From 0ab4458df0688955620b72cc2c72a32dffad3615 Mon Sep 17 00:00:00 2001
From: Monad <yanwqmonad@gmail.com>
Date: Tue, 7 May 2024 10:06:52 +0800
Subject: [PATCH 0001/1206] [InstCombine] Fold `cttz(lshr(-1, x) + 1)` to
 `width - x` (#91244)

Fold
``` llvm
define i64 @src(i64 %50) {
  %52 = lshr i64 -1, %50
  %53 = add i64 %52, 1
  %54 = call i64 @llvm.cttz.i64(i64 %53, i1 false)
  ret i64 %54
}
```
to
``` llvm
define i64 @tgt(i64 %50) {
  %52 = sub i64 64, %50
  ret i64 %52
}
```

as
https://github.com/llvm/llvm-project/pull/91171#pullrequestreview-2040663002
pointed out.

Alive2 proof: https://alive2.llvm.org/ce/z/2aHfYa

Note: the `ctlz` version of this pattern seems not exist in dtcxzyw's
benchmark, so put it aside for now.
---
 .../InstCombine/InstCombineCalls.cpp          |  7 +++
 llvm/test/Transforms/InstCombine/cttz.ll      | 61 +++++++++++++++++++
 2 files changed, 68 insertions(+)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 1913ef92c16c..d7433ad3599f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -562,6 +562,13 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) {
           IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, C, Op1);
       return BinaryOperator::CreateSub(ConstCttz, X);
     }
+
+    // cttz(add(lshr(UINT_MAX, %val), 1)) --> sub(width, %val)
+    if (match(Op0, m_Add(m_LShr(m_AllOnes(), m_Value(X)), m_One()))) {
+      Value *Width =
+          ConstantInt::get(II.getType(), II.getType()->getScalarSizeInBits());
+      return BinaryOperator::CreateSub(Width, X);
+    }
   } else {
     // ctlz(lshr(%const, %val), 1) --> add(ctlz(%const, 1), %val)
     if (match(Op0, m_LShr(m_ImmConstant(C), m_Value(X))) &&
diff --git a/llvm/test/Transforms/InstCombine/cttz.ll b/llvm/test/Transforms/InstCombine/cttz.ll
index 3595cff5f1ae..66b7a03fe5d7 100644
--- a/llvm/test/Transforms/InstCombine/cttz.ll
+++ b/llvm/test/Transforms/InstCombine/cttz.ll
@@ -215,3 +215,64 @@ define i32 @cttz_of_lowest_set_bit_wrong_intrinsic(i32 %x) {
   %tz = call i32 @llvm.ctlz.i32(i32 %and, i1 false)
   ret i32 %tz
 }
+
+define i32 @cttz_of_power_of_two(i32 %x) {
+; CHECK-LABEL: @cttz_of_power_of_two(
+; CHECK-NEXT:    [[R:%.*]] = sub i32 32, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %lshr = lshr i32 -1, %x
+  %add = add i32 %lshr, 1
+  %r = call i32 @llvm.cttz.i32(i32 %add, i1 false)
+  ret i32 %r
+}
+
+define i32 @cttz_of_power_of_two_zero_poison(i32 %x) {
+; CHECK-LABEL: @cttz_of_power_of_two_zero_poison(
+; CHECK-NEXT:    [[R:%.*]] = sub i32 32, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %lshr = lshr i32 -1, %x
+  %add = add i32 %lshr, 1
+  %r = call i32 @llvm.cttz.i32(i32 %add, i1 true)
+  ret i32 %r
+}
+
+define i32 @cttz_of_power_of_two_wrong_intrinsic(i32 %x) {
+; CHECK-LABEL: @cttz_of_power_of_two_wrong_intrinsic(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 -1, [[X:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[LSHR]], 1
+; CHECK-NEXT:    [[R:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[ADD]], i1 false)
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %lshr = lshr i32 -1, %x
+  %add = add i32 %lshr, 1
+  %r = call i32 @llvm.ctlz.i32(i32 %add, i1 false)
+  ret i32 %r
+}
+
+define i32 @cttz_of_power_of_two_wrong_constant_1(i32 %x) {
+; CHECK-LABEL: @cttz_of_power_of_two_wrong_constant_1(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 -2, [[X:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[LSHR]], 1
+; CHECK-NEXT:    [[R:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[ADD]], i1 true)
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %lshr = lshr i32 -2, %x
+  %add = add i32 %lshr, 1
+  %r = call i32 @llvm.cttz.i32(i32 %add, i1 false)
+  ret i32 %r
+}
+
+define i32 @cttz_of_power_of_two_wrong_constant_2(i32 %x) {
+; CHECK-LABEL: @cttz_of_power_of_two_wrong_constant_2(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 -1, [[X:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[LSHR]], -1
+; CHECK-NEXT:    [[R:%.*]] = call range(i32 1, 33) i32 @llvm.cttz.i32(i32 [[ADD]], i1 false)
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %lshr = lshr i32 -1, %x
+  %add = add i32 %lshr, -1
+  %r = call i32 @llvm.cttz.i32(i32 %add, i1 false)
+  ret i32 %r
+}
-- 
GitLab


From 178ff395006f204265b4f6fe72a3dbb2b9a79b47 Mon Sep 17 00:00:00 2001
From: "S. Bharadwaj Yadavalli" <Bharadwaj.Yadavalli@microsoft.com>
Date: Mon, 6 May 2024 22:21:37 -0400
Subject: [PATCH 0002/1206] Revert "[DirectX][DXIL] Set DXIL Version in DXIL
 target triple based on shader model version" (#91290)

Reverts llvm/llvm-project#90809

Need to investigate ASAN failures.
---
 clang/lib/Basic/Targets.cpp                   |  2 +-
 clang/lib/Driver/ToolChains/HLSL.cpp          | 44 +-----------
 clang/test/CodeGenHLSL/basic-target.c         |  2 +-
 clang/test/Driver/dxc_dxv_path.hlsl           |  6 +-
 .../enable_16bit_types_validation.hlsl        |  4 +-
 clang/unittests/Driver/DXCModeTest.cpp        | 22 +++---
 llvm/include/llvm/TargetParser/Triple.h       |  1 -
 llvm/lib/IR/Verifier.cpp                      |  4 +-
 llvm/lib/TargetParser/Triple.cpp              | 68 -------------------
 llvm/unittests/TargetParser/TripleTest.cpp    | 16 -----
 10 files changed, 21 insertions(+), 148 deletions(-)

diff --git a/clang/lib/Basic/Targets.cpp b/clang/lib/Basic/Targets.cpp
index dc1792b3471e..e3283510c6aa 100644
--- a/clang/lib/Basic/Targets.cpp
+++ b/clang/lib/Basic/Targets.cpp
@@ -760,7 +760,7 @@ using namespace clang::targets;
 TargetInfo *
 TargetInfo::CreateTargetInfo(DiagnosticsEngine &Diags,
                              const std::shared_ptr<TargetOptions> &Opts) {
-  llvm::Triple Triple(llvm::Triple::normalize(Opts->Triple));
+  llvm::Triple Triple(Opts->Triple);
 
   // Construct the target
   std::unique_ptr<TargetInfo> Target = AllocateTarget(Triple, *Opts);
diff --git a/clang/lib/Driver/ToolChains/HLSL.cpp b/clang/lib/Driver/ToolChains/HLSL.cpp
index 8286e3be2180..558e4db46f81 100644
--- a/clang/lib/Driver/ToolChains/HLSL.cpp
+++ b/clang/lib/Driver/ToolChains/HLSL.cpp
@@ -98,49 +98,9 @@ std::optional<std::string> tryParseProfile(StringRef Profile) {
   else if (llvm::getAsUnsignedInteger(Parts[2], 0, Minor))
     return std::nullopt;
 
-  // Determine DXIL version using the minor version number of Shader
-  // Model version specified in target profile. Prior to decoupling DXIL version
-  // numbering from that of Shader Model DXIL version 1.Y corresponds to SM 6.Y.
-  // E.g., dxilv1.Y-unknown-shadermodelX.Y-hull
+  // dxil-unknown-shadermodel-hull
   llvm::Triple T;
-  Triple::SubArchType SubArch = llvm::Triple::NoSubArch;
-  switch (Minor) {
-  case 0:
-    SubArch = llvm::Triple::DXILSubArch_v1_0;
-    break;
-  case 1:
-    SubArch = llvm::Triple::DXILSubArch_v1_1;
-    break;
-  case 2:
-    SubArch = llvm::Triple::DXILSubArch_v1_2;
-    break;
-  case 3:
-    SubArch = llvm::Triple::DXILSubArch_v1_3;
-    break;
-  case 4:
-    SubArch = llvm::Triple::DXILSubArch_v1_4;
-    break;
-  case 5:
-    SubArch = llvm::Triple::DXILSubArch_v1_5;
-    break;
-  case 6:
-    SubArch = llvm::Triple::DXILSubArch_v1_6;
-    break;
-  case 7:
-    SubArch = llvm::Triple::DXILSubArch_v1_7;
-    break;
-  case 8:
-    SubArch = llvm::Triple::DXILSubArch_v1_8;
-    break;
-  case OfflineLibMinor:
-    // Always consider minor version x as the latest supported DXIL version
-    SubArch = llvm::Triple::LatestDXILSubArch;
-    break;
-  default:
-    // No DXIL Version corresponding to specified Shader Model version found
-    return std::nullopt;
-  }
-  T.setArch(Triple::ArchType::dxil, SubArch);
+  T.setArch(Triple::ArchType::dxil);
   T.setOSName(Triple::getOSTypeName(Triple::OSType::ShaderModel).str() +
               VersionTuple(Major, Minor).getAsString());
   T.setEnvironment(Kind);
diff --git a/clang/test/CodeGenHLSL/basic-target.c b/clang/test/CodeGenHLSL/basic-target.c
index b97ebf90a7a1..8db711c3f2a5 100644
--- a/clang/test/CodeGenHLSL/basic-target.c
+++ b/clang/test/CodeGenHLSL/basic-target.c
@@ -7,4 +7,4 @@
 // RUN: %clang -target dxil-pc-shadermodel6.0-geometry -S -emit-llvm -o - %s | FileCheck %s
 
 // CHECK: target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
-// CHECK: target triple = "dxilv1.0-pc-shadermodel6.0-{{[a-z]+}}"
+// CHECK: target triple = "dxil-pc-shadermodel6.0-{{[a-z]+}}"
diff --git a/clang/test/Driver/dxc_dxv_path.hlsl b/clang/test/Driver/dxc_dxv_path.hlsl
index 4845de11d5b0..3d8e90d0d919 100644
--- a/clang/test/Driver/dxc_dxv_path.hlsl
+++ b/clang/test/Driver/dxc_dxv_path.hlsl
@@ -7,12 +7,12 @@
 // DXV_PATH:dxv{{(.exe)?}}" "-" "-o" "-"
 
 // RUN: %clang_dxc -I test -Vd -Tlib_6_3  -### %s 2>&1 | FileCheck %s --check-prefix=VD
-// VD:"-cc1"{{.*}}"-triple" "dxilv1.3-unknown-shadermodel6.3-library"
+// VD:"-cc1"{{.*}}"-triple" "dxil-unknown-shadermodel6.3-library"
 // VD-NOT:dxv not found
 
 // RUN: %clang_dxc -Tlib_6_3 -ccc-print-bindings --dxv-path=%T -Fo %t.dxo  %s 2>&1 | FileCheck %s --check-prefix=BINDINGS
-// BINDINGS: "dxilv1.3-unknown-shadermodel6.3-library" - "clang", inputs: ["[[INPUT:.+]]"], output: "[[DXC:.+]].dxo"
-// BINDINGS-NEXT: "dxilv1.3-unknown-shadermodel6.3-library" - "hlsl::Validator", inputs: ["[[DXC]].dxo"]
+// BINDINGS: "dxil-unknown-shadermodel6.3-library" - "clang", inputs: ["[[INPUT:.+]]"], output: "[[DXC:.+]].dxo"
+// BINDINGS-NEXT: "dxil-unknown-shadermodel6.3-library" - "hlsl::Validator", inputs: ["[[DXC]].dxo"]
 
 // RUN: %clang_dxc -Tlib_6_3 -ccc-print-phases --dxv-path=%T -Fo %t.dxc  %s 2>&1 | FileCheck %s --check-prefix=PHASES
 
diff --git a/clang/test/Options/enable_16bit_types_validation.hlsl b/clang/test/Options/enable_16bit_types_validation.hlsl
index bcb217e8982e..89fe26790c52 100644
--- a/clang/test/Options/enable_16bit_types_validation.hlsl
+++ b/clang/test/Options/enable_16bit_types_validation.hlsl
@@ -9,11 +9,11 @@
 // HV_invalid_2017: error: '-enable-16bit-types' option requires target HLSL Version >= 2018 and shader model >= 6.2, but HLSL Version is 'hlsl2017' and shader model is '6.4'
 // TP_invalid: error: '-enable-16bit-types' option requires target HLSL Version >= 2018 and shader model >= 6.2, but HLSL Version is 'hlsl2021' and shader model is '6.0'
 
-// valid_2021: "dxilv1.4-unknown-shadermodel6.4-library"
+// valid_2021: "dxil-unknown-shadermodel6.4-library"
 // valid_2021-SAME: "-std=hlsl2021"
 // valid_2021-SAME: "-fnative-half-type"
 
-// valid_2018: "dxilv1.4-unknown-shadermodel6.4-library"
+// valid_2018: "dxil-unknown-shadermodel6.4-library"
 // valid_2018-SAME: "-std=hlsl2018"
 // valid_2018-SAME: "-fnative-half-type"
 
diff --git a/clang/unittests/Driver/DXCModeTest.cpp b/clang/unittests/Driver/DXCModeTest.cpp
index 416723d498a2..b3767c042edb 100644
--- a/clang/unittests/Driver/DXCModeTest.cpp
+++ b/clang/unittests/Driver/DXCModeTest.cpp
@@ -68,27 +68,25 @@ TEST(DxcModeTest, TargetProfileValidation) {
   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
   DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagConsumer);
 
-  validateTargetProfile("-Tvs_6_0", "dxilv1.0--shadermodel6.0-vertex",
+  validateTargetProfile("-Tvs_6_0", "dxil--shadermodel6.0-vertex",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Ths_6_1", "dxilv1.1--shadermodel6.1-hull",
+  validateTargetProfile("-Ths_6_1", "dxil--shadermodel6.1-hull",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tds_6_2", "dxilv1.2--shadermodel6.2-domain",
+  validateTargetProfile("-Tds_6_2", "dxil--shadermodel6.2-domain",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tds_6_2", "dxilv1.2--shadermodel6.2-domain",
+  validateTargetProfile("-Tds_6_2", "dxil--shadermodel6.2-domain",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tgs_6_3", "dxilv1.3--shadermodel6.3-geometry",
+  validateTargetProfile("-Tgs_6_3", "dxil--shadermodel6.3-geometry",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tps_6_4", "dxilv1.4--shadermodel6.4-pixel",
+  validateTargetProfile("-Tps_6_4", "dxil--shadermodel6.4-pixel",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tcs_6_5", "dxilv1.5--shadermodel6.5-compute",
+  validateTargetProfile("-Tcs_6_5", "dxil--shadermodel6.5-compute",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tms_6_6", "dxilv1.6--shadermodel6.6-mesh",
+  validateTargetProfile("-Tms_6_6", "dxil--shadermodel6.6-mesh",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tas_6_7", "dxilv1.7--shadermodel6.7-amplification",
+  validateTargetProfile("-Tas_6_7", "dxil--shadermodel6.7-amplification",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tcs_6_8", "dxilv1.8--shadermodel6.8-compute",
-                        InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tlib_6_x", "dxilv1.8--shadermodel6.15-library",
+  validateTargetProfile("-Tlib_6_x", "dxil--shadermodel6.15-library",
                         InMemoryFileSystem, Diags);
 
   // Invalid tests.
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 4e357cddcf2a..7da30e6cf96f 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -176,7 +176,6 @@ public:
     DXILSubArch_v1_6,
     DXILSubArch_v1_7,
     DXILSubArch_v1_8,
-    LatestDXILSubArch = DXILSubArch_v1_8,
   };
   enum VendorType {
     UnknownVendor,
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index a463e672bee9..41d3fce7eef7 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -152,8 +152,8 @@ struct VerifierSupport {
   bool TreatBrokenDebugInfoAsError = true;
 
   explicit VerifierSupport(raw_ostream *OS, const Module &M)
-      : OS(OS), M(M), MST(&M), TT(Triple::normalize(M.getTargetTriple())),
-        DL(M.getDataLayout()), Context(M.getContext()) {}
+      : OS(OS), M(M), MST(&M), TT(M.getTargetTriple()), DL(M.getDataLayout()),
+        Context(M.getContext()) {}
 
 private:
   void Write(const Module *M) {
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index a6989790a98e..f3f244c814e7 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -115,31 +115,6 @@ StringRef Triple::getArchName(ArchType Kind, SubArchType SubArch) {
     if (SubArch == AArch64SubArch_arm64e)
       return "arm64e";
     break;
-  case Triple::dxil:
-    switch (SubArch) {
-    case Triple::NoSubArch:
-    case Triple::DXILSubArch_v1_0:
-      return "dxilv1.0";
-    case Triple::DXILSubArch_v1_1:
-      return "dxilv1.1";
-    case Triple::DXILSubArch_v1_2:
-      return "dxilv1.2";
-    case Triple::DXILSubArch_v1_3:
-      return "dxilv1.3";
-    case Triple::DXILSubArch_v1_4:
-      return "dxilv1.4";
-    case Triple::DXILSubArch_v1_5:
-      return "dxilv1.5";
-    case Triple::DXILSubArch_v1_6:
-      return "dxilv1.6";
-    case Triple::DXILSubArch_v1_7:
-      return "dxilv1.7";
-    case Triple::DXILSubArch_v1_8:
-      return "dxilv1.8";
-    default:
-      break;
-    }
-    break;
   default:
     break;
   }
@@ -1039,8 +1014,6 @@ Triple::Triple(const Twine &ArchStr, const Twine &VendorStr, const Twine &OSStr,
     ObjectFormat = getDefaultFormat(*this);
 }
 
-static VersionTuple parseVersionFromName(StringRef Name);
-
 std::string Triple::normalize(StringRef Str) {
   bool IsMinGW32 = false;
   bool IsCygwin = false;
@@ -1233,47 +1206,6 @@ std::string Triple::normalize(StringRef Str) {
     }
   }
 
-  // Normalize DXIL triple if it does not include DXIL version number.
-  // Determine DXIL version number using the minor version number of Shader
-  // Model version specified in target triple, if any. Prior to decoupling DXIL
-  // version numbering from that of Shader Model DXIL version 1.Y corresponds to
-  // SM 6.Y. E.g., dxilv1.Y-unknown-shadermodelX.Y-hull
-  if (Components[0] == "dxil") {
-    if (Components.size() > 4) {
-      Components.resize(4);
-    }
-    // Add DXIL version only if shadermodel is specified in the triple
-    if (OS == Triple::ShaderModel) {
-      VersionTuple Ver =
-          parseVersionFromName(Components[2].drop_front(strlen("shadermodel")));
-      // Default DXIL minor version when Shader Model version is anything other
-      // than 6.[0...8] or 6.x (which translates to latest current SM version)
-      // DXIL version corresponding to Shader Model version other than 6.x
-      // is 1.0
-      unsigned DXILMinor = 0;
-      const unsigned SMMajor = 6;
-      const unsigned LatestCurrentDXILMinor = 8;
-      if (!Ver.empty()) {
-        if (Ver.getMajor() == SMMajor) {
-          if (std::optional<unsigned> SMMinor = Ver.getMinor()) {
-            DXILMinor = *SMMinor;
-            // Ensure specified minor version is supported
-            if (DXILMinor > LatestCurrentDXILMinor) {
-              report_fatal_error("Unsupported Shader Model version", false);
-            }
-          }
-        }
-      } else {
-        // Special case: DXIL minor version is set to LatestCurrentDXILMinor for
-        // shadermodel6.x is
-        if (Components[2] == "shadermodel6.x") {
-          DXILMinor = LatestCurrentDXILMinor;
-        }
-      }
-      Components[0] =
-          Components[0].str().append("v1.").append(std::to_string(DXILMinor));
-    }
-  }
   // Stick the corrected components back together to form the normalized string.
   return join(Components, "-");
 }
diff --git a/llvm/unittests/TargetParser/TripleTest.cpp b/llvm/unittests/TargetParser/TripleTest.cpp
index 3112014d9efb..b8f5fbd87407 100644
--- a/llvm/unittests/TargetParser/TripleTest.cpp
+++ b/llvm/unittests/TargetParser/TripleTest.cpp
@@ -2454,20 +2454,4 @@ TEST(TripleTest, isArmMClass) {
     EXPECT_TRUE(T.isArmMClass());
   }
 }
-
-TEST(TripleTest, DXILNormaizeWithVersion) {
-  EXPECT_EQ("dxilv1.0-unknown-shadermodel6.0",
-            Triple::normalize("dxilv1.0--shadermodel6.0"));
-  EXPECT_EQ("dxilv1.0-unknown-shadermodel6.0",
-            Triple::normalize("dxil--shadermodel6.0"));
-  EXPECT_EQ("dxilv1.1-unknown-shadermodel6.1-library",
-            Triple::normalize("dxil-shadermodel6.1-unknown-library"));
-  EXPECT_EQ("dxilv1.8-unknown-shadermodel6.x-unknown",
-            Triple::normalize("dxil-unknown-shadermodel6.x-unknown"));
-  EXPECT_EQ("dxilv1.8-unknown-shadermodel6.x-unknown",
-            Triple::normalize("dxil-unknown-shadermodel6.x-unknown"));
-  EXPECT_EQ("dxil-unknown-unknown-unknown", Triple::normalize("dxil---"));
-  EXPECT_EQ("dxilv1.0-pc-shadermodel5.0-compute",
-            Triple::normalize("dxil-shadermodel5.0-pc-compute"));
-}
 } // end anonymous namespace
-- 
GitLab


From 3e54768d7a0e1cfa65e892b6602993192ecad91e Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 6 May 2024 21:23:04 -0500
Subject: [PATCH 0003/1206] [Offload] Detect target triple from preprocessor
 instead of CMake (#91283)

Summary:
This patch removes the special-case handling for the target triple
inside of the CMake. I moved it into the implementation so it's easier
to see and modify.
---
 offload/plugins-nextgen/host/CMakeLists.txt | 10 ------
 offload/plugins-nextgen/host/src/rtl.cpp    | 36 +++++++++++++++++----
 2 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/offload/plugins-nextgen/host/CMakeLists.txt b/offload/plugins-nextgen/host/CMakeLists.txt
index 6407f72e8db0..48e591bc894e 100644
--- a/offload/plugins-nextgen/host/CMakeLists.txt
+++ b/offload/plugins-nextgen/host/CMakeLists.txt
@@ -53,36 +53,26 @@ endif()
 # Define the target specific triples and ELF machine values.
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$")
   target_compile_definitions(omptarget.rtl.host PRIVATE TARGET_ELF_ID=EM_PPC64)
-  target_compile_definitions(omptarget.rtl.host PRIVATE
-      LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="powerpc64le-ibm-linux-gnu")
   list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
        "powerpc64le-ibm-linux-gnu" "powerpc64le-ibm-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64$")
   target_compile_definitions(omptarget.rtl.host PRIVATE TARGET_ELF_ID=EM_PPC64)
-  target_compile_definitions(omptarget.rtl.host PRIVATE
-      LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="powerpc64-ibm-linux-gnu")
   list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
        "powerpc64-ibm-linux-gnu" "powerpc64-ibm-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64$")
   target_compile_definitions(omptarget.rtl.host PRIVATE TARGET_ELF_ID=EM_X86_64)
-  target_compile_definitions(omptarget.rtl.host PRIVATE
-      LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="x86_64-pc-linux-gnu")
   list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
        "x86_64-pc-linux-gnu" "x86_64-pc-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64$")
   target_compile_definitions(omptarget.rtl.host PRIVATE TARGET_ELF_ID=EM_AARCH64)
-  target_compile_definitions(omptarget.rtl.host PRIVATE
-      LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="aarch64-unknown-linux-gnu")
   list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
        "aarch64-unknown-linux-gnu" "aarch64-unknown-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x$")
   target_compile_definitions(omptarget.rtl.host PRIVATE TARGET_ELF_ID=EM_S390)
-  target_compile_definitions(omptarget.rtl.host PRIVATE
-      LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="s390x-ibm-linux-gnu")
   list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
        "s390x-ibm-linux-gnu" "s390x-ibm-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index f0ce24249301..c929db6c22d8 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -30,6 +30,17 @@
 #include "llvm/Frontend/OpenMP/OMPGridValues.h"
 #include "llvm/Support/DynamicLibrary.h"
 
+#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) ||           \
+    !defined(__ORDER_BIG_ENDIAN__)
+#error "Missing preprocessor definitions for endianness detection."
+#endif
+
+#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#define LITTLEENDIAN_CPU
+#elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#define BIGENDIAN_CPU
+#endif
+
 // The number of devices in this plugin.
 #define NUM_DEVICES 4
 
@@ -38,11 +49,6 @@
 #define TARGET_ELF_ID EM_NONE
 #endif
 
-// The target triple should be defined at compile-time by the build system.
-#ifndef LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE
-#define LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE ""
-#endif
-
 namespace llvm {
 namespace omp {
 namespace target {
@@ -421,7 +427,25 @@ struct GenELF64PluginTy final : public GenericPluginTy {
   Expected<bool> isELFCompatible(StringRef) const override { return true; }
 
   Triple::ArchType getTripleArch() const override {
-    return llvm::Triple(LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE).getArch();
+#if defined(__x86_64__)
+    return llvm::Triple::x86_64;
+#elif defined(__s390x__)
+    return llvm::Triple::systemz;
+#elif defined(__aarch64__)
+#ifdef LITTLEENDIAN_CPU
+    return llvm::Triple::aarch64_le;
+#else
+    return llvm::Triple::aarch64_be;
+#endif
+#elif defined(__powerpc64__)
+#ifdef LITTLEENDIAN_CPU
+    return llvm::Triple::ppc64le;
+#else
+    return llvm::Triple::ppc64;
+#endif
+#else
+    return llvm::Triple::UnknownArch;
+#endif
   }
 };
 
-- 
GitLab


From 37fcb323f61efb8dfb74548a1b472fa20e829170 Mon Sep 17 00:00:00 2001
From: Jianjian Guan <jacquesguan@me.com>
Date: Tue, 7 May 2024 10:25:06 +0800
Subject: [PATCH 0004/1206] [RISCV] Add codegen support for Zvfbfmin (#87911)

This patch adds basic codegen support for Zvfbfmin extension.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  50 +++++++-
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    |  32 ++---
 .../Target/RISCV/RISCVInstrInfoVSDPatterns.td |  14 ++
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td |  30 +++++
 .../CodeGen/RISCV/rvv/extract-subvector.ll    |  58 ++++++++-
 .../RISCV/rvv/fixed-vectors-fpext-vp.ll       |  58 ++++++++-
 .../RISCV/rvv/fixed-vectors-fptrunc-vp.ll     |  58 ++++++++-
 .../RISCV/rvv/fixed-vectors-load-store.ll     |  16 ++-
 .../CodeGen/RISCV/rvv/fixed-vectors-load.ll   |  14 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-store.ll  |  14 +-
 ...fixed-vectors-vfpext-constrained-sdnode.ll |  79 +++++++++++-
 ...xed-vectors-vfptrunc-constrained-sdnode.ll |  83 +++++++++++-
 .../CodeGen/RISCV/rvv/insert-subvector.ll     |  63 ++++++++-
 .../RISCV/rvv/vfpext-constrained-sdnode.ll    | 104 ++++++++++++++-
 llvm/test/CodeGen/RISCV/rvv/vfpext-sdnode.ll  | 120 +++++++++++++++++-
 llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll      |  59 ++++++++-
 .../RISCV/rvv/vfptrunc-constrained-sdnode.ll  | 108 +++++++++++++++-
 .../test/CodeGen/RISCV/rvv/vfptrunc-sdnode.ll |  81 +++++++++++-
 llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll    |  58 ++++++++-
 19 files changed, 1032 insertions(+), 67 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 19ef1f2f18ec..2818e1911ee5 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1087,6 +1087,23 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       }
     }
 
+    // TODO: Could we merge some code with zvfhmin?
+    if (Subtarget.hasVInstructionsBF16()) {
+      for (MVT VT : BF16VecVTs) {
+        if (!isTypeLegal(VT))
+          continue;
+        setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
+        setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
+        setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
+                           Custom);
+        setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
+                            ISD::EXTRACT_SUBVECTOR},
+                           VT, Custom);
+        setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
+        // TODO: Promote to fp32.
+      }
+    }
+
     if (Subtarget.hasVInstructionsF32()) {
       for (MVT VT : F32VecVTs) {
         if (!isTypeLegal(VT))
@@ -1302,6 +1319,19 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
           continue;
         }
 
+        if (VT.getVectorElementType() == MVT::bf16) {
+          setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
+          setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
+          setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
+                             Custom);
+          setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
+                              ISD::EXTRACT_SUBVECTOR},
+                             VT, Custom);
+          setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
+          // TODO: Promote to fp32.
+          continue;
+        }
+
         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
         setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
                            Custom);
@@ -2561,6 +2591,10 @@ static bool useRVVForFixedLengthVectorVT(MVT VT,
     if (!Subtarget.hasVInstructionsF16Minimal())
       return false;
     break;
+  case MVT::bf16:
+    if (!Subtarget.hasVInstructionsBF16())
+      return false;
+    break;
   case MVT::f32:
     if (!Subtarget.hasVInstructionsF32())
       return false;
@@ -2612,6 +2646,7 @@ static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT,
   case MVT::i16:
   case MVT::i32:
   case MVT::i64:
+  case MVT::bf16:
   case MVT::f16:
   case MVT::f32:
   case MVT::f64: {
@@ -8101,8 +8136,10 @@ RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(SDValue Op,
 
   // RVV can only widen/truncate fp to types double/half the size as the source.
   if ((VT.getVectorElementType() == MVT::f64 &&
-       SrcVT.getVectorElementType() == MVT::f16) ||
-      (VT.getVectorElementType() == MVT::f16 &&
+       (SrcVT.getVectorElementType() == MVT::f16 ||
+        SrcVT.getVectorElementType() == MVT::bf16)) ||
+      ((VT.getVectorElementType() == MVT::f16 ||
+        VT.getVectorElementType() == MVT::bf16) &&
        SrcVT.getVectorElementType() == MVT::f64)) {
     // For double rounding, the intermediate rounding should be round-to-odd.
     unsigned InterConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
@@ -8146,9 +8183,12 @@ RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(SDValue Op,
   SDValue Src = Op.getOperand(0);
   MVT SrcVT = Src.getSimpleValueType();
 
-  bool IsDirectExtend = IsExtend && (VT.getVectorElementType() != MVT::f64 ||
-                                     SrcVT.getVectorElementType() != MVT::f16);
-  bool IsDirectTrunc = !IsExtend && (VT.getVectorElementType() != MVT::f16 ||
+  bool IsDirectExtend =
+      IsExtend && (VT.getVectorElementType() != MVT::f64 ||
+                   (SrcVT.getVectorElementType() != MVT::f16 &&
+                    SrcVT.getVectorElementType() != MVT::bf16));
+  bool IsDirectTrunc = !IsExtend && ((VT.getVectorElementType() != MVT::f16 &&
+                                      VT.getVectorElementType() != MVT::bf16) ||
                                      SrcVT.getVectorElementType() != MVT::f64);
 
   bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 22e548861784..4adc26f62891 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -355,24 +355,24 @@ defset list<VTypeInfo> AllVectors = {
                                  V_M8, f64, FPR64>;
     }
   }
-}
 
-defset list<VTypeInfo> AllBFloatVectors = {
-  defset list<VTypeInfo> NoGroupBFloatVectors = {
-    defset list<VTypeInfo> FractionalGroupBFloatVectors = {
-      def VBF16MF4: VTypeInfo<vbfloat16mf4_t, vbool64_t, 16, V_MF4, bf16, FPR16>;
-      def VBF16MF2: VTypeInfo<vbfloat16mf2_t, vbool32_t, 16, V_MF2, bf16, FPR16>;
+  defset list<VTypeInfo> AllBFloatVectors = {
+    defset list<VTypeInfo> NoGroupBFloatVectors = {
+      defset list<VTypeInfo> FractionalGroupBFloatVectors = {
+        def VBF16MF4: VTypeInfo<vbfloat16mf4_t, vbool64_t, 16, V_MF4, bf16, FPR16>;
+        def VBF16MF2: VTypeInfo<vbfloat16mf2_t, vbool32_t, 16, V_MF2, bf16, FPR16>;
+      }
+      def VBF16M1:  VTypeInfo<vbfloat16m1_t, vbool16_t, 16, V_M1, bf16, FPR16>;
+    }
+  
+    defset list<GroupVTypeInfo> GroupBFloatVectors = {
+      def VBF16M2: GroupVTypeInfo<vbfloat16m2_t, vbfloat16m1_t, vbool8_t, 16,
+                                  V_M2, bf16, FPR16>;
+      def VBF16M4: GroupVTypeInfo<vbfloat16m4_t, vbfloat16m1_t, vbool4_t, 16,
+                                  V_M4, bf16, FPR16>;
+      def VBF16M8: GroupVTypeInfo<vbfloat16m8_t, vbfloat16m1_t, vbool2_t, 16,
+                                  V_M8, bf16, FPR16>;
     }
-    def VBF16M1:  VTypeInfo<vbfloat16m1_t, vbool16_t, 16, V_M1, bf16, FPR16>;
-  }
-
-  defset list<GroupVTypeInfo> GroupBFloatVectors = {
-    def VBF16M2: GroupVTypeInfo<vbfloat16m2_t, vbfloat16m1_t, vbool8_t, 16,
-                                V_M2, bf16, FPR16>;
-    def VBF16M4: GroupVTypeInfo<vbfloat16m4_t, vbfloat16m1_t, vbool4_t, 16,
-                                V_M4, bf16, FPR16>;
-    def VBF16M8: GroupVTypeInfo<vbfloat16m8_t, vbfloat16m1_t, vbool2_t, 16,
-                                V_M8, bf16, FPR16>;
   }
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index b4af83a3cbf6..714f8cff7b63 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -1495,6 +1495,20 @@ foreach fvtiToFWti = AllWidenableFloatVectors in {
                 fvti.AVL, fvti.Log2SEW, TA_MA)>;
 }
 
+foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in {
+  defvar fvti = fvtiToFWti.Vti;
+  defvar fwti = fvtiToFWti.Wti;
+  let Predicates = [HasVInstructionsBF16] in
+  def : Pat<(fvti.Vector (fpround (fwti.Vector fwti.RegClass:$rs1))),
+            (!cast<Instruction>("PseudoVFNCVTBF16_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW)
+                (fvti.Vector (IMPLICIT_DEF)),
+                fwti.RegClass:$rs1,
+                // Value to indicate no rounding mode change in
+                // RISCVInsertReadWriteCSR
+                FRM_DYN,
+                fvti.AVL, fvti.Log2SEW, TA_MA)>;
+}
+
 //===----------------------------------------------------------------------===//
 // Vector Splats
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 6c6ecb604fd0..e10b8bf2767b 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -2670,6 +2670,20 @@ foreach fvtiToFWti = AllWidenableFloatVectors in {
                 GPR:$vl, fvti.Log2SEW, TA_MA)>;
 }
 
+foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in {
+  defvar fvti = fvtiToFWti.Vti;
+  defvar fwti = fvtiToFWti.Wti;
+  let Predicates = [HasVInstructionsBF16] in
+  def : Pat<(fwti.Vector (any_riscv_fpextend_vl
+                             (fvti.Vector fvti.RegClass:$rs1),
+                             (fvti.Mask V0),
+                             VLOpFrag)),
+            (!cast<Instruction>("PseudoVFWCVTBF16_F_F_V_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK")
+                (fwti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs1,
+                (fvti.Mask V0),
+                GPR:$vl, fvti.Log2SEW, TA_MA)>;
+}
+
 // 13.19 Narrowing Floating-Point/Integer Type-Convert Instructions
 defm : VPatNConvertFP2IVL_W_RM<riscv_vfcvt_xu_f_vl, "PseudoVFNCVT_XU_F_W">;
 defm : VPatNConvertFP2IVL_W_RM<riscv_vfcvt_x_f_vl, "PseudoVFNCVT_X_F_W">;
@@ -2714,6 +2728,22 @@ foreach fvtiToFWti = AllWidenableFloatVectors in {
   }
 }
 
+foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in {
+  defvar fvti = fvtiToFWti.Vti;
+  defvar fwti = fvtiToFWti.Wti;
+  let Predicates = [HasVInstructionsBF16] in
+    def : Pat<(fvti.Vector (any_riscv_fpround_vl
+                               (fwti.Vector fwti.RegClass:$rs1),
+                               (fwti.Mask V0), VLOpFrag)),
+              (!cast<Instruction>("PseudoVFNCVTBF16_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK")
+                  (fvti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1,
+                  (fwti.Mask V0),
+                  // Value to indicate no rounding mode change in
+                  // RISCVInsertReadWriteCSR
+                  FRM_DYN,
+                  GPR:$vl, fvti.Log2SEW, TA_MA)>;
+}
+
 // 14. Vector Reduction Operations
 
 // 14.1. Vector Single-Width Integer Reduction Instructions
diff --git a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
index e15e6452163b..4f1fcfbe8cc5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple riscv32 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple riscv64 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 4 x i32> @extract_nxv8i32_nxv4i32_0(<vscale x 8 x i32> %vec) {
 ; CHECK-LABEL: extract_nxv8i32_nxv4i32_0:
@@ -481,6 +481,60 @@ define <vscale x 6 x half> @extract_nxv6f16_nxv12f16_6(<vscale x 12 x half> %in)
   ret <vscale x 6 x half> %res
 }
 
+define <vscale x 2 x bfloat> @extract_nxv2bf16_nxv16bf16_0(<vscale x 16 x bfloat> %vec) {
+; CHECK-LABEL: extract_nxv2bf16_nxv16bf16_0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %c = call <vscale x 2 x bfloat> @llvm.vector.extract.nxv2bf16.nxv16bf16(<vscale x 16 x bfloat> %vec, i64 0)
+  ret <vscale x 2 x bfloat> %c
+}
+
+define <vscale x 2 x bfloat> @extract_nxv2bf16_nxv16bf16_2(<vscale x 16 x bfloat> %vec) {
+; CHECK-LABEL: extract_nxv2bf16_nxv16bf16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 2
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v8, a0
+; CHECK-NEXT:    ret
+  %c = call <vscale x 2 x bfloat> @llvm.vector.extract.nxv2bf16.nxv16bf16(<vscale x 16 x bfloat> %vec, i64 2)
+  ret <vscale x 2 x bfloat> %c
+}
+
+define <vscale x 2 x bfloat> @extract_nxv2bf16_nxv16bf16_4(<vscale x 16 x bfloat> %vec) {
+; CHECK-LABEL: extract_nxv2bf16_nxv16bf16_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %c = call <vscale x 2 x bfloat> @llvm.vector.extract.nxv2bf16.nxv16bf16(<vscale x 16 x bfloat> %vec, i64 4)
+  ret <vscale x 2 x bfloat> %c
+}
+
+define <vscale x 6 x bfloat> @extract_nxv6bf16_nxv12bf16_0(<vscale x 12 x bfloat> %in) {
+; CHECK-LABEL: extract_nxv6bf16_nxv12bf16_0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %res = call <vscale x 6 x bfloat> @llvm.vector.extract.nxv6bf16.nxv12bf16(<vscale x 12 x bfloat> %in, i64 0)
+  ret <vscale x 6 x bfloat> %res
+}
+
+define <vscale x 6 x bfloat> @extract_nxv6bf16_nxv12bf16_6(<vscale x 12 x bfloat> %in) {
+; CHECK-LABEL: extract_nxv6bf16_nxv12bf16_6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 2
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v13, v10, a0
+; CHECK-NEXT:    vslidedown.vx v12, v9, a0
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v12, v10, a0
+; CHECK-NEXT:    vmv2r.v v8, v12
+; CHECK-NEXT:    ret
+  %res = call <vscale x 6 x bfloat> @llvm.vector.extract.nxv6bf16.nxv12bf16(<vscale x 12 x bfloat> %in, i64 6)
+  ret <vscale x 6 x bfloat> %res
+}
+
 declare <vscale x 6 x half> @llvm.vector.extract.nxv6f16.nxv12f16(<vscale x 12 x half>, i64)
 
 declare <vscale x 1 x i8> @llvm.vector.extract.nxv1i8.nxv4i8(<vscale x 4 x i8> %vec, i64 %idx)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fpext-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fpext-vp.ll
index 51ac27acaf47..48cc3f17a626 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fpext-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fpext-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x float> @llvm.vp.fpext.v2f32.v2f16(<2 x half>, <2 x i1>, i32)
 
@@ -120,3 +120,53 @@ define <32 x double> @vfpext_v32f32_v32f64(<32 x float> %a, <32 x i1> %m, i32 ze
   %v = call <32 x double> @llvm.vp.fpext.v32f64.v32f32(<32 x float> %a, <32 x i1> %m, i32 %vl)
   ret <32 x double> %v
 }
+
+declare <2 x float> @llvm.vp.fpext.v2f32.v2bf16(<2 x bfloat>, <2 x i1>, i32)
+
+define <2 x float> @vfpext_v2bf16_v2f32(<2 x bfloat> %a, <2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vfpext_v2bf16_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8, v0.t
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <2 x float> @llvm.vp.fpext.v2f32.v2bf16(<2 x bfloat> %a, <2 x i1> %m, i32 %vl)
+  ret <2 x float> %v
+}
+
+define <2 x float> @vfpext_v2bf16_v2f32_unmasked(<2 x bfloat> %a, i32 zeroext %vl) {
+; CHECK-LABEL: vfpext_v2bf16_v2f32_unmasked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <2 x float> @llvm.vp.fpext.v2f32.v2bf16(<2 x bfloat> %a, <2 x i1> shufflevector (<2 x i1> insertelement (<2 x i1> undef, i1 true, i32 0), <2 x i1> undef, <2 x i32> zeroinitializer), i32 %vl)
+  ret <2 x float> %v
+}
+
+declare <2 x double> @llvm.vp.fpext.v2f64.v2bf16(<2 x bfloat>, <2 x i1>, i32)
+
+define <2 x double> @vfpext_v2bf16_v2f64(<2 x bfloat> %a, <2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vfpext_v2bf16_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v9, v0.t
+; CHECK-NEXT:    ret
+  %v = call <2 x double> @llvm.vp.fpext.v2f64.v2bf16(<2 x bfloat> %a, <2 x i1> %m, i32 %vl)
+  ret <2 x double> %v
+}
+
+define <2 x double> @vfpext_v2bf16_v2f64_unmasked(<2 x bfloat> %a, i32 zeroext %vl) {
+; CHECK-LABEL: vfpext_v2bf16_v2f64_unmasked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <2 x double> @llvm.vp.fpext.v2f64.v2bf16(<2 x bfloat> %a, <2 x i1> shufflevector (<2 x i1> insertelement (<2 x i1> undef, i1 true, i32 0), <2 x i1> undef, <2 x i32> zeroinitializer), i32 %vl)
+  ret <2 x double> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll
index de11f9e8a9fa..d890bf5412f9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fptrunc-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 
 declare <2 x half> @llvm.vp.fptrunc.v2f16.v2f32(<2 x float>, <2 x i1>, i32)
@@ -122,3 +122,53 @@ define <32 x float> @vfptrunc_v32f32_v32f64(<32 x double> %a, <32 x i1> %m, i32
   %v = call <32 x float> @llvm.vp.fptrunc.v32f64.v32f32(<32 x double> %a, <32 x i1> %m, i32 %vl)
   ret <32 x float> %v
 }
+
+declare <2 x bfloat> @llvm.vp.fptrunc.v2bf16.v2f32(<2 x float>, <2 x i1>, i32)
+
+define <2 x bfloat> @vfptrunc_v2bf16_v2f32(<2 x float> %a, <2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vfptrunc_v2bf16_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v9, v8, v0.t
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <2 x bfloat> @llvm.vp.fptrunc.v2bf16.v2f32(<2 x float> %a, <2 x i1> %m, i32 %vl)
+  ret <2 x bfloat> %v
+}
+
+define <2 x bfloat> @vfptrunc_v2bf16_v2f32_unmasked(<2 x float> %a, i32 zeroext %vl) {
+; CHECK-LABEL: vfptrunc_v2bf16_v2f32_unmasked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <2 x bfloat> @llvm.vp.fptrunc.v2bf16.v2f32(<2 x float> %a, <2 x i1> shufflevector (<2 x i1> insertelement (<2 x i1> undef, i1 true, i32 0), <2 x i1> undef, <2 x i32> zeroinitializer), i32 %vl)
+  ret <2 x bfloat> %v
+}
+
+declare <2 x bfloat> @llvm.vp.fptrunc.v2bf16.v2f64(<2 x double>, <2 x i1>, i32)
+
+define <2 x bfloat> @vfptrunc_v2bf16_v2f64(<2 x double> %a, <2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vfptrunc_v2bf16_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v9, v8, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9, v0.t
+; CHECK-NEXT:    ret
+  %v = call <2 x bfloat> @llvm.vp.fptrunc.v2bf16.v2f64(<2 x double> %a, <2 x i1> %m, i32 %vl)
+  ret <2 x bfloat> %v
+}
+
+define <2 x bfloat> @vfptrunc_v2bf16_v2f64_unmasked(<2 x double> %a, i32 zeroext %vl) {
+; CHECK-LABEL: vfptrunc_v2bf16_v2f64_unmasked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v9, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
+; CHECK-NEXT:    ret
+  %v = call <2 x bfloat> @llvm.vp.fptrunc.v2bf16.v2f64(<2 x double> %a, <2 x i1> shufflevector (<2 x i1> insertelement (<2 x i1> undef, i1 true, i32 0), <2 x i1> undef, <2 x i32> zeroinitializer), i32 %vl)
+  ret <2 x bfloat> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load-store.ll
index 38aee567e2b5..fbe8bcbc0d3c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load-store.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
 
 define void @v2i8(ptr %p, ptr %q) {
 ; CHECK-LABEL: v2i8:
@@ -301,3 +301,15 @@ define void @v2i8_volatile_store(ptr %p, ptr %q) {
   store volatile <2 x i8> %v, ptr %q
   ret void
 }
+
+define void @v8bf16(ptr %p, ptr %q) {
+; CHECK-LABEL: v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    vse16.v v8, (a1)
+; CHECK-NEXT:    ret
+  %v = load <8 x bfloat>, ptr %p
+  store <8 x bfloat> %v, ptr %q
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
index 791e6eb5ff30..d80d75d3d5d0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
 
 define <5 x i8> @load_v5i8(ptr %p) {
 ; CHECK-LABEL: load_v5i8:
@@ -181,3 +181,13 @@ define <16 x i64> @exact_vlen_i64_m8(ptr %p) vscale_range(2,2) {
   %v = load <16 x i64>, ptr %p
   ret <16 x i64> %v
 }
+
+define <8 x bfloat> @load_v8bf16(ptr %p) {
+; CHECK-LABEL: load_v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    ret
+  %x = load <8 x bfloat>, ptr %p
+  ret <8 x bfloat> %x
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
index b747d73ce353..6317a4977562 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+zvfh,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+zvfh,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s
 
 define void @store_v5i8(ptr %p, <5 x i8> %v) {
 ; CHECK-LABEL: store_v5i8:
@@ -294,6 +294,16 @@ define void @exact_vlen_i64_m8(ptr %p) vscale_range(2,2) {
   ret void
 }
 
+define void @store_v8bf16(ptr %p, <8 x bfloat> %v) {
+; CHECK-LABEL: store_v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vse16.v v8, (a0)
+; CHECK-NEXT:    ret
+  store <8 x bfloat> %v, ptr %p
+  ret void
+}
+
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; RV32: {{.*}}
 ; RV64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfpext-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfpext-constrained-sdnode.ll
index b0e6a6a56051..5d9076208988 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfpext-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfpext-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x float> @llvm.experimental.constrained.fpext.v2f32.v2f16(<2 x half>, metadata)
@@ -114,3 +114,78 @@ define <8 x double> @vfpext_v8f32_v8f64(<8 x float> %va) strictfp {
   %evec = call <8 x double> @llvm.experimental.constrained.fpext.v8f64.v8f32(<8 x float> %va, metadata !"fpexcept.strict")
   ret <8 x double> %evec
 }
+
+declare <2 x float> @llvm.experimental.constrained.fpext.v2f32.v2bf16(<2 x bfloat>, metadata)
+define <2 x float> @vfpext_v2bf16_v2f32(<2 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_v2bf16_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <2 x float> @llvm.experimental.constrained.fpext.v2f32.v2bf16(<2 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <2 x float> %evec
+}
+
+declare <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2bf16(<2 x bfloat>, metadata)
+define <2 x double> @vfpext_v2bf16_v2f64(<2 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_v2bf16_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <2 x double> @llvm.experimental.constrained.fpext.v2f64.v2bf16(<2 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <2 x double> %evec
+}
+
+declare <4 x float> @llvm.experimental.constrained.fpext.v4f32.v4bf16(<4 x bfloat>, metadata)
+define <4 x float> @vfpext_v4bf16_v4f32(<4 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_v4bf16_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <4 x float> @llvm.experimental.constrained.fpext.v4f32.v4bf16(<4 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <4 x float> %evec
+}
+
+declare <4 x double> @llvm.experimental.constrained.fpext.v4f64.v4bf16(<4 x bfloat>, metadata)
+define <4 x double> @vfpext_v4bf16_v4f64(<4 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_v4bf16_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v10
+; CHECK-NEXT:    ret
+  %evec = call <4 x double> @llvm.experimental.constrained.fpext.v4f64.v4bf16(<4 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <4 x double> %evec
+}
+
+declare <8 x float> @llvm.experimental.constrained.fpext.v8f32.v8bf16(<8 x bfloat>, metadata)
+define <8 x float> @vfpext_v8bf16_v8f32(<8 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_v8bf16_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
+; CHECK-NEXT:    vmv2r.v v8, v10
+; CHECK-NEXT:    ret
+  %evec = call <8 x float> @llvm.experimental.constrained.fpext.v8f32.v8bf16(<8 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <8 x float> %evec
+}
+
+declare <8 x double> @llvm.experimental.constrained.fpext.v8f64.v8bf16(<8 x bfloat>, metadata)
+define <8 x double> @vfpext_v8bf16_v8f64(<8 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_v8bf16_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v12
+; CHECK-NEXT:    ret
+  %evec = call <8 x double> @llvm.experimental.constrained.fpext.v8f64.v8bf16(<8 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <8 x double> %evec
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll
index fd53113741de..5781223a5326 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(<2 x double>, metadata, metadata)
@@ -118,3 +118,78 @@ define <8 x half> @vfptrunc_v8f32_v8f16(<8 x float> %va) strictfp {
   %evec = call <8 x half> @llvm.experimental.constrained.fptrunc.v8f16.v8f32(<8 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <8 x half> %evec
 }
+
+declare <2 x bfloat> @llvm.experimental.constrained.fptrunc.v2bf16.v2f64(<2 x double>, metadata, metadata)
+define <2 x bfloat> @vfptrunc_v2f64_v2bf16(<2 x double> %va) strictfp {
+; CHECK-LABEL: vfptrunc_v2f64_v2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v9, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <2 x bfloat> @llvm.experimental.constrained.fptrunc.v2bf16.v2f64(<2 x double> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <2 x bfloat> %evec
+}
+
+declare <2 x bfloat> @llvm.experimental.constrained.fptrunc.v2bf16.v2f32(<2 x float>, metadata, metadata)
+define <2 x bfloat> @vfptrunc_v2f32_v2bf16(<2 x float> %va) strictfp {
+; CHECK-LABEL: vfptrunc_v2f32_v2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <2 x bfloat> @llvm.experimental.constrained.fptrunc.v2bf16.v2f32(<2 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <2 x bfloat> %evec
+}
+
+declare <4 x bfloat> @llvm.experimental.constrained.fptrunc.v4bf16.v4f64(<4 x double>, metadata, metadata)
+define <4 x bfloat> @vfptrunc_v4f64_v4bf16(<4 x double> %va) strictfp {
+; CHECK-LABEL: vfptrunc_v4f64_v4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
+; CHECK-NEXT:    ret
+  %evec = call <4 x bfloat> @llvm.experimental.constrained.fptrunc.v4bf16.v4f64(<4 x double> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <4 x bfloat> %evec
+}
+
+declare <4 x bfloat> @llvm.experimental.constrained.fptrunc.v4bf16.v4f32(<4 x float>, metadata, metadata)
+define <4 x bfloat> @vfptrunc_v4f32_v4bf16(<4 x float> %va) strictfp {
+; CHECK-LABEL: vfptrunc_v4f32_v4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <4 x bfloat> @llvm.experimental.constrained.fptrunc.v4bf16.v4f32(<4 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <4 x bfloat> %evec
+}
+
+declare <8 x bfloat> @llvm.experimental.constrained.fptrunc.v8bf16.v8f64(<8 x double>, metadata, metadata)
+define <8 x bfloat> @vfptrunc_v8f64_v8bf16(<8 x double> %va) strictfp {
+; CHECK-LABEL: vfptrunc_v8f64_v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
+; CHECK-NEXT:    ret
+  %evec = call <8 x bfloat> @llvm.experimental.constrained.fptrunc.v8bf16.v8f64(<8 x double> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <8 x bfloat> %evec
+}
+
+declare <8 x bfloat> @llvm.experimental.constrained.fptrunc.v8bf16.v8f32(<8 x float>, metadata, metadata)
+define <8 x bfloat> @vfptrunc_v8f32_v8bf16(<8 x float> %va) strictfp {
+; CHECK-LABEL: vfptrunc_v8f32_v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v10, v8
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %evec = call <8 x bfloat> @llvm.experimental.constrained.fptrunc.v8bf16.v8f32(<8 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <8 x bfloat> %evec
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
index b15896580d42..0cd4f423a9df 100644
--- a/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple riscv32 -mattr=+m,+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple riscv64 -mattr=+m,+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple riscv32 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple riscv64 -mattr=+m,+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 8 x i32> @insert_nxv8i32_nxv4i32_0(<vscale x 8 x i32> %vec, <vscale x 4 x i32> %subvec) {
 ; CHECK-LABEL: insert_nxv8i32_nxv4i32_0:
@@ -531,6 +531,65 @@ define <vscale x 8 x i32> @insert_insert_combine2(<vscale x 2 x i32> %subvec) {
   ret <vscale x 8 x i32> %outer
 }
 
+define <vscale x 32 x bfloat> @insert_nxv32bf16_nxv2bf16_0(<vscale x 32 x bfloat> %vec, <vscale x 2 x bfloat> %subvec) {
+; CHECK-LABEL: insert_nxv32bf16_nxv2bf16_0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 2
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
+; CHECK-NEXT:    vmv.v.v v8, v16
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x bfloat> @llvm.vector.insert.nxv2bf16.nxv32bf16(<vscale x 32 x bfloat> %vec, <vscale x 2 x bfloat> %subvec, i64 0)
+  ret <vscale x 32 x bfloat> %v
+}
+
+define <vscale x 32 x bfloat> @insert_nxv32bf16_nxv2bf16_2(<vscale x 32 x bfloat> %vec, <vscale x 2 x bfloat> %subvec) {
+; CHECK-LABEL: insert_nxv32bf16_nxv2bf16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 2
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v16, a0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x bfloat> @llvm.vector.insert.nxv2bf16.nxv32bf16(<vscale x 32 x bfloat> %vec, <vscale x 2 x bfloat> %subvec, i64 2)
+  ret <vscale x 32 x bfloat> %v
+}
+
+define <vscale x 32 x bfloat> @insert_nxv32bf16_nxv2bf16_26(<vscale x 32 x bfloat> %vec, <vscale x 2 x bfloat> %subvec) {
+; CHECK-LABEL: insert_nxv32bf16_nxv2bf16_26:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a0, a0, 2
+; CHECK-NEXT:    add a1, a0, a0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v14, v16, a0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x bfloat> @llvm.vector.insert.nxv2bf16.nxv32bf16(<vscale x 32 x bfloat> %vec, <vscale x 2 x bfloat> %subvec, i64 26)
+  ret <vscale x 32 x bfloat> %v
+}
+
+define <vscale x 32 x bfloat> @insert_nxv32bf16_undef_nxv1bf16_0(<vscale x 1 x bfloat> %subvec) {
+; CHECK-LABEL: insert_nxv32bf16_undef_nxv1bf16_0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x bfloat> @llvm.vector.insert.nxv1bf16.nxv32bf16(<vscale x 32 x bfloat> undef, <vscale x 1 x bfloat> %subvec, i64 0)
+  ret <vscale x 32 x bfloat> %v
+}
+
+define <vscale x 32 x bfloat> @insert_nxv32bf16_undef_nxv1bf16_26(<vscale x 1 x bfloat> %subvec) {
+; CHECK-LABEL: insert_nxv32bf16_undef_nxv1bf16_26:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a1, a0, 3
+; CHECK-NEXT:    srli a0, a0, 2
+; CHECK-NEXT:    add a1, a0, a1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v14, v8, a0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 32 x bfloat> @llvm.vector.insert.nxv1bf16.nxv32bf16(<vscale x 32 x bfloat> undef, <vscale x 1 x bfloat> %subvec, i64 26)
+  ret <vscale x 32 x bfloat> %v
+}
 
 attributes #0 = { vscale_range(2,1024) }
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfpext-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfpext-constrained-sdnode.ll
index 5de309757c6d..8b49b720e851 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfpext-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfpext-constrained-sdnode.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.experimental.constrained.fpext.nxv1f32.nxv1f16(<vscale x 1 x half>, metadata)
@@ -151,3 +151,103 @@ define <vscale x 8 x double> @vfpext_nxv8f32_nxv8f64(<vscale x 8 x float> %va) s
   %evec = call <vscale x 8 x double> @llvm.experimental.constrained.fpext.nxv8f64.nxv8f32(<vscale x 8 x float> %va, metadata !"fpexcept.strict")
   ret <vscale x 8 x double> %evec
 }
+
+declare <vscale x 1 x float> @llvm.experimental.constrained.fpext.nxv1f32.nxv1bf16(<vscale x 1 x bfloat>, metadata)
+define <vscale x 1 x float> @vfpext_nxv1bf16_nxv1f32(<vscale x 1 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_nxv1bf16_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 1 x float> @llvm.experimental.constrained.fpext.nxv1f32.nxv1bf16(<vscale x 1 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <vscale x 1 x float> %evec
+}
+
+declare <vscale x 1 x double> @llvm.experimental.constrained.fpext.nxv1f64.nxv1bf16(<vscale x 1 x bfloat>, metadata)
+define <vscale x 1 x double> @vfpext_nxv1bf16_nxv1f64(<vscale x 1 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_nxv1bf16_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 1 x double> @llvm.experimental.constrained.fpext.nxv1f64.nxv1bf16(<vscale x 1 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <vscale x 1 x double> %evec
+}
+
+declare <vscale x 2 x float> @llvm.experimental.constrained.fpext.nxv2f32.nxv2bf16(<vscale x 2 x bfloat>, metadata)
+define <vscale x 2 x float> @vfpext_nxv2bf16_nxv2f32(<vscale x 2 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_nxv2bf16_nxv2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 2 x float> @llvm.experimental.constrained.fpext.nxv2f32.nxv2bf16(<vscale x 2 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <vscale x 2 x float> %evec
+}
+
+declare <vscale x 2 x double> @llvm.experimental.constrained.fpext.nxv2f64.nxv2bf16(<vscale x 2 x bfloat>, metadata)
+define <vscale x 2 x double> @vfpext_nxv2bf16_nxv2f64(<vscale x 2 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_nxv2bf16_nxv2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v10
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 2 x double> @llvm.experimental.constrained.fpext.nxv2f64.nxv2bf16(<vscale x 2 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <vscale x 2 x double> %evec
+}
+
+declare <vscale x 4 x float> @llvm.experimental.constrained.fpext.nxv4f32.nxv4bf16(<vscale x 4 x bfloat>, metadata)
+define <vscale x 4 x float> @vfpext_nxv4bf16_nxv4f32(<vscale x 4 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_nxv4bf16_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
+; CHECK-NEXT:    vmv2r.v v8, v10
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 4 x float> @llvm.experimental.constrained.fpext.nxv4f32.nxv4bf16(<vscale x 4 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <vscale x 4 x float> %evec
+}
+
+declare <vscale x 4 x double> @llvm.experimental.constrained.fpext.nxv4f64.nxv4bf16(<vscale x 4 x bfloat>, metadata)
+define <vscale x 4 x double> @vfpext_nxv4bf16_nxv4f64(<vscale x 4 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_nxv4bf16_nxv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v12
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 4 x double> @llvm.experimental.constrained.fpext.nxv4f64.nxv4bf16(<vscale x 4 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <vscale x 4 x double> %evec
+}
+
+declare <vscale x 8 x float> @llvm.experimental.constrained.fpext.nxv8f32.nxv8bf16(<vscale x 8 x bfloat>, metadata)
+define <vscale x 8 x float> @vfpext_nxv8bf16_nxv8f32(<vscale x 8 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_nxv8bf16_nxv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
+; CHECK-NEXT:    vmv4r.v v8, v12
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 8 x float> @llvm.experimental.constrained.fpext.nxv8f32.nxv8bf16(<vscale x 8 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <vscale x 8 x float> %evec
+}
+
+declare <vscale x 8 x double> @llvm.experimental.constrained.fpext.nxv8f64.nxv8bf16(<vscale x 8 x bfloat>, metadata)
+define <vscale x 8 x double> @vfpext_nxv8bf16_nxv8f64(<vscale x 8 x bfloat> %va) strictfp {
+; CHECK-LABEL: vfpext_nxv8bf16_nxv8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v16
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 8 x double> @llvm.experimental.constrained.fpext.nxv8f64.nxv8bf16(<vscale x 8 x bfloat> %va, metadata !"fpexcept.strict")
+  ret <vscale x 8 x double> %evec
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfpext-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfpext-sdnode.ll
index d805a103aafd..b002b8e76566 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfpext-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfpext-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x float> @vfpext_nxv1f16_nxv1f32(<vscale x 1 x half> %va) {
@@ -167,3 +167,115 @@ define <vscale x 8 x double> @vfpext_nxv8f32_nxv8f64(<vscale x 8 x float> %va) {
   %evec = fpext <vscale x 8 x float> %va to <vscale x 8 x double>
   ret <vscale x 8 x double> %evec
 }
+
+define <vscale x 1 x float> @vfpext_nxv1bf16_nxv1f32(<vscale x 1 x bfloat> %va) {
+;
+; CHECK-LABEL: vfpext_nxv1bf16_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = fpext <vscale x 1 x bfloat> %va to <vscale x 1 x float>
+  ret <vscale x 1 x float> %evec
+}
+
+define <vscale x 1 x double> @vfpext_nxv1bf16_nxv1f64(<vscale x 1 x bfloat> %va) {
+;
+; CHECK-LABEL: vfpext_nxv1bf16_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = fpext <vscale x 1 x bfloat> %va to <vscale x 1 x double>
+  ret <vscale x 1 x double> %evec
+}
+
+define <vscale x 2 x float> @vfpext_nxv2bf16_nxv2f32(<vscale x 2 x bfloat> %va) {
+;
+; CHECK-LABEL: vfpext_nxv2bf16_nxv2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = fpext <vscale x 2 x bfloat> %va to <vscale x 2 x float>
+  ret <vscale x 2 x float> %evec
+}
+
+define <vscale x 2 x double> @vfpext_nxv2bf16_nxv2f64(<vscale x 2 x bfloat> %va) {
+;
+; CHECK-LABEL: vfpext_nxv2bf16_nxv2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v10
+; CHECK-NEXT:    ret
+  %evec = fpext <vscale x 2 x bfloat> %va to <vscale x 2 x double>
+  ret <vscale x 2 x double> %evec
+}
+
+define <vscale x 4 x float> @vfpext_nxv4bf16_nxv4f32(<vscale x 4 x bfloat> %va) {
+;
+; CHECK-LABEL: vfpext_nxv4bf16_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
+; CHECK-NEXT:    vmv2r.v v8, v10
+; CHECK-NEXT:    ret
+  %evec = fpext <vscale x 4 x bfloat> %va to <vscale x 4 x float>
+  ret <vscale x 4 x float> %evec
+}
+
+define <vscale x 4 x double> @vfpext_nxv4bf16_nxv4f64(<vscale x 4 x bfloat> %va) {
+;
+; CHECK-LABEL: vfpext_nxv4bf16_nxv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v12
+; CHECK-NEXT:    ret
+  %evec = fpext <vscale x 4 x bfloat> %va to <vscale x 4 x double>
+  ret <vscale x 4 x double> %evec
+}
+
+define <vscale x 8 x float> @vfpext_nxv8bf16_nxv8f32(<vscale x 8 x bfloat> %va) {
+;
+; CHECK-LABEL: vfpext_nxv8bf16_nxv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
+; CHECK-NEXT:    vmv4r.v v8, v12
+; CHECK-NEXT:    ret
+  %evec = fpext <vscale x 8 x bfloat> %va to <vscale x 8 x float>
+  ret <vscale x 8 x float> %evec
+}
+
+define <vscale x 8 x double> @vfpext_nxv8bf16_nxv8f64(<vscale x 8 x bfloat> %va) {
+;
+; CHECK-LABEL: vfpext_nxv8bf16_nxv8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v16
+; CHECK-NEXT:    ret
+  %evec = fpext <vscale x 8 x bfloat> %va to <vscale x 8 x double>
+  ret <vscale x 8 x double> %evec
+}
+
+define <vscale x 16 x float> @vfpext_nxv16bf16_nxv16f32(<vscale x 16 x bfloat> %va) {
+;
+; CHECK-LABEL: vfpext_nxv16bf16_nxv16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v8
+; CHECK-NEXT:    vmv8r.v v8, v16
+; CHECK-NEXT:    ret
+  %evec = fpext <vscale x 16 x bfloat> %va to <vscale x 16 x float>
+  ret <vscale x 16 x float> %evec
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll
index 5cfa98916a2d..aaaf4ad46071 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfpext-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 2 x float> @llvm.vp.fpext.nxv2f32.nxv2f16(<vscale x 2 x half>, <vscale x 2 x i1>, i32)
 
@@ -120,3 +120,54 @@ define <vscale x 32 x float> @vfpext_nxv32f16_nxv32f32(<vscale x 32 x half> %a,
   %v = call <vscale x 32 x float> @llvm.vp.fpext.nxv32f32.nxv32f16(<vscale x 32 x half> %a, <vscale x 32 x i1> %m, i32 %vl)
   ret <vscale x 32 x float> %v
 }
+
+declare <vscale x 2 x float> @llvm.vp.fpext.nxv2f32.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x i1>, i32)
+
+define <vscale x 2 x float> @vfpext_nxv2bf16_nxv2f32(<vscale x 2 x bfloat> %a, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vfpext_nxv2bf16_nxv2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8, v0.t
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x float> @llvm.vp.fpext.nxv2f32.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x float> %v
+}
+
+define <vscale x 2 x float> @vfpext_nxv2bf16_nxv2f32_unmasked(<vscale x 2 x bfloat> %a, i32 zeroext %vl) {
+; CHECK-LABEL: vfpext_nxv2bf16_nxv2f32_unmasked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x float> @llvm.vp.fpext.nxv2f32.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x i1> splat (i1 true), i32 %vl)
+  ret <vscale x 2 x float> %v
+}
+
+declare <vscale x 2 x double> @llvm.vp.fpext.nxv2f64.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x i1>, i32)
+
+define <vscale x 2 x double> @vfpext_nxv2bf16_nxv2f64(<vscale x 2 x bfloat> %a, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vfpext_nxv2bf16_nxv2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v10, v0.t
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x double> @llvm.vp.fpext.nxv2f64.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x double> %v
+}
+
+define <vscale x 2 x double> @vfpext_nxv2bf16_nxv2f64_unmasked(<vscale x 2 x bfloat> %a, i32 zeroext %vl) {
+; CHECK-LABEL: vfpext_nxv2bf16_nxv2f64_unmasked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vfwcvt.f.f.v v8, v10
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x double> @llvm.vp.fpext.nxv2f64.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x i1> splat (i1 true), i32 %vl)
+  ret <vscale x 2 x double> %v
+}
+
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll
index 4404a275858f..4341f45dd6c7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.experimental.constrained.fptrunc.nxv1f32.nxv1f64(<vscale x 1 x double>, metadata, metadata)
@@ -155,3 +155,103 @@ define <vscale x 8 x half> @vfptrunc_nxv8f32_nxv8f16(<vscale x 8 x float> %va) s
   %evec = call <vscale x 8 x half> @llvm.experimental.constrained.fptrunc.nxv8f16.nxv8f32(<vscale x 8 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 8 x half> %evec
 }
+
+declare <vscale x 1 x bfloat> @llvm.experimental.constrained.fptrunc.nxv1bf16.nxv1f64(<vscale x 1 x double>, metadata, metadata)
+define <vscale x 1 x bfloat> @vfptrunc_nxv1f64_nxv1bf16(<vscale x 1 x double> %va) strictfp {
+; CHECK-LABEL: vfptrunc_nxv1f64_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v9, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 1 x bfloat> @llvm.experimental.constrained.fptrunc.nxv1bf16.nxv1f64(<vscale x 1 x double> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 1 x bfloat> %evec
+}
+
+declare <vscale x 1 x bfloat> @llvm.experimental.constrained.fptrunc.nxv1bf16.nxv1f32(<vscale x 1 x float>, metadata, metadata)
+define <vscale x 1 x bfloat> @vfptrunc_nxv1f32_nxv1bf16(<vscale x 1 x float> %va) strictfp {
+; CHECK-LABEL: vfptrunc_nxv1f32_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 1 x bfloat> @llvm.experimental.constrained.fptrunc.nxv1bf16.nxv1f32(<vscale x 1 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 1 x bfloat> %evec
+}
+
+declare <vscale x 2 x bfloat> @llvm.experimental.constrained.fptrunc.nxv2bf16.nxv2f64(<vscale x 2 x double>, metadata, metadata)
+define <vscale x 2 x bfloat> @vfptrunc_nxv2f64_nxv2bf16(<vscale x 2 x double> %va) strictfp {
+; CHECK-LABEL: vfptrunc_nxv2f64_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 2 x bfloat> @llvm.experimental.constrained.fptrunc.nxv2bf16.nxv2f64(<vscale x 2 x double> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 2 x bfloat> %evec
+}
+
+declare <vscale x 2 x bfloat> @llvm.experimental.constrained.fptrunc.nxv2bf16.nxv2f32(<vscale x 2 x float>, metadata, metadata)
+define <vscale x 2 x bfloat> @vfptrunc_nxv2f32_nxv2bf16(<vscale x 2 x float> %va) strictfp {
+; CHECK-LABEL: vfptrunc_nxv2f32_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 2 x bfloat> @llvm.experimental.constrained.fptrunc.nxv2bf16.nxv2f32(<vscale x 2 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 2 x bfloat> %evec
+}
+
+declare <vscale x 4 x bfloat> @llvm.experimental.constrained.fptrunc.nxv4bf16.nxv4f64(<vscale x 4 x double>, metadata, metadata)
+define <vscale x 4 x bfloat> @vfptrunc_nxv4f64_nxv4bf16(<vscale x 4 x double> %va) strictfp {
+; CHECK-LABEL: vfptrunc_nxv4f64_nxv4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v12
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 4 x bfloat> @llvm.experimental.constrained.fptrunc.nxv4bf16.nxv4f64(<vscale x 4 x double> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 4 x bfloat> %evec
+}
+
+declare <vscale x 4 x bfloat> @llvm.experimental.constrained.fptrunc.nxv4bf16.nxv4f32(<vscale x 4 x float>, metadata, metadata)
+define <vscale x 4 x bfloat> @vfptrunc_nxv4f32_nxv4bf16(<vscale x 4 x float> %va) strictfp {
+; CHECK-LABEL: vfptrunc_nxv4f32_nxv4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v10, v8
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 4 x bfloat> @llvm.experimental.constrained.fptrunc.nxv4bf16.nxv4f32(<vscale x 4 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 4 x bfloat> %evec
+}
+
+declare <vscale x 8 x bfloat> @llvm.experimental.constrained.fptrunc.nxv8bf16.nxv8f64(<vscale x 8 x double>, metadata, metadata)
+define <vscale x 8 x bfloat> @vfptrunc_nxv8f64_nxv8bf16(<vscale x 8 x double> %va) strictfp {
+; CHECK-LABEL: vfptrunc_nxv8f64_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v16, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v16
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 8 x bfloat> @llvm.experimental.constrained.fptrunc.nxv8bf16.nxv8f64(<vscale x 8 x double> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 8 x bfloat> %evec
+}
+
+declare <vscale x 8 x bfloat> @llvm.experimental.constrained.fptrunc.nxv8bf16.nxv8f32(<vscale x 8 x float>, metadata, metadata)
+define <vscale x 8 x bfloat> @vfptrunc_nxv8f32_nxv8bf16(<vscale x 8 x float> %va) strictfp {
+; CHECK-LABEL: vfptrunc_nxv8f32_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v12, v8
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %evec = call <vscale x 8 x bfloat> @llvm.experimental.constrained.fptrunc.nxv8bf16.nxv8f32(<vscale x 8 x float> %va, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 8 x bfloat> %evec
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-sdnode.ll
index d715b46e95fe..9148a79cb740 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-sdnode.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+experimental-zvfbfmin -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x half> @vfptrunc_nxv1f32_nxv1f16(<vscale x 1 x float> %va) {
@@ -167,3 +167,76 @@ define <vscale x 8 x float> @vfptrunc_nxv8f64_nxv8f32(<vscale x 8 x double> %va)
   %evec = fptrunc <vscale x 8 x double> %va to <vscale x 8 x float>
   ret <vscale x 8 x float> %evec
 }
+
+define <vscale x 1 x bfloat> @vfptrunc_nxv1f32_nxv1bf16(<vscale x 1 x float> %va) {
+;
+; CHECK-LABEL: vfptrunc_nxv1f32_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = fptrunc <vscale x 1 x float> %va to <vscale x 1 x bfloat>
+  ret <vscale x 1 x bfloat> %evec
+}
+
+define <vscale x 2 x bfloat> @vfptrunc_nxv2f32_nxv2bf16(<vscale x 2 x float> %va) {
+;
+; CHECK-LABEL: vfptrunc_nxv2f32_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %evec = fptrunc <vscale x 2 x float> %va to <vscale x 2 x bfloat>
+  ret <vscale x 2 x bfloat> %evec
+}
+
+define <vscale x 4 x bfloat> @vfptrunc_nxv4f32_nxv4bf16(<vscale x 4 x float> %va) {
+;
+; CHECK-LABEL: vfptrunc_nxv4f32_nxv4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v10, v8
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %evec = fptrunc <vscale x 4 x float> %va to <vscale x 4 x bfloat>
+  ret <vscale x 4 x bfloat> %evec
+}
+
+define <vscale x 8 x bfloat> @vfptrunc_nxv8f32_nxv8bf16(<vscale x 8 x float> %va) {
+;
+; CHECK-LABEL: vfptrunc_nxv8f32_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v12, v8
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %evec = fptrunc <vscale x 8 x float> %va to <vscale x 8 x bfloat>
+  ret <vscale x 8 x bfloat> %evec
+}
+
+define <vscale x 16 x bfloat> @vfptrunc_nxv16f32_nxv16bf16(<vscale x 16 x float> %va) {
+;
+; CHECK-LABEL: vfptrunc_nxv16f32_nxv16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v16, v8
+; CHECK-NEXT:    vmv.v.v v8, v16
+; CHECK-NEXT:    ret
+  %evec = fptrunc <vscale x 16 x float> %va to <vscale x 16 x bfloat>
+  ret <vscale x 16 x bfloat> %evec
+}
+
+define <vscale x 1 x bfloat> @vfptrunc_nxv1f64_nxv1bf16(<vscale x 1 x double> %va) {
+;
+; CHECK-LABEL: vfptrunc_nxv1f64_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v9, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v9
+; CHECK-NEXT:    ret
+  %evec = fptrunc <vscale x 1 x double> %va to <vscale x 1 x bfloat>
+  ret <vscale x 1 x bfloat> %evec
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll
index dd122f1f2511..0c3abe37af27 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-vp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+m,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m,+experimental-zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 2 x half> @llvm.vp.fptrunc.nxv2f16.nxv2f32(<vscale x 2 x float>, <vscale x 2 x i1>, i32)
 
@@ -218,3 +218,53 @@ define <vscale x 32 x float> @vfptrunc_nxv32f32_nxv32f64(<vscale x 32 x double>
   %v = call <vscale x 32 x float> @llvm.vp.fptrunc.nxv32f64.nxv32f32(<vscale x 32 x double> %a, <vscale x 32 x i1> %m, i32 %vl)
   ret <vscale x 32 x float> %v
 }
+
+declare <vscale x 2 x bfloat> @llvm.vp.fptrunc.nxv2bf16.nxv2f32(<vscale x 2 x float>, <vscale x 2 x i1>, i32)
+
+define <vscale x 2 x bfloat> @vfptrunc_nxv2bf16_nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vfptrunc_nxv2bf16_nxv2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v9, v8, v0.t
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x bfloat> @llvm.vp.fptrunc.nxv2bf16.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x bfloat> %v
+}
+
+define <vscale x 2 x bfloat> @vfptrunc_nxv2bf16_nxv2f32_unmasked(<vscale x 2 x float> %a, i32 zeroext %vl) {
+; CHECK-LABEL: vfptrunc_nxv2bf16_nxv2f32_unmasked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x bfloat> @llvm.vp.fptrunc.nxv2bf16.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x i1> splat (i1 true), i32 %vl)
+  ret <vscale x 2 x bfloat> %v
+}
+
+declare <vscale x 2 x bfloat> @llvm.vp.fptrunc.nxv2bf16.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, i32)
+
+define <vscale x 2 x bfloat> @vfptrunc_nxv2bf16_nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: vfptrunc_nxv2bf16_nxv2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v10, v8, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10, v0.t
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x bfloat> @llvm.vp.fptrunc.nxv2bf16.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x i1> %m, i32 %vl)
+  ret <vscale x 2 x bfloat> %v
+}
+
+define <vscale x 2 x bfloat> @vfptrunc_nxv2bf16_nxv2f64_unmasked(<vscale x 2 x double> %a, i32 zeroext %vl) {
+; CHECK-LABEL: vfptrunc_nxv2bf16_nxv2f64_unmasked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vfncvt.rod.f.f.w v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vfncvtbf16.f.f.w v8, v10
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x bfloat> @llvm.vp.fptrunc.nxv2bf16.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x i1> splat (i1 true), i32 %vl)
+  ret <vscale x 2 x bfloat> %v
+}
-- 
GitLab


From aac83fcf3ec6bbe5e0d83b76d2d236b1b4bfbe89 Mon Sep 17 00:00:00 2001
From: SahilPatidar <patidarsahil2001@gmail.com>
Date: Tue, 7 May 2024 08:14:29 +0530
Subject: [PATCH 0005/1206] [Reassociate] Adds test coverage for reassociation
 of scalar & vector boolean types (#89899)

First step for #64840.
---
 .../Transforms/Reassociate/reassoc_bool.ll    | 207 ++++++++++++++++
 .../Reassociate/reassoc_bool_vec.ll           | 227 ++++++++++++++++++
 2 files changed, 434 insertions(+)
 create mode 100644 llvm/test/Transforms/Reassociate/reassoc_bool.ll
 create mode 100644 llvm/test/Transforms/Reassociate/reassoc_bool_vec.ll

diff --git a/llvm/test/Transforms/Reassociate/reassoc_bool.ll b/llvm/test/Transforms/Reassociate/reassoc_bool.ll
new file mode 100644
index 000000000000..935a1e8c31a0
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/reassoc_bool.ll
@@ -0,0 +1,207 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=reassociate -S | FileCheck %s
+
+define i1 @scalar(i1 %b0, i1 %b1, i1 %b2, i1 %b3, i1 %b4, i1 %b5, i1 %b6, i1 %b7) {
+; CHECK-LABEL: define i1 @scalar(
+; CHECK-SAME: i1 [[B0:%.*]], i1 [[B1:%.*]], i1 [[B2:%.*]], i1 [[B3:%.*]], i1 [[B4:%.*]], i1 [[B5:%.*]], i1 [[B6:%.*]], i1 [[B7:%.*]]) {
+; CHECK-NEXT:    [[OR01:%.*]] = or i1 [[B0]], [[B1]]
+; CHECK-NEXT:    [[OR23:%.*]] = or i1 [[B2]], [[B3]]
+; CHECK-NEXT:    [[OR45:%.*]] = or i1 [[B4]], [[B5]]
+; CHECK-NEXT:    [[OR67:%.*]] = or i1 [[B6]], [[B7]]
+; CHECK-NEXT:    [[OR0123:%.*]] = or i1 [[OR01]], [[OR23]]
+; CHECK-NEXT:    [[OR4567:%.*]] = or i1 [[OR45]], [[OR67]]
+; CHECK-NEXT:    [[OR01234567:%.*]] = or i1 [[OR0123]], [[OR4567]]
+; CHECK-NEXT:    ret i1 [[OR01234567]]
+;
+  %or01 = or i1 %b0, %b1
+  %or23 = or i1 %b2, %b3
+  %or45 = or i1 %b4, %b5
+  %or67 = or i1 %b6, %b7
+  %or0123 = or i1 %or01, %or23
+  %or4567 = or i1 %or45, %or67
+  %or01234567 = or i1 %or0123, %or4567
+  ret i1 %or01234567
+}
+
+
+define i1 @scalar1(i1 %a, i1 %b0, i1 %b1, i1 %b2, i1 %b3, i1 %b4, i1 %b5, i1 %b6, i1 %b7) {
+; CHECK-LABEL: define i1 @scalar1(
+; CHECK-SAME: i1 [[A:%.*]], i1 [[B0:%.*]], i1 [[B1:%.*]], i1 [[B2:%.*]], i1 [[B3:%.*]], i1 [[B4:%.*]], i1 [[B5:%.*]], i1 [[B6:%.*]], i1 [[B7:%.*]]) {
+; CHECK-NEXT:    [[OR0:%.*]] = or i1 [[A]], [[B0]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i1 [[A]], [[B1]]
+; CHECK-NEXT:    [[OR2:%.*]] = or i1 [[A]], [[B2]]
+; CHECK-NEXT:    [[OR3:%.*]] = or i1 [[A]], [[B3]]
+; CHECK-NEXT:    [[OR4:%.*]] = or i1 [[A]], [[B4]]
+; CHECK-NEXT:    [[OR5:%.*]] = or i1 [[A]], [[B5]]
+; CHECK-NEXT:    [[OR6:%.*]] = or i1 [[A]], [[B6]]
+; CHECK-NEXT:    [[OR7:%.*]] = or i1 [[A]], [[B7]]
+; CHECK-NEXT:    [[XOR0:%.*]] = xor i1 [[OR0]], [[OR1]]
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i1 [[XOR0]], [[OR2]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i1 [[XOR1]], [[OR3]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i1 [[XOR2]], [[OR4]]
+; CHECK-NEXT:    [[XOR4:%.*]] = xor i1 [[XOR3]], [[OR5]]
+; CHECK-NEXT:    [[XOR5:%.*]] = xor i1 [[XOR4]], [[OR6]]
+; CHECK-NEXT:    [[XOR6:%.*]] = xor i1 [[XOR5]], [[OR7]]
+; CHECK-NEXT:    [[OR001:%.*]] = or i1 [[XOR0]], [[XOR1]]
+; CHECK-NEXT:    [[OR023:%.*]] = or i1 [[XOR2]], [[XOR3]]
+; CHECK-NEXT:    [[OR045:%.*]] = or i1 [[XOR4]], [[XOR5]]
+; CHECK-NEXT:    [[OR060:%.*]] = or i1 [[XOR0]], [[XOR6]]
+; CHECK-NEXT:    [[OR0123:%.*]] = or i1 [[OR001]], [[OR023]]
+; CHECK-NEXT:    [[OR4560:%.*]] = or i1 [[OR045]], [[OR060]]
+; CHECK-NEXT:    [[OR01234567:%.*]] = or i1 [[OR0123]], [[OR4560]]
+; CHECK-NEXT:    ret i1 [[OR01234567]]
+;
+  %or0 = or i1 %b0, %a
+  %or1 = or i1 %b1, %a
+  %or2 = or i1 %b2, %a
+  %or3 = or i1 %b3, %a
+  %or4 = or i1 %b4, %a
+  %or5 = or i1 %b5, %a
+  %or6 = or i1 %b6, %a
+  %or7 = or i1 %b7, %a
+  %xor0 = xor i1 %or0, %or1
+  %xor1 = xor i1 %xor0, %or2
+  %xor2 = xor i1 %xor1, %or3
+  %xor3 = xor i1 %xor2, %or4
+  %xor4 = xor i1 %xor3, %or5
+  %xor5 = xor i1 %xor4, %or6
+  %xor6 = xor i1 %xor5, %or7
+  %or001 = or i1 %xor0, %xor1
+  %or023 = or i1 %xor2, %xor3
+  %or045 = or i1 %xor4, %xor5
+  %or060 = or i1 %xor6, %xor0
+  %or0123 = or i1 %or001, %or023
+  %or4560 = or i1 %or045, %or060
+  %or01234567 = or i1 %or0123, %or4560
+  ret i1 %or01234567
+}
+
+define i1 @scalar2(i1 %a, i1 %b0, i1 %b1, i1 %b2, i1 %b3, i1 %b4, i1 %b5, i1 %b6, i1 %b7) {
+; CHECK-LABEL: define i1 @scalar2(
+; CHECK-SAME: i1 [[A:%.*]], i1 [[B0:%.*]], i1 [[B1:%.*]], i1 [[B2:%.*]], i1 [[B3:%.*]], i1 [[B4:%.*]], i1 [[B5:%.*]], i1 [[B6:%.*]], i1 [[B7:%.*]]) {
+; CHECK-NEXT:    [[OR0:%.*]] = or i1 [[A]], [[B0]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i1 [[A]], [[B1]]
+; CHECK-NEXT:    [[OR2:%.*]] = or i1 [[A]], [[B2]]
+; CHECK-NEXT:    [[OR3:%.*]] = or i1 [[A]], [[B3]]
+; CHECK-NEXT:    [[OR4:%.*]] = or i1 [[A]], [[B4]]
+; CHECK-NEXT:    [[OR5:%.*]] = or i1 [[A]], [[B5]]
+; CHECK-NEXT:    [[OR6:%.*]] = or i1 [[A]], [[B6]]
+; CHECK-NEXT:    [[OR7:%.*]] = or i1 [[A]], [[B7]]
+; CHECK-NEXT:    [[XOR0:%.*]] = xor i1 [[OR0]], [[OR1]]
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i1 [[OR2]], [[OR3]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i1 [[OR4]], [[OR5]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i1 [[OR6]], [[OR7]]
+; CHECK-NEXT:    [[OR01:%.*]] = xor i1 [[XOR0]], [[XOR1]]
+; CHECK-NEXT:    [[OR23:%.*]] = xor i1 [[XOR2]], [[XOR3]]
+; CHECK-NEXT:    [[OR0123:%.*]] = xor i1 [[OR01]], [[OR23]]
+; CHECK-NEXT:    ret i1 [[OR0123]]
+;
+  %or0 = or i1 %b0, %a
+  %or1 = or i1 %b1, %a
+  %or2 = or i1 %b2, %a
+  %or3 = or i1 %b3, %a
+  %or4 = or i1 %b4, %a
+  %or5 = or i1 %b5, %a
+  %or6 = or i1 %b6, %a
+  %or7 = or i1 %b7, %a
+  %xor0 = xor i1 %or0, %or1
+  %xor1 = xor i1 %or2, %or3
+  %xor2 = xor i1 %or4, %or5
+  %xor3 = xor i1 %or6, %or7
+  %or01 = xor i1 %xor0, %xor1
+  %or23 = xor i1 %xor2, %xor3
+  %or0123 = xor i1 %or01, %or23
+  ret i1 %or0123
+}
+
+define i1 @scalar3(i1 %a, i1 %b0, i1 %b1, i1 %b2, i1 %b3, i1 %b4, i1 %b5, i1 %b6, i1 %b7) {
+; CHECK-LABEL: define i1 @scalar3(
+; CHECK-SAME: i1 [[A:%.*]], i1 [[B0:%.*]], i1 [[B1:%.*]], i1 [[B2:%.*]], i1 [[B3:%.*]], i1 [[B4:%.*]], i1 [[B5:%.*]], i1 [[B6:%.*]], i1 [[B7:%.*]]) {
+; CHECK-NEXT:    [[XOR0:%.*]] = xor i1 [[A]], [[B0]]
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i1 [[A]], [[B1]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i1 [[A]], [[B2]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i1 [[A]], [[B3]]
+; CHECK-NEXT:    [[XOR4:%.*]] = xor i1 [[A]], [[B4]]
+; CHECK-NEXT:    [[XOR5:%.*]] = xor i1 [[A]], [[B5]]
+; CHECK-NEXT:    [[XOR6:%.*]] = xor i1 [[A]], [[B6]]
+; CHECK-NEXT:    [[XOR7:%.*]] = xor i1 [[A]], [[B7]]
+; CHECK-NEXT:    [[AND0:%.*]] = and i1 [[XOR0]], [[XOR1]]
+; CHECK-NEXT:    [[AND1:%.*]] = and i1 [[XOR2]], [[XOR3]]
+; CHECK-NEXT:    [[AND2:%.*]] = and i1 [[XOR4]], [[XOR5]]
+; CHECK-NEXT:    [[AND3:%.*]] = and i1 [[XOR6]], [[XOR7]]
+; CHECK-NEXT:    [[OR01:%.*]] = and i1 [[AND0]], [[AND1]]
+; CHECK-NEXT:    [[OR23:%.*]] = and i1 [[AND2]], [[AND3]]
+; CHECK-NEXT:    [[OR0123:%.*]] = and i1 [[OR01]], [[OR23]]
+; CHECK-NEXT:    ret i1 [[OR0123]]
+;
+  %xor0 = xor i1 %b0, %a
+  %xor1 = xor i1 %b1, %a
+  %xor2 = xor i1 %b2, %a
+  %xor3 = xor i1 %b3, %a
+  %xor4 = xor i1 %b4, %a
+  %xor5 = xor i1 %b5, %a
+  %xor6 = xor i1 %b6, %a
+  %xor7 = xor i1 %b7, %a
+  %and0 = and i1 %xor0, %xor1
+  %and1 = and i1 %xor2, %xor3
+  %and2 = and i1 %xor4, %xor5
+  %and3 = and i1 %xor6, %xor7
+  %or01 = and i1 %and0, %and1
+  %or23 = and i1 %and2, %and3
+  %or0123 = and i1 %or01, %or23
+  ret i1 %or0123
+}
+
+define i1 @scalar4(i1 %a, i1 %b0, i1 %b1, i1 %b2, i1 %b3, i1 %b4, i1 %b5, i1 %b6, i1 %b7) {
+; CHECK-LABEL: define i1 @scalar4(
+; CHECK-SAME: i1 [[A:%.*]], i1 [[B0:%.*]], i1 [[B1:%.*]], i1 [[B2:%.*]], i1 [[B3:%.*]], i1 [[B4:%.*]], i1 [[B5:%.*]], i1 [[B6:%.*]], i1 [[B7:%.*]]) {
+; CHECK-NEXT:    [[XOR0:%.*]] = xor i1 [[A]], [[B0]]
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i1 [[A]], [[B1]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i1 [[A]], [[B2]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i1 [[A]], [[B3]]
+; CHECK-NEXT:    [[XOR4:%.*]] = xor i1 [[A]], [[B4]]
+; CHECK-NEXT:    [[XOR5:%.*]] = xor i1 [[A]], [[B5]]
+; CHECK-NEXT:    [[XOR6:%.*]] = xor i1 [[A]], [[B6]]
+; CHECK-NEXT:    [[XOR7:%.*]] = xor i1 [[A]], [[B7]]
+; CHECK-NEXT:    [[OR0:%.*]] = or i1 [[XOR0]], [[XOR1]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i1 [[XOR2]], [[XOR3]]
+; CHECK-NEXT:    [[OR2:%.*]] = or i1 [[XOR4]], [[XOR5]]
+; CHECK-NEXT:    [[OR3:%.*]] = or i1 [[XOR6]], [[XOR7]]
+; CHECK-NEXT:    [[OR4:%.*]] = or i1 [[B0]], [[B1]]
+; CHECK-NEXT:    [[OR5:%.*]] = or i1 [[B2]], [[B3]]
+; CHECK-NEXT:    [[OR6:%.*]] = or i1 [[B4]], [[B5]]
+; CHECK-NEXT:    [[OR7:%.*]] = or i1 [[B6]], [[B7]]
+; CHECK-NEXT:    [[OR01:%.*]] = or i1 [[OR0]], [[OR1]]
+; CHECK-NEXT:    [[OR23:%.*]] = or i1 [[OR2]], [[OR3]]
+; CHECK-NEXT:    [[OR45:%.*]] = or i1 [[OR4]], [[OR5]]
+; CHECK-NEXT:    [[OR67:%.*]] = or i1 [[OR6]], [[OR7]]
+; CHECK-NEXT:    [[OR0123:%.*]] = or i1 [[OR01]], [[OR23]]
+; CHECK-NEXT:    [[OR4567:%.*]] = or i1 [[OR45]], [[OR67]]
+; CHECK-NEXT:    [[OR01234567:%.*]] = or i1 [[OR4567]], [[OR0123]]
+; CHECK-NEXT:    ret i1 [[OR01234567]]
+;
+  %xor0 = xor i1 %b0, %a
+  %xor1 = xor i1 %b1, %a
+  %xor2 = xor i1 %b2, %a
+  %xor3 = xor i1 %b3, %a
+  %xor4 = xor i1 %b4, %a
+  %xor5 = xor i1 %b5, %a
+  %xor6 = xor i1 %b6, %a
+  %xor7 = xor i1 %b7, %a
+  %or0 = or i1 %xor0, %xor1
+  %or1 = or i1 %xor2, %xor3
+  %or2 = or i1 %xor4, %xor5
+  %or3 = or i1 %xor6, %xor7
+  %or4 = or i1 %b0, %b1
+  %or5 = or i1 %b2, %b3
+  %or6 = or i1 %b4, %b5
+  %or7 = or i1 %b6, %b7
+  %or01 = or i1 %or0, %or1
+  %or23 = or i1 %or2, %or3
+  %or45 = or i1 %or4, %or5
+  %or67 = or i1 %or6, %or7
+  %or0123 = or i1 %or01, %or23
+  %or4567 = or i1 %or45, %or67
+  %or01234567 = or i1 %or0123, %or4567
+  ret i1 %or01234567
+}
diff --git a/llvm/test/Transforms/Reassociate/reassoc_bool_vec.ll b/llvm/test/Transforms/Reassociate/reassoc_bool_vec.ll
new file mode 100644
index 000000000000..fcedde23ecc7
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/reassoc_bool_vec.ll
@@ -0,0 +1,227 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=reassociate -S | FileCheck %s
+
+define <8 x i1> @vector0(<8 x i1> %b0, <8 x i1> %b1, <8 x i1> %b2, <8 x i1> %b3, <8 x i1> %b4, <8 x i1> %b5, <8 x i1> %b6, <8 x i1> %b7) {
+; CHECK-LABEL: define <8 x i1> @vector0(
+; CHECK-SAME: <8 x i1> [[B0:%.*]], <8 x i1> [[B1:%.*]], <8 x i1> [[B2:%.*]], <8 x i1> [[B3:%.*]], <8 x i1> [[B4:%.*]], <8 x i1> [[B5:%.*]], <8 x i1> [[B6:%.*]], <8 x i1> [[B7:%.*]]) {
+; CHECK-NEXT:    [[OR67:%.*]] = or <8 x i1> [[B1]], [[B0]]
+; CHECK-NEXT:    [[OR45:%.*]] = or <8 x i1> [[OR67]], [[B2]]
+; CHECK-NEXT:    [[OR4567:%.*]] = or <8 x i1> [[OR45]], [[B3]]
+; CHECK-NEXT:    [[OR23:%.*]] = or <8 x i1> [[OR4567]], [[B4]]
+; CHECK-NEXT:    [[OR01:%.*]] = or <8 x i1> [[OR23]], [[B5]]
+; CHECK-NEXT:    [[OR0123:%.*]] = or <8 x i1> [[OR01]], [[B6]]
+; CHECK-NEXT:    [[OR01234567:%.*]] = or <8 x i1> [[OR0123]], [[B7]]
+; CHECK-NEXT:    ret <8 x i1> [[OR01234567]]
+;
+  %or01 = or <8 x i1> %b0, %b1
+  %or23 = or <8 x i1> %b2, %b3
+  %or45 = or <8 x i1> %b4, %b5
+  %or67 = or <8 x i1> %b6, %b7
+  %or0123 = or <8 x i1> %or01, %or23
+  %or4567 = or <8 x i1> %or45, %or67
+  %or01234567 = or <8 x i1> %or0123, %or4567
+  ret <8 x i1> %or01234567
+}
+
+define <8 x i1> @vector1(<8 x i1> %b0, <8 x i1> %b1, <8 x i1> %b2, <8 x i1> %b3, <8 x i1> %b4, <8 x i1> %b5, <8 x i1> %b6, <8 x i1> %b7) {
+; CHECK-LABEL: define <8 x i1> @vector1(
+; CHECK-SAME: <8 x i1> [[B0:%.*]], <8 x i1> [[B1:%.*]], <8 x i1> [[B2:%.*]], <8 x i1> [[B3:%.*]], <8 x i1> [[B4:%.*]], <8 x i1> [[B5:%.*]], <8 x i1> [[B6:%.*]], <8 x i1> [[B7:%.*]]) {
+; CHECK-NEXT:    [[OR67:%.*]] = and <8 x i1> [[B1]], [[B0]]
+; CHECK-NEXT:    [[OR45:%.*]] = and <8 x i1> [[OR67]], [[B2]]
+; CHECK-NEXT:    [[OR4567:%.*]] = and <8 x i1> [[OR45]], [[B3]]
+; CHECK-NEXT:    [[OR23:%.*]] = and <8 x i1> [[OR4567]], [[B4]]
+; CHECK-NEXT:    [[OR01:%.*]] = and <8 x i1> [[OR23]], [[B5]]
+; CHECK-NEXT:    [[OR0123:%.*]] = and <8 x i1> [[OR01]], [[B6]]
+; CHECK-NEXT:    [[OR01234567:%.*]] = and <8 x i1> [[OR0123]], [[B7]]
+; CHECK-NEXT:    ret <8 x i1> [[OR01234567]]
+;
+  %or01 = and <8 x i1> %b0, %b1
+  %or23 = and <8 x i1> %b2, %b3
+  %or45 = and <8 x i1> %b4, %b5
+  %or67 = and <8 x i1> %b6, %b7
+  %or0123 = and <8 x i1> %or01, %or23
+  %or4567 = and <8 x i1> %or45, %or67
+  %or01234567 = and <8 x i1> %or0123, %or4567
+  ret <8 x i1> %or01234567
+}
+
+define <8 x i1> @vector2(<8 x i1> %a, <8 x i1> %b0, <8 x i1> %b1, <8 x i1> %b2, <8 x i1> %b3, <8 x i1> %b4, <8 x i1> %b5, <8 x i1> %b6, <8 x i1> %b7) {
+; CHECK-LABEL: define <8 x i1> @vector2(
+; CHECK-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B0:%.*]], <8 x i1> [[B1:%.*]], <8 x i1> [[B2:%.*]], <8 x i1> [[B3:%.*]], <8 x i1> [[B4:%.*]], <8 x i1> [[B5:%.*]], <8 x i1> [[B6:%.*]], <8 x i1> [[B7:%.*]]) {
+; CHECK-NEXT:    [[OR0:%.*]] = or <8 x i1> [[B0]], [[A]]
+; CHECK-NEXT:    [[OR1:%.*]] = or <8 x i1> [[B1]], [[A]]
+; CHECK-NEXT:    [[OR2:%.*]] = or <8 x i1> [[B2]], [[A]]
+; CHECK-NEXT:    [[OR3:%.*]] = or <8 x i1> [[B3]], [[A]]
+; CHECK-NEXT:    [[OR4:%.*]] = or <8 x i1> [[B4]], [[A]]
+; CHECK-NEXT:    [[OR5:%.*]] = or <8 x i1> [[B5]], [[A]]
+; CHECK-NEXT:    [[OR6:%.*]] = or <8 x i1> [[B6]], [[A]]
+; CHECK-NEXT:    [[OR7:%.*]] = or <8 x i1> [[B7]], [[A]]
+; CHECK-NEXT:    [[XOR0:%.*]] = xor <8 x i1> [[OR1]], [[OR0]]
+; CHECK-NEXT:    [[XOR1:%.*]] = xor <8 x i1> [[XOR0]], [[OR2]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor <8 x i1> [[XOR1]], [[OR3]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor <8 x i1> [[XOR2]], [[OR4]]
+; CHECK-NEXT:    [[XOR4:%.*]] = xor <8 x i1> [[XOR3]], [[OR5]]
+; CHECK-NEXT:    [[XOR5:%.*]] = xor <8 x i1> [[XOR4]], [[OR6]]
+; CHECK-NEXT:    [[XOR6:%.*]] = xor <8 x i1> [[XOR5]], [[OR7]]
+; CHECK-NEXT:    [[OR045:%.*]] = or <8 x i1> [[XOR1]], [[XOR0]]
+; CHECK-NEXT:    [[OR4560:%.*]] = or <8 x i1> [[OR045]], [[XOR2]]
+; CHECK-NEXT:    [[OR023:%.*]] = or <8 x i1> [[OR4560]], [[XOR3]]
+; CHECK-NEXT:    [[OR001:%.*]] = or <8 x i1> [[OR023]], [[XOR4]]
+; CHECK-NEXT:    [[OR0123:%.*]] = or <8 x i1> [[OR001]], [[XOR5]]
+; CHECK-NEXT:    [[OR01234567:%.*]] = or <8 x i1> [[OR0123]], [[XOR6]]
+; CHECK-NEXT:    ret <8 x i1> [[OR01234567]]
+;
+  %or0 = or <8 x i1> %b0, %a
+  %or1 = or <8 x i1> %b1, %a
+  %or2 = or <8 x i1> %b2, %a
+  %or3 = or <8 x i1> %b3, %a
+  %or4 = or <8 x i1> %b4, %a
+  %or5 = or <8 x i1> %b5, %a
+  %or6 = or <8 x i1> %b6, %a
+  %or7 = or <8 x i1> %b7, %a
+  %xor0 = xor <8 x i1> %or0, %or1
+  %xor1 = xor <8 x i1> %xor0, %or2
+  %xor2 = xor <8 x i1> %xor1, %or3
+  %xor3 = xor <8 x i1> %xor2, %or4
+  %xor4 = xor <8 x i1> %xor3, %or5
+  %xor5 = xor <8 x i1> %xor4, %or6
+  %xor6 = xor <8 x i1> %xor5, %or7
+  %or001 = or <8 x i1> %xor0, %xor1
+  %or023 = or <8 x i1> %xor2, %xor3
+  %or045 = or <8 x i1> %xor4, %xor5
+  %or060 = or <8 x i1> %xor6, %xor0
+  %or0123 = or <8 x i1> %or001, %or023
+  %or4560 = or <8 x i1> %or045, %or060
+  %or01234567 = or <8 x i1> %or0123, %or4560
+  ret <8 x i1> %or01234567
+}
+
+define <8 x i1> @vector3(<8 x i1> %a, <8 x i1> %b0, <8 x i1> %b1, <8 x i1> %b2, <8 x i1> %b3, <8 x i1> %b4, <8 x i1> %b5, <8 x i1> %b6, <8 x i1> %b7) {
+; CHECK-LABEL: define <8 x i1> @vector3(
+; CHECK-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B0:%.*]], <8 x i1> [[B1:%.*]], <8 x i1> [[B2:%.*]], <8 x i1> [[B3:%.*]], <8 x i1> [[B4:%.*]], <8 x i1> [[B5:%.*]], <8 x i1> [[B6:%.*]], <8 x i1> [[B7:%.*]]) {
+; CHECK-NEXT:    [[OR0:%.*]] = or <8 x i1> [[B0]], [[A]]
+; CHECK-NEXT:    [[OR1:%.*]] = or <8 x i1> [[B1]], [[A]]
+; CHECK-NEXT:    [[OR2:%.*]] = or <8 x i1> [[B2]], [[A]]
+; CHECK-NEXT:    [[OR3:%.*]] = or <8 x i1> [[B3]], [[A]]
+; CHECK-NEXT:    [[OR4:%.*]] = or <8 x i1> [[B4]], [[A]]
+; CHECK-NEXT:    [[OR5:%.*]] = or <8 x i1> [[B5]], [[A]]
+; CHECK-NEXT:    [[OR6:%.*]] = or <8 x i1> [[B6]], [[A]]
+; CHECK-NEXT:    [[OR7:%.*]] = or <8 x i1> [[B7]], [[A]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor <8 x i1> [[OR1]], [[OR0]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor <8 x i1> [[XOR3]], [[OR2]]
+; CHECK-NEXT:    [[XOR7:%.*]] = xor <8 x i1> [[XOR2]], [[OR3]]
+; CHECK-NEXT:    [[XOR0:%.*]] = xor <8 x i1> [[XOR7]], [[OR4]]
+; CHECK-NEXT:    [[XOR4:%.*]] = xor <8 x i1> [[XOR0]], [[OR5]]
+; CHECK-NEXT:    [[XOR5:%.*]] = xor <8 x i1> [[XOR4]], [[OR6]]
+; CHECK-NEXT:    [[OR4560:%.*]] = xor <8 x i1> [[XOR5]], [[OR7]]
+; CHECK-NEXT:    ret <8 x i1> [[OR4560]]
+;
+  %or0 = or <8 x i1> %b0, %a
+  %or1 = or <8 x i1> %b1, %a
+  %or2 = or <8 x i1> %b2, %a
+  %or3 = or <8 x i1> %b3, %a
+  %or4 = or <8 x i1> %b4, %a
+  %or5 = or <8 x i1> %b5, %a
+  %or6 = or <8 x i1> %b6, %a
+  %or7 = or <8 x i1> %b7, %a
+  %xor0 = xor <8 x i1> %or0, %or1
+  %xor1 = xor <8 x i1> %or2, %or3
+  %xor2 = xor <8 x i1> %or4, %or5
+  %xor3 = xor <8 x i1> %or6, %or7
+  %or01 = xor <8 x i1> %xor0, %xor1
+  %or23 = xor <8 x i1> %xor2, %xor3
+  %or0123 = xor <8 x i1> %or01, %or23
+  ret <8 x i1> %or0123
+}
+
+define <8 x i1> @vector4(<8 x i1> %a, <8 x i1> %b0, <8 x i1> %b1, <8 x i1> %b2, <8 x i1> %b3, <8 x i1> %b4, <8 x i1> %b5, <8 x i1> %b6, <8 x i1> %b7) {
+; CHECK-LABEL: define <8 x i1> @vector4(
+; CHECK-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B0:%.*]], <8 x i1> [[B1:%.*]], <8 x i1> [[B2:%.*]], <8 x i1> [[B3:%.*]], <8 x i1> [[B4:%.*]], <8 x i1> [[B5:%.*]], <8 x i1> [[B6:%.*]], <8 x i1> [[B7:%.*]]) {
+; CHECK-NEXT:    [[XOR0:%.*]] = xor <8 x i1> [[B0]], [[A]]
+; CHECK-NEXT:    [[XOR1:%.*]] = xor <8 x i1> [[B1]], [[A]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor <8 x i1> [[B2]], [[A]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor <8 x i1> [[B3]], [[A]]
+; CHECK-NEXT:    [[XOR4:%.*]] = xor <8 x i1> [[B4]], [[A]]
+; CHECK-NEXT:    [[XOR5:%.*]] = xor <8 x i1> [[B5]], [[A]]
+; CHECK-NEXT:    [[XOR6:%.*]] = xor <8 x i1> [[B6]], [[A]]
+; CHECK-NEXT:    [[XOR7:%.*]] = xor <8 x i1> [[B7]], [[A]]
+; CHECK-NEXT:    [[AND3:%.*]] = and <8 x i1> [[XOR1]], [[XOR0]]
+; CHECK-NEXT:    [[AND2:%.*]] = and <8 x i1> [[AND3]], [[XOR2]]
+; CHECK-NEXT:    [[OR23:%.*]] = and <8 x i1> [[AND2]], [[XOR3]]
+; CHECK-NEXT:    [[AND1:%.*]] = and <8 x i1> [[OR23]], [[XOR4]]
+; CHECK-NEXT:    [[AND0:%.*]] = and <8 x i1> [[AND1]], [[XOR5]]
+; CHECK-NEXT:    [[OR01:%.*]] = and <8 x i1> [[AND0]], [[XOR6]]
+; CHECK-NEXT:    [[OR0123:%.*]] = and <8 x i1> [[OR01]], [[XOR7]]
+; CHECK-NEXT:    ret <8 x i1> [[OR0123]]
+;
+  %xor0 = xor <8 x i1> %b0, %a
+  %xor1 = xor <8 x i1> %b1, %a
+  %xor2 = xor <8 x i1> %b2, %a
+  %xor3 = xor <8 x i1> %b3, %a
+  %xor4 = xor <8 x i1> %b4, %a
+  %xor5 = xor <8 x i1> %b5, %a
+  %xor6 = xor <8 x i1> %b6, %a
+  %xor7 = xor <8 x i1> %b7, %a
+  %and0 = and <8 x i1> %xor0, %xor1
+  %and1 = and <8 x i1> %xor2, %xor3
+  %and2 = and <8 x i1> %xor4, %xor5
+  %and3 = and <8 x i1> %xor6, %xor7
+  %or01 = and <8 x i1> %and0, %and1
+  %or23 = and <8 x i1> %and2, %and3
+  %or0123 = and <8 x i1> %or01, %or23
+  ret <8 x i1> %or0123
+}
+
+define <8 x i1> @vector5(<8 x i1> %a, <8 x i1> %b0, <8 x i1> %b1, <8 x i1> %b2, <8 x i1> %b3, <8 x i1> %b4, <8 x i1> %b5, <8 x i1> %b6, <8 x i1> %b7) {
+; CHECK-LABEL: define <8 x i1> @vector5(
+; CHECK-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B0:%.*]], <8 x i1> [[B1:%.*]], <8 x i1> [[B2:%.*]], <8 x i1> [[B3:%.*]], <8 x i1> [[B4:%.*]], <8 x i1> [[B5:%.*]], <8 x i1> [[B6:%.*]], <8 x i1> [[B7:%.*]]) {
+; CHECK-NEXT:    [[XOR0:%.*]] = xor <8 x i1> [[B0]], [[A]]
+; CHECK-NEXT:    [[XOR1:%.*]] = xor <8 x i1> [[B1]], [[A]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor <8 x i1> [[B2]], [[A]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor <8 x i1> [[B3]], [[A]]
+; CHECK-NEXT:    [[XOR4:%.*]] = xor <8 x i1> [[B4]], [[A]]
+; CHECK-NEXT:    [[XOR5:%.*]] = xor <8 x i1> [[B5]], [[A]]
+; CHECK-NEXT:    [[XOR6:%.*]] = xor <8 x i1> [[B6]], [[A]]
+; CHECK-NEXT:    [[XOR7:%.*]] = xor <8 x i1> [[B7]], [[A]]
+; CHECK-NEXT:    [[OR3:%.*]] = or <8 x i1> [[B1]], [[B0]]
+; CHECK-NEXT:    [[OR2:%.*]] = or <8 x i1> [[OR3]], [[XOR0]]
+; CHECK-NEXT:    [[OR23:%.*]] = or <8 x i1> [[OR2]], [[B2]]
+; CHECK-NEXT:    [[OR1:%.*]] = or <8 x i1> [[OR23]], [[XOR1]]
+; CHECK-NEXT:    [[OR0:%.*]] = or <8 x i1> [[OR1]], [[B3]]
+; CHECK-NEXT:    [[OR01:%.*]] = or <8 x i1> [[OR0]], [[XOR2]]
+; CHECK-NEXT:    [[OR0123:%.*]] = or <8 x i1> [[OR01]], [[B4]]
+; CHECK-NEXT:    [[OR7:%.*]] = or <8 x i1> [[OR0123]], [[XOR3]]
+; CHECK-NEXT:    [[OR6:%.*]] = or <8 x i1> [[OR7]], [[B5]]
+; CHECK-NEXT:    [[OR67:%.*]] = or <8 x i1> [[OR6]], [[XOR4]]
+; CHECK-NEXT:    [[OR5:%.*]] = or <8 x i1> [[OR67]], [[B6]]
+; CHECK-NEXT:    [[OR4:%.*]] = or <8 x i1> [[OR5]], [[XOR5]]
+; CHECK-NEXT:    [[OR45:%.*]] = or <8 x i1> [[OR4]], [[B7]]
+; CHECK-NEXT:    [[OR4567:%.*]] = or <8 x i1> [[OR45]], [[XOR6]]
+; CHECK-NEXT:    [[OR01234567:%.*]] = or <8 x i1> [[OR4567]], [[XOR7]]
+; CHECK-NEXT:    ret <8 x i1> [[OR01234567]]
+;
+  %xor0 = xor <8 x i1> %b0, %a
+  %xor1 = xor <8 x i1> %b1, %a
+  %xor2 = xor <8 x i1> %b2, %a
+  %xor3 = xor <8 x i1> %b3, %a
+  %xor4 = xor <8 x i1> %b4, %a
+  %xor5 = xor <8 x i1> %b5, %a
+  %xor6 = xor <8 x i1> %b6, %a
+  %xor7 = xor <8 x i1> %b7, %a
+  %or0 = or <8 x i1> %xor0, %xor1
+  %or1 = or <8 x i1> %xor2, %xor3
+  %or2 = or <8 x i1> %xor4, %xor5
+  %or3 = or <8 x i1> %xor6, %xor7
+  %or4 = or <8 x i1> %b0, %b1
+  %or5 = or <8 x i1> %b2, %b3
+  %or6 = or <8 x i1> %b4, %b5
+  %or7 = or <8 x i1> %b6, %b7
+  %or01 = or <8 x i1> %or0, %or1
+  %or23 = or <8 x i1> %or2, %or3
+  %or45 = or <8 x i1> %or4, %or5
+  %or67 = or <8 x i1> %or6, %or7
+  %or0123 = or <8 x i1> %or01, %or23
+  %or4567 = or <8 x i1> %or45, %or67
+  %or01234567 = or <8 x i1> %or0123, %or4567
+  ret <8 x i1> %or01234567
+}
-- 
GitLab


From 236b3e1aad45e2bab8ede0da6397b7b01f9cc9d8 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Mon, 6 May 2024 19:55:55 -0700
Subject: [PATCH 0006/1206] [clang-format] Handle Java switch expressions
 (#91112)

Also adds AllowShortCaseExpressionOnASingleLine option and
AlignCaseArrows suboption of AlignConsecutiveShortCaseStatements.

Fixes #55903.
---
 clang/docs/ClangFormatStyleOptions.rst        |  36 +++-
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/Format/Format.h           |  36 +++-
 clang/lib/Format/Format.cpp                   |   4 +
 clang/lib/Format/FormatToken.h                |   3 +
 clang/lib/Format/TokenAnnotator.cpp           |   2 +
 clang/lib/Format/UnwrappedLineFormatter.cpp   |   6 +
 clang/lib/Format/UnwrappedLineParser.cpp      |  46 ++++-
 clang/lib/Format/UnwrappedLineParser.h        |   2 +-
 clang/lib/Format/WhitespaceManager.cpp        |  22 ++-
 clang/lib/Format/WhitespaceManager.h          |   2 +-
 clang/unittests/Format/ConfigParseTest.cpp    |   2 +
 clang/unittests/Format/FormatTestJava.cpp     | 171 ++++++++++++++++++
 clang/unittests/Format/TokenAnnotatorTest.cpp |  18 ++
 14 files changed, 332 insertions(+), 21 deletions(-)

diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index ce9035a2770e..6d092219877f 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -861,7 +861,8 @@ the configuration (without a prefix: ``Auto``).
 
 **AlignConsecutiveShortCaseStatements** (``ShortCaseStatementsAlignmentStyle``) :versionbadge:`clang-format 17` :ref:`¶ <AlignConsecutiveShortCaseStatements>`
   Style of aligning consecutive short case labels.
-  Only applies if ``AllowShortCaseLabelsOnASingleLine`` is ``true``.
+  Only applies if ``AllowShortCaseExpressionOnASingleLine`` or
+  ``AllowShortCaseLabelsOnASingleLine`` is ``true``.
 
 
   .. code-block:: yaml
@@ -935,6 +936,24 @@ the configuration (without a prefix: ``Auto``).
       default: return "";
       }
 
+  * ``bool AlignCaseArrows`` Whether to align the case arrows when aligning short case expressions.
+
+    .. code-block:: java
+
+      true:
+      i = switch (day) {
+        case THURSDAY, SATURDAY -> 8;
+        case WEDNESDAY          -> 9;
+        default                 -> 0;
+      };
+
+      false:
+      i = switch (day) {
+        case THURSDAY, SATURDAY -> 8;
+        case WEDNESDAY ->          9;
+        default ->                 0;
+      };
+
   * ``bool AlignCaseColons`` Whether aligned case labels are aligned on the colon, or on the tokens
     after the colon.
 
@@ -1692,6 +1711,21 @@ the configuration (without a prefix: ``Auto``).
 
 
 
+.. _AllowShortCaseExpressionOnASingleLine:
+
+**AllowShortCaseExpressionOnASingleLine** (``Boolean``) :versionbadge:`clang-format 19` :ref:`¶ <AllowShortCaseExpressionOnASingleLine>`
+  Whether to merge a short switch labeled rule into a single line.
+
+  .. code-block:: java
+
+    true:                               false:
+    switch (a) {           vs.          switch (a) {
+    case 1 -> 1;                        case 1 ->
+    default -> 0;                         1;
+    };                                  default ->
+                                          0;
+                                        };
+
 .. _AllowShortCaseLabelsOnASingleLine:
 
 **AllowShortCaseLabelsOnASingleLine** (``Boolean``) :versionbadge:`clang-format 3.6` :ref:`¶ <AllowShortCaseLabelsOnASingleLine>`
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index b146a9b56884..a85095e424b6 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -834,6 +834,9 @@ clang-format
   ``BreakTemplateDeclarations``.
 - ``AlwaysBreakAfterReturnType`` is deprecated and renamed to
   ``BreakAfterReturnType``.
+- Handles Java ``switch`` expressions.
+- Adds ``AllowShortCaseExpressionOnASingleLine`` option.
+- Adds ``AlignCaseArrows`` suboption to ``AlignConsecutiveShortCaseStatements``.
 
 libclang
 --------
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 8ebdc86b9832..74893f23210c 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -375,6 +375,23 @@ struct FormatStyle {
     ///   }
     /// \endcode
     bool AcrossComments;
+    /// Whether to align the case arrows when aligning short case expressions.
+    /// \code{.java}
+    ///   true:
+    ///   i = switch (day) {
+    ///     case THURSDAY, SATURDAY -> 8;
+    ///     case WEDNESDAY          -> 9;
+    ///     default                 -> 0;
+    ///   };
+    ///
+    ///   false:
+    ///   i = switch (day) {
+    ///     case THURSDAY, SATURDAY -> 8;
+    ///     case WEDNESDAY ->          9;
+    ///     default ->                 0;
+    ///   };
+    /// \endcode
+    bool AlignCaseArrows;
     /// Whether aligned case labels are aligned on the colon, or on the tokens
     /// after the colon.
     /// \code
@@ -396,12 +413,14 @@ struct FormatStyle {
     bool operator==(const ShortCaseStatementsAlignmentStyle &R) const {
       return Enabled == R.Enabled && AcrossEmptyLines == R.AcrossEmptyLines &&
              AcrossComments == R.AcrossComments &&
+             AlignCaseArrows == R.AlignCaseArrows &&
              AlignCaseColons == R.AlignCaseColons;
     }
   };
 
   /// Style of aligning consecutive short case labels.
-  /// Only applies if ``AllowShortCaseLabelsOnASingleLine`` is ``true``.
+  /// Only applies if ``AllowShortCaseExpressionOnASingleLine`` or
+  /// ``AllowShortCaseLabelsOnASingleLine`` is ``true``.
   ///
   /// \code{.yaml}
   ///   # Example of usage:
@@ -724,6 +743,19 @@ struct FormatStyle {
   /// \version 3.5
   ShortBlockStyle AllowShortBlocksOnASingleLine;
 
+  /// Whether to merge a short switch labeled rule into a single line.
+  /// \code{.java}
+  ///   true:                               false:
+  ///   switch (a) {           vs.          switch (a) {
+  ///   case 1 -> 1;                        case 1 ->
+  ///   default -> 0;                         1;
+  ///   };                                  default ->
+  ///                                         0;
+  ///                                       };
+  /// \endcode
+  /// \version 19
+  bool AllowShortCaseExpressionOnASingleLine;
+
   /// If ``true``, short case labels will be contracted to a single line.
   /// \code
   ///   true:                                   false:
@@ -4923,6 +4955,8 @@ struct FormatStyle {
            AllowBreakBeforeNoexceptSpecifier ==
                R.AllowBreakBeforeNoexceptSpecifier &&
            AllowShortBlocksOnASingleLine == R.AllowShortBlocksOnASingleLine &&
+           AllowShortCaseExpressionOnASingleLine ==
+               R.AllowShortCaseExpressionOnASingleLine &&
            AllowShortCaseLabelsOnASingleLine ==
                R.AllowShortCaseLabelsOnASingleLine &&
            AllowShortCompoundRequirementOnASingleLine ==
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index c8d8ec3afbd9..c4eac1c99a66 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -100,6 +100,7 @@ struct MappingTraits<FormatStyle::ShortCaseStatementsAlignmentStyle> {
     IO.mapOptional("Enabled", Value.Enabled);
     IO.mapOptional("AcrossEmptyLines", Value.AcrossEmptyLines);
     IO.mapOptional("AcrossComments", Value.AcrossComments);
+    IO.mapOptional("AlignCaseArrows", Value.AlignCaseArrows);
     IO.mapOptional("AlignCaseColons", Value.AlignCaseColons);
   }
 };
@@ -911,6 +912,8 @@ template <> struct MappingTraits<FormatStyle> {
                    Style.AllowBreakBeforeNoexceptSpecifier);
     IO.mapOptional("AllowShortBlocksOnASingleLine",
                    Style.AllowShortBlocksOnASingleLine);
+    IO.mapOptional("AllowShortCaseExpressionOnASingleLine",
+                   Style.AllowShortCaseExpressionOnASingleLine);
     IO.mapOptional("AllowShortCaseLabelsOnASingleLine",
                    Style.AllowShortCaseLabelsOnASingleLine);
     IO.mapOptional("AllowShortCompoundRequirementOnASingleLine",
@@ -1423,6 +1426,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.AllowAllParametersOfDeclarationOnNextLine = true;
   LLVMStyle.AllowBreakBeforeNoexceptSpecifier = FormatStyle::BBNSS_Never;
   LLVMStyle.AllowShortBlocksOnASingleLine = FormatStyle::SBS_Never;
+  LLVMStyle.AllowShortCaseExpressionOnASingleLine = true;
   LLVMStyle.AllowShortCaseLabelsOnASingleLine = false;
   LLVMStyle.AllowShortCompoundRequirementOnASingleLine = true;
   LLVMStyle.AllowShortEnumsOnASingleLine = true;
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 28b6488e54a4..95f16fde5005 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -38,6 +38,7 @@ namespace format {
   /* l_brace of a block that is not the body of a (e.g. loop) statement. */    \
   TYPE(BlockLBrace)                                                            \
   TYPE(BracedListLBrace)                                                       \
+  TYPE(CaseLabelArrow)                                                         \
   /* The colon at the end of a case label. */                                  \
   TYPE(CaseLabelColon)                                                         \
   TYPE(CastRParen)                                                             \
@@ -148,6 +149,8 @@ namespace format {
   TYPE(StructLBrace)                                                           \
   TYPE(StructRBrace)                                                           \
   TYPE(StructuredBindingLSquare)                                               \
+  TYPE(SwitchExpressionLabel)                                                  \
+  TYPE(SwitchExpressionLBrace)                                                 \
   TYPE(TableGenBangOperator)                                                   \
   TYPE(TableGenCondOperator)                                                   \
   TYPE(TableGenCondOperatorColon)                                              \
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index d366ae2080bc..e935d3e2709c 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -5051,6 +5051,8 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
       return true; // "x! as string", "x! in y"
     }
   } else if (Style.Language == FormatStyle::LK_Java) {
+    if (Left.is(TT_CaseLabelArrow) || Right.is(TT_CaseLabelArrow))
+      return true;
     if (Left.is(tok::r_square) && Right.is(tok::l_brace))
       return true;
     // spaces inside square brackets.
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index 4ae54e56331b..4d53361aaf33 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -515,6 +515,12 @@ private:
       }
     }
 
+    if (TheLine->First->is(TT_SwitchExpressionLabel)) {
+      return Style.AllowShortCaseExpressionOnASingleLine
+                 ? tryMergeShortCaseLabels(I, E, Limit)
+                 : 0;
+    }
+
     if (TheLine->Last->is(tok::l_brace)) {
       bool ShouldMerge = false;
       // Try to merge records.
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index f71661d837ec..71557b127fb7 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -430,9 +430,9 @@ bool UnwrappedLineParser::parseLevel(const FormatToken *OpeningBrace,
       unsigned StoredPosition = Tokens->getPosition();
       auto *Next = Tokens->getNextNonComment();
       FormatTok = Tokens->setPosition(StoredPosition);
-      if (Next->isNot(tok::colon)) {
-        // default not followed by ':' is not a case label; treat it like
-        // an identifier.
+      if (!Next->isOneOf(tok::colon, tok::arrow)) {
+        // default not followed by `:` or `->` is not a case label; treat it
+        // like an identifier.
         parseStructuralElement();
         break;
       }
@@ -451,6 +451,7 @@ bool UnwrappedLineParser::parseLevel(const FormatToken *OpeningBrace,
       }
       if (!SwitchLabelEncountered &&
           (Style.IndentCaseLabels ||
+           (OpeningBrace && OpeningBrace->is(TT_SwitchExpressionLBrace)) ||
            (Line->InPPDirective && Line->Level == 1))) {
         ++Line->Level;
       }
@@ -1519,9 +1520,9 @@ void UnwrappedLineParser::parseStructuralElement(
       // 'switch: string' field declaration.
       break;
     }
-    parseSwitch();
+    parseSwitch(/*IsExpr=*/false);
     return;
-  case tok::kw_default:
+  case tok::kw_default: {
     // In Verilog default along with other labels are handled in the next loop.
     if (Style.isVerilog())
       break;
@@ -1529,14 +1530,22 @@ void UnwrappedLineParser::parseStructuralElement(
       // 'default: string' field declaration.
       break;
     }
+    auto *Default = FormatTok;
     nextToken();
     if (FormatTok->is(tok::colon)) {
       FormatTok->setFinalizedType(TT_CaseLabelColon);
       parseLabel();
       return;
     }
+    if (FormatTok->is(tok::arrow)) {
+      FormatTok->setFinalizedType(TT_CaseLabelArrow);
+      Default->setFinalizedType(TT_SwitchExpressionLabel);
+      parseLabel();
+      return;
+    }
     // e.g. "default void f() {}" in a Java interface.
     break;
+  }
   case tok::kw_case:
     // Proto: there are no switch/case statements.
     if (Style.Language == FormatStyle::LK_Proto) {
@@ -2062,6 +2071,11 @@ void UnwrappedLineParser::parseStructuralElement(
     case tok::kw_new:
       parseNew();
       break;
+    case tok::kw_switch:
+      if (Style.Language == FormatStyle::LK_Java)
+        parseSwitch(/*IsExpr=*/true);
+      nextToken();
+      break;
     case tok::kw_case:
       // Proto: there are no switch/case statements.
       if (Style.Language == FormatStyle::LK_Proto) {
@@ -2589,6 +2603,9 @@ bool UnwrappedLineParser::parseParens(TokenType AmpAmpTokenType) {
       else
         nextToken();
       break;
+    case tok::kw_switch:
+      parseSwitch(/*IsExpr=*/true);
+      break;
     case tok::kw_requires: {
       auto RequiresToken = FormatTok;
       nextToken();
@@ -3246,6 +3263,7 @@ void UnwrappedLineParser::parseLabel(bool LeftAlignLabel) {
 
 void UnwrappedLineParser::parseCaseLabel() {
   assert(FormatTok->is(tok::kw_case) && "'case' expected");
+  auto *Case = FormatTok;
 
   // FIXME: fix handling of complex expressions here.
   do {
@@ -3254,11 +3272,16 @@ void UnwrappedLineParser::parseCaseLabel() {
       FormatTok->setFinalizedType(TT_CaseLabelColon);
       break;
     }
+    if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::arrow)) {
+      FormatTok->setFinalizedType(TT_CaseLabelArrow);
+      Case->setFinalizedType(TT_SwitchExpressionLabel);
+      break;
+    }
   } while (!eof());
   parseLabel();
 }
 
-void UnwrappedLineParser::parseSwitch() {
+void UnwrappedLineParser::parseSwitch(bool IsExpr) {
   assert(FormatTok->is(tok::kw_switch) && "'switch' expected");
   nextToken();
   if (FormatTok->is(tok::l_paren))
@@ -3268,10 +3291,15 @@ void UnwrappedLineParser::parseSwitch() {
 
   if (FormatTok->is(tok::l_brace)) {
     CompoundStatementIndenter Indenter(this, Style, Line->Level);
-    FormatTok->setFinalizedType(TT_ControlStatementLBrace);
-    parseBlock();
+    FormatTok->setFinalizedType(IsExpr ? TT_SwitchExpressionLBrace
+                                       : TT_ControlStatementLBrace);
+    if (IsExpr)
+      parseChildBlock();
+    else
+      parseBlock();
     setPreviousRBraceType(TT_ControlStatementRBrace);
-    addUnwrappedLine();
+    if (!IsExpr)
+      addUnwrappedLine();
   } else {
     addUnwrappedLine();
     ++Line->Level;
diff --git a/clang/lib/Format/UnwrappedLineParser.h b/clang/lib/Format/UnwrappedLineParser.h
index e2cf28c0c065..2a0fe19d0957 100644
--- a/clang/lib/Format/UnwrappedLineParser.h
+++ b/clang/lib/Format/UnwrappedLineParser.h
@@ -157,7 +157,7 @@ private:
   void parseDoWhile();
   void parseLabel(bool LeftAlignLabel = false);
   void parseCaseLabel();
-  void parseSwitch();
+  void parseSwitch(bool IsExpr);
   void parseNamespace();
   bool parseModuleImport();
   void parseNew();
diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp
index 44fd807ec27e..ed06d6098a9f 100644
--- a/clang/lib/Format/WhitespaceManager.cpp
+++ b/clang/lib/Format/WhitespaceManager.cpp
@@ -107,7 +107,8 @@ const tooling::Replacements &WhitespaceManager::generateReplacements() {
   llvm::sort(Changes, Change::IsBeforeInFile(SourceMgr));
   calculateLineBreakInformation();
   alignConsecutiveMacros();
-  alignConsecutiveShortCaseStatements();
+  alignConsecutiveShortCaseStatements(/*IsExpr=*/true);
+  alignConsecutiveShortCaseStatements(/*IsExpr=*/false);
   alignConsecutiveDeclarations();
   alignConsecutiveBitFields();
   alignConsecutiveAssignments();
@@ -878,22 +879,27 @@ void WhitespaceManager::alignConsecutiveColons(
       Changes, /*StartAt=*/0, AlignStyle);
 }
 
-void WhitespaceManager::alignConsecutiveShortCaseStatements() {
+void WhitespaceManager::alignConsecutiveShortCaseStatements(bool IsExpr) {
   if (!Style.AlignConsecutiveShortCaseStatements.Enabled ||
-      !Style.AllowShortCaseLabelsOnASingleLine) {
+      !(IsExpr ? Style.AllowShortCaseExpressionOnASingleLine
+               : Style.AllowShortCaseLabelsOnASingleLine)) {
     return;
   }
 
+  const auto Type = IsExpr ? TT_CaseLabelArrow : TT_CaseLabelColon;
+  const auto &Option = Style.AlignConsecutiveShortCaseStatements;
+  const bool AlignArrowOrColon =
+      IsExpr ? Option.AlignCaseArrows : Option.AlignCaseColons;
+
   auto Matches = [&](const Change &C) {
-    if (Style.AlignConsecutiveShortCaseStatements.AlignCaseColons)
-      return C.Tok->is(TT_CaseLabelColon);
+    if (AlignArrowOrColon)
+      return C.Tok->is(Type);
 
     // Ignore 'IsInsideToken' to allow matching trailing comments which
     // need to be reflowed as that causes the token to appear in two
     // different changes, which will cause incorrect alignment as we'll
     // reflow early due to detecting multiple aligning tokens per line.
-    return !C.IsInsideToken && C.Tok->Previous &&
-           C.Tok->Previous->is(TT_CaseLabelColon);
+    return !C.IsInsideToken && C.Tok->Previous && C.Tok->Previous->is(Type);
   };
 
   unsigned MinColumn = 0;
@@ -944,7 +950,7 @@ void WhitespaceManager::alignConsecutiveShortCaseStatements() {
     if (Changes[I].Tok->isNot(tok::comment))
       LineIsComment = false;
 
-    if (Changes[I].Tok->is(TT_CaseLabelColon)) {
+    if (Changes[I].Tok->is(Type)) {
       LineIsEmptyCase =
           !Changes[I].Tok->Next || Changes[I].Tok->Next->isTrailingComment();
 
diff --git a/clang/lib/Format/WhitespaceManager.h b/clang/lib/Format/WhitespaceManager.h
index 98cf4a260cc4..7b91d8bf4db7 100644
--- a/clang/lib/Format/WhitespaceManager.h
+++ b/clang/lib/Format/WhitespaceManager.h
@@ -233,7 +233,7 @@ private:
   void alignChainedConditionals();
 
   /// Align consecutive short case statements over all \c Changes.
-  void alignConsecutiveShortCaseStatements();
+  void alignConsecutiveShortCaseStatements(bool IsExpr);
 
   /// Align consecutive TableGen DAGArg colon over all \c Changes.
   void alignConsecutiveTableGenBreakingDAGArgColons();
diff --git a/clang/unittests/Format/ConfigParseTest.cpp b/clang/unittests/Format/ConfigParseTest.cpp
index 8c74ed2d119a..82e72f08ffb5 100644
--- a/clang/unittests/Format/ConfigParseTest.cpp
+++ b/clang/unittests/Format/ConfigParseTest.cpp
@@ -153,6 +153,7 @@ TEST(ConfigParseTest, ParsesConfigurationBools) {
   Style.Language = FormatStyle::LK_Cpp;
   CHECK_PARSE_BOOL(AllowAllArgumentsOnNextLine);
   CHECK_PARSE_BOOL(AllowAllParametersOfDeclarationOnNextLine);
+  CHECK_PARSE_BOOL(AllowShortCaseExpressionOnASingleLine);
   CHECK_PARSE_BOOL(AllowShortCaseLabelsOnASingleLine);
   CHECK_PARSE_BOOL(AllowShortCompoundRequirementOnASingleLine);
   CHECK_PARSE_BOOL(AllowShortEnumsOnASingleLine);
@@ -205,6 +206,7 @@ TEST(ConfigParseTest, ParsesConfigurationBools) {
   CHECK_PARSE_NESTED_BOOL(AlignConsecutiveShortCaseStatements,
                           AcrossEmptyLines);
   CHECK_PARSE_NESTED_BOOL(AlignConsecutiveShortCaseStatements, AcrossComments);
+  CHECK_PARSE_NESTED_BOOL(AlignConsecutiveShortCaseStatements, AlignCaseArrows);
   CHECK_PARSE_NESTED_BOOL(AlignConsecutiveShortCaseStatements, AlignCaseColons);
   CHECK_PARSE_NESTED_BOOL(BraceWrapping, AfterCaseLabel);
   CHECK_PARSE_NESTED_BOOL(BraceWrapping, AfterClass);
diff --git a/clang/unittests/Format/FormatTestJava.cpp b/clang/unittests/Format/FormatTestJava.cpp
index 6da5f4fa2543..33998bc7ff85 100644
--- a/clang/unittests/Format/FormatTestJava.cpp
+++ b/clang/unittests/Format/FormatTestJava.cpp
@@ -618,6 +618,177 @@ TEST_F(FormatTestJava, ConfigurableSpacesInSquareBrackets) {
   verifyFormat("types[ i ] = arguments[ i ].getClass();", Spaces);
 }
 
+TEST_F(FormatTestJava, SwitchExpression) {
+  auto Style = getLLVMStyle(FormatStyle::LK_Java);
+  EXPECT_TRUE(Style.AllowShortCaseExpressionOnASingleLine);
+
+  verifyFormat("foo(switch (day) {\n"
+               "  case THURSDAY, SATURDAY -> 8;\n"
+               "  case WEDNESDAY -> 9;\n"
+               "  default -> 1;\n"
+               "});",
+               Style);
+
+  constexpr StringRef Code1{"i = switch (day) {\n"
+                            "  case THURSDAY, SATURDAY -> 8;\n"
+                            "  case WEDNESDAY -> 9;\n"
+                            "  default -> 0;\n"
+                            "};"};
+  verifyFormat(Code1, Style);
+
+  Style.IndentCaseLabels = true;
+  verifyFormat(Code1, Style);
+
+  constexpr StringRef Code2{"i = switch (day) {\n"
+                            "  case THURSDAY, SATURDAY -> {\n"
+                            "    foo();\n"
+                            "    yield 8;\n"
+                            "  }\n"
+                            "  case WEDNESDAY -> {\n"
+                            "    bar();\n"
+                            "    yield 9;\n"
+                            "  }\n"
+                            "  default -> {\n"
+                            "    yield 0;\n"
+                            "  }\n"
+                            "};"};
+  verifyFormat(Code2, Style);
+
+  Style.IndentCaseLabels = false;
+  verifyFormat(Code2, Style);
+
+  constexpr StringRef Code3{"switch (day) {\n"
+                            "case THURSDAY, SATURDAY -> i = 8;\n"
+                            "case WEDNESDAY -> i = 9;\n"
+                            "default -> i = 0;\n"
+                            "};"};
+  verifyFormat(Code3, Style);
+
+  Style.IndentCaseLabels = true;
+  verifyFormat("switch (day) {\n"
+               "  case THURSDAY, SATURDAY -> i = 8;\n"
+               "  case WEDNESDAY -> i = 9;\n"
+               "  default -> i = 0;\n"
+               "};",
+               Code3, Style);
+}
+
+TEST_F(FormatTestJava, ShortCaseExpression) {
+  auto Style = getLLVMStyle(FormatStyle::LK_Java);
+
+  verifyFormat("i = switch (a) {\n"
+               "  case 1 -> 1;\n"
+               "  case 2 -> // comment\n"
+               "    2;\n"
+               "  case 3 ->\n"
+               "    // comment\n"
+               "    3;\n"
+               "  case 4 -> 4; // comment\n"
+               "  default -> 0;\n"
+               "};",
+               Style);
+
+  verifyNoChange("i = switch (a) {\n"
+                 "  case 1 -> 1;\n"
+                 "  // comment\n"
+                 "  case 2 -> 2;\n"
+                 "  // comment 1\n"
+                 "  // comment 2\n"
+                 "  case 3 -> 3; /* comment */\n"
+                 "  case 4 -> /* comment */ 4;\n"
+                 "  case 5 -> x + /* comment */ 1;\n"
+                 "  default ->\n"
+                 "    0; // comment line 1\n"
+                 "       // comment line 2\n"
+                 "};",
+                 Style);
+
+  Style.ColumnLimit = 18;
+  verifyFormat("i = switch (a) {\n"
+               "  case Monday ->\n"
+               "    1;\n"
+               "  default -> 9999;\n"
+               "};",
+               Style);
+
+  Style.ColumnLimit = 80;
+  Style.AllowShortCaseExpressionOnASingleLine = false;
+  Style.IndentCaseLabels = true;
+  verifyFormat("i = switch (n) {\n"
+               "  default /*comments*/ ->\n"
+               "    1;\n"
+               "  case 0 ->\n"
+               "    0;\n"
+               "};",
+               Style);
+
+  Style.AllowShortCaseExpressionOnASingleLine = true;
+  Style.BreakBeforeBraces = FormatStyle::BS_Custom;
+  Style.BraceWrapping.AfterCaseLabel = true;
+  Style.BraceWrapping.AfterControlStatement = FormatStyle::BWACS_Always;
+  verifyFormat("i = switch (n)\n"
+               "{\n"
+               "  case 0 ->\n"
+               "  {\n"
+               "    yield 0;\n"
+               "  }\n"
+               "  default ->\n"
+               "  {\n"
+               "    yield 1;\n"
+               "  }\n"
+               "};",
+               Style);
+}
+
+TEST_F(FormatTestJava, AlignCaseArrows) {
+  auto Style = getLLVMStyle(FormatStyle::LK_Java);
+  Style.AlignConsecutiveShortCaseStatements.Enabled = true;
+
+  verifyFormat("foo(switch (day) {\n"
+               "  case THURSDAY, SATURDAY -> 8;\n"
+               "  case WEDNESDAY ->          9;\n"
+               "  default ->                 1;\n"
+               "});",
+               Style);
+
+  verifyFormat("i = switch (day) {\n"
+               "  case THURSDAY, SATURDAY -> 8;\n"
+               "  case WEDNESDAY ->          9;\n"
+               "  default ->                 0;\n"
+               "};",
+               Style);
+
+  verifyFormat("switch (day) {\n"
+               "case THURSDAY, SATURDAY -> i = 8;\n"
+               "case WEDNESDAY ->          i = 9;\n"
+               "default ->                 i = 0;\n"
+               "};",
+               Style);
+
+  Style.AlignConsecutiveShortCaseStatements.AlignCaseArrows = true;
+
+  verifyFormat("foo(switch (day) {\n"
+               "  case THURSDAY, SATURDAY -> 8;\n"
+               "  case WEDNESDAY          -> 9;\n"
+               "  default                 -> 1;\n"
+               "});",
+               Style);
+
+  verifyFormat("i = switch (day) {\n"
+               "  case THURSDAY, SATURDAY -> 8;\n"
+               "  case WEDNESDAY          -> 9;\n"
+               "  default                 -> 0;\n"
+               "};",
+               Style);
+
+  verifyFormat("switch (day) {\n"
+               "case THURSDAY, SATURDAY -> i = 8;\n"
+               "case WEDNESDAY          -> i = 9;\n"
+               "default                 -> i = 0;\n"
+               "};",
+               Style);
+}
+
 } // namespace
 } // namespace test
 } // namespace format
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index b424424b8577..51b475d37977 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -2981,6 +2981,24 @@ TEST_F(TokenAnnotatorTest, BlockLBrace) {
   EXPECT_BRACE_KIND(Tokens[5], BK_Block);
 }
 
+TEST_F(TokenAnnotatorTest, SwitchExpression) {
+  auto Style = getLLVMStyle(FormatStyle::LK_Java);
+  auto Tokens = annotate("i = switch (day) {\n"
+                         "  case THURSDAY, SATURDAY -> 8;\n"
+                         "  case WEDNESDAY -> 9;\n"
+                         "  default -> 1;\n"
+                         "};",
+                         Style);
+  ASSERT_EQ(Tokens.size(), 26u) << Tokens;
+  EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_SwitchExpressionLBrace);
+  EXPECT_TOKEN(Tokens[7], tok::kw_case, TT_SwitchExpressionLabel);
+  EXPECT_TOKEN(Tokens[11], tok::arrow, TT_CaseLabelArrow);
+  EXPECT_TOKEN(Tokens[14], tok::kw_case, TT_SwitchExpressionLabel);
+  EXPECT_TOKEN(Tokens[16], tok::arrow, TT_CaseLabelArrow);
+  EXPECT_TOKEN(Tokens[19], tok::kw_default, TT_SwitchExpressionLabel);
+  EXPECT_TOKEN(Tokens[20], tok::arrow, TT_CaseLabelArrow);
+}
+
 } // namespace
 } // namespace format
 } // namespace clang
-- 
GitLab


From f9d76197ff0099502cf001abe3f5310c5bc4532d Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Tue, 7 May 2024 10:48:47 +0800
Subject: [PATCH 0007/1206] [ASTContext] Profile Dependently-sized array types
 that do not have a specified number of elements

Close https://github.com/llvm/llvm-project/issues/91105

The root reason for the issue is that we always generate the
dependently-sized array types which don't specify a number of elements.

The original comment says:

> We do no canonicalization here at all, which is okay
> because they can't be used in most locations.

But now we find the locations.
---
 clang/lib/AST/ASTContext.cpp    | 34 ++++++++++++------------
 clang/lib/AST/Type.cpp          |  3 ++-
 clang/test/Modules/pr91105.cppm | 47 +++++++++++++++++++++++++++++++++
 3 files changed, 66 insertions(+), 18 deletions(-)
 create mode 100644 clang/test/Modules/pr91105.cppm

diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 5f96e86f803a..91e7a5f67a93 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -3797,33 +3797,33 @@ QualType ASTContext::getDependentSizedArrayType(QualType elementType,
           numElements->isValueDependent()) &&
          "Size must be type- or value-dependent!");
 
+  SplitQualType canonElementType = getCanonicalType(elementType).split();
+
+  void *insertPos = nullptr;
+  llvm::FoldingSetNodeID ID;
+  DependentSizedArrayType::Profile(
+      ID, *this, numElements ? QualType(canonElementType.Ty, 0) : elementType,
+      ASM, elementTypeQuals, numElements);
+
+  // Look for an existing type with these properties.
+  DependentSizedArrayType *canonTy =
+    DependentSizedArrayTypes.FindNodeOrInsertPos(ID, insertPos);
+
   // Dependently-sized array types that do not have a specified number
   // of elements will have their sizes deduced from a dependent
-  // initializer.  We do no canonicalization here at all, which is okay
-  // because they can't be used in most locations.
+  // initializer.
   if (!numElements) {
+    if (canonTy)
+      return QualType(canonTy, 0);
+
     auto *newType = new (*this, alignof(DependentSizedArrayType))
         DependentSizedArrayType(elementType, QualType(), numElements, ASM,
                                 elementTypeQuals, brackets);
+    DependentSizedArrayTypes.InsertNode(newType, insertPos);
     Types.push_back(newType);
     return QualType(newType, 0);
   }
 
-  // Otherwise, we actually build a new type every time, but we
-  // also build a canonical type.
-
-  SplitQualType canonElementType = getCanonicalType(elementType).split();
-
-  void *insertPos = nullptr;
-  llvm::FoldingSetNodeID ID;
-  DependentSizedArrayType::Profile(ID, *this,
-                                   QualType(canonElementType.Ty, 0),
-                                   ASM, elementTypeQuals, numElements);
-
-  // Look for an existing type with these properties.
-  DependentSizedArrayType *canonTy =
-    DependentSizedArrayTypes.FindNodeOrInsertPos(ID, insertPos);
-
   // If we don't have one, build one.
   if (!canonTy) {
     canonTy = new (*this, alignof(DependentSizedArrayType))
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 2385c5e02cb2..e31741cd4424 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -256,7 +256,8 @@ void DependentSizedArrayType::Profile(llvm::FoldingSetNodeID &ID,
   ID.AddPointer(ET.getAsOpaquePtr());
   ID.AddInteger(llvm::to_underlying(SizeMod));
   ID.AddInteger(TypeQuals);
-  E->Profile(ID, Context, true);
+  if (E)
+    E->Profile(ID, Context, true);
 }
 
 DependentVectorType::DependentVectorType(QualType ElementType,
diff --git a/clang/test/Modules/pr91105.cppm b/clang/test/Modules/pr91105.cppm
new file mode 100644
index 000000000000..0873962c3773
--- /dev/null
+++ b/clang/test/Modules/pr91105.cppm
@@ -0,0 +1,47 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/bar.cppm -emit-module-interface -o %t/bar.pcm
+// RUN: %clang_cc1 -std=c++20 %t/foo.cc -fmodule-file=bar=%t/bar.pcm -fsyntax-only -verify
+//
+// RUN: %clang_cc1 -std=c++20 -fskip-odr-check-in-gmf %t/bar.cppm -emit-module-interface \
+// RUN:     -o %t/bar.pcm
+// RUN: %clang_cc1 -std=c++20 -fskip-odr-check-in-gmf %t/foo.cc \
+// RUN:     -fmodule-file=bar=%t/bar.pcm -fsyntax-only -verify
+//
+// RUN: %clang_cc1 -std=c++20 %t/bar.cppm -emit-reduced-module-interface -o %t/bar.pcm
+// RUN: %clang_cc1 -std=c++20 %t/foo.cc -fmodule-file=bar=%t/bar.pcm -fsyntax-only -verify
+//
+// RUN: %clang_cc1 -std=c++20 -fskip-odr-check-in-gmf %t/bar.cppm -emit-reduced-module-interface \
+// RUN:     -o %t/bar.pcm
+// RUN: %clang_cc1 -std=c++20 -fskip-odr-check-in-gmf %t/foo.cc \
+// RUN:     -fmodule-file=bar=%t/bar.pcm -fsyntax-only -verify
+
+//--- h.hpp
+#pragma once
+
+struct T {
+    constexpr T(const char *) {}
+};
+template <char... c>
+struct t {
+    inline constexpr operator T() const { return {s}; }
+
+private:
+    inline static constexpr char s[]{c..., '\0'};
+};
+
+//--- bar.cppm
+module;
+#include "h.hpp"
+export module bar;
+export inline constexpr auto k = t<'k'>{};
+
+//--- foo.cc
+// expected-no-diagnostics
+#include "h.hpp"
+import bar;
+void f() {
+  T x = k;
+}
-- 
GitLab


From 4cce9fbb4e086170f69bfc8766f9613673b441c9 Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma@quicinc.com>
Date: Mon, 6 May 2024 20:04:55 -0700
Subject: [PATCH 0008/1206] [Arm64EC] Fix compilation of arm_acle.h (#91281)

---
 clang/lib/Headers/arm_acle.h         | 2 +-
 clang/test/Headers/arm-acle-header.c | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h
index 6e557eda1ddd..5785954c9171 100644
--- a/clang/lib/Headers/arm_acle.h
+++ b/clang/lib/Headers/arm_acle.h
@@ -109,7 +109,7 @@ __swp(uint32_t __x, volatile uint32_t *__p) {
 #endif
 
 /* 7.7 NOP */
-#if !defined(_MSC_VER) || !defined(__aarch64__)
+#if !defined(_MSC_VER) || (!defined(__aarch64__) && !defined(__arm64ec__))
 static __inline__ void __attribute__((__always_inline__, __nodebug__)) __nop(void) {
   __builtin_arm_nop();
 }
diff --git a/clang/test/Headers/arm-acle-header.c b/clang/test/Headers/arm-acle-header.c
index f04c7e1f0f35..fea8472183c8 100644
--- a/clang/test/Headers/arm-acle-header.c
+++ b/clang/test/Headers/arm-acle-header.c
@@ -7,6 +7,7 @@
 // RUN: %clang_cc1 -x c++ -triple thumbv7-windows -target-cpu cortex-a15 -fsyntax-only -ffreestanding -fms-extensions -fms-compatibility -fms-compatibility-version=19.11 %s
 // RUN: %clang_cc1 -x c++ -triple aarch64-windows -target-cpu cortex-a53 -fsyntax-only -ffreestanding -fms-extensions -fms-compatibility -fms-compatibility-version=19.11 %s
 // RUN: %clang_cc1 -x c++ -triple arm64-apple-ios -target-cpu apple-a7 -fsyntax-only -ffreestanding -fms-extensions %s
+// RUN: %clang_cc1 -x c++ -triple arm64ec-windows -target-cpu cortex-a53 -fsyntax-only -ffreestanding -fms-extensions -fms-compatibility -fms-compatibility-version=19.11 %s
 // expected-no-diagnostics
 
 #include <arm_acle.h>
-- 
GitLab


From dfa7ff97b24dc5a3dd714b45af288812c13d0110 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Tue, 7 May 2024 11:34:42 +0800
Subject: [PATCH 0009/1206] [C++20] [Modules] [Reduced BMI] Combine the
 signature of used modules into the current module

Following of https://github.com/llvm/llvm-project/pull/86912. After
https://github.com/llvm/llvm-project/pull/86912, with reduced BMI, the
BMI can keep unchange if the dependent modules only changes the
implementation (without introduing new decls). However, this is not
strictly correct.

For example:

```
// a.cppm
export module a;
export inline int a() { ... }

// b.cppm
export module b;
import a;
export inline int b() { return a(); }
```

Since both `a()` and `b()` are inline, we need to make sure the BMI of
`b.pcm` will change after the implementation of `a()` changes.

We can't get that naturally since we won't record the body of `a()`
during the writing process. We can't reuse ODRHash here since ODRHash
won't calculate the called function recursively. So ODRHash will be
problematic if `a()` calls other inline functions.

Probably we can solve this by a new hash mechanism. But the safety and
efficiency may a problem too. Here we just combine the hash value of the
used modules conservatively.
---
 clang/include/clang/Serialization/ASTWriter.h |  7 ++
 clang/lib/Serialization/ASTWriter.cpp         | 31 +++++-
 .../Modules/function-transitive-change.cppm   | 94 +++++++++++++++++++
 .../no-transitive-source-location-change.cppm | 24 -----
 4 files changed, 131 insertions(+), 25 deletions(-)
 create mode 100644 clang/test/Modules/function-transitive-change.cppm

diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index 6847c1db39c8..482e9dd168cc 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -357,6 +357,13 @@ private:
   /// contexts.
   llvm::DenseMap<const Decl *, unsigned> AnonymousDeclarationNumbers;
 
+  /// The external top level module during the writing process. Used to
+  /// generate signature for the module file being written.
+  ///
+  /// Only meaningful for standard C++ named modules. See the comments in
+  /// createSignatureForNamedModule() for details.
+  llvm::DenseSet<Module *> TouchedTopLevelModules;
+
   /// An update to a Decl.
   class DeclUpdate {
     /// A DeclUpdateKind.
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 8a0116fa8932..42da50abdc68 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -1200,6 +1200,31 @@ ASTFileSignature ASTWriter::createSignatureForNamedModule() const {
   for (auto [ExportImported, _] : WritingModule->Exports)
     Hasher.update(ExportImported->Signature);
 
+  // We combine all the used modules to make sure the signature is precise.
+  // Consider the case like:
+  //
+  // // a.cppm
+  // export module a;
+  // export inline int a() { ... }
+  //
+  // // b.cppm
+  // export module b;
+  // import a;
+  // export inline int b() { return a(); }
+  //
+  // Since both `a()` and `b()` are inline, we need to make sure the BMI of
+  // `b.pcm` will change after the implementation of `a()` changes. We can't
+  // get that naturally since we won't record the body of `a()` during the
+  // writing process. We can't reuse ODRHash here since ODRHash won't calculate
+  // the called function recursively. So ODRHash will be problematic if `a()`
+  // calls other inline functions.
+  //
+  // Probably we can solve this by a new hash mechanism. But the safety and
+  // efficiency may a problem too. Here we just combine the hash value of the
+  // used modules conservatively.
+  for (Module *M : TouchedTopLevelModules)
+    Hasher.update(M->Signature);
+
   return ASTFileSignature::create(Hasher.result());
 }
 
@@ -6112,8 +6137,12 @@ LocalDeclID ASTWriter::GetDeclRef(const Decl *D) {
 
   // If D comes from an AST file, its declaration ID is already known and
   // fixed.
-  if (D->isFromASTFile())
+  if (D->isFromASTFile()) {
+    if (isWritingStdCXXNamedModules() && D->getOwningModule())
+      TouchedTopLevelModules.insert(D->getOwningModule()->getTopLevelModule());
+
     return LocalDeclID(D->getGlobalID());
+  }
 
   assert(!(reinterpret_cast<uintptr_t>(D) & 0x01) && "Invalid decl pointer");
   LocalDeclID &ID = DeclIDs[D];
diff --git a/clang/test/Modules/function-transitive-change.cppm b/clang/test/Modules/function-transitive-change.cppm
new file mode 100644
index 000000000000..cfce669e3a7b
--- /dev/null
+++ b/clang/test/Modules/function-transitive-change.cppm
@@ -0,0 +1,94 @@
+// Test that, in C++20 modules reduced BMI, the implementation detail changes
+// in non-inline function may not propagate while the inline function changes
+// can get propagate.
+//
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: cd %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/a.v1.cppm -emit-reduced-module-interface -o %t/a.v1.pcm
+//
+// The BMI of A should differ since the different implementation.
+// RUN: not diff %t/a.pcm %t/a.v1.pcm &> /dev/null
+//
+// The BMI of B should change since the dependent inline function changes
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-reduced-module-interface -fmodule-file=a=%t/a.pcm \
+// RUN:     -o %t/b.pcm
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-reduced-module-interface -fmodule-file=a=%t/a.v1.pcm \
+// RUN:     -o %t/b.v1.pcm
+// RUN: not diff %t/b.v1.pcm %t/b.pcm  &> /dev/null
+//
+// Test the case with unused partitions.
+// RUN: %clang_cc1 -std=c++20 %t/M-A.cppm -emit-reduced-module-interface -o %t/M-A.pcm
+// RUN: %clang_cc1 -std=c++20 %t/M-B.cppm -emit-reduced-module-interface -o %t/M-B.pcm
+// RUN: %clang_cc1 -std=c++20 %t/M.cppm -emit-reduced-module-interface -o %t/M.pcm \
+// RUN:     -fmodule-file=M:partA=%t/M-A.pcm \
+// RUN:     -fmodule-file=M:partB=%t/M-B.pcm
+// RUN: %clang_cc1 -std=c++20 %t/N.cppm -emit-reduced-module-interface -o %t/N.pcm \
+// RUN:     -fmodule-file=M:partA=%t/M-A.pcm \
+// RUN:     -fmodule-file=M:partB=%t/M-B.pcm \
+// RUN:     -fmodule-file=M=%t/M.pcm
+//
+// Now we change `M-A.cppm` to `M-A.v1.cppm`.
+// RUN: %clang_cc1 -std=c++20 %t/M-A.v1.cppm -emit-reduced-module-interface -o %t/M-A.v1.pcm
+// RUN: %clang_cc1 -std=c++20 %t/M.cppm -emit-reduced-module-interface -o %t/M.v1.pcm \
+// RUN:     -fmodule-file=M:partA=%t/M-A.v1.pcm \
+// RUN:     -fmodule-file=M:partB=%t/M-B.pcm
+// RUN: %clang_cc1 -std=c++20 %t/N.cppm -emit-reduced-module-interface -o %t/N.v1.pcm \
+// RUN:     -fmodule-file=M:partA=%t/M-A.v1.pcm \
+// RUN:     -fmodule-file=M:partB=%t/M-B.pcm \
+// RUN:     -fmodule-file=M=%t/M.v1.pcm
+//
+// The BMI of N can keep unchanged since the N didn't use the changed partition unit 'M:A'.
+// RUN: diff %t/N.v1.pcm %t/N.pcm  &> /dev/null
+
+//--- a.cppm
+export module a;
+export inline int a() {
+    return 48;
+}
+
+//--- a.v1.cppm
+export module a;
+export inline int a() {
+    return 50;
+}
+
+//--- b.cppm
+export module b;
+import a;
+export inline int b() {
+    return a();
+}
+
+//--- M-A.cppm
+export module M:partA;
+export inline int a() {
+    return 43;
+}
+
+//--- M-A.v1.cppm
+export module M:partA;
+export inline int a() {
+    return 50;
+}
+
+//--- M-B.cppm
+export module M:partB;
+export inline int b() {
+    return 44;
+}
+
+//--- M.cppm
+export module M;
+export import :partA;
+export import :partB;
+
+//--- N.cppm
+export module N;
+import M;
+
+export inline int n() {
+    return b();
+}
diff --git a/clang/test/Modules/no-transitive-source-location-change.cppm b/clang/test/Modules/no-transitive-source-location-change.cppm
index 303142a1af89..c9d156a74ce8 100644
--- a/clang/test/Modules/no-transitive-source-location-change.cppm
+++ b/clang/test/Modules/no-transitive-source-location-change.cppm
@@ -1,30 +1,6 @@
 // Testing that adding a new line in a module interface unit won't cause the BMI
 // of consuming module unit changes.
 //
-// RUN: rm -rf %t
-// RUN: split-file %s %t
-//
-// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-module-interface -o %t/A.pcm
-// RUN: %clang_cc1 -std=c++20 %t/A.v1.cppm -emit-module-interface -o %t/A.v1.pcm
-//
-// The BMI may not be the same since the source location differs.
-// RUN: not diff %t/A.pcm %t/A.v1.pcm &> /dev/null
-//
-// The BMI of B shouldn't change since all the locations remain the same.
-// RUN: %clang_cc1 -std=c++20 %t/B.cppm -emit-module-interface -fmodule-file=A=%t/A.pcm \
-// RUN:     -o %t/B.pcm
-// RUN: %clang_cc1 -std=c++20 %t/B.cppm -emit-module-interface -fmodule-file=A=%t/A.v1.pcm \
-// RUN:     -o %t/B.v1.pcm
-// RUN: diff %t/B.v1.pcm %t/B.pcm  &> /dev/null
-//
-// The BMI of C may change since the locations for instantiations changes.
-// RUN: %clang_cc1 -std=c++20 %t/C.cppm -emit-module-interface -fmodule-file=A=%t/A.pcm \
-// RUN:     -o %t/C.pcm
-// RUN: %clang_cc1 -std=c++20 %t/C.cppm -emit-module-interface -fmodule-file=A=%t/A.v1.pcm \
-// RUN:     -o %t/C.v1.pcm
-// RUN: not diff %t/C.v1.pcm %t/C.pcm  &> /dev/null
-//
-// Test again with reduced BMI.
 // RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm
 // RUN: %clang_cc1 -std=c++20 %t/A.v1.cppm -emit-reduced-module-interface -o %t/A.v1.pcm
 //
-- 
GitLab


From 02ce8227ac28e0b83cf780716ae8f912d076eebe Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Mon, 6 May 2024 23:46:18 -0400
Subject: [PATCH 0010/1206] [NFC][OpenMP][OMPX] Move `declare variant` up

---
 openmp/runtime/src/include/ompx.h.var | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/openmp/runtime/src/include/ompx.h.var b/openmp/runtime/src/include/ompx.h.var
index 5dd8e8355e4c..579d31aa98c5 100644
--- a/openmp/runtime/src/include/ompx.h.var
+++ b/openmp/runtime/src/include/ompx.h.var
@@ -50,9 +50,12 @@ enum {
   ompx_dim_z = 2,
 };
 
+// TODO: The following implementation is for host fallback. We need to disable
+// generation of host fallback in kernel language mode.
+#pragma omp begin declare variant match(device = {kind(cpu)})
+
 /// ompx_{thread,block}_{id,dim}
 ///{
-#pragma omp begin declare variant match(device = {kind(cpu)})
 #define _TGT_KERNEL_LANGUAGE_HOST_IMPL_GRID_C(NAME, VALUE)                     \
   static inline int ompx_##NAME(int Dim) { return VALUE; }
 
-- 
GitLab


From 879245e2b5d48b629e8b085afacf69cc1fd6a6ec Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Mon, 6 May 2024 21:39:13 -0700
Subject: [PATCH 0011/1206] [NFC]Extract the heuristic to find vtable for an
 indirect call into a helper function  (#81024)

* This way the helper function could be re-used by
indirect-call-promotion pass to find out the vtable for an indirect call
and extract the value profiles if any.
* The parent patch is https://github.com/llvm/llvm-project/pull/80762
---
 .../llvm/Analysis/IndirectCallVisitor.h       | 53 ++++++++++---------
 1 file changed, 29 insertions(+), 24 deletions(-)

diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
index 50815f4e3e83..66c972572b06 100644
--- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h
+++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h
@@ -27,31 +27,21 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
   std::vector<Instruction *> ProfiledAddresses;
   PGOIndirectCallVisitor(InstructionType Type) : Type(Type) {}
 
-  void visitCallBase(CallBase &Call) {
-    if (!Call.isIndirectCall())
-      return;
-
-    if (Type == InstructionType::kIndirectCall) {
-      IndirectCalls.push_back(&Call);
-      return;
-    }
-
-    assert(Type == InstructionType::kVTableVal && "Control flow guaranteed");
+  // Given an indirect call instruction, try to find the the following pattern
+  //
+  // %vtable = load ptr, ptr %obj
+  // %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1
+  // %2 = load ptr, ptr %vfn
+  // $call = tail call i32 %2
+  //
+  // A heuristic is used to find the address feeding instructions.
+  static Instruction *tryGetVTableInstruction(CallBase *CB) {
+    assert(CB != nullptr && "Caller guaranteed");
+    LoadInst *LI = dyn_cast<LoadInst>(CB->getCalledOperand());
 
-    LoadInst *LI = dyn_cast<LoadInst>(Call.getCalledOperand());
-    // The code pattern to look for
-    //
-    // %vtable = load ptr, ptr %b
-    // %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1
-    // %2 = load ptr, ptr %vfn
-    // %call = tail call i32 %2(ptr %b)
-    //
-    // %vtable is the vtable address value to profile, and
-    // %2 is the indirect call target address to profile.
     if (LI != nullptr) {
-      Value *Ptr = LI->getPointerOperand();
-      Value *VTablePtr = Ptr->stripInBoundsConstantOffsets();
-      // This is a heuristic to find address feeding instructions.
+      Value *FuncPtr = LI->getPointerOperand(); // GEP (or bitcast)
+      Value *VTablePtr = FuncPtr->stripInBoundsConstantOffsets();
       // FIXME: Add support in the frontend so LLVM type intrinsics are
       // emitted without LTO. This way, added intrinsics could filter
       // non-vtable instructions and reduce instrumentation overhead.
@@ -63,7 +53,22 @@ struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
       // address is negligible if exists at all. Comparing loaded address
       // with symbol address guarantees correctness.
       if (VTablePtr != nullptr && isa<Instruction>(VTablePtr))
-        ProfiledAddresses.push_back(cast<Instruction>(VTablePtr));
+        return cast<Instruction>(VTablePtr);
+    }
+    return nullptr;
+  }
+
+  void visitCallBase(CallBase &Call) {
+    if (Call.isIndirectCall()) {
+      IndirectCalls.push_back(&Call);
+
+      if (Type != InstructionType::kVTableVal)
+        return;
+
+      Instruction *VPtr =
+          PGOIndirectCallVisitor::tryGetVTableInstruction(&Call);
+      if (VPtr)
+        ProfiledAddresses.push_back(VPtr);
     }
   }
 
-- 
GitLab


From b42f553af5179b26efe38bee2c1b7aa365b06517 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Tue, 7 May 2024 07:12:58 +0200
Subject: [PATCH 0012/1206] [GlobalIsel] Combine extract vector element
 (#90339)

look through shuffle vectors
---
 .../llvm/CodeGen/GlobalISel/CombinerHelper.h  |   5 +
 .../CodeGen/GlobalISel/GenericMachineInstrs.h |  12 ++
 .../include/llvm/Target/GlobalISel/Combine.td |   8 ++
 .../GlobalISel/CombinerHelperVectorOps.cpp    | 106 ++++++++++++++++++
 .../GlobalISel/combine-extract-vec-elt.mir    |  63 +++++++++++
 .../CodeGen/AArch64/extract-vector-elt.ll     |  19 +---
 6 files changed, 198 insertions(+), 15 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 76e8d1166ae0..4f1c9642e117 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -848,6 +848,11 @@ public:
   bool matchExtractVectorElementWithBuildVectorTrunc(const MachineOperand &MO,
                                                      BuildFnTy &MatchInfo);
 
+  /// Combine extract vector element with a shuffle vector on the vector
+  /// register.
+  bool matchExtractVectorElementWithShuffleVector(const MachineOperand &MO,
+                                                  BuildFnTy &MatchInfo);
+
   /// Combine extract vector element with a insert vector element on the vector
   /// register and different indices.
   bool matchExtractVectorElementWithDifferentIndices(const MachineOperand &MO,
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 25e47114e4a3..705ef0fa7f2b 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -294,6 +294,18 @@ public:
   }
 };
 
+/// Represents a G_SHUFFLE_VECTOR.
+class GShuffleVector : public GenericMachineInstr {
+public:
+  Register getSrc1Reg() const { return getOperand(1).getReg(); }
+  Register getSrc2Reg() const { return getOperand(2).getReg(); }
+  ArrayRef<int> getMask() const { return getOperand(3).getShuffleMask(); }
+
+  static bool classof(const MachineInstr *MI) {
+    return MI->getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR;
+  }
+};
+
 /// Represents a G_PTR_ADD.
 class GPtrAdd : public GenericMachineInstr {
 public:
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index f7895fbc6539..72c5de03f4e7 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1501,6 +1501,13 @@ def extract_vector_element_freeze : GICombineRule<
    [{ return Helper.matchExtractVectorElementWithFreeze(${root}, ${matchinfo}); }]),
    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
 
+def extract_vector_element_shuffle_vector : GICombineRule<
+   (defs root:$root, build_fn_matchinfo:$matchinfo),
+   (match (G_SHUFFLE_VECTOR $src, $src1, $src2, $mask),
+          (G_EXTRACT_VECTOR_ELT $root, $src, $idx),
+   [{ return Helper.matchExtractVectorElementWithShuffleVector(${root}, ${matchinfo}); }]),
+   (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
+
 // Combines concat operations
 def concat_matchinfo : GIDefMatchData<"SmallVector<Register>">;
 def combine_concat_vector : GICombineRule<
@@ -1578,6 +1585,7 @@ extract_vector_element_build_vector_trunc6,
 extract_vector_element_build_vector_trunc7,
 extract_vector_element_build_vector_trunc8,
 extract_vector_element_freeze,
+extract_vector_element_shuffle_vector,
 insert_vector_element_extract_vector_element
 ]>;
 
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp
index fb33801a3a33..21b1eb262817 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp
@@ -325,6 +325,112 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVectorTrunc(
   return true;
 }
 
+bool CombinerHelper::matchExtractVectorElementWithShuffleVector(
+    const MachineOperand &MO, BuildFnTy &MatchInfo) {
+  GExtractVectorElement *Extract =
+      cast<GExtractVectorElement>(getDefIgnoringCopies(MO.getReg(), MRI));
+
+  //
+  //  %zero:_(s64) = G_CONSTANT i64 0
+  //  %sv:_(<4 x s32>) = G_SHUFFLE_SHUFFLE %arg1(<4 x s32>), %arg2(<4 x s32>),
+  //                     shufflemask(0, 0, 0, 0)
+  //  %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %zero(s64)
+  //
+  //  -->
+  //
+  //  %zero1:_(s64) = G_CONSTANT i64 0
+  //  %extract:_(s32) = G_EXTRACT_VECTOR_ELT %arg1(<4 x s32>), %zero1(s64)
+  //
+  //
+  //
+  //
+  //  %three:_(s64) = G_CONSTANT i64 3
+  //  %sv:_(<4 x s32>) = G_SHUFFLE_SHUFFLE %arg1(<4 x s32>), %arg2(<4 x s32>),
+  //                     shufflemask(0, 0, 0, -1)
+  //  %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %three(s64)
+  //
+  //  -->
+  //
+  //  %extract:_(s32) = G_IMPLICIT_DEF
+  //
+  //
+  //
+  //
+  //
+  //  %sv:_(<4 x s32>) = G_SHUFFLE_SHUFFLE %arg1(<4 x s32>), %arg2(<4 x s32>),
+  //                     shufflemask(0, 0, 0, -1)
+  //  %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %opaque(s64)
+  //
+  //  -->
+  //
+  //  %sv:_(<4 x s32>) = G_SHUFFLE_SHUFFLE %arg1(<4 x s32>), %arg2(<4 x s32>),
+  //                     shufflemask(0, 0, 0, -1)
+  //  %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %opaque(s64)
+  //
+
+  // We try to get the value of the Index register.
+  std::optional<ValueAndVReg> MaybeIndex =
+      getIConstantVRegValWithLookThrough(Extract->getIndexReg(), MRI);
+  if (!MaybeIndex)
+    return false;
+
+  GShuffleVector *Shuffle =
+      cast<GShuffleVector>(getDefIgnoringCopies(Extract->getVectorReg(), MRI));
+
+  ArrayRef<int> Mask = Shuffle->getMask();
+
+  unsigned Offset = MaybeIndex->Value.getZExtValue();
+  int SrcIdx = Mask[Offset];
+
+  LLT Src1Type = MRI.getType(Shuffle->getSrc1Reg());
+  // At the IR level a <1 x ty> shuffle  vector is valid, but we want to extract
+  // from a vector.
+  assert(Src1Type.isVector() && "expected to extract from a vector");
+  unsigned LHSWidth = Src1Type.isVector() ? Src1Type.getNumElements() : 1;
+
+  // Note that there is no one use check.
+  Register Dst = Extract->getReg(0);
+  LLT DstTy = MRI.getType(Dst);
+
+  if (SrcIdx < 0 &&
+      isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) {
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Dst); };
+    return true;
+  }
+
+  // If the legality check failed, then we still have to abort.
+  if (SrcIdx < 0)
+    return false;
+
+  Register NewVector;
+
+  // We check in which vector and at what offset to look through.
+  if (SrcIdx < (int)LHSWidth) {
+    NewVector = Shuffle->getSrc1Reg();
+    // SrcIdx unchanged
+  } else { // SrcIdx >= LHSWidth
+    NewVector = Shuffle->getSrc2Reg();
+    SrcIdx -= LHSWidth;
+  }
+
+  LLT IdxTy = MRI.getType(Extract->getIndexReg());
+  LLT NewVectorTy = MRI.getType(NewVector);
+
+  // We check the legality of the look through.
+  if (!isLegalOrBeforeLegalizer(
+          {TargetOpcode::G_EXTRACT_VECTOR_ELT, {DstTy, NewVectorTy, IdxTy}}) ||
+      !isConstantLegalOrBeforeLegalizer({IdxTy}))
+    return false;
+
+  // We look through the shuffle vector.
+  MatchInfo = [=](MachineIRBuilder &B) {
+    auto Idx = B.buildConstant(IdxTy, SrcIdx);
+    B.buildExtractVectorElement(Dst, NewVector, Idx);
+  };
+
+  return true;
+}
+
 bool CombinerHelper::matchInsertVectorElementOOB(MachineInstr &MI,
                                                  BuildFnTy &MatchInfo) {
   GInsertVectorElement *Insert = cast<GInsertVectorElement>(&MI);
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir
index 587d53c300f8..d5d33742148a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir
@@ -571,3 +571,66 @@ body:             |
     RET_ReallyLR implicit $x0
 ...
 ---
+name:            extract_from_build_vector_shuffle_vector_undef
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: extract_from_build_vector_shuffle_vector_undef
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %extract:_(s32) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $w0 = COPY %extract(s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %arg1:_(<4 x s32>) = COPY $q0
+    %arg2:_(<4 x s32>) = COPY $q1
+    %idx:_(s64) = G_CONSTANT i64 0
+    %sv:_(<4 x s32>) = G_SHUFFLE_VECTOR %arg1(<4 x s32>), %arg2(<4 x s32>), shufflemask(-1, 0, 0, 0)
+    %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %idx(s64)
+    $w0 = COPY %extract(s32)
+    RET_ReallyLR implicit $x0
+...
+---
+name:            extract_from_build_vector_shuffle_vector_opaque
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: extract_from_build_vector_shuffle_vector_opaque
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %arg1:_(<4 x s32>) = COPY $q0
+    ; CHECK-NEXT: %arg2:_(<4 x s32>) = COPY $q1
+    ; CHECK-NEXT: %idx:_(s64) = COPY $x1
+    ; CHECK-NEXT: %sv:_(<4 x s32>) = G_SHUFFLE_VECTOR %arg1(<4 x s32>), %arg2, shufflemask(undef, 0, 0, 0)
+    ; CHECK-NEXT: %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %idx(s64)
+    ; CHECK-NEXT: $w0 = COPY %extract(s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %arg1:_(<4 x s32>) = COPY $q0
+    %arg2:_(<4 x s32>) = COPY $q1
+    %idx:_(s64) = COPY $x1
+    %sv:_(<4 x s32>) = G_SHUFFLE_VECTOR %arg1(<4 x s32>), %arg2(<4 x s32>), shufflemask(-1, 0, 0, 0)
+    %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %idx(s64)
+    $w0 = COPY %extract(s32)
+    RET_ReallyLR implicit $x0
+...
+---
+name:            extract_from_build_vector_shuffle_vector_const
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: extract_from_build_vector_shuffle_vector_const
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %arg1:_(<4 x s32>) = COPY $q0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: %extract:_(s32) = G_EXTRACT_VECTOR_ELT %arg1(<4 x s32>), [[C]](s64)
+    ; CHECK-NEXT: $w0 = COPY %extract(s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %arg1:_(<4 x s32>) = COPY $q0
+    %arg2:_(<4 x s32>) = COPY $q1
+    %idx:_(s64) = G_CONSTANT i64 0
+    %sv:_(<4 x s32>) = G_SHUFFLE_VECTOR %arg1(<4 x s32>), %arg2(<4 x s32>), shufflemask(3, 0, 0, 0)
+    %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %idx(s64)
+    $w0 = COPY %extract(s32)
+    RET_ReallyLR implicit $x0
+...
+---
diff --git a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
index 504222e0036e..0481d997d24f 100644
--- a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
+++ b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
@@ -938,21 +938,10 @@ entry:
 }
 
 define i32 @extract_v4i32_shuffle_const(<4 x i32> %a, <4 x i32> %b, i32 %c) {
-; CHECK-SD-LABEL: extract_v4i32_shuffle_const:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov w0, s1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: extract_v4i32_shuffle_const:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    adrp x8, .LCPI36_0
-; CHECK-GI-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI36_0]
-; CHECK-GI-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-GI-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
-; CHECK-GI-NEXT:    mov s0, v0.s[2]
-; CHECK-GI-NEXT:    fmov w0, s0
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: extract_v4i32_shuffle_const:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov w0, s1
+; CHECK-NEXT:    ret
 entry:
   %vector = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 3>
   %d = extractelement <4 x i32> %vector, i32 2
-- 
GitLab


From ad9f38d0e3a5e7e06c39dbd7da88a921a49aa805 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Tue, 7 May 2024 13:25:07 +0800
Subject: [PATCH 0013/1206] [NFC] Fix
 Modules/no-transitive-source-location-change.cppm after dfa7ff97b2

The test fails after dfa7ff97b2. I didn't find this locally due to
cache.
---
 clang/test/Modules/no-transitive-source-location-change.cppm | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/clang/test/Modules/no-transitive-source-location-change.cppm b/clang/test/Modules/no-transitive-source-location-change.cppm
index c9d156a74ce8..2a84ef6a912f 100644
--- a/clang/test/Modules/no-transitive-source-location-change.cppm
+++ b/clang/test/Modules/no-transitive-source-location-change.cppm
@@ -1,6 +1,9 @@
 // Testing that adding a new line in a module interface unit won't cause the BMI
 // of consuming module unit changes.
 //
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+//
 // RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm
 // RUN: %clang_cc1 -std=c++20 %t/A.v1.cppm -emit-reduced-module-interface -o %t/A.v1.pcm
 //
-- 
GitLab


From 05f4448d40f00b9fb2447e1c32cd18a7a9b8b011 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Tue, 7 May 2024 06:27:33 +0200
Subject: [PATCH 0014/1206] [clang][Interp][NFC] Add eval-order test

Demonstrate that this isn't yet working right.
---
 clang/test/AST/Interp/eval-order.cpp | 117 +++++++++++++++++++++++++++
 1 file changed, 117 insertions(+)
 create mode 100644 clang/test/AST/Interp/eval-order.cpp

diff --git a/clang/test/AST/Interp/eval-order.cpp b/clang/test/AST/Interp/eval-order.cpp
new file mode 100644
index 000000000000..695a43c9d235
--- /dev/null
+++ b/clang/test/AST/Interp/eval-order.cpp
@@ -0,0 +1,117 @@
+// RUN: %clang_cc1 -std=c++1z -verify %s -fcxx-exceptions -triple=x86_64-linux-gnu
+// RUN: %clang_cc1 -std=c++1z -verify %s -fcxx-exceptions -triple=x86_64-linux-gnu -fexperimental-new-constant-interpreter
+
+// ref-no-diagnostics
+// expected-no-diagnostics
+
+/// Check that assignment operators evaluate their operands right-to-left.
+/// Copied from test/SemaCXX/constant-expression-cxx1z.cpp
+///
+/// As you can see from the FIXME comments, some of these are not yet working correctly
+/// in the new interpreter.
+namespace EvalOrder {
+  template<typename T> struct lvalue {
+    T t;
+    constexpr T &get() { return t; }
+  };
+
+  struct UserDefined {
+    int n = 0;
+    constexpr UserDefined &operator=(const UserDefined&) { return *this; }
+    constexpr UserDefined &operator+=(const UserDefined&) { return *this; }
+    constexpr void operator<<(const UserDefined&) const {}
+    constexpr void operator>>(const UserDefined&) const {}
+    constexpr void operator+(const UserDefined&) const {}
+    constexpr void operator[](int) const {}
+  };
+  constexpr UserDefined ud;
+
+  struct NonMember {};
+  constexpr void operator+=(NonMember, NonMember) {}
+  constexpr void operator<<(NonMember, NonMember) {}
+  constexpr void operator>>(NonMember, NonMember) {}
+  constexpr void operator+(NonMember, NonMember) {}
+  constexpr NonMember nm;
+
+  constexpr void f(...) {}
+
+  // Helper to ensure that 'a' is evaluated before 'b'.
+  struct seq_checker {
+    bool done_a = false;
+    bool done_b = false;
+
+    template <typename T> constexpr T &&a(T &&v) {
+      done_a = true;
+      return (T &&)v;
+    }
+    template <typename T> constexpr T &&b(T &&v) {
+      if (!done_a)
+        throw "wrong";
+      done_b = true;
+      return (T &&)v;
+    }
+
+    constexpr bool ok() { return done_a && done_b; }
+  };
+
+  // SEQ(expr), where part of the expression is tagged A(...) and part is
+  // tagged B(...), checks that A is evaluated before B.
+  #define A sc.a
+  #define B sc.b
+  #define SEQ(...) static_assert([](seq_checker sc) { void(__VA_ARGS__); return sc.ok(); }({}))
+
+  // Longstanding sequencing rules.
+  SEQ((A(1), B(2)));
+  SEQ((A(true) ? B(2) : throw "huh?"));
+  SEQ((A(false) ? throw "huh?" : B(2)));
+  SEQ(A(true) && B(true));
+  SEQ(A(false) || B(true));
+
+  // From P0145R3:
+
+  // Rules 1 and 2 have no effect ('b' is not an expression).
+
+  // Rule 3: a->*b
+  // SEQ(A(ud).*B(&UserDefined::n)); FIXME
+  // SEQ(A(&ud)->*B(&UserDefined::n)); FIXME
+
+  // Rule 4: a(b1, b2, b3)
+  // SEQ(A(f)(B(1), B(2), B(3))); FIXME
+
+  // Rule 5: b = a, b @= a
+  // SEQ(B(lvalue<int>().get()) = A(0)); FIXME
+  // SEQ(B(lvalue<UserDefined>().get()) = A(ud)); FIXME
+  SEQ(B(lvalue<int>().get()) += A(0));
+  // SEQ(B(lvalue<UserDefined>().get()) += A(ud)); FIXME
+  // SEQ(B(lvalue<NonMember>().get()) += A(nm)); FIXME
+
+  // Rule 6: a[b]
+  constexpr int arr[3] = {};
+  SEQ(A(arr)[B(0)]);
+  SEQ(A(+arr)[B(0)]);
+  // SEQ(A(0)[B(arr)]); FIXME
+  // SEQ(A(0)[B(+arr)]); FIXME
+  SEQ(A(ud)[B(0)]);
+
+  // Rule 7: a << b
+  SEQ(A(1) << B(2));
+  SEQ(A(ud) << B(ud));
+  SEQ(A(nm) << B(nm));
+
+  // Rule 8: a >> b
+  SEQ(A(1) >> B(2));
+  SEQ(A(ud) >> B(ud));
+  SEQ(A(nm) >> B(nm));
+
+  // No particular order of evaluation is specified in other cases, but we in
+  // practice evaluate left-to-right.
+  // FIXME: Technically we're expected to check for undefined behavior due to
+  // unsequenced read and modification and treat it as non-constant due to UB.
+  SEQ(A(1) + B(2));
+  SEQ(A(ud) + B(ud));
+  SEQ(A(nm) + B(nm));
+  SEQ(f(A(1), B(2)));
+  #undef SEQ
+  #undef A
+  #undef B
+}
-- 
GitLab


From 5f2f3900138cc519e1cb807e99920337eede2b6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Tue, 7 May 2024 08:53:45 +0200
Subject: [PATCH 0015/1206] [clang][Interp][NFC] Allow Pointer assignment if
 both are zero

... even if the storage types are different.
---
 clang/lib/AST/Interp/Pointer.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp
index 5ef31671ae7b..12bef73f7e21 100644
--- a/clang/lib/AST/Interp/Pointer.cpp
+++ b/clang/lib/AST/Interp/Pointer.cpp
@@ -63,9 +63,8 @@ Pointer::~Pointer() {
 }
 
 void Pointer::operator=(const Pointer &P) {
-
   if (!this->isIntegralPointer() || !P.isBlockPointer())
-    assert(P.StorageKind == StorageKind);
+    assert(P.StorageKind == StorageKind || (this->isZero() && P.isZero()));
 
   bool WasBlockPointer = isBlockPointer();
   StorageKind = P.StorageKind;
@@ -92,7 +91,7 @@ void Pointer::operator=(const Pointer &P) {
 
 void Pointer::operator=(Pointer &&P) {
   if (!this->isIntegralPointer() || !P.isBlockPointer())
-    assert(P.StorageKind == StorageKind);
+    assert(P.StorageKind == StorageKind || (this->isZero() && P.isZero()));
 
   bool WasBlockPointer = isBlockPointer();
   StorageKind = P.StorageKind;
-- 
GitLab


From 2b9210d1aa9ce9c204b3af0158636c71a5a72e17 Mon Sep 17 00:00:00 2001
From: Abhishek Varma <abhvarma@amd.com>
Date: Tue, 7 May 2024 12:38:14 +0530
Subject: [PATCH 0016/1206] [MLIR][SCF] Add canonicalization pattern to fold
 away iter args of scf.forall (#90189)

-- This commit adds a canonicalization pattern to fold away iter args of
scf.forall if :-
   a. The corresponding tied result has no use.
   b. It is not being modified within the loop.

Signed-off-by: Abhishek Varma <avarma094@gmail.com>
---
 mlir/include/mlir/Dialect/SCF/IR/SCFOps.td |   4 +
 mlir/lib/Dialect/SCF/IR/SCF.cpp            | 188 ++++++++++++++++++++-
 mlir/test/Dialect/SCF/canonicalize.mlir    |  81 +++++++++
 3 files changed, 272 insertions(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
index 41a0b67c42a0..0b063aa772ba 100644
--- a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
+++ b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
@@ -609,6 +609,10 @@ def ForallOp : SCF_Op<"forall", [
 
     // Declare the shared_outs as inits/outs to DestinationStyleOpInterface.
     MutableOperandRange getDpsInitsMutable() { return getOutputsMutable(); }
+
+    /// Returns operations within scf.forall.in_parallel whose destination
+    /// operand is the block argument `bbArg`.
+    SmallVector<Operation*> getCombiningOps(BlockArgument bbArg);
   }];
 }
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 7a1aafc9f1c2..107fd0690f19 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -1415,6 +1415,19 @@ InParallelOp ForallOp::getTerminator() {
   return cast<InParallelOp>(getBody()->getTerminator());
 }
 
+SmallVector<Operation *> ForallOp::getCombiningOps(BlockArgument bbArg) {
+  SmallVector<Operation *> storeOps;
+  InParallelOp inParallelOp = getTerminator();
+  for (Operation &yieldOp : inParallelOp.getYieldingOps()) {
+    if (auto parallelInsertSliceOp =
+            dyn_cast<tensor::ParallelInsertSliceOp>(yieldOp);
+        parallelInsertSliceOp && parallelInsertSliceOp.getDest() == bbArg) {
+      storeOps.push_back(parallelInsertSliceOp);
+    }
+  }
+  return storeOps;
+}
+
 std::optional<Value> ForallOp::getSingleInductionVar() {
   if (getRank() != 1)
     return std::nullopt;
@@ -1509,6 +1522,179 @@ public:
   }
 };
 
+/// The following canonicalization pattern folds the iter arguments of
+/// scf.forall op if :-
+/// 1. The corresponding result has zero uses.
+/// 2. The iter argument is NOT being modified within the loop body.
+/// uses.
+///
+/// Example of first case :-
+///  INPUT:
+///   %res:3 = scf.forall ... shared_outs(%arg0 = %a, %arg1 = %b, %arg2 = %c)
+///            {
+///                ...
+///                <SOME USE OF %arg0>
+///                <SOME USE OF %arg1>
+///                <SOME USE OF %arg2>
+///                ...
+///                scf.forall.in_parallel {
+///                    <STORE OP WITH DESTINATION %arg1>
+///                    <STORE OP WITH DESTINATION %arg0>
+///                    <STORE OP WITH DESTINATION %arg2>
+///                }
+///             }
+///   return %res#1
+///
+///  OUTPUT:
+///   %res:3 = scf.forall ... shared_outs(%new_arg0 = %b)
+///            {
+///                ...
+///                <SOME USE OF %a>
+///                <SOME USE OF %new_arg0>
+///                <SOME USE OF %c>
+///                ...
+///                scf.forall.in_parallel {
+///                    <STORE OP WITH DESTINATION %new_arg0>
+///                }
+///             }
+///   return %res
+///
+/// NOTE: 1. All uses of the folded shared_outs (iter argument) within the
+///          scf.forall is replaced by their corresponding operands.
+///       2. Even if there are <STORE OP WITH DESTINATION *> ops within the body
+///          of the scf.forall besides within scf.forall.in_parallel terminator,
+///          this canonicalization remains valid. For more details, please refer
+///          to :
+///          https://github.com/llvm/llvm-project/pull/90189#discussion_r1589011124
+///       3. TODO(avarma): Generalize it for other store ops. Currently it
+///          handles tensor.parallel_insert_slice ops only.
+///
+/// Example of second case :-
+///  INPUT:
+///   %res:2 = scf.forall ... shared_outs(%arg0 = %a, %arg1 = %b)
+///            {
+///                ...
+///                <SOME USE OF %arg0>
+///                <SOME USE OF %arg1>
+///                ...
+///                scf.forall.in_parallel {
+///                    <STORE OP WITH DESTINATION %arg1>
+///                }
+///             }
+///   return %res#0, %res#1
+///
+///  OUTPUT:
+///   %res = scf.forall ... shared_outs(%new_arg0 = %b)
+///            {
+///                ...
+///                <SOME USE OF %a>
+///                <SOME USE OF %new_arg0>
+///                ...
+///                scf.forall.in_parallel {
+///                    <STORE OP WITH DESTINATION %new_arg0>
+///                }
+///             }
+///   return %a, %res
+struct ForallOpIterArgsFolder : public OpRewritePattern<ForallOp> {
+  using OpRewritePattern<ForallOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(ForallOp forallOp,
+                                PatternRewriter &rewriter) const final {
+    // Step 1: For a given i-th result of scf.forall, check the following :-
+    //         a. If it has any use.
+    //         b. If the corresponding iter argument is being modified within
+    //            the loop, i.e. has at least one store op with the iter arg as
+    //            its destination operand. For this we use
+    //            ForallOp::getCombiningOps(iter_arg).
+    //
+    //         Based on the check we maintain the following :-
+    //         a. `resultToDelete` - i-th result of scf.forall that'll be
+    //            deleted.
+    //         b. `resultToReplace` - i-th result of the old scf.forall
+    //            whose uses will be replaced by the new scf.forall.
+    //         c. `newOuts` - the shared_outs' operand of the new scf.forall
+    //            corresponding to the i-th result with at least one use.
+    SetVector<OpResult> resultToDelete;
+    SmallVector<Value> resultToReplace;
+    SmallVector<Value> newOuts;
+    for (OpResult result : forallOp.getResults()) {
+      OpOperand *opOperand = forallOp.getTiedOpOperand(result);
+      BlockArgument blockArg = forallOp.getTiedBlockArgument(opOperand);
+      if (result.use_empty() || forallOp.getCombiningOps(blockArg).empty()) {
+        resultToDelete.insert(result);
+      } else {
+        resultToReplace.push_back(result);
+        newOuts.push_back(opOperand->get());
+      }
+    }
+
+    // Return early if all results of scf.forall have at least one use and being
+    // modified within the loop.
+    if (resultToDelete.empty())
+      return failure();
+
+    // Step 2: For the the i-th result, do the following :-
+    //         a. Fetch the corresponding BlockArgument.
+    //         b. Look for store ops (currently tensor.parallel_insert_slice)
+    //            with the BlockArgument as its destination operand.
+    //         c. Remove the operations fetched in b.
+    for (OpResult result : resultToDelete) {
+      OpOperand *opOperand = forallOp.getTiedOpOperand(result);
+      BlockArgument blockArg = forallOp.getTiedBlockArgument(opOperand);
+      SmallVector<Operation *> combiningOps =
+          forallOp.getCombiningOps(blockArg);
+      for (Operation *combiningOp : combiningOps)
+        rewriter.eraseOp(combiningOp);
+    }
+
+    // Step 3. Create a new scf.forall op with the new shared_outs' operands
+    //         fetched earlier
+    auto newForallOp = rewriter.create<scf::ForallOp>(
+        forallOp.getLoc(), forallOp.getMixedLowerBound(),
+        forallOp.getMixedUpperBound(), forallOp.getMixedStep(), newOuts,
+        forallOp.getMapping(),
+        /*bodyBuilderFn =*/[](OpBuilder &, Location, ValueRange) {});
+
+    // Step 4. Merge the block of the old scf.forall into the newly created
+    //         scf.forall using the new set of arguments.
+    Block *loopBody = forallOp.getBody();
+    Block *newLoopBody = newForallOp.getBody();
+    ArrayRef<BlockArgument> newBbArgs = newLoopBody->getArguments();
+    // Form initial new bbArg list with just the control operands of the new
+    // scf.forall op.
+    SmallVector<Value> newBlockArgs =
+        llvm::map_to_vector(newBbArgs.take_front(forallOp.getRank()),
+                            [](BlockArgument b) -> Value { return b; });
+    Block::BlockArgListType newSharedOutsArgs = newForallOp.getRegionOutArgs();
+    unsigned index = 0;
+    // Take the new corresponding bbArg if the old bbArg was used as a
+    // destination in the in_parallel op. For all other bbArgs, use the
+    // corresponding init_arg from the old scf.forall op.
+    for (OpResult result : forallOp.getResults()) {
+      if (resultToDelete.count(result)) {
+        newBlockArgs.push_back(forallOp.getTiedOpOperand(result)->get());
+      } else {
+        newBlockArgs.push_back(newSharedOutsArgs[index++]);
+      }
+    }
+    rewriter.mergeBlocks(loopBody, newLoopBody, newBlockArgs);
+
+    // Step 5. Replace the uses of result of old scf.forall with that of the new
+    //         scf.forall.
+    for (auto &&[oldResult, newResult] :
+         llvm::zip(resultToReplace, newForallOp->getResults()))
+      rewriter.replaceAllUsesWith(oldResult, newResult);
+
+    // Step 6. Replace the uses of those values that either has no use or are
+    //         not being modified within the loop with the corresponding
+    //         OpOperand.
+    for (OpResult oldResult : resultToDelete)
+      rewriter.replaceAllUsesWith(oldResult,
+                                  forallOp.getTiedOpOperand(oldResult)->get());
+    return success();
+  }
+};
+
 struct ForallOpSingleOrZeroIterationDimsFolder
     : public OpRewritePattern<ForallOp> {
   using OpRewritePattern<ForallOp>::OpRewritePattern;
@@ -1667,7 +1853,7 @@ struct FoldTensorCastOfOutputIntoForallOp
 void ForallOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                            MLIRContext *context) {
   results.add<DimOfForallOp, FoldTensorCastOfOutputIntoForallOp,
-              ForallOpControlOperandsFolder,
+              ForallOpControlOperandsFolder, ForallOpIterArgsFolder,
               ForallOpSingleOrZeroIterationDimsFolder>(context);
 }
 
diff --git a/mlir/test/Dialect/SCF/canonicalize.mlir b/mlir/test/Dialect/SCF/canonicalize.mlir
index b4c9ed4db94e..459ccd73cfe6 100644
--- a/mlir/test/Dialect/SCF/canonicalize.mlir
+++ b/mlir/test/Dialect/SCF/canonicalize.mlir
@@ -1735,6 +1735,87 @@ func.func @do_not_fold_tensor_cast_from_dynamic_to_static_type_into_forall(
 
 // -----
 
+#map = affine_map<()[s0, s1] -> (s0 ceildiv s1)>
+#map1 = affine_map<(d0)[s0] -> (d0 * s0)>
+#map2 = affine_map<(d0)[s0, s1] -> (-(d0 * s1) + s0, s1)>
+module {
+  func.func @fold_iter_args_not_being_modified_within_scfforall(%arg0: index, %arg1: tensor<?xf32>, %arg2: tensor<?xf32>) -> (tensor<?xf32>, tensor<?xf32>) {
+    %c0 = arith.constant 0 : index
+    %cst = arith.constant 4.200000e+01 : f32
+    %0 = linalg.fill ins(%cst : f32) outs(%arg1 : tensor<?xf32>) -> tensor<?xf32>
+    %dim = tensor.dim %arg1, %c0 : tensor<?xf32>
+    %1 = affine.apply #map()[%dim, %arg0]
+    %2:2 = scf.forall (%arg3) in (%1) shared_outs(%arg4 = %arg1, %arg5 = %arg2) -> (tensor<?xf32>, tensor<?xf32>) {
+      %3 = affine.apply #map1(%arg3)[%arg0]
+      %4 = affine.min #map2(%arg3)[%dim, %arg0]
+      %extracted_slice0 = tensor.extract_slice %arg4[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
+      %extracted_slice1 = tensor.extract_slice %arg5[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
+      %5 = linalg.elemwise_unary ins(%extracted_slice0 : tensor<?xf32>) outs(%extracted_slice1 : tensor<?xf32>) -> tensor<?xf32>
+      scf.forall.in_parallel {
+        tensor.parallel_insert_slice %5 into %arg5[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
+      }
+    }
+    return %2#0, %2#1 : tensor<?xf32>, tensor<?xf32>
+  }
+}
+// CHECK-LABEL: @fold_iter_args_not_being_modified_within_scfforall
+//  CHECK-SAME:   (%{{.*}}: index, %[[ARG1:.*]]: tensor<?xf32>, %[[ARG2:.*]]: tensor<?xf32>) -> (tensor<?xf32>, tensor<?xf32>) {
+//       CHECK:    %[[RESULT:.*]] = scf.forall 
+//  CHECK-SAME:                       shared_outs(%[[ITER_ARG_5:.*]] = %[[ARG2]]) -> (tensor<?xf32>) {
+//       CHECK:      %[[OPERAND0:.*]] = tensor.extract_slice %[[ARG1]]
+//       CHECK:      %[[OPERAND1:.*]] = tensor.extract_slice %[[ITER_ARG_5]]
+//       CHECK:      %[[ELEM:.*]] = linalg.elemwise_unary ins(%[[OPERAND0]] : tensor<?xf32>) outs(%[[OPERAND1]] : tensor<?xf32>) -> tensor<?xf32>
+//       CHECK:      scf.forall.in_parallel {
+//  CHECK-NEXT:         tensor.parallel_insert_slice %[[ELEM]] into %[[ITER_ARG_5]]
+//  CHECK-NEXT:      }
+//  CHECK-NEXT:    }
+//  CHECK-NEXT:    return %[[ARG1]], %[[RESULT]]
+
+// -----
+
+#map = affine_map<()[s0, s1] -> (s0 ceildiv s1)>
+#map1 = affine_map<(d0)[s0] -> (d0 * s0)>
+#map2 = affine_map<(d0)[s0, s1] -> (-(d0 * s1) + s0, s1)>
+module {
+  func.func @fold_iter_args_with_no_use_of_result_scfforall(%arg0: index, %arg1: tensor<?xf32>, %arg2: tensor<?xf32>, %arg3: tensor<?xf32>) -> tensor<?xf32> {
+    %cst = arith.constant 4.200000e+01 : f32
+    %c0 = arith.constant 0 : index
+    %0 = linalg.fill ins(%cst : f32) outs(%arg1 : tensor<?xf32>) -> tensor<?xf32>
+    %dim = tensor.dim %arg1, %c0 : tensor<?xf32>
+    %1 = affine.apply #map()[%dim, %arg0]
+    %2:3 = scf.forall (%arg4) in (%1) shared_outs(%arg5 = %arg1, %arg6 = %arg2, %arg7 = %arg3) -> (tensor<?xf32>, tensor<?xf32>, tensor<?xf32>) {
+      %3 = affine.apply #map1(%arg4)[%arg0]
+      %4 = affine.min #map2(%arg4)[%dim, %arg0]
+      %extracted_slice = tensor.extract_slice %arg5[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
+      %extracted_slice_0 = tensor.extract_slice %arg6[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
+      %extracted_slice_1 = tensor.extract_slice %arg7[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
+      %extracted_slice_2 = tensor.extract_slice %0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
+      %5 = linalg.elemwise_unary ins(%extracted_slice : tensor<?xf32>) outs(%extracted_slice_1 : tensor<?xf32>) -> tensor<?xf32>
+      scf.forall.in_parallel {
+        tensor.parallel_insert_slice %5 into %arg6[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
+        tensor.parallel_insert_slice %extracted_slice into %arg5[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
+        tensor.parallel_insert_slice %extracted_slice_0 into %arg7[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
+        tensor.parallel_insert_slice %5 into %arg7[%4] [%3] [1] : tensor<?xf32> into tensor<?xf32>
+      }
+    }
+    return %2#1 : tensor<?xf32>
+  }
+}
+// CHECK-LABEL: @fold_iter_args_with_no_use_of_result_scfforall
+//  CHECK-SAME:   (%{{.*}}: index, %[[ARG1:.*]]: tensor<?xf32>, %[[ARG2:.*]]: tensor<?xf32>, %[[ARG3:.*]]: tensor<?xf32>) -> tensor<?xf32> {
+//       CHECK:    %[[RESULT:.*]] = scf.forall 
+//  CHECK-SAME:                       shared_outs(%[[ITER_ARG_6:.*]] = %[[ARG2]]) -> (tensor<?xf32>) {
+//       CHECK:      %[[OPERAND0:.*]] = tensor.extract_slice %[[ARG1]]
+//       CHECK:      %[[OPERAND1:.*]] = tensor.extract_slice %[[ARG3]]
+//       CHECK:      %[[ELEM:.*]] = linalg.elemwise_unary ins(%[[OPERAND0]] : tensor<?xf32>) outs(%[[OPERAND1]] : tensor<?xf32>) -> tensor<?xf32>
+//       CHECK:      scf.forall.in_parallel {
+//  CHECK-NEXT:         tensor.parallel_insert_slice %[[ELEM]] into %[[ITER_ARG_6]]
+//  CHECK-NEXT:      }
+//  CHECK-NEXT:    }
+//  CHECK-NEXT:    return %[[RESULT]]
+
+// -----
+
 func.func @index_switch_fold() -> (f32, f32) {
   %switch_cst = arith.constant 1: index
   %0 = scf.index_switch %switch_cst -> f32
-- 
GitLab


From fc866fd2a2cfca6d62f48dcf83778959fd24f559 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Tue, 7 May 2024 09:10:29 +0200
Subject: [PATCH 0017/1206] [clang] Don't preserve the typo expr in the
 recovery expr for invalid VarDecls (#90948)

With the commit d5308949cf884d8e4b971d51a8b4f73584c4adec, we now
preserve the initializer for invalid decls with the recovery-expr.

However there is a chance that the original init expr is a typo-expr, we
should not preserve it in the final AST, as typo-expr is an internal AST
node. We should use the one after the typo correction.

This is spotted by a clangd hover crash on the testcase.
---
 clang-tools-extra/clangd/unittests/HoverTests.cpp | 13 +++++++++++++
 clang/lib/Sema/SemaDecl.cpp                       |  7 +++++--
 clang/test/AST/ast-dump-recovery.cpp              |  5 +++++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/clang-tools-extra/clangd/unittests/HoverTests.cpp b/clang-tools-extra/clangd/unittests/HoverTests.cpp
index 5ead74748f55..28df24f34827 100644
--- a/clang-tools-extra/clangd/unittests/HoverTests.cpp
+++ b/clang-tools-extra/clangd/unittests/HoverTests.cpp
@@ -965,6 +965,19 @@ class Foo final {})cpp";
          // Bindings are in theory public members of an anonymous struct.
          HI.AccessSpecifier = "public";
        }},
+      {// Don't crash on invalid decl with invalid init expr.
+       R"cpp(
+          Unknown [[^abc]] = invalid;
+          // error-ok
+          )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "abc";
+         HI.Kind = index::SymbolKind::Variable;
+         HI.NamespaceScope = "";
+         HI.Definition = "int abc = <recovery - expr>()";
+         HI.Type = "int";
+         HI.AccessSpecifier = "public";
+       }},
       {// Extra info for function call.
        R"cpp(
           void fun(int arg_a, int &arg_b) {};
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 19968452f0d5..590f37837eb2 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -13530,9 +13530,12 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
   }
 
   if (VDecl->isInvalidDecl()) {
-    CorrectDelayedTyposInExpr(Init, VDecl);
+    ExprResult Res = CorrectDelayedTyposInExpr(Init, VDecl);
+    SmallVector<Expr *> SubExprs;
+    if (Res.isUsable())
+      SubExprs.push_back(Res.get());
     ExprResult Recovery =
-        CreateRecoveryExpr(Init->getBeginLoc(), Init->getEndLoc(), {Init});
+        CreateRecoveryExpr(Init->getBeginLoc(), Init->getEndLoc(), SubExprs);
     if (Expr *E = Recovery.get())
       VDecl->setInit(E);
     return;
diff --git a/clang/test/AST/ast-dump-recovery.cpp b/clang/test/AST/ast-dump-recovery.cpp
index 77527743fe85..a88dff471d9f 100644
--- a/clang/test/AST/ast-dump-recovery.cpp
+++ b/clang/test/AST/ast-dump-recovery.cpp
@@ -419,6 +419,11 @@ void InitializerOfInvalidDecl() {
   // CHECK:      VarDecl {{.*}} invalid InvalidDecl
   // CHECK-NEXT: `-RecoveryExpr {{.*}} '<dependent type>' contains-errors
   // CHECK-NEXT:   `-DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'ValidDecl'
+
+  Unknown InvalidDeclWithInvalidInit = Invalid;
+  // CHECK:      VarDecl {{.*}} invalid InvalidDeclWithInvalidInit
+  // CHECK-NEXT: `-RecoveryExpr {{.*}} '<dependent type>' contains-errors
+  // CHECK-NOT:    `-TypoExpr
 }
 
 void RecoverToAnInvalidDecl() {
-- 
GitLab


From 6ad37a41b5489ce66ea890bf92fca66ea1ae41e0 Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan@arm.com>
Date: Tue, 7 May 2024 08:13:32 +0100
Subject: [PATCH 0018/1206] [Flang][OpenMP] NFC: Trivial changes in
 OmpCycleChecker (#91024)

Cycle is associated with construct-names and not labels. Change name of
a few variables to reflect this. Also add appropriate comment to
describe the else case of error checking.
---
 flang/lib/Semantics/check-omp-structure.cpp | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index ab76fe59911b..70863c5f20e8 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -94,9 +94,10 @@ public:
 
   bool Pre(const parser::DoConstruct &dc) {
     cycleLevel_--;
-    const auto &labelName{std::get<0>(std::get<0>(dc.t).statement.t)};
-    if (labelName) {
-      labelNamesandLevels_.emplace(labelName.value().ToString(), cycleLevel_);
+    const auto &constructName{std::get<0>(std::get<0>(dc.t).statement.t)};
+    if (constructName) {
+      constructNamesAndLevels_.emplace(
+          constructName.value().ToString(), cycleLevel_);
     }
     return true;
   }
@@ -105,10 +106,14 @@ public:
     std::map<std::string, std::int64_t>::iterator it;
     bool err{false};
     if (cyclestmt.v) {
-      it = labelNamesandLevels_.find(cyclestmt.v->source.ToString());
-      err = (it != labelNamesandLevels_.end() && it->second > 0);
+      it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second > 0);
+    } else {
+      // If there is no label then the cycle statement is associated with the
+      // closest enclosing DO. Use its level for the checks.
+      err = cycleLevel_ > 0;
     }
-    if (cycleLevel_ > 0 || err) {
+    if (err) {
       context_.Say(*cycleSource_,
           "CYCLE statement to non-innermost associated loop of an OpenMP DO "
           "construct"_err_en_US);
@@ -125,7 +130,7 @@ private:
   SemanticsContext &context_;
   const parser::CharBlock *cycleSource_;
   std::int64_t cycleLevel_;
-  std::map<std::string, std::int64_t> labelNamesandLevels_;
+  std::map<std::string, std::int64_t> constructNamesAndLevels_;
 };
 
 bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
-- 
GitLab


From a62a7024164c2977cd0e77f77807f957802d204a Mon Sep 17 00:00:00 2001
From: jinchen <49575973+jinchen62@users.noreply.github.com>
Date: Tue, 7 May 2024 15:40:37 +0800
Subject: [PATCH 0019/1206] [mlir][math] Add expand patterns for acosh, asinh,
 atanh (#90718)

---
 .../mlir/Dialect/Math/Transforms/Passes.h     |   3 +
 .../Math/Transforms/ExpandPatterns.cpp        |  87 +++++++++++--
 mlir/test/lib/Dialect/Math/TestExpandMath.cpp |   3 +
 .../test-expand-math-approx.mlir              | 119 ++++++++++++++++++
 4 files changed, 200 insertions(+), 12 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
index e2c513047c77..24e6d9a8d98e 100644
--- a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
@@ -31,6 +31,9 @@ void populateExpandTanPattern(RewritePatternSet &patterns);
 void populateExpandSinhPattern(RewritePatternSet &patterns);
 void populateExpandCoshPattern(RewritePatternSet &patterns);
 void populateExpandTanhPattern(RewritePatternSet &patterns);
+void populateExpandAsinhPattern(RewritePatternSet &patterns);
+void populateExpandAcoshPattern(RewritePatternSet &patterns);
+void populateExpandAtanhPattern(RewritePatternSet &patterns);
 void populateExpandFmaFPattern(RewritePatternSet &patterns);
 void populateExpandFloorFPattern(RewritePatternSet &patterns);
 void populateExpandCeilFPattern(RewritePatternSet &patterns);
diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
index 42629e149e9f..5ccf3b6d72a2 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
@@ -73,14 +73,14 @@ static LogicalResult convertSinhOp(math::SinhOp op, PatternRewriter &rewriter) {
   ImplicitLocOpBuilder b(op->getLoc(), rewriter);
   Value operand = op.getOperand();
   Type opType = operand.getType();
-  Value exp = b.create<math::ExpOp>(operand);
 
-  Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
-  Value nexp = b.create<arith::DivFOp>(one, exp);
+  Value exp = b.create<math::ExpOp>(operand);
+  Value neg = b.create<arith::NegFOp>(operand);
+  Value nexp = b.create<math::ExpOp>(neg);
   Value sub = b.create<arith::SubFOp>(exp, nexp);
-  Value two = createFloatConst(op->getLoc(), opType, 2.0, rewriter);
-  Value div = b.create<arith::DivFOp>(sub, two);
-  rewriter.replaceOp(op, div);
+  Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
+  Value res = b.create<arith::MulFOp>(sub, half);
+  rewriter.replaceOp(op, res);
   return success();
 }
 
@@ -89,14 +89,14 @@ static LogicalResult convertCoshOp(math::CoshOp op, PatternRewriter &rewriter) {
   ImplicitLocOpBuilder b(op->getLoc(), rewriter);
   Value operand = op.getOperand();
   Type opType = operand.getType();
-  Value exp = b.create<math::ExpOp>(operand);
 
-  Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
-  Value nexp = b.create<arith::DivFOp>(one, exp);
+  Value exp = b.create<math::ExpOp>(operand);
+  Value neg = b.create<arith::NegFOp>(operand);
+  Value nexp = b.create<math::ExpOp>(neg);
   Value add = b.create<arith::AddFOp>(exp, nexp);
-  Value two = createFloatConst(op->getLoc(), opType, 2.0, rewriter);
-  Value div = b.create<arith::DivFOp>(add, two);
-  rewriter.replaceOp(op, div);
+  Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
+  Value res = b.create<arith::MulFOp>(add, half);
+  rewriter.replaceOp(op, res);
   return success();
 }
 
@@ -152,6 +152,57 @@ static LogicalResult convertTanOp(math::TanOp op, PatternRewriter &rewriter) {
   return success();
 }
 
+// asinh(float x) -> log(x + sqrt(x**2 + 1))
+static LogicalResult convertAsinhOp(math::AsinhOp op,
+                                    PatternRewriter &rewriter) {
+  ImplicitLocOpBuilder b(op->getLoc(), rewriter);
+  Value operand = op.getOperand();
+  Type opType = operand.getType();
+
+  Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
+  Value fma = b.create<math::FmaOp>(operand, operand, one);
+  Value sqrt = b.create<math::SqrtOp>(fma);
+  Value add = b.create<arith::AddFOp>(operand, sqrt);
+  Value res = b.create<math::LogOp>(add);
+  rewriter.replaceOp(op, res);
+  return success();
+}
+
+// acosh(float x) -> log(x + sqrt(x**2 - 1))
+static LogicalResult convertAcoshOp(math::AcoshOp op,
+                                    PatternRewriter &rewriter) {
+  ImplicitLocOpBuilder b(op->getLoc(), rewriter);
+  Value operand = op.getOperand();
+  Type opType = operand.getType();
+
+  Value negOne = createFloatConst(op->getLoc(), opType, -1.0, rewriter);
+  Value fma = b.create<math::FmaOp>(operand, operand, negOne);
+  Value sqrt = b.create<math::SqrtOp>(fma);
+  Value add = b.create<arith::AddFOp>(operand, sqrt);
+  Value res = b.create<math::LogOp>(add);
+  rewriter.replaceOp(op, res);
+  return success();
+}
+
+// atanh(float x) -> log((1 + x) / (1 - x)) / 2
+static LogicalResult convertAtanhOp(math::AtanhOp op,
+                                    PatternRewriter &rewriter) {
+  ImplicitLocOpBuilder b(op->getLoc(), rewriter);
+  Value operand = op.getOperand();
+  Type opType = operand.getType();
+
+  Value one = createFloatConst(op->getLoc(), opType, 1.0, rewriter);
+  Value add = b.create<arith::AddFOp>(operand, one);
+  Value neg = b.create<arith::NegFOp>(operand);
+  Value sub = b.create<arith::AddFOp>(neg, one);
+  Value div = b.create<arith::DivFOp>(add, sub);
+  Value log = b.create<math::LogOp>(div);
+  Value half = createFloatConst(op->getLoc(), opType, 0.5, rewriter);
+  Value res = b.create<arith::MulFOp>(log, half);
+  rewriter.replaceOp(op, res);
+  return success();
+}
+
 static LogicalResult convertFmaFOp(math::FmaOp op, PatternRewriter &rewriter) {
   ImplicitLocOpBuilder b(op->getLoc(), rewriter);
   Value operandA = op.getOperand(0);
@@ -584,6 +635,18 @@ void mlir::populateExpandTanhPattern(RewritePatternSet &patterns) {
   patterns.add(convertTanhOp);
 }
 
+void mlir::populateExpandAsinhPattern(RewritePatternSet &patterns) {
+  patterns.add(convertAsinhOp);
+}
+
+void mlir::populateExpandAcoshPattern(RewritePatternSet &patterns) {
+  patterns.add(convertAcoshOp);
+}
+
+void mlir::populateExpandAtanhPattern(RewritePatternSet &patterns) {
+  patterns.add(convertAtanhOp);
+}
+
 void mlir::populateExpandFmaFPattern(RewritePatternSet &patterns) {
   patterns.add(convertFmaFOp);
 }
diff --git a/mlir/test/lib/Dialect/Math/TestExpandMath.cpp b/mlir/test/lib/Dialect/Math/TestExpandMath.cpp
index 97600ad1ebe7..da48ccb6e5e0 100644
--- a/mlir/test/lib/Dialect/Math/TestExpandMath.cpp
+++ b/mlir/test/lib/Dialect/Math/TestExpandMath.cpp
@@ -42,6 +42,9 @@ void TestExpandMathPass::runOnOperation() {
   populateExpandSinhPattern(patterns);
   populateExpandCoshPattern(patterns);
   populateExpandTanhPattern(patterns);
+  populateExpandAsinhPattern(patterns);
+  populateExpandAcoshPattern(patterns);
+  populateExpandAtanhPattern(patterns);
   populateExpandFmaFPattern(patterns);
   populateExpandFloorFPattern(patterns);
   populateExpandCeilFPattern(patterns);
diff --git a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
index 340ef30bf59c..2b72acde6a3b 100644
--- a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
+++ b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
@@ -717,6 +717,122 @@ func.func @tanh() {
  return
 }
 
+// -------------------------------------------------------------------------- //
+// Asinh.
+// -------------------------------------------------------------------------- //
+
+func.func @asinh_f32(%a : f32) {
+  %r = math.asinh %a : f32
+  vector.print %r : f32
+  return
+}
+
+func.func @asinh_3xf32(%a : vector<3xf32>) {
+  %r = math.asinh %a : vector<3xf32>
+  vector.print %r : vector<3xf32>
+  return
+}
+
+func.func @asinh() {
+  // CHECK: 0
+  %zero = arith.constant 0.0 : f32
+  call @asinh_f32(%zero) : (f32) -> ()
+
+  // CHECK: 0.881374
+  %cst1 = arith.constant 1.0 : f32
+  call @asinh_f32(%cst1) : (f32) -> ()
+
+  // CHECK: -0.881374
+  %cst2 = arith.constant -1.0 : f32
+  call @asinh_f32(%cst2) : (f32) -> ()
+
+  // CHECK: 1.81845
+  %cst3 = arith.constant 3.0 : f32
+  call @asinh_f32(%cst3) : (f32) -> ()
+
+  // CHECK: 0.247466, 0.790169, 1.44364
+  %vec_x = arith.constant dense<[0.25, 0.875, 2.0]> : vector<3xf32>
+  call @asinh_3xf32(%vec_x) : (vector<3xf32>) -> ()
+
+  return
+}
+
+// -------------------------------------------------------------------------- //
+// Acosh.
+// -------------------------------------------------------------------------- //
+
+func.func @acosh_f32(%a : f32) {
+  %r = math.acosh %a : f32
+  vector.print %r : f32
+  return
+}
+
+func.func @acosh_3xf32(%a : vector<3xf32>) {
+  %r = math.acosh %a : vector<3xf32>
+  vector.print %r : vector<3xf32>
+  return
+}
+
+func.func @acosh() {
+  // CHECK: 0
+  %zero = arith.constant 1.0 : f32
+  call @acosh_f32(%zero) : (f32) -> ()
+
+  // CHECK: 1.31696
+  %cst1 = arith.constant 2.0 : f32
+  call @acosh_f32(%cst1) : (f32) -> ()
+
+  // CHECK: 2.99322
+  %cst2 = arith.constant 10.0 : f32
+  call @acosh_f32(%cst2) : (f32) -> ()
+
+  // CHECK: 0.962424, 1.76275, 2.47789
+  %vec_x = arith.constant dense<[1.5, 3.0, 6.0]> : vector<3xf32>
+  call @acosh_3xf32(%vec_x) : (vector<3xf32>) -> ()
+
+  return
+}
+
+// -------------------------------------------------------------------------- //
+// Atanh.
+// -------------------------------------------------------------------------- //
+
+func.func @atanh_f32(%a : f32) {
+  %r = math.atanh %a : f32
+  vector.print %r : f32
+  return
+}
+
+func.func @atanh_3xf32(%a : vector<3xf32>) {
+  %r = math.atanh %a : vector<3xf32>
+  vector.print %r : vector<3xf32>
+  return
+}
+
+func.func @atanh() {
+  // CHECK: 0
+  %zero = arith.constant 0.0 : f32
+  call @atanh_f32(%zero) : (f32) -> ()
+
+  // CHECK: 0.549306
+  %cst1 = arith.constant 0.5 : f32
+  call @atanh_f32(%cst1) : (f32) -> ()
+
+  // CHECK: -0.549306
+  %cst2 = arith.constant -0.5 : f32
+  call @atanh_f32(%cst2) : (f32) -> ()
+
+  // CHECK: inf
+  %cst3 = arith.constant 1.0 : f32
+  call @atanh_f32(%cst3) : (f32) -> ()
+
+  // CHECK: 0.255413, 0.394229, 2.99448
+  %vec_x = arith.constant dense<[0.25, 0.375, 0.995]> : vector<3xf32>
+  call @atanh_3xf32(%vec_x) : (vector<3xf32>) -> ()
+
+  return
+}
+
 func.func @main() {
   call @exp2f() : () -> ()
   call @roundf() : () -> ()
@@ -725,5 +841,8 @@ func.func @main() {
   call @sinh() : () -> ()
   call @cosh() : () -> ()
   call @tanh() : () -> ()
+  call @asinh() : () -> ()
+  call @acosh() : () -> ()
+  call @atanh() : () -> ()
   return
 }
-- 
GitLab


From ebde770c3e6f0dd9d297659cbaeb486cef9471d6 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Tue, 7 May 2024 15:40:49 +0800
Subject: [PATCH 0020/1206] [RISCV] Use IMPLICIT_DEF for undef GPR reg in
 vsetvli test. NFC

Only VRs should use $noreg, this GPR was accidentally changed in d392520c6
---
 llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
index c66eb5717048..ef834403fb4f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
@@ -499,7 +499,7 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY undef $noreg
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gpr = IMPLICIT_DEF
   ; CHECK-NEXT:   dead [[PseudoVSETVLIX0_:%[0-9]+]]:gpr = PseudoVSETVLIX0 killed $x0, 223 /* e64, mf2, ta, ma */, implicit-def $vl, implicit-def $vtype
   ; CHECK-NEXT:   [[PseudoVID_V_MF2_:%[0-9]+]]:vr = PseudoVID_V_MF2 undef $noreg, -1, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   dead [[PseudoVSETVLIX0_1:%[0-9]+]]:gpr = PseudoVSETVLIX0 killed $x0, 215 /* e32, mf2, ta, ma */, implicit-def $vl, implicit-def $vtype
@@ -514,8 +514,8 @@ body:             |
   ; CHECK-NEXT:   [[PseudoVLE32_V_MF2_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_MF2_MASK [[PseudoVMV_V_I_MF2_]], killed [[COPY]], $v0, -1, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 197 /* e8, mf8, ta, ma */, implicit-def $vl, implicit-def $vtype, implicit $vl
   ; CHECK-NEXT:   [[PseudoVCPOP_M_B1_:%[0-9]+]]:gpr = PseudoVCPOP_M_B1 [[PseudoVMSEQ_VI_MF2_]], -1, 0 /* e8 */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr = COPY $x0
-  ; CHECK-NEXT:   BEQ killed [[PseudoVCPOP_M_B1_]], [[COPY3]], %bb.3
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x0
+  ; CHECK-NEXT:   BEQ killed [[PseudoVCPOP_M_B1_]], [[COPY2]], %bb.3
   ; CHECK-NEXT:   PseudoBR %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
@@ -524,7 +524,7 @@ body:             |
   ; CHECK-NEXT:   [[LWU:%[0-9]+]]:gpr = LWU [[COPY1]], 0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3:
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gpr = PHI [[COPY2]], %bb.1, [[LWU]], %bb.2
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gpr = PHI [[DEF]], %bb.1, [[LWU]], %bb.2
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 215 /* e32, mf2, ta, ma */, implicit-def $vl, implicit-def $vtype, implicit $vl
   ; CHECK-NEXT:   [[PseudoVADD_VX_MF2_:%[0-9]+]]:vr = nsw PseudoVADD_VX_MF2 undef $noreg, [[PseudoVLE32_V_MF2_MASK]], [[PHI]], -1, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   $v0 = COPY [[PseudoVADD_VX_MF2_]]
@@ -535,7 +535,7 @@ body:             |
 
     %0:gpr = COPY $x11
     %1:gpr = COPY $x10
-    %2:gpr = COPY undef $noreg
+    %2:gpr = IMPLICIT_DEF
     %3:vr = PseudoVID_V_MF2 undef $noreg, -1, 6, 0
     %4:vrnov0 = PseudoVMV_V_I_MF2 undef $noreg, 0, -1, 5, 0
 
-- 
GitLab


From ad59967336d2279eee77fff3a92e52ec87010aae Mon Sep 17 00:00:00 2001
From: hev <wangrui@loongson.cn>
Date: Tue, 7 May 2024 15:46:11 +0800
Subject: [PATCH 0021/1206] [LoongArch] Optimize codegen for ISD::{ROTL,ROTR}
 (#91174)

---
 .../LoongArch/LoongArchISelLowering.cpp       | 17 ++++-----
 .../Target/LoongArch/LoongArchInstrInfo.td    |  7 ++--
 llvm/test/CodeGen/LoongArch/rotl-rotr.ll      | 35 +++++++------------
 3 files changed, 24 insertions(+), 35 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 46a6703f29d5..21d520656091 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -1671,10 +1671,9 @@ static LoongArchISD::NodeType getLoongArchWOpcode(unsigned Opcode) {
     return LoongArchISD::SRA_W;
   case ISD::SRL:
     return LoongArchISD::SRL_W;
+  case ISD::ROTL:
   case ISD::ROTR:
     return LoongArchISD::ROTR_W;
-  case ISD::ROTL:
-    return LoongArchISD::ROTL_W;
   case ISD::CTTZ:
     return LoongArchISD::CTZ_W;
   case ISD::CTLZ:
@@ -1704,6 +1703,10 @@ static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp,
   case 2: {
     NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
     SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
+    if (N->getOpcode() == ISD::ROTL) {
+      SDValue TmpOp = DAG.getConstant(32, DL, MVT::i64);
+      NewOp1 = DAG.getNode(ISD::SUB, DL, MVT::i64, TmpOp, NewOp1);
+    }
     NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
     break;
   }
@@ -1841,7 +1844,6 @@ void LoongArchTargetLowering::ReplaceNodeResults(
   case ISD::SHL:
   case ISD::SRA:
   case ISD::SRL:
-  case ISD::ROTR:
     assert(VT == MVT::i32 && Subtarget.is64Bit() &&
            "Unexpected custom legalisation");
     if (N->getOperand(1).getOpcode() != ISD::Constant) {
@@ -1850,11 +1852,10 @@ void LoongArchTargetLowering::ReplaceNodeResults(
     }
     break;
   case ISD::ROTL:
-    ConstantSDNode *CN;
-    if ((CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))) {
-      Results.push_back(customLegalizeToWOp(N, DAG, 2));
-      break;
-    }
+  case ISD::ROTR:
+    assert(VT == MVT::i32 && Subtarget.is64Bit() &&
+           "Unexpected custom legalisation");
+    Results.push_back(customLegalizeToWOp(N, DAG, 2));
     break;
   case ISD::FP_TO_SINT: {
     assert(VT == MVT::i32 && Subtarget.is64Bit() &&
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index a7f6eb9a79eb..f56f8f7e1179 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -85,7 +85,6 @@ def loongarch_sll_w : SDNode<"LoongArchISD::SLL_W", SDT_LoongArchIntBinOpW>;
 def loongarch_sra_w : SDNode<"LoongArchISD::SRA_W", SDT_LoongArchIntBinOpW>;
 def loongarch_srl_w : SDNode<"LoongArchISD::SRL_W", SDT_LoongArchIntBinOpW>;
 def loongarch_rotr_w : SDNode<"LoongArchISD::ROTR_W", SDT_LoongArchIntBinOpW>;
-def loongarch_rotl_w : SDNode<"LoongArchISD::ROTL_W", SDT_LoongArchIntBinOpW>;
 def loongarch_crc_w_b_w
     : SDNode<"LoongArchISD::CRC_W_B_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
 def loongarch_crc_w_h_w
@@ -1116,12 +1115,10 @@ def : PatGprGpr<srem, MOD_D>;
 def : PatGprGpr<urem, MOD_DU>;
 def : PatGprGpr<rotr, ROTR_D>;
 def : PatGprGpr<loongarch_rotr_w, ROTR_W>;
+def : PatGprGpr_32<rotr, ROTR_W>;
 def : PatGprImm<rotr, ROTRI_D, uimm6>;
 def : PatGprImm_32<rotr, ROTRI_W, uimm5>;
-def : Pat<(loongarch_rotl_w GPR:$rj, uimm5:$imm),
-          (ROTRI_W GPR:$rj, (ImmSubFrom32 uimm5:$imm))>;
-def : Pat<(sext_inreg (loongarch_rotl_w GPR:$rj, uimm5:$imm), i32),
-          (ROTRI_W GPR:$rj, (ImmSubFrom32 uimm5:$imm))>;
+def : PatGprImm<loongarch_rotr_w, ROTRI_W, uimm5>;
 // TODO: Select "_W[U]" instructions for i32xi32 if only lower 32 bits of the
 // product are used.
 def : PatGprGpr<mul, MUL_D>;
diff --git a/llvm/test/CodeGen/LoongArch/rotl-rotr.ll b/llvm/test/CodeGen/LoongArch/rotl-rotr.ll
index 8646771e5d48..b9fbd962e6bb 100644
--- a/llvm/test/CodeGen/LoongArch/rotl-rotr.ll
+++ b/llvm/test/CodeGen/LoongArch/rotl-rotr.ll
@@ -2,8 +2,6 @@
 ; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
 ; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
 
-;; TODO: Add optimization to ISD::ROTL
-
 define signext i32 @rotl_32(i32 signext %x, i32 signext %y) nounwind {
 ; LA32-LABEL: rotl_32:
 ; LA32:       # %bb.0:
@@ -14,10 +12,9 @@ define signext i32 @rotl_32(i32 signext %x, i32 signext %y) nounwind {
 ;
 ; LA64-LABEL: rotl_32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    sll.w $a2, $a0, $a1
-; LA64-NEXT:    sub.d $a1, $zero, $a1
-; LA64-NEXT:    srl.w $a0, $a0, $a1
-; LA64-NEXT:    or $a0, $a2, $a0
+; LA64-NEXT:    ori $a2, $zero, 32
+; LA64-NEXT:    sub.d $a1, $a2, $a1
+; LA64-NEXT:    rotr.w $a0, $a0, $a1
 ; LA64-NEXT:    ret
   %z = sub i32 32, %y
   %b = shl i32 %x, %y
@@ -152,10 +149,9 @@ define signext i32 @rotl_32_mask(i32 signext %x, i32 signext %y) nounwind {
 ;
 ; LA64-LABEL: rotl_32_mask:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    sll.w $a2, $a0, $a1
-; LA64-NEXT:    sub.d $a1, $zero, $a1
-; LA64-NEXT:    srl.w $a0, $a0, $a1
-; LA64-NEXT:    or $a0, $a2, $a0
+; LA64-NEXT:    ori $a2, $zero, 32
+; LA64-NEXT:    sub.d $a1, $a2, $a1
+; LA64-NEXT:    rotr.w $a0, $a0, $a1
 ; LA64-NEXT:    ret
   %z = sub i32 0, %y
   %and = and i32 %z, 31
@@ -174,10 +170,9 @@ define signext i32 @rotl_32_mask_and_63_and_31(i32 signext %x, i32 signext %y) n
 ;
 ; LA64-LABEL: rotl_32_mask_and_63_and_31:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    sll.w $a2, $a0, $a1
-; LA64-NEXT:    sub.d $a1, $zero, $a1
-; LA64-NEXT:    srl.w $a0, $a0, $a1
-; LA64-NEXT:    or $a0, $a2, $a0
+; LA64-NEXT:    ori $a2, $zero, 32
+; LA64-NEXT:    sub.d $a1, $a2, $a1
+; LA64-NEXT:    rotr.w $a0, $a0, $a1
 ; LA64-NEXT:    ret
   %a = and i32 %y, 63
   %b = shl i32 %x, %a
@@ -197,10 +192,9 @@ define signext i32 @rotl_32_mask_or_64_or_32(i32 signext %x, i32 signext %y) nou
 ;
 ; LA64-LABEL: rotl_32_mask_or_64_or_32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    sll.w $a2, $a0, $a1
-; LA64-NEXT:    sub.d $a1, $zero, $a1
-; LA64-NEXT:    srl.w $a0, $a0, $a1
-; LA64-NEXT:    or $a0, $a2, $a0
+; LA64-NEXT:    ori $a2, $zero, 32
+; LA64-NEXT:    sub.d $a1, $a2, $a1
+; LA64-NEXT:    rotr.w $a0, $a0, $a1
 ; LA64-NEXT:    ret
   %a = or i32 %y, 64
   %b = shl i32 %x, %a
@@ -591,10 +585,7 @@ define signext i32 @rotr_i32_fshr(i32 signext %a) nounwind {
 ;
 ; LA64-LABEL: rotr_i32_fshr:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    slli.d $a1, $a0, 20
-; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 12
-; LA64-NEXT:    or $a0, $a0, $a1
-; LA64-NEXT:    addi.w $a0, $a0, 0
+; LA64-NEXT:    rotri.w $a0, $a0, 12
 ; LA64-NEXT:    ret
   %or = tail call i32 @llvm.fshr.i32(i32 %a, i32 %a, i32 12)
   ret i32 %or
-- 
GitLab


From d9f2b9391887af95acdd91dfea2e72eb3a9d8d05 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Tue, 7 May 2024 16:00:53 +0800
Subject: [PATCH 0022/1206] [RISCV] Change more undef passthrus to $noreg in
 vector tests. NFC

---
 llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir |  3 +--
 llvm/test/CodeGen/RISCV/rvv/copyprop.mir             | 10 +++-------
 llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir       |  9 +++------
 llvm/test/CodeGen/RISCV/rvv/vxrm.mir                 |  3 +--
 4 files changed, 8 insertions(+), 17 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir b/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir
index a54da97d2548..f976adcfe931 100644
--- a/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/addi-scalable-offset.mir
@@ -57,8 +57,7 @@ body: |
     ; CHECK-NEXT: PseudoRET
     %1:gprnox0 = COPY $x11
     %0:gpr = COPY $x10
-    %pt:vr = IMPLICIT_DEF
-    %2:vr = PseudoVLE64_V_M1 %pt, %0, %1, 6, 0 :: (load unknown-size from %ir.pa, align 8)
+    %2:vr = PseudoVLE64_V_M1 undef $noreg, %0, %1, 6, 0 :: (load unknown-size from %ir.pa, align 8)
     %3:gpr = ADDI %stack.2, 0
     VS1R_V killed %2:vr, %3:gpr
     PseudoRET
diff --git a/llvm/test/CodeGen/RISCV/rvv/copyprop.mir b/llvm/test/CodeGen/RISCV/rvv/copyprop.mir
index 95c227518f5c..1718dc90eed4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/copyprop.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/copyprop.mir
@@ -43,16 +43,12 @@ body:             |
     %2:gpr = COPY $x11
     %1:gpr = COPY $x10
     %3:vr = COPY $v8
-    %pt5:vr = IMPLICIT_DEF
-    %17:vr = PseudoVSLL_VI_M1 %pt5, %3, 5, 1, 6 /* e64 */, 0
+    %17:vr = PseudoVSLL_VI_M1 undef $noreg, %3, 5, 1, 6 /* e64 */, 0
     %22:vr = PseudoVMSNE_VI_M1 %3, 0, 1, 6 /* e64 */
     $v0 = COPY %22
-    %26:vrnov0 = IMPLICIT_DEF
-    %25:vrnov0 = PseudoVMERGE_VIM_M1 %26, %17, -1, $v0, 1, 6 /* e64 */
-    %pt8:vr = IMPLICIT_DEF
+    %25:vrnov0 = PseudoVMERGE_VIM_M1 undef $noreg, %17, -1, $v0, 1, 6 /* e64 */
     %29:vr = PseudoVC_V_X_SE_M1 3, 31, %2, 1, 6 /* e64 */, implicit-def dead $vcix_state, implicit $vcix_state
-    %pt9:vr = IMPLICIT_DEF
-    %30:vr = PseudoVMV_V_I_M1 %pt9, 0, 1, 6 /* e64 */, 0
+    %30:vr = PseudoVMV_V_I_M1 undef $noreg, 0, 1, 6 /* e64 */, 0
     BGEU %1, $x0, %bb.2
 
   bb.1.entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir b/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir
index 0e207731e020..b891207341b3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir
@@ -27,13 +27,10 @@ body:             |
     %2:vr = COPY $v2
     %3:vr = COPY $v3
     %4:vmv0 = COPY %0
-    %pt1:vrnov0 = IMPLICIT_DEF
-    %5:vrnov0 = PseudoVMERGE_VIM_M1 %pt1, killed %2, 1, %4, 1, 3
+    %5:vrnov0 = PseudoVMERGE_VIM_M1 undef $noreg, killed %2, 1, %4, 1, 3
     %6:vmv0 = COPY %1
-    %pt2:vrnov0 = IMPLICIT_DEF
-    %7:vrnov0 = PseudoVMERGE_VIM_M1 %pt2, killed %3, 1, %6, 1, 3
-    %pt:vr = IMPLICIT_DEF
-    %8:vr = PseudoVADD_VV_M1 %pt, killed %5, killed %7, 1, 3, 0
+    %7:vrnov0 = PseudoVMERGE_VIM_M1 undef $noreg, killed %3, 1, %6, 1, 3
+    %8:vr = PseudoVADD_VV_M1 undef $noreg, killed %5, killed %7, 1, 3, 0
     $v0 = COPY %8
     PseudoRET implicit $v0
 ...
diff --git a/llvm/test/CodeGen/RISCV/rvv/vxrm.mir b/llvm/test/CodeGen/RISCV/rvv/vxrm.mir
index a588677bec8e..eac3cfca209e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vxrm.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vxrm.mir
@@ -24,7 +24,6 @@ body:     |
     %0:vr = COPY $v8
     %1:vr = COPY $v9
     %2:gprnox0 = COPY $x10
-    %pt:vr = IMPLICIT_DEF
-    renamable $v8 = PseudoVAADD_VV_MF8 %pt, %0, %1, 0, %2, 3 /* e8 */, 0
+    renamable $v8 = PseudoVAADD_VV_MF8 undef $noreg, %0, %1, 0, %2, 3 /* e8 */, 0
     PseudoRET implicit $v8
 ...
-- 
GitLab


From f3fbd21fa4e25496725c22d987e4e47e4c39c8b0 Mon Sep 17 00:00:00 2001
From: martinboehme <mboehme@google.com>
Date: Tue, 7 May 2024 10:12:23 +0200
Subject: [PATCH 0023/1206] [clang][dataflow] Strengthen pointer comparison.
 (#75170)

-  Instead of comparing the identity of the `PointerValue`s, compare the
   underlying `StorageLocation`s.

- If the `StorageLocation`s are the same, return a definite "true" as
the
result of the comparison. Before, if the `PointerValue`s were different,
we
would return an atom, even if the storage locations themselves were the
same.

- If the `StorageLocation`s are different, return an atom (as before).
Pointers
that have different storage locations may still alias, so we can't
return a
   definite "false" in this case.

The application-level gains from this are relatively modest. For the
Crubit
nullability check running on an internal codebase, this change reduces
the
number of functions on which the SAT solver times out from 223 to 221;
the
number of "pointer expression not modeled" errors reduces from 3815 to
3778.

Still, it seems that the gain in precision is generally worthwhile.

@Xazax-hun inspired me to think about this with his

[comments](https://github.com/llvm/llvm-project/pull/73860#pullrequestreview-1761484615)
on a different PR.
---
 clang/lib/Analysis/FlowSensitive/Transfer.cpp |  8 ++
 .../Analysis/FlowSensitive/TransferTest.cpp   | 88 +++++++++++++++++++
 2 files changed, 96 insertions(+)

diff --git a/clang/lib/Analysis/FlowSensitive/Transfer.cpp b/clang/lib/Analysis/FlowSensitive/Transfer.cpp
index 5a57f11b000d..4214488c98e5 100644
--- a/clang/lib/Analysis/FlowSensitive/Transfer.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Transfer.cpp
@@ -68,6 +68,14 @@ static BoolValue &evaluateBooleanEquality(const Expr &LHS, const Expr &RHS,
     if (auto *RHSBool = dyn_cast_or_null<BoolValue>(RHSValue))
       return Env.makeIff(*LHSBool, *RHSBool);
 
+  if (auto *LHSPtr = dyn_cast_or_null<PointerValue>(LHSValue))
+    if (auto *RHSPtr = dyn_cast_or_null<PointerValue>(RHSValue))
+      // If the storage locations are the same, the pointers definitely compare
+      // the same. If the storage locations are different, they may still alias,
+      // so we fall through to the case below that returns an atom.
+      if (&LHSPtr->getPointeeLoc() == &RHSPtr->getPointeeLoc())
+        return Env.getBoolLiteralValue(true);
+
   return Env.makeAtomicBoolValue();
 }
 
diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
index 6743e778a2ff..e1fb16b64fd6 100644
--- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
@@ -4709,6 +4709,94 @@ TEST(TransferTest, BooleanInequality) {
       });
 }
 
+TEST(TransferTest, PointerEquality) {
+  std::string Code = R"cc(
+    void target() {
+      int i = 0;
+      int i_other = 0;
+      int *p1 = &i;
+      int *p2 = &i;
+      int *p_other = &i_other;
+      int *null = nullptr;
+
+      bool p1_eq_p1 = (p1 == p1);
+      bool p1_eq_p2 = (p1 == p2);
+      bool p1_eq_p_other = (p1 == p_other);
+
+      bool p1_eq_null = (p1 == null);
+      bool p1_eq_nullptr = (p1 == nullptr);
+      bool null_eq_nullptr = (null == nullptr);
+      bool nullptr_eq_nullptr = (nullptr == nullptr);
+
+      // We won't duplicate all of the tests above with `!=`, as we know that
+      // the implementation simply negates the result of the `==` comparison.
+      // Instaed, just spot-check one case.
+      bool p1_ne_p1 = (p1 != p1);
+
+      (void)0; // [[p]]
+    }
+  )cc";
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        // Check the we have indeed set things up so that `p1` and `p2` have
+        // different pointer values.
+        EXPECT_NE(&getValueForDecl<PointerValue>(ASTCtx, Env, "p1"),
+                  &getValueForDecl<PointerValue>(ASTCtx, Env, "p2"));
+
+        EXPECT_EQ(&getValueForDecl<BoolValue>(ASTCtx, Env, "p1_eq_p1"),
+                  &Env.getBoolLiteralValue(true));
+        EXPECT_EQ(&getValueForDecl<BoolValue>(ASTCtx, Env, "p1_eq_p2"),
+                  &Env.getBoolLiteralValue(true));
+        EXPECT_TRUE(isa<AtomicBoolValue>(
+            getValueForDecl<BoolValue>(ASTCtx, Env, "p1_eq_p_other")));
+
+        EXPECT_TRUE(isa<AtomicBoolValue>(
+            getValueForDecl<BoolValue>(ASTCtx, Env, "p1_eq_null")));
+        EXPECT_TRUE(isa<AtomicBoolValue>(
+            getValueForDecl<BoolValue>(ASTCtx, Env, "p1_eq_nullptr")));
+        EXPECT_EQ(&getValueForDecl<BoolValue>(ASTCtx, Env, "null_eq_nullptr"),
+                  &Env.getBoolLiteralValue(true));
+        EXPECT_EQ(
+            &getValueForDecl<BoolValue>(ASTCtx, Env, "nullptr_eq_nullptr"),
+            &Env.getBoolLiteralValue(true));
+
+        EXPECT_EQ(&getValueForDecl<BoolValue>(ASTCtx, Env, "p1_ne_p1"),
+                  &Env.getBoolLiteralValue(false));
+      });
+}
+
+TEST(TransferTest, PointerEqualityUnionMembers) {
+  std::string Code = R"cc(
+    union U {
+      int i1;
+      int i2;
+    };
+    void target() {
+      U u;
+      bool i1_eq_i2 = (&u.i1 == &u.i2);
+
+      (void)0; // [[p]]
+    }
+  )cc";
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        // FIXME: By the standard, `u.i1` and `u.i2` should have the same
+        // address, but we don't yet model this property of union members
+        // correctly. The result is therefore weaker than it could be (just an
+        // atom rather than a true literal), though not wrong.
+        EXPECT_TRUE(isa<AtomicBoolValue>(
+            getValueForDecl<BoolValue>(ASTCtx, Env, "i1_eq_i2")));
+      });
+}
+
 TEST(TransferTest, IntegerLiteralEquality) {
   std::string Code = R"(
     void target() {
-- 
GitLab


From 1de0535e84f03941badc8021bbc87a8c674a379f Mon Sep 17 00:00:00 2001
From: Peter Waller <peter.waller@arm.com>
Date: Tue, 7 May 2024 09:13:44 +0100
Subject: [PATCH 0024/1206] [llvm-mca] Abort on parse error without
 -skip-unsupported-instructions (#90474)

[llvm-mca] Abort on parse error without -skip-unsupported-instructions

Prior to this patch, llvm-mca would continue executing after parse
errors. These errors can lead to some confusion since some analysis
results are printed on the standard output, and they're printed after
the errors, which could otherwise be easy to miss.

However it is still useful to be able to continue analysis after errors;
so extend the recently added -skip-unsupported-instructions to support
this.

Two tests which have parse errors for some of the 'RUN' branches are
updated to use -skip-unsupported-instructions so they can remain as-is.

Add a description of -skip-unsupported-instructions to the llvm-mca
command guide, and add it to the llvm-mca --help output:

```
  --skip-unsupported-instructions=<value> - Force analysis to continue in the presence of unsupported instructions
    =none                                 -   Exit with an error when an instruction is unsupported for any reason (default)
    =lack-sched                           -   Skip instructions on input which lack scheduling information
    =parse-failure                        -   Skip lines on the input which fail to parse for any reason
    =any                                  -   Skip instructions or lines on input which are unsupported for any reason
```

Tests within this patch are intended to cover each of the cases.

Reason        | Flag | Comment
--------------|------|-------
none          | none | Usual case, existing test suite
lack-sched    | none | Advises user to use -skip-unsupported-instructions=lack-sched, tested in llvm/test/tools/llvm-mca/X86/BtVer2/unsupported-instruction.s
parse-failure | none | Advises user to use -skip-unsupported-instructions=parse-failure, tested in llvm/test/tools/llvm-mca/bad-input.s
any           | none | (N/A, covered above)
lack-sched    | any  | Continues, prints warnings, tested in llvm/test/tools/llvm-mca/X86/BtVer2/unsupported-instruction.s
parse-failure | any  | Continues, prints errors, tested in llvm/test/tools/llvm-mca/bad-input.s
lack-sched    | parse-failure | Advises user to use -skip-unsupported-instructions=lack-sched, tested in llvm/test/tools/llvm-mca/X86/BtVer2/unsupported-instruction.s
parse-failure | lack-sched    | Advises user to use -skip-unsupported-instructions=parse-failure, tested in llvm/test/tools/llvm-mca/bad-input.s
none          | * | This would be any test case with skip-unsupported-instructions, coverage added in llvm/test/tools/llvm-mca/X86/BtVer2/simple-test.s
any           | * | (Logically covered by the other cases)
---
 llvm/docs/CommandGuide/llvm-mca.rst           | 10 ++++
 llvm/docs/ReleaseNotes.rst                    |  8 ++++
 .../AArch64/Exynos/float-divide-multiply.s    |  2 +-
 .../llvm-mca/AArch64/Exynos/float-integer.s   |  2 +-
 .../ARM/cortex-a57-basic-instructions.s       |  1 -
 .../tools/llvm-mca/ARM/cortex-a57-thumb.s     | 27 +++++++++--
 .../tools/llvm-mca/X86/BtVer2/simple-test.s   |  3 ++
 ...kip-unsupported-instructions-none-remain.s |  4 +-
 .../X86/BtVer2/unsupported-instruction.s      |  9 ++--
 llvm/test/tools/llvm-mca/bad-input.s          | 14 ++++++
 llvm/tools/llvm-mca/CodeRegionGenerator.cpp   | 13 ++++-
 llvm/tools/llvm-mca/CodeRegionGenerator.h     | 34 +++++++++-----
 llvm/tools/llvm-mca/llvm-mca.cpp              | 47 +++++++++++++++----
 13 files changed, 138 insertions(+), 36 deletions(-)
 create mode 100644 llvm/test/tools/llvm-mca/bad-input.s

diff --git a/llvm/docs/CommandGuide/llvm-mca.rst b/llvm/docs/CommandGuide/llvm-mca.rst
index eae5e1406b89..f610ea2f2168 100644
--- a/llvm/docs/CommandGuide/llvm-mca.rst
+++ b/llvm/docs/CommandGuide/llvm-mca.rst
@@ -234,6 +234,16 @@ option specifies "``-``", then the output will also be sent to standard output.
   no extra information, and InstrumentManager never overrides the default
   schedule class for a given instruction.
 
+.. option:: -skip-unsupported-instructions=<reason>
+
+  Force :program:`llvm-mca` to continue in the presence of instructions which do
+  not parse or lack key scheduling information. Note that the resulting analysis
+  is impacted since those unsupported instructions are ignored as-if they are
+  not supplied as a part of the input.
+
+  The choice of `<reason>` controls the when mca will report an error.
+  `<reason>` may be `none` (default), `lack-sched`, `parse-failure`, `any`.
+
 EXIT STATUS
 -----------
 
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 0f4e2759de08..3cf65aac1cc1 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -223,6 +223,14 @@ Changes to the LLVM tools
   (`#89162 <https://github.com/llvm/llvm-project/pull/89162>`_)
   ``--raw-relr`` has been removed.
 
+* llvm-mca now aborts by default if it is given bad input where previously it
+  would continue. Additionally, it can now continue when it encounters
+  instructions which lack scheduling information. The behaviour can be
+  controlled by the newly introduced
+  `--skip-unsupported-instructions=<none|lack-sched|parse-failure|any>`, as
+  documented in `--help` output and the command guide. (`#90474
+  <https://github.com/llvm/llvm-project/pull/90474>`)
+
 Changes to LLDB
 ---------------------------------
 
diff --git a/llvm/test/tools/llvm-mca/AArch64/Exynos/float-divide-multiply.s b/llvm/test/tools/llvm-mca/AArch64/Exynos/float-divide-multiply.s
index ecfd019452af..271bd836eb24 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Exynos/float-divide-multiply.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Exynos/float-divide-multiply.s
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
-# RUN: llvm-mca -march=aarch64 -mcpu=exynos-m3 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM3
+# RUN: llvm-mca -march=aarch64 -mcpu=exynos-m3 -resource-pressure=false -skip-unsupported-instructions=parse-failure < %s | FileCheck %s -check-prefixes=ALL,EM3
 # RUN: llvm-mca -march=aarch64 -mcpu=exynos-m4 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM4
 # RUN: llvm-mca -march=aarch64 -mcpu=exynos-m5 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM5
 
diff --git a/llvm/test/tools/llvm-mca/AArch64/Exynos/float-integer.s b/llvm/test/tools/llvm-mca/AArch64/Exynos/float-integer.s
index 16c710553f75..f95e530a41fe 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Exynos/float-integer.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Exynos/float-integer.s
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
-# RUN: llvm-mca -mtriple=aarch64-linux-gnu -mcpu=exynos-m3 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM3
+# RUN: llvm-mca -mtriple=aarch64-linux-gnu -mcpu=exynos-m3 -resource-pressure=false -skip-unsupported-instructions=parse-failure < %s | FileCheck %s -check-prefixes=ALL,EM3
 # RUN: llvm-mca -mtriple=aarch64-linux-gnu -mcpu=exynos-m4 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM4
 # RUN: llvm-mca -mtriple=aarch64-linux-gnu -mcpu=exynos-m5 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM5
 
diff --git a/llvm/test/tools/llvm-mca/ARM/cortex-a57-basic-instructions.s b/llvm/test/tools/llvm-mca/ARM/cortex-a57-basic-instructions.s
index d686293c9b43..9c2ae8fb2aa5 100644
--- a/llvm/test/tools/llvm-mca/ARM/cortex-a57-basic-instructions.s
+++ b/llvm/test/tools/llvm-mca/ARM/cortex-a57-basic-instructions.s
@@ -33,7 +33,6 @@
   adc	pc, r5, r6, ror #2
   adc	r4, r5, r6, ror #31
   adc	r6, r7, r8, lsl r9
-  adc	pc, r7, r8, lsl r9
   adc	r6, r7, r8, lsr r9
   adc	r6, r7, r8, asr r9
   adc	r6, r7, r8, ror r9
diff --git a/llvm/test/tools/llvm-mca/ARM/cortex-a57-thumb.s b/llvm/test/tools/llvm-mca/ARM/cortex-a57-thumb.s
index 21accd7e2e18..6c56e1dbf024 100644
--- a/llvm/test/tools/llvm-mca/ARM/cortex-a57-thumb.s
+++ b/llvm/test/tools/llvm-mca/ARM/cortex-a57-thumb.s
@@ -95,12 +95,13 @@
   itett	ne
   cmpne	r7, #243
   addeq	r7, r1, r2
+  addne	r7, r1, r2
+  uxthne	r7, r7
   itttt	lt
   cmplt	r7, #243
   uxthlt	r7, r1
   strhlt	r2, [r7, #22]
   lsrlt	r1, r6, #3
-  uxthne	r7, r7
   strh	r2, [r7, #22]
   asrs	r1, r6, #7
   lsrs	r1, r6, #31
@@ -253,7 +254,7 @@
   ldrd	r0, r1, [r2, #-0]!
   ldrd	r0, r1, [r2, #0]!
   ldrd	r0, r1, [r2, #-0]
-  ldrd	r1, r1, [r0], #0
+  ldrd	r1, r2, [r0], #0
   ldrex	r1, [r4]
   ldrex	r8, [r4]
   ldrex	r2, [sp, #128]
@@ -648,7 +649,7 @@
   str	r10, [r11], #0
   strd	r1, r1, [r0], #0
   strd	r6, r3, [r5], #-8
-  strd	r8, r5, [r5], #-0
+  strd	r8, r5, [r6], #-0
   strd	r7, r4, [r5], #-4
   strd	r0, r1, [r2, #-0]!
   strd	r0, r1, [r2, #0]!
@@ -1010,6 +1011,13 @@
 # CHECK-NEXT:  0      0     0.00                  U     itett	ne
 # CHECK-NEXT:  1      1     0.50                        cmpne	r7, #243
 # CHECK-NEXT:  1      1     0.50                        addeq	r7, r1, r2
+# CHECK-NEXT:  1      1     0.50                        addne	r7, r1, r2
+# CHECK-NEXT:  1      1     0.50                        uxthne	r7, r7
+# CHECK-NEXT:  0      0     0.00                  U     itttt	lt
+# CHECK-NEXT:  1      1     0.50                        cmplt	r7, #243
+# CHECK-NEXT:  1      1     0.50                        uxthlt	r7, r1
+# CHECK-NEXT:  1      1     1.00           *            strhlt	r2, [r7, #22]
+# CHECK-NEXT:  1      1     0.50                        lsrlt	r1, r6, #3
 # CHECK-NEXT:  1      1     1.00           *            strh	r2, [r7, #22]
 # CHECK-NEXT:  1      2     1.00                        asrs	r1, r6, #7
 # CHECK-NEXT:  1      2     1.00                        lsrs	r1, r6, #31
@@ -1162,6 +1170,7 @@
 # CHECK-NEXT:  4      4     2.00    *                   ldrd	r0, r1, [r2, #-0]!
 # CHECK-NEXT:  4      4     2.00    *                   ldrd	r0, r1, [r2, #0]!
 # CHECK-NEXT:  2      4     2.00    *                   ldrd	r0, r1, [r2, #-0]
+# CHECK-NEXT:  4      4     2.00    *                   ldrd	r1, r2, [r0], #0
 # CHECK-NEXT:  0      0     0.00    *      *      U     ldrex	r1, [r4]
 # CHECK-NEXT:  0      0     0.00    *      *      U     ldrex	r8, [r4]
 # CHECK-NEXT:  0      0     0.00    *      *      U     ldrex	r2, [sp, #128]
@@ -1556,6 +1565,7 @@
 # CHECK-NEXT:  2      1     1.00           *            str	r10, [r11], #0
 # CHECK-NEXT:  2      1     1.00           *            strd	r1, r1, [r0], #0
 # CHECK-NEXT:  2      1     1.00           *            strd	r6, r3, [r5], #-8
+# CHECK-NEXT:  2      1     1.00           *            strd	r8, r5, [r6], #-0
 # CHECK-NEXT:  2      1     1.00           *            strd	r7, r4, [r5], #-4
 # CHECK-NEXT:  2      1     1.00           *            strd	r0, r1, [r2, #-0]!
 # CHECK-NEXT:  2      1     1.00           *            strd	r0, r1, [r2, #0]!
@@ -1827,7 +1837,7 @@
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]
-# CHECK-NEXT: 12.00  164.00 164.00 221.00 313.00 44.00   -      -
+# CHECK-NEXT: 12.00  168.00 168.00 223.00 313.00 46.00   -      -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]    Instructions:
@@ -1924,6 +1934,13 @@
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     itett	ne
 # CHECK-NEXT:  -     0.50   0.50    -      -      -      -      -     cmpne	r7, #243
 # CHECK-NEXT:  -     0.50   0.50    -      -      -      -      -     addeq	r7, r1, r2
+# CHECK-NEXT:  -     0.50   0.50    -      -      -      -      -     addne	r7, r1, r2
+# CHECK-NEXT:  -     0.50   0.50    -      -      -      -      -     uxthne	r7, r7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     itttt	lt
+# CHECK-NEXT:  -     0.50   0.50    -      -      -      -      -     cmplt	r7, #243
+# CHECK-NEXT:  -     0.50   0.50    -      -      -      -      -     uxthlt	r7, r1
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     strhlt	r2, [r7, #22]
+# CHECK-NEXT:  -     0.50   0.50    -      -      -      -      -     lsrlt	r1, r6, #3
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     strh	r2, [r7, #22]
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -     asrs	r1, r6, #7
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -     lsrs	r1, r6, #31
@@ -2076,6 +2093,7 @@
 # CHECK-NEXT:  -     1.00   1.00   2.00    -      -      -      -     ldrd	r0, r1, [r2, #-0]!
 # CHECK-NEXT:  -     1.00   1.00   2.00    -      -      -      -     ldrd	r0, r1, [r2, #0]!
 # CHECK-NEXT:  -      -      -     2.00    -      -      -      -     ldrd	r0, r1, [r2, #-0]
+# CHECK-NEXT:  -     1.00   1.00   2.00    -      -      -      -     ldrd	r1, r2, [r0], #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     ldrex	r1, [r4]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     ldrex	r8, [r4]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     ldrex	r2, [sp, #128]
@@ -2470,6 +2488,7 @@
 # CHECK-NEXT:  -     0.50   0.50    -      -     1.00    -      -     str	r10, [r11], #0
 # CHECK-NEXT:  -     0.50   0.50    -      -     1.00    -      -     strd	r1, r1, [r0], #0
 # CHECK-NEXT:  -     0.50   0.50    -      -     1.00    -      -     strd	r6, r3, [r5], #-8
+# CHECK-NEXT:  -     0.50   0.50    -      -     1.00    -      -     strd	r8, r5, [r6], #-0
 # CHECK-NEXT:  -     0.50   0.50    -      -     1.00    -      -     strd	r7, r4, [r5], #-4
 # CHECK-NEXT:  -     0.50   0.50    -      -     1.00    -      -     strd	r0, r1, [r2, #-0]!
 # CHECK-NEXT:  -     0.50   0.50    -      -     1.00    -      -     strd	r0, r1, [r2, #0]!
diff --git a/llvm/test/tools/llvm-mca/X86/BtVer2/simple-test.s b/llvm/test/tools/llvm-mca/X86/BtVer2/simple-test.s
index d1285441de5e..715f3706ef88 100644
--- a/llvm/test/tools/llvm-mca/X86/BtVer2/simple-test.s
+++ b/llvm/test/tools/llvm-mca/X86/BtVer2/simple-test.s
@@ -1,5 +1,8 @@
 # NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
 # RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -iterations=100 < %s | FileCheck %s
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -iterations=100 -skip-unsupported-instructions=lack-sched < %s | FileCheck %s
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -iterations=100 -skip-unsupported-instructions=parse-failure < %s | FileCheck %s
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -iterations=100 -skip-unsupported-instructions=any < %s | FileCheck %s
 
 add %edi, %eax
 
diff --git a/llvm/test/tools/llvm-mca/X86/BtVer2/skip-unsupported-instructions-none-remain.s b/llvm/test/tools/llvm-mca/X86/BtVer2/skip-unsupported-instructions-none-remain.s
index 0d67f53e12f1..5bd6910369ee 100644
--- a/llvm/test/tools/llvm-mca/X86/BtVer2/skip-unsupported-instructions-none-remain.s
+++ b/llvm/test/tools/llvm-mca/X86/BtVer2/skip-unsupported-instructions-none-remain.s
@@ -1,4 +1,4 @@
-# RUN: not llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -skip-unsupported-instructions %s 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
+# RUN: not llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -skip-unsupported-instructions=lack-sched %s 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
 # RUN: not llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 %s 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-ERROR %s
 
 # Test defends that if all instructions are skipped leaving an empty input, an error is printed.
@@ -7,7 +7,7 @@ bzhi %eax, %ebx, %ecx
 
 # CHECK-ALL-NOT: error
 
-# CHECK-ERROR: error: found an unsupported instruction in the input assembly sequence, use -skip-unsupported-instructions to ignore.
+# CHECK-ERROR: error: found an unsupported instruction in the input assembly sequence, use -skip-unsupported-instructions=lack-sched to ignore these on the input.
 
 # CHECK-SKIP: warning: found an unsupported instruction in the input assembly sequence, skipping with -skip-unsupported-instructions, note accuracy will be impacted:
 # CHECK-SKIP: note: instruction:      bzhil   %eax, %ebx, %ecx
diff --git a/llvm/test/tools/llvm-mca/X86/BtVer2/unsupported-instruction.s b/llvm/test/tools/llvm-mca/X86/BtVer2/unsupported-instruction.s
index 3690a1101be9..7d3aee5e3bf9 100644
--- a/llvm/test/tools/llvm-mca/X86/BtVer2/unsupported-instruction.s
+++ b/llvm/test/tools/llvm-mca/X86/BtVer2/unsupported-instruction.s
@@ -1,10 +1,13 @@
-# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -skip-unsupported-instructions -timeline %s 2>&1 | FileCheck --check-prefix=CHECK-SKIP %s
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -skip-unsupported-instructions=any -timeline %s 2>&1 | FileCheck --check-prefix=CHECK-SKIP %s
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -skip-unsupported-instructions=lack-sched -timeline %s 2>&1 | FileCheck --check-prefix=CHECK-SKIP %s
+# RUN: not llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -skip-unsupported-instructions=parse-failure -timeline %s 2>&1 | FileCheck --check-prefix=CHECK-ERROR %s
 # RUN: not llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 %s 2>&1 | FileCheck --check-prefix=CHECK-ERROR %s
 
-# Test checks that unsupported instructions exit with an error, unless -skip-unsupported-instructions is passed, in which case the remaining instructions should be analysed.
+# Test checks that unsupported instructions exit with an error, unless -skip-unsupported-instructions=lack-sched is passed, in which case the remaining instructions should be analysed.
+# Additionally check that -skip-unsupported-instructions=parse-failure continues to raise the lack of scheduling information.
 
 # CHECK-SKIP: warning: found an unsupported instruction in the input assembly sequence, skipping with -skip-unsupported-instructions, note accuracy will be impacted:
-# CHECK-ERROR: error: found an unsupported instruction in the input assembly sequence, use -skip-unsupported-instructions to ignore.
+# CHECK-ERROR: error: found an unsupported instruction in the input assembly sequence, use -skip-unsupported-instructions=lack-sched to ignore these on the input.
 
 bzhi %eax, %ebx, %ecx
 
diff --git a/llvm/test/tools/llvm-mca/bad-input.s b/llvm/test/tools/llvm-mca/bad-input.s
new file mode 100644
index 000000000000..eaf69979cb20
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/bad-input.s
@@ -0,0 +1,14 @@
+# RUN: not llvm-mca %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -skip-unsupported-instructions=none %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -skip-unsupported-instructions=lack-sched %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -skip-unsupported-instructions=parse-failure %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
+# RUN: not llvm-mca -skip-unsupported-instructions=any %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
+
+# Test checks that MCA does not produce a total cycles estimate if it encounters parse errors.
+
+# CHECK-ALL-NOT: Total Cycles:
+
+# CHECK: error: Assembly input parsing had errors, use -skip-unsupported-instructions=parse-failure to drop failing lines from the input.
+# CHECK-SKIP: error: no assembly instructions found.
+
+This is not a valid assembly file for any architecture (by virtue of this text.)
diff --git a/llvm/tools/llvm-mca/CodeRegionGenerator.cpp b/llvm/tools/llvm-mca/CodeRegionGenerator.cpp
index 5241b584b746..863766cd777d 100644
--- a/llvm/tools/llvm-mca/CodeRegionGenerator.cpp
+++ b/llvm/tools/llvm-mca/CodeRegionGenerator.cpp
@@ -29,7 +29,7 @@ namespace mca {
 CodeRegionGenerator::~CodeRegionGenerator() {}
 
 Expected<const CodeRegions &> AsmCodeRegionGenerator::parseCodeRegions(
-    const std::unique_ptr<MCInstPrinter> &IP) {
+    const std::unique_ptr<MCInstPrinter> &IP, bool SkipFailures) {
   MCTargetOptions Opts;
   Opts.PreserveAsmComments = false;
   CodeRegions &Regions = getRegions();
@@ -61,7 +61,16 @@ Expected<const CodeRegions &> AsmCodeRegionGenerator::parseCodeRegions(
         "This target does not support assembly parsing.",
         inconvertibleErrorCode());
   Parser->setTargetParser(*TAP);
-  Parser->Run(false);
+  // Parser->Run() confusingly returns true on errors, in which case the errors
+  // were already shown to the user. SkipFailures implies continuing in the
+  // presence of any kind of failure within the parser, in which case failing
+  // input lines are not represented, but the rest of the input remains.
+  if (Parser->Run(false) && !SkipFailures) {
+    const char *Message = "Assembly input parsing had errors, use "
+                          "-skip-unsupported-instructions=parse-failure "
+                          "to drop failing lines from the input.";
+    return make_error<StringError>(Message, inconvertibleErrorCode());
+  }
 
   if (CCP->hadErr())
     return make_error<StringError>("There was an error parsing comments.",
diff --git a/llvm/tools/llvm-mca/CodeRegionGenerator.h b/llvm/tools/llvm-mca/CodeRegionGenerator.h
index 68da567f3e0f..12261e7656a4 100644
--- a/llvm/tools/llvm-mca/CodeRegionGenerator.h
+++ b/llvm/tools/llvm-mca/CodeRegionGenerator.h
@@ -148,7 +148,8 @@ protected:
   CodeRegionGenerator(const CodeRegionGenerator &) = delete;
   CodeRegionGenerator &operator=(const CodeRegionGenerator &) = delete;
   virtual Expected<const CodeRegions &>
-  parseCodeRegions(const std::unique_ptr<MCInstPrinter> &IP) = 0;
+  parseCodeRegions(const std::unique_ptr<MCInstPrinter> &IP,
+                   bool SkipFailures) = 0;
 
 public:
   CodeRegionGenerator() {}
@@ -164,7 +165,8 @@ public:
   AnalysisRegionGenerator(llvm::SourceMgr &SM) : Regions(SM) {}
 
   virtual Expected<const AnalysisRegions &>
-  parseAnalysisRegions(const std::unique_ptr<MCInstPrinter> &IP) = 0;
+  parseAnalysisRegions(const std::unique_ptr<MCInstPrinter> &IP,
+                       bool SkipFailures) = 0;
 };
 
 /// Abstract CodeRegionGenerator with InstrumentRegionsRegions member
@@ -176,7 +178,8 @@ public:
   InstrumentRegionGenerator(llvm::SourceMgr &SM) : Regions(SM) {}
 
   virtual Expected<const InstrumentRegions &>
-  parseInstrumentRegions(const std::unique_ptr<MCInstPrinter> &IP) = 0;
+  parseInstrumentRegions(const std::unique_ptr<MCInstPrinter> &IP,
+                         bool SkipFailures) = 0;
 };
 
 /// This abstract class is responsible for parsing input ASM and
@@ -202,7 +205,8 @@ public:
 
   unsigned getAssemblerDialect() const { return AssemblerDialect; }
   Expected<const CodeRegions &>
-  parseCodeRegions(const std::unique_ptr<MCInstPrinter> &IP) override;
+  parseCodeRegions(const std::unique_ptr<MCInstPrinter> &IP,
+                   bool SkipFailures) override;
 };
 
 class AsmAnalysisRegionGenerator final : public AnalysisRegionGenerator,
@@ -222,8 +226,10 @@ public:
   MCStreamerWrapper *getMCStreamer() override { return &Streamer; }
 
   Expected<const AnalysisRegions &>
-  parseAnalysisRegions(const std::unique_ptr<MCInstPrinter> &IP) override {
-    Expected<const CodeRegions &> RegionsOrErr = parseCodeRegions(IP);
+  parseAnalysisRegions(const std::unique_ptr<MCInstPrinter> &IP,
+                       bool SkipFailures) override {
+    Expected<const CodeRegions &> RegionsOrErr =
+        parseCodeRegions(IP, SkipFailures);
     if (!RegionsOrErr)
       return RegionsOrErr.takeError();
     else
@@ -231,8 +237,9 @@ public:
   }
 
   Expected<const CodeRegions &>
-  parseCodeRegions(const std::unique_ptr<MCInstPrinter> &IP) override {
-    return AsmCodeRegionGenerator::parseCodeRegions(IP);
+  parseCodeRegions(const std::unique_ptr<MCInstPrinter> &IP,
+                   bool SkipFailures) override {
+    return AsmCodeRegionGenerator::parseCodeRegions(IP, SkipFailures);
   }
 };
 
@@ -254,8 +261,10 @@ public:
   MCStreamerWrapper *getMCStreamer() override { return &Streamer; }
 
   Expected<const InstrumentRegions &>
-  parseInstrumentRegions(const std::unique_ptr<MCInstPrinter> &IP) override {
-    Expected<const CodeRegions &> RegionsOrErr = parseCodeRegions(IP);
+  parseInstrumentRegions(const std::unique_ptr<MCInstPrinter> &IP,
+                         bool SkipFailures) override {
+    Expected<const CodeRegions &> RegionsOrErr =
+        parseCodeRegions(IP, SkipFailures);
     if (!RegionsOrErr)
       return RegionsOrErr.takeError();
     else
@@ -263,8 +272,9 @@ public:
   }
 
   Expected<const CodeRegions &>
-  parseCodeRegions(const std::unique_ptr<MCInstPrinter> &IP) override {
-    return AsmCodeRegionGenerator::parseCodeRegions(IP);
+  parseCodeRegions(const std::unique_ptr<MCInstPrinter> &IP,
+                   bool SkipFailures) override {
+    return AsmCodeRegionGenerator::parseCodeRegions(IP, SkipFailures);
   }
 };
 
diff --git a/llvm/tools/llvm-mca/llvm-mca.cpp b/llvm/tools/llvm-mca/llvm-mca.cpp
index e037c06b12a3..03d7d7944b9c 100644
--- a/llvm/tools/llvm-mca/llvm-mca.cpp
+++ b/llvm/tools/llvm-mca/llvm-mca.cpp
@@ -135,6 +135,35 @@ static cl::opt<unsigned>
                                "(instructions per cycle)"),
                       cl::cat(ToolOptions), cl::init(0));
 
+enum class SkipType { NONE, LACK_SCHED, PARSE_FAILURE, ANY_FAILURE };
+
+static cl::opt<enum SkipType> SkipUnsupportedInstructions(
+    "skip-unsupported-instructions",
+    cl::desc("Force analysis to continue in the presence of unsupported "
+             "instructions"),
+    cl::values(
+        clEnumValN(SkipType::NONE, "none",
+                   "Exit with an error when an instruction is unsupported for "
+                   "any reason (default)"),
+        clEnumValN(
+            SkipType::LACK_SCHED, "lack-sched",
+            "Skip instructions on input which lack scheduling information"),
+        clEnumValN(
+            SkipType::PARSE_FAILURE, "parse-failure",
+            "Skip lines on the input which fail to parse for any reason"),
+        clEnumValN(SkipType::ANY_FAILURE, "any",
+                   "Skip instructions or lines on input which are unsupported "
+                   "for any reason")),
+    cl::init(SkipType::NONE), cl::cat(ViewOptions));
+
+bool shouldSkip(enum SkipType skipType) {
+  if (SkipUnsupportedInstructions == SkipType::NONE)
+    return false;
+  if (SkipUnsupportedInstructions == SkipType::ANY_FAILURE)
+    return true;
+  return skipType == SkipUnsupportedInstructions;
+}
+
 static cl::opt<bool>
     PrintRegisterFileStats("register-file-stats",
                            cl::desc("Print register file statistics"),
@@ -237,11 +266,6 @@ static cl::opt<bool> DisableInstrumentManager(
              "ignores instruments.)."),
     cl::cat(ViewOptions), cl::init(false));
 
-static cl::opt<bool> SkipUnsupportedInstructions(
-    "skip-unsupported-instructions",
-    cl::desc("Make unsupported instruction errors into warnings."),
-    cl::cat(ViewOptions), cl::init(false));
-
 namespace {
 
 const Target *getTarget(const char *ProgName) {
@@ -440,7 +464,8 @@ int main(int argc, char **argv) {
   mca::AsmAnalysisRegionGenerator CRG(*TheTarget, SrcMgr, ACtx, *MAI, *STI,
                                       *MCII);
   Expected<const mca::AnalysisRegions &> RegionsOrErr =
-      CRG.parseAnalysisRegions(std::move(IPtemp));
+      CRG.parseAnalysisRegions(std::move(IPtemp),
+                               shouldSkip(SkipType::PARSE_FAILURE));
   if (!RegionsOrErr) {
     if (auto Err =
             handleErrors(RegionsOrErr.takeError(), [](const StringError &E) {
@@ -482,7 +507,8 @@ int main(int argc, char **argv) {
   mca::AsmInstrumentRegionGenerator IRG(*TheTarget, SrcMgr, ICtx, *MAI, *STI,
                                         *MCII, *IM);
   Expected<const mca::InstrumentRegions &> InstrumentRegionsOrErr =
-      IRG.parseInstrumentRegions(std::move(IPtemp));
+      IRG.parseInstrumentRegions(std::move(IPtemp),
+                                 shouldSkip(SkipType::PARSE_FAILURE));
   if (!InstrumentRegionsOrErr) {
     if (auto Err = handleErrors(InstrumentRegionsOrErr.takeError(),
                                 [](const StringError &E) {
@@ -593,7 +619,7 @@ int main(int argc, char **argv) {
                 [&IP, &STI](const mca::InstructionError<MCInst> &IE) {
                   std::string InstructionStr;
                   raw_string_ostream SS(InstructionStr);
-                  if (SkipUnsupportedInstructions)
+                  if (shouldSkip(SkipType::LACK_SCHED))
                     WithColor::warning()
                         << IE.Message
                         << ", skipping with -skip-unsupported-instructions, "
@@ -601,7 +627,8 @@ int main(int argc, char **argv) {
                   else
                     WithColor::error()
                         << IE.Message
-                        << ", use -skip-unsupported-instructions to ignore.\n";
+                        << ", use -skip-unsupported-instructions=lack-sched to "
+                           "ignore these on the input.\n";
                   IP->printInst(&IE.Inst, 0, "", *STI, SS);
                   SS.flush();
                   WithColor::note()
@@ -610,7 +637,7 @@ int main(int argc, char **argv) {
           // Default case.
           WithColor::error() << toString(std::move(NewE));
         }
-        if (SkipUnsupportedInstructions) {
+        if (shouldSkip(SkipType::LACK_SCHED)) {
           DroppedInsts.insert(&MCI);
           continue;
         }
-- 
GitLab


From 1530f319311908b06fe935c89fca692d3e53184f Mon Sep 17 00:00:00 2001
From: Orlando Cazalet-Hyams <orlando.hyams@sony.com>
Date: Tue, 7 May 2024 09:15:16 +0100
Subject: [PATCH 0025/1206] [RemoveDIs] Update some unittests to the new format
 (#90476)

This patch updates the unittests that can be changed to the new format
after #89799 (which changes the default format everywhere) to avoid a
loss in coverage for the (new) default debug info format.
---
 .../Analysis/IRSimilarityIdentifierTest.cpp   |  42 +---
 llvm/unittests/Transforms/Utils/LocalTest.cpp | 197 ++++++++----------
 2 files changed, 96 insertions(+), 143 deletions(-)

diff --git a/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp b/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
index 0a08ca3cb99d..24f4f11db9a8 100644
--- a/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
+++ b/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
@@ -28,22 +28,6 @@ extern cl::opt<cl::boolOrDefault> PreserveInputDbgFormat;
 extern bool WriteNewDbgInfoFormatToBitcode;
 extern cl::opt<bool> WriteNewDbgInfoFormat;
 
-// Backup all of the existing settings that may be modified when
-// PreserveInputDbgFormat=true, so that when the test is finished we return them
-// (and the "preserve" setting) to their original values.
-static auto SaveDbgInfoFormat() {
-  return make_scope_exit(
-      [OldPreserveInputDbgFormat = PreserveInputDbgFormat.getValue(),
-       OldUseNewDbgInfoFormat = UseNewDbgInfoFormat.getValue(),
-       OldWriteNewDbgInfoFormatToBitcode = WriteNewDbgInfoFormatToBitcode,
-       OldWriteNewDbgInfoFormat = WriteNewDbgInfoFormat.getValue()] {
-        PreserveInputDbgFormat = OldPreserveInputDbgFormat;
-        UseNewDbgInfoFormat = OldUseNewDbgInfoFormat;
-        WriteNewDbgInfoFormatToBitcode = OldWriteNewDbgInfoFormatToBitcode;
-        WriteNewDbgInfoFormat = OldWriteNewDbgInfoFormat;
-      });
-}
-
 static std::unique_ptr<Module> makeLLVMModule(LLVMContext &Context,
                                               StringRef ModuleStr) {
   SMDiagnostic Err;
@@ -1328,25 +1312,19 @@ TEST(IRInstructionMapper, CallBrInstIllegal) {
   ASSERT_GT(UnsignedVec[0], Mapper.IllegalInstrNumber);
 }
 
-// Checks that an debuginfo intrinsics are mapped to be invisible.  Since they
+// Checks that an debuginfo records are mapped to be invisible. Since they
 // do not semantically change the program, they can be recognized as similar.
-// FIXME: PreserveInputDbgFormat is set to true because this test contains
-// malformed debug info that cannot be converted to the new debug info format;
-// this test should be updated later to use valid debug info.
 TEST(IRInstructionMapper, DebugInfoInvisible) {
   StringRef ModuleString = R"(
                           define i32 @f(i32 %a, i32 %b) {
                           then:
-                            %0 = add i32 %a, %b                    
-                            call void @llvm.dbg.value(metadata !0)
-                            %1 = add i32 %a, %b     
+                            %0 = add i32 %a, %b
+                              #dbg_value(i32 0, !0, !0, !0)
+                            %1 = add i32 %a, %b
                             ret i32 0
                           }
 
-                          declare void @llvm.dbg.value(metadata)
                           !0 = distinct !{!"test\00", i32 10})";
-  auto SettingGuard = SaveDbgInfoFormat();
-  PreserveInputDbgFormat = cl::boolOrDefault::BOU_TRUE;
   LLVMContext Context;
   std::unique_ptr<Module> M = makeLLVMModule(Context, ModuleString);
 
@@ -1941,22 +1919,19 @@ TEST(IRSimilarityCandidate, CheckRegionsDifferentTypes) {
   ASSERT_FALSE(longSimCandCompare(InstrList));
 }
 
-// Check that debug instructions do not impact similarity. They are marked as
+// Check that debug records do not impact similarity. They are marked as
 // invisible.
-// FIXME: PreserveInputDbgFormat is set to true because this test contains
-// malformed debug info that cannot be converted to the new debug info format;
-// this test should be updated later to use valid debug info.
 TEST(IRSimilarityCandidate, IdenticalWithDebug) {
   StringRef ModuleString = R"(
                           define i32 @f(i32 %a, i32 %b) {
                           bb0:
                              %0 = add i32 %a, %b
-                             call void @llvm.dbg.value(metadata !0)
+                               #dbg_value(i32 0, !0, !0, !0)
                              %1 = add i32 %b, %a
                              ret i32 0
                           bb1:
                              %2 = add i32 %a, %b
-                             call void @llvm.dbg.value(metadata !1)
+                               #dbg_value(i32 1, !1, !1, !1)
                              %3 = add i32 %b, %a
                              ret i32 0
                           bb2:
@@ -1965,11 +1940,8 @@ TEST(IRSimilarityCandidate, IdenticalWithDebug) {
                              ret i32 0       
                           }
 
-                          declare void @llvm.dbg.value(metadata)
                           !0 = distinct !{!"test\00", i32 10}
                           !1 = distinct !{!"test\00", i32 11})";
-  auto SettingGuard = SaveDbgInfoFormat();
-  PreserveInputDbgFormat = cl::boolOrDefault::BOU_TRUE;
   LLVMContext Context;
   std::unique_ptr<Module> M = makeLLVMModule(Context, ModuleString);
 
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index b28ba2b1b446..6052e58b697d 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -138,12 +138,6 @@ static std::unique_ptr<Module> parseIR(LLVMContext &C, const char *IR) {
 
 TEST(Local, ReplaceDbgDeclare) {
   LLVMContext C;
-  // FIXME: PreserveInputDbgFormat is set to true because this test has
-  // been written to expect debug intrinsics rather than debug records; use the
-  // intrinsic format until we update the test checks.
-  auto SettingGuard = SaveDbgInfoFormat();
-  PreserveInputDbgFormat = cl::boolOrDefault::BOU_TRUE;
-
   // Original C source to get debug info for a local variable:
   // void f() { int x; }
   std::unique_ptr<Module> M = parseIR(C,
@@ -151,11 +145,11 @@ TEST(Local, ReplaceDbgDeclare) {
       define void @f() !dbg !8 {
       entry:
         %x = alloca i32, align 4
-        call void @llvm.dbg.declare(metadata i32* %x, metadata !11, metadata !DIExpression()), !dbg !13
-        call void @llvm.dbg.declare(metadata i32* %x, metadata !11, metadata !DIExpression()), !dbg !13
+          #dbg_declare(ptr %x, !11, !DIExpression(), !13)
+          #dbg_declare(ptr %x, !11, !DIExpression(), !13)
         ret void, !dbg !14
       }
-      declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
       !llvm.dbg.cu = !{!0}
       !llvm.module.flags = !{!3, !4}
       !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
@@ -178,20 +172,18 @@ TEST(Local, ReplaceDbgDeclare) {
   Instruction *Inst = &F->front().front();
   auto *AI = dyn_cast<AllocaInst>(Inst);
   ASSERT_TRUE(AI);
-  Inst = Inst->getNextNode()->getNextNode();
-  ASSERT_TRUE(Inst);
-  auto *DII = dyn_cast<DbgDeclareInst>(Inst);
-  ASSERT_TRUE(DII);
+
   Value *NewBase = Constant::getNullValue(PointerType::getUnqual(C));
   DIBuilder DIB(*M);
   replaceDbgDeclare(AI, NewBase, DIB, DIExpression::ApplyOffset, 0);
 
-  // There should be exactly two dbg.declares.
-  int Declares = 0;
-  for (const Instruction &I : F->front())
-    if (isa<DbgDeclareInst>(I))
-      Declares++;
-  EXPECT_EQ(2, Declares);
+  // There should be exactly two dbg.declares, attached to the terminator.
+  Inst = F->front().getTerminator();
+  ASSERT_TRUE(Inst);
+  EXPECT_TRUE(Inst->hasDbgRecords());
+  EXPECT_EQ(range_size(Inst->getDbgRecordRange()), 2u);
+  for (DbgVariableRecord &DVR : filterDbgVars(Inst->getDbgRecordRange()))
+    EXPECT_EQ(DVR.getAddress(), NewBase);
 }
 
 /// Build the dominator tree for the function and run the Test.
@@ -520,25 +512,16 @@ struct SalvageDebugInfoTest : ::testing::Test {
   Function *F = nullptr;
 
   void SetUp() override {
-    // FIXME: PreserveInputDbgFormat is set to true because this test has
-    // been written to expect debug intrinsics rather than debug records; use
-    // the intrinsic format until we update the test checks. Note that the
-    // temporary setting of this flag only needs to cover the parsing step, not
-    // the test body itself.
-    auto SettingGuard = SaveDbgInfoFormat();
-    PreserveInputDbgFormat = cl::boolOrDefault::BOU_TRUE;
-
     M = parseIR(C,
                 R"(
       define void @f() !dbg !8 {
       entry:
         %x = add i32 0, 1
         %y = add i32 %x, 2
-        call void @llvm.dbg.value(metadata i32 %x, metadata !11, metadata !DIExpression()), !dbg !13
-        call void @llvm.dbg.value(metadata i32 %y, metadata !11, metadata !DIExpression()), !dbg !13
+          #dbg_value(i32 %x, !11, !DIExpression(), !13)
+          #dbg_value(i32 %y, !11, !DIExpression(), !13)
         ret void, !dbg !14
       }
-      declare void @llvm.dbg.value(metadata, metadata, metadata)
       !llvm.dbg.cu = !{!0}
       !llvm.module.flags = !{!3, !4}
       !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
@@ -561,49 +544,48 @@ struct SalvageDebugInfoTest : ::testing::Test {
     ASSERT_TRUE(F);
   }
 
-  bool doesDebugValueDescribeX(const DbgValueInst &DI) {
-    if (DI.getNumVariableLocationOps() != 1u)
+  bool doesDebugValueDescribeX(const DbgVariableRecord &DVR) {
+    if (DVR.getNumVariableLocationOps() != 1u)
       return false;
-    const auto &CI = *cast<ConstantInt>(DI.getValue(0));
+    const auto &CI = *cast<ConstantInt>(DVR.getValue(0));
     if (CI.isZero())
-      return DI.getExpression()->getElements().equals(
+      return DVR.getExpression()->getElements().equals(
           {dwarf::DW_OP_plus_uconst, 1, dwarf::DW_OP_stack_value});
     else if (CI.isOneValue())
-      return DI.getExpression()->getElements().empty();
+      return DVR.getExpression()->getElements().empty();
     return false;
   }
 
-  bool doesDebugValueDescribeY(const DbgValueInst &DI) {
-    if (DI.getNumVariableLocationOps() != 1u)
+  bool doesDebugValueDescribeY(const DbgVariableRecord &DVR) {
+    if (DVR.getNumVariableLocationOps() != 1u)
       return false;
-    const auto &CI = *cast<ConstantInt>(DI.getVariableLocationOp(0));
+    const auto &CI = *cast<ConstantInt>(DVR.getVariableLocationOp(0));
     if (CI.isZero())
-      return DI.getExpression()->getElements().equals(
+      return DVR.getExpression()->getElements().equals(
           {dwarf::DW_OP_plus_uconst, 1, dwarf::DW_OP_plus_uconst, 2,
            dwarf::DW_OP_stack_value});
     else if (CI.isOneValue())
-      return DI.getExpression()->getElements().equals(
+      return DVR.getExpression()->getElements().equals(
           {dwarf::DW_OP_plus_uconst, 2, dwarf::DW_OP_stack_value});
     return false;
   }
 
   void verifyDebugValuesAreSalvaged() {
+    // The function should only contain debug values and a terminator.
+    EXPECT_EQ(F->size(), 1u);
+    EXPECT_TRUE(F->begin()->begin()->isTerminator());
+
     // Check that the debug values for %x and %y are preserved.
     bool FoundX = false;
     bool FoundY = false;
-    for (const Instruction &I : F->front()) {
-      auto DI = dyn_cast<DbgValueInst>(&I);
-      if (!DI) {
-        // The function should only contain debug values and a terminator.
-        ASSERT_TRUE(I.isTerminator());
-        continue;
-      }
-      EXPECT_EQ(DI->getVariable()->getName(), "x");
-      FoundX |= doesDebugValueDescribeX(*DI);
-      FoundY |= doesDebugValueDescribeY(*DI);
+    for (DbgVariableRecord &DVR :
+         filterDbgVars(F->begin()->begin()->getDbgRecordRange())) {
+      EXPECT_EQ(DVR.getVariable()->getName(), "x");
+      FoundX |= doesDebugValueDescribeX(DVR);
+      FoundY |= doesDebugValueDescribeY(DVR);
     }
-    ASSERT_TRUE(FoundX);
-    ASSERT_TRUE(FoundY);
+    EXPECT_TRUE(FoundX);
+    EXPECT_TRUE(FoundY);
   }
 };
 
@@ -721,21 +703,14 @@ TEST(Local, ChangeToUnreachable) {
 
 TEST(Local, FindDbgUsers) {
   LLVMContext Ctx;
-  // FIXME: PreserveInputDbgFormat is set to true because this test has
-  // been written to expect debug intrinsics rather than debug records; use the
-  // intrinsic format until we update the test checks.
-  auto SettingGuard = SaveDbgInfoFormat();
-  PreserveInputDbgFormat = cl::boolOrDefault::BOU_TRUE;
   std::unique_ptr<Module> M = parseIR(Ctx,
                                       R"(
   define dso_local void @fun(ptr %a) #0 !dbg !11 {
   entry:
-    call void @llvm.dbg.assign(metadata ptr %a, metadata !16, metadata !DIExpression(), metadata !15, metadata ptr %a, metadata !DIExpression()), !dbg !19
+      #dbg_assign(ptr %a, !16, !DIExpression(), !15, ptr %a, !DIExpression(), !19)
     ret void
   }
 
-  declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata)
-
   !llvm.dbg.cu = !{!0}
   !llvm.module.flags = !{!2, !3, !9}
   !llvm.ident = !{!10}
@@ -762,9 +737,13 @@ TEST(Local, FindDbgUsers) {
   verifyModule(*M, &errs(), &BrokenDebugInfo);
   ASSERT_FALSE(BrokenDebugInfo);
 
+  // Convert to debug intrinsics as we want to test findDbgUsers and
+  // findDbgValue's debug-intrinsic-finding code here.
+  // TODO: Remove this test when debug intrinsics are removed.
+  M->convertFromNewDbgValues();
+
   Function &Fun = *cast<Function>(M->getNamedValue("fun"));
   Value *Arg = Fun.getArg(0);
-
   SmallVector<DbgVariableIntrinsic *> Users;
   // Arg (%a) is used twice by a single dbg.assign. Check findDbgUsers returns
   // only 1 pointer to it rather than 2.
@@ -785,7 +764,7 @@ TEST(Local, FindDbgRecords) {
                                       R"(
   define dso_local void @fun(ptr %a) #0 !dbg !11 {
   entry:
-    call void @llvm.dbg.assign(metadata ptr %a, metadata !16, metadata !DIExpression(), metadata !15, metadata ptr %a, metadata !DIExpression()), !dbg !19
+      #dbg_assign(ptr %a, !16, !DIExpression(), !15, ptr %a, !DIExpression(), !19)
     ret void
   }
 
@@ -837,13 +816,7 @@ TEST(Local, FindDbgRecords) {
 
 TEST(Local, ReplaceAllDbgUsesWith) {
   using namespace llvm::dwarf;
-
   LLVMContext Ctx;
-  // FIXME: PreserveInputDbgFormat is set to true because this test has
-  // been written to expect debug intrinsics rather than debug records; use the
-  // intrinsic format until we update the test checks.
-  auto SettingGuard = SaveDbgInfoFormat();
-  PreserveInputDbgFormat = cl::boolOrDefault::BOU_TRUE;
 
   // Note: The datalayout simulates Darwin/x86_64.
   std::unique_ptr<Module> M = parseIR(Ctx,
@@ -855,39 +828,36 @@ TEST(Local, ReplaceAllDbgUsesWith) {
     define void @f() !dbg !6 {
     entry:
       %a = add i32 0, 1, !dbg !15
-      call void @llvm.dbg.value(metadata i32 %a, metadata !9, metadata !DIExpression()), !dbg !15
 
+        #dbg_value(i32 %a, !9, !DIExpression(), !15)
       %b = add i64 0, 1, !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression()), !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul)), !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value)), !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_LLVM_fragment, 0, 8)), !dbg !16
-      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value, DW_OP_LLVM_fragment, 0, 8)), !dbg !16
 
-      %c = inttoptr i64 0 to i64*, !dbg !17
-      call void @llvm.dbg.declare(metadata i64* %c, metadata !13, metadata !DIExpression()), !dbg !17
+        #dbg_value(i64 %b, !11, !DIExpression(), !16)
+        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul), !16)
+        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value), !16)
+        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_LLVM_fragment, 0, 8), !16)
+        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_LLVM_fragment, 0, 8), !16)
+        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value, DW_OP_LLVM_fragment, 0, 8), !16)
+      %c = inttoptr i64 0 to ptr, !dbg !17
 
-      %d = inttoptr i64 0 to i32*, !dbg !18
-      call void @llvm.dbg.declare(metadata i32* %d, metadata !20, metadata !DIExpression()), !dbg !18
+        #dbg_declare(ptr %c, !13, !DIExpression(), !17)
+      %d = inttoptr i64 0 to ptr, !dbg !18
 
+        #dbg_declare(ptr %d,  !20,  !DIExpression(), !18)
       %e = add <2 x i16> zeroinitializer, zeroinitializer
-      call void @llvm.dbg.value(metadata <2 x i16> %e, metadata !14, metadata !DIExpression()), !dbg !18
 
+        #dbg_value(<2 x i16> %e, !14, !DIExpression(), !18)
       %f = call i32 @escape(i32 0)
-      call void @llvm.dbg.value(metadata i32 %f, metadata !9, metadata !DIExpression()), !dbg !15
 
+        #dbg_value(i32 %f, !9, !DIExpression(), !15)
       %barrier = call i32 @escape(i32 0)
 
       %g = call i32 @escape(i32 %f)
-      call void @llvm.dbg.value(metadata i32 %g, metadata !9, metadata !DIExpression()), !dbg !15
 
+        #dbg_value(i32 %g, !9, !DIExpression(), !15)
       ret void, !dbg !19
     }
 
-    declare void @llvm.dbg.declare(metadata, metadata, metadata)
-    declare void @llvm.dbg.value(metadata, metadata, metadata)
-
     !llvm.dbg.cu = !{!0}
     !llvm.module.flags = !{!5}
 
@@ -942,38 +912,47 @@ TEST(Local, ReplaceAllDbgUsesWith) {
   EXPECT_TRUE(replaceAllDbgUsesWith(D, C, C, DT));
 
   SmallVector<DbgVariableIntrinsic *, 2> CDbgVals;
-  findDbgUsers(CDbgVals, &C);
-  EXPECT_EQ(2U, CDbgVals.size());
-  EXPECT_TRUE(all_of(CDbgVals, [](DbgVariableIntrinsic *DII) {
-    return isa<DbgDeclareInst>(DII);
-  }));
+  SmallVector<DbgVariableRecord *, 2> CDbgRecords;
+  findDbgUsers(CDbgVals, &C, &CDbgRecords);
+  EXPECT_EQ(0U, CDbgVals.size());
+  EXPECT_EQ(2U, CDbgRecords.size());
+  EXPECT_TRUE(all_of(
+      CDbgRecords, [](DbgVariableRecord *DVR) { return DVR->isDbgDeclare(); }));
 
   EXPECT_TRUE(replaceAllDbgUsesWith(C, D, D, DT));
 
   SmallVector<DbgVariableIntrinsic *, 2> DDbgVals;
-  findDbgUsers(DDbgVals, &D);
-  EXPECT_EQ(2U, DDbgVals.size());
-  EXPECT_TRUE(all_of(DDbgVals, [](DbgVariableIntrinsic *DII) {
-    return isa<DbgDeclareInst>(DII);
-  }));
+  SmallVector<DbgVariableRecord *, 2> DDbgRecords;
+  findDbgUsers(DDbgVals, &D, &DDbgRecords);
+  EXPECT_EQ(0U, DDbgVals.size());
+  EXPECT_EQ(2U, DDbgRecords.size());
+  EXPECT_TRUE(all_of(
+      DDbgRecords, [](DbgVariableRecord *DVR) { return DVR->isDbgDeclare(); }));
 
   // Introduce a use-before-def. Check that the dbg.value for %a is salvaged.
   EXPECT_TRUE(replaceAllDbgUsesWith(A, F_, F_, DT));
 
-  auto *ADbgVal = cast<DbgValueInst>(A.getNextNode());
-  EXPECT_EQ(ADbgVal->getNumVariableLocationOps(), 1u);
-  EXPECT_EQ(ConstantInt::get(A.getType(), 0), ADbgVal->getVariableLocationOp(0));
+  EXPECT_FALSE(A.hasDbgRecords());
+  EXPECT_TRUE(B.hasDbgRecords());
+  DbgVariableRecord *BDbgVal =
+      cast<DbgVariableRecord>(&*B.getDbgRecordRange().begin());
+  EXPECT_EQ(BDbgVal->getNumVariableLocationOps(), 1u);
+  EXPECT_EQ(ConstantInt::get(A.getType(), 0),
+            BDbgVal->getVariableLocationOp(0));
 
   // Introduce a use-before-def. Check that the dbg.values for %f become undef.
   EXPECT_TRUE(replaceAllDbgUsesWith(F_, G, G, DT));
 
-  auto *FDbgVal = cast<DbgValueInst>(F_.getNextNode());
-  EXPECT_EQ(FDbgVal->getNumVariableLocationOps(), 1u);
-  EXPECT_TRUE(FDbgVal->isKillLocation());
+  DbgVariableRecord *BarrierDbgVal =
+      cast<DbgVariableRecord>(&*Barrier.getDbgRecordRange().begin());
+  EXPECT_EQ(BarrierDbgVal->getNumVariableLocationOps(), 1u);
+  EXPECT_TRUE(BarrierDbgVal->isKillLocation());
 
-  SmallVector<DbgValueInst *, 1> FDbgVals;
-  findDbgValues(FDbgVals, &F_);
-  EXPECT_EQ(0U, FDbgVals.size());
+  SmallVector<DbgValueInst *, 1> BarrierDbgVals;
+  SmallVector<DbgVariableRecord *, 8> BarrierDbgRecs;
+  findDbgValues(BarrierDbgVals, &F_, &BarrierDbgRecs);
+  EXPECT_EQ(0U, BarrierDbgVals.size());
+  EXPECT_EQ(0U, BarrierDbgRecs.size());
 
   // Simulate i32 -> i64 conversion to test sign-extension. Here are some
   // interesting cases to handle:
@@ -983,13 +962,15 @@ TEST(Local, ReplaceAllDbgUsesWith) {
   //  4-6) like (1-3), but with a fragment
   EXPECT_TRUE(replaceAllDbgUsesWith(B, A, A, DT));
 
-  SmallVector<DbgValueInst *, 8> ADbgVals;
-  findDbgValues(ADbgVals, &A);
-  EXPECT_EQ(6U, ADbgVals.size());
+  SmallVector<DbgValueInst *, 8> BDbgVals;
+  SmallVector<DbgVariableRecord *, 8> BDbgRecs;
+  findDbgValues(BDbgVals, &A, &BDbgRecs);
+  EXPECT_EQ(0U, BDbgVals.size());
+  EXPECT_EQ(6U, BDbgRecs.size());
 
   // Check that %a has a dbg.value with a DIExpression matching \p Ops.
   auto hasADbgVal = [&](ArrayRef<uint64_t> Ops) {
-    return any_of(ADbgVals, [&](DbgValueInst *DVI) {
+    return any_of(BDbgRecs, [&](DbgVariableRecord *DVI) {
       assert(DVI->getVariable()->getName() == "2");
       return DVI->getExpression()->getElements() == Ops;
     });
-- 
GitLab


From 1fd196c8df8e9fa4e0eddddc92b012824d8d1b0b Mon Sep 17 00:00:00 2001
From: ostannard <oliver.stannard@arm.com>
Date: Tue, 7 May 2024 09:17:05 +0100
Subject: [PATCH 0026/1206] [AArch64] Diagnose more functions when FP not
 enabled (#90832)

When using a hard-float ABI for a target without FP registers, it's not
possible to correctly generate code for functions with arguments which
must be passed in floating-point registers. This is diagnosed in CodeGen
instead of Sema, to more closely match GCC's behaviour around inline
functions, which is relied on by the Linux kernel.

Previously, this only checked function signatures as they were
code-generated, but this missed some cases:
* Calls to functions not defined in this translation unit.
* Calls through function pointers.
* Calls to variadic functions, where the variadic arguments have a
floating-point type.

This adds checks to function calls, as well as definitions, so that
these cases are correctly diagnosed.
---
 clang/lib/CodeGen/CGCall.cpp                  | 11 ++-
 clang/lib/CodeGen/TargetInfo.h                |  3 +-
 clang/lib/CodeGen/Targets/AArch64.cpp         | 92 ++++++++++++++-----
 clang/lib/CodeGen/Targets/X86.cpp             | 16 +++-
 .../CodeGen/aarch64-soft-float-abi-errors.c   | 24 +++++
 5 files changed, 114 insertions(+), 32 deletions(-)

diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 69548902dc43..0c7eef59db53 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -5050,13 +5050,14 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         (TargetDecl->hasAttr<TargetAttr>() ||
          (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>())))
       checkTargetFeatures(Loc, FD);
-
-    // Some architectures (such as x86-64) have the ABI changed based on
-    // attribute-target/features. Give them a chance to diagnose.
-    CGM.getTargetCodeGenInfo().checkFunctionCallABI(
-        CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs);
   }
 
+  // Some architectures (such as x86-64) have the ABI changed based on
+  // attribute-target/features. Give them a chance to diagnose.
+  CGM.getTargetCodeGenInfo().checkFunctionCallABI(
+      CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl),
+      dyn_cast_or_null<FunctionDecl>(TargetDecl), CallArgs, RetTy);
+
   // 1. Set up the arguments.
 
   // If we're using inalloca, insert the allocation after the stack save.
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index b1dfe5bf8f27..f242d9e36ed4 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -94,7 +94,8 @@ public:
   virtual void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
                                     const FunctionDecl *Caller,
                                     const FunctionDecl *Callee,
-                                    const CallArgList &Args) const {}
+                                    const CallArgList &Args,
+                                    QualType ReturnType) const {}
 
   /// Determines the size of struct _Unwind_Exception on this platform,
   /// in 8-bit units.  The Itanium ABI defines this as:
diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp b/clang/lib/CodeGen/Targets/AArch64.cpp
index 4c32f510101f..452dc049d51b 100644
--- a/clang/lib/CodeGen/Targets/AArch64.cpp
+++ b/clang/lib/CodeGen/Targets/AArch64.cpp
@@ -170,8 +170,22 @@ public:
 
   void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
                             const FunctionDecl *Caller,
-                            const FunctionDecl *Callee,
-                            const CallArgList &Args) const override;
+                            const FunctionDecl *Callee, const CallArgList &Args,
+                            QualType ReturnType) const override;
+
+private:
+  // Diagnose calls between functions with incompatible Streaming SVE
+  // attributes.
+  void checkFunctionCallABIStreaming(CodeGenModule &CGM, SourceLocation CallLoc,
+                                     const FunctionDecl *Caller,
+                                     const FunctionDecl *Callee) const;
+  // Diagnose calls which must pass arguments in floating-point registers when
+  // the selected target does not have floating-point registers.
+  void checkFunctionCallABISoftFloat(CodeGenModule &CGM, SourceLocation CallLoc,
+                                     const FunctionDecl *Caller,
+                                     const FunctionDecl *Callee,
+                                     const CallArgList &Args,
+                                     QualType ReturnType) const;
 };
 
 class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
@@ -853,37 +867,42 @@ static bool isStreamingCompatible(const FunctionDecl *F) {
   return false;
 }
 
+// Report an error if an argument or return value of type Ty would need to be
+// passed in a floating-point register.
+static void diagnoseIfNeedsFPReg(DiagnosticsEngine &Diags,
+                                 const StringRef ABIName,
+                                 const AArch64ABIInfo &ABIInfo,
+                                 const QualType &Ty, const NamedDecl *D) {
+  const Type *HABase = nullptr;
+  uint64_t HAMembers = 0;
+  if (Ty->isFloatingType() || Ty->isVectorType() ||
+      ABIInfo.isHomogeneousAggregate(Ty, HABase, HAMembers)) {
+    Diags.Report(D->getLocation(), diag::err_target_unsupported_type_for_abi)
+        << D->getDeclName() << Ty << ABIName;
+  }
+}
+
+// If we are using a hard-float ABI, but do not have floating point registers,
+// then report an error for any function arguments or returns which would be
+// passed in floating-pint registers.
 void AArch64TargetCodeGenInfo::checkFunctionABI(
     CodeGenModule &CGM, const FunctionDecl *FuncDecl) const {
   const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
   const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
 
-  // If we are using a hard-float ABI, but do not have floating point
-  // registers, then report an error for any function arguments or returns
-  // which would be passed in floating-pint registers.
-  auto CheckType = [&CGM, &TI, &ABIInfo](const QualType &Ty,
-                                         const NamedDecl *D) {
-    const Type *HABase = nullptr;
-    uint64_t HAMembers = 0;
-    if (Ty->isFloatingType() || Ty->isVectorType() ||
-        ABIInfo.isHomogeneousAggregate(Ty, HABase, HAMembers)) {
-      CGM.getDiags().Report(D->getLocation(),
-                            diag::err_target_unsupported_type_for_abi)
-          << D->getDeclName() << Ty << TI.getABI();
-    }
-  };
-
   if (!TI.hasFeature("fp") && !ABIInfo.isSoftFloat()) {
-    CheckType(FuncDecl->getReturnType(), FuncDecl);
+    diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo,
+                         FuncDecl->getReturnType(), FuncDecl);
     for (ParmVarDecl *PVD : FuncDecl->parameters()) {
-      CheckType(PVD->getType(), PVD);
+      diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, PVD->getType(),
+                           PVD);
     }
   }
 }
 
-void AArch64TargetCodeGenInfo::checkFunctionCallABI(
+void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming(
     CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
-    const FunctionDecl *Callee, const CallArgList &Args) const {
+    const FunctionDecl *Callee) const {
   if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>())
     return;
 
@@ -903,6 +922,37 @@ void AArch64TargetCodeGenInfo::checkFunctionCallABI(
           << Callee->getDeclName();
 }
 
+// If the target does not have floating-point registers, but we are using a
+// hard-float ABI, there is no way to pass floating-point, vector or HFA values
+// to functions, so we report an error.
+void AArch64TargetCodeGenInfo::checkFunctionCallABISoftFloat(
+    CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
+    const FunctionDecl *Callee, const CallArgList &Args,
+    QualType ReturnType) const {
+  const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
+  const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
+
+  if (!Caller || TI.hasFeature("fp") || ABIInfo.isSoftFloat())
+    return;
+
+  diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, ReturnType,
+                       Caller);
+
+  for (const CallArg &Arg : Args)
+    diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, Arg.getType(),
+                         Caller);
+}
+
+void AArch64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
+                                                    SourceLocation CallLoc,
+                                                    const FunctionDecl *Caller,
+                                                    const FunctionDecl *Callee,
+                                                    const CallArgList &Args,
+                                                    QualType ReturnType) const {
+  checkFunctionCallABIStreaming(CGM, CallLoc, Caller, Callee);
+  checkFunctionCallABISoftFloat(CGM, CallLoc, Caller, Callee, Args, ReturnType);
+}
+
 void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr,
                                              unsigned Index,
                                              raw_ostream &Out) const {
diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 94cf0d86f9be..717a27fc9c57 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -1482,8 +1482,8 @@ public:
 
   void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
                             const FunctionDecl *Caller,
-                            const FunctionDecl *Callee,
-                            const CallArgList &Args) const override;
+                            const FunctionDecl *Callee, const CallArgList &Args,
+                            QualType ReturnType) const override;
 };
 } // namespace
 
@@ -1558,9 +1558,15 @@ static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
   return false;
 }
 
-void X86_64TargetCodeGenInfo::checkFunctionCallABI(
-    CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
-    const FunctionDecl *Callee, const CallArgList &Args) const {
+void X86_64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
+                                                   SourceLocation CallLoc,
+                                                   const FunctionDecl *Caller,
+                                                   const FunctionDecl *Callee,
+                                                   const CallArgList &Args,
+                                                   QualType ReturnType) const {
+  if (!Callee)
+    return;
+
   llvm::StringMap<bool> CallerMap;
   llvm::StringMap<bool> CalleeMap;
   unsigned ArgIndex = 0;
diff --git a/clang/test/CodeGen/aarch64-soft-float-abi-errors.c b/clang/test/CodeGen/aarch64-soft-float-abi-errors.c
index 33c8d3bcd76f..95b7668aca1b 100644
--- a/clang/test/CodeGen/aarch64-soft-float-abi-errors.c
+++ b/clang/test/CodeGen/aarch64-soft-float-abi-errors.c
@@ -69,6 +69,7 @@ inline void test_float_arg_inline(float a) {}
 inline void test_float_arg_inline_used(float a) {}
 // nofp-hard-opt-error@-1 {{'a' requires 'float' type support, but ABI 'aapcs' does not support it}}
 void use_inline() { test_float_arg_inline_used(1.0f); }
+// nofp-hard-error@-1 {{'use_inline' requires 'float' type support, but ABI 'aapcs' does not support it}}
 
 // The always_inline attribute causes an inline function to always be
 // code-genned, even at -O0, so we always emit the error.
@@ -76,6 +77,7 @@ __attribute((always_inline))
 inline void test_float_arg_always_inline_used(float a) {}
 // nofp-hard-error@-1 {{'a' requires 'float' type support, but ABI 'aapcs' does not support it}}
 void use_always_inline() { test_float_arg_always_inline_used(1.0f); }
+// nofp-hard-error@-1 {{'use_always_inline' requires 'float' type support, but ABI 'aapcs' does not support it}}
 
 // Floating-point expressions, global variables and local variables do not
 // affect the ABI, so are allowed. GCC does reject some uses of floating point
@@ -97,3 +99,25 @@ int test_var_double(int a) {
   d *= 6.0;
   return (int)d;
 }
+
+extern void extern_float_arg(float);
+extern float extern_float_ret(void);
+void call_extern_float_arg() { extern_float_arg(1.0f); }
+// nofp-hard-error@-1 {{'call_extern_float_arg' requires 'float' type support, but ABI 'aapcs' does not support it}}
+void call_extern_float_ret() { extern_float_ret(); }
+// nofp-hard-error@-1 {{'call_extern_float_ret' requires 'float' type support, but ABI 'aapcs' does not support it}}
+
+// Definitions of variadic functions, and calls to them which only use integer
+// argument registers, are both fine.
+void variadic(int, ...);
+void call_variadic_int() { variadic(0, 1); }
+
+// Calls to variadic functions with floating-point arguments are an error,
+// since this would require floating-point registers.
+void call_variadic_double() { variadic(0, 1.0); }
+// nofp-hard-error@-1 {{'call_variadic_double' requires 'double' type support, but ABI 'aapcs' does not support it}}
+
+// Calls through function pointers are also diagnosed.
+void (*fptr)(float);
+void call_indirect() { fptr(1.0f); }
+// nofp-hard-error@-1 {{'call_indirect' requires 'float' type support, but ABI 'aapcs' does not support it}}
-- 
GitLab


From 50da7680d882dac122fac442348649c9951011a0 Mon Sep 17 00:00:00 2001
From: Ben Shi <2283975856@qq.com>
Date: Tue, 7 May 2024 16:42:46 +0800
Subject: [PATCH 0027/1206] [AVR][NFC] Improve format of target description
 files (#91296)

---
 llvm/lib/Target/AVR/AVRInstrInfo.td | 337 ++++++++--------------------
 1 file changed, 90 insertions(+), 247 deletions(-)

diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.td b/llvm/lib/Target/AVR/AVRInstrInfo.td
index 38ebfab64c61..88b1989ef917 100644
--- a/llvm/lib/Target/AVR/AVRInstrInfo.td
+++ b/llvm/lib/Target/AVR/AVRInstrInfo.td
@@ -536,208 +536,95 @@ let Constraints = "$src = $rd", Defs = [SREG] in {
   // Register-Register logic instructions (which have the
   // property of commutativity).
   let isCommutable = 1 in {
-    def ANDRdRr
-        : FRdRr<0b0010, 0b00,
-                (outs GPR8
-                 : $rd),
-                (ins GPR8
-                 : $src, GPR8
-                 : $rr),
-                "and\t$rd, $rr",
-                [(set i8
-                  : $rd, (and i8
-                          : $src, i8
-                          : $rr)),
-                 (implicit SREG)]>;
+    def ANDRdRr : FRdRr<0b0010, 0b00, (outs GPR8:$rd),
+                        (ins GPR8:$src, GPR8:$rr), "and\t$rd, $rr",
+                        [(set i8:$rd, (and i8:$src, i8:$rr)), (implicit SREG)]>;
 
     // ANDW Rd+1:Rd, Rr+1:Rr
     //
     // Expands to:
     // and Rd,   Rr
     // and Rd+1, Rr+1
-    def ANDWRdRr : Pseudo<(outs DREGS
-                           : $rd),
-                          (ins DREGS
-                           : $src, DREGS
-                           : $rr),
-                          "andw\t$rd, $rr", [
-                            (set i16
-                             : $rd, (and i16
-                                     : $src, i16
-                                     : $rr)),
-                            (implicit SREG)
-                          ]>;
-
-    def ORRdRr
-        : FRdRr<0b0010, 0b10,
-                (outs GPR8
-                 : $rd),
-                (ins GPR8
-                 : $src, GPR8
-                 : $rr),
-                "or\t$rd, $rr",
-                [(set i8
-                  : $rd, (or i8
-                          : $src, i8
-                          : $rr)),
-                 (implicit SREG)]>;
+    def ANDWRdRr : Pseudo<(outs DREGS:$rd), (ins DREGS:$src, DREGS:$rr),
+                          "andw\t$rd, $rr",
+                          [(set i16:$rd, (and i16:$src, i16:$rr)),
+                           (implicit SREG)]>;
+
+    def ORRdRr : FRdRr<0b0010, 0b10, (outs GPR8:$rd), (ins GPR8:$src, GPR8:$rr),
+                       "or\t$rd, $rr",
+                       [(set i8:$rd, (or i8:$src, i8:$rr)), (implicit SREG)]>;
 
     // ORW Rd+1:Rd, Rr+1:Rr
     //
     // Expands to:
     // or Rd,   Rr
     // or Rd+1, Rr+1
-    def ORWRdRr : Pseudo<(outs DREGS
-                          : $rd),
-                         (ins DREGS
-                          : $src, DREGS
-                          : $rr),
-                         "orw\t$rd, $rr", [
-                           (set i16
-                            : $rd, (or i16
-                                    : $src, i16
-                                    : $rr)),
-                           (implicit SREG)
-                         ]>;
-
-    def EORRdRr
-        : FRdRr<0b0010, 0b01,
-                (outs GPR8
-                 : $rd),
-                (ins GPR8
-                 : $src, GPR8
-                 : $rr),
-                "eor\t$rd, $rr",
-                [(set i8
-                  : $rd, (xor i8
-                          : $src, i8
-                          : $rr)),
-                 (implicit SREG)]>;
+    def ORWRdRr : Pseudo<(outs DREGS:$rd), (ins DREGS:$src, DREGS:$rr),
+                         "orw\t$rd, $rr",
+                         [(set i16:$rd, (or i16:$src, i16:$rr)),
+                          (implicit SREG)]>;
+
+    def EORRdRr : FRdRr<0b0010, 0b01, (outs GPR8:$rd),
+                        (ins GPR8:$src, GPR8:$rr), "eor\t$rd, $rr",
+                        [(set i8:$rd, (xor i8:$src, i8:$rr)), (implicit SREG)]>;
 
     // EORW Rd+1:Rd, Rr+1:Rr
     //
     // Expands to:
     // eor Rd,   Rr
     // eor Rd+1, Rr+1
-    def EORWRdRr : Pseudo<(outs DREGS
-                           : $rd),
-                          (ins DREGS
-                           : $src, DREGS
-                           : $rr),
-                          "eorw\t$rd, $rr", [
-                            (set i16
-                             : $rd, (xor i16
-                                     : $src, i16
-                                     : $rr)),
-                            (implicit SREG)
-                          ]>;
+    def EORWRdRr : Pseudo<(outs DREGS:$rd), (ins DREGS:$src, DREGS:$rr),
+                          "eorw\t$rd, $rr",
+                          [(set i16:$rd, (xor i16:$src, i16:$rr)),
+                           (implicit SREG)]>;
   }
 
-  def ANDIRdK
-      : FRdK<0b0111,
-             (outs LD8
-              : $rd),
-             (ins LD8
-              : $src, imm_ldi8
-              : $k),
-             "andi\t$rd, $k",
-             [(set i8
-               : $rd, (and i8
-                       : $src, imm
-                       : $k)),
-              (implicit SREG)]>;
+  def ANDIRdK : FRdK<0b0111, (outs LD8:$rd), (ins LD8:$src, imm_ldi8:$k),
+                     "andi\t$rd, $k",
+                     [(set i8:$rd, (and i8:$src, imm:$k)), (implicit SREG)]>;
 
   // ANDI Rd+1:Rd, K+1:K
   //
   // Expands to:
   // andi Rd,   K
   // andi Rd+1, K+1
-  def ANDIWRdK
-      : Pseudo<(outs DLDREGS
-                : $rd),
-               (ins DLDREGS
-                : $src, i16imm
-                : $k),
-               "andiw\t$rd, $k",
-               [(set i16
-                 : $rd, (and i16
-                         : $src, imm
-                         : $k)),
-                (implicit SREG)]>;
-
-  def ORIRdK
-      : FRdK<0b0110,
-             (outs LD8
-              : $rd),
-             (ins LD8
-              : $src, imm_ldi8
-              : $k),
-             "ori\t$rd, $k",
-             [(set i8
-               : $rd, (or i8
-                       : $src, imm
-                       : $k)),
-              (implicit SREG)]>;
+  def ANDIWRdK : Pseudo<(outs DLDREGS:$rd), (ins DLDREGS:$src, i16imm:$k),
+                        "andiw\t$rd, $k",
+                        [(set i16:$rd, (and i16:$src, imm:$k)),
+                         (implicit SREG)]>;
+
+  def ORIRdK : FRdK<0b0110, (outs LD8:$rd), (ins LD8:$src, imm_ldi8:$k),
+                    "ori\t$rd, $k",
+                    [(set i8:$rd, (or i8:$src, imm:$k)), (implicit SREG)]>;
 
   // ORIW Rd+1:Rd, K+1,K
   //
   // Expands to:
   // ori Rd,   K
   // ori Rd+1, K+1
-  def ORIWRdK
-      : Pseudo<(outs DLDREGS
-                : $rd),
-               (ins DLDREGS
-                : $src, i16imm
-                : $rr),
-               "oriw\t$rd, $rr",
-               [(set i16
-                 : $rd, (or i16
-                         : $src, imm
-                         : $rr)),
-                (implicit SREG)]>;
+  def ORIWRdK : Pseudo<(outs DLDREGS:$rd), (ins DLDREGS:$src, i16imm:$rr),
+                       "oriw\t$rd, $rr",
+                       [(set i16:$rd, (or i16:$src, imm:$rr)),
+                        (implicit SREG)]>;
 }
 
 //===----------------------------------------------------------------------===//
 // One's/Two's Complement
 //===----------------------------------------------------------------------===//
 let Constraints = "$src = $rd", Defs = [SREG] in {
-  def COMRd
-      : FRd<0b1001, 0b0100000,
-            (outs GPR8
-             : $rd),
-            (ins GPR8
-             : $src),
-            "com\t$rd", [(set i8
-                          : $rd, (not i8
-                                  : $src)),
-                         (implicit SREG)]>;
+  def COMRd : FRd<0b1001, 0b0100000, (outs GPR8:$rd), (ins GPR8:$src),
+                  "com\t$rd", [(set i8:$rd, (not i8:$src)), (implicit SREG)]>;
 
   // COMW Rd+1:Rd
   //
   // Expands to:
   // com Rd
   // com Rd+1
-  def COMWRd : Pseudo<(outs DREGS
-                       : $rd),
-                      (ins DREGS
-                       : $src),
-                      "comw\t$rd",
-                      [(set i16
-                        : $rd, (not i16
-                                : $src)),
-                       (implicit SREG)]>;
+  def COMWRd : Pseudo<(outs DREGS:$rd), (ins DREGS:$src), "comw\t$rd",
+                      [(set i16:$rd, (not i16:$src)), (implicit SREG)]>;
 
-  def NEGRd
-      : FRd<0b1001, 0b0100001,
-            (outs GPR8
-             : $rd),
-            (ins GPR8
-             : $src),
-            "neg\t$rd", [(set i8
-                          : $rd, (ineg i8
-                                  : $src)),
-                         (implicit SREG)]>;
+  def NEGRd : FRd<0b1001, 0b0100001, (outs GPR8:$rd), (ins GPR8:$src),
+                  "neg\t$rd", [(set i8:$rd, (ineg i8:$src)), (implicit SREG)]>;
 
   // NEGW Rd+1:Rd
   //
@@ -746,51 +633,37 @@ let Constraints = "$src = $rd", Defs = [SREG] in {
   // neg Rd
   // sbc Rd+1, r1
   let hasSideEffects=0 in
-  def NEGWRd : Pseudo<(outs DREGS:$rd),
-                      (ins DREGS:$src, GPR8:$zero),
-                      "negw\t$rd",
-                      []>;
+  def NEGWRd : Pseudo<(outs DREGS:$rd), (ins DREGS:$src, GPR8:$zero),
+                      "negw\t$rd", []>;
 }
 
 // TST Rd
 // Test for zero of minus.
 // This operation is identical to a `Rd AND Rd`.
-def : InstAlias<"tst\t$rd", (ANDRdRr GPR8 : $rd, GPR8 : $rd)>;
+def : InstAlias<"tst\t$rd", (ANDRdRr GPR8:$rd, GPR8:$rd)>;
 
 // SBR Rd, K
 //
 // Mnemonic alias to 'ORI Rd, K'. Same bit pattern, same operands,
 // same everything.
-def : InstAlias<"sbr\t$rd, $k",
-                (ORIRdK LD8
-                 : $rd, imm_ldi8
-                 : $k),
+def : InstAlias<"sbr\t$rd, $k", (ORIRdK LD8:$rd, imm_ldi8:$k),
                 /* Disable display, so we don't override ORI */ 0>;
 
 //===----------------------------------------------------------------------===//
 // Jump instructions
 //===----------------------------------------------------------------------===//
 let isBarrier = 1, isBranch = 1, isTerminator = 1 in {
-  def RJMPk : FBRk<0, (outs),
-                   (ins brtarget_13
-                    : $k),
-                   "rjmp\t$k", [(br bb
-                                 : $k)]>;
-
-  let isIndirectBranch = 1,
-      Uses = [R31R30] in def IJMP
-      : F16<0b1001010000001001, (outs), (ins), "ijmp", []>,
-      Requires<[HasIJMPCALL]>;
-
-  let isIndirectBranch = 1,
-      Uses = [R31R30] in def EIJMP
-      : F16<0b1001010000011001, (outs), (ins), "eijmp", []>,
-      Requires<[HasEIJMPCALL]>;
+  def RJMPk : FBRk<0, (outs), (ins brtarget_13:$k), "rjmp\t$k", [(br bb:$k)]>;
 
-  def JMPk : F32BRk<0b110, (outs),
-                    (ins call_target
-                     : $k),
-                    "jmp\t$k", []>,
+  let isIndirectBranch = 1, Uses = [R31R30] in
+  def IJMP : F16<0b1001010000001001, (outs), (ins), "ijmp", []>,
+             Requires<[HasIJMPCALL]>;
+
+  let isIndirectBranch = 1, Uses = [R31R30] in
+  def EIJMP : F16<0b1001010000011001, (outs), (ins), "eijmp", []>,
+              Requires<[HasEIJMPCALL]>;
+
+  def JMPk : F32BRk<0b110, (outs), (ins call_target:$k), "jmp\t$k", []>,
              Requires<[HasJMPCALL]>;
 }
 
@@ -800,19 +673,21 @@ let isBarrier = 1, isBranch = 1, isTerminator = 1 in {
 let isCall = 1 in {
   // SP is marked as a use to prevent stack-pointer assignments that appear
   // immediately before calls from potentially appearing dead.
-  let Uses = [SP] in def RCALLk : FBRk<1, (outs), (ins rcalltarget_13:$k),
-                                       "rcall\t$k", [(AVRcall imm:$k)]>;
+  let Uses = [SP] in
+  def RCALLk : FBRk<1, (outs), (ins rcalltarget_13:$k), "rcall\t$k",
+                    [(AVRcall imm:$k)]>;
 
   // SP is marked as a use to prevent stack-pointer assignments that appear
   // immediately before calls from potentially appearing dead.
-  let Uses = [SP, R31R30] in def ICALL
-      : F16<0b1001010100001001, (outs), (ins variable_ops), "icall", []>,
-      Requires<[HasIJMPCALL]>;
+  let Uses = [SP, R31R30] in
+  def ICALL : F16<0b1001010100001001, (outs), (ins variable_ops), "icall", []>,
+              Requires<[HasIJMPCALL]>;
 
   // SP is marked as a use to prevent stack-pointer assignments that appear
   // immediately before calls from potentially appearing dead.
-  let Uses = [SP, R31R30] in def EICALL
-      : F16<0b1001010100011001, (outs), (ins variable_ops), "eicall", []>,
+  let Uses = [SP, R31R30] in
+  def EICALL : F16<0b1001010100011001, (outs), (ins variable_ops), "eicall",
+                   []>,
       Requires<[HasEIJMPCALL]>;
 
   // SP is marked as a use to prevent stack-pointer assignments that appear
@@ -820,9 +695,10 @@ let isCall = 1 in {
   //
   // TODO: the imm field can be either 16 or 22 bits in devices with more
   // than 64k of ROM, fix it once we support the largest devices.
-  let Uses = [SP] in def CALLk : F32BRk<0b111, (outs), (ins call_target:$k),
-                                        "call\t$k", [(AVRcall imm:$k)]>,
-      Requires<[HasJMPCALL]>;
+  let Uses = [SP] in
+  def CALLk : F32BRk<0b111, (outs), (ins call_target:$k), "call\t$k",
+                     [(AVRcall imm:$k)]>,
+              Requires<[HasJMPCALL]>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -840,75 +716,42 @@ let isTerminator = 1, isReturn = 1, isBarrier = 1 in {
 let Defs = [SREG] in {
   // CPSE Rd, Rr
   // Compare Rd and Rr, skipping the next instruction if they are equal.
-  let isBarrier = 1, isBranch = 1,
-      isTerminator = 1 in def CPSE : FRdRr<0b0001, 0b00, (outs),
-                                           (ins GPR8
-                                            : $rd, GPR8
-                                            : $rr),
-                                           "cpse\t$rd, $rr", []>;
-
-  def CPRdRr
-      : FRdRr<0b0001, 0b01, (outs),
-              (ins GPR8
-               : $rd, GPR8
-               : $rr),
-              "cp\t$rd, $rr", [(AVRcmp i8
-                                : $rd, i8
-                                : $rr),
-                               (implicit SREG)]>;
+  let isBarrier = 1, isBranch = 1, isTerminator = 1 in
+  def CPSE : FRdRr<0b0001, 0b00, (outs), (ins GPR8:$rd, GPR8:$rr),
+                   "cpse\t$rd, $rr", []>;
+
+  def CPRdRr : FRdRr<0b0001, 0b01, (outs), (ins GPR8:$rd, GPR8:$rr),
+                     "cp\t$rd, $rr",
+                     [(AVRcmp i8:$rd, i8:$rr), (implicit SREG)]>;
 
   // CPW Rd+1:Rd, Rr+1:Rr
   //
   // Expands to:
   // cp  Rd,   Rr
   // cpc Rd+1, Rr+1
-  def CPWRdRr : Pseudo<(outs),
-                       (ins DREGS
-                        : $src, DREGS
-                        : $src2),
+  def CPWRdRr : Pseudo<(outs), (ins DREGS:$src, DREGS:$src2),
                        "cpw\t$src, $src2",
-                       [(AVRcmp i16
-                         : $src, i16
-                         : $src2),
-                        (implicit SREG)]>;
+                       [(AVRcmp i16:$src, i16:$src2), (implicit SREG)]>;
 
-  let Uses = [SREG] in def CPCRdRr
-      : FRdRr<0b0000, 0b01, (outs),
-              (ins GPR8
-               : $rd, GPR8
-               : $rr),
-              "cpc\t$rd, $rr", [(AVRcmpc i8
-                                 : $rd, i8
-                                 : $rr),
-                                (implicit SREG)]>;
+  let Uses = [SREG] in
+  def CPCRdRr : FRdRr<0b0000, 0b01, (outs), (ins GPR8:$rd, GPR8:$rr),
+                      "cpc\t$rd, $rr",
+                      [(AVRcmpc i8:$rd, i8:$rr), (implicit SREG)]>;
 
   // CPCW Rd+1:Rd. Rr+1:Rr
   //
   // Expands to:
   // cpc Rd,   Rr
   // cpc Rd+1, Rr+1
-  let Uses = [SREG] in def CPCWRdRr
-      : Pseudo<(outs),
-               (ins DREGS
-                : $src, DREGS
-                : $src2),
-               "cpcw\t$src, $src2",
-               [(AVRcmpc i16
-                 : $src, i16
-                 : $src2),
-                (implicit SREG)]>;
+  let Uses = [SREG] in
+  def CPCWRdRr : Pseudo<(outs), (ins DREGS:$src, DREGS:$src2),
+                        "cpcw\t$src, $src2",
+                        [(AVRcmpc i16:$src, i16:$src2), (implicit SREG)]>;
 
   // CPI Rd, K
   // Compares a register with an 8 bit immediate.
-  def CPIRdK
-      : FRdK<0b0011, (outs),
-             (ins LD8
-              : $rd, imm_ldi8
-              : $k),
-             "cpi\t$rd, $k", [(AVRcmp i8
-                               : $rd, imm
-                               : $k),
-                              (implicit SREG)]>;
+  def CPIRdK : FRdK<0b0011, (outs), (ins LD8:$rd, imm_ldi8:$k), "cpi\t$rd, $k",
+                    [(AVRcmp i8:$rd, imm:$k), (implicit SREG)]>;
 }
 
 //===----------------------------------------------------------------------===//
-- 
GitLab


From 6aed0ab6547f577cceaccfc6d710f96b645c3af7 Mon Sep 17 00:00:00 2001
From: Anthony Ha <anthonyha96@gmail.com>
Date: Tue, 7 May 2024 01:45:07 -0700
Subject: [PATCH 0028/1206] [lldb] Have lldb-server assign ports to children in
 platform mode (#88845)

Fixes #47549

`lldb-server`'s platform mode seems to have an issue with its
`--min-gdbserver-port` `--max-gdbserver-port` flags (and probably the
`--gdbserver-port` flag, but I didn't test it).

How the platform code seems to work is that it listens on a port, and
whenever there's an incoming connection, it forks the process to handle
the connection. To handle the port flags, the main process uses an
instance of the helper class
`GDBRemoteCommunicationServerPlatform::PortMap`, that can be configured
and track usages of ports. The child process handling the platform
connection, can then use the port map to allocate a port for the
gdb-server connection it will make (this is another process it spawns).

However, in the current code, this works only once. After the first
connection is handled by forking a child process, the main platform
listener code loops around, and then 'forgets' about the port map. This
is because this code:
```cpp
GDBRemoteCommunicationServerPlatform platform(
    acceptor_up->GetSocketProtocol(), acceptor_up->GetSocketScheme());
if (!gdbserver_portmap.empty()) {
  platform.SetPortMap(std::move(gdbserver_portmap));
}
```
is within the connection listening loop. This results in the
`gdbserver_portmap` being moved into the platform object at the
beginning of the first iteration of the loop, but on the second
iteration, after the first fork, the next instance of the platform
object will not have its platform port mapped.
The result of this bug is that subsequent connections to the platform,
when spawning the gdb-remote connection, will be supplied a random port
- which isn't bounded by the `--min-gdbserver-port` and
`--max-gdbserver--port` parameters passed in by the user.

This PR fixes this issue by having the port map be maintained by the
parent platform listener process. On connection, the listener allocates
a single available port from the port map, associates the child process
pid with the port, and lets the connection handling child use that
single port number.

Additionally, when cleaning up child processes, the main listener
process tracks the child that exited to deallocate the previously
associated port, so it can be reused for a new connection.
---
 lldb/docs/use/qemu-testing.rst           |  3 +-
 lldb/tools/lldb-server/lldb-platform.cpp | 49 +++++++++++++++++-------
 2 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/lldb/docs/use/qemu-testing.rst b/lldb/docs/use/qemu-testing.rst
index 6e282141864c..51a30b11717a 100644
--- a/lldb/docs/use/qemu-testing.rst
+++ b/lldb/docs/use/qemu-testing.rst
@@ -172,6 +172,7 @@ forwarded for this to work.
 
 .. note::
   These options are used to create a "port map" within ``lldb-server``.
-  Unfortunately this map is not shared across all the processes it may create,
+  Unfortunately this map is not cleaned up on Windows on connection close,
   and across a few uses you may run out of valid ports. To work around this,
   restart the platform every so often, especially after running a set of tests.
+  This is tracked here: https://github.com/llvm/llvm-project/issues/90923
diff --git a/lldb/tools/lldb-server/lldb-platform.cpp b/lldb/tools/lldb-server/lldb-platform.cpp
index 3e126584eb25..cfd0a3797d81 100644
--- a/lldb/tools/lldb-server/lldb-platform.cpp
+++ b/lldb/tools/lldb-server/lldb-platform.cpp
@@ -282,17 +282,12 @@ int main_platform(int argc, char *argv[]) {
     }
   }
 
-  do {
-    GDBRemoteCommunicationServerPlatform platform(
-        acceptor_up->GetSocketProtocol(), acceptor_up->GetSocketScheme());
-
-    if (port_offset > 0)
-      platform.SetPortOffset(port_offset);
-
-    if (!gdbserver_portmap.empty()) {
-      platform.SetPortMap(std::move(gdbserver_portmap));
-    }
+  GDBRemoteCommunicationServerPlatform platform(
+      acceptor_up->GetSocketProtocol(), acceptor_up->GetSocketScheme());
+  if (port_offset > 0)
+    platform.SetPortOffset(port_offset);
 
+  do {
     const bool children_inherit_accept_socket = true;
     Connection *conn = nullptr;
     error = acceptor_up->Accept(children_inherit_accept_socket, conn);
@@ -301,13 +296,37 @@ int main_platform(int argc, char *argv[]) {
       exit(socket_error);
     }
     printf("Connection established.\n");
+
     if (g_server) {
       // Collect child zombie processes.
 #if !defined(_WIN32)
-      while (waitpid(-1, nullptr, WNOHANG) > 0)
-        ;
+      ::pid_t waitResult;
+      while ((waitResult = waitpid(-1, nullptr, WNOHANG)) > 0) {
+        // waitResult is the child pid
+        gdbserver_portmap.FreePortForProcess(waitResult);
+      }
 #endif
-      if (fork()) {
+      // TODO: Clean up portmap for Windows when children die
+      // See https://github.com/llvm/llvm-project/issues/90923
+
+      // After collecting zombie ports, get the next available
+      GDBRemoteCommunicationServerPlatform::PortMap portmap_for_child;
+      llvm::Expected<uint16_t> available_port =
+          gdbserver_portmap.GetNextAvailablePort();
+      if (available_port)
+        portmap_for_child.AllowPort(*available_port);
+      else {
+        llvm::consumeError(available_port.takeError());
+        fprintf(stderr,
+                "no available gdbserver port for connection - dropping...\n");
+        delete conn;
+        continue;
+      }
+      platform.SetPortMap(std::move(portmap_for_child));
+
+      auto childPid = fork();
+      if (childPid) {
+        gdbserver_portmap.AssociatePortWithProcess(*available_port, childPid);
         // Parent doesn't need a connection to the lldb client
         delete conn;
 
@@ -323,7 +342,11 @@ int main_platform(int argc, char *argv[]) {
       // If not running as a server, this process will not accept
       // connections while a connection is active.
       acceptor_up.reset();
+
+      // When not running in server mode, use all available ports
+      platform.SetPortMap(std::move(gdbserver_portmap));
     }
+
     platform.SetConnection(std::unique_ptr<Connection>(conn));
 
     if (platform.IsConnected()) {
-- 
GitLab


From d838e5b3e86e7b3b4b2f75ee9c2854e23782888e Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 7 May 2024 10:00:00 +0100
Subject: [PATCH 0029/1206] [X86] Add FastImm16 tuning flag to Intel Atom + AMD
 Bobcat/Ryzen Families (#90635)

This patch limits the icmp_i16(x,c) -> icmp_i32(ext(x),ext(c)) fold to CPUs that aren't known to have fast handling for length-changing prefixes for imm16 operands.

We are always assuming that 66/67h length-changing prefixes cause severe stalls and we should always extend imm16 operands and use a i32 icmp instead, the only exception being Intel Bonnell CPUs.

Agner makes this clear (see microarchitecture.pdf) that there are no stalls for any of the Intel Atom family (at least as far as Tremont - not sure about Gracemont or later). This is also true for AMD Bobcat/Jaguar and Ryzen families.

Recent performance Intel CPUs are trickier - Core2/Nehalem and earlier could have a 6-11cy stall, while SandyBridge onwards this is reduced to 3cy or less. I'm not sure if we should accept this as fast or not, we only use this flag for the icmp_i16 case, so that might be acceptable? If so, we should add this to x86-64-v3/v4 tuning as well.

Part of #90355 + #62952
---
 llvm/lib/Target/X86/X86.td              |  12 ++
 llvm/lib/Target/X86/X86ISelLowering.cpp |   2 +-
 llvm/test/CodeGen/X86/cmp16.ll          | 239 ++++++++++++++++++++++++
 3 files changed, 252 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 9e731947893d..25ab08187cf1 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -739,6 +739,10 @@ def TuningFastMOVBE
     : SubtargetFeature<"fast-movbe", "HasFastMOVBE", "true",
     "Prefer a movbe over a single-use load + bswap / single-use bswap + store">;
 
+def TuningFastImm16
+    : SubtargetFeature<"fast-imm16", "HasFastImm16", "true",
+    "Prefer a i16 instruction with i16 immediate over extension to i32">;
+
 def TuningUseSLMArithCosts
     : SubtargetFeature<"use-slm-arith-costs", "UseSLMArithCosts", "true",
         "Use Silvermont specific arithmetic costs">;
@@ -1146,6 +1150,7 @@ def ProcessorFeatures {
                                        TuningSlowDivide32,
                                        TuningSlowDivide64,
                                        TuningSlowTwoMemOps,
+                                       TuningFastImm16,
                                        TuningLEAUsesAG,
                                        TuningPadShortFunctions,
                                        TuningInsertVZEROUPPER,
@@ -1166,6 +1171,7 @@ def ProcessorFeatures {
                                       TuningSlowPMULLD,
                                       TuningFast7ByteNOP,
                                       TuningFastMOVBE,
+                                      TuningFastImm16,
                                       TuningPOPCNTFalseDeps,
                                       TuningInsertVZEROUPPER,
                                       TuningNoDomainDelay];
@@ -1187,6 +1193,7 @@ def ProcessorFeatures {
                                       TuningSlowLEA,
                                       TuningSlowIncDec,
                                       TuningFastMOVBE,
+                                      TuningFastImm16,
                                       TuningPOPCNTFalseDeps,
                                       TuningInsertVZEROUPPER,
                                       TuningNoDomainDelay];
@@ -1201,6 +1208,7 @@ def ProcessorFeatures {
                                       TuningSlowLEA,
                                       TuningSlowIncDec,
                                       TuningFastMOVBE,
+                                      TuningFastImm16,
                                       TuningInsertVZEROUPPER,
                                       TuningNoDomainDelay];
   list<SubtargetFeature> GLPFeatures =
@@ -1321,6 +1329,7 @@ def ProcessorFeatures {
                                       TuningPreferMaskRegisters,
                                       TuningFastGather,
                                       TuningFastMOVBE,
+                                      TuningFastImm16,
                                       TuningSlowPMADDWD];
   // TODO Add AVX5124FMAPS/AVX5124VNNIW features
   list<SubtargetFeature> KNMFeatures =
@@ -1364,6 +1373,7 @@ def ProcessorFeatures {
                                          TuningFastScalarShiftMasks,
                                          TuningFastVectorShiftMasks,
                                          TuningSlowSHLD,
+                                         TuningFastImm16,
                                          TuningSBBDepBreaking,
                                          TuningInsertVZEROUPPER];
 
@@ -1384,6 +1394,7 @@ def ProcessorFeatures {
                                          TuningFastScalarShiftMasks,
                                          TuningFastVectorShiftMasks,
                                          TuningFastMOVBE,
+                                         TuningFastImm16,
                                          TuningSBBDepBreaking,
                                          TuningSlowSHLD];
   list<SubtargetFeature> BtVer2Features =
@@ -1488,6 +1499,7 @@ def ProcessorFeatures {
                                      TuningFastScalarShiftMasks,
                                      TuningFastVariablePerLaneShuffle,
                                      TuningFastMOVBE,
+                                     TuningFastImm16,
                                      TuningSlowSHLD,
                                      TuningSBBDepBreaking,
                                      TuningInsertVZEROUPPER,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 4638f7b70358..b7c14e50210e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -22690,7 +22690,7 @@ static SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
 
   // Only promote the compare up to I32 if it is a 16 bit operation
   // with an immediate.  16 bit immediates are to be avoided.
-  if (CmpVT == MVT::i16 && !Subtarget.isAtom() &&
+  if (CmpVT == MVT::i16 && !Subtarget.hasFastImm16() &&
       !DAG.getMachineFunction().getFunction().hasMinSize()) {
     ConstantSDNode *COp0 = dyn_cast<ConstantSDNode>(Op0);
     ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1);
diff --git a/llvm/test/CodeGen/X86/cmp16.ll b/llvm/test/CodeGen/X86/cmp16.ll
index 760c8e404499..699ea3e4dd47 100644
--- a/llvm/test/CodeGen/X86/cmp16.ll
+++ b/llvm/test/CodeGen/X86/cmp16.ll
@@ -1,8 +1,18 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc < %s -mtriple=i686-- | FileCheck %s --check-prefixes=X86,X86-GENERIC
 ; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s --check-prefixes=X64,X64-GENERIC
+; RUN: llc < %s -mtriple=i686-- -mattr=+fast-imm16 | FileCheck %s --check-prefixes=X86,X86-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+fast-imm16 | FileCheck %s --check-prefixes=X64,X64-FAST
 ; RUN: llc < %s -mtriple=i686-- -mcpu=atom | FileCheck %s --check-prefixes=X86,X86-ATOM
 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=atom | FileCheck %s --check-prefixes=X64,X64-ATOM
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=slm | FileCheck %s --check-prefixes=X64,X64-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=knl | FileCheck %s --check-prefixes=X64,X64-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=btver1 | FileCheck %s --check-prefixes=X64,X64-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=btver2 | FileCheck %s --check-prefixes=X64,X64-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver1 | FileCheck %s --check-prefixes=X64,X64-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver2 | FileCheck %s --check-prefixes=X64,X64-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=X64,X64-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=X64,X64-FAST
 
 define i1 @cmp16_reg_eq_reg(i16 %a0, i16 %a1) {
 ; X86-GENERIC-LABEL: cmp16_reg_eq_reg:
@@ -18,6 +28,19 @@ define i1 @cmp16_reg_eq_reg(i16 %a0, i16 %a1) {
 ; X64-GENERIC-NEXT:    sete %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_reg_eq_reg:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT:    cmpw {{[0-9]+}}(%esp), %ax
+; X86-FAST-NEXT:    sete %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_reg_eq_reg:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw %si, %di
+; X64-FAST-NEXT:    sete %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_reg_eq_reg:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
@@ -52,6 +75,18 @@ define i1 @cmp16_reg_eq_imm8(i16 %a0) {
 ; X64-GENERIC-NEXT:    sete %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_reg_eq_imm8:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    cmpw $15, {{[0-9]+}}(%esp)
+; X86-FAST-NEXT:    sete %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_reg_eq_imm8:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $15, %di
+; X64-FAST-NEXT:    sete %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_reg_eq_imm8:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    cmpw $15, {{[0-9]+}}(%esp)
@@ -90,6 +125,18 @@ define i1 @cmp16_reg_eq_imm16(i16 %a0) {
 ; X64-GENERIC-NEXT:    sete %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_reg_eq_imm16:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    cmpw $1024, {{[0-9]+}}(%esp) # imm = 0x400
+; X86-FAST-NEXT:    sete %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_reg_eq_imm16:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $1024, %di # imm = 0x400
+; X64-FAST-NEXT:    sete %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_reg_eq_imm16:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    cmpw $1024, {{[0-9]+}}(%esp) # imm = 0x400
@@ -144,6 +191,18 @@ define i1 @cmp16_reg_eq_imm16_optsize(i16 %a0) optsize {
 ; X64-GENERIC-NEXT:    sete %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_reg_eq_imm16_optsize:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    cmpw $1024, {{[0-9]+}}(%esp) # imm = 0x400
+; X86-FAST-NEXT:    sete %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_reg_eq_imm16_optsize:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $1024, %di # imm = 0x400
+; X64-FAST-NEXT:    sete %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_reg_eq_imm16_optsize:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    cmpw $1024, {{[0-9]+}}(%esp) # imm = 0x400
@@ -172,6 +231,18 @@ define i1 @cmp16_reg_sgt_imm8(i16 %a0) {
 ; X64-GENERIC-NEXT:    setge %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_reg_sgt_imm8:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    cmpw $16, {{[0-9]+}}(%esp)
+; X86-FAST-NEXT:    setge %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_reg_sgt_imm8:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $16, %di
+; X64-FAST-NEXT:    setge %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_reg_sgt_imm8:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    cmpw $16, {{[0-9]+}}(%esp)
@@ -210,6 +281,18 @@ define i1 @cmp16_reg_sgt_imm16(i16 %a0) {
 ; X64-GENERIC-NEXT:    setge %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_reg_sgt_imm16:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    cmpw $-1023, {{[0-9]+}}(%esp) # imm = 0xFC01
+; X86-FAST-NEXT:    setge %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_reg_sgt_imm16:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $-1023, %di # imm = 0xFC01
+; X64-FAST-NEXT:    setge %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_reg_sgt_imm16:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    cmpw $-1023, {{[0-9]+}}(%esp) # imm = 0xFC01
@@ -264,6 +347,18 @@ define i1 @cmp16_reg_sgt_imm16_optsize(i16 %a0) optsize {
 ; X64-GENERIC-NEXT:    setge %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_reg_sgt_imm16_optsize:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    cmpw $-1023, {{[0-9]+}}(%esp) # imm = 0xFC01
+; X86-FAST-NEXT:    setge %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_reg_sgt_imm16_optsize:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $-1023, %di # imm = 0xFC01
+; X64-FAST-NEXT:    setge %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_reg_sgt_imm16_optsize:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    cmpw $-1023, {{[0-9]+}}(%esp) # imm = 0xFC01
@@ -294,6 +389,18 @@ define i1 @cmp16_reg_uge_imm16(i16 %a0) {
 ; X64-GENERIC-NEXT:    setae %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_reg_uge_imm16:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    cmpw $-1024, {{[0-9]+}}(%esp) # imm = 0xFC00
+; X86-FAST-NEXT:    setae %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_reg_uge_imm16:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $-1024, %di # imm = 0xFC00
+; X64-FAST-NEXT:    setae %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_reg_uge_imm16:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    cmpw $-1024, {{[0-9]+}}(%esp) # imm = 0xFC00
@@ -348,6 +455,18 @@ define i1 @cmp16_reg_uge_imm16_optsize(i16 %a0) optsize {
 ; X64-GENERIC-NEXT:    setae %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_reg_uge_imm16_optsize:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    cmpw $-1024, {{[0-9]+}}(%esp) # imm = 0xFC00
+; X86-FAST-NEXT:    setae %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_reg_uge_imm16_optsize:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $-1024, %di # imm = 0xFC00
+; X64-FAST-NEXT:    setae %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_reg_uge_imm16_optsize:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    cmpw $-1024, {{[0-9]+}}(%esp) # imm = 0xFC00
@@ -380,6 +499,22 @@ define i1 @cmp16_load_ne_load(ptr %p0, ptr %p1) {
 ; X64-GENERIC-NEXT:    setne %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_load_ne_load:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-NEXT:    movzwl (%ecx), %ecx
+; X86-FAST-NEXT:    cmpw (%eax), %cx
+; X86-FAST-NEXT:    setne %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_load_ne_load:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    movzwl (%rdi), %eax
+; X64-FAST-NEXT:    cmpw (%rsi), %ax
+; X64-FAST-NEXT:    setne %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_load_ne_load:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
@@ -417,6 +552,19 @@ define i1 @cmp16_load_ne_imm8(ptr %p0) {
 ; X64-GENERIC-NEXT:    setne %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_load_ne_imm8:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT:    cmpw $15, (%eax)
+; X86-FAST-NEXT:    setne %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_load_ne_imm8:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $15, (%rdi)
+; X64-FAST-NEXT:    setne %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_load_ne_imm8:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -456,6 +604,19 @@ define i1 @cmp16_load_ne_imm16(ptr %p0) {
 ; X64-GENERIC-NEXT:    setne %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_load_ne_imm16:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT:    cmpw $512, (%eax) # imm = 0x200
+; X86-FAST-NEXT:    setne %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_load_ne_imm16:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $512, (%rdi) # imm = 0x200
+; X64-FAST-NEXT:    setne %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_load_ne_imm16:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -493,6 +654,19 @@ define i1 @cmp16_load_slt_imm8(ptr %p0) {
 ; X64-GENERIC-NEXT:    setl %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_load_slt_imm8:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT:    cmpw $15, (%eax)
+; X86-FAST-NEXT:    setl %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_load_slt_imm8:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $15, (%rdi)
+; X64-FAST-NEXT:    setl %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_load_slt_imm8:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -532,6 +706,19 @@ define i1 @cmp16_load_slt_imm16(ptr %p0) {
 ; X64-GENERIC-NEXT:    setl %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_load_slt_imm16:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT:    cmpw $512, (%eax) # imm = 0x200
+; X86-FAST-NEXT:    setl %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_load_slt_imm16:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $512, (%rdi) # imm = 0x200
+; X64-FAST-NEXT:    setl %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_load_slt_imm16:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -589,6 +776,19 @@ define i1 @cmp16_load_slt_imm16_optsize(ptr %p0) optsize {
 ; X64-GENERIC-NEXT:    setl %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_load_slt_imm16_optsize:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT:    cmpw $512, (%eax) # imm = 0x200
+; X86-FAST-NEXT:    setl %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_load_slt_imm16_optsize:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $512, (%rdi) # imm = 0x200
+; X64-FAST-NEXT:    setl %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_load_slt_imm16_optsize:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -620,6 +820,19 @@ define i1 @cmp16_load_ule_imm8(ptr %p0) {
 ; X64-GENERIC-NEXT:    setb %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_load_ule_imm8:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT:    cmpw $16, (%eax)
+; X86-FAST-NEXT:    setb %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_load_ule_imm8:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $16, (%rdi)
+; X64-FAST-NEXT:    setb %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_load_ule_imm8:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -659,6 +872,19 @@ define i1 @cmp16_load_ule_imm16(ptr %p0) {
 ; X64-GENERIC-NEXT:    setb %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_load_ule_imm16:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT:    cmpw $513, (%eax) # imm = 0x201
+; X86-FAST-NEXT:    setb %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_load_ule_imm16:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $513, (%rdi) # imm = 0x201
+; X64-FAST-NEXT:    setb %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_load_ule_imm16:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -716,6 +942,19 @@ define i1 @cmp16_load_ule_imm16_optsize(ptr %p0) optsize {
 ; X64-GENERIC-NEXT:    setb %al
 ; X64-GENERIC-NEXT:    retq
 ;
+; X86-FAST-LABEL: cmp16_load_ule_imm16_optsize:
+; X86-FAST:       # %bb.0:
+; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT:    cmpw $513, (%eax) # imm = 0x201
+; X86-FAST-NEXT:    setb %al
+; X86-FAST-NEXT:    retl
+;
+; X64-FAST-LABEL: cmp16_load_ule_imm16_optsize:
+; X64-FAST:       # %bb.0:
+; X64-FAST-NEXT:    cmpw $513, (%rdi) # imm = 0x201
+; X64-FAST-NEXT:    setb %al
+; X64-FAST-NEXT:    retq
+;
 ; X86-ATOM-LABEL: cmp16_load_ule_imm16_optsize:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
-- 
GitLab


From 6ce04747cff524b4c5c8738e25144659a5cf6691 Mon Sep 17 00:00:00 2001
From: Quentin Colombet <quentin.colombet@gmail.com>
Date: Tue, 7 May 2024 11:08:33 +0200
Subject: [PATCH 0030/1206] [SDISel] Teach the type legalizer about
 ADDRSPACECAST (#90969)

Vectorized ADDRSPACECASTs were not supported by the type legalizer.

This patch adds the support for:
- splitting the vector result: <2 x ptr> => 2 x <1 x ptr>
- scalarization: <1 x ptr> => ptr
- widening: <3 x ptr> => <4 x ptr>

This is all exercised by the added NVPTX tests.
---
 llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h |  3 +
 .../SelectionDAG/LegalizeVectorTypes.cpp      | 65 +++++++++++++
 llvm/test/CodeGen/NVPTX/addrspacecast.ll      | 92 +++++++++++++++++++
 3 files changed, 160 insertions(+)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 4b06e19656ce..f44916b741cc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -786,6 +786,7 @@ private:
   SDValue ScalarizeVecRes_InregOp(SDNode *N);
   SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
 
+  SDValue ScalarizeVecRes_ADDRSPACECAST(SDNode *N);
   SDValue ScalarizeVecRes_BITCAST(SDNode *N);
   SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N);
   SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N);
@@ -853,6 +854,7 @@ private:
   void SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
+  void SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_FFREXP(SDNode *N, unsigned ResNo, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -956,6 +958,7 @@ private:
   // Widen Vector Result Promotion.
   void WidenVectorResult(SDNode *N, unsigned ResNo);
   SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo);
+  SDValue WidenVecRes_ADDRSPACECAST(SDNode *N);
   SDValue WidenVecRes_AssertZext(SDNode* N);
   SDValue WidenVecRes_BITCAST(SDNode* N);
   SDValue WidenVecRes_BUILD_VECTOR(SDNode* N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index cab4dc5f3c15..43db9b8e6be9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -23,6 +23,7 @@
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/VectorUtils.h"
+#include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TypeSize.h"
@@ -116,6 +117,9 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::FCANONICALIZE:
     R = ScalarizeVecRes_UnaryOp(N);
     break;
+  case ISD::ADDRSPACECAST:
+    R = ScalarizeVecRes_ADDRSPACECAST(N);
+    break;
   case ISD::FFREXP:
     R = ScalarizeVecRes_FFREXP(N, ResNo);
     break;
@@ -475,6 +479,31 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
   llvm_unreachable("Illegal extend_vector_inreg opcode");
 }
 
+SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
+  EVT DestVT = N->getValueType(0).getVectorElementType();
+  SDValue Op = N->getOperand(0);
+  EVT OpVT = Op.getValueType();
+  SDLoc DL(N);
+  // The result needs scalarizing, but it's not a given that the source does.
+  // This is a workaround for targets where it's impossible to scalarize the
+  // result of a conversion, because the source type is legal.
+  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
+  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
+  // legal and was not scalarized.
+  // See the similar logic in ScalarizeVecRes_SETCC
+  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
+    Op = GetScalarizedVector(Op);
+  } else {
+    EVT VT = OpVT.getVectorElementType();
+    Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
+                     DAG.getVectorIdxConstant(0, DL));
+  }
+  auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(N);
+  unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
+  unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
+  return DAG.getAddrSpaceCast(DL, DestVT, Op, SrcAS, DestAS);
+}
+
 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
   // If the operand is wider than the vector element type then it is implicitly
   // truncated.  Make that explicit here.
@@ -1122,6 +1151,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::FCANONICALIZE:
     SplitVecRes_UnaryOp(N, Lo, Hi);
     break;
+  case ISD::ADDRSPACECAST:
+    SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
+    break;
   case ISD::FFREXP:
     SplitVecRes_FFREXP(N, ResNo, Lo, Hi);
     break;
@@ -2353,6 +2385,26 @@ void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
   Hi = DAG.getNode(Opcode, dl, HiVT, {Hi, MaskHi, EVLHi}, Flags);
 }
 
+void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
+                                                 SDValue &Hi) {
+  SDLoc dl(N);
+  auto [LoVT, HiVT] = DAG.GetSplitDestVTs(N->getValueType(0));
+
+  // If the input also splits, handle it directly for a compile time speedup.
+  // Otherwise split it by hand.
+  EVT InVT = N->getOperand(0).getValueType();
+  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
+    GetSplitVector(N->getOperand(0), Lo, Hi);
+  else
+    std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
+
+  auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(N);
+  unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
+  unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
+  Lo = DAG.getAddrSpaceCast(dl, LoVT, Lo, SrcAS, DestAS);
+  Hi = DAG.getAddrSpaceCast(dl, HiVT, Hi, SrcAS, DestAS);
+}
+
 void DAGTypeLegalizer::SplitVecRes_FFREXP(SDNode *N, unsigned ResNo,
                                           SDValue &Lo, SDValue &Hi) {
   SDLoc dl(N);
@@ -4121,6 +4173,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
     report_fatal_error("Do not know how to widen the result of this operator!");
 
   case ISD::MERGE_VALUES:      Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
+  case ISD::ADDRSPACECAST:
+    Res = WidenVecRes_ADDRSPACECAST(N);
+    break;
   case ISD::AssertZext:        Res = WidenVecRes_AssertZext(N); break;
   case ISD::BITCAST:           Res = WidenVecRes_BITCAST(N); break;
   case ISD::BUILD_VECTOR:      Res = WidenVecRes_BUILD_VECTOR(N); break;
@@ -5086,6 +5141,16 @@ SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
   return GetWidenedVector(WidenVec);
 }
 
+SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
+  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
+  SDValue InOp = GetWidenedVector(N->getOperand(0));
+  auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(N);
+
+  return DAG.getAddrSpaceCast(SDLoc(N), WidenVT, InOp,
+                              AddrSpaceCastN->getSrcAddressSpace(),
+                              AddrSpaceCastN->getDestAddressSpace());
+}
+
 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
   SDValue InOp = N->getOperand(0);
   EVT InVT = InOp.getValueType();
diff --git a/llvm/test/CodeGen/NVPTX/addrspacecast.ll b/llvm/test/CodeGen/NVPTX/addrspacecast.ll
index b680490ac5b1..85752bb95eb3 100644
--- a/llvm/test/CodeGen/NVPTX/addrspacecast.ll
+++ b/llvm/test/CodeGen/NVPTX/addrspacecast.ll
@@ -98,3 +98,95 @@ define i32 @conv8(ptr %ptr) {
   %val = load i32, ptr addrspace(5) %specptr
   ret i32 %val
 }
+
+; Check that we support addrspacecast when splitting the vector
+; result (<2 x ptr> => 2 x <1 x ptr>).
+; This also checks that scalarization works for addrspacecast
+; (when going from <1 x ptr> to ptr.)
+; ALL-LABEL: split1To0
+define void @split1To0(ptr nocapture noundef readonly %xs) {
+; CLS32: cvta.global.u32
+; CLS32: cvta.global.u32
+; CLS64: cvta.global.u64
+; CLS64: cvta.global.u64
+; ALL: st.u32
+; ALL: st.u32
+  %vec_addr = load <2 x ptr addrspace(1)>, ptr %xs, align 16
+  %addrspacecast = addrspacecast <2 x ptr addrspace(1)> %vec_addr to <2 x ptr>
+  %extractelement0 = extractelement <2 x ptr> %addrspacecast, i64 0
+  store float 0.5, ptr %extractelement0, align 4
+  %extractelement1 = extractelement <2 x ptr> %addrspacecast, i64 1
+  store float 1.0, ptr %extractelement1, align 4
+  ret void
+}
+
+; Same as split1To0 but from 0 to 1, to make sure the addrspacecast preserve
+; the source and destination addrspaces properly.
+; ALL-LABEL: split0To1
+define void @split0To1(ptr nocapture noundef readonly %xs) {
+; CLS32: cvta.to.global.u32
+; CLS32: cvta.to.global.u32
+; CLS64: cvta.to.global.u64
+; CLS64: cvta.to.global.u64
+; ALL: st.global.u32
+; ALL: st.global.u32
+  %vec_addr = load <2 x ptr>, ptr %xs, align 16
+  %addrspacecast = addrspacecast <2 x ptr> %vec_addr to <2 x ptr addrspace(1)>
+  %extractelement0 = extractelement <2 x ptr addrspace(1)> %addrspacecast, i64 0
+  store float 0.5, ptr addrspace(1) %extractelement0, align 4
+  %extractelement1 = extractelement <2 x ptr addrspace(1)> %addrspacecast, i64 1
+  store float 1.0, ptr addrspace(1) %extractelement1, align 4
+  ret void
+}
+
+; Check that we support addrspacecast when a widening is required
+; (3 x ptr => 4 x ptr).
+; ALL-LABEL: widen1To0
+define void @widen1To0(ptr nocapture noundef readonly %xs) {
+; CLS32: cvta.global.u32
+; CLS32: cvta.global.u32
+; CLS32: cvta.global.u32
+
+; CLS64: cvta.global.u64
+; CLS64: cvta.global.u64
+; CLS64: cvta.global.u64
+
+; ALL: st.u32
+; ALL: st.u32
+; ALL: st.u32
+  %vec_addr = load <3 x ptr addrspace(1)>, ptr %xs, align 16
+  %addrspacecast = addrspacecast <3 x ptr addrspace(1)> %vec_addr to <3 x ptr>
+  %extractelement0 = extractelement <3 x ptr> %addrspacecast, i64 0
+  store float 0.5, ptr %extractelement0, align 4
+  %extractelement1 = extractelement <3 x ptr> %addrspacecast, i64 1
+  store float 1.0, ptr %extractelement1, align 4
+  %extractelement2 = extractelement <3 x ptr> %addrspacecast, i64 2
+  store float 1.5, ptr %extractelement2, align 4
+  ret void
+}
+
+; Same as widen1To0 but from 0 to 1, to make sure the addrspacecast preserve
+; the source and destination addrspaces properly.
+; ALL-LABEL: widen0To1
+define void @widen0To1(ptr nocapture noundef readonly %xs) {
+; CLS32: cvta.to.global.u32
+; CLS32: cvta.to.global.u32
+; CLS32: cvta.to.global.u32
+
+; CLS64: cvta.to.global.u64
+; CLS64: cvta.to.global.u64
+; CLS64: cvta.to.global.u64
+
+; ALL: st.global.u32
+; ALL: st.global.u32
+; ALL: st.global.u32
+  %vec_addr = load <3 x ptr>, ptr %xs, align 16
+  %addrspacecast = addrspacecast <3 x ptr> %vec_addr to <3 x ptr addrspace(1)>
+  %extractelement0 = extractelement <3 x ptr addrspace(1)> %addrspacecast, i64 0
+  store float 0.5, ptr addrspace(1) %extractelement0, align 4
+  %extractelement1 = extractelement <3 x ptr addrspace(1)> %addrspacecast, i64 1
+  store float 1.0, ptr addrspace(1) %extractelement1, align 4
+  %extractelement2 = extractelement <3 x ptr addrspace(1)> %addrspacecast, i64 2
+  store float 1.5, ptr addrspace(1) %extractelement2, align 4
+  ret void
+}
-- 
GitLab


From abd314938dda1b117f289be5e630e43e68533929 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 7 May 2024 10:28:55 +0100
Subject: [PATCH 0031/1206] [X86] Use GFNI for vXi8 shifts/rotates (#89115)

As detailed here: https://github.com/InstLatx64/InstLatX64_Demo/blob/master/GFNI_Demo.h

We can use the gf2p8affine instruction to lower byte shifts/rotates as well as the existing bitreverse case.

Based off the original patch here: https://reviews.llvm.org/D137026
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |   53 +-
 .../lib/Target/X86/X86TargetTransformInfo.cpp |   21 +
 .../Analysis/CostModel/X86/fshl-codesize.ll   |   12 +-
 .../Analysis/CostModel/X86/fshl-latency.ll    |   12 +-
 .../CostModel/X86/fshl-sizelatency.ll         |   12 +-
 llvm/test/Analysis/CostModel/X86/fshl.ll      |    2 +-
 .../Analysis/CostModel/X86/fshr-codesize.ll   |   12 +-
 .../Analysis/CostModel/X86/fshr-latency.ll    |   12 +-
 .../CostModel/X86/fshr-sizelatency.ll         |   12 +-
 llvm/test/Analysis/CostModel/X86/fshr.ll      |    2 +-
 .../CostModel/X86/vshift-ashr-codesize.ll     |    6 +-
 .../CostModel/X86/vshift-ashr-latency.ll      |    6 +-
 .../CostModel/X86/vshift-ashr-sizelatency.ll  |    6 +-
 .../CostModel/X86/vshift-lshr-codesize.ll     |   30 +-
 .../CostModel/X86/vshift-lshr-latency.ll      |   18 +-
 .../CostModel/X86/vshift-lshr-sizelatency.ll  |   18 +-
 .../CostModel/X86/vshift-shl-codesize.ll      |   30 +-
 .../CostModel/X86/vshift-shl-latency.ll       |   18 +-
 .../CostModel/X86/vshift-shl-sizelatency.ll   |   18 +-
 llvm/test/CodeGen/X86/gfni-funnel-shifts.ll   |  909 ++++------
 llvm/test/CodeGen/X86/gfni-rotates.ll         | 1490 +++++++----------
 llvm/test/CodeGen/X86/gfni-shifts.ll          |  823 +++------
 .../CodeGen/X86/min-legal-vector-width.ll     |   75 +-
 23 files changed, 1474 insertions(+), 2123 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b7c14e50210e..8ec4984dfa55 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -29022,6 +29022,29 @@ SDValue X86TargetLowering::LowerWin64_INT128_TO_FP(SDValue Op,
   return IsStrict ? DAG.getMergeValues({Result, Chain}, dl) : Result;
 }
 
+// Generate a GFNI gf2p8affine bitmask for vXi8 bitreverse/shift/rotate.
+uint64_t getGFNICtrlImm(unsigned Opcode, unsigned Amt = 0) {
+  assert((Amt < 8) && "Shift/Rotation amount out of range");
+  switch (Opcode) {
+  case ISD::BITREVERSE:
+    return 0x8040201008040201ULL;
+  case ISD::SHL:
+    return ((0x0102040810204080ULL >> (Amt)) &
+            (0x0101010101010101ULL * (0xFF >> (Amt))));
+  case ISD::SRL:
+    return ((0x0102040810204080ULL << (Amt)) &
+            (0x0101010101010101ULL * ((0xFF << (Amt)) & 0xFF)));
+  case ISD::SRA:
+    return (getGFNICtrlImm(ISD::SRL, Amt) |
+            (0x8080808080808080ULL >> (64 - (8 * Amt))));
+  case ISD::ROTL:
+    return getGFNICtrlImm(ISD::SRL, 8 - Amt) | getGFNICtrlImm(ISD::SHL, Amt);
+  case ISD::ROTR:
+    return getGFNICtrlImm(ISD::SHL, 8 - Amt) | getGFNICtrlImm(ISD::SRL, Amt);
+  }
+  llvm_unreachable("Unsupported GFNI opcode");
+}
+
 // Return true if the required (according to Opcode) shift-imm form is natively
 // supported by the Subtarget
 static bool supportedVectorShiftWithImm(EVT VT, const X86Subtarget &Subtarget,
@@ -29209,6 +29232,14 @@ static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG,
     if (VT == MVT::v16i8 && Subtarget.hasXOP())
       return SDValue();
 
+    if (Subtarget.hasGFNI()) {
+      uint64_t ShiftMask = getGFNICtrlImm(Op.getOpcode(), ShiftAmt);
+      MVT MaskVT = MVT::getVectorVT(MVT::i64, NumElts / 8);
+      SDValue Mask = DAG.getBitcast(VT, DAG.getConstant(ShiftMask, dl, MaskVT));
+      return DAG.getNode(X86ISD::GF2P8AFFINEQB, dl, VT, R, Mask,
+                         DAG.getTargetConstant(0, dl, MVT::i8));
+    }
+
     if (Op.getOpcode() == ISD::SHL) {
       // Make a large shift.
       SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ShiftVT, R,
@@ -29892,13 +29923,15 @@ static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
       uint64_t ShXAmt = IsFSHR ? (EltSizeInBits - ShiftAmt) : ShiftAmt;
       uint64_t ShYAmt = IsFSHR ? ShiftAmt : (EltSizeInBits - ShiftAmt);
       assert((ShXAmt + ShYAmt) == EltSizeInBits && "Illegal funnel shift");
+      MVT WideVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
 
-      if (EltSizeInBits == 8 && ShXAmt > 1 &&
-          (Subtarget.hasXOP() || useVPTERNLOG(Subtarget, VT))) {
+      if (EltSizeInBits == 8 &&
+          (Subtarget.hasXOP() ||
+           (useVPTERNLOG(Subtarget, VT) &&
+            supportedVectorShiftWithImm(WideVT, Subtarget, ISD::SHL)))) {
         // For vXi8 cases on Subtargets that can perform VPCMOV/VPTERNLOG
         // bit-select - lower using vXi16 shifts and then perform the bitmask at
         // the original vector width to handle cases where we split.
-        MVT WideVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
         APInt MaskX = APInt::getHighBitsSet(8, 8 - ShXAmt);
         APInt MaskY = APInt::getLowBitsSet(8, 8 - ShYAmt);
         SDValue ShX =
@@ -30103,6 +30136,17 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
                          DAG.getNode(ISD::SUB, DL, VT, Z, Amt));
   }
 
+  // Attempt to use GFNI gf2p8affine to rotate vXi8 by an uniform constant.
+  if (IsCstSplat && Subtarget.hasGFNI() && VT.getScalarType() == MVT::i8 &&
+      DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
+    uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
+    uint64_t RotMask = getGFNICtrlImm(Opcode, RotAmt);
+    MVT MaskVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
+    SDValue Mask = DAG.getBitcast(VT, DAG.getConstant(RotMask, DL, MaskVT));
+    return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, R, Mask,
+                       DAG.getTargetConstant(0, DL, MVT::i8));
+  }
+
   // Split 256-bit integers on XOP/pre-AVX2 targets.
   if (VT.is256BitVector() && (Subtarget.hasXOP() || !Subtarget.hasAVX2()))
     return splitVectorIntBinary(Op, DAG, DL);
@@ -31426,7 +31470,8 @@ static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
   // If we have GFNI, we can use GF2P8AFFINEQB to reverse the bits.
   if (Subtarget.hasGFNI()) {
     MVT MatrixVT = MVT::getVectorVT(MVT::i64, NumElts / 8);
-    SDValue Matrix = DAG.getConstant(0x8040201008040201ULL, DL, MatrixVT);
+    SDValue Matrix =
+        DAG.getConstant(getGFNICtrlImm(ISD::BITREVERSE), DL, MatrixVT);
     Matrix = DAG.getBitcast(VT, Matrix);
     return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, In, Matrix,
                        DAG.getTargetConstant(0, DL, MVT::i8));
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index cb07c2a4b56a..2257370912bd 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -345,6 +345,24 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
                                   Op1Info.getNoProps(), Op2Info.getNoProps());
   }
 
+  static const CostKindTblEntry GFNIUniformConstCostTable[] = {
+    { ISD::SHL,  MVT::v16i8,  { 1, 6, 1, 2 } }, // gf2p8affineqb
+    { ISD::SRL,  MVT::v16i8,  { 1, 6, 1, 2 } }, // gf2p8affineqb
+    { ISD::SRA,  MVT::v16i8,  { 1, 6, 1, 2 } }, // gf2p8affineqb
+    { ISD::SHL,  MVT::v32i8,  { 1, 6, 1, 2 } }, // gf2p8affineqb
+    { ISD::SRL,  MVT::v32i8,  { 1, 6, 1, 2 } }, // gf2p8affineqb
+    { ISD::SRA,  MVT::v32i8,  { 1, 6, 1, 2 } }, // gf2p8affineqb
+    { ISD::SHL,  MVT::v64i8,  { 1, 6, 1, 2 } }, // gf2p8affineqb
+    { ISD::SRL,  MVT::v64i8,  { 1, 6, 1, 2 } }, // gf2p8affineqb
+    { ISD::SRA,  MVT::v64i8,  { 1, 6, 1, 2 } }, // gf2p8affineqb
+  };
+
+  if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasGFNI())
+    if (const auto *Entry =
+            CostTableLookup(GFNIUniformConstCostTable, ISD, LT.second))
+      if (auto KindCost = Entry->Cost[CostKind])
+        return LT.first * *KindCost;
+
   static const CostKindTblEntry AVX512BWUniformConstCostTable[] = {
     { ISD::SHL,  MVT::v16i8,  { 1, 7, 2, 3 } }, // psllw + pand.
     { ISD::SRL,  MVT::v16i8,  { 1, 7, 2, 3 } }, // psrlw + pand.
@@ -3869,6 +3887,9 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
     { ISD::BITREVERSE, MVT::v2i64,   {  1,  8,  2,  4 } }, // gf2p8affineqb
     { ISD::BITREVERSE, MVT::v4i64,   {  1,  9,  2,  4 } }, // gf2p8affineqb
     { ISD::BITREVERSE, MVT::v8i64,   {  1,  9,  2,  4 } }, // gf2p8affineqb
+    { X86ISD::VROTLI,  MVT::v16i8,   {  1,  6,  1,  2 } }, // gf2p8affineqb
+    { X86ISD::VROTLI,  MVT::v32i8,   {  1,  6,  1,  2 } }, // gf2p8affineqb
+    { X86ISD::VROTLI,  MVT::v64i8,   {  1,  6,  1,  2 } }, // gf2p8affineqb
   };
   static const CostKindTblEntry GLMCostTbl[] = {
     { ISD::FSQRT,      MVT::f32,     { 19, 20, 1, 1 } }, // sqrtss
diff --git a/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll b/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll
index a7585a4d9f39..71927002b599 100644
--- a/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll
@@ -1597,9 +1597,9 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_funnel_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
@@ -2871,9 +2871,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3)
diff --git a/llvm/test/Analysis/CostModel/X86/fshl-latency.ll b/llvm/test/Analysis/CostModel/X86/fshl-latency.ll
index 7105f713fdc3..c40394ba9a72 100644
--- a/llvm/test/Analysis/CostModel/X86/fshl-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshl-latency.ll
@@ -1549,9 +1549,9 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_funnel_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
@@ -2823,9 +2823,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3)
diff --git a/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll
index 5d7361e29317..7b0daf504855 100644
--- a/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll
@@ -1597,9 +1597,9 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_funnel_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
@@ -3111,9 +3111,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3)
diff --git a/llvm/test/Analysis/CostModel/X86/fshl.ll b/llvm/test/Analysis/CostModel/X86/fshl.ll
index 1cbdab09acd9..127dec0a1a6f 100644
--- a/llvm/test/Analysis/CostModel/X86/fshl.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshl.ll
@@ -2811,7 +2811,7 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3)
diff --git a/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll b/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll
index ecc861dd7f8e..92a20b938142 100644
--- a/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll
@@ -1597,9 +1597,9 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_funnel_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
@@ -2871,9 +2871,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3)
diff --git a/llvm/test/Analysis/CostModel/X86/fshr-latency.ll b/llvm/test/Analysis/CostModel/X86/fshr-latency.ll
index 0142ad77849c..33fadef536bf 100644
--- a/llvm/test/Analysis/CostModel/X86/fshr-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshr-latency.ll
@@ -1549,9 +1549,9 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_funnel_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
@@ -2823,9 +2823,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3)
diff --git a/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll
index 6dafb20a0aee..ef831328c480 100644
--- a/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll
@@ -1597,9 +1597,9 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_funnel_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
@@ -3111,9 +3111,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8'
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3)
diff --git a/llvm/test/Analysis/CostModel/X86/fshr.ll b/llvm/test/Analysis/CostModel/X86/fshr.ll
index ada1b9c5bdc4..3c233b51053d 100644
--- a/llvm/test/Analysis/CostModel/X86/fshr.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshr.ll
@@ -2811,7 +2811,7 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3)
diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll
index a3c24bdd1a88..9ff975665f13 100644
--- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll
@@ -1676,7 +1676,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
   %shift = ashr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
@@ -1713,7 +1713,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
   %shift = ashr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
@@ -1750,7 +1750,7 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
   %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll
index cd4189d4a7f8..ab300779b434 100644
--- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll
@@ -1806,7 +1806,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
   %shift = ashr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
@@ -1847,7 +1847,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %shift = ashr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
   %shift = ashr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
@@ -1888,7 +1888,7 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
   %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll
index 84ccad029415..1b51a2e0a1e6 100644
--- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll
@@ -1700,7 +1700,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
   %shift = ashr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
@@ -1741,7 +1741,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
   %shift = ashr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
@@ -1782,7 +1782,7 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
   %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll
index a0e15bb8ff73..644fcbbfefdf 100644
--- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll
@@ -1619,9 +1619,17 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
-; AVX512-LABEL: 'splatconstant_shift_v16i8'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+; AVX512F-LABEL: 'splatconstant_shift_v16i8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+;
+; AVX512BW-LABEL: 'splatconstant_shift_v16i8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+;
+; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8'
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
   %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <16 x i8> %shift
@@ -1652,9 +1660,17 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
-; AVX512-LABEL: 'splatconstant_shift_v32i8'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+; AVX512F-LABEL: 'splatconstant_shift_v32i8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+;
+; AVX512BW-LABEL: 'splatconstant_shift_v32i8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+;
+; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8'
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
   %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <32 x i8> %shift
@@ -1694,7 +1710,7 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
   %shift = lshr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll
index 61620e2cc97e..f879a09d067e 100644
--- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll
@@ -1778,7 +1778,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
   %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
@@ -1806,9 +1806,17 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
-; AVX512-LABEL: 'splatconstant_shift_v32i8'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+; AVX512F-LABEL: 'splatconstant_shift_v32i8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+;
+; AVX512BW-LABEL: 'splatconstant_shift_v32i8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+;
+; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8'
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
   %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <32 x i8> %shift
@@ -1844,7 +1852,7 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shift = lshr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
   %shift = lshr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll
index e6b6ac75b65d..fe472342e214 100644
--- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll
@@ -1635,9 +1635,17 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
-; AVX512-LABEL: 'splatconstant_shift_v16i8'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+; AVX512F-LABEL: 'splatconstant_shift_v16i8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+;
+; AVX512BW-LABEL: 'splatconstant_shift_v16i8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+;
+; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8'
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
   %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <16 x i8> %shift
@@ -1677,7 +1685,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
   %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
@@ -1718,7 +1726,7 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
   %shift = lshr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll
index 265658b1e3a2..1045b827da7c 100644
--- a/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll
@@ -1593,9 +1593,17 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
-; AVX512-LABEL: 'splatconstant_shift_v16i8'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+; AVX512F-LABEL: 'splatconstant_shift_v16i8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+;
+; AVX512BW-LABEL: 'splatconstant_shift_v16i8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+;
+; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8'
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
   %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <16 x i8> %shift
@@ -1626,9 +1634,17 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
-; AVX512-LABEL: 'splatconstant_shift_v32i8'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+; AVX512F-LABEL: 'splatconstant_shift_v32i8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+;
+; AVX512BW-LABEL: 'splatconstant_shift_v32i8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+;
+; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8'
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
   %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <32 x i8> %shift
@@ -1668,7 +1684,7 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
   %shift = shl <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll
index 42c91144ff6f..3ae71daf50a3 100644
--- a/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll
@@ -1738,7 +1738,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
   %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
@@ -1766,9 +1766,17 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
-; AVX512-LABEL: 'splatconstant_shift_v32i8'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+; AVX512F-LABEL: 'splatconstant_shift_v32i8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+;
+; AVX512BW-LABEL: 'splatconstant_shift_v32i8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
+;
+; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8'
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
   %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <32 x i8> %shift
@@ -1804,7 +1812,7 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
   %shift = shl <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll
index 47b24df063ef..4256a73a7cf7 100644
--- a/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll
@@ -1691,9 +1691,17 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
-; AVX512-LABEL: 'splatconstant_shift_v16i8'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+; AVX512F-LABEL: 'splatconstant_shift_v16i8'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+;
+; AVX512BW-LABEL: 'splatconstant_shift_v16i8'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
+;
+; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8'
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift
 ;
   %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <16 x i8> %shift
@@ -1729,7 +1737,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift
 ;
   %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
@@ -1766,7 +1774,7 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
 ; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8'
-; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
+; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
 ; AVX512GFNI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift
 ;
   %shift = shl <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
diff --git a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
index afe0ebb9dcb4..b3ca9fb04aeb 100644
--- a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
+++ b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
@@ -107,16 +107,15 @@ define <16 x i8> @var_fshl_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %amt) nou
 ; GFNIAVX512VL-LABEL: var_fshl_v16i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; GFNIAVX512VL-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; GFNIAVX512VL-NEXT:    vpandn %xmm3, %xmm2, %xmm4
 ; GFNIAVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
-; GFNIAVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; GFNIAVX512VL-NEXT:    vpsllvd %zmm4, %zmm0, %zmm0
-; GFNIAVX512VL-NEXT:    vpandn %xmm3, %xmm2, %xmm2
-; GFNIAVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %xmm1, %xmm1
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm1
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm1
 ; GFNIAVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; GFNIAVX512VL-NEXT:    vpsrlvd %zmm2, %zmm1, %zmm1
+; GFNIAVX512VL-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm1
+; GFNIAVX512VL-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; GFNIAVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; GFNIAVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; GFNIAVX512VL-NEXT:    vpsllvd %zmm2, %zmm0, %zmm0
 ; GFNIAVX512VL-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; GFNIAVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
 ; GFNIAVX512VL-NEXT:    vzeroupper
@@ -151,17 +150,14 @@ define <16 x i8> @var_fshr_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %amt) nou
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm4
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm6
-; GFNISSE-NEXT:    psrlw $4, %xmm6
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm6, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm6
-; GFNISSE-NEXT:    psrlw $2, %xmm6
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm6, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm6
-; GFNISSE-NEXT:    psrlw $1, %xmm6
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm6, %xmm1
@@ -171,13 +167,11 @@ define <16 x i8> @var_fshr_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %amt) nou
 ; GFNISSE-NEXT:    paddb %xmm3, %xmm4
 ; GFNISSE-NEXT:    paddb %xmm2, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psllw $4, %xmm5
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
-; GFNISSE-NEXT:    psllw $2, %xmm3
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
@@ -195,25 +189,20 @@ define <16 x i8> @var_fshr_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %amt) nou
 ; GFNIAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm4
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm4, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm4, %xmm4, %xmm5
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm1, %xmm6
-; GFNIAVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm6
 ; GFNIAVX1-NEXT:    vpblendvb %xmm4, %xmm6, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm4
 ; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
 ; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm0, %xmm4
-; GFNIAVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm4
 ; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm4, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm0, %xmm2
-; GFNIAVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
 ; GFNIAVX1-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
 ; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm3, %xmm3
@@ -227,25 +216,20 @@ define <16 x i8> @var_fshr_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %amt) nou
 ; GFNIAVX2-NEXT:    vpand %xmm3, %xmm2, %xmm4
 ; GFNIAVX2-NEXT:    vpsllw $5, %xmm4, %xmm4
 ; GFNIAVX2-NEXT:    vpaddb %xmm4, %xmm4, %xmm5
-; GFNIAVX2-NEXT:    vpsrlw $4, %xmm1, %xmm6
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6, %xmm6
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm6
 ; GFNIAVX2-NEXT:    vpblendvb %xmm4, %xmm6, %xmm1, %xmm1
-; GFNIAVX2-NEXT:    vpsrlw $2, %xmm1, %xmm4
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm4
 ; GFNIAVX2-NEXT:    vpblendvb %xmm5, %xmm4, %xmm1, %xmm1
-; GFNIAVX2-NEXT:    vpsrlw $1, %xmm1, %xmm4
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm4
 ; GFNIAVX2-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
 ; GFNIAVX2-NEXT:    vpblendvb %xmm5, %xmm4, %xmm1, %xmm1
 ; GFNIAVX2-NEXT:    vpandn %xmm3, %xmm2, %xmm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %xmm2, %xmm2
 ; GFNIAVX2-NEXT:    vpaddb %xmm2, %xmm2, %xmm3
 ; GFNIAVX2-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
-; GFNIAVX2-NEXT:    vpsllw $4, %xmm0, %xmm4
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm4
 ; GFNIAVX2-NEXT:    vpblendvb %xmm2, %xmm4, %xmm0, %xmm0
-; GFNIAVX2-NEXT:    vpsllw $2, %xmm0, %xmm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
 ; GFNIAVX2-NEXT:    vpblendvb %xmm3, %xmm2, %xmm0, %xmm0
 ; GFNIAVX2-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
 ; GFNIAVX2-NEXT:    vpaddb %xmm3, %xmm3, %xmm3
@@ -492,19 +476,15 @@ define <16 x i8> @constant_fshr_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <16 x i8> @splatconstant_fshl_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; GFNISSE-LABEL: splatconstant_fshl_v16i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psrlw $5, %xmm1
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; GFNISSE-NEXT:    psllw $3, %xmm0
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    por %xmm1, %xmm0
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1OR2-LABEL: splatconstant_fshl_v16i8:
 ; GFNIAVX1OR2:       # %bb.0:
-; GFNIAVX1OR2-NEXT:    vpsrlw $5, %xmm1, %xmm1
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; GFNIAVX1OR2-NEXT:    vpsllw $3, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; GFNIAVX1OR2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; GFNIAVX1OR2-NEXT:    retq
 ;
@@ -522,25 +502,23 @@ declare <16 x i8> @llvm.fshl.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)
 define <16 x i8> @splatconstant_fshr_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; GFNISSE-LABEL: splatconstant_fshr_v16i8:
 ; GFNISSE:       # %bb.0:
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm1
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; GFNISSE-NEXT:    por %xmm1, %xmm0
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1OR2-LABEL: splatconstant_fshr_v16i8:
 ; GFNIAVX1OR2:       # %bb.0:
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
 ; GFNIAVX1OR2-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
 ; GFNIAVX1OR2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_fshr_v16i8:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; GFNIAVX512-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
-; GFNIAVX512-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0
+; GFNIAVX512-NEXT:    vpaddw %xmm0, %xmm0, %xmm2
+; GFNIAVX512-NEXT:    vpsrlw $7, %xmm1, %xmm0
+; GFNIAVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0
 ; GFNIAVX512-NEXT:    retq
   %res = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>)
   ret <16 x i8> %res
@@ -721,28 +699,22 @@ define <32 x i8> @var_fshl_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; GFNIAVX512VL-NEXT:    vpandn %ymm3, %ymm2, %ymm4
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX512VL-NEXT:    vpand %ymm5, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm6
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm6, %ymm6
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm6
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm4, %ymm6, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm1, %ymm6
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm6, %ymm6
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm6
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm4, %ymm6, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm1, %ymm6
-; GFNIAVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm5
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm1, %ymm5
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm4, %ymm5, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm4
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
@@ -771,40 +743,35 @@ define <32 x i8> @var_fshr_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm4
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [1161999622361579520,1161999622361579520]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm9
-; GFNISSE-NEXT:    psrlw $4, %xmm9
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNISSE-NEXT:    pand %xmm8, %xmm9
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm9
 ; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
 ; GFNISSE-NEXT:    pand %xmm7, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm9, %xmm2
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [290499906672525312,290499906672525312]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm10
-; GFNISSE-NEXT:    psrlw $2, %xmm10
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNISSE-NEXT:    pand %xmm9, %xmm10
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm10
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm10, %xmm2
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [145249953336295424,145249953336295424]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm11
-; GFNISSE-NEXT:    psrlw $1, %xmm11
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNISSE-NEXT:    pand %xmm10, %xmm11
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm11
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm11, %xmm2
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
+; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm11 = [16909320,16909320]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm12
-; GFNISSE-NEXT:    psllw $4, %xmm12
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNISSE-NEXT:    pand %xmm11, %xmm12
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm12
 ; GFNISSE-NEXT:    pandn %xmm7, %xmm6
 ; GFNISSE-NEXT:    psllw $5, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm4
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm12 = [1108169199648,1108169199648]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm13
-; GFNISSE-NEXT:    psllw $2, %xmm13
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm12 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNISSE-NEXT:    pand %xmm12, %xmm13
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm13
 ; GFNISSE-NEXT:    paddb %xmm6, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm13, %xmm4
@@ -815,33 +782,28 @@ define <32 x i8> @var_fshr_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm13, %xmm4
 ; GFNISSE-NEXT:    por %xmm2, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm2
-; GFNISSE-NEXT:    psrlw $4, %xmm2
-; GFNISSE-NEXT:    pand %xmm8, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
 ; GFNISSE-NEXT:    pand %xmm7, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm2, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm2
-; GFNISSE-NEXT:    psrlw $2, %xmm2
-; GFNISSE-NEXT:    pand %xmm9, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm2
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm2, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm2
-; GFNISSE-NEXT:    psrlw $1, %xmm2
-; GFNISSE-NEXT:    pand %xmm10, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm2
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm2, %xmm3
 ; GFNISSE-NEXT:    paddb %xmm1, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm2
-; GFNISSE-NEXT:    psllw $4, %xmm2
-; GFNISSE-NEXT:    pand %xmm11, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm2
 ; GFNISSE-NEXT:    pandn %xmm7, %xmm5
 ; GFNISSE-NEXT:    psllw $5, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm2
-; GFNISSE-NEXT:    psllw $2, %xmm2
-; GFNISSE-NEXT:    pand %xmm12, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm2
 ; GFNISSE-NEXT:    paddb %xmm5, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
@@ -856,100 +818,95 @@ define <32 x i8> @var_fshr_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ;
 ; GFNIAVX1-LABEL: var_fshr_v32i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm5, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm6
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm4, %xmm6
 ; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; GFNIAVX1-NEXT:    vandps %ymm3, %ymm2, %ymm2
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm7, %xmm8
-; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm6, %xmm5, %xmm6
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm6, %xmm9
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm9, %xmm9
+; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm6, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [290499906672525312,290499906672525312]
+; GFNIAVX1-NEXT:    # xmm6 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm4, %xmm9
 ; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm9, %xmm6, %xmm9
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm9, %xmm10
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm10, %xmm10
+; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm9, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [145249953336295424,145249953336295424]
+; GFNIAVX1-NEXT:    # xmm9 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm4, %xmm10
 ; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm10, %xmm9, %xmm8
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm9
-; GFNIAVX1-NEXT:    vpaddb %xmm9, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm9, %xmm10
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm11 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX1-NEXT:    vpand %xmm11, %xmm10, %xmm10
+; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm10, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm8
+; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm10 = [16909320,16909320]
+; GFNIAVX1-NEXT:    # xmm10 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm10, %xmm8, %xmm11
 ; GFNIAVX1-NEXT:    vpxor %xmm3, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm7, %xmm7
-; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm10, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm9, %xmm10
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm12 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX1-NEXT:    vpand %xmm12, %xmm10, %xmm10
+; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm11, %xmm8, %xmm8
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm11 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    # xmm11 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm11, %xmm8, %xmm12
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
-; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm10, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpaddb %xmm9, %xmm9, %xmm10
+; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm12, %xmm8, %xmm8
+; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm12
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
-; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm10, %xmm9, %xmm7
-; GFNIAVX1-NEXT:    vpor %xmm7, %xmm8, %xmm7
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm1, %xmm8
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm8, %xmm4
-; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm8
-; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm12, %xmm8, %xmm7
+; GFNIAVX1-NEXT:    vpor %xmm4, %xmm7, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm1, %xmm5
+; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm7
+; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm5, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm1, %xmm5
+; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm6
+; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm5, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm1, %xmm5
+; GFNIAVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm5, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm0, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm11, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm10, %xmm0, %xmm5
 ; GFNIAVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm4, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm12, %xmm3
+; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm11, %xmm0, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm3, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm3, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: var_fshr_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; GFNIAVX2-NEXT:    vpand %ymm3, %ymm2, %ymm4
-; GFNIAVX2-NEXT:    vpsllw $5, %ymm4, %ymm4
-; GFNIAVX2-NEXT:    vpaddb %ymm4, %ymm4, %ymm5
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm1, %ymm6
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm6, %ymm6
-; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm6, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsrlw $2, %ymm1, %ymm4
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4
-; GFNIAVX2-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsrlw $1, %ymm1, %ymm4
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm1, %ymm3
+; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; GFNIAVX2-NEXT:    vpand %ymm4, %ymm2, %ymm5
+; GFNIAVX2-NEXT:    vpsllw $5, %ymm5, %ymm5
+; GFNIAVX2-NEXT:    vpblendvb %ymm5, %ymm3, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm1, %ymm3
 ; GFNIAVX2-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
-; GFNIAVX2-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpandn %ymm3, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $5, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
+; GFNIAVX2-NEXT:    vpblendvb %ymm5, %ymm3, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm1, %ymm3
+; GFNIAVX2-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
+; GFNIAVX2-NEXT:    vpblendvb %ymm5, %ymm3, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
+; GFNIAVX2-NEXT:    vpandn %ymm4, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpsllw $5, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm3, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
+; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm3, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
+; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm3, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
@@ -959,25 +916,20 @@ define <32 x i8> @var_fshr_v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %amt) nou
 ; GFNIAVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm4
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm4, %ymm4
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm4, %ymm4, %ymm5
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm6
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm6, %ymm6
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm6
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm4, %ymm6, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm1, %ymm4
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm4
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm1, %ymm4
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm4
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpandn %ymm3, %ymm2, %ymm2
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm4
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
@@ -1336,45 +1288,29 @@ define <32 x i8> @constant_fshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 define <32 x i8> @splatconstant_fshl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNISSE-LABEL: splatconstant_fshl_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psrlw $4, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNISSE-NEXT:    movdqa %xmm4, %xmm5
-; GFNISSE-NEXT:    pandn %xmm2, %xmm5
-; GFNISSE-NEXT:    psllw $4, %xmm0
-; GFNISSE-NEXT:    pand %xmm4, %xmm0
-; GFNISSE-NEXT:    por %xmm5, %xmm0
-; GFNISSE-NEXT:    psrlw $4, %xmm3
-; GFNISSE-NEXT:    psllw $4, %xmm1
-; GFNISSE-NEXT:    pand %xmm4, %xmm1
-; GFNISSE-NEXT:    pandn %xmm3, %xmm4
-; GFNISSE-NEXT:    por %xmm4, %xmm1
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
+; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm5 = [16909320,16909320]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm0
+; GFNISSE-NEXT:    por %xmm2, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm1
+; GFNISSE-NEXT:    por %xmm3, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_fshl_v32i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_fshl_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
@@ -1392,45 +1328,29 @@ declare <32 x i8> @llvm.fshl.v32i8(<32 x i8>, <32 x i8>, <32 x i8>)
 define <32 x i8> @splatconstant_fshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNISSE-LABEL: splatconstant_fshr_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psrlw $6, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNISSE-NEXT:    movdqa %xmm4, %xmm5
-; GFNISSE-NEXT:    pandn %xmm2, %xmm5
-; GFNISSE-NEXT:    psllw $2, %xmm0
-; GFNISSE-NEXT:    pand %xmm4, %xmm0
-; GFNISSE-NEXT:    por %xmm5, %xmm0
-; GFNISSE-NEXT:    psrlw $6, %xmm3
-; GFNISSE-NEXT:    psllw $2, %xmm1
-; GFNISSE-NEXT:    pand %xmm4, %xmm1
-; GFNISSE-NEXT:    pandn %xmm3, %xmm4
-; GFNISSE-NEXT:    por %xmm4, %xmm1
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [4647714815446351872,4647714815446351872]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm0
+; GFNISSE-NEXT:    por %xmm2, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm1
+; GFNISSE-NEXT:    por %xmm3, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_fshr_v32i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_fshr_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $6, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
@@ -1766,63 +1686,51 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNIAVX512VL-LABEL: var_fshl_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm5
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm5, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm7
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm8 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; GFNIAVX512VL-NEXT:    vpandq %zmm8, %zmm2, %zmm2
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpxor %ymm3, %ymm8, %ymm9
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm3, %ymm6
+; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; GFNIAVX512VL-NEXT:    vpandq %zmm7, %zmm2, %zmm2
+; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm2, %ymm8
+; GFNIAVX512VL-NEXT:    vpxor %ymm7, %ymm8, %ymm9
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm9, %ymm9
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm9, %ymm7, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm5, %ymm7
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNIAVX512VL-NEXT:    vpand %ymm7, %ymm10, %ymm7
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm9, %ymm6, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm3, %ymm10
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm9, %ymm9, %ymm9
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm9, %ymm7, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm5, %ymm7
-; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm7, %ymm7
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm9, %ymm10, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm10
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm9, %ymm9, %ymm9
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm9, %ymm7, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm7
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm7, %ymm6
-; GFNIAVX512VL-NEXT:    vpxor %ymm2, %ymm8, %ymm7
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm9, %ymm10, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm1, %ymm5
+; GFNIAVX512VL-NEXT:    vpxor %ymm7, %ymm2, %ymm7
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm7, %ymm7
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm7, %ymm6, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm1, %ymm6
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm10, %ymm6
-; GFNIAVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm7
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm7, %ymm6, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm1, %ymm6
-; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm6, %ymm4
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm1, %ymm5
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm6
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm4, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm4, %ymm5
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm5, %ymm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm4, %ymm5
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX512VL-NEXT:    vpand %ymm7, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm5, %ymm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpaddb %ymm4, %ymm4, %ymm5
-; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm5, %ymm4, %ymm3
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm5, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm4
+; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm5
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [16909320,16909320,16909320,16909320]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm5
+; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm8, %ymm6
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm5, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm3, %ymm7
+; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm7, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm7
+; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm7, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm4
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm4
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
@@ -1863,35 +1771,30 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm1
 ; GFNISSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm12
-; GFNISSE-NEXT:    psrlw $4, %xmm12
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm12
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm12
 ; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pand %xmm11, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm13
-; GFNISSE-NEXT:    psrlw $2, %xmm13
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm13
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm13
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm13, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm14
-; GFNISSE-NEXT:    psrlw $1, %xmm14
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm14
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm14
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm14, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm1, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm15
-; GFNISSE-NEXT:    psllw $4, %xmm15
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm15
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm15
 ; GFNISSE-NEXT:    movdqa %xmm11, %xmm12
 ; GFNISSE-NEXT:    pandn %xmm11, %xmm9
 ; GFNISSE-NEXT:    psllw $5, %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm15, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm8
-; GFNISSE-NEXT:    psllw $2, %xmm8
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm8
 ; GFNISSE-NEXT:    paddb %xmm9, %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
@@ -1902,38 +1805,33 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
 ; GFNISSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm8
-; GFNISSE-NEXT:    psrlw $4, %xmm8
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNISSE-NEXT:    pand %xmm11, %xmm8
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pand %xmm12, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm8
-; GFNISSE-NEXT:    psrlw $2, %xmm8
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm13 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNISSE-NEXT:    pand %xmm13, %xmm8
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm13 = [290499906672525312,290499906672525312]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm8
-; GFNISSE-NEXT:    psrlw $1, %xmm8
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm14 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNISSE-NEXT:    pand %xmm14, %xmm8
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm14 = [145249953336295424,145249953336295424]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm14, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm6
 ; GFNISSE-NEXT:    paddb %xmm2, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm8
-; GFNISSE-NEXT:    psllw $4, %xmm8
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm15 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNISSE-NEXT:    pand %xmm15, %xmm8
+; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm15 = [16909320,16909320]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm15, %xmm8
 ; GFNISSE-NEXT:    pandn %xmm12, %xmm9
 ; GFNISSE-NEXT:    psllw $5, %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm8
-; GFNISSE-NEXT:    psllw $2, %xmm8
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm0 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNISSE-NEXT:    pand %xmm0, %xmm8
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm0 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm0, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm9, %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm2
@@ -1944,33 +1842,28 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm2
 ; GFNISSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm7, %xmm8
-; GFNISSE-NEXT:    psrlw $4, %xmm8
-; GFNISSE-NEXT:    pand %xmm11, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pand %xmm12, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm7
 ; GFNISSE-NEXT:    movdqa %xmm7, %xmm8
-; GFNISSE-NEXT:    psrlw $2, %xmm8
-; GFNISSE-NEXT:    pand %xmm13, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm7
 ; GFNISSE-NEXT:    movdqa %xmm7, %xmm8
-; GFNISSE-NEXT:    psrlw $1, %xmm8
-; GFNISSE-NEXT:    pand %xmm14, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm14, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm7
 ; GFNISSE-NEXT:    paddb %xmm3, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
-; GFNISSE-NEXT:    psllw $4, %xmm8
-; GFNISSE-NEXT:    pand %xmm15, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm15, %xmm8
 ; GFNISSE-NEXT:    pandn %xmm12, %xmm9
 ; GFNISSE-NEXT:    psllw $5, %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm8
-; GFNISSE-NEXT:    psllw $2, %xmm8
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm8
 ; GFNISSE-NEXT:    paddb %xmm9, %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm3
@@ -1981,33 +1874,28 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm3
 ; GFNISSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm10, %xmm8
-; GFNISSE-NEXT:    psrlw $4, %xmm8
-; GFNISSE-NEXT:    pand %xmm11, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pand %xmm12, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm10
 ; GFNISSE-NEXT:    movdqa %xmm10, %xmm8
-; GFNISSE-NEXT:    psrlw $2, %xmm8
-; GFNISSE-NEXT:    pand %xmm13, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm10
 ; GFNISSE-NEXT:    movdqa %xmm10, %xmm8
-; GFNISSE-NEXT:    psrlw $1, %xmm8
-; GFNISSE-NEXT:    pand %xmm14, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm14, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm10
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm8
-; GFNISSE-NEXT:    psllw $4, %xmm8
-; GFNISSE-NEXT:    pand %xmm15, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm15, %xmm8
 ; GFNISSE-NEXT:    pandn %xmm12, %xmm9
 ; GFNISSE-NEXT:    psllw $5, %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm8
-; GFNISSE-NEXT:    psllw $2, %xmm8
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm8
 ; GFNISSE-NEXT:    paddb %xmm9, %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm4
@@ -2029,61 +1917,56 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNIAVX1-LABEL: var_fshr_v64i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm8
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm8, %xmm6
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm6, %xmm9
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    # xmm7 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm8, %xmm9
 ; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} ymm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; GFNIAVX1-NEXT:    vandps %ymm6, %ymm4, %ymm11
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm11, %xmm10
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm10, %xmm12
 ; GFNIAVX1-NEXT:    vpblendvb %xmm12, %xmm9, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm8, %xmm9
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm9, %xmm9
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [290499906672525312,290499906672525312]
+; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm8, %xmm9
 ; GFNIAVX1-NEXT:    vpaddb %xmm12, %xmm12, %xmm12
 ; GFNIAVX1-NEXT:    vpblendvb %xmm12, %xmm9, %xmm8, %xmm9
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm9, %xmm13
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm8 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX1-NEXT:    vpand %xmm8, %xmm13, %xmm13
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [145249953336295424,145249953336295424]
+; GFNIAVX1-NEXT:    # xmm8 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm9, %xmm13
 ; GFNIAVX1-NEXT:    vpaddb %xmm12, %xmm12, %xmm12
 ; GFNIAVX1-NEXT:    vpblendvb %xmm12, %xmm13, %xmm9, %xmm12
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm9
 ; GFNIAVX1-NEXT:    vpaddb %xmm9, %xmm9, %xmm13
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm13, %xmm14
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm9 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX1-NEXT:    vpand %xmm9, %xmm14, %xmm14
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [16909320,16909320]
+; GFNIAVX1-NEXT:    # xmm9 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm13, %xmm14
 ; GFNIAVX1-NEXT:    vpxor %xmm6, %xmm10, %xmm10
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm10, %xmm15
 ; GFNIAVX1-NEXT:    vpblendvb %xmm15, %xmm14, %xmm13, %xmm13
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm13, %xmm14
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm10 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX1-NEXT:    vpand %xmm10, %xmm14, %xmm14
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm10 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    # xmm10 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm10, %xmm13, %xmm14
 ; GFNIAVX1-NEXT:    vpaddb %xmm15, %xmm15, %xmm15
 ; GFNIAVX1-NEXT:    vpblendvb %xmm15, %xmm14, %xmm13, %xmm13
 ; GFNIAVX1-NEXT:    vpaddb %xmm13, %xmm13, %xmm14
 ; GFNIAVX1-NEXT:    vpaddb %xmm15, %xmm15, %xmm15
 ; GFNIAVX1-NEXT:    vpblendvb %xmm15, %xmm14, %xmm13, %xmm13
 ; GFNIAVX1-NEXT:    vpor %xmm12, %xmm13, %xmm12
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm2, %xmm13
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm13, %xmm13
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm2, %xmm13
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm11, %xmm14
 ; GFNIAVX1-NEXT:    vpblendvb %xmm14, %xmm13, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm2, %xmm13
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm13, %xmm13
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm13
 ; GFNIAVX1-NEXT:    vpaddb %xmm14, %xmm14, %xmm14
 ; GFNIAVX1-NEXT:    vpblendvb %xmm14, %xmm13, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm2, %xmm13
-; GFNIAVX1-NEXT:    vpand %xmm8, %xmm13, %xmm13
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm2, %xmm13
 ; GFNIAVX1-NEXT:    vpaddb %xmm14, %xmm14, %xmm14
 ; GFNIAVX1-NEXT:    vpblendvb %xmm14, %xmm13, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm0, %xmm13
-; GFNIAVX1-NEXT:    vpand %xmm9, %xmm13, %xmm13
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm0, %xmm13
 ; GFNIAVX1-NEXT:    vpxor %xmm6, %xmm11, %xmm11
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm11, %xmm11
 ; GFNIAVX1-NEXT:    vpblendvb %xmm11, %xmm13, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm0, %xmm13
-; GFNIAVX1-NEXT:    vpand %xmm10, %xmm13, %xmm13
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm10, %xmm0, %xmm13
 ; GFNIAVX1-NEXT:    vpaddb %xmm11, %xmm11, %xmm11
 ; GFNIAVX1-NEXT:    vpblendvb %xmm11, %xmm13, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm13
@@ -2092,55 +1975,45 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ; GFNIAVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm12, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm11
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm11, %xmm2
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm2, %xmm12
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm11, %xmm12
 ; GFNIAVX1-NEXT:    vandps %ymm6, %ymm5, %ymm2
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm5, %xmm13
 ; GFNIAVX1-NEXT:    vpblendvb %xmm13, %xmm12, %xmm11, %xmm11
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm11, %xmm12
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm12, %xmm12
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm11, %xmm12
 ; GFNIAVX1-NEXT:    vpaddb %xmm13, %xmm13, %xmm13
 ; GFNIAVX1-NEXT:    vpblendvb %xmm13, %xmm12, %xmm11, %xmm11
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm11, %xmm12
-; GFNIAVX1-NEXT:    vpand %xmm8, %xmm12, %xmm12
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm11, %xmm12
 ; GFNIAVX1-NEXT:    vpaddb %xmm13, %xmm13, %xmm13
 ; GFNIAVX1-NEXT:    vpblendvb %xmm13, %xmm12, %xmm11, %xmm11
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm12
 ; GFNIAVX1-NEXT:    vpaddb %xmm12, %xmm12, %xmm12
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm12, %xmm13
-; GFNIAVX1-NEXT:    vpand %xmm9, %xmm13, %xmm13
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm12, %xmm13
 ; GFNIAVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
 ; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm13, %xmm12, %xmm12
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm12, %xmm13
-; GFNIAVX1-NEXT:    vpand %xmm10, %xmm13, %xmm13
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm10, %xmm12, %xmm13
 ; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
 ; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm13, %xmm12, %xmm12
 ; GFNIAVX1-NEXT:    vpaddb %xmm12, %xmm12, %xmm13
 ; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
 ; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm13, %xmm12, %xmm5
 ; GFNIAVX1-NEXT:    vpor %xmm5, %xmm11, %xmm5
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm3, %xmm11
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm11, %xmm7
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm3, %xmm7
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm11
 ; GFNIAVX1-NEXT:    vpblendvb %xmm11, %xmm7, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm3, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm7, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm3, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm11, %xmm11, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm4, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm3, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm8, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm3, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm4, %xmm3, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm9, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm1, %xmm4
 ; GFNIAVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm10, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm10, %xmm1, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm4, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm4
@@ -2152,60 +2025,50 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ;
 ; GFNIAVX2-LABEL: var_fshr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm2, %ymm6
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNIAVX2-NEXT:    vpand %ymm7, %ymm6, %ymm8
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm2, %ymm8
 ; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; GFNIAVX2-NEXT:    vpand %ymm6, %ymm4, %ymm9
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm9, %ymm9
 ; GFNIAVX2-NEXT:    vpblendvb %ymm9, %ymm8, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpsrlw $2, %ymm2, %ymm8
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNIAVX2-NEXT:    vpand %ymm10, %ymm8, %ymm8
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm2, %ymm10
 ; GFNIAVX2-NEXT:    vpaddb %ymm9, %ymm9, %ymm9
-; GFNIAVX2-NEXT:    vpblendvb %ymm9, %ymm8, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpsrlw $1, %ymm2, %ymm8
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm11 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX2-NEXT:    vpand %ymm11, %ymm8, %ymm8
+; GFNIAVX2-NEXT:    vpblendvb %ymm9, %ymm10, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm10, %ymm2, %ymm11
 ; GFNIAVX2-NEXT:    vpaddb %ymm9, %ymm9, %ymm9
-; GFNIAVX2-NEXT:    vpblendvb %ymm9, %ymm8, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpblendvb %ymm9, %ymm11, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm0, %ymm8
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm9 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX2-NEXT:    vpand %ymm9, %ymm8, %ymm8
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm9, %ymm0, %ymm11
 ; GFNIAVX2-NEXT:    vpandn %ymm6, %ymm4, %ymm4
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm4, %ymm4
-; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm8, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm0, %ymm8
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm12 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX2-NEXT:    vpand %ymm12, %ymm8, %ymm8
+; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm11, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm11 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm11, %ymm0, %ymm12
 ; GFNIAVX2-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
-; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm8, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm8
+; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm12, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm12
 ; GFNIAVX2-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
-; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm8, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm12, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm3, %ymm2
-; GFNIAVX2-NEXT:    vpand %ymm7, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpand %ymm6, %ymm5, %ymm4
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm4, %ymm4
 ; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm2, %ymm3, %ymm2
-; GFNIAVX2-NEXT:    vpsrlw $2, %ymm2, %ymm3
-; GFNIAVX2-NEXT:    vpand %ymm3, %ymm10, %ymm3
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm2, %ymm3
 ; GFNIAVX2-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
 ; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm3, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpsrlw $1, %ymm2, %ymm3
-; GFNIAVX2-NEXT:    vpand %ymm3, %ymm11, %ymm3
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm10, %ymm2, %ymm3
 ; GFNIAVX2-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
 ; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm3, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm1, %ymm3
-; GFNIAVX2-NEXT:    vpand %ymm3, %ymm9, %ymm3
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm9, %ymm1, %ymm3
 ; GFNIAVX2-NEXT:    vpandn %ymm6, %ymm5, %ymm4
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm4, %ymm4
 ; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm3, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm1, %ymm3
-; GFNIAVX2-NEXT:    vpand %ymm3, %ymm12, %ymm3
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm11, %ymm1, %ymm3
 ; GFNIAVX2-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
 ; GFNIAVX2-NEXT:    vpblendvb %ymm4, %ymm3, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm3
@@ -2216,62 +2079,52 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
 ;
 ; GFNIAVX512VL-LABEL: var_fshr_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm4, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNIAVX512VL-NEXT:    vpand %ymm5, %ymm3, %ymm6
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; GFNIAVX512VL-NEXT:    vpandq %zmm7, %zmm2, %zmm2
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm8
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm8, %ymm6, %ymm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm4, %ymm6
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm9 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm9, %ymm6
+; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm5
+; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; GFNIAVX512VL-NEXT:    vpandq %zmm6, %zmm2, %zmm2
+; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm2, %ymm7
+; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm7, %ymm8
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm8, %ymm5, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm3, %ymm9
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm8, %ymm8, %ymm8
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm8, %ymm6, %ymm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm4, %ymm6
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm10 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm10, %ymm6
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm8, %ymm9, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm9, %ymm3, %ymm10
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm8, %ymm8, %ymm8
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm8, %ymm6, %ymm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm6
-; GFNIAVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm5
-; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm6
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm5, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm1, %ymm5
-; GFNIAVX512VL-NEXT:    vpand %ymm5, %ymm9, %ymm5
-; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm5, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm1, %ymm5
-; GFNIAVX512VL-NEXT:    vpand %ymm5, %ymm10, %ymm5
-; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm5, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm1
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm4, %ymm5
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpxor %ymm7, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm5, %ymm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm4, %ymm5
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX512VL-NEXT:    vpand %ymm5, %ymm8, %ymm5
-; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm5, %ymm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpaddb %ymm4, %ymm4, %ymm5
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm8, %ymm10, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm4
+; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm8
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm8, %ymm4, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm1, %ymm4
+; GFNIAVX512VL-NEXT:    vpaddb %ymm8, %ymm8, %ymm5
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm9, %ymm1, %ymm4
+; GFNIAVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm1, %ymm1
+; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm5, %ymm4, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [16909320,16909320,16909320,16909320]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm5
+; GFNIAVX512VL-NEXT:    vpxor %ymm6, %ymm7, %ymm7
+; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm7, %ymm7
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm7, %ymm5, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm3, %ymm8
+; GFNIAVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm7
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm7, %ymm8, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm8
+; GFNIAVX512VL-NEXT:    vpaddb %ymm7, %ymm7, %ymm7
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm7, %ymm8, %ymm3, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; GFNIAVX512VL-NEXT:    vpxor %ymm7, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm4
+; GFNIAVX512VL-NEXT:    vpxor %ymm6, %ymm2, %ymm2
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm8, %ymm4
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm4
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
@@ -2874,45 +2727,31 @@ define <64 x i8> @constant_fshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 define <64 x i8> @splatconstant_fshl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNISSE-LABEL: splatconstant_fshl_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psrlw $7, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNISSE-NEXT:    pand %xmm8, %xmm4
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm4
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    por %xmm4, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm5
-; GFNISSE-NEXT:    pand %xmm8, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm1, %xmm1
 ; GFNISSE-NEXT:    por %xmm5, %xmm1
-; GFNISSE-NEXT:    psrlw $7, %xmm6
-; GFNISSE-NEXT:    pand %xmm8, %xmm6
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm6
 ; GFNISSE-NEXT:    paddb %xmm2, %xmm2
 ; GFNISSE-NEXT:    por %xmm6, %xmm2
-; GFNISSE-NEXT:    psrlw $7, %xmm7
-; GFNISSE-NEXT:    pand %xmm7, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm7
 ; GFNISSE-NEXT:    paddb %xmm3, %xmm3
-; GFNISSE-NEXT:    por %xmm8, %xmm3
+; GFNISSE-NEXT:    por %xmm7, %xmm3
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_fshl_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
-; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm4
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm2
+; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm5
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm5, %ymm0
 ; GFNIAVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm2
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm3
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
@@ -2922,35 +2761,30 @@ define <64 x i8> @splatconstant_fshl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind
 ;
 ; GFNIAVX2-LABEL: splatconstant_fshl_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm3, %ymm2
-; GFNIAVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512VL-LABEL: splatconstant_fshl_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $7, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
-; GFNIAVX512VL-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm1
+; GFNIAVX512VL-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; GFNIAVX512VL-NEXT:    retq
 ;
 ; GFNIAVX512BW-LABEL: splatconstant_fshl_v64i8:
 ; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vpsrlw $7, %zmm1, %zmm1
-; GFNIAVX512BW-NEXT:    vpaddb %zmm0, %zmm0, %zmm0
-; GFNIAVX512BW-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
+; GFNIAVX512BW-NEXT:    vpaddw %zmm0, %zmm0, %zmm2
+; GFNIAVX512BW-NEXT:    vpsrlw $7, %zmm1, %zmm0
+; GFNIAVX512BW-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0
 ; GFNIAVX512BW-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
   ret <64 x i8> %res
@@ -2960,90 +2794,51 @@ declare <64 x i8> @llvm.fshl.v64i8(<64 x i8>, <64 x i8>, <64 x i8>)
 define <64 x i8> @splatconstant_fshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNISSE-LABEL: splatconstant_fshr_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psrlw $2, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192]
-; GFNISSE-NEXT:    movdqa %xmm8, %xmm9
-; GFNISSE-NEXT:    pandn %xmm4, %xmm9
-; GFNISSE-NEXT:    psllw $6, %xmm0
-; GFNISSE-NEXT:    pand %xmm8, %xmm0
-; GFNISSE-NEXT:    por %xmm9, %xmm0
-; GFNISSE-NEXT:    psrlw $2, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm8, %xmm4
-; GFNISSE-NEXT:    pandn %xmm5, %xmm4
-; GFNISSE-NEXT:    psllw $6, %xmm1
-; GFNISSE-NEXT:    pand %xmm8, %xmm1
-; GFNISSE-NEXT:    por %xmm4, %xmm1
-; GFNISSE-NEXT:    psrlw $2, %xmm6
-; GFNISSE-NEXT:    movdqa %xmm8, %xmm4
-; GFNISSE-NEXT:    pandn %xmm6, %xmm4
-; GFNISSE-NEXT:    psllw $6, %xmm2
-; GFNISSE-NEXT:    pand %xmm8, %xmm2
-; GFNISSE-NEXT:    por %xmm4, %xmm2
-; GFNISSE-NEXT:    psrlw $2, %xmm7
-; GFNISSE-NEXT:    psllw $6, %xmm3
-; GFNISSE-NEXT:    pand %xmm8, %xmm3
-; GFNISSE-NEXT:    pandn %xmm7, %xmm8
-; GFNISSE-NEXT:    por %xmm8, %xmm3
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [290499906672525312,290499906672525312]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm4
+; GFNISSE-NEXT:    pmovsxwq {{.*#+}} xmm9 = [258,258]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm0
+; GFNISSE-NEXT:    por %xmm4, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm1
+; GFNISSE-NEXT:    por %xmm5, %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm6
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm2
+; GFNISSE-NEXT:    por %xmm6, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm7
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm3
+; GFNISSE-NEXT:    por %xmm7, %xmm3
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_fshr_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; GFNIAVX1-NEXT:    vpsllw $6, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192]
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpsllw $6, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4,0,0,128,64,32,16,8,4]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm2
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,1,0,0,0,0,0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $6, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $6, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    vorps %ymm2, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_fshr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $2, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm4 = [192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192]
-; GFNIAVX2-NEXT:    vpandn %ymm2, %ymm4, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $6, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [258,258,258,258]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $2, %ymm3, %ymm2
-; GFNIAVX2-NEXT:    vpandn %ymm2, %ymm4, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $6, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm3, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512VL-LABEL: splatconstant_fshr_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vpsllw $6, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $6, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm2
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm1, %ymm0
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; GFNIAVX512VL-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm1
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512VL-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; GFNIAVX512VL-NEXT:    retq
 ;
 ; GFNIAVX512BW-LABEL: splatconstant_fshr_v64i8:
diff --git a/llvm/test/CodeGen/X86/gfni-rotates.ll b/llvm/test/CodeGen/X86/gfni-rotates.ll
index 96aff5b2af31..9ddadca380fe 100644
--- a/llvm/test/CodeGen/X86/gfni-rotates.ll
+++ b/llvm/test/CodeGen/X86/gfni-rotates.ll
@@ -14,28 +14,23 @@ define <16 x i8> @var_rotl_v16i8(<16 x i8> %a, <16 x i8> %amt) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm1
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm3
-; GFNISSE-NEXT:    psllw $4, %xmm3
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; GFNISSE-NEXT:    por %xmm0, %xmm3
 ; GFNISSE-NEXT:    psllw $5, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm3, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $6, %xmm0
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm3
-; GFNISSE-NEXT:    psllw $2, %xmm3
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; GFNISSE-NEXT:    por %xmm0, %xmm3
 ; GFNISSE-NEXT:    paddb %xmm2, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm3, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm3
 ; GFNISSE-NEXT:    paddb %xmm1, %xmm3
 ; GFNISSE-NEXT:    por %xmm0, %xmm3
@@ -47,22 +42,17 @@ define <16 x i8> @var_rotl_v16i8(<16 x i8> %a, <16 x i8> %amt) nounwind {
 ;
 ; GFNIAVX1OR2-LABEL: var_rotl_v16i8:
 ; GFNIAVX1OR2:       # %bb.0:
-; GFNIAVX1OR2-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; GFNIAVX1OR2-NEXT:    vpsllw $4, %xmm0, %xmm3
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
 ; GFNIAVX1OR2-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpsllw $5, %xmm1, %xmm1
 ; GFNIAVX1OR2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpsrlw $6, %xmm0, %xmm2
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; GFNIAVX1OR2-NEXT:    vpsllw $2, %xmm0, %xmm3
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
 ; GFNIAVX1OR2-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; GFNIAVX1OR2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpsrlw $7, %xmm0, %xmm2
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpaddb %xmm0, %xmm0, %xmm3
 ; GFNIAVX1OR2-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
@@ -103,28 +93,23 @@ define <16 x i8> @var_rotr_v16i8(<16 x i8> %a, <16 x i8> %amt) nounwind {
 ; GFNISSE-LABEL: var_rotr_v16i8:
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
-; GFNISSE-NEXT:    psllw $4, %xmm3
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; GFNISSE-NEXT:    por %xmm0, %xmm3
 ; GFNISSE-NEXT:    pxor %xmm0, %xmm0
 ; GFNISSE-NEXT:    psubb %xmm1, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm1
-; GFNISSE-NEXT:    psrlw $6, %xmm1
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
-; GFNISSE-NEXT:    psllw $2, %xmm3
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; GFNISSE-NEXT:    por %xmm1, %xmm3
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm1
-; GFNISSE-NEXT:    psrlw $7, %xmm1
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
 ; GFNISSE-NEXT:    paddb %xmm2, %xmm3
 ; GFNISSE-NEXT:    por %xmm1, %xmm3
@@ -135,24 +120,19 @@ define <16 x i8> @var_rotr_v16i8(<16 x i8> %a, <16 x i8> %amt) nounwind {
 ;
 ; GFNIAVX1OR2-LABEL: var_rotr_v16i8:
 ; GFNIAVX1OR2:       # %bb.0:
-; GFNIAVX1OR2-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; GFNIAVX1OR2-NEXT:    vpsllw $4, %xmm0, %xmm3
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
 ; GFNIAVX1OR2-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; GFNIAVX1OR2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; GFNIAVX1OR2-NEXT:    vpsllw $5, %xmm1, %xmm1
 ; GFNIAVX1OR2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpsrlw $6, %xmm0, %xmm2
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; GFNIAVX1OR2-NEXT:    vpsllw $2, %xmm0, %xmm3
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
 ; GFNIAVX1OR2-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; GFNIAVX1OR2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpsrlw $7, %xmm0, %xmm2
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpaddb %xmm0, %xmm0, %xmm3
 ; GFNIAVX1OR2-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
@@ -389,28 +369,17 @@ define <16 x i8> @constant_rotr_v16i8(<16 x i8> %a) nounwind {
 define <16 x i8> @splatconstant_rotl_v16i8(<16 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_rotl_v16i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa %xmm0, %xmm1
-; GFNISSE-NEXT:    psrlw $5, %xmm1
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; GFNISSE-NEXT:    psllw $3, %xmm0
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; GFNISSE-NEXT:    por %xmm1, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1OR2-LABEL: splatconstant_rotl_v16i8:
 ; GFNIAVX1OR2:       # %bb.0:
-; GFNIAVX1OR2-NEXT:    vpsrlw $5, %xmm0, %xmm1
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; GFNIAVX1OR2-NEXT:    vpsllw $3, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_rotl_v16i8:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vpsllw $3, %xmm0, %xmm1
-; GFNIAVX512-NEXT:    vpsrlw $5, %xmm0, %xmm0
-; GFNIAVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
 ; GFNIAVX512-NEXT:    retq
   %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %a, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
   ret <16 x i8> %res
@@ -420,26 +389,17 @@ declare <16 x i8> @llvm.fshl.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)
 define <16 x i8> @splatconstant_rotr_v16i8(<16 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_rotr_v16i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa %xmm0, %xmm1
-; GFNISSE-NEXT:    paddb %xmm0, %xmm1
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; GFNISSE-NEXT:    por %xmm1, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1OR2-LABEL: splatconstant_rotr_v16i8:
 ; GFNIAVX1OR2:       # %bb.0:
-; GFNIAVX1OR2-NEXT:    vpaddb %xmm0, %xmm0, %xmm1
-; GFNIAVX1OR2-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_rotr_v16i8:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vpsrlw $7, %xmm0, %xmm1
-; GFNIAVX512-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
-; GFNIAVX512-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
 ; GFNIAVX512-NEXT:    retq
   %res = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %a, <16 x i8> <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>)
   ret <16 x i8> %res
@@ -455,62 +415,52 @@ define <32 x i8> @var_rotl_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNISSE-NEXT:    movdqa %xmm5, %xmm6
-; GFNISSE-NEXT:    pandn %xmm0, %xmm6
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm0
+; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm6 = [16909320,16909320]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm7
-; GFNISSE-NEXT:    psllw $4, %xmm7
-; GFNISSE-NEXT:    pand %xmm5, %xmm7
-; GFNISSE-NEXT:    por %xmm6, %xmm7
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm7
+; GFNISSE-NEXT:    por %xmm0, %xmm7
 ; GFNISSE-NEXT:    psllw $5, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm7, %xmm2
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [4647714815446351872,4647714815446351872]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
-; GFNISSE-NEXT:    psrlw $6, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNISSE-NEXT:    movdqa %xmm6, %xmm7
-; GFNISSE-NEXT:    pandn %xmm0, %xmm7
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm8
-; GFNISSE-NEXT:    psllw $2, %xmm8
-; GFNISSE-NEXT:    pand %xmm6, %xmm8
-; GFNISSE-NEXT:    por %xmm7, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm9
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm9
+; GFNISSE-NEXT:    por %xmm0, %xmm9
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm2
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm9, %xmm2
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNISSE-NEXT:    pand %xmm7, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm8
-; GFNISSE-NEXT:    paddb %xmm2, %xmm8
-; GFNISSE-NEXT:    por %xmm0, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm10
+; GFNISSE-NEXT:    paddb %xmm2, %xmm10
+; GFNISSE-NEXT:    por %xmm0, %xmm10
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm2
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm10, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $4, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm4
-; GFNISSE-NEXT:    psllw $4, %xmm4
-; GFNISSE-NEXT:    pand %xmm5, %xmm4
-; GFNISSE-NEXT:    pandn %xmm0, %xmm5
-; GFNISSE-NEXT:    por %xmm4, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm4
+; GFNISSE-NEXT:    por %xmm0, %xmm4
 ; GFNISSE-NEXT:    psllw $5, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm1
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $6, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm4
-; GFNISSE-NEXT:    psllw $2, %xmm4
-; GFNISSE-NEXT:    pand %xmm6, %xmm4
-; GFNISSE-NEXT:    pandn %xmm0, %xmm6
-; GFNISSE-NEXT:    por %xmm4, %xmm6
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm4
+; GFNISSE-NEXT:    por %xmm0, %xmm4
 ; GFNISSE-NEXT:    paddb %xmm3, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm6, %xmm1
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    pand %xmm7, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm4
 ; GFNISSE-NEXT:    paddb %xmm1, %xmm4
 ; GFNISSE-NEXT:    por %xmm0, %xmm4
@@ -523,46 +473,43 @@ define <32 x i8> @var_rotl_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ; GFNIAVX1-LABEL: var_rotl_v32i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm2, %xmm5
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm5, %xmm5
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    # xmm3 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm2, %xmm4
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [16909320,16909320]
+; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm6
+; GFNIAVX1-NEXT:    vpor %xmm4, %xmm6, %xmm4
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
+; GFNIAVX1-NEXT:    vpsllw $5, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm4, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [4647714815446351872,4647714815446351872]
+; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm7
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    # xmm8 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm2, %xmm9
+; GFNIAVX1-NEXT:    vpor %xmm7, %xmm9, %xmm7
+; GFNIAVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm7, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [9223372036854775808,9223372036854775808]
+; GFNIAVX1-NEXT:    # xmm7 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm2, %xmm9
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm10
+; GFNIAVX1-NEXT:    vpor %xmm9, %xmm10, %xmm9
+; GFNIAVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm9, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm0, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm0, %xmm5
 ; GFNIAVX1-NEXT:    vpor %xmm3, %xmm5, %xmm3
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; GFNIAVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm6, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm7, %xmm7
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm7, %xmm3
-; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm8
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm8, %xmm3
-; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm0, %xmm5
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm5, %xmm4
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm6, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm0, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm0, %xmm4
 ; GFNIAVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm0, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm4
 ; GFNIAVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
@@ -572,22 +519,22 @@ define <32 x i8> @var_rotl_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_rotl_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm0, %ymm3
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $6, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm0, %ymm3
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
@@ -596,21 +543,21 @@ define <32 x i8> @var_rotl_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ;
 ; GFNIAVX512VL-LABEL: var_rotl_v32i8:
 ; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm3
+; GFNIAVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm3
+; GFNIAVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm2
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3
+; GFNIAVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    retq
 ;
 ; GFNIAVX512BW-LABEL: var_rotl_v32i8:
@@ -634,63 +581,53 @@ define <32 x i8> @var_rotr_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ; GFNISSE-LABEL: var_rotr_v32i8:
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm5
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNISSE-NEXT:    movdqa %xmm6, %xmm4
-; GFNISSE-NEXT:    pandn %xmm0, %xmm4
-; GFNISSE-NEXT:    movdqa %xmm5, %xmm7
-; GFNISSE-NEXT:    psllw $4, %xmm7
-; GFNISSE-NEXT:    pand %xmm6, %xmm7
-; GFNISSE-NEXT:    por %xmm4, %xmm7
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm0
+; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm7 = [16909320,16909320]
+; GFNISSE-NEXT:    movdqa %xmm5, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm8
+; GFNISSE-NEXT:    por %xmm0, %xmm8
 ; GFNISSE-NEXT:    pxor %xmm4, %xmm4
 ; GFNISSE-NEXT:    pxor %xmm0, %xmm0
 ; GFNISSE-NEXT:    psubb %xmm2, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm7, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm5, %xmm7
-; GFNISSE-NEXT:    psrlw $6, %xmm7
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm8
-; GFNISSE-NEXT:    pandn %xmm7, %xmm8
-; GFNISSE-NEXT:    movdqa %xmm5, %xmm7
-; GFNISSE-NEXT:    psllw $2, %xmm7
-; GFNISSE-NEXT:    pand %xmm2, %xmm7
-; GFNISSE-NEXT:    por %xmm8, %xmm7
-; GFNISSE-NEXT:    paddb %xmm0, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm7, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm5, %xmm8
-; GFNISSE-NEXT:    psrlw $7, %xmm8
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNISSE-NEXT:    pand %xmm7, %xmm8
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm5
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [4647714815446351872,4647714815446351872]
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm9
-; GFNISSE-NEXT:    paddb %xmm5, %xmm9
-; GFNISSE-NEXT:    por %xmm8, %xmm9
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm9
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa %xmm5, %xmm10
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm10
+; GFNISSE-NEXT:    por %xmm9, %xmm10
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm9, %xmm5
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm10, %xmm5
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; GFNISSE-NEXT:    movdqa %xmm5, %xmm10
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm10
+; GFNISSE-NEXT:    movdqa %xmm5, %xmm11
+; GFNISSE-NEXT:    paddb %xmm5, %xmm11
+; GFNISSE-NEXT:    por %xmm10, %xmm11
+; GFNISSE-NEXT:    paddb %xmm0, %xmm0
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm11, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm8
-; GFNISSE-NEXT:    psllw $4, %xmm8
-; GFNISSE-NEXT:    pand %xmm6, %xmm8
-; GFNISSE-NEXT:    pandn %xmm0, %xmm6
-; GFNISSE-NEXT:    por %xmm8, %xmm6
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm6
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm6
+; GFNISSE-NEXT:    por %xmm0, %xmm6
 ; GFNISSE-NEXT:    psubb %xmm3, %xmm4
 ; GFNISSE-NEXT:    psllw $5, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm6, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $6, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm3
-; GFNISSE-NEXT:    psllw $2, %xmm3
-; GFNISSE-NEXT:    pand %xmm2, %xmm3
-; GFNISSE-NEXT:    pandn %xmm0, %xmm2
-; GFNISSE-NEXT:    por %xmm3, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm8, %xmm2
+; GFNISSE-NEXT:    por %xmm0, %xmm2
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    pand %xmm7, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm2
 ; GFNISSE-NEXT:    paddb %xmm1, %xmm2
 ; GFNISSE-NEXT:    por %xmm0, %xmm2
@@ -703,49 +640,46 @@ define <32 x i8> @var_rotr_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ; GFNIAVX1-LABEL: var_rotr_v32i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm2, %xmm5
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm5, %xmm5
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    # xmm3 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm2, %xmm4
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [16909320,16909320]
+; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm6
+; GFNIAVX1-NEXT:    vpor %xmm4, %xmm6, %xmm4
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
+; GFNIAVX1-NEXT:    vpxor %xmm7, %xmm7, %xmm7
+; GFNIAVX1-NEXT:    vpsubb %xmm6, %xmm7, %xmm6
+; GFNIAVX1-NEXT:    vpsllw $5, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm4, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [4647714815446351872,4647714815446351872]
+; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm8
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    # xmm9 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm2, %xmm10
+; GFNIAVX1-NEXT:    vpor %xmm8, %xmm10, %xmm8
+; GFNIAVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm8, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
+; GFNIAVX1-NEXT:    # xmm8 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm2, %xmm10
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm11
+; GFNIAVX1-NEXT:    vpor %xmm10, %xmm11, %xmm10
+; GFNIAVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vpblendvb %xmm6, %xmm10, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm0, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm0, %xmm5
 ; GFNIAVX1-NEXT:    vpor %xmm3, %xmm5, %xmm3
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; GFNIAVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; GFNIAVX1-NEXT:    vpsubb %xmm5, %xmm6, %xmm5
-; GFNIAVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm7, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm2, %xmm8
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm8, %xmm3
-; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm8, %xmm3
-; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm9
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm9, %xmm3
-; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm0, %xmm5
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm5, %xmm4
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; GFNIAVX1-NEXT:    vpsubb %xmm1, %xmm6, %xmm1
+; GFNIAVX1-NEXT:    vpsubb %xmm1, %xmm7, %xmm1
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm7, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm0, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm0, %xmm4
 ; GFNIAVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm8, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm0, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm4
 ; GFNIAVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
@@ -755,24 +689,24 @@ define <32 x i8> @var_rotr_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_rotr_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm0, %ymm3
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; GFNIAVX2-NEXT:    vpsubb %ymm1, %ymm3, %ymm1
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $6, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm0, %ymm3
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
@@ -781,21 +715,21 @@ define <32 x i8> @var_rotr_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
 ;
 ; GFNIAVX512VL-LABEL: var_rotr_v32i8:
 ; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm3
+; GFNIAVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpsllw $6, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm3
+; GFNIAVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm3
+; GFNIAVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    retq
 ;
 ; GFNIAVX512BW-LABEL: var_rotr_v32i8:
@@ -1141,53 +1075,25 @@ define <32 x i8> @constant_rotr_v32i8(<32 x i8> %a) nounwind {
 define <32 x i8> @splatconstant_rotl_v32i8(<32 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_rotl_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
-; GFNISSE-NEXT:    psrlw $4, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm4
-; GFNISSE-NEXT:    pandn %xmm2, %xmm4
-; GFNISSE-NEXT:    psllw $4, %xmm0
-; GFNISSE-NEXT:    pand %xmm3, %xmm0
-; GFNISSE-NEXT:    por %xmm4, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm2
-; GFNISSE-NEXT:    psrlw $4, %xmm2
-; GFNISSE-NEXT:    psllw $4, %xmm1
-; GFNISSE-NEXT:    pand %xmm3, %xmm1
-; GFNISSE-NEXT:    pandn %xmm2, %xmm3
-; GFNISSE-NEXT:    por %xmm3, %xmm1
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [1161999622378488840,1161999622378488840]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_rotl_v32i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; GFNIAVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_rotl_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm0, %ymm1
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [1161999622378488840,1161999622378488840,1161999622378488840,1161999622378488840]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_rotl_v32i8:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vpsllw $4, %ymm0, %ymm1
-; GFNIAVX512-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; GFNIAVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
 ; GFNIAVX512-NEXT:    retq
   %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a, <32 x i8> %a, <32 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <32 x i8> %res
@@ -1197,53 +1103,25 @@ declare <32 x i8> @llvm.fshl.v32i8(<32 x i8>, <32 x i8>, <32 x i8>)
 define <32 x i8> @splatconstant_rotr_v32i8(<32 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_rotr_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
-; GFNISSE-NEXT:    psrlw $6, %xmm2
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm3 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm4
-; GFNISSE-NEXT:    pandn %xmm2, %xmm4
-; GFNISSE-NEXT:    psllw $2, %xmm0
-; GFNISSE-NEXT:    pand %xmm3, %xmm0
-; GFNISSE-NEXT:    por %xmm4, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm2
-; GFNISSE-NEXT:    psrlw $6, %xmm2
-; GFNISSE-NEXT:    psllw $2, %xmm1
-; GFNISSE-NEXT:    pand %xmm3, %xmm1
-; GFNISSE-NEXT:    pandn %xmm2, %xmm3
-; GFNISSE-NEXT:    por %xmm3, %xmm1
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [4647715923615551520,4647715923615551520]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_rotr_v32i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm1, %xmm2
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm0, %xmm2
-; GFNIAVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_rotr_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $6, %ymm0, %ymm1
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4647715923615551520,4647715923615551520,4647715923615551520,4647715923615551520]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_rotr_v32i8:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vpsllw $2, %ymm0, %ymm1
-; GFNIAVX512-NEXT:    vpsrlw $6, %ymm0, %ymm0
-; GFNIAVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
 ; GFNIAVX512-NEXT:    retq
   %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a, <32 x i8> %a, <32 x i8> <i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6>)
   ret <32 x i8> %res
@@ -1259,64 +1137,52 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm4
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNISSE-NEXT:    movdqa %xmm9, %xmm10
-; GFNISSE-NEXT:    pandn %xmm0, %xmm10
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm0
+; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm10 = [16909320,16909320]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm11
-; GFNISSE-NEXT:    psllw $4, %xmm11
-; GFNISSE-NEXT:    pand %xmm9, %xmm11
-; GFNISSE-NEXT:    por %xmm10, %xmm11
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm11
+; GFNISSE-NEXT:    por %xmm0, %xmm11
 ; GFNISSE-NEXT:    psllw $5, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm11, %xmm4
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [4647714815446351872,4647714815446351872]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
-; GFNISSE-NEXT:    psrlw $6, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNISSE-NEXT:    movdqa %xmm10, %xmm11
-; GFNISSE-NEXT:    pandn %xmm0, %xmm11
-; GFNISSE-NEXT:    movdqa %xmm4, %xmm12
-; GFNISSE-NEXT:    psllw $2, %xmm12
-; GFNISSE-NEXT:    pand %xmm10, %xmm12
-; GFNISSE-NEXT:    por %xmm11, %xmm12
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm0
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm12 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm13
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm13
+; GFNISSE-NEXT:    por %xmm0, %xmm13
 ; GFNISSE-NEXT:    paddb %xmm8, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm4
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm13, %xmm4
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm13 = [9223372036854775808,9223372036854775808]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNISSE-NEXT:    pand %xmm11, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm4, %xmm12
-; GFNISSE-NEXT:    paddb %xmm4, %xmm12
-; GFNISSE-NEXT:    por %xmm0, %xmm12
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm4, %xmm14
+; GFNISSE-NEXT:    paddb %xmm4, %xmm14
+; GFNISSE-NEXT:    por %xmm0, %xmm14
 ; GFNISSE-NEXT:    paddb %xmm8, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm4
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm14, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm9, %xmm8
-; GFNISSE-NEXT:    pandn %xmm0, %xmm8
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm12
-; GFNISSE-NEXT:    psllw $4, %xmm12
-; GFNISSE-NEXT:    pand %xmm9, %xmm12
-; GFNISSE-NEXT:    por %xmm8, %xmm12
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm8
+; GFNISSE-NEXT:    por %xmm0, %xmm8
 ; GFNISSE-NEXT:    psllw $5, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm1
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $6, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm10, %xmm8
-; GFNISSE-NEXT:    pandn %xmm0, %xmm8
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm12
-; GFNISSE-NEXT:    psllw $2, %xmm12
-; GFNISSE-NEXT:    pand %xmm10, %xmm12
-; GFNISSE-NEXT:    por %xmm8, %xmm12
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm8
+; GFNISSE-NEXT:    por %xmm0, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm5, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm1
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    pand %xmm11, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm1, %xmm8
 ; GFNISSE-NEXT:    por %xmm0, %xmm8
@@ -1324,30 +1190,23 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm9, %xmm5
-; GFNISSE-NEXT:    pandn %xmm0, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm8
-; GFNISSE-NEXT:    psllw $4, %xmm8
-; GFNISSE-NEXT:    pand %xmm9, %xmm8
-; GFNISSE-NEXT:    por %xmm5, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm5
+; GFNISSE-NEXT:    por %xmm0, %xmm5
 ; GFNISSE-NEXT:    psllw $5, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm2
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
-; GFNISSE-NEXT:    psrlw $6, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm10, %xmm5
-; GFNISSE-NEXT:    pandn %xmm0, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm8
-; GFNISSE-NEXT:    psllw $2, %xmm8
-; GFNISSE-NEXT:    pand %xmm10, %xmm8
-; GFNISSE-NEXT:    por %xmm5, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm5
+; GFNISSE-NEXT:    por %xmm0, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm6, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm2
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    pand %xmm11, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm2, %xmm5
 ; GFNISSE-NEXT:    por %xmm0, %xmm5
@@ -1355,28 +1214,23 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
-; GFNISSE-NEXT:    psrlw $4, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm5
-; GFNISSE-NEXT:    psllw $4, %xmm5
-; GFNISSE-NEXT:    pand %xmm9, %xmm5
-; GFNISSE-NEXT:    pandn %xmm0, %xmm9
-; GFNISSE-NEXT:    por %xmm5, %xmm9
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm5
+; GFNISSE-NEXT:    por %xmm0, %xmm5
 ; GFNISSE-NEXT:    psllw $5, %xmm7
 ; GFNISSE-NEXT:    movdqa %xmm7, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm9, %xmm3
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
-; GFNISSE-NEXT:    psrlw $6, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm5
-; GFNISSE-NEXT:    psllw $2, %xmm5
-; GFNISSE-NEXT:    pand %xmm10, %xmm5
-; GFNISSE-NEXT:    pandn %xmm0, %xmm10
-; GFNISSE-NEXT:    por %xmm5, %xmm10
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm5
+; GFNISSE-NEXT:    por %xmm0, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm7, %xmm7
 ; GFNISSE-NEXT:    movdqa %xmm7, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm10, %xmm3
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    pand %xmm11, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm3, %xmm5
 ; GFNISSE-NEXT:    por %xmm0, %xmm5
@@ -1388,90 +1242,77 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ;
 ; GFNIAVX1-LABEL: var_rotl_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm5, %xmm6
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX1-NEXT:    vpandn %xmm6, %xmm4, %xmm6
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm5, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm7, %xmm7
-; GFNIAVX1-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
-; GFNIAVX1-NEXT:    vpsllw $5, %xmm7, %xmm7
-; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm6, %xmm5, %xmm6
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm6, %xmm8
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX1-NEXT:    vpandn %xmm8, %xmm5, %xmm8
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm6, %xmm9
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpor %xmm8, %xmm9, %xmm8
-; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
-; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm8, %xmm6, %xmm8
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm8, %xmm9
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm10
-; GFNIAVX1-NEXT:    vpor %xmm9, %xmm10, %xmm9
-; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
-; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm9, %xmm8, %xmm7
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm0, %xmm8
-; GFNIAVX1-NEXT:    vpandn %xmm8, %xmm4, %xmm8
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm0, %xmm9
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpor %xmm8, %xmm9, %xmm8
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm6, %xmm7
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [16909320,16909320]
+; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm6, %xmm8
+; GFNIAVX1-NEXT:    vpor %xmm7, %xmm8, %xmm7
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm8
+; GFNIAVX1-NEXT:    vpsllw $5, %xmm8, %xmm8
+; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm7, %xmm6, %xmm9
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [4647714815446351872,4647714815446351872]
+; GFNIAVX1-NEXT:    # xmm6 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm9, %xmm10
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    # xmm7 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm9, %xmm11
+; GFNIAVX1-NEXT:    vpor %xmm10, %xmm11, %xmm10
+; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm11
+; GFNIAVX1-NEXT:    vpblendvb %xmm11, %xmm10, %xmm9, %xmm9
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
+; GFNIAVX1-NEXT:    # xmm8 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm9, %xmm10
+; GFNIAVX1-NEXT:    vpaddb %xmm9, %xmm9, %xmm12
+; GFNIAVX1-NEXT:    vpor %xmm10, %xmm12, %xmm10
+; GFNIAVX1-NEXT:    vpaddb %xmm11, %xmm11, %xmm11
+; GFNIAVX1-NEXT:    vpblendvb %xmm11, %xmm10, %xmm9, %xmm9
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm10
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm0, %xmm11
+; GFNIAVX1-NEXT:    vpor %xmm10, %xmm11, %xmm10
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm8, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm0, %xmm8
-; GFNIAVX1-NEXT:    vpandn %xmm8, %xmm5, %xmm8
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm0, %xmm9
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpor %xmm8, %xmm9, %xmm8
+; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm10, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm0, %xmm10
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm0, %xmm11
+; GFNIAVX1-NEXT:    vpor %xmm10, %xmm11, %xmm10
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm8, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm0, %xmm8
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm9
-; GFNIAVX1-NEXT:    vpor %xmm8, %xmm9, %xmm8
+; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm10, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm0, %xmm10
+; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm11
+; GFNIAVX1-NEXT:    vpor %xmm10, %xmm11, %xmm10
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm8, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm10, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vpandn %xmm7, %xmm4, %xmm7
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm2, %xmm8
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpor %xmm7, %xmm8, %xmm7
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm8
-; GFNIAVX1-NEXT:    vpsllw $5, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm7, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vpandn %xmm7, %xmm5, %xmm7
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm2, %xmm9
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpor %xmm7, %xmm9, %xmm7
-; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm7, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm7, %xmm7
-; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm9
-; GFNIAVX1-NEXT:    vpor %xmm7, %xmm9, %xmm7
-; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm7, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm1, %xmm7
-; GFNIAVX1-NEXT:    vpandn %xmm7, %xmm4, %xmm7
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm1, %xmm8
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm8, %xmm4
-; GFNIAVX1-NEXT:    vpor %xmm7, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm9
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm10
+; GFNIAVX1-NEXT:    vpor %xmm9, %xmm10, %xmm9
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm10
+; GFNIAVX1-NEXT:    vpsllw $5, %xmm10, %xmm10
+; GFNIAVX1-NEXT:    vpblendvb %xmm10, %xmm9, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm2, %xmm9
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm2, %xmm11
+; GFNIAVX1-NEXT:    vpor %xmm9, %xmm11, %xmm9
+; GFNIAVX1-NEXT:    vpaddb %xmm10, %xmm10, %xmm10
+; GFNIAVX1-NEXT:    vpblendvb %xmm10, %xmm9, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm2, %xmm9
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm11
+; GFNIAVX1-NEXT:    vpor %xmm9, %xmm11, %xmm9
+; GFNIAVX1-NEXT:    vpaddb %xmm10, %xmm10, %xmm10
+; GFNIAVX1-NEXT:    vpblendvb %xmm10, %xmm9, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm1, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm1, %xmm5
+; GFNIAVX1-NEXT:    vpor %xmm4, %xmm5, %xmm4
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm3, %xmm3
 ; GFNIAVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpandn %xmm4, %xmm5, %xmm4
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm1, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm7, %xmm5
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm1, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm1, %xmm5
 ; GFNIAVX1-NEXT:    vpor %xmm4, %xmm5, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm3, %xmm3
 ; GFNIAVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm1, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm5
 ; GFNIAVX1-NEXT:    vpor %xmm4, %xmm5, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm3, %xmm3
@@ -1481,45 +1322,37 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_rotl_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX2-NEXT:    vpandn %ymm4, %ymm5, %ymm4
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm0, %ymm6
-; GFNIAVX2-NEXT:    vpand %ymm5, %ymm6, %ymm6
-; GFNIAVX2-NEXT:    vpor %ymm4, %ymm6, %ymm4
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm5
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm0, %ymm7
+; GFNIAVX2-NEXT:    vpor %ymm5, %ymm7, %ymm5
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $6, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX2-NEXT:    vpandn %ymm4, %ymm6, %ymm4
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm0, %ymm7
-; GFNIAVX2-NEXT:    vpand %ymm6, %ymm7, %ymm7
-; GFNIAVX2-NEXT:    vpor %ymm4, %ymm7, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm5, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm7
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm0, %ymm9
+; GFNIAVX2-NEXT:    vpor %ymm7, %ymm9, %ymm7
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX2-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm8
-; GFNIAVX2-NEXT:    vpor %ymm4, %ymm8, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm7, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm0, %ymm9
+; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm10
+; GFNIAVX2-NEXT:    vpor %ymm9, %ymm10, %ymm9
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpandn %ymm2, %ymm5, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm1, %ymm4
-; GFNIAVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm9, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm1, %ymm4
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm3, %ymm3
 ; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpandn %ymm2, %ymm6, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm1, %ymm4
-; GFNIAVX2-NEXT:    vpand %ymm6, %ymm4, %ymm4
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm1, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm1, %ymm4
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
 ; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpand %ymm7, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm1, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
@@ -1529,40 +1362,42 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNIAVX512VL-LABEL: var_rotl_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm4
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160]
-; GFNIAVX512VL-NEXT:    vpternlogd $226, %ymm3, %ymm5, %ymm4
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm4, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpsrlw $6, %ymm2, %ymm4
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm2, %ymm6
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm7 = [4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268]
-; GFNIAVX512VL-NEXT:    vpternlogd $226, %ymm4, %ymm7, %ymm6
-; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm6, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpsrlw $7, %ymm2, %ymm4
-; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm6
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX512VL-NEXT:    vpternlogq $248, %ymm8, %ymm4, %ymm6
-; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm6, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpternlogd $226, %ymm3, %ymm5, %ymm4
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm2, %ymm4
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [16909320,16909320,16909320,16909320]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm2, %ymm6
+; GFNIAVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
+; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm6, %ymm6
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm4, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm7
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm2, %ymm9
+; GFNIAVX512VL-NEXT:    vpor %ymm7, %ymm9, %ymm7
+; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm7, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm2, %ymm9
+; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm10
+; GFNIAVX512VL-NEXT:    vpor %ymm9, %ymm10, %ymm9
+; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm9, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm5
+; GFNIAVX512VL-NEXT:    vpor %ymm3, %ymm5, %ymm3
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpternlogd $226, %ymm3, %ymm7, %ymm4
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm0, %ymm4
+; GFNIAVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm0, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpternlogq $248, %ymm8, %ymm3, %ymm4
+; GFNIAVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; GFNIAVX512VL-NEXT:    retq
 ;
@@ -1587,123 +1422,99 @@ define <64 x i8> @var_rotr_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNISSE-LABEL: var_rotr_v64i8:
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm9
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNISSE-NEXT:    movdqa %xmm10, %xmm8
-; GFNISSE-NEXT:    pandn %xmm0, %xmm8
-; GFNISSE-NEXT:    movdqa %xmm9, %xmm11
-; GFNISSE-NEXT:    psllw $4, %xmm11
-; GFNISSE-NEXT:    pand %xmm10, %xmm11
-; GFNISSE-NEXT:    por %xmm8, %xmm11
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm0
+; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm11 = [16909320,16909320]
+; GFNISSE-NEXT:    movdqa %xmm9, %xmm12
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm12
+; GFNISSE-NEXT:    por %xmm0, %xmm12
 ; GFNISSE-NEXT:    pxor %xmm8, %xmm8
 ; GFNISSE-NEXT:    pxor %xmm0, %xmm0
 ; GFNISSE-NEXT:    psubb %xmm4, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm11, %xmm9
-; GFNISSE-NEXT:    movdqa %xmm9, %xmm11
-; GFNISSE-NEXT:    psrlw $6, %xmm11
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNISSE-NEXT:    movdqa %xmm4, %xmm12
-; GFNISSE-NEXT:    pandn %xmm11, %xmm12
-; GFNISSE-NEXT:    movdqa %xmm9, %xmm11
-; GFNISSE-NEXT:    psllw $2, %xmm11
-; GFNISSE-NEXT:    pand %xmm4, %xmm11
-; GFNISSE-NEXT:    por %xmm12, %xmm11
-; GFNISSE-NEXT:    paddb %xmm0, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm11, %xmm9
-; GFNISSE-NEXT:    movdqa %xmm9, %xmm12
-; GFNISSE-NEXT:    psrlw $7, %xmm12
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNISSE-NEXT:    pand %xmm11, %xmm12
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm9
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [4647714815446351872,4647714815446351872]
 ; GFNISSE-NEXT:    movdqa %xmm9, %xmm13
-; GFNISSE-NEXT:    paddb %xmm9, %xmm13
-; GFNISSE-NEXT:    por %xmm12, %xmm13
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm13
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm12 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa %xmm9, %xmm14
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm14
+; GFNISSE-NEXT:    por %xmm13, %xmm14
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm13, %xmm9
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm14, %xmm9
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm13 = [9223372036854775808,9223372036854775808]
+; GFNISSE-NEXT:    movdqa %xmm9, %xmm14
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm14
+; GFNISSE-NEXT:    movdqa %xmm9, %xmm15
+; GFNISSE-NEXT:    paddb %xmm9, %xmm15
+; GFNISSE-NEXT:    por %xmm14, %xmm15
+; GFNISSE-NEXT:    paddb %xmm0, %xmm0
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm15, %xmm9
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm10, %xmm12
-; GFNISSE-NEXT:    pandn %xmm0, %xmm12
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm13
-; GFNISSE-NEXT:    psllw $4, %xmm13
-; GFNISSE-NEXT:    pand %xmm10, %xmm13
-; GFNISSE-NEXT:    por %xmm12, %xmm13
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm14
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm14
+; GFNISSE-NEXT:    por %xmm0, %xmm14
 ; GFNISSE-NEXT:    pxor %xmm0, %xmm0
 ; GFNISSE-NEXT:    psubb %xmm5, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm13, %xmm1
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm5
-; GFNISSE-NEXT:    psrlw $6, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm4, %xmm12
-; GFNISSE-NEXT:    pandn %xmm5, %xmm12
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm14, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm5
-; GFNISSE-NEXT:    psllw $2, %xmm5
-; GFNISSE-NEXT:    pand %xmm4, %xmm5
-; GFNISSE-NEXT:    por %xmm12, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm14
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm14
+; GFNISSE-NEXT:    por %xmm5, %xmm14
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm1
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm14, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm5
-; GFNISSE-NEXT:    psrlw $7, %xmm5
-; GFNISSE-NEXT:    pand %xmm11, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm12
-; GFNISSE-NEXT:    paddb %xmm1, %xmm12
-; GFNISSE-NEXT:    por %xmm5, %xmm12
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm1, %xmm14
+; GFNISSE-NEXT:    paddb %xmm1, %xmm14
+; GFNISSE-NEXT:    por %xmm5, %xmm14
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm1
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm14, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm0
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm10, %xmm5
-; GFNISSE-NEXT:    pandn %xmm0, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm12
-; GFNISSE-NEXT:    psllw $4, %xmm12
-; GFNISSE-NEXT:    pand %xmm10, %xmm12
-; GFNISSE-NEXT:    por %xmm5, %xmm12
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm5
+; GFNISSE-NEXT:    por %xmm0, %xmm5
 ; GFNISSE-NEXT:    pxor %xmm0, %xmm0
 ; GFNISSE-NEXT:    psubb %xmm6, %xmm0
 ; GFNISSE-NEXT:    psllw $5, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psrlw $6, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm4, %xmm6
-; GFNISSE-NEXT:    pandn %xmm5, %xmm6
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psllw $2, %xmm5
-; GFNISSE-NEXT:    pand %xmm4, %xmm5
-; GFNISSE-NEXT:    por %xmm6, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm5
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm6
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm6
+; GFNISSE-NEXT:    por %xmm5, %xmm6
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm6, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psrlw $7, %xmm5
-; GFNISSE-NEXT:    pand %xmm11, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm6
 ; GFNISSE-NEXT:    paddb %xmm2, %xmm6
 ; GFNISSE-NEXT:    por %xmm5, %xmm6
 ; GFNISSE-NEXT:    paddb %xmm0, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm6, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
-; GFNISSE-NEXT:    psrlw $4, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm5
-; GFNISSE-NEXT:    psllw $4, %xmm5
-; GFNISSE-NEXT:    pand %xmm10, %xmm5
-; GFNISSE-NEXT:    pandn %xmm0, %xmm10
-; GFNISSE-NEXT:    por %xmm5, %xmm10
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm5
+; GFNISSE-NEXT:    por %xmm0, %xmm5
 ; GFNISSE-NEXT:    psubb %xmm7, %xmm8
 ; GFNISSE-NEXT:    psllw $5, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm10, %xmm3
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
-; GFNISSE-NEXT:    psrlw $6, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm5
-; GFNISSE-NEXT:    psllw $2, %xmm5
-; GFNISSE-NEXT:    pand %xmm4, %xmm5
-; GFNISSE-NEXT:    pandn %xmm0, %xmm4
-; GFNISSE-NEXT:    por %xmm5, %xmm4
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
+; GFNISSE-NEXT:    movdqa %xmm3, %xmm4
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm12, %xmm4
+; GFNISSE-NEXT:    por %xmm0, %xmm4
 ; GFNISSE-NEXT:    paddb %xmm8, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm4, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    pand %xmm11, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm13, %xmm0
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm4
 ; GFNISSE-NEXT:    paddb %xmm3, %xmm4
 ; GFNISSE-NEXT:    por %xmm0, %xmm4
@@ -1715,95 +1526,82 @@ define <64 x i8> @var_rotr_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ;
 ; GFNIAVX1-LABEL: var_rotr_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm5, %xmm6
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX1-NEXT:    vpandn %xmm6, %xmm4, %xmm6
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm5, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm7, %xmm7
-; GFNIAVX1-NEXT:    vpor %xmm6, %xmm7, %xmm7
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm8
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm7, %xmm6
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [16909320,16909320]
+; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm7, %xmm8
+; GFNIAVX1-NEXT:    vpor %xmm6, %xmm8, %xmm8
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
 ; GFNIAVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; GFNIAVX1-NEXT:    vpsubb %xmm8, %xmm6, %xmm8
-; GFNIAVX1-NEXT:    vpsllw $5, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm7, %xmm5, %xmm7
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm7, %xmm9
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX1-NEXT:    vpandn %xmm9, %xmm5, %xmm9
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm7, %xmm10
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm10, %xmm10
-; GFNIAVX1-NEXT:    vpor %xmm9, %xmm10, %xmm9
-; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm9, %xmm7, %xmm9
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm9, %xmm10
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm10, %xmm10
-; GFNIAVX1-NEXT:    vpaddb %xmm9, %xmm9, %xmm11
-; GFNIAVX1-NEXT:    vpor %xmm10, %xmm11, %xmm10
-; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm10, %xmm9, %xmm8
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm0, %xmm9
-; GFNIAVX1-NEXT:    vpandn %xmm9, %xmm4, %xmm9
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm0, %xmm10
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm10, %xmm10
-; GFNIAVX1-NEXT:    vpor %xmm9, %xmm10, %xmm9
+; GFNIAVX1-NEXT:    vpsubb %xmm9, %xmm6, %xmm9
+; GFNIAVX1-NEXT:    vpsllw $5, %xmm9, %xmm9
+; GFNIAVX1-NEXT:    vpblendvb %xmm9, %xmm8, %xmm7, %xmm10
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [4647714815446351872,4647714815446351872]
+; GFNIAVX1-NEXT:    # xmm7 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm10, %xmm11
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm8 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    # xmm8 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm10, %xmm12
+; GFNIAVX1-NEXT:    vpor %xmm11, %xmm12, %xmm11
+; GFNIAVX1-NEXT:    vpaddb %xmm9, %xmm9, %xmm12
+; GFNIAVX1-NEXT:    vpblendvb %xmm12, %xmm11, %xmm10, %xmm10
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; GFNIAVX1-NEXT:    # xmm9 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm10, %xmm11
+; GFNIAVX1-NEXT:    vpaddb %xmm10, %xmm10, %xmm13
+; GFNIAVX1-NEXT:    vpor %xmm11, %xmm13, %xmm11
+; GFNIAVX1-NEXT:    vpaddb %xmm12, %xmm12, %xmm12
+; GFNIAVX1-NEXT:    vpblendvb %xmm12, %xmm11, %xmm10, %xmm10
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm11
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm0, %xmm12
+; GFNIAVX1-NEXT:    vpor %xmm11, %xmm12, %xmm11
 ; GFNIAVX1-NEXT:    vpsubb %xmm2, %xmm6, %xmm2
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm9, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm0, %xmm9
-; GFNIAVX1-NEXT:    vpandn %xmm9, %xmm5, %xmm9
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm0, %xmm10
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm10, %xmm10
-; GFNIAVX1-NEXT:    vpor %xmm9, %xmm10, %xmm9
+; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm11, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm0, %xmm11
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm0, %xmm12
+; GFNIAVX1-NEXT:    vpor %xmm11, %xmm12, %xmm11
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm9, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm0, %xmm9
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm10
-; GFNIAVX1-NEXT:    vpor %xmm9, %xmm10, %xmm9
+; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm11, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm0, %xmm11
+; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm12
+; GFNIAVX1-NEXT:    vpor %xmm11, %xmm12, %xmm11
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm9, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm11, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm10, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm2, %xmm8
-; GFNIAVX1-NEXT:    vpandn %xmm8, %xmm4, %xmm8
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm2, %xmm9
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpor %xmm8, %xmm9, %xmm8
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm9
-; GFNIAVX1-NEXT:    vpsubb %xmm9, %xmm6, %xmm9
-; GFNIAVX1-NEXT:    vpsllw $5, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpblendvb %xmm9, %xmm8, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm2, %xmm8
-; GFNIAVX1-NEXT:    vpandn %xmm8, %xmm5, %xmm8
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm2, %xmm10
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm10, %xmm10
-; GFNIAVX1-NEXT:    vpor %xmm8, %xmm10, %xmm8
-; GFNIAVX1-NEXT:    vpaddb %xmm9, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpblendvb %xmm9, %xmm8, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm2, %xmm8
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm8, %xmm8
-; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm10
-; GFNIAVX1-NEXT:    vpor %xmm8, %xmm10, %xmm8
-; GFNIAVX1-NEXT:    vpaddb %xmm9, %xmm9, %xmm9
-; GFNIAVX1-NEXT:    vpblendvb %xmm9, %xmm8, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm1, %xmm8
-; GFNIAVX1-NEXT:    vpandn %xmm8, %xmm4, %xmm8
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm1, %xmm9
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm9, %xmm4
-; GFNIAVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm10
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm11
+; GFNIAVX1-NEXT:    vpor %xmm10, %xmm11, %xmm10
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm11
+; GFNIAVX1-NEXT:    vpsubb %xmm11, %xmm6, %xmm11
+; GFNIAVX1-NEXT:    vpsllw $5, %xmm11, %xmm11
+; GFNIAVX1-NEXT:    vpblendvb %xmm11, %xmm10, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm2, %xmm10
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm2, %xmm12
+; GFNIAVX1-NEXT:    vpor %xmm10, %xmm12, %xmm10
+; GFNIAVX1-NEXT:    vpaddb %xmm11, %xmm11, %xmm11
+; GFNIAVX1-NEXT:    vpblendvb %xmm11, %xmm10, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm2, %xmm10
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm12
+; GFNIAVX1-NEXT:    vpor %xmm10, %xmm12, %xmm10
+; GFNIAVX1-NEXT:    vpaddb %xmm11, %xmm11, %xmm11
+; GFNIAVX1-NEXT:    vpblendvb %xmm11, %xmm10, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm1, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm1, %xmm5
+; GFNIAVX1-NEXT:    vpor %xmm4, %xmm5, %xmm4
 ; GFNIAVX1-NEXT:    vpsubb %xmm3, %xmm6, %xmm3
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm3, %xmm3
 ; GFNIAVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $6, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpandn %xmm4, %xmm5, %xmm4
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm1, %xmm6
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm6, %xmm5
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm7, %xmm1, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm8, %xmm1, %xmm5
 ; GFNIAVX1-NEXT:    vpor %xmm4, %xmm5, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm3, %xmm3
 ; GFNIAVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm9, %xmm1, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm5
 ; GFNIAVX1-NEXT:    vpor %xmm4, %xmm5, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm3, %xmm3
@@ -1813,48 +1611,40 @@ define <64 x i8> @var_rotr_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_rotr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX2-NEXT:    vpandn %ymm4, %ymm5, %ymm4
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm0, %ymm6
-; GFNIAVX2-NEXT:    vpand %ymm5, %ymm6, %ymm6
-; GFNIAVX2-NEXT:    vpor %ymm4, %ymm6, %ymm4
-; GFNIAVX2-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; GFNIAVX2-NEXT:    vpsubb %ymm2, %ymm6, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm5
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm0, %ymm7
+; GFNIAVX2-NEXT:    vpor %ymm5, %ymm7, %ymm5
+; GFNIAVX2-NEXT:    vpxor %xmm7, %xmm7, %xmm7
+; GFNIAVX2-NEXT:    vpsubb %ymm2, %ymm7, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $6, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX2-NEXT:    vpandn %ymm4, %ymm7, %ymm4
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm0, %ymm8
-; GFNIAVX2-NEXT:    vpand %ymm7, %ymm8, %ymm8
-; GFNIAVX2-NEXT:    vpor %ymm4, %ymm8, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm5, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [4647714815446351872,4647714815446351872,4647714815446351872,4647714815446351872]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm8
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm9, %ymm0, %ymm10
+; GFNIAVX2-NEXT:    vpor %ymm8, %ymm10, %ymm8
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX2-NEXT:    vpand %ymm4, %ymm8, %ymm4
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm9
-; GFNIAVX2-NEXT:    vpor %ymm4, %ymm9, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm8, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm0, %ymm10
+; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm11
+; GFNIAVX2-NEXT:    vpor %ymm10, %ymm11, %ymm10
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpandn %ymm2, %ymm5, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm1, %ymm4
-; GFNIAVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm10, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm1, %ymm4
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; GFNIAVX2-NEXT:    vpsubb %ymm3, %ymm6, %ymm3
+; GFNIAVX2-NEXT:    vpsubb %ymm3, %ymm7, %ymm3
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm3, %ymm3
 ; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpandn %ymm2, %ymm7, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm1, %ymm4
-; GFNIAVX2-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm1, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm9, %ymm1, %ymm4
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
 ; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpand %ymm2, %ymm8, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm1, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
 ; GFNIAVX2-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
@@ -1864,40 +1654,43 @@ define <64 x i8> @var_rotr_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
 ; GFNIAVX512VL-LABEL: var_rotr_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm4
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135]
-; GFNIAVX512VL-NEXT:    vpternlogd $226, %ymm3, %ymm5, %ymm4
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm4, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpsllw $6, %ymm2, %ymm4
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm2, %ymm6
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm7 = [1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567]
-; GFNIAVX512VL-NEXT:    vpternlogd $226, %ymm4, %ymm7, %ymm6
-; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm6, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpsllw $7, %ymm2, %ymm4
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm2, %ymm6
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm8 = [2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143]
-; GFNIAVX512VL-NEXT:    vpternlogd $226, %ymm4, %ymm8, %ymm6
-; GFNIAVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm3, %ymm6, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpternlogd $226, %ymm3, %ymm5, %ymm4
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [16909320,16909320,16909320,16909320]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm2, %ymm4
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm2, %ymm6
+; GFNIAVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
+; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
+; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm6, %ymm6
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm4, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [258,258,258,258]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm7
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm2, %ymm9
+; GFNIAVX512VL-NEXT:    vpor %ymm7, %ymm9, %ymm7
+; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm7, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [1,1,1,1]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm2, %ymm9
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm10, %ymm2, %ymm11
+; GFNIAVX512VL-NEXT:    vpor %ymm9, %ymm11, %ymm9
+; GFNIAVX512VL-NEXT:    vpaddb %ymm6, %ymm6, %ymm6
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm6, %ymm9, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm5
+; GFNIAVX512VL-NEXT:    vpor %ymm3, %ymm5, %ymm3
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $6, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpternlogd $226, %ymm3, %ymm7, %ymm4
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm8, %ymm0, %ymm4
+; GFNIAVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm0, %ymm4
-; GFNIAVX512VL-NEXT:    vpternlogd $226, %ymm3, %ymm8, %ymm4
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm7, %ymm0, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm10, %ymm0, %ymm4
+; GFNIAVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; GFNIAVX512VL-NEXT:    retq
 ;
@@ -2464,85 +2257,31 @@ define <64 x i8> @constant_rotr_v64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @splatconstant_rotl_v64i8(<64 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_rotl_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa %xmm0, %xmm4
-; GFNISSE-NEXT:    psrlw $7, %xmm4
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNISSE-NEXT:    pand %xmm5, %xmm4
-; GFNISSE-NEXT:    paddb %xmm0, %xmm0
-; GFNISSE-NEXT:    por %xmm4, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm4
-; GFNISSE-NEXT:    psrlw $7, %xmm4
-; GFNISSE-NEXT:    pand %xmm5, %xmm4
-; GFNISSE-NEXT:    paddb %xmm1, %xmm1
-; GFNISSE-NEXT:    por %xmm4, %xmm1
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm4
-; GFNISSE-NEXT:    psrlw $7, %xmm4
-; GFNISSE-NEXT:    pand %xmm5, %xmm4
-; GFNISSE-NEXT:    paddb %xmm2, %xmm2
-; GFNISSE-NEXT:    por %xmm4, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm4
-; GFNISSE-NEXT:    psrlw $7, %xmm4
-; GFNISSE-NEXT:    pand %xmm5, %xmm4
-; GFNISSE-NEXT:    paddb %xmm3, %xmm3
-; GFNISSE-NEXT:    por %xmm4, %xmm3
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [9223655728169885760,9223655728169885760]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm3
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_rotl_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm1, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [9223655728169885760,9223655728169885760,9223655728169885760,9223655728169885760]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_rotl_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223655728169885760,9223655728169885760,9223655728169885760,9223655728169885760]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
 ;
-; GFNIAVX512VL-LABEL: splatconstant_rotl_v64i8:
-; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm1
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpsrlw $7, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; GFNIAVX512VL-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
-; GFNIAVX512VL-NEXT:    retq
-;
-; GFNIAVX512BW-LABEL: splatconstant_rotl_v64i8:
-; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vpsrlw $7, %zmm0, %zmm1
-; GFNIAVX512BW-NEXT:    vpaddb %zmm0, %zmm0, %zmm0
-; GFNIAVX512BW-NEXT:    vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
-; GFNIAVX512BW-NEXT:    retq
+; GFNIAVX512-LABEL: splatconstant_rotl_v64i8:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a, <64 x i8> %a, <64 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
   ret <64 x i8> %res
 }
@@ -2551,98 +2290,31 @@ declare <64 x i8> @llvm.fshl.v64i8(<64 x i8>, <64 x i8>, <64 x i8>)
 define <64 x i8> @splatconstant_rotr_v64i8(<64 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_rotr_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    movdqa %xmm0, %xmm5
-; GFNISSE-NEXT:    psrlw $2, %xmm5
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192]
-; GFNISSE-NEXT:    movdqa %xmm4, %xmm6
-; GFNISSE-NEXT:    pandn %xmm5, %xmm6
-; GFNISSE-NEXT:    psllw $6, %xmm0
-; GFNISSE-NEXT:    pand %xmm4, %xmm0
-; GFNISSE-NEXT:    por %xmm6, %xmm0
-; GFNISSE-NEXT:    movdqa %xmm1, %xmm5
-; GFNISSE-NEXT:    psrlw $2, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm4, %xmm6
-; GFNISSE-NEXT:    pandn %xmm5, %xmm6
-; GFNISSE-NEXT:    psllw $6, %xmm1
-; GFNISSE-NEXT:    pand %xmm4, %xmm1
-; GFNISSE-NEXT:    por %xmm6, %xmm1
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psrlw $2, %xmm5
-; GFNISSE-NEXT:    movdqa %xmm4, %xmm6
-; GFNISSE-NEXT:    pandn %xmm5, %xmm6
-; GFNISSE-NEXT:    psllw $6, %xmm2
-; GFNISSE-NEXT:    pand %xmm4, %xmm2
-; GFNISSE-NEXT:    por %xmm6, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm3, %xmm5
-; GFNISSE-NEXT:    psrlw $2, %xmm5
-; GFNISSE-NEXT:    psllw $6, %xmm3
-; GFNISSE-NEXT:    pand %xmm4, %xmm3
-; GFNISSE-NEXT:    pandn %xmm5, %xmm4
-; GFNISSE-NEXT:    por %xmm4, %xmm3
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [290499906672525570,290499906672525570]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm3
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_rotr_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192]
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $6, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $6, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $6, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm1, %xmm3
-; GFNIAVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
-; GFNIAVX1-NEXT:    vpsllw $6, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [290499906672525570,290499906672525570,290499906672525570,290499906672525570]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_rotr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192]
-; GFNIAVX2-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $6, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $2, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; GFNIAVX2-NEXT:    vpsllw $6, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [290499906672525570,290499906672525570,290499906672525570,290499906672525570]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
 ;
-; GFNIAVX512VL-LABEL: splatconstant_rotr_v64i8:
-; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vpsllw $6, %ymm0, %ymm1
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpsllw $6, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; GFNIAVX512VL-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
-; GFNIAVX512VL-NEXT:    retq
-;
-; GFNIAVX512BW-LABEL: splatconstant_rotr_v64i8:
-; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vpsllw $6, %zmm0, %zmm1
-; GFNIAVX512BW-NEXT:    vpsrlw $2, %zmm0, %zmm0
-; GFNIAVX512BW-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
-; GFNIAVX512BW-NEXT:    retq
+; GFNIAVX512-LABEL: splatconstant_rotr_v64i8:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a, <64 x i8> %a, <64 x i8> <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>)
   ret <64 x i8> %res
 }
diff --git a/llvm/test/CodeGen/X86/gfni-shifts.ll b/llvm/test/CodeGen/X86/gfni-shifts.ll
index f79407d08ab0..6232488bea71 100644
--- a/llvm/test/CodeGen/X86/gfni-shifts.ll
+++ b/llvm/test/CodeGen/X86/gfni-shifts.ll
@@ -15,13 +15,11 @@ define <16 x i8> @var_shl_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
 ; GFNISSE-NEXT:    psllw $5, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm3
-; GFNISSE-NEXT:    psllw $4, %xmm3
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
-; GFNISSE-NEXT:    psllw $2, %xmm3
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; GFNISSE-NEXT:    paddb %xmm1, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
@@ -36,11 +34,9 @@ define <16 x i8> @var_shl_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; GFNIAVX1OR2-LABEL: var_shl_v16i8:
 ; GFNIAVX1OR2:       # %bb.0:
 ; GFNIAVX1OR2-NEXT:    vpsllw $5, %xmm1, %xmm1
-; GFNIAVX1OR2-NEXT:    vpsllw $4, %xmm0, %xmm2
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpsllw $2, %xmm0, %xmm2
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; GFNIAVX1OR2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
 ; GFNIAVX1OR2-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
@@ -75,19 +71,16 @@ define <16 x i8> @var_lshr_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
 ; GFNISSE-NEXT:    psllw $5, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm3
-; GFNISSE-NEXT:    psrlw $4, %xmm3
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
-; GFNISSE-NEXT:    psrlw $2, %xmm3
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; GFNISSE-NEXT:    paddb %xmm1, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm3
-; GFNISSE-NEXT:    psrlw $1, %xmm3
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; GFNISSE-NEXT:    paddb %xmm1, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
@@ -97,15 +90,12 @@ define <16 x i8> @var_lshr_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; GFNIAVX1OR2-LABEL: var_lshr_v16i8:
 ; GFNIAVX1OR2:       # %bb.0:
 ; GFNIAVX1OR2-NEXT:    vpsllw $5, %xmm1, %xmm1
-; GFNIAVX1OR2-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpsrlw $2, %xmm0, %xmm2
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; GFNIAVX1OR2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpsrlw $1, %xmm0, %xmm2
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
 ; GFNIAVX1OR2-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; GFNIAVX1OR2-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
 ; GFNIAVX1OR2-NEXT:    retq
@@ -562,20 +552,17 @@ define <16 x i8> @constant_ashr_v16i8(<16 x i8> %a) nounwind {
 define <16 x i8> @splatconstant_shl_v16i8(<16 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_shl_v16i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psllw $3, %xmm0
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1OR2-LABEL: splatconstant_shl_v16i8:
 ; GFNIAVX1OR2:       # %bb.0:
-; GFNIAVX1OR2-NEXT:    vpsllw $3, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_shl_v16i8:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vpsllw $3, %xmm0, %xmm0
-; GFNIAVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
 ; GFNIAVX512-NEXT:    retq
   %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <16 x i8> %shift
@@ -584,20 +571,17 @@ define <16 x i8> @splatconstant_shl_v16i8(<16 x i8> %a) nounwind {
 define <16 x i8> @splatconstant_lshr_v16i8(<16 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_lshr_v16i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1OR2-LABEL: splatconstant_lshr_v16i8:
 ; GFNIAVX1OR2:       # %bb.0:
-; GFNIAVX1OR2-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; GFNIAVX1OR2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; GFNIAVX1OR2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_lshr_v16i8:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; GFNIAVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
 ; GFNIAVX512-NEXT:    retq
   %shift = lshr <16 x i8> %a, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
   ret <16 x i8> %shift
@@ -606,46 +590,18 @@ define <16 x i8> @splatconstant_lshr_v16i8(<16 x i8> %a) nounwind {
 define <16 x i8> @splatconstant_ashr_v16i8(<16 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_ashr_v16i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psrlw $4, %xmm0
-; GFNISSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; GFNISSE-NEXT:    pxor %xmm1, %xmm0
-; GFNISSE-NEXT:    psubb %xmm1, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; GFNISSE-NEXT:    retq
 ;
-; GFNIAVX1-LABEL: splatconstant_ashr_v16i8:
-; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; GFNIAVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    retq
-;
-; GFNIAVX2-LABEL: splatconstant_ashr_v16i8:
-; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; GFNIAVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; GFNIAVX2-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
-; GFNIAVX2-NEXT:    retq
-;
-; GFNIAVX512VL-LABEL: splatconstant_ashr_v16i8:
-; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; GFNIAVX512VL-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0
-; GFNIAVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
-; GFNIAVX512VL-NEXT:    retq
+; GFNIAVX1OR2-LABEL: splatconstant_ashr_v16i8:
+; GFNIAVX1OR2:       # %bb.0:
+; GFNIAVX1OR2-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; GFNIAVX1OR2-NEXT:    retq
 ;
-; GFNIAVX512BW-LABEL: splatconstant_ashr_v16i8:
-; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; GFNIAVX512BW-NEXT:    vpbroadcastb {{.*#+}} xmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; GFNIAVX512BW-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0
-; GFNIAVX512BW-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
-; GFNIAVX512BW-NEXT:    retq
+; GFNIAVX512-LABEL: splatconstant_ashr_v16i8:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; GFNIAVX512-NEXT:    retq
   %shift = ashr <16 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   ret <16 x i8> %shift
 }
@@ -659,34 +615,30 @@ define <32 x i8> @var_shl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm0, %xmm5
-; GFNISSE-NEXT:    psllw $4, %xmm5
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNISSE-NEXT:    pand %xmm6, %xmm5
+; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm5 = [16909320,16909320]
+; GFNISSE-NEXT:    movdqa %xmm0, %xmm6
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm6
 ; GFNISSE-NEXT:    psllw $5, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psllw $2, %xmm5
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNISSE-NEXT:    pand %xmm7, %xmm5
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm6, %xmm2
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [1108169199648,1108169199648]
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm7
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm7
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    paddb %xmm2, %xmm5
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm7, %xmm2
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm7
+; GFNISSE-NEXT:    paddb %xmm2, %xmm7
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm7, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm4
-; GFNISSE-NEXT:    psllw $4, %xmm4
-; GFNISSE-NEXT:    pand %xmm6, %xmm4
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm4
 ; GFNISSE-NEXT:    psllw $5, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm4
-; GFNISSE-NEXT:    psllw $2, %xmm4
-; GFNISSE-NEXT:    pand %xmm7, %xmm4
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm4
 ; GFNISSE-NEXT:    paddb %xmm3, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
@@ -701,26 +653,24 @@ define <32 x i8> @var_shl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNIAVX1-LABEL: var_shl_v32i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [16909320,16909320]
+; GFNIAVX1-NEXT:    # xmm3 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm2, %xmm4
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm6
 ; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm3
+; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm6
 ; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm0, %xmm3
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm3
@@ -731,12 +681,12 @@ define <32 x i8> @var_shl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_shl_v32i8:
 ; GFNIAVX2:       # %bb.0:
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
@@ -747,11 +697,9 @@ define <32 x i8> @var_shl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNIAVX512VL-LABEL: var_shl_v32i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
@@ -775,42 +723,36 @@ define <32 x i8> @var_lshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm0, %xmm5
-; GFNISSE-NEXT:    psrlw $4, %xmm5
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNISSE-NEXT:    pand %xmm6, %xmm5
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [1161999622361579520,1161999622361579520]
+; GFNISSE-NEXT:    movdqa %xmm0, %xmm6
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm6
 ; GFNISSE-NEXT:    psllw $5, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psrlw $2, %xmm5
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNISSE-NEXT:    pand %xmm7, %xmm5
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm6, %xmm2
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm6 = [290499906672525312,290499906672525312]
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm7
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm7
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
-; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psrlw $1, %xmm5
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm8 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNISSE-NEXT:    pand %xmm8, %xmm5
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm7, %xmm2
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm7 = [145249953336295424,145249953336295424]
+; GFNISSE-NEXT:    movdqa %xmm2, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm4, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm0
-; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
+; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm4
-; GFNISSE-NEXT:    psrlw $4, %xmm4
-; GFNISSE-NEXT:    pand %xmm6, %xmm4
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm5, %xmm4
 ; GFNISSE-NEXT:    psllw $5, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm4
-; GFNISSE-NEXT:    psrlw $2, %xmm4
-; GFNISSE-NEXT:    pand %xmm7, %xmm4
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm6, %xmm4
 ; GFNISSE-NEXT:    paddb %xmm3, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm4
-; GFNISSE-NEXT:    psrlw $1, %xmm4
-; GFNISSE-NEXT:    pand %xmm8, %xmm4
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm7, %xmm4
 ; GFNISSE-NEXT:    paddb %xmm3, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm4, %xmm1
@@ -820,32 +762,29 @@ define <32 x i8> @var_lshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNIAVX1-LABEL: var_lshr_v32i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    # xmm3 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm2, %xmm4
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [290499906672525312,290499906672525312]
+; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm6
 ; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm2, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [145249953336295424,145249953336295424]
+; GFNIAVX1-NEXT:    # xmm6 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm2, %xmm7
 ; GFNIAVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpblendvb %xmm5, %xmm7, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm3, %xmm0, %xmm3
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm0, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm0, %xmm3
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
@@ -853,16 +792,16 @@ define <32 x i8> @var_lshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_lshr_v32i8:
 ; GFNIAVX2:       # %bb.0:
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
@@ -870,15 +809,12 @@ define <32 x i8> @var_lshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; GFNIAVX512VL-LABEL: var_lshr_v32i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    retq
@@ -1539,34 +1475,25 @@ define <32 x i8> @constant_ashr_v32i8(<32 x i8> %a) nounwind {
 define <32 x i8> @splatconstant_shl_v32i8(<32 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_shl_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psllw $6, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192]
-; GFNISSE-NEXT:    pand %xmm2, %xmm0
-; GFNISSE-NEXT:    psllw $6, %xmm1
-; GFNISSE-NEXT:    pand %xmm2, %xmm1
+; GFNISSE-NEXT:    pmovsxwq {{.*#+}} xmm2 = [258,258]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_shl_v32i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; GFNIAVX1-NEXT:    vpsllw $6, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [192,192,192,192,192,192,192,192,192,192,192,192,192,192,192,192]
-; GFNIAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsllw $6, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_shl_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsllw $6, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [258,258,258,258]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_shl_v32i8:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vpsllw $6, %ymm0, %ymm0
-; GFNIAVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
 ; GFNIAVX512-NEXT:    retq
   %shift = shl <32 x i8> %a, <i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6, i8 6>
   ret <32 x i8> %shift
@@ -1575,34 +1502,25 @@ define <32 x i8> @splatconstant_shl_v32i8(<32 x i8> %a) nounwind {
 define <32 x i8> @splatconstant_lshr_v32i8(<32 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_lshr_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psrlw $1, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNISSE-NEXT:    pand %xmm2, %xmm0
-; GFNISSE-NEXT:    psrlw $1, %xmm1
-; GFNISSE-NEXT:    pand %xmm2, %xmm1
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [145249953336295424,145249953336295424]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_lshr_v32i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_lshr_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $1, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
 ; GFNIAVX512-LABEL: splatconstant_lshr_v32i8:
 ; GFNIAVX512:       # %bb.0:
-; GFNIAVX512-NEXT:    vpsrlw $1, %ymm0, %ymm0
-; GFNIAVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
 ; GFNIAVX512-NEXT:    retq
   %shift = lshr <32 x i8> %a, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
   ret <32 x i8> %shift
@@ -1611,58 +1529,26 @@ define <32 x i8> @splatconstant_lshr_v32i8(<32 x i8> %a) nounwind {
 define <32 x i8> @splatconstant_ashr_v32i8(<32 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_ashr_v32i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psrlw $2, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNISSE-NEXT:    pand %xmm2, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; GFNISSE-NEXT:    pxor %xmm3, %xmm0
-; GFNISSE-NEXT:    psubb %xmm3, %xmm0
-; GFNISSE-NEXT:    psrlw $2, %xmm1
-; GFNISSE-NEXT:    pand %xmm2, %xmm1
-; GFNISSE-NEXT:    pxor %xmm3, %xmm1
-; GFNISSE-NEXT:    psubb %xmm3, %xmm1
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm2 = [290499906672558208,290499906672558208]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm2, %xmm1
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_ashr_v32i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNIAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; GFNIAVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_ashr_v32i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; GFNIAVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [290499906672558208,290499906672558208,290499906672558208,290499906672558208]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm1, %ymm0, %ymm0
 ; GFNIAVX2-NEXT:    retq
 ;
-; GFNIAVX512VL-LABEL: splatconstant_ashr_v32i8:
-; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; GFNIAVX512VL-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
-; GFNIAVX512VL-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    retq
-;
-; GFNIAVX512BW-LABEL: splatconstant_ashr_v32i8:
-; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vpsrlw $2, %ymm0, %ymm0
-; GFNIAVX512BW-NEXT:    vpbroadcastb {{.*#+}} ymm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; GFNIAVX512BW-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
-; GFNIAVX512BW-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
-; GFNIAVX512BW-NEXT:    retq
+; GFNIAVX512-LABEL: splatconstant_ashr_v32i8:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
+; GFNIAVX512-NEXT:    retq
   %shift = ashr <32 x i8> %a, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
   ret <32 x i8> %shift
 }
@@ -1676,17 +1562,15 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm4
+; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm9 = [16909320,16909320]
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm10
-; GFNISSE-NEXT:    psllw $4, %xmm10
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNISSE-NEXT:    pand %xmm9, %xmm10
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm10
 ; GFNISSE-NEXT:    psllw $5, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm10, %xmm4
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [1108169199648,1108169199648]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm11
-; GFNISSE-NEXT:    psllw $2, %xmm11
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNISSE-NEXT:    pand %xmm10, %xmm11
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm11
 ; GFNISSE-NEXT:    paddb %xmm8, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm11, %xmm4
@@ -1696,14 +1580,12 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm11, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm8
-; GFNISSE-NEXT:    psllw $4, %xmm8
-; GFNISSE-NEXT:    pand %xmm9, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm8
 ; GFNISSE-NEXT:    psllw $5, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm8
-; GFNISSE-NEXT:    psllw $2, %xmm8
-; GFNISSE-NEXT:    pand %xmm10, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm5, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
@@ -1713,14 +1595,12 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psllw $4, %xmm5
-; GFNISSE-NEXT:    pand %xmm9, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm5
 ; GFNISSE-NEXT:    psllw $5, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psllw $2, %xmm5
-; GFNISSE-NEXT:    pand %xmm10, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm6, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
@@ -1730,14 +1610,12 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm5
-; GFNISSE-NEXT:    psllw $4, %xmm5
-; GFNISSE-NEXT:    pand %xmm9, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm5
 ; GFNISSE-NEXT:    psllw $5, %xmm7
 ; GFNISSE-NEXT:    movdqa %xmm7, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm5
-; GFNISSE-NEXT:    psllw $2, %xmm5
-; GFNISSE-NEXT:    pand %xmm10, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm7, %xmm7
 ; GFNISSE-NEXT:    movdqa %xmm7, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm3
@@ -1752,26 +1630,24 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNIAVX1-LABEL: var_shl_v64i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm5, %xmm6
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [16909320,16909320]
+; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm5, %xmm6
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm6, %xmm5, %xmm6
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm6, %xmm8
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm8, %xmm8
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1108169199648,1108169199648]
+; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm6, %xmm8
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm8, %xmm6, %xmm6
 ; GFNIAVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm8
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm8, %xmm6, %xmm6
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm0, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm7, %xmm7
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm7
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm7, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm0, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm7, %xmm7
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm0, %xmm7
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm7, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm7
@@ -1779,24 +1655,20 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm7, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm2, %xmm6
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm6
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm7
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm6, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm2, %xmm6
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm6
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm6, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm6
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm6, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $4, %xmm1, %xmm6
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm6, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm1, %xmm4
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm3, %xmm3
 ; GFNIAVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsllw $2, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm1, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm3, %xmm3
 ; GFNIAVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm4
@@ -1807,25 +1679,21 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_shl_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [16909320,16909320,16909320,16909320]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm5
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX2-NEXT:    vpand %ymm6, %ymm4, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm5, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm6
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm6, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm6
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsllw $4, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm6, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm3, %ymm3
 ; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsllw $2, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm1, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
 ; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm2
@@ -1836,26 +1704,22 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNIAVX512VL-LABEL: var_shl_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [16909320,16909320,16909320,16909320]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm2, %ymm4
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [1108169199648,1108169199648,1108169199648,1108169199648]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm6
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm3
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm6, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm6
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm6, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
@@ -1866,16 +1730,12 @@ define <64 x i8> @var_shl_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ;
 ; GFNIAVX512BW-LABEL: var_shl_v64i8:
 ; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm2
-; GFNIAVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
 ; GFNIAVX512BW-NEXT:    vpsllw $5, %zmm1, %zmm1
 ; GFNIAVX512BW-NEXT:    vpmovb2m %zmm1, %k1
-; GFNIAVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm0 {%k1}
-; GFNIAVX512BW-NEXT:    vpsllw $2, %zmm0, %zmm2
-; GFNIAVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 {%k1}
 ; GFNIAVX512BW-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
 ; GFNIAVX512BW-NEXT:    vpmovb2m %zmm1, %k1
-; GFNIAVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm0 {%k1}
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 {%k1}
 ; GFNIAVX512BW-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
 ; GFNIAVX512BW-NEXT:    vpmovb2m %zmm1, %k1
 ; GFNIAVX512BW-NEXT:    vpaddb %zmm0, %zmm0, %zmm0 {%k1}
@@ -1889,78 +1749,66 @@ define <64 x i8> @var_lshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNISSE:       # %bb.0:
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm4
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [1161999622361579520,1161999622361579520]
 ; GFNISSE-NEXT:    movdqa %xmm0, %xmm10
-; GFNISSE-NEXT:    psrlw $4, %xmm10
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm9 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNISSE-NEXT:    pand %xmm9, %xmm10
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm10
 ; GFNISSE-NEXT:    psllw $5, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm10, %xmm4
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [290499906672525312,290499906672525312]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm11
-; GFNISSE-NEXT:    psrlw $2, %xmm11
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNISSE-NEXT:    pand %xmm10, %xmm11
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm11
 ; GFNISSE-NEXT:    paddb %xmm8, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm11, %xmm4
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [145249953336295424,145249953336295424]
 ; GFNISSE-NEXT:    movdqa %xmm4, %xmm12
-; GFNISSE-NEXT:    psrlw $1, %xmm12
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm11 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNISSE-NEXT:    pand %xmm11, %xmm12
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm12
 ; GFNISSE-NEXT:    paddb %xmm8, %xmm8
 ; GFNISSE-NEXT:    movdqa %xmm8, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm12, %xmm4
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm8
-; GFNISSE-NEXT:    psrlw $4, %xmm8
-; GFNISSE-NEXT:    pand %xmm9, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm8
 ; GFNISSE-NEXT:    psllw $5, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm8
-; GFNISSE-NEXT:    psrlw $2, %xmm8
-; GFNISSE-NEXT:    pand %xmm10, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm5, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm1, %xmm8
-; GFNISSE-NEXT:    psrlw $1, %xmm8
-; GFNISSE-NEXT:    pand %xmm11, %xmm8
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm8
 ; GFNISSE-NEXT:    paddb %xmm5, %xmm5
 ; GFNISSE-NEXT:    movdqa %xmm5, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm8, %xmm1
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psrlw $4, %xmm5
-; GFNISSE-NEXT:    pand %xmm9, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm5
 ; GFNISSE-NEXT:    psllw $5, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psrlw $2, %xmm5
-; GFNISSE-NEXT:    pand %xmm10, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm6, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm2, %xmm5
-; GFNISSE-NEXT:    psrlw $1, %xmm5
-; GFNISSE-NEXT:    pand %xmm11, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm6, %xmm6
 ; GFNISSE-NEXT:    movdqa %xmm6, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm2
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm5
-; GFNISSE-NEXT:    psrlw $4, %xmm5
-; GFNISSE-NEXT:    pand %xmm9, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm9, %xmm5
 ; GFNISSE-NEXT:    psllw $5, %xmm7
 ; GFNISSE-NEXT:    movdqa %xmm7, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm5
-; GFNISSE-NEXT:    psrlw $2, %xmm5
-; GFNISSE-NEXT:    pand %xmm10, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm10, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm7, %xmm7
 ; GFNISSE-NEXT:    movdqa %xmm7, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm3
 ; GFNISSE-NEXT:    movdqa %xmm3, %xmm5
-; GFNISSE-NEXT:    psrlw $1, %xmm5
-; GFNISSE-NEXT:    pand %xmm11, %xmm5
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm11, %xmm5
 ; GFNISSE-NEXT:    paddb %xmm7, %xmm7
 ; GFNISSE-NEXT:    movdqa %xmm7, %xmm0
 ; GFNISSE-NEXT:    pblendvb %xmm0, %xmm5, %xmm3
@@ -1970,59 +1818,50 @@ define <64 x i8> @var_lshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNIAVX1-LABEL: var_lshr_v64i8:
 ; GFNIAVX1:       # %bb.0:
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm5, %xmm6
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm6, %xmm6
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [1161999622361579520,1161999622361579520]
+; GFNIAVX1-NEXT:    # xmm4 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm5, %xmm6
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm6, %xmm5, %xmm6
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm6, %xmm8
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm8, %xmm8
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [290499906672525312,290499906672525312]
+; GFNIAVX1-NEXT:    # xmm5 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm6, %xmm8
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm8, %xmm6, %xmm8
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm8, %xmm9
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm9, %xmm9
+; GFNIAVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [145249953336295424,145249953336295424]
+; GFNIAVX1-NEXT:    # xmm6 = mem[0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm8, %xmm9
 ; GFNIAVX1-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
 ; GFNIAVX1-NEXT:    vpblendvb %xmm7, %xmm9, %xmm8, %xmm7
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm0, %xmm8
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm8, %xmm8
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm0, %xmm8
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm8, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm0, %xmm8
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm8, %xmm8
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm0, %xmm8
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm8, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm0, %xmm8
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm8, %xmm8
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm0, %xmm8
 ; GFNIAVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
 ; GFNIAVX1-NEXT:    vpblendvb %xmm2, %xmm8, %xmm0, %xmm0
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm7, %xmm7
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm2, %xmm7
 ; GFNIAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm8
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm8, %xmm8
 ; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm7, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm7, %xmm7
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm2, %xmm7
 ; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
 ; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm7, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm2, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm7, %xmm7
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm2, %xmm7
 ; GFNIAVX1-NEXT:    vpaddb %xmm8, %xmm8, %xmm8
 ; GFNIAVX1-NEXT:    vpblendvb %xmm8, %xmm7, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $4, %xmm1, %xmm7
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm7, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm4, %xmm1, %xmm4
 ; GFNIAVX1-NEXT:    vpsllw $5, %xmm3, %xmm3
 ; GFNIAVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $2, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm5, %xmm1, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm3, %xmm3
 ; GFNIAVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm1, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %xmm6, %xmm1, %xmm4
 ; GFNIAVX1-NEXT:    vpaddb %xmm3, %xmm3, %xmm3
 ; GFNIAVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm1, %xmm1
 ; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
@@ -2030,31 +1869,25 @@ define <64 x i8> @var_lshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ;
 ; GFNIAVX2-LABEL: var_lshr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNIAVX2-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm5
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $2, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNIAVX2-NEXT:    vpand %ymm6, %ymm4, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm5, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm0, %ymm6
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $1, %ymm0, %ymm4
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX2-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm6, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm0, %ymm7
 ; GFNIAVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vpblendvb %ymm2, %ymm7, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm1, %ymm2
 ; GFNIAVX2-NEXT:    vpsllw $5, %ymm3, %ymm3
 ; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsrlw $2, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm5, %ymm1, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
 ; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsrlw $1, %ymm1, %ymm2
-; GFNIAVX2-NEXT:    vpand %ymm7, %ymm2, %ymm2
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm1, %ymm2
 ; GFNIAVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
 ; GFNIAVX2-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
@@ -2062,32 +1895,26 @@ define <64 x i8> @var_lshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; GFNIAVX512VL-LABEL: var_lshr_v64i8:
 ; GFNIAVX512VL:       # %bb.0:
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1161999622361579520,1161999622361579520,1161999622361579520,1161999622361579520]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm2, %ymm4
 ; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm5
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm4, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [290499906672525312,290499906672525312,290499906672525312,290499906672525312]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm2, %ymm6
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm2, %ymm3
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX512VL-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm6, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [145249953336295424,145249953336295424,145249953336295424,145249953336295424]
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm2, %ymm7
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm5, %ymm5, %ymm5
-; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm2
-; GFNIAVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vpblendvb %ymm5, %ymm7, %ymm2, %ymm2
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm3, %ymm0, %ymm3
 ; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsrlw $2, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm4, %ymm0, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm0, %ymm3
-; GFNIAVX512VL-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; GFNIAVX512VL-NEXT:    vgf2p8affineqb $0, %ymm6, %ymm0, %ymm3
 ; GFNIAVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
 ; GFNIAVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
@@ -2095,21 +1922,15 @@ define <64 x i8> @var_lshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ;
 ; GFNIAVX512BW-LABEL: var_lshr_v64i8:
 ; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm2
-; GFNIAVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
 ; GFNIAVX512BW-NEXT:    vpsllw $5, %zmm1, %zmm1
 ; GFNIAVX512BW-NEXT:    vpmovb2m %zmm1, %k1
-; GFNIAVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm0 {%k1}
-; GFNIAVX512BW-NEXT:    vpsrlw $2, %zmm0, %zmm2
-; GFNIAVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 {%k1}
 ; GFNIAVX512BW-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
 ; GFNIAVX512BW-NEXT:    vpmovb2m %zmm1, %k1
-; GFNIAVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm0 {%k1}
-; GFNIAVX512BW-NEXT:    vpsrlw $1, %zmm0, %zmm2
-; GFNIAVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 {%k1}
 ; GFNIAVX512BW-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
 ; GFNIAVX512BW-NEXT:    vpmovb2m %zmm1, %k1
-; GFNIAVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm0 {%k1}
+; GFNIAVX512BW-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0 {%k1}
 ; GFNIAVX512BW-NEXT:    retq
   %shift = lshr <64 x i8> %a, %b
   ret <64 x i8> %shift
@@ -3214,57 +3035,31 @@ define <64 x i8> @constant_ashr_v64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @splatconstant_shl_v64i8(<64 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_shl_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psllw $5, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224]
-; GFNISSE-NEXT:    pand %xmm4, %xmm0
-; GFNISSE-NEXT:    psllw $5, %xmm1
-; GFNISSE-NEXT:    pand %xmm4, %xmm1
-; GFNISSE-NEXT:    psllw $5, %xmm2
-; GFNISSE-NEXT:    pand %xmm4, %xmm2
-; GFNISSE-NEXT:    psllw $5, %xmm3
-; GFNISSE-NEXT:    pand %xmm4, %xmm3
+; GFNISSE-NEXT:    pmovsxdq {{.*#+}} xmm4 = [66052,66052]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm3
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_shl_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224]
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $5, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [4,2,1,0,0,0,0,0,4,2,1,0,0,0,0,0,4,2,1,0,0,0,0,0,4,2,1,0,0,0,0,0]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_shl_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsllw $5, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm2 = [224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224]
-; GFNIAVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [66052,66052,66052,66052]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
 ;
-; GFNIAVX512VL-LABEL: splatconstant_shl_v64i8:
-; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm0, %ymm1
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsllw $5, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
-; GFNIAVX512VL-NEXT:    retq
-;
-; GFNIAVX512BW-LABEL: splatconstant_shl_v64i8:
-; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vpsllw $5, %zmm0, %zmm0
-; GFNIAVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
-; GFNIAVX512BW-NEXT:    retq
+; GFNIAVX512-LABEL: splatconstant_shl_v64i8:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512-NEXT:    retq
   %shift = shl <64 x i8> %a, <i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5>
   ret <64 x i8> %shift
 }
@@ -3272,57 +3067,31 @@ define <64 x i8> @splatconstant_shl_v64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @splatconstant_lshr_v64i8(<64 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_lshr_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psrlw $7, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNISSE-NEXT:    pand %xmm4, %xmm0
-; GFNISSE-NEXT:    psrlw $7, %xmm1
-; GFNISSE-NEXT:    pand %xmm4, %xmm1
-; GFNISSE-NEXT:    psrlw $7, %xmm2
-; GFNISSE-NEXT:    pand %xmm4, %xmm2
-; GFNISSE-NEXT:    psrlw $7, %xmm3
-; GFNISSE-NEXT:    pand %xmm4, %xmm3
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm3
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_lshr_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128,0,0,0,0,0,0,0,128]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_lshr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; GFNIAVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $7, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
 ;
-; GFNIAVX512VL-LABEL: splatconstant_lshr_v64i8:
-; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm1
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; GFNIAVX512VL-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
-; GFNIAVX512VL-NEXT:    retq
-;
-; GFNIAVX512BW-LABEL: splatconstant_lshr_v64i8:
-; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vpsrlw $7, %zmm0, %zmm0
-; GFNIAVX512BW-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
-; GFNIAVX512BW-NEXT:    retq
+; GFNIAVX512-LABEL: splatconstant_lshr_v64i8:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512-NEXT:    retq
   %shift = lshr <64 x i8> %a, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
   ret <64 x i8> %shift
 }
@@ -3330,87 +3099,31 @@ define <64 x i8> @splatconstant_lshr_v64i8(<64 x i8> %a) nounwind {
 define <64 x i8> @splatconstant_ashr_v64i8(<64 x i8> %a) nounwind {
 ; GFNISSE-LABEL: splatconstant_ashr_v64i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    psrlw $1, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNISSE-NEXT:    pand %xmm4, %xmm0
-; GFNISSE-NEXT:    movdqa {{.*#+}} xmm5 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; GFNISSE-NEXT:    pxor %xmm5, %xmm0
-; GFNISSE-NEXT:    psubb %xmm5, %xmm0
-; GFNISSE-NEXT:    psrlw $1, %xmm1
-; GFNISSE-NEXT:    pand %xmm4, %xmm1
-; GFNISSE-NEXT:    pxor %xmm5, %xmm1
-; GFNISSE-NEXT:    psubb %xmm5, %xmm1
-; GFNISSE-NEXT:    psrlw $1, %xmm2
-; GFNISSE-NEXT:    pand %xmm4, %xmm2
-; GFNISSE-NEXT:    pxor %xmm5, %xmm2
-; GFNISSE-NEXT:    psubb %xmm5, %xmm2
-; GFNISSE-NEXT:    psrlw $1, %xmm3
-; GFNISSE-NEXT:    pand %xmm4, %xmm3
-; GFNISSE-NEXT:    pxor %xmm5, %xmm3
-; GFNISSE-NEXT:    psubb %xmm5, %xmm3
+; GFNISSE-NEXT:    movdqa {{.*#+}} xmm4 = [145249953336295552,145249953336295552]
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm1
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm2
+; GFNISSE-NEXT:    gf2p8affineqb $0, %xmm4, %xmm3
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX1-LABEL: splatconstant_ashr_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; GFNIAVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsubb %xmm4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsubb %xmm4, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsubb %xmm4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsrlw $1, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsubb %xmm4, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [128,128,64,32,16,8,4,2,128,128,64,32,16,8,4,2,128,128,64,32,16,8,4,2,128,128,64,32,16,8,4,2]
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: splatconstant_ashr_v64i8:
 ; GFNIAVX2:       # %bb.0:
-; GFNIAVX2-NEXT:    vpsrlw $1, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; GFNIAVX2-NEXT:    vpxor %ymm3, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsubb %ymm3, %ymm0, %ymm0
-; GFNIAVX2-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm1
-; GFNIAVX2-NEXT:    vpsubb %ymm3, %ymm1, %ymm1
+; GFNIAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [145249953336295552,145249953336295552,145249953336295552,145249953336295552]
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm0, %ymm0
+; GFNIAVX2-NEXT:    vgf2p8affineqb $0, %ymm2, %ymm1, %ymm1
 ; GFNIAVX2-NEXT:    retq
 ;
-; GFNIAVX512VL-LABEL: splatconstant_ashr_v64i8:
-; GFNIAVX512VL:       # %bb.0:
-; GFNIAVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; GFNIAVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; GFNIAVX512VL-NEXT:    vpternlogq $108, %ymm2, %ymm3, %ymm1
-; GFNIAVX512VL-NEXT:    vpsubb %ymm3, %ymm1, %ymm1
-; GFNIAVX512VL-NEXT:    vpsrlw $1, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vpternlogq $108, %ymm2, %ymm3, %ymm0
-; GFNIAVX512VL-NEXT:    vpsubb %ymm3, %ymm0, %ymm0
-; GFNIAVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; GFNIAVX512VL-NEXT:    retq
-;
-; GFNIAVX512BW-LABEL: splatconstant_ashr_v64i8:
-; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vpsrlw $1, %zmm0, %zmm0
-; GFNIAVX512BW-NEXT:    vpbroadcastb {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
-; GFNIAVX512BW-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0
-; GFNIAVX512BW-NEXT:    vpsubb %zmm1, %zmm0, %zmm0
-; GFNIAVX512BW-NEXT:    retq
+; GFNIAVX512-LABEL: splatconstant_ashr_v64i8:
+; GFNIAVX512:       # %bb.0:
+; GFNIAVX512-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512-NEXT:    retq
   %shift = ashr <64 x i8> %a, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
   ret <64 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index a953c505cd8e..f3a8ca4de997 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -5,10 +5,10 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cascadelake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cooperlake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=cannonlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-client | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-server | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=cannonlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-VBMI1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-client | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-GFNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-server | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-GFNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-GFNI
 
 ; This file primarily contains tests for specific places in X86ISelLowering.cpp that needed be made aware of the legalizer not allowing 512-bit vectors due to prefer-256-bit even though AVX512 is enabled.
 
@@ -2006,12 +2006,31 @@ define <32 x i8> @constant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector
 }
 
 define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
-; CHECK-LABEL: splatconstant_rotate_v32i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsllw $4, %ymm0, %ymm1
-; CHECK-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; CHECK-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
-; CHECK-NEXT:    retq
+; CHECK-SKX-LABEL: splatconstant_rotate_v32i8:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vpsllw $4, %ymm0, %ymm1
+; CHECK-SKX-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; CHECK-SKX-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: splatconstant_rotate_v32i8:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vpsllw $4, %ymm0, %ymm1
+; CHECK-AVX512-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; CHECK-AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-VBMI1-LABEL: splatconstant_rotate_v32i8:
+; CHECK-VBMI1:       # %bb.0:
+; CHECK-VBMI1-NEXT:    vpsllw $4, %ymm0, %ymm1
+; CHECK-VBMI1-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; CHECK-VBMI1-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
+; CHECK-VBMI1-NEXT:    retq
+;
+; CHECK-GFNI-LABEL: splatconstant_rotate_v32i8:
+; CHECK-GFNI:       # %bb.0:
+; CHECK-GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
+; CHECK-GFNI-NEXT:    retq
   %shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %or = or <32 x i8> %shl, %lshr
@@ -2019,13 +2038,35 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-v
 }
 
 define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
-; CHECK-LABEL: splatconstant_rotate_mask_v32i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsllw $4, %ymm0, %ymm1
-; CHECK-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; CHECK-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
-; CHECK-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; CHECK-SKX-LABEL: splatconstant_rotate_mask_v32i8:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vpsllw $4, %ymm0, %ymm1
+; CHECK-SKX-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; CHECK-SKX-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
+; CHECK-SKX-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: splatconstant_rotate_mask_v32i8:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vpsllw $4, %ymm0, %ymm1
+; CHECK-AVX512-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; CHECK-AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
+; CHECK-AVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-VBMI1-LABEL: splatconstant_rotate_mask_v32i8:
+; CHECK-VBMI1:       # %bb.0:
+; CHECK-VBMI1-NEXT:    vpsllw $4, %ymm0, %ymm1
+; CHECK-VBMI1-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; CHECK-VBMI1-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0
+; CHECK-VBMI1-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; CHECK-VBMI1-NEXT:    retq
+;
+; CHECK-GFNI-LABEL: splatconstant_rotate_mask_v32i8:
+; CHECK-GFNI:       # %bb.0:
+; CHECK-GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
+; CHECK-GFNI-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; CHECK-GFNI-NEXT:    retq
   %shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %rmask = and <32 x i8> %lshr, <i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55>
-- 
GitLab


From e232659028365b51feb001565884b3b8e62cc2a9 Mon Sep 17 00:00:00 2001
From: Kristof Beyls <kristof.beyls@arm.com>
Date: Tue, 7 May 2024 12:01:14 +0200
Subject: [PATCH 0032/1206] [NFC][BOLT] Call EnsureAllocatorExists instead of
 copy pasting code

---
 bolt/lib/Core/ParallelUtilities.cpp | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/bolt/lib/Core/ParallelUtilities.cpp b/bolt/lib/Core/ParallelUtilities.cpp
index e1649565cc6e..a24c37c06f1a 100644
--- a/bolt/lib/Core/ParallelUtilities.cpp
+++ b/bolt/lib/Core/ParallelUtilities.cpp
@@ -231,12 +231,7 @@ void runOnEachFunctionWithUniqueAllocId(
     }
   }
 
-  if (!BC.MIB->checkAllocatorExists(AllocId)) {
-    MCPlusBuilder::AllocatorIdTy Id =
-        BC.MIB->initializeNewAnnotationAllocator();
-    (void)Id;
-    assert(AllocId == Id && "unexpected allocator id created");
-  }
+  EnsureAllocatorExists(AllocId);
 
   Pool.async(runBlock, BlockBegin, BC.getBinaryFunctions().end(), AllocId);
   Lock.unlock();
-- 
GitLab


From 235cea720c0fa6dcf0bf5aff15001de88b6042f9 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Tue, 7 May 2024 11:23:55 +0100
Subject: [PATCH 0033/1206] [NFC][LLVM] Refactor rounding mode detection of
 constrained fp intrinsic IDs (#90854)

I've refactored the code to genericise the implementation to better
allow for target specific constrained fp intrinsics.
---
 llvm/include/llvm/IR/IntrinsicInst.h          |  3 +-
 llvm/include/llvm/IR/Intrinsics.h             |  4 ++
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  7 +---
 .../SelectionDAG/SelectionDAGBuilder.cpp      | 12 +-----
 llvm/lib/IR/Function.cpp                      | 12 ++++++
 llvm/lib/IR/IRBuilder.cpp                     | 25 ++----------
 llvm/lib/IR/IntrinsicInst.cpp                 | 40 ++++++-------------
 llvm/lib/IR/Verifier.cpp                      | 29 ++++++--------
 llvm/lib/Transforms/Utils/CloneFunction.cpp   | 14 +------
 9 files changed, 51 insertions(+), 95 deletions(-)

diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index 2e99c9e2ee3e..fcd3a1025ac1 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -707,8 +707,7 @@ public:
 /// This is the common base class for constrained floating point intrinsics.
 class ConstrainedFPIntrinsic : public IntrinsicInst {
 public:
-  bool isUnaryOp() const;
-  bool isTernaryOp() const;
+  unsigned getNonMetadataArgCount() const;
   std::optional<RoundingMode> getRoundingMode() const;
   std::optional<fp::ExceptionBehavior> getExceptionBehavior() const;
   bool isDefaultFPEnvironment() const;
diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index 340c1c326d06..f79df522dc80 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -109,6 +109,10 @@ namespace Intrinsic {
   /// Floating-Point Intrinsics".
   bool isConstrainedFPIntrinsic(ID QID);
 
+  /// Returns true if the intrinsic ID is for one of the "Constrained
+  /// Floating-Point Intrinsics" that take rounding mode metadata.
+  bool hasConstrainedFPRoundingModeOperand(ID QID);
+
   /// This is a type descriptor which explains the type requirements of an
   /// intrinsic. This is returned by getIntrinsicInfoTableEntries.
   struct IITDescriptor {
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index e26c6ca3d616..77ee5e645288 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2053,11 +2053,8 @@ bool IRTranslator::translateConstrainedFPIntrinsic(
     Flags |= MachineInstr::NoFPExcept;
 
   SmallVector<llvm::SrcOp, 4> VRegs;
-  VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(0)));
-  if (!FPI.isUnaryOp())
-    VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(1)));
-  if (FPI.isTernaryOp())
-    VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(2)));
+  for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
+    VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(I)));
 
   MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);
   return true;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index cfd82a342433..f47aea29625f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7962,16 +7962,8 @@ void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
   SDValue Chain = DAG.getRoot();
   SmallVector<SDValue, 4> Opers;
   Opers.push_back(Chain);
-  if (FPI.isUnaryOp()) {
-    Opers.push_back(getValue(FPI.getArgOperand(0)));
-  } else if (FPI.isTernaryOp()) {
-    Opers.push_back(getValue(FPI.getArgOperand(0)));
-    Opers.push_back(getValue(FPI.getArgOperand(1)));
-    Opers.push_back(getValue(FPI.getArgOperand(2)));
-  } else {
-    Opers.push_back(getValue(FPI.getArgOperand(0)));
-    Opers.push_back(getValue(FPI.getArgOperand(1)));
-  }
+  for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
+    Opers.push_back(getValue(FPI.getArgOperand(I)));
 
   auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
     assert(Result.getNode()->getNumValues() == 2);
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index e42248da1742..7f1e832f8597 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -1493,7 +1493,19 @@ bool Intrinsic::isConstrainedFPIntrinsic(ID QID) {
 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
   case Intrinsic::INTRINSIC:
 #include "llvm/IR/ConstrainedOps.def"
+#undef INSTRUCTION
     return true;
+  default:
+    return false;
+  }
+}
+
+bool Intrinsic::hasConstrainedFPRoundingModeOperand(Intrinsic::ID QID) {
+  switch (QID) {
+#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
+  case Intrinsic::INTRINSIC:                                                   \
+    return ROUND_MODE == 1;
+#include "llvm/IR/ConstrainedOps.def"
 #undef INSTRUCTION
   default:
     return false;
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 9ec5a7deeec6..c6f20af0f1df 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -1029,17 +1029,7 @@ CallInst *IRBuilderBase::CreateConstrainedFPCast(
     UseFMF = FMFSource->getFastMathFlags();
 
   CallInst *C;
-  bool HasRoundingMD = false;
-  switch (ID) {
-  default:
-    break;
-#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
-  case Intrinsic::INTRINSIC:                                \
-    HasRoundingMD = ROUND_MODE;                             \
-    break;
-#include "llvm/IR/ConstrainedOps.def"
-  }
-  if (HasRoundingMD) {
+  if (Intrinsic::hasConstrainedFPRoundingModeOperand(ID)) {
     Value *RoundingV = getConstrainedFPRounding(Rounding);
     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
                         nullptr, Name);
@@ -1088,17 +1078,8 @@ CallInst *IRBuilderBase::CreateConstrainedFPCall(
   llvm::SmallVector<Value *, 6> UseArgs;
 
   append_range(UseArgs, Args);
-  bool HasRoundingMD = false;
-  switch (Callee->getIntrinsicID()) {
-  default:
-    break;
-#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
-  case Intrinsic::INTRINSIC:                                \
-    HasRoundingMD = ROUND_MODE;                             \
-    break;
-#include "llvm/IR/ConstrainedOps.def"
-  }
-  if (HasRoundingMD)
+
+  if (Intrinsic::hasConstrainedFPRoundingModeOperand(Callee->getIntrinsicID()))
     UseArgs.push_back(getConstrainedFPRounding(Rounding));
   UseArgs.push_back(getConstrainedFPExcept(Except));
 
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index 6743b315c74a..e17755c8ad57 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -365,37 +365,23 @@ FCmpInst::Predicate ConstrainedFPCmpIntrinsic::getPredicate() const {
   return getFPPredicateFromMD(getArgOperand(2));
 }
 
-bool ConstrainedFPIntrinsic::isUnaryOp() const {
-  switch (getIntrinsicID()) {
-  default:
-    return false;
-#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
-  case Intrinsic::INTRINSIC:                                                   \
-    return NARG == 1;
-#include "llvm/IR/ConstrainedOps.def"
-  }
-}
+unsigned ConstrainedFPIntrinsic::getNonMetadataArgCount() const {
+  // All constrained fp intrinsics have "fpexcept" metadata.
+  unsigned NumArgs = arg_size() - 1;
 
-bool ConstrainedFPIntrinsic::isTernaryOp() const {
-  switch (getIntrinsicID()) {
-  default:
-    return false;
-#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
-  case Intrinsic::INTRINSIC:                                                   \
-    return NARG == 3;
-#include "llvm/IR/ConstrainedOps.def"
-  }
+  // Some intrinsics have "round" metadata.
+  if (Intrinsic::hasConstrainedFPRoundingModeOperand(getIntrinsicID()))
+    NumArgs -= 1;
+
+  // Compare intrinsics take their predicate as metadata.
+  if (isa<ConstrainedFPCmpIntrinsic>(this))
+    NumArgs -= 1;
+
+  return NumArgs;
 }
 
 bool ConstrainedFPIntrinsic::classof(const IntrinsicInst *I) {
-  switch (I->getIntrinsicID()) {
-#define INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC)                        \
-  case Intrinsic::INTRINSIC:
-#include "llvm/IR/ConstrainedOps.def"
-    return true;
-  default:
-    return false;
-  }
+  return Intrinsic::isConstrainedFPIntrinsic(I->getIntrinsicID());
 }
 
 ElementCount VPIntrinsic::getStaticVectorLength() const {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 41d3fce7eef7..aa8160d18edd 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -5384,11 +5384,13 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
   }
 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
 #include "llvm/IR/VPIntrinsics.def"
+#undef BEGIN_REGISTER_VP_INTRINSIC
     visitVPIntrinsic(cast<VPIntrinsic>(Call));
     break;
 #define INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC)                        \
   case Intrinsic::INTRINSIC:
 #include "llvm/IR/ConstrainedOps.def"
+#undef INSTRUCTION
     visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(Call));
     break;
   case Intrinsic::dbg_declare: // llvm.dbg.declare
@@ -6527,19 +6529,13 @@ void Verifier::visitVPIntrinsic(VPIntrinsic &VPI) {
 }
 
 void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) {
-  unsigned NumOperands;
-  bool HasRoundingMD;
-  switch (FPI.getIntrinsicID()) {
-#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
-  case Intrinsic::INTRINSIC:                                                   \
-    NumOperands = NARG;                                                        \
-    HasRoundingMD = ROUND_MODE;                                                \
-    break;
-#include "llvm/IR/ConstrainedOps.def"
-  default:
-    llvm_unreachable("Invalid constrained FP intrinsic!");
-  }
+  unsigned NumOperands = FPI.getNonMetadataArgCount();
+  bool HasRoundingMD =
+      Intrinsic::hasConstrainedFPRoundingModeOperand(FPI.getIntrinsicID());
+
+  // Add the expected number of metadata operands.
   NumOperands += (1 + HasRoundingMD);
+
   // Compare intrinsics carry an extra predicate metadata operand.
   if (isa<ConstrainedFPCmpIntrinsic>(FPI))
     NumOperands += 1;
@@ -6553,8 +6549,8 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) {
     Type *ResultTy = FPI.getType();
     Check(!ValTy->isVectorTy() && !ResultTy->isVectorTy(),
           "Intrinsic does not support vectors", &FPI);
-  }
     break;
+  }
 
   case Intrinsic::experimental_constrained_lround:
   case Intrinsic::experimental_constrained_llround: {
@@ -6593,8 +6589,8 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) {
             "Intrinsic first argument and result vector lengths must be equal",
             &FPI);
     }
-  }
     break;
+  }
 
   case Intrinsic::experimental_constrained_sitofp:
   case Intrinsic::experimental_constrained_uitofp: {
@@ -6616,7 +6612,8 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) {
             "Intrinsic first argument and result vector lengths must be equal",
             &FPI);
     }
-  } break;
+    break;
+  }
 
   case Intrinsic::experimental_constrained_fptrunc:
   case Intrinsic::experimental_constrained_fpext: {
@@ -6645,8 +6642,8 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) {
             "Intrinsic first argument's type must be smaller than result type",
             &FPI);
     }
-  }
     break;
+  }
 
   default:
     break;
diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp
index 6f6dc63653c1..6a3b3faac77d 100644
--- a/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -386,18 +386,6 @@ public:
 };
 } // namespace
 
-static bool hasRoundingModeOperand(Intrinsic::ID CIID) {
-  switch (CIID) {
-#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
-  case Intrinsic::INTRINSIC:                                                   \
-    return ROUND_MODE == 1;
-#define FUNCTION INSTRUCTION
-#include "llvm/IR/ConstrainedOps.def"
-  default:
-    llvm_unreachable("Unexpected constrained intrinsic id");
-  }
-}
-
 Instruction *
 PruningFunctionCloner::cloneInstruction(BasicBlock::const_iterator II) {
   const Instruction &OldInst = *II;
@@ -455,7 +443,7 @@ PruningFunctionCloner::cloneInstruction(BasicBlock::const_iterator II) {
       // The last arguments of a constrained intrinsic are metadata that
       // represent rounding mode (absents in some intrinsics) and exception
       // behavior. The inlined function uses default settings.
-      if (hasRoundingModeOperand(CIID))
+      if (Intrinsic::hasConstrainedFPRoundingModeOperand(CIID))
         Args.push_back(
             MetadataAsValue::get(Ctx, MDString::get(Ctx, "round.tonearest")));
       Args.push_back(
-- 
GitLab


From 651bdb96b16d4e522f4611b60103234b1f890b24 Mon Sep 17 00:00:00 2001
From: Chris Copeland <chris@chrisnc.net>
Date: Tue, 7 May 2024 03:48:30 -0700
Subject: [PATCH 0034/1206] [ARM] Armv8-R does not require fp64 or neon.
 (#88287)

This was [addressed for AArch64
here](https://github.com/llvm/llvm-project/pull/79004), but the same
applies to ARM.

Move the enablement of neon+fp64 to `-mcpu=cortex-r52`, which optionally
supports these features.
---
 clang/test/Driver/arm-cortex-cpus-1.c              | 8 ++++----
 clang/test/Driver/arm-features.c                   | 2 +-
 clang/test/Preprocessor/arm-target-features.c      | 4 ++--
 llvm/docs/ReleaseNotes.rst                         | 2 ++
 llvm/include/llvm/TargetParser/ARMTargetParser.def | 4 ++--
 llvm/lib/Target/ARM/ARMArchitectures.td            | 5 ++---
 llvm/lib/Target/ARM/ARMProcessors.td               | 2 ++
 llvm/test/Analysis/CostModel/ARM/arith.ll          | 2 +-
 llvm/test/Analysis/CostModel/ARM/cast.ll           | 4 ++--
 llvm/test/Analysis/CostModel/ARM/cast_ldst.ll      | 4 ++--
 llvm/test/Analysis/CostModel/ARM/cmps.ll           | 4 ++--
 llvm/test/Analysis/CostModel/ARM/divrem.ll         | 2 +-
 llvm/test/CodeGen/ARM/cortex-a57-misched-basic.ll  | 2 +-
 llvm/test/CodeGen/ARM/fpconv.ll                    | 4 ++--
 llvm/test/CodeGen/ARM/half.ll                      | 4 ++--
 llvm/test/CodeGen/ARM/useaa.ll                     | 6 +-----
 llvm/unittests/TargetParser/TargetParserTest.cpp   | 4 ++--
 17 files changed, 31 insertions(+), 32 deletions(-)

diff --git a/clang/test/Driver/arm-cortex-cpus-1.c b/clang/test/Driver/arm-cortex-cpus-1.c
index 25abbe1e3a8a..6f0b64910f9b 100644
--- a/clang/test/Driver/arm-cortex-cpus-1.c
+++ b/clang/test/Driver/arm-cortex-cpus-1.c
@@ -153,23 +153,23 @@
 // RUN: %clang -target armv8r-linux-gnueabi -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V8R %s
 // RUN: %clang -target arm -march=armv8r -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V8R %s
 // RUN: %clang -target arm -march=armv8-r -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V8R %s
-// CHECK-V8R: "-cc1"{{.*}} "-triple" "armv8r-{{.*}} "-target-cpu" "cortex-r52"
+// CHECK-V8R: "-cc1"{{.*}} "-triple" "armv8r-{{.*}} "-target-cpu" "generic"
 
 // RUN: %clang -target armv8r-linux-gnueabi -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V8R-BIG %s
 // RUN: %clang -target arm -march=armv8r -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V8R-BIG %s
 // RUN: %clang -target arm -march=armv8-r -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V8R-BIG %s
-// CHECK-V8R-BIG: "-cc1"{{.*}} "-triple" "armebv8r-{{.*}} "-target-cpu" "cortex-r52"
+// CHECK-V8R-BIG: "-cc1"{{.*}} "-triple" "armebv8r-{{.*}} "-target-cpu" "generic"
 
 // RUN: %clang -target armv8r-linux-gnueabi -mthumb -### -c %s 2>&1 | \
 // RUN:     FileCheck -check-prefix=CHECK-V8R-THUMB %s
 // RUN: %clang -target arm -march=armv8r -mthumb -### -c %s 2>&1 | \
 // RUN:     FileCheck -check-prefix=CHECK-V8R-THUMB %s
-// CHECK-V8R-THUMB: "-cc1"{{.*}} "-triple" "thumbv8r-{{.*}} "-target-cpu" "cortex-r52"
+// CHECK-V8R-THUMB: "-cc1"{{.*}} "-triple" "thumbv8r-{{.*}} "-target-cpu" "generic"
 // RUN: %clang -target armv8r-linux-gnueabi -mthumb -mbig-endian -### -c %s 2>&1 | \
 // RUN:     FileCheck -check-prefix=CHECK-V8R-THUMB-BIG %s
 // RUN: %clang -target arm -march=armv8r -mthumb -mbig-endian -### -c %s 2>&1 | \
 // RUN:     FileCheck -check-prefix=CHECK-V8R-THUMB-BIG %s
-// CHECK-V8R-THUMB-BIG: "-cc1"{{.*}} "-triple" "thumbebv8r-{{.*}} "-target-cpu" "cortex-r52"
+// CHECK-V8R-THUMB-BIG: "-cc1"{{.*}} "-triple" "thumbebv8r-{{.*}} "-target-cpu" "generic"
 
 // RUN: %clang -mcpu=generic -target armv8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V8A-GENERIC %s
 // RUN: %clang -mcpu=generic -target arm -march=armv8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V8A-GENERIC %s
diff --git a/clang/test/Driver/arm-features.c b/clang/test/Driver/arm-features.c
index e043244f18a6..eb424f5f6111 100644
--- a/clang/test/Driver/arm-features.c
+++ b/clang/test/Driver/arm-features.c
@@ -74,7 +74,7 @@
 // Check +crypto for M and R profiles:
 //
 // RUN: %clang -target arm-arm-none-eabi -march=armv8-r+crypto   -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CRYPTO-R %s
-// CHECK-CRYPTO-R: "-cc1"{{.*}} "-target-cpu" "cortex-r52"{{.*}} "-target-feature" "+sha2" "-target-feature" "+aes"
+// CHECK-CRYPTO-R: "-cc1"{{.*}} "-target-cpu" "generic"{{.*}} "-target-feature" "+sha2" "-target-feature" "+aes"
 // RUN: %clang -target arm-arm-none-eabi -march=armv8-m.base+crypto -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-NOCRYPTO5 %s
 // RUN: %clang -target arm-arm-none-eabi -march=armv8-m.main+crypto -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-NOCRYPTO5 %s
 // RUN: %clang -target arm-arm-none-eabi -mcpu=cortex-m23+crypto -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-NOCRYPTO5 %s
diff --git a/clang/test/Preprocessor/arm-target-features.c b/clang/test/Preprocessor/arm-target-features.c
index 236c9f2479b7..2d65bfd4f439 100644
--- a/clang/test/Preprocessor/arm-target-features.c
+++ b/clang/test/Preprocessor/arm-target-features.c
@@ -88,8 +88,8 @@
 // CHECK-V8R: #define __ARM_FEATURE_NUMERIC_MAXMIN 1
 // CHECK-V8R-NOT: #define __ARM_FP 0x
 
-// RUN: %clang -target armv8r-none-linux-gnueabi -x c -E -dM %s -o - | FileCheck -match-full-lines --check-prefix=CHECK-V8R-ALLOW-FP-INSTR %s
-// RUN: %clang -target armv8r-none-linux-gnueabihf -x c -E -dM %s -o - | FileCheck -match-full-lines --check-prefix=CHECK-V8R-ALLOW-FP-INSTR %s
+// RUN: %clang -target armv8r-none-linux-gnueabi -mcpu=cortex-r52 -x c -E -dM %s -o - | FileCheck -match-full-lines --check-prefix=CHECK-V8R-ALLOW-FP-INSTR %s
+// RUN: %clang -target armv8r-none-linux-gnueabihf -mcpu=cortex-r52 -x c -E -dM %s -o - | FileCheck -match-full-lines --check-prefix=CHECK-V8R-ALLOW-FP-INSTR %s
 // CHECK-V8R-ALLOW-FP-INSTR: #define __ARMEL__ 1
 // CHECK-V8R-ALLOW-FP-INSTR: #define __ARM_ARCH 8
 // CHECK-V8R-ALLOW-FP-INSTR: #define __ARM_ARCH_8R__ 1
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 3cf65aac1cc1..9deae46d0233 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -85,7 +85,9 @@ Changes to the AMDGPU Backend
 
 Changes to the ARM Backend
 --------------------------
+
 * FEAT_F32MM is no longer activated by default when using `+sve` on v8.6-A or greater. The feature is still available and can be used by adding `+f32mm` to the command line options.
+* armv8-r now implies only fp-armv8d16sp, rather than neon and full fp-armv8. These features are still included by default for cortex-r52. The default cpu for armv8-r is now "generic", for compatibility with variants that do not include neon, fp64, and d32.
 
 Changes to the AVR Backend
 --------------------------
diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def
index b821d224d7a8..d7b77a6ef5b6 100644
--- a/llvm/include/llvm/TargetParser/ARMTargetParser.def
+++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def
@@ -183,7 +183,7 @@ ARM_ARCH("armv9.5-a", ARMV9_5A, "9.5-A", "+v9.5a", ARMBuildAttrs::CPUArch::v9_A,
           ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS |
           ARM::AEK_DOTPROD | ARM::AEK_BF16 | ARM::AEK_I8MM))
 ARM_ARCH("armv8-r", ARMV8R, "8-R", "+v8r", ARMBuildAttrs::CPUArch::v8_R,
-         FK_NEON_FP_ARMV8,
+         FK_FPV5_SP_D16,
          (ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
           ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC))
 ARM_ARCH("armv8-m.base", ARMV8MBaseline, "8-M.Baseline", "+v8m.base",
@@ -329,7 +329,7 @@ ARM_CPU_NAME("cortex-r7", ARMV7R, FK_VFPV3_D16_FP16, false,
              (ARM::AEK_MP | ARM::AEK_HWDIVARM))
 ARM_CPU_NAME("cortex-r8", ARMV7R, FK_VFPV3_D16_FP16, false,
              (ARM::AEK_MP | ARM::AEK_HWDIVARM))
-ARM_CPU_NAME("cortex-r52", ARMV8R, FK_NEON_FP_ARMV8, true, ARM::AEK_NONE)
+ARM_CPU_NAME("cortex-r52", ARMV8R, FK_NEON_FP_ARMV8, false, ARM::AEK_NONE)
 ARM_CPU_NAME("sc300", ARMV7M, FK_NONE, false, ARM::AEK_NONE)
 ARM_CPU_NAME("cortex-m3", ARMV7M, FK_NONE, true, ARM::AEK_NONE)
 ARM_CPU_NAME("cortex-m4", ARMV7EM, FK_FPV4_SP_D16, true, ARM::AEK_NONE)
diff --git a/llvm/lib/Target/ARM/ARMArchitectures.td b/llvm/lib/Target/ARM/ARMArchitectures.td
index daf54f457b3b..e1e90cdae188 100644
--- a/llvm/lib/Target/ARM/ARMArchitectures.td
+++ b/llvm/lib/Target/ARM/ARMArchitectures.td
@@ -293,9 +293,8 @@ def ARMv8r    : Architecture<"armv8-r",   "ARMv8r",   [HasV8Ops,
                                                        FeatureDSP,
                                                        FeatureCRC,
                                                        FeatureMP,
-                                                       FeatureVirtualization,
-                                                       FeatureFPARMv8,
-                                                       FeatureNEON]>;
+                                                       FeatureFPARMv8_D16_SP,
+                                                       FeatureVirtualization]>;
 
 def ARMv8mBaseline : Architecture<"armv8-m.base", "ARMv8mBaseline",
                                                       [HasV8MBaselineOps,
diff --git a/llvm/lib/Target/ARM/ARMProcessors.td b/llvm/lib/Target/ARM/ARMProcessors.td
index 2c5594976400..eb5ed41ae8a1 100644
--- a/llvm/lib/Target/ARM/ARMProcessors.td
+++ b/llvm/lib/Target/ARM/ARMProcessors.td
@@ -573,5 +573,7 @@ def : ProcNoItin<"kryo",                                [ARMv8a, ProcKryo,
                                                          FeatureCRC]>;
 
 def : ProcessorModel<"cortex-r52", CortexR52Model,      [ARMv8r, ProcR52,
+                                                         FeatureFPARMv8,
+                                                         FeatureNEON,
                                                          FeatureUseMISched,
                                                          FeatureFPAO]>;
diff --git a/llvm/test/Analysis/CostModel/ARM/arith.ll b/llvm/test/Analysis/CostModel/ARM/arith.ll
index 3a137a5af366..8f173596c3b9 100644
--- a/llvm/test/Analysis/CostModel/ARM/arith.ll
+++ b/llvm/test/Analysis/CostModel/ARM/arith.ll
@@ -4,7 +4,7 @@
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve,+mve4beat < %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MVE4
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8R
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R
 ; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=thumbv8.1m.main -mattr=+mve < %s | FileCheck %s --check-prefix=CHECK-MVE-SIZE
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
diff --git a/llvm/test/Analysis/CostModel/ARM/cast.ll b/llvm/test/Analysis/CostModel/ARM/cast.ll
index 60addd3077ed..ae0d2347ec8b 100644
--- a/llvm/test/Analysis/CostModel/ARM/cast.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cast.ll
@@ -3,11 +3,11 @@
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE-RECIP
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN-RECIP
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE-RECIP
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8R-RECIP
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R-RECIP
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE-SIZE
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN-SIZE
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE-SIZE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=armv8r-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8R-SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R-SIZE
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll b/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
index db700eb3baee..4a2f9a25dc15 100644
--- a/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
@@ -3,11 +3,11 @@
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE-RECIP
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN-RECIP
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE-RECIP
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8R-RECIP
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R-RECIP
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE-SIZE
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN-SIZE
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE-SIZE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=armv8r-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8R-SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R-SIZE
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Analysis/CostModel/ARM/cmps.ll b/llvm/test/Analysis/CostModel/ARM/cmps.ll
index 7f89f521e77c..184b7076d02b 100644
--- a/llvm/test/Analysis/CostModel/ARM/cmps.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cmps.ll
@@ -2,11 +2,11 @@
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE-RECIP
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN-RECIP
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE-RECIP
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8R-RECIP
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R-RECIP
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE-SIZE
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN-SIZE
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE-SIZE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=armv8r-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8R-SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R-SIZE
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Analysis/CostModel/ARM/divrem.ll b/llvm/test/Analysis/CostModel/ARM/divrem.ll
index b582a61c2a0f..36c258503232 100644
--- a/llvm/test/Analysis/CostModel/ARM/divrem.ll
+++ b/llvm/test/Analysis/CostModel/ARM/divrem.ll
@@ -3,7 +3,7 @@
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8R
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/CodeGen/ARM/cortex-a57-misched-basic.ll b/llvm/test/CodeGen/ARM/cortex-a57-misched-basic.ll
index 2e8a05417d43..ec4e37f0ba80 100644
--- a/llvm/test/CodeGen/ARM/cortex-a57-misched-basic.ll
+++ b/llvm/test/CodeGen/ARM/cortex-a57-misched-basic.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: asserts
 ; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=A57_SCHED
-; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=generic    -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
+; RUN: llc < %s -mtriple=armv8r-eabi -mattr=+neon,+fp-armv8 -mcpu=generic    -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
 
 ; Check the latency for instructions for both generic and cortex-a57.
 ; SDIV should be scheduled at the block's begin (20 cyc of independent M unit).
diff --git a/llvm/test/CodeGen/ARM/fpconv.ll b/llvm/test/CodeGen/ARM/fpconv.ll
index 929da5f18c81..7e6109f75201 100644
--- a/llvm/test/CodeGen/ARM/fpconv.ll
+++ b/llvm/test/CodeGen/ARM/fpconv.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=arm-eabi -mattr=+vfp2 %s -o - | FileCheck %s --check-prefix=CHECK-VFP
 ; RUN: llc -mtriple=arm-apple-darwin %s -o - | FileCheck %s
-; RUN: llc -mtriple=armv8r-none-none-eabi %s -o - | FileCheck %s --check-prefix=CHECK-VFP
-; RUN: llc -mtriple=armv8r-none-none-eabi -mattr=-fp64 %s -o - | FileCheck %s --check-prefix=CHECK-VFP-SP
+; RUN: llc -mtriple=armv8r-none-none-eabi -mattr=+neon,+fp-armv8 %s -o - | FileCheck %s --check-prefix=CHECK-VFP
+; RUN: llc -mtriple=armv8r-none-none-eabi %s -o - | FileCheck %s --check-prefix=CHECK-VFP-SP
 
 define float @f1(double %x) {
 ;CHECK-VFP-LABEL: f1:
diff --git a/llvm/test/CodeGen/ARM/half.ll b/llvm/test/CodeGen/ARM/half.ll
index 9b53dc77f227..9f8c552cf839 100644
--- a/llvm/test/CodeGen/ARM/half.ll
+++ b/llvm/test/CodeGen/ARM/half.ll
@@ -1,8 +1,8 @@
 ; RUN: llc < %s -mtriple=thumbv7-apple-ios7.0 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-OLD
 ; RUN: llc < %s -mtriple=thumbv7s-apple-ios7.0 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-F16
 ; RUN: llc < %s -mtriple=thumbv8-apple-ios7.0 | FileCheck %s --check-prefix=CHECK  --check-prefix=CHECK-V8
-; RUN: llc < %s -mtriple=armv8r-none-none-eabi | FileCheck %s --check-prefix=CHECK  --check-prefix=CHECK-V8
-; RUN: llc < %s -mtriple=armv8r-none-none-eabi -mattr=-fp64 | FileCheck %s --check-prefix=CHECK  --check-prefix=CHECK-V8-SP
+; RUN: llc < %s -mtriple=armv8r-none-none-eabi -mattr=+neon,+fp-armv8 | FileCheck %s --check-prefix=CHECK  --check-prefix=CHECK-V8
+; RUN: llc < %s -mtriple=armv8r-none-none-eabi | FileCheck %s --check-prefix=CHECK  --check-prefix=CHECK-V8-SP
 ; RUN: llc < %s -mtriple=armv8.1m-none-none-eabi -mattr=+fp-armv8 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-V8
 ; RUN: llc < %s -mtriple=armv8.1m-none-none-eabi -mattr=+fp-armv8,-fp64 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-V8-SP
 ; RUN: llc < %s -mtriple=armv8.1m-none-none-eabi -mattr=+mve.fp,+fp64 | FileCheck %s --check-prefix=CHECK-V8
diff --git a/llvm/test/CodeGen/ARM/useaa.ll b/llvm/test/CodeGen/ARM/useaa.ll
index f8207a1056e3..d70d24b3fd34 100644
--- a/llvm/test/CodeGen/ARM/useaa.ll
+++ b/llvm/test/CodeGen/ARM/useaa.ll
@@ -1,15 +1,11 @@
 ; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-r52 | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA
 ; RUN: llc < %s -mtriple=armv7m-eabi -mcpu=cortex-m4 | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA
 ; RUN: llc < %s -mtriple=armv8m-eabi -mcpu=cortex-m33 | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA
-; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=generic | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
+; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=generic | FileCheck %s --check-prefix=CHECK --check-prefix=USEAA
 
 ; Check we use AA during codegen, so can interleave these loads/stores.
 
 ; CHECK-LABEL: test
-; GENERIC: ldr
-; GENERIC: ldr
-; GENERIC: str
-; GENERIC: str
 ; USEAA: ldr
 ; USEAA: ldr
 ; USEAA: str
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 816aea44a9bc..cc098c264065 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -641,8 +641,8 @@ TEST(TargetParserTest, testARMArch) {
                           ARMBuildAttrs::CPUArch::v9_A));
   EXPECT_TRUE(testARMArch("armv9.5-a", "generic", "v9.5a",
                           ARMBuildAttrs::CPUArch::v9_A));
-  EXPECT_TRUE(testARMArch("armv8-r", "cortex-r52", "v8r",
-                          ARMBuildAttrs::CPUArch::v8_R));
+  EXPECT_TRUE(
+      testARMArch("armv8-r", "generic", "v8r", ARMBuildAttrs::CPUArch::v8_R));
   EXPECT_TRUE(testARMArch("armv8-m.base", "generic", "v8m.base",
                           ARMBuildAttrs::CPUArch::v8_M_Base));
   EXPECT_TRUE(testARMArch("armv8-m.main", "generic", "v8m.main",
-- 
GitLab


From 97dd8e3c4f38ef345b01fbbf0a2052c7875ff7e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Don=C3=A1t=20Nagy?= <donat.nagy@ericsson.com>
Date: Tue, 7 May 2024 13:06:11 +0200
Subject: [PATCH 0035/1206] [analyzer] Clean up apiModeling.llvm.ReturnValue
 (#91231)

This commit heavily refactors and simplifies the small and trivial
checker `apiModeling.llvm.ReturnValue`, which is responsible for
modeling the peculiar coding convention that in the LLVM/Clang codebase
certain Error() methods always return true.

Changes included in this commit:
- The call description mode is now specified explicitly (this is not the
most significant change, but it was the original reason for touching
this checker).
- Previously the code provided support for modeling functions that
always return `false`; but there was no need for that, so this commit
hardcodes that the return value is `true`.
- The overcomplicated constraint/state handling logic was simplified.
- The separate `checkEndFunction` callback was removed to simplify the
code. Admittedly this means that the note tag for the "<method> returns
false, breaking the convention" case is placed on the method call
instead of the `return` statement; but that case will _never_ appear in
practice, so this difference is mostly academical.
- The text of the note tags was clarified.
- The descriptions in the header comment and Checkers.td were clarified.
- Some minor cleanup was applied in the associated test file.

This change is very close to NFC because it only affects a hidden
`apiModeling.llvm` checker that's only relevant during the analysis of
the LLVM/Clang codebase, and even there it doesn't affect the normal
behavior of the checker.
---
 .../clang/StaticAnalyzer/Checkers/Checkers.td |   2 +-
 .../Checkers/ReturnValueChecker.cpp           | 156 +++++-------------
 .../test/Analysis/return-value-guaranteed.cpp |  66 ++++----
 3 files changed, 79 insertions(+), 145 deletions(-)

diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index 520286b57c9f..64414e3d37f7 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -1397,7 +1397,7 @@ def CastValueChecker : Checker<"CastValue">,
   Documentation<NotDocumented>;
 
 def ReturnValueChecker : Checker<"ReturnValue">,
-  HelpText<"Model the guaranteed boolean return value of function calls">,
+  HelpText<"Model certain Error() methods that always return true by convention">,
   Documentation<NotDocumented>;
 
 } // end "apiModeling.llvm"
diff --git a/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp
index c3112ebe4e79..3da571adfa44 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp
@@ -1,4 +1,4 @@
-//===- ReturnValueChecker - Applies guaranteed return values ----*- C++ -*-===//
+//===- ReturnValueChecker - Check methods always returning true -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This defines ReturnValueChecker, which checks for calls with guaranteed
-// boolean return value. It ensures the return value of each function call.
+// This defines ReturnValueChecker, which models a very specific coding
+// convention within the LLVM/Clang codebase: there several classes that have
+// Error() methods which always return true.
+// This checker was introduced to eliminate false positives caused by this
+// peculiar "always returns true" invariant. (Normally, the analyzer assumes
+// that a function returning `bool` can return both `true` and `false`, because
+// otherwise it could've been a `void` function.)
 //
 //===----------------------------------------------------------------------===//
 
@@ -18,43 +23,40 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/FormatVariadic.h"
 #include <optional>
 
 using namespace clang;
 using namespace ento;
+using llvm::formatv;
 
 namespace {
-class ReturnValueChecker : public Checker<check::PostCall, check::EndFunction> {
+class ReturnValueChecker : public Checker<check::PostCall> {
 public:
-  // It sets the predefined invariant ('CDM') if the current call not break it.
   void checkPostCall(const CallEvent &Call, CheckerContext &C) const;
 
-  // It reports whether a predefined invariant ('CDM') is broken.
-  void checkEndFunction(const ReturnStmt *RS, CheckerContext &C) const;
-
 private:
-  // The pairs are in the following form: {{{class, call}}, return value}
-  const CallDescriptionMap<bool> CDM = {
+  const CallDescriptionSet Methods = {
       // These are known in the LLVM project: 'Error()'
-      {{{"ARMAsmParser", "Error"}}, true},
-      {{{"HexagonAsmParser", "Error"}}, true},
-      {{{"LLLexer", "Error"}}, true},
-      {{{"LLParser", "Error"}}, true},
-      {{{"MCAsmParser", "Error"}}, true},
-      {{{"MCAsmParserExtension", "Error"}}, true},
-      {{{"TGParser", "Error"}}, true},
-      {{{"X86AsmParser", "Error"}}, true},
+      {CDM::CXXMethod, {"ARMAsmParser", "Error"}},
+      {CDM::CXXMethod, {"HexagonAsmParser", "Error"}},
+      {CDM::CXXMethod, {"LLLexer", "Error"}},
+      {CDM::CXXMethod, {"LLParser", "Error"}},
+      {CDM::CXXMethod, {"MCAsmParser", "Error"}},
+      {CDM::CXXMethod, {"MCAsmParserExtension", "Error"}},
+      {CDM::CXXMethod, {"TGParser", "Error"}},
+      {CDM::CXXMethod, {"X86AsmParser", "Error"}},
       // 'TokError()'
-      {{{"LLParser", "TokError"}}, true},
-      {{{"MCAsmParser", "TokError"}}, true},
-      {{{"MCAsmParserExtension", "TokError"}}, true},
-      {{{"TGParser", "TokError"}}, true},
+      {CDM::CXXMethod, {"LLParser", "TokError"}},
+      {CDM::CXXMethod, {"MCAsmParser", "TokError"}},
+      {CDM::CXXMethod, {"MCAsmParserExtension", "TokError"}},
+      {CDM::CXXMethod, {"TGParser", "TokError"}},
       // 'error()'
-      {{{"MIParser", "error"}}, true},
-      {{{"WasmAsmParser", "error"}}, true},
-      {{{"WebAssemblyAsmParser", "error"}}, true},
+      {CDM::CXXMethod, {"MIParser", "error"}},
+      {CDM::CXXMethod, {"WasmAsmParser", "error"}},
+      {CDM::CXXMethod, {"WebAssemblyAsmParser", "error"}},
       // Other
-      {{{"AsmParser", "printError"}}, true}};
+      {CDM::CXXMethod, {"AsmParser", "printError"}}};
 };
 } // namespace
 
@@ -68,100 +70,32 @@ static std::string getName(const CallEvent &Call) {
   return Name;
 }
 
-// The predefinitions ('CDM') could break due to the ever growing code base.
-// Check for the expected invariants and see whether they apply.
-static std::optional<bool> isInvariantBreak(bool ExpectedValue, SVal ReturnV,
-                                            CheckerContext &C) {
-  auto ReturnDV = ReturnV.getAs<DefinedOrUnknownSVal>();
-  if (!ReturnDV)
-    return std::nullopt;
-
-  if (ExpectedValue)
-    return C.getState()->isNull(*ReturnDV).isConstrainedTrue();
-
-  return C.getState()->isNull(*ReturnDV).isConstrainedFalse();
-}
-
 void ReturnValueChecker::checkPostCall(const CallEvent &Call,
                                        CheckerContext &C) const {
-  const bool *RawExpectedValue = CDM.lookup(Call);
-  if (!RawExpectedValue)
+  if (!Methods.contains(Call))
     return;
 
-  SVal ReturnV = Call.getReturnValue();
-  bool ExpectedValue = *RawExpectedValue;
-  std::optional<bool> IsInvariantBreak =
-      isInvariantBreak(ExpectedValue, ReturnV, C);
-  if (!IsInvariantBreak)
-    return;
+  auto ReturnV = Call.getReturnValue().getAs<DefinedOrUnknownSVal>();
 
-  // If the invariant is broken it is reported by 'checkEndFunction()'.
-  if (*IsInvariantBreak)
+  if (!ReturnV)
     return;
 
-  std::string Name = getName(Call);
-  const NoteTag *CallTag = C.getNoteTag(
-      [Name, ExpectedValue](PathSensitiveBugReport &) -> std::string {
-        SmallString<128> Msg;
-        llvm::raw_svector_ostream Out(Msg);
-
-        Out << '\'' << Name << "' returns "
-            << (ExpectedValue ? "true" : "false");
-        return std::string(Out.str());
-      },
-      /*IsPrunable=*/true);
-
   ProgramStateRef State = C.getState();
-  State = State->assume(ReturnV.castAs<DefinedOrUnknownSVal>(), ExpectedValue);
-  C.addTransition(State, CallTag);
-}
-
-void ReturnValueChecker::checkEndFunction(const ReturnStmt *RS,
-                                          CheckerContext &C) const {
-  if (!RS || !RS->getRetValue())
+  if (ProgramStateRef StTrue = State->assume(*ReturnV, true)) {
+    // The return value can be true, so transition to a state where it's true.
+    std::string Msg =
+        formatv("'{0}' returns true (by convention)", getName(Call));
+    C.addTransition(StTrue, C.getNoteTag(Msg, /*IsPrunable=*/true));
     return;
-
-  // We cannot get the caller in the top-frame.
-  const StackFrameContext *SFC = C.getStackFrame();
-  if (C.getStackFrame()->inTopFrame())
-    return;
-
-  ProgramStateRef State = C.getState();
-  CallEventManager &CMgr = C.getStateManager().getCallEventManager();
-  CallEventRef<> Call = CMgr.getCaller(SFC, State);
-  if (!Call)
-    return;
-
-  const bool *RawExpectedValue = CDM.lookup(*Call);
-  if (!RawExpectedValue)
-    return;
-
-  SVal ReturnV = State->getSVal(RS->getRetValue(), C.getLocationContext());
-  bool ExpectedValue = *RawExpectedValue;
-  std::optional<bool> IsInvariantBreak =
-      isInvariantBreak(ExpectedValue, ReturnV, C);
-  if (!IsInvariantBreak)
-    return;
-
-  // If the invariant is appropriate it is reported by 'checkPostCall()'.
-  if (!*IsInvariantBreak)
-    return;
-
-  std::string Name = getName(*Call);
-  const NoteTag *CallTag = C.getNoteTag(
-      [Name, ExpectedValue](BugReport &BR) -> std::string {
-        SmallString<128> Msg;
-        llvm::raw_svector_ostream Out(Msg);
-
-        // The following is swapped because the invariant is broken.
-        Out << '\'' << Name << "' returns "
-            << (ExpectedValue ? "false" : "true");
-
-        return std::string(Out.str());
-      },
-      /*IsPrunable=*/false);
-
-  C.addTransition(State, CallTag);
+  }
+  // Paranoia: if the return value is known to be false (which is highly
+  // unlikely, it's easy to ensure that the method always returns true), then
+  // produce a note that highlights that this unusual situation.
+  // Note that this checker is 'hidden' so it cannot produce a bug report.
+  std::string Msg = formatv("'{0}' returned false, breaking the convention "
+                            "that it always returns true",
+                            getName(Call));
+  C.addTransition(State, C.getNoteTag(Msg, /*IsPrunable=*/true));
 }
 
 void ento::registerReturnValueChecker(CheckerManager &Mgr) {
diff --git a/clang/test/Analysis/return-value-guaranteed.cpp b/clang/test/Analysis/return-value-guaranteed.cpp
index 367a8e5906af..3b010ffba360 100644
--- a/clang/test/Analysis/return-value-guaranteed.cpp
+++ b/clang/test/Analysis/return-value-guaranteed.cpp
@@ -1,91 +1,91 @@
 // RUN: %clang_analyze_cc1 \
 // RUN:  -analyzer-checker=core,apiModeling.llvm.ReturnValue \
-// RUN:  -analyzer-output=text -verify=class %s
+// RUN:  -analyzer-output=text -verify %s
 
 struct Foo { int Field; };
 bool problem();
 void doSomething();
 
-// We predefined the return value of 'MCAsmParser::Error' as true and we cannot
-// take the false-branches which leads to a "garbage value" false positive.
-namespace test_classes {
+// Test the normal case when the implementation of MCAsmParser::Error() (one of
+// the methods modeled by this checker) is opaque.
+namespace test_normal {
 struct MCAsmParser {
   static bool Error();
 };
 
 bool parseFoo(Foo &F) {
   if (problem()) {
-    // class-note@-1 {{Assuming the condition is false}}
-    // class-note@-2 {{Taking false branch}}
+    // expected-note@-1 {{Assuming the condition is false}}
+    // expected-note@-2 {{Taking false branch}}
     return MCAsmParser::Error();
   }
 
   F.Field = 0;
-  // class-note@-1 {{The value 0 is assigned to 'F.Field'}}
-  return !MCAsmParser::Error();
-  // class-note@-1 {{'MCAsmParser::Error' returns true}}
+  // expected-note@-1 {{The value 0 is assigned to 'F.Field'}}
+  return false;
 }
 
 bool parseFile() {
   Foo F;
   if (parseFoo(F)) {
-    // class-note@-1 {{Calling 'parseFoo'}}
-    // class-note@-2 {{Returning from 'parseFoo'}}
-    // class-note@-3 {{Taking false branch}}
+    // expected-note@-1 {{Calling 'parseFoo'}}
+    // expected-note@-2 {{Returning from 'parseFoo'}}
+    // expected-note@-3 {{Taking false branch}}
     return true;
   }
 
+  // The following expression would produce the false positive report
+  //    "The left operand of '==' is a garbage value"
+  // without the modeling done by apiModeling.llvm.ReturnValue:
   if (F.Field == 0) {
-    // class-note@-1 {{Field 'Field' is equal to 0}}
-    // class-note@-2 {{Taking true branch}}
-
-    // no-warning: "The left operand of '==' is a garbage value" was here.
+    // expected-note@-1 {{Field 'Field' is equal to 0}}
+    // expected-note@-2 {{Taking true branch}}
     doSomething();
   }
 
+  // Trigger a zero division to get path notes:
   (void)(1 / F.Field);
-  // class-warning@-1 {{Division by zero}}
-  // class-note@-2 {{Division by zero}}
+  // expected-warning@-1 {{Division by zero}}
+  // expected-note@-2 {{Division by zero}}
   return false;
 }
-} // namespace test_classes
+} // namespace test_normal
 
 
-// We predefined 'MCAsmParser::Error' as returning true, but now it returns
-// false, which breaks our invariant. Test the notes.
+// Sanity check for the highly unlikely case where the implementation of the
+// method breaks the convention.
 namespace test_break {
 struct MCAsmParser {
   static bool Error() {
-    return false; // class-note {{'MCAsmParser::Error' returns false}}
+    return false;
   }
 };
 
 bool parseFoo(Foo &F) {
   if (problem()) {
-    // class-note@-1 {{Assuming the condition is false}}
-    // class-note@-2 {{Taking false branch}}
+    // expected-note@-1 {{Assuming the condition is false}}
+    // expected-note@-2 {{Taking false branch}}
     return !MCAsmParser::Error();
   }
 
   F.Field = 0;
-  // class-note@-1 {{The value 0 is assigned to 'F.Field'}}
+  // expected-note@-1 {{The value 0 is assigned to 'F.Field'}}
   return MCAsmParser::Error();
-  // class-note@-1 {{Calling 'MCAsmParser::Error'}}
-  // class-note@-2 {{Returning from 'MCAsmParser::Error'}}
+  // expected-note@-1 {{'MCAsmParser::Error' returned false, breaking the convention that it always returns true}}
 }
 
 bool parseFile() {
   Foo F;
   if (parseFoo(F)) {
-    // class-note@-1 {{Calling 'parseFoo'}}
-    // class-note@-2 {{Returning from 'parseFoo'}}
-    // class-note@-3 {{Taking false branch}}
+    // expected-note@-1 {{Calling 'parseFoo'}}
+    // expected-note@-2 {{Returning from 'parseFoo'}}
+    // expected-note@-3 {{Taking false branch}}
     return true;
   }
 
   (void)(1 / F.Field);
-  // class-warning@-1 {{Division by zero}}
-  // class-note@-2 {{Division by zero}}
+  // expected-warning@-1 {{Division by zero}}
+  // expected-note@-2 {{Division by zero}}
   return false;
 }
-} // namespace test_classes
+} // namespace test_break
-- 
GitLab


From dcc7ef3ce87d7ea1ed9e64bb91e3bb2026df9644 Mon Sep 17 00:00:00 2001
From: Emma Pilkington <emma.pilkington95@gmail.com>
Date: Tue, 7 May 2024 07:38:58 -0400
Subject: [PATCH 0036/1206] [AMDGPU][MC] Disable sendmsg
 SYSMSG_OP_HOST_TRAP_ACK on gfx9+ (#90203)

This is no longer supported as of gfx9. Fixes #52903

This commit also includes some refactoring of sendmsg operand parsing:
  - Use CustomOperand for sendmsg operations, this allows them to be
    conditionally available based on a STI check (and automatically in
    sync with SIDefines.h).
  - Move CustomOperand table lookups from AMDGPUBaseInfo to
    AMDGPUAsmUtils. This cleans up an awkward interface where
    AMDGPUAsmUtils defined a table/size as globals that AMDGPUBaseInfo
    had to loop over.
  - Clean up a few of the operand lookup functions while moving them.
---
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      |   8 +-
 llvm/lib/Target/AMDGPU/SIDefines.h            |   3 -
 .../Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp    | 122 +++++++++++++++---
 llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h |  34 +++--
 .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp    | 119 ++---------------
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h |  19 +--
 llvm/test/MC/AMDGPU/gfx9-asm-err.s            |   3 +
 llvm/test/MC/AMDGPU/sopp-err.s                |   4 +
 llvm/test/MC/AMDGPU/sopp-gfx9.s               |   3 +
 9 files changed, 150 insertions(+), 165 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index e7930b68972e..5ac245ac3b63 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -7436,7 +7436,8 @@ AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &Msg,
     Op.IsDefined = true;
     Op.Loc = getLoc();
     if (isToken(AsmToken::Identifier) &&
-        (Op.Val = getMsgOpId(Msg.Val, getTokenStr())) >= 0) {
+        (Op.Val = getMsgOpId(Msg.Val, getTokenStr(), getSTI())) !=
+            OPR_ID_UNKNOWN) {
       lex(); // skip operation name
     } else if (!parseExpr(Op.Val, "an operation name")) {
       return false;
@@ -7484,7 +7485,10 @@ AMDGPUAsmParser::validateSendMsg(const OperandInfoTy &Msg,
     return false;
   }
   if (!isValidMsgOp(Msg.Val, Op.Val, getSTI(), Strict)) {
-    Error(Op.Loc, "invalid operation id");
+    if (Op.Val == OPR_ID_UNSUPPORTED)
+      Error(Op.Loc, "specified operation id is not supported on this GPU");
+    else
+      Error(Op.Loc, "invalid operation id");
     return false;
   }
   if (Strict && !msgSupportsStream(Msg.Val, Op.Val, getSTI()) &&
diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h
index 1f0207ddb0eb..6d0e0b3f4de2 100644
--- a/llvm/lib/Target/AMDGPU/SIDefines.h
+++ b/llvm/lib/Target/AMDGPU/SIDefines.h
@@ -468,7 +468,6 @@ enum Id { // Message ID, width(4) [3:0].
 };
 
 enum Op { // Both GS and SYS operation IDs.
-  OP_UNKNOWN_ = -1,
   OP_SHIFT_ = 4,
   OP_NONE_ = 0,
   // Bits used for operation encoding
@@ -479,14 +478,12 @@ enum Op { // Both GS and SYS operation IDs.
   OP_GS_CUT = 1,
   OP_GS_EMIT = 2,
   OP_GS_EMIT_CUT = 3,
-  OP_GS_LAST_,
   OP_GS_FIRST_ = OP_GS_NOP,
   // SYS operations are encoded in bits 6:4
   OP_SYS_ECC_ERR_INTERRUPT = 1,
   OP_SYS_REG_RD = 2,
   OP_SYS_HOST_TRAP_ACK = 3,
   OP_SYS_TTRACE_PC = 4,
-  OP_SYS_LAST_,
   OP_SYS_FIRST_ = OP_SYS_ECC_ERR_INTERRUPT,
 };
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
index d468b14d54d3..2e1db1665b9c 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
@@ -12,6 +12,60 @@
 namespace llvm {
 namespace AMDGPU {
 
+//===----------------------------------------------------------------------===//
+// Custom Operands.
+//
+// A table of custom operands shall describe "primary" operand names first
+// followed by aliases if any. It is not required but recommended to arrange
+// operands so that operand encoding match operand position in the table. This
+// will make getNameFromOperandTable() a bit more efficient. Unused slots in the
+// table shall have an empty name.
+//
+//===----------------------------------------------------------------------===//
+
+/// Map from the encoding of a sendmsg/hwreg asm operand to it's name.
+template <size_t N>
+static StringRef getNameFromOperandTable(const CustomOperand (&Table)[N],
+                                         unsigned Encoding,
+                                         const MCSubtargetInfo &STI) {
+  auto isValidIndexForEncoding = [&](size_t Idx) {
+    return Idx < N && Table[Idx].Encoding == Encoding &&
+           !Table[Idx].Name.empty() &&
+           (!Table[Idx].Cond || Table[Idx].Cond(STI));
+  };
+
+  // This is an optimization that should work in most cases. As a side effect,
+  // it may cause selection of an alias instead of a primary operand name in
+  // case of sparse tables.
+  if (isValidIndexForEncoding(Encoding))
+    return Table[Encoding].Name;
+
+  for (size_t Idx = 0; Idx != N; ++Idx)
+    if (isValidIndexForEncoding(Idx))
+      return Table[Idx].Name;
+
+  return "";
+}
+
+/// Map from a symbolic name for a sendmsg/hwreg asm operand to it's encoding.
+template <size_t N>
+static int64_t getEncodingFromOperandTable(const CustomOperand (&Table)[N],
+                                           StringRef Name,
+                                           const MCSubtargetInfo &STI) {
+  int64_t InvalidEncoding = OPR_ID_UNKNOWN;
+  for (const CustomOperand &Entry : Table) {
+    if (Entry.Name != Name)
+      continue;
+
+    if (!Entry.Cond || Entry.Cond(STI))
+      return Entry.Encoding;
+
+    InvalidEncoding = OPR_ID_UNSUPPORTED;
+  }
+
+  return InvalidEncoding;
+}
+
 namespace DepCtr {
 
 // NOLINTBEGIN
@@ -34,10 +88,11 @@ const int DEP_CTR_SIZE =
 
 namespace SendMsg {
 
-// Disable lint checking for this block since it makes the table unreadable.
+// Disable lint checking here since it makes these tables unreadable.
 // NOLINTBEGIN
 // clang-format off
-const CustomOperand<const MCSubtargetInfo &> Msg[] = {
+
+static constexpr CustomOperand MsgOperands[] = {
   {{""}},
   {{"MSG_INTERRUPT"},           ID_INTERRUPT},
   {{"MSG_GS"},                  ID_GS_PreGFX11,             isNotGFX11Plus},
@@ -63,27 +118,47 @@ const CustomOperand<const MCSubtargetInfo &> Msg[] = {
   {{"MSG_RTN_GET_TBA_TO_PC"},   ID_RTN_GET_TBA_TO_PC,       isGFX11Plus},
   {{"MSG_RTN_GET_SE_AID_ID"},   ID_RTN_GET_SE_AID_ID,       isGFX12Plus},
 };
+
+static constexpr CustomOperand SysMsgOperands[] = {
+  {{""}},
+  {{"SYSMSG_OP_ECC_ERR_INTERRUPT"},  OP_SYS_ECC_ERR_INTERRUPT},
+  {{"SYSMSG_OP_REG_RD"},             OP_SYS_REG_RD},
+  {{"SYSMSG_OP_HOST_TRAP_ACK"},      OP_SYS_HOST_TRAP_ACK,      isNotGFX9Plus},
+  {{"SYSMSG_OP_TTRACE_PC"},          OP_SYS_TTRACE_PC},
+};
+
+static constexpr CustomOperand StreamMsgOperands[] = {
+  {{"GS_OP_NOP"},       OP_GS_NOP},
+  {{"GS_OP_CUT"},       OP_GS_CUT},
+  {{"GS_OP_EMIT"},      OP_GS_EMIT},
+  {{"GS_OP_EMIT_CUT"},  OP_GS_EMIT_CUT},
+};
+
 // clang-format on
 // NOLINTEND
 
-const int MSG_SIZE = static_cast<int>(
-    sizeof(Msg) / sizeof(CustomOperand<const MCSubtargetInfo &>));
+int64_t getMsgId(StringRef Name, const MCSubtargetInfo &STI) {
+  return getEncodingFromOperandTable(MsgOperands, Name, STI);
+}
 
-// These two must be in sync with llvm::AMDGPU::SendMsg::Op enum members, see SIDefines.h.
-const char *const OpSysSymbolic[OP_SYS_LAST_] = {
-  nullptr,
-  "SYSMSG_OP_ECC_ERR_INTERRUPT",
-  "SYSMSG_OP_REG_RD",
-  "SYSMSG_OP_HOST_TRAP_ACK",
-  "SYSMSG_OP_TTRACE_PC"
-};
+StringRef getMsgName(uint64_t Encoding, const MCSubtargetInfo &STI) {
+  return getNameFromOperandTable(MsgOperands, Encoding, STI);
+}
 
-const char *const OpGsSymbolic[OP_GS_LAST_] = {
-  "GS_OP_NOP",
-  "GS_OP_CUT",
-  "GS_OP_EMIT",
-  "GS_OP_EMIT_CUT"
-};
+int64_t getMsgOpId(int64_t MsgId, StringRef Name, const MCSubtargetInfo &STI) {
+  if (MsgId == ID_SYSMSG)
+    return getEncodingFromOperandTable(SysMsgOperands, Name, STI);
+  return getEncodingFromOperandTable(StreamMsgOperands, Name, STI);
+}
+
+StringRef getMsgOpName(int64_t MsgId, uint64_t Encoding,
+                       const MCSubtargetInfo &STI) {
+  assert(msgRequiresOp(MsgId, STI) && "must have an operand");
+
+  if (MsgId == ID_SYSMSG)
+    return getNameFromOperandTable(SysMsgOperands, Encoding, STI);
+  return getNameFromOperandTable(StreamMsgOperands, Encoding, STI);
+}
 
 } // namespace SendMsg
 
@@ -92,7 +167,7 @@ namespace Hwreg {
 // Disable lint checking for this block since it makes the table unreadable.
 // NOLINTBEGIN
 // clang-format off
-const CustomOperand<const MCSubtargetInfo &> Opr[] = {
+static constexpr CustomOperand Operands[] = {
   {{""}},
   {{"HW_REG_MODE"},          ID_MODE},
   {{"HW_REG_STATUS"},        ID_STATUS},
@@ -155,8 +230,13 @@ const CustomOperand<const MCSubtargetInfo &> Opr[] = {
 // clang-format on
 // NOLINTEND
 
-const int OPR_SIZE = static_cast<int>(
-    sizeof(Opr) / sizeof(CustomOperand<const MCSubtargetInfo &>));
+int64_t getHwregId(StringRef Name, const MCSubtargetInfo &STI) {
+  return getEncodingFromOperandTable(Operands, Name, STI);
+}
+
+StringRef getHwreg(uint64_t Encoding, const MCSubtargetInfo &STI) {
+  return getNameFromOperandTable(Operands, Encoding, STI);
+}
 
 } // namespace Hwreg
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h
index 054e35e90f2f..069134a7ae7f 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h
@@ -25,10 +25,10 @@ const int OPR_ID_UNSUPPORTED = -2;
 const int OPR_ID_DUPLICATE = -3;
 const int OPR_VAL_INVALID = -4;
 
-template <class T> struct CustomOperand {
+struct CustomOperand {
   StringLiteral Name;
-  int Encoding = 0;
-  bool (*Cond)(T Context) = nullptr;
+  unsigned Encoding = 0;
+  bool (*Cond)(const MCSubtargetInfo &STI) = nullptr;
 };
 
 struct CustomOperandVal {
@@ -60,20 +60,34 @@ extern const int DEP_CTR_SIZE;
 
 } // namespace DepCtr
 
-namespace SendMsg { // Symbolic names for the sendmsg(...) syntax.
+// Symbolic names for the sendmsg(msg_id, operation, stream) syntax.
+namespace SendMsg {
+
+/// Map from a symbolic name for a msg_id to the message portion of the
+/// immediate encoding. A negative return value indicates that the Name was
+/// unknown or unsupported on this target.
+int64_t getMsgId(StringRef Name, const MCSubtargetInfo &STI);
+
+/// Map from an encoding to the symbolic name for a msg_id immediate. This is
+/// doing opposite of getMsgId().
+StringRef getMsgName(uint64_t Encoding, const MCSubtargetInfo &STI);
 
-extern const CustomOperand<const MCSubtargetInfo &> Msg[];
-extern const int MSG_SIZE;
+/// Map from a symbolic name for a sendmsg operation to the operation portion of
+/// the immediate encoding. A negative return value indicates that the Name was
+/// unknown or unsupported on this target.
+int64_t getMsgOpId(int64_t MsgId, StringRef Name, const MCSubtargetInfo &STI);
 
-extern const char *const OpSysSymbolic[OP_SYS_LAST_];
-extern const char *const OpGsSymbolic[OP_GS_LAST_];
+/// Map from an encoding to the symbolic name for a sendmsg operation. This is
+/// doing opposite of getMsgOpId().
+StringRef getMsgOpName(int64_t MsgId, uint64_t Encoding,
+                       const MCSubtargetInfo &STI);
 
 } // namespace SendMsg
 
 namespace Hwreg { // Symbolic names for the hwreg(...) syntax.
 
-extern const CustomOperand<const MCSubtargetInfo &> Opr[];
-extern const int OPR_SIZE;
+int64_t getHwregId(StringRef Name, const MCSubtargetInfo &STI);
+StringRef getHwreg(uint64_t Encoding, const MCSubtargetInfo &STI);
 
 } // namespace Hwreg
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 4e0074451aa5..2fae7a31d70b 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1495,62 +1495,6 @@ unsigned encodeStorecntDscnt(const IsaVersion &Version,
   return encodeStorecntDscnt(Version, Decoded.StoreCnt, Decoded.DsCnt);
 }
 
-//===----------------------------------------------------------------------===//
-// Custom Operands.
-//
-// A table of custom operands shall describe "primary" operand names
-// first followed by aliases if any. It is not required but recommended
-// to arrange operands so that operand encoding match operand position
-// in the table. This will make disassembly a bit more efficient.
-// Unused slots in the table shall have an empty name.
-//
-//===----------------------------------------------------------------------===//
-
-template <class T>
-static bool isValidOpr(int Idx, const CustomOperand<T> OpInfo[], int OpInfoSize,
-                       T Context) {
-  return 0 <= Idx && Idx < OpInfoSize && !OpInfo[Idx].Name.empty() &&
-         (!OpInfo[Idx].Cond || OpInfo[Idx].Cond(Context));
-}
-
-template <class T>
-static int getOprIdx(std::function<bool(const CustomOperand<T> &)> Test,
-                     const CustomOperand<T> OpInfo[], int OpInfoSize,
-                     T Context) {
-  int InvalidIdx = OPR_ID_UNKNOWN;
-  for (int Idx = 0; Idx < OpInfoSize; ++Idx) {
-    if (Test(OpInfo[Idx])) {
-      if (!OpInfo[Idx].Cond || OpInfo[Idx].Cond(Context))
-        return Idx;
-      InvalidIdx = OPR_ID_UNSUPPORTED;
-    }
-  }
-  return InvalidIdx;
-}
-
-template <class T>
-static int getOprIdx(const StringRef Name, const CustomOperand<T> OpInfo[],
-                     int OpInfoSize, T Context) {
-  auto Test = [=](const CustomOperand<T> &Op) { return Op.Name == Name; };
-  return getOprIdx<T>(Test, OpInfo, OpInfoSize, Context);
-}
-
-template <class T>
-static int getOprIdx(int Id, const CustomOperand<T> OpInfo[], int OpInfoSize,
-                     T Context, bool QuickCheck = true) {
-  auto Test = [=](const CustomOperand<T> &Op) {
-    return Op.Encoding == Id && !Op.Name.empty();
-  };
-  // This is an optimization that should work in most cases.
-  // As a side effect, it may cause selection of an alias
-  // instead of a primary operand name in case of sparse tables.
-  if (QuickCheck && isValidOpr<T>(Id, OpInfo, OpInfoSize, Context) &&
-      OpInfo[Id].Encoding == Id) {
-    return Id;
-  }
-  return getOprIdx<T>(Test, OpInfo, OpInfoSize, Context);
-}
-
 //===----------------------------------------------------------------------===//
 // Custom Operand Values
 //===----------------------------------------------------------------------===//
@@ -1701,24 +1645,6 @@ unsigned encodeFieldSaSdst(unsigned SaSdst) {
 
 } // namespace DepCtr
 
-//===----------------------------------------------------------------------===//
-// hwreg
-//===----------------------------------------------------------------------===//
-
-namespace Hwreg {
-
-int64_t getHwregId(const StringRef Name, const MCSubtargetInfo &STI) {
-  int Idx = getOprIdx<const MCSubtargetInfo &>(Name, Opr, OPR_SIZE, STI);
-  return (Idx < 0) ? Idx : Opr[Idx].Encoding;
-}
-
-StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) {
-  int Idx = getOprIdx<const MCSubtargetInfo &>(Id, Opr, OPR_SIZE, STI);
-  return (Idx < 0) ? "" : Opr[Idx].Name;
-}
-
-} // namespace Hwreg
-
 //===----------------------------------------------------------------------===//
 // exp tgt
 //===----------------------------------------------------------------------===//
@@ -1919,32 +1845,10 @@ static uint64_t getMsgIdMask(const MCSubtargetInfo &STI) {
   return isGFX11Plus(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_;
 }
 
-int64_t getMsgId(const StringRef Name, const MCSubtargetInfo &STI) {
-  int Idx = getOprIdx<const MCSubtargetInfo &>(Name, Msg, MSG_SIZE, STI);
-  return (Idx < 0) ? Idx : Msg[Idx].Encoding;
-}
-
 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) {
   return (MsgId & ~(getMsgIdMask(STI))) == 0;
 }
 
-StringRef getMsgName(int64_t MsgId, const MCSubtargetInfo &STI) {
-  int Idx = getOprIdx<const MCSubtargetInfo &>(MsgId, Msg, MSG_SIZE, STI);
-  return (Idx < 0) ? "" : Msg[Idx].Name;
-}
-
-int64_t getMsgOpId(int64_t MsgId, const StringRef Name) {
-  const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
-  const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
-  const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
-  for (int i = F; i < L; ++i) {
-    if (Name == S[i]) {
-      return i;
-    }
-  }
-  return OP_UNKNOWN_;
-}
-
 bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
                   bool Strict) {
   assert(isValidMsgId(MsgId, STI));
@@ -1952,23 +1856,14 @@ bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
   if (!Strict)
     return 0 <= OpId && isUInt<OP_WIDTH_>(OpId);
 
-  if (MsgId == ID_SYSMSG)
-    return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_;
-  if (!isGFX11Plus(STI)) {
-    switch (MsgId) {
-    case ID_GS_PreGFX11:
-      return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP;
-    case ID_GS_DONE_PreGFX11:
-      return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_;
-    }
+  if (msgRequiresOp(MsgId, STI)) {
+    if (MsgId == ID_GS_PreGFX11 && OpId == OP_GS_NOP)
+      return false;
+
+    return !getMsgOpName(MsgId, OpId, STI).empty();
   }
-  return OpId == OP_NONE_;
-}
 
-StringRef getMsgOpName(int64_t MsgId, int64_t OpId,
-                       const MCSubtargetInfo &STI) {
-  assert(msgRequiresOp(MsgId, STI));
-  return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId];
+  return OpId == OP_NONE_;
 }
 
 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
@@ -2186,6 +2081,8 @@ bool isGFX9Plus(const MCSubtargetInfo &STI) {
   return isGFX9(STI) || isGFX10Plus(STI);
 }
 
+bool isNotGFX9Plus(const MCSubtargetInfo &STI) { return !isGFX9Plus(STI); }
+
 bool isGFX10(const MCSubtargetInfo &STI) {
   return STI.hasFeature(AMDGPU::FeatureGFX10);
 }
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 943588fe701c..12d1b3a55ccc 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -1078,12 +1078,6 @@ struct HwregSize : EncodingField<15, 11, 32> {
 
 using HwregEncoding = EncodingFields<HwregId, HwregOffset, HwregSize>;
 
-LLVM_READONLY
-int64_t getHwregId(const StringRef Name, const MCSubtargetInfo &STI);
-
-LLVM_READNONE
-StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI);
-
 } // namespace Hwreg
 
 namespace DepCtr {
@@ -1173,18 +1167,6 @@ unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI);
 
 namespace SendMsg {
 
-LLVM_READONLY
-int64_t getMsgId(const StringRef Name, const MCSubtargetInfo &STI);
-
-LLVM_READONLY
-int64_t getMsgOpId(int64_t MsgId, const StringRef Name);
-
-LLVM_READNONE
-StringRef getMsgName(int64_t MsgId, const MCSubtargetInfo &STI);
-
-LLVM_READNONE
-StringRef getMsgOpName(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI);
-
 LLVM_READNONE
 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI);
 
@@ -1276,6 +1258,7 @@ bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI);
 bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI);
 bool isGFX8Plus(const MCSubtargetInfo &STI);
 bool isGFX9Plus(const MCSubtargetInfo &STI);
+bool isNotGFX9Plus(const MCSubtargetInfo &STI);
 bool isGFX10(const MCSubtargetInfo &STI);
 bool isGFX10_GFX11(const MCSubtargetInfo &STI);
 bool isGFX10Plus(const MCSubtargetInfo &STI);
diff --git a/llvm/test/MC/AMDGPU/gfx9-asm-err.s b/llvm/test/MC/AMDGPU/gfx9-asm-err.s
index 93138a829185..31e0d953b5bd 100644
--- a/llvm/test/MC/AMDGPU/gfx9-asm-err.s
+++ b/llvm/test/MC/AMDGPU/gfx9-asm-err.s
@@ -41,3 +41,6 @@ global_load_dword v[2:3], off
 
 scratch_load_dword v2, off, offset:256
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: too few operands for instruction
+
+s_sendmsg sendmsg(MSG_SYSMSG, SYSMSG_OP_HOST_TRAP_ACK)
+// GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: specified operation id is not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/sopp-err.s b/llvm/test/MC/AMDGPU/sopp-err.s
index bd044cb74340..8b7ff74b2105 100644
--- a/llvm/test/MC/AMDGPU/sopp-err.s
+++ b/llvm/test/MC/AMDGPU/sopp-err.s
@@ -199,6 +199,10 @@ s_sendmsg sendmsg(MSG_SYSMSG, 0)
 s_sendmsg sendmsg(MSG_SYSMSG, 5)
 // GCN: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id
 
+s_sendmsg sendmsg(MSG_SYSMSG, SYSMSG_OP_HOST_TRAP_ACK)
+// GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: specified operation id is not supported on this GPU
+// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified operation id is not supported on this GPU
+
 //===----------------------------------------------------------------------===//
 // waitcnt
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AMDGPU/sopp-gfx9.s b/llvm/test/MC/AMDGPU/sopp-gfx9.s
index e760d497896f..2ba6d0043f35 100644
--- a/llvm/test/MC/AMDGPU/sopp-gfx9.s
+++ b/llvm/test/MC/AMDGPU/sopp-gfx9.s
@@ -109,3 +109,6 @@ s_sendmsg 10
 
 s_sendmsg sendmsg(MSG_GET_DOORBELL)
 // GFX9: s_sendmsg sendmsg(MSG_GET_DOORBELL) ; encoding: [0x0a,0x00,0x90,0xbf]
+
+s_sendmsg sendmsg(15, 3, 0)
+// GFX9: s_sendmsg sendmsg(15, 3, 0) ; encoding: [0x3f,0x00,0x90,0xbf]
-- 
GitLab


From 6d64f8e1feee014e72730a78b62d9d415df112ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Don=C3=A1t=20Nagy?= <donat.nagy@ericsson.com>
Date: Tue, 7 May 2024 13:48:02 +0200
Subject: [PATCH 0037/1206] [analyzer] Use explicit call description mode in
 more checkers (#90974)

This commit explicitly specifies the matching mode (C library function,
any non-method function, or C++ method) for the `CallDescription`s
constructed in various checkers.

Some code was simplified to use `CallDescriptionSet`s instead of
individual `CallDescription`s.

This change won't cause major functional changes, but isn't NFC because
it ensures that e.g. call descriptions for a non-method function won't
accidentally match a method that has the same name.

Separate commits have already performed this change in other checkers:
- easy cases: e2f1cbae45f81f3cd9a4d3c2bcf69a094eb060fa
- MallocChecker: d6d84b5d1448e4f2e24b467a0abcf42fe9d543e9
- iterator checkers: 06eedffe0d2782922e63cc25cb927f4acdaf7b30
- InvalidPtr checker: 024281d4d26344f9613b9115ea1fcbdbdba23235

... and follow-up commits will handle the remaining checkers.

My goal is to ensure that the call description mode is always explicitly
specified and eliminate (or strongly restrict) the vague "may be either
a method or a simple function" mode that's the current default.
---
 .../BlockInCriticalSectionChecker.cpp         | 40 ++++++-------
 .../Checkers/CStringChecker.cpp               |  4 +-
 .../Checkers/InnerPointerChecker.cpp          | 58 ++++++++-----------
 .../Checkers/SmartPtrModeling.cpp             | 18 +++---
 .../StaticAnalyzer/Checkers/StreamChecker.cpp |  8 ++-
 5 files changed, 61 insertions(+), 67 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp
index e4373915410f..e138debd1361 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp
@@ -148,27 +148,28 @@ using MutexDescriptor =
 class BlockInCriticalSectionChecker : public Checker<check::PostCall> {
 private:
   const std::array<MutexDescriptor, 8> MutexDescriptors{
-      MemberMutexDescriptor(
-          CallDescription(/*QualifiedName=*/{"std", "mutex", "lock"},
-                          /*RequiredArgs=*/0),
-          CallDescription({"std", "mutex", "unlock"}, 0)),
-      FirstArgMutexDescriptor(CallDescription({"pthread_mutex_lock"}, 1),
-                              CallDescription({"pthread_mutex_unlock"}, 1)),
-      FirstArgMutexDescriptor(CallDescription({"mtx_lock"}, 1),
-                              CallDescription({"mtx_unlock"}, 1)),
-      FirstArgMutexDescriptor(CallDescription({"pthread_mutex_trylock"}, 1),
-                              CallDescription({"pthread_mutex_unlock"}, 1)),
-      FirstArgMutexDescriptor(CallDescription({"mtx_trylock"}, 1),
-                              CallDescription({"mtx_unlock"}, 1)),
-      FirstArgMutexDescriptor(CallDescription({"mtx_timedlock"}, 1),
-                              CallDescription({"mtx_unlock"}, 1)),
+      MemberMutexDescriptor({/*MatchAs=*/CDM::CXXMethod,
+                             /*QualifiedName=*/{"std", "mutex", "lock"},
+                             /*RequiredArgs=*/0},
+                            {CDM::CXXMethod, {"std", "mutex", "unlock"}, 0}),
+      FirstArgMutexDescriptor({CDM::CLibrary, {"pthread_mutex_lock"}, 1},
+                              {CDM::CLibrary, {"pthread_mutex_unlock"}, 1}),
+      FirstArgMutexDescriptor({CDM::CLibrary, {"mtx_lock"}, 1},
+                              {CDM::CLibrary, {"mtx_unlock"}, 1}),
+      FirstArgMutexDescriptor({CDM::CLibrary, {"pthread_mutex_trylock"}, 1},
+                              {CDM::CLibrary, {"pthread_mutex_unlock"}, 1}),
+      FirstArgMutexDescriptor({CDM::CLibrary, {"mtx_trylock"}, 1},
+                              {CDM::CLibrary, {"mtx_unlock"}, 1}),
+      FirstArgMutexDescriptor({CDM::CLibrary, {"mtx_timedlock"}, 1},
+                              {CDM::CLibrary, {"mtx_unlock"}, 1}),
       RAIIMutexDescriptor("lock_guard"),
       RAIIMutexDescriptor("unique_lock")};
 
-  const std::array<CallDescription, 5> BlockingFunctions{
-      ArrayRef{StringRef{"sleep"}}, ArrayRef{StringRef{"getc"}},
-      ArrayRef{StringRef{"fgets"}}, ArrayRef{StringRef{"read"}},
-      ArrayRef{StringRef{"recv"}}};
+  const CallDescriptionSet BlockingFunctions{{CDM::CLibrary, {"sleep"}},
+                                             {CDM::CLibrary, {"getc"}},
+                                             {CDM::CLibrary, {"fgets"}},
+                                             {CDM::CLibrary, {"read"}},
+                                             {CDM::CLibrary, {"recv"}}};
 
   const BugType BlockInCritSectionBugType{
       this, "Call to blocking function in critical section", "Blocking Error"};
@@ -291,8 +292,7 @@ void BlockInCriticalSectionChecker::handleUnlock(
 
 bool BlockInCriticalSectionChecker::isBlockingInCritSection(
     const CallEvent &Call, CheckerContext &C) const {
-  return llvm::any_of(BlockingFunctions,
-                      [&Call](auto &&Fn) { return Fn.matches(Call); }) &&
+  return BlockingFunctions.contains(Call) &&
          !C.getState()->get<ActiveCritSections>().isEmpty();
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
index f9548b5c3010..238e87a712a4 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
@@ -189,8 +189,8 @@ public:
   };
 
   // These require a bit of special handling.
-  CallDescription StdCopy{{"std", "copy"}, 3},
-      StdCopyBackward{{"std", "copy_backward"}, 3};
+  CallDescription StdCopy{CDM::SimpleFunc, {"std", "copy"}, 3},
+      StdCopyBackward{CDM::SimpleFunc, {"std", "copy_backward"}, 3};
 
   FnCheck identifyCall(const CallEvent &Call, CheckerContext &C) const;
   void evalMemcpy(CheckerContext &C, const CallEvent &Call, CharKind CK) const;
diff --git a/clang/lib/StaticAnalyzer/Checkers/InnerPointerChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/InnerPointerChecker.cpp
index b673b51c4623..261db2b2a704 100644
--- a/clang/lib/StaticAnalyzer/Checkers/InnerPointerChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/InnerPointerChecker.cpp
@@ -35,9 +35,28 @@ namespace {
 class InnerPointerChecker
     : public Checker<check::DeadSymbols, check::PostCall> {
 
-  CallDescription AppendFn, AssignFn, AddressofFn, AddressofFn_, ClearFn,
-      CStrFn, DataFn, DataMemberFn, EraseFn, InsertFn, PopBackFn, PushBackFn,
-      ReplaceFn, ReserveFn, ResizeFn, ShrinkToFitFn, SwapFn;
+  CallDescriptionSet InvalidatingMemberFunctions{
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "append"}),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "assign"}),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "clear"}),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "erase"}),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "insert"}),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "pop_back"}),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "push_back"}),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "replace"}),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "reserve"}),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "resize"}),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "shrink_to_fit"}),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "swap"})};
+
+  CallDescriptionSet AddressofFunctions{
+      CallDescription(CDM::SimpleFunc, {"std", "addressof"}),
+      CallDescription(CDM::SimpleFunc, {"std", "__addressof"})};
+
+  CallDescriptionSet InnerPointerAccessFunctions{
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "c_str"}),
+      CallDescription(CDM::SimpleFunc, {"std", "data"}, 1),
+      CallDescription(CDM::CXXMethod, {"std", "basic_string", "data"})};
 
 public:
   class InnerPointerBRVisitor : public BugReporterVisitor {
@@ -71,30 +90,10 @@ public:
     }
   };
 
-  InnerPointerChecker()
-      : AppendFn({"std", "basic_string", "append"}),
-        AssignFn({"std", "basic_string", "assign"}),
-        AddressofFn({"std", "addressof"}), AddressofFn_({"std", "__addressof"}),
-        ClearFn({"std", "basic_string", "clear"}),
-        CStrFn({"std", "basic_string", "c_str"}), DataFn({"std", "data"}, 1),
-        DataMemberFn({"std", "basic_string", "data"}),
-        EraseFn({"std", "basic_string", "erase"}),
-        InsertFn({"std", "basic_string", "insert"}),
-        PopBackFn({"std", "basic_string", "pop_back"}),
-        PushBackFn({"std", "basic_string", "push_back"}),
-        ReplaceFn({"std", "basic_string", "replace"}),
-        ReserveFn({"std", "basic_string", "reserve"}),
-        ResizeFn({"std", "basic_string", "resize"}),
-        ShrinkToFitFn({"std", "basic_string", "shrink_to_fit"}),
-        SwapFn({"std", "basic_string", "swap"}) {}
-
   /// Check whether the called member function potentially invalidates
   /// pointers referring to the container object's inner buffer.
   bool isInvalidatingMemberFunction(const CallEvent &Call) const;
 
-  /// Check whether the called function returns a raw inner pointer.
-  bool isInnerPointerAccessFunction(const CallEvent &Call) const;
-
   /// Mark pointer symbols associated with the given memory region released
   /// in the program state.
   void markPtrSymbolsReleased(const CallEvent &Call, ProgramStateRef State,
@@ -127,14 +126,7 @@ bool InnerPointerChecker::isInvalidatingMemberFunction(
     return false;
   }
   return isa<CXXDestructorCall>(Call) ||
-         matchesAny(Call, AppendFn, AssignFn, ClearFn, EraseFn, InsertFn,
-                    PopBackFn, PushBackFn, ReplaceFn, ReserveFn, ResizeFn,
-                    ShrinkToFitFn, SwapFn);
-}
-
-bool InnerPointerChecker::isInnerPointerAccessFunction(
-    const CallEvent &Call) const {
-  return matchesAny(Call, CStrFn, DataFn, DataMemberFn);
+         InvalidatingMemberFunctions.contains(Call);
 }
 
 void InnerPointerChecker::markPtrSymbolsReleased(const CallEvent &Call,
@@ -181,7 +173,7 @@ void InnerPointerChecker::checkFunctionArguments(const CallEvent &Call,
 
       // std::addressof functions accepts a non-const reference as an argument,
       // but doesn't modify it.
-      if (matchesAny(Call, AddressofFn, AddressofFn_))
+      if (AddressofFunctions.contains(Call))
         continue;
 
       markPtrSymbolsReleased(Call, State, ArgRegion, C);
@@ -221,7 +213,7 @@ void InnerPointerChecker::checkPostCall(const CallEvent &Call,
     }
   }
 
-  if (isInnerPointerAccessFunction(Call)) {
+  if (InnerPointerAccessFunctions.contains(Call)) {
 
     if (isa<SimpleFunctionCall>(Call)) {
       // NOTE: As of now, we only have one free access function: std::data.
diff --git a/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
index 268fc742f050..505020d4bb39 100644
--- a/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
@@ -86,14 +86,14 @@ private:
   using SmartPtrMethodHandlerFn =
       void (SmartPtrModeling::*)(const CallEvent &Call, CheckerContext &) const;
   CallDescriptionMap<SmartPtrMethodHandlerFn> SmartPtrMethodHandlers{
-      {{{"reset"}}, &SmartPtrModeling::handleReset},
-      {{{"release"}}, &SmartPtrModeling::handleRelease},
-      {{{"swap"}, 1}, &SmartPtrModeling::handleSwapMethod},
-      {{{"get"}}, &SmartPtrModeling::handleGet}};
-  const CallDescription StdSwapCall{{"std", "swap"}, 2};
-  const CallDescription StdMakeUniqueCall{{"std", "make_unique"}};
-  const CallDescription StdMakeUniqueForOverwriteCall{
-      {"std", "make_unique_for_overwrite"}};
+      {{CDM::CXXMethod, {"reset"}}, &SmartPtrModeling::handleReset},
+      {{CDM::CXXMethod, {"release"}}, &SmartPtrModeling::handleRelease},
+      {{CDM::CXXMethod, {"swap"}, 1}, &SmartPtrModeling::handleSwapMethod},
+      {{CDM::CXXMethod, {"get"}}, &SmartPtrModeling::handleGet}};
+  const CallDescription StdSwapCall{CDM::SimpleFunc, {"std", "swap"}, 2};
+  const CallDescriptionSet MakeUniqueVariants{
+      {CDM::SimpleFunc, {"std", "make_unique"}},
+      {CDM::SimpleFunc, {"std", "make_unique_for_overwrite"}}};
 };
 } // end of anonymous namespace
 
@@ -296,7 +296,7 @@ bool SmartPtrModeling::evalCall(const CallEvent &Call,
     return handleSwap(State, Call.getArgSVal(0), Call.getArgSVal(1), C);
   }
 
-  if (matchesAny(Call, StdMakeUniqueCall, StdMakeUniqueForOverwriteCall)) {
+  if (MakeUniqueVariants.contains(Call)) {
     if (!ModelSmartPtrDereference)
       return false;
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
index a0aa2316a7b4..a7b6f6c1fb55 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
@@ -388,17 +388,19 @@ private:
   };
 
   CallDescriptionMap<FnDescription> FnTestDescriptions = {
-      {{{"StreamTesterChecker_make_feof_stream"}, 1},
+      {{CDM::SimpleFunc, {"StreamTesterChecker_make_feof_stream"}, 1},
        {nullptr,
         std::bind(&StreamChecker::evalSetFeofFerror, _1, _2, _3, _4, ErrorFEof,
                   false),
         0}},
-      {{{"StreamTesterChecker_make_ferror_stream"}, 1},
+      {{CDM::SimpleFunc, {"StreamTesterChecker_make_ferror_stream"}, 1},
        {nullptr,
         std::bind(&StreamChecker::evalSetFeofFerror, _1, _2, _3, _4,
                   ErrorFError, false),
         0}},
-      {{{"StreamTesterChecker_make_ferror_indeterminate_stream"}, 1},
+      {{CDM::SimpleFunc,
+        {"StreamTesterChecker_make_ferror_indeterminate_stream"},
+        1},
        {nullptr,
         std::bind(&StreamChecker::evalSetFeofFerror, _1, _2, _3, _4,
                   ErrorFError, true),
-- 
GitLab


From afc10fc9b7ce3d23d9012f5a1496e849fe873ba2 Mon Sep 17 00:00:00 2001
From: Peter Waller <peter.waller@arm.com>
Date: Tue, 7 May 2024 12:42:21 +0100
Subject: [PATCH 0038/1206] [llvm-mca] Move bad-input.s test to be target
 specific

... for now.

This is a follow up to #90474 in response to build bot failures.

This test is intended to check a case where invalid assembly is passed
to llvm-mca.

Unfortunately it appears that a cross-toolchain built with
-DTOOLCHAIN_TARGET_TRIPLE does not have an llvm-mca which works out of
the box if the host target is not enabled.

As a quick fix to make the build bots green, move the test into AArch64
and X86 so that there is reasonable coverage for this test; later I hope
mca can be fixed to work out of the box in this configuration.
---
 llvm/test/tools/llvm-mca/{ => AArch64}/bad-input.s |  0
 llvm/test/tools/llvm-mca/X86/bad-input.s           | 14 ++++++++++++++
 2 files changed, 14 insertions(+)
 rename llvm/test/tools/llvm-mca/{ => AArch64}/bad-input.s (100%)
 create mode 100644 llvm/test/tools/llvm-mca/X86/bad-input.s

diff --git a/llvm/test/tools/llvm-mca/bad-input.s b/llvm/test/tools/llvm-mca/AArch64/bad-input.s
similarity index 100%
rename from llvm/test/tools/llvm-mca/bad-input.s
rename to llvm/test/tools/llvm-mca/AArch64/bad-input.s
diff --git a/llvm/test/tools/llvm-mca/X86/bad-input.s b/llvm/test/tools/llvm-mca/X86/bad-input.s
new file mode 100644
index 000000000000..eaf69979cb20
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/bad-input.s
@@ -0,0 +1,14 @@
+# RUN: not llvm-mca %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -skip-unsupported-instructions=none %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -skip-unsupported-instructions=lack-sched %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -skip-unsupported-instructions=parse-failure %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
+# RUN: not llvm-mca -skip-unsupported-instructions=any %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
+
+# Test checks that MCA does not produce a total cycles estimate if it encounters parse errors.
+
+# CHECK-ALL-NOT: Total Cycles:
+
+# CHECK: error: Assembly input parsing had errors, use -skip-unsupported-instructions=parse-failure to drop failing lines from the input.
+# CHECK-SKIP: error: no assembly instructions found.
+
+This is not a valid assembly file for any architecture (by virtue of this text.)
-- 
GitLab


From 458d70674190c4d043d5dfd2e41aecddff5cdb69 Mon Sep 17 00:00:00 2001
From: Peter Waller <peter.waller@arm.com>
Date: Tue, 7 May 2024 13:06:00 +0100
Subject: [PATCH 0039/1206] [llvm-mca] Make bad-input.s even more CPU specific

Note: This patch is distinct from the previous one titled
  "[llvm-mca] Move bad-input.s test to be target specific"

This is a followup to #90474 and commit
afc10fc9b7ce3d23d9012f5a1496e849fe873ba2

Context: Builders failing because they're unable to run the failure
test.

This still doesn't work in various circumstances, it seems MCA doesn't
want to run on a wide variety of hosts in various configurations, so
stick to the tried and tested method and pass -mtriple and -mcpu.
---
 .../tools/llvm-mca/AArch64/Neoverse/bad-input.s  | 16 ++++++++++++++++
 llvm/test/tools/llvm-mca/AArch64/bad-input.s     | 14 --------------
 llvm/test/tools/llvm-mca/X86/bad-input.s         | 12 +++++++-----
 3 files changed, 23 insertions(+), 19 deletions(-)
 create mode 100644 llvm/test/tools/llvm-mca/AArch64/Neoverse/bad-input.s
 delete mode 100644 llvm/test/tools/llvm-mca/AArch64/bad-input.s

diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/bad-input.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/bad-input.s
new file mode 100644
index 000000000000..41891adc6c0d
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/bad-input.s
@@ -0,0 +1,16 @@
+# This test is generic but not all builders have an llvm-mca which can run natively.
+
+# RUN: not llvm-mca -mtriple=aarch64 -mcpu=neoverse-v1 %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -mtriple=aarch64 -mcpu=neoverse-v1 -skip-unsupported-instructions=none %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -mtriple=aarch64 -mcpu=neoverse-v1 -skip-unsupported-instructions=lack-sched %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -mtriple=aarch64 -mcpu=neoverse-v1 -skip-unsupported-instructions=parse-failure %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
+# RUN: not llvm-mca -mtriple=aarch64 -mcpu=neoverse-v1 -skip-unsupported-instructions=any %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
+
+# Test checks that MCA does not produce a total cycles estimate if it encounters parse errors.
+
+# CHECK-ALL-NOT: Total Cycles:
+
+# CHECK: error: Assembly input parsing had errors, use -skip-unsupported-instructions=parse-failure to drop failing lines from the input.
+# CHECK-SKIP: error: no assembly instructions found.
+
+This is not a valid assembly file for any architecture (by virtue of this text.)
diff --git a/llvm/test/tools/llvm-mca/AArch64/bad-input.s b/llvm/test/tools/llvm-mca/AArch64/bad-input.s
deleted file mode 100644
index eaf69979cb20..000000000000
--- a/llvm/test/tools/llvm-mca/AArch64/bad-input.s
+++ /dev/null
@@ -1,14 +0,0 @@
-# RUN: not llvm-mca %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
-# RUN: not llvm-mca -skip-unsupported-instructions=none %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
-# RUN: not llvm-mca -skip-unsupported-instructions=lack-sched %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
-# RUN: not llvm-mca -skip-unsupported-instructions=parse-failure %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
-# RUN: not llvm-mca -skip-unsupported-instructions=any %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
-
-# Test checks that MCA does not produce a total cycles estimate if it encounters parse errors.
-
-# CHECK-ALL-NOT: Total Cycles:
-
-# CHECK: error: Assembly input parsing had errors, use -skip-unsupported-instructions=parse-failure to drop failing lines from the input.
-# CHECK-SKIP: error: no assembly instructions found.
-
-This is not a valid assembly file for any architecture (by virtue of this text.)
diff --git a/llvm/test/tools/llvm-mca/X86/bad-input.s b/llvm/test/tools/llvm-mca/X86/bad-input.s
index eaf69979cb20..b49bccc45b2d 100644
--- a/llvm/test/tools/llvm-mca/X86/bad-input.s
+++ b/llvm/test/tools/llvm-mca/X86/bad-input.s
@@ -1,8 +1,10 @@
-# RUN: not llvm-mca %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
-# RUN: not llvm-mca -skip-unsupported-instructions=none %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
-# RUN: not llvm-mca -skip-unsupported-instructions=lack-sched %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
-# RUN: not llvm-mca -skip-unsupported-instructions=parse-failure %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
-# RUN: not llvm-mca -skip-unsupported-instructions=any %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
+# This test is generic but not all builders have an llvm-mca which can run natively.
+
+# RUN: not llvm-mca -mtriple=x86_64 -mcpu=x86-64 %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -mtriple=x86_64 -mcpu=x86-64 -skip-unsupported-instructions=none %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -mtriple=x86_64 -mcpu=x86-64 -skip-unsupported-instructions=lack-sched %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK %s
+# RUN: not llvm-mca -mtriple=x86_64 -mcpu=x86-64 -skip-unsupported-instructions=parse-failure %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
+# RUN: not llvm-mca -mtriple=x86_64 -mcpu=x86-64 -skip-unsupported-instructions=any %s -o /dev/null 2>&1 | FileCheck --check-prefixes=CHECK-ALL,CHECK-SKIP %s
 
 # Test checks that MCA does not produce a total cycles estimate if it encounters parse errors.
 
-- 
GitLab


From 6f2997cefc1e32c11a891ede2e3a2d73310e6ce1 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Tue, 7 May 2024 14:25:10 +0200
Subject: [PATCH 0040/1206] [libc++][NFC] Remove unused include from
 <__type_traits/is_equality_comparable.h> (#90950)

---
 libcxx/include/__type_traits/is_equality_comparable.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libcxx/include/__type_traits/is_equality_comparable.h b/libcxx/include/__type_traits/is_equality_comparable.h
index 00316ed63778..d4142218b641 100644
--- a/libcxx/include/__type_traits/is_equality_comparable.h
+++ b/libcxx/include/__type_traits/is_equality_comparable.h
@@ -17,7 +17,6 @@
 #include <__type_traits/is_signed.h>
 #include <__type_traits/is_void.h>
 #include <__type_traits/remove_cv.h>
-#include <__type_traits/remove_cvref.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
 
-- 
GitLab


From b22a6f1eba8e27b2a21bf6b96a3bd349230cb80a Mon Sep 17 00:00:00 2001
From: Vincent Belliard <81770341+v-bulle@users.noreply.github.com>
Date: Tue, 7 May 2024 05:42:16 -0700
Subject: [PATCH 0041/1206] [lldb] fix step in AArch64 trampoline (#90783)

Detects AArch64 trampolines in order to be able to step in a function
through a trampoline on AArch64.

---------

Co-authored-by: Vincent Belliard <v-bulle@github.com>
---
 .../POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp     | 13 +++++++++++++
 .../Plugins/ObjectFile/ELF/ObjectFileELF.cpp  | 19 ++++++++++++++++++-
 .../StepIn/Inputs/aarch64_thunk.cc            | 15 +++++++++++++++
 .../StepIn/step_through-aarch64-thunk.test    | 17 +++++++++++++++++
 4 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 lldb/test/Shell/ExecControl/StepIn/Inputs/aarch64_thunk.cc
 create mode 100644 lldb/test/Shell/ExecControl/StepIn/step_through-aarch64-thunk.test

diff --git a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp
index 9fa245fc41d4..51e4b3e6728f 100644
--- a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp
+++ b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp
@@ -506,6 +506,19 @@ DynamicLoaderPOSIXDYLD::GetStepThroughTrampolinePlan(Thread &thread,
   Target &target = thread.GetProcess()->GetTarget();
   const ModuleList &images = target.GetImages();
 
+  llvm::StringRef target_name = sym_name.GetStringRef();
+  // On AArch64, the trampoline name has a prefix (__AArch64ADRPThunk_ or
+  // __AArch64AbsLongThunk_) added to the function name. If we detect a
+  // trampoline with the prefix, we need to remove the prefix to find the
+  // function symbol.
+  if (target_name.consume_front("__AArch64ADRPThunk_") ||
+      target_name.consume_front("__AArch64AbsLongThunk_")) {
+    // An empty target name can happen for trampolines generated for
+    // section-referencing relocations.
+    if (!target_name.empty()) {
+      sym_name = ConstString(target_name);
+    }
+  }
   images.FindSymbolsWithNameAndType(sym_name, eSymbolTypeCode, target_symbols);
   if (!target_symbols.GetSize())
     return thread_plan_sp;
diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
index 16f6d2e884b5..1646ee9aa34a 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
@@ -2356,13 +2356,30 @@ unsigned ObjectFileELF::ParseSymbols(Symtab *symtab, user_id_t start_id,
     bool symbol_size_valid =
         symbol.st_size != 0 || symbol.getType() != STT_FUNC;
 
+    bool is_trampoline = false;
+    if (arch.IsValid() && (arch.GetMachine() == llvm::Triple::aarch64)) {
+      // On AArch64, trampolines are registered as code.
+      // If we detect a trampoline (which starts with __AArch64ADRPThunk_ or
+      // __AArch64AbsLongThunk_) we register the symbol as a trampoline. This
+      // way we will be able to detect the trampoline when we step in a function
+      // and step through the trampoline.
+      if (symbol_type == eSymbolTypeCode) {
+        llvm::StringRef trampoline_name = mangled.GetName().GetStringRef();
+        if (trampoline_name.starts_with("__AArch64ADRPThunk_") ||
+            trampoline_name.starts_with("__AArch64AbsLongThunk_")) {
+          symbol_type = eSymbolTypeTrampoline;
+          is_trampoline = true;
+        }
+      }
+    }
+
     Symbol dc_symbol(
         i + start_id, // ID is the original symbol table index.
         mangled,
         symbol_type,                    // Type of this symbol
         is_global,                      // Is this globally visible?
         false,                          // Is this symbol debug info?
-        false,                          // Is this symbol a trampoline?
+        is_trampoline,                  // Is this symbol a trampoline?
         false,                          // Is this symbol artificial?
         AddressRange(symbol_section_sp, // Section in which this symbol is
                                         // defined or null.
diff --git a/lldb/test/Shell/ExecControl/StepIn/Inputs/aarch64_thunk.cc b/lldb/test/Shell/ExecControl/StepIn/Inputs/aarch64_thunk.cc
new file mode 100644
index 000000000000..02f3bef32a59
--- /dev/null
+++ b/lldb/test/Shell/ExecControl/StepIn/Inputs/aarch64_thunk.cc
@@ -0,0 +1,15 @@
+extern "C" int __attribute__((naked)) __AArch64ADRPThunk_step_here() {
+    asm (
+      "adrp x16, step_here\n"
+      "add x16, x16, :lo12:step_here\n"
+      "br x16"
+    );
+}
+
+extern "C" __attribute__((used)) int step_here() {
+    return 47;
+}
+
+int main() {
+  return __AArch64ADRPThunk_step_here();
+}
diff --git a/lldb/test/Shell/ExecControl/StepIn/step_through-aarch64-thunk.test b/lldb/test/Shell/ExecControl/StepIn/step_through-aarch64-thunk.test
new file mode 100644
index 000000000000..336a746fa3a4
--- /dev/null
+++ b/lldb/test/Shell/ExecControl/StepIn/step_through-aarch64-thunk.test
@@ -0,0 +1,17 @@
+# REQUIRES: native && target-aarch64
+
+# This test is specific to elf platforms.
+# UNSUPPORTED: system-windows, system-darwin
+
+# RUN: %clangxx_host %p/Inputs/aarch64_thunk.cc -g -o %t.out
+# RUN: %lldb %t.out -s %s | FileCheck %s
+
+b main
+# CHECK: Breakpoint 1: where = step_through-aarch64-thunk.test.tmp.out`main
+
+r
+# CHECK: stop reason = breakpoint 1.1
+
+s
+# CHECK: stop reason = step in
+# CHECK:     frame #0: {{.*}} step_through-aarch64-thunk.test.tmp.out`::step_here()
-- 
GitLab


From 66364e65405d4964709e67574abf1b519a55296c Mon Sep 17 00:00:00 2001
From: martinboehme <mboehme@google.com>
Date: Tue, 7 May 2024 14:58:57 +0200
Subject: [PATCH 0042/1206] [clang][dataflow] Add `reachedLimit()` to the
 `Solver` interface. (#91320)

We may want code to call this that doesn't know which specific solver
implementation it is dealing with.
---
 clang/include/clang/Analysis/FlowSensitive/Solver.h            | 3 +++
 .../clang/Analysis/FlowSensitive/WatchedLiteralsSolver.h       | 3 +--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Analysis/FlowSensitive/Solver.h b/clang/include/clang/Analysis/FlowSensitive/Solver.h
index 079f6802f241..6166a503ab41 100644
--- a/clang/include/clang/Analysis/FlowSensitive/Solver.h
+++ b/clang/include/clang/Analysis/FlowSensitive/Solver.h
@@ -87,6 +87,9 @@ public:
   ///
   ///  All elements in `Vals` must not be null.
   virtual Result solve(llvm::ArrayRef<const Formula *> Vals) = 0;
+
+  // Did the solver reach its resource limit?
+  virtual bool reachedLimit() const = 0;
 };
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const Solver::Result &);
diff --git a/clang/include/clang/Analysis/FlowSensitive/WatchedLiteralsSolver.h b/clang/include/clang/Analysis/FlowSensitive/WatchedLiteralsSolver.h
index 5448eecf6d41..b5cd7aa10fd7 100644
--- a/clang/include/clang/Analysis/FlowSensitive/WatchedLiteralsSolver.h
+++ b/clang/include/clang/Analysis/FlowSensitive/WatchedLiteralsSolver.h
@@ -48,8 +48,7 @@ public:
 
   Result solve(llvm::ArrayRef<const Formula *> Vals) override;
 
-  // The solver reached its maximum number of iterations.
-  bool reachedLimit() const { return MaxIterations == 0; }
+  bool reachedLimit() const override { return MaxIterations == 0; }
 };
 
 } // namespace dataflow
-- 
GitLab


From fff2db2e426ebe3a349bd0f00555d4a3dc8a6de7 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Tue, 7 May 2024 07:11:16 -0600
Subject: [PATCH 0043/1206] [libc++] Rename _LIBCPP_INTRODUCED_foo_MARKUP to
 _LIBCPP_INTRODUCED_foo_ATTRIBUTE (#91269)

This was discussed in #87563 and overlooked when I landed the patch.
---
 libcxx/include/__availability | 96 +++++++++++++++++------------------
 1 file changed, 48 insertions(+), 48 deletions(-)

diff --git a/libcxx/include/__availability b/libcxx/include/__availability
index 7a02ae00846b..e44ac1962df3 100644
--- a/libcxx/include/__availability
+++ b/libcxx/include/__availability
@@ -87,43 +87,43 @@
 #if defined(_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS)
 
 #  define _LIBCPP_INTRODUCED_IN_LLVM_4 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_4_MARKUP /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE /* nothing */
 
 #  define _LIBCPP_INTRODUCED_IN_LLVM_9 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP      /* nothing */
-#  define _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP_PUSH /* nothing */
-#  define _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP_POP  /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE      /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP  /* nothing */
 
 #  define _LIBCPP_INTRODUCED_IN_LLVM_10 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_10_MARKUP /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_10_ATTRIBUTE /* nothing */
 
 #  define _LIBCPP_INTRODUCED_IN_LLVM_12 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_12_MARKUP /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_12_ATTRIBUTE /* nothing */
 
 #  define _LIBCPP_INTRODUCED_IN_LLVM_14 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_14_MARKUP /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_14_ATTRIBUTE /* nothing */
 
 #  define _LIBCPP_INTRODUCED_IN_LLVM_15 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_15_MARKUP /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_15_ATTRIBUTE /* nothing */
 
 #  define _LIBCPP_INTRODUCED_IN_LLVM_16 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_16_MARKUP /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_16_ATTRIBUTE /* nothing */
 
 #  define _LIBCPP_INTRODUCED_IN_LLVM_18 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_18_MARKUP /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE /* nothing */
 
 #  define _LIBCPP_INTRODUCED_IN_LLVM_19 1
-#  define _LIBCPP_INTRODUCED_IN_LLVM_19_MARKUP /* nothing */
+#  define _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE /* nothing */
 
 #elif defined(__APPLE__)
 
 // LLVM 4
 #  if defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 50000
 #    define _LIBCPP_INTRODUCED_IN_LLVM_4 0
-#    define _LIBCPP_INTRODUCED_IN_LLVM_4_MARKUP __attribute__((availability(watchos, strict, introduced = 5.0)))
+#    define _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE __attribute__((availability(watchos, strict, introduced = 5.0)))
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_4 1
-#    define _LIBCPP_INTRODUCED_IN_LLVM_4_MARKUP /* nothing */
+#    define _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE /* nothing */
 #  endif
 
 // LLVM 9
@@ -134,18 +134,18 @@
       (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 60000)
 // clang-format on
 #    define _LIBCPP_INTRODUCED_IN_LLVM_9 0
-#    define _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP                                                                        \
+#    define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE                                                                     \
       __attribute__((availability(macos, strict, introduced = 10.15)))                                                 \
       __attribute__((availability(ios, strict, introduced = 13.0)))                                                    \
       __attribute__((availability(tvos, strict, introduced = 13.0)))                                                   \
       __attribute__((availability(watchos, strict, introduced = 6.0)))
 // clang-format off
-#    define _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP_PUSH                                                                               \
+#    define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH                                                                               \
       _Pragma("clang attribute push(__attribute__((availability(macos,strict,introduced=10.15))), apply_to=any(function,record))") \
       _Pragma("clang attribute push(__attribute__((availability(ios,strict,introduced=13.0))), apply_to=any(function,record))")    \
       _Pragma("clang attribute push(__attribute__((availability(tvos,strict,introduced=13.0))), apply_to=any(function,record))")   \
       _Pragma("clang attribute push(__attribute__((availability(watchos,strict,introduced=6.0))), apply_to=any(function,record))")
-#    define _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP_POP                                                                    \
+#    define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP                                                                    \
       _Pragma("clang attribute pop") \
       _Pragma("clang attribute pop") \
       _Pragma("clang attribute pop") \
@@ -153,9 +153,9 @@
 // clang-format on
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_9 1
-#    define _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP      /* nothing */
-#    define _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP_PUSH /* nothing */
-#    define _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP_POP  /* nothing */
+#    define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE      /* nothing */
+#    define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH /* nothing */
+#    define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP  /* nothing */
 #  endif
 
 // LLVM 10
@@ -166,14 +166,14 @@
       (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 70000)
 // clang-format on
 #    define _LIBCPP_INTRODUCED_IN_LLVM_10 0
-#    define _LIBCPP_INTRODUCED_IN_LLVM_10_MARKUP                                                                       \
+#    define _LIBCPP_INTRODUCED_IN_LLVM_10_ATTRIBUTE                                                                    \
       __attribute__((availability(macos, strict, introduced = 11.0)))                                                  \
       __attribute__((availability(ios, strict, introduced = 14.0)))                                                    \
       __attribute__((availability(tvos, strict, introduced = 14.0)))                                                   \
       __attribute__((availability(watchos, strict, introduced = 7.0)))
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_10 1
-#    define _LIBCPP_INTRODUCED_IN_LLVM_10_MARKUP /* nothing */
+#    define _LIBCPP_INTRODUCED_IN_LLVM_10_ATTRIBUTE /* nothing */
 #  endif
 
 // LLVM 12
@@ -184,14 +184,14 @@
       (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 80000)
 // clang-format on
 #    define _LIBCPP_INTRODUCED_IN_LLVM_12 0
-#    define _LIBCPP_INTRODUCED_IN_LLVM_12_MARKUP                                                                       \
+#    define _LIBCPP_INTRODUCED_IN_LLVM_12_ATTRIBUTE                                                                    \
       __attribute__((availability(macos, strict, introduced = 12.0)))                                                  \
       __attribute__((availability(ios, strict, introduced = 15.0)))                                                    \
       __attribute__((availability(tvos, strict, introduced = 15.0)))                                                   \
       __attribute__((availability(watchos, strict, introduced = 8.0)))
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_12 1
-#    define _LIBCPP_INTRODUCED_IN_LLVM_12_MARKUP /* nothing */
+#    define _LIBCPP_INTRODUCED_IN_LLVM_12_ATTRIBUTE /* nothing */
 #  endif
 
 // LLVM 14
@@ -202,19 +202,19 @@
       (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 90500)
 // clang-format on
 #    define _LIBCPP_INTRODUCED_IN_LLVM_14 0
-#    define _LIBCPP_INTRODUCED_IN_LLVM_14_MARKUP                                                                       \
+#    define _LIBCPP_INTRODUCED_IN_LLVM_14_ATTRIBUTE                                                                    \
       __attribute__((availability(macos, strict, introduced = 13.4)))                                                  \
       __attribute__((availability(ios, strict, introduced = 16.5)))                                                    \
       __attribute__((availability(tvos, strict, introduced = 16.5)))                                                   \
       __attribute__((availability(watchos, strict, introduced = 9.5)))
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_14 1
-#    define _LIBCPP_INTRODUCED_IN_LLVM_14_MARKUP /* nothing */
+#    define _LIBCPP_INTRODUCED_IN_LLVM_14_ATTRIBUTE /* nothing */
 #  endif
 
 // LLVM 15-16
 #  define _LIBCPP_INTRODUCED_IN_LLVM_15 _LIBCPP_INTRODUCED_IN_LLVM_16
-#  define _LIBCPP_INTRODUCED_IN_LLVM_15_MARKUP _LIBCPP_INTRODUCED_IN_LLVM_16_MARKUP
+#  define _LIBCPP_INTRODUCED_IN_LLVM_15_ATTRIBUTE _LIBCPP_INTRODUCED_IN_LLVM_16_ATTRIBUTE
 // clang-format off
 #  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 140000) ||   \
       (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 170000) || \
@@ -222,34 +222,34 @@
       (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 100000)
 // clang-format on
 #    define _LIBCPP_INTRODUCED_IN_LLVM_16 0
-#    define _LIBCPP_INTRODUCED_IN_LLVM_16_MARKUP                                                                       \
+#    define _LIBCPP_INTRODUCED_IN_LLVM_16_ATTRIBUTE                                                                    \
       __attribute__((availability(macos, strict, introduced = 14.0)))                                                  \
       __attribute__((availability(ios, strict, introduced = 17.0)))                                                    \
       __attribute__((availability(tvos, strict, introduced = 17.0)))                                                   \
       __attribute__((availability(watchos, strict, introduced = 10.0)))
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_16 1
-#    define _LIBCPP_INTRODUCED_IN_LLVM_16_MARKUP /* nothing */
+#    define _LIBCPP_INTRODUCED_IN_LLVM_16_ATTRIBUTE /* nothing */
 #  endif
 
 // LLVM 18
 // TODO: Fill this in
 #  if 1
 #    define _LIBCPP_INTRODUCED_IN_LLVM_18 0
-#    define _LIBCPP_INTRODUCED_IN_LLVM_18_MARKUP __attribute__((unavailable))
+#    define _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE __attribute__((unavailable))
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_18 1
-#    define _LIBCPP_INTRODUCED_IN_LLVM_18_MARKUP /* nothing */
+#    define _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE /* nothing */
 #  endif
 
 // LLVM 19
 // TODO: Fill this in
 #  if 1
 #    define _LIBCPP_INTRODUCED_IN_LLVM_19 0
-#    define _LIBCPP_INTRODUCED_IN_LLVM_19_MARKUP __attribute__((unavailable))
+#    define _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE __attribute__((unavailable))
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_19 1
-#    define _LIBCPP_INTRODUCED_IN_LLVM_19_MARKUP /* nothing */
+#    define _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE /* nothing */
 #  endif
 
 #else
@@ -270,27 +270,27 @@
 // these exceptions can be used even on older deployment targets, but those
 // methods will abort instead of throwing.
 #define _LIBCPP_AVAILABILITY_HAS_BAD_OPTIONAL_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4
-#define _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4_MARKUP
+#define _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE
 
 #define _LIBCPP_AVAILABILITY_HAS_BAD_VARIANT_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4
-#define _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4_MARKUP
+#define _LIBCPP_AVAILABILITY_BAD_VARIANT_ACCESS _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE
 
 #define _LIBCPP_AVAILABILITY_HAS_BAD_ANY_CAST _LIBCPP_INTRODUCED_IN_LLVM_4
-#define _LIBCPP_AVAILABILITY_BAD_ANY_CAST _LIBCPP_INTRODUCED_IN_LLVM_4_MARKUP
+#define _LIBCPP_AVAILABILITY_BAD_ANY_CAST _LIBCPP_INTRODUCED_IN_LLVM_4_ATTRIBUTE
 
 // These macros control the availability of all parts of <filesystem> that
 // depend on something in the dylib.
 #define _LIBCPP_AVAILABILITY_HAS_FILESYSTEM_LIBRARY _LIBCPP_INTRODUCED_IN_LLVM_9
-#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP
-#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP_PUSH
-#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP _LIBCPP_INTRODUCED_IN_LLVM_9_MARKUP_POP
+#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE
+#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH
+#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP
 
 // This controls the availability of the C++20 synchronization library,
 // which requires shared library support for various operations
 // (see libcxx/src/atomic.cpp). This includes <barier>, <latch>,
 // <semaphore>, and notification functions on std::atomic.
 #define _LIBCPP_AVAILABILITY_HAS_SYNC _LIBCPP_INTRODUCED_IN_LLVM_10
-#define _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INTRODUCED_IN_LLVM_10_MARKUP
+#define _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INTRODUCED_IN_LLVM_10_ATTRIBUTE
 
 // Enable additional explicit instantiations of iostreams components. This
 // reduces the number of weak definitions generated in programs that use
@@ -308,13 +308,13 @@
 // This controls the availability of floating-point std::to_chars functions.
 // These overloads were added later than the integer overloads.
 #define _LIBCPP_AVAILABILITY_HAS_TO_CHARS_FLOATING_POINT _LIBCPP_INTRODUCED_IN_LLVM_14
-#define _LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_INTRODUCED_IN_LLVM_14_MARKUP
+#define _LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_INTRODUCED_IN_LLVM_14_ATTRIBUTE
 
 // This controls whether the library claims to provide a default verbose
 // termination function, and consequently whether the headers will try
 // to use it when the mechanism isn't overriden at compile-time.
 #define _LIBCPP_AVAILABILITY_HAS_VERBOSE_ABORT _LIBCPP_INTRODUCED_IN_LLVM_15
-#define _LIBCPP_AVAILABILITY_VERBOSE_ABORT _LIBCPP_INTRODUCED_IN_LLVM_15_MARKUP
+#define _LIBCPP_AVAILABILITY_VERBOSE_ABORT _LIBCPP_INTRODUCED_IN_LLVM_15_ATTRIBUTE
 
 // This controls the availability of the C++17 std::pmr library,
 // which is implemented in large part in the built library.
@@ -330,27 +330,27 @@
 // in the built library, which std::make_exception_ptr might use
 // (see libcxx/include/__exception/exception_ptr.h).
 #define _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION _LIBCPP_INTRODUCED_IN_LLVM_18
-#define _LIBCPP_AVAILABILITY_INIT_PRIMARY_EXCEPTION _LIBCPP_INTRODUCED_IN_LLVM_18_MARKUP
+#define _LIBCPP_AVAILABILITY_INIT_PRIMARY_EXCEPTION _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE
 
 // This controls the availability of C++23 <print>, which
 // has a dependency on the built library (it needs access to
 // the underlying buffer types of std::cout, std::cerr, and std::clog.
 #define _LIBCPP_AVAILABILITY_HAS_PRINT _LIBCPP_INTRODUCED_IN_LLVM_18
-#define _LIBCPP_AVAILABILITY_PRINT _LIBCPP_INTRODUCED_IN_LLVM_18_MARKUP
+#define _LIBCPP_AVAILABILITY_PRINT _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE
 
 // This controls the availability of the C++20 time zone database.
 // The parser code is built in the library.
 #define _LIBCPP_AVAILABILITY_HAS_TZDB _LIBCPP_INTRODUCED_IN_LLVM_19
-#define _LIBCPP_AVAILABILITY_TZDB _LIBCPP_INTRODUCED_IN_LLVM_19_MARKUP
+#define _LIBCPP_AVAILABILITY_TZDB _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE
 
 // These macros determine whether we assume that std::bad_function_call and
 // std::bad_expected_access provide a key function in the dylib. This allows
 // centralizing their vtable and typeinfo instead of having all TUs provide
 // a weak definition that then gets deduplicated.
-#  define _LIBCPP_AVAILABILITY_HAS_BAD_FUNCTION_CALL_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19
-#  define _LIBCPP_AVAILABILITY_BAD_FUNCTION_CALL_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19_MARKUP
-#  define _LIBCPP_AVAILABILITY_HAS_BAD_EXPECTED_ACCESS_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19
-#  define _LIBCPP_AVAILABILITY_BAD_EXPECTED_ACCESS_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19_MARKUP
+#define _LIBCPP_AVAILABILITY_HAS_BAD_FUNCTION_CALL_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19
+#define _LIBCPP_AVAILABILITY_BAD_FUNCTION_CALL_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE
+#define _LIBCPP_AVAILABILITY_HAS_BAD_EXPECTED_ACCESS_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19
+#define _LIBCPP_AVAILABILITY_BAD_EXPECTED_ACCESS_KEY_FUNCTION _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE
 
 // Define availability attributes that depend on _LIBCPP_HAS_NO_EXCEPTIONS.
 // Those are defined in terms of the availability attributes above, and
-- 
GitLab


From 27becf0c3c1e7ac4a2f2e848b44d872f1aa1db9a Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Tue, 7 May 2024 15:20:43 +0200
Subject: [PATCH 0044/1206] [clang] CTAD: fix the aggregate deduction guide for
 alias templates. (#90894)

For alias templates, our current way of constructing their aggregate
deduction guides deviates from the standard approach. We should align it
with how we handle implicit deduction guides.

This patch has a refactoring change which pulls the construction logic
out from `DeclareImplicitDeductionGuidesForTypeAlia` and reusing it for
building aggregate deduction guides.
---
 clang/lib/Sema/SemaTemplate.cpp              | 452 +++++++++----------
 clang/test/SemaCXX/cxx20-ctad-type-alias.cpp |  14 +
 clang/test/SemaTemplate/deduction-guide.cpp  |   7 +
 3 files changed, 226 insertions(+), 247 deletions(-)

diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index e647ac267ab3..5c72270ff150 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -2803,7 +2803,207 @@ getRHSTemplateDeclAndArgs(Sema &SemaRef, TypeAliasTemplateDecl *AliasTemplate) {
   return {Template, AliasRhsTemplateArgs};
 }
 
-// Build deduction guides for a type alias template.
+// Build deduction guides for a type alias template from the given underlying
+// deduction guide F.
+FunctionTemplateDecl *
+BuildDeductionGuideForTypeAlias(Sema &SemaRef,
+                                TypeAliasTemplateDecl *AliasTemplate,
+                                FunctionTemplateDecl *F, SourceLocation Loc) {
+  LocalInstantiationScope Scope(SemaRef);
+  Sema::InstantiatingTemplate BuildingDeductionGuides(
+      SemaRef, AliasTemplate->getLocation(), F,
+      Sema::InstantiatingTemplate::BuildingDeductionGuidesTag{});
+  if (BuildingDeductionGuides.isInvalid())
+    return nullptr;
+
+  auto &Context = SemaRef.Context;
+  auto [Template, AliasRhsTemplateArgs] =
+      getRHSTemplateDeclAndArgs(SemaRef, AliasTemplate);
+
+  auto RType = F->getTemplatedDecl()->getReturnType();
+  // The (trailing) return type of the deduction guide.
+  const TemplateSpecializationType *FReturnType =
+      RType->getAs<TemplateSpecializationType>();
+  if (const auto *InjectedCNT = RType->getAs<InjectedClassNameType>())
+    // implicitly-generated deduction guide.
+    FReturnType = InjectedCNT->getInjectedTST();
+  else if (const auto *ET = RType->getAs<ElaboratedType>())
+    // explicit deduction guide.
+    FReturnType = ET->getNamedType()->getAs<TemplateSpecializationType>();
+  assert(FReturnType && "expected to see a return type");
+  // Deduce template arguments of the deduction guide f from the RHS of
+  // the alias.
+  //
+  // C++ [over.match.class.deduct]p3: ...For each function or function
+  // template f in the guides of the template named by the
+  // simple-template-id of the defining-type-id, the template arguments
+  // of the return type of f are deduced from the defining-type-id of A
+  // according to the process in [temp.deduct.type] with the exception
+  // that deduction does not fail if not all template arguments are
+  // deduced.
+  //
+  //
+  //  template<typename X, typename Y>
+  //  f(X, Y) -> f<Y, X>;
+  //
+  //  template<typename U>
+  //  using alias = f<int, U>;
+  //
+  // The RHS of alias is f<int, U>, we deduced the template arguments of
+  // the return type of the deduction guide from it: Y->int, X->U
+  sema::TemplateDeductionInfo TDeduceInfo(Loc);
+  // Must initialize n elements, this is required by DeduceTemplateArguments.
+  SmallVector<DeducedTemplateArgument> DeduceResults(
+      F->getTemplateParameters()->size());
+
+  // FIXME: DeduceTemplateArguments stops immediately at the first
+  // non-deducible template argument. However, this doesn't seem to casue
+  // issues for practice cases, we probably need to extend it to continue
+  // performing deduction for rest of arguments to align with the C++
+  // standard.
+  SemaRef.DeduceTemplateArguments(
+      F->getTemplateParameters(), FReturnType->template_arguments(),
+      AliasRhsTemplateArgs, TDeduceInfo, DeduceResults,
+      /*NumberOfArgumentsMustMatch=*/false);
+
+  SmallVector<TemplateArgument> DeducedArgs;
+  SmallVector<unsigned> NonDeducedTemplateParamsInFIndex;
+  // !!NOTE: DeduceResults respects the sequence of template parameters of
+  // the deduction guide f.
+  for (unsigned Index = 0; Index < DeduceResults.size(); ++Index) {
+    if (const auto &D = DeduceResults[Index]; !D.isNull()) // Deduced
+      DeducedArgs.push_back(D);
+    else
+      NonDeducedTemplateParamsInFIndex.push_back(Index);
+  }
+  auto DeducedAliasTemplateParams =
+      TemplateParamsReferencedInTemplateArgumentList(
+          AliasTemplate->getTemplateParameters()->asArray(), DeducedArgs);
+  // All template arguments null by default.
+  SmallVector<TemplateArgument> TemplateArgsForBuildingFPrime(
+      F->getTemplateParameters()->size());
+
+  // Create a template parameter list for the synthesized deduction guide f'.
+  //
+  // C++ [over.match.class.deduct]p3.2:
+  //   If f is a function template, f' is a function template whose template
+  //   parameter list consists of all the template parameters of A
+  //   (including their default template arguments) that appear in the above
+  //   deductions or (recursively) in their default template arguments
+  SmallVector<NamedDecl *> FPrimeTemplateParams;
+  // Store template arguments that refer to the newly-created template
+  // parameters, used for building `TemplateArgsForBuildingFPrime`.
+  SmallVector<TemplateArgument, 16> TransformedDeducedAliasArgs(
+      AliasTemplate->getTemplateParameters()->size());
+
+  for (unsigned AliasTemplateParamIdx : DeducedAliasTemplateParams) {
+    auto *TP =
+        AliasTemplate->getTemplateParameters()->getParam(AliasTemplateParamIdx);
+    // Rebuild any internal references to earlier parameters and reindex as
+    // we go.
+    MultiLevelTemplateArgumentList Args;
+    Args.setKind(TemplateSubstitutionKind::Rewrite);
+    Args.addOuterTemplateArguments(TransformedDeducedAliasArgs);
+    NamedDecl *NewParam = transformTemplateParameter(
+        SemaRef, AliasTemplate->getDeclContext(), TP, Args,
+        /*NewIndex=*/FPrimeTemplateParams.size());
+    FPrimeTemplateParams.push_back(NewParam);
+
+    auto NewTemplateArgument = Context.getCanonicalTemplateArgument(
+        Context.getInjectedTemplateArg(NewParam));
+    TransformedDeducedAliasArgs[AliasTemplateParamIdx] = NewTemplateArgument;
+  }
+  //   ...followed by the template parameters of f that were not deduced
+  //   (including their default template arguments)
+  for (unsigned FTemplateParamIdx : NonDeducedTemplateParamsInFIndex) {
+    auto *TP = F->getTemplateParameters()->getParam(FTemplateParamIdx);
+    MultiLevelTemplateArgumentList Args;
+    Args.setKind(TemplateSubstitutionKind::Rewrite);
+    // We take a shortcut here, it is ok to reuse the
+    // TemplateArgsForBuildingFPrime.
+    Args.addOuterTemplateArguments(TemplateArgsForBuildingFPrime);
+    NamedDecl *NewParam = transformTemplateParameter(
+        SemaRef, F->getDeclContext(), TP, Args, FPrimeTemplateParams.size());
+    FPrimeTemplateParams.push_back(NewParam);
+
+    assert(TemplateArgsForBuildingFPrime[FTemplateParamIdx].isNull() &&
+           "The argument must be null before setting");
+    TemplateArgsForBuildingFPrime[FTemplateParamIdx] =
+        Context.getCanonicalTemplateArgument(
+            Context.getInjectedTemplateArg(NewParam));
+  }
+
+  // To form a deduction guide f' from f, we leverage clang's instantiation
+  // mechanism, we construct a template argument list where the template
+  // arguments refer to the newly-created template parameters of f', and
+  // then apply instantiation on this template argument list to instantiate
+  // f, this ensures all template parameter occurrences are updated
+  // correctly.
+  //
+  // The template argument list is formed from the `DeducedArgs`, two parts:
+  //  1) appeared template parameters of alias: transfrom the deduced
+  //  template argument;
+  //  2) non-deduced template parameters of f: rebuild a
+  //  template argument;
+  //
+  // 2) has been built already (when rebuilding the new template
+  // parameters), we now perform 1).
+  MultiLevelTemplateArgumentList Args;
+  Args.setKind(TemplateSubstitutionKind::Rewrite);
+  Args.addOuterTemplateArguments(TransformedDeducedAliasArgs);
+  for (unsigned Index = 0; Index < DeduceResults.size(); ++Index) {
+    const auto &D = DeduceResults[Index];
+    if (D.isNull()) {
+      // 2): Non-deduced template parameter has been built already.
+      assert(!TemplateArgsForBuildingFPrime[Index].isNull() &&
+             "template arguments for non-deduced template parameters should "
+             "be been set!");
+      continue;
+    }
+    TemplateArgumentLoc Input =
+        SemaRef.getTrivialTemplateArgumentLoc(D, QualType(), SourceLocation{});
+    TemplateArgumentLoc Output;
+    if (!SemaRef.SubstTemplateArgument(Input, Args, Output)) {
+      assert(TemplateArgsForBuildingFPrime[Index].isNull() &&
+             "InstantiatedArgs must be null before setting");
+      TemplateArgsForBuildingFPrime[Index] = Output.getArgument();
+    }
+  }
+
+  auto *TemplateArgListForBuildingFPrime =
+      TemplateArgumentList::CreateCopy(Context, TemplateArgsForBuildingFPrime);
+  // Form the f' by substituting the template arguments into f.
+  if (auto *FPrime = SemaRef.InstantiateFunctionDeclaration(
+          F, TemplateArgListForBuildingFPrime, AliasTemplate->getLocation(),
+          Sema::CodeSynthesisContext::BuildingDeductionGuides)) {
+    auto *GG = cast<CXXDeductionGuideDecl>(FPrime);
+
+    Expr *RequiresClause =
+        transformRequireClause(SemaRef, F, TemplateArgsForBuildingFPrime);
+
+    // FIXME: implement the is_deducible constraint per C++
+    // [over.match.class.deduct]p3.3:
+    //    ... and a constraint that is satisfied if and only if the arguments
+    //    of A are deducible (see below) from the return type.
+    auto *FPrimeTemplateParamList = TemplateParameterList::Create(
+        Context, AliasTemplate->getTemplateParameters()->getTemplateLoc(),
+        AliasTemplate->getTemplateParameters()->getLAngleLoc(),
+        FPrimeTemplateParams,
+        AliasTemplate->getTemplateParameters()->getRAngleLoc(),
+        /*RequiresClause=*/RequiresClause);
+    FunctionTemplateDecl *Result = buildDeductionGuide(
+        SemaRef, AliasTemplate, FPrimeTemplateParamList,
+        GG->getCorrespondingConstructor(), GG->getExplicitSpecifier(),
+        GG->getTypeSourceInfo(), AliasTemplate->getBeginLoc(),
+        AliasTemplate->getLocation(), AliasTemplate->getEndLoc(),
+        F->isImplicit());
+    cast<CXXDeductionGuideDecl>(Result->getTemplatedDecl())
+        ->setDeductionCandidateKind(GG->getDeductionCandidateKind());
+    return Result;
+  }
+  return nullptr;
+}
+
 void DeclareImplicitDeductionGuidesForTypeAlias(
     Sema &SemaRef, TypeAliasTemplateDecl *AliasTemplate, SourceLocation Loc) {
   if (AliasTemplate->isInvalidDecl())
@@ -2831,197 +3031,13 @@ void DeclareImplicitDeductionGuidesForTypeAlias(
     if (!F)
       continue;
     // The **aggregate** deduction guides are handled in a different code path
-    // (DeclareImplicitDeductionGuideFromInitList), which involves the tricky
+    // (DeclareAggregateDeductionGuideFromInitList), which involves the tricky
     // cache.
     if (cast<CXXDeductionGuideDecl>(F->getTemplatedDecl())
             ->getDeductionCandidateKind() == DeductionCandidate::Aggregate)
       continue;
 
-    auto RType = F->getTemplatedDecl()->getReturnType();
-    // The (trailing) return type of the deduction guide.
-    const TemplateSpecializationType *FReturnType =
-        RType->getAs<TemplateSpecializationType>();
-    if (const auto *InjectedCNT = RType->getAs<InjectedClassNameType>())
-      // implicitly-generated deduction guide.
-      FReturnType = InjectedCNT->getInjectedTST();
-    else if (const auto *ET = RType->getAs<ElaboratedType>())
-      // explicit deduction guide.
-      FReturnType = ET->getNamedType()->getAs<TemplateSpecializationType>();
-    assert(FReturnType && "expected to see a return type");
-    // Deduce template arguments of the deduction guide f from the RHS of
-    // the alias.
-    //
-    // C++ [over.match.class.deduct]p3: ...For each function or function
-    // template f in the guides of the template named by the
-    // simple-template-id of the defining-type-id, the template arguments
-    // of the return type of f are deduced from the defining-type-id of A
-    // according to the process in [temp.deduct.type] with the exception
-    // that deduction does not fail if not all template arguments are
-    // deduced.
-    //
-    //
-    //  template<typename X, typename Y>
-    //  f(X, Y) -> f<Y, X>;
-    //
-    //  template<typename U>
-    //  using alias = f<int, U>;
-    //
-    // The RHS of alias is f<int, U>, we deduced the template arguments of
-    // the return type of the deduction guide from it: Y->int, X->U
-    sema::TemplateDeductionInfo TDeduceInfo(Loc);
-    // Must initialize n elements, this is required by DeduceTemplateArguments.
-    SmallVector<DeducedTemplateArgument> DeduceResults(
-        F->getTemplateParameters()->size());
-
-    // FIXME: DeduceTemplateArguments stops immediately at the first
-    // non-deducible template argument. However, this doesn't seem to casue
-    // issues for practice cases, we probably need to extend it to continue
-    // performing deduction for rest of arguments to align with the C++
-    // standard.
-    SemaRef.DeduceTemplateArguments(
-        F->getTemplateParameters(), FReturnType->template_arguments(),
-        AliasRhsTemplateArgs, TDeduceInfo, DeduceResults,
-        /*NumberOfArgumentsMustMatch=*/false);
-
-    SmallVector<TemplateArgument> DeducedArgs;
-    SmallVector<unsigned> NonDeducedTemplateParamsInFIndex;
-    // !!NOTE: DeduceResults respects the sequence of template parameters of
-    // the deduction guide f.
-    for (unsigned Index = 0; Index < DeduceResults.size(); ++Index) {
-      if (const auto &D = DeduceResults[Index]; !D.isNull()) // Deduced
-        DeducedArgs.push_back(D);
-      else
-        NonDeducedTemplateParamsInFIndex.push_back(Index);
-    }
-    auto DeducedAliasTemplateParams =
-        TemplateParamsReferencedInTemplateArgumentList(
-            AliasTemplate->getTemplateParameters()->asArray(), DeducedArgs);
-    // All template arguments null by default.
-    SmallVector<TemplateArgument> TemplateArgsForBuildingFPrime(
-        F->getTemplateParameters()->size());
-
-    Sema::InstantiatingTemplate BuildingDeductionGuides(
-        SemaRef, AliasTemplate->getLocation(), F,
-        Sema::InstantiatingTemplate::BuildingDeductionGuidesTag{});
-    if (BuildingDeductionGuides.isInvalid())
-      return;
-    LocalInstantiationScope Scope(SemaRef);
-
-    // Create a template parameter list for the synthesized deduction guide f'.
-    //
-    // C++ [over.match.class.deduct]p3.2:
-    //   If f is a function template, f' is a function template whose template
-    //   parameter list consists of all the template parameters of A
-    //   (including their default template arguments) that appear in the above
-    //   deductions or (recursively) in their default template arguments
-    SmallVector<NamedDecl *> FPrimeTemplateParams;
-    // Store template arguments that refer to the newly-created template
-    // parameters, used for building `TemplateArgsForBuildingFPrime`.
-    SmallVector<TemplateArgument, 16> TransformedDeducedAliasArgs(
-        AliasTemplate->getTemplateParameters()->size());
-
-    for (unsigned AliasTemplateParamIdx : DeducedAliasTemplateParams) {
-      auto *TP = AliasTemplate->getTemplateParameters()->getParam(
-          AliasTemplateParamIdx);
-      // Rebuild any internal references to earlier parameters and reindex as
-      // we go.
-      MultiLevelTemplateArgumentList Args;
-      Args.setKind(TemplateSubstitutionKind::Rewrite);
-      Args.addOuterTemplateArguments(TransformedDeducedAliasArgs);
-      NamedDecl *NewParam = transformTemplateParameter(
-          SemaRef, AliasTemplate->getDeclContext(), TP, Args,
-          /*NewIndex*/ FPrimeTemplateParams.size());
-      FPrimeTemplateParams.push_back(NewParam);
-
-      auto NewTemplateArgument = Context.getCanonicalTemplateArgument(
-          Context.getInjectedTemplateArg(NewParam));
-      TransformedDeducedAliasArgs[AliasTemplateParamIdx] = NewTemplateArgument;
-    }
-    //   ...followed by the template parameters of f that were not deduced
-    //   (including their default template arguments)
-    for (unsigned FTemplateParamIdx : NonDeducedTemplateParamsInFIndex) {
-      auto *TP = F->getTemplateParameters()->getParam(FTemplateParamIdx);
-      MultiLevelTemplateArgumentList Args;
-      Args.setKind(TemplateSubstitutionKind::Rewrite);
-      // We take a shortcut here, it is ok to reuse the
-      // TemplateArgsForBuildingFPrime.
-      Args.addOuterTemplateArguments(TemplateArgsForBuildingFPrime);
-      NamedDecl *NewParam = transformTemplateParameter(
-          SemaRef, F->getDeclContext(), TP, Args, FPrimeTemplateParams.size());
-      FPrimeTemplateParams.push_back(NewParam);
-
-      assert(TemplateArgsForBuildingFPrime[FTemplateParamIdx].isNull() &&
-             "The argument must be null before setting");
-      TemplateArgsForBuildingFPrime[FTemplateParamIdx] =
-          Context.getCanonicalTemplateArgument(
-              Context.getInjectedTemplateArg(NewParam));
-    }
-
-    // To form a deduction guide f' from f, we leverage clang's instantiation
-    // mechanism, we construct a template argument list where the template
-    // arguments refer to the newly-created template parameters of f', and
-    // then apply instantiation on this template argument list to instantiate
-    // f, this ensures all template parameter occurrences are updated
-    // correctly.
-    //
-    // The template argument list is formed from the `DeducedArgs`, two parts:
-    //  1) appeared template parameters of alias: transfrom the deduced
-    //  template argument;
-    //  2) non-deduced template parameters of f: rebuild a
-    //  template argument;
-    //
-    // 2) has been built already (when rebuilding the new template
-    // parameters), we now perform 1).
-    MultiLevelTemplateArgumentList Args;
-    Args.setKind(TemplateSubstitutionKind::Rewrite);
-    Args.addOuterTemplateArguments(TransformedDeducedAliasArgs);
-    for (unsigned Index = 0; Index < DeduceResults.size(); ++Index) {
-      const auto &D = DeduceResults[Index];
-      if (D.isNull()) {
-        // 2): Non-deduced template parameter has been built already.
-        assert(!TemplateArgsForBuildingFPrime[Index].isNull() &&
-               "template arguments for non-deduced template parameters should "
-               "be been set!");
-        continue;
-      }
-      TemplateArgumentLoc Input = SemaRef.getTrivialTemplateArgumentLoc(
-          D, QualType(), SourceLocation{});
-      TemplateArgumentLoc Output;
-      if (!SemaRef.SubstTemplateArgument(Input, Args, Output)) {
-        assert(TemplateArgsForBuildingFPrime[Index].isNull() &&
-               "InstantiatedArgs must be null before setting");
-        TemplateArgsForBuildingFPrime[Index] = (Output.getArgument());
-      }
-    }
-
-    auto *TemplateArgListForBuildingFPrime = TemplateArgumentList::CreateCopy(
-        Context, TemplateArgsForBuildingFPrime);
-    // Form the f' by substituting the template arguments into f.
-    if (auto *FPrime = SemaRef.InstantiateFunctionDeclaration(
-            F, TemplateArgListForBuildingFPrime, AliasTemplate->getLocation(),
-            Sema::CodeSynthesisContext::BuildingDeductionGuides)) {
-      auto *GG = cast<CXXDeductionGuideDecl>(FPrime);
-      // Substitute new template parameters into requires-clause if present.
-      Expr *RequiresClause =
-          transformRequireClause(SemaRef, F, TemplateArgsForBuildingFPrime);
-      // FIXME: implement the is_deducible constraint per C++
-      // [over.match.class.deduct]p3.3:
-      //    ... and a constraint that is satisfied if and only if the arguments
-      //    of A are deducible (see below) from the return type.
-      auto *FPrimeTemplateParamList = TemplateParameterList::Create(
-          Context, AliasTemplate->getTemplateParameters()->getTemplateLoc(),
-          AliasTemplate->getTemplateParameters()->getLAngleLoc(),
-          FPrimeTemplateParams,
-          AliasTemplate->getTemplateParameters()->getRAngleLoc(),
-          /*RequiresClause=*/RequiresClause);
-
-      buildDeductionGuide(SemaRef, AliasTemplate, FPrimeTemplateParamList,
-                          GG->getCorrespondingConstructor(),
-                          GG->getExplicitSpecifier(), GG->getTypeSourceInfo(),
-                          AliasTemplate->getBeginLoc(),
-                          AliasTemplate->getLocation(),
-                          AliasTemplate->getEndLoc(), F->isImplicit());
-    }
+    BuildDeductionGuideForTypeAlias(SemaRef, AliasTemplate, F, Loc);
   }
 }
 
@@ -3037,66 +3053,8 @@ FunctionTemplateDecl *DeclareAggregateDeductionGuideForTypeAlias(
       RHSTemplate, ParamTypes, Loc);
   if (!RHSDeductionGuide)
     return nullptr;
-
-  LocalInstantiationScope Scope(SemaRef);
-  Sema::InstantiatingTemplate BuildingDeductionGuides(
-      SemaRef, AliasTemplate->getLocation(), RHSDeductionGuide,
-      Sema::InstantiatingTemplate::BuildingDeductionGuidesTag{});
-  if (BuildingDeductionGuides.isInvalid())
-    return nullptr;
-
-  // Build a new template parameter list for the synthesized aggregate deduction
-  // guide by transforming the one from RHSDeductionGuide.
-  SmallVector<NamedDecl *> TransformedTemplateParams;
-  // Template args that refer to the rebuilt template parameters.
-  // All template arguments must be initialized in advance.
-  SmallVector<TemplateArgument> TransformedTemplateArgs(
-      RHSDeductionGuide->getTemplateParameters()->size());
-  for (auto *TP : *RHSDeductionGuide->getTemplateParameters()) {
-    // Rebuild any internal references to earlier parameters and reindex as
-    // we go.
-    MultiLevelTemplateArgumentList Args;
-    Args.setKind(TemplateSubstitutionKind::Rewrite);
-    Args.addOuterTemplateArguments(TransformedTemplateArgs);
-    NamedDecl *NewParam = transformTemplateParameter(
-        SemaRef, AliasTemplate->getDeclContext(), TP, Args,
-        /*NewIndex=*/TransformedTemplateParams.size());
-
-    TransformedTemplateArgs[TransformedTemplateParams.size()] =
-        SemaRef.Context.getCanonicalTemplateArgument(
-            SemaRef.Context.getInjectedTemplateArg(NewParam));
-    TransformedTemplateParams.push_back(NewParam);
-  }
-  // FIXME: implement the is_deducible constraint per C++
-  // [over.match.class.deduct]p3.3.
-  Expr *TransformedRequiresClause = transformRequireClause(
-      SemaRef, RHSDeductionGuide, TransformedTemplateArgs);
-  auto *TransformedTemplateParameterList = TemplateParameterList::Create(
-      SemaRef.Context, AliasTemplate->getTemplateParameters()->getTemplateLoc(),
-      AliasTemplate->getTemplateParameters()->getLAngleLoc(),
-      TransformedTemplateParams,
-      AliasTemplate->getTemplateParameters()->getRAngleLoc(),
-      TransformedRequiresClause);
-  auto *TransformedTemplateArgList = TemplateArgumentList::CreateCopy(
-      SemaRef.Context, TransformedTemplateArgs);
-
-  if (auto *TransformedDeductionGuide = SemaRef.InstantiateFunctionDeclaration(
-          RHSDeductionGuide, TransformedTemplateArgList,
-          AliasTemplate->getLocation(),
-          Sema::CodeSynthesisContext::BuildingDeductionGuides)) {
-    auto *GD =
-        llvm::dyn_cast<clang::CXXDeductionGuideDecl>(TransformedDeductionGuide);
-    FunctionTemplateDecl *Result = buildDeductionGuide(
-        SemaRef, AliasTemplate, TransformedTemplateParameterList,
-        GD->getCorrespondingConstructor(), GD->getExplicitSpecifier(),
-        GD->getTypeSourceInfo(), AliasTemplate->getBeginLoc(),
-        AliasTemplate->getLocation(), AliasTemplate->getEndLoc(),
-        GD->isImplicit());
-    cast<CXXDeductionGuideDecl>(Result->getTemplatedDecl())
-        ->setDeductionCandidateKind(DeductionCandidate::Aggregate);
-    return Result;
-  }
-  return nullptr;
+  return BuildDeductionGuideForTypeAlias(SemaRef, AliasTemplate,
+                                         RHSDeductionGuide, Loc);
 }
 
 } // namespace
diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
index 508a3a5da76a..e8b4383f53c5 100644
--- a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
+++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
@@ -307,3 +307,17 @@ using AFoo = Foo<int, Derived<U>>;
 
 AFoo a(Derived<int>{});
 } // namespace test22
+
+namespace test23 {
+// We have an aggregate deduction guide "G(T) -> G<T>".
+template<typename T>
+struct G { T t1; };
+
+template<typename X = int>
+using AG = G<int>;
+
+AG ag(1.0);
+// Verify that the aggregate deduction guide "AG(int) -> AG<int>" is built and
+// choosen.
+static_assert(__is_same(decltype(ag.t1), int));
+} // namespace test23
diff --git a/clang/test/SemaTemplate/deduction-guide.cpp b/clang/test/SemaTemplate/deduction-guide.cpp
index ff5e39216762..51e1eb49c5de 100644
--- a/clang/test/SemaTemplate/deduction-guide.cpp
+++ b/clang/test/SemaTemplate/deduction-guide.cpp
@@ -261,6 +261,13 @@ AG ag = {1};
 // CHECK:   | `-BuiltinType {{.*}} 'int'
 // CHECK:   `-ParmVarDecl {{.*}} 'int'
 
+template <typename X = int>
+using BG = G<int>;
+BG bg(1.0);
+// CHECK-LABEL: Dumping <deduction guide for BG>
+// CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for BG>
+// CHECK: |-CXXDeductionGuideDecl {{.*}} 'auto (int) -> G<int>' aggregate
+
 template <typename D>
 requires (sizeof(D) == 4)
 struct Foo {
-- 
GitLab


From 227fe1c1995dea1850483449e8510db2726bcbee Mon Sep 17 00:00:00 2001
From: David Truby <david.truby@arm.com>
Date: Tue, 7 May 2024 14:27:39 +0100
Subject: [PATCH 0045/1206] [flang] Remove C++ runtime dependency from Sleep
 extension (#84911)

The Sleep extension currently has a potential dependency on the C++
runtime. I run into this dependency using libc++ on Linux. This patch
uses the POSIX `sleep` function or the Windows `Sleep` function
instead to avoid this dependency.
---
 flang/runtime/extensions.cpp | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp
index 4b110cc10c84..be3833db88b0 100644
--- a/flang/runtime/extensions.cpp
+++ b/flang/runtime/extensions.cpp
@@ -23,6 +23,12 @@
 #include <thread>
 
 #ifdef _WIN32
+#define WIN32_LEAN_AND_MEAN
+#define NOMINMAX
+#include <windows.h>
+
+#include <synchapi.h>
+
 inline void CtimeBuffer(char *buffer, size_t bufsize, const time_t cur_time,
     Fortran::runtime::Terminator terminator) {
   int error{ctime_s(buffer, bufsize, &cur_time)};
@@ -136,7 +142,11 @@ void RTNAME(Sleep)(std::int64_t seconds) {
   if (seconds < 1) {
     return;
   }
-  std::this_thread::sleep_for(std::chrono::seconds(seconds));
+#if _WIN32
+  Sleep(seconds * 1000);
+#else
+  sleep(seconds);
+#endif
 }
 
 // TODO: not supported on Windows
-- 
GitLab


From 1d87465a0a95cee9accc5dce7abdabbbc3f3c122 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Tue, 7 May 2024 15:25:17 +0200
Subject: [PATCH 0046/1206] [libc][math] fmod: clear exceptions before the test
 instead of after

The test has no control over the CPU state before the test runs.

This test checks whether no exception flags are set, which may not be
true at the start of the test. This used to be not a problem because the
check was broken but that was fixed in ecfb5d9951554d8bdb6a499c958f48cc35f78a88
---
 libc/test/src/math/FModTest.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libc/test/src/math/FModTest.h b/libc/test/src/math/FModTest.h
index f1015d6497fc..32c009ab8828 100644
--- a/libc/test/src/math/FModTest.h
+++ b/libc/test/src/math/FModTest.h
@@ -18,10 +18,10 @@
 #include "hdr/math_macros.h"
 
 #define TEST_SPECIAL(x, y, expected, dom_err, expected_exception)              \
+  LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT);                         \
   EXPECT_FP_EQ(expected, f(x, y));                                             \
   EXPECT_MATH_ERRNO((dom_err) ? EDOM : 0);                                     \
-  EXPECT_FP_EXCEPTION(expected_exception);                                     \
-  LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT)
+  EXPECT_FP_EXCEPTION(expected_exception)
 
 #define TEST_REGULAR(x, y, expected) TEST_SPECIAL(x, y, expected, false, 0)
 
-- 
GitLab


From 41ca9104ac1e0bf248d4082f45c5ad03ddd55727 Mon Sep 17 00:00:00 2001
From: Jonathan Peyton <jonathan.l.peyton@intel.com>
Date: Tue, 7 May 2024 08:41:51 -0500
Subject: [PATCH 0047/1206] [OpenMP] Fix task state and taskteams for serial
 teams (#86859)

* Serial teams now use a stack (similar to dispatch buffers)
* Serial teams always use `t_task_team[0]` as the task team and the
second pointer is a next pointer for the stack

`t_task_team[1]` is interpreted as a stack of task teams where each
level is a nested level

```
 inner serial team                   outer serial team
[ t_task_team[0] ] -> (task_team)    [ t_task_team[0] ] -> (task_team)
[ next           ] ----------------> [ next           ] -> ...
```

* Remove the task state memo stack from thread structure.
* Instead of a thread-private stack, use team structure to store
th_task_state of the primary thread. When coming out of a parallel,
restore the primary thread's task state. The new field in the team
structure doesn't cause sizeof(team) to change and is in the cache line
which is only read/written by the primary thread.

Fixes: #50602
Fixes: #69368
Fixes: #69733
Fixes: #79416
---
 openmp/runtime/src/kmp.h                      |  29 +-
 openmp/runtime/src/kmp_barrier.cpp            |  15 +-
 openmp/runtime/src/kmp_csupport.cpp           |  11 +
 openmp/runtime/src/kmp_runtime.cpp            | 179 ++++------
 openmp/runtime/src/kmp_tasking.cpp            | 193 +++++------
 openmp/runtime/test/target/issue-81488.c      |  36 ++
 openmp/runtime/test/tasking/issue-50602.c     |  40 +++
 openmp/runtime/test/tasking/issue-69368.c     |  27 ++
 openmp/runtime/test/tasking/issue-69733.c     | 147 ++++++++
 openmp/runtime/test/tasking/issue-79416.c     |  33 ++
 .../test/tasking/task_teams_stress_test.cpp   | 318 ++++++++++++++++++
 11 files changed, 792 insertions(+), 236 deletions(-)
 create mode 100644 openmp/runtime/test/target/issue-81488.c
 create mode 100644 openmp/runtime/test/tasking/issue-50602.c
 create mode 100644 openmp/runtime/test/tasking/issue-69368.c
 create mode 100644 openmp/runtime/test/tasking/issue-69733.c
 create mode 100644 openmp/runtime/test/tasking/issue-79416.c
 create mode 100644 openmp/runtime/test/tasking/task_teams_stress_test.cpp

diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 18ccf10fe17d..64a3ea6d5be5 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2871,6 +2871,11 @@ union KMP_ALIGN_CACHE kmp_task_team {
   char tt_pad[KMP_PAD(kmp_base_task_team_t, CACHE_LINE)];
 };
 
+typedef struct kmp_task_team_list_t {
+  kmp_task_team_t *task_team;
+  kmp_task_team_list_t *next;
+} kmp_task_team_list_t;
+
 #if (USE_FAST_MEMORY == 3) || (USE_FAST_MEMORY == 5)
 // Free lists keep same-size free memory slots for fast memory allocation
 // routines
@@ -3008,10 +3013,6 @@ typedef struct KMP_ALIGN_CACHE kmp_base_info {
   kmp_task_team_t *th_task_team; // Task team struct
   kmp_taskdata_t *th_current_task; // Innermost Task being executed
   kmp_uint8 th_task_state; // alternating 0/1 for task team identification
-  kmp_uint8 *th_task_state_memo_stack; // Stack holding memos of th_task_state
-  // at nested levels
-  kmp_uint32 th_task_state_top; // Top element of th_task_state_memo_stack
-  kmp_uint32 th_task_state_stack_sz; // Size of th_task_state_memo_stack
   kmp_uint32 th_reap_state; // Non-zero indicates thread is not
   // tasking, thus safe to reap
 
@@ -3133,6 +3134,7 @@ typedef struct KMP_ALIGN_CACHE kmp_base_team {
   kmp_disp_t *t_dispatch; // thread's dispatch data
   kmp_task_team_t *t_task_team[2]; // Task team struct; switch between 2
   kmp_proc_bind_t t_proc_bind; // bind type for par region
+  int t_primary_task_state; // primary thread's task state saved
 #if USE_ITT_BUILD
   kmp_uint64 t_region_time; // region begin timestamp
 #endif /* USE_ITT_BUILD */
@@ -3204,6 +3206,12 @@ typedef struct KMP_ALIGN_CACHE kmp_base_team {
   distributedBarrier *b; // Distributed barrier data associated with team
 } kmp_base_team_t;
 
+// Assert that the list structure fits and aligns within
+// the double task team pointer
+KMP_BUILD_ASSERT(sizeof(kmp_task_team_t *[2]) == sizeof(kmp_task_team_list_t));
+KMP_BUILD_ASSERT(alignof(kmp_task_team_t *[2]) ==
+                 alignof(kmp_task_team_list_t));
+
 union KMP_ALIGN_CACHE kmp_team {
   kmp_base_team_t t;
   double t_align; /* use worst case alignment */
@@ -4114,9 +4122,10 @@ extern void __kmp_fulfill_event(kmp_event_t *event);
 extern void __kmp_free_task_team(kmp_info_t *thread,
                                  kmp_task_team_t *task_team);
 extern void __kmp_reap_task_teams(void);
+extern void __kmp_push_task_team_node(kmp_info_t *thread, kmp_team_t *team);
+extern void __kmp_pop_task_team_node(kmp_info_t *thread, kmp_team_t *team);
 extern void __kmp_wait_to_unref_task_teams(void);
-extern void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team,
-                                  int always);
+extern void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team);
 extern void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team);
 extern void __kmp_task_team_wait(kmp_info_t *this_thr, kmp_team_t *team
 #if USE_ITT_BUILD
@@ -4127,6 +4136,14 @@ extern void __kmp_task_team_wait(kmp_info_t *this_thr, kmp_team_t *team
                                  int wait = 1);
 extern void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread,
                                   int gtid);
+#if KMP_DEBUG
+#define KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, thr)                         \
+  KMP_DEBUG_ASSERT(                                                            \
+      __kmp_tasking_mode != tskm_task_teams || team->t.t_nproc == 1 ||         \
+      thr->th.th_task_team == team->t.t_task_team[thr->th.th_task_state])
+#else
+#define KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, thr) /* Nothing */
+#endif
 
 extern int __kmp_is_address_mapped(void *addr);
 extern kmp_uint64 __kmp_hardware_timestamp(void);
diff --git a/openmp/runtime/src/kmp_barrier.cpp b/openmp/runtime/src/kmp_barrier.cpp
index e9ab15f1723b..b381694c0953 100644
--- a/openmp/runtime/src/kmp_barrier.cpp
+++ b/openmp/runtime/src/kmp_barrier.cpp
@@ -1858,8 +1858,7 @@ static int __kmp_barrier_template(enum barrier_type bt, int gtid, int is_split,
     }
 
     if (KMP_MASTER_TID(tid) && __kmp_tasking_mode != tskm_immediate_exec)
-      // use 0 to only setup the current team if nthreads > 1
-      __kmp_task_team_setup(this_thr, team, 0);
+      __kmp_task_team_setup(this_thr, team);
 
     if (cancellable) {
       cancelled = __kmp_linear_barrier_gather_cancellable(
@@ -2042,7 +2041,7 @@ static int __kmp_barrier_template(enum barrier_type bt, int gtid, int is_split,
             this_thr->th.th_task_team->tt.tt_hidden_helper_task_encountered ==
                 TRUE);
         __kmp_task_team_wait(this_thr, team USE_ITT_BUILD_ARG(itt_sync_obj));
-        __kmp_task_team_setup(this_thr, team, 0);
+        __kmp_task_team_setup(this_thr, team);
 
 #if USE_ITT_BUILD
         if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
@@ -2243,9 +2242,7 @@ void __kmp_join_barrier(int gtid) {
                   __kmp_gtid_from_thread(this_thr), team_id,
                   team->t.t_task_team[this_thr->th.th_task_state],
                   this_thr->th.th_task_team));
-    if (this_thr->th.th_task_team)
-      KMP_DEBUG_ASSERT(this_thr->th.th_task_team ==
-                       team->t.t_task_team[this_thr->th.th_task_state]);
+    KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, this_thr);
   }
 #endif /* KMP_DEBUG */
 
@@ -2440,10 +2437,8 @@ void __kmp_fork_barrier(int gtid, int tid) {
     }
 #endif
 
-    if (__kmp_tasking_mode != tskm_immediate_exec) {
-      // 0 indicates setup current task team if nthreads > 1
-      __kmp_task_team_setup(this_thr, team, 0);
-    }
+    if (__kmp_tasking_mode != tskm_immediate_exec)
+      __kmp_task_team_setup(this_thr, team);
 
     /* The primary thread may have changed its blocktime between join barrier
        and fork barrier. Copy the blocktime info to the thread, where
diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp
index 0268f692ff7f..f45fe646d1d9 100644
--- a/openmp/runtime/src/kmp_csupport.cpp
+++ b/openmp/runtime/src/kmp_csupport.cpp
@@ -654,6 +654,12 @@ void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
         serial_team->t.t_dispatch->th_disp_buffer->next;
     __kmp_free(disp_buffer);
   }
+
+  /* pop the task team stack */
+  if (serial_team->t.t_serialized > 1) {
+    __kmp_pop_task_team_node(this_thr, serial_team);
+  }
+
   this_thr->th.th_def_allocator = serial_team->t.t_def_allocator; // restore
 
   --serial_team->t.t_serialized;
@@ -692,6 +698,11 @@ void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
     this_thr->th.th_current_task->td_flags.executing = 1;
 
     if (__kmp_tasking_mode != tskm_immediate_exec) {
+      // Restore task state from serial team structure
+      KMP_DEBUG_ASSERT(serial_team->t.t_primary_task_state == 0 ||
+                       serial_team->t.t_primary_task_state == 1);
+      this_thr->th.th_task_state =
+          (kmp_uint8)serial_team->t.t_primary_task_state;
       // Copy the task team from the new child / old parent team to the thread.
       this_thr->th.th_task_team =
           this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 95acf4dff4cb..4be67f3b5987 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -1042,6 +1042,41 @@ static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
     }
   }
 
+  // Take care of primary thread's task state
+  if (__kmp_tasking_mode != tskm_immediate_exec) {
+    if (use_hot_team) {
+      KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team->t.t_parent, master_th);
+      KA_TRACE(
+          20,
+          ("__kmp_fork_team_threads: Primary T#%d pushing task_team %p / team "
+           "%p, new task_team %p / team %p\n",
+           __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,
+           team->t.t_parent, team->t.t_task_team[master_th->th.th_task_state],
+           team));
+
+      // Store primary thread's current task state on new team
+      KMP_CHECK_UPDATE(team->t.t_primary_task_state,
+                       master_th->th.th_task_state);
+
+      // Restore primary thread's task state to hot team's state
+      // by using thread 1's task state
+      if (team->t.t_nproc > 1) {
+        KMP_DEBUG_ASSERT(team->t.t_threads[1]->th.th_task_state == 0 ||
+                         team->t.t_threads[1]->th.th_task_state == 1);
+        KMP_CHECK_UPDATE(master_th->th.th_task_state,
+                         team->t.t_threads[1]->th.th_task_state);
+      } else {
+        master_th->th.th_task_state = 0;
+      }
+    } else {
+      // Store primary thread's current task_state on new team
+      KMP_CHECK_UPDATE(team->t.t_primary_task_state,
+                       master_th->th.th_task_state);
+      // Are not using hot team, so set task state to 0.
+      master_th->th.th_task_state = 0;
+    }
+  }
+
   if (__kmp_display_affinity && team->t.t_display_affinity != 1) {
     for (i = 0; i < team->t.t_nproc; i++) {
       kmp_info_t *thr = team->t.t_threads[i];
@@ -1145,18 +1180,6 @@ void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
   KMP_DEBUG_ASSERT(serial_team);
   KMP_MB();
 
-  if (__kmp_tasking_mode != tskm_immediate_exec) {
-    KMP_DEBUG_ASSERT(
-        this_thr->th.th_task_team ==
-        this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]);
-    KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==
-                     NULL);
-    KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
-                  "team %p, new task_team = NULL\n",
-                  global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
-    this_thr->th.th_task_team = NULL;
-  }
-
   kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
   if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
     proc_bind = proc_bind_false;
@@ -1242,6 +1265,8 @@ void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
     serial_team->t.t_serialized = 1;
     serial_team->t.t_nproc = 1;
     serial_team->t.t_parent = this_thr->th.th_team;
+    // Save previous team's task state on serial team structure
+    serial_team->t.t_primary_task_state = this_thr->th.th_task_state;
     serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
     this_thr->th.th_team = serial_team;
     serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
@@ -1281,6 +1306,8 @@ void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
     this_thr->th.th_team_nproc = 1;
     this_thr->th.th_team_master = this_thr;
     this_thr->th.th_team_serialized = 1;
+    this_thr->th.th_task_team = NULL;
+    this_thr->th.th_task_state = 0;
 
     serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
     serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
@@ -1332,6 +1359,9 @@ void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
     }
     this_thr->th.th_dispatch = serial_team->t.t_dispatch;
 
+    /* allocate/push task team stack */
+    __kmp_push_task_team_node(this_thr, serial_team);
+
     KMP_MB();
   }
   KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq);
@@ -1985,17 +2015,12 @@ int __kmp_fork_call(ident_t *loc, int gtid,
                                  ap);
     } // End parallel closely nested in teams construct
 
-#if KMP_DEBUG
-    if (__kmp_tasking_mode != tskm_immediate_exec) {
-      KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
-                       parent_team->t.t_task_team[master_th->th.th_task_state]);
-    }
-#endif
-
     // Need this to happen before we determine the number of threads, not while
     // we are allocating the team
     //__kmp_push_current_task_to_thread(master_th, parent_team, 0);
 
+    KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(parent_team, master_th);
+
     // Determine the number of threads
     int enter_teams =
         __kmp_is_entering_teams(active_level, level, teams_level, ap);
@@ -2186,64 +2211,6 @@ int __kmp_fork_call(ident_t *loc, int gtid,
       ompd_bp_parallel_begin();
 #endif
 
-    if (__kmp_tasking_mode != tskm_immediate_exec) {
-      // Set primary thread's task team to team's task team. Unless this is hot
-      // team, it should be NULL.
-      KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
-                       parent_team->t.t_task_team[master_th->th.th_task_state]);
-      KA_TRACE(20, ("__kmp_fork_call: Primary T#%d pushing task_team %p / team "
-                    "%p, new task_team %p / team %p\n",
-                    __kmp_gtid_from_thread(master_th),
-                    master_th->th.th_task_team, parent_team,
-                    team->t.t_task_team[master_th->th.th_task_state], team));
-
-      if (active_level || master_th->th.th_task_team) {
-        // Take a memo of primary thread's task_state
-        KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
-        if (master_th->th.th_task_state_top >=
-            master_th->th.th_task_state_stack_sz) { // increase size
-          kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
-          kmp_uint8 *old_stack, *new_stack;
-          kmp_uint32 i;
-          new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
-          for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
-            new_stack[i] = master_th->th.th_task_state_memo_stack[i];
-          }
-          for (i = master_th->th.th_task_state_stack_sz; i < new_size;
-               ++i) { // zero-init rest of stack
-            new_stack[i] = 0;
-          }
-          old_stack = master_th->th.th_task_state_memo_stack;
-          master_th->th.th_task_state_memo_stack = new_stack;
-          master_th->th.th_task_state_stack_sz = new_size;
-          __kmp_free(old_stack);
-        }
-        // Store primary thread's task_state on stack
-        master_th->th
-            .th_task_state_memo_stack[master_th->th.th_task_state_top] =
-            master_th->th.th_task_state;
-        master_th->th.th_task_state_top++;
-#if KMP_NESTED_HOT_TEAMS
-        if (master_th->th.th_hot_teams &&
-            active_level < __kmp_hot_teams_max_level &&
-            team == master_th->th.th_hot_teams[active_level].hot_team) {
-          // Restore primary thread's nested state if nested hot team
-          master_th->th.th_task_state =
-              master_th->th
-                  .th_task_state_memo_stack[master_th->th.th_task_state_top];
-        } else {
-#endif
-          master_th->th.th_task_state = 0;
-#if KMP_NESTED_HOT_TEAMS
-        }
-#endif
-      }
-#if !KMP_NESTED_HOT_TEAMS
-      KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||
-                       (team == root->r.r_hot_team));
-#endif
-    }
-
     KA_TRACE(
         20,
         ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",
@@ -2451,8 +2418,7 @@ void __kmp_join_call(ident_t *loc, int gtid
                   __kmp_gtid_from_thread(master_th), team,
                   team->t.t_task_team[master_th->th.th_task_state],
                   master_th->th.th_task_team));
-    KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
-                     team->t.t_task_team[master_th->th.th_task_state]);
+    KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, master_th);
   }
 #endif
 
@@ -2690,24 +2656,11 @@ void __kmp_join_call(ident_t *loc, int gtid
   }
 
   if (__kmp_tasking_mode != tskm_immediate_exec) {
-    if (master_th->th.th_task_state_top >
-        0) { // Restore task state from memo stack
-      KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
-      // Remember primary thread's state if we re-use this nested hot team
-      master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
-          master_th->th.th_task_state;
-      --master_th->th.th_task_state_top; // pop
-      // Now restore state at this level
-      master_th->th.th_task_state =
-          master_th->th
-              .th_task_state_memo_stack[master_th->th.th_task_state_top];
-    } else if (team != root->r.r_hot_team) {
-      // Reset the task state of primary thread if we are not hot team because
-      // in this case all the worker threads will be free, and their task state
-      // will be reset. If not reset the primary's, the task state will be
-      // inconsistent.
-      master_th->th.th_task_state = 0;
-    }
+    // Restore primary thread's task state from team structure
+    KMP_DEBUG_ASSERT(team->t.t_primary_task_state == 0 ||
+                     team->t.t_primary_task_state == 1);
+    master_th->th.th_task_state = (kmp_uint8)team->t.t_primary_task_state;
+
     // Copy the task team from the parent team to the primary thread
     master_th->th.th_task_team =
         parent_team->t.t_task_team[master_th->th.th_task_state];
@@ -4396,17 +4349,6 @@ static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
 
   this_thr->th.th_next_pool = NULL;
 
-  if (!this_thr->th.th_task_state_memo_stack) {
-    size_t i;
-    this_thr->th.th_task_state_memo_stack =
-        (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8));
-    this_thr->th.th_task_state_top = 0;
-    this_thr->th.th_task_state_stack_sz = 4;
-    for (i = 0; i < this_thr->th.th_task_state_stack_sz;
-         ++i) // zero init the stack
-      this_thr->th.th_task_state_memo_stack[i] = 0;
-  }
-
   KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
   KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
 
@@ -4463,8 +4405,6 @@ kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
     TCW_4(__kmp_nth, __kmp_nth + 1);
 
     new_thr->th.th_task_state = 0;
-    new_thr->th.th_task_state_top = 0;
-    new_thr->th.th_task_state_stack_sz = 4;
 
     if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
       // Make sure pool thread has transitioned to waiting on own thread struct
@@ -5262,6 +5202,15 @@ __kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
         // Activate team threads via th_used_in_team
         __kmp_add_threads_to_team(team, new_nproc);
       }
+      // When decreasing team size, threads no longer in the team should
+      // unref task team.
+      if (__kmp_tasking_mode != tskm_immediate_exec) {
+        for (f = new_nproc; f < team->t.t_nproc; f++) {
+          kmp_info_t *th = team->t.t_threads[f];
+          KMP_DEBUG_ASSERT(th);
+          th->th.th_task_team = NULL;
+        }
+      }
 #if KMP_NESTED_HOT_TEAMS
       if (__kmp_hot_teams_mode == 0) {
         // AC: saved number of threads should correspond to team's value in this
@@ -5272,11 +5221,6 @@ __kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
         /* release the extra threads we don't need any more */
         for (f = new_nproc; f < team->t.t_nproc; f++) {
           KMP_DEBUG_ASSERT(team->t.t_threads[f]);
-          if (__kmp_tasking_mode != tskm_immediate_exec) {
-            // When decreasing team size, threads no longer in the team should
-            // unref task team.
-            team->t.t_threads[f]->th.th_task_team = NULL;
-          }
           __kmp_free_thread(team->t.t_threads[f]);
           team->t.t_threads[f] = NULL;
         }
@@ -6248,11 +6192,6 @@ static void __kmp_reap_thread(kmp_info_t *thread, int is_root) {
     thread->th.th_pri_common = NULL;
   }
 
-  if (thread->th.th_task_state_memo_stack != NULL) {
-    __kmp_free(thread->th.th_task_state_memo_stack);
-    thread->th.th_task_state_memo_stack = NULL;
-  }
-
 #if KMP_USE_BGET
   if (thread->th.th_local.bget_data != NULL) {
     __kmp_finalize_bget(thread);
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 6303bb0d63f0..a78202749449 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -1511,8 +1511,7 @@ kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
       KA_TRACE(30,
                ("T#%d creating task team in __kmp_task_alloc for proxy task\n",
                 gtid));
-      // 1 indicates setup the current team regardless of nthreads
-      __kmp_task_team_setup(thread, team, 1);
+      __kmp_task_team_setup(thread, team);
       thread->th.th_task_team = team->t.t_task_team[thread->th.th_task_state];
     }
     kmp_task_team_t *task_team = thread->th.th_task_team;
@@ -3390,8 +3389,6 @@ static inline int __kmp_execute_tasks_template(
 
   nthreads = task_team->tt.tt_nproc;
   unfinished_threads = &(task_team->tt.tt_unfinished_threads);
-  KMP_DEBUG_ASSERT(nthreads > 1 || task_team->tt.tt_found_proxy_tasks ||
-                   task_team->tt.tt_hidden_helper_task_encountered);
   KMP_DEBUG_ASSERT(*unfinished_threads >= 0);
 
   while (1) { // Outer loop keeps trying to find tasks in case of single thread
@@ -3943,6 +3940,20 @@ static void __kmp_free_task_pri_list(kmp_task_team_t *task_team) {
   __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
 }
 
+static inline void __kmp_task_team_init(kmp_task_team_t *task_team,
+                                        kmp_team_t *team) {
+  int team_nth = team->t.t_nproc;
+  // Only need to init if task team is isn't active or team size changed
+  if (!task_team->tt.tt_active || team_nth != task_team->tt.tt_nproc) {
+    TCW_4(task_team->tt.tt_found_tasks, FALSE);
+    TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
+    TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
+    TCW_4(task_team->tt.tt_nproc, team_nth);
+    KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, team_nth);
+    TCW_4(task_team->tt.tt_active, TRUE);
+  }
+}
+
 // __kmp_allocate_task_team:
 // Allocates a task team associated with a specific team, taking it from
 // the global task team free list if possible.  Also initializes data
@@ -3950,7 +3961,6 @@ static void __kmp_free_task_pri_list(kmp_task_team_t *task_team) {
 static kmp_task_team_t *__kmp_allocate_task_team(kmp_info_t *thread,
                                                  kmp_team_t *team) {
   kmp_task_team_t *task_team = NULL;
-  int nthreads;
 
   KA_TRACE(20, ("__kmp_allocate_task_team: T#%d entering; team = %p\n",
                 (thread ? __kmp_gtid_from_thread(thread) : -1), team));
@@ -3992,14 +4002,7 @@ static kmp_task_team_t *__kmp_allocate_task_team(kmp_info_t *thread,
     // task_team->tt.tt_next = NULL;
   }
 
-  TCW_4(task_team->tt.tt_found_tasks, FALSE);
-  TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
-  TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
-  task_team->tt.tt_nproc = nthreads = team->t.t_nproc;
-
-  KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, nthreads);
-  TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
-  TCW_4(task_team->tt.tt_active, TRUE);
+  __kmp_task_team_init(task_team, team);
 
   KA_TRACE(20, ("__kmp_allocate_task_team: T#%d exiting; task_team = %p "
                 "unfinished_threads init'd to %d\n",
@@ -4053,6 +4056,40 @@ void __kmp_reap_task_teams(void) {
   }
 }
 
+// View the array of two task team pointers as a pair of pointers:
+//  1) a single task_team pointer
+//  2) next pointer for stack
+// Serial teams can create a stack of task teams for nested serial teams.
+void __kmp_push_task_team_node(kmp_info_t *thread, kmp_team_t *team) {
+  KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
+  kmp_task_team_list_t *current =
+      (kmp_task_team_list_t *)(&team->t.t_task_team[0]);
+  kmp_task_team_list_t *node =
+      (kmp_task_team_list_t *)__kmp_allocate(sizeof(kmp_task_team_list_t));
+  node->task_team = current->task_team;
+  node->next = current->next;
+  thread->th.th_task_team = current->task_team = NULL;
+  current->next = node;
+}
+
+// Serial team pops a task team off the stack
+void __kmp_pop_task_team_node(kmp_info_t *thread, kmp_team_t *team) {
+  KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
+  kmp_task_team_list_t *current =
+      (kmp_task_team_list_t *)(&team->t.t_task_team[0]);
+  if (current->task_team) {
+    __kmp_free_task_team(thread, current->task_team);
+  }
+  kmp_task_team_list_t *next = current->next;
+  if (next) {
+    current->task_team = next->task_team;
+    current->next = next->next;
+    KMP_DEBUG_ASSERT(next != current);
+    __kmp_free(next);
+    thread->th.th_task_team = current->task_team;
+  }
+}
+
 // __kmp_wait_to_unref_task_teams:
 // Some threads could still be in the fork barrier release code, possibly
 // trying to steal tasks.  Wait for each thread to unreference its task team.
@@ -4117,55 +4154,34 @@ void __kmp_wait_to_unref_task_teams(void) {
   }
 }
 
-void __kmp_shift_task_state_stack(kmp_info_t *this_thr, kmp_uint8 value) {
-  // Shift values from th_task_state_top+1 to task_state_stack_sz
-  if (this_thr->th.th_task_state_top + 1 >=
-      this_thr->th.th_task_state_stack_sz) { // increase size
-    kmp_uint32 new_size = 2 * this_thr->th.th_task_state_stack_sz;
-    kmp_uint8 *old_stack, *new_stack;
-    kmp_uint32 i;
-    new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
-    for (i = 0; i <= this_thr->th.th_task_state_top; ++i) {
-      new_stack[i] = this_thr->th.th_task_state_memo_stack[i];
-    }
-    // If we need to reallocate do the shift at the same time.
-    for (; i < this_thr->th.th_task_state_stack_sz; ++i) {
-      new_stack[i + 1] = this_thr->th.th_task_state_memo_stack[i];
-    }
-    for (i = this_thr->th.th_task_state_stack_sz; i < new_size;
-         ++i) { // zero-init rest of stack
-      new_stack[i] = 0;
-    }
-    old_stack = this_thr->th.th_task_state_memo_stack;
-    this_thr->th.th_task_state_memo_stack = new_stack;
-    this_thr->th.th_task_state_stack_sz = new_size;
-    __kmp_free(old_stack);
-  } else {
-    kmp_uint8 *end;
-    kmp_uint32 i;
-
-    end = &this_thr->th
-               .th_task_state_memo_stack[this_thr->th.th_task_state_stack_sz];
-
-    for (i = this_thr->th.th_task_state_stack_sz - 1;
-         i > this_thr->th.th_task_state_top; i--, end--)
-      end[0] = end[-1];
-  }
-  this_thr->th.th_task_state_memo_stack[this_thr->th.th_task_state_top + 1] =
-      value;
-}
-
 // __kmp_task_team_setup:  Create a task_team for the current team, but use
 // an already created, unused one if it already exists.
-void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) {
+void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team) {
   KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
 
+  // For the serial and root teams, setup the first task team pointer to point
+  // to task team. The other pointer is a stack of task teams from previous
+  // serial levels.
+  if (team == this_thr->th.th_serial_team ||
+      team == this_thr->th.th_root->r.r_root_team) {
+    KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
+    if (team->t.t_task_team[0] == NULL) {
+      team->t.t_task_team[0] = __kmp_allocate_task_team(this_thr, team);
+      KA_TRACE(
+          20, ("__kmp_task_team_setup: Primary T#%d created new task_team %p"
+               " for serial/root team %p\n",
+               __kmp_gtid_from_thread(this_thr), team->t.t_task_team[0], team));
+
+    } else
+      __kmp_task_team_init(team->t.t_task_team[0], team);
+    return;
+  }
+
   // If this task_team hasn't been created yet, allocate it. It will be used in
   // the region after the next.
   // If it exists, it is the current task team and shouldn't be touched yet as
   // it may still be in use.
-  if (team->t.t_task_team[this_thr->th.th_task_state] == NULL &&
-      (always || team->t.t_nproc > 1)) {
+  if (team->t.t_task_team[this_thr->th.th_task_state] == NULL) {
     team->t.t_task_team[this_thr->th.th_task_state] =
         __kmp_allocate_task_team(this_thr, team);
     KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created new task_team %p"
@@ -4174,52 +4190,31 @@ void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) {
                   team->t.t_task_team[this_thr->th.th_task_state], team->t.t_id,
                   this_thr->th.th_task_state));
   }
-  if (this_thr->th.th_task_state == 1 && always && team->t.t_nproc == 1) {
-    // fix task state stack to adjust for proxy and helper tasks
-    KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d needs to shift stack"
-                  " for team %d at parity=%d\n",
-                  __kmp_gtid_from_thread(this_thr), team->t.t_id,
-                  this_thr->th.th_task_state));
-    __kmp_shift_task_state_stack(this_thr, this_thr->th.th_task_state);
-  }
 
   // After threads exit the release, they will call sync, and then point to this
   // other task_team; make sure it is allocated and properly initialized. As
   // threads spin in the barrier release phase, they will continue to use the
   // previous task_team struct(above), until they receive the signal to stop
   // checking for tasks (they can't safely reference the kmp_team_t struct,
-  // which could be reallocated by the primary thread). No task teams are formed
-  // for serialized teams.
-  if (team->t.t_nproc > 1) {
-    int other_team = 1 - this_thr->th.th_task_state;
-    KMP_DEBUG_ASSERT(other_team >= 0 && other_team < 2);
-    if (team->t.t_task_team[other_team] == NULL) { // setup other team as well
-      team->t.t_task_team[other_team] =
-          __kmp_allocate_task_team(this_thr, team);
-      KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created second new "
-                    "task_team %p for team %d at parity=%d\n",
-                    __kmp_gtid_from_thread(this_thr),
-                    team->t.t_task_team[other_team], team->t.t_id, other_team));
-    } else { // Leave the old task team struct in place for the upcoming region;
-      // adjust as needed
-      kmp_task_team_t *task_team = team->t.t_task_team[other_team];
-      if (!task_team->tt.tt_active ||
-          team->t.t_nproc != task_team->tt.tt_nproc) {
-        TCW_4(task_team->tt.tt_nproc, team->t.t_nproc);
-        TCW_4(task_team->tt.tt_found_tasks, FALSE);
-        TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
-        TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
-        KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads,
-                          team->t.t_nproc);
-        TCW_4(task_team->tt.tt_active, TRUE);
-      }
-      // if team size has changed, the first thread to enable tasking will
-      // realloc threads_data if necessary
-      KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d reset next task_team "
-                    "%p for team %d at parity=%d\n",
-                    __kmp_gtid_from_thread(this_thr),
-                    team->t.t_task_team[other_team], team->t.t_id, other_team));
-    }
+  // which could be reallocated by the primary thread).
+  int other_team = 1 - this_thr->th.th_task_state;
+  KMP_DEBUG_ASSERT(other_team >= 0 && other_team < 2);
+  if (team->t.t_task_team[other_team] == NULL) { // setup other team as well
+    team->t.t_task_team[other_team] = __kmp_allocate_task_team(this_thr, team);
+    KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created second new "
+                  "task_team %p for team %d at parity=%d\n",
+                  __kmp_gtid_from_thread(this_thr),
+                  team->t.t_task_team[other_team], team->t.t_id, other_team));
+  } else { // Leave the old task team struct in place for the upcoming region;
+    // adjust as needed
+    kmp_task_team_t *task_team = team->t.t_task_team[other_team];
+    __kmp_task_team_init(task_team, team);
+    // if team size has changed, the first thread to enable tasking will
+    // realloc threads_data if necessary
+    KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d reset next task_team "
+                  "%p for team %d at parity=%d\n",
+                  __kmp_gtid_from_thread(this_thr),
+                  team->t.t_task_team[other_team], team->t.t_id, other_team));
   }
 
   // For regular thread, task enabling should be called when the task is going
@@ -4245,9 +4240,11 @@ void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) {
 
 // __kmp_task_team_sync: Propagation of task team data from team to threads
 // which happens just after the release phase of a team barrier.  This may be
-// called by any thread, but only for teams with # threads > 1.
+// called by any thread. This is not called for serial or root teams.
 void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) {
   KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
+  KMP_DEBUG_ASSERT(team != this_thr->th.th_serial_team);
+  KMP_DEBUG_ASSERT(team != this_thr->th.th_root->r.r_root_team);
 
   // Toggle the th_task_state field, to switch which task_team this thread
   // refers to
@@ -4265,8 +4262,7 @@ void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) {
 }
 
 // __kmp_task_team_wait: Primary thread waits for outstanding tasks after the
-// barrier gather phase. Only called by primary thread if #threads in team > 1
-// or if proxy tasks were created.
+// barrier gather phase. Only called by the primary thread.
 //
 // wait is a flag that defaults to 1 (see kmp.h), but waiting can be turned off
 // by passing in 0 optionally as the last argument. When wait is zero, primary
@@ -4300,9 +4296,6 @@ void __kmp_task_team_wait(
         ("__kmp_task_team_wait: Primary T#%d deactivating task_team %p: "
          "setting active to false, setting local and team's pointer to NULL\n",
          __kmp_gtid_from_thread(this_thr), task_team));
-    KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1 ||
-                     task_team->tt.tt_found_proxy_tasks == TRUE ||
-                     task_team->tt.tt_hidden_helper_task_encountered == TRUE);
     TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE);
     TCW_SYNC_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
     KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0);
diff --git a/openmp/runtime/test/target/issue-81488.c b/openmp/runtime/test/target/issue-81488.c
new file mode 100644
index 000000000000..adac7d699446
--- /dev/null
+++ b/openmp/runtime/test/target/issue-81488.c
@@ -0,0 +1,36 @@
+// RUN: %libomp-compile
+// RUN: env OMP_NUM_THREADS=1 LIBOMP_USE_HIDDEN_HELPER_TASK=1 \
+// RUN:     LIBOMP_NUM_HIDDEN_HELPER_THREADS=8 %libomp-run
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <omp.h>
+
+#define Nz 8
+#define DEVICE_ID 0
+
+int a[Nz];
+
+int main(void) {
+  for (int n = 0; n < 10; ++n) {
+    for (int k = 0; k < Nz; ++k) {
+      a[k] = -1;
+    }
+#pragma omp parallel shared(a)
+    {
+#pragma omp single
+      {
+#pragma omp target teams distribute parallel for nowait device(DEVICE_ID)      \
+    map(tofrom : a[0 : 8])
+        for (int i = 0; i < Nz; ++i) {
+          a[i] = i;
+        }
+      }
+#pragma omp barrier
+    }
+    for (int k = 0; k < Nz; ++k) {
+      printf("a[%d] = %d\n", k, a[k]);
+    }
+  }
+  return 0;
+}
diff --git a/openmp/runtime/test/tasking/issue-50602.c b/openmp/runtime/test/tasking/issue-50602.c
new file mode 100644
index 000000000000..b691204c480e
--- /dev/null
+++ b/openmp/runtime/test/tasking/issue-50602.c
@@ -0,0 +1,40 @@
+// RUN: %libomp-compile-and-run
+// RUN: env OMP_NUM_THREADS=1 %libomp-run
+// RUN: %libomp-compile -DUSE_HIDDEN_HELPERS=1
+// RUN: %libomp-run
+// RUN: env OMP_NUM_THREADS=1 %libomp-run
+#include <omp.h>
+
+int main(int argc, char *argv[]) {
+  int i;
+
+  omp_set_max_active_levels(1);
+  omp_set_dynamic(0);
+
+  for (i = 0; i < 10; ++i) {
+#pragma omp parallel
+    {
+#ifndef USE_HIDDEN_HELPERS
+      omp_event_handle_t event;
+#endif
+      int a = 0;
+
+#ifdef USE_HIDDEN_HELPERS
+#pragma omp target map(tofrom : a) nowait
+#else
+#pragma omp task shared(a) detach(event)
+#endif
+      { a = 1; }
+
+#pragma omp parallel
+      { a = 2; }
+
+#ifndef USE_HIDDEN_HELPERS
+      omp_fulfill_event(event);
+#endif
+
+#pragma omp taskwait
+    }
+  }
+  return 0;
+}
diff --git a/openmp/runtime/test/tasking/issue-69368.c b/openmp/runtime/test/tasking/issue-69368.c
new file mode 100644
index 000000000000..57bd7412a51e
--- /dev/null
+++ b/openmp/runtime/test/tasking/issue-69368.c
@@ -0,0 +1,27 @@
+// RUN: %libomp-compile-and-run
+// RUN: env OMP_NUM_THREADS=1 %libomp-run
+
+int main() {
+  int i;
+  int a[2];
+  volatile int attempt = 0;
+
+  for (i = 0; i < 10; ++i) {
+    a[0] = a[1] = 0;
+#pragma omp parallel for
+    for (int i = 0; i < 2; i++) {
+      a[i] = 2;
+    }
+    if (a[0] != 2 || a[1] != 2)
+      return 1;
+
+#pragma omp teams distribute parallel for if (attempt >= 2)
+    for (int i = 0; i < 2; i++) {
+      a[i] = 1;
+    }
+    if (a[0] != 1 || a[1] != 1)
+      return 1;
+  }
+
+  return 0;
+}
diff --git a/openmp/runtime/test/tasking/issue-69733.c b/openmp/runtime/test/tasking/issue-69733.c
new file mode 100644
index 000000000000..5775b016b7b4
--- /dev/null
+++ b/openmp/runtime/test/tasking/issue-69733.c
@@ -0,0 +1,147 @@
+// RUN: %libomp-compile-and-run
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <omp.h>
+
+int a;
+
+void inc_a() {
+#pragma omp atomic
+  a++;
+}
+
+void root_team_detached() {
+  a = 0;
+  omp_event_handle_t ev;
+#pragma omp task detach(ev)
+  inc_a();
+  omp_fulfill_event(ev);
+  if (a != 1) {
+    fprintf(stderr, "error: root_team_detached(): a != 1\n");
+    exit(EXIT_FAILURE);
+  }
+}
+
+void root_team_hidden_helpers() {
+  a = 0;
+#pragma omp target nowait
+  inc_a();
+
+#pragma omp taskwait
+
+  if (a != 1) {
+    fprintf(stderr, "error: root_team_hidden_helpers(): a != 1\n");
+    exit(EXIT_FAILURE);
+  }
+}
+
+void parallel_detached(int nth1) {
+  a = 0;
+  omp_event_handle_t *evs =
+      (omp_event_handle_t *)malloc(sizeof(omp_event_handle_t) * nth1);
+#pragma omp parallel num_threads(nth1)
+  {
+    int tid = omp_get_thread_num();
+    omp_event_handle_t e = evs[tid];
+#pragma omp task detach(e)
+    inc_a();
+    omp_fulfill_event(e);
+  }
+  free(evs);
+  if (a != nth1) {
+    fprintf(stderr, "error: parallel_detached(): a (%d) != %d\n", a, nth1);
+    exit(EXIT_FAILURE);
+  }
+}
+
+void parallel_hidden_helpers(int nth1) {
+  a = 0;
+#pragma omp parallel num_threads(nth1)
+  {
+#pragma omp target nowait
+    inc_a();
+  }
+  if (a != nth1) {
+    fprintf(stderr, "error: parallel_hidden_helpers(): a (%d) != %d\n", a,
+            nth1);
+    exit(EXIT_FAILURE);
+  }
+}
+
+void nested_parallel_detached(int nth1, int nth2) {
+  a = 0;
+  omp_event_handle_t **evs =
+      (omp_event_handle_t **)malloc(sizeof(omp_event_handle_t *) * nth1);
+#pragma omp parallel num_threads(nth1)
+  {
+    int tid = omp_get_thread_num();
+    evs[tid] = (omp_event_handle_t *)malloc(sizeof(omp_event_handle_t) * nth2);
+#pragma omp parallel num_threads(nth2) shared(tid)
+    {
+      int tid2 = omp_get_thread_num();
+      omp_event_handle_t e = evs[tid][tid2];
+#pragma omp task detach(e)
+      inc_a();
+      omp_fulfill_event(e);
+    }
+    free(evs[tid]);
+  }
+  free(evs);
+  if (a != nth1 * nth2) {
+    fprintf(stderr, "error: nested_parallel_detached(): a (%d) != %d * %d\n", a,
+            nth1, nth2);
+    exit(EXIT_FAILURE);
+  }
+}
+
+void nested_parallel_hidden_helpers(int nth1, int nth2) {
+  a = 0;
+#pragma omp parallel num_threads(nth1)
+  {
+#pragma omp parallel num_threads(nth2)
+    {
+#pragma omp target nowait
+      inc_a();
+    }
+  }
+  if (a != nth1 * nth2) {
+    fprintf(stderr,
+            "error: nested_parallel_hidden_helpers(): a (%d) != %d * %d\n", a,
+            nth1, nth2);
+    exit(EXIT_FAILURE);
+  }
+}
+
+int main() {
+  int i, nth1, nth2;
+
+  omp_set_max_active_levels(2);
+  omp_set_dynamic(0);
+
+  for (i = 0; i < 10; ++i)
+    root_team_detached();
+
+  for (i = 0; i < 10; ++i)
+    root_team_hidden_helpers();
+
+  for (i = 0; i < 10; ++i)
+    for (nth1 = 1; nth1 <= 4; ++nth1)
+      parallel_detached(nth1);
+
+  for (i = 0; i < 10; ++i)
+    for (nth1 = 1; nth1 <= 4; ++nth1)
+      parallel_hidden_helpers(nth1);
+
+  for (i = 0; i < 10; ++i)
+    for (nth1 = 1; nth1 <= 4; ++nth1)
+      for (nth2 = 1; nth2 <= 4; ++nth2)
+        nested_parallel_detached(nth1, nth2);
+
+  for (i = 0; i < 10; ++i)
+    for (nth1 = 1; nth1 <= 4; ++nth1)
+      for (nth2 = 1; nth2 <= 4; ++nth2)
+        nested_parallel_hidden_helpers(nth1, nth2);
+
+  return 0;
+}
diff --git a/openmp/runtime/test/tasking/issue-79416.c b/openmp/runtime/test/tasking/issue-79416.c
new file mode 100644
index 000000000000..ee96fce80974
--- /dev/null
+++ b/openmp/runtime/test/tasking/issue-79416.c
@@ -0,0 +1,33 @@
+// RUN: %libomp-compile-and-run
+#include <stdio.h>
+#include <stdlib.h>
+
+int a;
+
+void run(int nteams, int nth) {
+  a = 0;
+#pragma omp teams num_teams(nteams)
+  {
+#pragma omp parallel num_threads(nth)
+    {
+#pragma omp task
+      {
+#pragma omp atomic
+        a++;
+      }
+    }
+  }
+  if (a == 0)
+    exit(EXIT_FAILURE);
+}
+
+int main() {
+  int i, nteams, nth;
+  for (nteams = 1; nteams <= 2; ++nteams)
+    for (nth = 1; nth <= 3; ++nth)
+      for (i = 0; i < 10; ++i) {
+        printf("run(%d, %d)\n", nteams, nth);
+        run(nteams, nth);
+      }
+  return EXIT_SUCCESS;
+}
diff --git a/openmp/runtime/test/tasking/task_teams_stress_test.cpp b/openmp/runtime/test/tasking/task_teams_stress_test.cpp
new file mode 100644
index 000000000000..e781a895d41f
--- /dev/null
+++ b/openmp/runtime/test/tasking/task_teams_stress_test.cpp
@@ -0,0 +1,318 @@
+// RUN: %libomp-cxx-compile
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=0 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=1 KMP_HOT_TEAMS_MODE=0 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=1 KMP_HOT_TEAMS_MODE=1 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=2 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=3 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=4 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=5 %libomp-run
+//
+// RUN: %libomp-cxx-compile -DUSE_HIDDEN_HELPERS=1
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=0 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=1 KMP_HOT_TEAMS_MODE=0 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=1 KMP_HOT_TEAMS_MODE=1 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=2 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=3 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=4 %libomp-run
+// RUN: env KMP_HOT_TEAMS_MAX_LEVEL=5 %libomp-run
+
+// This test stresses the task team mechanism by running a simple
+// increment task over and over with varying number of threads and nesting.
+// The test covers nested serial teams and mixing serial teams with
+// normal active teams.
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <omp.h>
+
+// The number of times to run each test
+#define NTIMES 5
+
+// Regular single increment task
+void task_inc_a(int *a) {
+#pragma omp task
+  {
+#pragma omp atomic
+    (*a)++;
+  }
+}
+
+// Splitting increment task that binary splits the incrementing task
+void task_inc_split_a(int *a, int low, int high) {
+#pragma omp task firstprivate(low, high)
+  {
+    if (low == high) {
+#pragma omp atomic
+      (*a)++;
+    } else if (low < high) {
+      int mid = (high - low) / 2 + low;
+      task_inc_split_a(a, low, mid);
+      task_inc_split_a(a, mid + 1, high);
+    }
+  }
+}
+
+#ifdef USE_HIDDEN_HELPERS
+// Hidden helper tasks force serial regions to create task teams
+void task_inc_a_hidden_helper(int *a) {
+#pragma omp target map(tofrom : a[0]) nowait
+  {
+#pragma omp atomic
+    (*a)++;
+  }
+}
+#else
+// Detached tasks force serial regions to create task teams
+void task_inc_a_detached(int *a, omp_event_handle_t handle) {
+#pragma omp task detach(handle)
+  {
+#pragma omp atomic
+    (*a)++;
+    omp_fulfill_event(handle);
+  }
+}
+#endif
+
+void check_a(int *a, int expected) {
+  if (*a != expected) {
+    fprintf(stderr,
+            "FAIL: a = %d instead of expected = %d. Compile with "
+            "-DVERBOSE for more verbose output.\n",
+            *a, expected);
+    exit(EXIT_FAILURE);
+  }
+}
+
+// Every thread creates a single "increment" task
+void test_tasks(omp_event_handle_t *handles, int expected, int *a) {
+  int tid = omp_get_thread_num();
+
+  task_inc_a(a);
+
+#pragma omp barrier
+  check_a(a, expected);
+#pragma omp barrier
+  check_a(a, expected);
+#pragma omp barrier
+
+#ifdef USE_HIDDEN_HELPERS
+  task_inc_a_hidden_helper(a);
+#else
+  task_inc_a_detached(a, handles[tid]);
+#endif
+
+#pragma omp barrier
+  check_a(a, 2 * expected);
+#pragma omp barrier
+  task_inc_a(a);
+#pragma omp barrier
+  check_a(a, 3 * expected);
+}
+
+// Testing single level of parallelism with increment tasks
+void test_base(int nthreads) {
+#ifdef VERBOSE
+#pragma omp master
+  printf("    test_base(%d)\n", nthreads);
+#endif
+  int a = 0;
+  omp_event_handle_t *handles;
+  handles = (omp_event_handle_t *)malloc(sizeof(omp_event_handle_t) * nthreads);
+#pragma omp parallel num_threads(nthreads) shared(a)
+  { test_tasks(handles, nthreads, &a); }
+  free(handles);
+}
+
+// Testing nested parallel with increment tasks
+// first = nthreads of outer parallel
+// second = nthreads of nested parallel
+void test_nest(int first, int second) {
+#ifdef VERBOSE
+#pragma omp master
+  printf("   test_nest(%d, %d)\n", first, second);
+#endif
+#pragma omp parallel num_threads(first)
+  { test_base(second); }
+}
+
+// Testing 2-level nested parallels with increment tasks
+// first = nthreads of outer parallel
+// second = nthreads of nested parallel
+// third = nthreads of second nested parallel
+void test_nest2(int first, int second, int third) {
+#ifdef VERBOSE
+#pragma omp master
+  printf("  test_nest2(%d, %d, %d)\n", first, second, third);
+#endif
+#pragma omp parallel num_threads(first)
+  { test_nest(second, third); }
+}
+
+// Testing 3-level nested parallels with increment tasks
+// first = nthreads of outer parallel
+// second = nthreads of nested parallel
+// third = nthreads of second nested parallel
+// fourth = nthreads of third nested parallel
+void test_nest3(int first, int second, int third, int fourth) {
+#ifdef VERBOSE
+#pragma omp master
+  printf(" test_nest3(%d, %d, %d, %d)\n", first, second, third, fourth);
+#endif
+#pragma omp parallel num_threads(first)
+  { test_nest2(second, third, fourth); }
+}
+
+// Testing 4-level nested parallels with increment tasks
+// first = nthreads of outer parallel
+// second = nthreads of nested parallel
+// third = nthreads of second nested parallel
+// fourth = nthreads of third nested parallel
+// fifth = nthreads of fourth nested parallel
+void test_nest4(int first, int second, int third, int fourth, int fifth) {
+#ifdef VERBOSE
+#pragma omp master
+  printf("test_nest4(%d, %d, %d, %d, %d)\n", first, second, third, fourth,
+         fifth);
+#endif
+#pragma omp parallel num_threads(first)
+  { test_nest3(second, third, fourth, fifth); }
+}
+
+// Single thread starts a binary splitting "increment" task
+// Detached tasks are still single "increment" task
+void test_tasks_split(omp_event_handle_t *handles, int expected, int *a) {
+  int tid = omp_get_thread_num();
+
+#pragma omp single
+  task_inc_split_a(a, 1, expected); // task team A
+
+#pragma omp barrier
+  check_a(a, expected);
+#pragma omp barrier
+  check_a(a, expected);
+#pragma omp barrier
+
+#ifdef USE_HIDDEN_HELPERS
+  task_inc_a_hidden_helper(a);
+#else
+  task_inc_a_detached(a, handles[tid]);
+#endif
+
+#pragma omp barrier
+  check_a(a, 2 * expected);
+#pragma omp barrier
+#pragma omp single
+  task_inc_split_a(a, 1, expected); // task team B
+#pragma omp barrier
+  check_a(a, 3 * expected);
+}
+
+// Testing single level of parallelism with splitting incrementing tasks
+void test_base_split(int nthreads) {
+#ifdef VERBOSE
+#pragma omp master
+  printf("  test_base_split(%d)\n", nthreads);
+#endif
+  int a = 0;
+  omp_event_handle_t *handles;
+  handles = (omp_event_handle_t *)malloc(sizeof(omp_event_handle_t) * nthreads);
+#pragma omp parallel num_threads(nthreads) shared(a)
+  { test_tasks_split(handles, nthreads, &a); }
+  free(handles);
+}
+
+// Testing nested parallels with splitting tasks
+// first = nthreads of outer parallel
+// second = nthreads of nested parallel
+void test_nest_split(int first, int second) {
+#ifdef VERBOSE
+#pragma omp master
+  printf(" test_nest_split(%d, %d)\n", first, second);
+#endif
+#pragma omp parallel num_threads(first)
+  { test_base_split(second); }
+}
+
+// Testing doubly nested parallels with splitting tasks
+// first = nthreads of outer parallel
+// second = nthreads of nested parallel
+// third = nthreads of second nested parallel
+void test_nest2_split(int first, int second, int third) {
+#ifdef VERBOSE
+#pragma omp master
+  printf("test_nest2_split(%d, %d, %d)\n", first, second, third);
+#endif
+#pragma omp parallel num_threads(first)
+  { test_nest_split(second, third); }
+}
+
+template <typename... Args>
+void run_ntimes(int n, void (*func)(Args...), Args... args) {
+  for (int i = 0; i < n; ++i) {
+    func(args...);
+  }
+}
+
+int main() {
+  omp_set_max_active_levels(5);
+
+  run_ntimes(NTIMES, test_base, 4);
+  run_ntimes(NTIMES, test_base, 1);
+  run_ntimes(NTIMES, test_base, 8);
+  run_ntimes(NTIMES, test_base, 2);
+  run_ntimes(NTIMES, test_base, 6);
+  run_ntimes(NTIMES, test_nest, 1, 1);
+  run_ntimes(NTIMES, test_nest, 1, 5);
+  run_ntimes(NTIMES, test_nest, 2, 6);
+  run_ntimes(NTIMES, test_nest, 1, 1);
+  run_ntimes(NTIMES, test_nest, 4, 3);
+  run_ntimes(NTIMES, test_nest, 3, 2);
+  run_ntimes(NTIMES, test_nest, 1, 1);
+  run_ntimes(NTIMES, test_nest2, 1, 1, 2);
+  run_ntimes(NTIMES, test_nest2, 1, 2, 1);
+  run_ntimes(NTIMES, test_nest2, 2, 2, 1);
+  run_ntimes(NTIMES, test_nest2, 2, 1, 1);
+  run_ntimes(NTIMES, test_nest2, 4, 2, 1);
+  run_ntimes(NTIMES, test_nest2, 4, 2, 2);
+  run_ntimes(NTIMES, test_nest2, 1, 1, 1);
+  run_ntimes(NTIMES, test_nest2, 4, 2, 2);
+  run_ntimes(NTIMES, test_nest3, 1, 1, 1, 1);
+  run_ntimes(NTIMES, test_nest3, 1, 2, 1, 1);
+  run_ntimes(NTIMES, test_nest3, 1, 1, 2, 1);
+  run_ntimes(NTIMES, test_nest3, 1, 1, 1, 2);
+  run_ntimes(NTIMES, test_nest3, 2, 1, 1, 1);
+  run_ntimes(NTIMES, test_nest4, 1, 1, 1, 1, 1);
+  run_ntimes(NTIMES, test_nest4, 2, 1, 1, 1, 1);
+  run_ntimes(NTIMES, test_nest4, 1, 2, 1, 1, 1);
+  run_ntimes(NTIMES, test_nest4, 1, 1, 2, 1, 1);
+  run_ntimes(NTIMES, test_nest4, 1, 1, 1, 2, 1);
+  run_ntimes(NTIMES, test_nest4, 1, 1, 1, 1, 2);
+  run_ntimes(NTIMES, test_nest4, 1, 1, 1, 1, 1);
+  run_ntimes(NTIMES, test_nest4, 1, 2, 1, 2, 1);
+
+  run_ntimes(NTIMES, test_base_split, 4);
+  run_ntimes(NTIMES, test_base_split, 2);
+
+  run_ntimes(NTIMES, test_base_split, 7);
+
+  run_ntimes(NTIMES, test_base_split, 1);
+  run_ntimes(NTIMES, test_nest_split, 4, 2);
+  run_ntimes(NTIMES, test_nest_split, 2, 1);
+
+  run_ntimes(NTIMES, test_nest_split, 7, 2);
+  run_ntimes(NTIMES, test_nest_split, 1, 1);
+  run_ntimes(NTIMES, test_nest_split, 1, 4);
+
+  run_ntimes(NTIMES, test_nest2_split, 1, 1, 2);
+  run_ntimes(NTIMES, test_nest2_split, 1, 2, 1);
+  run_ntimes(NTIMES, test_nest2_split, 2, 2, 1);
+  run_ntimes(NTIMES, test_nest2_split, 2, 1, 1);
+  run_ntimes(NTIMES, test_nest2_split, 4, 2, 1);
+  run_ntimes(NTIMES, test_nest2_split, 4, 2, 2);
+  run_ntimes(NTIMES, test_nest2_split, 1, 1, 1);
+  run_ntimes(NTIMES, test_nest2_split, 4, 2, 2);
+
+  printf("PASS\n");
+  return EXIT_SUCCESS;
+}
-- 
GitLab


From 5d9b549bb05ad31727cd019bcefeae7b94b2dbd2 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 7 May 2024 06:28:57 -0700
Subject: [PATCH 0048/1206] [SLP][NFC]Add a test showing incorrect signedness
 detection in sext nodes.

---
 .../AArch64/unsigned-after-sext-node.ll       | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll

diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll
new file mode 100644
index 000000000000..406e5b9b930d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -mtriple=aarch64 -passes=slp-vectorizer -S -slp-threshold=-100 < %s | FileCheck %s
+
+define i16 @test()  {
+; CHECK-LABEL: define i16 @test() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LNOT:%.*]] = xor i1 false, true
+; CHECK-NEXT:    [[LNOT_EXT:%.*]] = zext i1 [[LNOT]] to i16
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i16 0, [[LNOT_EXT]]
+; CHECK-NEXT:    [[LNOT5:%.*]] = xor i1 false, true
+; CHECK-NEXT:    [[LNOT_EXT6:%.*]] = zext i1 [[LNOT5]] to i16
+; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i16 [[ADD]], [[LNOT_EXT6]]
+; CHECK-NEXT:    ret i16 [[ADD7]]
+;
+entry:
+  %conv = sext i16 1 to i32
+  %cmp = icmp eq i32 %conv, 1
+  %lnot = xor i1 %cmp, true
+  %lnot.ext = zext i1 %lnot to i16
+  %add = add nsw i16 0, %lnot.ext
+  %conv2 = sext i16 1 to i32
+  %cmp3 = icmp eq i32 %conv2, 1
+  %lnot5 = xor i1 %cmp3, true
+  %lnot.ext6 = zext i1 %lnot5 to i16
+  %add7 = add nsw i16 %add, %lnot.ext6
+  ret i16 %add7
+}
-- 
GitLab


From a775455cdca78445ccfe4adb2a7c9e390ae46e10 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Tue, 7 May 2024 15:57:27 +0200
Subject: [PATCH 0049/1206] [bazel] Add `nobuildkite` tags for incompatible
 target

---
 utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index a9a74ae09b3b..0ffd562f3111 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -270,6 +270,7 @@ cc_library(
            }),
     hdrs = glob(["Platform/MacOSX/*.h"]),
     include_prefix = "Plugins",
+    tags = ["nobuildkite"],
     deps = [
         ":PlatformMacOSXProperties",
         ":PluginDynamicLoaderDarwinKernelHeaders",
-- 
GitLab


From 9eb91f45fb34353942b8f8154f229150a0d01456 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Tue, 7 May 2024 15:58:47 +0200
Subject: [PATCH 0050/1206] [bazel] Add `nobuildkite` tags for incompatible
 targets

---
 .../bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index 0ffd562f3111..9c8943e44f7b 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -1265,6 +1265,7 @@ cc_library(
     name = "PluginDynamicLoaderDarwinKernel",
     srcs = glob(["DynamicLoader/Darwin-Kernel/*.cpp"]),
     include_prefix = "Plugins",
+    tags = ["nobuildkite"],
     deps = [
         ":DynamicLoaderDarwinKernelProperties",
         ":PluginDynamicLoaderDarwinKernelHeaders",
@@ -2146,6 +2147,7 @@ cc_library(
     srcs = glob(["Process/mach-core/*.cpp"]),
     hdrs = glob(["Process/mach-core/*.h"]),
     include_prefix = "Plugins",
+    tags = ["nobuildkite"],
     deps = [
         ":PluginDynamicLoaderDarwinKernelHeaders",
         ":PluginDynamicLoaderMacOSXDYLD",
-- 
GitLab


From f548c4d83cdded0c19ca02ca9c071d8ced9ea4fd Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 7 May 2024 15:46:15 +0200
Subject: [PATCH 0051/1206] AMDGPU: Add mode register use to s_getreg_b32

This should fix reading the wrong mode after setting the mode.
Ideally we would have separate pseudos for the case that we know
does not read mode.
---
 llvm/lib/Target/AMDGPU/SOPInstructions.td     |   5 +-
 llvm/test/CodeGen/AMDGPU/fdiv.ll              |  44 +++---
 llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll | 127 ++++++++++++++++++
 3 files changed, 152 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 93b7e86b5f29..b05d0018201b 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -1110,14 +1110,15 @@ def S_CBRANCH_I_FORK : SOPK_Pseudo <
 
 // This is hasSideEffects to allow its use in readcyclecounter selection.
 // FIXME: Need to truncate immediate to 16-bits.
-// FIXME: Missing mode register use. Should have separate pseudos for
-// known may read MODE and only read MODE.
+// FIXME: Should have separate pseudos for known may read MODE and
+// only read MODE.
 def S_GETREG_B32 : SOPK_Pseudo <
   "s_getreg_b32",
   (outs SReg_32:$sdst), (ins hwreg:$simm16),
   "$sdst, $simm16",
   [(set i32:$sdst, (int_amdgcn_s_getreg (i32 timm:$simm16)))]> {
   let hasSideEffects = 1;
+  let Uses = [MODE];
 }
 
 let Defs = [MODE], Uses = [MODE] in {
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.ll b/llvm/test/CodeGen/AMDGPU/fdiv.ll
index 1e5f4c08c7a0..0468175c5df5 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.ll
@@ -2417,12 +2417,12 @@ define float @v_fdiv_f32_dynamic_denorm(float %a, float %b) #2 {
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX6-FASTFMA-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX6-FASTFMA-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -2455,12 +2455,12 @@ define float @v_fdiv_f32_dynamic_denorm(float %a, float %b) #2 {
 ; GFX7-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
 ; GFX7-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX7-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX7-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX7-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX7-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX7-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX7-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -2727,12 +2727,12 @@ define float @v_fdiv_f32_dynamic(float %x, float %y) #2 {
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX6-FASTFMA-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX6-FASTFMA-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -2765,12 +2765,12 @@ define float @v_fdiv_f32_dynamic(float %x, float %y) #2 {
 ; GFX7-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
 ; GFX7-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX7-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX7-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX7-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX7-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX7-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX7-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -3294,12 +3294,12 @@ define float @v_fdiv_f32_dynamic_contractable_user(float %x, float %y, float %z)
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v3, s[4:5], v1, v1, v0
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v4, v3
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v5, vcc, v0, v1, v0
+; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v6, -v3, v4, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v4, v6, v4, v4
 ; GFX6-FASTFMA-NEXT:    v_mul_f32_e32 v6, v5, v4
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v7, -v3, v6, v5
-; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v6, v7, v4, v6
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v3, -v3, v6, v5
 ; GFX6-FASTFMA-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -3334,12 +3334,12 @@ define float @v_fdiv_f32_dynamic_contractable_user(float %x, float %y, float %z)
 ; GFX7-NEXT:    v_div_scale_f32 v3, s[4:5], v1, v1, v0
 ; GFX7-NEXT:    v_rcp_f32_e32 v4, v3
 ; GFX7-NEXT:    v_div_scale_f32 v5, vcc, v0, v1, v0
+; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v6, -v3, v4, 1.0
 ; GFX7-NEXT:    v_fma_f32 v4, v6, v4, v4
 ; GFX7-NEXT:    v_mul_f32_e32 v6, v5, v4
 ; GFX7-NEXT:    v_fma_f32 v7, -v3, v6, v5
-; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    v_fma_f32 v6, v7, v4, v6
 ; GFX7-NEXT:    v_fma_f32 v3, -v3, v6, v5
 ; GFX7-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -3868,12 +3868,12 @@ define float @v_fdiv_f32_dynamic__nnan_ninf(float %x, float %y, float %z) #2 {
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX6-FASTFMA-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX6-FASTFMA-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -3906,12 +3906,12 @@ define float @v_fdiv_f32_dynamic__nnan_ninf(float %x, float %y, float %z) #2 {
 ; GFX7-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
 ; GFX7-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX7-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX7-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX7-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX7-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX7-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX7-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -4434,12 +4434,12 @@ define float @v_fdiv_f32_dynamic__nnan_ninf_contractable_user(float %x, float %y
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v3, s[4:5], v1, v1, v0
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v4, v3
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v5, vcc, v0, v1, v0
+; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v6, -v3, v4, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v4, v6, v4, v4
 ; GFX6-FASTFMA-NEXT:    v_mul_f32_e32 v6, v5, v4
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v7, -v3, v6, v5
-; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v6, v7, v4, v6
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v3, -v3, v6, v5
 ; GFX6-FASTFMA-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -4474,12 +4474,12 @@ define float @v_fdiv_f32_dynamic__nnan_ninf_contractable_user(float %x, float %y
 ; GFX7-NEXT:    v_div_scale_f32 v3, s[4:5], v1, v1, v0
 ; GFX7-NEXT:    v_rcp_f32_e32 v4, v3
 ; GFX7-NEXT:    v_div_scale_f32 v5, vcc, v0, v1, v0
+; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v6, -v3, v4, 1.0
 ; GFX7-NEXT:    v_fma_f32 v4, v6, v4, v4
 ; GFX7-NEXT:    v_mul_f32_e32 v6, v5, v4
 ; GFX7-NEXT:    v_fma_f32 v7, -v3, v6, v5
-; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    v_fma_f32 v6, v7, v4, v6
 ; GFX7-NEXT:    v_fma_f32 v3, -v3, v6, v5
 ; GFX7-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -5010,12 +5010,12 @@ define float @v_fdiv_neglhs_f32_dynamic(float %x, float %y) #2 {
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, -v0
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v4, vcc, -v0, v1, -v0
+; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX6-FASTFMA-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX6-FASTFMA-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -5048,12 +5048,12 @@ define float @v_fdiv_neglhs_f32_dynamic(float %x, float %y) #2 {
 ; GFX7-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, -v0
 ; GFX7-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX7-NEXT:    v_div_scale_f32 v4, vcc, -v0, v1, -v0
+; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX7-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX7-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX7-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX7-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX7-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -5569,12 +5569,12 @@ define float @v_fdiv_negrhs_f32_dynamic(float %x, float %y) #2 {
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v2, s[4:5], -v1, -v1, v0
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v4, vcc, v0, -v1, v0
+; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX6-FASTFMA-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX6-FASTFMA-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -5607,12 +5607,12 @@ define float @v_fdiv_negrhs_f32_dynamic(float %x, float %y) #2 {
 ; GFX7-NEXT:    v_div_scale_f32 v2, s[4:5], -v1, -v1, v0
 ; GFX7-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX7-NEXT:    v_div_scale_f32 v4, vcc, v0, -v1, v0
+; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX7-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX7-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX7-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX7-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX7-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -6113,12 +6113,12 @@ define float @v_fdiv_f32_constrhs0_dynamic(float %x) #2 {
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v1, s[4:5], s6, s6, v0
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v2, v1
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v3, vcc, v0, s6, v0
+; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v4, -v1, v2, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v2, v4, v2, v2
 ; GFX6-FASTFMA-NEXT:    v_mul_f32_e32 v4, v3, v2
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, -v1, v4, v3
-; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v4, v5, v2, v4
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v1, -v1, v4, v3
 ; GFX6-FASTFMA-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -6153,12 +6153,12 @@ define float @v_fdiv_f32_constrhs0_dynamic(float %x) #2 {
 ; GFX7-NEXT:    v_div_scale_f32 v1, s[4:5], s6, s6, v0
 ; GFX7-NEXT:    v_rcp_f32_e32 v2, v1
 ; GFX7-NEXT:    v_div_scale_f32 v3, vcc, v0, s6, v0
+; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v4, -v1, v2, 1.0
 ; GFX7-NEXT:    v_fma_f32 v2, v4, v2, v2
 ; GFX7-NEXT:    v_mul_f32_e32 v4, v3, v2
 ; GFX7-NEXT:    v_fma_f32 v5, -v1, v4, v3
-; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    v_fma_f32 v4, v5, v2, v4
 ; GFX7-NEXT:    v_fma_f32 v1, -v1, v4, v3
 ; GFX7-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -6619,12 +6619,12 @@ define float @v_fdiv_f32_constlhs0_dynamic(float %x) #2 {
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v1, s[4:5], v0, v0, s6
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v2, v1
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v3, vcc, s6, v0, s6
+; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v4, -v1, v2, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v2, v4, v2, v2
 ; GFX6-FASTFMA-NEXT:    v_mul_f32_e32 v4, v3, v2
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, -v1, v4, v3
-; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v4, v5, v2, v4
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v1, -v1, v4, v3
 ; GFX6-FASTFMA-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -6659,12 +6659,12 @@ define float @v_fdiv_f32_constlhs0_dynamic(float %x) #2 {
 ; GFX7-NEXT:    v_div_scale_f32 v1, s[4:5], v0, v0, s6
 ; GFX7-NEXT:    v_rcp_f32_e32 v2, v1
 ; GFX7-NEXT:    v_div_scale_f32 v3, vcc, s6, v0, s6
+; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v4, -v1, v2, 1.0
 ; GFX7-NEXT:    v_fma_f32 v2, v4, v2, v2
 ; GFX7-NEXT:    v_mul_f32_e32 v4, v3, v2
 ; GFX7-NEXT:    v_fma_f32 v5, -v1, v4, v3
-; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    v_fma_f32 v4, v5, v2, v4
 ; GFX7-NEXT:    v_fma_f32 v1, -v1, v4, v3
 ; GFX7-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -7168,12 +7168,12 @@ define float @v_fdiv_f32_dynamic_nodenorm_x(float nofpclass(sub) %x, float %y) #
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX6-FASTFMA-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX6-FASTFMA-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -7206,12 +7206,12 @@ define float @v_fdiv_f32_dynamic_nodenorm_x(float nofpclass(sub) %x, float %y) #
 ; GFX7-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
 ; GFX7-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX7-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX7-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX7-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX7-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX7-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX7-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -7721,12 +7721,12 @@ define float @v_fdiv_f32_dynamic_nodenorm_y(float %x, float nofpclass(sub) %y) #
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX6-FASTFMA-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX6-FASTFMA-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX6-FASTFMA-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
@@ -7759,12 +7759,12 @@ define float @v_fdiv_f32_dynamic_nodenorm_y(float %x, float nofpclass(sub) %y) #
 ; GFX7-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
 ; GFX7-NEXT:    v_rcp_f32_e32 v3, v2
 ; GFX7-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
 ; GFX7-NEXT:    v_fma_f32 v3, v5, v3, v3
 ; GFX7-NEXT:    v_mul_f32_e32 v5, v4, v3
 ; GFX7-NEXT:    v_fma_f32 v6, -v2, v5, v4
-; GFX7-NEXT:    s_getreg_b32 s4, hwreg(HW_REG_MODE, 4, 2)
 ; GFX7-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX7-NEXT:    v_fma_f32 v2, -v2, v5, v4
 ; GFX7-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll b/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
index 48abc49c41ae..6a9c4c8d41c2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
@@ -1661,5 +1661,132 @@ define amdgpu_gfx void @s_set_rounding_select_3_5(i32 inreg %cond) {
   ret void
 }
 
+define amdgpu_kernel void @get_rounding_after_set_rounding_1() {
+; GFX6-LABEL: get_rounding_after_set_rounding_1:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_nop 0
+; GFX6-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
+; GFX6-NEXT:    s_lshl_b32 s2, s0, 2
+; GFX6-NEXT:    s_mov_b32 s0, 0xeb24da71
+; GFX6-NEXT:    s_mov_b32 s1, 0xc96f385
+; GFX6-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX6-NEXT:    s_and_b32 s0, s0, 15
+; GFX6-NEXT:    s_add_i32 s1, s0, 4
+; GFX6-NEXT:    s_cmp_lt_u32 s0, 4
+; GFX6-NEXT:    s_cselect_b32 s4, s0, s1
+; GFX6-NEXT:    s_mov_b32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_mov_b32 s1, s0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: get_rounding_after_set_rounding_1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_nop 0
+; GFX7-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
+; GFX7-NEXT:    s_lshl_b32 s2, s0, 2
+; GFX7-NEXT:    s_mov_b32 s0, 0xeb24da71
+; GFX7-NEXT:    s_mov_b32 s1, 0xc96f385
+; GFX7-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX7-NEXT:    s_and_b32 s0, s0, 15
+; GFX7-NEXT:    s_add_i32 s1, s0, 4
+; GFX7-NEXT:    s_cmp_lt_u32 s0, 4
+; GFX7-NEXT:    s_cselect_b32 s4, s0, s1
+; GFX7-NEXT:    s_mov_b32 s0, 0
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_mov_b32 s1, s0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: get_rounding_after_set_rounding_1:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, 0
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
+; GFX8-NEXT:    s_lshl_b32 s2, s0, 2
+; GFX8-NEXT:    s_mov_b32 s0, 0xeb24da71
+; GFX8-NEXT:    s_mov_b32 s1, 0xc96f385
+; GFX8-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX8-NEXT:    s_and_b32 s0, s0, 15
+; GFX8-NEXT:    s_add_i32 s1, s0, 4
+; GFX8-NEXT:    s_cmp_lt_u32 s0, 4
+; GFX8-NEXT:    s_cselect_b32 s0, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: get_rounding_after_set_rounding_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
+; GFX9-NEXT:    s_lshl_b32 s2, s0, 2
+; GFX9-NEXT:    s_mov_b32 s0, 0xeb24da71
+; GFX9-NEXT:    s_mov_b32 s1, 0xc96f385
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX9-NEXT:    s_and_b32 s0, s0, 15
+; GFX9-NEXT:    s_add_i32 s1, s0, 4
+; GFX9-NEXT:    s_cmp_lt_u32 s0, 4
+; GFX9-NEXT:    s_cselect_b32 s0, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: get_rounding_after_set_rounding_1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_round_mode 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
+; GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-NEXT:    s_lshl_b32 s2, s0, 2
+; GFX10-NEXT:    s_mov_b32 s0, 0xeb24da71
+; GFX10-NEXT:    s_mov_b32 s1, 0xc96f385
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX10-NEXT:    s_and_b32 s0, s0, 15
+; GFX10-NEXT:    s_add_i32 s1, s0, 4
+; GFX10-NEXT:    s_cmp_lt_u32 s0, 4
+; GFX10-NEXT:    s_cselect_b32 s0, s0, s1
+; GFX10-NEXT:    v_mov_b32_e32 v2, s0
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: get_rounding_after_set_rounding_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_round_mode 0x0
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 4)
+; GFX11-NEXT:    s_lshl_b32 s2, s0, 2
+; GFX11-NEXT:    s_mov_b32 s0, 0xeb24da71
+; GFX11-NEXT:    s_mov_b32 s1, 0xc96f385
+; GFX11-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX11-NEXT:    s_and_b32 s0, s0, 15
+; GFX11-NEXT:    s_add_i32 s1, s0, 4
+; GFX11-NEXT:    s_cmp_lt_u32 s0, 4
+; GFX11-NEXT:    s_cselect_b32 s0, s0, s1
+; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT:    global_store_b32 v[0:1], v2, off dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+  tail call void @llvm.set.rounding(i32 1)
+  %set.mode = tail call i32 @llvm.get.rounding()
+  store volatile i32 %set.mode, ptr addrspace(1) null
+  ret void
+}
+
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; GCN: {{.*}}
-- 
GitLab


From 30cfe2b2ace51a8fa0eeb64f136e3999f87971ad Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Mon, 6 May 2024 12:02:15 -0700
Subject: [PATCH 0052/1206] [OpenACC] Implement 'async' clause sema for compute
 constructs

This is a pretty simple clause, it takes an 'async-argument', which
effectively needs to be just parsed as an 'int' argument, since it can
be an arbitrarly integer at runtime (and negative values are legal for
implementation defined values).

This patch also cleans up the async-argument parsing, so 'wait' got some
minor quality-of-life improvements for parsing (both clause and
    construct).
---
 clang/include/clang/AST/OpenACCClause.h       |  14 +-
 clang/include/clang/Basic/OpenACCClauses.def  |   1 +
 clang/include/clang/Parse/Parser.h            |   4 +-
 clang/include/clang/Sema/SemaOpenACC.h        |  10 ++
 clang/lib/AST/OpenACCClause.cpp               |  29 ++++
 clang/lib/AST/StmtProfile.cpp                 |   5 +
 clang/lib/AST/TextNodeDumper.cpp              |   1 +
 clang/lib/Parse/ParseOpenACC.cpp              |  35 +++--
 clang/lib/Sema/SemaOpenACC.cpp                |  40 ++++++
 clang/lib/Sema/TreeTransform.h                |  24 ++++
 clang/lib/Serialization/ASTReader.cpp         |   7 +-
 clang/lib/Serialization/ASTWriter.cpp         |   9 +-
 .../ast-print-openacc-compute-construct.cpp   |   8 ++
 clang/test/ParserOpenACC/parse-clauses.c      |   4 -
 clang/test/ParserOpenACC/parse-clauses.cpp    |   5 +-
 clang/test/ParserOpenACC/parse-wait-clause.c  |  18 ++-
 .../test/ParserOpenACC/parse-wait-construct.c |  22 ++-
 .../compute-construct-async-clause.c          |  41 ++++++
 .../compute-construct-async-clause.cpp        | 135 ++++++++++++++++++
 .../compute-construct-intexpr-clause-ast.cpp  |  72 ++++++++++
 clang/tools/libclang/CIndex.cpp               |   4 +
 21 files changed, 461 insertions(+), 27 deletions(-)
 create mode 100644 clang/test/SemaOpenACC/compute-construct-async-clause.c
 create mode 100644 clang/test/SemaOpenACC/compute-construct-async-clause.cpp

diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index ec6b4aebcb9f..e7b0b411b654 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -227,7 +227,8 @@ protected:
                                  SourceLocation EndLoc)
       : OpenACCClauseWithExprs(K, BeginLoc, LParenLoc, EndLoc),
         IntExpr(IntExpr) {
-    setExprs(MutableArrayRef<Expr *>{&this->IntExpr, 1});
+    if (IntExpr)
+      setExprs(MutableArrayRef<Expr *>{&this->IntExpr, 1});
   }
 
 public:
@@ -260,6 +261,17 @@ public:
          Expr *IntExpr, SourceLocation EndLoc);
 };
 
+class OpenACCAsyncClause : public OpenACCClauseWithSingleIntExpr {
+  OpenACCAsyncClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
+                     Expr *IntExpr, SourceLocation EndLoc);
+
+public:
+  static OpenACCAsyncClause *Create(const ASTContext &C,
+                                    SourceLocation BeginLoc,
+                                    SourceLocation LParenLoc, Expr *IntExpr,
+                                    SourceLocation EndLoc);
+};
+
 /// Represents a clause with one or more 'var' objects, represented as an expr,
 /// as its arguments. Var-list is expected to be stored in trailing storage.
 /// For now, we're just storing the original expression in its entirety, unlike
diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def
index c92e5eb1e1b6..8933e09b44f9 100644
--- a/clang/include/clang/Basic/OpenACCClauses.def
+++ b/clang/include/clang/Basic/OpenACCClauses.def
@@ -21,6 +21,7 @@
 #define CLAUSE_ALIAS(ALIAS_NAME, CLAUSE_NAME)
 #endif
 
+VISIT_CLAUSE(Async)
 VISIT_CLAUSE(Attach)
 VISIT_CLAUSE(Copy)
 CLAUSE_ALIAS(PCopy, Copy)
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index daefd4f28f01..532b5c125ef5 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3698,7 +3698,9 @@ private:
   bool ParseOpenACCDeviceTypeList();
   /// Parses the 'async-argument', which is an integral value with two
   /// 'special' values that are likely negative (but come from Macros).
-  ExprResult ParseOpenACCAsyncArgument();
+  OpenACCIntExprParseResult ParseOpenACCAsyncArgument(OpenACCDirectiveKind DK,
+                                                      OpenACCClauseKind CK,
+                                                      SourceLocation Loc);
   /// Parses the 'size-expr', which is an integral value, or an asterisk.
   bool ParseOpenACCSizeExpr();
   /// Parses a comma delimited list of 'size-expr's.
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 32d94ee8f33f..2cec2b73e918 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -101,16 +101,24 @@ public:
     unsigned getNumIntExprs() const {
       assert((ClauseKind == OpenACCClauseKind::NumGangs ||
               ClauseKind == OpenACCClauseKind::NumWorkers ||
+              ClauseKind == OpenACCClauseKind::Async ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
+      //
+      // 'async' has an optional IntExpr, so be tolerant of that.
+      if (ClauseKind == OpenACCClauseKind::Async &&
+          std::holds_alternative<std::monostate>(Details))
+        return 0;
       return std::get<IntExprDetails>(Details).IntExprs.size();
     }
 
     ArrayRef<Expr *> getIntExprs() {
       assert((ClauseKind == OpenACCClauseKind::NumGangs ||
               ClauseKind == OpenACCClauseKind::NumWorkers ||
+              ClauseKind == OpenACCClauseKind::Async ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
+
       return std::get<IntExprDetails>(Details).IntExprs;
     }
 
@@ -190,6 +198,7 @@ public:
     void setIntExprDetails(ArrayRef<Expr *> IntExprs) {
       assert((ClauseKind == OpenACCClauseKind::NumGangs ||
               ClauseKind == OpenACCClauseKind::NumWorkers ||
+              ClauseKind == OpenACCClauseKind::Async ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
       Details = IntExprDetails{{IntExprs.begin(), IntExprs.end()}};
@@ -197,6 +206,7 @@ public:
     void setIntExprDetails(llvm::SmallVector<Expr *> &&IntExprs) {
       assert((ClauseKind == OpenACCClauseKind::NumGangs ||
               ClauseKind == OpenACCClauseKind::NumWorkers ||
+              ClauseKind == OpenACCClauseKind::Async ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
       Details = IntExprDetails{std::move(IntExprs)};
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index c1affa97b781..ffa90884cef5 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -127,6 +127,26 @@ OpenACCVectorLengthClause::Create(const ASTContext &C, SourceLocation BeginLoc,
       OpenACCVectorLengthClause(BeginLoc, LParenLoc, IntExpr, EndLoc);
 }
 
+OpenACCAsyncClause::OpenACCAsyncClause(SourceLocation BeginLoc,
+                                       SourceLocation LParenLoc, Expr *IntExpr,
+                                       SourceLocation EndLoc)
+    : OpenACCClauseWithSingleIntExpr(OpenACCClauseKind::Async, BeginLoc,
+                                     LParenLoc, IntExpr, EndLoc) {
+  assert((!IntExpr || IntExpr->isInstantiationDependent() ||
+          IntExpr->getType()->isIntegerType()) &&
+         "Condition expression type not scalar/dependent");
+}
+
+OpenACCAsyncClause *OpenACCAsyncClause::Create(const ASTContext &C,
+                                               SourceLocation BeginLoc,
+                                               SourceLocation LParenLoc,
+                                               Expr *IntExpr,
+                                               SourceLocation EndLoc) {
+  void *Mem =
+      C.Allocate(sizeof(OpenACCAsyncClause), alignof(OpenACCAsyncClause));
+  return new (Mem) OpenACCAsyncClause(BeginLoc, LParenLoc, IntExpr, EndLoc);
+}
+
 OpenACCNumGangsClause *OpenACCNumGangsClause::Create(const ASTContext &C,
                                                      SourceLocation BeginLoc,
                                                      SourceLocation LParenLoc,
@@ -287,6 +307,15 @@ void OpenACCClausePrinter::VisitVectorLengthClause(
   OS << ")";
 }
 
+void OpenACCClausePrinter::VisitAsyncClause(const OpenACCAsyncClause &C) {
+  OS << "async";
+  if (C.hasIntExpr()) {
+    OS << "(";
+    printExpr(C.getIntExpr());
+    OS << ")";
+  }
+}
+
 void OpenACCClausePrinter::VisitPrivateClause(const OpenACCPrivateClause &C) {
   OS << "private(";
   llvm::interleaveComma(C.getVarList(), OS,
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 11d3f3d4cec4..0910471098c9 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2573,6 +2573,11 @@ void OpenACCClauseProfiler::VisitVectorLengthClause(
          "vector_length clause requires a valid int expr");
   Profiler.VisitStmt(Clause.getIntExpr());
 }
+
+void OpenACCClauseProfiler::VisitAsyncClause(const OpenACCAsyncClause &Clause) {
+  if (Clause.hasIntExpr())
+    Profiler.VisitStmt(Clause.getIntExpr());
+}
 } // namespace
 
 void StmtProfiler::VisitOpenACCComputeConstruct(
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 21167ca56e59..bf02d9545f84 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -397,6 +397,7 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
     case OpenACCClauseKind::Default:
       OS << '(' << cast<OpenACCDefaultClause>(C)->getDefaultClauseKind() << ')';
       break;
+    case OpenACCClauseKind::Async:
     case OpenACCClauseKind::Attach:
     case OpenACCClauseKind::Copy:
     case OpenACCClauseKind::PCopy:
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index b4b81e2ba13e..8c8330a5fad7 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -1081,7 +1081,12 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
         break;
       }
       case OpenACCClauseKind::Async: {
-        ExprResult AsyncArg = ParseOpenACCAsyncArgument();
+        ExprResult AsyncArg =
+            ParseOpenACCAsyncArgument(OpenACCDirectiveKind::Invalid,
+                                      OpenACCClauseKind::Async, ClauseLoc)
+                .first;
+        ParsedClause.setIntExprDetails(AsyncArg.isUsable() ? AsyncArg.get()
+                                                           : nullptr);
         if (AsyncArg.isInvalid()) {
           Parens.skipToEnd();
           return OpenACCCanContinue();
@@ -1120,8 +1125,10 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
 /// defined in the C header file and the Fortran openacc module. The special
 /// values are negative values, so as not to conflict with a user-specified
 /// nonnegative async-argument.
-ExprResult Parser::ParseOpenACCAsyncArgument() {
-  return getActions().CorrectDelayedTyposInExpr(ParseAssignmentExpression());
+Parser::OpenACCIntExprParseResult
+Parser::ParseOpenACCAsyncArgument(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
+                                  SourceLocation Loc) {
+  return ParseOpenACCIntExpr(DK, CK, Loc);
 }
 
 /// OpenACC 3.3, section 2.16:
@@ -1137,14 +1144,12 @@ bool Parser::ParseOpenACCWaitArgument(SourceLocation Loc, bool IsDirective) {
     // Consume colon.
     ConsumeToken();
 
-    ExprResult IntExpr =
-        ParseOpenACCIntExpr(IsDirective ? OpenACCDirectiveKind::Wait
-                                        : OpenACCDirectiveKind::Invalid,
-                            IsDirective ? OpenACCClauseKind::Invalid
-                                        : OpenACCClauseKind::Wait,
-                            Loc)
-            .first;
-    if (IntExpr.isInvalid())
+    OpenACCIntExprParseResult Res = ParseOpenACCIntExpr(
+        IsDirective ? OpenACCDirectiveKind::Wait
+                    : OpenACCDirectiveKind::Invalid,
+        IsDirective ? OpenACCClauseKind::Invalid : OpenACCClauseKind::Wait,
+        Loc);
+    if (Res.first.isInvalid() && Res.second == OpenACCParseCanContinue::Cannot)
       return true;
 
     if (ExpectAndConsume(tok::colon))
@@ -1172,9 +1177,13 @@ bool Parser::ParseOpenACCWaitArgument(SourceLocation Loc, bool IsDirective) {
     }
     FirstArg = false;
 
-    ExprResult CurArg = ParseOpenACCAsyncArgument();
+    OpenACCIntExprParseResult Res = ParseOpenACCAsyncArgument(
+        IsDirective ? OpenACCDirectiveKind::Wait
+                    : OpenACCDirectiveKind::Invalid,
+        IsDirective ? OpenACCClauseKind::Invalid : OpenACCClauseKind::Wait,
+        Loc);
 
-    if (CurArg.isInvalid())
+    if (Res.first.isInvalid() && Res.second == OpenACCParseCanContinue::Cannot)
       return true;
   }
 
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index 8cf829cf215b..b1086baa3ae2 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -198,6 +198,25 @@ bool doesClauseApplyToDirective(OpenACCDirectiveKind DirectiveKind,
     default:
       return false;
     }
+  case OpenACCClauseKind::Async:
+    switch (DirectiveKind) {
+    case OpenACCDirectiveKind::Parallel:
+    case OpenACCDirectiveKind::Serial:
+    case OpenACCDirectiveKind::Kernels:
+    case OpenACCDirectiveKind::Data:
+    case OpenACCDirectiveKind::EnterData:
+    case OpenACCDirectiveKind::ExitData:
+    case OpenACCDirectiveKind::Set:
+    case OpenACCDirectiveKind::Update:
+    case OpenACCDirectiveKind::Wait:
+    case OpenACCDirectiveKind::ParallelLoop:
+    case OpenACCDirectiveKind::SerialLoop:
+    case OpenACCDirectiveKind::KernelsLoop:
+      return true;
+    default:
+      return false;
+    }
+
   default:
     // Do nothing so we can go to the 'unimplemented' diagnostic instead.
     return true;
@@ -398,6 +417,27 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
         getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
         Clause.getIntExprs()[0], Clause.getEndLoc());
   }
+  case OpenACCClauseKind::Async: {
+    // Restrictions only properly implemented on 'compute' constructs, and
+    // 'compute' constructs are the only construct that can do anything with
+    // this yet, so skip/treat as unimplemented in this case.
+    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+      break;
+
+    // There is no prose in the standard that says duplicates aren't allowed,
+    // but this diagnostic is present in other compilers, as well as makes
+    // sense.
+    if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause))
+      return nullptr;
+
+    assert(Clause.getNumIntExprs() < 2 &&
+           "Invalid number of expressions for Async");
+
+    return OpenACCAsyncClause::Create(
+        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
+        Clause.getNumIntExprs() != 0 ? Clause.getIntExprs()[0] : nullptr,
+        Clause.getEndLoc());
+  }
   case OpenACCClauseKind::Private: {
     // Restrictions only properly implemented on 'compute' constructs, and
     // 'compute' constructs are the only construct that can do anything with
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index a4ca8b5771a9..2d6d6dae680c 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -11401,6 +11401,30 @@ void OpenACCClauseTransform<Derived>::VisitVectorLengthClause(
       ParsedClause.getLParenLoc(), ParsedClause.getIntExprs()[0],
       ParsedClause.getEndLoc());
 }
+
+template <typename Derived>
+void OpenACCClauseTransform<Derived>::VisitAsyncClause(
+    const OpenACCAsyncClause &C) {
+  if (C.hasIntExpr()) {
+    ExprResult Res = Self.TransformExpr(const_cast<Expr *>(C.getIntExpr()));
+    if (!Res.isUsable())
+      return;
+
+    Res = Self.getSema().OpenACC().ActOnIntExpr(OpenACCDirectiveKind::Invalid,
+                                                C.getClauseKind(),
+                                                C.getBeginLoc(), Res.get());
+    if (!Res.isUsable())
+      return;
+    ParsedClause.setIntExprDetails(Res.get());
+  }
+
+  NewClause = OpenACCAsyncClause::Create(
+      Self.getSema().getASTContext(), ParsedClause.getBeginLoc(),
+      ParsedClause.getLParenLoc(),
+      ParsedClause.getNumIntExprs() != 0 ? ParsedClause.getIntExprs()[0]
+                                         : nullptr,
+      ParsedClause.getEndLoc());
+}
 } // namespace
 template <typename Derived>
 OpenACCClause *TreeTransform<Derived>::TransformOpenACCClause(
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 81b78edd9c6c..b4b2f999d225 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11881,6 +11881,12 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
     return OpenACCCreateClause::Create(getContext(), ClauseKind, BeginLoc,
                                        LParenLoc, IsZero, VarList, EndLoc);
   }
+  case OpenACCClauseKind::Async: {
+    SourceLocation LParenLoc = readSourceLocation();
+    Expr *AsyncExpr = readBool() ? readSubExpr() : nullptr;
+    return OpenACCAsyncClause::Create(getContext(), BeginLoc, LParenLoc,
+                                      AsyncExpr, EndLoc);
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -11904,7 +11910,6 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
   case OpenACCClauseKind::DefaultAsync:
   case OpenACCClauseKind::DeviceType:
   case OpenACCClauseKind::DType:
-  case OpenACCClauseKind::Async:
   case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Wait:
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 42da50abdc68..ce2ea4e3d614 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -7908,6 +7908,14 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeOpenACCVarList(CC);
     return;
   }
+  case OpenACCClauseKind::Async: {
+    const auto *AC = cast<OpenACCAsyncClause>(C);
+    writeSourceLocation(AC->getLParenLoc());
+    writeBool(AC->hasIntExpr());
+    if (AC->hasIntExpr())
+      AddStmt(const_cast<Expr *>(AC->getIntExpr()));
+    return;
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -7931,7 +7939,6 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   case OpenACCClauseKind::DefaultAsync:
   case OpenACCClauseKind::DeviceType:
   case OpenACCClauseKind::DType:
-  case OpenACCClauseKind::Async:
   case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Wait:
diff --git a/clang/test/AST/ast-print-openacc-compute-construct.cpp b/clang/test/AST/ast-print-openacc-compute-construct.cpp
index 1ee1e15bdfc3..13597543e9b6 100644
--- a/clang/test/AST/ast-print-openacc-compute-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-compute-construct.cpp
@@ -75,5 +75,13 @@ void foo() {
   // CHECK: #pragma acc kernels deviceptr(iPtr, arrayPtr[0])
 #pragma acc kernels deviceptr(iPtr, arrayPtr[0])
   while(true);
+
+  // CHECK: #pragma acc kernels async(*iPtr)
+#pragma acc kernels async(*iPtr)
+  while(true);
+
+  // CHECK: #pragma acc kernels async
+#pragma acc kernels async
+  while(true);
 }
 
diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c
index 035b7ab4c1f4..51858b441e93 100644
--- a/clang/test/ParserOpenACC/parse-clauses.c
+++ b/clang/test/ParserOpenACC/parse-clauses.c
@@ -1233,7 +1233,6 @@ void device_type() {
 
 #define acc_async_sync -1
 void AsyncArgument() {
-  // expected-warning@+1{{OpenACC clause 'async' not yet implemented, clause ignored}}
 #pragma acc parallel async
   {}
 
@@ -1250,15 +1249,12 @@ void AsyncArgument() {
 #pragma acc parallel async(4, 3)
   {}
 
-  // expected-warning@+1{{OpenACC clause 'async' not yet implemented, clause ignored}}
 #pragma acc parallel async(returns_int())
   {}
 
-  // expected-warning@+1{{OpenACC clause 'async' not yet implemented, clause ignored}}
 #pragma acc parallel async(5)
   {}
 
-  // expected-warning@+1{{OpenACC clause 'async' not yet implemented, clause ignored}}
 #pragma acc parallel async(acc_async_sync)
   {}
 }
diff --git a/clang/test/ParserOpenACC/parse-clauses.cpp b/clang/test/ParserOpenACC/parse-clauses.cpp
index 8c1d64374799..702eb75ca890 100644
--- a/clang/test/ParserOpenACC/parse-clauses.cpp
+++ b/clang/test/ParserOpenACC/parse-clauses.cpp
@@ -18,13 +18,14 @@ void templ() {
 #pragma acc parallel vector_length(I)
   for(;;){}
 
-  // expected-warning@+1{{OpenACC clause 'async' not yet implemented, clause ignored}}
 #pragma acc parallel async(T::value)
   for(;;){}
 
-  // expected-warning@+1{{OpenACC clause 'async' not yet implemented, clause ignored}}
 #pragma acc parallel async(I)
   for(;;){}
+
+#pragma acc parallel async
+  for(;;){}
 }
 
 struct S {
diff --git a/clang/test/ParserOpenACC/parse-wait-clause.c b/clang/test/ParserOpenACC/parse-wait-clause.c
index f3e651de4583..64f5b9c8fd73 100644
--- a/clang/test/ParserOpenACC/parse-wait-clause.c
+++ b/clang/test/ParserOpenACC/parse-wait-clause.c
@@ -84,29 +84,35 @@ void func() {
   #pragma acc parallel wait (devnum: i + j:queues:) clause-list
     {}
 
-  // expected-error@+3{{use of undeclared identifier 'devnum'}}
+  // expected-error@+4{{use of undeclared identifier 'devnum'}}
+  // expected-error@+3{{expected ','}}
   // expected-error@+2{{expected ')'}}
   // expected-note@+1{{to match this '('}}
   #pragma acc parallel wait (queues:devnum: i + j
     {}
 
+  // expected-error@+2{{expected ','}}
   // expected-error@+1{{use of undeclared identifier 'devnum'}}
   #pragma acc parallel wait (queues:devnum: i + j)
     {}
 
+  // expected-error@+3{{expected ','}}
   // expected-error@+2{{use of undeclared identifier 'devnum'}}
   // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc parallel wait (queues:devnum: i + j) clause-list
     {}
 
+  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{expected ')'}}
   // expected-note@+1{{to match this '('}}
   #pragma acc parallel wait(i, j, 1+1, 3.3
     {}
 
+  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait(i, j, 1+1, 3.3)
     {}
+  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{invalid OpenACC clause 'clause'}}
   // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait(i, j, 1+1, 3.3) clause-list
@@ -127,45 +133,55 @@ void func() {
   #pragma acc parallel wait(,) clause-list
     {}
 
+  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{expected ')'}}
   // expected-note@+1{{to match this '('}}
   #pragma acc parallel wait(queues:i, j, 1+1, 3.3
     {}
 
+  // expected-error@+4{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+3{{expected expression}}
   // expected-error@+2{{expected ')'}}
   // expected-note@+1{{to match this '('}}
   #pragma acc parallel wait(queues:i, j, 1+1, 3.3,
     {}
 
+  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait(queues:i, j, 1+1, 3.3)
     {}
 
+  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{invalid OpenACC clause 'clause'}}
   // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait(queues:i, j, 1+1, 3.3) clause-list
     {}
 
+  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{expected ')'}}
   // expected-note@+1{{to match this '('}}
   #pragma acc parallel wait(devnum:3:i, j, 1+1, 3.3
     {}
+  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait(devnum:3:i, j, 1+1, 3.3)
     {}
+  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{invalid OpenACC clause 'clause'}}
   // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait(devnum:3:i, j, 1+1, 3.3) clause-list
     {}
 
+  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{expected ')'}}
   // expected-note@+1{{to match this '('}}
   #pragma acc parallel wait(devnum:3:queues:i, j, 1+1, 3.3
     {}
+  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait(devnum:3:queues:i, j, 1+1, 3.3)
     {}
+  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{invalid OpenACC clause 'clause'}}
   // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait(devnum:3:queues:i, j, 1+1, 3.3) clause-list
diff --git a/clang/test/ParserOpenACC/parse-wait-construct.c b/clang/test/ParserOpenACC/parse-wait-construct.c
index 30a9fc8c12a4..8f7ea8efd576 100644
--- a/clang/test/ParserOpenACC/parse-wait-construct.c
+++ b/clang/test/ParserOpenACC/parse-wait-construct.c
@@ -76,28 +76,34 @@ void func() {
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait (devnum: i + j:queues:) clause-list
 
-  // expected-error@+4{{use of undeclared identifier 'devnum'}}
+  // expected-error@+5{{use of undeclared identifier 'devnum'}}
+  // expected-error@+4{{expected ','}}
   // expected-error@+3{{expected ')'}}
   // expected-note@+2{{to match this '('}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait (queues:devnum: i + j
 
-  // expected-error@+2{{use of undeclared identifier 'devnum'}}
+  // expected-error@+3{{use of undeclared identifier 'devnum'}}
+  // expected-error@+2{{expected ','}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait (queues:devnum: i + j)
 
-  // expected-error@+3{{use of undeclared identifier 'devnum'}}
+  // expected-error@+4{{use of undeclared identifier 'devnum'}}
+  // expected-error@+3{{expected ','}}
   // expected-error@+2{{invalid OpenACC clause 'clause'}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait (queues:devnum: i + j) clause-list
 
+  // expected-error@+4{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+3{{expected ')'}}
   // expected-note@+2{{to match this '('}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(i, j, 1+1, 3.3
 
+  // expected-error@+2{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(i, j, 1+1, 3.3)
+  // expected-error@+3{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{invalid OpenACC clause 'clause'}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(i, j, 1+1, 3.3) clause-list
@@ -117,40 +123,50 @@ void func() {
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(,) clause-list
 
+  // expected-error@+4{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+3{{expected ')'}}
   // expected-note@+2{{to match this '('}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(queues:i, j, 1+1, 3.3
 
+  // expected-error@+5{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+4{{expected expression}}
   // expected-error@+3{{expected ')'}}
   // expected-note@+2{{to match this '('}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(queues:i, j, 1+1, 3.3,
 
+  // expected-error@+2{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(queues:i, j, 1+1, 3.3)
 
+  // expected-error@+3{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{invalid OpenACC clause 'clause'}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(queues:i, j, 1+1, 3.3) clause-list
 
+  // expected-error@+4{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+3{{expected ')'}}
   // expected-note@+2{{to match this '('}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(devnum:3:i, j, 1+1, 3.3
+  // expected-error@+2{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(devnum:3:i, j, 1+1, 3.3)
+  // expected-error@+3{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{invalid OpenACC clause 'clause'}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(devnum:3:i, j, 1+1, 3.3) clause-list
 
+  // expected-error@+4{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+3{{expected ')'}}
   // expected-note@+2{{to match this '('}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(devnum:3:queues:i, j, 1+1, 3.3
+  // expected-error@+2{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(devnum:3:queues:i, j, 1+1, 3.3)
+  // expected-error@+3{{OpenACC directive 'wait' requires expression of integer type ('double' invalid)}}
   // expected-error@+2{{invalid OpenACC clause 'clause'}}
   // expected-warning@+1{{OpenACC construct 'wait' not yet implemented, pragma ignored}}
   #pragma acc wait(devnum:3:queues:i, j, 1+1, 3.3) clause-list
diff --git a/clang/test/SemaOpenACC/compute-construct-async-clause.c b/clang/test/SemaOpenACC/compute-construct-async-clause.c
new file mode 100644
index 000000000000..a8af06bc0afd
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-async-clause.c
@@ -0,0 +1,41 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+short getS();
+
+void Test() {
+#pragma acc parallel async
+  while(1);
+#pragma acc parallel async(1)
+  while(1);
+#pragma acc kernels async(1)
+  while(1);
+#pragma acc kernels async(-51)
+  while(1);
+
+#pragma acc serial async(1)
+  while(1);
+
+  // expected-error@+2{{expected ')'}}
+  // expected-note@+1{{to match this '('}}
+#pragma acc serial async(1, 2)
+  while(1);
+
+  struct NotConvertible{} NC;
+  // expected-error@+1{{OpenACC clause 'async' requires expression of integer type ('struct NotConvertible' invalid)}}
+#pragma acc parallel async(NC)
+  while(1);
+
+#pragma acc kernels async(getS())
+  while(1);
+
+  struct Incomplete *SomeIncomplete;
+
+  // expected-error@+1{{OpenACC clause 'async' requires expression of integer type ('struct Incomplete' invalid)}}
+#pragma acc kernels async(*SomeIncomplete)
+  while(1);
+
+  enum E{A} SomeE;
+
+#pragma acc kernels async(SomeE)
+  while(1);
+}
diff --git a/clang/test/SemaOpenACC/compute-construct-async-clause.cpp b/clang/test/SemaOpenACC/compute-construct-async-clause.cpp
new file mode 100644
index 000000000000..a5da7c8f4e56
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-async-clause.cpp
@@ -0,0 +1,135 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct NotConvertible{} NC;
+struct Incomplete *SomeIncomplete; // #INCOMPLETE
+enum E{} SomeE;
+enum class E2{} SomeE2;
+
+struct CorrectConvert {
+  operator int();
+} Convert;
+
+struct ExplicitConvertOnly {
+  explicit operator int() const; // #EXPL_CONV
+} Explicit;
+
+struct AmbiguousConvert{
+  operator int(); // #AMBIG_INT
+  operator short(); // #AMBIG_SHORT
+  operator float();
+} Ambiguous;
+
+void Test() {
+#pragma acc parallel async
+  while(1);
+#pragma acc parallel async(1)
+  while(1);
+#pragma acc kernels async(-51)
+  while(1);
+
+  // expected-error@+1{{OpenACC clause 'async' requires expression of integer type ('struct NotConvertible' invalid}}
+#pragma acc parallel async(NC)
+  while(1);
+
+  // expected-error@+2{{OpenACC integer expression has incomplete class type 'struct Incomplete'}}
+  // expected-note@#INCOMPLETE{{forward declaration of 'Incomplete'}}
+#pragma acc kernels async(*SomeIncomplete)
+  while(1);
+
+#pragma acc parallel async(SomeE)
+  while(1);
+
+  // expected-error@+1{{OpenACC clause 'async' requires expression of integer type ('enum E2' invalid}}
+#pragma acc kernels async(SomeE2)
+  while(1);
+
+#pragma acc parallel async(Convert)
+  while(1);
+
+  // expected-error@+2{{OpenACC integer expression type 'struct ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+#pragma acc kernels async(Explicit)
+  while(1);
+
+  // expected-error@+3{{multiple conversions from expression type 'struct AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel async(Ambiguous)
+  while(1);
+}
+
+struct HasInt {
+  using IntTy = int;
+  using ShortTy = short;
+  static constexpr int value = 1;
+  static constexpr AmbiguousConvert ACValue;
+  static constexpr ExplicitConvertOnly EXValue;
+
+  operator char();
+};
+
+template<typename T>
+void TestInst() {
+
+  // expected-error@+1{{no member named 'Invalid' in 'HasInt'}}
+#pragma acc parallel async(HasInt::Invalid)
+  while (1);
+
+  // expected-error@+2{{no member named 'Invalid' in 'HasInt'}}
+  // expected-note@#INST{{in instantiation of function template specialization 'TestInst<HasInt>' requested here}}
+#pragma acc kernels async(T::Invalid)
+  while (1);
+
+  // expected-error@+3{{multiple conversions from expression type 'const AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel async(HasInt::ACValue)
+  while (1);
+
+  // expected-error@+3{{multiple conversions from expression type 'const AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc kernels async(T::ACValue)
+  while (1);
+
+  // expected-error@+2{{OpenACC integer expression type 'const ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+#pragma acc parallel async(HasInt::EXValue)
+  while (1);
+
+  // expected-error@+2{{OpenACC integer expression type 'const ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+#pragma acc kernels async(T::EXValue)
+  while (1);
+
+#pragma acc parallel async(HasInt::value)
+  while (1);
+
+#pragma acc kernels async(T::value)
+  while (1);
+
+#pragma acc parallel async(HasInt::IntTy{})
+  while (1);
+
+#pragma acc kernels async(typename T::ShortTy{})
+  while (1);
+
+#pragma acc parallel async(HasInt::IntTy{})
+  while (1);
+
+#pragma acc kernels async(typename T::ShortTy{})
+  while (1);
+
+  HasInt HI{};
+  T MyT{};
+
+#pragma acc parallel async(HI)
+  while (1);
+
+#pragma acc kernels async(MyT)
+  while (1);
+}
+
+void Inst() {
+  TestInst<HasInt>(); // #INST
+}
diff --git a/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp b/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp
index 5a4c9f05ee08..b85de56c7ae9 100644
--- a/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp
+++ b/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp
@@ -116,8 +116,28 @@ void NormalUses() {
   // CHECK-NEXT: WhileStmt
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: CompoundStmt
+
+#pragma acc kernels async(some_int())
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
+  // CHECK-NEXT: async clause
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc kernels async
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
+  // CHECK-NEXT: async clause
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
 }
 
+
 template<typename T, typename U>
 void TemplUses(T t, U u) {
   // CHECK-NEXT: FunctionTemplateDecl
@@ -235,6 +255,33 @@ void TemplUses(T t, U u) {
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: CompoundStmt
 
+#pragma acc kernels async
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
+  // CHECK-NEXT: async clause
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc kernels async(u)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
+  // CHECK-NEXT: async clause
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel async (U::value)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: async clause
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
 
   // CHECK-NEXT: DeclStmt
   // CHECK-NEXT: VarDecl{{.*}}EndMarker
@@ -365,6 +412,31 @@ void TemplUses(T t, U u) {
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: CompoundStmt
 
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
+  // CHECK-NEXT: async clause
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
+  // CHECK-NEXT: async clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: async clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
   // CHECK-NEXT: DeclStmt
   // CHECK-NEXT: VarDecl{{.*}}EndMarker
 }
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 6c07c4d2e307..b845a381d63b 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2847,6 +2847,10 @@ void OpenACCClauseEnqueue::VisitDevicePtrClause(
     const OpenACCDevicePtrClause &C) {
   VisitVarList(C);
 }
+void OpenACCClauseEnqueue::VisitAsyncClause(const OpenACCAsyncClause &C) {
+  if (C.hasIntExpr())
+    Visitor.AddStmt(C.getIntExpr());
+}
 } // namespace
 
 void EnqueueVisitor::EnqueueChildren(const OpenACCClause *C) {
-- 
GitLab


From 099417d617cf44711377d02eedc580a0c11297e9 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Tue, 7 May 2024 16:15:09 +0200
Subject: [PATCH 0053/1206] [mlir][NFC] Improve bufferization documentation
 (#89495)

* Add example for `test-analysis-only` and `print-conflicts`.
* Mention other bufferization-related passes.
* Update outdated documentation.
---
 mlir/docs/Bufferization.md                    | 304 ++++++++++++------
 .../includes/img/bufferization_passes.svg     |   1 +
 .../img/bufferization_tensor_insert_dst.svg   |   1 +
 3 files changed, 216 insertions(+), 90 deletions(-)
 create mode 100644 mlir/docs/includes/img/bufferization_passes.svg
 create mode 100644 mlir/docs/includes/img/bufferization_tensor_insert_dst.svg

diff --git a/mlir/docs/Bufferization.md b/mlir/docs/Bufferization.md
index 808535822212..6a49bea9c68c 100644
--- a/mlir/docs/Bufferization.md
+++ b/mlir/docs/Bufferization.md
@@ -5,35 +5,45 @@
 ## Overview
 
 Bufferization in MLIR is the process of converting ops with `tensor` semantics
-to ops with `memref` semantics. MLIR provides an infrastructure that bufferizes
-an entire program in a single pass (*One-Shot Bufferize*). This infrastructure
-bufferizes all ops that implement the
-[`BufferizableOpInterface`](https://github.com/llvm/llvm-project/blob/17a68065c378da74805e4e1b9a5b78cc9f83e580/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td)
-can be bufferized.
-
-MLIR has an older bufferization infrastructure built around
-[dialect conversion](DialectConversion.md). Most dialect conversion
-bufferization patterns have been migrated to One-Shot Bufferize, but some
-functionality such as function boundary bufferization still depends on dialect
-conversion and its type converter. New projects should use One-Shot Bufferize,
-as the dialect conversion-based bufferization will eventually be deprecated.
-Moreover, One-Shot Bufferize results in better bufferization with fewer memory
-allocations and buffer copies. This documentation is mostly about One-Shot
-Bufferize, but also describes how to gradually migrate a project from dialect
-conversion-based bufferization to One-Shot Bufferize.
+to ops with `memref` semantics. There are multiple MLIR passes that are related
+to bufferization. These passes typically run as one of the last steps in a
+pass pipeline, right before lowering to `memref` ops to LLVM. That is because
+many transformations are easier or only supported in tensor land; e.g.,
+[tile/fuse/… on tensors first](https://llvm.discourse.group/t/rfc-linalg-on-tensors-update-and-comprehensive-bufferization-rfc/3373),
+then bufferize the remaining IR.
+
+![bufferization passes](/includes/img/bufferization_passes.svg)
+
+The most important bufferization pass is *One-Shot Bufferize*: This pass
+rewrites `tensor` IR to `memref` IR. There are additional helper passes that
+preprocess IR (e.g., so that IR can be bufferized more efficiently), perform
+buffer-level optimizations such as allocation hoisting, and
+[insert buffer deallocation ops](OwnershipBasedBufferDeallocation.md) so that
+the resulting `memref` IR has no memory leaks.
+
+## Deprecated Passes
+
+The old dialect conversion-based bufferization passes have been deprecated and
+should not be used anymore. Most of those passes have already been removed from
+MLIR. One-Shot Bufferize produces in better bufferization results with fewer
+memory allocations and buffer copies.
+
+The buffer deallocation pass has been deprecated in favor of the ownership-based
+buffer deallocation pipeline. The deprecated pass has some limitations that may
+cause memory leaks in the resulting IR.
 
 ## What is One-Shot Bufferize?
 
-One-Shot Bufferize is a new tensor bufferization pass designed for IR in
+One-Shot Bufferize is a tensor bufferization pass designed for IR in
 [destination-passing style](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/11/dps-fhpc17.pdf),
 and with aggressive in-place bufferization.
 
 One-Shot Bufferize is:
 
-*   **Monolithic**: A single MLIR pass does the entire work, whereas the
-    previous bufferization in MLIR was split across multiple passes residing in
-    different dialects. In One-Shot Bufferize, `BufferizableOpInterface`
-    implementations are spread across different dialects.
+*   **Monolithic**: A single MLIR pass does the entire work.
+
+*   **Extensible** via an op interface: All ops that implement
+    `BufferizableOpInterface` can be bufferized.
 
 *   A **whole-function at a time analysis**. In-place bufferization decisions
     are made by analyzing SSA use-def chains on tensors. Op interface
@@ -41,10 +51,7 @@ One-Shot Bufferize is:
     ops, but also helper methods for One-Shot Bufferize's analysis to query
     information about an op's bufferization/memory semantics.
 
-*   **Extensible** via an op interface: All ops that implement
-    `BufferizableOpInterface` can be bufferized.
-
-*   **2-Pass**: Bufferization is internally broken down into 2 steps: First,
+*   **2-Phase**: Bufferization is internally broken down into 2 steps: First,
     analyze the entire IR and make bufferization decisions. Then, bufferize
     (rewrite) the IR. The analysis has access to exact SSA use-def information.
     It incrementally builds alias and equivalence sets and does not rely on a
@@ -60,27 +67,17 @@ One-Shot Bufferize is:
     of `AnalysisState` that implements a small number virtual functions can
     serve as a custom analysis. It is even possible to run One-Shot Bufferize
     without any analysis (`AlwaysCopyAnalysisState`), in which case One-Shot
-    Bufferize behaves exactly like the old dialect conversion-based
-    bufferization (i.e., copy every buffer before writing to it).
+    Bufferize copies every buffer before writing to it.
 
-To reduce complexity, One-Shot Bufferize should be
-[run after other transformations](https://llvm.discourse.group/t/rfc-linalg-on-tensors-update-and-comprehensive-bufferization-rfc/3373),
-typically as one of the last steps right before lowering memref ops. Many
-transformations are easier in tensor land; e.g., tile/fuse/… on tensors first,
-then bufferize the remaining IR.
-
-From an architecture perspective, One-Shot Bufferize consists of
-[BufferizableOpInterface](https://github.com/llvm/llvm-project/blob/17a68065c378da74805e4e1b9a5b78cc9f83e580/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td)
-(and its implementations) and an
-[analysis](https://github.com/llvm/llvm-project/blob/ae2764e835a26bad9774803eca0a6530df2a3e2d/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h#L164)
-of tensor SSA values that decides if a buffer can be used directly or must be
-copied. The [bufferize] method of the op interface inspects analysis results and
-rewrites tensor ops into memref ops.
+Note that One-Shot Bufferize does not deallocate buffers. That is done by the
+[Ownership-based Buffer Deallocation passes](OwnershipBasedBufferDeallocation.md).
 
 ## Goals of Bufferization
 
-The high-level goal of every bufferization technique is to: 1. Use as little
-memory as possible. 2. Copy as little memory as possible.
+The high-level goal of every bufferization technique is to:
+
+1. Use as little memory as possible.
+2. Copy as little memory as possible.
 
 This implies reusing already allocated buffers when possible, turning
 bufferization into an algorithmically complex problem with similarities to
@@ -102,40 +99,46 @@ choosing an already existing buffer, we must be careful not to accidentally
 overwrite data that is still needed later in the program.
 
 To simplify this problem, One-Shot Bufferize was designed to take advantage of
-*destination-passing style*. This form exists in itself independently of
-bufferization and is tied to SSA semantics: many ops are “updating” part of
-their input SSA variable. For example the LLVM instruction
+*destination-passing style* (DPS). In MLIR, DPS op should implement the
+[`DestinationStyleOpInterface`](https://github.com/llvm/llvm-project/blob/792d437b56adfb3416daf8105942d4899fb82763/mlir/include/mlir/Interfaces/DestinationStyleOpInterface.td).
+DPS exists in itself independently of bufferization and is tied to SSA
+semantics: many ops are "updating" a part of their input SSA variables. For
+example the LLVM instruction
 [`insertelement`](https://llvm.org/docs/LangRef.html#insertelement-instruction)
 is inserting an element inside a vector. Since SSA values are immutable, the
 operation returns a copy of the input vector with the element inserted.
-Another example in MLIR is `linalg.generic`, which always has an extra `outs`
-operand which provides the initial values to update (for example when the
-operation is doing a reduction).
+Another example in MLIR is `linalg.generic` on tensors, which always has an
+extra `outs` operand for each result, which provides the initial values to
+update (for example when the operation is doing a reduction).
 
-This input is referred to as "destination" in the following (quotes are
+`outs` operands are referred to as "destinations" in the following (quotes are
 important as this operand isn't modified in place but copied) and comes into
 place in the context of bufferization as a possible "anchor" for the
 bufferization algorithm. This allows the user to shape the input in a form that
 guarantees close to optimal bufferization result when carefully choosing the
 SSA value used as "destination".
 
-For every tensor result, a "destination-passing" style op has a corresponding
-tensor operand. If there aren't any other uses of this tensor, the bufferization
-can alias it with the op result and perform the operation "in-place" by reusing
-the buffer allocated for this "destination" input.
+For every tensor result, a DPS op has a corresponding tensor operand. If there
+aren't any other conflicting uses of this tensor, the bufferization can alias
+it with the op result and perform the operation "in-place" by reusing the buffer
+allocated for this "destination" input.
 
-As an example, consider the following op: `%0 = tensor.insert %cst into
-%t[%idx] : tensor<?xf32>`
+As an example, consider the following op: `%r = tensor.insert %f into
+%t[%idx] : tensor<5xf32>`
+
+![tensor.insert example](/includes/img/bufferization_tensor_insert_dst.svg)
 
 `%t` is the "destination" in this example. When choosing a buffer for the result
-`%0`, denoted as `buffer(%0)`, One-Shot Bufferize considers only two options:
+`%r`, denoted as `buffer(%r)`, One-Shot Bufferize considers only two options:
 
-1.  `buffer(%0) = buffer(%t)` : alias the "destination" tensor with the
-    result and perform the operation in-place.
-2.  `buffer(%0)` is a newly allocated buffer.
+1.  `buffer(%r) = buffer(%t)`: store the result in the existing `buffer(%t)`.
+    Note that this is not always possible. E.g., if the old contents of
+    `buffer(%t)` are still needed. One-Shot Bufferize's main task is to detect
+    such cases and fall back to the second option when necessary.
+2.  `buffer(%r)` is a newly allocated buffer.
 
 There may be other buffers in the same function that could potentially be used
-for `buffer(%0)`, but those are not considered by One-Shot Bufferize to keep the
+for `buffer(%r)`, but those are not considered by One-Shot Bufferize to keep the
 bufferization simple. One-Shot Bufferize could be extended to consider such
 buffers in the future to achieve a better quality of bufferization.
 
@@ -151,7 +154,7 @@ memory allocation. E.g.:
 ```
 
 The result of `tensor.generate` does not have a "destination" operand, so
-bufferization allocates a new buffer. This could be avoided by choosing an
+bufferization allocates a new buffer. This could be avoided by instead using an
 op such as `linalg.generic`, which can express the same computation with a
 "destination" operand, as specified behind outputs (`outs`):
 
@@ -198,12 +201,61 @@ e.g.:
 ```mlir
 %0 = "my_dialect.some_op"(%t) : (tensor<?xf32>) -> (tensor<?xf32>)
 %1 = "my_dialect.another_op"(%0) : (tensor<?xf32>) -> (tensor<?xf32>)
+
+// "yet_another_op" likely needs to read the data of %0, so "another_op" cannot
+// in-place write to buffer(%0).
 %2 = "my_dialect.yet_another_op"(%0) : (tensor<?xf32>) -> (tensor<?xf32>)
 ```
 
-One-Shot Bufferize has debug flags (`test-analysis-only print-conflicts`) that
-print the results of the analysis and explain to the user why buffer copies were
-inserted.
+## Tensor / MemRef Boundary
+
+The bufferization dialect provides a few helper ops to connect tensor IR (that
+should be bufferized) with existing buffers (that may be allocated/provided by
+a different runtime/library/etc.).
+
+`bufferization.to_memref %t` returns the future buffer of a tensor SSA value.
+`bufferization.to_tensor %m` returns a tensor SSA value for a given MemRef
+buffer. `bufferization.materialize_in_destination` indicates that a tensor value
+should materialize in a certain buffer.
+
+Consider the following example, where a TOSA matmul result should materialize in
+an existing buffer `%C`:
+
+```mlir
+// Batched TOSA matrix multiplication. %A and %B are the
+// inputs, %C is the output.
+func.func @test_matmul(%A: memref<1x17x19xf32>,
+                       %B: memref<1x19x29xf32>,
+                       %C: memref<1x17x29xf32>) {
+
+  %A_tensor = bufferization.to_tensor %A restrict : memref<1x17x19xf32>
+  %B_tensor = bufferization.to_tensor %B restrict : memref<1x19x29xf32>
+
+  %0 = tosa.matmul %A_tensor, %B_tensor
+      : (tensor<1x17x19xf32>, tensor<1x19x29xf32>) ->
+         tensor<1x17x29xf32>
+
+  bufferization.materialize_in_destination
+    %0 in restrict writable %C
+      : (tensor<1x17x29xf32>, memref<1x17x29xf32>) -> ()
+
+  return
+}
+```
+
+Note that all bufferization ops in this example have the `restrict` unit
+attribute set. This attribute is similar to the C restrict keyword and indicates
+that there is no other `to_tensor` or `materialize_in_destination` op with
+the same or an aliasing MemRef operand. Only such
+`to_tensor`/`materialize_in_destination` ops are supported. The `restrict`
+attribute gives strong aliasing guarantees to the bufferization analysis and
+allows us to look only at the tensor IR in a program. (Ops that do not operate
+on tensors are ignored by the One-Shot Bufferize.)
+
+Also note that `tosa.matmul` cannot be bufferized as is: there is no
+`BufferizableOpInterface` implementation for that op. However, the op can be
+lowered to a combination of `tensor.empty` and `linalg.matmul`, which can be
+bufferized.
 
 ## Using One-Shot Bufferize
 
@@ -221,17 +273,14 @@ By default, One-Shot Bufferize fails when it encounters an op with tensor
 semantics (i.e., tensor result or tensor operand) that is not bufferizable
 (i.e., does not implement `BufferizableOpInterface`). This can be avoided with
 `allow-unknown-ops`. In that case, One-Shot Bufferize inserts
-`to_memref`/`to_tensor` ops around the bufferization boundary. These ops are
-named versions of `unrealized_conversion_cast`. Note that One-Shot Bufferize's
-analysis can currently not analyze these ops, so input IR with such ops may fail
-bufferization. Therefore, running One-Shot Bufferize multiple times in a
-sequence is also not supported at the moment.
+`to_memref`/`to_tensor` ops around the bufferization boundary.
 
 One-Shot Bufferize can be configured to bufferize only ops from a set of
 dialects with `dialect-filter`. This can be useful for gradually migrating from
 dialect conversion-based bufferization to One-Shot Bufferize. One-Shot Bufferize
 must run first in such a case, because dialect conversion-based bufferization
-generates `to_tensor`/`to_memref` ops which One-Shot Bufferize cannot analyze.
+generates `to_tensor` ops without the `restrict` unit attribute, which One-Shot
+Bufferize cannot analyze.
 
 One-Shot Bufferize can also be called programmatically with
 [`bufferization::runOneShotBufferize`](https://github.com/llvm/llvm-project/blob/ae2764e835a26bad9774803eca0a6530df2a3e2d/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h#L167).
@@ -240,6 +289,14 @@ Alternatively,
 skips the analysis and inserts a copy on every buffer write, just like the
 dialect conversion-based bufferization.
 
+By default, function boundaries are not bufferized. This is because there are
+currently limitations around function graph bufferization: recursive
+calls are not supported. As long as there are no recursive calls, function
+boundary bufferization can be enabled with `bufferize-function-boundaries`. Each
+tensor function argument and tensor function result is then turned into a
+memref. The layout map of the memref type can be controlled with
+`function-boundary-type-conversion`.
+
 ## Memory Layouts
 
 One-Shot Bufferize bufferizes ops from top to bottom. This works well when all
@@ -319,6 +376,11 @@ To get a better intuition of the interface methods, we invite users to take a
 look at existing implementations in MLIR, e.g., the implementation of
 `tensor.insert` or `tensor.extract`.
 
+Interface implementations of DPS ops (that implement
+`DestinationStyleOpInterface`) can derive from
+`DstBufferizableOpInterfaceExternalModel`, which provides all necessary
+method implementations except for `bufferize`.
+
 ## Debugging Buffer Copies
 
 To get a better understanding of why One-Shot Bufferize introduced a buffer
@@ -338,14 +400,90 @@ There are two reasons why a buffer copy may be inserted.
 In the first case, `print-conflicts` illustrates the conflict in the form of a
 ("read", "conflicting write", "last write") tuple.
 
-## Understanding the SSA Use-Def Chain Analysis
+A RaW conflict consists of three parts, in the following order according to
+op dominance:
+
+1. **Definition:** A tensor `%t` is defined.
+2. **Conflicting Write:** An operation writes to `buffer(%t)`.
+3. **Read:** An operation reads `%t`.
+
+When such a RaW conflict is detected during the analysis phase, One-Shot
+Bufferize will insert a buffer copy for the conflicting write.
+
+**Example**
+
+```mlir
+// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only print-conflicts"
+func.func @test(%arg0: f32, %arg1: f32, %arg2: index, %arg3: index) -> (f32, tensor<3xf32>) {
+  // Create a new tensor with [%arg0, %arg0, %arg0].
+  %0 = tensor.from_elements %arg0, %arg0, %arg0 : tensor<3xf32>
+
+  // Insert something into the new tensor.
+  %1 = tensor.insert %arg1 into %0[%arg2] : tensor<3xf32>
+
+  // Read from the old tensor.
+  %r = tensor.extract %0[%arg3] : tensor<3xf32>
+
+  // Return the extracted value and the result of the insertion.
+  func.return %r, %1 : f32, tensor<3xf32>
+}
+```
+
+The output IR is as follows:
+
+```mlir
+func.func @test(%arg0: f32, %arg1: f32, %arg2: index, %arg3: index) -> (f32, tensor<3xf32>) {
+  %from_elements = tensor.from_elements %arg0, %arg0, %arg0 {"C_0[DEF: result 0]"} : tensor<3xf32>
+  %inserted = tensor.insert %arg1 into %from_elements[%arg2] {"C_0[CONFL-WRITE: 1]", __inplace_operands_attr__ = ["none", "false", "none"]} : tensor<3xf32>
+  %extracted = tensor.extract %from_elements[%arg3] {"C_0[READ: 0]", __inplace_operands_attr__ = ["true", "none"]} : tensor<3xf32>
+  return {__inplace_operands_attr__ = ["none", "true"]} %extracted, %inserted : f32, tensor<3xf32>
+}
+```
+
+Note that the IR was not bufferized. It was merely annotated with the results
+of the bufferization analysis. Every operation with tensor semantics has a
+`__inplace_operands_attr__` attribute with one value per operand. If an operand
+is not a tensor, the respective value is `none`. Otherwise, if the operand was
+decided to be bufferized in-place, the value is `true`. A value of `false`
+indicates a buffer copy. In the above example, a buffer copy would be inserted
+for `tensor.insert`, so that it does not overwrite `buffer(%from_elements)`,
+which is still needed for `tensor.extract`.
+
+For each RaW (there is only one in the example), three `C_i` attributes were
+added:
+
+* `C_0[DEF: result 0]`: A tensor is defined: 0-th result of
+  `tensor.from_elements`.
+* `C_0[CONFL-WRITE: 1]`: An operation (if bufferized in-place) would write into
+  the future buffer of the defined tensor: 1-st operand of `tensor.insert`.
+* `C_0[READ: 0]`: An operation reads the tensor definition: 0-th operand of
+  `tensor.extract`.
+
+The fully bufferized IR (with the inserted buffer copy) is as follows:
+
+```mlir
+func.func @test(%arg0: f32, %arg1: f32, %arg2: index, %arg3: index) -> (f32, memref<3xf32>) {
+  %c2 = arith.constant 2 : index
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %alloc = memref.alloc() {alignment = 64 : i64} : memref<3xf32>
+  memref.store %arg0, %alloc[%c0] : memref<3xf32>
+  memref.store %arg0, %alloc[%c1] : memref<3xf32>
+  memref.store %arg0, %alloc[%c2] : memref<3xf32>
+  %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<3xf32>
+  memref.copy %alloc, %alloc_0 : memref<3xf32> to memref<3xf32>
+  memref.store %arg1, %alloc_0[%arg2] : memref<3xf32>
+  %0 = memref.load %alloc[%arg3] : memref<3xf32>
+  return %0, %alloc_0 : f32, memref<3xf32>
+}
+```
 
 To get a better understanding of the SSA Use-Def Chain Analysis and the RaW
-conflict detection algorithm, we invite interested users to read the
-[design document](https://discourse.llvm.org/uploads/short-url/5kckJ3DftYwQokG252teFgw3sYa.pdf)
-and watch the corresponding [ODM talk](https://youtu.be/TXEo59CYS9A)
-([slides](https://mlir.llvm.org/OpenMeetings/2022-01-13-One-Shot-Bufferization.pdf)).
-can be used to bufferize a program in a single pass, as long as each op
+conflict detection algorithm, interested users may want to refer to:
+
+* [Original design document](https://discourse.llvm.org/uploads/short-url/5kckJ3DftYwQokG252teFgw3sYa.pdf)
+* [ODM talk](https://youtu.be/TXEo59CYS9A), ([slides](https://mlir.llvm.org/OpenMeetings/2022-01-13-One-Shot-Bufferization.pdf)).
+* [LLVM Dev Meeting 2023 tutorial slides](https://m-sp.org/downloads/llvm_dev_2023.pdf)
 
 ## Migrating from Dialect Conversion-based Bufferization
 
@@ -356,20 +494,6 @@ One-Shot Bufferize must run first because it cannot analyze those boundary ops.
 To update existing code step-by-step, it may be useful to specify a dialect
 filter for One-Shot Bufferize, so that dialects can be switched over one-by-one.
 
-## Bufferization Function Graphs
-
-One-Shot Bufferize does currently not support function graph bufferization.
-I.e., `CallOp`, `ReturnOp` and function bbArgs are not bufferizable. Users can
-run the existing `--func-bufferize` bufferization pass after One-Shot Bufferize.
-
-Alternatively, users can try
-[`ModuleBufferization`](https://github.com/llvm/llvm-project/blob/ae2764e835a26bad9774803eca0a6530df2a3e2d/mlir/include/mlir/Dialect/Linalg/ComprehensiveBufferize/ModuleBufferization.h#L31),
-which is an extension of One-Shot Bufferize. This bufferization is still under
-development and does not support arbitrary IR. In essence, returning a tensor
-from a function is not supported, unless it is equivalent to a function bbArg.
-In that case, the corresponding return value can simply be dropped during
-bufferization.
-
 ## Dialect Conversion-based Bufferization
 
 Disclaimer: Most dialect conversion-based bufferization has been migrated to
diff --git a/mlir/docs/includes/img/bufferization_passes.svg b/mlir/docs/includes/img/bufferization_passes.svg
new file mode 100644
index 000000000000..835726569227
--- /dev/null
+++ b/mlir/docs/includes/img/bufferization_passes.svg
@@ -0,0 +1 @@
+<svg version="1.1" viewBox="0.0 0.0 480.0 540.0" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l480.0 0l0 540.0l-480.0 0l0 -540.0z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#ffffff" d="m0 0l480.0 0l0 540.0l-480.0 0z" fill-rule="evenodd"/><path fill="#4285f4" d="m163.16655 122.94077l315.59058 0l0 29.039375l-315.59058 0z" fill-rule="evenodd"/><path stroke="#4285f4" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m163.16655 122.94077l315.59058 0l0 29.039375l-315.59058 0z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m244.41011 131.65561l153.10344 0l0 17.16098l-153.10344 0l0 -17.16098z" fill-rule="nonzero"/><path fill="#ffffff" d="m246.20699 139.41545l0 -1.546875l4.453125 0l0 1.546875l-4.453125 0zm13.80809 -0.5625q0 0.859375 -0.25 1.578125q-0.234375 0.703125 -0.703125 1.21875q-0.453125 0.5 -1.125 0.78125q-0.65625 0.28125 -1.515625 0.28125q-0.79689026 0 -1.4375153 -0.234375q-0.640625 -0.25 -1.09375 -0.71875q-0.4375 -0.46875 -0.671875 -1.171875q-0.234375 -0.703125 -0.234375 -1.640625q0 -0.859375 0.25 -1.5625q0.25 -0.71875 0.703125 -1.21875q0.46875 -0.5 1.125 -0.765625q0.671875 -0.28125 1.5000153 -0.28125q0.828125 0 1.453125 0.234375q0.640625 0.234375 1.078125 0.71875q0.453125 0.46875 0.6875 1.171875q0.234375 0.6875 0.234375 1.609375zm-1.828125 0.03125q0 -1.125 -0.421875 -1.6875q-0.421875 -0.578125 -1.25 -0.578125q-0.46875 0 -0.79689026 0.1875q-0.3125 0.171875 -0.515625 0.484375q-0.203125 0.3125 -0.3125 0.734375q-0.09375 0.40625 -0.09375 0.875q0 1.140625 0.453125 1.71875q0.46875 0.578125 1.2656403 0.578125q0.4375 0 0.75 -0.171875q0.328125 -0.1875 0.53125 -0.484375q0.203125 -0.3125 0.296875 -0.734375q0.09375 -0.421875 0.09375 -0.921875zm7.66745 3.65625l0 -4.734375q0 -1.203125 -0.890625 -1.203125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 4.59375l-1.734375 0l0 -7.265625l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.546875 0 0.953125 0.1875q0.421875 0.171875 0.703125 0.515625q0.28125 0.328125 0.421875 0.796875q0.140625 0.46875 0.140625 1.046875l0 4.875l-1.75 0zm10.058075 -4.1875q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm4.308075 1.3125l0 -1.546875l4.453125 0l0 1.546875l-4.453125 0zm13.245575 0.984375q0 0.625 -0.28125 1.0625q-0.265625 0.4375 -0.71875 0.71875q-0.453125 0.28125 -1.03125 0.40625q-0.578125 0.125 -1.171875 0.125q-0.796875 0 -1.453125 -0.078125q-0.640625 -0.078125 -1.21875 -0.21875l0 -1.578125q0.6875 0.28125 1.34375 0.40625q0.671875 0.125 1.265625 0.125q0.6875 0 1.03125 -0.21875q0.34375 -0.21875 0.34375 -0.5625q0 -0.171875 -0.078125 -0.296875q-0.078125 -0.140625 -0.28125 -0.265625q-0.1875 -0.125 -0.546875 -0.25q-0.359375 -0.140625 -0.9375 -0.3125q-0.53125 -0.15625 -0.9375 -0.34375q-0.40625 -0.1875 -0.671875 -0.4375q-0.265625 -0.265625 -0.40625 -0.59375q-0.125 -0.34375 -0.125 -0.8125q0 -0.4375 0.203125 -0.828125q0.203125 -0.390625 0.59375 -0.6875q0.40625 -0.296875 1.0 -0.46875q0.59375 -0.171875 1.390625 -0.171875q0.6875 0 1.21875 0.078125q0.53125 0.0625 0.9375 0.140625l0 1.421875q-0.625 -0.1875 -1.171875 -0.265625q-0.546875 -0.09375 -1.09375 -0.09375q-0.53125 0 -0.859375 0.203125q-0.328125 0.1875 -0.328125 0.53125q0 0.15625 0.0625 0.296875q0.078125 0.125 0.265625 0.25q0.1875 0.109375 0.515625 0.25q0.34375 0.125 0.90625 0.28125q0.625 0.1875 1.046875 0.390625q0.4375 0.203125 0.703125 0.46875q0.265625 0.25 0.375 0.578125q0.109375 0.328125 0.109375 0.75zm6.401825 2.140625l0 -4.734375q0 -1.203125 -0.890625 -1.203125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 4.59375l-1.734375 0l0 -10.109375l1.734375 0l0 2.484375l-0.078125 1.3125q0.21875 -0.25 0.4375 -0.453125q0.234375 -0.21875 0.5 -0.359375q0.265625 -0.15625 0.578125 -0.21875q0.3125 -0.078125 0.6875 -0.078125q0.546875 0 0.953125 0.1875q0.421875 0.171875 0.703125 0.515625q0.28125 0.328125 0.421875 0.796875q0.140625 0.46875 0.140625 1.046875l0 4.875l-1.75 0zm10.276825 -3.6875q0 0.859375 -0.25 1.578125q-0.234375 0.703125 -0.703125 1.21875q-0.453125 0.5 -1.125 0.78125q-0.65625 0.28125 -1.515625 0.28125q-0.796875 0 -1.4375 -0.234375q-0.640625 -0.25 -1.09375 -0.71875q-0.4375 -0.46875 -0.671875 -1.171875q-0.234375 -0.703125 -0.234375 -1.640625q0 -0.859375 0.25 -1.5625q0.25 -0.71875 0.703125 -1.21875q0.46875 -0.5 1.125 -0.765625q0.671875 -0.28125 1.5 -0.28125q0.828125 0 1.453125 0.234375q0.640625 0.234375 1.078125 0.71875q0.453125 0.46875 0.6875 1.171875q0.234375 0.6875 0.234375 1.609375zm-1.828125 0.03125q0 -1.125 -0.421875 -1.6875q-0.421875 -0.578125 -1.25 -0.578125q-0.46875 0 -0.796875 0.1875q-0.3125 0.171875 -0.515625 0.484375q-0.203125 0.3125 -0.3125 0.734375q-0.09375 0.40625 -0.09375 0.875q0 1.140625 0.453125 1.71875q0.46875 0.578125 1.265625 0.578125q0.4375 0 0.75 -0.171875q0.328125 -0.1875 0.53125 -0.484375q0.203125 -0.3125 0.296875 -0.734375q0.09375 -0.421875 0.09375 -0.921875zm9.3237 3.578125q-0.46875 0.109375 -0.953125 0.171875q-0.46875 0.078125 -0.90625 0.078125q-0.703125 0 -1.234375 -0.15625q-0.515625 -0.15625 -0.859375 -0.453125q-0.328125 -0.3125 -0.5 -0.78125q-0.15625 -0.484375 -0.15625 -1.140625l0 -3.546875l-1.953125 0l0 -1.359375l1.953125 0l0 -1.859375l1.796875 -0.46875l0 2.328125l2.8125 0l0 1.359375l-2.8125 0l0 3.421875q0 0.609375 0.28125 0.9375q0.28125 0.3125 0.953125 0.3125q0.4375 0 0.84375 -0.0625q0.40625 -0.078125 0.734375 -0.171875l0 1.390625zm2.870575 -3.046875l0 -1.546875l4.453125 0l0 1.546875l-4.453125 0zm13.79245 -0.640625q0 1.0 -0.28125 1.734375q-0.28125 0.734375 -0.78125 1.21875q-0.5 0.46875 -1.1875 0.703125q-0.6875 0.234375 -1.5 0.234375q-0.796875 0 -1.484375 -0.125q-0.671875 -0.109375 -1.296875 -0.3125l0 -9.796875l1.734375 0l0 2.375l-0.0625 1.421875q0.390625 -0.5 0.90625 -0.796875q0.53125 -0.3125 1.28125 -0.3125q0.640625 0 1.140625 0.265625q0.5 0.25 0.828125 0.71875q0.34375 0.46875 0.515625 1.15625q0.1875 0.671875 0.1875 1.515625zm-1.828125 0.078125q0 -0.609375 -0.09375 -1.03125q-0.078125 -0.421875 -0.25 -0.6875q-0.15625 -0.28125 -0.390625 -0.40625q-0.21875 -0.125 -0.53125 -0.125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 3.171875q0.21875 0.078125 0.515625 0.125q0.3125 0.046875 0.625 0.046875q0.40625 0 0.75 -0.171875q0.34375 -0.171875 0.578125 -0.484375q0.25 -0.3125 0.375 -0.765625q0.125 -0.453125 0.125 -1.015625zm5.089325 -3.578125l0 4.734375q0 0.59375 0.21875 0.90625q0.21875 0.296875 0.671875 0.296875q0.4375 0 0.828125 -0.359375q0.40625 -0.359375 0.859375 -0.984375l0 -4.59375l1.75 0l0 7.265625l-1.515625 0l-0.03125 -1.078125q-0.234375 0.28125 -0.46875 0.515625q-0.234375 0.21875 -0.515625 0.390625q-0.265625 0.15625 -0.59375 0.25q-0.328125 0.09375 -0.71875 0.09375q-0.5625 0 -0.984375 -0.1875q-0.421875 -0.1875 -0.703125 -0.515625q-0.265625 -0.34375 -0.40625 -0.8125q-0.140625 -0.46875 -0.140625 -1.046875l0 -4.875l1.75 0zm12.839325 -1.359375q-0.3125 -0.109375 -0.75 -0.171875q-0.421875 -0.0625 -0.875 -0.0625q-0.3125 0 -0.578125 0.078125q-0.265625 0.078125 -0.453125 0.265625q-0.1875 0.171875 -0.3125 0.453125q-0.109375 0.265625 -0.109375 0.65625l0 1.1875l2.859375 0l0 1.359375l-2.859375 0l0 4.859375l-1.75 0l0 -4.859375l-2.109375 0l0 -1.359375l2.109375 0l0 -1.109375q0 -0.765625 0.21875 -1.3125q0.234375 -0.5625 0.640625 -0.921875q0.40625 -0.359375 0.984375 -0.515625q0.578125 -0.171875 1.296875 -0.171875q0.484375 0 0.90625 0.0625q0.421875 0.0625 0.78125 0.140625l0 1.421875zm-7.515625 1.359375l0 0zm15.5737 -1.359375q-0.3125 -0.109375 -0.75 -0.171875q-0.421875 -0.0625 -0.875 -0.0625q-0.3125 0 -0.578125 0.078125q-0.265625 0.078125 -0.453125 0.265625q-0.1875 0.171875 -0.3125 0.453125q-0.109375 0.265625 -0.109375 0.65625l0 1.1875l2.859375 0l0 1.359375l-2.859375 0l0 4.859375l-1.75 0l0 -4.859375l-2.109375 0l0 -1.359375l2.109375 0l0 -1.109375q0 -0.765625 0.21875 -1.3125q0.234375 -0.5625 0.640625 -0.921875q0.40625 -0.359375 0.984375 -0.515625q0.578125 -0.171875 1.296875 -0.171875q0.484375 0 0.90625 0.0625q0.421875 0.0625 0.78125 0.140625l0 1.421875zm-7.515625 1.359375l0 0zm15.3862 3.078125q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm8.183075 0q0.015625 -0.421875 -0.046875 -0.703125q-0.046875 -0.28125 -0.171875 -0.46875q-0.109375 -0.1875 -0.28125 -0.265625q-0.171875 -0.09375 -0.390625 -0.09375q-0.390625 0 -0.796875 0.328125q-0.40625 0.3125 -0.90625 1.046875l0 4.59375l-1.796875 0l0 -7.265625l1.59375 0l0.0625 1.046875q0.171875 -0.265625 0.390625 -0.484375q0.234375 -0.234375 0.5 -0.390625q0.28125 -0.15625 0.609375 -0.234375q0.34375 -0.09375 0.75 -0.09375q0.546875 0 0.96875 0.1875q0.4375 0.1875 0.71875 0.5625q0.296875 0.359375 0.4375 0.921875q0.140625 0.5625 0.109375 1.3125l-1.75 0zm7.66745 -4.828125q0 0.234375 -0.09375 0.453125q-0.078125 0.203125 -0.234375 0.359375q-0.15625 0.15625 -0.375 0.25q-0.203125 0.078125 -0.4375 0.078125q-0.25 0 -0.46875 -0.078125q-0.203125 -0.09375 -0.359375 -0.25q-0.15625 -0.15625 -0.25 -0.359375q-0.078125 -0.21875 -0.078125 -0.453125q0 -0.234375 0.078125 -0.4375q0.09375 -0.203125 0.25 -0.359375q0.15625 -0.15625 0.359375 -0.25q0.21875 -0.09375 0.46875 -0.09375q0.234375 0 0.4375 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.234375 0.359375q0.09375 0.203125 0.09375 0.4375zm-1.9375 3.359375l-2.0625 0l0 -1.359375l3.84375 0l0 5.90625l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -4.546875zm5.79245 5.90625l0 -1.21875l3.546875 -4.5625l-3.46875 0l0 -1.484375l5.78125 0l0 1.21875l-3.484375 4.515625l3.53125 0l0 1.53125l-5.90625 0zm14.308075 -4.1875q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m397.51355 131.65561l0 0l0 17.16098l0 0l0 -17.16098z" fill-rule="nonzero"/><path fill="#ffffff" d="m244.41011 143.49379l153.10344 0l0 1.3200073l-153.10344 0l0 -1.3200073z" fill-rule="nonzero"/><path fill="#34a853" d="m163.16655 174.66765l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path stroke="#34a853" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m163.16655 174.66765l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path fill="#ffffff" d="m262.6691 190.95483l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm9.16745 -0.40625q0 0.890625 -0.25 1.59375q-0.234375 0.703125 -0.703125 1.203125q-0.453125 0.484375 -1.109375 0.75q-0.640625 0.25 -1.453125 0.25q-0.640625 0 -1.3125 -0.109375q-0.65625 -0.125 -1.3125 -0.390625l0 -9.6875l1.25 0l0 2.78125l-0.0625 1.328125q0.546875 -0.71875 1.15625 -1.015625q0.609375 -0.296875 1.3125 -0.296875q0.609375 0 1.078125 0.265625q0.46875 0.25 0.78125 0.71875q0.3125 0.46875 0.46875 1.140625q0.15625 0.65625 0.15625 1.46875zm-1.265625 0.046875q0 -0.5625 -0.09375 -1.03125q-0.078125 -0.46875 -0.265625 -0.796875q-0.171875 -0.34375 -0.453125 -0.53125q-0.265625 -0.1875 -0.65625 -0.1875q-0.234375 0 -0.484375 0.078125q-0.234375 0.0625 -0.5 0.25q-0.265625 0.171875 -0.5625 0.46875q-0.28125 0.28125 -0.609375 0.703125l0 3.484375q0.359375 0.140625 0.734375 0.234375q0.390625 0.078125 0.734375 0.078125q0.421875 0 0.8125 -0.125q0.390625 -0.140625 0.6875 -0.453125q0.296875 -0.328125 0.46875 -0.859375q0.1875 -0.53125 0.1875 -1.3125zm8.964325 3.671875l-1.125 0l-0.03125 -1.15625q-0.328125 0.375 -0.625 0.625q-0.28125 0.234375 -0.578125 0.390625q-0.28125 0.140625 -0.578125 0.203125q-0.28125 0.0625 -0.59375 0.0625q-1.109375 0 -1.6875 -0.640625q-0.5625 -0.65625 -0.5625 -1.96875l0 -4.703125l1.25 0l0 4.59375q0 1.65625 1.25 1.65625q0.21875 0 0.4375 -0.0625q0.21875 -0.078125 0.453125 -0.234375q0.25 -0.171875 0.515625 -0.453125q0.28125 -0.296875 0.625 -0.734375l0 -4.765625l1.25 0l0 7.1875zm8.651825 -8.953125q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.5737 -1.765625q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.276825 3.21875q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm3.5737 -3.03125l1.140625 0l0.03125 1.328125q0.640625 -0.765625 1.25 -1.109375q0.625 -0.34375 1.25 -0.34375q1.125 0 1.703125 0.71875q0.578125 0.71875 0.53125 2.15625l-1.265625 0q0.03125 -0.953125 -0.265625 -1.375q-0.296875 -0.421875 -0.875 -0.421875q-0.25 0 -0.515625 0.09375q-0.25 0.078125 -0.515625 0.28125q-0.265625 0.1875 -0.5625 0.5q-0.296875 0.3125 -0.640625 0.75l0 4.609375l-1.265625 0l0 -7.1875zm12.79245 3.875l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm8.808075 3.3125l-1.25 0l0 -4.59375q0 -0.828125 -0.3125 -1.234375q-0.3125 -0.40625 -0.890625 -0.40625q-0.25 0 -0.46875 0.0625q-0.21875 0.0625 -0.453125 0.234375q-0.234375 0.171875 -0.515625 0.453125q-0.28125 0.28125 -0.640625 0.71875l0 4.765625l-1.25 0l0 -10.109375l1.25 0l0 2.921875l-0.046875 1.140625q0.296875 -0.359375 0.578125 -0.59375q0.28125 -0.25 0.5625 -0.390625q0.28125 -0.15625 0.578125 -0.21875q0.296875 -0.0625 0.609375 -0.0625q1.078125 0 1.65625 0.65625q0.59375 0.65625 0.59375 1.96875l0 4.6875zm8.54245 -3.65625q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm5.526825 -2.5625l-2.125 0l0 -1.03125l3.390625 0l0 6.15625l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -5.125zm0.4375 -4.046875q0.203125 0 0.375 0.078125q0.1875 0.078125 0.3125 0.21875q0.140625 0.125 0.21875 0.3125q0.078125 0.171875 0.078125 0.375q0 0.203125 -0.078125 0.390625q-0.078125 0.171875 -0.21875 0.3125q-0.125 0.125 -0.3125 0.203125q-0.171875 0.078125 -0.375 0.078125q-0.203125 0 -0.390625 -0.078125q-0.171875 -0.078125 -0.3125 -0.203125q-0.125 -0.140625 -0.203125 -0.3125q-0.078125 -0.1875 -0.078125 -0.390625q0 -0.203125 0.078125 -0.375q0.078125 -0.1875 0.203125 -0.3125q0.140625 -0.140625 0.3125 -0.21875q0.1875 -0.078125 0.390625 -0.078125zm10.85495 8.25q0 0.375 -0.125 0.671875q-0.125 0.296875 -0.34375 0.53125q-0.21875 0.234375 -0.515625 0.40625q-0.296875 0.15625 -0.640625 0.265625q-0.328125 0.109375 -0.6875 0.15625q-0.34375 0.046875 -0.671875 0.046875q-0.734375 0 -1.34375 -0.0625q-0.609375 -0.0625 -1.203125 -0.203125l0 -1.140625q0.640625 0.171875 1.25 0.265625q0.625 0.09375 1.25 0.09375q0.890625 0 1.3125 -0.234375q0.4375 -0.25 0.4375 -0.703125q0 -0.1875 -0.078125 -0.34375q-0.0625 -0.15625 -0.25 -0.296875q-0.171875 -0.140625 -0.546875 -0.28125q-0.375 -0.15625 -1.015625 -0.359375q-0.5 -0.140625 -0.90625 -0.3125q-0.40625 -0.1875 -0.71875 -0.4375q-0.296875 -0.25 -0.46875 -0.578125q-0.171875 -0.34375 -0.171875 -0.8125q0 -0.296875 0.140625 -0.65625q0.140625 -0.359375 0.46875 -0.65625q0.34375 -0.3125 0.921875 -0.515625q0.578125 -0.203125 1.421875 -0.203125q0.421875 0 0.9375 0.046875q0.515625 0.046875 1.078125 0.15625l0 1.109375q-0.578125 -0.140625 -1.109375 -0.203125q-0.53125 -0.078125 -0.90625 -0.078125q-0.46875 0 -0.796875 0.078125q-0.3125 0.0625 -0.515625 0.203125q-0.1875 0.125 -0.28125 0.296875q-0.078125 0.15625 -0.078125 0.34375q0 0.203125 0.0625 0.359375q0.078125 0.15625 0.28125 0.3125q0.21875 0.140625 0.578125 0.28125q0.359375 0.140625 0.953125 0.3125q0.640625 0.1875 1.0625 0.390625q0.4375 0.203125 0.703125 0.46875q0.28125 0.25 0.390625 0.5625q0.125 0.3125 0.125 0.71875zm8.16745 1.859375q-0.421875 0.09375 -0.875 0.140625q-0.453125 0.046875 -0.921875 0.046875q-1.34375 0 -2.015625 -0.609375q-0.65625 -0.609375 -0.65625 -1.875l0 -3.75l-2.015625 0l0 -1.046875l2.015625 0l0 -1.96875l1.234375 -0.328125l0 2.296875l3.234375 0l0 1.046875l-3.234375 0l0 3.65625q0 0.765625 0.40625 1.15625q0.421875 0.375 1.21875 0.375q0.34375 0 0.75 -0.046875q0.40625 -0.0625 0.859375 -0.171875l0 1.078125zm4.714325 -6.0625l-2.125 0l0 -1.03125l3.390625 0l0 6.15625l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -5.125zm0.4375 -4.046875q0.203125 0 0.375 0.078125q0.1875 0.078125 0.3125 0.21875q0.140625 0.125 0.21875 0.3125q0.078125 0.171875 0.078125 0.375q0 0.203125 -0.078125 0.390625q-0.078125 0.171875 -0.21875 0.3125q-0.125 0.125 -0.3125 0.203125q-0.171875 0.078125 -0.375 0.078125q-0.203125 0 -0.390625 -0.078125q-0.171875 -0.078125 -0.3125 -0.203125q-0.125 -0.140625 -0.203125 -0.3125q-0.078125 -0.1875 -0.078125 -0.390625q0 -0.203125 0.078125 -0.375q0.078125 -0.1875 0.203125 -0.3125q0.140625 -0.140625 0.3125 -0.21875q0.1875 -0.078125 0.390625 -0.078125zm5.16745 3.015625l1.109375 0l0.046875 1.15625q0.328125 -0.359375 0.609375 -0.609375q0.296875 -0.25 0.578125 -0.390625q0.28125 -0.15625 0.578125 -0.21875q0.296875 -0.0625 0.609375 -0.0625q1.109375 0 1.671875 0.65625q0.578125 0.65625 0.578125 1.96875l0 4.6875l-1.25 0l0 -4.59375q0 -0.84375 -0.3125 -1.234375q-0.3125 -0.40625 -0.9375 -0.40625q-0.234375 0 -0.453125 0.0625q-0.21875 0.0625 -0.453125 0.234375q-0.234375 0.171875 -0.515625 0.453125q-0.265625 0.28125 -0.609375 0.71875l0 4.765625l-1.25 0l0 -7.1875zm13.22995 1.015625q0.203125 0.25 0.3125 0.59375q0.109375 0.328125 0.109375 0.71875q0 0.546875 -0.203125 1.015625q-0.203125 0.453125 -0.578125 0.78125q-0.359375 0.328125 -0.890625 0.515625q-0.515625 0.1875 -1.140625 0.1875q-0.4375 0 -0.84375 -0.09375q-0.390625 -0.109375 -0.609375 -0.25q-0.140625 0.203125 -0.234375 0.390625q-0.09375 0.171875 -0.09375 0.390625q0 0.28125 0.25 0.46875q0.265625 0.171875 0.703125 0.171875l1.890625 0.078125q0.53125 0.015625 0.984375 0.140625q0.453125 0.125 0.78125 0.359375q0.328125 0.21875 0.5 0.5625q0.1875 0.328125 0.1875 0.765625q0 0.484375 -0.203125 0.90625q-0.203125 0.4375 -0.640625 0.765625q-0.4375 0.328125 -1.109375 0.515625q-0.65625 0.1875 -1.5625 0.1875q-0.875 0 -1.5 -0.140625q-0.609375 -0.125 -1.0 -0.375q-0.390625 -0.25 -0.578125 -0.59375q-0.171875 -0.34375 -0.171875 -0.75q0 -0.515625 0.234375 -0.921875q0.25 -0.390625 0.765625 -0.75q-0.1875 -0.078125 -0.328125 -0.203125q-0.140625 -0.140625 -0.234375 -0.296875q-0.078125 -0.15625 -0.125 -0.328125q-0.03125 -0.171875 -0.03125 -0.328125q0 -0.46875 0.21875 -0.84375q0.21875 -0.390625 0.515625 -0.734375q-0.140625 -0.171875 -0.25 -0.328125q-0.09375 -0.15625 -0.171875 -0.328125q-0.078125 -0.1875 -0.125 -0.390625q-0.03125 -0.21875 -0.03125 -0.5q0 -0.5625 0.203125 -1.015625q0.21875 -0.46875 0.578125 -0.796875q0.375 -0.328125 0.890625 -0.5q0.515625 -0.1875 1.140625 -0.1875q0.265625 0 0.515625 0.03125q0.25 0.03125 0.421875 0.09375l2.609375 0l0 1.015625l-1.15625 0zm-4.3125 7.203125q0 0.515625 0.53125 0.75q0.53125 0.234375 1.46875 0.234375q0.59375 0 1.0 -0.109375q0.40625 -0.109375 0.640625 -0.28125q0.25 -0.171875 0.359375 -0.40625q0.109375 -0.21875 0.109375 -0.453125q0 -0.4375 -0.359375 -0.65625q-0.359375 -0.203125 -1.09375 -0.234375l-1.875 -0.0625q-0.234375 0.15625 -0.390625 0.296875q-0.15625 0.15625 -0.25 0.3125q-0.078125 0.15625 -0.109375 0.296875q-0.03125 0.15625 -0.03125 0.3125zm0.375 -5.875q0 0.34375 0.109375 0.640625q0.125 0.28125 0.328125 0.484375q0.203125 0.203125 0.484375 0.3125q0.28125 0.109375 0.625 0.109375q0.375 0 0.65625 -0.125q0.296875 -0.125 0.5 -0.34375q0.203125 -0.21875 0.296875 -0.484375q0.109375 -0.28125 0.109375 -0.59375q0 -0.34375 -0.125 -0.625q-0.109375 -0.28125 -0.3125 -0.484375q-0.203125 -0.203125 -0.5 -0.3125q-0.28125 -0.109375 -0.625 -0.109375q-0.359375 0 -0.65625 0.125q-0.28125 0.125 -0.484375 0.34375q-0.203125 0.203125 -0.3125 0.484375q-0.09375 0.28125 -0.09375 0.578125z" fill-rule="nonzero"/><path fill="#34a853" d="m163.16655 224.35252l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path stroke="#34a853" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m163.16655 224.35252l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path fill="#ffffff" d="m242.52391 240.63971l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm9.16745 -0.40625q0 0.890625 -0.25 1.59375q-0.234375 0.703125 -0.703125 1.203125q-0.453125 0.484375 -1.109375 0.75q-0.640625 0.25 -1.453125 0.25q-0.640625 0 -1.3125 -0.109375q-0.65625 -0.125 -1.3125 -0.390625l0 -9.6875l1.25 0l0 2.78125l-0.0625 1.328125q0.546875 -0.71875 1.15625 -1.015625q0.609375 -0.296875 1.3125 -0.296875q0.609375 0 1.078125 0.265625q0.46875 0.25 0.78125 0.71875q0.3125 0.46875 0.46875 1.140625q0.15625 0.65625 0.15625 1.46875zm-1.265625 0.046875q0 -0.5625 -0.09375 -1.03125q-0.078125 -0.46875 -0.265625 -0.796875q-0.171875 -0.34375 -0.453125 -0.53125q-0.265625 -0.1875 -0.65625 -0.1875q-0.234375 0 -0.484375 0.078125q-0.234375 0.0625 -0.5 0.25q-0.265625 0.171875 -0.5625 0.46875q-0.28125 0.28125 -0.609375 0.703125l0 3.484375q0.359375 0.140625 0.734375 0.234375q0.390625 0.078125 0.734375 0.078125q0.421875 0 0.8125 -0.125q0.390625 -0.140625 0.6875 -0.453125q0.296875 -0.328125 0.46875 -0.859375q0.1875 -0.53125 0.1875 -1.3125zm8.96434 3.671875l-1.125 0l-0.03125 -1.15625q-0.328125 0.375 -0.625 0.625q-0.28125 0.234375 -0.578125 0.390625q-0.28125 0.140625 -0.578125 0.203125q-0.28125 0.0625 -0.59376526 0.0625q-1.109375 0 -1.6875 -0.640625q-0.5625 -0.65625 -0.5625 -1.96875l0 -4.703125l1.25 0l0 4.59375q0 1.65625 1.2500153 1.65625q0.21875 0 0.4375 -0.0625q0.21875 -0.078125 0.453125 -0.234375q0.25 -0.171875 0.515625 -0.453125q0.28125 -0.296875 0.625 -0.734375l0 -4.765625l1.25 0l0 7.1875zm8.651825 -8.953125q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.5737 -1.765625q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.276825 3.21875q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm3.5737 -3.03125l1.140625 0l0.03125 1.328125q0.640625 -0.765625 1.25 -1.109375q0.625 -0.34375 1.25 -0.34375q1.125 0 1.703125 0.71875q0.578125 0.71875 0.53125 2.15625l-1.265625 0q0.03125 -0.953125 -0.265625 -1.375q-0.296875 -0.421875 -0.875 -0.421875q-0.25 0 -0.515625 0.09375q-0.25 0.078125 -0.515625 0.28125q-0.265625 0.1875 -0.5625 0.5q-0.296875 0.3125 -0.640625 0.75l0 4.609375l-1.265625 0l0 -7.1875zm12.79245 3.875l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm5.47995 -5.78125l-2.125 0l0 -1.015625l3.390625 0l0 9.078125l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -8.0625zm11.870575 5.4375q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm9.339325 -0.0625q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm9.214325 -0.125q0 0.953125 -0.265625 1.671875q-0.265625 0.71875 -0.75 1.1875q-0.46875 0.46875 -1.109375 0.703125q-0.640625 0.234375 -1.390625 0.234375q-0.34375 0 -0.6875 -0.03125q-0.34375 -0.03125 -0.6875 -0.125l0 3.015625l-1.25 0l0 -10.125l1.109375 0l0.078125 1.203125q0.546875 -0.734375 1.15625 -1.03125q0.609375 -0.296875 1.3125 -0.296875q0.609375 0 1.078125 0.265625q0.46875 0.25 0.78125 0.71875q0.3125 0.46875 0.46875 1.140625q0.15625 0.65625 0.15625 1.46875zm-1.265625 0.046875q0 -0.5625 -0.09375 -1.03125q-0.078125 -0.46875 -0.265625 -0.796875q-0.171875 -0.34375 -0.453125 -0.53125q-0.265625 -0.1875 -0.65625 -0.1875q-0.234375 0 -0.484375 0.078125q-0.234375 0.0625 -0.5 0.25q-0.265625 0.171875 -0.5625 0.46875q-0.28125 0.28125 -0.609375 0.703125l0 3.484375q0.34375 0.140625 0.71875 0.234375q0.390625 0.078125 0.75 0.078125q1.0 0 1.578125 -0.6875q0.578125 -0.6875 0.578125 -2.0625zm8.214325 0.359375l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm8.808075 3.3125l-1.25 0l0 -4.59375q0 -0.828125 -0.3125 -1.234375q-0.3125 -0.40625 -0.890625 -0.40625q-0.25 0 -0.46875 0.0625q-0.21875 0.0625 -0.453125 0.234375q-0.234375 0.171875 -0.515625 0.453125q-0.28125 0.28125 -0.640625 0.71875l0 4.765625l-1.25 0l0 -10.109375l1.25 0l0 2.921875l-0.046875 1.140625q0.296875 -0.359375 0.578125 -0.59375q0.28125 -0.25 0.5625 -0.390625q0.28125 -0.15625 0.578125 -0.21875q0.296875 -0.0625 0.609375 -0.0625q1.078125 0 1.65625 0.65625q0.59375 0.65625 0.59375 1.96875l0 4.6875zm8.54245 -3.65625q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm5.526825 -2.5625l-2.125 0l0 -1.03125l3.390625 0l0 6.15625l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -5.125zm0.4375 -4.046875q0.203125 0 0.375 0.078125q0.1875 0.078125 0.3125 0.21875q0.140625 0.125 0.21875 0.3125q0.078125 0.171875 0.078125 0.375q0 0.203125 -0.078125 0.390625q-0.078125 0.171875 -0.21875 0.3125q-0.125 0.125 -0.3125 0.203125q-0.171875 0.078125 -0.375 0.078125q-0.203125 0 -0.390625 -0.078125q-0.171875 -0.078125 -0.3125 -0.203125q-0.125 -0.140625 -0.203125 -0.3125q-0.078125 -0.1875 -0.078125 -0.390625q0 -0.203125 0.078125 -0.375q0.078125 -0.1875 0.203125 -0.3125q0.140625 -0.140625 0.3125 -0.21875q0.1875 -0.078125 0.390625 -0.078125zm10.85495 8.25q0 0.375 -0.125 0.671875q-0.125 0.296875 -0.34375 0.53125q-0.21875 0.234375 -0.515625 0.40625q-0.296875 0.15625 -0.640625 0.265625q-0.328125 0.109375 -0.6875 0.15625q-0.34375 0.046875 -0.671875 0.046875q-0.734375 0 -1.34375 -0.0625q-0.609375 -0.0625 -1.203125 -0.203125l0 -1.140625q0.640625 0.171875 1.25 0.265625q0.625 0.09375 1.25 0.09375q0.890625 0 1.3125 -0.234375q0.4375 -0.25 0.4375 -0.703125q0 -0.1875 -0.078125 -0.34375q-0.0625 -0.15625 -0.25 -0.296875q-0.171875 -0.140625 -0.546875 -0.28125q-0.375 -0.15625 -1.015625 -0.359375q-0.5 -0.140625 -0.90625 -0.3125q-0.40625 -0.1875 -0.71875 -0.4375q-0.296875 -0.25 -0.46875 -0.578125q-0.171875 -0.34375 -0.171875 -0.8125q0 -0.296875 0.140625 -0.65625q0.140625 -0.359375 0.46875 -0.65625q0.34375 -0.3125 0.921875 -0.515625q0.578125 -0.203125 1.421875 -0.203125q0.421875 0 0.9375 0.046875q0.515625 0.046875 1.078125 0.15625l0 1.109375q-0.578125 -0.140625 -1.109375 -0.203125q-0.53125 -0.078125 -0.90625 -0.078125q-0.46875 0 -0.796875 0.078125q-0.3125 0.0625 -0.515625 0.203125q-0.1875 0.125 -0.28125 0.296875q-0.078125 0.15625 -0.078125 0.34375q0 0.203125 0.0625 0.359375q0.078125 0.15625 0.28125 0.3125q0.21875 0.140625 0.578125 0.28125q0.359375 0.140625 0.953125 0.3125q0.640625 0.1875 1.0625 0.390625q0.4375 0.203125 0.703125 0.46875q0.28125 0.25 0.390625 0.5625q0.125 0.3125 0.125 0.71875zm8.16745 1.859375q-0.421875 0.09375 -0.875 0.140625q-0.453125 0.046875 -0.921875 0.046875q-1.34375 0 -2.015625 -0.609375q-0.65625 -0.609375 -0.65625 -1.875l0 -3.75l-2.015625 0l0 -1.046875l2.015625 0l0 -1.96875l1.234375 -0.328125l0 2.296875l3.234375 0l0 1.046875l-3.234375 0l0 3.65625q0 0.765625 0.40625 1.15625q0.421875 0.375 1.21875 0.375q0.34375 0 0.75 -0.046875q0.40625 -0.0625 0.859375 -0.171875l0 1.078125zm4.714325 -6.0625l-2.125 0l0 -1.03125l3.390625 0l0 6.15625l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -5.125zm0.4375 -4.046875q0.203125 0 0.375 0.078125q0.1875 0.078125 0.3125 0.21875q0.140625 0.125 0.21875 0.3125q0.078125 0.171875 0.078125 0.375q0 0.203125 -0.078125 0.390625q-0.078125 0.171875 -0.21875 0.3125q-0.125 0.125 -0.3125 0.203125q-0.171875 0.078125 -0.375 0.078125q-0.203125 0 -0.390625 -0.078125q-0.171875 -0.078125 -0.3125 -0.203125q-0.125 -0.140625 -0.203125 -0.3125q-0.078125 -0.1875 -0.078125 -0.390625q0 -0.203125 0.078125 -0.375q0.078125 -0.1875 0.203125 -0.3125q0.140625 -0.140625 0.3125 -0.21875q0.1875 -0.078125 0.390625 -0.078125zm5.16745 3.015625l1.109375 0l0.046875 1.15625q0.328125 -0.359375 0.609375 -0.609375q0.296875 -0.25 0.578125 -0.390625q0.28125 -0.15625 0.578125 -0.21875q0.296875 -0.0625 0.609375 -0.0625q1.109375 0 1.671875 0.65625q0.578125 0.65625 0.578125 1.96875l0 4.6875l-1.25 0l0 -4.59375q0 -0.84375 -0.3125 -1.234375q-0.3125 -0.40625 -0.9375 -0.40625q-0.234375 0 -0.453125 0.0625q-0.21875 0.0625 -0.453125 0.234375q-0.234375 0.171875 -0.515625 0.453125q-0.265625 0.28125 -0.609375 0.71875l0 4.765625l-1.25 0l0 -7.1875zm13.22995 1.015625q0.203125 0.25 0.3125 0.59375q0.109375 0.328125 0.109375 0.71875q0 0.546875 -0.203125 1.015625q-0.203125 0.453125 -0.578125 0.78125q-0.359375 0.328125 -0.890625 0.515625q-0.515625 0.1875 -1.140625 0.1875q-0.4375 0 -0.84375 -0.09375q-0.390625 -0.109375 -0.609375 -0.25q-0.140625 0.203125 -0.234375 0.390625q-0.09375 0.171875 -0.09375 0.390625q0 0.28125 0.25 0.46875q0.265625 0.171875 0.703125 0.171875l1.890625 0.078125q0.53125 0.015625 0.984375 0.140625q0.453125 0.125 0.78125 0.359375q0.328125 0.21875 0.5 0.5625q0.1875 0.328125 0.1875 0.765625q0 0.484375 -0.203125 0.90625q-0.203125 0.4375 -0.640625 0.765625q-0.4375 0.328125 -1.109375 0.515625q-0.65625 0.1875 -1.5625 0.1875q-0.875 0 -1.5 -0.140625q-0.609375 -0.125 -1.0 -0.375q-0.390625 -0.25 -0.578125 -0.59375q-0.171875 -0.34375 -0.171875 -0.75q0 -0.515625 0.234375 -0.921875q0.25 -0.390625 0.765625 -0.75q-0.1875 -0.078125 -0.328125 -0.203125q-0.140625 -0.140625 -0.234375 -0.296875q-0.078125 -0.15625 -0.125 -0.328125q-0.03125 -0.171875 -0.03125 -0.328125q0 -0.46875 0.21875 -0.84375q0.21875 -0.390625 0.515625 -0.734375q-0.140625 -0.171875 -0.25 -0.328125q-0.09375 -0.15625 -0.171875 -0.328125q-0.078125 -0.1875 -0.125 -0.390625q-0.03125 -0.21875 -0.03125 -0.5q0 -0.5625 0.203125 -1.015625q0.21875 -0.46875 0.578125 -0.796875q0.375 -0.328125 0.890625 -0.5q0.515625 -0.1875 1.140625 -0.1875q0.265625 0 0.515625 0.03125q0.25 0.03125 0.421875 0.09375l2.609375 0l0 1.015625l-1.15625 0zm-4.3125 7.203125q0 0.515625 0.53125 0.75q0.53125 0.234375 1.46875 0.234375q0.59375 0 1.0 -0.109375q0.40625 -0.109375 0.640625 -0.28125q0.25 -0.171875 0.359375 -0.40625q0.109375 -0.21875 0.109375 -0.453125q0 -0.4375 -0.359375 -0.65625q-0.359375 -0.203125 -1.09375 -0.234375l-1.875 -0.0625q-0.234375 0.15625 -0.390625 0.296875q-0.15625 0.15625 -0.25 0.3125q-0.078125 0.15625 -0.109375 0.296875q-0.03125 0.15625 -0.03125 0.3125zm0.375 -5.875q0 0.34375 0.109375 0.640625q0.125 0.28125 0.328125 0.484375q0.203125 0.203125 0.484375 0.3125q0.28125 0.109375 0.625 0.109375q0.375 0 0.65625 -0.125q0.296875 -0.125 0.5 -0.34375q0.203125 -0.21875 0.296875 -0.484375q0.109375 -0.28125 0.109375 -0.59375q0 -0.34375 -0.125 -0.625q-0.109375 -0.28125 -0.3125 -0.484375q-0.203125 -0.203125 -0.5 -0.3125q-0.28125 -0.109375 -0.625 -0.109375q-0.359375 0 -0.65625 0.125q-0.28125 0.125 -0.484375 0.34375q-0.203125 0.203125 -0.3125 0.484375q-0.09375 0.28125 -0.09375 0.578125z" fill-rule="nonzero"/><path fill="#34a853" d="m163.16655 274.0374l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path stroke="#34a853" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m163.16655 274.0374l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path fill="#ffffff" d="m210.29161 290.3246l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm9.16745 -0.40625q0 0.890625 -0.25 1.59375q-0.234375 0.703125 -0.703125 1.203125q-0.453125 0.484375 -1.109375 0.75q-0.640625 0.25 -1.453125 0.25q-0.640625 0 -1.3125 -0.109375q-0.65625 -0.125 -1.3125 -0.390625l0 -9.6875l1.25 0l0 2.78125l-0.0625 1.328125q0.546875 -0.71875 1.15625 -1.015625q0.609375 -0.296875 1.3125 -0.296875q0.609375 0 1.078125 0.265625q0.46875 0.25 0.78125 0.71875q0.3125 0.46875 0.46875 1.140625q0.15625 0.65625 0.15625 1.46875zm-1.265625 0.046875q0 -0.5625 -0.09375 -1.03125q-0.078125 -0.46875 -0.265625 -0.796875q-0.171875 -0.34375 -0.453125 -0.53125q-0.265625 -0.1875 -0.65625 -0.1875q-0.234375 0 -0.484375 0.078125q-0.234375 0.0625 -0.5 0.25q-0.265625 0.171875 -0.5625 0.46875q-0.28125 0.28125 -0.609375 0.703125l0 3.484375q0.359375 0.140625 0.734375 0.234375q0.390625 0.078125 0.734375 0.078125q0.421875 0 0.8125 -0.125q0.390625 -0.140625 0.6875 -0.453125q0.296875 -0.328125 0.46875 -0.859375q0.1875 -0.53125 0.1875 -1.3125zm8.964325 3.671875l-1.125 0l-0.03125 -1.15625q-0.328125 0.375 -0.625 0.625q-0.28125 0.234375 -0.578125 0.390625q-0.28125 0.140625 -0.578125 0.203125q-0.28125 0.0625 -0.59375 0.0625q-1.109375 0 -1.6875 -0.640625q-0.5625 -0.65625 -0.5625 -1.96875l0 -4.703125l1.25 0l0 4.59375q0 1.65625 1.25 1.65625q0.21875 0 0.4375 -0.0625q0.21875 -0.078125 0.453125 -0.234375q0.25 -0.171875 0.515625 -0.453125q0.28125 -0.296875 0.625 -0.734375l0 -4.765625l1.25 0l0 7.1875zm8.651825 -8.953125q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.5737 -1.765625q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.276825 3.21875q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm3.5737 -3.03125l1.140625 0l0.03125 1.328125q0.640625 -0.765625 1.2500153 -1.109375q0.625 -0.34375 1.25 -0.34375q1.125 0 1.703125 0.71875q0.578125 0.71875 0.53125 2.15625l-1.265625 0q0.03125 -0.953125 -0.265625 -1.375q-0.296875 -0.421875 -0.875 -0.421875q-0.25 0 -0.515625 0.09375q-0.25 0.078125 -0.515625 0.28125q-0.265625 0.1875 -0.56251526 0.5q-0.296875 0.3125 -0.640625 0.75l0 4.609375l-1.265625 0l0 -7.1875zm12.792465 3.875l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm3.3237 -3.875l1.140625 0l0.03125 1.328125q0.640625 -0.765625 1.25 -1.109375q0.625 -0.34375 1.25 -0.34375q1.125 0 1.703125 0.71875q0.578125 0.71875 0.53125 2.15625l-1.265625 0q0.03125 -0.953125 -0.265625 -1.375q-0.296875 -0.421875 -0.875 -0.421875q-0.25 0 -0.515625 0.09375q-0.25 0.078125 -0.515625 0.28125q-0.265625 0.1875 -0.5625 0.5q-0.296875 0.3125 -0.640625 0.75l0 4.609375l-1.265625 0l0 -7.1875zm13.839325 3.21875q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm8.964325 2.203125q0 0.375 -0.125 0.671875q-0.125 0.296875 -0.34375 0.53125q-0.21875 0.234375 -0.515625 0.40625q-0.296875 0.15625 -0.640625 0.265625q-0.328125 0.109375 -0.6875 0.15625q-0.34375 0.046875 -0.671875 0.046875q-0.734375 0 -1.34375 -0.0625q-0.609375 -0.0625 -1.203125 -0.203125l0 -1.140625q0.640625 0.171875 1.25 0.265625q0.625 0.09375 1.25 0.09375q0.890625 0 1.3125 -0.234375q0.4375 -0.25 0.4375 -0.703125q0 -0.1875 -0.078125 -0.34375q-0.0625 -0.15625 -0.25 -0.296875q-0.171875 -0.140625 -0.546875 -0.28125q-0.375 -0.15625 -1.015625 -0.359375q-0.5 -0.140625 -0.90625 -0.3125q-0.40625 -0.1875 -0.71875 -0.4375q-0.296875 -0.25 -0.46875 -0.578125q-0.171875 -0.34375 -0.171875 -0.8125q0 -0.296875 0.140625 -0.65625q0.140625 -0.359375 0.46875 -0.65625q0.34375 -0.3125 0.921875 -0.515625q0.578125 -0.203125 1.421875 -0.203125q0.421875 0 0.9375 0.046875q0.515625 0.046875 1.078125 0.15625l0 1.109375q-0.578125 -0.140625 -1.109375 -0.203125q-0.53125 -0.078125 -0.90625 -0.078125q-0.46875 0 -0.796875 0.078125q-0.3125 0.0625 -0.515625 0.203125q-0.1875 0.125 -0.28125 0.296875q-0.078125 0.15625 -0.078125 0.34375q0 0.203125 0.0625 0.359375q0.078125 0.15625 0.28125 0.3125q0.21875 0.140625 0.578125 0.28125q0.359375 0.140625 0.953125 0.3125q0.640625 0.1875 1.0625 0.390625q0.4375 0.203125 0.703125 0.46875q0.28125 0.25 0.390625 0.5625q0.125 0.3125 0.125 0.71875zm8.151825 1.953125l-1.125 0l-0.03125 -1.15625q-0.328125 0.375 -0.625 0.625q-0.28125 0.234375 -0.578125 0.390625q-0.28125 0.140625 -0.578125 0.203125q-0.28125 0.0625 -0.59375 0.0625q-1.109375 0 -1.6875 -0.640625q-0.5625 -0.65625 -0.5625 -1.96875l0 -4.703125l1.25 0l0 4.59375q0 1.65625 1.25 1.65625q0.21875 0 0.4375 -0.0625q0.21875 -0.078125 0.453125 -0.234375q0.25 -0.171875 0.515625 -0.453125q0.28125 -0.296875 0.625 -0.734375l0 -4.765625l1.25 0l0 7.1875zm4.72995 -9.09375l-2.125 0l0 -1.015625l3.390625 0l0 9.078125l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -8.0625zm11.401825 9.0q-0.421875 0.09375 -0.875 0.140625q-0.453125 0.046875 -0.921875 0.046875q-1.34375 0 -2.015625 -0.609375q-0.65625 -0.609375 -0.65625 -1.875l0 -3.75l-2.015625 0l0 -1.046875l2.015625 0l0 -1.96875l1.234375 -0.328125l0 2.296875l3.234375 0l0 1.046875l-3.234375 0l0 3.65625q0 0.765625 0.40625 1.15625q0.421875 0.375 1.21875 0.375q0.34375 0 0.75 -0.046875q0.40625 -0.0625 0.859375 -0.171875l0 1.078125zm7.9487 -1.859375q0 0.375 -0.125 0.671875q-0.125 0.296875 -0.34375 0.53125q-0.21875 0.234375 -0.515625 0.40625q-0.296875 0.15625 -0.640625 0.265625q-0.328125 0.109375 -0.6875 0.15625q-0.34375 0.046875 -0.671875 0.046875q-0.734375 0 -1.34375 -0.0625q-0.609375 -0.0625 -1.203125 -0.203125l0 -1.140625q0.640625 0.171875 1.25 0.265625q0.625 0.09375 1.25 0.09375q0.890625 0 1.3125 -0.234375q0.4375 -0.25 0.4375 -0.703125q0 -0.1875 -0.078125 -0.34375q-0.0625 -0.15625 -0.25 -0.296875q-0.171875 -0.140625 -0.546875 -0.28125q-0.375 -0.15625 -1.015625 -0.359375q-0.5 -0.140625 -0.90625 -0.3125q-0.40625 -0.1875 -0.71875 -0.4375q-0.296875 -0.25 -0.46875 -0.578125q-0.171875 -0.34375 -0.171875 -0.8125q0 -0.296875 0.140625 -0.65625q0.140625 -0.359375 0.46875 -0.65625q0.34375 -0.3125 0.921875 -0.515625q0.578125 -0.203125 1.421875 -0.203125q0.421875 0 0.9375 0.046875q0.515625 0.046875 1.078125 0.15625l0 1.109375q-0.578125 -0.140625 -1.109375 -0.203125q-0.53125 -0.078125 -0.90625 -0.078125q-0.46875 0 -0.796875 0.078125q-0.3125 0.0625 -0.515625 0.203125q-0.1875 0.125 -0.28125 0.296875q-0.078125 0.15625 -0.078125 0.34375q0 0.203125 0.0625 0.359375q0.078125 0.15625 0.28125 0.3125q0.21875 0.140625 0.578125 0.28125q0.359375 0.140625 0.953125 0.3125q0.640625 0.1875 1.0625 0.390625q0.4375 0.203125 0.703125 0.46875q0.28125 0.25 0.390625 0.5625q0.125 0.3125 0.125 0.71875zm7.401825 -1.359375l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm8.8237 3.21875q-0.421875 0.09375 -0.875 0.140625q-0.453125 0.046875 -0.921875 0.046875q-1.34375 0 -2.015625 -0.609375q-0.65625 -0.609375 -0.65625 -1.875l0 -3.75l-2.015625 0l0 -1.046875l2.015625 0l0 -1.96875l1.234375 -0.328125l0 2.296875l3.234375 0l0 1.046875l-3.234375 0l0 3.65625q0 0.765625 0.40625 1.15625q0.421875 0.375 1.21875 0.375q0.34375 0 0.75 -0.046875q0.40625 -0.0625 0.859375 -0.171875l0 1.078125zm8.526825 -3.5625q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm8.10495 0.28125l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm9.29245 -0.34375q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm8.85495 3.59375l-1.125 0l-0.03125 -1.15625q-0.328125 0.375 -0.625 0.625q-0.28125 0.234375 -0.578125 0.390625q-0.28125 0.140625 -0.578125 0.203125q-0.28125 0.0625 -0.59375 0.0625q-1.109375 0 -1.6875 -0.640625q-0.5625 -0.65625 -0.5625 -1.96875l0 -4.703125l1.25 0l0 4.59375q0 1.65625 1.25 1.65625q0.21875 0 0.4375 -0.0625q0.21875 -0.078125 0.453125 -0.234375q0.25 -0.171875 0.515625 -0.453125q0.28125 -0.296875 0.625 -0.734375l0 -4.765625l1.25 0l0 7.1875zm8.0737 -0.09375q-0.421875 0.09375 -0.875 0.140625q-0.453125 0.046875 -0.921875 0.046875q-1.34375 0 -2.015625 -0.609375q-0.65625 -0.609375 -0.65625 -1.875l0 -3.75l-2.015625 0l0 -1.046875l2.015625 0l0 -1.96875l1.234375 -0.328125l0 2.296875l3.234375 0l0 1.046875l-3.234375 0l0 3.65625q0 0.765625 0.40625 1.15625q0.421875 0.375 1.21875 0.375q0.34375 0 0.75 -0.046875q0.40625 -0.0625 0.859375 -0.171875l0 1.078125zm7.29245 -3.21875l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm9.16745 -0.40625q0 0.953125 -0.265625 1.671875q-0.265625 0.71875 -0.75 1.1875q-0.46875 0.46875 -1.109375 0.703125q-0.640625 0.234375 -1.390625 0.234375q-0.34375 0 -0.6875 -0.03125q-0.34375 -0.03125 -0.6875 -0.125l0 3.015625l-1.25 0l0 -10.125l1.109375 0l0.078125 1.203125q0.546875 -0.734375 1.15625 -1.03125q0.609375 -0.296875 1.3125 -0.296875q0.609375 0 1.078125 0.265625q0.46875 0.25 0.78125 0.71875q0.3125 0.46875 0.46875 1.140625q0.15625 0.65625 0.15625 1.46875zm-1.265625 0.046875q0 -0.5625 -0.09375 -1.03125q-0.078125 -0.46875 -0.265625 -0.796875q-0.171875 -0.34375 -0.453125 -0.53125q-0.265625 -0.1875 -0.65625 -0.1875q-0.234375 0 -0.484375 0.078125q-0.234375 0.0625 -0.5 0.25q-0.265625 0.171875 -0.5625 0.46875q-0.28125 0.28125 -0.609375 0.703125l0 3.484375q0.34375 0.140625 0.71875 0.234375q0.390625 0.078125 0.75 0.078125q1.0 0 1.578125 -0.6875q0.578125 -0.6875 0.578125 -2.0625zm7.808075 3.671875l-0.03125 -0.96875q-0.59375 0.578125 -1.203125 0.84375q-0.59375 0.25 -1.265625 0.25q-0.609375 0 -1.046875 -0.15625q-0.4375 -0.15625 -0.71875 -0.421875q-0.28125 -0.28125 -0.421875 -0.65625q-0.125 -0.375 -0.125 -0.8125q0 -1.078125 0.796875 -1.6875q0.8125 -0.609375 2.390625 -0.609375l1.484375 0l0 -0.640625q0 -0.625 -0.40625 -1.0q-0.40625 -0.390625 -1.25 -0.390625q-0.609375 0 -1.203125 0.140625q-0.578125 0.125 -1.21875 0.375l0 -1.125q0.25 -0.078125 0.53125 -0.15625q0.296875 -0.09375 0.609375 -0.15625q0.328125 -0.0625 0.671875 -0.09375q0.359375 -0.046875 0.71875 -0.046875q0.640625 0 1.15625 0.140625q0.53125 0.140625 0.890625 0.4375q0.359375 0.296875 0.546875 0.75q0.203125 0.4375 0.203125 1.03125l0 4.953125l-1.109375 0zm-0.140625 -3.265625l-1.578125 0q-0.46875 0 -0.8125 0.09375q-0.328125 0.09375 -0.546875 0.265625q-0.21875 0.171875 -0.328125 0.40625q-0.09375 0.234375 -0.09375 0.546875q0 0.203125 0.0625 0.390625q0.0625 0.1875 0.203125 0.34375q0.15625 0.140625 0.375 0.234375q0.234375 0.078125 0.5625 0.078125q0.4375 0 0.984375 -0.265625q0.5625 -0.265625 1.171875 -0.828125l0 -1.265625zm3.870575 -3.921875l1.140625 0l0.03125 1.328125q0.640625 -0.765625 1.25 -1.109375q0.625 -0.34375 1.25 -0.34375q1.125 0 1.703125 0.71875q0.578125 0.71875 0.53125 2.15625l-1.265625 0q0.03125 -0.953125 -0.265625 -1.375q-0.296875 -0.421875 -0.875 -0.421875q-0.25 0 -0.515625 0.09375q-0.25 0.078125 -0.515625 0.28125q-0.265625 0.1875 -0.5625 0.5q-0.296875 0.3125 -0.640625 0.75l0 4.609375l-1.265625 0l0 -7.1875zm12.3862 7.1875l-0.03125 -0.96875q-0.59375 0.578125 -1.203125 0.84375q-0.59375 0.25 -1.265625 0.25q-0.609375 0 -1.046875 -0.15625q-0.4375 -0.15625 -0.71875 -0.421875q-0.28125 -0.28125 -0.421875 -0.65625q-0.125 -0.375 -0.125 -0.8125q0 -1.078125 0.796875 -1.6875q0.8125 -0.609375 2.390625 -0.609375l1.484375 0l0 -0.640625q0 -0.625 -0.40625 -1.0q-0.40625 -0.390625 -1.25 -0.390625q-0.609375 0 -1.203125 0.140625q-0.578125 0.125 -1.21875 0.375l0 -1.125q0.25 -0.078125 0.53125 -0.15625q0.296875 -0.09375 0.609375 -0.15625q0.328125 -0.0625 0.671875 -0.09375q0.359375 -0.046875 0.71875 -0.046875q0.640625 0 1.15625 0.140625q0.53125 0.140625 0.890625 0.4375q0.359375 0.296875 0.546875 0.75q0.203125 0.4375 0.203125 1.03125l0 4.953125l-1.109375 0zm-0.140625 -3.265625l-1.578125 0q-0.46875 0 -0.8125 0.09375q-0.328125 0.09375 -0.546875 0.265625q-0.21875 0.171875 -0.328125 0.40625q-0.09375 0.234375 -0.09375 0.546875q0 0.203125 0.0625 0.390625q0.0625 0.1875 0.203125 0.34375q0.15625 0.140625 0.375 0.234375q0.234375 0.078125 0.5625 0.078125q0.4375 0 0.984375 -0.265625q0.5625 -0.265625 1.171875 -0.828125l0 -1.265625zm8.6987 3.265625l0 -5.15625q0 -0.34375 -0.03125 -0.546875q-0.015625 -0.21875 -0.078125 -0.34375q-0.046875 -0.125 -0.125 -0.171875q-0.078125 -0.0625 -0.203125 -0.0625q-0.140625 0 -0.265625 0.09375q-0.125 0.078125 -0.265625 0.28125q-0.140625 0.1875 -0.3125 0.515625q-0.15625 0.3125 -0.390625 0.78125l0 4.609375l-1.140625 0l0 -5.03125q0 -0.390625 -0.03125 -0.625q-0.015625 -0.25 -0.078125 -0.375q-0.046875 -0.140625 -0.140625 -0.1875q-0.078125 -0.0625 -0.203125 -0.0625q-0.125 0 -0.25 0.078125q-0.109375 0.0625 -0.25 0.25q-0.125 0.1875 -0.296875 0.515625q-0.171875 0.328125 -0.40625 0.828125l0 4.609375l-1.15625 0l0 -7.1875l0.953125 0l0.0625 1.375q0.1875 -0.40625 0.359375 -0.6875q0.171875 -0.296875 0.359375 -0.46875q0.1875 -0.1875 0.390625 -0.265625q0.21875 -0.078125 0.46875 -0.078125q0.59375 0 0.890625 0.375q0.296875 0.375 0.296875 1.171875q0.171875 -0.375 0.328125 -0.65625q0.171875 -0.296875 0.359375 -0.484375q0.1875 -0.203125 0.421875 -0.296875q0.234375 -0.109375 0.53125 -0.109375q1.34375 0 1.34375 2.078125l0 5.234375l-1.140625 0zm8.620575 -1.953125q0 0.375 -0.125 0.671875q-0.125 0.296875 -0.34375 0.53125q-0.21875 0.234375 -0.515625 0.40625q-0.296875 0.15625 -0.640625 0.265625q-0.328125 0.109375 -0.6875 0.15625q-0.34375 0.046875 -0.671875 0.046875q-0.734375 0 -1.34375 -0.0625q-0.609375 -0.0625 -1.203125 -0.203125l0 -1.140625q0.640625 0.171875 1.25 0.265625q0.625 0.09375 1.25 0.09375q0.890625 0 1.3125 -0.234375q0.4375 -0.25 0.4375 -0.703125q0 -0.1875 -0.078125 -0.34375q-0.0625 -0.15625 -0.25 -0.296875q-0.171875 -0.140625 -0.546875 -0.28125q-0.375 -0.15625 -1.015625 -0.359375q-0.5 -0.140625 -0.90625 -0.3125q-0.40625 -0.1875 -0.71875 -0.4375q-0.296875 -0.25 -0.46875 -0.578125q-0.171875 -0.34375 -0.171875 -0.8125q0 -0.296875 0.140625 -0.65625q0.140625 -0.359375 0.46875 -0.65625q0.34375 -0.3125 0.921875 -0.515625q0.578125 -0.203125 1.421875 -0.203125q0.421875 0 0.9375 0.046875q0.515625 0.046875 1.078125 0.15625l0 1.109375q-0.578125 -0.140625 -1.109375 -0.203125q-0.53125 -0.078125 -0.90625 -0.078125q-0.46875 0 -0.796875 0.078125q-0.3125 0.0625 -0.515625 0.203125q-0.1875 0.125 -0.28125 0.296875q-0.078125 0.15625 -0.078125 0.34375q0 0.203125 0.0625 0.359375q0.078125 0.15625 0.28125 0.3125q0.21875 0.140625 0.578125 0.28125q0.359375 0.140625 0.953125 0.3125q0.640625 0.1875 1.0625 0.390625q0.4375 0.203125 0.703125 0.46875q0.28125 0.25 0.390625 0.5625q0.125 0.3125 0.125 0.71875z" fill-rule="nonzero"/><path fill="#34a853" d="m163.16655 323.7223l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path stroke="#34a853" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m163.16655 323.7223l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path fill="#ffffff" d="m202.23354 340.0095l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm2.66745 -0.125q0 -0.921875 0.25 -1.625q0.25 -0.71875 0.703125 -1.203125q0.46875 -0.5 1.109375 -0.75q0.65625 -0.25 1.453125 -0.25q0.34375 0 0.671875 0.046875q0.328125 0.03125 0.65625 0.125l0 -3.015625l1.25 0l0 10.109375l-1.109375 0l-0.046875 -1.359375q-0.515625 0.765625 -1.125 1.125q-0.609375 0.359375 -1.328125 0.359375q-0.609375 0 -1.078125 -0.25q-0.46875 -0.265625 -0.78125 -0.734375q-0.3125 -0.46875 -0.46875 -1.125q-0.15625 -0.65625 -0.15625 -1.453125zm1.265625 -0.078125q0 1.296875 0.375 1.9375q0.390625 0.640625 1.09375 0.640625q0.484375 0 1.0 -0.421875q0.53125 -0.421875 1.109375 -1.25l0 -3.34375q-0.3125 -0.140625 -0.6875 -0.203125q-0.359375 -0.078125 -0.734375 -0.078125q-1.015625 0 -1.59375 0.65625q-0.5625 0.65625 -0.5625 2.0625zm7.4487 -3.671875l1.140625 0l0.03125 1.328125q0.640625 -0.765625 1.25 -1.109375q0.625 -0.34375 1.25 -0.34375q1.125 0 1.703125 0.71875q0.578125 0.71875 0.53125 2.15625l-1.265625 0q0.03125 -0.953125 -0.265625 -1.375q-0.296875 -0.421875 -0.875 -0.421875q-0.25 0 -0.515625 0.09375q-0.25 0.078125 -0.515625 0.28125q-0.265625 0.1875 -0.5625 0.5q-0.296875 0.3125 -0.640625 0.75l0 4.609375l-1.265625 0l0 -7.1875zm14.026825 3.53125q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm9.214325 -0.125q0 0.953125 -0.265625 1.671875q-0.265625 0.71875 -0.75 1.1875q-0.46875 0.46875 -1.109375 0.703125q-0.640625 0.234375 -1.390625 0.234375q-0.34375 0 -0.6875 -0.03125q-0.34375 -0.03125 -0.6875 -0.125l0 3.015625l-1.25 0l0 -10.125l1.109375 0l0.078125 1.203125q0.546875 -0.734375 1.15625 -1.03125q0.609375 -0.296875 1.3125 -0.296875q0.609375 0 1.078125 0.265625q0.46875 0.25 0.78125 0.71875q0.3125 0.46875 0.46875 1.140625q0.15625 0.65625 0.15625 1.46875zm-1.265625 0.046875q0 -0.5625 -0.09375 -1.03125q-0.078125 -0.46875 -0.265625 -0.796875q-0.171875 -0.34375 -0.453125 -0.53125q-0.265625 -0.1875 -0.65625 -0.1875q-0.234375 0 -0.484375 0.078125q-0.234375 0.0625 -0.5 0.25q-0.265625 0.171875 -0.5625 0.46875q-0.28125 0.28125 -0.609375 0.703125l0 3.484375q0.34375 0.140625 0.71875 0.234375q0.390625 0.078125 0.75 0.078125q1.0 0 1.578125 -0.6875q0.578125 -0.6875 0.578125 -2.0625zm8.214325 0.359375l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm9.10495 -0.65625q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm2.91745 0.71875q0 -0.78125 0.203125 -1.46875q0.203125 -0.703125 0.625 -1.21875q0.4375 -0.53125 1.09375 -0.828125q0.671875 -0.3125 1.5937653 -0.3125q0.359375 0 0.71875 0.0625q0.359375 0.046875 0.765625 0.171875l1.09375 -0.28125l0 10.25l-1.25 0l0 -2.71875l0.0625 -1.53125q-1.015625 1.4375 -2.4218903 1.4375q-0.625 0 -1.09375 -0.25q-0.46875 -0.265625 -0.78125 -0.734375q-0.296875 -0.46875 -0.453125 -1.125q-0.15625 -0.65625 -0.15625 -1.453125zm1.265625 -0.078125q0 0.59375 0.09375 1.078125q0.09375 0.484375 0.265625 0.8125q0.1875 0.328125 0.46875 0.515625q0.28125 0.171875 0.640625 0.171875q0.48439026 0 1.0000153 -0.421875q0.53125 -0.421875 1.109375 -1.25l0 -3.34375q-0.28125 -0.125 -0.65625 -0.203125q-0.359375 -0.09375 -0.765625 -0.09375q-1.0468903 0 -1.6093903 0.703125q-0.546875 0.6875 -0.546875 2.03125zm12.93309 3.515625l-1.125 0l-0.03125 -1.15625q-0.328125 0.375 -0.625 0.625q-0.28125 0.234375 -0.578125 0.390625q-0.28125 0.140625 -0.578125 0.203125q-0.28125 0.0625 -0.59375 0.0625q-1.109375 0 -1.6875 -0.640625q-0.5625 -0.65625 -0.5625 -1.96875l0 -4.703125l1.25 0l0 4.59375q0 1.65625 1.25 1.65625q0.21875 0 0.4375 -0.0625q0.21875 -0.078125 0.453125 -0.234375q0.25 -0.171875 0.515625 -0.453125q0.28125 -0.296875 0.625 -0.734375l0 -4.765625l1.25 0l0 7.1875zm4.72995 -6.15625l-2.125 0l0 -1.03125l3.390625 0l0 6.15625l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -5.125zm0.4375 -4.046875q0.203125 0 0.375 0.078125q0.1875 0.078125 0.3125 0.21875q0.140625 0.125 0.21875 0.3125q0.078125 0.171875 0.078125 0.375q0 0.203125 -0.078125 0.390625q-0.078125 0.171875 -0.21875 0.3125q-0.125 0.125 -0.3125 0.203125q-0.171875 0.078125 -0.375 0.078125q-0.203125 0 -0.390625 -0.078125q-0.171875 -0.078125 -0.3125 -0.203125q-0.125 -0.140625 -0.203125 -0.3125q-0.078125 -0.1875 -0.078125 -0.390625q0 -0.203125 0.078125 -0.375q0.078125 -0.1875 0.203125 -0.3125q0.140625 -0.140625 0.3125 -0.21875q0.1875 -0.078125 0.390625 -0.078125zm4.495575 3.015625l1.421875 0l1.765625 4.75l0.375 1.15625l0.390625 -1.1875l1.75 -4.71875l1.359375 0l-2.8125 7.1875l-1.4375 0l-2.8125 -7.1875zm13.35495 7.1875l-0.03125 -0.96875q-0.59375 0.578125 -1.203125 0.84375q-0.59375 0.25 -1.265625 0.25q-0.609375 0 -1.046875 -0.15625q-0.4375 -0.15625 -0.71875 -0.421875q-0.28125 -0.28125 -0.421875 -0.65625q-0.125 -0.375 -0.125 -0.8125q0 -1.078125 0.796875 -1.6875q0.8125 -0.609375 2.390625 -0.609375l1.484375 0l0 -0.640625q0 -0.625 -0.40625 -1.0q-0.40625 -0.390625 -1.25 -0.390625q-0.609375 0 -1.203125 0.140625q-0.578125 0.125 -1.21875 0.375l0 -1.125q0.25 -0.078125 0.53125 -0.15625q0.296875 -0.09375 0.609375 -0.15625q0.328125 -0.0625 0.671875 -0.09375q0.359375 -0.046875 0.71875 -0.046875q0.640625 0 1.15625 0.140625q0.53125 0.140625 0.890625 0.4375q0.359375 0.296875 0.546875 0.75q0.203125 0.4375 0.203125 1.03125l0 4.953125l-1.109375 0zm-0.140625 -3.265625l-1.578125 0q-0.46875 0 -0.8125 0.09375q-0.328125 0.09375 -0.546875 0.265625q-0.21875 0.171875 -0.328125 0.40625q-0.09375 0.234375 -0.09375 0.546875q0 0.203125 0.0625 0.390625q0.0625 0.1875 0.203125 0.34375q0.15625 0.140625 0.375 0.234375q0.234375 0.078125 0.5625 0.078125q0.4375 0 0.984375 -0.265625q0.5625 -0.265625 1.171875 -0.828125l0 -1.265625zm6.026825 -5.828125l-2.125 0l0 -1.015625l3.390625 0l0 9.078125l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -8.0625zm11.683075 5.125q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm3.276825 -3.03125l1.109375 0l0.046875 1.15625q0.328125 -0.359375 0.609375 -0.609375q0.296875 -0.25 0.578125 -0.390625q0.28125 -0.15625 0.578125 -0.21875q0.296875 -0.0625 0.609375 -0.0625q1.109375 0 1.671875 0.65625q0.578125 0.65625 0.578125 1.96875l0 4.6875l-1.25 0l0 -4.59375q0 -0.84375 -0.3125 -1.234375q-0.3125 -0.40625 -0.9375 -0.40625q-0.234375 0 -0.453125 0.0625q-0.21875 0.0625 -0.453125 0.234375q-0.234375 0.171875 -0.515625 0.453125q-0.265625 0.28125 -0.609375 0.71875l0 4.765625l-1.25 0l0 -7.1875zm13.85495 7.09375q-0.421875 0.09375 -0.875 0.140625q-0.453125 0.046875 -0.921875 0.046875q-1.34375 0 -2.015625 -0.609375q-0.65625 -0.609375 -0.65625 -1.875l0 -3.75l-2.015625 0l0 -1.046875l2.015625 0l0 -1.96875l1.234375 -0.328125l0 2.296875l3.234375 0l0 1.046875l-3.234375 0l0 3.65625q0 0.765625 0.40625 1.15625q0.421875 0.375 1.21875 0.375q0.34375 0 0.75 -0.046875q0.40625 -0.0625 0.859375 -0.171875l0 1.078125zm7.29245 -3.21875l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm9.16745 -0.40625q0 0.890625 -0.25 1.59375q-0.234375 0.703125 -0.703125 1.203125q-0.453125 0.484375 -1.109375 0.75q-0.640625 0.25 -1.453125 0.25q-0.640625 0 -1.3125 -0.109375q-0.65625 -0.125 -1.3125 -0.390625l0 -9.6875l1.25 0l0 2.78125l-0.0625 1.328125q0.546875 -0.71875 1.15625 -1.015625q0.609375 -0.296875 1.3125 -0.296875q0.609375 0 1.078125 0.265625q0.46875 0.25 0.78125 0.71875q0.3125 0.46875 0.46875 1.140625q0.15625 0.65625 0.15625 1.46875zm-1.265625 0.046875q0 -0.5625 -0.09375 -1.03125q-0.078125 -0.46875 -0.265625 -0.796875q-0.171875 -0.34375 -0.453125 -0.53125q-0.265625 -0.1875 -0.65625 -0.1875q-0.234375 0 -0.484375 0.078125q-0.234375 0.0625 -0.5 0.25q-0.265625 0.171875 -0.5625 0.46875q-0.28125 0.28125 -0.609375 0.703125l0 3.484375q0.359375 0.140625 0.734375 0.234375q0.390625 0.078125 0.734375 0.078125q0.421875 0 0.8125 -0.125q0.390625 -0.140625 0.6875 -0.453125q0.296875 -0.328125 0.46875 -0.859375q0.1875 -0.53125 0.1875 -1.3125zm8.964325 3.671875l-1.125 0l-0.03125 -1.15625q-0.328125 0.375 -0.625 0.625q-0.28125 0.234375 -0.578125 0.390625q-0.28125 0.140625 -0.578125 0.203125q-0.28125 0.0625 -0.59375 0.0625q-1.109375 0 -1.6875 -0.640625q-0.5625 -0.65625 -0.5625 -1.96875l0 -4.703125l1.25 0l0 4.59375q0 1.65625 1.25 1.65625q0.21875 0 0.4375 -0.0625q0.21875 -0.078125 0.453125 -0.234375q0.25 -0.171875 0.515625 -0.453125q0.28125 -0.296875 0.625 -0.734375l0 -4.765625l1.25 0l0 7.1875zm8.651825 -8.953125q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.5737 -1.765625q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.276825 3.21875q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm3.5737 -3.03125l1.140625 0l0.03125 1.328125q0.640625 -0.765625 1.25 -1.109375q0.625 -0.34375 1.25 -0.34375q1.125 0 1.703125 0.71875q0.578125 0.71875 0.53125 2.15625l-1.265625 0q0.03125 -0.953125 -0.265625 -1.375q-0.296875 -0.421875 -0.875 -0.421875q-0.25 0 -0.515625 0.09375q-0.25 0.078125 -0.515625 0.28125q-0.265625 0.1875 -0.5625 0.5q-0.296875 0.3125 -0.640625 0.75l0 4.609375l-1.265625 0l0 -7.1875zm12.79245 3.875l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm3.3237 -3.875l1.140625 0l0.03125 1.328125q0.640625 -0.765625 1.25 -1.109375q0.625 -0.34375 1.25 -0.34375q1.125 0 1.703125 0.71875q0.578125 0.71875 0.53125 2.15625l-1.265625 0q0.03125 -0.953125 -0.265625 -1.375q-0.296875 -0.421875 -0.875 -0.421875q-0.25 0 -0.515625 0.09375q-0.25 0.078125 -0.515625 0.28125q-0.265625 0.1875 -0.5625 0.5q-0.296875 0.3125 -0.640625 0.75l0 4.609375l-1.265625 0l0 -7.1875zm13.839325 3.21875q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm8.964325 2.203125q0 0.375 -0.125 0.671875q-0.125 0.296875 -0.34375 0.53125q-0.21875 0.234375 -0.515625 0.40625q-0.296875 0.15625 -0.640625 0.265625q-0.328125 0.109375 -0.6875 0.15625q-0.34375 0.046875 -0.671875 0.046875q-0.734375 0 -1.34375 -0.0625q-0.609375 -0.0625 -1.203125 -0.203125l0 -1.140625q0.640625 0.171875 1.25 0.265625q0.625 0.09375 1.25 0.09375q0.890625 0 1.3125 -0.234375q0.4375 -0.25 0.4375 -0.703125q0 -0.1875 -0.078125 -0.34375q-0.0625 -0.15625 -0.25 -0.296875q-0.171875 -0.140625 -0.546875 -0.28125q-0.375 -0.15625 -1.015625 -0.359375q-0.5 -0.140625 -0.90625 -0.3125q-0.40625 -0.1875 -0.71875 -0.4375q-0.296875 -0.25 -0.46875 -0.578125q-0.171875 -0.34375 -0.171875 -0.8125q0 -0.296875 0.140625 -0.65625q0.140625 -0.359375 0.46875 -0.65625q0.34375 -0.3125 0.921875 -0.515625q0.578125 -0.203125 1.421875 -0.203125q0.421875 0 0.9375 0.046875q0.515625 0.046875 1.078125 0.15625l0 1.109375q-0.578125 -0.140625 -1.109375 -0.203125q-0.53125 -0.078125 -0.90625 -0.078125q-0.46875 0 -0.796875 0.078125q-0.3125 0.0625 -0.515625 0.203125q-0.1875 0.125 -0.28125 0.296875q-0.078125 0.15625 -0.078125 0.34375q0 0.203125 0.0625 0.359375q0.078125 0.15625 0.28125 0.3125q0.21875 0.140625 0.578125 0.28125q0.359375 0.140625 0.953125 0.3125q0.640625 0.1875 1.0625 0.390625q0.4375 0.203125 0.703125 0.46875q0.28125 0.25 0.390625 0.5625q0.125 0.3125 0.125 0.71875zm8.151825 1.953125l-1.125 0l-0.03125 -1.15625q-0.328125 0.375 -0.625 0.625q-0.28125 0.234375 -0.578125 0.390625q-0.28125 0.140625 -0.578125 0.203125q-0.28125 0.0625 -0.59375 0.0625q-1.109375 0 -1.6875 -0.640625q-0.5625 -0.65625 -0.5625 -1.96875l0 -4.703125l1.25 0l0 4.59375q0 1.65625 1.25 1.65625q0.21875 0 0.4375 -0.0625q0.21875 -0.078125 0.453125 -0.234375q0.25 -0.171875 0.515625 -0.453125q0.28125 -0.296875 0.625 -0.734375l0 -4.765625l1.25 0l0 7.1875zm4.72995 -9.09375l-2.125 0l0 -1.015625l3.390625 0l0 9.078125l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -8.0625zm11.401825 9.0q-0.421875 0.09375 -0.875 0.140625q-0.453125 0.046875 -0.921875 0.046875q-1.34375 0 -2.015625 -0.609375q-0.65625 -0.609375 -0.65625 -1.875l0 -3.75l-2.015625 0l0 -1.046875l2.015625 0l0 -1.96875l1.234375 -0.328125l0 2.296875l3.234375 0l0 1.046875l-3.234375 0l0 3.65625q0 0.765625 0.40625 1.15625q0.421875 0.375 1.21875 0.375q0.34375 0 0.75 -0.046875q0.40625 -0.0625 0.859375 -0.171875l0 1.078125zm7.9487 -1.859375q0 0.375 -0.125 0.671875q-0.125 0.296875 -0.34375 0.53125q-0.21875 0.234375 -0.515625 0.40625q-0.296875 0.15625 -0.640625 0.265625q-0.328125 0.109375 -0.6875 0.15625q-0.34375 0.046875 -0.671875 0.046875q-0.734375 0 -1.34375 -0.0625q-0.609375 -0.0625 -1.203125 -0.203125l0 -1.140625q0.640625 0.171875 1.25 0.265625q0.625 0.09375 1.25 0.09375q0.890625 0 1.3125 -0.234375q0.4375 -0.25 0.4375 -0.703125q0 -0.1875 -0.078125 -0.34375q-0.0625 -0.15625 -0.25 -0.296875q-0.171875 -0.140625 -0.546875 -0.28125q-0.375 -0.15625 -1.015625 -0.359375q-0.5 -0.140625 -0.90625 -0.3125q-0.40625 -0.1875 -0.71875 -0.4375q-0.296875 -0.25 -0.46875 -0.578125q-0.171875 -0.34375 -0.171875 -0.8125q0 -0.296875 0.140625 -0.65625q0.140625 -0.359375 0.46875 -0.65625q0.34375 -0.3125 0.921875 -0.515625q0.578125 -0.203125 1.421875 -0.203125q0.421875 0 0.9375 0.046875q0.515625 0.046875 1.078125 0.15625l0 1.109375q-0.578125 -0.140625 -1.109375 -0.203125q-0.53125 -0.078125 -0.90625 -0.078125q-0.46875 0 -0.796875 0.078125q-0.3125 0.0625 -0.515625 0.203125q-0.1875 0.125 -0.28125 0.296875q-0.078125 0.15625 -0.078125 0.34375q0 0.203125 0.0625 0.359375q0.078125 0.15625 0.28125 0.3125q0.21875 0.140625 0.578125 0.28125q0.359375 0.140625 0.953125 0.3125q0.640625 0.1875 1.0625 0.390625q0.4375 0.203125 0.703125 0.46875q0.28125 0.25 0.390625 0.5625q0.125 0.3125 0.125 0.71875z" fill-rule="nonzero"/><path fill="#ea4335" d="m163.2269 19.56168l316.7874 0l0 29.039371l-316.7874 0z" fill-rule="evenodd"/><path stroke="#ea4335" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m163.2269 19.56168l316.7874 0l0 29.039371l-316.7874 0z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m176.57526 28.276524l290.0907 0l0 17.16098l-290.0907 0l0 -17.16098z" fill-rule="nonzero"/><path fill="#ffffff" d="m182.24713 34.723866q0.015625 -0.421875 -0.046875 -0.703125q-0.046875 -0.28125 -0.171875 -0.46875q-0.109375 -0.1875 -0.28125 -0.265625q-0.171875 -0.09375 -0.390625 -0.09375q-0.390625 0 -0.796875 0.328125q-0.40625 0.3125 -0.90625 1.046875l0 4.59375l-1.796875 0l0 -7.265625l1.59375 0l0.0625 1.046875q0.171875 -0.265625 0.390625 -0.484375q0.234375 -0.234375 0.5 -0.390625q0.28125 -0.15625 0.609375 -0.234375q0.34375 -0.09375 0.75 -0.09375q0.546875 0 0.96875 0.1875q0.4375 0.1875 0.71875 0.5625q0.296875 0.359375 0.4375 0.921875q0.140625 0.5625 0.109375 1.3125l-1.75 0zm9.714325 0.25q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm9.47995 4.4375l-1.9375 0l-0.78125 -2.296875l-0.234375 -0.8125l-0.234375 0.828125l-0.75 2.28125l-1.9375 0l-1.03125 -7.265625l1.65625 0l0.515625 4.109375l0.125 1.109375l0.296875 -1.0l0.734375 -2.21875l1.328125 0l0.78125 2.203125l0.328125 1.03125l0.140625 -1.140625l0.4375 -4.09375l1.578125 0l-1.015625 7.265625zm6.7612 -4.4375q0.015625 -0.421875 -0.046875 -0.703125q-0.046875 -0.28125 -0.171875 -0.46875q-0.109375 -0.1875 -0.28125 -0.265625q-0.171875 -0.09375 -0.390625 -0.09375q-0.390625 0 -0.796875 0.328125q-0.40625 0.3125 -0.90625 1.046875l0 4.59375l-1.796875 0l0 -7.265625l1.59375 0l0.0625 1.046875q0.171875 -0.265625 0.390625 -0.484375q0.234375 -0.234375 0.5 -0.390625q0.28125 -0.15625 0.609375 -0.234375q0.34375 -0.09375 0.75 -0.09375q0.546875 0 0.96875 0.1875q0.4375 0.1875 0.71875 0.5625q0.296875 0.359375 0.4375 0.921875q0.140625 0.5625 0.109375 1.3125l-1.75 0zm7.66745 -4.828125q0 0.234375 -0.09375 0.453125q-0.078125 0.203125 -0.234375 0.359375q-0.15625 0.15625 -0.375 0.25q-0.203125 0.078125 -0.4375 0.078125q-0.25 0 -0.46875 -0.078125q-0.203125 -0.09375 -0.359375 -0.25q-0.15625 -0.15625 -0.25 -0.359375q-0.078125 -0.21875 -0.078125 -0.453125q0 -0.234375 0.078125 -0.4375q0.09375 -0.203125 0.25 -0.359375q0.15625 -0.15625 0.359375 -0.25q0.21875 -0.09375 0.46875 -0.09375q0.234375 0 0.4375 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.234375 0.359375q0.09375 0.203125 0.09375 0.4375zm-1.9375 3.359375l-2.0625 0l0 -1.359375l3.84375 0l0 5.90625l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -4.546875zm11.6987 5.828125q-0.46875 0.109375 -0.953125 0.171875q-0.46875 0.078125 -0.90625 0.078125q-0.703125 0 -1.234375 -0.15625q-0.515625 -0.15625 -0.859375 -0.453125q-0.328125 -0.3125 -0.5 -0.78125q-0.15625 -0.484375 -0.15625 -1.140625l0 -3.546875l-1.953125 0l0 -1.359375l1.953125 0l0 -1.859375l1.796875 -0.46875l0 2.328125l2.8125 0l0 1.359375l-2.8125 0l0 3.421875q0 0.609375 0.28125 0.9375q0.28125 0.3125 0.953125 0.3125q0.4375 0 0.84375 -0.0625q0.40625 -0.078125 0.734375 -0.171875l0 1.390625zm8.401825 -4.109375q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm2.5112 7.375l0 -1.390625l8.0625 0l0 1.390625l-8.0625 0zm13.339325 -12.203125q0 0.234375 -0.09375 0.453125q-0.078125 0.203125 -0.234375 0.359375q-0.15625 0.15625 -0.375 0.25q-0.203125 0.078125 -0.4375 0.078125q-0.25 0 -0.46875 -0.078125q-0.203125 -0.09375 -0.359375 -0.25q-0.15625 -0.15625 -0.25 -0.359375q-0.078125 -0.21875 -0.078125 -0.453125q0 -0.234375 0.078125 -0.4375q0.09375 -0.203125 0.25 -0.359375q0.15625 -0.15625 0.359375 -0.25q0.21875 -0.09375 0.46875 -0.09375q0.234375 0 0.4375 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.234375 0.359375q0.09375 0.203125 0.09375 0.4375zm-1.9375 3.359375l-2.0625 0l0 -1.359375l3.84375 0l0 5.90625l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -4.546875zm10.04245 5.90625l0 -4.734375q0 -1.203125 -0.890625 -1.203125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 4.59375l-1.734375 0l0 -7.265625l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.546875 0 0.953125 0.1875q0.421875 0.171875 0.703125 0.515625q0.28125 0.328125 0.421875 0.796875q0.140625 0.46875 0.140625 1.046875l0 4.875l-1.75 0zm2.72995 2.9375l0 -1.390625l8.0625 0l0 1.390625l-8.0625 0zm13.620575 -2.9375l-0.03125 -1.078125q-0.234375 0.28125 -0.484375 0.515625q-0.234375 0.21875 -0.515625 0.390625q-0.28125 0.15625 -0.609375 0.25q-0.3125 0.09375 -0.6875 0.09375q-0.65625 0 -1.15625 -0.25q-0.5 -0.265625 -0.84375 -0.734375q-0.328125 -0.484375 -0.515625 -1.15625q-0.171875 -0.6875 -0.171875 -1.53125q0 -1.0 0.28125 -1.734375q0.28125 -0.734375 0.78125 -1.21875q0.5 -0.484375 1.1875 -0.71875q0.6875 -0.234375 1.5 -0.234375q0.265625 0 0.53125 0.046875q0.28125 0.03125 0.5 0.09375l0 -2.84375l1.75 0l0 10.109375l-1.515625 0zm-3.1875 -3.578125q0 0.59375 0.078125 1.03125q0.09375 0.421875 0.25 0.703125q0.15625 0.265625 0.390625 0.390625q0.234375 0.125 0.546875 0.125q0.4375 0 0.828125 -0.359375q0.40625 -0.359375 0.859375 -0.984375l0 -3.171875q-0.203125 -0.078125 -0.515625 -0.125q-0.296875 -0.0625 -0.609375 -0.0625q-0.421875 0 -0.765625 0.171875q-0.328125 0.171875 -0.5625 0.5q-0.234375 0.3125 -0.375 0.765625q-0.125 0.453125 -0.125 1.015625zm13.0112 -0.609375q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm9.495575 2.296875q0 0.625 -0.28125 1.0625q-0.265625 0.4375 -0.71875 0.71875q-0.453125 0.28125 -1.03125 0.40625q-0.578125 0.125 -1.171875 0.125q-0.796875 0 -1.453125 -0.078125q-0.640625 -0.078125 -1.21875 -0.21875l0 -1.578125q0.6875 0.28125 1.34375 0.40625q0.671875 0.125 1.265625 0.125q0.6875 0 1.03125 -0.21875q0.34375 -0.21875 0.34375 -0.5625q0 -0.171875 -0.078125 -0.296875q-0.078125 -0.140625 -0.28125 -0.265625q-0.1875 -0.125 -0.546875 -0.25q-0.359375 -0.140625 -0.9375 -0.3125q-0.53125 -0.15625 -0.9375 -0.34375q-0.40625 -0.1875 -0.671875 -0.4375q-0.265625 -0.265625 -0.40625 -0.59375q-0.125 -0.34375 -0.125 -0.8125q0 -0.4375 0.203125 -0.828125q0.203125 -0.390625 0.59375 -0.6875q0.40625 -0.296875 1.0 -0.46875q0.59375 -0.171875 1.390625 -0.171875q0.6875 0 1.21875 0.078125q0.53125 0.0625 0.9375 0.140625l0 1.421875q-0.625 -0.1875 -1.171875 -0.265625q-0.546875 -0.09375 -1.09375 -0.09375q-0.53125 0 -0.859375 0.203125q-0.328125 0.1875 -0.328125 0.53125q0 0.15625 0.0625 0.296875q0.078125 0.125 0.265625 0.25q0.1875 0.109375 0.515625 0.25q0.34375 0.125 0.90625 0.28125q0.625 0.1875 1.046875 0.390625q0.4375 0.203125 0.703125 0.46875q0.265625 0.25 0.375 0.578125q0.109375 0.328125 0.109375 0.75zm8.058075 2.0625q-0.46875 0.109375 -0.953125 0.171875q-0.46875 0.078125 -0.90625 0.078125q-0.703125 0 -1.234375 -0.15625q-0.515625 -0.15625 -0.859375 -0.453125q-0.328125 -0.3125 -0.5 -0.78125q-0.15625 -0.484375 -0.15625 -1.140625l0 -3.546875l-1.953125 0l0 -1.359375l1.953125 0l0 -1.859375l1.796875 -0.46875l0 2.328125l2.8125 0l0 1.359375l-2.8125 0l0 3.421875q0 0.609375 0.28125 0.9375q0.28125 0.3125 0.953125 0.3125q0.4375 0 0.84375 -0.0625q0.40625 -0.078125 0.734375 -0.171875l0 1.390625zm6.35495 -9.1875q0 0.234375 -0.09375 0.453125q-0.078125 0.203125 -0.234375 0.359375q-0.15625 0.15625 -0.375 0.25q-0.203125 0.078125 -0.4375 0.078125q-0.25 0 -0.46875 -0.078125q-0.203125 -0.09375 -0.359375 -0.25q-0.15625 -0.15625 -0.25 -0.359375q-0.078125 -0.21875 -0.078125 -0.453125q0 -0.234375 0.078125 -0.4375q0.09375 -0.203125 0.25 -0.359375q0.15625 -0.15625 0.359375 -0.25q0.21875 -0.09375 0.46875 -0.09375q0.234375 0 0.4375 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.234375 0.359375q0.09375 0.203125 0.09375 0.4375zm-1.9375 3.359375l-2.0625 0l0 -1.359375l3.84375 0l0 5.90625l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -4.546875zm10.04245 5.90625l0 -4.734375q0 -1.203125 -0.890625 -1.203125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 4.59375l-1.734375 0l0 -7.265625l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.546875 0 0.953125 0.1875q0.421875 0.171875 0.703125 0.515625q0.28125 0.328125 0.421875 0.796875q0.140625 0.46875 0.140625 1.046875l0 4.875l-1.75 0zm8.276825 0l-0.03125 -0.953125q-0.234375 0.25 -0.484375 0.453125q-0.25 0.203125 -0.5625 0.359375q-0.296875 0.140625 -0.65625 0.21875q-0.34375 0.09375 -0.765625 0.09375q-0.5625 0 -0.984375 -0.171875q-0.421875 -0.171875 -0.703125 -0.453125q-0.28125 -0.296875 -0.4375 -0.703125q-0.140625 -0.421875 -0.140625 -0.921875q0 -0.515625 0.21875 -0.953125q0.21875 -0.4375 0.65625 -0.75q0.453125 -0.3125 1.125 -0.484375q0.671875 -0.1875 1.5625 -0.1875l0.953125 0l0 -0.4375q0 -0.28125 -0.078125 -0.5q-0.078125 -0.234375 -0.25 -0.390625q-0.171875 -0.15625 -0.453125 -0.234375q-0.28125 -0.09375 -0.6875 -0.09375q-0.640625 0 -1.265625 0.15625q-0.625 0.140625 -1.21875 0.40625l0 -1.40625q0.53125 -0.203125 1.203125 -0.328125q0.6875 -0.140625 1.421875 -0.140625q0.8125 0 1.390625 0.15625q0.578125 0.140625 0.953125 0.453125q0.375 0.3125 0.546875 0.78125q0.1875 0.453125 0.1875 1.0625l0 4.96875l-1.5 0zm-0.25 -3.21875l-1.0625 0q-0.4375 0 -0.75 0.09375q-0.3125 0.078125 -0.5 0.234375q-0.1875 0.15625 -0.28125 0.359375q-0.09375 0.1875 -0.09375 0.40625q0 0.4375 0.28125 0.671875q0.296875 0.234375 0.78125 0.234375q0.375 0 0.765625 -0.265625q0.390625 -0.265625 0.859375 -0.75l0 -0.984375zm9.745575 3.140625q-0.46875 0.109375 -0.953125 0.171875q-0.46875 0.078125 -0.90625 0.078125q-0.703125 0 -1.234375 -0.15625q-0.515625 -0.15625 -0.859375 -0.453125q-0.328125 -0.3125 -0.5 -0.78125q-0.15625 -0.484375 -0.15625 -1.140625l0 -3.546875l-1.953125 0l0 -1.359375l1.953125 0l0 -1.859375l1.796875 -0.46875l0 2.328125l2.8125 0l0 1.359375l-2.8125 0l0 3.421875q0 0.609375 0.28125 0.9375q0.28125 0.3125 0.953125 0.3125q0.4375 0 0.84375 -0.0625q0.40625 -0.078125 0.734375 -0.171875l0 1.390625zm6.35495 -9.1875q0 0.234375 -0.09375 0.453125q-0.078125 0.203125 -0.234375 0.359375q-0.15625 0.15625 -0.375 0.25q-0.203125 0.078125 -0.4375 0.078125q-0.25 0 -0.46875 -0.078125q-0.203125 -0.09375 -0.359375 -0.25q-0.15625 -0.15625 -0.25 -0.359375q-0.078125 -0.21875 -0.078125 -0.453125q0 -0.234375 0.078125 -0.4375q0.09375 -0.203125 0.25 -0.359375q0.15625 -0.15625 0.359375 -0.25q0.21875 -0.09375 0.46875 -0.09375q0.234375 0 0.4375 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.234375 0.359375q0.09375 0.203125 0.09375 0.4375zm-1.9375 3.359375l-2.0625 0l0 -1.359375l3.84375 0l0 5.90625l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -4.546875zm12.2612 2.21875q0 0.859375 -0.25 1.578125q-0.234375 0.703125 -0.703125 1.21875q-0.453125 0.5 -1.125 0.78125q-0.65625 0.28125 -1.515625 0.28125q-0.796875 0 -1.4375 -0.234375q-0.640625 -0.25 -1.09375 -0.71875q-0.4375 -0.46875 -0.671875 -1.171875q-0.234375 -0.703125 -0.234375 -1.640625q0 -0.859375 0.25 -1.5625q0.25 -0.71875 0.703125 -1.21875q0.46875 -0.5 1.125 -0.765625q0.671875 -0.28125 1.5 -0.28125q0.828125 0 1.453125 0.234375q0.640625 0.234375 1.078125 0.71875q0.453125 0.46875 0.6875 1.171875q0.234375 0.6875 0.234375 1.609375zm-1.828125 0.03125q0 -1.125 -0.421875 -1.6875q-0.421875 -0.578125 -1.25 -0.578125q-0.46875 0 -0.796875 0.1875q-0.3125 0.171875 -0.515625 0.484375q-0.203125 0.3125 -0.3125 0.734375q-0.09375 0.40625 -0.09375 0.875q0 1.140625 0.453125 1.71875q0.46875 0.578125 1.265625 0.578125q0.4375 0 0.75 -0.171875q0.328125 -0.1875 0.53125 -0.484375q0.203125 -0.3125 0.296875 -0.734375q0.09375 -0.421875 0.09375 -0.921875zm7.66745 3.65625l0 -4.734375q0 -1.203125 -0.890625 -1.203125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 4.59375l-1.734375 0l0 -7.265625l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.546875 0 0.953125 0.1875q0.421875 0.171875 0.703125 0.515625q0.28125 0.328125 0.421875 0.796875q0.140625 0.46875 0.140625 1.046875l0 4.875l-1.75 0zm2.72995 2.9375l0 -1.390625l8.0625 0l0 1.390625l-8.0625 0zm15.589325 -6.703125q0 1.0 -0.28125 1.734375q-0.28125 0.734375 -0.78125 1.21875q-0.5 0.46875 -1.1875 0.703125q-0.6875 0.234375 -1.5 0.234375q-0.28125 0 -0.546875 -0.03125q-0.265625 -0.03125 -0.5 -0.09375l0 2.84375l-1.734375 0l0 -10.109375l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.640625 0 1.140625 0.265625q0.5 0.25 0.828125 0.71875q0.34375 0.46875 0.515625 1.15625q0.1875 0.671875 0.1875 1.515625zm-1.828125 0.078125q0 -0.609375 -0.09375 -1.03125q-0.078125 -0.421875 -0.25 -0.6875q-0.15625 -0.28125 -0.390625 -0.40625q-0.21875 -0.125 -0.53125 -0.125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 3.171875q0.21875 0.078125 0.515625 0.125q0.3125 0.046875 0.625 0.046875q0.40625 0 0.75 -0.171875q0.34375 -0.171875 0.578125 -0.484375q0.25 -0.3125 0.375 -0.765625q0.125 -0.453125 0.125 -1.015625zm7.901825 3.6875l-0.03125 -0.953125q-0.234375 0.25 -0.484375 0.453125q-0.25 0.203125 -0.5625 0.359375q-0.296875 0.140625 -0.65625 0.21875q-0.34375 0.09375 -0.765625 0.09375q-0.5625 0 -0.984375 -0.171875q-0.421875 -0.171875 -0.703125 -0.453125q-0.28125 -0.296875 -0.4375 -0.703125q-0.140625 -0.421875 -0.140625 -0.921875q0 -0.515625 0.21875 -0.953125q0.21875 -0.4375 0.65625 -0.75q0.453125 -0.3125 1.125 -0.484375q0.671875 -0.1875 1.5625 -0.1875l0.953125 0l0 -0.4375q0 -0.28125 -0.078125 -0.5q-0.078125 -0.234375 -0.25 -0.390625q-0.171875 -0.15625 -0.453125 -0.234375q-0.28125 -0.09375 -0.6875 -0.09375q-0.640625 0 -1.265625 0.15625q-0.625 0.140625 -1.21875 0.40625l0 -1.40625q0.53125 -0.203125 1.203125 -0.328125q0.6875 -0.140625 1.421875 -0.140625q0.8125 0 1.390625 0.15625q0.578125 0.140625 0.953125 0.453125q0.375 0.3125 0.546875 0.78125q0.1875 0.453125 0.1875 1.0625l0 4.96875l-1.5 0zm-0.25 -3.21875l-1.0625 0q-0.4375 0 -0.75 0.09375q-0.3125 0.078125 -0.5 0.234375q-0.1875 0.15625 -0.28125 0.359375q-0.09375 0.1875 -0.09375 0.40625q0 0.4375 0.28125 0.671875q0.296875 0.234375 0.78125 0.234375q0.375 0 0.765625 -0.265625q0.390625 -0.265625 0.859375 -0.75l0 -0.984375zm9.745575 1.078125q0 0.625 -0.28125 1.0625q-0.265625 0.4375 -0.71875 0.71875q-0.453125 0.28125 -1.03125 0.40625q-0.578125 0.125 -1.171875 0.125q-0.796875 0 -1.453125 -0.078125q-0.640625 -0.078125 -1.21875 -0.21875l0 -1.578125q0.6875 0.28125 1.34375 0.40625q0.671875 0.125 1.265625 0.125q0.6875 0 1.03125 -0.21875q0.34375 -0.21875 0.34375 -0.5625q0 -0.171875 -0.078125 -0.296875q-0.078125 -0.140625 -0.28125 -0.265625q-0.1875 -0.125 -0.546875 -0.25q-0.359375 -0.140625 -0.9375 -0.3125q-0.53125 -0.15625 -0.9375 -0.34375q-0.40625 -0.1875 -0.671875 -0.4375q-0.265625 -0.265625 -0.40625 -0.59375q-0.125 -0.34375 -0.125 -0.8125q0 -0.4375 0.203125 -0.828125q0.203125 -0.390625 0.59375 -0.6875q0.40625 -0.296875 1.0 -0.46875q0.59375 -0.171875 1.390625 -0.171875q0.6875 0 1.21875 0.078125q0.53125 0.0625 0.9375 0.140625l0 1.421875q-0.625 -0.1875 -1.171875 -0.265625q-0.546875 -0.09375 -1.09375 -0.09375q-0.53125 0 -0.859375 0.203125q-0.328125 0.1875 -0.328125 0.53125q0 0.15625 0.0625 0.296875q0.078125 0.125 0.265625 0.25q0.1875 0.109375 0.515625 0.25q0.34375 0.125 0.90625 0.28125q0.625 0.1875 1.046875 0.390625q0.4375 0.203125 0.703125 0.46875q0.265625 0.25 0.375 0.578125q0.109375 0.328125 0.109375 0.75zm8.058075 0q0 0.625 -0.28125 1.0625q-0.265625 0.4375 -0.71875 0.71875q-0.453125 0.28125 -1.03125 0.40625q-0.578125 0.125 -1.171875 0.125q-0.796875 0 -1.453125 -0.078125q-0.640625 -0.078125 -1.21875 -0.21875l0 -1.578125q0.6875 0.28125 1.34375 0.40625q0.671875 0.125 1.265625 0.125q0.6875 0 1.03125 -0.21875q0.34375 -0.21875 0.34375 -0.5625q0 -0.171875 -0.078125 -0.296875q-0.078125 -0.140625 -0.28125 -0.265625q-0.1875 -0.125 -0.546875 -0.25q-0.359375 -0.140625 -0.9375 -0.3125q-0.53125 -0.15625 -0.9375 -0.34375q-0.40625 -0.1875 -0.671875 -0.4375q-0.265625 -0.265625 -0.40625 -0.59375q-0.125 -0.34375 -0.125 -0.8125q0 -0.4375 0.203125 -0.828125q0.203125 -0.390625 0.59375 -0.6875q0.40625 -0.296875 1.0 -0.46875q0.59375 -0.171875 1.390625 -0.171875q0.6875 0 1.21875 0.078125q0.53125 0.0625 0.9375 0.140625l0 1.421875q-0.625 -0.1875 -1.171875 -0.265625q-0.546875 -0.09375 -1.09375 -0.09375q-0.53125 0 -0.859375 0.203125q-0.328125 0.1875 -0.328125 0.53125q0 0.15625 0.0625 0.296875q0.078125 0.125 0.265625 0.25q0.1875 0.109375 0.515625 0.25q0.34375 0.125 0.90625 0.28125q0.625 0.1875 1.046875 0.390625q0.4375 0.203125 0.703125 0.46875q0.265625 0.25 0.375 0.578125q0.109375 0.328125 0.109375 0.75zm6.35495 -7.125q0 0.234375 -0.09375 0.453125q-0.078125 0.203125 -0.234375 0.359375q-0.15625 0.15625 -0.375 0.25q-0.203125 0.078125 -0.4375 0.078125q-0.25 0 -0.46875 -0.078125q-0.203125 -0.09375 -0.359375 -0.25q-0.15625 -0.15625 -0.25 -0.359375q-0.078125 -0.21875 -0.078125 -0.453125q0 -0.234375 0.078125 -0.4375q0.09375 -0.203125 0.25 -0.359375q0.15625 -0.15625 0.359375 -0.25q0.21875 -0.09375 0.46875 -0.09375q0.234375 0 0.4375 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.234375 0.359375q0.09375 0.203125 0.09375 0.4375zm-1.9375 3.359375l-2.0625 0l0 -1.359375l3.84375 0l0 5.90625l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -4.546875zm10.04245 5.90625l0 -4.734375q0 -1.203125 -0.890625 -1.203125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 4.59375l-1.734375 0l0 -7.265625l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.546875 0 0.953125 0.1875q0.421875 0.171875 0.703125 0.515625q0.28125 0.328125 0.421875 0.796875q0.140625 0.46875 0.140625 1.046875l0 4.875l-1.75 0zm9.183075 -6.0q0.1875 0.25 0.265625 0.53125q0.09375 0.265625 0.09375 0.5625q0 0.609375 -0.21875 1.078125q-0.203125 0.453125 -0.59375 0.78125q-0.375 0.3125 -0.90625 0.46875q-0.53125 0.15625 -1.171875 0.15625q-0.390625 0 -0.71875 -0.0625q-0.3125 -0.078125 -0.46875 -0.1875q-0.125 0.125 -0.21875 0.28125q-0.078125 0.140625 -0.078125 0.328125q0 0.109375 0.046875 0.234375q0.0625 0.109375 0.171875 0.203125q0.109375 0.078125 0.265625 0.140625q0.15625 0.046875 0.34375 0.0625l1.734375 0.0625q0.578125 0.015625 1.046875 0.15625q0.46875 0.125 0.796875 0.375q0.34375 0.234375 0.515625 0.578125q0.1875 0.34375 0.1875 0.796875q0 0.5 -0.234375 0.953125q-0.21875 0.453125 -0.6875 0.78125q-0.453125 0.34375 -1.140625 0.53125q-0.6875 0.1875 -1.59375 0.1875q-0.890625 0 -1.53125 -0.140625q-0.625 -0.125 -1.03125 -0.390625q-0.390625 -0.25 -0.59375 -0.59375q-0.1875 -0.34375 -0.1875 -0.765625q0 -0.25 0.0625 -0.46875q0.0625 -0.203125 0.1875 -0.40625q0.140625 -0.1875 0.328125 -0.375q0.1875 -0.1875 0.453125 -0.375q-0.34375 -0.203125 -0.53125 -0.515625q-0.1875 -0.3125 -0.1875 -0.671875q0 -0.25 0.0625 -0.453125q0.078125 -0.21875 0.171875 -0.40625q0.109375 -0.1875 0.234375 -0.359375q0.140625 -0.1875 0.296875 -0.359375q-0.265625 -0.265625 -0.4375 -0.640625q-0.171875 -0.375 -0.171875 -0.9375q0 -0.609375 0.21875 -1.078125q0.21875 -0.484375 0.59375 -0.8125q0.390625 -0.328125 0.921875 -0.484375q0.53125 -0.171875 1.171875 -0.171875q0.3125 0 0.609375 0.03125q0.296875 0.03125 0.546875 0.109375l2.515625 0l0 1.265625l-1.140625 0zm-4.125 6.96875q0 0.421875 0.421875 0.625q0.4375 0.203125 1.203125 0.203125q0.5 0 0.828125 -0.09375q0.34375 -0.09375 0.5625 -0.25q0.21875 -0.15625 0.3125 -0.359375q0.09375 -0.1875 0.09375 -0.40625q0 -0.1875 -0.09375 -0.328125q-0.09375 -0.125 -0.25 -0.21875q-0.15625 -0.09375 -0.375 -0.140625q-0.21875 -0.046875 -0.484375 -0.0625l-1.546875 -0.015625q-0.203125 0.140625 -0.34375 0.265625q-0.125 0.125 -0.203125 0.25q-0.0625 0.125 -0.09375 0.25q-0.03125 0.140625 -0.03125 0.28125zm0.296875 -5.828125q0 0.578125 0.34375 0.921875q0.34375 0.328125 0.953125 0.328125q0.3125 0 0.546875 -0.109375q0.25 -0.109375 0.40625 -0.28125q0.15625 -0.171875 0.234375 -0.40625q0.078125 -0.25 0.078125 -0.5q0 -0.609375 -0.34375 -0.953125q-0.328125 -0.34375 -0.9375 -0.34375q-0.328125 0 -0.578125 0.109375q-0.234375 0.109375 -0.390625 0.296875q-0.15625 0.171875 -0.234375 0.421875q-0.078125 0.25 -0.078125 0.515625zm5.433075 7.796875l0 -1.390625l8.0625 0l0 1.390625l-8.0625 0zm15.04245 -5.078125q0 0.625 -0.28125 1.0625q-0.265625 0.4375 -0.71875 0.71875q-0.453125 0.28125 -1.03125 0.40625q-0.578125 0.125 -1.171875 0.125q-0.796875 0 -1.453125 -0.078125q-0.640625 -0.078125 -1.21875 -0.21875l0 -1.578125q0.6875 0.28125 1.34375 0.40625q0.671875 0.125 1.265625 0.125q0.6875 0 1.03125 -0.21875q0.34375 -0.21875 0.34375 -0.5625q0 -0.171875 -0.078125 -0.296875q-0.078125 -0.140625 -0.28125 -0.265625q-0.1875 -0.125 -0.546875 -0.25q-0.359375 -0.140625 -0.9375 -0.3125q-0.53125 -0.15625 -0.9375 -0.34375q-0.40625 -0.1875 -0.671875 -0.4375q-0.265625 -0.265625 -0.40625 -0.59375q-0.125 -0.34375 -0.125 -0.8125q0 -0.4375 0.203125 -0.828125q0.203125 -0.390625 0.59375 -0.6875q0.40625 -0.296875 1.0 -0.46875q0.59375 -0.171875 1.390625 -0.171875q0.6875 0 1.21875 0.078125q0.53125 0.0625 0.9375 0.140625l0 1.421875q-0.625 -0.1875 -1.171875 -0.265625q-0.546875 -0.09375 -1.09375 -0.09375q-0.53125 0 -0.859375 0.203125q-0.328125 0.1875 -0.328125 0.53125q0 0.15625 0.0625 0.296875q0.078125 0.125 0.265625 0.25q0.1875 0.109375 0.515625 0.25q0.34375 0.125 0.90625 0.28125q0.625 0.1875 1.046875 0.390625q0.4375 0.203125 0.703125 0.46875q0.265625 0.25 0.375 0.578125q0.109375 0.328125 0.109375 0.75zm8.058075 2.0625q-0.46875 0.109375 -0.953125 0.171875q-0.46875 0.078125 -0.90625 0.078125q-0.703125 0 -1.234375 -0.15625q-0.515625 -0.15625 -0.859375 -0.453125q-0.328125 -0.3125 -0.5 -0.78125q-0.15625 -0.484375 -0.15625 -1.140625l0 -3.546875l-1.953125 0l0 -1.359375l1.953125 0l0 -1.859375l1.796875 -0.46875l0 2.328125l2.8125 0l0 1.359375l-2.8125 0l0 3.421875q0 0.609375 0.28125 0.9375q0.28125 0.3125 0.953125 0.3125q0.4375 0 0.84375 -0.0625q0.40625 -0.078125 0.734375 -0.171875l0 1.390625zm6.4487 -0.71875q-0.390625 0.953125 -0.796875 1.65625q-0.40625 0.71875 -0.90625 1.1875q-0.5 0.484375 -1.109375 0.71875q-0.59375 0.234375 -1.34375 0.234375q-0.1875 0 -0.40625 -0.015625q-0.203125 -0.015625 -0.421875 -0.046875l0 -1.484375q0.09375 0.015625 0.203125 0.03125q0.109375 0.015625 0.21875 0.03125q0.125 0.015625 0.25 0.015625q0.125 0 0.234375 0q0.3125 0 0.5625 -0.109375q0.265625 -0.109375 0.484375 -0.3125q0.21875 -0.203125 0.390625 -0.484375q0.171875 -0.28125 0.3125 -0.625l-2.875 -7.265625l1.9375 0l1.515625 4.109375l0.453125 1.34375l0.4375 -1.28125l1.53125 -4.171875l1.875 0l-2.546875 6.46875zm6.026825 -7.953125l-2.0625 0l0 -1.359375l3.84375 0l0 8.75l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -7.390625zm12.04245 4.5625q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m466.66595 28.276524l0 0l0 17.16098l0 0l0 -17.16098z" fill-rule="nonzero"/><path fill="#ffffff" d="m176.57526 40.1147l290.0907 0l0 1.3199997l-290.0907 0l0 -1.3199997z" fill-rule="nonzero"/><path fill="#34a853" d="m163.16655 377.49115l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path stroke="#34a853" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m163.16655 377.49115l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path fill="#ffffff" d="m226.40776 393.77835l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm9.16745 -0.40625q0 0.953125 -0.265625 1.671875q-0.265625 0.71875 -0.75 1.1875q-0.46875 0.46875 -1.109375 0.703125q-0.640625 0.234375 -1.390625 0.234375q-0.34375 0 -0.6875 -0.03125q-0.34375 -0.03125 -0.6875 -0.125l0 3.015625l-1.25 0l0 -10.125l1.109375 0l0.078125 1.203125q0.546875 -0.734375 1.15625 -1.03125q0.609375 -0.296875 1.3125 -0.296875q0.609375 0 1.078125 0.265625q0.46875 0.25 0.78125 0.71875q0.3125 0.46875 0.46875 1.140625q0.15625 0.65625 0.15625 1.46875zm-1.265625 0.046875q0 -0.5625 -0.09375 -1.03125q-0.078125 -0.46875 -0.265625 -0.796875q-0.171875 -0.34375 -0.453125 -0.53125q-0.265625 -0.1875 -0.65625 -0.1875q-0.234375 0 -0.484375 0.078125q-0.234375 0.0625 -0.5 0.25q-0.265625 0.171875 -0.5625 0.46875q-0.28125 0.28125 -0.609375 0.703125l0 3.484375q0.34375 0.140625 0.71875 0.234375q0.390625 0.078125 0.75 0.078125q1.0 0 1.578125 -0.6875q0.578125 -0.6875 0.578125 -2.0625zm3.47995 -3.515625l1.140625 0l0.03125 1.328125q0.640625 -0.765625 1.25 -1.109375q0.625 -0.34375 1.25 -0.34375q1.125 0 1.703125 0.71875q0.578125 0.71875 0.53125 2.15625l-1.265625 0q0.03125 -0.953125 -0.265625 -1.375q-0.296875 -0.421875 -0.875 -0.421875q-0.25 0 -0.515625 0.09375q-0.25 0.078125 -0.515625 0.28125q-0.265625 0.1875 -0.5625 0.5q-0.296875 0.3125 -0.640625 0.75l0 4.609375l-1.265625 0l0 -7.1875zm14.026825 3.53125q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm8.198715 3.59375l0 -5.15625q0 -0.34375 -0.03125 -0.546875q-0.015625 -0.21875 -0.078125 -0.34375q-0.046875 -0.125 -0.125 -0.171875q-0.078125 -0.0625 -0.203125 -0.0625q-0.140625 0 -0.265625 0.09375q-0.125 0.078125 -0.265625 0.28125q-0.140625 0.1875 -0.3125 0.515625q-0.15625 0.3125 -0.390625 0.78125l0 4.609375l-1.1406403 0l0 -5.03125q0 -0.390625 -0.03125 -0.625q-0.015625 -0.25 -0.078125 -0.375q-0.046875 -0.140625 -0.140625 -0.1875q-0.078125 -0.0625 -0.203125 -0.0625q-0.125 0 -0.25 0.078125q-0.109375 0.0625 -0.25 0.25q-0.125 0.1875 -0.296875 0.515625q-0.171875 0.328125 -0.40625 0.828125l0 4.609375l-1.15625 0l0 -7.1875l0.953125 0l0.0625 1.375q0.1875 -0.40625 0.359375 -0.6875q0.171875 -0.296875 0.359375 -0.46875q0.1875 -0.1875 0.390625 -0.265625q0.21875 -0.078125 0.46875 -0.078125q0.59376526 0 0.89064026 0.375q0.296875 0.375 0.296875 1.171875q0.171875 -0.375 0.328125 -0.65625q0.171875 -0.296875 0.359375 -0.484375q0.1875 -0.203125 0.421875 -0.296875q0.234375 -0.109375 0.53125 -0.109375q1.34375 0 1.34375 2.078125l0 5.234375l-1.140625 0zm9.1987 -3.65625q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm8.870575 3.5q-0.421875 0.09375 -0.875 0.140625q-0.453125 0.046875 -0.921875 0.046875q-1.34375 0 -2.015625 -0.609375q-0.65625 -0.609375 -0.65625 -1.875l0 -3.75l-2.015625 0l0 -1.046875l2.015625 0l0 -1.96875l1.234375 -0.328125l0 2.296875l3.234375 0l0 1.046875l-3.234375 0l0 3.65625q0 0.765625 0.40625 1.15625q0.421875 0.375 1.21875 0.375q0.34375 0 0.75 -0.046875q0.40625 -0.0625 0.859375 -0.171875l0 1.078125zm8.339325 -3.875q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm8.308075 0.84375l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm9.16745 -0.40625q0 0.890625 -0.25 1.59375q-0.234375 0.703125 -0.703125 1.203125q-0.453125 0.484375 -1.109375 0.75q-0.640625 0.25 -1.453125 0.25q-0.640625 0 -1.3125 -0.109375q-0.65625 -0.125 -1.3125 -0.390625l0 -9.6875l1.25 0l0 2.78125l-0.0625 1.328125q0.546875 -0.71875 1.15625 -1.015625q0.609375 -0.296875 1.3125 -0.296875q0.609375 0 1.078125 0.265625q0.46875 0.25 0.78125 0.71875q0.3125 0.46875 0.46875 1.140625q0.15625 0.65625 0.15625 1.46875zm-1.265625 0.046875q0 -0.5625 -0.09375 -1.03125q-0.078125 -0.46875 -0.265625 -0.796875q-0.171875 -0.34375 -0.453125 -0.53125q-0.265625 -0.1875 -0.65625 -0.1875q-0.234375 0 -0.484375 0.078125q-0.234375 0.0625 -0.5 0.25q-0.265625 0.171875 -0.5625 0.46875q-0.28125 0.28125 -0.609375 0.703125l0 3.484375q0.359375 0.140625 0.734375 0.234375q0.390625 0.078125 0.734375 0.078125q0.421875 0 0.8125 -0.125q0.390625 -0.140625 0.6875 -0.453125q0.296875 -0.328125 0.46875 -0.859375q0.1875 -0.53125 0.1875 -1.3125zm8.964325 3.671875l-1.125 0l-0.03125 -1.15625q-0.328125 0.375 -0.625 0.625q-0.28125 0.234375 -0.578125 0.390625q-0.28125 0.140625 -0.578125 0.203125q-0.28125 0.0625 -0.59375 0.0625q-1.109375 0 -1.6875 -0.640625q-0.5625 -0.65625 -0.5625 -1.96875l0 -4.703125l1.25 0l0 4.59375q0 1.65625 1.25 1.65625q0.21875 0 0.4375 -0.0625q0.21875 -0.078125 0.453125 -0.234375q0.25 -0.171875 0.515625 -0.453125q0.28125 -0.296875 0.625 -0.734375l0 -4.765625l1.25 0l0 7.1875zm8.651825 -8.953125q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.5737 -1.765625q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.276825 3.21875q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm3.5737 -3.03125l1.140625 0l0.03125 1.328125q0.640625 -0.765625 1.25 -1.109375q0.625 -0.34375 1.25 -0.34375q1.125 0 1.703125 0.71875q0.578125 0.71875 0.53125 2.15625l-1.265625 0q0.03125 -0.953125 -0.265625 -1.375q-0.296875 -0.421875 -0.875 -0.421875q-0.25 0 -0.515625 0.09375q-0.25 0.078125 -0.515625 0.28125q-0.265625 0.1875 -0.5625 0.5q-0.296875 0.3125 -0.640625 0.75l0 4.609375l-1.265625 0l0 -7.1875zm13.4487 5.234375q0 0.375 -0.125 0.671875q-0.125 0.296875 -0.34375 0.53125q-0.21875 0.234375 -0.515625 0.40625q-0.296875 0.15625 -0.640625 0.265625q-0.328125 0.109375 -0.6875 0.15625q-0.34375 0.046875 -0.671875 0.046875q-0.734375 0 -1.34375 -0.0625q-0.609375 -0.0625 -1.203125 -0.203125l0 -1.140625q0.640625 0.171875 1.25 0.265625q0.625 0.09375 1.25 0.09375q0.890625 0 1.3125 -0.234375q0.4375 -0.25 0.4375 -0.703125q0 -0.1875 -0.078125 -0.34375q-0.0625 -0.15625 -0.25 -0.296875q-0.171875 -0.140625 -0.546875 -0.28125q-0.375 -0.15625 -1.015625 -0.359375q-0.5 -0.140625 -0.90625 -0.3125q-0.40625 -0.1875 -0.71875 -0.4375q-0.296875 -0.25 -0.46875 -0.578125q-0.171875 -0.34375 -0.171875 -0.8125q0 -0.296875 0.140625 -0.65625q0.140625 -0.359375 0.46875 -0.65625q0.34375 -0.3125 0.921875 -0.515625q0.578125 -0.203125 1.421875 -0.203125q0.421875 0 0.9375 0.046875q0.515625 0.046875 1.078125 0.15625l0 1.109375q-0.578125 -0.140625 -1.109375 -0.203125q-0.53125 -0.078125 -0.90625 -0.078125q-0.46875 0 -0.796875 0.078125q-0.3125 0.0625 -0.515625 0.203125q-0.1875 0.125 -0.28125 0.296875q-0.078125 0.15625 -0.078125 0.34375q0 0.203125 0.0625 0.359375q0.078125 0.15625 0.28125 0.3125q0.21875 0.140625 0.578125 0.28125q0.359375 0.140625 0.953125 0.3125q0.640625 0.1875 1.0625 0.390625q0.4375 0.203125 0.703125 0.46875q0.28125 0.25 0.390625 0.5625q0.125 0.3125 0.125 0.71875zm7.401825 -1.359375l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm8.8237 3.21875q-0.421875 0.09375 -0.875 0.140625q-0.453125 0.046875 -0.921875 0.046875q-1.34375 0 -2.015625 -0.609375q-0.65625 -0.609375 -0.65625 -1.875l0 -3.75l-2.015625 0l0 -1.046875l2.015625 0l0 -1.96875l1.234375 -0.328125l0 2.296875l3.234375 0l0 1.046875l-3.234375 0l0 3.65625q0 0.765625 0.40625 1.15625q0.421875 0.375 1.21875 0.375q0.34375 0 0.75 -0.046875q0.40625 -0.0625 0.859375 -0.171875l0 1.078125zm8.526825 -3.5625q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm8.10495 0.28125l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm8.714325 1.359375q0 0.375 -0.125 0.671875q-0.125 0.296875 -0.34375 0.53125q-0.21875 0.234375 -0.515625 0.40625q-0.296875 0.15625 -0.640625 0.265625q-0.328125 0.109375 -0.6875 0.15625q-0.34375 0.046875 -0.671875 0.046875q-0.734375 0 -1.34375 -0.0625q-0.609375 -0.0625 -1.203125 -0.203125l0 -1.140625q0.640625 0.171875 1.25 0.265625q0.625 0.09375 1.25 0.09375q0.890625 0 1.3125 -0.234375q0.4375 -0.25 0.4375 -0.703125q0 -0.1875 -0.078125 -0.34375q-0.0625 -0.15625 -0.25 -0.296875q-0.171875 -0.140625 -0.546875 -0.28125q-0.375 -0.15625 -1.015625 -0.359375q-0.5 -0.140625 -0.90625 -0.3125q-0.40625 -0.1875 -0.71875 -0.4375q-0.296875 -0.25 -0.46875 -0.578125q-0.171875 -0.34375 -0.171875 -0.8125q0 -0.296875 0.140625 -0.65625q0.140625 -0.359375 0.46875 -0.65625q0.34375 -0.3125 0.921875 -0.515625q0.578125 -0.203125 1.421875 -0.203125q0.421875 0 0.9375 0.046875q0.515625 0.046875 1.078125 0.15625l0 1.109375q-0.578125 -0.140625 -1.109375 -0.203125q-0.53125 -0.078125 -0.90625 -0.078125q-0.46875 0 -0.796875 0.078125q-0.3125 0.0625 -0.515625 0.203125q-0.1875 0.125 -0.28125 0.296875q-0.078125 0.15625 -0.078125 0.34375q0 0.203125 0.0625 0.359375q0.078125 0.15625 0.28125 0.3125q0.21875 0.140625 0.578125 0.28125q0.359375 0.140625 0.953125 0.3125q0.640625 0.1875 1.0625 0.390625q0.4375 0.203125 0.703125 0.46875q0.28125 0.25 0.390625 0.5625q0.125 0.3125 0.125 0.71875zm8.16745 1.859375q-0.421875 0.09375 -0.875 0.140625q-0.453125 0.046875 -0.921875 0.046875q-1.34375 0 -2.015625 -0.609375q-0.65625 -0.609375 -0.65625 -1.875l0 -3.75l-2.015625 0l0 -1.046875l2.015625 0l0 -1.96875l1.234375 -0.328125l0 2.296875l3.234375 0l0 1.046875l-3.234375 0l0 3.65625q0 0.765625 0.40625 1.15625q0.421875 0.375 1.21875 0.375q0.34375 0 0.75 -0.046875q0.40625 -0.0625 0.859375 -0.171875l0 1.078125zm6.8862 0.09375l-0.03125 -0.96875q-0.59375 0.578125 -1.203125 0.84375q-0.59375 0.25 -1.265625 0.25q-0.609375 0 -1.046875 -0.15625q-0.4375 -0.15625 -0.71875 -0.421875q-0.28125 -0.28125 -0.421875 -0.65625q-0.125 -0.375 -0.125 -0.8125q0 -1.078125 0.796875 -1.6875q0.8125 -0.609375 2.390625 -0.609375l1.484375 0l0 -0.640625q0 -0.625 -0.40625 -1.0q-0.40625 -0.390625 -1.25 -0.390625q-0.609375 0 -1.203125 0.140625q-0.578125 0.125 -1.21875 0.375l0 -1.125q0.25 -0.078125 0.53125 -0.15625q0.296875 -0.09375 0.609375 -0.15625q0.328125 -0.0625 0.671875 -0.09375q0.359375 -0.046875 0.71875 -0.046875q0.640625 0 1.15625 0.140625q0.53125 0.140625 0.890625 0.4375q0.359375 0.296875 0.546875 0.75q0.203125 0.4375 0.203125 1.03125l0 4.953125l-1.109375 0zm-0.140625 -3.265625l-1.578125 0q-0.46875 0 -0.8125 0.09375q-0.328125 0.09375 -0.546875 0.265625q-0.21875 0.171875 -0.328125 0.40625q-0.09375 0.234375 -0.09375 0.546875q0 0.203125 0.0625 0.390625q0.0625 0.1875 0.203125 0.34375q0.15625 0.140625 0.375 0.234375q0.234375 0.078125 0.5625 0.078125q0.4375 0 0.984375 -0.265625q0.5625 -0.265625 1.171875 -0.828125l0 -1.265625zm9.16745 3.0q-0.5 0.1875 -1.015625 0.265625q-0.5 0.09375 -1.046875 0.09375q-1.703125 0 -2.625 -0.921875q-0.921875 -0.921875 -0.921875 -2.6875q0 -0.859375 0.265625 -1.546875q0.265625 -0.703125 0.75 -1.1875q0.484375 -0.5 1.140625 -0.765625q0.671875 -0.265625 1.46875 -0.265625q0.5625 0 1.046875 0.078125q0.484375 0.078125 0.9375 0.25l0 1.1875q-0.46875 -0.234375 -0.953125 -0.34375q-0.484375 -0.109375 -1.0 -0.109375q-0.484375 0 -0.90625 0.1875q-0.421875 0.171875 -0.75 0.515625q-0.3125 0.34375 -0.5 0.84375q-0.1875 0.484375 -0.1875 1.109375q0 1.296875 0.625 1.953125q0.640625 0.640625 1.765625 0.640625q0.5 0 0.96875 -0.109375q0.484375 -0.109375 0.9375 -0.34375l0 1.15625zm8.933075 0.265625l-1.71875 0l-3.34375 -3.84375l0 3.84375l-1.25 0l0 -10.109375l1.25 0l0 6.203125l3.234375 -3.28125l1.640625 0l-3.375 3.3125l3.5625 3.875z" fill-rule="nonzero"/><path fill="#fbbc05" d="m163.16655 431.26l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path stroke="#fbbc05" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m163.16655 431.26l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m204.11974 439.97485l233.68419 0l0 17.16098l-233.68419 0l0 -17.16098z" fill-rule="nonzero"/><path fill="#000000" d="m205.91661 447.7347l0 -1.546875l4.453125 0l0 1.546875l-4.453125 0zm13.79245 -0.640625q0 1.0 -0.28125 1.734375q-0.28125 0.734375 -0.78125 1.21875q-0.5 0.46875 -1.1875 0.703125q-0.6875 0.234375 -1.5 0.234375q-0.796875 0 -1.484375 -0.125q-0.671875 -0.109375 -1.296875 -0.3125l0 -9.796875l1.734375 0l0 2.375l-0.0625 1.421875q0.390625 -0.5 0.90625 -0.796875q0.53125 -0.3125 1.28125 -0.3125q0.640625 0 1.140625 0.265625q0.5 0.25 0.828125 0.71875q0.34375 0.46875 0.515625 1.15625q0.1875 0.671875 0.1875 1.515625zm-1.828125 0.078125q0 -0.609375 -0.09375 -1.03125q-0.078125 -0.421875 -0.25 -0.6875q-0.15625 -0.28125 -0.390625 -0.40625q-0.21875 -0.125 -0.53125 -0.125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 3.171875q0.21875 0.078125 0.515625 0.125q0.3125 0.046875 0.625 0.046875q0.40625 0 0.75 -0.171875q0.34375 -0.171875 0.578125 -0.484375q0.25 -0.3125 0.375 -0.765625q0.125 -0.453125 0.125 -1.015625zm5.089325 -3.578125l0 4.734375q0 0.59375 0.21875 0.90625q0.21875 0.296875 0.671875 0.296875q0.4375 0 0.828125 -0.359375q0.40625 -0.359375 0.859375 -0.984375l0 -4.59375l1.75 0l0 7.265625l-1.515625 0l-0.03125 -1.078125q-0.234375 0.28125 -0.46875 0.515625q-0.234375 0.21875 -0.515625 0.390625q-0.265625 0.15625 -0.59375 0.25q-0.328125 0.09375 -0.71875 0.09375q-0.5625 0 -0.984375 -0.1875q-0.421875 -0.1875 -0.703125 -0.515625q-0.265625 -0.34375 -0.40625 -0.8125q-0.140625 -0.46875 -0.140625 -1.046875l0 -4.875l1.75 0zm12.839325 -1.359375q-0.3125 -0.109375 -0.75 -0.171875q-0.421875 -0.0625 -0.875 -0.0625q-0.3125 0 -0.578125 0.078125q-0.265625 0.078125 -0.453125 0.265625q-0.1875 0.171875 -0.3125 0.453125q-0.109375 0.265625 -0.109375 0.65625l0 1.1875l2.859375 0l0 1.359375l-2.859375 0l0 4.859375l-1.75 0l0 -4.859375l-2.109375 0l0 -1.359375l2.109375 0l0 -1.109375q0 -0.765625 0.21875 -1.3125q0.234375 -0.5625 0.640625 -0.921875q0.40625 -0.359375 0.984375 -0.515625q0.578125 -0.171875 1.296875 -0.171875q0.484375 0 0.90625 0.0625q0.421875 0.0625 0.78125 0.140625l0 1.421875zm-7.515625 1.359375l0 0zm15.5737 -1.359375q-0.3125 -0.109375 -0.75 -0.171875q-0.421875 -0.0625 -0.875 -0.0625q-0.3125 0 -0.578125 0.078125q-0.265625 0.078125 -0.453125 0.265625q-0.1875 0.171875 -0.3125 0.453125q-0.109375 0.265625 -0.109375 0.65625l0 1.1875l2.859375 0l0 1.359375l-2.859375 0l0 4.859375l-1.75 0l0 -4.859375l-2.109375 0l0 -1.359375l2.109375 0l0 -1.109375q0 -0.765625 0.21875 -1.3125q0.234375 -0.5625 0.640625 -0.921875q0.40625 -0.359375 0.984375 -0.515625q0.578125 -0.171875 1.296875 -0.171875q0.484375 0 0.90625 0.0625q0.421875 0.0625 0.78125 0.140625l0 1.421875zm-7.515625 1.359375l0 0zm15.3862 3.078125q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm8.18309 0q0.015625 -0.421875 -0.046875 -0.703125q-0.046875 -0.28125 -0.171875 -0.46875q-0.109375 -0.1875 -0.28125 -0.265625q-0.171875 -0.09375 -0.390625 -0.09375q-0.390625 0 -0.796875 0.328125q-0.40625 0.3125 -0.90626526 1.046875l0 4.59375l-1.796875 0l0 -7.265625l1.59375 0l0.0625 1.046875q0.171875 -0.265625 0.390625 -0.484375q0.23439026 -0.234375 0.50001526 -0.390625q0.28125 -0.15625 0.609375 -0.234375q0.34375 -0.09375 0.75 -0.09375q0.546875 0 0.96875 0.1875q0.4375 0.1875 0.71875 0.5625q0.296875 0.359375 0.4375 0.921875q0.140625 0.5625 0.109375 1.3125l-1.75 0zm4.183075 1.3125l0 -1.546875l4.453125 0l0 1.546875l-4.453125 0zm11.8237 3.125l-0.03125 -1.078125q-0.234375 0.28125 -0.484375 0.515625q-0.234375 0.21875 -0.515625 0.390625q-0.28125 0.15625 -0.609375 0.25q-0.3125 0.09375 -0.6875 0.09375q-0.65625 0 -1.15625 -0.25q-0.5 -0.265625 -0.84375 -0.734375q-0.328125 -0.484375 -0.515625 -1.15625q-0.171875 -0.6875 -0.171875 -1.53125q0 -1.0 0.28125 -1.734375q0.28125 -0.734375 0.78125 -1.21875q0.5 -0.484375 1.1875 -0.71875q0.6875 -0.234375 1.5 -0.234375q0.265625 0 0.53125 0.046875q0.28125 0.03125 0.5 0.09375l0 -2.84375l1.75 0l0 10.109375l-1.515625 0zm-3.1875 -3.578125q0 0.59375 0.078125 1.03125q0.09375 0.421875 0.25 0.703125q0.15625 0.265625 0.390625 0.390625q0.234375 0.125 0.546875 0.125q0.4375 0 0.828125 -0.359375q0.40625 -0.359375 0.859375 -0.984375l0 -3.171875q-0.203125 -0.078125 -0.515625 -0.125q-0.296875 -0.0625 -0.609375 -0.0625q-0.421875 0 -0.765625 0.171875q-0.328125 0.171875 -0.5625 0.5q-0.234375 0.3125 -0.375 0.765625q-0.125 0.453125 -0.125 1.015625zm13.0112 -0.609375q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm8.058075 4.4375l-0.03125 -0.953125q-0.234375 0.25 -0.484375 0.453125q-0.25 0.203125 -0.5625 0.359375q-0.296875 0.140625 -0.65625 0.21875q-0.34375 0.09375 -0.765625 0.09375q-0.5625 0 -0.984375 -0.171875q-0.421875 -0.171875 -0.703125 -0.453125q-0.28125 -0.296875 -0.4375 -0.703125q-0.140625 -0.421875 -0.140625 -0.921875q0 -0.515625 0.21875 -0.953125q0.21875 -0.4375 0.65625 -0.75q0.453125 -0.3125 1.125 -0.484375q0.671875 -0.1875 1.5625 -0.1875l0.953125 0l0 -0.4375q0 -0.28125 -0.078125 -0.5q-0.078125 -0.234375 -0.25 -0.390625q-0.171875 -0.15625 -0.453125 -0.234375q-0.28125 -0.09375 -0.6875 -0.09375q-0.640625 0 -1.265625 0.15625q-0.625 0.140625 -1.21875 0.40625l0 -1.40625q0.53125 -0.203125 1.203125 -0.328125q0.6875 -0.140625 1.421875 -0.140625q0.8125 0 1.390625 0.15625q0.578125 0.140625 0.953125 0.453125q0.375 0.3125 0.546875 0.78125q0.1875 0.453125 0.1875 1.0625l0 4.96875l-1.5 0zm-0.25 -3.21875l-1.0625 0q-0.4375 0 -0.75 0.09375q-0.3125 0.078125 -0.5 0.234375q-0.1875 0.15625 -0.28125 0.359375q-0.09375 0.1875 -0.09375 0.40625q0 0.4375 0.28125 0.671875q0.296875 0.234375 0.78125 0.234375q0.375 0 0.765625 -0.265625q0.390625 -0.265625 0.859375 -0.75l0 -0.984375zm6.10495 -5.53125l-2.0625 0l0 -1.359375l3.84375 0l0 8.75l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -7.390625zm8.058075 0l-2.0625 0l0 -1.359375l3.84375 0l0 8.75l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -7.390625zm12.2612 5.0625q0 0.859375 -0.25 1.578125q-0.234375 0.703125 -0.703125 1.21875q-0.453125 0.5 -1.125 0.78125q-0.65625 0.28125 -1.515625 0.28125q-0.796875 0 -1.4375 -0.234375q-0.640625 -0.25 -1.09375 -0.71875q-0.4375 -0.46875 -0.671875 -1.171875q-0.234375 -0.703125 -0.234375 -1.640625q0 -0.859375 0.25 -1.5625q0.25 -0.71875 0.703125 -1.21875q0.46875 -0.5 1.125 -0.765625q0.671875 -0.28125 1.5 -0.28125q0.828125 0 1.453125 0.234375q0.640625 0.234375 1.078125 0.71875q0.453125 0.46875 0.6875 1.171875q0.234375 0.6875 0.234375 1.609375zm-1.828125 0.03125q0 -1.125 -0.421875 -1.6875q-0.421875 -0.578125 -1.25 -0.578125q-0.46875 0 -0.796875 0.1875q-0.3125 0.171875 -0.515625 0.484375q-0.203125 0.3125 -0.3125 0.734375q-0.09375 0.40625 -0.09375 0.875q0 1.140625 0.453125 1.71875q0.46875 0.578125 1.265625 0.578125q0.4375 0 0.75 -0.171875q0.328125 -0.1875 0.53125 -0.484375q0.203125 -0.3125 0.296875 -0.734375q0.09375 -0.421875 0.09375 -0.921875zm9.214325 3.375q-0.515625 0.203125 -1.0625 0.296875q-0.546875 0.109375 -1.15625 0.109375q-0.859375 0 -1.546875 -0.234375q-0.6875 -0.234375 -1.171875 -0.703125q-0.484375 -0.46875 -0.734375 -1.140625q-0.25 -0.6875 -0.25 -1.59375q0 -0.875 0.265625 -1.578125q0.28125 -0.71875 0.78125 -1.21875q0.5 -0.5 1.203125 -0.765625q0.71875 -0.28125 1.578125 -0.28125q0.703125 0 1.1875 0.078125q0.484375 0.078125 0.875 0.1875l0 1.703125q-0.453125 -0.21875 -0.96875 -0.34375q-0.515625 -0.125 -0.984375 -0.125q-0.484375 0 -0.875 0.171875q-0.375 0.15625 -0.640625 0.453125q-0.265625 0.296875 -0.40625 0.71875q-0.140625 0.40625 -0.140625 0.90625q0 0.53125 0.140625 0.953125q0.15625 0.40625 0.421875 0.703125q0.28125 0.28125 0.671875 0.4375q0.390625 0.15625 0.859375 0.15625q0.234375 0 0.484375 -0.03125q0.25 -0.03125 0.5 -0.09375q0.265625 -0.0625 0.5 -0.140625q0.25 -0.078125 0.46875 -0.171875l0 1.546875zm6.72995 0.28125l-0.03125 -0.953125q-0.234375 0.25 -0.484375 0.453125q-0.25 0.203125 -0.5625 0.359375q-0.296875 0.140625 -0.65625 0.21875q-0.34375 0.09375 -0.765625 0.09375q-0.5625 0 -0.984375 -0.171875q-0.421875 -0.171875 -0.703125 -0.453125q-0.28125 -0.296875 -0.4375 -0.703125q-0.140625 -0.421875 -0.140625 -0.921875q0 -0.515625 0.21875 -0.953125q0.21875 -0.4375 0.65625 -0.75q0.453125 -0.3125 1.125 -0.484375q0.671875 -0.1875 1.5625 -0.1875l0.953125 0l0 -0.4375q0 -0.28125 -0.078125 -0.5q-0.078125 -0.234375 -0.25 -0.390625q-0.171875 -0.15625 -0.453125 -0.234375q-0.28125 -0.09375 -0.6875 -0.09375q-0.640625 0 -1.265625 0.15625q-0.625 0.140625 -1.21875 0.40625l0 -1.40625q0.53125 -0.203125 1.203125 -0.328125q0.6875 -0.140625 1.421875 -0.140625q0.8125 0 1.390625 0.15625q0.578125 0.140625 0.953125 0.453125q0.375 0.3125 0.546875 0.78125q0.1875 0.453125 0.1875 1.0625l0 4.96875l-1.5 0zm-0.25 -3.21875l-1.0625 0q-0.4375 0 -0.75 0.09375q-0.3125 0.078125 -0.5 0.234375q-0.1875 0.15625 -0.28125 0.359375q-0.09375 0.1875 -0.09375 0.40625q0 0.4375 0.28125 0.671875q0.296875 0.234375 0.78125 0.234375q0.375 0 0.765625 -0.265625q0.390625 -0.265625 0.859375 -0.75l0 -0.984375zm9.745575 3.140625q-0.46875 0.109375 -0.953125 0.171875q-0.46875 0.078125 -0.90625 0.078125q-0.703125 0 -1.234375 -0.15625q-0.515625 -0.15625 -0.859375 -0.453125q-0.328125 -0.3125 -0.5 -0.78125q-0.15625 -0.484375 -0.15625 -1.140625l0 -3.546875l-1.953125 0l0 -1.359375l1.953125 0l0 -1.859375l1.796875 -0.46875l0 2.328125l2.8125 0l0 1.359375l-2.8125 0l0 3.421875q0 0.609375 0.28125 0.9375q0.28125 0.3125 0.953125 0.3125q0.4375 0 0.84375 -0.0625q0.40625 -0.078125 0.734375 -0.171875l0 1.390625zm6.35495 -9.1875q0 0.234375 -0.09375 0.453125q-0.078125 0.203125 -0.234375 0.359375q-0.15625 0.15625 -0.375 0.25q-0.203125 0.078125 -0.4375 0.078125q-0.25 0 -0.46875 -0.078125q-0.203125 -0.09375 -0.359375 -0.25q-0.15625 -0.15625 -0.25 -0.359375q-0.078125 -0.21875 -0.078125 -0.453125q0 -0.234375 0.078125 -0.4375q0.09375 -0.203125 0.25 -0.359375q0.15625 -0.15625 0.359375 -0.25q0.21875 -0.09375 0.46875 -0.09375q0.234375 0 0.4375 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.234375 0.359375q0.09375 0.203125 0.09375 0.4375zm-1.9375 3.359375l-2.0625 0l0 -1.359375l3.84375 0l0 5.90625l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -4.546875zm12.2612 2.21875q0 0.859375 -0.25 1.578125q-0.234375 0.703125 -0.703125 1.21875q-0.453125 0.5 -1.125 0.78125q-0.65625 0.28125 -1.515625 0.28125q-0.796875 0 -1.4375 -0.234375q-0.640625 -0.25 -1.09375 -0.71875q-0.4375 -0.46875 -0.671875 -1.171875q-0.234375 -0.703125 -0.234375 -1.640625q0 -0.859375 0.25 -1.5625q0.25 -0.71875 0.703125 -1.21875q0.46875 -0.5 1.125 -0.765625q0.671875 -0.28125 1.5 -0.28125q0.828125 0 1.453125 0.234375q0.640625 0.234375 1.078125 0.71875q0.453125 0.46875 0.6875 1.171875q0.234375 0.6875 0.234375 1.609375zm-1.828125 0.03125q0 -1.125 -0.421875 -1.6875q-0.421875 -0.578125 -1.25 -0.578125q-0.46875 0 -0.796875 0.1875q-0.3125 0.171875 -0.515625 0.484375q-0.203125 0.3125 -0.3125 0.734375q-0.09375 0.40625 -0.09375 0.875q0 1.140625 0.453125 1.71875q0.46875 0.578125 1.265625 0.578125q0.4375 0 0.75 -0.171875q0.328125 -0.1875 0.53125 -0.484375q0.203125 -0.3125 0.296875 -0.734375q0.09375 -0.421875 0.09375 -0.921875zm7.66745 3.65625l0 -4.734375q0 -1.203125 -0.890625 -1.203125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 4.59375l-1.734375 0l0 -7.265625l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.546875 0 0.953125 0.1875q0.421875 0.171875 0.703125 0.515625q0.28125 0.328125 0.421875 0.796875q0.140625 0.46875 0.140625 1.046875l0 4.875l-1.75 0zm4.526825 -3.125l0 -1.546875l4.453125 0l0 1.546875l-4.453125 0zm13.79245 -0.640625q0 1.0 -0.28125 1.734375q-0.28125 0.734375 -0.78125 1.21875q-0.5 0.46875 -1.1875 0.703125q-0.6875 0.234375 -1.5 0.234375q-0.28125 0 -0.546875 -0.03125q-0.265625 -0.03125 -0.5 -0.09375l0 2.84375l-1.734375 0l0 -10.109375l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.640625 0 1.140625 0.265625q0.5 0.25 0.828125 0.71875q0.34375 0.46875 0.515625 1.15625q0.1875 0.671875 0.1875 1.515625zm-1.828125 0.078125q0 -0.609375 -0.09375 -1.03125q-0.078125 -0.421875 -0.25 -0.6875q-0.15625 -0.28125 -0.390625 -0.40625q-0.21875 -0.125 -0.53125 -0.125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 3.171875q0.21875 0.078125 0.515625 0.125q0.3125 0.046875 0.625 0.046875q0.40625 0 0.75 -0.171875q0.34375 -0.171875 0.578125 -0.484375q0.25 -0.3125 0.375 -0.765625q0.125 -0.453125 0.125 -1.015625zm7.6362 -5.578125q0 0.234375 -0.09375 0.453125q-0.078125 0.203125 -0.234375 0.359375q-0.15625 0.15625 -0.375 0.25q-0.203125 0.078125 -0.4375 0.078125q-0.25 0 -0.46875 -0.078125q-0.203125 -0.09375 -0.359375 -0.25q-0.15625 -0.15625 -0.25 -0.359375q-0.078125 -0.21875 -0.078125 -0.453125q0 -0.234375 0.078125 -0.4375q0.09375 -0.203125 0.25 -0.359375q0.15625 -0.15625 0.359375 -0.25q0.21875 -0.09375 0.46875 -0.09375q0.234375 0 0.4375 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.234375 0.359375q0.09375 0.203125 0.09375 0.4375zm-1.9375 3.359375l-2.0625 0l0 -1.359375l3.84375 0l0 5.90625l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -4.546875zm12.245575 2.140625q0 1.0 -0.28125 1.734375q-0.28125 0.734375 -0.78125 1.21875q-0.5 0.46875 -1.1875 0.703125q-0.6875 0.234375 -1.5 0.234375q-0.28125 0 -0.546875 -0.03125q-0.265625 -0.03125 -0.5 -0.09375l0 2.84375l-1.734375 0l0 -10.109375l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.640625 0 1.140625 0.265625q0.5 0.25 0.828125 0.71875q0.34375 0.46875 0.515625 1.15625q0.1875 0.671875 0.1875 1.515625zm-1.828125 0.078125q0 -0.609375 -0.09375 -1.03125q-0.078125 -0.421875 -0.25 -0.6875q-0.15625 -0.28125 -0.390625 -0.40625q-0.21875 -0.125 -0.53125 -0.125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 3.171875q0.21875 0.078125 0.515625 0.125q0.3125 0.046875 0.625 0.046875q0.40625 0 0.75 -0.171875q0.34375 -0.171875 0.578125 -0.484375q0.25 -0.3125 0.375 -0.765625q0.125 -0.453125 0.125 -1.015625zm9.683075 -0.5q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm5.85495 -4.3125l-2.0625 0l0 -1.359375l3.84375 0l0 8.75l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -7.390625zm9.995575 -0.515625q0 0.234375 -0.09375 0.453125q-0.078125 0.203125 -0.234375 0.359375q-0.15625 0.15625 -0.375 0.25q-0.203125 0.078125 -0.4375 0.078125q-0.25 0 -0.46875 -0.078125q-0.203125 -0.09375 -0.359375 -0.25q-0.15625 -0.15625 -0.25 -0.359375q-0.078125 -0.21875 -0.078125 -0.453125q0 -0.234375 0.078125 -0.4375q0.09375 -0.203125 0.25 -0.359375q0.15625 -0.15625 0.359375 -0.25q0.21875 -0.09375 0.46875 -0.09375q0.234375 0 0.4375 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.234375 0.359375q0.09375 0.203125 0.09375 0.4375zm-1.9375 3.359375l-2.0625 0l0 -1.359375l3.84375 0l0 5.90625l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -4.546875zm10.04245 5.90625l0 -4.734375q0 -1.203125 -0.890625 -1.203125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 4.59375l-1.734375 0l0 -7.265625l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.546875 0 0.953125 0.1875q0.421875 0.171875 0.703125 0.515625q0.28125 0.328125 0.421875 0.796875q0.140625 0.46875 0.140625 1.046875l0 4.875l-1.75 0zm10.058075 -4.1875q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m437.80392 439.97485l0 0l0 17.16098l0 0l0 -17.16098z" fill-rule="nonzero"/><path fill="#000000" d="m204.11974 451.81302l233.68419 0l0 1.3200073l-233.68419 0l0 -1.3200073z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m0 34.706036l161.41733 0l0 47.968502l-161.41733 0z" fill-rule="evenodd"/><path fill="#000000" d="m22.033333 61.626034l0 -13.359375l4.484375 0q1.125 0 2.078125 0.515625q0.953125 0.5 1.515625 1.390625q0.578125 0.875 0.578125 2.046875q0 1.15625 -0.578125 2.0625q-0.5625 0.890625 -1.515625 1.390625q-0.953125 0.5 -2.078125 0.5l-3.65625 0l0 -1.5l3.703125 0q0.78125 0 1.328125 -0.359375q0.5625 -0.375 0.875 -0.9375q0.3125 -0.5625 0.3125 -1.15625q0 -0.59375 -0.3125 -1.15625q-0.3125 -0.5625 -0.875 -0.921875q-0.546875 -0.359375 -1.328125 -0.359375l-2.953125 0l0 11.84375l-1.578125 0zm10.417877 0l0 -9.515625l1.515625 0l0 1.53125l0.078125 0q0.1875 -0.546875 0.625 -0.9375q0.4375 -0.40625 1.0 -0.640625q0.578125 -0.234375 1.125 -0.234375q0.4375 0 0.671875 0.046875q0.25 0.046875 0.453125 0.140625l0 1.71875q-0.296875 -0.15625 -0.640625 -0.21875q-0.34375 -0.078125 -0.703125 -0.078125q-0.6875 0 -1.265625 0.390625q-0.578125 0.390625 -0.921875 1.046875q-0.34375 0.65625 -0.34375 1.4375l0 5.3125l-1.59375 0zm10.670471 0.296875q-1.375 0 -2.453125 -0.640625q-1.0625 -0.65625 -1.671875 -1.796875q-0.609375 -1.140625 -0.609375 -2.59375q0 -1.359375 0.5625 -2.515625q0.578125 -1.15625 1.609375 -1.859375q1.03125 -0.703125 2.4375 -0.703125q1.421875 0 2.4375 0.625q1.015625 0.625 1.5625 1.734375q0.546875 1.09375 0.546875 2.515625q0 0.125 -0.015625 0.265625q0 0.125 -0.015625 0.21875l-8.1875 0l0 -1.3125l6.546875 0q-0.015625 -0.390625 -0.1875 -0.84375q-0.15625 -0.46875 -0.5 -0.859375q-0.34375 -0.40625 -0.875 -0.65625q-0.53125 -0.25 -1.3125 -0.25q-0.9375 0 -1.625 0.484375q-0.671875 0.46875 -1.046875 1.296875q-0.359375 0.8125 -0.359375 1.859375q0 1.203125 0.46875 2.015625q0.46875 0.796875 1.203125 1.1875q0.75 0.390625 1.546875 0.390625q1.046875 0 1.71875 -0.484375q0.6875 -0.5 1.09375 -1.234375l1.34375 0.65625q-0.5625 1.078125 -1.609375 1.796875q-1.03125 0.703125 -2.609375 0.703125zm6.2690887 3.7343712l0 -13.546871l1.515625 0l0 1.421875l0.078125 0q0.25 -0.453125 0.71875 -0.84375q0.484375 -0.390625 1.125 -0.625q0.640625 -0.25 1.390625 -0.25q1.3125 0 2.328125 0.65625q1.03125 0.65625 1.625 1.796875q0.609375 1.125 0.609375 2.609375q0 1.46875 -0.609375 2.609375q-0.59375 1.140625 -1.625 1.796875q-1.015625 0.640625 -2.328125 0.640625q-1.125 0 -1.984375 -0.515625q-0.859375 -0.53125 -1.25 -1.1875l-0.078125 0l0.078125 1.3125l0 4.124996l-1.59375 0zm4.671875 -5.171871q0.8125 0 1.53125 -0.4375q0.71875 -0.4375 1.15625 -1.25q0.4375 -0.828125 0.4375 -1.921875q0 -1.125 -0.4375 -1.9375q-0.4375 -0.8125 -1.15625 -1.25q-0.71875 -0.4375 -1.53125 -0.4375q-0.828125 0 -1.546875 0.4375q-0.71875 0.4375 -1.171875 1.25q-0.4375 0.8125 -0.4375 1.9375q0 1.109375 0.4375 1.921875q0.453125 0.8125 1.171875 1.25q0.71875 0.4375 1.546875 0.4375zm6.465851 1.140625l0 -9.515625l1.515625 0l0 1.53125l0.078125 0q0.1875 -0.546875 0.625 -0.9375q0.4375 -0.40625 1.0 -0.640625q0.578125 -0.234375 1.125 -0.234375q0.4375 0 0.671875 0.046875q0.25 0.046875 0.453125 0.140625l0 1.71875q-0.296875 -0.15625 -0.640625 -0.21875q-0.34375 -0.078125 -0.703125 -0.078125q-0.6875 0 -1.265625 0.390625q-0.578125 0.390625 -0.921875 1.046875q-0.34375 0.65625 -0.34375 1.4375l0 5.3125l-1.59375 0zm10.826721 0.296875q-1.4375 0 -2.546875 -0.671875q-1.09375 -0.671875 -1.71875 -1.8125q-0.625 -1.15625 -0.625 -2.5625q0 -1.421875 0.625 -2.5625q0.625 -1.15625 1.71875 -1.828125q1.109375 -0.671875 2.546875 -0.671875q1.4375 0 2.53125 0.6875q1.109375 0.671875 1.734375 1.828125q0.625 1.140625 0.625 2.546875q0 1.40625 -0.625 2.5625q-0.625 1.140625 -1.734375 1.8125q-1.09375 0.671875 -2.53125 0.671875zm0 -1.4375q0.859375 0 1.609375 -0.421875q0.75 -0.4375 1.21875 -1.25q0.46875 -0.8125 0.46875 -1.9375q0 -1.140625 -0.46875 -1.953125q-0.46875 -0.8125 -1.21875 -1.234375q-0.75 -0.4375 -1.609375 -0.4375q-0.859375 0 -1.625 0.4375q-0.765625 0.421875 -1.234375 1.234375q-0.46875 0.8125 -0.46875 1.953125q0 1.125 0.46875 1.9375q0.46875 0.8125 1.234375 1.25q0.765625 0.421875 1.625 0.421875zm11.041 1.4375q-1.40625 0 -2.5 -0.65625q-1.078125 -0.671875 -1.703125 -1.8125q-0.609375 -1.15625 -0.609375 -2.578125q0 -1.46875 0.609375 -2.59375q0.625 -1.140625 1.703125 -1.796875q1.09375 -0.671875 2.5 -0.671875q1.609375 0 2.640625 0.734375q1.03125 0.734375 1.46875 1.890625l-1.4375 0.609375q-0.359375 -0.890625 -1.0625 -1.34375q-0.6875 -0.453125 -1.6875 -0.453125q-0.828125 0 -1.546875 0.453125q-0.71875 0.4375 -1.171875 1.25q-0.453125 0.8125 -0.453125 1.921875q0 1.078125 0.453125 1.90625q0.453125 0.8125 1.171875 1.265625q0.71875 0.4375 1.546875 0.4375q1.015625 0 1.734375 -0.46875q0.734375 -0.46875 1.09375 -1.3125l1.40625 0.59375q-0.46875 1.09375 -1.515625 1.859375q-1.03125 0.765625 -2.640625 0.765625zm10.014847 0q-1.375 0 -2.453125 -0.640625q-1.0625 -0.65625 -1.671875 -1.796875q-0.609375 -1.140625 -0.609375 -2.59375q0 -1.359375 0.5625 -2.515625q0.578125 -1.15625 1.609375 -1.859375q1.03125 -0.703125 2.4375 -0.703125q1.421875 0 2.4375 0.625q1.015625 0.625 1.5625 1.734375q0.546875 1.09375 0.546875 2.515625q0 0.125 -0.015625 0.265625q0 0.125 -0.015625 0.21875l-8.1875 0l0 -1.3125l6.546875 0q-0.015625 -0.390625 -0.1875 -0.84375q-0.15625 -0.46875 -0.5 -0.859375q-0.34375 -0.40625 -0.875 -0.65625q-0.53125 -0.25 -1.3125 -0.25q-0.9375 0 -1.625 0.484375q-0.671875 0.46875 -1.046875 1.296875q-0.359375 0.8125 -0.359375 1.859375q0 1.203125 0.46875 2.015625q0.46875 0.796875 1.203125 1.1875q0.75 0.390625 1.546875 0.390625q1.046875 0 1.71875 -0.484375q0.6875 -0.5 1.09375 -1.234375l1.34375 0.65625q-0.5625 1.078125 -1.609375 1.796875q-1.03125 0.703125 -2.609375 0.703125zm9.690552 0q-1.0625 0 -1.875 -0.34375q-0.8125 -0.34375 -1.34375 -0.921875q-0.53125 -0.59375 -0.796875 -1.28125l1.421875 -0.640625q0.375 0.859375 1.078125 1.328125q0.703125 0.46875 1.625 0.46875q0.875 0 1.453125 -0.359375q0.59375 -0.359375 0.59375 -1.046875q0 -0.421875 -0.25 -0.703125q-0.234375 -0.296875 -0.703125 -0.5q-0.453125 -0.21875 -1.125 -0.375l-1.15625 -0.3125q-0.671875 -0.1875 -1.28125 -0.515625q-0.59375 -0.34375 -0.953125 -0.875q-0.359375 -0.53125 -0.359375 -1.296875q0 -0.84375 0.5 -1.453125q0.5 -0.625 1.3125 -0.953125q0.828125 -0.328125 1.765625 -0.328125q0.8125 0 1.53125 0.234375q0.71875 0.234375 1.265625 0.6875q0.546875 0.453125 0.8125 1.125l-1.375 0.640625q-0.359375 -0.703125 -0.953125 -0.984375q-0.59375 -0.28125 -1.328125 -0.28125q-0.78125 0 -1.359375 0.34375q-0.578125 0.34375 -0.578125 0.9375q0 0.59375 0.46875 0.90625q0.484375 0.296875 1.171875 0.46875l1.390625 0.359375q1.390625 0.359375 2.09375 1.0625q0.71875 0.6875 0.71875 1.703125q0 0.890625 -0.515625 1.546875q-0.5 0.65625 -1.359375 1.015625q-0.84375 0.34375 -1.890625 0.34375zm8.824402 0q-1.0625 0 -1.875 -0.34375q-0.8125 -0.34375 -1.34375 -0.921875q-0.53125 -0.59375 -0.796875 -1.28125l1.421875 -0.640625q0.375 0.859375 1.078125 1.328125q0.703125 0.46875 1.625 0.46875q0.875 0 1.453125 -0.359375q0.59375 -0.359375 0.59375 -1.046875q0 -0.421875 -0.25 -0.703125q-0.234375 -0.296875 -0.703125 -0.5q-0.453125 -0.21875 -1.125 -0.375l-1.15625 -0.3125q-0.671875 -0.1875 -1.28125 -0.515625q-0.59375 -0.34375 -0.953125 -0.875q-0.359375 -0.53125 -0.359375 -1.296875q0 -0.84375 0.5 -1.453125q0.5 -0.625 1.3125 -0.953125q0.828125 -0.328125 1.765625 -0.328125q0.8125 0 1.53125 0.234375q0.71875 0.234375 1.265625 0.6875q0.546875 0.453125 0.8125 1.125l-1.375 0.640625q-0.359375 -0.703125 -0.953125 -0.984375q-0.59375 -0.28125 -1.328125 -0.28125q-0.78125 0 -1.359375 0.34375q-0.578125 0.34375 -0.578125 0.9375q0 0.59375 0.46875 0.90625q0.484375 0.296875 1.171875 0.46875l1.390625 0.359375q1.390625 0.359375 2.09375 1.0625q0.71875 0.6875 0.71875 1.703125q0 0.890625 -0.515625 1.546875q-0.5 0.65625 -1.359375 1.015625q-0.84375 0.34375 -1.890625 0.34375zm5.559189 -0.296875l0 -9.515625l1.578125 0l0 9.515625l-1.578125 0zm0.78125 -11.265625q-0.46875 0 -0.8125 -0.328125q-0.328125 -0.34375 -0.328125 -0.8125q0 -0.484375 0.328125 -0.8125q0.34375 -0.328125 0.8125 -0.328125q0.484375 0 0.8125 0.328125q0.328125 0.328125 0.328125 0.8125q0 0.46875 -0.328125 0.8125q-0.328125 0.328125 -0.8125 0.328125zm3.2788086 11.265625l0 -9.515625l1.515625 0l0 1.40625l0.078125 0q0.375 -0.703125 1.21875 -1.203125q0.84375 -0.5 1.859375 -0.5q1.75 0 2.625 1.015625q0.8906174 1.015625 0.8906174 2.703125l0 6.09375l-1.5781174 0l0 -5.859375q0 -1.375 -0.671875 -1.9375q-0.65625 -0.578125 -1.703125 -0.578125q-0.78125 0 -1.375 0.4375q-0.59375 0.4375 -0.9375 1.125q-0.328125 0.6875 -0.328125 1.453125l0 5.359375l-1.59375 0zm14.560326 4.328121q-1.21875 0 -2.09375 -0.40625q-0.859375 -0.390625 -1.390625 -1.015625q-0.53125 -0.6093712 -0.75 -1.2499962l1.453125 -0.609375q0.296875 0.78125 1.015625 1.3125q0.71875 0.5312462 1.765625 0.5312462q1.484375 0 2.28125 -0.8749962q0.8125 -0.859375 0.8125 -2.40625l0 -1.0625l-0.078125 0q-0.453125 0.6875 -1.296875 1.171875q-0.828125 0.46875 -1.953125 0.46875q-1.21875 0 -2.25 -0.625q-1.015625 -0.640625 -1.609375 -1.765625q-0.59375 -1.140625 -0.59375 -2.609375q0 -1.46875 0.59375 -2.59375q0.59375 -1.140625 1.609375 -1.765625q1.03125 -0.640625 2.25 -0.640625q1.125 0 1.953125 0.484375q0.84375 0.46875 1.296875 1.171875l0.078125 0l0 -1.359375l1.515625 0l0 9.140625q0 1.59375 -0.609375 2.625q-0.609375 1.0468712 -1.65625 1.5624962q-1.03125 0.515625 -2.34375 0.515625zm0 -5.578121q0.828125 0 1.53125 -0.421875q0.71875 -0.421875 1.140625 -1.21875q0.421875 -0.796875 0.421875 -1.921875q0 -1.15625 -0.421875 -1.953125q-0.421875 -0.796875 -1.140625 -1.203125q-0.703125 -0.40625 -1.53125 -0.40625q-0.84375 0 -1.5625 0.421875q-0.703125 0.421875 -1.140625 1.21875q-0.421875 0.78125 -0.421875 1.921875q0 1.140625 0.421875 1.9375q0.4375 0.78125 1.140625 1.203125q0.71875 0.421875 1.5625 0.421875z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m0 117.82413l161.41733 0l0 47.968506l-161.41733 0z" fill-rule="evenodd"/><path fill="#000000" d="m28.693481 144.74413l0 -13.359375l4.78125 0q1.03125 0 1.890625 0.46875q0.875 0.453125 1.40625 1.265625q0.53125 0.796875 0.53125 1.875q0 0.984375 -0.5 1.71875q-0.5 0.71875 -1.328125 1.09375l0 0.078125q0.984375 0.3125 1.625 1.140625q0.65625 0.8125 0.65625 1.9375q0 1.109375 -0.5625 1.96875q-0.5625 0.84375 -1.484375 1.328125q-0.90625 0.484375 -1.96875 0.484375l-5.046875 0zm0.828125 -6.078125l0 -1.515625l3.953125 0q0.71875 0 1.21875 -0.328125q0.5 -0.328125 0.765625 -0.8125q0.28125 -0.484375 0.28125 -0.984375q0 -0.5 -0.265625 -0.984375q-0.265625 -0.484375 -0.75 -0.8125q-0.484375 -0.328125 -1.171875 -0.328125l-3.28125 0l0 10.328125l3.5625 0q0.703125 0 1.21875 -0.34375q0.53125 -0.34375 0.8125 -0.875q0.28125 -0.53125 0.28125 -1.078125q0 -0.546875 -0.296875 -1.0625q-0.28125 -0.53125 -0.8125 -0.859375q-0.53125 -0.34375 -1.28125 -0.34375l-4.234375 0zm13.60257 6.375q-1.75 0 -2.640625 -1.015625q-0.890625 -1.03125 -0.890625 -2.8125l0 -5.984375l1.59375 0l0 5.75q0 1.421875 0.65625 2.03125q0.65625 0.59375 1.625 0.59375q0.828125 0 1.4375 -0.4375q0.625 -0.4375 0.953125 -1.125q0.34375 -0.6875 0.34375 -1.4375l0 -5.375l1.59375 0l0 9.515625l-1.515625 0l0 -1.375l-0.078125 0q-0.234375 0.4375 -0.71875 0.828125q-0.46875 0.375 -1.078125 0.609375q-0.609375 0.234375 -1.28125 0.234375zm12.356369 -8.375l0 -1.4375l5.796875 0l0 1.4375l-5.796875 0zm1.765625 8.078125l0 -10.546875q0 -0.921875 0.40625 -1.578125q0.40625 -0.671875 1.109375 -1.015625q0.703125 -0.359375 1.515625 -0.359375q0.546875 0 0.859375 0.0625q0.328125 0.0625 0.546875 0.15625l0 1.546875q-0.25 -0.09375 -0.5625 -0.1875q-0.3125 -0.109375 -0.78125 -0.109375q-0.578125 0 -1.046875 0.421875q-0.46875 0.421875 -0.46875 1.234375l0 10.375l-1.578125 0zm-7.671875 -8.078125l0 -1.4375l7.6875 0l0 1.4375l-7.6875 0zm3.34375 -2.296875l0 10.375l-1.578125 0l0 -10.546875q0 -0.921875 0.40625 -1.578125q0.40625 -0.671875 1.109375 -1.015625q0.703125 -0.359375 1.515625 -0.359375q0.546875 0 0.859375 0.0625q0.328125 0.0625 0.546875 0.15625l0 1.546875q-0.25 -0.09375 -0.5625 -0.1875q-0.3125 -0.109375 -0.78125 -0.109375q-0.375 0 -0.734375 0.1875q-0.34375 0.1875 -0.5625 0.546875q-0.21875 0.359375 -0.21875 0.921875zm13.952972 10.671875q-1.375 0 -2.453125 -0.640625q-1.0625 -0.65625 -1.671875 -1.796875q-0.609375 -1.140625 -0.609375 -2.59375q0 -1.359375 0.5625 -2.515625q0.578125 -1.15625 1.609375 -1.859375q1.03125 -0.703125 2.4375 -0.703125q1.421875 0 2.4375 0.625q1.015625 0.625 1.5625 1.734375q0.546875 1.09375 0.546875 2.515625q0 0.125 -0.015625 0.265625q0 0.125 -0.015625 0.21875l-8.1875 0l0 -1.3125l6.546875 0q-0.015625 -0.390625 -0.1875 -0.84375q-0.15625 -0.46875 -0.5 -0.859375q-0.34375 -0.40625 -0.875 -0.65625q-0.53125 -0.25 -1.3125 -0.25q-0.9375 0 -1.625 0.484375q-0.671875 0.46875 -1.046875 1.296875q-0.359375 0.8125 -0.359375 1.859375q0 1.203125 0.46875 2.015625q0.46875 0.796875 1.203125 1.1875q0.75 0.390625 1.546875 0.390625q1.046875 0 1.71875 -0.484375q0.6875 -0.5 1.09375 -1.234375l1.34375 0.65625q-0.5625 1.078125 -1.609375 1.796875q-1.03125 0.703125 -2.609375 0.703125zm6.2690887 -0.296875l0 -9.515625l1.515625 0l0 1.53125l0.078125 0q0.1875 -0.546875 0.625 -0.9375q0.4375 -0.40625 1.0 -0.640625q0.578125 -0.234375 1.125 -0.234375q0.4375 0 0.671875 0.046875q0.25 0.046875 0.453125 0.140625l0 1.71875q-0.296875 -0.15625 -0.640625 -0.21875q-0.34375 -0.078125 -0.703125 -0.078125q-0.6875 0 -1.265625 0.390625q-0.578125 0.390625 -0.921875 1.046875q-0.34375 0.65625 -0.34375 1.4375l0 5.3125l-1.59375 0zm7.1523438 0l0 -9.515625l1.578125 0l0 9.515625l-1.578125 0zm0.78125 -11.265625q-0.46875 0 -0.8125 -0.328125q-0.328125 -0.34375 -0.328125 -0.8125q0 -0.484375 0.328125 -0.8125q0.34375 -0.328125 0.8125 -0.328125q0.484375 0 0.8125 0.328125q0.328125 0.328125 0.328125 0.8125q0 0.46875 -0.328125 0.8125q-0.328125 0.328125 -0.8125 0.328125zm2.7944336 11.265625l0 -1.359375l5.625 -6.71875l-5.453125 0l0 -1.4375l7.390625 0l0 1.359375l-5.625 6.71875l5.765625 0l0 1.4375l-7.703125 0zm12.430191 0.296875q-1.0625 0 -1.875 -0.40625q-0.796875 -0.40625 -1.25 -1.125q-0.453125 -0.71875 -0.453125 -1.640625q0 -1.046875 0.53125 -1.765625q0.546875 -0.71875 1.453125 -1.078125q0.921875 -0.359375 2.015625 -0.359375q0.640625 0 1.171875 0.109375q0.546875 0.09375 0.9375 0.234375q0.40625 0.140625 0.625 0.265625l0 -0.578125q0 -1.078125 -0.765625 -1.703125q-0.765625 -0.640625 -1.875 -0.640625q-0.78125 0 -1.46875 0.34375q-0.671875 0.34375 -1.0625 0.953125l-1.203125 -0.890625q0.375 -0.5625 0.9375 -0.96875q0.5625 -0.40625 1.28125 -0.625q0.71875 -0.234375 1.515625 -0.234375q1.9375 0 3.03125 1.03125q1.109375 1.015625 1.109375 2.75l0 6.03125l-1.5 0l0 -1.359375l-0.078125 0q-0.25 0.40625 -0.703125 0.796875q-0.4375 0.375 -1.046875 0.609375q-0.609375 0.25 -1.328125 0.25zm0.140625 -1.390625q0.828125 0 1.5 -0.40625q0.6875 -0.421875 1.09375 -1.109375q0.421875 -0.6875 0.421875 -1.515625q-0.4375 -0.296875 -1.078125 -0.484375q-0.640625 -0.1875 -1.40625 -0.1875q-1.359375 0 -2.0 0.5625q-0.640625 0.5625 -0.640625 1.375q0 0.78125 0.59375 1.28125q0.609375 0.484375 1.515625 0.484375zm5.876053 -8.421875l5.578125 0l0 1.4375l-5.578125 0l0 -1.4375zm1.671875 7.015625l0 -9.703125l1.578125 0l0 9.3125q0 0.75 0.3125 1.15625q0.3125 0.40625 1.015625 0.40625q0.3125 0 0.578125 -0.09375q0.265625 -0.09375 0.46875 -0.21875l0 1.546875q-0.25 0.109375 -0.546875 0.171875q-0.28125 0.078125 -0.765625 0.078125q-1.1875 0 -1.921875 -0.703125q-0.71875 -0.703125 -0.71875 -1.953125zm6.043503 2.5l0 -9.515625l1.578125 0l0 9.515625l-1.578125 0zm0.78125 -11.265625q-0.46875 0 -0.8125 -0.328125q-0.328125 -0.34375 -0.328125 -0.8125q0 -0.484375 0.328125 -0.8125q0.34375 -0.328125 0.8125 -0.328125q0.484375 0 0.8125 0.328125q0.328125 0.328125 0.328125 0.8125q0 0.46875 -0.328125 0.8125q-0.328125 0.328125 -0.8125 0.328125zm7.5760956 11.5625q-1.4375 0 -2.546875 -0.671875q-1.09375 -0.671875 -1.71875 -1.8125q-0.625 -1.15625 -0.625 -2.5625q0 -1.421875 0.625 -2.5625q0.625 -1.15625 1.71875 -1.828125q1.109375 -0.671875 2.546875 -0.671875q1.4375 0 2.53125 0.6875q1.109375 0.671875 1.734375 1.828125q0.625 1.140625 0.625 2.546875q0 1.40625 -0.625 2.5625q-0.625 1.140625 -1.734375 1.8125q-1.09375 0.671875 -2.53125 0.671875zm0 -1.4375q0.859375 0 1.609375 -0.421875q0.75 -0.4375 1.21875 -1.25q0.46875 -0.8125 0.46875 -1.9375q0 -1.140625 -0.46875 -1.953125q-0.46875 -0.8125 -1.21875 -1.234375q-0.75 -0.4375 -1.609375 -0.4375q-0.859375 0 -1.625 0.4375q-0.765625 0.421875 -1.234375 1.234375q-0.46875 0.8125 -0.46875 1.953125q0 1.125 0.46875 1.9375q0.46875 0.8125 1.234375 1.25q0.765625 0.421875 1.625 0.421875zm6.5418396 1.140625l0 -9.515625l1.515625 0l0 1.40625l0.078125 0q0.375 -0.703125 1.21875 -1.203125q0.84375 -0.5 1.859375 -0.5q1.75 0 2.625 1.015625q0.890625 1.015625 0.890625 2.703125l0 6.09375l-1.578125 0l0 -5.859375q0 -1.375 -0.671875 -1.9375q-0.65625 -0.578125 -1.703125 -0.578125q-0.78125 0 -1.375 0.4375q-0.59375 0.4375 -0.9375 1.125q-0.328125 0.6875 -0.328125 1.453125l0 5.359375l-1.59375 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m0 308.9212l161.41733 0l0 47.968506l-161.41733 0z" fill-rule="evenodd"/><path fill="#000000" d="m30.540504 335.84122l0 -13.359375l4.78125 0q1.03125 0 1.890625 0.46875q0.875 0.453125 1.40625 1.265625q0.53125 0.796875 0.53125 1.875q0 0.984375 -0.5 1.71875q-0.5 0.71875 -1.328125 1.09375l0 0.078125q0.984375 0.3125 1.625 1.140625q0.65625 0.8125 0.65625 1.9375q0 1.109375 -0.5625 1.96875q-0.5625 0.84375 -1.484375 1.328125q-0.90625 0.484375 -1.96875 0.484375l-5.046875 0zm0.828125 -6.078125l0 -1.515625l3.953125 0q0.71875 0 1.21875 -0.328125q0.5 -0.328125 0.765625 -0.8125q0.28125 -0.484375 0.28125 -0.984375q0 -0.5 -0.265625 -0.984375q-0.265625 -0.484375 -0.75 -0.8125q-0.484375 -0.328125 -1.171875 -0.328125l-3.28125 0l0 10.328125l3.5625 0q0.703125 0 1.21875 -0.34375q0.53125 -0.34375 0.8125 -0.875q0.28125 -0.53125 0.28125 -1.078125q0 -0.546875 -0.296875 -1.0625q-0.28125 -0.53125 -0.8125 -0.859375q-0.53125 -0.34375 -1.28125 -0.34375l-4.234375 0zm13.60257 6.375q-1.75 0 -2.640625 -1.015625q-0.890625 -1.03125 -0.890625 -2.8125l0 -5.984375l1.59375 0l0 5.75q0 1.421875 0.65625 2.03125q0.65625 0.59375 1.625 0.59375q0.828125 0 1.4375 -0.4375q0.625 -0.4375 0.953125 -1.125q0.34375 -0.6875 0.34375 -1.4375l0 -5.375l1.59375 0l0 9.515625l-1.515625 0l0 -1.375l-0.078125 0q-0.234375 0.4375 -0.71875 0.828125q-0.46875 0.375 -1.078125 0.609375q-0.609375 0.234375 -1.28125 0.234375zm12.356369 -8.375l0 -1.4375l5.796875 0l0 1.4375l-5.796875 0zm1.765625 8.078125l0 -10.546875q0 -0.921875 0.40625 -1.578125q0.40625 -0.671875 1.109375 -1.015625q0.703125 -0.359375 1.515625 -0.359375q0.546875 0 0.859375 0.0625q0.328125 0.0625 0.546875 0.15625l0 1.546875q-0.25 -0.09375 -0.5625 -0.1875q-0.3125 -0.109375 -0.78125 -0.109375q-0.578125 0 -1.046875 0.421875q-0.46875 0.421875 -0.46875 1.234375l0 10.375l-1.578125 0zm-7.671875 -8.078125l0 -1.4375l7.6875 0l0 1.4375l-7.6875 0zm3.34375 -2.296875l0 10.375l-1.578125 0l0 -10.546875q0 -0.921875 0.40625 -1.578125q0.40625 -0.671875 1.109375 -1.015625q0.703125 -0.359375 1.515625 -0.359375q0.546875 0 0.859375 0.0625q0.328125 0.0625 0.546875 0.15625l0 1.546875q-0.25 -0.09375 -0.5625 -0.1875q-0.3125 -0.109375 -0.78125 -0.109375q-0.375 0 -0.734375 0.1875q-0.34375 0.1875 -0.5625 0.546875q-0.21875 0.359375 -0.21875 0.921875zm13.952972 10.671875q-1.375 0 -2.453125 -0.640625q-1.0625 -0.65625 -1.671875 -1.796875q-0.609375 -1.140625 -0.609375 -2.59375q0 -1.359375 0.5625 -2.515625q0.578125 -1.15625 1.609375 -1.859375q1.03125 -0.703125 2.4375 -0.703125q1.421875 0 2.4375 0.625q1.015625 0.625 1.5625 1.734375q0.546875 1.09375 0.546875 2.515625q0 0.125 -0.015625 0.265625q0 0.125 -0.015625 0.21875l-8.1875 0l0 -1.3125l6.546875 0q-0.015625 -0.390625 -0.1875 -0.84375q-0.15625 -0.46875 -0.5 -0.859375q-0.34375 -0.40625 -0.875 -0.65625q-0.53125 -0.25 -1.3125 -0.25q-0.9375 0 -1.625 0.484375q-0.671875 0.46875 -1.046875 1.296875q-0.359375 0.8125 -0.359375 1.859375q0 1.203125 0.46875 2.015625q0.46875 0.796875 1.203125 1.1875q0.75 0.390625 1.546875 0.390625q1.046875 0 1.71875 -0.484375q0.6875 -0.5 1.09375 -1.234375l1.34375 0.65625q-0.5625 1.078125 -1.609375 1.796875q-1.03125 0.703125 -2.609375 0.703125zm6.2690887 -0.296875l0 -9.515625l1.515625 0l0 1.53125l0.078125 0q0.1875 -0.546875 0.625 -0.9375q0.4375 -0.40625 1.0 -0.640625q0.578125 -0.234375 1.125 -0.234375q0.4375 0 0.671875 0.046875q0.25 0.046875 0.453125 0.140625l0 1.71875q-0.296875 -0.15625 -0.640625 -0.21875q-0.34375 -0.078125 -0.703125 -0.078125q-0.6875 0 -1.265625 0.390625q-0.578125 0.390625 -0.921875 1.046875q-0.34375 0.65625 -0.34375 1.4375l0 5.3125l-1.59375 0zm6.9804688 -5.125l0 -1.3125l5.921875 0l0 1.3125l-5.921875 0zm8.571152 5.125l0 -13.359375l1.578125 0l0 11.84375l5.84375 0l0 1.515625l-7.421875 0zm13.178284 0.296875q-1.375 0 -2.453125 -0.640625q-1.0625 -0.65625 -1.671875 -1.796875q-0.609375 -1.140625 -0.609375 -2.59375q0 -1.359375 0.5625 -2.515625q0.578125 -1.15625 1.609375 -1.859375q1.03125 -0.703125 2.4375 -0.703125q1.421875 0 2.4375 0.625q1.015625 0.625 1.5625 1.734375q0.546875 1.09375 0.546875 2.515625q0 0.125 -0.015625 0.265625q0 0.125 -0.015625 0.21875l-8.1875 0l0 -1.3125l6.546875 0q-0.015625 -0.390625 -0.1875 -0.84375q-0.15625 -0.46875 -0.5 -0.859375q-0.34375 -0.40625 -0.875 -0.65625q-0.53125 -0.25 -1.3125 -0.25q-0.9375 0 -1.625 0.484375q-0.671875 0.46875 -1.046875 1.296875q-0.359375 0.8125 -0.359375 1.859375q0 1.203125 0.46875 2.015625q0.46875 0.796875 1.203125 1.1875q0.75 0.390625 1.546875 0.390625q1.046875 0 1.71875 -0.484375q0.6875 -0.5 1.09375 -1.234375l1.34375 0.65625q-0.5625 1.078125 -1.609375 1.796875q-1.03125 0.703125 -2.609375 0.703125zm8.879303 -0.296875l-3.84375 -9.515625l1.703125 0l2.953125 7.65625l0.03125 0l2.984375 -7.65625l1.65625 0l-3.875 9.515625l-1.609375 0zm10.746811 0.296875q-1.375 0 -2.453125 -0.640625q-1.0625 -0.65625 -1.671875 -1.796875q-0.609375 -1.140625 -0.609375 -2.59375q0 -1.359375 0.5625 -2.515625q0.578125 -1.15625 1.609375 -1.859375q1.03125 -0.703125 2.4375 -0.703125q1.421875 0 2.4375 0.625q1.015625 0.625 1.5625 1.734375q0.546875 1.09375 0.546875 2.515625q0 0.125 -0.015625 0.265625q0 0.125 -0.015625 0.21875l-8.1875 0l0 -1.3125l6.546875 0q-0.015625 -0.390625 -0.1875 -0.84375q-0.15625 -0.46875 -0.5 -0.859375q-0.34375 -0.40625 -0.875 -0.65625q-0.53125 -0.25 -1.3125 -0.25q-0.9375 0 -1.625 0.484375q-0.671875 0.46875 -1.046875 1.296875q-0.359375 0.8125 -0.359375 1.859375q0 1.203125 0.46875 2.015625q0.46875 0.796875 1.203125 1.1875q0.75 0.390625 1.546875 0.390625q1.046875 0 1.71875 -0.484375q0.6875 -0.5 1.09375 -1.234375l1.34375 0.65625q-0.5625 1.078125 -1.609375 1.796875q-1.03125 0.703125 -2.609375 0.703125zm6.2690964 -0.296875l0 -13.359375l1.59375 0l0 13.359375l-1.59375 0z" fill-rule="nonzero"/><path fill="#000000" d="m29.563927 358.5381q-1.46875 0 -2.71875 -0.53125q-1.25 -0.546875 -2.1875 -1.5q-0.921875 -0.96875 -1.4375 -2.21875q-0.5 -1.265625 -0.5 -2.71875q0 -1.46875 0.5 -2.71875q0.515625 -1.265625 1.4375 -2.21875q0.9375 -0.96875 2.1875 -1.5q1.25 -0.546875 2.71875 -0.546875q1.46875 0 2.71875 0.546875q1.25 0.53125 2.171875 1.5q0.921875 0.953125 1.4375 2.21875q0.515625 1.25 0.515625 2.71875q0 1.453125 -0.515625 2.71875q-0.515625 1.25 -1.4375 2.21875q-0.921875 0.953125 -2.171875 1.5q-1.25 0.53125 -2.71875 0.53125zm0 -1.515625q1.453125 0 2.640625 -0.6875q1.1875 -0.6875 1.890625 -1.921875q0.703125 -1.25 0.703125 -2.84375q0 -1.625 -0.703125 -2.859375q-0.703125 -1.234375 -1.890625 -1.921875q-1.1875 -0.6875 -2.640625 -0.6875q-1.4375 0 -2.640625 0.6875q-1.1875 0.6875 -1.90625 1.921875q-0.703125 1.234375 -0.703125 2.859375q0 1.59375 0.703125 2.84375q0.71875 1.234375 1.90625 1.921875q1.203125 0.6875 2.640625 0.6875zm8.857132 5.25l0 -13.546875l1.515625 0l0 1.421875l0.078125 0q0.25 -0.453125 0.71875 -0.84375q0.484375 -0.390625 1.125 -0.625q0.640625 -0.25 1.390625 -0.25q1.3125 0 2.328125 0.65625q1.03125 0.65625 1.625 1.796875q0.609375 1.125 0.609375 2.609375q0 1.46875 -0.609375 2.609375q-0.59375 1.140625 -1.625 1.796875q-1.015625 0.640625 -2.328125 0.640625q-1.125 0 -1.984375 -0.515625q-0.859375 -0.53125 -1.25 -1.1875l-0.078125 0l0.078125 1.3125l0 4.125l-1.59375 0zm4.671875 -5.171875q0.8125 0 1.53125 -0.4375q0.71875 -0.4375 1.15625 -1.25q0.4375 -0.828125 0.4375 -1.921875q0 -1.125 -0.4375 -1.9375q-0.4375 -0.8125 -1.15625 -1.25q-0.71875 -0.4375 -1.53125 -0.4375q-0.828125 0 -1.546875 0.4375q-0.71875 0.4375 -1.171875 1.25q-0.4375 0.8125 -0.4375 1.9375q0 1.109375 0.4375 1.921875q0.453125 0.8125 1.171875 1.25q0.71875 0.4375 1.546875 0.4375zm5.529602 -8.375l5.578125 0l0 1.4375l-5.578125 0l0 -1.4375zm1.671875 7.015625l0 -9.703125l1.578125 0l0 9.3125q0 0.75 0.3125 1.15625q0.3125 0.40625 1.015625 0.40625q0.3125 0 0.578125 -0.09375q0.265625 -0.09375 0.46875 -0.21875l0 1.546875q-0.25 0.109375 -0.546875 0.171875q-0.28125 0.078125 -0.765625 0.078125q-1.1875 0 -1.921875 -0.703125q-0.71875 -0.703125 -0.71875 -1.953125zm6.043503 2.5l0 -9.515625l1.578125 0l0 9.515625l-1.578125 0zm0.78125 -11.265625q-0.46875 0 -0.8125 -0.328125q-0.328125 -0.34375 -0.328125 -0.8125q0 -0.484375 0.328125 -0.8125q0.34375 -0.328125 0.8125 -0.328125q0.484375 0 0.8125 0.328125q0.328125 0.328125 0.328125 0.8125q0 0.46875 -0.328125 0.8125q-0.328125 0.328125 -0.8125 0.328125zm3.2788086 11.265625l0 -9.515625l1.515625 0l0 1.40625l0.078125 0q0.25 -0.46875 0.703125 -0.84375q0.46875 -0.390625 1.046875 -0.625q0.59375 -0.234375 1.21875 -0.234375q1.0625 0 1.828125 0.515625q0.765625 0.515625 1.09375 1.328125q0.484375 -0.796875 1.296875 -1.3125q0.828125 -0.53125 1.96875 -0.53125q1.703125 0 2.5 1.03125q0.796875 1.015625 0.796875 2.6875l0 6.09375l-1.5625 0l0 -5.859375q0 -1.375 -0.5625 -1.9375q-0.5625 -0.578125 -1.578125 -0.578125q-0.734375 0 -1.3125 0.421875q-0.5625 0.421875 -0.890625 1.109375q-0.3125 0.671875 -0.3125 1.46875l0 5.375l-1.59375 0l0 -5.84375q0 -1.375 -0.5625 -1.953125q-0.5625 -0.578125 -1.5625 -0.578125q-0.734375 0 -1.296875 0.4375q-0.5625 0.421875 -0.890625 1.109375q-0.328125 0.6875 -0.328125 1.5l0 5.328125l-1.59375 0zm16.480469 0l0 -9.515625l1.578125 0l0 9.515625l-1.578125 0zm0.78125 -11.265625q-0.46875 0 -0.8125 -0.328125q-0.328125 -0.34375 -0.328125 -0.8125q0 -0.484375 0.328125 -0.8125q0.34375 -0.328125 0.8125 -0.328125q0.484375 0 0.8125 0.328125q0.328125 0.328125 0.328125 0.8125q0 0.46875 -0.328125 0.8125q-0.328125 0.328125 -0.8125 0.328125zm2.7944336 11.265625l0 -1.359375l5.625 -6.71875l-5.453125 0l0 -1.4375l7.390625 0l0 1.359375l-5.625 6.71875l5.765625 0l0 1.4375l-7.703125 0zm12.430191 0.296875q-1.0625 0 -1.875 -0.40625q-0.796875 -0.40625 -1.25 -1.125q-0.453125 -0.71875 -0.453125 -1.640625q0 -1.046875 0.53125 -1.765625q0.546875 -0.71875 1.453125 -1.078125q0.921875 -0.359375 2.015625 -0.359375q0.640625 0 1.171875 0.109375q0.546875 0.09375 0.9375 0.234375q0.40625 0.140625 0.625 0.265625l0 -0.578125q0 -1.078125 -0.765625 -1.703125q-0.765625 -0.640625 -1.875 -0.640625q-0.78125 0 -1.46875 0.34375q-0.671875 0.34375 -1.0625 0.953125l-1.203125 -0.890625q0.375 -0.5625 0.9375 -0.96875q0.5625 -0.40625 1.28125 -0.625q0.71875 -0.234375 1.515625 -0.234375q1.9375 0 3.03125 1.03125q1.109375 1.015625 1.109375 2.75l0 6.03125l-1.5 0l0 -1.359375l-0.078125 0q-0.25 0.40625 -0.703125 0.796875q-0.4375 0.375 -1.046875 0.609375q-0.609375 0.25 -1.328125 0.25zm0.140625 -1.390625q0.828125 0 1.5 -0.40625q0.6875 -0.421875 1.09375 -1.109375q0.421875 -0.6875 0.421875 -1.515625q-0.4375 -0.296875 -1.078125 -0.484375q-0.640625 -0.1875 -1.40625 -0.1875q-1.359375 0 -2.0 0.5625q-0.640625 0.5625 -0.640625 1.375q0 0.78125 0.59375 1.28125q0.609375 0.484375 1.515625 0.484375zm5.876053 -8.421875l5.578125 0l0 1.4375l-5.578125 0l0 -1.4375zm1.671875 7.015625l0 -9.703125l1.578125 0l0 9.3125q0 0.75 0.3125 1.15625q0.3125 0.40625 1.015625 0.40625q0.3125 0 0.578125 -0.09375q0.265625 -0.09375 0.46875 -0.21875l0 1.546875q-0.25 0.109375 -0.546875 0.171875q-0.28125 0.078125 -0.765625 0.078125q-1.1875 0 -1.921875 -0.703125q-0.71875 -0.703125 -0.71875 -1.953125zm6.043503 2.5l0 -9.515625l1.578125 0l0 9.515625l-1.578125 0zm0.78125 -11.265625q-0.46875 0 -0.8125 -0.328125q-0.328125 -0.34375 -0.328125 -0.8125q0 -0.484375 0.328125 -0.8125q0.34375 -0.328125 0.8125 -0.328125q0.484375 0 0.8125 0.328125q0.328125 0.328125 0.328125 0.8125q0 0.46875 -0.328125 0.8125q-0.328125 0.328125 -0.8125 0.328125zm7.5760956 11.5625q-1.4375 0 -2.546875 -0.671875q-1.09375 -0.671875 -1.71875 -1.8125q-0.625 -1.15625 -0.625 -2.5625q0 -1.421875 0.625 -2.5625q0.625 -1.15625 1.71875 -1.828125q1.109375 -0.671875 2.546875 -0.671875q1.4375 0 2.53125 0.6875q1.109375 0.671875 1.734375 1.828125q0.625 1.140625 0.625 2.546875q0 1.40625 -0.625 2.5625q-0.625 1.140625 -1.734375 1.8125q-1.09375 0.671875 -2.53125 0.671875zm0 -1.4375q0.859375 0 1.609375 -0.421875q0.75 -0.4375 1.21875 -1.25q0.46875 -0.8125 0.46875 -1.9375q0 -1.140625 -0.46875 -1.953125q-0.46875 -0.8125 -1.21875 -1.234375q-0.75 -0.4375 -1.609375 -0.4375q-0.859375 0 -1.625 0.4375q-0.765625 0.421875 -1.234375 1.234375q-0.46875 0.8125 -0.46875 1.953125q0 1.125 0.46875 1.9375q0.46875 0.8125 1.234375 1.25q0.765625 0.421875 1.625 0.421875zm6.5418396 1.140625l0 -9.515625l1.515625 0l0 1.40625l0.078125 0q0.375 -0.703125 1.21875 -1.203125q0.84375 -0.5 1.859375 -0.5q1.75 0 2.6250076 1.015625q0.890625 1.015625 0.890625 2.703125l0 6.09375l-1.578125 0l0 -5.859375q0 -1.375 -0.6718826 -1.9375q-0.65625 -0.578125 -1.703125 -0.578125q-0.78125 0 -1.375 0.4375q-0.59375 0.4375 -0.9375 1.125q-0.328125 0.6875 -0.328125 1.453125l0 5.359375l-1.59375 0zm13.700935 0.296875q-1.0625 0 -1.875 -0.34375q-0.8125 -0.34375 -1.34375 -0.921875q-0.53125 -0.59375 -0.796875 -1.28125l1.421875 -0.640625q0.375 0.859375 1.078125 1.328125q0.703125 0.46875 1.625 0.46875q0.875 0 1.453125 -0.359375q0.59375 -0.359375 0.59375 -1.046875q0 -0.421875 -0.25 -0.703125q-0.234375 -0.296875 -0.703125 -0.5q-0.453125 -0.21875 -1.125 -0.375l-1.15625 -0.3125q-0.671875 -0.1875 -1.28125 -0.515625q-0.59375 -0.34375 -0.953125 -0.875q-0.359375 -0.53125 -0.359375 -1.296875q0 -0.84375 0.5 -1.453125q0.5 -0.625 1.3125 -0.953125q0.828125 -0.328125 1.765625 -0.328125q0.8125 0 1.53125 0.234375q0.71875 0.234375 1.265625 0.6875q0.546875 0.453125 0.8125 1.125l-1.375 0.640625q-0.359375 -0.703125 -0.953125 -0.984375q-0.59375 -0.28125 -1.328125 -0.28125q-0.78125 0 -1.359375 0.34375q-0.578125 0.34375 -0.578125 0.9375q0 0.59375 0.46875 0.90625q0.484375 0.296875 1.171875 0.46875l1.390625 0.359375q1.390625 0.359375 2.09375 1.0625q0.71875 0.6875 0.71875 1.703125q0 0.890625 -0.515625 1.546875q-0.5 0.65625 -1.359375 1.015625q-0.84375 0.34375 -1.890625 0.34375z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m0 454.21594l161.41733 0l0 47.968506l-161.41733 0z" fill-rule="evenodd"/><path fill="#000000" d="m29.971382 481.13596l0 -13.359375l4.015625 0q2.09375 0 3.609375 0.859375q1.515625 0.84375 2.328125 2.359375q0.828125 1.5 0.828125 3.46875q0 1.953125 -0.828125 3.453125q-0.8125 1.5 -2.328125 2.359375q-1.515625 0.859375 -3.609375 0.859375l-4.015625 0zm1.578125 -1.515625l2.4375 0q1.578125 0 2.734375 -0.59375q1.15625 -0.609375 1.78125 -1.765625q0.640625 -1.15625 0.640625 -2.796875q0 -1.65625 -0.640625 -2.8125q-0.625 -1.15625 -1.78125 -1.75q-1.15625 -0.609375 -2.734375 -0.609375l-2.4375 0l0 10.328125zm15.163467 1.8125q-1.375 0 -2.453125 -0.640625q-1.0625 -0.65625 -1.671875 -1.796875q-0.609375 -1.140625 -0.609375 -2.59375q0 -1.359375 0.5625 -2.515625q0.578125 -1.15625 1.609375 -1.859375q1.03125 -0.703125 2.4375 -0.703125q1.421875 0 2.4375 0.625q1.015625 0.625 1.5625 1.734375q0.546875 1.09375 0.546875 2.515625q0 0.125 -0.015625 0.265625q0 0.125 -0.015625 0.21875l-8.1875 0l0 -1.3125l6.546875 0q-0.015625 -0.390625 -0.1875 -0.84375q-0.15625 -0.46875 -0.5 -0.859375q-0.34375 -0.40625 -0.875 -0.65625q-0.53125 -0.25 -1.3125 -0.25q-0.9375 0 -1.625 0.484375q-0.671875 0.46875 -1.046875 1.296875q-0.359375 0.8125 -0.359375 1.859375q0 1.203125 0.46875 2.015625q0.46875 0.796875 1.203125 1.1875q0.75 0.390625 1.546875 0.390625q1.046875 0 1.71875 -0.484375q0.6875 -0.5 1.09375 -1.234375l1.34375 0.65625q-0.5625 1.078125 -1.609375 1.796875q-1.03125 0.703125 -2.609375 0.703125zm9.409302 0q-1.0625 0 -1.875 -0.40625q-0.796875 -0.40625 -1.25 -1.125q-0.453125 -0.71875 -0.453125 -1.640625q0 -1.046875 0.53125 -1.765625q0.546875 -0.71875 1.453125 -1.078125q0.921875 -0.359375 2.015625 -0.359375q0.640625 0 1.171875 0.109375q0.546875 0.09375 0.9375 0.234375q0.40625 0.140625 0.625 0.265625l0 -0.578125q0 -1.078125 -0.765625 -1.703125q-0.765625 -0.640625 -1.875 -0.640625q-0.78125 0 -1.46875 0.34375q-0.671875 0.34375 -1.0625 0.953125l-1.203125 -0.890625q0.375 -0.5625 0.9375 -0.96875q0.5625 -0.40625 1.28125 -0.625q0.71875 -0.234375 1.515625 -0.234375q1.9375 0 3.03125 1.03125q1.109375 1.015625 1.109375 2.75l0 6.03125l-1.5 0l0 -1.359375l-0.078125 0q-0.25 0.40625 -0.703125 0.796875q-0.4375 0.375 -1.046875 0.609375q-0.609375 0.25 -1.328125 0.25zm0.140625 -1.390625q0.828125 0 1.5 -0.40625q0.6875 -0.421875 1.09375 -1.109375q0.421875 -0.6875 0.421875 -1.515625q-0.4375 -0.296875 -1.078125 -0.484375q-0.640625 -0.1875 -1.40625 -0.1875q-1.359375 0 -2.0 0.5625q-0.640625 0.5625 -0.640625 1.375q0 0.78125 0.59375 1.28125q0.609375 0.484375 1.515625 0.484375zm6.7189636 1.09375l0 -13.359375l1.59375 0l0 13.359375l-1.59375 0zm3.9364624 0l0 -13.359375l1.59375 0l0 13.359375l-1.59375 0zm8.233749 0.296875q-1.4375 0 -2.546875 -0.671875q-1.09375 -0.671875 -1.71875 -1.8125q-0.625 -1.15625 -0.625 -2.5625q0 -1.421875 0.625 -2.5625q0.625 -1.15625 1.71875 -1.828125q1.109375 -0.671875 2.546875 -0.671875q1.4375 0 2.53125 0.6875q1.109375 0.671875 1.734375 1.828125q0.625 1.140625 0.625 2.546875q0 1.40625 -0.625 2.5625q-0.625 1.140625 -1.734375 1.8125q-1.09375 0.671875 -2.53125 0.671875zm0 -1.4375q0.859375 0 1.609375 -0.421875q0.75 -0.4375 1.21875 -1.25q0.46875 -0.8125 0.46875 -1.9375q0 -1.140625 -0.46875 -1.953125q-0.46875 -0.8125 -1.21875 -1.234375q-0.75 -0.4375 -1.609375 -0.4375q-0.859375 0 -1.625 0.4375q-0.765625 0.421875 -1.234375 1.234375q-0.46875 0.8125 -0.46875 1.953125q0 1.125 0.46875 1.9375q0.46875 0.8125 1.234375 1.25q0.765625 0.421875 1.625 0.421875zm11.041 1.4375q-1.40625 0 -2.5 -0.65625q-1.078125 -0.671875 -1.703125 -1.8125q-0.609375 -1.15625 -0.609375 -2.578125q0 -1.46875 0.609375 -2.59375q0.625 -1.140625 1.703125 -1.796875q1.09375 -0.671875 2.5 -0.671875q1.609375 0 2.640625 0.734375q1.03125 0.734375 1.46875 1.890625l-1.4375 0.609375q-0.359375 -0.890625 -1.0625 -1.34375q-0.6875 -0.453125 -1.6875 -0.453125q-0.828125 0 -1.546875 0.453125q-0.71875 0.4375 -1.171875 1.25q-0.453125 0.8125 -0.453125 1.921875q0 1.078125 0.453125 1.90625q0.453125 0.8125 1.171875 1.265625q0.71875 0.4375 1.546875 0.4375q1.015625 0 1.734375 -0.46875q0.734375 -0.46875 1.09375 -1.3125l1.40625 0.59375q-0.46875 1.09375 -1.515625 1.859375q-1.03125 0.765625 -2.640625 0.765625zm8.827347 0q-1.0625 0 -1.875 -0.40625q-0.796875 -0.40625 -1.25 -1.125q-0.453125 -0.71875 -0.453125 -1.640625q0 -1.046875 0.53125 -1.765625q0.546875 -0.71875 1.453125 -1.078125q0.921875 -0.359375 2.015625 -0.359375q0.640625 0 1.171875 0.109375q0.546875 0.09375 0.9375 0.234375q0.40625 0.140625 0.625 0.265625l0 -0.578125q0 -1.078125 -0.765625 -1.703125q-0.765625 -0.640625 -1.875 -0.640625q-0.78125 0 -1.46875 0.34375q-0.671875 0.34375 -1.0625 0.953125l-1.203125 -0.890625q0.375 -0.5625 0.9375 -0.96875q0.5625 -0.40625 1.28125 -0.625q0.71875 -0.234375 1.515625 -0.234375q1.9375 0 3.03125 1.03125q1.109375 1.015625 1.109375 2.75l0 6.03125l-1.5 0l0 -1.359375l-0.078125 0q-0.25 0.40625 -0.703125 0.796875q-0.4375 0.375 -1.046875 0.609375q-0.609375 0.25 -1.328125 0.25zm0.140625 -1.390625q0.828125 0 1.5 -0.40625q0.6875 -0.421875 1.09375 -1.109375q0.421875 -0.6875 0.421875 -1.515625q-0.4375 -0.296875 -1.078125 -0.484375q-0.640625 -0.1875 -1.40625 -0.1875q-1.359375 0 -2.0 0.5625q-0.640625 0.5625 -0.640625 1.375q0 0.78125 0.59375 1.28125q0.609375 0.484375 1.515625 0.484375zm5.876053 -8.421875l5.578125 0l0 1.4375l-5.578125 0l0 -1.4375zm1.671875 7.015625l0 -9.703125l1.578125 0l0 9.3125q0 0.75 0.3125 1.15625q0.3125 0.40625 1.015625 0.40625q0.3125 0 0.578125 -0.09375q0.265625 -0.09375 0.46875 -0.21875l0 1.546875q-0.25 0.109375 -0.546875 0.171875q-0.28125 0.078125 -0.765625 0.078125q-1.1875 0 -1.921875 -0.703125q-0.71875 -0.703125 -0.71875 -1.953125zm6.043503 2.5l0 -9.515625l1.578125 0l0 9.515625l-1.578125 0zm0.78125 -11.265625q-0.46875 0 -0.8125 -0.328125q-0.328125 -0.34375 -0.328125 -0.8125q0 -0.484375 0.328125 -0.8125q0.34375 -0.328125 0.8125 -0.328125q0.484375 0 0.8125 0.328125q0.328125 0.328125 0.328125 0.8125q0 0.46875 -0.328125 0.8125q-0.328125 0.328125 -0.8125 0.328125zm7.5760956 11.5625q-1.4375 0 -2.546875 -0.671875q-1.09375 -0.671875 -1.71875 -1.8125q-0.625 -1.15625 -0.625 -2.5625q0 -1.421875 0.625 -2.5625q0.625 -1.15625 1.71875 -1.828125q1.109375 -0.671875 2.546875 -0.671875q1.4375 0 2.53125 0.6875q1.109375 0.671875 1.734375 1.828125q0.625 1.140625 0.625 2.546875q0 1.40625 -0.625 2.5625q-0.625 1.140625 -1.734375 1.8125q-1.09375 0.671875 -2.53125 0.671875zm0 -1.4375q0.859375 0 1.609375 -0.421875q0.75 -0.4375 1.21875 -1.25q0.46875 -0.8125 0.46875 -1.9375q0 -1.140625 -0.46875 -1.953125q-0.46875 -0.8125 -1.21875 -1.234375q-0.75 -0.4375 -1.609375 -0.4375q-0.859375 0 -1.625 0.4375q-0.765625 0.421875 -1.234375 1.234375q-0.46875 0.8125 -0.46875 1.953125q0 1.125 0.46875 1.9375q0.46875 0.8125 1.234375 1.25q0.765625 0.421875 1.625 0.421875zm6.5418396 1.140625l0 -9.515625l1.515625 0l0 1.40625l0.078125 0q0.375 -0.703125 1.21875 -1.203125q0.84375 -0.5 1.8593826 -0.5q1.75 0 2.625 1.015625q0.890625 1.015625 0.890625 2.703125l0 6.09375l-1.578125 0l0 -5.859375q0 -1.375 -0.671875 -1.9375q-0.65625 -0.578125 -1.7031326 -0.578125q-0.78125 0 -1.375 0.4375q-0.59375 0.4375 -0.9375 1.125q-0.328125 0.6875 -0.328125 1.453125l0 5.359375l-1.59375 0z" fill-rule="nonzero"/><path fill="#fbbc05" d="m163.16655 485.02887l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path stroke="#fbbc05" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m163.16655 485.02887l315.59058 0l0 29.039368l-315.59058 0z" fill-rule="evenodd"/><path fill="#000000" d="m246.55295 501.31607l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm9.16745 -0.40625q0 0.890625 -0.25 1.59375q-0.234375 0.703125 -0.703125 1.203125q-0.453125 0.484375 -1.109375 0.75q-0.640625 0.25 -1.453125 0.25q-0.640625 0 -1.3125 -0.109375q-0.65625 -0.125 -1.3125 -0.390625l0 -9.6875l1.25 0l0 2.78125l-0.0625 1.328125q0.546875 -0.71875 1.15625 -1.015625q0.609375 -0.296875 1.3125 -0.296875q0.609375 0 1.078125 0.265625q0.46875 0.25 0.78125 0.71875q0.3125 0.46875 0.46875 1.140625q0.15625 0.65625 0.15625 1.46875zm-1.265625 0.046875q0 -0.5625 -0.09375 -1.03125q-0.078125 -0.46875 -0.265625 -0.796875q-0.171875 -0.34375 -0.453125 -0.53125q-0.265625 -0.1875 -0.65625 -0.1875q-0.234375 0 -0.484375 0.078125q-0.234375 0.0625 -0.5 0.25q-0.265625 0.171875 -0.5625 0.46875q-0.28125 0.28125 -0.609375 0.703125l0 3.484375q0.359375 0.140625 0.734375 0.234375q0.390625 0.078125 0.734375 0.078125q0.421875 0 0.8125 -0.125q0.390625 -0.140625 0.6875 -0.453125q0.296875 -0.328125 0.46875 -0.859375q0.1875 -0.53125 0.1875 -1.3125zm8.964325 3.671875l-1.125 0l-0.03125 -1.15625q-0.328125 0.375 -0.625 0.625q-0.28125 0.234375 -0.578125 0.390625q-0.28125 0.140625 -0.578125 0.203125q-0.28125 0.0625 -0.59375 0.0625q-1.109375 0 -1.6875 -0.640625q-0.5625 -0.65625 -0.5625 -1.96875l0 -4.703125l1.25 0l0 4.59375q0 1.65625 1.25 1.65625q0.21875 0 0.4375 -0.0625q0.21875 -0.078125 0.453125 -0.234375q0.25 -0.171875 0.515625 -0.453125q0.28125 -0.296875 0.625 -0.734375l0 -4.765625l1.25 0l0 7.1875zm8.651825 -8.953125q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.5737 -1.765625q-0.96875 -0.203125 -1.6875 -0.203125q-1.671875 0 -1.671875 1.75l0 1.265625l3.140625 0l0 1.03125l-3.140625 0l0 5.109375l-1.265625 0l0 -5.109375l-2.3125 0l0 -1.03125l2.3125 0l0 -1.1875q0 -2.875 2.984375 -2.875q0.75 0 1.640625 0.171875l0 1.078125zm-7.515625 1.765625l0 0zm15.276825 3.21875q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm3.5737 -3.03125l1.140625 0l0.03125 1.328125q0.640625 -0.765625 1.25 -1.109375q0.625 -0.34375 1.25 -0.34375q1.125 0 1.703125 0.71875q0.578125 0.71875 0.53125 2.15625l-1.265625 0q0.03125 -0.953125 -0.265625 -1.375q-0.296875 -0.421875 -0.875 -0.421875q-0.25 0 -0.515625 0.09375q-0.25 0.078125 -0.515625 0.28125q-0.265625 0.1875 -0.5625 0.5q-0.296875 0.3125 -0.640625 0.75l0 4.609375l-1.265625 0l0 -7.1875zm12.79245 3.875l-4.28125 0l0 -1.171875l4.28125 0l0 1.171875zm2.66745 -0.125q0 -0.921875 0.25 -1.625q0.25 -0.71875 0.703125 -1.203125q0.46875 -0.5 1.109375 -0.75q0.65625 -0.25 1.453125 -0.25q0.34375 0 0.671875 0.046875q0.328125 0.03125 0.65625 0.125l0 -3.015625l1.25 0l0 10.109375l-1.109375 0l-0.046875 -1.359375q-0.515625 0.765625 -1.125 1.125q-0.609375 0.359375 -1.328125 0.359375q-0.609375 0 -1.078125 -0.25q-0.46875 -0.265625 -0.78125 -0.734375q-0.3125 -0.46875 -0.46875 -1.125q-0.15625 -0.65625 -0.15625 -1.453125zm1.265625 -0.078125q0 1.296875 0.375 1.9375q0.390625 0.640625 1.09375 0.640625q0.484375 0 1.0 -0.421875q0.53125 -0.421875 1.109375 -1.25l0 -3.34375q-0.3125 -0.140625 -0.6875 -0.203125q-0.359375 -0.078125 -0.734375 -0.078125q-1.015625 0 -1.59375 0.65625q-0.5625 0.65625 -0.5625 2.0625zm13.22995 -0.453125q0 0.265625 -0.015625 0.453125q0 0.171875 -0.015625 0.328125l-5.046875 0q0 1.09375 0.609375 1.6875q0.625 0.59375 1.78125 0.59375q0.3125 0 0.625 -0.015625q0.3125 -0.03125 0.609375 -0.078125q0.296875 -0.046875 0.5625 -0.09375q0.265625 -0.0625 0.5 -0.125l0 1.03125q-0.515625 0.140625 -1.15625 0.21875q-0.640625 0.09375 -1.328125 0.09375q-0.921875 0 -1.59375 -0.25q-0.65625 -0.25 -1.078125 -0.71875q-0.421875 -0.484375 -0.625 -1.171875q-0.203125 -0.6875 -0.203125 -1.5625q0 -0.765625 0.21875 -1.4375q0.21875 -0.671875 0.625 -1.1875q0.421875 -0.515625 1.03125 -0.8125q0.609375 -0.296875 1.375 -0.296875q0.765625 0 1.34375 0.234375q0.578125 0.234375 0.96875 0.671875q0.40625 0.4375 0.609375 1.0625q0.203125 0.609375 0.203125 1.375zm-1.296875 -0.1875q0.015625 -0.46875 -0.109375 -0.859375q-0.109375 -0.40625 -0.34375 -0.6875q-0.234375 -0.296875 -0.59375 -0.453125q-0.359375 -0.15625 -0.828125 -0.15625q-0.40625 0 -0.75 0.15625q-0.328125 0.15625 -0.578125 0.4375q-0.25 0.28125 -0.40625 0.6875q-0.140625 0.40625 -0.171875 0.875l3.78125 0zm7.901825 4.15625l-0.03125 -0.96875q-0.59375 0.578125 -1.203125 0.84375q-0.59375 0.25 -1.265625 0.25q-0.609375 0 -1.046875 -0.15625q-0.4375 -0.15625 -0.71875 -0.421875q-0.28125 -0.28125 -0.421875 -0.65625q-0.125 -0.375 -0.125 -0.8125q0 -1.078125 0.796875 -1.6875q0.8125 -0.609375 2.390625 -0.609375l1.484375 0l0 -0.640625q0 -0.625 -0.40625 -1.0q-0.40625 -0.390625 -1.25 -0.390625q-0.609375 0 -1.203125 0.140625q-0.578125 0.125 -1.21875 0.375l0 -1.125q0.25 -0.078125 0.53125 -0.15625q0.296875 -0.09375 0.609375 -0.15625q0.328125 -0.0625 0.671875 -0.09375q0.359375 -0.046875 0.71875 -0.046875q0.640625 0 1.15625 0.140625q0.53125 0.140625 0.890625 0.4375q0.359375 0.296875 0.546875 0.75q0.203125 0.4375 0.203125 1.03125l0 4.953125l-1.109375 0zm-0.140625 -3.265625l-1.578125 0q-0.46875 0 -0.8125 0.09375q-0.328125 0.09375 -0.546875 0.265625q-0.21875 0.171875 -0.328125 0.40625q-0.09375 0.234375 -0.09375 0.546875q0 0.203125 0.0625 0.390625q0.0625 0.1875 0.203125 0.34375q0.15625 0.140625 0.375 0.234375q0.234375 0.078125 0.5625 0.078125q0.4375 0 0.984375 -0.265625q0.5625 -0.265625 1.171875 -0.828125l0 -1.265625zm6.026825 -5.828125l-2.125 0l0 -1.015625l3.390625 0l0 9.078125l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -8.0625zm8.058075 0l-2.125 0l0 -1.015625l3.390625 0l0 9.078125l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -8.0625zm11.870575 5.4375q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm8.66745 3.328125q-0.5 0.1875 -1.015625 0.265625q-0.5 0.09375 -1.046875 0.09375q-1.703125 0 -2.625 -0.921875q-0.921875 -0.921875 -0.921875 -2.6875q0 -0.859375 0.265625 -1.546875q0.265625 -0.703125 0.75 -1.1875q0.484375 -0.5 1.140625 -0.765625q0.671875 -0.265625 1.46875 -0.265625q0.5625 0 1.046875 0.078125q0.484375 0.078125 0.9375 0.25l0 1.1875q-0.46875 -0.234375 -0.953125 -0.34375q-0.484375 -0.109375 -1.0 -0.109375q-0.484375 0 -0.90625 0.1875q-0.421875 0.171875 -0.75 0.515625q-0.3125 0.34375 -0.5 0.84375q-0.1875 0.484375 -0.1875 1.109375q0 1.296875 0.625 1.953125q0.640625 0.640625 1.765625 0.640625q0.5 0 0.96875 -0.109375q0.484375 -0.109375 0.9375 -0.34375l0 1.15625zm7.089325 0.265625l-0.03125 -0.96875q-0.59375 0.578125 -1.203125 0.84375q-0.59375 0.25 -1.265625 0.25q-0.609375 0 -1.046875 -0.15625q-0.4375 -0.15625 -0.71875 -0.421875q-0.28125 -0.28125 -0.421875 -0.65625q-0.125 -0.375 -0.125 -0.8125q0 -1.078125 0.796875 -1.6875q0.8125 -0.609375 2.390625 -0.609375l1.484375 0l0 -0.640625q0 -0.625 -0.40625 -1.0q-0.40625 -0.390625 -1.25 -0.390625q-0.609375 0 -1.203125 0.140625q-0.578125 0.125 -1.21875 0.375l0 -1.125q0.25 -0.078125 0.53125 -0.15625q0.296875 -0.09375 0.609375 -0.15625q0.328125 -0.0625 0.671875 -0.09375q0.359375 -0.046875 0.71875 -0.046875q0.640625 0 1.15625 0.140625q0.53125 0.140625 0.890625 0.4375q0.359375 0.296875 0.546875 0.75q0.203125 0.4375 0.203125 1.03125l0 4.953125l-1.109375 0zm-0.140625 -3.265625l-1.578125 0q-0.46875 0 -0.8125 0.09375q-0.328125 0.09375 -0.546875 0.265625q-0.21875 0.171875 -0.328125 0.40625q-0.09375 0.234375 -0.09375 0.546875q0 0.203125 0.0625 0.390625q0.0625 0.1875 0.203125 0.34375q0.15625 0.140625 0.375 0.234375q0.234375 0.078125 0.5625 0.078125q0.4375 0 0.984375 -0.265625q0.5625 -0.265625 1.171875 -0.828125l0 -1.265625zm9.370575 3.171875q-0.421875 0.09375 -0.875 0.140625q-0.453125 0.046875 -0.921875 0.046875q-1.34375 0 -2.015625 -0.609375q-0.65625 -0.609375 -0.65625 -1.875l0 -3.75l-2.015625 0l0 -1.046875l2.015625 0l0 -1.96875l1.234375 -0.328125l0 2.296875l3.234375 0l0 1.046875l-3.234375 0l0 3.65625q0 0.765625 0.40625 1.15625q0.421875 0.375 1.21875 0.375q0.34375 0 0.75 -0.046875q0.40625 -0.0625 0.859375 -0.171875l0 1.078125zm4.714325 -6.0625l-2.125 0l0 -1.03125l3.390625 0l0 6.15625l2.125 0l0 1.03125l-5.75 0l0 -1.03125l2.359375 0l0 -5.125zm0.4375 -4.046875q0.203125 0 0.375 0.078125q0.1875 0.078125 0.3125 0.21875q0.140625 0.125 0.21875 0.3125q0.078125 0.171875 0.078125 0.375q0 0.203125 -0.078125 0.390625q-0.078125 0.171875 -0.21875 0.3125q-0.125 0.125 -0.3125 0.203125q-0.171875 0.078125 -0.375 0.078125q-0.203125 0 -0.390625 -0.078125q-0.171875 -0.078125 -0.3125 -0.203125q-0.125 -0.140625 -0.203125 -0.3125q-0.078125 -0.1875 -0.078125 -0.390625q0 -0.203125 0.078125 -0.375q0.078125 -0.1875 0.203125 -0.3125q0.140625 -0.140625 0.3125 -0.21875q0.1875 -0.078125 0.390625 -0.078125zm11.433075 6.546875q0 0.84375 -0.25 1.546875q-0.234375 0.6875 -0.671875 1.1875q-0.4375 0.5 -1.078125 0.78125q-0.640625 0.265625 -1.453125 0.265625q-0.765625 0 -1.390625 -0.234375q-0.609375 -0.234375 -1.03125 -0.703125q-0.421875 -0.46875 -0.65625 -1.15625q-0.21875 -0.6875 -0.21875 -1.578125q0 -0.84375 0.234375 -1.53125q0.234375 -0.6875 0.671875 -1.1875q0.453125 -0.5 1.09375 -0.765625q0.640625 -0.28125 1.4375 -0.28125q0.78125 0 1.390625 0.25q0.609375 0.234375 1.03125 0.703125q0.4375 0.453125 0.65625 1.140625q0.234375 0.6875 0.234375 1.5625zm-1.28125 0.0625q0 -0.671875 -0.15625 -1.15625q-0.140625 -0.5 -0.421875 -0.828125q-0.265625 -0.34375 -0.65625 -0.5q-0.375 -0.171875 -0.859375 -0.171875q-0.5625 0 -0.96875 0.21875q-0.390625 0.21875 -0.640625 0.578125q-0.25 0.359375 -0.375 0.84375q-0.109375 0.484375 -0.109375 1.015625q0 0.671875 0.140625 1.171875q0.140625 0.5 0.40625 0.828125q0.28125 0.328125 0.671875 0.5q0.390625 0.171875 0.875 0.171875q0.5625 0 0.953125 -0.21875q0.390625 -0.21875 0.640625 -0.578125q0.265625 -0.375 0.375 -0.859375q0.125 -0.484375 0.125 -1.015625zm3.0737 -3.59375l1.109375 0l0.046875 1.15625q0.328125 -0.359375 0.609375 -0.609375q0.296875 -0.25 0.578125 -0.390625q0.28125 -0.15625 0.578125 -0.21875q0.296875 -0.0625 0.609375 -0.0625q1.109375 0 1.671875 0.65625q0.578125 0.65625 0.578125 1.96875l0 4.6875l-1.25 0l0 -4.59375q0 -0.84375 -0.3125 -1.234375q-0.3125 -0.40625 -0.9375 -0.40625q-0.234375 0 -0.453125 0.0625q-0.21875 0.0625 -0.453125 0.234375q-0.234375 0.171875 -0.515625 0.453125q-0.265625 0.28125 -0.609375 0.71875l0 4.765625l-1.25 0l0 -7.1875z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m162.69989 484.61154l316.7874 30.11026" fill-rule="evenodd"/><path stroke="#595959" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m162.69989 484.61154l316.7874 30.11026" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m163.29161 514.21515l314.39368 -27.559082" fill-rule="evenodd"/><path stroke="#595959" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m163.29161 514.21515l314.39368 -27.559082" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m77.98062 65.06823l0 59.590553" fill-rule="evenodd"/><path stroke="#595959" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m77.98062 65.06823l0 53.590553" fill-rule="evenodd"/><path fill="#595959" stroke="#595959" stroke-width="1.0" stroke-linecap="butt" d="m76.32889 118.65878l1.6517334 4.5380936l1.6517334 -4.5380936z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m78.50435 160.0866l0 156.88188" fill-rule="evenodd"/><path stroke="#595959" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m78.50435 160.08658l0 150.8819" fill-rule="evenodd"/><path fill="#595959" stroke="#595959" stroke-width="1.0" stroke-linecap="butt" d="m76.852615 310.96848l1.6517334 4.5381165l1.6517334 -4.5381165z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m81.09268 363.24664l0 104.787415" fill-rule="evenodd"/><path stroke="#595959" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m81.09268 363.24664l0 98.787384" fill-rule="evenodd"/><path fill="#595959" stroke="#595959" stroke-width="1.0" stroke-linecap="butt" d="m79.44095 462.03403l1.6517334 4.5381165l1.6517334 -4.5381165z" fill-rule="evenodd"/><path fill="#ea4335" d="m163.3038 71.25196l315.59055 0l0 29.039368l-315.59055 0z" fill-rule="evenodd"/><path stroke="#ea4335" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m163.3038 71.25196l315.59055 0l0 29.039368l-315.59055 0z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m224.40218 79.966805l193.39381 0l0 17.16098l-193.39381 0l0 -17.16098z" fill-rule="nonzero"/><path fill="#ffffff" d="m226.19905 87.72665l0 -1.546875l4.453125 0l0 1.546875l-4.453125 0zm13.589325 -1.0625q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm5.85495 -4.3125l-2.0625 0l0 -1.359375l3.84375 0l0 8.75l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -7.390625zm9.995575 -0.515625q0 0.234375 -0.09375 0.453125q-0.078125 0.203125 -0.234375 0.359375q-0.15625 0.15625 -0.375 0.25q-0.203125 0.078125 -0.4375 0.078125q-0.25 0 -0.46875 -0.078125q-0.203125 -0.09375 -0.359375 -0.25q-0.15625 -0.15625 -0.25 -0.359375q-0.078125 -0.21875 -0.078125 -0.453125q0 -0.234375 0.078125 -0.4375q0.09375 -0.203125 0.25 -0.359375q0.15625 -0.15625 0.359375 -0.25q0.21875 -0.09375 0.46875 -0.09375q0.234375 0 0.4375 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.234375 0.359375q0.09375 0.203125 0.09375 0.4375zm-1.9375 3.359375l-2.0625 0l0 -1.359375l3.84375 0l0 5.90625l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -4.546875zm10.83934 5.90625l0 -4.953125q0 -0.265625 -0.015625 -0.46875q-0.015625 -0.203125 -0.0625 -0.34375q-0.03125 -0.140625 -0.125 -0.203125q-0.078125 -0.078125 -0.21875 -0.078125q-0.21875 0 -0.4375 0.3125q-0.21875 0.296875 -0.484375 1.015625l0 4.71875l-1.5 0l0 -4.953125q0 -0.265625 -0.015625 -0.46875q-0.015625 -0.203125 -0.0625 -0.34375q-0.03125 -0.140625 -0.125 -0.203125q-0.078125 -0.078125 -0.21875 -0.078125q-0.1875 0 -0.421875 0.3125q-0.21875 0.296875 -0.5 1.015625l0 4.71875l-1.515625 0l0 -7.265625l1.296875 0l0.03125 1.046875q0.15625 -0.3125 0.3125 -0.53125q0.15625 -0.234375 0.328125 -0.375q0.1875 -0.15625 0.390625 -0.21875q0.203125 -0.0625 0.453125 -0.0625q0.609375 0 0.921875 0.3125q0.328125 0.3125 0.375 0.875q0.171875 -0.3125 0.328125 -0.53125q0.171875 -0.234375 0.359375 -0.375q0.1875 -0.15625 0.390625 -0.21875q0.21875 -0.0625 0.46875 -0.0625q0.796875 0 1.171875 0.546875q0.390625 0.546875 0.390625 1.640625l0 5.21875l-1.515625 0zm7.214325 -9.265625q0 0.234375 -0.09375 0.453125q-0.078125 0.203125 -0.234375 0.359375q-0.15625 0.15625 -0.375 0.25q-0.203125 0.078125 -0.4375 0.078125q-0.25 0 -0.46875 -0.078125q-0.203125 -0.09375 -0.359375 -0.25q-0.15625 -0.15625 -0.25 -0.359375q-0.078125 -0.21875 -0.078125 -0.453125q0 -0.234375 0.078125 -0.4375q0.09375 -0.203125 0.25 -0.359375q0.15625 -0.15625 0.359375 -0.25q0.21875 -0.09375 0.46875 -0.09375q0.234375 0 0.4375 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.234375 0.359375q0.09375 0.203125 0.09375 0.4375zm-1.9375 3.359375l-2.0625 0l0 -1.359375l3.84375 0l0 5.90625l2.109375 0l0 1.359375l-6.203125 0l0 -1.359375l2.3125 0l0 -4.546875zm10.04245 5.90625l0 -4.734375q0 -1.203125 -0.890625 -1.203125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 4.59375l-1.734375 0l0 -7.265625l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.546875 0 0.953125 0.1875q0.421875 0.171875 0.703125 0.515625q0.28125 0.328125 0.421875 0.796875q0.140625 0.46875 0.140625 1.046875l0 4.875l-1.75 0zm8.276825 0l-0.03125 -0.953125q-0.234375 0.25 -0.484375 0.453125q-0.25 0.203125 -0.5625 0.359375q-0.296875 0.140625 -0.65625 0.21875q-0.34375 0.09375 -0.765625 0.09375q-0.5625 0 -0.984375 -0.171875q-0.421875 -0.171875 -0.703125 -0.453125q-0.28125 -0.296875 -0.4375 -0.703125q-0.140625 -0.421875 -0.140625 -0.921875q0 -0.515625 0.21875 -0.953125q0.21875 -0.4375 0.65625 -0.75q0.453125 -0.3125 1.125 -0.484375q0.671875 -0.1875 1.5625 -0.1875l0.953125 0l0 -0.4375q0 -0.28125 -0.078125 -0.5q-0.078125 -0.234375 -0.25 -0.390625q-0.171875 -0.15625 -0.453125 -0.234375q-0.28125 -0.09375 -0.6875 -0.09375q-0.640625 0 -1.265625 0.15625q-0.625 0.140625 -1.21875 0.40625l0 -1.40625q0.53125 -0.203125 1.203125 -0.328125q0.6875 -0.140625 1.421875 -0.140625q0.8125 0 1.390625 0.15625q0.578125 0.140625 0.953125 0.453125q0.375 0.3125 0.546875 0.78125q0.1875 0.453125 0.1875 1.0625l0 4.96875l-1.5 0zm-0.25 -3.21875l-1.0625 0q-0.4375 0 -0.75 0.09375q-0.3125 0.078125 -0.5 0.234375q-0.1875 0.15625 -0.28125 0.359375q-0.09375 0.1875 -0.09375 0.40625q0 0.4375 0.28125 0.671875q0.296875 0.234375 0.78125 0.234375q0.375 0 0.765625 -0.265625q0.390625 -0.265625 0.859375 -0.75l0 -0.984375zm9.745575 3.140625q-0.46875 0.109375 -0.953125 0.171875q-0.46875 0.078125 -0.90625 0.078125q-0.703125 0 -1.234375 -0.15625q-0.515625 -0.15625 -0.859375 -0.453125q-0.328125 -0.3125 -0.5 -0.78125q-0.15625 -0.484375 -0.15625 -1.140625l0 -3.546875l-1.953125 0l0 -1.359375l1.953125 0l0 -1.859375l1.796875 -0.46875l0 2.328125l2.8125 0l0 1.359375l-2.8125 0l0 3.421875q0 0.609375 0.28125 0.9375q0.28125 0.3125 0.953125 0.3125q0.4375 0 0.84375 -0.0625q0.40625 -0.078125 0.734375 -0.171875l0 1.390625zm8.401825 -4.109375q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm4.308075 1.3125l0 -1.546875l4.453125 0l0 1.546875l-4.453125 0zm13.589325 -1.0625q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm8.6362 4.4375l0 -4.953125q0 -0.265625 -0.015625 -0.46875q-0.015625 -0.203125 -0.0625 -0.34375q-0.03125 -0.140625 -0.125 -0.203125q-0.078125 -0.078125 -0.21875 -0.078125q-0.21875 0 -0.4375 0.3125q-0.21875 0.296875 -0.484375 1.015625l0 4.71875l-1.5 0l0 -4.953125q0 -0.265625 -0.015625 -0.46875q-0.015625 -0.203125 -0.0625 -0.34375q-0.03125 -0.140625 -0.125 -0.203125q-0.078125 -0.078125 -0.21875 -0.078125q-0.1875 0 -0.421875 0.3125q-0.21875 0.296875 -0.5 1.015625l0 4.71875l-1.515625 0l0 -7.265625l1.296875 0l0.03125 1.046875q0.15625 -0.3125 0.3125 -0.53125q0.15625 -0.234375 0.328125 -0.375q0.1875 -0.15625 0.390625 -0.21875q0.203125 -0.0625 0.453125 -0.0625q0.609375 0 0.921875 0.3125q0.328125 0.3125 0.375 0.875q0.171875 -0.3125 0.328125 -0.53125q0.171875 -0.234375 0.359375 -0.375q0.1875 -0.15625 0.390625 -0.21875q0.21875 -0.0625 0.46875 -0.0625q0.796875 0 1.171875 0.546875q0.390625 0.546875 0.390625 1.640625l0 5.21875l-1.515625 0zm9.464325 -3.765625q0 1.0 -0.28125 1.734375q-0.28125 0.734375 -0.78125 1.21875q-0.5 0.46875 -1.1875 0.703125q-0.6875 0.234375 -1.5 0.234375q-0.28125 0 -0.546875 -0.03125q-0.265625 -0.03125 -0.5 -0.09375l0 2.84375l-1.734375 0l0 -10.109375l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.640625 0 1.140625 0.265625q0.5 0.25 0.828125 0.71875q0.34375 0.46875 0.515625 1.15625q0.1875 0.671875 0.1875 1.515625zm-1.828125 0.078125q0 -0.609375 -0.09375 -1.03125q-0.078125 -0.421875 -0.25 -0.6875q-0.15625 -0.28125 -0.390625 -0.40625q-0.21875 -0.125 -0.53125 -0.125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 3.171875q0.21875 0.078125 0.515625 0.125q0.3125 0.046875 0.625 0.046875q0.40625 0 0.75 -0.171875q0.34375 -0.171875 0.578125 -0.484375q0.25 -0.3125 0.375 -0.765625q0.125 -0.453125 0.125 -1.015625zm9.339325 3.609375q-0.46875 0.109375 -0.953125 0.171875q-0.46875 0.078125 -0.90625 0.078125q-0.703125 0 -1.234375 -0.15625q-0.515625 -0.15625 -0.859375 -0.453125q-0.328125 -0.3125 -0.5 -0.78125q-0.15625 -0.484375 -0.15625 -1.140625l0 -3.546875l-1.953125 0l0 -1.359375l1.953125 0l0 -1.859375l1.796875 -0.46875l0 2.328125l2.8125 0l0 1.359375l-2.8125 0l0 3.421875q0 0.609375 0.28125 0.9375q0.28125 0.3125 0.953125 0.3125q0.4375 0 0.84375 -0.0625q0.40625 -0.078125 0.734375 -0.171875l0 1.390625zm6.4487 -0.71875q-0.390625 0.953125 -0.796875 1.65625q-0.40625 0.71875 -0.90625 1.1875q-0.5 0.484375 -1.109375 0.71875q-0.59375 0.234375 -1.34375 0.234375q-0.1875 0 -0.40625 -0.015625q-0.203125 -0.015625 -0.421875 -0.046875l0 -1.484375q0.09375 0.015625 0.203125 0.03125q0.109375 0.015625 0.21875 0.03125q0.125 0.015625 0.25 0.015625q0.125 0 0.234375 0q0.3125 0 0.5625 -0.109375q0.265625 -0.109375 0.484375 -0.3125q0.21875 -0.203125 0.390625 -0.484375q0.171875 -0.28125 0.3125 -0.625l-2.875 -7.265625l1.9375 0l1.515625 4.109375l0.453125 1.34375l0.4375 -1.28125l1.53125 -4.171875l1.875 0l-2.546875 6.46875zm4.47995 -2.328125l0 -1.546875l4.453125 0l0 1.546875l-4.453125 0zm13.245575 3.046875q-0.46875 0.109375 -0.953125 0.171875q-0.46875 0.078125 -0.90625 0.078125q-0.703125 0 -1.234375 -0.15625q-0.515625 -0.15625 -0.859375 -0.453125q-0.328125 -0.3125 -0.5 -0.78125q-0.15625 -0.484375 -0.15625 -1.140625l0 -3.546875l-1.953125 0l0 -1.359375l1.953125 0l0 -1.859375l1.796875 -0.46875l0 2.328125l2.8125 0l0 1.359375l-2.8125 0l0 3.421875q0 0.609375 0.28125 0.9375q0.28125 0.3125 0.953125 0.3125q0.4375 0 0.84375 -0.0625q0.40625 -0.078125 0.734375 -0.171875l0 1.390625zm8.401825 -4.109375q0 0.171875 -0.015625 0.46875q0 0.28125 -0.03125 0.53125l-4.78125 0q0 0.46875 0.140625 0.84375q0.15625 0.359375 0.421875 0.609375q0.265625 0.25 0.640625 0.375q0.390625 0.125 0.84375 0.125q0.53125 0 1.125 -0.078125q0.609375 -0.078125 1.265625 -0.265625l0 1.390625q-0.28125 0.078125 -0.609375 0.140625q-0.328125 0.0625 -0.671875 0.109375q-0.34375 0.046875 -0.703125 0.078125q-0.34375 0.03125 -0.671875 0.03125q-0.828125 0 -1.5 -0.234375q-0.65625 -0.25 -1.109375 -0.703125q-0.453125 -0.46875 -0.703125 -1.140625q-0.234375 -0.6875 -0.234375 -1.578125q0 -0.890625 0.234375 -1.609375q0.25 -0.734375 0.6875 -1.25q0.453125 -0.515625 1.078125 -0.796875q0.640625 -0.28125 1.421875 -0.28125q0.765625 0 1.359375 0.234375q0.59375 0.234375 0.984375 0.671875q0.40625 0.421875 0.609375 1.015625q0.21875 0.59375 0.21875 1.3125zm-1.78125 -0.25q0.015625 -0.4375 -0.109375 -0.75q-0.109375 -0.328125 -0.3125 -0.53125q-0.1875 -0.203125 -0.453125 -0.296875q-0.25 -0.109375 -0.5625 -0.109375q-0.65625 0 -1.078125 0.4375q-0.421875 0.4375 -0.5 1.25l3.015625 0zm7.839325 4.4375l0 -4.734375q0 -1.203125 -0.890625 -1.203125q-0.4375 0 -0.84375 0.359375q-0.40625 0.359375 -0.859375 0.984375l0 4.59375l-1.734375 0l0 -7.265625l1.5 0l0.046875 1.0625q0.21875 -0.265625 0.453125 -0.5q0.25 -0.234375 0.53125 -0.390625q0.28125 -0.15625 0.59375 -0.234375q0.328125 -0.09375 0.734375 -0.09375q0.546875 0 0.953125 0.1875q0.421875 0.171875 0.703125 0.515625q0.28125 0.328125 0.421875 0.796875q0.140625 0.46875 0.140625 1.046875l0 4.875l-1.75 0zm9.714325 -2.140625q0 0.625 -0.28125 1.0625q-0.265625 0.4375 -0.71875 0.71875q-0.453125 0.28125 -1.03125 0.40625q-0.578125 0.125 -1.171875 0.125q-0.796875 0 -1.453125 -0.078125q-0.640625 -0.078125 -1.21875 -0.21875l0 -1.578125q0.6875 0.28125 1.34375 0.40625q0.671875 0.125 1.265625 0.125q0.6875 0 1.03125 -0.21875q0.34375 -0.21875 0.34375 -0.5625q0 -0.171875 -0.078125 -0.296875q-0.078125 -0.140625 -0.28125 -0.265625q-0.1875 -0.125 -0.546875 -0.25q-0.359375 -0.140625 -0.9375 -0.3125q-0.53125 -0.15625 -0.9375 -0.34375q-0.40625 -0.1875 -0.671875 -0.4375q-0.265625 -0.265625 -0.40625 -0.59375q-0.125 -0.34375 -0.125 -0.8125q0 -0.4375 0.203125 -0.828125q0.203125 -0.390625 0.59375 -0.6875q0.40625 -0.296875 1.0 -0.46875q0.59375 -0.171875 1.390625 -0.171875q0.6875 0 1.21875 0.078125q0.53125 0.0625 0.9375 0.140625l0 1.421875q-0.625 -0.1875 -1.171875 -0.265625q-0.546875 -0.09375 -1.09375 -0.09375q-0.53125 0 -0.859375 0.203125q-0.328125 0.1875 -0.328125 0.53125q0 0.15625 0.0625 0.296875q0.078125 0.125 0.265625 0.25q0.1875 0.109375 0.515625 0.25q0.34375 0.125 0.90625 0.28125q0.625 0.1875 1.046875 0.390625q0.4375 0.203125 0.703125 0.46875q0.265625 0.25 0.375 0.578125q0.109375 0.328125 0.109375 0.75zm8.620575 -1.546875q0 0.859375 -0.25 1.578125q-0.234375 0.703125 -0.703125 1.21875q-0.453125 0.5 -1.125 0.78125q-0.65625 0.28125 -1.515625 0.28125q-0.796875 0 -1.4375 -0.234375q-0.640625 -0.25 -1.09375 -0.71875q-0.4375 -0.46875 -0.671875 -1.171875q-0.234375 -0.703125 -0.234375 -1.640625q0 -0.859375 0.25 -1.5625q0.25 -0.71875 0.703125 -1.21875q0.46875 -0.5 1.125 -0.765625q0.671875 -0.28125 1.5 -0.28125q0.828125 0 1.453125 0.234375q0.640625 0.234375 1.078125 0.71875q0.453125 0.46875 0.6875 1.171875q0.234375 0.6875 0.234375 1.609375zm-1.828125 0.03125q0 -1.125 -0.421875 -1.6875q-0.421875 -0.578125 -1.25 -0.578125q-0.46875 0 -0.796875 0.1875q-0.3125 0.171875 -0.515625 0.484375q-0.203125 0.3125 -0.3125 0.734375q-0.09375 0.40625 -0.09375 0.875q0 1.140625 0.453125 1.71875q0.46875 0.578125 1.265625 0.578125q0.4375 0 0.75 -0.171875q0.328125 -0.1875 0.53125 -0.484375q0.203125 -0.3125 0.296875 -0.734375q0.09375 -0.421875 0.09375 -0.921875zm8.0112 -0.78125q0.015625 -0.421875 -0.046875 -0.703125q-0.046875 -0.28125 -0.171875 -0.46875q-0.109375 -0.1875 -0.28125 -0.265625q-0.171875 -0.09375 -0.390625 -0.09375q-0.390625 0 -0.796875 0.328125q-0.40625 0.3125 -0.90625 1.046875l0 4.59375l-1.796875 0l0 -7.265625l1.59375 0l0.0625 1.046875q0.171875 -0.265625 0.390625 -0.484375q0.234375 -0.234375 0.5 -0.390625q0.28125 -0.15625 0.609375 -0.234375q0.34375 -0.09375 0.75 -0.09375q0.546875 0 0.96875 0.1875q0.4375 0.1875 0.71875 0.5625q0.296875 0.359375 0.4375 0.921875q0.140625 0.5625 0.109375 1.3125l-1.75 0zm9.370575 2.296875q0 0.625 -0.28125 1.0625q-0.265625 0.4375 -0.71875 0.71875q-0.453125 0.28125 -1.03125 0.40625q-0.578125 0.125 -1.171875 0.125q-0.796875 0 -1.453125 -0.078125q-0.640625 -0.078125 -1.21875 -0.21875l0 -1.578125q0.6875 0.28125 1.34375 0.40625q0.671875 0.125 1.265625 0.125q0.6875 0 1.03125 -0.21875q0.34375 -0.21875 0.34375 -0.5625q0 -0.171875 -0.078125 -0.296875q-0.078125 -0.140625 -0.28125 -0.265625q-0.1875 -0.125 -0.546875 -0.25q-0.359375 -0.140625 -0.9375 -0.3125q-0.53125 -0.15625 -0.9375 -0.34375q-0.40625 -0.1875 -0.671875 -0.4375q-0.265625 -0.265625 -0.40625 -0.59375q-0.125 -0.34375 -0.125 -0.8125q0 -0.4375 0.203125 -0.828125q0.203125 -0.390625 0.59375 -0.6875q0.40625 -0.296875 1.0 -0.46875q0.59375 -0.171875 1.390625 -0.171875q0.6875 0 1.21875 0.078125q0.53125 0.0625 0.9375 0.140625l0 1.421875q-0.625 -0.1875 -1.171875 -0.265625q-0.546875 -0.09375 -1.09375 -0.09375q-0.53125 0 -0.859375 0.203125q-0.328125 0.1875 -0.328125 0.53125q0 0.15625 0.0625 0.296875q0.078125 0.125 0.265625 0.25q0.1875 0.109375 0.515625 0.25q0.34375 0.125 0.90625 0.28125q0.625 0.1875 1.046875 0.390625q0.4375 0.203125 0.703125 0.46875q0.265625 0.25 0.375 0.578125q0.109375 0.328125 0.109375 0.75z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m417.796 79.966805l0 0l0 17.16098l0 0l0 -17.16098z" fill-rule="nonzero"/><path fill="#ffffff" d="m224.40218 91.80498l193.39381 0l0 1.3199997l-193.39381 0l0 -1.3199997z" fill-rule="nonzero"/></g></svg>
\ No newline at end of file
diff --git a/mlir/docs/includes/img/bufferization_tensor_insert_dst.svg b/mlir/docs/includes/img/bufferization_tensor_insert_dst.svg
new file mode 100644
index 000000000000..228b19e92299
--- /dev/null
+++ b/mlir/docs/includes/img/bufferization_tensor_insert_dst.svg
@@ -0,0 +1 @@
+<svg version="1.1" viewBox="0.0 0.0 624.0 192.0" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l624.0 0l0 192.0l-624.0 0l0 -192.0z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l624.0 0l0 192.0l-624.0 0z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m81.125984 50.27559l681.22833 0l0 73.732285l-681.22833 0z" fill-rule="evenodd"/><path fill="#616161" d="m100.20411 64.32059l-8.40625 12.875l-1.484375 0l8.375 -12.875l1.515625 0zm-4.859375 2.546875q0 0.59375 -0.171875 1.109375q-0.171875 0.5 -0.5 0.875q-0.3125 0.359375 -0.78125 0.578125q-0.453125 0.21875 -1.03125 0.21875q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.515625q-0.3125 -0.34375 -0.484375 -0.84375q-0.15625 -0.5 -0.15625 -1.15625q0 -0.59375 0.171875 -1.09375q0.171875 -0.5 0.484375 -0.875q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.21875 1.03125 -0.21875q0.5625 0 1.0 0.171875q0.4375 0.171875 0.75 0.515625q0.3125 0.328125 0.484375 0.828125q0.171875 0.5 0.171875 1.15625zm-1.34375 0.046875q0 -0.796875 -0.296875 -1.171875q-0.28125 -0.390625 -0.796875 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.34375 0.328125q-0.140625 0.203125 -0.21875 0.5q-0.078125 0.28125 -0.078125 0.609375q0 0.8125 0.296875 1.203125q0.296875 0.375 0.8125 0.375q0.265625 0 0.46875 -0.125q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.203125 0.203125 -0.484375q0.078125 -0.296875 0.078125 -0.640625zm6.046875 7.65625q0 0.578125 -0.171875 1.09375q-0.171875 0.5 -0.5 0.875q-0.328125 0.359375 -0.796875 0.578125q-0.453125 0.203125 -1.015625 0.203125q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.5q-0.3125 -0.34375 -0.484375 -0.84375q-0.171875 -0.515625 -0.171875 -1.171875q0 -0.578125 0.171875 -1.078125q0.1875 -0.515625 0.5 -0.890625q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.203125 1.03125 -0.203125q0.5625 0 1.0 0.171875q0.453125 0.171875 0.765625 0.515625q0.3125 0.328125 0.46875 0.84375q0.171875 0.5 0.171875 1.15625zm-1.328125 0.03125q0 -0.796875 -0.296875 -1.171875q-0.296875 -0.390625 -0.8125 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.359375 0.328125q-0.140625 0.203125 -0.21875 0.484375q-0.0625 0.28125 -0.0625 0.625q0 0.796875 0.28125 1.1875q0.296875 0.375 0.828125 0.375q0.265625 0 0.46875 -0.109375q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.21875 0.21875 -0.5q0.078125 -0.296875 0.078125 -0.625z" fill-rule="nonzero"/><path fill="#c53929" d="m102.2114 68.05496l1.453125 0l0.046875 1.6875q0.8125 -0.984375 1.59375 -1.421875q0.796875 -0.4375 1.59375 -0.4375q1.421875 0 2.15625 0.921875q0.734375 0.921875 0.671875 2.734375l-1.59375 0q0.015625 -1.203125 -0.359375 -1.734375q-0.375 -0.546875 -1.109375 -0.546875q-0.3125 0 -0.640625 0.109375q-0.328125 0.109375 -0.671875 0.359375q-0.328125 0.25 -0.71875 0.640625q-0.375 0.390625 -0.8125 0.953125l0 5.875l-1.609375 0l0 -9.140625z" fill-rule="nonzero"/><path fill="#616161" d="m129.94475 71.30496l-7.8281326 0l0 -1.328125l7.8281326 0l0 1.328125zm0 3.171875l-7.8281326 0l0 -1.328125l7.8281326 0l0 1.328125z" fill-rule="nonzero"/><path fill="#000000" d="m150.24059 77.07059q-0.53125 0.140625 -1.109375 0.1875q-0.578125 0.0625 -1.171875 0.0625q-1.71875 0 -2.5625 -0.78125q-0.84375 -0.78125 -0.84375 -2.390625l0 -4.765625l-2.5625 0l0 -1.328125l2.5625 0l0 -2.515625l1.578125 -0.40625l0 2.921875l4.109375 0l0 1.328125l-4.109375 0l0 4.640625q0 0.984375 0.515625 1.46875q0.53125 0.484375 1.546875 0.484375q0.4375 0 0.953125 -0.0625q0.53125 -0.0625 1.09375 -0.21875l0 1.375zm10.616669 -4.9375q0 0.34375 -0.015625 0.578125q0 0.21875 -0.03125 0.421875l-6.421875 0q0 1.40625 0.78125 2.15625q0.796875 0.75 2.265625 0.75q0.40625 0 0.796875 -0.03125q0.40625 -0.03125 0.78125 -0.078125q0.375 -0.0625 0.71875 -0.125q0.34375 -0.078125 0.625 -0.15625l0 1.296875q-0.640625 0.1875 -1.46875 0.296875q-0.8125 0.109375 -1.6875 0.109375q-1.171875 0 -2.015625 -0.3125q-0.84375 -0.3125 -1.390625 -0.921875q-0.546875 -0.609375 -0.8125 -1.484375q-0.25 -0.875 -0.25 -2.0q0 -0.953125 0.28125 -1.8125q0.28125 -0.859375 0.8125 -1.515625q0.53125 -0.65625 1.296875 -1.03125q0.78125 -0.390625 1.765625 -0.390625q0.953125 0 1.6875 0.3125q0.75 0.296875 1.25 0.84375q0.5 0.546875 0.765625 1.34375q0.265625 0.78125 0.265625 1.75zm-1.65625 -0.21875q0.03125 -0.609375 -0.125 -1.109375q-0.140625 -0.515625 -0.453125 -0.875q-0.296875 -0.375 -0.75 -0.578125q-0.453125 -0.203125 -1.0625 -0.203125q-0.515625 0 -0.953125 0.203125q-0.421875 0.203125 -0.734375 0.5625q-0.296875 0.359375 -0.5 0.875q-0.1875 0.515625 -0.234375 1.125l4.8125 0zm4.1791687 -3.859375l1.421875 0l0.0625 1.46875q0.390625 -0.46875 0.765625 -0.78125q0.375 -0.3125 0.734375 -0.5q0.359375 -0.203125 0.734375 -0.28125q0.375 -0.078125 0.78125 -0.078125q1.40625 0 2.125 0.84375q0.734375 0.828125 0.734375 2.5l0 5.96875l-1.59375 0l0 -5.84375q0 -1.078125 -0.40625 -1.578125q-0.390625 -0.515625 -1.1875 -0.515625q-0.28125 0 -0.5625 0.09375q-0.28125 0.078125 -0.578125 0.296875q-0.296875 0.203125 -0.65625 0.578125q-0.34375 0.359375 -0.78125 0.90625l0 6.0625l-1.59375 0l0 -9.140625zm17.507294 6.640625q0 0.484375 -0.171875 0.875q-0.15625 0.375 -0.4375 0.671875q-0.28125 0.296875 -0.65625 0.515625q-0.375 0.203125 -0.796875 0.34375q-0.421875 0.125 -0.875 0.1875q-0.453125 0.0625 -0.875 0.0625q-0.921875 0 -1.703125 -0.078125q-0.78125 -0.078125 -1.53125 -0.265625l0 -1.453125q0.796875 0.234375 1.59375 0.359375q0.796875 0.109375 1.578125 0.109375q1.140625 0 1.6875 -0.3125q0.546875 -0.3125 0.546875 -0.890625q0 -0.234375 -0.09375 -0.421875q-0.078125 -0.203125 -0.3125 -0.375q-0.21875 -0.1875 -0.703125 -0.375q-0.484375 -0.203125 -1.3125 -0.4375q-0.609375 -0.1875 -1.140625 -0.421875q-0.515625 -0.234375 -0.90625 -0.546875q-0.375 -0.328125 -0.59375 -0.75q-0.21875 -0.421875 -0.21875 -1.015625q0 -0.375 0.171875 -0.828125q0.171875 -0.453125 0.59375 -0.84375q0.4375 -0.40625 1.171875 -0.65625q0.734375 -0.265625 1.8125 -0.265625q0.546875 0 1.203125 0.0625q0.65625 0.0625 1.359375 0.203125l0 1.40625q-0.75 -0.171875 -1.421875 -0.25q-0.65625 -0.09375 -1.15625 -0.09375q-0.59375 0 -1.0 0.09375q-0.390625 0.078125 -0.65625 0.25q-0.25 0.15625 -0.359375 0.375q-0.109375 0.203125 -0.109375 0.453125q0 0.25 0.09375 0.453125q0.109375 0.1875 0.359375 0.375q0.265625 0.1875 0.71875 0.375q0.46875 0.1875 1.21875 0.40625q0.8125 0.234375 1.359375 0.5q0.5625 0.25 0.90625 0.578125q0.359375 0.3125 0.5 0.71875q0.15625 0.40625 0.15625 0.90625zm10.976044 -2.140625q0 1.0625 -0.3125 1.953125q-0.296875 0.890625 -0.859375 1.53125q-0.5625 0.625 -1.375 0.96875q-0.8125 0.34375 -1.84375 0.34375q-0.984375 0 -1.765625 -0.296875q-0.765625 -0.3125 -1.3125 -0.890625q-0.546875 -0.59375 -0.828125 -1.46875q-0.28125 -0.875 -0.28125 -2.015625q0 -1.0625 0.296875 -1.9375q0.296875 -0.890625 0.859375 -1.515625q0.5625 -0.640625 1.375 -0.984375q0.8125 -0.359375 1.84375 -0.359375q0.984375 0 1.765625 0.3125q0.78125 0.296875 1.3125 0.890625q0.546875 0.578125 0.828125 1.453125q0.296875 0.875 0.296875 2.015625zm-1.625 0.0625q0 -0.84375 -0.1875 -1.46875q-0.1875 -0.640625 -0.53125 -1.0625q-0.34375 -0.421875 -0.84375 -0.640625q-0.5 -0.21875 -1.109375 -0.21875q-0.703125 0 -1.21875 0.28125q-0.5 0.28125 -0.828125 0.75q-0.3125 0.453125 -0.46875 1.078125q-0.15625 0.609375 -0.15625 1.28125q0 0.859375 0.1875 1.5q0.1875 0.625 0.53125 1.046875q0.359375 0.421875 0.84375 0.640625q0.5 0.21875 1.109375 0.21875q0.71875 0 1.21875 -0.28125q0.5 -0.28125 0.828125 -0.734375q0.328125 -0.46875 0.46875 -1.078125q0.15625 -0.625 0.15625 -1.3125zm4.2885437 -4.5625l1.453125 0l0.046875 1.6875q0.8125 -0.984375 1.59375 -1.421875q0.796875 -0.4375 1.59375 -0.4375q1.421875 0 2.15625 0.921875q0.734375 0.921875 0.671875 2.734375l-1.59375 0q0.015625 -1.203125 -0.359375 -1.734375q-0.375 -0.546875 -1.109375 -0.546875q-0.3125 0 -0.640625 0.109375q-0.328125 0.109375 -0.671875 0.359375q-0.328125 0.25 -0.71875 0.640625q-0.375 0.390625 -0.8125 0.953125l0 5.875l-1.609375 0l0 -9.140625z" fill-rule="nonzero"/><path fill="#616161" d="m208.01872 74.32059q0.296875 0 0.578125 0.125q0.28125 0.125 0.484375 0.328125q0.203125 0.203125 0.3125 0.484375q0.125 0.28125 0.125 0.59375q0 0.3125 -0.125 0.59375q-0.109375 0.265625 -0.3125 0.46875q-0.203125 0.203125 -0.484375 0.3125q-0.28125 0.125 -0.578125 0.125q-0.328125 0 -0.609375 -0.125q-0.265625 -0.109375 -0.46875 -0.3125q-0.203125 -0.203125 -0.328125 -0.46875q-0.109375 -0.28125 -0.109375 -0.59375q0 -0.3125 0.109375 -0.59375q0.125 -0.28125 0.328125 -0.484375q0.203125 -0.203125 0.46875 -0.328125q0.28125 -0.125 0.609375 -0.125z" fill-rule="nonzero"/><path fill="#000000" d="m217.79164 69.36746l-2.703125 0l0 -1.3125l4.296875 0l0 7.8125l2.734375 0l0 1.328125l-7.34375 0l0 -1.328125l3.015625 0l0 -6.5zm0.546875 -5.15625q0.265625 0 0.484375 0.09375q0.234375 0.09375 0.40625 0.28125q0.171875 0.171875 0.265625 0.390625q0.09375 0.21875 0.09375 0.484375q0 0.25 -0.09375 0.484375q-0.09375 0.234375 -0.265625 0.40625q-0.171875 0.171875 -0.40625 0.265625q-0.21875 0.09375 -0.484375 0.09375q-0.265625 0 -0.5 -0.09375q-0.21875 -0.09375 -0.390625 -0.265625q-0.15625 -0.171875 -0.265625 -0.40625q-0.09375 -0.234375 -0.09375 -0.484375q0 -0.265625 0.09375 -0.484375q0.109375 -0.21875 0.265625 -0.390625q0.171875 -0.1875 0.390625 -0.28125q0.234375 -0.09375 0.5 -0.09375zm6.5854187 3.84375l1.421875 0l0.0625 1.46875q0.390625 -0.46875 0.765625 -0.78125q0.375 -0.3125 0.734375 -0.5q0.359375 -0.203125 0.734375 -0.28125q0.375 -0.078125 0.78125 -0.078125q1.40625 0 2.125 0.84375q0.734375 0.828125 0.734375 2.5l0 5.96875l-1.59375 0l0 -5.84375q0 -1.078125 -0.40625 -1.578125q-0.390625 -0.515625 -1.1875 -0.515625q-0.28125 0 -0.5625 0.09375q-0.28125 0.078125 -0.578125 0.296875q-0.296875 0.203125 -0.65625 0.578125q-0.34375 0.359375 -0.78125 0.90625l0 6.0625l-1.59375 0l0 -9.140625zm17.507294 6.640625q0 0.484375 -0.171875 0.875q-0.15625 0.375 -0.4375 0.671875q-0.28125 0.296875 -0.65625 0.515625q-0.375 0.203125 -0.796875 0.34375q-0.421875 0.125 -0.875 0.1875q-0.453125 0.0625 -0.875 0.0625q-0.921875 0 -1.703125 -0.078125q-0.78125 -0.078125 -1.53125 -0.265625l0 -1.453125q0.796875 0.234375 1.59375 0.359375q0.796875 0.109375 1.578125 0.109375q1.140625 0 1.6875 -0.3125q0.546875 -0.3125 0.546875 -0.890625q0 -0.234375 -0.09375 -0.421875q-0.078125 -0.203125 -0.3125 -0.375q-0.21875 -0.1875 -0.703125 -0.375q-0.484375 -0.203125 -1.3125 -0.4375q-0.609375 -0.1875 -1.140625 -0.421875q-0.515625 -0.234375 -0.90625 -0.546875q-0.375 -0.328125 -0.59375 -0.75q-0.21875 -0.421875 -0.21875 -1.015625q0 -0.375 0.171875 -0.828125q0.171875 -0.453125 0.59375 -0.84375q0.4375 -0.40625 1.171875 -0.65625q0.734375 -0.265625 1.8125 -0.265625q0.546875 0 1.203125 0.0625q0.65625 0.0625 1.359375 0.203125l0 1.40625q-0.75 -0.171875 -1.421875 -0.25q-0.65625 -0.09375 -1.15625 -0.09375q-0.59375 0 -1.0 0.09375q-0.390625 0.078125 -0.65625 0.25q-0.25 0.15625 -0.359375 0.375q-0.109375 0.203125 -0.109375 0.453125q0 0.25 0.09375 0.453125q0.109375 0.1875 0.359375 0.375q0.265625 0.1875 0.71875 0.375q0.46875 0.1875 1.21875 0.40625q0.8125 0.234375 1.359375 0.5q0.5625 0.25 0.90625 0.578125q0.359375 0.3125 0.5 0.71875q0.15625 0.40625 0.15625 0.90625zm10.741669 -2.5625q0 0.34375 -0.015625 0.578125q0 0.21875 -0.03125 0.421875l-6.421875 0q0 1.40625 0.78125 2.15625q0.796875 0.75 2.265625 0.75q0.40625 0 0.796875 -0.03125q0.40625 -0.03125 0.78125 -0.078125q0.375 -0.0625 0.71875 -0.125q0.34375 -0.078125 0.625 -0.15625l0 1.296875q-0.640625 0.1875 -1.46875 0.296875q-0.8125 0.109375 -1.6875 0.109375q-1.171875 0 -2.015625 -0.3125q-0.84375 -0.3125 -1.390625 -0.921875q-0.546875 -0.609375 -0.8125 -1.484375q-0.25 -0.875 -0.25 -2.0q0 -0.953125 0.28125 -1.8125q0.28125 -0.859375 0.8125 -1.515625q0.53125 -0.65625 1.296875 -1.03125q0.78125 -0.390625 1.765625 -0.390625q0.953125 0 1.6875 0.3125q0.75 0.296875 1.25 0.84375q0.5 0.546875 0.765625 1.34375q0.265625 0.78125 0.265625 1.75zm-1.65625 -0.21875q0.03125 -0.609375 -0.125 -1.109375q-0.140625 -0.515625 -0.453125 -0.875q-0.296875 -0.375 -0.75 -0.578125q-0.453125 -0.203125 -1.0625 -0.203125q-0.515625 0 -0.953125 0.203125q-0.421875 0.203125 -0.734375 0.5625q-0.296875 0.359375 -0.5 0.875q-0.1875 0.515625 -0.234375 1.125l4.8125 0zm4.5541534 -3.859375l1.453125 0l0.046875 1.6875q0.8125 -0.984375 1.59375 -1.421875q0.796875 -0.4375 1.59375 -0.4375q1.421875 0 2.15625 0.921875q0.734375 0.921875 0.671875 2.734375l-1.59375 0q0.015625 -1.203125 -0.359375 -1.734375q-0.375 -0.546875 -1.109375 -0.546875q-0.3125 0 -0.640625 0.109375q-0.328125 0.109375 -0.671875 0.359375q-0.328125 0.25 -0.71875 0.640625q-0.375 0.390625 -0.8125 0.953125l0 5.875l-1.609375 0l0 -9.140625zm17.257294 9.015625q-0.53125 0.140625 -1.109375 0.1875q-0.578125 0.0625 -1.171875 0.0625q-1.71875 0 -2.5625 -0.78125q-0.84375 -0.78125 -0.84375 -2.390625l0 -4.765625l-2.5625 0l0 -1.328125l2.5625 0l0 -2.515625l1.578125 -0.40625l0 2.921875l4.109375 0l0 1.328125l-4.109375 0l0 4.640625q0 0.984375 0.515625 1.46875q0.53125 0.484375 1.546875 0.484375q0.4375 0 0.953125 -0.0625q0.53125 -0.0625 1.09375 -0.21875l0 1.375z" fill-rule="nonzero"/><path fill="#616161" d="m295.09268 64.32059l-8.40625 12.875l-1.484375 0l8.375 -12.875l1.515625 0zm-4.859375 2.546875q0 0.59375 -0.171875 1.109375q-0.171875 0.5 -0.5 0.875q-0.3125 0.359375 -0.78125 0.578125q-0.453125 0.21875 -1.03125 0.21875q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.515625q-0.3125 -0.34375 -0.484375 -0.84375q-0.15625 -0.5 -0.15625 -1.15625q0 -0.59375 0.171875 -1.09375q0.171875 -0.5 0.484375 -0.875q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.21875 1.03125 -0.21875q0.5625 0 1.0 0.171875q0.4375 0.171875 0.75 0.515625q0.3125 0.328125 0.484375 0.828125q0.171875 0.5 0.171875 1.15625zm-1.34375 0.046875q0 -0.796875 -0.296875 -1.171875q-0.28125 -0.390625 -0.796875 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.34375 0.328125q-0.140625 0.203125 -0.21875 0.5q-0.078125 0.28125 -0.078125 0.609375q0 0.8125 0.296875 1.203125q0.296875 0.375 0.8125 0.375q0.265625 0 0.46875 -0.125q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.203125 0.203125 -0.484375q0.078125 -0.296875 0.078125 -0.640625zm6.046875 7.65625q0 0.578125 -0.171875 1.09375q-0.171875 0.5 -0.5 0.875q-0.328125 0.359375 -0.796875 0.578125q-0.453125 0.203125 -1.015625 0.203125q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.5q-0.3125 -0.34375 -0.484375 -0.84375q-0.171875 -0.515625 -0.171875 -1.171875q0 -0.578125 0.171875 -1.078125q0.1875 -0.515625 0.5 -0.890625q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.203125 1.03125 -0.203125q0.5625 0 1.0 0.171875q0.453125 0.171875 0.765625 0.515625q0.3125 0.328125 0.46875 0.84375q0.171875 0.5 0.171875 1.15625zm-1.328125 0.03125q0 -0.796875 -0.296875 -1.171875q-0.296875 -0.390625 -0.8125 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.359375 0.328125q-0.140625 0.203125 -0.21875 0.484375q-0.0625 0.28125 -0.0625 0.625q0 0.796875 0.28125 1.1875q0.296875 0.375 0.828125 0.375q0.265625 0 0.46875 -0.109375q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.21875 0.21875 -0.5q0.078125 -0.296875 0.078125 -0.625z" fill-rule="nonzero"/><path fill="#c53929" d="m304.84998 65.80496q-1.25 -0.265625 -2.15625 -0.265625q-2.140625 0 -2.140625 2.234375l0 1.609375l4.015625 0l0 1.3125l-4.015625 0l0 6.5l-1.609375 0l0 -6.5l-2.9375 0l0 -1.3125l2.9375 0l0 -1.515625q0 -3.65625 3.8125 -3.65625q0.9375 0 2.09375 0.21875l0 1.375zm-9.578125 2.25l0 0z" fill-rule="nonzero"/><path fill="#9c27b0" d="m320.36456 69.36746l-2.703125 0l0 -1.3125l4.296875 0l0 7.8125l2.734375 0l0 1.328125l-7.34375 0l0 -1.328125l3.015625 0l0 -6.5zm0.546875 -5.15625q0.265625 0 0.484375 0.09375q0.234375 0.09375 0.40625 0.28125q0.171875 0.171875 0.265625 0.390625q0.09375 0.21875 0.09375 0.484375q0 0.25 -0.09375 0.484375q-0.09375 0.234375 -0.265625 0.40625q-0.171875 0.171875 -0.40625 0.265625q-0.21875 0.09375 -0.484375 0.09375q-0.265625 0 -0.5 -0.09375q-0.21875 -0.09375 -0.390625 -0.265625q-0.15625 -0.171875 -0.265625 -0.40625q-0.09375 -0.234375 -0.09375 -0.484375q0 -0.265625 0.09375 -0.484375q0.109375 -0.21875 0.265625 -0.390625q0.171875 -0.1875 0.390625 -0.28125q0.234375 -0.09375 0.5 -0.09375zm6.5854187 3.84375l1.421875 0l0.0625 1.46875q0.390625 -0.46875 0.765625 -0.78125q0.375 -0.3125 0.734375 -0.5q0.359375 -0.203125 0.734375 -0.28125q0.375 -0.078125 0.78125 -0.078125q1.40625 0 2.125 0.84375q0.734375 0.828125 0.734375 2.5l0 5.96875l-1.59375 0l0 -5.84375q0 -1.078125 -0.40625 -1.578125q-0.390625 -0.515625 -1.1875 -0.515625q-0.28125 0 -0.5625 0.09375q-0.28125 0.078125 -0.578125 0.296875q-0.296875 0.203125 -0.65625 0.578125q-0.34375 0.359375 -0.78125 0.90625l0 6.0625l-1.59375 0l0 -9.140625zm17.632294 9.015625q-0.53125 0.140625 -1.109375 0.1875q-0.578125 0.0625 -1.171875 0.0625q-1.71875 0 -2.5625 -0.78125q-0.84375 -0.78125 -0.84375 -2.390625l0 -4.765625l-2.5625 0l0 -1.328125l2.5625 0l0 -2.515625l1.578125 -0.40625l0 2.921875l4.109375 0l0 1.328125l-4.109375 0l0 4.640625q0 0.984375 0.515625 1.46875q0.53125 0.484375 1.546875 0.484375q0.4375 0 0.953125 -0.0625q0.53125 -0.0625 1.09375 -0.21875l0 1.375zm10.851044 -4.515625q0 1.0625 -0.3125 1.953125q-0.296875 0.890625 -0.859375 1.53125q-0.5625 0.625 -1.375 0.96875q-0.8125 0.34375 -1.84375 0.34375q-0.984375 0 -1.765625 -0.296875q-0.765625 -0.3125 -1.3125 -0.890625q-0.546875 -0.59375 -0.828125 -1.46875q-0.28125 -0.875 -0.28125 -2.015625q0 -1.0625 0.296875 -1.9375q0.296875 -0.890625 0.859375 -1.515625q0.5625 -0.640625 1.375 -0.984375q0.8125 -0.359375 1.84375 -0.359375q0.984375 0 1.765625 0.3125q0.78125 0.296875 1.3125 0.890625q0.546875 0.578125 0.828125 1.453125q0.296875 0.875 0.296875 2.015625zm-1.625 0.0625q0 -0.84375 -0.1875 -1.46875q-0.1875 -0.640625 -0.53125 -1.0625q-0.34375 -0.421875 -0.84375 -0.640625q-0.5 -0.21875 -1.109375 -0.21875q-0.703125 0 -1.21875 0.28125q-0.5 0.28125 -0.828125 0.75q-0.3125 0.453125 -0.46875 1.078125q-0.15625 0.609375 -0.15625 1.28125q0 0.859375 0.1875 1.5q0.1875 0.625 0.53125 1.046875q0.359375 0.421875 0.84375 0.640625q0.5 0.21875 1.109375 0.21875q0.71875 0 1.21875 -0.28125q0.5 -0.28125 0.828125 -0.734375q0.328125 -0.46875 0.46875 -1.078125q0.15625 -0.625 0.15625 -1.3125z" fill-rule="nonzero"/><path fill="#616161" d="m377.15103 64.32059l-8.40625 12.875l-1.484375 0l8.375 -12.875l1.515625 0zm-4.859375 2.546875q0 0.59375 -0.171875 1.109375q-0.171875 0.5 -0.5 0.875q-0.3125 0.359375 -0.78125 0.578125q-0.453125 0.21875 -1.03125 0.21875q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.515625q-0.3125 -0.34375 -0.484375 -0.84375q-0.15625 -0.5 -0.15625 -1.15625q0 -0.59375 0.171875 -1.09375q0.171875 -0.5 0.484375 -0.875q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.21875 1.03125 -0.21875q0.5625 0 1.0 0.171875q0.4375 0.171875 0.75 0.515625q0.3125 0.328125 0.484375 0.828125q0.171875 0.5 0.171875 1.15625zm-1.34375 0.046875q0 -0.796875 -0.296875 -1.171875q-0.28125 -0.390625 -0.796875 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.34375 0.328125q-0.140625 0.203125 -0.21875 0.5q-0.078125 0.28125 -0.078125 0.609375q0 0.8125 0.296875 1.203125q0.296875 0.375 0.8125 0.375q0.265625 0 0.46875 -0.125q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.203125 0.203125 -0.484375q0.078125 -0.296875 0.078125 -0.640625zm6.046875 7.65625q0 0.578125 -0.171875 1.09375q-0.171875 0.5 -0.5 0.875q-0.328125 0.359375 -0.796875 0.578125q-0.453125 0.203125 -1.015625 0.203125q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.5q-0.3125 -0.34375 -0.484375 -0.84375q-0.171875 -0.515625 -0.171875 -1.171875q0 -0.578125 0.171875 -1.078125q0.1875 -0.515625 0.5 -0.890625q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.203125 1.03125 -0.203125q0.5625 0 1.0 0.171875q0.453125 0.171875 0.765625 0.515625q0.3125 0.328125 0.46875 0.84375q0.171875 0.5 0.171875 1.15625zm-1.328125 0.03125q0 -0.796875 -0.296875 -1.171875q-0.296875 -0.390625 -0.8125 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.359375 0.328125q-0.140625 0.203125 -0.21875 0.484375q-0.0625 0.28125 -0.0625 0.625q0 0.796875 0.28125 1.1875q0.296875 0.375 0.828125 0.375q0.265625 0 0.46875 -0.109375q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.21875 0.21875 -0.5q0.078125 -0.296875 0.078125 -0.625z" fill-rule="nonzero"/><path fill="#c53929" d="m386.15833 77.07059q-0.53125 0.140625 -1.109375 0.1875q-0.578125 0.0625 -1.171875 0.0625q-1.71875 0 -2.5625 -0.78125q-0.84375 -0.78125 -0.84375 -2.390625l0 -4.765625l-2.5625 0l0 -1.328125l2.5625 0l0 -2.515625l1.578125 -0.40625l0 2.921875l4.109375 0l0 1.328125l-4.109375 0l0 4.640625q0 0.984375 0.515625 1.46875q0.53125 0.484375 1.546875 0.484375q0.4375 0 0.953125 -0.0625q0.53125 -0.0625 1.09375 -0.21875l0 1.375z" fill-rule="nonzero"/><path fill="#616161" d="m395.22812 80.92996l-4.5 0l0 -16.921875l4.5 0l0 1.265625l-2.984375 0l0 14.375l2.984375 0l0 1.28125zm12.694794 -16.609375l-8.40625 12.875l-1.484375 0l8.375 -12.875l1.515625 0zm-4.859375 2.546875q0 0.59375 -0.171875 1.109375q-0.171875 0.5 -0.5 0.875q-0.3125 0.359375 -0.78125 0.578125q-0.453125 0.21875 -1.03125 0.21875q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.515625q-0.3125 -0.34375 -0.484375 -0.84375q-0.15625 -0.5 -0.15625 -1.15625q0 -0.59375 0.171875 -1.09375q0.171875 -0.5 0.484375 -0.875q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.21875 1.03125 -0.21875q0.5625 0 1.0 0.171875q0.4375 0.171875 0.75 0.515625q0.3125 0.328125 0.484375 0.828125q0.171875 0.5 0.171875 1.15625zm-1.34375 0.046875q0 -0.796875 -0.296875 -1.171875q-0.28125 -0.390625 -0.796875 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.34375 0.328125q-0.140625 0.203125 -0.21875 0.5q-0.078125 0.28125 -0.078125 0.609375q0 0.8125 0.296875 1.203125q0.296875 0.375 0.8125 0.375q0.265625 0 0.46875 -0.125q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.203125 0.203125 -0.484375q0.078125 -0.296875 0.078125 -0.640625zm6.046875 7.65625q0 0.578125 -0.171875 1.09375q-0.171875 0.5 -0.5 0.875q-0.328125 0.359375 -0.796875 0.578125q-0.453125 0.203125 -1.015625 0.203125q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.5q-0.3125 -0.34375 -0.484375 -0.84375q-0.171875 -0.515625 -0.171875 -1.171875q0 -0.578125 0.171875 -1.078125q0.1875 -0.515625 0.5 -0.890625q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.203125 1.03125 -0.203125q0.5625 0 1.0 0.171875q0.453125 0.171875 0.765625 0.515625q0.3125 0.328125 0.46875 0.84375q0.171875 0.5 0.171875 1.15625zm-1.328125 0.03125q0 -0.796875 -0.296875 -1.171875q-0.296875 -0.390625 -0.8125 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.359375 0.328125q-0.140625 0.203125 -0.21875 0.484375q-0.0625 0.28125 -0.0625 0.625q0 0.796875 0.28125 1.1875q0.296875 0.375 0.828125 0.375q0.265625 0 0.46875 -0.109375q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.21875 0.21875 -0.5q0.078125 -0.296875 0.078125 -0.625z" fill-rule="nonzero"/><path fill="#c53929" d="m412.6802 69.36746l-2.703125 0l0 -1.3125l4.296875 0l0 7.8125l2.734375 0l0 1.328125l-7.34375 0l0 -1.328125l3.015625 0l0 -6.5zm0.546875 -5.15625q0.265625 0 0.484375 0.09375q0.234375 0.09375 0.40625 0.28125q0.171875 0.171875 0.265625 0.390625q0.09375 0.21875 0.09375 0.484375q0 0.25 -0.09375 0.484375q-0.09375 0.234375 -0.265625 0.40625q-0.171875 0.171875 -0.40625 0.265625q-0.21875 0.09375 -0.484375 0.09375q-0.265625 0 -0.5 -0.09375q-0.21875 -0.09375 -0.390625 -0.265625q-0.15625 -0.171875 -0.265625 -0.40625q-0.09375 -0.234375 -0.09375 -0.484375q0 -0.265625 0.09375 -0.484375q0.109375 -0.21875 0.265625 -0.390625q0.171875 -0.1875 0.390625 -0.28125q0.234375 -0.09375 0.5 -0.09375zm6.1322937 8.609375q0 -1.171875 0.3125 -2.078125q0.3125 -0.90625 0.90625 -1.53125q0.59375 -0.625 1.40625 -0.9375q0.828125 -0.328125 1.84375 -0.328125q0.4375 0 0.859375 0.0625q0.421875 0.046875 0.828125 0.15625l0 -3.84375l1.59375 0l0 12.875l-1.421875 0l-0.046875 -1.734375q-0.671875 0.96875 -1.453125 1.4375q-0.765625 0.453125 -1.671875 0.453125q-0.78125 0 -1.375 -0.3125q-0.59375 -0.328125 -1.0 -0.921875q-0.390625 -0.609375 -0.59375 -1.4375q-0.1875 -0.84375 -0.1875 -1.859375zm1.609375 -0.109375q0 1.65625 0.484375 2.484375q0.5 0.8125 1.390625 0.8125q0.59375 0 1.265625 -0.53125q0.671875 -0.546875 1.40625 -1.609375l0 -4.234375q-0.390625 -0.1875 -0.859375 -0.28125q-0.46875 -0.09375 -0.9375 -0.09375q-1.296875 0 -2.03125 0.84375q-0.71875 0.828125 -0.71875 2.609375zm17.351044 4.484375l-2.109375 0l-2.46875 -3.484375l-2.4375 3.484375l-2.046875 0l3.546875 -4.59375l-3.390625 -4.546875l2.03125 0l2.40625 3.515625l2.359375 -3.515625l1.96875 0l-3.4375 4.578125l3.578125 4.5625z" fill-rule="nonzero"/><path fill="#616161" d="m445.98334 80.92996l-4.484375 0l0 -1.28125l2.953125 0l0 -14.375l-2.953125 0l0 -1.265625l4.484375 0l0 16.921875z" fill-rule="nonzero"/><path fill="#616161" d="m464.51355 67.88309q0.28125 0 0.53125 0.125q0.265625 0.109375 0.453125 0.296875q0.1875 0.1875 0.296875 0.453125q0.125 0.25 0.125 0.53125q0 0.296875 -0.125 0.546875q-0.109375 0.25 -0.296875 0.4375q-0.1875 0.1875 -0.453125 0.296875q-0.25 0.109375 -0.53125 0.109375q-0.28125 0 -0.53125 -0.109375q-0.25 -0.109375 -0.453125 -0.296875q-0.1875 -0.1875 -0.296875 -0.4375q-0.109375 -0.25 -0.109375 -0.546875q0 -0.28125 0.109375 -0.53125q0.109375 -0.265625 0.296875 -0.453125q0.203125 -0.1875 0.453125 -0.296875q0.25 -0.125 0.53125 -0.125zm0 6.6875q0.28125 0 0.53125 0.109375q0.265625 0.109375 0.453125 0.3125q0.1875 0.1875 0.296875 0.4375q0.125 0.25 0.125 0.53125q0 0.296875 -0.125 0.546875q-0.109375 0.25 -0.296875 0.453125q-0.1875 0.1875 -0.453125 0.28125q-0.25 0.109375 -0.53125 0.109375q-0.28125 0 -0.53125 -0.109375q-0.25 -0.09375 -0.453125 -0.28125q-0.1875 -0.203125 -0.296875 -0.453125q-0.109375 -0.25 -0.109375 -0.546875q0 -0.28125 0.109375 -0.53125q0.109375 -0.25 0.296875 -0.4375q0.203125 -0.203125 0.453125 -0.3125q0.25 -0.109375 0.53125 -0.109375z" fill-rule="nonzero"/><path fill="#000000" d="m488.73126 77.07059q-0.53125 0.140625 -1.109375 0.1875q-0.578125 0.0625 -1.171875 0.0625q-1.71875 0 -2.5625 -0.78125q-0.84375 -0.78125 -0.84375 -2.390625l0 -4.765625l-2.5625 0l0 -1.328125l2.5625 0l0 -2.515625l1.578125 -0.40625l0 2.921875l4.109375 0l0 1.328125l-4.109375 0l0 4.640625q0 0.984375 0.515625 1.46875q0.53125 0.484375 1.546875 0.484375q0.4375 0 0.953125 -0.0625q0.53125 -0.0625 1.09375 -0.21875l0 1.375zm10.616669 -4.9375q0 0.34375 -0.015625 0.578125q0 0.21875 -0.03125 0.421875l-6.421875 0q0 1.40625 0.78125 2.15625q0.796875 0.75 2.265625 0.75q0.40625 0 0.796875 -0.03125q0.40625 -0.03125 0.78125 -0.078125q0.375 -0.0625 0.71875 -0.125q0.34375 -0.078125 0.625 -0.15625l0 1.296875q-0.640625 0.1875 -1.46875 0.296875q-0.8125 0.109375 -1.6875 0.109375q-1.171875 0 -2.015625 -0.3125q-0.84375 -0.3125 -1.390625 -0.921875q-0.546875 -0.609375 -0.8125 -1.484375q-0.25 -0.875 -0.25 -2.0q0 -0.953125 0.28125 -1.8125q0.28125 -0.859375 0.8125 -1.515625q0.53125 -0.65625 1.296875 -1.03125q0.78125 -0.390625 1.765625 -0.390625q0.953125 0 1.6875 0.3125q0.75 0.296875 1.25 0.84375q0.5 0.546875 0.765625 1.34375q0.265625 0.78125 0.265625 1.75zm-1.65625 -0.21875q0.03125 -0.609375 -0.125 -1.109375q-0.140625 -0.515625 -0.453125 -0.875q-0.296875 -0.375 -0.75 -0.578125q-0.453125 -0.203125 -1.0625 -0.203125q-0.515625 0 -0.953125 0.203125q-0.421875 0.203125 -0.734375 0.5625q-0.296875 0.359375 -0.5 0.875q-0.1875 0.515625 -0.234375 1.125l4.8125 0zm4.1791687 -3.859375l1.421875 0l0.0625 1.46875q0.390625 -0.46875 0.765625 -0.78125q0.375 -0.3125 0.734375 -0.5q0.359375 -0.203125 0.734375 -0.28125q0.375 -0.078125 0.78125 -0.078125q1.40625 0 2.125 0.84375q0.734375 0.828125 0.734375 2.5l0 5.96875l-1.59375 0l0 -5.84375q0 -1.078125 -0.40625 -1.578125q-0.390625 -0.515625 -1.1875 -0.515625q-0.28125 0 -0.5625 0.09375q-0.28125 0.078125 -0.578125 0.296875q-0.296875 0.203125 -0.65625 0.578125q-0.34375 0.359375 -0.78125 0.90625l0 6.0625l-1.59375 0l0 -9.140625zm17.507324 6.640625q0 0.484375 -0.171875 0.875q-0.15625 0.375 -0.4375 0.671875q-0.28125 0.296875 -0.65625 0.515625q-0.375 0.203125 -0.796875 0.34375q-0.421875 0.125 -0.875 0.1875q-0.453125 0.0625 -0.875 0.0625q-0.921875 0 -1.703125 -0.078125q-0.78125 -0.078125 -1.53125 -0.265625l0 -1.453125q0.796875 0.234375 1.59375 0.359375q0.796875 0.109375 1.578125 0.109375q1.140625 0 1.6875 -0.3125q0.546875 -0.3125 0.546875 -0.890625q0 -0.234375 -0.09375 -0.421875q-0.078125 -0.203125 -0.3125 -0.375q-0.21875 -0.1875 -0.703125 -0.375q-0.484375 -0.203125 -1.3125 -0.4375q-0.609375 -0.1875 -1.140625 -0.421875q-0.515625 -0.234375 -0.90625 -0.546875q-0.375 -0.328125 -0.59375 -0.75q-0.21875 -0.421875 -0.21875 -1.015625q0 -0.375 0.171875 -0.828125q0.171875 -0.453125 0.59375 -0.84375q0.4375 -0.40625 1.171875 -0.65625q0.734375 -0.265625 1.8125 -0.265625q0.546875 0 1.203125 0.0625q0.65625 0.0625 1.359375 0.203125l0 1.40625q-0.75 -0.171875 -1.421875 -0.25q-0.65625 -0.09375 -1.15625 -0.09375q-0.59375 0 -1.0 0.09375q-0.390625 0.078125 -0.65625 0.25q-0.25 0.15625 -0.359375 0.375q-0.109375 0.203125 -0.109375 0.453125q0 0.25 0.09375 0.453125q0.109375 0.1875 0.359375 0.375q0.265625 0.1875 0.71875 0.375q0.46875 0.1875 1.21875 0.40625q0.8125 0.234375 1.359375 0.5q0.5625 0.25 0.90625 0.578125q0.359375 0.3125 0.5 0.71875q0.15625 0.40625 0.15625 0.90625zm10.976013 -2.140625q0 1.0625 -0.3125 1.953125q-0.296875 0.890625 -0.859375 1.53125q-0.5625 0.625 -1.375 0.96875q-0.8125 0.34375 -1.84375 0.34375q-0.984375 0 -1.765625 -0.296875q-0.765625 -0.3125 -1.3125 -0.890625q-0.546875 -0.59375 -0.828125 -1.46875q-0.28125 -0.875 -0.28125 -2.015625q0 -1.0625 0.296875 -1.9375q0.296875 -0.890625 0.859375 -1.515625q0.5625 -0.640625 1.375 -0.984375q0.8125 -0.359375 1.84375 -0.359375q0.984375 0 1.765625 0.3125q0.78125 0.296875 1.3125 0.890625q0.546875 0.578125 0.828125 1.453125q0.296875 0.875 0.296875 2.015625zm-1.625 0.0625q0 -0.84375 -0.1875 -1.46875q-0.1875 -0.640625 -0.53125 -1.0625q-0.34375 -0.421875 -0.84375 -0.640625q-0.5 -0.21875 -1.109375 -0.21875q-0.703125 0 -1.21875 0.28125q-0.5 0.28125 -0.828125 0.75q-0.3125 0.453125 -0.46875 1.078125q-0.15625 0.609375 -0.15625 1.28125q0 0.859375 0.1875 1.5q0.1875 0.625 0.53125 1.046875q0.359375 0.421875 0.84375 0.640625q0.5 0.21875 1.109375 0.21875q0.71875 0 1.21875 -0.28125q0.5 -0.28125 0.828125 -0.734375q0.328125 -0.46875 0.46875 -1.078125q0.15625 -0.625 0.15625 -1.3125zm4.288574 -4.5625l1.453125 0l0.046875 1.6875q0.8125 -0.984375 1.59375 -1.421875q0.796875 -0.4375 1.59375 -0.4375q1.421875 0 2.15625 0.921875q0.734375 0.921875 0.671875 2.734375l-1.59375 0q0.015625 -1.203125 -0.359375 -1.734375q-0.375 -0.546875 -1.109375 -0.546875q-0.3125 0 -0.640625 0.109375q-0.328125 0.109375 -0.671875 0.359375q-0.328125 0.25 -0.71875 0.640625q-0.375 0.390625 -0.8125 0.953125l0 5.875l-1.609375 0l0 -9.140625z" fill-rule="nonzero"/><path fill="#616161" d="m549.8063 76.33621l-0.984375 1.0l-6.125 -5.109375l6.125 -5.109375l0.984375 1.015625l-4.96875 4.078125l4.96875 4.125z" fill-rule="nonzero"/><path fill="#c53929" d="m560.5792 73.38309q0 0.890625 -0.390625 1.625q-0.375 0.71875 -1.046875 1.25q-0.65625 0.515625 -1.578125 0.8125q-0.90625 0.28125 -1.953125 0.28125q-0.265625 0 -0.578125 -0.015625q-0.296875 0 -0.609375 -0.03125q-0.296875 -0.015625 -0.59375 -0.046875q-0.28125 -0.03125 -0.5 -0.0625l0 -1.4375q0.484375 0.109375 1.109375 0.171875q0.640625 0.046875 1.296875 0.046875q0.71875 0 1.296875 -0.171875q0.59375 -0.171875 1.0 -0.484375q0.40625 -0.328125 0.625 -0.78125q0.21875 -0.46875 0.21875 -1.046875q0 -1.109375 -0.796875 -1.609375q-0.796875 -0.515625 -2.28125 -0.515625l-2.265625 0l0 -6.078125l6.421875 0l0 1.390625l-4.921875 0l0 3.34375l1.03125 0q0.859375 0 1.671875 0.15625q0.8125 0.140625 1.4375 0.53125q0.640625 0.390625 1.015625 1.046875q0.390625 0.640625 0.390625 1.625zm11.085388 3.8125l-2.109375 0l-2.46875 -3.484375l-2.4375 3.484375l-2.046875 0l3.546875 -4.59375l-3.390625 -4.546875l2.03125 0l2.40625 3.515625l2.359375 -3.515625l1.96875 0l-3.4375 4.578125l3.578125 4.5625zm10.132324 -11.390625q-1.25 -0.265625 -2.15625 -0.265625q-2.140625 0 -2.140625 2.234375l0 1.609375l4.015625 0l0 1.3125l-4.015625 0l0 6.5l-1.609375 0l0 -6.5l-2.9375 0l0 -1.3125l2.9375 0l0 -1.515625q0 -3.65625 3.8125 -3.65625q0.9375 0 2.09375 0.21875l0 1.375zm-9.578125 2.25l0 0zm19.210388 5.53125q0 0.78125 -0.328125 1.46875q-0.3125 0.6875 -0.953125 1.203125q-0.625 0.515625 -1.59375 0.8125q-0.96875 0.28125 -2.25 0.28125q-0.71875 0 -1.28125 -0.046875q-0.5625 -0.03125 -1.046875 -0.109375l0 -1.421875q0.5625 0.09375 1.1875 0.15625q0.625 0.046875 1.28125 0.046875q0.890625 0 1.515625 -0.15625q0.640625 -0.15625 1.046875 -0.453125q0.40625 -0.296875 0.578125 -0.71875q0.1875 -0.4375 0.1875 -0.984375q0 -0.484375 -0.21875 -0.84375q-0.21875 -0.375 -0.625 -0.609375q-0.390625 -0.25 -0.9375 -0.375q-0.546875 -0.125 -1.203125 -0.125l-1.359375 0l0 -1.296875l1.375 0q0.53125 0 0.96875 -0.140625q0.453125 -0.140625 0.765625 -0.40625q0.3125 -0.265625 0.484375 -0.65625q0.171875 -0.390625 0.171875 -0.875q0 -0.953125 -0.59375 -1.390625q-0.578125 -0.4375 -1.703125 -0.4375q-0.609375 0 -1.25 0.125q-0.625 0.109375 -1.359375 0.34375l0 -1.390625q0.3125 -0.109375 0.65625 -0.1875q0.359375 -0.09375 0.703125 -0.140625q0.359375 -0.0625 0.703125 -0.09375q0.34375 -0.03125 0.65625 -0.03125q0.953125 0 1.671875 0.203125q0.71875 0.203125 1.203125 0.59375q0.484375 0.375 0.71875 0.921875q0.25 0.546875 0.25 1.21875q0 1.03125 -0.53125 1.71875q-0.515625 0.6875 -1.421875 1.109375q0.46875 0.0625 0.90625 0.28125q0.453125 0.21875 0.8125 0.5625q0.375 0.34375 0.59375 0.8125q0.21875 0.453125 0.21875 1.03125zm10.491699 3.609375l-7.84375 0l0 -1.421875l3.078125 -3.0625q0.75 -0.75 1.21875 -1.296875q0.484375 -0.546875 0.75 -0.984375q0.265625 -0.453125 0.34375 -0.859375q0.09375 -0.40625 0.09375 -0.859375q0 -0.4375 -0.125 -0.828125q-0.109375 -0.40625 -0.359375 -0.703125q-0.234375 -0.3125 -0.640625 -0.484375q-0.390625 -0.171875 -0.9375 -0.171875q-0.75 0 -1.375 0.34375q-0.609375 0.328125 -1.125 0.875l-0.875 -1.046875q0.671875 -0.71875 1.546875 -1.140625q0.875 -0.421875 2.046875 -0.421875q0.796875 0 1.4375 0.234375q0.65625 0.234375 1.125 0.6875q0.484375 0.4375 0.734375 1.09375q0.265625 0.640625 0.265625 1.453125q0 0.671875 -0.1875 1.265625q-0.171875 0.578125 -0.546875 1.15625q-0.375 0.578125 -0.9375 1.203125q-0.546875 0.609375 -1.3125 1.359375l-2.171875 2.09375l5.796875 0l0 1.515625z" fill-rule="nonzero"/><path fill="#616161" d="m604.8969 68.13309l0.984375 -1.015625l6.125 5.109375l-6.125 5.109375l-0.984375 -1.0l4.96875 -4.09375l-4.96875 -4.109375z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m376.13647 58.68504l-12.75589 -23.464565" fill-rule="evenodd"/><path stroke="#ff0000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m376.13647 58.68504l-12.75589 -23.464565" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m297.02362 4.5826774l154.14172 0l0 23.464567l-154.14172 0z" fill-rule="evenodd"/><path fill="#000000" d="m308.58612 17.846426l0.734375 0.484375l-0.90625 1.40625q0.3125 0.046875 0.578125 0.328125q0.28125 0.265625 0.28125 0.734375q0 0.5 -0.34375 0.84375q-0.34375 0.34375 -0.828125 0.34375q-0.25 0 -0.484375 -0.09375q-0.21875 -0.09375 -0.375 -0.25q-0.15625 -0.15625 -0.25 -0.375q-0.09375 -0.234375 -0.09375 -0.46875q0 -0.234375 0.0625 -0.4375q0.0625 -0.21875 0.1875 -0.421875q0.125 -0.203125 0.25 -0.390625l1.1875 -1.703125zm3.15625 0l0.734375 0.484375l-0.90625 1.40625q0.3125 0.046875 0.578125 0.328125q0.28125 0.265625 0.28125 0.734375q0 0.5 -0.34375 0.84375q-0.34375 0.34375 -0.828125 0.34375q-0.25 0 -0.484375 -0.09375q-0.21875 -0.09375 -0.375 -0.25q-0.15625 -0.15625 -0.25 -0.375q-0.09375 -0.234375 -0.09375 -0.46875q0 -0.234375 0.0625 -0.4375q0.0625 -0.21875 0.1875 -0.421875q0.125 -0.203125 0.25 -0.390625l1.1875 -1.703125zm5.711914 13.953125q-1.28125 0 -2.328125 -0.640625q-1.03125 -0.65625 -1.625 -1.796875q-0.59375 -1.140625 -0.59375 -2.609375q0 -1.484375 0.59375 -2.609375q0.59375 -1.140625 1.625 -1.796875q1.046875 -0.65625 2.328125 -0.65625q0.765625 0 1.390625 0.25q0.640625 0.234375 1.109375 0.625q0.484375 0.390625 0.75 0.84375l0.078125 0l-0.078125 -1.328125l0 -3.9375l1.59375 0l0 13.359375l-1.515625 0l0 -1.40625l-0.078125 0q-0.265625 0.4375 -0.75 0.828125q-0.46875 0.390625 -1.109375 0.625q-0.625 0.25 -1.390625 0.25zm0.171875 -1.4375q0.828125 0 1.546875 -0.4375q0.71875 -0.4375 1.15625 -1.25q0.453125 -0.8125 0.453125 -1.921875q0 -1.125 -0.453125 -1.9375q-0.4375 -0.8125 -1.15625 -1.25q-0.71875 -0.4375 -1.546875 -0.4375q-0.8125 0 -1.53125 0.4375q-0.71875 0.4375 -1.171875 1.25q-0.453125 0.8125 -0.453125 1.9375q0 1.09375 0.453125 1.921875q0.453125 0.8125 1.171875 1.25q0.71875 0.4375 1.53125 0.4375zm11.153351 1.4375q-1.375 0 -2.453125 -0.640625q-1.0625 -0.65625 -1.671875 -1.796875q-0.609375 -1.140625 -0.609375 -2.59375q0 -1.359375 0.5625 -2.515625q0.578125 -1.15625 1.609375 -1.859375q1.03125 -0.703125 2.4375 -0.703125q1.421875 0 2.4375 0.625q1.015625 0.625 1.5625 1.734375q0.546875 1.09375 0.546875 2.515625q0 0.125 -0.015625 0.265625q0 0.125 -0.015625 0.21875l-8.1875 0l0 -1.3125l6.546875 0q-0.015625 -0.390625 -0.1875 -0.84375q-0.15625 -0.46875 -0.5 -0.859375q-0.34375 -0.40625 -0.875 -0.65625q-0.53125 -0.25 -1.3125 -0.25q-0.9375 0 -1.625 0.484375q-0.671875 0.46875 -1.046875 1.296875q-0.359375 0.8125 -0.359375 1.859375q0 1.203125 0.46875 2.015625q0.46875 0.796875 1.203125 1.1875q0.75 0.390625 1.546875 0.390625q1.046875 0 1.71875 -0.484375q0.6875 -0.5 1.09375 -1.234375l1.34375 0.65625q-0.5625 1.078125 -1.609375 1.796875q-1.03125 0.703125 -2.609375 0.703125zm9.690552 0q-1.0625 0 -1.875 -0.34375q-0.8125 -0.34375 -1.34375 -0.921875q-0.53125 -0.59375 -0.796875 -1.28125l1.421875 -0.640625q0.375 0.859375 1.078125 1.328125q0.703125 0.46875 1.625 0.46875q0.875 0 1.453125 -0.359375q0.59375 -0.359375 0.59375 -1.046875q0 -0.421875 -0.25 -0.703125q-0.234375 -0.296875 -0.703125 -0.5q-0.453125 -0.21875 -1.125 -0.375l-1.15625 -0.3125q-0.671875 -0.1875 -1.28125 -0.515625q-0.59375 -0.34375 -0.953125 -0.875q-0.359375 -0.53125 -0.359375 -1.296875q0 -0.84375 0.5 -1.453125q0.5 -0.625 1.3125 -0.953125q0.828125 -0.328125 1.765625 -0.328125q0.8125 0 1.53125 0.234375q0.71875 0.234375 1.265625 0.6875q0.546875 0.453125 0.8125 1.125l-1.375 0.640625q-0.359375 -0.703125 -0.953125 -0.984375q-0.59375 -0.28125 -1.328125 -0.28125q-0.78125 0 -1.359375 0.34375q-0.578125 0.34375 -0.578125 0.9375q0 0.59375 0.46875 0.90625q0.484375 0.296875 1.171875 0.46875l1.390625 0.359375q1.390625 0.359375 2.09375 1.0625q0.71875 0.6875 0.71875 1.703125q0 0.890625 -0.515625 1.546875q-0.5 0.65625 -1.359375 1.015625q-0.84375 0.34375 -1.890625 0.34375zm4.746704 -9.8125l5.578125 0l0 1.4375l-5.578125 0l0 -1.4375zm1.671875 7.015625l0 -9.703125l1.578125 0l0 9.3125q0 0.75 0.3125 1.15625q0.3125 0.40625 1.015625 0.40625q0.3125 0 0.578125 -0.09375q0.265625 -0.09375 0.46875 -0.21875l0 1.546875q-0.25 0.109375 -0.546875 0.171875q-0.28125 0.078125 -0.765625 0.078125q-1.1875 0 -1.921875 -0.703125q-0.71875 -0.703125 -0.71875 -1.953125zm6.0434875 2.5l0 -9.515625l1.578125 0l0 9.515625l-1.578125 0zm0.78125 -11.265625q-0.46875 0 -0.8125 -0.328125q-0.328125 -0.34375 -0.328125 -0.8125q0 -0.484375 0.328125 -0.8125q0.34375 -0.328125 0.8125 -0.328125q0.484375 0 0.8125 0.328125q0.328125 0.328125 0.328125 0.8125q0 0.46875 -0.328125 0.8125q-0.328125 0.328125 -0.8125 0.328125zm3.2788086 11.265625l0 -9.515625l1.515625 0l0 1.40625l0.078125 0q0.375 -0.703125 1.21875 -1.203125q0.84375 -0.5 1.859375 -0.5q1.75 0 2.625 1.015625q0.890625 1.015625 0.890625 2.703125l0 6.09375l-1.578125 0l0 -5.859375q0 -1.375 -0.671875 -1.9375q-0.65625 -0.578125 -1.703125 -0.578125q-0.78125 0 -1.375 0.4375q-0.59375 0.4375 -0.9375 1.125q-0.328125 0.6875 -0.328125 1.453125l0 5.359375l-1.59375 0zm13.419708 0.296875q-1.0625 0 -1.875 -0.40625q-0.796875 -0.40625 -1.25 -1.125q-0.453125 -0.71875 -0.453125 -1.640625q0 -1.046875 0.53125 -1.765625q0.546875 -0.71875 1.453125 -1.078125q0.921875 -0.359375 2.015625 -0.359375q0.640625 0 1.171875 0.109375q0.546875 0.09375 0.9375 0.234375q0.40625 0.140625 0.625 0.265625l0 -0.578125q0 -1.078125 -0.765625 -1.703125q-0.765625 -0.640625 -1.875 -0.640625q-0.78125 0 -1.46875 0.34375q-0.671875 0.34375 -1.0625 0.953125l-1.203125 -0.890625q0.375 -0.5625 0.9375 -0.96875q0.5625 -0.40625 1.28125 -0.625q0.71875 -0.234375 1.515625 -0.234375q1.9375 0 3.03125 1.03125q1.109375 1.015625 1.109375 2.75l0 6.03125l-1.5 0l0 -1.359375l-0.078125 0q-0.25 0.40625 -0.703125 0.796875q-0.4375 0.375 -1.046875 0.609375q-0.609375 0.25 -1.328125 0.25zm0.140625 -1.390625q0.828125 0 1.5 -0.40625q0.6875 -0.421875 1.09375 -1.109375q0.421875 -0.6875 0.421875 -1.515625q-0.4375 -0.296875 -1.078125 -0.484375q-0.640625 -0.1875 -1.40625 -0.1875q-1.359375 0 -2.0 0.5625q-0.640625 0.5625 -0.640625 1.375q0 0.78125 0.59375 1.28125q0.609375 0.484375 1.515625 0.484375zm5.8760376 -8.421875l5.578125 0l0 1.4375l-5.578125 0l0 -1.4375zm1.671875 7.015625l0 -9.703125l1.578125 0l0 9.3125q0 0.75 0.3125 1.15625q0.3125 0.40625 1.015625 0.40625q0.3125 0 0.578125 -0.09375q0.265625 -0.09375 0.46875 -0.21875l0 1.546875q-0.25 0.109375 -0.546875 0.171875q-0.28125 0.078125 -0.765625 0.078125q-1.1875 0 -1.921875 -0.703125q-0.71875 -0.703125 -0.71875 -1.953125zm6.043518 2.5l0 -9.515625l1.578125 0l0 9.515625l-1.578125 0zm0.78125 -11.265625q-0.46875 0 -0.8125 -0.328125q-0.328125 -0.34375 -0.328125 -0.8125q0 -0.484375 0.328125 -0.8125q0.34375 -0.328125 0.8125 -0.328125q0.484375 0 0.8125 0.328125q0.328125 0.328125 0.328125 0.8125q0 0.46875 -0.328125 0.8125q-0.328125 0.328125 -0.8125 0.328125zm7.5760803 11.5625q-1.4375 0 -2.546875 -0.671875q-1.09375 -0.671875 -1.71875 -1.8125q-0.625 -1.15625 -0.625 -2.5625q0 -1.421875 0.625 -2.5625q0.625 -1.15625 1.71875 -1.828125q1.109375 -0.671875 2.546875 -0.671875q1.4375 0 2.53125 0.6875q1.109375 0.671875 1.734375 1.828125q0.625 1.140625 0.625 2.546875q0 1.40625 -0.625 2.5625q-0.625 1.140625 -1.734375 1.8125q-1.09375 0.671875 -2.53125 0.671875zm0 -1.4375q0.859375 0 1.609375 -0.421875q0.75 -0.4375 1.21875 -1.25q0.46875 -0.8125 0.46875 -1.9375q0 -1.140625 -0.46875 -1.953125q-0.46875 -0.8125 -1.21875 -1.234375q-0.75 -0.4375 -1.609375 -0.4375q-0.859375 0 -1.625 0.4375q-0.765625 0.421875 -1.234375 1.234375q-0.46875 0.8125 -0.46875 1.953125q0 1.125 0.46875 1.9375q0.46875 0.8125 1.234375 1.25q0.765625 0.421875 1.625 0.421875zm6.5418396 1.140625l0 -9.515625l1.515625 0l0 1.40625l0.078125 0q0.375 -0.703125 1.21875 -1.203125q0.84375 -0.5 1.859375 -0.5q1.75 0 2.625 1.015625q0.890625 1.015625 0.890625 2.703125l0 6.09375l-1.578125 0l0 -5.859375q0 -1.375 -0.671875 -1.9375q-0.65625 -0.578125 -1.703125 -0.578125q-0.78125 0 -1.375 0.4375q-0.59375 0.4375 -0.9375 1.125q-0.328125 0.6875 -0.328125 1.453125l0 5.359375l-1.59375 0zm9.795959 -10.0l0.984375 -1.40625q-0.34375 -0.0625 -0.640625 -0.328125q-0.28125 -0.265625 -0.28125 -0.734375q0 -0.5 0.34375 -0.84375q0.34375 -0.34375 0.828125 -0.34375q0.265625 0 0.46875 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.25 0.375q0.09375 0.21875 0.09375 0.46875q0 0.234375 -0.078125 0.4375q-0.0625 0.203125 -0.1875 0.40625q-0.109375 0.203125 -0.234375 0.40625l-1.171875 1.703125l-0.75 -0.484375zm3.15625 0l0.984375 -1.40625q-0.34375 -0.0625 -0.640625 -0.328125q-0.28125 -0.265625 -0.28125 -0.734375q0 -0.5 0.34375 -0.84375q0.34375 -0.34375 0.828125 -0.34375q0.265625 0 0.46875 0.09375q0.21875 0.09375 0.375 0.25q0.15625 0.15625 0.25 0.375q0.09375 0.21875 0.09375 0.46875q0 0.234375 -0.078125 0.4375q-0.0625 0.203125 -0.1875 0.40625q-0.109375 0.203125 -0.234375 0.40625l-1.171875 1.703125l-0.75 -0.484375z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m100.52231 63.587925l54.61418 -21.952755" fill-rule="evenodd"/><path stroke="#ff0000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m100.52231 63.587925l54.61418 -21.952755" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m366.43832 63.07874l-146.48819 -20.944881" fill-rule="evenodd"/><path stroke="#ff0000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m366.43832 63.07874l-146.48819 -20.944881" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m-3.0367453 9.677165l388.15747 0l0 23.464565l-388.15747 0z" fill-rule="evenodd"/><path fill="#000000" d="m6.4788795 27.08154l5.5781255 0l0 1.4375l-5.5781255 0l0 -1.4375zm1.6718755 7.015625l0 -9.703125l1.578125 0l0 9.3125q0 0.75 0.3125 1.15625q0.3125 0.40625 1.015625 0.40625q0.3125 0 0.578125 -0.09375q0.265625 -0.09375 0.46875 -0.21875l0 1.546875q-0.25 0.109375 -0.546875 0.171875q-0.28125 0.078125 -0.765625 0.078125q-1.1875 0 -1.921875 -0.703125q-0.71875 -0.703125 -0.71875 -1.953125zm6.043503 2.5l0 -9.515625l1.578125 0l0 9.515625l-1.578125 0zm0.78125 -11.265625q-0.46875 0 -0.8125 -0.328125q-0.328125 -0.34375 -0.328125 -0.8125q0 -0.484375 0.328125 -0.8125q0.34375 -0.328125 0.8125 -0.328125q0.484375 0 0.8125 0.328125q0.328125 0.328125 0.328125 0.8125q0 0.46875 -0.328125 0.8125q-0.328125 0.328125 -0.8125 0.328125zm7.4198456 11.5625q-1.375 0 -2.453125 -0.640625q-1.0625 -0.65625 -1.671875 -1.796875q-0.609375 -1.140625 -0.609375 -2.59375q0 -1.359375 0.5625 -2.515625q0.578125 -1.15625 1.609375 -1.859375q1.03125 -0.703125 2.4375 -0.703125q1.421875 0 2.4375 0.625q1.015625 0.625 1.5625 1.734375q0.546875 1.09375 0.546875 2.515625q0 0.125 -0.015625 0.265625q0 0.125 -0.015625 0.21875l-8.1875 0l0 -1.3125l6.546875 0q-0.015625 -0.390625 -0.1875 -0.84375q-0.15625 -0.46875 -0.5 -0.859375q-0.34375 -0.40625 -0.875 -0.65625q-0.53125 -0.25 -1.3125 -0.25q-0.9375 0 -1.625 0.484375q-0.671875 0.46875 -1.046875 1.296875q-0.359375 0.8125 -0.359375 1.859375q0 1.203125 0.46875 2.015625q0.46875 0.796875 1.203125 1.1875q0.75 0.390625 1.546875 0.390625q1.046875 0 1.71875 -0.484375q0.6875 -0.5 1.09375 -1.234375l1.34375 0.65625q-0.5625 1.078125 -1.609375 1.796875q-1.03125 0.703125 -2.609375 0.703125zm10.4093 0q-1.2812481 0 -2.328123 -0.640625q-1.03125 -0.65625 -1.625 -1.796875q-0.59375 -1.140625 -0.59375 -2.609375q0 -1.484375 0.59375 -2.609375q0.59375 -1.140625 1.625 -1.796875q1.046875 -0.65625 2.328123 -0.65625q0.765625 0 1.390625 0.25q0.640625 0.234375 1.109375 0.625q0.484375 0.390625 0.75 0.84375l0.078125 0l-0.078125 -1.328125l0 -3.9375l1.59375 0l0 13.359375l-1.515625 0l0 -1.40625l-0.078125 0q-0.265625 0.4375 -0.75 0.828125q-0.46875 0.390625 -1.109375 0.625q-0.625 0.25 -1.390625 0.25zm0.171875 -1.4375q0.828125 0 1.546875 -0.4375q0.71875 -0.4375 1.15625 -1.25q0.453125 -0.8125 0.453125 -1.921875q0 -1.125 -0.453125 -1.9375q-0.4375 -0.8125 -1.15625 -1.25q-0.71875 -0.4375 -1.546875 -0.4375q-0.8125 0 -1.5312481 0.4375q-0.71875 0.4375 -1.171875 1.25q-0.453125 0.8125 -0.453125 1.9375q0 1.09375 0.453125 1.921875q0.453125 0.8125 1.171875 1.25q0.7187481 0.4375 1.5312481 0.4375zm17.48285 1.4375q-1.46875 0 -2.71875 -0.53125q-1.25 -0.546875 -2.1875 -1.5q-0.921875 -0.96875 -1.4375 -2.21875q-0.5 -1.265625 -0.5 -2.71875q0 -1.46875 0.5 -2.71875q0.515625 -1.265625 1.4375 -2.21875q0.9375 -0.96875 2.1875 -1.5q1.25 -0.546875 2.71875 -0.546875q1.46875 0 2.71875 0.546875q1.25 0.53125 2.171875 1.5q0.921875 0.953125 1.4375 2.21875q0.515625 1.25 0.515625 2.71875q0 1.453125 -0.515625 2.71875q-0.515625 1.25 -1.4375 2.21875q-0.921875 0.953125 -2.171875 1.5q-1.25 0.53125 -2.71875 0.53125zm0 -1.515625q1.453125 0 2.640625 -0.6875q1.1875 -0.6875 1.890625 -1.921875q0.703125 -1.25 0.703125 -2.84375q0 -1.625 -0.703125 -2.859375q-0.703125 -1.234375 -1.890625 -1.921875q-1.1875 -0.6875 -2.640625 -0.6875q-1.4375 0 -2.640625 0.6875q-1.1875 0.6875 -1.90625 1.921875q-0.703125 1.234375 -0.703125 2.859375q0 1.59375 0.703125 2.84375q0.71875 1.234375 1.90625 1.921875q1.203125 0.6875 2.640625 0.6875zm8.857132 5.25l0 -13.546875l1.515625 0l0 1.421875l0.078125 0q0.25 -0.453125 0.71875 -0.84375q0.484375 -0.390625 1.125 -0.625q0.640625 -0.25 1.3906288 -0.25q1.3125 0 2.328125 0.65625q1.03125 0.65625 1.625 1.796875q0.609375 1.125 0.609375 2.609375q0 1.46875 -0.609375 2.609375q-0.59375 1.140625 -1.625 1.796875q-1.015625 0.640625 -2.328125 0.640625q-1.1250038 0 -1.9843788 -0.515625q-0.859375 -0.53125 -1.25 -1.1875l-0.078125 0l0.078125 1.3125l0 4.125l-1.59375 0zm4.671875 -5.171875q0.8125038 0 1.5312538 -0.4375q0.71875 -0.4375 1.15625 -1.25q0.4375 -0.828125 0.4375 -1.921875q0 -1.125 -0.4375 -1.9375q-0.4375 -0.8125 -1.15625 -1.25q-0.71875 -0.4375 -1.5312538 -0.4375q-0.828125 0 -1.546875 0.4375q-0.71875 0.4375 -1.171875 1.25q-0.4375 0.8125 -0.4375 1.9375q0 1.109375 0.4375 1.921875q0.453125 0.8125 1.171875 1.25q0.71875 0.4375 1.546875 0.4375zm13.074818 1.4375q-1.46875 0 -2.71875 -0.53125q-1.25 -0.546875 -2.1875 -1.5q-0.921875 -0.96875 -1.4375 -2.21875q-0.5 -1.265625 -0.5 -2.71875q0 -1.46875 0.5 -2.71875q0.515625 -1.265625 1.4375 -2.21875q0.9375 -0.96875 2.1875 -1.5q1.25 -0.546875 2.71875 -0.546875q1.46875 0 2.71875 0.546875q1.25 0.53125 2.171875 1.5q0.921875 0.953125 1.4375 2.21875q0.515625 1.25 0.515625 2.71875q0 1.453125 -0.515625 2.71875q-0.515625 1.25 -1.4375 2.21875q-0.921875 0.953125 -2.171875 1.5q-1.25 0.53125 -2.71875 0.53125zm0 -1.515625q1.453125 0 2.640625 -0.6875q1.1875 -0.6875 1.890625 -1.921875q0.703125 -1.25 0.703125 -2.84375q0 -1.625 -0.703125 -2.859375q-0.703125 -1.234375 -1.890625 -1.921875q-1.1875 -0.6875 -2.640625 -0.6875q-1.4375 0 -2.640625 0.6875q-1.1875 0.6875 -1.90625 1.921875q-0.703125 1.234375 -0.703125 2.859375q0 1.59375 0.703125 2.84375q0.71875 1.234375 1.90625 1.921875q1.203125 0.6875 2.640625 0.6875zm8.857132 5.25l0 -13.546875l1.515625 0l0 1.421875l0.078125 0q0.25 -0.453125 0.71875 -0.84375q0.484375 -0.390625 1.125 -0.625q0.640625 -0.25 1.390625 -0.25q1.3125 0 2.328125 0.65625q1.03125 0.65625 1.625 1.796875q0.609375 1.125 0.609375 2.609375q0 1.46875 -0.609375 2.609375q-0.59375 1.140625 -1.625 1.796875q-1.015625 0.640625 -2.328125 0.640625q-1.125 0 -1.984375 -0.515625q-0.859375 -0.53125 -1.25 -1.1875l-0.078125 0l0.078125 1.3125l0 4.125l-1.59375 0zm4.671875 -5.171875q0.8125 0 1.53125 -0.4375q0.71875 -0.4375 1.15625 -1.25q0.4375 -0.828125 0.4375 -1.921875q0 -1.125 -0.4375 -1.9375q-0.4375 -0.8125 -1.15625 -1.25q-0.71875 -0.4375 -1.53125 -0.4375q-0.828125 0 -1.546875 0.4375q-0.71875 0.4375 -1.171875 1.25q-0.4375 0.8125 -0.4375 1.9375q0 1.109375 0.4375 1.921875q0.453125 0.8125 1.171875 1.25q0.71875 0.4375 1.546875 0.4375zm10.793564 1.4375q-1.375 0 -2.453125 -0.640625q-1.0625 -0.65625 -1.671875 -1.796875q-0.609375 -1.140625 -0.609375 -2.59375q0 -1.359375 0.5625 -2.515625q0.578125 -1.15625 1.609375 -1.859375q1.03125 -0.703125 2.4375 -0.703125q1.421875 0 2.4375 0.625q1.015625 0.625 1.5625 1.734375q0.546875 1.09375 0.546875 2.515625q0 0.125 -0.015625 0.265625q0 0.125 -0.015625 0.21875l-8.1875 0l0 -1.3125l6.546875 0q-0.015625 -0.390625 -0.1875 -0.84375q-0.15625 -0.46875 -0.5 -0.859375q-0.34375 -0.40625 -0.875 -0.65625q-0.53125 -0.25 -1.3125 -0.25q-0.9375 0 -1.625 0.484375q-0.671875 0.46875 -1.046875 1.296875q-0.359375 0.8125 -0.359375 1.859375q0 1.203125 0.46875 2.015625q0.46875 0.796875 1.203125 1.1875q0.75 0.390625 1.546875 0.390625q1.046875 0 1.71875 -0.484375q0.6875 -0.5 1.09375 -1.234375l1.34375 0.65625q-0.5625 1.078125 -1.609375 1.796875q-1.03125 0.703125 -2.609375 0.703125zm6.2690887 -0.296875l0 -9.515625l1.515625 0l0 1.53125l0.078125 0q0.1875 -0.546875 0.625 -0.9375q0.4375 -0.40625 1.0 -0.640625q0.578125 -0.234375 1.125 -0.234375q0.4375 0 0.671875 0.046875q0.25 0.046875 0.453125 0.140625l0 1.71875q-0.296875 -0.15625 -0.640625 -0.21875q-0.34375 -0.078125 -0.703125 -0.078125q-0.6875 0 -1.265625 0.390625q-0.578125 0.390625 -0.921875 1.046875q-0.34375 0.65625 -0.34375 1.4375l0 5.3125l-1.59375 0zm9.669632 0.296875q-1.0625 0 -1.875 -0.40625q-0.796875 -0.40625 -1.25 -1.125q-0.453125 -0.71875 -0.453125 -1.640625q0 -1.046875 0.53125 -1.765625q0.546875 -0.71875 1.453125 -1.078125q0.921875 -0.359375 2.015625 -0.359375q0.640625 0 1.171875 0.109375q0.546875 0.09375 0.9375 0.234375q0.40625 0.140625 0.625 0.265625l0 -0.578125q0 -1.078125 -0.765625 -1.703125q-0.765625 -0.640625 -1.875 -0.640625q-0.78125 0 -1.46875 0.34375q-0.671875 0.34375 -1.0625 0.953125l-1.203125 -0.890625q0.375 -0.5625 0.9375 -0.96875q0.5625 -0.40625 1.28125 -0.625q0.71875 -0.234375 1.515625 -0.234375q1.9375 0 3.03125 1.03125q1.109375 1.015625 1.109375 2.75l0 6.03125l-1.5 0l0 -1.359375l-0.078125 0q-0.25 0.40625 -0.703125 0.796875q-0.4375 0.375 -1.046875 0.609375q-0.609375 0.25 -1.328125 0.25zm0.140625 -1.390625q0.828125 0 1.5 -0.40625q0.6875 -0.421875 1.09375 -1.109375q0.421875 -0.6875 0.421875 -1.515625q-0.4375 -0.296875 -1.078125 -0.484375q-0.640625 -0.1875 -1.40625 -0.1875q-1.359375 0 -2.0 0.5625q-0.640625 0.5625 -0.640625 1.375q0 0.78125 0.59375 1.28125q0.609375 0.484375 1.515625 0.484375zm6.7189636 1.09375l0 -9.515625l1.515625 0l0 1.40625l0.078125 0q0.375 -0.703125 1.21875 -1.203125q0.84375 -0.5 1.859375 -0.5q1.75 0 2.625 1.015625q0.890625 1.015625 0.890625 2.703125l0 6.09375l-1.578125 0l0 -5.859375q0 -1.375 -0.671875 -1.9375q-0.65625 -0.578125 -1.703125 -0.578125q-0.78125 0 -1.375 0.4375q-0.59375 0.4375 -0.9375 1.125q-0.328125 0.6875 -0.328125 1.453125l0 5.359375l-1.59375 0zm14.513031 0.296875q-1.28125 0 -2.328125 -0.640625q-1.03125 -0.65625 -1.625 -1.796875q-0.59375 -1.140625 -0.59375 -2.609375q0 -1.484375 0.59375 -2.609375q0.59375 -1.140625 1.625 -1.796875q1.046875 -0.65625 2.328125 -0.65625q0.765625 0 1.390625 0.25q0.640625 0.234375 1.109375 0.625q0.484375 0.390625 0.75 0.84375l0.078125 0l-0.078125 -1.328125l0 -3.9375l1.59375 0l0 13.359375l-1.515625 0l0 -1.40625l-0.078125 0q-0.265625 0.4375 -0.75 0.828125q-0.46875 0.390625 -1.109375 0.625q-0.625 0.25 -1.390625 0.25zm0.171875 -1.4375q0.828125 0 1.546875 -0.4375q0.71875 -0.4375 1.15625 -1.25q0.453125 -0.8125 0.453125 -1.921875q0 -1.125 -0.453125 -1.9375q-0.4375 -0.8125 -1.15625 -1.25q-0.71875 -0.4375 -1.546875 -0.4375q-0.8125 0 -1.53125 0.4375q-0.71875 0.4375 -1.171875 1.25q-0.453125 0.8125 -0.453125 1.9375q0 1.09375 0.453125 1.921875q0.453125 0.8125 1.171875 1.25q0.71875 0.4375 1.53125 0.4375zm5.871689 1.140625l4.078125 -13.359375l1.453125 0l-4.0625 13.359375l-1.46875 0zm12.879791 0.296875q-1.46875 0 -2.71875 -0.53125q-1.25 -0.546875 -2.1875 -1.5q-0.921875 -0.96875 -1.4375 -2.21875q-0.5 -1.265625 -0.5 -2.71875q0 -1.46875 0.5 -2.71875q0.515625 -1.265625 1.4375 -2.21875q0.9375 -0.96875 2.1875 -1.5q1.25 -0.546875 2.71875 -0.546875q1.46875 0 2.71875 0.546875q1.25 0.53125 2.171875 1.5q0.921875 0.953125 1.4375 2.21875q0.515625 1.25 0.515625 2.71875q0 1.453125 -0.515625 2.71875q-0.515625 1.25 -1.4375 2.21875q-0.921875 0.953125 -2.171875 1.5q-1.25 0.53125 -2.71875 0.53125zm0 -1.515625q1.453125 0 2.640625 -0.6875q1.1875 -0.6875 1.890625 -1.921875q0.703125 -1.25 0.703125 -2.84375q0 -1.625 -0.703125 -2.859375q-0.703125 -1.234375 -1.890625 -1.921875q-1.1875 -0.6875 -2.640625 -0.6875q-1.4375 0 -2.640625 0.6875q-1.1875 0.6875 -1.90625 1.921875q-0.703125 1.234375 -0.703125 2.859375q0 1.59375 0.703125 2.84375q0.71875 1.234375 1.90625 1.921875q1.203125 0.6875 2.640625 0.6875zm8.857132 5.25l0 -13.546875l1.515625 0l0 1.421875l0.078125 0q0.25 -0.453125 0.71875 -0.84375q0.484375 -0.390625 1.125 -0.625q0.640625 -0.25 1.390625 -0.25q1.3125 0 2.328125 0.65625q1.03125 0.65625 1.625 1.796875q0.609375 1.125 0.609375 2.609375q0 1.46875 -0.609375 2.609375q-0.59375 1.140625 -1.625 1.796875q-1.015625 0.640625 -2.328125 0.640625q-1.125 0 -1.984375 -0.515625q-0.859375 -0.53125 -1.25 -1.1875l-0.078125 0l0.078125 1.3125l0 4.125l-1.59375 0zm4.671875 -5.171875q0.8125 0 1.53125 -0.4375q0.71875 -0.4375 1.15625 -1.25q0.4375 -0.828125 0.4375 -1.921875q0 -1.125 -0.4375 -1.9375q-0.4375 -0.8125 -1.15625 -1.25q-0.71875 -0.4375 -1.53125 -0.4375q-0.828125 0 -1.546875 0.4375q-0.71875 0.4375 -1.171875 1.25q-0.4375 0.8125 -0.4375 1.9375q0 1.109375 0.4375 1.921875q0.453125 0.8125 1.171875 1.25q0.71875 0.4375 1.546875 0.4375zm6.887314 1.140625l0 -13.359375l4.5 0q1.125 0 2.0625 0.515625q0.953125 0.5 1.515625 1.390625q0.578125 0.875 0.578125 2.046875q0 0.71875 -0.265625 1.421875q-0.265625 0.6875 -0.8125 1.265625q-0.546875 0.578125 -1.40625 0.921875q-0.84375 0.34375 -2.015625 0.34375l-3.328125 0l0 -1.46875l3.578125 0q0.703125 0 1.296875 -0.3125q0.59375 -0.328125 0.96875 -0.875q0.375 -0.5625 0.375 -1.296875q0 -0.59375 -0.3125 -1.15625q-0.3125 -0.5625 -0.875 -0.921875q-0.546875 -0.359375 -1.328125 -0.359375l-2.953125 0l0 11.84375l-1.578125 0zm3.21875 -6.125l1.734375 -0.09375l4.21875 6.140625l0 0.078125l-1.859375 0l-4.09375 -6.125zm11.433411 6.421875q-1.375 0 -2.453125 -0.640625q-1.0625 -0.65625 -1.671875 -1.796875q-0.609375 -1.140625 -0.609375 -2.59375q0 -1.359375 0.5625 -2.515625q0.578125 -1.15625 1.609375 -1.859375q1.03125 -0.703125 2.4375 -0.703125q1.421875 0 2.4375 0.625q1.015625 0.625 1.5625 1.734375q0.546875 1.09375 0.546875 2.515625q0 0.125 -0.015625 0.265625q0 0.125 -0.015625 0.21875l-8.1875 0l0 -1.3125l6.546875 0q-0.015625 -0.390625 -0.1875 -0.84375q-0.15625 -0.46875 -0.5 -0.859375q-0.34375 -0.40625 -0.875 -0.65625q-0.53125 -0.25 -1.3125 -0.25q-0.9375 0 -1.625 0.484375q-0.671875 0.46875 -1.046875 1.296875q-0.359375 0.8125 -0.359375 1.859375q0 1.203125 0.46875 2.015625q0.46875 0.796875 1.203125 1.1875q0.75 0.390625 1.546875 0.390625q1.046875 0 1.71875 -0.484375q0.6875 -0.5 1.09375 -1.234375l1.34375 0.65625q-0.5625 1.078125 -1.609375 1.796875q-1.03125 0.703125 -2.609375 0.703125zm9.690552 0q-1.0625 0 -1.875 -0.34375q-0.8125 -0.34375 -1.34375 -0.921875q-0.53125 -0.59375 -0.796875 -1.28125l1.421875 -0.640625q0.375 0.859375 1.078125 1.328125q0.703125 0.46875 1.625 0.46875q0.875 0 1.453125 -0.359375q0.59375 -0.359375 0.59375 -1.046875q0 -0.421875 -0.25 -0.703125q-0.234375 -0.296875 -0.703125 -0.5q-0.453125 -0.21875 -1.125 -0.375l-1.15625 -0.3125q-0.671875 -0.1875 -1.28125 -0.515625q-0.59375 -0.34375 -0.953125 -0.875q-0.359375 -0.53125 -0.359375 -1.296875q0 -0.84375 0.5 -1.453125q0.5 -0.625 1.3125 -0.953125q0.828125 -0.328125 1.765625 -0.328125q0.8125 0 1.53125 0.234375q0.71875 0.234375 1.265625 0.6875q0.546875 0.453125 0.8125 1.125l-1.375 0.640625q-0.359375 -0.703125 -0.953125 -0.984375q-0.59375 -0.28125 -1.328125 -0.28125q-0.78125 0 -1.359375 0.34375q-0.578125 0.34375 -0.578125 0.9375q0 0.59375 0.46875 0.90625q0.484375 0.296875 1.171875 0.46875l1.390625 0.359375q1.390625 0.359375 2.09375 1.0625q0.71875 0.6875 0.71875 1.703125q0 0.890625 -0.515625 1.546875q-0.5 0.65625 -1.359375 1.015625q-0.84375 0.34375 -1.890625 0.34375zm8.949402 0q-1.75 0 -2.640625 -1.015625q-0.890625 -1.03125 -0.890625 -2.8125l0 -5.984375l1.59375 0l0 5.75q0 1.421875 0.65625 2.03125q0.65625 0.59375 1.625 0.59375q0.828125 0 1.4375 -0.4375q0.625 -0.4375 0.953125 -1.125q0.34375 -0.6875 0.34375 -1.4375l0 -5.375l1.59375 0l0 9.515625l-1.515625 0l0 -1.375l-0.078125 0q-0.234375 0.4375 -0.71875 0.828125q-0.46875 0.375 -1.078125 0.609375q-0.609375 0.234375 -1.28125 0.234375zm7.013031 -0.296875l0 -13.359375l1.59375 0l0 13.359375l-1.59375 0zm3.3735504 -9.515625l5.578125 0l0 1.4375l-5.578125 0l0 -1.4375zm1.671875 7.015625l0 -9.703125l1.578125 0l0 9.3125q0 0.75 0.3125 1.15625q0.3125 0.40625 1.015625 0.40625q0.3125 0 0.578125 -0.09375q0.265625 -0.09375 0.46875 -0.21875l0 1.546875q-0.25 0.109375 -0.546875 0.171875q-0.28125 0.078125 -0.765625 0.078125q-1.1875 0 -1.921875 -0.703125q-0.71875 -0.703125 -0.71875 -1.953125zm10.122162 6.53125l0 -13.546875l1.515625 0l0 1.421875l0.078125 0q0.25 -0.453125 0.71875 -0.84375q0.484375 -0.390625 1.125 -0.625q0.640625 -0.25 1.390625 -0.25q1.3125 0 2.328125 0.65625q1.03125 0.65625 1.625 1.796875q0.609375 1.125 0.609375 2.609375q0 1.46875 -0.609375 2.609375q-0.59375 1.140625 -1.625 1.796875q-1.015625 0.640625 -2.328125 0.640625q-1.125 0 -1.984375 -0.515625q-0.859375 -0.53125 -1.25 -1.1875l-0.078125 0l0.078125 1.3125l0 4.125l-1.59375 0zm4.671875 -5.171875q0.8125 0 1.53125 -0.4375q0.71875 -0.4375 1.15625 -1.25q0.4375 -0.828125 0.4375 -1.921875q0 -1.125 -0.4375 -1.9375q-0.4375 -0.8125 -1.15625 -1.25q-0.71875 -0.4375 -1.53125 -0.4375q-0.828125 0 -1.546875 0.4375q-0.71875 0.4375 -1.171875 1.25q-0.4375 0.8125 -0.4375 1.9375q0 1.109375 0.4375 1.921875q0.453125 0.8125 1.171875 1.25q0.71875 0.4375 1.546875 0.4375zm9.606064 1.4375q-1.0625 0 -1.875 -0.40625q-0.796875 -0.40625 -1.25 -1.125q-0.453125 -0.71875 -0.453125 -1.640625q0 -1.046875 0.53125 -1.765625q0.546875 -0.71875 1.453125 -1.078125q0.921875 -0.359375 2.015625 -0.359375q0.640625 0 1.171875 0.109375q0.546875 0.09375 0.9375 0.234375q0.40625 0.140625 0.625 0.265625l0 -0.578125q0 -1.078125 -0.765625 -1.703125q-0.765625 -0.640625 -1.875 -0.640625q-0.78125 0 -1.46875 0.34375q-0.671875 0.34375 -1.0625 0.953125l-1.203125 -0.890625q0.375 -0.5625 0.9375 -0.96875q0.5625 -0.40625 1.28125 -0.625q0.71875 -0.234375 1.515625 -0.234375q1.9375 0 3.03125 1.03125q1.109375 1.015625 1.109375 2.75l0 6.03125l-1.5 0l0 -1.359375l-0.078125 0q-0.25 0.40625 -0.703125 0.796875q-0.4375 0.375 -1.046875 0.609375q-0.609375 0.25 -1.328125 0.25zm0.140625 -1.390625q0.828125 0 1.5 -0.40625q0.6875 -0.421875 1.09375 -1.109375q0.421875 -0.6875 0.421875 -1.515625q-0.4375 -0.296875 -1.078125 -0.484375q-0.640625 -0.1875 -1.40625 -0.1875q-1.359375 0 -2.0 0.5625q-0.640625 0.5625 -0.640625 1.375q0 0.78125 0.59375 1.28125q0.609375 0.484375 1.515625 0.484375zm6.8751984 1.09375l0 -9.515625l1.578125 0l0 9.515625l-1.578125 0zm0.78125 -11.265625q-0.46875 0 -0.8125 -0.328125q-0.328125 -0.34375 -0.328125 -0.8125q0 -0.484375 0.328125 -0.8125q0.34375 -0.328125 0.8125 -0.328125q0.484375 0 0.8125 0.328125q0.328125 0.328125 0.328125 0.8125q0 0.46875 -0.328125 0.8125q-0.328125 0.328125 -0.8125 0.328125zm3.2788086 11.265625l0 -9.515625l1.515625 0l0 1.53125l0.078125 0q0.1875 -0.546875 0.625 -0.9375q0.4375 -0.40625 1.0 -0.640625q0.578125 -0.234375 1.125 -0.234375q0.4375 0 0.671875 0.046875q0.25 0.046875 0.453125 0.140625l0 1.71875q-0.296875 -0.15625 -0.640625 -0.21875q-0.34375 -0.078125 -0.703125 -0.078125q-0.6875 0 -1.265625 0.390625q-0.578125 0.390625 -0.921875 1.046875q-0.34375 0.65625 -0.34375 1.4375l0 5.3125l-1.59375 0z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m114.61155 124.00787l-11.527557 -43.244095" fill-rule="evenodd"/><path stroke="#ff0000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m114.61155 124.00787l-9.982109 -37.44654" fill-rule="evenodd"/><path fill="#ff0000" stroke="#ff0000" stroke-width="1.0" stroke-linecap="butt" d="m106.22544 86.13589l-2.7649002 -3.9595337l-0.42710114 4.810417z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m51.220474 125.220474l604.50397 0l0 32.72441l-604.50397 0z" fill-rule="evenodd"/><path fill="#000000" d="m60.845474 152.14047l2.78125 -13.359375l1.65625 0l-1.0 4.78125q0.78125 -0.71875 1.421875 -1.015625q0.640625 -0.296875 1.328125 -0.296875q1.359375 0 2.265625 1.015625q0.90625 1.0 0.90625 2.9375q0 1.28125 -0.375 2.359375q-0.359375 1.0625 -0.890625 1.78125q-0.53125 0.71875 -1.109375 1.15625q-0.578125 0.4375 -1.1875 0.640625q-0.59375 0.21875 -1.140625 0.21875q-0.96875 0 -1.703125 -0.5q-0.71875 -0.515625 -1.125 -1.546875l-0.390625 1.828125l-1.4375 0zm2.4375 -3.96875l-0.015625 0.3125q0 1.234375 0.59375 1.890625q0.59375 0.640625 1.484375 0.640625q0.859375 0 1.578125 -0.609375q0.734375 -0.609375 1.1875 -1.890625q0.46875 -1.28125 0.46875 -2.375q0 -1.21875 -0.59375 -1.890625q-0.578125 -0.671875 -1.4375 -0.671875q-0.90625 0 -1.65625 0.6875q-0.734375 0.6875 -1.234375 2.125q-0.375 1.0625 -0.375 1.78125zm14.531967 2.21875q-1.734375 1.96875 -3.5625 1.96875q-1.109375 0 -1.796875 -0.640625q-0.6875 -0.640625 -0.6875 -1.578125q0 -0.609375 0.296875 -2.09375l1.171875 -5.578125l1.65625 0l-1.296875 6.1875q-0.171875 0.765625 -0.171875 1.203125q0 0.546875 0.328125 0.859375q0.34375 0.296875 0.984375 0.296875q0.703125 0 1.359375 -0.328125q0.65625 -0.34375 1.125 -0.921875q0.484375 -0.578125 0.796875 -1.359375q0.1875 -0.5 0.453125 -1.765625l0.875 -4.171875l1.65625 0l-2.03125 9.671875l-1.515625 0l0.359375 -1.75zm4.000717 1.75l1.765625 -8.40625l-1.484375 0l0.265625 -1.265625l1.484375 0l0.28125 -1.375q0.21875 -1.03125 0.4375 -1.484375q0.234375 -0.453125 0.75 -0.75q0.53125 -0.296875 1.4375 -0.296875q0.625 0 1.828125 0.265625l-0.296875 1.4375q-0.84375 -0.21875 -1.40625 -0.21875q-0.484375 0 -0.734375 0.25q-0.25 0.234375 -0.4375 1.125l-0.21875 1.046875l1.84375 0l-0.265625 1.265625l-1.84375 0l-1.75 8.40625l-1.65625 0zm5.183304 0l1.765625 -8.40625l-1.484375 0l0.265625 -1.265625l1.484375 0l0.28125 -1.375q0.21875 -1.03125 0.4375 -1.484375q0.234375 -0.453125 0.75 -0.75q0.53125 -0.296875 1.4375 -0.296875q0.625 0 1.828125 0.265625l-0.296875 1.4375q-0.84375 -0.21875 -1.40625 -0.21875q-0.484375 0 -0.734375 0.25q-0.25 0.234375 -0.4375 1.125l-0.21875 1.046875l1.84375 0l-0.265625 1.265625l-1.84375 0l-1.75 8.40625l-1.65625 0zm12.058304 -3.28125l1.609375 0.15625q-0.34375 1.1875 -1.59375 2.265625q-1.234375 1.078125 -2.96875 1.078125q-1.0625 0 -1.96875 -0.5q-0.890625 -0.5 -1.359375 -1.4375q-0.46875 -0.953125 -0.46875 -2.15625q0 -1.59375 0.734375 -3.078125q0.734375 -1.484375 1.890625 -2.203125q1.171875 -0.734375 2.53125 -0.734375q1.734375 0 2.765625 1.078125q1.03125 1.0625 1.03125 2.921875q0 0.71875 -0.125 1.46875l-7.125 0q-0.046875 0.28125 -0.046875 0.5q0 1.359375 0.625 2.078125q0.625 0.71875 1.53125 0.71875q0.84375 0 1.65625 -0.546875q0.828125 -0.5625 1.28125 -1.609375zm-4.78125 -2.40625l5.421875 0q0.015625 -0.25 0.015625 -0.359375q0 -1.234375 -0.625 -1.890625q-0.625 -0.671875 -1.59375 -0.671875q-1.0625 0 -1.9375 0.734375q-0.859375 0.71875 -1.28125 2.1875zm8.063217 5.6875l2.015625 -9.671875l1.453125 0l-0.40625 1.96875q0.75 -1.109375 1.453125 -1.640625q0.71875 -0.546875 1.46875 -0.546875q0.5 0 1.21875 0.359375l-0.671875 1.53125q-0.4375 -0.3125 -0.9375 -0.3125q-0.875 0 -1.78125 0.96875q-0.90625 0.953125 -1.4375 3.46875l-0.8125 3.875l-1.5625 0z" fill-rule="nonzero"/><path fill="#000000" d="m112.286285 156.06235q-1.359375 -1.703125 -2.296875 -4.0q-0.9375 -2.296875 -0.9375 -4.765625q0 -2.15625 0.703125 -4.140625q0.828125 -2.3125 2.53125 -4.59375l1.171875 0q-1.09375 1.890625 -1.453125 2.703125q-0.546875 1.25 -0.875 2.625q-0.390625 1.703125 -0.390625 3.421875q0 4.375 2.71875 8.75l-1.171875 0z" fill-rule="nonzero"/><path fill="#000000" d="m124.21771 139.26547l-8.40625 12.875l-1.484375 0l8.375 -12.875l1.515625 0zm-4.859375 2.546875q0 0.59375 -0.171875 1.109375q-0.171875 0.5 -0.5 0.875q-0.3125 0.359375 -0.78125 0.578125q-0.453125 0.21875 -1.03125 0.21875q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.515625q-0.3125 -0.34375 -0.484375 -0.84375q-0.15625 -0.5 -0.15625 -1.15625q0 -0.59375 0.171875 -1.09375q0.171875 -0.5 0.484375 -0.875q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.21875 1.03125 -0.21875q0.5625 0 1.0 0.171875q0.4375 0.171875 0.75 0.515625q0.3125 0.328125 0.484375 0.828125q0.171875 0.5 0.171875 1.15625zm-1.34375 0.046875q0 -0.796875 -0.296875 -1.171875q-0.28125 -0.390625 -0.796875 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.34375 0.328125q-0.140625 0.203125 -0.21875 0.5q-0.078125 0.28125 -0.078125 0.609375q0 0.8125 0.296875 1.203125q0.296875 0.375 0.8125 0.375q0.265625 0 0.46875 -0.125q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.203125 0.203125 -0.484375q0.078125 -0.296875 0.078125 -0.640625zm6.046875 7.65625q0 0.578125 -0.171875 1.09375q-0.171875 0.5 -0.5 0.875q-0.328125 0.359375 -0.796875 0.578125q-0.453125 0.203125 -1.015625 0.203125q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.5q-0.3125 -0.34375 -0.484375 -0.84375q-0.171875 -0.515625 -0.171875 -1.171875q0 -0.578125 0.171875 -1.078125q0.1875 -0.515625 0.5 -0.890625q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.203125 1.03125 -0.203125q0.5625 0 1.0 0.171875q0.453125 0.171875 0.765625 0.515625q0.3125 0.328125 0.46875 0.84375q0.171875 0.5 0.171875 1.15625zm-1.328125 0.03125q0 -0.796875 -0.296875 -1.171875q-0.296875 -0.390625 -0.8125 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.359375 0.328125q-0.140625 0.203125 -0.21875 0.484375q-0.0625 0.28125 -0.0625 0.625q0 0.796875 0.28125 1.1875q0.296875 0.375 0.828125 0.375q0.265625 0 0.46875 -0.109375q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.21875 0.21875 -0.5q0.078125 -0.296875 0.078125 -0.625zm3.4916687 -6.546875l1.453125 0l0.046875 1.6875q0.8125 -0.984375 1.59375 -1.421875q0.796875 -0.4375 1.59375 -0.4375q1.421875 0 2.15625 0.921875q0.734375 0.921875 0.671875 2.734375l-1.59375 0q0.015625 -1.203125 -0.359375 -1.734375q-0.375 -0.546875 -1.109375 -0.546875q-0.3125 0 -0.640625 0.109375q-0.328125 0.109375 -0.671875 0.359375q-0.328125 0.25 -0.71875 0.640625q-0.375 0.390625 -0.8125 0.953125l0 5.875l-1.609375 0l0 -9.140625z" fill-rule="nonzero"/><path fill="#000000" d="m136.96667 156.06235l-1.1875 0q2.734375 -4.375 2.734375 -8.75q0 -1.71875 -0.390625 -3.390625q-0.3125 -1.375 -0.875 -2.625q-0.359375 -0.828125 -1.46875 -2.734375l1.1875 0q1.703125 2.28125 2.53125 4.59375q0.6875 1.984375 0.6875 4.140625q0 2.46875 -0.9375 4.765625q-0.9375 2.296875 -2.28125 4.0zm10.770981 -11.734375l0 -1.859375l1.859375 0l0 1.859375l-1.859375 0zm0 7.8125l0 -1.875l1.859375 0l0 1.875l-1.859375 0zm13.355179 -7.859375l-8.828125 0l0 -1.515625l8.828125 0l0 1.515625zm0 4.0625l-8.828125 0l0 -1.53125l8.828125 0l0 1.53125z" fill-rule="nonzero"/><path fill="#000000" d="m167.93672 152.14047l2.78125 -13.359375l1.65625 0l-1.0 4.78125q0.78125 -0.71875 1.421875 -1.015625q0.640625 -0.296875 1.328125 -0.296875q1.359375 0 2.265625 1.015625q0.90625 1.0 0.90625 2.9375q0 1.28125 -0.375 2.359375q-0.359375 1.0625 -0.890625 1.78125q-0.53125 0.71875 -1.109375 1.15625q-0.578125 0.4375 -1.1875 0.640625q-0.59375 0.21875 -1.140625 0.21875q-0.96875 0 -1.703125 -0.5q-0.71875 -0.515625 -1.125 -1.546875l-0.390625 1.828125l-1.4375 0zm2.4375 -3.96875l-0.015625 0.3125q0 1.234375 0.59375 1.890625q0.59375 0.640625 1.484375 0.640625q0.859375 0 1.578125 -0.609375q0.734375 -0.609375 1.1875 -1.890625q0.46875 -1.28125 0.46875 -2.375q0 -1.21875 -0.59375 -1.890625q-0.578125 -0.671875 -1.4375 -0.671875q-0.90625 0 -1.65625 0.6875q-0.734375 0.6875 -1.234375 2.125q-0.375 1.0625 -0.375 1.78125zm14.531967 2.21875q-1.734375 1.96875 -3.5625 1.96875q-1.109375 0 -1.796875 -0.640625q-0.6875 -0.640625 -0.6875 -1.578125q0 -0.609375 0.296875 -2.09375l1.171875 -5.578125l1.65625 0l-1.296875 6.1875q-0.171875 0.765625 -0.171875 1.203125q0 0.546875 0.328125 0.859375q0.34375 0.296875 0.984375 0.296875q0.703125 0 1.359375 -0.328125q0.65625 -0.34375 1.125 -0.921875q0.484375 -0.578125 0.796875 -1.359375q0.1875 -0.5 0.453125 -1.765625l0.875 -4.171875l1.65625 0l-2.03125 9.671875l-1.515625 0l0.359375 -1.75zm4.000717 1.75l1.765625 -8.40625l-1.484375 0l0.265625 -1.265625l1.484375 0l0.28125 -1.375q0.21875 -1.03125 0.4375 -1.484375q0.234375 -0.453125 0.75 -0.75q0.53125 -0.296875 1.4375 -0.296875q0.625 0 1.828125 0.265625l-0.296875 1.4375q-0.84375 -0.21875 -1.40625 -0.21875q-0.484375 0 -0.734375 0.25q-0.25 0.234375 -0.4375 1.125l-0.21875 1.046875l1.84375 0l-0.265625 1.265625l-1.84375 0l-1.75 8.40625l-1.65625 0zm5.183304 0l1.765625 -8.40625l-1.484375 0l0.265625 -1.265625l1.484375 0l0.28125 -1.375q0.21875 -1.03125 0.4375 -1.484375q0.234375 -0.453125 0.75 -0.75q0.53125 -0.296875 1.4375 -0.296875q0.625 0 1.828125 0.265625l-0.296875 1.4375q-0.84375 -0.21875 -1.40625 -0.21875q-0.484375 0 -0.734375 0.25q-0.25 0.234375 -0.4375 1.125l-0.21875 1.046875l1.84375 0l-0.265625 1.265625l-1.84375 0l-1.75 8.40625l-1.65625 0zm12.058304 -3.28125l1.609375 0.15625q-0.34375 1.1875 -1.59375 2.265625q-1.234375 1.078125 -2.96875 1.078125q-1.0625 0 -1.96875 -0.5q-0.890625 -0.5 -1.359375 -1.4375q-0.46875 -0.953125 -0.46875 -2.15625q0 -1.59375 0.734375 -3.078125q0.734375 -1.484375 1.890625 -2.203125q1.171875 -0.734375 2.53125 -0.734375q1.734375 0 2.765625 1.078125q1.03125 1.0625 1.03125 2.921875q0 0.71875 -0.125 1.46875l-7.125 0q-0.046875 0.28125 -0.046875 0.5q0 1.359375 0.625 2.078125q0.625 0.71875 1.53125 0.71875q0.84375 0 1.65625 -0.546875q0.828125 -0.5625 1.28125 -1.609375zm-4.78125 -2.40625l5.421875 0q0.015625 -0.25 0.015625 -0.359375q0 -1.234375 -0.625 -1.890625q-0.625 -0.671875 -1.59375 -0.671875q-1.0625 0 -1.9375 0.734375q-0.859375 0.71875 -1.28125 2.1875zm8.063217 5.6875l2.015625 -9.671875l1.453125 0l-0.40625 1.96875q0.75 -1.109375 1.453125 -1.640625q0.71875 -0.546875 1.46875 -0.546875q0.5 0 1.21875 0.359375l-0.671875 1.53125q-0.4375 -0.3125 -0.9375 -0.3125q-0.875 0 -1.78125 0.96875q-0.90625 0.953125 -1.4375 3.46875l-0.8125 3.875l-1.5625 0z" fill-rule="nonzero"/><path fill="#000000" d="m219.37753 156.06235q-1.359375 -1.703125 -2.296875 -4.0q-0.9375 -2.296875 -0.9375 -4.765625q0 -2.15625 0.703125 -4.140625q0.828125 -2.3125 2.53125 -4.59375l1.171875 0q-1.09375 1.890625 -1.453125 2.703125q-0.546875 1.25 -0.875 2.625q-0.390625 1.703125 -0.390625 3.421875q0 4.375 2.71875 8.75l-1.171875 0z" fill-rule="nonzero"/><path fill="#000000" d="m231.30896 139.26547l-8.40625 12.875l-1.484375 0l8.375 -12.875l1.515625 0zm-4.859375 2.546875q0 0.59375 -0.171875 1.109375q-0.171875 0.5 -0.5 0.875q-0.3125 0.359375 -0.78125 0.578125q-0.453125 0.21875 -1.03125 0.21875q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.515625q-0.3125 -0.34375 -0.484375 -0.84375q-0.15625 -0.5 -0.15625 -1.15625q0 -0.59375 0.171875 -1.09375q0.171875 -0.5 0.484375 -0.875q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.21875 1.03125 -0.21875q0.5625 0 1.0 0.171875q0.4375 0.171875 0.75 0.515625q0.3125 0.328125 0.484375 0.828125q0.171875 0.5 0.171875 1.15625zm-1.34375 0.046875q0 -0.796875 -0.296875 -1.171875q-0.28125 -0.390625 -0.796875 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.34375 0.328125q-0.140625 0.203125 -0.21875 0.5q-0.078125 0.28125 -0.078125 0.609375q0 0.8125 0.296875 1.203125q0.296875 0.375 0.8125 0.375q0.265625 0 0.46875 -0.125q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.203125 0.203125 -0.484375q0.078125 -0.296875 0.078125 -0.640625zm6.046875 7.65625q0 0.578125 -0.171875 1.09375q-0.171875 0.5 -0.5 0.875q-0.328125 0.359375 -0.796875 0.578125q-0.453125 0.203125 -1.015625 0.203125q-0.546875 0 -1.0 -0.171875q-0.4375 -0.171875 -0.75 -0.5q-0.3125 -0.34375 -0.484375 -0.84375q-0.171875 -0.515625 -0.171875 -1.171875q0 -0.578125 0.171875 -1.078125q0.1875 -0.515625 0.5 -0.890625q0.328125 -0.375 0.78125 -0.578125q0.46875 -0.203125 1.03125 -0.203125q0.5625 0 1.0 0.171875q0.453125 0.171875 0.765625 0.515625q0.3125 0.328125 0.46875 0.84375q0.171875 0.5 0.171875 1.15625zm-1.328125 0.03125q0 -0.796875 -0.296875 -1.171875q-0.296875 -0.390625 -0.8125 -0.390625q-0.265625 0 -0.46875 0.125q-0.203125 0.109375 -0.359375 0.328125q-0.140625 0.203125 -0.21875 0.484375q-0.0625 0.28125 -0.0625 0.625q0 0.796875 0.28125 1.1875q0.296875 0.375 0.828125 0.375q0.265625 0 0.46875 -0.109375q0.203125 -0.125 0.34375 -0.328125q0.140625 -0.21875 0.21875 -0.5q0.078125 -0.296875 0.078125 -0.625zm10.491669 2.46875q-0.53125 0.140625 -1.109375 0.1875q-0.578125 0.0625 -1.171875 0.0625q-1.71875 0 -2.5625 -0.78125q-0.84375 -0.78125 -0.84375 -2.390625l0 -4.765625l-2.5625 0l0 -1.328125l2.5625 0l0 -2.515625l1.578125 -0.40625l0 2.921875l4.109375 0l0 1.328125l-4.109375 0l0 4.640625q0 0.984375 0.515625 1.46875q0.53125 0.484375 1.546875 0.484375q0.4375 0 0.953125 -0.0625q0.53125 -0.0625 1.09375 -0.21875l0 1.375z" fill-rule="nonzero"/><path fill="#000000" d="m244.05792 156.06235l-1.1875 0q2.734375 -4.375 2.734375 -8.75q0 -1.71875 -0.390625 -3.390625q-0.3125 -1.375 -0.875 -2.625q-0.359375 -0.828125 -1.46875 -2.734375l1.1875 0q1.703125 2.28125 2.53125 4.59375q0.6875 1.984375 0.6875 4.140625q0 2.46875 -0.9375 4.765625q-0.9375 2.296875 -2.28125 4.0zm9.708481 -8.765625q0 -2.6875 1.484375 -3.96875q1.2499847 -1.078125 3.0468597 -1.078125q2.0 0 3.265625 1.3125q1.265625 1.296875 1.265625 3.609375q0 1.859375 -0.5625 2.9375q-0.5625 1.0625 -1.640625 1.65625q-1.0625 0.59375 -2.328125 0.59375q-2.03125 0 -3.2812347 -1.296875q-1.25 -1.3125 -1.25 -3.765625zm1.6875 0q0 1.859375 0.79685974 2.796875q0.8125 0.921875 2.046875 0.921875q1.21875 0 2.03125 -0.921875q0.8125 -0.9375 0.8125 -2.84375q0 -1.796875 -0.8125 -2.71875q-0.8125 -0.921875 -2.03125 -0.921875q-1.234375 0 -2.046875 0.921875q-0.79685974 0.90625 -0.79685974 2.765625zm9.281967 4.84375l0 -9.671875l1.46875 0l0 1.46875q0.5625 -1.03125 1.03125 -1.359375q0.484375 -0.328125 1.0625 -0.328125q0.828125 0 1.6875 0.53125l-0.5625 1.515625q-0.609375 -0.359375 -1.203125 -0.359375q-0.546875 0 -0.96875 0.328125q-0.421875 0.328125 -0.609375 0.890625q-0.28125 0.875 -0.28125 1.921875l0 5.0625l-1.625 0zm11.411591 0l0 -9.671875l1.46875 0l0 1.375q1.0625 -1.59375 3.078125 -1.59375q0.875 0 1.609375 0.3125q0.734375 0.3125 1.09375 0.828125q0.375 0.5 0.515625 1.203125q0.09375 0.453125 0.09375 1.59375l0 5.953125l-1.640625 0l0 -5.890625q0 -1.0 -0.203125 -1.484375q-0.1875 -0.5 -0.671875 -0.796875q-0.484375 -0.296875 -1.140625 -0.296875q-1.046875 0 -1.8125 0.671875q-0.75 0.65625 -0.75 2.515625l0 5.28125l-1.640625 0zm17.000732 -3.109375l1.6875 0.203125q-0.40625 1.484375 -1.484375 2.3125q-1.078125 0.8125 -2.765625 0.8125q-2.125 0 -3.375 -1.296875q-1.234375 -1.3125 -1.234375 -3.671875q0 -2.453125 1.25 -3.796875q1.265625 -1.34375 3.265625 -1.34375q1.9375 0 3.15625 1.328125q1.234375 1.3125 1.234375 3.703125q0 0.15625 0 0.4375l-7.21875 0q0.09375 1.59375 0.90625 2.453125q0.8125 0.84375 2.015625 0.84375q0.90625 0 1.546875 -0.46875q0.640625 -0.484375 1.015625 -1.515625zm-5.390625 -2.65625l5.40625 0q-0.109375 -1.21875 -0.625 -1.828125q-0.78125 -0.953125 -2.03125 -0.953125q-1.125 0 -1.90625 0.765625q-0.765625 0.75 -0.84375 2.015625zm10.922577 5.765625l-2.96875 -9.671875l1.703125 0l1.53125 5.578125l0.578125 2.078125q0.046875 -0.15625 0.5 -2.0l1.546875 -5.65625l1.6875 0l1.4375 5.609375l0.484375 1.84375l0.5625 -1.859375l1.65625 -5.59375l1.59375 0l-3.03125 9.671875l-1.703125 0l-1.53125 -5.796875l-0.375 -1.640625l-1.953125 7.4375l-1.71875 0zm23.1875 -1.1875q-0.921875 0.765625 -1.765625 1.09375q-0.828125 0.3125 -1.796875 0.3125q-1.59375 0 -2.453125 -0.78125q-0.859375 -0.78125 -0.859375 -1.984375q0 -0.71875 0.328125 -1.296875q0.328125 -0.59375 0.84375 -0.9375q0.53125 -0.359375 1.1875 -0.546875q0.46875 -0.125 1.453125 -0.25q1.984375 -0.234375 2.921875 -0.5625q0.015625 -0.34375 0.015625 -0.421875q0 -1.0 -0.46875 -1.421875q-0.625 -0.546875 -1.875 -0.546875q-1.15625 0 -1.703125 0.40625q-0.546875 0.40625 -0.8125 1.421875l-1.609375 -0.21875q0.21875 -1.015625 0.71875 -1.640625q0.5 -0.640625 1.453125 -0.984375q0.953125 -0.34375 2.1875 -0.34375q1.25 0 2.015625 0.296875q0.78125 0.28125 1.140625 0.734375q0.375 0.4375 0.515625 1.109375q0.078125 0.421875 0.078125 1.515625l0 2.1875q0 2.28125 0.109375 2.890625q0.109375 0.59375 0.40625 1.15625l-1.703125 0q-0.265625 -0.515625 -0.328125 -1.1875zm-0.140625 -3.671875q-0.890625 0.375 -2.671875 0.625q-1.015625 0.140625 -1.4375 0.328125q-0.421875 0.1875 -0.65625 0.53125q-0.21875 0.34375 -0.21875 0.78125q0 0.65625 0.5 1.09375q0.5 0.4375 1.453125 0.4375q0.9375 0 1.671875 -0.40625q0.75 -0.421875 1.09375 -1.140625q0.265625 -0.5625 0.265625 -1.640625l0 -0.609375zm4.156952 4.859375l0 -13.359375l1.640625 0l0 13.359375l-1.640625 0zm4.1448364 0l0 -13.359375l1.640625 0l0 13.359375l-1.640625 0zm3.5823364 -4.84375q0 -2.6875 1.484375 -3.96875q1.25 -1.078125 3.046875 -1.078125q2.0 0 3.265625 1.3125q1.265625 1.296875 1.265625 3.609375q0 1.859375 -0.5625 2.9375q-0.5625 1.0625 -1.640625 1.65625q-1.0625 0.59375 -2.328125 0.59375q-2.03125 0 -3.28125 -1.296875q-1.25 -1.3125 -1.25 -3.765625zm1.6875 0q0 1.859375 0.796875 2.796875q0.8125 0.921875 2.046875 0.921875q1.21875 0 2.03125 -0.921875q0.8125 -0.9375 0.8125 -2.84375q0 -1.796875 -0.8125 -2.71875q-0.8125 -0.921875 -2.03125 -0.921875q-1.234375 0 -2.046875 0.921875q-0.796875 0.90625 -0.796875 2.765625zm15.610077 1.296875l1.609375 0.21875q-0.265625 1.65625 -1.359375 2.609375q-1.078125 0.9375 -2.671875 0.9375q-1.984375 0 -3.1875 -1.296875q-1.203125 -1.296875 -1.203125 -3.71875q0 -1.578125 0.515625 -2.75q0.515625 -1.171875 1.578125 -1.75q1.0625 -0.59375 2.3125 -0.59375q1.578125 0 2.578125 0.796875q1.0 0.796875 1.28125 2.265625l-1.59375 0.234375q-0.234375 -0.96875 -0.8125 -1.453125q-0.578125 -0.5 -1.390625 -0.5q-1.234375 0 -2.015625 0.890625q-0.78125 0.890625 -0.78125 2.8125q0 1.953125 0.75 2.84375q0.75 0.875 1.953125 0.875q0.96875 0 1.609375 -0.59375q0.65625 -0.59375 0.828125 -1.828125z" fill-rule="nonzero"/></g></svg>
\ No newline at end of file
-- 
GitLab


From ab3a9e724d87a4272782f76b90fb0872a6a86939 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Tue, 7 May 2024 10:47:41 -0400
Subject: [PATCH 0054/1206] [libc] clean up futex usage (#91163)

# Motivation

Futex syscalls are widely used in our codebase as synchronization
mechanism. Hence, it may be worthy to abstract out the most common
routines (wait and wake). On the other hand, C++20 also provides
`std::atomic_notify_one/std::atomic_wait/std::atomic_notify_all` which
align with such functionalities. This PR introduces `Futex` as a subtype
of `cpp::Atomic<FutexWordType>` with additional
`notify_one/notify_all/wait` operations.

Providing such wrappers also make future porting easier. For example,
FreeBSD's `_umtx_op` and Darwin's `ulock` can be wrapped in a similar
manner.

### Similar Examples

1. [bionic
futex](https://android.googlesource.com/platform/bionic/+/refs/heads/main/libc/bionic/bionic_futex.cpp)
2. [futex in Rust's
std](https://github.com/rust-lang/rust/blob/8cef37dbb67e9c80702925f19cf298c4203991e4/library/std/src/sys/pal/unix/futex.rs#L21)
---
 .../__support/threads/linux/CMakeLists.txt    | 24 +++--
 libc/src/__support/threads/linux/callonce.cpp | 24 ++---
 .../src/__support/threads/linux/futex_utils.h | 90 +++++++++++++++++++
 libc/src/__support/threads/linux/futex_word.h |  1 -
 libc/src/__support/threads/linux/mutex.h      | 22 ++---
 libc/src/__support/threads/linux/thread.cpp   | 25 +++---
 libc/src/__support/threads/mutex.h            |  4 +-
 libc/src/__support/threads/thread.cpp         |  4 +-
 libc/src/threads/linux/CMakeLists.txt         |  2 +-
 libc/src/threads/linux/CndVar.h               |  9 +-
 10 files changed, 136 insertions(+), 69 deletions(-)
 create mode 100644 libc/src/__support/threads/linux/futex_utils.h

diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index 87a7a66ac6ea..b277c2a37f2d 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -9,14 +9,25 @@ if(NOT TARGET libc.src.__support.OSUtil.osutil)
 endif()
 
 add_header_library(
-  mutex
+  futex_utils
   HDRS
-    mutex.h
+    futex_utils.h
   DEPENDS
     .futex_word_type
     libc.include.sys_syscall
-    libc.src.__support.CPP.atomic
     libc.src.__support.OSUtil.osutil
+    libc.src.__support.CPP.atomic
+    libc.src.__support.CPP.limits
+    libc.src.__support.CPP.optional
+    libc.hdr.types.struct_timespec
+)
+
+add_header_library(
+  mutex
+  HDRS
+    mutex.h
+  DEPENDS
+    .futex
     libc.src.__support.threads.mutex_common
 )
 
@@ -25,7 +36,7 @@ add_object_library(
   SRCS
     thread.cpp
   DEPENDS
-    .futex_word_type
+    .futex_utils
     libc.config.linux.app_h
     libc.include.sys_syscall
     libc.src.errno.errno
@@ -50,8 +61,5 @@ add_object_library(
   HDRS
     ../callonce.h
   DEPENDS
-    libc.include.sys_syscall
-    libc.src.__support.CPP.atomic
-    libc.src.__support.CPP.limits
-    libc.src.__support.OSUtil.osutil
+    .futex_utils
 )
diff --git a/libc/src/__support/threads/linux/callonce.cpp b/libc/src/__support/threads/linux/callonce.cpp
index b6a5ab8c0d07..1c29db5f5c1a 100644
--- a/libc/src/__support/threads/linux/callonce.cpp
+++ b/libc/src/__support/threads/linux/callonce.cpp
@@ -6,15 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "futex_word.h"
-
-#include "src/__support/CPP/atomic.h"
-#include "src/__support/CPP/limits.h"     // INT_MAX
-#include "src/__support/OSUtil/syscall.h" // For syscall functions.
 #include "src/__support/threads/callonce.h"
-
-#include <linux/futex.h>
-#include <sys/syscall.h> // For syscall numbers.
+#include "src/__support/threads/linux/futex_utils.h"
 
 namespace LIBC_NAMESPACE {
 
@@ -24,7 +17,7 @@ static constexpr FutexWordType WAITING = 0x22;
 static constexpr FutexWordType FINISH = 0x33;
 
 int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
-  auto *futex_word = reinterpret_cast<cpp::Atomic<FutexWordType> *>(flag);
+  auto *futex_word = reinterpret_cast<Futex *>(flag);
 
   FutexWordType not_called = NOT_CALLED;
 
@@ -33,22 +26,15 @@ int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
   if (futex_word->compare_exchange_strong(not_called, START)) {
     func();
     auto status = futex_word->exchange(FINISH);
-    if (status == WAITING) {
-      LIBC_NAMESPACE::syscall_impl<long>(FUTEX_SYSCALL_ID, &futex_word->val,
-                                         FUTEX_WAKE_PRIVATE,
-                                         INT_MAX, // Wake all waiters.
-                                         0, 0, 0);
-    }
+    if (status == WAITING)
+      futex_word->notify_all();
     return 0;
   }
 
   FutexWordType status = START;
   if (futex_word->compare_exchange_strong(status, WAITING) ||
       status == WAITING) {
-    LIBC_NAMESPACE::syscall_impl<long>(
-        FUTEX_SYSCALL_ID, &futex_word->val, FUTEX_WAIT_PRIVATE,
-        WAITING, // Block only if status is still |WAITING|.
-        0, 0, 0);
+    futex_word->wait(WAITING);
   }
 
   return 0;
diff --git a/libc/src/__support/threads/linux/futex_utils.h b/libc/src/__support/threads/linux/futex_utils.h
new file mode 100644
index 000000000000..1fbce4f7bf43
--- /dev/null
+++ b/libc/src/__support/threads/linux/futex_utils.h
@@ -0,0 +1,90 @@
+//===--- Futex Wrapper ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_UTILS_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_UTILS_H
+
+#include "hdr/types/struct_timespec.h"
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/OSUtil/syscall.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/threads/linux/futex_word.h"
+#include <linux/errno.h>
+#include <linux/futex.h>
+
+namespace LIBC_NAMESPACE {
+class Futex : public cpp::Atomic<FutexWordType> {
+public:
+  struct Timeout {
+    timespec abs_time;
+    bool is_realtime;
+  };
+  LIBC_INLINE constexpr Futex(FutexWordType value)
+      : cpp::Atomic<FutexWordType>(value) {}
+  LIBC_INLINE Futex &operator=(FutexWordType value) {
+    cpp::Atomic<FutexWordType>::store(value);
+    return *this;
+  }
+  LIBC_INLINE long wait(FutexWordType expected,
+                        cpp::optional<Timeout> timeout = cpp::nullopt,
+                        bool is_shared = false) {
+    // use bitset variants to enforce abs_time
+    uint32_t op = is_shared ? FUTEX_WAIT_BITSET : FUTEX_WAIT_BITSET_PRIVATE;
+    if (timeout && timeout->is_realtime) {
+      op |= FUTEX_CLOCK_REALTIME;
+    }
+    for (;;) {
+      if (this->load(cpp::MemoryOrder::RELAXED) != expected)
+        return 0;
+
+      long ret = syscall_impl<long>(
+          /* syscall number */ FUTEX_SYSCALL_ID,
+          /* futex address */ this,
+          /* futex operation  */ op,
+          /* expected value */ expected,
+          /* timeout */ timeout ? &timeout->abs_time : nullptr,
+          /* ignored */ nullptr,
+          /* bitset */ FUTEX_BITSET_MATCH_ANY);
+
+      // continue waiting if interrupted; otherwise return the result
+      // which should normally be 0 or -ETIMEOUT
+      if (ret == -EINTR)
+        continue;
+
+      return ret;
+    }
+  }
+  LIBC_INLINE long notify_one(bool is_shared = false) {
+    return syscall_impl<long>(
+        /* syscall number */ FUTEX_SYSCALL_ID,
+        /* futex address */ this,
+        /* futex operation  */ is_shared ? FUTEX_WAKE : FUTEX_WAKE_PRIVATE,
+        /* wake up limit */ 1,
+        /* ignored */ nullptr,
+        /* ignored */ nullptr,
+        /* ignored */ 0);
+  }
+  LIBC_INLINE long notify_all(bool is_shared = false) {
+    return syscall_impl<long>(
+        /* syscall number */ FUTEX_SYSCALL_ID,
+        /* futex address */ this,
+        /* futex operation  */ is_shared ? FUTEX_WAKE : FUTEX_WAKE_PRIVATE,
+        /* wake up limit */ cpp::numeric_limits<int>::max(),
+        /* ignored */ nullptr,
+        /* ignored */ nullptr,
+        /* ignored */ 0);
+  }
+};
+
+static_assert(__is_standard_layout(Futex),
+              "Futex must be a standard layout type.");
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_UTILS_H
diff --git a/libc/src/__support/threads/linux/futex_word.h b/libc/src/__support/threads/linux/futex_word.h
index 67159b81b561..acdd33bcdaaf 100644
--- a/libc/src/__support/threads/linux/futex_word.h
+++ b/libc/src/__support/threads/linux/futex_word.h
@@ -11,7 +11,6 @@
 
 #include <stdint.h>
 #include <sys/syscall.h>
-
 namespace LIBC_NAMESPACE {
 
 // Futexes are 32 bits in size on all platforms, including 64-bit platforms.
diff --git a/libc/src/__support/threads/linux/mutex.h b/libc/src/__support/threads/linux/mutex.h
index 618698db0d25..6702de465168 100644
--- a/libc/src/__support/threads/linux/mutex.h
+++ b/libc/src/__support/threads/linux/mutex.h
@@ -9,17 +9,10 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
 #define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
 
-#include "src/__support/CPP/atomic.h"
-#include "src/__support/OSUtil/syscall.h" // For syscall functions.
-#include "src/__support/threads/linux/futex_word.h"
+#include "src/__support/threads/linux/futex_utils.h"
 #include "src/__support/threads/mutex_common.h"
 
-#include <linux/futex.h>
-#include <stdint.h>
-#include <sys/syscall.h> // For syscall numbers.
-
 namespace LIBC_NAMESPACE {
-
 struct Mutex {
   unsigned char timed;
   unsigned char recursive;
@@ -28,7 +21,7 @@ struct Mutex {
   void *owner;
   unsigned long long lock_count;
 
-  cpp::Atomic<FutexWordType> futex_word;
+  Futex futex_word;
 
   enum class LockState : FutexWordType {
     Free,
@@ -76,9 +69,7 @@ public:
         // futex syscall will block if the futex data is still
         // `LockState::Waiting` (the 4th argument to the syscall function
         // below.)
-        LIBC_NAMESPACE::syscall_impl<long>(
-            FUTEX_SYSCALL_ID, &futex_word.val, FUTEX_WAIT_PRIVATE,
-            FutexWordType(LockState::Waiting), 0, 0, 0);
+        futex_word.wait(FutexWordType(LockState::Waiting));
         was_waiting = true;
         // Once woken up/unblocked, try everything all over.
         continue;
@@ -91,9 +82,7 @@ public:
           // we will wait for the futex to be woken up. Note again that the
           // following syscall will block only if the futex data is still
           // `LockState::Waiting`.
-          LIBC_NAMESPACE::syscall_impl<long>(
-              FUTEX_SYSCALL_ID, &futex_word, FUTEX_WAIT_PRIVATE,
-              FutexWordType(LockState::Waiting), 0, 0, 0);
+          futex_word.wait(FutexWordType(LockState::Waiting));
           was_waiting = true;
         }
         continue;
@@ -110,8 +99,7 @@ public:
       if (futex_word.compare_exchange_strong(mutex_status,
                                              FutexWordType(LockState::Free))) {
         // If any thread is waiting to be woken up, then do it.
-        LIBC_NAMESPACE::syscall_impl<long>(FUTEX_SYSCALL_ID, &futex_word,
-                                           FUTEX_WAKE_PRIVATE, 1, 0, 0, 0);
+        futex_word.notify_one();
         return MutexError::NONE;
       }
 
diff --git a/libc/src/__support/threads/linux/thread.cpp b/libc/src/__support/threads/linux/thread.cpp
index fcf87cc587a5..1d986ff38cff 100644
--- a/libc/src/__support/threads/linux/thread.cpp
+++ b/libc/src/__support/threads/linux/thread.cpp
@@ -14,15 +14,14 @@
 #include "src/__support/OSUtil/syscall.h" // For syscall functions.
 #include "src/__support/common.h"
 #include "src/__support/error_or.h"
-#include "src/__support/threads/linux/futex_word.h" // For FutexWordType
-#include "src/errno/libc_errno.h"                   // For error macros
+#include "src/__support/threads/linux/futex_utils.h" // For FutexWordType
+#include "src/errno/libc_errno.h"                    // For error macros
 
 #ifdef LIBC_TARGET_ARCH_IS_AARCH64
 #include <arm_acle.h>
 #endif
 
 #include <fcntl.h>
-#include <linux/futex.h>
 #include <linux/param.h> // For EXEC_PAGESIZE.
 #include <linux/prctl.h> // For PR_SET_NAME
 #include <linux/sched.h> // For CLONE_* flags.
@@ -247,8 +246,7 @@ int Thread::run(ThreadStyle style, ThreadRunner runner, void *arg, void *stack,
   // stack memory.
 
   static constexpr size_t INTERNAL_STACK_DATA_SIZE =
-      sizeof(StartArgs) + sizeof(ThreadAttributes) +
-      sizeof(cpp::Atomic<FutexWordType>);
+      sizeof(StartArgs) + sizeof(ThreadAttributes) + sizeof(Futex);
 
   // This is pretty arbitrary, but at the moment we don't adjust user provided
   // stacksize (or default) to account for this data as its assumed minimal. If
@@ -288,9 +286,9 @@ int Thread::run(ThreadStyle style, ThreadRunner runner, void *arg, void *stack,
   start_args->runner = runner;
   start_args->arg = arg;
 
-  auto clear_tid = reinterpret_cast<cpp::Atomic<FutexWordType> *>(
+  auto clear_tid = reinterpret_cast<Futex *>(
       adjusted_stack + sizeof(StartArgs) + sizeof(ThreadAttributes));
-  clear_tid->val = CLEAR_TID_VALUE;
+  clear_tid->set(CLEAR_TID_VALUE);
   attrib->platform_data = clear_tid;
 
   // The clone syscall takes arguments in an architecture specific order.
@@ -374,14 +372,11 @@ void Thread::wait() {
   // The kernel should set the value at the clear tid address to zero.
   // If not, it is a spurious wake and we should continue to wait on
   // the futex.
-  auto *clear_tid =
-      reinterpret_cast<cpp::Atomic<FutexWordType> *>(attrib->platform_data);
-  while (clear_tid->load() != 0) {
-    // We cannot do a FUTEX_WAIT_PRIVATE here as the kernel does a
-    // FUTEX_WAKE and not a FUTEX_WAKE_PRIVATE.
-    LIBC_NAMESPACE::syscall_impl<long>(FUTEX_SYSCALL_ID, &clear_tid->val,
-                                       FUTEX_WAIT, CLEAR_TID_VALUE, nullptr);
-  }
+  auto *clear_tid = reinterpret_cast<Futex *>(attrib->platform_data);
+  // We cannot do a FUTEX_WAIT_PRIVATE here as the kernel does a
+  // FUTEX_WAKE and not a FUTEX_WAKE_PRIVATE.
+  while (clear_tid->load() != 0)
+    clear_tid->wait(CLEAR_TID_VALUE, cpp::nullopt, true);
 }
 
 bool Thread::operator==(const Thread &thread) const {
diff --git a/libc/src/__support/threads/mutex.h b/libc/src/__support/threads/mutex.h
index fa2bd64b6b51..9dded2e3f952 100644
--- a/libc/src/__support/threads/mutex.h
+++ b/libc/src/__support/threads/mutex.h
@@ -38,9 +38,9 @@
 // want the constructors of the Mutex classes to be constexprs.
 
 #if defined(__linux__)
-#include "linux/mutex.h"
+#include "src/__support/threads/linux/mutex.h"
 #elif defined(LIBC_TARGET_ARCH_IS_GPU)
-#include "gpu/mutex.h"
+#include "src/__support/threads/gpu/mutex.h"
 #endif // __linux__
 
 namespace LIBC_NAMESPACE {
diff --git a/libc/src/__support/threads/thread.cpp b/libc/src/__support/threads/thread.cpp
index 62aa86b7aef7..c1785343671c 100644
--- a/libc/src/__support/threads/thread.cpp
+++ b/libc/src/__support/threads/thread.cpp
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "thread.h"
-#include "mutex.h"
+#include "src/__support/threads/thread.h"
+#include "src/__support/threads/mutex.h"
 
 #include "src/__support/CPP/array.h"
 #include "src/__support/CPP/optional.h"
diff --git a/libc/src/threads/linux/CMakeLists.txt b/libc/src/threads/linux/CMakeLists.txt
index be5407031aad..d372bd9e18c4 100644
--- a/libc/src/threads/linux/CMakeLists.txt
+++ b/libc/src/threads/linux/CMakeLists.txt
@@ -9,7 +9,7 @@ add_header_library(
     libc.src.__support.CPP.atomic
     libc.src.__support.OSUtil.osutil
     libc.src.__support.threads.mutex
-    libc.src.__support.threads.linux.futex_word_type
+    libc.src.__support.threads.linux.futex_utils
 )
 
 add_entrypoint_object(
diff --git a/libc/src/threads/linux/CndVar.h b/libc/src/threads/linux/CndVar.h
index b4afdef9f9eb..525a8f0f2b53 100644
--- a/libc/src/threads/linux/CndVar.h
+++ b/libc/src/threads/linux/CndVar.h
@@ -10,8 +10,9 @@
 #define LLVM_LIBC_SRC_THREADS_LINUX_CNDVAR_H
 
 #include "src/__support/CPP/atomic.h"
+#include "src/__support/CPP/optional.h"
 #include "src/__support/OSUtil/syscall.h" // For syscall functions.
-#include "src/__support/threads/linux/futex_word.h"
+#include "src/__support/threads/linux/futex_utils.h"
 #include "src/__support/threads/mutex.h"
 
 #include <linux/futex.h> // For futex operations.
@@ -28,7 +29,7 @@ struct CndVar {
   };
 
   struct CndWaiter {
-    cpp::Atomic<uint32_t> futex_word = WS_Waiting;
+    Futex futex_word = WS_Waiting;
     CndWaiter *next = nullptr;
   };
 
@@ -84,8 +85,7 @@ struct CndVar {
       }
     }
 
-    LIBC_NAMESPACE::syscall_impl<long>(FUTEX_SYSCALL_ID, &waiter.futex_word.val,
-                                       FUTEX_WAIT, WS_Waiting, 0, 0, 0);
+    waiter.futex_word.wait(WS_Waiting, cpp::nullopt, true);
 
     // At this point, if locking |m| fails, we can simply return as the
     // queued up waiter would have been removed from the queue.
@@ -109,6 +109,7 @@ struct CndVar {
 
     qmtx.futex_word = FutexWordType(Mutex::LockState::Free);
 
+    // this is a special WAKE_OP, so we use syscall directly
     LIBC_NAMESPACE::syscall_impl<long>(
         FUTEX_SYSCALL_ID, &qmtx.futex_word.val, FUTEX_WAKE_OP, 1, 1,
         &first->futex_word.val,
-- 
GitLab


From 41dd07bf5cbfb800797821d1ad32226e5339bcfb Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 7 May 2024 15:48:36 +0100
Subject: [PATCH 0055/1206] [AArch64] Add test coverage for
 bitreverse(logicalshift(bitreverse(x),y)) -> logicalshift(x,y) fold

DAG already performs this fold (#89897), GISel is currently missing it (patch incoming)
---
 .../GlobalISel/combine-bitreverse-shift.ll    | 164 ++++++++++++++++++
 1 file changed, 164 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-bitreverse-shift.ll

diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-bitreverse-shift.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combine-bitreverse-shift.ll
new file mode 100644
index 000000000000..3ce94e2c40a9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-bitreverse-shift.ll
@@ -0,0 +1,164 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefixes=SDAG
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown -global-isel | FileCheck %s --check-prefixes=GISEL
+
+; These tests can be optimised
+;       fold (bitreverse(srl (bitreverse c), x)) -> (shl c, x)
+;       fold (bitreverse(shl (bitreverse c), x)) -> (srl c, x)
+
+declare i8 @llvm.bitreverse.i8(i8)
+declare i16 @llvm.bitreverse.i16(i16)
+declare i32 @llvm.bitreverse.i32(i32)
+declare i64 @llvm.bitreverse.i64(i64)
+
+define i8 @test_bitreverse_srli_bitreverse_i8(i8 %a) nounwind {
+; SDAG-LABEL: test_bitreverse_srli_bitreverse_i8:
+; SDAG:       // %bb.0:
+; SDAG-NEXT:    lsl w0, w0, #3
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: test_bitreverse_srli_bitreverse_i8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    rbit w8, w0
+; GISEL-NEXT:    lsr w8, w8, #24
+; GISEL-NEXT:    lsr w8, w8, #3
+; GISEL-NEXT:    rbit w8, w8
+; GISEL-NEXT:    lsr w0, w8, #24
+; GISEL-NEXT:    ret
+  %1 = call i8 @llvm.bitreverse.i8(i8 %a)
+  %2 = lshr i8 %1, 3
+  %3 = call i8 @llvm.bitreverse.i8(i8 %2)
+  ret i8 %3
+}
+
+define i16 @test_bitreverse_srli_bitreverse_i16(i16 %a) nounwind {
+; SDAG-LABEL: test_bitreverse_srli_bitreverse_i16:
+; SDAG:       // %bb.0:
+; SDAG-NEXT:    lsl w0, w0, #7
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: test_bitreverse_srli_bitreverse_i16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    rbit w8, w0
+; GISEL-NEXT:    lsr w8, w8, #16
+; GISEL-NEXT:    lsr w8, w8, #7
+; GISEL-NEXT:    rbit w8, w8
+; GISEL-NEXT:    lsr w0, w8, #16
+; GISEL-NEXT:    ret
+  %1 = call i16 @llvm.bitreverse.i16(i16 %a)
+  %2 = lshr i16 %1, 7
+  %3 = call i16 @llvm.bitreverse.i16(i16 %2)
+  ret i16 %3
+}
+
+define i32 @test_bitreverse_srli_bitreverse_i32(i32 %a) nounwind {
+; SDAG-LABEL: test_bitreverse_srli_bitreverse_i32:
+; SDAG:       // %bb.0:
+; SDAG-NEXT:    lsl w0, w0, #15
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: test_bitreverse_srli_bitreverse_i32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    rbit w8, w0
+; GISEL-NEXT:    lsr w8, w8, #15
+; GISEL-NEXT:    rbit w0, w8
+; GISEL-NEXT:    ret
+  %1 = call i32 @llvm.bitreverse.i32(i32 %a)
+  %2 = lshr i32 %1, 15
+  %3 = call i32 @llvm.bitreverse.i32(i32 %2)
+  ret i32 %3
+}
+
+define i64 @test_bitreverse_srli_bitreverse_i64(i64 %a) nounwind {
+; SDAG-LABEL: test_bitreverse_srli_bitreverse_i64:
+; SDAG:       // %bb.0:
+; SDAG-NEXT:    lsl x0, x0, #33
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: test_bitreverse_srli_bitreverse_i64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    rbit x8, x0
+; GISEL-NEXT:    lsr x8, x8, #33
+; GISEL-NEXT:    rbit x0, x8
+; GISEL-NEXT:    ret
+  %1 = call i64 @llvm.bitreverse.i64(i64 %a)
+  %2 = lshr i64 %1, 33
+  %3 = call i64 @llvm.bitreverse.i64(i64 %2)
+  ret i64 %3
+}
+
+define i8 @test_bitreverse_shli_bitreverse_i8(i8 %a) nounwind {
+; SDAG-LABEL: test_bitreverse_shli_bitreverse_i8:
+; SDAG:       // %bb.0:
+; SDAG-NEXT:    ubfx w0, w0, #3, #5
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: test_bitreverse_shli_bitreverse_i8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    rbit w8, w0
+; GISEL-NEXT:    lsr w8, w8, #24
+; GISEL-NEXT:    lsl w8, w8, #3
+; GISEL-NEXT:    rbit w8, w8
+; GISEL-NEXT:    lsr w0, w8, #24
+; GISEL-NEXT:    ret
+  %1 = call i8 @llvm.bitreverse.i8(i8 %a)
+  %2 = shl i8 %1, 3
+  %3 = call i8 @llvm.bitreverse.i8(i8 %2)
+  ret i8 %3
+}
+
+define i16 @test_bitreverse_shli_bitreverse_i16(i16 %a) nounwind {
+; SDAG-LABEL: test_bitreverse_shli_bitreverse_i16:
+; SDAG:       // %bb.0:
+; SDAG-NEXT:    ubfx w0, w0, #7, #9
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: test_bitreverse_shli_bitreverse_i16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    rbit w8, w0
+; GISEL-NEXT:    lsr w8, w8, #16
+; GISEL-NEXT:    lsl w8, w8, #7
+; GISEL-NEXT:    rbit w8, w8
+; GISEL-NEXT:    lsr w0, w8, #16
+; GISEL-NEXT:    ret
+  %1 = call i16 @llvm.bitreverse.i16(i16 %a)
+  %2 = shl i16 %1, 7
+  %3 = call i16 @llvm.bitreverse.i16(i16 %2)
+  ret i16 %3
+}
+
+define i32 @test_bitreverse_shli_bitreverse_i32(i32 %a) nounwind {
+; SDAG-LABEL: test_bitreverse_shli_bitreverse_i32:
+; SDAG:       // %bb.0:
+; SDAG-NEXT:    lsr w0, w0, #15
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: test_bitreverse_shli_bitreverse_i32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    rbit w8, w0
+; GISEL-NEXT:    lsl w8, w8, #15
+; GISEL-NEXT:    rbit w0, w8
+; GISEL-NEXT:    ret
+  %1 = call i32 @llvm.bitreverse.i32(i32 %a)
+  %2 = shl i32 %1, 15
+  %3 = call i32 @llvm.bitreverse.i32(i32 %2)
+  ret i32 %3
+}
+
+define i64 @test_bitreverse_shli_bitreverse_i64(i64 %a) nounwind {
+; SDAG-LABEL: test_bitreverse_shli_bitreverse_i64:
+; SDAG:       // %bb.0:
+; SDAG-NEXT:    lsr x0, x0, #33
+; SDAG-NEXT:    ret
+;
+; GISEL-LABEL: test_bitreverse_shli_bitreverse_i64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    rbit x8, x0
+; GISEL-NEXT:    lsl x8, x8, #33
+; GISEL-NEXT:    rbit x0, x8
+; GISEL-NEXT:    ret
+  %1 = call i64 @llvm.bitreverse.i64(i64 %a)
+  %2 = shl i64 %1, 33
+  %3 = call i64 @llvm.bitreverse.i64(i64 %2)
+  ret i64 %3
+}
-- 
GitLab


From de117dd533547f8bc8d00ea989252021ec1e877e Mon Sep 17 00:00:00 2001
From: Ulrich Weigand <ulrich.weigand@de.ibm.com>
Date: Tue, 7 May 2024 16:55:33 +0200
Subject: [PATCH 0056/1206] [SystemZ] Add some more atomic load/store tests

Verify atomic load/store of f128 on z14 where the type
lives in VRs.
---
 llvm/test/CodeGen/SystemZ/atomic-load-09.ll  | 78 +++++++++++++++++++
 llvm/test/CodeGen/SystemZ/atomic-store-09.ll | 81 ++++++++++++++++++++
 2 files changed, 159 insertions(+)
 create mode 100644 llvm/test/CodeGen/SystemZ/atomic-load-09.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/atomic-store-09.ll

diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-09.ll b/llvm/test/CodeGen/SystemZ/atomic-load-09.ll
new file mode 100644
index 000000000000..61b8e2f0efa8
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-09.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; Test long double atomic loads on z14.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lpq %r0, 0(%r3)
+; CHECK-NEXT:    stg %r1, 8(%r2)
+; CHECK-NEXT:    stg %r0, 0(%r2)
+; CHECK-NEXT:    br %r14
+  %val = load atomic fp128, ptr %src seq_cst, align 16
+  store fp128 %val, ptr %ret, align 8
+  ret void
+}
+
+define void @f1_fpuse(ptr %ret, ptr %src) {
+; CHECK-LABEL: f1_fpuse:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lpq %r0, 0(%r3)
+; CHECK-NEXT:    vlvgp %v0, %r0, %r1
+; CHECK-NEXT:    wfaxb %v0, %v0, %v0
+; CHECK-NEXT:    vst %v0, 0(%r2), 3
+; CHECK-NEXT:    br %r14
+  %val = load atomic fp128, ptr %src seq_cst, align 16
+  %use = fadd fp128 %val, %val
+  store fp128 %use, ptr %ret, align 8
+  ret void
+}
+
+define void @f2(ptr %ret, ptr %src) {
+; CHECK-LABEL: f2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r13, %r15, 104(%r15)
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -176
+; CHECK-NEXT:    .cfi_def_cfa_offset 336
+; CHECK-NEXT:    lgr %r13, %r2
+; CHECK-NEXT:    la %r4, 160(%r15)
+; CHECK-NEXT:    lghi %r2, 16
+; CHECK-NEXT:    lhi %r5, 5
+; CHECK-NEXT:    brasl %r14, __atomic_load@PLT
+; CHECK-NEXT:    vl %v0, 160(%r15), 3
+; CHECK-NEXT:    vst %v0, 0(%r13), 3
+; CHECK-NEXT:    lmg %r13, %r15, 280(%r15)
+; CHECK-NEXT:    br %r14
+  %val = load atomic fp128, ptr %src seq_cst, align 8
+  store fp128 %val, ptr %ret, align 8
+  ret void
+}
+
+define void @f2_fpuse(ptr %ret, ptr %src) {
+; CHECK-LABEL: f2_fpuse:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r13, %r15, 104(%r15)
+; CHECK-NEXT:    .cfi_offset %r13, -56
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -176
+; CHECK-NEXT:    .cfi_def_cfa_offset 336
+; CHECK-NEXT:    lgr %r13, %r2
+; CHECK-NEXT:    la %r4, 160(%r15)
+; CHECK-NEXT:    lghi %r2, 16
+; CHECK-NEXT:    lhi %r5, 5
+; CHECK-NEXT:    brasl %r14, __atomic_load@PLT
+; CHECK-NEXT:    vl %v0, 160(%r15), 3
+; CHECK-NEXT:    wfaxb %v0, %v0, %v0
+; CHECK-NEXT:    vst %v0, 0(%r13), 3
+; CHECK-NEXT:    lmg %r13, %r15, 280(%r15)
+; CHECK-NEXT:    br %r14
+  %val = load atomic fp128, ptr %src seq_cst, align 8
+  %use = fadd fp128 %val, %val
+  store fp128 %use, ptr %ret, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-09.ll b/llvm/test/CodeGen/SystemZ/atomic-store-09.ll
new file mode 100644
index 000000000000..3af16490b34b
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-09.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; Test long double atomic stores on z14.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+
+define void @f1(ptr %dst, ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lg %r1, 8(%r3)
+; CHECK-NEXT:    lg %r0, 0(%r3)
+; CHECK-NEXT:    stpq %r0, 0(%r2)
+; CHECK-NEXT:    bcr 14, %r0
+; CHECK-NEXT:    br %r14
+  %val = load fp128, ptr %src, align 8
+  store atomic fp128 %val, ptr %dst seq_cst, align 16
+  ret void
+}
+
+define void @f1_fpsrc(ptr %dst, ptr %src) {
+; CHECK-LABEL: f1_fpsrc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl %v0, 0(%r3), 3
+; CHECK-NEXT:    wfaxb %v0, %v0, %v0
+; CHECK-NEXT:    vlgvg %r1, %v0, 1
+; CHECK-NEXT:    vlgvg %r0, %v0, 0
+; CHECK-NEXT:    stpq %r0, 0(%r2)
+; CHECK-NEXT:    bcr 14, %r0
+; CHECK-NEXT:    br %r14
+  %val = load fp128, ptr %src, align 8
+  %add = fadd fp128 %val, %val
+  store atomic fp128 %add, ptr %dst seq_cst, align 16
+  ret void
+}
+
+define void @f2(ptr %dst, ptr %src) {
+; CHECK-LABEL: f2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -176
+; CHECK-NEXT:    .cfi_def_cfa_offset 336
+; CHECK-NEXT:    vl %v0, 0(%r3), 3
+; CHECK-NEXT:    lgr %r0, %r2
+; CHECK-NEXT:    la %r4, 160(%r15)
+; CHECK-NEXT:    lghi %r2, 16
+; CHECK-NEXT:    lgr %r3, %r0
+; CHECK-NEXT:    lhi %r5, 5
+; CHECK-NEXT:    vst %v0, 160(%r15), 3
+; CHECK-NEXT:    brasl %r14, __atomic_store@PLT
+; CHECK-NEXT:    lmg %r14, %r15, 288(%r15)
+; CHECK-NEXT:    br %r14
+  %val = load fp128, ptr %src, align 8
+  store atomic fp128 %val, ptr %dst seq_cst, align 8
+  ret void
+}
+
+define void @f2_fpuse(ptr %dst, ptr %src) {
+; CHECK-LABEL: f2_fpuse:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    stmg %r14, %r15, 112(%r15)
+; CHECK-NEXT:    .cfi_offset %r14, -48
+; CHECK-NEXT:    .cfi_offset %r15, -40
+; CHECK-NEXT:    aghi %r15, -176
+; CHECK-NEXT:    .cfi_def_cfa_offset 336
+; CHECK-NEXT:    vl %v0, 0(%r3), 3
+; CHECK-NEXT:    wfaxb %v0, %v0, %v0
+; CHECK-NEXT:    lgr %r0, %r2
+; CHECK-NEXT:    la %r4, 160(%r15)
+; CHECK-NEXT:    lghi %r2, 16
+; CHECK-NEXT:    lgr %r3, %r0
+; CHECK-NEXT:    lhi %r5, 5
+; CHECK-NEXT:    vst %v0, 160(%r15), 3
+; CHECK-NEXT:    brasl %r14, __atomic_store@PLT
+; CHECK-NEXT:    lmg %r14, %r15, 288(%r15)
+; CHECK-NEXT:    br %r14
+  %val = load fp128, ptr %src, align 8
+  %add = fadd fp128 %val, %val
+  store atomic fp128 %add, ptr %dst seq_cst, align 8
+  ret void
+}
-- 
GitLab


From 45fed80b15df85cee53d3d31a7a46ae0daa91a3f Mon Sep 17 00:00:00 2001
From: Jake Egan <Jake.egan@ibm.com>
Date: Tue, 7 May 2024 10:57:51 -0400
Subject: [PATCH 0057/1206] [AIX][libc++] Enable clang_modules_include.gen.py
 tests (#90971)

Enable these tests on AIX since they're passing.
---
 libcxx/test/libcxx/clang_modules_include.gen.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/libcxx/test/libcxx/clang_modules_include.gen.py b/libcxx/test/libcxx/clang_modules_include.gen.py
index a823a47fd19b..7ba4bf032624 100644
--- a/libcxx/test/libcxx/clang_modules_include.gen.py
+++ b/libcxx/test/libcxx/clang_modules_include.gen.py
@@ -31,9 +31,6 @@ for header in public_headers:
 // UNSUPPORTED: windows
 // UNSUPPORTED: buildhost=windows
 
-// The AIX headers don't appear to be compatible with modules
-// UNSUPPORTED: LIBCXX-AIX-FIXME
-
 // The Android headers don't appear to be compatible with modules yet
 // UNSUPPORTED: LIBCXX-ANDROID-FIXME
 
@@ -61,9 +58,6 @@ print(f"""\
 // UNSUPPORTED: windows
 // UNSUPPORTED: buildhost=windows
 
-// The AIX headers don't appear to be compatible with modules
-// UNSUPPORTED: LIBCXX-AIX-FIXME
-
 // The Android headers don't appear to be compatible with modules yet
 // UNSUPPORTED: LIBCXX-ANDROID-FIXME
 
-- 
GitLab


From d5cabf8d89a5f5faa5255283821cb080bebbff86 Mon Sep 17 00:00:00 2001
From: srcarroll <50210727+srcarroll@users.noreply.github.com>
Date: Tue, 7 May 2024 10:26:30 -0500
Subject: [PATCH 0058/1206] Keep attribute when bufferizing `scf.forall` op
 (#91236)

---
 .../lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp | 3 +++
 mlir/test/Dialect/SCF/one-shot-bufferize.mlir                  | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index 2a16b10bbaf8..cf40443ff383 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -1267,6 +1267,9 @@ struct ForallOpInterface
         forallOp.getMixedUpperBound(), forallOp.getMixedStep(),
         /*outputs=*/ValueRange(), forallOp.getMapping());
 
+    // Keep discardable attributes from the original op.
+    newForallOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
+
     rewriter.eraseOp(newForallOp.getBody()->getTerminator());
 
     // Move over block contents of the old op.
diff --git a/mlir/test/Dialect/SCF/one-shot-bufferize.mlir b/mlir/test/Dialect/SCF/one-shot-bufferize.mlir
index 485fdd9b0e59..bb9f7dfdba83 100644
--- a/mlir/test/Dialect/SCF/one-shot-bufferize.mlir
+++ b/mlir/test/Dialect/SCF/one-shot-bufferize.mlir
@@ -499,7 +499,8 @@ func.func @parallel_insert_slice_no_conflict(
         tensor.parallel_insert_slice %8 into %o[5] [%idx] [%c1] :
           tensor<?xf32> into tensor<?xf32>
       }
-  }
+  } {keep_this_attribute}
+  // CHECK: keep_this_attribute
 
   // CHECK: %[[load:.*]] = memref.load %[[arg2]]
   %f = tensor.extract %2[%c0] : tensor<?xf32>
-- 
GitLab


From f72454086af9d3f91a86e10dc1923849c5f670a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Tue, 7 May 2024 08:29:21 -0700
Subject: [PATCH 0059/1206] [flang][cuda] Fix retrieval of nested evaluation in
 cuf kernel (#91298)

`loopEval` was declared inside the for loop to iterate over the nested
loops so the same loop control was redeclared for each level of the loop
nest. Make sure we are iterating over all the loops by putting
`loopEval` declaration ouside of the for loop.
---
 flang/lib/Lower/Bridge.cpp                           | 5 ++---
 flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf | 8 ++++++--
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ae8679afc603..b0fc26332651 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2585,11 +2585,10 @@ private:
     llvm::SmallVector<mlir::Type> ivTypes;
     llvm::SmallVector<mlir::Location> ivLocs;
     llvm::SmallVector<mlir::Value> ivValues;
+    Fortran::lower::pft::Evaluation *loopEval =
+        &getEval().getFirstNestedEvaluation();
     for (unsigned i = 0; i < nestedLoops; ++i) {
       const Fortran::parser::LoopControl *loopControl;
-      Fortran::lower::pft::Evaluation *loopEval =
-          &getEval().getFirstNestedEvaluation();
-
       mlir::Location crtLoc = loc;
       if (i == 0) {
         loopControl = &*outerDoConstruct->GetLoopControl();
diff --git a/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf b/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
index d80542f76c92..e1cc35772618 100644
--- a/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
+++ b/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
@@ -11,7 +11,7 @@ subroutine sub1()
 
 ! CHECK-LABEL: func.func @_QPsub1()
 ! CHECK: %[[IV:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsub1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-
+! CHECK: %[[IV_J:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsub1Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   !$cuf kernel do <<< 1, 2 >>>
   do i = 1, n
     a(i) = a(i) * b(i)
@@ -41,7 +41,11 @@ subroutine sub1()
     end do
   end do
 
-! CHECK: fir.cuda_kernel<<<%c1{{.*}}, (%c256{{.*}}, %c1{{.*}})>>> (%{{.*}} : index, %{{.*}} : index) = (%{{.*}}, %{{.*}} : index, index) to (%{{.*}}, %{{.*}} : index, index) step (%{{.*}}, %{{.*}} : index, index)
+! CHECK: fir.cuda_kernel<<<%c1{{.*}}, (%c256{{.*}}, %c1{{.*}})>>> (%[[ARG0:.*]] : index, %[[ARG1:.*]] : index) = (%{{.*}}, %{{.*}} : index, index) to (%{{.*}}, %{{.*}} : index, index) step (%{{.*}}, %{{.*}} : index, index)
+! CHECK: %[[ARG0_I32:.*]] = fir.convert %[[ARG0]] : (index) -> i32
+! CHECK: fir.store %[[ARG0_I32]] to %[[IV]]#1 : !fir.ref<i32>
+! CHECK: %[[ARG1_I32:.*]] = fir.convert %[[ARG1]] : (index) -> i32
+! CHECK: fir.store %[[ARG1_I32]] to %[[IV_J]]#1 : !fir.ref<i32>
 ! CHECK: {n = 2 : i64}
 
   !$cuf kernel do(2) <<< (1,*), (256,1) >>>
-- 
GitLab


From 6a6fcbffbb31f83fab7425d43e28eb6aa39dbfe9 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Tue, 7 May 2024 15:03:26 +0000
Subject: [PATCH 0060/1206] [Clang][AArch64] NFC: Add IsArmStreamingFunction.

Simple refactoring to make a single interface that checks if a
FunctionDecl is a __arm[_locally]_streaming function.
---
 clang/include/clang/AST/Decl.h        |  5 +++++
 clang/lib/AST/Decl.cpp                | 15 +++++++++++++++
 clang/lib/CodeGen/Targets/AArch64.cpp | 15 +++++----------
 clang/lib/Sema/SemaDeclAttr.cpp       | 13 ++++---------
 4 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index a53c27a99a8c..de8b923645f8 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -5049,6 +5049,11 @@ static constexpr StringRef getOpenMPVariantManglingSeparatorStr() {
   return "$ompvariant";
 }
 
+/// Returns whether the given FunctionDecl has an __arm[_locally]_streaming
+/// attribute.
+bool IsArmStreamingFunction(const FunctionDecl *FD,
+                            bool IncludeLocallyStreaming);
+
 } // namespace clang
 
 #endif // LLVM_CLANG_AST_DECL_H
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index e7e95c16b697..ec851c9371e1 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -5758,3 +5758,18 @@ ExportDecl *ExportDecl::Create(ASTContext &C, DeclContext *DC,
 ExportDecl *ExportDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID) {
   return new (C, ID) ExportDecl(nullptr, SourceLocation());
 }
+
+bool clang::IsArmStreamingFunction(const FunctionDecl *FD,
+                                   bool IncludeLocallyStreaming) {
+  if (IncludeLocallyStreaming)
+    if (FD->hasAttr<ArmLocallyStreamingAttr>())
+      return true;
+
+  if (const Type *Ty = FD->getType().getTypePtrOrNull())
+    if (const auto *FPT = Ty->getAs<FunctionProtoType>())
+      if (FPT->getAArch64SMEAttributes() &
+          FunctionType::SME_PStateSMEnabledMask)
+        return true;
+
+  return false;
+}
diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp b/clang/lib/CodeGen/Targets/AArch64.cpp
index 452dc049d51b..e32b060ebeb9 100644
--- a/clang/lib/CodeGen/Targets/AArch64.cpp
+++ b/clang/lib/CodeGen/Targets/AArch64.cpp
@@ -8,6 +8,7 @@
 
 #include "ABIInfoImpl.h"
 #include "TargetInfo.h"
+#include "clang/AST/Decl.h"
 #include "clang/Basic/DiagnosticFrontend.h"
 #include "llvm/TargetParser/AArch64TargetParser.h"
 
@@ -852,14 +853,6 @@ Address AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
                           /*allowHigherAlign*/ false);
 }
 
-static bool isStreaming(const FunctionDecl *F) {
-  if (F->hasAttr<ArmLocallyStreamingAttr>())
-    return true;
-  if (const auto *T = F->getType()->getAs<FunctionProtoType>())
-    return T->getAArch64SMEAttributes() & FunctionType::SME_PStateSMEnabledMask;
-  return false;
-}
-
 static bool isStreamingCompatible(const FunctionDecl *F) {
   if (const auto *T = F->getType()->getAs<FunctionProtoType>())
     return T->getAArch64SMEAttributes() &
@@ -906,8 +899,10 @@ void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming(
   if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>())
     return;
 
-  bool CallerIsStreaming = isStreaming(Caller);
-  bool CalleeIsStreaming = isStreaming(Callee);
+  bool CallerIsStreaming =
+      IsArmStreamingFunction(Caller, /*IncludeLocallyStreaming=*/true);
+  bool CalleeIsStreaming =
+      IsArmStreamingFunction(Callee, /*IncludeLocallyStreaming=*/true);
   bool CallerIsStreamingCompatible = isStreamingCompatible(Caller);
   bool CalleeIsStreamingCompatible = isStreamingCompatible(Callee);
 
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 363ae93cb62d..6ca42856459f 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -3538,13 +3538,6 @@ bool Sema::checkTargetAttr(SourceLocation LiteralLoc, StringRef AttrStr) {
   return false;
 }
 
-static bool hasArmStreamingInterface(const FunctionDecl *FD) {
-  if (const auto *T = FD->getType()->getAs<FunctionProtoType>())
-    if (T->getAArch64SMEAttributes() & FunctionType::SME_PStateSMEnabledMask)
-      return true;
-  return false;
-}
-
 // Check Target Version attrs
 bool Sema::checkTargetVersionAttr(SourceLocation LiteralLoc, Decl *D,
                                   StringRef &AttrStr, bool &isDefault) {
@@ -3563,7 +3556,8 @@ bool Sema::checkTargetVersionAttr(SourceLocation LiteralLoc, Decl *D,
       return Diag(LiteralLoc, diag::warn_unsupported_target_attribute)
              << Unsupported << None << CurFeature << TargetVersion;
   }
-  if (hasArmStreamingInterface(cast<FunctionDecl>(D)))
+  if (IsArmStreamingFunction(cast<FunctionDecl>(D),
+                             /*IncludeLocallyStreaming=*/false))
     return Diag(LiteralLoc, diag::err_sme_streaming_cannot_be_multiversioned);
   return false;
 }
@@ -3665,7 +3659,8 @@ bool Sema::checkTargetClonesAttrString(
           HasNotDefault = true;
         }
       }
-      if (hasArmStreamingInterface(cast<FunctionDecl>(D)))
+      if (IsArmStreamingFunction(cast<FunctionDecl>(D),
+                                 /*IncludeLocallyStreaming=*/false))
         return Diag(LiteralLoc,
                     diag::err_sme_streaming_cannot_be_multiversioned);
     } else {
-- 
GitLab


From e84fae837c0b154153bd9b9a3255ec5a67b1ea61 Mon Sep 17 00:00:00 2001
From: Andrew Sukach <134116196+soukatch@users.noreply.github.com>
Date: Tue, 7 May 2024 11:40:26 -0400
Subject: [PATCH 0061/1206] [clang] MangledSymbol: remove pointless copy of
 vector (#90012)

This pr addresses #87255 adds a std::move call to the names in
MangledSymbol's constructor.
---
 clang/lib/Frontend/InterfaceStubFunctionsConsumer.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Frontend/InterfaceStubFunctionsConsumer.cpp b/clang/lib/Frontend/InterfaceStubFunctionsConsumer.cpp
index f8dced5dbafb..d7cfd23bb0a7 100644
--- a/clang/lib/Frontend/InterfaceStubFunctionsConsumer.cpp
+++ b/clang/lib/Frontend/InterfaceStubFunctionsConsumer.cpp
@@ -33,7 +33,8 @@ class InterfaceStubFunctionsConsumer : public ASTConsumer {
 
     MangledSymbol(const std::string &ParentName, uint8_t Type, uint8_t Binding,
                   std::vector<std::string> Names)
-        : ParentName(ParentName), Type(Type), Binding(Binding), Names(Names) {}
+        : ParentName(ParentName), Type(Type), Binding(Binding),
+          Names(std::move(Names)) {}
   };
   using MangledSymbols = std::map<const NamedDecl *, MangledSymbol>;
 
-- 
GitLab


From 57175533da0f3ea2054550c2e4d3e831e93bb4df Mon Sep 17 00:00:00 2001
From: Scott Manley <rscottmanley@gmail.com>
Date: Tue, 7 May 2024 10:45:28 -0500
Subject: [PATCH 0062/1206] [MLIR][IR] add -mlir-print-unique-ssa-ids to
 AsmPrinter (#91241)

Add an option to unique the numbers of values, block arguments and
naming conflicts when requested and/or printing generic op form. This is
helpful when debugging. For example, if you have:

    scf.for
      %0 =
      %1 = opA %0

    scf.for
      %0 =
      %1 = opB %0

And you get a verifier error which says opB's "operand #0 does not
dominate this use", it looks like %0 does dominate the use. This is not
intuitive. If these were numbered uniquely, it would look like:

    scf.for
      %0 =
      %1 = opA %0

    scf.for
      %2 =
      %3 = opB %0

And thus, much clearer as to why you are getting the error since %0 is
out of scope. Since generic op form should aim to give you the most
possible information, it seems like a good idea to use unique numbers in
this situation. Adding an option also gives those an option to use it
outside of generic op form.

Co-authored-by: Scott Manley <scmanley@nvidia.com>
---
 mlir/include/mlir/IR/OperationSupport.h |  6 +++++
 mlir/lib/IR/AsmPrinter.cpp              | 23 ++++++++++++++++---
 mlir/test/IR/print-unique-ssa-ids.mlir  | 30 +++++++++++++++++++++++++
 3 files changed, 56 insertions(+), 3 deletions(-)
 create mode 100644 mlir/test/IR/print-unique-ssa-ids.mlir

diff --git a/mlir/include/mlir/IR/OperationSupport.h b/mlir/include/mlir/IR/OperationSupport.h
index e661bb87a27e..f8ab5338107f 100644
--- a/mlir/include/mlir/IR/OperationSupport.h
+++ b/mlir/include/mlir/IR/OperationSupport.h
@@ -1219,6 +1219,9 @@ public:
   /// Return if the printer should print users of values.
   bool shouldPrintValueUsers() const;
 
+  /// Return if printer should use unique SSA IDs.
+  bool shouldPrintUniqueSSAIDs() const;
+
 private:
   /// Elide large elements attributes if the number of elements is larger than
   /// the upper limit.
@@ -1249,6 +1252,9 @@ private:
 
   /// Print users of values.
   bool printValueUsersFlag : 1;
+
+  /// Print unique SSA IDs for values, block arguments and naming conflicts
+  bool printUniqueSSAIDsFlag : 1;
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index e915b97d9ff1..9a5c51ba738f 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -189,6 +189,11 @@ struct AsmPrinterOptions {
       "mlir-print-value-users", llvm::cl::init(false),
       llvm::cl::desc(
           "Print users of operation results and block arguments as a comment")};
+
+  llvm::cl::opt<bool> printUniqueSSAIDs{
+      "mlir-print-unique-ssa-ids", llvm::cl::init(false),
+      llvm::cl::desc("Print unique SSA ID numbers for values, block arguments "
+                     "and naming conflicts across all regions")};
 };
 } // namespace
 
@@ -206,7 +211,7 @@ OpPrintingFlags::OpPrintingFlags()
     : printDebugInfoFlag(false), printDebugInfoPrettyFormFlag(false),
       printGenericOpFormFlag(false), skipRegionsFlag(false),
       assumeVerifiedFlag(false), printLocalScope(false),
-      printValueUsersFlag(false) {
+      printValueUsersFlag(false), printUniqueSSAIDsFlag(false) {
   // Initialize based upon command line options, if they are available.
   if (!clOptions.isConstructed())
     return;
@@ -224,6 +229,7 @@ OpPrintingFlags::OpPrintingFlags()
   printLocalScope = clOptions->printLocalScopeOpt;
   skipRegionsFlag = clOptions->skipRegionsOpt;
   printValueUsersFlag = clOptions->printValueUsers;
+  printUniqueSSAIDsFlag = clOptions->printUniqueSSAIDs;
 }
 
 /// Enable the elision of large elements attributes, by printing a '...'
@@ -350,6 +356,11 @@ bool OpPrintingFlags::shouldPrintValueUsers() const {
   return printValueUsersFlag;
 }
 
+/// Return if the printer should use unique IDs.
+bool OpPrintingFlags::shouldPrintUniqueSSAIDs() const {
+  return printUniqueSSAIDsFlag || shouldPrintGenericOpForm();
+}
+
 //===----------------------------------------------------------------------===//
 // NewLineCounter
 //===----------------------------------------------------------------------===//
@@ -1369,8 +1380,14 @@ SSANameState::SSANameState(Operation *op, const OpPrintingFlags &printerFlags)
   while (!nameContext.empty()) {
     Region *region;
     UsedNamesScopeTy *parentScope;
-    std::tie(region, nextValueID, nextArgumentID, nextConflictID, parentScope) =
-        nameContext.pop_back_val();
+
+    if (printerFlags.shouldPrintUniqueSSAIDs())
+      // To print unique SSA IDs, ignore saved ID counts from parent regions
+      std::tie(region, std::ignore, std::ignore, std::ignore, parentScope) =
+          nameContext.pop_back_val();
+    else
+      std::tie(region, nextValueID, nextArgumentID, nextConflictID,
+               parentScope) = nameContext.pop_back_val();
 
     // When we switch from one subtree to another, pop the scopes(needless)
     // until the parent scope.
diff --git a/mlir/test/IR/print-unique-ssa-ids.mlir b/mlir/test/IR/print-unique-ssa-ids.mlir
new file mode 100644
index 000000000000..a2d2d9bb7907
--- /dev/null
+++ b/mlir/test/IR/print-unique-ssa-ids.mlir
@@ -0,0 +1,30 @@
+// RUN: mlir-opt -mlir-print-unique-ssa-ids %s | FileCheck %s
+// RUN: mlir-opt -mlir-print-op-generic %s | FileCheck %s
+// RUN: mlir-opt %s | FileCheck %s --check-prefix=LOCAL_SCOPE
+
+// CHECK: %arg3
+// CHECK: %7
+// LOCAL_SCOPE-NOT: %arg3
+// LOCAL_SCOPE-NOT: %7
+module {
+  func.func @uniqueSSAIDs(%arg0 : memref<i32>, %arg1 : memref<i32>) {
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %c8 = arith.constant 8 : index
+    scf.for %arg2 = %c0 to %c8 step %c1 {
+      %a = memref.load %arg0[] : memref<i32>
+      %b = memref.load %arg1[] : memref<i32>
+      %0 = arith.addi %a, %b : i32
+      %1 = arith.subi %a, %b : i32
+      scf.yield
+    }
+    scf.for %arg2 = %c0 to %c8 step %c1 {
+      %a = memref.load %arg0[] : memref<i32>
+      %b = memref.load %arg1[] : memref<i32>
+      %0 = arith.addi %a, %b : i32
+      %1 = arith.subi %a, %b : i32
+      scf.yield
+    }
+    return
+  }
+}
-- 
GitLab


From b2477765dbf9bd28bd2d1813c41ae12613f87717 Mon Sep 17 00:00:00 2001
From: Duo Wang <duow1@uci.edu>
Date: Tue, 7 May 2024 08:51:12 -0700
Subject: [PATCH 0063/1206] [clang][test] Fix instantiation-depth-default.cpp
 under ubsan config on Windows (#91021)

Clang test `instantiation-depth-default.cpp` fails on Windows when built
with `ubsan` due to extra warnings printed by the compiler:
```console
File instantiation-depth-default.cpp Line 11: stack nearly exhausted; compilation time may suffer, and crashes due to stack overflow are likely
```

Originally in https://github.com/llvm/llvm-project/pull/75254 this test
was enabled for `asan` as well but later started to cause failures in
Linux ASAN buildbots. I have excluded `asan` from this change.
---
 clang/test/SemaTemplate/instantiation-depth-default.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/SemaTemplate/instantiation-depth-default.cpp b/clang/test/SemaTemplate/instantiation-depth-default.cpp
index f5835b86b3a3..5934d4e542ee 100644
--- a/clang/test/SemaTemplate/instantiation-depth-default.cpp
+++ b/clang/test/SemaTemplate/instantiation-depth-default.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -ftemplate-backtrace-limit=2 %s
+// RUN: %clang_cc1 -fsyntax-only -verify -ftemplate-backtrace-limit=2 %if {{ubsan}} %{ -Wno-stack-exhausted %} %s
 //
 // FIXME: Disable this test when Clang was built with ASan, because ASan
 // increases our per-frame stack usage enough that this test no longer fits
-- 
GitLab


From f00f2941307e04d3b7320969ee3fec9af31246ba Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 7 May 2024 08:55:50 -0700
Subject: [PATCH 0064/1206] [SLP]Fix PR91309: Do not consider SExt as always
 producing signed result.

Still need to do the full analysis of the signedness of the values
rather than rely on Instruction opcode, if the opcode is SExt. Still may
produce unsigned result.
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp               | 3 +--
 .../SLPVectorizer/AArch64/unsigned-after-sext-node.ll         | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d6a2273d0f18..98561f9ca044 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15487,8 +15487,7 @@ void BoUpSLP::computeMinimumValueSizes() {
       TreeEntry *TE = VectorizableTree[Idx].get();
       if (MinBWs.contains(TE))
         continue;
-      bool IsSigned = TE->getOpcode() == Instruction::SExt ||
-                      any_of(TE->Scalars, [&](Value *R) {
+      bool IsSigned = any_of(TE->Scalars, [&](Value *R) {
                         return !isKnownNonNegative(R, SimplifyQuery(*DL));
                       });
       MinBWs.try_emplace(TE, MaxBitWidth, IsSigned);
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll
index 406e5b9b930d..96ed3e77d987 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll
@@ -4,10 +4,10 @@
 define i16 @test()  {
 ; CHECK-LABEL: define i16 @test() {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LNOT:%.*]] = xor i1 false, true
+; CHECK-NEXT:    [[LNOT:%.*]] = xor i1 true, true
 ; CHECK-NEXT:    [[LNOT_EXT:%.*]] = zext i1 [[LNOT]] to i16
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i16 0, [[LNOT_EXT]]
-; CHECK-NEXT:    [[LNOT5:%.*]] = xor i1 false, true
+; CHECK-NEXT:    [[LNOT5:%.*]] = xor i1 true, true
 ; CHECK-NEXT:    [[LNOT_EXT6:%.*]] = zext i1 [[LNOT5]] to i16
 ; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i16 [[ADD]], [[LNOT_EXT6]]
 ; CHECK-NEXT:    ret i16 [[ADD7]]
-- 
GitLab


From e74a7a9fd79a74073277471243a44527c71eb4a9 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 7 May 2024 09:15:52 -0700
Subject: [PATCH 0065/1206] cc1: Report an error for multiple actions unless
 separated by -main-file-name (#91140)

When multiple actions are specified, the last one is used and others are
overridden. This might lead to confusion if the user is used to driver's
`-S -emit-llvm` behavior.

```
%clang_cc1 -S -emit-llvm a.c     # -S is overridden
%clang_cc1 -emit-llvm -S a.c     # -emit-llvm is overridden
%clang_cc1 -fsyntax-only -S a.c  # -fsyntax-only is overridden
```

However, we want to continue supporting overriding the driver action
with -Xclang:

* `clang -c -Xclang -ast-dump a.c` (`%clang -cc1 -emit-obj ...
-main-file-name a.c ... -ast-dump`)
* `clang -c -xc++ -Xclang -emit-module stl.modulemap`

As an exception, we allow -ast-dump* options to be composed together
(e.g. `-ast-dump -ast-dump-lookups` in AST/ast-dump-lookups.cpp).
---
 .../clang/Basic/DiagnosticFrontendKinds.td    |  2 ++
 clang/lib/Frontend/CompilerInvocation.cpp     | 24 +++++++++++++++++++
 clang/test/Frontend/multiple-actions.c        |  7 ++++++
 3 files changed, 33 insertions(+)
 create mode 100644 clang/test/Frontend/multiple-actions.c

diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
index fcffadacc8e6..e456ec2cac46 100644
--- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td
+++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
@@ -134,6 +134,8 @@ def err_fe_no_pch_in_dir : Error<
     "no suitable precompiled header file found in directory '%0'">;
 def err_fe_action_not_available : Error<
     "action %0 not compiled in">;
+def err_fe_invalid_multiple_actions : Error<
+    "'%0' action ignored; '%1' action specified previously">;
 def err_fe_invalid_alignment : Error<
     "invalid value '%1' in '%0'; alignment must be a power of 2">;
 def err_fe_invalid_exception_model
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 8312abc36039..948fe08c863a 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -2841,6 +2841,30 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     }
 
     Opts.ProgramAction = *ProgramAction;
+
+    // Catch common mistakes when multiple actions are specified for cc1 (e.g.
+    // -S -emit-llvm means -emit-llvm while -emit-llvm -S means -S). However, to
+    // support driver `-c -Xclang ACTION` (-cc1 -emit-llvm file -main-file-name
+    // X ACTION), we suppress the error when the two actions are separated by
+    // -main-file-name.
+    //
+    // As an exception, accept composable -ast-dump*.
+    if (!A->getSpelling().starts_with("-ast-dump")) {
+      const Arg *SavedAction = nullptr;
+      for (const Arg *AA :
+           Args.filtered(OPT_Action_Group, OPT_main_file_name)) {
+        if (AA->getOption().matches(OPT_main_file_name)) {
+          SavedAction = nullptr;
+        } else if (!SavedAction) {
+          SavedAction = AA;
+        } else {
+          if (!A->getOption().matches(OPT_ast_dump_EQ))
+            Diags.Report(diag::err_fe_invalid_multiple_actions)
+                << SavedAction->getSpelling() << A->getSpelling();
+          break;
+        }
+      }
+    }
   }
 
   if (const Arg* A = Args.getLastArg(OPT_plugin)) {
diff --git a/clang/test/Frontend/multiple-actions.c b/clang/test/Frontend/multiple-actions.c
new file mode 100644
index 000000000000..284e8f0467bf
--- /dev/null
+++ b/clang/test/Frontend/multiple-actions.c
@@ -0,0 +1,7 @@
+// RUN: not %clang_cc1 -S -emit-llvm -main-file-name %s 2>&1 | FileCheck %s --check-prefix=ERR1 --implicit-check-not=error:
+// ERR1: error: '-S' action ignored; '-emit-llvm' action specified previously
+
+// RUN: not %clang_cc1 -main-file-name %s -emit-llvm-only -emit-llvm -S 2>&1 | FileCheck %s --check-prefix=ERR2 --implicit-check-not=error:
+// ERR2: error: '-emit-llvm-only' action ignored; '-S' action specified previously
+
+// RUN: %clang_cc1 -S -main-file-name %s -emit-llvm -o /dev/null
-- 
GitLab


From 8bcb0737056163686e967821bea7f9e87c57cdfc Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Tue, 7 May 2024 09:18:55 -0700
Subject: [PATCH 0066/1206] [Clang] -fseparate-named-sections option (#91028)

When set, the compiler will use separate unique sections for global
symbols in named special sections (e.g. symbols that are annotated with
__attribute__((section(...)))). Doing so enables linker GC to collect
unused symbols without having to use a different section per-symbol.
---
 clang/docs/ReleaseNotes.rst                   |  5 +++
 clang/include/clang/Basic/CodeGenOptions.def  |  1 +
 clang/include/clang/Driver/Options.td         |  5 +++
 clang/lib/CodeGen/BackendUtil.cpp             |  1 +
 clang/test/CodeGen/fseparate-named-sections.c | 28 +++++++++++++++
 llvm/include/llvm/CodeGen/CommandFlags.h      |  2 ++
 llvm/include/llvm/Target/TargetMachine.h      |  4 +++
 llvm/include/llvm/Target/TargetOptions.h      | 21 ++++++-----
 llvm/lib/CodeGen/CommandFlags.cpp             |  8 +++++
 .../CodeGen/TargetLoweringObjectFileImpl.cpp  | 11 ++++--
 .../X86/elf-separate-named-sections.ll        | 36 +++++++++++++++++++
 11 files changed, 110 insertions(+), 12 deletions(-)
 create mode 100644 clang/test/CodeGen/fseparate-named-sections.c
 create mode 100644 llvm/test/CodeGen/X86/elf-separate-named-sections.ll

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index a85095e424b6..cc3108bf41d6 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -303,6 +303,11 @@ New Compiler Flags
   allow late parsing certain attributes in specific contexts where they would
   not normally be late parsed.
 
+- ``-fseparate-named-sections`` uses separate unique sections for global
+  symbols in named special sections (i.e. symbols annotated with
+  ``__attribute__((section(...)))``. This enables linker GC to collect unused
+  symbols without having to use a per-symbol section.
+
 Deprecated Compiler Flags
 -------------------------
 
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index 340b08dd7e2a..b964e4557478 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -57,6 +57,7 @@ CODEGENOPT(UniqueSectionNames, 1, 1) ///< Set for -funique-section-names.
 CODEGENOPT(UniqueBasicBlockSectionNames, 1, 1) ///< Set for -funique-basic-block-section-names,
                                                ///< Produce unique section names with
                                                ///< basic block sections.
+CODEGENOPT(SeparateNamedSections, 1, 0) ///< Set for -fseparate-named-sections.
 CODEGENOPT(EnableAIXExtendedAltivecABI, 1, 0) ///< Set for -mabi=vec-extabi. Enables the extended Altivec ABI on AIX.
 CODEGENOPT(XCOFFReadOnlyPointers, 1, 0) ///< Set for -mxcoff-roptr.
 CODEGENOPT(AllTocData, 1, 0) ///< AIX -mtocdata
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index c9f7c4e5f718..2c319ba38a29 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -4159,6 +4159,11 @@ defm unique_section_names : BoolFOption<"unique-section-names",
   NegFlag<SetFalse, [], [ClangOption, CC1Option],
           "Don't use unique names for text and data sections">,
   PosFlag<SetTrue>>;
+defm separate_named_sections : BoolFOption<"separate-named-sections",
+  CodeGenOpts<"SeparateNamedSections">, DefaultFalse,
+  PosFlag<SetTrue, [], [ClangOption, CC1Option],
+          "Use separate unique sections for named sections (ELF Only)">,
+  NegFlag<SetFalse>>;
 
 defm split_machine_functions: BoolFOption<"split-machine-functions",
   CodeGenOpts<"SplitMachineFunctions">, DefaultFalse,
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 22c3f8642ad8..119ec4704002 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -423,6 +423,7 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
   Options.UniqueSectionNames = CodeGenOpts.UniqueSectionNames;
   Options.UniqueBasicBlockSectionNames =
       CodeGenOpts.UniqueBasicBlockSectionNames;
+  Options.SeparateNamedSections = CodeGenOpts.SeparateNamedSections;
   Options.TLSSize = CodeGenOpts.TLSSize;
   Options.EnableTLSDESC = CodeGenOpts.EnableTLSDESC;
   Options.EmulatedTLS = CodeGenOpts.EmulatedTLS;
diff --git a/clang/test/CodeGen/fseparate-named-sections.c b/clang/test/CodeGen/fseparate-named-sections.c
new file mode 100644
index 000000000000..7a247dbd085c
--- /dev/null
+++ b/clang/test/CodeGen/fseparate-named-sections.c
@@ -0,0 +1,28 @@
+// REQUIRES: x86-registered-target
+
+// RUN: %clang_cc1 -triple x86_64-pc-linux -S -o - < %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux -S -fseparate-named-sections -o - < %s | FileCheck %s --check-prefix=SEPARATE
+
+__attribute__((section("custom_text"))) void f(void) {}
+__attribute__((section("custom_text"))) void g(void) {}
+
+// CHECK: .section custom_text,"ax",@progbits{{$}}
+// CHECK: f:
+// CHECK: g:
+
+// SEPARATE: .section custom_text,"ax",@progbits,unique,1{{$}}
+// SEPARATE: f:
+// SEPARATE: .section custom_text,"ax",@progbits,unique,2{{$}}
+// SEPARATE: g:
+
+__attribute__((section("custom_data"))) int i = 0;
+__attribute__((section("custom_data"))) int j = 0;
+
+// CHECK: .section custom_data,"aw",@progbits{{$}}
+// CHECK: i:
+// CHECK: j:
+
+// SEPARATE: .section custom_data,"aw",@progbits,unique,3{{$}}
+// SEPARATE: i:
+// SEPARATE: .section custom_data,"aw",@progbits,unique,4{{$}}
+// SEPARATE: j:
diff --git a/llvm/include/llvm/CodeGen/CommandFlags.h b/llvm/include/llvm/CodeGen/CommandFlags.h
index 244dabd38cf6..d5448d781363 100644
--- a/llvm/include/llvm/CodeGen/CommandFlags.h
+++ b/llvm/include/llvm/CodeGen/CommandFlags.h
@@ -122,6 +122,8 @@ bool getUniqueSectionNames();
 
 bool getUniqueBasicBlockSectionNames();
 
+bool getSeparateNamedSections();
+
 llvm::EABI getEABIVersion();
 
 llvm::DebuggerKind getDebuggerTuningOpt();
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index 48ea3cfe0277..1ba99730ca70 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -288,6 +288,10 @@ public:
     return Options.UniqueBasicBlockSectionNames;
   }
 
+  bool getSeparateNamedSections() const {
+    return Options.SeparateNamedSections;
+  }
+
   /// Return true if data objects should be emitted into their own section,
   /// corresponds to -fdata-sections.
   bool getDataSections() const {
diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h
index d37e9d9576ba..98a8b7ba337c 100644
--- a/llvm/include/llvm/Target/TargetOptions.h
+++ b/llvm/include/llvm/Target/TargetOptions.h
@@ -144,15 +144,15 @@ namespace llvm {
           DisableIntegratedAS(false), FunctionSections(false),
           DataSections(false), IgnoreXCOFFVisibility(false),
           XCOFFTracebackTable(true), UniqueSectionNames(true),
-          UniqueBasicBlockSectionNames(false), TrapUnreachable(false),
-          NoTrapAfterNoreturn(false), TLSSize(0), EmulatedTLS(false),
-          EnableTLSDESC(false), EnableIPRA(false), EmitStackSizeSection(false),
-          EnableMachineOutliner(false), EnableMachineFunctionSplitter(false),
-          SupportsDefaultOutlining(false), EmitAddrsig(false), BBAddrMap(false),
-          EmitCallSiteInfo(false), SupportsDebugEntryValues(false),
-          EnableDebugEntryValues(false), ValueTrackingVariableLocations(false),
-          ForceDwarfFrameSection(false), XRayFunctionIndex(true),
-          DebugStrictDwarf(false), Hotpatch(false),
+          UniqueBasicBlockSectionNames(false), SeparateNamedSections(false),
+          TrapUnreachable(false), NoTrapAfterNoreturn(false), TLSSize(0),
+          EmulatedTLS(false), EnableTLSDESC(false), EnableIPRA(false),
+          EmitStackSizeSection(false), EnableMachineOutliner(false),
+          EnableMachineFunctionSplitter(false), SupportsDefaultOutlining(false),
+          EmitAddrsig(false), BBAddrMap(false), EmitCallSiteInfo(false),
+          SupportsDebugEntryValues(false), EnableDebugEntryValues(false),
+          ValueTrackingVariableLocations(false), ForceDwarfFrameSection(false),
+          XRayFunctionIndex(true), DebugStrictDwarf(false), Hotpatch(false),
           PPCGenScalarMASSEntries(false), JMCInstrument(false),
           EnableCFIFixup(false), MisExpect(false), XCOFFReadOnlyPointers(false),
           FPDenormalMode(DenormalMode::IEEE, DenormalMode::IEEE) {}
@@ -277,6 +277,9 @@ namespace llvm {
     /// Use unique names for basic block sections.
     unsigned UniqueBasicBlockSectionNames : 1;
 
+    /// Emit named sections with the same name into different sections.
+    unsigned SeparateNamedSections : 1;
+
     /// Emit target-specific trap instruction for 'unreachable' IR instructions.
     unsigned TrapUnreachable : 1;
 
diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp
index 14ac4b2102c2..677460a2d8e4 100644
--- a/llvm/lib/CodeGen/CommandFlags.cpp
+++ b/llvm/lib/CodeGen/CommandFlags.cpp
@@ -96,6 +96,7 @@ CGOPT_EXP(bool, EmulatedTLS)
 CGOPT_EXP(bool, EnableTLSDESC)
 CGOPT(bool, UniqueSectionNames)
 CGOPT(bool, UniqueBasicBlockSectionNames)
+CGOPT(bool, SeparateNamedSections)
 CGOPT(EABI, EABIVersion)
 CGOPT(DebuggerKind, DebuggerTuningOpt)
 CGOPT(bool, EnableStackSizeSection)
@@ -419,6 +420,12 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() {
       cl::init(false));
   CGBINDOPT(UniqueBasicBlockSectionNames);
 
+  static cl::opt<bool> SeparateNamedSections(
+      "separate-named-sections",
+      cl::desc("Use separate unique sections for named sections"),
+      cl::init(false));
+  CGBINDOPT(SeparateNamedSections);
+
   static cl::opt<EABI> EABIVersion(
       "meabi", cl::desc("Set EABI type (default depends on triple):"),
       cl::init(EABI::Default),
@@ -569,6 +576,7 @@ codegen::InitTargetOptionsFromCodeGenFlags(const Triple &TheTriple) {
   Options.BBSections = getBBSectionsMode(Options);
   Options.UniqueSectionNames = getUniqueSectionNames();
   Options.UniqueBasicBlockSectionNames = getUniqueBasicBlockSectionNames();
+  Options.SeparateNamedSections = getSeparateNamedSections();
   Options.TLSSize = getTLSSize();
   Options.EmulatedTLS =
       getExplicitEmulatedTLS().value_or(TheTriple.hasDefaultEmulatedTLS());
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 2a77a683a901..81f3864ee4d0 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -733,15 +733,20 @@ calcUniqueIDUpdateFlagsAndSize(const GlobalObject *GO, StringRef SectionName,
       Ctx.isELFGenericMergeableSection(SectionName);
   // If this is the first ocurrence of this section name, treat it as the
   // generic section
-  if (!SymbolMergeable && !SeenSectionNameBefore)
-    return MCContext::GenericSectionID;
+  if (!SymbolMergeable && !SeenSectionNameBefore) {
+    if (TM.getSeparateNamedSections())
+      return NextUniqueID++;
+    else
+      return MCContext::GenericSectionID;
+  }
 
   // Symbols must be placed into sections with compatible entry sizes. Generate
   // unique sections for symbols that have not been assigned to compatible
   // sections.
   const auto PreviousID =
       Ctx.getELFUniqueIDForEntsize(SectionName, Flags, EntrySize);
-  if (PreviousID)
+  if (PreviousID && (!TM.getSeparateNamedSections() ||
+                     *PreviousID == MCContext::GenericSectionID))
     return *PreviousID;
 
   // If the user has specified the same section name as would be created
diff --git a/llvm/test/CodeGen/X86/elf-separate-named-sections.ll b/llvm/test/CodeGen/X86/elf-separate-named-sections.ll
new file mode 100644
index 000000000000..18efc20aa945
--- /dev/null
+++ b/llvm/test/CodeGen/X86/elf-separate-named-sections.ll
@@ -0,0 +1,36 @@
+; Test that global values with explicit sections are placed into unique sections.
+
+; RUN: llc < %s | FileCheck %s
+; RUN: llc -separate-named-sections < %s | FileCheck %s --check-prefix=SEPARATE
+target triple="x86_64-unknown-unknown-elf"
+
+define i32 @f() section "custom_text" {
+    entry:
+    ret i32 0
+}
+
+define i32 @g() section "custom_text" {
+    entry:
+    ret i32 0
+}
+
+; CHECK: .section custom_text,"ax",@progbits{{$}}
+; CHECK: f:
+; CHECK: g:
+
+; SEPARATE: .section custom_text,"ax",@progbits,unique,1{{$}}
+; SEPARATE: f:
+; SEPARATE: .section custom_text,"ax",@progbits,unique,2{{$}}
+; SEPARATE: g:
+
+@i = global i32 0, section "custom_data", align 8
+@j = global i32 0, section "custom_data", align 8
+
+; CHECK: .section custom_data,"aw",@progbits{{$}}
+; CHECK: i:
+; CHECK: j:
+
+; SEPARATE: .section custom_data,"aw",@progbits,unique,3{{$}}
+; SEPARATE: i:
+; SEPARATE: .section custom_data,"aw",@progbits,unique,4{{$}}
+; SEPARATE: j:
-- 
GitLab


From 5c5116556f58d90353aa3e3a34214cdc5ff0b2f2 Mon Sep 17 00:00:00 2001
From: Aart Bik <ajcbik@google.com>
Date: Tue, 7 May 2024 09:20:56 -0700
Subject: [PATCH 0067/1206] [mlir][sparse] force a properly sized view on
 pos/crd/val under codegen (#91288)

Codegen "vectors" for pos/crd/val use the capacity as memref size, not
the actual used size. Although the sparsifier itself always uses just
the defined pos/crd/val parts, printing these and passing them back to a
runtime environment could benefit from wrapping the basic pos/crd/val
getters into a proper memref view that sets the right size.
---
 .../Transforms/SparseTensorCodegen.cpp        |  45 +++--
 .../Dialect/SparseTensor/binary_valued.mlir   |  69 +++----
 mlir/test/Dialect/SparseTensor/codegen.mlir   |  23 ++-
 .../SparseTensor/sparse_matmul_codegen.mlir   | 170 ++++++++----------
 .../Dialect/SparseTensor/CPU/sparse_ds.mlir   |  18 +-
 .../SparseTensor/CPU/sparse_empty.mlir        |  22 +--
 .../SparseTensor/CPU/sparse_print.mlir        | 102 +++++------
 .../GPU/CUDA/sparse-gemm-lib.mlir             |   6 +-
 .../GPU/CUDA/sparse-sampled-matmul-lib.mlir   |  12 +-
 .../GPU/CUDA/sparse-sddmm-lib.mlir            |  12 +-
 10 files changed, 240 insertions(+), 239 deletions(-)

diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index 5679f277e148..d9b203a88648 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -1050,10 +1050,14 @@ public:
   matchAndRewrite(ToPositionsOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Replace the requested position access with corresponding field.
-    // The cast_op is inserted by type converter to intermix 1:N type
-    // conversion.
+    // The view is restricted to the actual size to ensure clients
+    // of this operation truly observe size, not capacity!
+    Location loc = op.getLoc();
+    Level lvl = op.getLevel();
     auto desc = getDescriptorFromTensorTuple(adaptor.getTensor());
-    rewriter.replaceOp(op, desc.getPosMemRef(op.getLevel()));
+    auto mem = desc.getPosMemRef(lvl);
+    auto size = desc.getPosMemSize(rewriter, loc, lvl);
+    rewriter.replaceOp(op, genSliceToSize(rewriter, loc, mem, size));
     return success();
   }
 };
@@ -1068,12 +1072,17 @@ public:
   matchAndRewrite(ToCoordinatesOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Replace the requested coordinates access with corresponding field.
-    // The cast_op is inserted by type converter to intermix 1:N type
-    // conversion.
+    // The view is restricted to the actual size to ensure clients
+    // of this operation truly observe size, not capacity!
+    Location loc = op.getLoc();
+    Level lvl = op.getLevel();
     auto desc = getDescriptorFromTensorTuple(adaptor.getTensor());
-    rewriter.replaceOp(
-        op, desc.getCrdMemRefOrView(rewriter, op.getLoc(), op.getLevel()));
-
+    auto mem = desc.getCrdMemRefOrView(rewriter, loc, lvl);
+    if (lvl < getSparseTensorType(op.getTensor()).getAoSCOOStart()) {
+      auto size = desc.getCrdMemSize(rewriter, loc, lvl);
+      mem = genSliceToSize(rewriter, loc, mem, size);
+    }
+    rewriter.replaceOp(op, mem);
     return success();
   }
 };
@@ -1088,11 +1097,14 @@ public:
   matchAndRewrite(ToCoordinatesBufferOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Replace the requested coordinates access with corresponding field.
-    // The cast_op is inserted by type converter to intermix 1:N type
-    // conversion.
+    // The view is restricted to the actual size to ensure clients
+    // of this operation truly observe size, not capacity!
+    Location loc = op.getLoc();
+    Level lvl = getSparseTensorType(op.getTensor()).getAoSCOOStart();
     auto desc = getDescriptorFromTensorTuple(adaptor.getTensor());
-    rewriter.replaceOp(op, desc.getAOSMemRef());
-
+    auto mem = desc.getAOSMemRef();
+    auto size = desc.getCrdMemSize(rewriter, loc, lvl);
+    rewriter.replaceOp(op, genSliceToSize(rewriter, loc, mem, size));
     return success();
   }
 };
@@ -1106,10 +1118,13 @@ public:
   matchAndRewrite(ToValuesOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     // Replace the requested values access with corresponding field.
-    // The cast_op is inserted by type converter to intermix 1:N type
-    // conversion.
+    // The view is restricted to the actual size to ensure clients
+    // of this operation truly observe size, not capacity!
+    Location loc = op.getLoc();
     auto desc = getDescriptorFromTensorTuple(adaptor.getTensor());
-    rewriter.replaceOp(op, desc.getValMemRef());
+    auto mem = desc.getValMemRef();
+    auto size = desc.getValMemSize(rewriter, loc);
+    rewriter.replaceOp(op, genSliceToSize(rewriter, loc, mem, size));
     return success();
   }
 };
diff --git a/mlir/test/Dialect/SparseTensor/binary_valued.mlir b/mlir/test/Dialect/SparseTensor/binary_valued.mlir
index e2d410b126a7..dd9b60a6488b 100755
--- a/mlir/test/Dialect/SparseTensor/binary_valued.mlir
+++ b/mlir/test/Dialect/SparseTensor/binary_valued.mlir
@@ -26,12 +26,11 @@
 //
 // Make sure X += A * A => X += 1 in single loop.
 //
-//
 // CHECK-LABEL:   func.func @sum_squares(
 // CHECK-SAME:      %[[VAL_0:.*0]]: memref<?xindex>,
 // CHECK-SAME:      %[[VAL_1:.*1]]: memref<?xindex>,
 // CHECK-SAME:      %[[VAL_2:.*2]]: memref<?xf32>,
-// CHECK-SAME:      %[[VAL_3:.*3]]: !sparse_tensor.storage_specifier<#{{.*}}>) -> memref<f32> {
+// CHECK-SAME:      %[[VAL_3:.*]]: !sparse_tensor.storage_specifier<#{{.*}}>) -> memref<f32> {
 // CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 1 : index
 // CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 0 : index
@@ -40,23 +39,25 @@
 // CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
 // CHECK:           %[[VAL_10:.*]] = memref.alloc() {alignment = 64 : i64} : memref<f32>
 // CHECK:           linalg.fill ins(%[[VAL_9]] : f32) outs(%[[VAL_10]] : memref<f32>)
-// CHECK:           %[[VAL_11:.*]] = memref.load %[[VAL_10]][] : memref<f32>
-// CHECK:           %[[VAL_12:.*]] = scf.for %[[VAL_13:.*]] = %[[VAL_6]] to %[[VAL_8]] step %[[VAL_5]] iter_args(%[[VAL_14:.*]] = %[[VAL_11]]) -> (f32) {
-// CHECK:             %[[VAL_15:.*]] = arith.muli %[[VAL_13]], %[[VAL_7]] : index
-// CHECK:             %[[VAL_16:.*]] = scf.for %[[VAL_17:.*]] = %[[VAL_6]] to %[[VAL_7]] step %[[VAL_5]] iter_args(%[[VAL_18:.*]] = %[[VAL_14]]) -> (f32) {
-// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_17]], %[[VAL_15]] : index
-// CHECK:               %[[VAL_20:.*]] = memref.load %[[VAL_0]]{{\[}}%[[VAL_19]]] : memref<?xindex>
-// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_19]], %[[VAL_5]] : index
-// CHECK:               %[[VAL_22:.*]] = memref.load %[[VAL_0]]{{\[}}%[[VAL_21]]] : memref<?xindex>
-// CHECK:               %[[VAL_23:.*]] = scf.for %[[VAL_24:.*]] = %[[VAL_20]] to %[[VAL_22]] step %[[VAL_5]] iter_args(%[[VAL_25:.*]] = %[[VAL_18]]) -> (f32) {
-// CHECK:                 %[[VAL_26:.*]] = arith.addf %[[VAL_25]], %[[VAL_4]] : f32
-// CHECK:                 scf.yield %[[VAL_26]] : f32
+// CHECK:           %[[VAL_11:.*]] = sparse_tensor.storage_specifier.get %[[VAL_3]]
+// CHECK:           %[[VAL_12:.*]] = memref.subview %[[VAL_0]][0] {{\[}}%[[VAL_11]]] [1] : memref<?xindex> to memref<?xindex>
+// CHECK:           %[[VAL_13:.*]] = memref.load %[[VAL_10]][] : memref<f32>
+// CHECK:           %[[VAL_14:.*]] = scf.for %[[VAL_15:.*]] = %[[VAL_6]] to %[[VAL_8]] step %[[VAL_5]] iter_args(%[[VAL_16:.*]] = %[[VAL_13]]) -> (f32) {
+// CHECK:             %[[VAL_17:.*]] = arith.muli %[[VAL_15]], %[[VAL_7]] : index
+// CHECK:             %[[VAL_18:.*]] = scf.for %[[VAL_19:.*]] = %[[VAL_6]] to %[[VAL_7]] step %[[VAL_5]] iter_args(%[[VAL_20:.*]] = %[[VAL_16]]) -> (f32) {
+// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_19]], %[[VAL_17]] : index
+// CHECK:               %[[VAL_22:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_21]]] : memref<?xindex>
+// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_21]], %[[VAL_5]] : index
+// CHECK:               %[[VAL_24:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_23]]] : memref<?xindex>
+// CHECK:               %[[VAL_25:.*]] = scf.for %[[VAL_26:.*]] = %[[VAL_22]] to %[[VAL_24]] step %[[VAL_5]] iter_args(%[[VAL_27:.*]] = %[[VAL_20]]) -> (f32) {
+// CHECK:                 %[[VAL_28:.*]] = arith.addf %[[VAL_27]], %[[VAL_4]] : f32
+// CHECK:                 scf.yield %[[VAL_28]] : f32
 // CHECK:               } {"Emitted from" = "linalg.generic"}
-// CHECK:               scf.yield %[[VAL_23]] : f32
+// CHECK:               scf.yield %[[VAL_25]] : f32
 // CHECK:             } {"Emitted from" = "linalg.generic"}
-// CHECK:             scf.yield %[[VAL_16]] : f32
+// CHECK:             scf.yield %[[VAL_18]] : f32
 // CHECK:           } {"Emitted from" = "linalg.generic"}
-// CHECK:           memref.store %[[VAL_12]], %[[VAL_10]][] : memref<f32>
+// CHECK:           memref.store %[[VAL_14]], %[[VAL_10]][] : memref<f32>
 // CHECK:           return %[[VAL_10]] : memref<f32>
 // CHECK:         }
 //
@@ -99,25 +100,29 @@ func.func @sum_squares(%a: tensor<2x3x8xf32, #Sparse>) -> tensor<f32> {
 // CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
 // CHECK:           %[[VAL_10:.*]] = memref.alloc() {alignment = 64 : i64} : memref<f32>
 // CHECK:           linalg.fill ins(%[[VAL_9]] : f32) outs(%[[VAL_10]] : memref<f32>)
-// CHECK:           %[[VAL_11:.*]] = memref.load %[[VAL_10]][] : memref<f32>
-// CHECK:           %[[VAL_12:.*]] = scf.for %[[VAL_13:.*]] = %[[VAL_6]] to %[[VAL_8]] step %[[VAL_5]] iter_args(%[[VAL_14:.*]] = %[[VAL_11]]) -> (f32) {
-// CHECK:             %[[VAL_15:.*]] = arith.muli %[[VAL_13]], %[[VAL_7]] : index
-// CHECK:             %[[VAL_16:.*]] = scf.for %[[VAL_17:.*]] = %[[VAL_6]] to %[[VAL_7]] step %[[VAL_5]] iter_args(%[[VAL_18:.*]] = %[[VAL_14]]) -> (f32) {
-// CHECK:               %[[VAL_19:.*]] = arith.addi %[[VAL_17]], %[[VAL_15]] : index
-// CHECK:               %[[VAL_20:.*]] = memref.load %[[VAL_0]]{{\[}}%[[VAL_19]]] : memref<?xindex>
-// CHECK:               %[[VAL_21:.*]] = arith.addi %[[VAL_19]], %[[VAL_5]] : index
-// CHECK:               %[[VAL_22:.*]] = memref.load %[[VAL_0]]{{\[}}%[[VAL_21]]] : memref<?xindex>
-// CHECK:               %[[VAL_23:.*]] = scf.for %[[VAL_24:.*]] = %[[VAL_20]] to %[[VAL_22]] step %[[VAL_5]] iter_args(%[[VAL_25:.*]] = %[[VAL_18]]) -> (f32) {
-// CHECK:                 %[[VAL_26:.*]] = memref.load %[[VAL_1]]{{\[}}%[[VAL_24]]] : memref<?xindex>
-// CHECK:                 %[[VAL_27:.*]] = memref.load %[[VAL_4]]{{\[}}%[[VAL_13]], %[[VAL_17]], %[[VAL_26]]] : memref<2x3x8xf32>
-// CHECK:                 %[[VAL_28:.*]] = arith.addf %[[VAL_27]], %[[VAL_25]] : f32
-// CHECK:                 scf.yield %[[VAL_28]] : f32
+// CHECK:           %[[VAL_11:.*]] = sparse_tensor.storage_specifier.get %[[VAL_3]]
+// CHECK:           %[[VAL_12:.*]] = memref.subview %[[VAL_0]][0] {{\[}}%[[VAL_11]]] [1] : memref<?xindex> to memref<?xindex>
+// CHECK:           %[[VAL_13:.*]] = sparse_tensor.storage_specifier.get %[[VAL_3]]
+// CHECK:           %[[VAL_14:.*]] = memref.subview %[[VAL_1]][0] {{\[}}%[[VAL_13]]] [1] : memref<?xindex> to memref<?xindex>
+// CHECK:           %[[VAL_15:.*]] = memref.load %[[VAL_10]][] : memref<f32>
+// CHECK:           %[[VAL_16:.*]] = scf.for %[[VAL_17:.*]] = %[[VAL_6]] to %[[VAL_8]] step %[[VAL_5]] iter_args(%[[VAL_18:.*]] = %[[VAL_15]]) -> (f32) {
+// CHECK:             %[[VAL_19:.*]] = arith.muli %[[VAL_17]], %[[VAL_7]] : index
+// CHECK:             %[[VAL_20:.*]] = scf.for %[[VAL_21:.*]] = %[[VAL_6]] to %[[VAL_7]] step %[[VAL_5]] iter_args(%[[VAL_22:.*]] = %[[VAL_18]]) -> (f32) {
+// CHECK:               %[[VAL_23:.*]] = arith.addi %[[VAL_21]], %[[VAL_19]] : index
+// CHECK:               %[[VAL_24:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_23]]] : memref<?xindex>
+// CHECK:               %[[VAL_25:.*]] = arith.addi %[[VAL_23]], %[[VAL_5]] : index
+// CHECK:               %[[VAL_26:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_25]]] : memref<?xindex>
+// CHECK:               %[[VAL_27:.*]] = scf.for %[[VAL_28:.*]] = %[[VAL_24]] to %[[VAL_26]] step %[[VAL_5]] iter_args(%[[VAL_29:.*]] = %[[VAL_22]]) -> (f32) {
+// CHECK:                 %[[VAL_30:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_28]]] : memref<?xindex>
+// CHECK:                 %[[VAL_31:.*]] = memref.load %[[VAL_4]]{{\[}}%[[VAL_17]], %[[VAL_21]], %[[VAL_30]]] : memref<2x3x8xf32>
+// CHECK:                 %[[VAL_32:.*]] = arith.addf %[[VAL_31]], %[[VAL_29]] : f32
+// CHECK:                 scf.yield %[[VAL_32]] : f32
 // CHECK:               } {"Emitted from" = "linalg.generic"}
-// CHECK:               scf.yield %[[VAL_23]] : f32
+// CHECK:               scf.yield %[[VAL_27]] : f32
 // CHECK:             } {"Emitted from" = "linalg.generic"}
-// CHECK:             scf.yield %[[VAL_16]] : f32
+// CHECK:             scf.yield %[[VAL_20]] : f32
 // CHECK:           } {"Emitted from" = "linalg.generic"}
-// CHECK:           memref.store %[[VAL_12]], %[[VAL_10]][] : memref<f32>
+// CHECK:           memref.store %[[VAL_16]], %[[VAL_10]][] : memref<f32>
 // CHECK:           return %[[VAL_10]] : memref<f32>
 // CHECK:         }
 //
diff --git a/mlir/test/Dialect/SparseTensor/codegen.mlir b/mlir/test/Dialect/SparseTensor/codegen.mlir
index 40bfa1e4e2a5..af78458f1093 100644
--- a/mlir/test/Dialect/SparseTensor/codegen.mlir
+++ b/mlir/test/Dialect/SparseTensor/codegen.mlir
@@ -266,7 +266,9 @@ func.func @sparse_dense_3d_dyn(%arg0: tensor<?x?x?xf64, #Dense3D>) -> index {
 //  CHECK-SAME: %[[A3:.*3]]: memref<?xi64>,
 //  CHECK-SAME: %[[A4:.*4]]: memref<?xf64>,
 //  CHECK-SAME: %[[A5:.*5]]: !sparse_tensor.storage_specifier
-//       CHECK: return %[[A2]] : memref<?xi32>
+//       CHECK: %[[S:.*]] = sparse_tensor.storage_specifier.get %[[A5]] pos_mem_sz at 1
+//       CHECK: %[[V:.*]] = memref.subview %[[A2]][0] [%[[S]]] [1]
+//       CHECK: return %[[V]] : memref<?xi32>
 func.func @sparse_positions_dcsr(%arg0: tensor<?x?xf64, #DCSR>) -> memref<?xi32> {
   %0 = sparse_tensor.positions %arg0 { level = 1 : index } : tensor<?x?xf64, #DCSR> to memref<?xi32>
   return %0 : memref<?xi32>
@@ -279,7 +281,9 @@ func.func @sparse_positions_dcsr(%arg0: tensor<?x?xf64, #DCSR>) -> memref<?xi32>
 //  CHECK-SAME: %[[A3:.*3]]: memref<?xi64>,
 //  CHECK-SAME: %[[A4:.*4]]: memref<?xf64>,
 //  CHECK-SAME: %[[A5:.*5]]: !sparse_tensor.storage_specifier
-//       CHECK: return %[[A3]] : memref<?xi64>
+//       CHECK: %[[S:.*]] = sparse_tensor.storage_specifier.get %[[A5]] crd_mem_sz at 1
+//       CHECK: %[[V:.*]] = memref.subview %[[A3]][0] [%[[S]]] [1]
+//       CHECK: return %[[V]] : memref<?xi64>
 func.func @sparse_indices_dcsr(%arg0: tensor<?x?xf64, #DCSR>) -> memref<?xi64> {
   %0 = sparse_tensor.coordinates %arg0 { level = 1 : index } : tensor<?x?xf64, #DCSR> to memref<?xi64>
   return %0 : memref<?xi64>
@@ -292,7 +296,9 @@ func.func @sparse_indices_dcsr(%arg0: tensor<?x?xf64, #DCSR>) -> memref<?xi64> {
 //  CHECK-SAME: %[[A3:.*3]]: memref<?xi64>,
 //  CHECK-SAME: %[[A4:.*4]]: memref<?xf64>,
 //  CHECK-SAME: %[[A5:.*5]]: !sparse_tensor.storage_specifier
-//       CHECK: return %[[A4]] : memref<?xf64>
+//       CHECK: %[[S:.*]] = sparse_tensor.storage_specifier.get %[[A5]] val_mem_sz
+//       CHECK: %[[V:.*]] = memref.subview %[[A4]][0] [%[[S]]] [1]
+//       CHECK: return %[[V]] : memref<?xf64>
 func.func @sparse_values_dcsr(%arg0: tensor<?x?xf64, #DCSR>) -> memref<?xf64> {
   %0 = sparse_tensor.values %arg0 : tensor<?x?xf64, #DCSR> to memref<?xf64>
   return %0 : memref<?xf64>
@@ -305,13 +311,14 @@ func.func @sparse_values_dcsr(%arg0: tensor<?x?xf64, #DCSR>) -> memref<?xf64> {
 //  CHECK-SAME: %[[A3:.*3]]: memref<?xindex>,
 //  CHECK-SAME: %[[A4:.*4]]: memref<?xf64>,
 //  CHECK-SAME: %[[A5:.*5]]: !sparse_tensor.storage_specifier
-//       CHECK: return %[[A4]] : memref<?xf64>
+//       CHECK: %[[S:.*]] = sparse_tensor.storage_specifier.get %[[A5]] val_mem_sz
+//       CHECK: %[[V:.*]] = memref.subview %[[A4]][0] [%[[S]]] [1]
+//       CHECK: return %[[V]] : memref<?xf64>
 func.func @sparse_values_coo(%arg0: tensor<?x?x?xf64, #ccoo>) -> memref<?xf64> {
   %0 = sparse_tensor.values %arg0 : tensor<?x?x?xf64, #ccoo> to memref<?xf64>
   return %0 : memref<?xf64>
 }
 
-
 // CHECK-LABEL: func.func @sparse_indices_coo(
 //  CHECK-SAME: %[[A0:.*0]]: memref<?xindex>,
 //  CHECK-SAME: %[[A1:.*1]]: memref<?xindex>,
@@ -320,7 +327,7 @@ func.func @sparse_values_coo(%arg0: tensor<?x?x?xf64, #ccoo>) -> memref<?xf64> {
 //  CHECK-SAME: %[[A4:.*4]]: memref<?xf64>,
 //  CHECK-SAME: %[[A5:.*5]]: !sparse_tensor.storage_specifier
 //       CHECK: %[[C2:.*]] = arith.constant 2 : index
-//       CHECK: %[[S0:.*]] = sparse_tensor.storage_specifier.get %[[A5]]  crd_mem_sz at 1
+//       CHECK: %[[S0:.*]] = sparse_tensor.storage_specifier.get %[[A5]] crd_mem_sz at 1
 //       CHECK: %[[S2:.*]] = arith.divui %[[S0]], %[[C2]] : index
 //       CHECK: %[[R1:.*]] = memref.subview %[[A3]][0] {{\[}}%[[S2]]] [2] : memref<?xindex> to memref<?xindex, strided<[2]>>
 //       CHECK: %[[R2:.*]] = memref.cast %[[R1]] : memref<?xindex, strided<[2]>> to memref<?xindex, strided<[?], offset: ?>>
@@ -337,7 +344,9 @@ func.func @sparse_indices_coo(%arg0: tensor<?x?x?xf64, #ccoo>) -> memref<?xindex
 //  CHECK-SAME: %[[A3:.*3]]: memref<?xindex>,
 //  CHECK-SAME: %[[A4:.*4]]: memref<?xf64>,
 //  CHECK-SAME: %[[A5:.*5]]: !sparse_tensor.storage_specifier
-//       CHECK: return %[[A3]] : memref<?xindex>
+//       CHECK: %[[S:.*]] = sparse_tensor.storage_specifier.get %[[A5]] crd_mem_sz at 1
+//       CHECK: %[[V:.*]] = memref.subview %[[A3]][0] [%[[S]]] [1]
+//       CHECK: return %[[V]] : memref<?xindex>
 func.func @sparse_indices_buffer_coo(%arg0: tensor<?x?x?xf64, #ccoo>) -> memref<?xindex> {
   %0 = sparse_tensor.coordinates_buffer  %arg0 : tensor<?x?x?xf64, #ccoo> to memref<?xindex>
   return %0 : memref<?xindex>
diff --git a/mlir/test/Dialect/SparseTensor/sparse_matmul_codegen.mlir b/mlir/test/Dialect/SparseTensor/sparse_matmul_codegen.mlir
index 5145d6c1dcfc..ad12b637d0c5 100644
--- a/mlir/test/Dialect/SparseTensor/sparse_matmul_codegen.mlir
+++ b/mlir/test/Dialect/SparseTensor/sparse_matmul_codegen.mlir
@@ -1,5 +1,3 @@
-// NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
-
 // RUN: mlir-opt %s --linalg-generalize-named-ops \
 // RUN:  --sparse-reinterpret-map --sparsification --sparse-tensor-codegen \
 // RUN:  --canonicalize --cse | FileCheck %s
@@ -11,45 +9,6 @@
 //
 // Computes C = A x B with all matrices sparse (SpMSpM) in CSR.
 //
-// CHECK-LABEL:   func.func private @_insert_dense_compressed_4_4_f64_0_0(
-// CHECK-SAME:      %[[VAL_0:.*0]]: memref<?xindex>,
-// CHECK-SAME:      %[[VAL_1:.*1]]: memref<?xindex>,
-// CHECK-SAME:      %[[VAL_2:.*2]]: memref<?xf64>,
-// CHECK-SAME:      %[[VAL_3:.*3]]: !sparse_tensor.storage_specifier
-// CHECK-SAME:      %[[VAL_4:.*4]]: index,
-// CHECK-SAME:      %[[VAL_5:.*5]]: index,
-// CHECK-SAME:      %[[VAL_6:.*6]]: f64) -> (memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
-// CHECK:           %[[VAL_7:.*]] = arith.constant false
-// CHECK:           %[[VAL_8:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_9:.*]] = arith.addi %[[VAL_4]], %[[VAL_8]] : index
-// CHECK:           %[[VAL_10:.*]] = memref.load %[[VAL_0]]{{\[}}%[[VAL_4]]] : memref<?xindex>
-// CHECK:           %[[VAL_11:.*]] = memref.load %[[VAL_0]]{{\[}}%[[VAL_9]]] : memref<?xindex>
-// CHECK:           %[[VAL_13:.*]] = sparse_tensor.storage_specifier.get %[[VAL_3]]  crd_mem_sz at 1 : !sparse_tensor.storage_specifier
-// CHECK:           %[[VAL_14:.*]] = arith.subi %[[VAL_11]], %[[VAL_8]] : index
-// CHECK:           %[[VAL_15:.*]] = arith.cmpi ult, %[[VAL_10]], %[[VAL_11]] : index
-// CHECK:           %[[VAL_16:.*]] = scf.if %[[VAL_15]] -> (i1) {
-// CHECK:             %[[VAL_17:.*]] = memref.load %[[VAL_1]]{{\[}}%[[VAL_14]]] : memref<?xindex>
-// CHECK:             %[[VAL_18:.*]] = arith.cmpi eq, %[[VAL_17]], %[[VAL_5]] : index
-// CHECK:             scf.yield %[[VAL_18]] : i1
-// CHECK:           } else {
-// CHECK:             memref.store %[[VAL_13]], %[[VAL_0]]{{\[}}%[[VAL_4]]] : memref<?xindex>
-// CHECK:             scf.yield %[[VAL_7]] : i1
-// CHECK:           }
-// CHECK:           %[[VAL_19:.*]]:2 = scf.if %[[VAL_20:.*]] -> (memref<?xindex>, !sparse_tensor.storage_specifier
-// CHECK:             scf.yield %[[VAL_1]], %[[VAL_3]] : memref<?xindex>, !sparse_tensor.storage_specifier
-// CHECK:           } else {
-// CHECK:             %[[VAL_21:.*]] = arith.addi %[[VAL_13]], %[[VAL_8]] : index
-// CHECK:             memref.store %[[VAL_21]], %[[VAL_0]]{{\[}}%[[VAL_9]]] : memref<?xindex>
-// CHECK:             %[[VAL_22:.*]], %[[VAL_24:.*]] = sparse_tensor.push_back %[[VAL_13]], %[[VAL_1]], %[[VAL_5]] : index, memref<?xindex>, index
-// CHECK:             %[[VAL_25:.*]] = sparse_tensor.storage_specifier.set %[[VAL_3]]  crd_mem_sz at 1 with %[[VAL_24]] : !sparse_tensor.storage_specifier
-// CHECK:             scf.yield %[[VAL_22]], %[[VAL_25]] : memref<?xindex>, !sparse_tensor.storage_specifier
-// CHECK:           }
-// CHECK:           %[[VAL_28:.*]] = sparse_tensor.storage_specifier.get %[[VAL_27:.*]]#1  val_mem_sz : !sparse_tensor.storage_specifier
-// CHECK:           %[[VAL_29:.*]], %[[VAL_30:.*]] = sparse_tensor.push_back %[[VAL_28]], %[[VAL_2]], %[[VAL_6]] : index, memref<?xf64>, f64
-// CHECK:           %[[VAL_32:.*]] = sparse_tensor.storage_specifier.set %[[VAL_27]]#1  val_mem_sz with %[[VAL_30]] : !sparse_tensor.storage_specifier
-// CHECK:           return %[[VAL_0]], %[[VAL_27]]#0, %[[VAL_29]], %[[VAL_32]] : memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
-// CHECK:         }
-
 // CHECK-LABEL:   func.func @matmul(
 // CHECK-SAME:      %[[VAL_0:.*0]]: memref<?xindex>,
 // CHECK-SAME:      %[[VAL_1:.*1]]: memref<?xindex>,
@@ -59,12 +18,12 @@
 // CHECK-SAME:      %[[VAL_5:.*5]]: memref<?xindex>,
 // CHECK-SAME:      %[[VAL_6:.*6]]: memref<?xf64>,
 // CHECK-SAME:      %[[VAL_7:.*7]]: !sparse_tensor.storage_specifier
-// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 4 : index
-// CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f64
-// CHECK-DAG:       %[[VAL_10:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 0.000000e+00 : f64
+// CHECK-DAG:       %[[VAL_9:.*]] = arith.constant true
+// CHECK-DAG:       %[[VAL_10:.*]] = arith.constant false
 // CHECK-DAG:       %[[VAL_11:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_12:.*]] = arith.constant false
-// CHECK-DAG:       %[[VAL_13:.*]] = arith.constant true
+// CHECK-DAG:       %[[VAL_12:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[VAL_13:.*]] = arith.constant 4 : index
 // CHECK:           %[[VAL_14:.*]] = memref.alloc() : memref<16xindex>
 // CHECK:           %[[VAL_15:.*]] = memref.cast %[[VAL_14]] : memref<16xindex> to memref<?xindex>
 // CHECK:           %[[VAL_16:.*]] = memref.alloc() : memref<16xindex>
@@ -72,76 +31,89 @@
 // CHECK:           %[[VAL_18:.*]] = memref.alloc() : memref<16xf64>
 // CHECK:           %[[VAL_19:.*]] = memref.cast %[[VAL_18]] : memref<16xf64> to memref<?xf64>
 // CHECK:           %[[VAL_20:.*]] = sparse_tensor.storage_specifier.init : !sparse_tensor.storage_specifier
-// CHECK:           %[[VAL_21:.*]] = sparse_tensor.storage_specifier.set %[[VAL_20]]  lvl_sz at 0 with %[[VAL_8]] : !sparse_tensor.storage_specifier
-// CHECK:           %[[VAL_22:.*]] = sparse_tensor.storage_specifier.set %[[VAL_21]]  lvl_sz at 1 with %[[VAL_8]] : !sparse_tensor.storage_specifier
+// CHECK:           %[[VAL_21:.*]] = sparse_tensor.storage_specifier.set %[[VAL_20]]  lvl_sz at 0 with %[[VAL_13]] : !sparse_tensor.storage_specifier
+// CHECK:           %[[VAL_22:.*]] = sparse_tensor.storage_specifier.set %[[VAL_21]]  lvl_sz at 1 with %[[VAL_13]] : !sparse_tensor.storage_specifier
 // CHECK:           %[[VAL_23:.*]] = sparse_tensor.storage_specifier.get %[[VAL_22]]  pos_mem_sz at 1 : !sparse_tensor.storage_specifier
-// CHECK:           %[[VAL_24:.*]], %[[VAL_25:.*]] = sparse_tensor.push_back %[[VAL_23]], %[[VAL_15]], %[[VAL_10]] : index, memref<?xindex>, index
+// CHECK:           %[[VAL_24:.*]], %[[VAL_25:.*]] = sparse_tensor.push_back %[[VAL_23]], %[[VAL_15]], %[[VAL_12]] : index, memref<?xindex>, index
 // CHECK:           %[[VAL_26:.*]] = sparse_tensor.storage_specifier.set %[[VAL_22]]  pos_mem_sz at 1 with %[[VAL_25]] : !sparse_tensor.storage_specifier
-// CHECK:           %[[VAL_27:.*]], %[[VAL_28:.*]] = sparse_tensor.push_back %[[VAL_25]], %[[VAL_24]], %[[VAL_10]], %[[VAL_8]] : index, memref<?xindex>, index, index
+// CHECK:           %[[VAL_27:.*]], %[[VAL_28:.*]] = sparse_tensor.push_back %[[VAL_25]], %[[VAL_24]], %[[VAL_12]], %[[VAL_13]] : index, memref<?xindex>, index, index
 // CHECK:           %[[VAL_29:.*]] = sparse_tensor.storage_specifier.set %[[VAL_26]]  pos_mem_sz at 1 with %[[VAL_28]] : !sparse_tensor.storage_specifier
 // CHECK:           %[[VAL_30:.*]] = memref.alloc() : memref<4xf64>
-// CHECK:           %[[VAL_31:.*]] = m
+// CHECK:           %[[VAL_31:.*]] = memref.alloc() : memref<4xi1>
 // CHECK:           %[[VAL_32:.*]] = memref.alloc() : memref<4xindex>
 // CHECK:           %[[VAL_33:.*]] = memref.cast %[[VAL_32]] : memref<4xindex> to memref<?xindex>
-// CHECK:           linalg.fill ins(%[[VAL_9]] : f64) outs(%[[VAL_30]] : memref<4xf64>)
-// CHECK:           linalg.fill ins(%[[VAL_12]] : i1) outs(%[[VAL_31]] : memref<4xi1>)
-// CHECK:           %[[VAL_34:.*]]:4 = scf.for %[[VAL_35:.*]] = %[[VAL_10]] to %[[VAL_8]] step %[[VAL_11]] iter_args(%[[VAL_36:.*]] = %[[VAL_27]], %[[VAL_37:.*]] = %[[VAL_17]], %[[VAL_38:.*]] = %[[VAL_19]], %[[VAL_39:.*]] = %[[VAL_29]]) -> (memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
-// CHECK:             %[[VAL_40:.*]] = memref.load %[[VAL_0]]{{\[}}%[[VAL_35]]] : memref<?xindex>
-// CHECK:             %[[VAL_41:.*]] = arith.addi %[[VAL_35]], %[[VAL_11]] : index
-// CHECK:             %[[VAL_42:.*]] = memref.load %[[VAL_0]]{{\[}}%[[VAL_41]]] : memref<?xindex>
-// CHECK:             %[[VAL_43:.*]] = scf.for %[[VAL_44:.*]] = %[[VAL_40]] to %[[VAL_42]] step %[[VAL_11]] iter_args(%[[VAL_45:.*]] = %[[VAL_10]]) -> (index) {
-// CHECK:               %[[VAL_46:.*]] = memref.load %[[VAL_1]]{{\[}}%[[VAL_44]]] : memref<?xindex>
-// CHECK:               %[[VAL_47:.*]] = memref.load %[[VAL_2]]{{\[}}%[[VAL_44]]] : memref<?xf64>
-// CHECK:               %[[VAL_48:.*]] = memref.load %[[VAL_4]]{{\[}}%[[VAL_46]]] : memref<?xindex>
-// CHECK:               %[[VAL_49:.*]] = arith.addi %[[VAL_46]], %[[VAL_11]] : index
-// CHECK:               %[[VAL_50:.*]] = memref.load %[[VAL_4]]{{\[}}%[[VAL_49]]] : memref<?xindex>
-// CHECK:               %[[VAL_51:.*]] = scf.for %[[VAL_52:.*]] = %[[VAL_48]] to %[[VAL_50]] step %[[VAL_11]] iter_args(%[[VAL_53:.*]] = %[[VAL_45]]) -> (index) {
-// CHECK:                 %[[VAL_54:.*]] = memref.load %[[VAL_5]]{{\[}}%[[VAL_52]]] : memref<?xindex>
-// CHECK:                 %[[VAL_55:.*]] = memref.load %[[VAL_30]]{{\[}}%[[VAL_54]]] : memref<4xf64>
-// CHECK:                 %[[VAL_56:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_52]]] : memref<?xf64>
-// CHECK:                 %[[VAL_57:.*]] = arith.mulf %[[VAL_47]], %[[VAL_56]] : f64
-// CHECK:                 %[[VAL_58:.*]] = arith.addf %[[VAL_55]], %[[VAL_57]] : f64
-// CHECK:                 %[[VAL_59:.*]] = memref.load %[[VAL_31]]{{\[}}%[[VAL_54]]] : memref<4xi1>
-// CHECK:                 %[[VAL_60:.*]] = arith.cmpi eq, %[[VAL_59]], %[[VAL_12]] : i1
-// CHECK:                 %[[VAL_61:.*]] = scf.if %[[VAL_60]] -> (index) {
-// CHECK:                   memref.store %[[VAL_13]], %[[VAL_31]]{{\[}}%[[VAL_54]]] : memref<4xi1>
-// CHECK:                   memref.store %[[VAL_54]], %[[VAL_32]]{{\[}}%[[VAL_53]]] : memref<4xindex>
-// CHECK:                   %[[VAL_62:.*]] = arith.addi %[[VAL_53]], %[[VAL_11]] : index
-// CHECK:                   scf.yield %[[VAL_62]] : index
+// CHECK:           linalg.fill ins(%[[VAL_8]] : f64) outs(%[[VAL_30]] : memref<4xf64>)
+// CHECK:           linalg.fill ins(%[[VAL_10]] : i1) outs(%[[VAL_31]] : memref<4xi1>)
+// CHECK:           %[[VAL_34:.*]] = sparse_tensor.storage_specifier.get %[[VAL_3]]  pos_mem_sz at 1 : !sparse_tensor.storage_specifier
+// CHECK:           %[[VAL_35:.*]] = memref.subview %[[VAL_0]][0] {{\[}}%[[VAL_34]]] [1] : memref<?xindex> to memref<?xindex>
+// CHECK:           %[[VAL_36:.*]] = sparse_tensor.storage_specifier.get %[[VAL_3]]  crd_mem_sz at 1 : !sparse_tensor.storage_specifier
+// CHECK:           %[[VAL_37:.*]] = memref.subview %[[VAL_1]][0] {{\[}}%[[VAL_36]]] [1] : memref<?xindex> to memref<?xindex>
+// CHECK:           %[[VAL_38:.*]] = sparse_tensor.storage_specifier.get %[[VAL_3]]  val_mem_sz : !sparse_tensor.storage_specifier
+// CHECK:           %[[VAL_39:.*]] = memref.subview %[[VAL_2]][0] {{\[}}%[[VAL_38]]] [1] : memref<?xf64> to memref<?xf64>
+// CHECK:           %[[VAL_40:.*]] = sparse_tensor.storage_specifier.get %[[VAL_7]]  pos_mem_sz at 1 : !sparse_tensor.storage_specifier
+// CHECK:           %[[VAL_41:.*]] = memref.subview %[[VAL_4]][0] {{\[}}%[[VAL_40]]] [1] : memref<?xindex> to memref<?xindex>
+// CHECK:           %[[VAL_42:.*]] = sparse_tensor.storage_specifier.get %[[VAL_7]]  crd_mem_sz at 1 : !sparse_tensor.storage_specifier
+// CHECK:           %[[VAL_43:.*]] = memref.subview %[[VAL_5]][0] {{\[}}%[[VAL_42]]] [1] : memref<?xindex> to memref<?xindex>
+// CHECK:           %[[VAL_44:.*]] = sparse_tensor.storage_specifier.get %[[VAL_7]]  val_mem_sz : !sparse_tensor.storage_specifier
+// CHECK:           %[[VAL_45:.*]] = memref.subview %[[VAL_6]][0] {{\[}}%[[VAL_44]]] [1] : memref<?xf64> to memref<?xf64>
+// CHECK:           %[[VAL_46:.*]]:4 = scf.for %[[VAL_47:.*]] = %[[VAL_12]] to %[[VAL_13]] step %[[VAL_11]] iter_args(%[[VAL_48:.*]] = %[[VAL_27]], %[[VAL_49:.*]] = %[[VAL_17]], %[[VAL_50:.*]] = %[[VAL_19]], %[[VAL_51:.*]] = %[[VAL_29]]) -> (memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
+// CHECK:             %[[VAL_52:.*]] = memref.load %[[VAL_35]]{{\[}}%[[VAL_47]]] : memref<?xindex>
+// CHECK:             %[[VAL_53:.*]] = arith.addi %[[VAL_47]], %[[VAL_11]] : index
+// CHECK:             %[[VAL_54:.*]] = memref.load %[[VAL_35]]{{\[}}%[[VAL_53]]] : memref<?xindex>
+// CHECK:             %[[VAL_55:.*]] = scf.for %[[VAL_56:.*]] = %[[VAL_52]] to %[[VAL_54]] step %[[VAL_11]] iter_args(%[[VAL_57:.*]] = %[[VAL_12]]) -> (index) {
+// CHECK:               %[[VAL_58:.*]] = memref.load %[[VAL_37]]{{\[}}%[[VAL_56]]] : memref<?xindex>
+// CHECK:               %[[VAL_59:.*]] = memref.load %[[VAL_39]]{{\[}}%[[VAL_56]]] : memref<?xf64>
+// CHECK:               %[[VAL_60:.*]] = memref.load %[[VAL_41]]{{\[}}%[[VAL_58]]] : memref<?xindex>
+// CHECK:               %[[VAL_61:.*]] = arith.addi %[[VAL_58]], %[[VAL_11]] : index
+// CHECK:               %[[VAL_62:.*]] = memref.load %[[VAL_41]]{{\[}}%[[VAL_61]]] : memref<?xindex>
+// CHECK:               %[[VAL_63:.*]] = scf.for %[[VAL_64:.*]] = %[[VAL_60]] to %[[VAL_62]] step %[[VAL_11]] iter_args(%[[VAL_65:.*]] = %[[VAL_57]]) -> (index) {
+// CHECK:                 %[[VAL_66:.*]] = memref.load %[[VAL_43]]{{\[}}%[[VAL_64]]] : memref<?xindex>
+// CHECK:                 %[[VAL_67:.*]] = memref.load %[[VAL_30]]{{\[}}%[[VAL_66]]] : memref<4xf64>
+// CHECK:                 %[[VAL_68:.*]] = memref.load %[[VAL_45]]{{\[}}%[[VAL_64]]] : memref<?xf64>
+// CHECK:                 %[[VAL_69:.*]] = arith.mulf %[[VAL_59]], %[[VAL_68]] : f64
+// CHECK:                 %[[VAL_70:.*]] = arith.addf %[[VAL_67]], %[[VAL_69]] : f64
+// CHECK:                 %[[VAL_71:.*]] = memref.load %[[VAL_31]]{{\[}}%[[VAL_66]]] : memref<4xi1>
+// CHECK:                 %[[VAL_72:.*]] = arith.cmpi eq, %[[VAL_71]], %[[VAL_10]] : i1
+// CHECK:                 %[[VAL_73:.*]] = scf.if %[[VAL_72]] -> (index) {
+// CHECK:                   memref.store %[[VAL_9]], %[[VAL_31]]{{\[}}%[[VAL_66]]] : memref<4xi1>
+// CHECK:                   memref.store %[[VAL_66]], %[[VAL_32]]{{\[}}%[[VAL_65]]] : memref<4xindex>
+// CHECK:                   %[[VAL_74:.*]] = arith.addi %[[VAL_65]], %[[VAL_11]] : index
+// CHECK:                   scf.yield %[[VAL_74]] : index
 // CHECK:                 } else {
-// CHECK:                   scf.yield %[[VAL_53]] : index
+// CHECK:                   scf.yield %[[VAL_65]] : index
 // CHECK:                 }
-// CHECK:                 memref.store %[[VAL_58]], %[[VAL_30]]{{\[}}%[[VAL_54]]] : memref<4xf64>
-// CHECK:                 scf.yield %[[VAL_63:.*]] : index
+// CHECK:                 memref.store %[[VAL_70]], %[[VAL_30]]{{\[}}%[[VAL_66]]] : memref<4xf64>
+// CHECK:                 scf.yield %[[VAL_73]] : index
 // CHECK:               } {"Emitted from" = "linalg.generic"}
-// CHECK:               scf.yield %[[VAL_64:.*]] : index
+// CHECK:               scf.yield %[[VAL_63]] : index
 // CHECK:             } {"Emitted from" = "linalg.generic"}
-// CHECK:             sparse_tensor.sort  hybrid_quick_sort %[[VAL_65:.*]], %[[VAL_33]]
-// CHECK:             %[[VAL_66:.*]]:4 = scf.for %[[VAL_67:.*]] = %[[VAL_10]] to %[[VAL_65]] step %[[VAL_11]] iter_args(%[[VAL_68:.*]] = %[[VAL_36]], %[[VAL_69:.*]] = %[[VAL_37]], %[[VAL_70:.*]] = %[[VAL_38]], %[[VAL_71:.*]] = %[[VAL_39]]) -> (memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
-// CHECK:               %[[VAL_72:.*]] = memref.load %[[VAL_32]]{{\[}}%[[VAL_67]]] : memref<4xindex>
-// CHECK:               %[[VAL_73:.*]] = memref.load %[[VAL_30]]{{\[}}%[[VAL_72]]] : memref<4xf64>
-// CHECK:               %[[VAL_74:.*]]:4 = func.call @_insert_dense_compressed_4_4_f64_0_0(%[[VAL_68]], %[[VAL_69]], %[[VAL_70]], %[[VAL_71]], %[[VAL_35]], %[[VAL_72]], %[[VAL_73]]) : (memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifie
-// CHECK:               memref.store %[[VAL_9]], %[[VAL_30]]{{\[}}%[[VAL_72]]] : memref<4xf64>
-// CHECK:               memref.store %[[VAL_12]], %[[VAL_31]]{{\[}}%[[VAL_72]]] : memref<4xi1>
-// CHECK:               scf.yield %[[VAL_74]]#0, %[[VAL_74]]#1, %[[VAL_74]]#2, %[[VAL_74]]#3 : memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
+// CHECK:             sparse_tensor.sort  hybrid_quick_sort %[[VAL_55]], %[[VAL_33]]
+// CHECK:             %[[VAL_75:.*]]:4 = scf.for %[[VAL_76:.*]] = %[[VAL_12]] to %[[VAL_55]] step %[[VAL_11]] iter_args(%[[VAL_77:.*]] = %[[VAL_48]], %[[VAL_78:.*]] = %[[VAL_49]], %[[VAL_79:.*]] = %[[VAL_50]], %[[VAL_80:.*]] = %[[VAL_51]]) -> (memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
+// CHECK:               %[[VAL_81:.*]] = memref.load %[[VAL_32]]{{\[}}%[[VAL_76]]] : memref<4xindex>
+// CHECK:               %[[VAL_82:.*]] = memref.load %[[VAL_30]]{{\[}}%[[VAL_81]]] : memref<4xf64>
+// CHECK:               %[[VAL_83:.*]]:4 = func.call @_insert_dense_compressed_4_4_f64_0_0(%[[VAL_77]], %[[VAL_78]], %[[VAL_79]], %[[VAL_80]], %[[VAL_47]], %[[VAL_81]], %[[VAL_82]]) : (memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
+// CHECK:               memref.store %[[VAL_8]], %[[VAL_30]]{{\[}}%[[VAL_81]]] : memref<4xf64>
+// CHECK:               memref.store %[[VAL_10]], %[[VAL_31]]{{\[}}%[[VAL_81]]] : memref<4xi1>
+// CHECK:               scf.yield %[[VAL_83]]#0, %[[VAL_83]]#1, %[[VAL_83]]#2, %[[VAL_83]]#3 : memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
 // CHECK:             }
-// CHECK:             scf.yield %[[VAL_75:.*]]#0, %[[VAL_75]]#1, %[[VAL_75]]#2, %[[VAL_75]]#3 : memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
+// CHECK:             scf.yield %[[VAL_84:.*]]#0, %[[VAL_84]]#1, %[[VAL_84]]#2, %[[VAL_84]]#3 : memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
 // CHECK:           } {"Emitted from" = "linalg.generic"}
 // CHECK:           memref.dealloc %[[VAL_30]] : memref<4xf64>
 // CHECK:           memref.dealloc %[[VAL_31]] : memref<4xi1>
 // CHECK:           memref.dealloc %[[VAL_32]] : memref<4xindex>
-// CHECK:           %[[VAL_76:.*]] = sparse_tensor.storage_specifier.get %[[VAL_77:.*]]#3  pos_mem_sz at 1 : !sparse_tensor.storage_specifier
-// CHECK:           %[[VAL_78:.*]] = memref.load %[[VAL_77]]#0{{\[}}%[[VAL_10]]] : memref<?xindex>
-// CHECK:           %[[VAL_79:.*]] = scf.for %[[VAL_80:.*]] = %[[VAL_11]] to %[[VAL_76]] step %[[VAL_11]] iter_args(%[[VAL_81:.*]] = %[[VAL_78]]) -> (index) {
-// CHECK:             %[[VAL_82:.*]] = memref.load %[[VAL_77]]#0{{\[}}%[[VAL_80]]] : memref<?xindex>
-// CHECK:             %[[VAL_83:.*]] = arith.cmpi eq, %[[VAL_82]], %[[VAL_10]] : index
-// CHECK:             %[[VAL_84:.*]] = arith.select %[[VAL_83]], %[[VAL_81]], %[[VAL_82]] : index
-// CHECK:             scf.if %[[VAL_83]] {
-// CHECK:               memref.store %[[VAL_81]], %[[VAL_77]]#0{{\[}}%[[VAL_80]]] : memref<?xindex>
+// CHECK:           %[[VAL_85:.*]] = sparse_tensor.storage_specifier.get %[[VAL_86:.*]]#3  pos_mem_sz at 1 : !sparse_tensor.storage_specifier
+// CHECK:           %[[VAL_87:.*]] = memref.load %[[VAL_86]]#0{{\[}}%[[VAL_12]]] : memref<?xindex>
+// CHECK:           %[[VAL_88:.*]] = scf.for %[[VAL_89:.*]] = %[[VAL_11]] to %[[VAL_85]] step %[[VAL_11]] iter_args(%[[VAL_90:.*]] = %[[VAL_87]]) -> (index) {
+// CHECK:             %[[VAL_91:.*]] = memref.load %[[VAL_86]]#0{{\[}}%[[VAL_89]]] : memref<?xindex>
+// CHECK:             %[[VAL_92:.*]] = arith.cmpi eq, %[[VAL_91]], %[[VAL_12]] : index
+// CHECK:             %[[VAL_93:.*]] = arith.select %[[VAL_92]], %[[VAL_90]], %[[VAL_91]] : index
+// CHECK:             scf.if %[[VAL_92]] {
+// CHECK:               memref.store %[[VAL_90]], %[[VAL_86]]#0{{\[}}%[[VAL_89]]] : memref<?xindex>
 // CHECK:             }
-// CHECK:             scf.yield %[[VAL_84]] : index
+// CHECK:             scf.yield %[[VAL_93]] : index
 // CHECK:           }
-// CHECK:           return %[[VAL_77]]#0, %[[VAL_77]]#1, %[[VAL_77]]#2, %[[VAL_77]]#3 : memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
+// CHECK:           return %[[VAL_86]]#0, %[[VAL_86]]#1, %[[VAL_86]]#2, %[[VAL_86]]#3 : memref<?xindex>, memref<?xindex>, memref<?xf64>, !sparse_tensor.storage_specifier
+// CHECK:         }
 func.func @matmul(%A: tensor<4x8xf64, #CSR>,
                   %B: tensor<8x4xf64, #CSR>) -> tensor<4x4xf64, #CSR> {
   %C = tensor.empty() : tensor<4x4xf64, #CSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
index 37d8a42a2990..f4ae33a42d06 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
@@ -79,9 +79,9 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 8 )
     // CHECK-NEXT: lvl = ( 3, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12,
-    // CHECK-NEXT: crd[1] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12,  )
+    // CHECK-NEXT: crd[1] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5,  )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,  )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %A1 : tensor<?x?xf64, #CSR>
@@ -94,8 +94,8 @@ module {
     // CHECK-NEXT: dim = ( 3, 8 )
     // CHECK-NEXT: lvl = ( 3, 8 )
     // CHECK-NEXT: pos[1] : ( 0, 4, 4, 8, 8, 12,
-    // CHECK-NEXT: crd[1] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
+    // CHECK-NEXT: crd[1] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5,  )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,  )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %A2 : tensor<?x?xf64, #CSR_hi>
@@ -107,8 +107,8 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 8 )
     // CHECK-NEXT: lvl = ( 3, 2, 4 )
-    // CHECK-NEXT: crd[2] : ( 2, 3, 1, 3, 1, 2, 0, 3, 0, 2, 0, 1,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
+    // CHECK-NEXT: crd[2] : ( 2, 3, 1, 3, 1, 2, 0, 3, 0, 2, 0, 1,  )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,  )
     // CHECK-NEXT: ----
     // CHECK-NEXT: ---- Sparse Tensor ----
     //
@@ -120,8 +120,8 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 8 )
     // CHECK-NEXT: lvl = ( 3, 1, 8 )
-    // CHECK-NEXT: crd[2] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
+    // CHECK-NEXT: crd[2] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5,  )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,  )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %A4 : tensor<?x?xf64, #NV_58>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
index bcd71f7bd674..7fc37eade720 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
@@ -98,9 +98,9 @@ module {
     // CHECK-NEXT: nse = 0
     // CHECK-NEXT: dim = ( 10 )
     // CHECK-NEXT: lvl = ( 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 0,
-    // CHECK-NEXT: crd[0] : (
-    // CHECK-NEXT: values : (
+    // CHECK-NEXT: pos[0] : ( 0, 0,  )
+    // CHECK-NEXT: crd[0] : (  )
+    // CHECK-NEXT: values : (  )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
@@ -108,26 +108,26 @@ module {
     // CHECK-NEXT: dim = ( 10 )
     // CHECK-NEXT: lvl = ( 10 )
     // CHECK-NEXT: pos[0] : ( 0, 0,
-    // CHECK-NEXT: crd[0] : (
-    // CHECK-NEXT: values : (
+    // CHECK-NEXT: crd[0] : (  )
+    // CHECK-NEXT: values : (  )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 0
     // CHECK-NEXT: dim = ( 10 )
     // CHECK-NEXT: lvl = ( 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 0,
-    // CHECK-NEXT: crd[0] : (
-    // CHECK-NEXT: values : (
+    // CHECK-NEXT: pos[0] : ( 0, 0,  )
+    // CHECK-NEXT: crd[0] : (  )
+    // CHECK-NEXT: values : (  )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 10
     // CHECK-NEXT: dim = ( 10 )
     // CHECK-NEXT: lvl = ( 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 10,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
-    // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    // CHECK-NEXT: pos[0] : ( 0, 10,  )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,  )
+    // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<10xf32, #SV>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
index 7758ca77dce9..b664b7f99944 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
@@ -147,7 +147,7 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 5, 0, 0,
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 5, 0, 0,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %XO : tensor<4x8xi32, #AllDense>
 
@@ -155,7 +155,7 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 8, 4 )
-    // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0,
+    // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %XT : tensor<4x8xi32, #AllDenseT>
 
@@ -176,9 +176,9 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 2, 2, 5,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,
+    // CHECK-NEXT: pos[1] : ( 0, 2, 2, 2, 5,  )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5,  )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %a : tensor<4x8xi32, #CSR>
 
@@ -186,11 +186,11 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,
-    // CHECK-NEXT: crd[0] : ( 0, 3,
-    // CHECK-NEXT: pos[1] : ( 0, 2, 5,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,
+    // CHECK-NEXT: pos[0] : ( 0, 2,  )
+    // CHECK-NEXT: crd[0] : ( 0, 3,  )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 5,  )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5,  )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %b : tensor<4x8xi32, #DCSR>
 
@@ -198,9 +198,9 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 8, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 1, 3, 4, 4, 5, 5, 5,
-    // CHECK-NEXT: crd[1] : ( 0, 0, 3, 3, 3,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,
+    // CHECK-NEXT: pos[1] : ( 0, 1, 1, 3, 4, 4, 5, 5, 5,  )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 3, 3, 3,  )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %c : tensor<4x8xi32, #CSC>
 
@@ -208,11 +208,11 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 8, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3, 5,
-    // CHECK-NEXT: pos[1] : ( 0, 1, 3, 4, 5,
-    // CHECK-NEXT: crd[1] : ( 0, 0, 3, 3, 3,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,
+    // CHECK-NEXT: pos[0] : ( 0, 4,  )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3, 5,  )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 3, 4, 5,  )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 3, 3, 3,  )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %d : tensor<4x8xi32, #DCSC>
 
@@ -220,11 +220,11 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 2, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,
-    // CHECK-NEXT: crd[0] : ( 0, 1,
-    // CHECK-NEXT: pos[1] : ( 0, 1, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 0, 1,
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,
+    // CHECK-NEXT: pos[0] : ( 0, 2,  )
+    // CHECK-NEXT: crd[0] : ( 0, 1,  )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 3,  )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 1,  )
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %e : tensor<4x8xi32, #BSR>
 
@@ -232,11 +232,11 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 4, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,
-    // CHECK-NEXT: crd[0] : ( 0, 1,
-    // CHECK-NEXT: pos[1] : ( 0, 1, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 0, 1,
-    // CHECK-NEXT: values : ( 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 5, 0, 0, 0, 0,
+    // CHECK-NEXT: pos[0] : ( 0, 2,  )
+    // CHECK-NEXT: crd[0] : ( 0, 1,  )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 3,  )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 1,  )
+    // CHECK-NEXT: values : ( 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 5, 0, 0, 0, 0,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %f : tensor<4x8xi32, #BSRC>
 
@@ -244,11 +244,11 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 2, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,
-    // CHECK-NEXT: crd[0] : ( 0, 1,
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 1, 1,
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,
+    // CHECK-NEXT: pos[0] : ( 0, 2,  )
+    // CHECK-NEXT: crd[0] : ( 0, 1,  )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3,  )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 1,  )
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %g : tensor<4x8xi32, #BSC>
 
@@ -256,11 +256,11 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 4, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,
-    // CHECK-NEXT: crd[0] : ( 0, 1,
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 1, 1,
-    // CHECK-NEXT: values : ( 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 5, 0, 0, 0, 0,
+    // CHECK-NEXT: pos[0] : ( 0, 2,  )
+    // CHECK-NEXT: crd[0] : ( 0, 1,  )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3,  )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 1,  )
+    // CHECK-NEXT: values : ( 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 5, 0, 0, 0, 0,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %h : tensor<4x8xi32, #BSCC>
 
@@ -268,9 +268,9 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 2, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 0, 1,
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,
+    // CHECK-NEXT: pos[1] : ( 0, 1, 3,  )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 1,  )
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %i : tensor<4x8xi32, #BSR0>
 
@@ -278,9 +278,9 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 2, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 1, 1,
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3,  )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 1,  )
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %j : tensor<4x8xi32, #BSC0>
 
@@ -288,9 +288,9 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 5,
-    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 2, 3, 2, 3, 3, 3, 5,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,
+    // CHECK-NEXT: pos[0] : ( 0, 5,  )
+    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 2, 3, 2, 3, 3, 3, 5,  )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %AoS : tensor<4x8xi32, #COOAoS>
 
@@ -298,10 +298,10 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 5,
-    // CHECK-NEXT: crd[0] : ( 0, 0, 3, 3, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,
+    // CHECK-NEXT: pos[0] : ( 0, 5,  )
+    // CHECK-NEXT: crd[0] : ( 0, 0, 3, 3, 3,  )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5,  )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %SoA : tensor<4x8xi32, #COOSoA>
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir
index da78452d94fd..9413119509c6 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir
@@ -68,9 +68,9 @@ module {
     // CHECK-NEXT: nse = 20
     // CHECK-NEXT: dim = ( 8, 8 )
     // CHECK-NEXT: lvl = ( 8, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 5, 5, 6, 7, 8, 12, 16, 20,
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 6, 7, 2, 3, 4, 1, 2, 6, 7, 1, 2, 6, 7, 1, 2, 6, 7,
-    // CHECK-NEXT: values : ( 1, 39, 52, 45, 51, 16, 25, 36, 117, 158, 135, 144, 156, 318, 301, 324, 208, 430, 405, 436,
+    // CHECK-NEXT: pos[1] : ( 0, 5, 5, 6, 7, 8, 12, 16, 20,  )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 6, 7, 2, 3, 4, 1, 2, 6, 7, 1, 2, 6, 7, 1, 2, 6, 7,  )
+    // CHECK-NEXT: values : ( 1, 39, 52, 45, 51, 16, 25, 36, 117, 158, 135, 144, 156, 318, 301, 324, 208, 430, 405, 436,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %Ccsr : tensor<8x8xf32, #CSR>
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir
index 3d17b719732f..3b3d074f7e2a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir
@@ -117,9 +117,9 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 5, 5 )
     // CHECK-NEXT: lvl = ( 5, 5 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5, 7, 9,
-    // CHECK-NEXT: crd[1] : ( 0, 3, 1, 4, 2, 0, 3, 1, 4,
-    // CHECK-NEXT: values : ( 11, 41.4, 42, 102.5, 93, 44.1, 164, 105.2, 255,
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5, 7, 9,  )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 1, 4, 2, 0, 3, 1, 4,  )
+    // CHECK-NEXT: values : ( 11, 41.4, 42, 102.5, 93, 44.1, 164, 105.2, 255,  )
     // CHECK-NEXT: ----
     sparse_tensor.print %0 : tensor<?x?xf32, #CSR>
 
@@ -145,9 +145,9 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 8, 8 )
     // CHECK-NEXT: lvl = ( 8, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 3, 4, 4, 4, 4, 5,
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 4, 7,
-    // CHECK-NEXT: values : ( 17, 18, 19, 20, 21,
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 3, 4, 4, 4, 4, 5,  )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 4, 7,  )
+    // CHECK-NEXT: values : ( 17, 18, 19, 20, 21,  )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<?x?xf32, #CSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir
index 68bb32891f34..18f59f59a9f0 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir
@@ -170,18 +170,18 @@ module {
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 4, 6 )
     // CHECK-NEXT: lvl = ( 4, 6 )
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7, 8,
-    // CHECK-NEXT: crd[1] : ( 0, 1, 4, 1, 5, 2, 3, 2,
-    // CHECK-NEXT: values : ( 5, 10, 24, 19, 53, 42, 55, 56,
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7, 8,  )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 4, 1, 5, 2, 3, 2,  )
+    // CHECK-NEXT: values : ( 5, 10, 24, 19, 53, 42, 55, 56,  )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 4, 6 )
     // CHECK-NEXT: lvl = ( 2, 3, 2, 2 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 1,
-    // CHECK-NEXT: values : ( 5, 10, 8, 19, 24, 24, 40, 53, 42, 55, 56, 64,
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3,  )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 1,  )
+    // CHECK-NEXT: values : ( 5, 10, 8, 19, 24, 24, 40, 53, 42, 55, 56, 64,  )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?x?xf32, #CSR>
-- 
GitLab


From 7927bcdb8a32646f78c01535050ada6ddc23f4f5 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 7 May 2024 18:26:32 +0200
Subject: [PATCH 0068/1206] AMDGPU: Do not bitcast atomicrmw in IR (#90045)

This is the first step to eliminating shouldCastAtomicRMWIInIR. This and
the other atomic expand casting hooks should be removed. This adds
duplicate legalization machinery and interfaces. This is already what
codegen is supposed to do, and already does for the promotion case.

In the case of atomicrmw xchg, there seems to be some benefit to having
the bitcasts moved outside of the cmpxchg loop on targets with separate
int and FP registers, which we should be able to deal with by directly
checking for the legality of the underlying operation.

The casting path was also losing metadata when it recreated the
instruction.
---
 llvm/lib/CodeGen/AtomicExpandPass.cpp         |  5 +--
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp |  7 ++++
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h   |  4 +++
 .../AMDGPU/expand-atomic-f32-agent.ll         | 32 ++++++++++-------
 .../AMDGPU/expand-atomic-f32-system.ll        | 32 ++++++++++-------
 .../AMDGPU/expand-atomic-f64-agent.ll         | 32 ++++++++++-------
 .../AMDGPU/expand-atomic-f64-system.ll        | 32 ++++++++++-------
 .../AMDGPU/expand-atomic-i16-system.ll        | 36 +++++++++----------
 .../AtomicExpand/AMDGPU/expand-atomic-i16.ll  | 36 +++++++++----------
 9 files changed, 126 insertions(+), 90 deletions(-)

diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index f3b8097396e2..ee44e9353d04 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -909,9 +909,10 @@ void AtomicExpandImpl::expandPartwordAtomicRMW(
   Value *ValOperand_Shifted = nullptr;
   if (Op == AtomicRMWInst::Xchg || Op == AtomicRMWInst::Add ||
       Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Nand) {
+    Value *ValOp = Builder.CreateBitCast(AI->getValOperand(), PMV.IntValueType);
     ValOperand_Shifted =
-        Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
-                          PMV.ShiftAmt, "ValOperand_Shifted");
+        Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
+                          "ValOperand_Shifted");
   }
 
   auto PerformPartwordOp = [&](IRBuilderBase &Builder, Value *Loaded) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index b95acdb3550b..5ca7f8ef5345 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5988,6 +5988,13 @@ AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
   case AtomicRMWInst::FMax:
   case AtomicRMWInst::FMin:
     return AtomicExpansionKind::CmpXChg;
+  case AtomicRMWInst::Xchg: {
+    const DataLayout &DL = RMW->getFunction()->getParent()->getDataLayout();
+    unsigned ValSize = DL.getTypeSizeInBits(RMW->getType());
+    if (ValSize == 32 || ValSize == 64)
+      return AtomicExpansionKind::None;
+    return AtomicExpansionKind::CmpXChg;
+  }
   default: {
     if (auto *IntTy = dyn_cast<IntegerType>(RMW->getType())) {
       unsigned Size = IntTy->getBitWidth();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 269c414521db..16c4f53d6344 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -236,6 +236,10 @@ public:
     return AtomicExpansionKind::None;
   }
 
+  AtomicExpansionKind shouldCastAtomicRMWIInIR(AtomicRMWInst *) const override {
+    return AtomicExpansionKind::None;
+  }
+
   static CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg);
   static CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg);
 
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f32-agent.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f32-agent.ll
index 70389bbb26d3..31da626e01f0 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f32-agent.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f32-agent.ll
@@ -16,9 +16,7 @@
 define float @test_atomicrmw_xchg_f32_global_agent(ptr addrspace(1) %ptr, float %value) {
 ; COMMON-LABEL: define float @test_atomicrmw_xchg_f32_global_agent(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0:[0-9]+]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast float [[VALUE]] to i32
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i32 [[TMP1]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i32 [[TMP2]] to float
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4
 ; COMMON-NEXT:    ret float [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, float %value syncscope("agent") seq_cst
@@ -29,9 +27,7 @@ define float @test_atomicrmw_xchg_f32_global_agent(ptr addrspace(1) %ptr, float
 define float @test_atomicrmw_xchg_f32_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %value) {
 ; COMMON-LABEL: define float @test_atomicrmw_xchg_f32_global_agent__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast float [[VALUE]] to i32
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i32 [[TMP1]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i32 [[TMP2]] to float
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
 ; COMMON-NEXT:    ret float [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, float %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -42,9 +38,7 @@ define float @test_atomicrmw_xchg_f32_global_agent__amdgpu_no_fine_grained_memor
 define float @test_atomicrmw_xchg_f32_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, float %value) {
 ; COMMON-LABEL: define float @test_atomicrmw_xchg_f32_global_agent__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast float [[VALUE]] to i32
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i32 [[TMP1]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i32 [[TMP2]] to float
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret float [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, float %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
@@ -55,9 +49,7 @@ define float @test_atomicrmw_xchg_f32_global_agent__amdgpu_no_remote_memory(ptr
 define float @test_atomicrmw_xchg_f32_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, float %value) {
 ; COMMON-LABEL: define float @test_atomicrmw_xchg_f32_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast float [[VALUE]] to i32
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i32 [[TMP1]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i32 [[TMP2]] to float
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret float [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, float %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -268,7 +260,7 @@ define float @test_atomicrmw_fadd_f32_global_agent__amdgpu_no_fine_grained_memor
 ;
 ; GFX940-LABEL: define float @test_atomicrmw_fadd_f32_global_agent__amdgpu_no_fine_grained_memory(
 ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX940-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX940-NEXT:    ret float [[RES]]
 ;
 ; GFX10-LABEL: define float @test_atomicrmw_fadd_f32_global_agent__amdgpu_no_fine_grained_memory(
@@ -3713,5 +3705,19 @@ attributes #1 = { "denormal-fp-mode-f32"="dynamic,dynamic" }
 
 !0 = !{}
 ;.
+; GFX803: [[META0]] = !{}
+;.
+; GFX906: [[META0]] = !{}
+;.
+; GFX908: [[META0]] = !{}
+;.
+; GFX90A: [[META0]] = !{}
+;.
 ; GFX940: [[META0]] = !{}
 ;.
+; GFX10: [[META0]] = !{}
+;.
+; GFX11: [[META0]] = !{}
+;.
+; GFX12: [[META0]] = !{}
+;.
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f32-system.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f32-system.ll
index a4f81efa8961..35c546322e63 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f32-system.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f32-system.ll
@@ -16,9 +16,7 @@
 define float @test_atomicrmw_xchg_f32_global_system(ptr addrspace(1) %ptr, float %value) {
 ; COMMON-LABEL: define float @test_atomicrmw_xchg_f32_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0:[0-9]+]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast float [[VALUE]] to i32
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i32 [[TMP1]] seq_cst, align 4
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i32 [[TMP2]] to float
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], float [[VALUE]] seq_cst, align 4
 ; COMMON-NEXT:    ret float [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, float %value seq_cst
@@ -29,9 +27,7 @@ define float @test_atomicrmw_xchg_f32_global_system(ptr addrspace(1) %ptr, float
 define float @test_atomicrmw_xchg_f32_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, float %value) {
 ; COMMON-LABEL: define float @test_atomicrmw_xchg_f32_global_system__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast float [[VALUE]] to i32
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i32 [[TMP1]] seq_cst, align 4
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i32 [[TMP2]] to float
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], float [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
 ; COMMON-NEXT:    ret float [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, float %value seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -42,9 +38,7 @@ define float @test_atomicrmw_xchg_f32_global_system__amdgpu_no_fine_grained_memo
 define float @test_atomicrmw_xchg_f32_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, float %value) {
 ; COMMON-LABEL: define float @test_atomicrmw_xchg_f32_global_system__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast float [[VALUE]] to i32
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i32 [[TMP1]] seq_cst, align 4
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i32 [[TMP2]] to float
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], float [[VALUE]] seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret float [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, float %value seq_cst, !amdgpu.no.remote.memory !0
@@ -55,9 +49,7 @@ define float @test_atomicrmw_xchg_f32_global_system__amdgpu_no_remote_memory(ptr
 define float @test_atomicrmw_xchg_f32_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, float %value) {
 ; COMMON-LABEL: define float @test_atomicrmw_xchg_f32_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast float [[VALUE]] to i32
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i32 [[TMP1]] seq_cst, align 4
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i32 [[TMP2]] to float
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], float [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret float [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, float %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -268,7 +260,7 @@ define float @test_atomicrmw_fadd_f32_global_system__amdgpu_no_fine_grained_memo
 ;
 ; GFX940-LABEL: define float @test_atomicrmw_fadd_f32_global_system__amdgpu_no_fine_grained_memory(
 ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX940-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], float [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], float [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX940-NEXT:    ret float [[RES]]
 ;
 ; GFX10-LABEL: define float @test_atomicrmw_fadd_f32_global_system__amdgpu_no_fine_grained_memory(
@@ -3713,5 +3705,19 @@ attributes #1 = { "denormal-fp-mode-f32"="dynamic,dynamic" }
 
 !0 = !{}
 ;.
+; GFX803: [[META0]] = !{}
+;.
+; GFX906: [[META0]] = !{}
+;.
+; GFX908: [[META0]] = !{}
+;.
+; GFX90A: [[META0]] = !{}
+;.
 ; GFX940: [[META0]] = !{}
 ;.
+; GFX10: [[META0]] = !{}
+;.
+; GFX11: [[META0]] = !{}
+;.
+; GFX12: [[META0]] = !{}
+;.
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-agent.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-agent.ll
index 8810d1552290..a5830bd8d7c3 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-agent.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-agent.ll
@@ -16,9 +16,7 @@
 define double @test_atomicrmw_xchg_f64_global_agent(ptr addrspace(1) %ptr, double %value) {
 ; COMMON-LABEL: define double @test_atomicrmw_xchg_f64_global_agent(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0:[0-9]+]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast double [[VALUE]] to i64
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[TMP1]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i64 [[TMP2]] to double
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8
 ; COMMON-NEXT:    ret double [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst
@@ -29,9 +27,7 @@ define double @test_atomicrmw_xchg_f64_global_agent(ptr addrspace(1) %ptr, doubl
 define double @test_atomicrmw_xchg_f64_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
 ; COMMON-LABEL: define double @test_atomicrmw_xchg_f64_global_agent__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast double [[VALUE]] to i64
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[TMP1]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i64 [[TMP2]] to double
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
 ; COMMON-NEXT:    ret double [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -42,9 +38,7 @@ define double @test_atomicrmw_xchg_f64_global_agent__amdgpu_no_fine_grained_memo
 define double @test_atomicrmw_xchg_f64_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
 ; COMMON-LABEL: define double @test_atomicrmw_xchg_f64_global_agent__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast double [[VALUE]] to i64
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[TMP1]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i64 [[TMP2]] to double
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret double [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
@@ -55,9 +49,7 @@ define double @test_atomicrmw_xchg_f64_global_agent__amdgpu_no_remote_memory(ptr
 define double @test_atomicrmw_xchg_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
 ; COMMON-LABEL: define double @test_atomicrmw_xchg_f64_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast double [[VALUE]] to i64
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[TMP1]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i64 [[TMP2]] to double
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret double [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -268,7 +260,7 @@ define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_no_fine_grained_memo
 ;
 ; GFX940-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_no_fine_grained_memory(
 ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX940-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX940-NEXT:    ret double [[RES]]
 ;
 ; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_agent__amdgpu_no_fine_grained_memory(
@@ -1681,5 +1673,19 @@ attributes #1 = { "denormal-fp-mode"="dynamic,dynamic" }
 
 !0 = !{}
 ;.
+; GFX803: [[META0]] = !{}
+;.
+; GFX906: [[META0]] = !{}
+;.
+; GFX908: [[META0]] = !{}
+;.
+; GFX90A: [[META0]] = !{}
+;.
 ; GFX940: [[META0]] = !{}
 ;.
+; GFX10: [[META0]] = !{}
+;.
+; GFX11: [[META0]] = !{}
+;.
+; GFX12: [[META0]] = !{}
+;.
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-system.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-system.ll
index bd126322836b..4489b639b678 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-system.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-system.ll
@@ -16,9 +16,7 @@
 define double @test_atomicrmw_xchg_f64_global_system(ptr addrspace(1) %ptr, double %value) {
 ; COMMON-LABEL: define double @test_atomicrmw_xchg_f64_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0:[0-9]+]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast double [[VALUE]] to i64
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[TMP1]] seq_cst, align 8
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i64 [[TMP2]] to double
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8
 ; COMMON-NEXT:    ret double [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, double %value seq_cst
@@ -29,9 +27,7 @@ define double @test_atomicrmw_xchg_f64_global_system(ptr addrspace(1) %ptr, doub
 define double @test_atomicrmw_xchg_f64_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, double %value) {
 ; COMMON-LABEL: define double @test_atomicrmw_xchg_f64_global_system__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast double [[VALUE]] to i64
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[TMP1]] seq_cst, align 8
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i64 [[TMP2]] to double
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
 ; COMMON-NEXT:    ret double [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -42,9 +38,7 @@ define double @test_atomicrmw_xchg_f64_global_system__amdgpu_no_fine_grained_mem
 define double @test_atomicrmw_xchg_f64_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
 ; COMMON-LABEL: define double @test_atomicrmw_xchg_f64_global_system__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast double [[VALUE]] to i64
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[TMP1]] seq_cst, align 8
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i64 [[TMP2]] to double
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret double [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.remote.memory !0
@@ -55,9 +49,7 @@ define double @test_atomicrmw_xchg_f64_global_system__amdgpu_no_remote_memory(pt
 define double @test_atomicrmw_xchg_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, double %value) {
 ; COMMON-LABEL: define double @test_atomicrmw_xchg_f64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = bitcast double [[VALUE]] to i64
-; COMMON-NEXT:    [[TMP2:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], i64 [[TMP1]] seq_cst, align 8
-; COMMON-NEXT:    [[RES:%.*]] = bitcast i64 [[TMP2]] to double
+; COMMON-NEXT:    [[RES:%.*]] = atomicrmw xchg ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret double [[RES]]
 ;
   %res = atomicrmw xchg ptr addrspace(1) %ptr, double %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -268,7 +260,7 @@ define double @test_atomicrmw_fadd_f64_global_system__amdgpu_no_fine_grained_mem
 ;
 ; GFX940-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_no_fine_grained_memory(
 ; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {
-; GFX940-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR]], double [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; GFX940-NEXT:    ret double [[RES]]
 ;
 ; GFX10-LABEL: define double @test_atomicrmw_fadd_f64_global_system__amdgpu_no_fine_grained_memory(
@@ -1681,5 +1673,19 @@ attributes #1 = { "denormal-fp-mode"="dynamic,dynamic" }
 
 !0 = !{}
 ;.
+; GFX803: [[META0]] = !{}
+;.
+; GFX906: [[META0]] = !{}
+;.
+; GFX908: [[META0]] = !{}
+;.
+; GFX90A: [[META0]] = !{}
+;.
 ; GFX940: [[META0]] = !{}
 ;.
+; GFX10: [[META0]] = !{}
+;.
+; GFX11: [[META0]] = !{}
+;.
+; GFX12: [[META0]] = !{}
+;.
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16-system.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16-system.ll
index 78468b933ff5..050c0170270a 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16-system.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16-system.ll
@@ -697,15 +697,15 @@ define i16 @test_atomicrmw_dec_i16_flat_system_align4(ptr %ptr, i16 %value) {
 
 define half @test_atomicrmw_xchg_f16_global_system(ptr addrspace(1) %ptr, half %value) {
 ; CHECK-LABEL: @test_atomicrmw_xchg_f16_global_system(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[VALUE:%.*]] to i16
 ; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP2]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
 ; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
 ; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast half [[VALUE:%.*]] to i16
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -752,15 +752,15 @@ define half @test_atomicrmw_xchg_f16_global_system_align4(ptr addrspace(1) %ptr,
 
 define half @test_atomicrmw_xchg_f16_flat_system(ptr %ptr, half %value) {
 ; CHECK-LABEL: @test_atomicrmw_xchg_f16_flat_system(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[VALUE:%.*]] to i16
 ; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP2]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
 ; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
 ; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast half [[VALUE:%.*]] to i16
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -807,15 +807,15 @@ define half @test_atomicrmw_xchg_f16_flat_system_align4(ptr %ptr, half %value) {
 
 define bfloat @test_atomicrmw_xchg_bf16_flat_system(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: @test_atomicrmw_xchg_bf16_flat_system(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast bfloat [[VALUE:%.*]] to i16
 ; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP2]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
 ; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
 ; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast bfloat [[VALUE:%.*]] to i16
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
index 9d1e9a8fd8b8..ce8524c70af6 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
@@ -1016,15 +1016,15 @@ define i16 @test_atomicrmw_dec_i16_flat_agent_align4(ptr %ptr, i16 %value) {
 
 define half @test_atomicrmw_xchg_f16_global_agent(ptr addrspace(1) %ptr, half %value) {
 ; CHECK-LABEL: @test_atomicrmw_xchg_f16_global_agent(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[VALUE:%.*]] to i16
 ; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP2]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
 ; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
 ; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast half [[VALUE:%.*]] to i16
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1071,15 +1071,15 @@ define half @test_atomicrmw_xchg_f16_global_agent_align4(ptr addrspace(1) %ptr,
 
 define half @test_atomicrmw_xchg_f16_flat_agent(ptr %ptr, half %value) {
 ; CHECK-LABEL: @test_atomicrmw_xchg_f16_flat_agent(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[VALUE:%.*]] to i16
 ; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP2]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
 ; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
 ; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast half [[VALUE:%.*]] to i16
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
@@ -1126,15 +1126,15 @@ define half @test_atomicrmw_xchg_f16_flat_agent_align4(ptr %ptr, half %value) {
 
 define bfloat @test_atomicrmw_xchg_bf16_global_agent(ptr addrspace(1) %ptr, bfloat %value) {
 ; CHECK-LABEL: @test_atomicrmw_xchg_bf16_global_agent(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast bfloat [[VALUE:%.*]] to i16
 ; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP2]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
 ; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
 ; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast bfloat [[VALUE:%.*]] to i16
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
-- 
GitLab


From 63ceb9afc693209964efd4ac4844c9c0712c312d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 7 May 2024 17:21:21 +0100
Subject: [PATCH 0069/1206] [X86] sext-subreg.ll - regenerate checks

---
 llvm/test/CodeGen/X86/sext-subreg.ll | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/llvm/test/CodeGen/X86/sext-subreg.ll b/llvm/test/CodeGen/X86/sext-subreg.ll
index 3e54f24d13af..20451ff208cc 100644
--- a/llvm/test/CodeGen/X86/sext-subreg.ll
+++ b/llvm/test/CodeGen/X86/sext-subreg.ll
@@ -1,16 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 ; rdar://7529457
 
 define i64 @t(i64 %A, i64 %B, ptr %P, ptr%P2) nounwind {
 ; CHECK-LABEL: t:
-; CHECK: movslq %e{{.*}}, %rax
-; CHECK: movq %rax
-; CHECK: movl %eax
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq %rsi, %rdi
+; CHECK-NEXT:    movl %edi, (%rdx)
+; CHECK-NEXT:    movslq %edi, %rax
+; CHECK-NEXT:    movq %rax, (%rcx)
+; CHECK-NEXT:    movl %eax, (%rdx)
+; CHECK-NEXT:    retq
   %C = add i64 %A, %B
   %D = trunc i64 %C to i32
   store volatile i32 %D, ptr %P
   %E = shl i64 %C, 32
-  %F = ashr i64 %E, 32  
+  %F = ashr i64 %E, 32
   store volatile i64 %F, ptr%P2
   store volatile i32 %D, ptr %P
   ret i64 undef
-- 
GitLab


From 7198b8a39a062215aaf4ad8d2df23f7a10eaf6ae Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 7 May 2024 17:22:51 +0100
Subject: [PATCH 0070/1206] [X86] x86-64-extend-shift.ll - regenerate checks

---
 llvm/test/CodeGen/X86/x86-64-extend-shift.ll | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/llvm/test/CodeGen/X86/x86-64-extend-shift.ll b/llvm/test/CodeGen/X86/x86-64-extend-shift.ll
index 6ebaeee36697..b73da1625969 100644
--- a/llvm/test/CodeGen/X86/x86-64-extend-shift.ll
+++ b/llvm/test/CodeGen/X86/x86-64-extend-shift.ll
@@ -1,10 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
 ; Formerly there were two shifts.
 
 define i64 @baz(i32 %A) nounwind {
-; CHECK:  shlq  $49, %r
-        %tmp1 = shl i32 %A, 17
-        %tmp2 = zext i32 %tmp1 to i64
-        %tmp3 = shl i64 %tmp2, 32
-        ret i64 %tmp3
+; CHECK-LABEL: baz:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shlq $49, %rax
+; CHECK-NEXT:    retq
+  %tmp1 = shl i32 %A, 17
+  %tmp2 = zext i32 %tmp1 to i64
+  %tmp3 = shl i64 %tmp2, 32
+  ret i64 %tmp3
 }
-- 
GitLab


From 1a96179596099b8a3839050dbff02bfed94502e5 Mon Sep 17 00:00:00 2001
From: Tacet <advenam.tacet@trailofbits.com>
Date: Tue, 7 May 2024 18:35:25 +0200
Subject: [PATCH 0071/1206] [ASan][libc++] Turn on ASan annotations for short
 strings (#79536)

This pull request is the third iteration aiming to integrate short
string annotations. This commit includes:
- Enabling basic_string annotations for short strings.
- Setting a value of `__trivially_relocatable` in `std::basic_string` to
`false_type` when compiling with ASan (nothing changes when compiling
without ASan). Short string annotations make `std::basic_string` to not
be trivially relocatable, because memory has to be unpoisoned.
- Adding a `_LIBCPP_STRING_INTERNAL_MEMORY_ACCESS` modifier to two
functions.
- Creating a macro `_LIBCPP_ASAN_VOLATILE_WRAPPER` to prevent
problematic stack optimizations (the macro modifies code behavior only
when compiling with ASan).

Previously we had issues with compiler optimization, which we understand
thanks to @vitalybuka. This commit also addresses smaller changes in
short string, since previous upstream attempts.

Problematic optimization was loading two values in code similar to:
```
__is_long() ? __get_long_size() : __get_short_size();
```
We aim to resolve it with the volatile wrapper.

This commit is built on top of two previous attempts which descriptions
are below.

Additionally, in the meantime, annotations were updated (but it
shouldn't have any impact on anything):
- https://github.com/llvm/llvm-project/pull/79292

---

Previous PR: https://github.com/llvm/llvm-project/pull/79049
Reverted:
https://github.com/llvm/llvm-project/commit/a16f81f5e3313e88f96de35e5edfe8bee463d308

Previous description:

Originally merged here: https://github.com/llvm/llvm-project/pull/75882
Reverted here: https://github.com/llvm/llvm-project/pull/78627

Reverted due to failing buildbots. The problem was not caused by the
annotations code, but by code in the `UniqueFunctionBase` class and in
the `JSON.h` file. That code caused the program to write to memory that
was already being used by string objects, which resulted in an ASan
error.

Fixes are implemented in:
- https://github.com/llvm/llvm-project/pull/79065
- https://github.com/llvm/llvm-project/pull/79066

Problematic code from `UniqueFunctionBase` for example:
```cpp
    // In debug builds, we also scribble across the rest of the storage.
    memset(RHS.getInlineStorage(), 0xAD, InlineStorageSize);
```

---

Original description:

This commit turns on ASan annotations in `std::basic_string` for short
stings (SSO case).

Originally suggested here: https://reviews.llvm.org/D147680

String annotations added here:
https://github.com/llvm/llvm-project/pull/72677

Requires to pass CI without fails:
- https://github.com/llvm/llvm-project/pull/75845
- https://github.com/llvm/llvm-project/pull/75858

Annotating `std::basic_string` with default allocator is implemented in
https://github.com/llvm/llvm-project/pull/72677 but annotations for
short strings (SSO - Short String Optimization) are turned off there.
This commit turns them on. This also removes
`_LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED`, because we do not plan to
support turning on and off short string annotations.

Support in ASan API exists since
https://github.com/llvm/llvm-project/commit/dd1b7b797a116eed588fd752fbe61d34deeb24e4.
You can turn off annotations for a specific allocator based on changes
from
https://github.com/llvm/llvm-project/commit/2fa1bec7a20bb23f2e6620085adb257dafaa3be0.

This PR is a part of a series of patches extending AddressSanitizer C++
container overflow detection capabilities by adding annotations, similar
to those existing in `std::vector` and `std::deque` collections. These
enhancements empower ASan to effectively detect instances where the
instrumented program attempts to access memory within a collection's
internal allocation that remains unused. This includes cases where
access occurs before or after the stored elements in `std::deque`, or
between the `std::basic_string`'s size (including the null terminator)
and capacity bounds.

The introduction of these annotations was spurred by a real-world
software bug discovered by Trail of Bits, involving an out-of-bounds
memory access during the comparison of two strings using the
`std::equals` function. This function was taking iterators
(`iter1_begin`, `iter1_end`, `iter2_begin`) to perform the comparison,
using a custom comparison function. When the `iter1` object exceeded the
length of `iter2`, an out-of-bounds read could occur on the `iter2`
object. Container sanitization, upon enabling these annotations, would
effectively identify and flag this potential vulnerability.

If you have any questions, please email:

- advenam.tacet@trailofbits.com
- disconnect3d@trailofbits.com
---
 libcxx/include/string                         |  60 ++++--
 .../asan_deque_integration.pass.cpp           | 182 ++++++++++++++++++
 .../strings/basic.string/asan_short.pass.cpp  |  56 ++++++
 .../asan_vector_integration.pass.cpp          | 182 ++++++++++++++++++
 .../is_trivially_relocatable.compile.pass.cpp |   3 +-
 libcxx/test/support/asan_testing.h            |  29 +--
 6 files changed, 471 insertions(+), 41 deletions(-)
 create mode 100644 libcxx/test/libcxx/containers/strings/basic.string/asan_deque_integration.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/strings/basic.string/asan_short.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/strings/basic.string/asan_vector_integration.pass.cpp

diff --git a/libcxx/include/string b/libcxx/include/string
index 4e3dd278c12b..8f629d8bf13c 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -662,7 +662,6 @@ _LIBCPP_PUSH_MACROS
 #else
 #  define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
 #endif
-#define _LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED false
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -736,10 +735,44 @@ public:
   //
   // This string implementation doesn't contain any references into itself. It only contains a bit that says whether
   // it is in small or large string mode, so the entire structure is trivially relocatable if its members are.
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+  // When compiling with AddressSanitizer (ASan), basic_string cannot be trivially
+  // relocatable. Because the object's memory might be poisoned when its content
+  // is kept inside objects memory (short string optimization), instead of in allocated
+  // external memory. In such cases, the destructor is responsible for unpoisoning
+  // the memory to avoid triggering false positives.
+  // Therefore it's crucial to ensure the destructor is called
+  using __trivially_relocatable = false_type;
+#else
   using __trivially_relocatable = __conditional_t<
       __libcpp_is_trivially_relocatable<allocator_type>::value && __libcpp_is_trivially_relocatable<pointer>::value,
       basic_string,
       void>;
+#endif
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  pointer __asan_volatile_wrapper(pointer const &__ptr) const {
+    if (__libcpp_is_constant_evaluated())
+      return __ptr;
+
+    pointer volatile __copy_ptr = __ptr;
+
+    return const_cast<pointer &>(__copy_ptr);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  const_pointer __asan_volatile_wrapper(const_pointer const &__ptr) const {
+    if (__libcpp_is_constant_evaluated())
+      return __ptr;
+
+    const_pointer volatile __copy_ptr = __ptr;
+
+    return const_cast<const_pointer &>(__copy_ptr);
+  }
+#define _LIBCPP_ASAN_VOLATILE_WRAPPER(PTR) __asan_volatile_wrapper(PTR)
+#else
+#define _LIBCPP_ASAN_VOLATILE_WRAPPER(PTR) PTR
+#endif
 
   static_assert((!is_array<value_type>::value), "Character type of basic_string must not be an array");
   static_assert((is_standard_layout<value_type>::value), "Character type of basic_string must be standard-layout");
@@ -1886,16 +1919,16 @@ private:
     __r_.first().__l.__data_ = __p;
   }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __get_long_pointer() _NOEXCEPT {
-    return __r_.first().__l.__data_;
+    return _LIBCPP_ASAN_VOLATILE_WRAPPER(__r_.first().__l.__data_);
   }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_pointer __get_long_pointer() const _NOEXCEPT {
-    return __r_.first().__l.__data_;
+    return _LIBCPP_ASAN_VOLATILE_WRAPPER(__r_.first().__l.__data_);
   }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __get_short_pointer() _NOEXCEPT {
-    return pointer_traits<pointer>::pointer_to(__r_.first().__s.__data_[0]);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS pointer __get_short_pointer() _NOEXCEPT {
+    return _LIBCPP_ASAN_VOLATILE_WRAPPER(pointer_traits<pointer>::pointer_to(__r_.first().__s.__data_[0]));
   }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_pointer __get_short_pointer() const _NOEXCEPT {
-    return pointer_traits<const_pointer>::pointer_to(__r_.first().__s.__data_[0]);
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS const_pointer __get_short_pointer() const _NOEXCEPT {
+    return _LIBCPP_ASAN_VOLATILE_WRAPPER(pointer_traits<const_pointer>::pointer_to(__r_.first().__s.__data_[0]));
   }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __get_pointer() _NOEXCEPT {
     return __is_long() ? __get_long_pointer() : __get_short_pointer();
@@ -1914,22 +1947,17 @@ private:
 #endif
   }
 
-  // ASan: short string is poisoned if and only if this function returns true.
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __asan_short_string_is_annotated() const _NOEXCEPT {
-    return _LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED && !__libcpp_is_constant_evaluated();
-  }
-
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_new(size_type __current_size) const _NOEXCEPT {
     (void)__current_size;
 #if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
-    if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
+    if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + capacity() + 1, data() + __current_size + 1);
 #endif
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_delete() const _NOEXCEPT {
 #if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
-    if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
+    if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + size() + 1, data() + capacity() + 1);
 #endif
   }
@@ -1937,7 +1965,7 @@ private:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_increase(size_type __n) const _NOEXCEPT {
     (void)__n;
 #if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
-    if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
+    if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + size() + 1, data() + size() + 1 + __n);
 #endif
   }
@@ -1945,7 +1973,7 @@ private:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_shrink(size_type __old_size) const _NOEXCEPT {
     (void)__old_size;
 #if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
-    if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
+    if (!__libcpp_is_constant_evaluated())
       __annotate_contiguous_container(data() + __old_size + 1, data() + size() + 1);
 #endif
   }
diff --git a/libcxx/test/libcxx/containers/strings/basic.string/asan_deque_integration.pass.cpp b/libcxx/test/libcxx/containers/strings/basic.string/asan_deque_integration.pass.cpp
new file mode 100644
index 000000000000..1205190b3a6e
--- /dev/null
+++ b/libcxx/test/libcxx/containers/strings/basic.string/asan_deque_integration.pass.cpp
@@ -0,0 +1,182 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: asan
+// UNSUPPORTED: c++03
+
+#include <cassert>
+#include <string>
+#include <array>
+#include <deque>
+#include "test_macros.h"
+#include "asan_testing.h"
+#include "min_allocator.h"
+
+// This tests exists to check if strings work well with deque, as those
+// may be partialy annotated, we cannot simply call
+// is_double_ended_contiguous_container_asan_correct, as it assumes that
+// object memory inside is not annotated, so we check everything in a more careful way.
+
+template <typename D>
+void verify_inside(D const& d) {
+  for (size_t i = 0; i < d.size(); ++i) {
+    assert(is_string_asan_correct(d[i]));
+  }
+}
+
+template <typename S, size_t N>
+S get_s(char c) {
+  S s;
+  for (size_t i = 0; i < N; ++i)
+    s.push_back(c);
+
+  return s;
+}
+
+template <class C, class S>
+void test_string() {
+  size_t const N = sizeof(S) < 256 ? (4096 / sizeof(S)) : 16;
+
+  {
+    C d1a(1), d1b(N), d1c(N + 1), d1d(5 * N);
+    verify_inside(d1a);
+    verify_inside(d1b);
+    verify_inside(d1c);
+    verify_inside(d1d);
+  }
+  {
+    C d2;
+    for (size_t i = 0; i < 3 * N + 2; ++i) {
+      d2.push_back(get_s<S, 1>(i % 10 + 'a'));
+      verify_inside(d2);
+      d2.push_back(get_s<S, 22>(i % 10 + 'b'));
+      verify_inside(d2);
+
+      d2.pop_front();
+      verify_inside(d2);
+    }
+  }
+  {
+    C d3;
+    for (size_t i = 0; i < 3 * N + 2; ++i) {
+      d3.push_front(get_s<S, 1>(i % 10 + 'a'));
+      verify_inside(d3);
+      d3.push_front(get_s<S, 28>(i % 10 + 'b'));
+      verify_inside(d3);
+
+      d3.pop_back();
+      verify_inside(d3);
+    }
+  }
+  {
+    C d4;
+    for (size_t i = 0; i < 3 * N + 2; ++i) {
+      // When there is no SSO, all elements inside should not be poisoned,
+      // so we can verify deque poisoning.
+      d4.push_front(get_s<S, 33>(i % 10 + 'a'));
+      verify_inside(d4);
+      assert(is_double_ended_contiguous_container_asan_correct(d4));
+      d4.push_back(get_s<S, 28>(i % 10 + 'b'));
+      verify_inside(d4);
+      assert(is_double_ended_contiguous_container_asan_correct(d4));
+    }
+  }
+  {
+    C d5;
+    for (size_t i = 0; i < 3 * N + 2; ++i) {
+      // In d4 we never had poisoned memory inside deque.
+      // Here we start with SSO, so part of the inside of the container,
+      // will be poisoned.
+      d5.push_front(S());
+      verify_inside(d5);
+    }
+    for (size_t i = 0; i < d5.size(); ++i) {
+      // We change the size to have long string.
+      // Memory owne by deque should not be poisoned by string.
+      d5[i].resize(100);
+      verify_inside(d5);
+    }
+
+    assert(is_double_ended_contiguous_container_asan_correct(d5));
+
+    d5.erase(d5.begin() + 2);
+    verify_inside(d5);
+
+    d5.erase(d5.end() - 2);
+    verify_inside(d5);
+
+    assert(is_double_ended_contiguous_container_asan_correct(d5));
+  }
+  {
+    C d6a;
+    assert(is_double_ended_contiguous_container_asan_correct(d6a));
+
+    C d6b(N + 2, get_s<S, 100>('a'));
+    d6b.push_front(get_s<S, 101>('b'));
+    while (!d6b.empty()) {
+      d6b.pop_back();
+      assert(is_double_ended_contiguous_container_asan_correct(d6b));
+    }
+
+    C d6c(N + 2, get_s<S, 102>('c'));
+    while (!d6c.empty()) {
+      d6c.pop_back();
+      assert(is_double_ended_contiguous_container_asan_correct(d6c));
+    }
+  }
+  {
+    C d7(9 * N + 2);
+
+    d7.insert(d7.begin() + 1, S());
+    verify_inside(d7);
+
+    d7.insert(d7.end() - 3, S());
+    verify_inside(d7);
+
+    d7.insert(d7.begin() + 2 * N, get_s<S, 1>('a'));
+    verify_inside(d7);
+
+    d7.insert(d7.end() - 2 * N, get_s<S, 1>('b'));
+    verify_inside(d7);
+
+    d7.insert(d7.begin() + 2 * N, 3 * N, get_s<S, 1>('c'));
+    verify_inside(d7);
+
+    // It may not be short for big element types, but it will be checked correctly:
+    d7.insert(d7.end() - 2 * N, 3 * N, get_s<S, 2>('d'));
+    verify_inside(d7);
+
+    d7.erase(d7.begin() + 2);
+    verify_inside(d7);
+
+    d7.erase(d7.end() - 2);
+    verify_inside(d7);
+  }
+}
+
+template <class S>
+void test_container() {
+  test_string<std::deque<S, std::allocator<S>>, S>();
+  test_string<std::deque<S, min_allocator<S>>, S>();
+  test_string<std::deque<S, safe_allocator<S>>, S>();
+}
+
+int main(int, char**) {
+  // Those tests support only types based on std::basic_string.
+  test_container<std::string>();
+  test_container<std::wstring>();
+#if TEST_STD_VER >= 11
+  test_container<std::u16string>();
+  test_container<std::u32string>();
+#endif
+#if TEST_STD_VER >= 20
+  test_container<std::u8string>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/strings/basic.string/asan_short.pass.cpp b/libcxx/test/libcxx/containers/strings/basic.string/asan_short.pass.cpp
new file mode 100644
index 000000000000..53c70bed189b
--- /dev/null
+++ b/libcxx/test/libcxx/containers/strings/basic.string/asan_short.pass.cpp
@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: asan
+// UNSUPPORTED: c++03
+
+// <string>
+
+// Basic test if ASan annotations work for short strings.
+
+#include <string>
+#include <cassert>
+#include <cstdlib>
+
+#include "asan_testing.h"
+#include "min_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
+
+extern "C" void __sanitizer_set_death_callback(void (*callback)(void));
+
+void do_exit() { exit(0); }
+
+int main(int, char**) {
+  {
+    typedef cpp17_input_iterator<char*> MyInputIter;
+    // Should not trigger ASan.
+    std::basic_string<char, std::char_traits<char>, safe_allocator<char>> v;
+    char i[] = {'a', 'b', 'c', 'd'};
+
+    v.insert(v.begin(), MyInputIter(i), MyInputIter(i + 4));
+    assert(v[0] == 'a');
+    assert(is_string_asan_correct(v));
+  }
+
+  __sanitizer_set_death_callback(do_exit);
+  {
+    using T     = char;
+    using C     = std::basic_string<T, std::char_traits<T>, safe_allocator<T>>;
+    const T t[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g'};
+    C c(std::begin(t), std::end(t));
+    assert(is_string_asan_correct(c));
+    assert(__sanitizer_verify_contiguous_container(c.data(), c.data() + c.size() + 1, c.data() + c.capacity() + 1) !=
+           0);
+    volatile T foo = c[c.size() + 1]; // should trigger ASAN. Use volatile to prevent being optimized away.
+    assert(false);                    // if we got here, ASAN didn't trigger
+    ((void)foo);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/strings/basic.string/asan_vector_integration.pass.cpp b/libcxx/test/libcxx/containers/strings/basic.string/asan_vector_integration.pass.cpp
new file mode 100644
index 000000000000..b7d95b706908
--- /dev/null
+++ b/libcxx/test/libcxx/containers/strings/basic.string/asan_vector_integration.pass.cpp
@@ -0,0 +1,182 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: asan
+// UNSUPPORTED: c++03
+
+#include <cassert>
+#include <string>
+#include <vector>
+#include <array>
+#include "test_macros.h"
+#include "asan_testing.h"
+#include "min_allocator.h"
+
+// This tests exists to check if strings work well with vector, as those
+// may be partialy annotated, we cannot simply call
+// is_contiguous_container_asan_correct, as it assumes that
+// object memory inside is not annotated, so we check everything in a more careful way.
+
+template <typename D>
+void verify_inside(D const& d) {
+  for (size_t i = 0; i < d.size(); ++i) {
+    assert(is_string_asan_correct(d[i]));
+  }
+}
+
+template <typename S, size_t N>
+S get_s(char c) {
+  S s;
+  for (size_t i = 0; i < N; ++i)
+    s.push_back(c);
+
+  return s;
+}
+
+template <class C, class S>
+void test_string() {
+  size_t const N = sizeof(S) < 256 ? (4096 / sizeof(S)) : 16;
+
+  {
+    C d1a(1), d1b(N), d1c(N + 1), d1d(5 * N);
+    verify_inside(d1a);
+    verify_inside(d1b);
+    verify_inside(d1c);
+    verify_inside(d1d);
+  }
+  {
+    C d2;
+    for (size_t i = 0; i < 3 * N + 2; ++i) {
+      d2.push_back(get_s<S, 1>(i % 10 + 'a'));
+      verify_inside(d2);
+      d2.push_back(get_s<S, 28>(i % 10 + 'b'));
+      verify_inside(d2);
+
+      d2.erase(d2.cbegin());
+      verify_inside(d2);
+    }
+  }
+  {
+    C d3;
+    for (size_t i = 0; i < 3 * N + 2; ++i) {
+      d3.push_back(get_s<S, 1>(i % 10 + 'a'));
+      verify_inside(d3);
+      d3.push_back(get_s<S, 28>(i % 10 + 'b'));
+      verify_inside(d3);
+
+      d3.pop_back();
+      verify_inside(d3);
+    }
+  }
+  {
+    C d4;
+    for (size_t i = 0; i < 3 * N + 2; ++i) {
+      // When there is no SSO, all elements inside should not be poisoned,
+      // so we can verify vector poisoning.
+      d4.push_back(get_s<S, 33>(i % 10 + 'a'));
+      verify_inside(d4);
+      assert(is_contiguous_container_asan_correct(d4));
+      d4.push_back(get_s<S, 28>(i % 10 + 'b'));
+      verify_inside(d4);
+      assert(is_contiguous_container_asan_correct(d4));
+    }
+  }
+  {
+    C d5;
+    for (size_t i = 0; i < 3 * N + 2; ++i) {
+      // In d4 we never had poisoned memory inside vector.
+      // Here we start with SSO, so part of the inside of the container,
+      // will be poisoned.
+      d5.push_back(S());
+      verify_inside(d5);
+    }
+    for (size_t i = 0; i < d5.size(); ++i) {
+      // We change the size to have long string.
+      // Memory owne by vector should not be poisoned by string.
+      d5[i].resize(100);
+      verify_inside(d5);
+    }
+
+    assert(is_contiguous_container_asan_correct(d5));
+
+    d5.erase(d5.begin() + 2);
+    verify_inside(d5);
+
+    d5.erase(d5.end() - 2);
+    verify_inside(d5);
+
+    assert(is_contiguous_container_asan_correct(d5));
+  }
+  {
+    C d6a;
+    assert(is_contiguous_container_asan_correct(d6a));
+
+    C d6b(N + 2, get_s<S, 100>('a'));
+    d6b.push_back(get_s<S, 101>('b'));
+    while (!d6b.empty()) {
+      d6b.pop_back();
+      assert(is_contiguous_container_asan_correct(d6b));
+    }
+
+    C d6c(N + 2, get_s<S, 102>('c'));
+    while (!d6c.empty()) {
+      d6c.pop_back();
+      assert(is_contiguous_container_asan_correct(d6c));
+    }
+  }
+  {
+    C d7(9 * N + 2);
+
+    d7.insert(d7.begin() + 1, S());
+    verify_inside(d7);
+
+    d7.insert(d7.end() - 3, S());
+    verify_inside(d7);
+
+    d7.insert(d7.begin() + 2 * N, get_s<S, 1>('a'));
+    verify_inside(d7);
+
+    d7.insert(d7.end() - 2 * N, get_s<S, 1>('b'));
+    verify_inside(d7);
+
+    d7.insert(d7.begin() + 2 * N, 3 * N, get_s<S, 1>('c'));
+    verify_inside(d7);
+
+    // It may not be short for big element types, but it will be checked correctly:
+    d7.insert(d7.end() - 2 * N, 3 * N, get_s<S, 2>('d'));
+    verify_inside(d7);
+
+    d7.erase(d7.begin() + 2);
+    verify_inside(d7);
+
+    d7.erase(d7.end() - 2);
+    verify_inside(d7);
+  }
+}
+
+template <class S>
+void test_container() {
+  test_string<std::vector<S, std::allocator<S>>, S>();
+  test_string<std::vector<S, min_allocator<S>>, S>();
+  test_string<std::vector<S, safe_allocator<S>>, S>();
+}
+
+int main(int, char**) {
+  // Those tests support only types based on std::basic_string.
+  test_container<std::string>();
+  test_container<std::wstring>();
+#if TEST_STD_VER >= 11
+  test_container<std::u16string>();
+  test_container<std::u32string>();
+#endif
+#if TEST_STD_VER >= 20
+  test_container<std::u8string>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp b/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
index 389816bb23aa..4d1a8ad9e229 100644
--- a/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
+++ b/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp
@@ -48,6 +48,7 @@ static_assert(!std::__libcpp_is_trivially_relocatable<MoveOnlyTriviallyCopyable>
 // ----------------------
 
 // basic_string
+#if defined(_LIBCPP_HAS_NO_ASAN) || !defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
 struct MyChar {
   char c;
 };
@@ -78,7 +79,7 @@ static_assert(
     !std::__libcpp_is_trivially_relocatable<
         std::basic_string<MyChar, NotTriviallyRelocatableCharTraits<MyChar>, test_allocator<MyChar> > >::value,
     "");
-
+#endif
 // unique_ptr
 struct NotTriviallyRelocatableDeleter {
   NotTriviallyRelocatableDeleter(const NotTriviallyRelocatableDeleter&);
diff --git a/libcxx/test/support/asan_testing.h b/libcxx/test/support/asan_testing.h
index 6bfc8280a4ea..3785c1f9c20d 100644
--- a/libcxx/test/support/asan_testing.h
+++ b/libcxx/test/support/asan_testing.h
@@ -56,35 +56,16 @@ TEST_CONSTEXPR bool is_double_ended_contiguous_container_asan_correct(const std:
 #endif
 
 #if TEST_HAS_FEATURE(address_sanitizer)
-template <typename S>
-bool is_string_short(S const& s) {
-  // We do not have access to __is_long(), but we can check if strings
-  // buffer is inside strings memory. If strings memory contains its content,
-  // SSO is in use. To check it, we can just confirm that the beginning is in
-  // the string object memory block.
-  // &s    - beginning of objects memory
-  // &s[0] - beginning of the buffer
-  // (&s+1) - end of objects memory
-  return (void*)std::addressof(s) <= (void*)std::addressof(s[0]) &&
-         (void*)std::addressof(s[0]) < (void*)(std::addressof(s) + 1);
-}
-
 template <typename ChrT, typename TraitsT, typename Alloc>
 TEST_CONSTEXPR bool is_string_asan_correct(const std::basic_string<ChrT, TraitsT, Alloc>& c) {
   if (TEST_IS_CONSTANT_EVALUATED)
     return true;
 
-  if (!is_string_short(c) || _LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED) {
-    if (std::__asan_annotate_container_with_allocator<Alloc>::value)
-      return __sanitizer_verify_contiguous_container(c.data(), c.data() + c.size() + 1, c.data() + c.capacity() + 1) !=
-             0;
-    else
-      return __sanitizer_verify_contiguous_container(
-                 c.data(), c.data() + c.capacity() + 1, c.data() + c.capacity() + 1) != 0;
-  } else {
-    return __sanitizer_verify_contiguous_container(std::addressof(c), std::addressof(c) + 1, std::addressof(c) + 1) !=
-           0;
-  }
+  if (std::__asan_annotate_container_with_allocator<Alloc>::value)
+    return __sanitizer_verify_contiguous_container(c.data(), c.data() + c.size() + 1, c.data() + c.capacity() + 1) != 0;
+  else
+    return __sanitizer_verify_contiguous_container(
+               c.data(), c.data() + c.capacity() + 1, c.data() + c.capacity() + 1) != 0;
 }
 #else
 #  include <string>
-- 
GitLab


From 1318230587c30acb82324f851734a40341847a50 Mon Sep 17 00:00:00 2001
From: Yeting Kuo <46629943+yetingk@users.noreply.github.com>
Date: Wed, 8 May 2024 00:40:18 +0800
Subject: [PATCH 0072/1206] [RISCV][NFC] Remove redundant test cases. (#91324)

PR #89727 added the two test cases to verify `.option arch` should only
work when having -menable-experimental-extensions. And the test idea
could be splitted to
1. When having menable-experimental-extensions, clang passes
+experimental.
2. `.option arch` only enabled when +experimental enabled.

And we already had the two kind of tests.
---
 clang/test/Driver/riscv-option-arch.c | 8 --------
 clang/test/Driver/riscv-option-arch.s | 6 ------
 2 files changed, 14 deletions(-)
 delete mode 100644 clang/test/Driver/riscv-option-arch.c
 delete mode 100644 clang/test/Driver/riscv-option-arch.s

diff --git a/clang/test/Driver/riscv-option-arch.c b/clang/test/Driver/riscv-option-arch.c
deleted file mode 100644
index 9f0e037cd12e..000000000000
--- a/clang/test/Driver/riscv-option-arch.c
+++ /dev/null
@@ -1,8 +0,0 @@
-// REQUIRES: riscv-registered-target
-// RUN: %clang --target=riscv64 -menable-experimental-extensions -c -o /dev/null %s
-// RUN: ! %clang --target=riscv64 -c -o /dev/null %s 2>&1 | FileCheck -check-prefixes=CHECK-ERR %s
-
-void foo() {
-  asm volatile (".option arch, +zicfiss");
-  // CHECK-ERR: Unexpected experimental extensions.
-}
diff --git a/clang/test/Driver/riscv-option-arch.s b/clang/test/Driver/riscv-option-arch.s
deleted file mode 100644
index c4ca4aa459ce..000000000000
--- a/clang/test/Driver/riscv-option-arch.s
+++ /dev/null
@@ -1,6 +0,0 @@
-# REQUIRES: riscv-registered-target
-# RUN: %clang --target=riscv64 -menable-experimental-extensions -c -o /dev/null %s
-# RUN: ! %clang --target=riscv64 -c -o /dev/null %s 2>&1 | FileCheck -check-prefixes=CHECK-ERR %s
-
-.option arch, +zicfiss
-# CHECK-ERR: Unexpected experimental extensions.
-- 
GitLab


From 486695d154b23d0f66f3a5e054963b78d7d08d5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Tue, 7 May 2024 17:48:28 +0100
Subject: [PATCH 0073/1206] [flang] Remove driver-help-hidden.f90 (#91307)

This file was originally removed in #89504 and then accidentally
re-added in #89938.
---
 flang/test/Driver/driver-help-hidden.f90 | 172 -----------------------
 1 file changed, 172 deletions(-)
 delete mode 100644 flang/test/Driver/driver-help-hidden.f90

diff --git a/flang/test/Driver/driver-help-hidden.f90 b/flang/test/Driver/driver-help-hidden.f90
deleted file mode 100644
index 706b2cb6c245..000000000000
--- a/flang/test/Driver/driver-help-hidden.f90
+++ /dev/null
@@ -1,172 +0,0 @@
-
-!--------------------------
-! FLANG DRIVER (flang-new)
-!--------------------------
-! RUN: %flang --help-hidden 2>&1 | FileCheck %s
-! RUN: not %flang  -help-hidden 2>&1 | FileCheck %s --check-prefix=ERROR-FLANG
-
-!----------------------------------------
-! FLANG FRONTEND DRIVER (flang-new -fc1)
-!----------------------------------------
-! RUN: not %flang_fc1 --help-hidden 2>&1 | FileCheck %s --check-prefix=ERROR-FLANG-FC1
-! RUN: not %flang_fc1  -help-hidden 2>&1 | FileCheck %s --check-prefix=ERROR-FLANG-FC1
-
-! CHECK:USAGE: flang-new
-! CHECK-EMPTY:
-! CHECK-NEXT: DRIVER OPTIONS:
-! CHECK-NEXT:  --driver-mode=<value> Set the driver mode to either 'gcc', 'g++', 'cpp', 'cl' or 'flang'
-! CHECK-EMPTY:
-! CHECK-NEXT:OPTIONS:
-! CHECK-NEXT: -###                    Print (but do not run) the commands to run for this compilation
-! CHECK-NEXT: -ccc-print-phases       Dump list of actions to perform
-! CHECK-NEXT: -cpp                    Enable predefined and command line preprocessor macros
-! CHECK-NEXT: -c                      Only run preprocess, compile, and assemble steps
-! CHECK-NEXT: -dM                     Print macro definitions in -E mode instead of normal output
-! CHECK-NEXT: -dumpmachine            Display the compiler's target processor
-! CHECK-NEXT: -dumpversion            Display the version of the compiler
-! CHECK-NEXT: -D <macro>=<value>      Define <macro> to <value> (or 1 if <value> omitted)
-! CHECK-NEXT: -emit-llvm              Use the LLVM representation for assembler and object files
-! CHECK-NEXT: -E                      Only run the preprocessor
-! CHECK-NEXT: -falternative-parameter-statement
-! CHECK-NEXT:                         Enable the old style PARAMETER statement
-! CHECK-NEXT: -fapprox-func           Allow certain math function calls to be replaced with an approximately equivalent calculation
-! CHECK-NEXT: -fbackslash             Specify that backslash in string introduces an escape character
-! CHECK-NEXT: -fcolor-diagnostics     Enable colors in diagnostics
-! CHECK-NEXT: -fconvert=<value>       Set endian conversion of data for unformatted files
-! CHECK-NEXT: -fdefault-double-8      Set the default double precision kind to an 8 byte wide type
-! CHECK-NEXT: -fdefault-integer-8     Set the default integer and logical kind to an 8 byte wide type
-! CHECK-NEXT: -fdefault-real-8        Set the default real kind to an 8 byte wide type
-! CHECK-NEXT: -ffast-math             Allow aggressive, lossy floating-point optimizations
-! CHECK-NEXT: -ffixed-form            Process source files in fixed form
-! CHECK-NEXT: -ffixed-line-length=<value>
-! CHECK-NEXT:                         Use <value> as character line width in fixed mode
-! CHECK-NEXT: -ffp-contract=<value>   Form fused FP ops (e.g. FMAs)
-! CHECK-NEXT: -ffree-form             Process source files in free form
-! CHECK-NEXT: -fhonor-infinities      Specify that floating-point optimizations are not allowed that assume arguments and results are not +-inf.
-! CHECK-NEXT: -fhonor-nans            Specify that floating-point optimizations are not allowed that assume arguments and results are not NANs.
-! CHECK-NEXT: -fimplicit-none         No implicit typing allowed unless overridden by IMPLICIT statements
-! CHECK-NEXT: -finput-charset=<value> Specify the default character set for source files
-! CHECK-NEXT: -fintegrated-as         Enable the integrated assembler
-! CHECK-NEXT: -fintrinsic-modules-path <dir>
-! CHECK-NEXT:                         Specify where to find the compiled intrinsic modules
-! CHECK-NEXT: -flang-deprecated-no-hlfir
-! CHECK-NEXT:                         Do not use HLFIR lowering (deprecated)
-! CHECK-NEXT: -flang-experimental-hlfir
-! CHECK-NEXT:                         Use HLFIR lowering (experimental)
-! CHECK-NEXT: -flarge-sizes           Use INTEGER(KIND=8) for the result type in size-related intrinsics
-! CHECK-NEXT: -flogical-abbreviations Enable logical abbreviations
-! CHECK-NEXT: -flto=auto              Enable LTO in 'full' mode
-! CHECK-NEXT: -flto=jobserver         Enable LTO in 'full' mode
-! CHECK-NEXT: -flto=<value>           Set LTO mode
-! CHECK-NEXT: -flto                   Enable LTO in 'full' mode
-! CHECK-NEXT: -fms-runtime-lib=<value>
-! CHECK-NEXT:                         Select Windows run-time library
-! CHECK-NEXT: -fno-automatic          Implies the SAVE attribute for non-automatic local objects in subprograms unless RECURSIVE
-! CHECK-NEXT: -fno-color-diagnostics  Disable colors in diagnostics
-! CHECK-NEXT: -fno-integrated-as      Disable the integrated assembler
-! CHECK-NEXT: -fno-lto                Disable LTO mode (default)
-! CHECK-NEXT: -fno-ppc-native-vector-element-order
-! CHECK-NEXT:                         Specifies PowerPC non-native vector element order
-! CHECK-NEXT: -fno-rtlib-add-rpath Do not add -rpath with architecture-specific resource directory to the linker flags. When --hip-link is specified, do not add -rpath with HIP runtime library directory to the linker flags
-! CHECK-NEXT: -fno-signed-zeros       Allow optimizations that ignore the sign of floating point zeros
-! CHECK-NEXT: -fno-stack-arrays       Allocate array temporaries on the heap (default)
-! CHECK-NEXT: -fno-version-loops-for-stride
-! CHECK-NEXT:                         Do not create unit-strided loops (default)
-! CHECK-NEXT: -fomit-frame-pointer    Omit the frame pointer from functions that don't need it. Some stack unwinding cases, such as profilers and sanitizers, may prefer specifying -fno-omit-frame-pointer. On many targets, -O1 and higher omit the frame pointer by default. -m[no-]omit-leaf-frame-pointer takes precedence for leaf functions
-! CHECK-NEXT: -fopenacc               Enable OpenACC
-! CHECK-NEXT: -fopenmp-assume-no-nested-parallelism
-! CHECK-NEXT:                         Assert no nested parallel regions in the GPU
-! CHECK-NEXT: -fopenmp-assume-no-thread-state
-! CHECK-NEXT:                         Assert no thread in a parallel region modifies an ICV
-! CHECK-NEXT: -fopenmp-target-debug   Enable debugging in the OpenMP offloading device RTL
-! CHECK-NEXT: -fopenmp-targets=<value>
-! CHECK-NEXT:                         Specify comma-separated list of triples OpenMP offloading targets to be supported
-! CHECK-NEXT: -fopenmp-version=<value>
-! CHECK-NEXT:                         Set OpenMP version (e.g. 45 for OpenMP 4.5, 51 for OpenMP 5.1). Default value is 11 for Flang
-! CHECK-NEXT: -fopenmp                Parse OpenMP pragmas and generate parallel code.
-! CHECK-NEXT: -foptimization-record-file=<file>
-! CHECK-NEXT:                         Specify the output name of the file containing the optimization remarks. Implies -fsave-optimization-record. On Darwin platforms, this cannot be used with multiple -arch <arch> options.
-! CHECK-NEXT: -foptimization-record-passes=<regex>
-! CHECK-NEXT:                         Only include passes which match a specified regular expression in the generated optimization record (by default, include all passes)
-! CHECK-NEXT: -fpass-plugin=<dsopath> Load pass plugin from a dynamic shared object file (only with new pass manager).
-! CHECK-NEXT: -fppc-native-vector-element-order
-! CHECK-NEXT:                         Specifies PowerPC native vector element order (default)
-! CHECK-NEXT: -freciprocal-math       Allow division operations to be reassociated
-! CHECK-NEXT: -fropi                  Generate read-only position independent code (ARM only)
-! CHECK-NEXT: -frtlib-add-rpath Add -rpath with architecture-specific resource directory to the linker flags. When --hip-link is specified, also add -rpath with HIP runtime library directory to the linker flags
-! CHECK-NEXT: -frwpi                  Generate read-write position independent code (ARM only)
-! CHECK-NEXT: -fsave-optimization-record=<format>
-! CHECK-NEXT:                         Generate an optimization record file in a specific format
-! CHECK-NEXT: -fsave-optimization-record
-! CHECK-NEXT:                         Generate a YAML optimization record file
-! CHECK-NEXT: -fstack-arrays          Attempt to allocate array temporaries on the stack, no matter their size
-! CHECK-NEXT: -fsyntax-only           Run the preprocessor, parser and semantic analysis stages
-! CHECK-NEXT: -funderscoring          Appends one trailing underscore to external names
-! CHECK-NEXT: -fveclib=<value>        Use the given vector functions library
-! CHECK-NEXT: -fversion-loops-for-stride
-! CHECK-NEXT:                         Create unit-strided versions of loops
-! CHECK-NEXT: -fxor-operator          Enable .XOR. as a synonym of .NEQV.
-! CHECK-NEXT: --gcc-install-dir=<value>
-! CHECK-NEXT:                         Use GCC installation in the specified directory. The directory ends with path components like 'lib{,32,64}/gcc{,-cross}/$triple/$version'. Note: executables (e.g. ld) used by the compiler are not overridden by the selected GCC installation
-! CHECK-NEXT: --gcc-toolchain=<value> Specify a directory where Flang can find 'lib{,32,64}/gcc{,-cross}/$triple/$version'. Flang will use the GCC installation with the largest version
-! CHECK-NEXT: -gline-directives-only  Emit debug line info directives only
-! CHECK-NEXT: -gline-tables-only      Emit debug line number tables only
-! CHECK-NEXT: -gpulibc                Link the LLVM C Library for GPUs
-! CHECK-NEXT: -g                      Generate source-level debug information
-! CHECK-NEXT: --help-hidden           Display help for hidden options
-! CHECK-NEXT: -help                   Display available options
-! CHECK-NEXT: -isysroot <dir>         Set the system root directory (usually /)
-! CHECK-NEXT: -I <dir>                Add directory to the end of the list of include search paths
-! CHECK-NEXT: -L <dir>                Add directory to library search path
-! CHECK-NEXT: -march=<value>          For a list of available architectures for the target use '-mcpu=help'
-! CHECK-NEXT: -mcode-object-version=<value>
-! CHECK-NEXT:                         Specify code object ABI version. Defaults to 5. (AMDGPU only)
-! CHECK-NEXT: -mcpu=<value>           For a list of available CPUs for the target use '-mcpu=help'
-! CHECK-NEXT: -mllvm=<arg>            Alias for -mllvm
-! CHECK-NEXT: -mllvm <value>          Additional arguments to forward to LLVM's option processing
-! CHECK-NEXT: -mmlir <value>          Additional arguments to forward to MLIR's option processing
-! CHECK-NEXT: -mno-outline-atomics    Don't generate local calls to out-of-line atomic operations
-! CHECK-NEXT: -module-dir <dir>       Put MODULE files in <dir>
-! CHECK-NEXT: -moutline-atomics       Generate local calls to out-of-line atomic operations
-! CHECK-NEXT: -mrvv-vector-bits=<value>
-! CHECK-NEXT:                         Specify the size in bits of an RVV vector register
-! CHECK-NEXT: -msve-vector-bits=<value>
-! CHECK-NEXT:                          Specify the size in bits of an SVE vector register. Defaults to the vector length agnostic value of "scalable". (AArch64 only)
-! CHECK-NEXT: --no-offload-arch=<value>
-! CHECK-NEXT:                         Remove CUDA/HIP offloading device architecture (e.g. sm_35, gfx906) from the list of devices to compile for. 'all' resets the list to its default value.
-! CHECK-NEXT: -nocpp                  Disable predefined and command line preprocessor macros
-! CHECK-NEXT: -nogpulib               Do not link device library for CUDA/HIP device compilation
-! CHECK-NEXT: --offload-arch=<value>  Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). If 'native' is used the compiler will detect locally installed architectures. For HIP offloading, the device architecture can be followed by target ID features delimited by a colon (e.g. gfx908:xnack+:sramecc-). May be specified more than once.
-! CHECK-NEXT: --offload-device-only   Only compile for the offloading device.
-! CHECK-NEXT: --offload-host-device   Compile for both the offloading host and device (default).
-! CHECK-NEXT: --offload-host-only     Only compile for the offloading host.
-! CHECK-NEXT: -o <file>               Write output to <file>
-! CHECK-NEXT: -pedantic               Warn on language extensions
-! CHECK-NEXT: -print-effective-triple Print the effective target triple
-! CHECK-NEXT: -print-target-triple    Print the normalized target triple
-! CHECK-NEXT: -pthread                Support POSIX threads in generated code
-! CHECK-NEXT: -P                      Disable linemarker output in -E mode
-! CHECK-NEXT: -resource-dir <value>   The directory which holds the compiler resource files
-! CHECK-NEXT: --rocm-path=<value> ROCm installation path, used for finding and automatically linking required bitcode libraries.
-! CHECK-NEXT: -Rpass-analysis=<value> Report transformation analysis from optimization passes whose name matches the given POSIX regular expression
-! CHECK-NEXT: -Rpass-missed=<value>   Report missed transformations by optimization passes whose name matches the given POSIX regular expression
-! CHECK-NEXT: -Rpass=<value>          Report transformations performed by optimization passes whose name matches the given POSIX regular expression
-! CHECK-NEXT: -R<remark>              Enable the specified remark
-! CHECK-NEXT: -save-temps=<value>     Save intermediate compilation results.
-! CHECK-NEXT: -save-temps             Alias for --save-temps=cwd
-! CHECK-NEXT: -std=<value>            Language standard to compile for
-! CHECK-NEXT: -S                      Only run preprocess and compilation steps
-! CHECK-NEXT: --target=<value>        Generate code for the given target
-! CHECK-NEXT: -U <macro>              Undefine macro <macro>
-! CHECK-NEXT: --version               Print version information
-! CHECK-NEXT: -v                      Show commands to run and use verbose output
-! CHECK-NEXT: -Wl,<arg>               Pass the comma separated arguments in <arg> to the linker
-! CHECK-NEXT: -W<warning>             Enable the specified warning
-! CHECK-NEXT: -Xflang <arg>           Pass <arg> to the flang compiler
-! CHECK-NEXT: -x <language>           Treat subsequent input files as having type <language>
-
-
-! ERROR-FLANG: error: unknown argument '-help-hidden'; did you mean '--help-hidden'?
-
-! Frontend driver -help-hidden is not supported
-! ERROR-FLANG-FC1: error: unknown argument: '{{.*}}'
-- 
GitLab


From 72085698a244e10780a6f115269a2f88455c8cab Mon Sep 17 00:00:00 2001
From: Prashant Kumar <pk5561@gmail.com>
Date: Tue, 7 May 2024 22:19:28 +0530
Subject: [PATCH 0074/1206] [mlir][math] Add Polynomial Approximation for acos,
 asin op (#90962)

Adds the Polynomial Approximation for math.acos and math.asin op. Also,
it adds integration tests.
The Approximation has been borrowed from
https://stackoverflow.com/a/42683455
I added this script:
https://gist.github.com/pashu123/cd3e682b21a64ac306f650fb842a422b to
test 50 values between -1 and 1. The results are
https://gist.github.com/pashu123/8acb233bd045bacabfa8c992d4040465. It's
well within the bounds.
---
 .../Transforms/PolynomialApproximation.cpp    | 160 +++++++++++++++++-
 .../math-polynomial-approx.mlir               |  80 +++++++++
 2 files changed, 234 insertions(+), 6 deletions(-)

diff --git a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
index 428c1c37c4e8..f4fae68da63b 100644
--- a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
@@ -821,6 +821,153 @@ Log1pApproximation::matchAndRewrite(math::Log1pOp op,
   return success();
 }
 
+//----------------------------------------------------------------------------//
+// Asin approximation.
+//----------------------------------------------------------------------------//
+
+// Approximates asin(x).
+// This approximation is based on the following stackoverflow post:
+// https://stackoverflow.com/a/42683455
+namespace {
+struct AsinPolynomialApproximation : public OpRewritePattern<math::AsinOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(math::AsinOp op,
+                                PatternRewriter &rewriter) const final;
+};
+} // namespace
+LogicalResult
+AsinPolynomialApproximation::matchAndRewrite(math::AsinOp op,
+                                             PatternRewriter &rewriter) const {
+  Value operand = op.getOperand();
+  Type elementType = getElementTypeOrSelf(operand);
+
+  if (!(elementType.isF32() || elementType.isF16()))
+    return rewriter.notifyMatchFailure(op,
+                                       "only f32 and f16 type is supported.");
+  VectorShape shape = vectorShape(operand);
+
+  ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
+  auto bcast = [&](Value value) -> Value {
+    return broadcast(builder, value, shape);
+  };
+
+  auto fma = [&](Value a, Value b, Value c) -> Value {
+    return builder.create<math::FmaOp>(a, b, c);
+  };
+
+  auto mul = [&](Value a, Value b) -> Value {
+    return builder.create<arith::MulFOp>(a, b);
+  };
+
+  Value s = mul(operand, operand);
+  Value q = mul(s, s);
+  Value r = bcast(floatCst(builder, 5.5579749017470502e-2, elementType));
+  Value t = bcast(floatCst(builder, -6.2027913464120114e-2, elementType));
+
+  r = fma(r, q, bcast(floatCst(builder, 5.4224464349245036e-2, elementType)));
+  t = fma(t, q, bcast(floatCst(builder, -1.1326992890324464e-2, elementType)));
+  r = fma(r, q, bcast(floatCst(builder, 1.5268872539397656e-2, elementType)));
+  t = fma(t, q, bcast(floatCst(builder, 1.0493798473372081e-2, elementType)));
+  r = fma(r, q, bcast(floatCst(builder, 1.4106045900607047e-2, elementType)));
+  t = fma(t, q, bcast(floatCst(builder, 1.7339776384962050e-2, elementType)));
+  r = fma(r, q, bcast(floatCst(builder, 2.2372961589651054e-2, elementType)));
+  t = fma(t, q, bcast(floatCst(builder, 3.0381912707941005e-2, elementType)));
+  r = fma(r, q, bcast(floatCst(builder, 4.4642857881094775e-2, elementType)));
+  t = fma(t, q, bcast(floatCst(builder, 7.4999999991367292e-2, elementType)));
+  r = fma(r, s, t);
+  r = fma(r, s, bcast(floatCst(builder, 1.6666666666670193e-1, elementType)));
+  t = mul(operand, s);
+  r = fma(r, t, operand);
+
+  rewriter.replaceOp(op, r);
+  return success();
+}
+
+//----------------------------------------------------------------------------//
+// Acos approximation.
+//----------------------------------------------------------------------------//
+
+// Approximates acos(x).
+// This approximation is based on the following stackoverflow post:
+// https://stackoverflow.com/a/42683455
+namespace {
+struct AcosPolynomialApproximation : public OpRewritePattern<math::AcosOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(math::AcosOp op,
+                                PatternRewriter &rewriter) const final;
+};
+} // namespace
+LogicalResult
+AcosPolynomialApproximation::matchAndRewrite(math::AcosOp op,
+                                             PatternRewriter &rewriter) const {
+  Value operand = op.getOperand();
+  Type elementType = getElementTypeOrSelf(operand);
+
+  if (!(elementType.isF32() || elementType.isF16()))
+    return rewriter.notifyMatchFailure(op,
+                                       "only f32 and f16 type is supported.");
+  VectorShape shape = vectorShape(operand);
+
+  ImplicitLocOpBuilder builder(op->getLoc(), rewriter);
+  auto bcast = [&](Value value) -> Value {
+    return broadcast(builder, value, shape);
+  };
+
+  auto fma = [&](Value a, Value b, Value c) -> Value {
+    return builder.create<math::FmaOp>(a, b, c);
+  };
+
+  auto mul = [&](Value a, Value b) -> Value {
+    return builder.create<arith::MulFOp>(a, b);
+  };
+
+  Value negOperand = builder.create<arith::NegFOp>(operand);
+  Value zero = bcast(floatCst(builder, 0.0, elementType));
+  Value half = bcast(floatCst(builder, 0.5, elementType));
+  Value negOne = bcast(floatCst(builder, -1.0, elementType));
+  Value selR =
+      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, operand, zero);
+  Value r = builder.create<arith::SelectOp>(selR, negOperand, operand);
+  Value chkConst = bcast(floatCst(builder, -0.5625, elementType));
+  Value firstPred =
+      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, r, chkConst);
+
+  Value trueVal =
+      fma(bcast(floatCst(builder, 9.3282184640716537e-1, elementType)),
+          bcast(floatCst(builder, 1.6839188885261840e+0, elementType)),
+          builder.create<math::AsinOp>(r));
+
+  Value falseVal = builder.create<math::SqrtOp>(fma(half, r, half));
+  falseVal = builder.create<math::AsinOp>(falseVal);
+  falseVal = mul(bcast(floatCst(builder, 2.0, elementType)), falseVal);
+
+  r = builder.create<arith::SelectOp>(firstPred, trueVal, falseVal);
+
+  // Check whether the operand lies in between [-1.0, 0.0).
+  Value greaterThanNegOne =
+      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGE, operand, negOne);
+
+  Value lessThanZero =
+      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, operand, zero);
+
+  Value betweenNegOneZero =
+      builder.create<arith::AndIOp>(greaterThanNegOne, lessThanZero);
+
+  trueVal = fma(bcast(floatCst(builder, 1.8656436928143307e+0, elementType)),
+                bcast(floatCst(builder, 1.6839188885261840e+0, elementType)),
+                builder.create<arith::NegFOp>(r));
+
+  Value finalVal =
+      builder.create<arith::SelectOp>(betweenNegOneZero, trueVal, r);
+
+  rewriter.replaceOp(op, finalVal);
+  return success();
+}
+
 //----------------------------------------------------------------------------//
 // Erf approximation.
 //----------------------------------------------------------------------------//
@@ -1505,12 +1652,13 @@ void mlir::populateMathPolynomialApproximationPatterns(
            ReuseF32Expansion<math::SinOp>, ReuseF32Expansion<math::CosOp>>(
           patterns.getContext());
 
-  patterns.add<AtanApproximation, Atan2Approximation, TanhApproximation,
-               LogApproximation, Log2Approximation, Log1pApproximation,
-               ErfPolynomialApproximation, ExpApproximation, ExpM1Approximation,
-               CbrtApproximation, SinAndCosApproximation<true, math::SinOp>,
-               SinAndCosApproximation<false, math::CosOp>>(
-      patterns.getContext());
+  patterns
+      .add<AtanApproximation, Atan2Approximation, TanhApproximation,
+           LogApproximation, Log2Approximation, Log1pApproximation,
+           ErfPolynomialApproximation, AsinPolynomialApproximation,
+           AcosPolynomialApproximation, ExpApproximation, ExpM1Approximation,
+           CbrtApproximation, SinAndCosApproximation<true, math::SinOp>,
+           SinAndCosApproximation<false, math::CosOp>>(patterns.getContext());
   if (options.enableAvx2) {
     patterns.add<RsqrtApproximation, ReuseF32Expansion<math::RsqrtOp>>(
         patterns.getContext());
diff --git a/mlir/test/mlir-cpu-runner/math-polynomial-approx.mlir b/mlir/test/mlir-cpu-runner/math-polynomial-approx.mlir
index d3b19be9ecaf..370c5baa0ade 100644
--- a/mlir/test/mlir-cpu-runner/math-polynomial-approx.mlir
+++ b/mlir/test/mlir-cpu-runner/math-polynomial-approx.mlir
@@ -461,6 +461,84 @@ func.func @cos() {
   return
 }
 
+// -------------------------------------------------------------------------- //
+// Asin.
+// -------------------------------------------------------------------------- //
+func.func @asin_f32(%a : f32) {
+  %r = math.asin %a : f32
+  vector.print %r : f32
+  return
+}
+
+func.func @asin_3xf32(%a : vector<3xf32>) {
+  %r = math.asin %a : vector<3xf32>
+  vector.print %r : vector<3xf32>
+  return
+}
+
+func.func @asin() {
+  // CHECK: 0
+  %zero = arith.constant 0.0 : f32
+  call @asin_f32(%zero) : (f32) -> ()
+
+  // CHECK: -0.597406
+  %cst1 = arith.constant -0.5625 : f32
+  call @asin_f32(%cst1) : (f32) -> ()
+
+  // CHECK: -0.384397
+  %cst2 = arith.constant -0.375 : f32
+  call @asin_f32(%cst2) : (f32) -> ()
+
+  // CHECK: -0.25268
+  %cst3 = arith.constant -0.25 : f32
+  call @asin_f32(%cst3) : (f32) -> ()
+
+  // CHECK: 0.25268, 0.384397, 0.597406
+  %vec_x = arith.constant dense<[0.25, 0.375, 0.5625]> : vector<3xf32>
+  call @asin_3xf32(%vec_x) : (vector<3xf32>) -> ()
+
+  return
+}
+
+// -------------------------------------------------------------------------- //
+// Acos.
+// -------------------------------------------------------------------------- //
+func.func @acos_f32(%a : f32) {
+  %r = math.acos %a : f32
+  vector.print %r : f32
+  return
+}
+
+func.func @acos_3xf32(%a : vector<3xf32>) {
+  %r = math.acos %a : vector<3xf32>
+  vector.print %r : vector<3xf32>
+  return
+}
+
+func.func @acos() {
+  // CHECK: 1.5708
+  %zero = arith.constant 0.0 : f32
+  call @acos_f32(%zero) : (f32) -> ()
+
+  // CHECK: 2.1682
+  %cst1 = arith.constant -0.5625 : f32
+  call @acos_f32(%cst1) : (f32) -> ()
+
+  // CHECK: 1.95519
+  %cst2 = arith.constant -0.375 : f32
+  call @acos_f32(%cst2) : (f32) -> ()
+
+  // CHECK: 1.82348
+  %cst3 = arith.constant -0.25 : f32
+  call @acos_f32(%cst3) : (f32) -> ()
+
+  // CHECK: 1.31812, 1.1864, 0.97339
+  %vec_x = arith.constant dense<[0.25, 0.375, 0.5625]> : vector<3xf32>
+  call @acos_3xf32(%vec_x) : (vector<3xf32>) -> ()
+
+  return
+}
+
 // -------------------------------------------------------------------------- //
 // Atan.
 // -------------------------------------------------------------------------- //
@@ -694,6 +772,8 @@ func.func @main() {
   call @expm1(): () -> ()
   call @sin(): () -> ()
   call @cos(): () -> ()
+  call @asin(): () -> ()
+  call @acos(): () -> ()
   call @atan() : () -> ()
   call @atan2() : () -> ()
   call @cbrt() : () -> ()
-- 
GitLab


From 026a29e8b38aad79568de033d0e8e5d2e6bb4250 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 7 May 2024 10:20:10 -0700
Subject: [PATCH 0075/1206] [Analysis, CodeGen, DebugInfo] Use
 StringRef::operator== instead of StringRef::equals (NFC) (#91304)

I'm planning to remove StringRef::equals in favor of
StringRef::operator==.

- StringRef::operator==/!= outnumber StringRef::equals by a factor of
  53 under llvm/ in terms of their usage.

- The elimination of StringRef::equals brings StringRef closer to
  std::string_view, which has operator== but not equals.

- S == "foo" is more readable than S.equals("foo"), especially for
  !Long.Expression.equals("str") vs Long.Expression != "str".
---
 llvm/lib/Analysis/BlockFrequencyInfo.cpp               |  5 ++---
 llvm/lib/Analysis/BranchProbabilityInfo.cpp            |  5 ++---
 llvm/lib/Analysis/LoopInfo.cpp                         |  2 +-
 llvm/lib/Analysis/MemoryProfileInfo.cpp                |  4 ++--
 llvm/lib/CodeGen/MIRSampleProfile.cpp                  |  4 ++--
 llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp         |  5 ++---
 llvm/lib/CodeGen/MachineBlockPlacement.cpp             |  2 +-
 llvm/lib/CodeGen/TargetLoweringBase.cpp                |  4 ++--
 llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp      |  2 +-
 llvm/lib/DebugInfo/LogicalView/Core/LVOptions.cpp      |  2 +-
 .../DebugInfo/LogicalView/Readers/LVBinaryReader.cpp   |  5 ++---
 .../LogicalView/Readers/LVCodeViewVisitor.cpp          | 10 +++++-----
 .../lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp |  2 +-
 13 files changed, 24 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Analysis/BlockFrequencyInfo.cpp b/llvm/lib/Analysis/BlockFrequencyInfo.cpp
index ebad8388cbe4..d1b21e8c83f2 100644
--- a/llvm/lib/Analysis/BlockFrequencyInfo.cpp
+++ b/llvm/lib/Analysis/BlockFrequencyInfo.cpp
@@ -188,12 +188,11 @@ void BlockFrequencyInfo::calculate(const Function &F,
     BFI.reset(new ImplType);
   BFI->calculate(F, BPI, LI);
   if (ViewBlockFreqPropagationDAG != GVDT_None &&
-      (ViewBlockFreqFuncName.empty() ||
-       F.getName().equals(ViewBlockFreqFuncName))) {
+      (ViewBlockFreqFuncName.empty() || F.getName() == ViewBlockFreqFuncName)) {
     view();
   }
   if (PrintBFI &&
-      (PrintBFIFuncName.empty() || F.getName().equals(PrintBFIFuncName))) {
+      (PrintBFIFuncName.empty() || F.getName() == PrintBFIFuncName)) {
     print(dbgs());
   }
 }
diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index 6448ed66dc51..36a2df645913 100644
--- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -1273,9 +1273,8 @@ void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LoopI,
   EstimatedBlockWeight.clear();
   SccI.reset();
 
-  if (PrintBranchProb &&
-      (PrintBranchProbFuncName.empty() ||
-       F.getName().equals(PrintBranchProbFuncName))) {
+  if (PrintBranchProb && (PrintBranchProbFuncName.empty() ||
+                          F.getName() == PrintBranchProbFuncName)) {
     print(dbgs());
   }
 }
diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp
index 3075e5190f8e..369ab087ffc0 100644
--- a/llvm/lib/Analysis/LoopInfo.cpp
+++ b/llvm/lib/Analysis/LoopInfo.cpp
@@ -1032,7 +1032,7 @@ MDNode *llvm::findOptionMDForLoopID(MDNode *LoopID, StringRef Name) {
     if (!S)
       continue;
     // Return the operand node if MDString holds expected metadata.
-    if (Name.equals(S->getString()))
+    if (Name == S->getString())
       return MD;
   }
 
diff --git a/llvm/lib/Analysis/MemoryProfileInfo.cpp b/llvm/lib/Analysis/MemoryProfileInfo.cpp
index 8f5bc24747b1..5c09ba946271 100644
--- a/llvm/lib/Analysis/MemoryProfileInfo.cpp
+++ b/llvm/lib/Analysis/MemoryProfileInfo.cpp
@@ -86,9 +86,9 @@ AllocationType llvm::memprof::getMIBAllocType(const MDNode *MIB) {
   // types that can be applied based on the allocation profile data.
   auto *MDS = dyn_cast<MDString>(MIB->getOperand(1));
   assert(MDS);
-  if (MDS->getString().equals("cold")) {
+  if (MDS->getString() == "cold") {
     return AllocationType::Cold;
-  } else if (MDS->getString().equals("hot")) {
+  } else if (MDS->getString() == "hot") {
     return AllocationType::Hot;
   }
   return AllocationType::NotCold;
diff --git a/llvm/lib/CodeGen/MIRSampleProfile.cpp b/llvm/lib/CodeGen/MIRSampleProfile.cpp
index 42d0aba4b166..6faa1ad1a779 100644
--- a/llvm/lib/CodeGen/MIRSampleProfile.cpp
+++ b/llvm/lib/CodeGen/MIRSampleProfile.cpp
@@ -372,7 +372,7 @@ bool MIRProfileLoaderPass::runOnMachineFunction(MachineFunction &MF) {
   MF.RenumberBlocks();
   if (ViewBFIBefore && ViewBlockLayoutWithBFI != GVDT_None &&
       (ViewBlockFreqFuncName.empty() ||
-       MF.getFunction().getName().equals(ViewBlockFreqFuncName))) {
+       MF.getFunction().getName() == ViewBlockFreqFuncName)) {
     MBFI->view("MIR_Prof_loader_b." + MF.getName(), false);
   }
 
@@ -382,7 +382,7 @@ bool MIRProfileLoaderPass::runOnMachineFunction(MachineFunction &MF) {
 
   if (ViewBFIAfter && ViewBlockLayoutWithBFI != GVDT_None &&
       (ViewBlockFreqFuncName.empty() ||
-       MF.getFunction().getName().equals(ViewBlockFreqFuncName))) {
+       MF.getFunction().getName() == ViewBlockFreqFuncName)) {
     MBFI->view("MIR_prof_loader_a." + MF.getName(), false);
   }
 
diff --git a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
index cbebdd87398e..7ebecc6beb17 100644
--- a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
+++ b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
@@ -198,12 +198,11 @@ void MachineBlockFrequencyInfo::calculate(
     MBFI.reset(new ImplType);
   MBFI->calculate(F, MBPI, MLI);
   if (ViewMachineBlockFreqPropagationDAG != GVDT_None &&
-      (ViewBlockFreqFuncName.empty() ||
-       F.getName().equals(ViewBlockFreqFuncName))) {
+      (ViewBlockFreqFuncName.empty() || F.getName() == ViewBlockFreqFuncName)) {
     view("MachineBlockFrequencyDAGS." + F.getName());
   }
   if (PrintMachineBlockFreq &&
-      (PrintBFIFuncName.empty() || F.getName().equals(PrintBFIFuncName))) {
+      (PrintBFIFuncName.empty() || F.getName() == PrintBFIFuncName)) {
     MBFI->print(dbgs());
   }
 }
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index ef34e920aed5..c0cdeab25f1c 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -3500,7 +3500,7 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) {
   }
   if (ViewBlockLayoutWithBFI != GVDT_None &&
       (ViewBlockFreqFuncName.empty() ||
-       F->getFunction().getName().equals(ViewBlockFreqFuncName))) {
+       F->getFunction().getName() == ViewBlockFreqFuncName)) {
     if (RenumberBlocksBeforeView)
       MF.RenumberBlocks();
     MBFI->view("MBP." + MF.getName(), false);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 6e7b67ded23c..75b3f14e9622 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -2249,7 +2249,7 @@ static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) {
     if (IsDisabled)
       RecipType = RecipType.substr(1);
 
-    if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
+    if (RecipType == VTName || RecipType == VTNameNoSize)
       return IsDisabled ? TargetLoweringBase::ReciprocalEstimate::Disabled
                         : TargetLoweringBase::ReciprocalEstimate::Enabled;
   }
@@ -2299,7 +2299,7 @@ static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) {
       continue;
 
     RecipType = RecipType.substr(0, RefPos);
-    if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
+    if (RecipType == VTName || RecipType == VTNameNoSize)
       return RefSteps;
   }
 
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 81f3864ee4d0..622773cc73f7 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -1036,7 +1036,7 @@ MCSection *TargetLoweringObjectFileELF::getSectionForMachineBasicBlock(
   // name, or a unique ID for the section.
   SmallString<128> Name;
   StringRef FunctionSectionName = MBB.getParent()->getSection()->getName();
-  if (FunctionSectionName.equals(".text") ||
+  if (FunctionSectionName == ".text" ||
       FunctionSectionName.starts_with(".text.")) {
     // Function is in a regular .text section.
     StringRef FunctionName = MBB.getParent()->getName();
diff --git a/llvm/lib/DebugInfo/LogicalView/Core/LVOptions.cpp b/llvm/lib/DebugInfo/LogicalView/Core/LVOptions.cpp
index 265237ee21dc..c8789cb959fb 100644
--- a/llvm/lib/DebugInfo/LogicalView/Core/LVOptions.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Core/LVOptions.cpp
@@ -512,7 +512,7 @@ bool LVPatterns::matchPattern(StringRef Input, const LVMatchInfo &MatchInfo) {
   for (const LVMatch &Match : MatchInfo) {
     switch (Match.Mode) {
     case LVMatchMode::Match:
-      Matched = Input.equals(Match.Pattern);
+      Matched = Input == Match.Pattern;
       break;
     case LVMatchMode::NoCase:
       Matched = Input.equals_insensitive(Match.Pattern);
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp
index 2d46414a6986..c45f0e91c435 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp
@@ -184,9 +184,8 @@ void LVBinaryReader::mapVirtualAddress(const object::ObjectFile &Obj) {
       consumeError(SectionNameOrErr.takeError());
       continue;
     }
-    if ((*SectionNameOrErr).equals(".text") ||
-        (*SectionNameOrErr).equals("CODE") ||
-        (*SectionNameOrErr).equals(".code")) {
+    if (*SectionNameOrErr == ".text" || *SectionNameOrErr == "CODE" ||
+        *SectionNameOrErr == ".code") {
       DotTextSectionIndex = Section.getIndex();
       // If the object is WebAssembly, update the address offset that
       // will be added to DWARF DW_AT_* attributes.
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
index 1d0178532882..e89664d360a9 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
@@ -834,7 +834,7 @@ Error LVSymbolVisitor::visitKnownRecord(CVSymbol &Record,
     // Symbol was created as 'variable'; determine its real kind.
     Symbol->resetIsVariable();
 
-    if (Local.Name.equals("this")) {
+    if (Local.Name == "this") {
       Symbol->setIsParameter();
       Symbol->setIsArtificial();
     } else {
@@ -885,7 +885,7 @@ Error LVSymbolVisitor::visitKnownRecord(CVSymbol &Record,
     Symbol->resetIsVariable();
 
     // Check for the 'this' symbol.
-    if (Local.Name.equals("this")) {
+    if (Local.Name == "this") {
       Symbol->setIsArtificial();
       Symbol->setIsParameter();
     } else {
@@ -1429,7 +1429,7 @@ Error LVSymbolVisitor::visitKnownRecord(CVSymbol &Record, LocalSym &Local) {
 
     // Be sure the 'this' symbol is marked as 'compiler generated'.
     if (bool(Local.Flags & LocalSymFlags::IsCompilerGenerated) ||
-        Local.Name.equals("this")) {
+        Local.Name == "this") {
       Symbol->setIsArtificial();
       Symbol->setIsParameter();
     } else {
@@ -1669,7 +1669,7 @@ Error LVSymbolVisitor::visitKnownRecord(CVSymbol &Record, UDTSym &UDT) {
       Type->resetIncludeInPrint();
     else {
       StringRef RecordName = getRecordName(Types, UDT.Type);
-      if (UDT.Name.equals(RecordName))
+      if (UDT.Name == RecordName)
         Type->resetIncludeInPrint();
       Type->setType(LogicalVisitor->getElement(StreamTPI, UDT.Type));
     }
@@ -2740,7 +2740,7 @@ Error LVLogicalVisitor::visitKnownMember(CVMemberRecord &Record,
             getInnerComponent(NestedTypeName);
         // We have an already created nested type. Add it to the current scope
         // and update all its children if any.
-        if (OuterComponent.size() && OuterComponent.equals(RecordName)) {
+        if (OuterComponent.size() && OuterComponent == RecordName) {
           if (!NestedType->getIsScopedAlready()) {
             Scope->addElement(NestedType);
             NestedType->setIsScopedAlready();
diff --git a/llvm/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp b/llvm/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp
index d4fc48e146f6..02a9555858e4 100644
--- a/llvm/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp
+++ b/llvm/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp
@@ -355,7 +355,7 @@ std::vector<object::SectionedAddress>
 SymbolizableObjectFile::findSymbol(StringRef Symbol, uint64_t Offset) const {
   std::vector<object::SectionedAddress> Result;
   for (const SymbolDesc &Sym : Symbols) {
-    if (Sym.Name.equals(Symbol)) {
+    if (Sym.Name == Symbol) {
       uint64_t Addr = Sym.Addr;
       if (Offset < Sym.Size)
         Addr += Offset;
-- 
GitLab


From 873431a68a3aa3ec4fed5d2dc98ef527230b0d21 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 7 May 2024 12:57:54 -0500
Subject: [PATCH 0076/1206] [libc] Add __FE_DENORM to the fenv macros (#91353)

Summary:
Some targets support denormals as floating point exceptions. This is
provided as an extension in the GNU headers as __FE_DENORM.

This provides it in our headers, however I'm unsure if we should make it
internal or external. I do not think it should be in all exception as it
doesn't represent an exceptional behavior as far as the standard is
concerned, but I'm not an expert.
---
 libc/hdr/fenv_macros.h                      |  5 +++++
 libc/include/llvm-libc-macros/fenv-macros.h | 11 ++++++-----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/libc/hdr/fenv_macros.h b/libc/hdr/fenv_macros.h
index 1ad28cc278a9..041fca5f224b 100644
--- a/libc/hdr/fenv_macros.h
+++ b/libc/hdr/fenv_macros.h
@@ -17,6 +17,11 @@
 
 #include <fenv.h>
 
+// If this is not provided by the system, define it for use internally.
+#ifndef __FE_DENORM
+#define __FE_DENORM (1 << 6)
+#endif
+
 #endif // LLVM_LIBC_FULL_BUILD
 
 #endif // LLVM_LIBC_HDR_FENV_MACROS_H
diff --git a/libc/include/llvm-libc-macros/fenv-macros.h b/libc/include/llvm-libc-macros/fenv-macros.h
index 72ac660cd98c..1826723f9349 100644
--- a/libc/include/llvm-libc-macros/fenv-macros.h
+++ b/libc/include/llvm-libc-macros/fenv-macros.h
@@ -9,11 +9,12 @@
 #ifndef LLVM_LIBC_MACROS_FENV_MACROS_H
 #define LLVM_LIBC_MACROS_FENV_MACROS_H
 
-#define FE_DIVBYZERO 1
-#define FE_INEXACT 2
-#define FE_INVALID 4
-#define FE_OVERFLOW 8
-#define FE_UNDERFLOW 16
+#define FE_DIVBYZERO 0x1
+#define FE_INEXACT 0x2
+#define FE_INVALID 0x4
+#define FE_OVERFLOW 0x8
+#define FE_UNDERFLOW 0x10
+#define __FE_DENORM 0x20
 #define FE_ALL_EXCEPT                                                          \
   (FE_DIVBYZERO | FE_INEXACT | FE_INVALID | FE_OVERFLOW | FE_UNDERFLOW)
 
-- 
GitLab


From cf58c58e0967dbf812ef84944efd923ea366583a Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Tue, 7 May 2024 11:19:52 -0700
Subject: [PATCH 0077/1206] [bazel] Move HostMacOSXPrivateHeaders to macOS only
 dep (#91354)

---
 .../bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index 9c8943e44f7b..6c45cdf25cac 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -281,7 +281,6 @@ cc_library(
         "//lldb:Core",
         "//lldb:Headers",
         "//lldb:Host",
-        "//lldb:HostMacOSXPrivateHeaders",
         "//lldb:InterpreterHeaders",
         "//lldb:SymbolHeaders",
         "//lldb:TargetHeaders",
@@ -292,6 +291,7 @@ cc_library(
         "@platforms//os:macos": [
             ":PluginPlatformMacOSXObjCXX",
             ":PluginPlatformMacOSXObjCXXHeaders",
+            "//lldb:HostMacOSXPrivateHeaders",
         ],
         "//conditions:default": [],
     }),
-- 
GitLab


From 1a2a1fbd7c03381fe5e4f459f7081bef13366ef4 Mon Sep 17 00:00:00 2001
From: Brendan Dahl <brendan.dahl@gmail.com>
Date: Tue, 7 May 2024 11:33:10 -0700
Subject: [PATCH 0078/1206] [WebAssembly] Implement prototype f32.load_f16
 instruction. (#90906)

Adds a builtin and intrinsic for the f32.load_f16 instruction.

The instruction loads an f16 value from memory and puts it in an f32.
Specified at:

https://github.com/WebAssembly/half-precision/blob/29a9b9462c9285d4ccc1a5dc39214ddfd1892658/proposals/half-precision/Overview.md

Note: the current spec has f32.load_f16 as opcode 0xFD0120, but this is
incorrect and will be changed to 0xFC30 soon.
---
 .../clang/Basic/BuiltinsWebAssembly.def       |  3 ++
 clang/lib/CodeGen/CGBuiltin.cpp               |  5 ++++
 clang/test/CodeGen/builtins-wasm.c            |  9 ++++--
 llvm/include/llvm/IR/IntrinsicsWebAssembly.td | 12 ++++++++
 .../MCTargetDesc/WebAssemblyMCTargetDesc.h    |  1 +
 .../WebAssembly/WebAssemblyISelLowering.cpp   |  8 ++++++
 .../WebAssembly/WebAssemblyInstrMemory.td     |  5 ++++
 .../CodeGen/WebAssembly/half-precision.ll     | 12 ++++++++
 llvm/test/CodeGen/WebAssembly/offset.ll       | 28 ++++++++++++++++++-
 llvm/test/MC/WebAssembly/simd-encodings.s     |  5 +++-
 10 files changed, 84 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/WebAssembly/half-precision.ll

diff --git a/clang/include/clang/Basic/BuiltinsWebAssembly.def b/clang/include/clang/Basic/BuiltinsWebAssembly.def
index 7e950914ad94..cf54f8f4422f 100644
--- a/clang/include/clang/Basic/BuiltinsWebAssembly.def
+++ b/clang/include/clang/Basic/BuiltinsWebAssembly.def
@@ -190,6 +190,9 @@ TARGET_BUILTIN(__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8, "V8sV16ScV16Sc",
 TARGET_BUILTIN(__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4, "V4iV16ScV16ScV4i", "nc", "relaxed-simd")
 TARGET_BUILTIN(__builtin_wasm_relaxed_dot_bf16x8_add_f32_f32x4, "V4fV8UsV8UsV4f", "nc", "relaxed-simd")
 
+// Half-Precision (fp16)
+TARGET_BUILTIN(__builtin_wasm_loadf16_f32, "fh*", "nU", "half-precision")
+
 // Reference Types builtins
 // Some builtins are custom type-checked - see 't' as part of the third argument,
 // in which case the argument spec (second argument) is unused.
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 8e31652f4dab..e8a6bd050e17 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -21303,6 +21303,11 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
         CGM.getIntrinsic(Intrinsic::wasm_relaxed_dot_bf16x8_add_f32);
     return Builder.CreateCall(Callee, {LHS, RHS, Acc});
   }
+  case WebAssembly::BI__builtin_wasm_loadf16_f32: {
+    Value *Addr = EmitScalarExpr(E->getArg(0));
+    Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_loadf16_f32);
+    return Builder.CreateCall(Callee, {Addr});
+  }
   case WebAssembly::BI__builtin_wasm_table_get: {
     assert(E->getArg(0)->getType()->isArrayType());
     Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
diff --git a/clang/test/CodeGen/builtins-wasm.c b/clang/test/CodeGen/builtins-wasm.c
index 9a323da9a8e8..ab1c6cd494ae 100644
--- a/clang/test/CodeGen/builtins-wasm.c
+++ b/clang/test/CodeGen/builtins-wasm.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple wasm32-unknown-unknown -target-feature +reference-types -target-feature +simd128 -target-feature +relaxed-simd -target-feature +nontrapping-fptoint -target-feature +exception-handling -target-feature +bulk-memory -target-feature +atomics -flax-vector-conversions=none -O3 -emit-llvm -o - %s | FileCheck %s -check-prefixes WEBASSEMBLY,WEBASSEMBLY32
-// RUN: %clang_cc1 -triple wasm64-unknown-unknown -target-feature +reference-types -target-feature +simd128 -target-feature +relaxed-simd -target-feature +nontrapping-fptoint -target-feature +exception-handling -target-feature +bulk-memory -target-feature +atomics -flax-vector-conversions=none -O3 -emit-llvm -o - %s | FileCheck %s -check-prefixes WEBASSEMBLY,WEBASSEMBLY64
+// RUN: %clang_cc1 -triple wasm32-unknown-unknown -target-feature +reference-types -target-feature +simd128 -target-feature +relaxed-simd -target-feature +nontrapping-fptoint -target-feature +exception-handling -target-feature +bulk-memory -target-feature +atomics -target-feature +half-precision -flax-vector-conversions=none -O3 -emit-llvm -o - %s | FileCheck %s -check-prefixes WEBASSEMBLY,WEBASSEMBLY32
+// RUN: %clang_cc1 -triple wasm64-unknown-unknown -target-feature +reference-types -target-feature +simd128 -target-feature +relaxed-simd -target-feature +nontrapping-fptoint -target-feature +exception-handling -target-feature +bulk-memory -target-feature +atomics -target-feature +half-precision -flax-vector-conversions=none -O3 -emit-llvm -o - %s | FileCheck %s -check-prefixes WEBASSEMBLY,WEBASSEMBLY64
 // RUN: not %clang_cc1 -triple wasm64-unknown-unknown -target-feature +reference-types -target-feature +nontrapping-fptoint -target-feature +exception-handling -target-feature +bulk-memory -target-feature +atomics -flax-vector-conversions=none -O3 -emit-llvm -o - %s 2>&1 | FileCheck %s -check-prefixes MISSING-SIMD
 
 // SIMD convenience types
@@ -802,6 +802,11 @@ f32x4 relaxed_dot_bf16x8_add_f32_f32x4(u16x8 a, u16x8 b, f32x4 c) {
   // WEBASSEMBLY-NEXT: ret
 }
 
+float load_f16_f32(__fp16 *addr) {
+  return __builtin_wasm_loadf16_f32(addr);
+  // WEBASSEMBLY: call float @llvm.wasm.loadf16.f32(ptr %{{.*}})
+}
+
 __externref_t externref_null() {
   return __builtin_wasm_ref_null_extern();
   // WEBASSEMBLY: tail call ptr addrspace(10) @llvm.wasm.ref.null.extern()
diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
index b93a5e7be1b5..f8142a8ca9e9 100644
--- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
+++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -321,6 +321,18 @@ def int_wasm_relaxed_dot_bf16x8_add_f32:
                         [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v4f32_ty],
                         [IntrNoMem, IntrSpeculatable]>;
 
+//===----------------------------------------------------------------------===//
+// Half-precision intrinsics (experimental)
+//===----------------------------------------------------------------------===//
+
+// TODO: Replace these intrinsic with normal ISel patterns once the XXX
+// instructions are merged to the proposal.
+def int_wasm_loadf16_f32:
+  Intrinsic<[llvm_float_ty],
+            [llvm_ptr_ty],
+            [IntrReadMem, IntrArgMemOnly],
+             "", [SDNPMemOperand]>;
+
 
 //===----------------------------------------------------------------------===//
 // Thread-local storage intrinsics
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
index 15aeaaeb8c4a..d3b496ae5917 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -206,6 +206,7 @@ inline unsigned GetDefaultP2AlignAny(unsigned Opc) {
   WASM_LOAD_STORE(LOAD16_SPLAT)
   WASM_LOAD_STORE(LOAD_LANE_I16x8)
   WASM_LOAD_STORE(STORE_LANE_I16x8)
+  WASM_LOAD_STORE(LOAD_F16_F32)
   return 1;
   WASM_LOAD_STORE(LOAD_I32)
   WASM_LOAD_STORE(LOAD_F32)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 64bcadf3f567..ed52fe53bc60 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -906,6 +906,14 @@ bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.align = Align(8);
     Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
     return true;
+  case Intrinsic::wasm_loadf16_f32:
+    Info.opc = ISD::INTRINSIC_W_CHAIN;
+    Info.memVT = MVT::f16;
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.align = Align(2);
+    Info.flags = MachineMemOperand::MOLoad;
+    return true;
   default:
     return false;
   }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
index 01c0909af72e..e4baf842462a 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
@@ -72,6 +72,9 @@ defm LOAD16_U_I64 : WebAssemblyLoad<I64, "i64.load16_u", 0x33, []>;
 defm LOAD32_S_I64 : WebAssemblyLoad<I64, "i64.load32_s", 0x34, []>;
 defm LOAD32_U_I64 : WebAssemblyLoad<I64, "i64.load32_u", 0x35, []>;
 
+// Half Precision
+defm LOAD_F16_F32 : WebAssemblyLoad<F32, "f32.load_f16", 0xfc30, [HasHalfPrecision]>;
+
 // Pattern matching
 
 multiclass LoadPat<ValueType ty, SDPatternOperator kind, string Name> {
@@ -111,6 +114,8 @@ defm : LoadPat<i64, extloadi8, "LOAD8_U_I64">;
 defm : LoadPat<i64, extloadi16, "LOAD16_U_I64">;
 defm : LoadPat<i64, extloadi32, "LOAD32_U_I64">;
 
+defm : LoadPat<f32, int_wasm_loadf16_f32, "LOAD_F16_F32">;
+
 // Defines atomic and non-atomic stores, regular and truncating
 multiclass WebAssemblyStore<WebAssemblyRegClass rc, string Name, int Opcode,
                             list<Predicate> reqs = []> {
diff --git a/llvm/test/CodeGen/WebAssembly/half-precision.ll b/llvm/test/CodeGen/WebAssembly/half-precision.ll
new file mode 100644
index 000000000000..582771d3f95f
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/half-precision.ll
@@ -0,0 +1,12 @@
+; RUN: llc < %s --mtriple=wasm32-unknown-unknown -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+half-precision | FileCheck %s
+; RUN: llc < %s --mtriple=wasm64-unknown-unknown -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+half-precision | FileCheck %s
+
+declare float @llvm.wasm.loadf32.f16(ptr)
+
+; CHECK-LABEL: ldf16_32:
+; CHECK:      f32.load_f16 $push[[NUM0:[0-9]+]]=, 0($0){{$}}
+; CHECK-NEXT: return $pop[[NUM0]]{{$}}
+define float @ldf16_32(ptr %p) {
+  %v = call float @llvm.wasm.loadf16.f32(ptr %p)
+  ret float %v
+}
diff --git a/llvm/test/CodeGen/WebAssembly/offset.ll b/llvm/test/CodeGen/WebAssembly/offset.ll
index 0d9fcf05ab1b..b497ddd7273a 100644
--- a/llvm/test/CodeGen/WebAssembly/offset.ll
+++ b/llvm/test/CodeGen/WebAssembly/offset.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -wasm-disable-explicit-locals -wasm-keep-registers -disable-wasm-fallthrough-return-opt | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -wasm-disable-explicit-locals -wasm-keep-registers -disable-wasm-fallthrough-return-opt -mattr=+half-precision | FileCheck %s
 
 ; Test constant load and store address offsets.
 
@@ -666,3 +666,29 @@ define {i32,i32,i32,i32} @aggregate_return() {
 define {i64,i32,i16,i8} @aggregate_return_without_merge() {
   ret {i64,i32,i16,i8} zeroinitializer
 }
+
+;===----------------------------------------------------------------------------
+; Loads: Half Precision
+;===----------------------------------------------------------------------------
+
+; Fold an offset into a zero-extending load.
+
+; CHECK-LABEL: load_f16_f32_with_folded_offset:
+; CHECK: f32.load_f16 $push0=, 24($0){{$}}
+define float @load_f16_f32_with_folded_offset(ptr %p) {
+  %q = ptrtoint ptr %p to i32
+  %r = add nuw i32 %q, 24
+  %s = inttoptr i32 %r to ptr
+  %t = call float @llvm.wasm.loadf16.f32(ptr %s)
+  ret float %t
+}
+
+; Fold a gep offset into a zero-extending load.
+
+; CHECK-LABEL: load_f16_f32_with_folded_gep_offset:
+; CHECK: f32.load_f16 $push0=, 24($0){{$}}
+define float @load_f16_f32_with_folded_gep_offset(ptr %p) {
+  %s = getelementptr inbounds i8, ptr %p, i32 24
+  %t = call float @llvm.wasm.loadf16.f32(ptr %s)
+  ret float %t
+}
diff --git a/llvm/test/MC/WebAssembly/simd-encodings.s b/llvm/test/MC/WebAssembly/simd-encodings.s
index c6c554990c2c..e7c3761f381d 100644
--- a/llvm/test/MC/WebAssembly/simd-encodings.s
+++ b/llvm/test/MC/WebAssembly/simd-encodings.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -no-type-check -show-encoding -triple=wasm32-unknown-unknown -mattr=+simd128,+relaxed-simd < %s | FileCheck %s
+# RUN: llvm-mc -no-type-check -show-encoding -triple=wasm32-unknown-unknown -mattr=+simd128,+relaxed-simd,+half-precision < %s | FileCheck %s
 
 main:
     .functype main () -> ()
@@ -839,4 +839,7 @@ main:
     # CHECK: i32x4.relaxed_dot_i8x16_i7x16_add_s # encoding: [0xfd,0x93,0x02]
     i32x4.relaxed_dot_i8x16_i7x16_add_s
 
+    # CHECK: f32.load_f16 48 # encoding: [0xfc,0x30,0x01,0x30]
+    f32.load_f16 48
+
     end_function
-- 
GitLab


From 7115ed0fff027b65fa76fdfae215ed1382ed1473 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Tue, 7 May 2024 14:45:52 -0400
Subject: [PATCH 0079/1206] [Clang] Unify interface for accessing template
 arguments as written for class/variable template specializations (#81642)

Our current method of storing the template arguments as written for
`(Class/Var)Template(Partial)SpecializationDecl` suffers from a number
of flaws:
- We use `TypeSourceInfo` to store `TemplateArgumentLocs` for class
template/variable template partial/explicit specializations. For
variable template specializations, this is a rather unintuitive hack (as
we store a non-type specialization as a type). Moreover, we don't ever
*need* the type as written -- in almost all cases, we only want the
template arguments (e.g. in tooling use-cases).
- The template arguments as written are stored in a number of redundant
data members. For example, `(Class/Var)TemplatePartialSpecialization`
have their own `ArgsAsWritten` member that stores an
`ASTTemplateArgumentListInfo` (the template arguments).
`VarTemplateSpecializationDecl` has yet _another_ redundant member
"`TemplateArgsInfo`" that also stores an `ASTTemplateArgumentListInfo`.

This patch eliminates all
`(Class/Var)Template(Partial)SpecializationDecl` members which store the
template arguments as written, and turns the `ExplicitInfo` member into
a `llvm::PointerUnion<const ASTTemplateArgumentListInfo*,
ExplicitInstantiationInfo*>` (to avoid unnecessary allocations when the
declaration isn't an explicit instantiation). The template arguments as
written are now accessed via `getTemplateArgsWritten` in all cases.

The "most breaking" change is to AST Matchers, insofar that `hasTypeLoc`
will no longer match class template specializations (since they no
longer store the type as written).
---
 clang-tools-extra/clangd/AST.cpp              |  37 +-
 .../clangd/SemanticHighlighting.cpp           |  13 +-
 .../include-cleaner/lib/WalkAST.cpp           |  13 +-
 clang/docs/LibASTMatchersReference.html       | 364 +++++++++++++-----
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/AST/DeclTemplate.h        | 220 +++++------
 clang/include/clang/AST/RecursiveASTVisitor.h |  27 +-
 clang/include/clang/ASTMatchers/ASTMatchers.h |  74 ++--
 .../clang/ASTMatchers/ASTMatchersInternal.h   |  50 ++-
 clang/lib/AST/ASTImporter.cpp                 |  68 ++--
 clang/lib/AST/DeclPrinter.cpp                 |  18 +-
 clang/lib/AST/DeclTemplate.cpp                | 198 +++++-----
 clang/lib/AST/TypePrinter.cpp                 |  25 +-
 clang/lib/Index/IndexDecl.cpp                 |   9 +-
 clang/lib/Sema/Sema.cpp                       |   2 +-
 clang/lib/Sema/SemaTemplate.cpp               |  54 ++-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  | 157 +++-----
 clang/lib/Serialization/ASTReaderDecl.cpp     |  28 +-
 clang/lib/Serialization/ASTWriterDecl.cpp     |  36 +-
 clang/lib/Tooling/Syntax/BuildTree.cpp        |   3 +-
 clang/test/AST/ast-dump-template-decls.cpp    |  18 +-
 clang/test/Index/Core/index-source.cpp        |  24 +-
 clang/test/Index/index-refs.cpp               |   1 -
 clang/tools/libclang/CIndex.cpp               |  29 +-
 .../ASTMatchers/ASTMatchersNodeTest.cpp       |  12 -
 .../ASTMatchers/ASTMatchersTraversalTest.cpp  |  92 ++---
 26 files changed, 846 insertions(+), 729 deletions(-)

diff --git a/clang-tools-extra/clangd/AST.cpp b/clang-tools-extra/clangd/AST.cpp
index 1b86ea19cf28..fda1e5fdf8d8 100644
--- a/clang-tools-extra/clangd/AST.cpp
+++ b/clang-tools-extra/clangd/AST.cpp
@@ -50,16 +50,11 @@ getTemplateSpecializationArgLocs(const NamedDecl &ND) {
     if (const ASTTemplateArgumentListInfo *Args =
             Func->getTemplateSpecializationArgsAsWritten())
       return Args->arguments();
-  } else if (auto *Cls =
-                 llvm::dyn_cast<ClassTemplatePartialSpecializationDecl>(&ND)) {
+  } else if (auto *Cls = llvm::dyn_cast<ClassTemplateSpecializationDecl>(&ND)) {
     if (auto *Args = Cls->getTemplateArgsAsWritten())
       return Args->arguments();
-  } else if (auto *Var =
-                 llvm::dyn_cast<VarTemplatePartialSpecializationDecl>(&ND)) {
-    if (auto *Args = Var->getTemplateArgsAsWritten())
-      return Args->arguments();
   } else if (auto *Var = llvm::dyn_cast<VarTemplateSpecializationDecl>(&ND)) {
-    if (auto *Args = Var->getTemplateArgsInfo())
+    if (auto *Args = Var->getTemplateArgsAsWritten())
       return Args->arguments();
   }
   // We return std::nullopt for ClassTemplateSpecializationDecls because it does
@@ -270,22 +265,10 @@ std::string printTemplateSpecializationArgs(const NamedDecl &ND) {
           getTemplateSpecializationArgLocs(ND)) {
     printTemplateArgumentList(OS, *Args, Policy);
   } else if (auto *Cls = llvm::dyn_cast<ClassTemplateSpecializationDecl>(&ND)) {
-    if (const TypeSourceInfo *TSI = Cls->getTypeAsWritten()) {
-      // ClassTemplateSpecializationDecls do not contain
-      // TemplateArgumentTypeLocs, they only have TemplateArgumentTypes. So we
-      // create a new argument location list from TypeSourceInfo.
-      auto STL = TSI->getTypeLoc().getAs<TemplateSpecializationTypeLoc>();
-      llvm::SmallVector<TemplateArgumentLoc> ArgLocs;
-      ArgLocs.reserve(STL.getNumArgs());
-      for (unsigned I = 0; I < STL.getNumArgs(); ++I)
-        ArgLocs.push_back(STL.getArgLoc(I));
-      printTemplateArgumentList(OS, ArgLocs, Policy);
-    } else {
-      // FIXME: Fix cases when getTypeAsWritten returns null inside clang AST,
-      // e.g. friend decls. Currently we fallback to Template Arguments without
-      // location information.
-      printTemplateArgumentList(OS, Cls->getTemplateArgs().asArray(), Policy);
-    }
+    // FIXME: Fix cases when getTypeAsWritten returns null inside clang AST,
+    // e.g. friend decls. Currently we fallback to Template Arguments without
+    // location information.
+    printTemplateArgumentList(OS, Cls->getTemplateArgs().asArray(), Policy);
   }
   OS.flush();
   return TemplateArgs;
@@ -453,10 +436,12 @@ bool hasReservedScope(const DeclContext &DC) {
 }
 
 QualType declaredType(const TypeDecl *D) {
+  ASTContext &Context = D->getASTContext();
   if (const auto *CTSD = llvm::dyn_cast<ClassTemplateSpecializationDecl>(D))
-    if (const auto *TSI = CTSD->getTypeAsWritten())
-      return TSI->getType();
-  return D->getASTContext().getTypeDeclType(D);
+    if (const auto *Args = CTSD->getTemplateArgsAsWritten())
+      return Context.getTemplateSpecializationType(
+          TemplateName(CTSD->getSpecializedTemplate()), Args->arguments());
+  return Context.getTypeDeclType(D);
 }
 
 namespace {
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index 08f99e11ac9b..eb025f21f361 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -693,17 +693,22 @@ public:
     return true;
   }
 
+  bool
+  VisitClassTemplateSpecializationDecl(ClassTemplateSpecializationDecl *D) {
+    if (auto *Args = D->getTemplateArgsAsWritten())
+      H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
+    return true;
+  }
+
   bool VisitClassTemplatePartialSpecializationDecl(
       ClassTemplatePartialSpecializationDecl *D) {
     if (auto *TPL = D->getTemplateParameters())
       H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
-    if (auto *Args = D->getTemplateArgsAsWritten())
-      H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
     return true;
   }
 
   bool VisitVarTemplateSpecializationDecl(VarTemplateSpecializationDecl *D) {
-    if (auto *Args = D->getTemplateArgsInfo())
+    if (auto *Args = D->getTemplateArgsAsWritten())
       H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
     return true;
   }
@@ -712,8 +717,6 @@ public:
       VarTemplatePartialSpecializationDecl *D) {
     if (auto *TPL = D->getTemplateParameters())
       H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
-    if (auto *Args = D->getTemplateArgsAsWritten())
-      H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
     return true;
   }
 
diff --git a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
index 878067aca017..f7cc9d191236 100644
--- a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
+++ b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
@@ -267,18 +267,21 @@ public:
     return true;
   }
 
-  // Report a reference from explicit specializations to the specialized
-  // template. Implicit ones are filtered out by RAV and explicit instantiations
-  // are already traversed through typelocs.
+  // Report a reference from explicit specializations/instantiations to the
+  // specialized template. Implicit ones are filtered out by RAV.
   bool
   VisitClassTemplateSpecializationDecl(ClassTemplateSpecializationDecl *CTSD) {
-    if (CTSD->isExplicitSpecialization())
+    // if (CTSD->isExplicitSpecialization())
+    if (clang::isTemplateExplicitInstantiationOrSpecialization(
+            CTSD->getTemplateSpecializationKind()))
       report(CTSD->getLocation(),
              CTSD->getSpecializedTemplate()->getTemplatedDecl());
     return true;
   }
   bool VisitVarTemplateSpecializationDecl(VarTemplateSpecializationDecl *VTSD) {
-    if (VTSD->isExplicitSpecialization())
+    // if (VTSD->isExplicitSpecialization())
+    if (clang::isTemplateExplicitInstantiationOrSpecialization(
+            VTSD->getTemplateSpecializationKind()))
       report(VTSD->getLocation(),
              VTSD->getSpecializedTemplate()->getTemplatedDecl());
     return true;
diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html
index bb1b68f6671b..a16b9c44ef0e 100644
--- a/clang/docs/LibASTMatchersReference.html
+++ b/clang/docs/LibASTMatchersReference.html
@@ -3546,33 +3546,35 @@ cxxMethodDecl(isConst()) matches A::foo() but not A::bar()
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isExplicitObjectMemberFunction0')"><a name="isExplicitObjectMemberFunction0Anchor">isExplicitObjectMemberFunction</a></td><td></td></tr>
-<tr><td colspan="4" class="doc" id="isExplicitObjectMemberFunction0"><pre>Matches if the given method declaration declares a member function with an explicit object parameter.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isCopyAssignmentOperator0')"><a name="isCopyAssignmentOperator0Anchor">isCopyAssignmentOperator</a></td><td></td></tr>
+<tr><td colspan="4" class="doc" id="isCopyAssignmentOperator0"><pre>Matches if the given method declaration declares a copy assignment
+operator.
 
 Given
 struct A {
-  int operator-(this A, int);
-  void fun(this A &&self);
-  static int operator()(int);
-  int operator+(int);
+  A &amp;operator=(const A &amp;);
+  A &amp;operator=(A &amp;&amp;);
 };
 
-cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two methods but not the last two.
+cxxMethodDecl(isCopyAssignmentOperator()) matches the first method but not
+the second one.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isCopyAssignmentOperator0')"><a name="isCopyAssignmentOperator0Anchor">isCopyAssignmentOperator</a></td><td></td></tr>
-<tr><td colspan="4" class="doc" id="isCopyAssignmentOperator0"><pre>Matches if the given method declaration declares a copy assignment
-operator.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isExplicitObjectMemberFunction0')"><a name="isExplicitObjectMemberFunction0Anchor">isExplicitObjectMemberFunction</a></td><td></td></tr>
+<tr><td colspan="4" class="doc" id="isExplicitObjectMemberFunction0"><pre>Matches if the given method declaration declares a member function with an
+explicit object parameter.
 
 Given
 struct A {
-  A &amp;operator=(const A &amp;);
-  A &amp;operator=(A &amp;&amp;);
+ int operator-(this A, int);
+ void fun(this A &amp;&amp;self);
+ static int operator()(int);
+ int operator+(int);
 };
 
-cxxMethodDecl(isCopyAssignmentOperator()) matches the first method but not
-the second one.
+cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two
+methods but not the last two.
 </pre></td></tr>
 
 
@@ -6713,7 +6715,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -6757,7 +6759,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -6985,7 +6987,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7219,7 +7221,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7416,7 +7418,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7620,7 +7622,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7677,7 +7679,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7875,9 +7877,10 @@ int a = b ?: 1;
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument0')"><a name="forEachTemplateArgument0Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="forEachTemplateArgument0"><pre>Matches classTemplateSpecialization, templateSpecializationType and
-functionDecl nodes where the template argument matches the inner matcher.
-This matcher may produce multiple matches.
+<tr><td colspan="4" class="doc" id="forEachTemplateArgument0"><pre>Matches templateSpecializationType, class template specialization,
+variable template specialization, and function template specialization
+nodes where the template argument matches the inner matcher. This matcher
+may produce multiple matches.
 
 Given
   template &lt;typename T, unsigned N, unsigned M&gt;
@@ -7899,10 +7902,25 @@ functionDecl(forEachTemplateArgument(refersToType(builtinType())))
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc0')"><a name="hasAnyTemplateArgumentLoc0Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+that have at least one `TemplateArgumentLoc` matching the given
+`InnerMatcher`.
+
+Given
+  template&lt;typename T&gt; class A {};
+  A&lt;int&gt; a;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+  hasTypeLoc(loc(asString("int")))))))
+  matches `A&lt;int&gt; a`.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument0')"><a name="hasAnyTemplateArgument0Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument0"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl that have at least one TemplateArgument matching the given
-InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument0"><pre>Matches templateSpecializationTypes, class template specializations,
+variable template specializations, and function template specializations
+that have at least one TemplateArgument matching the given InnerMatcher.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -7933,9 +7951,25 @@ classTemplateSpecializationDecl(hasSpecializedTemplate(classTemplateDecl()))
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc0')"><a name="hasTemplateArgumentLoc0Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
+
+Given
+  template&lt;typename T, typename U&gt; class A {};
+  A&lt;double, int&gt; b;
+  A&lt;int, double&gt; c;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+  hasTypeLoc(loc(asString("double")))))))
+  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument0')"><a name="hasTemplateArgument0Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgument0"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasTemplateArgument0"><pre>Matches templateSpecializationType, class template specializations,
+variable template specializations, and function template specializations
+where the n'th TemplateArgument matches the given InnerMatcher.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -7953,34 +7987,6 @@ functionDecl(hasTemplateArgument(0, refersToType(asString("int"))))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc7')"><a name="hasTypeLoc7Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc7"><pre>Matches if the type location of a node matches the inner matcher.
-
-Examples:
-  int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
-
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
-
-struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
-
-Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
-</pre></td></tr>
-
-
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ComplexType.html">ComplexType</a>&gt;</td><td class="name" onclick="toggle('hasElementType1')"><a name="hasElementType1Anchor">hasElementType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td></tr>
 <tr><td colspan="4" class="doc" id="hasElementType1"><pre>Matches arrays and C99 complex types that have a specific element
 type.
@@ -7996,8 +8002,8 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Arra
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc8')"><a name="hasTypeLoc8Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc8"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc7')"><a name="hasTypeLoc7Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc7"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -8017,7 +8023,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -8066,6 +8072,21 @@ with compoundStmt()
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc3')"><a name="hasAnyTemplateArgumentLoc3Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc3"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+that have at least one `TemplateArgumentLoc` matching the given
+`InnerMatcher`.
+
+Given
+  template&lt;typename T&gt; class A {};
+  A&lt;int&gt; a;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+  hasTypeLoc(loc(asString("int")))))))
+  matches `A&lt;int&gt; a`.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration11')"><a name="hasDeclaration11Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration11"><pre>Matches a node if the declaration associated with that node
 matches the given matcher.
@@ -8100,9 +8121,10 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Addr
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc0')"><a name="hasTemplateArgumentLoc0Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s where the n'th
-`TemplateArgumentLoc` matches the given `InnerMatcher`.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc3')"><a name="hasTemplateArgumentLoc3Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc3"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -8176,8 +8198,8 @@ declStmt(hasSingleDecl(anything()))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc9')"><a name="hasTypeLoc9Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc9"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc8')"><a name="hasTypeLoc8Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc8"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -8197,7 +8219,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -8393,8 +8415,8 @@ actual casts "explicit" casts.)
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc10')"><a name="hasTypeLoc10Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc10"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc9')"><a name="hasTypeLoc9Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc9"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -8414,7 +8436,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -8707,9 +8729,10 @@ Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument2')"><a name="forEachTemplateArgument2Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="forEachTemplateArgument2"><pre>Matches classTemplateSpecialization, templateSpecializationType and
-functionDecl nodes where the template argument matches the inner matcher.
-This matcher may produce multiple matches.
+<tr><td colspan="4" class="doc" id="forEachTemplateArgument2"><pre>Matches templateSpecializationType, class template specialization,
+variable template specialization, and function template specialization
+nodes where the template argument matches the inner matcher. This matcher
+may produce multiple matches.
 
 Given
   template &lt;typename T, unsigned N, unsigned M&gt;
@@ -8778,10 +8801,25 @@ matching y.
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc2')"><a name="hasAnyTemplateArgumentLoc2Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc2"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+that have at least one `TemplateArgumentLoc` matching the given
+`InnerMatcher`.
+
+Given
+  template&lt;typename T&gt; class A {};
+  A&lt;int&gt; a;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+  hasTypeLoc(loc(asString("int")))))))
+  matches `A&lt;int&gt; a`.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument2')"><a name="hasAnyTemplateArgument2Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument2"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl that have at least one TemplateArgument matching the given
-InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument2"><pre>Matches templateSpecializationTypes, class template specializations,
+variable template specializations, and function template specializations
+that have at least one TemplateArgument matching the given InnerMatcher.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -8878,9 +8916,25 @@ functionDecl(hasReturnTypeLoc(loc(asString("int"))))
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc2')"><a name="hasTemplateArgumentLoc2Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc2"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
+
+Given
+  template&lt;typename T, typename U&gt; class A {};
+  A&lt;double, int&gt; b;
+  A&lt;int, double&gt; c;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+  hasTypeLoc(loc(asString("double")))))))
+  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument2')"><a name="hasTemplateArgument2Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgument2"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasTemplateArgument2"><pre>Matches templateSpecializationType, class template specializations,
+variable template specializations, and function template specializations
+where the n'th TemplateArgument matches the given InnerMatcher.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -9473,8 +9527,8 @@ matching y.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc11')"><a name="hasTypeLoc11Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc11"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc10')"><a name="hasTypeLoc10Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc10"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -9494,7 +9548,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -9919,8 +9973,8 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Addr
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc12')"><a name="hasTypeLoc12Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc12"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc11')"><a name="hasTypeLoc11Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc11"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -9940,7 +9994,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -10014,9 +10068,11 @@ matches the specialization of struct A generated by A&lt;X&gt;.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc0')"><a name="hasAnyTemplateArgumentLoc0Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s that have at least one
-`TemplateArgumentLoc` matching the given `InnerMatcher`.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc4')"><a name="hasAnyTemplateArgumentLoc4Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc4"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+that have at least one `TemplateArgumentLoc` matching the given
+`InnerMatcher`.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -10027,9 +10083,10 @@ varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc1')"><a name="hasTemplateArgumentLoc1Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc1"><pre>Matches template specialization `TypeLoc`s where the n'th
-`TemplateArgumentLoc` matches the given `InnerMatcher`.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc4')"><a name="hasTemplateArgumentLoc4Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc4"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -10041,10 +10098,11 @@ varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument1')"><a name="forEachTemplateArgument1Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="forEachTemplateArgument1"><pre>Matches classTemplateSpecialization, templateSpecializationType and
-functionDecl nodes where the template argument matches the inner matcher.
-This matcher may produce multiple matches.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument3')"><a name="forEachTemplateArgument3Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="forEachTemplateArgument3"><pre>Matches templateSpecializationType, class template specialization,
+variable template specialization, and function template specialization
+nodes where the template argument matches the inner matcher. This matcher
+may produce multiple matches.
 
 Given
   template &lt;typename T, unsigned N, unsigned M&gt;
@@ -10066,10 +10124,10 @@ functionDecl(forEachTemplateArgument(refersToType(builtinType())))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument1')"><a name="hasAnyTemplateArgument1Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument1"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl that have at least one TemplateArgument matching the given
-InnerMatcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument3')"><a name="hasAnyTemplateArgument3Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument3"><pre>Matches templateSpecializationTypes, class template specializations,
+variable template specializations, and function template specializations
+that have at least one TemplateArgument matching the given InnerMatcher.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -10122,9 +10180,10 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Addr
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument1')"><a name="hasTemplateArgument1Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgument1"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument3')"><a name="hasTemplateArgument3Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgument3"><pre>Matches templateSpecializationType, class template specializations,
+variable template specializations, and function template specializations
+where the n'th TemplateArgument matches the given InnerMatcher.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -10182,8 +10241,8 @@ QualType-matcher matches.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc13')"><a name="hasTypeLoc13Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc13"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc12')"><a name="hasTypeLoc12Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc12"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -10203,7 +10262,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -10449,6 +10508,105 @@ Example matches x (matcher = varDecl(hasInitializer(callExpr())))
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument1')"><a name="forEachTemplateArgument1Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="forEachTemplateArgument1"><pre>Matches templateSpecializationType, class template specialization,
+variable template specialization, and function template specialization
+nodes where the template argument matches the inner matcher. This matcher
+may produce multiple matches.
+
+Given
+  template &lt;typename T, unsigned N, unsigned M&gt;
+  struct Matrix {};
+
+  constexpr unsigned R = 2;
+  Matrix&lt;int, R * 2, R * 4&gt; M;
+
+  template &lt;typename T, typename U&gt;
+  void f(T&amp;&amp; t, U&amp;&amp; u) {}
+
+  bool B = false;
+  f(R, B);
+templateSpecializationType(forEachTemplateArgument(isExpr(expr())))
+  matches twice, with expr() matching 'R * 2' and 'R * 4'
+functionDecl(forEachTemplateArgument(refersToType(builtinType())))
+  matches the specialization f&lt;unsigned, bool&gt; twice, for 'unsigned'
+  and 'bool'
+</pre></td></tr>
+
+
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc1')"><a name="hasAnyTemplateArgumentLoc1Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc1"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+that have at least one `TemplateArgumentLoc` matching the given
+`InnerMatcher`.
+
+Given
+  template&lt;typename T&gt; class A {};
+  A&lt;int&gt; a;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+  hasTypeLoc(loc(asString("int")))))))
+  matches `A&lt;int&gt; a`.
+</pre></td></tr>
+
+
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument1')"><a name="hasAnyTemplateArgument1Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument1"><pre>Matches templateSpecializationTypes, class template specializations,
+variable template specializations, and function template specializations
+that have at least one TemplateArgument matching the given InnerMatcher.
+
+Given
+  template&lt;typename T&gt; class A {};
+  template&lt;&gt; class A&lt;double&gt; {};
+  A&lt;int&gt; a;
+
+  template&lt;typename T&gt; f() {};
+  void func() { f&lt;int&gt;(); };
+
+classTemplateSpecializationDecl(hasAnyTemplateArgument(
+    refersToType(asString("int"))))
+  matches the specialization A&lt;int&gt;
+
+functionDecl(hasAnyTemplateArgument(refersToType(asString("int"))))
+  matches the specialization f&lt;int&gt;
+</pre></td></tr>
+
+
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc1')"><a name="hasTemplateArgumentLoc1Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc1"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
+
+Given
+  template&lt;typename T, typename U&gt; class A {};
+  A&lt;double, int&gt; b;
+  A&lt;int, double&gt; c;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+  hasTypeLoc(loc(asString("double")))))))
+  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
+</pre></td></tr>
+
+
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument1')"><a name="hasTemplateArgument1Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgument1"><pre>Matches templateSpecializationType, class template specializations,
+variable template specializations, and function template specializations
+where the n'th TemplateArgument matches the given InnerMatcher.
+
+Given
+  template&lt;typename T, typename U&gt; class A {};
+  A&lt;bool, int&gt; b;
+  A&lt;int, bool&gt; c;
+
+  template&lt;typename T&gt; void f() {}
+  void func() { f&lt;int&gt;(); };
+classTemplateSpecializationDecl(hasTemplateArgument(
+    1, refersToType(asString("int"))))
+  matches the specialization A&lt;bool, int&gt;
+
+functionDecl(hasTemplateArgument(0, refersToType(asString("int"))))
+  matches the specialization f&lt;int&gt;
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VariableArrayType.html">VariableArrayType</a>&gt;</td><td class="name" onclick="toggle('hasSizeExpr0')"><a name="hasSizeExpr0Anchor">hasSizeExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasSizeExpr0"><pre>Matches VariableArrayType nodes that have a specific size
 expression.
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index cc3108bf41d6..2fae5731566d 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -111,6 +111,9 @@ Clang Frontend Potentially Breaking Changes
     $ clang --target=<your target triple> -print-target-triple
     <the normalized target triple>
 
+- The ``hasTypeLoc`` AST matcher will no longer match a ``classTemplateSpecializationDecl``;
+  existing uses should switch to ``templateArgumentLoc`` or ``hasAnyTemplateArgumentLoc`` instead.
+
 What's New in Clang |release|?
 ==============================
 Some of the major new features and improvements to Clang are listed
diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index 3ee03eebdb8c..36fb7ec80c17 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -1776,6 +1776,25 @@ public:
   BuiltinTemplateKind getBuiltinTemplateKind() const { return BTK; }
 };
 
+/// Provides information about an explicit instantiation of a variable or class
+/// template.
+struct ExplicitInstantiationInfo {
+  /// The template arguments as written..
+  const ASTTemplateArgumentListInfo *TemplateArgsAsWritten = nullptr;
+
+  /// The location of the extern keyword.
+  SourceLocation ExternKeywordLoc;
+
+  /// The location of the template keyword.
+  SourceLocation TemplateKeywordLoc;
+
+  ExplicitInstantiationInfo() = default;
+};
+
+using SpecializationOrInstantiationInfo =
+    llvm::PointerUnion<const ASTTemplateArgumentListInfo *,
+                       ExplicitInstantiationInfo *>;
+
 /// Represents a class template specialization, which refers to
 /// a class template with a given set of template arguments.
 ///
@@ -1789,8 +1808,8 @@ public:
 /// template<>
 /// class array<bool> { }; // class template specialization array<bool>
 /// \endcode
-class ClassTemplateSpecializationDecl
-  : public CXXRecordDecl, public llvm::FoldingSetNode {
+class ClassTemplateSpecializationDecl : public CXXRecordDecl,
+                                        public llvm::FoldingSetNode {
   /// Structure that stores information about a class template
   /// specialization that was instantiated from a class template partial
   /// specialization.
@@ -1808,23 +1827,9 @@ class ClassTemplateSpecializationDecl
   llvm::PointerUnion<ClassTemplateDecl *, SpecializedPartialSpecialization *>
     SpecializedTemplate;
 
-  /// Further info for explicit template specialization/instantiation.
-  struct ExplicitSpecializationInfo {
-    /// The type-as-written.
-    TypeSourceInfo *TypeAsWritten = nullptr;
-
-    /// The location of the extern keyword.
-    SourceLocation ExternLoc;
-
-    /// The location of the template keyword.
-    SourceLocation TemplateKeywordLoc;
-
-    ExplicitSpecializationInfo() = default;
-  };
-
   /// Further info for explicit template specialization/instantiation.
   /// Does not apply to implicit specializations.
-  ExplicitSpecializationInfo *ExplicitInfo = nullptr;
+  SpecializationOrInstantiationInfo ExplicitInfo = nullptr;
 
   /// The template arguments used to describe this specialization.
   const TemplateArgumentList *TemplateArgs;
@@ -2001,44 +2006,49 @@ public:
     SpecializedTemplate = TemplDecl;
   }
 
-  /// Sets the type of this specialization as it was written by
-  /// the user. This will be a class template specialization type.
-  void setTypeAsWritten(TypeSourceInfo *T) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->TypeAsWritten = T;
+  /// Retrieve the template argument list as written in the sources,
+  /// if any.
+  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->TemplateArgsAsWritten;
+    return ExplicitInfo.get<const ASTTemplateArgumentListInfo *>();
   }
 
-  /// Gets the type of this specialization as it was written by
-  /// the user, if it was so written.
-  TypeSourceInfo *getTypeAsWritten() const {
-    return ExplicitInfo ? ExplicitInfo->TypeAsWritten : nullptr;
+  /// Set the template argument list as written in the sources.
+  void
+  setTemplateArgsAsWritten(const ASTTemplateArgumentListInfo *ArgsWritten) {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      Info->TemplateArgsAsWritten = ArgsWritten;
+    else
+      ExplicitInfo = ArgsWritten;
   }
 
-  /// Gets the location of the extern keyword, if present.
-  SourceLocation getExternLoc() const {
-    return ExplicitInfo ? ExplicitInfo->ExternLoc : SourceLocation();
+  /// Set the template argument list as written in the sources.
+  void setTemplateArgsAsWritten(const TemplateArgumentListInfo &ArgsInfo) {
+    setTemplateArgsAsWritten(
+        ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo));
   }
 
-  /// Sets the location of the extern keyword.
-  void setExternLoc(SourceLocation Loc) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->ExternLoc = Loc;
+  /// Gets the location of the extern keyword, if present.
+  SourceLocation getExternKeywordLoc() const {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->ExternKeywordLoc;
+    return SourceLocation();
   }
 
-  /// Sets the location of the template keyword.
-  void setTemplateKeywordLoc(SourceLocation Loc) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->TemplateKeywordLoc = Loc;
-  }
+  /// Sets the location of the extern keyword.
+  void setExternKeywordLoc(SourceLocation Loc);
 
   /// Gets the location of the template keyword, if present.
   SourceLocation getTemplateKeywordLoc() const {
-    return ExplicitInfo ? ExplicitInfo->TemplateKeywordLoc : SourceLocation();
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->TemplateKeywordLoc;
+    return SourceLocation();
   }
 
+  /// Sets the location of the template keyword.
+  void setTemplateKeywordLoc(SourceLocation Loc);
+
   SourceRange getSourceRange() const override LLVM_READONLY;
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
@@ -2066,10 +2076,6 @@ class ClassTemplatePartialSpecializationDecl
   /// The list of template parameters
   TemplateParameterList* TemplateParams = nullptr;
 
-  /// The source info for the template arguments as written.
-  /// FIXME: redundant with TypeAsWritten?
-  const ASTTemplateArgumentListInfo *ArgsAsWritten = nullptr;
-
   /// The class template partial specialization from which this
   /// class template partial specialization was instantiated.
   ///
@@ -2078,15 +2084,11 @@ class ClassTemplatePartialSpecializationDecl
   llvm::PointerIntPair<ClassTemplatePartialSpecializationDecl *, 1, bool>
       InstantiatedFromMember;
 
-  ClassTemplatePartialSpecializationDecl(ASTContext &Context, TagKind TK,
-                                         DeclContext *DC,
-                                         SourceLocation StartLoc,
-                                         SourceLocation IdLoc,
-                                         TemplateParameterList *Params,
-                                         ClassTemplateDecl *SpecializedTemplate,
-                                         ArrayRef<TemplateArgument> Args,
-                               const ASTTemplateArgumentListInfo *ArgsAsWritten,
-                               ClassTemplatePartialSpecializationDecl *PrevDecl);
+  ClassTemplatePartialSpecializationDecl(
+      ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
+      SourceLocation IdLoc, TemplateParameterList *Params,
+      ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
+      ClassTemplatePartialSpecializationDecl *PrevDecl);
 
   ClassTemplatePartialSpecializationDecl(ASTContext &C)
     : ClassTemplateSpecializationDecl(C, ClassTemplatePartialSpecialization),
@@ -2101,11 +2103,8 @@ public:
   static ClassTemplatePartialSpecializationDecl *
   Create(ASTContext &Context, TagKind TK, DeclContext *DC,
          SourceLocation StartLoc, SourceLocation IdLoc,
-         TemplateParameterList *Params,
-         ClassTemplateDecl *SpecializedTemplate,
-         ArrayRef<TemplateArgument> Args,
-         const TemplateArgumentListInfo &ArgInfos,
-         QualType CanonInjectedType,
+         TemplateParameterList *Params, ClassTemplateDecl *SpecializedTemplate,
+         ArrayRef<TemplateArgument> Args, QualType CanonInjectedType,
          ClassTemplatePartialSpecializationDecl *PrevDecl);
 
   static ClassTemplatePartialSpecializationDecl *
@@ -2136,11 +2135,6 @@ public:
     return TemplateParams->hasAssociatedConstraints();
   }
 
-  /// Get the template arguments as written.
-  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
-    return ArgsAsWritten;
-  }
-
   /// Retrieve the member class template partial specialization from
   /// which this particular class template partial specialization was
   /// instantiated.
@@ -2613,27 +2607,12 @@ class VarTemplateSpecializationDecl : public VarDecl,
   llvm::PointerUnion<VarTemplateDecl *, SpecializedPartialSpecialization *>
   SpecializedTemplate;
 
-  /// Further info for explicit template specialization/instantiation.
-  struct ExplicitSpecializationInfo {
-    /// The type-as-written.
-    TypeSourceInfo *TypeAsWritten = nullptr;
-
-    /// The location of the extern keyword.
-    SourceLocation ExternLoc;
-
-    /// The location of the template keyword.
-    SourceLocation TemplateKeywordLoc;
-
-    ExplicitSpecializationInfo() = default;
-  };
-
   /// Further info for explicit template specialization/instantiation.
   /// Does not apply to implicit specializations.
-  ExplicitSpecializationInfo *ExplicitInfo = nullptr;
+  SpecializationOrInstantiationInfo ExplicitInfo = nullptr;
 
   /// The template arguments used to describe this specialization.
   const TemplateArgumentList *TemplateArgs;
-  const ASTTemplateArgumentListInfo *TemplateArgsInfo = nullptr;
 
   /// The point where this template was instantiated (if any).
   SourceLocation PointOfInstantiation;
@@ -2687,14 +2666,6 @@ public:
   /// specialization.
   const TemplateArgumentList &getTemplateArgs() const { return *TemplateArgs; }
 
-  // TODO: Always set this when creating the new specialization?
-  void setTemplateArgsInfo(const TemplateArgumentListInfo &ArgsInfo);
-  void setTemplateArgsInfo(const ASTTemplateArgumentListInfo *ArgsInfo);
-
-  const ASTTemplateArgumentListInfo *getTemplateArgsInfo() const {
-    return TemplateArgsInfo;
-  }
-
   /// Determine the kind of specialization that this
   /// declaration represents.
   TemplateSpecializationKind getSpecializationKind() const {
@@ -2798,44 +2769,49 @@ public:
     SpecializedTemplate = TemplDecl;
   }
 
-  /// Sets the type of this specialization as it was written by
-  /// the user.
-  void setTypeAsWritten(TypeSourceInfo *T) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->TypeAsWritten = T;
+  /// Retrieve the template argument list as written in the sources,
+  /// if any.
+  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->TemplateArgsAsWritten;
+    return ExplicitInfo.get<const ASTTemplateArgumentListInfo *>();
+  }
+
+  /// Set the template argument list as written in the sources.
+  void
+  setTemplateArgsAsWritten(const ASTTemplateArgumentListInfo *ArgsWritten) {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      Info->TemplateArgsAsWritten = ArgsWritten;
+    else
+      ExplicitInfo = ArgsWritten;
   }
 
-  /// Gets the type of this specialization as it was written by
-  /// the user, if it was so written.
-  TypeSourceInfo *getTypeAsWritten() const {
-    return ExplicitInfo ? ExplicitInfo->TypeAsWritten : nullptr;
+  /// Set the template argument list as written in the sources.
+  void setTemplateArgsAsWritten(const TemplateArgumentListInfo &ArgsInfo) {
+    setTemplateArgsAsWritten(
+        ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo));
   }
 
   /// Gets the location of the extern keyword, if present.
-  SourceLocation getExternLoc() const {
-    return ExplicitInfo ? ExplicitInfo->ExternLoc : SourceLocation();
+  SourceLocation getExternKeywordLoc() const {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->ExternKeywordLoc;
+    return SourceLocation();
   }
 
   /// Sets the location of the extern keyword.
-  void setExternLoc(SourceLocation Loc) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->ExternLoc = Loc;
-  }
-
-  /// Sets the location of the template keyword.
-  void setTemplateKeywordLoc(SourceLocation Loc) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->TemplateKeywordLoc = Loc;
-  }
+  void setExternKeywordLoc(SourceLocation Loc);
 
   /// Gets the location of the template keyword, if present.
   SourceLocation getTemplateKeywordLoc() const {
-    return ExplicitInfo ? ExplicitInfo->TemplateKeywordLoc : SourceLocation();
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->TemplateKeywordLoc;
+    return SourceLocation();
   }
 
+  /// Sets the location of the template keyword.
+  void setTemplateKeywordLoc(SourceLocation Loc);
+
   SourceRange getSourceRange() const override LLVM_READONLY;
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
@@ -2863,10 +2839,6 @@ class VarTemplatePartialSpecializationDecl
   /// The list of template parameters
   TemplateParameterList *TemplateParams = nullptr;
 
-  /// The source info for the template arguments as written.
-  /// FIXME: redundant with TypeAsWritten?
-  const ASTTemplateArgumentListInfo *ArgsAsWritten = nullptr;
-
   /// The variable template partial specialization from which this
   /// variable template partial specialization was instantiated.
   ///
@@ -2879,8 +2851,7 @@ class VarTemplatePartialSpecializationDecl
       ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
       SourceLocation IdLoc, TemplateParameterList *Params,
       VarTemplateDecl *SpecializedTemplate, QualType T, TypeSourceInfo *TInfo,
-      StorageClass S, ArrayRef<TemplateArgument> Args,
-      const ASTTemplateArgumentListInfo *ArgInfos);
+      StorageClass S, ArrayRef<TemplateArgument> Args);
 
   VarTemplatePartialSpecializationDecl(ASTContext &Context)
       : VarTemplateSpecializationDecl(VarTemplatePartialSpecialization,
@@ -2897,8 +2868,8 @@ public:
   Create(ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
          SourceLocation IdLoc, TemplateParameterList *Params,
          VarTemplateDecl *SpecializedTemplate, QualType T,
-         TypeSourceInfo *TInfo, StorageClass S, ArrayRef<TemplateArgument> Args,
-         const TemplateArgumentListInfo &ArgInfos);
+         TypeSourceInfo *TInfo, StorageClass S,
+         ArrayRef<TemplateArgument> Args);
 
   static VarTemplatePartialSpecializationDecl *
   CreateDeserialized(ASTContext &C, GlobalDeclID ID);
@@ -2914,11 +2885,6 @@ public:
     return TemplateParams;
   }
 
-  /// Get the template arguments as written.
-  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
-    return ArgsAsWritten;
-  }
-
   /// \brief All associated constraints of this partial specialization,
   /// including the requires clause and any constraints derived from
   /// constrained-parameters.
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index f9b145b4e86a..782f60844506 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -2030,6 +2030,15 @@ DEF_TRAVERSE_DECL(RecordDecl, { TRY_TO(TraverseRecordHelper(D)); })
 
 DEF_TRAVERSE_DECL(CXXRecordDecl, { TRY_TO(TraverseCXXRecordHelper(D)); })
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::TraverseTemplateArgumentLocsHelper(
+    const TemplateArgumentLoc *TAL, unsigned Count) {
+  for (unsigned I = 0; I < Count; ++I) {
+    TRY_TO(TraverseTemplateArgumentLoc(TAL[I]));
+  }
+  return true;
+}
+
 #define DEF_TRAVERSE_TMPL_SPEC_DECL(TMPLDECLKIND, DECLKIND)                    \
   DEF_TRAVERSE_DECL(TMPLDECLKIND##TemplateSpecializationDecl, {                \
     /* For implicit instantiations ("set<int> x;"), we don't want to           \
@@ -2039,9 +2048,12 @@ DEF_TRAVERSE_DECL(CXXRecordDecl, { TRY_TO(TraverseCXXRecordHelper(D)); })
        TemplateSpecializationType).  For explicit instantiations               \
        ("template set<int>;"), we do need a callback, since this               \
        is the only callback that's made for this instantiation.                \
-       We use getTypeAsWritten() to distinguish. */                            \
-    if (TypeSourceInfo *TSI = D->getTypeAsWritten())                           \
-      TRY_TO(TraverseTypeLoc(TSI->getTypeLoc()));                              \
+       We use getTemplateArgsAsWritten() to distinguish. */                    \
+    if (const auto *ArgsWritten = D->getTemplateArgsAsWritten()) {             \
+      /* The args that remains unspecialized. */                               \
+      TRY_TO(TraverseTemplateArgumentLocsHelper(                               \
+          ArgsWritten->getTemplateArgs(), ArgsWritten->NumTemplateArgs));      \
+    }                                                                          \
                                                                                \
     if (getDerived().shouldVisitTemplateInstantiations() ||                    \
         D->getTemplateSpecializationKind() == TSK_ExplicitSpecialization) {    \
@@ -2061,15 +2073,6 @@ DEF_TRAVERSE_DECL(CXXRecordDecl, { TRY_TO(TraverseCXXRecordHelper(D)); })
 DEF_TRAVERSE_TMPL_SPEC_DECL(Class, CXXRecord)
 DEF_TRAVERSE_TMPL_SPEC_DECL(Var, Var)
 
-template <typename Derived>
-bool RecursiveASTVisitor<Derived>::TraverseTemplateArgumentLocsHelper(
-    const TemplateArgumentLoc *TAL, unsigned Count) {
-  for (unsigned I = 0; I < Count; ++I) {
-    TRY_TO(TraverseTemplateArgumentLoc(TAL[I]));
-  }
-  return true;
-}
-
 #define DEF_TRAVERSE_TMPL_PART_SPEC_DECL(TMPLDECLKIND, DECLKIND)               \
   DEF_TRAVERSE_DECL(TMPLDECLKIND##TemplatePartialSpecializationDecl, {         \
     /* The partial specialization. */                                          \
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 8a2bbfff9e9e..0f3257db6f41 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -764,9 +764,9 @@ AST_POLYMORPHIC_MATCHER(isImplicit,
   return Node.isImplicit();
 }
 
-/// Matches classTemplateSpecializations, templateSpecializationType and
-/// functionDecl that have at least one TemplateArgument matching the given
-/// InnerMatcher.
+/// Matches templateSpecializationTypes, class template specializations,
+/// variable template specializations, and function template specializations
+/// that have at least one TemplateArgument matching the given InnerMatcher.
 ///
 /// Given
 /// \code
@@ -788,8 +788,8 @@ AST_POLYMORPHIC_MATCHER(isImplicit,
 AST_POLYMORPHIC_MATCHER_P(
     hasAnyTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
-                                    TemplateSpecializationType,
-                                    FunctionDecl),
+                                    VarTemplateSpecializationDecl, FunctionDecl,
+                                    TemplateSpecializationType),
     internal::Matcher<TemplateArgument>, InnerMatcher) {
   ArrayRef<TemplateArgument> List =
       internal::getTemplateSpecializationArgs(Node);
@@ -1047,8 +1047,9 @@ AST_MATCHER(Expr, isTypeDependent) { return Node.isTypeDependent(); }
 /// expr(isValueDependent()) matches return Size
 AST_MATCHER(Expr, isValueDependent) { return Node.isValueDependent(); }
 
-/// Matches classTemplateSpecializations, templateSpecializationType and
-/// functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
+/// Matches templateSpecializationType, class template specializations,
+/// variable template specializations, and function template specializations
+/// where the n'th TemplateArgument matches the given InnerMatcher.
 ///
 /// Given
 /// \code
@@ -1068,8 +1069,8 @@ AST_MATCHER(Expr, isValueDependent) { return Node.isValueDependent(); }
 AST_POLYMORPHIC_MATCHER_P2(
     hasTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
-                                    TemplateSpecializationType,
-                                    FunctionDecl),
+                                    VarTemplateSpecializationDecl, FunctionDecl,
+                                    TemplateSpecializationType),
     unsigned, N, internal::Matcher<TemplateArgument>, InnerMatcher) {
   ArrayRef<TemplateArgument> List =
       internal::getTemplateSpecializationArgs(Node);
@@ -4066,7 +4067,7 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 ///   Matcher<CXXCtorInitializer>, Matcher<CXXFunctionalCastExpr>,
 ///   Matcher<CXXNewExpr>, Matcher<CXXTemporaryObjectExpr>,
 ///   Matcher<CXXUnresolvedConstructExpr>,
-///   Matcher<ClassTemplateSpecializationDecl>, Matcher<CompoundLiteralExpr>,
+///   Matcher<CompoundLiteralExpr>,
 ///   Matcher<DeclaratorDecl>, Matcher<ExplicitCastExpr>,
 ///   Matcher<ObjCPropertyDecl>, Matcher<TemplateArgumentLoc>,
 ///   Matcher<TypedefNameDecl>
@@ -4075,9 +4076,8 @@ AST_POLYMORPHIC_MATCHER_P(
     AST_POLYMORPHIC_SUPPORTED_TYPES(
         BlockDecl, CXXBaseSpecifier, CXXCtorInitializer, CXXFunctionalCastExpr,
         CXXNewExpr, CXXTemporaryObjectExpr, CXXUnresolvedConstructExpr,
-        ClassTemplateSpecializationDecl, CompoundLiteralExpr, DeclaratorDecl,
-        ExplicitCastExpr, ObjCPropertyDecl, TemplateArgumentLoc,
-        TypedefNameDecl),
+        CompoundLiteralExpr, DeclaratorDecl, ExplicitCastExpr, ObjCPropertyDecl,
+        TemplateArgumentLoc, TypedefNameDecl),
     internal::Matcher<TypeLoc>, Inner) {
   TypeSourceInfo *source = internal::GetTypeSourceInfo(Node);
   if (source == nullptr) {
@@ -5304,9 +5304,10 @@ AST_POLYMORPHIC_MATCHER_P(parameterCountIs,
   return Node.getNumParams() == N;
 }
 
-/// Matches classTemplateSpecialization, templateSpecializationType and
-/// functionDecl nodes where the template argument matches the inner matcher.
-/// This matcher may produce multiple matches.
+/// Matches templateSpecializationType, class template specialization,
+/// variable template specialization, and function template specialization
+/// nodes where the template argument matches the inner matcher. This matcher
+/// may produce multiple matches.
 ///
 /// Given
 /// \code
@@ -5330,7 +5331,8 @@ AST_POLYMORPHIC_MATCHER_P(parameterCountIs,
 AST_POLYMORPHIC_MATCHER_P(
     forEachTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
-                                    TemplateSpecializationType, FunctionDecl),
+                                    VarTemplateSpecializationDecl, FunctionDecl,
+                                    TemplateSpecializationType),
     internal::Matcher<TemplateArgument>, InnerMatcher) {
   ArrayRef<TemplateArgument> TemplateArgs =
       clang::ast_matchers::internal::getTemplateSpecializationArgs(Node);
@@ -6905,8 +6907,10 @@ extern const internal::VariadicDynCastAllOfMatcher<
     TypeLoc, TemplateSpecializationTypeLoc>
     templateSpecializationTypeLoc;
 
-/// Matches template specialization `TypeLoc`s that have at least one
-/// `TemplateArgumentLoc` matching the given `InnerMatcher`.
+/// Matches template specialization `TypeLoc`s, class template specializations,
+/// variable template specializations, and function template specializations
+/// that have at least one `TemplateArgumentLoc` matching the given
+/// `InnerMatcher`.
 ///
 /// Given
 /// \code
@@ -6916,20 +6920,21 @@ extern const internal::VariadicDynCastAllOfMatcher<
 /// varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
 ///   hasTypeLoc(loc(asString("int")))))))
 ///   matches `A<int> a`.
-AST_MATCHER_P(TemplateSpecializationTypeLoc, hasAnyTemplateArgumentLoc,
-              internal::Matcher<TemplateArgumentLoc>, InnerMatcher) {
-  for (unsigned Index = 0, N = Node.getNumArgs(); Index < N; ++Index) {
-    clang::ast_matchers::internal::BoundNodesTreeBuilder Result(*Builder);
-    if (InnerMatcher.matches(Node.getArgLoc(Index), Finder, &Result)) {
-      *Builder = std::move(Result);
-      return true;
-    }
-  }
+AST_POLYMORPHIC_MATCHER_P(
+    hasAnyTemplateArgumentLoc,
+    AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
+                                    VarTemplateSpecializationDecl, FunctionDecl,
+                                    DeclRefExpr, TemplateSpecializationTypeLoc),
+    internal::Matcher<TemplateArgumentLoc>, InnerMatcher) {
+  auto Args = internal::getTemplateArgsWritten(Node);
+  return matchesFirstInRange(InnerMatcher, Args.begin(), Args.end(), Finder,
+                             Builder) != Args.end();
   return false;
 }
 
-/// Matches template specialization `TypeLoc`s where the n'th
-/// `TemplateArgumentLoc` matches the given `InnerMatcher`.
+/// Matches template specialization `TypeLoc`s, class template specializations,
+/// variable template specializations, and function template specializations
+/// where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
 ///
 /// Given
 /// \code
@@ -6942,10 +6947,13 @@ AST_MATCHER_P(TemplateSpecializationTypeLoc, hasAnyTemplateArgumentLoc,
 ///   matches `A<double, int> b`, but not `A<int, double> c`.
 AST_POLYMORPHIC_MATCHER_P2(
     hasTemplateArgumentLoc,
-    AST_POLYMORPHIC_SUPPORTED_TYPES(DeclRefExpr, TemplateSpecializationTypeLoc),
+    AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
+                                    VarTemplateSpecializationDecl, FunctionDecl,
+                                    DeclRefExpr, TemplateSpecializationTypeLoc),
     unsigned, Index, internal::Matcher<TemplateArgumentLoc>, InnerMatcher) {
-  return internal::MatchTemplateArgLocAt(Node, Index, InnerMatcher, Finder,
-                                         Builder);
+  auto Args = internal::getTemplateArgsWritten(Node);
+  return Index < Args.size() &&
+         InnerMatcher.matches(Args[Index], Finder, Builder);
 }
 
 /// Matches C or C++ elaborated `TypeLoc`s.
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
index 47d912c73dd7..c1cc63fdb743 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
@@ -186,10 +186,6 @@ inline TypeSourceInfo *GetTypeSourceInfo(const BlockDecl &Node) {
 inline TypeSourceInfo *GetTypeSourceInfo(const CXXNewExpr &Node) {
   return Node.getAllocatedTypeSourceInfo();
 }
-inline TypeSourceInfo *
-GetTypeSourceInfo(const ClassTemplateSpecializationDecl &Node) {
-  return Node.getTypeAsWritten();
-}
 
 /// Unifies obtaining the FunctionProtoType pointer from both
 /// FunctionProtoType and FunctionDecl nodes..
@@ -1939,6 +1935,11 @@ getTemplateSpecializationArgs(const ClassTemplateSpecializationDecl &D) {
   return D.getTemplateArgs().asArray();
 }
 
+inline ArrayRef<TemplateArgument>
+getTemplateSpecializationArgs(const VarTemplateSpecializationDecl &D) {
+  return D.getTemplateArgs().asArray();
+}
+
 inline ArrayRef<TemplateArgument>
 getTemplateSpecializationArgs(const TemplateSpecializationType &T) {
   return T.template_arguments();
@@ -1948,7 +1949,46 @@ inline ArrayRef<TemplateArgument>
 getTemplateSpecializationArgs(const FunctionDecl &FD) {
   if (const auto* TemplateArgs = FD.getTemplateSpecializationArgs())
     return TemplateArgs->asArray();
-  return ArrayRef<TemplateArgument>();
+  return std::nullopt;
+}
+
+inline ArrayRef<TemplateArgumentLoc>
+getTemplateArgsWritten(const ClassTemplateSpecializationDecl &D) {
+  if (const ASTTemplateArgumentListInfo *Args = D.getTemplateArgsAsWritten())
+    return Args->arguments();
+  return std::nullopt;
+}
+
+inline ArrayRef<TemplateArgumentLoc>
+getTemplateArgsWritten(const VarTemplateSpecializationDecl &D) {
+  if (const ASTTemplateArgumentListInfo *Args = D.getTemplateArgsAsWritten())
+    return Args->arguments();
+  return std::nullopt;
+}
+
+inline ArrayRef<TemplateArgumentLoc>
+getTemplateArgsWritten(const FunctionDecl &FD) {
+  if (const auto *Args = FD.getTemplateSpecializationArgsAsWritten())
+    return Args->arguments();
+  return std::nullopt;
+}
+
+inline ArrayRef<TemplateArgumentLoc>
+getTemplateArgsWritten(const DeclRefExpr &DRE) {
+  if (const auto *Args = DRE.getTemplateArgs())
+    return {Args, DRE.getNumTemplateArgs()};
+  return std::nullopt;
+}
+
+inline SmallVector<TemplateArgumentLoc>
+getTemplateArgsWritten(const TemplateSpecializationTypeLoc &T) {
+  SmallVector<TemplateArgumentLoc> Args;
+  if (!T.isNull()) {
+    Args.reserve(T.getNumArgs());
+    for (unsigned I = 0; I < T.getNumArgs(); ++I)
+      Args.emplace_back(T.getArgLoc(I));
+  }
+  return Args;
 }
 
 struct NotEqualsBoundNodePredicate {
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 60f213322b34..9ff8e1ea78d8 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -443,8 +443,9 @@ namespace clang {
     Expected<FunctionTemplateAndArgsTy>
     ImportFunctionTemplateWithTemplateArgsFromSpecialization(
         FunctionDecl *FromFD);
-    Error ImportTemplateParameterLists(const DeclaratorDecl *FromD,
-                                       DeclaratorDecl *ToD);
+
+    template <typename DeclTy>
+    Error ImportTemplateParameterLists(const DeclTy *FromD, DeclTy *ToD);
 
     Error ImportTemplateInformation(FunctionDecl *FromFD, FunctionDecl *ToFD);
 
@@ -3322,8 +3323,9 @@ ExpectedDecl ASTNodeImporter::VisitEnumConstantDecl(EnumConstantDecl *D) {
   return ToEnumerator;
 }
 
-Error ASTNodeImporter::ImportTemplateParameterLists(const DeclaratorDecl *FromD,
-                                                    DeclaratorDecl *ToD) {
+template <typename DeclTy>
+Error ASTNodeImporter::ImportTemplateParameterLists(const DeclTy *FromD,
+                                                    DeclTy *ToD) {
   unsigned int Num = FromD->getNumTemplateParameterLists();
   if (Num == 0)
     return Error::success();
@@ -6210,15 +6212,16 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
   if (!IdLocOrErr)
     return IdLocOrErr.takeError();
 
+  // Import TemplateArgumentListInfo.
+  TemplateArgumentListInfo ToTAInfo;
+  if (const auto *ASTTemplateArgs = D->getTemplateArgsAsWritten()) {
+    if (Error Err = ImportTemplateArgumentListInfo(*ASTTemplateArgs, ToTAInfo))
+      return std::move(Err);
+  }
+
   // Create the specialization.
   ClassTemplateSpecializationDecl *D2 = nullptr;
   if (PartialSpec) {
-    // Import TemplateArgumentListInfo.
-    TemplateArgumentListInfo ToTAInfo;
-    const auto &ASTTemplateArgs = *PartialSpec->getTemplateArgsAsWritten();
-    if (Error Err = ImportTemplateArgumentListInfo(ASTTemplateArgs, ToTAInfo))
-      return std::move(Err);
-
     QualType CanonInjType;
     if (Error Err = importInto(
         CanonInjType, PartialSpec->getInjectedSpecializationType()))
@@ -6228,7 +6231,7 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
     if (GetImportedOrCreateDecl<ClassTemplatePartialSpecializationDecl>(
             D2, D, Importer.getToContext(), D->getTagKind(), DC, *BeginLocOrErr,
             *IdLocOrErr, ToTPList, ClassTemplate,
-            llvm::ArrayRef(TemplateArgs.data(), TemplateArgs.size()), ToTAInfo,
+            llvm::ArrayRef(TemplateArgs.data(), TemplateArgs.size()),
             CanonInjType,
             cast_or_null<ClassTemplatePartialSpecializationDecl>(PrevDecl)))
       return D2;
@@ -6276,28 +6279,27 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
   else
     return BraceRangeOrErr.takeError();
 
+  if (Error Err = ImportTemplateParameterLists(D, D2))
+    return std::move(Err);
+
   // Import the qualifier, if any.
   if (auto LocOrErr = import(D->getQualifierLoc()))
     D2->setQualifierInfo(*LocOrErr);
   else
     return LocOrErr.takeError();
 
-  if (auto *TSI = D->getTypeAsWritten()) {
-    if (auto TInfoOrErr = import(TSI))
-      D2->setTypeAsWritten(*TInfoOrErr);
-    else
-      return TInfoOrErr.takeError();
+  if (D->getTemplateArgsAsWritten())
+    D2->setTemplateArgsAsWritten(ToTAInfo);
 
-    if (auto LocOrErr = import(D->getTemplateKeywordLoc()))
-      D2->setTemplateKeywordLoc(*LocOrErr);
-    else
-      return LocOrErr.takeError();
+  if (auto LocOrErr = import(D->getTemplateKeywordLoc()))
+    D2->setTemplateKeywordLoc(*LocOrErr);
+  else
+    return LocOrErr.takeError();
 
-    if (auto LocOrErr = import(D->getExternLoc()))
-      D2->setExternLoc(*LocOrErr);
-    else
-      return LocOrErr.takeError();
-  }
+  if (auto LocOrErr = import(D->getExternKeywordLoc()))
+    D2->setExternKeywordLoc(*LocOrErr);
+  else
+    return LocOrErr.takeError();
 
   if (D->getPointOfInstantiation().isValid()) {
     if (auto POIOrErr = import(D->getPointOfInstantiation()))
@@ -6517,7 +6519,7 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
   VarTemplateSpecializationDecl *D2 = nullptr;
 
   TemplateArgumentListInfo ToTAInfo;
-  if (const ASTTemplateArgumentListInfo *Args = D->getTemplateArgsInfo()) {
+  if (const auto *Args = D->getTemplateArgsAsWritten()) {
     if (Error Err = ImportTemplateArgumentListInfo(*Args, ToTAInfo))
       return std::move(Err);
   }
@@ -6525,14 +6527,6 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
   using PartVarSpecDecl = VarTemplatePartialSpecializationDecl;
   // Create a new specialization.
   if (auto *FromPartial = dyn_cast<PartVarSpecDecl>(D)) {
-    // Import TemplateArgumentListInfo
-    TemplateArgumentListInfo ArgInfos;
-    const auto *FromTAArgsAsWritten = FromPartial->getTemplateArgsAsWritten();
-    // NOTE: FromTAArgsAsWritten and template parameter list are non-null.
-    if (Error Err =
-            ImportTemplateArgumentListInfo(*FromTAArgsAsWritten, ArgInfos))
-      return std::move(Err);
-
     auto ToTPListOrErr = import(FromPartial->getTemplateParameters());
     if (!ToTPListOrErr)
       return ToTPListOrErr.takeError();
@@ -6541,7 +6535,7 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
     if (GetImportedOrCreateDecl(ToPartial, D, Importer.getToContext(), DC,
                                 *BeginLocOrErr, *IdLocOrErr, *ToTPListOrErr,
                                 VarTemplate, QualType(), nullptr,
-                                D->getStorageClass(), TemplateArgs, ArgInfos))
+                                D->getStorageClass(), TemplateArgs))
       return ToPartial;
 
     if (Expected<PartVarSpecDecl *> ToInstOrErr =
@@ -6584,7 +6578,9 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
   }
 
   D2->setSpecializationKind(D->getSpecializationKind());
-  D2->setTemplateArgsInfo(ToTAInfo);
+
+  if (D->getTemplateArgsAsWritten())
+    D2->setTemplateArgsAsWritten(ToTAInfo);
 
   if (auto LocOrErr = import(D->getQualifierLoc()))
     D2->setQualifierInfo(*LocOrErr);
diff --git a/clang/lib/AST/DeclPrinter.cpp b/clang/lib/AST/DeclPrinter.cpp
index 599d379340ab..c5868256b440 100644
--- a/clang/lib/AST/DeclPrinter.cpp
+++ b/clang/lib/AST/DeclPrinter.cpp
@@ -1083,15 +1083,15 @@ void DeclPrinter::VisitCXXRecordDecl(CXXRecordDecl *D) {
       NNS->print(Out, Policy);
     Out << *D;
 
-    if (auto S = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
-      ArrayRef<TemplateArgument> Args = S->getTemplateArgs().asArray();
-      if (!Policy.PrintCanonicalTypes)
-        if (const auto* TSI = S->getTypeAsWritten())
-          if (const auto *TST =
-                  dyn_cast<TemplateSpecializationType>(TSI->getType()))
-            Args = TST->template_arguments();
-      printTemplateArguments(
-          Args, S->getSpecializedTemplate()->getTemplateParameters());
+    if (auto *S = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
+      const TemplateParameterList *TParams =
+          S->getSpecializedTemplate()->getTemplateParameters();
+      const ASTTemplateArgumentListInfo *TArgAsWritten =
+          S->getTemplateArgsAsWritten();
+      if (TArgAsWritten && !Policy.PrintCanonicalTypes)
+        printTemplateArguments(TArgAsWritten->arguments(), TParams);
+      else
+        printTemplateArguments(S->getTemplateArgs().asArray(), TParams);
     }
   }
 
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index d27a30e0c5fc..af2d8d728e3e 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -985,41 +985,63 @@ ClassTemplateSpecializationDecl::getSpecializedTemplate() const {
 
 SourceRange
 ClassTemplateSpecializationDecl::getSourceRange() const {
-  if (ExplicitInfo) {
-    SourceLocation Begin = getTemplateKeywordLoc();
-    if (Begin.isValid()) {
-      // Here we have an explicit (partial) specialization or instantiation.
-      assert(getSpecializationKind() == TSK_ExplicitSpecialization ||
-             getSpecializationKind() == TSK_ExplicitInstantiationDeclaration ||
-             getSpecializationKind() == TSK_ExplicitInstantiationDefinition);
-      if (getExternLoc().isValid())
-        Begin = getExternLoc();
-      SourceLocation End = getBraceRange().getEnd();
-      if (End.isInvalid())
-        End = getTypeAsWritten()->getTypeLoc().getEndLoc();
-      return SourceRange(Begin, End);
-    }
-    // An implicit instantiation of a class template partial specialization
-    // uses ExplicitInfo to record the TypeAsWritten, but the source
-    // locations should be retrieved from the instantiation pattern.
-    using CTPSDecl = ClassTemplatePartialSpecializationDecl;
-    auto *ctpsd = const_cast<CTPSDecl *>(cast<CTPSDecl>(this));
-    CTPSDecl *inst_from = ctpsd->getInstantiatedFromMember();
-    assert(inst_from != nullptr);
-    return inst_from->getSourceRange();
-  }
-  else {
+  if (getSpecializationKind() == TSK_ExplicitInstantiationDeclaration) {
+    return SourceRange(getExternKeywordLoc(),
+                       getTemplateArgsAsWritten()->getRAngleLoc());
+  } else if (getSpecializationKind() == TSK_ExplicitInstantiationDefinition) {
+    return SourceRange(getTemplateKeywordLoc(),
+                       getTemplateArgsAsWritten()->getRAngleLoc());
+  } else if (!isExplicitSpecialization()) {
     // No explicit info available.
     llvm::PointerUnion<ClassTemplateDecl *,
                        ClassTemplatePartialSpecializationDecl *>
-      inst_from = getInstantiatedFrom();
-    if (inst_from.isNull())
+        InstFrom = getInstantiatedFrom();
+    if (InstFrom.isNull())
       return getSpecializedTemplate()->getSourceRange();
-    if (const auto *ctd = inst_from.dyn_cast<ClassTemplateDecl *>())
-      return ctd->getSourceRange();
-    return inst_from.get<ClassTemplatePartialSpecializationDecl *>()
-      ->getSourceRange();
+    if (const auto *CTD = InstFrom.dyn_cast<ClassTemplateDecl *>())
+      return CTD->getSourceRange();
+    return InstFrom.get<ClassTemplatePartialSpecializationDecl *>()
+        ->getSourceRange();
+  }
+  SourceLocation Begin = TagDecl::getOuterLocStart();
+  if (const auto *CTPSD =
+          dyn_cast<ClassTemplatePartialSpecializationDecl>(this)) {
+    if (const auto *InstFrom = CTPSD->getInstantiatedFromMember())
+      return InstFrom->getSourceRange();
+    else if (!getNumTemplateParameterLists())
+      Begin = CTPSD->getTemplateParameters()->getTemplateLoc();
+  }
+  SourceLocation End = getBraceRange().getEnd();
+  if (End.isInvalid())
+    End = getTemplateArgsAsWritten()->getRAngleLoc();
+  return SourceRange(Begin, End);
+}
+
+void ClassTemplateSpecializationDecl::setExternKeywordLoc(SourceLocation Loc) {
+  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  if (!Info) {
+    // Don't allocate if the location is invalid.
+    if (Loc.isInvalid())
+      return;
+    Info = new (getASTContext()) ExplicitInstantiationInfo;
+    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
+    ExplicitInfo = Info;
   }
+  Info->ExternKeywordLoc = Loc;
+}
+
+void ClassTemplateSpecializationDecl::setTemplateKeywordLoc(
+    SourceLocation Loc) {
+  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  if (!Info) {
+    // Don't allocate if the location is invalid.
+    if (Loc.isInvalid())
+      return;
+    Info = new (getASTContext()) ExplicitInstantiationInfo;
+    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
+    ExplicitInfo = Info;
+  }
+  Info->TemplateKeywordLoc = Loc;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1087,43 +1109,29 @@ void ImplicitConceptSpecializationDecl::setTemplateArguments(
 //===----------------------------------------------------------------------===//
 void ClassTemplatePartialSpecializationDecl::anchor() {}
 
-ClassTemplatePartialSpecializationDecl::
-ClassTemplatePartialSpecializationDecl(ASTContext &Context, TagKind TK,
-                                       DeclContext *DC,
-                                       SourceLocation StartLoc,
-                                       SourceLocation IdLoc,
-                                       TemplateParameterList *Params,
-                                       ClassTemplateDecl *SpecializedTemplate,
-                                       ArrayRef<TemplateArgument> Args,
-                               const ASTTemplateArgumentListInfo *ArgInfos,
-                               ClassTemplatePartialSpecializationDecl *PrevDecl)
-    : ClassTemplateSpecializationDecl(Context,
-                                      ClassTemplatePartialSpecialization,
-                                      TK, DC, StartLoc, IdLoc,
-                                      SpecializedTemplate, Args, PrevDecl),
-      TemplateParams(Params), ArgsAsWritten(ArgInfos),
-      InstantiatedFromMember(nullptr, false) {
+ClassTemplatePartialSpecializationDecl::ClassTemplatePartialSpecializationDecl(
+    ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
+    SourceLocation IdLoc, TemplateParameterList *Params,
+    ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
+    ClassTemplatePartialSpecializationDecl *PrevDecl)
+    : ClassTemplateSpecializationDecl(
+          Context, ClassTemplatePartialSpecialization, TK, DC, StartLoc, IdLoc,
+          SpecializedTemplate, Args, PrevDecl),
+      TemplateParams(Params), InstantiatedFromMember(nullptr, false) {
   if (AdoptTemplateParameterList(Params, this))
     setInvalidDecl();
 }
 
 ClassTemplatePartialSpecializationDecl *
-ClassTemplatePartialSpecializationDecl::
-Create(ASTContext &Context, TagKind TK,DeclContext *DC,
-       SourceLocation StartLoc, SourceLocation IdLoc,
-       TemplateParameterList *Params,
-       ClassTemplateDecl *SpecializedTemplate,
-       ArrayRef<TemplateArgument> Args,
-       const TemplateArgumentListInfo &ArgInfos,
-       QualType CanonInjectedType,
-       ClassTemplatePartialSpecializationDecl *PrevDecl) {
-  const ASTTemplateArgumentListInfo *ASTArgInfos =
-    ASTTemplateArgumentListInfo::Create(Context, ArgInfos);
-
-  auto *Result = new (Context, DC)
-      ClassTemplatePartialSpecializationDecl(Context, TK, DC, StartLoc, IdLoc,
-                                             Params, SpecializedTemplate, Args,
-                                             ASTArgInfos, PrevDecl);
+ClassTemplatePartialSpecializationDecl::Create(
+    ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
+    SourceLocation IdLoc, TemplateParameterList *Params,
+    ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
+    QualType CanonInjectedType,
+    ClassTemplatePartialSpecializationDecl *PrevDecl) {
+  auto *Result = new (Context, DC) ClassTemplatePartialSpecializationDecl(
+      Context, TK, DC, StartLoc, IdLoc, Params, SpecializedTemplate, Args,
+      PrevDecl);
   Result->setSpecializationKind(TSK_ExplicitSpecialization);
   Result->setMayHaveOutOfDateDef(false);
 
@@ -1371,26 +1379,47 @@ VarTemplateDecl *VarTemplateSpecializationDecl::getSpecializedTemplate() const {
   return SpecializedTemplate.get<VarTemplateDecl *>();
 }
 
-void VarTemplateSpecializationDecl::setTemplateArgsInfo(
-    const TemplateArgumentListInfo &ArgsInfo) {
-  TemplateArgsInfo =
-      ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo);
-}
-
-void VarTemplateSpecializationDecl::setTemplateArgsInfo(
-    const ASTTemplateArgumentListInfo *ArgsInfo) {
-  TemplateArgsInfo =
-      ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo);
-}
-
 SourceRange VarTemplateSpecializationDecl::getSourceRange() const {
   if (isExplicitSpecialization() && !hasInit()) {
-    if (const ASTTemplateArgumentListInfo *Info = getTemplateArgsInfo())
+    if (const ASTTemplateArgumentListInfo *Info = getTemplateArgsAsWritten())
       return SourceRange(getOuterLocStart(), Info->getRAngleLoc());
+  } else if (getTemplateSpecializationKind() ==
+             TSK_ExplicitInstantiationDeclaration) {
+    if (const ASTTemplateArgumentListInfo *Info = getTemplateArgsAsWritten())
+      return SourceRange(getExternKeywordLoc(), Info->getRAngleLoc());
+  } else if (getTemplateSpecializationKind() ==
+             TSK_ExplicitInstantiationDefinition) {
+    if (const ASTTemplateArgumentListInfo *Info = getTemplateArgsAsWritten())
+      return SourceRange(getTemplateKeywordLoc(), Info->getRAngleLoc());
   }
   return VarDecl::getSourceRange();
 }
 
+void VarTemplateSpecializationDecl::setExternKeywordLoc(SourceLocation Loc) {
+  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  if (!Info) {
+    // Don't allocate if the location is invalid.
+    if (Loc.isInvalid())
+      return;
+    Info = new (getASTContext()) ExplicitInstantiationInfo;
+    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
+    ExplicitInfo = Info;
+  }
+  Info->ExternKeywordLoc = Loc;
+}
+
+void VarTemplateSpecializationDecl::setTemplateKeywordLoc(SourceLocation Loc) {
+  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  if (!Info) {
+    // Don't allocate if the location is invalid.
+    if (Loc.isInvalid())
+      return;
+    Info = new (getASTContext()) ExplicitInstantiationInfo;
+    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
+    ExplicitInfo = Info;
+  }
+  Info->TemplateKeywordLoc = Loc;
+}
 
 //===----------------------------------------------------------------------===//
 // VarTemplatePartialSpecializationDecl Implementation
@@ -1402,13 +1431,11 @@ VarTemplatePartialSpecializationDecl::VarTemplatePartialSpecializationDecl(
     ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
     SourceLocation IdLoc, TemplateParameterList *Params,
     VarTemplateDecl *SpecializedTemplate, QualType T, TypeSourceInfo *TInfo,
-    StorageClass S, ArrayRef<TemplateArgument> Args,
-    const ASTTemplateArgumentListInfo *ArgInfos)
+    StorageClass S, ArrayRef<TemplateArgument> Args)
     : VarTemplateSpecializationDecl(VarTemplatePartialSpecialization, Context,
                                     DC, StartLoc, IdLoc, SpecializedTemplate, T,
                                     TInfo, S, Args),
-      TemplateParams(Params), ArgsAsWritten(ArgInfos),
-      InstantiatedFromMember(nullptr, false) {
+      TemplateParams(Params), InstantiatedFromMember(nullptr, false) {
   if (AdoptTemplateParameterList(Params, DC))
     setInvalidDecl();
 }
@@ -1418,15 +1445,10 @@ VarTemplatePartialSpecializationDecl::Create(
     ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
     SourceLocation IdLoc, TemplateParameterList *Params,
     VarTemplateDecl *SpecializedTemplate, QualType T, TypeSourceInfo *TInfo,
-    StorageClass S, ArrayRef<TemplateArgument> Args,
-    const TemplateArgumentListInfo &ArgInfos) {
-  const ASTTemplateArgumentListInfo *ASTArgInfos
-    = ASTTemplateArgumentListInfo::Create(Context, ArgInfos);
-
-  auto *Result =
-      new (Context, DC) VarTemplatePartialSpecializationDecl(
-          Context, DC, StartLoc, IdLoc, Params, SpecializedTemplate, T, TInfo,
-          S, Args, ASTArgInfos);
+    StorageClass S, ArrayRef<TemplateArgument> Args) {
+  auto *Result = new (Context, DC) VarTemplatePartialSpecializationDecl(
+      Context, DC, StartLoc, IdLoc, Params, SpecializedTemplate, T, TInfo, S,
+      Args);
   Result->setSpecializationKind(TSK_ExplicitSpecialization);
   return Result;
 }
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 9602f448e942..87f0a8728d85 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -1472,21 +1472,18 @@ void TypePrinter::printTag(TagDecl *D, raw_ostream &OS) {
 
   // If this is a class template specialization, print the template
   // arguments.
-  if (const auto *Spec = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
-    ArrayRef<TemplateArgument> Args;
-    TypeSourceInfo *TAW = Spec->getTypeAsWritten();
-    if (!Policy.PrintCanonicalTypes && TAW) {
-      const TemplateSpecializationType *TST =
-        cast<TemplateSpecializationType>(TAW->getType());
-      Args = TST->template_arguments();
-    } else {
-      const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
-      Args = TemplateArgs.asArray();
-    }
+  if (auto *S = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
+    const TemplateParameterList *TParams =
+        S->getSpecializedTemplate()->getTemplateParameters();
+    const ASTTemplateArgumentListInfo *TArgAsWritten =
+        S->getTemplateArgsAsWritten();
     IncludeStrongLifetimeRAII Strong(Policy);
-    printTemplateArgumentList(
-        OS, Args, Policy,
-        Spec->getSpecializedTemplate()->getTemplateParameters());
+    if (TArgAsWritten && !Policy.PrintCanonicalTypes)
+      printTemplateArgumentList(OS, TArgAsWritten->arguments(), Policy,
+                                TParams);
+    else
+      printTemplateArgumentList(OS, S->getTemplateArgs().asArray(), Policy,
+                                TParams);
   }
 
   spaceBeforePlaceHolder(OS);
diff --git a/clang/lib/Index/IndexDecl.cpp b/clang/lib/Index/IndexDecl.cpp
index 1c04aa17d53f..8eb88f5a1e94 100644
--- a/clang/lib/Index/IndexDecl.cpp
+++ b/clang/lib/Index/IndexDecl.cpp
@@ -673,9 +673,12 @@ public:
     IndexCtx.indexTagDecl(
         D, SymbolRelation(SymbolRoleSet(SymbolRole::RelationSpecializationOf),
                           SpecializationOf));
-    if (TypeSourceInfo *TSI = D->getTypeAsWritten())
-      IndexCtx.indexTypeSourceInfo(TSI, /*Parent=*/nullptr,
-                                   D->getLexicalDeclContext());
+    // Template specialization arguments.
+    if (const ASTTemplateArgumentListInfo *TemplateArgInfo =
+            D->getTemplateArgsAsWritten()) {
+      for (const auto &Arg : TemplateArgInfo->arguments())
+        handleTemplateArgumentLoc(Arg, D, D->getLexicalDeclContext());
+    }
     return true;
   }
 
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index a1e32d391ed0..0febf4e1d454 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -1408,7 +1408,7 @@ void Sema::ActOnEndOfTranslationUnit() {
         SourceRange DiagRange = DiagD->getLocation();
         if (const auto *VTSD = dyn_cast<VarTemplateSpecializationDecl>(DiagD)) {
           if (const ASTTemplateArgumentListInfo *ASTTAL =
-                  VTSD->getTemplateArgsInfo())
+                  VTSD->getTemplateArgsAsWritten())
             DiagRange.setEnd(ASTTAL->RAngleLoc);
         }
         if (DiagD->isReferenced()) {
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 5c72270ff150..b268d7c405df 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -5166,7 +5166,8 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
         VarTemplatePartialSpecializationDecl::Create(
             Context, VarTemplate->getDeclContext(), TemplateKWLoc,
             TemplateNameLoc, TemplateParams, VarTemplate, DI->getType(), DI, SC,
-            CanonicalConverted, TemplateArgs);
+            CanonicalConverted);
+    Partial->setTemplateArgsAsWritten(TemplateArgs);
 
     if (!PrevPartial)
       VarTemplate->AddPartialSpecialization(Partial, InsertPos);
@@ -5184,7 +5185,7 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
     Specialization = VarTemplateSpecializationDecl::Create(
         Context, VarTemplate->getDeclContext(), TemplateKWLoc, TemplateNameLoc,
         VarTemplate, DI->getType(), DI, SC, CanonicalConverted);
-    Specialization->setTemplateArgsInfo(TemplateArgs);
+    Specialization->setTemplateArgsAsWritten(TemplateArgs);
 
     if (!PrevDecl)
       VarTemplate->AddSpecialization(Specialization, InsertPos);
@@ -5219,7 +5220,6 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
     }
   }
 
-  Specialization->setTemplateKeywordLoc(TemplateKWLoc);
   Specialization->setLexicalDeclContext(CurContext);
 
   // Add the specialization into its lexical context, so that it can
@@ -9489,7 +9489,8 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
         ClassTemplatePartialSpecializationDecl::Create(
             Context, Kind, ClassTemplate->getDeclContext(), KWLoc,
             TemplateNameLoc, TemplateParams, ClassTemplate, CanonicalConverted,
-            TemplateArgs, CanonType, PrevPartial);
+            CanonType, PrevPartial);
+    Partial->setTemplateArgsAsWritten(TemplateArgs);
     SetNestedNameSpecifier(*this, Partial, SS);
     if (TemplateParameterLists.size() > 1 && SS.isSet()) {
       Partial->setTemplateParameterListsInfo(
@@ -9512,6 +9513,7 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     Specialization = ClassTemplateSpecializationDecl::Create(
         Context, Kind, ClassTemplate->getDeclContext(), KWLoc, TemplateNameLoc,
         ClassTemplate, CanonicalConverted, PrevDecl);
+    Specialization->setTemplateArgsAsWritten(TemplateArgs);
     SetNestedNameSpecifier(*this, Specialization, SS);
     if (TemplateParameterLists.size() > 0) {
       Specialization->setTemplateParameterListsInfo(Context,
@@ -9595,21 +9597,6 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
       << (isPartialSpecialization? 1 : 0)
       << FixItHint::CreateRemoval(ModulePrivateLoc);
 
-  // Build the fully-sugared type for this class template
-  // specialization as the user wrote in the specialization
-  // itself. This means that we'll pretty-print the type retrieved
-  // from the specialization's declaration the way that the user
-  // actually wrote the specialization, rather than formatting the
-  // name based on the "canonical" representation used to store the
-  // template arguments in the specialization.
-  TypeSourceInfo *WrittenTy
-    = Context.getTemplateSpecializationTypeInfo(Name, TemplateNameLoc,
-                                                TemplateArgs, CanonType);
-  if (TUK != TUK_Friend) {
-    Specialization->setTypeAsWritten(WrittenTy);
-    Specialization->setTemplateKeywordLoc(TemplateKWLoc);
-  }
-
   // C++ [temp.expl.spec]p9:
   //   A template explicit specialization is in the scope of the
   //   namespace in which the template was defined.
@@ -9625,6 +9612,15 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     Specialization->startDefinition();
 
   if (TUK == TUK_Friend) {
+    // Build the fully-sugared type for this class template
+    // specialization as the user wrote in the specialization
+    // itself. This means that we'll pretty-print the type retrieved
+    // from the specialization's declaration the way that the user
+    // actually wrote the specialization, rather than formatting the
+    // name based on the "canonical" representation used to store the
+    // template arguments in the specialization.
+    TypeSourceInfo *WrittenTy = Context.getTemplateSpecializationTypeInfo(
+        Name, TemplateNameLoc, TemplateArgs, CanonType);
     FriendDecl *Friend = FriendDecl::Create(Context, CurContext,
                                             TemplateNameLoc,
                                             WrittenTy,
@@ -10830,21 +10826,10 @@ DeclResult Sema::ActOnExplicitInstantiation(
     }
   }
 
-  // Build the fully-sugared type for this explicit instantiation as
-  // the user wrote in the explicit instantiation itself. This means
-  // that we'll pretty-print the type retrieved from the
-  // specialization's declaration the way that the user actually wrote
-  // the explicit instantiation, rather than formatting the name based
-  // on the "canonical" representation used to store the template
-  // arguments in the specialization.
-  TypeSourceInfo *WrittenTy
-    = Context.getTemplateSpecializationTypeInfo(Name, TemplateNameLoc,
-                                                TemplateArgs,
-                                  Context.getTypeDeclType(Specialization));
-  Specialization->setTypeAsWritten(WrittenTy);
+  Specialization->setTemplateArgsAsWritten(TemplateArgs);
 
   // Set source locations for keywords.
-  Specialization->setExternLoc(ExternLoc);
+  Specialization->setExternKeywordLoc(ExternLoc);
   Specialization->setTemplateKeywordLoc(TemplateLoc);
   Specialization->setBraceRange(SourceRange());
 
@@ -11257,6 +11242,11 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     if (!HasNoEffect) {
       // Instantiate static data member or variable template.
       Prev->setTemplateSpecializationKind(TSK, D.getIdentifierLoc());
+      if (auto *VTSD = dyn_cast<VarTemplatePartialSpecializationDecl>(Prev)) {
+        VTSD->setExternKeywordLoc(ExternLoc);
+        VTSD->setTemplateKeywordLoc(TemplateLoc);
+      }
+
       // Merge attributes.
       ProcessDeclAttributeList(S, Prev, D.getDeclSpec().getAttributes());
       if (PrevTemplate)
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index d544cfac55ba..5315b143215e 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -3858,15 +3858,16 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
 
   // Substitute into the template arguments of the class template explicit
   // specialization.
-  TemplateSpecializationTypeLoc Loc = D->getTypeAsWritten()->getTypeLoc().
-                                        castAs<TemplateSpecializationTypeLoc>();
-  TemplateArgumentListInfo InstTemplateArgs(Loc.getLAngleLoc(),
-                                            Loc.getRAngleLoc());
-  SmallVector<TemplateArgumentLoc, 4> ArgLocs;
-  for (unsigned I = 0; I != Loc.getNumArgs(); ++I)
-    ArgLocs.push_back(Loc.getArgLoc(I));
-  if (SemaRef.SubstTemplateArguments(ArgLocs, TemplateArgs, InstTemplateArgs))
-    return nullptr;
+  TemplateArgumentListInfo InstTemplateArgs;
+  if (const ASTTemplateArgumentListInfo *TemplateArgsInfo =
+          D->getTemplateArgsAsWritten()) {
+    InstTemplateArgs.setLAngleLoc(TemplateArgsInfo->getLAngleLoc());
+    InstTemplateArgs.setRAngleLoc(TemplateArgsInfo->getRAngleLoc());
+
+    if (SemaRef.SubstTemplateArguments(TemplateArgsInfo->arguments(),
+                                       TemplateArgs, InstTemplateArgs))
+      return nullptr;
+  }
 
   // Check that the template argument list is well-formed for this
   // class template.
@@ -3920,6 +3921,7 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
       ClassTemplateSpecializationDecl::Create(
           SemaRef.Context, D->getTagKind(), Owner, D->getBeginLoc(),
           D->getLocation(), InstClassTemplate, CanonicalConverted, PrevDecl);
+  InstD->setTemplateArgsAsWritten(InstTemplateArgs);
 
   // Add this partial specialization to the set of class template partial
   // specializations.
@@ -3936,22 +3938,10 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
       TemplateName(InstClassTemplate), CanonicalConverted,
       SemaRef.Context.getRecordType(InstD));
 
-  // Build the fully-sugared type for this class template
-  // specialization as the user wrote in the specialization
-  // itself. This means that we'll pretty-print the type retrieved
-  // from the specialization's declaration the way that the user
-  // actually wrote the specialization, rather than formatting the
-  // name based on the "canonical" representation used to store the
-  // template arguments in the specialization.
-  TypeSourceInfo *WrittenTy = SemaRef.Context.getTemplateSpecializationTypeInfo(
-      TemplateName(InstClassTemplate), D->getLocation(), InstTemplateArgs,
-      CanonType);
-
   InstD->setAccess(D->getAccess());
   InstD->setInstantiationOfMemberClass(D, TSK_ImplicitInstantiation);
   InstD->setSpecializationKind(D->getSpecializationKind());
-  InstD->setTypeAsWritten(WrittenTy);
-  InstD->setExternLoc(D->getExternLoc());
+  InstD->setExternKeywordLoc(D->getExternKeywordLoc());
   InstD->setTemplateKeywordLoc(D->getTemplateKeywordLoc());
 
   Owner->addDecl(InstD);
@@ -3985,7 +3975,7 @@ Decl *TemplateDeclInstantiator::VisitVarTemplateSpecializationDecl(
 
   // Substitute the current template arguments.
   if (const ASTTemplateArgumentListInfo *TemplateArgsInfo =
-          D->getTemplateArgsInfo()) {
+          D->getTemplateArgsAsWritten()) {
     VarTemplateArgsInfo.setLAngleLoc(TemplateArgsInfo->getLAngleLoc());
     VarTemplateArgsInfo.setRAngleLoc(TemplateArgsInfo->getRAngleLoc());
 
@@ -4043,7 +4033,7 @@ Decl *TemplateDeclInstantiator::VisitVarTemplateSpecializationDecl(
   VarTemplateSpecializationDecl *Var = VarTemplateSpecializationDecl::Create(
       SemaRef.Context, Owner, D->getInnerLocStart(), D->getLocation(),
       VarTemplate, DI->getType(), DI, D->getStorageClass(), Converted);
-  Var->setTemplateArgsInfo(TemplateArgsInfo);
+  Var->setTemplateArgsAsWritten(TemplateArgsInfo);
   if (!PrevDecl) {
     void *InsertPos = nullptr;
     VarTemplate->findSpecialization(Converted, InsertPos);
@@ -4285,19 +4275,21 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
   QualType CanonType = SemaRef.Context.getTemplateSpecializationType(
       TemplateName(ClassTemplate), CanonicalConverted);
 
-  // Build the fully-sugared type for this class template
-  // specialization as the user wrote in the specialization
-  // itself. This means that we'll pretty-print the type retrieved
-  // from the specialization's declaration the way that the user
-  // actually wrote the specialization, rather than formatting the
-  // name based on the "canonical" representation used to store the
-  // template arguments in the specialization.
-  TypeSourceInfo *WrittenTy
-    = SemaRef.Context.getTemplateSpecializationTypeInfo(
-                                                    TemplateName(ClassTemplate),
-                                                    PartialSpec->getLocation(),
-                                                    InstTemplateArgs,
-                                                    CanonType);
+  // Create the class template partial specialization declaration.
+  ClassTemplatePartialSpecializationDecl *InstPartialSpec =
+      ClassTemplatePartialSpecializationDecl::Create(
+          SemaRef.Context, PartialSpec->getTagKind(), Owner,
+          PartialSpec->getBeginLoc(), PartialSpec->getLocation(), InstParams,
+          ClassTemplate, CanonicalConverted, CanonType,
+          /*PrevDecl=*/nullptr);
+
+  InstPartialSpec->setTemplateArgsAsWritten(InstTemplateArgs);
+
+  // Substitute the nested name specifier, if any.
+  if (SubstQualifier(PartialSpec, InstPartialSpec))
+    return nullptr;
+
+  InstPartialSpec->setInstantiatedFromMember(PartialSpec);
 
   if (PrevDecl) {
     // We've already seen a partial specialization with the same template
@@ -4315,28 +4307,14 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
     //
     //   Outer<int, int> outer; // error: the partial specializations of Inner
     //                          // have the same signature.
-    SemaRef.Diag(PartialSpec->getLocation(), diag::err_partial_spec_redeclared)
-      << WrittenTy->getType();
+    SemaRef.Diag(InstPartialSpec->getLocation(),
+                 diag::err_partial_spec_redeclared)
+        << InstPartialSpec;
     SemaRef.Diag(PrevDecl->getLocation(), diag::note_prev_partial_spec_here)
       << SemaRef.Context.getTypeDeclType(PrevDecl);
     return nullptr;
   }
 
-
-  // Create the class template partial specialization declaration.
-  ClassTemplatePartialSpecializationDecl *InstPartialSpec =
-      ClassTemplatePartialSpecializationDecl::Create(
-          SemaRef.Context, PartialSpec->getTagKind(), Owner,
-          PartialSpec->getBeginLoc(), PartialSpec->getLocation(), InstParams,
-          ClassTemplate, CanonicalConverted, InstTemplateArgs, CanonType,
-          nullptr);
-  // Substitute the nested name specifier, if any.
-  if (SubstQualifier(PartialSpec, InstPartialSpec))
-    return nullptr;
-
-  InstPartialSpec->setInstantiatedFromMember(PartialSpec);
-  InstPartialSpec->setTypeAsWritten(WrittenTy);
-
   // Check the completed partial specialization.
   SemaRef.CheckTemplatePartialSpecialization(InstPartialSpec);
 
@@ -4405,46 +4383,6 @@ TemplateDeclInstantiator::InstantiateVarTemplatePartialSpecialization(
       VarTemplate->findPartialSpecialization(CanonicalConverted, InstParams,
                                              InsertPos);
 
-  // Build the canonical type that describes the converted template
-  // arguments of the variable template partial specialization.
-  QualType CanonType = SemaRef.Context.getTemplateSpecializationType(
-      TemplateName(VarTemplate), CanonicalConverted);
-
-  // Build the fully-sugared type for this variable template
-  // specialization as the user wrote in the specialization
-  // itself. This means that we'll pretty-print the type retrieved
-  // from the specialization's declaration the way that the user
-  // actually wrote the specialization, rather than formatting the
-  // name based on the "canonical" representation used to store the
-  // template arguments in the specialization.
-  TypeSourceInfo *WrittenTy = SemaRef.Context.getTemplateSpecializationTypeInfo(
-      TemplateName(VarTemplate), PartialSpec->getLocation(), InstTemplateArgs,
-      CanonType);
-
-  if (PrevDecl) {
-    // We've already seen a partial specialization with the same template
-    // parameters and template arguments. This can happen, for example, when
-    // substituting the outer template arguments ends up causing two
-    // variable template partial specializations of a member variable template
-    // to have identical forms, e.g.,
-    //
-    //   template<typename T, typename U>
-    //   struct Outer {
-    //     template<typename X, typename Y> pair<X,Y> p;
-    //     template<typename Y> pair<T, Y> p;
-    //     template<typename Y> pair<U, Y> p;
-    //   };
-    //
-    //   Outer<int, int> outer; // error: the partial specializations of Inner
-    //                          // have the same signature.
-    SemaRef.Diag(PartialSpec->getLocation(),
-                 diag::err_var_partial_spec_redeclared)
-        << WrittenTy->getType();
-    SemaRef.Diag(PrevDecl->getLocation(),
-                 diag::note_var_prev_partial_spec_here);
-    return nullptr;
-  }
-
   // Do substitution on the type of the declaration
   TypeSourceInfo *DI = SemaRef.SubstType(
       PartialSpec->getTypeSourceInfo(), TemplateArgs,
@@ -4464,16 +4402,39 @@ TemplateDeclInstantiator::InstantiateVarTemplatePartialSpecialization(
       VarTemplatePartialSpecializationDecl::Create(
           SemaRef.Context, Owner, PartialSpec->getInnerLocStart(),
           PartialSpec->getLocation(), InstParams, VarTemplate, DI->getType(),
-          DI, PartialSpec->getStorageClass(), CanonicalConverted,
-          InstTemplateArgs);
+          DI, PartialSpec->getStorageClass(), CanonicalConverted);
+
+  InstPartialSpec->setTemplateArgsAsWritten(InstTemplateArgs);
 
   // Substitute the nested name specifier, if any.
   if (SubstQualifier(PartialSpec, InstPartialSpec))
     return nullptr;
 
   InstPartialSpec->setInstantiatedFromMember(PartialSpec);
-  InstPartialSpec->setTypeAsWritten(WrittenTy);
 
+  if (PrevDecl) {
+    // We've already seen a partial specialization with the same template
+    // parameters and template arguments. This can happen, for example, when
+    // substituting the outer template arguments ends up causing two
+    // variable template partial specializations of a member variable template
+    // to have identical forms, e.g.,
+    //
+    //   template<typename T, typename U>
+    //   struct Outer {
+    //     template<typename X, typename Y> pair<X,Y> p;
+    //     template<typename Y> pair<T, Y> p;
+    //     template<typename Y> pair<U, Y> p;
+    //   };
+    //
+    //   Outer<int, int> outer; // error: the partial specializations of Inner
+    //                          // have the same signature.
+    SemaRef.Diag(PartialSpec->getLocation(),
+                 diag::err_var_partial_spec_redeclared)
+        << InstPartialSpec;
+    SemaRef.Diag(PrevDecl->getLocation(),
+                 diag::note_var_prev_partial_spec_here);
+    return nullptr;
+  }
   // Check the completed partial specialization.
   SemaRef.CheckTemplatePartialSpecialization(InstPartialSpec);
 
@@ -5735,7 +5696,7 @@ void Sema::InstantiateVariableDefinition(SourceLocation PointOfInstantiation,
 
     TemplateArgumentListInfo TemplateArgInfo;
     if (const ASTTemplateArgumentListInfo *ArgInfo =
-            VarSpec->getTemplateArgsInfo()) {
+            VarSpec->getTemplateArgsAsWritten()) {
       TemplateArgInfo.setLAngleLoc(ArgInfo->getLAngleLoc());
       TemplateArgInfo.setRAngleLoc(ArgInfo->getRAngleLoc());
       for (const TemplateArgumentLoc &Arg : ArgInfo->arguments())
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 089ede4f4926..0c647086e304 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -2548,16 +2548,17 @@ ASTDeclReader::VisitClassTemplateSpecializationDeclImpl(
     }
   }
 
-  // Explicit info.
-  if (TypeSourceInfo *TyInfo = readTypeSourceInfo()) {
-    auto *ExplicitInfo =
-        new (C) ClassTemplateSpecializationDecl::ExplicitSpecializationInfo;
-    ExplicitInfo->TypeAsWritten = TyInfo;
-    ExplicitInfo->ExternLoc = readSourceLocation();
+  // extern/template keyword locations for explicit instantiations
+  if (Record.readBool()) {
+    auto *ExplicitInfo = new (C) ExplicitInstantiationInfo;
+    ExplicitInfo->ExternKeywordLoc = readSourceLocation();
     ExplicitInfo->TemplateKeywordLoc = readSourceLocation();
     D->ExplicitInfo = ExplicitInfo;
   }
 
+  if (Record.readBool())
+    D->setTemplateArgsAsWritten(Record.readASTTemplateArgumentListInfo());
+
   return Redecl;
 }
 
@@ -2567,7 +2568,6 @@ void ASTDeclReader::VisitClassTemplatePartialSpecializationDecl(
   // need them for profiling
   TemplateParameterList *Params = Record.readTemplateParameterList();
   D->TemplateParams = Params;
-  D->ArgsAsWritten = Record.readASTTemplateArgumentListInfo();
 
   RedeclarableResult Redecl = VisitClassTemplateSpecializationDeclImpl(D);
 
@@ -2617,16 +2617,17 @@ ASTDeclReader::VisitVarTemplateSpecializationDeclImpl(
     }
   }
 
-  // Explicit info.
-  if (TypeSourceInfo *TyInfo = readTypeSourceInfo()) {
-    auto *ExplicitInfo =
-        new (C) VarTemplateSpecializationDecl::ExplicitSpecializationInfo;
-    ExplicitInfo->TypeAsWritten = TyInfo;
-    ExplicitInfo->ExternLoc = readSourceLocation();
+  // extern/template keyword locations for explicit instantiations
+  if (Record.readBool()) {
+    auto *ExplicitInfo = new (C) ExplicitInstantiationInfo;
+    ExplicitInfo->ExternKeywordLoc = readSourceLocation();
     ExplicitInfo->TemplateKeywordLoc = readSourceLocation();
     D->ExplicitInfo = ExplicitInfo;
   }
 
+  if (Record.readBool())
+    D->setTemplateArgsAsWritten(Record.readASTTemplateArgumentListInfo());
+
   SmallVector<TemplateArgument, 8> TemplArgs;
   Record.readTemplateArgumentList(TemplArgs, /*Canonicalize*/ true);
   D->TemplateArgs = TemplateArgumentList::CreateCopy(C, TemplArgs);
@@ -2666,7 +2667,6 @@ void ASTDeclReader::VisitVarTemplatePartialSpecializationDecl(
     VarTemplatePartialSpecializationDecl *D) {
   TemplateParameterList *Params = Record.readTemplateParameterList();
   D->TemplateParams = Params;
-  D->ArgsAsWritten = Record.readASTTemplateArgumentListInfo();
 
   RedeclarableResult Redecl = VisitVarTemplateSpecializationDeclImpl(D);
 
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 6201d284f0e0..c2f1d1b44241 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -1765,20 +1765,28 @@ void ASTDeclWriter::VisitClassTemplateSpecializationDecl(
     Record.AddDeclRef(D->getSpecializedTemplate()->getCanonicalDecl());
   }
 
-  // Explicit info.
-  Record.AddTypeSourceInfo(D->getTypeAsWritten());
-  if (D->getTypeAsWritten()) {
-    Record.AddSourceLocation(D->getExternLoc());
+  bool ExplicitInstantiation =
+      D->getTemplateSpecializationKind() ==
+          TSK_ExplicitInstantiationDeclaration ||
+      D->getTemplateSpecializationKind() == TSK_ExplicitInstantiationDefinition;
+  Record.push_back(ExplicitInstantiation);
+  if (ExplicitInstantiation) {
+    Record.AddSourceLocation(D->getExternKeywordLoc());
     Record.AddSourceLocation(D->getTemplateKeywordLoc());
   }
 
+  const ASTTemplateArgumentListInfo *ArgsWritten =
+      D->getTemplateArgsAsWritten();
+  Record.push_back(!!ArgsWritten);
+  if (ArgsWritten)
+    Record.AddASTTemplateArgumentListInfo(ArgsWritten);
+
   Code = serialization::DECL_CLASS_TEMPLATE_SPECIALIZATION;
 }
 
 void ASTDeclWriter::VisitClassTemplatePartialSpecializationDecl(
                                     ClassTemplatePartialSpecializationDecl *D) {
   Record.AddTemplateParameterList(D->getTemplateParameters());
-  Record.AddASTTemplateArgumentListInfo(D->getTemplateArgsAsWritten());
 
   VisitClassTemplateSpecializationDecl(D);
 
@@ -1812,13 +1820,22 @@ void ASTDeclWriter::VisitVarTemplateSpecializationDecl(
     Record.AddTemplateArgumentList(&D->getTemplateInstantiationArgs());
   }
 
-  // Explicit info.
-  Record.AddTypeSourceInfo(D->getTypeAsWritten());
-  if (D->getTypeAsWritten()) {
-    Record.AddSourceLocation(D->getExternLoc());
+  bool ExplicitInstantiation =
+      D->getTemplateSpecializationKind() ==
+          TSK_ExplicitInstantiationDeclaration ||
+      D->getTemplateSpecializationKind() == TSK_ExplicitInstantiationDefinition;
+  Record.push_back(ExplicitInstantiation);
+  if (ExplicitInstantiation) {
+    Record.AddSourceLocation(D->getExternKeywordLoc());
     Record.AddSourceLocation(D->getTemplateKeywordLoc());
   }
 
+  const ASTTemplateArgumentListInfo *ArgsWritten =
+      D->getTemplateArgsAsWritten();
+  Record.push_back(!!ArgsWritten);
+  if (ArgsWritten)
+    Record.AddASTTemplateArgumentListInfo(ArgsWritten);
+
   Record.AddTemplateArgumentList(&D->getTemplateArgs());
   Record.AddSourceLocation(D->getPointOfInstantiation());
   Record.push_back(D->getSpecializationKind());
@@ -1839,7 +1856,6 @@ void ASTDeclWriter::VisitVarTemplateSpecializationDecl(
 void ASTDeclWriter::VisitVarTemplatePartialSpecializationDecl(
     VarTemplatePartialSpecializationDecl *D) {
   Record.AddTemplateParameterList(D->getTemplateParameters());
-  Record.AddASTTemplateArgumentListInfo(D->getTemplateArgsAsWritten());
 
   VisitVarTemplateSpecializationDecl(D);
 
diff --git a/clang/lib/Tooling/Syntax/BuildTree.cpp b/clang/lib/Tooling/Syntax/BuildTree.cpp
index cd0261989495..3e50d67f4d6e 100644
--- a/clang/lib/Tooling/Syntax/BuildTree.cpp
+++ b/clang/lib/Tooling/Syntax/BuildTree.cpp
@@ -735,7 +735,8 @@ public:
     auto *Declaration =
         cast<syntax::SimpleDeclaration>(handleFreeStandingTagDecl(C));
     foldExplicitTemplateInstantiation(
-        Builder.getTemplateRange(C), Builder.findToken(C->getExternLoc()),
+        Builder.getTemplateRange(C),
+        Builder.findToken(C->getExternKeywordLoc()),
         Builder.findToken(C->getTemplateKeywordLoc()), Declaration, C);
     return true;
   }
diff --git a/clang/test/AST/ast-dump-template-decls.cpp b/clang/test/AST/ast-dump-template-decls.cpp
index 142bc9e6ad9a..37f6d8a0472d 100644
--- a/clang/test/AST/ast-dump-template-decls.cpp
+++ b/clang/test/AST/ast-dump-template-decls.cpp
@@ -1,12 +1,12 @@
 // Test without serialization:
 // RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown -ast-dump %s \
-// RUN: | FileCheck -strict-whitespace %s --check-prefix=DIRECT
+// RUN: | FileCheck -strict-whitespace %s
 //
 // Test with serialization:
 // RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -x c++ -std=c++17 -triple x86_64-unknown-unknown -include-pch %t -ast-dump-all /dev/null \
 // RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
-// RUN: | FileCheck --strict-whitespace %s --check-prefix=SERIALIZED
+// RUN: | FileCheck --strict-whitespace %s
 
 template <typename Ty>
 // CHECK: FunctionTemplateDecl 0x{{[^ ]*}} <{{.*}}:1, line:[[@LINE+2]]:10> col:6 a
@@ -189,15 +189,13 @@ T unTempl = 1;
 
 template<>
 int unTempl<int>;
-// FIXME (#61680) - serializing and loading AST should not affect reported source range
-// DIRECT:     VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:16> col:5 unTempl 'int'
-// SERIALIZED: VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:5> col:5 unTempl 'int'
+// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:16> col:5 unTempl 'int'
 // CHECK-NEXT: `-TemplateArgument type 'int'
 // CHECK-NEXT: `-BuiltinType 0x{{[^ ]*}} 'int'
 
 template<>
 float unTempl<float> = 1;
-// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:24> col:7 unTempl 'float' cinit
+// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:24> col:7 unTempl 'float'
 // CHECK-NEXT: |-TemplateArgument type 'float'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'float'
 // CHECK-NEXT: `-ImplicitCastExpr 0x{{[^ ]*}} <col:24> 'float' <IntegralToFloating>
@@ -222,7 +220,7 @@ int binTempl<int, U>;
 
 template<class U>
 float binTempl<float, U> = 1;
-// CHECK:      VarTemplatePartialSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:24> col:7 binTempl 'float' cinit
+// CHECK:      VarTemplatePartialSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:28> col:7 binTempl 'float'
 // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:10, col:16> col:16 referenced class depth 0 index 0 U
 // CHECK-NEXT: |-TemplateArgument type 'float'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'float'
@@ -233,9 +231,7 @@ float binTempl<float, U> = 1;
 
 template<>
 int binTempl<int, int>;
-// FIXME (#61680) - serializing and loading AST should not affect reported source range
-// DIRECT:     VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:22> col:5 binTempl 'int'
-// SERIALIZED: VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:5> col:5 binTempl 'int'
+// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:22> col:5 binTempl 'int'
 // CHECK-NEXT: |-TemplateArgument type 'int'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'int'
 // CHECK-NEXT: `-TemplateArgument type 'int'
@@ -243,7 +239,7 @@ int binTempl<int, int>;
 
 template<>
 float binTempl<float, float> = 1;
-// CHECK:     VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:32> col:7 binTempl 'float' cinit
+// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:32> col:7 binTempl 'float'
 // CHECK-NEXT: |-TemplateArgument type 'float'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'float'
 // CHECK-NEXT: |-TemplateArgument type 'float'
diff --git a/clang/test/Index/Core/index-source.cpp b/clang/test/Index/Core/index-source.cpp
index 8f9fbc4c8d29..043e616a1d36 100644
--- a/clang/test/Index/Core/index-source.cpp
+++ b/clang/test/Index/Core/index-source.cpp
@@ -285,20 +285,17 @@ template<>
 class SpecializationDecl<int>;
 // CHECK: [[@LINE-1]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Decl,RelSpecialization | rel: 1
 // CHECK-NEXT: RelSpecialization | SpecializationDecl | c:@ST>1#T@SpecializationDecl
-// CHECK: [[@LINE-3]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Ref | rel: 0
 
 template<>
 class SpecializationDecl<int> { };
 // CHECK: [[@LINE-1]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Def,RelSpecialization | rel: 1
 // CHECK-NEXT: RelSpecialization | SpecializationDecl | c:@ST>1#T@SpecializationDecl
-// CHECK-NEXT: [[@LINE-3]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Ref | rel: 0
 
 template<typename T>
 class PartialSpecilizationClass<Cls, T>;
 // CHECK: [[@LINE-1]]:7 | class(Gen,TPS)/C++ | PartialSpecilizationClass | c:@SP>1#T@PartialSpecilizationClass>#$@S@Cls#t0.0 | <no-cgname> | Decl,RelSpecialization | rel: 1
 // CHECK-NEXT: RelSpecialization | PartialSpecilizationClass | c:@ST>2#T#T@PartialSpecilizationClass
-// CHECK: [[@LINE-3]]:7 | class(Gen)/C++ | PartialSpecilizationClass | c:@ST>2#T#T@PartialSpecilizationClass | <no-cgname> | Ref | rel: 0
-// CHECK-NEXT: [[@LINE-4]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
+// CHECK-NEXT: [[@LINE-3]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
 
 template<>
 class PartialSpecilizationClass<Cls, Cls> : Cls { };
@@ -306,9 +303,10 @@ class PartialSpecilizationClass<Cls, Cls> : Cls { };
 // CHECK-NEXT: RelSpecialization | PartialSpecilizationClass | c:@ST>2#T#T@PartialSpecilizationClass
 // CHECK-NEXT: [[@LINE-3]]:45 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelBase,RelCont | rel: 1
 // CHECK-NEXT: RelBase,RelCont | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_
-// CHECK-NEXT: [[@LINE-5]]:7 | class(Gen,TS)/C++ | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_ | <no-cgname> | Ref | rel: 0
-// CHECK-NEXT: [[@LINE-6]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
-// CHECK-NEXT: [[@LINE-7]]:38 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
+// CHECK-NEXT: [[@LINE-5]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK-NEXT: RelCont | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_
+// CHECK-NEXT: [[@LINE-7]]:38 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK-NEXT: RelCont | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_
 
 template<typename T, int x>
 void functionSp() { }
@@ -332,10 +330,14 @@ class ClassWithCorrectSpecialization { };
 
 template<>
 class ClassWithCorrectSpecialization<SpecializationDecl<Cls>, Record::C> { };
-// CHECK: [[@LINE-1]]:38 | class(Gen)/C++ | SpecializationDecl | c:@ST>1#T@SpecializationDecl | <no-cgname> | Ref | rel: 0
-// CHECK: [[@LINE-2]]:57 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
-// CHECK: [[@LINE-3]]:71 | static-property/C++ | C | c:@S@Record@C | __ZN6Record1CE | Ref,Read | rel: 0
-// CHECK: [[@LINE-4]]:63 | struct/C++ | Record | c:@S@Record | <no-cgname> | Ref | rel: 0
+// CHECK: [[@LINE-1]]:38 | class(Gen)/C++ | SpecializationDecl | c:@ST>1#T@SpecializationDecl | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
+// CHECK-NEXT: [[@LINE-3]]:57 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
+// CHECK-NEXT: [[@LINE-5]]:71 | static-property/C++ | C | c:@S@Record@C | __ZN6Record1CE | Ref,Read,RelCont | rel: 1
+// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
+// CHECK-NEXT: [[@LINE-7]]:63 | struct/C++ | Record | c:@S@Record | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
 
 namespace ns {
 // CHECK: [[@LINE-1]]:11 | namespace/C++ | ns | c:@N@ns | <no-cgname> | Decl | rel: 0
diff --git a/clang/test/Index/index-refs.cpp b/clang/test/Index/index-refs.cpp
index 0e613e48522b..14946849777d 100644
--- a/clang/test/Index/index-refs.cpp
+++ b/clang/test/Index/index-refs.cpp
@@ -108,7 +108,6 @@ int ginitlist[] = {EnumVal};
 // CHECK:      [indexDeclaration]: kind: c++-class-template | name: TS | {{.*}} | loc: 47:8
 // CHECK-NEXT: [indexDeclaration]: kind: struct-template-partial-spec | name: TS | USR: c:@SP>1#T@TS>#t0.0#I | {{.*}} | loc: 50:8
 // CHECK-NEXT: [indexDeclaration]: kind: typedef | name: MyInt | USR: c:index-refs.cpp@SP>1#T@TS>#t0.0#I@T@MyInt | {{.*}} | loc: 51:15 | semantic-container: [TS:50:8] | lexical-container: [TS:50:8]
-// CHECK-NEXT: [indexEntityReference]: kind: c++-class-template | name: TS | USR: c:@ST>2#T#T@TS | lang: C++ | cursor: TemplateRef=TS:47:8 | loc: 50:8 | <parent>:: <<NULL>> | container: [TU] | refkind: direct | role: ref
 /* when indexing implicit instantiations
   [indexDeclaration]: kind: struct-template-spec | name: TS | USR: c:@S@TS>#I | {{.*}} | loc: 50:8
   [indexDeclaration]: kind: typedef | name: MyInt | USR: c:index-refs.cpp@593@S@TS>#I@T@MyInt | {{.*}} | loc: 51:15 | semantic-container: [TS:50:8] | lexical-container: [TS:50:8]
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index b845a381d63b..60241afd8776 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -743,14 +743,10 @@ bool CursorVisitor::VisitClassTemplateSpecializationDecl(
   }
 
   // Visit the template arguments used in the specialization.
-  if (TypeSourceInfo *SpecType = D->getTypeAsWritten()) {
-    TypeLoc TL = SpecType->getTypeLoc();
-    if (TemplateSpecializationTypeLoc TSTLoc =
-            TL.getAs<TemplateSpecializationTypeLoc>()) {
-      for (unsigned I = 0, N = TSTLoc.getNumArgs(); I != N; ++I)
-        if (VisitTemplateArgumentLoc(TSTLoc.getArgLoc(I)))
-          return true;
-    }
+  if (const auto *ArgsWritten = D->getTemplateArgsAsWritten()) {
+    for (const TemplateArgumentLoc &Arg : ArgsWritten->arguments())
+      if (VisitTemplateArgumentLoc(Arg))
+        return true;
   }
 
   return ShouldVisitBody && VisitCXXRecordDecl(D);
@@ -5659,16 +5655,19 @@ CXString clang_getCursorDisplayName(CXCursor C) {
 
   if (const ClassTemplateSpecializationDecl *ClassSpec =
           dyn_cast<ClassTemplateSpecializationDecl>(D)) {
-    // If the type was explicitly written, use that.
-    if (TypeSourceInfo *TSInfo = ClassSpec->getTypeAsWritten())
-      return cxstring::createDup(TSInfo->getType().getAsString(Policy));
-
     SmallString<128> Str;
     llvm::raw_svector_ostream OS(Str);
     OS << *ClassSpec;
-    printTemplateArgumentList(
-        OS, ClassSpec->getTemplateArgs().asArray(), Policy,
-        ClassSpec->getSpecializedTemplate()->getTemplateParameters());
+    // If the template arguments were written explicitly, use them..
+    if (const auto *ArgsWritten = ClassSpec->getTemplateArgsAsWritten()) {
+      printTemplateArgumentList(
+          OS, ArgsWritten->arguments(), Policy,
+          ClassSpec->getSpecializedTemplate()->getTemplateParameters());
+    } else {
+      printTemplateArgumentList(
+          OS, ClassSpec->getTemplateArgs().asArray(), Policy,
+          ClassSpec->getSpecializedTemplate()->getTemplateParameters());
+    }
     return cxstring::createDup(OS.str());
   }
 
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index b76627cb9be6..65df513d2713 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -2213,18 +2213,6 @@ TEST_P(ASTMatchersTest, ReferenceTypeLocTest_BindsToAnyRvalueReferenceTypeLoc) {
   EXPECT_TRUE(matches("float&& r = 3.0;", matcher));
 }
 
-TEST_P(
-    ASTMatchersTest,
-    TemplateSpecializationTypeLocTest_BindsToTemplateSpecializationExplicitInstantiation) {
-  if (!GetParam().isCXX()) {
-    return;
-  }
-  EXPECT_TRUE(
-      matches("template <typename T> class C {}; template class C<int>;",
-              classTemplateSpecializationDecl(
-                  hasName("C"), hasTypeLoc(templateSpecializationTypeLoc()))));
-}
-
 TEST_P(ASTMatchersTest,
        TemplateSpecializationTypeLocTest_BindsToVarDeclTemplateSpecialization) {
   if (!GetParam().isCXX()) {
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
index f198dc71eb83..af99c73f1945 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
@@ -430,12 +430,6 @@ TEST(HasTypeLoc, MatchesCXXUnresolvedConstructExpr) {
               cxxUnresolvedConstructExpr(hasTypeLoc(loc(asString("T"))))));
 }
 
-TEST(HasTypeLoc, MatchesClassTemplateSpecializationDecl) {
-  EXPECT_TRUE(matches(
-      "template <typename T> class Foo; template <> class Foo<int> {};",
-      classTemplateSpecializationDecl(hasTypeLoc(loc(asString("Foo<int>"))))));
-}
-
 TEST(HasTypeLoc, MatchesCompoundLiteralExpr) {
   EXPECT_TRUE(
       matches("int* x = (int[2]) { 0, 1 };",
@@ -6384,8 +6378,7 @@ TEST(HasAnyTemplateArgumentLoc, BindsToExplicitSpecializationWithIntArgument) {
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(
-              hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))))));
+          hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc,
@@ -6394,8 +6387,7 @@ TEST(HasAnyTemplateArgumentLoc,
       "template<typename T> class A {}; template<> class A<double> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-              hasTypeLoc(loc(asString("double")))))))));
+          hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("double")))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
@@ -6405,24 +6397,20 @@ TEST(HasAnyTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
   )";
   EXPECT_TRUE(
       matches(code, classTemplateSpecializationDecl(
-                        hasName("A"), hasTypeLoc(templateSpecializationTypeLoc(
-                                          hasAnyTemplateArgumentLoc(hasTypeLoc(
-                                              loc(asString("double")))))))));
+                        hasName("A"), hasAnyTemplateArgumentLoc(hasTypeLoc(
+                                          loc(asString("double")))))));
+
   EXPECT_TRUE(matches(
-      code,
-      classTemplateSpecializationDecl(
-          hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(
-              hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))))));
+      code, classTemplateSpecializationDecl(
+                hasName("A"),
+                hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc, DoesNotBindToSpecializationWithIntArgument) {
-  EXPECT_TRUE(notMatches(
-      "template<typename T> class A {}; A<int> a;",
-      classTemplateSpecializationDecl(
-          hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-              hasTypeLoc(loc(asString("double")))))))));
+  EXPECT_TRUE(notMatches("template<typename T> class A {}; A<int> a;",
+                         classTemplateSpecializationDecl(
+                             hasName("A"), hasAnyTemplateArgumentLoc(hasTypeLoc(
+                                               loc(asString("double")))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc,
@@ -6431,8 +6419,7 @@ TEST(HasAnyTemplateArgumentLoc,
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-              hasTypeLoc(loc(asString("double")))))))));
+          hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("double")))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToSpecializationWithIntArgument) {
@@ -6453,13 +6440,21 @@ TEST(HasTemplateArgumentLoc, BindsToSpecializationWithDoubleArgument) {
                               0, hasTypeLoc(loc(asString("double")))))))))));
 }
 
+TEST(HasTemplateArgumentLoc, DoesNotBindToSpecializationWithIntArgument) {
+  EXPECT_TRUE(notMatches(
+      "template<typename T> class A {}; A<int> a;",
+      varDecl(hasName("a"),
+              hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+                  templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+                      0, hasTypeLoc(loc(asString("double")))))))))));
+}
+
 TEST(HasTemplateArgumentLoc, BindsToExplicitSpecializationWithIntArgument) {
   EXPECT_TRUE(matches(
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(
-              hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("int")))))))));
+          hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToExplicitSpecializationWithDoubleArgument) {
@@ -6467,8 +6462,7 @@ TEST(HasTemplateArgumentLoc, BindsToExplicitSpecializationWithDoubleArgument) {
       "template<typename T> class A {}; template<> class A<double> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-              0, hasTypeLoc(loc(asString("double")))))))));
+          hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("double")))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
@@ -6478,23 +6472,12 @@ TEST(HasTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
   )";
   EXPECT_TRUE(matches(
       code, classTemplateSpecializationDecl(
-                hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    0, hasTypeLoc(loc(asString("double")))))))));
+                hasName("A"), hasTemplateArgumentLoc(
+                                  0, hasTypeLoc(loc(asString("double")))))));
   EXPECT_TRUE(matches(
       code, classTemplateSpecializationDecl(
                 hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    1, hasTypeLoc(loc(asString("int")))))))));
-}
-
-TEST(HasTemplateArgumentLoc, DoesNotBindToSpecializationWithIntArgument) {
-  EXPECT_TRUE(notMatches(
-      "template<typename T> class A {}; A<int> a;",
-      classTemplateSpecializationDecl(
-          hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-              0, hasTypeLoc(loc(asString("double")))))))));
+                hasTemplateArgumentLoc(1, hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasTemplateArgumentLoc,
@@ -6503,8 +6486,7 @@ TEST(HasTemplateArgumentLoc,
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-              0, hasTypeLoc(loc(asString("double")))))))));
+          hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("double")))))));
 }
 
 TEST(HasTemplateArgumentLoc,
@@ -6515,14 +6497,12 @@ TEST(HasTemplateArgumentLoc,
   )";
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
-                hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    1, hasTypeLoc(loc(asString("double")))))))));
+                hasName("A"), hasTemplateArgumentLoc(
+                                  1, hasTypeLoc(loc(asString("double")))))));
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
                 hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    0, hasTypeLoc(loc(asString("int")))))))));
+                hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasTemplateArgumentLoc, DoesNotBindWithBadIndex) {
@@ -6532,14 +6512,12 @@ TEST(HasTemplateArgumentLoc, DoesNotBindWithBadIndex) {
   )";
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
-                hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    -1, hasTypeLoc(loc(asString("double")))))))));
+                hasName("A"), hasTemplateArgumentLoc(
+                                  -1, hasTypeLoc(loc(asString("double")))))));
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
-                hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    100, hasTypeLoc(loc(asString("int")))))))));
+                hasName("A"), hasTemplateArgumentLoc(
+                                  100, hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToDeclRefExprWithIntArgument) {
-- 
GitLab


From b1bc1dbea6d0423813bb73d625c6eedc040007ed Mon Sep 17 00:00:00 2001
From: Edwin Vane <revane@google.com>
Date: Tue, 7 May 2024 15:06:51 -0400
Subject: [PATCH 0080/1206] [clang-tidy] Refactor how NamedDecl are renamed
 (#88735)

The handling of renaming failures and multiple usages related to those
failures is currently spread over several functions. Identifying the
failure NamedDecl for a given usage is also duplicated, once when
creating failures and again when identify usages. There are currently
two ways to a failed NamedDecl from a usage: use the canonical decl or
use the overridden method. With new methods about to be added, a cleanup
was in order.

The data flow is simplified as follows:
* The visitor always forwards NamedDecls to addUsage(NamedDecl).
* addUsage(NamedDecl) determines the failed NamedDecl and determines
potential new names based on that failure. Usages are registered using
addUsage(NamingCheckId).
* addUsage(NamingCheckId) is now protected and its single responsibility
is maintaining the integrity of the failure/usage map.
---
 .../bugprone/ReservedIdentifierCheck.cpp      |   5 +-
 .../readability/IdentifierNamingCheck.cpp     |   4 +
 .../utils/RenamerClangTidyCheck.cpp           | 196 ++++++++++--------
 .../clang-tidy/utils/RenamerClangTidyCheck.h  |  14 +-
 4 files changed, 121 insertions(+), 98 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp
index f6714d056518..53956661d57d 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp
@@ -178,8 +178,11 @@ std::optional<RenamerClangTidyCheck::FailureInfo>
 ReservedIdentifierCheck::getDeclFailureInfo(const NamedDecl *Decl,
                                             const SourceManager &) const {
   assert(Decl && Decl->getIdentifier() && !Decl->getName().empty() &&
-         !Decl->isImplicit() &&
          "Decl must be an explicit identifier with a name.");
+  // Implicit identifiers cannot fail.
+  if (Decl->isImplicit())
+    return std::nullopt;
+
   return getFailureInfoImpl(
       Decl->getName(), isa<TranslationUnitDecl>(Decl->getDeclContext()),
       /*IsMacro = */ false, getLangOpts(), Invert, AllowedIdentifiers);
diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index dc30531ebda0..27a12bfc5806 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -1374,6 +1374,10 @@ IdentifierNamingCheck::getFailureInfo(
 std::optional<RenamerClangTidyCheck::FailureInfo>
 IdentifierNamingCheck::getDeclFailureInfo(const NamedDecl *Decl,
                                           const SourceManager &SM) const {
+  // Implicit identifiers cannot be renamed.
+  if (Decl->isImplicit())
+    return std::nullopt;
+
   SourceLocation Loc = Decl->getLocation();
   const FileStyle &FileStyle = getStyleForFile(SM.getFilename(Loc));
   if (!FileStyle.isActive())
diff --git a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
index 962a243ce94d..f5ed61736540 100644
--- a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
@@ -61,6 +61,7 @@ struct DenseMapInfo<clang::tidy::RenamerClangTidyCheck::NamingCheckId> {
 namespace clang::tidy {
 
 namespace {
+
 class NameLookup {
   llvm::PointerIntPair<const NamedDecl *, 1, bool> Data;
 
@@ -78,6 +79,7 @@ public:
   operator bool() const { return !hasMultipleResolutions(); }
   const NamedDecl *operator*() const { return getDecl(); }
 };
+
 } // namespace
 
 static const NamedDecl *findDecl(const RecordDecl &RecDecl,
@@ -91,6 +93,44 @@ static const NamedDecl *findDecl(const RecordDecl &RecDecl,
   return nullptr;
 }
 
+/// Returns the function that \p Method is overridding. If There are none or
+/// multiple overrides it returns nullptr. If the overridden function itself is
+/// overridding then it will recurse up to find the first decl of the function.
+static const CXXMethodDecl *getOverrideMethod(const CXXMethodDecl *Method) {
+  if (Method->size_overridden_methods() != 1)
+    return nullptr;
+
+  while (true) {
+    Method = *Method->begin_overridden_methods();
+    assert(Method && "Overridden method shouldn't be null");
+    unsigned NumOverrides = Method->size_overridden_methods();
+    if (NumOverrides == 0)
+      return Method;
+    if (NumOverrides > 1)
+      return nullptr;
+  }
+}
+
+static bool hasNoName(const NamedDecl *Decl) {
+  return !Decl->getIdentifier() || Decl->getName().empty();
+}
+
+static const NamedDecl *getFailureForNamedDecl(const NamedDecl *ND) {
+  const auto *Canonical = cast<NamedDecl>(ND->getCanonicalDecl());
+  if (Canonical != ND)
+    return Canonical;
+
+  if (const auto *Method = dyn_cast<CXXMethodDecl>(ND)) {
+    if (const CXXMethodDecl *Overridden = getOverrideMethod(Method))
+      Canonical = cast<NamedDecl>(Overridden->getCanonicalDecl());
+
+    if (Canonical != ND)
+      return Canonical;
+  }
+
+  return ND;
+}
+
 /// Returns a decl matching the \p DeclName in \p Parent or one of its base
 /// classes. If \p AggressiveTemplateLookup is `true` then it will check
 /// template dependent base classes as well.
@@ -132,24 +172,6 @@ static NameLookup findDeclInBases(const CXXRecordDecl &Parent,
   return NameLookup(Found); // If nullptr, decl wasn't found.
 }
 
-/// Returns the function that \p Method is overridding. If There are none or
-/// multiple overrides it returns nullptr. If the overridden function itself is
-/// overridding then it will recurse up to find the first decl of the function.
-static const CXXMethodDecl *getOverrideMethod(const CXXMethodDecl *Method) {
-  if (Method->size_overridden_methods() != 1)
-    return nullptr;
-
-  while (true) {
-    Method = *Method->begin_overridden_methods();
-    assert(Method && "Overridden method shouldn't be null");
-    unsigned NumOverrides = Method->size_overridden_methods();
-    if (NumOverrides == 0)
-      return Method;
-    if (NumOverrides > 1)
-      return nullptr;
-  }
-}
-
 namespace {
 
 /// Callback supplies macros to RenamerClangTidyCheck::checkMacro
@@ -192,10 +214,6 @@ public:
       : Check(Check), SM(SM),
         AggressiveDependentMemberLookup(AggressiveDependentMemberLookup) {}
 
-  static bool hasNoName(const NamedDecl *Decl) {
-    return !Decl->getIdentifier() || Decl->getName().empty();
-  }
-
   bool shouldVisitTemplateInstantiations() const { return true; }
 
   bool shouldVisitImplicitCode() const { return false; }
@@ -246,29 +264,10 @@ public:
   }
 
   bool VisitNamedDecl(NamedDecl *Decl) {
-    if (hasNoName(Decl))
-      return true;
-
-    const auto *Canonical = cast<NamedDecl>(Decl->getCanonicalDecl());
-    if (Canonical != Decl) {
-      Check->addUsage(Canonical, Decl->getLocation(), SM);
-      return true;
-    }
-
-    // Fix overridden methods
-    if (const auto *Method = dyn_cast<CXXMethodDecl>(Decl)) {
-      if (const CXXMethodDecl *Overridden = getOverrideMethod(Method)) {
-        Check->addUsage(Overridden, Method->getLocation(), SM);
-        return true; // Don't try to add the actual decl as a Failure.
-      }
-    }
-
-    // Ignore ClassTemplateSpecializationDecl which are creating duplicate
-    // replacements with CXXRecordDecl.
-    if (isa<ClassTemplateSpecializationDecl>(Decl))
-      return true;
-
-    Check->checkNamedDecl(Decl, SM);
+    SourceRange UsageRange =
+        DeclarationNameInfo(Decl->getDeclName(), Decl->getLocation())
+            .getSourceRange();
+    Check->addUsage(Decl, UsageRange, SM);
     return true;
   }
 
@@ -413,82 +412,97 @@ void RenamerClangTidyCheck::registerPPCallbacks(
       std::make_unique<RenamerClangTidyCheckPPCallbacks>(SM, this));
 }
 
-void RenamerClangTidyCheck::addUsage(
-    const RenamerClangTidyCheck::NamingCheckId &Decl, SourceRange Range,
-    const SourceManager &SourceMgr) {
+std::pair<RenamerClangTidyCheck::NamingCheckFailureMap::iterator, bool>
+RenamerClangTidyCheck::addUsage(
+    const RenamerClangTidyCheck::NamingCheckId &FailureId,
+    SourceRange UsageRange, const SourceManager &SourceMgr) {
   // Do nothing if the provided range is invalid.
-  if (Range.isInvalid())
-    return;
+  if (UsageRange.isInvalid())
+    return {NamingCheckFailures.end(), false};
 
-  // If we have a source manager, use it to convert to the spelling location for
-  // performing the fix. This is necessary because macros can map the same
-  // spelling location to different source locations, and we only want to fix
-  // the token once, before it is expanded by the macro.
-  SourceLocation FixLocation = Range.getBegin();
+  // Get the spelling location for performing the fix. This is necessary because
+  // macros can map the same spelling location to different source locations,
+  // and we only want to fix the token once, before it is expanded by the macro.
+  SourceLocation FixLocation = UsageRange.getBegin();
   FixLocation = SourceMgr.getSpellingLoc(FixLocation);
   if (FixLocation.isInvalid())
-    return;
+    return {NamingCheckFailures.end(), false};
+
+  auto EmplaceResult = NamingCheckFailures.try_emplace(FailureId);
+  NamingCheckFailure &Failure = EmplaceResult.first->second;
 
   // Try to insert the identifier location in the Usages map, and bail out if it
   // is already in there
-  RenamerClangTidyCheck::NamingCheckFailure &Failure =
-      NamingCheckFailures[Decl];
   if (!Failure.RawUsageLocs.insert(FixLocation).second)
-    return;
+    return EmplaceResult;
 
-  if (!Failure.shouldFix())
-    return;
+  if (Failure.FixStatus != RenamerClangTidyCheck::ShouldFixStatus::ShouldFix)
+    return EmplaceResult;
 
   if (SourceMgr.isWrittenInScratchSpace(FixLocation))
     Failure.FixStatus = RenamerClangTidyCheck::ShouldFixStatus::InsideMacro;
 
-  if (!utils::rangeCanBeFixed(Range, &SourceMgr))
+  if (!utils::rangeCanBeFixed(UsageRange, &SourceMgr))
     Failure.FixStatus = RenamerClangTidyCheck::ShouldFixStatus::InsideMacro;
+
+  return EmplaceResult;
 }
 
-void RenamerClangTidyCheck::addUsage(const NamedDecl *Decl, SourceRange Range,
+void RenamerClangTidyCheck::addUsage(const NamedDecl *Decl,
+                                     SourceRange UsageRange,
                                      const SourceManager &SourceMgr) {
-  // Don't keep track for non-identifier names.
-  auto *II = Decl->getIdentifier();
-  if (!II)
+  if (hasNoName(Decl))
+    return;
+
+  // Ignore ClassTemplateSpecializationDecl which are creating duplicate
+  // replacements with CXXRecordDecl.
+  if (isa<ClassTemplateSpecializationDecl>(Decl))
     return;
-  if (const auto *Method = dyn_cast<CXXMethodDecl>(Decl)) {
-    if (const CXXMethodDecl *Overridden = getOverrideMethod(Method))
-      Decl = Overridden;
-  }
-  Decl = cast<NamedDecl>(Decl->getCanonicalDecl());
-  return addUsage(
-      RenamerClangTidyCheck::NamingCheckId(Decl->getLocation(), II->getName()),
-      Range, SourceMgr);
-}
 
-void RenamerClangTidyCheck::checkNamedDecl(const NamedDecl *Decl,
-                                           const SourceManager &SourceMgr) {
-  std::optional<FailureInfo> MaybeFailure = getDeclFailureInfo(Decl, SourceMgr);
+  // We don't want to create a failure for every NamedDecl we find. Ideally
+  // there is just one NamedDecl in every group of "related" NamedDecls that
+  // becomes the failure. This NamedDecl and all of its related NamedDecls
+  // become usages. E.g. Since NamedDecls are Redeclarable, only the canonical
+  // NamedDecl becomes the failure and all redeclarations become usages.
+  const NamedDecl *FailureDecl = getFailureForNamedDecl(Decl);
+
+  std::optional<FailureInfo> MaybeFailure =
+      getDeclFailureInfo(FailureDecl, SourceMgr);
   if (!MaybeFailure)
     return;
 
-  FailureInfo &Info = *MaybeFailure;
-  NamingCheckFailure &Failure =
-      NamingCheckFailures[NamingCheckId(Decl->getLocation(), Decl->getName())];
-  SourceRange Range =
-      DeclarationNameInfo(Decl->getDeclName(), Decl->getLocation())
-          .getSourceRange();
-
-  const IdentifierTable &Idents = Decl->getASTContext().Idents;
-  auto CheckNewIdentifier = Idents.find(Info.Fixup);
+  NamingCheckId FailureId(FailureDecl->getLocation(), FailureDecl->getName());
+
+  auto [FailureIter, NewFailure] = addUsage(FailureId, UsageRange, SourceMgr);
+
+  if (FailureIter == NamingCheckFailures.end()) {
+    // Nothing to do if the usage wasn't accepted.
+    return;
+  }
+  if (!NewFailure) {
+    // FailureInfo has already been provided.
+    return;
+  }
+
+  // Update the stored failure with info regarding the FailureDecl.
+  NamingCheckFailure &Failure = FailureIter->second;
+  Failure.Info = std::move(*MaybeFailure);
+
+  // Don't overwritte the failure status if it was already set.
+  if (!Failure.shouldFix()) {
+    return;
+  }
+  const IdentifierTable &Idents = FailureDecl->getASTContext().Idents;
+  auto CheckNewIdentifier = Idents.find(Failure.Info.Fixup);
   if (CheckNewIdentifier != Idents.end()) {
     const IdentifierInfo *Ident = CheckNewIdentifier->second;
     if (Ident->isKeyword(getLangOpts()))
       Failure.FixStatus = ShouldFixStatus::ConflictsWithKeyword;
     else if (Ident->hasMacroDefinition())
       Failure.FixStatus = ShouldFixStatus::ConflictsWithMacroDefinition;
-  } else if (!isValidAsciiIdentifier(Info.Fixup)) {
+  } else if (!isValidAsciiIdentifier(Failure.Info.Fixup)) {
     Failure.FixStatus = ShouldFixStatus::FixInvalidIdentifier;
   }
-
-  Failure.Info = std::move(Info);
-  addUsage(Decl, Range, SourceMgr);
 }
 
 void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h
index be5b6f0c7f76..3d5721b789ac 100644
--- a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h
+++ b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h
@@ -115,15 +115,9 @@ public:
   void expandMacro(const Token &MacroNameTok, const MacroInfo *MI,
                    const SourceManager &SourceMgr);
 
-  void addUsage(const RenamerClangTidyCheck::NamingCheckId &Decl,
-                SourceRange Range, const SourceManager &SourceMgr);
-
-  /// Convenience method when the usage to be added is a NamedDecl.
   void addUsage(const NamedDecl *Decl, SourceRange Range,
                 const SourceManager &SourceMgr);
 
-  void checkNamedDecl(const NamedDecl *Decl, const SourceManager &SourceMgr);
-
 protected:
   /// Overridden by derived classes, returns information about if and how a Decl
   /// failed the check. A 'std::nullopt' result means the Decl did not fail the
@@ -158,6 +152,14 @@ protected:
                                const NamingCheckFailure &Failure) const = 0;
 
 private:
+  // Manage additions to the Failure/usage map
+  //
+  // return the result of NamingCheckFailures::try_emplace() if the usage was
+  // accepted.
+  std::pair<NamingCheckFailureMap::iterator, bool>
+  addUsage(const RenamerClangTidyCheck::NamingCheckId &FailureId,
+           SourceRange UsageRange, const SourceManager &SourceMgr);
+
   NamingCheckFailureMap NamingCheckFailures;
   const bool AggressiveDependentMemberLookup;
 };
-- 
GitLab


From 62bed56efdde1bed5dcebec5ceb375ffce223691 Mon Sep 17 00:00:00 2001
From: Benoit Jacob <jacob.benoit.1@gmail.com>
Date: Tue, 7 May 2024 15:07:06 -0400
Subject: [PATCH 0081/1206] [mlir][tensor] Remove assertion in
 ExpandShapeOp::build (#91361)

Unblocking downstream integrate where an expected-to-fail test was
expecting this to be a runtime verifier error, not a compiler crash:
https://github.com/llvm/torch-mlir/pull/3279.
---
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 4c65045084dc..7a13f7a7d135 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -1676,10 +1676,12 @@ void ExpandShapeOp::build(OpBuilder &builder, OperationState &result,
   auto tensorResultTy = cast<RankedTensorType>(resultType);
   FailureOr<SmallVector<OpFoldResult>> outputShape = inferOutputShape(
       builder, result.location, tensorResultTy, reassociation, inputShape);
-  // Failure of this assertion usually indicates presence of multiple
-  // dynamic dimensions in the same reassociation group.
-  assert(succeeded(outputShape) && "unable to infer output shape");
-  build(builder, result, tensorResultTy, src, reassociation, *outputShape);
+  SmallVector<OpFoldResult> outputShapeOrEmpty;
+  if (succeeded(outputShape)) {
+    outputShapeOrEmpty = *outputShape;
+  }
+  build(builder, result, tensorResultTy, src, reassociation,
+        outputShapeOrEmpty);
 }
 
 SmallVector<AffineMap, 4> CollapseShapeOp::getReassociationMaps() {
-- 
GitLab


From 6cba93f25dc2014b5d8c71c739f17be1d8c3763a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 7 May 2024 11:48:53 -0700
Subject: [PATCH 0082/1206] [RISCV] Add partial validation of S and X extension
 names to RISCVISAInfo::parseNormalizedArchString.

Extensions starting with 's' or 'x' should always be followed by an
alphabetical character.  I don't know of any crashes from this currently,
but it seemed better to be defensive.
---
 llvm/lib/TargetParser/RISCVISAInfo.cpp           |  6 ++++--
 .../ELF/RISCV/unknown-arch-attr.test             |  6 +++---
 llvm/unittests/TargetParser/RISCVISAInfoTest.cpp | 16 ++++++++++++++++
 3 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 6ab5ee3508a6..9c2ac8c3893f 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -486,9 +486,11 @@ RISCVISAInfo::parseNormalizedArchString(StringRef Arch) {
       return createStringError(errc::invalid_argument,
                                "failed to parse major version number");
 
-    if (ExtName[0] == 'z' && (ExtName.size() == 1 || isDigit(ExtName[1])))
+    if ((ExtName[0] == 'z' || ExtName[0] == 's' || ExtName[0] == 'x') &&
+        (ExtName.size() == 1 || isDigit(ExtName[1])))
       return createStringError(errc::invalid_argument,
-                               "'z' must be followed by a letter");
+                               "'" + Twine(ExtName[0]) +
+                                   "' must be followed by a letter");
 
     ISAInfo->addExtension(ExtName, {MajorVersion, MinorVersion});
   }
diff --git a/llvm/test/tools/llvm-objdump/ELF/RISCV/unknown-arch-attr.test b/llvm/test/tools/llvm-objdump/ELF/RISCV/unknown-arch-attr.test
index 35c8c6240d84..704c9d4add0d 100644
--- a/llvm/test/tools/llvm-objdump/ELF/RISCV/unknown-arch-attr.test
+++ b/llvm/test/tools/llvm-objdump/ELF/RISCV/unknown-arch-attr.test
@@ -3,7 +3,7 @@
 ## The expected behavior is to ignore the unrecognized arch feature and
 ## continue to process the following arch features.
 ##
-## The object file has the "rv32i2p0_m2p0_x1p0" arch feature. "x1p0" is an
+## The object file has the "rv32i2p0_m2p0_y1p0" arch feature. "y1p0" is an
 ## unrecognized architecture extension. llvm-objdump will ignore it and decode
 ## "mul" instruction correctly according to "m2p0" in the arch feature.
 ##
@@ -34,5 +34,5 @@ Sections:
     Content: 3385C502
   - Name:    .riscv.attributes
     Type:    SHT_RISCV_ATTRIBUTES
-## The content is the encoding of the arch feature "rv32i2p0_m2p0_x1p0"
-    Content: 412300000072697363760001190000000572763332693270305F6D3270305F7831703000
+## The content is the encoding of the arch feature "rv32i2p0_m2p0_y1p0"
+    Content: 412300000072697363760001190000000572763332693270305F6D3270305F7931703000
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index d813a8d7185f..a6c21c18c0ec 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -62,6 +62,22 @@ TEST(ParseNormalizedArchString, RejectsBadZ) {
   }
 }
 
+TEST(ParseNormalizedArchString, RejectsBadS) {
+  for (StringRef Input : {"rv64i2p0_s1p0", "rv32i2p0_s2a1p0"}) {
+    EXPECT_EQ(
+        toString(RISCVISAInfo::parseNormalizedArchString(Input).takeError()),
+        "'s' must be followed by a letter");
+  }
+}
+
+TEST(ParseNormalizedArchString, RejectsBadX) {
+  for (StringRef Input : {"rv64i2p0_x1p0", "rv32i2p0_x2a1p0"}) {
+    EXPECT_EQ(
+        toString(RISCVISAInfo::parseNormalizedArchString(Input).takeError()),
+        "'x' must be followed by a letter");
+  }
+}
+
 TEST(ParseNormalizedArchString, AcceptsValidBaseISAsAndSetsXLen) {
   auto MaybeRV32I = RISCVISAInfo::parseNormalizedArchString("rv32i2p0");
   ASSERT_THAT_EXPECTED(MaybeRV32I, Succeeded());
-- 
GitLab


From 1e36c96dc0998e886644d6fc76aa475d88d9645c Mon Sep 17 00:00:00 2001
From: AtariDreams <gfunni234@gmail.com>
Date: Tue, 7 May 2024 15:17:56 -0400
Subject: [PATCH 0083/1206] [InstCombine] Fold ((X << nuw Z) binop nuw Y) >>u Z
 --> X binop nuw (Y >>u Z) (#88193)

Proofs:
https://alive2.llvm.org/ce/z/N9dRzP
https://alive2.llvm.org/ce/z/Xrpc-Y
https://alive2.llvm.org/ce/z/BagBM6
---
 .../InstCombine/InstCombineShifts.cpp         |  50 +++-
 llvm/test/Transforms/InstCombine/lshr.ll      | 240 ++++++++++++++++++
 2 files changed, 288 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 1cb21a1d81af..8847de366713 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -1259,6 +1259,54 @@ Instruction *InstCombinerImpl::visitLShr(BinaryOperator &I) {
       match(Op1, m_SpecificIntAllowPoison(BitWidth - 1)))
     return new ZExtInst(Builder.CreateIsNotNeg(X, "isnotneg"), Ty);
 
+  // ((X << nuw Z) sub nuw Y) >>u exact Z --> X sub nuw (Y >>u exact Z),
+  Value *Y;
+  if (I.isExact() &&
+      match(Op0, m_OneUse(m_NUWSub(m_NUWShl(m_Value(X), m_Specific(Op1)),
+                                   m_Value(Y))))) {
+    Value *NewLshr = Builder.CreateLShr(Y, Op1, "", /*isExact=*/true);
+    auto *NewSub = BinaryOperator::CreateNUWSub(X, NewLshr);
+    NewSub->setHasNoSignedWrap(
+        cast<OverflowingBinaryOperator>(Op0)->hasNoSignedWrap());
+    return NewSub;
+  }
+
+  auto isSuitableBinOpcode = [](Instruction::BinaryOps BinOpcode) {
+    switch (BinOpcode) {
+    default:
+      return false;
+    case Instruction::Add:
+    case Instruction::And:
+    case Instruction::Or:
+    case Instruction::Xor:
+      // And does not work here, and sub is handled separately.
+      return true;
+    }
+  };
+
+  // If both the binop and the shift are nuw, then:
+  // ((X << nuw Z) binop nuw Y) >>u Z --> X binop nuw (Y >>u Z)
+  if (match(Op0, m_OneUse(m_c_BinOp(m_NUWShl(m_Value(X), m_Specific(Op1)),
+                                    m_Value(Y))))) {
+    BinaryOperator *Op0OB = cast<BinaryOperator>(Op0);
+    if (isSuitableBinOpcode(Op0OB->getOpcode())) {
+      if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Op0);
+          !OBO || OBO->hasNoUnsignedWrap()) {
+        Value *NewLshr = Builder.CreateLShr(
+            Y, Op1, "", I.isExact() && Op0OB->getOpcode() != Instruction::And);
+        auto *NewBinOp = BinaryOperator::Create(Op0OB->getOpcode(), NewLshr, X);
+        if (OBO) {
+          NewBinOp->setHasNoUnsignedWrap(true);
+          NewBinOp->setHasNoSignedWrap(OBO->hasNoSignedWrap());
+        } else if (auto *Disjoint = dyn_cast<PossiblyDisjointInst>(Op0)) {
+          cast<PossiblyDisjointInst>(NewBinOp)->setIsDisjoint(
+              Disjoint->isDisjoint());
+        }
+        return NewBinOp;
+      }
+    }
+  }
+
   if (match(Op1, m_APInt(C))) {
     unsigned ShAmtC = C->getZExtValue();
     auto *II = dyn_cast<IntrinsicInst>(Op0);
@@ -1275,7 +1323,6 @@ Instruction *InstCombinerImpl::visitLShr(BinaryOperator &I) {
       return new ZExtInst(Cmp, Ty);
     }
 
-    Value *X;
     const APInt *C1;
     if (match(Op0, m_Shl(m_Value(X), m_APInt(C1))) && C1->ult(BitWidth)) {
       if (C1->ult(ShAmtC)) {
@@ -1320,7 +1367,6 @@ Instruction *InstCombinerImpl::visitLShr(BinaryOperator &I) {
     // ((X << C) + Y) >>u C --> (X + (Y >>u C)) & (-1 >>u C)
     // TODO: Consolidate with the more general transform that starts from shl
     //       (the shifts are in the opposite order).
-    Value *Y;
     if (match(Op0,
               m_OneUse(m_c_Add(m_OneUse(m_Shl(m_Value(X), m_Specific(Op1))),
                                m_Value(Y))))) {
diff --git a/llvm/test/Transforms/InstCombine/lshr.ll b/llvm/test/Transforms/InstCombine/lshr.ll
index 7d611ba188d6..563e669f9035 100644
--- a/llvm/test/Transforms/InstCombine/lshr.ll
+++ b/llvm/test/Transforms/InstCombine/lshr.ll
@@ -163,6 +163,17 @@ define <2 x i8> @lshr_exact_splat_vec(<2 x i8> %x) {
   ret <2 x i8> %lshr
 }
 
+define <2 x i8> @lshr_exact_splat_vec_nuw(<2 x i8> %x) {
+; CHECK-LABEL: @lshr_exact_splat_vec_nuw(
+; CHECK-NEXT:    [[LSHR:%.*]] = add nuw <2 x i8> [[X:%.*]], <i8 1, i8 1>
+; CHECK-NEXT:    ret <2 x i8> [[LSHR]]
+;
+  %shl = shl nuw <2 x i8> %x, <i8 2, i8 2>
+  %add = add nuw <2 x i8> %shl, <i8 4, i8 4>
+  %lshr = lshr <2 x i8> %add, <i8 2, i8 2>
+  ret <2 x i8> %lshr
+}
+
 define i8 @shl_add(i8 %x, i8 %y) {
 ; CHECK-LABEL: @shl_add(
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[Y:%.*]], 2
@@ -360,8 +371,222 @@ define <3 x i14> @mul_splat_fold_vec(<3 x i14> %x) {
   ret <3 x i14> %t
 }
 
+define i32 @shl_add_lshr_flag_preservation(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_add_lshr_flag_preservation(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = add nuw nsw i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %add = add nuw nsw i32 %shl, %y
+  %lshr = lshr exact i32 %add, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_add_lshr(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_add_lshr(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = add nuw i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %add = add nuw i32 %shl, %y
+  %lshr = lshr i32 %add, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_add_lshr_comm(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_add_lshr_comm(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = add nuw i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %add = add nuw i32 %y, %shl
+  %lshr = lshr i32 %add, %c
+  ret i32 %lshr
+}
+
 ; Negative test
 
+define i32 @shl_add_lshr_no_nuw(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_add_lshr_no_nuw(
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 [[X:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[SHL]], [[Y:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[ADD]], [[C]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %add = add i32 %shl, %y
+  %lshr = lshr i32 %add, %c
+  ret i32 %lshr
+}
+
+; Negative test
+
+define i32 @shl_sub_lshr_not_exact(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_sub_lshr_not_exact(
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 [[X:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 [[SHL]], [[Y:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[SUB]], [[C]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %sub = sub nuw i32 %shl, %y
+  %lshr = lshr i32 %sub, %c
+  ret i32 %lshr
+}
+
+; Negative test
+
+define i32 @shl_sub_lshr_no_nuw(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_sub_lshr_no_nuw(
+; CHECK-NEXT:    [[SHL:%.*]] = shl nsw i32 [[X:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[SHL]], [[Y:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr exact i32 [[SUB]], [[C]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nsw i32 %x, %c
+  %sub = sub nsw i32 %shl, %y
+  %lshr = lshr exact i32 %sub, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_sub_lshr(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_sub_lshr(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = sub nuw nsw i32 [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %sub = sub nuw nsw i32 %shl, %y
+  %lshr = lshr exact i32 %sub, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_or_lshr(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_or_lshr(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = or i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %or = or i32 %shl, %y
+  %lshr = lshr i32 %or, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_or_disjoint_lshr(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_or_disjoint_lshr(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = or disjoint i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %or = or disjoint i32 %shl, %y
+  %lshr = lshr i32 %or, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_or_lshr_comm(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_or_lshr_comm(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = or i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %or = or i32 %y, %shl
+  %lshr = lshr i32 %or, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_or_disjoint_lshr_comm(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_or_disjoint_lshr_comm(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = or disjoint i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %or = or disjoint i32 %y, %shl
+  %lshr = lshr i32 %or, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_xor_lshr(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_xor_lshr(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = xor i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %xor = xor i32 %shl, %y
+  %lshr = lshr i32 %xor, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_xor_lshr_comm(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_xor_lshr_comm(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = xor i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %xor = xor i32 %y, %shl
+  %lshr = lshr i32 %xor, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_and_lshr(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_and_lshr(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = and i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %and = and i32 %shl, %y
+  %lshr = lshr i32 %and, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_and_lshr_comm(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_and_lshr_comm(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[LSHR:%.*]] = and i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[LSHR]]
+;
+  %shl = shl nuw i32 %x, %c
+  %and = and i32 %y, %shl
+  %lshr = lshr i32 %and, %c
+  ret i32 %lshr
+}
+
+define i32 @shl_lshr_and_exact(i32 %x, i32 %c, i32 %y) {
+; CHECK-LABEL: @shl_lshr_and_exact(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %2 = shl nuw i32 %x, %c
+  %3 = and i32 %2, %y
+  %4 = lshr exact i32 %3, %c
+  ret i32 %4
+}
+
+; Negative test
+
+define i32 @shl_add_lshr_neg(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @shl_add_lshr_neg(
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[SHL]], [[Z:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = lshr exact i32 [[ADD]], [[Z]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %shl = shl nuw i32 %x, %y
+  %add = add nuw nsw i32 %shl, %z
+  %res = lshr exact i32 %add, %z
+  ret i32 %res
+}
+
 define i32 @mul_splat_fold_wrong_mul_const(i32 %x) {
 ; CHECK-LABEL: @mul_splat_fold_wrong_mul_const(
 ; CHECK-NEXT:    [[M:%.*]] = mul nuw i32 [[X:%.*]], 65538
@@ -375,6 +600,21 @@ define i32 @mul_splat_fold_wrong_mul_const(i32 %x) {
 
 ; Negative test
 
+define i32 @shl_add_lshr_multiuse(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @shl_add_lshr_multiuse(
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[SHL]], [[Z:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[ADD]])
+; CHECK-NEXT:    [[RES:%.*]] = lshr exact i32 [[ADD]], [[Z]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %shl = shl nuw i32 %x, %y
+  %add = add nuw nsw i32 %shl, %z
+  call void @use (i32 %add)
+  %res = lshr exact i32 %add, %z
+  ret i32 %res
+}
+
 define i32 @mul_splat_fold_wrong_lshr_const(i32 %x) {
 ; CHECK-LABEL: @mul_splat_fold_wrong_lshr_const(
 ; CHECK-NEXT:    [[M:%.*]] = mul nuw i32 [[X:%.*]], 65537
-- 
GitLab


From 2a4f1f4a8ff60d55da69b4654360cf947b5b20f7 Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Tue, 7 May 2024 12:23:00 -0700
Subject: [PATCH 0084/1206] Document FP relative offsets (#91031)

---
 llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index fa661b17c13a..fca1824165e7 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -1271,6 +1271,9 @@ Value *HWAddressSanitizer::getFrameRecordInfo(IRBuilder<> &IRB) {
   // FP is 0xfffffffffffFFFF0  (4 lower bits are zero)
   // We only really need ~20 lower non-zero bits (FFFF), so we mix like this:
   //       0xFFFFPPPPPPPPPPPP
+  //
+  // FP works because in AArch64FrameLowering::getFrameIndexReference, we
+  // prefer FP-relative offsets for functions compiled with HWASan.
   FP = IRB.CreateShl(FP, 44);
   return IRB.CreateOr(PC, FP);
 }
-- 
GitLab


From c9ab1d890586bd8a6a194e6a37968538b80f81bd Mon Sep 17 00:00:00 2001
From: Sean Perry <39927768+perry-ca@users.noreply.github.com>
Date: Tue, 7 May 2024 15:23:50 -0400
Subject: [PATCH 0085/1206] Mark test cases as unsupported on z/OS (#90990)

These test cases are testing features not available when either
targeting the s390x-ibm-zos target or use tools/features not available
on the z/OS operating system. In a couple cases the lit test had a
number of subtests with one or two that aren't supported on z/OS. Rather
than mark the entire test as unsupported I split out the unsupported
tests into a separate test case.
---
 clang/test/AST/Interp/cxx23.cpp                  |  1 +
 clang/test/CodeGen/ffp-contract-option.c         |  1 +
 clang/test/CodeGen/ffp-model.c                   |  1 +
 clang/test/CodeGen/fp-matrix-pragma.c            |  1 +
 .../Driver/clang-offload-bundler-asserts-on.c    |  2 +-
 .../Driver/clang-offload-bundler-standardize.c   |  2 +-
 clang/test/Driver/clang-offload-bundler-zlib.c   |  2 +-
 clang/test/Driver/clang-offload-bundler-zstd.c   |  2 +-
 clang/test/Driver/clang-offload-bundler.c        |  2 +-
 clang/test/Driver/std-trigraph-override.c        |  7 +++++++
 clang/test/Driver/std.c                          |  4 ----
 clang/test/FixIt/fixit-c++2a-tls.cpp             | 16 ++++++++++++++++
 clang/test/FixIt/fixit-c++2a.cpp                 |  4 ----
 clang/test/Interpreter/const.cpp                 |  2 +-
 clang/test/Lexer/unicode.c                       |  1 +
 clang/test/Modules/cstd.m                        |  1 +
 .../Modules/merge-objc-protocol-visibility.m     |  2 +-
 clang/test/PCH/chain-openmp-threadprivate.cpp    |  1 +
 clang/test/Sema/thread_local.c                   |  1 +
 llvm/test/MC/AsmParser/layout-interdependency.s  |  1 +
 llvm/test/Object/archive-big-extract.test        |  1 +
 llvm/test/Object/archive-extract.test            |  1 +
 22 files changed, 41 insertions(+), 15 deletions(-)
 create mode 100644 clang/test/Driver/std-trigraph-override.c
 create mode 100644 clang/test/FixIt/fixit-c++2a-tls.cpp

diff --git a/clang/test/AST/Interp/cxx23.cpp b/clang/test/AST/Interp/cxx23.cpp
index 55807f0e0f11..c91d52c552b1 100644
--- a/clang/test/AST/Interp/cxx23.cpp
+++ b/clang/test/AST/Interp/cxx23.cpp
@@ -1,3 +1,4 @@
+// UNSUPPORTED:  target={{.*}}-zos{{.*}}
 // RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=ref20,all,all20 %s
 // RUN: %clang_cc1 -std=c++23 -fsyntax-only -fcxx-exceptions -verify=ref23,all %s
 // RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=expected20,all,all20 %s -fexperimental-new-constant-interpreter
diff --git a/clang/test/CodeGen/ffp-contract-option.c b/clang/test/CodeGen/ffp-contract-option.c
index cd777ac9b43c..2a6443032a4e 100644
--- a/clang/test/CodeGen/ffp-contract-option.c
+++ b/clang/test/CodeGen/ffp-contract-option.c
@@ -1,4 +1,5 @@
 // REQUIRES: x86-registered-target
+// UNSUPPORTED: target={{.*}}-zos{{.*}}
 // RUN: %clang_cc1 -triple=x86_64 %s -emit-llvm -o - \
 // RUN:| FileCheck --check-prefixes CHECK,CHECK-DEFAULT  %s
 
diff --git a/clang/test/CodeGen/ffp-model.c b/clang/test/CodeGen/ffp-model.c
index 780603284a99..4ed9b9dc0a78 100644
--- a/clang/test/CodeGen/ffp-model.c
+++ b/clang/test/CodeGen/ffp-model.c
@@ -1,4 +1,5 @@
 // REQUIRES: x86-registered-target
+// UNSUPPORTED: target={{.*}}-zos{{.*}}
 // RUN: %clang -S -emit-llvm -fenable-matrix -ffp-model=fast %s -o - \
 // RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-FAST
 
diff --git a/clang/test/CodeGen/fp-matrix-pragma.c b/clang/test/CodeGen/fp-matrix-pragma.c
index 45ad6e657daf..5c9909bf60e0 100644
--- a/clang/test/CodeGen/fp-matrix-pragma.c
+++ b/clang/test/CodeGen/fp-matrix-pragma.c
@@ -1,4 +1,5 @@
 // RUN: %clang -emit-llvm -S -fenable-matrix -mllvm -disable-llvm-optzns %s -o - | FileCheck %s
+// UNSUPPORTED: target={{.*}}-zos{{.*}}
 
 typedef float fx2x2_t __attribute__((matrix_type(2, 2)));
 typedef int ix2x2_t __attribute__((matrix_type(2, 2)));
diff --git a/clang/test/Driver/clang-offload-bundler-asserts-on.c b/clang/test/Driver/clang-offload-bundler-asserts-on.c
index eb11d5fbbee4..55060c2c42e7 100644
--- a/clang/test/Driver/clang-offload-bundler-asserts-on.c
+++ b/clang/test/Driver/clang-offload-bundler-asserts-on.c
@@ -1,6 +1,6 @@
 // REQUIRES: x86-registered-target
 // REQUIRES: asserts
-// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}}
+// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}}, target={{.*}}-zos{{.*}}
 
 // Generate the file we can bundle.
 // RUN: %clang -O0 -target %itanium_abi_triple %s -c -o %t.o
diff --git a/clang/test/Driver/clang-offload-bundler-standardize.c b/clang/test/Driver/clang-offload-bundler-standardize.c
index 91dc8947aabb..52f5ea038e47 100644
--- a/clang/test/Driver/clang-offload-bundler-standardize.c
+++ b/clang/test/Driver/clang-offload-bundler-standardize.c
@@ -1,6 +1,6 @@
 // REQUIRES: x86-registered-target
 // REQUIRES: asserts
-// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}}
+// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}}, target={{.*}}-zos{{.*}}
 // REQUIRES: asserts
 
 // Generate the file we can bundle.
diff --git a/clang/test/Driver/clang-offload-bundler-zlib.c b/clang/test/Driver/clang-offload-bundler-zlib.c
index 15b60341a8db..fff7a0f54568 100644
--- a/clang/test/Driver/clang-offload-bundler-zlib.c
+++ b/clang/test/Driver/clang-offload-bundler-zlib.c
@@ -1,6 +1,6 @@
 // REQUIRES: zlib && !zstd
 // REQUIRES: x86-registered-target
-// UNSUPPORTED: target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}}
+// UNSUPPORTED: target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}}, target={{.*}}-zos{{.*}}
 
 //
 // Generate the host binary to be bundled.
diff --git a/clang/test/Driver/clang-offload-bundler-zstd.c b/clang/test/Driver/clang-offload-bundler-zstd.c
index a424981c6971..d01d9659a68d 100644
--- a/clang/test/Driver/clang-offload-bundler-zstd.c
+++ b/clang/test/Driver/clang-offload-bundler-zstd.c
@@ -1,6 +1,6 @@
 // REQUIRES: zstd
 // REQUIRES: x86-registered-target
-// UNSUPPORTED: target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}}
+// UNSUPPORTED: target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}}, target={{.*}}-zos{{.*}}
 
 //
 // Generate the host binary to be bundled.
diff --git a/clang/test/Driver/clang-offload-bundler.c b/clang/test/Driver/clang-offload-bundler.c
index a56a5424abf8..e492da31abb7 100644
--- a/clang/test/Driver/clang-offload-bundler.c
+++ b/clang/test/Driver/clang-offload-bundler.c
@@ -1,5 +1,5 @@
 // REQUIRES: x86-registered-target
-// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}}
+// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}}, target={{.*}}-zos{{.*}}
 
 //
 // Generate all the types of files we can bundle.
diff --git a/clang/test/Driver/std-trigraph-override.c b/clang/test/Driver/std-trigraph-override.c
new file mode 100644
index 000000000000..e4b83ffcf823
--- /dev/null
+++ b/clang/test/Driver/std-trigraph-override.c
@@ -0,0 +1,7 @@
+// UNSUPPORTED: target={{.*-zos.*}}
+// RUN: %clang -w -std=c99 -trigraphs -std=gnu99 %s -E -o - | FileCheck -check-prefix=OVERRIDE %s
+// OVERRIDE: ??(??)
+// RUN: %clang -w -std=c99 -ftrigraphs -std=gnu99 %s -E -o - | FileCheck -check-prefix=FOVERRIDE %s
+// FOVERRIDE: ??(??)
+
+??(??)
diff --git a/clang/test/Driver/std.c b/clang/test/Driver/std.c
index 54f746cc63d0..fe0c4671d9d6 100644
--- a/clang/test/Driver/std.c
+++ b/clang/test/Driver/std.c
@@ -1,7 +1,3 @@
-// RUN: %clang -w -std=c99 -trigraphs -std=gnu99 %s -E -o - | FileCheck -check-prefix=OVERRIDE %s
-// OVERRIDE: ??(??)
-// RUN: %clang -w -std=c99 -ftrigraphs -std=gnu99 %s -E -o - | FileCheck -check-prefix=FOVERRIDE %s
-// FOVERRIDE: ??(??)
 // RUN: %clang -w -ansi %s -E -o - | FileCheck -check-prefix=ANSI %s
 // ANSI: []
 // RUN: %clang -w -ansi %s -fno-trigraphs -E -o - | FileCheck -check-prefix=ANSI-OVERRIDE %s
diff --git a/clang/test/FixIt/fixit-c++2a-tls.cpp b/clang/test/FixIt/fixit-c++2a-tls.cpp
new file mode 100644
index 000000000000..97f2899c9083
--- /dev/null
+++ b/clang/test/FixIt/fixit-c++2a-tls.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -verify -std=c++2a -pedantic-errors %s
+// RUN: cp %s %t
+// RUN: %clang_cc1 -x c++ -std=c++2a -fixit %t
+// RUN: %clang_cc1 -Wall -pedantic-errors -x c++ -std=c++2a %t
+// RUN: cat %t | FileCheck %s
+// UNSUPPORTED: target={{.*-zos.*}}
+
+/* This is a test of the various code modification hints that only
+   apply in C++2a. */
+
+namespace constinit_mismatch {
+  extern thread_local constinit int a; // expected-note {{declared constinit here}}
+  thread_local int a = 123; // expected-error {{'constinit' specifier missing on initializing declaration of 'a'}}
+  // CHECK: {{^}}  constinit thread_local int a = 123;
+}
+
diff --git a/clang/test/FixIt/fixit-c++2a.cpp b/clang/test/FixIt/fixit-c++2a.cpp
index 6fe05dabf079..a21dd701ec74 100644
--- a/clang/test/FixIt/fixit-c++2a.cpp
+++ b/clang/test/FixIt/fixit-c++2a.cpp
@@ -16,10 +16,6 @@ template<typename ...T> void init_capture_pack(T ...a) {
 }
 
 namespace constinit_mismatch {
-  extern thread_local constinit int a; // expected-note {{declared constinit here}}
-  thread_local int a = 123; // expected-error {{'constinit' specifier missing on initializing declaration of 'a'}}
-  // CHECK: {{^}}  constinit thread_local int a = 123;
-
   int b = 123; // expected-note {{add the 'constinit' specifier}}
   extern constinit int b; // expected-error {{'constinit' specifier added after initialization of variable}}
   // CHECK: {{^}}  extern int b;
diff --git a/clang/test/Interpreter/const.cpp b/clang/test/Interpreter/const.cpp
index 86358c1a54fb..57fd880400e6 100644
--- a/clang/test/Interpreter/const.cpp
+++ b/clang/test/Interpreter/const.cpp
@@ -1,4 +1,4 @@
-// UNSUPPORTED: system-aix
+// UNSUPPORTED: system-aix, system-zos
 // see https://github.com/llvm/llvm-project/issues/68092
 // XFAIL: host={{.*}}-windows-msvc
 
diff --git a/clang/test/Lexer/unicode.c b/clang/test/Lexer/unicode.c
index 909b5b424443..e7c7d4b5dad5 100644
--- a/clang/test/Lexer/unicode.c
+++ b/clang/test/Lexer/unicode.c
@@ -3,6 +3,7 @@
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx -x c++ -std=c++11 %s
 // RUN: %clang_cc1 -std=c99 -E -DPP_ONLY=1 %s | FileCheck %s --strict-whitespace
 // RUN: %clang_cc1 -E -DPP_ONLY=1 %s | FileCheck %s --strict-whitespace
+// UNSUPPORTED: system-zos
 
 // This file contains Unicode characters; please do not "fix" them!
 
diff --git a/clang/test/Modules/cstd.m b/clang/test/Modules/cstd.m
index 6b81b9013e9d..2155037400bd 100644
--- a/clang/test/Modules/cstd.m
+++ b/clang/test/Modules/cstd.m
@@ -1,5 +1,6 @@
 // RUN: rm -rf %t
 // RUN: %clang_cc1 -fsyntax-only -internal-isystem %S/Inputs/System/usr/include -fmodules -fimplicit-module-maps -fbuiltin-headers-in-system-modules -fmodules-cache-path=%t -D__need_wint_t -Werror=implicit-function-declaration %s
+// UNSUPPORTED: target={{.*}}-zos{{.*}}
 
 @import uses_other_constants;
 const double other_value = DBL_MAX;
diff --git a/clang/test/Modules/merge-objc-protocol-visibility.m b/clang/test/Modules/merge-objc-protocol-visibility.m
index f5f048b36902..074c3b1ca668 100644
--- a/clang/test/Modules/merge-objc-protocol-visibility.m
+++ b/clang/test/Modules/merge-objc-protocol-visibility.m
@@ -1,4 +1,4 @@
-// UNSUPPORTED: target={{.*}}-aix{{.*}}
+// UNSUPPORTED: target={{.*}}-aix{{.*}}, target={{.*}}-zos{{.*}}
 // RUN: rm -rf %t
 // RUN: split-file %s %t
 // RUN: %clang_cc1 -emit-llvm -o %t/test.bc -F%t/Frameworks %t/test.m -Werror=objc-method-access -DHIDDEN_FIRST=1 \
diff --git a/clang/test/PCH/chain-openmp-threadprivate.cpp b/clang/test/PCH/chain-openmp-threadprivate.cpp
index 05cd65063789..21b9f6868cc3 100644
--- a/clang/test/PCH/chain-openmp-threadprivate.cpp
+++ b/clang/test/PCH/chain-openmp-threadprivate.cpp
@@ -8,6 +8,7 @@
 // with PCH
 // RUN: %clang_cc1 -fopenmp -emit-llvm -chain-include %s -chain-include %s %s -o - | FileCheck %s -check-prefix=CHECK-TLS-1
 // RUN: %clang_cc1 -fopenmp -emit-llvm -chain-include %s -chain-include %s %s -o - | FileCheck %s -check-prefix=CHECK-TLS-2
+// // UNSUPPORTED: target={{.*}}-zos{{.*}}
 
 #if !defined(PASS1)
 #define PASS1
diff --git a/clang/test/Sema/thread_local.c b/clang/test/Sema/thread_local.c
index a0de0aa4e39a..b65f1119c738 100644
--- a/clang/test/Sema/thread_local.c
+++ b/clang/test/Sema/thread_local.c
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c23 %s -verify
+// UNSUPPORTED: target={{.*}}-zos{{.*}}
 
 // Ensure that thread_local and _Thread_local are synonyms in C23 and both
 // restrict local variables to be explicitly static or extern.
diff --git a/llvm/test/MC/AsmParser/layout-interdependency.s b/llvm/test/MC/AsmParser/layout-interdependency.s
index f26149ced766..d275614e87e7 100644
--- a/llvm/test/MC/AsmParser/layout-interdependency.s
+++ b/llvm/test/MC/AsmParser/layout-interdependency.s
@@ -1,5 +1,6 @@
 # RUN: not llvm-mc --filetype=obj %s -o /dev/null 2>&1 | FileCheck %s
 # REQUIRES: object-emission
+# UNSUPPORTED: target={{.*}}-zos{{.*}}
 
 fct_end:
 
diff --git a/llvm/test/Object/archive-big-extract.test b/llvm/test/Object/archive-big-extract.test
index a1d7f0c731c0..3de09d8fb106 100644
--- a/llvm/test/Object/archive-big-extract.test
+++ b/llvm/test/Object/archive-big-extract.test
@@ -1,4 +1,5 @@
 ## Test extract xcoff object file from AIX big archive.
+# UNSUPPORTED: target={{.*}}-zos{{.*}}
 # RUN: rm -rf %t && mkdir -p %t/extracted/ && cd %t/extracted/
 # RUN: llvm-ar x %p/Inputs/aix-big-archive.a
 # RUN: echo "content_of_evenlen" > evenlen_1
diff --git a/llvm/test/Object/archive-extract.test b/llvm/test/Object/archive-extract.test
index 57b3c8f6795a..d4edece8fc45 100644
--- a/llvm/test/Object/archive-extract.test
+++ b/llvm/test/Object/archive-extract.test
@@ -1,6 +1,7 @@
 ; This test just makes sure that llvm-ar can extract bytecode members
 ; from various style archives.
 
+; UNSUPPORTED: target={{.*}}-zos{{.*}}
 ; RUN: rm -rf %t && mkdir -p %t && cd %t
 
 ; RUN: rm -f very_long_bytecode_file_name.bc
-- 
GitLab


From 057de4d26425c8b9840912e40ce025626f45d8d6 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Tue, 7 May 2024 12:35:06 -0700
Subject: [PATCH 0086/1206] [lldb] Request crash report when prompting for a
 bug report on Darwin (#91371)

On Darwin platforms, the system will generate a crash report in
~/Library/Logs/DiagnosticReports/ when a process crashes.

These reports are much more useful than the "pretty backtraces" printed
by LLVM and are preferred when filing bug reports on Darwin.
---
 lldb/tools/driver/Driver.cpp     | 6 ++++++
 lldb/tools/lldb-dap/lldb-dap.cpp | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/lldb/tools/driver/Driver.cpp b/lldb/tools/driver/Driver.cpp
index a821699c5e2e..14371da64f2f 100644
--- a/lldb/tools/driver/Driver.cpp
+++ b/lldb/tools/driver/Driver.cpp
@@ -733,8 +733,14 @@ int main(int argc, char const *argv[]) {
   // Setup LLVM signal handlers and make sure we call llvm_shutdown() on
   // destruction.
   llvm::InitLLVM IL(argc, argv, /*InstallPipeSignalExitHandler=*/false);
+#if !defined(__APPLE__)
   llvm::setBugReportMsg("PLEASE submit a bug report to " LLDB_BUG_REPORT_URL
                         " and include the crash backtrace.\n");
+#else
+  llvm::setBugReportMsg("PLEASE submit a bug report to " LLDB_BUG_REPORT_URL
+                        " and include the crash report from "
+                        "~/Library/Logs/DiagnosticReports/.\n");
+#endif
 
   // Parse arguments.
   LLDBOptTable T;
diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp
index cf52a22b18cc..f35abd665e84 100644
--- a/lldb/tools/lldb-dap/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/lldb-dap.cpp
@@ -4192,8 +4192,14 @@ int SetupStdoutStderrRedirection() {
 
 int main(int argc, char *argv[]) {
   llvm::InitLLVM IL(argc, argv, /*InstallPipeSignalExitHandler=*/false);
+#if !defined(__APPLE__)
   llvm::setBugReportMsg("PLEASE submit a bug report to " LLDB_BUG_REPORT_URL
                         " and include the crash backtrace.\n");
+#else
+  llvm::setBugReportMsg("PLEASE submit a bug report to " LLDB_BUG_REPORT_URL
+                        " and include the crash report from "
+                        "~/Library/Logs/DiagnosticReports/.\n");
+#endif
 
   llvm::SmallString<256> program_path(argv[0]);
   llvm::sys::fs::make_absolute(program_path);
-- 
GitLab


From 65e2fab401a2da55c51d3caceae8478c33f3c60f Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 7 May 2024 12:40:19 -0700
Subject: [PATCH 0087/1206] [Sema] Fix warnings

This patch fixes:

  clang/lib/Sema/SemaTemplateInstantiateDecl.cpp:3937:12: error:
  unused variable 'CanonType' [-Werror,-Wunused-variable]

  clang/lib/Sema/SemaTemplate.cpp:9279:18: error: unused variable
  'TemplateKWLoc' [-Werror,-Wunused-variable]
---
 clang/lib/Sema/SemaTemplate.cpp                | 2 --
 clang/lib/Sema/SemaTemplateInstantiateDecl.cpp | 6 ------
 2 files changed, 8 deletions(-)

diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index b268d7c405df..6231b65bd842 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -9276,8 +9276,6 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
 
   // NOTE: KWLoc is the location of the tag keyword. This will instead
   // store the location of the outermost template keyword in the declaration.
-  SourceLocation TemplateKWLoc = TemplateParameterLists.size() > 0
-    ? TemplateParameterLists[0]->getTemplateLoc() : KWLoc;
   SourceLocation TemplateNameLoc = TemplateId.TemplateNameLoc;
   SourceLocation LAngleLoc = TemplateId.LAngleLoc;
   SourceLocation RAngleLoc = TemplateId.RAngleLoc;
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 5315b143215e..884e98a300f5 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -3932,12 +3932,6 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
   if (SubstQualifier(D, InstD))
     return nullptr;
 
-  // Build the canonical type that describes the converted template
-  // arguments of the class template explicit specialization.
-  QualType CanonType = SemaRef.Context.getTemplateSpecializationType(
-      TemplateName(InstClassTemplate), CanonicalConverted,
-      SemaRef.Context.getRecordType(InstD));
-
   InstD->setAccess(D->getAccess());
   InstD->setInstantiationOfMemberClass(D, TSK_ImplicitInstantiation);
   InstD->setSpecializationKind(D->getSpecializationKind());
-- 
GitLab


From c76ccf0f1e05d649449c8ff6908b0b6329eb2612 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 7 May 2024 20:41:55 +0100
Subject: [PATCH 0088/1206] [LV] Add test case for #91369.

Add tests for https://github.com/llvm/llvm-project/issues/91369.
---
 .../version-stride-with-integer-casts.ll      | 137 ++++++++++++++++++
 1 file changed, 137 insertions(+)

diff --git a/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll b/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
index d09066fa2d70..45745f85de95 100644
--- a/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
+++ b/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
@@ -412,6 +412,139 @@ loop:
 exit:
   ret void
 }
+
+; Test case to make sure that uses of versioned strides of type i1 are properly
+; extended. From https://github.com/llvm/llvm-project/issues/91369.
+; FIXME: Currently miscompiled.
+define void @zext_of_i1_stride(i1 %g, ptr %dst) mustprogress {
+; CHECK-LABEL: define void @zext_of_i1_stride(
+; CHECK-SAME: i1 [[G:%.*]], ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[G_16:%.*]] = zext i1 [[G]] to i16
+; CHECK-NEXT:    [[G_64:%.*]] = zext i1 [[G]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 15, [[G_64]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK:       vector.scevcheck:
+; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i1 [[G]], true
+; CHECK-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], [[G_64]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[G_64]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 0, [[G_64]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    store <4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1>, ptr [[TMP5]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i16 [[G_16]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], [[G_64]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[IV_NEXT]], 16
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %g.16 = zext i1 %g to i16
+  %g.64 = zext i1 %g to i64
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i16, ptr %dst, i64 %iv
+  store i16 %g.16, ptr %gep, align 2
+  %iv.next = add nuw nsw i64 %iv, %g.64
+  %cmp = icmp ult i64 %iv.next, 16
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Test case to make sure that uses of versioned strides of type i1 are properly
+; extended.
+define void @sext_of_i1_stride(i1 %g, ptr %dst) mustprogress {
+; CHECK-LABEL: define void @sext_of_i1_stride(
+; CHECK-SAME: i1 [[G:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[G_16:%.*]] = sext i1 [[G]] to i16
+; CHECK-NEXT:    [[G_64:%.*]] = sext i1 [[G]] to i64
+; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[G_64]], i64 16)
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], [[G_64]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK:       vector.scevcheck:
+; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i1 [[G]], true
+; CHECK-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], [[G_64]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[G_64]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 0, [[G_64]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[TMP6]], i32 -3
+; CHECK-NEXT:    store <4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1>, ptr [[TMP7]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i16 [[G_16]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], [[G_64]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[IV_NEXT]], 16
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %g.16 = sext i1 %g to i16
+  %g.64 = sext i1 %g to i64
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i16, ptr %dst, i64 %iv
+  store i16 %g.16, ptr %gep, align 2
+  %iv.next = add nuw nsw i64 %iv, %g.64
+  %cmp = icmp ult i64 %iv.next, 16
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+
 ;.
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -425,4 +558,8 @@ exit:
 ; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
 ; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
 ; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]]}
 ;.
-- 
GitLab


From 82bb2534d4de16abb7a51fc646d5c31d6cec5eeb Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 7 May 2024 21:43:22 +0200
Subject: [PATCH 0089/1206] AMDGPU: Don't bitcast float typed atomic store in
 IR (#90116)

Implement the promotion in the DAG.

Depends #90113
---
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 29 +++++++----
 .../SelectionDAG/LegalizeFloatTypes.cpp       | 34 +++++++++++++
 llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h |  2 +
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 12 +++++
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h   |  4 ++
 .../AMDGPU/no-expand-atomic-store.ll          | 51 +++++++------------
 6 files changed, 91 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index bfc3e08c1632..b3ae419b20fe 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -5006,7 +5006,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
       Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
     OVT = Node->getOperand(0).getSimpleValueType();
   }
-  if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
+  if (Node->getOpcode() == ISD::ATOMIC_STORE ||
+      Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
       Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
       Node->getOpcode() == ISD::STRICT_FSETCC ||
       Node->getOpcode() == ISD::STRICT_FSETCCS ||
@@ -5622,7 +5623,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     Results.push_back(CvtVec);
     break;
   }
-  case ISD::ATOMIC_SWAP: {
+  case ISD::ATOMIC_SWAP:
+  case ISD::ATOMIC_STORE: {
     AtomicSDNode *AM = cast<AtomicSDNode>(Node);
     SDLoc SL(Node);
     SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
@@ -5631,13 +5633,22 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
            "unexpected atomic_swap with illegal type");
 
-    SDValue NewAtomic
-      = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
-                      DAG.getVTList(NVT, MVT::Other),
-                      { AM->getChain(), AM->getBasePtr(), CastVal },
-                      AM->getMemOperand());
-    Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
-    Results.push_back(NewAtomic.getValue(1));
+    SDValue Op0 = AM->getBasePtr();
+    SDValue Op1 = CastVal;
+
+    // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
+    // but really it should merge with ISD::STORE.
+    if (AM->getOpcode() == ISD::ATOMIC_STORE)
+      std::swap(Op0, Op1);
+
+    SDValue NewAtomic = DAG.getAtomic(AM->getOpcode(), SL, NVT, AM->getChain(),
+                                      Op0, Op1, AM->getMemOperand());
+
+    if (AM->getOpcode() != ISD::ATOMIC_STORE) {
+      Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
+      Results.push_back(NewAtomic.getValue(1));
+    } else
+      Results.push_back(NewAtomic);
     break;
   }
   case ISD::ATOMIC_LOAD: {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index bf87437b8dfd..fc96ecdc6628 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -2287,6 +2287,7 @@ bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
     case ISD::SELECT_CC:  R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
     case ISD::SETCC:      R = PromoteFloatOp_SETCC(N, OpNo); break;
     case ISD::STORE:      R = PromoteFloatOp_STORE(N, OpNo); break;
+    case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
   }
   // clang-format on
 
@@ -2409,6 +2410,23 @@ SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
                       ST->getMemOperand());
 }
 
+SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
+                                                      unsigned OpNo) {
+  AtomicSDNode *ST = cast<AtomicSDNode>(N);
+  SDValue Val = ST->getVal();
+  SDLoc DL(N);
+
+  SDValue Promoted = GetPromotedFloat(Val);
+  EVT VT = ST->getOperand(1).getValueType();
+  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
+
+  SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT),
+                               DL, IVT, Promoted);
+
+  return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal,
+                       ST->getBasePtr(), ST->getMemOperand());
+}
+
 //===----------------------------------------------------------------------===//
 //  Float Result Promotion
 //===----------------------------------------------------------------------===//
@@ -3238,6 +3256,9 @@ bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
   case ISD::SELECT_CC:  Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
   case ISD::SETCC:      Res = SoftPromoteHalfOp_SETCC(N); break;
   case ISD::STORE:      Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
+  case ISD::ATOMIC_STORE:
+    Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
+    break;
   case ISD::STACKMAP:
     Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
     break;
@@ -3391,6 +3412,19 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
                       ST->getMemOperand());
 }
 
+SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
+                                                         unsigned OpNo) {
+  assert(OpNo == 1 && "Can only soften the stored value!");
+  AtomicSDNode *ST = cast<AtomicSDNode>(N);
+  SDValue Val = ST->getVal();
+  SDLoc dl(N);
+
+  SDValue Promoted = GetSoftPromotedHalf(Val);
+  return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(),
+                       ST->getChain(), Promoted, ST->getBasePtr(),
+                       ST->getMemOperand());
+}
+
 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
   assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
   SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index f44916b741cc..d925089d5689 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -712,6 +712,7 @@ private:
   SDValue PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
+  SDValue PromoteFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo);
 
@@ -757,6 +758,7 @@ private:
   SDValue SoftPromoteHalfOp_SETCC(SDNode *N);
   SDValue SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo);
   SDValue SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo);
+  SDValue SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);
   SDValue SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo);
   SDValue SoftPromoteHalfOp_PATCHPOINT(SDNode *N, unsigned OpNo);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 5ca7f8ef5345..1e9132bcfaf9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -161,6 +161,18 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::ATOMIC_LOAD, MVT::bf16, Promote);
   AddPromotedToType(ISD::ATOMIC_LOAD, MVT::bf16, MVT::i16);
 
+  setOperationAction(ISD::ATOMIC_STORE, MVT::f32, Promote);
+  AddPromotedToType(ISD::ATOMIC_STORE, MVT::f32, MVT::i32);
+
+  setOperationAction(ISD::ATOMIC_STORE, MVT::f64, Promote);
+  AddPromotedToType(ISD::ATOMIC_STORE, MVT::f64, MVT::i64);
+
+  setOperationAction(ISD::ATOMIC_STORE, MVT::f16, Promote);
+  AddPromotedToType(ISD::ATOMIC_STORE, MVT::f16, MVT::i16);
+
+  setOperationAction(ISD::ATOMIC_STORE, MVT::bf16, Promote);
+  AddPromotedToType(ISD::ATOMIC_STORE, MVT::bf16, MVT::i16);
+
   // There are no 64-bit extloads. These should be done as a 32-bit extload and
   // an extension to 64-bit.
   for (MVT VT : MVT::integer_valuetypes())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 16c4f53d6344..3814b56a4d56 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -236,6 +236,10 @@ public:
     return AtomicExpansionKind::None;
   }
 
+  AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const override {
+    return AtomicExpansionKind::None;
+  }
+
   AtomicExpansionKind shouldCastAtomicRMWIInIR(AtomicRMWInst *) const override {
     return AtomicExpansionKind::None;
   }
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/no-expand-atomic-store.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/no-expand-atomic-store.ll
index db0c3a20e62f..9159393ab887 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/no-expand-atomic-store.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/no-expand-atomic-store.ll
@@ -4,8 +4,7 @@
 define void @store_atomic_f32_global_system(float %val, ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f32_global_system(
 ; CHECK-SAME: float [[VAL:%.*]], ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[VAL]] to i32
-; CHECK-NEXT:    store atomic i32 [[TMP1]], ptr addrspace(1) [[PTR]] seq_cst, align 4
+; CHECK-NEXT:    store atomic float [[VAL]], ptr addrspace(1) [[PTR]] seq_cst, align 4, !some.unknown.md [[META0:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic float %val, ptr addrspace(1) %ptr seq_cst, align 4, !some.unknown.md !0
@@ -15,8 +14,7 @@ define void @store_atomic_f32_global_system(float %val, ptr addrspace(1) %ptr) {
 define void @store_atomic_f32_global_agent(float %val, ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f32_global_agent(
 ; CHECK-SAME: float [[VAL:%.*]], ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[VAL]] to i32
-; CHECK-NEXT:    store atomic i32 [[TMP1]], ptr addrspace(1) [[PTR]] syncscope("agent") seq_cst, align 4
+; CHECK-NEXT:    store atomic float [[VAL]], ptr addrspace(1) [[PTR]] syncscope("agent") seq_cst, align 4, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic float %val, ptr addrspace(1) %ptr syncscope("agent") seq_cst, align 4, !some.unknown.md !0
@@ -26,8 +24,7 @@ define void @store_atomic_f32_global_agent(float %val, ptr addrspace(1) %ptr) {
 define void @store_atomic_f32_local(float %val, ptr addrspace(3) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f32_local(
 ; CHECK-SAME: float [[VAL:%.*]], ptr addrspace(3) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[VAL]] to i32
-; CHECK-NEXT:    store atomic i32 [[TMP1]], ptr addrspace(3) [[PTR]] seq_cst, align 4
+; CHECK-NEXT:    store atomic float [[VAL]], ptr addrspace(3) [[PTR]] seq_cst, align 4, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic float %val, ptr addrspace(3) %ptr seq_cst, align 4, !some.unknown.md !0
@@ -37,8 +34,7 @@ define void @store_atomic_f32_local(float %val, ptr addrspace(3) %ptr) {
 define void @store_atomic_f32_flat(float %val, ptr %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f32_flat(
 ; CHECK-SAME: float [[VAL:%.*]], ptr [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[VAL]] to i32
-; CHECK-NEXT:    store atomic i32 [[TMP1]], ptr [[PTR]] seq_cst, align 4
+; CHECK-NEXT:    store atomic float [[VAL]], ptr [[PTR]] seq_cst, align 4, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic float %val, ptr %ptr seq_cst, align 4, !some.unknown.md !0
@@ -48,8 +44,7 @@ define void @store_atomic_f32_flat(float %val, ptr %ptr) {
 define void @store_atomic_f16_global_system(half %val, ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f16_global_system(
 ; CHECK-SAME: half [[VAL:%.*]], ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[VAL]] to i16
-; CHECK-NEXT:    store atomic i16 [[TMP1]], ptr addrspace(1) [[PTR]] seq_cst, align 4
+; CHECK-NEXT:    store atomic half [[VAL]], ptr addrspace(1) [[PTR]] seq_cst, align 4, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic half %val, ptr addrspace(1) %ptr seq_cst, align 4, !some.unknown.md !0
@@ -59,8 +54,7 @@ define void @store_atomic_f16_global_system(half %val, ptr addrspace(1) %ptr) {
 define void @store_atomic_f16_global_agent(half %val, ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f16_global_agent(
 ; CHECK-SAME: half [[VAL:%.*]], ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[VAL]] to i16
-; CHECK-NEXT:    store atomic i16 [[TMP1]], ptr addrspace(1) [[PTR]] syncscope("agent") seq_cst, align 4
+; CHECK-NEXT:    store atomic half [[VAL]], ptr addrspace(1) [[PTR]] syncscope("agent") seq_cst, align 4, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic half %val, ptr addrspace(1) %ptr syncscope("agent") seq_cst, align 4, !some.unknown.md !0
@@ -70,8 +64,7 @@ define void @store_atomic_f16_global_agent(half %val, ptr addrspace(1) %ptr) {
 define void @store_atomic_f16_local(half %val, ptr addrspace(3) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f16_local(
 ; CHECK-SAME: half [[VAL:%.*]], ptr addrspace(3) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[VAL]] to i16
-; CHECK-NEXT:    store atomic i16 [[TMP1]], ptr addrspace(3) [[PTR]] seq_cst, align 4
+; CHECK-NEXT:    store atomic half [[VAL]], ptr addrspace(3) [[PTR]] seq_cst, align 4, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic half %val, ptr addrspace(3) %ptr seq_cst, align 4, !some.unknown.md !0
@@ -81,8 +74,7 @@ define void @store_atomic_f16_local(half %val, ptr addrspace(3) %ptr) {
 define void @store_atomic_f16_flat(half %val, ptr %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f16_flat(
 ; CHECK-SAME: half [[VAL:%.*]], ptr [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[VAL]] to i16
-; CHECK-NEXT:    store atomic i16 [[TMP1]], ptr [[PTR]] seq_cst, align 4
+; CHECK-NEXT:    store atomic half [[VAL]], ptr [[PTR]] seq_cst, align 4, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic half %val, ptr %ptr seq_cst, align 4, !some.unknown.md !0
@@ -92,8 +84,7 @@ define void @store_atomic_f16_flat(half %val, ptr %ptr) {
 define void @store_atomic_bf16_global_system(bfloat %val, ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_bf16_global_system(
 ; CHECK-SAME: bfloat [[VAL:%.*]], ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast bfloat [[VAL]] to i16
-; CHECK-NEXT:    store atomic i16 [[TMP1]], ptr addrspace(1) [[PTR]] seq_cst, align 4
+; CHECK-NEXT:    store atomic bfloat [[VAL]], ptr addrspace(1) [[PTR]] seq_cst, align 4, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic bfloat %val, ptr addrspace(1) %ptr seq_cst, align 4, !some.unknown.md !0
@@ -103,8 +94,7 @@ define void @store_atomic_bf16_global_system(bfloat %val, ptr addrspace(1) %ptr)
 define void @store_atomic_bf16_global_agent(bfloat %val, ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_bf16_global_agent(
 ; CHECK-SAME: bfloat [[VAL:%.*]], ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast bfloat [[VAL]] to i16
-; CHECK-NEXT:    store atomic i16 [[TMP1]], ptr addrspace(1) [[PTR]] syncscope("agent") seq_cst, align 4
+; CHECK-NEXT:    store atomic bfloat [[VAL]], ptr addrspace(1) [[PTR]] syncscope("agent") seq_cst, align 4, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic bfloat %val, ptr addrspace(1) %ptr syncscope("agent") seq_cst, align 4, !some.unknown.md !0
@@ -114,8 +104,7 @@ define void @store_atomic_bf16_global_agent(bfloat %val, ptr addrspace(1) %ptr)
 define void @store_atomic_bf16_local(bfloat %val, ptr addrspace(3) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_bf16_local(
 ; CHECK-SAME: bfloat [[VAL:%.*]], ptr addrspace(3) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast bfloat [[VAL]] to i16
-; CHECK-NEXT:    store atomic i16 [[TMP1]], ptr addrspace(3) [[PTR]] seq_cst, align 4
+; CHECK-NEXT:    store atomic bfloat [[VAL]], ptr addrspace(3) [[PTR]] seq_cst, align 4, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic bfloat %val, ptr addrspace(3) %ptr seq_cst, align 4, !some.unknown.md !0
@@ -125,8 +114,7 @@ define void @store_atomic_bf16_local(bfloat %val, ptr addrspace(3) %ptr) {
 define void @store_atomic_bf16_flat(bfloat %val, ptr %ptr) {
 ; CHECK-LABEL: define void @store_atomic_bf16_flat(
 ; CHECK-SAME: bfloat [[VAL:%.*]], ptr [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast bfloat [[VAL]] to i16
-; CHECK-NEXT:    store atomic i16 [[TMP1]], ptr [[PTR]] seq_cst, align 4
+; CHECK-NEXT:    store atomic bfloat [[VAL]], ptr [[PTR]] seq_cst, align 4, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic bfloat %val, ptr %ptr seq_cst, align 4, !some.unknown.md !0
@@ -135,8 +123,7 @@ define void @store_atomic_bf16_flat(bfloat %val, ptr %ptr) {
 define void @store_atomic_f64_global_system(double %val, ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f64_global_system(
 ; CHECK-SAME: double [[VAL:%.*]], ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double [[VAL]] to i64
-; CHECK-NEXT:    store atomic i64 [[TMP1]], ptr addrspace(1) [[PTR]] seq_cst, align 8
+; CHECK-NEXT:    store atomic double [[VAL]], ptr addrspace(1) [[PTR]] seq_cst, align 8, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic double %val, ptr addrspace(1) %ptr seq_cst, align 8, !some.unknown.md !0
@@ -146,8 +133,7 @@ define void @store_atomic_f64_global_system(double %val, ptr addrspace(1) %ptr)
 define void @store_atomic_f64_global_agent(double %val, ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f64_global_agent(
 ; CHECK-SAME: double [[VAL:%.*]], ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double [[VAL]] to i64
-; CHECK-NEXT:    store atomic i64 [[TMP1]], ptr addrspace(1) [[PTR]] syncscope("agent") seq_cst, align 8
+; CHECK-NEXT:    store atomic double [[VAL]], ptr addrspace(1) [[PTR]] syncscope("agent") seq_cst, align 8, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic double %val, ptr addrspace(1) %ptr syncscope("agent") seq_cst, align 8, !some.unknown.md !0
@@ -157,8 +143,7 @@ define void @store_atomic_f64_global_agent(double %val, ptr addrspace(1) %ptr) {
 define void @store_atomic_f64_local(double %val, ptr addrspace(3) %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f64_local(
 ; CHECK-SAME: double [[VAL:%.*]], ptr addrspace(3) [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double [[VAL]] to i64
-; CHECK-NEXT:    store atomic i64 [[TMP1]], ptr addrspace(3) [[PTR]] seq_cst, align 8
+; CHECK-NEXT:    store atomic double [[VAL]], ptr addrspace(3) [[PTR]] seq_cst, align 8, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic double %val, ptr addrspace(3) %ptr seq_cst, align 8, !some.unknown.md !0
@@ -168,8 +153,7 @@ define void @store_atomic_f64_local(double %val, ptr addrspace(3) %ptr) {
 define void @store_atomic_f64_flat(double %val, ptr %ptr) {
 ; CHECK-LABEL: define void @store_atomic_f64_flat(
 ; CHECK-SAME: double [[VAL:%.*]], ptr [[PTR:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double [[VAL]] to i64
-; CHECK-NEXT:    store atomic i64 [[TMP1]], ptr [[PTR]] seq_cst, align 8
+; CHECK-NEXT:    store atomic double [[VAL]], ptr [[PTR]] seq_cst, align 8, !some.unknown.md [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   store atomic double %val, ptr %ptr seq_cst, align 8, !some.unknown.md !0
@@ -177,3 +161,6 @@ define void @store_atomic_f64_flat(double %val, ptr %ptr) {
 }
 
 !0 = !{}
+;.
+; CHECK: [[META0]] = !{}
+;.
-- 
GitLab


From 31dd0ef73c99b1bc9825ddfc58ddff0b134608fb Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Thu, 21 Mar 2024 01:28:09 -0500
Subject: [PATCH 0090/1206] [CVP] Add tests for adding `nneg` flag to `uitofp`
 and converting `sitofp` -> `uitofp nneg`; NFC

---
 .../CorrelatedValuePropagation/sitofp.ll      | 99 +++++++++++++++++++
 .../CorrelatedValuePropagation/uitofp.ll      | 98 ++++++++++++++++++
 2 files changed, 197 insertions(+)
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/sitofp.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/uitofp.ll

diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/sitofp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/sitofp.ll
new file mode 100644
index 000000000000..4bc649245d52
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/sitofp.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=correlated-propagation -S | FileCheck %s
+
+declare void @use.f32(float)
+
+define void @test1(i32 %n) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -1
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[EXT_WIDE:%.*]] = sitofp i32 [[N]] to float
+; CHECK-NEXT:    call void @use.f32(float [[EXT_WIDE]])
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i32 %n, -1
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %ext.wide = sitofp i32 %n to float
+  call void @use.f32(float %ext.wide)
+  br label %exit
+
+exit:
+  ret void
+}
+
+
+define void @test2_fail(i32 %n) {
+; CHECK-LABEL: @test2_fail(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -2
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[EXT_WIDE:%.*]] = sitofp i32 [[N]] to float
+; CHECK-NEXT:    call void @use.f32(float [[EXT_WIDE]])
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i32 %n, -2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %ext.wide = sitofp i32 %n to float
+  call void @use.f32(float %ext.wide)
+  br label %exit
+
+exit:
+  ret void
+}
+
+define float @may_including_undef(i1 %c.1, i1 %c.2) {
+; CHECK-LABEL: @may_including_undef(
+; CHECK-NEXT:    br i1 [[C_1:%.*]], label [[TRUE_1:%.*]], label [[FALSE:%.*]]
+; CHECK:       true.1:
+; CHECK-NEXT:    br i1 [[C_2:%.*]], label [[TRUE_2:%.*]], label [[EXIT:%.*]]
+; CHECK:       true.2:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       false:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 0, [[TRUE_1]] ], [ 1, [[TRUE_2]] ], [ undef, [[FALSE]] ]
+; CHECK-NEXT:    [[EXT:%.*]] = sitofp i32 [[P]] to float
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  br i1 %c.1, label %true.1, label %false
+
+true.1:
+  br i1 %c.2, label %true.2, label %exit
+
+true.2:
+  br label %exit
+
+false:
+  br label %exit
+
+exit:
+  %p = phi i32 [ 0, %true.1 ], [ 1, %true.2], [ undef, %false ]
+  %ext = sitofp i32 %p to float
+  ret float %ext
+}
+
+define double @test_infer_at_use(i32 noundef %n) {
+; CHECK-LABEL: @test_infer_at_use(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -1
+; CHECK-NEXT:    [[EXT:%.*]] = sitofp i32 [[N]] to double
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], double [[EXT]], double 0.000000e+00
+; CHECK-NEXT:    ret double [[SELECT]]
+;
+  %cmp = icmp sgt i32 %n, -1
+  %ext = sitofp i32 %n to double
+  %select = select i1 %cmp, double %ext, double 0.0
+  ret double %select
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/uitofp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/uitofp.ll
new file mode 100644
index 000000000000..0558ec61e636
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/uitofp.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=correlated-propagation -S | FileCheck %s
+
+declare void @use.f32(float)
+
+define void @test1(i32 %n) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -1
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[EXT_WIDE:%.*]] = uitofp i32 [[N]] to float
+; CHECK-NEXT:    call void @use.f32(float [[EXT_WIDE]])
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i32 %n, -1
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %ext.wide = uitofp i32 %n to float
+  call void @use.f32(float %ext.wide)
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @test2_fail(i32 %n) {
+; CHECK-LABEL: @test2_fail(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -2
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[EXT_WIDE:%.*]] = uitofp i32 [[N]] to float
+; CHECK-NEXT:    call void @use.f32(float [[EXT_WIDE]])
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i32 %n, -2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+  %ext.wide = uitofp i32 %n to float
+  call void @use.f32(float %ext.wide)
+  br label %exit
+
+exit:
+  ret void
+}
+
+define float @may_including_undef(i1 %c.1, i1 %c.2) {
+; CHECK-LABEL: @may_including_undef(
+; CHECK-NEXT:    br i1 [[C_1:%.*]], label [[TRUE_1:%.*]], label [[FALSE:%.*]]
+; CHECK:       true.1:
+; CHECK-NEXT:    br i1 [[C_2:%.*]], label [[TRUE_2:%.*]], label [[EXIT:%.*]]
+; CHECK:       true.2:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       false:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 0, [[TRUE_1]] ], [ 1, [[TRUE_2]] ], [ undef, [[FALSE]] ]
+; CHECK-NEXT:    [[EXT:%.*]] = uitofp i32 [[P]] to float
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  br i1 %c.1, label %true.1, label %false
+
+true.1:
+  br i1 %c.2, label %true.2, label %exit
+
+true.2:
+  br label %exit
+
+false:
+  br label %exit
+
+exit:
+  %p = phi i32 [ 0, %true.1 ], [ 1, %true.2], [ undef, %false ]
+  %ext = uitofp i32 %p to float
+  ret float %ext
+}
+
+define double @test_infer_at_use(i32 noundef %n) {
+; CHECK-LABEL: @test_infer_at_use(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -1
+; CHECK-NEXT:    [[EXT:%.*]] = uitofp i32 [[N]] to double
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], double [[EXT]], double 0.000000e+00
+; CHECK-NEXT:    ret double [[SELECT]]
+;
+  %cmp = icmp sgt i32 %n, -1
+  %ext = uitofp i32 %n to double
+  %select = select i1 %cmp, double %ext, double 0.0
+  ret double %select
+}
-- 
GitLab


From 925a11128c903c8554921c2b5700caf191ae61d6 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed, 20 Mar 2024 17:00:47 -0500
Subject: [PATCH 0091/1206] [CVP] Convert `sitofp` -> `uitofp nneg` and add
 `nneg` flag to `uitofp`

Similiar to the `InstCombine` changes, just furthering the scope of
the canonicalization/`uitofp nneg` support
---
 .../Scalar/CorrelatedValuePropagation.cpp     | 50 ++++++++++++++++---
 .../CorrelatedValuePropagation/sitofp.ll      |  4 +-
 .../CorrelatedValuePropagation/uitofp.ll      |  4 +-
 3 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index 715cdaff9727..50b5fdb56720 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -62,6 +62,7 @@ STATISTIC(NumAShrsConverted, "Number of ashr converted to lshr");
 STATISTIC(NumAShrsRemoved, "Number of ashr removed");
 STATISTIC(NumSRems,     "Number of srem converted to urem");
 STATISTIC(NumSExt,      "Number of sext converted to zext");
+STATISTIC(NumSIToFP,    "Number of sitofp converted to uitofp");
 STATISTIC(NumSICmps,    "Number of signed icmp preds simplified to unsigned");
 STATISTIC(NumAnd,       "Number of ands removed");
 STATISTIC(NumNW,        "Number of no-wrap deductions");
@@ -89,7 +90,7 @@ STATISTIC(NumSMinMax,
           "Number of llvm.s{min,max} intrinsics simplified to unsigned");
 STATISTIC(NumUDivURemsNarrowedExpanded,
           "Number of bound udiv's/urem's expanded");
-STATISTIC(NumZExt, "Number of non-negative deductions");
+STATISTIC(NumNNeg, "Number of zext/uitofp non-negative deductions");
 
 static Constant *getConstantAt(Value *V, Instruction *At, LazyValueInfo *LVI) {
   if (Constant *C = LVI->getConstant(V, At))
@@ -1075,20 +1076,49 @@ static bool processSExt(SExtInst *SDI, LazyValueInfo *LVI) {
   return true;
 }
 
-static bool processZExt(ZExtInst *ZExt, LazyValueInfo *LVI) {
-  if (ZExt->getType()->isVectorTy())
+static bool processPossibleNonNeg(PossiblyNonNegInst *I, LazyValueInfo *LVI) {
+  if (I->getType()->isVectorTy())
     return false;
 
-  if (ZExt->hasNonNeg())
+  if (I->hasNonNeg())
     return false;
 
-  const Use &Base = ZExt->getOperandUse(0);
+  const Use &Base = I->getOperandUse(0);
   if (!LVI->getConstantRangeAtUse(Base, /*UndefAllowed*/ false)
            .isAllNonNegative())
     return false;
 
-  ++NumZExt;
-  ZExt->setNonNeg();
+  ++NumNNeg;
+  I->setNonNeg();
+
+  return true;
+}
+
+static bool processZExt(ZExtInst *ZExt, LazyValueInfo *LVI) {
+  return processPossibleNonNeg(cast<PossiblyNonNegInst>(ZExt), LVI);
+}
+
+static bool processUIToFP(UIToFPInst *UIToFP, LazyValueInfo *LVI) {
+  return processPossibleNonNeg(cast<PossiblyNonNegInst>(UIToFP), LVI);
+}
+
+static bool processSIToFP(SIToFPInst *SIToFP, LazyValueInfo *LVI) {
+  if (SIToFP->getType()->isVectorTy())
+    return false;
+
+  const Use &Base = SIToFP->getOperandUse(0);
+  if (!LVI->getConstantRangeAtUse(Base, /*UndefAllowed*/ false)
+           .isAllNonNegative())
+    return false;
+
+  ++NumSIToFP;
+  auto *UIToFP = CastInst::Create(Instruction::UIToFP, Base, SIToFP->getType(),
+                                  "", SIToFP->getIterator());
+  UIToFP->takeName(SIToFP);
+  UIToFP->setDebugLoc(SIToFP->getDebugLoc());
+  UIToFP->setNonNeg();
+  SIToFP->replaceAllUsesWith(UIToFP);
+  SIToFP->eraseFromParent();
 
   return true;
 }
@@ -1197,6 +1227,12 @@ static bool runImpl(Function &F, LazyValueInfo *LVI, DominatorTree *DT,
       case Instruction::ZExt:
         BBChanged |= processZExt(cast<ZExtInst>(&II), LVI);
         break;
+      case Instruction::UIToFP:
+        BBChanged |= processUIToFP(cast<UIToFPInst>(&II), LVI);
+        break;
+      case Instruction::SIToFP:
+        BBChanged |= processSIToFP(cast<SIToFPInst>(&II), LVI);
+        break;
       case Instruction::Add:
       case Instruction::Sub:
       case Instruction::Mul:
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/sitofp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/sitofp.ll
index 4bc649245d52..83533290e2f6 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/sitofp.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/sitofp.ll
@@ -9,7 +9,7 @@ define void @test1(i32 %n) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -1
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
 ; CHECK:       bb:
-; CHECK-NEXT:    [[EXT_WIDE:%.*]] = sitofp i32 [[N]] to float
+; CHECK-NEXT:    [[EXT_WIDE:%.*]] = uitofp nneg i32 [[N]] to float
 ; CHECK-NEXT:    call void @use.f32(float [[EXT_WIDE]])
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -88,7 +88,7 @@ exit:
 define double @test_infer_at_use(i32 noundef %n) {
 ; CHECK-LABEL: @test_infer_at_use(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -1
-; CHECK-NEXT:    [[EXT:%.*]] = sitofp i32 [[N]] to double
+; CHECK-NEXT:    [[EXT:%.*]] = uitofp nneg i32 [[N]] to double
 ; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], double [[EXT]], double 0.000000e+00
 ; CHECK-NEXT:    ret double [[SELECT]]
 ;
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/uitofp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/uitofp.ll
index 0558ec61e636..32d0f5b4d338 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/uitofp.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/uitofp.ll
@@ -9,7 +9,7 @@ define void @test1(i32 %n) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -1
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[EXIT:%.*]]
 ; CHECK:       bb:
-; CHECK-NEXT:    [[EXT_WIDE:%.*]] = uitofp i32 [[N]] to float
+; CHECK-NEXT:    [[EXT_WIDE:%.*]] = uitofp nneg i32 [[N]] to float
 ; CHECK-NEXT:    call void @use.f32(float [[EXT_WIDE]])
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -87,7 +87,7 @@ exit:
 define double @test_infer_at_use(i32 noundef %n) {
 ; CHECK-LABEL: @test_infer_at_use(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], -1
-; CHECK-NEXT:    [[EXT:%.*]] = uitofp i32 [[N]] to double
+; CHECK-NEXT:    [[EXT:%.*]] = uitofp nneg i32 [[N]] to double
 ; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], double [[EXT]], double 0.000000e+00
 ; CHECK-NEXT:    ret double [[SELECT]]
 ;
-- 
GitLab


From 6243395d7f1da6a2ea813f5d86ba71f91e1070bf Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed, 20 Mar 2024 17:05:03 -0500
Subject: [PATCH 0092/1206] [SCCP] Add `nneg` flag to `uitofp` if its operand
 is non-negative

Similiar to the `InstCombine` changes, just furthering the support of
the `uitofp nneg` support.

Closes #86154
---
 llvm/lib/Transforms/Utils/SCCPSolver.cpp     | 12 +++++++-----
 llvm/test/Transforms/SCCP/ip-ranges-casts.ll |  6 +++---
 llvm/test/Transforms/SCCP/sitofp.ll          |  8 ++++----
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
index c6029b428ed3..ce40e8b31b76 100644
--- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp
+++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
@@ -143,7 +143,7 @@ static bool refineInstruction(SCCPSolver &Solver,
         Changed = true;
       }
     }
-  } else if (isa<ZExtInst>(Inst) && !Inst.hasNonNeg()) {
+  } else if (isa<PossiblyNonNegInst>(Inst) && !Inst.hasNonNeg()) {
     auto Range = GetRange(Inst.getOperand(0));
     if (Range.isAllNonNegative()) {
       Inst.setNonNeg();
@@ -191,14 +191,16 @@ static bool replaceSignedInst(SCCPSolver &Solver,
 
   Instruction *NewInst = nullptr;
   switch (Inst.getOpcode()) {
-  // Note: We do not fold sitofp -> uitofp here because that could be more
-  // expensive in codegen and may not be reversible in the backend.
+  case Instruction::SIToFP:
   case Instruction::SExt: {
-    // If the source value is not negative, this is a zext.
+    // If the source value is not negative, this is a zext/uitofp.
     Value *Op0 = Inst.getOperand(0);
     if (InsertedValues.count(Op0) || !isNonNegative(Op0))
       return false;
-    NewInst = new ZExtInst(Op0, Inst.getType(), "", Inst.getIterator());
+    NewInst = CastInst::Create(Inst.getOpcode() == Instruction::SExt
+                                   ? Instruction::ZExt
+                                   : Instruction::UIToFP,
+                               Op0, Inst.getType(), "", Inst.getIterator());
     NewInst->setNonNeg();
     break;
   }
diff --git a/llvm/test/Transforms/SCCP/ip-ranges-casts.ll b/llvm/test/Transforms/SCCP/ip-ranges-casts.ll
index 05fa04a9fbe0..e8d417546def 100644
--- a/llvm/test/Transforms/SCCP/ip-ranges-casts.ll
+++ b/llvm/test/Transforms/SCCP/ip-ranges-casts.ll
@@ -167,7 +167,7 @@ define i1 @caller.sext() {
 define internal i1 @f.fptosi(i32 %x) {
 ; CHECK-LABEL: define internal i1 @f.fptosi(
 ; CHECK-SAME: i32 [[X:%.*]]) {
-; CHECK-NEXT:    [[TO_DOUBLE:%.*]] = sitofp i32 [[X]] to double
+; CHECK-NEXT:    [[TO_DOUBLE:%.*]] = uitofp nneg i32 [[X]] to double
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd double 0.000000e+00, [[TO_DOUBLE]]
 ; CHECK-NEXT:    [[TO_I32:%.*]] = fptosi double [[ADD]] to i32
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sgt i32 [[TO_I32]], 300
@@ -209,7 +209,7 @@ define i1 @caller.fptosi() {
 define internal i1 @f.fpext(i16 %x) {
 ; CHECK-LABEL: define internal i1 @f.fpext(
 ; CHECK-SAME: i16 [[X:%.*]]) {
-; CHECK-NEXT:    [[TO_FLOAT:%.*]] = sitofp i16 [[X]] to float
+; CHECK-NEXT:    [[TO_FLOAT:%.*]] = uitofp nneg i16 [[X]] to float
 ; CHECK-NEXT:    [[TO_DOUBLE:%.*]] = fpext float [[TO_FLOAT]] to double
 ; CHECK-NEXT:    [[TO_I64:%.*]] = fptoui float [[TO_FLOAT]] to i64
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sgt i64 [[TO_I64]], 300
@@ -293,7 +293,7 @@ define i1 @int_range_to_double_cast(i32 %a) {
 ; CHECK-LABEL: define i1 @int_range_to_double_cast(
 ; CHECK-SAME: i32 [[A:%.*]]) {
 ; CHECK-NEXT:    [[R:%.*]] = and i32 [[A]], 255
-; CHECK-NEXT:    [[T4:%.*]] = sitofp i32 [[R]] to double
+; CHECK-NEXT:    [[T4:%.*]] = uitofp nneg i32 [[R]] to double
 ; CHECK-NEXT:    [[T10:%.*]] = fadd double 0.000000e+00, [[T4]]
 ; CHECK-NEXT:    [[T11:%.*]] = fcmp olt double [[T4]], [[T10]]
 ; CHECK-NEXT:    ret i1 [[T11]]
diff --git a/llvm/test/Transforms/SCCP/sitofp.ll b/llvm/test/Transforms/SCCP/sitofp.ll
index b635263a5726..24f04ae1fccb 100644
--- a/llvm/test/Transforms/SCCP/sitofp.ll
+++ b/llvm/test/Transforms/SCCP/sitofp.ll
@@ -4,7 +4,7 @@
 define float @sitofp_and(i8 %x) {
 ; CHECK-LABEL: @sitofp_and(
 ; CHECK-NEXT:    [[PX:%.*]] = and i8 [[X:%.*]], 127
-; CHECK-NEXT:    [[R:%.*]] = sitofp i8 [[PX]] to float
+; CHECK-NEXT:    [[R:%.*]] = uitofp nneg i8 [[PX]] to float
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %px = and i8 %x, 127
@@ -23,7 +23,7 @@ define half @sitofp_const(i8 %x) {
 define double @sitofp_zext(i7 %x) {
 ; CHECK-LABEL: @sitofp_zext(
 ; CHECK-NEXT:    [[PX:%.*]] = zext i7 [[X:%.*]] to i8
-; CHECK-NEXT:    [[R:%.*]] = sitofp i8 [[PX]] to double
+; CHECK-NEXT:    [[R:%.*]] = uitofp nneg i8 [[PX]] to double
 ; CHECK-NEXT:    ret double [[R]]
 ;
   %px = zext i7 %x to i8
@@ -52,7 +52,7 @@ define float @dominating_condition(i32 %x) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[T:%.*]], label [[F:%.*]]
 ; CHECK:       t:
-; CHECK-NEXT:    [[A:%.*]] = sitofp i32 [[X]] to float
+; CHECK-NEXT:    [[A:%.*]] = uitofp nneg i32 [[X]] to float
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       f:
 ; CHECK-NEXT:    br label [[EXIT]]
@@ -86,7 +86,7 @@ define float @dominating_condition_alt(i32 %x) {
 ; CHECK:       t:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       f:
-; CHECK-NEXT:    [[A:%.*]] = sitofp i32 [[X]] to float
+; CHECK-NEXT:    [[A:%.*]] = uitofp nneg i32 [[X]] to float
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[COND:%.*]] = phi float [ -4.200000e+01, [[T]] ], [ [[A]], [[F]] ]
-- 
GitLab


From 117bda523ea15510d2289020decabef57d89acc0 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 7 May 2024 12:57:01 -0700
Subject: [PATCH 0093/1206] [RISCV] Add unittests for MinVLen/MaxELen/MaxElenFp
 for ParseArchString. NFC

We had tests for ParseNormalizedArchString, but not ParseArchString.
The ParseNormalizedArchString test was not checking MaxElenFp.
---
 .../TargetParser/RISCVISAInfoTest.cpp         | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index a6c21c18c0ec..83b52d0527c3 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -139,6 +139,7 @@ TEST(ParseNormalizedArchString, UpdatesFLenMinVLenMaxELen) {
   EXPECT_EQ(Info.getFLen(), 64U);
   EXPECT_EQ(Info.getMinVLen(), 64U);
   EXPECT_EQ(Info.getMaxELen(), 64U);
+  EXPECT_EQ(Info.getMaxELenFp(), 64U);
 }
 
 TEST(ParseArchString, RejectsInvalidChars) {
@@ -181,6 +182,9 @@ TEST(ParseArchString, AcceptsSupportedBaseISAsAndSetsXLenAndFLen) {
   EXPECT_TRUE(ExtsRV32I.at("i") == (RISCVISAUtils::ExtensionVersion{2, 1}));
   EXPECT_EQ(InfoRV32I.getXLen(), 32U);
   EXPECT_EQ(InfoRV32I.getFLen(), 0U);
+  EXPECT_EQ(InfoRV32I.getMinVLen(), 0U);
+  EXPECT_EQ(InfoRV32I.getMaxELen(), 0U);
+  EXPECT_EQ(InfoRV32I.getMaxELenFp(), 0U);
 
   auto MaybeRV32E = RISCVISAInfo::parseArchString("rv32e", true);
   ASSERT_THAT_EXPECTED(MaybeRV32E, Succeeded());
@@ -190,6 +194,9 @@ TEST(ParseArchString, AcceptsSupportedBaseISAsAndSetsXLenAndFLen) {
   EXPECT_TRUE(ExtsRV32E.at("e") == (RISCVISAUtils::ExtensionVersion{2, 0}));
   EXPECT_EQ(InfoRV32E.getXLen(), 32U);
   EXPECT_EQ(InfoRV32E.getFLen(), 0U);
+  EXPECT_EQ(InfoRV32E.getMinVLen(), 0U);
+  EXPECT_EQ(InfoRV32E.getMaxELen(), 0U);
+  EXPECT_EQ(InfoRV32E.getMaxELenFp(), 0U);
 
   auto MaybeRV32G = RISCVISAInfo::parseArchString("rv32g", true);
   ASSERT_THAT_EXPECTED(MaybeRV32G, Succeeded());
@@ -206,6 +213,9 @@ TEST(ParseArchString, AcceptsSupportedBaseISAsAndSetsXLenAndFLen) {
               (RISCVISAUtils::ExtensionVersion{2, 0}));
   EXPECT_EQ(InfoRV32G.getXLen(), 32U);
   EXPECT_EQ(InfoRV32G.getFLen(), 64U);
+  EXPECT_EQ(InfoRV32G.getMinVLen(), 0U);
+  EXPECT_EQ(InfoRV32G.getMaxELen(), 0U);
+  EXPECT_EQ(InfoRV32G.getMaxELenFp(), 0U);
 
   auto MaybeRV64I = RISCVISAInfo::parseArchString("rv64i", true);
   ASSERT_THAT_EXPECTED(MaybeRV64I, Succeeded());
@@ -215,6 +225,9 @@ TEST(ParseArchString, AcceptsSupportedBaseISAsAndSetsXLenAndFLen) {
   EXPECT_TRUE(ExtsRV64I.at("i") == (RISCVISAUtils::ExtensionVersion{2, 1}));
   EXPECT_EQ(InfoRV64I.getXLen(), 64U);
   EXPECT_EQ(InfoRV64I.getFLen(), 0U);
+  EXPECT_EQ(InfoRV64I.getMinVLen(), 0U);
+  EXPECT_EQ(InfoRV64I.getMaxELen(), 0U);
+  EXPECT_EQ(InfoRV64I.getMaxELenFp(), 0U);
 
   auto MaybeRV64E = RISCVISAInfo::parseArchString("rv64e", true);
   ASSERT_THAT_EXPECTED(MaybeRV64E, Succeeded());
@@ -224,6 +237,9 @@ TEST(ParseArchString, AcceptsSupportedBaseISAsAndSetsXLenAndFLen) {
   EXPECT_TRUE(ExtsRV64E.at("e") == (RISCVISAUtils::ExtensionVersion{2, 0}));
   EXPECT_EQ(InfoRV64E.getXLen(), 64U);
   EXPECT_EQ(InfoRV64E.getFLen(), 0U);
+  EXPECT_EQ(InfoRV64E.getMinVLen(), 0U);
+  EXPECT_EQ(InfoRV64E.getMaxELen(), 0U);
+  EXPECT_EQ(InfoRV64E.getMaxELenFp(), 0U);
 
   auto MaybeRV64G = RISCVISAInfo::parseArchString("rv64g", true);
   ASSERT_THAT_EXPECTED(MaybeRV64G, Succeeded());
@@ -240,6 +256,38 @@ TEST(ParseArchString, AcceptsSupportedBaseISAsAndSetsXLenAndFLen) {
               (RISCVISAUtils::ExtensionVersion{2, 0}));
   EXPECT_EQ(InfoRV64G.getXLen(), 64U);
   EXPECT_EQ(InfoRV64G.getFLen(), 64U);
+  EXPECT_EQ(InfoRV64G.getMinVLen(), 0U);
+  EXPECT_EQ(InfoRV64G.getMaxELen(), 0U);
+  EXPECT_EQ(InfoRV64G.getMaxELenFp(), 0U);
+
+  auto MaybeRV64GCV = RISCVISAInfo::parseArchString("rv64gcv", true);
+  ASSERT_THAT_EXPECTED(MaybeRV64GCV, Succeeded());
+  RISCVISAInfo &InfoRV64GCV = **MaybeRV64GCV;
+  const auto &ExtsRV64GCV = InfoRV64GCV.getExtensions();
+  EXPECT_EQ(ExtsRV64GCV.size(), 17UL);
+  EXPECT_TRUE(ExtsRV64GCV.at("i") == (RISCVISAUtils::ExtensionVersion{2, 1}));
+  EXPECT_TRUE(ExtsRV64GCV.at("m") == (RISCVISAUtils::ExtensionVersion{2, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("a") == (RISCVISAUtils::ExtensionVersion{2, 1}));
+  EXPECT_TRUE(ExtsRV64GCV.at("f") == (RISCVISAUtils::ExtensionVersion{2, 2}));
+  EXPECT_TRUE(ExtsRV64GCV.at("d") == (RISCVISAUtils::ExtensionVersion{2, 2}));
+  EXPECT_TRUE(ExtsRV64GCV.at("c") == (RISCVISAUtils::ExtensionVersion{2, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("zicsr") == (RISCVISAUtils::ExtensionVersion{2, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("zifencei") ==
+              (RISCVISAUtils::ExtensionVersion{2, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("v") == (RISCVISAUtils::ExtensionVersion{1, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("zve32x") == (RISCVISAUtils::ExtensionVersion{1, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("zve32f") == (RISCVISAUtils::ExtensionVersion{1, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("zve64x") == (RISCVISAUtils::ExtensionVersion{1, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("zve64f") == (RISCVISAUtils::ExtensionVersion{1, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("zve64d") == (RISCVISAUtils::ExtensionVersion{1, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("zvl32b") == (RISCVISAUtils::ExtensionVersion{1, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("zvl64b") == (RISCVISAUtils::ExtensionVersion{1, 0}));
+  EXPECT_TRUE(ExtsRV64GCV.at("zvl128b") == (RISCVISAUtils::ExtensionVersion{1, 0}));
+  EXPECT_EQ(InfoRV64GCV.getXLen(), 64U);
+  EXPECT_EQ(InfoRV64GCV.getFLen(), 64U);
+  EXPECT_EQ(InfoRV64GCV.getMinVLen(), 128U);
+  EXPECT_EQ(InfoRV64GCV.getMaxELen(), 64U);
+  EXPECT_EQ(InfoRV64GCV.getMaxELenFp(), 64U);
 }
 
 TEST(ParseArchString, RejectsUnrecognizedExtensionNamesByDefault) {
-- 
GitLab


From dca3a6e562e012940c2b62a4d8dae3afec09caa4 Mon Sep 17 00:00:00 2001
From: Heejin Ahn <aheejin@gmail.com>
Date: Tue, 7 May 2024 13:02:31 -0700
Subject: [PATCH 0094/1206] [WebAssembly] Make EH depend on multivalue and
 reference-types (#91299)

This PR turns on multivalue and reference-types features when
exception-handling feature is turned on, and errors out when disabling
of those dependent features is explicitly requested.

I think doing this would be safe anyway regardless of whether or when we
end up turning on reference-types by default.

We currently don't yet have a experimental flag for the Clang and LLVM
for the new experimental EH yet. But I think it should be fine to turn
those features on even if the LLVM does not yet generate the new EH
instructions, for the same reason we tried to turn them on by default
and the browsers that support EH also support multivalue and
reference-types anyway.
---
 clang/lib/Driver/ToolChains/WebAssembly.cpp | 34 ++++++++++++++++++
 clang/test/Driver/wasm-toolchain.c          | 39 +++++++++++++++++----
 2 files changed, 67 insertions(+), 6 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/WebAssembly.cpp b/clang/lib/Driver/ToolChains/WebAssembly.cpp
index b7c6efab83e8..5b763df9b332 100644
--- a/clang/lib/Driver/ToolChains/WebAssembly.cpp
+++ b/clang/lib/Driver/ToolChains/WebAssembly.cpp
@@ -347,6 +347,23 @@ void WebAssembly::addClangTargetOptions(const ArgList &DriverArgs,
     // Backend needs -wasm-enable-eh to enable Wasm EH
     CC1Args.push_back("-mllvm");
     CC1Args.push_back("-wasm-enable-eh");
+
+    // New Wasm EH spec (adopted in Oct 2023) requires multivalue and
+    // reference-types.
+    if (DriverArgs.hasFlag(options::OPT_mno_multivalue,
+                           options::OPT_mmultivalue, false)) {
+      getDriver().Diag(diag::err_drv_argument_not_allowed_with)
+          << "-fwasm-exceptions" << "-mno-multivalue";
+    }
+    if (DriverArgs.hasFlag(options::OPT_mno_reference_types,
+                           options::OPT_mreference_types, false)) {
+      getDriver().Diag(diag::err_drv_argument_not_allowed_with)
+          << "-fwasm-exceptions" << "-mno-reference-types";
+    }
+    CC1Args.push_back("-target-feature");
+    CC1Args.push_back("+multivalue");
+    CC1Args.push_back("-target-feature");
+    CC1Args.push_back("+reference-types");
   }
 
   for (const Arg *A : DriverArgs.filtered(options::OPT_mllvm)) {
@@ -408,6 +425,23 @@ void WebAssembly::addClangTargetOptions(const ArgList &DriverArgs,
       CC1Args.push_back("+exception-handling");
       // Backend needs '-exception-model=wasm' to use Wasm EH instructions
       CC1Args.push_back("-exception-model=wasm");
+
+      // New Wasm EH spec (adopted in Oct 2023) requires multivalue and
+      // reference-types.
+      if (DriverArgs.hasFlag(options::OPT_mno_multivalue,
+                             options::OPT_mmultivalue, false)) {
+        getDriver().Diag(diag::err_drv_argument_not_allowed_with)
+            << "-mllvm -wasm-enable-sjlj" << "-mno-multivalue";
+      }
+      if (DriverArgs.hasFlag(options::OPT_mno_reference_types,
+                             options::OPT_mreference_types, false)) {
+        getDriver().Diag(diag::err_drv_argument_not_allowed_with)
+            << "-mllvm -wasm-enable-sjlj" << "-mno-reference-types";
+      }
+      CC1Args.push_back("-target-feature");
+      CC1Args.push_back("+multivalue");
+      CC1Args.push_back("-target-feature");
+      CC1Args.push_back("+reference-types");
     }
   }
 }
diff --git a/clang/test/Driver/wasm-toolchain.c b/clang/test/Driver/wasm-toolchain.c
index dabf0ac2433b..7c26c2c13c0b 100644
--- a/clang/test/Driver/wasm-toolchain.c
+++ b/clang/test/Driver/wasm-toolchain.c
@@ -120,11 +120,12 @@
 // RUN:   | FileCheck -check-prefix=EMSCRIPTEN_EH_ALLOWED_WO_ENABLE %s
 // EMSCRIPTEN_EH_ALLOWED_WO_ENABLE: invalid argument '-mllvm -emscripten-cxx-exceptions-allowed' only allowed with '-mllvm -enable-emscripten-cxx-exceptions'
 
-// '-fwasm-exceptions' sets +exception-handling and '-mllvm -wasm-enable-eh'
+// '-fwasm-exceptions' sets +exception-handling, -multivalue, -reference-types
+// and '-mllvm -wasm-enable-eh'
 // RUN: %clang -### --target=wasm32-unknown-unknown \
 // RUN:    --sysroot=/foo %s -fwasm-exceptions 2>&1 \
 // RUN:  | FileCheck -check-prefix=WASM_EXCEPTIONS %s
-// WASM_EXCEPTIONS: "-cc1" {{.*}} "-target-feature" "+exception-handling" "-mllvm" "-wasm-enable-eh"
+// WASM_EXCEPTIONS: "-cc1" {{.*}} "-target-feature" "+exception-handling" "-mllvm" "-wasm-enable-eh" "-target-feature" "+multivalue" "-target-feature" "+reference-types"
 
 // '-fwasm-exceptions' not allowed with '-mno-exception-handling'
 // RUN: not %clang -### --target=wasm32-unknown-unknown \
@@ -132,19 +133,32 @@
 // RUN:   | FileCheck -check-prefix=WASM_EXCEPTIONS_NO_EH %s
 // WASM_EXCEPTIONS_NO_EH: invalid argument '-fwasm-exceptions' not allowed with '-mno-exception-handling'
 
-// '-fwasm-exceptions' not allowed with '-mllvm -enable-emscripten-cxx-exceptions'
+// '-fwasm-exceptions' not allowed with
+// '-mllvm -enable-emscripten-cxx-exceptions'
 // RUN: not %clang -### --target=wasm32-unknown-unknown \
 // RUN:     --sysroot=/foo %s -fwasm-exceptions \
 // RUN:     -mllvm -enable-emscripten-cxx-exceptions 2>&1 \
 // RUN:   | FileCheck -check-prefix=WASM_EXCEPTIONS_EMSCRIPTEN_EH %s
 // WASM_EXCEPTIONS_EMSCRIPTEN_EH: invalid argument '-fwasm-exceptions' not allowed with '-mllvm -enable-emscripten-cxx-exceptions'
 
-// '-mllvm -wasm-enable-sjlj' sets +exception-handling and
-// '-exception-model=wasm'
+// '-fwasm-exceptions' not allowed with '-mno-multivalue'
+// RUN: not %clang -### --target=wasm32-unknown-unknown \
+// RUN:     --sysroot=/foo %s -fwasm-exceptions -mno-multivalue 2>&1 \
+// RUN:   | FileCheck -check-prefix=WASM_EXCEPTIONS_NO_MULTIVALUE %s
+// WASM_EXCEPTIONS_NO_MULTIVALUE: invalid argument '-fwasm-exceptions' not allowed with '-mno-multivalue'
+
+// '-fwasm-exceptions' not allowed with '-mno-reference-types'
+// RUN: not %clang -### --target=wasm32-unknown-unknown \
+// RUN:     --sysroot=/foo %s -fwasm-exceptions -mno-reference-types 2>&1 \
+// RUN:   | FileCheck -check-prefix=WASM_EXCEPTIONS_NO_REFERENCE_TYPES %s
+// WASM_EXCEPTIONS_NO_REFERENCE_TYPES: invalid argument '-fwasm-exceptions' not allowed with '-mno-reference-types'
+
+// '-mllvm -wasm-enable-sjlj' sets +exception-handling, +multivalue,
+// +reference-types  and '-exception-model=wasm'
 // RUN: %clang -### --target=wasm32-unknown-unknown \
 // RUN:    --sysroot=/foo %s -mllvm -wasm-enable-sjlj 2>&1 \
 // RUN:  | FileCheck -check-prefix=WASM_SJLJ %s
-// WASM_SJLJ: "-cc1" {{.*}} "-target-feature" "+exception-handling" "-exception-model=wasm"
+// WASM_SJLJ: "-cc1" {{.*}} "-target-feature" "+exception-handling" "-exception-model=wasm" "-target-feature" "+multivalue" "-target-feature" "+reference-types"
 
 // '-mllvm -wasm-enable-sjlj' not allowed with '-mno-exception-handling'
 // RUN: not %clang -### --target=wasm32-unknown-unknown \
@@ -168,6 +182,19 @@
 // RUN:   | FileCheck -check-prefix=WASM_SJLJ_EMSCRIPTEN_SJLJ %s
 // WASM_SJLJ_EMSCRIPTEN_SJLJ: invalid argument '-mllvm -wasm-enable-sjlj' not allowed with '-mllvm -enable-emscripten-sjlj'
 
+// '-mllvm -wasm-enable-sjlj' not allowed with '-mno-multivalue'
+// RUN: not %clang -### --target=wasm32-unknown-unknown \
+// RUN:     --sysroot=/foo %s -mllvm -wasm-enable-sjlj -mno-multivalue 2>&1 \
+// RUN:   | FileCheck -check-prefix=WASM_SJLJ_NO_MULTIVALUE %s
+// WASM_SJLJ_NO_MULTIVALUE: invalid argument '-mllvm -wasm-enable-sjlj' not allowed with '-mno-multivalue'
+
+// '-mllvm -wasm-enable-sjlj' not allowed with '-mno-reference-types'
+// RUN: not %clang -### --target=wasm32-unknown-unknown \
+// RUN:     --sysroot=/foo %s -mllvm -wasm-enable-sjlj \
+// RUN:     -mno-reference-types 2>&1 \
+// RUN:   | FileCheck -check-prefix=WASM_SJLJ_NO_REFERENCE_TYPES %s
+// WASM_SJLJ_NO_REFERENCE_TYPES: invalid argument '-mllvm -wasm-enable-sjlj' not allowed with '-mno-reference-types'
+
 // RUN: %clang -### %s -fsanitize=address --target=wasm32-unknown-emscripten 2>&1 | FileCheck -check-prefix=CHECK-ASAN-EMSCRIPTEN %s
 // CHECK-ASAN-EMSCRIPTEN: "-fsanitize=address"
 // CHECK-ASAN-EMSCRIPTEN: "-fsanitize-address-globals-dead-stripping"
-- 
GitLab


From 2e4abfae57f81e2bb23fc654d6edbaeae51ae10a Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Tue, 7 May 2024 13:02:04 -0700
Subject: [PATCH 0095/1206] Revert "[Sema] Fix warnings"

This reverts commit 65e2fab401a2da55c51d3caceae8478c33f3c60f because I'm also reverting 7115ed0fff027b65fa76fdfae215ed1382ed1473.
---
 clang/lib/Sema/SemaTemplate.cpp                | 2 ++
 clang/lib/Sema/SemaTemplateInstantiateDecl.cpp | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 6231b65bd842..b268d7c405df 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -9276,6 +9276,8 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
 
   // NOTE: KWLoc is the location of the tag keyword. This will instead
   // store the location of the outermost template keyword in the declaration.
+  SourceLocation TemplateKWLoc = TemplateParameterLists.size() > 0
+    ? TemplateParameterLists[0]->getTemplateLoc() : KWLoc;
   SourceLocation TemplateNameLoc = TemplateId.TemplateNameLoc;
   SourceLocation LAngleLoc = TemplateId.LAngleLoc;
   SourceLocation RAngleLoc = TemplateId.RAngleLoc;
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 884e98a300f5..5315b143215e 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -3932,6 +3932,12 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
   if (SubstQualifier(D, InstD))
     return nullptr;
 
+  // Build the canonical type that describes the converted template
+  // arguments of the class template explicit specialization.
+  QualType CanonType = SemaRef.Context.getTemplateSpecializationType(
+      TemplateName(InstClassTemplate), CanonicalConverted,
+      SemaRef.Context.getRecordType(InstD));
+
   InstD->setAccess(D->getAccess());
   InstD->setInstantiationOfMemberClass(D, TSK_ImplicitInstantiation);
   InstD->setSpecializationKind(D->getSpecializationKind());
-- 
GitLab


From c6855ab24e63feb432aac4f86eb70ac16d76c921 Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Tue, 7 May 2024 13:02:17 -0700
Subject: [PATCH 0096/1206] Revert "[Clang] Unify interface for accessing
 template arguments as written for class/variable template specializations
 (#81642)"

This reverts commit 7115ed0fff027b65fa76fdfae215ed1382ed1473.

This commit broke several LLDB tests.

https://green.lab.llvm.org/job/llvm.org/view/LLDB/job/as-lldb-cmake/3480/
---
 clang-tools-extra/clangd/AST.cpp              |  37 +-
 .../clangd/SemanticHighlighting.cpp           |  13 +-
 .../include-cleaner/lib/WalkAST.cpp           |  13 +-
 clang/docs/LibASTMatchersReference.html       | 364 +++++-------------
 clang/docs/ReleaseNotes.rst                   |   3 -
 clang/include/clang/AST/DeclTemplate.h        | 220 ++++++-----
 clang/include/clang/AST/RecursiveASTVisitor.h |  27 +-
 clang/include/clang/ASTMatchers/ASTMatchers.h |  74 ++--
 .../clang/ASTMatchers/ASTMatchersInternal.h   |  50 +--
 clang/lib/AST/ASTImporter.cpp                 |  68 ++--
 clang/lib/AST/DeclPrinter.cpp                 |  18 +-
 clang/lib/AST/DeclTemplate.cpp                | 198 +++++-----
 clang/lib/AST/TypePrinter.cpp                 |  25 +-
 clang/lib/Index/IndexDecl.cpp                 |   9 +-
 clang/lib/Sema/Sema.cpp                       |   2 +-
 clang/lib/Sema/SemaTemplate.cpp               |  54 +--
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  | 157 +++++---
 clang/lib/Serialization/ASTReaderDecl.cpp     |  28 +-
 clang/lib/Serialization/ASTWriterDecl.cpp     |  36 +-
 clang/lib/Tooling/Syntax/BuildTree.cpp        |   3 +-
 clang/test/AST/ast-dump-template-decls.cpp    |  18 +-
 clang/test/Index/Core/index-source.cpp        |  24 +-
 clang/test/Index/index-refs.cpp               |   1 +
 clang/tools/libclang/CIndex.cpp               |  29 +-
 .../ASTMatchers/ASTMatchersNodeTest.cpp       |  12 +
 .../ASTMatchers/ASTMatchersTraversalTest.cpp  |  92 +++--
 26 files changed, 729 insertions(+), 846 deletions(-)

diff --git a/clang-tools-extra/clangd/AST.cpp b/clang-tools-extra/clangd/AST.cpp
index fda1e5fdf8d8..1b86ea19cf28 100644
--- a/clang-tools-extra/clangd/AST.cpp
+++ b/clang-tools-extra/clangd/AST.cpp
@@ -50,12 +50,17 @@ getTemplateSpecializationArgLocs(const NamedDecl &ND) {
     if (const ASTTemplateArgumentListInfo *Args =
             Func->getTemplateSpecializationArgsAsWritten())
       return Args->arguments();
-  } else if (auto *Cls = llvm::dyn_cast<ClassTemplateSpecializationDecl>(&ND)) {
+  } else if (auto *Cls =
+                 llvm::dyn_cast<ClassTemplatePartialSpecializationDecl>(&ND)) {
     if (auto *Args = Cls->getTemplateArgsAsWritten())
       return Args->arguments();
-  } else if (auto *Var = llvm::dyn_cast<VarTemplateSpecializationDecl>(&ND)) {
+  } else if (auto *Var =
+                 llvm::dyn_cast<VarTemplatePartialSpecializationDecl>(&ND)) {
     if (auto *Args = Var->getTemplateArgsAsWritten())
       return Args->arguments();
+  } else if (auto *Var = llvm::dyn_cast<VarTemplateSpecializationDecl>(&ND)) {
+    if (auto *Args = Var->getTemplateArgsInfo())
+      return Args->arguments();
   }
   // We return std::nullopt for ClassTemplateSpecializationDecls because it does
   // not contain TemplateArgumentLoc information.
@@ -265,10 +270,22 @@ std::string printTemplateSpecializationArgs(const NamedDecl &ND) {
           getTemplateSpecializationArgLocs(ND)) {
     printTemplateArgumentList(OS, *Args, Policy);
   } else if (auto *Cls = llvm::dyn_cast<ClassTemplateSpecializationDecl>(&ND)) {
-    // FIXME: Fix cases when getTypeAsWritten returns null inside clang AST,
-    // e.g. friend decls. Currently we fallback to Template Arguments without
-    // location information.
-    printTemplateArgumentList(OS, Cls->getTemplateArgs().asArray(), Policy);
+    if (const TypeSourceInfo *TSI = Cls->getTypeAsWritten()) {
+      // ClassTemplateSpecializationDecls do not contain
+      // TemplateArgumentTypeLocs, they only have TemplateArgumentTypes. So we
+      // create a new argument location list from TypeSourceInfo.
+      auto STL = TSI->getTypeLoc().getAs<TemplateSpecializationTypeLoc>();
+      llvm::SmallVector<TemplateArgumentLoc> ArgLocs;
+      ArgLocs.reserve(STL.getNumArgs());
+      for (unsigned I = 0; I < STL.getNumArgs(); ++I)
+        ArgLocs.push_back(STL.getArgLoc(I));
+      printTemplateArgumentList(OS, ArgLocs, Policy);
+    } else {
+      // FIXME: Fix cases when getTypeAsWritten returns null inside clang AST,
+      // e.g. friend decls. Currently we fallback to Template Arguments without
+      // location information.
+      printTemplateArgumentList(OS, Cls->getTemplateArgs().asArray(), Policy);
+    }
   }
   OS.flush();
   return TemplateArgs;
@@ -436,12 +453,10 @@ bool hasReservedScope(const DeclContext &DC) {
 }
 
 QualType declaredType(const TypeDecl *D) {
-  ASTContext &Context = D->getASTContext();
   if (const auto *CTSD = llvm::dyn_cast<ClassTemplateSpecializationDecl>(D))
-    if (const auto *Args = CTSD->getTemplateArgsAsWritten())
-      return Context.getTemplateSpecializationType(
-          TemplateName(CTSD->getSpecializedTemplate()), Args->arguments());
-  return Context.getTypeDeclType(D);
+    if (const auto *TSI = CTSD->getTypeAsWritten())
+      return TSI->getType();
+  return D->getASTContext().getTypeDeclType(D);
 }
 
 namespace {
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index eb025f21f361..08f99e11ac9b 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -693,22 +693,17 @@ public:
     return true;
   }
 
-  bool
-  VisitClassTemplateSpecializationDecl(ClassTemplateSpecializationDecl *D) {
-    if (auto *Args = D->getTemplateArgsAsWritten())
-      H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
-    return true;
-  }
-
   bool VisitClassTemplatePartialSpecializationDecl(
       ClassTemplatePartialSpecializationDecl *D) {
     if (auto *TPL = D->getTemplateParameters())
       H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
+    if (auto *Args = D->getTemplateArgsAsWritten())
+      H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
     return true;
   }
 
   bool VisitVarTemplateSpecializationDecl(VarTemplateSpecializationDecl *D) {
-    if (auto *Args = D->getTemplateArgsAsWritten())
+    if (auto *Args = D->getTemplateArgsInfo())
       H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
     return true;
   }
@@ -717,6 +712,8 @@ public:
       VarTemplatePartialSpecializationDecl *D) {
     if (auto *TPL = D->getTemplateParameters())
       H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
+    if (auto *Args = D->getTemplateArgsAsWritten())
+      H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
     return true;
   }
 
diff --git a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
index f7cc9d191236..878067aca017 100644
--- a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
+++ b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
@@ -267,21 +267,18 @@ public:
     return true;
   }
 
-  // Report a reference from explicit specializations/instantiations to the
-  // specialized template. Implicit ones are filtered out by RAV.
+  // Report a reference from explicit specializations to the specialized
+  // template. Implicit ones are filtered out by RAV and explicit instantiations
+  // are already traversed through typelocs.
   bool
   VisitClassTemplateSpecializationDecl(ClassTemplateSpecializationDecl *CTSD) {
-    // if (CTSD->isExplicitSpecialization())
-    if (clang::isTemplateExplicitInstantiationOrSpecialization(
-            CTSD->getTemplateSpecializationKind()))
+    if (CTSD->isExplicitSpecialization())
       report(CTSD->getLocation(),
              CTSD->getSpecializedTemplate()->getTemplatedDecl());
     return true;
   }
   bool VisitVarTemplateSpecializationDecl(VarTemplateSpecializationDecl *VTSD) {
-    // if (VTSD->isExplicitSpecialization())
-    if (clang::isTemplateExplicitInstantiationOrSpecialization(
-            VTSD->getTemplateSpecializationKind()))
+    if (VTSD->isExplicitSpecialization())
       report(VTSD->getLocation(),
              VTSD->getSpecializedTemplate()->getTemplatedDecl());
     return true;
diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html
index a16b9c44ef0e..bb1b68f6671b 100644
--- a/clang/docs/LibASTMatchersReference.html
+++ b/clang/docs/LibASTMatchersReference.html
@@ -3546,35 +3546,33 @@ cxxMethodDecl(isConst()) matches A::foo() but not A::bar()
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isCopyAssignmentOperator0')"><a name="isCopyAssignmentOperator0Anchor">isCopyAssignmentOperator</a></td><td></td></tr>
-<tr><td colspan="4" class="doc" id="isCopyAssignmentOperator0"><pre>Matches if the given method declaration declares a copy assignment
-operator.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isExplicitObjectMemberFunction0')"><a name="isExplicitObjectMemberFunction0Anchor">isExplicitObjectMemberFunction</a></td><td></td></tr>
+<tr><td colspan="4" class="doc" id="isExplicitObjectMemberFunction0"><pre>Matches if the given method declaration declares a member function with an explicit object parameter.
 
 Given
 struct A {
-  A &amp;operator=(const A &amp;);
-  A &amp;operator=(A &amp;&amp;);
+  int operator-(this A, int);
+  void fun(this A &&self);
+  static int operator()(int);
+  int operator+(int);
 };
 
-cxxMethodDecl(isCopyAssignmentOperator()) matches the first method but not
-the second one.
+cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two methods but not the last two.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isExplicitObjectMemberFunction0')"><a name="isExplicitObjectMemberFunction0Anchor">isExplicitObjectMemberFunction</a></td><td></td></tr>
-<tr><td colspan="4" class="doc" id="isExplicitObjectMemberFunction0"><pre>Matches if the given method declaration declares a member function with an
-explicit object parameter.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isCopyAssignmentOperator0')"><a name="isCopyAssignmentOperator0Anchor">isCopyAssignmentOperator</a></td><td></td></tr>
+<tr><td colspan="4" class="doc" id="isCopyAssignmentOperator0"><pre>Matches if the given method declaration declares a copy assignment
+operator.
 
 Given
 struct A {
- int operator-(this A, int);
- void fun(this A &amp;&amp;self);
- static int operator()(int);
- int operator+(int);
+  A &amp;operator=(const A &amp;);
+  A &amp;operator=(A &amp;&amp;);
 };
 
-cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two
-methods but not the last two.
+cxxMethodDecl(isCopyAssignmentOperator()) matches the first method but not
+the second one.
 </pre></td></tr>
 
 
@@ -6715,7 +6713,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -6759,7 +6757,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -6987,7 +6985,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7221,7 +7219,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7418,7 +7416,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7622,7 +7620,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7679,7 +7677,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7877,10 +7875,9 @@ int a = b ?: 1;
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument0')"><a name="forEachTemplateArgument0Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="forEachTemplateArgument0"><pre>Matches templateSpecializationType, class template specialization,
-variable template specialization, and function template specialization
-nodes where the template argument matches the inner matcher. This matcher
-may produce multiple matches.
+<tr><td colspan="4" class="doc" id="forEachTemplateArgument0"><pre>Matches classTemplateSpecialization, templateSpecializationType and
+functionDecl nodes where the template argument matches the inner matcher.
+This matcher may produce multiple matches.
 
 Given
   template &lt;typename T, unsigned N, unsigned M&gt;
@@ -7902,25 +7899,10 @@ functionDecl(forEachTemplateArgument(refersToType(builtinType())))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc0')"><a name="hasAnyTemplateArgumentLoc0Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s, class template specializations,
-variable template specializations, and function template specializations
-that have at least one `TemplateArgumentLoc` matching the given
-`InnerMatcher`.
-
-Given
-  template&lt;typename T&gt; class A {};
-  A&lt;int&gt; a;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-  hasTypeLoc(loc(asString("int")))))))
-  matches `A&lt;int&gt; a`.
-</pre></td></tr>
-
-
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument0')"><a name="hasAnyTemplateArgument0Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument0"><pre>Matches templateSpecializationTypes, class template specializations,
-variable template specializations, and function template specializations
-that have at least one TemplateArgument matching the given InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument0"><pre>Matches classTemplateSpecializations, templateSpecializationType and
+functionDecl that have at least one TemplateArgument matching the given
+InnerMatcher.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -7951,25 +7933,9 @@ classTemplateSpecializationDecl(hasSpecializedTemplate(classTemplateDecl()))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc0')"><a name="hasTemplateArgumentLoc0Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s, class template specializations,
-variable template specializations, and function template specializations
-where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
-
-Given
-  template&lt;typename T, typename U&gt; class A {};
-  A&lt;double, int&gt; b;
-  A&lt;int, double&gt; c;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
-  hasTypeLoc(loc(asString("double")))))))
-  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
-</pre></td></tr>
-
-
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument0')"><a name="hasTemplateArgument0Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgument0"><pre>Matches templateSpecializationType, class template specializations,
-variable template specializations, and function template specializations
-where the n'th TemplateArgument matches the given InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasTemplateArgument0"><pre>Matches classTemplateSpecializations, templateSpecializationType and
+functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -7987,6 +7953,34 @@ functionDecl(hasTemplateArgument(0, refersToType(asString("int"))))
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc7')"><a name="hasTypeLoc7Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc7"><pre>Matches if the type location of a node matches the inner matcher.
+
+Examples:
+  int x;
+declaratorDecl(hasTypeLoc(loc(asString("int"))))
+  matches int x
+
+auto x = int(3);
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
+  matches int(3)
+
+struct Foo { Foo(int, int); };
+auto x = Foo(1, 2);
+cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
+  matches Foo(1, 2)
+
+Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ComplexType.html">ComplexType</a>&gt;</td><td class="name" onclick="toggle('hasElementType1')"><a name="hasElementType1Anchor">hasElementType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td></tr>
 <tr><td colspan="4" class="doc" id="hasElementType1"><pre>Matches arrays and C99 complex types that have a specific element
 type.
@@ -8002,8 +7996,8 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Arra
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc7')"><a name="hasTypeLoc7Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc7"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc8')"><a name="hasTypeLoc8Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc8"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -8023,7 +8017,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -8072,21 +8066,6 @@ with compoundStmt()
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc3')"><a name="hasAnyTemplateArgumentLoc3Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc3"><pre>Matches template specialization `TypeLoc`s, class template specializations,
-variable template specializations, and function template specializations
-that have at least one `TemplateArgumentLoc` matching the given
-`InnerMatcher`.
-
-Given
-  template&lt;typename T&gt; class A {};
-  A&lt;int&gt; a;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-  hasTypeLoc(loc(asString("int")))))))
-  matches `A&lt;int&gt; a`.
-</pre></td></tr>
-
-
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration11')"><a name="hasDeclaration11Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration11"><pre>Matches a node if the declaration associated with that node
 matches the given matcher.
@@ -8121,10 +8100,9 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Addr
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc3')"><a name="hasTemplateArgumentLoc3Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc3"><pre>Matches template specialization `TypeLoc`s, class template specializations,
-variable template specializations, and function template specializations
-where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc0')"><a name="hasTemplateArgumentLoc0Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s where the n'th
+`TemplateArgumentLoc` matches the given `InnerMatcher`.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -8198,8 +8176,8 @@ declStmt(hasSingleDecl(anything()))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc8')"><a name="hasTypeLoc8Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc8"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc9')"><a name="hasTypeLoc9Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc9"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -8219,7 +8197,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -8415,8 +8393,8 @@ actual casts "explicit" casts.)
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc9')"><a name="hasTypeLoc9Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc9"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc10')"><a name="hasTypeLoc10Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc10"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -8436,7 +8414,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -8729,10 +8707,9 @@ Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument2')"><a name="forEachTemplateArgument2Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="forEachTemplateArgument2"><pre>Matches templateSpecializationType, class template specialization,
-variable template specialization, and function template specialization
-nodes where the template argument matches the inner matcher. This matcher
-may produce multiple matches.
+<tr><td colspan="4" class="doc" id="forEachTemplateArgument2"><pre>Matches classTemplateSpecialization, templateSpecializationType and
+functionDecl nodes where the template argument matches the inner matcher.
+This matcher may produce multiple matches.
 
 Given
   template &lt;typename T, unsigned N, unsigned M&gt;
@@ -8801,25 +8778,10 @@ matching y.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc2')"><a name="hasAnyTemplateArgumentLoc2Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc2"><pre>Matches template specialization `TypeLoc`s, class template specializations,
-variable template specializations, and function template specializations
-that have at least one `TemplateArgumentLoc` matching the given
-`InnerMatcher`.
-
-Given
-  template&lt;typename T&gt; class A {};
-  A&lt;int&gt; a;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-  hasTypeLoc(loc(asString("int")))))))
-  matches `A&lt;int&gt; a`.
-</pre></td></tr>
-
-
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument2')"><a name="hasAnyTemplateArgument2Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument2"><pre>Matches templateSpecializationTypes, class template specializations,
-variable template specializations, and function template specializations
-that have at least one TemplateArgument matching the given InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument2"><pre>Matches classTemplateSpecializations, templateSpecializationType and
+functionDecl that have at least one TemplateArgument matching the given
+InnerMatcher.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -8916,25 +8878,9 @@ functionDecl(hasReturnTypeLoc(loc(asString("int"))))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc2')"><a name="hasTemplateArgumentLoc2Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc2"><pre>Matches template specialization `TypeLoc`s, class template specializations,
-variable template specializations, and function template specializations
-where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
-
-Given
-  template&lt;typename T, typename U&gt; class A {};
-  A&lt;double, int&gt; b;
-  A&lt;int, double&gt; c;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
-  hasTypeLoc(loc(asString("double")))))))
-  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
-</pre></td></tr>
-
-
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument2')"><a name="hasTemplateArgument2Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgument2"><pre>Matches templateSpecializationType, class template specializations,
-variable template specializations, and function template specializations
-where the n'th TemplateArgument matches the given InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasTemplateArgument2"><pre>Matches classTemplateSpecializations, templateSpecializationType and
+functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -9527,8 +9473,8 @@ matching y.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc10')"><a name="hasTypeLoc10Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc10"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc11')"><a name="hasTypeLoc11Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc11"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -9548,7 +9494,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -9973,8 +9919,8 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Addr
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc11')"><a name="hasTypeLoc11Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc11"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc12')"><a name="hasTypeLoc12Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc12"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -9994,7 +9940,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -10068,11 +10014,9 @@ matches the specialization of struct A generated by A&lt;X&gt;.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc4')"><a name="hasAnyTemplateArgumentLoc4Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc4"><pre>Matches template specialization `TypeLoc`s, class template specializations,
-variable template specializations, and function template specializations
-that have at least one `TemplateArgumentLoc` matching the given
-`InnerMatcher`.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc0')"><a name="hasAnyTemplateArgumentLoc0Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s that have at least one
+`TemplateArgumentLoc` matching the given `InnerMatcher`.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -10083,10 +10027,9 @@ varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc4')"><a name="hasTemplateArgumentLoc4Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc4"><pre>Matches template specialization `TypeLoc`s, class template specializations,
-variable template specializations, and function template specializations
-where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc1')"><a name="hasTemplateArgumentLoc1Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc1"><pre>Matches template specialization `TypeLoc`s where the n'th
+`TemplateArgumentLoc` matches the given `InnerMatcher`.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -10098,11 +10041,10 @@ varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument3')"><a name="forEachTemplateArgument3Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="forEachTemplateArgument3"><pre>Matches templateSpecializationType, class template specialization,
-variable template specialization, and function template specialization
-nodes where the template argument matches the inner matcher. This matcher
-may produce multiple matches.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument1')"><a name="forEachTemplateArgument1Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="forEachTemplateArgument1"><pre>Matches classTemplateSpecialization, templateSpecializationType and
+functionDecl nodes where the template argument matches the inner matcher.
+This matcher may produce multiple matches.
 
 Given
   template &lt;typename T, unsigned N, unsigned M&gt;
@@ -10124,10 +10066,10 @@ functionDecl(forEachTemplateArgument(refersToType(builtinType())))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument3')"><a name="hasAnyTemplateArgument3Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument3"><pre>Matches templateSpecializationTypes, class template specializations,
-variable template specializations, and function template specializations
-that have at least one TemplateArgument matching the given InnerMatcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument1')"><a name="hasAnyTemplateArgument1Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument1"><pre>Matches classTemplateSpecializations, templateSpecializationType and
+functionDecl that have at least one TemplateArgument matching the given
+InnerMatcher.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -10180,10 +10122,9 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Addr
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument3')"><a name="hasTemplateArgument3Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgument3"><pre>Matches templateSpecializationType, class template specializations,
-variable template specializations, and function template specializations
-where the n'th TemplateArgument matches the given InnerMatcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument1')"><a name="hasTemplateArgument1Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgument1"><pre>Matches classTemplateSpecializations, templateSpecializationType and
+functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -10241,8 +10182,8 @@ QualType-matcher matches.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc12')"><a name="hasTypeLoc12Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc12"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc13')"><a name="hasTypeLoc13Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc13"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -10262,7 +10203,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -10508,105 +10449,6 @@ Example matches x (matcher = varDecl(hasInitializer(callExpr())))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument1')"><a name="forEachTemplateArgument1Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="forEachTemplateArgument1"><pre>Matches templateSpecializationType, class template specialization,
-variable template specialization, and function template specialization
-nodes where the template argument matches the inner matcher. This matcher
-may produce multiple matches.
-
-Given
-  template &lt;typename T, unsigned N, unsigned M&gt;
-  struct Matrix {};
-
-  constexpr unsigned R = 2;
-  Matrix&lt;int, R * 2, R * 4&gt; M;
-
-  template &lt;typename T, typename U&gt;
-  void f(T&amp;&amp; t, U&amp;&amp; u) {}
-
-  bool B = false;
-  f(R, B);
-templateSpecializationType(forEachTemplateArgument(isExpr(expr())))
-  matches twice, with expr() matching 'R * 2' and 'R * 4'
-functionDecl(forEachTemplateArgument(refersToType(builtinType())))
-  matches the specialization f&lt;unsigned, bool&gt; twice, for 'unsigned'
-  and 'bool'
-</pre></td></tr>
-
-
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc1')"><a name="hasAnyTemplateArgumentLoc1Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc1"><pre>Matches template specialization `TypeLoc`s, class template specializations,
-variable template specializations, and function template specializations
-that have at least one `TemplateArgumentLoc` matching the given
-`InnerMatcher`.
-
-Given
-  template&lt;typename T&gt; class A {};
-  A&lt;int&gt; a;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-  hasTypeLoc(loc(asString("int")))))))
-  matches `A&lt;int&gt; a`.
-</pre></td></tr>
-
-
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument1')"><a name="hasAnyTemplateArgument1Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument1"><pre>Matches templateSpecializationTypes, class template specializations,
-variable template specializations, and function template specializations
-that have at least one TemplateArgument matching the given InnerMatcher.
-
-Given
-  template&lt;typename T&gt; class A {};
-  template&lt;&gt; class A&lt;double&gt; {};
-  A&lt;int&gt; a;
-
-  template&lt;typename T&gt; f() {};
-  void func() { f&lt;int&gt;(); };
-
-classTemplateSpecializationDecl(hasAnyTemplateArgument(
-    refersToType(asString("int"))))
-  matches the specialization A&lt;int&gt;
-
-functionDecl(hasAnyTemplateArgument(refersToType(asString("int"))))
-  matches the specialization f&lt;int&gt;
-</pre></td></tr>
-
-
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc1')"><a name="hasTemplateArgumentLoc1Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc1"><pre>Matches template specialization `TypeLoc`s, class template specializations,
-variable template specializations, and function template specializations
-where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
-
-Given
-  template&lt;typename T, typename U&gt; class A {};
-  A&lt;double, int&gt; b;
-  A&lt;int, double&gt; c;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
-  hasTypeLoc(loc(asString("double")))))))
-  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
-</pre></td></tr>
-
-
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument1')"><a name="hasTemplateArgument1Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgument1"><pre>Matches templateSpecializationType, class template specializations,
-variable template specializations, and function template specializations
-where the n'th TemplateArgument matches the given InnerMatcher.
-
-Given
-  template&lt;typename T, typename U&gt; class A {};
-  A&lt;bool, int&gt; b;
-  A&lt;int, bool&gt; c;
-
-  template&lt;typename T&gt; void f() {}
-  void func() { f&lt;int&gt;(); };
-classTemplateSpecializationDecl(hasTemplateArgument(
-    1, refersToType(asString("int"))))
-  matches the specialization A&lt;bool, int&gt;
-
-functionDecl(hasTemplateArgument(0, refersToType(asString("int"))))
-  matches the specialization f&lt;int&gt;
-</pre></td></tr>
-
-
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VariableArrayType.html">VariableArrayType</a>&gt;</td><td class="name" onclick="toggle('hasSizeExpr0')"><a name="hasSizeExpr0Anchor">hasSizeExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasSizeExpr0"><pre>Matches VariableArrayType nodes that have a specific size
 expression.
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 2fae5731566d..cc3108bf41d6 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -111,9 +111,6 @@ Clang Frontend Potentially Breaking Changes
     $ clang --target=<your target triple> -print-target-triple
     <the normalized target triple>
 
-- The ``hasTypeLoc`` AST matcher will no longer match a ``classTemplateSpecializationDecl``;
-  existing uses should switch to ``templateArgumentLoc`` or ``hasAnyTemplateArgumentLoc`` instead.
-
 What's New in Clang |release|?
 ==============================
 Some of the major new features and improvements to Clang are listed
diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index 36fb7ec80c17..3ee03eebdb8c 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -1776,25 +1776,6 @@ public:
   BuiltinTemplateKind getBuiltinTemplateKind() const { return BTK; }
 };
 
-/// Provides information about an explicit instantiation of a variable or class
-/// template.
-struct ExplicitInstantiationInfo {
-  /// The template arguments as written..
-  const ASTTemplateArgumentListInfo *TemplateArgsAsWritten = nullptr;
-
-  /// The location of the extern keyword.
-  SourceLocation ExternKeywordLoc;
-
-  /// The location of the template keyword.
-  SourceLocation TemplateKeywordLoc;
-
-  ExplicitInstantiationInfo() = default;
-};
-
-using SpecializationOrInstantiationInfo =
-    llvm::PointerUnion<const ASTTemplateArgumentListInfo *,
-                       ExplicitInstantiationInfo *>;
-
 /// Represents a class template specialization, which refers to
 /// a class template with a given set of template arguments.
 ///
@@ -1808,8 +1789,8 @@ using SpecializationOrInstantiationInfo =
 /// template<>
 /// class array<bool> { }; // class template specialization array<bool>
 /// \endcode
-class ClassTemplateSpecializationDecl : public CXXRecordDecl,
-                                        public llvm::FoldingSetNode {
+class ClassTemplateSpecializationDecl
+  : public CXXRecordDecl, public llvm::FoldingSetNode {
   /// Structure that stores information about a class template
   /// specialization that was instantiated from a class template partial
   /// specialization.
@@ -1827,9 +1808,23 @@ class ClassTemplateSpecializationDecl : public CXXRecordDecl,
   llvm::PointerUnion<ClassTemplateDecl *, SpecializedPartialSpecialization *>
     SpecializedTemplate;
 
+  /// Further info for explicit template specialization/instantiation.
+  struct ExplicitSpecializationInfo {
+    /// The type-as-written.
+    TypeSourceInfo *TypeAsWritten = nullptr;
+
+    /// The location of the extern keyword.
+    SourceLocation ExternLoc;
+
+    /// The location of the template keyword.
+    SourceLocation TemplateKeywordLoc;
+
+    ExplicitSpecializationInfo() = default;
+  };
+
   /// Further info for explicit template specialization/instantiation.
   /// Does not apply to implicit specializations.
-  SpecializationOrInstantiationInfo ExplicitInfo = nullptr;
+  ExplicitSpecializationInfo *ExplicitInfo = nullptr;
 
   /// The template arguments used to describe this specialization.
   const TemplateArgumentList *TemplateArgs;
@@ -2006,49 +2001,44 @@ public:
     SpecializedTemplate = TemplDecl;
   }
 
-  /// Retrieve the template argument list as written in the sources,
-  /// if any.
-  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
-      return Info->TemplateArgsAsWritten;
-    return ExplicitInfo.get<const ASTTemplateArgumentListInfo *>();
+  /// Sets the type of this specialization as it was written by
+  /// the user. This will be a class template specialization type.
+  void setTypeAsWritten(TypeSourceInfo *T) {
+    if (!ExplicitInfo)
+      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
+    ExplicitInfo->TypeAsWritten = T;
   }
 
-  /// Set the template argument list as written in the sources.
-  void
-  setTemplateArgsAsWritten(const ASTTemplateArgumentListInfo *ArgsWritten) {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
-      Info->TemplateArgsAsWritten = ArgsWritten;
-    else
-      ExplicitInfo = ArgsWritten;
-  }
-
-  /// Set the template argument list as written in the sources.
-  void setTemplateArgsAsWritten(const TemplateArgumentListInfo &ArgsInfo) {
-    setTemplateArgsAsWritten(
-        ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo));
+  /// Gets the type of this specialization as it was written by
+  /// the user, if it was so written.
+  TypeSourceInfo *getTypeAsWritten() const {
+    return ExplicitInfo ? ExplicitInfo->TypeAsWritten : nullptr;
   }
 
   /// Gets the location of the extern keyword, if present.
-  SourceLocation getExternKeywordLoc() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
-      return Info->ExternKeywordLoc;
-    return SourceLocation();
+  SourceLocation getExternLoc() const {
+    return ExplicitInfo ? ExplicitInfo->ExternLoc : SourceLocation();
   }
 
   /// Sets the location of the extern keyword.
-  void setExternKeywordLoc(SourceLocation Loc);
+  void setExternLoc(SourceLocation Loc) {
+    if (!ExplicitInfo)
+      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
+    ExplicitInfo->ExternLoc = Loc;
+  }
+
+  /// Sets the location of the template keyword.
+  void setTemplateKeywordLoc(SourceLocation Loc) {
+    if (!ExplicitInfo)
+      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
+    ExplicitInfo->TemplateKeywordLoc = Loc;
+  }
 
   /// Gets the location of the template keyword, if present.
   SourceLocation getTemplateKeywordLoc() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
-      return Info->TemplateKeywordLoc;
-    return SourceLocation();
+    return ExplicitInfo ? ExplicitInfo->TemplateKeywordLoc : SourceLocation();
   }
 
-  /// Sets the location of the template keyword.
-  void setTemplateKeywordLoc(SourceLocation Loc);
-
   SourceRange getSourceRange() const override LLVM_READONLY;
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
@@ -2076,6 +2066,10 @@ class ClassTemplatePartialSpecializationDecl
   /// The list of template parameters
   TemplateParameterList* TemplateParams = nullptr;
 
+  /// The source info for the template arguments as written.
+  /// FIXME: redundant with TypeAsWritten?
+  const ASTTemplateArgumentListInfo *ArgsAsWritten = nullptr;
+
   /// The class template partial specialization from which this
   /// class template partial specialization was instantiated.
   ///
@@ -2084,11 +2078,15 @@ class ClassTemplatePartialSpecializationDecl
   llvm::PointerIntPair<ClassTemplatePartialSpecializationDecl *, 1, bool>
       InstantiatedFromMember;
 
-  ClassTemplatePartialSpecializationDecl(
-      ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
-      SourceLocation IdLoc, TemplateParameterList *Params,
-      ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
-      ClassTemplatePartialSpecializationDecl *PrevDecl);
+  ClassTemplatePartialSpecializationDecl(ASTContext &Context, TagKind TK,
+                                         DeclContext *DC,
+                                         SourceLocation StartLoc,
+                                         SourceLocation IdLoc,
+                                         TemplateParameterList *Params,
+                                         ClassTemplateDecl *SpecializedTemplate,
+                                         ArrayRef<TemplateArgument> Args,
+                               const ASTTemplateArgumentListInfo *ArgsAsWritten,
+                               ClassTemplatePartialSpecializationDecl *PrevDecl);
 
   ClassTemplatePartialSpecializationDecl(ASTContext &C)
     : ClassTemplateSpecializationDecl(C, ClassTemplatePartialSpecialization),
@@ -2103,8 +2101,11 @@ public:
   static ClassTemplatePartialSpecializationDecl *
   Create(ASTContext &Context, TagKind TK, DeclContext *DC,
          SourceLocation StartLoc, SourceLocation IdLoc,
-         TemplateParameterList *Params, ClassTemplateDecl *SpecializedTemplate,
-         ArrayRef<TemplateArgument> Args, QualType CanonInjectedType,
+         TemplateParameterList *Params,
+         ClassTemplateDecl *SpecializedTemplate,
+         ArrayRef<TemplateArgument> Args,
+         const TemplateArgumentListInfo &ArgInfos,
+         QualType CanonInjectedType,
          ClassTemplatePartialSpecializationDecl *PrevDecl);
 
   static ClassTemplatePartialSpecializationDecl *
@@ -2135,6 +2136,11 @@ public:
     return TemplateParams->hasAssociatedConstraints();
   }
 
+  /// Get the template arguments as written.
+  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
+    return ArgsAsWritten;
+  }
+
   /// Retrieve the member class template partial specialization from
   /// which this particular class template partial specialization was
   /// instantiated.
@@ -2607,12 +2613,27 @@ class VarTemplateSpecializationDecl : public VarDecl,
   llvm::PointerUnion<VarTemplateDecl *, SpecializedPartialSpecialization *>
   SpecializedTemplate;
 
+  /// Further info for explicit template specialization/instantiation.
+  struct ExplicitSpecializationInfo {
+    /// The type-as-written.
+    TypeSourceInfo *TypeAsWritten = nullptr;
+
+    /// The location of the extern keyword.
+    SourceLocation ExternLoc;
+
+    /// The location of the template keyword.
+    SourceLocation TemplateKeywordLoc;
+
+    ExplicitSpecializationInfo() = default;
+  };
+
   /// Further info for explicit template specialization/instantiation.
   /// Does not apply to implicit specializations.
-  SpecializationOrInstantiationInfo ExplicitInfo = nullptr;
+  ExplicitSpecializationInfo *ExplicitInfo = nullptr;
 
   /// The template arguments used to describe this specialization.
   const TemplateArgumentList *TemplateArgs;
+  const ASTTemplateArgumentListInfo *TemplateArgsInfo = nullptr;
 
   /// The point where this template was instantiated (if any).
   SourceLocation PointOfInstantiation;
@@ -2666,6 +2687,14 @@ public:
   /// specialization.
   const TemplateArgumentList &getTemplateArgs() const { return *TemplateArgs; }
 
+  // TODO: Always set this when creating the new specialization?
+  void setTemplateArgsInfo(const TemplateArgumentListInfo &ArgsInfo);
+  void setTemplateArgsInfo(const ASTTemplateArgumentListInfo *ArgsInfo);
+
+  const ASTTemplateArgumentListInfo *getTemplateArgsInfo() const {
+    return TemplateArgsInfo;
+  }
+
   /// Determine the kind of specialization that this
   /// declaration represents.
   TemplateSpecializationKind getSpecializationKind() const {
@@ -2769,49 +2798,44 @@ public:
     SpecializedTemplate = TemplDecl;
   }
 
-  /// Retrieve the template argument list as written in the sources,
-  /// if any.
-  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
-      return Info->TemplateArgsAsWritten;
-    return ExplicitInfo.get<const ASTTemplateArgumentListInfo *>();
-  }
-
-  /// Set the template argument list as written in the sources.
-  void
-  setTemplateArgsAsWritten(const ASTTemplateArgumentListInfo *ArgsWritten) {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
-      Info->TemplateArgsAsWritten = ArgsWritten;
-    else
-      ExplicitInfo = ArgsWritten;
+  /// Sets the type of this specialization as it was written by
+  /// the user.
+  void setTypeAsWritten(TypeSourceInfo *T) {
+    if (!ExplicitInfo)
+      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
+    ExplicitInfo->TypeAsWritten = T;
   }
 
-  /// Set the template argument list as written in the sources.
-  void setTemplateArgsAsWritten(const TemplateArgumentListInfo &ArgsInfo) {
-    setTemplateArgsAsWritten(
-        ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo));
+  /// Gets the type of this specialization as it was written by
+  /// the user, if it was so written.
+  TypeSourceInfo *getTypeAsWritten() const {
+    return ExplicitInfo ? ExplicitInfo->TypeAsWritten : nullptr;
   }
 
   /// Gets the location of the extern keyword, if present.
-  SourceLocation getExternKeywordLoc() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
-      return Info->ExternKeywordLoc;
-    return SourceLocation();
+  SourceLocation getExternLoc() const {
+    return ExplicitInfo ? ExplicitInfo->ExternLoc : SourceLocation();
   }
 
   /// Sets the location of the extern keyword.
-  void setExternKeywordLoc(SourceLocation Loc);
+  void setExternLoc(SourceLocation Loc) {
+    if (!ExplicitInfo)
+      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
+    ExplicitInfo->ExternLoc = Loc;
+  }
+
+  /// Sets the location of the template keyword.
+  void setTemplateKeywordLoc(SourceLocation Loc) {
+    if (!ExplicitInfo)
+      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
+    ExplicitInfo->TemplateKeywordLoc = Loc;
+  }
 
   /// Gets the location of the template keyword, if present.
   SourceLocation getTemplateKeywordLoc() const {
-    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
-      return Info->TemplateKeywordLoc;
-    return SourceLocation();
+    return ExplicitInfo ? ExplicitInfo->TemplateKeywordLoc : SourceLocation();
   }
 
-  /// Sets the location of the template keyword.
-  void setTemplateKeywordLoc(SourceLocation Loc);
-
   SourceRange getSourceRange() const override LLVM_READONLY;
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
@@ -2839,6 +2863,10 @@ class VarTemplatePartialSpecializationDecl
   /// The list of template parameters
   TemplateParameterList *TemplateParams = nullptr;
 
+  /// The source info for the template arguments as written.
+  /// FIXME: redundant with TypeAsWritten?
+  const ASTTemplateArgumentListInfo *ArgsAsWritten = nullptr;
+
   /// The variable template partial specialization from which this
   /// variable template partial specialization was instantiated.
   ///
@@ -2851,7 +2879,8 @@ class VarTemplatePartialSpecializationDecl
       ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
       SourceLocation IdLoc, TemplateParameterList *Params,
       VarTemplateDecl *SpecializedTemplate, QualType T, TypeSourceInfo *TInfo,
-      StorageClass S, ArrayRef<TemplateArgument> Args);
+      StorageClass S, ArrayRef<TemplateArgument> Args,
+      const ASTTemplateArgumentListInfo *ArgInfos);
 
   VarTemplatePartialSpecializationDecl(ASTContext &Context)
       : VarTemplateSpecializationDecl(VarTemplatePartialSpecialization,
@@ -2868,8 +2897,8 @@ public:
   Create(ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
          SourceLocation IdLoc, TemplateParameterList *Params,
          VarTemplateDecl *SpecializedTemplate, QualType T,
-         TypeSourceInfo *TInfo, StorageClass S,
-         ArrayRef<TemplateArgument> Args);
+         TypeSourceInfo *TInfo, StorageClass S, ArrayRef<TemplateArgument> Args,
+         const TemplateArgumentListInfo &ArgInfos);
 
   static VarTemplatePartialSpecializationDecl *
   CreateDeserialized(ASTContext &C, GlobalDeclID ID);
@@ -2885,6 +2914,11 @@ public:
     return TemplateParams;
   }
 
+  /// Get the template arguments as written.
+  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
+    return ArgsAsWritten;
+  }
+
   /// \brief All associated constraints of this partial specialization,
   /// including the requires clause and any constraints derived from
   /// constrained-parameters.
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 782f60844506..f9b145b4e86a 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -2030,15 +2030,6 @@ DEF_TRAVERSE_DECL(RecordDecl, { TRY_TO(TraverseRecordHelper(D)); })
 
 DEF_TRAVERSE_DECL(CXXRecordDecl, { TRY_TO(TraverseCXXRecordHelper(D)); })
 
-template <typename Derived>
-bool RecursiveASTVisitor<Derived>::TraverseTemplateArgumentLocsHelper(
-    const TemplateArgumentLoc *TAL, unsigned Count) {
-  for (unsigned I = 0; I < Count; ++I) {
-    TRY_TO(TraverseTemplateArgumentLoc(TAL[I]));
-  }
-  return true;
-}
-
 #define DEF_TRAVERSE_TMPL_SPEC_DECL(TMPLDECLKIND, DECLKIND)                    \
   DEF_TRAVERSE_DECL(TMPLDECLKIND##TemplateSpecializationDecl, {                \
     /* For implicit instantiations ("set<int> x;"), we don't want to           \
@@ -2048,12 +2039,9 @@ bool RecursiveASTVisitor<Derived>::TraverseTemplateArgumentLocsHelper(
        TemplateSpecializationType).  For explicit instantiations               \
        ("template set<int>;"), we do need a callback, since this               \
        is the only callback that's made for this instantiation.                \
-       We use getTemplateArgsAsWritten() to distinguish. */                    \
-    if (const auto *ArgsWritten = D->getTemplateArgsAsWritten()) {             \
-      /* The args that remains unspecialized. */                               \
-      TRY_TO(TraverseTemplateArgumentLocsHelper(                               \
-          ArgsWritten->getTemplateArgs(), ArgsWritten->NumTemplateArgs));      \
-    }                                                                          \
+       We use getTypeAsWritten() to distinguish. */                            \
+    if (TypeSourceInfo *TSI = D->getTypeAsWritten())                           \
+      TRY_TO(TraverseTypeLoc(TSI->getTypeLoc()));                              \
                                                                                \
     if (getDerived().shouldVisitTemplateInstantiations() ||                    \
         D->getTemplateSpecializationKind() == TSK_ExplicitSpecialization) {    \
@@ -2073,6 +2061,15 @@ bool RecursiveASTVisitor<Derived>::TraverseTemplateArgumentLocsHelper(
 DEF_TRAVERSE_TMPL_SPEC_DECL(Class, CXXRecord)
 DEF_TRAVERSE_TMPL_SPEC_DECL(Var, Var)
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::TraverseTemplateArgumentLocsHelper(
+    const TemplateArgumentLoc *TAL, unsigned Count) {
+  for (unsigned I = 0; I < Count; ++I) {
+    TRY_TO(TraverseTemplateArgumentLoc(TAL[I]));
+  }
+  return true;
+}
+
 #define DEF_TRAVERSE_TMPL_PART_SPEC_DECL(TMPLDECLKIND, DECLKIND)               \
   DEF_TRAVERSE_DECL(TMPLDECLKIND##TemplatePartialSpecializationDecl, {         \
     /* The partial specialization. */                                          \
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 0f3257db6f41..8a2bbfff9e9e 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -764,9 +764,9 @@ AST_POLYMORPHIC_MATCHER(isImplicit,
   return Node.isImplicit();
 }
 
-/// Matches templateSpecializationTypes, class template specializations,
-/// variable template specializations, and function template specializations
-/// that have at least one TemplateArgument matching the given InnerMatcher.
+/// Matches classTemplateSpecializations, templateSpecializationType and
+/// functionDecl that have at least one TemplateArgument matching the given
+/// InnerMatcher.
 ///
 /// Given
 /// \code
@@ -788,8 +788,8 @@ AST_POLYMORPHIC_MATCHER(isImplicit,
 AST_POLYMORPHIC_MATCHER_P(
     hasAnyTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
-                                    VarTemplateSpecializationDecl, FunctionDecl,
-                                    TemplateSpecializationType),
+                                    TemplateSpecializationType,
+                                    FunctionDecl),
     internal::Matcher<TemplateArgument>, InnerMatcher) {
   ArrayRef<TemplateArgument> List =
       internal::getTemplateSpecializationArgs(Node);
@@ -1047,9 +1047,8 @@ AST_MATCHER(Expr, isTypeDependent) { return Node.isTypeDependent(); }
 /// expr(isValueDependent()) matches return Size
 AST_MATCHER(Expr, isValueDependent) { return Node.isValueDependent(); }
 
-/// Matches templateSpecializationType, class template specializations,
-/// variable template specializations, and function template specializations
-/// where the n'th TemplateArgument matches the given InnerMatcher.
+/// Matches classTemplateSpecializations, templateSpecializationType and
+/// functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
 ///
 /// Given
 /// \code
@@ -1069,8 +1068,8 @@ AST_MATCHER(Expr, isValueDependent) { return Node.isValueDependent(); }
 AST_POLYMORPHIC_MATCHER_P2(
     hasTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
-                                    VarTemplateSpecializationDecl, FunctionDecl,
-                                    TemplateSpecializationType),
+                                    TemplateSpecializationType,
+                                    FunctionDecl),
     unsigned, N, internal::Matcher<TemplateArgument>, InnerMatcher) {
   ArrayRef<TemplateArgument> List =
       internal::getTemplateSpecializationArgs(Node);
@@ -4067,7 +4066,7 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 ///   Matcher<CXXCtorInitializer>, Matcher<CXXFunctionalCastExpr>,
 ///   Matcher<CXXNewExpr>, Matcher<CXXTemporaryObjectExpr>,
 ///   Matcher<CXXUnresolvedConstructExpr>,
-///   Matcher<CompoundLiteralExpr>,
+///   Matcher<ClassTemplateSpecializationDecl>, Matcher<CompoundLiteralExpr>,
 ///   Matcher<DeclaratorDecl>, Matcher<ExplicitCastExpr>,
 ///   Matcher<ObjCPropertyDecl>, Matcher<TemplateArgumentLoc>,
 ///   Matcher<TypedefNameDecl>
@@ -4076,8 +4075,9 @@ AST_POLYMORPHIC_MATCHER_P(
     AST_POLYMORPHIC_SUPPORTED_TYPES(
         BlockDecl, CXXBaseSpecifier, CXXCtorInitializer, CXXFunctionalCastExpr,
         CXXNewExpr, CXXTemporaryObjectExpr, CXXUnresolvedConstructExpr,
-        CompoundLiteralExpr, DeclaratorDecl, ExplicitCastExpr, ObjCPropertyDecl,
-        TemplateArgumentLoc, TypedefNameDecl),
+        ClassTemplateSpecializationDecl, CompoundLiteralExpr, DeclaratorDecl,
+        ExplicitCastExpr, ObjCPropertyDecl, TemplateArgumentLoc,
+        TypedefNameDecl),
     internal::Matcher<TypeLoc>, Inner) {
   TypeSourceInfo *source = internal::GetTypeSourceInfo(Node);
   if (source == nullptr) {
@@ -5304,10 +5304,9 @@ AST_POLYMORPHIC_MATCHER_P(parameterCountIs,
   return Node.getNumParams() == N;
 }
 
-/// Matches templateSpecializationType, class template specialization,
-/// variable template specialization, and function template specialization
-/// nodes where the template argument matches the inner matcher. This matcher
-/// may produce multiple matches.
+/// Matches classTemplateSpecialization, templateSpecializationType and
+/// functionDecl nodes where the template argument matches the inner matcher.
+/// This matcher may produce multiple matches.
 ///
 /// Given
 /// \code
@@ -5331,8 +5330,7 @@ AST_POLYMORPHIC_MATCHER_P(parameterCountIs,
 AST_POLYMORPHIC_MATCHER_P(
     forEachTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
-                                    VarTemplateSpecializationDecl, FunctionDecl,
-                                    TemplateSpecializationType),
+                                    TemplateSpecializationType, FunctionDecl),
     internal::Matcher<TemplateArgument>, InnerMatcher) {
   ArrayRef<TemplateArgument> TemplateArgs =
       clang::ast_matchers::internal::getTemplateSpecializationArgs(Node);
@@ -6907,10 +6905,8 @@ extern const internal::VariadicDynCastAllOfMatcher<
     TypeLoc, TemplateSpecializationTypeLoc>
     templateSpecializationTypeLoc;
 
-/// Matches template specialization `TypeLoc`s, class template specializations,
-/// variable template specializations, and function template specializations
-/// that have at least one `TemplateArgumentLoc` matching the given
-/// `InnerMatcher`.
+/// Matches template specialization `TypeLoc`s that have at least one
+/// `TemplateArgumentLoc` matching the given `InnerMatcher`.
 ///
 /// Given
 /// \code
@@ -6920,21 +6916,20 @@ extern const internal::VariadicDynCastAllOfMatcher<
 /// varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
 ///   hasTypeLoc(loc(asString("int")))))))
 ///   matches `A<int> a`.
-AST_POLYMORPHIC_MATCHER_P(
-    hasAnyTemplateArgumentLoc,
-    AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
-                                    VarTemplateSpecializationDecl, FunctionDecl,
-                                    DeclRefExpr, TemplateSpecializationTypeLoc),
-    internal::Matcher<TemplateArgumentLoc>, InnerMatcher) {
-  auto Args = internal::getTemplateArgsWritten(Node);
-  return matchesFirstInRange(InnerMatcher, Args.begin(), Args.end(), Finder,
-                             Builder) != Args.end();
+AST_MATCHER_P(TemplateSpecializationTypeLoc, hasAnyTemplateArgumentLoc,
+              internal::Matcher<TemplateArgumentLoc>, InnerMatcher) {
+  for (unsigned Index = 0, N = Node.getNumArgs(); Index < N; ++Index) {
+    clang::ast_matchers::internal::BoundNodesTreeBuilder Result(*Builder);
+    if (InnerMatcher.matches(Node.getArgLoc(Index), Finder, &Result)) {
+      *Builder = std::move(Result);
+      return true;
+    }
+  }
   return false;
 }
 
-/// Matches template specialization `TypeLoc`s, class template specializations,
-/// variable template specializations, and function template specializations
-/// where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
+/// Matches template specialization `TypeLoc`s where the n'th
+/// `TemplateArgumentLoc` matches the given `InnerMatcher`.
 ///
 /// Given
 /// \code
@@ -6947,13 +6942,10 @@ AST_POLYMORPHIC_MATCHER_P(
 ///   matches `A<double, int> b`, but not `A<int, double> c`.
 AST_POLYMORPHIC_MATCHER_P2(
     hasTemplateArgumentLoc,
-    AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
-                                    VarTemplateSpecializationDecl, FunctionDecl,
-                                    DeclRefExpr, TemplateSpecializationTypeLoc),
+    AST_POLYMORPHIC_SUPPORTED_TYPES(DeclRefExpr, TemplateSpecializationTypeLoc),
     unsigned, Index, internal::Matcher<TemplateArgumentLoc>, InnerMatcher) {
-  auto Args = internal::getTemplateArgsWritten(Node);
-  return Index < Args.size() &&
-         InnerMatcher.matches(Args[Index], Finder, Builder);
+  return internal::MatchTemplateArgLocAt(Node, Index, InnerMatcher, Finder,
+                                         Builder);
 }
 
 /// Matches C or C++ elaborated `TypeLoc`s.
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
index c1cc63fdb743..47d912c73dd7 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
@@ -186,6 +186,10 @@ inline TypeSourceInfo *GetTypeSourceInfo(const BlockDecl &Node) {
 inline TypeSourceInfo *GetTypeSourceInfo(const CXXNewExpr &Node) {
   return Node.getAllocatedTypeSourceInfo();
 }
+inline TypeSourceInfo *
+GetTypeSourceInfo(const ClassTemplateSpecializationDecl &Node) {
+  return Node.getTypeAsWritten();
+}
 
 /// Unifies obtaining the FunctionProtoType pointer from both
 /// FunctionProtoType and FunctionDecl nodes..
@@ -1935,11 +1939,6 @@ getTemplateSpecializationArgs(const ClassTemplateSpecializationDecl &D) {
   return D.getTemplateArgs().asArray();
 }
 
-inline ArrayRef<TemplateArgument>
-getTemplateSpecializationArgs(const VarTemplateSpecializationDecl &D) {
-  return D.getTemplateArgs().asArray();
-}
-
 inline ArrayRef<TemplateArgument>
 getTemplateSpecializationArgs(const TemplateSpecializationType &T) {
   return T.template_arguments();
@@ -1949,46 +1948,7 @@ inline ArrayRef<TemplateArgument>
 getTemplateSpecializationArgs(const FunctionDecl &FD) {
   if (const auto* TemplateArgs = FD.getTemplateSpecializationArgs())
     return TemplateArgs->asArray();
-  return std::nullopt;
-}
-
-inline ArrayRef<TemplateArgumentLoc>
-getTemplateArgsWritten(const ClassTemplateSpecializationDecl &D) {
-  if (const ASTTemplateArgumentListInfo *Args = D.getTemplateArgsAsWritten())
-    return Args->arguments();
-  return std::nullopt;
-}
-
-inline ArrayRef<TemplateArgumentLoc>
-getTemplateArgsWritten(const VarTemplateSpecializationDecl &D) {
-  if (const ASTTemplateArgumentListInfo *Args = D.getTemplateArgsAsWritten())
-    return Args->arguments();
-  return std::nullopt;
-}
-
-inline ArrayRef<TemplateArgumentLoc>
-getTemplateArgsWritten(const FunctionDecl &FD) {
-  if (const auto *Args = FD.getTemplateSpecializationArgsAsWritten())
-    return Args->arguments();
-  return std::nullopt;
-}
-
-inline ArrayRef<TemplateArgumentLoc>
-getTemplateArgsWritten(const DeclRefExpr &DRE) {
-  if (const auto *Args = DRE.getTemplateArgs())
-    return {Args, DRE.getNumTemplateArgs()};
-  return std::nullopt;
-}
-
-inline SmallVector<TemplateArgumentLoc>
-getTemplateArgsWritten(const TemplateSpecializationTypeLoc &T) {
-  SmallVector<TemplateArgumentLoc> Args;
-  if (!T.isNull()) {
-    Args.reserve(T.getNumArgs());
-    for (unsigned I = 0; I < T.getNumArgs(); ++I)
-      Args.emplace_back(T.getArgLoc(I));
-  }
-  return Args;
+  return ArrayRef<TemplateArgument>();
 }
 
 struct NotEqualsBoundNodePredicate {
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 9ff8e1ea78d8..60f213322b34 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -443,9 +443,8 @@ namespace clang {
     Expected<FunctionTemplateAndArgsTy>
     ImportFunctionTemplateWithTemplateArgsFromSpecialization(
         FunctionDecl *FromFD);
-
-    template <typename DeclTy>
-    Error ImportTemplateParameterLists(const DeclTy *FromD, DeclTy *ToD);
+    Error ImportTemplateParameterLists(const DeclaratorDecl *FromD,
+                                       DeclaratorDecl *ToD);
 
     Error ImportTemplateInformation(FunctionDecl *FromFD, FunctionDecl *ToFD);
 
@@ -3323,9 +3322,8 @@ ExpectedDecl ASTNodeImporter::VisitEnumConstantDecl(EnumConstantDecl *D) {
   return ToEnumerator;
 }
 
-template <typename DeclTy>
-Error ASTNodeImporter::ImportTemplateParameterLists(const DeclTy *FromD,
-                                                    DeclTy *ToD) {
+Error ASTNodeImporter::ImportTemplateParameterLists(const DeclaratorDecl *FromD,
+                                                    DeclaratorDecl *ToD) {
   unsigned int Num = FromD->getNumTemplateParameterLists();
   if (Num == 0)
     return Error::success();
@@ -6212,16 +6210,15 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
   if (!IdLocOrErr)
     return IdLocOrErr.takeError();
 
-  // Import TemplateArgumentListInfo.
-  TemplateArgumentListInfo ToTAInfo;
-  if (const auto *ASTTemplateArgs = D->getTemplateArgsAsWritten()) {
-    if (Error Err = ImportTemplateArgumentListInfo(*ASTTemplateArgs, ToTAInfo))
-      return std::move(Err);
-  }
-
   // Create the specialization.
   ClassTemplateSpecializationDecl *D2 = nullptr;
   if (PartialSpec) {
+    // Import TemplateArgumentListInfo.
+    TemplateArgumentListInfo ToTAInfo;
+    const auto &ASTTemplateArgs = *PartialSpec->getTemplateArgsAsWritten();
+    if (Error Err = ImportTemplateArgumentListInfo(ASTTemplateArgs, ToTAInfo))
+      return std::move(Err);
+
     QualType CanonInjType;
     if (Error Err = importInto(
         CanonInjType, PartialSpec->getInjectedSpecializationType()))
@@ -6231,7 +6228,7 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
     if (GetImportedOrCreateDecl<ClassTemplatePartialSpecializationDecl>(
             D2, D, Importer.getToContext(), D->getTagKind(), DC, *BeginLocOrErr,
             *IdLocOrErr, ToTPList, ClassTemplate,
-            llvm::ArrayRef(TemplateArgs.data(), TemplateArgs.size()),
+            llvm::ArrayRef(TemplateArgs.data(), TemplateArgs.size()), ToTAInfo,
             CanonInjType,
             cast_or_null<ClassTemplatePartialSpecializationDecl>(PrevDecl)))
       return D2;
@@ -6279,27 +6276,28 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
   else
     return BraceRangeOrErr.takeError();
 
-  if (Error Err = ImportTemplateParameterLists(D, D2))
-    return std::move(Err);
-
   // Import the qualifier, if any.
   if (auto LocOrErr = import(D->getQualifierLoc()))
     D2->setQualifierInfo(*LocOrErr);
   else
     return LocOrErr.takeError();
 
-  if (D->getTemplateArgsAsWritten())
-    D2->setTemplateArgsAsWritten(ToTAInfo);
+  if (auto *TSI = D->getTypeAsWritten()) {
+    if (auto TInfoOrErr = import(TSI))
+      D2->setTypeAsWritten(*TInfoOrErr);
+    else
+      return TInfoOrErr.takeError();
 
-  if (auto LocOrErr = import(D->getTemplateKeywordLoc()))
-    D2->setTemplateKeywordLoc(*LocOrErr);
-  else
-    return LocOrErr.takeError();
+    if (auto LocOrErr = import(D->getTemplateKeywordLoc()))
+      D2->setTemplateKeywordLoc(*LocOrErr);
+    else
+      return LocOrErr.takeError();
 
-  if (auto LocOrErr = import(D->getExternKeywordLoc()))
-    D2->setExternKeywordLoc(*LocOrErr);
-  else
-    return LocOrErr.takeError();
+    if (auto LocOrErr = import(D->getExternLoc()))
+      D2->setExternLoc(*LocOrErr);
+    else
+      return LocOrErr.takeError();
+  }
 
   if (D->getPointOfInstantiation().isValid()) {
     if (auto POIOrErr = import(D->getPointOfInstantiation()))
@@ -6519,7 +6517,7 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
   VarTemplateSpecializationDecl *D2 = nullptr;
 
   TemplateArgumentListInfo ToTAInfo;
-  if (const auto *Args = D->getTemplateArgsAsWritten()) {
+  if (const ASTTemplateArgumentListInfo *Args = D->getTemplateArgsInfo()) {
     if (Error Err = ImportTemplateArgumentListInfo(*Args, ToTAInfo))
       return std::move(Err);
   }
@@ -6527,6 +6525,14 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
   using PartVarSpecDecl = VarTemplatePartialSpecializationDecl;
   // Create a new specialization.
   if (auto *FromPartial = dyn_cast<PartVarSpecDecl>(D)) {
+    // Import TemplateArgumentListInfo
+    TemplateArgumentListInfo ArgInfos;
+    const auto *FromTAArgsAsWritten = FromPartial->getTemplateArgsAsWritten();
+    // NOTE: FromTAArgsAsWritten and template parameter list are non-null.
+    if (Error Err =
+            ImportTemplateArgumentListInfo(*FromTAArgsAsWritten, ArgInfos))
+      return std::move(Err);
+
     auto ToTPListOrErr = import(FromPartial->getTemplateParameters());
     if (!ToTPListOrErr)
       return ToTPListOrErr.takeError();
@@ -6535,7 +6541,7 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
     if (GetImportedOrCreateDecl(ToPartial, D, Importer.getToContext(), DC,
                                 *BeginLocOrErr, *IdLocOrErr, *ToTPListOrErr,
                                 VarTemplate, QualType(), nullptr,
-                                D->getStorageClass(), TemplateArgs))
+                                D->getStorageClass(), TemplateArgs, ArgInfos))
       return ToPartial;
 
     if (Expected<PartVarSpecDecl *> ToInstOrErr =
@@ -6578,9 +6584,7 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
   }
 
   D2->setSpecializationKind(D->getSpecializationKind());
-
-  if (D->getTemplateArgsAsWritten())
-    D2->setTemplateArgsAsWritten(ToTAInfo);
+  D2->setTemplateArgsInfo(ToTAInfo);
 
   if (auto LocOrErr = import(D->getQualifierLoc()))
     D2->setQualifierInfo(*LocOrErr);
diff --git a/clang/lib/AST/DeclPrinter.cpp b/clang/lib/AST/DeclPrinter.cpp
index c5868256b440..599d379340ab 100644
--- a/clang/lib/AST/DeclPrinter.cpp
+++ b/clang/lib/AST/DeclPrinter.cpp
@@ -1083,15 +1083,15 @@ void DeclPrinter::VisitCXXRecordDecl(CXXRecordDecl *D) {
       NNS->print(Out, Policy);
     Out << *D;
 
-    if (auto *S = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
-      const TemplateParameterList *TParams =
-          S->getSpecializedTemplate()->getTemplateParameters();
-      const ASTTemplateArgumentListInfo *TArgAsWritten =
-          S->getTemplateArgsAsWritten();
-      if (TArgAsWritten && !Policy.PrintCanonicalTypes)
-        printTemplateArguments(TArgAsWritten->arguments(), TParams);
-      else
-        printTemplateArguments(S->getTemplateArgs().asArray(), TParams);
+    if (auto S = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
+      ArrayRef<TemplateArgument> Args = S->getTemplateArgs().asArray();
+      if (!Policy.PrintCanonicalTypes)
+        if (const auto* TSI = S->getTypeAsWritten())
+          if (const auto *TST =
+                  dyn_cast<TemplateSpecializationType>(TSI->getType()))
+            Args = TST->template_arguments();
+      printTemplateArguments(
+          Args, S->getSpecializedTemplate()->getTemplateParameters());
     }
   }
 
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index af2d8d728e3e..d27a30e0c5fc 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -985,63 +985,41 @@ ClassTemplateSpecializationDecl::getSpecializedTemplate() const {
 
 SourceRange
 ClassTemplateSpecializationDecl::getSourceRange() const {
-  if (getSpecializationKind() == TSK_ExplicitInstantiationDeclaration) {
-    return SourceRange(getExternKeywordLoc(),
-                       getTemplateArgsAsWritten()->getRAngleLoc());
-  } else if (getSpecializationKind() == TSK_ExplicitInstantiationDefinition) {
-    return SourceRange(getTemplateKeywordLoc(),
-                       getTemplateArgsAsWritten()->getRAngleLoc());
-  } else if (!isExplicitSpecialization()) {
+  if (ExplicitInfo) {
+    SourceLocation Begin = getTemplateKeywordLoc();
+    if (Begin.isValid()) {
+      // Here we have an explicit (partial) specialization or instantiation.
+      assert(getSpecializationKind() == TSK_ExplicitSpecialization ||
+             getSpecializationKind() == TSK_ExplicitInstantiationDeclaration ||
+             getSpecializationKind() == TSK_ExplicitInstantiationDefinition);
+      if (getExternLoc().isValid())
+        Begin = getExternLoc();
+      SourceLocation End = getBraceRange().getEnd();
+      if (End.isInvalid())
+        End = getTypeAsWritten()->getTypeLoc().getEndLoc();
+      return SourceRange(Begin, End);
+    }
+    // An implicit instantiation of a class template partial specialization
+    // uses ExplicitInfo to record the TypeAsWritten, but the source
+    // locations should be retrieved from the instantiation pattern.
+    using CTPSDecl = ClassTemplatePartialSpecializationDecl;
+    auto *ctpsd = const_cast<CTPSDecl *>(cast<CTPSDecl>(this));
+    CTPSDecl *inst_from = ctpsd->getInstantiatedFromMember();
+    assert(inst_from != nullptr);
+    return inst_from->getSourceRange();
+  }
+  else {
     // No explicit info available.
     llvm::PointerUnion<ClassTemplateDecl *,
                        ClassTemplatePartialSpecializationDecl *>
-        InstFrom = getInstantiatedFrom();
-    if (InstFrom.isNull())
+      inst_from = getInstantiatedFrom();
+    if (inst_from.isNull())
       return getSpecializedTemplate()->getSourceRange();
-    if (const auto *CTD = InstFrom.dyn_cast<ClassTemplateDecl *>())
-      return CTD->getSourceRange();
-    return InstFrom.get<ClassTemplatePartialSpecializationDecl *>()
-        ->getSourceRange();
-  }
-  SourceLocation Begin = TagDecl::getOuterLocStart();
-  if (const auto *CTPSD =
-          dyn_cast<ClassTemplatePartialSpecializationDecl>(this)) {
-    if (const auto *InstFrom = CTPSD->getInstantiatedFromMember())
-      return InstFrom->getSourceRange();
-    else if (!getNumTemplateParameterLists())
-      Begin = CTPSD->getTemplateParameters()->getTemplateLoc();
-  }
-  SourceLocation End = getBraceRange().getEnd();
-  if (End.isInvalid())
-    End = getTemplateArgsAsWritten()->getRAngleLoc();
-  return SourceRange(Begin, End);
-}
-
-void ClassTemplateSpecializationDecl::setExternKeywordLoc(SourceLocation Loc) {
-  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
-  if (!Info) {
-    // Don't allocate if the location is invalid.
-    if (Loc.isInvalid())
-      return;
-    Info = new (getASTContext()) ExplicitInstantiationInfo;
-    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
-    ExplicitInfo = Info;
+    if (const auto *ctd = inst_from.dyn_cast<ClassTemplateDecl *>())
+      return ctd->getSourceRange();
+    return inst_from.get<ClassTemplatePartialSpecializationDecl *>()
+      ->getSourceRange();
   }
-  Info->ExternKeywordLoc = Loc;
-}
-
-void ClassTemplateSpecializationDecl::setTemplateKeywordLoc(
-    SourceLocation Loc) {
-  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
-  if (!Info) {
-    // Don't allocate if the location is invalid.
-    if (Loc.isInvalid())
-      return;
-    Info = new (getASTContext()) ExplicitInstantiationInfo;
-    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
-    ExplicitInfo = Info;
-  }
-  Info->TemplateKeywordLoc = Loc;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1109,29 +1087,43 @@ void ImplicitConceptSpecializationDecl::setTemplateArguments(
 //===----------------------------------------------------------------------===//
 void ClassTemplatePartialSpecializationDecl::anchor() {}
 
-ClassTemplatePartialSpecializationDecl::ClassTemplatePartialSpecializationDecl(
-    ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
-    SourceLocation IdLoc, TemplateParameterList *Params,
-    ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
-    ClassTemplatePartialSpecializationDecl *PrevDecl)
-    : ClassTemplateSpecializationDecl(
-          Context, ClassTemplatePartialSpecialization, TK, DC, StartLoc, IdLoc,
-          SpecializedTemplate, Args, PrevDecl),
-      TemplateParams(Params), InstantiatedFromMember(nullptr, false) {
+ClassTemplatePartialSpecializationDecl::
+ClassTemplatePartialSpecializationDecl(ASTContext &Context, TagKind TK,
+                                       DeclContext *DC,
+                                       SourceLocation StartLoc,
+                                       SourceLocation IdLoc,
+                                       TemplateParameterList *Params,
+                                       ClassTemplateDecl *SpecializedTemplate,
+                                       ArrayRef<TemplateArgument> Args,
+                               const ASTTemplateArgumentListInfo *ArgInfos,
+                               ClassTemplatePartialSpecializationDecl *PrevDecl)
+    : ClassTemplateSpecializationDecl(Context,
+                                      ClassTemplatePartialSpecialization,
+                                      TK, DC, StartLoc, IdLoc,
+                                      SpecializedTemplate, Args, PrevDecl),
+      TemplateParams(Params), ArgsAsWritten(ArgInfos),
+      InstantiatedFromMember(nullptr, false) {
   if (AdoptTemplateParameterList(Params, this))
     setInvalidDecl();
 }
 
 ClassTemplatePartialSpecializationDecl *
-ClassTemplatePartialSpecializationDecl::Create(
-    ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
-    SourceLocation IdLoc, TemplateParameterList *Params,
-    ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
-    QualType CanonInjectedType,
-    ClassTemplatePartialSpecializationDecl *PrevDecl) {
-  auto *Result = new (Context, DC) ClassTemplatePartialSpecializationDecl(
-      Context, TK, DC, StartLoc, IdLoc, Params, SpecializedTemplate, Args,
-      PrevDecl);
+ClassTemplatePartialSpecializationDecl::
+Create(ASTContext &Context, TagKind TK,DeclContext *DC,
+       SourceLocation StartLoc, SourceLocation IdLoc,
+       TemplateParameterList *Params,
+       ClassTemplateDecl *SpecializedTemplate,
+       ArrayRef<TemplateArgument> Args,
+       const TemplateArgumentListInfo &ArgInfos,
+       QualType CanonInjectedType,
+       ClassTemplatePartialSpecializationDecl *PrevDecl) {
+  const ASTTemplateArgumentListInfo *ASTArgInfos =
+    ASTTemplateArgumentListInfo::Create(Context, ArgInfos);
+
+  auto *Result = new (Context, DC)
+      ClassTemplatePartialSpecializationDecl(Context, TK, DC, StartLoc, IdLoc,
+                                             Params, SpecializedTemplate, Args,
+                                             ASTArgInfos, PrevDecl);
   Result->setSpecializationKind(TSK_ExplicitSpecialization);
   Result->setMayHaveOutOfDateDef(false);
 
@@ -1379,47 +1371,26 @@ VarTemplateDecl *VarTemplateSpecializationDecl::getSpecializedTemplate() const {
   return SpecializedTemplate.get<VarTemplateDecl *>();
 }
 
+void VarTemplateSpecializationDecl::setTemplateArgsInfo(
+    const TemplateArgumentListInfo &ArgsInfo) {
+  TemplateArgsInfo =
+      ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo);
+}
+
+void VarTemplateSpecializationDecl::setTemplateArgsInfo(
+    const ASTTemplateArgumentListInfo *ArgsInfo) {
+  TemplateArgsInfo =
+      ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo);
+}
+
 SourceRange VarTemplateSpecializationDecl::getSourceRange() const {
   if (isExplicitSpecialization() && !hasInit()) {
-    if (const ASTTemplateArgumentListInfo *Info = getTemplateArgsAsWritten())
+    if (const ASTTemplateArgumentListInfo *Info = getTemplateArgsInfo())
       return SourceRange(getOuterLocStart(), Info->getRAngleLoc());
-  } else if (getTemplateSpecializationKind() ==
-             TSK_ExplicitInstantiationDeclaration) {
-    if (const ASTTemplateArgumentListInfo *Info = getTemplateArgsAsWritten())
-      return SourceRange(getExternKeywordLoc(), Info->getRAngleLoc());
-  } else if (getTemplateSpecializationKind() ==
-             TSK_ExplicitInstantiationDefinition) {
-    if (const ASTTemplateArgumentListInfo *Info = getTemplateArgsAsWritten())
-      return SourceRange(getTemplateKeywordLoc(), Info->getRAngleLoc());
   }
   return VarDecl::getSourceRange();
 }
 
-void VarTemplateSpecializationDecl::setExternKeywordLoc(SourceLocation Loc) {
-  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
-  if (!Info) {
-    // Don't allocate if the location is invalid.
-    if (Loc.isInvalid())
-      return;
-    Info = new (getASTContext()) ExplicitInstantiationInfo;
-    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
-    ExplicitInfo = Info;
-  }
-  Info->ExternKeywordLoc = Loc;
-}
-
-void VarTemplateSpecializationDecl::setTemplateKeywordLoc(SourceLocation Loc) {
-  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
-  if (!Info) {
-    // Don't allocate if the location is invalid.
-    if (Loc.isInvalid())
-      return;
-    Info = new (getASTContext()) ExplicitInstantiationInfo;
-    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
-    ExplicitInfo = Info;
-  }
-  Info->TemplateKeywordLoc = Loc;
-}
 
 //===----------------------------------------------------------------------===//
 // VarTemplatePartialSpecializationDecl Implementation
@@ -1431,11 +1402,13 @@ VarTemplatePartialSpecializationDecl::VarTemplatePartialSpecializationDecl(
     ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
     SourceLocation IdLoc, TemplateParameterList *Params,
     VarTemplateDecl *SpecializedTemplate, QualType T, TypeSourceInfo *TInfo,
-    StorageClass S, ArrayRef<TemplateArgument> Args)
+    StorageClass S, ArrayRef<TemplateArgument> Args,
+    const ASTTemplateArgumentListInfo *ArgInfos)
     : VarTemplateSpecializationDecl(VarTemplatePartialSpecialization, Context,
                                     DC, StartLoc, IdLoc, SpecializedTemplate, T,
                                     TInfo, S, Args),
-      TemplateParams(Params), InstantiatedFromMember(nullptr, false) {
+      TemplateParams(Params), ArgsAsWritten(ArgInfos),
+      InstantiatedFromMember(nullptr, false) {
   if (AdoptTemplateParameterList(Params, DC))
     setInvalidDecl();
 }
@@ -1445,10 +1418,15 @@ VarTemplatePartialSpecializationDecl::Create(
     ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
     SourceLocation IdLoc, TemplateParameterList *Params,
     VarTemplateDecl *SpecializedTemplate, QualType T, TypeSourceInfo *TInfo,
-    StorageClass S, ArrayRef<TemplateArgument> Args) {
-  auto *Result = new (Context, DC) VarTemplatePartialSpecializationDecl(
-      Context, DC, StartLoc, IdLoc, Params, SpecializedTemplate, T, TInfo, S,
-      Args);
+    StorageClass S, ArrayRef<TemplateArgument> Args,
+    const TemplateArgumentListInfo &ArgInfos) {
+  const ASTTemplateArgumentListInfo *ASTArgInfos
+    = ASTTemplateArgumentListInfo::Create(Context, ArgInfos);
+
+  auto *Result =
+      new (Context, DC) VarTemplatePartialSpecializationDecl(
+          Context, DC, StartLoc, IdLoc, Params, SpecializedTemplate, T, TInfo,
+          S, Args, ASTArgInfos);
   Result->setSpecializationKind(TSK_ExplicitSpecialization);
   return Result;
 }
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 87f0a8728d85..9602f448e942 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -1472,18 +1472,21 @@ void TypePrinter::printTag(TagDecl *D, raw_ostream &OS) {
 
   // If this is a class template specialization, print the template
   // arguments.
-  if (auto *S = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
-    const TemplateParameterList *TParams =
-        S->getSpecializedTemplate()->getTemplateParameters();
-    const ASTTemplateArgumentListInfo *TArgAsWritten =
-        S->getTemplateArgsAsWritten();
+  if (const auto *Spec = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
+    ArrayRef<TemplateArgument> Args;
+    TypeSourceInfo *TAW = Spec->getTypeAsWritten();
+    if (!Policy.PrintCanonicalTypes && TAW) {
+      const TemplateSpecializationType *TST =
+        cast<TemplateSpecializationType>(TAW->getType());
+      Args = TST->template_arguments();
+    } else {
+      const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
+      Args = TemplateArgs.asArray();
+    }
     IncludeStrongLifetimeRAII Strong(Policy);
-    if (TArgAsWritten && !Policy.PrintCanonicalTypes)
-      printTemplateArgumentList(OS, TArgAsWritten->arguments(), Policy,
-                                TParams);
-    else
-      printTemplateArgumentList(OS, S->getTemplateArgs().asArray(), Policy,
-                                TParams);
+    printTemplateArgumentList(
+        OS, Args, Policy,
+        Spec->getSpecializedTemplate()->getTemplateParameters());
   }
 
   spaceBeforePlaceHolder(OS);
diff --git a/clang/lib/Index/IndexDecl.cpp b/clang/lib/Index/IndexDecl.cpp
index 8eb88f5a1e94..1c04aa17d53f 100644
--- a/clang/lib/Index/IndexDecl.cpp
+++ b/clang/lib/Index/IndexDecl.cpp
@@ -673,12 +673,9 @@ public:
     IndexCtx.indexTagDecl(
         D, SymbolRelation(SymbolRoleSet(SymbolRole::RelationSpecializationOf),
                           SpecializationOf));
-    // Template specialization arguments.
-    if (const ASTTemplateArgumentListInfo *TemplateArgInfo =
-            D->getTemplateArgsAsWritten()) {
-      for (const auto &Arg : TemplateArgInfo->arguments())
-        handleTemplateArgumentLoc(Arg, D, D->getLexicalDeclContext());
-    }
+    if (TypeSourceInfo *TSI = D->getTypeAsWritten())
+      IndexCtx.indexTypeSourceInfo(TSI, /*Parent=*/nullptr,
+                                   D->getLexicalDeclContext());
     return true;
   }
 
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 0febf4e1d454..a1e32d391ed0 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -1408,7 +1408,7 @@ void Sema::ActOnEndOfTranslationUnit() {
         SourceRange DiagRange = DiagD->getLocation();
         if (const auto *VTSD = dyn_cast<VarTemplateSpecializationDecl>(DiagD)) {
           if (const ASTTemplateArgumentListInfo *ASTTAL =
-                  VTSD->getTemplateArgsAsWritten())
+                  VTSD->getTemplateArgsInfo())
             DiagRange.setEnd(ASTTAL->RAngleLoc);
         }
         if (DiagD->isReferenced()) {
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index b268d7c405df..5c72270ff150 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -5166,8 +5166,7 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
         VarTemplatePartialSpecializationDecl::Create(
             Context, VarTemplate->getDeclContext(), TemplateKWLoc,
             TemplateNameLoc, TemplateParams, VarTemplate, DI->getType(), DI, SC,
-            CanonicalConverted);
-    Partial->setTemplateArgsAsWritten(TemplateArgs);
+            CanonicalConverted, TemplateArgs);
 
     if (!PrevPartial)
       VarTemplate->AddPartialSpecialization(Partial, InsertPos);
@@ -5185,7 +5184,7 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
     Specialization = VarTemplateSpecializationDecl::Create(
         Context, VarTemplate->getDeclContext(), TemplateKWLoc, TemplateNameLoc,
         VarTemplate, DI->getType(), DI, SC, CanonicalConverted);
-    Specialization->setTemplateArgsAsWritten(TemplateArgs);
+    Specialization->setTemplateArgsInfo(TemplateArgs);
 
     if (!PrevDecl)
       VarTemplate->AddSpecialization(Specialization, InsertPos);
@@ -5220,6 +5219,7 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
     }
   }
 
+  Specialization->setTemplateKeywordLoc(TemplateKWLoc);
   Specialization->setLexicalDeclContext(CurContext);
 
   // Add the specialization into its lexical context, so that it can
@@ -9489,8 +9489,7 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
         ClassTemplatePartialSpecializationDecl::Create(
             Context, Kind, ClassTemplate->getDeclContext(), KWLoc,
             TemplateNameLoc, TemplateParams, ClassTemplate, CanonicalConverted,
-            CanonType, PrevPartial);
-    Partial->setTemplateArgsAsWritten(TemplateArgs);
+            TemplateArgs, CanonType, PrevPartial);
     SetNestedNameSpecifier(*this, Partial, SS);
     if (TemplateParameterLists.size() > 1 && SS.isSet()) {
       Partial->setTemplateParameterListsInfo(
@@ -9513,7 +9512,6 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     Specialization = ClassTemplateSpecializationDecl::Create(
         Context, Kind, ClassTemplate->getDeclContext(), KWLoc, TemplateNameLoc,
         ClassTemplate, CanonicalConverted, PrevDecl);
-    Specialization->setTemplateArgsAsWritten(TemplateArgs);
     SetNestedNameSpecifier(*this, Specialization, SS);
     if (TemplateParameterLists.size() > 0) {
       Specialization->setTemplateParameterListsInfo(Context,
@@ -9597,6 +9595,21 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
       << (isPartialSpecialization? 1 : 0)
       << FixItHint::CreateRemoval(ModulePrivateLoc);
 
+  // Build the fully-sugared type for this class template
+  // specialization as the user wrote in the specialization
+  // itself. This means that we'll pretty-print the type retrieved
+  // from the specialization's declaration the way that the user
+  // actually wrote the specialization, rather than formatting the
+  // name based on the "canonical" representation used to store the
+  // template arguments in the specialization.
+  TypeSourceInfo *WrittenTy
+    = Context.getTemplateSpecializationTypeInfo(Name, TemplateNameLoc,
+                                                TemplateArgs, CanonType);
+  if (TUK != TUK_Friend) {
+    Specialization->setTypeAsWritten(WrittenTy);
+    Specialization->setTemplateKeywordLoc(TemplateKWLoc);
+  }
+
   // C++ [temp.expl.spec]p9:
   //   A template explicit specialization is in the scope of the
   //   namespace in which the template was defined.
@@ -9612,15 +9625,6 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     Specialization->startDefinition();
 
   if (TUK == TUK_Friend) {
-    // Build the fully-sugared type for this class template
-    // specialization as the user wrote in the specialization
-    // itself. This means that we'll pretty-print the type retrieved
-    // from the specialization's declaration the way that the user
-    // actually wrote the specialization, rather than formatting the
-    // name based on the "canonical" representation used to store the
-    // template arguments in the specialization.
-    TypeSourceInfo *WrittenTy = Context.getTemplateSpecializationTypeInfo(
-        Name, TemplateNameLoc, TemplateArgs, CanonType);
     FriendDecl *Friend = FriendDecl::Create(Context, CurContext,
                                             TemplateNameLoc,
                                             WrittenTy,
@@ -10826,10 +10830,21 @@ DeclResult Sema::ActOnExplicitInstantiation(
     }
   }
 
-  Specialization->setTemplateArgsAsWritten(TemplateArgs);
+  // Build the fully-sugared type for this explicit instantiation as
+  // the user wrote in the explicit instantiation itself. This means
+  // that we'll pretty-print the type retrieved from the
+  // specialization's declaration the way that the user actually wrote
+  // the explicit instantiation, rather than formatting the name based
+  // on the "canonical" representation used to store the template
+  // arguments in the specialization.
+  TypeSourceInfo *WrittenTy
+    = Context.getTemplateSpecializationTypeInfo(Name, TemplateNameLoc,
+                                                TemplateArgs,
+                                  Context.getTypeDeclType(Specialization));
+  Specialization->setTypeAsWritten(WrittenTy);
 
   // Set source locations for keywords.
-  Specialization->setExternKeywordLoc(ExternLoc);
+  Specialization->setExternLoc(ExternLoc);
   Specialization->setTemplateKeywordLoc(TemplateLoc);
   Specialization->setBraceRange(SourceRange());
 
@@ -11242,11 +11257,6 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     if (!HasNoEffect) {
       // Instantiate static data member or variable template.
       Prev->setTemplateSpecializationKind(TSK, D.getIdentifierLoc());
-      if (auto *VTSD = dyn_cast<VarTemplatePartialSpecializationDecl>(Prev)) {
-        VTSD->setExternKeywordLoc(ExternLoc);
-        VTSD->setTemplateKeywordLoc(TemplateLoc);
-      }
-
       // Merge attributes.
       ProcessDeclAttributeList(S, Prev, D.getDeclSpec().getAttributes());
       if (PrevTemplate)
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 5315b143215e..d544cfac55ba 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -3858,16 +3858,15 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
 
   // Substitute into the template arguments of the class template explicit
   // specialization.
-  TemplateArgumentListInfo InstTemplateArgs;
-  if (const ASTTemplateArgumentListInfo *TemplateArgsInfo =
-          D->getTemplateArgsAsWritten()) {
-    InstTemplateArgs.setLAngleLoc(TemplateArgsInfo->getLAngleLoc());
-    InstTemplateArgs.setRAngleLoc(TemplateArgsInfo->getRAngleLoc());
-
-    if (SemaRef.SubstTemplateArguments(TemplateArgsInfo->arguments(),
-                                       TemplateArgs, InstTemplateArgs))
-      return nullptr;
-  }
+  TemplateSpecializationTypeLoc Loc = D->getTypeAsWritten()->getTypeLoc().
+                                        castAs<TemplateSpecializationTypeLoc>();
+  TemplateArgumentListInfo InstTemplateArgs(Loc.getLAngleLoc(),
+                                            Loc.getRAngleLoc());
+  SmallVector<TemplateArgumentLoc, 4> ArgLocs;
+  for (unsigned I = 0; I != Loc.getNumArgs(); ++I)
+    ArgLocs.push_back(Loc.getArgLoc(I));
+  if (SemaRef.SubstTemplateArguments(ArgLocs, TemplateArgs, InstTemplateArgs))
+    return nullptr;
 
   // Check that the template argument list is well-formed for this
   // class template.
@@ -3921,7 +3920,6 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
       ClassTemplateSpecializationDecl::Create(
           SemaRef.Context, D->getTagKind(), Owner, D->getBeginLoc(),
           D->getLocation(), InstClassTemplate, CanonicalConverted, PrevDecl);
-  InstD->setTemplateArgsAsWritten(InstTemplateArgs);
 
   // Add this partial specialization to the set of class template partial
   // specializations.
@@ -3938,10 +3936,22 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
       TemplateName(InstClassTemplate), CanonicalConverted,
       SemaRef.Context.getRecordType(InstD));
 
+  // Build the fully-sugared type for this class template
+  // specialization as the user wrote in the specialization
+  // itself. This means that we'll pretty-print the type retrieved
+  // from the specialization's declaration the way that the user
+  // actually wrote the specialization, rather than formatting the
+  // name based on the "canonical" representation used to store the
+  // template arguments in the specialization.
+  TypeSourceInfo *WrittenTy = SemaRef.Context.getTemplateSpecializationTypeInfo(
+      TemplateName(InstClassTemplate), D->getLocation(), InstTemplateArgs,
+      CanonType);
+
   InstD->setAccess(D->getAccess());
   InstD->setInstantiationOfMemberClass(D, TSK_ImplicitInstantiation);
   InstD->setSpecializationKind(D->getSpecializationKind());
-  InstD->setExternKeywordLoc(D->getExternKeywordLoc());
+  InstD->setTypeAsWritten(WrittenTy);
+  InstD->setExternLoc(D->getExternLoc());
   InstD->setTemplateKeywordLoc(D->getTemplateKeywordLoc());
 
   Owner->addDecl(InstD);
@@ -3975,7 +3985,7 @@ Decl *TemplateDeclInstantiator::VisitVarTemplateSpecializationDecl(
 
   // Substitute the current template arguments.
   if (const ASTTemplateArgumentListInfo *TemplateArgsInfo =
-          D->getTemplateArgsAsWritten()) {
+          D->getTemplateArgsInfo()) {
     VarTemplateArgsInfo.setLAngleLoc(TemplateArgsInfo->getLAngleLoc());
     VarTemplateArgsInfo.setRAngleLoc(TemplateArgsInfo->getRAngleLoc());
 
@@ -4033,7 +4043,7 @@ Decl *TemplateDeclInstantiator::VisitVarTemplateSpecializationDecl(
   VarTemplateSpecializationDecl *Var = VarTemplateSpecializationDecl::Create(
       SemaRef.Context, Owner, D->getInnerLocStart(), D->getLocation(),
       VarTemplate, DI->getType(), DI, D->getStorageClass(), Converted);
-  Var->setTemplateArgsAsWritten(TemplateArgsInfo);
+  Var->setTemplateArgsInfo(TemplateArgsInfo);
   if (!PrevDecl) {
     void *InsertPos = nullptr;
     VarTemplate->findSpecialization(Converted, InsertPos);
@@ -4275,21 +4285,19 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
   QualType CanonType = SemaRef.Context.getTemplateSpecializationType(
       TemplateName(ClassTemplate), CanonicalConverted);
 
-  // Create the class template partial specialization declaration.
-  ClassTemplatePartialSpecializationDecl *InstPartialSpec =
-      ClassTemplatePartialSpecializationDecl::Create(
-          SemaRef.Context, PartialSpec->getTagKind(), Owner,
-          PartialSpec->getBeginLoc(), PartialSpec->getLocation(), InstParams,
-          ClassTemplate, CanonicalConverted, CanonType,
-          /*PrevDecl=*/nullptr);
-
-  InstPartialSpec->setTemplateArgsAsWritten(InstTemplateArgs);
-
-  // Substitute the nested name specifier, if any.
-  if (SubstQualifier(PartialSpec, InstPartialSpec))
-    return nullptr;
-
-  InstPartialSpec->setInstantiatedFromMember(PartialSpec);
+  // Build the fully-sugared type for this class template
+  // specialization as the user wrote in the specialization
+  // itself. This means that we'll pretty-print the type retrieved
+  // from the specialization's declaration the way that the user
+  // actually wrote the specialization, rather than formatting the
+  // name based on the "canonical" representation used to store the
+  // template arguments in the specialization.
+  TypeSourceInfo *WrittenTy
+    = SemaRef.Context.getTemplateSpecializationTypeInfo(
+                                                    TemplateName(ClassTemplate),
+                                                    PartialSpec->getLocation(),
+                                                    InstTemplateArgs,
+                                                    CanonType);
 
   if (PrevDecl) {
     // We've already seen a partial specialization with the same template
@@ -4307,14 +4315,28 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
     //
     //   Outer<int, int> outer; // error: the partial specializations of Inner
     //                          // have the same signature.
-    SemaRef.Diag(InstPartialSpec->getLocation(),
-                 diag::err_partial_spec_redeclared)
-        << InstPartialSpec;
+    SemaRef.Diag(PartialSpec->getLocation(), diag::err_partial_spec_redeclared)
+      << WrittenTy->getType();
     SemaRef.Diag(PrevDecl->getLocation(), diag::note_prev_partial_spec_here)
       << SemaRef.Context.getTypeDeclType(PrevDecl);
     return nullptr;
   }
 
+
+  // Create the class template partial specialization declaration.
+  ClassTemplatePartialSpecializationDecl *InstPartialSpec =
+      ClassTemplatePartialSpecializationDecl::Create(
+          SemaRef.Context, PartialSpec->getTagKind(), Owner,
+          PartialSpec->getBeginLoc(), PartialSpec->getLocation(), InstParams,
+          ClassTemplate, CanonicalConverted, InstTemplateArgs, CanonType,
+          nullptr);
+  // Substitute the nested name specifier, if any.
+  if (SubstQualifier(PartialSpec, InstPartialSpec))
+    return nullptr;
+
+  InstPartialSpec->setInstantiatedFromMember(PartialSpec);
+  InstPartialSpec->setTypeAsWritten(WrittenTy);
+
   // Check the completed partial specialization.
   SemaRef.CheckTemplatePartialSpecialization(InstPartialSpec);
 
@@ -4383,6 +4405,46 @@ TemplateDeclInstantiator::InstantiateVarTemplatePartialSpecialization(
       VarTemplate->findPartialSpecialization(CanonicalConverted, InstParams,
                                              InsertPos);
 
+  // Build the canonical type that describes the converted template
+  // arguments of the variable template partial specialization.
+  QualType CanonType = SemaRef.Context.getTemplateSpecializationType(
+      TemplateName(VarTemplate), CanonicalConverted);
+
+  // Build the fully-sugared type for this variable template
+  // specialization as the user wrote in the specialization
+  // itself. This means that we'll pretty-print the type retrieved
+  // from the specialization's declaration the way that the user
+  // actually wrote the specialization, rather than formatting the
+  // name based on the "canonical" representation used to store the
+  // template arguments in the specialization.
+  TypeSourceInfo *WrittenTy = SemaRef.Context.getTemplateSpecializationTypeInfo(
+      TemplateName(VarTemplate), PartialSpec->getLocation(), InstTemplateArgs,
+      CanonType);
+
+  if (PrevDecl) {
+    // We've already seen a partial specialization with the same template
+    // parameters and template arguments. This can happen, for example, when
+    // substituting the outer template arguments ends up causing two
+    // variable template partial specializations of a member variable template
+    // to have identical forms, e.g.,
+    //
+    //   template<typename T, typename U>
+    //   struct Outer {
+    //     template<typename X, typename Y> pair<X,Y> p;
+    //     template<typename Y> pair<T, Y> p;
+    //     template<typename Y> pair<U, Y> p;
+    //   };
+    //
+    //   Outer<int, int> outer; // error: the partial specializations of Inner
+    //                          // have the same signature.
+    SemaRef.Diag(PartialSpec->getLocation(),
+                 diag::err_var_partial_spec_redeclared)
+        << WrittenTy->getType();
+    SemaRef.Diag(PrevDecl->getLocation(),
+                 diag::note_var_prev_partial_spec_here);
+    return nullptr;
+  }
+
   // Do substitution on the type of the declaration
   TypeSourceInfo *DI = SemaRef.SubstType(
       PartialSpec->getTypeSourceInfo(), TemplateArgs,
@@ -4402,39 +4464,16 @@ TemplateDeclInstantiator::InstantiateVarTemplatePartialSpecialization(
       VarTemplatePartialSpecializationDecl::Create(
           SemaRef.Context, Owner, PartialSpec->getInnerLocStart(),
           PartialSpec->getLocation(), InstParams, VarTemplate, DI->getType(),
-          DI, PartialSpec->getStorageClass(), CanonicalConverted);
-
-  InstPartialSpec->setTemplateArgsAsWritten(InstTemplateArgs);
+          DI, PartialSpec->getStorageClass(), CanonicalConverted,
+          InstTemplateArgs);
 
   // Substitute the nested name specifier, if any.
   if (SubstQualifier(PartialSpec, InstPartialSpec))
     return nullptr;
 
   InstPartialSpec->setInstantiatedFromMember(PartialSpec);
+  InstPartialSpec->setTypeAsWritten(WrittenTy);
 
-  if (PrevDecl) {
-    // We've already seen a partial specialization with the same template
-    // parameters and template arguments. This can happen, for example, when
-    // substituting the outer template arguments ends up causing two
-    // variable template partial specializations of a member variable template
-    // to have identical forms, e.g.,
-    //
-    //   template<typename T, typename U>
-    //   struct Outer {
-    //     template<typename X, typename Y> pair<X,Y> p;
-    //     template<typename Y> pair<T, Y> p;
-    //     template<typename Y> pair<U, Y> p;
-    //   };
-    //
-    //   Outer<int, int> outer; // error: the partial specializations of Inner
-    //                          // have the same signature.
-    SemaRef.Diag(PartialSpec->getLocation(),
-                 diag::err_var_partial_spec_redeclared)
-        << InstPartialSpec;
-    SemaRef.Diag(PrevDecl->getLocation(),
-                 diag::note_var_prev_partial_spec_here);
-    return nullptr;
-  }
   // Check the completed partial specialization.
   SemaRef.CheckTemplatePartialSpecialization(InstPartialSpec);
 
@@ -5696,7 +5735,7 @@ void Sema::InstantiateVariableDefinition(SourceLocation PointOfInstantiation,
 
     TemplateArgumentListInfo TemplateArgInfo;
     if (const ASTTemplateArgumentListInfo *ArgInfo =
-            VarSpec->getTemplateArgsAsWritten()) {
+            VarSpec->getTemplateArgsInfo()) {
       TemplateArgInfo.setLAngleLoc(ArgInfo->getLAngleLoc());
       TemplateArgInfo.setRAngleLoc(ArgInfo->getRAngleLoc());
       for (const TemplateArgumentLoc &Arg : ArgInfo->arguments())
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 0c647086e304..089ede4f4926 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -2548,17 +2548,16 @@ ASTDeclReader::VisitClassTemplateSpecializationDeclImpl(
     }
   }
 
-  // extern/template keyword locations for explicit instantiations
-  if (Record.readBool()) {
-    auto *ExplicitInfo = new (C) ExplicitInstantiationInfo;
-    ExplicitInfo->ExternKeywordLoc = readSourceLocation();
+  // Explicit info.
+  if (TypeSourceInfo *TyInfo = readTypeSourceInfo()) {
+    auto *ExplicitInfo =
+        new (C) ClassTemplateSpecializationDecl::ExplicitSpecializationInfo;
+    ExplicitInfo->TypeAsWritten = TyInfo;
+    ExplicitInfo->ExternLoc = readSourceLocation();
     ExplicitInfo->TemplateKeywordLoc = readSourceLocation();
     D->ExplicitInfo = ExplicitInfo;
   }
 
-  if (Record.readBool())
-    D->setTemplateArgsAsWritten(Record.readASTTemplateArgumentListInfo());
-
   return Redecl;
 }
 
@@ -2568,6 +2567,7 @@ void ASTDeclReader::VisitClassTemplatePartialSpecializationDecl(
   // need them for profiling
   TemplateParameterList *Params = Record.readTemplateParameterList();
   D->TemplateParams = Params;
+  D->ArgsAsWritten = Record.readASTTemplateArgumentListInfo();
 
   RedeclarableResult Redecl = VisitClassTemplateSpecializationDeclImpl(D);
 
@@ -2617,17 +2617,16 @@ ASTDeclReader::VisitVarTemplateSpecializationDeclImpl(
     }
   }
 
-  // extern/template keyword locations for explicit instantiations
-  if (Record.readBool()) {
-    auto *ExplicitInfo = new (C) ExplicitInstantiationInfo;
-    ExplicitInfo->ExternKeywordLoc = readSourceLocation();
+  // Explicit info.
+  if (TypeSourceInfo *TyInfo = readTypeSourceInfo()) {
+    auto *ExplicitInfo =
+        new (C) VarTemplateSpecializationDecl::ExplicitSpecializationInfo;
+    ExplicitInfo->TypeAsWritten = TyInfo;
+    ExplicitInfo->ExternLoc = readSourceLocation();
     ExplicitInfo->TemplateKeywordLoc = readSourceLocation();
     D->ExplicitInfo = ExplicitInfo;
   }
 
-  if (Record.readBool())
-    D->setTemplateArgsAsWritten(Record.readASTTemplateArgumentListInfo());
-
   SmallVector<TemplateArgument, 8> TemplArgs;
   Record.readTemplateArgumentList(TemplArgs, /*Canonicalize*/ true);
   D->TemplateArgs = TemplateArgumentList::CreateCopy(C, TemplArgs);
@@ -2667,6 +2666,7 @@ void ASTDeclReader::VisitVarTemplatePartialSpecializationDecl(
     VarTemplatePartialSpecializationDecl *D) {
   TemplateParameterList *Params = Record.readTemplateParameterList();
   D->TemplateParams = Params;
+  D->ArgsAsWritten = Record.readASTTemplateArgumentListInfo();
 
   RedeclarableResult Redecl = VisitVarTemplateSpecializationDeclImpl(D);
 
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index c2f1d1b44241..6201d284f0e0 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -1765,28 +1765,20 @@ void ASTDeclWriter::VisitClassTemplateSpecializationDecl(
     Record.AddDeclRef(D->getSpecializedTemplate()->getCanonicalDecl());
   }
 
-  bool ExplicitInstantiation =
-      D->getTemplateSpecializationKind() ==
-          TSK_ExplicitInstantiationDeclaration ||
-      D->getTemplateSpecializationKind() == TSK_ExplicitInstantiationDefinition;
-  Record.push_back(ExplicitInstantiation);
-  if (ExplicitInstantiation) {
-    Record.AddSourceLocation(D->getExternKeywordLoc());
+  // Explicit info.
+  Record.AddTypeSourceInfo(D->getTypeAsWritten());
+  if (D->getTypeAsWritten()) {
+    Record.AddSourceLocation(D->getExternLoc());
     Record.AddSourceLocation(D->getTemplateKeywordLoc());
   }
 
-  const ASTTemplateArgumentListInfo *ArgsWritten =
-      D->getTemplateArgsAsWritten();
-  Record.push_back(!!ArgsWritten);
-  if (ArgsWritten)
-    Record.AddASTTemplateArgumentListInfo(ArgsWritten);
-
   Code = serialization::DECL_CLASS_TEMPLATE_SPECIALIZATION;
 }
 
 void ASTDeclWriter::VisitClassTemplatePartialSpecializationDecl(
                                     ClassTemplatePartialSpecializationDecl *D) {
   Record.AddTemplateParameterList(D->getTemplateParameters());
+  Record.AddASTTemplateArgumentListInfo(D->getTemplateArgsAsWritten());
 
   VisitClassTemplateSpecializationDecl(D);
 
@@ -1820,22 +1812,13 @@ void ASTDeclWriter::VisitVarTemplateSpecializationDecl(
     Record.AddTemplateArgumentList(&D->getTemplateInstantiationArgs());
   }
 
-  bool ExplicitInstantiation =
-      D->getTemplateSpecializationKind() ==
-          TSK_ExplicitInstantiationDeclaration ||
-      D->getTemplateSpecializationKind() == TSK_ExplicitInstantiationDefinition;
-  Record.push_back(ExplicitInstantiation);
-  if (ExplicitInstantiation) {
-    Record.AddSourceLocation(D->getExternKeywordLoc());
+  // Explicit info.
+  Record.AddTypeSourceInfo(D->getTypeAsWritten());
+  if (D->getTypeAsWritten()) {
+    Record.AddSourceLocation(D->getExternLoc());
     Record.AddSourceLocation(D->getTemplateKeywordLoc());
   }
 
-  const ASTTemplateArgumentListInfo *ArgsWritten =
-      D->getTemplateArgsAsWritten();
-  Record.push_back(!!ArgsWritten);
-  if (ArgsWritten)
-    Record.AddASTTemplateArgumentListInfo(ArgsWritten);
-
   Record.AddTemplateArgumentList(&D->getTemplateArgs());
   Record.AddSourceLocation(D->getPointOfInstantiation());
   Record.push_back(D->getSpecializationKind());
@@ -1856,6 +1839,7 @@ void ASTDeclWriter::VisitVarTemplateSpecializationDecl(
 void ASTDeclWriter::VisitVarTemplatePartialSpecializationDecl(
     VarTemplatePartialSpecializationDecl *D) {
   Record.AddTemplateParameterList(D->getTemplateParameters());
+  Record.AddASTTemplateArgumentListInfo(D->getTemplateArgsAsWritten());
 
   VisitVarTemplateSpecializationDecl(D);
 
diff --git a/clang/lib/Tooling/Syntax/BuildTree.cpp b/clang/lib/Tooling/Syntax/BuildTree.cpp
index 3e50d67f4d6e..cd0261989495 100644
--- a/clang/lib/Tooling/Syntax/BuildTree.cpp
+++ b/clang/lib/Tooling/Syntax/BuildTree.cpp
@@ -735,8 +735,7 @@ public:
     auto *Declaration =
         cast<syntax::SimpleDeclaration>(handleFreeStandingTagDecl(C));
     foldExplicitTemplateInstantiation(
-        Builder.getTemplateRange(C),
-        Builder.findToken(C->getExternKeywordLoc()),
+        Builder.getTemplateRange(C), Builder.findToken(C->getExternLoc()),
         Builder.findToken(C->getTemplateKeywordLoc()), Declaration, C);
     return true;
   }
diff --git a/clang/test/AST/ast-dump-template-decls.cpp b/clang/test/AST/ast-dump-template-decls.cpp
index 37f6d8a0472d..142bc9e6ad9a 100644
--- a/clang/test/AST/ast-dump-template-decls.cpp
+++ b/clang/test/AST/ast-dump-template-decls.cpp
@@ -1,12 +1,12 @@
 // Test without serialization:
 // RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown -ast-dump %s \
-// RUN: | FileCheck -strict-whitespace %s
+// RUN: | FileCheck -strict-whitespace %s --check-prefix=DIRECT
 //
 // Test with serialization:
 // RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -x c++ -std=c++17 -triple x86_64-unknown-unknown -include-pch %t -ast-dump-all /dev/null \
 // RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
-// RUN: | FileCheck --strict-whitespace %s
+// RUN: | FileCheck --strict-whitespace %s --check-prefix=SERIALIZED
 
 template <typename Ty>
 // CHECK: FunctionTemplateDecl 0x{{[^ ]*}} <{{.*}}:1, line:[[@LINE+2]]:10> col:6 a
@@ -189,13 +189,15 @@ T unTempl = 1;
 
 template<>
 int unTempl<int>;
-// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:16> col:5 unTempl 'int'
+// FIXME (#61680) - serializing and loading AST should not affect reported source range
+// DIRECT:     VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:16> col:5 unTempl 'int'
+// SERIALIZED: VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:5> col:5 unTempl 'int'
 // CHECK-NEXT: `-TemplateArgument type 'int'
 // CHECK-NEXT: `-BuiltinType 0x{{[^ ]*}} 'int'
 
 template<>
 float unTempl<float> = 1;
-// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:24> col:7 unTempl 'float'
+// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:24> col:7 unTempl 'float' cinit
 // CHECK-NEXT: |-TemplateArgument type 'float'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'float'
 // CHECK-NEXT: `-ImplicitCastExpr 0x{{[^ ]*}} <col:24> 'float' <IntegralToFloating>
@@ -220,7 +222,7 @@ int binTempl<int, U>;
 
 template<class U>
 float binTempl<float, U> = 1;
-// CHECK:      VarTemplatePartialSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:28> col:7 binTempl 'float'
+// CHECK:      VarTemplatePartialSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:24> col:7 binTempl 'float' cinit
 // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:10, col:16> col:16 referenced class depth 0 index 0 U
 // CHECK-NEXT: |-TemplateArgument type 'float'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'float'
@@ -231,7 +233,9 @@ float binTempl<float, U> = 1;
 
 template<>
 int binTempl<int, int>;
-// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:22> col:5 binTempl 'int'
+// FIXME (#61680) - serializing and loading AST should not affect reported source range
+// DIRECT:     VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:22> col:5 binTempl 'int'
+// SERIALIZED: VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:5> col:5 binTempl 'int'
 // CHECK-NEXT: |-TemplateArgument type 'int'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'int'
 // CHECK-NEXT: `-TemplateArgument type 'int'
@@ -239,7 +243,7 @@ int binTempl<int, int>;
 
 template<>
 float binTempl<float, float> = 1;
-// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:32> col:7 binTempl 'float'
+// CHECK:     VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:32> col:7 binTempl 'float' cinit
 // CHECK-NEXT: |-TemplateArgument type 'float'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'float'
 // CHECK-NEXT: |-TemplateArgument type 'float'
diff --git a/clang/test/Index/Core/index-source.cpp b/clang/test/Index/Core/index-source.cpp
index 043e616a1d36..8f9fbc4c8d29 100644
--- a/clang/test/Index/Core/index-source.cpp
+++ b/clang/test/Index/Core/index-source.cpp
@@ -285,17 +285,20 @@ template<>
 class SpecializationDecl<int>;
 // CHECK: [[@LINE-1]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Decl,RelSpecialization | rel: 1
 // CHECK-NEXT: RelSpecialization | SpecializationDecl | c:@ST>1#T@SpecializationDecl
+// CHECK: [[@LINE-3]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Ref | rel: 0
 
 template<>
 class SpecializationDecl<int> { };
 // CHECK: [[@LINE-1]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Def,RelSpecialization | rel: 1
 // CHECK-NEXT: RelSpecialization | SpecializationDecl | c:@ST>1#T@SpecializationDecl
+// CHECK-NEXT: [[@LINE-3]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Ref | rel: 0
 
 template<typename T>
 class PartialSpecilizationClass<Cls, T>;
 // CHECK: [[@LINE-1]]:7 | class(Gen,TPS)/C++ | PartialSpecilizationClass | c:@SP>1#T@PartialSpecilizationClass>#$@S@Cls#t0.0 | <no-cgname> | Decl,RelSpecialization | rel: 1
 // CHECK-NEXT: RelSpecialization | PartialSpecilizationClass | c:@ST>2#T#T@PartialSpecilizationClass
-// CHECK-NEXT: [[@LINE-3]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK: [[@LINE-3]]:7 | class(Gen)/C++ | PartialSpecilizationClass | c:@ST>2#T#T@PartialSpecilizationClass | <no-cgname> | Ref | rel: 0
+// CHECK-NEXT: [[@LINE-4]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
 
 template<>
 class PartialSpecilizationClass<Cls, Cls> : Cls { };
@@ -303,10 +306,9 @@ class PartialSpecilizationClass<Cls, Cls> : Cls { };
 // CHECK-NEXT: RelSpecialization | PartialSpecilizationClass | c:@ST>2#T#T@PartialSpecilizationClass
 // CHECK-NEXT: [[@LINE-3]]:45 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelBase,RelCont | rel: 1
 // CHECK-NEXT: RelBase,RelCont | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_
-// CHECK-NEXT: [[@LINE-5]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
-// CHECK-NEXT: RelCont | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_
-// CHECK-NEXT: [[@LINE-7]]:38 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
-// CHECK-NEXT: RelCont | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_
+// CHECK-NEXT: [[@LINE-5]]:7 | class(Gen,TS)/C++ | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_ | <no-cgname> | Ref | rel: 0
+// CHECK-NEXT: [[@LINE-6]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
+// CHECK-NEXT: [[@LINE-7]]:38 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
 
 template<typename T, int x>
 void functionSp() { }
@@ -330,14 +332,10 @@ class ClassWithCorrectSpecialization { };
 
 template<>
 class ClassWithCorrectSpecialization<SpecializationDecl<Cls>, Record::C> { };
-// CHECK: [[@LINE-1]]:38 | class(Gen)/C++ | SpecializationDecl | c:@ST>1#T@SpecializationDecl | <no-cgname> | Ref,RelCont | rel: 1
-// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
-// CHECK-NEXT: [[@LINE-3]]:57 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
-// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
-// CHECK-NEXT: [[@LINE-5]]:71 | static-property/C++ | C | c:@S@Record@C | __ZN6Record1CE | Ref,Read,RelCont | rel: 1
-// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
-// CHECK-NEXT: [[@LINE-7]]:63 | struct/C++ | Record | c:@S@Record | <no-cgname> | Ref,RelCont | rel: 1
-// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
+// CHECK: [[@LINE-1]]:38 | class(Gen)/C++ | SpecializationDecl | c:@ST>1#T@SpecializationDecl | <no-cgname> | Ref | rel: 0
+// CHECK: [[@LINE-2]]:57 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
+// CHECK: [[@LINE-3]]:71 | static-property/C++ | C | c:@S@Record@C | __ZN6Record1CE | Ref,Read | rel: 0
+// CHECK: [[@LINE-4]]:63 | struct/C++ | Record | c:@S@Record | <no-cgname> | Ref | rel: 0
 
 namespace ns {
 // CHECK: [[@LINE-1]]:11 | namespace/C++ | ns | c:@N@ns | <no-cgname> | Decl | rel: 0
diff --git a/clang/test/Index/index-refs.cpp b/clang/test/Index/index-refs.cpp
index 14946849777d..0e613e48522b 100644
--- a/clang/test/Index/index-refs.cpp
+++ b/clang/test/Index/index-refs.cpp
@@ -108,6 +108,7 @@ int ginitlist[] = {EnumVal};
 // CHECK:      [indexDeclaration]: kind: c++-class-template | name: TS | {{.*}} | loc: 47:8
 // CHECK-NEXT: [indexDeclaration]: kind: struct-template-partial-spec | name: TS | USR: c:@SP>1#T@TS>#t0.0#I | {{.*}} | loc: 50:8
 // CHECK-NEXT: [indexDeclaration]: kind: typedef | name: MyInt | USR: c:index-refs.cpp@SP>1#T@TS>#t0.0#I@T@MyInt | {{.*}} | loc: 51:15 | semantic-container: [TS:50:8] | lexical-container: [TS:50:8]
+// CHECK-NEXT: [indexEntityReference]: kind: c++-class-template | name: TS | USR: c:@ST>2#T#T@TS | lang: C++ | cursor: TemplateRef=TS:47:8 | loc: 50:8 | <parent>:: <<NULL>> | container: [TU] | refkind: direct | role: ref
 /* when indexing implicit instantiations
   [indexDeclaration]: kind: struct-template-spec | name: TS | USR: c:@S@TS>#I | {{.*}} | loc: 50:8
   [indexDeclaration]: kind: typedef | name: MyInt | USR: c:index-refs.cpp@593@S@TS>#I@T@MyInt | {{.*}} | loc: 51:15 | semantic-container: [TS:50:8] | lexical-container: [TS:50:8]
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 60241afd8776..b845a381d63b 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -743,10 +743,14 @@ bool CursorVisitor::VisitClassTemplateSpecializationDecl(
   }
 
   // Visit the template arguments used in the specialization.
-  if (const auto *ArgsWritten = D->getTemplateArgsAsWritten()) {
-    for (const TemplateArgumentLoc &Arg : ArgsWritten->arguments())
-      if (VisitTemplateArgumentLoc(Arg))
-        return true;
+  if (TypeSourceInfo *SpecType = D->getTypeAsWritten()) {
+    TypeLoc TL = SpecType->getTypeLoc();
+    if (TemplateSpecializationTypeLoc TSTLoc =
+            TL.getAs<TemplateSpecializationTypeLoc>()) {
+      for (unsigned I = 0, N = TSTLoc.getNumArgs(); I != N; ++I)
+        if (VisitTemplateArgumentLoc(TSTLoc.getArgLoc(I)))
+          return true;
+    }
   }
 
   return ShouldVisitBody && VisitCXXRecordDecl(D);
@@ -5655,19 +5659,16 @@ CXString clang_getCursorDisplayName(CXCursor C) {
 
   if (const ClassTemplateSpecializationDecl *ClassSpec =
           dyn_cast<ClassTemplateSpecializationDecl>(D)) {
+    // If the type was explicitly written, use that.
+    if (TypeSourceInfo *TSInfo = ClassSpec->getTypeAsWritten())
+      return cxstring::createDup(TSInfo->getType().getAsString(Policy));
+
     SmallString<128> Str;
     llvm::raw_svector_ostream OS(Str);
     OS << *ClassSpec;
-    // If the template arguments were written explicitly, use them..
-    if (const auto *ArgsWritten = ClassSpec->getTemplateArgsAsWritten()) {
-      printTemplateArgumentList(
-          OS, ArgsWritten->arguments(), Policy,
-          ClassSpec->getSpecializedTemplate()->getTemplateParameters());
-    } else {
-      printTemplateArgumentList(
-          OS, ClassSpec->getTemplateArgs().asArray(), Policy,
-          ClassSpec->getSpecializedTemplate()->getTemplateParameters());
-    }
+    printTemplateArgumentList(
+        OS, ClassSpec->getTemplateArgs().asArray(), Policy,
+        ClassSpec->getSpecializedTemplate()->getTemplateParameters());
     return cxstring::createDup(OS.str());
   }
 
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index 65df513d2713..b76627cb9be6 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -2213,6 +2213,18 @@ TEST_P(ASTMatchersTest, ReferenceTypeLocTest_BindsToAnyRvalueReferenceTypeLoc) {
   EXPECT_TRUE(matches("float&& r = 3.0;", matcher));
 }
 
+TEST_P(
+    ASTMatchersTest,
+    TemplateSpecializationTypeLocTest_BindsToTemplateSpecializationExplicitInstantiation) {
+  if (!GetParam().isCXX()) {
+    return;
+  }
+  EXPECT_TRUE(
+      matches("template <typename T> class C {}; template class C<int>;",
+              classTemplateSpecializationDecl(
+                  hasName("C"), hasTypeLoc(templateSpecializationTypeLoc()))));
+}
+
 TEST_P(ASTMatchersTest,
        TemplateSpecializationTypeLocTest_BindsToVarDeclTemplateSpecialization) {
   if (!GetParam().isCXX()) {
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
index af99c73f1945..f198dc71eb83 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
@@ -430,6 +430,12 @@ TEST(HasTypeLoc, MatchesCXXUnresolvedConstructExpr) {
               cxxUnresolvedConstructExpr(hasTypeLoc(loc(asString("T"))))));
 }
 
+TEST(HasTypeLoc, MatchesClassTemplateSpecializationDecl) {
+  EXPECT_TRUE(matches(
+      "template <typename T> class Foo; template <> class Foo<int> {};",
+      classTemplateSpecializationDecl(hasTypeLoc(loc(asString("Foo<int>"))))));
+}
+
 TEST(HasTypeLoc, MatchesCompoundLiteralExpr) {
   EXPECT_TRUE(
       matches("int* x = (int[2]) { 0, 1 };",
@@ -6378,7 +6384,8 @@ TEST(HasAnyTemplateArgumentLoc, BindsToExplicitSpecializationWithIntArgument) {
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))));
+          hasTypeLoc(templateSpecializationTypeLoc(
+              hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc,
@@ -6387,7 +6394,8 @@ TEST(HasAnyTemplateArgumentLoc,
       "template<typename T> class A {}; template<> class A<double> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("double")))))));
+          hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+              hasTypeLoc(loc(asString("double")))))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
@@ -6397,20 +6405,24 @@ TEST(HasAnyTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
   )";
   EXPECT_TRUE(
       matches(code, classTemplateSpecializationDecl(
-                        hasName("A"), hasAnyTemplateArgumentLoc(hasTypeLoc(
-                                          loc(asString("double")))))));
-
+                        hasName("A"), hasTypeLoc(templateSpecializationTypeLoc(
+                                          hasAnyTemplateArgumentLoc(hasTypeLoc(
+                                              loc(asString("double")))))))));
   EXPECT_TRUE(matches(
-      code, classTemplateSpecializationDecl(
-                hasName("A"),
-                hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))));
+      code,
+      classTemplateSpecializationDecl(
+          hasName("A"),
+          hasTypeLoc(templateSpecializationTypeLoc(
+              hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc, DoesNotBindToSpecializationWithIntArgument) {
-  EXPECT_TRUE(notMatches("template<typename T> class A {}; A<int> a;",
-                         classTemplateSpecializationDecl(
-                             hasName("A"), hasAnyTemplateArgumentLoc(hasTypeLoc(
-                                               loc(asString("double")))))));
+  EXPECT_TRUE(notMatches(
+      "template<typename T> class A {}; A<int> a;",
+      classTemplateSpecializationDecl(
+          hasName("A"),
+          hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+              hasTypeLoc(loc(asString("double")))))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc,
@@ -6419,7 +6431,8 @@ TEST(HasAnyTemplateArgumentLoc,
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("double")))))));
+          hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+              hasTypeLoc(loc(asString("double")))))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToSpecializationWithIntArgument) {
@@ -6440,21 +6453,13 @@ TEST(HasTemplateArgumentLoc, BindsToSpecializationWithDoubleArgument) {
                               0, hasTypeLoc(loc(asString("double")))))))))));
 }
 
-TEST(HasTemplateArgumentLoc, DoesNotBindToSpecializationWithIntArgument) {
-  EXPECT_TRUE(notMatches(
-      "template<typename T> class A {}; A<int> a;",
-      varDecl(hasName("a"),
-              hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
-                  templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                      0, hasTypeLoc(loc(asString("double")))))))))));
-}
-
 TEST(HasTemplateArgumentLoc, BindsToExplicitSpecializationWithIntArgument) {
   EXPECT_TRUE(matches(
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("int")))))));
+          hasTypeLoc(templateSpecializationTypeLoc(
+              hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("int")))))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToExplicitSpecializationWithDoubleArgument) {
@@ -6462,7 +6467,8 @@ TEST(HasTemplateArgumentLoc, BindsToExplicitSpecializationWithDoubleArgument) {
       "template<typename T> class A {}; template<> class A<double> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("double")))))));
+          hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+              0, hasTypeLoc(loc(asString("double")))))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
@@ -6472,12 +6478,23 @@ TEST(HasTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
   )";
   EXPECT_TRUE(matches(
       code, classTemplateSpecializationDecl(
-                hasName("A"), hasTemplateArgumentLoc(
-                                  0, hasTypeLoc(loc(asString("double")))))));
+                hasName("A"),
+                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+                    0, hasTypeLoc(loc(asString("double")))))))));
   EXPECT_TRUE(matches(
       code, classTemplateSpecializationDecl(
                 hasName("A"),
-                hasTemplateArgumentLoc(1, hasTypeLoc(loc(asString("int")))))));
+                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+                    1, hasTypeLoc(loc(asString("int")))))))));
+}
+
+TEST(HasTemplateArgumentLoc, DoesNotBindToSpecializationWithIntArgument) {
+  EXPECT_TRUE(notMatches(
+      "template<typename T> class A {}; A<int> a;",
+      classTemplateSpecializationDecl(
+          hasName("A"),
+          hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+              0, hasTypeLoc(loc(asString("double")))))))));
 }
 
 TEST(HasTemplateArgumentLoc,
@@ -6486,7 +6503,8 @@ TEST(HasTemplateArgumentLoc,
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("double")))))));
+          hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+              0, hasTypeLoc(loc(asString("double")))))))));
 }
 
 TEST(HasTemplateArgumentLoc,
@@ -6497,12 +6515,14 @@ TEST(HasTemplateArgumentLoc,
   )";
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
-                hasName("A"), hasTemplateArgumentLoc(
-                                  1, hasTypeLoc(loc(asString("double")))))));
+                hasName("A"),
+                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+                    1, hasTypeLoc(loc(asString("double")))))))));
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
                 hasName("A"),
-                hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("int")))))));
+                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+                    0, hasTypeLoc(loc(asString("int")))))))));
 }
 
 TEST(HasTemplateArgumentLoc, DoesNotBindWithBadIndex) {
@@ -6512,12 +6532,14 @@ TEST(HasTemplateArgumentLoc, DoesNotBindWithBadIndex) {
   )";
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
-                hasName("A"), hasTemplateArgumentLoc(
-                                  -1, hasTypeLoc(loc(asString("double")))))));
+                hasName("A"),
+                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+                    -1, hasTypeLoc(loc(asString("double")))))))));
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
-                hasName("A"), hasTemplateArgumentLoc(
-                                  100, hasTypeLoc(loc(asString("int")))))));
+                hasName("A"),
+                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+                    100, hasTypeLoc(loc(asString("int")))))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToDeclRefExprWithIntArgument) {
-- 
GitLab


From dad11097096c05564758e539f9f03ef883365fdd Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Tue, 7 May 2024 13:12:22 -0700
Subject: [PATCH 0097/1206] [lldb] Reinstate lldb-sbapi-dwarf-enums target
 (NFC) (#91390)

Alex pointed out in #91254 that we only need the custom target if we had
more than one target depending on it. This isn't the case upstream, but
on our downstream fork, we have a second dependency. Reintroduce the
target so that everything can depend on that, without the
single-dependency foot-gun.
---
 lldb/source/API/CMakeLists.txt | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lldb/source/API/CMakeLists.txt b/lldb/source/API/CMakeLists.txt
index 798a92874f13..aa31caddfde3 100644
--- a/lldb/source/API/CMakeLists.txt
+++ b/lldb/source/API/CMakeLists.txt
@@ -20,7 +20,7 @@ if(LLDB_ENABLE_LUA)
   set(lldb_lua_wrapper ${lua_bindings_dir}/LLDBWrapLua.cpp)
 endif()
 
-# Target to generate SBLanguages.h from Dwarf.def.
+# Generate SBLanguages.h from Dwarf.def.
 set(sb_languages_file
   ${CMAKE_CURRENT_BINARY_DIR}/../../include/lldb/API/SBLanguages.h)
 add_custom_command(
@@ -33,6 +33,8 @@ add_custom_command(
   DEPENDS ${LLVM_MAIN_INCLUDE_DIR}/llvm/BinaryFormat/Dwarf.def
   WORKING_DIRECTORY ${LLVM_LIBRARY_OUTPUT_INTDIR}
 )
+add_custom_target(lldb-sbapi-dwarf-enums
+  DEPENDS ${sb_languages_file})
 
 add_lldb_library(liblldb SHARED ${option_framework}
   SBAddress.cpp
@@ -113,7 +115,9 @@ add_lldb_library(liblldb SHARED ${option_framework}
   SystemInitializerFull.cpp
   ${lldb_python_wrapper}
   ${lldb_lua_wrapper}
-  ${sb_languages_file}
+
+  DEPENDS
+    lldb-sbapi-dwarf-enums
 
   LINK_LIBS
     lldbBreakpoint
-- 
GitLab


From 5e9dd8827b3ccd03f8499b610deb6accd2d71d21 Mon Sep 17 00:00:00 2001
From: Xiang Li <python3kgae@outlook.com>
Date: Tue, 7 May 2024 13:19:52 -0700
Subject: [PATCH 0098/1206] [DirectX] remove string function attribute DXIL not
 allowed (#90778)

Remove string function attribute other than
"waveops-include-helper-lanes" and "fp32-denorm-mode".

Move DXILPrepareModulePass after DXILTranslateMetadataPass since
DXILTranslateMetadataPass needs to use attribute like hlsl.numthreads.

Fixes #90773
---
 llvm/lib/Target/DirectX/DXILMetadata.cpp      |  9 ++++
 llvm/lib/Target/DirectX/DXILMetadata.h        |  1 +
 llvm/lib/Target/DirectX/DXILPrepare.cpp       | 50 ++++++++++++++++++-
 .../Target/DirectX/DirectXTargetMachine.cpp   |  2 +-
 .../Metadata/shaderModel-cs-val-ver-0.0.ll    | 16 ++++++
 .../DirectX/Metadata/shaderModel-cs.ll        |  7 ++-
 llvm/test/tools/dxil-dis/attribute-filter.ll  |  8 +--
 7 files changed, 86 insertions(+), 7 deletions(-)
 create mode 100644 llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs-val-ver-0.0.ll

diff --git a/llvm/lib/Target/DirectX/DXILMetadata.cpp b/llvm/lib/Target/DirectX/DXILMetadata.cpp
index 2d94490a7f24..03758dc76e7e 100644
--- a/llvm/lib/Target/DirectX/DXILMetadata.cpp
+++ b/llvm/lib/Target/DirectX/DXILMetadata.cpp
@@ -40,6 +40,15 @@ void ValidatorVersionMD::update(VersionTuple ValidatorVer) {
 
 bool ValidatorVersionMD::isEmpty() { return Entry->getNumOperands() == 0; }
 
+VersionTuple ValidatorVersionMD::getAsVersionTuple() {
+  if (isEmpty())
+    return VersionTuple(1, 0);
+  auto *ValVerMD = cast<MDNode>(Entry->getOperand(0));
+  auto *MajorMD = mdconst::extract<ConstantInt>(ValVerMD->getOperand(0));
+  auto *MinorMD = mdconst::extract<ConstantInt>(ValVerMD->getOperand(1));
+  return VersionTuple(MajorMD->getZExtValue(), MinorMD->getZExtValue());
+}
+
 static StringRef getShortShaderStage(Triple::EnvironmentType Env) {
   switch (Env) {
   case Triple::Pixel:
diff --git a/llvm/lib/Target/DirectX/DXILMetadata.h b/llvm/lib/Target/DirectX/DXILMetadata.h
index 2f5d7d9fe768..cd9f4c83fbd0 100644
--- a/llvm/lib/Target/DirectX/DXILMetadata.h
+++ b/llvm/lib/Target/DirectX/DXILMetadata.h
@@ -30,6 +30,7 @@ public:
   void update(VersionTuple ValidatorVer);
 
   bool isEmpty();
+  VersionTuple getAsVersionTuple();
 };
 
 void createShaderModelMD(Module &M);
diff --git a/llvm/lib/Target/DirectX/DXILPrepare.cpp b/llvm/lib/Target/DirectX/DXILPrepare.cpp
index 026911946b47..24be644d9fc0 100644
--- a/llvm/lib/Target/DirectX/DXILPrepare.cpp
+++ b/llvm/lib/Target/DirectX/DXILPrepare.cpp
@@ -11,10 +11,14 @@
 /// Language (DXIL).
 //===----------------------------------------------------------------------===//
 
+#include "DXILMetadata.h"
+#include "DXILResourceAnalysis.h"
+#include "DXILShaderFlags.h"
 #include "DirectX.h"
 #include "DirectXIRPasses/PointerTypeAnalysis.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/IR/AttributeMask.h"
 #include "llvm/IR/IRBuilder.h"
@@ -23,6 +27,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/VersionTuple.h"
 
 #define DEBUG_TYPE "dxil-prepare"
 
@@ -80,6 +85,37 @@ constexpr bool isValidForDXIL(Attribute::AttrKind Attr) {
                       Attr);
 }
 
+static void collectDeadStringAttrs(AttributeMask &DeadAttrs, AttributeSet &&AS,
+                                   const StringSet<> &LiveKeys,
+                                   bool AllowExperimental) {
+  for (auto &Attr : AS) {
+    if (!Attr.isStringAttribute())
+      continue;
+    StringRef Key = Attr.getKindAsString();
+    if (LiveKeys.contains(Key))
+      continue;
+    if (AllowExperimental && Key.starts_with("exp-"))
+      continue;
+    DeadAttrs.addAttribute(Key);
+  }
+}
+
+static void removeStringFunctionAttributes(Function &F,
+                                           bool AllowExperimental) {
+  AttributeList Attrs = F.getAttributes();
+  const StringSet<> LiveKeys = {"waveops-include-helper-lanes",
+                                "fp32-denorm-mode"};
+  // Collect DeadKeys in FnAttrs.
+  AttributeMask DeadAttrs;
+  collectDeadStringAttrs(DeadAttrs, Attrs.getFnAttrs(), LiveKeys,
+                         AllowExperimental);
+  collectDeadStringAttrs(DeadAttrs, Attrs.getRetAttrs(), LiveKeys,
+                         AllowExperimental);
+
+  F.removeFnAttrs(DeadAttrs);
+  F.removeRetAttrs(DeadAttrs);
+}
+
 class DXILPrepareModule : public ModulePass {
 
   static Value *maybeGenerateBitcast(IRBuilder<> &Builder,
@@ -110,9 +146,18 @@ public:
       if (!isValidForDXIL(I))
         AttrMask.addAttribute(I);
     }
+
+    dxil::ValidatorVersionMD ValVerMD(M);
+    VersionTuple ValVer = ValVerMD.getAsVersionTuple();
+    bool SkipValidation = ValVer.getMajor() == 0 && ValVer.getMinor() == 0;
+
     for (auto &F : M.functions()) {
       F.removeFnAttrs(AttrMask);
       F.removeRetAttrs(AttrMask);
+      // Only remove string attributes if we are not skipping validation.
+      // This will reserve the experimental attributes when validation version
+      // is 0.0 for experiment mode.
+      removeStringFunctionAttributes(F, SkipValidation);
       for (size_t Idx = 0, End = F.arg_size(); Idx < End; ++Idx)
         F.removeParamAttrs(Idx, AttrMask);
 
@@ -172,7 +217,10 @@ public:
   }
 
   DXILPrepareModule() : ModulePass(ID) {}
-
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addPreserved<ShaderFlagsAnalysisWrapper>();
+    AU.addPreserved<DXILResourceWrapper>();
+  }
   static char ID; // Pass identification.
 };
 char DXILPrepareModule::ID = 0;
diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
index bebca0675522..c853393e4282 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
@@ -79,8 +79,8 @@ public:
   void addCodeGenPrepare() override {
     addPass(createDXILIntrinsicExpansionLegacyPass());
     addPass(createDXILOpLoweringLegacyPass());
-    addPass(createDXILPrepareModulePass());
     addPass(createDXILTranslateMetadataPass());
+    addPass(createDXILPrepareModulePass());
   }
 };
 
diff --git a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs-val-ver-0.0.ll b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs-val-ver-0.0.ll
new file mode 100644
index 000000000000..a85dc43ac2f6
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs-val-ver-0.0.ll
@@ -0,0 +1,16 @@
+; RUN: opt -S -dxil-prepare  %s | FileCheck %s 
+
+target triple = "dxil-pc-shadermodel6.6-compute"
+
+define void @entry() #0 {
+entry:
+  ret void
+}
+
+; Make sure experimental attribute is left when validation version is 0.0.
+; CHECK:attributes #0 = { noinline nounwind "exp-shader"="cs" } 
+attributes #0 = { noinline nounwind "exp-shader"="cs" "hlsl.numthreads"="1,2,1" "hlsl.shader"="compute" }
+
+!dx.valver = !{!0}
+
+!0 = !{i32 0, i32 0}
diff --git a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs.ll b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs.ll
index be4b46f22ef2..343f190d994f 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs.ll
@@ -1,4 +1,6 @@
 ; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+; RUN: opt -S -dxil-prepare  %s | FileCheck %s  --check-prefix=REMOVE_EXTRA_ATTRIBUTE
+
 target triple = "dxil-pc-shadermodel6.6-compute"
 
 ; CHECK: !dx.shaderModel = !{![[SM:[0-9]+]]}
@@ -9,4 +11,7 @@ entry:
   ret void
 }
 
-attributes #0 = { noinline nounwind "hlsl.numthreads"="1,2,1" "hlsl.shader"="compute" }
+; Make sure extra attribute like hlsl.numthreads are removed.
+; And experimental attribute is removed when validator version is not 0.0.
+; REMOVE_EXTRA_ATTRIBUTE:attributes #0 = { noinline nounwind } 
+attributes #0 = { noinline nounwind "exp-shader"="cs" "hlsl.numthreads"="1,2,1" "hlsl.shader"="compute" }
diff --git a/llvm/test/tools/dxil-dis/attribute-filter.ll b/llvm/test/tools/dxil-dis/attribute-filter.ll
index 432a5a1b7101..27590e10d79b 100644
--- a/llvm/test/tools/dxil-dis/attribute-filter.ll
+++ b/llvm/test/tools/dxil-dis/attribute-filter.ll
@@ -19,8 +19,8 @@ define float @fma2(float %0, float %1, float %2) #1 {
   ret float %5
 }
 
-; CHECK: attributes #0 = { nounwind readnone "disable-tail-calls"="false" }
-attributes #0 = { norecurse nounwind readnone willreturn "disable-tail-calls"="false" }
+; CHECK: attributes #0 = { nounwind readnone "fp32-denorm-mode"="any" "waveops-include-helper-lanes" }
+attributes #0 = { norecurse nounwind readnone willreturn "disable-tail-calls"="false" "waveops-include-helper-lanes" "fp32-denorm-mode"="any" }
 
-; CHECK: attributes #1 = { readnone "disable-tail-calls"="false" }
-attributes #1 = { norecurse memory(none) willreturn "disable-tail-calls"="false" }
+; CHECK: attributes #1 = { readnone "fp32-denorm-mode"="ftz" "waveops-include-helper-lanes" }
+attributes #1 = { norecurse memory(none) willreturn "disable-tail-calls"="false" "waveops-include-helper-lanes" "fp32-denorm-mode"="ftz" }
-- 
GitLab


From 9a28814f59e8f52cc63ae3d17023cee8348d9b53 Mon Sep 17 00:00:00 2001
From: Maryam Moghadas <m.moghadas@gmail.com>
Date: Tue, 7 May 2024 16:23:37 -0400
Subject: [PATCH 0099/1206] [PowerPC] Spill non-volatile registers required for
 traceback table (#71115)

On AIX we need to spill all [rfv]N-[rfv]31 when a function clobbers
[rfv]N so that the traceback table contains accurate information.
---
 llvm/lib/Target/PowerPC/PPCFrameLowering.cpp  |   59 +
 llvm/lib/Target/PowerPC/PPCFrameLowering.h    |    1 +
 .../CodeGen/PowerPC/aix-csr-vector-extabi.ll  | 1199 +++++++++++++----
 llvm/test/CodeGen/PowerPC/aix-csr-vector.ll   |  198 ++-
 llvm/test/CodeGen/PowerPC/aix-csr.ll          |  809 +++++++++--
 .../test/CodeGen/PowerPC/aix-spills-for-eh.ll |  301 +++++
 llvm/test/CodeGen/PowerPC/aix32-crsave.mir    |   34 +-
 .../CodeGen/PowerPC/ppc-shrink-wrapping.ll    |   24 +-
 llvm/test/CodeGen/PowerPC/ppc64-crsave.mir    |  105 +-
 9 files changed, 2237 insertions(+), 493 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-spills-for-eh.ll

diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
index 04e9f9e2366e..8444266459c4 100644
--- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -1966,6 +1966,8 @@ void PPCFrameLowering::determineCalleeSaves(MachineFunction &MF,
                                             BitVector &SavedRegs,
                                             RegScavenger *RS) const {
   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
+  if (Subtarget.isAIXABI())
+    updateCalleeSaves(MF, SavedRegs);
 
   const PPCRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
 
@@ -2725,6 +2727,63 @@ bool PPCFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
   return !MF.getSubtarget<PPCSubtarget>().is32BitELFABI();
 }
 
+void PPCFrameLowering::updateCalleeSaves(const MachineFunction &MF,
+                                         BitVector &SavedRegs) const {
+  // The AIX ABI uses traceback tables for EH which require that if callee-saved
+  // register N is used, all registers N-31 must be saved/restored.
+  // NOTE: The check for AIX is not actually what is relevant. Traceback tables
+  // on Linux have the same requirements. It is just that AIX is the only ABI
+  // for which we actually use traceback tables. If another ABI needs to be
+  // supported that also uses them, we can add a check such as
+  // Subtarget.usesTraceBackTables().
+  assert(Subtarget.isAIXABI() &&
+         "Function updateCalleeSaves should only be called for AIX.");
+
+  // If there are no callee saves then there is nothing to do.
+  if (SavedRegs.none())
+    return;
+
+  const MCPhysReg *CSRegs =
+      Subtarget.getRegisterInfo()->getCalleeSavedRegs(&MF);
+  MCPhysReg LowestGPR = PPC::R31;
+  MCPhysReg LowestG8R = PPC::X31;
+  MCPhysReg LowestFPR = PPC::F31;
+  MCPhysReg LowestVR = PPC::V31;
+
+  // Traverse the CSRs twice so as not to rely on ascending ordering of
+  // registers in the array. The first pass finds the lowest numbered
+  // register and the second pass marks all higher numbered registers
+  // for spilling.
+  for (int i = 0; CSRegs[i]; i++) {
+    // Get the lowest numbered register for each class that actually needs
+    // to be saved.
+    MCPhysReg Cand = CSRegs[i];
+    if (!SavedRegs.test(Cand))
+      continue;
+    if (PPC::GPRCRegClass.contains(Cand) && Cand < LowestGPR)
+      LowestGPR = Cand;
+    else if (PPC::G8RCRegClass.contains(Cand) && Cand < LowestG8R)
+      LowestG8R = Cand;
+    else if ((PPC::F4RCRegClass.contains(Cand) ||
+              PPC::F8RCRegClass.contains(Cand)) &&
+             Cand < LowestFPR)
+      LowestFPR = Cand;
+    else if (PPC::VRRCRegClass.contains(Cand) && Cand < LowestVR)
+      LowestVR = Cand;
+  }
+
+  for (int i = 0; CSRegs[i]; i++) {
+    MCPhysReg Cand = CSRegs[i];
+    if ((PPC::GPRCRegClass.contains(Cand) && Cand > LowestGPR) ||
+        (PPC::G8RCRegClass.contains(Cand) && Cand > LowestG8R) ||
+        ((PPC::F4RCRegClass.contains(Cand) ||
+          PPC::F8RCRegClass.contains(Cand)) &&
+         Cand > LowestFPR) ||
+        (PPC::VRRCRegClass.contains(Cand) && Cand > LowestVR))
+      SavedRegs.set(Cand);
+  }
+}
+
 uint64_t PPCFrameLowering::getStackThreshold() const {
   // On PPC64, we use `stux r1, r1, <scratch_reg>` to extend the stack;
   // use `add r1, r1, <scratch_reg>` to release the stack frame.
diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.h b/llvm/lib/Target/PowerPC/PPCFrameLowering.h
index e19087ce0e18..d74c87428326 100644
--- a/llvm/lib/Target/PowerPC/PPCFrameLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.h
@@ -173,6 +173,7 @@ public:
   /// function prologue/epilogue.
   bool canUseAsPrologue(const MachineBasicBlock &MBB) const override;
   bool canUseAsEpilogue(const MachineBasicBlock &MBB) const override;
+  void updateCalleeSaves(const MachineFunction &MF, BitVector &SavedRegs) const;
 
   uint64_t getStackThreshold() const override;
 };
diff --git a/llvm/test/CodeGen/PowerPC/aix-csr-vector-extabi.ll b/llvm/test/CodeGen/PowerPC/aix-csr-vector-extabi.ll
index 67397e4adf4e..b99ef4904d54 100644
--- a/llvm/test/CodeGen/PowerPC/aix-csr-vector-extabi.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-csr-vector-extabi.ll
@@ -23,92 +23,259 @@ entry:
 
 ; MIR32:         name:            vec_regs
 
-; MIR32-LABEL:   fixedStack:
-; MIR32-NEXT:    - { id: 0, type: spill-slot, offset: -16, size: 16, alignment: 16, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$v31', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 1, type: spill-slot, offset: -96, size: 16, alignment: 16, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$v26', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 2, type: spill-slot, offset: -192, size: 16, alignment: 16, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$v20', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
+; MIR32-LABEL:  fixedStack:
+; MIR32-NEXT:     - { id: 0, type: spill-slot, offset: -16, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:         callee-saved-register: '$v31', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:         debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:     - { id: 1, type: spill-slot, offset: -32, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:         callee-saved-register: '$v30', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:         debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:     - { id: 2, type: spill-slot, offset: -48, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:         callee-saved-register: '$v29', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:         debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:     - { id: 3, type: spill-slot, offset: -64, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:         callee-saved-register: '$v28', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:         debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:     - { id: 4, type: spill-slot, offset: -80, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:         callee-saved-register: '$v27', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:         debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:     - { id: 5, type: spill-slot, offset: -96, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:         callee-saved-register: '$v26', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:         debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:     - { id: 6, type: spill-slot, offset: -112, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:         callee-saved-register: '$v25', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:         debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:     - { id: 7, type: spill-slot, offset: -128, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:         callee-saved-register: '$v24', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:         debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:     - { id: 8, type: spill-slot, offset: -144, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:         callee-saved-register: '$v23', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:         debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:     - { id: 9, type: spill-slot, offset: -160, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:         callee-saved-register: '$v22', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:         debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:     - { id: 10, type: spill-slot, offset: -176, size: 16, alignment: 16,
+; MIR32-NEXT:         stack-id: default, callee-saved-register: '$v21', callee-saved-restored: true,
+; MIR32-NEXT:         debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:     - { id: 11, type: spill-slot, offset: -192, size: 16, alignment: 16,
+; MIR32-NEXT:         stack-id: default, callee-saved-register: '$v20', callee-saved-restored: true,
+; MIR32-NEXT:         debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
 ; MIR32-NEXT:    stack:
 
-; MIR32:         liveins: $v20, $v26, $v31
+; MIR32: liveins: $v20, $v21, $v22, $v23, $v24, $v25, $v26, $v27, $v28, $v29, $v30, $v31
 
-; MIR32-DAG:     STXVD2X killed $v20, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.2)
-; MIR32-DAG:     STXVD2X killed $v26, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.1)
+; MIR32-DAG:     STXVD2X killed $v20, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.11)
+; MIR32-DAG:     STXVD2X killed $v21, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.10)
+; MIR32-DAG:     STXVD2X killed $v22, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.9)
+; MIR32-DAG:     STXVD2X killed $v23, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.8)
+; MIR32-DAG:     STXVD2X killed $v24, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.7)
+; MIR32-DAG:     STXVD2X killed $v25, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.6)
+; MIR32-DAG:     STXVD2X killed $v26, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.5)
+; MIR32-DAG:     STXVD2X killed $v27, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.4)
+; MIR32-DAG:     STXVD2X killed $v28, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.3)
+; MIR32-DAG:     STXVD2X killed $v29, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.2)
+; MIR32-DAG:     STXVD2X killed $v30, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.1)
 ; MIR32-DAG:     STXVD2X killed $v31, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.0)
 
 ; MIR32:         INLINEASM
 
-; MIR32-DAG:     $v20 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.2)
-; MIR32-DAG:     $v26 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.1)
 ; MIR32-DAG:     $v31 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.0)
+; MIR32-DAG:     $v30 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.1)
+; MIR32-DAG:     $v29 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.2)
+; MIR32-DAG:     $v28 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.3)
+; MIR32-DAG:     $v27 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.4)
+; MIR32-DAG:     $v26 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.5)
+; MIR32-DAG:     $v25 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.6)
+; MIR32-DAG:     $v24 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.7)
+; MIR32-DAG:     $v23 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.8)
+; MIR32-DAG:     $v22 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.9)
+; MIR32-DAG:     $v21 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.10)
+; MIR32-DAG:     $v20 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.11)
 ; MIR32:         BLR implicit $lr, implicit $rm
 
 ; MIR64:         name:            vec_regs
 
 ; MIR64-LABEL:   fixedStack:
-; MIR64-NEXT:    - { id: 0, type: spill-slot, offset: -16, size: 16, alignment: 16, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$v31', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 1, type: spill-slot, offset: -96, size: 16, alignment: 16, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$v26', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 2, type: spill-slot, offset: -192, size: 16, alignment: 16, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$v20', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 0, type: spill-slot, offset: -16, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v31', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 1, type: spill-slot, offset: -32, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v30', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 2, type: spill-slot, offset: -48, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v29', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 3, type: spill-slot, offset: -64, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v28', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 4, type: spill-slot, offset: -80, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v27', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 5, type: spill-slot, offset: -96, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v26', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 6, type: spill-slot, offset: -112, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v25', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 7, type: spill-slot, offset: -128, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v24', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 8, type: spill-slot, offset: -144, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v23', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 9, type: spill-slot, offset: -160, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v22', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 10, type: spill-slot, offset: -176, size: 16, alignment: 16,
+; MIR64-DAG:           stack-id: default, callee-saved-register: '$v21', callee-saved-restored: true,
+; MIR64-DAG:           debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 11, type: spill-slot, offset: -192, size: 16, alignment: 16,
+; MIR64-DAG:           stack-id: default, callee-saved-register: '$v20', callee-saved-restored: true,
+; MIR64-DAG:           debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
 ; MIR64-NEXT:    stack:
 
-; MIR64:         liveins: $v20, $v26, $v31
+; MIR64: liveins: $v20, $v21, $v22, $v23, $v24, $v25, $v26, $v27, $v28, $v29, $v30, $v31
 
-; MIR64-DAG:     STXVD2X killed $v20, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.2)
-; MIR64-DAG:     STXVD2X killed $v26, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.1)
-; MIR64-DAG:     STXVD2X killed $v31, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.0)
+; MIR64-DAG:   STXVD2X killed $v20, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.11)
+; MIR64-DAG:   STXVD2X killed $v21, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.10)
+; MIR64-DAG:   STXVD2X killed $v22, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.9)
+; MIR64-DAG:   STXVD2X killed $v23, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.8)
+; MIR64-DAG:   STXVD2X killed $v24, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.7)
+; MIR64-DAG:   STXVD2X killed $v25, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.6)
+; MIR64-DAG:   STXVD2X killed $v26, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.5)
+; MIR64-DAG:   STXVD2X killed $v27, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.4)
+; MIR64-DAG:   STXVD2X killed $v28, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.3)
+; MIR64-DAG:   STXVD2X killed $v29, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.2)
+; MIR64-DAG:   STXVD2X killed $v30, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.1)
+; MIR64-DAG:   STXVD2X killed $v31, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.0)
 
-; MIR64:         INLINEASM
+; MIR64:       INLINEASM
 
-; MIR64-DAG:     $v20 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.2)
-; MIR64-DAG:     $v26 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.1)
-; MIR64-DAG:     $v31 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.0)
-; MIR64:         BLR8 implicit $lr8, implicit $rm
+; MIR64-DAG:   $v31 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.0)
+; MIR64-DAG:   $v30 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.1)
+; MIR64-DAG:   $v29 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.2)
+; MIR64-DAG:   $v28 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.3)
+; MIR64-DAG:   $v27 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.4)
+; MIR64-DAG:   $v26 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.5)
+; MIR64-DAG:   $v25 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.6)
+; MIR64-DAG:   $v24 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.7)
+; MIR64-DAG:   $v23 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.8)
+; MIR64-DAG:   $v22 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.9)
+; MIR64-DAG:   $v21 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.10)
+; MIR64-DAG:   $v20 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.11)
+; MIR64:       BLR8 implicit $lr8, implicit $rm
 
 
 ; ASM32-LABEL:   .vec_regs:
 
-; ASM32:         li {{[0-9]+}}, -192
-; ASM32-DAG:     stxvd2x 52, 1, {{[0-9]+}}               # 16-byte Folded Spill
-; ASM32-DAG:     li {{[0-9]+}}, -96
-; ASM32-DAG:     stxvd2x 58, 1, {{[0-9]+}}               # 16-byte Folded Spill
-; ASM32-DAG:     li {{[0-9]+}}, -16
-; ASM32-DAG:     stxvd2x 63, 1, {{[0-9]+}}               # 16-byte Folded Spill
-; ASM32:         #APP
-; ASM32-DAG:     #NO_APP
-; ASM32-DAG:     lxvd2x 63, 1, {{[0-9]+}}       # 16-byte Folded Reload
-; ASM32-DAG:     li {{[0-9]+}}, -96
-; ASM32-DAG:     lxvd2x 58, 1, {{[0-9]+}}       # 16-byte Folded Reload
-; ASM32-DAG:     li {{[0-9]+}}, -192
-; ASM32-DAG:     lxvd2x 52, 1, {{[0-9]+}}       # 16-byte Folded Reload
-; ASM32:         blr
+; ASM32-DAG:       li [[FIXEDSTACK11:[0-9]+]], -192
+; ASM32-DAG:       stxvd2x 52, 1, [[FIXEDSTACK11]]                       # 16-byte Folded Spill
+; ASM32-DAG:       li [[FIXEDSTACK10:[0-9]+]], -176
+; ASM32-DAG:       stxvd2x 53, 1, [[FIXEDSTACK10]]                       # 16-byte Folded Spill
+; ASM32-DAG:       li [[FIXEDSTACK9:[0-9]+]], -160
+; ASM32-DAG:       stxvd2x 54, 1, [[FIXEDSTACK9]]                       # 16-byte Folded Spill
+; ASM32-DAG:       li [[FIXEDSTACK8:[0-9]+]], -144
+; ASM32-DAG:       stxvd2x 55, 1, [[FIXEDSTACK8]]                       # 16-byte Folded Spill
+; ASM32-DAG:       li [[FIXEDSTACK7:[0-9]+]], -128
+; ASM32-DAG:       stxvd2x 56, 1, [[FIXEDSTACK7]]                       # 16-byte Folded Spill
+; ASM32-DAG:       li [[FIXEDSTACK6:[0-9]+]], -112
+; ASM32-DAG:       stxvd2x 57, 1, [[FIXEDSTACK6]]                       # 16-byte Folded Spill
+; ASM32-DAG:       li [[FIXEDSTACK5:[0-9]+]], -96
+; ASM32-DAG:       stxvd2x 58, 1, [[FIXEDSTACK5]]                       # 16-byte Folded Spill
+; ASM32-DAG:       li [[FIXEDSTACK4:[0-9]+]], -80
+; ASM32-DAG:       stxvd2x 59, 1, [[FIXEDSTACK4]]                       # 16-byte Folded Spill
+; ASM32-DAG:       li [[FIXEDSTACK3:[0-9]+]], -64
+; ASM32-DAG:       stxvd2x 60, 1, [[FIXEDSTACK3]]                       # 16-byte Folded Spill
+; ASM32-DAG:       li [[FIXEDSTACK2:[0-9]+]], -48
+; ASM32-DAG:       stxvd2x 61, 1, [[FIXEDSTACK2]]                       # 16-byte Folded Spill
+; ASM32-DAG:       li [[FIXEDSTACK1:[0-9]+]], -32
+; ASM32-DAG:       stxvd2x 62, 1, [[FIXEDSTACK1]]                       # 16-byte Folded Spill
+; ASM32-DAG:       li [[FIXEDSTACK0:[0-9]+]], -16
+; ASM32-DAG:       stxvd2x 63, 1, [[FIXEDSTACK0]]                       # 16-byte Folded Spill
+
+; ASM32:           #APP
+; ASM32-NEXT:      #NO_APP
+
+; ASM32-DAG:       lxvd2x 63, 1, [[FIXEDSTACK0]]                        # 16-byte Folded Reload
+; ASM32-DAG:       li [[FIXEDSTACK1:[0-9]+]], -32
+; ASM32-DAG:       lxvd2x 62, 1, [[FIXEDSTACK1]]                        # 16-byte Folded Reload
+; ASM32-DAG:       li [[FIXEDSTACK2:[0-9]+]], -48
+; ASM32-DAG:       lxvd2x 61, 1, [[FIXEDSTACK2]]                        # 16-byte Folded Reload
+; ASM32-DAG:       li [[FIXEDSTACK3:[0-9]+]], -64
+; ASM32-DAG:       lxvd2x 60, 1, [[FIXEDSTACK3]]                        # 16-byte Folded Reload
+; ASM32-DAG:       li [[FIXEDSTACK4:[0-9]+]], -80
+; ASM32-DAG:       lxvd2x 59, 1, [[FIXEDSTACK4]]                        # 16-byte Folded Reload
+; ASM32-DAG:       li [[FIXEDSTACK5:[0-9]+]], -96
+; ASM32-DAG:       lxvd2x 58, 1, [[FIXEDSTACK5]]                        # 16-byte Folded Reload
+; ASM32-DAG:       li [[FIXEDSTACK6:[0-9]+]], -112
+; ASM32-DAG:       lxvd2x 57, 1, [[FIXEDSTACK6]]                        # 16-byte Folded Reload
+; ASM32-DAG:       li [[FIXEDSTACK7:[0-9]+]], -128
+; ASM32-DAG:       lxvd2x 56, 1, [[FIXEDSTACK7]]                        # 16-byte Folded Reload
+; ASM32-DAG:       li [[FIXEDSTACK8:[0-9]+]], -144
+; ASM32-DAG:       lxvd2x 55, 1, [[FIXEDSTACK8]]                        # 16-byte Folded Reload
+; ASM32-DAG:       li [[FIXEDSTACK9:[0-9]+]], -160
+; ASM32-DAG:       lxvd2x 54, 1, [[FIXEDSTACK9]]                        # 16-byte Folded Reload
+; ASM32-DAG:       li [[FIXEDSTACK10:[0-9]+]], -176
+; ASM32-DAG:       lxvd2x 53, 1, [[FIXEDSTACK10]]                       # 16-byte Folded Reload
+; ASM32-DAG:       li [[FIXEDSTACK11:[0-9]+]], -192
+; ASM32-DAG:       lxvd2x 52, 1, [[FIXEDSTACK11]]                       # 16-byte Folded Reload
+; ASM32:           blr
 
 ; ASM64-LABEL:   .vec_regs:
 
-; ASM64-DAG:     li {{[0-9]+}}, -192
-; ASM64-DAG:     stxvd2x 52, 1, {{[0-9]+}}               # 16-byte Folded Spill
-; ASM64-DAG:     li {{[0-9]+}}, -96
-; ASM64-DAG:     stxvd2x 58, 1, {{[0-9]+}}               # 16-byte Folded Spill
-; ASM64-DAG:     li {{[0-9]+}}, -16
-; ASM64-DAG:     stxvd2x {{[0-9]+}}, 1, {{[0-9]+}}      # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK11:[0-9]+]], -192
+; ASM64-DAG:       stxvd2x 52, 1, [[FIXEDSTACK11]]                   # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK10:[0-9]+]], -176
+; ASM64-DAG:       stxvd2x 53, 1, [[FIXEDSTACK10]]                   # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK9:[0-9]+]], -160
+; ASM64-DAG:       stxvd2x 54, 1, [[FIXEDSTACK9]]                    # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK8:[0-9]+]], -144
+; ASM64-DAG:       stxvd2x 55, 1, [[FIXEDSTACK8]]                    # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK7:[0-9]+]], -128
+; ASM64-DAG:       stxvd2x 56, 1, [[FIXEDSTACK7]]                    # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK6:[0-9]+]], -112
+; ASM64-DAG:       stxvd2x 57, 1, [[FIXEDSTACK6]]                    # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK5:[0-9]+]], -96
+; ASM64-DAG:       stxvd2x 58, 1, [[FIXEDSTACK5]]                    # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK4:[0-9]+]], -80
+; ASM64-DAG:       stxvd2x 59, 1, [[FIXEDSTACK4]]                    # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK3:[0-9]+]], -64
+; ASM64-DAG:       stxvd2x 60, 1, [[FIXEDSTACK3]]                    # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK2:[0-9]+]], -48
+; ASM64-DAG:       stxvd2x 61, 1, [[FIXEDSTACK2]]                    # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK1:[0-9]+]], -32
+; ASM64-DAG:       stxvd2x 62, 1, [[FIXEDSTACK1]]                    # 16-byte Folded Spill
+; ASM64-DAG:       li [[FIXEDSTACK0:[0-9]+]], -16
+; ASM64-DAG:       stxvd2x 63, 1, [[FIXEDSTACK0]]                    # 16-byte Folded Spill
+
 ; ASM64-DAG:     #APP
 ; ASM64-DAG:     #NO_APP
-; ASM64-DAG:     lxvd2x {{[0-9]+}}, 1, {{[0-9]+}}       # 16-byte Folded Reload
-; ASM64-DAG:     li {{[0-9]+}}, -96
-; ASM64-DAG:     lxvd2x 58, 1, {{[0-9]+}}                # 16-byte Folded Reload
-; ASM64-DAG:     li {{[0-9]+}}, -192
-; ASM64-DAG:     lxvd2x 52, 1, {{[0-9]+}}                # 16-byte Folded Reload
-; ASM64-DAG:     blr
+
+; ASM64-DAG:     lxvd2x 63, 1, [[FIXEDSTACK0]]                         # 16-byte Folded Reload
+; ASM64-DAG:     li [[FIXEDSTACK1:[0-9]+]], -32
+; ASM64-DAG:     lxvd2x 62, 1, [[FIXEDSTACK1]]                         # 16-byte Folded Reload
+; ASM64-DAG:     li [[FIXEDSTACK2:[0-9]+]], -48
+; ASM64-DAG:     lxvd2x 61, 1, [[FIXEDSTACK2]]                         # 16-byte Folded Reload
+; ASM64-DAG:     li [[FIXEDSTACK3:[0-9]+]], -64
+; ASM64-DAG:     lxvd2x 60, 1, [[FIXEDSTACK3]]                         # 16-byte Folded Reload
+; ASM64-DAG:     li [[FIXEDSTACK4:[0-9]+]], -80
+; ASM64-DAG:     lxvd2x 59, 1, [[FIXEDSTACK4]]                         # 16-byte Folded Reload
+; ASM64-DAG:     li [[FIXEDSTACK5:[0-9]+]], -96
+; ASM64-DAG:     lxvd2x 58, 1, [[FIXEDSTACK5]]                         # 16-byte Folded Reload
+; ASM64-DAG:     li [[FIXEDSTACK6:[0-9]+]], -112
+; ASM64-DAG:     lxvd2x 57, 1, [[FIXEDSTACK6]]                         # 16-byte Folded Reload
+; ASM64-DAG:     li [[FIXEDSTACK7:[0-9]+]], -128
+; ASM64-DAG:     lxvd2x 56, 1, [[FIXEDSTACK7]]                         # 16-byte Folded Reload
+; ASM64-DAG:     li [[FIXEDSTACK8:[0-9]+]], -144
+; ASM64-DAG:     lxvd2x 55, 1, [[FIXEDSTACK8]]                         # 16-byte Folded Reload
+; ASM64-DAG:     li [[FIXEDSTACK9:[0-9]+]], -160
+; ASM64-DAG:     lxvd2x 54, 1, [[FIXEDSTACK9]]                         # 16-byte Folded Reload
+; ASM64-DAG:     li [[FIXEDSTACK10:[0-9]+]], -176
+; ASM64-DAG:     lxvd2x 53, 1, [[FIXEDSTACK10]]                        # 16-byte Folded Reload
+; ASM64-DAG:     li [[FIXEDSTACK11:[0-9]+]], -192
+; ASM64-DAG:     lxvd2x 52, 1, [[FIXEDSTACK11]]                        # 16-byte Folded Reload
+
+; ASM64:         blr
 
 define dso_local void @fprs_gprs_vecregs() {
   call void asm sideeffect "", "~{r14},~{r25},~{r31},~{f14},~{f21},~{f31},~{v20},~{v26},~{v31}"()
@@ -118,191 +285,767 @@ define dso_local void @fprs_gprs_vecregs() {
 ; MIR32:         name:            fprs_gprs_vecregs
 
 ; MIR32-LABEL:   fixedStack:
-; MIR32-NEXT:    - { id: 0, type: spill-slot, offset: -240, size: 16, alignment: 16, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$v31', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 1, type: spill-slot, offset: -320, size: 16, alignment: 16, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$v26', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 2, type: spill-slot, offset: -416, size: 16, alignment: 16, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$v20', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 3, type: spill-slot, offset: -8, size: 8, alignment: 8, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$f31', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 4, type: spill-slot, offset: -88, size: 8, alignment: 8, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$f21', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 5, type: spill-slot, offset: -144, size: 8, alignment: 16, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$f14', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 6, type: spill-slot, offset: -148, size: 4, alignment: 4, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$r31', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 7, type: spill-slot, offset: -172, size: 4, alignment: 4, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$r25', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 8, type: spill-slot, offset: -216, size: 4, alignment: 8, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$r14', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 0, type: spill-slot, offset: -240, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$v31', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 1, type: spill-slot, offset: -256, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$v30', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 2, type: spill-slot, offset: -272, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$v29', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 3, type: spill-slot, offset: -288, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$v28', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 4, type: spill-slot, offset: -304, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$v27', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 5, type: spill-slot, offset: -320, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$v26', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 6, type: spill-slot, offset: -336, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$v25', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 7, type: spill-slot, offset: -352, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$v24', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 8, type: spill-slot, offset: -368, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$v23', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 9, type: spill-slot, offset: -384, size: 16, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$v22', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 10, type: spill-slot, offset: -400, size: 16, alignment: 16,
+; MIR32-NEXT:          stack-id: default, callee-saved-register: '$v21', callee-saved-restored: true,
+; MIR32-NEXT:          debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 11, type: spill-slot, offset: -416, size: 16, alignment: 16,
+; MIR32-NEXT:          stack-id: default, callee-saved-register: '$v20', callee-saved-restored: true,
+; MIR32-NEXT:          debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 12, type: spill-slot, offset: -8, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f31', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 13, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f30', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 14, type: spill-slot, offset: -24, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f29', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 15, type: spill-slot, offset: -32, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f28', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 16, type: spill-slot, offset: -40, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f27', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 17, type: spill-slot, offset: -48, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f26', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 18, type: spill-slot, offset: -56, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f25', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 19, type: spill-slot, offset: -64, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f24', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 20, type: spill-slot, offset: -72, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f23', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 21, type: spill-slot, offset: -80, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f22', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 22, type: spill-slot, offset: -88, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f21', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 23, type: spill-slot, offset: -96, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f20', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 24, type: spill-slot, offset: -104, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f19', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 25, type: spill-slot, offset: -112, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f18', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 26, type: spill-slot, offset: -120, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f17', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 27, type: spill-slot, offset: -128, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f16', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 28, type: spill-slot, offset: -136, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f15', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 29, type: spill-slot, offset: -144, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$f14', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 30, type: spill-slot, offset: -148, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r31', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 31, type: spill-slot, offset: -152, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r30', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 32, type: spill-slot, offset: -156, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r29', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 33, type: spill-slot, offset: -160, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r28', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 34, type: spill-slot, offset: -164, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r27', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 35, type: spill-slot, offset: -168, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r26', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 36, type: spill-slot, offset: -172, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r25', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 37, type: spill-slot, offset: -176, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r24', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 38, type: spill-slot, offset: -180, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r23', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 39, type: spill-slot, offset: -184, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r22', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 40, type: spill-slot, offset: -188, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r21', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 41, type: spill-slot, offset: -192, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r20', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 42, type: spill-slot, offset: -196, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r19', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 43, type: spill-slot, offset: -200, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r18', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 44, type: spill-slot, offset: -204, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r17', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 45, type: spill-slot, offset: -208, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r16', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 46, type: spill-slot, offset: -212, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r15', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:      - { id: 47, type: spill-slot, offset: -216, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:          callee-saved-register: '$r14', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:          debug-info-expression: '', debug-info-location: '' }
 ; MIR32-NEXT:    stack:
 
-; MIR32:         liveins: $r14, $r25, $r31, $f14, $f21, $f31, $v20, $v26, $v31
-
-; MIR32-DAG:     STW killed $r14, 232, $r1 :: (store (s32) into %fixed-stack.8, align 8)
-; MIR32-DAG:     STW killed $r25, 276, $r1 :: (store (s32) into %fixed-stack.7)
-; MIR32-DAG:     STW killed $r31, 300, $r1 :: (store (s32) into %fixed-stack.6)
-; MIR32-DAG:     STFD killed $f14, 304, $r1 :: (store (s64) into %fixed-stack.5, align 16)
-; MIR32-DAG:     STFD killed $f21, 360, $r1 :: (store (s64) into %fixed-stack.4)
-; MIR32-DAG:     STFD killed $f31, 440, $r1 :: (store (s64) into %fixed-stack.3)
-; MIR32-DAG:     $r{{[0-9]+}} = LI 32
-; MIR32-DAG:     STXVD2X killed $v20, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.2)
-; MIR32-DAG:     $r{{[0-9]+}} = LI 128
-; MIR32-DAG:     STXVD2X killed $v26, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.1)
-; MIR32-DAG:     $r{{[0-9]+}} = LI 208
+; MIR32: liveins: $r14, $r15, $r16, $r17, $r18, $r19, $r20, $r21, $r22, $r23, $r24, $r25, $r26, $r27, $r28, $r29, $r30, $r31, $f14, $f15, $f16, $f17, $f18, $f19, $f20, $f21, $f22, $f23, $f24, $f25, $f26, $f27, $f28, $f29, $f30, $f31, $v20, $v21, $v22, $v23, $v24, $v25, $v26, $v27, $v28, $v29, $v30, $v31
+
+; MIR32-DAG:     STW killed $r14, 232, $r1 :: (store (s32) into %fixed-stack.47, align 8)
+; MIR32-DAG:     STW killed $r15, 236, $r1 :: (store (s32) into %fixed-stack.46)
+; MIR32-DAG:     STW killed $r16, 240, $r1 :: (store (s32) into %fixed-stack.45, align 16)
+; MIR32-DAG:     STW killed $r17, 244, $r1 :: (store (s32) into %fixed-stack.44)
+; MIR32-DAG:     STW killed $r18, 248, $r1 :: (store (s32) into %fixed-stack.43, align 8)
+; MIR32-DAG:     STW killed $r19, 252, $r1 :: (store (s32) into %fixed-stack.42)
+; MIR32-DAG:     STW killed $r20, 256, $r1 :: (store (s32) into %fixed-stack.41, align 16)
+; MIR32-DAG:     STW killed $r21, 260, $r1 :: (store (s32) into %fixed-stack.40)
+; MIR32-DAG:     STW killed $r22, 264, $r1 :: (store (s32) into %fixed-stack.39, align 8)
+; MIR32-DAG:     STW killed $r23, 268, $r1 :: (store (s32) into %fixed-stack.38)
+; MIR32-DAG:     STW killed $r24, 272, $r1 :: (store (s32) into %fixed-stack.37, align 16)
+; MIR32-DAG:     STW killed $r25, 276, $r1 :: (store (s32) into %fixed-stack.36)
+; MIR32-DAG:     STW killed $r26, 280, $r1 :: (store (s32) into %fixed-stack.35, align 8)
+; MIR32-DAG:     STW killed $r27, 284, $r1 :: (store (s32) into %fixed-stack.34)
+; MIR32-DAG:     STW killed $r28, 288, $r1 :: (store (s32) into %fixed-stack.33, align 16)
+; MIR32-DAG:     STW killed $r29, 292, $r1 :: (store (s32) into %fixed-stack.32)
+; MIR32-DAG:     STW killed $r30, 296, $r1 :: (store (s32) into %fixed-stack.31, align 8)
+; MIR32-DAG:     STW killed $r31, 300, $r1 :: (store (s32) into %fixed-stack.30)
+; MIR32-DAG:     STFD killed $f14, 304, $r1 :: (store (s64) into %fixed-stack.29, align 16)
+; MIR32-DAG:     STFD killed $f15, 312, $r1 :: (store (s64) into %fixed-stack.28)
+; MIR32-DAG:     STFD killed $f16, 320, $r1 :: (store (s64) into %fixed-stack.27, align 16)
+; MIR32-DAG:     STFD killed $f17, 328, $r1 :: (store (s64) into %fixed-stack.26)
+; MIR32-DAG:     STFD killed $f18, 336, $r1 :: (store (s64) into %fixed-stack.25, align 16)
+; MIR32-DAG:     STFD killed $f19, 344, $r1 :: (store (s64) into %fixed-stack.24)
+; MIR32-DAG:     STFD killed $f20, 352, $r1 :: (store (s64) into %fixed-stack.23, align 16)
+; MIR32-DAG:     STFD killed $f21, 360, $r1 :: (store (s64) into %fixed-stack.22)
+; MIR32-DAG:     STFD killed $f22, 368, $r1 :: (store (s64) into %fixed-stack.21, align 16)
+; MIR32-DAG:     STFD killed $f23, 376, $r1 :: (store (s64) into %fixed-stack.20)
+; MIR32-DAG:     STFD killed $f24, 384, $r1 :: (store (s64) into %fixed-stack.19, align 16)
+; MIR32-DAG:     STFD killed $f25, 392, $r1 :: (store (s64) into %fixed-stack.18)
+; MIR32-DAG:     STFD killed $f26, 400, $r1 :: (store (s64) into %fixed-stack.17, align 16)
+; MIR32-DAG:     STFD killed $f27, 408, $r1 :: (store (s64) into %fixed-stack.16)
+; MIR32-DAG:     STFD killed $f28, 416, $r1 :: (store (s64) into %fixed-stack.15, align 16)
+; MIR32-DAG:     STFD killed $f29, 424, $r1 :: (store (s64) into %fixed-stack.14)
+; MIR32-DAG:     STFD killed $f30, 432, $r1 :: (store (s64) into %fixed-stack.13, align 16)
+; MIR32-DAG:     STFD killed $f31, 440, $r1 :: (store (s64) into %fixed-stack.12)
+; MIR32-DAG:     STXVD2X killed $v20, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.11)
+; MIR32-DAG:     STXVD2X killed $v21, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.10)
+; MIR32-DAG:     STXVD2X killed $v22, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.9)
+; MIR32-DAG:     STXVD2X killed $v23, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.8)
+; MIR32-DAG:     STXVD2X killed $v24, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.7)
+; MIR32-DAG:     STXVD2X killed $v25, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.6)
+; MIR32-DAG:     STXVD2X killed $v26, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.5)
+; MIR32-DAG:     STXVD2X killed $v27, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.4)
+; MIR32-DAG:     STXVD2X killed $v28, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.3)
+; MIR32-DAG:     STXVD2X killed $v29, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.2)
+; MIR32-DAG:     STXVD2X killed $v30, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.1)
 ; MIR32-DAG:     STXVD2X killed $v31, $r1, killed $r{{[0-9]+}} :: (store (s128) into %fixed-stack.0)
-; MIR32-DAG:     $r1 = STWU $r1, -448, $r1
 
 ; MIR32:         INLINEASM
 
-; MIR32-DAG:     $r14 = LWZ 232, $r1 :: (load (s32) from %fixed-stack.8, align 8)
-; MIR32-DAG:     $r25 = LWZ 276, $r1 :: (load (s32) from %fixed-stack.7)
-; MIR32-DAG:     $r31 = LWZ 300, $r1 :: (load (s32) from %fixed-stack.6)
-; MIR32-DAG:     $f14 = LFD 304, $r1 :: (load (s64) from %fixed-stack.5, align 16)
-; MIR32-DAG:     $f21 = LFD 360, $r1 :: (load (s64) from %fixed-stack.4)
-; MIR32-DAG:     $f31 = LFD 440, $r1 :: (load (s64) from %fixed-stack.3)
-; MIR32-DAG:     $v20 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.2)
-; MIR32-DAG:     $r{{[0-9]+}} = LI 32
-; MIR32-DAG:     $v26 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.1)
-; MIR32-DAG:     $r{{[0-9]+}} = LI 128
-; MIR32-DAG:     $v31 = LXVD2X $r1, killed $r{{[0-9]+}} :: (load (s128) from %fixed-stack.0)
-; MIR32-DAG:     $r{{[0-9]+}} = LI 208
-; MIR32-DAG:     $r1 = ADDI $r1, 448
-; MIR32-DAG:     BLR implicit $lr, implicit $rm
+; MIR32-DAG:     $v31 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.0)
+; MIR32-DAG:     $v30 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.1)
+; MIR32-DAG:     $v29 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.2)
+; MIR32-DAG:     $v28 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.3)
+; MIR32-DAG:     $v27 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.4)
+; MIR32-DAG:     $v26 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.5)
+; MIR32-DAG:     $v25 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.6)
+; MIR32-DAG:     $v24 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.7)
+; MIR32-DAG:     $v23 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.8)
+; MIR32-DAG:     $v22 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.9)
+; MIR32-DAG:     $v21 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.10)
+; MIR32-DAG:     $v20 = LXVD2X $r1, killed $r3 :: (load (s128) from %fixed-stack.11)
+; MIR32-DAG:     $f31 = LFD 440, $r1 :: (load (s64) from %fixed-stack.12)
+; MIR32-DAG:     $f30 = LFD 432, $r1 :: (load (s64) from %fixed-stack.13, align 16)
+; MIR32-DAG:     $f29 = LFD 424, $r1 :: (load (s64) from %fixed-stack.14)
+; MIR32-DAG:     $f28 = LFD 416, $r1 :: (load (s64) from %fixed-stack.15, align 16)
+; MIR32-DAG:     $f27 = LFD 408, $r1 :: (load (s64) from %fixed-stack.16)
+; MIR32-DAG:     $f26 = LFD 400, $r1 :: (load (s64) from %fixed-stack.17, align 16)
+; MIR32-DAG:     $f25 = LFD 392, $r1 :: (load (s64) from %fixed-stack.18)
+; MIR32-DAG:     $f24 = LFD 384, $r1 :: (load (s64) from %fixed-stack.19, align 16)
+; MIR32-DAG:     $f23 = LFD 376, $r1 :: (load (s64) from %fixed-stack.20)
+; MIR32-DAG:     $f22 = LFD 368, $r1 :: (load (s64) from %fixed-stack.21, align 16)
+; MIR32-DAG:     $f21 = LFD 360, $r1 :: (load (s64) from %fixed-stack.22)
+; MIR32-DAG:     $f20 = LFD 352, $r1 :: (load (s64) from %fixed-stack.23, align 16)
+; MIR32-DAG:     $f19 = LFD 344, $r1 :: (load (s64) from %fixed-stack.24)
+; MIR32-DAG:     $f18 = LFD 336, $r1 :: (load (s64) from %fixed-stack.25, align 16)
+; MIR32-DAG:     $f17 = LFD 328, $r1 :: (load (s64) from %fixed-stack.26)
+; MIR32-DAG:     $f16 = LFD 320, $r1 :: (load (s64) from %fixed-stack.27, align 16)
+; MIR32-DAG:     $f15 = LFD 312, $r1 :: (load (s64) from %fixed-stack.28)
+; MIR32-DAG:     $f14 = LFD 304, $r1 :: (load (s64) from %fixed-stack.29, align 16)
+; MIR32-DAG:     $r31 = LWZ 300, $r1 :: (load (s32) from %fixed-stack.30)
+; MIR32-DAG:     $r30 = LWZ 296, $r1 :: (load (s32) from %fixed-stack.31, align 8)
+; MIR32-DAG:     $r29 = LWZ 292, $r1 :: (load (s32) from %fixed-stack.32)
+; MIR32-DAG:     $r28 = LWZ 288, $r1 :: (load (s32) from %fixed-stack.33, align 16)
+; MIR32-DAG:     $r27 = LWZ 284, $r1 :: (load (s32) from %fixed-stack.34)
+; MIR32-DAG:     $r26 = LWZ 280, $r1 :: (load (s32) from %fixed-stack.35, align 8)
+; MIR32-DAG:     $r25 = LWZ 276, $r1 :: (load (s32) from %fixed-stack.36)
+; MIR32-DAG:     $r24 = LWZ 272, $r1 :: (load (s32) from %fixed-stack.37, align 16)
+; MIR32-DAG:     $r23 = LWZ 268, $r1 :: (load (s32) from %fixed-stack.38)
+; MIR32-DAG:     $r22 = LWZ 264, $r1 :: (load (s32) from %fixed-stack.39, align 8)
+; MIR32-DAG:     $r21 = LWZ 260, $r1 :: (load (s32) from %fixed-stack.40)
+; MIR32-DAG:     $r20 = LWZ 256, $r1 :: (load (s32) from %fixed-stack.41, align 16)
+; MIR32-DAG:     $r19 = LWZ 252, $r1 :: (load (s32) from %fixed-stack.42)
+; MIR32-DAG:     $r18 = LWZ 248, $r1 :: (load (s32) from %fixed-stack.43, align 8)
+; MIR32-DAG:     $r17 = LWZ 244, $r1 :: (load (s32) from %fixed-stack.44)
+; MIR32-DAG:     $r16 = LWZ 240, $r1 :: (load (s32) from %fixed-stack.45, align 16)
+; MIR32-DAG:     $r15 = LWZ 236, $r1 :: (load (s32) from %fixed-stack.46)
+; MIR32-DAG:     $r14 = LWZ 232, $r1 :: (load (s32) from %fixed-stack.47, align 8)
+; MIR32:         $r1 = ADDI $r1, 448
+; MIR32-NEXT:    BLR implicit $lr, implicit $rm
+
 
 ; MIR64:         name:            fprs_gprs_vecregs
 
 ; MIR64-LABEL:   fixedStack:
-; MIR64-NEXT:    - { id: 0, type: spill-slot, offset: -304, size: 16, alignment: 16, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$v31', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 1, type: spill-slot, offset: -384, size: 16, alignment: 16, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$v26', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 2, type: spill-slot, offset: -480, size: 16, alignment: 16, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$v20', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 3, type: spill-slot, offset: -8, size: 8, alignment: 8, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$f31', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 4, type: spill-slot, offset: -88, size: 8, alignment: 8, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$f21', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 5, type: spill-slot, offset: -144, size: 8, alignment: 16, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$f14', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 6, type: spill-slot, offset: -152, size: 8, alignment: 8, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$x31', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 7, type: spill-slot, offset: -200, size: 8, alignment: 8, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$x25', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 8, type: spill-slot, offset: -288, size: 8, alignment: 16, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$x14', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 0, type: spill-slot, offset: -304, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v31', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 1, type: spill-slot, offset: -320, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v30', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 2, type: spill-slot, offset: -336, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v29', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 3, type: spill-slot, offset: -352, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v28', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 4, type: spill-slot, offset: -368, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v27', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 5, type: spill-slot, offset: -384, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v26', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 6, type: spill-slot, offset: -400, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v25', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 7, type: spill-slot, offset: -416, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v24', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 8, type: spill-slot, offset: -432, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v23', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 9, type: spill-slot, offset: -448, size: 16, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$v22', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 10, type: spill-slot, offset: -464, size: 16, alignment: 16,
+; MIR64-DAG:           stack-id: default, callee-saved-register: '$v21', callee-saved-restored: true,
+; MIR64-DAG:           debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 11, type: spill-slot, offset: -480, size: 16, alignment: 16,
+; MIR64-DAG:           stack-id: default, callee-saved-register: '$v20', callee-saved-restored: true,
+; MIR64-DAG:           debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 12, type: spill-slot, offset: -8, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f31', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 13, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f30', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 14, type: spill-slot, offset: -24, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f29', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 15, type: spill-slot, offset: -32, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f28', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 16, type: spill-slot, offset: -40, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f27', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 17, type: spill-slot, offset: -48, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f26', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 18, type: spill-slot, offset: -56, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f25', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 19, type: spill-slot, offset: -64, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f24', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 20, type: spill-slot, offset: -72, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f23', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 21, type: spill-slot, offset: -80, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f22', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 22, type: spill-slot, offset: -88, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f21', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 23, type: spill-slot, offset: -96, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f20', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 24, type: spill-slot, offset: -104, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f19', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 25, type: spill-slot, offset: -112, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f18', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 26, type: spill-slot, offset: -120, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f17', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 27, type: spill-slot, offset: -128, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f16', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 28, type: spill-slot, offset: -136, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f15', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 29, type: spill-slot, offset: -144, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$f14', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 30, type: spill-slot, offset: -152, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x31', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 31, type: spill-slot, offset: -160, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x30', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 32, type: spill-slot, offset: -168, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x29', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 33, type: spill-slot, offset: -176, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x28', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 34, type: spill-slot, offset: -184, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x27', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 35, type: spill-slot, offset: -192, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x26', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 36, type: spill-slot, offset: -200, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x25', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 37, type: spill-slot, offset: -208, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x24', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 38, type: spill-slot, offset: -216, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x23', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 39, type: spill-slot, offset: -224, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x22', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 40, type: spill-slot, offset: -232, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x21', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 41, type: spill-slot, offset: -240, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x20', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 42, type: spill-slot, offset: -248, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x19', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 43, type: spill-slot, offset: -256, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x18', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 44, type: spill-slot, offset: -264, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x17', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 45, type: spill-slot, offset: -272, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x16', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 46, type: spill-slot, offset: -280, size: 8, alignment: 8, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x15', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
+; MIR64-DAG:       - { id: 47, type: spill-slot, offset: -288, size: 8, alignment: 16, stack-id: default,
+; MIR64-DAG:           callee-saved-register: '$x14', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-DAG:           debug-info-expression: '', debug-info-location: '' }
 ; MIR64-NEXT:    stack:
 
-; MIR64:         liveins: $x14, $x25, $x31, $f14, $f21, $f31, $v20, $v26, $v31
-
-; MIR64-DAG:     $x1 = STDU $x1, -544, $x1
-; MIR64-DAG:     STD killed $x14, 256, $x1 :: (store (s64) into %fixed-stack.8, align 16)
-; MIR64-DAG:     STD killed $x25, 344, $x1 :: (store (s64) into %fixed-stack.7)
-; MIR64-DAG:     STD killed $x31, 392, $x1 :: (store (s64) into %fixed-stack.6)
-; MIR64-DAG:     STFD killed $f14, 400, $x1 :: (store (s64) into %fixed-stack.5, align 16)
-; MIR64-DAG:     STFD killed $f21, 456, $x1 :: (store (s64) into %fixed-stack.4)
-; MIR64-DAG:     STFD killed $f31, 536, $x1 :: (store (s64) into %fixed-stack.3)
-; MIR64-DAG:     $x{{[0-9]+}} = LI8 64
-; MIR64-DAG:     STXVD2X killed $v20, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.2)
-; MIR64-DAG:     $x{{[0-9]+}} = LI8 160
-; MIR64-DAG:     STXVD2X killed $v26, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.1)
-; MIR64-DAG:     $x{{[0-9]+}} = LI8 240
-; MIR64-DAG:     STXVD2X killed $v31, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.0)
+; MIR64: liveins: $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $x29, $x30, $x31, $f14, $f15, $f16, $f17, $f18, $f19, $f20, $f21, $f22, $f23, $f24, $f25, $f26, $f27, $f28, $f29, $f30, $f31, $v20, $v21, $v22, $v23, $v24, $v25, $v26, $v27, $v28, $v29, $v30, $v31
+
+; MIR64:         $x1 = STDU $x1, -544, $x1
+;MIR64-DAG:      STD killed $x14, 256, $x1 :: (store (s64) into %fixed-stack.47, align 16)
+;MIR64-DAG:      STD killed $x15, 264, $x1 :: (store (s64) into %fixed-stack.46)
+;MIR64-DAG:      STD killed $x16, 272, $x1 :: (store (s64) into %fixed-stack.45, align 16)
+;MIR64-DAG:      STD killed $x17, 280, $x1 :: (store (s64) into %fixed-stack.44)
+;MIR64-DAG:      STD killed $x18, 288, $x1 :: (store (s64) into %fixed-stack.43, align 16)
+;MIR64-DAG:      STD killed $x19, 296, $x1 :: (store (s64) into %fixed-stack.42)
+;MIR64-DAG:      STD killed $x20, 304, $x1 :: (store (s64) into %fixed-stack.41, align 16)
+;MIR64-DAG:      STD killed $x21, 312, $x1 :: (store (s64) into %fixed-stack.40)
+;MIR64-DAG:      STD killed $x22, 320, $x1 :: (store (s64) into %fixed-stack.39, align 16)
+;MIR64-DAG:      STD killed $x23, 328, $x1 :: (store (s64) into %fixed-stack.38)
+;MIR64-DAG:      STD killed $x24, 336, $x1 :: (store (s64) into %fixed-stack.37, align 16)
+;MIR64-DAG:      STD killed $x25, 344, $x1 :: (store (s64) into %fixed-stack.36)
+;MIR64-DAG:      STD killed $x26, 352, $x1 :: (store (s64) into %fixed-stack.35, align 16)
+;MIR64-DAG:      STD killed $x27, 360, $x1 :: (store (s64) into %fixed-stack.34)
+;MIR64-DAG:      STD killed $x28, 368, $x1 :: (store (s64) into %fixed-stack.33, align 16)
+;MIR64-DAG:      STD killed $x29, 376, $x1 :: (store (s64) into %fixed-stack.32)
+;MIR64-DAG:      STD killed $x30, 384, $x1 :: (store (s64) into %fixed-stack.31, align 16)
+;MIR64-DAG:      STD killed $x31, 392, $x1 :: (store (s64) into %fixed-stack.30)
+;MIR64-DAG:      STFD killed $f14, 400, $x1 :: (store (s64) into %fixed-stack.29, align 16)
+;MIR64-DAG:      STFD killed $f15, 408, $x1 :: (store (s64) into %fixed-stack.28)
+;MIR64-DAG:      STFD killed $f16, 416, $x1 :: (store (s64) into %fixed-stack.27, align 16)
+;MIR64-DAG:      STFD killed $f17, 424, $x1 :: (store (s64) into %fixed-stack.26)
+;MIR64-DAG:      STFD killed $f18, 432, $x1 :: (store (s64) into %fixed-stack.25, align 16)
+;MIR64-DAG:      STFD killed $f19, 440, $x1 :: (store (s64) into %fixed-stack.24)
+;MIR64-DAG:      STFD killed $f20, 448, $x1 :: (store (s64) into %fixed-stack.23, align 16)
+;MIR64-DAG:      STFD killed $f21, 456, $x1 :: (store (s64) into %fixed-stack.22)
+;MIR64-DAG:      STFD killed $f22, 464, $x1 :: (store (s64) into %fixed-stack.21, align 16)
+;MIR64-DAG:      STFD killed $f23, 472, $x1 :: (store (s64) into %fixed-stack.20)
+;MIR64-DAG:      STFD killed $f24, 480, $x1 :: (store (s64) into %fixed-stack.19, align 16)
+;MIR64-DAG:      STFD killed $f25, 488, $x1 :: (store (s64) into %fixed-stack.18)
+;MIR64-DAG:      STFD killed $f26, 496, $x1 :: (store (s64) into %fixed-stack.17, align 16)
+;MIR64-DAG:      STFD killed $f27, 504, $x1 :: (store (s64) into %fixed-stack.16)
+;MIR64-DAG:      STFD killed $f28, 512, $x1 :: (store (s64) into %fixed-stack.15, align 16)
+;MIR64-DAG:      STFD killed $f29, 520, $x1 :: (store (s64) into %fixed-stack.14)
+;MIR64-DAG:      STFD killed $f30, 528, $x1 :: (store (s64) into %fixed-stack.13, align 16)
+;MIR64-DAG:      STFD killed $f31, 536, $x1 :: (store (s64) into %fixed-stack.12)
+;MIR64-DAG:      STXVD2X killed $v20, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.11)
+;MIR64-DAG:      STXVD2X killed $v21, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.10)
+;MIR64-DAG:      STXVD2X killed $v22, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.9)
+;MIR64-DAG:      STXVD2X killed $v23, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.8)
+;MIR64-DAG:      STXVD2X killed $v24, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.7)
+;MIR64-DAG:      STXVD2X killed $v25, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.6)
+;MIR64-DAG:      STXVD2X killed $v26, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.5)
+;MIR64-DAG:      STXVD2X killed $v27, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.4)
+;MIR64-DAG:      STXVD2X killed $v28, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.3)
+;MIR64-DAG:      STXVD2X killed $v29, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.2)
+;MIR64-DAG:      STXVD2X killed $v30, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.1)
+;MIR64-DAG:      STXVD2X killed $v31, $x1, killed $x{{[0-9]+}} :: (store (s128) into %fixed-stack.0)
 
 ; MIR64:         INLINEASM
 
-; MIR64-DAG:     $x14 = LD 256, $x1 :: (load (s64) from %fixed-stack.8, align 16)
-; MIR64-DAG:     $x25 = LD 344, $x1 :: (load (s64) from %fixed-stack.7)
-; MIR64-DAG:     $x31 = LD 392, $x1 :: (load (s64) from %fixed-stack.6)
-; MIR64-DAG:     $f14 = LFD 400, $x1 :: (load (s64) from %fixed-stack.5, align 16)
-; MIR64-DAG:     $f21 = LFD 456, $x1 :: (load (s64) from %fixed-stack.4)
-; MIR64-DAG:     $f31 = LFD 536, $x1 :: (load (s64) from %fixed-stack.3)
-; MIR64-DAG:     $v20 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.2)
-; MIR64-DAG:     $x{{[0-9]+}} = LI8 64
-; MIR64-DAG:     $v26 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.1)
-; MIR64-DAG:     $x{{[0-9]+}} = LI8 160
 ; MIR64-DAG:     $v31 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.0)
-; MIR64-DAG:     $x{{[0-9]+}} = LI8 240
-; MIR64-DAG:     $x1 = ADDI8 $x1, 544
-; MIR64-DAG:     BLR8 implicit $lr8, implicit $rm
-
-; ASM32-LABEL:   .fprs_gprs_vecregs:
-
-; ASM32:         stwu 1, -448(1)
-; ASM32-DAG:     li {{[0-9]+}}, 32
-; ASM32-DAG:     stw 14, 232(1)                          # 4-byte Folded Spill
-; ASM32-DAG:     stfd 14, 304(1)                         # 8-byte Folded Spill
-; ASM32-DAG:     stxvd2x 52, 1, {{[0-9]+}}               # 16-byte Folded Spill
-; ASM32-DAG:     li {{[0-9]+}}, 128
-; ASM32-DAG:     stw 25, 276(1)                          # 4-byte Folded Spill
-; ASM32-DAG:     stxvd2x 58, 1, {{[0-9]+}}               # 16-byte Folded Spill
-; ASM32-DAG:     li {{[0-9]+}}, 208
-; ASM32-DAG:     stw 31, 300(1)                          # 4-byte Folded Spill
-; ASM32-DAG:     stfd 21, 360(1)                         # 8-byte Folded Spill
-; ASM32-DAG:     stfd 31, 440(1)                         # 8-byte Folded Spill
-; ASM32-DAG:     stxvd2x 63, 1, {{[0-9]+}}               # 16-byte Folded Spill
-; ASM32-DAG:     #APP
-; ASM32-DAG:     #NO_APP
-; ASM32-DAG:     lxvd2x 63, 1, {{[0-9]+}}                # 16-byte Folded Reload
-; ASM32-DAG:     li {{[0-9]+}}, 128
-; ASM32-DAG:     lfd 31, 440(1)                          # 8-byte Folded Reload
-; ASM32-DAG:     lxvd2x 58, 1, {{[0-9]+}}                # 16-byte Folded Reload
-; ASM32-DAG:     li {{[0-9]+}}, 32
-; ASM32-DAG:     lfd 21, 360(1)                          # 8-byte Folded Reload
-; ASM32-DAG:     lxvd2x 52, 1, {{[0-9]+}}                # 16-byte Folded Reload
-; ASM32-DAG:     lfd 14, 304(1)                          # 8-byte Folded Reload
-; ASM32-DAG:     lwz 31, 300(1)                          # 4-byte Folded Reload
-; ASM32-DAG:     lwz 25, 276(1)                          # 4-byte Folded Reload
-; ASM32-DAG:     lwz 14, 232(1)                          # 4-byte Folded Reload
-; ASM32-DAG:     addi 1, 1, 448
-; ASM32:         blr
+; MIR64-DAG:     $v30 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.1)
+; MIR64-DAG:     $v29 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.2)
+; MIR64-DAG:     $v28 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.3)
+; MIR64-DAG:     $v27 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.4)
+; MIR64-DAG:     $v26 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.5)
+; MIR64-DAG:     $v25 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.6)
+; MIR64-DAG:     $v24 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.7)
+; MIR64-DAG:     $v23 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.8)
+; MIR64-DAG:     $v22 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.9)
+; MIR64-DAG:     $v21 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.10)
+; MIR64-DAG:     $v20 = LXVD2X $x1, killed $x{{[0-9]+}} :: (load (s128) from %fixed-stack.11)
+; MIR64-DAG:     $f31 = LFD 536, $x1 :: (load (s64) from %fixed-stack.12)
+; MIR64-DAG:     $f30 = LFD 528, $x1 :: (load (s64) from %fixed-stack.13, align 16)
+; MIR64-DAG:     $f29 = LFD 520, $x1 :: (load (s64) from %fixed-stack.14)
+; MIR64-DAG:     $f28 = LFD 512, $x1 :: (load (s64) from %fixed-stack.15, align 16)
+; MIR64-DAG:     $f27 = LFD 504, $x1 :: (load (s64) from %fixed-stack.16)
+; MIR64-DAG:     $f26 = LFD 496, $x1 :: (load (s64) from %fixed-stack.17, align 16)
+; MIR64-DAG:     $f25 = LFD 488, $x1 :: (load (s64) from %fixed-stack.18)
+; MIR64-DAG:     $f24 = LFD 480, $x1 :: (load (s64) from %fixed-stack.19, align 16)
+; MIR64-DAG:     $f23 = LFD 472, $x1 :: (load (s64) from %fixed-stack.20)
+; MIR64-DAG:     $f22 = LFD 464, $x1 :: (load (s64) from %fixed-stack.21, align 16)
+; MIR64-DAG:     $f21 = LFD 456, $x1 :: (load (s64) from %fixed-stack.22)
+; MIR64-DAG:     $f20 = LFD 448, $x1 :: (load (s64) from %fixed-stack.23, align 16)
+; MIR64-DAG:     $f19 = LFD 440, $x1 :: (load (s64) from %fixed-stack.24)
+; MIR64-DAG:     $f18 = LFD 432, $x1 :: (load (s64) from %fixed-stack.25, align 16)
+; MIR64-DAG:     $f17 = LFD 424, $x1 :: (load (s64) from %fixed-stack.26)
+; MIR64-DAG:     $f16 = LFD 416, $x1 :: (load (s64) from %fixed-stack.27, align 16)
+; MIR64-DAG:     $f15 = LFD 408, $x1 :: (load (s64) from %fixed-stack.28)
+; MIR64-DAG:     $f14 = LFD 400, $x1 :: (load (s64) from %fixed-stack.29, align 16)
+; MIR64-DAG:     $x31 = LD 392, $x1 :: (load (s64) from %fixed-stack.30)
+; MIR64-DAG:     $x30 = LD 384, $x1 :: (load (s64) from %fixed-stack.31, align 16)
+; MIR64-DAG:     $x29 = LD 376, $x1 :: (load (s64) from %fixed-stack.32)
+; MIR64-DAG:     $x28 = LD 368, $x1 :: (load (s64) from %fixed-stack.33, align 16)
+; MIR64-DAG:     $x27 = LD 360, $x1 :: (load (s64) from %fixed-stack.34)
+; MIR64-DAG:     $x26 = LD 352, $x1 :: (load (s64) from %fixed-stack.35, align 16)
+; MIR64-DAG:     $x25 = LD 344, $x1 :: (load (s64) from %fixed-stack.36)
+; MIR64-DAG:     $x24 = LD 336, $x1 :: (load (s64) from %fixed-stack.37, align 16)
+; MIR64-DAG:     $x23 = LD 328, $x1 :: (load (s64) from %fixed-stack.38)
+; MIR64-DAG:     $x22 = LD 320, $x1 :: (load (s64) from %fixed-stack.39, align 16)
+; MIR64-DAG:     $x21 = LD 312, $x1 :: (load (s64) from %fixed-stack.40)
+; MIR64-DAG:     $x20 = LD 304, $x1 :: (load (s64) from %fixed-stack.41, align 16)
+; MIR64-DAG:     $x19 = LD 296, $x1 :: (load (s64) from %fixed-stack.42)
+; MIR64-DAG:     $x18 = LD 288, $x1 :: (load (s64) from %fixed-stack.43, align 16)
+; MIR64-DAG:     $x17 = LD 280, $x1 :: (load (s64) from %fixed-stack.44)
+; MIR64-DAG:     $x16 = LD 272, $x1 :: (load (s64) from %fixed-stack.45, align 16)
+; MIR64-DAG:     $x15 = LD 264, $x1 :: (load (s64) from %fixed-stack.46)
+; MIR64-DAG:     $x14 = LD 256, $x1 :: (load (s64) from %fixed-stack.47, align 16)
+; MIR64:         $x1 = ADDI8 $x1, 544
+; MIR64-NEXT:    BLR8 implicit $lr8, implicit $rm
+
+; ASM32-LABEL:  .fprs_gprs_vecregs:
+
+; ASM32:          stwu 1, -448(1)
+; ASM32-DAG:      li [[FIXEDSTACK11:[0-9]+]], 32
+; ASM32-DAG:      stxvd2x 52, 1, [[FIXEDSTACK11]]                      # 16-byte Folded Spill
+; ASM32-DAG:      li [[FIXEDSTACK10:[0-9]+]], 48
+; ASM32-DAG:      stxvd2x 53, 1, [[FIXEDSTACK10]]                      # 16-byte Folded Spill
+; ASM32-DAG:      li [[FIXEDSTACK9:[0-9]+]], 64
+; ASM32-DAG:      stxvd2x 54, 1, [[FIXEDSTACK9]]                       # 16-byte Folded Spill
+; ASM32-DAG:      li [[FIXEDSTACK8:[0-9]+]], 80
+; ASM32-DAG:      stxvd2x 55, 1, [[FIXEDSTACK8]]                       # 16-byte Folded Spill
+; ASM32-DAG:      li [[FIXEDSTACK7:[0-9]+]], 96
+; ASM32-DAG:      stxvd2x 56, 1, [[FIXEDSTACK7]]                       # 16-byte Folded Spill
+; ASM32-DAG:      li [[FIXEDSTACK6:[0-9]+]], 112
+; ASM32-DAG:      stxvd2x 57, 1, [[FIXEDSTACK6]]                       # 16-byte Folded Spill
+; ASM32-DAG:      li [[FIXEDSTACK5:[0-9]+]], 128
+; ASM32-DAG:      stxvd2x 58, 1, [[FIXEDSTACK5]]                       # 16-byte Folded Spill
+; ASM32-DAG:      li [[FIXEDSTACK4:[0-9]+]], 144
+; ASM32-DAG:      stxvd2x 59, 1, [[FIXEDSTACK4]]                       # 16-byte Folded Spill
+; ASM32-DAG:      li [[FIXEDSTACK3:[0-9]+]], 160
+; ASM32-DAG:      stxvd2x 60, 1, [[FIXEDSTACK3]]                       # 16-byte Folded Spill
+; ASM32-DAG:      li [[FIXEDSTACK2:[0-9]+]], 176
+; ASM32-DAG:      stxvd2x 61, 1, [[FIXEDSTACK2]]                       # 16-byte Folded Spill
+; ASM32-DAG:      li [[FIXEDSTACK1:[0-9]+]], 192
+; ASM32-DAG:      stxvd2x 62, 1, [[FIXEDSTACK1]]                       # 16-byte Folded Spill
+; ASM32-DAG:      li [[FIXEDSTACK0:[0-9]+]], 208
+; ASM32-DAG:      stxvd2x 63, 1, [[FIXEDSTACK0]]                       # 16-byte Folded Spill
+; ASM32-DAG:      stw 14, 232(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 15, 236(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 16, 240(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 17, 244(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 18, 248(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 19, 252(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 20, 256(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 21, 260(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 22, 264(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 23, 268(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 24, 272(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 25, 276(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 26, 280(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 27, 284(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 28, 288(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 29, 292(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 30, 296(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stw 31, 300(1)                          # 4-byte Folded Spill
+; ASM32-DAG:      stfd 14, 304(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 15, 312(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 16, 320(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 17, 328(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 18, 336(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 19, 344(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 20, 352(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 21, 360(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 22, 368(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 23, 376(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 24, 384(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 25, 392(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 26, 400(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 27, 408(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 28, 416(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 29, 424(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 30, 432(1)                         # 8-byte Folded Spill
+; ASM32-DAG:      stfd 31, 440(1)                         # 8-byte Folded Spill
+
+; ASM32:          #APP
+; ASM32-NEXT:     #NO_APP
+
+; ASM32-DAG:      lxvd2x 63, 1, [[FIXEDSTACK0]]                        # 16-byte Folded Reload
+; ASM32-DAG:      li [[FIXEDSTACK1:[0-9]+]], 192
+; ASM32-DAG:      lxvd2x 62, 1, [[FIXEDSTACK1]]                        # 16-byte Folded Reload
+; ASM32-DAG:      li [[FIXEDSTACK2:[0-9]+]], 176
+; ASM32-DAG:      lxvd2x 61, 1, [[FIXEDSTACK2]]                        # 16-byte Folded Reload
+; ASM32-DAG:      li [[FIXEDSTACK3:[0-9]+]], 160
+; ASM32-DAG:      lxvd2x 60, 1, [[FIXEDSTACK3]]                        # 16-byte Folded Reload
+; ASM32-DAG:      li [[FIXEDSTACK4:[0-9]+]], 144
+; ASM32-DAG:      lxvd2x 59, 1, [[FIXEDSTACK4]]                        # 16-byte Folded Reload
+; ASM32-DAG:      li [[FIXEDSTACK5:[0-9]+]], 128
+; ASM32-DAG:      lxvd2x 58, 1, [[FIXEDSTACK5]]                        # 16-byte Folded Reload
+; ASM32-DAG:      li [[FIXEDSTACK6:[0-9]+]], 112
+; ASM32-DAG:      lxvd2x 57, 1, [[FIXEDSTACK6]]                        # 16-byte Folded Reload
+; ASM32-DAG:      li [[FIXEDSTACK7:[0-9]+]], 96
+; ASM32-DAG:      lxvd2x 56, 1, [[FIXEDSTACK7]]                        # 16-byte Folded Reload
+; ASM32-DAG:      li [[FIXEDSTACK8:[0-9]+]], 80
+; ASM32-DAG:      lxvd2x 55, 1, [[FIXEDSTACK8]]                        # 16-byte Folded Reload
+; ASM32-DAG:      li [[FIXEDSTACK9:[0-9]+]], 64
+; ASM32-DAG:      lxvd2x 54, 1, [[FIXEDSTACK9]]                        # 16-byte Folded Reload
+; ASM32-DAG:      li [[FIXEDSTACK10:[0-9]+]], 48
+; ASM32-DAG:      lxvd2x 53, 1, [[FIXEDSTACK10]]                        # 16-byte Folded Reload
+; ASM32-DAG:      li [[FIXEDSTACK11:[0-9]+]], 32
+; ASM32-DAG:      lxvd2x 52, 1, [[FIXEDSTACK11]]                        # 16-byte Folded Reload
+; ASM32-DAG:      lfd 31, 440(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 30, 432(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 29, 424(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 28, 416(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 27, 408(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 26, 400(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 25, 392(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 24, 384(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 23, 376(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 22, 368(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 21, 360(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 20, 352(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 19, 344(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 18, 336(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 17, 328(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 16, 320(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 15, 312(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lfd 14, 304(1)                          # 8-byte Folded Reload
+; ASM32-DAG:      lwz 31, 300(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 30, 296(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 29, 292(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 28, 288(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 27, 284(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 26, 280(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 25, 276(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 24, 272(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 23, 268(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 22, 264(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 21, 260(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 20, 256(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 19, 252(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 18, 248(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 17, 244(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 16, 240(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 15, 236(1)                          # 4-byte Folded Reload
+; ASM32-DAG:      lwz 14, 232(1)                          # 4-byte Folded Reload
+
+; ASM32:          addi 1, 1, 448
+; ASM32-NEXT:     blr
 
 ; ASM64-LABEL:    .fprs_gprs_vecregs:
 
-; ASM64:         stdu 1, -544(1)
-; ASM64-DAG:     li {{[0-9]+}}, 64
-; ASM64-DAG:     std 14, 256(1)                          # 8-byte Folded Spill
-; ASM64-DAG:     stfd 14, 400(1)                         # 8-byte Folded Spill
-; ASM64-DAG:     stxvd2x 52, 1, {{[0-9]+}}               # 16-byte Folded Spill
-; ASM64-DAG:     li {{[0-9]+}}, 160
-; ASM64-DAG:     std 25, 344(1)                          # 8-byte Folded Spill
-; ASM64-DAG:     stxvd2x 58, 1, {{[0-9]+}}               # 16-byte Folded Spill
-; ASM64-DAG:     li {{[0-9]+}}, 240
-; ASM64-DAG:     std 31, 392(1)                          # 8-byte Folded Spill
-; ASM64-DAG:     stfd 21, 456(1)                         # 8-byte Folded Spill
-; ASM64-DAG:     stfd 31, 536(1)                         # 8-byte Folded Spill
-; ASM64-DAG:     stxvd2x 63, 1, {{[0-9]+}}               # 16-byte Folded Spill
-; ASM64-DAG:     #APP
-; ASM64-DAG:     #NO_APP
-; ASM64-DAG:     lxvd2x 63, 1, {{[0-9]+}}                # 16-byte Folded Reload
-; ASM64-DAG:     li {{[0-9]+}}, 160
-; ASM64-DAG:     lfd 31, 536(1)                          # 8-byte Folded Reload
-; ASM64-DAG:     lxvd2x 58, 1, {{[0-9]+}}                # 16-byte Folded Reload
-; ASM64-DAG:     li {{[0-9]+}}, 64
-; ASM64-DAG:     lfd 21, 456(1)                          # 8-byte Folded Reload
-; ASM64-DAG:     lxvd2x 52, 1, {{[0-9]+}}                # 16-byte Folded Reload
-; ASM64-DAG:     lfd 14, 400(1)                          # 8-byte Folded Reload
-; ASM64-DAG:     ld 31, 392(1)                           # 8-byte Folded Reload
-; ASM64-DAG:     ld 25, 344(1)                           # 8-byte Folded Reload
-; ASM64-DAG:     ld 14, 256(1)                           # 8-byte Folded Reload
-; ASM64-DAG:     addi 1, 1, 544
-; ASM64:         blr
+; ASM64:            stdu 1, -544(1)
+; ASM64-DAG:        li [[FIXEDSTACK11:[0-9]+]], 64
+; ASM64-DAG:        stxvd2x 52, 1, [[FIXEDSTACK11]]                       # 16-byte Folded Spill
+; ASM64-DAG:        li [[FIXEDSTACK10:[0-9]+]], 80
+; ASM64-DAG:        stxvd2x 53, 1, [[FIXEDSTACK10]]                       # 16-byte Folded Spill
+; ASM64-DAG:        li [[FIXEDSTACK9:[0-9]+]], 96
+; ASM64-DAG:        stxvd2x 54, 1, [[FIXEDSTACK9]]                        # 16-byte Folded Spill
+; ASM64-DAG:        li [[FIXEDSTACK8:[0-9]+]], 112
+; ASM64-DAG:        stxvd2x 55, 1, [[FIXEDSTACK8]]                        # 16-byte Folded Spill
+; ASM64-DAG:        li [[FIXEDSTACK7:[0-9]+]], 128
+; ASM64-DAG:        stxvd2x 56, 1, [[FIXEDSTACK7]]                        # 16-byte Folded Spill
+; ASM64-DAG:        li [[FIXEDSTACK6:[0-9]+]], 144
+; ASM64-DAG:        stxvd2x 57, 1, [[FIXEDSTACK6]]                        # 16-byte Folded Spill
+; ASM64-DAG:        li [[FIXEDSTACK5:[0-9]+]], 160
+; ASM64-DAG:        stxvd2x 58, 1, [[FIXEDSTACK5]]                        # 16-byte Folded Spill
+; ASM64-DAG:        li [[FIXEDSTACK4:[0-9]+]], 176
+; ASM64-DAG:        stxvd2x 59, 1, [[FIXEDSTACK4]]                        # 16-byte Folded Spill
+; ASM64-DAG:        li [[FIXEDSTACK3:[0-9]+]], 192
+; ASM64-DAG:        stxvd2x 60, 1, [[FIXEDSTACK3]]                        # 16-byte Folded Spill
+; ASM64-DAG:        li [[FIXEDSTACK2:[0-9]+]], 208
+; ASM64-DAG:        stxvd2x 61, 1, [[FIXEDSTACK2]]                        # 16-byte Folded Spill
+; ASM64-DAG:        li [[FIXEDSTACK1:[0-9]+]], 224
+; ASM64-DAG:        stxvd2x 62, 1, [[FIXEDSTACK1]]                        # 16-byte Folded Spill
+; ASM64-DAG:        li [[FIXEDSTACK0:[0-9]+]], 240
+; ASM64-DAG:        stxvd2x 63, 1, [[FIXEDSTACK0]]                        # 16-byte Folded Spill
+; ASM64-DAG:        std 14, 256(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 15, 264(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 16, 272(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 17, 280(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 18, 288(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 19, 296(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 20, 304(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 21, 312(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 22, 320(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 23, 328(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 24, 336(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 25, 344(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 26, 352(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 27, 360(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 28, 368(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 29, 376(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 30, 384(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        std 31, 392(1)                          # 8-byte Folded Spill
+; ASM64-DAG:        stfd 14, 400(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 15, 408(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 16, 416(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 17, 424(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 18, 432(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 19, 440(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 20, 448(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 21, 456(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 22, 464(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 23, 472(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 24, 480(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 25, 488(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 26, 496(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 27, 504(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 28, 512(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 29, 520(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 30, 528(1)                         # 8-byte Folded Spill
+; ASM64-DAG:        stfd 31, 536(1)                         # 8-byte Folded Spill
+
+; ASM64:            #APP
+; ASM64-NEXT:       #NO_APP
+
+; ASM64-DAG:        lxvd2x 63, 1, [[FIXEDSTACK0]]                         # 16-byte Folded Reload
+; ASM64-DAG:        li [[FIXEDSTACK1:[0-9]+]], 224
+; ASM64-DAG:        lxvd2x 62, 1, [[FIXEDSTACK1]]                         # 16-byte Folded Reload
+; ASM64-DAG:        li [[FIXEDSTACK2:[0-9]+]], 208
+; ASM64-DAG:        lxvd2x 61, 1, [[FIXEDSTACK2]]                         # 16-byte Folded Reload
+; ASM64-DAG:        li [[FIXEDSTACK3:[0-9]+]], 192
+; ASM64-DAG:        lxvd2x 60, 1, [[FIXEDSTACK3]]                         # 16-byte Folded Reload
+; ASM64-DAG:        li [[FIXEDSTACK4:[0-9]+]], 176
+; ASM64-DAG:        lxvd2x 59, 1, [[FIXEDSTACK4]]                         # 16-byte Folded Reload
+; ASM64-DAG:        li [[FIXEDSTACK5:[0-9]+]], 160
+; ASM64-DAG:        lxvd2x 58, 1, [[FIXEDSTACK5]]                         # 16-byte Folded Reload
+; ASM64-DAG:        li [[FIXEDSTACK6:[0-9]+]], 144
+; ASM64-DAG:        lxvd2x 57, 1, [[FIXEDSTACK6]]                         # 16-byte Folded Reload
+; ASM64-DAG:        li [[FIXEDSTACK7:[0-9]+]], 128
+; ASM64-DAG:        lxvd2x 56, 1, [[FIXEDSTACK7]]                         # 16-byte Folded Reload
+; ASM64-DAG:        li [[FIXEDSTACK8:[0-9]+]], 112
+; ASM64-DAG:        lxvd2x 55, 1, [[FIXEDSTACK8]]                         # 16-byte Folded Reload
+; ASM64-DAG:        li [[FIXEDSTACK9:[0-9]+]], 96
+; ASM64-DAG:        lxvd2x 54, 1, [[FIXEDSTACK9]]                         # 16-byte Folded Reload
+; ASM64-DAG:        li [[FIXEDSTACK10:[0-9]+]], 80
+; ASM64-DAG:        lxvd2x 53, 1, [[FIXEDSTACK10]]                        # 16-byte Folded Reload
+; ASM64-DAG:        li [[FIXEDSTACK11:[0-9]+]], 64
+; ASM64-DAG:        lxvd2x 52, 1, [[FIXEDSTACK11]]                        # 16-byte Folded Reload
+; ASM64-DAG:        lfd 31, 536(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 30, 528(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 29, 520(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 28, 512(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 27, 504(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 26, 496(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 25, 488(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 24, 480(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 23, 472(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 22, 464(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 21, 456(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 20, 448(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 19, 440(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 18, 432(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 17, 424(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 16, 416(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 15, 408(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        lfd 14, 400(1)                          # 8-byte Folded Reload
+; ASM64-DAG:        ld 31, 392(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 30, 384(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 29, 376(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 28, 368(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 27, 360(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 26, 352(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 25, 344(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 24, 336(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 23, 328(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 22, 320(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 21, 312(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 20, 304(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 19, 296(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 18, 288(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 17, 280(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 16, 272(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 15, 264(1)                           # 8-byte Folded Reload
+; ASM64-DAG:        ld 14, 256(1)                           # 8-byte Folded Reload
+
+; ASM64:            addi 1, 1, 544
+; ASM64-NEXT:       blr
diff --git a/llvm/test/CodeGen/PowerPC/aix-csr-vector.ll b/llvm/test/CodeGen/PowerPC/aix-csr-vector.ll
index 45ec7357656b..9dc06dca3d3b 100644
--- a/llvm/test/CodeGen/PowerPC/aix-csr-vector.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-csr-vector.ll
@@ -63,24 +63,34 @@ define dso_local void @vec_regs() {
 ; ASM64:         blr
 
 define dso_local void @fprs_gprs_vecregs() {
-    call void asm sideeffect "", "~{r14},~{r25},~{r31},~{f14},~{f21},~{f31},~{v20},~{v26},~{v31}"()
+    call void asm sideeffect "", "~{r25},~{r28},~{r31},~{f21},~{f25},~{f31},~{v20},~{v26},~{v31}"()
       ret void
 }
 
 ; MIR32-LABEL:   name:            fprs_gprs_vecregs
 
-; MIR32:         fixedStack:
-
-; MIR32:         liveins: $r14, $r25, $r31, $f14, $f21, $f31
+; MIR32: liveins: $r25, $r26, $r27, $r28, $r29, $r30, $r31, $f21, $f22, $f23, $f24, $f25, $f26, $f27, $f28, $f29, $f30, $f31
 
 ; MIR32-NOT:     STXVD2X killed $v20
 ; MIR32-NOT:     STXVD2X killed $v26
 ; MIR32-NOT:     STXVD2X killed $v31
-; MIR32-DAG:     STW killed $r14, -216, $r1 :: (store (s32) into %fixed-stack.5, align 8)
-; MIR32-DAG:     STW killed $r25, -172, $r1 :: (store (s32) into %fixed-stack.4)
-; MIR32-DAG:     STW killed $r31, -148, $r1 :: (store (s32) into %fixed-stack.3)
-; MIR32-DAG:     STFD killed $f14, -144, $r1 :: (store (s64) into %fixed-stack.2, align 16)
-; MIR32-DAG:     STFD killed $f21, -88, $r1 :: (store (s64) into %fixed-stack.1)
+; MIR32-DAG:     STW killed $r25, -116, $r1 :: (store (s32) into %fixed-stack.17)
+; MIR32-DAG:     STW killed $r26, -112, $r1 :: (store (s32) into %fixed-stack.16, align 8)
+; MIR32-DAG:     STW killed $r27, -108, $r1 :: (store (s32) into %fixed-stack.15)
+; MIR32-DAG:     STW killed $r28, -104, $r1 :: (store (s32) into %fixed-stack.14, align 16)
+; MIR32-DAG:     STW killed $r29, -100, $r1 :: (store (s32) into %fixed-stack.13)
+; MIR32-DAG:     STW killed $r30, -96, $r1 :: (store (s32) into %fixed-stack.12, align 8)
+; MIR32-DAG:     STW killed $r31, -92, $r1 :: (store (s32) into %fixed-stack.11)
+; MIR32-DAG:     STFD killed $f21, -88, $r1 :: (store (s64) into %fixed-stack.10)
+; MIR32-DAG:     STFD killed $f22, -80, $r1 :: (store (s64) into %fixed-stack.9, align 16)
+; MIR32-DAG:     STFD killed $f23, -72, $r1 :: (store (s64) into %fixed-stack.8)
+; MIR32-DAG:     STFD killed $f24, -64, $r1 :: (store (s64) into %fixed-stack.7, align 16)
+; MIR32-DAG:     STFD killed $f25, -56, $r1 :: (store (s64) into %fixed-stack.6)
+; MIR32-DAG:     STFD killed $f26, -48, $r1 :: (store (s64) into %fixed-stack.5, align 16)
+; MIR32-DAG:     STFD killed $f27, -40, $r1 :: (store (s64) into %fixed-stack.4)
+; MIR32-DAG:     STFD killed $f28, -32, $r1 :: (store (s64) into %fixed-stack.3, align 16)
+; MIR32-DAG:     STFD killed $f29, -24, $r1 :: (store (s64) into %fixed-stack.2)
+; MIR32-DAG:     STFD killed $f30, -16, $r1 :: (store (s64) into %fixed-stack.1, align 16)
 ; MIR32-DAG:     STFD killed $f31, -8, $r1 :: (store (s64) into %fixed-stack.0)
 
 ; MIR32-LABEL:   INLINEASM
@@ -88,28 +98,50 @@ define dso_local void @fprs_gprs_vecregs() {
 ; MIR32-NOT:     $v20 = LXVD2X
 ; MIR32-NOT:     $v26 = LXVD2X
 ; MIR32-NOT:     $v31 = LXVD2X
-; MIR32-DAG:     $r14 = LWZ -216, $r1 :: (load (s32) from %fixed-stack.5, align 8)
-; MIR32-DAG:     $r25 = LWZ -172, $r1 :: (load (s32) from %fixed-stack.4)
-; MIR32-DAG:     $r31 = LWZ -148, $r1 :: (load (s32) from %fixed-stack.3)
-; MIR32-DAG:     $f14 = LFD -144, $r1 :: (load (s64) from %fixed-stack.2, align 16)
-; MIR32-DAG:     $f21 = LFD -88, $r1 :: (load (s64) from %fixed-stack.1)
 ; MIR32-DAG:     $f31 = LFD -8, $r1 :: (load (s64) from %fixed-stack.0)
-; MIR32-DAG:     BLR implicit $lr, implicit $rm
+; MIR32-DAG:     $f30 = LFD -16, $r1 :: (load (s64) from %fixed-stack.1, align 16)
+; MIR32-DAG:     $f29 = LFD -24, $r1 :: (load (s64) from %fixed-stack.2)
+; MIR32-DAG:     $f28 = LFD -32, $r1 :: (load (s64) from %fixed-stack.3, align 16)
+; MIR32-DAG:     $f27 = LFD -40, $r1 :: (load (s64) from %fixed-stack.4)
+; MIR32-DAG:     $f26 = LFD -48, $r1 :: (load (s64) from %fixed-stack.5, align 16)
+; MIR32-DAG:     $f25 = LFD -56, $r1 :: (load (s64) from %fixed-stack.6)
+; MIR32-DAG:     $f24 = LFD -64, $r1 :: (load (s64) from %fixed-stack.7, align 16)
+; MIR32-DAG:     $f23 = LFD -72, $r1 :: (load (s64) from %fixed-stack.8)
+; MIR32-DAG:     $f22 = LFD -80, $r1 :: (load (s64) from %fixed-stack.9, align 16)
+; MIR32-DAG:     $f21 = LFD -88, $r1 :: (load (s64) from %fixed-stack.10)
+; MIR32-DAG:     $r31 = LWZ -92, $r1 :: (load (s32) from %fixed-stack.11)
+; MIR32-DAG:     $r30 = LWZ -96, $r1 :: (load (s32) from %fixed-stack.12, align 8)
+; MIR32-DAG:     $r29 = LWZ -100, $r1 :: (load (s32) from %fixed-stack.13)
+; MIR32-DAG:     $r28 = LWZ -104, $r1 :: (load (s32) from %fixed-stack.14, align 16)
+; MIR32-DAG:     $r27 = LWZ -108, $r1 :: (load (s32) from %fixed-stack.15)
+; MIR32-DAG:     $r26 = LWZ -112, $r1 :: (load (s32) from %fixed-stack.16, align 8)
+; MIR32-DAG:     $r25 = LWZ -116, $r1 :: (load (s32) from %fixed-stack.17)
+; MIR32:         BLR implicit $lr, implicit $rm
 
 ; MIR64-LABEL:   name:            fprs_gprs_vecregs
 
-; MIR64:         fixedStack:
-
-; MIR64:         liveins: $x14, $x25, $x31, $f14, $f21, $f31
+; MIR64: liveins: $x25, $x26, $x27, $x28, $x29, $x30, $x31, $f21, $f22, $f23, $f24, $f25, $f26, $f27, $f28, $f29, $f30, $f31
 
 ; MIR64-NOT:     STXVD2X killed $v20
 ; MIR64-NOT:     STXVD2X killed $v26
 ; MIR64-NOT:     STXVD2X killed $v31
-; MIR64-DAG:     STD killed $x14, -288, $x1 :: (store (s64) into %fixed-stack.5, align 16)
-; MIR64-DAG:     STD killed $x25, -200, $x1 :: (store (s64) into %fixed-stack.4)
-; MIR64-DAG:     STD killed $x31, -152, $x1 :: (store (s64) into %fixed-stack.3)
-; MIR64-DAG:     STFD killed $f14, -144, $x1 :: (store (s64) into %fixed-stack.2, align 16)
-; MIR64-DAG:     STFD killed $f21, -88, $x1 :: (store (s64) into %fixed-stack.1)
+; MIR64-DAG:     STD killed $x25, -144, $x1 :: (store (s64) into %fixed-stack.17)
+; MIR64-DAG:     STD killed $x26, -136, $x1 :: (store (s64) into %fixed-stack.16, align 16)
+; MIR64-DAG:     STD killed $x27, -128, $x1 :: (store (s64) into %fixed-stack.15)
+; MIR64-DAG:     STD killed $x28, -120, $x1 :: (store (s64) into %fixed-stack.14, align 16)
+; MIR64-DAG:     STD killed $x29, -112, $x1 :: (store (s64) into %fixed-stack.13)
+; MIR64-DAG:     STD killed $x30, -104, $x1 :: (store (s64) into %fixed-stack.12, align 16)
+; MIR64-DAG:     STD killed $x31, -96, $x1 :: (store (s64) into %fixed-stack.11)
+; MIR64-DAG:     STFD killed $f21, -88, $x1 :: (store (s64) into %fixed-stack.10)
+; MIR64-DAG:     STFD killed $f22, -80, $x1 :: (store (s64) into %fixed-stack.9, align 16)
+; MIR64-DAG:     STFD killed $f23, -72, $x1 :: (store (s64) into %fixed-stack.8)
+; MIR64-DAG:     STFD killed $f24, -64, $x1 :: (store (s64) into %fixed-stack.7, align 16)
+; MIR64-DAG:     STFD killed $f25, -56, $x1 :: (store (s64) into %fixed-stack.6)
+; MIR64-DAG:     STFD killed $f26, -48, $x1 :: (store (s64) into %fixed-stack.5, align 16)
+; MIR64-DAG:     STFD killed $f27, -40, $x1 :: (store (s64) into %fixed-stack.4)
+; MIR64-DAG:     STFD killed $f28, -32, $x1 :: (store (s64) into %fixed-stack.3, align 16)
+; MIR64-DAG:     STFD killed $f29, -24, $x1 :: (store (s64) into %fixed-stack.2)
+; MIR64-DAG:     STFD killed $f30, -16, $x1 :: (store (s64) into %fixed-stack.1, align 16)
 ; MIR64-DAG:     STFD killed $f31, -8, $x1 :: (store (s64) into %fixed-stack.0)
 
 ; MIR64-LABEL:   INLINEASM
@@ -117,12 +149,25 @@ define dso_local void @fprs_gprs_vecregs() {
 ; MIR64-NOT:     $v20 = LXVD2X
 ; MIR64-NOT:     $v26 = LXVD2X
 ; MIR64-NOT:     $v31 = LXVD2X
-; MIR64-DAG:     $x14 = LD -288, $x1 :: (load (s64) from %fixed-stack.5, align 16)
-; MIR64-DAG:     $x25 = LD -200, $x1 :: (load (s64) from %fixed-stack.4)
-; MIR64-DAG:     $x31 = LD -152, $x1 :: (load (s64) from %fixed-stack.3)
-; MIR64-DAG:     $f14 = LFD -144, $x1 :: (load (s64) from %fixed-stack.2, align 16)
-; MIR64-DAG:     $f21 = LFD -88, $x1 :: (load (s64) from %fixed-stack.1)
 ; MIR64-DAG:     $f31 = LFD -8, $x1 :: (load (s64) from %fixed-stack.0)
+; MIR64-DAG:     $f30 = LFD -16, $x1 :: (load (s64) from %fixed-stack.1, align 16)
+; MIR64-DAG:     $f29 = LFD -24, $x1 :: (load (s64) from %fixed-stack.2)
+; MIR64-DAG:     $f28 = LFD -32, $x1 :: (load (s64) from %fixed-stack.3, align 16)
+; MIR64-DAG:     $f27 = LFD -40, $x1 :: (load (s64) from %fixed-stack.4)
+; MIR64-DAG:     $f26 = LFD -48, $x1 :: (load (s64) from %fixed-stack.5, align 16)
+; MIR64-DAG:     $f25 = LFD -56, $x1 :: (load (s64) from %fixed-stack.6)
+; MIR64-DAG:     $f24 = LFD -64, $x1 :: (load (s64) from %fixed-stack.7, align 16)
+; MIR64-DAG:     $f23 = LFD -72, $x1 :: (load (s64) from %fixed-stack.8)
+; MIR64-DAG:     $f22 = LFD -80, $x1 :: (load (s64) from %fixed-stack.9, align 16)
+; MIR64-DAG:     $f21 = LFD -88, $x1 :: (load (s64) from %fixed-stack.10)
+; MIR64-DAG:     $x31 = LD -96, $x1 :: (load (s64) from %fixed-stack.11)
+; MIR64-DAG:     $x30 = LD -104, $x1 :: (load (s64) from %fixed-stack.12, align 16)
+; MIR64-DAG:     $x29 = LD -112, $x1 :: (load (s64) from %fixed-stack.13)
+; MIR64-DAG:     $x28 = LD -120, $x1 :: (load (s64) from %fixed-stack.14, align 16)
+; MIR64-DAG:     $x27 = LD -128, $x1 :: (load (s64) from %fixed-stack.15)
+; MIR64-DAG:     $x26 = LD -136, $x1 :: (load (s64) from %fixed-stack.16, align 16)
+; MIR64-DAG:     $x25 = LD -144, $x1 :: (load (s64) from %fixed-stack.17)
+
 ; MIR64:         BLR8 implicit $lr8, implicit $rm
 
 ;; We don't have -ppc-full-reg-names on AIX so can't reliably check-not for
@@ -130,38 +175,87 @@ define dso_local void @fprs_gprs_vecregs() {
 
 ; ASM32-LABEL:   .fprs_gprs_vecregs:
 
-; ASM32-DAG:     stw 14, -216(1)                         # 4-byte Folded Spill
-; ASM32-DAG:     stw 25, -172(1)                         # 4-byte Folded Spill
-; ASM32-DAG:     stw 31, -148(1)                         # 4-byte Folded Spill
-; ASM32-DAG:     stfd 14, -144(1)                        # 8-byte Folded Spill
-; ASM32-DAG:     stfd 21, -88(1)                         # 8-byte Folded Spill
-; ASM32-DAG:     stfd 31, -8(1)                          # 8-byte Folded Spill
-; ASM32-DAG:     #APP
-; ASM32-DAG:     #NO_APP
-; ASM32-DAG:     lfd 31, -8(1)                           # 8-byte Folded Reload
-; ASM32-DAG:     lfd 21, -88(1)                          # 8-byte Folded Reload
-; ASM32-DAG:     lfd 14, -144(1)                         # 8-byte Folded Reload
-; ASM32-DAG:     lwz 31, -148(1)                         # 4-byte Folded Reload
-; ASM32-DAG:     lwz 25, -172(1)                         # 4-byte Folded Reload
-; ASM32-DAG:     lwz 14, -216(1)                         # 4-byte Folded Reload
+; ASM32-DAG:   stw 25, -116(1)                         # 4-byte Folded Spill
+; ASM32-DAG:   stw 26, -112(1)                         # 4-byte Folded Spill
+; ASM32-DAG:   stw 27, -108(1)                         # 4-byte Folded Spill
+; ASM32-DAG:   stw 28, -104(1)                         # 4-byte Folded Spill
+; ASM32-DAG:   stw 29, -100(1)                         # 4-byte Folded Spill
+; ASM32-DAG:   stw 30, -96(1)                          # 4-byte Folded Spill
+; ASM32-DAG:   stw 31, -92(1)                          # 4-byte Folded Spill
+; ASM32-DAG:   stfd 21, -88(1)                         # 8-byte Folded Spill
+; ASM32-DAG:   stfd 22, -80(1)                         # 8-byte Folded Spill
+; ASM32-DAG:   stfd 23, -72(1)                         # 8-byte Folded Spill
+; ASM32-DAG:   stfd 24, -64(1)                         # 8-byte Folded Spill
+; ASM32-DAG:   stfd 25, -56(1)                         # 8-byte Folded Spill
+; ASM32-DAG:   stfd 26, -48(1)                         # 8-byte Folded Spill
+; ASM32-DAG:   stfd 27, -40(1)                         # 8-byte Folded Spill
+; ASM32-DAG:   stfd 28, -32(1)                         # 8-byte Folded Spill
+; ASM32-DAG:   stfd 29, -24(1)                         # 8-byte Folded Spill
+; ASM32-DAG:   stfd 30, -16(1)                         # 8-byte Folded Spill
+; ASM32-DAG:   stfd 31, -8(1)                          # 8-byte Folded Spill
+; ASM32:       #APP
+; ASM32-NEXT:  #NO_APP
+; ASM32-DAG:   lfd 31, -8(1)                           # 8-byte Folded Reload
+; ASM32-DAG:   lfd 30, -16(1)                          # 8-byte Folded Reload
+; ASM32-DAG:   lfd 29, -24(1)                          # 8-byte Folded Reload
+; ASM32-DAG:   lfd 28, -32(1)                          # 8-byte Folded Reload
+; ASM32-DAG:   lfd 27, -40(1)                          # 8-byte Folded Reload
+; ASM32-DAG:   lfd 26, -48(1)                          # 8-byte Folded Reload
+; ASM32-DAG:   lfd 25, -56(1)                          # 8-byte Folded Reload
+; ASM32-DAG:   lfd 24, -64(1)                          # 8-byte Folded Reload
+; ASM32-DAG:   lfd 23, -72(1)                          # 8-byte Folded Reload
+; ASM32-DAG:   lfd 22, -80(1)                          # 8-byte Folded Reload
+; ASM32-DAG:   lfd 21, -88(1)                          # 8-byte Folded Reload
+; ASM32-DAG:   lwz 31, -92(1)                          # 4-byte Folded Reload
+; ASM32-DAG:   lwz 30, -96(1)                          # 4-byte Folded Reload
+; ASM32-DAG:   lwz 29, -100(1)                         # 4-byte Folded Reload
+; ASM32-DAG:   lwz 28, -104(1)                         # 4-byte Folded Reload
+; ASM32-DAG:   lwz 27, -108(1)                         # 4-byte Folded Reload
+; ASM32-DAG:   lwz 26, -112(1)                         # 4-byte Folded Reload
+; ASM32-DAG:   lwz 25, -116(1)                         # 4-byte Folded Reload
 ; ASM32:         blr
 
 ; ASM64-LABEL:    .fprs_gprs_vecregs:
 
-; ASM64-DAG:     std 14, -288(1)                         # 8-byte Folded Spill
-; ASM64-DAG:     std 25, -200(1)                         # 8-byte Folded Spill
-; ASM64-DAG:     std 31, -152(1)                         # 8-byte Folded Spill
-; ASM64-DAG:     stfd 14, -144(1)                        # 8-byte Folded Spill
+; ASM64-DAG:     std 25, -144(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     std 26, -136(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     std 27, -128(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     std 28, -120(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     std 29, -112(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     std 30, -104(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     std 31, -96(1)                          # 8-byte Folded Spill
 ; ASM64-DAG:     stfd 21, -88(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 22, -80(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 23, -72(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 24, -64(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 25, -56(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 26, -48(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 27, -40(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 28, -32(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 29, -24(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 30, -16(1)                         # 8-byte Folded Spill
 ; ASM64-DAG:     stfd 31, -8(1)                          # 8-byte Folded Spill
-; ASM64-DAG:     #APP
-; ASM64-DAG:     #NO_APP
+; ASM64:         #APP
+; ASM64-NEXT:    #NO_APP
 ; ASM64-DAG:     lfd 31, -8(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     lfd 30, -16(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 29, -24(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 28, -32(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 27, -40(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 26, -48(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 25, -56(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 24, -64(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 23, -72(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 22, -80(1)                          # 8-byte Folded Reload
 ; ASM64-DAG:     lfd 21, -88(1)                          # 8-byte Folded Reload
-; ASM64-DAG:     lfd 14, -144(1)                         # 8-byte Folded Reload
-; ASM64-DAG:     ld 31, -152(1)                          # 8-byte Folded Reload
-; ASM64-DAG:     ld 25, -200(1)                          # 8-byte Folded Reload
-; ASM64-DAG:     ld 14, -288(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     ld 31, -96(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 30, -104(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     ld 29, -112(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     ld 28, -120(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     ld 27, -128(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     ld 26, -136(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     ld 25, -144(1)                          # 8-byte Folded Reload
+
 ; ASM64:         blr
 
 define dso_local void @all_fprs_and_vecregs() {
diff --git a/llvm/test/CodeGen/PowerPC/aix-csr.ll b/llvm/test/CodeGen/PowerPC/aix-csr.ll
index a9a85c8be5a1..1dadacf1faab 100644
--- a/llvm/test/CodeGen/PowerPC/aix-csr.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-csr.ll
@@ -20,77 +20,260 @@ entry:
 
 ; MIR64:       name:            gprs_only
 ; MIR64-LABEL: fixedStack:
-; MIR64-NEXT:   - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default,
-; MIR64-NEXT:       callee-saved-register: '$x30', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:       debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:   - { id: 1, type: spill-slot, offset: -80, size: 8, alignment: 16, stack-id: default,
-; MIR64-NEXT:       callee-saved-register: '$x22', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:       debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:   - { id: 2, type: spill-slot, offset: -128, size: 8, alignment: 16, stack-id: default,
-; MIR64-NEXT:       callee-saved-register: '$x16', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:       debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 0, type: spill-slot, offset: -8, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x31', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 1, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x30', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 2, type: spill-slot, offset: -24, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x29', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 3, type: spill-slot, offset: -32, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x28', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 4, type: spill-slot, offset: -40, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x27', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 5, type: spill-slot, offset: -48, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x26', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 6, type: spill-slot, offset: -56, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x25', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 7, type: spill-slot, offset: -64, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x24', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 8, type: spill-slot, offset: -72, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x23', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 9, type: spill-slot, offset: -80, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x22', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 10, type: spill-slot, offset: -88, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x21', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 11, type: spill-slot, offset: -96, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x20', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 12, type: spill-slot, offset: -104, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x19', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 13, type: spill-slot, offset: -112, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x18', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 14, type: spill-slot, offset: -120, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x17', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT:  - { id: 15, type: spill-slot, offset: -128, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:      callee-saved-register: '$x16', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:      debug-info-expression: '', debug-info-location: '' }
 ; MIR64-NEXT:  stack:           []
 
 ; MIR32:       name:            gprs_only
 ; MIR32-LABEL: fixedStack:
-; MIR32:        - { id: 0, type: spill-slot, offset: -8, size: 4, alignment: 8, stack-id: default,
-; MIR32-NEXT:       callee-saved-register: '$r30', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:       debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:   - { id: 1, type: spill-slot, offset: -40, size: 4, alignment: 8, stack-id: default,
-; MIR32-NEXT:       callee-saved-register: '$r22', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:       debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:   - { id: 2, type: spill-slot, offset: -64, size: 4, alignment: 16, stack-id: default,
-; MIR32-NEXT:       callee-saved-register: '$r16', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:       debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 0, type: spill-slot, offset: -4, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r31', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 1, type: spill-slot, offset: -8, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r30', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 2, type: spill-slot, offset: -12, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r29', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 3, type: spill-slot, offset: -16, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r28', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 4, type: spill-slot, offset: -20, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r27', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 5, type: spill-slot, offset: -24, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r26', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 6, type: spill-slot, offset: -28, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r25', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 7, type: spill-slot, offset: -32, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r24', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 8, type: spill-slot, offset: -36, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r23', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 9, type: spill-slot, offset: -40, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r22', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 10, type: spill-slot, offset: -44, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r21', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 11, type: spill-slot, offset: -48, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r20', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 12, type: spill-slot, offset: -52, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r19', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 13, type: spill-slot, offset: -56, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r18', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 14, type: spill-slot, offset: -60, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r17', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 15, type: spill-slot, offset: -64, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r16', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
 ; MIR32-NEXT:  stack:           []
 
 
-; MIR64: liveins: $x3, $x16, $x22, $x30
-
-; MIR64-DAG: STD killed $x16, -128, $x1 :: (store (s64) into %fixed-stack.2, align 16)
-; MIR64-DAG: STD killed $x22, -80, $x1 :: (store (s64) into %fixed-stack.1, align 16)
-; MIR64-DAG: STD killed $x30, -16, $x1 :: (store (s64) into %fixed-stack.0, align 16)
+; MIR64: liveins: $x3, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $x29, $x30, $x31
+
+; MIR64-DAG:       STD killed $x16, -128, $x1 :: (store (s64) into %fixed-stack.15, align 16)
+; MIR64-DAG:  STD killed $x17, -120, $x1 :: (store (s64) into %fixed-stack.14)
+; MIR64-DAG:  STD killed $x18, -112, $x1 :: (store (s64) into %fixed-stack.13, align 16)
+; MIR64-DAG:  STD killed $x19, -104, $x1 :: (store (s64) into %fixed-stack.12)
+; MIR64-DAG:  STD killed $x20, -96, $x1 :: (store (s64) into %fixed-stack.11, align 16)
+; MIR64-DAG:  STD killed $x21, -88, $x1 :: (store (s64) into %fixed-stack.10)
+; MIR64-DAG:  STD killed $x22, -80, $x1 :: (store (s64) into %fixed-stack.9, align 16)
+; MIR64-DAG:  STD killed $x23, -72, $x1 :: (store (s64) into %fixed-stack.8)
+; MIR64-DAG:  STD killed $x24, -64, $x1 :: (store (s64) into %fixed-stack.7, align 16)
+; MIR64-DAG:  STD killed $x25, -56, $x1 :: (store (s64) into %fixed-stack.6)
+; MIR64-DAG:  STD killed $x26, -48, $x1 :: (store (s64) into %fixed-stack.5, align 16)
+; MIR64-DAG:  STD killed $x27, -40, $x1 :: (store (s64) into %fixed-stack.4)
+; MIR64-DAG:  STD killed $x28, -32, $x1 :: (store (s64) into %fixed-stack.3, align 16)
+; MIR64-DAG:  STD killed $x29, -24, $x1 :: (store (s64) into %fixed-stack.2)
+; MIR64-DAG:  STD killed $x30, -16, $x1 :: (store (s64) into %fixed-stack.1, align 16)
+; MIR64-DAG:  STD killed $x31, -8, $x1 :: (store (s64) into %fixed-stack.0)
 
 ; MIR64:     INLINEASM
 
-; MIR64-DAG: $x30 = LD -16, $x1 :: (load (s64) from %fixed-stack.0, align 16)
-; MIR64-DAG: $x22 = LD -80, $x1 :: (load (s64) from %fixed-stack.1, align 16)
-; MIR64-DAG: $x16 = LD -128, $x1 :: (load (s64) from %fixed-stack.2, align 16)
-; MIR64:     BLR8 implicit $lr8, implicit $rm, implicit $x3
-
-
-; MIR32: liveins: $r3, $r16, $r22, $r30
 
-; MIR32-DAG: STW killed $r16, -64, $r1 :: (store (s32) into %fixed-stack.2, align 16)
-; MIR32-DAG: STW killed $r22, -40, $r1 :: (store (s32) into %fixed-stack.1, align 8)
-; MIR32-DAG: STW killed $r30, -8, $r1 :: (store (s32) into %fixed-stack.0, align 8)
+; MIR64-DAG:    $x31 = LD -8, $x1 :: (load (s64) from %fixed-stack.0)
+; MIR64-DAG:    $x30 = LD -16, $x1 :: (load (s64) from %fixed-stack.1, align 16)
+; MIR64-DAG:    $x29 = LD -24, $x1 :: (load (s64) from %fixed-stack.2)
+; MIR64-DAG:    $x28 = LD -32, $x1 :: (load (s64) from %fixed-stack.3, align 16)
+; MIR64-DAG:    $x27 = LD -40, $x1 :: (load (s64) from %fixed-stack.4)
+; MIR64-DAG:    $x26 = LD -48, $x1 :: (load (s64) from %fixed-stack.5, align 16)
+; MIR64-DAG:    $x25 = LD -56, $x1 :: (load (s64) from %fixed-stack.6)
+; MIR64-DAG:    $x24 = LD -64, $x1 :: (load (s64) from %fixed-stack.7, align 16)
+; MIR64-DAG:    $x23 = LD -72, $x1 :: (load (s64) from %fixed-stack.8)
+; MIR64-DAG:    $x22 = LD -80, $x1 :: (load (s64) from %fixed-stack.9, align 16)
+; MIR64-DAG:    $x21 = LD -88, $x1 :: (load (s64) from %fixed-stack.10)
+; MIR64-DAG:    $x20 = LD -96, $x1 :: (load (s64) from %fixed-stack.11, align 16)
+; MIR64-DAG:    $x19 = LD -104, $x1 :: (load (s64) from %fixed-stack.12)
+; MIR64-DAG:    $x18 = LD -112, $x1 :: (load (s64) from %fixed-stack.13, align 16)
+; MIR64-DAG:    $x17 = LD -120, $x1 :: (load (s64) from %fixed-stack.14)
+; MIR64-DAG:    $x16 = LD -128, $x1 :: (load (s64) from %fixed-stack.15, align 16)
+; MIR64:        BLR8 implicit $lr8, implicit $rm, implicit $x3
+
+
+; MIR32:  liveins: $r3, $r16, $r17, $r18, $r19, $r20, $r21, $r22, $r23, $r24, $r25, $r26, $r27, $r28, $r29, $r30, $r31
+
+; MIR32-DAG:  STW killed $r16, -64, $r1 :: (store (s32) into %fixed-stack.15, align 16)
+; MIR32-DAG:  STW killed $r17, -60, $r1 :: (store (s32) into %fixed-stack.14)
+; MIR32-DAG:  STW killed $r18, -56, $r1 :: (store (s32) into %fixed-stack.13, align 8)
+; MIR32-DAG:  STW killed $r19, -52, $r1 :: (store (s32) into %fixed-stack.12)
+; MIR32-DAG:  STW killed $r20, -48, $r1 :: (store (s32) into %fixed-stack.11, align 16)
+; MIR32-DAG:  STW killed $r21, -44, $r1 :: (store (s32) into %fixed-stack.10)
+; MIR32-DAG:  STW killed $r22, -40, $r1 :: (store (s32) into %fixed-stack.9, align 8)
+; MIR32-DAG:  STW killed $r23, -36, $r1 :: (store (s32) into %fixed-stack.8)
+; MIR32-DAG:  STW killed $r24, -32, $r1 :: (store (s32) into %fixed-stack.7, align 16)
+; MIR32-DAG:  STW killed $r25, -28, $r1 :: (store (s32) into %fixed-stack.6)
+; MIR32-DAG:  STW killed $r26, -24, $r1 :: (store (s32) into %fixed-stack.5, align 8)
+; MIR32-DAG:  STW killed $r27, -20, $r1 :: (store (s32) into %fixed-stack.4)
+; MIR32-DAG:  STW killed $r28, -16, $r1 :: (store (s32) into %fixed-stack.3, align 16)
+; MIR32-DAG:  STW killed $r29, -12, $r1 :: (store (s32) into %fixed-stack.2)
+; MIR32-DAG:  STW killed $r30, -8, $r1 :: (store (s32) into %fixed-stack.1, align 8)
+; MIR32-DAG:  STW killed $r31, -4, $r1 :: (store (s32) into %fixed-stack.0)
 
-; MIR32:     INLINEASM
+; MIR32:      INLINEASM
 
-; MIR32-DAG: $r30 = LWZ -8, $r1 :: (load (s32) from %fixed-stack.0, align 8)
-; MIR32-DAG: $r22 = LWZ -40, $r1 :: (load (s32) from %fixed-stack.1, align 8)
-; MIR32-DAG: $r16 = LWZ -64, $r1 :: (load (s32) from %fixed-stack.2, align 16)
-; MIR32:     BLR implicit $lr, implicit $rm, implicit $r3
+; MIR32-DAG:  $r31 = LWZ -4, $r1 :: (load (s32) from %fixed-stack.0)
+; MIR32-DAG:  $r30 = LWZ -8, $r1 :: (load (s32) from %fixed-stack.1, align 8)
+; MIR32-DAG:  $r29 = LWZ -12, $r1 :: (load (s32) from %fixed-stack.2)
+; MIR32-DAG:  $r28 = LWZ -16, $r1 :: (load (s32) from %fixed-stack.3, align 16)
+; MIR32-DAG:  $r27 = LWZ -20, $r1 :: (load (s32) from %fixed-stack.4)
+; MIR32-DAG:  $r26 = LWZ -24, $r1 :: (load (s32) from %fixed-stack.5, align 8)
+; MIR32-DAG:  $r25 = LWZ -28, $r1 :: (load (s32) from %fixed-stack.6)
+; MIR32-DAG:  $r24 = LWZ -32, $r1 :: (load (s32) from %fixed-stack.7, align 16)
+; MIR32-DAG:  $r23 = LWZ -36, $r1 :: (load (s32) from %fixed-stack.8)
+; MIR32-DAG:  $r22 = LWZ -40, $r1 :: (load (s32) from %fixed-stack.9, align 8)
+; MIR32-DAG:  $r21 = LWZ -44, $r1 :: (load (s32) from %fixed-stack.10)
+; MIR32-DAG:  $r20 = LWZ -48, $r1 :: (load (s32) from %fixed-stack.11, align 16)
+; MIR32-DAG:  $r19 = LWZ -52, $r1 :: (load (s32) from %fixed-stack.12)
+; MIR32-DAG:  $r18 = LWZ -56, $r1 :: (load (s32) from %fixed-stack.13, align 8)
+; MIR32-DAG:  $r17 = LWZ -60, $r1 :: (load (s32) from %fixed-stack.14)
+; MIR32-DAG:  $r16 = LWZ -64, $r1 :: (load (s32) from %fixed-stack.15, align 16)
+; MIR32:      BLR implicit $lr, implicit $rm, implicit $r3
 
 
 ; ASM64-LABEL: .gprs_only:
-; ASM64-DAG:      std 16, -128(1)                 # 8-byte Folded Spill
-; ASM64-DAG:      std 22, -80(1)                  # 8-byte Folded Spill
-; ASM64-DAG:      std 30, -16(1)                  # 8-byte Folded Spill
-; ASM64:          #APP
-; ASM64-DAG:      ld 30, -16(1)                   # 8-byte Folded Reload
-; ASM64-DAG:      ld 22, -80(1)                   # 8-byte Folded Reload
-; ASM64-DAG:      ld 16, -128(1)                  # 8-byte Folded Reload
+; ASM64-DAG:     std 16, -128(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     std 17, -120(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     std 18, -112(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     std 19, -104(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     std 20, -96(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 21, -88(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 22, -80(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 23, -72(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 24, -64(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 25, -56(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 26, -48(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 27, -40(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 28, -32(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 29, -24(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 30, -16(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 31, -8(1)                           # 8-byte Folded Spill
+; ASM64:         #APP
+; AMS64-DAG:     ld 31, -8(1)                            # 8-byte Folded Reload
+; ASM64-DAG:     ld 30, -16(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 29, -24(1)                           # 8-byte Folded Reload
+; ASM64-DAG:      ld 28, -32(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 27, -40(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 26, -48(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 25, -56(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 24, -64(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 23, -72(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 22, -80(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 21, -88(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 20, -96(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 19, -104(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     ld 18, -112(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     ld 17, -120(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     ld 16, -128(1)                          # 8-byte Folded Reload
 ; ASM64:          blr
 
 ; ASM32-LABEL: .gprs_only:
-; ASM32-DAG:     stw 16, -64(1)                  # 4-byte Folded Spill
-; ASM32-DAG:     stw 22, -40(1)                  # 4-byte Folded Spill
-; ASM32-DAG:     stw 30, -8(1)                   # 4-byte Folded Spill
+; ASM32-DAG:     stw 16, -64(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 17, -60(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 18, -56(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 19, -52(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 20, -48(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 21, -44(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 22, -40(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 23, -36(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 24, -32(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 25, -28(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 26, -24(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 27, -20(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 28, -16(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 29, -12(1)                          # 4-byte Folded Spill
+; ASM32-DAG:     stw 30, -8(1)                           # 4-byte Folded Spill
+; ASM32-DAG:     stw 31, -4(1)                           # 4-byte Folded Spill
 ; ASM32:         #APP
-; ASM32-DAG:     lwz 30, -8(1)                   # 4-byte Folded Reload
-; ASM32-DAG:     lwz 22, -40(1)                  # 4-byte Folded Reload
-; ASM32-DAG:     lwz 16, -64(1)                  # 4-byte Folded Reload
+; ASM32-DAG:     lwz 31, -4(1)                           # 4-byte Folded Reload
+; ASM32-DAG:     lwz 30, -8(1)                           # 4-byte Folded Reload
+; ASM32-DAG:     lwz 29, -12(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 28, -16(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 27, -20(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 26, -24(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 25, -28(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 24, -32(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 23, -36(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 22, -40(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 21, -44(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 20, -48(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 19, -52(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 18, -56(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 17, -60(1)                          # 4-byte Folded Reload
+; ASM32-DAG:     lwz 16, -64(1)                          # 4-byte Folded Reload
 ; ASM32-DAG:     blr
 
 
@@ -104,112 +287,402 @@ define dso_local double @fprs_and_gprs(i32 signext %i) {
 
 ; MIR64:       name:            fprs_and_gprs
 ; MIR64-LABEL: fixedStack:
-; MIR64-NEXT:    - { id: 0, type: spill-slot, offset: -8, size: 8, alignment: 8, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$f31', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 1, type: spill-slot, offset: -88, size: 8, alignment: 8, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$f21', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 2, type: spill-slot, offset: -104, size: 8, alignment: 8, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$f19', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 3, type: spill-slot, offset: -144, size: 8, alignment: 16, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$f14', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 4, type: spill-slot, offset: -152, size: 8, alignment: 8, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$x31', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 5, type: spill-slot, offset: -200, size: 8, alignment: 8, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$x25', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR64-NEXT:    - { id: 6, type: spill-slot, offset: -288, size: 8, alignment: 16, stack-id: default,
-; MIR64-NEXT:        callee-saved-register: '$x14', callee-saved-restored: true, debug-info-variable: '',
-; MIR64-NEXT:        debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 0, type: spill-slot, offset: -8, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f31', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 1, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f30', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 2, type: spill-slot, offset: -24, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f29', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 3, type: spill-slot, offset: -32, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f28', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 4, type: spill-slot, offset: -40, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f27', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 5, type: spill-slot, offset: -48, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f26', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 6, type: spill-slot, offset: -56, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f25', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 7, type: spill-slot, offset: -64, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f24', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 8, type: spill-slot, offset: -72, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f23', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 9, type: spill-slot, offset: -80, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f22', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 10, type: spill-slot, offset: -88, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f21', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 11, type: spill-slot, offset: -96, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f20', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 12, type: spill-slot, offset: -104, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f19', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 13, type: spill-slot, offset: -112, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f18', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 14, type: spill-slot, offset: -120, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f17', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 15, type: spill-slot, offset: -128, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f16', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 16, type: spill-slot, offset: -136, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f15', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 17, type: spill-slot, offset: -144, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$f14', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 18, type: spill-slot, offset: -152, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x31', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 19, type: spill-slot, offset: -160, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x30', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 20, type: spill-slot, offset: -168, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x29', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 21, type: spill-slot, offset: -176, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x28', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 22, type: spill-slot, offset: -184, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x27', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 23, type: spill-slot, offset: -192, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x26', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 24, type: spill-slot, offset: -200, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x25', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 25, type: spill-slot, offset: -208, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x24', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 26, type: spill-slot, offset: -216, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x23', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 27, type: spill-slot, offset: -224, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x22', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 28, type: spill-slot, offset: -232, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x21', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 29, type: spill-slot, offset: -240, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x20', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 30, type: spill-slot, offset: -248, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x19', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 31, type: spill-slot, offset: -256, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x18', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 32, type: spill-slot, offset: -264, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x17', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 33, type: spill-slot, offset: -272, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x16', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 34, type: spill-slot, offset: -280, size: 8, alignment: 8, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x15', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
+; MIR64-NEXT: - { id: 35, type: spill-slot, offset: -288, size: 8, alignment: 16, stack-id: default,
+; MIR64-NEXT:     callee-saved-register: '$x14', callee-saved-restored: true, debug-info-variable: '',
+; MIR64-NEXT:     debug-info-expression: '', debug-info-location: '' }
 ; MIR64-NEXT:  stack:           []
 
 ; MIR32:       name:            fprs_and_gprs
 ; MIR32-LABEL: fixedStack:
-; MIR32-NEXT:    - { id: 0, type: spill-slot, offset: -8, size: 8, alignment: 8, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$f31', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 1, type: spill-slot, offset: -88, size: 8, alignment: 8, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$f21', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 2, type: spill-slot, offset: -104, size: 8, alignment: 8, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$f19', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 3, type: spill-slot, offset: -144, size: 8, alignment: 16, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$f14', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 4, type: spill-slot, offset: -148, size: 4, alignment: 4, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$r31', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 5, type: spill-slot, offset: -172, size: 4, alignment: 4, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$r25', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 6, type: spill-slot, offset: -216, size: 4, alignment: 8, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$r14', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
-; MIR32-NEXT:    - { id: 7, type: spill-slot, offset: -220, size: 4, alignment: 4, stack-id: default,
-; MIR32-NEXT:        callee-saved-register: '$r13', callee-saved-restored: true, debug-info-variable: '',
-; MIR32-NEXT:        debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 0, type: spill-slot, offset: -8, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f31', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 1, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f30', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 2, type: spill-slot, offset: -24, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f29', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 3, type: spill-slot, offset: -32, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f28', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 4, type: spill-slot, offset: -40, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f27', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 5, type: spill-slot, offset: -48, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f26', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 6, type: spill-slot, offset: -56, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f25', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 7, type: spill-slot, offset: -64, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f24', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 8, type: spill-slot, offset: -72, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f23', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 9, type: spill-slot, offset: -80, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f22', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 10, type: spill-slot, offset: -88, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f21', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 11, type: spill-slot, offset: -96, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f20', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 12, type: spill-slot, offset: -104, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f19', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 13, type: spill-slot, offset: -112, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f18', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 14, type: spill-slot, offset: -120, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f17', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 15, type: spill-slot, offset: -128, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f16', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 16, type: spill-slot, offset: -136, size: 8, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f15', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 17, type: spill-slot, offset: -144, size: 8, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$f14', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 18, type: spill-slot, offset: -148, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r31', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 19, type: spill-slot, offset: -152, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r30', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 20, type: spill-slot, offset: -156, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r29', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 21, type: spill-slot, offset: -160, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r28', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 22, type: spill-slot, offset: -164, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r27', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 23, type: spill-slot, offset: -168, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r26', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 24, type: spill-slot, offset: -172, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r25', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 25, type: spill-slot, offset: -176, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r24', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 26, type: spill-slot, offset: -180, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r23', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 27, type: spill-slot, offset: -184, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r22', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 28, type: spill-slot, offset: -188, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r21', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 29, type: spill-slot, offset: -192, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r20', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 30, type: spill-slot, offset: -196, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r19', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 31, type: spill-slot, offset: -200, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r18', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 32, type: spill-slot, offset: -204, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r17', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 33, type: spill-slot, offset: -208, size: 4, alignment: 16, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r16', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 34, type: spill-slot, offset: -212, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r15', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 35, type: spill-slot, offset: -216, size: 4, alignment: 8, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r14', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
+; MIR32-NEXT:  - { id: 36, type: spill-slot, offset: -220, size: 4, alignment: 4, stack-id: default,
+; MIR32-NEXT:      callee-saved-register: '$r13', callee-saved-restored: true, debug-info-variable: '',
+; MIR32-NEXT:      debug-info-expression: '', debug-info-location: '' }
 ; MIR32-NEXT:  stack:           []
 
 
-; MIR64: liveins: $x3, $x14, $x25, $x31, $f14, $f19, $f21, $f31
+; MIR64: liveins: $x3, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $x29, $x30, $x31, $f14, $f15, $f16, $f17, $f18, $f19, $f20, $f21, $f22, $f23, $f24, $f25, $f26, $f27, $f28, $f29, $f30, $f31
 
 ; MIR64:       $x0 = MFLR8 implicit $lr8
 ; MIR64-NEXT:  $x1 = STDU $x1, -400, $x1
 ; MIR64-NEXT:  STD killed $x0, 416, $x1
-; MIR64-DAG:   STD killed $x14, 112, $x1 :: (store (s64) into %fixed-stack.6, align 16)
-; MIR64-DAG:   STD killed $x25, 200, $x1 :: (store (s64) into %fixed-stack.5)
-; MIR64-DAG:   STD killed $x31, 248, $x1 :: (store (s64) into %fixed-stack.4)
-; MIR64-DAG:   STFD killed $f14, 256, $x1 :: (store (s64) into %fixed-stack.3, align 16)
-; MIR64-DAG:   STFD killed $f19, 296, $x1 :: (store (s64) into %fixed-stack.2)
-; MIR64-DAG:   STFD killed $f21, 312, $x1 :: (store (s64) into %fixed-stack.1)
+; MIR64-DAG:   STD killed $x14, 112, $x1 :: (store (s64) into %fixed-stack.35, align 16)
+; MIR64-DAG:   STD killed $x15, 120, $x1 :: (store (s64) into %fixed-stack.34)
+; MIR64-DAG:   STD killed $x16, 128, $x1 :: (store (s64) into %fixed-stack.33, align 16)
+; MIR64-DAG:   STD killed $x17, 136, $x1 :: (store (s64) into %fixed-stack.32)
+; MIR64-DAG:   STD killed $x18, 144, $x1 :: (store (s64) into %fixed-stack.31, align 16)
+; MIR64-DAG:   STD killed $x19, 152, $x1 :: (store (s64) into %fixed-stack.30)
+; MIR64-DAG:   STD killed $x20, 160, $x1 :: (store (s64) into %fixed-stack.29, align 16)
+; MIR64-DAG:   STD killed $x21, 168, $x1 :: (store (s64) into %fixed-stack.28)
+; MIR64-DAG:   STD killed $x22, 176, $x1 :: (store (s64) into %fixed-stack.27, align 16)
+; MIR64-DAG:   STD killed $x23, 184, $x1 :: (store (s64) into %fixed-stack.26)
+; MIR64-DAG:   STD killed $x24, 192, $x1 :: (store (s64) into %fixed-stack.25, align 16)
+; MIR64-DAG:   STD killed $x25, 200, $x1 :: (store (s64) into %fixed-stack.24)
+; MIR64-DAG:   STD killed $x26, 208, $x1 :: (store (s64) into %fixed-stack.23, align 16)
+; MIR64-DAG:   STD killed $x27, 216, $x1 :: (store (s64) into %fixed-stack.22)
+; MIR64-DAG:   STD killed $x28, 224, $x1 :: (store (s64) into %fixed-stack.21, align 16)
+; MIR64-DAG:   STD killed $x29, 232, $x1 :: (store (s64) into %fixed-stack.20)
+; MIR64-DAG:   STD killed $x30, 240, $x1 :: (store (s64) into %fixed-stack.19, align 16)
+; MIR64-DAG:   STD killed $x31, 248, $x1 :: (store (s64) into %fixed-stack.18)
+; MIR64-DAG:   STFD killed $f14, 256, $x1 :: (store (s64) into %fixed-stack.17, align 16)
+; MIR64-DAG:   STFD killed $f15, 264, $x1 :: (store (s64) into %fixed-stack.16)
+; MIR64-DAG:   STFD killed $f16, 272, $x1 :: (store (s64) into %fixed-stack.15, align 16)
+; MIR64-DAG:   STFD killed $f17, 280, $x1 :: (store (s64) into %fixed-stack.14)
+; MIR64-DAG:   STFD killed $f18, 288, $x1 :: (store (s64) into %fixed-stack.13, align 16)
+; MIR64-DAG:   STFD killed $f19, 296, $x1 :: (store (s64) into %fixed-stack.12)
+; MIR64-DAG:   STFD killed $f20, 304, $x1 :: (store (s64) into %fixed-stack.11, align 16)
+; MIR64-DAG:   STFD killed $f21, 312, $x1 :: (store (s64) into %fixed-stack.10)
+; MIR64-DAG:   STFD killed $f22, 320, $x1 :: (store (s64) into %fixed-stack.9, align 16)
+; MIR64-DAG:   STFD killed $f23, 328, $x1 :: (store (s64) into %fixed-stack.8)
+; MIR64-DAG:   STFD killed $f24, 336, $x1 :: (store (s64) into %fixed-stack.7, align 16)
+; MIR64-DAG:   STFD killed $f25, 344, $x1 :: (store (s64) into %fixed-stack.6)
+; MIR64-DAG:   STFD killed $f26, 352, $x1 :: (store (s64) into %fixed-stack.5, align 16)
+; MIR64-DAG:   STFD killed $f27, 360, $x1 :: (store (s64) into %fixed-stack.4)
+; MIR64-DAG:   STFD killed $f28, 368, $x1 :: (store (s64) into %fixed-stack.3, align 16)
+; MIR64-DAG:   STFD killed $f29, 376, $x1 :: (store (s64) into %fixed-stack.2)
+; MIR64-DAG:   STFD killed $f30, 384, $x1 :: (store (s64) into %fixed-stack.1, align 16)
 ; MIR64-DAG:   STFD killed $f31, 392, $x1 :: (store (s64) into %fixed-stack.0)
 
 ; MIR64:       INLINEASM
 ; MIR64-NEXT:  BL8_NOP
 
 ; MIR64-DAG:   $f31 = LFD 392, $x1 :: (load (s64) from %fixed-stack.0)
-; MIR64-DAG:   $f21 = LFD 312, $x1 :: (load (s64) from %fixed-stack.1)
-; MIR64-DAG:   $f19 = LFD 296, $x1 :: (load (s64) from %fixed-stack.2)
-; MIR64-DAG:   $f14 = LFD 256, $x1 :: (load (s64) from %fixed-stack.3, align 16)
-; MIR64-DAG:   $x31 = LD 248, $x1 :: (load (s64) from %fixed-stack.4)
-; MIR64-DAG:   $x25 = LD 200, $x1 :: (load (s64) from %fixed-stack.5)
-; MIR64-DAG:   $x14 = LD 112, $x1 :: (load (s64) from %fixed-stack.6, align 16)
+; MIR64-DAG:   $f30 = LFD 384, $x1 :: (load (s64) from %fixed-stack.1, align 16)
+; MIR64-DAG:   $f29 = LFD 376, $x1 :: (load (s64) from %fixed-stack.2)
+; MIR64-DAG:   $f28 = LFD 368, $x1 :: (load (s64) from %fixed-stack.3, align 16)
+; MIR64-DAG:   $f27 = LFD 360, $x1 :: (load (s64) from %fixed-stack.4)
+; MIR64-DAG:   $f26 = LFD 352, $x1 :: (load (s64) from %fixed-stack.5, align 16)
+; MIR64-DAG:   $f25 = LFD 344, $x1 :: (load (s64) from %fixed-stack.6)
+; MIR64-DAG:   $f24 = LFD 336, $x1 :: (load (s64) from %fixed-stack.7, align 16)
+; MIR64-DAG:   $f23 = LFD 328, $x1 :: (load (s64) from %fixed-stack.8)
+; MIR64-DAG:   $f22 = LFD 320, $x1 :: (load (s64) from %fixed-stack.9, align 16)
+; MIR64-DAG:   $f21 = LFD 312, $x1 :: (load (s64) from %fixed-stack.10)
+; MIR64-DAG:   $f20 = LFD 304, $x1 :: (load (s64) from %fixed-stack.11, align 16)
+; MIR64-DAG:   $f19 = LFD 296, $x1 :: (load (s64) from %fixed-stack.12)
+; MIR64-DAG:   $f18 = LFD 288, $x1 :: (load (s64) from %fixed-stack.13, align 16)
+; MIR64-DAG:   $f17 = LFD 280, $x1 :: (load (s64) from %fixed-stack.14)
+; MIR64-DAG:   $f16 = LFD 272, $x1 :: (load (s64) from %fixed-stack.15, align 16)
+; MIR64-DAG:   $f15 = LFD 264, $x1 :: (load (s64) from %fixed-stack.16)
+; MIR64-DAG:   $f14 = LFD 256, $x1 :: (load (s64) from %fixed-stack.17, align 16)
+; MIR64-DAG:   $x31 = LD 248, $x1 :: (load (s64) from %fixed-stack.18)
+; MIR64-DAG:   $x30 = LD 240, $x1 :: (load (s64) from %fixed-stack.19, align 16)
+; MIR64-DAG:   $x29 = LD 232, $x1 :: (load (s64) from %fixed-stack.20)
+; MIR64-DAG:   $x28 = LD 224, $x1 :: (load (s64) from %fixed-stack.21, align 16)
+; MIR64-DAG:   $x27 = LD 216, $x1 :: (load (s64) from %fixed-stack.22)
+; MIR64-DAG:   $x26 = LD 208, $x1 :: (load (s64) from %fixed-stack.23, align 16)
+; MIR64-DAG:   $x25 = LD 200, $x1 :: (load (s64) from %fixed-stack.24)
+; MIR64-DAG:   $x24 = LD 192, $x1 :: (load (s64) from %fixed-stack.25, align 16)
+; MIR64-DAG:   $x23 = LD 184, $x1 :: (load (s64) from %fixed-stack.26)
+; MIR64-DAG:   $x22 = LD 176, $x1 :: (load (s64) from %fixed-stack.27, align 16)
+; MIR64-DAG:   $x21 = LD 168, $x1 :: (load (s64) from %fixed-stack.28)
+; MIR64-DAG:   $x20 = LD 160, $x1 :: (load (s64) from %fixed-stack.29, align 16)
+; MIR64-DAG:   $x19 = LD 152, $x1 :: (load (s64) from %fixed-stack.30)
+; MIR64-DAG:   $x18 = LD 144, $x1 :: (load (s64) from %fixed-stack.31, align 16)
+; MIR64-DAG:   $x17 = LD 136, $x1 :: (load (s64) from %fixed-stack.32)
+; MIR64-DAG:   $x16 = LD 128, $x1 :: (load (s64) from %fixed-stack.33, align 16)
+; MIR64-DAG:   $x15 = LD 120, $x1 :: (load (s64) from %fixed-stack.34)
+; MIR64-DAG:   $x14 = LD 112, $x1 :: (load (s64) from %fixed-stack.35, align 16)
+
 ; MIR64:       $x1 = ADDI8 $x1, 400
 ; MIR64-NEXT:  $x0 = LD 16, $x1
 ; MIR64-NEXT:  MTLR8 $x0, implicit-def $lr8
 ; MIR64-NEXT:  BLR8 implicit $lr8, implicit $rm, implicit $f1
 
-
-; MIR32: liveins: $r3, $r13, $r14, $r25, $r31, $f14, $f19, $f21, $f31
+; MIR32: liveins: $r3, $r13, $r14, $r15, $r16, $r17, $r18, $r19, $r20, $r21, $r22, $r23, $r24, $r25, $r26, $r27, $r28, $r29, $r30, $r31, $f14, $f15, $f16, $f17, $f18, $f19, $f20, $f21, $f22, $f23, $f24, $f25, $f26, $f27, $f28, $f29, $f30, $f31
 
 ; MIR32:      $r0 = MFLR implicit $lr
 ; MIR32-NEXT: $r1 = STWU $r1, -288, $r1
 ; MIR32-NEXT: STW killed $r0, 296, $r1
-; MIR32-DAG:  STW killed $r13, 68, $r1 :: (store (s32) into %fixed-stack.7)
-; MIR32-DAG:  STW killed $r14, 72, $r1 :: (store (s32) into %fixed-stack.6, align 8)
-; MIR32-DAG:  STW killed $r25, 116, $r1 :: (store (s32) into %fixed-stack.5)
-; MIR32-DAG:  STW killed $r31, 140, $r1 :: (store (s32) into %fixed-stack.4)
-; MIR32-DAG:  STFD killed $f14, 144, $r1 :: (store (s64) into %fixed-stack.3, align 16)
-; MIR32-DAG:  STFD killed $f19, 184, $r1 :: (store (s64) into %fixed-stack.2)
-; MIR32-DAG:  STFD killed $f21, 200, $r1 :: (store (s64) into %fixed-stack.1)
+; MIR32-DAG:  STW killed $r13, 68, $r1 :: (store (s32) into %fixed-stack.36)
+; MIR32-DAG:  STW killed $r14, 72, $r1 :: (store (s32) into %fixed-stack.35, align 8)
+; MIR32-DAG:  STW killed $r15, 76, $r1 :: (store (s32) into %fixed-stack.34)
+; MIR32-DAG:  STW killed $r16, 80, $r1 :: (store (s32) into %fixed-stack.33, align 16)
+; MIR32-DAG:  STW killed $r17, 84, $r1 :: (store (s32) into %fixed-stack.32)
+; MIR32-DAG:  STW killed $r18, 88, $r1 :: (store (s32) into %fixed-stack.31, align 8)
+; MIR32-DAG:  STW killed $r19, 92, $r1 :: (store (s32) into %fixed-stack.30)
+; MIR32-DAG:  STW killed $r20, 96, $r1 :: (store (s32) into %fixed-stack.29, align 16)
+; MIR32-DAG:  STW killed $r21, 100, $r1 :: (store (s32) into %fixed-stack.28)
+; MIR32-DAG:  STW killed $r22, 104, $r1 :: (store (s32) into %fixed-stack.27, align 8)
+; MIR32-DAG:  STW killed $r23, 108, $r1 :: (store (s32) into %fixed-stack.26)
+; MIR32-DAG:  STW killed $r24, 112, $r1 :: (store (s32) into %fixed-stack.25, align 16)
+; MIR32-DAG:  STW killed $r25, 116, $r1 :: (store (s32) into %fixed-stack.24)
+; MIR32-DAG:  STW killed $r26, 120, $r1 :: (store (s32) into %fixed-stack.23, align 8)
+; MIR32-DAG:  STW killed $r27, 124, $r1 :: (store (s32) into %fixed-stack.22)
+; MIR32-DAG:  STW killed $r28, 128, $r1 :: (store (s32) into %fixed-stack.21, align 16)
+; MIR32-DAG:  STW killed $r29, 132, $r1 :: (store (s32) into %fixed-stack.20)
+; MIR32-DAG:  STW killed $r30, 136, $r1 :: (store (s32) into %fixed-stack.19, align 8)
+; MIR32-DAG:  STW killed $r31, 140, $r1 :: (store (s32) into %fixed-stack.18)
+; MIR32-DAG:  STFD killed $f14, 144, $r1 :: (store (s64) into %fixed-stack.17, align 16)
+; MIR32-DAG:  STFD killed $f15, 152, $r1 :: (store (s64) into %fixed-stack.16)
+; MIR32-DAG:  STFD killed $f16, 160, $r1 :: (store (s64) into %fixed-stack.15, align 16)
+; MIR32-DAG:  STFD killed $f17, 168, $r1 :: (store (s64) into %fixed-stack.14)
+; MIR32-DAG:  STFD killed $f18, 176, $r1 :: (store (s64) into %fixed-stack.13, align 16)
+; MIR32-DAG:  STFD killed $f19, 184, $r1 :: (store (s64) into %fixed-stack.12)
+; MIR32-DAG:  STFD killed $f20, 192, $r1 :: (store (s64) into %fixed-stack.11, align 16)
+; MIR32-DAG:  STFD killed $f21, 200, $r1 :: (store (s64) into %fixed-stack.10)
+; MIR32-DAG:  STFD killed $f22, 208, $r1 :: (store (s64) into %fixed-stack.9, align 16)
+; MIR32-DAG:  STFD killed $f23, 216, $r1 :: (store (s64) into %fixed-stack.8)
+; MIR32-DAG:  STFD killed $f24, 224, $r1 :: (store (s64) into %fixed-stack.7, align 16)
+; MIR32-DAG:  STFD killed $f25, 232, $r1 :: (store (s64) into %fixed-stack.6)
+; MIR32-DAG:  STFD killed $f26, 240, $r1 :: (store (s64) into %fixed-stack.5, align 16)
+; MIR32-DAG:  STFD killed $f27, 248, $r1 :: (store (s64) into %fixed-stack.4)
+; MIR32-DAG:  STFD killed $f28, 256, $r1 :: (store (s64) into %fixed-stack.3, align 16)
+; MIR32-DAG:  STFD killed $f29, 264, $r1 :: (store (s64) into %fixed-stack.2)
+; MIR32-DAG:  STFD killed $f30, 272, $r1 :: (store (s64) into %fixed-stack.1, align 16)
 ; MIR32-DAG:  STFD killed $f31, 280, $r1 :: (store (s64) into %fixed-stack.0)
 
 ; MIR32:      INLINEASM
 ; MIR32:      BL_NOP
 
 ; MIR32-DAG:  $f31 = LFD 280, $r1 :: (load (s64) from %fixed-stack.0)
-; MIR32-DAG:  $f21 = LFD 200, $r1 :: (load (s64) from %fixed-stack.1)
-; MIR32-DAG:  $f19 = LFD 184, $r1 :: (load (s64) from %fixed-stack.2)
-; MIR32-DAG:  $f14 = LFD 144, $r1 :: (load (s64) from %fixed-stack.3, align 16)
-; MIR32-DAG:  $r31 = LWZ 140, $r1 :: (load (s32) from %fixed-stack.4)
-; MIR32-DAG:  $r25 = LWZ 116, $r1 :: (load (s32) from %fixed-stack.5)
-; MIR32-DAG:  $r14 = LWZ 72, $r1 :: (load (s32) from %fixed-stack.6, align 8)
-; MIR32-DAG:  $r13 = LWZ 68, $r1 :: (load (s32) from %fixed-stack.7)
+; MIR32-DAG:  $f30 = LFD 272, $r1 :: (load (s64) from %fixed-stack.1, align 16)
+; MIR32-DAG:  $f29 = LFD 264, $r1 :: (load (s64) from %fixed-stack.2)
+; MIR32-DAG:  $f28 = LFD 256, $r1 :: (load (s64) from %fixed-stack.3, align 16)
+; MIR32-DAG:  $f27 = LFD 248, $r1 :: (load (s64) from %fixed-stack.4)
+; MIR32-DAG:  $f26 = LFD 240, $r1 :: (load (s64) from %fixed-stack.5, align 16)
+; MIR32-DAG:  $f25 = LFD 232, $r1 :: (load (s64) from %fixed-stack.6)
+; MIR32-DAG:  $f24 = LFD 224, $r1 :: (load (s64) from %fixed-stack.7, align 16)
+; MIR32-DAG:  $f23 = LFD 216, $r1 :: (load (s64) from %fixed-stack.8)
+; MIR32-DAG:  $f22 = LFD 208, $r1 :: (load (s64) from %fixed-stack.9, align 16)
+; MIR32-DAG:  $f21 = LFD 200, $r1 :: (load (s64) from %fixed-stack.10)
+; MIR32-DAG:  $f20 = LFD 192, $r1 :: (load (s64) from %fixed-stack.11, align 16)
+; MIR32-DAG:  $f19 = LFD 184, $r1 :: (load (s64) from %fixed-stack.12)
+; MIR32-DAG:  $f18 = LFD 176, $r1 :: (load (s64) from %fixed-stack.13, align 16)
+; MIR32-DAG:  $f17 = LFD 168, $r1 :: (load (s64) from %fixed-stack.14)
+; MIR32-DAG:  $f16 = LFD 160, $r1 :: (load (s64) from %fixed-stack.15, align 16)
+; MIR32-DAG:  $f15 = LFD 152, $r1 :: (load (s64) from %fixed-stack.16)
+; MIR32-DAG:  $f14 = LFD 144, $r1 :: (load (s64) from %fixed-stack.17, align 16)
+; MIR32-DAG:  $r31 = LWZ 140, $r1 :: (load (s32) from %fixed-stack.18)
+; MIR32-DAG:  $r30 = LWZ 136, $r1 :: (load (s32) from %fixed-stack.19, align 8)
+; MIR32-DAG:  $r29 = LWZ 132, $r1 :: (load (s32) from %fixed-stack.20)
+; MIR32-DAG:  $r28 = LWZ 128, $r1 :: (load (s32) from %fixed-stack.21, align 16)
+; MIR32-DAG:  $r27 = LWZ 124, $r1 :: (load (s32) from %fixed-stack.22)
+; MIR32-DAG:  $r26 = LWZ 120, $r1 :: (load (s32) from %fixed-stack.23, align 8)
+; MIR32-DAG:  $r25 = LWZ 116, $r1 :: (load (s32) from %fixed-stack.24)
+; MIR32-DAG:  $r24 = LWZ 112, $r1 :: (load (s32) from %fixed-stack.25, align 16)
+; MIR32-DAG:  $r23 = LWZ 108, $r1 :: (load (s32) from %fixed-stack.26)
+; MIR32-DAG:  $r22 = LWZ 104, $r1 :: (load (s32) from %fixed-stack.27, align 8)
+; MIR32-DAG:  $r21 = LWZ 100, $r1 :: (load (s32) from %fixed-stack.28)
+; MIR32-DAG:  $r20 = LWZ 96, $r1 :: (load (s32) from %fixed-stack.29, align 16)
+; MIR32-DAG:  $r19 = LWZ 92, $r1 :: (load (s32) from %fixed-stack.30)
+; MIR32-DAG:  $r18 = LWZ 88, $r1 :: (load (s32) from %fixed-stack.31, align 8)
+; MIR32-DAG:  $r17 = LWZ 84, $r1 :: (load (s32) from %fixed-stack.32)
+; MIR32-DAG:  $r16 = LWZ 80, $r1 :: (load (s32) from %fixed-stack.33, align 16)
+; MIR32-DAG:  $r15 = LWZ 76, $r1 :: (load (s32) from %fixed-stack.34)
+; MIR32-DAG:  $r14 = LWZ 72, $r1 :: (load (s32) from %fixed-stack.35, align 8)
+; MIR32-DAG:  $r13 = LWZ 68, $r1 :: (load (s32) from %fixed-stack.36)
 ; MIR32:      $r1 = ADDI $r1, 288
 ; MIR32-NEXT: $r0 = LWZ 8, $r1
 ; MIR32-NEXT: MTLR $r0, implicit-def $lr
@@ -219,23 +692,81 @@ define dso_local double @fprs_and_gprs(i32 signext %i) {
 ; ASM64:         mflr 0
 ; ASM64-NEXT:    stdu 1, -400(1)
 ; ASM64-NEXT:    std 0, 416(1)
-; ASM64-DAG:     std 14, 112(1)                  # 8-byte Folded Spill
-; ASM64-DAG:     std 25, 200(1)                  # 8-byte Folded Spill
-; ASM64-DAG:     std 31, 248(1)                  # 8-byte Folded Spill
-; ASM64-DAG:     stfd 14, 256(1)                 # 8-byte Folded Spill
-; ASM64-DAG:     stfd 19, 296(1)                 # 8-byte Folded Spill
-; ASM64-DAG:     stfd 21, 312(1)                 # 8-byte Folded Spill
-; ASM64-DAG:     stfd 31, 392(1)                 # 8-byte Folded Spill
+; ASM64-DAG:     std 14, 112(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 15, 120(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 16, 128(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 17, 136(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 18, 144(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 19, 152(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 20, 160(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 21, 168(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 22, 176(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 23, 184(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 24, 192(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 25, 200(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 26, 208(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 27, 216(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 28, 224(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 29, 232(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 30, 240(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     std 31, 248(1)                          # 8-byte Folded Spill
+; ASM64-DAG:     stfd 14, 256(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 15, 264(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 16, 272(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 17, 280(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 18, 288(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 19, 296(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 20, 304(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 21, 312(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 22, 320(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 23, 328(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 24, 336(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 25, 344(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 26, 352(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 27, 360(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 28, 368(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 29, 376(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 30, 384(1)                         # 8-byte Folded Spill
+; ASM64-DAG:     stfd 31, 392(1)                         # 8-byte Folded Spill
 
 ; ASM64:         bl .dummy
+; ASM64-DAG:     lfd 31, 392(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 30, 384(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 29, 376(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 28, 368(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 27, 360(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 26, 352(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 25, 344(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 24, 336(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 23, 328(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 22, 320(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 21, 312(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 20, 304(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 19, 296(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 18, 288(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 17, 280(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 16, 272(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 15, 264(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     lfd 14, 256(1)                          # 8-byte Folded Reload
+; ASM64-DAG:     ld 31, 248(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 30, 240(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 29, 232(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 28, 224(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 27, 216(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 26, 208(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 25, 200(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 24, 192(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 23, 184(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 22, 176(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 21, 168(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 20, 160(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 19, 152(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 18, 144(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 17, 136(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 16, 128(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 15, 120(1)                           # 8-byte Folded Reload
+; ASM64-DAG:     ld 14, 112(1)                           # 8-byte Folded Reload
 
-; ASM64-DAG:     lfd 31, 392(1)                  # 8-byte Folded Reload
-; ASM64-DAG:     lfd 21, 312(1)                  # 8-byte Folded Reload
-; ASM64-DAG:     lfd 19, 296(1)                  # 8-byte Folded Reload
-; ASM64-DAG:     lfd 14, 256(1)                  # 8-byte Folded Reload
-; ASM64-DAG:     ld 31, 248(1)                   # 8-byte Folded Reload
-; ASM64-DAG:     ld 25, 200(1)                   # 8-byte Folded Reload
-; ASM64-DAG:     ld 14, 112(1)                   # 8-byte Folded Reload
 ; ASM64:         addi 1, 1, 400
 ; ASM64-NEXT:    ld 0, 16(1)
 ; ASM64-NEXT:    mtlr 0
diff --git a/llvm/test/CodeGen/PowerPC/aix-spills-for-eh.ll b/llvm/test/CodeGen/PowerPC/aix-spills-for-eh.ll
new file mode 100644
index 000000000000..73004e875873
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-spills-for-eh.ll
@@ -0,0 +1,301 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr9 -mattr=+altivec -verify-machineinstrs --vec-extabi \
+; RUN:   -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc-unknown-aix < %s  | FileCheck %s --check-prefix 32BIT
+
+; RUN: llc -mcpu=pwr9 -mattr=+altivec -verify-machineinstrs --vec-extabi \
+; RUN:   -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64-unknown-aix < %s | FileCheck %s --check-prefix 64BIT
+
+@_ZTIi = external constant ptr
+
+; Function Attrs: uwtable mustprogress
+define dso_local signext i32 @_Z5test2iPPKc(i32 signext %argc, ptr nocapture readnone %argv) local_unnamed_addr #0 personality ptr @__gxx_personality_v0{
+; 32BIT-LABEL: _Z5test2iPPKc:
+; 32BIT:       # %bb.0: # %entry
+; 32BIT-NEXT:    mflr r0
+; 32BIT-NEXT:    stwu r1, -464(r1)
+; 32BIT-NEXT:    stw r0, 472(r1)
+; 32BIT-NEXT:    stw r30, 320(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    li r30, 0
+; 32BIT-NEXT:    stxv v20, 64(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stxv v21, 80(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stw r31, 324(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    mr r31, r3
+; 32BIT-NEXT:    stw r14, 256(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stxv v22, 96(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stw r15, 260(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stxv v23, 112(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stw r16, 264(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stxv v24, 128(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stw r17, 268(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stw r18, 272(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stxv v25, 144(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stw r19, 276(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stxv v26, 160(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stw r20, 280(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stxv v27, 176(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stw r21, 284(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stw r22, 288(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stxv v28, 192(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stw r23, 292(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stxv v29, 208(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stw r24, 296(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stxv v30, 224(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stw r25, 300(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stw r26, 304(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stxv v31, 240(r1) # 16-byte Folded Spill
+; 32BIT-NEXT:    stw r27, 308(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stw r28, 312(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stw r29, 316(r1) # 4-byte Folded Spill
+; 32BIT-NEXT:    stfd f15, 328(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f16, 336(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f17, 344(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f18, 352(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f19, 360(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f20, 368(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f21, 376(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f22, 384(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f23, 392(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f24, 400(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f25, 408(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f26, 416(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f27, 424(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f28, 432(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f29, 440(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f30, 448(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    stfd f31, 456(r1) # 8-byte Folded Spill
+; 32BIT-NEXT:    #APP
+; 32BIT-NEXT:    nop
+; 32BIT-NEXT:    #NO_APP
+; 32BIT-NEXT:  L..tmp0:
+; 32BIT-NEXT:    bl ._Z4testi[PR]
+; 32BIT-NEXT:    nop
+; 32BIT-NEXT:  L..tmp1:
+; 32BIT-NEXT:  L..BB0_1: # %return
+; 32BIT-NEXT:    lxv v31, 240(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    lxv v30, 224(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    lxv v29, 208(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    lxv v28, 192(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    mr r3, r30
+; 32BIT-NEXT:    lxv v27, 176(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    lxv v26, 160(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    lxv v25, 144(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    lxv v24, 128(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    lxv v23, 112(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    lxv v22, 96(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    lxv v21, 80(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    lxv v20, 64(r1) # 16-byte Folded Reload
+; 32BIT-NEXT:    lfd f31, 456(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lfd f30, 448(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lfd f29, 440(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lfd f28, 432(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lwz r31, 324(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r30, 320(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r29, 316(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lfd f27, 424(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lwz r28, 312(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r27, 308(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r26, 304(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lfd f26, 416(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lwz r25, 300(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r24, 296(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r23, 292(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lfd f25, 408(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lwz r22, 288(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r21, 284(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r20, 280(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lfd f24, 400(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lwz r19, 276(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r18, 272(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r17, 268(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lfd f23, 392(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lwz r16, 264(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r15, 260(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lwz r14, 256(r1) # 4-byte Folded Reload
+; 32BIT-NEXT:    lfd f22, 384(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lfd f21, 376(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lfd f20, 368(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lfd f19, 360(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lfd f18, 352(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lfd f17, 344(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lfd f16, 336(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    lfd f15, 328(r1) # 8-byte Folded Reload
+; 32BIT-NEXT:    addi r1, r1, 464
+; 32BIT-NEXT:    lwz r0, 8(r1)
+; 32BIT-NEXT:    mtlr r0
+; 32BIT-NEXT:    blr
+; 32BIT-NEXT:  L..BB0_2: # %lpad
+; 32BIT-NEXT:  L..tmp2:
+; 32BIT-NEXT:    bl .__cxa_begin_catch[PR]
+; 32BIT-NEXT:    nop
+; 32BIT-NEXT:    lwz r3, 0(r3)
+; 32BIT-NEXT:    add r30, r3, r31
+; 32BIT-NEXT:    bl .__cxa_end_catch[PR]
+; 32BIT-NEXT:    nop
+; 32BIT-NEXT:    b L..BB0_1
+;
+; 64BIT-LABEL: _Z5test2iPPKc:
+; 64BIT:       # %bb.0: # %entry
+; 64BIT-NEXT:    mflr r0
+; 64BIT-NEXT:    stdu r1, -592(r1)
+; 64BIT-NEXT:    std r0, 608(r1)
+; 64BIT-NEXT:    std r30, 440(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    li r30, 0
+; 64BIT-NEXT:    stxv v20, 112(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    stxv v21, 128(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    std r31, 448(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    mr r31, r3
+; 64BIT-NEXT:    std r14, 312(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stxv v22, 144(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    std r15, 320(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stxv v23, 160(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    std r16, 328(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stxv v24, 176(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    std r17, 336(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    std r18, 344(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stxv v25, 192(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    std r19, 352(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stxv v26, 208(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    std r20, 360(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stxv v27, 224(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    std r21, 368(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    std r22, 376(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stxv v28, 240(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    std r23, 384(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stxv v29, 256(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    std r24, 392(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stxv v30, 272(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    std r25, 400(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    std r26, 408(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stxv v31, 288(r1) # 16-byte Folded Spill
+; 64BIT-NEXT:    std r27, 416(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    std r28, 424(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    std r29, 432(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f15, 456(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f16, 464(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f17, 472(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f18, 480(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f19, 488(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f20, 496(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f21, 504(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f22, 512(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f23, 520(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f24, 528(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f25, 536(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f26, 544(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f27, 552(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f28, 560(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f29, 568(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f30, 576(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    stfd f31, 584(r1) # 8-byte Folded Spill
+; 64BIT-NEXT:    #APP
+; 64BIT-NEXT:    nop
+; 64BIT-NEXT:    #NO_APP
+; 64BIT-NEXT:  L..tmp0:
+; 64BIT-NEXT:    bl ._Z4testi[PR]
+; 64BIT-NEXT:    nop
+; 64BIT-NEXT:  L..tmp1:
+; 64BIT-NEXT:  L..BB0_1: # %return
+; 64BIT-NEXT:    lxv v31, 288(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    lxv v30, 272(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    lxv v29, 256(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    lxv v28, 240(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    extsw r3, r30
+; 64BIT-NEXT:    lxv v27, 224(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    lxv v26, 208(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    lxv v25, 192(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    lxv v24, 176(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    lxv v23, 160(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    lxv v22, 144(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    lxv v21, 128(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    lxv v20, 112(r1) # 16-byte Folded Reload
+; 64BIT-NEXT:    lfd f31, 584(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f30, 576(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f29, 568(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f28, 560(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r31, 448(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r30, 440(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r29, 432(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f27, 552(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r28, 424(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r27, 416(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r26, 408(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f26, 544(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r25, 400(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r24, 392(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r23, 384(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f25, 536(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r22, 376(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r21, 368(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r20, 360(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f24, 528(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r19, 352(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r18, 344(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r17, 336(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f23, 520(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r16, 328(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r15, 320(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    ld r14, 312(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f22, 512(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f21, 504(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f20, 496(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f19, 488(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f18, 480(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f17, 472(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f16, 464(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    lfd f15, 456(r1) # 8-byte Folded Reload
+; 64BIT-NEXT:    addi r1, r1, 592
+; 64BIT-NEXT:    ld r0, 16(r1)
+; 64BIT-NEXT:    mtlr r0
+; 64BIT-NEXT:    blr
+; 64BIT-NEXT:  L..BB0_2: # %lpad
+; 64BIT-NEXT:  L..tmp2:
+; 64BIT-NEXT:    bl .__cxa_begin_catch[PR]
+; 64BIT-NEXT:    nop
+; 64BIT-NEXT:    lwz r3, 0(r3)
+; 64BIT-NEXT:    add r30, r3, r31
+; 64BIT-NEXT:    bl .__cxa_end_catch[PR]
+; 64BIT-NEXT:    nop
+; 64BIT-NEXT:    b L..BB0_1
+entry:
+  tail call void asm sideeffect "nop", "~{r14},~{f15},~{v20}"()
+  %call = invoke signext i32 @_Z4testi(i32 signext %argc)
+          to label %return unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { ptr, i32 }
+          catch ptr @_ZTIi
+  %1 = extractvalue { ptr, i32 } %0, 1
+  %2 = tail call i32 @llvm.eh.typeid.for(ptr @_ZTIi) #3
+  %matches = icmp eq i32 %1, %2
+  br i1 %matches, label %catch, label %eh.resume
+
+catch:                                            ; preds = %lpad
+  %3 = extractvalue { ptr, i32 } %0, 0
+  %4 = tail call ptr @__cxa_begin_catch(ptr %3) #3
+  %5 = load i32, ptr %4, align 4
+  %add = add nsw i32 %5, %argc
+  tail call void @__cxa_end_catch()
+  br label %return
+
+return:                                           ; preds = %entry, %catch
+  %retval.0 = phi i32 [ %add, %catch ], [ 0, %entry ]
+  ret i32 %retval.0
+
+eh.resume:                                        ; preds = %lpad
+  resume { ptr, i32 } %0
+}
+
+declare signext i32 @_Z4testi(i32 signext) local_unnamed_addr
+
+declare i32 @__gxx_personality_v0(...)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.eh.typeid.for(ptr)
+
+declare ptr @__cxa_begin_catch(ptr) local_unnamed_addr
+
+declare void @__cxa_end_catch() local_unnamed_addr
+
+attributes #0 = { uwtable }
diff --git a/llvm/test/CodeGen/PowerPC/aix32-crsave.mir b/llvm/test/CodeGen/PowerPC/aix32-crsave.mir
index cf51f79c7e98..73736d6d5353 100644
--- a/llvm/test/CodeGen/PowerPC/aix32-crsave.mir
+++ b/llvm/test/CodeGen/PowerPC/aix32-crsave.mir
@@ -18,23 +18,33 @@ body:             |
     BLR implicit $lr, implicit $rm, implicit $r3
 
     ; CHECK-LABEL:  fixedStack:
-    ; CHECK-NEXT:   - { id: 0, type: spill-slot, offset: -12, size: 4, alignment: 4, stack-id: default,
+    ; CHECK-NEXT:   - { id: 0, type: spill-slot, offset: -4, size: 4, alignment: 4, stack-id: default, 
+    ; CHECK-NEXT:       callee-saved-register: '$r31', callee-saved-restored: true, debug-info-variable: '', 
+    ; CHECK-NEXT:       debug-info-expression: '', debug-info-location: '' }
+    ; CHECK-NEXT:   - { id: 1, type: spill-slot, offset: -8, size: 4, alignment: 8, stack-id: default, 
+    ; CHECK-NEXT:       callee-saved-register: '$r30', callee-saved-restored: true, debug-info-variable: '', 
+    ; CHECK-NEXT:       debug-info-expression: '', debug-info-location: '' }
+    ; CHECK-NEXT:   - { id: 2, type: spill-slot, offset: -12, size: 4, alignment: 4, stack-id: default,
     ; CHECK-NEXT:       callee-saved-register: '$r29', callee-saved-restored: true, debug-info-variable: '',
     ; CHECK-NEXT:       debug-info-expression: '', debug-info-location: '' }
-    ; CHECK-NEXT:   - { id: 1, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
+    ; CHECK-NEXT:   - { id: 3, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
     ; CHECK-NEXT:       isImmutable: true, isAliased: false, callee-saved-register: '$cr4',
     ; CHECK-NEXT:       callee-saved-restored: true, debug-info-variable: '', debug-info-expression: '',
     ; CHECK-NEXT:       debug-info-location: '' }
     ; CHECK-LABEL:  stack:
 
     ; CHECK:      bb.0.entry:
-    ; CHECK-NEXT:  liveins: $r3, $r29, $cr2, $cr4
+    ; CHECK-NEXT:  liveins: $r3, $r29, $r30, $r31, $cr2, $cr4
 
     ; CHECK:      $r12 = MFCR implicit killed $cr2, implicit killed $cr4
     ; CHECK-NEXT: STW killed $r12, 4, $r1
-    ; CHECK-NEXT: STW killed $r29, -12, $r1 :: (store (s32) into %fixed-stack.0)
+    ; CHECK-NEXT: STW killed $r29, -12, $r1 :: (store (s32) into %fixed-stack.2)
+    ; CHECK-NEXT: STW killed $r30, -8, $r1 :: (store (s32) into %fixed-stack.1, align 8)
+    ; CHECK-NEXT: STW killed $r31, -4, $r1 :: (store (s32) into %fixed-stack.0)
 
-    ; CHECK:      $r29 = LWZ -12, $r1 :: (load (s32) from %fixed-stack.0)
+    ; CHECK:      $r31 = LWZ -4, $r1 :: (load (s32) from %fixed-stack.0)
+    ; CHECK-NEXT: $r30 = LWZ -8, $r1 :: (load (s32) from %fixed-stack.1, align 8)
+    ; CHECK-NEXT: $r29 = LWZ -12, $r1 :: (load (s32) from %fixed-stack.2)
     ; CHECK-NEXT: $r12 = LWZ 4, $r1
     ; CHECK-NEXT: $cr2 = MTOCRF $r12
     ; CHECK-NEXT: $cr4 = MTOCRF killed $r12
@@ -49,14 +59,14 @@ liveins:
 body:             |
   bb.0.entry:
     liveins: $r3
-    renamable $r14 = ANDI_rec killed renamable $r3, 1, implicit-def dead $cr0, implicit-def $cr0gt
+    renamable $r31 = ANDI_rec killed renamable $r3, 1, implicit-def dead $cr0, implicit-def $cr0gt
     renamable $cr3lt = COPY $cr0gt
-    renamable $r3 = COPY $r14
+    renamable $r3 = COPY $r31
     BLR implicit $lr, implicit $rm, implicit $r3
 
     ; CHECK-LABEL: fixedStack:
-    ; CHECK-NEXT:  - { id: 0, type: spill-slot, offset: -72, size: 4, alignment: 8, stack-id: default,
-    ; CHECK-NEXT:      callee-saved-register: '$r14', callee-saved-restored: true, debug-info-variable: '',
+    ; CHECK-NEXT:  - { id: 0, type: spill-slot, offset: -4, size: 4, alignment: 4, stack-id: default,
+    ; CHECK-NEXT:      callee-saved-register: '$r31', callee-saved-restored: true, debug-info-variable: '',
     ; CHECK-NEXT:      debug-info-expression: '', debug-info-location: '' }
     ; CHECK-NEXT:  - { id: 1, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
     ; CHECK-NEXT:      isImmutable: true, isAliased: false, callee-saved-register: '$cr3',
@@ -65,12 +75,12 @@ body:             |
     ; CHECK-LABEL: stack:
 
     ; CHECK:      bb.0.entry:
-    ; CHECK-NEXT:   liveins: $r3, $r14, $cr3
+    ; CHECK-NEXT:   liveins: $r3, $r31, $cr3
 
     ; CHECK:      $r12 = MFCR implicit killed $cr3
     ; CHECK-NEXT: STW killed $r12, 4, $r1
-    ; CHECK-NEXT: STW killed $r14, -72, $r1 :: (store (s32) into %fixed-stack.0, align 8)
+    ; CHECK-NEXT: STW killed $r31, -4, $r1 :: (store (s32) into %fixed-stack.0)
 
-    ; CHECK:      $r14 = LWZ -72, $r1 :: (load (s32) from %fixed-stack.0, align 8)
+    ; CHECK:      $r31 = LWZ -4, $r1 :: (load (s32) from %fixed-stack.0)
     ; CHECK-NEXT: $r12 = LWZ 4, $r1
     ; CHECK-NEXT: $cr3 = MTOCRF killed $r12
diff --git a/llvm/test/CodeGen/PowerPC/ppc-shrink-wrapping.ll b/llvm/test/CodeGen/PowerPC/ppc-shrink-wrapping.ll
index f22aeffdbb46..412cb758ad60 100644
--- a/llvm/test/CodeGen/PowerPC/ppc-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc-shrink-wrapping.ll
@@ -31,7 +31,7 @@
 ; After the prologue is set.
 ; DISABLE: cmpw 3, 4
 ; DISABLE-32: stw 0,
-; DISABLE-64-AIX: std 0, 
+; DISABLE-64-AIX: std 0,
 ; DISABLE-NEXT: bge 0, {{.*}}[[EXIT_LABEL:BB[0-9_]+]]
 ;
 ; Store %a on the stack
@@ -421,14 +421,14 @@ entry:
 ; ENABLE-NEXT: beq 0, {{.*}}[[ELSE_LABEL:BB[0-9_]+]]
 ;
 ; Prologue code.
-; Make sure we save the CSR used in the inline asm: r14
+; Make sure we save the CSR used in the inline asm: r31
 ; ENABLE-DAG: li [[IV:[0-9]+]], 10
-; ENABLE-64-DAG: std 14, -[[STACK_OFFSET:[0-9]+]](1) # 8-byte Folded Spill
-; ENABLE-32-DAG: stw 14, -[[STACK_OFFSET:[0-9]+]](1) # 4-byte Folded Spill
+; ENABLE-64-DAG: std 31, -[[STACK_OFFSET:[0-9]+]](1) # 8-byte Folded Spill
+; ENABLE-32-DAG: stw 31, -[[STACK_OFFSET:[0-9]+]](1) # 4-byte Folded Spill
 ;
 ; DISABLE: cmplwi 3, 0
-; DISABLE-64-NEXT: std 14, -[[STACK_OFFSET:[0-9]+]](1) # 8-byte Folded Spill
-; DISABLE-32-NEXT: stw 14, -[[STACK_OFFSET:[0-9]+]](1) # 4-byte Folded Spill
+; DISABLE-64-NEXT: std 31, -[[STACK_OFFSET:[0-9]+]](1) # 8-byte Folded Spill
+; DISABLE-32-NEXT: stw 31, -[[STACK_OFFSET:[0-9]+]](1) # 4-byte Folded Spill
 ; DISABLE-NEXT: beq 0, {{.*}}[[ELSE_LABEL:BB[0-9_]+]]
 ; DISABLE: li [[IV:[0-9]+]], 10
 ;
@@ -437,20 +437,20 @@ entry:
 ;
 ; CHECK: {{.*}}[[LOOP_LABEL:BB[0-9_]+]]: # %for.body
 ; Inline asm statement.
-; CHECK: addi 14, 14, 1
+; CHECK: addi 31, 14, 1
 ; CHECK: bdnz {{.*}}[[LOOP_LABEL]]
 ;
 ; Epilogue code.
 ; CHECK: li 3, 0
-; CHECK-64-DAG: ld 14, -[[STACK_OFFSET]](1) # 8-byte Folded Reload
-; CHECK-32-DAG: lwz 14, -[[STACK_OFFSET]](1) # 4-byte Folded Reload
+; CHECK-64-DAG: ld 31, -[[STACK_OFFSET]](1) # 8-byte Folded Reload
+; CHECK-32-DAG: lwz 31, -[[STACK_OFFSET]](1) # 4-byte Folded Reload
 ; CHECK-DAG: nop
 ; CHECK: blr
 ;
 ; CHECK: [[ELSE_LABEL]]
 ; CHECK-NEXT: slwi 3, 4, 1
-; DISABLE-64-NEXT: ld 14, -[[STACK_OFFSET]](1) # 8-byte Folded Reload
-; DISABLE-32-NEXT: lwz 14, -[[STACK_OFFSET]](1) # 4-byte Folded Reload
+; DISABLE-64-NEXT: ld 31, -[[STACK_OFFSET]](1) # 8-byte Folded Reload
+; DISABLE-32-NEXT: lwz 31, -[[STACK_OFFSET]](1) # 4-byte Folded Reload
 ; CHECK-NEXT: blr
 define i32 @inlineAsm(i32 %cond, i32 %N) {
 entry:
@@ -463,7 +463,7 @@ for.preheader:
 
 for.body:                                         ; preds = %entry, %for.body
   %i.03 = phi i32 [ %inc, %for.body ], [ 0, %for.preheader ]
-  tail call void asm "addi 14, 14, 1", "~{r14}"()
+  tail call void asm "addi 31, 14, 1", "~{r31}"()
   %inc = add nuw nsw i32 %i.03, 1
   %exitcond = icmp eq i32 %inc, 10
   br i1 %exitcond, label %for.exit, label %for.body
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-crsave.mir b/llvm/test/CodeGen/PowerPC/ppc64-crsave.mir
index f4af2ad21a56..196ad134bfa5 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-crsave.mir
+++ b/llvm/test/CodeGen/PowerPC/ppc64-crsave.mir
@@ -1,15 +1,15 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple powerpc64le-unknown-linux-gnu -x mir -mcpu=pwr8 -mattr=-altivec \
-# RUN: -run-pass=prologepilog --verify-machineinstrs < %s | \
-# RUN: FileCheck %s --check-prefixes=CHECK,SAVEONE
+# RUN: -run-pass=prologepilog --verify-machineinstrs %s -o - | \
+# RUN: FileCheck %s --check-prefix=SAVEONE
 
 # RUN: llc -mtriple powerpc64-unknown-linux-gnu -x mir -mcpu=pwr7 -mattr=-altivec \
-# RUN: -run-pass=prologepilog --verify-machineinstrs < %s | \
-# RUN: FileCheck %s --check-prefixes=CHECK,SAVEALL
-
+# RUN: -run-pass=prologepilog --verify-machineinstrs %s -o - | \
+# RUN: FileCheck %s --check-prefix=SAVEALL
 
 # RUN: llc -mtriple powerpc64-unknown-aix-xcoff -x mir -mcpu=pwr4 -mattr=-altivec \
-# RUN: -run-pass=prologepilog --verify-machineinstrs < %s | \
-# RUN: FileCheck %s --check-prefixes=CHECK,SAVEALL
+# RUN: -run-pass=prologepilog --verify-machineinstrs %s -o - | \
+# RUN: FileCheck %s --check-prefix=SAVEALL
 
 ---
 name:            CRAllSave
@@ -20,33 +20,39 @@ liveins:
 body:             |
   bb.0.entry:
     liveins: $x3
-    renamable $x29 = ANDI8_rec killed renamable $x3, 1, implicit-def dead $cr0, implicit-def $cr0gt
+    ; SAVEONE-LABEL: name: CRAllSave
+    ; SAVEONE: liveins: $x3, $cr2, $cr4
+    ; SAVEONE-NEXT: {{  $}}
+    ; SAVEONE-NEXT: $x12 = MFCR8 implicit killed $cr2, implicit killed $cr4
+    ; SAVEONE-NEXT: STW8 killed $x12, 8, $x1
+    ; SAVEONE-NEXT: renamable $x3 = ANDI8_rec killed renamable $x3, 1, implicit-def dead $cr0, implicit-def $cr0gt
+    ; SAVEONE-NEXT: renamable $cr2lt = COPY $cr0gt
+    ; SAVEONE-NEXT: renamable $cr4lt = COPY $cr0gt
+    ; SAVEONE-NEXT: $x12 = LWZ8 8, $x1
+    ; SAVEONE-NEXT: $cr2 = MTOCRF8 $x12
+    ; SAVEONE-NEXT: $cr4 = MTOCRF8 killed $x12
+    ; SAVEONE-NEXT: BLR8 implicit $lr8, implicit $rm, implicit $x3
+    ;
+    ; SAVEALL-LABEL: name: CRAllSave
+    ; SAVEALL: liveins: $x3, $cr2, $cr4
+    ; SAVEALL-NEXT: {{  $}}
+    ; SAVEALL-NEXT: $x12 = MFCR8 implicit killed $cr2, implicit killed $cr4
+    ; SAVEALL-NEXT: STW8 killed $x12, 8, $x1
+    ; SAVEALL-NEXT: renamable $x3 = ANDI8_rec killed renamable $x3, 1, implicit-def dead $cr0, implicit-def $cr0gt
+    ; SAVEALL-NEXT: renamable $cr2lt = COPY $cr0gt
+    ; SAVEALL-NEXT: renamable $cr4lt = COPY $cr0gt
+    ; SAVEALL-NEXT: $x12 = LWZ8 8, $x1
+    ; SAVEALL-NEXT: $cr2 = MTOCRF8 $x12
+    ; SAVEALL-NEXT: $cr4 = MTOCRF8 killed $x12
+    ; SAVEALL-NEXT: BLR8 implicit $lr8, implicit $rm, implicit $x3
+    renamable $x3 = ANDI8_rec killed renamable $x3, 1, implicit-def dead $cr0, implicit-def $cr0gt
     renamable $cr2lt = COPY $cr0gt
     renamable $cr4lt = COPY $cr0gt
-    renamable $x3 = COPY $x29
     BLR8 implicit $lr8, implicit $rm, implicit $x3
 
-    ; CHECK-LABEL: fixedStack:
-    ; CHECK-NEXT:     - { id: 0, type: spill-slot, offset: -24, size: 8, alignment: 8, stack-id: default,
-    ; CHECK-NEXT:         callee-saved-register: '$x29', callee-saved-restored: true, debug-info-variable: '',
-    ; CHECK-NEXT:         debug-info-expression: '', debug-info-location: '' }
-    ; CHECK-NEXT:     - { id: 1, type: default, offset: 8, size: 4, alignment: 8, stack-id: default,
-    ; CHECK-NEXT:         isImmutable: true, isAliased: false, callee-saved-register: '$cr4',
-    ; CHECK-NEXT:         callee-saved-restored: true, debug-info-variable: '', debug-info-expression: '',
-    ; CHECK-NEXT:         debug-info-location: '' }
-    ; CHECK-LABEL:  stack:
 
-    ; Verify the proper live-ins have been added in the prologue.
-    ; CHECK:    liveins: $x3, $x29, $cr2, $cr4
 
-    ; CHECK:     $x12 = MFCR8 implicit killed $cr2, implicit killed $cr4
-    ; CHECK-DAG: STD killed $x29, -24, $x1 :: (store (s64) into %fixed-stack.0)
-    ; CHECK-DAG: STW8 killed $x12, 8, $x1
 
-    ; CHECK:     $x29 = LD -24, $x1 :: (load (s64) from %fixed-stack.0)
-    ; CHECK:     $x12 = LWZ8 8, $x1
-    ; CHECK:     $cr2 = MTOCRF8 $x12
-    ; CHECK:     $cr4 = MTOCRF8 killed $x12
 
 ...
 ---
@@ -58,37 +64,36 @@ liveins:
 body:             |
   bb.0.entry:
     liveins: $x3
-    renamable $x14 = ANDI8_rec killed renamable $x3, 1, implicit-def dead $cr0, implicit-def $cr0gt
+    ; SAVEONE-LABEL: name: CR2Save
+    ; SAVEONE: liveins: $x3, $cr2
+    ; SAVEONE-NEXT: {{  $}}
+    ; SAVEONE-NEXT: $x12 = MFOCRF8 killed $cr2
+    ; SAVEONE-NEXT: STW8 killed $x12, 8, $x1
+    ; SAVEONE-NEXT: renamable $x3 = ANDI8_rec killed renamable $x3, 1, implicit-def dead $cr0, implicit-def $cr0gt
+    ; SAVEONE-NEXT: renamable $cr2lt = COPY $cr0gt
+    ; SAVEONE-NEXT: $x12 = LWZ8 8, $x1
+    ; SAVEONE-NEXT: $cr2 = MTOCRF8 killed $x12
+    ; SAVEONE-NEXT: BLR8 implicit $lr8, implicit $rm, implicit $x3
+    ;
+    ; SAVEALL-LABEL: name: CR2Save
+    ; SAVEALL: liveins: $x3, $cr2
+    ; SAVEALL-NEXT: {{  $}}
+    ; SAVEALL-NEXT: $x12 = MFCR8 implicit killed $cr2
+    ; SAVEALL-NEXT: STW8 killed $x12, 8, $x1
+    ; SAVEALL-NEXT: renamable $x3 = ANDI8_rec killed renamable $x3, 1, implicit-def dead $cr0, implicit-def $cr0gt
+    ; SAVEALL-NEXT: renamable $cr2lt = COPY $cr0gt
+    ; SAVEALL-NEXT: $x12 = LWZ8 8, $x1
+    ; SAVEALL-NEXT: $cr2 = MTOCRF8 killed $x12
+    ; SAVEALL-NEXT: BLR8 implicit $lr8, implicit $rm, implicit $x3
+    renamable $x3 = ANDI8_rec killed renamable $x3, 1, implicit-def dead $cr0, implicit-def $cr0gt
     renamable $cr2lt = COPY $cr0gt
-    renamable $x3 = COPY $x14
     BLR8 implicit $lr8, implicit $rm, implicit $x3
 
-    ; CHECK-LABEL: CR2Save
 
-    ; CHECK-LABEL: fixedStack:
-    ; CHECK-NEXT:   - { id: 0, type: spill-slot, offset: -144, size: 8, alignment: 16, stack-id: default,
-    ; CHECK-NEXT:       callee-saved-register: '$x14', callee-saved-restored: true, debug-info-variable: '',
-    ; CHECK-NEXT:       debug-info-expression: '', debug-info-location: '' }
-    ; CHECK-NEXT:   - { id: 1, type: default, offset: 8, size: 4, alignment: 8, stack-id: default,
-    ; CHECK-NEXT:       isImmutable: true, isAliased: false, callee-saved-register: '$cr2',
-    ; CHECK-NEXT:       callee-saved-restored: true, debug-info-variable: '', debug-info-expression: '',
-    ; CHECK-NEXT:       debug-info-location: '' }
-    ; CHECK-LABEL:  stack:
 
-    ; Verify the proper live-ins have been added in the prologue.
-    ; CHECK:    liveins: $x3, $x14, $cr2
 
-    ; ELF V2 ABI allows saving only the clobbered cr fields,
-    ; whereas the other ABIs do not.
-    ; SAVEONE:     $x12 = MFOCRF8 killed $cr2
-    ; SAVEALL:     $x12 = MFCR8 implicit killed $cr2
 
-    ; CHECK-DAG: STD killed $x14, -144, $x1 :: (store (s64) into %fixed-stack.0, align 16)
-    ; CHECK-DAG: STW8 killed $x12, 8, $x1
 
-    ; CHECK:     $x14 = LD -144, $x1 :: (load (s64) from %fixed-stack.0, align 16)
-    ; CHECK:     $x12 = LWZ8 8, $x1
-    ; CHECK:     $cr2 = MTOCRF8 killed $x12
 
 
 ...
-- 
GitLab


From 082c81ae4ab9db6bb0acd52098288223dd58501a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 7 May 2024 21:31:40 +0100
Subject: [PATCH 0100/1206] [LV] Properly extend versioned constant strides.

We only version unknown strides to 1. If the original type is i1, then
the sign of the extension matters. Properly extend the stride value
before replacing it.

Fixes https://github.com/llvm/llvm-project/issues/91369.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp             | 6 ++++--
 .../LoopVectorize/version-stride-with-integer-casts.ll      | 3 +--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3be0102bea3e..261933966b74 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8841,8 +8841,10 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
       VPValue *StrideVPV = Plan->getLiveIn(U);
       if (!StrideVPV)
         continue;
-      VPValue *CI = Plan->getOrAddLiveIn(ConstantInt::get(
-          U->getType(), ScevStride->getAPInt().getSExtValue()));
+      unsigned BW = U->getType()->getScalarSizeInBits();
+      APInt C = isa<SExtInst>(U) ? ScevStride->getAPInt().sext(BW)
+                                 : ScevStride->getAPInt().zext(BW);
+      VPValue *CI = Plan->getOrAddLiveIn(ConstantInt::get(U->getType(), C));
       StrideVPV->replaceAllUsesWith(CI);
     }
   }
diff --git a/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll b/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
index 45745f85de95..45596169da3c 100644
--- a/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
+++ b/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
@@ -415,7 +415,6 @@ exit:
 
 ; Test case to make sure that uses of versioned strides of type i1 are properly
 ; extended. From https://github.com/llvm/llvm-project/issues/91369.
-; FIXME: Currently miscompiled.
 define void @zext_of_i1_stride(i1 %g, ptr %dst) mustprogress {
 ; CHECK-LABEL: define void @zext_of_i1_stride(
 ; CHECK-SAME: i1 [[G:%.*]], ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
@@ -441,7 +440,7 @@ define void @zext_of_i1_stride(i1 %g, ptr %dst) mustprogress {
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[TMP4]], i32 0
-; CHECK-NEXT:    store <4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1>, ptr [[TMP5]], align 2
+; CHECK-NEXT:    store <4 x i16> <i16 1, i16 1, i16 1, i16 1>, ptr [[TMP5]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-- 
GitLab


From 0b50d095bccbd47c77e5ad2b03b09b41b696c4a0 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Tue, 7 May 2024 16:44:00 -0400
Subject: [PATCH 0101/1206] [AMDGPU] Don't optimize agpr phis if the operand
 doesn't have subreg use (#91267)

If the operand doesn't have any subreg use, the optimization could
potentially
generate `V_ACCVGPR_READ_B32_e64` with wrong register class. The
following example demonstrates the issue.

Input MIR:

```
bb.0:
  %0:sgpr_32 = S_MOV_B32 0
  %1:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %0:sgpr_32, %subreg.sub1, %0:sgpr_32, %subreg.sub2, %0:sgpr_32, %subreg.sub3
  %2:vreg_128 = COPY %1:sgpr_128
  %3:areg_128 = COPY %2:vreg_128, implicit $exec

bb.1:
  %4:areg_128 = PHI %3:areg_128, %bb.0, %6:areg_128, %bb.1
  %5:areg_128 = PHI %3:areg_128, %bb.0, %7:areg_128, %bb.1
  ...
```

Output of current implementation:

```
bb.0:
  %0:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
  %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
  %2:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
  %3:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
  %4:areg_128 = REG_SEQUENCE %0:agpr_32, %subreg.sub0, %1:agpr_32, %subreg.sub1, %2:agpr_32, %subreg.sub2, %3:agpr_32, %subreg.sub3
  %5:vreg_128 = V_ACCVGPR_READ_B32_e64 %4:areg_128, implicit $exec
  %6:areg_128 = COPY %46:vreg_128

bb.1:
  %7:areg_128 = PHI %6:areg_128, %bb.0, %9:areg_128, %bb.1
  %8:areg_128 = PHI %6:areg_128, %bb.0, %10:areg_128, %bb.1
  ...
```

The problem is the generated `V_ACCVGPR_READ_B32_e64` instruction.
Apparently the operand `%4:areg_128` is not valid for this.

In this patch, we don't count the none-subreg use because
`V_ACCVGPR_READ_B32_e64` can't handle none-32-bit operand.

Fixes: SWDEV-459556
---
 llvm/lib/Target/AMDGPU/SIFoldOperands.cpp   |  2 +
 llvm/test/CodeGen/AMDGPU/fold-agpr-phis.mir | 83 ++++++++++++++++++++-
 2 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index cb448aaafa4c..5c411a095587 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -2106,6 +2106,8 @@ bool SIFoldOperands::tryOptimizeAGPRPhis(MachineBasicBlock &MBB) {
 
     for (unsigned K = 1; K < MI.getNumOperands(); K += 2) {
       MachineOperand &PhiMO = MI.getOperand(K);
+      if (!PhiMO.getSubReg())
+        continue;
       RegToMO[{PhiMO.getReg(), PhiMO.getSubReg()}].push_back(&PhiMO);
     }
   }
diff --git a/llvm/test/CodeGen/AMDGPU/fold-agpr-phis.mir b/llvm/test/CodeGen/AMDGPU/fold-agpr-phis.mir
index a32b3d0f1e6b..e94546fd5e8a 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-agpr-phis.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-agpr-phis.mir
@@ -465,7 +465,6 @@ body: |
   ; GFX90A-NEXT: bb.2:
   ; GFX90A-NEXT:   S_ENDPGM 0
   bb.0:
-    ; Tests that tryOptimizeAGPRPhis kicks in for GFX908.
     liveins: $sgpr0, $scc
     successors: %bb.1
 
@@ -715,3 +714,85 @@ body: |
   bb.3:
     S_ENDPGM 0
 ...
+
+---
+name:            skip_optimize_agpr_phi_without_subreg_use
+tracksRegLiveness: true
+body:             |
+  ; GFX908-LABEL: name: skip_optimize_agpr_phi_without_subreg_use
+  ; GFX908: bb.0:
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $scc
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+  ; GFX908-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_2:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_3:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:areg_128_align2 = REG_SEQUENCE [[V_ACCVGPR_WRITE_B32_e64_]], %subreg.sub0, [[V_ACCVGPR_WRITE_B32_e64_1]], %subreg.sub1, [[V_ACCVGPR_WRITE_B32_e64_2]], %subreg.sub2, [[V_ACCVGPR_WRITE_B32_e64_3]], %subreg.sub3
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; GFX908-NEXT:   liveins: $scc
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[PHI:%[0-9]+]]:areg_128_align2 = PHI [[REG_SEQUENCE]], %bb.0, %7, %bb.1
+  ; GFX908-NEXT:   [[V_MFMA_F32_16X16X4F32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X4F32_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], [[PHI]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[V_MFMA_F32_16X16X4F32_e64_]], implicit $exec
+  ; GFX908-NEXT:   S_CBRANCH_SCC1 %bb.1, implicit $scc
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX90A-LABEL: name: skip_optimize_agpr_phi_without_subreg_use
+  ; GFX90A: bb.0:
+  ; GFX90A-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX90A-NEXT:   liveins: $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+  ; GFX90A-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX90A-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX90A-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_2:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX90A-NEXT:   [[V_ACCVGPR_WRITE_B32_e64_3:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+  ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:areg_128_align2 = REG_SEQUENCE [[V_ACCVGPR_WRITE_B32_e64_]], %subreg.sub0, [[V_ACCVGPR_WRITE_B32_e64_1]], %subreg.sub1, [[V_ACCVGPR_WRITE_B32_e64_2]], %subreg.sub2, [[V_ACCVGPR_WRITE_B32_e64_3]], %subreg.sub3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.1:
+  ; GFX90A-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; GFX90A-NEXT:   liveins: $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   [[PHI:%[0-9]+]]:areg_128_align2 = PHI [[REG_SEQUENCE]], %bb.0, %7, %bb.1
+  ; GFX90A-NEXT:   [[V_MFMA_F32_16X16X4F32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X4F32_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], [[PHI]], 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX90A-NEXT:   [[COPY:%[0-9]+]]:areg_128_align2 = COPY [[V_MFMA_F32_16X16X4F32_e64_]], implicit $exec
+  ; GFX90A-NEXT:   S_CBRANCH_SCC1 %bb.1, implicit $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.2:
+  ; GFX90A-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $scc
+    successors: %bb.1
+
+    %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %1:sgpr_32 = S_MOV_B32 0
+    %2:sgpr_128 = REG_SEQUENCE %1, %subreg.sub0, %1, %subreg.sub1, %1, %subreg.sub2, %1, %subreg.sub3
+    %3:vreg_128 = COPY %2
+    %4:sreg_64 = S_MOV_B64 0
+    %5:areg_128_align2 = COPY %3, implicit $exec
+
+  bb.1:
+    liveins: $scc
+    successors: %bb.1, %bb.2
+
+    %9:areg_128_align2 = PHI %5, %bb.0, %10, %bb.1
+    %11:areg_128_align2 = V_MFMA_F32_16X16X4F32_e64 %0:vgpr_32, %0:vgpr_32, %9:areg_128_align2, 0, 0, 0, implicit $mode, implicit $exec
+    %12:vgpr_32 = COPY %11.sub3
+    %13:vgpr_32 = COPY %11.sub2
+    %14:vgpr_32 = COPY %11.sub1
+    %15:vgpr_32 = COPY %11.sub0
+    %10:areg_128_align2 = COPY %11, implicit $exec
+    S_CBRANCH_SCC1 %bb.1, implicit $scc
+
+  bb.2:
+    S_ENDPGM 0
+
+...
-- 
GitLab


From 272ea28bdec93b33527dc54edbdef8f43c51df47 Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Tue, 7 May 2024 12:57:43 -0700
Subject: [PATCH 0102/1206] Remove else-after-break (NFC)

---
 lldb/source/Expression/UserExpression.cpp | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/lldb/source/Expression/UserExpression.cpp b/lldb/source/Expression/UserExpression.cpp
index 5658426c8891..06fdb7007ced 100644
--- a/lldb/source/Expression/UserExpression.cpp
+++ b/lldb/source/Expression/UserExpression.cpp
@@ -308,17 +308,16 @@ UserExpression::Evaluate(ExecutionContext &exe_ctx,
           diagnostic_manager.Clear();
           user_expression_sp = fixed_expression_sp;
           break;
+        }
+        // The fixed expression also didn't parse. Let's check for any new
+        // fixits we could try.
+        if (!fixed_expression_sp->GetFixedText().empty()) {
+          *fixed_expression = fixed_expression_sp->GetFixedText().str();
         } else {
-          // The fixed expression also didn't parse. Let's check for any new
-          // Fix-Its we could try.
-          if (!fixed_expression_sp->GetFixedText().empty()) {
-            *fixed_expression = fixed_expression_sp->GetFixedText().str();
-          } else {
-            // Fixed expression didn't compile without a fixit, don't retry and
-            // don't tell the user about it.
-            fixed_expression->clear();
-            break;
-          }
+          // Fixed expression didn't compile without a fixit, don't retry and
+          // don't tell the user about it.
+          fixed_expression->clear();
+          break;
         }
       }
     }
-- 
GitLab


From 8c4d7989c2b4a7e251afc3b13002611646de90b6 Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Tue, 7 May 2024 12:58:20 -0700
Subject: [PATCH 0103/1206] Add a missing check for nullptr

This can't happen with Clang, but I've seen a crash report from the
Swift plugin where this happened.

rdar://126564844
---
 lldb/source/Expression/UserExpression.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lldb/source/Expression/UserExpression.cpp b/lldb/source/Expression/UserExpression.cpp
index 06fdb7007ced..b78f43995767 100644
--- a/lldb/source/Expression/UserExpression.cpp
+++ b/lldb/source/Expression/UserExpression.cpp
@@ -300,6 +300,8 @@ UserExpression::Evaluate(ExecutionContext &exe_ctx,
             target->GetUserExpressionForLanguage(
                 fixed_expression->c_str(), full_prefix, language, desired_type,
                 options, ctx_obj, error));
+        if (!fixed_expression_sp)
+          break;
         DiagnosticManager fixed_diagnostic_manager;
         parse_success = fixed_expression_sp->Parse(
             fixed_diagnostic_manager, exe_ctx, execution_policy,
-- 
GitLab


From a70ad96b3cc5275246f7f007d1892bb867b75bc0 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Tue, 7 May 2024 13:45:58 -0700
Subject: [PATCH 0104/1206] [AMDGPU] Fix condition in VOP3_Real_Base. NFCI.
 (#91373)

---
 llvm/lib/Target/AMDGPU/VOPInstructions.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index 74988ac634c3..d974aacd7d45 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -1410,7 +1410,7 @@ multiclass VOP3_Real_Base<GFXGen Gen, bits<10> op, string opName = NAME,
       def _e64#Gen.Suffix :
         VOP3_Real_Gen<ps, Gen>,
         VOP3FP8OpSel_dst_bytesel_gfx11_gfx12<op, ps.Pfl>;
-    } if ps.Pfl.HasOpSel then {
+    } else if ps.Pfl.HasOpSel then {
       def _e64#Gen.Suffix :
         VOP3_Real_Gen<ps, Gen>,
         VOP3OpSel_gfx11_gfx12<op, ps.Pfl>;
-- 
GitLab


From 2ad6917c4c524576405f2146424911fd9adb3528 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellis.sparky.hoag@gmail.com>
Date: Tue, 7 May 2024 13:55:44 -0700
Subject: [PATCH 0105/1206] [modules] Accept equivalent module caches from
 different symlink (#90925)

Use `VFS.equivalent()`, which follows symlinks, to check if two module
cache paths are equivalent. This prevents a PCH error when building from
a different path that is a symlink of the original.

```
error: PCH was compiled with module cache path '/home/foo/blah/ModuleCache/2IBP1TNT8OR8D', but the path is currently '/data/users/foo/blah/ModuleCache/2IBP1TNT8OR8D'
1 error generated.
```
---
 clang/lib/Serialization/ASTReader.cpp         | 53 ++++++++++---------
 clang/test/Modules/module-symlink.m           | 14 +++++
 llvm/include/llvm/Support/VirtualFileSystem.h |  4 ++
 llvm/lib/Support/VirtualFileSystem.cpp        | 10 ++++
 4 files changed, 55 insertions(+), 26 deletions(-)
 create mode 100644 clang/test/Modules/module-symlink.m

diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index b4b2f999d225..856c743086c5 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -829,36 +829,37 @@ bool SimpleASTReaderListener::ReadPreprocessorOptions(
                                   OptionValidateNone);
 }
 
-/// Check the header search options deserialized from the control block
-/// against the header search options in an existing preprocessor.
+/// Check that the specified and the existing module cache paths are equivalent.
 ///
 /// \param Diags If non-null, produce diagnostics for any mismatches incurred.
-static bool checkHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
-                                     StringRef SpecificModuleCachePath,
-                                     StringRef ExistingModuleCachePath,
-                                     DiagnosticsEngine *Diags,
-                                     const LangOptions &LangOpts,
-                                     const PreprocessorOptions &PPOpts) {
-  if (LangOpts.Modules) {
-    if (SpecificModuleCachePath != ExistingModuleCachePath &&
-        !PPOpts.AllowPCHWithDifferentModulesCachePath) {
-      if (Diags)
-        Diags->Report(diag::err_pch_modulecache_mismatch)
-          << SpecificModuleCachePath << ExistingModuleCachePath;
-      return true;
-    }
-  }
-
-  return false;
+/// \returns true when the module cache paths differ.
+static bool checkModuleCachePath(llvm::vfs::FileSystem &VFS,
+                                 StringRef SpecificModuleCachePath,
+                                 StringRef ExistingModuleCachePath,
+                                 DiagnosticsEngine *Diags,
+                                 const LangOptions &LangOpts,
+                                 const PreprocessorOptions &PPOpts) {
+  if (!LangOpts.Modules || PPOpts.AllowPCHWithDifferentModulesCachePath ||
+      SpecificModuleCachePath == ExistingModuleCachePath)
+    return false;
+  auto EqualOrErr =
+      VFS.equivalent(SpecificModuleCachePath, ExistingModuleCachePath);
+  if (EqualOrErr && *EqualOrErr)
+    return false;
+  if (Diags)
+    Diags->Report(diag::err_pch_modulecache_mismatch)
+        << SpecificModuleCachePath << ExistingModuleCachePath;
+  return true;
 }
 
 bool PCHValidator::ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
                                            StringRef SpecificModuleCachePath,
                                            bool Complain) {
-  return checkHeaderSearchOptions(HSOpts, SpecificModuleCachePath,
-                                  PP.getHeaderSearchInfo().getModuleCachePath(),
-                                  Complain ? &Reader.Diags : nullptr,
-                                  PP.getLangOpts(), PP.getPreprocessorOpts());
+  return checkModuleCachePath(Reader.getFileManager().getVirtualFileSystem(),
+                              SpecificModuleCachePath,
+                              PP.getHeaderSearchInfo().getModuleCachePath(),
+                              Complain ? &Reader.Diags : nullptr,
+                              PP.getLangOpts(), PP.getPreprocessorOpts());
 }
 
 void PCHValidator::ReadCounter(const ModuleFile &M, unsigned Value) {
@@ -5376,9 +5377,9 @@ namespace {
     bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
                                  StringRef SpecificModuleCachePath,
                                  bool Complain) override {
-      return checkHeaderSearchOptions(HSOpts, SpecificModuleCachePath,
-                                      ExistingModuleCachePath, nullptr,
-                                      ExistingLangOpts, ExistingPPOpts);
+      return checkModuleCachePath(
+          FileMgr.getVirtualFileSystem(), SpecificModuleCachePath,
+          ExistingModuleCachePath, nullptr, ExistingLangOpts, ExistingPPOpts);
     }
 
     bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
diff --git a/clang/test/Modules/module-symlink.m b/clang/test/Modules/module-symlink.m
new file mode 100644
index 000000000000..efdaf3db0dfe
--- /dev/null
+++ b/clang/test/Modules/module-symlink.m
@@ -0,0 +1,14 @@
+// REQUIRES: shell
+
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -fmodules-cache-path=%t/modules -fmodules -fimplicit-module-maps -I %S/Inputs -emit-pch -o %t.pch %s -verify
+
+// RUN: ln -s %t/modules %t/modules.symlink
+// RUN: %clang_cc1 -fmodules-cache-path=%t/modules.symlink -fmodules -fimplicit-module-maps -I %S/Inputs -include-pch %t.pch %s -verify
+// RUN: not %clang_cc1 -fmodules-cache-path=%t/modules.dne -fmodules -fimplicit-module-maps -I %S/Inputs -include-pch %t.pch %s -verify
+
+// expected-no-diagnostics
+
+@import ignored_macros;
+
+struct Point p;
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 49e67e7555a0..a1e38de74dfc 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -320,6 +320,10 @@ public:
   ///          platform-specific error_code.
   virtual std::error_code makeAbsolute(SmallVectorImpl<char> &Path) const;
 
+  /// \returns true if \p A and \p B represent the same file, or an error or
+  /// false if they do not.
+  llvm::ErrorOr<bool> equivalent(const Twine &A, const Twine &B);
+
   enum class PrintType { Summary, Contents, RecursiveContents };
   void print(raw_ostream &OS, PrintType Type = PrintType::Contents,
              unsigned IndentLevel = 0) const {
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 152fcfe695b2..54b9c38f7609 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -151,6 +151,16 @@ bool FileSystem::exists(const Twine &Path) {
   return Status && Status->exists();
 }
 
+llvm::ErrorOr<bool> FileSystem::equivalent(const Twine &A, const Twine &B) {
+  auto StatusA = status(A);
+  if (!StatusA)
+    return StatusA.getError();
+  auto StatusB = status(B);
+  if (!StatusB)
+    return StatusB.getError();
+  return StatusA->equivalent(*StatusB);
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void FileSystem::dump() const { print(dbgs(), PrintType::RecursiveContents); }
 #endif
-- 
GitLab


From bc8a42762057d7036f6871211e62b1c3efb2738a Mon Sep 17 00:00:00 2001
From: Marian Buschsieweke <maribu@users.noreply.github.com>
Date: Tue, 7 May 2024 22:58:13 +0200
Subject: [PATCH 0106/1206] [MSP430][Clang] Update list of MCUs (#91258)

This updates the list of MSP430 MCUs from TI's devices.csv obtained from [1] under the "Header and Support Files" link. A simple python script has been used to generate this list and is included as well.

[1]: https://www.ti.com/tool/MSP430-GCC-OPENSOURCE#downloads
---
 clang/include/clang/Basic/MSP430Target.def    | 583 +++++++++++++++---
 .../Basic/Target/MSP430/gen-msp430-def.py     | 126 ++++
 2 files changed, 626 insertions(+), 83 deletions(-)
 create mode 100755 clang/include/clang/Basic/Target/MSP430/gen-msp430-def.py

diff --git a/clang/include/clang/Basic/MSP430Target.def b/clang/include/clang/Basic/MSP430Target.def
index 7a10be1d54c8..8fd44e3ba8e7 100644
--- a/clang/include/clang/Basic/MSP430Target.def
+++ b/clang/include/clang/Basic/MSP430Target.def
@@ -8,6 +8,10 @@
 //
 // This file defines the MSP430 devices and their features.
 //
+// Generated from TI's devices.csv in version 1.212 using the script in
+// Target/MSP430/gen-msp430-def.py - use this tool rather than adding
+// new MCUs by hand.
+//
 //===----------------------------------------------------------------------===//
 
 #ifndef MSP430_MCU_FEAT
@@ -24,7 +28,19 @@ MSP430_MCU("msp430c112")
 MSP430_MCU("msp430c1121")
 MSP430_MCU("msp430c1331")
 MSP430_MCU("msp430c1351")
+MSP430_MCU("msp430c311s")
+MSP430_MCU("msp430c312")
+MSP430_MCU("msp430c313")
+MSP430_MCU("msp430c314")
+MSP430_MCU("msp430c315")
+MSP430_MCU("msp430c323")
+MSP430_MCU("msp430c325")
+MSP430_MCU("msp430c412")
+MSP430_MCU("msp430c413")
 MSP430_MCU("msp430e112")
+MSP430_MCU("msp430e313")
+MSP430_MCU("msp430e315")
+MSP430_MCU("msp430e325")
 MSP430_MCU("msp430f110")
 MSP430_MCU("msp430f1101")
 MSP430_MCU("msp430f1101a")
@@ -44,7 +60,6 @@ MSP430_MCU("msp430f135")
 MSP430_MCU("msp430f155")
 MSP430_MCU("msp430f156")
 MSP430_MCU("msp430f157")
-MSP430_MCU("msp430p112")
 MSP430_MCU("msp430f2001")
 MSP430_MCU("msp430f2011")
 MSP430_MCU("msp430f2002")
@@ -64,6 +79,58 @@ MSP430_MCU("msp430f2272")
 MSP430_MCU("msp430f2234")
 MSP430_MCU("msp430f2254")
 MSP430_MCU("msp430f2274")
+MSP430_MCU("msp430f412")
+MSP430_MCU("msp430f413")
+MSP430_MCU("msp430f415")
+MSP430_MCU("msp430f417")
+MSP430_MCU("msp430f4132")
+MSP430_MCU("msp430f4152")
+MSP430_MCU("msp430f435")
+MSP430_MCU("msp430f436")
+MSP430_MCU("msp430f437")
+MSP430_MCU("msp430f4351")
+MSP430_MCU("msp430f4361")
+MSP430_MCU("msp430f4371")
+MSP430_MCU("msp430fe423")
+MSP430_MCU("msp430fe425")
+MSP430_MCU("msp430fe427")
+MSP430_MCU("msp430fe423a")
+MSP430_MCU("msp430fe425a")
+MSP430_MCU("msp430fe427a")
+MSP430_MCU("msp430fe4232")
+MSP430_MCU("msp430fe4242")
+MSP430_MCU("msp430fe4252")
+MSP430_MCU("msp430fe4272")
+MSP430_MCU("msp430f4250")
+MSP430_MCU("msp430f4260")
+MSP430_MCU("msp430f4270")
+MSP430_MCU("msp430fg4250")
+MSP430_MCU("msp430fg4260")
+MSP430_MCU("msp430fg4270")
+MSP430_MCU("msp430fw423")
+MSP430_MCU("msp430fw425")
+MSP430_MCU("msp430fw427")
+MSP430_MCU("msp430fw428")
+MSP430_MCU("msp430fw429")
+MSP430_MCU("msp430fg437")
+MSP430_MCU("msp430fg438")
+MSP430_MCU("msp430fg439")
+MSP430_MCU("msp430f438")
+MSP430_MCU("msp430f439")
+MSP430_MCU("msp430f477")
+MSP430_MCU("msp430f478")
+MSP430_MCU("msp430f479")
+MSP430_MCU("msp430fg477")
+MSP430_MCU("msp430fg478")
+MSP430_MCU("msp430fg479")
+MSP430_MCU("msp430p112")
+MSP430_MCU("msp430p313")
+MSP430_MCU("msp430p315")
+MSP430_MCU("msp430p315s")
+MSP430_MCU("msp430p325")
+MSP430_MCU("msp430l092")
+MSP430_MCU("msp430c091")
+MSP430_MCU("msp430c092")
 MSP430_MCU("msp430g2211")
 MSP430_MCU("msp430g2201")
 MSP430_MCU("msp430g2111")
@@ -115,68 +182,32 @@ MSP430_MCU("msp430g2855")
 MSP430_MCU("msp430g2955")
 MSP430_MCU("msp430g2230")
 MSP430_MCU("msp430g2210")
-MSP430_MCU("msp430c311s")
-MSP430_MCU("msp430c312")
-MSP430_MCU("msp430c313")
-MSP430_MCU("msp430c314")
-MSP430_MCU("msp430c315")
-MSP430_MCU("msp430c323")
-MSP430_MCU("msp430c325")
-MSP430_MCU("msp430c412")
-MSP430_MCU("msp430c413")
-MSP430_MCU("msp430e313")
-MSP430_MCU("msp430e315")
-MSP430_MCU("msp430e325")
-MSP430_MCU("msp430p313")
-MSP430_MCU("msp430p315")
-MSP430_MCU("msp430p315s")
-MSP430_MCU("msp430p325")
-MSP430_MCU("msp430f412")
-MSP430_MCU("msp430f413")
-MSP430_MCU("msp430f415")
-MSP430_MCU("msp430f417")
-MSP430_MCU("msp430f4132")
-MSP430_MCU("msp430f4152")
-MSP430_MCU("msp430f435")
-MSP430_MCU("msp430f436")
-MSP430_MCU("msp430f437")
-MSP430_MCU("msp430f4351")
-MSP430_MCU("msp430f4361")
-MSP430_MCU("msp430f4371")
-MSP430_MCU("msp430fe423")
-MSP430_MCU("msp430fe425")
-MSP430_MCU("msp430fe427")
-MSP430_MCU("msp430fe423a")
-MSP430_MCU("msp430fe425a")
-MSP430_MCU("msp430fe427a")
-MSP430_MCU("msp430fe4232")
-MSP430_MCU("msp430fe4242")
-MSP430_MCU("msp430fe4252")
-MSP430_MCU("msp430fe4272")
-MSP430_MCU("msp430f4250")
-MSP430_MCU("msp430f4260")
-MSP430_MCU("msp430f4270")
-MSP430_MCU("msp430fg4250")
-MSP430_MCU("msp430fg4260")
-MSP430_MCU("msp430fg4270")
-MSP430_MCU("msp430fw423")
-MSP430_MCU("msp430fw425")
-MSP430_MCU("msp430fw427")
-MSP430_MCU("msp430fw428")
-MSP430_MCU("msp430fw429")
-MSP430_MCU("msp430fg437")
-MSP430_MCU("msp430fg438")
-MSP430_MCU("msp430fg439")
-MSP430_MCU("msp430f438")
-MSP430_MCU("msp430f439")
-MSP430_MCU("msp430f477")
-MSP430_MCU("msp430f478")
-MSP430_MCU("msp430f479")
-MSP430_MCU("msp430fg477")
-MSP430_MCU("msp430fg478")
-MSP430_MCU("msp430fg479")
+MSP430_MCU("rf430frl152h")
+MSP430_MCU("rf430frl153h")
+MSP430_MCU("rf430frl154h")
+MSP430_MCU("rf430frl152h_rom")
+MSP430_MCU("rf430frl153h_rom")
+MSP430_MCU("rf430frl154h_rom")
+MSP430_MCU("msp430fr4131")
+MSP430_MCU("msp430fr4132")
+MSP430_MCU("msp430fr4133")
+MSP430_MCU("msp430fr2032")
+MSP430_MCU("msp430fr2033")
+MSP430_MCU("msp430fr2110")
+MSP430_MCU("msp430fr2111")
+MSP430_MCU("msp430fr2310")
+MSP430_MCU("msp430fr2311")
+MSP430_MCU("msp430fr2100")
+MSP430_MCU("msp430fr2000")
 
 // With 16-bit hardware multiplier
+MSP430_MCU_FEAT("msp430c336", "16bit")
+MSP430_MCU_FEAT("msp430c337", "16bit")
+MSP430_MCU_FEAT("msp430cg4616", "16bit")
+MSP430_MCU_FEAT("msp430cg4617", "16bit")
+MSP430_MCU_FEAT("msp430cg4618", "16bit")
+MSP430_MCU_FEAT("msp430cg4619", "16bit")
+MSP430_MCU_FEAT("msp430e337", "16bit")
 MSP430_MCU_FEAT("msp430f147", "16bit")
 MSP430_MCU_FEAT("msp430f148", "16bit")
 MSP430_MCU_FEAT("msp430f149", "16bit")
@@ -189,21 +220,6 @@ MSP430_MCU_FEAT("msp430f169", "16bit")
 MSP430_MCU_FEAT("msp430f1610", "16bit")
 MSP430_MCU_FEAT("msp430f1611", "16bit")
 MSP430_MCU_FEAT("msp430f1612", "16bit")
-MSP430_MCU_FEAT("msp430c336", "16bit")
-MSP430_MCU_FEAT("msp430c337", "16bit")
-MSP430_MCU_FEAT("msp430e337", "16bit")
-MSP430_MCU_FEAT("msp430p337", "16bit")
-MSP430_MCU_FEAT("msp430f423", "16bit")
-MSP430_MCU_FEAT("msp430f425", "16bit")
-MSP430_MCU_FEAT("msp430f427", "16bit")
-MSP430_MCU_FEAT("msp430f423a", "16bit")
-MSP430_MCU_FEAT("msp430f425a", "16bit")
-MSP430_MCU_FEAT("msp430f427a", "16bit")
-MSP430_MCU_FEAT("msp430f4481", "16bit")
-MSP430_MCU_FEAT("msp430f4491", "16bit")
-MSP430_MCU_FEAT("msp430f447", "16bit")
-MSP430_MCU_FEAT("msp430f448", "16bit")
-MSP430_MCU_FEAT("msp430f449", "16bit")
 MSP430_MCU_FEAT("msp430f2330", "16bit")
 MSP430_MCU_FEAT("msp430f2350", "16bit")
 MSP430_MCU_FEAT("msp430f2370", "16bit")
@@ -216,12 +232,38 @@ MSP430_MCU_FEAT("msp430f2410", "16bit")
 MSP430_MCU_FEAT("msp430f2471", "16bit")
 MSP430_MCU_FEAT("msp430f2481", "16bit")
 MSP430_MCU_FEAT("msp430f2491", "16bit")
-MSP430_MCU_FEAT("msp430i2020", "16bit")
-MSP430_MCU_FEAT("msp430i2021", "16bit")
-MSP430_MCU_FEAT("msp430i2030", "16bit")
-MSP430_MCU_FEAT("msp430i2031", "16bit")
-MSP430_MCU_FEAT("msp430i2040", "16bit")
-MSP430_MCU_FEAT("msp430i2041", "16bit")
+MSP430_MCU_FEAT("msp430f2416", "16bit")
+MSP430_MCU_FEAT("msp430f2417", "16bit")
+MSP430_MCU_FEAT("msp430f2418", "16bit")
+MSP430_MCU_FEAT("msp430f2419", "16bit")
+MSP430_MCU_FEAT("msp430f2616", "16bit")
+MSP430_MCU_FEAT("msp430f2617", "16bit")
+MSP430_MCU_FEAT("msp430f2618", "16bit")
+MSP430_MCU_FEAT("msp430f2619", "16bit")
+MSP430_MCU_FEAT("msp430f423", "16bit")
+MSP430_MCU_FEAT("msp430f425", "16bit")
+MSP430_MCU_FEAT("msp430f427", "16bit")
+MSP430_MCU_FEAT("msp430f423a", "16bit")
+MSP430_MCU_FEAT("msp430f425a", "16bit")
+MSP430_MCU_FEAT("msp430f427a", "16bit")
+MSP430_MCU_FEAT("msp430f4481", "16bit")
+MSP430_MCU_FEAT("msp430f4491", "16bit")
+MSP430_MCU_FEAT("msp430f447", "16bit")
+MSP430_MCU_FEAT("msp430f448", "16bit")
+MSP430_MCU_FEAT("msp430f449", "16bit")
+MSP430_MCU_FEAT("msp430f46161", "16bit")
+MSP430_MCU_FEAT("msp430f46171", "16bit")
+MSP430_MCU_FEAT("msp430f46181", "16bit")
+MSP430_MCU_FEAT("msp430f46191", "16bit")
+MSP430_MCU_FEAT("msp430f4616", "16bit")
+MSP430_MCU_FEAT("msp430f4617", "16bit")
+MSP430_MCU_FEAT("msp430f4618", "16bit")
+MSP430_MCU_FEAT("msp430f4619", "16bit")
+MSP430_MCU_FEAT("msp430fg4616", "16bit")
+MSP430_MCU_FEAT("msp430fg4617", "16bit")
+MSP430_MCU_FEAT("msp430fg4618", "16bit")
+MSP430_MCU_FEAT("msp430fg4619", "16bit")
+MSP430_MCU_FEAT("msp430p337", "16bit")
 MSP430_MCU_FEAT("msp430afe221", "16bit")
 MSP430_MCU_FEAT("msp430afe231", "16bit")
 MSP430_MCU_FEAT("msp430afe251", "16bit")
@@ -231,12 +273,387 @@ MSP430_MCU_FEAT("msp430afe252", "16bit")
 MSP430_MCU_FEAT("msp430afe223", "16bit")
 MSP430_MCU_FEAT("msp430afe233", "16bit")
 MSP430_MCU_FEAT("msp430afe253", "16bit")
+MSP430_MCU_FEAT("msp430i2020", "16bit")
+MSP430_MCU_FEAT("msp430i2021", "16bit")
+MSP430_MCU_FEAT("msp430i2030", "16bit")
+MSP430_MCU_FEAT("msp430i2031", "16bit")
+MSP430_MCU_FEAT("msp430i2040", "16bit")
+MSP430_MCU_FEAT("msp430i2041", "16bit")
 
-// With 32 Bit Hardware Multiplier
+// With 32-bit hardware multiplier
 MSP430_MCU_FEAT("msp430f4783", "32bit")
 MSP430_MCU_FEAT("msp430f4793", "32bit")
 MSP430_MCU_FEAT("msp430f4784", "32bit")
 MSP430_MCU_FEAT("msp430f4794", "32bit")
+MSP430_MCU_FEAT("msp430f47126", "32bit")
+MSP430_MCU_FEAT("msp430f47127", "32bit")
+MSP430_MCU_FEAT("msp430f47163", "32bit")
+MSP430_MCU_FEAT("msp430f47173", "32bit")
+MSP430_MCU_FEAT("msp430f47183", "32bit")
+MSP430_MCU_FEAT("msp430f47193", "32bit")
+MSP430_MCU_FEAT("msp430f47166", "32bit")
+MSP430_MCU_FEAT("msp430f47176", "32bit")
+MSP430_MCU_FEAT("msp430f47186", "32bit")
+MSP430_MCU_FEAT("msp430f47196", "32bit")
+MSP430_MCU_FEAT("msp430f47167", "32bit")
+MSP430_MCU_FEAT("msp430f47177", "32bit")
+MSP430_MCU_FEAT("msp430f47187", "32bit")
+MSP430_MCU_FEAT("msp430f47197", "32bit")
+MSP430_MCU_FEAT("msp430f5418", "32bit")
+MSP430_MCU_FEAT("msp430f5419", "32bit")
+MSP430_MCU_FEAT("msp430f5435", "32bit")
+MSP430_MCU_FEAT("msp430f5436", "32bit")
+MSP430_MCU_FEAT("msp430f5437", "32bit")
+MSP430_MCU_FEAT("msp430f5438", "32bit")
+MSP430_MCU_FEAT("msp430f5418a", "32bit")
+MSP430_MCU_FEAT("msp430f5419a", "32bit")
+MSP430_MCU_FEAT("msp430f5435a", "32bit")
+MSP430_MCU_FEAT("msp430f5436a", "32bit")
+MSP430_MCU_FEAT("msp430f5437a", "32bit")
+MSP430_MCU_FEAT("msp430f5438a", "32bit")
+MSP430_MCU_FEAT("msp430f5212", "32bit")
+MSP430_MCU_FEAT("msp430f5213", "32bit")
+MSP430_MCU_FEAT("msp430f5214", "32bit")
+MSP430_MCU_FEAT("msp430f5217", "32bit")
+MSP430_MCU_FEAT("msp430f5218", "32bit")
+MSP430_MCU_FEAT("msp430f5219", "32bit")
+MSP430_MCU_FEAT("msp430f5222", "32bit")
+MSP430_MCU_FEAT("msp430f5223", "32bit")
+MSP430_MCU_FEAT("msp430f5224", "32bit")
+MSP430_MCU_FEAT("msp430f5227", "32bit")
+MSP430_MCU_FEAT("msp430f5228", "32bit")
+MSP430_MCU_FEAT("msp430f5229", "32bit")
+MSP430_MCU_FEAT("msp430f5232", "32bit")
+MSP430_MCU_FEAT("msp430f5234", "32bit")
+MSP430_MCU_FEAT("msp430f5237", "32bit")
+MSP430_MCU_FEAT("msp430f5239", "32bit")
+MSP430_MCU_FEAT("msp430f5242", "32bit")
+MSP430_MCU_FEAT("msp430f5244", "32bit")
+MSP430_MCU_FEAT("msp430f5247", "32bit")
+MSP430_MCU_FEAT("msp430f5249", "32bit")
+MSP430_MCU_FEAT("msp430f5304", "32bit")
+MSP430_MCU_FEAT("msp430f5308", "32bit")
+MSP430_MCU_FEAT("msp430f5309", "32bit")
+MSP430_MCU_FEAT("msp430f5310", "32bit")
+MSP430_MCU_FEAT("msp430f5340", "32bit")
+MSP430_MCU_FEAT("msp430f5341", "32bit")
+MSP430_MCU_FEAT("msp430f5342", "32bit")
+MSP430_MCU_FEAT("msp430f5324", "32bit")
+MSP430_MCU_FEAT("msp430f5325", "32bit")
+MSP430_MCU_FEAT("msp430f5326", "32bit")
+MSP430_MCU_FEAT("msp430f5327", "32bit")
+MSP430_MCU_FEAT("msp430f5328", "32bit")
+MSP430_MCU_FEAT("msp430f5329", "32bit")
+MSP430_MCU_FEAT("msp430f5500", "32bit")
+MSP430_MCU_FEAT("msp430f5501", "32bit")
+MSP430_MCU_FEAT("msp430f5502", "32bit")
+MSP430_MCU_FEAT("msp430f5503", "32bit")
+MSP430_MCU_FEAT("msp430f5504", "32bit")
+MSP430_MCU_FEAT("msp430f5505", "32bit")
+MSP430_MCU_FEAT("msp430f5506", "32bit")
+MSP430_MCU_FEAT("msp430f5507", "32bit")
+MSP430_MCU_FEAT("msp430f5508", "32bit")
+MSP430_MCU_FEAT("msp430f5509", "32bit")
+MSP430_MCU_FEAT("msp430f5510", "32bit")
+MSP430_MCU_FEAT("msp430f5513", "32bit")
+MSP430_MCU_FEAT("msp430f5514", "32bit")
+MSP430_MCU_FEAT("msp430f5515", "32bit")
+MSP430_MCU_FEAT("msp430f5517", "32bit")
+MSP430_MCU_FEAT("msp430f5519", "32bit")
+MSP430_MCU_FEAT("msp430f5521", "32bit")
+MSP430_MCU_FEAT("msp430f5522", "32bit")
+MSP430_MCU_FEAT("msp430f5524", "32bit")
+MSP430_MCU_FEAT("msp430f5525", "32bit")
+MSP430_MCU_FEAT("msp430f5526", "32bit")
+MSP430_MCU_FEAT("msp430f5527", "32bit")
+MSP430_MCU_FEAT("msp430f5528", "32bit")
+MSP430_MCU_FEAT("msp430f5529", "32bit")
+MSP430_MCU_FEAT("cc430f5133", "32bit")
+MSP430_MCU_FEAT("cc430f5135", "32bit")
+MSP430_MCU_FEAT("cc430f5137", "32bit")
+MSP430_MCU_FEAT("cc430f6125", "32bit")
+MSP430_MCU_FEAT("cc430f6126", "32bit")
+MSP430_MCU_FEAT("cc430f6127", "32bit")
+MSP430_MCU_FEAT("cc430f6135", "32bit")
+MSP430_MCU_FEAT("cc430f6137", "32bit")
+MSP430_MCU_FEAT("cc430f5123", "32bit")
+MSP430_MCU_FEAT("cc430f5125", "32bit")
+MSP430_MCU_FEAT("cc430f5143", "32bit")
+MSP430_MCU_FEAT("cc430f5145", "32bit")
+MSP430_MCU_FEAT("cc430f5147", "32bit")
+MSP430_MCU_FEAT("cc430f6143", "32bit")
+MSP430_MCU_FEAT("cc430f6145", "32bit")
+MSP430_MCU_FEAT("cc430f6147", "32bit")
+MSP430_MCU_FEAT("msp430f5333", "32bit")
+MSP430_MCU_FEAT("msp430f5335", "32bit")
+MSP430_MCU_FEAT("msp430f5336", "32bit")
+MSP430_MCU_FEAT("msp430f5338", "32bit")
+MSP430_MCU_FEAT("msp430f5630", "32bit")
+MSP430_MCU_FEAT("msp430f5631", "32bit")
+MSP430_MCU_FEAT("msp430f5632", "32bit")
+MSP430_MCU_FEAT("msp430f5633", "32bit")
+MSP430_MCU_FEAT("msp430f5634", "32bit")
+MSP430_MCU_FEAT("msp430f5635", "32bit")
+MSP430_MCU_FEAT("msp430f5636", "32bit")
+MSP430_MCU_FEAT("msp430f5637", "32bit")
+MSP430_MCU_FEAT("msp430f5638", "32bit")
+MSP430_MCU_FEAT("msp430f6433", "32bit")
+MSP430_MCU_FEAT("msp430f6435", "32bit")
+MSP430_MCU_FEAT("msp430f6436", "32bit")
+MSP430_MCU_FEAT("msp430f6438", "32bit")
+MSP430_MCU_FEAT("msp430f6630", "32bit")
+MSP430_MCU_FEAT("msp430f6631", "32bit")
+MSP430_MCU_FEAT("msp430f6632", "32bit")
+MSP430_MCU_FEAT("msp430f6633", "32bit")
+MSP430_MCU_FEAT("msp430f6634", "32bit")
+MSP430_MCU_FEAT("msp430f6635", "32bit")
+MSP430_MCU_FEAT("msp430f6636", "32bit")
+MSP430_MCU_FEAT("msp430f6637", "32bit")
+MSP430_MCU_FEAT("msp430f6638", "32bit")
+MSP430_MCU_FEAT("msp430f5358", "32bit")
+MSP430_MCU_FEAT("msp430f5359", "32bit")
+MSP430_MCU_FEAT("msp430f5658", "32bit")
+MSP430_MCU_FEAT("msp430f5659", "32bit")
+MSP430_MCU_FEAT("msp430f6458", "32bit")
+MSP430_MCU_FEAT("msp430f6459", "32bit")
+MSP430_MCU_FEAT("msp430f6658", "32bit")
+MSP430_MCU_FEAT("msp430f6659", "32bit")
+MSP430_MCU_FEAT("msp430fg6425", "32bit")
+MSP430_MCU_FEAT("msp430fg6426", "32bit")
+MSP430_MCU_FEAT("msp430fg6625", "32bit")
+MSP430_MCU_FEAT("msp430fg6626", "32bit")
+MSP430_MCU_FEAT("msp430f5131", "32bit")
+MSP430_MCU_FEAT("msp430f5151", "32bit")
+MSP430_MCU_FEAT("msp430f5171", "32bit")
+MSP430_MCU_FEAT("msp430f5132", "32bit")
+MSP430_MCU_FEAT("msp430f5152", "32bit")
+MSP430_MCU_FEAT("msp430f5172", "32bit")
+MSP430_MCU_FEAT("msp430f6720", "32bit")
+MSP430_MCU_FEAT("msp430f6721", "32bit")
+MSP430_MCU_FEAT("msp430f6723", "32bit")
+MSP430_MCU_FEAT("msp430f6724", "32bit")
+MSP430_MCU_FEAT("msp430f6725", "32bit")
+MSP430_MCU_FEAT("msp430f6726", "32bit")
+MSP430_MCU_FEAT("msp430f6730", "32bit")
+MSP430_MCU_FEAT("msp430f6731", "32bit")
+MSP430_MCU_FEAT("msp430f6733", "32bit")
+MSP430_MCU_FEAT("msp430f6734", "32bit")
+MSP430_MCU_FEAT("msp430f6735", "32bit")
+MSP430_MCU_FEAT("msp430f6736", "32bit")
+MSP430_MCU_FEAT("msp430f67621", "32bit")
+MSP430_MCU_FEAT("msp430f67641", "32bit")
+MSP430_MCU_FEAT("msp430f6720a", "32bit")
+MSP430_MCU_FEAT("msp430f6721a", "32bit")
+MSP430_MCU_FEAT("msp430f6723a", "32bit")
+MSP430_MCU_FEAT("msp430f6724a", "32bit")
+MSP430_MCU_FEAT("msp430f6725a", "32bit")
+MSP430_MCU_FEAT("msp430f6726a", "32bit")
+MSP430_MCU_FEAT("msp430f6730a", "32bit")
+MSP430_MCU_FEAT("msp430f6731a", "32bit")
+MSP430_MCU_FEAT("msp430f6733a", "32bit")
+MSP430_MCU_FEAT("msp430f6734a", "32bit")
+MSP430_MCU_FEAT("msp430f6735a", "32bit")
+MSP430_MCU_FEAT("msp430f6736a", "32bit")
+MSP430_MCU_FEAT("msp430f67621a", "32bit")
+MSP430_MCU_FEAT("msp430f67641a", "32bit")
+MSP430_MCU_FEAT("msp430f67451", "32bit")
+MSP430_MCU_FEAT("msp430f67651", "32bit")
+MSP430_MCU_FEAT("msp430f67751", "32bit")
+MSP430_MCU_FEAT("msp430f67461", "32bit")
+MSP430_MCU_FEAT("msp430f67661", "32bit")
+MSP430_MCU_FEAT("msp430f67761", "32bit")
+MSP430_MCU_FEAT("msp430f67471", "32bit")
+MSP430_MCU_FEAT("msp430f67671", "32bit")
+MSP430_MCU_FEAT("msp430f67771", "32bit")
+MSP430_MCU_FEAT("msp430f67481", "32bit")
+MSP430_MCU_FEAT("msp430f67681", "32bit")
+MSP430_MCU_FEAT("msp430f67781", "32bit")
+MSP430_MCU_FEAT("msp430f67491", "32bit")
+MSP430_MCU_FEAT("msp430f67691", "32bit")
+MSP430_MCU_FEAT("msp430f67791", "32bit")
+MSP430_MCU_FEAT("msp430f6745", "32bit")
+MSP430_MCU_FEAT("msp430f6765", "32bit")
+MSP430_MCU_FEAT("msp430f6775", "32bit")
+MSP430_MCU_FEAT("msp430f6746", "32bit")
+MSP430_MCU_FEAT("msp430f6766", "32bit")
+MSP430_MCU_FEAT("msp430f6776", "32bit")
+MSP430_MCU_FEAT("msp430f6747", "32bit")
+MSP430_MCU_FEAT("msp430f6767", "32bit")
+MSP430_MCU_FEAT("msp430f6777", "32bit")
+MSP430_MCU_FEAT("msp430f6748", "32bit")
+MSP430_MCU_FEAT("msp430f6768", "32bit")
+MSP430_MCU_FEAT("msp430f6778", "32bit")
+MSP430_MCU_FEAT("msp430f6749", "32bit")
+MSP430_MCU_FEAT("msp430f6769", "32bit")
+MSP430_MCU_FEAT("msp430f6779", "32bit")
+MSP430_MCU_FEAT("msp430f67451a", "32bit")
+MSP430_MCU_FEAT("msp430f67651a", "32bit")
+MSP430_MCU_FEAT("msp430f67751a", "32bit")
+MSP430_MCU_FEAT("msp430f67461a", "32bit")
+MSP430_MCU_FEAT("msp430f67661a", "32bit")
+MSP430_MCU_FEAT("msp430f67761a", "32bit")
+MSP430_MCU_FEAT("msp430f67471a", "32bit")
+MSP430_MCU_FEAT("msp430f67671a", "32bit")
+MSP430_MCU_FEAT("msp430f67771a", "32bit")
+MSP430_MCU_FEAT("msp430f67481a", "32bit")
+MSP430_MCU_FEAT("msp430f67681a", "32bit")
+MSP430_MCU_FEAT("msp430f67781a", "32bit")
+MSP430_MCU_FEAT("msp430f67491a", "32bit")
+MSP430_MCU_FEAT("msp430f67691a", "32bit")
+MSP430_MCU_FEAT("msp430f67791a", "32bit")
+MSP430_MCU_FEAT("msp430f6745a", "32bit")
+MSP430_MCU_FEAT("msp430f6765a", "32bit")
+MSP430_MCU_FEAT("msp430f6775a", "32bit")
+MSP430_MCU_FEAT("msp430f6746a", "32bit")
+MSP430_MCU_FEAT("msp430f6766a", "32bit")
+MSP430_MCU_FEAT("msp430f6776a", "32bit")
+MSP430_MCU_FEAT("msp430f6747a", "32bit")
+MSP430_MCU_FEAT("msp430f6767a", "32bit")
+MSP430_MCU_FEAT("msp430f6777a", "32bit")
+MSP430_MCU_FEAT("msp430f6748a", "32bit")
+MSP430_MCU_FEAT("msp430f6768a", "32bit")
+MSP430_MCU_FEAT("msp430f6778a", "32bit")
+MSP430_MCU_FEAT("msp430f6749a", "32bit")
+MSP430_MCU_FEAT("msp430f6769a", "32bit")
+MSP430_MCU_FEAT("msp430f6779a", "32bit")
+MSP430_MCU_FEAT("msp430fr5720", "32bit")
+MSP430_MCU_FEAT("msp430fr5721", "32bit")
+MSP430_MCU_FEAT("msp430fr5722", "32bit")
+MSP430_MCU_FEAT("msp430fr5723", "32bit")
+MSP430_MCU_FEAT("msp430fr5724", "32bit")
+MSP430_MCU_FEAT("msp430fr5725", "32bit")
+MSP430_MCU_FEAT("msp430fr5726", "32bit")
+MSP430_MCU_FEAT("msp430fr5727", "32bit")
+MSP430_MCU_FEAT("msp430fr5728", "32bit")
+MSP430_MCU_FEAT("msp430fr5729", "32bit")
+MSP430_MCU_FEAT("msp430fr5730", "32bit")
+MSP430_MCU_FEAT("msp430fr5731", "32bit")
+MSP430_MCU_FEAT("msp430fr5732", "32bit")
+MSP430_MCU_FEAT("msp430fr5733", "32bit")
+MSP430_MCU_FEAT("msp430fr5734", "32bit")
+MSP430_MCU_FEAT("msp430fr5735", "32bit")
+MSP430_MCU_FEAT("msp430fr5736", "32bit")
+MSP430_MCU_FEAT("msp430fr5737", "32bit")
+MSP430_MCU_FEAT("msp430fr5738", "32bit")
+MSP430_MCU_FEAT("msp430fr5739", "32bit")
+MSP430_MCU_FEAT("msp430bt5190", "32bit")
+MSP430_MCU_FEAT("msp430fr5857", "32bit")
+MSP430_MCU_FEAT("msp430fr5858", "32bit")
+MSP430_MCU_FEAT("msp430fr5859", "32bit")
+MSP430_MCU_FEAT("msp430fr5847", "32bit")
+MSP430_MCU_FEAT("msp430fr58471", "32bit")
+MSP430_MCU_FEAT("msp430fr5848", "32bit")
+MSP430_MCU_FEAT("msp430fr5849", "32bit")
+MSP430_MCU_FEAT("msp430fr5867", "32bit")
+MSP430_MCU_FEAT("msp430fr58671", "32bit")
+MSP430_MCU_FEAT("msp430fr5868", "32bit")
+MSP430_MCU_FEAT("msp430fr5869", "32bit")
+MSP430_MCU_FEAT("msp430fr5957", "32bit")
+MSP430_MCU_FEAT("msp430fr5958", "32bit")
+MSP430_MCU_FEAT("msp430fr5959", "32bit")
+MSP430_MCU_FEAT("msp430fr5947", "32bit")
+MSP430_MCU_FEAT("msp430fr59471", "32bit")
+MSP430_MCU_FEAT("msp430fr5948", "32bit")
+MSP430_MCU_FEAT("msp430fr5949", "32bit")
+MSP430_MCU_FEAT("msp430fr5967", "32bit")
+MSP430_MCU_FEAT("msp430fr5968", "32bit")
+MSP430_MCU_FEAT("msp430fr5969", "32bit")
+MSP430_MCU_FEAT("msp430fr59691", "32bit")
+MSP430_MCU_FEAT("rf430f5175", "32bit")
+MSP430_MCU_FEAT("rf430f5155", "32bit")
+MSP430_MCU_FEAT("rf430f5144", "32bit")
+MSP430_MCU_FEAT("msp430fr69271", "32bit")
+MSP430_MCU_FEAT("msp430fr68791", "32bit")
+MSP430_MCU_FEAT("msp430fr69791", "32bit")
+MSP430_MCU_FEAT("msp430fr6927", "32bit")
+MSP430_MCU_FEAT("msp430fr6928", "32bit")
+MSP430_MCU_FEAT("msp430fr6877", "32bit")
+MSP430_MCU_FEAT("msp430fr6977", "32bit")
+MSP430_MCU_FEAT("msp430fr6879", "32bit")
+MSP430_MCU_FEAT("msp430fr6979", "32bit")
+MSP430_MCU_FEAT("msp430fr58891", "32bit")
+MSP430_MCU_FEAT("msp430fr68891", "32bit")
+MSP430_MCU_FEAT("msp430fr59891", "32bit")
+MSP430_MCU_FEAT("msp430fr69891", "32bit")
+MSP430_MCU_FEAT("msp430fr5887", "32bit")
+MSP430_MCU_FEAT("msp430fr5888", "32bit")
+MSP430_MCU_FEAT("msp430fr5889", "32bit")
+MSP430_MCU_FEAT("msp430fr6887", "32bit")
+MSP430_MCU_FEAT("msp430fr6888", "32bit")
+MSP430_MCU_FEAT("msp430fr6889", "32bit")
+MSP430_MCU_FEAT("msp430fr5986", "32bit")
+MSP430_MCU_FEAT("msp430fr5987", "32bit")
+MSP430_MCU_FEAT("msp430fr5988", "32bit")
+MSP430_MCU_FEAT("msp430fr5989", "32bit")
+MSP430_MCU_FEAT("msp430fr6987", "32bit")
+MSP430_MCU_FEAT("msp430fr6988", "32bit")
+MSP430_MCU_FEAT("msp430fr6989", "32bit")
+MSP430_MCU_FEAT("msp430fr5922", "32bit")
+MSP430_MCU_FEAT("msp430fr5870", "32bit")
+MSP430_MCU_FEAT("msp430fr5970", "32bit")
+MSP430_MCU_FEAT("msp430fr5872", "32bit")
+MSP430_MCU_FEAT("msp430fr5972", "32bit")
+MSP430_MCU_FEAT("msp430fr6820", "32bit")
+MSP430_MCU_FEAT("msp430fr6920", "32bit")
+MSP430_MCU_FEAT("msp430fr6822", "32bit")
+MSP430_MCU_FEAT("msp430fr6922", "32bit")
+MSP430_MCU_FEAT("msp430fr6870", "32bit")
+MSP430_MCU_FEAT("msp430fr6970", "32bit")
+MSP430_MCU_FEAT("msp430fr6872", "32bit")
+MSP430_MCU_FEAT("msp430fr6972", "32bit")
+MSP430_MCU_FEAT("msp430fr59221", "32bit")
+MSP430_MCU_FEAT("msp430fr58721", "32bit")
+MSP430_MCU_FEAT("msp430fr59721", "32bit")
+MSP430_MCU_FEAT("msp430fr68221", "32bit")
+MSP430_MCU_FEAT("msp430fr69221", "32bit")
+MSP430_MCU_FEAT("msp430fr68721", "32bit")
+MSP430_MCU_FEAT("msp430fr69721", "32bit")
+MSP430_MCU_FEAT("msp430sl5438a", "32bit")
+MSP430_MCU_FEAT("msp430fr2433", "32bit")
+MSP430_MCU_FEAT("msp430fr2532", "32bit")
+MSP430_MCU_FEAT("msp430fr2533", "32bit")
+MSP430_MCU_FEAT("msp430fr2632", "32bit")
+MSP430_MCU_FEAT("msp430fr2633", "32bit")
+MSP430_MCU_FEAT("msp430f5252", "32bit")
+MSP430_MCU_FEAT("msp430f5253", "32bit")
+MSP430_MCU_FEAT("msp430f5254", "32bit")
+MSP430_MCU_FEAT("msp430f5255", "32bit")
+MSP430_MCU_FEAT("msp430f5256", "32bit")
+MSP430_MCU_FEAT("msp430f5257", "32bit")
+MSP430_MCU_FEAT("msp430f5258", "32bit")
+MSP430_MCU_FEAT("msp430f5259", "32bit")
+MSP430_MCU_FEAT("msp430fr5962", "32bit")
+MSP430_MCU_FEAT("msp430fr5964", "32bit")
+MSP430_MCU_FEAT("msp430fr5992", "32bit")
+MSP430_MCU_FEAT("msp430fr5994", "32bit")
+MSP430_MCU_FEAT("msp430fr59941", "32bit")
+MSP430_MCU_FEAT("msp430fr2355", "32bit")
+MSP430_MCU_FEAT("msp430fr2155", "32bit")
+MSP430_MCU_FEAT("msp430fr2353", "32bit")
+MSP430_MCU_FEAT("msp430fr2153", "32bit")
+MSP430_MCU_FEAT("msp430fr2522", "32bit")
+MSP430_MCU_FEAT("msp430fr2512", "32bit")
+MSP430_MCU_FEAT("msp430fr2422", "32bit")
+MSP430_MCU_FEAT("msp430fr2676", "32bit")
+MSP430_MCU_FEAT("msp430fr2476", "32bit")
+MSP430_MCU_FEAT("msp430fr2675", "32bit")
+MSP430_MCU_FEAT("msp430fr2673", "32bit")
+MSP430_MCU_FEAT("msp430fr2475", "32bit")
+MSP430_MCU_FEAT("msp430fr2672", "32bit")
+MSP430_MCU_FEAT("msp430fr6043", "32bit")
+MSP430_MCU_FEAT("msp430fr5043", "32bit")
+MSP430_MCU_FEAT("msp430fr6041", "32bit")
+MSP430_MCU_FEAT("msp430fr60431", "32bit")
+MSP430_MCU_FEAT("msp430fr5041", "32bit")
+MSP430_MCU_FEAT("msp430fr50431", "32bit")
+MSP430_MCU_FEAT("msp430fr6005", "32bit")
+MSP430_MCU_FEAT("msp430fr6047", "32bit")
+MSP430_MCU_FEAT("msp430fr6037", "32bit")
+MSP430_MCU_FEAT("msp430fr6045", "32bit")
+MSP430_MCU_FEAT("msp430fr60471", "32bit")
+MSP430_MCU_FEAT("msp430fr6035", "32bit")
+MSP430_MCU_FEAT("msp430fr6007", "32bit")
+MSP430_MCU_FEAT("msp430fr60371", "32bit")
 
 // Generic MCUs
 MSP430_MCU("msp430i2xxgeneric")
diff --git a/clang/include/clang/Basic/Target/MSP430/gen-msp430-def.py b/clang/include/clang/Basic/Target/MSP430/gen-msp430-def.py
new file mode 100755
index 000000000000..3ae6fdd9d5c6
--- /dev/null
+++ b/clang/include/clang/Basic/Target/MSP430/gen-msp430-def.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+"""
+Script to generate MSP430 definitions from TI's devices.csv
+
+Download the devices.csv from [1] using the link "Header and Support Files".
+
+[1]: https://www.ti.com/tool/MSP430-GCC-OPENSOURCE#downloads
+"""
+import csv
+import sys
+
+DEVICE_COLUMN = 0
+MULTIPLIER_COLUMN = 3
+
+MULTIPLIER_SW = "0"
+MULTIPLIER_HW_16 = ("1", "2")
+MULTIPLIER_HW_32 = ("4", "8")
+
+PREFIX = """//===--- MSP430Target.def - MSP430 Feature/Processor Database----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the MSP430 devices and their features.
+//
+// Generated from TI's devices.csv in version {} using the script in
+// Target/MSP430/gen-msp430-def.py - use this tool rather than adding
+// new MCUs by hand.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MSP430_MCU_FEAT
+#define MSP430_MCU_FEAT(NAME, HWMULT) MSP430_MCU(NAME)
+#endif
+
+#ifndef MSP430_MCU
+#define MSP430_MCU(NAME)
+#endif
+
+"""
+
+SUFFIX = """
+// Generic MCUs
+MSP430_MCU("msp430i2xxgeneric")
+
+#undef MSP430_MCU
+#undef MSP430_MCU_FEAT
+"""
+
+
+def csv2def(csv_path, def_path):
+    """
+    Parse the devices.csv file at the given path, generate the definitions and
+    write them to the given path.
+
+    :param csv_path: Path to the devices.csv to parse
+    :type csv_path: str
+    :param def_path: Path to the output file to write the definitions to
+    "type def_path: str
+    """
+
+    mcus_multiplier_sw = []
+    mcus_multiplier_hw_16 = []
+    mcus_multiplier_hw_32 = []
+    version = "unknown"
+
+    with open(csv_path) as csv_file:
+        csv_reader = csv.reader(csv_file)
+        while True:
+            row = next(csv_reader)
+            if len(row) < MULTIPLIER_COLUMN:
+                continue
+
+            if row[DEVICE_COLUMN] == "# Device Name":
+                assert row[MULTIPLIER_COLUMN] == "MPY_TYPE", "File format changed"
+                break
+
+            if row[0] == "Version:":
+                version = row[1]
+
+        for row in csv_reader:
+            if row[DEVICE_COLUMN].endswith("generic"):
+                continue
+            if row[MULTIPLIER_COLUMN] == MULTIPLIER_SW:
+                mcus_multiplier_sw.append(row[DEVICE_COLUMN])
+            elif row[MULTIPLIER_COLUMN] in MULTIPLIER_HW_16:
+                mcus_multiplier_hw_16.append(row[DEVICE_COLUMN])
+            elif row[MULTIPLIER_COLUMN] in MULTIPLIER_HW_32:
+                mcus_multiplier_hw_32.append(row[DEVICE_COLUMN])
+            else:
+                assert 0, "Unknown multiplier type"
+
+    with open(def_path, "w") as def_file:
+        def_file.write(PREFIX.format(version))
+
+        for mcu in mcus_multiplier_sw:
+            def_file.write(f'MSP430_MCU("{mcu}")\n')
+
+        def_file.write("\n// With 16-bit hardware multiplier\n")
+
+        for mcu in mcus_multiplier_hw_16:
+            def_file.write(f'MSP430_MCU_FEAT("{mcu}", "16bit")\n')
+
+        def_file.write("\n// With 32-bit hardware multiplier\n")
+
+        for mcu in mcus_multiplier_hw_32:
+            def_file.write(f'MSP430_MCU_FEAT("{mcu}", "32bit")\n')
+
+        def_file.write(SUFFIX)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        sys.exit(f"Usage: {sys.argv[0]} <CSV_FILE> <DEF_FILE>")
+
+    csv2def(sys.argv[1], sys.argv[2])
-- 
GitLab


From 2a3903fa0e88d7149df11aa37d4ba87c5e5f0913 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Tue, 7 May 2024 14:20:13 -0700
Subject: [PATCH 0107/1206] [AMDGPU] Prevent FMINIMUM and FMAXIMUM beeing fully
 scalarized (#91378)

This is the same logic as with FMINNUM_IEEE/FMAXNUM_IEEE.
---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp    |  8 +++-
 llvm/test/CodeGen/AMDGPU/fmaximum.ll         | 39 ++++++++++++--------
 llvm/test/CodeGen/AMDGPU/fminimum.ll         | 39 ++++++++++++--------
 llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll |  8 ++--
 llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll |  8 ++--
 5 files changed, 63 insertions(+), 39 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index ed41c10b50d3..33bdd6195a04 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -854,9 +854,13 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
   if (Subtarget->hasPrefetch())
     setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
-  if (Subtarget->hasIEEEMinMax())
+  if (Subtarget->hasIEEEMinMax()) {
     setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM},
                        {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Legal);
+    setOperationAction({ISD::FMINIMUM, ISD::FMAXIMUM},
+                       {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
+                       Custom);
+  }
 
   setOperationAction(ISD::INTRINSIC_WO_CHAIN,
                      {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
@@ -5821,6 +5825,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::FMUL:
   case ISD::FMINNUM_IEEE:
   case ISD::FMAXNUM_IEEE:
+  case ISD::FMINIMUM:
+  case ISD::FMAXIMUM:
   case ISD::UADDSAT:
   case ISD::USUBSAT:
   case ISD::SADDSAT:
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index dd685a6169d8..87ac95a1cd73 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -148,23 +148,35 @@ define amdgpu_ps <2 x half> @test_fmaximum_v2f16_ss(<2 x half> inreg %a, <2 x ha
 }
 
 define amdgpu_ps <3 x half> @test_fmaximum_v3f16_vv(<3 x half> %a, <3 x half> %b) {
-; GCN-LABEL: test_fmaximum_v3f16_vv:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_pk_maximum_f16 v0, v0, v2
-; GCN-NEXT:    v_maximum_f16 v1, v1, v3
-; GCN-NEXT:    ; return to shader part epilog
+; GFX12-SDAG-LABEL: test_fmaximum_v3f16_vv:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    v_pk_maximum_f16 v0, v0, v2
+; GFX12-SDAG-NEXT:    v_pk_maximum_f16 v1, v1, v3
+; GFX12-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: test_fmaximum_v3f16_vv:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    v_pk_maximum_f16 v0, v0, v2
+; GFX12-GISEL-NEXT:    v_maximum_f16 v1, v1, v3
+; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %val = call <3 x half> @llvm.maximum.v3f16(<3 x half> %a, <3 x half> %b)
   ret <3 x half> %val
 }
 
 define amdgpu_ps <3 x half> @test_fmaximum_v3f16_ss(<3 x half> inreg %a, <3 x half> inreg %b) {
-; GCN-LABEL: test_fmaximum_v3f16_ss:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_pk_maximum_f16 v0, s0, s2
-; GCN-NEXT:    s_maximum_f16 s0, s1, s3
-; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
-; GCN-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX12-SDAG-LABEL: test_fmaximum_v3f16_ss:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    v_pk_maximum_f16 v0, s0, s2
+; GFX12-SDAG-NEXT:    v_pk_maximum_f16 v1, s1, s3
+; GFX12-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: test_fmaximum_v3f16_ss:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    v_pk_maximum_f16 v0, s0, s2
+; GFX12-GISEL-NEXT:    s_maximum_f16 s0, s1, s3
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, s0
+; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %val = call <3 x half> @llvm.maximum.v3f16(<3 x half> %a, <3 x half> %b)
   ret <3 x half> %val
 }
@@ -306,6 +318,3 @@ declare <4 x half> @llvm.maximum.v4f16(<4 x half>, <4 x half>)
 declare double @llvm.maximum.f64(double, double)
 declare <2 x double> @llvm.maximum.v2f64(<2 x double>, <2 x double>)
 declare <4 x double> @llvm.maximum.v4f64(<4 x double>, <4 x double>)
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX12-GISEL: {{.*}}
-; GFX12-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index 2b3cc4fd7385..45f6bff10f45 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -148,23 +148,35 @@ define amdgpu_ps <2 x half> @test_fminimum_v2f16_ss(<2 x half> inreg %a, <2 x ha
 }
 
 define amdgpu_ps <3 x half> @test_fminimum_v3f16_vv(<3 x half> %a, <3 x half> %b) {
-; GCN-LABEL: test_fminimum_v3f16_vv:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_pk_minimum_f16 v0, v0, v2
-; GCN-NEXT:    v_minimum_f16 v1, v1, v3
-; GCN-NEXT:    ; return to shader part epilog
+; GFX12-SDAG-LABEL: test_fminimum_v3f16_vv:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    v_pk_minimum_f16 v0, v0, v2
+; GFX12-SDAG-NEXT:    v_pk_minimum_f16 v1, v1, v3
+; GFX12-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: test_fminimum_v3f16_vv:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    v_pk_minimum_f16 v0, v0, v2
+; GFX12-GISEL-NEXT:    v_minimum_f16 v1, v1, v3
+; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %val = call <3 x half> @llvm.minimum.v3f16(<3 x half> %a, <3 x half> %b)
   ret <3 x half> %val
 }
 
 define amdgpu_ps <3 x half> @test_fminimum_v3f16_ss(<3 x half> inreg %a, <3 x half> inreg %b) {
-; GCN-LABEL: test_fminimum_v3f16_ss:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_pk_minimum_f16 v0, s0, s2
-; GCN-NEXT:    s_minimum_f16 s0, s1, s3
-; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
-; GCN-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX12-SDAG-LABEL: test_fminimum_v3f16_ss:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    v_pk_minimum_f16 v0, s0, s2
+; GFX12-SDAG-NEXT:    v_pk_minimum_f16 v1, s1, s3
+; GFX12-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: test_fminimum_v3f16_ss:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    v_pk_minimum_f16 v0, s0, s2
+; GFX12-GISEL-NEXT:    s_minimum_f16 s0, s1, s3
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, s0
+; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %val = call <3 x half> @llvm.minimum.v3f16(<3 x half> %a, <3 x half> %b)
   ret <3 x half> %val
 }
@@ -306,6 +318,3 @@ declare <4 x half> @llvm.minimum.v4f16(<4 x half>, <4 x half>)
 declare double @llvm.minimum.f64(double, double)
 declare <2 x double> @llvm.minimum.v2f64(<2 x double>, <2 x double>)
 declare <4 x double> @llvm.minimum.v4f64(<4 x double>, <4 x double>)
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX12-GISEL: {{.*}}
-; GFX12-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
index c49e6a9a9f25..c476208ed8f4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
@@ -1794,7 +1794,7 @@ define <3 x half> @v_maximum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_pk_maximum_f16 v0, v0, v2
-; GFX12-NEXT:    v_maximum_f16 v1, v1, v3
+; GFX12-NEXT:    v_pk_maximum_f16 v1, v1, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %op = call <3 x half> @llvm.maximum.v3f16(<3 x half> %src0, <3 x half> %src1)
   ret <3 x half> %op
@@ -2013,7 +2013,7 @@ define <3 x half> @v_maximum_v3f16__nnan(<3 x half> %src0, <3 x half> %src1) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_pk_maximum_f16 v0, v0, v2
-; GFX12-NEXT:    v_maximum_f16 v1, v1, v3
+; GFX12-NEXT:    v_pk_maximum_f16 v1, v1, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %op = call nnan <3 x half> @llvm.maximum.v3f16(<3 x half> %src0, <3 x half> %src1)
   ret <3 x half> %op
@@ -2163,7 +2163,7 @@ define <3 x half> @v_maximum_v3f16__nsz(<3 x half> %src0, <3 x half> %src1) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_pk_maximum_f16 v0, v0, v2
-; GFX12-NEXT:    v_maximum_f16 v1, v1, v3
+; GFX12-NEXT:    v_pk_maximum_f16 v1, v1, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %op = call nsz <3 x half> @llvm.maximum.v3f16(<3 x half> %src0, <3 x half> %src1)
   ret <3 x half> %op
@@ -2260,7 +2260,7 @@ define <3 x half> @v_maximum_v3f16__nnan_nsz(<3 x half> %src0, <3 x half> %src1)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_pk_maximum_f16 v0, v0, v2
-; GFX12-NEXT:    v_maximum_f16 v1, v1, v3
+; GFX12-NEXT:    v_pk_maximum_f16 v1, v1, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %op = call nnan nsz <3 x half> @llvm.maximum.v3f16(<3 x half> %src0, <3 x half> %src1)
   ret <3 x half> %op
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
index 7281c3fd64d4..66f3a48b13ee 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
@@ -1461,7 +1461,7 @@ define <3 x half> @v_minimum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_pk_minimum_f16 v0, v0, v2
-; GFX12-NEXT:    v_minimum_f16 v1, v1, v3
+; GFX12-NEXT:    v_pk_minimum_f16 v1, v1, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %op = call <3 x half> @llvm.minimum.v3f16(<3 x half> %src0, <3 x half> %src1)
   ret <3 x half> %op
@@ -1635,7 +1635,7 @@ define <3 x half> @v_minimum_v3f16__nnan(<3 x half> %src0, <3 x half> %src1) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_pk_minimum_f16 v0, v0, v2
-; GFX12-NEXT:    v_minimum_f16 v1, v1, v3
+; GFX12-NEXT:    v_pk_minimum_f16 v1, v1, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %op = call nnan <3 x half> @llvm.minimum.v3f16(<3 x half> %src0, <3 x half> %src1)
   ret <3 x half> %op
@@ -1740,7 +1740,7 @@ define <3 x half> @v_minimum_v3f16__nsz(<3 x half> %src0, <3 x half> %src1) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_pk_minimum_f16 v0, v0, v2
-; GFX12-NEXT:    v_minimum_f16 v1, v1, v3
+; GFX12-NEXT:    v_pk_minimum_f16 v1, v1, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %op = call nsz <3 x half> @llvm.minimum.v3f16(<3 x half> %src0, <3 x half> %src1)
   ret <3 x half> %op
@@ -1792,7 +1792,7 @@ define <3 x half> @v_minimum_v3f16__nnan_nsz(<3 x half> %src0, <3 x half> %src1)
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_pk_minimum_f16 v0, v0, v2
-; GFX12-NEXT:    v_minimum_f16 v1, v1, v3
+; GFX12-NEXT:    v_pk_minimum_f16 v1, v1, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %op = call nnan nsz <3 x half> @llvm.minimum.v3f16(<3 x half> %src0, <3 x half> %src1)
   ret <3 x half> %op
-- 
GitLab


From a5044e6d505deb79f1b00bb39d11096d29b9c910 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Tue, 7 May 2024 17:36:58 -0400
Subject: [PATCH 0108/1206] [libc] fix typo due to futex renaming (#91379)

---
 libc/src/__support/threads/linux/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index b277c2a37f2d..9bee30206f1b 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -27,7 +27,7 @@ add_header_library(
   HDRS
     mutex.h
   DEPENDS
-    .futex
+    .futex_utils
     libc.src.__support.threads.mutex_common
 )
 
-- 
GitLab


From ccf765cfd578c4ea4f710386e19cb8d1ef1859ce Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Tue, 7 May 2024 15:01:15 -0700
Subject: [PATCH 0109/1206] [compiler-rt][ctx_profile] Add the instrumented
 contextual profiling APIs (#89838)

APIs for contextual profiling. `ContextNode` is the call context-specific counter buffer. `ContextRoot` is associated to those functions that constitute roots into interesting call graphs, and is the object on which we hang off `Arena`s for allocating `ContextNode`s, as well as the `ContextNode` corresponding to such functions. Graphs of `ContextNode`s are accessible by one thread at a time.

(Tracking Issue: #89287, more details in the RFC referenced there)
---
 .../lib/ctx_profile/CtxInstrProfiling.cpp     | 283 +++++++++++++++++-
 .../lib/ctx_profile/CtxInstrProfiling.h       | 208 +++++++++++++
 .../tests/CtxInstrProfilingTest.cpp           | 192 ++++++++++++
 3 files changed, 681 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
index 7620ce92f7eb..68bfe5c1ae61 100644
--- a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
+++ b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
@@ -10,20 +10,115 @@
 #include "sanitizer_common/sanitizer_allocator_internal.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_dense_map.h"
+#include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_mutex.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
 #include "sanitizer_common/sanitizer_thread_safety.h"
+#include "sanitizer_common/sanitizer_vector.h"
 
 #include <assert.h>
 
 using namespace __ctx_profile;
 
+namespace {
+// Keep track of all the context roots we actually saw, so we can then traverse
+// them when the user asks for the profile in __llvm_ctx_profile_fetch
+__sanitizer::SpinMutex AllContextsMutex;
+SANITIZER_GUARDED_BY(AllContextsMutex)
+__sanitizer::Vector<ContextRoot *> AllContextRoots;
+
+// utility to taint a pointer by setting the LSB. There is an assumption
+// throughout that the addresses of contexts are even (really, they should be
+// align(8), but "even"-ness is the minimum assumption)
+// "scratch contexts" are buffers that we return in certain cases - they are
+// large enough to allow for memory safe counter access, but they don't link
+// subcontexts below them (the runtime recognizes them and enforces that)
+ContextNode *markAsScratch(const ContextNode *Ctx) {
+  return reinterpret_cast<ContextNode *>(reinterpret_cast<uint64_t>(Ctx) | 1);
+}
+
+// Used when getting the data from TLS. We don't *really* need to reset, but
+// it's a simpler system if we do.
+template <typename T> inline T consume(T &V) {
+  auto R = V;
+  V = {0};
+  return R;
+}
+
+// We allocate at least kBuffSize Arena pages. The scratch buffer is also that
+// large.
+constexpr size_t kPower = 20;
+constexpr size_t kBuffSize = 1 << kPower;
+
+// Highly unlikely we need more than kBuffSize for a context.
+size_t getArenaAllocSize(size_t Needed) {
+  if (Needed >= kBuffSize)
+    return 2 * Needed;
+  return kBuffSize;
+}
+
+// verify the structural integrity of the context
+bool validate(const ContextRoot *Root) {
+  // all contexts should be laid out in some arena page. Go over each arena
+  // allocated for this Root, and jump over contained contexts based on
+  // self-reported sizes.
+  __sanitizer::DenseMap<uint64_t, bool> ContextStartAddrs;
+  for (const auto *Mem = Root->FirstMemBlock; Mem; Mem = Mem->next()) {
+    const auto *Pos = Mem->start();
+    while (Pos < Mem->pos()) {
+      const auto *Ctx = reinterpret_cast<const ContextNode *>(Pos);
+      if (!ContextStartAddrs.insert({reinterpret_cast<uint64_t>(Ctx), true})
+               .second)
+        return false;
+      Pos += Ctx->size();
+    }
+  }
+
+  // Now traverse the contexts again the same way, but validate all nonull
+  // subcontext addresses appear in the set computed above.
+  for (const auto *Mem = Root->FirstMemBlock; Mem; Mem = Mem->next()) {
+    const auto *Pos = Mem->start();
+    while (Pos < Mem->pos()) {
+      const auto *Ctx = reinterpret_cast<const ContextNode *>(Pos);
+      for (uint32_t I = 0; I < Ctx->callsites_size(); ++I)
+        for (auto *Sub = Ctx->subContexts()[I]; Sub; Sub = Sub->next())
+          if (!ContextStartAddrs.find(reinterpret_cast<uint64_t>(Sub)))
+            return false;
+
+      Pos += Ctx->size();
+    }
+  }
+  return true;
+}
+} // namespace
+
+// the scratch buffer - what we give when we can't produce a real context (the
+// scratch isn't "real" in that it's expected to be clobbered carelessly - we
+// don't read it). The other important thing is that the callees from a scratch
+// context also get a scratch context.
+// Eventually this can be replaced with per-function buffers, a'la the typical
+// (flat) instrumented FDO buffers. The clobbering aspect won't apply there, but
+// the part about determining the nature of the subcontexts does.
+__thread char __Buffer[kBuffSize] = {0};
+
+#define TheScratchContext                                                      \
+  markAsScratch(reinterpret_cast<ContextNode *>(__Buffer))
+
+// init the TLSes
+__thread void *volatile __llvm_ctx_profile_expected_callee[2] = {nullptr,
+                                                                 nullptr};
+__thread ContextNode **volatile __llvm_ctx_profile_callsite[2] = {0, 0};
+
+__thread ContextRoot *volatile __llvm_ctx_profile_current_context_root =
+    nullptr;
+
 // FIXME(mtrofin): use malloc / mmap instead of sanitizer common APIs to reduce
 // the dependency on the latter.
 Arena *Arena::allocateNewArena(size_t Size, Arena *Prev) {
   assert(!Prev || Prev->Next == nullptr);
-  Arena *NewArena =
-      new (__sanitizer::InternalAlloc(Size + sizeof(Arena))) Arena(Size);
+  Arena *NewArena = new (__sanitizer::InternalAlloc(
+      Size + sizeof(Arena), /*cache=*/nullptr, /*alignment=*/ExpectedAlignment))
+      Arena(Size);
   if (Prev)
     Prev->Next = NewArena;
   return NewArena;
@@ -38,3 +133,187 @@ void Arena::freeArenaList(Arena *&A) {
   }
   A = nullptr;
 }
+
+inline ContextNode *ContextNode::alloc(char *Place, GUID Guid,
+                                       uint32_t NrCounters,
+                                       uint32_t NrCallsites,
+                                       ContextNode *Next) {
+  assert(reinterpret_cast<uint64_t>(Place) % ExpectedAlignment == 0);
+  return new (Place) ContextNode(Guid, NrCounters, NrCallsites, Next);
+}
+
+void ContextNode::reset() {
+  // FIXME(mtrofin): this is std::memset, which we can probably use if we
+  // drop/reduce the dependency on sanitizer_common.
+  for (uint32_t I = 0; I < NrCounters; ++I)
+    counters()[I] = 0;
+  for (uint32_t I = 0; I < NrCallsites; ++I)
+    for (auto *Next = subContexts()[I]; Next; Next = Next->Next)
+      Next->reset();
+}
+
+// If this is the first time we hit a callsite with this (Guid) particular
+// callee, we need to allocate.
+ContextNode *getCallsiteSlow(uint64_t Guid, ContextNode **InsertionPoint,
+                             uint32_t NrCounters, uint32_t NrCallsites) {
+  auto AllocSize = ContextNode::getAllocSize(NrCounters, NrCallsites);
+  auto *Mem = __llvm_ctx_profile_current_context_root->CurrentMem;
+  char *AllocPlace = Mem->tryBumpAllocate(AllocSize);
+  if (!AllocPlace) {
+    // if we failed to allocate on the current arena, allocate a new arena,
+    // and place it on __llvm_ctx_profile_current_context_root->CurrentMem so we
+    // find it from now on for other cases when we need to getCallsiteSlow.
+    // Note that allocateNewArena will link the allocated memory in the list of
+    // Arenas.
+    __llvm_ctx_profile_current_context_root->CurrentMem = Mem =
+        Mem->allocateNewArena(getArenaAllocSize(AllocSize), Mem);
+    AllocPlace = Mem->tryBumpAllocate(AllocSize);
+  }
+  auto *Ret = ContextNode::alloc(AllocPlace, Guid, NrCounters, NrCallsites,
+                                 *InsertionPoint);
+  *InsertionPoint = Ret;
+  return Ret;
+}
+
+ContextNode *__llvm_ctx_profile_get_context(void *Callee, GUID Guid,
+                                            uint32_t NrCounters,
+                                            uint32_t NrCallsites) {
+  // fast "out" if we're not even doing contextual collection.
+  if (!__llvm_ctx_profile_current_context_root)
+    return TheScratchContext;
+
+  // also fast "out" if the caller is scratch. We can see if it's scratch by
+  // looking at the interior pointer into the subcontexts vector that the caller
+  // provided, which, if the context is scratch, so is that interior pointer
+  // (because all the address calculations are using even values. Or more
+  // precisely, aligned - 8 values)
+  auto **CallsiteContext = consume(__llvm_ctx_profile_callsite[0]);
+  if (!CallsiteContext || isScratch(CallsiteContext))
+    return TheScratchContext;
+
+  // if the callee isn't the expected one, return scratch.
+  // Signal handler(s) could have been invoked at any point in the execution.
+  // Should that have happened, and had it (the handler) be built with
+  // instrumentation, its __llvm_ctx_profile_get_context would have failed here.
+  // Its sub call graph would have then populated
+  // __llvm_ctx_profile_{expected_callee | callsite} at index 1.
+  // The normal call graph may be impacted in that, if the signal handler
+  // happened somewhere before we read the TLS here, we'd see the TLS reset and
+  // we'd also fail here. That would just mean we would loose counter values for
+  // the normal subgraph, this time around. That should be very unlikely, but if
+  // it happens too frequently, we should be able to detect discrepancies in
+  // entry counts (caller-callee). At the moment, the design goes on the
+  // assumption that is so unfrequent, though, that it's not worth doing more
+  // for that case.
+  auto *ExpectedCallee = consume(__llvm_ctx_profile_expected_callee[0]);
+  if (ExpectedCallee != Callee)
+    return TheScratchContext;
+
+  auto *Callsite = *CallsiteContext;
+  // in the case of indirect calls, we will have all seen targets forming a
+  // linked list here. Find the one corresponding to this callee.
+  while (Callsite && Callsite->guid() != Guid) {
+    Callsite = Callsite->next();
+  }
+  auto *Ret = Callsite ? Callsite
+                       : getCallsiteSlow(Guid, CallsiteContext, NrCounters,
+                                         NrCallsites);
+  if (Ret->callsites_size() != NrCallsites ||
+      Ret->counters_size() != NrCounters)
+    __sanitizer::Printf("[ctxprof] Returned ctx differs from what's asked: "
+                        "Context: %p, Asked: %lu %u %u, Got: %lu %u %u \n",
+                        Ret, Guid, NrCallsites, NrCounters, Ret->guid(),
+                        Ret->callsites_size(), Ret->counters_size());
+  Ret->onEntry();
+  return Ret;
+}
+
+// This should be called once for a Root. Allocate the first arena, set up the
+// first context.
+void setupContext(ContextRoot *Root, GUID Guid, uint32_t NrCounters,
+                  uint32_t NrCallsites) {
+  __sanitizer::GenericScopedLock<__sanitizer::SpinMutex> Lock(
+      &AllContextsMutex);
+  // Re-check - we got here without having had taken a lock.
+  if (Root->FirstMemBlock)
+    return;
+  const auto Needed = ContextNode::getAllocSize(NrCounters, NrCallsites);
+  auto *M = Arena::allocateNewArena(getArenaAllocSize(Needed));
+  Root->FirstMemBlock = M;
+  Root->CurrentMem = M;
+  Root->FirstNode = ContextNode::alloc(M->tryBumpAllocate(Needed), Guid,
+                                       NrCounters, NrCallsites);
+  AllContextRoots.PushBack(Root);
+}
+
+ContextNode *__llvm_ctx_profile_start_context(
+    ContextRoot *Root, GUID Guid, uint32_t Counters,
+    uint32_t Callsites) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+  if (!Root->FirstMemBlock) {
+    setupContext(Root, Guid, Counters, Callsites);
+  }
+  if (Root->Taken.TryLock()) {
+    __llvm_ctx_profile_current_context_root = Root;
+    Root->FirstNode->onEntry();
+    return Root->FirstNode;
+  }
+  // If this thread couldn't take the lock, return scratch context.
+  __llvm_ctx_profile_current_context_root = nullptr;
+  return TheScratchContext;
+}
+
+void __llvm_ctx_profile_release_context(ContextRoot *Root)
+    SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+  if (__llvm_ctx_profile_current_context_root) {
+    __llvm_ctx_profile_current_context_root = nullptr;
+    Root->Taken.Unlock();
+  }
+}
+
+void __llvm_ctx_profile_start_collection() {
+  size_t NrMemUnits = 0;
+  __sanitizer::GenericScopedLock<__sanitizer::SpinMutex> Lock(
+      &AllContextsMutex);
+  for (uint32_t I = 0; I < AllContextRoots.Size(); ++I) {
+    auto *Root = AllContextRoots[I];
+    __sanitizer::GenericScopedLock<__sanitizer::StaticSpinMutex> Lock(
+        &Root->Taken);
+    for (auto *Mem = Root->FirstMemBlock; Mem; Mem = Mem->next())
+      ++NrMemUnits;
+
+    Root->FirstNode->reset();
+  }
+  __sanitizer::Printf("[ctxprof] Initial NrMemUnits: %zu \n", NrMemUnits);
+}
+
+bool __llvm_ctx_profile_fetch(
+    void *Data, bool (*Writer)(void *W, const __ctx_profile::ContextNode &)) {
+  assert(Writer);
+  __sanitizer::GenericScopedLock<__sanitizer::SpinMutex> Lock(
+      &AllContextsMutex);
+
+  for (int I = 0, E = AllContextRoots.Size(); I < E; ++I) {
+    auto *Root = AllContextRoots[I];
+    __sanitizer::GenericScopedLock<__sanitizer::StaticSpinMutex> TakenLock(
+        &Root->Taken);
+    if (!validate(Root)) {
+      __sanitizer::Printf("[ctxprof] Contextual Profile is %s\n", "invalid");
+      return false;
+    }
+    if (!Writer(Data, *Root->FirstNode))
+      return false;
+  }
+  return true;
+}
+
+void __llvm_ctx_profile_free() {
+  __sanitizer::GenericScopedLock<__sanitizer::SpinMutex> Lock(
+      &AllContextsMutex);
+  for (int I = 0, E = AllContextRoots.Size(); I < E; ++I)
+    for (auto *A = AllContextRoots[I]->FirstMemBlock; A;) {
+      auto *C = A;
+      A = A->next();
+      __sanitizer::InternalFree(C);
+    }
+  AllContextRoots.Reset();
+}
diff --git a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
index c1789c32a64c..8c4be5d8a23a 100644
--- a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
+++ b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
@@ -9,9 +9,16 @@
 #ifndef CTX_PROFILE_CTXINSTRPROFILING_H_
 #define CTX_PROFILE_CTXINSTRPROFILING_H_
 
+#include "sanitizer_common/sanitizer_mutex.h"
 #include <sanitizer/common_interface_defs.h>
 
 namespace __ctx_profile {
+using GUID = uint64_t;
+static constexpr size_t ExpectedAlignment = 8;
+// We really depend on this, see further below. We currently support x86_64.
+// When we want to support other archs, we need to trace the places Alignment is
+// used and adjust accordingly.
+static_assert(sizeof(void *) == ExpectedAlignment);
 
 /// Arena (bump allocator) forming a linked list. Intentionally not thread safe.
 /// Allocation and de-allocation happen using sanitizer APIs. We make that
@@ -51,5 +58,206 @@ private:
   const uint64_t Size;
 };
 
+// The memory available for allocation follows the Arena header, and we expect
+// it to be thus aligned.
+static_assert(alignof(Arena) == ExpectedAlignment);
+
+/// The contextual profile is a directed tree where each node has one parent. A
+/// node (ContextNode) corresponds to a function activation. The root of the
+/// tree is at a function that was marked as entrypoint to the compiler. A node
+/// stores counter values for edges and a vector of subcontexts. These are the
+/// contexts of callees. The index in the subcontext vector corresponds to the
+/// index of the callsite (as was instrumented via llvm.instrprof.callsite). At
+/// that index we find a linked list, potentially empty, of ContextNodes. Direct
+/// calls will have 0 or 1 values in the linked list, but indirect callsites may
+/// have more.
+///
+/// The ContextNode has a fixed sized header describing it - the GUID of the
+/// function, the size of the counter and callsite vectors. It is also an
+/// (intrusive) linked list for the purposes of the indirect call case above.
+///
+/// Allocation is expected to happen on an Arena. The allocation lays out inline
+/// the counter and subcontexts vectors. The class offers APIs to correctly
+/// reference the latter.
+///
+/// The layout is as follows:
+///
+/// [[declared fields][counters vector][vector of ptrs to subcontexts]]
+///
+/// See also documentation on the counters and subContexts members below.
+///
+/// The structure of the ContextNode is known to LLVM, because LLVM needs to:
+///   (1) increment counts, and
+///   (2) form a GEP for the position in the subcontext list of a callsite
+/// This means changes to LLVM contextual profile lowering and changes here
+/// must be coupled.
+/// Note: the header content isn't interesting to LLVM (other than its size)
+///
+/// Part of contextual collection is the notion of "scratch contexts". These are
+/// buffers that are "large enough" to allow for memory-safe acceses during
+/// counter increments - meaning the counter increment code in LLVM doesn't need
+/// to be concerned with memory safety. Their subcontexts never get populated,
+/// though. The runtime code here produces and recognizes them.
+class ContextNode final {
+  const GUID Guid;
+  ContextNode *const Next;
+  const uint32_t NrCounters;
+  const uint32_t NrCallsites;
+
+public:
+  ContextNode(GUID Guid, uint32_t NrCounters, uint32_t NrCallsites,
+              ContextNode *Next = nullptr)
+      : Guid(Guid), Next(Next), NrCounters(NrCounters),
+        NrCallsites(NrCallsites) {}
+  static inline ContextNode *alloc(char *Place, GUID Guid, uint32_t NrCounters,
+                                   uint32_t NrCallsites,
+                                   ContextNode *Next = nullptr);
+
+  static inline size_t getAllocSize(uint32_t NrCounters, uint32_t NrCallsites) {
+    return sizeof(ContextNode) + sizeof(uint64_t) * NrCounters +
+           sizeof(ContextNode *) * NrCallsites;
+  }
+
+  // The counters vector starts right after the static header.
+  uint64_t *counters() {
+    ContextNode *addr_after = &(this[1]);
+    return reinterpret_cast<uint64_t *>(addr_after);
+  }
+
+  uint32_t counters_size() const { return NrCounters; }
+  uint32_t callsites_size() const { return NrCallsites; }
+
+  const uint64_t *counters() const {
+    return const_cast<ContextNode *>(this)->counters();
+  }
+
+  // The subcontexts vector starts right after the end of the counters vector.
+  ContextNode **subContexts() {
+    return reinterpret_cast<ContextNode **>(&(counters()[NrCounters]));
+  }
+
+  ContextNode *const *subContexts() const {
+    return const_cast<ContextNode *>(this)->subContexts();
+  }
+
+  GUID guid() const { return Guid; }
+  ContextNode *next() { return Next; }
+
+  size_t size() const { return getAllocSize(NrCounters, NrCallsites); }
+
+  void reset();
+
+  // since we go through the runtime to get a context back to LLVM, in the entry
+  // basic block, might as well handle incrementing the entry basic block
+  // counter.
+  void onEntry() { ++counters()[0]; }
+
+  uint64_t entrycount() const { return counters()[0]; }
+};
+
+// Verify maintenance to ContextNode doesn't change this invariant, which makes
+// sure the inlined vectors are appropriately aligned.
+static_assert(alignof(ContextNode) == ExpectedAlignment);
+
+/// ContextRoots are allocated by LLVM for entrypoints. LLVM is only concerned
+/// with allocating and zero-initializing the global value (as in, GlobalValue)
+/// for it.
+struct ContextRoot {
+  ContextNode *FirstNode = nullptr;
+  Arena *FirstMemBlock = nullptr;
+  Arena *CurrentMem = nullptr;
+  // This is init-ed by the static zero initializer in LLVM.
+  // Taken is used to ensure only one thread traverses the contextual graph -
+  // either to read it or to write it. On server side, the same entrypoint will
+  // be entered by numerous threads, but over time, the profile aggregated by
+  // collecting sequentially on one thread at a time is expected to converge to
+  // the aggregate profile that may have been observable on all the threads.
+  // Note that this is node-by-node aggregation, i.e. summing counters of nodes
+  // at the same position in the graph, not flattening.
+  // Threads that cannot lock Taken (fail TryLock) are given a "scratch context"
+  // - a buffer they can clobber, safely from a memory access perspective.
+  //
+  // Note about "scratch"-ness: we currently ignore the data written in them
+  // (which is anyway clobbered). The design allows for that not be the case -
+  // because "scratch"-ness is first and foremost about not trying to build
+  // subcontexts, and is captured by tainting the pointer value (pointer to the
+  // memory treated as context), but right now, we drop that info.
+  //
+  // We could consider relaxing the requirement of more than one thread
+  // entering by holding a few context trees per entrypoint and then aggregating
+  // them (as explained above) at the end of the profile collection - it's a
+  // tradeoff between collection time and memory use: higher precision can be
+  // obtained with either less concurrent collections but more collection time,
+  // or with more concurrent collections (==more memory) and less collection
+  // time. Note that concurrent collection does happen for different
+  // entrypoints, regardless.
+  ::__sanitizer::StaticSpinMutex Taken;
+
+  // If (unlikely) StaticSpinMutex internals change, we need to modify the LLVM
+  // instrumentation lowering side because it is responsible for allocating and
+  // zero-initializing ContextRoots.
+  static_assert(sizeof(Taken) == 1);
+};
+
+/// This API is exposed for testing. See the APIs below about the contract with
+/// LLVM.
+inline bool isScratch(const void *Ctx) {
+  return (reinterpret_cast<uint64_t>(Ctx) & 1);
+}
+
 } // namespace __ctx_profile
+
+extern "C" {
+
+// LLVM fills these in when lowering a llvm.instrprof.callsite intrinsic.
+// position 0 is used when the current context isn't scratch, 1 when it is. They
+// are volatile because of signal handlers - we mean to specifically control
+// when the data is loaded.
+//
+/// TLS where LLVM stores the pointer of the called value, as part of lowering a
+/// llvm.instrprof.callsite
+extern __thread void *volatile __llvm_ctx_profile_expected_callee[2];
+/// TLS where LLVM stores the pointer inside a caller's subcontexts vector that
+/// corresponds to the callsite being lowered.
+extern __thread __ctx_profile::ContextNode *
+    *volatile __llvm_ctx_profile_callsite[2];
+
+// __llvm_ctx_profile_current_context_root is exposed for unit testing,
+// othwerise it's only used internally by compiler-rt/ctx_profile.
+extern __thread __ctx_profile::ContextRoot
+    *volatile __llvm_ctx_profile_current_context_root;
+
+/// called by LLVM in the entry BB of a "entry point" function. The returned
+/// pointer may be "tainted" - its LSB set to 1 - to indicate it's scratch.
+__ctx_profile::ContextNode *
+__llvm_ctx_profile_start_context(__ctx_profile::ContextRoot *Root,
+                                 __ctx_profile::GUID Guid, uint32_t Counters,
+                                 uint32_t Callsites);
+
+/// paired with __llvm_ctx_profile_start_context, and called at the exit of the
+/// entry point function.
+void __llvm_ctx_profile_release_context(__ctx_profile::ContextRoot *Root);
+
+/// called for any other function than entry points, in the entry BB of such
+/// function. Same consideration about LSB of returned value as .._start_context
+__ctx_profile::ContextNode *
+__llvm_ctx_profile_get_context(void *Callee, __ctx_profile::GUID Guid,
+                               uint32_t NrCounters, uint32_t NrCallsites);
+
+/// Prepares for collection. Currently this resets counter values but preserves
+/// internal context tree structure.
+void __llvm_ctx_profile_start_collection();
+
+/// Completely free allocated memory.
+void __llvm_ctx_profile_free();
+
+/// Used to obtain the profile. The Writer is called for each root ContextNode,
+/// with the ContextRoot::Taken taken. The Writer is responsible for traversing
+/// the structure underneath.
+/// The Writer's first parameter plays the role of closure for Writer, and is
+/// what the caller of __llvm_ctx_profile_fetch passes as the Data parameter.
+/// The second parameter is the root of a context tree.
+bool __llvm_ctx_profile_fetch(
+    void *Data, bool (*Writer)(void *, const __ctx_profile::ContextNode &));
+}
 #endif // CTX_PROFILE_CTXINSTRPROFILING_H_
diff --git a/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp b/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
index 44f37d257632..f6ebe6ab2e50 100644
--- a/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
+++ b/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
@@ -1,8 +1,17 @@
 #include "../CtxInstrProfiling.h"
 #include "gtest/gtest.h"
+#include <thread>
 
 using namespace __ctx_profile;
 
+class ContextTest : public ::testing::Test {
+  void SetUp() override { memset(&Root, 0, sizeof(ContextRoot)); }
+  void TearDown() override { __llvm_ctx_profile_free(); }
+
+public:
+  ContextRoot Root;
+};
+
 TEST(ArenaTest, Basic) {
   Arena *A = Arena::allocateNewArena(1024);
   EXPECT_EQ(A->size(), 1024U);
@@ -20,3 +29,186 @@ TEST(ArenaTest, Basic) {
   Arena::freeArenaList(A);
   EXPECT_EQ(A, nullptr);
 }
+
+TEST_F(ContextTest, Basic) {
+  auto *Ctx = __llvm_ctx_profile_start_context(&Root, 1, 10, 4);
+  ASSERT_NE(Ctx, nullptr);
+  EXPECT_NE(Root.CurrentMem, nullptr);
+  EXPECT_EQ(Root.FirstMemBlock, Root.CurrentMem);
+  EXPECT_EQ(Ctx->size(), sizeof(ContextNode) + 10 * sizeof(uint64_t) +
+                             4 * sizeof(ContextNode *));
+  EXPECT_EQ(Ctx->counters_size(), 10U);
+  EXPECT_EQ(Ctx->callsites_size(), 4U);
+  EXPECT_EQ(__llvm_ctx_profile_current_context_root, &Root);
+  Root.Taken.CheckLocked();
+  EXPECT_FALSE(Root.Taken.TryLock());
+  __llvm_ctx_profile_release_context(&Root);
+  EXPECT_EQ(__llvm_ctx_profile_current_context_root, nullptr);
+  EXPECT_TRUE(Root.Taken.TryLock());
+  Root.Taken.Unlock();
+}
+
+TEST_F(ContextTest, Callsite) {
+  auto *Ctx = __llvm_ctx_profile_start_context(&Root, 1, 10, 4);
+  int FakeCalleeAddress = 0;
+  const bool IsScratch = isScratch(Ctx);
+  EXPECT_FALSE(IsScratch);
+  // This is the sequence the caller performs - it's the lowering of the
+  // instrumentation of the callsite "2". "2" is arbitrary here.
+  __llvm_ctx_profile_expected_callee[0] = &FakeCalleeAddress;
+  __llvm_ctx_profile_callsite[0] = &Ctx->subContexts()[2];
+  // This is what the callee does
+  auto *Subctx = __llvm_ctx_profile_get_context(&FakeCalleeAddress, 2, 3, 1);
+  // We expect the subcontext to be appropriately placed and dimensioned
+  EXPECT_EQ(Ctx->subContexts()[2], Subctx);
+  EXPECT_EQ(Subctx->counters_size(), 3U);
+  EXPECT_EQ(Subctx->callsites_size(), 1U);
+  // We reset these in _get_context.
+  EXPECT_EQ(__llvm_ctx_profile_expected_callee[0], nullptr);
+  EXPECT_EQ(__llvm_ctx_profile_callsite[0], nullptr);
+
+  EXPECT_EQ(Subctx->size(), sizeof(ContextNode) + 3 * sizeof(uint64_t) +
+                                1 * sizeof(ContextNode *));
+  __llvm_ctx_profile_release_context(&Root);
+}
+
+TEST_F(ContextTest, ScratchNoCollection) {
+  EXPECT_EQ(__llvm_ctx_profile_current_context_root, nullptr);
+  int FakeCalleeAddress = 0;
+  // this would be the very first function executing this. the TLS is empty,
+  // too.
+  auto *Ctx = __llvm_ctx_profile_get_context(&FakeCalleeAddress, 2, 3, 1);
+  // We never entered a context (_start_context was never called) - so the
+  // returned context must be scratch.
+  EXPECT_TRUE(isScratch(Ctx));
+}
+
+TEST_F(ContextTest, ScratchDuringCollection) {
+  auto *Ctx = __llvm_ctx_profile_start_context(&Root, 1, 10, 4);
+  int FakeCalleeAddress = 0;
+  int OtherFakeCalleeAddress = 0;
+  __llvm_ctx_profile_expected_callee[0] = &FakeCalleeAddress;
+  __llvm_ctx_profile_callsite[0] = &Ctx->subContexts()[2];
+  auto *Subctx =
+      __llvm_ctx_profile_get_context(&OtherFakeCalleeAddress, 2, 3, 1);
+  // We expected a different callee - so return scratch. It mimics what happens
+  // in the case of a signal handler - in this case, OtherFakeCalleeAddress is
+  // the signal handler.
+  EXPECT_TRUE(isScratch(Subctx));
+  EXPECT_EQ(__llvm_ctx_profile_expected_callee[0], nullptr);
+  EXPECT_EQ(__llvm_ctx_profile_callsite[0], nullptr);
+
+  int ThirdFakeCalleeAddress = 0;
+  __llvm_ctx_profile_expected_callee[1] = &ThirdFakeCalleeAddress;
+  __llvm_ctx_profile_callsite[1] = &Subctx->subContexts()[0];
+
+  auto *Subctx2 =
+      __llvm_ctx_profile_get_context(&ThirdFakeCalleeAddress, 3, 0, 0);
+  // We again expect scratch because the '0' position is where the runtime
+  // looks, so it doesn't matter the '1' position is populated correctly.
+  EXPECT_TRUE(isScratch(Subctx2));
+
+  __llvm_ctx_profile_expected_callee[0] = &ThirdFakeCalleeAddress;
+  __llvm_ctx_profile_callsite[0] = &Subctx->subContexts()[0];
+  auto *Subctx3 =
+      __llvm_ctx_profile_get_context(&ThirdFakeCalleeAddress, 3, 0, 0);
+  // We expect scratch here, too, because the value placed in
+  // __llvm_ctx_profile_callsite is scratch
+  EXPECT_TRUE(isScratch(Subctx3));
+
+  __llvm_ctx_profile_release_context(&Root);
+}
+
+TEST_F(ContextTest, NeedMoreMemory) {
+  auto *Ctx = __llvm_ctx_profile_start_context(&Root, 1, 10, 4);
+  int FakeCalleeAddress = 0;
+  const bool IsScratch = isScratch(Ctx);
+  EXPECT_FALSE(IsScratch);
+  const auto *CurrentMem = Root.CurrentMem;
+  __llvm_ctx_profile_expected_callee[0] = &FakeCalleeAddress;
+  __llvm_ctx_profile_callsite[0] = &Ctx->subContexts()[2];
+  // Allocate a massive subcontext to force new arena allocation
+  auto *Subctx =
+      __llvm_ctx_profile_get_context(&FakeCalleeAddress, 3, 1 << 20, 1);
+  EXPECT_EQ(Ctx->subContexts()[2], Subctx);
+  EXPECT_NE(CurrentMem, Root.CurrentMem);
+  EXPECT_NE(Root.CurrentMem, nullptr);
+}
+
+TEST_F(ContextTest, ConcurrentRootCollection) {
+  std::atomic<int> NonScratch = 0;
+  std::atomic<int> Executions = 0;
+
+  __sanitizer::Semaphore GotCtx;
+
+  auto Entrypoint = [&]() {
+    ++Executions;
+    auto *Ctx = __llvm_ctx_profile_start_context(&Root, 1, 10, 4);
+    GotCtx.Post();
+    const bool IS = isScratch(Ctx);
+    NonScratch += (!IS);
+    if (!IS) {
+      GotCtx.Wait();
+      GotCtx.Wait();
+    }
+    __llvm_ctx_profile_release_context(&Root);
+  };
+  std::thread T1(Entrypoint);
+  std::thread T2(Entrypoint);
+  T1.join();
+  T2.join();
+  EXPECT_EQ(NonScratch, 1);
+  EXPECT_EQ(Executions, 2);
+}
+
+TEST_F(ContextTest, Dump) {
+  auto *Ctx = __llvm_ctx_profile_start_context(&Root, 1, 10, 4);
+  int FakeCalleeAddress = 0;
+  __llvm_ctx_profile_expected_callee[0] = &FakeCalleeAddress;
+  __llvm_ctx_profile_callsite[0] = &Ctx->subContexts()[2];
+  auto *Subctx = __llvm_ctx_profile_get_context(&FakeCalleeAddress, 2, 3, 1);
+  (void)Subctx;
+  __llvm_ctx_profile_release_context(&Root);
+
+  struct Writer {
+    ContextRoot *const Root;
+    const size_t Entries;
+    bool State = false;
+    Writer(ContextRoot *Root, size_t Entries) : Root(Root), Entries(Entries) {}
+
+    bool write(const ContextNode &Node) {
+      EXPECT_FALSE(Root->Taken.TryLock());
+      EXPECT_EQ(Node.guid(), 1);
+      EXPECT_EQ(Node.counters()[0], Entries);
+      EXPECT_EQ(Node.counters_size(), 10);
+      EXPECT_EQ(Node.callsites_size(), 4);
+      EXPECT_EQ(Node.subContexts()[0], nullptr);
+      EXPECT_EQ(Node.subContexts()[1], nullptr);
+      EXPECT_NE(Node.subContexts()[2], nullptr);
+      EXPECT_EQ(Node.subContexts()[3], nullptr);
+      const auto &SN = *Node.subContexts()[2];
+      EXPECT_EQ(SN.guid(), 2);
+      EXPECT_EQ(SN.counters()[0], Entries);
+      EXPECT_EQ(SN.counters_size(), 3);
+      EXPECT_EQ(SN.callsites_size(), 1);
+      EXPECT_EQ(SN.subContexts()[0], nullptr);
+      State = true;
+      return true;
+    }
+  };
+  Writer W(&Root, 1);
+  EXPECT_FALSE(W.State);
+  __llvm_ctx_profile_fetch(&W, [](void *W, const ContextNode &Node) -> bool {
+    return reinterpret_cast<Writer *>(W)->write(Node);
+  });
+  EXPECT_TRUE(W.State);
+
+  // this resets all counters but not the internal structure.
+  __llvm_ctx_profile_start_collection();
+  Writer W2(&Root, 0);
+  EXPECT_FALSE(W2.State);
+  __llvm_ctx_profile_fetch(&W2, [](void *W, const ContextNode &Node) -> bool {
+    return reinterpret_cast<Writer *>(W)->write(Node);
+  });
+  EXPECT_TRUE(W2.State);
+}
-- 
GitLab


From 8fc68879badc2dc83e8b9a575992af285d4a1057 Mon Sep 17 00:00:00 2001
From: Ryosuke Niwa <rniwa@webkit.org>
Date: Tue, 7 May 2024 15:16:29 -0700
Subject: [PATCH 0110/1206] Fix a typo in webkit.NoUncountedMemberChecker.
 (#91402)

Co-authored-by: Brianna Fan <bfan2@apple.com>
---
 .../Checkers/WebKit/NoUncountedMembersChecker.cpp               | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/NoUncountedMembersChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/NoUncountedMembersChecker.cpp
index c753ed84a700..69a0eb3086ab 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/NoUncountedMembersChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/NoUncountedMembersChecker.cpp
@@ -34,7 +34,7 @@ private:
 public:
   NoUncountedMemberChecker()
       : Bug(this,
-            "Member variable is a raw-poiner/reference to reference-countable "
+            "Member variable is a raw-pointer/reference to reference-countable "
             "type",
             "WebKit coding guidelines") {}
 
-- 
GitLab


From ff0c5ccbe8879ccad9cb3548b69b114872c33ebb Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks@fb.com>
Date: Tue, 7 May 2024 16:05:10 -0700
Subject: [PATCH 0111/1206] [BOLT] Add a test for BOLT-reserved space in a
 binary (#91399)

Test case for #90300.
---
 bolt/test/runtime/bolt-reserved.cpp | 40 +++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 bolt/test/runtime/bolt-reserved.cpp

diff --git a/bolt/test/runtime/bolt-reserved.cpp b/bolt/test/runtime/bolt-reserved.cpp
new file mode 100644
index 000000000000..5e93b4f7c3d4
--- /dev/null
+++ b/bolt/test/runtime/bolt-reserved.cpp
@@ -0,0 +1,40 @@
+// REQUIRES: system-linux
+
+/*
+ * Check that llvm-bolt uses reserved space in a binary for allocating
+ * new sections.
+ */
+
+// RUN: %clang %s -o %t.exe -Wl,-q
+// RUN: llvm-bolt %t.exe -o %t.bolt.exe 2>&1 | FileCheck %s
+// RUN: %t.bolt.exe
+
+// CHECK: BOLT-INFO: using reserved space
+
+/*
+ * Check that llvm-bolt detects a condition when the reserved space is
+ * not enough for allocating new sections.
+ */
+
+// RUN: %clang %s -o %t.exe -Wl,--no-eh-frame-hdr -Wl,-q -DTINY
+// RUN: not llvm-bolt %t.exe -o %t.bolt.exe 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=CHECK-TINY
+
+// CHECK-TINY: BOLT-ERROR: reserved space (1 byte) is smaller than required
+
+#ifdef TINY
+#define RSIZE "1"
+#else
+#define RSIZE "8192 * 1024"
+#endif
+
+asm(".pushsection .text \n\
+       .globl __bolt_reserved_start \n\
+       .type __bolt_reserved_start, @object \n\
+       __bolt_reserved_start: \n\
+       .space " RSIZE " \n\
+       .globl __bolt_reserved_end \n\
+       __bolt_reserved_end: \n\
+     .popsection");
+
+int main() { return 0; }
-- 
GitLab


From 54401b43494a57baae9d3663cd7c694b040ef01c Mon Sep 17 00:00:00 2001
From: Prathamesh Tagore <63031630+meshtag@users.noreply.github.com>
Date: Wed, 8 May 2024 04:49:55 +0530
Subject: [PATCH 0112/1206] [mlir][memref.expand_shape] Add verifier check to
 ensure correct output_shape is provided by user (#91245)

The verifier was not checking for the case when the user provided shape
in output_shape is different than the one inferred from output type. Fix
this.
---
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp | 10 ++++++++++
 mlir/test/Dialect/MemRef/invalid.mlir    | 11 +++++++++++
 2 files changed, 21 insertions(+)

diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index 393f73dc65cd..78201ae29cd9 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -2353,6 +2353,16 @@ LogicalResult ExpandShapeOp::verify() {
            << " dynamic dims while output_shape has " << getOutputShape().size()
            << " values";
 
+  // Verify if provided output shapes are in agreement with output type.
+  DenseI64ArrayAttr staticOutputShapes = getStaticOutputShapeAttr();
+  ArrayRef<int64_t> resShape = getResult().getType().getShape();
+  unsigned staticShapeNum = 0;
+
+  for (auto [pos, shape] : llvm::enumerate(resShape))
+    if (!ShapedType::isDynamic(shape) &&
+        shape != staticOutputShapes[staticShapeNum++])
+      emitOpError("invalid output shape provided at pos ") << pos;
+
   return success();
 }
 
diff --git a/mlir/test/Dialect/MemRef/invalid.mlir b/mlir/test/Dialect/MemRef/invalid.mlir
index 70c96aad9555..0f533cb95a0c 100644
--- a/mlir/test/Dialect/MemRef/invalid.mlir
+++ b/mlir/test/Dialect/MemRef/invalid.mlir
@@ -1103,3 +1103,14 @@ func.func @subview_invalid_strides_rank_reduction(%m: memref<7x22x333x4444xi32>)
       : memref<7x22x333x4444xi32> to memref<7x11x4444xi32>
   return
 }
+
+// -----
+
+func.func @expand_shape_invalid_output_shape(
+    %arg0: memref<30x20xf32, strided<[4000, 2], offset: 100>>) {
+  // expected-error @+1 {{invalid output shape provided at pos 2}}
+  %0 = memref.expand_shape %arg0 [[0, 1], [2]] output_shape [2, 15, 21] :
+      memref<30x20xf32, strided<[4000, 2], offset: 100>>
+      into memref<2x15x20xf32, strided<[60000, 4000, 2], offset: 100>>
+  return
+}
-- 
GitLab


From c0d9efd35d6a44258466349a7ba3a10c693b8c9c Mon Sep 17 00:00:00 2001
From: Youngsuk Kim <youngsuk.kim@hpe.com>
Date: Tue, 7 May 2024 19:20:26 -0400
Subject: [PATCH 0113/1206] [llvm][NVPTX] Remove outdated comments (NFC)
 (#91409)

---
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 142dd64ddea9..393fa29ff051 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -2497,10 +2497,6 @@ defm FSetNE : FSET_FORMAT<setne, CmpNE, CmpNE_FTZ>;
 defm FSetNUM : FSET_FORMAT<seto, CmpNUM, CmpNUM_FTZ>;
 defm FSetNAN : FSET_FORMAT<setuo, CmpNAN, CmpNAN_FTZ>;
 
-// FIXME: What is this doing here?  Can it be deleted?
-// def ld_param         : SDNode<"NVPTXISD::LOAD_PARAM", SDTLoad,
-//                         [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
-
 def SDTDeclareParamProfile :
   SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>]>;
 def SDTDeclareScalarParamProfile :
-- 
GitLab


From 3f37397c959a85f4cad91b655ea03a5d2450ab38 Mon Sep 17 00:00:00 2001
From: Max Winkler <max.enrico.winkler@gmail.com>
Date: Tue, 7 May 2024 19:46:19 -0400
Subject: [PATCH 0114/1206] [clang][CodeGen] Fix MSVC ABI for classes with a
 deleted copy assignment operator (#90547)

For global functions and static methods the MSVC ABI returns
structs/classes with a deleted copy assignment operator indirectly.
From local testing this ABI holds true for all currently supported
architectures including ARM64EC.
---
 clang/docs/ReleaseNotes.rst                   |  3 +
 clang/lib/CodeGen/MicrosoftCXXABI.cpp         | 22 ++++-
 .../test/CodeGen/x64-microsoft-arguments.cpp  | 92 +++++++++++++++++++
 3 files changed, 116 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/CodeGen/x64-microsoft-arguments.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index cc3108bf41d6..106b1e6f9945 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -76,6 +76,9 @@ ABI Changes in This Version
   returning a class in a register. This affects some uses of std::pair.
   (#GH86384).
 
+- Fixed Microsoft calling convention when returning classes that have a deleted
+  copy assignment operator. Such a class should be returned indirectly.
+
 AST Dumping Potentially Breaking Changes
 ----------------------------------------
 
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index d47927745759..e4f798f6a97d 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -1122,7 +1122,22 @@ static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty,
   //   No base classes
   //   No virtual functions
   // Additionally, we need to ensure that there is a trivial copy assignment
-  // operator, a trivial destructor and no user-provided constructors.
+  // operator, a trivial destructor, no user-provided constructors and no
+  // deleted copy assignment operator.
+
+  // We need to cover two cases when checking for a deleted copy assignment
+  // operator.
+  //
+  // struct S { int& r; };
+  // The above will have an implicit copy assignment operator that is deleted
+  // and there will not be a `CXXMethodDecl` for the copy assignment operator.
+  // This is handled by the `needsImplicitCopyAssignment()` check below.
+  //
+  // struct S { S& operator=(const S&) = delete; int i; };
+  // The above will not have an implicit copy assignment operator that is
+  // deleted but there is a deleted `CXXMethodDecl` for the declared copy
+  // assignment operator. This is handled by the `isDeleted()` check below.
+
   if (RD->hasProtectedFields() || RD->hasPrivateFields())
     return false;
   if (RD->getNumBases() > 0)
@@ -1131,6 +1146,8 @@ static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty,
     return false;
   if (RD->hasNonTrivialCopyAssignment())
     return false;
+  if (RD->needsImplicitCopyAssignment() && !RD->hasSimpleCopyAssignment())
+    return false;
   for (const Decl *D : RD->decls()) {
     if (auto *Ctor = dyn_cast<CXXConstructorDecl>(D)) {
       if (Ctor->isUserProvided())
@@ -1138,6 +1155,9 @@ static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty,
     } else if (auto *Template = dyn_cast<FunctionTemplateDecl>(D)) {
       if (isa<CXXConstructorDecl>(Template->getTemplatedDecl()))
         return false;
+    } else if (auto *MethodDecl = dyn_cast<CXXMethodDecl>(D)) {
+      if (MethodDecl->isCopyAssignmentOperator() && MethodDecl->isDeleted())
+        return false;
     }
   }
   if (RD->hasNonTrivialDestructor())
diff --git a/clang/test/CodeGen/x64-microsoft-arguments.cpp b/clang/test/CodeGen/x64-microsoft-arguments.cpp
new file mode 100644
index 000000000000..c666c92ad2db
--- /dev/null
+++ b/clang/test/CodeGen/x64-microsoft-arguments.cpp
@@ -0,0 +1,92 @@
+// RUN: %clang_cc1 -triple x86_64-windows-msvc -ffreestanding -emit-llvm -O0 \
+// RUN: -x c++ -o - %s | FileCheck %s
+
+int global_i = 0;
+
+// Pass and return object with a reference type (pass directly, return indirectly).
+// CHECK: define dso_local void @"?f1@@YA?AUS1@@XZ"(ptr dead_on_unwind noalias writable sret(%struct.S1) align 8 {{.*}})
+// CHECK: call void @"?func1@@YA?AUS1@@U1@@Z"(ptr dead_on_unwind writable sret(%struct.S1) align 8 {{.*}}, i64 {{.*}})
+struct S1 {
+  int& r;
+};
+
+S1 func1(S1 x);
+S1 f1() {
+  S1 x{ global_i };
+  return func1(x);
+}
+
+// Pass and return object with a reference type within an inner struct (pass directly, return indirectly).
+// CHECK: define dso_local void @"?f2@@YA?AUS2@@XZ"(ptr dead_on_unwind noalias writable sret(%struct.S2) align 8 {{.*}})
+// CHECK: call void @"?func2@@YA?AUS2@@U1@@Z"(ptr dead_on_unwind writable sret(%struct.S2) align 8 {{.*}}, i64 {{.*}})
+struct Inner {
+  int& r;
+};
+
+struct S2 {
+  Inner i;
+};
+
+S2 func2(S2 x);
+S2 f2() {
+  S2 x{ { global_i } };
+  return func2(x);
+}
+
+// Pass and return object with a reference type (pass directly, return indirectly).
+// CHECK: define dso_local void @"?f3@@YA?AUS3@@XZ"(ptr dead_on_unwind noalias writable sret(%struct.S3) align 8 {{.*}})
+// CHECK: call void @"?func3@@YA?AUS3@@U1@@Z"(ptr dead_on_unwind writable sret(%struct.S3) align 8 {{.*}}, i64 {{.*}})
+struct S3 {
+  const int& r;
+};
+
+S3 func3(S3 x);
+S3 f3() {
+  S3 x{ global_i };
+  return func3(x);
+}
+
+// Pass and return object with a reference type within an inner struct (pass directly, return indirectly).
+// CHECK: define dso_local void @"?f4@@YA?AUS4@@XZ"(ptr dead_on_unwind noalias writable sret(%struct.S4) align 8 {{.*}})
+// CHECK: call void @"?func4@@YA?AUS4@@U1@@Z"(ptr dead_on_unwind writable sret(%struct.S4) align 8 {{.*}}, i64 {{.*}})
+struct InnerConst {
+  const int& r;
+};
+
+struct S4 {
+  InnerConst i;
+};
+
+S4 func4(S4 x);
+S4 f4() {
+  S4 x{ { global_i } };
+  return func4(x);
+}
+
+// Pass and return an object with an explicitly deleted copy assignment operator (pass directly, return indirectly).
+// CHECK: define dso_local void @"?f5@@YA?AUS5@@XZ"(ptr dead_on_unwind noalias writable sret(%struct.S5) align 4 {{.*}})
+// CHECK: call void @"?func5@@YA?AUS5@@U1@@Z"(ptr dead_on_unwind writable sret(%struct.S5) align 4 {{.*}}, i32 {{.*}})
+struct S5 {
+  S5& operator=(const S5&) = delete;
+  int i;
+};
+
+S5 func5(S5 x);
+S5 f5() {
+  S5 x{ 1 };
+  return func5(x);
+}
+
+// Pass and return an object with an explicitly defaulted copy assignment operator that is implicitly deleted (pass directly, return indirectly).
+// CHECK: define dso_local void @"?f6@@YA?AUS6@@XZ"(ptr dead_on_unwind noalias writable sret(%struct.S6) align 8 {{.*}})
+// CHECK: call void @"?func6@@YA?AUS6@@U1@@Z"(ptr dead_on_unwind writable sret(%struct.S6) align 8 {{.*}}, i64 {{.*}})
+struct S6 {
+  S6& operator=(const S6&) = default;
+  int& i;
+};
+
+S6 func6(S6 x);
+S6 f6() {
+  S6 x{ global_i };
+  return func6(x);
+}
-- 
GitLab


From 04d0a691af9e116f651d233c5689863f614d3adf Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 7 May 2024 16:56:45 -0700
Subject: [PATCH 0115/1206] [ELF] Fix --compress-debug-sections=zstd when zlib
 is disabled

---
 lld/ELF/Options.td         | 2 +-
 lld/ELF/OutputSections.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 73a4f9662a56..b9e05a4b1fd5 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -68,7 +68,7 @@ defm compress_debug_sections:
   MetaVarName<"[none,zlib,zstd]">;
 
 defm compress_sections: EEq<"compress-sections",
-  "Compress output sections that match the glob and do not have the SHF_ALLOC flag."
+  "Compress output sections that match the glob and do not have the SHF_ALLOC flag. "
   "The compression level is <level> (if specified) or a default speed-focused level">,
   MetaVarName<"<section-glob>={none,zlib,zstd}[:level]">;
 
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 2dbbff06a890..9c667241360f 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -438,10 +438,10 @@ template <class ELFT> void OutputSection::maybeCompress() {
     compressed.type = ELFCOMPRESS_ZLIB;
     compressed.checksum = checksum;
   }
+#endif
 
   compressed.shards = std::move(shardsOut);
   flags |= SHF_COMPRESSED;
-#endif
 }
 
 static void writeInt(uint8_t *buf, uint64_t data, uint64_t size) {
-- 
GitLab


From 77c5cea78eac3f20d0ba79f5892235e5aac82603 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Tue, 7 May 2024 20:04:57 -0400
Subject: [PATCH 0116/1206] [Clang][Sema] Explicit template arguments are not
 substituted into the exception specification of a function (#90760)

[temp.deduct.general] p6 states:
> At certain points in the template argument deduction process it is
necessary to take a function type that makes use of template parameters
and replace those template parameters with the corresponding template
arguments.
This is done at the beginning of template argument deduction when any
explicitly specified template arguments are substituted into the
function type, and again at the end of template argument deduction when
any template arguments that were deduced or obtained from default
arguments are substituted.

[temp.deduct.general] p7 goes on to say:
> The _deduction substitution loci_ are
> - the function type outside of the _noexcept-specifier_,
> - the explicit-specifier,
> - the template parameter declarations, and
> - the template argument list of a partial specialization
>
> The substitution occurs in all types and expressions that are used in
the deduction substitution loci. [...]

Consider the following:
```cpp
struct A
{
    static constexpr bool x = true;
};

template<typename T, typename U>
void f(T, U) noexcept(T::x); // #1

template<typename T, typename U>
void f(T, U*) noexcept(T::y); // #2

template<>
void f<A>(A, int*) noexcept; // clang currently accepts, GCC and EDG reject
```

Currently, `Sema::SubstituteExplicitTemplateArguments` will substitute
into the _noexcept-specifier_ when deducing template arguments from a
function declaration or when deducing template arguments for taking the
address of a function template (and the substitution is treated as a
SFINAE context). In the above example, `#1` is selected as the primary
template because substitution of the explicit template arguments into
the _noexcept-specifier_ of `#2` failed, which resulted in the candidate
being ignored.

This behavior is incorrect ([temp.deduct.general] note 4 says as much), and
this patch corrects it by deferring all substitution into the
_noexcept-specifier_ until it is instantiated.

As part of the necessary changes to make this patch work, the
instantiation of the exception specification of a function template
specialization when taking the address of a function template is changed
to only occur for the function selected by overload resolution per
[except.spec] p13.1 (as opposed to being instantiated for every candidate).
---
 clang/docs/ReleaseNotes.rst                   |  2 ++
 clang/lib/Sema/SemaInit.cpp                   | 24 ++++++++++----
 clang/lib/Sema/SemaTemplateDeduction.cpp      | 32 ++-----------------
 clang/test/CXX/drs/dr13xx.cpp                 | 13 +++-----
 clang/test/CXX/except/except.spec/p13.cpp     | 27 ++++++++++++++++
 clang/test/CXX/temp/temp.deduct/p7.cpp        | 14 ++++++++
 .../SemaCXX/cxx1z-noexcept-function-type.cpp  |  4 +--
 clang/test/SemaTemplate/temp_arg_type.cpp     | 10 +++---
 8 files changed, 75 insertions(+), 51 deletions(-)
 create mode 100644 clang/test/CXX/temp/temp.deduct/p7.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 106b1e6f9945..c4a9501ca15c 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -691,6 +691,8 @@ Bug Fixes to C++ Support
 - Fix an assertion failure when parsing an invalid members of an anonymous class. (#GH85447)
 - Fixed a misuse of ``UnresolvedLookupExpr`` for ill-formed templated expressions. Fixes (#GH48673), (#GH63243)
   and (#GH88832).
+- Clang now defers all substitution into the exception specification of a function template specialization
+  until the noexcept-specifier is instantiated.
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 7d9eaf672046..c8049ae581f8 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -6576,12 +6576,12 @@ void InitializationSequence::InitializeFrom(Sema &S,
 
     AddPassByIndirectCopyRestoreStep(DestType, ShouldCopy);
   } else if (ICS.isBad()) {
-    DeclAccessPair dap;
-    if (isLibstdcxxPointerReturnFalseHack(S, Entity, Initializer)) {
+    if (isLibstdcxxPointerReturnFalseHack(S, Entity, Initializer))
       AddZeroInitializationStep(Entity.getType());
-    } else if (Initializer->getType() == Context.OverloadTy &&
-               !S.ResolveAddressOfOverloadedFunction(Initializer, DestType,
-                                                     false, dap))
+    else if (DeclAccessPair Found;
+             Initializer->getType() == Context.OverloadTy &&
+             !S.ResolveAddressOfOverloadedFunction(Initializer, DestType,
+                                                   /*Complain=*/false, Found))
       SetFailed(InitializationSequence::FK_AddressOfOverloadFailed);
     else if (Initializer->getType()->isFunctionType() &&
              isExprAnUnaddressableFunction(S, Initializer))
@@ -9641,6 +9641,8 @@ bool InitializationSequence::Diagnose(Sema &S,
   if (!Failed())
     return false;
 
+  QualType DestType = Entity.getType();
+
   // When we want to diagnose only one element of a braced-init-list,
   // we need to factor it out.
   Expr *OnlyArg;
@@ -9650,11 +9652,21 @@ bool InitializationSequence::Diagnose(Sema &S,
       OnlyArg = List->getInit(0);
     else
       OnlyArg = Args[0];
+
+    if (OnlyArg->getType() == S.Context.OverloadTy) {
+      DeclAccessPair Found;
+      if (FunctionDecl *FD = S.ResolveAddressOfOverloadedFunction(
+              OnlyArg, DestType.getNonReferenceType(), /*Complain=*/false,
+              Found)) {
+        if (Expr *Resolved =
+                S.FixOverloadedFunctionReference(OnlyArg, Found, FD).get())
+          OnlyArg = Resolved;
+      }
+    }
   }
   else
     OnlyArg = nullptr;
 
-  QualType DestType = Entity.getType();
   switch (Failure) {
   case FK_TooManyInitsForReference:
     // FIXME: Customize for the initialized entity?
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 9f9e44228271..dcaea4a77bff 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -1323,13 +1323,11 @@ bool Sema::isSameOrCompatibleFunctionType(QualType P, QualType A) {
     return Context.hasSameType(P, A);
 
   // Noreturn and noexcept adjustment.
-  QualType AdjustedParam;
-  if (IsFunctionConversion(P, A, AdjustedParam))
-    return Context.hasSameType(AdjustedParam, A);
+  if (QualType AdjustedParam; IsFunctionConversion(P, A, AdjustedParam))
+    P = AdjustedParam;
 
   // FIXME: Compatible calling conventions.
-
-  return Context.hasSameType(P, A);
+  return Context.hasSameFunctionTypeIgnoringExceptionSpec(P, A);
 }
 
 /// Get the index of the first template parameter that was originally from the
@@ -3509,23 +3507,6 @@ TemplateDeductionResult Sema::SubstituteExplicitTemplateArguments(
   if (FunctionType) {
     auto EPI = Proto->getExtProtoInfo();
     EPI.ExtParameterInfos = ExtParamInfos.getPointerOrNull(ParamTypes.size());
-
-    // In C++1z onwards, exception specifications are part of the function type,
-    // so substitution into the type must also substitute into the exception
-    // specification.
-    SmallVector<QualType, 4> ExceptionStorage;
-    if (getLangOpts().CPlusPlus17 &&
-        SubstExceptionSpec(
-            Function->getLocation(), EPI.ExceptionSpec, ExceptionStorage,
-            getTemplateInstantiationArgs(
-                FunctionTemplate, nullptr, /*Final=*/true,
-                /*Innermost=*/SugaredExplicitArgumentList->asArray(),
-                /*RelativeToPrimary=*/false,
-                /*Pattern=*/nullptr,
-                /*ForConstraintInstantiation=*/false,
-                /*SkipForSpecialization=*/true)))
-      return TemplateDeductionResult::SubstitutionFailure;
-
     *FunctionType = BuildFunctionType(ResultType, ParamTypes,
                                       Function->getLocation(),
                                       Function->getDeclName(),
@@ -4705,13 +4686,6 @@ TemplateDeductionResult Sema::DeduceTemplateArguments(
                                                Info.getLocation()))
     return TemplateDeductionResult::MiscellaneousDeductionFailure;
 
-  auto *SpecializationFPT =
-      Specialization->getType()->castAs<FunctionProtoType>();
-  if (IsAddressOfFunction && getLangOpts().CPlusPlus17 &&
-      isUnresolvedExceptionSpec(SpecializationFPT->getExceptionSpecType()) &&
-      !ResolveExceptionSpec(Info.getLocation(), SpecializationFPT))
-    return TemplateDeductionResult::MiscellaneousDeductionFailure;
-
   // Adjust the exception specification of the argument to match the
   // substituted and resolved type we just formed. (Calling convention and
   // noreturn can't be dependent, so we don't actually need this for them
diff --git a/clang/test/CXX/drs/dr13xx.cpp b/clang/test/CXX/drs/dr13xx.cpp
index dad82c4e2829..a334b6d01acf 100644
--- a/clang/test/CXX/drs/dr13xx.cpp
+++ b/clang/test/CXX/drs/dr13xx.cpp
@@ -281,13 +281,10 @@ namespace cwg1330 { // cwg1330: 4 c++11
   decltype(f<char>()) f2; // #cwg1330-f-char
   bool f3 = noexcept(f<float>()); /// #cwg1330-f-float
 #endif
-  // In C++17 onwards, substituting explicit template arguments into the
-  // function type substitutes into the exception specification (because it's
-  // part of the type). In earlier languages, we don't notice there's a problem
-  // until we've already started to instantiate.
   template int f<short>(); // #cwg1330-f-short
-  // since-cxx17-error@-1 {{explicit instantiation of 'f' does not refer to a function template, variable template, member function, member class, or static data member}}
-  //   since-cxx17-note@#cwg1330-f {{candidate template ignored: substitution failure [with T = short]: type 'short' cannot be used prior to '::' because it has no members}}
+  // since-cxx17-error@#cwg1330-f {{type 'short' cannot be used prior to '::' because it has no members}}
+  //   since-cxx17-note@#cwg1330-f {{in instantiation of exception specification for 'f<short>' requested here}}
+  //   since-cxx17-note@#cwg1330-f-short {{in instantiation of function template specialization 'cwg1330::f<short>' requested here}}
 
   template<typename T> struct C {
     C() throw(typename T::type); // #cwg1330-C
@@ -500,7 +497,7 @@ namespace cwg1359 { // cwg1359: 3.5
   union B { constexpr B() = default; int a; }; // #cwg1359-B
   // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr before C++23}}
   union C { constexpr C() = default; int a, b; }; // #cwg1359-C
-  // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} 
+  // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}}
   struct X { constexpr X() = default; union {}; };
   // since-cxx11-error@-1 {{declaration does not declare anything}}
   struct Y { constexpr Y() = default; union { int a; }; }; // #cwg1359-Y
@@ -720,7 +717,7 @@ struct A {
 } // namespace cwg1397
 
 namespace cwg1399 { // cwg1399: dup 1388
-  template<typename ...T> void f(T..., int, T...) {} // #cwg1399-f 
+  template<typename ...T> void f(T..., int, T...) {} // #cwg1399-f
   // cxx98-error@-1 {{variadic templates are a C++11 extension}}
   void g() {
     f(0);
diff --git a/clang/test/CXX/except/except.spec/p13.cpp b/clang/test/CXX/except/except.spec/p13.cpp
index 61cdb74f21ec..29390c277c52 100644
--- a/clang/test/CXX/except/except.spec/p13.cpp
+++ b/clang/test/CXX/except/except.spec/p13.cpp
@@ -72,3 +72,30 @@ template<>
 void f(A, int***); // expected-error {{'f<A, int>' is missing exception specification 'noexcept'}}
 
 }
+
+namespace N3 {
+
+template<typename T, typename U>
+void f(T, U) noexcept(T::y); // #1
+
+template<typename T, typename U> // #2
+void f(T, U*) noexcept(T::x);
+
+// Deduction should succeed for both candidates, and #2 should be selected by overload resolution.
+// Only the exception specification of #2 should be instantiated.
+void (*x)(A, int*) = f;
+}
+
+namespace N4 {
+
+template<typename T, typename U>
+void f(T, U) noexcept(T::x); // #1
+
+template<typename T, typename U>
+void f(T, U*) noexcept(T::y); // #2
+// expected-error@-1 {{no member named 'y' in 'A'}}
+
+// Deduction should succeed for both candidates, and #2 should be selected by overload resolution.
+// Only the exception specification of #2 should be instantiated.
+void (*x)(A, int*) = f; // expected-note {{in instantiation of exception specification for 'f<A, int>' requested here}}
+}
diff --git a/clang/test/CXX/temp/temp.deduct/p7.cpp b/clang/test/CXX/temp/temp.deduct/p7.cpp
new file mode 100644
index 000000000000..cf6d17fc51ac
--- /dev/null
+++ b/clang/test/CXX/temp/temp.deduct/p7.cpp
@@ -0,0 +1,14 @@
+// RUN:  %clang_cc1 -verify %s
+
+struct A {
+  static constexpr bool x = true;
+};
+
+template<typename T, typename U>
+void f(T, U) noexcept(T::x);
+
+template<typename T, typename U>
+void f(T, U*) noexcept(T::y); // expected-error {{no member named 'y' in 'A'}}
+
+template<>
+void f<A>(A, int*); // expected-note {{in instantiation of exception specification}}
diff --git a/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp b/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp
index 5e56f19477d6..c8204c21523a 100644
--- a/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp
+++ b/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp
@@ -18,7 +18,7 @@ template<typename A, typename B> void redecl3() throw(B); // expected-error {{do
 
 typedef int I;
 template<bool B> void redecl4(I) noexcept(B);
-template<bool B> void redecl4(I) noexcept(B); // expected-note {{could not match 'void (I) noexcept(false)' (aka 'void (int) noexcept(false)') against 'void (int) noexcept'}}
+template<bool B> void redecl4(I) noexcept(B);
 
 void (*init_with_exact_type_a)(int) noexcept = redecl4<true>;
 void (*init_with_mismatched_type_a)(int) = redecl4<true>;
@@ -27,7 +27,7 @@ using DeducedType_a = decltype(deduce_auto_from_noexcept_function_ptr_a);
 using DeducedType_a = void (*)(int) noexcept;
 
 void (*init_with_exact_type_b)(int) = redecl4<false>;
-void (*init_with_mismatched_type_b)(int) noexcept = redecl4<false>; // expected-error {{does not match required type}}
+void (*init_with_mismatched_type_b)(int) noexcept = redecl4<false>; // expected-error {{cannot initialize a variable of type}}
 auto deduce_auto_from_noexcept_function_ptr_b = redecl4<false>;
 using DeducedType_b = decltype(deduce_auto_from_noexcept_function_ptr_b);
 using DeducedType_b = void (*)(int);
diff --git a/clang/test/SemaTemplate/temp_arg_type.cpp b/clang/test/SemaTemplate/temp_arg_type.cpp
index 9069f63e0224..cdbcf281125e 100644
--- a/clang/test/SemaTemplate/temp_arg_type.cpp
+++ b/clang/test/SemaTemplate/temp_arg_type.cpp
@@ -11,7 +11,7 @@ A<0> *a1; // expected-error{{template argument for template type parameter must
 A<A> *a2; // expected-error{{use of class template 'A' requires template arguments}}
 
 A<int> *a3;
-A<int()> *a4; 
+A<int()> *a4;
 A<int(float)> *a5;
 A<A<int> > *a6;
 
@@ -95,15 +95,13 @@ namespace deduce_noexcept {
   template void dep() noexcept(true); // expected-error {{does not refer to a function template}}
   template void dep() noexcept(false); // expected-error {{does not refer to a function template}}
 
-  // FIXME: It's also not clear whether this should be valid: do we substitute
-  // into the function type (including the exception specification) or not?
-  template<typename T> typename T::type1 f() noexcept(T::a);
-  template<typename T> typename T::type2 f() noexcept(T::b) {}
+  template<typename T> typename T::type1 f() noexcept(T::a); // expected-note {{candidate}}
+  template<typename T> typename T::type2 f() noexcept(T::b) {} // expected-note {{candidate}}
   struct X {
     static constexpr bool b = true;
     using type1 = void;
     using type2 = void;
   };
-  template void f<X>();
+  template void f<X>(); // expected-error {{partial ordering for explicit instantiation of 'f' is ambiguous}}
 }
 #endif
-- 
GitLab


From d4cf20ca37160cb062a9db773d0e6255d6bbc31a Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Tue, 7 May 2024 20:09:19 -0400
Subject: [PATCH 0117/1206] [Clang][Sema] Don't set instantiated from function
 when rewriting operator<=> (#91339)

The following snippet causes a crash:
```
template<typename T>
struct A
{
    bool operator<=>(const A&) const requires true = default;
};

bool f(A<int> a)
{
    return a != A<int>();
}
```
This occurs because during the rewrite from `operator<=>` to
`operator==`, the "pattern" `operator<=>` function is set as the
instantiated from function for the newly created `operator==` function.
This is obviously incorrect, and this patch fixes it.
---
 .../clangd/unittests/FindTargetTests.cpp      |  5 +---
 .../clangd/unittests/HoverTests.cpp           |  4 +--
 clang/docs/ReleaseNotes.rst                   |  2 ++
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  | 24 +++++++++--------
 .../class.compare.default/p4.cpp              | 27 ++++++++++++++++---
 5 files changed, 41 insertions(+), 21 deletions(-)

diff --git a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
index 94437857cecc..0b2273f0a9a6 100644
--- a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
@@ -642,10 +642,7 @@ TEST_F(TargetDeclTest, RewrittenBinaryOperator) {
     bool x = (Foo(1) [[!=]] Foo(2));
   )cpp";
   EXPECT_DECLS("CXXRewrittenBinaryOperator",
-               {"std::strong_ordering operator<=>(const Foo &) const = default",
-                Rel::TemplatePattern},
-               {"bool operator==(const Foo &) const noexcept = default",
-                Rel::TemplateInstantiation});
+               {"bool operator==(const Foo &) const noexcept = default"});
 }
 
 TEST_F(TargetDeclTest, FunctionTemplate) {
diff --git a/clang-tools-extra/clangd/unittests/HoverTests.cpp b/clang-tools-extra/clangd/unittests/HoverTests.cpp
index 28df24f34827..d9e97e5215a2 100644
--- a/clang-tools-extra/clangd/unittests/HoverTests.cpp
+++ b/clang-tools-extra/clangd/unittests/HoverTests.cpp
@@ -3091,7 +3091,7 @@ TEST(Hover, All) {
             HI.NamespaceScope = "";
             HI.Definition =
                 "bool operator==(const Foo &) const noexcept = default";
-            HI.Documentation = "Foo spaceship";
+            HI.Documentation = "";
           }},
   };
 
@@ -3894,7 +3894,7 @@ TEST(Hover, SpaceshipTemplateNoCrash) {
   TU.ExtraArgs.push_back("-std=c++20");
   auto AST = TU.build();
   auto HI = getHover(AST, T.point(), format::getLLVMStyle(), nullptr);
-  EXPECT_EQ(HI->Documentation, "Foo bar baz");
+  EXPECT_EQ(HI->Documentation, "");
 }
 
 TEST(Hover, ForwardStructNoCrash) {
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index c4a9501ca15c..c8ef2e8d614a 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -693,6 +693,8 @@ Bug Fixes to C++ Support
   and (#GH88832).
 - Clang now defers all substitution into the exception specification of a function template specialization
   until the noexcept-specifier is instantiated.
+- Fix a crash when an implicitly declared ``operator==`` function with a trailing requires-clause has its
+  constraints compared to that of another declaration.
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index d544cfac55ba..fde2d920c785 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -2269,16 +2269,18 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(
                             TemplateArgumentList::CreateCopy(SemaRef.Context,
                                                              Innermost),
                                                 /*InsertPos=*/nullptr);
-  } else if (isFriend && D->isThisDeclarationADefinition()) {
-    // Do not connect the friend to the template unless it's actually a
-    // definition. We don't want non-template functions to be marked as being
-    // template instantiations.
-    Function->setInstantiationOfMemberFunction(D, TSK_ImplicitInstantiation);
-  } else if (!isFriend) {
-    // If this is not a function template, and this is not a friend (that is,
-    // this is a locally declared function), save the instantiation relationship
-    // for the purposes of constraint instantiation.
-    Function->setInstantiatedFromDecl(D);
+  } else if (FunctionRewriteKind == RewriteKind::None) {
+    if (isFriend && D->isThisDeclarationADefinition()) {
+      // Do not connect the friend to the template unless it's actually a
+      // definition. We don't want non-template functions to be marked as being
+      // template instantiations.
+      Function->setInstantiationOfMemberFunction(D, TSK_ImplicitInstantiation);
+    } else if (!isFriend) {
+      // If this is not a function template, and this is not a friend (that is,
+      // this is a locally declared function), save the instantiation
+      // relationship for the purposes of constraint instantiation.
+      Function->setInstantiatedFromDecl(D);
+    }
   }
 
   if (isFriend) {
@@ -2669,7 +2671,7 @@ Decl *TemplateDeclInstantiator::VisitCXXMethodDecl(
                          TemplateArgumentList::CreateCopy(SemaRef.Context,
                                                           Innermost),
                                               /*InsertPos=*/nullptr);
-  } else if (!isFriend) {
+  } else if (!isFriend && FunctionRewriteKind == RewriteKind::None) {
     // Record that this is an instantiation of a member function.
     Method->setInstantiationOfMemberFunction(D, TSK_ImplicitInstantiation);
   }
diff --git a/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp b/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp
index 534c3b34d883..53a8bfc9a4f4 100644
--- a/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp
+++ b/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp
@@ -18,14 +18,22 @@ namespace std {
 
 namespace N {
   struct A {
-    friend constexpr std::strong_ordering operator<=>(const A&, const A&) = default;
+    friend constexpr std::strong_ordering operator<=>(const A&, const A&) = default; // expected-note 2{{declared here}}
   };
 
-  constexpr bool (*test_a_not_found)(const A&, const A&) = &operator==; // expected-error {{undeclared}}
+  constexpr std::strong_ordering (*test_a_threeway_not_found)(const A&, const A&) = &operator<=>; // expected-error {{undeclared}}
+
+  constexpr std::strong_ordering operator<=>(const A&, const A&) noexcept;
+  constexpr std::strong_ordering (*test_a_threeway)(const A&, const A&) = &operator<=>;
+  static_assert(!(*test_a_threeway)(A(), A())); // expected-error {{static assertion expression is not an integral constant expression}}
+                                               // expected-note@-1 {{undefined function 'operator<=>' cannot be used in a constant expression}}
+
+  constexpr bool (*test_a_equal_not_found)(const A&, const A&) = &operator==; // expected-error {{undeclared}}
 
   constexpr bool operator==(const A&, const A&) noexcept;
-  constexpr bool (*test_a)(const A&, const A&) noexcept = &operator==;
-  static_assert((*test_a)(A(), A()));
+  constexpr bool (*test_a_equal)(const A&, const A&) noexcept = &operator==;
+  static_assert((*test_a_equal)(A(), A())); // expected-error {{static assertion expression is not an integral constant expression}}
+                                            // expected-note@-1 {{undefined function 'operator==' cannot be used in a constant expression}}
 }
 
 struct B1 {
@@ -161,3 +169,14 @@ struct non_constexpr_type {
 
 my_struct<non_constexpr_type> obj; // cxx2a-note {{in instantiation of template class 'GH61238::my_struct<GH61238::non_constexpr_type>' requested here}}
 }
+
+namespace Constrained {
+  template<typename T>
+  struct A {
+    std::strong_ordering operator<=>(const A&) const requires true = default;
+  };
+
+  bool f(A<int> a) {
+    return a != A<int>();
+  }
+}
-- 
GitLab


From 83f3b1cb480b41e3347035aff14fd4bc2ba21d24 Mon Sep 17 00:00:00 2001
From: Yinying Li <yinyingli@google.com>
Date: Tue, 7 May 2024 20:28:39 -0400
Subject: [PATCH 0118/1206] [mlir][sparse] Add verification for
 explicit/implicit value (#90111)

1. Verify that the type of explicit/implicit values should be the same
as the tensor element type.
2. Verify that implicit value could only be zero.
3. Verify that explicit/implicit values should be numeric.
4. Fix the type change issue caused by SparseTensorType(enc).
---
 .../Dialect/SparseTensor/IR/SparseTensor.h    | 13 +++
 .../SparseTensor/IR/SparseTensorAttrDefs.td   | 15 ++++
 .../SparseTensor/IR/SparseTensorType.h        | 25 +-----
 .../SparseTensor/IR/SparseTensorDialect.cpp   | 78 +++++++++++------
 .../SparseTensor/invalid_encoding.mlir        | 85 +++++++++++++++++++
 5 files changed, 169 insertions(+), 47 deletions(-)

diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
index b182b4c72b95..3cf81d2e58f2 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
@@ -41,6 +41,19 @@ using Level = uint64_t;
 /// including the value `ShapedType::kDynamic` (for shapes).
 using Size = int64_t;
 
+/// A simple structure that encodes a range of levels in the sparse tensors
+/// that forms a COO segment.
+struct COOSegment {
+  std::pair<Level, Level> lvlRange; // [low, high)
+  bool isSoA;
+
+  bool isAoS() const { return !isSoA; }
+  bool isSegmentStart(Level l) const { return l == lvlRange.first; }
+  bool inSegment(Level l) const {
+    return l >= lvlRange.first && l < lvlRange.second;
+  }
+};
+
 } // namespace sparse_tensor
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
index eefa4c71bbd2..53dd8e39438c 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
@@ -502,9 +502,24 @@ def SparseTensorEncodingAttr : SparseTensor_Attr<"SparseTensorEncoding",
     //
     // Helper function to translate between level/dimension space.
     //
+
     SmallVector<int64_t> translateShape(::mlir::ArrayRef<int64_t> srcShape, ::mlir::sparse_tensor::CrdTransDirectionKind) const;
     ValueRange translateCrds(::mlir::OpBuilder &builder, ::mlir::Location loc, ::mlir::ValueRange crds, ::mlir::sparse_tensor::CrdTransDirectionKind) const;
 
+    //
+    // COO methods.
+    //
+
+    /// Returns the starting level of this sparse tensor type for a
+    /// trailing COO region that spans **at least** two levels. If
+    /// no such COO region is found, then returns the level-rank.
+    ///
+    /// DEPRECATED: use getCOOSegment instead;
+    Level getAoSCOOStart() const;
+
+    /// Returns a list of COO segments in the sparse tensor types.
+    SmallVector<COOSegment> getCOOSegments() const;
+
     //
     // Printing methods.
     //
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h
index ea3d8013b456..a154d7fa5fb6 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h
@@ -18,19 +18,6 @@
 namespace mlir {
 namespace sparse_tensor {
 
-/// A simple structure that encodes a range of levels in the sparse tensors that
-/// forms a COO segment.
-struct COOSegment {
-  std::pair<Level, Level> lvlRange; // [low, high)
-  bool isSoA;
-
-  bool isAoS() const { return !isSoA; }
-  bool isSegmentStart(Level l) const { return l == lvlRange.first; }
-  bool inSegment(Level l) const {
-    return l >= lvlRange.first && l < lvlRange.second;
-  }
-};
-
 //===----------------------------------------------------------------------===//
 /// A wrapper around `RankedTensorType`, which has three goals:
 ///
@@ -73,12 +60,6 @@ public:
       : SparseTensorType(
             RankedTensorType::get(stp.getShape(), stp.getElementType(), enc)) {}
 
-  // TODO: remove?
-  SparseTensorType(SparseTensorEncodingAttr enc)
-      : SparseTensorType(RankedTensorType::get(
-            SmallVector<Size>(enc.getDimRank(), ShapedType::kDynamic),
-            Float32Type::get(enc.getContext()), enc)) {}
-
   SparseTensorType &operator=(const SparseTensorType &) = delete;
   SparseTensorType(const SparseTensorType &) = default;
 
@@ -369,13 +350,15 @@ public:
   /// no such COO region is found, then returns the level-rank.
   ///
   /// DEPRECATED: use getCOOSegment instead;
-  Level getAoSCOOStart() const;
+  Level getAoSCOOStart() const { return getEncoding().getAoSCOOStart(); };
 
   /// Returns [un]ordered COO type for this sparse tensor type.
   RankedTensorType getCOOType(bool ordered) const;
 
   /// Returns a list of COO segments in the sparse tensor types.
-  SmallVector<COOSegment> getCOOSegments() const;
+  SmallVector<COOSegment> getCOOSegments() const {
+    return getEncoding().getCOOSegments();
+  }
 
 private:
   // These two must be const, to ensure coherence of the memoized fields.
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index de3d3006ebaa..4cc6ee971d4a 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -104,7 +104,7 @@ void StorageLayout::foreachField(
         callback) const {
   const auto lvlTypes = enc.getLvlTypes();
   const Level lvlRank = enc.getLvlRank();
-  SmallVector<COOSegment> cooSegs = SparseTensorType(enc).getCOOSegments();
+  SmallVector<COOSegment> cooSegs = enc.getCOOSegments();
   FieldIndex fieldIdx = kDataFieldStartingIdx;
 
   ArrayRef cooSegsRef = cooSegs;
@@ -211,7 +211,7 @@ StorageLayout::getFieldIndexAndStride(SparseTensorFieldKind kind,
   unsigned stride = 1;
   if (kind == SparseTensorFieldKind::CrdMemRef) {
     assert(lvl.has_value());
-    const Level cooStart = SparseTensorType(enc).getAoSCOOStart();
+    const Level cooStart = enc.getAoSCOOStart();
     const Level lvlRank = enc.getLvlRank();
     if (lvl.value() >= cooStart && lvl.value() < lvlRank) {
       lvl = cooStart;
@@ -912,46 +912,53 @@ LogicalResult SparseTensorEncodingAttr::verifyEncoding(
     return emitError()
            << "dimension-rank mismatch between encoding and tensor shape: "
            << getDimRank() << " != " << dimRank;
+  if (auto expVal = getExplicitVal()) {
+    Type attrType = llvm::dyn_cast<TypedAttr>(expVal).getType();
+    if (attrType != elementType) {
+      return emitError() << "explicit value type mismatch between encoding and "
+                         << "tensor element type: " << attrType
+                         << " != " << elementType;
+    }
+  }
+  if (auto impVal = getImplicitVal()) {
+    Type attrType = llvm::dyn_cast<TypedAttr>(impVal).getType();
+    if (attrType != elementType) {
+      return emitError() << "implicit value type mismatch between encoding and "
+                         << "tensor element type: " << attrType
+                         << " != " << elementType;
+    }
+    // Currently, we only support zero as the implicit value.
+    auto impFVal = llvm::dyn_cast<FloatAttr>(impVal);
+    auto impIntVal = llvm::dyn_cast<IntegerAttr>(impVal);
+    auto impComplexVal = llvm::dyn_cast<complex::NumberAttr>(impVal);
+    if ((impFVal && impFVal.getValue().isNonZero()) ||
+        (impIntVal && !impIntVal.getValue().isZero()) ||
+        (impComplexVal && (impComplexVal.getImag().isNonZero() ||
+                           impComplexVal.getReal().isNonZero()))) {
+      return emitError() << "implicit value must be zero";
+    }
+  }
   return success();
 }
 
-//===----------------------------------------------------------------------===//
-// SparseTensorType Methods.
-//===----------------------------------------------------------------------===//
-
-bool mlir::sparse_tensor::SparseTensorType::isCOOType(Level startLvl,
-                                                      bool isUnique) const {
-  if (!hasEncoding())
-    return false;
-  if (!isCompressedLvl(startLvl) && !isLooseCompressedLvl(startLvl))
-    return false;
-  for (Level l = startLvl + 1; l < lvlRank; ++l)
-    if (!isSingletonLvl(l))
-      return false;
-  // If isUnique is true, then make sure that the last level is unique,
-  // that is, when lvlRank == 1, the only compressed level is unique,
-  // and when lvlRank > 1, the last singleton is unique.
-  return !isUnique || isUniqueLvl(lvlRank - 1);
-}
-
-Level mlir::sparse_tensor::SparseTensorType::getAoSCOOStart() const {
+Level mlir::sparse_tensor::SparseTensorEncodingAttr::getAoSCOOStart() const {
   SmallVector<COOSegment> coo = getCOOSegments();
   assert(coo.size() == 1 || coo.empty());
   if (!coo.empty() && coo.front().isAoS()) {
     return coo.front().lvlRange.first;
   }
-  return lvlRank;
+  return getLvlRank();
 }
 
 SmallVector<COOSegment>
-mlir::sparse_tensor::SparseTensorType::getCOOSegments() const {
+mlir::sparse_tensor::SparseTensorEncodingAttr::getCOOSegments() const {
   SmallVector<COOSegment> ret;
-  if (!hasEncoding() || lvlRank <= 1)
+  if (getLvlRank() <= 1)
     return ret;
 
   ArrayRef<LevelType> lts = getLvlTypes();
   Level l = 0;
-  while (l < lvlRank) {
+  while (l < getLvlRank()) {
     auto lt = lts[l];
     if (lt.isa<LevelFormat::Compressed, LevelFormat::LooseCompressed>()) {
       auto cur = lts.begin() + l;
@@ -975,6 +982,25 @@ mlir::sparse_tensor::SparseTensorType::getCOOSegments() const {
   return ret;
 }
 
+//===----------------------------------------------------------------------===//
+// SparseTensorType Methods.
+//===----------------------------------------------------------------------===//
+
+bool mlir::sparse_tensor::SparseTensorType::isCOOType(Level startLvl,
+                                                      bool isUnique) const {
+  if (!hasEncoding())
+    return false;
+  if (!isCompressedLvl(startLvl) && !isLooseCompressedLvl(startLvl))
+    return false;
+  for (Level l = startLvl + 1; l < lvlRank; ++l)
+    if (!isSingletonLvl(l))
+      return false;
+  // If isUnique is true, then make sure that the last level is unique,
+  // that is, when lvlRank == 1, the only compressed level is unique,
+  // and when lvlRank > 1, the last singleton is unique.
+  return !isUnique || isUniqueLvl(lvlRank - 1);
+}
+
 RankedTensorType
 mlir::sparse_tensor::SparseTensorType::getCOOType(bool ordered) const {
   SmallVector<LevelType> lvlTypes;
diff --git a/mlir/test/Dialect/SparseTensor/invalid_encoding.mlir b/mlir/test/Dialect/SparseTensor/invalid_encoding.mlir
index 8096c010ac93..a3f72bd3ae97 100644
--- a/mlir/test/Dialect/SparseTensor/invalid_encoding.mlir
+++ b/mlir/test/Dialect/SparseTensor/invalid_encoding.mlir
@@ -443,3 +443,88 @@ func.func private @NOutOfM(%arg0: tensor<?x?x?xf64, #NOutOfM>) {
 func.func private @NOutOfM(%arg0: tensor<?x?x?xf64, #NOutOfM>) {
   return
 }
+
+// -----
+
+#CSR_ExpType = #sparse_tensor.encoding<{
+  map = (d0, d1) -> (d0 : dense, d1 : compressed),
+  posWidth = 32,
+  crdWidth = 32,
+  explicitVal = 1 : i32,
+  implicitVal = 0.0 : f32
+}>
+
+// expected-error@+1 {{explicit value type mismatch between encoding and tensor element type: 'i32' != 'f32'}}
+func.func private @sparse_csr(tensor<?x?xf32, #CSR_ExpType>)
+
+// -----
+
+#CSR_ImpType = #sparse_tensor.encoding<{
+  map = (d0, d1) -> (d0 : dense, d1 : compressed),
+  posWidth = 32,
+  crdWidth = 32,
+  explicitVal = 1 : i32,
+  implicitVal = 0.0 : f32
+}>
+
+// expected-error@+1 {{implicit value type mismatch between encoding and tensor element type: 'f32' != 'i32'}}
+func.func private @sparse_csr(tensor<?x?xi32, #CSR_ImpType>)
+
+// -----
+
+// expected-error@+1 {{expected a numeric value for explicitVal}}
+#CSR_ExpType = #sparse_tensor.encoding<{
+  map = (d0, d1) -> (d0 : dense, d1 : compressed),
+  posWidth = 32,
+  crdWidth = 32,
+  explicitVal = "str"
+}>
+func.func private @sparse_csr(tensor<?x?xi32, #CSR_ExpType>)
+
+// -----
+
+// expected-error@+1 {{expected a numeric value for implicitVal}}
+#CSR_ImpType = #sparse_tensor.encoding<{
+  map = (d0, d1) -> (d0 : dense, d1 : compressed),
+  posWidth = 32,
+  crdWidth = 32,
+  implicitVal = "str"
+}>
+func.func private @sparse_csr(tensor<?x?xi32, #CSR_ImpType>)
+
+// -----
+
+#CSR_ImpVal = #sparse_tensor.encoding<{
+  map = (d0, d1) -> (d0 : dense, d1 : compressed),
+  posWidth = 32,
+  crdWidth = 32,
+  implicitVal = 1 : i32
+}>
+
+// expected-error@+1 {{implicit value must be zero}}
+func.func private @sparse_csr(tensor<?x?xi32, #CSR_ImpVal>)
+
+// -----
+
+#CSR_ImpVal = #sparse_tensor.encoding<{
+  map = (d0, d1) -> (d0 : dense, d1 : compressed),
+  posWidth = 32,
+  crdWidth = 32,
+  implicitVal = 1.0 : f32
+}>
+
+// expected-error@+1 {{implicit value must be zero}}
+func.func private @sparse_csr(tensor<?x?xf32, #CSR_ImpVal>)
+
+// -----
+
+#CSR_OnlyOnes = #sparse_tensor.encoding<{
+  map = (d0, d1) -> (d0 : dense, d1 : compressed),
+  posWidth = 64,
+  crdWidth = 64,
+  explicitVal = #complex.number<:f32 1.0, 0.0>,
+  implicitVal = #complex.number<:f32 1.0, 0.0>
+}>
+
+// expected-error@+1 {{implicit value must be zero}}
+func.func private @sparse_csr(tensor<?x?xcomplex<f32>, #CSR_OnlyOnes>)
-- 
GitLab


From 34ae2265e88c8a04350de5a244d0d888e74a8388 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Tue, 7 May 2024 21:41:33 -0400
Subject: [PATCH 0119/1206] [Clang][Sema] Improve support for explicit
 specializations of constrained member functions & member function templates
 (#88963)

Consider the following snippet from the discussion of CWG2847 on the core reflector:
```
template<typename T>
concept C = sizeof(T) <= sizeof(long);

template<typename T>
struct A
{
    template<typename U>
    void f(U) requires C<U>; // #1, declares a function template

    void g() requires C<T>; // #2, declares a function

    template<>
    void f(char);  // #3, an explicit specialization of a function template that declares a function
};

template<>
template<typename U>
void A<short>::f(U) requires C<U>; // #4, an explicit specialization of a function template that declares a function template

template<>
template<>
void A<int>::f(int); // #5, an explicit specialization of a function template that declares a function

template<>
void A<long>::g(); // #6, an explicit specialization of a function that declares a function
```

A number of problems exist:
- Clang rejects `#4` because the trailing _requires-clause_ has `U`
substituted with the wrong template parameter depth when
`Sema::AreConstraintExpressionsEqual` is called to determine whether it
matches the trailing _requires-clause_ of the implicitly instantiated
function template.
- Clang rejects `#5` because the function template specialization
instantiated from `A<int>::f` has a trailing _requires-clause_, but `#5`
does not (nor can it have one as it isn't a templated function).
- Clang rejects `#6` for the same reasons it rejects `#5`.

This patch resolves these issues by making the following changes:
- To fix `#4`, `Sema::AreConstraintExpressionsEqual` is passed
`FunctionTemplateDecl`s when comparing the trailing _requires-clauses_
of `#4` and the function template instantiated from `#1`.
- To fix `#5` and `#6`, the trailing _requires-clauses_ are not compared
for explicit specializations that declare functions.

In addition to these changes, `CheckMemberSpecialization` now considers
constraint satisfaction/constraint partial ordering when determining
which member function is specialized by an explicit specialization of a
member function for an implicit instantiation of a class template (we
previously would select the first function that has the same type as the
explicit specialization). With constraints taken under consideration, we
match EDG's behavior for these declarations.
---
 clang/docs/ReleaseNotes.rst                   |  4 +
 .../clang/Basic/DiagnosticSemaKinds.td        |  5 ++
 clang/include/clang/Sema/Sema.h               |  3 +
 clang/lib/Sema/SemaConcept.cpp                |  2 +-
 clang/lib/Sema/SemaOverload.cpp               | 72 +++++-------------
 clang/lib/Sema/SemaTemplate.cpp               | 57 ++++++++++----
 clang/lib/Sema/SemaTemplateDeduction.cpp      | 32 ++++++++
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |  7 ++
 .../temp/temp.spec/temp.expl.spec/p14-23.cpp  | 60 +++++++++++++++
 .../CXX/temp/temp.spec/temp.expl.spec/p8.cpp  | 74 +++++++++++++++++++
 10 files changed, 248 insertions(+), 68 deletions(-)
 create mode 100644 clang/test/CXX/temp/temp.spec/temp.expl.spec/p14-23.cpp
 create mode 100644 clang/test/CXX/temp/temp.spec/temp.expl.spec/p8.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index c8ef2e8d614a..0f9728c00e64 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -695,6 +695,10 @@ Bug Fixes to C++ Support
   until the noexcept-specifier is instantiated.
 - Fix a crash when an implicitly declared ``operator==`` function with a trailing requires-clause has its
   constraints compared to that of another declaration.
+- Fix a bug where explicit specializations of member functions/function templates would have substitution
+  performed incorrectly when checking constraints. Fixes (#GH90349).
+- Clang now allows constrained member functions to be explicitly specialized for an implicit instantiation
+  of a class template.
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 9a0bae9c216d..9317ae675c72 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -5437,6 +5437,11 @@ def note_function_template_spec_matched : Note<
 def err_function_template_partial_spec : Error<
     "function template partial specialization is not allowed">;
 
+def err_function_member_spec_ambiguous : Error<
+    "ambiguous member function specialization %q0 of %q1">;
+def note_function_member_spec_matched : Note<
+    "member function specialization matches %0">;
+
 // C++ Template Instantiation
 def err_template_recursion_depth_exceeded : Error<
   "recursive template instantiation exceeded maximum depth of %0">,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index a80ac6dbc761..ddb3de2b6602 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -9739,6 +9739,9 @@ public:
                      const PartialDiagnostic &CandidateDiag,
                      bool Complain = true, QualType TargetType = QualType());
 
+  FunctionDecl *getMoreConstrainedFunction(FunctionDecl *FD1,
+                                           FunctionDecl *FD2);
+
   ///@}
 
   //
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index e00c97260282..7bfec4e11f7a 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -811,7 +811,7 @@ static const Expr *SubstituteConstraintExpressionWithoutSatisfaction(
   // this may happen while we're comparing two templates' constraint
   // equivalence.
   LocalInstantiationScope ScopeForParameters(S);
-  if (auto *FD = llvm::dyn_cast<FunctionDecl>(DeclInfo.getDecl()))
+  if (auto *FD = DeclInfo.getDecl()->getAsFunction())
     for (auto *PVD : FD->parameters())
       ScopeForParameters.InstantiatedLocal(PVD, PVD);
 
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index a416df2e97c4..f173300b5c96 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1303,6 +1303,8 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
   if (New->isMSVCRTEntryPoint())
     return false;
 
+  NamedDecl *OldDecl = Old;
+  NamedDecl *NewDecl = New;
   FunctionTemplateDecl *OldTemplate = Old->getDescribedFunctionTemplate();
   FunctionTemplateDecl *NewTemplate = New->getDescribedFunctionTemplate();
 
@@ -1347,6 +1349,8 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
   // references to non-instantiated entities during constraint substitution.
   // GH78101.
   if (NewTemplate) {
+    OldDecl = OldTemplate;
+    NewDecl = NewTemplate;
     // C++ [temp.over.link]p4:
     //   The signature of a function template consists of its function
     //   signature, its return type and its template parameter list. The names
@@ -1506,13 +1510,14 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
     }
   }
 
-  if (!UseOverrideRules) {
+  if (!UseOverrideRules &&
+      New->getTemplateSpecializationKind() != TSK_ExplicitSpecialization) {
     Expr *NewRC = New->getTrailingRequiresClause(),
          *OldRC = Old->getTrailingRequiresClause();
     if ((NewRC != nullptr) != (OldRC != nullptr))
       return true;
-
-    if (NewRC && !SemaRef.AreConstraintExpressionsEqual(Old, OldRC, New, NewRC))
+    if (NewRC &&
+        !SemaRef.AreConstraintExpressionsEqual(OldDecl, OldRC, NewDecl, NewRC))
       return true;
   }
 
@@ -10695,29 +10700,10 @@ bool clang::isBetterOverloadCandidate(
   //   -— F1 and F2 are non-template functions with the same
   //      parameter-type-lists, and F1 is more constrained than F2 [...],
   if (!Cand1IsSpecialization && !Cand2IsSpecialization &&
-      sameFunctionParameterTypeLists(S, Cand1, Cand2)) {
-    FunctionDecl *Function1 = Cand1.Function;
-    FunctionDecl *Function2 = Cand2.Function;
-    if (FunctionDecl *MF = Function1->getInstantiatedFromMemberFunction())
-      Function1 = MF;
-    if (FunctionDecl *MF = Function2->getInstantiatedFromMemberFunction())
-      Function2 = MF;
-
-    const Expr *RC1 = Function1->getTrailingRequiresClause();
-    const Expr *RC2 = Function2->getTrailingRequiresClause();
-    if (RC1 && RC2) {
-      bool AtLeastAsConstrained1, AtLeastAsConstrained2;
-      if (S.IsAtLeastAsConstrained(Function1, RC1, Function2, RC2,
-                                   AtLeastAsConstrained1) ||
-          S.IsAtLeastAsConstrained(Function2, RC2, Function1, RC1,
-                                   AtLeastAsConstrained2))
-        return false;
-      if (AtLeastAsConstrained1 != AtLeastAsConstrained2)
-        return AtLeastAsConstrained1;
-    } else if (RC1 || RC2) {
-      return RC1 != nullptr;
-    }
-  }
+      sameFunctionParameterTypeLists(S, Cand1, Cand2) &&
+      S.getMoreConstrainedFunction(Cand1.Function, Cand2.Function) ==
+          Cand1.Function)
+    return true;
 
   //   -- F1 is a constructor for a class D, F2 is a constructor for a base
   //      class B of D, and for all arguments the corresponding parameters of
@@ -13385,25 +13371,6 @@ Sema::resolveAddressOfSingleOverloadCandidate(Expr *E, DeclAccessPair &Pair) {
            static_cast<int>(CUDA().IdentifyPreference(Caller, FD2));
   };
 
-  auto CheckMoreConstrained = [&](FunctionDecl *FD1,
-                                  FunctionDecl *FD2) -> std::optional<bool> {
-    if (FunctionDecl *MF = FD1->getInstantiatedFromMemberFunction())
-      FD1 = MF;
-    if (FunctionDecl *MF = FD2->getInstantiatedFromMemberFunction())
-      FD2 = MF;
-    SmallVector<const Expr *, 1> AC1, AC2;
-    FD1->getAssociatedConstraints(AC1);
-    FD2->getAssociatedConstraints(AC2);
-    bool AtLeastAsConstrained1, AtLeastAsConstrained2;
-    if (IsAtLeastAsConstrained(FD1, AC1, FD2, AC2, AtLeastAsConstrained1))
-      return std::nullopt;
-    if (IsAtLeastAsConstrained(FD2, AC2, FD1, AC1, AtLeastAsConstrained2))
-      return std::nullopt;
-    if (AtLeastAsConstrained1 == AtLeastAsConstrained2)
-      return std::nullopt;
-    return AtLeastAsConstrained1;
-  };
-
   // Don't use the AddressOfResolver because we're specifically looking for
   // cases where we have one overload candidate that lacks
   // enable_if/pass_object_size/...
@@ -13440,15 +13407,14 @@ Sema::resolveAddressOfSingleOverloadCandidate(Expr *E, DeclAccessPair &Pair) {
       }
       // FD has the same CUDA prefernece than Result. Continue check
       // constraints.
-      std::optional<bool> MoreConstrainedThanPrevious =
-          CheckMoreConstrained(FD, Result);
-      if (!MoreConstrainedThanPrevious) {
-        IsResultAmbiguous = true;
-        AmbiguousDecls.push_back(FD);
+      FunctionDecl *MoreConstrained = getMoreConstrainedFunction(FD, Result);
+      if (MoreConstrained != FD) {
+        if (!MoreConstrained) {
+          IsResultAmbiguous = true;
+          AmbiguousDecls.push_back(FD);
+        }
         continue;
       }
-      if (!*MoreConstrainedThanPrevious)
-        continue;
       // FD is more constrained - replace Result with it.
     }
     FoundBetter();
@@ -13467,7 +13433,7 @@ Sema::resolveAddressOfSingleOverloadCandidate(Expr *E, DeclAccessPair &Pair) {
       // constraints.
       if (getLangOpts().CUDA && CheckCUDAPreference(Skipped, Result) != 0)
         continue;
-      if (!CheckMoreConstrained(Skipped, Result))
+      if (!getMoreConstrainedFunction(Skipped, Result))
         return nullptr;
     }
     Pair = DAP;
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 5c72270ff150..7e57fa069672 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -10339,24 +10339,53 @@ Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
   if (Previous.empty()) {
     // Nowhere to look anyway.
   } else if (FunctionDecl *Function = dyn_cast<FunctionDecl>(Member)) {
+    SmallVector<FunctionDecl *> Candidates;
+    bool Ambiguous = false;
     for (LookupResult::iterator I = Previous.begin(), E = Previous.end();
            I != E; ++I) {
-      NamedDecl *D = (*I)->getUnderlyingDecl();
-      if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D)) {
-        QualType Adjusted = Function->getType();
-        if (!hasExplicitCallingConv(Adjusted))
-          Adjusted = adjustCCAndNoReturn(Adjusted, Method->getType());
-        // This doesn't handle deduced return types, but both function
-        // declarations should be undeduced at this point.
-        if (Context.hasSameType(Adjusted, Method->getType())) {
-          FoundInstantiation = *I;
-          Instantiation = Method;
-          InstantiatedFrom = Method->getInstantiatedFromMemberFunction();
-          MSInfo = Method->getMemberSpecializationInfo();
-          break;
-        }
+      CXXMethodDecl *Method =
+          dyn_cast<CXXMethodDecl>((*I)->getUnderlyingDecl());
+      if (!Method)
+        continue;
+      QualType Adjusted = Function->getType();
+      if (!hasExplicitCallingConv(Adjusted))
+        Adjusted = adjustCCAndNoReturn(Adjusted, Method->getType());
+      // This doesn't handle deduced return types, but both function
+      // declarations should be undeduced at this point.
+      if (!Context.hasSameType(Adjusted, Method->getType()))
+        continue;
+      if (ConstraintSatisfaction Satisfaction;
+          Method->getTrailingRequiresClause() &&
+          (CheckFunctionConstraints(Method, Satisfaction,
+                                    /*UsageLoc=*/Member->getLocation(),
+                                    /*ForOverloadResolution=*/true) ||
+           !Satisfaction.IsSatisfied))
+        continue;
+      Candidates.push_back(Method);
+      FunctionDecl *MoreConstrained =
+          Instantiation ? getMoreConstrainedFunction(
+                              Method, cast<FunctionDecl>(Instantiation))
+                        : Method;
+      if (!MoreConstrained) {
+        Ambiguous = true;
+        continue;
+      }
+      if (MoreConstrained == Method) {
+        Ambiguous = false;
+        FoundInstantiation = *I;
+        Instantiation = Method;
+        InstantiatedFrom = Method->getInstantiatedFromMemberFunction();
+        MSInfo = Method->getMemberSpecializationInfo();
       }
     }
+    if (Ambiguous) {
+      Diag(Member->getLocation(), diag::err_function_member_spec_ambiguous)
+          << Member << (InstantiatedFrom ? InstantiatedFrom : Instantiation);
+      for (FunctionDecl *Candidate : Candidates)
+        Diag(Candidate->getLocation(), diag::note_function_member_spec_matched)
+            << Candidate;
+      return true;
+    }
   } else if (isa<VarDecl>(Member)) {
     VarDecl *PrevVar;
     if (Previous.isSingleResult() &&
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index dcaea4a77bff..fe7e35d84151 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -5852,6 +5852,38 @@ UnresolvedSetIterator Sema::getMostSpecialized(
   return SpecEnd;
 }
 
+/// Returns the more constrained function according to the rules of
+/// partial ordering by constraints (C++ [temp.constr.order]).
+///
+/// \param FD1 the first function
+///
+/// \param FD2 the second function
+///
+/// \returns the more constrained function. If neither function is
+/// more constrained, returns NULL.
+FunctionDecl *Sema::getMoreConstrainedFunction(FunctionDecl *FD1,
+                                               FunctionDecl *FD2) {
+  assert(!FD1->getDescribedTemplate() && !FD2->getDescribedTemplate() &&
+         "not for function templates");
+  FunctionDecl *F1 = FD1;
+  if (FunctionDecl *MF = FD1->getInstantiatedFromMemberFunction())
+    F1 = MF;
+  FunctionDecl *F2 = FD2;
+  if (FunctionDecl *MF = FD2->getInstantiatedFromMemberFunction())
+    F2 = MF;
+  llvm::SmallVector<const Expr *, 1> AC1, AC2;
+  F1->getAssociatedConstraints(AC1);
+  F2->getAssociatedConstraints(AC2);
+  bool AtLeastAsConstrained1, AtLeastAsConstrained2;
+  if (IsAtLeastAsConstrained(F1, AC1, F2, AC2, AtLeastAsConstrained1))
+    return nullptr;
+  if (IsAtLeastAsConstrained(F2, AC2, F1, AC1, AtLeastAsConstrained2))
+    return nullptr;
+  if (AtLeastAsConstrained1 == AtLeastAsConstrained2)
+    return nullptr;
+  return AtLeastAsConstrained1 ? FD1 : FD2;
+}
+
 /// Determine whether one partial specialization, P1, is at least as
 /// specialized than another, P2.
 ///
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 3a9fd906b7af..07626058c797 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -275,6 +275,13 @@ Response HandleFunction(Sema &SemaRef, const FunctionDecl *Function,
                                      TemplateArgs->asArray(),
                                      /*Final=*/false);
 
+    if (RelativeToPrimary &&
+        (Function->getTemplateSpecializationKind() ==
+             TSK_ExplicitSpecialization ||
+         (Function->getFriendObjectKind() &&
+          !Function->getPrimaryTemplate()->getFriendObjectKind())))
+      return Response::UseNextDecl(Function);
+
     // If this function was instantiated from a specialized member that is
     // a function template, we're done.
     assert(Function->getPrimaryTemplate() && "No function template?");
diff --git a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p14-23.cpp b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p14-23.cpp
new file mode 100644
index 000000000000..dc17cea99d43
--- /dev/null
+++ b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p14-23.cpp
@@ -0,0 +1,60 @@
+// RUN: %clang_cc1 -std=c++20 -verify %s
+
+template<int I>
+concept C = I >= 4;
+
+template<int I>
+concept D = I < 8;
+
+template<int I>
+struct A {
+  constexpr static int f() { return 0; }
+  constexpr static int f() requires C<I> && D<I> { return 1; }
+  constexpr static int f() requires C<I> { return 2; }
+
+  constexpr static int g() requires C<I> { return 0; } // #candidate-0
+  constexpr static int g() requires D<I> { return 1; } // #candidate-1
+
+  constexpr static int h() requires C<I> { return 0; } // expected-note {{member declaration nearly matches}}
+};
+
+template<>
+constexpr int A<2>::f() { return 3; }
+
+template<>
+constexpr int A<4>::f() { return 4; }
+
+template<>
+constexpr int A<8>::f() { return 5; }
+
+static_assert(A<3>::f() == 0);
+static_assert(A<5>::f() == 1);
+static_assert(A<9>::f() == 2);
+static_assert(A<2>::f() == 3);
+static_assert(A<4>::f() == 4);
+static_assert(A<8>::f() == 5);
+
+template<>
+constexpr int A<0>::g() { return 2; }
+
+template<>
+constexpr int A<8>::g() { return 3; }
+
+template<>
+constexpr int A<6>::g() { return 4; } // expected-error {{ambiguous member function specialization 'A<6>::g' of 'A::g'}}
+                                      // expected-note@#candidate-0 {{member function specialization matches 'g'}}
+                                      // expected-note@#candidate-1 {{member function specialization matches 'g'}}
+
+static_assert(A<9>::g() == 0);
+static_assert(A<1>::g() == 1);
+static_assert(A<0>::g() == 2);
+static_assert(A<8>::g() == 3);
+
+template<>
+constexpr int A<4>::h() { return 1; }
+
+template<>
+constexpr int A<0>::h() { return 2; } // expected-error {{out-of-line definition of 'h' does not match any declaration in 'A<0>'}}
+
+static_assert(A<5>::h() == 0);
+static_assert(A<4>::h() == 1);
diff --git a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p8.cpp b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p8.cpp
new file mode 100644
index 000000000000..87e10d10e4b4
--- /dev/null
+++ b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p8.cpp
@@ -0,0 +1,74 @@
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+template<typename T>
+concept C = sizeof(T) <= sizeof(long);
+
+template<typename T>
+struct A {
+  template<typename U>
+  void f(U) requires C<U>;
+
+  void g() requires C<T>;
+
+  template<typename U>
+  void h(U) requires C<T>;
+
+  constexpr int i() requires C<T> {
+    return 0;
+  }
+
+  constexpr int i() requires C<T> && true {
+    return 1;
+  }
+
+  template<>
+  void f(char);
+};
+
+template<>
+template<typename U>
+void A<short>::f(U) requires C<U>;
+
+template<>
+template<typename U>
+void A<short>::h(U) requires C<short>;
+
+template<>
+template<>
+void A<int>::f(int);
+
+template<>
+void A<long>::g();
+
+template<>
+constexpr int A<long>::i() {
+  return 2;
+}
+
+static_assert(A<long>().i() == 2);
+
+template<typename T>
+struct D {
+  template<typename U>
+  static constexpr int f(U);
+
+  template<typename U>
+  static constexpr int f(U) requires (sizeof(T) == 1);
+
+  template<>
+  constexpr int f(int) {
+    return 1;
+  }
+};
+
+template<>
+template<typename U>
+constexpr int D<signed char>::f(U) requires (sizeof(signed char) == 1) {
+  return 0;
+}
+
+static_assert(D<char>::f(0) == 1);
+static_assert(D<char[2]>::f(0) == 1);
+static_assert(D<signed char>::f(0) == 1);
+static_assert(D<signed char>::f(0.0) == 0);
-- 
GitLab


From 584253c4e2f788f870488fc32193b52d67ddaccc Mon Sep 17 00:00:00 2001
From: Benji Smith <6193112+Benjins@users.noreply.github.com>
Date: Tue, 7 May 2024 21:59:53 -0400
Subject: [PATCH 0120/1206] [C API] Add getters and build function for CallBr
 (#91154)

This adds LLVMBuildCallBr to create CallBr instructions, and getters for
the CallBr-specific data. The remainder of its data, e.g.
arguments/function, can be accessed using existing getters.
---
 llvm/docs/ReleaseNotes.rst        |  7 ++++++
 llvm/include/llvm-c/Core.h        | 28 ++++++++++++++++++++++
 llvm/lib/IR/Core.cpp              | 35 +++++++++++++++++++++++++++
 llvm/test/Bindings/llvm-c/echo.ll | 26 ++++++++++++++++++++
 llvm/tools/llvm-c-test/echo.cpp   | 40 +++++++++++++++++++++++++++++++
 5 files changed, 136 insertions(+)

diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 9deae46d0233..26f1d33f6800 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -168,6 +168,13 @@ Changes to the C API
 
 * Added ``LLVMCreateConstantRangeAttribute`` function for creating ConstantRange Attributes.
 
+* Added the following functions for creating and accessing data for CallBr instructions:
+
+  * ``LLVMBuildCallBr``
+  * ``LLVMGetCallBrDefaultDest``
+  * ``LLVMGetCallBrNumIndirectDests``
+  * ``LLVMGetCallBrIndirectDest``
+
 Changes to the CodeGen infrastructure
 -------------------------------------
 
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index ba02ca482575..9d09546513f0 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -3737,6 +3737,28 @@ void LLVMSetNormalDest(LLVMValueRef InvokeInst, LLVMBasicBlockRef B);
  */
 void LLVMSetUnwindDest(LLVMValueRef InvokeInst, LLVMBasicBlockRef B);
 
+/**
+ * Get the default destination of a CallBr instruction.
+ *
+ * @see llvm::CallBrInst::getDefaultDest()
+ */
+LLVMBasicBlockRef LLVMGetCallBrDefaultDest(LLVMValueRef CallBr);
+
+/**
+ * Get the number of indirect destinations of a CallBr instruction.
+ *
+ * @see llvm::CallBrInst::getNumIndirectDests()
+
+ */
+unsigned LLVMGetCallBrNumIndirectDests(LLVMValueRef CallBr);
+
+/**
+ * Get the indirect destination of a CallBr instruction at the given index.
+ *
+ * @see llvm::CallBrInst::getIndirectDest()
+ */
+LLVMBasicBlockRef LLVMGetCallBrIndirectDest(LLVMValueRef CallBr, unsigned Idx);
+
 /**
  * @}
  */
@@ -4023,6 +4045,12 @@ LLVMValueRef LLVMBuildSwitch(LLVMBuilderRef, LLVMValueRef V,
                              LLVMBasicBlockRef Else, unsigned NumCases);
 LLVMValueRef LLVMBuildIndirectBr(LLVMBuilderRef B, LLVMValueRef Addr,
                                  unsigned NumDests);
+LLVMValueRef LLVMBuildCallBr(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
+                             LLVMBasicBlockRef DefaultDest,
+                             LLVMBasicBlockRef *IndirectDests,
+                             unsigned NumIndirectDests, LLVMValueRef *Args,
+                             unsigned NumArgs, LLVMOperandBundleRef *Bundles,
+                             unsigned NumBundles, const char *Name);
 LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Fn,
                               LLVMValueRef *Args, unsigned NumArgs,
                               LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 1b84527d5e87..df90b8834112 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -47,6 +47,10 @@ using namespace llvm;
 
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OperandBundleDef, LLVMOperandBundleRef)
 
+inline BasicBlock **unwrap(LLVMBasicBlockRef *BBs) {
+  return reinterpret_cast<BasicBlock **>(BBs);
+}
+
 #define DEBUG_TYPE "ir"
 
 void llvm::initializeCore(PassRegistry &Registry) {
@@ -3031,6 +3035,18 @@ void LLVMSetUnwindDest(LLVMValueRef Invoke, LLVMBasicBlockRef B) {
   unwrap<InvokeInst>(Invoke)->setUnwindDest(unwrap(B));
 }
 
+LLVMBasicBlockRef LLVMGetCallBrDefaultDest(LLVMValueRef CallBr) {
+  return wrap(unwrap<CallBrInst>(CallBr)->getDefaultDest());
+}
+
+unsigned LLVMGetCallBrNumIndirectDests(LLVMValueRef CallBr) {
+  return unwrap<CallBrInst>(CallBr)->getNumIndirectDests();
+}
+
+LLVMBasicBlockRef LLVMGetCallBrIndirectDest(LLVMValueRef CallBr, unsigned Idx) {
+  return wrap(unwrap<CallBrInst>(CallBr)->getIndirectDest(Idx));
+}
+
 /*--.. Operations on terminators ...........................................--*/
 
 unsigned LLVMGetNumSuccessors(LLVMValueRef Term) {
@@ -3258,6 +3274,25 @@ LLVMValueRef LLVMBuildIndirectBr(LLVMBuilderRef B, LLVMValueRef Addr,
   return wrap(unwrap(B)->CreateIndirectBr(unwrap(Addr), NumDests));
 }
 
+LLVMValueRef LLVMBuildCallBr(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
+                             LLVMBasicBlockRef DefaultDest,
+                             LLVMBasicBlockRef *IndirectDests,
+                             unsigned NumIndirectDests, LLVMValueRef *Args,
+                             unsigned NumArgs, LLVMOperandBundleRef *Bundles,
+                             unsigned NumBundles, const char *Name) {
+
+  SmallVector<OperandBundleDef, 8> OBs;
+  for (auto *Bundle : ArrayRef(Bundles, NumBundles)) {
+    OperandBundleDef *OB = unwrap(Bundle);
+    OBs.push_back(*OB);
+  }
+
+  return wrap(unwrap(B)->CreateCallBr(
+      unwrap<FunctionType>(Ty), unwrap(Fn), unwrap(DefaultDest),
+      ArrayRef(unwrap(IndirectDests), NumIndirectDests),
+      ArrayRef<Value *>(unwrap(Args), NumArgs), OBs, Name));
+}
+
 LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
                               LLVMValueRef *Args, unsigned NumArgs,
                               LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
diff --git a/llvm/test/Bindings/llvm-c/echo.ll b/llvm/test/Bindings/llvm-c/echo.ll
index 953a16b7e624..bb5fae0dcd12 100644
--- a/llvm/test/Bindings/llvm-c/echo.ll
+++ b/llvm/test/Bindings/llvm-c/echo.ll
@@ -348,6 +348,32 @@ define void @test_func_prologue_data_01() prologue %func_prolog_struct <{ i8 235
   ret void
 }
 
+
+define void @test_call_br_01(i32 %input) {
+entry:
+  callbr void asm "nop", "r,!i"(i32 %input) to label %bb_01 [label %bb_02]
+
+bb_01:
+  ret void
+bb_02:
+  ret void
+}
+
+define void @test_call_br_02(i32 %input0, i32 %input1) {
+entry:
+  ; Multiple indirect destinations, operand bundles, and arguments
+  callbr void asm "nop", "r,r,!i,!i"(i32 %input0, i32 %input1)
+    ["op0"(i32 %input1), "op1"(label %bb_02)]
+    to label %bb_01 [label %bb_03, label %bb_02]
+
+bb_01:
+  ret void
+bb_02:
+  ret void
+bb_03:
+  ret void
+}
+
 !llvm.dbg.cu = !{!0, !2}
 !llvm.module.flags = !{!3}
 
diff --git a/llvm/tools/llvm-c-test/echo.cpp b/llvm/tools/llvm-c-test/echo.cpp
index 347863638849..518716168c42 100644
--- a/llvm/tools/llvm-c-test/echo.cpp
+++ b/llvm/tools/llvm-c-test/echo.cpp
@@ -570,6 +570,46 @@ struct FunCloner {
           LLVMDisposeOperandBundle(Bundle);
         break;
       }
+      case LLVMCallBr: {
+        LLVMTypeRef FnTy = CloneType(LLVMGetCalledFunctionType(Src));
+        LLVMValueRef Fn = CloneValue(LLVMGetCalledValue(Src));
+
+        LLVMBasicBlockRef DefaultDest =
+            DeclareBB(LLVMGetCallBrDefaultDest(Src));
+
+        // Clone indirect destinations
+        SmallVector<LLVMBasicBlockRef, 8> IndirectDests;
+        unsigned IndirectDestCount = LLVMGetCallBrNumIndirectDests(Src);
+        for (unsigned i = 0; i < IndirectDestCount; ++i)
+          IndirectDests.push_back(DeclareBB(LLVMGetCallBrIndirectDest(Src, i)));
+
+        // Clone input arguments
+        SmallVector<LLVMValueRef, 8> Args;
+        unsigned ArgCount = LLVMGetNumArgOperands(Src);
+        for (unsigned i = 0; i < ArgCount; ++i)
+          Args.push_back(CloneValue(LLVMGetOperand(Src, i)));
+
+        // Clone operand bundles
+        SmallVector<LLVMOperandBundleRef, 8> Bundles;
+        unsigned BundleCount = LLVMGetNumOperandBundles(Src);
+        for (unsigned i = 0; i < BundleCount; ++i) {
+          auto Bundle = LLVMGetOperandBundleAtIndex(Src, i);
+          Bundles.push_back(CloneOB(Bundle));
+          LLVMDisposeOperandBundle(Bundle);
+        }
+
+        Dst = LLVMBuildCallBr(Builder, FnTy, Fn, DefaultDest,
+                              IndirectDests.data(), IndirectDests.size(),
+                              Args.data(), Args.size(), Bundles.data(),
+                              Bundles.size(), Name);
+
+        CloneAttrs(Src, Dst);
+
+        for (auto Bundle : Bundles)
+          LLVMDisposeOperandBundle(Bundle);
+
+        break;
+      }
       case LLVMUnreachable:
         Dst = LLVMBuildUnreachable(Builder);
         break;
-- 
GitLab


From c4e5a8a4d3ef0948384d9411ea1e44fc113e5b5c Mon Sep 17 00:00:00 2001
From: Aart Bik <ajcbik@google.com>
Date: Tue, 7 May 2024 19:01:36 -0700
Subject: [PATCH 0121/1206] [mlir][sparse] support 'batch' dimensions in
 sparse_tensor.print (#91411)

---
 .../Transforms/SparseTensorCodegen.cpp        | 12 ++-
 .../Transforms/SparseTensorRewriting.cpp      | 66 ++++++++++-------
 .../SparseTensor/CPU/sparse_pack_d.mlir       | 12 +--
 .../SparseTensor/CPU/sparse_print_3d.mlir     | 74 +++++++++++++++++++
 4 files changed, 130 insertions(+), 34 deletions(-)
 create mode 100755 mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir

diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index d9b203a88648..164e722c45db 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -417,11 +417,17 @@ static void genEndInsert(OpBuilder &builder, Location loc,
 /// Generates a subview into the sizes.
 static Value genSliceToSize(OpBuilder &builder, Location loc, Value mem,
                             Value sz) {
-  auto elemTp = llvm::cast<MemRefType>(mem.getType()).getElementType();
+  auto memTp = llvm::cast<MemRefType>(mem.getType());
+  // For higher-dimensional memrefs, we assume that the innermost
+  // dimension is always of the right size.
+  // TODO: generate complex truncating view here too?
+  if (memTp.getRank() > 1)
+    return mem;
+  // Truncate linear memrefs to given size.
   return builder
       .create<memref::SubViewOp>(
-          loc, MemRefType::get({ShapedType::kDynamic}, elemTp), mem,
-          ValueRange{}, ValueRange{sz}, ValueRange{},
+          loc, MemRefType::get({ShapedType::kDynamic}, memTp.getElementType()),
+          mem, ValueRange{}, ValueRange{sz}, ValueRange{},
           ArrayRef<int64_t>{0},                    // static offset
           ArrayRef<int64_t>{ShapedType::kDynamic}, // dynamic size
           ArrayRef<int64_t>{1})                    // static stride
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index 7d469198a653..025fd3331ba8 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -785,45 +785,61 @@ public:
   }
 
 private:
-  // Helper to print contents of a single memref. Note that for the "push_back"
-  // vectors, this prints the full capacity, not just the size. This is done
-  // on purpose, so that clients see how much storage has been allocated in
-  // total. Contents of the extra capacity in the buffer may be uninitialized
-  // (unless the flag enable-buffer-initialization is set to true).
+  // Helper to print contents of a single memref. For "push_back" vectors,
+  // we assume that the previous getters for pos/crd/val have added a
+  // slice-to-size view to make sure we just print the size and not the
+  // full capacity.
   //
-  // Generates code to print:
+  // Generates code to print (1-dim or higher):
   //    ( a0, a1, ... )
   static void printContents(PatternRewriter &rewriter, Location loc,
                             Value vec) {
+    auto shape = cast<ShapedType>(vec.getType()).getShape();
+    SmallVector<Value> idxs;
+    printContentsLevel(rewriter, loc, vec, 0, shape, idxs);
+    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::NewLine);
+  }
+
+  // Helper to the helper.
+  static void printContentsLevel(PatternRewriter &rewriter, Location loc,
+                                 Value vec, unsigned i, ArrayRef<int64_t> shape,
+                                 SmallVectorImpl<Value> &idxs) {
     // Open bracket.
     rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
-    // For loop over elements.
+    // Generate for loop.
     auto zero = constantIndex(rewriter, loc, 0);
-    auto size = rewriter.create<memref::DimOp>(loc, vec, zero);
+    auto index = constantIndex(rewriter, loc, i);
+    auto size = rewriter.create<memref::DimOp>(loc, vec, index);
     auto step = constantIndex(rewriter, loc, 1);
     auto forOp = rewriter.create<scf::ForOp>(loc, zero, size, step);
+    idxs.push_back(forOp.getInductionVar());
     rewriter.setInsertionPointToStart(forOp.getBody());
-    auto idx = forOp.getInductionVar();
-    auto val = rewriter.create<memref::LoadOp>(loc, vec, idx);
-    if (llvm::isa<ComplexType>(val.getType())) {
-      // Since the vector dialect does not support complex types in any op,
-      // we split those into (real, imag) pairs here.
-      Value real = rewriter.create<complex::ReOp>(loc, val);
-      Value imag = rewriter.create<complex::ImOp>(loc, val);
-      rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
-      rewriter.create<vector::PrintOp>(loc, real,
-                                       vector::PrintPunctuation::Comma);
-      rewriter.create<vector::PrintOp>(loc, imag,
-                                       vector::PrintPunctuation::Close);
-      rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Comma);
+    if (i < shape.size() - 1) {
+      // Enter deeper loop nest.
+      printContentsLevel(rewriter, loc, vec, i + 1, shape, idxs);
     } else {
-      rewriter.create<vector::PrintOp>(loc, val,
-                                       vector::PrintPunctuation::Comma);
+      // Actual contents printing.
+      auto val = rewriter.create<memref::LoadOp>(loc, vec, idxs);
+      if (llvm::isa<ComplexType>(val.getType())) {
+        // Since the vector dialect does not support complex types in any op,
+        // we split those into (real, imag) pairs here.
+        Value real = rewriter.create<complex::ReOp>(loc, val);
+        Value imag = rewriter.create<complex::ImOp>(loc, val);
+        rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Open);
+        rewriter.create<vector::PrintOp>(loc, real,
+                                         vector::PrintPunctuation::Comma);
+        rewriter.create<vector::PrintOp>(loc, imag,
+                                         vector::PrintPunctuation::Close);
+        rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Comma);
+      } else {
+        rewriter.create<vector::PrintOp>(loc, val,
+                                         vector::PrintPunctuation::Comma);
+      }
     }
+    idxs.pop_back();
     rewriter.setInsertionPointAfter(forOp);
-    // Close bracket and end of line.
+    // Close bracket.
     rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Close);
-    rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::NewLine);
   }
 
   // Helper method to print run-time lvl/dim sizes.
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
index 20ae7e86285c..467a77f30777 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
@@ -29,7 +29,7 @@
   crdWidth = 32
 }>
 
-#BatchedCSR = #sparse_tensor.encoding<{
+#DenseCSR = #sparse_tensor.encoding<{
   map = (d0, d1, d2) -> (d0 : dense, d1 : dense, d2 : compressed),
   posWidth = 64,
   crdWidth = 32
@@ -42,7 +42,7 @@
 }>
 
 //
-// Test assembly operation with CCC, batched-CSR and CSR-dense.
+// Test assembly operation with CCC, dense-CSR and CSR-dense.
 //
 module {
   //
@@ -77,7 +77,7 @@ module {
         tensor<6xi64>, tensor<8xi32>), tensor<8xf32> to tensor<4x3x2xf32, #CCC>
 
     //
-    // Setup BatchedCSR.
+    // Setup DenseCSR.
     //
 
     %data1 = arith.constant dense<
@@ -88,7 +88,7 @@ module {
     %crd1 = arith.constant dense<
        [ 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1]> : tensor<16xi32>
 
-    %s1 = sparse_tensor.assemble (%pos1, %crd1), %data1 : (tensor<13xi64>, tensor<16xi32>), tensor<16xf32> to tensor<4x3x2xf32, #BatchedCSR>
+    %s1 = sparse_tensor.assemble (%pos1, %crd1), %data1 : (tensor<13xi64>, tensor<16xi32>), tensor<16xf32> to tensor<4x3x2xf32, #DenseCSR>
 
     //
     // Setup CSRDense.
@@ -137,7 +137,7 @@ module {
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %s0 : tensor<4x3x2xf32, #CCC>
-    sparse_tensor.print %s1 : tensor<4x3x2xf32, #BatchedCSR>
+    sparse_tensor.print %s1 : tensor<4x3x2xf32, #DenseCSR>
     sparse_tensor.print %s2 : tensor<4x3x2xf32, #CSRDense>
 
     // TODO: This check is no longer needed once the codegen path uses the
@@ -148,7 +148,7 @@ module {
       // sparse_tensor.assemble copies buffers when running with the runtime
       // library. Deallocations are not needed when running in codegen mode.
       bufferization.dealloc_tensor %s0 : tensor<4x3x2xf32, #CCC>
-      bufferization.dealloc_tensor %s1 : tensor<4x3x2xf32, #BatchedCSR>
+      bufferization.dealloc_tensor %s1 : tensor<4x3x2xf32, #DenseCSR>
       bufferization.dealloc_tensor %s2 : tensor<4x3x2xf32, #CSRDense>
     }
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir
new file mode 100755
index 000000000000..98dee304fa51
--- /dev/null
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir
@@ -0,0 +1,74 @@
+//--------------------------------------------------------------------------------------------------
+// WHEN CREATING A NEW TEST, PLEASE JUST COPY & PASTE WITHOUT EDITS.
+//
+// Set-up that's shared across all tests in this directory. In principle, this
+// config could be moved to lit.local.cfg. However, there are downstream users that
+//  do not use these LIT config files. Hence why this is kept inline.
+//
+// DEFINE: %{sparsifier_opts} = enable-runtime-library=true
+// DEFINE: %{sparsifier_opts_sve} = enable-arm-sve=true %{sparsifier_opts}
+// DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
+// DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
+// DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_opts} = -e main -entry-point-result=void
+// DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+//
+// DEFINE: %{env} =
+//--------------------------------------------------------------------------------------------------
+
+// TODO: make this work with libgen
+
+// Do the same run, but now with direct IR generation.
+// REDEFINE: %{sparsifier_opts} = enable-runtime-library=false enable-buffer-initialization=true
+// RUN: %{compile} | %{run} | FileCheck %s
+//
+
+#BatchedCSR = #sparse_tensor.encoding<{
+  map = (d0, d1, d2) -> (d0 : batch, d1 : dense, d2 : compressed)
+}>
+
+module {
+
+  //
+  // Main driver that tests 3-D sparse tensor printing.
+  //
+  func.func @main() {
+
+    %pos = arith.constant dense<
+      [[ 0, 8, 16, 24, 32],
+       [ 0, 8, 16, 24, 32]]
+    > : tensor<2x5xindex>
+
+    %crd = arith.constant dense<
+      [[0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7],
+       [0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7]]
+    > : tensor<2x32xindex>
+
+    %val = arith.constant dense<
+      [[ 1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9., 10., 11.,
+        12., 13., 14., 15., 16., 17., 18., 19., 20., 21., 22.,
+        23., 24., 25., 26., 27., 28., 29., 30., 31., 32.],
+       [33., 34., 35., 36., 37., 38., 39., 40., 41., 42., 43.,
+        44., 45., 46., 47., 48., 49., 50., 51., 52., 53., 54.,
+        55., 56., 57., 58., 59., 60., 61., 62., 63., 64.]]
+    > : tensor<2x32xf64>
+
+    %X = sparse_tensor.assemble (%pos, %crd), %val
+      : (tensor<2x5xindex>, tensor<2x32xindex>), tensor<2x32xf64> to tensor<2x4x8xf64, #BatchedCSR>
+
+    // CHECK:      ---- Sparse Tensor ----
+    // CHECK-NEXT: nse = 32
+    // CHECK-NEXT: dim = ( 2, 4, 8 )
+    // CHECK-NEXT: lvl = ( 2, 4, 8 )
+    // CHECK-NEXT: pos[2] : ( ( 0, 8, 16, 24, 32,  )( 0, 8, 16, 24, 32,  ) )
+    // CHECK-NEXT: crd[2] : ( ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,  )
+    // CHECK-SAME:            ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,  ) )
+    // CHECK-NEXT: values : ( ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,  )
+    // CHECK-SAME:            ( 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64,  ) )
+    // CHECK-NEXT: ----
+    sparse_tensor.print %X : tensor<2x4x8xf64, #BatchedCSR>
+
+    return
+  }
+}
-- 
GitLab


From 2dade0041a62b192e9bde24ae6bbe6208f027523 Mon Sep 17 00:00:00 2001
From: Jinsong Ji <jinsong.ji@intel.com>
Date: Tue, 7 May 2024 19:02:10 -0700
Subject: [PATCH 0122/1206] [Analysis] Attribute Range should not prevent tail
 call optimization (#91122)

- Remove Range attr when comparing for tailcall
- Add test for testcall with range
---
 llvm/lib/CodeGen/Analysis.cpp                 |  7 +--
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |  7 +--
 llvm/test/CodeGen/X86/tailcall-range.ll       | 53 +++++++++++++++++++
 3 files changed, 61 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/tailcall-range.ll

diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index af7643d93591..e693cdbd0ccc 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -593,9 +593,10 @@ bool llvm::attributesPermitTailCall(const Function *F, const Instruction *I,
 
   // Following attributes are completely benign as far as calling convention
   // goes, they shouldn't affect whether the call is a tail call.
-  for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
-                           Attribute::DereferenceableOrNull, Attribute::NoAlias,
-                           Attribute::NonNull, Attribute::NoUndef}) {
+  for (const auto &Attr :
+       {Attribute::Alignment, Attribute::Dereferenceable,
+        Attribute::DereferenceableOrNull, Attribute::NoAlias,
+        Attribute::NonNull, Attribute::NoUndef, Attribute::Range}) {
     CallerAttrs.removeAttribute(Attr);
     CalleeAttrs.removeAttribute(Attr);
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 336d89fbcf63..9ec3ac4f9991 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -62,9 +62,10 @@ bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
   // the return. Ignore following attributes because they don't affect the
   // call sequence.
   AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
-  for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
-                           Attribute::DereferenceableOrNull, Attribute::NoAlias,
-                           Attribute::NonNull, Attribute::NoUndef})
+  for (const auto &Attr :
+       {Attribute::Alignment, Attribute::Dereferenceable,
+        Attribute::DereferenceableOrNull, Attribute::NoAlias,
+        Attribute::NonNull, Attribute::NoUndef, Attribute::Range})
     CallerAttrs.removeAttribute(Attr);
 
   if (CallerAttrs.hasAttributes())
diff --git a/llvm/test/CodeGen/X86/tailcall-range.ll b/llvm/test/CodeGen/X86/tailcall-range.ll
new file mode 100644
index 000000000000..6ae7405ebc4a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailcall-range.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=x86_64-linux < %s | FileCheck %s
+
+define range(i32 0, 2) i32 @foo(ptr %this) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movzbl (%rdi), %eax
+; CHECK-NEXT:    retq
+entry:
+  %call = load volatile i1, ptr %this, align 1
+  %spec.select = zext i1 %call to i32
+  ret i32 %spec.select
+}
+
+define range(i32 0, 2) i32 @bar(ptr %this) {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    jmp foo@PLT # TAILCALL
+entry:
+  %ret = musttail call i32 @foo(ptr null)
+  ret i32 %ret
+}
+
+declare i64 @llvm.llround.f32(float) nounwind readnone
+define range(i64 0, 8) i64 @testmsxs(float %x) {
+; CHECK-LABEL: testmsxs:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    jmp llroundf@PLT # TAILCALL
+entry:
+  %ret = tail call i64 @llvm.llround.f32(float %x)
+  ret i64 %ret
+}
+
+declare i32 @callee()
+
+define range(i32 0, 2) i32 @func_with_range_attr() {
+; CHECK-LABEL: func_with_range_attr:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    jmp callee@PLT # TAILCALL
+entry:
+  %ret = musttail call i32 @callee()
+  ret i32 %ret
+}
+
+define i32 @call_with_range_attr() {
+; CHECK-LABEL: call_with_range_attr:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    jmp callee@PLT # TAILCALL
+entry:
+  %ret = musttail call range(i32 0, 2) i32 @callee()
+  ret i32 %ret
+}
-- 
GitLab


From 7098cd215b95286794d9e0c822e8323ad0509750 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Wed, 8 May 2024 11:02:00 +0900
Subject: [PATCH 0123/1206] [NFC] Add myself as code owner for llvm/IR/Core.cpp

In practice I end up reviewing most changes to the C API.
---
 .github/CODEOWNERS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 0f178df1d18f..ad81bf1684b6 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -23,6 +23,7 @@
 /llvm/lib/Analysis/ScalarEvolution.cpp @nikic
 /llvm/lib/Analysis/ValueTracking.cpp @nikic
 /llvm/lib/IR/ConstantRange.cpp @nikic
+/llvm/lib/IR/Core.cpp @nikic
 /llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp @nikic
 /llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp @nikic
 /llvm/lib/Transforms/InstCombine/ @nikic
-- 
GitLab


From d085b42cbbefe79a41113abcd2b1e1f2a203acef Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Wed, 8 May 2024 10:04:09 +0800
Subject: [PATCH 0124/1206] [InstSimplify] Do not simplify freeze in
 `simplifyWithOpReplaced` (#91215)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

See the LangRef:
> All uses of a value returned by the same ‘freeze’ instruction are
guaranteed to always observe the same value, while different ‘freeze’
instructions may yield different values.

It is incorrect to replace freezes with the simplified value.

Proof:
https://alive2.llvm.org/ce/z/3Dn9Cd
https://alive2.llvm.org/ce/z/Qyh5h6

Fixes https://github.com/llvm/llvm-project/issues/91178
---
 llvm/lib/Analysis/InstructionSimplify.cpp  |  4 +++
 llvm/test/Transforms/InstCombine/icmp.ll   | 15 ++++++++++
 llvm/test/Transforms/InstCombine/select.ll | 32 ++++++++++++++++++++++
 llvm/test/Transforms/PGOProfile/chr.ll     |  7 +++--
 4 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 4061dae83c10..37a7259a5cd0 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -4312,6 +4312,10 @@ static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
   if (match(I, m_Intrinsic<Intrinsic::is_constant>()))
     return nullptr;
 
+  // Don't simplify freeze.
+  if (isa<FreezeInst>(I))
+    return nullptr;
+
   // Replace Op with RepOp in instruction operands.
   SmallVector<Value *, 8> NewOps;
   bool AnyReplaced = false;
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index 31093c7ca103..2d786c8f4883 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -5183,3 +5183,18 @@ entry:
   %cmp = icmp eq i8 %add2, %add1
   ret i1 %cmp
 }
+
+define i1 @icmp_freeze_sext(i16 %x, i16 %y) {
+; CHECK-LABEL: @icmp_freeze_sext(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i16 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP1_FR:%.*]] = freeze i1 [[CMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i16 [[Y]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = or i1 [[TMP1]], [[CMP1_FR]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp uge i16 %x, %y
+  %ext = sext i1 %cmp1 to i16
+  %ext.fr = freeze i16 %ext
+  %cmp2 = icmp uge i16 %ext.fr, %y
+  ret i1 %cmp2
+}
diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll
index 2efe2742ca49..2ade6faa99be 100644
--- a/llvm/test/Transforms/InstCombine/select.ll
+++ b/llvm/test/Transforms/InstCombine/select.ll
@@ -4580,3 +4580,35 @@ define i32 @sequence_select_with_same_cond_extra_use(i1 %c1, i1 %c2){
   %s3 = select i1 %c1, i32 789, i32 %s2
   ret i32 %s3
 }
+
+define i8 @test_replace_freeze_multiuse(i1 %x, i8 %y) {
+; CHECK-LABEL: @test_replace_freeze_multiuse(
+; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[X:%.*]] to i8
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i8 [[EXT]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHL_FR:%.*]] = freeze i8 [[SHL]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X]], i8 0, i8 [[SHL_FR]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[SHL_FR]], [[SEL]]
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %ext = zext i1 %x to i8
+  %shl = shl nuw i8 %ext, %y
+  %shl.fr = freeze i8 %shl
+  %sel = select i1 %x, i8 0, i8 %shl.fr
+  %add = add i8 %shl.fr, %sel
+  ret i8 %add
+}
+
+define i8 @test_replace_freeze_oneuse(i1 %x, i8 %y) {
+; CHECK-LABEL: @test_replace_freeze_oneuse(
+; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[X:%.*]] to i8
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i8 [[EXT]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHL_FR:%.*]] = freeze i8 [[SHL]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X]], i8 0, i8 [[SHL_FR]]
+; CHECK-NEXT:    ret i8 [[SEL]]
+;
+  %ext = zext i1 %x to i8
+  %shl = shl nuw i8 %ext, %y
+  %shl.fr = freeze i8 %shl
+  %sel = select i1 %x, i8 0, i8 %shl.fr
+  ret i8 %sel
+}
diff --git a/llvm/test/Transforms/PGOProfile/chr.ll b/llvm/test/Transforms/PGOProfile/chr.ll
index 0551a171091c..38e8f8536a19 100644
--- a/llvm/test/Transforms/PGOProfile/chr.ll
+++ b/llvm/test/Transforms/PGOProfile/chr.ll
@@ -1298,11 +1298,12 @@ define i32 @test_chr_14(ptr %i, ptr %j, i32 %sum0, i1 %pred, i32 %z) !prof !14 {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[Z_FR:%.*]] = freeze i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[I0:%.*]] = load i32, ptr [[I:%.*]], align 4
-; CHECK-NEXT:    [[V1:%.*]] = icmp eq i32 [[Z_FR]], 1
-; CHECK-NEXT:    br i1 [[V1]], label [[BB1:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof [[PROF15]]
+; CHECK-NEXT:    [[V1_NOT:%.*]] = icmp eq i32 [[Z_FR]], 1
+; CHECK-NEXT:    br i1 [[V1_NOT]], label [[BB1:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof [[PROF15]]
 ; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[PRED_FR:%.*]] = freeze i1 [[PRED:%.*]]
 ; CHECK-NEXT:    [[V0:%.*]] = icmp eq i32 [[Z_FR]], 0
-; CHECK-NEXT:    [[V3_NONCHR:%.*]] = and i1 [[V0]], [[PRED:%.*]]
+; CHECK-NEXT:    [[V3_NONCHR:%.*]] = and i1 [[V0]], [[PRED_FR]]
 ; CHECK-NEXT:    br i1 [[V3_NONCHR]], label [[BB0_NONCHR:%.*]], label [[BB1]], !prof [[PROF16]]
 ; CHECK:       bb0.nonchr:
 ; CHECK-NEXT:    call void @foo()
-- 
GitLab


From bb01b89cda71fe1594a87f81b3f3c01f66fcac59 Mon Sep 17 00:00:00 2001
From: Ryosuke Niwa <rniwa@webkit.org>
Date: Tue, 7 May 2024 19:10:50 -0700
Subject: [PATCH 0125/1206] [analyzer] Ignore system headers in WebKit
 checkers. (#91103)

---
 .../WebKit/UncountedCallArgsChecker.cpp         |  3 +++
 .../WebKit/UncountedLocalVarsChecker.cpp        |  3 +++
 .../Checkers/WebKit/mock-system-header.h        | 17 +++++++++++++++++
 .../Checkers/WebKit/uncounted-local-vars.cpp    | 11 +++++++++++
 .../Checkers/WebKit/uncounted-members.cpp       | 10 ++++++++++
 .../Checkers/WebKit/uncounted-obj-arg.cpp       |  6 ++++++
 6 files changed, 50 insertions(+)
 create mode 100644 clang/test/Analysis/Checkers/WebKit/mock-system-header.h

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
index 0f40ecc7ba30..9a178a690ff2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
@@ -150,6 +150,9 @@ public:
   bool shouldSkipCall(const CallExpr *CE) const {
     const auto *Callee = CE->getDirectCallee();
 
+    if (BR->getSourceManager().isInSystemHeader(CE->getExprLoc()))
+      return true;
+
     if (Callee && TFA.isTrivial(Callee))
       return true;
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
index 6036ad58cf25..98a73810b7af 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
@@ -230,6 +230,9 @@ public:
     if (!V->isLocalVarDecl())
       return true;
 
+    if (BR->getSourceManager().isInSystemHeader(V->getLocation()))
+      return true;
+
     return false;
   }
 
diff --git a/clang/test/Analysis/Checkers/WebKit/mock-system-header.h b/clang/test/Analysis/Checkers/WebKit/mock-system-header.h
new file mode 100644
index 000000000000..a1d30957b19c
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/mock-system-header.h
@@ -0,0 +1,17 @@
+#pragma clang system_header
+
+template <typename T, typename CreateFunction>
+void callMethod(CreateFunction createFunction) {
+  createFunction()->method();
+}
+
+template <typename T, typename CreateFunction>
+inline void localVar(CreateFunction createFunction) {
+  T* obj = createFunction();
+  obj->method();
+}
+
+template <typename T>
+struct MemberVariable {
+    T* obj { nullptr };
+};
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
index 00673e91f471..8da1dc557a5a 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
@@ -1,6 +1,7 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UncountedLocalVarsChecker -verify %s
 
 #include "mock-types.h"
+#include "mock-system-header.h"
 
 void someFunction();
 
@@ -187,3 +188,13 @@ void bar() {
 }
 
 } // namespace ignore_for_if
+
+namespace ignore_system_headers {
+
+RefCountable *provide_ref_ctnbl();
+
+void system_header() {
+  localVar<RefCountable>(provide_ref_ctnbl);
+}
+
+} // ignore_system_headers
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-members.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-members.cpp
index 108d5effdd2e..bca7b3bad3a1 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-members.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-members.cpp
@@ -1,6 +1,7 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=webkit.NoUncountedMemberChecker -verify %s
 
 #include "mock-types.h"
+#include "mock-system-header.h"
 
 namespace members {
   struct Foo {
@@ -50,3 +51,12 @@ namespace ignore_unions {
 
   void forceTmplToInstantiate(RefPtr<RefCountable>) {}
 }
+
+namespace ignore_system_header {
+
+void foo(RefCountable* t) {
+  MemberVariable<RefCountable> var { t };
+  var.obj->method();
+}
+
+} // ignore_system_header
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
index 63a68a994a5c..e75d42b9f149 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
@@ -1,6 +1,7 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UncountedCallArgsChecker -verify %s
 
 #include "mock-types.h"
+#include "mock-system-header.h"
 
 void WTFBreakpointTrap();
 void WTFCrashWithInfo(int, const char*, const char*, int);
@@ -147,6 +148,7 @@ public:
   void ref() const;
   void deref() const;
 
+  void method();
   void someFunction();
   int otherFunction();
 
@@ -399,3 +401,7 @@ void someFunction(const RefCounted&);
 void test2() {
     someFunction(*object());
 }
+
+void system_header() {
+  callMethod<RefCountable>(object);
+}
-- 
GitLab


From 0af448b71116ae93eae1cb9c3121cb94be076fc3 Mon Sep 17 00:00:00 2001
From: Menooker <yijie.mei@intel.com>
Date: Wed, 8 May 2024 10:14:52 +0800
Subject: [PATCH 0126/1206] [MLIR][Bufferization] BufferResultsToOutParams: Add
 an option to eliminate AllocOp and avoid Copy (#90011)

Add an option hoist-static-allocs to remove the unnecessary memref.alloc
and memref.copy after this pass, when the memref in ReturnOp is
allocated by memref.alloc and is statically shaped. Instead, it replaces
the uses of the allocated memref with the memref in the out argument.
By default, BufferResultsToOutParams will result in a memcpy operation
to copy the originally returned memref to the output argument memref.
This is inefficient when the source of memcpy (the returned memref in
the original ReturnOp) is from a local AllocOp. The pass can use the
output argument memref to replace the locally allocated memref for
better performance.hoist-static-allocs avoids dynamic allocation and
memory movement.
This option will be critical for performance-sensivtive applications,
which require BufferResultsToOutParams pass for a caller-owned output
buffer calling convension.
---
 .../Dialect/Bufferization/Transforms/Passes.h |  4 ++
 .../Bufferization/Transforms/Passes.td        |  9 +++++
 .../Transforms/BufferResultsToOutParams.cpp   | 21 ++++++++---
 .../buffer-results-to-out-params-elim.mlir    | 37 +++++++++++++++++++
 4 files changed, 65 insertions(+), 6 deletions(-)
 create mode 100644 mlir/test/Transforms/buffer-results-to-out-params-elim.mlir

diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h
index a729bc99b987..459c252b7071 100644
--- a/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h
@@ -166,6 +166,10 @@ struct BufferResultsToOutParamsOpts {
   /// If true, the pass adds a "bufferize.result" attribute to each output
   /// parameter.
   bool addResultAttribute = false;
+
+  /// If true, the pass eliminates the memref.alloc and memcpy if the returned
+  /// memref is allocated in the current function.
+  bool hoistStaticAllocs = false;
 };
 
 /// Creates a pass that converts memref function results to out-params.
diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td b/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td
index 1303dc2c9ae1..75ce85c9128c 100644
--- a/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td
@@ -315,11 +315,20 @@ def BufferResultsToOutParams : Pass<"buffer-results-to-out-params", "ModuleOp">
     The main issue with this pass (and the out-param calling convention) is that
     buffers for results need to be allocated in the caller. This currently only
     works for static shaped memrefs.
+
+    If the hoist-static-allocs option is on, the pass tries to eliminate the
+    allocation for the returned memref and avoid the memory-copy if possible.
+    This optimization applies on the returned memref which has static shape and
+    is allocated by memref.alloc in the function. It will use the memref given
+    in function argument to replace the allocated memref.
   }];
   let options = [
     Option<"addResultAttribute", "add-result-attr", "bool",
        /*default=*/"false",
        "Add the attribute 'bufferize.result' to all output parameters.">,
+    Option<"hoistStaticAllocs", "hoist-static-allocs",
+       "bool", /*default=*/"false",
+       "Hoist static allocations to call sites.">,
   ];
   let constructor = "mlir::bufferization::createBufferResultsToOutParamsPass()";
   let dependentDialects = ["memref::MemRefDialect"];
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
index a2222e169c4d..a5f01eadb213 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
@@ -107,7 +107,8 @@ updateFuncOp(func::FuncOp func,
 // the given out-params.
 static LogicalResult updateReturnOps(func::FuncOp func,
                                      ArrayRef<BlockArgument> appendedEntryArgs,
-                                     MemCpyFn memCpyFn) {
+                                     MemCpyFn memCpyFn,
+                                     bool hoistStaticAllocs) {
   auto res = func.walk([&](func::ReturnOp op) {
     SmallVector<Value, 6> copyIntoOutParams;
     SmallVector<Value, 6> keepAsReturnOperands;
@@ -118,10 +119,15 @@ static LogicalResult updateReturnOps(func::FuncOp func,
         keepAsReturnOperands.push_back(operand);
     }
     OpBuilder builder(op);
-    for (auto t : llvm::zip(copyIntoOutParams, appendedEntryArgs)) {
-      if (failed(
-              memCpyFn(builder, op.getLoc(), std::get<0>(t), std::get<1>(t))))
-        return WalkResult::interrupt();
+    for (auto [orig, arg] : llvm::zip(copyIntoOutParams, appendedEntryArgs)) {
+      if (hoistStaticAllocs && isa<memref::AllocOp>(orig.getDefiningOp()) &&
+          orig.getType().cast<MemRefType>().hasStaticShape()) {
+        orig.replaceAllUsesWith(arg);
+        orig.getDefiningOp()->erase();
+      } else {
+        if (failed(memCpyFn(builder, op.getLoc(), orig, arg)))
+          return WalkResult::interrupt();
+      }
     }
     builder.create<func::ReturnOp>(op.getLoc(), keepAsReturnOperands);
     op.erase();
@@ -212,7 +218,8 @@ LogicalResult mlir::bufferization::promoteBufferResultsToOutParams(
       return success();
     };
     if (failed(updateReturnOps(func, appendedEntryArgs,
-                               options.memCpyFn.value_or(defaultMemCpyFn)))) {
+                               options.memCpyFn.value_or(defaultMemCpyFn),
+                               options.hoistStaticAllocs))) {
       return failure();
     }
   }
@@ -233,6 +240,8 @@ struct BufferResultsToOutParamsPass
     // Convert from pass options in tablegen to BufferResultsToOutParamsOpts.
     if (addResultAttribute)
       options.addResultAttribute = true;
+    if (hoistStaticAllocs)
+      options.hoistStaticAllocs = true;
 
     if (failed(bufferization::promoteBufferResultsToOutParams(getOperation(),
                                                               options)))
diff --git a/mlir/test/Transforms/buffer-results-to-out-params-elim.mlir b/mlir/test/Transforms/buffer-results-to-out-params-elim.mlir
new file mode 100644
index 000000000000..f77dbfaa6cb1
--- /dev/null
+++ b/mlir/test/Transforms/buffer-results-to-out-params-elim.mlir
@@ -0,0 +1,37 @@
+// RUN: mlir-opt -allow-unregistered-dialect -p 'builtin.module(buffer-results-to-out-params{hoist-static-allocs})'  %s | FileCheck %s
+
+// CHECK-LABEL:   func @basic(
+// CHECK-SAME:                %[[ARG:.*]]: memref<8x64xf32>) {
+// CHECK-NOT:        memref.alloc()
+// CHECK:           "test.source"(%[[ARG]])  : (memref<8x64xf32>) -> ()
+// CHECK:           return
+// CHECK:         }
+func.func @basic() -> (memref<8x64xf32>) {
+  %b = memref.alloc() : memref<8x64xf32>
+  "test.source"(%b)  : (memref<8x64xf32>) -> ()
+  return %b : memref<8x64xf32>
+}
+
+// CHECK-LABEL:   func @basic_no_change(
+// CHECK-SAME:                %[[ARG:.*]]: memref<f32>) {
+// CHECK:           %[[RESULT:.*]] = "test.source"() : () -> memref<f32>
+// CHECK:           memref.copy %[[RESULT]], %[[ARG]]  : memref<f32> to memref<f32>
+// CHECK:           return
+// CHECK:         }
+func.func @basic_no_change() -> (memref<f32>) {
+  %0 = "test.source"() : () -> (memref<f32>)
+  return %0 : memref<f32>
+}
+
+// CHECK-LABEL:   func @basic_dynamic(
+// CHECK-SAME:                %[[D:.*]]: index, %[[ARG:.*]]: memref<?xf32>) {
+// CHECK:           %[[RESULT:.*]] = memref.alloc(%[[D]]) : memref<?xf32>
+// CHECK:           "test.source"(%[[RESULT]])  : (memref<?xf32>) -> ()
+// CHECK:           memref.copy %[[RESULT]], %[[ARG]]
+// CHECK:           return
+// CHECK:         }
+func.func @basic_dynamic(%d: index) -> (memref<?xf32>) {
+  %b = memref.alloc(%d) : memref<?xf32>
+  "test.source"(%b)  : (memref<?xf32>) -> ()
+  return %b : memref<?xf32>
+}
\ No newline at end of file
-- 
GitLab


From 1c8c2fdd289075d6ef448f60db9dd30caf7f78df Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Wed, 8 May 2024 10:38:34 +0800
Subject: [PATCH 0127/1206] [mlir] Fix -Wdeprecated-declarations in
 BufferResultsToOutParams.cpp (NFC)

/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp:124:26:
error: 'cast' is deprecated: Use mlir::cast<U>() instead [-Werror,-Wdeprecated-declarations]
  124 |           orig.getType().cast<MemRefType>().hasStaticShape()) {
      |
---
 .../Bufferization/Transforms/BufferResultsToOutParams.cpp       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
index a5f01eadb213..b19636adaa69 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp
@@ -121,7 +121,7 @@ static LogicalResult updateReturnOps(func::FuncOp func,
     OpBuilder builder(op);
     for (auto [orig, arg] : llvm::zip(copyIntoOutParams, appendedEntryArgs)) {
       if (hoistStaticAllocs && isa<memref::AllocOp>(orig.getDefiningOp()) &&
-          orig.getType().cast<MemRefType>().hasStaticShape()) {
+          mlir::cast<MemRefType>(orig.getType()).hasStaticShape()) {
         orig.replaceAllUsesWith(arg);
         orig.getDefiningOp()->erase();
       } else {
-- 
GitLab


From 31b45a9d0d91cc3a78446ee379abc6f2a3000065 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Tue, 7 May 2024 22:54:15 -0400
Subject: [PATCH 0128/1206] [clang][hlsl] Add tan intrinsic part 1 (#90276)

This change is an implementation of #87367's investigation on supporting
IEEE math operations as intrinsics.
Which was discussed in this RFC:
https://discourse.llvm.org/t/rfc-all-the-math-intrinsics/78294

If you want an overarching view of how this will all connect see:
https://github.com/llvm/llvm-project/pull/90088

Changes:
- `clang/docs/LanguageExtensions.rst` - Document the new elementwise tan
builtin.
-  `clang/include/clang/Basic/Builtins.td` - Implement the tan builtin.
- `clang/lib/CodeGen/CGBuiltin.cpp` - invoke the tan intrinsic on uses
of the builtin
- `clang/lib/Headers/hlsl/hlsl_intrinsics.h` - Associate the tan builtin
with the equivalent hlsl apis
- `clang/lib/Sema/SemaChecking.cpp` - Add generic sema checks as well as
HLSL specifc sema checks to the tan builtin
-  `llvm/include/llvm/IR/Intrinsics.td` - Create the tan intrinsic
-  `llvm/docs/LangRef.rst` - Document the tan intrinsic
---
 clang/docs/LanguageExtensions.rst             |  1 +
 clang/include/clang/Basic/Builtins.td         |  6 ++
 clang/lib/CodeGen/CGBuiltin.cpp               |  4 +-
 clang/lib/Headers/hlsl/hlsl_intrinsics.h      | 23 ++++++++
 clang/lib/Sema/SemaChecking.cpp               |  2 +
 .../test/CodeGen/builtins-elementwise-math.c  | 16 +++++
 .../CodeGen/strictfp-elementwise-bulitins.cpp | 10 ++++
 clang/test/CodeGenHLSL/builtins/tan.hlsl      | 59 +++++++++++++++++++
 clang/test/Sema/aarch64-sve-vector-trig-ops.c | 42 +++++++------
 clang/test/Sema/builtins-elementwise-math.c   | 21 +++++++
 clang/test/Sema/riscv-rvv-vector-trig-ops.c   | 44 ++++++++------
 .../SemaCXX/builtins-elementwise-math.cpp     |  7 +++
 .../BuiltIns/half-float-only-errors.hlsl      |  1 +
 llvm/docs/LangRef.rst                         | 37 ++++++++++++
 llvm/include/llvm/IR/Intrinsics.td            |  1 +
 15 files changed, 236 insertions(+), 38 deletions(-)
 create mode 100644 clang/test/CodeGenHLSL/builtins/tan.hlsl

diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index c2e90f4e7d58..3627a780886a 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -656,6 +656,7 @@ Unless specified otherwise operation(±0) = ±0 and operation(±infinity) = ±in
  T __builtin_elementwise_ceil(T x)           return the smallest integral value greater than or equal to x    floating point types
  T __builtin_elementwise_sin(T x)            return the sine of x interpreted as an angle in radians          floating point types
  T __builtin_elementwise_cos(T x)            return the cosine of x interpreted as an angle in radians        floating point types
+ T __builtin_elementwise_tan(T x)            return the tangent of x interpreted as an angle in radians       floating point types
  T __builtin_elementwise_floor(T x)          return the largest integral value less than or equal to x        floating point types
  T __builtin_elementwise_log(T x)            return the natural logarithm of x                                floating point types
  T __builtin_elementwise_log2(T x)           return the base 2 logarithm of x                                 floating point types
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index de721a87b334..11982af3fa60 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1326,6 +1326,12 @@ def ElementwiseSqrt : Builtin {
   let Prototype = "void(...)";
 }
 
+def ElementwiseTan : Builtin {
+  let Spellings = ["__builtin_elementwise_tan"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
 def ElementwiseTrunc : Builtin {
   let Spellings = ["__builtin_elementwise_trunc"];
   let Attributes = [NoThrow, Const, CustomTypeChecking];
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index e8a6bd050e17..4b03b8b0e093 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -3822,7 +3822,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_elementwise_sin:
     return RValue::get(
         emitUnaryBuiltin(*this, E, llvm::Intrinsic::sin, "elt.sin"));
-
+  case Builtin::BI__builtin_elementwise_tan:
+    return RValue::get(
+        emitUnaryBuiltin(*this, E, llvm::Intrinsic::tan, "elt.tan"));
   case Builtin::BI__builtin_elementwise_trunc:
     return RValue::get(
         emitUnaryBuiltin(*this, E, llvm::Intrinsic::trunc, "elt.trunc"));
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index 06409c6fc774..3390f0962f67 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -1441,6 +1441,29 @@ float3 sqrt(float3);
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_sqrt)
 float4 sqrt(float4);
 
+//===----------------------------------------------------------------------===//
+// tan builtins
+//===----------------------------------------------------------------------===//
+#ifdef __HLSL_ENABLE_16_BIT
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
+half tan(half);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
+half2 tan(half2);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
+half3 tan(half3);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
+half4 tan(half4);
+#endif
+
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
+float tan(float);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
+float2 tan(float2);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
+float3 tan(float3);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
+float4 tan(float4);
+
 //===----------------------------------------------------------------------===//
 // trunc builtins
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 3179d542b1f9..e8e74467208c 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -3047,6 +3047,7 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
   case Builtin::BI__builtin_elementwise_nearbyint:
   case Builtin::BI__builtin_elementwise_sin:
   case Builtin::BI__builtin_elementwise_sqrt:
+  case Builtin::BI__builtin_elementwise_tan:
   case Builtin::BI__builtin_elementwise_trunc:
   case Builtin::BI__builtin_elementwise_canonicalize: {
     if (PrepareBuiltinElementwiseMathOneArgCall(TheCall))
@@ -5677,6 +5678,7 @@ bool Sema::CheckHLSLBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   case Builtin::BI__builtin_elementwise_roundeven:
   case Builtin::BI__builtin_elementwise_sin:
   case Builtin::BI__builtin_elementwise_sqrt:
+  case Builtin::BI__builtin_elementwise_tan:
   case Builtin::BI__builtin_elementwise_trunc: {
     if (CheckFloatOrHalfRepresentations(this, TheCall))
       return true;
diff --git a/clang/test/CodeGen/builtins-elementwise-math.c b/clang/test/CodeGen/builtins-elementwise-math.c
index 1c667e5bff1e..1b5466abd347 100644
--- a/clang/test/CodeGen/builtins-elementwise-math.c
+++ b/clang/test/CodeGen/builtins-elementwise-math.c
@@ -604,6 +604,22 @@ void test_builtin_elementwise_sqrt(float f1, float f2, double d1, double d2,
   vf2 = __builtin_elementwise_sqrt(vf1);
 }
 
+void test_builtin_elementwise_tan(float f1, float f2, double d1, double d2,
+                                  float4 vf1, float4 vf2) {
+  // CHECK-LABEL: define void @test_builtin_elementwise_tan(
+  // CHECK:      [[F1:%.+]] = load float, ptr %f1.addr, align 4
+  // CHECK-NEXT:  call float @llvm.tan.f32(float [[F1]])
+  f2 = __builtin_elementwise_tan(f1);
+
+  // CHECK:      [[D1:%.+]] = load double, ptr %d1.addr, align 8
+  // CHECK-NEXT: call double @llvm.tan.f64(double [[D1]])
+  d2 = __builtin_elementwise_tan(d1);
+
+  // CHECK:      [[VF1:%.+]] = load <4 x float>, ptr %vf1.addr, align 16
+  // CHECK-NEXT: call <4 x float> @llvm.tan.v4f32(<4 x float> [[VF1]])
+  vf2 = __builtin_elementwise_tan(vf1);
+}
+
 void test_builtin_elementwise_trunc(float f1, float f2, double d1, double d2,
                                     float4 vf1, float4 vf2) {
   // CHECK-LABEL: define void @test_builtin_elementwise_trunc(
diff --git a/clang/test/CodeGen/strictfp-elementwise-bulitins.cpp b/clang/test/CodeGen/strictfp-elementwise-bulitins.cpp
index fdf865ebbe89..c72d59499169 100644
--- a/clang/test/CodeGen/strictfp-elementwise-bulitins.cpp
+++ b/clang/test/CodeGen/strictfp-elementwise-bulitins.cpp
@@ -187,6 +187,16 @@ float4 strict_elementwise_sqrt(float4 a) {
   return __builtin_elementwise_sqrt(a);
 }
 
+// CHECK-LABEL: define dso_local noundef <4 x float> @_Z22strict_elementwise_tanDv4_f
+// CHECK-SAME: (<4 x float> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ELT_TAN:%.*]] = tail call <4 x float> @llvm.tan.v4f32(<4 x float> [[A]]) #[[ATTR4]]
+// CHECK-NEXT:    ret <4 x float> [[ELT_TAN]]
+//
+float4 strict_elementwise_tan(float4 a) {
+  return __builtin_elementwise_tan(a);
+}
+
 // CHECK-LABEL: define dso_local noundef <4 x float> @_Z24strict_elementwise_truncDv4_f
 // CHECK-SAME: (<4 x float> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // CHECK-NEXT:  entry:
diff --git a/clang/test/CodeGenHLSL/builtins/tan.hlsl b/clang/test/CodeGenHLSL/builtins/tan.hlsl
new file mode 100644
index 000000000000..aa542fac226d
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/tan.hlsl
@@ -0,0 +1,59 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   dxil-pc-shadermodel6.3-library %s -fnative-half-type \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ 
+// RUN:   --check-prefixes=CHECK,NATIVE_HALF
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF
+
+// CHECK-LABEL: test_tan_half
+// NATIVE_HALF: call half @llvm.tan.f16
+// NO_HALF: call float @llvm.tan.f32
+half test_tan_half ( half p0 ) {
+  return tan ( p0 );
+}
+
+// CHECK-LABEL: test_tan_half2
+// NATIVE_HALF: call <2 x half> @llvm.tan.v2f16
+// NO_HALF: call <2 x float> @llvm.tan.v2f32
+half2 test_tan_half2 ( half2 p0 ) {
+  return tan ( p0 );
+}
+
+// CHECK-LABEL: test_tan_half3
+// NATIVE_HALF: call <3 x half> @llvm.tan.v3f16
+// NO_HALF: call <3 x float> @llvm.tan.v3f32
+half3 test_tan_half3 ( half3 p0 ) {
+  return tan ( p0 );
+}
+
+// CHECK-LABEL: test_tan_half4
+// NATIVE_HALF: call <4 x half> @llvm.tan.v4f16
+// NO_HALF: call <4 x float> @llvm.tan.v4f32
+half4 test_tan_half4 ( half4 p0 ) {
+  return tan ( p0 );
+}
+
+// CHECK-LABEL: test_tan_float
+// CHECK: call float @llvm.tan.f32
+float test_tan_float ( float p0 ) {
+  return tan ( p0 );
+}
+
+// CHECK-LABEL: test_tan_float2
+// CHECK: call <2 x float> @llvm.tan.v2f32
+float2 test_tan_float2 ( float2 p0 ) {
+  return tan ( p0 );
+}
+
+// CHECK-LABEL: test_tan_float3
+// CHECK: call <3 x float> @llvm.tan.v3f32
+float3 test_tan_float3 ( float3 p0 ) {
+  return tan ( p0 );
+}
+
+// CHECK-LABEL: test_tan_float4
+// CHECK: call <4 x float> @llvm.tan.v4f32
+float4 test_tan_float4 ( float4 p0 ) {
+  return tan ( p0 );
+}
diff --git a/clang/test/Sema/aarch64-sve-vector-trig-ops.c b/clang/test/Sema/aarch64-sve-vector-trig-ops.c
index 70832e77fdf2..6863f32b5948 100644
--- a/clang/test/Sema/aarch64-sve-vector-trig-ops.c
+++ b/clang/test/Sema/aarch64-sve-vector-trig-ops.c
@@ -1,18 +1,24 @@
-// RUN: %clang_cc1 -triple aarch64 -target-feature +sve \
-// RUN:   -disable-O0-optnone -o - -fsyntax-only %s -verify
-// REQUIRES: aarch64-registered-target
-
-#include <arm_sve.h>
-
-
-svfloat32_t test_sin_vv_i8mf8(svfloat32_t v) {
-
-  return __builtin_elementwise_sin(v);
-  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
-}
-
-svfloat32_t test_cos_vv_i8mf8(svfloat32_t v) {
-
-  return __builtin_elementwise_cos(v);
-  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
-}
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sve \
+// RUN:   -disable-O0-optnone -o - -fsyntax-only %s -verify
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sve.h>
+
+
+svfloat32_t test_sin_vv_i8mf8(svfloat32_t v) {
+
+  return __builtin_elementwise_sin(v);
+  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
+}
+
+svfloat32_t test_cos_vv_i8mf8(svfloat32_t v) {
+
+  return __builtin_elementwise_cos(v);
+  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
+}
+
+svfloat32_t test_tan_vv_i8mf8(svfloat32_t v) {
+
+  return __builtin_elementwise_tan(v);
+  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
+}
diff --git a/clang/test/Sema/builtins-elementwise-math.c b/clang/test/Sema/builtins-elementwise-math.c
index 2e05337273ee..2e4319d158e7 100644
--- a/clang/test/Sema/builtins-elementwise-math.c
+++ b/clang/test/Sema/builtins-elementwise-math.c
@@ -626,6 +626,27 @@ void test_builtin_elementwise_sqrt(int i, float f, double d, float4 v, int3 iv,
   // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned4' (vector of 4 'unsigned int' values))}}
 }
 
+void test_builtin_elementwise_tan(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) {
+
+  struct Foo s = __builtin_elementwise_tan(f);
+  // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'float'}}
+
+  i = __builtin_elementwise_tan();
+  // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+
+  i = __builtin_elementwise_tan(i);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'int')}}
+
+  i = __builtin_elementwise_tan(f, f);
+  // expected-error@-1 {{too many arguments to function call, expected 1, have 2}}
+
+  u = __builtin_elementwise_tan(u);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned int')}}
+
+  uv = __builtin_elementwise_tan(uv);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned4' (vector of 4 'unsigned int' values))}}
+}
+
 void test_builtin_elementwise_trunc(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) {
 
   struct Foo s = __builtin_elementwise_trunc(f);
diff --git a/clang/test/Sema/riscv-rvv-vector-trig-ops.c b/clang/test/Sema/riscv-rvv-vector-trig-ops.c
index 9879b3ca4be6..459582fe2839 100644
--- a/clang/test/Sema/riscv-rvv-vector-trig-ops.c
+++ b/clang/test/Sema/riscv-rvv-vector-trig-ops.c
@@ -1,19 +1,25 @@
-// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d \
-// RUN:   -target-feature +v -target-feature +zfh -target-feature +zvfh \
-// RUN:   -disable-O0-optnone -o - -fsyntax-only %s -verify
-// REQUIRES: riscv-registered-target
-
-#include <riscv_vector.h>
-
-
-vfloat32mf2_t test_sin_vv_i8mf8(vfloat32mf2_t v) {
-
-  return __builtin_elementwise_sin(v);
-  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
-}
-
-vfloat32mf2_t test_cos_vv_i8mf8(vfloat32mf2_t v) {
-
-  return __builtin_elementwise_cos(v);
-  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
-}
+// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d \
+// RUN:   -target-feature +v -target-feature +zfh -target-feature +zvfh \
+// RUN:   -disable-O0-optnone -o - -fsyntax-only %s -verify
+// REQUIRES: riscv-registered-target
+
+#include <riscv_vector.h>
+
+
+vfloat32mf2_t test_sin_vv_i8mf8(vfloat32mf2_t v) {
+
+  return __builtin_elementwise_sin(v);
+  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
+}
+
+vfloat32mf2_t test_cos_vv_i8mf8(vfloat32mf2_t v) {
+
+  return __builtin_elementwise_cos(v);
+  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
+}
+
+vfloat32mf2_t test_tan_vv_i8mf8(vfloat32mf2_t v) {
+
+  return __builtin_elementwise_tan(v);
+  // expected-error@-1 {{1st argument must be a vector, integer or floating point type}}
+}
diff --git a/clang/test/SemaCXX/builtins-elementwise-math.cpp b/clang/test/SemaCXX/builtins-elementwise-math.cpp
index 44a44ab055e9..499f2795ddb2 100644
--- a/clang/test/SemaCXX/builtins-elementwise-math.cpp
+++ b/clang/test/SemaCXX/builtins-elementwise-math.cpp
@@ -111,6 +111,13 @@ void test_builtin_elementwise_sin() {
   static_assert(!is_const<decltype(__builtin_elementwise_sin(b))>::value);
 }
 
+void test_builtin_elementwise_tan() {
+  const float a = 42.0;
+  float b = 42.3;
+  static_assert(!is_const<decltype(__builtin_elementwise_tan(a))>::value);
+  static_assert(!is_const<decltype(__builtin_elementwise_tan(b))>::value);
+}
+
 void test_builtin_elementwise_sqrt() {
   const float a = 42.0;
   float b = 42.3;
diff --git a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
index ef0928f8fef0..4089188134d3 100644
--- a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
@@ -9,6 +9,7 @@
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_sin
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_sqrt
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_roundeven
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_tan
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_trunc
 
 double2 test_double_builtin(double2 p0) {
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index ff0fc55860de..cc7094116b8b 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -15272,6 +15272,43 @@ trapping or setting ``errno``.
 When specified with the fast-math-flag 'afn', the result may be approximated
 using a less accurate calculation.
 
+'``llvm.tan.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This is an overloaded intrinsic. You can use ``llvm.tan`` on any
+floating-point or vector of floating-point type. Not all targets support
+all types however.
+
+::
+
+      declare float     @llvm.tan.f32(float  %Val)
+      declare double    @llvm.tan.f64(double %Val)
+      declare x86_fp80  @llvm.tan.f80(x86_fp80  %Val)
+      declare fp128     @llvm.tan.f128(fp128 %Val)
+      declare ppc_fp128 @llvm.tan.ppcf128(ppc_fp128  %Val)
+
+Overview:
+"""""""""
+
+The '``llvm.tan.*``' intrinsics return the tangent of the operand.
+
+Arguments:
+""""""""""
+
+The argument and return value are floating-point numbers of the same type.
+
+Semantics:
+""""""""""
+
+Return the same value as a corresponding libm '``tan``' function but without
+trapping or setting ``errno``.
+
+When specified with the fast-math-flag 'afn', the result may be approximated
+using a less accurate calculation.
+
 '``llvm.pow.*``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 28116e5316c9..29143123193b 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1025,6 +1025,7 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in {
   def int_powi : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, llvm_anyint_ty]>;
   def int_sin  : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
   def int_cos  : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
+  def int_tan  : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
   def int_pow  : DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
                            [LLVMMatchType<0>, LLVMMatchType<0>]>;
   def int_log  : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
-- 
GitLab


From b438a817bd863699715116ee7d85b454f3289c08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jhonatan=20Cl=C3=A9to?=
 <52751492+cl3to@users.noreply.github.com>
Date: Wed, 8 May 2024 00:21:32 -0300
Subject: [PATCH 0129/1206] [Offload] Fix dataDelete op for TARGET_ALLOC_HOST
 memory type (#91134)

Summary:
The `GenericDeviceTy::dataDelete` method doesn't verify the
`TargetAllocTy` of the of the device pointer. Because of this, it can
use the `MemoryManager` to free the ptr. However, the
`TARGET_ALLOC_HOST` and `TARGET_ALLOC_SHARED` types are not allocated
using the `MemoryManager` in the `GenericDeviceTy::dataAlloc` method.
Since the `MemoryManager` uses the `DeviceAllocatorTy::free` operation
without specifying the type of the ptr, some plugins may use incorrect
operations to free ptrs of certain types. In particular, this bug causes
the CUDA plugin to use the `cuMemFree` operation on ptrs of type
`TARGET_ALLOC_HOST`, resulting in an unchecked error, as shown in the
output snippet of the test
`offload/test/api/omp_host_pinned_memory_alloc.c`:

```
omptarget --> Notifying about an unmapping: HstPtr=0x00007c6114200000
omptarget --> Call to llvm_omp_target_free_host for device 0 and address 0x00007c6114200000
omptarget --> Call to omp_get_num_devices returning 1
omptarget --> Call to omp_get_initial_device returning 1
PluginInterface --> MemoryManagerTy::free: target memory 0x00007c6114200000.
PluginInterface --> Cannot find its node. Delete it on device directly.
TARGET CUDA RTL --> Failure to free memory: Error in cuMemFree[Host]: invalid argument
omptarget --> omp_target_free deallocated device ptr
```

This patch fixes this by adding the check of the device pointer type
before calling the appropriate operation for each type.
---
 .../common/src/PluginInterface.cpp            | 26 ++++++++++++++-----
 offload/src/omptarget.cpp                     |  4 ++-
 2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index b5f3c45c835f..8de93ba17a56 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -1348,13 +1348,27 @@ Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) {
     return Plugin::success();
 
   int Res;
-  if (MemoryManager)
-    Res = MemoryManager->free(TgtPtr);
-  else
+  switch (Kind) {
+  case TARGET_ALLOC_DEFAULT:
+  case TARGET_ALLOC_DEVICE_NON_BLOCKING:
+  case TARGET_ALLOC_DEVICE:
+    if (MemoryManager) {
+      Res = MemoryManager->free(TgtPtr);
+      if (Res)
+        return Plugin::error(
+            "Failure to deallocate device pointer %p via memory manager",
+            TgtPtr);
+      break;
+    }
+    [[fallthrough]];
+  case TARGET_ALLOC_HOST:
+  case TARGET_ALLOC_SHARED:
     Res = free(TgtPtr, Kind);
-
-  if (Res)
-    return Plugin::error("Failure to deallocate device pointer %p", TgtPtr);
+    if (Res)
+      return Plugin::error(
+          "Failure to deallocate device pointer %p via device deallocator",
+          TgtPtr);
+  }
 
   // Unregister deallocated pinned memory buffer if the type is host memory.
   if (Kind == TARGET_ALLOC_HOST)
diff --git a/offload/src/omptarget.cpp b/offload/src/omptarget.cpp
index 803e941fe838..5d5c6b05051b 100644
--- a/offload/src/omptarget.cpp
+++ b/offload/src/omptarget.cpp
@@ -461,7 +461,9 @@ void targetFreeExplicit(void *DevicePtr, int DeviceNum, int Kind,
   if (!DeviceOrErr)
     FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
 
-  DeviceOrErr->deleteData(DevicePtr, Kind);
+  if (DeviceOrErr->deleteData(DevicePtr, Kind) == OFFLOAD_FAIL)
+    FATAL_MESSAGE(DeviceNum, "%s", "Failed to deallocate device ptr");
+
   DP("omp_target_free deallocated device ptr\n");
 }
 
-- 
GitLab


From f4d2f7a3b7984795d61ff45daf37c76bf3fc8604 Mon Sep 17 00:00:00 2001
From: Liao Chunyu <chunyu@iscas.ac.cn>
Date: Wed, 8 May 2024 11:22:16 +0800
Subject: [PATCH 0130/1206] [RISCV] Codegen support for XCVbi extension
 (#89719)

spec:
https://github.com/openhwgroup/cv32e40p/blob/master/docs/source/instruction_set_extensions.rst#immediate-branching-operations

Contributors: @CharKeaney, @jeremybennett, @lewis-revill,
@NandniJamnadas,
@PaoloS02, @simonpcook, @xingmingjie, @realqhc, @PhilippvK,@melonedo
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  26 +-
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      |  23 +-
 llvm/lib/Target/RISCV/RISCVInstrInfo.h        |   4 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td    |  26 ++
 .../RISCV/RISCVRedundantCopyElimination.cpp   |   6 +-
 llvm/test/CodeGen/RISCV/xcvbi.ll              | 248 ++++++++++++++++++
 6 files changed, 315 insertions(+), 18 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/xcvbi.ll

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 2818e1911ee5..3536eb4c0ba4 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -17663,6 +17663,7 @@ static bool isSelectPseudo(MachineInstr &MI) {
   default:
     return false;
   case RISCV::Select_GPR_Using_CC_GPR:
+  case RISCV::Select_GPR_Using_CC_Imm:
   case RISCV::Select_FPR16_Using_CC_GPR:
   case RISCV::Select_FPR16INX_Using_CC_GPR:
   case RISCV::Select_FPR32_Using_CC_GPR:
@@ -17846,7 +17847,9 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
   // is checked here and handled by a separate function -
   // EmitLoweredCascadedSelect.
   Register LHS = MI.getOperand(1).getReg();
-  Register RHS = MI.getOperand(2).getReg();
+  Register RHS;
+  if (MI.getOperand(2).isReg())
+    RHS = MI.getOperand(2).getReg();
   auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
 
   SmallVector<MachineInstr *, 4> SelectDebugValues;
@@ -17855,8 +17858,9 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
 
   MachineInstr *LastSelectPseudo = &MI;
   auto Next = next_nodbg(MI.getIterator(), BB->instr_end());
-  if (MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR && Next != BB->end() &&
-      Next->getOpcode() == MI.getOpcode() &&
+  if ((MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
+       MI.getOpcode() != RISCV::Select_GPR_Using_CC_Imm) &&
+      Next != BB->end() && Next->getOpcode() == MI.getOpcode() &&
       Next->getOperand(5).getReg() == MI.getOperand(0).getReg() &&
       Next->getOperand(5).isKill()) {
     return EmitLoweredCascadedSelect(MI, *Next, BB, Subtarget);
@@ -17868,6 +17872,7 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
       continue;
     if (isSelectPseudo(*SequenceMBBI)) {
       if (SequenceMBBI->getOperand(1).getReg() != LHS ||
+          !SequenceMBBI->getOperand(2).isReg() ||
           SequenceMBBI->getOperand(2).getReg() != RHS ||
           SequenceMBBI->getOperand(3).getImm() != CC ||
           SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
@@ -17917,10 +17922,16 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
   HeadMBB->addSuccessor(TailMBB);
 
   // Insert appropriate branch.
-  BuildMI(HeadMBB, DL, TII.getBrCond(CC))
-    .addReg(LHS)
-    .addReg(RHS)
-    .addMBB(TailMBB);
+  if (MI.getOperand(2).isImm())
+    BuildMI(HeadMBB, DL, TII.getBrCond(CC, MI.getOperand(2).isImm()))
+        .addReg(LHS)
+        .addImm(MI.getOperand(2).getImm())
+        .addMBB(TailMBB);
+  else
+    BuildMI(HeadMBB, DL, TII.getBrCond(CC))
+        .addReg(LHS)
+        .addReg(RHS)
+        .addMBB(TailMBB);
 
   // IfFalseMBB just falls through to TailMBB.
   IfFalseMBB->addSuccessor(TailMBB);
@@ -18166,6 +18177,7 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
            "ReadCounterWide is only to be used on riscv32");
     return emitReadCounterWidePseudo(MI, BB);
   case RISCV::Select_GPR_Using_CC_GPR:
+  case RISCV::Select_GPR_Using_CC_Imm:
   case RISCV::Select_FPR16_Using_CC_GPR:
   case RISCV::Select_FPR16INX_Using_CC_GPR:
   case RISCV::Select_FPR32_Using_CC_GPR:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 8cb9a40a98bc..444b9076005c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -833,6 +833,10 @@ static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc) {
   switch (Opc) {
   default:
     return RISCVCC::COND_INVALID;
+  case RISCV::CV_BEQIMM:
+    return RISCVCC::COND_EQ;
+  case RISCV::CV_BNEIMM:
+    return RISCVCC::COND_NE;
   case RISCV::BEQ:
     return RISCVCC::COND_EQ;
   case RISCV::BNE:
@@ -863,14 +867,14 @@ static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,
   Cond.push_back(LastInst.getOperand(1));
 }
 
-unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC) {
+unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC, bool Imm) {
   switch (CC) {
   default:
     llvm_unreachable("Unknown condition code!");
   case RISCVCC::COND_EQ:
-    return RISCV::BEQ;
+    return Imm ? RISCV::CV_BEQIMM : RISCV::BEQ;
   case RISCVCC::COND_NE:
-    return RISCV::BNE;
+    return Imm ? RISCV::CV_BNEIMM : RISCV::BNE;
   case RISCVCC::COND_LT:
     return RISCV::BLT;
   case RISCVCC::COND_GE:
@@ -882,8 +886,9 @@ unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC) {
   }
 }
 
-const MCInstrDesc &RISCVInstrInfo::getBrCond(RISCVCC::CondCode CC) const {
-  return get(RISCVCC::getBrCond(CC));
+const MCInstrDesc &RISCVInstrInfo::getBrCond(RISCVCC::CondCode CC,
+                                             bool Imm) const {
+  return get(RISCVCC::getBrCond(CC, Imm));
 }
 
 RISCVCC::CondCode RISCVCC::getOppositeBranchCondition(RISCVCC::CondCode CC) {
@@ -1032,8 +1037,10 @@ unsigned RISCVInstrInfo::insertBranch(
 
   // Either a one or two-way conditional branch.
   auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
-  MachineInstr &CondMI =
-      *BuildMI(&MBB, DL, getBrCond(CC)).add(Cond[1]).add(Cond[2]).addMBB(TBB);
+  MachineInstr &CondMI = *BuildMI(&MBB, DL, getBrCond(CC, Cond[2].isImm()))
+                              .add(Cond[1])
+                              .add(Cond[2])
+                              .addMBB(TBB);
   if (BytesAdded)
     *BytesAdded += getInstSizeInBytes(CondMI);
 
@@ -1257,6 +1264,8 @@ bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
   case RISCV::BGE:
   case RISCV::BLTU:
   case RISCV::BGEU:
+  case RISCV::CV_BEQIMM:
+  case RISCV::CV_BNEIMM:
     return isIntN(13, BrOffset);
   case RISCV::JAL:
   case RISCV::PseudoBR:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 170f813eb10d..e069717aaef2 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -45,7 +45,7 @@ enum CondCode {
 };
 
 CondCode getOppositeBranchCondition(CondCode);
-unsigned getBrCond(CondCode CC);
+unsigned getBrCond(CondCode CC, bool Imm = false);
 
 } // end of namespace RISCVCC
 
@@ -65,7 +65,7 @@ public:
   explicit RISCVInstrInfo(RISCVSubtarget &STI);
 
   MCInst getNop() const override;
-  const MCInstrDesc &getBrCond(RISCVCC::CondCode CC) const;
+  const MCInstrDesc &getBrCond(RISCVCC::CondCode CC, bool Imm = false) const;
 
   Register isLoadFromStackSlot(const MachineInstr &MI,
                                int &FrameIndex) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
index 924e91e15c34..6dae8ca8f7a8 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
@@ -704,3 +704,29 @@ let Predicates = [HasVendorXCVbitmanip, IsRV32] in {
             (CV_BITREV GPR:$rs1, cv_tuimm2:$radix, cv_tuimm5:$pts)>;
   def : Pat<(bitreverse (XLenVT GPR:$rs)), (CV_BITREV GPR:$rs, 0, 0)>;
 }
+
+//===----------------------------------------------------------------------===//
+// Patterns for immediate branching operations 
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasVendorXCVbi, IsRV32], AddedComplexity = 2 in {
+  def : Pat<(riscv_brcc GPR:$rs1, simm5:$imm5, SETEQ, bb:$imm12),
+            (CV_BEQIMM GPR:$rs1, simm5:$imm5, simm13_lsb0:$imm12)>;
+  def : Pat<(riscv_brcc GPR:$rs1, simm5:$imm5, SETNE, bb:$imm12),
+            (CV_BNEIMM GPR:$rs1, simm5:$imm5, simm13_lsb0:$imm12)>;
+
+  let usesCustomInserter = 1 in
+  def Select_GPR_Using_CC_Imm : Pseudo<(outs GPR:$dst),
+                             (ins GPR:$lhs, simm5:$imm5, ixlenimm:$cc,
+                              GPR:$truev, GPR:$falsev), []>;
+
+
+  class Selectbi<CondCode Cond>
+      : Pat<(riscv_selectcc_frag:$cc (i32 GPR:$lhs), simm5:$Constant, Cond,
+                                     (i32 GPR:$truev), GPR:$falsev),
+            (Select_GPR_Using_CC_Imm GPR:$lhs, simm5:$Constant,
+             (IntCCtoRISCVCC $cc), GPR:$truev, GPR:$falsev)>;
+
+  def : Selectbi<SETEQ>;
+  def : Selectbi<SETNE>;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVRedundantCopyElimination.cpp b/llvm/lib/Target/RISCV/RISCVRedundantCopyElimination.cpp
index 61d605fda3f5..65ff67b42479 100644
--- a/llvm/lib/Target/RISCV/RISCVRedundantCopyElimination.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRedundantCopyElimination.cpp
@@ -77,9 +77,11 @@ guaranteesZeroRegInBlock(MachineBasicBlock &MBB,
   assert(Cond.size() == 3 && "Unexpected number of operands");
   assert(TBB != nullptr && "Expected branch target basic block");
   auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
-  if (CC == RISCVCC::COND_EQ && Cond[2].getReg() == RISCV::X0 && TBB == &MBB)
+  if (CC == RISCVCC::COND_EQ && Cond[2].isReg() &&
+      Cond[2].getReg() == RISCV::X0 && TBB == &MBB)
     return true;
-  if (CC == RISCVCC::COND_NE && Cond[2].getReg() == RISCV::X0 && TBB != &MBB)
+  if (CC == RISCVCC::COND_NE && Cond[2].isReg() &&
+      Cond[2].getReg() == RISCV::X0 && TBB != &MBB)
     return true;
   return false;
 }
diff --git a/llvm/test/CodeGen/RISCV/xcvbi.ll b/llvm/test/CodeGen/RISCV/xcvbi.ll
new file mode 100644
index 000000000000..afd30faa56f9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xcvbi.ll
@@ -0,0 +1,248 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 -mtriple=riscv32 -mattr=+xcvbi -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK_NOPT
+; RUN: llc -O3 -mtriple=riscv32 -mattr=+xcvbi -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK_OPT
+
+define i32 @beqimm(i32 %a) {
+; CHECK_NOPT-LABEL: beqimm:
+; CHECK_NOPT:       # %bb.0:
+; CHECK_NOPT-NEXT:    cv.beqimm a0, 5, .LBB0_2
+; CHECK_NOPT-NEXT:    j .LBB0_1
+; CHECK_NOPT-NEXT:  .LBB0_1: # %f
+; CHECK_NOPT-NEXT:    li a0, 0
+; CHECK_NOPT-NEXT:    ret
+; CHECK_NOPT-NEXT:  .LBB0_2: # %t
+; CHECK_NOPT-NEXT:    li a0, 1
+; CHECK_NOPT-NEXT:    ret
+;
+; CHECK_OPT-LABEL: beqimm:
+; CHECK_OPT:       # %bb.0:
+; CHECK_OPT-NEXT:    cv.bneimm a0, 5, .LBB0_2
+; CHECK_OPT-NEXT:  # %bb.1: # %t
+; CHECK_OPT-NEXT:    li a0, 1
+; CHECK_OPT-NEXT:    ret
+; CHECK_OPT-NEXT:  .LBB0_2: # %f
+; CHECK_OPT-NEXT:    li a0, 0
+; CHECK_OPT-NEXT:    ret
+  %1 = icmp eq i32 %a, 5
+  br i1 %1, label %t, label %f
+f:
+  ret i32 0
+t:
+  ret i32 1
+}
+
+define i32 @bneimm(i32 %a) {
+; CHECK_NOPT-LABEL: bneimm:
+; CHECK_NOPT:       # %bb.0:
+; CHECK_NOPT-NEXT:    cv.bneimm a0, 5, .LBB1_2
+; CHECK_NOPT-NEXT:    j .LBB1_1
+; CHECK_NOPT-NEXT:  .LBB1_1: # %f
+; CHECK_NOPT-NEXT:    li a0, 0
+; CHECK_NOPT-NEXT:    ret
+; CHECK_NOPT-NEXT:  .LBB1_2: # %t
+; CHECK_NOPT-NEXT:    li a0, 1
+; CHECK_NOPT-NEXT:    ret
+;
+; CHECK_OPT-LABEL: bneimm:
+; CHECK_OPT:       # %bb.0:
+; CHECK_OPT-NEXT:    cv.beqimm a0, 5, .LBB1_2
+; CHECK_OPT-NEXT:  # %bb.1: # %t
+; CHECK_OPT-NEXT:    li a0, 1
+; CHECK_OPT-NEXT:    ret
+; CHECK_OPT-NEXT:  .LBB1_2: # %f
+; CHECK_OPT-NEXT:    li a0, 0
+; CHECK_OPT-NEXT:    ret
+  %1 = icmp ne i32 %a, 5
+  br i1 %1, label %t, label %f
+f:
+  ret i32 0
+t:
+  ret i32 1
+}
+
+define i32 @select_beqimm_1(i32 %a, i32 %x, i32 %y) {
+; CHECK_NOPT-LABEL: select_beqimm_1:
+; CHECK_NOPT:       # %bb.0: # %entry
+; CHECK_NOPT-NEXT:    addi sp, sp, -16
+; CHECK_NOPT-NEXT:    .cfi_def_cfa_offset 16
+; CHECK_NOPT-NEXT:    sw a1, 8(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    sw a2, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    cv.beqimm a0, -16, .LBB2_2
+; CHECK_NOPT-NEXT:  # %bb.1: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:  .LBB2_2: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    addi sp, sp, 16
+; CHECK_NOPT-NEXT:    ret
+;
+; CHECK_OPT-LABEL: select_beqimm_1:
+; CHECK_OPT:       # %bb.0: # %entry
+; CHECK_OPT-NEXT:    cv.beqimm a0, -16, .LBB2_2
+; CHECK_OPT-NEXT:  # %bb.1: # %entry
+; CHECK_OPT-NEXT:    mv a2, a1
+; CHECK_OPT-NEXT:  .LBB2_2: # %entry
+; CHECK_OPT-NEXT:    mv a0, a2
+; CHECK_OPT-NEXT:    ret
+entry:
+  %cmp.not = icmp eq i32 %a, -16
+  %cond = select i1 %cmp.not, i32 %y, i32 %x
+  ret i32 %cond
+}
+
+define i32 @select_beqimm_2(i32 %a, i32 %x, i32 %y) {
+; CHECK_NOPT-LABEL: select_beqimm_2:
+; CHECK_NOPT:       # %bb.0: # %entry
+; CHECK_NOPT-NEXT:    addi sp, sp, -16
+; CHECK_NOPT-NEXT:    .cfi_def_cfa_offset 16
+; CHECK_NOPT-NEXT:    sw a1, 8(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    sw a2, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    cv.beqimm a0, 0, .LBB3_2
+; CHECK_NOPT-NEXT:  # %bb.1: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:  .LBB3_2: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    addi sp, sp, 16
+; CHECK_NOPT-NEXT:    ret
+;
+; CHECK_OPT-LABEL: select_beqimm_2:
+; CHECK_OPT:       # %bb.0: # %entry
+; CHECK_OPT-NEXT:    cv.beqimm a0, 0, .LBB3_2
+; CHECK_OPT-NEXT:  # %bb.1: # %entry
+; CHECK_OPT-NEXT:    mv a2, a1
+; CHECK_OPT-NEXT:  .LBB3_2: # %entry
+; CHECK_OPT-NEXT:    mv a0, a2
+; CHECK_OPT-NEXT:    ret
+entry:
+  %cmp.not = icmp eq i32 %a, 0
+  %cond = select i1 %cmp.not, i32 %y, i32 %x
+  ret i32 %cond
+}
+
+define i32 @select_beqimm_3(i32 %a, i32 %x, i32 %y) {
+; CHECK_NOPT-LABEL: select_beqimm_3:
+; CHECK_NOPT:       # %bb.0: # %entry
+; CHECK_NOPT-NEXT:    addi sp, sp, -16
+; CHECK_NOPT-NEXT:    .cfi_def_cfa_offset 16
+; CHECK_NOPT-NEXT:    sw a1, 8(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    sw a2, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    cv.beqimm a0, 15, .LBB4_2
+; CHECK_NOPT-NEXT:  # %bb.1: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:  .LBB4_2: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    addi sp, sp, 16
+; CHECK_NOPT-NEXT:    ret
+;
+; CHECK_OPT-LABEL: select_beqimm_3:
+; CHECK_OPT:       # %bb.0: # %entry
+; CHECK_OPT-NEXT:    cv.beqimm a0, 15, .LBB4_2
+; CHECK_OPT-NEXT:  # %bb.1: # %entry
+; CHECK_OPT-NEXT:    mv a2, a1
+; CHECK_OPT-NEXT:  .LBB4_2: # %entry
+; CHECK_OPT-NEXT:    mv a0, a2
+; CHECK_OPT-NEXT:    ret
+entry:
+  %cmp.not = icmp eq i32 %a, 15
+  %cond = select i1 %cmp.not, i32 %y, i32 %x
+  ret i32 %cond
+}
+
+define i32 @select_no_beqimm_1(i32 %a, i32 %x, i32 %y) {
+; CHECK_NOPT-LABEL: select_no_beqimm_1:
+; CHECK_NOPT:       # %bb.0: # %entry
+; CHECK_NOPT-NEXT:    addi sp, sp, -16
+; CHECK_NOPT-NEXT:    .cfi_def_cfa_offset 16
+; CHECK_NOPT-NEXT:    sw a1, 8(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    li a1, -17
+; CHECK_NOPT-NEXT:    sw a2, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    beq a0, a1, .LBB5_2
+; CHECK_NOPT-NEXT:  # %bb.1: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:  .LBB5_2: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    addi sp, sp, 16
+; CHECK_NOPT-NEXT:    ret
+;
+; CHECK_OPT-LABEL: select_no_beqimm_1:
+; CHECK_OPT:       # %bb.0: # %entry
+; CHECK_OPT-NEXT:    li a3, -17
+; CHECK_OPT-NEXT:    beq a0, a3, .LBB5_2
+; CHECK_OPT-NEXT:  # %bb.1: # %entry
+; CHECK_OPT-NEXT:    mv a2, a1
+; CHECK_OPT-NEXT:  .LBB5_2: # %entry
+; CHECK_OPT-NEXT:    mv a0, a2
+; CHECK_OPT-NEXT:    ret
+entry:
+  %cmp.not = icmp eq i32 %a, -17
+  %cond = select i1 %cmp.not, i32 %y, i32 %x
+  ret i32 %cond
+}
+
+define i32 @select_no_beqimm_2(i32 %a, i32 %x, i32 %y) {
+; CHECK_NOPT-LABEL: select_no_beqimm_2:
+; CHECK_NOPT:       # %bb.0: # %entry
+; CHECK_NOPT-NEXT:    addi sp, sp, -16
+; CHECK_NOPT-NEXT:    .cfi_def_cfa_offset 16
+; CHECK_NOPT-NEXT:    sw a1, 8(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    li a1, 16
+; CHECK_NOPT-NEXT:    sw a2, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    beq a0, a1, .LBB6_2
+; CHECK_NOPT-NEXT:  # %bb.1: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:  .LBB6_2: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    addi sp, sp, 16
+; CHECK_NOPT-NEXT:    ret
+;
+; CHECK_OPT-LABEL: select_no_beqimm_2:
+; CHECK_OPT:       # %bb.0: # %entry
+; CHECK_OPT-NEXT:    li a3, 16
+; CHECK_OPT-NEXT:    beq a0, a3, .LBB6_2
+; CHECK_OPT-NEXT:  # %bb.1: # %entry
+; CHECK_OPT-NEXT:    mv a2, a1
+; CHECK_OPT-NEXT:  .LBB6_2: # %entry
+; CHECK_OPT-NEXT:    mv a0, a2
+; CHECK_OPT-NEXT:    ret
+entry:
+  %cmp.not = icmp eq i32 %a, 16
+  %cond = select i1 %cmp.not, i32 %y, i32 %x
+  ret i32 %cond
+}
+
+define i32 @select_bneimm_1(i32 %a, i32 %x, i32 %y) {
+; CHECK_NOPT-LABEL: select_bneimm_1:
+; CHECK_NOPT:       # %bb.0: # %entry
+; CHECK_NOPT-NEXT:    addi sp, sp, -16
+; CHECK_NOPT-NEXT:    .cfi_def_cfa_offset 16
+; CHECK_NOPT-NEXT:    sw a1, 8(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    sw a2, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:    cv.bneimm a0, 0, .LBB7_2
+; CHECK_NOPT-NEXT:  # %bb.1: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK_NOPT-NEXT:  .LBB7_2: # %entry
+; CHECK_NOPT-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK_NOPT-NEXT:    addi sp, sp, 16
+; CHECK_NOPT-NEXT:    ret
+;
+; CHECK_OPT-LABEL: select_bneimm_1:
+; CHECK_OPT:       # %bb.0: # %entry
+; CHECK_OPT-NEXT:    cv.bneimm a0, 0, .LBB7_2
+; CHECK_OPT-NEXT:  # %bb.1: # %entry
+; CHECK_OPT-NEXT:    mv a2, a1
+; CHECK_OPT-NEXT:  .LBB7_2: # %entry
+; CHECK_OPT-NEXT:    mv a0, a2
+; CHECK_OPT-NEXT:    ret
+entry:
+  %cmp.not = icmp ne i32 %a, 0
+  %cond = select i1 %cmp.not, i32 %y, i32 %x
+  ret i32 %cond
+}
+
-- 
GitLab


From 48b6f4a18255816df51fcab7648c5a7f205dfe14 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Wed, 8 May 2024 11:33:05 +0800
Subject: [PATCH 0131/1206] [RISCV] Rewrite spill-fpr-scalar.ll test to not use
 vsetvli. NFC (#91428)

It was relying on the fact that vsetvlis have side effects to prevent
reordering, but #91319 proposes to remove the side effects. This reworks
it to use volatile loads and stores instead.
---
 llvm/test/CodeGen/RISCV/spill-fpr-scalar.ll | 73 ++++++++-------------
 1 file changed, 28 insertions(+), 45 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/spill-fpr-scalar.ll b/llvm/test/CodeGen/RISCV/spill-fpr-scalar.ll
index 48fb21dc5a8a..6b9b88d90de6 100644
--- a/llvm/test/CodeGen/RISCV/spill-fpr-scalar.ll
+++ b/llvm/test/CodeGen/RISCV/spill-fpr-scalar.ll
@@ -1,75 +1,58 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+v,+d,+zfh,+zvfh -target-abi=lp64 \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh -target-abi=lp64 \
 ; RUN:   -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s
 
-declare half @llvm.riscv.vfmv.f.s.nxv1f16(<vscale x 1 x half>)
-declare float @llvm.riscv.vfmv.f.s.nxv1f32(<vscale x 1 x float>)
-declare double @llvm.riscv.vfmv.f.s.nxv1f64(<vscale x 1 x double>)
-
-declare <vscale x 1 x half> @llvm.riscv.vfmv.v.f.nxv1f16(<vscale x 1 x half>, half, i64);
-declare <vscale x 1 x float> @llvm.riscv.vfmv.v.f.nxv1f32(<vscale x 1 x float>, float, i64);
-declare <vscale x 1 x double> @llvm.riscv.vfmv.v.f.nxv1f64(<vscale x 1 x double>, double, i64);
-
-define <vscale x 1 x half> @intrinsic_vfmv.f.s_s_nxv1f16(<vscale x 1 x half> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vfmv.f.s_s_nxv1f16:
-; CHECK:       # %bb.0: # %entry
+define void @spill_half(ptr) nounwind {
+; CHECK-LABEL: spill_half:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
-; CHECK-NEXT:    vfmv.f.s fa5, v8
+; CHECK-NEXT:    flh fa5, 0(a0)
 ; CHECK-NEXT:    fsh fa5, 14(sp) # 2-byte Folded Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    flh fa5, 14(sp) # 2-byte Folded Reload
-; CHECK-NEXT:    vfmv.v.f v8, fa5
+; CHECK-NEXT:    fsh fa5, 0(a0)
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
-entry:
-  %a = call half @llvm.riscv.vfmv.f.s.nxv1f16(<vscale x 1 x half> %0)
-  tail call void asm sideeffect "", "~{f0_d},~{f1_d},~{f2_d},~{f3_d},~{f4_d},~{f5_d},~{f6_d},~{f7_d},~{f8_d},~{f9_d},~{f10_d},~{f11_d},~{f12_d},~{f13_d},~{f14_d},~{f15_d},~{f16_d},~{f17_d},~{f18_d},~{f19_d},~{f20_d},~{f21_d},~{f22_d},~{f23_d},~{f24_d},~{f25_d},~{f26_d},~{f27_d},~{f28_d},~{f29_d},~{f30_d},~{f31_d}"()
-  %b = call <vscale x 1 x half> @llvm.riscv.vfmv.v.f.nxv1f16(<vscale x 1 x half> undef, half %a, i64 %1)
-  ret <vscale x 1 x half> %b
+  %2 = load volatile half, ptr %0
+  call void asm sideeffect "", "~{f0_d},~{f1_d},~{f2_d},~{f3_d},~{f4_d},~{f5_d},~{f6_d},~{f7_d},~{f8_d},~{f9_d},~{f10_d},~{f11_d},~{f12_d},~{f13_d},~{f14_d},~{f15_d},~{f16_d},~{f17_d},~{f18_d},~{f19_d},~{f20_d},~{f21_d},~{f22_d},~{f23_d},~{f24_d},~{f25_d},~{f26_d},~{f27_d},~{f28_d},~{f29_d},~{f30_d},~{f31_d}"()
+  store volatile half %2, ptr %0
+  ret void
 }
 
-define <vscale x 1 x float> @intrinsic_vfmv.f.s_s_nxv1f32(<vscale x 1 x float> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vfmv.f.s_s_nxv1f32:
-; CHECK:       # %bb.0: # %entry
+define void @spill_float(ptr) nounwind {
+; CHECK-LABEL: spill_float:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; CHECK-NEXT:    vfmv.f.s fa5, v8
+; CHECK-NEXT:    flw fa5, 0(a0)
 ; CHECK-NEXT:    fsw fa5, 12(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    flw fa5, 12(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    vfmv.v.f v8, fa5
+; CHECK-NEXT:    fsw fa5, 0(a0)
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
-entry:
-  %a = call float @llvm.riscv.vfmv.f.s.nxv1f32(<vscale x 1 x float> %0)
-  tail call void asm sideeffect "", "~{f0_d},~{f1_d},~{f2_d},~{f3_d},~{f4_d},~{f5_d},~{f6_d},~{f7_d},~{f8_d},~{f9_d},~{f10_d},~{f11_d},~{f12_d},~{f13_d},~{f14_d},~{f15_d},~{f16_d},~{f17_d},~{f18_d},~{f19_d},~{f20_d},~{f21_d},~{f22_d},~{f23_d},~{f24_d},~{f25_d},~{f26_d},~{f27_d},~{f28_d},~{f29_d},~{f30_d},~{f31_d}"()
-  %b = call <vscale x 1 x float> @llvm.riscv.vfmv.v.f.nxv1f32(<vscale x 1 x float> undef, float %a, i64 %1)
-  ret <vscale x 1 x float> %b
+  %2 = load volatile float, ptr %0
+  call void asm sideeffect "", "~{f0_d},~{f1_d},~{f2_d},~{f3_d},~{f4_d},~{f5_d},~{f6_d},~{f7_d},~{f8_d},~{f9_d},~{f10_d},~{f11_d},~{f12_d},~{f13_d},~{f14_d},~{f15_d},~{f16_d},~{f17_d},~{f18_d},~{f19_d},~{f20_d},~{f21_d},~{f22_d},~{f23_d},~{f24_d},~{f25_d},~{f26_d},~{f27_d},~{f28_d},~{f29_d},~{f30_d},~{f31_d}"()
+  store volatile float %2, ptr %0
+  ret void
 }
 
-define <vscale x 1 x double> @intrinsic_vfmv.f.s_s_nxv1f64(<vscale x 1 x double> %0, i64 %1) nounwind {
-; CHECK-LABEL: intrinsic_vfmv.f.s_s_nxv1f64:
-; CHECK:       # %bb.0: # %entry
+define void @spill_double(ptr) nounwind {
+; CHECK-LABEL: spill_double:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.f.s fa5, v8
+; CHECK-NEXT:    fld fa5, 0(a0)
 ; CHECK-NEXT:    fsd fa5, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    fld fa5, 8(sp) # 8-byte Folded Reload
-; CHECK-NEXT:    vfmv.v.f v8, fa5
+; CHECK-NEXT:    fsd fa5, 0(a0)
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
-entry:
-  %a = call double @llvm.riscv.vfmv.f.s.nxv1f64(<vscale x 1 x double> %0)
-  tail call void asm sideeffect "", "~{f0_d},~{f1_d},~{f2_d},~{f3_d},~{f4_d},~{f5_d},~{f6_d},~{f7_d},~{f8_d},~{f9_d},~{f10_d},~{f11_d},~{f12_d},~{f13_d},~{f14_d},~{f15_d},~{f16_d},~{f17_d},~{f18_d},~{f19_d},~{f20_d},~{f21_d},~{f22_d},~{f23_d},~{f24_d},~{f25_d},~{f26_d},~{f27_d},~{f28_d},~{f29_d},~{f30_d},~{f31_d}"()
-  %b = call <vscale x 1 x double> @llvm.riscv.vfmv.v.f.nxv1f64(<vscale x 1 x double> undef, double %a, i64 %1)
-  ret <vscale x 1 x double> %b
+  %2 = load volatile double, ptr %0
+  call void asm sideeffect "", "~{f0_d},~{f1_d},~{f2_d},~{f3_d},~{f4_d},~{f5_d},~{f6_d},~{f7_d},~{f8_d},~{f9_d},~{f10_d},~{f11_d},~{f12_d},~{f13_d},~{f14_d},~{f15_d},~{f16_d},~{f17_d},~{f18_d},~{f19_d},~{f20_d},~{f21_d},~{f22_d},~{f23_d},~{f24_d},~{f25_d},~{f26_d},~{f27_d},~{f28_d},~{f29_d},~{f30_d},~{f31_d}"()
+  store volatile double %2, ptr %0
+  ret void
 }
-- 
GitLab


From 812c3025ec033ad1f306aff7f8b6e6695a79ee35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Wed, 8 May 2024 05:35:23 +0200
Subject: [PATCH 0132/1206] [GlobalIsel][AArch64] legalize ptr add (#89218)

LLVM ERROR: unable to legalize instruction: %275:_(<4 x p0>) = G_PTR_ADD
%268:_, %274:_(<4 x s64>) (in function: prepare_for_pass)
---
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |  4 +-
 .../AArch64/GlobalISel/legalize-ptr-add.mir   | 59 ++++++++++++++++++-
 2 files changed, 59 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 243891249668..d4aac94d24f1 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -177,7 +177,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
 
   getActionDefinitionsBuilder(G_PTR_ADD)
       .legalFor({{p0, s64}, {v2p0, v2s64}})
-      .clampScalar(1, s64, s64);
+      .clampScalar(1, s64, s64)
+      .clampNumElements(0, v2p0, v2p0)
+      .clampNumElements(1, v2s64, v2s64);
 
   getActionDefinitionsBuilder(G_PTRMASK).legalFor({{p0, s64}});
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ptr-add.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ptr-add.mir
index 1ecd36b55380..1d3f7eab79d6 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ptr-add.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-ptr-add.mir
@@ -6,12 +6,65 @@ body:             |
   bb.0.entry:
     ; CHECK-LABEL: name: test_ptr_add_vec_p0
     ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p0>) = COPY $q0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[PTR_ADD:%[0-9]+]]:_(<2 x p0>) = G_PTR_ADD [[COPY]], [[COPY1]](<2 x s64>)
-    ; CHECK: $q0 = COPY [[PTR_ADD]](<2 x p0>)
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(<2 x p0>) = G_PTR_ADD [[COPY]], [[COPY1]](<2 x s64>)
+    ; CHECK-NEXT: $q0 = COPY [[PTR_ADD]](<2 x p0>)
     %0:_(<2 x p0>) = COPY $q0
     %1:_(<2 x s64>) = COPY $q1
     %3:_(<2 x p0>) = G_PTR_ADD %0, %1(<2 x s64>)
     $q0 = COPY %3(<2 x p0>)
 
 ...
+---
+name:            test_ptr_add_vec_4xp0
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_ptr_add_vec_4xp0
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(p0) = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(p0) = COPY $x3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x4
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x5
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x6
+    ; CHECK-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x7
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p0>) = G_BUILD_VECTOR [[COPY]](p0), [[COPY1]](p0)
+    ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x p0>) = G_BUILD_VECTOR [[COPY2]](p0), [[COPY3]](p0)
+    ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[COPY4]](s64), [[COPY5]](s64)
+    ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[COPY6]](s64), [[COPY7]](s64)
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(<2 x p0>) = G_PTR_ADD [[BUILD_VECTOR]], [[BUILD_VECTOR2]](<2 x s64>)
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(<2 x p0>) = G_PTR_ADD [[BUILD_VECTOR1]], [[BUILD_VECTOR3]](<2 x s64>)
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %one:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: %extract0:_(p0) = G_EXTRACT_VECTOR_ELT [[PTR_ADD]](<2 x p0>), %zero(s64)
+    ; CHECK-NEXT: %extract1:_(p0) = G_EXTRACT_VECTOR_ELT [[PTR_ADD]](<2 x p0>), %one(s64)
+    ; CHECK-NEXT: %extract2:_(p0) = G_EXTRACT_VECTOR_ELT [[PTR_ADD1]](<2 x p0>), %zero(s64)
+    ; CHECK-NEXT: %extract3:_(p0) = G_EXTRACT_VECTOR_ELT [[PTR_ADD1]](<2 x p0>), %one(s64)
+    ; CHECK-NEXT: $x0 = COPY %extract0(p0)
+    ; CHECK-NEXT: $x1 = COPY %extract1(p0)
+    ; CHECK-NEXT: $x2 = COPY %extract2(p0)
+    ; CHECK-NEXT: $x3 = COPY %extract3(p0)
+    %0:_(p0) = COPY $x0
+    %1:_(p0) = COPY $x1
+    %2:_(p0) = COPY $x2
+    %3:_(p0) = COPY $x3
+    %4:_(s64) = COPY $x4
+    %5:_(s64) = COPY $x5
+    %6:_(s64) = COPY $x6
+    %7:_(s64) = COPY $x7
+    %ptr:_(<4 x p0>) = G_BUILD_VECTOR %0(p0), %1(p0), %2(p0), %3(p0)
+    %add:_(<4 x s64>) = G_BUILD_VECTOR %4(s64), %5(s64), %6(s64), %7(s64)
+    %res:_(<4 x p0>) = G_PTR_ADD %ptr, %add(<4 x s64>)
+    %zero:_(s64) = G_CONSTANT i64 0
+    %one:_(s64) = G_CONSTANT i64 1
+    %two:_(s64) = G_CONSTANT i64 2
+    %three:_(s64) = G_CONSTANT i64 3
+    %extract0:_(p0) = G_EXTRACT_VECTOR_ELT %res(<4 x p0>), %zero(s64)
+    %extract1:_(p0) = G_EXTRACT_VECTOR_ELT %res(<4 x p0>), %one(s64)
+    %extract2:_(p0) = G_EXTRACT_VECTOR_ELT %res(<4 x p0>), %two(s64)
+    %extract3:_(p0) = G_EXTRACT_VECTOR_ELT %res(<4 x p0>), %three(s64)
+    $x0 = COPY %extract0(p0)
+    $x1 = COPY %extract1(p0)
+    $x2 = COPY %extract2(p0)
+    $x3 = COPY %extract3(p0)
+...
-- 
GitLab


From 2c209957819328481554e7c5929d134502b4972a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 7 May 2024 20:56:55 -0700
Subject: [PATCH 0133/1206] [RISCV] Detect duplicate extensions in
 parseNormalizedArchString. (#91416)

This detects the same extension name being added twice. Mostly I'm
worried about the case that the same string appears with two different
versions. We will only preserve one of the versions.

We could allow the same version to be repeated, but that doesn't seem
useful at the moment.

I've updated addExtension to use map::emplace instead of
map::operator[]. This means we only keep the first version if there are
duplicates. Previously we kept the last version, but that shouldn't matter
now that we don't allow duplicates. parseArchString already doesn't allow
duplicates.
---
 llvm/include/llvm/TargetParser/RISCVISAInfo.h    | 2 +-
 llvm/lib/TargetParser/RISCVISAInfo.cpp           | 8 +++++---
 llvm/unittests/TargetParser/RISCVISAInfoTest.cpp | 8 ++++++++
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/TargetParser/RISCVISAInfo.h b/llvm/include/llvm/TargetParser/RISCVISAInfo.h
index 36617a9b6259..12f6b46fb3ce 100644
--- a/llvm/include/llvm/TargetParser/RISCVISAInfo.h
+++ b/llvm/include/llvm/TargetParser/RISCVISAInfo.h
@@ -87,7 +87,7 @@ private:
 
   RISCVISAUtils::OrderedExtensionMap Exts;
 
-  void addExtension(StringRef ExtName, RISCVISAUtils::ExtensionVersion Version);
+  bool addExtension(StringRef ExtName, RISCVISAUtils::ExtensionVersion Version);
 
   Error checkDependency();
 
diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 9c2ac8c3893f..96590745b2eb 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -159,9 +159,9 @@ findDefaultVersion(StringRef ExtName) {
   return std::nullopt;
 }
 
-void RISCVISAInfo::addExtension(StringRef ExtName,
+bool RISCVISAInfo::addExtension(StringRef ExtName,
                                 RISCVISAUtils::ExtensionVersion Version) {
-  Exts[ExtName.str()] = Version;
+  return Exts.emplace(ExtName, Version).second;
 }
 
 static StringRef getExtensionTypeDesc(StringRef Ext) {
@@ -492,7 +492,9 @@ RISCVISAInfo::parseNormalizedArchString(StringRef Arch) {
                                "'" + Twine(ExtName[0]) +
                                    "' must be followed by a letter");
 
-    ISAInfo->addExtension(ExtName, {MajorVersion, MinorVersion});
+    if (!ISAInfo->addExtension(ExtName, {MajorVersion, MinorVersion}))
+      return createStringError(errc::invalid_argument,
+                               "duplicate extension '" + ExtName + "'");
   }
   ISAInfo->updateImpliedLengths();
   return std::move(ISAInfo);
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 83b52d0527c3..0e807cfb8e3b 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -78,6 +78,14 @@ TEST(ParseNormalizedArchString, RejectsBadX) {
   }
 }
 
+TEST(ParseNormalizedArchString, DuplicateExtension) {
+  for (StringRef Input : {"rv64i2p0_a2p0_a1p0"}) {
+    EXPECT_EQ(
+        toString(RISCVISAInfo::parseNormalizedArchString(Input).takeError()),
+        "duplicate extension 'a'");
+  }
+}
+
 TEST(ParseNormalizedArchString, AcceptsValidBaseISAsAndSetsXLen) {
   auto MaybeRV32I = RISCVISAInfo::parseNormalizedArchString("rv32i2p0");
   ASSERT_THAT_EXPECTED(MaybeRV32I, Succeeded());
-- 
GitLab


From 85ef6b7c364f3b57c13c179bf278fe47366287a2 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Wed, 8 May 2024 00:10:08 -0400
Subject: [PATCH 0134/1206] [DXIL] Add tan intrinsic part 2 (#90277)

This change is an implementation of #87367's investigation on supporting
IEEE math operations as intrinsics.
Which was discussed in this RFC:
https://discourse.llvm.org/t/rfc-all-the-math-intrinsics/78294

If you want an overarching view of how this will all connect see:
https://github.com/llvm/llvm-project/pull/90088

Changes:
-  `llvm/include/llvm/IR/Intrinsics.td` - Create the tan intrinsic
- `llvm/lib/Target/DirectX/DXIL.td` - Map `int_tan` (the tan intrinsic)
to the equivalent DXIL Op.
---
 llvm/lib/Target/DirectX/DXIL.td        |  3 +++
 llvm/test/CodeGen/DirectX/tan.ll       | 20 ++++++++++++++++++++
 llvm/test/CodeGen/DirectX/tan_error.ll | 10 ++++++++++
 3 files changed, 33 insertions(+)
 create mode 100644 llvm/test/CodeGen/DirectX/tan.ll
 create mode 100644 llvm/test/CodeGen/DirectX/tan_error.ll

diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index cd388ed3e319..24a0c8524230 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -266,6 +266,9 @@ def Cos  : DXILOpMapping<12, unary, int_cos,
 def Sin  : DXILOpMapping<13, unary, int_sin,
                          "Returns sine(theta) for theta in radians.",
                          [llvm_halforfloat_ty, LLVMMatchType<0>]>;
+def Tan  : DXILOpMapping<14, unary, int_tan,
+                         "Returns tangent(theta) for theta in radians.",
+                         [llvm_halforfloat_ty, LLVMMatchType<0>]>;
 def Exp2 : DXILOpMapping<21, unary, int_exp2,
                          "Returns the base 2 exponential, or 2**x, of the specified value."
                          "exp2(x) = 2**x.",
diff --git a/llvm/test/CodeGen/DirectX/tan.ll b/llvm/test/CodeGen/DirectX/tan.ll
new file mode 100644
index 000000000000..567ab02d40f9
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/tan.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -dxil-op-lower < %s | FileCheck %s
+
+; Make sure dxil operation function calls for tan are generated for float and half.
+
+define noundef float @tan_float(float noundef %a) #0 {
+entry:
+; CHECK:call float @dx.op.unary.f32(i32 14, float %{{.*}})
+  %elt.tan = call float @llvm.tan.f32(float %a)
+  ret float %elt.tan
+}
+
+define noundef half @tan_half(half noundef %a) #0 {
+entry:
+; CHECK:call half @dx.op.unary.f16(i32 14, half %{{.*}})
+  %elt.tan = call half @llvm.tan.f16(half %a)
+  ret half %elt.tan
+}
+
+declare half @llvm.tan.f16(half)
+declare float @llvm.tan.f32(float)
diff --git a/llvm/test/CodeGen/DirectX/tan_error.ll b/llvm/test/CodeGen/DirectX/tan_error.ll
new file mode 100644
index 000000000000..c870c36f5492
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/tan_error.ll
@@ -0,0 +1,10 @@
+; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s
+
+; DXIL operation tan does not support double overload type
+; CHECK: LLVM ERROR: Invalid Overload
+
+define noundef double @tan_double(double noundef %a) #0 {
+entry:
+  %1 = call double @llvm.tan.f64(double %a)
+  ret double %1
+}
-- 
GitLab


From ef84452571b8e8f4a38a173e6adf6a5ecbbde97e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 7 May 2024 21:17:50 -0700
Subject: [PATCH 0135/1206] [DAGCombiner] Be more careful about looking through
 extends and truncates in mergeTruncStores. (#91375)

Previously we recursively looked through extends and truncates on both
SourceValue and WideVal.

SourceValue is the largest source found for each of the stores we are
combining. WideVal is the source for the current store.

Previously we could incorrectly look through a (zext (trunc X)) pair and
incorrectly believe X to be a good source.

I think we could also look through a zext on one store and a sext on
another store and arbitrarily pick one of the extends as the final
source.

With this patch we only look through one level of extend or truncate.
And we don't look through extends/truncs on both SourceValue and WideVal
at the same time.

This may lose some optimization cases, but keeps everything we had tests
for.

Fixes #90936.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 21 +++++++++++--------
 llvm/test/CodeGen/AArch64/pr90936.ll          |  8 +++++--
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 05ab6e2e4820..e835bd950a7b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -8728,15 +8728,16 @@ static std::optional<bool> isBigEndian(const ArrayRef<int64_t> ByteOffsets,
   return BigEndian;
 }
 
+// Look through one layer of truncate or extend.
 static SDValue stripTruncAndExt(SDValue Value) {
   switch (Value.getOpcode()) {
   case ISD::TRUNCATE:
   case ISD::ZERO_EXTEND:
   case ISD::SIGN_EXTEND:
   case ISD::ANY_EXTEND:
-    return stripTruncAndExt(Value.getOperand(0));
+    return Value.getOperand(0);
   }
-  return Value;
+  return SDValue();
 }
 
 /// Match a pattern where a wide type scalar value is stored by several narrow
@@ -8849,16 +8850,18 @@ SDValue DAGCombiner::mergeTruncStores(StoreSDNode *N) {
     }
 
     // Stores must share the same source value with different offsets.
-    // Truncate and extends should be stripped to get the single source value.
     if (!SourceValue)
       SourceValue = WideVal;
-    else if (stripTruncAndExt(SourceValue) != stripTruncAndExt(WideVal))
-      return SDValue();
-    else if (SourceValue.getValueType() != WideVT) {
-      if (WideVal.getValueType() == WideVT ||
-          WideVal.getScalarValueSizeInBits() >
-              SourceValue.getScalarValueSizeInBits())
+    else if (SourceValue != WideVal) {
+      // Truncate and extends can be stripped to see if the values are related.
+      if (stripTruncAndExt(SourceValue) != WideVal &&
+          stripTruncAndExt(WideVal) != SourceValue)
+        return SDValue();
+
+      if (WideVal.getScalarValueSizeInBits() >
+          SourceValue.getScalarValueSizeInBits())
         SourceValue = WideVal;
+
       // Give up if the source value type is smaller than the store size.
       if (SourceValue.getScalarValueSizeInBits() < WideVT.getScalarSizeInBits())
         return SDValue();
diff --git a/llvm/test/CodeGen/AArch64/pr90936.ll b/llvm/test/CodeGen/AArch64/pr90936.ll
index cd816cdbf735..3ed8468b37f4 100644
--- a/llvm/test/CodeGen/AArch64/pr90936.ll
+++ b/llvm/test/CodeGen/AArch64/pr90936.ll
@@ -22,8 +22,12 @@ bb:
 define void @g(i32 %arg, ptr %arg1) {
 ; CHECK-LABEL: g:
 ; CHECK:       // %bb.0: // %bb
-; CHECK-NEXT:    and w8, w0, #0xff
-; CHECK-NEXT:    str w8, [x1]
+; CHECK-NEXT:    lsr w8, w0, #8
+; CHECK-NEXT:    lsr w9, w0, #16
+; CHECK-NEXT:    strb w0, [x1]
+; CHECK-NEXT:    strb wzr, [x1, #3]
+; CHECK-NEXT:    strb w8, [x1, #1]
+; CHECK-NEXT:    strb w9, [x1, #2]
 ; CHECK-NEXT:    ret
 bb:
   %i = trunc i32 %arg to i8
-- 
GitLab


From 0d93b01c3b1e2e543acec3f36db639b8b7b0b20d Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 7 May 2024 21:18:28 -0700
Subject: [PATCH 0136/1206] [RISCV] Don't crash if parseNormalizedArchString
 encounters a multi-letter extension with an unknown prefix. (#91398)

The sorting code previously asserted if a prefix was multiple letters,
but didn't start with s, x, or z.

Replace the assert with an explicit check and sort the multi-letter
extension after the known multi-letter prefixes.
---
 llvm/lib/Support/RISCVISAUtils.cpp               | 11 +++++++----
 llvm/unittests/TargetParser/RISCVISAInfoTest.cpp |  8 ++++++++
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Support/RISCVISAUtils.cpp b/llvm/lib/Support/RISCVISAUtils.cpp
index 46efe9369507..d6b002e66e7a 100644
--- a/llvm/lib/Support/RISCVISAUtils.cpp
+++ b/llvm/lib/Support/RISCVISAUtils.cpp
@@ -24,13 +24,15 @@ using namespace llvm;
 // -Multi-letter extensions starting with 's' in alphabetical order.
 // -(TODO) Multi-letter extensions starting with 'zxm' in alphabetical order.
 // -X extensions in alphabetical order.
+// -Unknown multi-letter extensions in alphabetical order.
 // These flags are used to indicate the category. The first 6 bits store the
 // single letter extension rank for single letter and multi-letter extensions
 // starting with 'z'.
 enum RankFlags {
   RF_Z_EXTENSION = 1 << 6,
-  RF_S_EXTENSION = 1 << 7,
-  RF_X_EXTENSION = 1 << 8,
+  RF_S_EXTENSION = 2 << 6,
+  RF_X_EXTENSION = 3 << 6,
+  RF_UNKNOWN_MULTILETTER_EXTENSION = 4 << 6,
 };
 
 // Get the rank for single-letter extension, lower value meaning higher
@@ -68,8 +70,9 @@ static unsigned getExtensionRank(const std::string &ExtName) {
   case 'x':
     return RF_X_EXTENSION;
   default:
-    assert(ExtName.size() == 1);
-    return singleLetterExtensionRank(ExtName[0]);
+    if (ExtName.size() == 1)
+      return singleLetterExtensionRank(ExtName[0]);
+    return RF_UNKNOWN_MULTILETTER_EXTENSION;
   }
 }
 
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 0e807cfb8e3b..f9e386a85fea 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -150,6 +150,14 @@ TEST(ParseNormalizedArchString, UpdatesFLenMinVLenMaxELen) {
   EXPECT_EQ(Info.getMaxELenFp(), 64U);
 }
 
+TEST(ParseNormalizedArchString, AcceptsUnknownMultiletter) {
+  auto MaybeISAInfo = RISCVISAInfo::parseNormalizedArchString(
+      "rv64i2p0_f2p0_d2p0_zicsr2p0_ykk1p0");
+  ASSERT_THAT_EXPECTED(MaybeISAInfo, Succeeded());
+  RISCVISAInfo &Info = **MaybeISAInfo;
+  EXPECT_EQ(Info.toString(), "rv64i2p0_f2p0_d2p0_zicsr2p0_ykk1p0");
+}
+
 TEST(ParseArchString, RejectsInvalidChars) {
   for (StringRef Input : {"RV32", "rV64", "rv32i2P0", "rv64i2p0_A2p0"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-- 
GitLab


From 8296f061aafb844bf3b9b002b7791ade7a1d3006 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Wed, 8 May 2024 12:33:01 +0800
Subject: [PATCH 0137/1206] [RISCV] Add invariants that registers always have
 definitions. NFC (#90587)

For vector merge operands, we check if it's a NoRegister beforehand so
any other register type should have a definition.

For VL operands, they don't get replaced with NoRegisters since they're
scalar and should also always have a definition, even if it's an
implicit_def.

All the definitions at this stage should also be unique, this will
change in #70549
---
 llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 42 +++++++++++---------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index eaebdc2e54be..06456f97f5eb 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -194,19 +194,22 @@ static bool hasUndefinedMergeOp(const MachineInstr &MI,
   if (UseMO.getReg().isPhysical())
     return false;
 
-  if (MachineInstr *UseMI = MRI.getVRegDef(UseMO.getReg())) {
-    if (UseMI->isImplicitDef())
-      return true;
+  MachineInstr *UseMI = MRI.getUniqueVRegDef(UseMO.getReg());
+  assert(UseMI);
+  if (UseMI->isImplicitDef())
+    return true;
 
-    if (UseMI->isRegSequence()) {
-      for (unsigned i = 1, e = UseMI->getNumOperands(); i < e; i += 2) {
-        MachineInstr *SourceMI = MRI.getVRegDef(UseMI->getOperand(i).getReg());
-        if (!SourceMI || !SourceMI->isImplicitDef())
-          return false;
-      }
-      return true;
+  if (UseMI->isRegSequence()) {
+    for (unsigned i = 1, e = UseMI->getNumOperands(); i < e; i += 2) {
+      MachineInstr *SourceMI =
+          MRI.getUniqueVRegDef(UseMI->getOperand(i).getReg());
+      assert(SourceMI);
+      if (!SourceMI->isImplicitDef())
+        return false;
     }
+    return true;
   }
+
   return false;
 }
 
@@ -886,7 +889,7 @@ static VSETVLIInfo getInfoForVSETVLI(const MachineInstr &MI,
     if (AVLReg == RISCV::X0)
       NewInfo.setAVLVLMAX();
     else
-      NewInfo.setAVLRegDef(MRI.getVRegDef(AVLReg), AVLReg);
+      NewInfo.setAVLRegDef(MRI.getUniqueVRegDef(AVLReg), AVLReg);
   }
   NewInfo.setVTYPE(MI.getOperand(2).getImm());
 
@@ -958,7 +961,8 @@ static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags,
       else
         InstrInfo.setAVLImm(Imm);
     } else {
-      InstrInfo.setAVLRegDef(MRI->getVRegDef(VLOp.getReg()), VLOp.getReg());
+      InstrInfo.setAVLRegDef(MRI->getUniqueVRegDef(VLOp.getReg()),
+                             VLOp.getReg());
     }
   } else {
     assert(isScalarExtractInstr(MI));
@@ -1231,7 +1235,7 @@ void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
 
   if (RISCV::isFaultFirstLoad(MI)) {
     // Update AVL to vl-output of the fault first load.
-    Info.setAVLRegDef(MRI->getVRegDef(MI.getOperand(1).getReg()),
+    Info.setAVLRegDef(MRI->getUniqueVRegDef(MI.getOperand(1).getReg()),
                       MI.getOperand(1).getReg());
     return;
   }
@@ -1338,8 +1342,9 @@ bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,
     const VSETVLIInfo &PBBExit = BlockInfo[PBB->getNumber()].Exit;
 
     // We need the PHI input to the be the output of a VSET(I)VLI.
-    MachineInstr *DefMI = MRI->getVRegDef(InReg);
-    if (!DefMI || !isVectorConfigInstr(*DefMI))
+    MachineInstr *DefMI = MRI->getUniqueVRegDef(InReg);
+    assert(DefMI);
+    if (!isVectorConfigInstr(*DefMI))
       return true;
 
     // We found a VSET(I)VLI make sure it matches the output of the
@@ -1399,7 +1404,8 @@ void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {
         MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));
         if (VLOp.isReg()) {
           Register Reg = VLOp.getReg();
-          MachineInstr *VLOpDef = MRI->getVRegDef(Reg);
+          MachineInstr *VLOpDef = MRI->getUniqueVRegDef(Reg);
+          assert(VLOpDef);
 
           // Erase the AVL operand from the instruction.
           VLOp.setReg(RISCV::NoRegister);
@@ -1409,8 +1415,7 @@ void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {
           // as an ADDI. However, the ADDI might not have been used in the
           // vsetvli, or a vsetvli might not have been emitted, so it may be
           // dead now.
-          if (VLOpDef && TII->isAddImmediate(*VLOpDef, Reg) &&
-              MRI->use_nodbg_empty(Reg))
+          if (TII->isAddImmediate(*VLOpDef, Reg) && MRI->use_nodbg_empty(Reg))
             VLOpDef->eraseFromParent();
         }
         MI.addOperand(MachineOperand::CreateReg(RISCV::VL, /*isDef*/ false,
@@ -1682,6 +1687,7 @@ void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
     MachineInstr &MI = *I++;
     if (RISCV::isFaultFirstLoad(MI)) {
       Register VLOutput = MI.getOperand(1).getReg();
+      assert(VLOutput.isVirtual());
       if (!MRI->use_nodbg_empty(VLOutput))
         BuildMI(MBB, I, MI.getDebugLoc(), TII->get(RISCV::PseudoReadVL),
                 VLOutput);
-- 
GitLab


From 3e82442ff7288b4c41bb77888bc2cfea2c34d6ee Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Wed, 8 May 2024 00:57:39 -0400
Subject: [PATCH 0138/1206] [SPIRV] Add tan intrinsic part 3 (#90278)

This change is an implementation of #87367's investigation on supporting
IEEE math operations as intrinsics.
Which was discussed in this RFC:
https://discourse.llvm.org/t/rfc-all-the-math-intrinsics/78294

If you want an overarching view of how this will all connect see:
https://github.com/llvm/llvm-project/pull/90088
Changes:
- `llvm/docs/GlobalISel/GenericOpcode.rst` - Document the `G_FTAN`
opcode
-  `llvm/include/llvm/IR/Intrinsics.td` - Create the tan intrinsic
- `llvm/include/llvm/Support/TargetOpcodes.def` - Create a `G_FTAN`
Opcode handler
- `llvm/include/llvm/Target/GenericOpcodes.td` - Define the `G_FTAN`
Opcode
- `llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp` Map the tan intrinsic
to `G_FTAN` Opcode
- `llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp` - Map the
`G_FTAN` opcode to the GLSL 4.5 and openCL tan instructions.
- `llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp` - Define `G_FTAN` as a
legal spirv target opcode.
---
 llvm/docs/GlobalISel/GenericOpcode.rst        |  4 +-
 llvm/include/llvm/Support/TargetOpcodes.def   |  3 ++
 llvm/include/llvm/Target/GenericOpcodes.td    |  7 +++
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  2 +
 .../Target/SPIRV/SPIRVInstructionSelector.cpp |  2 +
 llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp  |  1 +
 .../GlobalISel/legalizer-info-validation.mir  |  3 ++
 .../test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll | 45 +++++++++++++++++++
 8 files changed, 65 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll

diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index 492d30280f47..52dc039df777 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -592,8 +592,8 @@ G_FLOG, G_FLOG2, G_FLOG10
 
 Calculate the base-e, base-2, or base-10 respectively.
 
-G_FCEIL, G_FCOS, G_FSIN, G_FSQRT, G_FFLOOR, G_FRINT, G_FNEARBYINT
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+G_FCEIL, G_FCOS, G_FSIN, G_FTAN, G_FSQRT, G_FFLOOR, G_FRINT, G_FNEARBYINT
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 These correspond to the standard C functions of the same name.
 
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index cb98f96af522..559a588c2514 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -781,6 +781,9 @@ HANDLE_TARGET_OPCODE(G_FCOS)
 /// Floating point sine.
 HANDLE_TARGET_OPCODE(G_FSIN)
 
+/// Floating point Tangent.
+HANDLE_TARGET_OPCODE(G_FTAN)
+
 /// Floating point square root.
 HANDLE_TARGET_OPCODE(G_FSQRT)
 
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 8380d2738d16..c40498e55421 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -988,6 +988,13 @@ def G_FSIN : GenericInstruction {
   let hasSideEffects = false;
 }
 
+// Floating point tangent of a value.
+def G_FTAN : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1);
+  let hasSideEffects = false;
+}
+
 // Floating point square root of a value.
 // This returns NaN for negative nonzero values.
 // NOTE: Unlike libm sqrt(), this never sets errno. In all other respects it's
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 77ee5e645288..6661127162e5 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1945,6 +1945,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
       return TargetOpcode::G_FSIN;
     case Intrinsic::sqrt:
       return TargetOpcode::G_FSQRT;
+    case Intrinsic::tan:
+      return TargetOpcode::G_FTAN;
     case Intrinsic::trunc:
       return TargetOpcode::G_INTRINSIC_TRUNC;
     case Intrinsic::readcyclecounter:
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 9994a966c82c..2051cdc7e01f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -467,6 +467,8 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
     return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos);
   case TargetOpcode::G_FSIN:
     return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin);
+  case TargetOpcode::G_FTAN:
+    return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan);
 
   case TargetOpcode::G_FSQRT:
     return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt);
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 4b871bdd5d07..e7b35555293a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -277,6 +277,7 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
                                G_FCEIL,
                                G_FCOS,
                                G_FSIN,
+                               G_FTAN,
                                G_FSQRT,
                                G_FFLOOR,
                                G_FRINT,
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 20133158e4fa..d71111b57efe 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -674,6 +674,9 @@
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: G_FTAN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FSQRT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll
new file mode 100644
index 000000000000..7bdce99dbfaa
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/tan.ll
@@ -0,0 +1,45 @@
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
+; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
+; CHECK-DAG: %[[#float_16:]] = OpTypeFloat 16
+; CHECK-DAG: %[[#vec4_float_32:]] = OpTypeVector %[[#float_32]] 4
+; CHECK-DAG: %[[#vec4_float_16:]] = OpTypeVector %[[#float_16]] 4
+
+define noundef float @tan_float(float noundef %a) {
+entry:
+; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+; CHECK: %[[#]] = OpExtInst %[[#float_32]] %[[#op_ext_glsl]] Tan %[[#arg0]]
+  %elt.tan = call float @llvm.tan.f32(float %a)
+  ret float %elt.tan
+}
+
+define noundef half @tan_half(half noundef %a) {
+entry:
+; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+; CHECK: %[[#]] = OpExtInst %[[#float_16]] %[[#op_ext_glsl]] Tan %[[#arg0]]
+  %elt.tan = call half @llvm.tan.f16(half %a)
+  ret half %elt.tan
+}
+
+define noundef <4 x float> @tan_float4(<4 x float> noundef %a) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpExtInst %[[#vec4_float_32]] %[[#op_ext_glsl]] Tan %[[#arg0]]
+  %elt.tan = call <4 x float> @llvm.tan.v4f32(<4 x float> %a)
+  ret <4 x float> %elt.tan
+}
+
+define noundef <4 x half> @tan_half4(<4 x half> noundef %a) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpExtInst %[[#vec4_float_16]] %[[#op_ext_glsl]] Tan %[[#arg0]]
+  %elt.tan = call <4 x half> @llvm.tan.v4f16(<4 x half> %a)
+  ret <4 x half> %elt.tan
+}
+
+declare half @llvm.tan.f16(half)
+declare float @llvm.tan.f32(float)
+declare <4 x half> @llvm.tan.v4f16(<4 x half>)
+declare <4 x float> @llvm.tan.v4f32(<4 x float>)
-- 
GitLab


From 084e2b53d22c11e013b0a495b65d39aa7f934048 Mon Sep 17 00:00:00 2001
From: Christian Ulmann <christianulmann@gmail.com>
Date: Wed, 8 May 2024 07:40:15 +0200
Subject: [PATCH 0139/1206] [MLIR][Interfaces] Change MemorySlotInterface to
 use OpBuilder (#91341)

This commit changes the `MemorySlotInterface` back to using `OpBuilder`
instead of a rewriter. This was originally introduced in
https://reviews.llvm.org/D150432 but it was shown that patterns are a
bad idea for both Mem2Reg and SROA.
Mem2Reg suffers from the usage of a rewriter due to being forced to
create new basic blocks. This is an issue, as it leads to the
invalidation of the dominance information, which can be expensive to
recompute.
---
 .../mlir/Interfaces/MemorySlotInterfaces.td   |  63 ++---
 mlir/include/mlir/Transforms/Mem2Reg.h        |   2 +-
 mlir/include/mlir/Transforms/SROA.h           |   2 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp | 248 +++++++++---------
 .../Dialect/MemRef/IR/MemRefMemorySlot.cpp    |  52 ++--
 mlir/lib/Transforms/Mem2Reg.cpp               |  90 +++----
 mlir/lib/Transforms/SROA.cpp                  |  26 +-
 7 files changed, 216 insertions(+), 267 deletions(-)

diff --git a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td
index 764fa6d547b2..adf182ac7069 100644
--- a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td
+++ b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td
@@ -40,42 +40,40 @@ def PromotableAllocationOpInterface
         Provides the default Value of this memory slot. The provided Value
         will be used as the reaching definition of loads done before any store.
         This Value must outlive the promotion and dominate all the uses of this
-        slot's pointer. The provided rewriter can be used to create the default
+        slot's pointer. The provided builder can be used to create the default
         value on the fly.
 
-        The rewriter is located at the beginning of the block where the slot
-        pointer is defined. All IR mutations must happen through the rewriter.
+        The builder is located at the beginning of the block where the slot
+        pointer is defined.
       }], "::mlir::Value", "getDefaultValue",
       (ins
         "const ::mlir::MemorySlot &":$slot,
-        "::mlir::RewriterBase &":$rewriter)
+        "::mlir::OpBuilder &":$builder)
     >,
     InterfaceMethod<[{
         Hook triggered for every new block argument added to a block.
         This will only be called for slots declared by this operation.
 
-        The rewriter is located at the beginning of the block on call. All IR
-        mutations must happen through the rewriter.
+        The builder is located at the beginning of the block on call. All IR
+        mutations must happen through the builder.
       }],
       "void", "handleBlockArgument",
       (ins
         "const ::mlir::MemorySlot &":$slot,
         "::mlir::BlockArgument":$argument,
-        "::mlir::RewriterBase &":$rewriter
+        "::mlir::OpBuilder &":$builder
       )
     >,
     InterfaceMethod<[{
         Hook triggered once the promotion of a slot is complete. This can
         also clean up the created default value if necessary.
         This will only be called for slots declared by this operation.
-
-        All IR mutations must happen through the rewriter.
       }],
       "void", "handlePromotionComplete",
       (ins
         "const ::mlir::MemorySlot &":$slot, 
         "::mlir::Value":$defaultValue,
-        "::mlir::RewriterBase &":$rewriter)
+        "::mlir::OpBuilder &":$builder)
     >,
   ];
 }
@@ -119,15 +117,14 @@ def PromotableMemOpInterface : OpInterface<"PromotableMemOpInterface"> {
         The returned value must dominate all operations dominated by the storing
         operation.
 
-        If IR must be mutated to extract a concrete value being stored, mutation
-        must happen through the provided rewriter. The rewriter is located
-        immediately after the memory operation on call. No IR deletion is
-        allowed in this method. IR mutations must not introduce new uses of the
-        memory slot. Existing control flow must not be modified.
+        The builder is located immediately after the memory operation on call.
+        No IR deletion is allowed in this method. IR mutations must not
+        introduce new uses of the memory slot. Existing control flow must not
+        be modified.
       }],
       "::mlir::Value", "getStored",
       (ins "const ::mlir::MemorySlot &":$slot,
-           "::mlir::RewriterBase &":$rewriter,
+           "::mlir::OpBuilder &":$builder,
            "::mlir::Value":$reachingDef,
            "const ::mlir::DataLayout &":$dataLayout)
     >,
@@ -166,14 +163,13 @@ def PromotableMemOpInterface : OpInterface<"PromotableMemOpInterface"> {
         have been done at the point of calling this method, but it will be done
         eventually.
 
-        The rewriter is located after the promotable operation on call. All IR
-        mutations must happen through the rewriter.
+        The builder is located after the promotable operation on call.
       }],
       "::mlir::DeletionKind",
       "removeBlockingUses",
       (ins "const ::mlir::MemorySlot &":$slot,
            "const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> &":$blockingUses,
-           "::mlir::RewriterBase &":$rewriter,
+           "::mlir::OpBuilder &":$builder,
            "::mlir::Value":$reachingDefinition,
            "const ::mlir::DataLayout &":$dataLayout)
     >,
@@ -224,13 +220,12 @@ def PromotableOpInterface : OpInterface<"PromotableOpInterface"> {
         have been done at the point of calling this method, but it will be done
         eventually.
 
-        The rewriter is located after the promotable operation on call. All IR
-        mutations must happen through the rewriter.
+        The builder is located after the promotable operation on call.
       }],
       "::mlir::DeletionKind",
       "removeBlockingUses",
       (ins "const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> &":$blockingUses,
-           "::mlir::RewriterBase &":$rewriter)
+           "::mlir::OpBuilder &":$builder)
     >,
     InterfaceMethod<[{
         This method allows the promoted operation to visit the SSA values used
@@ -254,13 +249,12 @@ def PromotableOpInterface : OpInterface<"PromotableOpInterface"> {
         scheduled for removal and if `requiresReplacedValues` returned
         true.
 
-        The rewriter is located after the promotable operation on call. All IR
-        mutations must happen through the rewriter. During the transformation,
-        *no operation should be deleted*.
+        The builder is located after the promotable operation on call. During
+        the transformation, *no operation should be deleted*.
       }],
       "void", "visitReplacedValues",
       (ins "::llvm::ArrayRef<std::pair<::mlir::Operation*, ::mlir::Value>>":$mutatedDefs,
-           "::mlir::RewriterBase &":$rewriter), [{}], [{ return; }]
+           "::mlir::OpBuilder &":$builder), [{}], [{ return; }]
     >,
   ];
 }
@@ -293,25 +287,23 @@ def DestructurableAllocationOpInterface
         at the end of this call. Only generates subslots for the indices found in
         `usedIndices` since all other subslots are unused.
 
-        The rewriter is located at the beginning of the block where the slot
-        pointer is defined. All IR mutations must happen through the rewriter.
+        The builder is located at the beginning of the block where the slot
+        pointer is defined.
       }],
       "::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot>",
       "destructure",
       (ins "const ::mlir::DestructurableMemorySlot &":$slot,
            "const ::llvm::SmallPtrSetImpl<::mlir::Attribute> &":$usedIndices,
-           "::mlir::RewriterBase &":$rewriter)
+           "::mlir::OpBuilder &":$builder)
     >,
     InterfaceMethod<[{
         Hook triggered once the destructuring of a slot is complete, meaning the
         original slot is no longer being refered to and could be deleted.
         This will only be called for slots declared by this operation.
-
-        All IR mutations must happen through the rewriter.
       }],
       "void", "handleDestructuringComplete",
       (ins "const ::mlir::DestructurableMemorySlot &":$slot,
-           "::mlir::RewriterBase &":$rewriter)
+           "::mlir::OpBuilder &":$builder)
     >,
   ];
 }
@@ -376,15 +368,14 @@ def DestructurableAccessorOpInterface
         Rewires the use of a slot to the generated subslots, without deleting
         any operation. Returns whether the accessor should be deleted.
 
-        All IR mutations must happen through the rewriter. Deletion of
-        operations is not allowed, only the accessor can be scheduled for
-        deletion by returning the appropriate value.
+        Deletion of operations is not allowed, only the accessor can be
+        scheduled for deletion by returning the appropriate value.
       }],
       "::mlir::DeletionKind",
       "rewire",
       (ins "const ::mlir::DestructurableMemorySlot &":$slot,
            "::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> &":$subslots,
-           "::mlir::RewriterBase &":$rewriter,
+           "::mlir::OpBuilder &":$builder,
            "const ::mlir::DataLayout &":$dataLayout)
     >
   ];
diff --git a/mlir/include/mlir/Transforms/Mem2Reg.h b/mlir/include/mlir/Transforms/Mem2Reg.h
index ed10644e26a5..b4f939d65414 100644
--- a/mlir/include/mlir/Transforms/Mem2Reg.h
+++ b/mlir/include/mlir/Transforms/Mem2Reg.h
@@ -27,7 +27,7 @@ struct Mem2RegStatistics {
 /// at least one memory slot was promoted.
 LogicalResult
 tryToPromoteMemorySlots(ArrayRef<PromotableAllocationOpInterface> allocators,
-                        RewriterBase &rewriter, const DataLayout &dataLayout,
+                        OpBuilder &builder, const DataLayout &dataLayout,
                         Mem2RegStatistics statistics = {});
 
 } // namespace mlir
diff --git a/mlir/include/mlir/Transforms/SROA.h b/mlir/include/mlir/Transforms/SROA.h
index d09a7989edea..fa84fb1eae73 100644
--- a/mlir/include/mlir/Transforms/SROA.h
+++ b/mlir/include/mlir/Transforms/SROA.h
@@ -31,7 +31,7 @@ struct SROAStatistics {
 /// failure if no slot was destructured.
 LogicalResult tryToDestructureMemorySlots(
     ArrayRef<DestructurableAllocationOpInterface> allocators,
-    RewriterBase &rewriter, const DataLayout &dataLayout,
+    OpBuilder &builder, const DataLayout &dataLayout,
     SROAStatistics statistics = {});
 
 } // namespace mlir
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index 6eeb13ebffb5..70102e1c8192 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -36,26 +36,26 @@ llvm::SmallVector<MemorySlot> LLVM::AllocaOp::getPromotableSlots() {
 }
 
 Value LLVM::AllocaOp::getDefaultValue(const MemorySlot &slot,
-                                      RewriterBase &rewriter) {
-  return rewriter.create<LLVM::UndefOp>(getLoc(), slot.elemType);
+                                      OpBuilder &builder) {
+  return builder.create<LLVM::UndefOp>(getLoc(), slot.elemType);
 }
 
 void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
                                          BlockArgument argument,
-                                         RewriterBase &rewriter) {
+                                         OpBuilder &builder) {
   for (Operation *user : getOperation()->getUsers())
     if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
-      rewriter.create<LLVM::DbgValueOp>(declareOp.getLoc(), argument,
-                                        declareOp.getVarInfo(),
-                                        declareOp.getLocationExpr());
+      builder.create<LLVM::DbgValueOp>(declareOp.getLoc(), argument,
+                                       declareOp.getVarInfo(),
+                                       declareOp.getLocationExpr());
 }
 
 void LLVM::AllocaOp::handlePromotionComplete(const MemorySlot &slot,
                                              Value defaultValue,
-                                             RewriterBase &rewriter) {
+                                             OpBuilder &builder) {
   if (defaultValue && defaultValue.use_empty())
-    rewriter.eraseOp(defaultValue.getDefiningOp());
-  rewriter.eraseOp(*this);
+    defaultValue.getDefiningOp()->erase();
+  this->erase();
 }
 
 SmallVector<DestructurableMemorySlot> LLVM::AllocaOp::getDestructurableSlots() {
@@ -78,16 +78,16 @@ SmallVector<DestructurableMemorySlot> LLVM::AllocaOp::getDestructurableSlots() {
 DenseMap<Attribute, MemorySlot>
 LLVM::AllocaOp::destructure(const DestructurableMemorySlot &slot,
                             const SmallPtrSetImpl<Attribute> &usedIndices,
-                            RewriterBase &rewriter) {
+                            OpBuilder &builder) {
   assert(slot.ptr == getResult());
-  rewriter.setInsertionPointAfter(*this);
+  builder.setInsertionPointAfter(*this);
 
   auto destructurableType = cast<DestructurableTypeInterface>(getElemType());
   DenseMap<Attribute, MemorySlot> slotMap;
   for (Attribute index : usedIndices) {
     Type elemType = destructurableType.getTypeAtIndex(index);
     assert(elemType && "used index must exist");
-    auto subAlloca = rewriter.create<LLVM::AllocaOp>(
+    auto subAlloca = builder.create<LLVM::AllocaOp>(
         getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType,
         getArraySize());
     slotMap.try_emplace<MemorySlot>(index, {subAlloca.getResult(), elemType});
@@ -97,9 +97,9 @@ LLVM::AllocaOp::destructure(const DestructurableMemorySlot &slot,
 }
 
 void LLVM::AllocaOp::handleDestructuringComplete(
-    const DestructurableMemorySlot &slot, RewriterBase &rewriter) {
+    const DestructurableMemorySlot &slot, OpBuilder &builder) {
   assert(slot.ptr == getResult());
-  rewriter.eraseOp(*this);
+  this->erase();
 }
 
 //===----------------------------------------------------------------------===//
@@ -112,7 +112,7 @@ bool LLVM::LoadOp::loadsFrom(const MemorySlot &slot) {
 
 bool LLVM::LoadOp::storesTo(const MemorySlot &slot) { return false; }
 
-Value LLVM::LoadOp::getStored(const MemorySlot &slot, RewriterBase &rewriter,
+Value LLVM::LoadOp::getStored(const MemorySlot &slot, OpBuilder &builder,
                               Value reachingDef, const DataLayout &dataLayout) {
   llvm_unreachable("getStored should not be called on LoadOp");
 }
@@ -175,7 +175,7 @@ static bool isBigEndian(const DataLayout &dataLayout) {
 
 /// Converts a value to an integer type of the same size.
 /// Assumes that the type can be converted.
-static Value castToSameSizedInt(RewriterBase &rewriter, Location loc, Value val,
+static Value castToSameSizedInt(OpBuilder &builder, Location loc, Value val,
                                 const DataLayout &dataLayout) {
   Type type = val.getType();
   assert(isSupportedTypeForConversion(type) &&
@@ -185,15 +185,15 @@ static Value castToSameSizedInt(RewriterBase &rewriter, Location loc, Value val,
     return val;
 
   uint64_t typeBitSize = dataLayout.getTypeSizeInBits(type);
-  IntegerType valueSizeInteger = rewriter.getIntegerType(typeBitSize);
+  IntegerType valueSizeInteger = builder.getIntegerType(typeBitSize);
 
   if (isa<LLVM::LLVMPointerType>(type))
-    return rewriter.createOrFold<LLVM::PtrToIntOp>(loc, valueSizeInteger, val);
-  return rewriter.createOrFold<LLVM::BitcastOp>(loc, valueSizeInteger, val);
+    return builder.createOrFold<LLVM::PtrToIntOp>(loc, valueSizeInteger, val);
+  return builder.createOrFold<LLVM::BitcastOp>(loc, valueSizeInteger, val);
 }
 
 /// Converts a value with an integer type to `targetType`.
-static Value castIntValueToSameSizedType(RewriterBase &rewriter, Location loc,
+static Value castIntValueToSameSizedType(OpBuilder &builder, Location loc,
                                          Value val, Type targetType) {
   assert(isa<IntegerType>(val.getType()) &&
          "expected value to have an integer type");
@@ -202,13 +202,13 @@ static Value castIntValueToSameSizedType(RewriterBase &rewriter, Location loc,
   if (val.getType() == targetType)
     return val;
   if (isa<LLVM::LLVMPointerType>(targetType))
-    return rewriter.createOrFold<LLVM::IntToPtrOp>(loc, targetType, val);
-  return rewriter.createOrFold<LLVM::BitcastOp>(loc, targetType, val);
+    return builder.createOrFold<LLVM::IntToPtrOp>(loc, targetType, val);
+  return builder.createOrFold<LLVM::BitcastOp>(loc, targetType, val);
 }
 
 /// Constructs operations that convert `srcValue` into a new value of type
 /// `targetType`. Assumes the types have the same bitsize.
-static Value castSameSizedTypes(RewriterBase &rewriter, Location loc,
+static Value castSameSizedTypes(OpBuilder &builder, Location loc,
                                 Value srcValue, Type targetType,
                                 const DataLayout &dataLayout) {
   Type srcType = srcValue.getType();
@@ -226,18 +226,18 @@ static Value castSameSizedTypes(RewriterBase &rewriter, Location loc,
   // provenance.
   if (isa<LLVM::LLVMPointerType>(targetType) &&
       isa<LLVM::LLVMPointerType>(srcType))
-    return rewriter.createOrFold<LLVM::AddrSpaceCastOp>(loc, targetType,
-                                                        srcValue);
+    return builder.createOrFold<LLVM::AddrSpaceCastOp>(loc, targetType,
+                                                       srcValue);
 
   // For all other castable types, casting through integers is necessary.
-  Value replacement = castToSameSizedInt(rewriter, loc, srcValue, dataLayout);
-  return castIntValueToSameSizedType(rewriter, loc, replacement, targetType);
+  Value replacement = castToSameSizedInt(builder, loc, srcValue, dataLayout);
+  return castIntValueToSameSizedType(builder, loc, replacement, targetType);
 }
 
 /// Constructs operations that convert `srcValue` into a new value of type
 /// `targetType`. Performs bit-level extraction if the source type is larger
 /// than the target type. Assumes that this conversion is possible.
-static Value createExtractAndCast(RewriterBase &rewriter, Location loc,
+static Value createExtractAndCast(OpBuilder &builder, Location loc,
                                   Value srcValue, Type targetType,
                                   const DataLayout &dataLayout) {
   // Get the types of the source and target values.
@@ -249,31 +249,31 @@ static Value createExtractAndCast(RewriterBase &rewriter, Location loc,
   uint64_t srcTypeSize = dataLayout.getTypeSizeInBits(srcType);
   uint64_t targetTypeSize = dataLayout.getTypeSizeInBits(targetType);
   if (srcTypeSize == targetTypeSize)
-    return castSameSizedTypes(rewriter, loc, srcValue, targetType, dataLayout);
+    return castSameSizedTypes(builder, loc, srcValue, targetType, dataLayout);
 
   // First, cast the value to a same-sized integer type.
-  Value replacement = castToSameSizedInt(rewriter, loc, srcValue, dataLayout);
+  Value replacement = castToSameSizedInt(builder, loc, srcValue, dataLayout);
 
   // Truncate the integer if the size of the target is less than the value.
   if (isBigEndian(dataLayout)) {
     uint64_t shiftAmount = srcTypeSize - targetTypeSize;
-    auto shiftConstant = rewriter.create<LLVM::ConstantOp>(
-        loc, rewriter.getIntegerAttr(srcType, shiftAmount));
+    auto shiftConstant = builder.create<LLVM::ConstantOp>(
+        loc, builder.getIntegerAttr(srcType, shiftAmount));
     replacement =
-        rewriter.createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
+        builder.createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
   }
 
-  replacement = rewriter.create<LLVM::TruncOp>(
-      loc, rewriter.getIntegerType(targetTypeSize), replacement);
+  replacement = builder.create<LLVM::TruncOp>(
+      loc, builder.getIntegerType(targetTypeSize), replacement);
 
   // Now cast the integer to the actual target type if required.
-  return castIntValueToSameSizedType(rewriter, loc, replacement, targetType);
+  return castIntValueToSameSizedType(builder, loc, replacement, targetType);
 }
 
 /// Constructs operations that insert the bits of `srcValue` into the
 /// "beginning" of `reachingDef` (beginning is endianness dependent).
 /// Assumes that this conversion is possible.
-static Value createInsertAndCast(RewriterBase &rewriter, Location loc,
+static Value createInsertAndCast(OpBuilder &builder, Location loc,
                                  Value srcValue, Value reachingDef,
                                  const DataLayout &dataLayout) {
 
@@ -284,27 +284,27 @@ static Value createInsertAndCast(RewriterBase &rewriter, Location loc,
   uint64_t valueTypeSize = dataLayout.getTypeSizeInBits(srcValue.getType());
   uint64_t slotTypeSize = dataLayout.getTypeSizeInBits(reachingDef.getType());
   if (slotTypeSize == valueTypeSize)
-    return castSameSizedTypes(rewriter, loc, srcValue, reachingDef.getType(),
+    return castSameSizedTypes(builder, loc, srcValue, reachingDef.getType(),
                               dataLayout);
 
   // In the case where the store only overwrites parts of the memory,
   // bit fiddling is required to construct the new value.
 
   // First convert both values to integers of the same size.
-  Value defAsInt = castToSameSizedInt(rewriter, loc, reachingDef, dataLayout);
-  Value valueAsInt = castToSameSizedInt(rewriter, loc, srcValue, dataLayout);
+  Value defAsInt = castToSameSizedInt(builder, loc, reachingDef, dataLayout);
+  Value valueAsInt = castToSameSizedInt(builder, loc, srcValue, dataLayout);
   // Extend the value to the size of the reaching definition.
   valueAsInt =
-      rewriter.createOrFold<LLVM::ZExtOp>(loc, defAsInt.getType(), valueAsInt);
+      builder.createOrFold<LLVM::ZExtOp>(loc, defAsInt.getType(), valueAsInt);
   uint64_t sizeDifference = slotTypeSize - valueTypeSize;
   if (isBigEndian(dataLayout)) {
     // On big endian systems, a store to the base pointer overwrites the most
     // significant bits. To accomodate for this, the stored value needs to be
     // shifted into the according position.
-    Value bigEndianShift = rewriter.create<LLVM::ConstantOp>(
-        loc, rewriter.getIntegerAttr(defAsInt.getType(), sizeDifference));
+    Value bigEndianShift = builder.create<LLVM::ConstantOp>(
+        loc, builder.getIntegerAttr(defAsInt.getType(), sizeDifference));
     valueAsInt =
-        rewriter.createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
+        builder.createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
   }
 
   // Construct the mask that is used to erase the bits that are overwritten by
@@ -322,23 +322,23 @@ static Value createInsertAndCast(RewriterBase &rewriter, Location loc,
   }
 
   // Mask out the affected bits ...
-  Value mask = rewriter.create<LLVM::ConstantOp>(
-      loc, rewriter.getIntegerAttr(defAsInt.getType(), maskValue));
-  Value masked = rewriter.createOrFold<LLVM::AndOp>(loc, defAsInt, mask);
+  Value mask = builder.create<LLVM::ConstantOp>(
+      loc, builder.getIntegerAttr(defAsInt.getType(), maskValue));
+  Value masked = builder.createOrFold<LLVM::AndOp>(loc, defAsInt, mask);
 
   // ... and combine the result with the new value.
-  Value combined = rewriter.createOrFold<LLVM::OrOp>(loc, masked, valueAsInt);
+  Value combined = builder.createOrFold<LLVM::OrOp>(loc, masked, valueAsInt);
 
-  return castIntValueToSameSizedType(rewriter, loc, combined,
+  return castIntValueToSameSizedType(builder, loc, combined,
                                      reachingDef.getType());
 }
 
-Value LLVM::StoreOp::getStored(const MemorySlot &slot, RewriterBase &rewriter,
+Value LLVM::StoreOp::getStored(const MemorySlot &slot, OpBuilder &builder,
                                Value reachingDef,
                                const DataLayout &dataLayout) {
   assert(reachingDef && reachingDef.getType() == slot.elemType &&
          "expected the reaching definition's type to match the slot's type");
-  return createInsertAndCast(rewriter, getLoc(), getValue(), reachingDef,
+  return createInsertAndCast(builder, getLoc(), getValue(), reachingDef,
                              dataLayout);
 }
 
@@ -360,13 +360,13 @@ bool LLVM::LoadOp::canUsesBeRemoved(
 
 DeletionKind LLVM::LoadOp::removeBlockingUses(
     const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
-    RewriterBase &rewriter, Value reachingDefinition,
+    OpBuilder &builder, Value reachingDefinition,
     const DataLayout &dataLayout) {
   // `canUsesBeRemoved` checked this blocking use must be the loaded slot
   // pointer.
-  Value newResult = createExtractAndCast(rewriter, getLoc(), reachingDefinition,
+  Value newResult = createExtractAndCast(builder, getLoc(), reachingDefinition,
                                          getResult().getType(), dataLayout);
-  rewriter.replaceAllUsesWith(getResult(), newResult);
+  getResult().replaceAllUsesWith(newResult);
   return DeletionKind::Delete;
 }
 
@@ -390,7 +390,7 @@ bool LLVM::StoreOp::canUsesBeRemoved(
 
 DeletionKind LLVM::StoreOp::removeBlockingUses(
     const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
-    RewriterBase &rewriter, Value reachingDefinition,
+    OpBuilder &builder, Value reachingDefinition,
     const DataLayout &dataLayout) {
   return DeletionKind::Delete;
 }
@@ -452,14 +452,13 @@ bool LLVM::LoadOp::canRewire(const DestructurableMemorySlot &slot,
 
 DeletionKind LLVM::LoadOp::rewire(const DestructurableMemorySlot &slot,
                                   DenseMap<Attribute, MemorySlot> &subslots,
-                                  RewriterBase &rewriter,
+                                  OpBuilder &builder,
                                   const DataLayout &dataLayout) {
   auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0);
   auto it = subslots.find(index);
   assert(it != subslots.end());
 
-  rewriter.modifyOpInPlace(
-      *this, [&]() { getAddrMutable().set(it->getSecond().ptr); });
+  getAddrMutable().set(it->getSecond().ptr);
   return DeletionKind::Keep;
 }
 
@@ -491,14 +490,13 @@ bool LLVM::StoreOp::canRewire(const DestructurableMemorySlot &slot,
 
 DeletionKind LLVM::StoreOp::rewire(const DestructurableMemorySlot &slot,
                                    DenseMap<Attribute, MemorySlot> &subslots,
-                                   RewriterBase &rewriter,
+                                   OpBuilder &builder,
                                    const DataLayout &dataLayout) {
   auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0);
   auto it = subslots.find(index);
   assert(it != subslots.end());
 
-  rewriter.modifyOpInPlace(
-      *this, [&]() { getAddrMutable().set(it->getSecond().ptr); });
+  getAddrMutable().set(it->getSecond().ptr);
   return DeletionKind::Keep;
 }
 
@@ -523,7 +521,7 @@ bool LLVM::BitcastOp::canUsesBeRemoved(
 }
 
 DeletionKind LLVM::BitcastOp::removeBlockingUses(
-    const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+    const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
   return DeletionKind::Delete;
 }
 
@@ -535,7 +533,7 @@ bool LLVM::AddrSpaceCastOp::canUsesBeRemoved(
 }
 
 DeletionKind LLVM::AddrSpaceCastOp::removeBlockingUses(
-    const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+    const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
   return DeletionKind::Delete;
 }
 
@@ -547,7 +545,7 @@ bool LLVM::LifetimeStartOp::canUsesBeRemoved(
 }
 
 DeletionKind LLVM::LifetimeStartOp::removeBlockingUses(
-    const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+    const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
   return DeletionKind::Delete;
 }
 
@@ -559,7 +557,7 @@ bool LLVM::LifetimeEndOp::canUsesBeRemoved(
 }
 
 DeletionKind LLVM::LifetimeEndOp::removeBlockingUses(
-    const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+    const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
   return DeletionKind::Delete;
 }
 
@@ -571,7 +569,7 @@ bool LLVM::InvariantStartOp::canUsesBeRemoved(
 }
 
 DeletionKind LLVM::InvariantStartOp::removeBlockingUses(
-    const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+    const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
   return DeletionKind::Delete;
 }
 
@@ -583,7 +581,7 @@ bool LLVM::InvariantEndOp::canUsesBeRemoved(
 }
 
 DeletionKind LLVM::InvariantEndOp::removeBlockingUses(
-    const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+    const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
   return DeletionKind::Delete;
 }
 
@@ -595,7 +593,7 @@ bool LLVM::DbgDeclareOp::canUsesBeRemoved(
 }
 
 DeletionKind LLVM::DbgDeclareOp::removeBlockingUses(
-    const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+    const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
   return DeletionKind::Delete;
 }
 
@@ -611,28 +609,27 @@ bool LLVM::DbgValueOp::canUsesBeRemoved(
 }
 
 DeletionKind LLVM::DbgValueOp::removeBlockingUses(
-    const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
-  // Rewriter by default is after '*this', but we need it before '*this'.
-  rewriter.setInsertionPoint(*this);
+    const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
+  // builder by default is after '*this', but we need it before '*this'.
+  builder.setInsertionPoint(*this);
 
   // Rather than dropping the debug value, replace it with undef to preserve the
   // debug local variable info. This allows the debugger to inform the user that
   // the variable has been optimized out.
   auto undef =
-      rewriter.create<UndefOp>(getValue().getLoc(), getValue().getType());
-  rewriter.modifyOpInPlace(*this, [&] { getValueMutable().assign(undef); });
+      builder.create<UndefOp>(getValue().getLoc(), getValue().getType());
+  getValueMutable().assign(undef);
   return DeletionKind::Keep;
 }
 
 bool LLVM::DbgDeclareOp::requiresReplacedValues() { return true; }
 
 void LLVM::DbgDeclareOp::visitReplacedValues(
-    ArrayRef<std::pair<Operation *, Value>> definitions,
-    RewriterBase &rewriter) {
+    ArrayRef<std::pair<Operation *, Value>> definitions, OpBuilder &builder) {
   for (auto [op, value] : definitions) {
-    rewriter.setInsertionPointAfter(op);
-    rewriter.create<LLVM::DbgValueOp>(getLoc(), value, getVarInfo(),
-                                      getLocationExpr());
+    builder.setInsertionPointAfter(op);
+    builder.create<LLVM::DbgValueOp>(getLoc(), value, getVarInfo(),
+                                     getLocationExpr());
   }
 }
 
@@ -658,7 +655,7 @@ bool LLVM::GEPOp::canUsesBeRemoved(
 }
 
 DeletionKind LLVM::GEPOp::removeBlockingUses(
-    const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+    const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
   return DeletionKind::Delete;
 }
 
@@ -855,7 +852,7 @@ bool LLVM::GEPOp::canRewire(const DestructurableMemorySlot &slot,
 
 DeletionKind LLVM::GEPOp::rewire(const DestructurableMemorySlot &slot,
                                  DenseMap<Attribute, MemorySlot> &subslots,
-                                 RewriterBase &rewriter,
+                                 OpBuilder &builder,
                                  const DataLayout &dataLayout) {
   std::optional<SubslotAccessInfo> accessInfo =
       getSubslotAccessInfo(slot, dataLayout, *this);
@@ -864,11 +861,11 @@ DeletionKind LLVM::GEPOp::rewire(const DestructurableMemorySlot &slot,
       IntegerAttr::get(IntegerType::get(getContext(), 32), accessInfo->index);
   const MemorySlot &newSlot = subslots.at(indexAttr);
 
-  auto byteType = IntegerType::get(rewriter.getContext(), 8);
-  auto newPtr = rewriter.createOrFold<LLVM::GEPOp>(
+  auto byteType = IntegerType::get(builder.getContext(), 8);
+  auto newPtr = builder.createOrFold<LLVM::GEPOp>(
       getLoc(), getResult().getType(), byteType, newSlot.ptr,
       ArrayRef<GEPArg>(accessInfo->subslotOffset), getInbounds());
-  rewriter.replaceAllUsesWith(getResult(), newPtr);
+  getResult().replaceAllUsesWith(newPtr);
   return DeletionKind::Delete;
 }
 
@@ -938,7 +935,7 @@ bool LLVM::MemsetOp::storesTo(const MemorySlot &slot) {
   return getDst() == slot.ptr;
 }
 
-Value LLVM::MemsetOp::getStored(const MemorySlot &slot, RewriterBase &rewriter,
+Value LLVM::MemsetOp::getStored(const MemorySlot &slot, OpBuilder &builder,
                                 Value reachingDef,
                                 const DataLayout &dataLayout) {
   // TODO: Support non-integer types.
@@ -953,14 +950,14 @@ Value LLVM::MemsetOp::getStored(const MemorySlot &slot, RewriterBase &rewriter,
         // or-ing it with the previous value.
         uint64_t coveredBits = 8;
         Value currentValue =
-            rewriter.create<LLVM::ZExtOp>(getLoc(), intType, getVal());
+            builder.create<LLVM::ZExtOp>(getLoc(), intType, getVal());
         while (coveredBits < intType.getWidth()) {
           Value shiftBy =
-              rewriter.create<LLVM::ConstantOp>(getLoc(), intType, coveredBits);
+              builder.create<LLVM::ConstantOp>(getLoc(), intType, coveredBits);
           Value shifted =
-              rewriter.create<LLVM::ShlOp>(getLoc(), currentValue, shiftBy);
+              builder.create<LLVM::ShlOp>(getLoc(), currentValue, shiftBy);
           currentValue =
-              rewriter.create<LLVM::OrOp>(getLoc(), currentValue, shifted);
+              builder.create<LLVM::OrOp>(getLoc(), currentValue, shifted);
           coveredBits *= 2;
         }
 
@@ -994,7 +991,7 @@ bool LLVM::MemsetOp::canUsesBeRemoved(
 
 DeletionKind LLVM::MemsetOp::removeBlockingUses(
     const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
-    RewriterBase &rewriter, Value reachingDefinition,
+    OpBuilder &builder, Value reachingDefinition,
     const DataLayout &dataLayout) {
   return DeletionKind::Delete;
 }
@@ -1026,7 +1023,7 @@ bool LLVM::MemsetOp::canRewire(const DestructurableMemorySlot &slot,
 
 DeletionKind LLVM::MemsetOp::rewire(const DestructurableMemorySlot &slot,
                                     DenseMap<Attribute, MemorySlot> &subslots,
-                                    RewriterBase &rewriter,
+                                    OpBuilder &builder,
                                     const DataLayout &dataLayout) {
   std::optional<DenseMap<Attribute, Type>> types =
       cast<DestructurableTypeInterface>(slot.elemType).getSubelementIndexMap();
@@ -1063,15 +1060,14 @@ DeletionKind LLVM::MemsetOp::rewire(const DestructurableMemorySlot &slot,
       uint64_t newMemsetSize = std::min(memsetLen - covered, typeSize);
 
       Value newMemsetSizeValue =
-          rewriter
+          builder
               .create<LLVM::ConstantOp>(
                   getLen().getLoc(),
                   IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize))
               .getResult();
 
-      rewriter.create<LLVM::MemsetOp>(getLoc(), subslots.at(index).ptr,
-                                      getVal(), newMemsetSizeValue,
-                                      getIsVolatile());
+      builder.create<LLVM::MemsetOp>(getLoc(), subslots.at(index).ptr, getVal(),
+                                     newMemsetSizeValue, getIsVolatile());
     }
 
     covered += typeSize;
@@ -1096,8 +1092,8 @@ static bool memcpyStoresTo(MemcpyLike op, const MemorySlot &slot) {
 
 template <class MemcpyLike>
 static Value memcpyGetStored(MemcpyLike op, const MemorySlot &slot,
-                             RewriterBase &rewriter) {
-  return rewriter.create<LLVM::LoadOp>(op.getLoc(), slot.elemType, op.getSrc());
+                             OpBuilder &builder) {
+  return builder.create<LLVM::LoadOp>(op.getLoc(), slot.elemType, op.getSrc());
 }
 
 template <class MemcpyLike>
@@ -1122,10 +1118,9 @@ template <class MemcpyLike>
 static DeletionKind
 memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot,
                          const SmallPtrSetImpl<OpOperand *> &blockingUses,
-                         RewriterBase &rewriter, Value reachingDefinition) {
+                         OpBuilder &builder, Value reachingDefinition) {
   if (op.loadsFrom(slot))
-    rewriter.create<LLVM::StoreOp>(op.getLoc(), reachingDefinition,
-                                   op.getDst());
+    builder.create<LLVM::StoreOp>(op.getLoc(), reachingDefinition, op.getDst());
   return DeletionKind::Delete;
 }
 
@@ -1168,23 +1163,23 @@ static bool memcpyCanRewire(MemcpyLike op, const DestructurableMemorySlot &slot,
 namespace {
 
 template <class MemcpyLike>
-void createMemcpyLikeToReplace(RewriterBase &rewriter, const DataLayout &layout,
+void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                MemcpyLike toReplace, Value dst, Value src,
                                Type toCpy, bool isVolatile) {
-  Value memcpySize = rewriter.create<LLVM::ConstantOp>(
+  Value memcpySize = builder.create<LLVM::ConstantOp>(
       toReplace.getLoc(), IntegerAttr::get(toReplace.getLen().getType(),
                                            layout.getTypeSize(toCpy)));
-  rewriter.create<MemcpyLike>(toReplace.getLoc(), dst, src, memcpySize,
-                              isVolatile);
+  builder.create<MemcpyLike>(toReplace.getLoc(), dst, src, memcpySize,
+                             isVolatile);
 }
 
 template <>
-void createMemcpyLikeToReplace(RewriterBase &rewriter, const DataLayout &layout,
+void createMemcpyLikeToReplace(OpBuilder &builder, const DataLayout &layout,
                                LLVM::MemcpyInlineOp toReplace, Value dst,
                                Value src, Type toCpy, bool isVolatile) {
   Type lenType = IntegerType::get(toReplace->getContext(),
                                   toReplace.getLen().getBitWidth());
-  rewriter.create<LLVM::MemcpyInlineOp>(
+  builder.create<LLVM::MemcpyInlineOp>(
       toReplace.getLoc(), dst, src,
       IntegerAttr::get(lenType, layout.getTypeSize(toCpy)), isVolatile);
 }
@@ -1196,7 +1191,7 @@ void createMemcpyLikeToReplace(RewriterBase &rewriter, const DataLayout &layout,
 template <class MemcpyLike>
 static DeletionKind
 memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot,
-             DenseMap<Attribute, MemorySlot> &subslots, RewriterBase &rewriter,
+             DenseMap<Attribute, MemorySlot> &subslots, OpBuilder &builder,
              const DataLayout &dataLayout) {
   if (subslots.empty())
     return DeletionKind::Delete;
@@ -1226,12 +1221,12 @@ memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot,
     SmallVector<LLVM::GEPArg> gepIndices{
         0, static_cast<int32_t>(
                cast<IntegerAttr>(index).getValue().getZExtValue())};
-    Value subslotPtrInOther = rewriter.create<LLVM::GEPOp>(
+    Value subslotPtrInOther = builder.create<LLVM::GEPOp>(
         op.getLoc(), LLVM::LLVMPointerType::get(op.getContext()), slot.elemType,
         isDst ? op.getSrc() : op.getDst(), gepIndices);
 
     // Then create a new memcpy out of this source pointer.
-    createMemcpyLikeToReplace(rewriter, dataLayout, op,
+    createMemcpyLikeToReplace(builder, dataLayout, op,
                               isDst ? subslot.ptr : subslotPtrInOther,
                               isDst ? subslotPtrInOther : subslot.ptr,
                               subslot.elemType, op.getIsVolatile());
@@ -1250,10 +1245,10 @@ bool LLVM::MemcpyOp::storesTo(const MemorySlot &slot) {
   return memcpyStoresTo(*this, slot);
 }
 
-Value LLVM::MemcpyOp::getStored(const MemorySlot &slot, RewriterBase &rewriter,
+Value LLVM::MemcpyOp::getStored(const MemorySlot &slot, OpBuilder &builder,
                                 Value reachingDef,
                                 const DataLayout &dataLayout) {
-  return memcpyGetStored(*this, slot, rewriter);
+  return memcpyGetStored(*this, slot, builder);
 }
 
 bool LLVM::MemcpyOp::canUsesBeRemoved(
@@ -1266,9 +1261,9 @@ bool LLVM::MemcpyOp::canUsesBeRemoved(
 
 DeletionKind LLVM::MemcpyOp::removeBlockingUses(
     const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
-    RewriterBase &rewriter, Value reachingDefinition,
+    OpBuilder &builder, Value reachingDefinition,
     const DataLayout &dataLayout) {
-  return memcpyRemoveBlockingUses(*this, slot, blockingUses, rewriter,
+  return memcpyRemoveBlockingUses(*this, slot, blockingUses, builder,
                                   reachingDefinition);
 }
 
@@ -1288,9 +1283,9 @@ bool LLVM::MemcpyOp::canRewire(const DestructurableMemorySlot &slot,
 
 DeletionKind LLVM::MemcpyOp::rewire(const DestructurableMemorySlot &slot,
                                     DenseMap<Attribute, MemorySlot> &subslots,
-                                    RewriterBase &rewriter,
+                                    OpBuilder &builder,
                                     const DataLayout &dataLayout) {
-  return memcpyRewire(*this, slot, subslots, rewriter, dataLayout);
+  return memcpyRewire(*this, slot, subslots, builder, dataLayout);
 }
 
 bool LLVM::MemcpyInlineOp::loadsFrom(const MemorySlot &slot) {
@@ -1302,9 +1297,9 @@ bool LLVM::MemcpyInlineOp::storesTo(const MemorySlot &slot) {
 }
 
 Value LLVM::MemcpyInlineOp::getStored(const MemorySlot &slot,
-                                      RewriterBase &rewriter, Value reachingDef,
+                                      OpBuilder &builder, Value reachingDef,
                                       const DataLayout &dataLayout) {
-  return memcpyGetStored(*this, slot, rewriter);
+  return memcpyGetStored(*this, slot, builder);
 }
 
 bool LLVM::MemcpyInlineOp::canUsesBeRemoved(
@@ -1317,9 +1312,9 @@ bool LLVM::MemcpyInlineOp::canUsesBeRemoved(
 
 DeletionKind LLVM::MemcpyInlineOp::removeBlockingUses(
     const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
-    RewriterBase &rewriter, Value reachingDefinition,
+    OpBuilder &builder, Value reachingDefinition,
     const DataLayout &dataLayout) {
-  return memcpyRemoveBlockingUses(*this, slot, blockingUses, rewriter,
+  return memcpyRemoveBlockingUses(*this, slot, blockingUses, builder,
                                   reachingDefinition);
 }
 
@@ -1341,9 +1336,8 @@ bool LLVM::MemcpyInlineOp::canRewire(
 DeletionKind
 LLVM::MemcpyInlineOp::rewire(const DestructurableMemorySlot &slot,
                              DenseMap<Attribute, MemorySlot> &subslots,
-                             RewriterBase &rewriter,
-                             const DataLayout &dataLayout) {
-  return memcpyRewire(*this, slot, subslots, rewriter, dataLayout);
+                             OpBuilder &builder, const DataLayout &dataLayout) {
+  return memcpyRewire(*this, slot, subslots, builder, dataLayout);
 }
 
 bool LLVM::MemmoveOp::loadsFrom(const MemorySlot &slot) {
@@ -1354,10 +1348,10 @@ bool LLVM::MemmoveOp::storesTo(const MemorySlot &slot) {
   return memcpyStoresTo(*this, slot);
 }
 
-Value LLVM::MemmoveOp::getStored(const MemorySlot &slot, RewriterBase &rewriter,
+Value LLVM::MemmoveOp::getStored(const MemorySlot &slot, OpBuilder &builder,
                                  Value reachingDef,
                                  const DataLayout &dataLayout) {
-  return memcpyGetStored(*this, slot, rewriter);
+  return memcpyGetStored(*this, slot, builder);
 }
 
 bool LLVM::MemmoveOp::canUsesBeRemoved(
@@ -1370,9 +1364,9 @@ bool LLVM::MemmoveOp::canUsesBeRemoved(
 
 DeletionKind LLVM::MemmoveOp::removeBlockingUses(
     const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
-    RewriterBase &rewriter, Value reachingDefinition,
+    OpBuilder &builder, Value reachingDefinition,
     const DataLayout &dataLayout) {
-  return memcpyRemoveBlockingUses(*this, slot, blockingUses, rewriter,
+  return memcpyRemoveBlockingUses(*this, slot, blockingUses, builder,
                                   reachingDefinition);
 }
 
@@ -1392,9 +1386,9 @@ bool LLVM::MemmoveOp::canRewire(const DestructurableMemorySlot &slot,
 
 DeletionKind LLVM::MemmoveOp::rewire(const DestructurableMemorySlot &slot,
                                      DenseMap<Attribute, MemorySlot> &subslots,
-                                     RewriterBase &rewriter,
+                                     OpBuilder &builder,
                                      const DataLayout &dataLayout) {
-  return memcpyRewire(*this, slot, subslots, rewriter, dataLayout);
+  return memcpyRewire(*this, slot, subslots, builder, dataLayout);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
index 958c5f0c8dbc..dca07e84ea73 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
@@ -83,30 +83,30 @@ SmallVector<MemorySlot> memref::AllocaOp::getPromotableSlots() {
 }
 
 Value memref::AllocaOp::getDefaultValue(const MemorySlot &slot,
-                                        RewriterBase &rewriter) {
+                                        OpBuilder &builder) {
   assert(isSupportedElementType(slot.elemType));
   // TODO: support more types.
   return TypeSwitch<Type, Value>(slot.elemType)
       .Case([&](MemRefType t) {
-        return rewriter.create<memref::AllocaOp>(getLoc(), t);
+        return builder.create<memref::AllocaOp>(getLoc(), t);
       })
       .Default([&](Type t) {
-        return rewriter.create<arith::ConstantOp>(getLoc(), t,
-                                                  rewriter.getZeroAttr(t));
+        return builder.create<arith::ConstantOp>(getLoc(), t,
+                                                 builder.getZeroAttr(t));
       });
 }
 
 void memref::AllocaOp::handlePromotionComplete(const MemorySlot &slot,
                                                Value defaultValue,
-                                               RewriterBase &rewriter) {
+                                               OpBuilder &builder) {
   if (defaultValue.use_empty())
-    rewriter.eraseOp(defaultValue.getDefiningOp());
-  rewriter.eraseOp(*this);
+    defaultValue.getDefiningOp()->erase();
+  this->erase();
 }
 
 void memref::AllocaOp::handleBlockArgument(const MemorySlot &slot,
                                            BlockArgument argument,
-                                           RewriterBase &rewriter) {}
+                                           OpBuilder &builder) {}
 
 SmallVector<DestructurableMemorySlot>
 memref::AllocaOp::getDestructurableSlots() {
@@ -127,8 +127,8 @@ memref::AllocaOp::getDestructurableSlots() {
 DenseMap<Attribute, MemorySlot>
 memref::AllocaOp::destructure(const DestructurableMemorySlot &slot,
                               const SmallPtrSetImpl<Attribute> &usedIndices,
-                              RewriterBase &rewriter) {
-  rewriter.setInsertionPointAfter(*this);
+                              OpBuilder &builder) {
+  builder.setInsertionPointAfter(*this);
 
   DenseMap<Attribute, MemorySlot> slotMap;
 
@@ -136,7 +136,7 @@ memref::AllocaOp::destructure(const DestructurableMemorySlot &slot,
   for (Attribute usedIndex : usedIndices) {
     Type elemType = memrefType.getTypeAtIndex(usedIndex);
     MemRefType elemPtr = MemRefType::get({}, elemType);
-    auto subAlloca = rewriter.create<memref::AllocaOp>(getLoc(), elemPtr);
+    auto subAlloca = builder.create<memref::AllocaOp>(getLoc(), elemPtr);
     slotMap.try_emplace<MemorySlot>(usedIndex,
                                     {subAlloca.getResult(), elemType});
   }
@@ -145,9 +145,9 @@ memref::AllocaOp::destructure(const DestructurableMemorySlot &slot,
 }
 
 void memref::AllocaOp::handleDestructuringComplete(
-    const DestructurableMemorySlot &slot, RewriterBase &rewriter) {
+    const DestructurableMemorySlot &slot, OpBuilder &builder) {
   assert(slot.ptr == getResult());
-  rewriter.eraseOp(*this);
+  this->erase();
 }
 
 //===----------------------------------------------------------------------===//
@@ -160,7 +160,7 @@ bool memref::LoadOp::loadsFrom(const MemorySlot &slot) {
 
 bool memref::LoadOp::storesTo(const MemorySlot &slot) { return false; }
 
-Value memref::LoadOp::getStored(const MemorySlot &slot, RewriterBase &rewriter,
+Value memref::LoadOp::getStored(const MemorySlot &slot, OpBuilder &builder,
                                 Value reachingDef,
                                 const DataLayout &dataLayout) {
   llvm_unreachable("getStored should not be called on LoadOp");
@@ -179,11 +179,11 @@ bool memref::LoadOp::canUsesBeRemoved(
 
 DeletionKind memref::LoadOp::removeBlockingUses(
     const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
-    RewriterBase &rewriter, Value reachingDefinition,
+    OpBuilder &builder, Value reachingDefinition,
     const DataLayout &dataLayout) {
   // `canUsesBeRemoved` checked this blocking use must be the loaded slot
   // pointer.
-  rewriter.replaceAllUsesWith(getResult(), reachingDefinition);
+  getResult().replaceAllUsesWith(reachingDefinition);
   return DeletionKind::Delete;
 }
 
@@ -224,15 +224,13 @@ bool memref::LoadOp::canRewire(const DestructurableMemorySlot &slot,
 
 DeletionKind memref::LoadOp::rewire(const DestructurableMemorySlot &slot,
                                     DenseMap<Attribute, MemorySlot> &subslots,
-                                    RewriterBase &rewriter,
+                                    OpBuilder &builder,
                                     const DataLayout &dataLayout) {
   Attribute index = getAttributeIndexFromIndexOperands(
       getContext(), getIndices(), getMemRefType());
   const MemorySlot &memorySlot = subslots.at(index);
-  rewriter.modifyOpInPlace(*this, [&]() {
-    setMemRef(memorySlot.ptr);
-    getIndicesMutable().clear();
-  });
+  setMemRef(memorySlot.ptr);
+  getIndicesMutable().clear();
   return DeletionKind::Keep;
 }
 
@@ -242,7 +240,7 @@ bool memref::StoreOp::storesTo(const MemorySlot &slot) {
   return getMemRef() == slot.ptr;
 }
 
-Value memref::StoreOp::getStored(const MemorySlot &slot, RewriterBase &rewriter,
+Value memref::StoreOp::getStored(const MemorySlot &slot, OpBuilder &builder,
                                  Value reachingDef,
                                  const DataLayout &dataLayout) {
   return getValue();
@@ -261,7 +259,7 @@ bool memref::StoreOp::canUsesBeRemoved(
 
 DeletionKind memref::StoreOp::removeBlockingUses(
     const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
-    RewriterBase &rewriter, Value reachingDefinition,
+    OpBuilder &builder, Value reachingDefinition,
     const DataLayout &dataLayout) {
   return DeletionKind::Delete;
 }
@@ -282,15 +280,13 @@ bool memref::StoreOp::canRewire(const DestructurableMemorySlot &slot,
 
 DeletionKind memref::StoreOp::rewire(const DestructurableMemorySlot &slot,
                                      DenseMap<Attribute, MemorySlot> &subslots,
-                                     RewriterBase &rewriter,
+                                     OpBuilder &builder,
                                      const DataLayout &dataLayout) {
   Attribute index = getAttributeIndexFromIndexOperands(
       getContext(), getIndices(), getMemRefType());
   const MemorySlot &memorySlot = subslots.at(index);
-  rewriter.modifyOpInPlace(*this, [&]() {
-    setMemRef(memorySlot.ptr);
-    getIndicesMutable().clear();
-  });
+  setMemRef(memorySlot.ptr);
+  getIndicesMutable().clear();
   return DeletionKind::Keep;
 }
 
diff --git a/mlir/lib/Transforms/Mem2Reg.cpp b/mlir/lib/Transforms/Mem2Reg.cpp
index 71ba5bc076f0..1d7ba4ca4f83 100644
--- a/mlir/lib/Transforms/Mem2Reg.cpp
+++ b/mlir/lib/Transforms/Mem2Reg.cpp
@@ -164,7 +164,7 @@ private:
 class MemorySlotPromoter {
 public:
   MemorySlotPromoter(MemorySlot slot, PromotableAllocationOpInterface allocator,
-                     RewriterBase &rewriter, DominanceInfo &dominance,
+                     OpBuilder &builder, DominanceInfo &dominance,
                      const DataLayout &dataLayout, MemorySlotPromotionInfo info,
                      const Mem2RegStatistics &statistics);
 
@@ -195,7 +195,7 @@ private:
 
   MemorySlot slot;
   PromotableAllocationOpInterface allocator;
-  RewriterBase &rewriter;
+  OpBuilder &builder;
   /// Potentially non-initialized default value. Use `getOrCreateDefaultValue`
   /// to initialize it on demand.
   Value defaultValue;
@@ -213,12 +213,10 @@ private:
 
 MemorySlotPromoter::MemorySlotPromoter(
     MemorySlot slot, PromotableAllocationOpInterface allocator,
-    RewriterBase &rewriter, DominanceInfo &dominance,
-    const DataLayout &dataLayout, MemorySlotPromotionInfo info,
-    const Mem2RegStatistics &statistics)
-    : slot(slot), allocator(allocator), rewriter(rewriter),
-      dominance(dominance), dataLayout(dataLayout), info(std::move(info)),
-      statistics(statistics) {
+    OpBuilder &builder, DominanceInfo &dominance, const DataLayout &dataLayout,
+    MemorySlotPromotionInfo info, const Mem2RegStatistics &statistics)
+    : slot(slot), allocator(allocator), builder(builder), dominance(dominance),
+      dataLayout(dataLayout), info(std::move(info)), statistics(statistics) {
 #ifndef NDEBUG
   auto isResultOrNewBlockArgument = [&]() {
     if (BlockArgument arg = dyn_cast<BlockArgument>(slot.ptr))
@@ -236,9 +234,9 @@ Value MemorySlotPromoter::getOrCreateDefaultValue() {
   if (defaultValue)
     return defaultValue;
 
-  RewriterBase::InsertionGuard guard(rewriter);
-  rewriter.setInsertionPointToStart(slot.ptr.getParentBlock());
-  return defaultValue = allocator.getDefaultValue(slot, rewriter);
+  OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPointToStart(slot.ptr.getParentBlock());
+  return defaultValue = allocator.getDefaultValue(slot, builder);
 }
 
 LogicalResult MemorySlotPromotionAnalyzer::computeBlockingUses(
@@ -437,8 +435,8 @@ Value MemorySlotPromoter::computeReachingDefInBlock(Block *block,
         reachingDefs.insert({memOp, reachingDef});
 
       if (memOp.storesTo(slot)) {
-        rewriter.setInsertionPointAfter(memOp);
-        Value stored = memOp.getStored(slot, rewriter, reachingDef, dataLayout);
+        builder.setInsertionPointAfter(memOp);
+        Value stored = memOp.getStored(slot, builder, reachingDef, dataLayout);
         assert(stored && "a memory operation storing to a slot must provide a "
                          "new definition of the slot");
         reachingDef = stored;
@@ -475,33 +473,10 @@ void MemorySlotPromoter::computeReachingDefInRegion(Region *region,
     Block *block = job.block->getBlock();
 
     if (info.mergePoints.contains(block)) {
-      // If the block is a merge point, we need to add a block argument to hold
-      // the selected reaching definition. This has to be a bit complicated
-      // because of RewriterBase limitations: we need to create a new block with
-      // the extra block argument, move the content of the block to the new
-      // block, and replace the block with the new block in the merge point set.
-      SmallVector<Type> argTypes;
-      SmallVector<Location> argLocs;
-      for (BlockArgument arg : block->getArguments()) {
-        argTypes.push_back(arg.getType());
-        argLocs.push_back(arg.getLoc());
-      }
-      argTypes.push_back(slot.elemType);
-      argLocs.push_back(slot.ptr.getLoc());
-      Block *newBlock = rewriter.createBlock(block, argTypes, argLocs);
-
-      info.mergePoints.erase(block);
-      info.mergePoints.insert(newBlock);
-
-      rewriter.replaceAllUsesWith(block, newBlock);
-      rewriter.mergeBlocks(block, newBlock,
-                           newBlock->getArguments().drop_back());
-
-      block = newBlock;
-
-      BlockArgument blockArgument = block->getArguments().back();
-      rewriter.setInsertionPointToStart(block);
-      allocator.handleBlockArgument(slot, blockArgument, rewriter);
+      BlockArgument blockArgument =
+          block->addArgument(slot.elemType, slot.ptr.getLoc());
+      builder.setInsertionPointToStart(block);
+      allocator.handleBlockArgument(slot, blockArgument, builder);
       job.reachingDef = blockArgument;
 
       if (statistics.newBlockArgumentAmount)
@@ -514,10 +489,8 @@ void MemorySlotPromoter::computeReachingDefInRegion(Region *region,
     if (auto terminator = dyn_cast<BranchOpInterface>(block->getTerminator())) {
       for (BlockOperand &blockOperand : terminator->getBlockOperands()) {
         if (info.mergePoints.contains(blockOperand.get())) {
-          rewriter.modifyOpInPlace(terminator, [&]() {
-            terminator.getSuccessorOperands(blockOperand.getOperandNumber())
-                .append(job.reachingDef);
-          });
+          terminator.getSuccessorOperands(blockOperand.getOperandNumber())
+              .append(job.reachingDef);
         }
       }
     }
@@ -569,9 +542,9 @@ void MemorySlotPromoter::removeBlockingUses() {
       if (!reachingDef)
         reachingDef = getOrCreateDefaultValue();
 
-      rewriter.setInsertionPointAfter(toPromote);
+      builder.setInsertionPointAfter(toPromote);
       if (toPromoteMemOp.removeBlockingUses(
-              slot, info.userToBlockingUses[toPromote], rewriter, reachingDef,
+              slot, info.userToBlockingUses[toPromote], builder, reachingDef,
               dataLayout) == DeletionKind::Delete)
         toErase.push_back(toPromote);
       if (toPromoteMemOp.storesTo(slot))
@@ -581,20 +554,20 @@ void MemorySlotPromoter::removeBlockingUses() {
     }
 
     auto toPromoteBasic = cast<PromotableOpInterface>(toPromote);
-    rewriter.setInsertionPointAfter(toPromote);
+    builder.setInsertionPointAfter(toPromote);
     if (toPromoteBasic.removeBlockingUses(info.userToBlockingUses[toPromote],
-                                          rewriter) == DeletionKind::Delete)
+                                          builder) == DeletionKind::Delete)
       toErase.push_back(toPromote);
     if (toPromoteBasic.requiresReplacedValues())
       toVisit.push_back(toPromoteBasic);
   }
   for (PromotableOpInterface op : toVisit) {
-    rewriter.setInsertionPointAfter(op);
-    op.visitReplacedValues(replacedValuesList, rewriter);
+    builder.setInsertionPointAfter(op);
+    op.visitReplacedValues(replacedValuesList, builder);
   }
 
   for (Operation *toEraseOp : toErase)
-    rewriter.eraseOp(toEraseOp);
+    toEraseOp->erase();
 
   assert(slot.ptr.use_empty() &&
          "after promotion, the slot pointer should not be used anymore");
@@ -617,8 +590,7 @@ void MemorySlotPromoter::promoteSlot() {
       assert(succOperands.size() == mergePoint->getNumArguments() ||
              succOperands.size() + 1 == mergePoint->getNumArguments());
       if (succOperands.size() + 1 == mergePoint->getNumArguments())
-        rewriter.modifyOpInPlace(
-            user, [&]() { succOperands.append(getOrCreateDefaultValue()); });
+        succOperands.append(getOrCreateDefaultValue());
     }
   }
 
@@ -628,13 +600,12 @@ void MemorySlotPromoter::promoteSlot() {
   if (statistics.promotedAmount)
     (*statistics.promotedAmount)++;
 
-  allocator.handlePromotionComplete(slot, defaultValue, rewriter);
+  allocator.handlePromotionComplete(slot, defaultValue, builder);
 }
 
 LogicalResult mlir::tryToPromoteMemorySlots(
-    ArrayRef<PromotableAllocationOpInterface> allocators,
-    RewriterBase &rewriter, const DataLayout &dataLayout,
-    Mem2RegStatistics statistics) {
+    ArrayRef<PromotableAllocationOpInterface> allocators, OpBuilder &builder,
+    const DataLayout &dataLayout, Mem2RegStatistics statistics) {
   bool promotedAny = false;
 
   for (PromotableAllocationOpInterface allocator : allocators) {
@@ -646,7 +617,7 @@ LogicalResult mlir::tryToPromoteMemorySlots(
       MemorySlotPromotionAnalyzer analyzer(slot, dominance, dataLayout);
       std::optional<MemorySlotPromotionInfo> info = analyzer.computeInfo();
       if (info) {
-        MemorySlotPromoter(slot, allocator, rewriter, dominance, dataLayout,
+        MemorySlotPromoter(slot, allocator, builder, dominance, dataLayout,
                            std::move(*info), statistics)
             .promoteSlot();
         promotedAny = true;
@@ -674,7 +645,6 @@ struct Mem2Reg : impl::Mem2RegBase<Mem2Reg> {
         continue;
 
       OpBuilder builder(&region.front(), region.front().begin());
-      IRRewriter rewriter(builder);
 
       // Promoting a slot can allow for further promotion of other slots,
       // promotion is tried until no promotion succeeds.
@@ -689,7 +659,7 @@ struct Mem2Reg : impl::Mem2RegBase<Mem2Reg> {
         const DataLayout &dataLayout = dataLayoutAnalysis.getAtOrAbove(scopeOp);
 
         // Attempt promoting until no promotion succeeds.
-        if (failed(tryToPromoteMemorySlots(allocators, rewriter, dataLayout,
+        if (failed(tryToPromoteMemorySlots(allocators, builder, dataLayout,
                                            statistics)))
           break;
 
diff --git a/mlir/lib/Transforms/SROA.cpp b/mlir/lib/Transforms/SROA.cpp
index f24cbb7b1725..4e28fa687ffd 100644
--- a/mlir/lib/Transforms/SROA.cpp
+++ b/mlir/lib/Transforms/SROA.cpp
@@ -134,15 +134,14 @@ computeDestructuringInfo(DestructurableMemorySlot &slot,
 /// subslots as specified by its allocator.
 static void destructureSlot(DestructurableMemorySlot &slot,
                             DestructurableAllocationOpInterface allocator,
-                            RewriterBase &rewriter,
-                            const DataLayout &dataLayout,
+                            OpBuilder &builder, const DataLayout &dataLayout,
                             MemorySlotDestructuringInfo &info,
                             const SROAStatistics &statistics) {
-  RewriterBase::InsertionGuard guard(rewriter);
+  OpBuilder::InsertionGuard guard(builder);
 
-  rewriter.setInsertionPointToStart(slot.ptr.getParentBlock());
+  builder.setInsertionPointToStart(slot.ptr.getParentBlock());
   DenseMap<Attribute, MemorySlot> subslots =
-      allocator.destructure(slot, info.usedIndices, rewriter);
+      allocator.destructure(slot, info.usedIndices, builder);
 
   if (statistics.slotsWithMemoryBenefit &&
       slot.elementPtrs.size() != info.usedIndices.size())
@@ -160,9 +159,9 @@ static void destructureSlot(DestructurableMemorySlot &slot,
 
   llvm::SmallVector<Operation *> toErase;
   for (Operation *toRewire : llvm::reverse(usersToRewire)) {
-    rewriter.setInsertionPointAfter(toRewire);
+    builder.setInsertionPointAfter(toRewire);
     if (auto accessor = dyn_cast<DestructurableAccessorOpInterface>(toRewire)) {
-      if (accessor.rewire(slot, subslots, rewriter, dataLayout) ==
+      if (accessor.rewire(slot, subslots, builder, dataLayout) ==
           DeletionKind::Delete)
         toErase.push_back(accessor);
       continue;
@@ -170,12 +169,12 @@ static void destructureSlot(DestructurableMemorySlot &slot,
 
     auto promotable = cast<PromotableOpInterface>(toRewire);
     if (promotable.removeBlockingUses(info.userToBlockingUses[promotable],
-                                      rewriter) == DeletionKind::Delete)
+                                      builder) == DeletionKind::Delete)
       toErase.push_back(promotable);
   }
 
   for (Operation *toEraseOp : toErase)
-    rewriter.eraseOp(toEraseOp);
+    toEraseOp->erase();
 
   assert(slot.ptr.use_empty() && "after destructuring, the original slot "
                                  "pointer should no longer be used");
@@ -186,12 +185,12 @@ static void destructureSlot(DestructurableMemorySlot &slot,
   if (statistics.destructuredAmount)
     (*statistics.destructuredAmount)++;
 
-  allocator.handleDestructuringComplete(slot, rewriter);
+  allocator.handleDestructuringComplete(slot, builder);
 }
 
 LogicalResult mlir::tryToDestructureMemorySlots(
     ArrayRef<DestructurableAllocationOpInterface> allocators,
-    RewriterBase &rewriter, const DataLayout &dataLayout,
+    OpBuilder &builder, const DataLayout &dataLayout,
     SROAStatistics statistics) {
   bool destructuredAny = false;
 
@@ -202,7 +201,7 @@ LogicalResult mlir::tryToDestructureMemorySlots(
       if (!info)
         continue;
 
-      destructureSlot(slot, allocator, rewriter, dataLayout, *info, statistics);
+      destructureSlot(slot, allocator, builder, dataLayout, *info, statistics);
       destructuredAny = true;
     }
   }
@@ -230,7 +229,6 @@ struct SROA : public impl::SROABase<SROA> {
         continue;
 
       OpBuilder builder(&region.front(), region.front().begin());
-      IRRewriter rewriter(builder);
 
       // Destructuring a slot can allow for further destructuring of other
       // slots, destructuring is tried until no destructuring succeeds.
@@ -243,7 +241,7 @@ struct SROA : public impl::SROABase<SROA> {
           allocators.emplace_back(allocator);
         });
 
-        if (failed(tryToDestructureMemorySlots(allocators, rewriter, dataLayout,
+        if (failed(tryToDestructureMemorySlots(allocators, builder, dataLayout,
                                                statistics)))
           break;
 
-- 
GitLab


From a99ce615f19fec6fbb835490b89f53cba3cf9eff Mon Sep 17 00:00:00 2001
From: jyu2-git <jennifer.yu@intel.com>
Date: Tue, 7 May 2024 23:11:07 -0700
Subject: [PATCH 0140/1206] =?UTF-8?q?Revert=20"Revert=20"[OpenMP][TR12]=20?=
 =?UTF-8?q?change=20property=20of=20map-type=20modifier."=E2=80=A6=20(#911?=
 =?UTF-8?q?41)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… (#90885)"

This reverts commit eea81aa29848361eb5b24f24d2af643fdeb9adfd.

Also change isMapType as @vitalybuka suggested. Hope this fix sanitizer
build problem.
---
 .../clang/Basic/DiagnosticParseKinds.td       |   5 +
 clang/lib/Parse/ParseOpenMP.cpp               |  51 +++++++--
 clang/test/OpenMP/target_ast_print.cpp        |  58 ++++++++++
 clang/test/OpenMP/target_map_messages.cpp     | 105 ++++++++++--------
 4 files changed, 165 insertions(+), 54 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index fdffb35ea0d9..44bc4e0e130d 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1438,6 +1438,9 @@ def err_omp_decl_in_declare_simd_variant : Error<
 def err_omp_sink_and_source_iteration_not_allowd: Error<" '%0 %select{sink:|source:}1' must be with '%select{omp_cur_iteration - 1|omp_cur_iteration}1'">;
 def err_omp_unknown_map_type : Error<
   "incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'">;
+def err_omp_more_one_map_type : Error<"map type is already specified">;
+def note_previous_map_type_specified_here
+    : Note<"map type '%0' is previous specified here">;
 def err_omp_unknown_map_type_modifier : Error<
   "incorrect map type modifier, expected one of: 'always', 'close', 'mapper'"
   "%select{|, 'present'|, 'present', 'iterator'}0%select{|, 'ompx_hold'}1">;
@@ -1445,6 +1448,8 @@ def err_omp_map_type_missing : Error<
   "missing map type">;
 def err_omp_map_type_modifier_missing : Error<
   "missing map type modifier">;
+def err_omp_map_modifier_specification_list : Error<
+  "empty modifier-specification-list is not allowed">;
 def err_omp_declare_simd_inbranch_notinbranch : Error<
   "unexpected '%0' clause, '%1' is specified already">;
 def err_omp_expected_clause_argument
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 18ba1185ee8d..5265d8f1922c 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -4228,13 +4228,20 @@ bool Parser::parseMapperModifier(SemaOpenMP::OpenMPVarListDataTy &Data) {
   return T.consumeClose();
 }
 
+static OpenMPMapClauseKind isMapType(Parser &P);
+
 /// Parse map-type-modifiers in map clause.
-/// map([ [map-type-modifier[,] [map-type-modifier[,] ...] map-type : ] list)
+/// map([ [map-type-modifier[,] [map-type-modifier[,] ...] [map-type] : ] list)
 /// where, map-type-modifier ::= always | close | mapper(mapper-identifier) |
 /// present
+/// where, map-type ::= alloc | delete | from | release | to | tofrom
 bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
+  bool HasMapType = false;
+  SourceLocation PreMapLoc = Tok.getLocation();
+  StringRef PreMapName = "";
   while (getCurToken().isNot(tok::colon)) {
     OpenMPMapModifierKind TypeModifier = isMapModifier(*this);
+    OpenMPMapClauseKind MapKind = isMapType(*this);
     if (TypeModifier == OMPC_MAP_MODIFIER_always ||
         TypeModifier == OMPC_MAP_MODIFIER_close ||
         TypeModifier == OMPC_MAP_MODIFIER_present ||
@@ -4257,6 +4264,19 @@ bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
         Diag(Data.MapTypeModifiersLoc.back(), diag::err_omp_missing_comma)
             << "map type modifier";
 
+    } else if (getLangOpts().OpenMP >= 60 && MapKind != OMPC_MAP_unknown) {
+      if (!HasMapType) {
+        HasMapType = true;
+        Data.ExtraModifier = MapKind;
+        MapKind = OMPC_MAP_unknown;
+        PreMapLoc = Tok.getLocation();
+        PreMapName = Tok.getIdentifierInfo()->getName();
+      } else {
+        Diag(Tok, diag::err_omp_more_one_map_type);
+        Diag(PreMapLoc, diag::note_previous_map_type_specified_here)
+            << PreMapName;
+      }
+      ConsumeToken();
     } else {
       // For the case of unknown map-type-modifier or a map-type.
       // Map-type is followed by a colon; the function returns when it
@@ -4267,8 +4287,14 @@ bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
         continue;
       }
       // Potential map-type token as it is followed by a colon.
-      if (PP.LookAhead(0).is(tok::colon))
-        return false;
+      if (PP.LookAhead(0).is(tok::colon)) {
+        if (getLangOpts().OpenMP >= 60) {
+          break;
+        } else {
+          return false;
+        }
+      }
+
       Diag(Tok, diag::err_omp_unknown_map_type_modifier)
           << (getLangOpts().OpenMP >= 51 ? (getLangOpts().OpenMP >= 52 ? 2 : 1)
                                          : 0)
@@ -4278,6 +4304,14 @@ bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
     if (getCurToken().is(tok::comma))
       ConsumeToken();
   }
+  if (getLangOpts().OpenMP >= 60 && !HasMapType) {
+    if (!Tok.is(tok::colon)) {
+      Diag(Tok, diag::err_omp_unknown_map_type);
+      ConsumeToken();
+    } else {
+      Data.ExtraModifier = OMPC_MAP_unknown;
+    }
+  }
   return false;
 }
 
@@ -4289,13 +4323,12 @@ static OpenMPMapClauseKind isMapType(Parser &P) {
   if (!Tok.isOneOf(tok::identifier, tok::kw_delete))
     return OMPC_MAP_unknown;
   Preprocessor &PP = P.getPreprocessor();
-  OpenMPMapClauseKind MapType =
-      static_cast<OpenMPMapClauseKind>(getOpenMPSimpleClauseType(
-          OMPC_map, PP.getSpelling(Tok), P.getLangOpts()));
+  unsigned MapType =
+      getOpenMPSimpleClauseType(OMPC_map, PP.getSpelling(Tok), P.getLangOpts());
   if (MapType == OMPC_MAP_to || MapType == OMPC_MAP_from ||
       MapType == OMPC_MAP_tofrom || MapType == OMPC_MAP_alloc ||
       MapType == OMPC_MAP_delete || MapType == OMPC_MAP_release)
-    return MapType;
+    return static_cast<OpenMPMapClauseKind>(MapType);
   return OMPC_MAP_unknown;
 }
 
@@ -4679,8 +4712,10 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     // Only parse map-type-modifier[s] and map-type if a colon is present in
     // the map clause.
     if (ColonPresent) {
+      if (getLangOpts().OpenMP >= 60 && getCurToken().is(tok::colon))
+        Diag(Tok, diag::err_omp_map_modifier_specification_list);
       IsInvalidMapperModifier = parseMapTypeModifiers(Data);
-      if (!IsInvalidMapperModifier)
+      if (getLangOpts().OpenMP < 60 && !IsInvalidMapperModifier)
         parseMapType(*this, Data);
       else
         SkipUntil(tok::colon, tok::annot_pragma_openmp_end, StopBeforeMatch);
diff --git a/clang/test/OpenMP/target_ast_print.cpp b/clang/test/OpenMP/target_ast_print.cpp
index f4c10fe3a181..ac5ed285d97e 100644
--- a/clang/test/OpenMP/target_ast_print.cpp
+++ b/clang/test/OpenMP/target_ast_print.cpp
@@ -1201,6 +1201,64 @@ foo();
 }
 #endif // OMP52
 
+#ifdef OMP60
+
+///==========================================================================///
+// RUN: %clang_cc1 -DOMP60 -verify -Wno-vla -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix OMP60
+// RUN: %clang_cc1 -DOMP60 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP60
+
+// RUN: %clang_cc1 -DOMP60 -verify -Wno-vla -fopenmp-simd -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix OMP60
+// RUN: %clang_cc1 -DOMP60 -fopenmp-simd -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp-simd -fopenmp-version=60 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP60
+
+void foo() {}
+template <typename T, int C>
+T tmain(T argc, T *argv) {
+  T i;
+#pragma omp target map(from always: i)
+  foo();
+#pragma omp target map(from, close: i)
+  foo();
+#pragma omp target map(always,close: i)
+  foo();
+  return 0;
+}
+//OMP60: template <typename T, int C> T tmain(T argc, T *argv) {
+//OMP60-NEXT: T i;
+//OMP60-NEXT: #pragma omp target map(always,from: i)
+//OMP60-NEXT:     foo();
+//OMP60-NEXT: #pragma omp target map(close,from: i)
+//OMP60-NEXT:     foo();
+//OMP60-NEXT: #pragma omp target map(always,close,tofrom: i)
+//OMP60-NEXT:     foo();
+//OMP60-NEXT: return 0;
+//OMP60-NEXT:}
+//OMP60:  template<> int tmain<int, 5>(int argc, int *argv) {
+//OMP60-NEXT:  int i;
+//OMP60-NEXT:  #pragma omp target map(always,from: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  #pragma omp target map(close,from: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  #pragma omp target map(always,close,tofrom: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  return 0;
+//OMP60-NEXT:}
+//OMP60:  template<> char tmain<char, 1>(char argc, char *argv) {
+//OMP60-NEXT:  char i;
+//OMP60-NEXT:  #pragma omp target map(always,from: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  #pragma omp target map(close,from: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  #pragma omp target map(always,close,tofrom: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  return 0;
+//OMP60-NEXT:}
+int main (int argc, char **argv) {
+  return tmain<int, 5>(argc, &argc) + tmain<char, 1>(argv[0][0], argv[0]);
+}
+#endif // OMP60
+
 #ifdef OMPX
 
 // RUN: %clang_cc1 -DOMPX -verify -Wno-vla -fopenmp -fopenmp-extensions -ast-print %s | FileCheck %s --check-prefix=OMPX
diff --git a/clang/test/OpenMP/target_map_messages.cpp b/clang/test/OpenMP/target_map_messages.cpp
index a6776ee12c0e..3bd432b47e63 100644
--- a/clang/test/OpenMP/target_map_messages.cpp
+++ b/clang/test/OpenMP/target_map_messages.cpp
@@ -1,34 +1,35 @@
 // -fopenmp, -fno-openmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,omp,ge51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=51 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,ge52,omp,ge52-omp,omp52 -fopenmp -fno-openmp-extensions -fopenmp-version=52 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,omp,ge51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=51 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,ge52,lt60,omp,ge52-omp,omp52 -fopenmp -fno-openmp-extensions -fopenmp-version=52 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge52,ge60,omp,ge60-omp,omp60 -fopenmp -fno-openmp-extensions -fopenmp-version=60 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized -Wno-vla
 
 // -fopenmp-simd, -fno-openmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,omp,ge51-omp -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,omp,ge51-omp -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
 
 // -fopenmp -fopenmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,ompx,ge51-ompx -fopenmp -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,ompx,ge51-ompx -fopenmp -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized -Wno-vla
 
 // -fopenmp-simd -fopenmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,ompx,ge51-ompx -fopenmp-simd -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,ompx,ge51-ompx -fopenmp-simd -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
 
 // Check
@@ -113,7 +114,7 @@ struct SA {
     #pragma omp target map(b[true:true])
     {}
 
-    #pragma omp target map(: c,f) // expected-error {{missing map type}}
+    #pragma omp target map(: c,f) // lt60-error {{missing map type}} // ge60-error {{empty modifier-specification-list is not allowed}}
     {}
     #pragma omp target map(always, tofrom: c,f)
     {}
@@ -159,28 +160,28 @@ struct SA {
     // expected-error@+1 {{use of undeclared identifier 'present'}}
     #pragma omp target map(present)
     {}
-    // ge52-omp-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c,f)
     {}
-    // ge52-omp-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c[1:2],f)
     {}
-    // ge52-omp-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c,f[1:2])
     {}
-    // ge52-omp-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+3 {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c[:],f)
     {}
-    // ge52-omp-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+3 {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
@@ -193,19 +194,19 @@ struct SA {
     {}
     #pragma omp target map(always, close, always, close, tofrom: a)   // expected-error 2 {{same map type modifier has been specified more than once}}
     {}
+    // ge60-error@+3 {{same map type modifier has been specified more than once}}
     // ge51-error@+2 {{same map type modifier has been specified more than once}}
     // lt51-error@+1 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(present, present, tofrom: a)
     {}
-    // ge52-omp-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
-    // ge52-omp-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+4 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ompx-error@+3 {{same map type modifier has been specified more than once}}
     // ge51-omp-error@+2 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, ompx_hold, tofrom: a)
     {}
-    // ge52-omp-error@+9 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
-    // ge52-omp-error@+8 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge60-error@+9 {{same map type modifier has been specified more than once}}
+    // ge52-error@+8 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+7 2 {{same map type modifier has been specified more than once}}
     // ge51-error@+6 {{same map type modifier has been specified more than once}}
     // lt51-ompx-error@+5 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'ompx_hold'}}
@@ -219,34 +220,45 @@ struct SA {
     {}
     #pragma omp target map( , , tofrom: a)   // expected-error {{missing map type modifier}} expected-error {{missing map type modifier}}
     {}
-    #pragma omp target map( , , : a)   // expected-error {{missing map type modifier}} expected-error {{missing map type modifier}} expected-error {{missing map type}}
+    #pragma omp target map( , , : a)   // expected-error {{missing map type modifier}} expected-error {{missing map type modifier}} lt60-error {{missing map type}}
     {}
+    // ge60-error@+4 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-error@+3 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+2 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     // expected-error@+1 {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
     #pragma omp target map( d, f, bf: a)
     {}
+    // ge60-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator}}
     // expected-error@+4 {{missing map type modifier}}
     // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-    // expected-error@+1 {{missing map type}}
+    // lt60-error@+1 {{missing map type}}
     #pragma omp target map( , f, : a)
     {}
-    #pragma omp target map(always close: a)   // expected-error {{missing map type}} omp52-error{{missing ',' after map type modifier}}
+    #pragma omp target map(always close: a)   // lt60-error {{missing map type}} ge52-error{{missing ',' after map type modifier}}
     {}
-    #pragma omp target map(always close bf: a)   // omp52-error 2 {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}} 
+    #pragma omp target map(always close bf: a)   // ge52-error 2 {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
     {}
-    // omp52-error@+4 {{missing ',' after map type modifier}}
+    // ge52-error@+4 {{missing ',' after map type modifier}}
     // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-    // expected-error@+1 {{missing map type}}
+    // lt60-error@+1 {{missing map type}}
     #pragma omp target map(always tofrom close: a)
     {}
+    // ge60-note@+4 {{map type 'tofrom' is previous specified here}}
+    // ge60-error@+3 {{map type is already specified}}
     // ge51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(tofrom from: a)
     {}
-    #pragma omp target map(close bf: a)   // omp52-error {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+    // ge60-note@+5 {{map type 'to' is previous specified here}}
+    // ge60-error@+4 {{map type is already specified}}
+    // ge52-error@+3 {{missing ',' after map type modifier}}
+    // ge51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
+    // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
+    #pragma omp target map(to always from: a)
+    {}
+    #pragma omp target map(close bf: a)   // ge52-error {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
     {}
     #pragma omp target map(([b[I]][bf])f)  // lt50-error {{expected ',' or ']' in lambda capture list}} lt50-error {{expected ')'}} lt50-note {{to match this '('}}
     {}
@@ -266,6 +278,7 @@ struct SA {
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(iterator(it=0:10, it=0:20), tofrom:a)
     {}
+    // ge60-error@+7 {{expected '(' after 'iterator'}}
     // ge51-ompx-error@+6 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'ompx_hold'}}
     // lt51-ompx-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'ompx_hold'}}
     // lt51-error@+4 {{expected '(' after 'iterator'}}
@@ -694,20 +707,20 @@ T tmain(T argc) {
   foo();
 
 #pragma omp target data map(always, tofrom: x)
-#pragma omp target data map(always: x) // expected-error {{missing map type}}
+#pragma omp target data map(always: x) // lt60-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// expected-error@+1 {{missing map type}}
+// lt60-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, always: x)
 #pragma omp target data map(always, tofrom: always, tofrom, x)
 #pragma omp target map(tofrom j) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
 
 #pragma omp target data map(close, tofrom: x)
-#pragma omp target data map(close: x) // expected-error {{missing map type}}
+#pragma omp target data map(close: x) // lt60-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// expected-error@+1 {{missing map type}}
+// lt60-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, close: x)
 #pragma omp target data map(close, tofrom: close, tofrom, x)
   foo();
@@ -829,19 +842,19 @@ int main(int argc, char **argv) {
   foo();
 
 #pragma omp target data map(always, tofrom: x)
-#pragma omp target data map(always: x) // expected-error {{missing map type}}
+#pragma omp target data map(always: x) // lt60-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// expected-error@+1 {{missing map type}}
+// lt60-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, always: x)
 #pragma omp target data map(always, tofrom: always, tofrom, x)
 #pragma omp target map(tofrom j) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
 #pragma omp target data map(close, tofrom: x)
-#pragma omp target data map(close: x) // expected-error {{missing map type}}
+#pragma omp target data map(close: x) // lt60-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// expected-error@+1 {{missing map type}}
+// lt60-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, close: x)
   foo();
 // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-- 
GitLab


From 8755d24cb34b902557469445e1983850e0ce7cc7 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Tue, 7 May 2024 23:25:33 -0700
Subject: [PATCH 0141/1206] [compiler-rt][ctx_profile] Fix signed-ness warnings
 in test

Follow-up from PR ##89838. Some build bots warn-as-error
about signed/unsigned comparison in CtxInstrProfilingTest.

Example: https://lab.llvm.org/buildbot/#/builders/37/builds/34610
---
 .../lib/ctx_profile/tests/CtxInstrProfilingTest.cpp  | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp b/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
index f6ebe6ab2e50..1e96aea19ce4 100644
--- a/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
+++ b/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
@@ -178,19 +178,19 @@ TEST_F(ContextTest, Dump) {
 
     bool write(const ContextNode &Node) {
       EXPECT_FALSE(Root->Taken.TryLock());
-      EXPECT_EQ(Node.guid(), 1);
+      EXPECT_EQ(Node.guid(), 1U);
       EXPECT_EQ(Node.counters()[0], Entries);
-      EXPECT_EQ(Node.counters_size(), 10);
-      EXPECT_EQ(Node.callsites_size(), 4);
+      EXPECT_EQ(Node.counters_size(), 10U);
+      EXPECT_EQ(Node.callsites_size(), 4U);
       EXPECT_EQ(Node.subContexts()[0], nullptr);
       EXPECT_EQ(Node.subContexts()[1], nullptr);
       EXPECT_NE(Node.subContexts()[2], nullptr);
       EXPECT_EQ(Node.subContexts()[3], nullptr);
       const auto &SN = *Node.subContexts()[2];
-      EXPECT_EQ(SN.guid(), 2);
+      EXPECT_EQ(SN.guid(), 2U);
       EXPECT_EQ(SN.counters()[0], Entries);
-      EXPECT_EQ(SN.counters_size(), 3);
-      EXPECT_EQ(SN.callsites_size(), 1);
+      EXPECT_EQ(SN.counters_size(), 3U);
+      EXPECT_EQ(SN.callsites_size(), 1U);
       EXPECT_EQ(SN.subContexts()[0], nullptr);
       State = true;
       return true;
-- 
GitLab


From 23ae482bd01d7c966f871ddd620e9a26d6d66299 Mon Sep 17 00:00:00 2001
From: martinboehme <mboehme@google.com>
Date: Wed, 8 May 2024 08:36:53 +0200
Subject: [PATCH 0142/1206] [clang][dataflow] Allow `DataflowAnalysisContext`
 to use a non-owned `Solver`. (#91316)

For some callers (see change in DataflowAnalysis.h), this is more
convenient.
---
 .../Analysis/FlowSensitive/DataflowAnalysis.h |  5 ++--
 .../FlowSensitive/DataflowAnalysisContext.h   | 24 +++++++++++++++++--
 .../FlowSensitive/DataflowAnalysisContext.cpp | 10 ++++----
 3 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h
index 67eccdd030dc..763af2445476 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h
@@ -283,9 +283,8 @@ llvm::Expected<llvm::SmallVector<Diagnostic>> diagnoseFunction(
   if (!Context)
     return Context.takeError();
 
-  auto OwnedSolver = std::make_unique<WatchedLiteralsSolver>(MaxSATIterations);
-  const WatchedLiteralsSolver *Solver = OwnedSolver.get();
-  DataflowAnalysisContext AnalysisContext(std::move(OwnedSolver));
+  auto Solver = std::make_unique<WatchedLiteralsSolver>(MaxSATIterations);
+  DataflowAnalysisContext AnalysisContext(*Solver);
   Environment Env(AnalysisContext, FuncDecl);
   AnalysisT Analysis = createAnalysis<AnalysisT>(ASTCtx, Env);
   llvm::SmallVector<Diagnostic> Diagnostics;
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
index aa2c366cb164..5be4a1145f40 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
@@ -67,7 +67,19 @@ public:
   DataflowAnalysisContext(std::unique_ptr<Solver> S,
                           Options Opts = Options{
                               /*ContextSensitiveOpts=*/std::nullopt,
-                              /*Logger=*/nullptr});
+                              /*Logger=*/nullptr})
+      : DataflowAnalysisContext(*S, std::move(S), Opts) {}
+
+  /// Constructs a dataflow analysis context.
+  ///
+  /// Requirements:
+  ///
+  ///  `S` must outlive the `DataflowAnalysisContext`.
+  DataflowAnalysisContext(Solver &S, Options Opts = Options{
+                                         /*ContextSensitiveOpts=*/std::nullopt,
+                                         /*Logger=*/nullptr})
+      : DataflowAnalysisContext(S, nullptr, Opts) {}
+
   ~DataflowAnalysisContext();
 
   /// Sets a callback that returns the names and types of the synthetic fields
@@ -209,6 +221,13 @@ private:
     using DenseMapInfo::isEqual;
   };
 
+  /// `S` is the solver to use. `OwnedSolver` may be:
+  /// *  Null (in which case `S` is non-onwed and must outlive this object), or
+  /// *  Non-null (in which case it must refer to `S`, and the
+  ///    `DataflowAnalysisContext will take ownership of `OwnedSolver`).
+  DataflowAnalysisContext(Solver &S, std::unique_ptr<Solver> &&OwnedSolver,
+                          Options Opts);
+
   // Extends the set of modeled field declarations.
   void addModeledFields(const FieldSet &Fields);
 
@@ -232,7 +251,8 @@ private:
            Solver::Result::Status::Unsatisfiable;
   }
 
-  std::unique_ptr<Solver> S;
+  Solver &S;
+  std::unique_ptr<Solver> OwnedSolver;
   std::unique_ptr<Arena> A;
 
   // Maps from program declarations and statements to storage locations that are
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
index e94fd39c45dc..4b86daa56d7b 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
@@ -170,7 +170,7 @@ DataflowAnalysisContext::joinFlowConditions(Atom FirstToken,
 
 Solver::Result DataflowAnalysisContext::querySolver(
     llvm::SetVector<const Formula *> Constraints) {
-  return S->solve(Constraints.getArrayRef());
+  return S.solve(Constraints.getArrayRef());
 }
 
 bool DataflowAnalysisContext::flowConditionImplies(Atom Token,
@@ -338,10 +338,10 @@ static std::unique_ptr<Logger> makeLoggerFromCommandLine() {
   return Logger::html(std::move(StreamFactory));
 }
 
-DataflowAnalysisContext::DataflowAnalysisContext(std::unique_ptr<Solver> S,
-                                                 Options Opts)
-    : S(std::move(S)), A(std::make_unique<Arena>()), Opts(Opts) {
-  assert(this->S != nullptr);
+DataflowAnalysisContext::DataflowAnalysisContext(
+    Solver &S, std::unique_ptr<Solver> &&OwnedSolver, Options Opts)
+    : S(S), OwnedSolver(std::move(OwnedSolver)), A(std::make_unique<Arena>()),
+      Opts(Opts) {
   // If the -dataflow-log command-line flag was set, synthesize a logger.
   // This is ugly but provides a uniform method for ad-hoc debugging dataflow-
   // based tools.
-- 
GitLab


From e44600f3ab58b0e93a2a80f18e17181c2bc007a4 Mon Sep 17 00:00:00 2001
From: Freddy Ye <freddy.ye@intel.com>
Date: Wed, 8 May 2024 15:07:18 +0800
Subject: [PATCH 0143/1206] [X86][CFE] Support EGPR in GCCRegNames. (#91323)

---
 clang/lib/Basic/Targets/X86.cpp              |  19 ++-
 clang/test/CodeGen/X86/inline-asm-gcc-regs.c | 121 +++++++++++++++++++
 2 files changed, 139 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/CodeGen/X86/inline-asm-gcc-regs.c

diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index bf1767c87fe1..67e2126cf766 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -64,6 +64,8 @@ static const char *const GCCRegNames[] = {
     "dr0",   "dr1",   "dr2",   "dr3",   "dr6",     "dr7",
     "bnd0",  "bnd1",  "bnd2",  "bnd3",
     "tmm0",  "tmm1",  "tmm2",  "tmm3",  "tmm4",    "tmm5",  "tmm6",  "tmm7",
+    "r16",   "r17",   "r18",   "r19",   "r20",     "r21",   "r22",   "r23",
+    "r24",   "r25",   "r26",   "r27",   "r28",     "r29",   "r30",   "r31",
 };
 
 const TargetInfo::AddlRegName AddlRegNames[] = {
@@ -83,8 +85,23 @@ const TargetInfo::AddlRegName AddlRegNames[] = {
     {{"r13d", "r13w", "r13b"}, 43},
     {{"r14d", "r14w", "r14b"}, 44},
     {{"r15d", "r15w", "r15b"}, 45},
+    {{"r16d", "r16w", "r16b"}, 165},
+    {{"r17d", "r17w", "r17b"}, 166},
+    {{"r18d", "r18w", "r18b"}, 167},
+    {{"r19d", "r19w", "r19b"}, 168},
+    {{"r20d", "r20w", "r20b"}, 169},
+    {{"r21d", "r21w", "r21b"}, 170},
+    {{"r22d", "r22w", "r22b"}, 171},
+    {{"r23d", "r23w", "r23b"}, 172},
+    {{"r24d", "r24w", "r24b"}, 173},
+    {{"r25d", "r25w", "r25b"}, 174},
+    {{"r26d", "r26w", "r26b"}, 175},
+    {{"r27d", "r27w", "r27b"}, 176},
+    {{"r28d", "r28w", "r28b"}, 177},
+    {{"r29d", "r29w", "r29b"}, 178},
+    {{"r30d", "r30w", "r30b"}, 179},
+    {{"r31d", "r31w", "r31b"}, 180},
 };
-
 } // namespace targets
 } // namespace clang
 
diff --git a/clang/test/CodeGen/X86/inline-asm-gcc-regs.c b/clang/test/CodeGen/X86/inline-asm-gcc-regs.c
new file mode 100644
index 000000000000..17adbdc20a40
--- /dev/null
+++ b/clang/test/CodeGen/X86/inline-asm-gcc-regs.c
@@ -0,0 +1,121 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -O2 %s -o - | FileCheck %s
+
+// CHECK-LABEL: @test_r15
+// CHECK: call void asm sideeffect "", "{r15},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r15() {
+    register int a asm ("r15");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r16
+// CHECK: call void asm sideeffect "", "{r16},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r16() {
+    register int a asm ("r16");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r17
+// CHECK: call void asm sideeffect "", "{r17},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r17() {
+    register int a asm ("r17");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r18
+// CHECK: call void asm sideeffect "", "{r18},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r18() {
+    register int a asm ("r18");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r19
+// CHECK: call void asm sideeffect "", "{r19},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r19() {
+    register int a asm ("r19");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r20
+// CHECK: call void asm sideeffect "", "{r20},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r20() {
+    register int a asm ("r20");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r21
+// CHECK: call void asm sideeffect "", "{r21},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r21() {
+    register int a asm ("r21");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r22
+// CHECK: call void asm sideeffect "", "{r22},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r22() {
+    register int a asm ("r22");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r23
+// CHECK: call void asm sideeffect "", "{r23},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r23() {
+    register int a asm ("r23");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r24
+// CHECK: call void asm sideeffect "", "{r24},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r24() {
+    register int a asm ("r24");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r25
+// CHECK: call void asm sideeffect "", "{r25},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r25() {
+    register int a asm ("r25");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r26
+// CHECK: call void asm sideeffect "", "{r26},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r26() {
+    register int a asm ("r26");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r27
+// CHECK: call void asm sideeffect "", "{r27},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r27() {
+    register int a asm ("r27");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r28
+// CHECK: call void asm sideeffect "", "{r28},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r28() {
+    register int a asm ("r28");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r29
+// CHECK: call void asm sideeffect "", "{r29},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r29() {
+    register int a asm ("r29");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r30
+// CHECK: call void asm sideeffect "", "{r30},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r30() {
+    register int a asm ("r30");
+    asm ("" :: "r" (a));
+}
+
+// CHECK-LABEL: @test_r31
+// CHECK: call void asm sideeffect "", "{r31},~{dirflag},~{fpsr},~{flags}"(i32 undef)
+void test_r31() {
+    register int a asm ("r31");
+    asm ("" :: "r" (a));
+}
+
-- 
GitLab


From bbd6a2d85c44d99e66b471d251a742f7551a0c61 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Wed, 8 May 2024 15:38:13 +0800
Subject: [PATCH 0144/1206] [RISCV] Convert implicit_def tuples to noreg in
 post-isel peephole (#91173)

If a segmented load has an undefined passthru then it will be selected
as a reg_sequence with implicit_def operands, which currently slips
through the implicit_def -> noreg peephole.

This patch fixes this so we're able to infer if the passthru is
undefined without the need for looking through vreg definitions with
MachineRegisterInfo, which will help with moving RISCVInsertVSETVLI to
LiveIntervals in #70549
---
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp   | 11 ++++-
 llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp  | 45 +++++--------------
 .../CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll   | 38 +++-------------
 .../CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll   | 38 +++-------------
 .../RISCV/rvv/vleff-vlseg2ff-output.ll        |  7 +--
 5 files changed, 30 insertions(+), 109 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index dc3ad5ac5908..e73a3af92af6 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -3478,8 +3478,15 @@ static bool usesAllOnesMask(SDNode *N, unsigned MaskOpIdx) {
 }
 
 static bool isImplicitDef(SDValue V) {
-  return V.isMachineOpcode() &&
-         V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
+  if (!V.isMachineOpcode())
+    return false;
+  if (V.getMachineOpcode() == TargetOpcode::REG_SEQUENCE) {
+    for (unsigned I = 1; I < V.getNumOperands(); I += 2)
+      if (!isImplicitDef(V.getOperand(I)))
+        return false;
+    return true;
+  }
+  return V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
 }
 
 // Optimize masked RVV pseudo instructions with a known all-ones mask to their
diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 06456f97f5eb..5f8b610e5233 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -173,8 +173,7 @@ static bool isMaskRegOp(const MachineInstr &MI) {
 /// Note that this is different from "agnostic" as defined by the vector
 /// specification.  Agnostic requires each lane to either be undisturbed, or
 /// take the value -1; no other value is allowed.
-static bool hasUndefinedMergeOp(const MachineInstr &MI,
-                                const MachineRegisterInfo &MRI) {
+static bool hasUndefinedMergeOp(const MachineInstr &MI) {
 
   unsigned UseOpIdx;
   if (!MI.isRegTiedToUseOperand(0, &UseOpIdx))
@@ -182,35 +181,10 @@ static bool hasUndefinedMergeOp(const MachineInstr &MI,
     // lanes are undefined.
     return true;
 
-  // If the tied operand is NoReg, an IMPLICIT_DEF, or a REG_SEQEUENCE whose
-  // operands are solely IMPLICIT_DEFS, then the pass through lanes are
-  // undefined.
+  // All undefined passthrus should be $noreg: see
+  // RISCVDAGToDAGISel::doPeepholeNoRegPassThru
   const MachineOperand &UseMO = MI.getOperand(UseOpIdx);
-  if (UseMO.getReg() == RISCV::NoRegister)
-    return true;
-
-  if (UseMO.isUndef())
-    return true;
-  if (UseMO.getReg().isPhysical())
-    return false;
-
-  MachineInstr *UseMI = MRI.getUniqueVRegDef(UseMO.getReg());
-  assert(UseMI);
-  if (UseMI->isImplicitDef())
-    return true;
-
-  if (UseMI->isRegSequence()) {
-    for (unsigned i = 1, e = UseMI->getNumOperands(); i < e; i += 2) {
-      MachineInstr *SourceMI =
-          MRI.getUniqueVRegDef(UseMI->getOperand(i).getReg());
-      assert(SourceMI);
-      if (!SourceMI->isImplicitDef())
-        return false;
-    }
-    return true;
-  }
-
-  return false;
+  return UseMO.getReg() == RISCV::NoRegister || UseMO.isUndef();
 }
 
 /// Which subfields of VL or VTYPE have values we need to preserve?
@@ -429,7 +403,7 @@ DemandedFields getDemanded(const MachineInstr &MI,
     // this for any tail agnostic operation, but we can't as TA requires
     // tail lanes to either be the original value or -1.  We are writing
     // unknown bits to the lanes here.
-    if (hasUndefinedMergeOp(MI, *MRI)) {
+    if (hasUndefinedMergeOp(MI)) {
       if (isFloatScalarMoveOrScalarSplatInstr(MI) && !ST->hasVInstructionsF64())
         Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
       else
@@ -913,7 +887,7 @@ static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags,
 
   bool TailAgnostic = true;
   bool MaskAgnostic = true;
-  if (!hasUndefinedMergeOp(MI, *MRI)) {
+  if (!hasUndefinedMergeOp(MI)) {
     // Start with undisturbed.
     TailAgnostic = false;
     MaskAgnostic = false;
@@ -1109,7 +1083,7 @@ bool RISCVInsertVSETVLI::needVSETVLI(const MachineInstr &MI,
   // * The LMUL1 restriction is for machines whose latency may depend on VL.
   // * As above, this is only legal for tail "undefined" not "agnostic".
   if (isVSlideInstr(MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
-      isLMUL1OrSmaller(CurInfo.getVLMUL()) && hasUndefinedMergeOp(MI, *MRI)) {
+      isLMUL1OrSmaller(CurInfo.getVLMUL()) && hasUndefinedMergeOp(MI)) {
     Used.VLAny = false;
     Used.VLZeroness = true;
     Used.LMUL = false;
@@ -1121,8 +1095,9 @@ bool RISCVInsertVSETVLI::needVSETVLI(const MachineInstr &MI,
   // immediate form of vmv.s.x, and thus frequently use vmv.v.i in it's place.
   // Since a splat is non-constant time in LMUL, we do need to be careful to not
   // increase the number of active vector registers (unlike for vmv.s.x.)
-  if (isScalarSplatInstr(MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
-      isLMUL1OrSmaller(CurInfo.getVLMUL()) && hasUndefinedMergeOp(MI, *MRI)) {
+  if (isScalarSplatInstr(MI) && Require.hasAVLImm() &&
+      Require.getAVLImm() == 1 && isLMUL1OrSmaller(CurInfo.getVLMUL()) &&
+      hasUndefinedMergeOp(MI)) {
     Used.LMUL = false;
     Used.SEWLMULRatio = false;
     Used.VLAny = false;
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
index 407c782d3377..e7913fc53df0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
@@ -13,13 +13,8 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    # implicit-def: $v8
-; SPILL-O0-NEXT:    # implicit-def: $v9
-; SPILL-O0-NEXT:    # implicit-def: $v10
-; SPILL-O0-NEXT:    # implicit-def: $v9
-; SPILL-O0-NEXT:    # kill: def $v8 killed $v8 def $v8_v9
-; SPILL-O0-NEXT:    vmv1r.v v9, v10
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; SPILL-O0-NEXT:    # implicit-def: $v8_v9
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -95,13 +90,8 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    # implicit-def: $v8
-; SPILL-O0-NEXT:    # implicit-def: $v9
-; SPILL-O0-NEXT:    # implicit-def: $v10
-; SPILL-O0-NEXT:    # implicit-def: $v9
-; SPILL-O0-NEXT:    # kill: def $v8 killed $v8 def $v8_v9
-; SPILL-O0-NEXT:    vmv1r.v v9, v10
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; SPILL-O0-NEXT:    # implicit-def: $v8_v9
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -177,13 +167,8 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    # implicit-def: $v8m2
-; SPILL-O0-NEXT:    # implicit-def: $v10m2
-; SPILL-O0-NEXT:    # implicit-def: $v12m2
-; SPILL-O0-NEXT:    # implicit-def: $v10m2
-; SPILL-O0-NEXT:    # kill: def $v8m2 killed $v8m2 def $v8m2_v10m2
-; SPILL-O0-NEXT:    vmv2r.v v10, v12
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -262,13 +247,8 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 2
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    # implicit-def: $v8m4
-; SPILL-O0-NEXT:    # implicit-def: $v12m4
-; SPILL-O0-NEXT:    # implicit-def: $v16m4
-; SPILL-O0-NEXT:    # implicit-def: $v12m4
-; SPILL-O0-NEXT:    # kill: def $v8m4 killed $v8m4 def $v8m4_v12m4
-; SPILL-O0-NEXT:    vmv4r.v v12, v16
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; SPILL-O0-NEXT:    # implicit-def: $v8m4_v12m4
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv4r.v v8, v12
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -347,16 +327,8 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    # implicit-def: $v8m2
-; SPILL-O0-NEXT:    # implicit-def: $v10m2
-; SPILL-O0-NEXT:    # implicit-def: $v16m2
-; SPILL-O0-NEXT:    # implicit-def: $v10m2
-; SPILL-O0-NEXT:    # implicit-def: $v14m2
-; SPILL-O0-NEXT:    # implicit-def: $v10m2
-; SPILL-O0-NEXT:    # kill: def $v8m2 killed $v8m2 def $v8m2_v10m2_v12m2
-; SPILL-O0-NEXT:    vmv2r.v v10, v16
-; SPILL-O0-NEXT:    vmv2r.v v12, v14
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2_v12m2
 ; SPILL-O0-NEXT:    vlseg3e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
index 1c1544b4efa0..dd575b3fceb5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
@@ -13,13 +13,8 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    # implicit-def: $v8
-; SPILL-O0-NEXT:    # implicit-def: $v9
-; SPILL-O0-NEXT:    # implicit-def: $v10
-; SPILL-O0-NEXT:    # implicit-def: $v9
-; SPILL-O0-NEXT:    # kill: def $v8 killed $v8 def $v8_v9
-; SPILL-O0-NEXT:    vmv1r.v v9, v10
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
+; SPILL-O0-NEXT:    # implicit-def: $v8_v9
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -95,13 +90,8 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    # implicit-def: $v8
-; SPILL-O0-NEXT:    # implicit-def: $v9
-; SPILL-O0-NEXT:    # implicit-def: $v10
-; SPILL-O0-NEXT:    # implicit-def: $v9
-; SPILL-O0-NEXT:    # kill: def $v8 killed $v8 def $v8_v9
-; SPILL-O0-NEXT:    vmv1r.v v9, v10
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; SPILL-O0-NEXT:    # implicit-def: $v8_v9
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -177,13 +167,8 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    # implicit-def: $v8m2
-; SPILL-O0-NEXT:    # implicit-def: $v10m2
-; SPILL-O0-NEXT:    # implicit-def: $v12m2
-; SPILL-O0-NEXT:    # implicit-def: $v10m2
-; SPILL-O0-NEXT:    # kill: def $v8m2 killed $v8m2 def $v8m2_v10m2
-; SPILL-O0-NEXT:    vmv2r.v v10, v12
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -262,13 +247,8 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 2
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    # implicit-def: $v8m4
-; SPILL-O0-NEXT:    # implicit-def: $v12m4
-; SPILL-O0-NEXT:    # implicit-def: $v16m4
-; SPILL-O0-NEXT:    # implicit-def: $v12m4
-; SPILL-O0-NEXT:    # kill: def $v8m4 killed $v8m4 def $v8m4_v12m4
-; SPILL-O0-NEXT:    vmv4r.v v12, v16
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
+; SPILL-O0-NEXT:    # implicit-def: $v8m4_v12m4
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv4r.v v8, v12
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -347,16 +327,8 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    # implicit-def: $v8m2
-; SPILL-O0-NEXT:    # implicit-def: $v10m2
-; SPILL-O0-NEXT:    # implicit-def: $v16m2
-; SPILL-O0-NEXT:    # implicit-def: $v10m2
-; SPILL-O0-NEXT:    # implicit-def: $v14m2
-; SPILL-O0-NEXT:    # implicit-def: $v10m2
-; SPILL-O0-NEXT:    # kill: def $v8m2 killed $v8m2 def $v8m2_v10m2_v12m2
-; SPILL-O0-NEXT:    vmv2r.v v10, v16
-; SPILL-O0-NEXT:    vmv2r.v v12, v14
 ; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2_v12m2
 ; SPILL-O0-NEXT:    vlseg3e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll b/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll
index 15cb42bacf17..390647fd9e6c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll
@@ -66,12 +66,7 @@ define i64 @test_vlseg2ff_nxv8i8(ptr %base, i64 %vl, ptr %outvl) {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
-  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[DEF2:%[0-9]+]]:vr = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[DEF3:%[0-9]+]]:vr = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vrn2m1 = REG_SEQUENCE [[DEF]], %subreg.sub_vrm1_0, [[DEF2]], %subreg.sub_vrm1_1
-  ; CHECK-NEXT:   [[PseudoVLSEG2E8FF_V_M1_:%[0-9]+]]:vrn2m1, [[PseudoVLSEG2E8FF_V_M1_1:%[0-9]+]]:gpr = PseudoVLSEG2E8FF_V_M1 [[REG_SEQUENCE]], [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */, implicit-def dead $vl :: (load unknown-size from %ir.base, align 1)
+  ; CHECK-NEXT:   [[PseudoVLSEG2E8FF_V_M1_:%[0-9]+]]:vrn2m1, [[PseudoVLSEG2E8FF_V_M1_1:%[0-9]+]]:gpr = PseudoVLSEG2E8FF_V_M1 $noreg, [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */, implicit-def dead $vl :: (load unknown-size from %ir.base, align 1)
   ; CHECK-NEXT:   $x10 = COPY [[PseudoVLSEG2E8FF_V_M1_1]]
   ; CHECK-NEXT:   PseudoRET implicit $x10
 entry:
-- 
GitLab


From 81d304566b3b22781ca42303ec534bdc0ac5418c Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Wed, 8 May 2024 15:54:14 +0800
Subject: [PATCH 0145/1206] [RISCV] Remove unused arg in getDemanded in
 RISCVInsertVSETVLI. NFC

---
 llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 5f8b610e5233..7a8ff84995ea 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -337,9 +337,7 @@ static bool areCompatibleVTYPEs(uint64_t CurVType, uint64_t NewVType,
 }
 
 /// Return the fields and properties demanded by the provided instruction.
-DemandedFields getDemanded(const MachineInstr &MI,
-                           const MachineRegisterInfo *MRI,
-                           const RISCVSubtarget *ST) {
+DemandedFields getDemanded(const MachineInstr &MI, const RISCVSubtarget *ST) {
   // Warning: This function has to work on both the lowered (i.e. post
   // emitVSETVLIs) and pre-lowering forms.  The main implication of this is
   // that it can't use the value of a SEW, VL, or Policy operand as they might
@@ -1072,7 +1070,7 @@ bool RISCVInsertVSETVLI::needVSETVLI(const MachineInstr &MI,
   if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
     return true;
 
-  DemandedFields Used = getDemanded(MI, MRI, ST);
+  DemandedFields Used = getDemanded(MI, ST);
 
   // A slidedown/slideup with an *undefined* merge op can freely clobber
   // elements not copied from the source vector (e.g. masked off, tail, or
@@ -1163,7 +1161,7 @@ void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
   if (!Info.isValid() || Info.isUnknown())
     Info = NewInfo;
 
-  DemandedFields Demanded = getDemanded(MI, MRI, ST);
+  DemandedFields Demanded = getDemanded(MI, ST);
   const VSETVLIInfo IncomingInfo = adjustIncoming(PrevInfo, NewInfo, Demanded);
 
   // If MI only demands that VL has the same zeroness, we only need to set the
@@ -1572,7 +1570,7 @@ bool RISCVCoalesceVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) {
   for (MachineInstr &MI : make_range(MBB.rbegin(), MBB.rend())) {
 
     if (!isVectorConfigInstr(MI)) {
-      Used.doUnion(getDemanded(MI, MRI, ST));
+      Used.doUnion(getDemanded(MI, ST));
       if (MI.isCall() || MI.isInlineAsm() ||
           MI.modifiesRegister(RISCV::VL, /*TRI=*/nullptr) ||
           MI.modifiesRegister(RISCV::VTYPE, /*TRI=*/nullptr))
@@ -1645,7 +1643,7 @@ bool RISCVCoalesceVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) {
       }
     }
     NextMI = &MI;
-    Used = getDemanded(MI, MRI, ST);
+    Used = getDemanded(MI, ST);
   }
 
   NumCoalescedVSETVL += ToDelete.size();
-- 
GitLab


From bafbe39778a972d0f2869980de22fb00c03a6a35 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 8 May 2024 10:22:02 +0200
Subject: [PATCH 0146/1206] [NVPTX] Add support for atomic add for bf16 type
 (#89586)

atom.add.noftz.bf16 is supported since SM 9.0 and PTX 7.8
---
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp |   3 +
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td    |  11 +-
 llvm/test/CodeGen/NVPTX/atomics-sm90.ll     | 151 ++++++++++++++++++++
 3 files changed, 164 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/atomics-sm90.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 44b61a937d64..b03803f52b78 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -6125,6 +6125,9 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
       if (Ty->isHalfTy() && STI.getSmVersion() >= 70 &&
           STI.getPTXVersion() >= 63)
         return AtomicExpansionKind::None;
+      if (Ty->isBFloatTy() && STI.getSmVersion() >= 90 &&
+          STI.getPTXVersion() >= 78)
+        return AtomicExpansionKind::None;
       if (Ty->isFloatTy())
         return AtomicExpansionKind::None;
       if (Ty->isDoubleTy() && STI.hasAtomAddF64())
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 5f6e28283c5d..440af085cb8e 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1545,7 +1545,7 @@ multiclass F_ATOMIC_2_imp<ValueType ptrT, NVPTXRegClass ptrclass,
   def imm : NVPTXInst<(outs regclass:$dst), (ins ptrclass:$addr, IMMType:$b),
     !strconcat("atom", SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b;", ""),
     [(set (regT regclass:$dst), (IntOp (ptrT ptrclass:$addr), IMM:$b))]>,
-  Requires<!if(!eq(TypeStr, ".f16"), [Predicate<"false">], Pred)>;
+  Requires<!if(!or(!eq(TypeStr, ".f16"), !eq(TypeStr, ".bf16")), [Predicate<"false">], Pred)>;
 }
 multiclass F_ATOMIC_2<ValueType regT, NVPTXRegClass regclass, string SpaceStr, string TypeStr,
   string OpcStr, PatFrag IntOp, Operand IMMType, SDNode IMM,
@@ -1662,6 +1662,13 @@ defm INT_PTX_ATOM_ADD_S_F16 : F_ATOMIC_2<f16, Int16Regs, ".shared", ".f16", ".ad
 defm INT_PTX_ATOM_ADD_GEN_F16 : F_ATOMIC_2<f16, Int16Regs, "", ".f16", ".add.noftz",
   atomic_load_add_gen, f16imm, fpimm, [hasSM<70>, hasPTX<63>]>;
 
+defm INT_PTX_ATOM_ADD_G_BF16 : F_ATOMIC_2<bf16, Int16Regs, ".global", ".bf16", ".add.noftz",
+  atomic_load_add_g, bf16imm, fpimm, [hasSM<90>, hasPTX<78>]>;
+defm INT_PTX_ATOM_ADD_S_BF16 : F_ATOMIC_2<bf16, Int16Regs, ".shared", ".bf16", ".add.noftz",
+  atomic_load_add_s, bf16imm, fpimm, [hasSM<90>, hasPTX<78>]>;
+defm INT_PTX_ATOM_ADD_GEN_BF16 : F_ATOMIC_2<bf16, Int16Regs, "", ".bf16", ".add.noftz",
+  atomic_load_add_gen, bf16imm, fpimm, [hasSM<90>, hasPTX<78>]>;
+
 defm INT_PTX_ATOM_ADD_G_F32 : F_ATOMIC_2<f32, Float32Regs, ".global", ".f32", ".add",
   atomic_load_add_g, f32imm, fpimm>;
 defm INT_PTX_ATOM_ADD_S_F32 : F_ATOMIC_2<f32, Float32Regs, ".shared", ".f32", ".add",
@@ -2174,6 +2181,8 @@ multiclass ATOM2_add_impl<string OpStr> {
    defm _s32  : ATOM2S_impl<OpStr, "i", "s32", i32, Int32Regs, i32imm, imm, i32, []>;
    defm _u32  : ATOM2S_impl<OpStr, "i", "u32", i32, Int32Regs, i32imm, imm, i32, []>;
    defm _u64  : ATOM2S_impl<OpStr, "i", "u64", i64, Int64Regs, i64imm, imm, i64, []>;
+   defm _bf16  : ATOM2S_impl<OpStr, "f", "bf16", bf16, Int16Regs, bf16imm, fpimm, bf16,
+                            [hasSM<90>, hasPTX<78>]>;
    defm _f16  : ATOM2S_impl<OpStr, "f", "f16", f16, Int16Regs, f16imm, fpimm, f16,
                             [hasSM<70>, hasPTX<63>]>;
    defm _f32  : ATOM2S_impl<OpStr, "f", "f32", f32, Float32Regs, f32imm, fpimm, f32,
diff --git a/llvm/test/CodeGen/NVPTX/atomics-sm90.ll b/llvm/test/CodeGen/NVPTX/atomics-sm90.ll
new file mode 100644
index 000000000000..9301ea44c693
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/atomics-sm90.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -march=nvptx -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s --check-prefixes=CHECK
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s --check-prefixes=CHECK64
+; RUN: llc < %s -march=nvptx -mcpu=sm_86 -mattr=+ptx71 | FileCheck %s --check-prefixes=CHECKPTX71
+; RUN: %if ptxas && !ptxas-12.0 %{ llc < %s -march=nvptx -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas && !ptxas-12.0 %{ llc < %s -march=nvptx -mcpu=sm_86 -mattr=+ptx71 | %ptxas-verify -arch=sm_86 %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+define void @test(ptr %dp0, ptr addrspace(1) %dp1, ptr addrspace(3) %dp3, bfloat %val) {
+; CHECK-LABEL: test(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<7>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [test_param_0];
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_param_3];
+; CHECK-NEXT:    atom.add.noftz.bf16 %rs2, [%r1], %rs1;
+; CHECK-NEXT:    ld.param.u32 %r2, [test_param_1];
+; CHECK-NEXT:    mov.b16 %rs3, 0x3F80;
+; CHECK-NEXT:    atom.add.noftz.bf16 %rs4, [%r1], %rs3;
+; CHECK-NEXT:    ld.param.u32 %r3, [test_param_2];
+; CHECK-NEXT:    atom.global.add.noftz.bf16 %rs5, [%r2], %rs1;
+; CHECK-NEXT:    atom.shared.add.noftz.bf16 %rs6, [%r3], %rs1;
+; CHECK-NEXT:    ret;
+;
+; CHECK64-LABEL: test(
+; CHECK64:       {
+; CHECK64-NEXT:    .reg .b16 %rs<7>;
+; CHECK64-NEXT:    .reg .b64 %rd<4>;
+; CHECK64-EMPTY:
+; CHECK64-NEXT:  // %bb.0:
+; CHECK64-NEXT:    ld.param.u64 %rd1, [test_param_0];
+; CHECK64-NEXT:    ld.param.b16 %rs1, [test_param_3];
+; CHECK64-NEXT:    atom.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; CHECK64-NEXT:    ld.param.u64 %rd2, [test_param_1];
+; CHECK64-NEXT:    mov.b16 %rs3, 0x3F80;
+; CHECK64-NEXT:    atom.add.noftz.bf16 %rs4, [%rd1], %rs3;
+; CHECK64-NEXT:    ld.param.u64 %rd3, [test_param_2];
+; CHECK64-NEXT:    atom.global.add.noftz.bf16 %rs5, [%rd2], %rs1;
+; CHECK64-NEXT:    atom.shared.add.noftz.bf16 %rs6, [%rd3], %rs1;
+; CHECK64-NEXT:    ret;
+;
+; CHECKPTX71-LABEL: test(
+; CHECKPTX71:       {
+; CHECKPTX71-NEXT:    .reg .pred %p<5>;
+; CHECKPTX71-NEXT:    .reg .b16 %rs<18>;
+; CHECKPTX71-NEXT:    .reg .b32 %r<58>;
+; CHECKPTX71-NEXT:    .reg .f32 %f<12>;
+; CHECKPTX71-EMPTY:
+; CHECKPTX71-NEXT:  // %bb.0:
+; CHECKPTX71-NEXT:    ld.param.b16 %rs1, [test_param_3];
+; CHECKPTX71-NEXT:    ld.param.u32 %r23, [test_param_2];
+; CHECKPTX71-NEXT:    ld.param.u32 %r22, [test_param_1];
+; CHECKPTX71-NEXT:    ld.param.u32 %r24, [test_param_0];
+; CHECKPTX71-NEXT:    and.b32 %r1, %r24, -4;
+; CHECKPTX71-NEXT:    and.b32 %r25, %r24, 3;
+; CHECKPTX71-NEXT:    shl.b32 %r2, %r25, 3;
+; CHECKPTX71-NEXT:    mov.b32 %r26, 65535;
+; CHECKPTX71-NEXT:    shl.b32 %r27, %r26, %r2;
+; CHECKPTX71-NEXT:    not.b32 %r3, %r27;
+; CHECKPTX71-NEXT:    ld.u32 %r54, [%r1];
+; CHECKPTX71-NEXT:    cvt.f32.bf16 %f2, %rs1;
+; CHECKPTX71-NEXT:  $L__BB0_1: // %atomicrmw.start
+; CHECKPTX71-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECKPTX71-NEXT:    shr.u32 %r28, %r54, %r2;
+; CHECKPTX71-NEXT:    cvt.u16.u32 %rs2, %r28;
+; CHECKPTX71-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; CHECKPTX71-NEXT:    add.rn.f32 %f3, %f1, %f2;
+; CHECKPTX71-NEXT:    cvt.rn.bf16.f32 %rs4, %f3;
+; CHECKPTX71-NEXT:    cvt.u32.u16 %r29, %rs4;
+; CHECKPTX71-NEXT:    shl.b32 %r30, %r29, %r2;
+; CHECKPTX71-NEXT:    and.b32 %r31, %r54, %r3;
+; CHECKPTX71-NEXT:    or.b32 %r32, %r31, %r30;
+; CHECKPTX71-NEXT:    atom.cas.b32 %r6, [%r1], %r54, %r32;
+; CHECKPTX71-NEXT:    setp.ne.s32 %p1, %r6, %r54;
+; CHECKPTX71-NEXT:    mov.u32 %r54, %r6;
+; CHECKPTX71-NEXT:    @%p1 bra $L__BB0_1;
+; CHECKPTX71-NEXT:  // %bb.2: // %atomicrmw.end
+; CHECKPTX71-NEXT:    ld.u32 %r55, [%r1];
+; CHECKPTX71-NEXT:  $L__BB0_3: // %atomicrmw.start9
+; CHECKPTX71-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECKPTX71-NEXT:    shr.u32 %r33, %r55, %r2;
+; CHECKPTX71-NEXT:    cvt.u16.u32 %rs6, %r33;
+; CHECKPTX71-NEXT:    cvt.f32.bf16 %f4, %rs6;
+; CHECKPTX71-NEXT:    add.rn.f32 %f5, %f4, 0f3F800000;
+; CHECKPTX71-NEXT:    cvt.rn.bf16.f32 %rs8, %f5;
+; CHECKPTX71-NEXT:    cvt.u32.u16 %r34, %rs8;
+; CHECKPTX71-NEXT:    shl.b32 %r35, %r34, %r2;
+; CHECKPTX71-NEXT:    and.b32 %r36, %r55, %r3;
+; CHECKPTX71-NEXT:    or.b32 %r37, %r36, %r35;
+; CHECKPTX71-NEXT:    atom.cas.b32 %r9, [%r1], %r55, %r37;
+; CHECKPTX71-NEXT:    setp.ne.s32 %p2, %r9, %r55;
+; CHECKPTX71-NEXT:    mov.u32 %r55, %r9;
+; CHECKPTX71-NEXT:    @%p2 bra $L__BB0_3;
+; CHECKPTX71-NEXT:  // %bb.4: // %atomicrmw.end8
+; CHECKPTX71-NEXT:    and.b32 %r10, %r22, -4;
+; CHECKPTX71-NEXT:    shl.b32 %r38, %r22, 3;
+; CHECKPTX71-NEXT:    and.b32 %r11, %r38, 24;
+; CHECKPTX71-NEXT:    shl.b32 %r40, %r26, %r11;
+; CHECKPTX71-NEXT:    not.b32 %r12, %r40;
+; CHECKPTX71-NEXT:    ld.global.u32 %r56, [%r10];
+; CHECKPTX71-NEXT:  $L__BB0_5: // %atomicrmw.start27
+; CHECKPTX71-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECKPTX71-NEXT:    shr.u32 %r41, %r56, %r11;
+; CHECKPTX71-NEXT:    cvt.u16.u32 %rs10, %r41;
+; CHECKPTX71-NEXT:    cvt.f32.bf16 %f6, %rs10;
+; CHECKPTX71-NEXT:    add.rn.f32 %f8, %f6, %f2;
+; CHECKPTX71-NEXT:    cvt.rn.bf16.f32 %rs12, %f8;
+; CHECKPTX71-NEXT:    cvt.u32.u16 %r42, %rs12;
+; CHECKPTX71-NEXT:    shl.b32 %r43, %r42, %r11;
+; CHECKPTX71-NEXT:    and.b32 %r44, %r56, %r12;
+; CHECKPTX71-NEXT:    or.b32 %r45, %r44, %r43;
+; CHECKPTX71-NEXT:    atom.global.cas.b32 %r15, [%r10], %r56, %r45;
+; CHECKPTX71-NEXT:    setp.ne.s32 %p3, %r15, %r56;
+; CHECKPTX71-NEXT:    mov.u32 %r56, %r15;
+; CHECKPTX71-NEXT:    @%p3 bra $L__BB0_5;
+; CHECKPTX71-NEXT:  // %bb.6: // %atomicrmw.end26
+; CHECKPTX71-NEXT:    and.b32 %r16, %r23, -4;
+; CHECKPTX71-NEXT:    shl.b32 %r46, %r23, 3;
+; CHECKPTX71-NEXT:    and.b32 %r17, %r46, 24;
+; CHECKPTX71-NEXT:    shl.b32 %r48, %r26, %r17;
+; CHECKPTX71-NEXT:    not.b32 %r18, %r48;
+; CHECKPTX71-NEXT:    ld.shared.u32 %r57, [%r16];
+; CHECKPTX71-NEXT:  $L__BB0_7: // %atomicrmw.start45
+; CHECKPTX71-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECKPTX71-NEXT:    shr.u32 %r49, %r57, %r17;
+; CHECKPTX71-NEXT:    cvt.u16.u32 %rs14, %r49;
+; CHECKPTX71-NEXT:    cvt.f32.bf16 %f9, %rs14;
+; CHECKPTX71-NEXT:    add.rn.f32 %f11, %f9, %f2;
+; CHECKPTX71-NEXT:    cvt.rn.bf16.f32 %rs16, %f11;
+; CHECKPTX71-NEXT:    cvt.u32.u16 %r50, %rs16;
+; CHECKPTX71-NEXT:    shl.b32 %r51, %r50, %r17;
+; CHECKPTX71-NEXT:    and.b32 %r52, %r57, %r18;
+; CHECKPTX71-NEXT:    or.b32 %r53, %r52, %r51;
+; CHECKPTX71-NEXT:    atom.shared.cas.b32 %r21, [%r16], %r57, %r53;
+; CHECKPTX71-NEXT:    setp.ne.s32 %p4, %r21, %r57;
+; CHECKPTX71-NEXT:    mov.u32 %r57, %r21;
+; CHECKPTX71-NEXT:    @%p4 bra $L__BB0_7;
+; CHECKPTX71-NEXT:  // %bb.8: // %atomicrmw.end44
+; CHECKPTX71-NEXT:    ret;
+  %r1 = atomicrmw fadd ptr %dp0, bfloat %val seq_cst
+  %r2 = atomicrmw fadd ptr %dp0, bfloat 1.0 seq_cst
+  %r3 = atomicrmw fadd ptr addrspace(1) %dp1, bfloat %val seq_cst
+  %r4 = atomicrmw fadd ptr addrspace(3) %dp3, bfloat %val seq_cst
+  ret void
+}
+
+attributes #1 = { argmemonly nounwind }
-- 
GitLab


From 3aba4b5b4fe1634fc6f9919f987ddf1bb5a57813 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Wed, 8 May 2024 10:34:55 +0200
Subject: [PATCH 0147/1206] [libc++][NFC] Refactor __is_transparent to be a
 variable template (#90865)

---
 libcxx/include/__functional/is_transparent.h |  5 +--
 libcxx/include/map                           | 40 ++++++++++----------
 libcxx/include/set                           | 40 ++++++++++----------
 libcxx/include/unordered_map                 | 32 ++++++----------
 libcxx/include/unordered_set                 | 36 ++++++------------
 5 files changed, 66 insertions(+), 87 deletions(-)

diff --git a/libcxx/include/__functional/is_transparent.h b/libcxx/include/__functional/is_transparent.h
index 13fc94f71c6b..b2d62f2e3ead 100644
--- a/libcxx/include/__functional/is_transparent.h
+++ b/libcxx/include/__functional/is_transparent.h
@@ -11,7 +11,6 @@
 #define _LIBCPP___FUNCTIONAL_IS_TRANSPARENT
 
 #include <__config>
-#include <__type_traits/integral_constant.h>
 #include <__type_traits/void_t.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -23,10 +22,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 #if _LIBCPP_STD_VER >= 14
 
 template <class _Tp, class, class = void>
-struct __is_transparent : false_type {};
+inline const bool __is_transparent_v = false;
 
 template <class _Tp, class _Up>
-struct __is_transparent<_Tp, _Up, __void_t<typename _Tp::is_transparent> > : true_type {};
+inline const bool __is_transparent_v<_Tp, _Up, __void_t<typename _Tp::is_transparent> > = true;
 
 #endif
 
diff --git a/libcxx/include/map b/libcxx/include/map
index 2276cc043709..1d1c062a0267 100644
--- a/libcxx/include/map
+++ b/libcxx/include/map
@@ -1367,11 +1367,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __tree_.find(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __tree_.find(__k);
   }
@@ -1379,7 +1379,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_unique(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __tree_.__count_multi(__k);
   }
@@ -1387,7 +1387,7 @@ public:
 
 #if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
@@ -1396,12 +1396,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
     return __tree_.lower_bound(__k);
   }
 
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
     return __tree_.lower_bound(__k);
   }
@@ -1410,11 +1410,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
     return __tree_.upper_bound(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
     return __tree_.upper_bound(__k);
   }
@@ -1427,11 +1427,11 @@ public:
     return __tree_.__equal_range_unique(__k);
   }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __tree_.__equal_range_multi(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __tree_.__equal_range_multi(__k);
   }
@@ -1959,11 +1959,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __tree_.find(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __tree_.find(__k);
   }
@@ -1971,7 +1971,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_multi(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __tree_.__count_multi(__k);
   }
@@ -1979,7 +1979,7 @@ public:
 
 #if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
@@ -1988,12 +1988,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
     return __tree_.lower_bound(__k);
   }
 
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
     return __tree_.lower_bound(__k);
   }
@@ -2002,11 +2002,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
     return __tree_.upper_bound(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
     return __tree_.upper_bound(__k);
   }
@@ -2019,11 +2019,11 @@ public:
     return __tree_.__equal_range_multi(__k);
   }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __tree_.__equal_range_multi(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __tree_.__equal_range_multi(__k);
   }
diff --git a/libcxx/include/set b/libcxx/include/set
index 763c26cea01f..d9377ee6c332 100644
--- a/libcxx/include/set
+++ b/libcxx/include/set
@@ -825,11 +825,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __tree_.find(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __tree_.find(__k);
   }
@@ -837,7 +837,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_unique(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __tree_.__count_multi(__k);
   }
@@ -845,7 +845,7 @@ public:
 
 #if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
@@ -854,12 +854,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
     return __tree_.lower_bound(__k);
   }
 
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
     return __tree_.lower_bound(__k);
   }
@@ -868,11 +868,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
     return __tree_.upper_bound(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
     return __tree_.upper_bound(__k);
   }
@@ -885,11 +885,11 @@ public:
     return __tree_.__equal_range_unique(__k);
   }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __tree_.__equal_range_multi(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __tree_.__equal_range_multi(__k);
   }
@@ -1283,11 +1283,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __tree_.find(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __tree_.find(__k);
   }
@@ -1295,7 +1295,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_multi(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __tree_.__count_multi(__k);
   }
@@ -1303,7 +1303,7 @@ public:
 
 #if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
@@ -1312,12 +1312,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
     return __tree_.lower_bound(__k);
   }
 
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
     return __tree_.lower_bound(__k);
   }
@@ -1326,11 +1326,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
     return __tree_.upper_bound(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
     return __tree_.upper_bound(__k);
   }
@@ -1343,11 +1343,11 @@ public:
     return __tree_.__equal_range_multi(__k);
   }
 #if _LIBCPP_STD_VER >= 14
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __tree_.__equal_range_multi(__k);
   }
-  template <typename _K2, enable_if_t<__is_transparent<_Compare, _K2>::value, int> = 0>
+  template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __tree_.__equal_range_multi(__k);
   }
diff --git a/libcxx/include/unordered_map b/libcxx/include/unordered_map
index 8c21d703a5c0..c838cd96b112 100644
--- a/libcxx/include/unordered_map
+++ b/libcxx/include/unordered_map
@@ -1384,13 +1384,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
 #if _LIBCPP_STD_VER >= 20
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __table_.find(__k);
   }
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __table_.find(__k);
   }
@@ -1398,8 +1396,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_unique(__k); }
 #if _LIBCPP_STD_VER >= 20
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __table_.__count_unique(__k);
   }
@@ -1408,8 +1405,7 @@ public:
 #if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
 
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
@@ -1423,12 +1419,12 @@ public:
   }
 #if _LIBCPP_STD_VER >= 20
   template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+            enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __table_.__equal_range_unique(__k);
   }
   template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+            enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __table_.__equal_range_unique(__k);
   }
@@ -2135,13 +2131,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
 #if _LIBCPP_STD_VER >= 20
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __table_.find(__k);
   }
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __table_.find(__k);
   }
@@ -2149,8 +2143,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_multi(__k); }
 #if _LIBCPP_STD_VER >= 20
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __table_.__count_multi(__k);
   }
@@ -2159,8 +2152,7 @@ public:
 #if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
 
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
@@ -2174,12 +2166,12 @@ public:
   }
 #if _LIBCPP_STD_VER >= 20
   template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+            enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __table_.__equal_range_multi(__k);
   }
   template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+            enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __table_.__equal_range_multi(__k);
   }
diff --git a/libcxx/include/unordered_set b/libcxx/include/unordered_set
index 69fe6b768788..5de1458beb1e 100644
--- a/libcxx/include/unordered_set
+++ b/libcxx/include/unordered_set
@@ -839,13 +839,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
 #if _LIBCPP_STD_VER >= 20
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __table_.find(__k);
   }
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __table_.find(__k);
   }
@@ -853,8 +851,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_unique(__k); }
 #if _LIBCPP_STD_VER >= 20
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __table_.__count_unique(__k);
   }
@@ -863,8 +860,7 @@ public:
 #if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
 
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
@@ -877,13 +873,11 @@ public:
     return __table_.__equal_range_unique(__k);
   }
 #if _LIBCPP_STD_VER >= 20
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __table_.__equal_range_unique(__k);
   }
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __table_.__equal_range_unique(__k);
   }
@@ -1442,13 +1436,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
 #if _LIBCPP_STD_VER >= 20
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
     return __table_.find(__k);
   }
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
     return __table_.find(__k);
   }
@@ -1456,8 +1448,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_multi(__k); }
 #if _LIBCPP_STD_VER >= 20
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
     return __table_.__count_multi(__k);
   }
@@ -1466,8 +1457,7 @@ public:
 #if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
 
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
     return find(__k) != end();
   }
@@ -1480,13 +1470,11 @@ public:
     return __table_.__equal_range_multi(__k);
   }
 #if _LIBCPP_STD_VER >= 20
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
     return __table_.__equal_range_multi(__k);
   }
-  template <class _K2,
-            enable_if_t<__is_transparent<hasher, _K2>::value && __is_transparent<key_equal, _K2>::value>* = nullptr>
+  template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
     return __table_.__equal_range_multi(__k);
   }
-- 
GitLab


From 9ef28cf88ca6e45c3ecb75c649463f8797db68d2 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <r@artagnon.com>
Date: Wed, 8 May 2024 09:43:49 +0100
Subject: [PATCH 0148/1206] VectorCombine: add test for crash #88796 (#91200)

---
 llvm/test/Transforms/VectorCombine/pr88796.ll | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 llvm/test/Transforms/VectorCombine/pr88796.ll

diff --git a/llvm/test/Transforms/VectorCombine/pr88796.ll b/llvm/test/Transforms/VectorCombine/pr88796.ll
new file mode 100644
index 000000000000..d5cd52e11d39
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/pr88796.ll
@@ -0,0 +1,11 @@
+; REQUIRES: asserts
+; RUN: not --crash opt -passes=vector-combine -disable-output %s
+
+define i32 @test() {
+entry:
+  %0 = tail call i16 @llvm.vector.reduce.and.nxv8i16(<vscale x 8 x i16> trunc (<vscale x 8 x i32> shufflevector (<vscale x 8 x i32> insertelement (<vscale x 8 x i32> poison, i32 268435456, i64 0), <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer) to <vscale x 8 x i16>))
+  ret i32 0
+}
+
+declare i16 @llvm.vector.reduce.and.nxv8i16(<vscale x 8 x i16>)
+
-- 
GitLab


From 57b9c15227ec15a5e2abf4587d7d0ad536cff9e6 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <r@artagnon.com>
Date: Wed, 8 May 2024 09:47:55 +0100
Subject: [PATCH 0149/1206] VectorCombine: fix logical error after m_Trunc
 match (#91201)

The matcher m_Trunc() matches an Operator with a given Opcode, which
could either be an Instruction or ConstExpr.
VectorCombine::foldTruncFromReductions() incorrectly assumes that the
pattern matched is always an Instruction, and attempts a cast. Fix this.

Fixes #88796.
---
 llvm/lib/Transforms/Vectorize/VectorCombine.cpp | 12 ++++++------
 llvm/test/Transforms/VectorCombine/pr88796.ll   |  9 +++++++--
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index bbb70134870a..8573a8adf53b 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -1961,17 +1961,17 @@ bool VectorCombine::foldTruncFromReductions(Instruction &I) {
   if (!match(ReductionSrc, m_OneUse(m_Trunc(m_Value(TruncSrc)))))
     return false;
 
-  auto *Trunc = cast<CastInst>(ReductionSrc);
   auto *TruncSrcTy = cast<VectorType>(TruncSrc->getType());
   auto *ReductionSrcTy = cast<VectorType>(ReductionSrc->getType());
   Type *ResultTy = I.getType();
 
   TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
-  InstructionCost OldCost =
-      TTI.getCastInstrCost(Instruction::Trunc, ReductionSrcTy, TruncSrcTy,
-                           TTI::CastContextHint::None, CostKind, Trunc) +
-      TTI.getArithmeticReductionCost(ReductionOpc, ReductionSrcTy, std::nullopt,
-                                     CostKind);
+  InstructionCost OldCost = TTI.getArithmeticReductionCost(
+      ReductionOpc, ReductionSrcTy, std::nullopt, CostKind);
+  if (auto *Trunc = dyn_cast<CastInst>(ReductionSrc))
+    OldCost +=
+        TTI.getCastInstrCost(Instruction::Trunc, ReductionSrcTy, TruncSrcTy,
+                             TTI::CastContextHint::None, CostKind, Trunc);
   InstructionCost NewCost =
       TTI.getArithmeticReductionCost(ReductionOpc, TruncSrcTy, std::nullopt,
                                      CostKind) +
diff --git a/llvm/test/Transforms/VectorCombine/pr88796.ll b/llvm/test/Transforms/VectorCombine/pr88796.ll
index d5cd52e11d39..4f26f5dcbb92 100644
--- a/llvm/test/Transforms/VectorCombine/pr88796.ll
+++ b/llvm/test/Transforms/VectorCombine/pr88796.ll
@@ -1,7 +1,12 @@
-; REQUIRES: asserts
-; RUN: not --crash opt -passes=vector-combine -disable-output %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=vector-combine -S %s | FileCheck %s
 
 define i32 @test() {
+; CHECK-LABEL: define i32 @test() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call i16 @llvm.vector.reduce.and.nxv8i16(<vscale x 8 x i16> trunc (<vscale x 8 x i32> shufflevector (<vscale x 8 x i32> insertelement (<vscale x 8 x i32> poison, i32 268435456, i64 0), <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer) to <vscale x 8 x i16>))
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   %0 = tail call i16 @llvm.vector.reduce.and.nxv8i16(<vscale x 8 x i16> trunc (<vscale x 8 x i32> shufflevector (<vscale x 8 x i32> insertelement (<vscale x 8 x i32> poison, i32 268435456, i64 0), <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer) to <vscale x 8 x i16>))
   ret i32 0
-- 
GitLab


From 746bf297e2f0f637d2e1c197bf04a32ab04b669a Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan@arm.com>
Date: Wed, 8 May 2024 10:00:03 +0100
Subject: [PATCH 0150/1206] [Flang][OpenMP] Add checks for EXIT from associated
 loops (#91315)

Extend the checker that deals with CYCLE to handle EXIT also. The
difference for EXIT is that it is not allowed to EXIT from the innermost
associated loops while it is OK to CYCLE in the innermost associated
loop. Also add an incrementer on leaving the DO loop for EXIT checks.
---
 flang/lib/Semantics/check-omp-structure.cpp | 47 ++++++++++++++-------
 flang/test/Semantics/OpenMP/do08.f90        | 31 ++++++++++++++
 2 files changed, 63 insertions(+), 15 deletions(-)

diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 70863c5f20e8..2493eb3ed367 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -84,52 +84,69 @@ private:
   parser::CharBlock source_;
 };
 
-class OmpCycleChecker {
+class OmpCycleAndExitChecker {
 public:
-  OmpCycleChecker(SemanticsContext &context, std::int64_t cycleLevel)
-      : context_{context}, cycleLevel_{cycleLevel} {}
+  OmpCycleAndExitChecker(SemanticsContext &context, std::int64_t level)
+      : context_{context}, level_{level} {}
 
   template <typename T> bool Pre(const T &) { return true; }
   template <typename T> void Post(const T &) {}
 
   bool Pre(const parser::DoConstruct &dc) {
-    cycleLevel_--;
+    level_--;
     const auto &constructName{std::get<0>(std::get<0>(dc.t).statement.t)};
     if (constructName) {
       constructNamesAndLevels_.emplace(
-          constructName.value().ToString(), cycleLevel_);
+          constructName.value().ToString(), level_);
     }
     return true;
   }
 
+  void Post(const parser::DoConstruct &dc) { level_++; }
+
   bool Pre(const parser::CycleStmt &cyclestmt) {
     std::map<std::string, std::int64_t>::iterator it;
     bool err{false};
     if (cyclestmt.v) {
       it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
       err = (it != constructNamesAndLevels_.end() && it->second > 0);
-    } else {
-      // If there is no label then the cycle statement is associated with the
-      // closest enclosing DO. Use its level for the checks.
-      err = cycleLevel_ > 0;
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ > 0;
     }
     if (err) {
-      context_.Say(*cycleSource_,
+      context_.Say(*source_,
           "CYCLE statement to non-innermost associated loop of an OpenMP DO "
           "construct"_err_en_US);
     }
     return true;
   }
 
+  bool Pre(const parser::ExitStmt &exitStmt) {
+    std::map<std::string, std::int64_t>::iterator it;
+    bool err{false};
+    if (exitStmt.v) {
+      it = constructNamesAndLevels_.find(exitStmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second >= 0);
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ >= 0;
+    }
+    if (err) {
+      context_.Say(*source_,
+          "EXIT statement terminates associated loop of an OpenMP DO "
+          "construct"_err_en_US);
+    }
+    return true;
+  }
+
   bool Pre(const parser::Statement<parser::ActionStmt> &actionstmt) {
-    cycleSource_ = &actionstmt.source;
+    source_ = &actionstmt.source;
     return true;
   }
 
 private:
   SemanticsContext &context_;
-  const parser::CharBlock *cycleSource_;
-  std::int64_t cycleLevel_;
+  const parser::CharBlock *source_;
+  std::int64_t level_;
   std::map<std::string, std::int64_t> constructNamesAndLevels_;
 };
 
@@ -657,8 +674,8 @@ std::int64_t OmpStructureChecker::GetOrdCollapseLevel(
 void OmpStructureChecker::CheckCycleConstraints(
     const parser::OpenMPLoopConstruct &x) {
   std::int64_t ordCollapseLevel{GetOrdCollapseLevel(x)};
-  OmpCycleChecker ompCycleChecker{context_, ordCollapseLevel};
-  parser::Walk(x, ompCycleChecker);
+  OmpCycleAndExitChecker checker{context_, ordCollapseLevel};
+  parser::Walk(x, checker);
 }
 
 void OmpStructureChecker::CheckDistLinear(
diff --git a/flang/test/Semantics/OpenMP/do08.f90 b/flang/test/Semantics/OpenMP/do08.f90
index 3ba63072a80b..5143dff0dd31 100644
--- a/flang/test/Semantics/OpenMP/do08.f90
+++ b/flang/test/Semantics/OpenMP/do08.f90
@@ -4,6 +4,8 @@
 
 program omp
   integer i, j, k
+  logical cond(10,10,10)
+  cond = .false.
 
   !ERROR: The value of the parameter in the COLLAPSE or ORDERED clause must not be larger than the number of nested loops following the construct.
   !$omp do  collapse(3)
@@ -135,4 +137,33 @@ program omp
   end do foo
   !$omp end do
 
+  !$omp do collapse(3)
+  loopk: do k=1,10
+    loopj: do j=1,10
+      loopi: do i=1,10
+        ifi : if (.true.) then
+          !ERROR: EXIT statement terminates associated loop of an OpenMP DO construct
+          if (cond(i,j,k)) exit
+          if (cond(i,j,k)) exit ifi
+          !ERROR: EXIT statement terminates associated loop of an OpenMP DO construct
+          if (cond(i,j,k)) exit loopi
+          !ERROR: EXIT statement terminates associated loop of an OpenMP DO construct
+          if (cond(i,j,k)) exit loopj
+        end if ifi
+      end do loopi
+    end do loopj
+  end do loopk
+  !$omp end do
+
+  !$omp do collapse(2)
+  loopk: do k=1,10
+    loopj: do j=1,10
+      do i=1,10
+      end do
+      !ERROR: EXIT statement terminates associated loop of an OpenMP DO construct
+      if (cond(i,j,k)) exit
+    end do loopj
+  end do loopk
+  !$omp end do
+
 end program omp
-- 
GitLab


From 602df270a9bfcb52980a93c85eb615c0d91eba0c Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan@arm.com>
Date: Wed, 8 May 2024 10:00:48 +0100
Subject: [PATCH 0151/1206] [Flang] RFC: Add support for -w option 1/n (#90420)

Add support for the -w option to switch OFF all Flang
warnings. This patch only supports switching OFF the
frontend warnings.

TODO : Support for MLIR, LLVM and Driver warnings.
TODO : Support interactions between -w, -pedantic, -Wall
---
 clang/include/clang/Driver/Options.td         |  2 +-
 clang/lib/Driver/ToolChains/Flang.cpp         |  4 +++
 .../flang/Frontend/CompilerInvocation.h       |  8 +++++
 flang/lib/Frontend/CompilerInvocation.cpp     | 10 ++++++
 flang/test/Driver/w-option.f90                | 31 +++++++++++++++++++
 5 files changed, 54 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Driver/w-option.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 2c319ba38a29..734ae7833f5c 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5703,7 +5703,7 @@ def whatsloaded : Flag<["-"], "whatsloaded">;
 def why_load : Flag<["-"], "why_load">;
 def whyload : Flag<["-"], "whyload">, Alias<why_load>;
 def w : Flag<["-"], "w">, HelpText<"Suppress all warnings">,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   MarshallingInfoFlag<DiagnosticOpts<"IgnoreWarnings">>;
 def x : JoinedOrSeparate<["-"], "x">,
 Flags<[NoXarchOption]>,
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 8955b9fb653c..436a9c418a5f 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -748,6 +748,10 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   // Add other compile options
   addOtherOptions(Args, CmdArgs);
 
+  // Disable all warnings
+  // TODO: Handle interactions between -w, -pedantic, -Wall, -WOption
+  Args.AddLastArg(CmdArgs, options::OPT_w);
+
   // Forward flags for OpenMP. We don't do this if the current action is an
   // device offloading action other than OpenMP.
   if (Args.hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index 4924d090eaf9..0fefaecfe4f0 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -114,8 +114,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // Fortran Dialect options
   Fortran::common::IntrinsicTypeDefaultKinds defaultKinds;
 
+  // Fortran Warning options
   bool enableConformanceChecks = false;
   bool enableUsageChecks = false;
+  bool disableWarnings = false;
 
   /// Used in e.g. unparsing to dump the analyzed rather than the original
   /// parse-tree objects.
@@ -197,6 +199,9 @@ public:
   bool &getEnableUsageChecks() { return enableUsageChecks; }
   const bool &getEnableUsageChecks() const { return enableUsageChecks; }
 
+  bool &getDisableWarnings() { return disableWarnings; }
+  const bool &getDisableWarnings() const { return disableWarnings; }
+
   Fortran::parser::AnalyzedObjectsAsFortran &getAsFortran() {
     return asFortran;
   }
@@ -226,6 +231,9 @@ public:
   // Enables the usage checks
   void setEnableUsageChecks() { enableUsageChecks = true; }
 
+  // Disables all Warnings
+  void setDisableWarnings() { disableWarnings = true; }
+
   /// Useful setters
   void setArgv0(const char *dir) { argv0 = dir; }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index f1b7b5397539..4318286e7415 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -975,6 +975,11 @@ static bool parseDialectArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
     res.setEnableConformanceChecks();
     res.setEnableUsageChecks();
   }
+
+  // -w
+  if (args.hasArg(clang::driver::options::OPT_w))
+    res.setDisableWarnings();
+
   // -std=f2018
   // TODO: Set proper options when more fortran standards
   // are supported.
@@ -1403,6 +1408,11 @@ void CompilerInvocation::setFortranOpts() {
 
   if (getEnableUsageChecks())
     fortranOptions.features.WarnOnAllUsage();
+
+  if (getDisableWarnings()) {
+    fortranOptions.features.DisableAllNonstandardWarnings();
+    fortranOptions.features.DisableAllUsageWarnings();
+  }
 }
 
 std::unique_ptr<Fortran::semantics::SemanticsContext>
diff --git a/flang/test/Driver/w-option.f90 b/flang/test/Driver/w-option.f90
new file mode 100644
index 000000000000..e34cddaab373
--- /dev/null
+++ b/flang/test/Driver/w-option.f90
@@ -0,0 +1,31 @@
+! Test the default setting. Emit warnings only.
+! RUN: %flang -c %s 2>&1 | FileCheck %s -check-prefix=DEFAULT
+
+! Test that the warnings are not generated with `-w` option.
+! RUN: %flang -c -w %s 2>&1 | FileCheck --allow-empty %s -check-prefix=WARNING
+
+! Test that warnings are portability messages are generated.
+! RUN: %flang -c -pedantic %s 2>&1 | FileCheck %s -check-prefixes=DEFAULT,PORTABILITY
+
+! Test that warnings and portability messages are not generated.
+! TODO: Support the last flag wins behaviour.
+! RUN: %flang -c -pedantic -w %s 2>&1 | FileCheck --allow-empty %s -check-prefixes=WARNING,PORTABILITY-WARNING
+! RUN: %flang -c -w -pedantic %s 2>&1 | FileCheck --allow-empty %s -check-prefixes=WARNING,PORTABILITY-WARNING
+! DEFAULT: warning: Label '40' is in a construct that should not be used as a branch target here
+! DEFAULT: warning: Label '50' is in a construct that should not be used as a branch target here
+! WARNING-NOT: warning
+! PORTABILITY: portability: Statement function 'sf1' should not contain an array constructor
+! PORTABILITY-WARNING-NOT: portability
+
+subroutine sub01(n)
+  integer n
+  GOTO (40,50,60) n
+  if (n .eq. 1) then
+40   print *, "xyz"
+50 end if
+60 continue
+end subroutine sub01
+
+subroutine sub02
+  sf1(n) = sum([(j,j=1,n)])
+end subroutine sub02
-- 
GitLab


From 1a498103ee5c4d101e70dc49db11938d8b87b518 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Wed, 8 May 2024 10:01:47 +0100
Subject: [PATCH 0152/1206] [mlir][ArmSME][test] Prepare tests for tile
 allocation changes (#91358)

This patch:

 1. Removes some duplicate test cases
 2. Removes unnecessary uses of `-convert-arm-sme-to-llvm`
 3. Ensures tile values have uses via `test.some_use()`

1 and 2 will make these tests easier to update. 3 will be needed as
ArmSME operations will be pure.
---
 .../ArmSMEToLLVM/arm-sme-to-llvm.mlir         |  24 +-
 .../Conversion/ArmSMEToLLVM/unsupported.mlir  |   2 +-
 .../ArmSMEToSCF/arm-sme-to-scf.mlir           |   6 +
 ...cation.mlir => basic-tile-allocation.mlir} | 297 ++++++++++--------
 mlir/test/Dialect/ArmSME/enable-arm-za.mlir   |  20 +-
 .../Dialect/ArmSME/outer-product-fusion.mlir  |   7 +-
 mlir/test/Dialect/ArmSME/tile-zero-masks.mlir |  43 ++-
 7 files changed, 234 insertions(+), 165 deletions(-)
 rename mlir/test/Dialect/ArmSME/{tile-allocation.mlir => basic-tile-allocation.mlir} (52%)

diff --git a/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir b/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir
index 81087cc02099..f48046a8d799 100644
--- a/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir
+++ b/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir
@@ -25,6 +25,7 @@ func.func @arm_sme_load_tile_slice_hor_i8(%src : memref<?x?xi8>, %mask : vector<
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[16]x[16]xi8>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index : memref<?x?xi8>, vector<[16]xi1>, vector<[16]x[16]xi8>
+  "test.some_use" (%tile_update) : (vector<[16]x[16]xi8>) -> ()
   return
 }
 
@@ -36,6 +37,7 @@ func.func @arm_sme_load_tile_slice_hor_i16(%src : memref<?x?xi16>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[8]x[8]xi16>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index : memref<?x?xi16>, vector<[8]xi1>, vector<[8]x[8]xi16>
+  "test.some_use" (%tile_update) : (vector<[8]x[8]xi16>) -> ()
   return
 }
 
@@ -47,6 +49,7 @@ func.func @arm_sme_load_tile_slice_hor_i32(%src : memref<?x?xi32>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[4]x[4]xi32>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index : memref<?x?xi32>, vector<[4]xi1>, vector<[4]x[4]xi32>
+  "test.some_use" (%tile_update) : (vector<[4]x[4]xi32>) -> ()
   return
 }
 
@@ -58,6 +61,7 @@ func.func @arm_sme_load_tile_slice_hor_i64(%src : memref<?x?xi64>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[2]x[2]xi64>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index : memref<?x?xi64>, vector<[2]xi1>, vector<[2]x[2]xi64>
+  "test.some_use" (%tile_update) : (vector<[2]x[2]xi64>) -> ()
   return
 }
 
@@ -69,6 +73,7 @@ func.func @arm_sme_load_tile_slice_hor_i128(%src : memref<?x?xi128>, %mask : vec
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[1]x[1]xi128>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index : memref<?x?xi128>, vector<[1]xi1>, vector<[1]x[1]xi128>
+  "test.some_use" (%tile_update) : (vector<[1]x[1]xi128>) -> ()
   return
 }
 
@@ -80,6 +85,7 @@ func.func @arm_sme_load_tile_slice_hor_f16(%src : memref<?x?xf16>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[8]x[8]xf16>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index : memref<?x?xf16>, vector<[8]xi1>, vector<[8]x[8]xf16>
+  "test.some_use" (%tile_update) : (vector<[8]x[8]xf16>) -> ()
   return
 }
 
@@ -91,6 +97,7 @@ func.func @arm_sme_load_tile_slice_hor_bf16(%src : memref<?x?xbf16>, %mask : vec
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[8]x[8]xbf16>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index : memref<?x?xbf16>, vector<[8]xi1>, vector<[8]x[8]xbf16>
+  "test.some_use" (%tile_update) : (vector<[8]x[8]xbf16>) -> ()
   return
 }
 
@@ -102,6 +109,7 @@ func.func @arm_sme_load_tile_slice_hor_f32(%src : memref<?x?xf32>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[4]x[4]xf32>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index : memref<?x?xf32>, vector<[4]xi1>, vector<[4]x[4]xf32>
+  "test.some_use" (%tile_update) : (vector<[4]x[4]xf32>) -> ()
   return
 }
 
@@ -113,6 +121,7 @@ func.func @arm_sme_load_tile_slice_hor_f64(%src : memref<?x?xf64>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[2]x[2]xf64>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index : memref<?x?xf64>, vector<[2]xi1>, vector<[2]x[2]xf64>
+  "test.some_use" (%tile_update) : (vector<[2]x[2]xf64>) -> ()
   return
 }
 
@@ -124,6 +133,7 @@ func.func @arm_sme_load_tile_slice_ver_i8(%src : memref<?x?xi8>, %mask : vector<
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[16]x[16]xi8>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index layout<vertical> : memref<?x?xi8>, vector<[16]xi1>, vector<[16]x[16]xi8>
+  "test.some_use" (%tile_update) : (vector<[16]x[16]xi8>) -> ()
   return
 }
 
@@ -135,6 +145,7 @@ func.func @arm_sme_load_tile_slice_ver_i16(%src : memref<?x?xi16>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[8]x[8]xi16>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index layout<vertical> : memref<?x?xi16>, vector<[8]xi1>, vector<[8]x[8]xi16>
+  "test.some_use" (%tile_update) : (vector<[8]x[8]xi16>) -> ()
   return
 }
 
@@ -146,6 +157,7 @@ func.func @arm_sme_load_tile_slice_ver_i32(%src : memref<?x?xi32>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[4]x[4]xi32>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index layout<vertical> : memref<?x?xi32>, vector<[4]xi1>, vector<[4]x[4]xi32>
+  "test.some_use" (%tile_update) : (vector<[4]x[4]xi32>) -> ()
   return
 }
 
@@ -157,6 +169,7 @@ func.func @arm_sme_load_tile_slice_ver_i64(%src : memref<?x?xi64>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[2]x[2]xi64>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index layout<vertical> : memref<?x?xi64>, vector<[2]xi1>, vector<[2]x[2]xi64>
+  "test.some_use" (%tile_update) : (vector<[2]x[2]xi64>) -> ()
   return
 }
 
@@ -168,6 +181,7 @@ func.func @arm_sme_load_tile_slice_ver_i128(%src : memref<?x?xi128>, %mask : vec
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[1]x[1]xi128>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index layout<vertical> : memref<?x?xi128>, vector<[1]xi1>, vector<[1]x[1]xi128>
+  "test.some_use" (%tile_update) : (vector<[1]x[1]xi128>) -> ()
   return
 }
 
@@ -179,6 +193,7 @@ func.func @arm_sme_load_tile_slice_ver_f16(%src : memref<?x?xf16>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[8]x[8]xf16>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index layout<vertical> : memref<?x?xf16>, vector<[8]xi1>, vector<[8]x[8]xf16>
+  "test.some_use" (%tile_update) : (vector<[8]x[8]xf16>) -> ()
   return
 }
 
@@ -190,6 +205,7 @@ func.func @arm_sme_load_tile_slice_ver_bf16(%src : memref<?x?xbf16>, %mask : vec
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[8]x[8]xbf16>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index layout<vertical> : memref<?x?xbf16>, vector<[8]xi1>, vector<[8]x[8]xbf16>
+  "test.some_use" (%tile_update) : (vector<[8]x[8]xbf16>) -> ()
   return
 }
 
@@ -201,6 +217,7 @@ func.func @arm_sme_load_tile_slice_ver_f32(%src : memref<?x?xf32>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[4]x[4]xf32>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index layout<vertical> : memref<?x?xf32>, vector<[4]xi1>, vector<[4]x[4]xf32>
+  "test.some_use" (%tile_update) : (vector<[4]x[4]xf32>) -> ()
   return
 }
 
@@ -212,6 +229,7 @@ func.func @arm_sme_load_tile_slice_ver_f64(%src : memref<?x?xf64>, %mask : vecto
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[2]x[2]xf64>
   %tile_update = arm_sme.load_tile_slice %src[%c0], %mask, %tile, %tile_slice_index layout<vertical> : memref<?x?xf64>, vector<[2]xi1>, vector<[2]x[2]xf64>
+  "test.some_use" (%tile_update) : (vector<[2]x[2]xf64>) -> ()
   return
 }
 
@@ -441,7 +459,8 @@ func.func @arm_sme_store_tile_slice_ver_f64(%tile_slice_index : index, %mask : v
 func.func @arm_sme_move_vector_to_tile_slice_hor_i32(%vector : vector<[4]xi32>, %tile_slice_index : index) -> () {
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[4]x[4]xi32>
-  arm_sme.move_vector_to_tile_slice %vector, %tile, %tile_slice_index : vector<[4]xi32> into vector<[4]x[4]xi32>
+  %tile_update = arm_sme.move_vector_to_tile_slice %vector, %tile, %tile_slice_index : vector<[4]xi32> into vector<[4]x[4]xi32>
+  "test.some_use" (%tile_update) : (vector<[4]x[4]xi32>) -> ()
   return
 }
 
@@ -452,7 +471,8 @@ func.func @arm_sme_move_vector_to_tile_slice_hor_i32(%vector : vector<[4]xi32>,
 func.func @arm_sme_move_vector_to_tile_slice_ver_bf16(%vector : vector<[8]xbf16>, %tile_slice_index : index) -> () {
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[8]x[8]xbf16>
-  arm_sme.move_vector_to_tile_slice %vector, %tile, %tile_slice_index layout<vertical> : vector<[8]xbf16> into vector<[8]x[8]xbf16>
+  %tile_update =  arm_sme.move_vector_to_tile_slice %vector, %tile, %tile_slice_index layout<vertical> : vector<[8]xbf16> into vector<[8]x[8]xbf16>
+  "test.some_use" (%tile_update) : (vector<[8]x[8]xbf16>) -> ()
   return
 }
 
diff --git a/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir b/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir
index 59665c471921..15767ff1dec3 100644
--- a/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir
+++ b/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir
@@ -9,6 +9,6 @@ func.func @arm_sme_outerproduct_unsupported_type(%lhs : vector<[16]xi8>, %rhs :
   // expected-error@+2 {{failed to legalize operation 'arm_sme.outerproduct'}}
   // expected-error@+1 {{unsupported type}}
   %0 = arm_sme.outerproduct %lhs, %rhs  acc(%acc) : vector<[16]xi8>, vector<[16]xi8>
-  "prevent.dce"(%0) : (vector<[16]x[16]xi8>) -> ()
+  "test.some_use"(%0) : (vector<[16]x[16]xi8>) -> ()
 }
 
diff --git a/mlir/test/Conversion/ArmSMEToSCF/arm-sme-to-scf.mlir b/mlir/test/Conversion/ArmSMEToSCF/arm-sme-to-scf.mlir
index 6c393bc38af9..a2f2beff78c4 100644
--- a/mlir/test/Conversion/ArmSMEToSCF/arm-sme-to-scf.mlir
+++ b/mlir/test/Conversion/ArmSMEToSCF/arm-sme-to-scf.mlir
@@ -20,6 +20,7 @@
 func.func @arm_sme_tile_load_hor(%src : memref<?x?xi32>) {
   %c0 = arith.constant 0 : index
   %tile = arm_sme.tile_load %src[%c0, %c0] : memref<?x?xi32>, vector<[4]x[4]xi32>
+  "test.some_use" (%tile) : (vector<[4]x[4]xi32>) -> ()
   return
 }
 
@@ -30,6 +31,7 @@ func.func @arm_sme_tile_load_hor(%src : memref<?x?xi32>) {
 func.func @arm_sme_tile_load_ver(%src : memref<?x?xi32>) {
   %c0 = arith.constant 0 : index
   %tile = arm_sme.tile_load %src[%c0, %c0] layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  "test.some_use" (%tile) : (vector<[4]x[4]xi32>) -> ()
   return
 }
 
@@ -60,6 +62,7 @@ func.func @arm_sme_tile_load_hor_with_mask_and_pad_zero(%src : memref<?x?xi32>)
   %pad = arith.constant 0 : i32
   %mask = vector.create_mask %c3, %c2 : vector<[4]x[4]xi1>
   %tile = arm_sme.tile_load %src[%c0, %c0], %pad, %mask : memref<?x?xi32>, vector<[4]x[4]xi32>
+  "test.some_use" (%tile) : (vector<[4]x[4]xi32>) -> ()
   return
 }
 
@@ -94,6 +97,7 @@ func.func @arm_sme_tile_load_hor_with_mask_and_nonzero_pad(%src : memref<?x?xi32
   %c3 = arith.constant 3 : index
   %mask = vector.create_mask %c3, %c2 : vector<[4]x[4]xi1>
   %tile = arm_sme.tile_load %src[%c0, %c0], %pad, %mask : memref<?x?xi32>, vector<[4]x[4]xi32>
+  "test.some_use" (%tile) : (vector<[4]x[4]xi32>) -> ()
   return
 }
 
@@ -104,6 +108,7 @@ func.func @arm_sme_tile_load_zero_pad__unsupported_mask_op(%src : memref<?x?xi32
   %pad = arith.constant 0 : i32
   // expected-error@+1 {{failed to legalize operation 'arm_sme.tile_load' that was explicitly marked illegal}}
   %tile = arm_sme.tile_load %src[%c0, %c0], %pad, %mask : memref<?x?xi32>, vector<[4]x[4]xi32>
+  "test.some_use" (%tile) : (vector<[4]x[4]xi32>) -> ()
   return
 }
 
@@ -113,6 +118,7 @@ func.func @arm_sme_tile_load_nonzero_pad__unsupported_mask_op(%src : memref<?x?x
   %c0 = arith.constant 0 : index
   // expected-error@+1 {{failed to legalize operation 'arm_sme.tile_load' that was explicitly marked illegal}}
   %tile = arm_sme.tile_load %src[%c0, %c0], %pad, %mask : memref<?x?xi32>, vector<[4]x[4]xi32>
+  "test.some_use" (%tile) : (vector<[4]x[4]xi32>) -> ()
   return
 }
 
diff --git a/mlir/test/Dialect/ArmSME/tile-allocation.mlir b/mlir/test/Dialect/ArmSME/basic-tile-allocation.mlir
similarity index 52%
rename from mlir/test/Dialect/ArmSME/tile-allocation.mlir
rename to mlir/test/Dialect/ArmSME/basic-tile-allocation.mlir
index 9c368dd4fa23..e144bac970a7 100644
--- a/mlir/test/Dialect/ArmSME/tile-allocation.mlir
+++ b/mlir/test/Dialect/ArmSME/basic-tile-allocation.mlir
@@ -1,9 +1,10 @@
-// RUN: mlir-opt %s -allocate-arm-sme-tiles -split-input-file -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt %s -allocate-arm-sme-tiles -split-input-file | FileCheck %s
 
 // -----
 
+// Note: Tile IDs >= 16 are in-memory tile IDs (i.e. spills).
+
 // CHECK-LABEL: mixed_tiles
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 65534 : i32}
 func.func @mixed_tiles() {
   // ZA0.Q, ZA2.Q, ZA4.Q, ZA6.Q, ZA8.Q, ZA10.Q, ZA12.Q, ZA14.Q
   // CHECK-NEXT: tile_id = 0
@@ -18,76 +19,61 @@ func.func @mixed_tiles() {
   // CHECK-NEXT: tile_id = 7
   %za7_q = arm_sme.get_tile : vector<[1]x[1]xi128>
   // ZA15.Q is still free.
+  "test.some_use"(%za0_h) : (vector<[8]x[8]xi16>) -> ()
+  "test.some_use"(%za1_s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%za3_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za7_q) : (vector<[1]x[1]xi128>) -> ()
   return
 }
 
 // -----
 
 // CHECK-LABEL: za_b
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 65535 : i32}
 func.func @za_b() {
   // CHECK-NEXT: tile_id = 0
   %za0_b = arm_sme.get_tile : vector<[16]x[16]xi8>
-  return
-}
-
-// -----
-
-func.func @za_b__out_of_tiles() {
-  %za0_b = arm_sme.get_tile : vector<[16]x[16]xi8>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // Next tile is in-memory:
+  // CHECK-NEXT: tile_id = 16
   %next_tile = arm_sme.get_tile : vector<[16]x[16]xi8>
+  "test.some_use"(%za0_b) : (vector<[16]x[16]xi8>) -> ()
+  "test.some_use"(%next_tile) : (vector<[16]x[16]xi8>) -> ()
   return
 }
 
 // -----
 
+// CHECK-LABEL: za_b_overlapping_za_q
 func.func @za_b_overlapping_za_q() {
+  // CHECK-NEXT: tile_id = 0
   %za0_b = arm_sme.get_tile : vector<[16]x[16]xi8>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // Next tile is in-memory:
+  // CHECK-NEXT: tile_id = 16
   %next_tile = arm_sme.get_tile : vector<[1]x[1]xi128>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: za0_h
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 43690 : i32}
-func.func @za0_h() {
-  // CHECK-NEXT: tile_id = 0
-  %za0_h = arm_sme.get_tile : vector<[8]x[8]xi16>
+  "test.some_use"(%za0_b) : (vector<[16]x[16]xi8>) -> ()
+  "test.some_use"(%next_tile) : (vector<[1]x[1]xi128>) -> ()
   return
 }
 
 // -----
 
 // CHECK-LABEL: za_h
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 65535 : i32}
 func.func @za_h() {
   // CHECK-NEXT: tile_id = 0
   %za0_h = arm_sme.get_tile : vector<[8]x[8]xi16>
   // CHECK-NEXT: tile_id = 1
   %za1_h = arm_sme.get_tile : vector<[8]x[8]xi16>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: za_h__out_of_tiles
-func.func @za_h__out_of_tiles() {
-  // CHECK-NEXT: tile_id = 0
-  %za0_h = arm_sme.get_tile : vector<[8]x[8]xi16>
-  // CHECK-NEXT: tile_id = 1
-  %za1_h = arm_sme.get_tile : vector<[8]x[8]xi16>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // Next tile is in-memory:
+  // CHECK-NEXT: tile_id = 16
   %next_tile = arm_sme.get_tile : vector<[8]x[8]xi16>
+  "test.some_use"(%za0_h) : (vector<[8]x[8]xi16>) -> ()
+  "test.some_use"(%za1_h) : (vector<[8]x[8]xi16>) -> ()
+  "test.some_use"(%next_tile) : (vector<[8]x[8]xi16>) -> ()
   return
 }
 
 // -----
 
 // CHECK-LABEL: za_h_overlapping_za_s
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 65535 : i32}
 func.func @za_h_overlapping_za_s() {
   // ZA0.Q, ZA2.Q, ZA4.Q, ZA6.Q, ZA8.Q, ZA10.Q, ZA12.Q, ZA14.Q
   // CHECK-NEXT: tile_id = 0
@@ -98,13 +84,15 @@ func.func @za_h_overlapping_za_s() {
   // ZA3.Q, ZA7.Q, ZA11.Q, ZA15.Q
   // CHECK-NEXT: tile_id = 3
   %za3_s = arm_sme.get_tile : vector<[4]x[4]xi32>
+  "test.some_use"(%za0_h) : (vector<[8]x[8]xi16>) -> ()
+  "test.some_use"(%za1_s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%za3_s) : (vector<[4]x[4]xi32>) -> ()
   return
 }
 
 // -----
 
 // CHECK-LABEL: za_h_overlapping_za_d
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 65535 : i32}
 func.func @za_h_overlapping_za_d() {
   // ZA0.Q, ZA2.Q, ZA4.Q, ZA6.Q, ZA8.Q, ZA10.Q, ZA12.Q, ZA14.Q
   // CHECK-NEXT: tile_id = 0
@@ -121,40 +109,55 @@ func.func @za_h_overlapping_za_d() {
   // ZA7.Q, ZA15.Q
   // CHECK-NEXT: tile_id = 7
   %za7_d = arm_sme.get_tile : vector<[2]x[2]xi64>
+  "test.some_use"(%za0_h) : (vector<[8]x[8]xi16>) -> ()
+  "test.some_use"(%za1_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za3_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za5_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za7_d) : (vector<[2]x[2]xi64>) -> ()
   return
 }
 
 // -----
 
+// CHECK-LABEL: za_h_overlapping_za_q
 func.func @za_h_overlapping_za_q() {
+  // CHECK-NEXT: tile_id = 0
   %za0_h = arm_sme.get_tile : vector<[8]x[8]xi16>
-  %za0_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za2_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za4_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za6_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za8_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za10_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za12_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za14_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // CHECK-NEXT: tile_id = 1
+  %za1_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 3
+  %za3_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 5
+  %za5_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 7
+  %za7_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 9
+  %za9_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 11
+  %za11_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 13
+  %za13_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 15
+  %za15_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // Next tile is in-memory:
+  // CHECK-NEXT: tile_id = 16
   %next_tile = arm_sme.get_tile : vector<[1]x[1]xi128>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: za0_s
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 34952 : i32}
-func.func @za0_s() {
-  // CHECK-NEXT: tile_id = 0
-  %za0_s = arm_sme.get_tile : vector<[4]x[4]xi32>
+  "test.some_use"(%za0_h) : (vector<[8]x[8]xi16>) -> ()
+  "test.some_use"(%za1_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za3_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za5_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za7_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za9_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za11_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za13_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za15_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%next_tile) : (vector<[1]x[1]xi128>) -> ()
   return
 }
 
 // -----
 
 // CHECK-LABEL: za_s
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 65535 : i32}
 func.func @za_s() {
   // CHECK-NEXT: tile_id = 0
   %za0_s = arm_sme.get_tile : vector<[4]x[4]xi32>
@@ -164,25 +167,20 @@ func.func @za_s() {
   %za2_s = arm_sme.get_tile : vector<[4]x[4]xi32>
   // CHECK-NEXT: tile_id = 3
   %za3_s = arm_sme.get_tile : vector<[4]x[4]xi32>
-  return
-}
-
-// -----
-
-func.func @za_s__out_of_tiles() {
-  %za0_s = arm_sme.get_tile : vector<[4]x[4]xi32>
-  %za1_s = arm_sme.get_tile : vector<[4]x[4]xi32>
-  %za2_s = arm_sme.get_tile : vector<[4]x[4]xi32>
-  %za3_s = arm_sme.get_tile : vector<[4]x[4]xi32>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // Next tile is in-memory:
+  // CHECK-NEXT: tile_id = 16
   %next_tile = arm_sme.get_tile : vector<[4]x[4]xi32>
+  "test.some_use"(%za0_s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%za1_s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%za2_s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%za3_s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%next_tile) : (vector<[4]x[4]xi32>) -> ()
   return
 }
 
 // -----
 
 // CHECK-LABEL: za_s_overlapping_za_d
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 65535 : i32}
 func.func @za_s_overlapping_za_d() {
   // ZA0.Q, ZA4.Q, ZA8.Q, ZA12.Q
   // CHECK-NEXT: tile_id = 0
@@ -199,44 +197,67 @@ func.func @za_s_overlapping_za_d() {
   // ZA7.Q, ZA15.Q
   // CHECK-NEXT: tile_id = 7
   %za7_d = arm_sme.get_tile : vector<[2]x[2]xi64>
+  "test.some_use"(%za0_s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%za1_s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%za2_s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%za3_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za7_d) : (vector<[2]x[2]xi64>) -> ()
   return
 }
 
 // -----
 
+// CHECK-LABEL: za_s_overlapping_za_q
 func.func @za_s_overlapping_za_q() {
+  // CHECK-NEXT: tile_id = 0
   %za0_s = arm_sme.get_tile : vector<[4]x[4]xi32>
+  // CHECK-NEXT: tile_id = 1
   %za1_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 2
   %za2_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 3
   %za3_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 5
   %za5_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 6
   %za6_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 7
   %za7_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 9
   %za9_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 10
   %za10_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 11
   %za11_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 13
   %za13_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 14
   %za14_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 15
   %za15_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // Next tile is in-memory:
+  // CHECK-NEXT: tile_id = 16
   %next_tile = arm_sme.get_tile : vector<[1]x[1]xi128>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: za0_d
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 32896 : i32}
-func.func @za0_d() {
-  // CHECK-NEXT: tile_id = 0
-  %za0_d = arm_sme.get_tile : vector<[2]x[2]xi64>
+  "test.some_use"(%za0_s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%za1_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za2_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za3_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za5_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za6_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za7_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za9_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za10_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za11_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za13_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za14_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za15_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%next_tile) : (vector<[1]x[1]xi128>) -> ()
   return
 }
 
 // -----
 
 // CHECK-LABEL: za_d
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 65535 : i32}
 func.func @za_d() {
   // CHECK-NEXT: tile_id = 0
   %za0_d = arm_sme.get_tile : vector<[2]x[2]xi64>
@@ -254,62 +275,80 @@ func.func @za_d() {
   %za6_d = arm_sme.get_tile : vector<[2]x[2]xi64>
   // CHECK-NEXT: tile_id = 7
   %za7_d = arm_sme.get_tile : vector<[2]x[2]xi64>
-  return
-}
-
-// -----
-
-func.func @za_d__out_of_tiles() {
-  %za0_d = arm_sme.get_tile : vector<[2]x[2]xi64>
-  %za1_d = arm_sme.get_tile : vector<[2]x[2]xi64>
-  %za2_d = arm_sme.get_tile : vector<[2]x[2]xi64>
-  %za3_d = arm_sme.get_tile : vector<[2]x[2]xi64>
-  %za4_d = arm_sme.get_tile : vector<[2]x[2]xi64>
-  %za5_d = arm_sme.get_tile : vector<[2]x[2]xi64>
-  %za6_d = arm_sme.get_tile : vector<[2]x[2]xi64>
-  %za7_d = arm_sme.get_tile : vector<[2]x[2]xi64>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // Next tile is in-memory:
+  // CHECK-NEXT: tile_id = 16
   %next_tile = arm_sme.get_tile : vector<[2]x[2]xi64>
+  "test.some_use"(%za0_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za1_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za2_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za3_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za4_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za5_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za6_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za7_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%next_tile) : (vector<[2]x[2]xi64>) -> ()
   return
 }
 
 // -----
 
+// CHECK-LABEL: za_d_overlapping_za_q
 func.func @za_d_overlapping_za_q() {
+  // CHECK-NEXT: tile_id = 0
   %za0_d = arm_sme.get_tile : vector<[2]x[2]xi64>
+  // CHECK-NEXT: tile_id = 1
   %za1_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 2
   %za2_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 3
   %za3_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 4
   %za4_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 5
   %za5_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 6
   %za6_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 7
   %za7_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 9
   %za9_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 10
   %za10_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 11
   %za11_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 12
   %za12_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 13
   %za13_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 14
   %za14_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  // CHECK-NEXT: tile_id = 15
   %za15_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // Next tile is in-memory:
+  // CHECK-NEXT: tile_id = 16
   %next_tile = arm_sme.get_tile : vector<[1]x[1]xi128>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: za0_q
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 32768 : i32}
-func.func @za0_q() {
-  // CHECK-NEXT: tile_id = 0
-  %za0_q = arm_sme.get_tile : vector<[1]x[1]xi128>
+  "test.some_use"(%za0_d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%za1_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za2_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za3_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za4_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za5_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za6_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za7_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za9_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za10_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za11_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za12_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za13_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za14_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za15_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%next_tile) : (vector<[1]x[1]xi128>) -> ()
   return
 }
 
 // -----
 
 // CHECK-LABEL: za_q
-// CHECK-SAME: attributes {arm_sme.tiles_in_use = 65535 : i32}
 func.func @za_q() {
   // CHECK-NEXT: tile_id = 0
   %za0_q = arm_sme.get_tile : vector<[1]x[1]xi128>
@@ -343,29 +382,25 @@ func.func @za_q() {
   %za14_q = arm_sme.get_tile : vector<[1]x[1]xi128>
   // CHECK-NEXT: tile_id = 15
   %za15_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  return
-}
-
-// -----
-
-func.func @za_q__out_of_tiles() {
-  %za0_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za1_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za2_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za3_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za4_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za5_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za6_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za7_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za8_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za9_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za10_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za11_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za12_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za13_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za14_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  %za15_q = arm_sme.get_tile : vector<[1]x[1]xi128>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // Next tile is in-memory:
+  // CHECK-NEXT: tile_id = 16
   %next_tile = arm_sme.get_tile : vector<[1]x[1]xi128>
+  "test.some_use"(%za0_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za1_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za2_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za3_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za4_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za5_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za6_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za7_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za8_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za9_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za10_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za11_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za12_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za13_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za14_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%za15_q) : (vector<[1]x[1]xi128>) -> ()
+  "test.some_use"(%next_tile) : (vector<[1]x[1]xi128>) -> ()
   return
 }
diff --git a/mlir/test/Dialect/ArmSME/enable-arm-za.mlir b/mlir/test/Dialect/ArmSME/enable-arm-za.mlir
index a20203d7e557..d3325513a848 100644
--- a/mlir/test/Dialect/ArmSME/enable-arm-za.mlir
+++ b/mlir/test/Dialect/ArmSME/enable-arm-za.mlir
@@ -1,10 +1,9 @@
-// RUN: mlir-opt %s -enable-arm-streaming=za-mode=new-za -convert-arm-sme-to-llvm | FileCheck %s -check-prefix=ENABLE-ZA
-// RUN: mlir-opt %s -enable-arm-streaming -convert-arm-sme-to-llvm | FileCheck %s -check-prefix=DISABLE-ZA
-// RUN: mlir-opt %s -enable-arm-streaming=za-mode=in-za -convert-arm-sme-to-llvm | FileCheck %s -check-prefix=IN-ZA
-// RUN: mlir-opt %s -enable-arm-streaming=za-mode=out-za -convert-arm-sme-to-llvm | FileCheck %s -check-prefix=OUT-ZA
-// RUN: mlir-opt %s -enable-arm-streaming=za-mode=inout-za -convert-arm-sme-to-llvm | FileCheck %s -check-prefix=INOUT-ZA
-// RUN: mlir-opt %s -enable-arm-streaming=za-mode=preserves-za -convert-arm-sme-to-llvm | FileCheck %s -check-prefix=PRESERVES-ZA
-// RUN: mlir-opt %s -convert-arm-sme-to-llvm | FileCheck %s -check-prefix=NO-ARM-STREAMING
+// RUN: mlir-opt %s -enable-arm-streaming=za-mode=new-za | FileCheck %s -check-prefix=ENABLE-ZA
+// RUN: mlir-opt %s -enable-arm-streaming | FileCheck %s -check-prefix=DISABLE-ZA
+// RUN: mlir-opt %s -enable-arm-streaming=za-mode=in-za | FileCheck %s -check-prefix=IN-ZA
+// RUN: mlir-opt %s -enable-arm-streaming=za-mode=out-za | FileCheck %s -check-prefix=OUT-ZA
+// RUN: mlir-opt %s -enable-arm-streaming=za-mode=inout-za | FileCheck %s -check-prefix=INOUT-ZA
+// RUN: mlir-opt %s -enable-arm-streaming=za-mode=preserves-za | FileCheck %s -check-prefix=PRESERVES-ZA
 
 // CHECK-LABEL: @declaration
 func.func private @declaration()
@@ -22,11 +21,4 @@ func.func private @declaration()
 // DISABLE-ZA-LABEL: @arm_new_za
 // DISABLE-ZA-NOT: arm_new_za
 // DISABLE-ZA-SAME: attributes {arm_streaming}
-// NO-ARM-STREAMING-LABEL: @arm_new_za
-// NO-ARM-STREAMING-NOT: arm_new_za
-// NO-ARM-STREAMING-NOT: arm_streaming
-// NO-ARM-STREAMING-NOT: arm_in_za
-// NO-ARM-STREAMING-NOT: arm_out_za
-// NO-ARM-STREAMING-NOT: arm_inout_za
-// NO-ARM-STREAMING-NOT: arm_preserves_za
 func.func @arm_new_za() { return }
diff --git a/mlir/test/Dialect/ArmSME/outer-product-fusion.mlir b/mlir/test/Dialect/ArmSME/outer-product-fusion.mlir
index 01f54a4cf186..4887d611643f 100644
--- a/mlir/test/Dialect/ArmSME/outer-product-fusion.mlir
+++ b/mlir/test/Dialect/ArmSME/outer-product-fusion.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -arm-sme-outer-product-fusion -cse -split-input-file -allow-unregistered-dialect | FileCheck %s
+// RUN: mlir-opt %s -arm-sme-outer-product-fusion -cse -split-input-file | FileCheck %s
 
 // CHECK-LABEL: @outerproduct_add_widening_2way_f16f16f32
 // CHECK-SAME:    %[[A0:.*]]: vector<[4]xf16>, %[[B0:.*]]: vector<[4]xf16>, %[[A1:.*]]: vector<[4]xf16>, %[[B1:.*]]: vector<[4]xf16>,
@@ -929,6 +929,7 @@ func.func @outerproduct_widening_4way__missing_acc(
   %2 = arm_sme.outerproduct %a2_ext, %b2_ext acc(%1) : vector<[4]xi32>, vector<[4]xi32>
   // Missing accumulator breaks use-def chain.
   %3 = arm_sme.outerproduct %a3_ext, %b3_ext : vector<[4]xi32>, vector<[4]xi32>
+  "test.some_use"(%2) : (vector<[4]x[4]xi32>) -> ()
 
   return %3 : vector<[4]x[4]xi32>
 }
@@ -1014,7 +1015,7 @@ func.func @outerproduct_widening_2way__cant_erase(
 
   %acc = arith.constant dense<1.0> : vector<[4]x[4]xf32>
   %0 = arm_sme.outerproduct %a0_ext, %b0_ext acc(%acc) : vector<[4]xf32>, vector<[4]xf32>
-  "fake.use"(%0) : (vector<[4]x[4]xf32>) -> ()
+  "test.some_use"(%0) : (vector<[4]x[4]xf32>) -> ()
   %1 = arm_sme.outerproduct %a1_ext, %b1_ext acc(%0) : vector<[4]xf32>, vector<[4]xf32>
 
   return %1 : vector<[4]x[4]xf32>
@@ -1048,7 +1049,7 @@ func.func @outerproduct_widening_4way__multi_use_cant_erase(
 
   %0 = arm_sme.outerproduct %a0_ext, %b0_ext : vector<[4]xi32>, vector<[4]xi32>
   %1 = arm_sme.outerproduct %a1_ext, %b1_ext acc(%0) : vector<[4]xi32>, vector<[4]xi32>
-  "fake.use"(%1) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%1) : (vector<[4]x[4]xi32>) -> ()
   %2 = arm_sme.outerproduct %a2_ext, %b2_ext acc(%1) : vector<[4]xi32>, vector<[4]xi32>
   %3 = arm_sme.outerproduct %a3_ext, %b3_ext acc(%2) : vector<[4]xi32>, vector<[4]xi32>
 
diff --git a/mlir/test/Dialect/ArmSME/tile-zero-masks.mlir b/mlir/test/Dialect/ArmSME/tile-zero-masks.mlir
index 04412e4db1c5..cac2dcc24d10 100644
--- a/mlir/test/Dialect/ArmSME/tile-zero-masks.mlir
+++ b/mlir/test/Dialect/ArmSME/tile-zero-masks.mlir
@@ -9,6 +9,7 @@
 func.func @zero_za_b() {
   // CHECK: "arm_sme.intr.zero"() <{tile_mask = 255 : i32}> : () -> ()
   %zero_za0b = arm_sme.zero : vector<[16]x[16]xi8>
+  "test.some_use"(%zero_za0b) : (vector<[16]x[16]xi8>) -> ()
   return
 }
 
@@ -16,10 +17,12 @@ func.func @zero_za_b() {
 
 // CHECK-LABEL: zero_za_h
 func.func @zero_za_h() {
-  // CHECK:      "arm_sme.intr.zero"() <{tile_mask = 85 : i32}> : () -> ()
+  // CHECK: "arm_sme.intr.zero"() <{tile_mask = 85 : i32}> : () -> ()
   %zero_za0h = arm_sme.zero : vector<[8]x[8]xi16>
-  // CHECK-NEXT: "arm_sme.intr.zero"() <{tile_mask = 170 : i32}> : () -> ()
+  // CHECK: "arm_sme.intr.zero"() <{tile_mask = 170 : i32}> : () -> ()
   %zero_za1h = arm_sme.zero : vector<[8]x[8]xf16>
+  "test.some_use"(%zero_za0h) : (vector<[8]x[8]xi16>) -> ()
+  "test.some_use"(%zero_za1h) : (vector<[8]x[8]xf16>) -> ()
   return
 }
 
@@ -27,14 +30,18 @@ func.func @zero_za_h() {
 
 // CHECK-LABEL: zero_za_s
 func.func @zero_za_s() {
-  // CHECK:      arm_sme.intr.zero"() <{tile_mask = 17 : i32}> : () -> ()
+  // CHECK: arm_sme.intr.zero"() <{tile_mask = 17 : i32}> : () -> ()
   %zero_za0s = arm_sme.zero : vector<[4]x[4]xi32>
-  // CHECK-NEXT: arm_sme.intr.zero"() <{tile_mask = 34 : i32}> : () -> ()
+  // CHECK: arm_sme.intr.zero"() <{tile_mask = 34 : i32}> : () -> ()
   %zero_za1s = arm_sme.zero : vector<[4]x[4]xi32>
-  // CHECK-NEXT: arm_sme.intr.zero"() <{tile_mask = 68 : i32}> : () -> ()
+  // CHECK: arm_sme.intr.zero"() <{tile_mask = 68 : i32}> : () -> ()
   %zero_za2s = arm_sme.zero : vector<[4]x[4]xi32>
-  // CHECK-NEXT: arm_sme.intr.zero"() <{tile_mask = 136 : i32}> : () -> ()
+  // CHECK: arm_sme.intr.zero"() <{tile_mask = 136 : i32}> : () -> ()
   %zero_za3s = arm_sme.zero : vector<[4]x[4]xf32>
+  "test.some_use"(%zero_za0s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%zero_za1s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%zero_za2s) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%zero_za3s) : (vector<[4]x[4]xf32>) -> ()
   return
 }
 
@@ -42,21 +49,29 @@ func.func @zero_za_s() {
 
 // CHECK-LABEL: zero_za_d
 func.func @zero_za_d() {
-  // CHECK:      "arm_sme.intr.zero"() <{tile_mask = 1 : i32}> : () -> ()
+  // CHECK: "arm_sme.intr.zero"() <{tile_mask = 1 : i32}> : () -> ()
   %zero_za0d = arm_sme.zero : vector<[2]x[2]xi64>
-  // CHECK-NEXT: "arm_sme.intr.zero"() <{tile_mask = 2 : i32}> : () -> ()
+  // CHECK: "arm_sme.intr.zero"() <{tile_mask = 2 : i32}> : () -> ()
   %zero_za1d = arm_sme.zero : vector<[2]x[2]xi64>
-  // CHECK-NEXT: "arm_sme.intr.zero"() <{tile_mask = 4 : i32}> : () -> ()
+  // CHECK: "arm_sme.intr.zero"() <{tile_mask = 4 : i32}> : () -> ()
   %zero_za2d = arm_sme.zero : vector<[2]x[2]xi64>
-  // CHECK-NEXT: "arm_sme.intr.zero"() <{tile_mask = 8 : i32}> : () -> ()
+  // CHECK: "arm_sme.intr.zero"() <{tile_mask = 8 : i32}> : () -> ()
   %zero_za3d = arm_sme.zero : vector<[2]x[2]xi64>
-  // CHECK-NEXT: "arm_sme.intr.zero"() <{tile_mask = 16 : i32}> : () -> ()
+  // CHECK: "arm_sme.intr.zero"() <{tile_mask = 16 : i32}> : () -> ()
   %zero_za4d = arm_sme.zero : vector<[2]x[2]xi64>
-  // CHECK-NEXT: "arm_sme.intr.zero"() <{tile_mask = 32 : i32}> : () -> ()
+  // CHECK: "arm_sme.intr.zero"() <{tile_mask = 32 : i32}> : () -> ()
   %zero_za5d = arm_sme.zero : vector<[2]x[2]xi64>
-  // CHECK-NEXT: "arm_sme.intr.zero"() <{tile_mask = 64 : i32}> : () -> ()
+  // CHECK: "arm_sme.intr.zero"() <{tile_mask = 64 : i32}> : () -> ()
   %zero_za6d = arm_sme.zero : vector<[2]x[2]xi64>
-  // CHECK-NEXT: "arm_sme.intr.zero"() <{tile_mask = 128 : i32}> : () -> ()
+  // CHECK: "arm_sme.intr.zero"() <{tile_mask = 128 : i32}> : () -> ()
   %zero_za7d = arm_sme.zero : vector<[2]x[2]xf64>
+  "test.some_use"(%zero_za0d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%zero_za1d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%zero_za2d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%zero_za3d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%zero_za4d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%zero_za5d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%zero_za6d) : (vector<[2]x[2]xi64>) -> ()
+  "test.some_use"(%zero_za7d) : (vector<[2]x[2]xf64>) -> ()
   return
 }
-- 
GitLab


From dd4bf22b9380e797362fac1415a1796da338b2db Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 8 May 2024 10:21:41 +0100
Subject: [PATCH 0153/1206] [X86] combineBlendOfPermutes - don't introduce
 lane-crossing permutes without AVX2 support.

Fixes #91433
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 19 ++++++++++------
 .../test/CodeGen/X86/vector-shuffle-256-v4.ll | 22 +++++++++++++++++++
 .../X86/vector-shuffle-combining-avx.ll       |  3 ++-
 3 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 8ec4984dfa55..3ae68c438aa7 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -40078,10 +40078,10 @@ static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL,
 
 // Attempt to fold BLEND(PERMUTE(X),PERMUTE(Y)) -> PERMUTE(BLEND(X,Y))
 // iff we don't demand the same element index for both X and Y.
-static SDValue combineBlendOfPermutes(MVT VT, SDValue N0, SDValue N1,
-                                      ArrayRef<int> BlendMask,
-                                      const APInt &DemandedElts,
-                                      SelectionDAG &DAG, const SDLoc &DL) {
+static SDValue
+combineBlendOfPermutes(MVT VT, SDValue N0, SDValue N1, ArrayRef<int> BlendMask,
+                       const APInt &DemandedElts, SelectionDAG &DAG,
+                       const X86Subtarget &Subtarget, const SDLoc &DL) {
   assert(isBlendOrUndef(BlendMask) && "Blend shuffle expected");
   if (!N0.hasOneUse() || !N1.hasOneUse())
     return SDValue();
@@ -40156,6 +40156,11 @@ static SDValue combineBlendOfPermutes(MVT VT, SDValue N0, SDValue N1,
       return SDValue();
   }
 
+  // Don't introduce lane-crossing permutes without AVX2.
+  if (VT.is256BitVector() && !Subtarget.hasAVX2() &&
+      isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), NewPermuteMask))
+    return SDValue();
+
   SDValue NewBlend =
       DAG.getVectorShuffle(VT, DL, DAG.getBitcast(VT, Ops0[0]),
                            DAG.getBitcast(VT, Ops1[0]), NewBlendMask);
@@ -41918,9 +41923,9 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
   case X86ISD::BLENDI: {
     SmallVector<int, 16> BlendMask;
     DecodeBLENDMask(NumElts, Op.getConstantOperandVal(2), BlendMask);
-    if (SDValue R = combineBlendOfPermutes(VT.getSimpleVT(), Op.getOperand(0),
-                                           Op.getOperand(1), BlendMask,
-                                           DemandedElts, TLO.DAG, SDLoc(Op)))
+    if (SDValue R = combineBlendOfPermutes(
+            VT.getSimpleVT(), Op.getOperand(0), Op.getOperand(1), BlendMask,
+            DemandedElts, TLO.DAG, Subtarget, SDLoc(Op)))
       return TLO.CombineTo(Op, R);
     break;
   }
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
index bc95fd42e6b8..ced9304f4c59 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
@@ -699,6 +699,28 @@ define <4 x double> @shuffle_v4f64_0437(<4 x double> %a, <4 x double> %b) {
   ret <4 x double> %shuffle
 }
 
+; PR91433
+define <4 x double> @shuffle_v4f64_2303(<4 x double> %a) {
+; AVX1-LABEL: shuffle_v4f64_2303:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shuffle_v4f64_2303:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,3]
+; AVX2-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v4f64_2303:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,3]
+; AVX512VL-NEXT:    retq
+  %shuffle = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 3>
+  ret <4 x double> %shuffle
+}
+
 define <4 x double> @shuffle_v4f64_0z3z(<4 x double> %a, <4 x double> %b) {
 ; ALL-LABEL: shuffle_v4f64_0z3z:
 ; ALL:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
index 81ce14132c87..0c65f756f296 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
@@ -308,8 +308,9 @@ define <4 x float> @combine_vpermilvar_4f32_as_insertps(<4 x float> %a0) {
 define <8 x i32> @combine_blend_of_permutes_v8i32(<4 x i64> %a0, <4 x i64> %a1) {
 ; AVX1-LABEL: combine_blend_of_permutes_v8i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4],ymm0[5,6],ymm1[7]
 ; AVX1-NEXT:    ret{{[l|q]}}
 ;
 ; AVX2-LABEL: combine_blend_of_permutes_v8i32:
-- 
GitLab


From 8f21294897befee48f9f72734ea1b0ad4c920aa0 Mon Sep 17 00:00:00 2001
From: YunQiang Su <syq@debian.org>
Date: Wed, 8 May 2024 17:30:14 +0800
Subject: [PATCH 0154/1206] MIPS: Use pcrel|sdata4 for eh_frame (#91291)

Gas uses encoding DW_EH_PE_absptr for PIC, and gnu ld converts it to
DW_EH_PE_sdata4|DW_EH_PE_pcrel.
LLD doesn't have this workarounding, thus complains
```
  relocation R_MIPS_32 cannot be used against local symbol; recompile with -fPIC
  relocation R_MIPS_64 cannot be used against local symbol; recompile with -fPIC
```

So, let's generates asm/obj files with `DW_EH_PE_sdata4|DW_EH_PE_pcrel`
encoding. In fact, GNU ld supports such OBJs well.

For N64, maybe we should use sdata8, while GNU ld doesn't support it
well, and in fact sdata4 is enough now. So we just ignore the `Large`
for `MCObjectFileInfo::initELFMCObjectFileInfo`. Maybe we should switch
back to sdata8 once GNU LD supports it well.

Fixes: #58377.
---
 lld/test/ELF/mips-eh_frame-pic.s                  | 8 +++++---
 llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp | 4 +---
 llvm/lib/MC/MCObjectFileInfo.cpp                  | 4 +++-
 llvm/test/CodeGen/Mips/ehframe-indirect.ll        | 6 +++---
 llvm/test/DebugInfo/Mips/eh_frame.ll              | 6 +++---
 llvm/test/MC/Mips/eh-frame.s                      | 9 ++++-----
 6 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/lld/test/ELF/mips-eh_frame-pic.s b/lld/test/ELF/mips-eh_frame-pic.s
index c04dbdf57b08..79076e74a7e3 100644
--- a/lld/test/ELF/mips-eh_frame-pic.s
+++ b/lld/test/ELF/mips-eh_frame-pic.s
@@ -16,7 +16,7 @@
 # RUN: llvm-mc -filetype=obj -triple=mips64-unknown-linux --position-independent %s -o %t-pic.o
 # RUN: llvm-readobj -r %t-pic.o | FileCheck %s --check-prefixes=RELOCS,PIC64-RELOCS
 # RUN: ld.lld -shared %t-pic.o -o %t-pic.so
-# RUN: llvm-dwarfdump --eh-frame %t-pic.so | FileCheck %s --check-prefix=PIC-EH-FRAME
+# RUN: llvm-dwarfdump --eh-frame %t-pic.so | FileCheck %s --check-prefix=PIC64-EH-FRAME
 
 ## Also check MIPS32:
 # RUN: llvm-mc -filetype=obj -triple=mips-unknown-linux %s -o %t-nopic32.o
@@ -31,7 +31,7 @@
 # RUN: llvm-mc -filetype=obj -triple=mips-unknown-linux --position-independent %s -o %t-pic32.o
 # RUN: llvm-readobj -r %t-pic32.o | FileCheck %s --check-prefixes=RELOCS,PIC32-RELOCS
 # RUN: ld.lld -shared %t-pic32.o -o %t-pic32.so
-# RUN: llvm-dwarfdump --eh-frame %t-pic32.so | FileCheck %s --check-prefix=PIC-EH-FRAME
+# RUN: llvm-dwarfdump --eh-frame %t-pic32.so | FileCheck %s --check-prefix=PIC32-EH-FRAME
 
 # RELOCS:            .rel{{a?}}.eh_frame {
 # ABS32-RELOCS-NEXT:   0x1C R_MIPS_32 .text
@@ -44,7 +44,9 @@
 ##                                   ^^ fde pointer encoding: DW_EH_PE_sdata8
 # ABS32-EH-FRAME: Augmentation data: 0B
 ##                                   ^^ fde pointer encoding: DW_EH_PE_sdata4
-# PIC-EH-FRAME: Augmentation data: 1B
+# PIC32-EH-FRAME: Augmentation data: 1B
+##                                 ^^ fde pointer encoding: DW_EH_PE_pcrel | DW_EH_PE_sdata4
+# PIC64-EH-FRAME: Augmentation data: 1B
 ##                                 ^^ fde pointer encoding: DW_EH_PE_pcrel | DW_EH_PE_sdata4
 ## Note: ld.bfd converts the R_MIPS_64 relocs to DW_EH_PE_pcrel | DW_EH_PE_sdata8
 ## for N64 ABI (and DW_EH_PE_pcrel | DW_EH_PE_sdata4 for MIPS32)
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 622773cc73f7..3e1897ce670a 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -212,13 +212,11 @@ void TargetLoweringObjectFileELF::Initialize(MCContext &Ctx,
     //        identify N64 from just a triple.
     TTypeEncoding = dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel |
                     dwarf::DW_EH_PE_sdata4;
-    // We don't support PC-relative LSDA references in GAS so we use the default
-    // DW_EH_PE_absptr for those.
 
     // FreeBSD must be explicit about the data size and using pcrel since it's
     // assembler/linker won't do the automatic conversion that the Linux tools
     // do.
-    if (TgtM.getTargetTriple().isOSFreeBSD()) {
+    if (isPositionIndependent() || TgtM.getTargetTriple().isOSFreeBSD()) {
       PersonalityEncoding |= dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4;
       LSDAEncoding = dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4;
     }
diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp
index 1f8f8ec55727..045b566aae78 100644
--- a/llvm/lib/MC/MCObjectFileInfo.cpp
+++ b/llvm/lib/MC/MCObjectFileInfo.cpp
@@ -343,7 +343,9 @@ void MCObjectFileInfo::initELFMCObjectFileInfo(const Triple &T, bool Large) {
   case Triple::mips64el:
     // We cannot use DW_EH_PE_sdata8 for the large PositionIndependent case
     // since there is no R_MIPS_PC64 relocation (only a 32-bit version).
-    if (PositionIndependent && !Large)
+    // In fact DW_EH_PE_sdata4 is enough for us now, and GNU ld doesn't
+    // support pcrel|sdata8 well. Let's use sdata4 for now.
+    if (PositionIndependent)
       FDECFIEncoding = dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4;
     else
       FDECFIEncoding = Ctx->getAsmInfo()->getCodePointerSize() == 4
diff --git a/llvm/test/CodeGen/Mips/ehframe-indirect.ll b/llvm/test/CodeGen/Mips/ehframe-indirect.ll
index e36fa2f9ce42..1cd2b86a8e15 100644
--- a/llvm/test/CodeGen/Mips/ehframe-indirect.ll
+++ b/llvm/test/CodeGen/Mips/ehframe-indirect.ll
@@ -17,9 +17,9 @@ define i32 @main() personality ptr @__gxx_personality_v0 {
 ; ALL: .cfi_startproc
 
 ; Linux must rely on the assembler/linker converting the encodings.
-; LINUX: .cfi_personality 128, DW.ref.__gxx_personality_v0
-; LINUX-O32: .cfi_lsda 0, $exception0
-; LINUX-NEW: .cfi_lsda 0, .Lexception0
+; LINUX: .cfi_personality 155, DW.ref.__gxx_personality_v0
+; LINUX-O32: .cfi_lsda 27, $exception0
+; LINUX-NEW: .cfi_lsda 27, .Lexception0
 
 ; FreeBSD can (and must) be more direct about the encodings it wants.
 ; FREEBSD: .cfi_personality 155, DW.ref.__gxx_personality_v0
diff --git a/llvm/test/DebugInfo/Mips/eh_frame.ll b/llvm/test/DebugInfo/Mips/eh_frame.ll
index 60d4dc76777e..d53bc156ef29 100644
--- a/llvm/test/DebugInfo/Mips/eh_frame.ll
+++ b/llvm/test/DebugInfo/Mips/eh_frame.ll
@@ -17,9 +17,9 @@
 ; STATIC-DAG: R_MIPS_32 00000000 .gcc_except_table
 
 ; PIC-LABEL: Relocation section '.rel.eh_frame'
-; PIC-DAG: R_MIPS_32   00000000 DW.ref.__gxx_personality_v0
-; PIC-DAG: R_MIPS_PC32
-; PIC-DAG: R_MIPS_32   00000000 .gcc_except_table
+; PIC-DAG: R_MIPS_PC32   00000000 DW.ref.__gxx_personality_v0
+; PIC-DAG: R_MIPS_PC32   00000000 .L0
+; PIC-DAG: R_MIPS_PC32   00000000 .L0
 
 ; CHECK-READELF: DW.ref.__gxx_personality_v0
 ; CHECK-READELF-STATIC-NEXT: R_MIPS_32 00000000 .text
diff --git a/llvm/test/MC/Mips/eh-frame.s b/llvm/test/MC/Mips/eh-frame.s
index fd145317bf4d..dac142325f9f 100644
--- a/llvm/test/MC/Mips/eh-frame.s
+++ b/llvm/test/MC/Mips/eh-frame.s
@@ -33,14 +33,13 @@
 // RUN: llvm-readobj -r %t.o | FileCheck --check-prefixes=RELOCS,PIC64 %s
 // RUN: llvm-dwarfdump -eh-frame %t.o | FileCheck --check-prefixes=DWARF64,DWARF64_PIC %s
 
-/// However using the large code model forces R_MIPS_64 since there is no R_MIPS_PC64 relocation:
 // RUN: llvm-mc -filetype=obj %s -o %t.o -triple mips64-unknown-linux-gnu --position-independent --large-code-model
-// RUN: llvm-readobj -r %t.o | FileCheck --check-prefixes=RELOCS,ABS64 %s
-// RUN: llvm-dwarfdump -eh-frame %t.o | FileCheck --check-prefixes=DWARF64,DWARF64_ABS %s
+// RUN: llvm-readobj -r %t.o | FileCheck --check-prefixes=RELOCS,PIC64 %s
+// RUN: llvm-dwarfdump -eh-frame %t.o | FileCheck --check-prefixes=DWARF64,DWARF64_PIC %s
 
 // RUN: llvm-mc -filetype=obj %s -o %t.o -triple mips64el-unknown-linux-gnu --position-independent  --large-code-model
-// RUN: llvm-readobj -r %t.o | FileCheck --check-prefixes=RELOCS,ABS64 %s
-// RUN: llvm-dwarfdump -eh-frame %t.o | FileCheck --check-prefixes=DWARF64,DWARF64_ABS %s
+// RUN: llvm-readobj -r %t.o | FileCheck --check-prefixes=RELOCS,PIC64 %s
+// RUN: llvm-dwarfdump -eh-frame %t.o | FileCheck --check-prefixes=DWARF64,DWARF64_PIC %s
 
 func:
 	.cfi_startproc
-- 
GitLab


From df311a27629618f6ba645bfe3f46e981000cb5da Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <42888396+aleks-tmb@users.noreply.github.com>
Date: Wed, 8 May 2024 11:54:49 +0200
Subject: [PATCH 0155/1206] Add interface to check if a call has a deopt bundle
 (NFC) (#91348)

Encapsulate check that a call has a deopt bundle to make it easier to
change the deopt scheme.
---
 llvm/include/llvm/IR/InstrTypes.h                      | 5 +++++
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp           | 2 +-
 llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp  | 4 ++--
 llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp | 3 +--
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index b9af3a6ca42c..eaade9ce4755 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -2614,6 +2614,11 @@ public:
   op_iterator populateBundleOperandInfos(ArrayRef<OperandBundleDef> Bundles,
                                          const unsigned BeginIndex);
 
+  /// Return true if the call has deopt state bundle.
+  bool hasDeoptState() const {
+    return getOperandBundle(LLVMContext::OB_deopt).has_value();
+  }
+
 public:
   /// Return the BundleOpInfo for the operand at index OpIdx.
   ///
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 6661127162e5..5289b993476d 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2851,7 +2851,7 @@ bool IRTranslator::translateInvoke(const User &U,
     return false;
 
   // FIXME: support whatever these are.
-  if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
+  if (I.hasDeoptState())
     return false;
 
   // FIXME: support control flow guard targets.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index f47aea29625f..55aed43070df 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3357,7 +3357,7 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
       break;
     }
     }
-  } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
+  } else if (I.hasDeoptState()) {
     // Currently we do not lower any intrinsic calls with deopt operand bundles.
     // Eventually we will support lowering the @llvm.experimental.deoptimize
     // intrinsic, and right now there are no plans to support other intrinsics
@@ -9197,7 +9197,7 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
 
   SDValue Callee = getValue(I.getCalledOperand());
 
-  if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
+  if (I.hasDeoptState())
     LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
   else
     // Check if we can potentially perform a tail call. More detailed checking
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index 286273c897aa..858e54c4a9bc 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -3046,8 +3046,7 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
       // which doesn't know how to produce a proper deopt state. So if we see a
       // non-leaf memcpy/memmove without deopt state just treat it as a leaf
       // copy and don't produce a statepoint.
-      if (!AllowStatepointWithNoDeoptInfo &&
-          !Call->getOperandBundle(LLVMContext::OB_deopt)) {
+      if (!AllowStatepointWithNoDeoptInfo && !Call->hasDeoptState()) {
         assert((isa<AtomicMemCpyInst>(Call) || isa<AtomicMemMoveInst>(Call)) &&
                "Don't expect any other calls here!");
         return false;
-- 
GitLab


From d4fef93724e290a82d498f0d8df1a84a5ff50ab3 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Wed, 8 May 2024 11:57:19 +0200
Subject: [PATCH 0156/1206] [bazel][libc] Split up mutex libraries like
 ab3a9e724d87a4272782f76b90fb0872a6a86939 did

---
 .../llvm-project-overlay/libc/BUILD.bazel     | 34 ++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index aa9f665c350a..055630cb6a00 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1090,6 +1090,38 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_threads_linux_futex_word_type",
+    hdrs = [
+        "src/__support/threads/linux/futex_word.h",
+    ],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    deps = [
+        ":__support_osutil_syscall",
+    ],
+)
+
+libc_support_library(
+    name = "__support_threads_linux_futex_utils",
+    hdrs = [
+        "src/__support/threads/linux/futex_utils.h",
+    ],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    deps = [
+        ":__support_cpp_atomic",
+        ":__support_cpp_optional",
+        ":__support_osutil_syscall",
+        ":__support_threads_linux_futex_word_type",
+        ":types_struct_timespec",
+    ],
+)
+
 libc_support_library(
     name = "__support_threads_mutex",
     hdrs = [
@@ -1102,11 +1134,11 @@ libc_support_library(
     }),
     textual_hdrs = [
         "src/__support/threads/linux/mutex.h",
-        "src/__support/threads/linux/futex_word.h",
     ],
     deps = [
         ":__support_cpp_atomic",
         ":__support_osutil_syscall",
+        ":__support_threads_linux_futex_utils",
     ],
 )
 
-- 
GitLab


From aefad851672e6dd17592895066a39aa5b388e5db Mon Sep 17 00:00:00 2001
From: Kadir Cetinkaya <kadircet@google.com>
Date: Wed, 8 May 2024 10:23:39 +0200
Subject: [PATCH 0157/1206] [clangd] Fix data race surfaced in clangd-tsan
 buildbot

We can have concurrent accesses to same PreambleData (e.g.
code-completion and ast-builds). Hence we need to
deep copy TargetOpts.
---
 clang-tools-extra/clangd/Preamble.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp
index d5818e0ca309..ecd490145dd3 100644
--- a/clang-tools-extra/clangd/Preamble.cpp
+++ b/clang-tools-extra/clangd/Preamble.cpp
@@ -918,7 +918,9 @@ void PreamblePatch::apply(CompilerInvocation &CI) const {
   // no guarantees around using arbitrary options when reusing PCHs, and
   // different target opts can result in crashes, see
   // ParsedASTTest.PreambleWithDifferentTarget.
-  CI.TargetOpts = Baseline->TargetOpts;
+  // Make sure this is a deep copy, as the same Baseline might be used
+  // concurrently.
+  *CI.TargetOpts = *Baseline->TargetOpts;
 
   // No need to map an empty file.
   if (PatchContents.empty())
-- 
GitLab


From 341aecc2dd0f6debcbe9f251a6d2e8a60d327eea Mon Sep 17 00:00:00 2001
From: Weaver <Tom.Weaver@sony.com>
Date: Wed, 8 May 2024 11:12:14 +0100
Subject: [PATCH 0158/1206] =?UTF-8?q?Revert=20"Revert=20"Revert=20"[OpenMP?=
 =?UTF-8?q?][TR12]=20change=20property=20of=20map-type=20modifier."?=
 =?UTF-8?q?=E2=80=A6=20(#91141)"?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit a99ce615f19fec6fbb835490b89f53cba3cf9eff.

Caused test failure on following buildbot:
https://lab.llvm.org/buildbot/#/builders/139/builds/65066
---
 .../clang/Basic/DiagnosticParseKinds.td       |   5 -
 clang/lib/Parse/ParseOpenMP.cpp               |  51 ++-------
 clang/test/OpenMP/target_ast_print.cpp        |  58 ----------
 clang/test/OpenMP/target_map_messages.cpp     | 105 ++++++++----------
 4 files changed, 54 insertions(+), 165 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 44bc4e0e130d..fdffb35ea0d9 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1438,9 +1438,6 @@ def err_omp_decl_in_declare_simd_variant : Error<
 def err_omp_sink_and_source_iteration_not_allowd: Error<" '%0 %select{sink:|source:}1' must be with '%select{omp_cur_iteration - 1|omp_cur_iteration}1'">;
 def err_omp_unknown_map_type : Error<
   "incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'">;
-def err_omp_more_one_map_type : Error<"map type is already specified">;
-def note_previous_map_type_specified_here
-    : Note<"map type '%0' is previous specified here">;
 def err_omp_unknown_map_type_modifier : Error<
   "incorrect map type modifier, expected one of: 'always', 'close', 'mapper'"
   "%select{|, 'present'|, 'present', 'iterator'}0%select{|, 'ompx_hold'}1">;
@@ -1448,8 +1445,6 @@ def err_omp_map_type_missing : Error<
   "missing map type">;
 def err_omp_map_type_modifier_missing : Error<
   "missing map type modifier">;
-def err_omp_map_modifier_specification_list : Error<
-  "empty modifier-specification-list is not allowed">;
 def err_omp_declare_simd_inbranch_notinbranch : Error<
   "unexpected '%0' clause, '%1' is specified already">;
 def err_omp_expected_clause_argument
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 5265d8f1922c..18ba1185ee8d 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -4228,20 +4228,13 @@ bool Parser::parseMapperModifier(SemaOpenMP::OpenMPVarListDataTy &Data) {
   return T.consumeClose();
 }
 
-static OpenMPMapClauseKind isMapType(Parser &P);
-
 /// Parse map-type-modifiers in map clause.
-/// map([ [map-type-modifier[,] [map-type-modifier[,] ...] [map-type] : ] list)
+/// map([ [map-type-modifier[,] [map-type-modifier[,] ...] map-type : ] list)
 /// where, map-type-modifier ::= always | close | mapper(mapper-identifier) |
 /// present
-/// where, map-type ::= alloc | delete | from | release | to | tofrom
 bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
-  bool HasMapType = false;
-  SourceLocation PreMapLoc = Tok.getLocation();
-  StringRef PreMapName = "";
   while (getCurToken().isNot(tok::colon)) {
     OpenMPMapModifierKind TypeModifier = isMapModifier(*this);
-    OpenMPMapClauseKind MapKind = isMapType(*this);
     if (TypeModifier == OMPC_MAP_MODIFIER_always ||
         TypeModifier == OMPC_MAP_MODIFIER_close ||
         TypeModifier == OMPC_MAP_MODIFIER_present ||
@@ -4264,19 +4257,6 @@ bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
         Diag(Data.MapTypeModifiersLoc.back(), diag::err_omp_missing_comma)
             << "map type modifier";
 
-    } else if (getLangOpts().OpenMP >= 60 && MapKind != OMPC_MAP_unknown) {
-      if (!HasMapType) {
-        HasMapType = true;
-        Data.ExtraModifier = MapKind;
-        MapKind = OMPC_MAP_unknown;
-        PreMapLoc = Tok.getLocation();
-        PreMapName = Tok.getIdentifierInfo()->getName();
-      } else {
-        Diag(Tok, diag::err_omp_more_one_map_type);
-        Diag(PreMapLoc, diag::note_previous_map_type_specified_here)
-            << PreMapName;
-      }
-      ConsumeToken();
     } else {
       // For the case of unknown map-type-modifier or a map-type.
       // Map-type is followed by a colon; the function returns when it
@@ -4287,14 +4267,8 @@ bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
         continue;
       }
       // Potential map-type token as it is followed by a colon.
-      if (PP.LookAhead(0).is(tok::colon)) {
-        if (getLangOpts().OpenMP >= 60) {
-          break;
-        } else {
-          return false;
-        }
-      }
-
+      if (PP.LookAhead(0).is(tok::colon))
+        return false;
       Diag(Tok, diag::err_omp_unknown_map_type_modifier)
           << (getLangOpts().OpenMP >= 51 ? (getLangOpts().OpenMP >= 52 ? 2 : 1)
                                          : 0)
@@ -4304,14 +4278,6 @@ bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
     if (getCurToken().is(tok::comma))
       ConsumeToken();
   }
-  if (getLangOpts().OpenMP >= 60 && !HasMapType) {
-    if (!Tok.is(tok::colon)) {
-      Diag(Tok, diag::err_omp_unknown_map_type);
-      ConsumeToken();
-    } else {
-      Data.ExtraModifier = OMPC_MAP_unknown;
-    }
-  }
   return false;
 }
 
@@ -4323,12 +4289,13 @@ static OpenMPMapClauseKind isMapType(Parser &P) {
   if (!Tok.isOneOf(tok::identifier, tok::kw_delete))
     return OMPC_MAP_unknown;
   Preprocessor &PP = P.getPreprocessor();
-  unsigned MapType =
-      getOpenMPSimpleClauseType(OMPC_map, PP.getSpelling(Tok), P.getLangOpts());
+  OpenMPMapClauseKind MapType =
+      static_cast<OpenMPMapClauseKind>(getOpenMPSimpleClauseType(
+          OMPC_map, PP.getSpelling(Tok), P.getLangOpts()));
   if (MapType == OMPC_MAP_to || MapType == OMPC_MAP_from ||
       MapType == OMPC_MAP_tofrom || MapType == OMPC_MAP_alloc ||
       MapType == OMPC_MAP_delete || MapType == OMPC_MAP_release)
-    return static_cast<OpenMPMapClauseKind>(MapType);
+    return MapType;
   return OMPC_MAP_unknown;
 }
 
@@ -4712,10 +4679,8 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     // Only parse map-type-modifier[s] and map-type if a colon is present in
     // the map clause.
     if (ColonPresent) {
-      if (getLangOpts().OpenMP >= 60 && getCurToken().is(tok::colon))
-        Diag(Tok, diag::err_omp_map_modifier_specification_list);
       IsInvalidMapperModifier = parseMapTypeModifiers(Data);
-      if (getLangOpts().OpenMP < 60 && !IsInvalidMapperModifier)
+      if (!IsInvalidMapperModifier)
         parseMapType(*this, Data);
       else
         SkipUntil(tok::colon, tok::annot_pragma_openmp_end, StopBeforeMatch);
diff --git a/clang/test/OpenMP/target_ast_print.cpp b/clang/test/OpenMP/target_ast_print.cpp
index ac5ed285d97e..f4c10fe3a181 100644
--- a/clang/test/OpenMP/target_ast_print.cpp
+++ b/clang/test/OpenMP/target_ast_print.cpp
@@ -1201,64 +1201,6 @@ foo();
 }
 #endif // OMP52
 
-#ifdef OMP60
-
-///==========================================================================///
-// RUN: %clang_cc1 -DOMP60 -verify -Wno-vla -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix OMP60
-// RUN: %clang_cc1 -DOMP60 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP60 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP60
-
-// RUN: %clang_cc1 -DOMP60 -verify -Wno-vla -fopenmp-simd -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix OMP60
-// RUN: %clang_cc1 -DOMP60 -fopenmp-simd -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP60 -fopenmp-simd -fopenmp-version=60 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP60
-
-void foo() {}
-template <typename T, int C>
-T tmain(T argc, T *argv) {
-  T i;
-#pragma omp target map(from always: i)
-  foo();
-#pragma omp target map(from, close: i)
-  foo();
-#pragma omp target map(always,close: i)
-  foo();
-  return 0;
-}
-//OMP60: template <typename T, int C> T tmain(T argc, T *argv) {
-//OMP60-NEXT: T i;
-//OMP60-NEXT: #pragma omp target map(always,from: i)
-//OMP60-NEXT:     foo();
-//OMP60-NEXT: #pragma omp target map(close,from: i)
-//OMP60-NEXT:     foo();
-//OMP60-NEXT: #pragma omp target map(always,close,tofrom: i)
-//OMP60-NEXT:     foo();
-//OMP60-NEXT: return 0;
-//OMP60-NEXT:}
-//OMP60:  template<> int tmain<int, 5>(int argc, int *argv) {
-//OMP60-NEXT:  int i;
-//OMP60-NEXT:  #pragma omp target map(always,from: i)
-//OMP60-NEXT:      foo();
-//OMP60-NEXT:  #pragma omp target map(close,from: i)
-//OMP60-NEXT:      foo();
-//OMP60-NEXT:  #pragma omp target map(always,close,tofrom: i)
-//OMP60-NEXT:      foo();
-//OMP60-NEXT:  return 0;
-//OMP60-NEXT:}
-//OMP60:  template<> char tmain<char, 1>(char argc, char *argv) {
-//OMP60-NEXT:  char i;
-//OMP60-NEXT:  #pragma omp target map(always,from: i)
-//OMP60-NEXT:      foo();
-//OMP60-NEXT:  #pragma omp target map(close,from: i)
-//OMP60-NEXT:      foo();
-//OMP60-NEXT:  #pragma omp target map(always,close,tofrom: i)
-//OMP60-NEXT:      foo();
-//OMP60-NEXT:  return 0;
-//OMP60-NEXT:}
-int main (int argc, char **argv) {
-  return tmain<int, 5>(argc, &argc) + tmain<char, 1>(argv[0][0], argv[0]);
-}
-#endif // OMP60
-
 #ifdef OMPX
 
 // RUN: %clang_cc1 -DOMPX -verify -Wno-vla -fopenmp -fopenmp-extensions -ast-print %s | FileCheck %s --check-prefix=OMPX
diff --git a/clang/test/OpenMP/target_map_messages.cpp b/clang/test/OpenMP/target_map_messages.cpp
index 3bd432b47e63..a6776ee12c0e 100644
--- a/clang/test/OpenMP/target_map_messages.cpp
+++ b/clang/test/OpenMP/target_map_messages.cpp
@@ -1,35 +1,34 @@
 // -fopenmp, -fno-openmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,omp,ge51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=51 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,ge52,lt60,omp,ge52-omp,omp52 -fopenmp -fno-openmp-extensions -fopenmp-version=52 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge52,ge60,omp,ge60-omp,omp60 -fopenmp -fno-openmp-extensions -fopenmp-version=60 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,omp,ge51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=51 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,ge52,omp,ge52-omp,omp52 -fopenmp -fno-openmp-extensions -fopenmp-version=52 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized -Wno-vla
 
 // -fopenmp-simd, -fno-openmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,omp,ge51-omp -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,omp,ge51-omp -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
 
 // -fopenmp -fopenmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,ompx,ge51-ompx -fopenmp -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,ompx,ge51-ompx -fopenmp -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized -Wno-vla
 
 // -fopenmp-simd -fopenmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,ompx,ge51-ompx -fopenmp-simd -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,ompx,ge51-ompx -fopenmp-simd -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
 
 // Check
@@ -114,7 +113,7 @@ struct SA {
     #pragma omp target map(b[true:true])
     {}
 
-    #pragma omp target map(: c,f) // lt60-error {{missing map type}} // ge60-error {{empty modifier-specification-list is not allowed}}
+    #pragma omp target map(: c,f) // expected-error {{missing map type}}
     {}
     #pragma omp target map(always, tofrom: c,f)
     {}
@@ -160,28 +159,28 @@ struct SA {
     // expected-error@+1 {{use of undeclared identifier 'present'}}
     #pragma omp target map(present)
     {}
-    // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-omp-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c,f)
     {}
-    // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-omp-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c[1:2],f)
     {}
-    // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-omp-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c,f[1:2])
     {}
-    // ge52-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-omp-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+3 {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c[:],f)
     {}
-    // ge52-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-omp-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+3 {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
@@ -194,19 +193,19 @@ struct SA {
     {}
     #pragma omp target map(always, close, always, close, tofrom: a)   // expected-error 2 {{same map type modifier has been specified more than once}}
     {}
-    // ge60-error@+3 {{same map type modifier has been specified more than once}}
     // ge51-error@+2 {{same map type modifier has been specified more than once}}
     // lt51-error@+1 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(present, present, tofrom: a)
     {}
-    // ge52-error@+4 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-omp-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-omp-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ompx-error@+3 {{same map type modifier has been specified more than once}}
     // ge51-omp-error@+2 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, ompx_hold, tofrom: a)
     {}
-    // ge60-error@+9 {{same map type modifier has been specified more than once}}
-    // ge52-error@+8 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-omp-error@+9 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-omp-error@+8 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+7 2 {{same map type modifier has been specified more than once}}
     // ge51-error@+6 {{same map type modifier has been specified more than once}}
     // lt51-ompx-error@+5 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'ompx_hold'}}
@@ -220,45 +219,34 @@ struct SA {
     {}
     #pragma omp target map( , , tofrom: a)   // expected-error {{missing map type modifier}} expected-error {{missing map type modifier}}
     {}
-    #pragma omp target map( , , : a)   // expected-error {{missing map type modifier}} expected-error {{missing map type modifier}} lt60-error {{missing map type}}
+    #pragma omp target map( , , : a)   // expected-error {{missing map type modifier}} expected-error {{missing map type modifier}} expected-error {{missing map type}}
     {}
-    // ge60-error@+4 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-error@+3 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+2 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     // expected-error@+1 {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
     #pragma omp target map( d, f, bf: a)
     {}
-    // ge60-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator}}
     // expected-error@+4 {{missing map type modifier}}
     // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-    // lt60-error@+1 {{missing map type}}
+    // expected-error@+1 {{missing map type}}
     #pragma omp target map( , f, : a)
     {}
-    #pragma omp target map(always close: a)   // lt60-error {{missing map type}} ge52-error{{missing ',' after map type modifier}}
+    #pragma omp target map(always close: a)   // expected-error {{missing map type}} omp52-error{{missing ',' after map type modifier}}
     {}
-    #pragma omp target map(always close bf: a)   // ge52-error 2 {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+    #pragma omp target map(always close bf: a)   // omp52-error 2 {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}} 
     {}
-    // ge52-error@+4 {{missing ',' after map type modifier}}
+    // omp52-error@+4 {{missing ',' after map type modifier}}
     // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-    // lt60-error@+1 {{missing map type}}
+    // expected-error@+1 {{missing map type}}
     #pragma omp target map(always tofrom close: a)
     {}
-    // ge60-note@+4 {{map type 'tofrom' is previous specified here}}
-    // ge60-error@+3 {{map type is already specified}}
     // ge51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(tofrom from: a)
     {}
-    // ge60-note@+5 {{map type 'to' is previous specified here}}
-    // ge60-error@+4 {{map type is already specified}}
-    // ge52-error@+3 {{missing ',' after map type modifier}}
-    // ge51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
-    // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-    #pragma omp target map(to always from: a)
-    {}
-    #pragma omp target map(close bf: a)   // ge52-error {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+    #pragma omp target map(close bf: a)   // omp52-error {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
     {}
     #pragma omp target map(([b[I]][bf])f)  // lt50-error {{expected ',' or ']' in lambda capture list}} lt50-error {{expected ')'}} lt50-note {{to match this '('}}
     {}
@@ -278,7 +266,6 @@ struct SA {
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(iterator(it=0:10, it=0:20), tofrom:a)
     {}
-    // ge60-error@+7 {{expected '(' after 'iterator'}}
     // ge51-ompx-error@+6 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'ompx_hold'}}
     // lt51-ompx-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'ompx_hold'}}
     // lt51-error@+4 {{expected '(' after 'iterator'}}
@@ -707,20 +694,20 @@ T tmain(T argc) {
   foo();
 
 #pragma omp target data map(always, tofrom: x)
-#pragma omp target data map(always: x) // lt60-error {{missing map type}}
+#pragma omp target data map(always: x) // expected-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// lt60-error@+1 {{missing map type}}
+// expected-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, always: x)
 #pragma omp target data map(always, tofrom: always, tofrom, x)
 #pragma omp target map(tofrom j) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
 
 #pragma omp target data map(close, tofrom: x)
-#pragma omp target data map(close: x) // lt60-error {{missing map type}}
+#pragma omp target data map(close: x) // expected-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// lt60-error@+1 {{missing map type}}
+// expected-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, close: x)
 #pragma omp target data map(close, tofrom: close, tofrom, x)
   foo();
@@ -842,19 +829,19 @@ int main(int argc, char **argv) {
   foo();
 
 #pragma omp target data map(always, tofrom: x)
-#pragma omp target data map(always: x) // lt60-error {{missing map type}}
+#pragma omp target data map(always: x) // expected-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// lt60-error@+1 {{missing map type}}
+// expected-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, always: x)
 #pragma omp target data map(always, tofrom: always, tofrom, x)
 #pragma omp target map(tofrom j) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
 #pragma omp target data map(close, tofrom: x)
-#pragma omp target data map(close: x) // lt60-error {{missing map type}}
+#pragma omp target data map(close: x) // expected-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// lt60-error@+1 {{missing map type}}
+// expected-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, close: x)
   foo();
 // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-- 
GitLab


From 927913fac74b671b5202eb00a52907d8445c7691 Mon Sep 17 00:00:00 2001
From: Harald van Dijk <harald.vandijk@codeplay.com>
Date: Wed, 8 May 2024 11:33:47 +0100
Subject: [PATCH 0159/1206] [RemoveDIs] Fix remapping of DbgLabelRecords.
 (#91447)

We already remapped DILocations for DbgVariableRecords, but
DbgLabelRecords have debug locations too that need to be mapped the same
way.
---
 llvm/lib/Transforms/Utils/ValueMapper.cpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp
index 6ebdd85d37b4..1c877ee937fb 100644
--- a/llvm/lib/Transforms/Utils/ValueMapper.cpp
+++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp
@@ -538,17 +538,20 @@ Value *Mapper::mapValue(const Value *V) {
 }
 
 void Mapper::remapDbgRecord(DbgRecord &DR) {
+  // Remap DILocations.
+  auto *MappedDILoc = mapMetadata(DR.getDebugLoc());
+  DR.setDebugLoc(DebugLoc(cast<DILocation>(MappedDILoc)));
+
   if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
+    // Remap labels.
     DLR->setLabel(cast<DILabel>(mapMetadata(DLR->getLabel())));
     return;
   }
 
   DbgVariableRecord &V = cast<DbgVariableRecord>(DR);
-  // Remap variables and DILocations.
+  // Remap variables.
   auto *MappedVar = mapMetadata(V.getVariable());
-  auto *MappedDILoc = mapMetadata(V.getDebugLoc());
   V.setVariable(cast<DILocalVariable>(MappedVar));
-  V.setDebugLoc(DebugLoc(cast<DILocation>(MappedDILoc)));
 
   bool IgnoreMissingLocals = Flags & RF_IgnoreMissingLocals;
 
-- 
GitLab


From 19220110acf5ed5cf8be035b7e4a7aed69f0adb1 Mon Sep 17 00:00:00 2001
From: "Felix (Ting Wang)" <Ting.Wang.SH@ibm.com>
Date: Wed, 8 May 2024 18:37:51 +0800
Subject: [PATCH 0160/1206] [PowerPC][AIX] Refactor existing logic to handle
 non-zero offsets for aix-small-local-dynamic-tls (#89182)

To enable optimized small local-dynamic access sequence for non-zero
offsets, this patch refactors existing
2a50921553798d2db52ca6330c89f0f8a5bc2215.
---
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp     |  45 +++---
 llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp   |  89 ++++++-----
 ...aix-small-local-dynamic-tls-largeaccess.ll | 148 +++++++-----------
 .../aix-small-local-dynamic-tls-types.ll      |   6 +-
 4 files changed, 129 insertions(+), 159 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 405326f4530a..a63824735490 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -205,8 +205,8 @@ public:
   void LowerPATCHPOINT(StackMaps &SM, const MachineInstr &MI);
   void EmitTlsCall(const MachineInstr *MI, MCSymbolRefExpr::VariantKind VK);
   void EmitAIXTlsCallHelper(const MachineInstr *MI);
-  const MCExpr *getAdjustedLocalExecExpr(const MachineOperand &MO,
-                                         int64_t Offset);
+  const MCExpr *getAdjustedFasterLocalExpr(const MachineOperand &MO,
+                                           int64_t Offset);
   bool runOnMachineFunction(MachineFunction &MF) override {
     Subtarget = &MF.getSubtarget<PPCSubtarget>();
     bool Changed = AsmPrinter::runOnMachineFunction(MF);
@@ -1598,7 +1598,8 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     // machine operand (which is a TargetGlobalTLSAddress) is expected to be
     // the same operand for both loads and stores.
     for (const MachineOperand &TempMO : MI->operands()) {
-      if (((TempMO.getTargetFlags() == PPCII::MO_TPREL_FLAG)) &&
+      if (((TempMO.getTargetFlags() == PPCII::MO_TPREL_FLAG ||
+            TempMO.getTargetFlags() == PPCII::MO_TLSLD_FLAG)) &&
           TempMO.getOperandNo() == 1)
         OpNum = 1;
     }
@@ -1634,8 +1635,8 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
   case PPC::ADDI8: {
     // A faster non-TOC-based local-[exec|dynamic] sequence is represented by
     // `addi` or a load/store instruction (that directly loads or stores off of
-    // the thread pointer) with an immediate operand having the MO_TPREL_FLAG.
-    // Such instructions do not otherwise arise.
+    // the thread pointer) with an immediate operand having the
+    // [MO_TPREL_FLAG|MO_TLSLD_FLAG]. Such instructions do not otherwise arise.
     if (!HasAIXSmallLocalTLS)
       break;
     bool IsMIADDI8 = MI->getOpcode() == PPC::ADDI8;
@@ -1647,7 +1648,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
         Flag == PPCII::MO_TPREL_PCREL_FLAG || Flag == PPCII::MO_TLSLD_FLAG) {
       LowerPPCMachineInstrToMCInst(MI, TmpInst, *this);
 
-      const MCExpr *Expr = getAdjustedLocalExecExpr(MO, MO.getOffset());
+      const MCExpr *Expr = getAdjustedFasterLocalExpr(MO, MO.getOffset());
       if (Expr)
         TmpInst.getOperand(OpNum) = MCOperand::createExpr(Expr);
 
@@ -1677,14 +1678,15 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
   EmitToStreamer(*OutStreamer, TmpInst);
 }
 
-// For non-TOC-based local-exec variables that have a non-zero offset,
+// For non-TOC-based local-[exec|dynamic] variables that have a non-zero offset,
 // we need to create a new MCExpr that adds the non-zero offset to the address
-// of the local-exec variable that will be used in either an addi, load or
-// store. However, the final displacement for these instructions must be
+// of the local-[exec|dynamic] variable that will be used in either an addi,
+// load or store. However, the final displacement for these instructions must be
 // between [-32768, 32768), so if the TLS address + its non-zero offset is
 // greater than 32KB, a new MCExpr is produced to accommodate this situation.
-const MCExpr *PPCAsmPrinter::getAdjustedLocalExecExpr(const MachineOperand &MO,
-                                                      int64_t Offset) {
+const MCExpr *
+PPCAsmPrinter::getAdjustedFasterLocalExpr(const MachineOperand &MO,
+                                          int64_t Offset) {
   // Non-zero offsets (for loads, stores or `addi`) require additional handling.
   // When the offset is zero, there is no need to create an adjusted MCExpr.
   if (!Offset)
@@ -1692,13 +1694,9 @@ const MCExpr *PPCAsmPrinter::getAdjustedLocalExecExpr(const MachineOperand &MO,
 
   assert(MO.isGlobal() && "Only expecting a global MachineOperand here!");
   const GlobalValue *GValue = MO.getGlobal();
-  // TODO: Handle the aix-small-local-dynamic-tls non-zero offset case.
   TLSModel::Model Model = TM.getTLSModel(GValue);
-  if (Model == TLSModel::LocalDynamic) {
-    return nullptr;
-  }
-  assert(Model == TLSModel::LocalExec &&
-         "Only local-exec accesses are handled!");
+  assert((Model == TLSModel::LocalExec || Model == TLSModel::LocalDynamic) &&
+         "Only local-[exec|dynamic] accesses are handled!");
 
   bool IsGlobalADeclaration = GValue->isDeclarationForLinker();
   // Find the GlobalVariable that corresponds to the particular TLS variable
@@ -1719,7 +1717,10 @@ const MCExpr *PPCAsmPrinter::getAdjustedLocalExecExpr(const MachineOperand &MO,
   // For when TLS variables are extern, this is safe to do because we can
   // assume that the address of extern TLS variables are zero.
   const MCExpr *Expr = MCSymbolRefExpr::create(
-      getSymbol(GValue), MCSymbolRefExpr::VK_PPC_AIX_TLSLE, OutContext);
+      getSymbol(GValue),
+      Model == TLSModel::LocalExec ? MCSymbolRefExpr::VK_PPC_AIX_TLSLE
+                                   : MCSymbolRefExpr::VK_PPC_AIX_TLSLD,
+      OutContext);
   Expr = MCBinaryExpr::createAdd(
       Expr, MCConstantExpr::create(Offset, OutContext), OutContext);
   if (FinalAddress >= 32768) {
@@ -1732,10 +1733,10 @@ const MCExpr *PPCAsmPrinter::getAdjustedLocalExecExpr(const MachineOperand &MO,
     ptrdiff_t Delta = ((FinalAddress + 32768) & ~0xFFFF);
     // Check that the total instruction displacement fits within [-32768,32768).
     [[maybe_unused]] ptrdiff_t InstDisp = TLSVarAddress + Offset - Delta;
-    assert(((InstDisp < 32768) &&
-            (InstDisp >= -32768)) &&
-               "Expecting the instruction displacement for local-exec TLS "
-               "variables to be between [-32768, 32768)!");
+    assert(
+        ((InstDisp < 32768) && (InstDisp >= -32768)) &&
+        "Expecting the instruction displacement for local-[exec|dynamic] TLS "
+        "variables to be between [-32768, 32768)!");
     Expr = MCBinaryExpr::createAdd(
         Expr, MCConstantExpr::create(-Delta, OutContext), OutContext);
   }
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 48ee8b5d8d81..68621558e3fa 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -7587,29 +7587,23 @@ static bool hasAIXSmallTLSAttr(SDValue Val) {
   return false;
 }
 
-// Is an ADDI eligible for folding for non-TOC-based local-exec accesses?
-static bool isEligibleToFoldADDIForLocalExecAccesses(SelectionDAG *DAG,
-                                                     SDValue ADDIToFold) {
+// Is an ADDI eligible for folding for non-TOC-based local-[exec|dynamic]
+// accesses?
+static bool isEligibleToFoldADDIForFasterLocalAccesses(SelectionDAG *DAG,
+                                                       SDValue ADDIToFold) {
   // Check if ADDIToFold (the ADDI that we want to fold into local-exec
   // accesses), is truly an ADDI.
   if (!ADDIToFold.isMachineOpcode() ||
       (ADDIToFold.getMachineOpcode() != PPC::ADDI8))
     return false;
 
-  // Folding is only allowed for the AIX small-local-exec TLS target attribute
-  // or when the 'aix-small-tls' global variable attribute is present.
+  // Folding is only allowed for the AIX small-local-[exec|dynamic] TLS target
+  // attribute or when the 'aix-small-tls' global variable attribute is present.
   const PPCSubtarget &Subtarget =
       DAG->getMachineFunction().getSubtarget<PPCSubtarget>();
   SDValue TLSVarNode = ADDIToFold.getOperand(1);
-  if (!(Subtarget.hasAIXSmallLocalExecTLS() || hasAIXSmallTLSAttr(TLSVarNode)))
-    return false;
-
-  // The first operand of the ADDIToFold should be the thread pointer.
-  // This transformation is only performed if the first operand of the
-  // addi is the thread pointer.
-  SDValue TPRegNode = ADDIToFold.getOperand(0);
-  RegisterSDNode *TPReg = dyn_cast<RegisterSDNode>(TPRegNode.getNode());
-  if (!TPReg || (TPReg->getReg() != Subtarget.getThreadPointerRegister()))
+  if (!(Subtarget.hasAIXSmallLocalDynamicTLS() ||
+        Subtarget.hasAIXSmallLocalExecTLS() || hasAIXSmallTLSAttr(TLSVarNode)))
     return false;
 
   // The second operand of the ADDIToFold should be the global TLS address
@@ -7619,24 +7613,36 @@ static bool isEligibleToFoldADDIForLocalExecAccesses(SelectionDAG *DAG,
   if (!GA)
     return false;
 
-  // The local-exec TLS variable should only have the MO_TPREL_FLAG target flag,
-  // so this optimization is not performed otherwise if the flag is not set.
+  if (DAG->getTarget().getTLSModel(GA->getGlobal()) == TLSModel::LocalExec) {
+    // The first operand of the ADDIToFold should be the thread pointer.
+    // This transformation is only performed if the first operand of the
+    // addi is the thread pointer.
+    SDValue TPRegNode = ADDIToFold.getOperand(0);
+    RegisterSDNode *TPReg = dyn_cast<RegisterSDNode>(TPRegNode.getNode());
+    if (!TPReg || (TPReg->getReg() != Subtarget.getThreadPointerRegister()))
+      return false;
+  }
+
+  // The local-[exec|dynamic] TLS variable should only have the
+  // [MO_TPREL_FLAG|MO_TLSLD_FLAG] target flags, so this optimization is not
+  // performed otherwise if the flag is not set.
   unsigned TargetFlags = GA->getTargetFlags();
-  if (TargetFlags != PPCII::MO_TPREL_FLAG)
+  if (!(TargetFlags == PPCII::MO_TPREL_FLAG ||
+        TargetFlags == PPCII::MO_TLSLD_FLAG))
     return false;
 
   // If all conditions are satisfied, the ADDI is valid for folding.
   return true;
 }
 
-// For non-TOC-based local-exec access where an addi is feeding into another
-// addi, fold this sequence into a single addi if possible.
-// Before this optimization, the sequence appears as:
-//    addi rN, r13, sym@le
+// For non-TOC-based local-[exec|dynamic] access where an addi is feeding into
+// another addi, fold this sequence into a single addi if possible. Before this
+// optimization, the sequence appears as:
+//    addi rN, r13, sym@[le|ld]
 //    addi rM, rN, imm
 // After this optimization, we can fold the two addi into a single one:
-//    addi rM, r13, sym@le + imm
-static void foldADDIForLocalExecAccesses(SDNode *N, SelectionDAG *DAG) {
+//    addi rM, r13, sym@[le|ld] + imm
+static void foldADDIForFasterLocalAccesses(SDNode *N, SelectionDAG *DAG) {
   if (N->getMachineOpcode() != PPC::ADDI8)
     return;
 
@@ -7644,27 +7650,17 @@ static void foldADDIForLocalExecAccesses(SDNode *N, SelectionDAG *DAG) {
   // we want optimized out.
   SDValue InitialADDI = N->getOperand(0);
 
-  if (!isEligibleToFoldADDIForLocalExecAccesses(DAG, InitialADDI))
+  if (!isEligibleToFoldADDIForFasterLocalAccesses(DAG, InitialADDI))
     return;
 
-  // At this point, InitialADDI can be folded into a non-TOC-based local-exec
-  // access. The first operand of InitialADDI should be the thread pointer,
-  // which has been checked in isEligibleToFoldADDIForLocalExecAccesses().
-  SDValue TPRegNode = InitialADDI.getOperand(0);
-  [[maybe_unused]] RegisterSDNode *TPReg = dyn_cast<RegisterSDNode>(TPRegNode.getNode());
-  [[maybe_unused]] const PPCSubtarget &Subtarget =
-      DAG->getMachineFunction().getSubtarget<PPCSubtarget>();
-  assert((TPReg && (TPReg->getReg() == Subtarget.getThreadPointerRegister())) &&
-         "Expecting the first operand to be a thread pointer for folding addi "
-         "in local-exec accesses!");
-
   // The second operand of the InitialADDI should be the global TLS address
-  // (the local-exec TLS variable), with the MO_TPREL_FLAG target flag.
-  // This has been checked in isEligibleToFoldADDIForLocalExecAccesses().
+  // (the local-[exec|dynamic] TLS variable), with the
+  // [MO_TPREL_FLAG|MO_TLSLD_FLAG] target flag. This has been checked in
+  // isEligibleToFoldADDIForFasterLocalAccesses().
   SDValue TLSVarNode = InitialADDI.getOperand(1);
   GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(TLSVarNode);
   assert(GA && "Expecting a valid GlobalAddressSDNode when folding addi into "
-               "local-exec accesses!");
+               "local-[exec|dynamic] accesses!");
   unsigned TargetFlags = GA->getTargetFlags();
 
   // The second operand of the addi that we want to preserve will be an
@@ -7676,7 +7672,7 @@ static void foldADDIForLocalExecAccesses(SDNode *N, SelectionDAG *DAG) {
   TLSVarNode = DAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(GA), MVT::i64,
                                            Offset, TargetFlags);
 
-  (void)DAG->UpdateNodeOperands(N, TPRegNode, TLSVarNode);
+  (void)DAG->UpdateNodeOperands(N, InitialADDI.getOperand(0), TLSVarNode);
   if (InitialADDI.getNode()->use_empty())
     DAG->RemoveDeadNode(InitialADDI.getNode());
 }
@@ -7693,8 +7689,9 @@ void PPCDAGToDAGISel::PeepholePPC64() {
     if (isVSXSwap(SDValue(N, 0)))
       reduceVSXSwap(N, CurDAG);
 
-    // This optimization is performed for non-TOC-based local-exec accesses.
-    foldADDIForLocalExecAccesses(N, CurDAG);
+    // This optimization is performed for non-TOC-based local-[exec|dynamic]
+    // accesses.
+    foldADDIForFasterLocalAccesses(N, CurDAG);
 
     unsigned FirstOp;
     unsigned StorageOpcode = N->getMachineOpcode();
@@ -7852,13 +7849,15 @@ void PPCDAGToDAGISel::PeepholePPC64() {
         ImmOpnd = CurDAG->getTargetConstant(Offset, SDLoc(ImmOpnd),
                                             ImmOpnd.getValueType());
       } else if (Offset != 0) {
-        // This optimization is performed for non-TOC-based local-exec accesses.
-        if (isEligibleToFoldADDIForLocalExecAccesses(CurDAG, Base)) {
+        // This optimization is performed for non-TOC-based local-[exec|dynamic]
+        // accesses.
+        if (isEligibleToFoldADDIForFasterLocalAccesses(CurDAG, Base)) {
           // Add the non-zero offset information into the load or store
-          // instruction to be used for non-TOC-based local-exec accesses.
+          // instruction to be used for non-TOC-based local-[exec|dynamic]
+          // accesses.
           GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd);
           assert(GA && "Expecting a valid GlobalAddressSDNode when folding "
-                       "addi into local-exec accesses!");
+                       "addi into local-[exec|dynamic] accesses!");
           ImmOpnd = CurDAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(GA),
                                                    MVT::i64, Offset,
                                                    GA->getTargetFlags());
diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll
index d3fa94779dd7..44d62124ac58 100644
--- a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll
@@ -39,23 +39,18 @@ define signext i32 @test1() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r6, 4
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r5, 1
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r4, ElementIntTLSv1[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r5, ElementIntTLSv1[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r5, ElementIntTLS2[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r6, 24(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r5, 4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, ElementIntTLSv1[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 2
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 320(r5)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r4, ElementIntTLS3[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r5, 3
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r5, 324(r4)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r4, ElementIntTLS4[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementIntTLS5[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r6, 328(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r5, ElementIntTLSv1[TL]@ld+24(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r5, (ElementIntTLS4[TL]@ld+328)-65536(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, (ElementIntTLS2[TL]@ld+320)-65536(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, (ElementIntTLS3[TL]@ld+324)-65536(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 88
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 332(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, (ElementIntTLS5[TL]@ld+332)-65536(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 102
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
@@ -68,24 +63,19 @@ define signext i32 @test1() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r6, 4
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r5, 1
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r4, ElementIntTLSv1[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, ElementIntTLSv1[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r5, ElementIntTLS2[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r6, 24(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 1
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r5, 4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, ElementIntTLSv1[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 2
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 320(r5)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r4, ElementIntTLS3[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r5, 3
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, 324(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r4, ElementIntTLS4[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementIntTLS5[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r6, 328(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, ElementIntTLSv1[TL]@ld+24(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, (ElementIntTLS4[TL]@ld+328)-65536(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, (ElementIntTLS2[TL]@ld+320)-65536(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, (ElementIntTLS3[TL]@ld+324)-65536(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 88
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 332(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, (ElementIntTLS5[TL]@ld+332)-65536(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 102
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
@@ -132,26 +122,21 @@ define i64 @test2() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mr r6, r3
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementLongTLS6[UL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 212
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 424(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementLongTLS2[TL]@ld(r6)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 212
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 203
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 1200(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tlsgdm) @MyTLSGDVar
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tlsgd) @MyTLSGDVar
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, (ElementLongTLS2[TL]@ld+1200)-131072(r6)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C1(r2) # target-flags(ppc-tlsgd) @MyTLSGDVar
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r3, ElementLongTLS6[UL]@ld+424(r6)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C2(r2) # target-flags(ppc-tlsgdm) @MyTLSGDVar
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_addr[PR]
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 44
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 440(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementLongTLS3[TL]@ld(r6)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 6
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 2000(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementLongTLS4[TL]@ld(r6)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 6
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 100
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 6800(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementLongTLS5[TL]@ld(r6)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 882
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 8400(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r3, (ElementLongTLS3[TL]@ld+2000)-196608(r6)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 882
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, (ElementLongTLS4[TL]@ld+6800)-196608(r6)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r3, (ElementLongTLS5[TL]@ld+8400)-196608(r6)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 1191
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
@@ -166,29 +151,24 @@ define i64 @test2() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 212
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r4, L..C1@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mr r6, r3
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementLongTLS6[UL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 424(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementLongTLS2[TL]@ld(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 203
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 1200(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r4, L..C2@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C1@l(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C2@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 212
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C1@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r3, ElementLongTLS6[UL]@ld+424(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 203
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r3, (ElementLongTLS2[TL]@ld+1200)-131072(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C2@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C2@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_addr[PR]
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 44
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 440(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementLongTLS3[TL]@ld(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 6
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 2000(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementLongTLS4[TL]@ld(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 6
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 100
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 6800(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementLongTLS5[TL]@ld(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 882
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 8400(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r3, (ElementLongTLS3[TL]@ld+2000)-196608(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 882
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, (ElementLongTLS4[TL]@ld+6800)-196608(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r3, (ElementLongTLS5[TL]@ld+8400)-196608(r6)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 1191
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
@@ -230,23 +210,19 @@ define signext i32 @test3() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r6, 2
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r6, 3
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r5, 2
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 1
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r5, ElementIntTLS2[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r7, ElementIntTLS3[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r6, 320(r5)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r5, 3
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r6, ElementIntTLS4[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r5, 324(r7)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r5, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLSv2
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r7, ElementIntTLS5[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stwux r4, r3, r5
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 4
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 24(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 88
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 328(r6)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 332(r7)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r6, (ElementIntTLS3[TL]@ld+324)-65536(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r6, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLSv2
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r5, (ElementIntTLS2[TL]@ld+320)-65536(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r5, 88
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r5, (ElementIntTLS5[TL]@ld+332)-65536(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r5, 4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r5, (ElementIntTLS4[TL]@ld+328)-65536(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stwux r4, r3, r6
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r5, 24(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 102
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
@@ -262,22 +238,18 @@ define signext i32 @test3() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C3@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r6, L..C3@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r7, 3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r5, 2
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r4, ElementIntTLS2[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, 320(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 1
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r5, ElementIntTLS3[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r7, 324(r5)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r5, ElementIntTLS4[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r7, ElementIntTLS5[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, (ElementIntTLS2[TL]@ld+320)-65536(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r5, 3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, (ElementIntTLS3[TL]@ld+324)-65536(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r5, 88
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, (ElementIntTLS5[TL]@ld+332)-65536(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r5, 4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, (ElementIntTLS4[TL]@ld+328)-65536(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stwux r4, r3, r6
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 4
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 24(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 88
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 328(r5)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 332(r7)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, 24(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 102
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll
index 161a58a90296..489260b4e0ae 100644
--- a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll
@@ -51,8 +51,7 @@ define nonnull ptr @AddrTest1() local_unnamed_addr {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, a[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r3, r3, 12
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, a[TL]@ld+12(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
@@ -66,8 +65,7 @@ define nonnull ptr @AddrTest1() local_unnamed_addr {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, a[TL]@ld(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r3, r3, 12
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, a[TL]@ld+12(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
-- 
GitLab


From 6fa09616da7436f85eb7e1e1fd74e1ac078ddb0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Don=C3=A1t=20Nagy?= <donat.nagy@ericsson.com>
Date: Wed, 8 May 2024 12:38:33 +0200
Subject: [PATCH 0161/1206] [analyzer] Use explicit call description mode in
 MIGChecker (#91331)

This commit explicitly specifies the matching mode (C library function,
any non-method function, or C++ method) for the `CallDescription`s
constructed in the checker `osx.MIG`.

The code was simplified to use a `CallDescriptionMap` instead of a raw
vector of pairs.

This change won't cause major functional changes, but isn't NFC because
it ensures that e.g. call descriptions for a non-method function won't
accidentally match a method that has the same name.

Separate commits have already performed this change in other checkers:
- easy cases: e2f1cbae45f81f3cd9a4d3c2bcf69a094eb060fa,
    6d64f8e1feee014e72730a78b62d9d415df112ff
- MallocChecker: d6d84b5d1448e4f2e24b467a0abcf42fe9d543e9
- iterator checkers: 06eedffe0d2782922e63cc25cb927f4acdaf7b30
- InvalidPtr checker: 024281d4d26344f9613b9115ea1fcbdbdba23235
- apiModeling.llvm.ReturnValue: 97dd8e3c4f38ef345b01fbbf0a2052c7875ff7e0

... and follow-up commits will handle the remaining few checkers.

My goal is to ensure that the call description mode is always explicitly
specified and eliminate (or strongly restrict) the vague "may be either
a method or a simple function" mode that's the current default.
---
 .../StaticAnalyzer/Checkers/MIGChecker.cpp    | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/MIGChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MIGChecker.cpp
index 153a0a51e980..9757a00f1fb2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MIGChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MIGChecker.cpp
@@ -46,13 +46,13 @@ class MIGChecker : public Checker<check::PostCall, check::PreStmt<ReturnStmt>,
   // additionally an argument of a MIG routine, the checker keeps track of that
   // information and issues a warning when an error is returned from the
   // respective routine.
-  std::vector<std::pair<CallDescription, unsigned>> Deallocators = {
+  CallDescriptionMap<unsigned> Deallocators = {
 #define CALL(required_args, deallocated_arg, ...)                              \
-  {{{__VA_ARGS__}, required_args}, deallocated_arg}
-      // E.g., if the checker sees a C function 'vm_deallocate' that is
-      // defined on class 'IOUserClient' that has exactly 3 parameters, it knows
-      // that argument #1 (starting from 0, i.e. the second argument) is going
-      // to be consumed in the sense of the MIG consume-on-success convention.
+  {{CDM::SimpleFunc, {__VA_ARGS__}, required_args}, deallocated_arg}
+      // E.g., if the checker sees a C function 'vm_deallocate' that has
+      // exactly 3 parameters, it knows that argument #1 (starting from 0, i.e.
+      // the second argument) is going to be consumed in the sense of the MIG
+      // consume-on-success convention.
       CALL(3, 1, "vm_deallocate"),
       CALL(3, 1, "mach_vm_deallocate"),
       CALL(2, 0, "mig_deallocate"),
@@ -78,6 +78,9 @@ class MIGChecker : public Checker<check::PostCall, check::PreStmt<ReturnStmt>,
       CALL(1, 0, "thread_inspect_deallocate"),
       CALL(1, 0, "upl_deallocate"),
       CALL(1, 0, "vm_map_deallocate"),
+#undef CALL
+#define CALL(required_args, deallocated_arg, ...)                              \
+  {{CDM::CXXMethod, {__VA_ARGS__}, required_args}, deallocated_arg}
       // E.g., if the checker sees a method 'releaseAsyncReference64()' that is
       // defined on class 'IOUserClient' that takes exactly 1 argument, it knows
       // that the argument is going to be consumed in the sense of the MIG
@@ -87,7 +90,7 @@ class MIGChecker : public Checker<check::PostCall, check::PreStmt<ReturnStmt>,
 #undef CALL
   };
 
-  CallDescription OsRefRetain{{"os_ref_retain"}, 1};
+  CallDescription OsRefRetain{CDM::SimpleFunc, {"os_ref_retain"}, 1};
 
   void checkReturnAux(const ReturnStmt *RS, CheckerContext &C) const;
 
@@ -198,15 +201,12 @@ void MIGChecker::checkPostCall(const CallEvent &Call, CheckerContext &C) const {
   if (!isInMIGCall(C))
     return;
 
-  auto I = llvm::find_if(Deallocators,
-                         [&](const std::pair<CallDescription, unsigned> &Item) {
-                           return Item.first.matches(Call);
-                         });
-  if (I == Deallocators.end())
+  const unsigned *ArgIdxPtr = Deallocators.lookup(Call);
+  if (!ArgIdxPtr)
     return;
 
   ProgramStateRef State = C.getState();
-  unsigned ArgIdx = I->second;
+  unsigned ArgIdx = *ArgIdxPtr;
   SVal Arg = Call.getArgSVal(ArgIdx);
   const ParmVarDecl *PVD = getOriginParam(Arg, C);
   if (!PVD || State->contains<RefCountedParameters>(PVD))
-- 
GitLab


From 943617d12ccbd3cf317f0bbec03d9efc700f3953 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Wed, 8 May 2024 08:03:22 -0400
Subject: [PATCH 0162/1206] Typo fix; NFC

---
 clang/lib/CodeGen/CGExprAgg.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index cd9936a6dc0b..6172eb9cdc1b 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -1736,7 +1736,7 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
       for (const auto *Field : record->fields())
         assert(
             (Field->isUnnamedBitField() || Field->isAnonymousStructOrUnion()) &&
-            "Only unnamed bitfields or ananymous class allowed");
+            "Only unnamed bitfields or anonymous class allowed");
 #endif
       return;
     }
-- 
GitLab


From c5509fedc5757fffece385d9d068e36b26793ade Mon Sep 17 00:00:00 2001
From: Xiang Li <python3kgae@outlook.com>
Date: Wed, 8 May 2024 05:26:34 -0700
Subject: [PATCH 0163/1206] [HLSL] Support packoffset attribute in AST (#89836)

Add HLSLPackOffsetAttr to save packoffset in AST.

Since we have to parse the attribute manually in ParseHLSLAnnotations,
we could create the ParsedAttribute with a integer offset parameter
instead of string. This approach avoids parsing the string if the offset
is saved as a string in HLSLPackOffsetAttr.

For #57914.
---
 clang/include/clang/Basic/Attr.td             |  12 ++
 clang/include/clang/Basic/AttrDocs.td         |  20 +++
 clang/include/clang/Basic/DiagnosticGroups.td |   3 +
 .../clang/Basic/DiagnosticParseKinds.td       |   2 +
 .../clang/Basic/DiagnosticSemaKinds.td        |   5 +
 clang/lib/Parse/ParseHLSL.cpp                 |  88 +++++++++++++
 clang/lib/Sema/SemaDeclAttr.cpp               |  52 ++++++++
 clang/lib/Sema/SemaHLSL.cpp                   |  80 ++++++++++++
 clang/test/AST/HLSL/packoffset.hlsl           | 100 ++++++++++++++
 clang/test/SemaHLSL/packoffset-invalid.hlsl   | 122 ++++++++++++++++++
 10 files changed, 484 insertions(+)
 create mode 100644 clang/test/AST/HLSL/packoffset.hlsl
 create mode 100644 clang/test/SemaHLSL/packoffset-invalid.hlsl

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 0225598cbbe8..52552ba48856 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4415,6 +4415,18 @@ def HLSLResourceBinding: InheritableAttr {
   let Documentation = [HLSLResourceBindingDocs];
 }
 
+def HLSLPackOffset: HLSLAnnotationAttr {
+  let Spellings = [HLSLAnnotation<"packoffset">];
+  let LangOpts = [HLSL];
+  let Args = [IntArgument<"Subcomponent">, IntArgument<"Component">];
+  let Documentation = [HLSLPackOffsetDocs];
+  let AdditionalMembers = [{
+      unsigned getOffset() {
+        return subcomponent * 4 + component;
+      }
+  }];
+}
+
 def HLSLSV_DispatchThreadID: HLSLAnnotationAttr {
   let Spellings = [HLSLAnnotation<"SV_DispatchThreadID">];
   let Subjects = SubjectList<[ParmVar, Field]>;
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 8e6faabfae64..f351822ac74b 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -7408,6 +7408,26 @@ The full documentation is available here: https://docs.microsoft.com/en-us/windo
   }];
 }
 
+def HLSLPackOffsetDocs : Documentation {
+  let Category = DocCatFunction;
+  let Content = [{
+The packoffset attribute is used to change the layout of a cbuffer.
+Attribute spelling in HLSL is: ``packoffset( c[Subcomponent][.component] )``.
+A subcomponent is a register number, which is an integer. A component is in the form of [.xyzw].
+
+Examples:
+
+.. code-block:: c++
+
+  cbuffer A {
+    float3 a : packoffset(c0.y);
+    float4 b : packoffset(c4);
+  }
+
+The full documentation is available here: https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-variable-packoffset
+  }];
+}
+
 def HLSLSV_DispatchThreadIDDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 60f87da2a738..2beb1d45124b 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -1507,6 +1507,9 @@ def BranchProtection : DiagGroup<"branch-protection">;
 // Warnings for HLSL Clang extensions
 def HLSLExtension : DiagGroup<"hlsl-extensions">;
 
+// Warning for mix packoffset and non-packoffset.
+def HLSLMixPackOffset : DiagGroup<"mix-packoffset">;
+
 // Warnings for DXIL validation
 def DXILValidation : DiagGroup<"dxil-validation">;
 
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index fdffb35ea0d9..bc9d7cacc50b 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1754,5 +1754,7 @@ def err_hlsl_separate_attr_arg_and_number : Error<"wrong argument format for hls
 def ext_hlsl_access_specifiers : ExtWarn<
   "access specifiers are a clang HLSL extension">,
   InGroup<HLSLExtension>;
+def err_hlsl_unsupported_component : Error<"invalid component '%0' used; expected 'x', 'y', 'z', or 'w'">;
+def err_hlsl_packoffset_invalid_reg : Error<"invalid resource class specifier '%0' for packoffset, expected 'c'">;
 
 } // end of Parser diagnostics
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 9317ae675c72..d6863f90edb6 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12184,6 +12184,11 @@ def err_hlsl_init_priority_unsupported : Error<
 def err_hlsl_unsupported_register_type : Error<"invalid resource class specifier '%0' used; expected 'b', 's', 't', or 'u'">;
 def err_hlsl_unsupported_register_number : Error<"register number should be an integer">;
 def err_hlsl_expected_space : Error<"invalid space specifier '%0' used; expected 'space' followed by an integer, like space1">;
+def warn_hlsl_packoffset_mix : Warning<"cannot mix packoffset elements with nonpackoffset elements in a cbuffer">,
+    InGroup<HLSLMixPackOffset>;
+def err_hlsl_packoffset_overlap : Error<"packoffset overlap between %0, %1">;
+def err_hlsl_packoffset_cross_reg_boundary : Error<"packoffset cannot cross register boundary">;
+def err_hlsl_packoffset_alignment_mismatch : Error<"packoffset at 'y' not match alignment %0 required by %1">;
 def err_hlsl_pointers_unsupported : Error<
   "%select{pointers|references}0 are unsupported in HLSL">;
 
diff --git a/clang/lib/Parse/ParseHLSL.cpp b/clang/lib/Parse/ParseHLSL.cpp
index f4cbece31f18..e9c8d6dca7bf 100644
--- a/clang/lib/Parse/ParseHLSL.cpp
+++ b/clang/lib/Parse/ParseHLSL.cpp
@@ -183,6 +183,94 @@ void Parser::ParseHLSLAnnotations(ParsedAttributes &Attrs,
       return;
     }
   } break;
+  case ParsedAttr::AT_HLSLPackOffset: {
+    // Parse 'packoffset( c[Subcomponent][.component] )'.
+    // Check '('.
+    if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after)) {
+      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+      return;
+    }
+    // Check c[Subcomponent] as an identifier.
+    if (!Tok.is(tok::identifier)) {
+      Diag(Tok.getLocation(), diag::err_expected) << tok::identifier;
+      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+      return;
+    }
+    StringRef OffsetStr = Tok.getIdentifierInfo()->getName();
+    SourceLocation SubComponentLoc = Tok.getLocation();
+    if (OffsetStr[0] != 'c') {
+      Diag(Tok.getLocation(), diag::err_hlsl_packoffset_invalid_reg)
+          << OffsetStr;
+      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+      return;
+    }
+    OffsetStr = OffsetStr.substr(1);
+    unsigned SubComponent = 0;
+    if (!OffsetStr.empty()) {
+      // Make sure SubComponent is a number.
+      if (OffsetStr.getAsInteger(10, SubComponent)) {
+        Diag(SubComponentLoc.getLocWithOffset(1),
+             diag::err_hlsl_unsupported_register_number);
+        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+        return;
+      }
+    }
+    unsigned Component = 0;
+    ConsumeToken(); // consume identifier.
+    SourceLocation ComponentLoc;
+    if (Tok.is(tok::period)) {
+      ConsumeToken(); // consume period.
+      if (!Tok.is(tok::identifier)) {
+        Diag(Tok.getLocation(), diag::err_expected) << tok::identifier;
+        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+        return;
+      }
+      StringRef ComponentStr = Tok.getIdentifierInfo()->getName();
+      ComponentLoc = Tok.getLocation();
+      ConsumeToken(); // consume identifier.
+      // Make sure Component is a single character.
+      if (ComponentStr.size() != 1) {
+        Diag(ComponentLoc, diag::err_hlsl_unsupported_component)
+            << ComponentStr;
+        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+        return;
+      }
+      switch (ComponentStr[0]) {
+      case 'x':
+      case 'r':
+        Component = 0;
+        break;
+      case 'y':
+      case 'g':
+        Component = 1;
+        break;
+      case 'z':
+      case 'b':
+        Component = 2;
+        break;
+      case 'w':
+      case 'a':
+        Component = 3;
+        break;
+      default:
+        Diag(ComponentLoc, diag::err_hlsl_unsupported_component)
+            << ComponentStr;
+        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+        return;
+      }
+    }
+    ASTContext &Ctx = Actions.getASTContext();
+    QualType SizeTy = Ctx.getSizeType();
+    uint64_t SizeTySize = Ctx.getTypeSize(SizeTy);
+    ArgExprs.push_back(IntegerLiteral::Create(
+        Ctx, llvm::APInt(SizeTySize, SubComponent), SizeTy, SubComponentLoc));
+    ArgExprs.push_back(IntegerLiteral::Create(
+        Ctx, llvm::APInt(SizeTySize, Component), SizeTy, ComponentLoc));
+    if (ExpectAndConsume(tok::r_paren, diag::err_expected)) {
+      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+      return;
+    }
+  } break;
   case ParsedAttr::UnknownAttribute:
     Diag(Loc, diag::err_unknown_hlsl_semantic) << II;
     return;
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 6ca42856459f..6d957ac09e1c 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -7309,6 +7309,55 @@ static void handleHLSLSV_DispatchThreadIDAttr(Sema &S, Decl *D,
   D->addAttr(::new (S.Context) HLSLSV_DispatchThreadIDAttr(S.Context, AL));
 }
 
+static void handleHLSLPackOffsetAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
+  if (!isa<VarDecl>(D) || !isa<HLSLBufferDecl>(D->getDeclContext())) {
+    S.Diag(AL.getLoc(), diag::err_hlsl_attr_invalid_ast_node)
+        << AL << "shader constant in a constant buffer";
+    return;
+  }
+
+  uint32_t SubComponent;
+  if (!checkUInt32Argument(S, AL, AL.getArgAsExpr(0), SubComponent))
+    return;
+  uint32_t Component;
+  if (!checkUInt32Argument(S, AL, AL.getArgAsExpr(1), Component))
+    return;
+
+  QualType T = cast<VarDecl>(D)->getType().getCanonicalType();
+  // Check if T is an array or struct type.
+  // TODO: mark matrix type as aggregate type.
+  bool IsAggregateTy = (T->isArrayType() || T->isStructureType());
+
+  // Check Component is valid for T.
+  if (Component) {
+    unsigned Size = S.getASTContext().getTypeSize(T);
+    if (IsAggregateTy || Size > 128) {
+      S.Diag(AL.getLoc(), diag::err_hlsl_packoffset_cross_reg_boundary);
+      return;
+    } else {
+      // Make sure Component + sizeof(T) <= 4.
+      if ((Component * 32 + Size) > 128) {
+        S.Diag(AL.getLoc(), diag::err_hlsl_packoffset_cross_reg_boundary);
+        return;
+      }
+      QualType EltTy = T;
+      if (const auto *VT = T->getAs<VectorType>())
+        EltTy = VT->getElementType();
+      unsigned Align = S.getASTContext().getTypeAlign(EltTy);
+      if (Align > 32 && Component == 1) {
+        // NOTE: Component 3 will hit err_hlsl_packoffset_cross_reg_boundary.
+        // So we only need to check Component 1 here.
+        S.Diag(AL.getLoc(), diag::err_hlsl_packoffset_alignment_mismatch)
+            << Align << EltTy;
+        return;
+      }
+    }
+  }
+
+  D->addAttr(::new (S.Context)
+                 HLSLPackOffsetAttr(S.Context, AL, SubComponent, Component));
+}
+
 static void handleHLSLShaderAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   StringRef Str;
   SourceLocation ArgLoc;
@@ -9730,6 +9779,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_HLSLSV_DispatchThreadID:
     handleHLSLSV_DispatchThreadIDAttr(S, D, AL);
     break;
+  case ParsedAttr::AT_HLSLPackOffset:
+    handleHLSLPackOffsetAttr(S, D, AL);
+    break;
   case ParsedAttr::AT_HLSLShader:
     handleHLSLShaderAttr(S, D, AL);
     break;
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index bb9e37f18d37..6a12c417e2f3 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -39,9 +39,89 @@ Decl *SemaHLSL::ActOnStartBuffer(Scope *BufferScope, bool CBuffer,
   return Result;
 }
 
+// Calculate the size of a legacy cbuffer type based on
+// https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-packing-rules
+static unsigned calculateLegacyCbufferSize(const ASTContext &Context,
+                                           QualType T) {
+  unsigned Size = 0;
+  constexpr unsigned CBufferAlign = 128;
+  if (const RecordType *RT = T->getAs<RecordType>()) {
+    const RecordDecl *RD = RT->getDecl();
+    for (const FieldDecl *Field : RD->fields()) {
+      QualType Ty = Field->getType();
+      unsigned FieldSize = calculateLegacyCbufferSize(Context, Ty);
+      unsigned FieldAlign = 32;
+      if (Ty->isAggregateType())
+        FieldAlign = CBufferAlign;
+      Size = llvm::alignTo(Size, FieldAlign);
+      Size += FieldSize;
+    }
+  } else if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T)) {
+    if (unsigned ElementCount = AT->getSize().getZExtValue()) {
+      unsigned ElementSize =
+          calculateLegacyCbufferSize(Context, AT->getElementType());
+      unsigned AlignedElementSize = llvm::alignTo(ElementSize, CBufferAlign);
+      Size = AlignedElementSize * (ElementCount - 1) + ElementSize;
+    }
+  } else if (const VectorType *VT = T->getAs<VectorType>()) {
+    unsigned ElementCount = VT->getNumElements();
+    unsigned ElementSize =
+        calculateLegacyCbufferSize(Context, VT->getElementType());
+    Size = ElementSize * ElementCount;
+  } else {
+    Size = Context.getTypeSize(T);
+  }
+  return Size;
+}
+
 void SemaHLSL::ActOnFinishBuffer(Decl *Dcl, SourceLocation RBrace) {
   auto *BufDecl = cast<HLSLBufferDecl>(Dcl);
   BufDecl->setRBraceLoc(RBrace);
+
+  // Validate packoffset.
+  llvm::SmallVector<std::pair<VarDecl *, HLSLPackOffsetAttr *>> PackOffsetVec;
+  bool HasPackOffset = false;
+  bool HasNonPackOffset = false;
+  for (auto *Field : BufDecl->decls()) {
+    VarDecl *Var = dyn_cast<VarDecl>(Field);
+    if (!Var)
+      continue;
+    if (Field->hasAttr<HLSLPackOffsetAttr>()) {
+      PackOffsetVec.emplace_back(Var, Field->getAttr<HLSLPackOffsetAttr>());
+      HasPackOffset = true;
+    } else {
+      HasNonPackOffset = true;
+    }
+  }
+
+  if (HasPackOffset && HasNonPackOffset)
+    Diag(BufDecl->getLocation(), diag::warn_hlsl_packoffset_mix);
+
+  if (HasPackOffset) {
+    ASTContext &Context = getASTContext();
+    // Make sure no overlap in packoffset.
+    // Sort PackOffsetVec by offset.
+    std::sort(PackOffsetVec.begin(), PackOffsetVec.end(),
+              [](const std::pair<VarDecl *, HLSLPackOffsetAttr *> &LHS,
+                 const std::pair<VarDecl *, HLSLPackOffsetAttr *> &RHS) {
+                return LHS.second->getOffset() < RHS.second->getOffset();
+              });
+
+    for (unsigned i = 0; i < PackOffsetVec.size() - 1; i++) {
+      VarDecl *Var = PackOffsetVec[i].first;
+      HLSLPackOffsetAttr *Attr = PackOffsetVec[i].second;
+      unsigned Size = calculateLegacyCbufferSize(Context, Var->getType());
+      unsigned Begin = Attr->getOffset() * 32;
+      unsigned End = Begin + Size;
+      unsigned NextBegin = PackOffsetVec[i + 1].second->getOffset() * 32;
+      if (End > NextBegin) {
+        VarDecl *NextVar = PackOffsetVec[i + 1].first;
+        Diag(NextVar->getLocation(), diag::err_hlsl_packoffset_overlap)
+            << NextVar << Var;
+      }
+    }
+  }
+
   SemaRef.PopDeclContext();
 }
 
diff --git a/clang/test/AST/HLSL/packoffset.hlsl b/clang/test/AST/HLSL/packoffset.hlsl
new file mode 100644
index 000000000000..9cfd88eeec33
--- /dev/null
+++ b/clang/test/AST/HLSL/packoffset.hlsl
@@ -0,0 +1,100 @@
+// RUN: %clang_cc1 -triple dxil-unknown-shadermodel6.3-library -S -finclude-default-header  -ast-dump  -x hlsl %s | FileCheck %s
+
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer A
+cbuffer A
+{
+    // CHECK-NEXT: VarDecl {{.*}} A1 'float4'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 0
+    float4 A1 : packoffset(c);
+    // CHECK-NEXT: VarDecl {{.*}} col:11 A2 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 0
+    float A2 : packoffset(c1);
+    // CHECK-NEXT: VarDecl {{.*}} col:11 A3 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 1
+    float A3 : packoffset(c1.y);
+}
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer B
+cbuffer B
+{
+    // CHECK: VarDecl {{.*}} B0 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
+    float B0 : packoffset(c0.g);
+    // CHECK-NEXT: VarDecl {{.*}} B1 'double'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 2
+	double B1 : packoffset(c0.b);
+    // CHECK-NEXT: VarDecl {{.*}} B2 'half'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 0
+	half B2 : packoffset(c0.r);
+}
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer C
+cbuffer C
+{
+    // CHECK: VarDecl {{.*}} C0 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1
+    float C0 : packoffset(c0.y);
+    // CHECK-NEXT: VarDecl {{.*}} C1 'float2'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2
+	float2 C1 : packoffset(c0.z);
+    // CHECK-NEXT: VarDecl {{.*}} C2 'half'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0
+	half C2 : packoffset(c0.x);
+}
+
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer D
+cbuffer D
+{
+    // CHECK: VarDecl {{.*}} D0 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
+    float D0 : packoffset(c0.y);
+    // CHECK-NEXT: VarDecl {{.*}} D1 'float[2]'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 0
+	float D1[2] : packoffset(c1.x);
+    // CHECK-NEXT: VarDecl {{.*}} D2 'half3'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2 1
+	half3 D2 : packoffset(c2.y);
+    // CHECK-NEXT: VarDecl {{.*}} D3 'double'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 2
+	double D3 : packoffset(c0.z);
+}
+
+struct ST {
+  float a;
+  float2 b;
+  half c;
+};
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer S
+cbuffer S {
+    // CHECK: VarDecl {{.*}} S0 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
+  float S0 : packoffset(c0.y);
+    // CHECK: VarDecl {{.*}} S1 'ST'
+    // CHECK: HLSLPackOffsetAttr {{.*}} 1 0
+  ST S1 : packoffset(c1);
+    // CHECK: VarDecl {{.*}} S2 'double2'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2 0
+  double2 S2 : packoffset(c2);
+}
+
+struct ST2 {
+  float s0;
+  ST s1;
+  half s2;
+};
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer S2
+cbuffer S2 {
+    // CHECK: VarDecl {{.*}} S20 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 3
+  float S20 : packoffset(c0.a);
+    // CHECK: VarDecl {{.*}} S21 'ST2'
+    // CHECK: HLSLPackOffsetAttr {{.*}} 1 0
+  ST2 S21 : packoffset(c1);
+    // CHECK: VarDecl {{.*}} S22 'half'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 3 1
+  half S22 : packoffset(c3.y);
+}
diff --git a/clang/test/SemaHLSL/packoffset-invalid.hlsl b/clang/test/SemaHLSL/packoffset-invalid.hlsl
new file mode 100644
index 000000000000..c5983f6fd7e0
--- /dev/null
+++ b/clang/test/SemaHLSL/packoffset-invalid.hlsl
@@ -0,0 +1,122 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library -verify %s
+
+// expected-warning@+1{{cannot mix packoffset elements with nonpackoffset elements in a cbuffer}}
+cbuffer Mix
+{
+    float4 M1 : packoffset(c0);
+    float M2;
+    float M3 : packoffset(c1.y);
+}
+
+// expected-warning@+1{{cannot mix packoffset elements with nonpackoffset elements in a cbuffer}}
+cbuffer Mix2
+{
+    float4 M4;
+    float M5 : packoffset(c1.y);
+    float M6 ;
+}
+
+// expected-error@+1{{attribute 'packoffset' only applies to shader constant in a constant buffer}}
+float4 g : packoffset(c0);
+
+cbuffer IllegalOffset
+{
+    // expected-error@+1{{invalid resource class specifier 't2' for packoffset, expected 'c'}}
+    float4 i1 : packoffset(t2);
+    // expected-error@+1{{invalid component 'm' used; expected 'x', 'y', 'z', or 'w'}}
+    float i2 : packoffset(c1.m);
+}
+
+cbuffer Overlap
+{
+    float4 o1 : packoffset(c0);
+    // expected-error@+1{{packoffset overlap between 'o2', 'o1'}}
+    float2 o2 : packoffset(c0.z);
+}
+
+cbuffer CrossReg
+{
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+    float4 c1 : packoffset(c0.y);
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+    float2 c2 : packoffset(c1.w);
+}
+
+struct ST {
+  float s;
+};
+
+cbuffer Aggregate
+{
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+    ST A1 : packoffset(c0.y);
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+    float A2[2] : packoffset(c1.w);
+}
+
+cbuffer Double {
+    // expected-error@+1{{packoffset at 'y' not match alignment 64 required by 'double'}}
+    double d : packoffset(c.y);
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+	double2 d2 : packoffset(c.z);
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+	double3 d3 : packoffset(c.z);
+}
+
+cbuffer ParsingFail {
+// expected-error@+1{{expected identifier}}
+float pf0 : packoffset();
+// expected-error@+1{{expected identifier}}
+float pf1 : packoffset((c0));
+// expected-error@+1{{expected ')'}}
+float pf2 : packoffset(c0, x);
+// expected-error@+1{{invalid component 'X' used}}
+float pf3 : packoffset(c.X);
+// expected-error@+1{{expected '(' after ''}}
+float pf4 : packoffset;
+// expected-error@+1{{expected identifier}}
+float pf5 : packoffset(;
+// expected-error@+1{{expected '(' after '}}
+float pf6 : packoffset);
+// expected-error@+1{{expected '(' after '}}
+float pf7 : packoffset c0.x;
+
+// expected-error@+1{{invalid component 'xy' used}}
+float pf8 : packoffset(c0.xy);
+// expected-error@+1{{invalid component 'rg' used}}
+float pf9 : packoffset(c0.rg);
+// expected-error@+1{{invalid component 'yes' used}}
+float pf10 : packoffset(c0.yes);
+// expected-error@+1{{invalid component 'woo'}}
+float pf11 : packoffset(c0.woo);
+// expected-error@+1{{invalid component 'xr' used}}
+float pf12 : packoffset(c0.xr);
+}
+
+struct ST2 {
+  float a;
+  float2 b;
+};
+
+cbuffer S {
+  float S0 : packoffset(c0.y);
+  ST2 S1[2] : packoffset(c1);
+  // expected-error@+1{{packoffset overlap between 'S2', 'S1'}}
+  half2 S2 : packoffset(c1.w);
+  half2 S3 : packoffset(c2.w);
+}
+
+struct ST23 {
+  float s0;
+  ST2 s1;
+};
+
+cbuffer S2 {
+  float S20 : packoffset(c0.y);
+  ST2 S21 : packoffset(c1);
+  half2 S22 : packoffset(c2.w);
+  double S23[2] : packoffset(c3);
+  // expected-error@+1{{packoffset overlap between 'S24', 'S23'}}
+  float S24 : packoffset(c3.z);
+  float S25 : packoffset(c4.z);
+}
-- 
GitLab


From 737e0bcfe344bd9a8c4e4c3c2e80fdbe93bfaafb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Wed, 8 May 2024 14:27:02 +0200
Subject: [PATCH 0164/1206] [GlobalIsel] combine ext of trunc with flags
 (#87115)

https://github.com/llvm/llvm-project/pull/85592

https://discourse.llvm.org/t/rfc-add-nowrap-flags-to-trunc/77453

https://github.com/llvm/llvm-project/pull/88609
---
 .../llvm/CodeGen/GlobalISel/CombinerHelper.h  |  13 +-
 .../CodeGen/GlobalISel/GenericMachineInstrs.h |  53 +++
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |   6 +-
 .../include/llvm/Target/GlobalISel/Combine.td |  18 +-
 .../lib/CodeGen/GlobalISel/CombinerHelper.cpp |  83 +++-
 .../lib/CodeGen/GlobalISel/GISelKnownBits.cpp |   5 +-
 .../CodeGen/GlobalISel/MachineIRBuilder.cpp   |  12 +-
 .../AArch64/GlobalISel/combine-with-flags.mir | 353 ++++++++++++++++++
 8 files changed, 522 insertions(+), 21 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-with-flags.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 4f1c9642e117..ecaece8b6834 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -599,10 +599,6 @@ public:
   /// This variant does not erase \p MI after calling the build function.
   void applyBuildFnNoErase(MachineInstr &MI, BuildFnTy &MatchInfo);
 
-  /// Use a function which takes in a MachineIRBuilder to perform a combine.
-  /// By default, it erases the instruction \p MI from the function.
-  void applyBuildFnMO(const MachineOperand &MO, BuildFnTy &MatchInfo);
-
   bool matchOrShiftToFunnelShift(MachineInstr &MI, BuildFnTy &MatchInfo);
   bool matchFunnelShiftToRotate(MachineInstr &MI);
   void applyFunnelShiftToRotate(MachineInstr &MI);
@@ -814,6 +810,12 @@ public:
   /// Match constant LHS ops that should be commuted.
   bool matchCommuteConstantToRHS(MachineInstr &MI);
 
+  /// Combine sext of trunc.
+  bool matchSextOfTrunc(const MachineOperand &MO, BuildFnTy &MatchInfo);
+
+  /// Combine zext of trunc.
+  bool matchZextOfTrunc(const MachineOperand &MO, BuildFnTy &MatchInfo);
+
   /// Match constant LHS FP ops that should be commuted.
   bool matchCommuteFPConstantToRHS(MachineInstr &MI);
 
@@ -857,6 +859,9 @@ public:
   /// register and different indices.
   bool matchExtractVectorElementWithDifferentIndices(const MachineOperand &MO,
                                                      BuildFnTy &MatchInfo);
+  /// Use a function which takes in a MachineIRBuilder to perform a combine.
+  /// By default, it erases the instruction def'd on \p MO from the function.
+  void applyBuildFnMO(const MachineOperand &MO, BuildFnTy &MatchInfo);
 
   /// Combine insert vector element OOB.
   bool matchInsertVectorElementOOB(MachineInstr &MI, BuildFnTy &MatchInfo);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 705ef0fa7f2b..2a3145b635e6 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -792,6 +792,59 @@ public:
   }
 };
 
+/// Represents a cast operation.
+/// It models the llvm::CastInst concept.
+/// The exception is bitcast.
+class GCastOp : public GenericMachineInstr {
+public:
+  Register getSrcReg() const { return getOperand(1).getReg(); }
+
+  static bool classof(const MachineInstr *MI) {
+    switch (MI->getOpcode()) {
+    case TargetOpcode::G_ADDRSPACE_CAST:
+    case TargetOpcode::G_FPEXT:
+    case TargetOpcode::G_FPTOSI:
+    case TargetOpcode::G_FPTOUI:
+    case TargetOpcode::G_FPTRUNC:
+    case TargetOpcode::G_INTTOPTR:
+    case TargetOpcode::G_PTRTOINT:
+    case TargetOpcode::G_SEXT:
+    case TargetOpcode::G_SITOFP:
+    case TargetOpcode::G_TRUNC:
+    case TargetOpcode::G_UITOFP:
+    case TargetOpcode::G_ZEXT:
+    case TargetOpcode::G_ANYEXT:
+      return true;
+    default:
+      return false;
+    }
+  };
+};
+
+/// Represents a sext.
+class GSext : public GCastOp {
+public:
+  static bool classof(const MachineInstr *MI) {
+    return MI->getOpcode() == TargetOpcode::G_SEXT;
+  };
+};
+
+/// Represents a zext.
+class GZext : public GCastOp {
+public:
+  static bool classof(const MachineInstr *MI) {
+    return MI->getOpcode() == TargetOpcode::G_ZEXT;
+  };
+};
+
+/// Represents a trunc.
+class GTrunc : public GCastOp {
+public:
+  static bool classof(const MachineInstr *MI) {
+    return MI->getOpcode() == TargetOpcode::G_TRUNC;
+  };
+};
+
 } // namespace llvm
 
 #endif // LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index e15f7a7172e1..92e05ee858a7 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -746,7 +746,8 @@ public:
   /// \pre \p Op must be smaller than \p Res
   ///
   /// \return The newly created instruction.
-  MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op);
+  MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op,
+                                std::optional<unsigned> Flags = std::nullopt);
 
   /// Build and insert \p Res = G_SEXT \p Op, \p Res = G_TRUNC \p Op, or
   /// \p Res = COPY \p Op depending on the differing sizes of \p Res and \p Op.
@@ -1231,7 +1232,8 @@ public:
   /// \pre \p Res must be smaller than \p Op
   ///
   /// \return The newly created instruction.
-  MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op);
+  MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op,
+                                 std::optional<unsigned> Flags = std::nullopt);
 
   /// Build and insert a \p Res = G_ICMP \p Pred, \p Op0, \p Op1
   ///
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 72c5de03f4e7..d0e125390347 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -180,6 +180,8 @@ def FmContract  : MIFlagEnum<"FmContract">;
 def FmAfn       : MIFlagEnum<"FmAfn">;
 def FmReassoc   : MIFlagEnum<"FmReassoc">;
 def IsExact     : MIFlagEnum<"IsExact">;
+def NoSWrap     : MIFlagEnum<"NoSWrap">;
+def NoUWrap     : MIFlagEnum<"NoUWrap">;
 
 def MIFlags;
 // def not; -> Already defined as a SDNode
@@ -1501,6 +1503,20 @@ def extract_vector_element_freeze : GICombineRule<
    [{ return Helper.matchExtractVectorElementWithFreeze(${root}, ${matchinfo}); }]),
    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
 
+def sext_trunc : GICombineRule<
+   (defs root:$root, build_fn_matchinfo:$matchinfo),
+   (match (G_TRUNC $src, $x, (MIFlags NoSWrap)),
+          (G_SEXT $root, $src),
+   [{ return Helper.matchSextOfTrunc(${root}, ${matchinfo}); }]),
+   (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
+
+def zext_trunc : GICombineRule<
+   (defs root:$root, build_fn_matchinfo:$matchinfo),
+   (match (G_TRUNC $src, $x, (MIFlags NoUWrap)),
+          (G_ZEXT $root, $src),
+   [{ return Helper.matchZextOfTrunc(${root}, ${matchinfo}); }]),
+   (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
+
 def extract_vector_element_shuffle_vector : GICombineRule<
    (defs root:$root, build_fn_matchinfo:$matchinfo),
    (match (G_SHUFFLE_VECTOR $src, $src1, $src2, $mask),
@@ -1666,7 +1682,7 @@ def all_combines : GICombineGroup<[trivial_combines, vector_ops_combines,
     sub_add_reg, select_to_minmax, redundant_binop_in_equality,
     fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
     combine_concat_vector, double_icmp_zero_and_or_combine, match_addos,
-    combine_shuffle_concat]>;
+    sext_trunc, zext_trunc, combine_shuffle_concat]>;
 
 // A combine group used to for prelegalizer combiners at -O0. The combines in
 // this group have been selected based on experiments to balance code size and
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 653e7689b577..9999776b9826 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -4137,14 +4137,6 @@ void CombinerHelper::applyBuildFn(
   MI.eraseFromParent();
 }
 
-void CombinerHelper::applyBuildFnMO(const MachineOperand &MO,
-                                    BuildFnTy &MatchInfo) {
-  MachineInstr *Root = getDefIgnoringCopies(MO.getReg(), MRI);
-  Builder.setInstrAndDebugLoc(*Root);
-  MatchInfo(Builder);
-  Root->eraseFromParent();
-}
-
 void CombinerHelper::applyBuildFnNoErase(
     MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) {
   MatchInfo(Builder);
@@ -7252,3 +7244,78 @@ bool CombinerHelper::matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo) {
 
   return false;
 }
+
+void CombinerHelper::applyBuildFnMO(const MachineOperand &MO,
+                                    BuildFnTy &MatchInfo) {
+  MachineInstr *Root = getDefIgnoringCopies(MO.getReg(), MRI);
+  MatchInfo(Builder);
+  Root->eraseFromParent();
+}
+
+bool CombinerHelper::matchSextOfTrunc(const MachineOperand &MO,
+                                      BuildFnTy &MatchInfo) {
+  GSext *Sext = cast<GSext>(getDefIgnoringCopies(MO.getReg(), MRI));
+  GTrunc *Trunc = cast<GTrunc>(getDefIgnoringCopies(Sext->getSrcReg(), MRI));
+
+  Register Dst = Sext->getReg(0);
+  Register Src = Trunc->getSrcReg();
+
+  LLT DstTy = MRI.getType(Dst);
+  LLT SrcTy = MRI.getType(Src);
+
+  if (DstTy == SrcTy) {
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildCopy(Dst, Src); };
+    return true;
+  }
+
+  if (DstTy.getScalarSizeInBits() < SrcTy.getScalarSizeInBits() &&
+      isLegalOrBeforeLegalizer({TargetOpcode::G_TRUNC, {DstTy, SrcTy}})) {
+    MatchInfo = [=](MachineIRBuilder &B) {
+      B.buildTrunc(Dst, Src, MachineInstr::MIFlag::NoSWrap);
+    };
+    return true;
+  }
+
+  if (DstTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits() &&
+      isLegalOrBeforeLegalizer({TargetOpcode::G_SEXT, {DstTy, SrcTy}})) {
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildSExt(Dst, Src); };
+    return true;
+  }
+
+  return false;
+}
+
+bool CombinerHelper::matchZextOfTrunc(const MachineOperand &MO,
+                                      BuildFnTy &MatchInfo) {
+  GZext *Zext = cast<GZext>(getDefIgnoringCopies(MO.getReg(), MRI));
+  GTrunc *Trunc = cast<GTrunc>(getDefIgnoringCopies(Zext->getSrcReg(), MRI));
+
+  Register Dst = Zext->getReg(0);
+  Register Src = Trunc->getSrcReg();
+
+  LLT DstTy = MRI.getType(Dst);
+  LLT SrcTy = MRI.getType(Src);
+
+  if (DstTy == SrcTy) {
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildCopy(Dst, Src); };
+    return true;
+  }
+
+  if (DstTy.getScalarSizeInBits() < SrcTy.getScalarSizeInBits() &&
+      isLegalOrBeforeLegalizer({TargetOpcode::G_TRUNC, {DstTy, SrcTy}})) {
+    MatchInfo = [=](MachineIRBuilder &B) {
+      B.buildTrunc(Dst, Src, MachineInstr::MIFlag::NoUWrap);
+    };
+    return true;
+  }
+
+  if (DstTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits() &&
+      isLegalOrBeforeLegalizer({TargetOpcode::G_ZEXT, {DstTy, SrcTy}})) {
+    MatchInfo = [=](MachineIRBuilder &B) {
+      B.buildZExt(Dst, Src, MachineInstr::MIFlag::NonNeg);
+    };
+    return true;
+  }
+
+  return false;
+}
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index 529e50c8ebe0..c8199a42d15c 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -64,8 +64,11 @@ KnownBits GISelKnownBits::getKnownBits(MachineInstr &MI) {
 
 KnownBits GISelKnownBits::getKnownBits(Register R) {
   const LLT Ty = MRI.getType(R);
+  // Since the number of lanes in a scalable vector is unknown at compile time,
+  // we track one bit which is implicitly broadcast to all lanes.  This means
+  // that all lanes in a scalable vector are considered demanded.
   APInt DemandedElts =
-      Ty.isVector() ? APInt::getAllOnes(Ty.getNumElements()) : APInt(1, 1);
+      Ty.isFixedVector() ? APInt::getAllOnes(Ty.getNumElements()) : APInt(1, 1);
   return getKnownBits(R, DemandedElts);
 }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 2e8407813ba6..afe270356940 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -490,8 +490,9 @@ MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res,
 }
 
 MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
-                                                const SrcOp &Op) {
-  return buildInstr(TargetOpcode::G_ZEXT, Res, Op);
+                                                const SrcOp &Op,
+                                                std::optional<unsigned> Flags) {
+  return buildInstr(TargetOpcode::G_ZEXT, Res, Op, Flags);
 }
 
 unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
@@ -869,9 +870,10 @@ MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
   return buildIntrinsic(ID, Results, HasSideEffects, isConvergent);
 }
 
-MachineInstrBuilder MachineIRBuilder::buildTrunc(const DstOp &Res,
-                                                 const SrcOp &Op) {
-  return buildInstr(TargetOpcode::G_TRUNC, Res, Op);
+MachineInstrBuilder
+MachineIRBuilder::buildTrunc(const DstOp &Res, const SrcOp &Op,
+                             std::optional<unsigned> Flags) {
+  return buildInstr(TargetOpcode::G_TRUNC, Res, Op, Flags);
 }
 
 MachineInstrBuilder
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-with-flags.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-with-flags.mir
new file mode 100644
index 000000000000..6eece5c56258
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-with-flags.mir
@@ -0,0 +1,353 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs -mtriple aarch64-unknown-unknown %s -o - | FileCheck %s
+
+---
+name:            zext_trunc_nuw
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: zext_trunc_nuw
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: $x1 = COPY [[COPY]](s64)
+    %0:_(s64) = COPY $x0
+    %2:_(s32) = nuw G_TRUNC %0
+    %3:_(s64) = G_ZEXT  %2
+    $x1 = COPY %3
+...
+---
+name:            zext_trunc_nsw
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: zext_trunc_nsw
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = nsw G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s32)
+    ; CHECK-NEXT: $x1 = COPY [[ZEXT]](s64)
+    %0:_(s64) = COPY $x0
+    %2:_(s32) = nsw G_TRUNC %0
+    %3:_(s64) = G_ZEXT  %2
+    $x1 = COPY %3
+...
+---
+name:            zext_trunc
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: zext_trunc
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s32)
+    ; CHECK-NEXT: $x1 = COPY [[ZEXT]](s64)
+    %0:_(s64) = COPY $x0
+    %2:_(s32) = G_TRUNC %0
+    %3:_(s64) = G_ZEXT  %2
+    $x1 = COPY %3
+...
+---
+name:            zext_trunc_nuw_vector
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: zext_trunc_nuw_vector
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK-NEXT: %bv0:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY]](s32), [[COPY1]](s32)
+    ; CHECK-NEXT: $q0 = COPY %bv0(<4 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:_(s32) = COPY $w0
+    %1:_(s32) = COPY $w1
+    %2:_(s32) = COPY $w2
+    %3:_(s32) = COPY $w3
+    %bv0:_(<4 x s32>) = G_BUILD_VECTOR %0:_(s32), %1:_(s32), %0:_(s32), %1:_(s32)
+    %trunc:_(<4 x s16>) = nuw G_TRUNC %bv0
+    %zext:_(<4 x s32>) = G_ZEXT  %trunc
+    $q0 = COPY %zext(<4 x s32>)
+    RET_ReallyLR implicit $w0
+...
+---
+name:            sext_trunc_nsw
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: sext_trunc_nsw
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: $x1 = COPY [[COPY]](s64)
+    %0:_(s64) = COPY $x0
+    %2:_(s32) = nsw G_TRUNC %0
+    %3:_(s64) = G_SEXT  %2
+    $x1 = COPY %3
+...
+---
+name:            sext_trunc_nuw
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: sext_trunc_nuw
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = nuw G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC]](s32)
+    ; CHECK-NEXT: $x1 = COPY [[SEXT]](s64)
+    %0:_(s64) = COPY $x0
+    %2:_(s32) = nuw G_TRUNC %0
+    %3:_(s64) = G_SEXT  %2
+    $x1 = COPY %3
+...
+---
+name:            sext_trunc
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: sext_trunc
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC]](s32)
+    ; CHECK-NEXT: $x1 = COPY [[SEXT]](s64)
+    %0:_(s64) = COPY $x0
+    %2:_(s32) = G_TRUNC %0
+    %3:_(s64) = G_SEXT  %2
+    $x1 = COPY %3
+...
+---
+name:            sext_trunc_nsw_types_wrong
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: sext_trunc_nsw_types_wrong
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = nsw G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: $w1 = COPY [[TRUNC]](s32)
+    %0:_(s64) = COPY $x0
+    %2:_(s16) = nsw G_TRUNC %0
+    %3:_(s32) = G_SEXT  %2
+    $w1 = COPY %3
+...
+---
+name:            sext_trunc_nsw_nuw
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: sext_trunc_nsw_nuw
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: $x1 = COPY [[COPY]](s64)
+    %0:_(s64) = COPY $x0
+    %2:_(s32) = nsw nuw G_TRUNC %0
+    %3:_(s64) = G_SEXT  %2
+    $x1 = COPY %3
+...
+---
+name:            sext_trunc_nsw_nuw_vector
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: sext_trunc_nsw_nuw_vector
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $w3
+    ; CHECK-NEXT: %bv0:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: $q0 = COPY %bv0(<4 x s32>)
+    %0:_(s32) = COPY $w0
+    %1:_(s32) = COPY $w1
+    %2:_(s32) = COPY $w2
+    %3:_(s32) = COPY $w3
+    %bv0:_(<4 x s32>) = G_BUILD_VECTOR %0:_(s32), %1:_(s32), %2:_(s32), %3:_(s32)
+    %t:_(<4 x s16>) = nsw nuw G_TRUNC %bv0
+    %s:_(<4 x s32>) = G_SEXT  %t
+    $q0 = COPY %s
+...
+---
+name:            zext_trunc_vector
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: zext_trunc_vector
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $w3
+    ; CHECK-NEXT: %bv0:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: %t:_(<4 x s16>) = G_TRUNC %bv0(<4 x s32>)
+    ; CHECK-NEXT: %z:_(<4 x s32>) = G_ZEXT %t(<4 x s16>)
+    ; CHECK-NEXT: $q0 = COPY %z(<4 x s32>)
+    %0:_(s32) = COPY $w0
+    %1:_(s32) = COPY $w1
+    %2:_(s32) = COPY $w2
+    %3:_(s32) = COPY $w3
+    %bv0:_(<4 x s32>) = G_BUILD_VECTOR %0:_(s32), %1:_(s32), %2:_(s32), %3:_(s32)
+    %t:_(<4 x s16>) = G_TRUNC %bv0
+    %z:_(<4 x s32>) = G_ZEXT  %t
+    $q0 = COPY %z
+...
+---
+name:            zext_trunc_nsw_vector
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: zext_trunc_nsw_vector
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $w3
+    ; CHECK-NEXT: %bv0:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: %t:_(<4 x s16>) = nsw G_TRUNC %bv0(<4 x s32>)
+    ; CHECK-NEXT: %z:_(<4 x s32>) = G_ZEXT %t(<4 x s16>)
+    ; CHECK-NEXT: $q0 = COPY %z(<4 x s32>)
+    %0:_(s32) = COPY $w0
+    %1:_(s32) = COPY $w1
+    %2:_(s32) = COPY $w2
+    %3:_(s32) = COPY $w3
+    %bv0:_(<4 x s32>) = G_BUILD_VECTOR %0:_(s32), %1:_(s32), %2:_(s32), %3:_(s32)
+    %t:_(<4 x s16>) = nsw G_TRUNC %bv0
+    %z:_(<4 x s32>) = G_ZEXT  %t
+    $q0 = COPY %z
+...
+---
+name:            zext_trunc_nuw_vector2
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: zext_trunc_nuw_vector2
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $w3
+    ; CHECK-NEXT: %bv0:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: $q0 = COPY %bv0(<4 x s32>)
+    %0:_(s32) = COPY $w0
+    %1:_(s32) = COPY $w1
+    %2:_(s32) = COPY $w2
+    %3:_(s32) = COPY $w3
+    %bv0:_(<4 x s32>) = G_BUILD_VECTOR %0:_(s32), %1:_(s32), %2:_(s32), %3:_(s32)
+    %t:_(<4 x s16>) = nuw G_TRUNC %bv0
+    %z:_(<4 x s32>) = G_ZEXT  %t
+    $q0 = COPY %z
+...
+---
+name:            zext_trunc_nuw_vector_wrong_type
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: zext_trunc_nuw_vector_wrong_type
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK-NEXT: %bv0:_(<2 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64)
+    ; CHECK-NEXT: %z:_(<2 x s32>) = nuw G_TRUNC %bv0(<2 x s64>)
+    ; CHECK-NEXT: $d0 = COPY %z(<2 x s32>)
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %bv0:_(<2 x s64>) = G_BUILD_VECTOR %0:_(s64), %1:_(s64)
+    %t:_(<2 x s16>) = nuw G_TRUNC %bv0
+    %z:_(<2 x s32>) = G_ZEXT  %t
+    $d0 = COPY %z
+...
+---
+name:            zext_trunc_nuw_scalable_vector
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: zext_trunc_nuw_scalable_vector
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: %sv0:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR [[COPY]](s64)
+    ; CHECK-NEXT: $z0 = COPY %sv0(<vscale x 2 x s64>)
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %sv0:_(<vscale x 2 x s64>) = G_SPLAT_VECTOR %0:_(s64)
+    %t:_(<vscale x 2 x s32>) = nuw G_TRUNC %sv0
+    %z:_(<vscale x 2 x s64>) = G_ZEXT  %t
+    $z0 = COPY %z
+...
+---
+name:            zext_trunc_nuw_to_zext
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: zext_trunc_nuw_to_zext
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK-NEXT: %2:_(s64) = nneg G_ZEXT [[COPY]](s32)
+    ; CHECK-NEXT: $x1 = COPY %2(s64)
+    %0:_(s32) = COPY $w0
+    %2:_(s16) = nuw G_TRUNC %0
+    %3:_(s64) = G_ZEXT  %2
+    $x1 = COPY %3
+...
+---
+name:            zext_trunc_nuw_to_trunc
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: zext_trunc_nuw_to_trunc
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = nuw G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: $w1 = COPY [[TRUNC]](s32)
+    %0:_(s64) = COPY $x0
+    %2:_(s16) = nuw G_TRUNC %0
+    %3:_(s32) = G_ZEXT  %2
+    $w1 = COPY %3
+...
+---
+name:            sext_trunc_nsw_to_sext
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: sext_trunc_nsw_to_sext
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[COPY]](s32)
+    ; CHECK-NEXT: $x1 = COPY [[SEXT]](s64)
+    %0:_(s32) = COPY $w0
+    %2:_(s16) = nsw G_TRUNC %0
+    %3:_(s64) = G_SEXT  %2
+    $x1 = COPY %3
+...
+---
+name:            sext_trunc_nsw_to_trunc
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+    ; CHECK-LABEL: name: sext_trunc_nsw_to_trunc
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = nsw G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: $w1 = COPY [[TRUNC]](s32)
+    %0:_(s64) = COPY $x0
+    %2:_(s16) = nsw G_TRUNC %0
+    %3:_(s32) = G_SEXT  %2
+    $w1 = COPY %3
+...
-- 
GitLab


From db4cf7c0fc713c09bbe10dd2be4a3d0fb081014c Mon Sep 17 00:00:00 2001
From: Renato Golin <rengolin@systemcall.eu>
Date: Wed, 8 May 2024 13:33:20 +0100
Subject: [PATCH 0165/1206] Update CODEOWNERS

Adding myself to linalg dialect
---
 .github/CODEOWNERS | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index ad81bf1684b6..e25b2f50b1b4 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -64,8 +64,8 @@ clang/test/AST/Interp/ @tbaederr
 /mlir/Dialect/*/Transforms/Bufferize.cpp @matthias-springer
 
 # Linalg Dialect in MLIR.
-/mlir/include/mlir/Dialect/Linalg/* @dcaballe @nicolasvasilache
-/mlir/lib/Dialect/Linalg/* @dcaballe @nicolasvasilache
+/mlir/include/mlir/Dialect/Linalg/* @dcaballe @nicolasvasilache @rengolin
+/mlir/lib/Dialect/Linalg/* @dcaballe @nicolasvasilache @rengolin
 /mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp @MaheshRavishankar @nicolasvasilache
 /mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp @MaheshRavishankar @nicolasvasilache
 /mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp @MaheshRavishankar @nicolasvasilache
-- 
GitLab


From c84c74e67839a5207d7c6318fc37e607f088a994 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Fri, 3 May 2024 15:55:24 +0100
Subject: [PATCH 0166/1206] [LLVM][CodeGen][SVE] Add tests for vector extracts
 from unpacked types.

---
 .../sve-extract-fixed-from-scalable-vector.ll | 78 ++++++++++++++++++-
 1 file changed, 76 insertions(+), 2 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
index b9c531fe3352..e91aac430110 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
@@ -307,11 +307,85 @@ define <4 x i64> @extract_v4i64_nxv8i64_0(<vscale x 8 x i64> %arg) {
   ret <4 x i64> %ext
 }
 
+define <4 x half> @extract_v4f16_nxv2f16_0(<vscale x 2 x half> %arg) {
+; CHECK-LABEL: extract_v4f16_nxv2f16_0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    cntd x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    addpl x9, sp, #6
+; CHECK-NEXT:    subs x8, x8, #4
+; CHECK-NEXT:    csel x8, xzr, x8, lo
+; CHECK-NEXT:    st1h { z0.d }, p0, [sp, #3, mul vl]
+; CHECK-NEXT:    cmp x8, #0
+; CHECK-NEXT:    csel x8, x8, xzr, lo
+; CHECK-NEXT:    lsl x8, x8, #1
+; CHECK-NEXT:    ldr d0, [x9, x8]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ext = call <4 x half> @llvm.vector.extract.v4f16.nxv2f16(<vscale x 2 x half> %arg, i64 0)
+  ret <4 x half> %ext
+}
+
+define <4 x half> @extract_v4f16_nxv2f16_4(<vscale x 2 x half> %arg) {
+; CHECK-LABEL: extract_v4f16_nxv2f16_4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    cntd x8
+; CHECK-NEXT:    mov w9, #4 // =0x4
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    subs x8, x8, #4
+; CHECK-NEXT:    csel x8, xzr, x8, lo
+; CHECK-NEXT:    st1h { z0.d }, p0, [sp, #3, mul vl]
+; CHECK-NEXT:    cmp x8, #4
+; CHECK-NEXT:    csel x8, x8, x9, lo
+; CHECK-NEXT:    addpl x9, sp, #6
+; CHECK-NEXT:    lsl x8, x8, #1
+; CHECK-NEXT:    ldr d0, [x9, x8]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ext = call <4 x half> @llvm.vector.extract.v4f16.nxv2f16(<vscale x 2 x half> %arg, i64 4)
+  ret <4 x half> %ext
+}
+
+define <2 x half> @extract_v2f16_nxv4f16_2(<vscale x 4 x half> %arg) {
+; CHECK-LABEL: extract_v2f16_nxv4f16_2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z1.s, z0.s[3]
+; CHECK-NEXT:    mov z0.s, z0.s[2]
+; CHECK-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    ret
+  %ext = call <2 x half> @llvm.vector.extract.v2f16.nxv4f16(<vscale x 4 x half> %arg, i64 2)
+  ret <2 x half> %ext
+}
+
+define <2 x half> @extract_v2f16_nxv4f16_6(<vscale x 4 x half> %arg) {
+; CHECK-LABEL: extract_v2f16_nxv4f16_6:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z1.s, z0.s[7]
+; CHECK-NEXT:    mov z0.s, z0.s[6]
+; CHECK-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    ret
+  %ext = call <2 x half> @llvm.vector.extract.v2f16.nxv4f16(<vscale x 4 x half> %arg, i64 6)
+  ret <2 x half> %ext
+}
 
-declare <2 x i64> @llvm.vector.extract.v2i64.nxv8i64(<vscale x 8 x i64>, i64)
-declare <4 x i64> @llvm.vector.extract.v4i64.nxv8i64(<vscale x 8 x i64>, i64)
 declare <4 x float> @llvm.vector.extract.v4f32.nxv16f32(<vscale x 16 x float>, i64)
 declare <2 x float> @llvm.vector.extract.v2f32.nxv16f32(<vscale x 16 x float>, i64)
+declare <4 x half> @llvm.vector.extract.v4f16.nxv2f16(<vscale x 2 x half>, i64);
+declare <2 x half> @llvm.vector.extract.v2f16.nxv4f16(<vscale x 4 x half>, i64);
+declare <2 x i64> @llvm.vector.extract.v2i64.nxv8i64(<vscale x 8 x i64>, i64)
+declare <4 x i64> @llvm.vector.extract.v4i64.nxv8i64(<vscale x 8 x i64>, i64)
 declare <4 x i32> @llvm.vector.extract.v4i32.nxv16i32(<vscale x 16 x i32>, i64)
 declare <2 x i32> @llvm.vector.extract.v2i32.nxv16i32(<vscale x 16 x i32>, i64)
 declare <8 x i16> @llvm.vector.extract.v8i16.nxv32i16(<vscale x 32 x i16>, i64)
-- 
GitLab


From 686a206b2665b6ba9245980489daa79dc8547079 Mon Sep 17 00:00:00 2001
From: Nabeel Omer <nabeel.omer@sony.com>
Date: Wed, 8 May 2024 14:32:56 +0100
Subject: [PATCH 0167/1206] [SampleProfileLoader] Fix integer overflow in
 generateMDProfMetadata (#90217)

This patch fixes an integer overflow in the SampleProfileLoader pass.
The issue occurs when weights are saturated and Profi isn't being used.

This patch also adds a newline to a debug message to make it more
readable.
---
 llvm/lib/Transforms/IPO/SampleProfile.cpp     |  6 +-
 .../SampleProfile/Inputs/overflow.proftext    |  2 +
 .../test/Transforms/SampleProfile/overflow.ll | 77 +++++++++++++++++++
 3 files changed, 83 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/overflow.proftext
 create mode 100644 llvm/test/Transforms/SampleProfile/overflow.ll

diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 6cbd138842c8..0920179fb76b 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -1715,13 +1715,15 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
       // if needed. Sample counts in profiles are 64-bit unsigned values,
       // but internally branch weights are expressed as 32-bit values.
       if (Weight > std::numeric_limits<uint32_t>::max()) {
-        LLVM_DEBUG(dbgs() << " (saturated due to uint32_t overflow)");
+        LLVM_DEBUG(dbgs() << " (saturated due to uint32_t overflow)\n");
         Weight = std::numeric_limits<uint32_t>::max();
       }
       if (!SampleProfileUseProfi) {
         // Weight is added by one to avoid propagation errors introduced by
         // 0 weights.
-        Weights.push_back(static_cast<uint32_t>(Weight + 1));
+        Weights.push_back(static_cast<uint32_t>(
+            Weight == std::numeric_limits<uint32_t>::max() ? Weight
+                                                           : Weight + 1));
       } else {
         // Profi creates proper weights that do not require "+1" adjustments but
         // we evenly split the weight among branches with the same destination.
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/overflow.proftext b/llvm/test/Transforms/SampleProfile/Inputs/overflow.proftext
new file mode 100644
index 000000000000..753294a49e99
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/overflow.proftext
@@ -0,0 +1,2 @@
+_Z3testi:29600000000:29600000000
+ 5: 29600000000
diff --git a/llvm/test/Transforms/SampleProfile/overflow.ll b/llvm/test/Transforms/SampleProfile/overflow.ll
new file mode 100644
index 000000000000..06be3ce50023
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/overflow.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
+
+; Checks that we are able to handle overflowing counters correctly.
+
+; RUN: opt < %s -passes='sample-profile,print<branch-prob>' -sample-profile-file=%S/Inputs/overflow.proftext -disable-output 2>&1 | FileCheck %s
+
+; Original Source:
+; int sqrt(int);
+; int test(int i) {
+;    if (i == 5) {
+;        return 42;
+;    }
+;    else {
+;        return sqrt(i);
+;    }
+;}
+
+define dso_local noundef i32 @_Z3testi(i32 noundef %i) local_unnamed_addr #0 !dbg !10 {
+; CHECK-LABEL: '_Z3testi'
+; CHECK-NEXT:  ---- Branch Probabilities ----
+; CHECK-NEXT:    edge %entry -> %return probability is 0x00000000 / 0x80000000 = 0.00%
+; CHECK-NEXT:    edge %entry -> %if.else probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+; CHECK-NEXT:    edge %if.else -> %return probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+;
+entry:
+  tail call void @llvm.dbg.value(metadata i32 %i, metadata !16, metadata !DIExpression()), !dbg !17
+  %cmp = icmp eq i32 %i, 5, !dbg !18
+  br i1 %cmp, label %return, label %if.else, !dbg !20
+
+if.else:                                          ; preds = %entry
+  %call = tail call noundef i32 @_Z4sqrti(i32 noundef %i), !dbg !21
+  br label %return, !dbg !23
+
+return:                                           ; preds = %entry, %if.else
+  %retval.0 = phi i32 [ %call, %if.else ], [ 42, %entry ], !dbg !24
+  ret i32 %retval.0, !dbg !25
+}
+
+declare !dbg !26 noundef i32 @_Z4sqrti(i32 noundef)
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+attributes #0 = { "use-sample-profile" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/", checksumkind: CSK_MD5, checksum: "cb38d90153a7ebdd6ecf3058eb0524c7")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 8, !"PIC Level", i32 2}
+!6 = !{i32 7, !"PIE Level", i32 2}
+!7 = !{i32 7, !"uwtable", i32 2}
+!8 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
+!9 = !{!"clang"}
+!10 = distinct !DISubprogram(name: "test", linkageName: "_Z3loli", scope: !11, file: !11, line: 3, type: !12, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !15)
+!11 = !DIFile(filename: "./test.cpp", directory: "/", checksumkind: CSK_MD5, checksum: "cb38d90153a7ebdd6ecf3058eb0524c7")
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !14}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !{!16}
+!16 = !DILocalVariable(name: "i", arg: 1, scope: !10, file: !11, line: 3, type: !14)
+!17 = !DILocation(line: 0, scope: !10)
+!18 = !DILocation(line: 4, column: 11, scope: !19)
+!19 = distinct !DILexicalBlock(scope: !10, file: !11, line: 4, column: 9)
+!20 = !DILocation(line: 4, column: 9, scope: !10)
+!21 = !DILocation(line: 8, column: 16, scope: !22)
+!22 = distinct !DILexicalBlock(scope: !19, file: !11, line: 7, column: 10)
+!23 = !DILocation(line: 8, column: 9, scope: !22)
+!24 = !DILocation(line: 0, scope: !19)
+!25 = !DILocation(line: 10, column: 1, scope: !10)
+!26 = !DISubprogram(name: "sqrt", linkageName: "_Z4sqrti", scope: !11, file: !11, line: 1, type: !12, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized)
+
-- 
GitLab


From 3ceacd8b9567a25308f7aaa73d266ee3b4c6ab5f Mon Sep 17 00:00:00 2001
From: Paul T Robinson <paul.robinson@sony.com>
Date: Wed, 8 May 2024 06:37:24 -0700
Subject: [PATCH 0168/1206] [Coro] Relax a debug-info test (#91401)

Debug-info metadata does not have a strictly defined order. Check that
elements are linked to each other correctly, not that metadata appears
in a particular order.
---
 clang/test/CodeGenCoroutines/coro-dwarf.cpp | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/clang/test/CodeGenCoroutines/coro-dwarf.cpp b/clang/test/CodeGenCoroutines/coro-dwarf.cpp
index f951b63dc117..0ab70ef55c1d 100644
--- a/clang/test/CodeGenCoroutines/coro-dwarf.cpp
+++ b/clang/test/CodeGenCoroutines/coro-dwarf.cpp
@@ -71,14 +71,10 @@ void f_coro(int val, MoveOnly moParam, MoveAndCopy mcParam) {
 // CHECK: !{{[0-9]+}} = !DILocalVariable(name: "mcParam", arg: 3, scope: ![[SP]], file: !{{[0-9]+}}, line: {{[0-9]+}}, type: !{{[0-9]+}})
 // CHECK: !{{[0-9]+}} = !DILocalVariable(name: "__promise",
 
-// CHECK: !{{[0-9]+}} = distinct !DISubprogram(linkageName: "_Z6f_coroi8MoveOnly11MoveAndCopy.__await_suspend_wrapper__init"
-// CHECK-NEXT: !{{[0-9]+}} = !DIFile
-// CHECK-NEXT: !{{[0-9]+}} = !DISubroutineType
-// CHECK-NEXT: !{{[0-9]+}} = !DILocalVariable(arg: 1,
-// CHECK-NEXT: !{{[0-9]+}} = !DILocation
-// CHECK-NEXT: !{{[0-9]+}} = !DILocalVariable(arg: 2,
+// CHECK: ![[INIT:[0-9]+]] = distinct !DISubprogram(linkageName: "_Z6f_coroi8MoveOnly11MoveAndCopy.__await_suspend_wrapper__init"
+// CHECK: !{{[0-9]+}} = !DILocalVariable(arg: 1, scope: ![[INIT]]
+// CHECK: !{{[0-9]+}} = !DILocalVariable(arg: 2, scope: ![[INIT]]
 
-// CHECK: !{{[0-9]+}} = distinct !DISubprogram(linkageName: "_Z6f_coroi8MoveOnly11MoveAndCopy.__await_suspend_wrapper__final"
-// CHECK-NEXT: !{{[0-9]+}} = !DILocalVariable(arg: 1,
-// CHECK-NEXT: !{{[0-9]+}} = !DILocation
-// CHECK-NEXT: !{{[0-9]+}} = !DILocalVariable(arg: 2,
+// CHECK: ![[FINAL:[0-9]+]] = distinct !DISubprogram(linkageName: "_Z6f_coroi8MoveOnly11MoveAndCopy.__await_suspend_wrapper__final"
+// CHECK: !{{[0-9]+}} = !DILocalVariable(arg: 1, scope: ![[FINAL]]
+// CHECK: !{{[0-9]+}} = !DILocalVariable(arg: 2, scope: ![[FINAL]]
-- 
GitLab


From 665af09a86b8d80af33f170fca89ced9986cf1e5 Mon Sep 17 00:00:00 2001
From: Xiang Li <python3kgae@outlook.com>
Date: Wed, 8 May 2024 06:40:06 -0700
Subject: [PATCH 0169/1206] [DirectX backend] emits metadata for DXIL version.
 (#88350)

Emit named metadata "dx.version" for DXIL version.

Default to DXIL 1.0
---
 llvm/include/llvm/TargetParser/Triple.h       |  4 +
 llvm/lib/Target/DirectX/DXILMetadata.cpp      | 12 +++
 llvm/lib/Target/DirectX/DXILMetadata.h        |  1 +
 .../Target/DirectX/DXILTranslateMetadata.cpp  |  1 +
 llvm/lib/TargetParser/Triple.cpp              | 11 +++
 .../CodeGen/DirectX/Metadata/dxilVer-1.0.ll   | 12 +++
 .../CodeGen/DirectX/Metadata/dxilVer-1.8.ll   | 12 +++
 llvm/unittests/TargetParser/TripleTest.cpp    | 79 +++++++++++++++++++
 8 files changed, 132 insertions(+)
 create mode 100644 llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.0.ll
 create mode 100644 llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.8.ll

diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 7da30e6cf96f..20cca5928782 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -428,6 +428,10 @@ public:
   /// (SubArch).  This should only be called with Vulkan SPIR-V triples.
   VersionTuple getVulkanVersion() const;
 
+  /// Parse the DXIL version number from the DXIL version
+  /// (SubArch).  This should only be called with DXIL triples.
+  VersionTuple getDXILVersion() const;
+
   /// @}
   /// @name Direct Component Access
   /// @{
diff --git a/llvm/lib/Target/DirectX/DXILMetadata.cpp b/llvm/lib/Target/DirectX/DXILMetadata.cpp
index 03758dc76e7e..ed0434ac98a1 100644
--- a/llvm/lib/Target/DirectX/DXILMetadata.cpp
+++ b/llvm/lib/Target/DirectX/DXILMetadata.cpp
@@ -90,6 +90,18 @@ void dxil::createShaderModelMD(Module &M) {
   Entry->addOperand(MDNode::get(Ctx, Vals));
 }
 
+void dxil::createDXILVersionMD(Module &M) {
+  Triple TT(Triple::normalize(M.getTargetTriple()));
+  VersionTuple Ver = TT.getDXILVersion();
+  LLVMContext &Ctx = M.getContext();
+  IRBuilder<> B(Ctx);
+  NamedMDNode *Entry = M.getOrInsertNamedMetadata("dx.version");
+  Metadata *Vals[2];
+  Vals[0] = ConstantAsMetadata::get(B.getInt32(Ver.getMajor()));
+  Vals[1] = ConstantAsMetadata::get(B.getInt32(Ver.getMinor().value_or(0)));
+  Entry->addOperand(MDNode::get(Ctx, Vals));
+}
+
 static uint32_t getShaderStage(Triple::EnvironmentType Env) {
   return (uint32_t)Env - (uint32_t)llvm::Triple::Pixel;
 }
diff --git a/llvm/lib/Target/DirectX/DXILMetadata.h b/llvm/lib/Target/DirectX/DXILMetadata.h
index cd9f4c83fbd0..e05db8d5370d 100644
--- a/llvm/lib/Target/DirectX/DXILMetadata.h
+++ b/llvm/lib/Target/DirectX/DXILMetadata.h
@@ -34,6 +34,7 @@ public:
 };
 
 void createShaderModelMD(Module &M);
+void createDXILVersionMD(Module &M);
 void createEntryMD(Module &M, const uint64_t ShaderFlags);
 
 } // namespace dxil
diff --git a/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp b/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp
index 80d94bf0c9d4..ae6d6f96904c 100644
--- a/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp
+++ b/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp
@@ -48,6 +48,7 @@ bool DXILTranslateMetadata::runOnModule(Module &M) {
   if (ValVerMD.isEmpty())
     ValVerMD.update(VersionTuple(1, 0));
   dxil::createShaderModelMD(M);
+  dxil::createDXILVersionMD(M);
 
   const dxil::Resources &Res =
       getAnalysis<DXILResourceWrapper>().getDXILResource();
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index f3f244c814e7..18ec60296810 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -1420,6 +1420,17 @@ VersionTuple Triple::getVulkanVersion() const {
   return VersionTuple(0);
 }
 
+VersionTuple Triple::getDXILVersion() const {
+  if (getArch() != dxil || getOS() != ShaderModel)
+    llvm_unreachable("invalid DXIL triple");
+  StringRef Arch = getArchName();
+  Arch.consume_front("dxilv");
+  VersionTuple DXILVersion = parseVersionFromName(Arch);
+  // FIXME: validate DXIL version against Shader Model version.
+  // Tracked by https://github.com/llvm/llvm-project/issues/91388
+  return DXILVersion;
+}
+
 void Triple::setTriple(const Twine &Str) {
   *this = Triple(Str);
 }
diff --git a/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.0.ll b/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.0.ll
new file mode 100644
index 000000000000..254479e5f94c
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.0.ll
@@ -0,0 +1,12 @@
+; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+target triple = "dxil-pc-shadermodel6.0-vertex"
+
+; CHECK: !dx.version = !{![[DXVER:[0-9]+]]}
+; CHECK: ![[DXVER]] = !{i32 1, i32 0}
+
+define void @entry() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { noinline nounwind "hlsl.shader"="vertex" }
diff --git a/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.8.ll b/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.8.ll
new file mode 100644
index 000000000000..efeb5a1b2486
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.8.ll
@@ -0,0 +1,12 @@
+; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+target triple = "dxil-pc-shadermodel6.8-compute"
+
+; CHECK: !dx.version = !{![[DXVER:[0-9]+]]}
+; CHECK: ![[DXVER]] = !{i32 1, i32 8}
+
+define void @entry() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { noinline nounwind "hlsl.numthreads"="1,2,1" "hlsl.shader"="compute" }
diff --git a/llvm/unittests/TargetParser/TripleTest.cpp b/llvm/unittests/TargetParser/TripleTest.cpp
index b8f5fbd87407..8e90ee6858f4 100644
--- a/llvm/unittests/TargetParser/TripleTest.cpp
+++ b/llvm/unittests/TargetParser/TripleTest.cpp
@@ -437,6 +437,85 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(VersionTuple(1, 3), T.getVulkanVersion());
   EXPECT_EQ(Triple::Compute, T.getEnvironment());
 
+  T = Triple("dxilv1.0--shadermodel6.0-pixel");
+  EXPECT_EQ(Triple::dxil, T.getArch());
+  EXPECT_EQ(Triple::DXILSubArch_v1_0, T.getSubArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::ShaderModel, T.getOS());
+  EXPECT_EQ(VersionTuple(1, 0), T.getDXILVersion());
+  EXPECT_EQ(Triple::Pixel, T.getEnvironment());
+
+  T = Triple("dxilv1.1--shadermodel6.1-vertex");
+  EXPECT_EQ(Triple::dxil, T.getArch());
+  EXPECT_EQ(Triple::DXILSubArch_v1_1, T.getSubArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::ShaderModel, T.getOS());
+  EXPECT_EQ(VersionTuple(1, 1), T.getDXILVersion());
+  EXPECT_EQ(Triple::Vertex, T.getEnvironment());
+
+  T = Triple("dxilv1.2--shadermodel6.2-geometry");
+  EXPECT_EQ(Triple::dxil, T.getArch());
+  EXPECT_EQ(Triple::DXILSubArch_v1_2, T.getSubArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::ShaderModel, T.getOS());
+  EXPECT_EQ(VersionTuple(1, 2), T.getDXILVersion());
+  EXPECT_EQ(Triple::Geometry, T.getEnvironment());
+
+  T = Triple("dxilv1.3--shadermodel6.3-library");
+  EXPECT_EQ(Triple::dxil, T.getArch());
+  EXPECT_EQ(Triple::DXILSubArch_v1_3, T.getSubArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::ShaderModel, T.getOS());
+  EXPECT_EQ(VersionTuple(1, 3), T.getDXILVersion());
+  EXPECT_EQ(Triple::Library, T.getEnvironment());
+
+  T = Triple("dxilv1.4--shadermodel6.4-hull");
+  EXPECT_EQ(Triple::dxil, T.getArch());
+  EXPECT_EQ(Triple::DXILSubArch_v1_4, T.getSubArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::ShaderModel, T.getOS());
+  EXPECT_EQ(VersionTuple(1, 4), T.getDXILVersion());
+  EXPECT_EQ(Triple::Hull, T.getEnvironment());
+
+  T = Triple("dxilv1.5--shadermodel6.5-domain");
+  EXPECT_EQ(Triple::dxil, T.getArch());
+  EXPECT_EQ(Triple::DXILSubArch_v1_5, T.getSubArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::ShaderModel, T.getOS());
+  EXPECT_EQ(VersionTuple(1, 5), T.getDXILVersion());
+  EXPECT_EQ(Triple::Domain, T.getEnvironment());
+
+  T = Triple("dxilv1.6--shadermodel6.6-compute");
+  EXPECT_EQ(Triple::dxil, T.getArch());
+  EXPECT_EQ(Triple::DXILSubArch_v1_6, T.getSubArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::ShaderModel, T.getOS());
+  EXPECT_EQ(VersionTuple(1, 6), T.getDXILVersion());
+  EXPECT_EQ(Triple::Compute, T.getEnvironment());
+
+  T = Triple("dxilv1.7-unknown-shadermodel6.7-mesh");
+  EXPECT_EQ(Triple::dxil, T.getArch());
+  EXPECT_EQ(Triple::DXILSubArch_v1_7, T.getSubArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::ShaderModel, T.getOS());
+  EXPECT_EQ(VersionTuple(1, 7), T.getDXILVersion());
+  EXPECT_EQ(Triple::Mesh, T.getEnvironment());
+
+  T = Triple("dxilv1.8-unknown-shadermodel6.8-amplification");
+  EXPECT_EQ(Triple::dxil, T.getArch());
+  EXPECT_EQ(Triple::DXILSubArch_v1_8, T.getSubArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::ShaderModel, T.getOS());
+  EXPECT_EQ(VersionTuple(1, 8), T.getDXILVersion());
+  EXPECT_EQ(Triple::Amplification, T.getEnvironment());
+
+  T = Triple("dxilv1.8-unknown-shadermodel6.15-library");
+  EXPECT_EQ(Triple::dxil, T.getArch());
+  EXPECT_EQ(Triple::DXILSubArch_v1_8, T.getSubArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::ShaderModel, T.getOS());
+  EXPECT_EQ(VersionTuple(1, 8), T.getDXILVersion());
+
   T = Triple("x86_64-unknown-fuchsia");
   EXPECT_EQ(Triple::x86_64, T.getArch());
   EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
-- 
GitLab


From 40b322baef143df271d8b9142028aaeeec2a4d78 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 8 May 2024 14:46:27 +0100
Subject: [PATCH 0170/1206] [SCEV] Add tests for missed NSW preservation during
 loop guard handling.

Add test coverage for missed simplification.
---
 .../backedge-taken-count-guard-info.ll        | 34 ++++++++++++++
 ...count-expansion-loop-guard-preserve-nsw.ll | 47 +++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 llvm/test/Transforms/IndVarSimplify/trip-count-expansion-loop-guard-preserve-nsw.ll

diff --git a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll
index 1f475e80e562..da4487ce9cd4 100644
--- a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll
+++ b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll
@@ -69,4 +69,38 @@ exit:
   ret void
 }
 
+declare void @use(i32)
+
+define void @rewrite_preserve_add_nsw(i32 %a) {
+; CHECK-LABEL: 'rewrite_preserve_add_nsw'
+; CHECK-NEXT:  Classifying expressions for: @rewrite_preserve_add_nsw
+; CHECK-NEXT:    %add = add nsw i32 %a, 4
+; CHECK-NEXT:    --> (4 + %a)<nsw> U: [-2147483644,-2147483648) S: [-2147483644,-2147483648)
+; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<nuw><%loop> U: [0,-2147483648) S: [0,-2147483648) Exits: (0 smax (4 + %a)<nsw>) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i32 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<nuw><%loop> U: [1,-2147483647) S: [1,-2147483647) Exits: (1 + (0 smax (4 + %a)<nsw>))<nuw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @rewrite_preserve_add_nsw
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (0 smax (4 + %a)<nsw>)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 2147483647
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (0 smax (4 + %a)<nsw>)
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+entry:
+  %add = add nsw i32 %a, 4
+  call void @use(i32 noundef %add)
+  %pre = icmp sgt i32 %a, -4
+  br i1 %pre, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  call void @clobber()
+  %iv.next = add i32 %iv, 1
+  %ec = icmp slt i32 %iv, %add
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  ret void
+}
+
 declare void @clobber()
diff --git a/llvm/test/Transforms/IndVarSimplify/trip-count-expansion-loop-guard-preserve-nsw.ll b/llvm/test/Transforms/IndVarSimplify/trip-count-expansion-loop-guard-preserve-nsw.ll
new file mode 100644
index 000000000000..f86639ea4c50
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/trip-count-expansion-loop-guard-preserve-nsw.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=indvars -S %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128-Fn32"
+
+define void @rewrite_preserve_add_nsw(i32 %a) {
+; CHECK-LABEL: define void @rewrite_preserve_add_nsw(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[A]], 4
+; CHECK-NEXT:    call void @use(i32 noundef [[ADD]])
+; CHECK-NEXT:    [[PRE:%.*]] = icmp sgt i32 [[A]], -4
+; CHECK-NEXT:    br i1 [[PRE]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[ADD]], i32 0)
+; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i32 [[SMAX]], 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @clobber()
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EC]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add = add nsw i32 %a, 4
+  call void @use(i32 noundef %add)
+  %pre = icmp sgt i32 %a, -4
+  br i1 %pre, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  call void @clobber()
+  %iv.next = add i32 %iv, 1
+  %ec = icmp slt i32 %iv, %add
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+declare void @clobber()
+declare void @use(i32)
-- 
GitLab


From f97f04ec4392c79163179331211ea8c48e61799b Mon Sep 17 00:00:00 2001
From: Daniel Chen <cdchen@ca.ibm.com>
Date: Wed, 8 May 2024 09:47:53 -0400
Subject: [PATCH 0171/1206] [mlir] Fixing a regression that '-D' option of
 llvm-tblgen is unregistered. (#91329)

PR #89664 introduced a regression that it unregistered llvm-tblgen
option `-D` for macros. The test `TestOps.cpp` failed due to passing a
macros to llvm-tblgen.

It caused our internal build to fail because we append `-DLOCAL_NAME`
into `LLVM_TABLEGEN_FLANGS` in `llvm/lib/cmake/llvm/TableGen.cmake` as

```
list(APPEND LLVM_TABLEGEN_FLAGS "-DLOCAL_NAME")
```

And in `./llvm/lib/Target/PowerPC/PPC.td`, we check it for some
downstream code as:

```
...
#ifdef LOCAL_NAME
...
#endif
```
Now we got error message from mlir-src-sharder as
```
mlir-src-sharder -op-shard-index=1 -DLOCAL_NAME llvm-project/mlir/test/lib/Dialect/Test/TestOps.cpp --write-if-changed -o tools/mlir/test/lib/Dialect/Test/TestOps.1.cpp -d tools/mlir/test/lib/Dialect/Test/TestOps.1.cpp.d
mlir-src-sharder: Unknown command line argument '-DLOCAL_NAME'.  Try: 'llvm-project/build/bin/mlir-src-sharder --help'
mlir-src-sharder: Did you mean '-I'?
```

This PR is to fix the regression.
---
 mlir/tools/mlir-src-sharder/mlir-src-sharder.cpp | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/mlir/tools/mlir-src-sharder/mlir-src-sharder.cpp b/mlir/tools/mlir-src-sharder/mlir-src-sharder.cpp
index dc1e2939c7d2..5bfc24ef3b47 100644
--- a/mlir/tools/mlir-src-sharder/mlir-src-sharder.cpp
+++ b/mlir/tools/mlir-src-sharder/mlir-src-sharder.cpp
@@ -62,6 +62,16 @@ int main(int argc, char **argv) {
       "write-if-changed",
       llvm::cl::desc("Only write to the output file if it changed"));
 
+  // `ResetCommandLineParser` at the above unregistered the "D" option
+  // of `llvm-tblgen`, which caused `TestOps.cpp` to fail due to
+  // "Unknnown command line argument '-D...`" when a macros name is
+  // present. The following is a workaround to re-register it again.
+  llvm::cl::list<std::string> MacroNames(
+      "D",
+      llvm::cl::desc(
+          "Name of the macro to be defined -- ignored by mlir-src-sharder"),
+      llvm::cl::value_desc("macro name"), llvm::cl::Prefix);
+
   llvm::InitLLVM y(argc, argv);
   llvm::cl::ParseCommandLineOptions(argc, argv);
 
-- 
GitLab


From 9c09b0840e82490ed194207adc03d3e7284b8764 Mon Sep 17 00:00:00 2001
From: Xiang Li <python3kgae@outlook.com>
Date: Wed, 8 May 2024 06:48:04 -0700
Subject: [PATCH 0172/1206] Revert "[HLSL] Support packoffset attribute in AST
 (#89836)" (#91473)

This reverts commit c5509fedc5757fffece385d9d068e36b26793ade.
---
 clang/include/clang/Basic/Attr.td             |  12 --
 clang/include/clang/Basic/AttrDocs.td         |  20 ---
 clang/include/clang/Basic/DiagnosticGroups.td |   3 -
 .../clang/Basic/DiagnosticParseKinds.td       |   2 -
 .../clang/Basic/DiagnosticSemaKinds.td        |   5 -
 clang/lib/Parse/ParseHLSL.cpp                 |  88 -------------
 clang/lib/Sema/SemaDeclAttr.cpp               |  52 --------
 clang/lib/Sema/SemaHLSL.cpp                   |  80 ------------
 clang/test/AST/HLSL/packoffset.hlsl           | 100 --------------
 clang/test/SemaHLSL/packoffset-invalid.hlsl   | 122 ------------------
 10 files changed, 484 deletions(-)
 delete mode 100644 clang/test/AST/HLSL/packoffset.hlsl
 delete mode 100644 clang/test/SemaHLSL/packoffset-invalid.hlsl

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 52552ba48856..0225598cbbe8 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4415,18 +4415,6 @@ def HLSLResourceBinding: InheritableAttr {
   let Documentation = [HLSLResourceBindingDocs];
 }
 
-def HLSLPackOffset: HLSLAnnotationAttr {
-  let Spellings = [HLSLAnnotation<"packoffset">];
-  let LangOpts = [HLSL];
-  let Args = [IntArgument<"Subcomponent">, IntArgument<"Component">];
-  let Documentation = [HLSLPackOffsetDocs];
-  let AdditionalMembers = [{
-      unsigned getOffset() {
-        return subcomponent * 4 + component;
-      }
-  }];
-}
-
 def HLSLSV_DispatchThreadID: HLSLAnnotationAttr {
   let Spellings = [HLSLAnnotation<"SV_DispatchThreadID">];
   let Subjects = SubjectList<[ParmVar, Field]>;
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index f351822ac74b..8e6faabfae64 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -7408,26 +7408,6 @@ The full documentation is available here: https://docs.microsoft.com/en-us/windo
   }];
 }
 
-def HLSLPackOffsetDocs : Documentation {
-  let Category = DocCatFunction;
-  let Content = [{
-The packoffset attribute is used to change the layout of a cbuffer.
-Attribute spelling in HLSL is: ``packoffset( c[Subcomponent][.component] )``.
-A subcomponent is a register number, which is an integer. A component is in the form of [.xyzw].
-
-Examples:
-
-.. code-block:: c++
-
-  cbuffer A {
-    float3 a : packoffset(c0.y);
-    float4 b : packoffset(c4);
-  }
-
-The full documentation is available here: https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-variable-packoffset
-  }];
-}
-
 def HLSLSV_DispatchThreadIDDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 2beb1d45124b..60f87da2a738 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -1507,9 +1507,6 @@ def BranchProtection : DiagGroup<"branch-protection">;
 // Warnings for HLSL Clang extensions
 def HLSLExtension : DiagGroup<"hlsl-extensions">;
 
-// Warning for mix packoffset and non-packoffset.
-def HLSLMixPackOffset : DiagGroup<"mix-packoffset">;
-
 // Warnings for DXIL validation
 def DXILValidation : DiagGroup<"dxil-validation">;
 
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index bc9d7cacc50b..fdffb35ea0d9 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1754,7 +1754,5 @@ def err_hlsl_separate_attr_arg_and_number : Error<"wrong argument format for hls
 def ext_hlsl_access_specifiers : ExtWarn<
   "access specifiers are a clang HLSL extension">,
   InGroup<HLSLExtension>;
-def err_hlsl_unsupported_component : Error<"invalid component '%0' used; expected 'x', 'y', 'z', or 'w'">;
-def err_hlsl_packoffset_invalid_reg : Error<"invalid resource class specifier '%0' for packoffset, expected 'c'">;
 
 } // end of Parser diagnostics
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index d6863f90edb6..9317ae675c72 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12184,11 +12184,6 @@ def err_hlsl_init_priority_unsupported : Error<
 def err_hlsl_unsupported_register_type : Error<"invalid resource class specifier '%0' used; expected 'b', 's', 't', or 'u'">;
 def err_hlsl_unsupported_register_number : Error<"register number should be an integer">;
 def err_hlsl_expected_space : Error<"invalid space specifier '%0' used; expected 'space' followed by an integer, like space1">;
-def warn_hlsl_packoffset_mix : Warning<"cannot mix packoffset elements with nonpackoffset elements in a cbuffer">,
-    InGroup<HLSLMixPackOffset>;
-def err_hlsl_packoffset_overlap : Error<"packoffset overlap between %0, %1">;
-def err_hlsl_packoffset_cross_reg_boundary : Error<"packoffset cannot cross register boundary">;
-def err_hlsl_packoffset_alignment_mismatch : Error<"packoffset at 'y' not match alignment %0 required by %1">;
 def err_hlsl_pointers_unsupported : Error<
   "%select{pointers|references}0 are unsupported in HLSL">;
 
diff --git a/clang/lib/Parse/ParseHLSL.cpp b/clang/lib/Parse/ParseHLSL.cpp
index e9c8d6dca7bf..f4cbece31f18 100644
--- a/clang/lib/Parse/ParseHLSL.cpp
+++ b/clang/lib/Parse/ParseHLSL.cpp
@@ -183,94 +183,6 @@ void Parser::ParseHLSLAnnotations(ParsedAttributes &Attrs,
       return;
     }
   } break;
-  case ParsedAttr::AT_HLSLPackOffset: {
-    // Parse 'packoffset( c[Subcomponent][.component] )'.
-    // Check '('.
-    if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after)) {
-      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
-      return;
-    }
-    // Check c[Subcomponent] as an identifier.
-    if (!Tok.is(tok::identifier)) {
-      Diag(Tok.getLocation(), diag::err_expected) << tok::identifier;
-      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
-      return;
-    }
-    StringRef OffsetStr = Tok.getIdentifierInfo()->getName();
-    SourceLocation SubComponentLoc = Tok.getLocation();
-    if (OffsetStr[0] != 'c') {
-      Diag(Tok.getLocation(), diag::err_hlsl_packoffset_invalid_reg)
-          << OffsetStr;
-      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
-      return;
-    }
-    OffsetStr = OffsetStr.substr(1);
-    unsigned SubComponent = 0;
-    if (!OffsetStr.empty()) {
-      // Make sure SubComponent is a number.
-      if (OffsetStr.getAsInteger(10, SubComponent)) {
-        Diag(SubComponentLoc.getLocWithOffset(1),
-             diag::err_hlsl_unsupported_register_number);
-        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
-        return;
-      }
-    }
-    unsigned Component = 0;
-    ConsumeToken(); // consume identifier.
-    SourceLocation ComponentLoc;
-    if (Tok.is(tok::period)) {
-      ConsumeToken(); // consume period.
-      if (!Tok.is(tok::identifier)) {
-        Diag(Tok.getLocation(), diag::err_expected) << tok::identifier;
-        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
-        return;
-      }
-      StringRef ComponentStr = Tok.getIdentifierInfo()->getName();
-      ComponentLoc = Tok.getLocation();
-      ConsumeToken(); // consume identifier.
-      // Make sure Component is a single character.
-      if (ComponentStr.size() != 1) {
-        Diag(ComponentLoc, diag::err_hlsl_unsupported_component)
-            << ComponentStr;
-        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
-        return;
-      }
-      switch (ComponentStr[0]) {
-      case 'x':
-      case 'r':
-        Component = 0;
-        break;
-      case 'y':
-      case 'g':
-        Component = 1;
-        break;
-      case 'z':
-      case 'b':
-        Component = 2;
-        break;
-      case 'w':
-      case 'a':
-        Component = 3;
-        break;
-      default:
-        Diag(ComponentLoc, diag::err_hlsl_unsupported_component)
-            << ComponentStr;
-        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
-        return;
-      }
-    }
-    ASTContext &Ctx = Actions.getASTContext();
-    QualType SizeTy = Ctx.getSizeType();
-    uint64_t SizeTySize = Ctx.getTypeSize(SizeTy);
-    ArgExprs.push_back(IntegerLiteral::Create(
-        Ctx, llvm::APInt(SizeTySize, SubComponent), SizeTy, SubComponentLoc));
-    ArgExprs.push_back(IntegerLiteral::Create(
-        Ctx, llvm::APInt(SizeTySize, Component), SizeTy, ComponentLoc));
-    if (ExpectAndConsume(tok::r_paren, diag::err_expected)) {
-      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
-      return;
-    }
-  } break;
   case ParsedAttr::UnknownAttribute:
     Diag(Loc, diag::err_unknown_hlsl_semantic) << II;
     return;
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 6d957ac09e1c..6ca42856459f 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -7309,55 +7309,6 @@ static void handleHLSLSV_DispatchThreadIDAttr(Sema &S, Decl *D,
   D->addAttr(::new (S.Context) HLSLSV_DispatchThreadIDAttr(S.Context, AL));
 }
 
-static void handleHLSLPackOffsetAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
-  if (!isa<VarDecl>(D) || !isa<HLSLBufferDecl>(D->getDeclContext())) {
-    S.Diag(AL.getLoc(), diag::err_hlsl_attr_invalid_ast_node)
-        << AL << "shader constant in a constant buffer";
-    return;
-  }
-
-  uint32_t SubComponent;
-  if (!checkUInt32Argument(S, AL, AL.getArgAsExpr(0), SubComponent))
-    return;
-  uint32_t Component;
-  if (!checkUInt32Argument(S, AL, AL.getArgAsExpr(1), Component))
-    return;
-
-  QualType T = cast<VarDecl>(D)->getType().getCanonicalType();
-  // Check if T is an array or struct type.
-  // TODO: mark matrix type as aggregate type.
-  bool IsAggregateTy = (T->isArrayType() || T->isStructureType());
-
-  // Check Component is valid for T.
-  if (Component) {
-    unsigned Size = S.getASTContext().getTypeSize(T);
-    if (IsAggregateTy || Size > 128) {
-      S.Diag(AL.getLoc(), diag::err_hlsl_packoffset_cross_reg_boundary);
-      return;
-    } else {
-      // Make sure Component + sizeof(T) <= 4.
-      if ((Component * 32 + Size) > 128) {
-        S.Diag(AL.getLoc(), diag::err_hlsl_packoffset_cross_reg_boundary);
-        return;
-      }
-      QualType EltTy = T;
-      if (const auto *VT = T->getAs<VectorType>())
-        EltTy = VT->getElementType();
-      unsigned Align = S.getASTContext().getTypeAlign(EltTy);
-      if (Align > 32 && Component == 1) {
-        // NOTE: Component 3 will hit err_hlsl_packoffset_cross_reg_boundary.
-        // So we only need to check Component 1 here.
-        S.Diag(AL.getLoc(), diag::err_hlsl_packoffset_alignment_mismatch)
-            << Align << EltTy;
-        return;
-      }
-    }
-  }
-
-  D->addAttr(::new (S.Context)
-                 HLSLPackOffsetAttr(S.Context, AL, SubComponent, Component));
-}
-
 static void handleHLSLShaderAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   StringRef Str;
   SourceLocation ArgLoc;
@@ -9779,9 +9730,6 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_HLSLSV_DispatchThreadID:
     handleHLSLSV_DispatchThreadIDAttr(S, D, AL);
     break;
-  case ParsedAttr::AT_HLSLPackOffset:
-    handleHLSLPackOffsetAttr(S, D, AL);
-    break;
   case ParsedAttr::AT_HLSLShader:
     handleHLSLShaderAttr(S, D, AL);
     break;
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 6a12c417e2f3..bb9e37f18d37 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -39,89 +39,9 @@ Decl *SemaHLSL::ActOnStartBuffer(Scope *BufferScope, bool CBuffer,
   return Result;
 }
 
-// Calculate the size of a legacy cbuffer type based on
-// https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-packing-rules
-static unsigned calculateLegacyCbufferSize(const ASTContext &Context,
-                                           QualType T) {
-  unsigned Size = 0;
-  constexpr unsigned CBufferAlign = 128;
-  if (const RecordType *RT = T->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
-    for (const FieldDecl *Field : RD->fields()) {
-      QualType Ty = Field->getType();
-      unsigned FieldSize = calculateLegacyCbufferSize(Context, Ty);
-      unsigned FieldAlign = 32;
-      if (Ty->isAggregateType())
-        FieldAlign = CBufferAlign;
-      Size = llvm::alignTo(Size, FieldAlign);
-      Size += FieldSize;
-    }
-  } else if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T)) {
-    if (unsigned ElementCount = AT->getSize().getZExtValue()) {
-      unsigned ElementSize =
-          calculateLegacyCbufferSize(Context, AT->getElementType());
-      unsigned AlignedElementSize = llvm::alignTo(ElementSize, CBufferAlign);
-      Size = AlignedElementSize * (ElementCount - 1) + ElementSize;
-    }
-  } else if (const VectorType *VT = T->getAs<VectorType>()) {
-    unsigned ElementCount = VT->getNumElements();
-    unsigned ElementSize =
-        calculateLegacyCbufferSize(Context, VT->getElementType());
-    Size = ElementSize * ElementCount;
-  } else {
-    Size = Context.getTypeSize(T);
-  }
-  return Size;
-}
-
 void SemaHLSL::ActOnFinishBuffer(Decl *Dcl, SourceLocation RBrace) {
   auto *BufDecl = cast<HLSLBufferDecl>(Dcl);
   BufDecl->setRBraceLoc(RBrace);
-
-  // Validate packoffset.
-  llvm::SmallVector<std::pair<VarDecl *, HLSLPackOffsetAttr *>> PackOffsetVec;
-  bool HasPackOffset = false;
-  bool HasNonPackOffset = false;
-  for (auto *Field : BufDecl->decls()) {
-    VarDecl *Var = dyn_cast<VarDecl>(Field);
-    if (!Var)
-      continue;
-    if (Field->hasAttr<HLSLPackOffsetAttr>()) {
-      PackOffsetVec.emplace_back(Var, Field->getAttr<HLSLPackOffsetAttr>());
-      HasPackOffset = true;
-    } else {
-      HasNonPackOffset = true;
-    }
-  }
-
-  if (HasPackOffset && HasNonPackOffset)
-    Diag(BufDecl->getLocation(), diag::warn_hlsl_packoffset_mix);
-
-  if (HasPackOffset) {
-    ASTContext &Context = getASTContext();
-    // Make sure no overlap in packoffset.
-    // Sort PackOffsetVec by offset.
-    std::sort(PackOffsetVec.begin(), PackOffsetVec.end(),
-              [](const std::pair<VarDecl *, HLSLPackOffsetAttr *> &LHS,
-                 const std::pair<VarDecl *, HLSLPackOffsetAttr *> &RHS) {
-                return LHS.second->getOffset() < RHS.second->getOffset();
-              });
-
-    for (unsigned i = 0; i < PackOffsetVec.size() - 1; i++) {
-      VarDecl *Var = PackOffsetVec[i].first;
-      HLSLPackOffsetAttr *Attr = PackOffsetVec[i].second;
-      unsigned Size = calculateLegacyCbufferSize(Context, Var->getType());
-      unsigned Begin = Attr->getOffset() * 32;
-      unsigned End = Begin + Size;
-      unsigned NextBegin = PackOffsetVec[i + 1].second->getOffset() * 32;
-      if (End > NextBegin) {
-        VarDecl *NextVar = PackOffsetVec[i + 1].first;
-        Diag(NextVar->getLocation(), diag::err_hlsl_packoffset_overlap)
-            << NextVar << Var;
-      }
-    }
-  }
-
   SemaRef.PopDeclContext();
 }
 
diff --git a/clang/test/AST/HLSL/packoffset.hlsl b/clang/test/AST/HLSL/packoffset.hlsl
deleted file mode 100644
index 9cfd88eeec33..000000000000
--- a/clang/test/AST/HLSL/packoffset.hlsl
+++ /dev/null
@@ -1,100 +0,0 @@
-// RUN: %clang_cc1 -triple dxil-unknown-shadermodel6.3-library -S -finclude-default-header  -ast-dump  -x hlsl %s | FileCheck %s
-
-
-// CHECK: HLSLBufferDecl {{.*}} cbuffer A
-cbuffer A
-{
-    // CHECK-NEXT: VarDecl {{.*}} A1 'float4'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 0
-    float4 A1 : packoffset(c);
-    // CHECK-NEXT: VarDecl {{.*}} col:11 A2 'float'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 0
-    float A2 : packoffset(c1);
-    // CHECK-NEXT: VarDecl {{.*}} col:11 A3 'float'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 1
-    float A3 : packoffset(c1.y);
-}
-
-// CHECK: HLSLBufferDecl {{.*}} cbuffer B
-cbuffer B
-{
-    // CHECK: VarDecl {{.*}} B0 'float'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
-    float B0 : packoffset(c0.g);
-    // CHECK-NEXT: VarDecl {{.*}} B1 'double'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 2
-	double B1 : packoffset(c0.b);
-    // CHECK-NEXT: VarDecl {{.*}} B2 'half'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 0
-	half B2 : packoffset(c0.r);
-}
-
-// CHECK: HLSLBufferDecl {{.*}} cbuffer C
-cbuffer C
-{
-    // CHECK: VarDecl {{.*}} C0 'float'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1
-    float C0 : packoffset(c0.y);
-    // CHECK-NEXT: VarDecl {{.*}} C1 'float2'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2
-	float2 C1 : packoffset(c0.z);
-    // CHECK-NEXT: VarDecl {{.*}} C2 'half'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0
-	half C2 : packoffset(c0.x);
-}
-
-
-// CHECK: HLSLBufferDecl {{.*}} cbuffer D
-cbuffer D
-{
-    // CHECK: VarDecl {{.*}} D0 'float'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
-    float D0 : packoffset(c0.y);
-    // CHECK-NEXT: VarDecl {{.*}} D1 'float[2]'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 0
-	float D1[2] : packoffset(c1.x);
-    // CHECK-NEXT: VarDecl {{.*}} D2 'half3'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2 1
-	half3 D2 : packoffset(c2.y);
-    // CHECK-NEXT: VarDecl {{.*}} D3 'double'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 2
-	double D3 : packoffset(c0.z);
-}
-
-struct ST {
-  float a;
-  float2 b;
-  half c;
-};
-
-// CHECK: HLSLBufferDecl {{.*}} cbuffer S
-cbuffer S {
-    // CHECK: VarDecl {{.*}} S0 'float'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
-  float S0 : packoffset(c0.y);
-    // CHECK: VarDecl {{.*}} S1 'ST'
-    // CHECK: HLSLPackOffsetAttr {{.*}} 1 0
-  ST S1 : packoffset(c1);
-    // CHECK: VarDecl {{.*}} S2 'double2'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2 0
-  double2 S2 : packoffset(c2);
-}
-
-struct ST2 {
-  float s0;
-  ST s1;
-  half s2;
-};
-
-// CHECK: HLSLBufferDecl {{.*}} cbuffer S2
-cbuffer S2 {
-    // CHECK: VarDecl {{.*}} S20 'float'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 3
-  float S20 : packoffset(c0.a);
-    // CHECK: VarDecl {{.*}} S21 'ST2'
-    // CHECK: HLSLPackOffsetAttr {{.*}} 1 0
-  ST2 S21 : packoffset(c1);
-    // CHECK: VarDecl {{.*}} S22 'half'
-    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 3 1
-  half S22 : packoffset(c3.y);
-}
diff --git a/clang/test/SemaHLSL/packoffset-invalid.hlsl b/clang/test/SemaHLSL/packoffset-invalid.hlsl
deleted file mode 100644
index c5983f6fd7e0..000000000000
--- a/clang/test/SemaHLSL/packoffset-invalid.hlsl
+++ /dev/null
@@ -1,122 +0,0 @@
-// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library -verify %s
-
-// expected-warning@+1{{cannot mix packoffset elements with nonpackoffset elements in a cbuffer}}
-cbuffer Mix
-{
-    float4 M1 : packoffset(c0);
-    float M2;
-    float M3 : packoffset(c1.y);
-}
-
-// expected-warning@+1{{cannot mix packoffset elements with nonpackoffset elements in a cbuffer}}
-cbuffer Mix2
-{
-    float4 M4;
-    float M5 : packoffset(c1.y);
-    float M6 ;
-}
-
-// expected-error@+1{{attribute 'packoffset' only applies to shader constant in a constant buffer}}
-float4 g : packoffset(c0);
-
-cbuffer IllegalOffset
-{
-    // expected-error@+1{{invalid resource class specifier 't2' for packoffset, expected 'c'}}
-    float4 i1 : packoffset(t2);
-    // expected-error@+1{{invalid component 'm' used; expected 'x', 'y', 'z', or 'w'}}
-    float i2 : packoffset(c1.m);
-}
-
-cbuffer Overlap
-{
-    float4 o1 : packoffset(c0);
-    // expected-error@+1{{packoffset overlap between 'o2', 'o1'}}
-    float2 o2 : packoffset(c0.z);
-}
-
-cbuffer CrossReg
-{
-    // expected-error@+1{{packoffset cannot cross register boundary}}
-    float4 c1 : packoffset(c0.y);
-    // expected-error@+1{{packoffset cannot cross register boundary}}
-    float2 c2 : packoffset(c1.w);
-}
-
-struct ST {
-  float s;
-};
-
-cbuffer Aggregate
-{
-    // expected-error@+1{{packoffset cannot cross register boundary}}
-    ST A1 : packoffset(c0.y);
-    // expected-error@+1{{packoffset cannot cross register boundary}}
-    float A2[2] : packoffset(c1.w);
-}
-
-cbuffer Double {
-    // expected-error@+1{{packoffset at 'y' not match alignment 64 required by 'double'}}
-    double d : packoffset(c.y);
-    // expected-error@+1{{packoffset cannot cross register boundary}}
-	double2 d2 : packoffset(c.z);
-    // expected-error@+1{{packoffset cannot cross register boundary}}
-	double3 d3 : packoffset(c.z);
-}
-
-cbuffer ParsingFail {
-// expected-error@+1{{expected identifier}}
-float pf0 : packoffset();
-// expected-error@+1{{expected identifier}}
-float pf1 : packoffset((c0));
-// expected-error@+1{{expected ')'}}
-float pf2 : packoffset(c0, x);
-// expected-error@+1{{invalid component 'X' used}}
-float pf3 : packoffset(c.X);
-// expected-error@+1{{expected '(' after ''}}
-float pf4 : packoffset;
-// expected-error@+1{{expected identifier}}
-float pf5 : packoffset(;
-// expected-error@+1{{expected '(' after '}}
-float pf6 : packoffset);
-// expected-error@+1{{expected '(' after '}}
-float pf7 : packoffset c0.x;
-
-// expected-error@+1{{invalid component 'xy' used}}
-float pf8 : packoffset(c0.xy);
-// expected-error@+1{{invalid component 'rg' used}}
-float pf9 : packoffset(c0.rg);
-// expected-error@+1{{invalid component 'yes' used}}
-float pf10 : packoffset(c0.yes);
-// expected-error@+1{{invalid component 'woo'}}
-float pf11 : packoffset(c0.woo);
-// expected-error@+1{{invalid component 'xr' used}}
-float pf12 : packoffset(c0.xr);
-}
-
-struct ST2 {
-  float a;
-  float2 b;
-};
-
-cbuffer S {
-  float S0 : packoffset(c0.y);
-  ST2 S1[2] : packoffset(c1);
-  // expected-error@+1{{packoffset overlap between 'S2', 'S1'}}
-  half2 S2 : packoffset(c1.w);
-  half2 S3 : packoffset(c2.w);
-}
-
-struct ST23 {
-  float s0;
-  ST2 s1;
-};
-
-cbuffer S2 {
-  float S20 : packoffset(c0.y);
-  ST2 S21 : packoffset(c1);
-  half2 S22 : packoffset(c2.w);
-  double S23[2] : packoffset(c3);
-  // expected-error@+1{{packoffset overlap between 'S24', 'S23'}}
-  float S24 : packoffset(c3.z);
-  float S25 : packoffset(c4.z);
-}
-- 
GitLab


From c6efcc925c9969d616bc463171c0423d6a9766af Mon Sep 17 00:00:00 2001
From: Christian Ulmann <christianulmann@gmail.com>
Date: Wed, 8 May 2024 15:53:14 +0200
Subject: [PATCH 0173/1206] [MLIR][Mem2Reg] Improve performance by avoiding
 recomputations (#91444)

This commit ensures that Mem2Reg reuses the `DominanceInfo` as well as
block index maps to avoid expensive recomputations. Due to the recent
migration to `OpBuilder`, the promotion of a slot does no longer replace
blocks. Having stable blocks makes the `DominanceInfo` preservable and
additionally allows to cache block index maps between different
promotions.

Performance measurements on very large functions show an up to 4x
speedup by these changes.
---
 mlir/include/mlir/Transforms/Mem2Reg.h |  1 +
 mlir/lib/Transforms/Mem2Reg.cpp        | 65 ++++++++++++++++++--------
 2 files changed, 47 insertions(+), 19 deletions(-)

diff --git a/mlir/include/mlir/Transforms/Mem2Reg.h b/mlir/include/mlir/Transforms/Mem2Reg.h
index b4f939d65414..fee7fb312750 100644
--- a/mlir/include/mlir/Transforms/Mem2Reg.h
+++ b/mlir/include/mlir/Transforms/Mem2Reg.h
@@ -28,6 +28,7 @@ struct Mem2RegStatistics {
 LogicalResult
 tryToPromoteMemorySlots(ArrayRef<PromotableAllocationOpInterface> allocators,
                         OpBuilder &builder, const DataLayout &dataLayout,
+                        DominanceInfo &dominance,
                         Mem2RegStatistics statistics = {});
 
 } // namespace mlir
diff --git a/mlir/lib/Transforms/Mem2Reg.cpp b/mlir/lib/Transforms/Mem2Reg.cpp
index 1d7ba4ca4f83..8adbbcd01cb4 100644
--- a/mlir/lib/Transforms/Mem2Reg.cpp
+++ b/mlir/lib/Transforms/Mem2Reg.cpp
@@ -18,7 +18,6 @@
 #include "mlir/Transforms/Passes.h"
 #include "mlir/Transforms/RegionUtils.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/GenericIteratedDominanceFrontier.h"
 
 namespace mlir {
@@ -158,6 +157,8 @@ private:
   const DataLayout &dataLayout;
 };
 
+using BlockIndexCache = DenseMap<Region *, DenseMap<Block *, size_t>>;
+
 /// The MemorySlotPromoter handles the state of promoting a memory slot. It
 /// wraps a slot and its associated allocator. This will perform the mutation of
 /// IR.
@@ -166,7 +167,8 @@ public:
   MemorySlotPromoter(MemorySlot slot, PromotableAllocationOpInterface allocator,
                      OpBuilder &builder, DominanceInfo &dominance,
                      const DataLayout &dataLayout, MemorySlotPromotionInfo info,
-                     const Mem2RegStatistics &statistics);
+                     const Mem2RegStatistics &statistics,
+                     BlockIndexCache &blockIndexCache);
 
   /// Actually promotes the slot by mutating IR. Promoting a slot DOES
   /// invalidate the MemorySlotPromotionInfo of other slots. Preparation of
@@ -207,6 +209,9 @@ private:
   const DataLayout &dataLayout;
   MemorySlotPromotionInfo info;
   const Mem2RegStatistics &statistics;
+
+  /// Shared cache of block indices of specific regions.
+  BlockIndexCache &blockIndexCache;
 };
 
 } // namespace
@@ -214,9 +219,11 @@ private:
 MemorySlotPromoter::MemorySlotPromoter(
     MemorySlot slot, PromotableAllocationOpInterface allocator,
     OpBuilder &builder, DominanceInfo &dominance, const DataLayout &dataLayout,
-    MemorySlotPromotionInfo info, const Mem2RegStatistics &statistics)
+    MemorySlotPromotionInfo info, const Mem2RegStatistics &statistics,
+    BlockIndexCache &blockIndexCache)
     : slot(slot), allocator(allocator), builder(builder), dominance(dominance),
-      dataLayout(dataLayout), info(std::move(info)), statistics(statistics) {
+      dataLayout(dataLayout), info(std::move(info)), statistics(statistics),
+      blockIndexCache(blockIndexCache) {
 #ifndef NDEBUG
   auto isResultOrNewBlockArgument = [&]() {
     if (BlockArgument arg = dyn_cast<BlockArgument>(slot.ptr))
@@ -500,15 +507,29 @@ void MemorySlotPromoter::computeReachingDefInRegion(Region *region,
   }
 }
 
+/// Gets or creates a block index mapping for `region`.
+static const DenseMap<Block *, size_t> &
+getOrCreateBlockIndices(BlockIndexCache &blockIndexCache, Region *region) {
+  auto [it, inserted] = blockIndexCache.try_emplace(region);
+  if (!inserted)
+    return it->second;
+
+  DenseMap<Block *, size_t> &blockIndices = it->second;
+  SetVector<Block *> topologicalOrder = getTopologicallySortedBlocks(*region);
+  for (auto [index, block] : llvm::enumerate(topologicalOrder))
+    blockIndices[block] = index;
+  return blockIndices;
+}
+
 /// Sorts `ops` according to dominance. Relies on the topological order of basic
-/// blocks to get a deterministic ordering.
-static void dominanceSort(SmallVector<Operation *> &ops, Region &region) {
+/// blocks to get a deterministic ordering. Uses `blockIndexCache` to avoid the
+/// potentially expensive recomputation of a block index map.
+static void dominanceSort(SmallVector<Operation *> &ops, Region &region,
+                          BlockIndexCache &blockIndexCache) {
   // Produce a topological block order and construct a map to lookup the indices
   // of blocks.
-  DenseMap<Block *, size_t> topoBlockIndices;
-  SetVector<Block *> topologicalOrder = getTopologicallySortedBlocks(region);
-  for (auto [index, block] : llvm::enumerate(topologicalOrder))
-    topoBlockIndices[block] = index;
+  const DenseMap<Block *, size_t> &topoBlockIndices =
+      getOrCreateBlockIndices(blockIndexCache, &region);
 
   // Combining the topological order of the basic blocks together with block
   // internal operation order guarantees a deterministic, dominance respecting
@@ -527,7 +548,8 @@ void MemorySlotPromoter::removeBlockingUses() {
       llvm::make_first_range(info.userToBlockingUses));
 
   // Sort according to dominance.
-  dominanceSort(usersToRemoveUses, *slot.ptr.getParentBlock()->getParent());
+  dominanceSort(usersToRemoveUses, *slot.ptr.getParentBlock()->getParent(),
+                blockIndexCache);
 
   llvm::SmallVector<Operation *> toErase;
   // List of all replaced values in the slot.
@@ -605,20 +627,25 @@ void MemorySlotPromoter::promoteSlot() {
 
 LogicalResult mlir::tryToPromoteMemorySlots(
     ArrayRef<PromotableAllocationOpInterface> allocators, OpBuilder &builder,
-    const DataLayout &dataLayout, Mem2RegStatistics statistics) {
+    const DataLayout &dataLayout, DominanceInfo &dominance,
+    Mem2RegStatistics statistics) {
   bool promotedAny = false;
 
+  // A cache that stores deterministic block indices which are used to determine
+  // a valid operation modification order. The block index maps are computed
+  // lazily and cached to avoid expensive recomputation.
+  BlockIndexCache blockIndexCache;
+
   for (PromotableAllocationOpInterface allocator : allocators) {
     for (MemorySlot slot : allocator.getPromotableSlots()) {
       if (slot.ptr.use_empty())
         continue;
 
-      DominanceInfo dominance;
       MemorySlotPromotionAnalyzer analyzer(slot, dominance, dataLayout);
       std::optional<MemorySlotPromotionInfo> info = analyzer.computeInfo();
       if (info) {
         MemorySlotPromoter(slot, allocator, builder, dominance, dataLayout,
-                           std::move(*info), statistics)
+                           std::move(*info), statistics, blockIndexCache)
             .promoteSlot();
         promotedAny = true;
       }
@@ -640,6 +667,10 @@ struct Mem2Reg : impl::Mem2RegBase<Mem2Reg> {
 
     bool changed = false;
 
+    auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>();
+    const DataLayout &dataLayout = dataLayoutAnalysis.getAtOrAbove(scopeOp);
+    auto &dominance = getAnalysis<DominanceInfo>();
+
     for (Region &region : scopeOp->getRegions()) {
       if (region.getBlocks().empty())
         continue;
@@ -655,16 +686,12 @@ struct Mem2Reg : impl::Mem2RegBase<Mem2Reg> {
           allocators.emplace_back(allocator);
         });
 
-        auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>();
-        const DataLayout &dataLayout = dataLayoutAnalysis.getAtOrAbove(scopeOp);
-
         // Attempt promoting until no promotion succeeds.
         if (failed(tryToPromoteMemorySlots(allocators, builder, dataLayout,
-                                           statistics)))
+                                           dominance, statistics)))
           break;
 
         changed = true;
-        getAnalysisManager().invalidate({});
       }
     }
     if (!changed)
-- 
GitLab


From 2c1c67674cb3beb4e091a9f446de5858631cf8ae Mon Sep 17 00:00:00 2001
From: srcarroll <50210727+srcarroll@users.noreply.github.com>
Date: Wed, 8 May 2024 09:11:53 -0500
Subject: [PATCH 0174/1206] [mlir][transform] Consistent `linalg` `transform`
 op syntax for dynamic index lists (#90897)

This patch is a first pass at making consistent syntax across the
`LinalgTransformOp`s that use dynamic index lists for size parameters.
Previously, there were two different forms: inline types in the list, or
place them in the functional style tuple. This patch goes for the
latter.

In order to do this, the `printPackedOrDynamicIndexList`,
`printDynamicIndexList` and their `parse` counterparts were modified so
that the types can be optionally provided to the corresponding custom
directives.

All affected ops now use tablegen `assemblyFormat`, so custom
`parse`/`print` functions have been removed. There are a couple ops that
will likely add dynamic size support, and once that happens it should be
made sure that the assembly remains consistent with the changes in this
patch.

The affected ops are as follows: `pack`, `pack_greedily`,
`tile_using_forall`. The `tile_using_for` and `vectorize` ops already
used this syntax, but their custom assembly was removed.

---------

Co-authored-by: Oleksandr "Alex" Zinenko <ftynse@gmail.com>
---
 .../Linalg/TransformOps/LinalgTransformOps.td |  41 +++--
 .../mlir/Dialect/Transform/Utils/Utils.h      |  16 +-
 .../mlir/Interfaces/ViewLikeInterface.h       |  11 +-
 .../TransformOps/LinalgTransformOps.cpp       | 154 ------------------
 mlir/lib/Dialect/Transform/Utils/Utils.cpp    |  19 ++-
 mlir/lib/Interfaces/ViewLikeInterface.cpp     |   2 +-
 mlir/test/Dialect/LLVM/transform-e2e.mlir     |   2 +-
 .../Linalg/generalize-tensor-pack-tile.mlir   |   6 +-
 .../Linalg/generalize-tensor-unpack-tile.mlir |   6 +-
 .../Linalg/matmul-shared-memory-padding.mlir  |   4 +-
 .../Dialect/Linalg/multisize-tiling-full.mlir |  16 +-
 mlir/test/Dialect/Linalg/promote.mlir         |   4 +-
 .../Dialect/Linalg/promotion_options.mlir     |   2 +-
 mlir/test/Dialect/Linalg/tile-conv.mlir       |   2 +-
 mlir/test/Dialect/Linalg/tile-indexed.mlir    |   4 +-
 mlir/test/Dialect/Linalg/tile-softmax.mlir    |   4 +-
 mlir/test/Dialect/Linalg/tile-tensors.mlir    |   8 +-
 mlir/test/Dialect/Linalg/tile-to-forall.mlir  |  24 +--
 ...compose-masked-vectorize-and-cleanups.mlir |   4 +-
 .../Dialect/Linalg/transform-op-fuse.mlir     |   2 +-
 ...-op-hoist-pad-build-packing-loop-nest.mlir |  10 +-
 .../Linalg/transform-op-hoist-pad.mlir        |  10 +-
 .../Linalg/transform-op-mmt4d-to-fma.mlir     |   4 +-
 .../Dialect/Linalg/transform-op-pack.mlir     |   4 +-
 .../transform-op-peel-and-vectorize-conv.mlir |   4 +-
 .../transform-op-peel-and-vectorize.mlir      |   2 +-
 .../Linalg/transform-op-scalarize.mlir        |   2 +-
 .../Dialect/Linalg/transform-op-tile.mlir     |  14 +-
 .../Dialect/Linalg/transform-ops-invalid.mlir |   2 +-
 mlir/test/Dialect/Linalg/transform-ops.mlir   |   6 +-
 .../Dialect/Linalg/transform-patterns.mlir    |  20 +--
 .../vectorize-tensor-extract-masked.mlir      |  12 +-
 mlir/test/Dialect/Tensor/tiling.mlir          |  34 ++--
 mlir/test/Dialect/Transform/ops.mlir          |   8 +-
 .../Transform/selective-targeting.mlir        |   2 +-
 .../test/Dialect/Vector/transform-vector.mlir |   2 +-
 .../Linalg/CPU/ArmSME/matmul-transpose-a.mlir |   2 +-
 .../Dialect/Linalg/CPU/ArmSME/matmul.mlir     |   2 +-
 .../ArmSME/multi-tile-matmul-mixed-types.mlir |   2 +-
 .../Linalg/CPU/ArmSME/multi-tile-matmul.mlir  |   2 +-
 .../Dialect/Linalg/CPU/ArmSVE/matmul.mlir     |   2 +-
 .../Integration/Dialect/Linalg/CPU/mmt4d.mlir |   4 +-
 .../Dialect/Linalg/CPU/pack-unpack-mmt4d.mlir |   4 +-
 .../Dialect/Linalg/CPU/test-conv-1d-call.mlir |   2 +-
 .../Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir |   2 +-
 .../Dialect/Linalg/CPU/test-conv-2d-call.mlir |   2 +-
 .../CPU/test-conv-2d-nhwc-hwcf-call.mlir      |   2 +-
 .../Dialect/Linalg/CPU/test-conv-3d-call.mlir |   2 +-
 .../CPU/test-conv-3d-ndhwc-dhwcf-call.mlir    |   2 +-
 .../Linalg/CPU/test-tensor-matmul.mlir        |   2 +-
 .../tile-pad-using-interface.mlir             |  12 +-
 .../TilingInterface/tile-using-interface.mlir |  18 +-
 .../dialects/transform_structured_ext.py      |   4 +-
 53 files changed, 210 insertions(+), 323 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 55d82fd5825b..5585ba27fdad 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -783,10 +783,9 @@ def PackOp : Op<Transform_Dialect, "structured.pack", [
   let assemblyFormat = [{
     $target
     `packed_sizes` `=` custom<DynamicIndexList>($packed_sizes,
-                                                $static_packed_sizes,
-                                                type($packed_sizes))
+                                                $static_packed_sizes)
     attr-dict
-    `:` functional-type($target, results)
+    `:` functional-type(operands, results)
   }];
 
   let builders = [
@@ -890,14 +889,13 @@ def PackGreedilyOp : Op<Transform_Dialect, "structured.pack_greedily", [
     $target
     oilist(
       `matmul_packed_sizes` `=` custom<DynamicIndexList>($matmul_packed_sizes,
-                                                         $static_matmul_packed_sizes,
-                                                         type($matmul_packed_sizes))
+                                                         $static_matmul_packed_sizes)
       (`matmul_padded_sizes_next_multiple_of` `=`
         $matmul_padded_sizes_next_multiple_of^)?
       `matmul_inner_dims_order` `=` $matmul_inner_dims_order
     )
     attr-dict
-    `:` functional-type($target, results)
+    `:` functional-type(operands, results)
   }];
   let hasVerifier = 1;
 
@@ -1899,7 +1897,17 @@ def TileUsingForOp : Op<Transform_Dialect, "structured.tile_using_for",
                       $scalableSizes)>,
   ];
 
-  let hasCustomAssemblyFormat = 1;
+  let assemblyFormat = [{
+    $target
+      `tile_sizes` custom<DynamicIndexList>(
+        $dynamic_sizes,
+        $static_sizes,
+        $scalable_sizes)
+      (`interchange` `=` $interchange^)?
+    attr-dict
+    `:` functional-type(operands, results)
+  }];
+
   let hasVerifier = 1;
 
   let extraClassDeclaration = [{
@@ -2017,17 +2025,13 @@ def TileUsingForallOp :
   let assemblyFormat = [{
     $target oilist(
         `num_threads` custom<PackedOrDynamicIndexList>($packed_num_threads,
-                                                       type($packed_num_threads),
                                                        $num_threads,
-                                                       type($num_threads),
                                                        $static_num_threads) |
          `tile_sizes` custom<PackedOrDynamicIndexList>($packed_tile_sizes,
-                                                       type($packed_tile_sizes),
                                                        $tile_sizes,
-                                                       type($tile_sizes),
                                                        $static_tile_sizes))
     (`(` `mapping` `=` $mapping^ `)`)? attr-dict
-    `:` functional-type($target, results)
+    `:` functional-type(operands, results)
   }];
   let hasVerifier = 1;
 
@@ -2162,7 +2166,18 @@ def VectorizeOp : Op<Transform_Dialect, "structured.vectorize",
 
   let results = (outs);
 
-  let hasCustomAssemblyFormat = 1;
+  // We use oilist here to elide the optional `vector_sizes` when empty list
+  // is passed.
+  let assemblyFormat = [{
+    $target oilist(
+      `vector_sizes` custom<DynamicIndexList>(
+        $vector_sizes,
+        $static_vector_sizes,
+        $scalable_sizes))
+    attr-dict
+    `:` type($target)(`,`type($vector_sizes)^)? 
+  }];
+
   let hasVerifier = 1;
 
   let extraClassDeclaration = [{
diff --git a/mlir/include/mlir/Dialect/Transform/Utils/Utils.h b/mlir/include/mlir/Dialect/Transform/Utils/Utils.h
index 868054e5e2ae..be31f5beea8c 100644
--- a/mlir/include/mlir/Dialect/Transform/Utils/Utils.h
+++ b/mlir/include/mlir/Dialect/Transform/Utils/Utils.h
@@ -37,6 +37,12 @@ void printPackedOrDynamicIndexList(OpAsmPrinter &printer, Operation *op,
                                    Value packed, Type packedType,
                                    OperandRange values, TypeRange valueTypes,
                                    DenseI64ArrayAttr integers);
+inline void printPackedOrDynamicIndexList(OpAsmPrinter &printer, Operation *op,
+                                          Value packed, OperandRange values,
+                                          DenseI64ArrayAttr integers) {
+  printPackedOrDynamicIndexList(printer, op, packed, Type(), values,
+                                TypeRange{}, integers);
+}
 
 /// Parser hook for custom directive in assemblyFormat.
 ///
@@ -47,7 +53,15 @@ void printPackedOrDynamicIndexList(OpAsmPrinter &printer, Operation *op,
 ParseResult parsePackedOrDynamicIndexList(
     OpAsmParser &parser, std::optional<OpAsmParser::UnresolvedOperand> &packed,
     Type &packedType, SmallVectorImpl<OpAsmParser::UnresolvedOperand> &values,
-    SmallVectorImpl<Type> &valueTypes, DenseI64ArrayAttr &integers);
+    SmallVectorImpl<Type> *valueTypes, DenseI64ArrayAttr &integers);
+inline ParseResult parsePackedOrDynamicIndexList(
+    OpAsmParser &parser, std::optional<OpAsmParser::UnresolvedOperand> &packed,
+    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &values,
+    DenseI64ArrayAttr &integers) {
+  Type packedType;
+  return parsePackedOrDynamicIndexList(parser, packed, packedType, values,
+                                       nullptr, integers);
+}
 } // namespace transform
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Interfaces/ViewLikeInterface.h b/mlir/include/mlir/Interfaces/ViewLikeInterface.h
index 931309b0c596..d6479143a0a5 100644
--- a/mlir/include/mlir/Interfaces/ViewLikeInterface.h
+++ b/mlir/include/mlir/Interfaces/ViewLikeInterface.h
@@ -106,9 +106,16 @@ public:
 /// empty then assume that all indices are non-scalable.
 void printDynamicIndexList(
     OpAsmPrinter &printer, Operation *op, OperandRange values,
-    ArrayRef<int64_t> integers, TypeRange valueTypes = TypeRange(),
-    ArrayRef<bool> scalables = {},
+    ArrayRef<int64_t> integers, ArrayRef<bool> scalables,
+    TypeRange valueTypes = TypeRange(),
     AsmParser::Delimiter delimiter = AsmParser::Delimiter::Square);
+inline void printDynamicIndexList(
+    OpAsmPrinter &printer, Operation *op, OperandRange values,
+    ArrayRef<int64_t> integers, TypeRange valueTypes = TypeRange(),
+    AsmParser::Delimiter delimiter = AsmParser::Delimiter::Square) {
+  return printDynamicIndexList(printer, op, values, integers, {}, valueTypes,
+                               delimiter);
+}
 
 /// Parser hook for custom directive in assemblyFormat.
 ///
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index ae04281032d7..13582a140a96 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -2823,86 +2823,6 @@ SmallVector<OpFoldResult> transform::TileUsingForOp::getMixedSizes() {
   return results;
 }
 
-// We want to parse `DenseI64ArrayAttr` using the short form without the
-// `array` prefix to be consistent in the IR with `parseDynamicIndexList`.
-ParseResult parseOptionalInterchange(OpAsmParser &parser,
-                                     OperationState &result) {
-  if (failed(parser.parseOptionalKeyword("interchange")))
-    return success();
-  if (failed(parser.parseEqual()))
-    return failure();
-  result.addAttribute(
-      transform::TileUsingForOp::getInterchangeAttrName(result.name),
-      DenseI64ArrayAttr::parse(parser, Type{}));
-  return success();
-}
-
-void printOptionalInterchange(OpAsmPrinter &p,
-                              ArrayRef<int64_t> interchangeVals) {
-  if (!interchangeVals.empty()) {
-    p << " interchange = [";
-    llvm::interleaveComma(interchangeVals, p,
-                          [&](int64_t integer) { p << integer; });
-    p << "]";
-  }
-}
-
-ParseResult transform::TileUsingForOp::parse(OpAsmParser &parser,
-                                             OperationState &result) {
-  OpAsmParser::UnresolvedOperand target;
-  SmallVector<OpAsmParser::UnresolvedOperand> dynamicSizes;
-  DenseI64ArrayAttr staticSizes;
-  FunctionType functionalType;
-  llvm::SMLoc operandLoc;
-  DenseBoolArrayAttr scalableVals;
-
-  if (parser.parseOperand(target) || parser.getCurrentLocation(&operandLoc) ||
-      parseDynamicIndexList(parser, dynamicSizes, staticSizes, scalableVals) ||
-      parseOptionalInterchange(parser, result) ||
-      parser.parseOptionalAttrDict(result.attributes) ||
-      parser.parseColonType(functionalType))
-    return ParseResult::failure();
-
-  size_t numExpectedLoops =
-      staticSizes.size() - llvm::count(staticSizes.asArrayRef(), 0);
-  if (functionalType.getNumResults() != numExpectedLoops + 1) {
-    return parser.emitError(parser.getNameLoc())
-           << "expected " << (numExpectedLoops + 1) << " result type(s)";
-  }
-  if (functionalType.getNumInputs() != dynamicSizes.size() + 1) {
-    return parser.emitError(operandLoc)
-           << "expected " << dynamicSizes.size() + 1 << " operand type(s)";
-  }
-  if (parser.resolveOperand(target, functionalType.getInputs().front(),
-                            result.operands) ||
-      parser.resolveOperands(dynamicSizes,
-                             functionalType.getInputs().drop_front(),
-                             operandLoc, result.operands)) {
-    return failure();
-  }
-
-  result.addAttribute(getScalableSizesAttrName(result.name), scalableVals);
-
-  result.addAttribute(getStaticSizesAttrName(result.name), staticSizes);
-  result.addTypes(functionalType.getResults());
-  return success();
-}
-
-void TileUsingForOp::print(OpAsmPrinter &p) {
-  p << ' ' << getTarget();
-  printDynamicIndexList(p, getOperation(), getDynamicSizes(), getStaticSizes(),
-                        /*valueTypes=*/{}, getScalableSizesAttr(),
-                        OpAsmParser::Delimiter::Square);
-  printOptionalInterchange(p, getInterchange());
-  p.printOptionalAttrDict(
-      (*this)->getAttrs(),
-      /*elidedAttrs=*/{getInterchangeAttrName(getOperation()->getName()),
-                       getScalableSizesAttrName(getOperation()->getName()),
-                       getStaticSizesAttrName(getOperation()->getName())});
-  p << " : ";
-  p.printFunctionalType(getOperands().getTypes(), getResults().getTypes());
-}
-
 void transform::TileUsingForOp::getEffects(
     SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
   consumesHandle(getTarget(), effects);
@@ -3219,80 +3139,6 @@ transform::VectorizeChildrenAndApplyPatternsOp::applyToOne(
 // VectorizeOp
 //===----------------------------------------------------------------------===//
 
-static const StringLiteral kVectorSizesKeyword = "vector_sizes";
-
-ParseResult transform::VectorizeOp::parse(OpAsmParser &parser,
-                                          OperationState &result) {
-  OpAsmParser::UnresolvedOperand target;
-  SmallVector<OpAsmParser::UnresolvedOperand> dynamicSizes;
-  DenseI64ArrayAttr staticSizes;
-  SmallVector<Type> operandTypes;
-  llvm::SMLoc operandLoc;
-  DenseBoolArrayAttr scalableVals;
-
-  if (parser.parseOperand(target) || parser.getCurrentLocation(&operandLoc))
-    return ParseResult::failure();
-
-  if (succeeded(parser.parseOptionalKeyword(kVectorSizesKeyword))) {
-    if (failed(parseDynamicIndexList(parser, dynamicSizes, staticSizes,
-                                     scalableVals)))
-      return ParseResult::failure();
-  }
-
-  if (succeeded(parser.parseOptionalKeyword(
-          getVectorizeNdExtractAttrName(result.name))))
-    result.addAttribute(getVectorizeNdExtractAttrName(result.name),
-                        parser.getBuilder().getUnitAttr());
-
-  if (parser.parseOptionalAttrDict(result.attributes) ||
-      parser.parseColonTypeList(operandTypes))
-    return ParseResult::failure();
-
-  if (operandTypes.size() != dynamicSizes.size() + 1) {
-    return parser.emitError(operandLoc)
-           << "expected " << dynamicSizes.size() + 1 << " operand type(s)";
-  }
-  if (parser.resolveOperand(target, operandTypes.front(), result.operands) ||
-      parser.resolveOperands(dynamicSizes, ArrayRef(operandTypes).drop_front(),
-                             operandLoc, result.operands)) {
-    return failure();
-  }
-
-  if (scalableVals)
-    result.addAttribute(getScalableSizesAttrName(result.name), scalableVals);
-  if (staticSizes)
-    result.addAttribute(getStaticVectorSizesAttrName(result.name), staticSizes);
-
-  return success();
-}
-
-void transform::VectorizeOp::print(OpAsmPrinter &p) {
-  p << ' ' << getTarget() << ' ';
-  if (!getMixedVectorSizes().empty()) {
-    p << kVectorSizesKeyword << ' ';
-    printDynamicIndexList(p, getOperation(), getVectorSizes(),
-                          getStaticVectorSizesAttr(),
-                          /*valueTypes=*/{}, getScalableSizesAttr(),
-                          OpAsmParser::Delimiter::Square);
-  }
-
-  if (getVectorizeNdExtract())
-    p << getVectorizeNdExtractAttrName() << ' ';
-
-  p.printOptionalAttrDict(
-      (*this)->getAttrs(),
-      /*elidedAttrs=*/{
-          getScalableSizesAttrName(getOperation()->getName()),
-          getStaticVectorSizesAttrName(getOperation()->getName())});
-  p << " : ";
-  p << getTarget().getType();
-  if (!getVectorSizes().empty()) {
-    p << ", ";
-    llvm::interleaveComma(getVectorSizes(), p,
-                          [&](Value operand) { p << operand.getType(); });
-  }
-}
-
 DiagnosedSilenceableFailure transform::VectorizeOp::apply(
     transform::TransformRewriter &rewriter,
     mlir::transform::TransformResults &transformResults,
diff --git a/mlir/lib/Dialect/Transform/Utils/Utils.cpp b/mlir/lib/Dialect/Transform/Utils/Utils.cpp
index 08068d285b4c..2ce21fe8a9c1 100644
--- a/mlir/lib/Dialect/Transform/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Transform/Utils/Utils.cpp
@@ -20,7 +20,11 @@ void mlir::transform::printPackedOrDynamicIndexList(
   if (packed) {
     assert(values.empty() && (!integers || integers.empty()) &&
            "expected no values/integers");
-    printer << "*(" << packed << " : " << packedType << ")";
+    printer << "*(" << packed;
+    if (packedType) {
+      printer << " : " << packedType;
+    }
+    printer << ")";
     return;
   }
   printDynamicIndexList(printer, op, values, integers, valueTypes);
@@ -29,19 +33,20 @@ void mlir::transform::printPackedOrDynamicIndexList(
 ParseResult mlir::transform::parsePackedOrDynamicIndexList(
     OpAsmParser &parser, std::optional<OpAsmParser::UnresolvedOperand> &packed,
     Type &packedType, SmallVectorImpl<OpAsmParser::UnresolvedOperand> &values,
-    SmallVectorImpl<Type> &valueTypes, DenseI64ArrayAttr &integers) {
+    SmallVectorImpl<Type> *valueTypes, DenseI64ArrayAttr &integers) {
   OpAsmParser::UnresolvedOperand packedOperand;
   if (parser.parseOptionalStar().succeeded()) {
     if (parser.parseLParen().failed() ||
-        parser.parseOperand(packedOperand).failed() ||
-        parser.parseColonType(packedType).failed() ||
-        parser.parseRParen().failed()) {
+        parser.parseOperand(packedOperand).failed())
+      return failure();
+    if (packedType && (parser.parseColonType(packedType).failed()))
+      return failure();
+    if (parser.parseRParen().failed())
       return failure();
-    }
     packed.emplace(packedOperand);
     integers = parser.getBuilder().getDenseI64ArrayAttr({});
     return success();
   }
 
-  return parseDynamicIndexList(parser, values, integers, &valueTypes);
+  return parseDynamicIndexList(parser, values, integers, valueTypes);
 }
diff --git a/mlir/lib/Interfaces/ViewLikeInterface.cpp b/mlir/lib/Interfaces/ViewLikeInterface.cpp
index 6d1ff03756ac..ca33636336bf 100644
--- a/mlir/lib/Interfaces/ViewLikeInterface.cpp
+++ b/mlir/lib/Interfaces/ViewLikeInterface.cpp
@@ -113,7 +113,7 @@ static char getRightDelimiter(AsmParser::Delimiter delimiter) {
 void mlir::printDynamicIndexList(OpAsmPrinter &printer, Operation *op,
                                  OperandRange values,
                                  ArrayRef<int64_t> integers,
-                                 TypeRange valueTypes, ArrayRef<bool> scalables,
+                                 ArrayRef<bool> scalables, TypeRange valueTypes,
                                  AsmParser::Delimiter delimiter) {
   char leftDelimiter = getLeftDelimiter(delimiter);
   char rightDelimiter = getRightDelimiter(delimiter);
diff --git a/mlir/test/Dialect/LLVM/transform-e2e.mlir b/mlir/test/Dialect/LLVM/transform-e2e.mlir
index adbbbba32a40..c00b47fb936e 100644
--- a/mlir/test/Dialect/LLVM/transform-e2e.mlir
+++ b/mlir/test/Dialect/LLVM/transform-e2e.mlir
@@ -15,7 +15,7 @@ func.func @matmul_tensors(
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.consumed}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %module_op : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [2, 2, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [2, 2, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     %2 = transform.get_parent_op %1 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
     transform.structured.vectorize_children_and_apply_patterns %2 : (!transform.any_op) -> !transform.any_op
     %b = transform.bufferization.one_shot_bufferize layout{IdentityLayoutMap}
diff --git a/mlir/test/Dialect/Linalg/generalize-tensor-pack-tile.mlir b/mlir/test/Dialect/Linalg/generalize-tensor-pack-tile.mlir
index 0a197a0ee9fa..d0c53ae46800 100644
--- a/mlir/test/Dialect/Linalg/generalize-tensor-pack-tile.mlir
+++ b/mlir/test/Dialect/Linalg/generalize-tensor-pack-tile.mlir
@@ -27,7 +27,7 @@ func.func @KCRS_to_KCRSsr(%arg0: tensor<1x1x128x64xf32>, %arg1: tensor<1x1x4x8x8
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:4 = transform.structured.tile_using_for %0 [1, 1, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:4 = transform.structured.tile_using_for %0 tile_sizes [1, 1, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -54,7 +54,7 @@ func.func @pad_and_pack(%arg0: tensor<13x15xf32>, %arg1: tensor<2x8x8x2xf32>, %a
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -85,7 +85,7 @@ func.func @KC_to_CKkc(%arg0: tensor<128x256xf32>, %arg1: tensor<32x4x32x8xf32>)
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/generalize-tensor-unpack-tile.mlir b/mlir/test/Dialect/Linalg/generalize-tensor-unpack-tile.mlir
index 7d64331c9878..c15859d898ec 100644
--- a/mlir/test/Dialect/Linalg/generalize-tensor-unpack-tile.mlir
+++ b/mlir/test/Dialect/Linalg/generalize-tensor-unpack-tile.mlir
@@ -8,7 +8,7 @@ func.func @KCRSsr_to_KCRS(%arg0: tensor<1x1x4x8x8x32xf32>, %arg1: tensor<1x1x128
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["tensor.unpack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:4 = transform.structured.tile_using_for %0 [1, 1, 32, 8] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:4 = transform.structured.tile_using_for %0 tile_sizes [1, 1, 32, 8] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -68,7 +68,7 @@ func.func @unpack_and_extract_slice(%arg0: tensor<2x8x8x2xf32>, %arg1: tensor<13
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["tensor.unpack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:2 = transform.structured.tile_using_for %0 [8, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [8, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -100,7 +100,7 @@ func.func @CKkc_to_KC(%arg0: tensor<32x4x32x8xf32>, %arg1: tensor<128x256xf32>)
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["tensor.unpack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:2 = transform.structured.tile_using_for %0 [32, 8] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [32, 8] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/matmul-shared-memory-padding.mlir b/mlir/test/Dialect/Linalg/matmul-shared-memory-padding.mlir
index c3ac69f65b7c..3f8d2ea06641 100644
--- a/mlir/test/Dialect/Linalg/matmul-shared-memory-padding.mlir
+++ b/mlir/test/Dialect/Linalg/matmul-shared-memory-padding.mlir
@@ -52,7 +52,7 @@ module attributes {transform.with_named_sequence} {
         : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     // Tile linalg.matmul a second time.
-    %tiled_linalg_op, %loops = transform.structured.tile_using_for %tiled_matmul_op[0, 0, 16] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %tiled_linalg_op, %loops = transform.structured.tile_using_for %tiled_matmul_op tile_sizes [0, 0, 16] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     // Pad linalg.matmul.
     %padded, %pad, %copy_back = transform.structured.pad %tiled_linalg_op
@@ -171,7 +171,7 @@ module attributes {transform.with_named_sequence} {
         : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     // Tile linalg.matmul a second time.
-    %tiled_linalg_op, %loops = transform.structured.tile_using_for %tiled_matmul_op[0, 0, 16] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %tiled_linalg_op, %loops = transform.structured.tile_using_for %tiled_matmul_op tile_sizes [0, 0, 16] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     // Pad linalg.matmul.
     %padded, %pad, %copy_back = transform.structured.pad %tiled_linalg_op
diff --git a/mlir/test/Dialect/Linalg/multisize-tiling-full.mlir b/mlir/test/Dialect/Linalg/multisize-tiling-full.mlir
index 592eb781cd4f..15b24b56608e 100644
--- a/mlir/test/Dialect/Linalg/multisize-tiling-full.mlir
+++ b/mlir/test/Dialect/Linalg/multisize-tiling-full.mlir
@@ -8,13 +8,13 @@ module attributes {transform.with_named_sequence} {
     %1:3 = transform.structured.multitile_sizes %0 { dimension = 0, target_size = 3} : (!transform.any_op) -> !transform.any_op
     %t:3 = transform.structured.multitile_sizes %0 { dimension = 1, target_size = 10} : (!transform.any_op) -> !transform.any_op
     %2:2 = transform.structured.split %0 after %1#2 { dimension = 0 } : !transform.any_op, !transform.any_op
-    %3:2 = transform.structured.tile_using_for %2#0 [%1#0] : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
-    %4:2 = transform.structured.tile_using_for %2#1 [%1#1] : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %3:2 = transform.structured.tile_using_for %2#0 tile_sizes [%1#0] : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %4:2 = transform.structured.tile_using_for %2#1 tile_sizes [%1#1] : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
     %5 = transform.merge_handles %3#0, %4#0 : !transform.any_op
     %tt:3 = transform.replicate num(%5) %t#0, %t#1, %t#2 : !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op
     %6:2 = transform.structured.split %5 after %tt#2 { dimension = 1 } : !transform.any_op, !transform.any_op
-    transform.structured.tile_using_for %6#0 [0, %tt#0] : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
-    transform.structured.tile_using_for %6#1 [0, %tt#1] : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+    transform.structured.tile_using_for %6#0 tile_sizes [0, %tt#0] : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+    transform.structured.tile_using_for %6#1 tile_sizes [0, %tt#1] : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -110,13 +110,13 @@ module attributes {transform.with_named_sequence} {
     %1:3 = transform.structured.multitile_sizes %0 { dimension = 0, target_size = 3} : (!transform.any_op) -> !transform.param<i64>
     %t:3 = transform.structured.multitile_sizes %0 { dimension = 1, target_size = 10} : (!transform.any_op) -> !transform.param<i64>
     %2:2 = transform.structured.split %0 after %1#2 { dimension = 0 } : !transform.any_op, !transform.param<i64>
-    %3:2 = transform.structured.tile_using_for %2#0 [%1#0] : (!transform.any_op, !transform.param<i64>) -> (!transform.any_op, !transform.any_op)
-    %4:2 = transform.structured.tile_using_for %2#1 [%1#1] : (!transform.any_op, !transform.param<i64>) -> (!transform.any_op, !transform.any_op)
+    %3:2 = transform.structured.tile_using_for %2#0 tile_sizes [%1#0] : (!transform.any_op, !transform.param<i64>) -> (!transform.any_op, !transform.any_op)
+    %4:2 = transform.structured.tile_using_for %2#1 tile_sizes [%1#1] : (!transform.any_op, !transform.param<i64>) -> (!transform.any_op, !transform.any_op)
     %5 = transform.merge_handles %3#0, %4#0 : !transform.any_op
     %tt:3 = transform.replicate num(%5) %t#0, %t#1, %t#2 : !transform.any_op, !transform.param<i64>, !transform.param<i64>, !transform.param<i64>
     %6:2 = transform.structured.split %5 after %tt#2 { dimension = 1 } : !transform.any_op, !transform.param<i64>
-    transform.structured.tile_using_for %6#0 [0, %tt#0] : (!transform.any_op, !transform.param<i64>) -> (!transform.any_op, !transform.any_op)
-    transform.structured.tile_using_for %6#1 [0, %tt#1] : (!transform.any_op, !transform.param<i64>) -> (!transform.any_op, !transform.any_op)
+    transform.structured.tile_using_for %6#0 tile_sizes [0, %tt#0] : (!transform.any_op, !transform.param<i64>) -> (!transform.any_op, !transform.any_op)
+    transform.structured.tile_using_for %6#1 tile_sizes [0, %tt#1] : (!transform.any_op, !transform.param<i64>) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/promote.mlir b/mlir/test/Dialect/Linalg/promote.mlir
index fb5f357f3faa..2d640057df34 100644
--- a/mlir/test/Dialect/Linalg/promote.mlir
+++ b/mlir/test/Dialect/Linalg/promote.mlir
@@ -183,7 +183,7 @@ func.func @gemm_shared(%a : memref<?x?xf32>, %b : memref<?x?xf32>, %c : memref<?
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [16, 16, 16] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [16, 16, 16] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     %2 = transform.structured.promote %1 { operands_to_promote = [0, 1], mapping = [#gpu.memory_space<workgroup>] } : (!transform.any_op) -> !transform.any_op
     transform.yield
   }
@@ -227,7 +227,7 @@ func.func @gemm_private(%a : memref<?x?xf32>, %b : memref<?x?xf32>, %c : memref<
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [16, 16, 16] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [16, 16, 16] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     %2 = transform.structured.promote %1 { operands_to_promote = [0, 1], mapping = [#gpu.memory_space<private>] } : (!transform.any_op) -> !transform.any_op
     transform.yield
   }
diff --git a/mlir/test/Dialect/Linalg/promotion_options.mlir b/mlir/test/Dialect/Linalg/promotion_options.mlir
index 3bf74b708cb8..caa72ba24316 100644
--- a/mlir/test/Dialect/Linalg/promotion_options.mlir
+++ b/mlir/test/Dialect/Linalg/promotion_options.mlir
@@ -37,7 +37,7 @@ func.func @gemm(%a : memref<?x?xf32>, %b : memref<?x?xf32>, %c : memref<?x?xf32>
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [16, 16, 16] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [16, 16, 16] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     %2 = transform.structured.promote %1 { operands_to_promote = [0, 2], force_full_tiles = [false, false], use_full_tiles_by_default } : (!transform.any_op) -> !transform.any_op
     transform.yield
   }
diff --git a/mlir/test/Dialect/Linalg/tile-conv.mlir b/mlir/test/Dialect/Linalg/tile-conv.mlir
index c42bdbe982c4..f674996e42f3 100644
--- a/mlir/test/Dialect/Linalg/tile-conv.mlir
+++ b/mlir/test/Dialect/Linalg/tile-conv.mlir
@@ -12,7 +12,7 @@ func.func @conv(%arg0 : memref<?x?xf32>, %arg1 : memref<?x?xf32>, %arg2 : memref
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_2d"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loop:2 = transform.structured.tile_using_for %0 [2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loop:2 = transform.structured.tile_using_for %0 tile_sizes [2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/tile-indexed.mlir b/mlir/test/Dialect/Linalg/tile-indexed.mlir
index c176dc19c7e9..b4aa0a33bc59 100644
--- a/mlir/test/Dialect/Linalg/tile-indexed.mlir
+++ b/mlir/test/Dialect/Linalg/tile-indexed.mlir
@@ -14,7 +14,7 @@ func.func @indexed_vector(%arg0: memref<50xindex>) {
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loop = transform.structured.tile_using_for %0 [10] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+      %1, %loop = transform.structured.tile_using_for %0 tile_sizes [10] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -46,7 +46,7 @@ func.func @indexed_matrix(%arg0: memref<50x50xindex>) {
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loop:2 = transform.structured.tile_using_for %0 [10, 25] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loop:2 = transform.structured.tile_using_for %0 tile_sizes [10, 25] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/tile-softmax.mlir b/mlir/test/Dialect/Linalg/tile-softmax.mlir
index ec848e2deb74..7d201b58a8c3 100644
--- a/mlir/test/Dialect/Linalg/tile-softmax.mlir
+++ b/mlir/test/Dialect/Linalg/tile-softmax.mlir
@@ -39,7 +39,7 @@ func.func @softmax(%arg0: tensor<16x64x256xf32>) -> tensor<16x64x256xf32> {
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.softmax"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loop:2 = transform.structured.tile_using_for %0 [2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loop:2 = transform.structured.tile_using_for %0 tile_sizes [2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -149,7 +149,7 @@ func.func @softmax_memref(%arg0: memref<16x64x256xf32>, %arg1: memref<16x64x256x
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.softmax"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loop:2 = transform.structured.tile_using_for %0 [2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loop:2 = transform.structured.tile_using_for %0 tile_sizes [2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/tile-tensors.mlir b/mlir/test/Dialect/Linalg/tile-tensors.mlir
index cdef71ded8b2..89183813c080 100644
--- a/mlir/test/Dialect/Linalg/tile-tensors.mlir
+++ b/mlir/test/Dialect/Linalg/tile-tensors.mlir
@@ -30,7 +30,7 @@ func.func @matmul_tensors(
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [2, 3, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [2, 3, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -57,7 +57,7 @@ func.func @matmul_tensors_with_size_zeros(
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.structured.tile_using_for %0 [0, 0, 0] : (!transform.any_op) -> (!transform.any_op)
+    %1 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 0] : (!transform.any_op) -> (!transform.any_op)
     transform.yield
   }
 }
@@ -90,7 +90,7 @@ func.func @generic_op_tensors(
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [2, 3, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [2, 3, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -163,7 +163,7 @@ func.func @fold_extract_slice(
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [2, 3, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [2, 3, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/tile-to-forall.mlir b/mlir/test/Dialect/Linalg/tile-to-forall.mlir
index 12e2dea5530b..8545dfd25ecc 100644
--- a/mlir/test/Dialect/Linalg/tile-to-forall.mlir
+++ b/mlir/test/Dialect/Linalg/tile-to-forall.mlir
@@ -130,8 +130,8 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %sz = transform.structured.match ops{["test.dummy"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1:2 = transform.structured.tile_using_forall %0 tile_sizes *(%sz : !transform.any_op)
-           : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %1:2 = transform.structured.tile_using_forall %0 tile_sizes *(%sz)
+           : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -333,8 +333,8 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %sz = transform.structured.match ops{["test.dummy"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1:2 = transform.structured.tile_using_forall %0 tile_sizes [%sz : !transform.any_op, 20]
-           : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %1:2 = transform.structured.tile_using_forall %0 tile_sizes [%sz, 20]
+           : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -492,8 +492,8 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %sz = transform.param.constant 10 : i64 -> !transform.param<i64>
-    %1:2 = transform.structured.tile_using_forall %0 tile_sizes [%sz : !transform.param<i64>, 20]
-           : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %1:2 = transform.structured.tile_using_forall %0 tile_sizes [%sz, 20]
+           : (!transform.any_op, !transform.param<i64>) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -513,8 +513,8 @@ module attributes {transform.with_named_sequence} {
     %c20 = transform.param.constant 20 : i64 -> !transform.param<i64>
     %sz = transform.merge_handles %c10, %c20 : !transform.param<i64>
     // expected-error @below {{requires exactly one parameter associated}}
-    %1:2 = transform.structured.tile_using_forall %0 tile_sizes [%sz : !transform.param<i64>, 20]
-           : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %1:2 = transform.structured.tile_using_forall %0 tile_sizes [%sz, 20]
+           : (!transform.any_op, !transform.param<i64>) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -562,8 +562,8 @@ module attributes {transform.with_named_sequence} {
     %c10 = transform.param.constant 10 : i64 -> !transform.any_param
     %c20 = transform.param.constant 20 : i64 -> !transform.any_param
     %sz = transform.merge_handles %c10, %c20 : !transform.any_param
-    %1:2 = transform.structured.tile_using_forall %0 tile_sizes *(%sz : !transform.any_param)
-           : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %1:2 = transform.structured.tile_using_forall %0 tile_sizes *(%sz)
+           : (!transform.any_op, !transform.any_param) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -581,8 +581,8 @@ module attributes {transform.with_named_sequence} {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %sz = transform.param.constant "[10 : i64, 20 : i64]" -> !transform.any_param
     // expected-error @below {{expected the parameter to be associated with an integer attribute}}
-    %1:2 = transform.structured.tile_using_forall %0 tile_sizes *(%sz : !transform.any_param)
-           : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %1:2 = transform.structured.tile_using_forall %0 tile_sizes *(%sz)
+           : (!transform.any_op, !transform.any_param) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/transform-op-compose-masked-vectorize-and-cleanups.mlir b/mlir/test/Dialect/Linalg/transform-op-compose-masked-vectorize-and-cleanups.mlir
index 477261882421..61fe3da34e1d 100644
--- a/mlir/test/Dialect/Linalg/transform-op-compose-masked-vectorize-and-cleanups.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-compose-masked-vectorize-and-cleanups.mlir
@@ -22,9 +22,9 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%module: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %module
       : (!transform.any_op) -> !transform.any_op
-    %tiled_linalg_op, %loops:3 = transform.structured.tile_using_for %0[64, 128, 256]
+    %tiled_linalg_op, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [64, 128, 256]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
-    %tiled_linalg_op_0, %loops_1:3 = transform.structured.tile_using_for %tiled_linalg_op[8, 8, 8]
+    %tiled_linalg_op_0, %loops_1:3 = transform.structured.tile_using_for %tiled_linalg_op tile_sizes [8, 8, 8]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.structured.vectorize %tiled_linalg_op_0 vector_sizes [8, 8, 8]
       : !transform.any_op
diff --git a/mlir/test/Dialect/Linalg/transform-op-fuse.mlir b/mlir/test/Dialect/Linalg/transform-op-fuse.mlir
index 69daf8c80a16..3a023deb1132 100644
--- a/mlir/test/Dialect/Linalg/transform-op-fuse.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-fuse.mlir
@@ -95,7 +95,7 @@ module attributes {transform.with_named_sequence} {
     %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [5, 0, 7], tile_interchange = [0, 2, 1]}
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
-    %2, %loops_2 = transform.structured.tile_using_for %1 [0, 4]
+    %2, %loops_2 = transform.structured.tile_using_for %1 tile_sizes [0, 4]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
       transform.yield
   }
diff --git a/mlir/test/Dialect/Linalg/transform-op-hoist-pad-build-packing-loop-nest.mlir b/mlir/test/Dialect/Linalg/transform-op-hoist-pad-build-packing-loop-nest.mlir
index 1be5bf098c33..ae63ed5f1a41 100644
--- a/mlir/test/Dialect/Linalg/transform-op-hoist-pad-build-packing-loop-nest.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-hoist-pad-build-packing-loop-nest.mlir
@@ -15,7 +15,7 @@ module attributes {transform.with_named_sequence} {
     %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
 
-    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul tile_sizes [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     %matmul_padded, %0, %copy_back = transform.structured.pad %matmul_l1 {
       padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
@@ -49,7 +49,7 @@ module attributes {transform.with_named_sequence} {
     %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
 
-    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul tile_sizes [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     %matmul_padded, %0, %copy_back = transform.structured.pad %matmul_l1 {
       padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
@@ -89,7 +89,7 @@ module attributes {transform.with_named_sequence} {
     %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
 
-    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul tile_sizes [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     %matmul_padded, %0, %copy_back = transform.structured.pad %matmul_l1 {
       padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
@@ -129,7 +129,7 @@ module attributes {transform.with_named_sequence} {
     %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
 
-    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul tile_sizes [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     %matmul_padded, %0, %copy_back = transform.structured.pad %matmul_l1 {
       padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
@@ -167,7 +167,7 @@ module attributes {transform.with_named_sequence} {
     %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
 
-    %matmul_l1, %loops_l1:2 = transform.structured.tile_using_for %matmul [5, 0, 7] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %matmul_l1, %loops_l1:2 = transform.structured.tile_using_for %matmul tile_sizes [5, 0, 7] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
 
     %matmul_padded, %0, %copy_back = transform.structured.pad %matmul_l1 {
       padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
diff --git a/mlir/test/Dialect/Linalg/transform-op-hoist-pad.mlir b/mlir/test/Dialect/Linalg/transform-op-hoist-pad.mlir
index 37cb9b2376fb..499d9904c06b 100644
--- a/mlir/test/Dialect/Linalg/transform-op-hoist-pad.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-hoist-pad.mlir
@@ -15,7 +15,7 @@ module attributes {transform.with_named_sequence} {
       : (!transform.any_op) -> !transform.any_op
 
 
-    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul tile_sizes [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     %matmul_padded, %0, %copy_back = transform.structured.pad %matmul_l1 {
       padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
@@ -53,7 +53,7 @@ module attributes {transform.with_named_sequence} {
       : (!transform.any_op) -> !transform.any_op
 
 
-    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul tile_sizes [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     %matmul_padded, %0, %copy_back = transform.structured.pad %matmul_l1 {
       padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
@@ -98,7 +98,7 @@ module attributes {transform.with_named_sequence} {
       : (!transform.any_op) -> !transform.any_op
 
 
-    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul tile_sizes [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     %matmul_padded, %0, %copy_back = transform.structured.pad %matmul_l1 {
       padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
@@ -145,7 +145,7 @@ module attributes {transform.with_named_sequence} {
       : (!transform.any_op) -> !transform.any_op
 
 
-    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul tile_sizes [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     %matmul_padded, %0, %copy_back = transform.structured.pad %matmul_l1 {
       padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
@@ -191,7 +191,7 @@ module attributes {transform.with_named_sequence} {
       : (!transform.any_op) -> !transform.any_op
 
 
-    %matmul_l1, %loops_l1:2 = transform.structured.tile_using_for %matmul [5, 0, 7] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %matmul_l1, %loops_l1:2 = transform.structured.tile_using_for %matmul tile_sizes [5, 0, 7] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
 
     %matmul_padded, %0, %copy_back = transform.structured.pad %matmul_l1 {
       padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
diff --git a/mlir/test/Dialect/Linalg/transform-op-mmt4d-to-fma.mlir b/mlir/test/Dialect/Linalg/transform-op-mmt4d-to-fma.mlir
index 6aba2b3bb368..b5c6e610f58f 100644
--- a/mlir/test/Dialect/Linalg/transform-op-mmt4d-to-fma.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-mmt4d-to-fma.mlir
@@ -20,10 +20,10 @@ module attributes {transform.with_named_sequence} {
 
     // Step 1: Tile
     // Tile parallel dims
-    %tiled_linalg_op_p, %loops:4 = transform.structured.tile_using_for %mmt4d[1, 1, 0, 8, 8, 0]
+    %tiled_linalg_op_p, %loops:4 = transform.structured.tile_using_for %mmt4d tile_sizes [1, 1, 0, 8, 8, 0]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     // Tile reduction dims
-    %tiled_linalg_op_r, %loops2:2 = transform.structured.tile_using_for %tiled_linalg_op_p[0, 0, 1, 0, 0, 1]
+    %tiled_linalg_op_r, %loops2:2 = transform.structured.tile_using_for %tiled_linalg_op_p tile_sizes [0, 0, 1, 0, 0, 1]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
 
     // Step 2: Vectorize
diff --git a/mlir/test/Dialect/Linalg/transform-op-pack.mlir b/mlir/test/Dialect/Linalg/transform-op-pack.mlir
index cf6339ce3de8..6c26ebd0a5b8 100644
--- a/mlir/test/Dialect/Linalg/transform-op-pack.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-pack.mlir
@@ -372,8 +372,8 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
       %sz = transform.structured.match ops{["some_tile_size"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1 = transform.structured.pack %0 packed_sizes = [0, %sz : !transform.any_op, %sz : !transform.any_op]
-        : (!transform.any_op) -> (!transform.op<"linalg.generic">)
+      %1 = transform.structured.pack %0 packed_sizes = [0, %sz, %sz]
+        : (!transform.any_op, !transform.any_op, !transform.any_op) -> (!transform.op<"linalg.generic">)
         transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/transform-op-peel-and-vectorize-conv.mlir b/mlir/test/Dialect/Linalg/transform-op-peel-and-vectorize-conv.mlir
index 7f3997633a30..4bb40bef9fba 100644
--- a/mlir/test/Dialect/Linalg/transform-op-peel-and-vectorize-conv.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-peel-and-vectorize-conv.mlir
@@ -61,11 +61,11 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%root: !transform.any_op {transform.consume}) {
     // 1. Tile parallel dims
     %1 = transform.structured.match ops{["linalg.depthwise_conv_2d_nhwc_hwc"]} in %root : (!transform.any_op) -> !transform.any_op
-    %tiled_linalg_op_0, %loops_1:4 = transform.structured.tile_using_for %1[1, 1, 4, [4], 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.op<"scf.for">, !transform.op<"scf.for">, !transform.op<"scf.for">)
+    %tiled_linalg_op_0, %loops_1:4 = transform.structured.tile_using_for %1 tile_sizes [1, 1, 4, [4], 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.op<"scf.for">, !transform.op<"scf.for">, !transform.op<"scf.for">)
 
     // 2. Tile reduction dims
     %2 = transform.structured.match ops{["linalg.depthwise_conv_2d_nhwc_hwc"]} in %loops_1#3 : (!transform.op<"scf.for">) -> !transform.any_op
-    %tiled_linalg_op_1, %loops_2:2 = transform.structured.tile_using_for %2[0, 0, 0, 0, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %tiled_linalg_op_1, %loops_2:2 = transform.structured.tile_using_for %2 tile_sizes [0, 0, 0, 0, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
 
     // 3. Decompose 2D conv into 2 x 1D conv
     %3 = transform.structured.match ops{["linalg.depthwise_conv_2d_nhwc_hwc"]} in %loops_1#3 : (!transform.op<"scf.for">) -> !transform.any_op
diff --git a/mlir/test/Dialect/Linalg/transform-op-peel-and-vectorize.mlir b/mlir/test/Dialect/Linalg/transform-op-peel-and-vectorize.mlir
index b7e316f8925d..05a032b1ece0 100644
--- a/mlir/test/Dialect/Linalg/transform-op-peel-and-vectorize.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-peel-and-vectorize.mlir
@@ -67,7 +67,7 @@ module attributes {transform.with_named_sequence} {
     %matmul = transform.structured.match ops{["linalg.matmul"]} in %root : (!transform.any_op) -> !transform.any_op
     // 1. Scalable tiling
     %_, %loop_1, %loop_2, %loop_3 =
-      transform.structured.tile_using_for %matmul [8, [16], 1] : (!transform.any_op)
+      transform.structured.tile_using_for %matmul tile_sizes [8, [16], 1] : (!transform.any_op)
       -> (!transform.any_op, !transform.op<"scf.for">, !transform.op<"scf.for">,!transform.op<"scf.for">)
 
     // 2. Loop peeling (only the middle dimension)
diff --git a/mlir/test/Dialect/Linalg/transform-op-scalarize.mlir b/mlir/test/Dialect/Linalg/transform-op-scalarize.mlir
index 7d642c8995f0..91949f58931a 100644
--- a/mlir/test/Dialect/Linalg/transform-op-scalarize.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-scalarize.mlir
@@ -21,7 +21,7 @@ func.func @scalarize(%arg0: tensor<24x12xf32>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops = transform.structured.tile_using_for %0 [10, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %1, %loops = transform.structured.tile_using_for %0 tile_sizes [10, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
     %2 = transform.structured.scalarize %1 : (!transform.any_op) -> !transform.any_op
     transform.yield
   }
diff --git a/mlir/test/Dialect/Linalg/transform-op-tile.mlir b/mlir/test/Dialect/Linalg/transform-op-tile.mlir
index ea8c5e612479..d244670f7375 100644
--- a/mlir/test/Dialect/Linalg/transform-op-tile.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-tile.mlir
@@ -3,7 +3,7 @@
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [4, 4, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [4, 4, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -42,7 +42,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1 = transform.structured.match ops{["func.call"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %2, %loops:3 = transform.structured.tile_using_for %0 [%1, %1, 4] : (!transform.any_op, !transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %2, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [%1, %1, 4] : (!transform.any_op, !transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -86,7 +86,7 @@ module attributes {transform.with_named_sequence} {
     // expected-note @below {{for this parameter}}
     %1 = transform.test_produce_param (0 : i64) : !transform.param<i64>
     // expected-error @below {{expected as many parameter values (0) as target ops (2)}}
-    transform.structured.tile_using_for %0 [%1, %1, %1]
+    transform.structured.tile_using_for %0 tile_sizes [%1, %1, %1]
       : (!transform.any_op, !transform.param<i64>, !transform.param<i64>, !transform.param<i64>)
       -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
@@ -113,7 +113,7 @@ module attributes {transform.with_named_sequence} {
     // expected-note @below {{for this handle}}
     %1 = transform.structured.match ops{["arith.constant"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     // expected-error @below {{expected as many dynamic size-producing operations (0) as target ops (2)}}
-    transform.structured.tile_using_for %0 [%1, %1, 1]
+    transform.structured.tile_using_for %0 tile_sizes [%1, %1, 1]
       : (!transform.any_op, !transform.any_op, !transform.any_op)
       -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
@@ -194,7 +194,7 @@ module {
   module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loop = transform.structured.tile_using_for %0 [[4]] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+      %1, %loop = transform.structured.tile_using_for %0 tile_sizes [[4]] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
       transform.yield
   }
   }
@@ -230,7 +230,7 @@ func.func @scalable_and_fixed_length_tile(
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [4, 4, [4]] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [4, 4, [4]] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
@@ -249,7 +249,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     // expected-error @below {{too many tiles provided, expected at most 3 found 4}}
-    %1, %loops = transform.structured.tile_using_for %0 [1, 0, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %1, %loops = transform.structured.tile_using_for %0 tile_sizes [1, 0, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/transform-ops-invalid.mlir b/mlir/test/Dialect/Linalg/transform-ops-invalid.mlir
index e7d9815ab222..e86d4962530a 100644
--- a/mlir/test/Dialect/Linalg/transform-ops-invalid.mlir
+++ b/mlir/test/Dialect/Linalg/transform-ops-invalid.mlir
@@ -77,7 +77,7 @@ transform.sequence failures(propagate) {
 transform.sequence failures(propagate) {
 ^bb0(%arg0: !transform.any_op):
   %0 = transform.param.constant 2 : i64 -> !transform.param<i64>
-  // expected-error@below {{custom op 'transform.structured.vectorize' expected 2 operand type(s)}}
+  // expected-error@below {{custom op 'transform.structured.vectorize' 1 operands present, but expected 2}}
   transform.structured.vectorize %arg0 vector_sizes [%0, 2] : !transform.any_op, !transform.param<i64>, !transform.param<i64>
 
 }
diff --git a/mlir/test/Dialect/Linalg/transform-ops.mlir b/mlir/test/Dialect/Linalg/transform-ops.mlir
index 8f6274fd22c2..733f305f850c 100644
--- a/mlir/test/Dialect/Linalg/transform-ops.mlir
+++ b/mlir/test/Dialect/Linalg/transform-ops.mlir
@@ -3,7 +3,7 @@
 transform.sequence failures(propagate) {
 ^bb1(%arg0: !transform.any_op):
   // CHECK %{{.*}}, %{{.*}}:2 = transform.structured.tile
-  %0, %1:2 = transform.structured.tile_using_for %arg0 [2, 0, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+  %0, %1:2 = transform.structured.tile_using_for %arg0 tile_sizes [2, 0, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
 }
 
 // check that the Attributes of `tile_using_for` are preserved through printing
@@ -11,9 +11,9 @@ transform.sequence failures(propagate) {
 transform.sequence failures(propagate) {
 ^bb1(%arg0: !transform.any_op):
   // CHECK %{{.*}}, %{{.*}}:2 = transform.structured.tile %arg0 [2, 0, 3] interchange = [2, 1] {test_attr1 = 1 : i64, test_attr2}
-  %0, %1:2 = transform.structured.tile_using_for %arg0 [2, 0, 3] interchange = [2, 1] {test_attr1 = 1 : i64, test_attr2}: (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+  %0, %1:2 = transform.structured.tile_using_for %arg0 tile_sizes [2, 0, 3] interchange = [2, 1] {test_attr1 = 1 : i64, test_attr2}: (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
   // CHECK %{{.*}}, %{{.*}}:2 = transform.structured.tile %arg0 [4, 5, 3] {test_attr3 = 1 : i64, test_attr4}
-  %2, %3:2 = transform.structured.tile_using_for %0 [0, 5, 3] {test_attr3 = 1 : i64, test_attr4}: (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+  %2, %3:2 = transform.structured.tile_using_for %0 tile_sizes [0, 5, 3] {test_attr3 = 1 : i64, test_attr4}: (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
 }
 
 transform.sequence failures(propagate) {
diff --git a/mlir/test/Dialect/Linalg/transform-patterns.mlir b/mlir/test/Dialect/Linalg/transform-patterns.mlir
index 5a9b490c07ff..87b7664198da 100644
--- a/mlir/test/Dialect/Linalg/transform-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/transform-patterns.mlir
@@ -12,7 +12,7 @@ func.func @dot(%x: memref<?xf32, strided<[1], offset: ?>>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["linalg.dot"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loop = transform.structured.tile_using_for %0 [8000] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+      %1, %loop = transform.structured.tile_using_for %0 tile_sizes [8000] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -38,7 +38,7 @@ func.func @matvec(%A: memref<?x?xf32, strided<[?, 1], offset: ?>>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["linalg.matvec"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [5, 6] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [5, 6] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -67,10 +67,10 @@ func.func @matmul(%A: memref<?x?xf32, strided<[?, 1], offset: ?>>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:3 = transform.structured.tile_using_for %0 [2000, 3000, 4000] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
-      %2, %loops_2:3 = transform.structured.tile_using_for %1 [200, 300, 400] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
-      %3, %loops_3:3 = transform.structured.tile_using_for %2 [20, 30, 40] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
-      %4, %loops_4:3 = transform.structured.tile_using_for %3 [2, 3, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [2000, 3000, 4000] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+      %2, %loops_2:3 = transform.structured.tile_using_for %1 tile_sizes [200, 300, 400] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+      %3, %loops_3:3 = transform.structured.tile_using_for %2 tile_sizes [20, 30, 40] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+      %4, %loops_4:3 = transform.structured.tile_using_for %3 tile_sizes [2, 3, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -170,7 +170,7 @@ func.func @matvec_perm(%A: memref<?x?xf32, strided<[?, 1], offset: ?>>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["linalg.matvec"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [5, 6] interchange = [1, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [5, 6] interchange = [1, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -199,9 +199,9 @@ func.func @matmul_perm(%A: memref<?x?xf32, strided<[?, 1], offset: ?>>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:3 = transform.structured.tile_using_for %0 [2000, 3000, 4000] interchange = [1, 2, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
-      %2, %loops_2:3 = transform.structured.tile_using_for %1 [200, 300, 400] interchange = [1, 0, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
-      %3, %loops_3:3 = transform.structured.tile_using_for %2 [20, 30, 40] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [2000, 3000, 4000] interchange = [1, 2, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+      %2, %loops_2:3 = transform.structured.tile_using_for %1 tile_sizes [200, 300, 400] interchange = [1, 0, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+      %3, %loops_3:3 = transform.structured.tile_using_for %2 tile_sizes [20, 30, 40] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/vectorize-tensor-extract-masked.mlir b/mlir/test/Dialect/Linalg/vectorize-tensor-extract-masked.mlir
index edc38b42f5cd..e68d297dc41f 100644
--- a/mlir/test/Dialect/Linalg/vectorize-tensor-extract-masked.mlir
+++ b/mlir/test/Dialect/Linalg/vectorize-tensor-extract-masked.mlir
@@ -28,7 +28,7 @@ func.func @masked_static_vectorize_nd_tensor_extract_with_affine_apply_contiguou
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
      %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-     transform.structured.vectorize %0 vector_sizes [1, 4] vectorize_nd_extract : !transform.any_op
+     transform.structured.vectorize %0 vector_sizes [1, 4] {vectorize_nd_extract} : !transform.any_op
      transform.yield
    }
 }
@@ -85,7 +85,7 @@ func.func @masked_dynamic_vectorize_nd_tensor_extract_with_affine_apply_contiguo
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
      %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-     transform.structured.vectorize %0 vector_sizes [1, 4] vectorize_nd_extract : !transform.any_op
+     transform.structured.vectorize %0 vector_sizes [1, 4] {vectorize_nd_extract} : !transform.any_op
      transform.yield
   }
 }
@@ -125,7 +125,7 @@ func.func @masked_vectorize_nd_tensor_extract_with_affine_apply_gather(%6: tenso
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
      %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-     transform.structured.vectorize %0 vector_sizes [1, 4] vectorize_nd_extract : !transform.any_op
+     transform.structured.vectorize %0 vector_sizes [1, 4] {vectorize_nd_extract} : !transform.any_op
      transform.yield
    }
 }
@@ -182,7 +182,7 @@ func.func @masked_dynamic_vectorize_nd_tensor_extract_with_affine_apply_gather(%
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
      %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-     transform.structured.vectorize %0 vector_sizes [1, 4] vectorize_nd_extract : !transform.any_op
+     transform.structured.vectorize %0 vector_sizes [1, 4] {vectorize_nd_extract} : !transform.any_op
      transform.yield
    }
 }
@@ -234,7 +234,7 @@ func.func @extract_masked_vectorize(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf3
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
      %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-     transform.structured.vectorize %0 vector_sizes [3, 3] vectorize_nd_extract : !transform.any_op
+     transform.structured.vectorize %0 vector_sizes [3, 3] {vectorize_nd_extract} : !transform.any_op
      transform.yield
    }
 }
@@ -279,7 +279,7 @@ func.func @tensor_extract_dynamic_shape(%arg1: tensor<123x321xf32>, %arg2: tenso
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
      %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-     transform.structured.vectorize %0 vector_sizes [1, 3, 8] vectorize_nd_extract : !transform.any_op
+     transform.structured.vectorize %0 vector_sizes [1, 3, 8] {vectorize_nd_extract} : !transform.any_op
      transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Tensor/tiling.mlir b/mlir/test/Dialect/Tensor/tiling.mlir
index 1afbd3d0504f..e02ab06a9d53 100644
--- a/mlir/test/Dialect/Tensor/tiling.mlir
+++ b/mlir/test/Dialect/Tensor/tiling.mlir
@@ -34,7 +34,7 @@ func.func @dynamic_pad_tensor_3_4(%input_tensor: tensor<?x?xf32>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -73,7 +73,7 @@ func.func @dynamic_pad_tensor_0_3(%input_tensor: tensor<?x?xf32>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loop = transform.structured.tile_using_for %0 [0, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+      %1, %loop = transform.structured.tile_using_for %0 tile_sizes [0, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -109,7 +109,7 @@ func.func @static_pad_tensor_3_4(%input_tensor: tensor<7x9xf32>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -142,7 +142,7 @@ func.func @static_pad_tensor_0_3(%input_tensor: tensor<7x9xf32>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loop = transform.structured.tile_using_for %0 [0, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+      %1, %loop = transform.structured.tile_using_for %0 tile_sizes [0, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -179,7 +179,7 @@ func.func @static_pad_tile_evenly_0_3(%input_tensor: tensor<7x9xf32>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loop = transform.structured.tile_using_for %0 [0, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+      %1, %loop = transform.structured.tile_using_for %0 tile_sizes [0, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -217,7 +217,7 @@ func.func @NC_to_NCnc(%arg0: tensor<128x256xf32>, %arg1: tensor<4x8x32x32xf32>)
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -247,7 +247,7 @@ func.func @KC_to_CKkc(%arg0: tensor<128x256xf32>, %arg1: tensor<32x4x32x8xf32>)
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -284,7 +284,7 @@ func.func @pad_and_pack_static(%input: tensor<13x15xf32>, %output: tensor<2x8x8x
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -335,7 +335,7 @@ func.func @pad_and_pack_partially_dynamic(%input: tensor<?x?xf32>, %output: tens
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -391,7 +391,7 @@ func.func @pad_and_pack_fully_dynamic(%source: tensor<?x?xf32>, %dest: tensor<?x
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -440,7 +440,7 @@ func.func @NCnc_to_NC(%source: tensor<8x8x32x16xf32>, %dest: tensor<256x128xf32>
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.unpack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -488,7 +488,7 @@ func.func @CKkc_to_KC(%source: tensor<32x4x32x8xf32>, %dest: tensor<128x256xf32>
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.unpack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -526,7 +526,7 @@ func.func @perfect_CKkc_to_KC(%source: tensor<32x4x2x4xf32>, %dest: tensor<8x128
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.unpack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -570,7 +570,7 @@ func.func @dynamic_perfect_CKkc_to_KC(%source: tensor<?x?x2x2xf32>, %dest: tenso
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.unpack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -607,7 +607,7 @@ func.func @perfect_NKPQk_to_NPQK(%source: tensor<1x4x6x6x2xf32>, %dest: tensor<1
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.unpack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:4 = transform.structured.tile_using_for %0 [1, 1, 1, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:4 = transform.structured.tile_using_for %0 tile_sizes [1, 1, 1, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -635,7 +635,7 @@ func.func @fully_dynamic_unpack(%source: tensor<?x?x?x?xf32>, %dest: tensor<?x?x
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.unpack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:2 = transform.structured.tile_using_for %0 [4, 8] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [4, 8] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
@@ -671,7 +671,7 @@ func.func @perfect_NPQK_to_NKPQk(%source: tensor<1x6x6x8xf32>, %dest: tensor<1x4
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
       %0 = transform.structured.match ops{["tensor.pack"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-      %1, %loops:4 = transform.structured.tile_using_for %0 [1, 1, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+      %1, %loops:4 = transform.structured.tile_using_for %0 tile_sizes [1, 1, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Transform/ops.mlir b/mlir/test/Dialect/Transform/ops.mlir
index ecef7e181e90..b03a9f4d760d 100644
--- a/mlir/test/Dialect/Transform/ops.mlir
+++ b/mlir/test/Dialect/Transform/ops.mlir
@@ -101,19 +101,19 @@ transform.sequence failures(propagate) {
 }
 
 // CHECK: transform.sequence
-// CHECK: transform.structured.tile_using_for %0[4, 4, [4]]
+// CHECK: transform.structured.tile_using_for %0 tile_sizes [4, 4, [4]]
 transform.sequence failures(propagate) {
 ^bb0(%arg1: !transform.any_op):
   %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-  transform.structured.tile_using_for %0 [4, 4, [4]] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+  transform.structured.tile_using_for %0 tile_sizes [4, 4, [4]] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
 }
 
 // CHECK: transform.sequence
-// CHECK: transform.structured.tile_using_for %0{{\[}}[2], 4, 8]
+// CHECK: transform.structured.tile_using_for %0 tile_sizes {{\[}}[2], 4, 8]
 transform.sequence failures(propagate) {
 ^bb0(%arg1: !transform.any_op):
   %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-  transform.structured.tile_using_for %0 [[2], 4, 8] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+  transform.structured.tile_using_for %0 tile_sizes [[2], 4, 8] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
 }
 
 // CHECK: transform.sequence
diff --git a/mlir/test/Dialect/Transform/selective-targeting.mlir b/mlir/test/Dialect/Transform/selective-targeting.mlir
index e88104315649..69342100935c 100644
--- a/mlir/test/Dialect/Transform/selective-targeting.mlir
+++ b/mlir/test/Dialect/Transform/selective-targeting.mlir
@@ -79,7 +79,7 @@ module attributes {transform.with_named_sequence} {
       transform.sequence %arg0 : !transform.any_op failures(propagate) {
       ^bb1(%arg1: !transform.any_op):
         %0 = pdl_match @pdl_target_attrA in %arg1 : (!transform.any_op) -> !transform.any_op
-        transform.structured.tile_using_for %0 [4, 4, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+        transform.structured.tile_using_for %0 tile_sizes [4, 4, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
         %1 = pdl_match @pdl_target_attrC in %arg1 : (!transform.any_op) -> !transform.any_op
         %2 = get_parent_op %1 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
         transform.structured.vectorize_children_and_apply_patterns %2 : (!transform.any_op) -> !transform.any_op
diff --git a/mlir/test/Dialect/Vector/transform-vector.mlir b/mlir/test/Dialect/Vector/transform-vector.mlir
index a0ca8c2fa9b6..75b29e22b4d2 100644
--- a/mlir/test/Dialect/Vector/transform-vector.mlir
+++ b/mlir/test/Dialect/Vector/transform-vector.mlir
@@ -16,7 +16,7 @@ func.func @matmul_tensors(
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.consumed}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %module_op : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [8, 4, 2]
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [8, 4, 2]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     %2 = transform.get_parent_op %1 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
     transform.structured.vectorize_children_and_apply_patterns %2 : (!transform.any_op) -> !transform.any_op
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
index 34c5351c8703..a8b6457d64be 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
@@ -61,7 +61,7 @@ module attributes {transform.with_named_sequence} {
 
     // Step 1: Tile for size [4] x [4], which corresponds to SVLs x SVLs, where
     //         SVLs is the number of 32-bit elements in a vector of SVL bits.
-    %tiled_linalg_op, %loops:3 = transform.structured.tile_using_for %matmul_transpose_a[[4], [4], 1]
+    %tiled_linalg_op, %loops:3 = transform.structured.tile_using_for %matmul_transpose_a tile_sizes [[4], [4], 1]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
 
     // Step 2: Vectorize.
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
index 2bfdaa8e8a2b..091665223188 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
@@ -59,7 +59,7 @@ module attributes {transform.with_named_sequence} {
 
     // Step 1: Tile for size [4] x [4], which corresponds to SVLs x SVLs, where
     // SVLs is the number of 32-bit elements in a vector of SVL bits.
-    %tiled_linalg_op, %loops:3 = transform.structured.tile_using_for %matmul[[4], [4], 1]
+    %tiled_linalg_op, %loops:3 = transform.structured.tile_using_for %matmul tile_sizes [[4], [4], 1]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
 
     // Step 2: Vectorize.
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
index 9f06226a4f65..10ffed268817 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
@@ -84,7 +84,7 @@ module attributes {transform.with_named_sequence} {
     // Step 1: Tile for size [8] x [8] (unrolled by 4), which corresponds to
     // (2 x SVLs) x (2 x SVLs), where SVLs is the number of 32-bit elements in a
     // vector of SVL bits. This uses all four 32-bit SME virtual tiles.
-    %tiled_linalg_op, %loop_i, %loop_j, %loop_k = transform.structured.tile_using_for %matmul[[8], [8], 4]
+    %tiled_linalg_op, %loop_i, %loop_j, %loop_k = transform.structured.tile_using_for %matmul tile_sizes [[8], [8], 4]
       : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.op<"scf.for">, !transform.op<"scf.for">)
 
     // Step 2: Vectorize.
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir
index e376bdde24a1..ada744b322fe 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir
@@ -72,7 +72,7 @@ module attributes {transform.with_named_sequence} {
     // Step 1: Tile for size [8] x [8] (unrolled by 4), which corresponds to
     // (2 x SVLs) x (2 x SVLs), where SVLs is the number of 32-bit elements in a
     // vector of SVL bits. This uses all four 32-bit SME virtual tiles.
-    %tiled_linalg_op, %loop_i, %loop_j, %loop_k = transform.structured.tile_using_for %matmul[[8], [8], 4]
+    %tiled_linalg_op, %loop_i, %loop_j, %loop_k = transform.structured.tile_using_for %matmul tile_sizes [[8], [8], 4]
       : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.op<"scf.for">, !transform.op<"scf.for">)
 
     // Step 2: Vectorize.
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
index 68e474fe5cef..edb9de922808 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
@@ -96,7 +96,7 @@ module attributes {transform.with_named_sequence} {
       : (!transform.op<"func.func">) -> !transform.any_op
 
     // Step 1: Tile
-    %tiled_matmul, %loops:3 = transform.structured.tile_using_for %matmul [2, [4], 1]
+    %tiled_matmul, %loops:3 = transform.structured.tile_using_for %matmul tile_sizes [2, [4], 1]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
 
     // Step 2: Vectorize
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/mmt4d.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/mmt4d.mlir
index 92c7039c8496..183625f9748c 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/mmt4d.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/mmt4d.mlir
@@ -70,10 +70,10 @@ module @transforms attributes { transform.with_named_sequence } {
 
    // Step 1: Tile
    // Tile parallel dims
-   %tiled_linalg_op_p, %loops:4 = transform.structured.tile_using_for %mmt4d[1, 1, 0, 3, 3, 0]
+   %tiled_linalg_op_p, %loops:4 = transform.structured.tile_using_for %mmt4d tile_sizes [1, 1, 0, 3, 3, 0]
      : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
    // Tile reduction dims
-   %tiled_linalg_op_r, %loops2:2 = transform.structured.tile_using_for %tiled_linalg_op_p[0, 0, 1, 0, 0, 1]
+   %tiled_linalg_op_r, %loops2:2 = transform.structured.tile_using_for %tiled_linalg_op_p tile_sizes [0, 0, 1, 0, 0, 1]
      : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
 
    // Step 2: Vectorize
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/pack-unpack-mmt4d.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/pack-unpack-mmt4d.mlir
index 5680882dccb1..10b29dd70177 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/pack-unpack-mmt4d.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/pack-unpack-mmt4d.mlir
@@ -107,10 +107,10 @@ module @transforms attributes { transform.with_named_sequence } {
 
    // Step 1: Tile
    // Tile parallel dims
-   %tiled_linalg_op_p, %loops:4 = transform.structured.tile_using_for %mmt4d[1, 1, 0, 8, 8, 0]
+   %tiled_linalg_op_p, %loops:4 = transform.structured.tile_using_for %mmt4d tile_sizes [1, 1, 0, 8, 8, 0]
      : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
    // Tile reduction dims
-   %tiled_linalg_op_r, %loops2:2 = transform.structured.tile_using_for %tiled_linalg_op_p[0, 0, 1, 0, 0, 1]
+   %tiled_linalg_op_r, %loops2:2 = transform.structured.tile_using_for %tiled_linalg_op_p tile_sizes [0, 0, 1, 0, 0, 1]
      : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
 
    // Step 2: Vectorize
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir
index 443963fb8c59..9b46056918b5 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir
@@ -27,7 +27,7 @@ func.func @conv_1d(%arg0: memref<?xf32>, %arg1: memref<?xf32>, %arg2: memref<?xf
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_1d"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loop = transform.structured.tile_using_for %0 [4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %1, %loop = transform.structured.tile_using_for %0 tile_sizes [4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir
index f652d707de05..d6726fe1a6b4 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir
@@ -29,7 +29,7 @@ func.func @conv_1d_nwc_wcf(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32>, %
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_1d_nwc_wcf"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:2 = transform.structured.tile_using_for %0 [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir
index 2eaba8233d69..bb77d5eb9b8d 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir
@@ -27,7 +27,7 @@ func.func @conv_2d(%arg0: memref<?x?xf32>, %arg1: memref<?x?xf32>, %arg2: memref
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_2d"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:2 = transform.structured.tile_using_for %0 [2, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:2 = transform.structured.tile_using_for %0 tile_sizes [2, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir
index eac8d8a6ea43..39415dff1cbb 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir
@@ -29,7 +29,7 @@ func.func @conv_2d_nhwc_hwcf(%arg0: memref<?x?x?x?xf32>, %arg1: memref<?x?x?x?xf
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_2d_nhwc_hwcf"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:4 = transform.structured.tile_using_for %0 [2, 3, 3, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:4 = transform.structured.tile_using_for %0 tile_sizes [2, 3, 3, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir
index d5584cd67702..ece054ac7176 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir
@@ -27,7 +27,7 @@ func.func @conv_3d(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32>, %arg2: me
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_3d"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [2, 2, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [2, 2, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir
index 7dca79334565..ce169ee470c3 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir
@@ -29,7 +29,7 @@ func.func @conv_3d_ndhwc_dhwcf(%arg0: memref<?x?x?x?x?xf32>, %arg1: memref<?x?x?
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_3d_ndhwc_dhwcf"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [0, 5, 5, 5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [0, 5, 5, 5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir
index fda7ffb0c753..41296cdfcb2d 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir
@@ -39,7 +39,7 @@ func.func @main() {
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1, %loops:3 = transform.structured.tile_using_for %0 [1, 2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    %1, %loops:3 = transform.structured.tile_using_for %0 tile_sizes [1, 2, 3] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Interfaces/TilingInterface/tile-pad-using-interface.mlir b/mlir/test/Interfaces/TilingInterface/tile-pad-using-interface.mlir
index ba56206f03d7..7d247aefcf6b 100644
--- a/mlir/test/Interfaces/TilingInterface/tile-pad-using-interface.mlir
+++ b/mlir/test/Interfaces/TilingInterface/tile-pad-using-interface.mlir
@@ -14,7 +14,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %pad = transform.structured.match ops{["tensor.pad"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b, %c = transform.structured.tile_using_for %pad [2, 3]
+    %a, %b, %c = transform.structured.tile_using_for %pad tile_sizes [2, 3]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -57,7 +57,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %pad = transform.structured.match ops{["tensor.pad"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b = transform.structured.tile_using_for %pad [0, 3]
+    %a, %b = transform.structured.tile_using_for %pad tile_sizes [0, 3]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -97,7 +97,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %pad = transform.structured.match ops{["tensor.pad"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b, %c = transform.structured.tile_using_for %pad [2, 3]
+    %a, %b, %c = transform.structured.tile_using_for %pad tile_sizes [2, 3]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -134,7 +134,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %pad = transform.structured.match ops{["tensor.pad"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b = transform.structured.tile_using_for %pad [0, 3]
+    %a, %b = transform.structured.tile_using_for %pad tile_sizes [0, 3]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -170,7 +170,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %pad = transform.structured.match ops{["tensor.pad"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b, %c = transform.structured.tile_using_for %pad [2, 3]
+    %a, %b, %c = transform.structured.tile_using_for %pad tile_sizes [2, 3]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -192,7 +192,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %pad = transform.structured.match ops{["tensor.pad"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b = transform.structured.tile_using_for %pad [0, 3]
+    %a, %b = transform.structured.tile_using_for %pad tile_sizes [0, 3]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
     transform.yield
   }
diff --git a/mlir/test/Interfaces/TilingInterface/tile-using-interface.mlir b/mlir/test/Interfaces/TilingInterface/tile-using-interface.mlir
index 607836faafb7..488a52e8e3e9 100644
--- a/mlir/test/Interfaces/TilingInterface/tile-using-interface.mlir
+++ b/mlir/test/Interfaces/TilingInterface/tile-using-interface.mlir
@@ -11,7 +11,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b, %c = transform.structured.tile_using_for %matmul [10, 20]
+    %a, %b, %c = transform.structured.tile_using_for %matmul tile_sizes [10, 20]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -63,7 +63,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b, %c, %d = transform.structured.tile_using_for %matmul [10, 20, 30]
+    %a, %b, %c, %d = transform.structured.tile_using_for %matmul tile_sizes [10, 20, 30]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -122,7 +122,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %generic = transform.structured.match ops{["linalg.generic"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b, %c = transform.structured.tile_using_for %generic [10, 0, 20]
+    %a, %b, %c = transform.structured.tile_using_for %generic tile_sizes [10, 0, 20]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -175,7 +175,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %conv = transform.structured.match ops{["linalg.conv_2d_nhwc_hwcf"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b, %c, %d = transform.structured.tile_using_for %conv [0, 0, 0, 0, 10, 20, 30]
+    %a, %b, %c, %d = transform.structured.tile_using_for %conv tile_sizes [0, 0, 0, 0, 10, 20, 30]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -254,7 +254,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %generic = transform.structured.match ops{["linalg.generic"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b, %c = transform.structured.tile_using_for %generic [10, 20]
+    %a, %b, %c = transform.structured.tile_using_for %generic tile_sizes [10, 20]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -282,7 +282,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b, %c, %d = transform.structured.tile_using_for %matmul [10, 20, 30] interchange = [1, 2, 0]
+    %a, %b, %c, %d = transform.structured.tile_using_for %matmul tile_sizes [10, 20, 30] interchange = [1, 2, 0]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -338,7 +338,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %copy = transform.structured.match ops{["linalg.copy"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a, %b, %c = transform.structured.tile_using_for %copy [10, 20]
+    %a, %b, %c = transform.structured.tile_using_for %copy tile_sizes [10, 20]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     transform.yield
   }
@@ -369,7 +369,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %generic = transform.structured.match ops{["linalg.generic"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a = transform.structured.tile_using_for %generic []
+    %a = transform.structured.tile_using_for %generic tile_sizes []
       : (!transform.any_op) -> (!transform.any_op)
     transform.yield
   }
@@ -396,7 +396,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
     %generic = transform.structured.match ops{["linalg.generic"]} in %arg1
       : (!transform.any_op) -> !transform.any_op
-    %a = transform.structured.tile_using_for %generic []
+    %a = transform.structured.tile_using_for %generic tile_sizes []
       : (!transform.any_op) -> (!transform.any_op)
     transform.yield
   }
diff --git a/mlir/test/python/dialects/transform_structured_ext.py b/mlir/test/python/dialects/transform_structured_ext.py
index f4c092ba9ee9..935534edba7a 100644
--- a/mlir/test/python/dialects/transform_structured_ext.py
+++ b/mlir/test/python/dialects/transform_structured_ext.py
@@ -501,7 +501,7 @@ def testTileToForallMixedDynamic(target):
     structured.TileUsingForallOp(target, num_threads=[n, 3, 4])
     # CHECK-LABEL: TEST: testTileToForallMixedDynamic
     # CHECK: = transform.structured.tile_using_forall
-    # CHECK-SAME: num_threads [%{{.*}} : !transform.any_op, 3, 4]
+    # CHECK-SAME: num_threads [%{{.*}}, 3, 4] : (!transform.any_op, !transform.any_op)
 
 
 @run
@@ -511,7 +511,7 @@ def testTileToForallPackedDynamic(target):
     structured.TileUsingForallOp(target, num_threads=n)
     # CHECK-LABEL: TEST: testTileToForallPackedDynamic
     # CHECK: = transform.structured.tile_using_forall
-    # CHECK-SAME: num_threads *(%0 : !transform.any_op)
+    # CHECK-SAME: num_threads *(%0) : (!transform.any_op, !transform.any_op)
 
 
 @run
-- 
GitLab


From d6d613aaebc0ae503409ba7719a43b4a55e1ee70 Mon Sep 17 00:00:00 2001
From: martinboehme <mboehme@google.com>
Date: Wed, 8 May 2024 16:12:53 +0200
Subject: [PATCH 0175/1206] [clang][dataflow] Make `SolverTest` a
 type-parameterized test. (#91455)

This allows the tests to be run against any implementation of `Solver`
instead
of begin specific to `WatchedLiteralsSolver` as they currently are.
---
 clang/docs/tools/clang-formatted-files.txt    |   3 +-
 .../Analysis/FlowSensitive/CMakeLists.txt     |   2 +-
 .../{SolverTest.cpp => SolverTest.h}          | 166 ++++++++++--------
 .../WatchedLiteralsSolverTest.cpp             |  26 +++
 .../unittests/Analysis/FlowSensitive/BUILD.gn |   2 +-
 5 files changed, 124 insertions(+), 75 deletions(-)
 rename clang/unittests/Analysis/FlowSensitive/{SolverTest.cpp => SolverTest.h} (62%)
 create mode 100644 clang/unittests/Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp

diff --git a/clang/docs/tools/clang-formatted-files.txt b/clang/docs/tools/clang-formatted-files.txt
index 2252d0ccde96..eaeadf2656b0 100644
--- a/clang/docs/tools/clang-formatted-files.txt
+++ b/clang/docs/tools/clang-formatted-files.txt
@@ -632,11 +632,12 @@ clang/unittests/Analysis/FlowSensitive/MapLatticeTest.cpp
 clang/unittests/Analysis/FlowSensitive/MatchSwitchTest.cpp
 clang/unittests/Analysis/FlowSensitive/MultiVarConstantPropagationTest.cpp
 clang/unittests/Analysis/FlowSensitive/SingleVarConstantPropagationTest.cpp
-clang/unittests/Analysis/FlowSensitive/SolverTest.cpp
+clang/unittests/Analysis/FlowSensitive/SolverTest.h
 clang/unittests/Analysis/FlowSensitive/TestingSupport.cpp
 clang/unittests/Analysis/FlowSensitive/TestingSupport.h
 clang/unittests/Analysis/FlowSensitive/TestingSupportTest.cpp
 clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp
+clang/unittests/Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp
 clang/unittests/AST/ASTImporterFixtures.cpp
 clang/unittests/AST/ASTImporterFixtures.h
 clang/unittests/AST/ASTImporterObjCTest.cpp
diff --git a/clang/unittests/Analysis/FlowSensitive/CMakeLists.txt b/clang/unittests/Analysis/FlowSensitive/CMakeLists.txt
index 94160d949637..cfabb80576bc 100644
--- a/clang/unittests/Analysis/FlowSensitive/CMakeLists.txt
+++ b/clang/unittests/Analysis/FlowSensitive/CMakeLists.txt
@@ -19,7 +19,6 @@ add_clang_unittest(ClangAnalysisFlowSensitiveTests
   SignAnalysisTest.cpp
   SimplifyConstraintsTest.cpp
   SingleVarConstantPropagationTest.cpp
-  SolverTest.cpp
   TestingSupport.cpp
   TestingSupportTest.cpp
   TransferBranchTest.cpp
@@ -27,6 +26,7 @@ add_clang_unittest(ClangAnalysisFlowSensitiveTests
   TypeErasedDataflowAnalysisTest.cpp
   UncheckedOptionalAccessModelTest.cpp
   ValueTest.cpp
+  WatchedLiteralsSolverTest.cpp
   )
 
 clang_target_link_libraries(ClangAnalysisFlowSensitiveTests
diff --git a/clang/unittests/Analysis/FlowSensitive/SolverTest.cpp b/clang/unittests/Analysis/FlowSensitive/SolverTest.h
similarity index 62%
rename from clang/unittests/Analysis/FlowSensitive/SolverTest.cpp
rename to clang/unittests/Analysis/FlowSensitive/SolverTest.h
index 71f6da93594e..b37534438121 100644
--- a/clang/unittests/Analysis/FlowSensitive/SolverTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/SolverTest.h
@@ -1,4 +1,4 @@
-//===- unittests/Analysis/FlowSensitive/SolverTest.cpp --------------------===//
+//===--- SolverTest.h - Type-parameterized test for solvers ---------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,43 +6,53 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <utility>
+#ifndef LLVM_CLANG_ANALYSIS_FLOW_SENSITIVE_SOLVER_TEST_H_
+#define LLVM_CLANG_ANALYSIS_FLOW_SENSITIVE_SOLVER_TEST_H_
 
 #include "TestingSupport.h"
-#include "clang/Analysis/FlowSensitive/Arena.h"
-#include "clang/Analysis/FlowSensitive/Formula.h"
 #include "clang/Analysis/FlowSensitive/Solver.h"
-#include "clang/Analysis/FlowSensitive/WatchedLiteralsSolver.h"
-#include "clang/Basic/LLVM.h"
-#include "llvm/ADT/ArrayRef.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
-#include <vector>
+
+namespace clang::dataflow::test {
 
 namespace {
 
-using namespace clang;
-using namespace dataflow;
+constexpr auto AssignedTrue = Solver::Result::Assignment::AssignedTrue;
+constexpr auto AssignedFalse = Solver::Result::Assignment::AssignedFalse;
 
-using test::ConstraintContext;
-using test::parseFormulas;
 using testing::_;
 using testing::AnyOf;
 using testing::Pair;
 using testing::UnorderedElementsAre;
 
-constexpr auto AssignedTrue = Solver::Result::Assignment::AssignedTrue;
-constexpr auto AssignedFalse = Solver::Result::Assignment::AssignedFalse;
+} // namespace
 
-// Checks if the conjunction of `Vals` is satisfiable and returns the
-// corresponding result.
-Solver::Result solve(llvm::ArrayRef<const Formula *> Vals) {
-  return WatchedLiteralsSolver().solve(Vals);
-}
+/// Type-parameterized test for implementations of the `Solver` interface.
+/// To use:
+/// 1.  Implement a specialization of `createSolverWithLowTimeout()` for the
+///     solver you want to test.
+/// 2.  Instantiate the test suite for the solver you want to test using
+///     `INSTANTIATE_TYPED_TEST_SUITE_P()`.
+/// See WatchedLiteralsSolverTest.cpp for an example.
+template <typename SolverT> class SolverTest : public ::testing::Test {
+protected:
+  // Checks if the conjunction of `Vals` is satisfiable and returns the
+  // corresponding result.
+  Solver::Result solve(llvm::ArrayRef<const Formula *> Vals) {
+    return SolverT().solve(Vals);
+  }
+
+  // Create a specialization for the solver type to test.
+  SolverT createSolverWithLowTimeout();
+};
+
+TYPED_TEST_SUITE_P(SolverTest);
 
 MATCHER(unsat, "") {
   return arg.getStatus() == Solver::Result::Status::Unsatisfiable;
 }
+
 MATCHER_P(sat, SolutionMatcher,
           "is satisfiable, where solution " +
               (testing::DescribeMatcher<
@@ -55,57 +65,57 @@ MATCHER_P(sat, SolutionMatcher,
                                      result_listener);
 }
 
-TEST(SolverTest, Var) {
+TYPED_TEST_P(SolverTest, Var) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
 
   // X
-  EXPECT_THAT(solve({X}),
+  EXPECT_THAT(this->solve({X}),
               sat(UnorderedElementsAre(Pair(X->getAtom(), AssignedTrue))));
 }
 
-TEST(SolverTest, NegatedVar) {
+TYPED_TEST_P(SolverTest, NegatedVar) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto NotX = Ctx.neg(X);
 
   // !X
-  EXPECT_THAT(solve({NotX}),
+  EXPECT_THAT(this->solve({NotX}),
               sat(UnorderedElementsAre(Pair(X->getAtom(), AssignedFalse))));
 }
 
-TEST(SolverTest, UnitConflict) {
+TYPED_TEST_P(SolverTest, UnitConflict) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto NotX = Ctx.neg(X);
 
   // X ^ !X
-  EXPECT_THAT(solve({X, NotX}), unsat());
+  EXPECT_THAT(this->solve({X, NotX}), unsat());
 }
 
-TEST(SolverTest, DistinctVars) {
+TYPED_TEST_P(SolverTest, DistinctVars) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto Y = Ctx.atom();
   auto NotY = Ctx.neg(Y);
 
   // X ^ !Y
-  EXPECT_THAT(solve({X, NotY}),
+  EXPECT_THAT(this->solve({X, NotY}),
               sat(UnorderedElementsAre(Pair(X->getAtom(), AssignedTrue),
                                        Pair(Y->getAtom(), AssignedFalse))));
 }
 
-TEST(SolverTest, DoubleNegation) {
+TYPED_TEST_P(SolverTest, DoubleNegation) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto NotX = Ctx.neg(X);
   auto NotNotX = Ctx.neg(NotX);
 
   // !!X ^ !X
-  EXPECT_THAT(solve({NotNotX, NotX}), unsat());
+  EXPECT_THAT(this->solve({NotNotX, NotX}), unsat());
 }
 
-TEST(SolverTest, NegatedDisjunction) {
+TYPED_TEST_P(SolverTest, NegatedDisjunction) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto Y = Ctx.atom();
@@ -113,10 +123,10 @@ TEST(SolverTest, NegatedDisjunction) {
   auto NotXOrY = Ctx.neg(XOrY);
 
   // !(X v Y) ^ (X v Y)
-  EXPECT_THAT(solve({NotXOrY, XOrY}), unsat());
+  EXPECT_THAT(this->solve({NotXOrY, XOrY}), unsat());
 }
 
-TEST(SolverTest, NegatedConjunction) {
+TYPED_TEST_P(SolverTest, NegatedConjunction) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto Y = Ctx.atom();
@@ -124,48 +134,48 @@ TEST(SolverTest, NegatedConjunction) {
   auto NotXAndY = Ctx.neg(XAndY);
 
   // !(X ^ Y) ^ (X ^ Y)
-  EXPECT_THAT(solve({NotXAndY, XAndY}), unsat());
+  EXPECT_THAT(this->solve({NotXAndY, XAndY}), unsat());
 }
 
-TEST(SolverTest, DisjunctionSameVarWithNegation) {
+TYPED_TEST_P(SolverTest, DisjunctionSameVarWithNegation) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto NotX = Ctx.neg(X);
   auto XOrNotX = Ctx.disj(X, NotX);
 
   // X v !X
-  EXPECT_THAT(solve({XOrNotX}), sat(_));
+  EXPECT_THAT(this->solve({XOrNotX}), sat(_));
 }
 
-TEST(SolverTest, DisjunctionSameVar) {
+TYPED_TEST_P(SolverTest, DisjunctionSameVar) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto XOrX = Ctx.disj(X, X);
 
   // X v X
-  EXPECT_THAT(solve({XOrX}), sat(_));
+  EXPECT_THAT(this->solve({XOrX}), sat(_));
 }
 
-TEST(SolverTest, ConjunctionSameVarsConflict) {
+TYPED_TEST_P(SolverTest, ConjunctionSameVarsConflict) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto NotX = Ctx.neg(X);
   auto XAndNotX = Ctx.conj(X, NotX);
 
   // X ^ !X
-  EXPECT_THAT(solve({XAndNotX}), unsat());
+  EXPECT_THAT(this->solve({XAndNotX}), unsat());
 }
 
-TEST(SolverTest, ConjunctionSameVar) {
+TYPED_TEST_P(SolverTest, ConjunctionSameVar) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto XAndX = Ctx.conj(X, X);
 
   // X ^ X
-  EXPECT_THAT(solve({XAndX}), sat(_));
+  EXPECT_THAT(this->solve({XAndX}), sat(_));
 }
 
-TEST(SolverTest, PureVar) {
+TYPED_TEST_P(SolverTest, PureVar) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto Y = Ctx.atom();
@@ -175,12 +185,12 @@ TEST(SolverTest, PureVar) {
   auto NotXOrNotY = Ctx.disj(NotX, NotY);
 
   // (!X v Y) ^ (!X v !Y)
-  EXPECT_THAT(solve({NotXOrY, NotXOrNotY}),
+  EXPECT_THAT(this->solve({NotXOrY, NotXOrNotY}),
               sat(UnorderedElementsAre(Pair(X->getAtom(), AssignedFalse),
                                        Pair(Y->getAtom(), _))));
 }
 
-TEST(SolverTest, MustAssumeVarIsFalse) {
+TYPED_TEST_P(SolverTest, MustAssumeVarIsFalse) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto Y = Ctx.atom();
@@ -191,12 +201,12 @@ TEST(SolverTest, MustAssumeVarIsFalse) {
   auto NotXOrNotY = Ctx.disj(NotX, NotY);
 
   // (X v Y) ^ (!X v Y) ^ (!X v !Y)
-  EXPECT_THAT(solve({XOrY, NotXOrY, NotXOrNotY}),
+  EXPECT_THAT(this->solve({XOrY, NotXOrY, NotXOrNotY}),
               sat(UnorderedElementsAre(Pair(X->getAtom(), AssignedFalse),
                                        Pair(Y->getAtom(), AssignedTrue))));
 }
 
-TEST(SolverTest, DeepConflict) {
+TYPED_TEST_P(SolverTest, DeepConflict) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto Y = Ctx.atom();
@@ -208,10 +218,10 @@ TEST(SolverTest, DeepConflict) {
   auto XOrNotY = Ctx.disj(X, NotY);
 
   // (X v Y) ^ (!X v Y) ^ (!X v !Y) ^ (X v !Y)
-  EXPECT_THAT(solve({XOrY, NotXOrY, NotXOrNotY, XOrNotY}), unsat());
+  EXPECT_THAT(this->solve({XOrY, NotXOrY, NotXOrNotY, XOrNotY}), unsat());
 }
 
-TEST(SolverTest, IffIsEquivalentToDNF) {
+TYPED_TEST_P(SolverTest, IffIsEquivalentToDNF) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto Y = Ctx.atom();
@@ -222,19 +232,19 @@ TEST(SolverTest, IffIsEquivalentToDNF) {
   auto NotEquivalent = Ctx.neg(Ctx.iff(XIffY, XIffYDNF));
 
   // !((X <=> Y) <=> ((X ^ Y) v (!X ^ !Y)))
-  EXPECT_THAT(solve({NotEquivalent}), unsat());
+  EXPECT_THAT(this->solve({NotEquivalent}), unsat());
 }
 
-TEST(SolverTest, IffSameVars) {
+TYPED_TEST_P(SolverTest, IffSameVars) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto XEqX = Ctx.iff(X, X);
 
   // X <=> X
-  EXPECT_THAT(solve({XEqX}), sat(_));
+  EXPECT_THAT(this->solve({XEqX}), sat(_));
 }
 
-TEST(SolverTest, IffDistinctVars) {
+TYPED_TEST_P(SolverTest, IffDistinctVars) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto Y = Ctx.atom();
@@ -242,36 +252,36 @@ TEST(SolverTest, IffDistinctVars) {
 
   // X <=> Y
   EXPECT_THAT(
-      solve({XEqY}),
+      this->solve({XEqY}),
       sat(AnyOf(UnorderedElementsAre(Pair(X->getAtom(), AssignedTrue),
                                      Pair(Y->getAtom(), AssignedTrue)),
                 UnorderedElementsAre(Pair(X->getAtom(), AssignedFalse),
                                      Pair(Y->getAtom(), AssignedFalse)))));
 }
 
-TEST(SolverTest, IffWithUnits) {
+TYPED_TEST_P(SolverTest, IffWithUnits) {
   ConstraintContext Ctx;
   auto X = Ctx.atom();
   auto Y = Ctx.atom();
   auto XEqY = Ctx.iff(X, Y);
 
   // (X <=> Y) ^ X ^ Y
-  EXPECT_THAT(solve({XEqY, X, Y}),
+  EXPECT_THAT(this->solve({XEqY, X, Y}),
               sat(UnorderedElementsAre(Pair(X->getAtom(), AssignedTrue),
                                        Pair(Y->getAtom(), AssignedTrue))));
 }
 
-TEST(SolverTest, IffWithUnitsConflict) {
+TYPED_TEST_P(SolverTest, IffWithUnitsConflict) {
   Arena A;
   auto Constraints = parseFormulas(A, R"(
      (V0 = V1)
      V0
      !V1
   )");
-  EXPECT_THAT(solve(Constraints), unsat());
+  EXPECT_THAT(this->solve(Constraints), unsat());
 }
 
-TEST(SolverTest, IffTransitiveConflict) {
+TYPED_TEST_P(SolverTest, IffTransitiveConflict) {
   Arena A;
   auto Constraints = parseFormulas(A, R"(
      (V0 = V1)
@@ -279,63 +289,63 @@ TEST(SolverTest, IffTransitiveConflict) {
      V2
      !V0
   )");
-  EXPECT_THAT(solve(Constraints), unsat());
+  EXPECT_THAT(this->solve(Constraints), unsat());
 }
 
-TEST(SolverTest, DeMorgan) {
+TYPED_TEST_P(SolverTest, DeMorgan) {
   Arena A;
   auto Constraints = parseFormulas(A, R"(
      (!(V0 | V1) = (!V0 & !V1))
      (!(V2 & V3) = (!V2 | !V3))
   )");
-  EXPECT_THAT(solve(Constraints), sat(_));
+  EXPECT_THAT(this->solve(Constraints), sat(_));
 }
 
-TEST(SolverTest, RespectsAdditionalConstraints) {
+TYPED_TEST_P(SolverTest, RespectsAdditionalConstraints) {
   Arena A;
   auto Constraints = parseFormulas(A, R"(
      (V0 = V1)
      V0
      !V1
   )");
-  EXPECT_THAT(solve(Constraints), unsat());
+  EXPECT_THAT(this->solve(Constraints), unsat());
 }
 
-TEST(SolverTest, ImplicationIsEquivalentToDNF) {
+TYPED_TEST_P(SolverTest, ImplicationIsEquivalentToDNF) {
   Arena A;
   auto Constraints = parseFormulas(A, R"(
      !((V0 => V1) = (!V0 | V1))
   )");
-  EXPECT_THAT(solve(Constraints), unsat());
+  EXPECT_THAT(this->solve(Constraints), unsat());
 }
 
-TEST(SolverTest, ImplicationConflict) {
+TYPED_TEST_P(SolverTest, ImplicationConflict) {
   Arena A;
   auto Constraints = parseFormulas(A, R"(
      (V0 => V1)
      (V0 & !V1)
   )");
-  EXPECT_THAT(solve(Constraints), unsat());
+  EXPECT_THAT(this->solve(Constraints), unsat());
 }
 
-TEST(SolverTest, ReachedLimitsReflectsTimeouts) {
+TYPED_TEST_P(SolverTest, ReachedLimitsReflectsTimeouts) {
   Arena A;
   auto Constraints = parseFormulas(A, R"(
      (!(V0 | V1) = (!V0 & !V1))
      (!(V2 & V3) = (!V2 & !V3))
   )");
-  WatchedLiteralsSolver solver(10);
+  TypeParam solver = this->createSolverWithLowTimeout();
   ASSERT_EQ(solver.solve(Constraints).getStatus(),
             Solver::Result::Status::TimedOut);
   EXPECT_TRUE(solver.reachedLimit());
 }
 
-TEST(SolverTest, SimpleButLargeContradiction) {
+TYPED_TEST_P(SolverTest, SimpleButLargeContradiction) {
   // This test ensures that the solver takes a short-cut on known
   // contradictory inputs, without using max_iterations. At the time
   // this test is added, formulas that are easily recognized to be
   // contradictory at CNF construction time would lead to timeout.
-  WatchedLiteralsSolver solver(10);
+  TypeParam solver = this->createSolverWithLowTimeout();
   ConstraintContext Ctx;
   auto first = Ctx.atom();
   auto last = first;
@@ -358,4 +368,16 @@ TEST(SolverTest, SimpleButLargeContradiction) {
   EXPECT_FALSE(solver.reachedLimit());
 }
 
-} // namespace
+REGISTER_TYPED_TEST_SUITE_P(
+    SolverTest, Var, NegatedVar, UnitConflict, DistinctVars, DoubleNegation,
+    NegatedDisjunction, NegatedConjunction, DisjunctionSameVarWithNegation,
+    DisjunctionSameVar, ConjunctionSameVarsConflict, ConjunctionSameVar,
+    PureVar, MustAssumeVarIsFalse, DeepConflict, IffIsEquivalentToDNF,
+    IffSameVars, IffDistinctVars, IffWithUnits, IffWithUnitsConflict,
+    IffTransitiveConflict, DeMorgan, RespectsAdditionalConstraints,
+    ImplicationIsEquivalentToDNF, ImplicationConflict,
+    ReachedLimitsReflectsTimeouts, SimpleButLargeContradiction);
+
+} // namespace clang::dataflow::test
+
+#endif // LLVM_CLANG_ANALYSIS_FLOW_SENSITIVE_TESTING_SUPPORT_H_
diff --git a/clang/unittests/Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp b/clang/unittests/Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp
new file mode 100644
index 000000000000..0a2514a2d7c1
--- /dev/null
+++ b/clang/unittests/Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp
@@ -0,0 +1,26 @@
+//===- unittests/Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Analysis/FlowSensitive/WatchedLiteralsSolver.h"
+#include "SolverTest.h"
+
+namespace clang::dataflow::test {
+
+template <>
+WatchedLiteralsSolver
+SolverTest<WatchedLiteralsSolver>::createSolverWithLowTimeout() {
+  return WatchedLiteralsSolver(10);
+}
+
+namespace {
+
+INSTANTIATE_TYPED_TEST_SUITE_P(WatchedLiteralsSolverTest, SolverTest,
+                               WatchedLiteralsSolver);
+
+} // namespace
+} // namespace clang::dataflow::test
diff --git a/llvm/utils/gn/secondary/clang/unittests/Analysis/FlowSensitive/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Analysis/FlowSensitive/BUILD.gn
index df5b4587bf1c..e16ca31b81a8 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Analysis/FlowSensitive/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Analysis/FlowSensitive/BUILD.gn
@@ -33,7 +33,6 @@ unittest("ClangAnalysisFlowSensitiveTests") {
     "SignAnalysisTest.cpp",
     "SimplifyConstraintsTest.cpp",
     "SingleVarConstantPropagationTest.cpp",
-    "SolverTest.cpp",
     "TestingSupport.cpp",
     "TestingSupportTest.cpp",
     "TransferBranchTest.cpp",
@@ -41,5 +40,6 @@ unittest("ClangAnalysisFlowSensitiveTests") {
     "TypeErasedDataflowAnalysisTest.cpp",
     "UncheckedOptionalAccessModelTest.cpp",
     "ValueTest.cpp",
+    "WatchedLiteralsSolverTest.cpp",
   ]
 }
-- 
GitLab


From b5afda8d760998641cf08a6d229252924b0ad146 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 8 May 2024 15:46:11 +0200
Subject: [PATCH 0176/1206] AMDGPU: Add some more ctlz_zero_undef tests

---
 llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll | 473 ++++++++++++++++++++
 1 file changed, 473 insertions(+)

diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
index 21aff62b9226..54adde38d6d2 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
@@ -4,9 +4,17 @@
 ; RUN: llc -mtriple=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG %s
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-GISEL %s
 
+declare i7 @llvm.ctlz.i7(i7, i1) nounwind readnone
+declare <2 x i7> @llvm.ctlz.v2i7(<2 x i7>, i1) nounwind readnone
 declare i8 @llvm.ctlz.i8(i8, i1) nounwind readnone
+declare <2 x i8> @llvm.ctlz.v2i8(<2 x i8>, i1) nounwind readnone
 
 declare i16 @llvm.ctlz.i16(i16, i1) nounwind readnone
+declare i18 @llvm.ctlz.i18(i18, i1) nounwind readnone
+
+declare <2 x i16> @llvm.ctlz.v2i16(<2 x i16>, i1) nounwind readnone
+declare <3 x i16> @llvm.ctlz.v3i16(<3 x i16>, i1) nounwind readnone
+declare <4 x i16> @llvm.ctlz.v4i16(<4 x i16>, i1) nounwind readnone
 
 declare i32 @llvm.ctlz.i32(i32, i1) nounwind readnone
 declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1) nounwind readnone
@@ -2158,3 +2166,468 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i32_sel_ne_cmp_non0(ptr addrspace(1
   store i32 %sel, ptr addrspace(1) %out
   ret void
 }
+
+define i7 @v_ctlz_zero_undef_i7(i7 %val) {
+; SI-LABEL: v_ctlz_zero_undef_i7:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 0x7f, v0
+; SI-NEXT:    v_ffbh_u32_e32 v0, v0
+; SI-NEXT:    v_subrev_i32_e32 v0, vcc, 25, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_ctlz_zero_undef_i7:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 0x7f, v0
+; VI-NEXT:    v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, -16, v0
+; VI-NEXT:    v_add_u16_e32 v0, -9, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; EG-LABEL: v_ctlz_zero_undef_i7:
+; EG:       ; %bb.0:
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i7:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_and_b32_e32 v0, 0x7f, v0
+; GFX9-GISEL-NEXT:    v_ffbh_u32_e32 v0, v0
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v0, 25, v0
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %ctlz = call i7 @llvm.ctlz.i7(i7 %val, i1 true)
+  ret i7 %ctlz
+}
+
+define amdgpu_kernel void @s_ctlz_zero_undef_i18(ptr addrspace(1) noalias %out, i18 %val) nounwind {
+; SI-LABEL: s_ctlz_zero_undef_i18:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s2, s[0:1], 0xb
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_and_b32 s2, s2, 0x3ffff
+; SI-NEXT:    s_flbit_i32_b32 s2, s2
+; SI-NEXT:    s_add_i32 s4, s2, -14
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    s_bfe_u32 s4, s4, 0x20010
+; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:2
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_ctlz_zero_undef_i18:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s2, s2, 0x3ffff
+; VI-NEXT:    s_flbit_i32_b32 s2, s2
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    s_add_i32 s2, s2, -14
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    s_add_u32 s0, s0, 2
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_addc_u32 s1, s1, 0
+; VI-NEXT:    s_bfe_u32 s2, s2, 0x20010
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    flat_store_byte v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; EG-LABEL: s_ctlz_zero_undef_i18:
+; EG:       ; %bb.0:
+; EG-NEXT:    ALU 30, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T1.XW, T3.X
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T2.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    ALU clause starting at 4:
+; EG-NEXT:     AND_INT * T0.W, KC0[2].Z, literal.x,
+; EG-NEXT:    262143(3.673406e-40), 0(0.000000e+00)
+; EG-NEXT:     FFBH_UINT T0.W, PV.W,
+; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     ADD_INT * T0.W, PV.W, literal.x,
+; EG-NEXT:    -14(nan), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T2.W, PV.W, literal.x,
+; EG-NEXT:     LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT:     LSHL T1.X, PV.W, PS,
+; EG-NEXT:     LSHL * T1.W, literal.x, PS,
+; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT:     MOV T1.Y, 0.0,
+; EG-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T3.W, PV.W, literal.x,
+; EG-NEXT:     MOV * T4.W, literal.y,
+; EG-NEXT:    3(4.203895e-45), 2(2.802597e-45)
+; EG-NEXT:     BFE_UINT T0.W, T0.W, literal.x, PS,
+; EG-NEXT:     LSHL * T3.W, PV.W, literal.y,
+; EG-NEXT:    16(2.242078e-44), 3(4.203895e-45)
+; EG-NEXT:     LSHL T0.X, PV.W, PS,
+; EG-NEXT:     LSHL * T0.W, literal.x, PS,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV T1.Z, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR T2.X, T2.W, literal.x,
+; EG-NEXT:     LSHR * T3.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i18:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GFX9-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    s_and_b32 s0, s4, 0x3ffff
+; GFX9-GISEL-NEXT:    s_flbit_i32_b32 s0, s0
+; GFX9-GISEL-NEXT:    s_sub_i32 s0, s0, 14
+; GFX9-GISEL-NEXT:    s_and_b32 s0, s0, 0x3ffff
+; GFX9-GISEL-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-GISEL-NEXT:    global_store_short v0, v1, s[2:3]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-GISEL-NEXT:    global_store_byte v0, v1, s[2:3] offset:2
+; GFX9-GISEL-NEXT:    s_endpgm
+  %ctlz = call i18 @llvm.ctlz.i18(i18 %val, i1 true) nounwind readnone
+  store i18 %ctlz, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define i18 @v_ctlz_zero_undef_i18(i18 %val) {
+; SI-LABEL: v_ctlz_zero_undef_i18:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 0x3ffff, v0
+; SI-NEXT:    v_ffbh_u32_e32 v0, v0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -14, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_ctlz_zero_undef_i18:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v0, 0x3ffff, v0
+; VI-NEXT:    v_ffbh_u32_e32 v0, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, -14, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; EG-LABEL: v_ctlz_zero_undef_i18:
+; EG:       ; %bb.0:
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i18:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ffff, v0
+; GFX9-GISEL-NEXT:    v_ffbh_u32_e32 v0, v0
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v0, 14, v0
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %ctlz = call i18 @llvm.ctlz.i18(i18 %val, i1 true)
+  ret i18 %ctlz
+}
+
+define <2 x i18> @v_ctlz_zero_undef_v2i18(<2 x i18> %val) {
+; SI-LABEL: v_ctlz_zero_undef_v2i18:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 0x3ffff, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0x3ffff, v0
+; SI-NEXT:    v_ffbh_u32_e32 v0, v0
+; SI-NEXT:    v_ffbh_u32_e32 v1, v1
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -14, v0
+; SI-NEXT:    v_add_i32_e32 v1, vcc, -14, v1
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_ctlz_zero_undef_v2i18:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v1, 0x3ffff, v1
+; VI-NEXT:    v_and_b32_e32 v0, 0x3ffff, v0
+; VI-NEXT:    v_ffbh_u32_e32 v0, v0
+; VI-NEXT:    v_ffbh_u32_e32 v1, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, -14, v0
+; VI-NEXT:    v_add_u32_e32 v1, vcc, -14, v1
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; EG-LABEL: v_ctlz_zero_undef_v2i18:
+; EG:       ; %bb.0:
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v2i18:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ffff, v0
+; GFX9-GISEL-NEXT:    v_and_b32_e32 v1, 0x3ffff, v1
+; GFX9-GISEL-NEXT:    v_ffbh_u32_e32 v0, v0
+; GFX9-GISEL-NEXT:    v_ffbh_u32_e32 v1, v1
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v0, 14, v0
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v1, 14, v1
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %ctlz = call <2 x i18> @llvm.ctlz.v2i18(<2 x i18> %val, i1 true)
+  ret <2 x i18> %ctlz
+}
+
+define <2 x i16> @v_ctlz_zero_undef_v2i16(<2 x i16> %val) {
+; SI-LABEL: v_ctlz_zero_undef_v2i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_ffbh_u32_e32 v1, v1
+; SI-NEXT:    v_ffbh_u32_e32 v0, v0
+; SI-NEXT:    v_add_i32_e32 v1, vcc, -16, v1
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -16, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_or_b32_e32 v0, v0, v2
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_ctlz_zero_undef_v2i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_ffbh_u32_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-NEXT:    v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, -16, v0
+; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 0xfff00000, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; EG-LABEL: v_ctlz_zero_undef_v2i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v2i16:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_ffbh_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT:    v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX9-GISEL-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %ctlz = call <2 x i16> @llvm.ctlz.v2i16(<2 x i16> %val, i1 true)
+  ret <2 x i16> %ctlz
+}
+
+define <3 x i16> @v_ctlz_zero_undef_v3i16(<3 x i16> %val) {
+; SI-LABEL: v_ctlz_zero_undef_v3i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    v_ffbh_u32_e32 v1, v1
+; SI-NEXT:    v_ffbh_u32_e32 v0, v0
+; SI-NEXT:    v_ffbh_u32_e32 v2, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -16, v0
+; SI-NEXT:    v_add_i32_e32 v3, vcc, -16, v2
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v3
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 0xfff00000, v0
+; SI-NEXT:    v_or_b32_e32 v2, 0x100000, v2
+; SI-NEXT:    v_alignbit_b32 v1, v3, v0, 16
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_ctlz_zero_undef_v3i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_ffbh_u32_sdwa v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-NEXT:    v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, -16, v0
+; VI-NEXT:    v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-NEXT:    v_add_u32_e32 v1, vcc, -16, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 0xfff00000, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; EG-LABEL: v_ctlz_zero_undef_v3i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v3i16:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_ffbh_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT:    v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT:    v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-GISEL-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT:    v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %ctlz = call <3 x i16> @llvm.ctlz.v3i16(<3 x i16> %val, i1 true)
+  ret <3 x i16> %ctlz
+}
+
+define <4 x i16> @v_ctlz_zero_undef_v4i16(<4 x i16> %val) {
+; SI-LABEL: v_ctlz_zero_undef_v4i16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_ffbh_u32_e32 v3, v3
+; SI-NEXT:    v_ffbh_u32_e32 v2, v2
+; SI-NEXT:    v_ffbh_u32_e32 v1, v1
+; SI-NEXT:    v_ffbh_u32_e32 v0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_add_i32_e32 v2, vcc, -16, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -16, v0
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_or_b32_e32 v2, v3, v2
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 0xfff00000, v2
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 0xfff00000, v0
+; SI-NEXT:    v_alignbit_b32 v1, v2, v0, 16
+; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_ctlz_zero_undef_v4i16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_ffbh_u32_sdwa v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-NEXT:    v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT:    v_ffbh_u32_sdwa v3, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-NEXT:    v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; VI-NEXT:    v_add_u32_e32 v1, vcc, -16, v1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, -16, v0
+; VI-NEXT:    v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 0xfff00000, v0
+; VI-NEXT:    v_add_u32_e32 v1, vcc, 0xfff00000, v1
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; EG-LABEL: v_ctlz_zero_undef_v4i16:
+; EG:       ; %bb.0:
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v4i16:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_ffbh_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v2, 16, v2
+; GFX9-GISEL-NEXT:    v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-GISEL-NEXT:    v_ffbh_u32_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v0, 16, v0
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v3, 16, v3
+; GFX9-GISEL-NEXT:    v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-GISEL-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT:    v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-GISEL-NEXT:    v_and_b32_e32 v2, 0xffff, v3
+; GFX9-GISEL-NEXT:    v_lshl_or_b32 v1, v1, 16, v2
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %ctlz = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %val, i1 true)
+  ret <4 x i16> %ctlz
+}
+
+define <2 x i8> @v_ctlz_zero_undef_v2i8(<2 x i8> %val) {
+; SI-LABEL: v_ctlz_zero_undef_v2i8:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    v_ffbh_u32_e32 v1, v1
+; SI-NEXT:    v_ffbh_u32_e32 v0, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT:    v_subrev_i32_e32 v0, vcc, 24, v0
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    v_or_b32_e32 v0, v1, v0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 0xffffe800, v0
+; SI-NEXT:    v_bfe_u32 v1, v0, 8, 8
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_ctlz_zero_undef_v2i8:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_ffbh_u32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; VI-NEXT:    v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; VI-NEXT:    v_add_u16_e32 v1, 0xe800, v1
+; VI-NEXT:    v_subrev_u16_e32 v0, 24, v0
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT:    v_lshrrev_b16_e32 v1, 8, v1
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; EG-LABEL: v_ctlz_zero_undef_v2i8:
+; EG:       ; %bb.0:
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v2i8:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX9-GISEL-NEXT:    v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v0, 24, v0
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v1, 24, v1
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %ctlz = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %val, i1 true)
+  ret <2 x i8> %ctlz
+}
+
+define <2 x i7> @v_ctlz_zero_undef_v2i7(<2 x i7> %val) {
+; SI-LABEL: v_ctlz_zero_undef_v2i7:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 0x7f, v1
+; SI-NEXT:    v_and_b32_e32 v0, 0x7f, v0
+; SI-NEXT:    v_ffbh_u32_e32 v0, v0
+; SI-NEXT:    v_ffbh_u32_e32 v1, v1
+; SI-NEXT:    v_subrev_i32_e32 v0, vcc, 25, v0
+; SI-NEXT:    v_subrev_i32_e32 v1, vcc, 25, v1
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_ctlz_zero_undef_v2i7:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_and_b32_e32 v2, 0x7f007f, v0
+; VI-NEXT:    v_bfe_u32 v0, v0, 16, 7
+; VI-NEXT:    v_ffbh_u32_e32 v0, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, -16, v0
+; VI-NEXT:    v_add_u16_e32 v1, -9, v0
+; VI-NEXT:    v_and_b32_e32 v0, 0x7f, v2
+; VI-NEXT:    v_ffbh_u32_e32 v0, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, -16, v0
+; VI-NEXT:    v_add_u16_e32 v0, -9, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; EG-LABEL: v_ctlz_zero_undef_v2i7:
+; EG:       ; %bb.0:
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+;
+; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v2i7:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_and_b32_e32 v0, 0x7f, v0
+; GFX9-GISEL-NEXT:    v_and_b32_e32 v1, 0x7f, v1
+; GFX9-GISEL-NEXT:    v_ffbh_u32_e32 v0, v0
+; GFX9-GISEL-NEXT:    v_ffbh_u32_e32 v1, v1
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v0, 25, v0
+; GFX9-GISEL-NEXT:    v_subrev_u32_e32 v1, 25, v1
+; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %ctlz = call <2 x i7> @llvm.ctlz.v2i7(<2 x i7> %val, i1 true)
+  ret <2 x i7> %ctlz
+}
-- 
GitLab


From 27a062e9ca7c92e89ed4084c3c3affb9fa39aabb Mon Sep 17 00:00:00 2001
From: serge-sans-paille <sguelton@mozilla.com>
Date: Wed, 8 May 2024 14:21:31 +0000
Subject: [PATCH 0177/1206] [libc++] Implement std::gcd using the binary
 version (#77747)

The binary version is four times faster than current implementation in
my setup, and generally considered a better implementation.

Code inspired by https://en.algorithmica.org/hpc/algorithms/gcd/ which
itself is inspired by
https://lemire.me/blog/2013/12/26/fastest-way-to-compute-the-greatest-common-divisor/

Fix #77648
---
 libcxx/benchmarks/CMakeLists.txt              |  3 +-
 libcxx/benchmarks/numeric/gcd.bench.cpp       | 53 +++++++++++
 libcxx/include/__numeric/gcd_lcm.h            | 44 ++++++++-
 .../test/libcxx/transitive_includes/cxx03.csv |  1 +
 .../test/libcxx/transitive_includes/cxx11.csv |  1 +
 .../test/libcxx/transitive_includes/cxx14.csv |  1 +
 .../test/libcxx/transitive_includes/cxx17.csv |  1 +
 .../test/libcxx/transitive_includes/cxx20.csv |  1 +
 .../test/libcxx/transitive_includes/cxx26.csv | 23 +++++
 .../numeric.ops/numeric.ops.gcd/gcd.pass.cpp  | 89 ++++++++++++++++++-
 10 files changed, 213 insertions(+), 4 deletions(-)
 create mode 100644 libcxx/benchmarks/numeric/gcd.bench.cpp

diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt
index 5dc3be0c367e..93b549a316e3 100644
--- a/libcxx/benchmarks/CMakeLists.txt
+++ b/libcxx/benchmarks/CMakeLists.txt
@@ -122,7 +122,7 @@ endif()
 add_library(           cxx-benchmarks-flags-libcxx INTERFACE)
 target_link_libraries( cxx-benchmarks-flags-libcxx INTERFACE cxx-benchmarks-flags)
 target_compile_options(cxx-benchmarks-flags-libcxx INTERFACE ${SANITIZER_FLAGS} -Wno-user-defined-literals -Wno-suggest-override)
-target_link_options(   cxx-benchmarks-flags-libcxx INTERFACE -nostdlib++ "-L${BENCHMARK_LIBCXX_INSTALL}/lib" "-L${BENCHMARK_LIBCXX_INSTALL}/lib64" ${SANITIZER_FLAGS})
+target_link_options(   cxx-benchmarks-flags-libcxx INTERFACE -lm -nostdlib++ "-L${BENCHMARK_LIBCXX_INSTALL}/lib" "-L${BENCHMARK_LIBCXX_INSTALL}/lib64" ${SANITIZER_FLAGS})
 
 set(libcxx_benchmark_targets)
 
@@ -220,6 +220,7 @@ set(BENCHMARK_TESTS
     lexicographical_compare_three_way.bench.cpp
     map.bench.cpp
     monotonic_buffer.bench.cpp
+    numeric/gcd.bench.cpp
     ordered_set.bench.cpp
     shared_mutex_vs_mutex.bench.cpp
     stop_token.bench.cpp
diff --git a/libcxx/benchmarks/numeric/gcd.bench.cpp b/libcxx/benchmarks/numeric/gcd.bench.cpp
new file mode 100644
index 000000000000..f8b6a856cd0d
--- /dev/null
+++ b/libcxx/benchmarks/numeric/gcd.bench.cpp
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <array>
+#include <benchmark/benchmark.h>
+#include <cstring>
+#include <numeric>
+#include <random>
+
+template <class T>
+static std::array<T, 1000> generate(std::uniform_int_distribution<T> distribution = std::uniform_int_distribution<T>{
+                                        std::numeric_limits<T>::min() + 1, std::numeric_limits<T>::max()}) {
+  std::mt19937 generator;
+  std::array<T, 1000> result;
+  std::generate_n(result.begin(), result.size(), [&] { return distribution(generator); });
+  return result;
+}
+
+static void bm_gcd_random(benchmark::State& state) {
+  std::array data = generate<int>();
+  while (state.KeepRunningBatch(data.size()))
+    for (auto v0 : data)
+      for (auto v1 : data)
+        benchmark::DoNotOptimize(std::gcd(v0, v1));
+}
+BENCHMARK(bm_gcd_random);
+
+static void bm_gcd_trivial(benchmark::State& state) {
+  int lhs = ~static_cast<int>(0), rhs = 1;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(lhs);
+    benchmark::DoNotOptimize(rhs);
+    benchmark::DoNotOptimize(std::gcd(lhs, rhs));
+  }
+}
+BENCHMARK(bm_gcd_trivial);
+
+static void bm_gcd_complex(benchmark::State& state) {
+  int lhs = 2971215073, rhs = 1836311903;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(lhs);
+    benchmark::DoNotOptimize(rhs);
+    benchmark::DoNotOptimize(std::gcd(lhs, rhs));
+  }
+}
+BENCHMARK(bm_gcd_complex);
+
+BENCHMARK_MAIN();
diff --git a/libcxx/include/__numeric/gcd_lcm.h b/libcxx/include/__numeric/gcd_lcm.h
index 48df2338051e..5d735a51a47e 100644
--- a/libcxx/include/__numeric/gcd_lcm.h
+++ b/libcxx/include/__numeric/gcd_lcm.h
@@ -10,7 +10,9 @@
 #ifndef _LIBCPP___NUMERIC_GCD_LCM_H
 #define _LIBCPP___NUMERIC_GCD_LCM_H
 
+#include <__algorithm/min.h>
 #include <__assert>
+#include <__bit/countr.h>
 #include <__config>
 #include <__type_traits/common_type.h>
 #include <__type_traits/is_integral.h>
@@ -50,9 +52,47 @@ struct __ct_abs<_Result, _Source, false> {
 };
 
 template <class _Tp>
-_LIBCPP_CONSTEXPR _LIBCPP_HIDDEN _Tp __gcd(_Tp __m, _Tp __n) {
+_LIBCPP_CONSTEXPR _LIBCPP_HIDDEN _Tp __gcd(_Tp __a, _Tp __b) {
   static_assert((!is_signed<_Tp>::value), "");
-  return __n == 0 ? __m : std::__gcd<_Tp>(__n, __m % __n);
+
+  // From: https://lemire.me/blog/2013/12/26/fastest-way-to-compute-the-greatest-common-divisor
+  //
+  // If power of two divides both numbers, we can push it out.
+  // - gcd( 2^x * a, 2^x * b) = 2^x * gcd(a, b)
+  //
+  // If and only if exactly one number is even, we can divide that number by that power.
+  // - if a, b are odd, then gcd(2^x * a, b) = gcd(a, b)
+  //
+  // And standard gcd algorithm where instead of modulo, minus is used.
+
+  if (__a < __b) {
+    _Tp __tmp = __b;
+    __b       = __a;
+    __a       = __tmp;
+  }
+  if (__b == 0)
+    return __a;
+  __a %= __b; // Make both argument of the same size, and early result in the easy case.
+  if (__a == 0)
+    return __b;
+
+  int __az    = std::__countr_zero(__a);
+  int __bz    = std::__countr_zero(__b);
+  int __shift = std::min(__az, __bz);
+  __a >>= __az;
+  __b >>= __bz;
+  do {
+    _Tp __diff = __a - __b;
+    if (__a > __b) {
+      __a = __b;
+      __b = __diff;
+    } else {
+      __b = __b - __a;
+    }
+    if (__diff != 0)
+      __b >>= std::__countr_zero(__diff);
+  } while (__b != 0);
+  return __a << __shift;
 }
 
 template <class _Tp, class _Up>
diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv
index cf0af3b8bb39..92601fab5b77 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx03.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv
@@ -570,6 +570,7 @@ numeric cstddef
 numeric cstdint
 numeric execution
 numeric functional
+numeric initializer_list
 numeric iterator
 numeric limits
 numeric new
diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv
index f514ee3028f6..c05eb42deb9a 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx11.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv
@@ -575,6 +575,7 @@ numeric cstddef
 numeric cstdint
 numeric execution
 numeric functional
+numeric initializer_list
 numeric iterator
 numeric limits
 numeric new
diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv
index 43e3f996adba..09252b7b7d2d 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx14.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv
@@ -578,6 +578,7 @@ numeric cstddef
 numeric cstdint
 numeric execution
 numeric functional
+numeric initializer_list
 numeric iterator
 numeric limits
 numeric new
diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv
index 43e3f996adba..09252b7b7d2d 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx17.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv
@@ -578,6 +578,7 @@ numeric cstddef
 numeric cstdint
 numeric execution
 numeric functional
+numeric initializer_list
 numeric iterator
 numeric limits
 numeric new
diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv
index 8463f17db411..ce4ccc3d1161 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx20.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv
@@ -589,6 +589,7 @@ numeric cstddef
 numeric cstdint
 numeric execution
 numeric functional
+numeric initializer_list
 numeric iterator
 numeric limits
 numeric new
diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv
index 62d931c0eeba..f68249aeec78 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx26.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv
@@ -176,6 +176,29 @@ experimental/simd limits
 experimental/type_traits initializer_list
 experimental/type_traits type_traits
 experimental/utility utility
+experimental/vector experimental/memory_resource
+experimental/vector vector
+ext/hash_map algorithm
+ext/hash_map cmath
+ext/hash_map cstddef
+ext/hash_map cstdint
+ext/hash_map cstring
+ext/hash_map functional
+ext/hash_map initializer_list
+ext/hash_map limits
+ext/hash_map new
+ext/hash_map stdexcept
+ext/hash_map string
+ext/hash_set algorithm
+ext/hash_set cmath
+ext/hash_set cstddef
+ext/hash_set cstdint
+ext/hash_set cstring
+ext/hash_set functional
+ext/hash_set initializer_list
+ext/hash_set limits
+ext/hash_set new
+ext/hash_set string
 filesystem compare
 filesystem cstddef
 filesystem cstdint
diff --git a/libcxx/test/std/numerics/numeric.ops/numeric.ops.gcd/gcd.pass.cpp b/libcxx/test/std/numerics/numeric.ops/numeric.ops.gcd/gcd.pass.cpp
index 831c226f9c8e..212804356a05 100644
--- a/libcxx/test/std/numerics/numeric.ops/numeric.ops.gcd/gcd.pass.cpp
+++ b/libcxx/test/std/numerics/numeric.ops/numeric.ops.gcd/gcd.pass.cpp
@@ -17,6 +17,7 @@
 #include <cassert>
 #include <climits>
 #include <cstdint>
+#include <random>
 #include <type_traits>
 
 #include "test_macros.h"
@@ -48,6 +49,74 @@ constexpr bool test0(int in1, int in2, int out)
     return true;
 }
 
+template <typename T>
+T basic_gcd_(T m, T n) {
+  return n == 0 ? m : basic_gcd_<T>(n, m % n);
+}
+
+template <typename T>
+T basic_gcd(T m, T n) {
+  using Tp = std::make_unsigned_t<T>;
+  if (m < 0 && m != std::numeric_limits<T>::min())
+    m = -m;
+  if (n < 0 && n != std::numeric_limits<T>::min())
+    n = -n;
+  return basic_gcd_(static_cast<Tp>(m), static_cast<Tp>(n));
+}
+
+template <typename Input>
+void do_fuzzy_tests() {
+  std::mt19937 gen(1938);
+  std::uniform_int_distribution<Input> distrib;
+
+  constexpr int nb_rounds = 10000;
+  for (int i = 0; i < nb_rounds; ++i) {
+    Input n = distrib(gen);
+    Input m = distrib(gen);
+    assert(std::gcd(n, m) == basic_gcd(n, m));
+  }
+}
+
+template <typename Input>
+void do_limit_tests() {
+  Input inputs[] = {
+      // The behavior of std::gcd is undefined if the absolute value of one of its
+      // operand is not representable in the result type.
+      std::numeric_limits<Input>::min() + (std::is_signed<Input>::value ? 3 : 0),
+      std::numeric_limits<Input>::min() + 1,
+      std::numeric_limits<Input>::min() + 2,
+      std::numeric_limits<Input>::max(),
+      std::numeric_limits<Input>::max() - 1,
+      std::numeric_limits<Input>::max() - 2,
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      (Input)-1,
+      (Input)-2,
+      (Input)-3,
+      (Input)-4,
+      (Input)-5,
+      (Input)-6,
+      (Input)-7,
+      (Input)-8,
+      (Input)-9,
+      (Input)-10,
+  };
+
+  for (auto n : inputs) {
+    for (auto m : inputs) {
+      assert(std::gcd(n, m) == basic_gcd(n, m));
+    }
+  }
+}
 
 template <typename Input1, typename Input2 = Input1>
 constexpr bool do_test(int = 0)
@@ -143,5 +212,23 @@ int main(int argc, char**)
     assert(res == 2);
     }
 
-  return 0;
+    do_fuzzy_tests<std::int8_t>();
+    do_fuzzy_tests<std::int16_t>();
+    do_fuzzy_tests<std::int32_t>();
+    do_fuzzy_tests<std::int64_t>();
+    do_fuzzy_tests<std::uint8_t>();
+    do_fuzzy_tests<std::uint16_t>();
+    do_fuzzy_tests<std::uint32_t>();
+    do_fuzzy_tests<std::uint64_t>();
+
+    do_limit_tests<std::int8_t>();
+    do_limit_tests<std::int16_t>();
+    do_limit_tests<std::int32_t>();
+    do_limit_tests<std::int64_t>();
+    do_limit_tests<std::uint8_t>();
+    do_limit_tests<std::uint16_t>();
+    do_limit_tests<std::uint32_t>();
+    do_limit_tests<std::uint64_t>();
+
+    return 0;
 }
-- 
GitLab


From 73bb8d9d92f689863c94d48517e89d35dae0ebcf Mon Sep 17 00:00:00 2001
From: Jonathan Peyton <jonathan.l.peyton@intel.com>
Date: Wed, 8 May 2024 09:23:50 -0500
Subject: [PATCH 0178/1206] [OpenMP] Fix child processes to use affinity_none
 (#91391)

When a child process is forked with OpenMP already initialized, the
child process resets its affinity mask and sets proc-bind-var to false
so that the entire original affinity mask is used. This patch corrects
an issue with the affinity initialization code setting affinity to
compact instead of none for this special case of forked children.

The test trying to catch this only testing explicit setting of
KMP_AFFINITY=none. Add test run for no KMP_AFFINITY setting.

Fixes: #91098
---
 openmp/runtime/src/kmp_settings.cpp     | 2 ++
 openmp/runtime/test/affinity/redetect.c | 1 +
 2 files changed, 3 insertions(+)

diff --git a/openmp/runtime/src/kmp_settings.cpp b/openmp/runtime/src/kmp_settings.cpp
index b9c8289b5c51..8b6092cb1085 100644
--- a/openmp/runtime/src/kmp_settings.cpp
+++ b/openmp/runtime/src/kmp_settings.cpp
@@ -6420,6 +6420,8 @@ void __kmp_env_initialize(char const *string) {
         }
         if ((__kmp_nested_proc_bind.bind_types[0] != proc_bind_intel) &&
             (__kmp_nested_proc_bind.bind_types[0] != proc_bind_default)) {
+          if (__kmp_nested_proc_bind.bind_types[0] == proc_bind_false)
+            __kmp_affinity.type = affinity_none;
           if (__kmp_affinity.type == affinity_default) {
             __kmp_affinity.type = affinity_compact;
             __kmp_affinity.flags.dups = FALSE;
diff --git a/openmp/runtime/test/affinity/redetect.c b/openmp/runtime/test/affinity/redetect.c
index dba83b72cc42..4b96d1bd92ee 100644
--- a/openmp/runtime/test/affinity/redetect.c
+++ b/openmp/runtime/test/affinity/redetect.c
@@ -1,4 +1,5 @@
 // RUN: %libomp-compile
+// RUN: %libomp-run
 // RUN: env KMP_AFFINITY=none %libomp-run
 // REQUIRES: linux
 
-- 
GitLab


From 6ed8434edc5934210a38be99f33b6baed83df85c Mon Sep 17 00:00:00 2001
From: Prathamesh Tagore <63031630+meshtag@users.noreply.github.com>
Date: Wed, 8 May 2024 19:54:43 +0530
Subject: [PATCH 0179/1206] [mlir][fold-memref-alias-ops] Add support for
 folding memref.expand_shape involving dynamic dims (#89093)

`fold-memref-alias-ops` bails out in presence of dynamic shapes in
`memref.expand_shape` op. Handle this case.
---
 .../mlir/Dialect/MemRef/Utils/MemRefUtils.h   | 29 +++++++
 .../MemRef/Transforms/FoldMemRefAliasOps.cpp  | 85 ++++++++++++++-----
 mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp | 23 +++++
 .../Dialect/MemRef/fold-memref-alias-ops.mlir | 81 +++++++++++++-----
 4 files changed, 180 insertions(+), 38 deletions(-)

diff --git a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
index 7d9a5e6ca759..46003ed84686 100644
--- a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
+++ b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
@@ -64,6 +64,35 @@ getLinearizedMemRefOffsetAndSize(OpBuilder &builder, Location loc, int srcBits,
 // it means both the allocations and associated stores can be removed.
 void eraseDeadAllocAndStores(RewriterBase &rewriter, Operation *parentOp);
 
+/// Given a set of sizes, return the suffix product.
+///
+/// When applied to slicing, this is the calculation needed to derive the
+/// strides (i.e. the number of linear indices to skip along the (k-1) most
+/// minor dimensions to get the next k-slice).
+///
+/// This is the basis to linearize an n-D offset confined to `[0 ... sizes]`.
+///
+/// Assuming `sizes` is `[s0, .. sn]`, return the vector<Value>
+///   `[s1 * ... * sn, s2 * ... * sn, ..., sn, 1]`.
+///
+/// It is the caller's responsibility to provide valid OpFoldResult type values
+/// and construct valid IR in the end.
+///
+/// `sizes` elements are asserted to be non-negative.
+///
+/// Return an empty vector if `sizes` is empty.
+///
+/// The function emits an IR block which computes suffix product for provided
+/// sizes.
+SmallVector<OpFoldResult>
+computeSuffixProductIRBlock(Location loc, OpBuilder &builder,
+                            ArrayRef<OpFoldResult> sizes);
+inline SmallVector<OpFoldResult>
+computeStridesIRBlock(Location loc, OpBuilder &builder,
+                      ArrayRef<OpFoldResult> sizes) {
+  return computeSuffixProductIRBlock(loc, builder, sizes);
+}
+
 } // namespace memref
 } // namespace mlir
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
index aa44455ada7f..29a5bc9a7ae5 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
@@ -19,6 +19,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/MemRef/Transforms/Passes.h"
 #include "mlir/Dialect/MemRef/Transforms/Transforms.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
 #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
@@ -63,39 +64,85 @@ resolveSourceIndicesExpandShape(Location loc, PatternRewriter &rewriter,
                                 memref::ExpandShapeOp expandShapeOp,
                                 ValueRange indices,
                                 SmallVectorImpl<Value> &sourceIndices) {
-  // The below implementation uses computeSuffixProduct method, which only
-  // allows int64_t values (i.e., static shape). Bail out if it has dynamic
-  // shapes.
-  if (!expandShapeOp.getResultType().hasStaticShape())
+  // Record the rewriter context for constructing ops later.
+  MLIRContext *ctx = rewriter.getContext();
+
+  // Capture expand_shape's input dimensions as `SmallVector<OpFoldResult>`.
+  // This is done for the purpose of inferring the output shape via
+  // `inferExpandOutputShape` which will in turn be used for suffix product
+  // calculation later.
+  SmallVector<OpFoldResult> srcShape;
+  MemRefType srcType = expandShapeOp.getSrcType();
+
+  for (int64_t i = 0, e = srcType.getRank(); i < e; ++i) {
+    if (srcType.isDynamicDim(i)) {
+      srcShape.push_back(
+          rewriter.create<memref::DimOp>(loc, expandShapeOp.getSrc(), i)
+              .getResult());
+    } else {
+      srcShape.push_back(rewriter.getIndexAttr(srcType.getShape()[i]));
+    }
+  }
+
+  auto outputShape = inferExpandShapeOutputShape(
+      rewriter, loc, expandShapeOp.getResultType(),
+      expandShapeOp.getReassociationIndices(), srcShape);
+  if (!outputShape.has_value())
     return failure();
 
-  MLIRContext *ctx = rewriter.getContext();
+  // Traverse all reassociation groups to determine the appropriate indices
+  // corresponding to each one of them post op folding.
   for (ArrayRef<int64_t> groups : expandShapeOp.getReassociationIndices()) {
     assert(!groups.empty() && "association indices groups cannot be empty");
+    // Flag to indicate the presence of dynamic dimensions in current
+    // reassociation group.
     int64_t groupSize = groups.size();
 
-    // Construct the expression for the index value w.r.t to expand shape op
-    // source corresponding the indices wrt to expand shape op result.
-    SmallVector<int64_t> sizes(groupSize);
-    for (int64_t i = 0; i < groupSize; ++i)
-      sizes[i] = expandShapeOp.getResultType().getDimSize(groups[i]);
-    SmallVector<int64_t> suffixProduct = computeSuffixProduct(sizes);
-    SmallVector<AffineExpr> dims(groupSize);
-    bindDimsList(ctx, MutableArrayRef{dims});
-    AffineExpr srcIndexExpr = linearize(ctx, dims, suffixProduct);
+    // Group output dimensions utilized in this reassociation group for suffix
+    // product calculation.
+    SmallVector<OpFoldResult> sizesVal(groupSize);
+    for (int64_t i = 0; i < groupSize; ++i) {
+      sizesVal[i] = (*outputShape)[groups[i]];
+    }
+
+    // Calculate suffix product of relevant output dimension sizes.
+    SmallVector<OpFoldResult> suffixProduct =
+        memref::computeSuffixProductIRBlock(loc, rewriter, sizesVal);
+
+    // Create affine expression variables for dimensions and symbols in the
+    // newly constructed affine map.
+    SmallVector<AffineExpr> dims(groupSize), symbols(groupSize);
+    bindDimsList<AffineExpr>(ctx, dims);
+    bindSymbolsList<AffineExpr>(ctx, symbols);
 
-    /// Apply permutation and create AffineApplyOp.
+    // Linearize binded dimensions and symbols to construct the resultant
+    // affine expression for this indice.
+    AffineExpr srcIndexExpr = linearize(ctx, dims, symbols);
+
+    // Record the load index corresponding to each dimension in the
+    // reassociation group. These are later supplied as operands to the affine
+    // map used for calulating relevant index post op folding.
     SmallVector<OpFoldResult> dynamicIndices(groupSize);
     for (int64_t i = 0; i < groupSize; i++)
       dynamicIndices[i] = indices[groups[i]];
 
-    // Creating maximally folded and composd affine.apply composes better with
-    // other transformations without interleaving canonicalization passes.
+    // Supply suffix product results followed by load op indices as operands
+    // to the map.
+    SmallVector<OpFoldResult> mapOperands;
+    llvm::append_range(mapOperands, suffixProduct);
+    llvm::append_range(mapOperands, dynamicIndices);
+
+    // Creating maximally folded and composed affine.apply composes better
+    // with other transformations without interleaving canonicalization
+    // passes.
     OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
         rewriter, loc,
         AffineMap::get(/*numDims=*/groupSize,
-                       /*numSymbols=*/0, srcIndexExpr),
-        dynamicIndices);
+                       /*numSymbols=*/groupSize, /*expression=*/srcIndexExpr),
+        mapOperands);
+
+    // Push index value in the op post folding corresponding to this
+    // reassociation group.
     sourceIndices.push_back(
         getValueOrCreateConstantIndexOp(rewriter, loc, ofr));
   }
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
index 556a82de2166..c93e5a9dcd39 100644
--- a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "llvm/ADT/STLExtras.h"
 
 namespace mlir {
 namespace memref {
@@ -155,5 +156,27 @@ void eraseDeadAllocAndStores(RewriterBase &rewriter, Operation *parentOp) {
     rewriter.eraseOp(op);
 }
 
+static SmallVector<OpFoldResult>
+computeSuffixProductIRBlockImpl(Location loc, OpBuilder &builder,
+                                ArrayRef<OpFoldResult> sizes,
+                                OpFoldResult unit) {
+  SmallVector<OpFoldResult> strides(sizes.size(), unit);
+  AffineExpr s0, s1;
+  bindSymbols(builder.getContext(), s0, s1);
+
+  for (int64_t r = strides.size() - 1; r > 0; --r) {
+    strides[r - 1] = affine::makeComposedFoldedAffineApply(
+        builder, loc, s0 * s1, {strides[r], sizes[r]});
+  }
+  return strides;
+}
+
+SmallVector<OpFoldResult>
+computeSuffixProductIRBlock(Location loc, OpBuilder &builder,
+                            ArrayRef<OpFoldResult> sizes) {
+  OpFoldResult unit = builder.getIndexAttr(1);
+  return computeSuffixProductIRBlockImpl(loc, builder, sizes, unit);
+}
+
 } // namespace memref
 } // namespace mlir
diff --git a/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir b/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir
index 254cd4015eed..99b5f78b03fb 100644
--- a/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir
+++ b/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir
@@ -468,23 +468,66 @@ func.func @fold_static_stride_subview_with_affine_load_store_expand_shape_3d(%ar
 
 // -----
 
-// CHECK-LABEL: fold_dynamic_subview_with_memref_load_store_expand_shape
-// CHECK-SAME: (%[[ARG0:.*]]: memref<16x?xf32, strided<[16, 1]>>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index, %[[SZ0:.*]]: index)
-func.func @fold_dynamic_subview_with_memref_load_store_expand_shape(%arg0 : memref<16x?xf32, strided<[16, 1]>>, %arg1 : index, %arg2 : index, %sz0: index) -> f32 {
+// CHECK-LABEL: fold_dynamic_subview_with_memref_load_expand_shape
+// CHECK-SAME: (%[[ARG0:.*]]: memref<16x?xf32, strided<[16, 1]>>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index) -> f32
+func.func @fold_dynamic_subview_with_memref_load_expand_shape(%arg0 : memref<16x?xf32, strided<[16, 1]>>, %arg1 : index, %arg2 : index, %sz0: index) -> f32 {
   %c0 = arith.constant 0 : index
   %expand_shape = memref.expand_shape %arg0 [[0, 1], [2, 3]] output_shape [1, 16, %sz0, 1] : memref<16x?xf32, strided<[16, 1]>> into memref<1x16x?x1xf32, strided<[256, 16, 1, 1]>>
   %0 = memref.load %expand_shape[%c0, %arg1, %arg2, %c0] : memref<1x16x?x1xf32, strided<[256, 16, 1, 1]>>
   return %0 : f32
 }
-// CHECK: %[[C0:.*]] = arith.constant 0 : index
-// CHECK: %[[EXPAND_SHAPE:.*]] = memref.expand_shape %[[ARG0]] {{\[\[}}0, 1], [2, 3]] output_shape [1, 16, %[[SZ0]], 1] : memref<16x?xf32, strided<[16, 1]>> into memref<1x16x?x1xf32, strided<[256, 16, 1, 1]>>
-// CHECK: %[[VAL_0:.*]] = memref.load %[[EXPAND_SHAPE]][%[[C0]], %[[ARG1]], %[[ARG2]], %[[C0]]] : memref<1x16x?x1xf32, strided<[256, 16, 1, 1]>>
-// CHECK: return %[[VAL_0]] : f32
+// CHECK-NEXT: %[[VAL1:.*]] = memref.load %[[ARG0]][%[[ARG1]], %[[ARG2]]] : memref<16x?xf32, strided<[16, 1]>>
+// CHECK-NEXT: return %[[VAL1]] : f32
 
 // -----
 
-// CHECK-DAG: #[[$MAP0:.*]] = affine_map<(d0, d1) -> (d0 * 1024 + d1)>
-// CHECK-DAG: #[[$MAP1:.*]] = affine_map<(d0, d1) -> (d0 + d1)>
+// CHECK-LABEL: fold_dynamic_subview_with_memref_store_expand_shape
+// CHECK-SAME: (%[[ARG0:.*]]: memref<16x?xf32, strided<[16, 1]>>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index)
+func.func @fold_dynamic_subview_with_memref_store_expand_shape(%arg0 : memref<16x?xf32, strided<[16, 1]>>, %arg1 : index, %arg2 : index, %sz0 : index) {
+  %c0 = arith.constant 0 : index
+  %c1f32 = arith.constant 1.0 : f32
+  %expand_shape = memref.expand_shape %arg0 [[0, 1], [2, 3]] output_shape [1, 16, %sz0, 1] : memref<16x?xf32, strided<[16, 1]>> into memref<1x16x?x1xf32, strided<[256, 16, 1, 1]>>
+  memref.store %c1f32, %expand_shape[%c0, %arg1, %arg2, %c0] : memref<1x16x?x1xf32, strided<[256, 16, 1, 1]>>
+  return
+}
+// CHECK: %[[C1F32:.*]] = arith.constant 1.000000e+00 : f32
+// CHECK-NEXT: memref.store %[[C1F32]], %[[ARG0]][%[[ARG1]], %[[ARG2]]] : memref<16x?xf32, strided<[16, 1]>>
+// CHECK-NEXT: return
+
+// -----
+
+// CHECK-DAG: #[[$MAP0:.*]] = affine_map<()[s0, s1] -> (s0 + s1)>
+// CHECK-DAG: #[[$MAP1:.*]] = affine_map<()[s0] -> (s0 * 3)>
+// CHECK-LABEL: fold_memref_alias_expand_shape_subview_load_store_dynamic_dim
+// CHECK-SAME: (%[[ARG0:.*]]: memref<2048x16xf32>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index)
+func.func @fold_memref_alias_expand_shape_subview_load_store_dynamic_dim(%alloc: memref<2048x16xf32>, %c10: index, %c5: index, %c0: index, %sz0: index) {
+  %subview = memref.subview %alloc[%c5, 0] [%c10, 16] [1, 1] : memref<2048x16xf32> to memref<?x16xf32, strided<[16, 1], offset: ?>>
+  %expand_shape = memref.expand_shape %subview [[0], [1, 2, 3]] output_shape [1, 16, %sz0, 1] : memref<?x16xf32, strided<[16, 1], offset: ?>> into memref<?x1x8x2xf32, strided<[16, 16, 2, 1], offset: ?>>
+  %dim = memref.dim %expand_shape, %c0 : memref<?x1x8x2xf32, strided<[16, 16, 2, 1], offset: ?>>
+
+  affine.for %arg6 = 0 to %dim step 64 {
+    affine.for %arg7 = 0 to 16 step 16 {
+      %dummy_load = affine.load %expand_shape[%arg6, 0, %arg7, %arg7] : memref<?x1x8x2xf32, strided<[16, 16, 2, 1], offset: ?>>
+      affine.store %dummy_load, %subview[%arg6, %arg7] : memref<?x16xf32, strided<[16, 1], offset: ?>>
+    }
+  }
+  return
+}
+// CHECK-NEXT:   memref.subview
+// CHECK-NEXT:   %[[EXPAND_SHAPE:.*]] = memref.expand_shape
+// CHECK-NEXT:   %[[DIM:.*]] = memref.dim %[[EXPAND_SHAPE]], %[[ARG3]] : memref<?x1x8x2xf32, strided<[16, 16, 2, 1], offset: ?>>
+// CHECK-NEXT:   affine.for %[[ARG4:.*]] = 0 to %[[DIM]] step 64 {
+// CHECK-NEXT:   affine.for %[[ARG5:.*]] = 0 to 16 step 16 {
+// CHECK-NEXT:   %[[VAL0:.*]] = affine.apply #[[$MAP0]]()[%[[ARG2]], %[[ARG4]]]
+// CHECK-NEXT:   %[[VAL1:.*]] = affine.apply #[[$MAP1]]()[%[[ARG5]]]
+// CHECK-NEXT:   %[[VAL2:.*]] = affine.load %[[ARG0]][%[[VAL0]], %[[VAL1]]] : memref<2048x16xf32>
+// CHECK-NEXT:   %[[VAL3:.*]] = affine.apply #[[$MAP0]]()[%[[ARG2]], %[[ARG4]]]
+// CHECK-NEXT:   affine.store %[[VAL2]], %[[ARG0]][%[[VAL3]], %[[ARG5]]] : memref<2048x16xf32>
+
+// -----
+
+// CHECK-DAG: #[[$MAP0:.*]] = affine_map<()[s0, s1] -> (s0 * 1024 + s1)>
+// CHECK-DAG: #[[$MAP1:.*]] = affine_map<()[s0, s1] -> (s0 + s1)>
 // CHECK-LABEL: fold_static_stride_subview_with_affine_load_store_expand_shape
 // CHECK-SAME: (%[[ARG0:.*]]: memref<1024x1024xf32>, %[[ARG1:.*]]: memref<1xf32>, %[[ARG2:.*]]: index)
 func.func @fold_static_stride_subview_with_affine_load_store_expand_shape(%arg0: memref<1024x1024xf32>, %arg1: memref<1xf32>, %arg2: index) -> f32 {
@@ -506,14 +549,14 @@ func.func @fold_static_stride_subview_with_affine_load_store_expand_shape(%arg0:
 // CHECK-NEXT:  affine.for %[[ARG4:.*]] = 0 to 1024 {
 // CHECK-NEXT:   affine.for %[[ARG5:.*]] = 0 to 1020 {
 // CHECK-NEXT:    affine.for %[[ARG6:.*]] = 0 to 1 {
-// CHECK-NEXT:     %[[IDX1:.*]] = affine.apply #[[$MAP0]](%[[ARG3]], %[[ARG4]])
-// CHECK-NEXT:     %[[IDX2:.*]] = affine.apply #[[$MAP1]](%[[ARG5]], %[[ARG6]])
+// CHECK-NEXT:     %[[IDX1:.*]] = affine.apply #[[$MAP0]]()[%[[ARG3]], %[[ARG4]]]
+// CHECK-NEXT:     %[[IDX2:.*]] = affine.apply #[[$MAP1]]()[%[[ARG5]], %[[ARG6]]]
 // CHECK-NEXT:     affine.load %[[ARG0]][%[[IDX1]], %[[IDX2]]] : memref<1024x1024xf32>
 
 // -----
 
-// CHECK-DAG: #[[$MAP0:.*]] = affine_map<(d0, d1) -> (d0 * 1025 + d1)>
-// CHECK-DAG: #[[$MAP1:.*]] = affine_map<(d0, d1) -> (d0 + d1)>
+// CHECK-DAG: #[[$MAP0:.*]] = affine_map<(d0, d1)[s0] -> (d0 + d1 + s0 * 1024)>
+// CHECK-DAG: #[[$MAP1:.*]] = affine_map<()[s0, s1] -> (s0 + s1)>
 // CHECK-LABEL: fold_static_stride_subview_with_affine_load_store_expand_shape_when_access_index_is_an_expression
 // CHECK-SAME: (%[[ARG0:.*]]: memref<1024x1024xf32>, %[[ARG1:.*]]: memref<1xf32>, %[[ARG2:.*]]: index)
 func.func @fold_static_stride_subview_with_affine_load_store_expand_shape_when_access_index_is_an_expression(%arg0: memref<1024x1024xf32>, %arg1: memref<1xf32>, %arg2: index) -> f32 {
@@ -535,14 +578,14 @@ func.func @fold_static_stride_subview_with_affine_load_store_expand_shape_when_a
 // CHECK-NEXT:  affine.for %[[ARG4:.*]] = 0 to 1024 {
 // CHECK-NEXT:   affine.for %[[ARG5:.*]] = 0 to 1020 {
 // CHECK-NEXT:    affine.for %[[ARG6:.*]] = 0 to 1 {
-// CHECK-NEXT:      %[[TMP1:.*]] = affine.apply #[[$MAP0]](%[[ARG3]], %[[ARG4]])
-// CHECK-NEXT:      %[[TMP3:.*]] = affine.apply #[[$MAP1]](%[[ARG5]], %[[ARG6]])
+// CHECK-NEXT:      %[[TMP1:.*]] = affine.apply #[[$MAP0]](%[[ARG3]], %[[ARG4]])[%[[ARG3]]]
+// CHECK-NEXT:      %[[TMP3:.*]] = affine.apply #[[$MAP1]]()[%[[ARG5]], %[[ARG6]]]
 // CHECK-NEXT:      affine.load %[[ARG0]][%[[TMP1]], %[[TMP3]]] : memref<1024x1024xf32>
 
 // -----
 
-// CHECK-DAG: #[[$MAP0:.*]] = affine_map<(d0) -> (d0 * 1024)>
-// CHECK-DAG: #[[$MAP1:.*]] = affine_map<(d0, d1) -> (d0 + d1)>
+// CHECK-DAG: #[[$MAP0:.*]] = affine_map<()[s0] -> (s0 * 1024)>
+// CHECK-DAG: #[[$MAP1:.*]] = affine_map<()[s0, s1] -> (s0 + s1)>
 // CHECK-LABEL: fold_static_stride_subview_with_affine_load_store_expand_shape_with_constant_access_index
 // CHECK-SAME: (%[[ARG0:.*]]: memref<1024x1024xf32>, %[[ARG1:.*]]: memref<1xf32>, %[[ARG2:.*]]: index)
 func.func @fold_static_stride_subview_with_affine_load_store_expand_shape_with_constant_access_index(%arg0: memref<1024x1024xf32>, %arg1: memref<1xf32>, %arg2: index) -> f32 {
@@ -565,8 +608,8 @@ func.func @fold_static_stride_subview_with_affine_load_store_expand_shape_with_c
 // CHECK-NEXT:   affine.for %[[ARG4:.*]] = 0 to 1024 {
 // CHECK-NEXT:    affine.for %[[ARG5:.*]] = 0 to 1020 {
 // CHECK-NEXT:     affine.for %[[ARG6:.*]] = 0 to 1 {
-// CHECK-NEXT:      %[[TMP1:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
-// CHECK-NEXT:      %[[TMP2:.*]] = affine.apply #[[$MAP1]](%[[ARG5]], %[[ARG6]])
+// CHECK-NEXT:      %[[TMP1:.*]] = affine.apply #[[$MAP0]]()[%[[ARG3]]]
+// CHECK-NEXT:      %[[TMP2:.*]] = affine.apply #[[$MAP1]]()[%[[ARG5]], %[[ARG6]]]
 // CHECK-NEXT:      memref.load %[[ARG0]][%[[TMP1]], %[[TMP2]]] : memref<1024x1024xf32>
 
 // -----
-- 
GitLab


From 2475efa91d8b4fa8f1a2d16052cb6d14be7d5dc6 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Wed, 8 May 2024 06:53:12 -0700
Subject: [PATCH 0180/1206] [SLP]Fix PR91467: Look through scalar cast, when
 trying to cast to another type.

Need to look through the SExt/ZExt scalars to be gathered, when trying
to reduce their width after minbitwidth analysis to prevent permanent
attempts to revectorize such gathered instructions.
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp          | 6 +++++-
 .../SLPVectorizer/AArch64/gather-with-minbith-user.ll    | 9 +--------
 .../SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll  | 7 +------
 .../SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll      | 3 +--
 4 files changed, 8 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 98561f9ca044..cc9219ca02cf 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11419,8 +11419,12 @@ Value *BoUpSLP::gather(ArrayRef<Value *> VL, Value *Root, Type *ScalarTy) {
     if (Scalar->getType() != Ty) {
       assert(Scalar->getType()->isIntegerTy() && Ty->isIntegerTy() &&
              "Expected integer types only.");
+      Value *V = Scalar;
+      if (auto *CI = dyn_cast<CastInst>(Scalar);
+          isa_and_nonnull<SExtInst, ZExtInst>(CI))
+        V = CI->getOperand(0);
       Scalar = Builder.CreateIntCast(
-          Scalar, Ty, !isKnownNonNegative(Scalar, SimplifyQuery(*DL)));
+          V, Ty, !isKnownNonNegative(Scalar, SimplifyQuery(*DL)));
     }
 
     Vec = Builder.CreateInsertElement(Vec, Scalar, Builder.getInt32(Pos));
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll
index 76bb882171b1..3ebe920d1734 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll
@@ -5,14 +5,7 @@ define void @h() {
 ; CHECK-LABEL: define void @h() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, ptr null, i64 16
-; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 0 to i1
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 poison, i1 false, i1 false, i1 false>, i1 [[TMP6]], i32 4
-; CHECK-NEXT:    [[TMP1:%.*]] = sub <8 x i1> [[TMP0]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = add <8 x i1> [[TMP0]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i1> [[TMP3]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i1> [[TMP5]] to <8 x i16>
-; CHECK-NEXT:    store <8 x i16> [[TMP4]], ptr [[ARRAYIDX2]], align 2
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr [[ARRAYIDX2]], align 2
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll
index 2ab6e919c23b..6404cf4a2cd1 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll
@@ -5,12 +5,7 @@ define void @h() {
 ; CHECK-LABEL: define void @h() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, ptr null, i64 16
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 0 to i1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 poison, i1 false, i1 false, i1 false>, i1 [[TMP0]], i32 4
-; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i1> zeroinitializer, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i1> zeroinitializer, [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i1> [[TMP3]] to <8 x i16>
-; CHECK-NEXT:    store <8 x i16> [[TMP4]], ptr [[ARRAYIDX2]], align 2
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr [[ARRAYIDX2]], align 2
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll
index 1bb87bf6205f..3c8e98485ffc 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll
@@ -4,10 +4,9 @@
 define void @test(ptr %a, i8 %0, i16 %b.promoted.i) {
 ; CHECK-LABEL: define void @test(
 ; CHECK-SAME: ptr [[A:%.*]], i8 [[TMP0:%.*]], i16 [[B_PROMOTED_I:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP0]] to i128
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i16> poison, i16 [[B_PROMOTED_I]], i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP2]] to i16
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i8 [[TMP0]] to i16
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i16> poison, i16 [[TMP5]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[TMP6]], <4 x i16> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i16> [[TMP4]], [[TMP7]]
-- 
GitLab


From 2868e26d0a6f257d3a8f303c7918f37c690e35a4 Mon Sep 17 00:00:00 2001
From: Matthias Braun <matze@braunis.de>
Date: Wed, 8 May 2024 07:35:47 -0700
Subject: [PATCH 0181/1206] Use cmake to find perl executable (#91275)

`clang/tools/scan-build` is implemented in `perl`. However given `perl`
is not mentioned as a required dependency in `GettingStarted.rst` we
should make this optional.

This adds a `find_package(Perl)` check to cmake and disables the
`scan-build` tests when no perl executable is found.

Ideally we would also check if dependent perl modules like `Hash::Util`
are present on the system, but I don't see any pre-existing cmake macros
to easily test this. So for now I go with a plain check for the `perl`
package, at least this allows to use `cmake
-DCMAKE_DISABLE_FIND_PACKAGE_Perl=ON` to manually disable `perl` and the
tests.
---
 clang/CMakeLists.txt                                |  2 ++
 clang/test/Analysis/scan-build/deduplication.test   |  1 -
 .../Analysis/scan-build/exclude_directories.test    |  3 ---
 clang/test/Analysis/scan-build/help.test            |  3 ---
 clang/test/Analysis/scan-build/html_output.test     |  1 -
 clang/test/Analysis/scan-build/lit.local.cfg        | 13 ++++++++-----
 .../test/Analysis/scan-build/plist_html_output.test |  1 -
 clang/test/Analysis/scan-build/plist_output.test    |  1 -
 .../scan-build/rebuild_index/rebuild_index.test     |  3 ---
 .../Analysis/scan-build/silence-core-checkers.test  |  3 ---
 clang/test/lit.site.cfg.py.in                       |  1 +
 11 files changed, 11 insertions(+), 21 deletions(-)

diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index cf97e3c6e851..c20ce47a12ab 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -523,6 +523,8 @@ endif()
 
 
 if( CLANG_INCLUDE_TESTS )
+  find_package(Perl)
+
   add_subdirectory(unittests)
   list(APPEND CLANG_TEST_DEPS ClangUnitTests)
   list(APPEND CLANG_TEST_PARAMS
diff --git a/clang/test/Analysis/scan-build/deduplication.test b/clang/test/Analysis/scan-build/deduplication.test
index 56d888e5fc12..2ec3061701fc 100644
--- a/clang/test/Analysis/scan-build/deduplication.test
+++ b/clang/test/Analysis/scan-build/deduplication.test
@@ -1,4 +1,3 @@
-// FIXME: Actually, "perl".
 REQUIRES: shell
 
 RUN: rm -rf %t.output_dir && mkdir %t.output_dir
diff --git a/clang/test/Analysis/scan-build/exclude_directories.test b/clang/test/Analysis/scan-build/exclude_directories.test
index c161e51b6d26..2c79ed842af1 100644
--- a/clang/test/Analysis/scan-build/exclude_directories.test
+++ b/clang/test/Analysis/scan-build/exclude_directories.test
@@ -1,6 +1,3 @@
-// FIXME: Actually, "perl".
-REQUIRES: shell
-
 RUN: rm -rf %t.output_dir && mkdir %t.output_dir
 RUN: %scan-build -o %t.output_dir %clang -S \
 RUN:     %S/Inputs/multidirectory_project/directory1/file1.c \
diff --git a/clang/test/Analysis/scan-build/help.test b/clang/test/Analysis/scan-build/help.test
index 61915d326094..d1f17cd69f51 100644
--- a/clang/test/Analysis/scan-build/help.test
+++ b/clang/test/Analysis/scan-build/help.test
@@ -1,6 +1,3 @@
-// FIXME: Actually, "perl".
-REQUIRES: shell
-
 RUN: %scan-build -h | FileCheck %s
 RUN: %scan-build --help | FileCheck %s
 
diff --git a/clang/test/Analysis/scan-build/html_output.test b/clang/test/Analysis/scan-build/html_output.test
index add35d83b958..c2b509d9ef66 100644
--- a/clang/test/Analysis/scan-build/html_output.test
+++ b/clang/test/Analysis/scan-build/html_output.test
@@ -1,4 +1,3 @@
-// FIXME: Actually, "perl".
 REQUIRES: shell
 
 RUN: rm -rf %t.output_dir && mkdir %t.output_dir
diff --git a/clang/test/Analysis/scan-build/lit.local.cfg b/clang/test/Analysis/scan-build/lit.local.cfg
index fab52b1c7bd6..aed76ca0e808 100644
--- a/clang/test/Analysis/scan-build/lit.local.cfg
+++ b/clang/test/Analysis/scan-build/lit.local.cfg
@@ -1,8 +1,8 @@
 # -*- Python -*-
 
-import lit.util
 import lit.formats
 import os
+import platform
 
 use_lit_shell = os.environ.get("LIT_USE_INTERNAL_SHELL")
 config.test_format = lit.formats.ShTest(use_lit_shell == "0")
@@ -12,13 +12,16 @@ clang_path = config.clang if config.have_llvm_driver else os.path.realpath(confi
 config.substitutions.append(
     (
         "%scan-build",
-        "'%s' --use-analyzer=%s "
+        "'%s' '%s' --use-analyzer=%s "
         % (
-            lit.util.which(
-                "scan-build",
-                os.path.join(config.clang_src_dir, "tools", "scan-build", "bin"),
+            config.perl_executable,
+            os.path.join(
+                config.clang_src_dir, "tools", "scan-build", "bin", "scan-build"
             ),
             clang_path,
         ),
     )
 )
+
+if not config.perl_executable or platform.system() == "Windows":
+    config.unsupported = True
diff --git a/clang/test/Analysis/scan-build/plist_html_output.test b/clang/test/Analysis/scan-build/plist_html_output.test
index c07891e35fbf..ca9c5256b9d7 100644
--- a/clang/test/Analysis/scan-build/plist_html_output.test
+++ b/clang/test/Analysis/scan-build/plist_html_output.test
@@ -1,4 +1,3 @@
-// FIXME: Actually, "perl".
 REQUIRES: shell
 
 RUN: rm -rf %t.output_dir && mkdir %t.output_dir
diff --git a/clang/test/Analysis/scan-build/plist_output.test b/clang/test/Analysis/scan-build/plist_output.test
index 0112e84630ed..4d01640bff6e 100644
--- a/clang/test/Analysis/scan-build/plist_output.test
+++ b/clang/test/Analysis/scan-build/plist_output.test
@@ -1,4 +1,3 @@
-// FIXME: Actually, "perl".
 REQUIRES: shell
 
 RUN: rm -rf %t.output_dir && mkdir %t.output_dir
diff --git a/clang/test/Analysis/scan-build/rebuild_index/rebuild_index.test b/clang/test/Analysis/scan-build/rebuild_index/rebuild_index.test
index ab70435c6054..711a74f3fd02 100644
--- a/clang/test/Analysis/scan-build/rebuild_index/rebuild_index.test
+++ b/clang/test/Analysis/scan-build/rebuild_index/rebuild_index.test
@@ -1,6 +1,3 @@
-// FIXME: Actually, "perl".
-REQUIRES: shell
-
 RUN: rm -rf %t.output_dir && mkdir %t.output_dir
 RUN: cp %S/report-1.html %t.output_dir
 RUN: cp %S/report-2.html %t.output_dir
diff --git a/clang/test/Analysis/scan-build/silence-core-checkers.test b/clang/test/Analysis/scan-build/silence-core-checkers.test
index 6d9a3017fcd6..7ffa744a545c 100644
--- a/clang/test/Analysis/scan-build/silence-core-checkers.test
+++ b/clang/test/Analysis/scan-build/silence-core-checkers.test
@@ -1,6 +1,3 @@
-// FIXME: Actually, "perl".
-REQUIRES: shell
-
 RUN: rm -rf %t.output_dir && mkdir %t.output_dir
 RUN: %scan-build -o %t.output_dir \
 RUN:   %clang -S %S/Inputs/null_dereference_and_division_by_zero.c \
diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in
index 6641811c5883..ec6d30e6c220 100644
--- a/clang/test/lit.site.cfg.py.in
+++ b/clang/test/lit.site.cfg.py.in
@@ -34,6 +34,7 @@ config.enable_backtrace = @ENABLE_BACKTRACES@
 config.enable_threads = @LLVM_ENABLE_THREADS@
 config.reverse_iteration = @LLVM_ENABLE_REVERSE_ITERATION@
 config.host_arch = "@HOST_ARCH@"
+config.perl_executable = "@PERL_EXECUTABLE@"
 config.python_executable = "@Python3_EXECUTABLE@"
 config.use_z3_solver = lit_config.params.get('USE_Z3_SOLVER', "@USE_Z3_SOLVER@")
 config.has_plugins = @CLANG_PLUGIN_SUPPORT@
-- 
GitLab


From 3a8316216807d64a586b971f51695e23883331f7 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Wed, 8 May 2024 16:38:38 +0200
Subject: [PATCH 0182/1206] [bazel] Add missing dependency for
 6ed8434edc5934210a38be99f33b6baed83df85c

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index b75ed99df5b2..6a7bc5c9fea0 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -12945,6 +12945,7 @@ cc_library(
         ":ArithUtils",
         ":MemRefDialect",
         ":VectorDialect",
+        "//llvm:Support",
     ],
 )
 
-- 
GitLab


From 50b45b24220ead33cf5cedc49c13e0336297e4eb Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 8 May 2024 15:45:38 +0100
Subject: [PATCH 0183/1206] [LAA] Add tests with forward dependences known via
 assumes.

---
 .../offset-range-known-via-assume.ll          | 244 ++++++++++++++++++
 1 file changed, 244 insertions(+)
 create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/offset-range-known-via-assume.ll

diff --git a/llvm/test/Analysis/LoopAccessAnalysis/offset-range-known-via-assume.ll b/llvm/test/Analysis/LoopAccessAnalysis/offset-range-known-via-assume.ll
new file mode 100644
index 000000000000..7e36da78d6aa
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/offset-range-known-via-assume.ll
@@ -0,0 +1,244 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+declare void @llvm.assume(i1)
+
+declare void @use(ptr noundef)
+
+; TODO: %offset is known positive via assume, so we should be able to detect the
+; forward dependence.
+define void @offset_i8_known_positive_via_assume_forward_dep_1(ptr %A, i64 %offset, i64 %N) {
+; CHECK-LABEL: 'offset_i8_known_positive_via_assume_forward_dep_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP1:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+; CHECK-NEXT:        Against group ([[GRP2:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.off = getelementptr inbounds i8, ptr %off, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP1]]:
+; CHECK-NEXT:          (Low: %A High: (%N + %A))
+; CHECK-NEXT:            Member: {%A,+,1}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP2]]:
+; CHECK-NEXT:          (Low: (%offset + %A) High: (%offset + %N + %A))
+; CHECK-NEXT:            Member: {(%offset + %A),+,1}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %c = icmp sgt i64 %offset, 0
+  call void @llvm.assume(i1 %c)
+  %off = getelementptr inbounds i8, ptr %A, i64 %offset
+  call void @use(ptr noundef %off)
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.off = getelementptr inbounds i8, ptr %off, i64 %iv
+  %l = load i8 , ptr %gep.off, align 4
+  %add = add nsw i8 %l, 5
+  %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+  store i8 %add, ptr %gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @offset_i32_known_positive_via_assume_forward_dep_1(ptr %A, i64 %offset, i64 %N) {
+; CHECK-LABEL: 'offset_i32_known_positive_via_assume_forward_dep_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP3:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+; CHECK-NEXT:        Against group ([[GRP4:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.off = getelementptr inbounds i32, ptr %off, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP3]]:
+; CHECK-NEXT:          (Low: %A High: (-3 + (4 * %N) + %A))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP4]]:
+; CHECK-NEXT:          (Low: ((4 * %offset)<nsw> + %A) High: (-3 + (4 * %offset)<nsw> + (4 * %N) + %A))
+; CHECK-NEXT:            Member: {((4 * %offset)<nsw> + %A),+,4}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-NEXT:      {((4 * %offset)<nsw> + %A),+,4}<nw><%loop> Added Flags: <nusw>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %c = icmp sgt i64 %offset, 0
+  call void @llvm.assume(i1 %c)
+  %off = getelementptr inbounds i32, ptr %A, i64 %offset
+  call void @use(ptr noundef %off)
+
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.off = getelementptr inbounds i32, ptr %off, i64 %iv
+  %l = load i8 , ptr %gep.off, align 4
+  %add = add nsw i8 %l, 5
+  %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+  store i8 %add, ptr %gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; TODO: %offset is known positive via assume, so we should be able to detect the
+; forward dependence.
+define void @offset_known_positive_via_assume_forward_dep_2(ptr %A, i64 %offset, i64 %N) {
+; CHECK-LABEL: 'offset_known_positive_via_assume_forward_dep_2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP5:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+; CHECK-NEXT:        Against group ([[GRP6:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.off = getelementptr inbounds i32, ptr %off, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP5]]:
+; CHECK-NEXT:          (Low: %A High: ((4 * %N) + %A))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP6]]:
+; CHECK-NEXT:          (Low: ((4 * %offset)<nsw> + %A) High: ((4 * %offset)<nsw> + (4 * %N) + %A))
+; CHECK-NEXT:            Member: {((4 * %offset)<nsw> + %A),+,4}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %c = icmp sgt i64 %offset, 0
+  call void @llvm.assume(i1 %c)
+  %c.2 = icmp slt i64 %offset, 20
+  call void @llvm.assume(i1 %c.2)
+  %off = getelementptr inbounds i32, ptr %A, i64 %offset
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.off = getelementptr inbounds i32, ptr %off, i64 %iv
+  %l = load i32, ptr %gep.off, align 4
+  %add = add nsw i32 %l, 5
+  %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+  store i32 %add, ptr %gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; The range of %offset is known via assumes, but it may be positive or negative.
+define void @offset_may_be_negative_via_assume_unknown_dep(ptr %A, i64 %offset, i64 %N) {
+; CHECK-LABEL: 'offset_may_be_negative_via_assume_unknown_dep'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP7:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.mul.2 = getelementptr inbounds i32, ptr %A, i64 %iv
+; CHECK-NEXT:        Against group ([[GRP8:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep = getelementptr inbounds i32, ptr %off, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP7]]:
+; CHECK-NEXT:          (Low: %A High: ((4 * %N) + %A))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP8]]:
+; CHECK-NEXT:          (Low: ((4 * %offset)<nsw> + %A) High: ((4 * %offset)<nsw> + (4 * %N) + %A))
+; CHECK-NEXT:            Member: {((4 * %offset)<nsw> + %A),+,4}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %c = icmp sgt i64 %offset, -4
+  call void @llvm.assume(i1 %c)
+  %c.2 = icmp slt i64 %offset, 20
+  call void @llvm.assume(i1 %c.2)
+  %off = getelementptr inbounds i32, ptr %A, i64 %offset
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i32, ptr %off, i64 %iv
+  %l = load i32, ptr %gep, align 4
+  %add = add nsw i32 %l, 5
+  %gep.mul.2 = getelementptr inbounds i32, ptr %A, i64 %iv
+  store i32 %add, ptr %gep.mul.2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @offset_no_assumes(ptr %A, i64 %offset, i64 %N) {
+; CHECK-LABEL: 'offset_no_assumes'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP9:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+; CHECK-NEXT:        Against group ([[GRP10:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.off = getelementptr inbounds i32, ptr %off, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP9]]:
+; CHECK-NEXT:          (Low: %A High: ((4 * %N) + %A))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP10]]:
+; CHECK-NEXT:          (Low: ((4 * %offset)<nsw> + %A) High: ((4 * %offset)<nsw> + (4 * %N) + %A))
+; CHECK-NEXT:            Member: {((4 * %offset)<nsw> + %A),+,4}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %off = getelementptr inbounds i32, ptr %A, i64 %offset
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.off = getelementptr inbounds i32, ptr %off, i64 %iv
+  %l = load i32, ptr %gep.off, align 4
+  %add = add nsw i32 %l, 5
+  %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+  store i32 %add, ptr %gep, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
-- 
GitLab


From 11a6799740f824282650aa9ec249b55dcf1a8aae Mon Sep 17 00:00:00 2001
From: Jacob Lambert <jacob.lambert@amd.com>
Date: Wed, 8 May 2024 08:11:15 -0700
Subject: [PATCH 0184/1206] [clang][CodeGen] Omit pre-opt link when post-opt is
 link requested (#85672)

Currently, when the -relink-builtin-bitcodes-postop option is used we
link builtin bitcodes twice: once before optimization, and again after
optimization.

With this change, we omit the pre-opt linking when the option is set,
and we rename the option to the following:

  -Xclang -mlink-builtin-bitcodes-postopt
 (-Xclang -mno-link-builtin-bitcodes-postopt)

The goal of this change is to reduce compile time. We do lose the
theoretical benefits of pre-opt linking, but in practice these are small
than the overhead of linking twice. However we may be able to address
this in a future patch by adjusting the position of the builtin-bitcode
linking pass.

Compilations not setting the option are unaffected
---
 clang/include/clang/Basic/CodeGenOptions.def  |  1 +
 clang/include/clang/Driver/Options.td         |  5 +++
 clang/lib/CodeGen/BackendConsumer.h           |  4 --
 clang/lib/CodeGen/BackendUtil.cpp             | 12 +-----
 clang/lib/CodeGen/CodeGenAction.cpp           | 37 +------------------
 clang/lib/CodeGen/LinkInModulesPass.cpp       |  8 +---
 .../test/CodeGen/linking-bitcode-postopt.cpp  | 31 ++++++++++++++++
 7 files changed, 43 insertions(+), 55 deletions(-)
 create mode 100644 clang/test/CodeGen/linking-bitcode-postopt.cpp

diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index b964e4557478..07b0ca1691a6 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -309,6 +309,7 @@ CODEGENOPT(UnrollLoops       , 1, 0) ///< Control whether loops are unrolled.
 CODEGENOPT(RerollLoops       , 1, 0) ///< Control whether loops are rerolled.
 CODEGENOPT(NoUseJumpTables   , 1, 0) ///< Set when -fno-jump-tables is enabled.
 VALUE_CODEGENOPT(UnwindTables, 2, 0) ///< Unwind tables (1) or asynchronous unwind tables (2)
+CODEGENOPT(LinkBitcodePostopt, 1, 0) ///< Link builtin bitcodes after optimization pipeline.
 CODEGENOPT(VectorizeLoop     , 1, 0) ///< Run loop vectorizer.
 CODEGENOPT(VectorizeSLP      , 1, 0) ///< Run SLP vectorizer.
 CODEGENOPT(ProfileSampleAccurate, 1, 0) ///< Sample profile is accurate.
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 734ae7833f5c..322cc12af34a 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -7092,6 +7092,11 @@ def mlink_bitcode_file : Separate<["-"], "mlink-bitcode-file">,
 def mlink_builtin_bitcode : Separate<["-"], "mlink-builtin-bitcode">,
   HelpText<"Link and internalize needed symbols from the given bitcode file "
            "before performing optimizations.">;
+defm link_builtin_bitcode_postopt: BoolMOption<"link-builtin-bitcode-postopt",
+  CodeGenOpts<"LinkBitcodePostopt">, DefaultFalse,
+  PosFlag<SetTrue, [], [ClangOption], "Link builtin bitcodes after the "
+  "optimization pipeline">,
+  NegFlag<SetFalse, [], [ClangOption]>>;
 def vectorize_loops : Flag<["-"], "vectorize-loops">,
   HelpText<"Run the Loop vectorization passes">,
   MarshallingInfoFlag<CodeGenOpts<"VectorizeLoop">>;
diff --git a/clang/lib/CodeGen/BackendConsumer.h b/clang/lib/CodeGen/BackendConsumer.h
index fd0f1984d6c0..f9edbe901bb8 100644
--- a/clang/lib/CodeGen/BackendConsumer.h
+++ b/clang/lib/CodeGen/BackendConsumer.h
@@ -115,10 +115,6 @@ public:
   // Links each entry in LinkModules into our module.  Returns true on error.
   bool LinkInModules(llvm::Module *M, bool ShouldLinkFiles = true);
 
-  // Load a bitcode module from -mlink-builtin-bitcode option using
-  // methods from a BackendConsumer instead of CompilerInstance
-  bool ReloadModules(llvm::Module *M);
-
   /// Get the best possible source location to represent a diagnostic that
   /// may have associated debug info.
   const FullSourceLoc getBestLocationFromDebugLoc(
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 119ec4704002..90985c08fe7f 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -120,11 +120,6 @@ static cl::opt<PGOOptions::ColdFuncOpt> ClPGOColdFuncAttr(
                           "Mark cold functions with optnone.")));
 
 extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate;
-
-// Re-link builtin bitcodes after optimization
-cl::opt<bool> ClRelinkBuiltinBitcodePostop(
-    "relink-builtin-bitcode-postop", cl::Optional,
-    cl::desc("Re-link builtin bitcodes after optimization."));
 } // namespace llvm
 
 namespace {
@@ -1055,11 +1050,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
     }
   }
 
-  // Re-link against any bitcodes supplied via the -mlink-builtin-bitcode option
-  // Some optimizations may generate new function calls that would not have
-  // been linked pre-optimization (i.e. fused sincos calls generated by
-  // AMDGPULibCalls::fold_sincos.)
-  if (ClRelinkBuiltinBitcodePostop)
+  // Link against bitcodes supplied via the -mlink-builtin-bitcode option
+  if (CodeGenOpts.LinkBitcodePostopt)
     MPM.addPass(LinkInModulesPass(BC, false));
 
   // Add a verifier pass if requested. We don't have to do this if the action
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index 1a6b628016f7..0255f05b1f90 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -60,10 +60,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "codegenaction"
 
-namespace llvm {
-extern cl::opt<bool> ClRelinkBuiltinBitcodePostop;
-}
-
 namespace clang {
 class BackendConsumer;
 class ClangDiagnosticHandler final : public DiagnosticHandler {
@@ -232,35 +228,6 @@ void BackendConsumer::HandleInterestingDecl(DeclGroupRef D) {
     HandleTopLevelDecl(D);
 }
 
-bool BackendConsumer::ReloadModules(llvm::Module *M) {
-  for (const CodeGenOptions::BitcodeFileToLink &F :
-       CodeGenOpts.LinkBitcodeFiles) {
-    auto BCBuf = FileMgr.getBufferForFile(F.Filename);
-    if (!BCBuf) {
-      Diags.Report(diag::err_cannot_open_file)
-          << F.Filename << BCBuf.getError().message();
-      LinkModules.clear();
-      return true;
-    }
-
-    LLVMContext &Ctx = getModule()->getContext();
-    Expected<std::unique_ptr<llvm::Module>> ModuleOrErr =
-        getOwningLazyBitcodeModule(std::move(*BCBuf), Ctx);
-
-    if (!ModuleOrErr) {
-      handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
-        Diags.Report(diag::err_cannot_open_file) << F.Filename << EIB.message();
-      });
-      LinkModules.clear();
-      return true;
-    }
-    LinkModules.push_back({std::move(ModuleOrErr.get()), F.PropagateAttrs,
-                           F.Internalize, F.LinkFlags});
-  }
-
-  return false; // success
-}
-
 // Links each entry in LinkModules into our module.  Returns true on error.
 bool BackendConsumer::LinkInModules(llvm::Module *M, bool ShouldLinkFiles) {
   for (auto &LM : LinkModules) {
@@ -362,7 +329,7 @@ void BackendConsumer::HandleTranslationUnit(ASTContext &C) {
   }
 
   // Link each LinkModule into our module.
-  if (LinkInModules(getModule()))
+  if (!CodeGenOpts.LinkBitcodePostopt && LinkInModules(getModule()))
     return;
 
   for (auto &F : getModule()->functions()) {
@@ -1232,7 +1199,7 @@ void CodeGenAction::ExecuteAction() {
                          std::move(LinkModules), *VMContext, nullptr);
 
   // Link in each pending link module.
-  if (Result.LinkInModules(&*TheModule))
+  if (!CodeGenOpts.LinkBitcodePostopt && Result.LinkInModules(&*TheModule))
     return;
 
   // PR44896: Force DiscardValueNames as false. DiscardValueNames cannot be
diff --git a/clang/lib/CodeGen/LinkInModulesPass.cpp b/clang/lib/CodeGen/LinkInModulesPass.cpp
index 929539cc8f33..c3831aae13b6 100644
--- a/clang/lib/CodeGen/LinkInModulesPass.cpp
+++ b/clang/lib/CodeGen/LinkInModulesPass.cpp
@@ -28,12 +28,8 @@ PreservedAnalyses LinkInModulesPass::run(Module &M, ModuleAnalysisManager &AM) {
   if (!BC)
     return PreservedAnalyses::all();
 
-  // Re-load bitcode modules from files
-  if (BC->ReloadModules(&M))
-    report_fatal_error("Bitcode module re-loading failed, aborted!");
-
   if (BC->LinkInModules(&M, ShouldLinkFiles))
-    report_fatal_error("Bitcode module re-linking failed, aborted!");
+    report_fatal_error("Bitcode module postopt linking failed, aborted!");
 
-  return PreservedAnalyses::all();
+  return PreservedAnalyses::none();
 }
diff --git a/clang/test/CodeGen/linking-bitcode-postopt.cpp b/clang/test/CodeGen/linking-bitcode-postopt.cpp
new file mode 100644
index 000000000000..a0486ed0c9a8
--- /dev/null
+++ b/clang/test/CodeGen/linking-bitcode-postopt.cpp
@@ -0,0 +1,31 @@
+// REQUIRES: amdgpu-registered-target
+
+// Test that -mlink-bitcode-postopt correctly enables LinkInModulesPass
+
+// RUN: %clang_cc1 -triple amdgcn-- -emit-llvm-bc -o /dev/null \
+// RUN:   -mllvm -print-pipeline-passes \
+// RUN: %s 2>&1 | FileCheck --check-prefixes=DEFAULT %s
+
+// DEFAULT-NOT: LinkInModulesPass
+
+// RUN: %clang_cc1 -triple amdgcn-- -emit-llvm-bc -o /dev/null \
+// RUN:   -mllvm -print-pipeline-passes \
+// RUN:   -mlink-builtin-bitcode-postopt \
+// RUN: %s 2>&1 | FileCheck --check-prefixes=OPTION-POSITIVE %s
+
+// OPTION-POSITIVE: LinkInModulesPass
+
+// RUN: %clang_cc1 -triple amdgcn-- -emit-llvm-bc -o /dev/null \
+// RUN:   -mllvm -print-pipeline-passes \
+// RUN:   -mno-link-builtin-bitcode-postopt \
+// RUN: %s 2>&1 | FileCheck --check-prefixes=OPTION-NEGATIVE %s
+
+// OPTION-NEGATIVE-NOT: LinkInModulesPass
+
+// RUN: %clang_cc1 -triple amdgcn-- -emit-llvm-bc -o /dev/null \
+// RUN:   -mllvm -print-pipeline-passes \
+// RUN:   -mlink-builtin-bitcode-postopt \
+// RUN:   -mno-link-builtin-bitcode-postopt \
+// RUN: %s 2>&1 | FileCheck --check-prefixes=OPTION-POSITIVE-NEGATIVE %s
+
+// OPTION-POSITIVE-NEGATIVE-NOT: LinkInModulesPass
-- 
GitLab


From a55127281b2ed5f24f848b9e5c70870ad170bc3f Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 8 May 2024 13:43:27 +0100
Subject: [PATCH 0185/1206] [CostModel][X86] getGSVectorCost - add cost kind
 support

Don't just assume gather/scatter non-throughput costs are 1 - latency and sizelatency (#uops) costs will be high, and codesize (#instructions) needs to account splitting.
---
 .../lib/Target/X86/X86TargetTransformInfo.cpp |  31 ++--
 .../X86/masked-intrinsic-codesize.ll          |  38 ++---
 .../CostModel/X86/masked-intrinsic-latency.ll | 132 +++++++++---------
 .../X86/masked-intrinsic-sizelatency.ll       | 132 +++++++++---------
 4 files changed, 167 insertions(+), 166 deletions(-)

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 2257370912bd..6b7cddc6d72e 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5828,14 +5828,17 @@ InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode,
                                          Alignment, AddressSpace);
   }
 
+  // If we didn't split, this will be a single gather/scatter instruction.
+  if (CostKind == TTI::TCK_CodeSize)
+    return 1;
+
   // The gather / scatter cost is given by Intel architects. It is a rough
   // number since we are looking at one instruction in a time.
-  const int GSOverhead = (Opcode == Instruction::Load)
-                             ? getGatherOverhead()
-                             : getScatterOverhead();
+  const int GSOverhead = (Opcode == Instruction::Load) ? getGatherOverhead()
+                                                       : getScatterOverhead();
   return GSOverhead + VF * getMemoryOpCost(Opcode, SrcVTy->getScalarType(),
                                            MaybeAlign(Alignment), AddressSpace,
-                                           TTI::TCK_RecipThroughput);
+                                           CostKind);
 }
 
 /// Return the cost of full scalarization of gather / scatter operation.
@@ -5892,19 +5895,17 @@ InstructionCost X86TTIImpl::getGatherScatterOpCost(
     unsigned Opcode, Type *SrcVTy, const Value *Ptr, bool VariableMask,
     Align Alignment, TTI::TargetCostKind CostKind,
     const Instruction *I = nullptr) {
-  if (CostKind != TTI::TCK_RecipThroughput) {
-    if ((Opcode == Instruction::Load &&
-         isLegalMaskedGather(SrcVTy, Align(Alignment)) &&
-         !forceScalarizeMaskedGather(cast<VectorType>(SrcVTy),
-                                     Align(Alignment))) ||
-        (Opcode == Instruction::Store &&
-         isLegalMaskedScatter(SrcVTy, Align(Alignment)) &&
-         !forceScalarizeMaskedScatter(cast<VectorType>(SrcVTy),
-                                      Align(Alignment))))
-      return 1;
+  if (CostKind != TTI::TCK_RecipThroughput &&
+      ((Opcode == Instruction::Load &&
+        (!isLegalMaskedGather(SrcVTy, Align(Alignment)) ||
+         forceScalarizeMaskedGather(cast<VectorType>(SrcVTy),
+                                    Align(Alignment)))) ||
+       (Opcode == Instruction::Store &&
+        (!isLegalMaskedScatter(SrcVTy, Align(Alignment)) ||
+         forceScalarizeMaskedScatter(cast<VectorType>(SrcVTy),
+                                     Align(Alignment))))))
     return BaseT::getGatherScatterOpCost(Opcode, SrcVTy, Ptr, VariableMask,
                                          Alignment, CostKind, I);
-  }
 
   assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter");
   PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-codesize.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-codesize.ll
index 320774175944..1e5c02afc2b3 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-codesize.ll
@@ -840,20 +840,20 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_gather'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
@@ -871,7 +871,7 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
@@ -879,7 +879,7 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
@@ -898,7 +898,7 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
@@ -906,7 +906,7 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
@@ -1094,7 +1094,7 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
@@ -1102,7 +1102,7 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
@@ -1121,7 +1121,7 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
@@ -1129,7 +1129,7 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
@@ -1909,7 +1909,7 @@ define <16 x float> @test_gather_16f32_const_mask(ptr %base, <16 x i32> %ind) {
 ; SKL-LABEL: 'test_gather_16f32_const_mask'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_const_mask'
@@ -1953,7 +1953,7 @@ define <16 x float> @test_gather_16f32_var_mask(ptr %base, <16 x i32> %ind, <16
 ; SKL-LABEL: 'test_gather_16f32_var_mask'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_var_mask'
@@ -1997,13 +1997,13 @@ define <16 x float> @test_gather_16f32_ra_var_mask(<16 x ptr> %ptrs, <16 x i32>
 ; SKL-LABEL: 'test_gather_16f32_ra_var_mask'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_ra_var_mask'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
@@ -2051,7 +2051,7 @@ define <16 x float> @test_gather_16f32_const_mask2(ptr %base, <16 x i32> %ind) {
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x ptr> %broadcast.splat, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_const_mask2'
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-latency.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-latency.ll
index 781a000149e4..14dc561edc34 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-latency.ll
@@ -840,22 +840,22 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_gather'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
@@ -867,20 +867,20 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; KNL-LABEL: 'masked_gather'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
@@ -894,21 +894,21 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKX-LABEL: 'masked_gather'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
@@ -1090,20 +1090,20 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; KNL-LABEL: 'masked_scatter'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
@@ -1117,21 +1117,21 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKX-LABEL: 'masked_scatter'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
@@ -1804,7 +1804,7 @@ define <2 x double> @test_gather_2f64(<2 x ptr> %ptrs, <2 x i1> %mask, <2 x doub
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x double> %res
 ;
 ; SKL-LABEL: 'test_gather_2f64'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x double> %res
 ;
 ; AVX512-LABEL: 'test_gather_2f64'
@@ -1833,7 +1833,7 @@ define <4 x i32> @test_gather_4i32(<4 x ptr> %ptrs, <4 x i1> %mask, <4 x i32> %s
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; SKL-LABEL: 'test_gather_4i32'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; KNL-LABEL: 'test_gather_4i32'
@@ -1841,7 +1841,7 @@ define <4 x i32> @test_gather_4i32(<4 x ptr> %ptrs, <4 x i1> %mask, <4 x i32> %s
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; SKX-LABEL: 'test_gather_4i32'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
   %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
@@ -1866,7 +1866,7 @@ define <4 x i32> @test_gather_4i32_const_mask(<4 x ptr> %ptrs, <4 x i32> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; SKL-LABEL: 'test_gather_4i32_const_mask'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; KNL-LABEL: 'test_gather_4i32_const_mask'
@@ -1874,7 +1874,7 @@ define <4 x i32> @test_gather_4i32_const_mask(<4 x ptr> %ptrs, <4 x i32> %src0)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; SKX-LABEL: 'test_gather_4i32_const_mask'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
   %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
@@ -1909,13 +1909,13 @@ define <16 x float> @test_gather_16f32_const_mask(ptr %base, <16 x i32> %ind) {
 ; SKL-LABEL: 'test_gather_16f32_const_mask'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_const_mask'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
@@ -1953,13 +1953,13 @@ define <16 x float> @test_gather_16f32_var_mask(ptr %base, <16 x i32> %ind, <16
 ; SKL-LABEL: 'test_gather_16f32_var_mask'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_var_mask'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
@@ -1997,13 +1997,13 @@ define <16 x float> @test_gather_16f32_ra_var_mask(<16 x ptr> %ptrs, <16 x i32>
 ; SKL-LABEL: 'test_gather_16f32_ra_var_mask'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_ra_var_mask'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
@@ -2051,7 +2051,7 @@ define <16 x float> @test_gather_16f32_const_mask2(ptr %base, <16 x i32> %ind) {
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x ptr> %broadcast.splat, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_const_mask2'
@@ -2059,7 +2059,7 @@ define <16 x float> @test_gather_16f32_const_mask2(ptr %base, <16 x i32> %ind) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x ptr> %broadcast.splat, <16 x i64> %sext_ind
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
   %broadcast.splatinsert = insertelement <16 x ptr> undef, ptr %base, i32 0
@@ -2118,7 +2118,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %broadcast.splatinsert = insertelement <16 x ptr> undef, ptr %base, i32 0
@@ -2144,7 +2144,7 @@ define void @test_scatter_8i32(<8 x i32>%a1, <8 x ptr> %ptr, <8 x i1>%mask) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_scatter_8i32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
@@ -2169,7 +2169,7 @@ define void @test_scatter_4i32(<4 x i32>%a1, <4 x ptr> %ptr, <4 x i1>%mask) {
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SKX-LABEL: 'test_scatter_4i32'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
@@ -2204,7 +2204,7 @@ define <4 x float> @test_gather_4f32(ptr %ptr, <4 x i32> %ind, <4 x i1>%mask) {
 ; SKL-LABEL: 'test_gather_4f32'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x float> %res
 ;
 ; KNL-LABEL: 'test_gather_4f32'
@@ -2216,7 +2216,7 @@ define <4 x float> @test_gather_4f32(ptr %ptr, <4 x i32> %ind, <4 x i1>%mask) {
 ; SKX-LABEL: 'test_gather_4f32'
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x float> %res
 ;
   %sext_ind = sext <4 x i32> %ind to <4 x i64>
@@ -2254,7 +2254,7 @@ define <4 x float> @test_gather_4f32_const_mask(ptr %ptr, <4 x i32> %ind) {
 ; SKL-LABEL: 'test_gather_4f32_const_mask'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x float> %res
 ;
 ; KNL-LABEL: 'test_gather_4f32_const_mask'
@@ -2266,7 +2266,7 @@ define <4 x float> @test_gather_4f32_const_mask(ptr %ptr, <4 x i32> %ind) {
 ; SKX-LABEL: 'test_gather_4f32_const_mask'
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x float> %res
 ;
   %sext_ind = sext <4 x i32> %ind to <4 x i64>
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-sizelatency.ll
index 6d8b344c0002..a030068dfaf5 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-sizelatency.ll
@@ -840,22 +840,22 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_gather'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
@@ -867,20 +867,20 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; KNL-LABEL: 'masked_gather'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
@@ -894,21 +894,21 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKX-LABEL: 'masked_gather'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
@@ -1090,20 +1090,20 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; KNL-LABEL: 'masked_scatter'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
@@ -1117,21 +1117,21 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 0
 ;
 ; SKX-LABEL: 'masked_scatter'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
@@ -1804,7 +1804,7 @@ define <2 x double> @test_gather_2f64(<2 x ptr> %ptrs, <2 x i1> %mask, <2 x doub
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x double> %res
 ;
 ; SKL-LABEL: 'test_gather_2f64'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x double> %res
 ;
 ; AVX512-LABEL: 'test_gather_2f64'
@@ -1833,7 +1833,7 @@ define <4 x i32> @test_gather_4i32(<4 x ptr> %ptrs, <4 x i1> %mask, <4 x i32> %s
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; SKL-LABEL: 'test_gather_4i32'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; KNL-LABEL: 'test_gather_4i32'
@@ -1841,7 +1841,7 @@ define <4 x i32> @test_gather_4i32(<4 x ptr> %ptrs, <4 x i1> %mask, <4 x i32> %s
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; SKX-LABEL: 'test_gather_4i32'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
   %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
@@ -1866,7 +1866,7 @@ define <4 x i32> @test_gather_4i32_const_mask(<4 x ptr> %ptrs, <4 x i32> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; SKL-LABEL: 'test_gather_4i32_const_mask'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; KNL-LABEL: 'test_gather_4i32_const_mask'
@@ -1874,7 +1874,7 @@ define <4 x i32> @test_gather_4i32_const_mask(<4 x ptr> %ptrs, <4 x i32> %src0)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
 ; SKX-LABEL: 'test_gather_4i32_const_mask'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %res
 ;
   %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
@@ -1909,13 +1909,13 @@ define <16 x float> @test_gather_16f32_const_mask(ptr %base, <16 x i32> %ind) {
 ; SKL-LABEL: 'test_gather_16f32_const_mask'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_const_mask'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
@@ -1953,13 +1953,13 @@ define <16 x float> @test_gather_16f32_var_mask(ptr %base, <16 x i32> %ind, <16
 ; SKL-LABEL: 'test_gather_16f32_var_mask'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_var_mask'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
@@ -1997,13 +1997,13 @@ define <16 x float> @test_gather_16f32_ra_var_mask(<16 x ptr> %ptrs, <16 x i32>
 ; SKL-LABEL: 'test_gather_16f32_ra_var_mask'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_ra_var_mask'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
@@ -2051,7 +2051,7 @@ define <16 x float> @test_gather_16f32_const_mask2(ptr %base, <16 x i32> %ind) {
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x ptr> %broadcast.splat, <16 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
 ; AVX512-LABEL: 'test_gather_16f32_const_mask2'
@@ -2059,7 +2059,7 @@ define <16 x float> @test_gather_16f32_const_mask2(ptr %base, <16 x i32> %ind) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x ptr> %broadcast.splat, <16 x i64> %sext_ind
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x float> %res
 ;
   %broadcast.splatinsert = insertelement <16 x ptr> undef, ptr %base, i32 0
@@ -2118,7 +2118,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %broadcast.splatinsert = insertelement <16 x ptr> undef, ptr %base, i32 0
@@ -2144,7 +2144,7 @@ define void @test_scatter_8i32(<8 x i32>%a1, <8 x ptr> %ptr, <8 x i1>%mask) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_scatter_8i32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
@@ -2169,7 +2169,7 @@ define void @test_scatter_4i32(<4 x i32>%a1, <4 x ptr> %ptr, <4 x i1>%mask) {
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SKX-LABEL: 'test_scatter_4i32'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
@@ -2204,7 +2204,7 @@ define <4 x float> @test_gather_4f32(ptr %ptr, <4 x i32> %ind, <4 x i1>%mask) {
 ; SKL-LABEL: 'test_gather_4f32'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x float> %res
 ;
 ; KNL-LABEL: 'test_gather_4f32'
@@ -2216,7 +2216,7 @@ define <4 x float> @test_gather_4f32(ptr %ptr, <4 x i32> %ind, <4 x i1>%mask) {
 ; SKX-LABEL: 'test_gather_4f32'
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x float> %res
 ;
   %sext_ind = sext <4 x i32> %ind to <4 x i64>
@@ -2254,7 +2254,7 @@ define <4 x float> @test_gather_4f32_const_mask(ptr %ptr, <4 x i32> %ind) {
 ; SKL-LABEL: 'test_gather_4f32_const_mask'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x float> %res
 ;
 ; KNL-LABEL: 'test_gather_4f32_const_mask'
@@ -2266,7 +2266,7 @@ define <4 x float> @test_gather_4f32_const_mask(ptr %ptr, <4 x i32> %ind) {
 ; SKX-LABEL: 'test_gather_4f32_const_mask'
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x float> %res
 ;
   %sext_ind = sext <4 x i32> %ind to <4 x i64>
-- 
GitLab


From 29b7eb8400f48fe7d8de3cb3741584c329ec597c Mon Sep 17 00:00:00 2001
From: Juergen Ributzka <juergen@ributzka.de>
Date: Wed, 8 May 2024 08:58:55 -0700
Subject: [PATCH 0186/1206] [llvm][stackmaps] Include pristine registers for
 liveness computation. (#90529)

Users of stackmaps and patchpoints need to add all pristine registers to
the
spill set, even so they don't need to be all preserved.

This fixes the liveness computation for stackmaps to include pristine
registers.

This fixes rdar://21228337.
---
 llvm/lib/CodeGen/StackMapLivenessAnalysis.cpp |   3 +-
 .../test/CodeGen/AArch64/stackmap-liveness.ll |  80 +++++++++++-
 llvm/test/CodeGen/X86/stackmap-liveness.ll    | 115 +++++++++++++++---
 3 files changed, 175 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/CodeGen/StackMapLivenessAnalysis.cpp b/llvm/lib/CodeGen/StackMapLivenessAnalysis.cpp
index 778ac1f5701c..687acd90b405 100644
--- a/llvm/lib/CodeGen/StackMapLivenessAnalysis.cpp
+++ b/llvm/lib/CodeGen/StackMapLivenessAnalysis.cpp
@@ -126,8 +126,7 @@ bool StackMapLiveness::calculateLiveness(MachineFunction &MF) {
   for (auto &MBB : MF) {
     LLVM_DEBUG(dbgs() << "****** BB " << MBB.getName() << " ******\n");
     LiveRegs.init(*TRI);
-    // FIXME: This should probably be addLiveOuts().
-    LiveRegs.addLiveOutsNoPristines(MBB);
+    LiveRegs.addLiveOuts(MBB);
     bool HasStackMap = false;
     // Reverse iterate over all instructions and add the current live register
     // set to an instruction if we encounter a patchpoint instruction.
diff --git a/llvm/test/CodeGen/AArch64/stackmap-liveness.ll b/llvm/test/CodeGen/AArch64/stackmap-liveness.ll
index e1f9ffe42a77..c19c2623e322 100644
--- a/llvm/test/CodeGen/AArch64/stackmap-liveness.ll
+++ b/llvm/test/CodeGen/AArch64/stackmap-liveness.ll
@@ -27,16 +27,88 @@ define i64 @stackmap_liveness(i1 %c) {
 ; Padding
 ; CHECK-NEXT:   .p2align  3
 ; CHECK-NEXT:   .short  0
-; Num LiveOut Entries: 1
-; CHECK-NEXT:   .short  2
-; LiveOut Entry 0: X0
+; Num LiveOut Entries: 20
+; CHECK-NEXT:   .short  20
+; LiveOut Entry 1: X0
 ; CHECK-NEXT:   .short 0
 ; CHECK-NEXT:   .byte 0
 ; CHECK-NEXT:   .byte 8
-; LiveOut Entry 1: SP
+; LiveOut Entry 2:
+; CHECK-NEXT:   .short 19
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 3:
+; CHECK-NEXT:   .short 20
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 4:
+; CHECK-NEXT:   .short 21
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 5:
+; CHECK-NEXT:   .short 22
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 6:
+; CHECK-NEXT:   .short 23
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 7:
+; CHECK-NEXT:   .short 24
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 8:
+; CHECK-NEXT:   .short 25
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 9:
+; CHECK-NEXT:   .short 26
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 10:
+; CHECK-NEXT:   .short 27
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 11:
+; CHECK-NEXT:   .short 28
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 12: SP
 ; CHECK-NEXT:   .short 31
 ; CHECK-NEXT:   .byte 0
 ; CHECK-NEXT:   .byte 8
+; LiveOut Entry 13:
+; CHECK-NEXT:   .short 72
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 14:
+; CHECK-NEXT:   .short 73
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 15:
+; CHECK-NEXT:   .short 74
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 16:
+; CHECK-NEXT:   .short 75
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 17:
+; CHECK-NEXT:   .short 76
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 18:
+; CHECK-NEXT:   .short 77
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 19:
+; CHECK-NEXT:   .short 78
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
+; LiveOut Entry 20:
+; CHECK-NEXT:   .short 79
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .byte 8
 ; Align
 ; CHECK-NEXT:   .p2align  3
   %1 = select i1 %c, i64 1, i64 2
diff --git a/llvm/test/CodeGen/X86/stackmap-liveness.ll b/llvm/test/CodeGen/X86/stackmap-liveness.ll
index 798eab9249df..10a8f950baeb 100644
--- a/llvm/test/CodeGen/X86/stackmap-liveness.ll
+++ b/llvm/test/CodeGen/X86/stackmap-liveness.ll
@@ -46,9 +46,29 @@ entry:
 ; Padding
 ; PATCH-NEXT:   .p2align  3
 ; PATCH-NEXT:   .short  0
-; Num LiveOut Entries: 1
-; PATCH-NEXT:   .short  1
-; LiveOut Entry 1: %ymm2 (16 bytes) --> %xmm2
+; Num LiveOut Entries: 6
+; PATCH-NEXT:   .short  6
+; LiveOut Entry 1:
+; PATCH-NEXT:   .short 3
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 2:
+; PATCH-NEXT:   .short 12
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 3:
+; PATCH-NEXT:   .short 13
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 4:
+; PATCH-NEXT:   .short 14
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 5:
+; PATCH-NEXT:   .short 15
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 6: %ymm2 (16 bytes) --> %xmm2
 ; PATCH-NEXT:   .short  19
 ; PATCH-NEXT:   .byte 0
 ; PATCH-NEXT:   .byte 16
@@ -79,25 +99,46 @@ entry:
 ; Padding
 ; PATCH-NEXT:   .p2align  3
 ; PATCH-NEXT:   .short  0
-; Num LiveOut Entries: 5
-; PATCH-NEXT:   .short  5
+; Num LiveOut Entries: 10
+; PATCH-NEXT:   .short 10
+
 ; LiveOut Entry 1: %rax (1 bytes) --> %al or %ah
 ; PATCH-NEXT:   .short  0
 ; PATCH-NEXT:   .byte 0
 ; PATCH-NEXT:   .byte 1
-; LiveOut Entry 2: %r8 (8 bytes)
+; LiveOut Entry 2:
+; PATCH-NEXT:   .short 3
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 3: %r8 (8 bytes)
 ; PATCH-NEXT:   .short  8
 ; PATCH-NEXT:   .byte 0
 ; PATCH-NEXT:   .byte 8
-; LiveOut Entry 3: %ymm0 (32 bytes)
+; LiveOut Entry 4:
+; PATCH-NEXT:   .short 12
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 5:
+; PATCH-NEXT:   .short 13
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 6:
+; PATCH-NEXT:   .short 14
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 7:
+; PATCH-NEXT:   .short 15
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 8: %ymm0 (32 bytes)
 ; PATCH-NEXT:   .short  17
 ; PATCH-NEXT:   .byte 0
 ; PATCH-NEXT:   .byte 32
-; LiveOut Entry 4: %ymm1 (32 bytes)
+; LiveOut Entry 9: %ymm1 (32 bytes)
 ; PATCH-NEXT:   .short  18
 ; PATCH-NEXT:   .byte 0
 ; PATCH-NEXT:   .byte 32
-; LiveOut Entry 5: %ymm2 (16 bytes) --> %xmm2
+; LiveOut Entry 10: %ymm2 (16 bytes) --> %xmm2
 ; PATCH-NEXT:   .short  19
 ; PATCH-NEXT:   .byte 0
 ; PATCH-NEXT:   .byte 16
@@ -125,13 +166,33 @@ entry:
 ; Padding
 ; PATCH-NEXT:   .p2align  3
 ; PATCH-NEXT:   .short  0
-; Num LiveOut Entries: 2
-; PATCH-NEXT:   .short  2
-; LiveOut Entry 1: %rsp (8 bytes)
+; Num LiveOut Entries: 7
+; PATCH-NEXT:   .short 7
+; LiveOut Entry 1:
+; PATCH-NEXT:   .short 3
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 2: %rsp (8 bytes)
 ; PATCH-NEXT:   .short  7
 ; PATCH-NEXT:   .byte 0
 ; PATCH-NEXT:   .byte 8
-; LiveOut Entry 2: %ymm2 (16 bytes) --> %xmm2
+; LiveOut Entry 3:
+; PATCH-NEXT:   .short 12
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 4:
+; PATCH-NEXT:   .short 13
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 5:
+; PATCH-NEXT:   .short 14
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 6:
+; PATCH-NEXT:   .short 15
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 7: %ymm2 (16 bytes) --> %xmm2
 ; PATCH-NEXT:   .short  19
 ; PATCH-NEXT:   .byte 0
 ; PATCH-NEXT:   .byte 16
@@ -164,13 +225,33 @@ entry:
 ; Padding
 ; PATCH-NEXT:   .p2align  3
 ; PATCH-NEXT:   .short  0
-; Num LiveOut Entries: 2
-; PATCH-NEXT:   .short  2
-; LiveOut Entry 1: %rsp (8 bytes)
+; Num LiveOut Entries: 7
+; PATCH-NEXT:   .short 7
+; LiveOut Entry 1:
+; PATCH-NEXT:   .short 3
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 2: %rsp (8 bytes)
 ; PATCH-NEXT:   .short  7
 ; PATCH-NEXT:   .byte 0
 ; PATCH-NEXT:   .byte 8
-; LiveOut Entry 2: %ymm2 (16 bytes) --> %xmm2
+; LiveOut Entry 3:
+; PATCH-NEXT:   .short 12
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 4:
+; PATCH-NEXT:   .short 13
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 5:
+; PATCH-NEXT:   .short 14
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 6:
+; PATCH-NEXT:   .short 15
+; PATCH-NEXT:   .byte 0
+; PATCH-NEXT:   .byte 8
+; LiveOut Entry 7: %ymm2 (16 bytes) --> %xmm2
 ; PATCH-NEXT:   .short  19
 ; PATCH-NEXT:   .byte 0
 ; PATCH-NEXT:   .byte 16
-- 
GitLab


From 9263318f9da22e6632f1aae3e85717ed88fde64f Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Wed, 8 May 2024 12:01:22 -0400
Subject: [PATCH 0187/1206] Revise the modules document for clarity (#90237)

The intention isn't to add or change the information provided, but to
improve clarity through some grammar fixes, improvements to the
markdown, and so forth.
---
 clang/docs/StandardCPlusPlusModules.rst | 1215 +++++++++++------------
 1 file changed, 604 insertions(+), 611 deletions(-)

diff --git a/clang/docs/StandardCPlusPlusModules.rst b/clang/docs/StandardCPlusPlusModules.rst
index ee57fb5da648..1c3c4d319c0e 100644
--- a/clang/docs/StandardCPlusPlusModules.rst
+++ b/clang/docs/StandardCPlusPlusModules.rst
@@ -8,109 +8,92 @@ Standard C++ Modules
 Introduction
 ============
 
-The term ``modules`` has a lot of meanings. For the users of Clang, modules may
-refer to ``Objective-C Modules``, ``Clang C++ Modules`` (or ``Clang Header Modules``,
-etc.) or ``Standard C++ Modules``. The implementation of all these kinds of modules in Clang
-has a lot of shared code, but from the perspective of users, their semantics and
-command line interfaces are very different. This document focuses on
-an introduction of how to use standard C++ modules in Clang.
-
-There is already a detailed document about `Clang modules <Modules.html>`_, it
-should be helpful to read `Clang modules <Modules.html>`_ if you want to know
-more about the general idea of modules. Since standard C++ modules have different semantics
-(and work flows) from `Clang modules`, this page describes the background and use of
-Clang with standard C++ modules.
-
-Modules exist in two forms in the C++ Language Specification. They can refer to
-either "Named Modules" or to "Header Units". This document covers both forms.
+The term ``module`` is ambiguous, as it is used to mean multiple things in
+Clang. For Clang users, a module may refer to an ``Objective-C Module``,
+`Clang Module <Modules.html>`_ (also called a ``Clang Header Module``) or a
+``C++20 Module`` (or a ``Standard C++ Module``). The implementation of all
+these kinds of modules in Clang shares a lot of code, but from the perspective
+of users their semantics and command line interfaces are very different. This
+document is an introduction to the use of C++20 modules in Clang. In the
+remainder of this document, the term ``module`` will refer to Standard C++20
+modules and the term ``Clang module`` will refer to the Clang Modules
+extension.
+
+In terms of the C++ Standard, modules consist of two components: "Named
+Modules" or "Header Units". This document covers both.
 
 Standard C++ Named modules
 ==========================
 
-This document was intended to be a manual first and foremost, however, we consider it helpful to
-introduce some language background here for readers who are not familiar with
-the new language feature. This document is not intended to be a language
-tutorial; it will only introduce necessary concepts about the
-structure and building of the project.
+In order to better understand the compiler's behavior, it is helpful to
+understand some terms and definitions for readers who are not familiar with the
+C++ feature. This document is not a tutorial on C++; it only introduces
+necessary concepts to better understand use of modules in a project.
 
 Background and terminology
 --------------------------
 
-Modules
-~~~~~~~
-
-In this document, the term ``Modules``/``modules`` refers to standard C++ modules
-feature if it is not decorated by ``Clang``.
-
-Clang Modules
-~~~~~~~~~~~~~
-
-In this document, the term ``Clang Modules``/``Clang modules`` refer to Clang
-c++ modules extension. These are also known as ``Clang header modules``,
-``Clang module map modules`` or ``Clang c++ modules``.
-
 Module and module unit
 ~~~~~~~~~~~~~~~~~~~~~~
 
-A module consists of one or more module units. A module unit is a special
-translation unit. Every module unit must have a module declaration. The syntax
-of the module declaration is:
+A module consists of one or more module units. A module unit is a special kind
+of translation unit. A module unit should almost always start with a module
+declaration. The syntax of the module declaration is:
 
 .. code-block:: c++
 
   [export] module module_name[:partition_name];
 
-Terms enclosed in ``[]`` are optional. The syntax of ``module_name`` and ``partition_name``
-in regex form corresponds to ``[a-zA-Z_][a-zA-Z_0-9\.]*``. In particular, a literal dot ``.``
-in the name has no semantic meaning (e.g. implying a hierarchy).
+Terms enclosed in ``[]`` are optional. ``module_name`` and ``partition_name``
+follow the rules for a C++ identifier, except that they may contain one or more
+period (``.``) characters. Note that a ``.`` in the name has no semantic
+meaning and does not imply any hierarchy.
 
-In this document, module units are classified into:
+In this document, module units are classified as:
 
-* Primary module interface unit.
-
-* Module implementation unit.
-
-* Module interface partition unit.
-
-* Internal module partition unit.
+* Primary module interface unit
+* Module implementation unit
+* Module partition interface unit
+* Internal module partition unit
 
 A primary module interface unit is a module unit whose module declaration is
-``export module module_name;``. The ``module_name`` here denotes the name of the
+``export module module_name;`` where ``module_name`` denotes the name of the
 module. A module should have one and only one primary module interface unit.
 
 A module implementation unit is a module unit whose module declaration is
-``module module_name;``. A module could have multiple module implementation
-units with the same declaration.
+``module module_name;``. Multiple module implementation units can be declared
+in the same module.
 
-A module interface partition unit is a module unit whose module declaration is
+A module partition interface unit is a module unit whose module declaration is
 ``export module module_name:partition_name;``. The ``partition_name`` should be
 unique within any given module.
 
-An internal module partition unit is a module unit whose module declaration
-is ``module module_name:partition_name;``. The ``partition_name`` should be
-unique within any given module.
+An internal module partition unit is a module unit whose module
+declaration is ``module module_name:partition_name;``. The ``partition_name``
+should be unique within any given module.
 
-In this document, we use the following umbrella terms:
+In this document, we use the following terms:
 
 * A ``module interface unit`` refers to either a ``primary module interface unit``
-  or a ``module interface partition unit``.
+  or a ``module partition interface unit``.
 
-* An ``importable module unit`` refers to either a ``module interface unit``
-  or a ``internal module partition unit``.
+* An ``importable module unit`` refers to either a ``module interface unit`` or
+  an ``internal module partition unit``.
 
-* A ``module partition unit`` refers to either a ``module interface partition unit``
-  or a ``internal module partition unit``.
+* A ``module partition unit`` refers to either a ``module partition interface unit``
+  or an ``internal module partition unit``.
 
-Built Module Interface file
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Built Module Interface
+~~~~~~~~~~~~~~~~~~~~~~
 
-A ``Built Module Interface file`` stands for the precompiled result of an importable module unit.
-It is also called the acronym ``BMI`` generally.
+A ``Built Module Interface`` (or ``BMI``) is the precompiled result of an
+importable module unit.
 
 Global module fragment
 ~~~~~~~~~~~~~~~~~~~~~~
 
-In a module unit, the section from ``module;`` to the module declaration is called the global module fragment.
+The ``global module fragment`` (or ``GMF``) is the code between the ``module;``
+and the module declaration within a module unit.
 
 
 How to build projects using modules
@@ -138,7 +121,7 @@ Let's see a "hello world" example that uses modules.
     return 0;
   }
 
-Then we type:
+Then, on the command line, invoke Clang like:
 
 .. code-block:: console
 
@@ -148,9 +131,9 @@ Then we type:
   Hello World!
 
 In this example, we make and use a simple module ``Hello`` which contains only a
-primary module interface unit ``Hello.cppm``.
+primary module interface unit named ``Hello.cppm``.
 
-Then let's see a little bit more complex "hello world" example which uses the 4 kinds of module units.
+A more complex "hello world" example which uses the 4 kinds of module units is:
 
 .. code-block:: c++
 
@@ -192,7 +175,7 @@ Then let's see a little bit more complex "hello world" example which uses the 4
     return 0;
   }
 
-Then we are able to compile the example by the following command:
+Then, back on the command line, invoke Clang with:
 
 .. code-block:: console
 
@@ -216,51 +199,57 @@ We explain the options in the following sections.
 How to enable standard C++ modules
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Currently, standard C++ modules are enabled automatically
-if the language standard is ``-std=c++20`` or newer.
+Standard C++ modules are enabled automatically when the language standard mode
+is ``-std=c++20`` or newer.
 
 How to produce a BMI
 ~~~~~~~~~~~~~~~~~~~~
 
-We can generate a BMI for an importable module unit by either ``--precompile``
-or ``-fmodule-output`` flags.
+To generate a BMI for an importable module unit, use either the ``--precompile``
+or ``-fmodule-output`` command line options.
 
-The ``--precompile`` option generates the BMI as the output of the compilation and the output path
-can be specified using the ``-o`` option.
+The ``--precompile`` option generates the BMI as the output of the compilation
+with the output path specified using the ``-o`` option.
 
-The ``-fmodule-output`` option generates the BMI as a by-product of the compilation.
-If ``-fmodule-output=`` is specified, the BMI will be emitted the specified location. Then if
-``-fmodule-output`` and ``-c`` are specified, the BMI will be emitted in the directory of the
-output file with the name of the input file with the new extension ``.pcm``. Otherwise, the BMI
-will be emitted in the working directory with the name of the input file with the new extension
+The ``-fmodule-output`` option generates the BMI as a by-product of the
+compilation. If ``-fmodule-output=`` is specified, the BMI will be emitted to
+the specified location. If ``-fmodule-output`` and ``-c`` are specified, the
+BMI will be emitted in the directory of the output file with the name of the
+input file with the extension ``.pcm``. Otherwise, the BMI will be emitted in
+the working directory with the name of the input file with the extension
 ``.pcm``.
 
-The style to generate BMIs by ``--precompile`` is called two-phase compilation since it takes
-2 steps to compile a source file to an object file. The style to generate BMIs by ``-fmodule-output``
-is called one-phase compilation respectively. The one-phase compilation model is simpler
-for build systems to implement and the two-phase compilation has the potential to compile faster due
-to higher parallelism. As an example, if there are two module units A and B, and B depends on A, the
-one-phase compilation model would need to compile them serially, whereas the two-phase compilation
-model may be able to compile them simultaneously if the compilation from A.pcm to A.o takes a long
-time.
-
-File name requirement
-~~~~~~~~~~~~~~~~~~~~~
-
-The file name of an ``importable module unit`` should end with ``.cppm``
-(or ``.ccm``, ``.cxxm``, ``.c++m``). The file name of a ``module implementation unit``
-should end with ``.cpp`` (or ``.cc``, ``.cxx``, ``.c++``).
-
-The file name of BMIs should end with ``.pcm``.
-The file name of the BMI of a ``primary module interface unit`` should be ``module_name.pcm``.
-The file name of BMIs of ``module partition unit`` should be ``module_name-partition_name.pcm``.
-
-If the file names use different extensions, Clang may fail to build the module.
-For example, if the filename of an ``importable module unit`` ends with ``.cpp`` instead of ``.cppm``,
-then we can't generate a BMI for the ``importable module unit`` by ``--precompile`` option
-since ``--precompile`` option now would only run preprocessor, which is equal to `-E` now.
-If we want the filename of an ``importable module unit`` ends with other suffixes instead of ``.cppm``,
-we could put ``-x c++-module`` in front of the file. For example,
+Generating BMIs with ``--precompile`` is referred to as two-phase compilation
+because it takes two steps to compile a source file to an object file.
+Generating BMIs with ``-fmodule-output`` is called one-phase compilation. The
+one-phase compilation model is simpler for build systems to implement while the
+two-phase compilation has the potential to compile faster due to higher
+parallelism. As an example, if there are two module units ``A`` and ``B``, and
+``B`` depends on ``A``, the one-phase compilation model needs to compile them
+serially, whereas the two-phase compilation model is able to be compiled as
+soon as ``A.pcm`` is available, and thus can be compiled simultaneously as the
+``A.pcm`` to ``A.o`` compilation step.
+
+File name requirements
+~~~~~~~~~~~~~~~~~~~~~~
+
+By convention, ``importable module unit`` files should use ``.cppm`` (or
+``.ccm``, ``.cxxm``, or ``.c++m``) as a file extension.
+``Module implementation unit`` files should use ``.cpp`` (or ``.cc``, ``.cxx``,
+or ``.c++``) as a file extension.
+
+A BMI should use ``.pcm`` as a file extension. The file name of the BMI for a
+``primary module interface unit`` should be ``module_name.pcm``. The file name
+of a BMI for a ``module partition unit`` should be
+``module_name-partition_name.pcm``.
+
+Clang may fail to build the module if different extensions are used. For
+example, if the filename of an ``importable module unit`` ends with ``.cpp``
+instead of ``.cppm``, then Clang cannot generate a BMI for the
+``importable module unit`` with the ``--precompile`` option because the
+``--precompile`` option would only run the preprocessor (``-E``). If using a
+different extension than the conventional one for an ``importable module unit``
+you can specify ``-x c++-module`` before the file. For example,
 
 .. code-block:: c++
 
@@ -279,8 +268,9 @@ we could put ``-x c++-module`` in front of the file. For example,
     return 0;
   }
 
-Now the filename of the ``module interface`` ends with ``.cpp`` instead of ``.cppm``,
-we can't compile them by the original command lines. But we are still able to do it by:
+In this example, the extension used by the ``module interface`` is ``.cpp``
+instead of ``.cppm``, so it cannot be compiled like the previous example, but
+it can be compiled with:
 
 .. code-block:: console
 
@@ -289,12 +279,12 @@ we can't compile them by the original command lines. But we are still able to do
   $ ./Hello.out
   Hello World!
 
-Module name requirement
-~~~~~~~~~~~~~~~~~~~~~~~
+Module name requirements
+~~~~~~~~~~~~~~~~~~~~~~~~
 
-[module.unit]p1 says:
+..
 
-.. code-block:: text
+  [module.unit]p1:
 
   All module-names either beginning with an identifier consisting of std followed by zero
   or more digits or containing a reserved identifier ([lex.name]) are reserved and shall not
@@ -302,7 +292,7 @@ Module name requirement
   module-name is a reserved identifier, the module name is reserved for use by C++ implementations;
   otherwise it is reserved for future standardization.
 
-So all of the following name is not valid by default:
+Therefore, none of the following names are valid by default:
 
 .. code-block:: text
 
@@ -312,75 +302,74 @@ So all of the following name is not valid by default:
     __test
     // and so on ...
 
-If you still want to use the reserved module names for any reason, use
-``-Wno-reserved-module-identifier`` to suppress the warning.
+Using a reserved module name is strongly discouraged, but
+``-Wno-reserved-module-identifier`` can be used to suppress the warning.
 
-How to specify the dependent BMIs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Specifying dependent BMIs
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
-There are 3 methods to specify the dependent BMIs:
+There are 3 ways to specify a dependent BMI:
 
-* (1) ``-fprebuilt-module-path=<path/to/directory>``.
-* (2) ``-fmodule-file=<path/to/BMI>`` (Deprecated).
-* (3) ``-fmodule-file=<module-name>=<path/to/BMI>``.
+1. ``-fprebuilt-module-path=<path/to/directory>``.
+2. ``-fmodule-file=<path/to/BMI>`` (Deprecated).
+3. ``-fmodule-file=<module-name>=<path/to/BMI>``.
 
-The option ``-fprebuilt-module-path`` tells the compiler the path where to search for dependent BMIs.
-It may be used multiple times just like ``-I`` for specifying paths for header files. The look up rule here is:
+The ``-fprebuilt-module-path`` option specifies the path to search for
+dependent BMIs. Multiple paths may be specified, similar to using ``-I`` to
+specify a search path for header files. When importing a module ``M``, the
+compiler looks for ``M.pcm`` in the directories specified by
+``-fprebuilt-module-path``. Similarly, when importing a partition module unit
+``M:P``, the compiler looks for ``M-P.pcm`` in the directories specified by
+``-fprebuilt-module-path``.
 
-* (1) When we import module M. The compiler would look up M.pcm in the directories specified
-  by ``-fprebuilt-module-path``.
-* (2) When we import partition module unit M:P. The compiler would look up M-P.pcm in the
-  directories specified by ``-fprebuilt-module-path``.
-
-The option ``-fmodule-file=<path/to/BMI>`` tells the compiler to load the specified BMI directly.
-The option ``-fmodule-file=<module-name>=<path/to/BMI>`` tells the compiler to load the specified BMI
-for the module specified by ``<module-name>`` when necessary. The main difference is that
+The ``-fmodule-file=<path/to/BMI>`` option causes the compiler to load the
+specified BMI directly. The ``-fmodule-file=<module-name>=<path/to/BMI>``
+option causes the compiler to load the specified BMI for the module specified
+by ``<module-name>`` when necessary. The main difference is that
 ``-fmodule-file=<path/to/BMI>`` will load the BMI eagerly, whereas
-``-fmodule-file=<module-name>=<path/to/BMI>`` will only load the BMI lazily, which is similar
-with ``-fprebuilt-module-path``. The option ``-fmodule-file=<path/to/BMI>`` for named modules is deprecated
-and is planning to be removed in future versions.
+``-fmodule-file=<module-name>=<path/to/BMI>`` will only load the BMI lazily,
+as will ``-fprebuilt-module-path``. The ``-fmodule-file=<path/to/BMI>`` option
+for named modules is deprecated and will be removed in a future version of
+Clang.
 
-In case all ``-fprebuilt-module-path=<path/to/directory>``, ``-fmodule-file=<path/to/BMI>`` and
-``-fmodule-file=<module-name>=<path/to/BMI>`` exist, the ``-fmodule-file=<path/to/BMI>`` option
-takes highest precedence and ``-fmodule-file=<module-name>=<path/to/BMI>`` will take the second
-highest precedence.
+When these options are specified in the same invocation of the compiler, the
+``-fmodule-file=<path/to/BMI>`` option takes precedence over
+``-fmodule-file=<module-name>=<path/to/BMI>``, which takes precedence over
+``-fprebuilt-module-path=<path/to/directory>``.
 
-We need to specify all the dependent (directly and indirectly) BMIs.
-See https://github.com/llvm/llvm-project/issues/62707 for detail.
+Note: all dependant BMIs must be specified explicitly, either directly or
+indirectly dependent BMIs explicitly. See
+https://github.com/llvm/llvm-project/issues/62707 for details.
 
-When we compile a ``module implementation unit``, we must specify the BMI of the corresponding
-``primary module interface unit``.
-Since the language specification says a module implementation unit implicitly imports
-the primary module interface unit.
+When compiling a ``module implementation unit``, the BMI of the corresponding
+``primary module interface unit`` must be specified because a module
+implementation unit implicitly imports the primary module interface unit.
 
   [module.unit]p8
 
   A module-declaration that contains neither an export-keyword nor a module-partition implicitly
   imports the primary module interface unit of the module as if by a module-import-declaration.
 
-All of the 3 options ``-fprebuilt-module-path=<path/to/directory>``, ``-fmodule-file=<path/to/BMI>``
-and ``-fmodule-file=<module-name>=<path/to/BMI>`` may occur multiple times.
-For example, the command line to compile ``M.cppm`` in
-the above example could be rewritten into:
+The ``-fprebuilt-module-path=<path/to/directory>``, ``-fmodule-file=<path/to/BMI>``,
+and ``-fmodule-file=<module-name>=<path/to/BMI>`` options may be specified
+multiple times. For example, the command line to compile ``M.cppm`` in
+the previous example could be rewritten as:
 
 .. code-block:: console
 
   $ clang++ -std=c++20 M.cppm --precompile -fmodule-file=M:interface_part=M-interface_part.pcm -fmodule-file=M:impl_part=M-impl_part.pcm -o M.pcm
 
 When there are multiple ``-fmodule-file=<module-name>=`` options for the same
-``<module-name>``, the last ``-fmodule-file=<module-name>=`` will override the previous
-``-fmodule-file=<module-name>=`` options.
-
-``-fprebuilt-module-path`` is more convenient and ``-fmodule-file`` is faster since
-it saves time for file lookup.
+``<module-name>``, the last ``-fmodule-file=<module-name>=`` overrides the
+previous ``-fmodule-file=<module-name>=`` option.
 
 Remember that module units still have an object counterpart to the BMI
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-It is easy to forget to compile BMIs at first since we may envision module interfaces like headers.
-However, this is not true.
-Module units are translation units. We need to compile them to object files
-and link the object files like the example shows.
+While module interfaces resemble traditional header files, they still require
+compilation. Module units are translation units, and need to be compiled to
+object files, which then need to be linked together as the following examples
+show.
 
 For example, the traditional compilation processes for headers are like:
 
@@ -400,24 +389,27 @@ And the compilation process for module units are like:
                 mod1.cppm -> clang++ mod1.cppm ... -> mod1.pcm --,--> clang++ mod1.pcm ... -> mod1.o -+
                 src2.cpp ----------------------------------------+> clang++ src2.cpp -------> src2.o -'
 
-As the diagrams show, we need to compile the BMI from module units to object files and link the object files.
-(But we can't do this for the BMI from header units. See the later section for the definition of header units)
+As the diagrams show, we need to compile the BMI from module units to object
+files and then link the object files. (However, this cannot be done for the BMI
+from header units. See the section on :ref:`header units <header-units>` for
+more details.
 
-If we want to create a module library, we can't just ship the BMIs in an archive.
-We must compile these BMIs(``*.pcm``) into object files(``*.o``) and add those object files to the archive instead.
+BMIs cannot be shipped in an archive to create a module library. Instead, the
+BMIs(``*.pcm``) are compiled into object files(``*.o``) and those object files
+are added to the archive instead.
 
-Consistency Requirement
-~~~~~~~~~~~~~~~~~~~~~~~
+Consistency Requirements
+~~~~~~~~~~~~~~~~~~~~~~~~
 
-If we envision modules as a cache to speed up compilation, then - as with other caching techniques -
-it is important to keep cache consistency.
-So **currently** Clang will do very strict check for consistency.
+Modules can be viewed as a kind of cache to speed up compilation. Thus, like
+other caching techniques, it is important to maintain cache consistency which
+is why Clang does very strict checking for consistency.
 
 Options consistency
 ^^^^^^^^^^^^^^^^^^^
 
-The language option of module units and their non-module-unit users should be consistent.
-The following example is not allowed:
+Compiler options related to the language dialect for a module unit and its
+non-module-unit uses need to be consistent. Consider the following example:
 
 .. code-block:: c++
 
@@ -432,9 +424,8 @@ The following example is not allowed:
   $ clang++ -std=c++20 M.cppm --precompile -o M.pcm
   $ clang++ -std=c++23 Use.cpp -fprebuilt-module-path=.
 
-The compiler would reject the example due to the inconsistent language options.
-Not all options are language options.
-For example, the following example is allowed:
+Clang rejects the example due to the inconsistent language standard modes. Not
+all compiler options are language dialect options, though. For example:
 
 .. code-block:: console
 
@@ -444,9 +435,12 @@ For example, the following example is allowed:
   # Inconsistent debugging level.
   $ clang++ -std=c++20 -g Use.cpp -fprebuilt-module-path=.
 
-Although the two examples have inconsistent optimization and debugging level, both of them are accepted.
+Although the optimization and debugging levels are inconsistent, these
+compilations are accepted because the compiler options do not impact the
+language dialect.
 
-Note that **currently** the compiler doesn't consider inconsistent macro definition a problem. For example:
+Note that the compiler **currently** doesn't reject inconsistent macro
+definitions (this may change in the future). For example:
 
 .. code-block:: console
 
@@ -454,43 +448,43 @@ Note that **currently** the compiler doesn't consider inconsistent macro definit
   # Inconsistent optimization level.
   $ clang++ -std=c++20 -O3 -DNDEBUG Use.cpp -fprebuilt-module-path=.
 
-Currently Clang would accept the above example. But it may produce surprising results if the
-debugging code depends on consistent use of ``NDEBUG`` also in other translation units.
+Currently, Clang accepts the above example, though it may produce surprising
+results if the debugging code depends on consistent use of ``NDEBUG`` in other
+translation units.
 
-Definitions consistency
-^^^^^^^^^^^^^^^^^^^^^^^
+Object definition consistency
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The C++ language requires that declarations of the same entity in different
+translation units have the same definition, which is known as the One
+Definition Rule (ODR). Without modules, the compiler cannot perform strong ODR
+violation checking because it only sees one translation unit at a time. With
+the use of modules, the compiler can perform checks for ODR violations across
+translation units.
 
-The C++ language defines that same declarations in different translation units should have
-the same definition, as known as ODR (One Definition Rule). Prior to modules, the translation
-units don't dependent on each other and the compiler itself can't perform a strong
-ODR violation check. With the introduction of modules, now the compiler have
-the chance to perform ODR violations with language semantics across translation units.
-
-However, in the practice, we found the existing ODR checking mechanism is not stable
-enough. Many people suffers from the false positive ODR violation diagnostics, AKA,
-the compiler are complaining two identical declarations have different definitions
-incorrectly. Also the true positive ODR violations are rarely reported.
-Also we learned that MSVC don't perform ODR check for declarations in the global module
-fragment.
-
-So in order to get better user experience, save the time checking ODR and keep consistent
-behavior with MSVC, we disabled the ODR check for the declarations in the global module
-fragment by default. Users who want more strict check can still use the
-``-Xclang -fno-skip-odr-check-in-gmf`` flag to get the ODR check enabled. It is also
-encouraged to report issues if users find false positive ODR violations or false negative ODR
-violations with the flag enabled.
+However, the current ODR checking mechanisms are not perfect. There are a
+significant number of false positive ODR violation diagnostics, where the
+compiler incorrectly diagnoses two identical declarations as having different
+definitions. Further, true positive ODR violations are not always reported.
+
+To give a better user experience, improve compilation performance, and for
+consistency with MSVC, ODR checking of declarations in the global module
+fragment is disabled by default. These checks can be enabled by specifying
+``-Xclang -fno-skip-odr-check-in-gmf`` when compiling. If the check is enabled
+and you encounter incorrect or missing diagnostics, please report them via the
+`community issue tracker <https://github.com/llvm/llvm-project/issues/>`_.
 
 ABI Impacts
 -----------
 
-This section describes the new ABI changes brought by modules.
-
-Only Itanium C++ ABI related change are mentioned
+This section describes the new ABI changes brought by modules. Only changes to
+the Itanium C++ ABI are covered.
 
-Mangling Names
-~~~~~~~~~~~~~~
+Name Mangling
+~~~~~~~~~~~~~
 
-The declarations in a module unit which are not in the global module fragment have new linkage names.
+The declarations in a module unit which are not in the global module fragment
+have new linkage names.
 
 For example,
 
@@ -501,22 +495,24 @@ For example,
     export int foo();
   }
 
-The linkage name of ``NS::foo()`` would be ``_ZN2NSW1M3fooEv``.
-This couldn't be demangled by previous versions of the debugger or demangler.
-As of LLVM 15.x, users can utilize ``llvm-cxxfilt`` to demangle this:
+The linkage name of ``NS::foo()`` is ``_ZN2NSW1M3fooEv``. This couldn't be
+demangled by previous versions of the debugger or demangler. As of LLVM 15.x,
+``llvm-cxxfilt`` can be used to demangle this:
 
 .. code-block:: console
 
   $ llvm-cxxfilt _ZN2NSW1M3fooEv
+    NS::foo@M()
 
-The result would be ``NS::foo@M()``, which reads as ``NS::foo()`` in module ``M``.
+The result should be read as ``NS::foo()`` in module ``M``.
 
-The ABI implies that we can't declare something in a module unit and define it in a non-module unit (or vice-versa),
-as this would result in linking errors.
+The ABI implies that something cannot be declared in a module unit and defined
+in a non-module unit (or vice-versa), as this would result in linking errors.
 
-If we still want to implement declarations within the compatible ABI in module unit,
-we can use the language-linkage specifier. Since the declarations in the language-linkage specifier
-is attached to the global module fragments. For example:
+Despite this, it is possible to implement declarations with a compatible ABI in
+a module unit by using a language linkage specifier because the declarations in
+the language linkage specifier are attached to the global module fragment. For
+example:
 
 .. code-block:: c++
 
@@ -530,43 +526,47 @@ Now the linkage name of ``NS::foo()`` will be ``_ZN2NS3fooEv``.
 Module Initializers
 ~~~~~~~~~~~~~~~~~~~
 
-All the importable module units are required to emit an initializer function.
-The initializer function should contain calls to importing modules first and
-all the dynamic-initializers in the current module unit then.
+All importable module units are required to emit an initializer function to
+handle the dynamic initialization of non-inline variables in the module unit.
+The importable module unit has to emit the initializer even if there is no
+dynamic initialization; otherwise, the importer may call a nonexistent
+function. The initializer function emits calls to imported modules first
+followed by calls to all to of the dynamic initializers in the current module
+unit.
 
-Translation units explicitly or implicitly importing named modules must call
-the initializer functions of the imported named modules within the sequence of
-the dynamic-initializers in the TU. Initializations of entities at namespace
-scope are appearance-ordered. This (recursively) extends into imported modules
-at the point of appearance of the import declaration.
+Translation units that explicitly or implicitly import a named module must call
+the initializer functions of the imported named module within the sequence of
+the dynamic initializers in the translation unit. Initializations of entities
+at namespace scope are appearance-ordered. This (recursively) extends to
+imported modules at the point of appearance of the import declaration.
 
-It is allowed to omit calls to importing modules if it is known empty.
-
-It is allowed to omit calls to importing modules for which is known to be called.
+If the imported module is known to be empty, the call to its initializer may be
+omitted. Additionally, if the imported module is known to have already been
+imported, the call to its initializer may be omitted.
 
 Reduced BMI
 -----------
 
-To support the 2 phase compilation model, Clang chose to put everything needed to
-produce an object into the BMI. But every consumer of the BMI, except itself, doesn't
-need such informations. It makes the BMI to larger and so may introduce unnecessary
-dependencies into the BMI. To mitigate the problem, we decided to reduce the information
-contained in the BMI.
-
-To be clear, we call the default BMI as Full BMI and the new introduced BMI as Reduced
-BMI.
+To support the two-phase compilation model, Clang puts everything needed to
+produce an object into the BMI. However, other consumers of the BMI generally
+don't need that information. This makes the BMI larger and may introduce
+unnecessary dependencies for the BMI. To mitigate the problem, Clang has a
+compiler option to reduce the information contained in the BMI. These two
+formats are known as Full BMI and Reduced BMI, respectively.
 
-Users can use ``-fexperimental-modules-reduced-bmi`` flag to enable the Reduced BMI.
+Users can use the ``-fexperimental-modules-reduced-bmi`` option to produce a
+Reduced BMI.
 
-For one phase compilation model (CMake implements this model), with
-``-fexperimental-modules-reduced-bmi``, the generated BMI will be Reduced BMI automatically.
-(The output path of the BMI is specified by ``-fmodule-output=`` as usual one phase
-compilation model).
+For the one-phase compilation model (CMake implements this model), with
+``-fexperimental-modules-reduced-bmi``, the generated BMI will be a Reduced
+BMI automatically. (The output path of the BMI is specified by
+``-fmodule-output=`` as usual with the one-phase compilation model).
 
-It is still possible to support Reduced BMI in two phase compilation model. With
-``-fexperimental-modules-reduced-bmi``, ``--precompile`` and ``-fmodule-output=`` specified,
-the generated BMI specified by ``-o`` will be full BMI and the BMI specified by
-``-fmodule-output=`` will be Reduced BMI. The dependency graph may be:
+It is also possible to produce a Reduced BMI with the two-phase compilation
+model. When ``-fexperimental-modules-reduced-bmi``, ``--precompile``, and
+``-fmodule-output=`` are specified, the generated BMI specified by ``-o`` will
+be a full BMI and the BMI specified by ``-fmodule-output=`` will be a Reduced
+BMI. The dependency graph in this case would look like:
 
 .. code-block:: none
 
@@ -577,15 +577,16 @@ the generated BMI specified by ``-o`` will be full BMI and the BMI specified by
                                                -> ...
                                                -> consumer_n.cpp
 
-We don't emit diagnostics if ``-fexperimental-modules-reduced-bmi`` is used with a non-module
-unit. This design helps the end users of one phase compilation model to perform experiments
-early without asking for the help of build systems. The users of build systems which supports
-two phase compilation model still need helps from build systems.
+Clang does not emit diagnostics when ``-fexperimental-modules-reduced-bmi`` is
+used with a non-module unit. This design permits users of the one-phase
+compilation model to try using reduced BMIs without needing to modify the build
+system. The two-phase compilation module requires build system support.
 
-Within Reduced BMI, we won't write unreachable entities from GMF, definitions of non-inline
-functions and non-inline variables. This may not be a transparent change.
-`[module.global.frag]ex2 <https://eel.is/c++draft/module.global.frag#example-2>`_ may be a good
-example:
+In a Reduced BMI, Clang does not emit unreachable entities from the global
+module fragment, or definitions of non-inline functions and non-inline
+variables. This may not be a transparent change.
+
+Consider the following example:
 
 .. code-block:: c++
 
@@ -633,22 +634,23 @@ example:
                                   // module M's interface, so is discarded
   int c = use_h<int>();           // OK
 
-In the above example, the function definition of ``N::g`` is elided from the Reduced
-BMI of ``M.cppm``. Then the use of ``use_g<int>`` in ``M-impl.cpp`` fails
-to instantiate. For such issues, users can add references to ``N::g`` in the module purview
-of ``M.cppm`` to make sure it is reachable, e.g., ``using N::g;``.
+In the above example, the function definition of ``N::g`` is elided from the
+Reduced BMI of ``M.cppm``. Then the use of ``use_g<int>`` in ``M-impl.cpp``
+fails to instantiate. For such issues, users can add references to ``N::g`` in
+the `module purview <https://eel.is/c++draft/module.unit#5>`_ of ``M.cppm`` to
+ensure it is reachable, e.g. ``using N::g;``.
 
-We think the Reduced BMI is the correct direction. But given it is a drastic change,
-we'd like to make it experimental first to avoid breaking existing users. The roadmap
-of Reduced BMI may be:
+Support for Reduced BMIs is still experimental, but it may become the default
+in the future. The expected roadmap for Reduced BMIs as of Clang 19.x is:
 
-1. ``-fexperimental-modules-reduced-bmi`` is opt in for 1~2 releases. The period depends
-on testing feedbacks.
-2. We would announce Reduced BMI is not experimental and introduce ``-fmodules-reduced-bmi``.
-and suggest users to enable this mode. This may takes 1~2 releases too.
-3. Finally we will enable this by default. When that time comes, the term BMI will refer to
-the reduced BMI today and the Full BMI will only be meaningful to build systems which
-loves to support two phase compilations.
+1. ``-fexperimental-modules-reduced-bmi`` is opt-in for 1~2 releases. The period depends
+   on user feedback and may be extended.
+2. Announce that Reduced BMIs are no longer experimental and introduce
+   ``-fmodules-reduced-bmi`` as a new option, and recommend use of the new
+   option. This transition is expected to take 1~2 additional releases as well.
+3. Finally, ``-fmodules-reduced-bmi`` will be the default. When that time
+   comes, the term BMI will refer to the Reduced BMI and the Full BMI will only
+   be meaningful to build systems which elect to support two-phase compilation.
 
 Performance Tips
 ----------------
@@ -656,13 +658,11 @@ Performance Tips
 Reduce duplications
 ~~~~~~~~~~~~~~~~~~~
 
-While it is legal to have duplicated declarations in the global module fragments
-of different module units, it is not free for clang to deal with the duplicated
-declarations. In other word, for a translation unit, it will compile slower if the
-translation unit itself and its importing module units contains a lot duplicated
-declarations.
-
-For example,
+While it is valid to have duplicated declarations in the global module fragments
+of different module units, it is not free for Clang to deal with the duplicated
+declarations. A translation unit will compile more slowly if there is a lot of
+duplicated declarations between the translation unit and modules it imports.
+For example:
 
 .. code-block:: c++
 
@@ -698,9 +698,9 @@ For example,
   import M;
   ... // use declarations from module M.
 
-When ``big.header.h`` is big enough and there are a lot of partitions,
-the compilation of ``use.cpp`` may be slower than
-the following style significantly:
+When ``big.header.h`` is big enough and there are a lot of partitions, the
+compilation of ``use.cpp`` may be significantly slower than the following
+approach:
 
 .. code-block:: c++
 
@@ -738,22 +738,21 @@ the following style significantly:
   import M;
   ... // use declarations from module M.
 
-The key part of the tip is to reduce the duplications from the text includes.
-
-Ideas for converting to modules
--------------------------------
+Reducing the duplication from textual includes is what improves compile-time
+performance.
 
-For new libraries, we encourage them to use modules completely from day one if possible.
-This will be pretty helpful to make the whole ecosystems to get ready.
+Transitioning to modules
+------------------------
 
-For many existing libraries, it may be a breaking change to refactor themselves
-into modules completely. So that many existing libraries need to provide headers and module
-interfaces for a while to not break existing users.
-Here we provide some ideas to ease the transition process for existing libraries.
-**Note that the this section is only about helping ideas instead of requirement from clang**.
+It is best for new code and libraries to use modules from the start if
+possible. However, it may be a breaking change for existing code or libraries
+to switch to modules. As a result, many existing libraries need to provide
+both headers and module interfaces for a while to not break existing users.
 
-Let's start with the case that there is no dependency or no dependent libraries providing
-modules for your library.
+This section suggests some suggestions on how to ease the transition process
+for existing libraries. **Note that this information is only intended as
+guidance, rather than as requirements to use modules in Clang.** It presumes
+the project is starting with no module-based dependencies.
 
 ABI non-breaking styles
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -776,9 +775,9 @@ export-using style
     using decl_n;
   }
 
-As the example shows, you need to include all the headers containing declarations needs
-to be exported and `using` such declarations in an `export` block. Then, basically,
-we're done.
+This example shows how to include all the headers containing declarations which
+need to be exported, and uses `using` declarations in an `export` block to
+produce the module interface.
 
 export extern-C++ style
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -799,7 +798,7 @@ export extern-C++ style
     #include "header_n.h"
   }
 
-Then in your headers (from ``header_1.h`` to ``header_n.h``), you need to define the macro:
+Headers (from ``header_1.h`` to ``header_n.h``) need to define the macro:
 
 .. code-block:: c++
 
@@ -809,9 +808,10 @@ Then in your headers (from ``header_1.h`` to ``header_n.h``), you need to define
   #define EXPORT
   #endif
 
-And you should put ``EXPORT`` to the beginning of the declarations you want to export.
+and put ``EXPORT`` on the declarations you want to export.
 
-Also it is suggested to refactor your headers to include thirdparty headers conditionally:
+Also, it is recommended to refactor headers to include third-party headers
+conditionally:
 
 .. code-block:: c++
 
@@ -823,26 +823,25 @@ Also it is suggested to refactor your headers to include thirdparty headers cond
 
   ...
 
-This may be helpful to get better diagnostic messages if you forgot to update your module
-interface unit file during maintaining.
+This can be helpful because it gives better diagnostic messages if the module
+interface unit is not properly updated when modifying code.
 
-The reasoning for the practice is that the declarations in the language linkage are considered
-to be attached to the global module. So the ABI of your library in the modular version
-wouldn't change.
+This approach works because the declarations with language linkage are attached
+to the global module. Thus, the ABI of the modular form of the library does not
+change.
 
-While this style looks not as convenient as the export-using style, it is easier to convert
-to other styles.
+While this style is more involved than the export-using style, it makes it
+easier to further refactor the library to other styles.
 
 ABI breaking style
 ~~~~~~~~~~~~~~~~~~
 
-The term ``ABI breaking`` sounds terrifying generally. But you may want it here if you want
-to force your users to introduce your library in a consistent way. E.g., they either include
-your headers all the way or import your modules all the way.
-The style prevents the users to include your headers and import your modules at the same time
-in the same repo.
+The term ``ABI breaking`` may sound like a bad approach. However, this style
+forces consumers of the library use it in a consistent way. e.g., either always
+include headers for the library or always import modules. The style prevents
+the ability to mix includes and imports for the library.
 
-The pattern for ABI breaking style is similar with export extern-C++ style.
+The pattern for ABI breaking style is similar to the export extern-C++ style.
 
 .. code-block:: c++
 
@@ -865,7 +864,7 @@ The pattern for ABI breaking style is similar with export extern-C++ style.
   ...
   #include "source_n.cpp"
   #else // the number of .cpp files in your project are a lot
-  // Using all the declarations from thirdparty libraries which are
+  // Using all the declarations from third-party libraries which are
   // used in the .cpp files.
   namespace third_party_namespace {
     using third_party_decl_used_in_cpp_1;
@@ -875,11 +874,11 @@ The pattern for ABI breaking style is similar with export extern-C++ style.
   }
   #endif
 
-(And add `EXPORT` and conditional include to the headers as suggested in the export
-extern-C++ style section)
+(And add `EXPORT` and conditional include to the headers as suggested in the
+export extern-C++ style section.)
 
-Remember that the ABI get changed and we need to compile our source files into the
-new ABI format. This is the job of the additional part of the interface unit:
+The ABI with modules is different and thus we need to compile the source files
+into the new ABI. This is done by an additional part of the interface unit:
 
 .. code-block:: c++
 
@@ -890,7 +889,7 @@ new ABI format. This is the job of the additional part of the interface unit:
   ...
   #include "source_n.cpp"
   #else // the number of .cpp files in your project are a lot
-  // Using all the declarations from thirdparty libraries which are
+  // Using all the declarations from third-party libraries which are
   // used in the .cpp files.
   namespace third_party_namespace {
     using third_party_decl_used_in_cpp_1;
@@ -900,16 +899,17 @@ new ABI format. This is the job of the additional part of the interface unit:
   }
   #endif
 
-In case the number of your source files are small, we may put everything in the private
-module fragment directly. (it is suggested to add conditional include to the source
-files too). But it will make the compilation of the module interface unit to be slow
-when the number of the source files are not small enough.
+If the number of source files is small, everything can be put in the private
+module fragment directly (it is recommended to add conditional includes to the
+source files as well). However, compile time performance will be bad if there
+are a lot of source files to compile.
 
-**Note that the private module fragment can only be in the primary module interface unit
-and the primary module interface unit containing private module fragment should be the only
-module unit of the corresponding module.**
+**Note that the private module fragment can only be in the primary module
+interface unit and the primary module interface unit containing the private
+module fragment should be the only module unit of the corresponding module.**
 
-In that case, you need to convert your source files (.cpp files) to module implementation units:
+In this case, source files (.cpp files) must be converted to module
+implementation units:
 
 .. code-block:: c++
 
@@ -925,45 +925,40 @@ In that case, you need to convert your source files (.cpp files) to module imple
   // Following off should be unchanged.
   ...
 
-The module implementation unit will import the primary module implicitly.
-We don't include any headers in the module implementation units
-here since we want to avoid duplicated declarations between translation units.
-This is the reason why we add non-exported using declarations from the third
-party libraries in the primary module interface unit.
-
-And if you provide your library as ``libyour_library.so``, you probably need to
-provide a modular one ``libyour_library_modules.so`` since you changed the ABI.
-
-What if there are headers only inclued by the source files
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The module implementation unit will import the primary module implicitly. Do
+not include any headers in the module implementation units as it avoids
+duplicated declarations between translation units. This is why non-exported
+using declarations should be added from third-party libraries in the primary
+module interface unit.
 
-The above practice may be problematic if there are headers only included by the source
-files. If you're using private module fragment, you may solve the issue by including them
-in the private module fragment. While it is OK to solve it by including the implementation
-headers in the module purview if you're using implementation module units, it may be
-suboptimal since the primary module interface units now containing entities not belongs
-to the interface.
+If the library is provided as ``libyour_library.so``, a modular library (e.g.,
+``libyour_library_modules.so``) may also need to be provided for ABI
+compatibility.
 
-If you're a perfectionist, maybe you can improve it by introducing internal module partition unit.
+What if there are headers only included by the source files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The internal module partition unit is an importable module unit which is internal
-to the module itself. The concept just meets the headers only included by the source files.
+The above practice may be problematic if there are headers only included by the
+source files. When using a private module fragment, this issue may be solved by
+including those headers in the private module fragment. While it is OK to solve
+it by including the implementation headers in the module purview when using
+implementation module units, it may be suboptimal because the primary module
+interface units now contain entities that do not belong to the interface.
 
-We don't show code snippet since it may be too verbose or not good or not general.
-But it may not be too hard if you can understand the points of the section.
+This can potentially be improved by introducing a module partition
+implementation unit. An internal module partition unit is an importable
+module unit which is internal to the module itself.
 
 Providing a header to skip parsing redundant headers
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-It is a problem for clang to handle redeclarations between translation units.
-Also there is a long standing issue in clang (`problematic include after import <https://github.com/llvm/llvm-project/issues/61465>`_).
-But even if the issue get fixed in clang someday, the users may still get slower compilation speed
-and larger BMI size. So it is suggested to not include headers after importing the corresponding
-library.
-
-However, it is not easy for users if your library are included by other dependencies.
-
-So the users may have to write codes like:
+Many redeclarations shared between translation units causes Clang to have
+slower compile-time performance. Further, there are known issues with
+`include after import <https://github.com/llvm/llvm-project/issues/61465>`_.
+Even when that issue is resolved, users may still get slower compilation speed
+and larger BMIs. For these reasons, it is recommended to not include headers
+after importing the corresponding module. However, it is not always easy if the
+library is included by other dependencies, as in:
 
 .. code-block:: c++
 
@@ -977,9 +972,9 @@ or
   import your_library;
   #include "third_party/A.h" // #include "your_library/a_header.h"
 
-For such cases, we suggest the libraries providing modules and the headers at the same time
-to provide a header to skip parsing all the headers in your libraries. So the users can
-import your library as the following style to skip redundant handling:
+For such cases, it is best if the library providing both module and header
+interfaces also provides a header which skips parsing so that the library can
+be imported with the following approach that skips redundant redeclarations:
 
 .. code-block:: c++
 
@@ -987,9 +982,9 @@ import your library as the following style to skip redundant handling:
   #include "your_library_imported.h"
   #include "third_party/A.h" // #include "your_library/a_header.h" but got skipped
 
-The implementation of ``your_library_imported.h`` can be a set of controlling macros or
-an overall controlling macro if you're using `#pragma once`. So you can convert your
-headers to:
+The implementation of ``your_library_imported.h`` can be a set of controlling
+macros or an overall controlling macro if using `#pragma once`. Then headers
+can be refactored to:
 
 .. code-block:: c++
 
@@ -998,25 +993,24 @@ headers to:
   ...
   #endif
 
-If the modules imported by your library provides such headers too, remember to add them to
-your ``your_library_imported.h`` too.
+If the modules imported by the library provide such headers, remember to add
+them to ``your_library_imported.h`` too.
 
 Importing modules
 ~~~~~~~~~~~~~~~~~
 
-When there are dependent libraries providing modules, we suggest you to import that in
-your module.
-
-Most of the existing libraries would fall into this catagory once the std module gets available.
+When there are dependent libraries providing modules, they should be imported
+in your module as well. Many existing libraries will fall into this category
+once the ``std`` module is more widely available.
 
 All dependent libraries providing modules
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Life gets easier if all the dependent libraries providing modules.
+Of course, most of the complexity disappears if all the dependent libraries
+provide modules.
 
-You need to convert your headers to include thirdparty headers conditionally.
-
-Then for export-using style:
+Headers need to be converted to include third-party headers conditionally. Then,
+for the export-using style:
 
 .. code-block:: c++
 
@@ -1035,7 +1029,7 @@ Then for export-using style:
     using decl_n;
   }
 
-For export extern-C++ style:
+or, for the export extern-C++ style:
 
 .. code-block:: c++
 
@@ -1049,7 +1043,7 @@ For export extern-C++ style:
     #include "header_n.h"
   }
 
-For ABI breaking style,
+or, for the ABI-breaking style,
 
 .. code-block:: c++
 
@@ -1069,35 +1063,39 @@ For ABI breaking style,
   #include "source_n.cpp"
   #endif
 
-We don't need the non-exported using declarations if we're using implementation module
-units now. We can import thirdparty modules directly in the implementation module
-units.
+Non-exported ``using`` declarations are unnecessary if using implementation
+module units. Instead, third-party modules can be imported directly in
+implementation module units.
 
 Partial dependent libraries providing modules
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-In this case, we have to mix the use of ``include`` and ``import`` in the module of our
-library. The key point here is still to remove duplicated declarations in translation
-units as much as possible. If the imported modules provide headers to skip parsing their
-headers, we should include that after the including. If the imported modules don't provide
-the headers, we can make it ourselves if we still want to optimize it.
-
-Known Problems
---------------
-
-The following describes issues in the current implementation of modules.
-Please see https://github.com/llvm/llvm-project/labels/clang%3Amodules for more issues
-or file a new issue if you don't find an existing one.
-If you're going to create a new issue for standard C++ modules,
-please start the title with ``[C++20] [Modules]`` (or ``[C++23] [Modules]``, etc)
-and add the label ``clang:modules`` (if you have permissions for that).
-
-For higher level support for proposals, you could visit https://clang.llvm.org/cxx_status.html.
-
-Including headers after import is problematic
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+If the library has to mix the use of ``include`` and ``import`` in its module,
+the primary goal is still the removal of duplicated declarations in translation
+units as much as possible. If the imported modules provide headers to skip
+parsing their headers, those should be included after the import. If the
+imported modules don't provide such a header, one can be made manually for
+improved compile time performance.
+
+Known Issues
+------------
+
+The following describes issues in the current implementation of modules. Please
+see
+`the issues list for modules <https://github.com/llvm/llvm-project/labels/clang%3Amodules>`_
+for a list of issues or to file a new issue if you don't find an existing one.
+When creating a new issue for standard C++ modules, please start the title with
+``[C++20] [Modules]`` (or ``[C++23] [Modules]``, etc) and add the label
+``clang:modules`` if possible.
+
+A high-level overview of support for standards features, including modules, can
+be found on the `C++ Feature Status <https://clang.llvm.org/cxx_status.html>`_
+page.
+
+Including headers after import is not well-supported
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-For example, the following example can be accept:
+The following example is accepted:
 
 .. code-block:: c++
 
@@ -1110,8 +1108,8 @@ For example, the following example can be accept:
       return 0;
   }
 
-but it will get rejected if we reverse the order of ``#include <iostream>`` and
-``import foo;``:
+but if the order of ``#include <iostream>`` and ``import foo;`` is reversed,
+then the code is currently rejected:
 
 .. code-block:: c++
 
@@ -1126,33 +1124,31 @@ but it will get rejected if we reverse the order of ``#include <iostream>`` and
 
 Both of the above examples should be accepted.
 
-This is a limitation in the implementation. In the first example,
-the compiler will see and parse <iostream> first then the compiler will see the import.
-So the ODR Checking and declarations merging will happen in the deserializer.
-In the second example, the compiler will see the import first and the include second.
-As a result, the ODR Checking and declarations merging will happen in the semantic analyzer.
+This is a limitation of the implementation. In the first example, the compiler
+will see and parse ``<iostream>`` first then it will see the ``import``. In
+this case, ODR checking and declaration merging will happen in the
+deserializer. In the second example, the compiler will see the ``import`` first
+and the ``#include`` second which results in ODR checking and declarations
+merging happening in the semantic analyzer. This is due to a divergence in the
+implementation path. This is tracked by
+`#61465 <https://github.com/llvm/llvm-project/issues/61465>`_.
 
-So there is divergence in the implementation path. It might be understandable that why
-the orders matter here in the case.
-(Note that "understandable" is different from "makes sense").
-
-This is tracked in: https://github.com/llvm/llvm-project/issues/61465
-
-Ignored PreferredName Attribute
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Due to a tricky problem, when Clang writes BMIs, Clang will ignore the ``preferred_name`` attribute, if any.
-This implies that the ``preferred_name`` wouldn't show in debugger or dumping.
+Ignored ``preferred_name`` Attribute
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-This is tracked in: https://github.com/llvm/llvm-project/issues/56490
+When Clang writes BMIs, it will ignore the ``preferred_name`` attribute on
+declarations which use it. Thus, the preferred name will not be displayed in
+the debugger as expected. This is tracked by
+`#56490 <https://github.com/llvm/llvm-project/issues/56490>`_.
 
 Don't emit macros about module declaration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-This is covered by P1857R3. We mention it again here since users may abuse it before we implement it.
+This is covered by `P1857R3 <https://wg21.link/P1857R3>`_. It is mentioned here
+because we want users to be aware that we don't yet implement it.
 
-Someone may want to write code which could be compiled both by modules or non-modules.
-A direct idea would be use macros like:
+A direct approach to write code that can be compiled by both modules and
+non-module builds may look like:
 
 .. code-block:: c++
 
@@ -1162,39 +1158,37 @@ A direct idea would be use macros like:
   IMPORT header_name
   EXPORT ...
 
-So this file could be triggered like a module unit or a non-module unit depending on the definition
-of some macros.
-However, this kind of usage is forbidden by P1857R3 but we haven't implemented P1857R3 yet.
-This means that is possible to write illegal modules code now, and obviously this will stop working
-once P1857R3 is implemented.
-A simple suggestion would be "Don't play macro tricks with module declarations".
+The intent of this is that this file can be compiled like a module unit or a
+non-module unit depending on the definition of some macros. However, this usage
+is forbidden by P1857R3 which is not yet implemented in Clang. This means that
+is possible to write invalid modules which will no longer be accepted once
+P1857R3 is implemented. This is tracked by
+`#56917 <https://github.com/llvm/llvm-project/issues/56917>`_.
+
+Until then, it is recommended not to mix macros with module declarations.
 
-This is tracked in: https://github.com/llvm/llvm-project/issues/56917
 
 In consistent filename suffix requirement for importable module units
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Currently, clang requires the file name of an ``importable module unit`` should end with ``.cppm``
-(or ``.ccm``, ``.cxxm``, ``.c++m``). However, the behavior is inconsistent with other compilers.
-
-This is tracked in: https://github.com/llvm/llvm-project/issues/57416
-
-clang-cl is not compatible with the standard C++ modules
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Now we can't use the `/clang:-fmodule-file` or `/clang:-fprebuilt-module-path` to specify
-the BMI within ``clang-cl.exe``.
+Currently, Clang requires the file name of an ``importable module unit`` to
+have ``.cppm`` (or ``.ccm``, ``.cxxm``, ``.c++m``) as the file extension.
+However, the behavior is inconsistent with other compilers. This is tracked by
+`#57416 <https://github.com/llvm/llvm-project/issues/57416>`_.
 
-This is tracked in: https://github.com/llvm/llvm-project/issues/64118
+clang-cl is not compatible with standard C++ modules
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-false positive ODR violation diagnostic due to using inconsistent qualified but the same type
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``/clang:-fmodule-file`` and ``/clang:-fprebuilt-module-path`` cannot be used
+to specify the BMI with ``clang-cl.exe``. This is tracked by
+`#64118 <https://github.com/llvm/llvm-project/issues/64118>`_.
 
-ODR violation is a pretty common issue when using modules.
-Sometimes the program violated the One Definition Rule actually.
-But sometimes it shows the compiler gives false positive diagnostics.
+Incorrect ODR violation diagnostics
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-One often reported example is:
+ODR violations are a common issue when using modules. Clang sometimes produces
+false-positive diagnostics or fails to produce true-positive diagnostics of the
+One Definition Rule. One often-reported example is:
 
 .. code-block:: c++
 
@@ -1222,51 +1216,49 @@ One often reported example is:
   export module repro;
   export import :part;
 
-Currently the compiler complains about the inconsistent definition of `fun()` in
-2 module units. This is incorrect. Since both definitions of `fun()` has the same
-spelling and `T` refers to the same type entity finally. So the program should be
-fine.
-
-This is tracked in https://github.com/llvm/llvm-project/issues/78850.
+Currently the compiler incorrectly diagnoses the inconsistent definition of
+``fun()`` in two module units. Because both definitions of ``fun()`` have the
+same spelling and ``T`` refers to the same type entity, there is no ODR
+violation. This is tracked by
+`#78850 <https://github.com/llvm/llvm-project/issues/78850>`_.
 
 Using TU-local entity in other units
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Module units are translation units. So the entities which should only be local to the
-module unit itself shouldn't be used by other units in any means.
+Module units are translation units, so the entities which should be local to
+the module unit itself should never be used by other units.
 
-In the language side, to address the idea formally, the language specification defines
-the concept of ``TU-local`` and ``exposure`` in
+The C++ standard defines the concept of ``TU-local`` and ``exposure`` in
 `basic.link/p14 <https://eel.is/c++draft/basic.link#14>`_,
 `basic.link/p15 <https://eel.is/c++draft/basic.link#15>`_,
 `basic.link/p16 <https://eel.is/c++draft/basic.link#16>`_,
-`basic.link/p17 <https://eel.is/c++draft/basic.link#17>`_ and
+`basic.link/p17 <https://eel.is/c++draft/basic.link#17>`_, and
 `basic.link/p18 <https://eel.is/c++draft/basic.link#18>`_.
 
-However, the compiler doesn't support these 2 ideas formally.
-This results in unclear and confusing diagnostic messages.
-And it is worse that the compiler may import TU-local entities to other units without any
-diagnostics.
+However, Clang doesn't formally support these two concepts. This results in
+unclear or confusing diagnostic messages. Further, Clang may import
+``TU-local`` entities to other units without any diagnostics. This is tracked
+by `#78173 <https://github.com/llvm/llvm-project/issues/78173>`_.
 
-This is tracked in https://github.com/llvm/llvm-project/issues/78173.
+.. _header-units:
 
 Header Units
 ============
 
-How to build projects using header unit
----------------------------------------
+How to build projects using header units
+----------------------------------------
 
 .. warning::
 
-   The user interfaces of header units is highly experimental. There are still
-   many unanswered question about how tools should interact with header units.
-   The user interfaces described here may change after we have progress on how
-   tools should support for header units.
+   The support for header units, including related command line options, is
+   experimental. There are still many unanswered question about how tools
+   should interact with header units. The details described here may change in
+   the future.
 
 Quick Start
 ~~~~~~~~~~~
 
-For the following example,
+The following example:
 
 .. code-block:: c++
 
@@ -1275,7 +1267,7 @@ For the following example,
     std::cout << "Hello World.\n";
   }
 
-we could compile it as
+could be compiled with:
 
 .. code-block:: console
 
@@ -1285,14 +1277,14 @@ we could compile it as
 How to produce BMIs
 ~~~~~~~~~~~~~~~~~~~
 
-Similar to named modules, we could use ``--precompile`` to produce the BMI.
-But we need to specify that the input file is a header by ``-xc++-system-header`` or ``-xc++-user-header``.
+Similar to named modules, ``--precompile`` can be used to produce a BMI.
+However, that requires specifying that the input file is a header by using
+``-xc++-system-header`` or ``-xc++-user-header``.
 
-Also we could use `-fmodule-header={user,system}` option to produce the BMI for header units
-which has suffix like `.h` or `.hh`.
-The value of `-fmodule-header` means the user search path or the system search path.
-The default value for `-fmodule-header` is `user`.
-For example,
+The ``-fmodule-header={user,system}`` option can also be used to produce a BMI
+for header units which have a file extension like `.h` or `.hh`. The argument to
+``-fmodule-header`` specifies either the user search path or the system search
+path. The default value for ``-fmodule-header`` is ``user``. For example:
 
 .. code-block:: c++
 
@@ -1308,16 +1300,16 @@ For example,
     Hello();
   }
 
-We could compile it as:
+could be compiled with:
 
 .. code-block:: console
 
   $ clang++ -std=c++20 -fmodule-header foo.h -o foo.pcm
   $ clang++ -std=c++20 -fmodule-file=foo.pcm use.cpp
 
-For headers which don't have a suffix, we need to pass ``-xc++-header``
-(or ``-xc++-system-header`` or ``-xc++-user-header``) to mark it as a header.
-For example,
+For headers which do not have a file extension, ``-xc++-header`` (or
+``-xc++-system-header``, ``-xc++-user-header``) must be used to specify the
+file as a header. For example:
 
 .. code-block:: c++
 
@@ -1332,23 +1324,25 @@ For example,
   $ clang++ -std=c++20 -fmodule-header=system -xc++-header iostream -o iostream.pcm
   $ clang++ -std=c++20 -fmodule-file=iostream.pcm use.cpp
 
-How to specify the dependent BMIs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+How to specify dependent BMIs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-We could use ``-fmodule-file`` to specify the BMIs, and this option may occur multiple times as well.
+``-fmodule-file`` can be used to specify a dependent BMI (or multiple times for
+more than one dependent BMI).
 
-With the existing implementation ``-fprebuilt-module-path`` cannot be used for header units
-(since they are nominally anonymous).
-For header units, use  ``-fmodule-file`` to include the relevant PCM file for each header unit.
+With the existing implementation, ``-fprebuilt-module-path`` cannot be used for
+header units (because they are nominally anonymous). For header units, use
+``-fmodule-file`` to include the relevant PCM file for each header unit.
 
-This is expect to be solved in future editions of the compiler either by the tooling finding and specifying
-the -fmodule-file or by the use of a module-mapper that understands how to map the header name to their PCMs.
+This is expect to be solved in a future version of Clang either by the compiler
+finding and specifying ``-fmodule-file`` automatically, or by the use of a
+module-mapper that understands how to map the header name to their PCMs.
 
-Don't compile the BMI
-~~~~~~~~~~~~~~~~~~~~~
+Compiling a header unit to an object file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Another difference with modules is that we can't compile the BMI from a header unit.
-For example:
+A header unit cannot be compiled to an object file due to the semantics of
+header units. For example:
 
 .. code-block:: console
 
@@ -1356,15 +1350,13 @@ For example:
   # This is not allowed!
   $ clang++ iostream.pcm -c -o iostream.o
 
-It makes sense due to the semantics of header units, which are just like headers.
-
 Include translation
 ~~~~~~~~~~~~~~~~~~~
 
-The C++ spec allows the vendors to convert ``#include header-name`` to ``import header-name;`` when possible.
-Currently, Clang would do this translation for the ``#include`` in the global module fragment.
-
-For example, the following two examples are the same:
+The C++ standard allows vendors to convert ``#include header-name`` to
+``import header-name;`` when possible. Currently, Clang does this translation
+for the ``#include`` in the global module fragment. For example, the following
+example:
 
 .. code-block:: c++
 
@@ -1375,7 +1367,7 @@ For example, the following two examples are the same:
     std::cout << "Hello.\n";
   }
 
-with the following one:
+is the same as this example:
 
 .. code-block:: c++
 
@@ -1391,17 +1383,17 @@ with the following one:
   $ clang++ -std=c++20 -xc++-system-header --precompile iostream -o iostream.pcm
   $ clang++ -std=c++20 -fmodule-file=iostream.pcm --precompile M.cppm -o M.cpp
 
-In the latter example, the Clang could find the BMI for the ``<iostream>``
-so it would try to replace the ``#include <iostream>`` to ``import <iostream>;`` automatically.
+In the latter example, Clang can find the BMI for ``<iostream>`` and so it
+tries to replace the ``#include <iostream>`` with ``import <iostream>;``
+automatically.
 
 
-Relationships between Clang modules
------------------------------------
+Differences between Clang modules and header units
+--------------------------------------------------
 
-Header units have pretty similar semantics with Clang modules.
-The semantics of both of them are like headers.
-
-In fact, we could even "mimic" the sytle of header units by Clang modules:
+Header units have similar semantics to Clang modules. The semantics of both are
+like headers. Therefore, header units can be mimicked by Clang modules as in
+the following example:
 
 .. code-block:: c++
 
@@ -1414,46 +1406,45 @@ In fact, we could even "mimic" the sytle of header units by Clang modules:
 
   $ clang++ -std=c++20 -fimplicit-modules -fmodule-map-file=.modulemap main.cpp
 
-It would be simpler if we are using libcxx:
+This example is simplified when using libc++:
 
 .. code-block:: console
 
   $ clang++ -std=c++20 main.cpp -fimplicit-modules -fimplicit-module-maps
 
-Since there is already one
-`module map <https://github.com/llvm/llvm-project/blob/main/libcxx/include/module.modulemap.in>`_
-in the source of libcxx.
-
-Then immediately leads to the question: why don't we implement header units through Clang header modules?
+because libc++ already supplies a
+`module map <https://github.com/llvm/llvm-project/blob/main/libcxx/include/module.modulemap.in>`_.
 
-The main reason for this is that Clang modules have more semantics like hierarchy or
-wrapping multiple headers together as a big module.
-However, these things are not part of Standard C++ Header units,
-and we want to avoid the impression that these additional semantics get interpreted as Standard C++ behavior.
+This raises the question: why are header units not implemented through Clang
+modules?
 
-Another reason is that there are proposals to introduce module mappers to the C++ standard
-(for example, https://wg21.link/p1184r2).
-If we decide to reuse Clang's modulemap, we may get in trouble once we need to introduce another module mapper.
+This is primarily because Clang modules have more hierarchical semantics when
+wrapping multiple headers together as one module, which is not supported by
+Standard C++ Header units. We want to avoid the impression that these
+additional semantics get interpreted as Standard C++ behavior.
 
-So the final answer for why we don't reuse the interface of Clang modules for header units is that
-there are some differences between header units and Clang modules and that ignoring those
-differences now would likely become a problem in the future.
+Another reason is that there are proposals to introduce module mappers to the
+C++ standard (for example, https://wg21.link/p1184r2). Reusing Clang's
+``modulemap`` may be more difficult if we need to introduce another module
+mapper.
 
-Discover Dependencies
-=====================
+Discovering Dependencies
+========================
 
-Prior to modules, all the translation units can be compiled parallelly.
-But it is not true for the module units. The presence of module units requires
-us to compile the translation units in a (topological) order.
+Without use of modules, all the translation units in a project can be compiled
+in parallel. However, the presence of module units requires compiling the
+translation units in a topological order.
 
-The clang-scan-deps scanner implemented
-`P1689 paper <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html>`_
-to describe the order. Only named modules are supported now.
+The ``clang-scan-deps`` tool can extract dependency information and produce a
+JSON file conforming to the specification described in
+`P1689 <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html>`_.
+Only named modules are supported currently.
 
-We need a compilation database to use clang-scan-deps. See
+A compilation database is needed when using ``clang-scan-deps``. See
 `JSON Compilation Database Format Specification <JSONCompilationDatabase.html>`_
-for example. Note that the ``output`` entry is necessary for clang-scan-deps
-to scan P1689 format. Here is an example:
+for more information about compilation databases. Note that the ``output``
+JSON attribute is necessary for ``clang-scan-deps`` to scan using the P1689
+format. For example:
 
 .. code-block:: c++
 
@@ -1533,13 +1524,13 @@ And here is the compilation database:
   }
   ]
 
-And we can get the dependency information in P1689 format by:
+To get the dependency information in P1689 format, use:
 
 .. code-block:: console
 
   $ clang-scan-deps -format=p1689 -compilation-database P1689.json
 
-And we will get:
+to get:
 
 .. code-block:: text
 
@@ -1619,14 +1610,14 @@ And we will get:
 
 See the P1689 paper for the meaning of the fields.
 
-And if the user want a finer-grained control for any reason, e.g., to scan the generated source files,
-the user can choose to get the dependency information per file. For example:
+Getting dependency information per file with finer-grained control (such as
+scanning generated source files) is possible. For example:
 
 .. code-block:: console
 
   $ clang-scan-deps -format=p1689 -- <path-to-compiler-executable>/clang++ -std=c++20 impl_part.cppm -c -o impl_part.o
 
-And we'll get:
+will produce:
 
 .. code-block:: text
 
@@ -1652,22 +1643,23 @@ And we'll get:
     "version": 1
   }
 
-In this way, we can pass the single command line options after the ``--``.
-Then clang-scan-deps will extract the necessary information from the options.
-Note that we need to specify the path to the compiler executable instead of saying
-``clang++`` simply.
+Individual command line options can be specified after ``--``.
+``clang-scan-deps`` will extract the necessary information from the specified
+options. Note that the path to the compiler executable needs to be specified
+explicitly instead of using ``clang++`` directly.
 
-The users may want the scanner to get the transitional dependency information for headers.
-Otherwise, the users have to scan twice for the project, once for headers and once for modules.
-To address the requirement, clang-scan-deps will recognize the specified preprocessor options
-in the given command line and generate the corresponding dependency information. For example,
+Users may want the scanner to get the transitional dependency information for
+headers. Otherwise, the project has to be scanned twice, once for headers and
+once for modules. To address this, ``clang-scan-deps`` will recognize the
+specified preprocessor options in the given command line and generate the
+corresponding dependency information. For example:
 
 .. code-block:: console
 
   $ clang-scan-deps -format=p1689 -- ../bin/clang++ -std=c++20 impl_part.cppm -c -o impl_part.o -MD -MT impl_part.ddi -MF impl_part.dep
   $ cat impl_part.dep
 
-We will get:
+will produce:
 
 .. code-block:: text
 
@@ -1679,41 +1671,41 @@ We will get:
     /usr/include/bits/types/__locale_t.h \
     ...
 
-When clang-scan-deps detects ``-MF`` option, clang-scan-deps will try to write the
+When ``clang-scan-deps`` detects the ``-MF`` option, it will try to write the
 dependency information for headers to the file specified by ``-MF``.
 
 Possible Issues: Failed to find system headers
 ----------------------------------------------
 
-In case the users encounter errors like ``fatal error: 'stddef.h' file not found``,
-probably the specified ``<path-to-compiler-executable>/clang++`` refers to a symlink
-instead a real binary. There are 4 potential solutions to the problem:
-
-* (1) End users can resolve the issue by pointing the specified compiler executable to
-  the real binary instead of the symlink.
-* (2) End users can invoke ``<path-to-compiler-executable>/clang++ -print-resource-dir``
-  to get the corresponding resource directory for your compiler and add that directory
-  to the include search paths manually in the build scripts.
-* (3) Build systems that use a compilation database as the input for clang-scan-deps
-  scanner, the build system can add the flag ``--resource-dir-recipe invoke-compiler`` to
-  the clang-scan-deps scanner to calculate the resources directory dynamically.
-  The calculation happens only once for a unique ``<path-to-compiler-executable>/clang++``.
-* (4) For build systems that invokes the clang-scan-deps scanner per file, repeatedly
-  calculating the resource directory may be inefficient. In such cases, the build
-  system can cache the resource directory by itself and pass ``-resource-dir <resource-dir>``
-  explicitly in the command line options:
+If encountering an error like ``fatal error: 'stddef.h' file not found``,
+the specified ``<path-to-compiler-executable>/clang++`` probably refers to a
+symlink instead a real binary. There are four potential solutions to the
+problem:
 
-.. code-block:: console
+1. Point the specified compiler executable to the real binary instead of the
+   symlink.
+2. Invoke ``<path-to-compiler-executable>/clang++ -print-resource-dir`` to get
+   the corresponding resource directory for your compiler and add that
+   directory to the include search paths manually in the build scripts.
+3. For build systems that use a compilation database as the input for
+   ``clang-scan-deps``, the build system can add the
+   ``--resource-dir-recipe invoke-compiler`` option when executing
+   ``clang-scan-deps`` to calculate the resource directory dynamically.
+   The calculation happens only once for a unique ``<path-to-compiler-executable>/clang++``.
+4. For build systems that invoke ``clang-scan-deps`` per file, repeatedly
+   calculating the resource directory may be inefficient. In such cases, the
+   build system can cache the resource directory and specify
+   ``-resource-dir <resource-dir>`` explicitly, as in:
+
+   .. code-block:: console
 
-  $ clang-scan-deps -format=p1689 -- <path-to-compiler-executable>/clang++ -std=c++20 -resource-dir <resource-dir> mod.cppm -c -o mod.o
+     $ clang-scan-deps -format=p1689 -- <path-to-compiler-executable>/clang++ -std=c++20 -resource-dir <resource-dir> mod.cppm -c -o mod.o
 
 
 Import modules with clang-repl
 ==============================
 
-We're able to import C++20 named modules with clang-repl.
-
-Let's start with a simple example:
+``clang-repl`` supports importing C++20 named modules. For example:
 
 .. code-block:: c++
 
@@ -1723,7 +1715,7 @@ Let's start with a simple example:
       return "Hello Interpreter for Modules!";
   }
 
-We still need to compile the named module in ahead.
+The named module still needs to be compiled ahead of time.
 
 .. code-block:: console
 
@@ -1731,10 +1723,9 @@ We still need to compile the named module in ahead.
   $ clang++ M.pcm -c -o M.o
   $ clang++ -shared M.o -o libM.so
 
-Note that we need to compile the module unit into a dynamic library so that the clang-repl
-can load the object files of the module units.
-
-Then we are able to import module ``M`` in clang-repl.
+Note that the module unit needs to be compiled as a dynamic library so that
+``clang-repl`` can load the object files of the module units. Then it is
+possible to import module ``M`` in clang-repl.
 
 .. code-block:: console
 
@@ -1753,17 +1744,18 @@ Possible Questions
 How modules speed up compilation
 --------------------------------
 
-A classic theory for the reason why modules speed up the compilation is:
-if there are ``n`` headers and ``m`` source files and each header is included by each source file,
-then the complexity of the compilation is ``O(n*m)``;
-But if there are ``n`` module interfaces and ``m`` source files, the complexity of the compilation is
-``O(n+m)``. So, using modules would be a big win when scaling.
-In a simpler word, we could get rid of many redundant compilations by using modules.
+A classic theory for the reason why modules speed up the compilation is: if
+there are ``n`` headers and ``m`` source files and each header is included by
+each source file, then the complexity of the compilation is ``O(n*m)``.
+However, if there are ``n`` module interfaces and ``m`` source files, the
+complexity of the compilation is ``O(n+m)``. Therefore, using modules would be
+a significant improvement at scale. More simply, use of modules causes many of
+the redundant compilations to no longer be necessary.
 
-Roughly, this theory is correct. But the problem is that it is too rough.
-The behavior depends on the optimization level, as we will illustrate below.
+While this is accurate at a high level, this depends greatly on the
+optimization level, as illustrated below.
 
-First is ``O0``. The compilation process is described in the following graph.
+First is ``-O0``. The compilation process is described in the following graph.
 
 .. code-block:: none
 
@@ -1771,13 +1763,13 @@ First is ``O0``. The compilation process is described in the following graph.
   │                               │                                       │               │
   └---parsing----sema----codegen--┴----- transformations ---- codegen ----┴---- codegen --┘
 
-  ┌---------------------------------------------------------------------------------------┐
+  ├---------------------------------------------------------------------------------------┐
   |                                                                                       │
   |                                     source file                                       │
   |                                                                                       │
   └---------------------------------------------------------------------------------------┘
 
-              ┌--------┐
+              ├--------┐
               │        │
               │imported│
               │        │
@@ -1785,18 +1777,17 @@ First is ``O0``. The compilation process is described in the following graph.
               │        │
               └--------┘
 
-Here we can see that the source file (could be a non-module unit or a module unit) would get processed by the
-whole pipeline.
-But the imported code would only get involved in semantic analysis, which is mainly about name lookup,
-overload resolution and template instantiation.
-All of these processes are fast relative to the whole compilation process.
-More importantly, the imported code only needs to be processed once in frontend code generation,
-as well as the whole middle end and backend.
-So we could get a big win for the compilation time in O0.
+In this case, the source file (which could be a non-module unit or a module
+unit) would get processed by the entire pipeline. However, the imported code
+would only get involved in semantic analysis, which, for the most part, is name
+lookup, overload resolution, and template instantiation. All of these processes
+are fast relative to the whole compilation process. More importantly, the
+imported code only needs to be processed once during frontend code generation,
+as well as the whole middle end and backend. So we could get a big win for the
+compilation time in ``-O0``.
 
-But with optimizations, things are different:
-
-(we omit ``code generation`` part for each end due to the limited space)
+But with optimizations, things are different (the ``code generation`` part for
+each end is omitted due to limited space):
 
 .. code-block:: none
 
@@ -1804,12 +1795,12 @@ But with optimizations, things are different:
   │                           │                                               │                   │
   └--- parsing ---- sema -----┴--- optimizations --- IPO ---- optimizations---┴--- optimizations -┘
 
-  ┌-----------------------------------------------------------------------------------------------┐
+  ├-----------------------------------------------------------------------------------------------┐
   │                                                                                               │
   │                                         source file                                           │
   │                                                                                               │
   └-----------------------------------------------------------------------------------------------┘
-                ┌---------------------------------------┐
+                ├---------------------------------------┐
                 │                                       │
                 │                                       │
                 │            imported code              │
@@ -1817,27 +1808,29 @@ But with optimizations, things are different:
                 │                                       │
                 └---------------------------------------┘
 
-It would be very unfortunate if we end up with worse performance after using modules.
-The main concern is that when we compile a source file, the compiler needs to see the function body
-of imported module units so that it can perform IPO (InterProcedural Optimization, primarily inlining
-in practice) to optimize functions in current source file with the help of the information provided by
-the imported module units.
-In other words, the imported code would be processed again and again in importee units
-by optimizations (including IPO itself).
-The optimizations before IPO and the IPO itself are the most time-consuming part in whole compilation process.
-So from this perspective, we might not be able to get the improvements described in the theory.
-But we could still save the time for optimizations after IPO and the whole backend.
-
-Overall, at ``O0`` the implementations of functions defined in a module will not impact module users,
-but at higher optimization levels the definitions of such functions are provided to user compilations for the
-purposes of optimization (but definitions of these functions are still not included in the use's object file)-
-this means the build speedup at higher optimization levels may be lower than expected given ``O0`` experience,
-but does provide by more optimization opportunities.
+It would be very unfortunate if we end up with worse performance when using
+modules. The main concern is that when a source file is compiled, the compiler
+needs to see the body of imported module units so that it can perform IPO
+(InterProcedural Optimization, primarily inlining in practice) to optimize
+functions in the current source file with the help of the information provided
+by the imported module units. In other words, the imported code would be
+processed again and again in importee units by optimizations (including IPO
+itself). The optimizations before IPO and IPO itself are the most time-consuming
+part in whole compilation process. So from this perspective, it might not be
+possible to get the compile time improvements described, but there could be
+time savings for optimizations after IPO and the whole backend.
+
+Overall, at ``-O0`` the implementations of functions defined in a module will
+not impact module users, but at higher optimization levels the definitions of
+such functions are provided to user compilations for the purposes of
+optimization (but definitions of these functions are still not included in the
+use's object file). This means the build speedup at higher optimization levels
+may be lower than expected given ``-O0`` experience, but does provide more
+optimization opportunities.
 
 Interoperability with Clang Modules
 -----------------------------------
 
-We **wish** to support clang modules and standard c++ modules at the same time,
-but the mixed using form is not well used/tested yet.
-
-Please file new github issues as you find interoperability problems.
+We **wish** to support Clang modules and standard C++ modules at the same time,
+but the mixing them together is not well used/tested yet. Please file new
+GitHub issues as you find interoperability problems.
-- 
GitLab


From 2ceb1291ef3ddb87cb58030cd61d965f4030338f Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 8 May 2024 16:51:45 +0100
Subject: [PATCH 0188/1206] [X86] Add canScaleShuffleElements helper. NFC.

Returns true if the shuffle mask can be rescaled to the requested number of elements.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 3ae68c438aa7..126577a4d59e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3738,6 +3738,11 @@ static bool scaleShuffleElements(ArrayRef<int> Mask, unsigned NumDstElts,
   return false;
 }
 
+static bool canScaleShuffleElements(ArrayRef<int> Mask, unsigned NumDstElts) {
+  SmallVector<int, 32> ScaledMask;
+  return scaleShuffleElements(Mask, NumDstElts, ScaledMask);
+}
+
 /// Returns true if Elt is a constant zero or a floating point constant +0.0.
 bool X86::isZeroNode(SDValue Elt) {
   return isNullConstant(Elt) || isNullFPConstant(Elt);
@@ -46684,14 +46689,14 @@ static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
   // To address this, we check that we can scale the shuffle mask to MOVMSK
   // element width (this will ensure "high" elements match). Its slightly overly
   // conservative, but fine for an edge case fold.
-  SmallVector<int, 32> ShuffleMask, ScaledMaskUnused;
+  SmallVector<int, 32> ShuffleMask;
   SmallVector<SDValue, 2> ShuffleInputs;
   if (NumElts <= CmpBits &&
       getTargetShuffleInputs(peekThroughBitcasts(Vec), ShuffleInputs,
                              ShuffleMask, DAG) &&
       ShuffleInputs.size() == 1 && isCompletePermute(ShuffleMask) &&
       ShuffleInputs[0].getValueSizeInBits() == VecVT.getSizeInBits() &&
-      scaleShuffleElements(ShuffleMask, NumElts, ScaledMaskUnused)) {
+      canScaleShuffleElements(ShuffleMask, NumElts)) {
     SDLoc DL(EFLAGS);
     SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]);
     Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
-- 
GitLab


From a6171900a446c85c3b53a4a9deba16b746f9f77f Mon Sep 17 00:00:00 2001
From: Harald van Dijk <harald.vandijk@codeplay.com>
Date: Wed, 8 May 2024 17:02:25 +0100
Subject: [PATCH 0189/1206] [RemoveDIs] Change remapDbgVariableRecord to
 remapDbgRecord (#91456)

We need to remap any DbgRecord, not just DbgVariableRecords.

This is the followup to #91447.

Co-authored-by: PietroGhg <pietro.ghiglio@codeplay.com>
---
 .../llvm/Transforms/Utils/ValueMapper.h       | 34 ++++++++---------
 .../Transforms/Scalar/SimpleLoopUnswitch.cpp  |  5 +--
 llvm/lib/Transforms/Utils/CloneFunction.cpp   | 17 ++++-----
 .../Transforms/Utils/LoopRotationUtils.cpp    | 10 ++---
 .../Transforms/Utils/LoopUnrollRuntime.cpp    |  5 +--
 llvm/lib/Transforms/Utils/SimplifyCFG.cpp     |  9 ++---
 llvm/lib/Transforms/Utils/ValueMapper.cpp     | 10 ++---
 .../Transforms/Utils/CloningTest.cpp          | 37 +++++++++++++++++++
 8 files changed, 78 insertions(+), 49 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/ValueMapper.h b/llvm/include/llvm/Transforms/Utils/ValueMapper.h
index 54e3e62dc3af..743cfeb7ef3f 100644
--- a/llvm/include/llvm/Transforms/Utils/ValueMapper.h
+++ b/llvm/include/llvm/Transforms/Utils/ValueMapper.h
@@ -180,9 +180,8 @@ public:
   Constant *mapConstant(const Constant &C);
 
   void remapInstruction(Instruction &I);
-  void remapDbgVariableRecord(Module *M, DbgVariableRecord &V);
-  void remapDbgVariableRecordRange(Module *M,
-                                   iterator_range<DbgRecordIterator> Range);
+  void remapDbgRecord(Module *M, DbgRecord &V);
+  void remapDbgRecordRange(Module *M, iterator_range<DbgRecordIterator> Range);
   void remapFunction(Function &F);
   void remapGlobalObjectMetadata(GlobalObject &GO);
 
@@ -268,26 +267,25 @@ inline void RemapInstruction(Instruction *I, ValueToValueMapTy &VM,
   ValueMapper(VM, Flags, TypeMapper, Materializer).remapInstruction(*I);
 }
 
-/// Remap the Values used in the DbgVariableRecord \a V using the value map \a
+/// Remap the Values used in the DbgRecord \a DR using the value map \a
 /// VM.
-inline void RemapDbgVariableRecord(Module *M, DbgVariableRecord *V,
-                                   ValueToValueMapTy &VM,
-                                   RemapFlags Flags = RF_None,
-                                   ValueMapTypeRemapper *TypeMapper = nullptr,
-                                   ValueMaterializer *Materializer = nullptr) {
-  ValueMapper(VM, Flags, TypeMapper, Materializer)
-      .remapDbgVariableRecord(M, *V);
+inline void RemapDbgRecord(Module *M, DbgRecord *DR, ValueToValueMapTy &VM,
+                           RemapFlags Flags = RF_None,
+                           ValueMapTypeRemapper *TypeMapper = nullptr,
+                           ValueMaterializer *Materializer = nullptr) {
+  ValueMapper(VM, Flags, TypeMapper, Materializer).remapDbgRecord(M, *DR);
 }
 
-/// Remap the Values used in the DbgVariableRecord \a V using the value map \a
+/// Remap the Values used in the DbgRecords \a Range using the value map \a
 /// VM.
-inline void
-RemapDbgVariableRecordRange(Module *M, iterator_range<DbgRecordIterator> Range,
-                            ValueToValueMapTy &VM, RemapFlags Flags = RF_None,
-                            ValueMapTypeRemapper *TypeMapper = nullptr,
-                            ValueMaterializer *Materializer = nullptr) {
+inline void RemapDbgRecordRange(Module *M,
+                                iterator_range<DbgRecordIterator> Range,
+                                ValueToValueMapTy &VM,
+                                RemapFlags Flags = RF_None,
+                                ValueMapTypeRemapper *TypeMapper = nullptr,
+                                ValueMaterializer *Materializer = nullptr) {
   ValueMapper(VM, Flags, TypeMapper, Materializer)
-      .remapDbgVariableRecordRange(M, Range);
+      .remapDbgRecordRange(M, Range);
 }
 
 /// Remap the operands, metadata, arguments, and instructions of a function.
diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index d763b1ee0aa1..002ed381a4fd 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -1261,9 +1261,8 @@ static BasicBlock *buildClonedLoopBlocks(
   Module *M = ClonedPH->getParent()->getParent();
   for (auto *ClonedBB : NewBlocks)
     for (Instruction &I : *ClonedBB) {
-      RemapDbgVariableRecordRange(M, I.getDbgRecordRange(), VMap,
-                                  RF_NoModuleLevelChanges |
-                                      RF_IgnoreMissingLocals);
+      RemapDbgRecordRange(M, I.getDbgRecordRange(), VMap,
+                          RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
       RemapInstruction(&I, VMap,
                        RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
       if (auto *II = dyn_cast<AssumeInst>(&I))
diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp
index 6a3b3faac77d..981183682b8b 100644
--- a/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -278,8 +278,8 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
     // attached debug-info records.
     for (Instruction &II : *BB) {
       RemapInstruction(&II, VMap, RemapFlag, TypeMapper, Materializer);
-      RemapDbgVariableRecordRange(II.getModule(), II.getDbgRecordRange(), VMap,
-                                  RemapFlag, TypeMapper, Materializer);
+      RemapDbgRecordRange(II.getModule(), II.getDbgRecordRange(), VMap,
+                          RemapFlag, TypeMapper, Materializer);
     }
 
   // Only update !llvm.dbg.cu for DifferentModule (not CloneModule). In the
@@ -867,10 +867,10 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
   Function::iterator Begin = cast<BasicBlock>(VMap[StartingBB])->getIterator();
   for (BasicBlock &BB : make_range(Begin, NewFunc->end())) {
     for (Instruction &I : BB) {
-      RemapDbgVariableRecordRange(I.getModule(), I.getDbgRecordRange(), VMap,
-                                  ModuleLevelChanges ? RF_None
-                                                     : RF_NoModuleLevelChanges,
-                                  TypeMapper, Materializer);
+      RemapDbgRecordRange(I.getModule(), I.getDbgRecordRange(), VMap,
+                          ModuleLevelChanges ? RF_None
+                                             : RF_NoModuleLevelChanges,
+                          TypeMapper, Materializer);
     }
   }
 
@@ -969,9 +969,8 @@ void llvm::remapInstructionsInBlocks(ArrayRef<BasicBlock *> Blocks,
   // Rewrite the code to refer to itself.
   for (auto *BB : Blocks) {
     for (auto &Inst : *BB) {
-      RemapDbgVariableRecordRange(
-          Inst.getModule(), Inst.getDbgRecordRange(), VMap,
-          RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
+      RemapDbgRecordRange(Inst.getModule(), Inst.getDbgRecordRange(), VMap,
+                          RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
       RemapInstruction(&Inst, VMap,
                        RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
     }
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 5cd96412a322..08ba65d9483e 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -639,9 +639,8 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
             !NextDbgInsts.empty()) {
           auto DbgValueRange =
               LoopEntryBranch->cloneDebugInfoFrom(Inst, NextDbgInsts.begin());
-          RemapDbgVariableRecordRange(M, DbgValueRange, ValueMap,
-                                      RF_NoModuleLevelChanges |
-                                          RF_IgnoreMissingLocals);
+          RemapDbgRecordRange(M, DbgValueRange, ValueMap,
+                              RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
           // Erase anything we've seen before.
           for (DbgVariableRecord &DVR :
                make_early_inc_range(filterDbgVars(DbgValueRange)))
@@ -666,9 +665,8 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
       if (LoopEntryBranch->getParent()->IsNewDbgInfoFormat &&
           !NextDbgInsts.empty()) {
         auto Range = C->cloneDebugInfoFrom(Inst, NextDbgInsts.begin());
-        RemapDbgVariableRecordRange(M, Range, ValueMap,
-                                    RF_NoModuleLevelChanges |
-                                        RF_IgnoreMissingLocals);
+        RemapDbgRecordRange(M, Range, ValueMap,
+                            RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
         NextDbgInsts = DbgMarker::getEmptyDbgRecordRange();
         // Erase anything we've seen before.
         for (DbgVariableRecord &DVR :
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index 2d5b5f967ffb..e1af02829c1d 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -917,9 +917,8 @@ bool llvm::UnrollRuntimeLoopRemainder(
     for (Instruction &I : *BB) {
       RemapInstruction(&I, VMap,
                        RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
-      RemapDbgVariableRecordRange(M, I.getDbgRecordRange(), VMap,
-                                  RF_NoModuleLevelChanges |
-                                      RF_IgnoreMissingLocals);
+      RemapDbgRecordRange(M, I.getDbgRecordRange(), VMap,
+                          RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
     }
   }
 
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 5a44a11ecfd2..23a896c59bf6 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1124,9 +1124,8 @@ static void CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(
 
     NewBonusInst->insertInto(PredBlock, PTI->getIterator());
     auto Range = NewBonusInst->cloneDebugInfoFrom(&BonusInst);
-    RemapDbgVariableRecordRange(NewBonusInst->getModule(), Range, VMap,
-                                RF_NoModuleLevelChanges |
-                                    RF_IgnoreMissingLocals);
+    RemapDbgRecordRange(NewBonusInst->getModule(), Range, VMap,
+                        RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
 
     if (isa<DbgInfoIntrinsic>(BonusInst))
       continue;
@@ -3860,8 +3859,8 @@ static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
     PredBlock->getTerminator()->cloneDebugInfoFrom(BB->getTerminator());
     for (DbgVariableRecord &DVR :
          filterDbgVars(PredBlock->getTerminator()->getDbgRecordRange())) {
-      RemapDbgVariableRecord(M, &DVR, VMap,
-                             RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
+      RemapDbgRecord(M, &DVR, VMap,
+                     RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
     }
   }
 
diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp
index 1c877ee937fb..1696e9c72673 100644
--- a/llvm/lib/Transforms/Utils/ValueMapper.cpp
+++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp
@@ -1236,14 +1236,14 @@ void ValueMapper::remapInstruction(Instruction &I) {
   FlushingMapper(pImpl)->remapInstruction(&I);
 }
 
-void ValueMapper::remapDbgVariableRecord(Module *M, DbgVariableRecord &V) {
-  FlushingMapper(pImpl)->remapDbgRecord(V);
+void ValueMapper::remapDbgRecord(Module *M, DbgRecord &DR) {
+  FlushingMapper(pImpl)->remapDbgRecord(DR);
 }
 
-void ValueMapper::remapDbgVariableRecordRange(
+void ValueMapper::remapDbgRecordRange(
     Module *M, iterator_range<DbgRecord::self_iterator> Range) {
-  for (DbgVariableRecord &DVR : filterDbgVars(Range)) {
-    remapDbgVariableRecord(M, DVR);
+  for (DbgRecord &DR : Range) {
+    remapDbgRecord(M, DR);
   }
 }
 
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index 6f4e860d6046..1d0d56a2099c 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -1122,4 +1122,41 @@ TEST_F(CloneModule, IFunc) {
   EXPECT_EQ("resolver", Resolver->getName());
   EXPECT_EQ(GlobalValue::PrivateLinkage, Resolver->getLinkage());
 }
+
+TEST_F(CloneModule, CloneDbgLabel) {
+  LLVMContext Context;
+
+  std::unique_ptr<Module> M = parseIR(Context,
+                                      R"M(
+define void @noop(ptr nocapture noundef writeonly align 4 %dst) local_unnamed_addr !dbg !3 {
+entry:
+  %call = tail call spir_func i64 @foo(i32 noundef 0)
+    #dbg_label(!11, !12)
+  store i64 %call, ptr %dst, align 4
+  ret void
+}
+
+declare i64 @foo(i32 noundef) local_unnamed_addr
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 19.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "<stdin>", directory: "foo")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "noop", scope: !4, file: !4, line: 17, type: !5, scopeLine: 17, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !9)
+!4 = !DIFile(filename: "file", directory: "foo")
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7}
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !8, size: 64)
+!8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!9 = !{}
+!11 = !DILabel(scope: !3, name: "foo", file: !4, line: 23)
+!12 = !DILocation(line: 23, scope: !3)
+)M");
+
+  ASSERT_FALSE(verifyModule(*M, &errs()));
+  auto NewM = llvm::CloneModule(*M);
+  EXPECT_FALSE(verifyModule(*NewM, &errs()));
 }
+} // namespace
-- 
GitLab


From 5636eb89bd69f9c55f4e4aeafaa8c04aa99e5c84 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 8 May 2024 17:05:33 +0100
Subject: [PATCH 0190/1206] [X86] combineBlendOfPermutes - allow whole-lane
 permutation on AVX1 targets.

dd4bf22b9380e797362fac1415a1796da338b2db fixed #91433 but meant we couldn't use vperm2f128 to permute entire 128-bit lanes - if the new 256-bit permutation mask can be scaled to 2x128-bit elements, then we can still fold.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp               | 7 +++++--
 llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll | 3 +--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 126577a4d59e..0410cc33ca33 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -40161,9 +40161,12 @@ combineBlendOfPermutes(MVT VT, SDValue N0, SDValue N1, ArrayRef<int> BlendMask,
       return SDValue();
   }
 
-  // Don't introduce lane-crossing permutes without AVX2.
+  // Don't introduce lane-crossing permutes without AVX2, unless it can be
+  // widened to a lane permute (vperm2f128).
   if (VT.is256BitVector() && !Subtarget.hasAVX2() &&
-      isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), NewPermuteMask))
+      isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(),
+                                NewPermuteMask) &&
+      !canScaleShuffleElements(NewPermuteMask, 2))
     return SDValue();
 
   SDValue NewBlend =
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
index 0c65f756f296..81ce14132c87 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
@@ -308,9 +308,8 @@ define <4 x float> @combine_vpermilvar_4f32_as_insertps(<4 x float> %a0) {
 define <8 x i32> @combine_blend_of_permutes_v8i32(<4 x i64> %a0, <4 x i64> %a1) {
 ; AVX1-LABEL: combine_blend_of_permutes_v8i32:
 ; AVX1:       # %bb.0:
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
 ; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,0,1]
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4],ymm0[5,6],ymm1[7]
 ; AVX1-NEXT:    ret{{[l|q]}}
 ;
 ; AVX2-LABEL: combine_blend_of_permutes_v8i32:
-- 
GitLab


From 6d8901488f160dd92aea5b98fcc21c7fa7c1cbe6 Mon Sep 17 00:00:00 2001
From: "S. Bharadwaj Yadavalli" <Bharadwaj.Yadavalli@microsoft.com>
Date: Wed, 8 May 2024 12:20:41 -0400
Subject: [PATCH 0191/1206] [DXIL] Set DXIL Version in DXIL target triple based
 on shader model version (#91407)

This change set restores commit 080978dd2067d0c9ea7e229aa7696c2480d89ef1 that was reverted to address ASAN
failures and includes a fix for the ASAN failures.

Following is the description of the change:

An earlier commit provided a way to decouple DXIL version from Shader
Model version by representing the DXIL version as `SubArch` in the DXIL
Target Triple and adding corresponding valid DXIL Arch types.

This change constructs DXIL target triple with DXIL version that is
deduced from Shader Model version specified in the following scenarios:

1. When compilation target profile is specified:
    For e.g., DXIL target triple `dxilv1.8-unknown-shader6.8-library` is
    constructed when `-T lib_6_8` is specified.
2. When DXIL target triple without DXIL version is specified:
    For e.g., DXIL target triple `dxilv1.8-pc-shadermodel6.8-library` is
    constructed when `-mtriple=dxil-pc-shadermodel6.8-library` is specified.

Updated relevant HLSL tests that check for target triple.
---
 clang/lib/Basic/Targets.cpp                   |  2 +-
 clang/lib/Driver/ToolChains/HLSL.cpp          | 44 +++++++++-
 clang/test/CodeGenHLSL/basic-target.c         |  2 +-
 clang/test/Driver/dxc_dxv_path.hlsl           |  6 +-
 .../enable_16bit_types_validation.hlsl        |  4 +-
 clang/unittests/Driver/DXCModeTest.cpp        | 22 ++---
 llvm/include/llvm/TargetParser/Triple.h       |  1 +
 llvm/lib/IR/Verifier.cpp                      |  4 +-
 llvm/lib/TargetParser/Triple.cpp              | 86 +++++++++++++++++++
 llvm/unittests/TargetParser/TripleTest.cpp    | 16 ++++
 10 files changed, 166 insertions(+), 21 deletions(-)

diff --git a/clang/lib/Basic/Targets.cpp b/clang/lib/Basic/Targets.cpp
index e3283510c6aa..dc1792b3471e 100644
--- a/clang/lib/Basic/Targets.cpp
+++ b/clang/lib/Basic/Targets.cpp
@@ -760,7 +760,7 @@ using namespace clang::targets;
 TargetInfo *
 TargetInfo::CreateTargetInfo(DiagnosticsEngine &Diags,
                              const std::shared_ptr<TargetOptions> &Opts) {
-  llvm::Triple Triple(Opts->Triple);
+  llvm::Triple Triple(llvm::Triple::normalize(Opts->Triple));
 
   // Construct the target
   std::unique_ptr<TargetInfo> Target = AllocateTarget(Triple, *Opts);
diff --git a/clang/lib/Driver/ToolChains/HLSL.cpp b/clang/lib/Driver/ToolChains/HLSL.cpp
index 558e4db46f81..8286e3be2180 100644
--- a/clang/lib/Driver/ToolChains/HLSL.cpp
+++ b/clang/lib/Driver/ToolChains/HLSL.cpp
@@ -98,9 +98,49 @@ std::optional<std::string> tryParseProfile(StringRef Profile) {
   else if (llvm::getAsUnsignedInteger(Parts[2], 0, Minor))
     return std::nullopt;
 
-  // dxil-unknown-shadermodel-hull
+  // Determine DXIL version using the minor version number of Shader
+  // Model version specified in target profile. Prior to decoupling DXIL version
+  // numbering from that of Shader Model DXIL version 1.Y corresponds to SM 6.Y.
+  // E.g., dxilv1.Y-unknown-shadermodelX.Y-hull
   llvm::Triple T;
-  T.setArch(Triple::ArchType::dxil);
+  Triple::SubArchType SubArch = llvm::Triple::NoSubArch;
+  switch (Minor) {
+  case 0:
+    SubArch = llvm::Triple::DXILSubArch_v1_0;
+    break;
+  case 1:
+    SubArch = llvm::Triple::DXILSubArch_v1_1;
+    break;
+  case 2:
+    SubArch = llvm::Triple::DXILSubArch_v1_2;
+    break;
+  case 3:
+    SubArch = llvm::Triple::DXILSubArch_v1_3;
+    break;
+  case 4:
+    SubArch = llvm::Triple::DXILSubArch_v1_4;
+    break;
+  case 5:
+    SubArch = llvm::Triple::DXILSubArch_v1_5;
+    break;
+  case 6:
+    SubArch = llvm::Triple::DXILSubArch_v1_6;
+    break;
+  case 7:
+    SubArch = llvm::Triple::DXILSubArch_v1_7;
+    break;
+  case 8:
+    SubArch = llvm::Triple::DXILSubArch_v1_8;
+    break;
+  case OfflineLibMinor:
+    // Always consider minor version x as the latest supported DXIL version
+    SubArch = llvm::Triple::LatestDXILSubArch;
+    break;
+  default:
+    // No DXIL Version corresponding to specified Shader Model version found
+    return std::nullopt;
+  }
+  T.setArch(Triple::ArchType::dxil, SubArch);
   T.setOSName(Triple::getOSTypeName(Triple::OSType::ShaderModel).str() +
               VersionTuple(Major, Minor).getAsString());
   T.setEnvironment(Kind);
diff --git a/clang/test/CodeGenHLSL/basic-target.c b/clang/test/CodeGenHLSL/basic-target.c
index 8db711c3f2a5..b97ebf90a7a1 100644
--- a/clang/test/CodeGenHLSL/basic-target.c
+++ b/clang/test/CodeGenHLSL/basic-target.c
@@ -7,4 +7,4 @@
 // RUN: %clang -target dxil-pc-shadermodel6.0-geometry -S -emit-llvm -o - %s | FileCheck %s
 
 // CHECK: target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
-// CHECK: target triple = "dxil-pc-shadermodel6.0-{{[a-z]+}}"
+// CHECK: target triple = "dxilv1.0-pc-shadermodel6.0-{{[a-z]+}}"
diff --git a/clang/test/Driver/dxc_dxv_path.hlsl b/clang/test/Driver/dxc_dxv_path.hlsl
index 3d8e90d0d919..4845de11d5b0 100644
--- a/clang/test/Driver/dxc_dxv_path.hlsl
+++ b/clang/test/Driver/dxc_dxv_path.hlsl
@@ -7,12 +7,12 @@
 // DXV_PATH:dxv{{(.exe)?}}" "-" "-o" "-"
 
 // RUN: %clang_dxc -I test -Vd -Tlib_6_3  -### %s 2>&1 | FileCheck %s --check-prefix=VD
-// VD:"-cc1"{{.*}}"-triple" "dxil-unknown-shadermodel6.3-library"
+// VD:"-cc1"{{.*}}"-triple" "dxilv1.3-unknown-shadermodel6.3-library"
 // VD-NOT:dxv not found
 
 // RUN: %clang_dxc -Tlib_6_3 -ccc-print-bindings --dxv-path=%T -Fo %t.dxo  %s 2>&1 | FileCheck %s --check-prefix=BINDINGS
-// BINDINGS: "dxil-unknown-shadermodel6.3-library" - "clang", inputs: ["[[INPUT:.+]]"], output: "[[DXC:.+]].dxo"
-// BINDINGS-NEXT: "dxil-unknown-shadermodel6.3-library" - "hlsl::Validator", inputs: ["[[DXC]].dxo"]
+// BINDINGS: "dxilv1.3-unknown-shadermodel6.3-library" - "clang", inputs: ["[[INPUT:.+]]"], output: "[[DXC:.+]].dxo"
+// BINDINGS-NEXT: "dxilv1.3-unknown-shadermodel6.3-library" - "hlsl::Validator", inputs: ["[[DXC]].dxo"]
 
 // RUN: %clang_dxc -Tlib_6_3 -ccc-print-phases --dxv-path=%T -Fo %t.dxc  %s 2>&1 | FileCheck %s --check-prefix=PHASES
 
diff --git a/clang/test/Options/enable_16bit_types_validation.hlsl b/clang/test/Options/enable_16bit_types_validation.hlsl
index 89fe26790c52..bcb217e8982e 100644
--- a/clang/test/Options/enable_16bit_types_validation.hlsl
+++ b/clang/test/Options/enable_16bit_types_validation.hlsl
@@ -9,11 +9,11 @@
 // HV_invalid_2017: error: '-enable-16bit-types' option requires target HLSL Version >= 2018 and shader model >= 6.2, but HLSL Version is 'hlsl2017' and shader model is '6.4'
 // TP_invalid: error: '-enable-16bit-types' option requires target HLSL Version >= 2018 and shader model >= 6.2, but HLSL Version is 'hlsl2021' and shader model is '6.0'
 
-// valid_2021: "dxil-unknown-shadermodel6.4-library"
+// valid_2021: "dxilv1.4-unknown-shadermodel6.4-library"
 // valid_2021-SAME: "-std=hlsl2021"
 // valid_2021-SAME: "-fnative-half-type"
 
-// valid_2018: "dxil-unknown-shadermodel6.4-library"
+// valid_2018: "dxilv1.4-unknown-shadermodel6.4-library"
 // valid_2018-SAME: "-std=hlsl2018"
 // valid_2018-SAME: "-fnative-half-type"
 
diff --git a/clang/unittests/Driver/DXCModeTest.cpp b/clang/unittests/Driver/DXCModeTest.cpp
index b3767c042edb..416723d498a2 100644
--- a/clang/unittests/Driver/DXCModeTest.cpp
+++ b/clang/unittests/Driver/DXCModeTest.cpp
@@ -68,25 +68,27 @@ TEST(DxcModeTest, TargetProfileValidation) {
   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
   DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagConsumer);
 
-  validateTargetProfile("-Tvs_6_0", "dxil--shadermodel6.0-vertex",
+  validateTargetProfile("-Tvs_6_0", "dxilv1.0--shadermodel6.0-vertex",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Ths_6_1", "dxil--shadermodel6.1-hull",
+  validateTargetProfile("-Ths_6_1", "dxilv1.1--shadermodel6.1-hull",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tds_6_2", "dxil--shadermodel6.2-domain",
+  validateTargetProfile("-Tds_6_2", "dxilv1.2--shadermodel6.2-domain",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tds_6_2", "dxil--shadermodel6.2-domain",
+  validateTargetProfile("-Tds_6_2", "dxilv1.2--shadermodel6.2-domain",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tgs_6_3", "dxil--shadermodel6.3-geometry",
+  validateTargetProfile("-Tgs_6_3", "dxilv1.3--shadermodel6.3-geometry",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tps_6_4", "dxil--shadermodel6.4-pixel",
+  validateTargetProfile("-Tps_6_4", "dxilv1.4--shadermodel6.4-pixel",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tcs_6_5", "dxil--shadermodel6.5-compute",
+  validateTargetProfile("-Tcs_6_5", "dxilv1.5--shadermodel6.5-compute",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tms_6_6", "dxil--shadermodel6.6-mesh",
+  validateTargetProfile("-Tms_6_6", "dxilv1.6--shadermodel6.6-mesh",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tas_6_7", "dxil--shadermodel6.7-amplification",
+  validateTargetProfile("-Tas_6_7", "dxilv1.7--shadermodel6.7-amplification",
                         InMemoryFileSystem, Diags);
-  validateTargetProfile("-Tlib_6_x", "dxil--shadermodel6.15-library",
+  validateTargetProfile("-Tcs_6_8", "dxilv1.8--shadermodel6.8-compute",
+                        InMemoryFileSystem, Diags);
+  validateTargetProfile("-Tlib_6_x", "dxilv1.8--shadermodel6.15-library",
                         InMemoryFileSystem, Diags);
 
   // Invalid tests.
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 20cca5928782..8f9d99816931 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -176,6 +176,7 @@ public:
     DXILSubArch_v1_6,
     DXILSubArch_v1_7,
     DXILSubArch_v1_8,
+    LatestDXILSubArch = DXILSubArch_v1_8,
   };
   enum VendorType {
     UnknownVendor,
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index aa8160d18edd..50f8d6ec8420 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -152,8 +152,8 @@ struct VerifierSupport {
   bool TreatBrokenDebugInfoAsError = true;
 
   explicit VerifierSupport(raw_ostream *OS, const Module &M)
-      : OS(OS), M(M), MST(&M), TT(M.getTargetTriple()), DL(M.getDataLayout()),
-        Context(M.getContext()) {}
+      : OS(OS), M(M), MST(&M), TT(Triple::normalize(M.getTargetTriple())),
+        DL(M.getDataLayout()), Context(M.getContext()) {}
 
 private:
   void Write(const Module *M) {
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index 18ec60296810..ef40ccf36806 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -115,6 +115,31 @@ StringRef Triple::getArchName(ArchType Kind, SubArchType SubArch) {
     if (SubArch == AArch64SubArch_arm64e)
       return "arm64e";
     break;
+  case Triple::dxil:
+    switch (SubArch) {
+    case Triple::NoSubArch:
+    case Triple::DXILSubArch_v1_0:
+      return "dxilv1.0";
+    case Triple::DXILSubArch_v1_1:
+      return "dxilv1.1";
+    case Triple::DXILSubArch_v1_2:
+      return "dxilv1.2";
+    case Triple::DXILSubArch_v1_3:
+      return "dxilv1.3";
+    case Triple::DXILSubArch_v1_4:
+      return "dxilv1.4";
+    case Triple::DXILSubArch_v1_5:
+      return "dxilv1.5";
+    case Triple::DXILSubArch_v1_6:
+      return "dxilv1.6";
+    case Triple::DXILSubArch_v1_7:
+      return "dxilv1.7";
+    case Triple::DXILSubArch_v1_8:
+      return "dxilv1.8";
+    default:
+      break;
+    }
+    break;
   default:
     break;
   }
@@ -1014,6 +1039,53 @@ Triple::Triple(const Twine &ArchStr, const Twine &VendorStr, const Twine &OSStr,
     ObjectFormat = getDefaultFormat(*this);
 }
 
+static VersionTuple parseVersionFromName(StringRef Name);
+
+static StringRef getDXILArchNameFromShaderModel(StringRef ShaderModelStr) {
+  VersionTuple Ver =
+      parseVersionFromName(ShaderModelStr.drop_front(strlen("shadermodel")));
+  // Default DXIL minor version when Shader Model version is anything other
+  // than 6.[0...8] or 6.x (which translates to latest current SM version)
+  const unsigned SMMajor = 6;
+  if (!Ver.empty()) {
+    if (Ver.getMajor() == SMMajor) {
+      if (std::optional<unsigned> SMMinor = Ver.getMinor()) {
+        switch (*SMMinor) {
+        case 0:
+          return Triple::getArchName(Triple::dxil, Triple::DXILSubArch_v1_0);
+        case 1:
+          return Triple::getArchName(Triple::dxil, Triple::DXILSubArch_v1_1);
+        case 2:
+          return Triple::getArchName(Triple::dxil, Triple::DXILSubArch_v1_2);
+        case 3:
+          return Triple::getArchName(Triple::dxil, Triple::DXILSubArch_v1_3);
+        case 4:
+          return Triple::getArchName(Triple::dxil, Triple::DXILSubArch_v1_4);
+        case 5:
+          return Triple::getArchName(Triple::dxil, Triple::DXILSubArch_v1_5);
+        case 6:
+          return Triple::getArchName(Triple::dxil, Triple::DXILSubArch_v1_6);
+        case 7:
+          return Triple::getArchName(Triple::dxil, Triple::DXILSubArch_v1_7);
+        case 8:
+          return Triple::getArchName(Triple::dxil, Triple::DXILSubArch_v1_8);
+        default:
+          report_fatal_error("Unsupported Shader Model version", false);
+        }
+      }
+    }
+  } else {
+    // Special case: DXIL minor version is set to LatestCurrentDXILMinor for
+    // shadermodel6.x is
+    if (ShaderModelStr == "shadermodel6.x") {
+      return Triple::getArchName(Triple::dxil, Triple::LatestDXILSubArch);
+    }
+  }
+  // DXIL version corresponding to Shader Model version other than 6.Minor
+  // is 1.0
+  return Triple::getArchName(Triple::dxil, Triple::DXILSubArch_v1_0);
+}
+
 std::string Triple::normalize(StringRef Str) {
   bool IsMinGW32 = false;
   bool IsCygwin = false;
@@ -1206,6 +1278,20 @@ std::string Triple::normalize(StringRef Str) {
     }
   }
 
+  // Normalize DXIL triple if it does not include DXIL version number.
+  // Determine DXIL version number using the minor version number of Shader
+  // Model version specified in target triple, if any. Prior to decoupling DXIL
+  // version numbering from that of Shader Model DXIL version 1.Y corresponds to
+  // SM 6.Y. E.g., dxilv1.Y-unknown-shadermodelX.Y-hull
+  if (Components[0] == "dxil") {
+    if (Components.size() > 4) {
+      Components.resize(4);
+    }
+    // Add DXIL version only if shadermodel is specified in the triple
+    if (OS == Triple::ShaderModel) {
+      Components[0] = getDXILArchNameFromShaderModel(Components[2]);
+    }
+  }
   // Stick the corrected components back together to form the normalized string.
   return join(Components, "-");
 }
diff --git a/llvm/unittests/TargetParser/TripleTest.cpp b/llvm/unittests/TargetParser/TripleTest.cpp
index 8e90ee6858f4..f93dc3671197 100644
--- a/llvm/unittests/TargetParser/TripleTest.cpp
+++ b/llvm/unittests/TargetParser/TripleTest.cpp
@@ -2533,4 +2533,20 @@ TEST(TripleTest, isArmMClass) {
     EXPECT_TRUE(T.isArmMClass());
   }
 }
+
+TEST(TripleTest, DXILNormaizeWithVersion) {
+  EXPECT_EQ("dxilv1.0-unknown-shadermodel6.0",
+            Triple::normalize("dxilv1.0--shadermodel6.0"));
+  EXPECT_EQ("dxilv1.0-unknown-shadermodel6.0",
+            Triple::normalize("dxil--shadermodel6.0"));
+  EXPECT_EQ("dxilv1.1-unknown-shadermodel6.1-library",
+            Triple::normalize("dxil-shadermodel6.1-unknown-library"));
+  EXPECT_EQ("dxilv1.8-unknown-shadermodel6.x-unknown",
+            Triple::normalize("dxil-unknown-shadermodel6.x-unknown"));
+  EXPECT_EQ("dxilv1.8-unknown-shadermodel6.x-unknown",
+            Triple::normalize("dxil-unknown-shadermodel6.x-unknown"));
+  EXPECT_EQ("dxil-unknown-unknown-unknown", Triple::normalize("dxil---"));
+  EXPECT_EQ("dxilv1.0-pc-shadermodel5.0-compute",
+            Triple::normalize("dxil-shadermodel5.0-pc-compute"));
+}
 } // end anonymous namespace
-- 
GitLab


From efad14954c9d5bdfaddaca948be6cd7e71a1d1b0 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Wed, 8 May 2024 09:22:30 -0700
Subject: [PATCH 0192/1206] [Support] Add end/error to decode[US]LEB128AndInc

Follow-up to #85739 to encourage error checking. We make `end` mandatory
and add decodeULEB128AndIncUnsafe to be used without `end`.

Pull Request: https://github.com/llvm/llvm-project/pull/90006
---
 llvm/include/llvm/Support/LEB128.h     | 14 ++++++++++----
 llvm/unittests/Support/LEB128Test.cpp  | 25 ++++++++++++++++++++++++-
 llvm/utils/TableGen/DecoderEmitter.cpp | 16 ++++++++--------
 3 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/llvm/include/llvm/Support/LEB128.h b/llvm/include/llvm/Support/LEB128.h
index c4e741549f3f..a15b73bc14dc 100644
--- a/llvm/include/llvm/Support/LEB128.h
+++ b/llvm/include/llvm/Support/LEB128.h
@@ -200,20 +200,26 @@ inline int64_t decodeSLEB128(const uint8_t *p, unsigned *n = nullptr,
   return Value;
 }
 
-inline uint64_t decodeULEB128AndInc(const uint8_t *&p) {
+inline uint64_t decodeULEB128AndInc(const uint8_t *&p, const uint8_t *end,
+                                    const char **error = nullptr) {
   unsigned n;
-  auto ret = decodeULEB128(p, &n);
+  auto ret = decodeULEB128(p, &n, end, error);
   p += n;
   return ret;
 }
 
-inline int64_t decodeSLEB128AndInc(const uint8_t *&p) {
+inline int64_t decodeSLEB128AndInc(const uint8_t *&p, const uint8_t *end,
+                                   const char **error = nullptr) {
   unsigned n;
-  auto ret = decodeSLEB128(p, &n);
+  auto ret = decodeSLEB128(p, &n, end, error);
   p += n;
   return ret;
 }
 
+inline uint64_t decodeULEB128AndIncUnsafe(const uint8_t *&p) {
+  return decodeULEB128AndInc(p, nullptr);
+}
+
 /// Utility function to get the size of the ULEB128-encoded value.
 extern unsigned getULEB128Size(uint64_t Value);
 
diff --git a/llvm/unittests/Support/LEB128Test.cpp b/llvm/unittests/Support/LEB128Test.cpp
index 08b8c5573ce6..60f5ddd568ca 100644
--- a/llvm/unittests/Support/LEB128Test.cpp
+++ b/llvm/unittests/Support/LEB128Test.cpp
@@ -155,6 +155,12 @@ TEST(LEB128Test, DecodeInvalidULEB128) {
     EXPECT_NE(Error, nullptr);                                                 \
     EXPECT_EQ(0ul, Actual);                                                    \
     EXPECT_EQ(ERROR_OFFSET, ErrorOffset);                                      \
+    Value = reinterpret_cast<const uint8_t *>(VALUE);                          \
+    Error = nullptr;                                                           \
+    Actual = decodeULEB128AndInc(Value, Value + strlen(VALUE), &Error);        \
+    EXPECT_NE(Error, nullptr);                                                 \
+    EXPECT_EQ(0ul, Actual);                                                    \
+    EXPECT_EQ(ERROR_OFFSET, Value - reinterpret_cast<const uint8_t *>(VALUE)); \
   } while (0)
 
   // Buffer overflow.
@@ -224,6 +230,12 @@ TEST(LEB128Test, DecodeInvalidSLEB128) {
     EXPECT_NE(Error, nullptr);                                                 \
     EXPECT_EQ(0ul, Actual);                                                    \
     EXPECT_EQ(ERROR_OFFSET, ErrorOffset);                                      \
+    Value = reinterpret_cast<const uint8_t *>(VALUE);                          \
+    Error = nullptr;                                                           \
+    Actual = decodeSLEB128AndInc(Value, Value + strlen(VALUE), &Error);        \
+    EXPECT_NE(Error, nullptr);                                                 \
+    EXPECT_EQ(0ul, Actual);                                                    \
+    EXPECT_EQ(ERROR_OFFSET, Value - reinterpret_cast<const uint8_t *>(VALUE)); \
   } while (0)
 
   // Buffer overflow.
@@ -246,7 +258,7 @@ TEST(LEB128Test, DecodeAndInc) {
 #define EXPECT_LEB128(FUN, VALUE, SIZE)                                        \
   do {                                                                         \
     const uint8_t *V = reinterpret_cast<const uint8_t *>(VALUE), *P = V;       \
-    auto Expected = FUN(P), Actual = FUN##AndInc(P);                           \
+    auto Expected = FUN(P), Actual = FUN##AndInc(P, P + strlen(VALUE));        \
     EXPECT_EQ(Actual, Expected);                                               \
     EXPECT_EQ(P - V, SIZE);                                                    \
   } while (0)
@@ -255,6 +267,17 @@ TEST(LEB128Test, DecodeAndInc) {
   EXPECT_LEB128(decodeSLEB128, "\x7f", 1);
   EXPECT_LEB128(decodeSLEB128, "\x80\x01", 2);
 #undef EXPECT_LEB128
+
+#define EXPECT_LEB128(FUN, VALUE, SIZE)                                        \
+  do {                                                                         \
+    const uint8_t *V = reinterpret_cast<const uint8_t *>(VALUE), *P = V;       \
+    auto Expected = FUN(P), Actual = FUN##AndIncUnsafe(P);                     \
+    EXPECT_EQ(Actual, Expected);                                               \
+    EXPECT_EQ(P - V, SIZE);                                                    \
+  } while (0)
+  EXPECT_LEB128(decodeULEB128, "\x7f", 1);
+  EXPECT_LEB128(decodeULEB128, "\x80\x01", 2);
+#undef EXPECT_LEB128
 }
 
 TEST(LEB128Test, SLEB128Size) {
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 3bd7f432ff9a..c303322e63b4 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -2301,7 +2301,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     }
     case MCD::OPC_CheckField: {
       // Decode the start value.
-      unsigned Start = decodeULEB128AndInc(++Ptr);
+      unsigned Start = decodeULEB128AndIncUnsafe(++Ptr);
       unsigned Len = *Ptr;)";
   if (IsVarLenInst)
     OS << "\n      makeUp(insn, Start + Len);";
@@ -2328,7 +2328,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     }
     case MCD::OPC_CheckPredicate: {
       // Decode the Predicate Index value.
-      unsigned PIdx = decodeULEB128AndInc(++Ptr);
+      unsigned PIdx = decodeULEB128AndIncUnsafe(++Ptr);
       // NumToSkip is a plain 24-bit integer.
       unsigned NumToSkip = *Ptr++;
       NumToSkip |= (*Ptr++) << 8;
@@ -2345,8 +2345,8 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     }
     case MCD::OPC_Decode: {
       // Decode the Opcode value.
-      unsigned Opc = decodeULEB128AndInc(++Ptr);
-      unsigned DecodeIdx = decodeULEB128AndInc(Ptr);
+      unsigned Opc = decodeULEB128AndIncUnsafe(++Ptr);
+      unsigned DecodeIdx = decodeULEB128AndIncUnsafe(Ptr);
 
       MI.clear();
       MI.setOpcode(Opc);
@@ -2366,8 +2366,8 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     }
     case MCD::OPC_TryDecode: {
       // Decode the Opcode value.
-      unsigned Opc = decodeULEB128AndInc(++Ptr);
-      unsigned DecodeIdx = decodeULEB128AndInc(Ptr);
+      unsigned Opc = decodeULEB128AndIncUnsafe(++Ptr);
+      unsigned DecodeIdx = decodeULEB128AndIncUnsafe(Ptr);
       // NumToSkip is a plain 24-bit integer.
       unsigned NumToSkip = *Ptr++;
       NumToSkip |= (*Ptr++) << 8;
@@ -2399,8 +2399,8 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
     }
     case MCD::OPC_SoftFail: {
       // Decode the mask values.
-      uint64_t PositiveMask = decodeULEB128AndInc(++Ptr);
-      uint64_t NegativeMask = decodeULEB128AndInc(Ptr);
+      uint64_t PositiveMask = decodeULEB128AndIncUnsafe(++Ptr);
+      uint64_t NegativeMask = decodeULEB128AndIncUnsafe(Ptr);
       bool Fail = (insn & PositiveMask) != 0 || (~insn & NegativeMask) != 0;
       if (Fail)
         S = MCDisassembler::SoftFail;
-- 
GitLab


From 8d9b15497d70ac782d7d01a2d606f9fec7e7f642 Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Wed, 8 May 2024 10:28:54 -0600
Subject: [PATCH 0193/1206] Fix unused private field warning (#91500)

After 11a6799740f8 "[clang][CodeGen] Omit pre-opt link when post-opt is
link requested (#85672)" I'm seeing a new warning:

> BackendConsumer.h:37:22: error: private field 'FileMgr' is not used
[-Werror,-Wunused-private-field]

Remove the field since it's no longer used.
---
 clang/lib/CodeGen/BackendConsumer.h |  7 +++----
 clang/lib/CodeGen/CodeGenAction.cpp | 19 ++++++++-----------
 2 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/clang/lib/CodeGen/BackendConsumer.h b/clang/lib/CodeGen/BackendConsumer.h
index f9edbe901bb8..0fe9929dca2b 100644
--- a/clang/lib/CodeGen/BackendConsumer.h
+++ b/clang/lib/CodeGen/BackendConsumer.h
@@ -34,7 +34,6 @@ class BackendConsumer : public ASTConsumer {
   const CodeGenOptions &CodeGenOpts;
   const TargetOptions &TargetOpts;
   const LangOptions &LangOpts;
-  const FileManager &FileMgr;
   std::unique_ptr<raw_pwrite_stream> AsmOutStream;
   ASTContext *Context;
   IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS;
@@ -76,7 +75,7 @@ public:
                   const PreprocessorOptions &PPOpts,
                   const CodeGenOptions &CodeGenOpts,
                   const TargetOptions &TargetOpts, const LangOptions &LangOpts,
-                  const FileManager &FileMgr, const std::string &InFile,
+                  const std::string &InFile,
                   SmallVector<LinkModule, 4> LinkModules,
                   std::unique_ptr<raw_pwrite_stream> OS, llvm::LLVMContext &C,
                   CoverageSourceInfo *CoverageInfo = nullptr);
@@ -90,8 +89,8 @@ public:
                   const PreprocessorOptions &PPOpts,
                   const CodeGenOptions &CodeGenOpts,
                   const TargetOptions &TargetOpts, const LangOptions &LangOpts,
-                  const FileManager &FileMgr, llvm::Module *Module,
-                  SmallVector<LinkModule, 4> LinkModules, llvm::LLVMContext &C,
+                  llvm::Module *Module, SmallVector<LinkModule, 4> LinkModules,
+                  llvm::LLVMContext &C,
                   CoverageSourceInfo *CoverageInfo = nullptr);
 
   llvm::Module *getModule() const;
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index 0255f05b1f90..6d3efdb5ffe3 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -114,13 +114,12 @@ BackendConsumer::BackendConsumer(
     const HeaderSearchOptions &HeaderSearchOpts,
     const PreprocessorOptions &PPOpts, const CodeGenOptions &CodeGenOpts,
     const TargetOptions &TargetOpts, const LangOptions &LangOpts,
-    const FileManager &FileMgr, const std::string &InFile,
-    SmallVector<LinkModule, 4> LinkModules,
+    const std::string &InFile, SmallVector<LinkModule, 4> LinkModules,
     std::unique_ptr<raw_pwrite_stream> OS, LLVMContext &C,
     CoverageSourceInfo *CoverageInfo)
     : Diags(Diags), Action(Action), HeaderSearchOpts(HeaderSearchOpts),
       CodeGenOpts(CodeGenOpts), TargetOpts(TargetOpts), LangOpts(LangOpts),
-      FileMgr(FileMgr), AsmOutStream(std::move(OS)), Context(nullptr), FS(VFS),
+      AsmOutStream(std::move(OS)), Context(nullptr), FS(VFS),
       LLVMIRGeneration("irgen", "LLVM IR Generation Time"),
       LLVMIRGenerationRefCount(0),
       Gen(CreateLLVMCodeGen(Diags, InFile, std::move(VFS), HeaderSearchOpts,
@@ -140,12 +139,11 @@ BackendConsumer::BackendConsumer(
     const HeaderSearchOptions &HeaderSearchOpts,
     const PreprocessorOptions &PPOpts, const CodeGenOptions &CodeGenOpts,
     const TargetOptions &TargetOpts, const LangOptions &LangOpts,
-    const FileManager &FileMgr, llvm::Module *Module,
-    SmallVector<LinkModule, 4> LinkModules, LLVMContext &C,
-    CoverageSourceInfo *CoverageInfo)
+    llvm::Module *Module, SmallVector<LinkModule, 4> LinkModules,
+    LLVMContext &C, CoverageSourceInfo *CoverageInfo)
     : Diags(Diags), Action(Action), HeaderSearchOpts(HeaderSearchOpts),
       CodeGenOpts(CodeGenOpts), TargetOpts(TargetOpts), LangOpts(LangOpts),
-      FileMgr(FileMgr), Context(nullptr), FS(VFS),
+      Context(nullptr), FS(VFS),
       LLVMIRGeneration("irgen", "LLVM IR Generation Time"),
       LLVMIRGenerationRefCount(0),
       Gen(CreateLLVMCodeGen(Diags, "", std::move(VFS), HeaderSearchOpts, PPOpts,
@@ -1022,9 +1020,8 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   std::unique_ptr<BackendConsumer> Result(new BackendConsumer(
       BA, CI.getDiagnostics(), &CI.getVirtualFileSystem(),
       CI.getHeaderSearchOpts(), CI.getPreprocessorOpts(), CI.getCodeGenOpts(),
-      CI.getTargetOpts(), CI.getLangOpts(), CI.getFileManager(),
-      std::string(InFile), std::move(LinkModules), std::move(OS), *VMContext,
-      CoverageInfo));
+      CI.getTargetOpts(), CI.getLangOpts(), std::string(InFile),
+      std::move(LinkModules), std::move(OS), *VMContext, CoverageInfo));
   BEConsumer = Result.get();
 
   // Enable generating macro debug info only when debug info is not disabled and
@@ -1195,7 +1192,7 @@ void CodeGenAction::ExecuteAction() {
   BackendConsumer Result(BA, CI.getDiagnostics(), &CI.getVirtualFileSystem(),
                          CI.getHeaderSearchOpts(), CI.getPreprocessorOpts(),
                          CI.getCodeGenOpts(), CI.getTargetOpts(),
-                         CI.getLangOpts(), CI.getFileManager(), TheModule.get(),
+                         CI.getLangOpts(), TheModule.get(),
                          std::move(LinkModules), *VMContext, nullptr);
 
   // Link in each pending link module.
-- 
GitLab


From b59461ac63aa1770a617f96bab31010442bd2090 Mon Sep 17 00:00:00 2001
From: Aleksandr Platonov <platonov.aleksandr@huawei.com>
Date: Wed, 8 May 2024 19:40:16 +0300
Subject: [PATCH 0195/1206] [ADT] Add back ability to compare StringSet
 (#91374)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

StringSet comparison was broken after moving from llvm::Optional to
std::optional because std::nullopt_t is not equality-comparable. Without
this patch a try to compare objects of StringSet type leads to
compilation error:
```
llvm-project/llvm/include/llvm/ADT/StringMap.h:294:33: error: no match for ‘operator==’ (operand types are ‘std::nullopt_t’ and ‘std::nullopt_t’)
294 |       if (!(KeyValue.getValue() == FindInRHS->getValue()))
```
---
 llvm/include/llvm/ADT/StringMap.h    | 6 ++++--
 llvm/unittests/ADT/StringSetTest.cpp | 8 ++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/ADT/StringMap.h b/llvm/include/llvm/ADT/StringMap.h
index 453d91349e35..daaf82654e09 100644
--- a/llvm/include/llvm/ADT/StringMap.h
+++ b/llvm/include/llvm/ADT/StringMap.h
@@ -291,8 +291,10 @@ public:
       if (FindInRHS == RHS.end())
         return false;
 
-      if (!(KeyValue.getValue() == FindInRHS->getValue()))
-        return false;
+      if constexpr (!std::is_same_v<ValueTy, std::nullopt_t>) {
+        if (!(KeyValue.getValue() == FindInRHS->getValue()))
+          return false;
+      }
     }
 
     return true;
diff --git a/llvm/unittests/ADT/StringSetTest.cpp b/llvm/unittests/ADT/StringSetTest.cpp
index e3703f6f0150..a804c1f17d1c 100644
--- a/llvm/unittests/ADT/StringSetTest.cpp
+++ b/llvm/unittests/ADT/StringSetTest.cpp
@@ -73,4 +73,12 @@ TEST_F(StringSetTest, Contains) {
   EXPECT_FALSE(Set.contains("test"));
 }
 
+TEST_F(StringSetTest, Equal) {
+  StringSet<> A = {"A"};
+  StringSet<> B = {"B"};
+  ASSERT_TRUE(A != B);
+  ASSERT_FALSE(A == B);
+  ASSERT_TRUE(A == A);
+}
+
 } // end anonymous namespace
-- 
GitLab


From 4298fc5eb5c483fb72db6fce062352087dfd0acf Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Wed, 8 May 2024 10:13:01 -0700
Subject: [PATCH 0196/1206] [RISCV] Move strength reduction of mul X, 3/5/9*2^N
 to combine (#89966)

This moves our last major category tablegen driven multiply strength
reduction into the post legalize combine framework. The one slightly
tricky bit is making sure that we use a leading shl if we can form a
slli.uw, and trailing shl otherwise. Having the trailing shl is critical
for shNadd matching, and folding any following sext.w.

As can be seen in the TD deltas, this allows us to kill off both the
actual multiply patterns and the explicit add (mul X, C) Y patterns. The
later are now handled by the generic shNadd matching code, with the
exception of the THead only C=200 case because we don't (yet) have a
multiply expansion with two shNadd + a shift.

---------

Co-authored-by: Yingwei Zheng <dtcxzyw@qq.com>
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 25 ++++++-
 llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td | 29 --------
 llvm/lib/Target/RISCV/RISCVInstrInfoZb.td     | 74 -------------------
 llvm/test/CodeGen/RISCV/addimm-mulimm.ll      |  9 ++-
 .../CodeGen/RISCV/rv64-legal-i32/rv64zba.ll   |  6 +-
 llvm/test/CodeGen/RISCV/rv64zba.ll            |  6 +-
 6 files changed, 32 insertions(+), 117 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 3536eb4c0ba4..846768f6d631 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -13565,10 +13565,27 @@ static SDValue expandMul(SDNode *N, SelectionDAG &DAG,
     if (MulAmt % Divisor != 0)
       continue;
     uint64_t MulAmt2 = MulAmt / Divisor;
-    // 3/5/9 * 2^N -> shXadd (sll X, C), (sll X, C)
-    // Matched in tablegen, avoid perturbing patterns.
-    if (isPowerOf2_64(MulAmt2))
-      return SDValue();
+    // 3/5/9 * 2^N ->  shl (shXadd X, X), N
+    if (isPowerOf2_64(MulAmt2)) {
+      SDLoc DL(N);
+      SDValue X = N->getOperand(0);
+      // Put the shift first if we can fold a zext into the
+      // shift forming a slli.uw.
+      if (X.getOpcode() == ISD::AND && isa<ConstantSDNode>(X.getOperand(1)) &&
+          X.getConstantOperandVal(1) == UINT64_C(0xffffffff)) {
+        SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, X,
+                                  DAG.getConstant(Log2_64(MulAmt2), DL, VT));
+        return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Shl,
+                           DAG.getConstant(Log2_64(Divisor - 1), DL, VT), Shl);
+      }
+      // Otherwise, put rhe shl second so that it can fold with following
+      // instructions (e.g. sext or add).
+      SDValue Mul359 =
+          DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
+                      DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
+      return DAG.getNode(ISD::SHL, DL, VT, Mul359,
+                         DAG.getConstant(Log2_64(MulAmt2), DL, VT));
+    }
 
     // 3/5/9 * 3/5/9 -> shXadd (shYadd X, X), (shYadd X, X)
     if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
index b398c5e7fec2..bc14f165d962 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
@@ -549,40 +549,11 @@ def : Pat<(add_non_imm12 sh2add_op:$rs1, (XLenVT GPR:$rs2)),
 def : Pat<(add_non_imm12 sh3add_op:$rs1, (XLenVT GPR:$rs2)),
           (TH_ADDSL GPR:$rs2, sh3add_op:$rs1, 3)>;
 
-def : Pat<(add (mul_oneuse GPR:$rs1, (XLenVT 6)), GPR:$rs2),
-          (TH_ADDSL GPR:$rs2, (XLenVT (TH_ADDSL GPR:$rs1, GPR:$rs1, 1)), 1)>;
-def : Pat<(add (mul_oneuse GPR:$rs1, (XLenVT 10)), GPR:$rs2),
-          (TH_ADDSL GPR:$rs2, (XLenVT (TH_ADDSL GPR:$rs1, GPR:$rs1, 2)), 1)>;
-def : Pat<(add (mul_oneuse GPR:$rs1, (XLenVT 18)), GPR:$rs2),
-          (TH_ADDSL GPR:$rs2, (XLenVT (TH_ADDSL GPR:$rs1, GPR:$rs1, 3)), 1)>;
-def : Pat<(add (mul_oneuse GPR:$rs1, (XLenVT 12)), GPR:$rs2),
-          (TH_ADDSL GPR:$rs2, (XLenVT (TH_ADDSL GPR:$rs1, GPR:$rs1, 1)), 2)>;
-def : Pat<(add (mul_oneuse GPR:$rs1, (XLenVT 20)), GPR:$rs2),
-          (TH_ADDSL GPR:$rs2, (XLenVT (TH_ADDSL GPR:$rs1, GPR:$rs1, 2)), 2)>;
-def : Pat<(add (mul_oneuse GPR:$rs1, (XLenVT 36)), GPR:$rs2),
-          (TH_ADDSL GPR:$rs2, (XLenVT (TH_ADDSL GPR:$rs1, GPR:$rs1, 3)), 2)>;
-def : Pat<(add (mul_oneuse GPR:$rs1, (XLenVT 24)), GPR:$rs2),
-          (TH_ADDSL GPR:$rs2, (XLenVT (TH_ADDSL GPR:$rs1, GPR:$rs1, 1)), 3)>;
-def : Pat<(add (mul_oneuse GPR:$rs1, (XLenVT 40)), GPR:$rs2),
-          (TH_ADDSL GPR:$rs2, (XLenVT (TH_ADDSL GPR:$rs1, GPR:$rs1, 2)), 3)>;
-def : Pat<(add (mul_oneuse GPR:$rs1, (XLenVT 72)), GPR:$rs2),
-          (TH_ADDSL GPR:$rs2, (XLenVT (TH_ADDSL GPR:$rs1, GPR:$rs1, 3)), 3)>;
-
 def : Pat<(add (XLenVT GPR:$r), CSImm12MulBy4:$i),
           (TH_ADDSL GPR:$r, (XLenVT (ADDI (XLenVT X0), (SimmShiftRightBy2XForm CSImm12MulBy4:$i))), 2)>;
 def : Pat<(add (XLenVT GPR:$r), CSImm12MulBy8:$i),
           (TH_ADDSL GPR:$r, (XLenVT (ADDI (XLenVT X0), (SimmShiftRightBy3XForm CSImm12MulBy8:$i))), 3)>;
 
-def : Pat<(mul (XLenVT GPR:$r), C3LeftShift:$i),
-          (SLLI (XLenVT (TH_ADDSL GPR:$r, GPR:$r, 1)),
-                (TrailingZeros C3LeftShift:$i))>;
-def : Pat<(mul (XLenVT GPR:$r), C5LeftShift:$i),
-          (SLLI (XLenVT (TH_ADDSL GPR:$r, GPR:$r, 2)),
-                (TrailingZeros C5LeftShift:$i))>;
-def : Pat<(mul (XLenVT GPR:$r), C9LeftShift:$i),
-          (SLLI (XLenVT (TH_ADDSL GPR:$r, GPR:$r, 3)),
-                (TrailingZeros C9LeftShift:$i))>;
-
 def : Pat<(mul_const_oneuse GPR:$r, (XLenVT 200)),
           (SLLI (XLenVT (TH_ADDSL (XLenVT (TH_ADDSL GPR:$r, GPR:$r, 2)),
                                   (XLenVT (TH_ADDSL GPR:$r, GPR:$r, 2)), 2)), 3)>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index ffe2b7e27120..8a0bbf6abd33 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -173,42 +173,6 @@ def BCLRIANDIMaskLow : SDNodeXForm<imm, [{
                                    SDLoc(N), N->getValueType(0));
 }]>;
 
-def C3LeftShift : PatLeaf<(imm), [{
-  uint64_t C = N->getZExtValue();
-  return C > 3 && (C >> llvm::countr_zero(C)) == 3;
-}]>;
-
-def C5LeftShift : PatLeaf<(imm), [{
-  uint64_t C = N->getZExtValue();
-  return C > 5 && (C >> llvm::countr_zero(C)) == 5;
-}]>;
-
-def C9LeftShift : PatLeaf<(imm), [{
-  uint64_t C = N->getZExtValue();
-  return C > 9 && (C >> llvm::countr_zero(C)) == 9;
-}]>;
-
-// Constant of the form (3 << C) where C is less than 32.
-def C3LeftShiftUW : PatLeaf<(imm), [{
-  uint64_t C = N->getZExtValue();
-  unsigned Shift = llvm::countr_zero(C);
-  return 1 <= Shift && Shift < 32 && (C >> Shift) == 3;
-}]>;
-
-// Constant of the form (5 << C) where C is less than 32.
-def C5LeftShiftUW : PatLeaf<(imm), [{
-  uint64_t C = N->getZExtValue();
-  unsigned Shift = llvm::countr_zero(C);
-  return 1 <= Shift && Shift < 32 && (C >> Shift) == 5;
-}]>;
-
-// Constant of the form (9 << C) where C is less than 32.
-def C9LeftShiftUW : PatLeaf<(imm), [{
-  uint64_t C = N->getZExtValue();
-  unsigned Shift = llvm::countr_zero(C);
-  return 1 <= Shift && Shift < 32 && (C >> Shift) == 9;
-}]>;
-
 def CSImm12MulBy4 : PatLeaf<(imm), [{
   if (!N->hasOneUse())
     return false;
@@ -693,25 +657,6 @@ foreach i = {1,2,3} in {
             (shxadd pat:$rs1, GPR:$rs2)>;
 }
 
-def : Pat<(add_like (mul_oneuse GPR:$rs1, (XLenVT 6)), GPR:$rs2),
-          (SH1ADD (XLenVT (SH1ADD GPR:$rs1, GPR:$rs1)), GPR:$rs2)>;
-def : Pat<(add_like (mul_oneuse GPR:$rs1, (XLenVT 10)), GPR:$rs2),
-          (SH1ADD (XLenVT (SH2ADD GPR:$rs1, GPR:$rs1)), GPR:$rs2)>;
-def : Pat<(add_like (mul_oneuse GPR:$rs1, (XLenVT 18)), GPR:$rs2),
-          (SH1ADD (XLenVT (SH3ADD GPR:$rs1, GPR:$rs1)), GPR:$rs2)>;
-def : Pat<(add_like (mul_oneuse GPR:$rs1, (XLenVT 12)), GPR:$rs2),
-          (SH2ADD (XLenVT (SH1ADD GPR:$rs1, GPR:$rs1)), GPR:$rs2)>;
-def : Pat<(add_like (mul_oneuse GPR:$rs1, (XLenVT 20)), GPR:$rs2),
-          (SH2ADD (XLenVT (SH2ADD GPR:$rs1, GPR:$rs1)), GPR:$rs2)>;
-def : Pat<(add_like (mul_oneuse GPR:$rs1, (XLenVT 36)), GPR:$rs2),
-          (SH2ADD (XLenVT (SH3ADD GPR:$rs1, GPR:$rs1)), GPR:$rs2)>;
-def : Pat<(add_like (mul_oneuse GPR:$rs1, (XLenVT 24)), GPR:$rs2),
-          (SH3ADD (XLenVT (SH1ADD GPR:$rs1, GPR:$rs1)), GPR:$rs2)>;
-def : Pat<(add_like (mul_oneuse GPR:$rs1, (XLenVT 40)), GPR:$rs2),
-          (SH3ADD (XLenVT (SH2ADD GPR:$rs1, GPR:$rs1)), GPR:$rs2)>;
-def : Pat<(add_like (mul_oneuse GPR:$rs1, (XLenVT 72)), GPR:$rs2),
-          (SH3ADD (XLenVT (SH3ADD GPR:$rs1, GPR:$rs1)), GPR:$rs2)>;
-
 def : Pat<(add_like (XLenVT GPR:$r), CSImm12MulBy4:$i),
           (SH2ADD (XLenVT (ADDI (XLenVT X0), (SimmShiftRightBy2XForm CSImm12MulBy4:$i))),
                   GPR:$r)>;
@@ -719,16 +664,6 @@ def : Pat<(add_like (XLenVT GPR:$r), CSImm12MulBy8:$i),
           (SH3ADD (XLenVT (ADDI (XLenVT X0), (SimmShiftRightBy3XForm CSImm12MulBy8:$i))),
                   GPR:$r)>;
 
-def : Pat<(mul (XLenVT GPR:$r), C3LeftShift:$i),
-          (SLLI (XLenVT (SH1ADD GPR:$r, GPR:$r)),
-                (TrailingZeros C3LeftShift:$i))>;
-def : Pat<(mul (XLenVT GPR:$r), C5LeftShift:$i),
-          (SLLI (XLenVT (SH2ADD GPR:$r, GPR:$r)),
-                (TrailingZeros C5LeftShift:$i))>;
-def : Pat<(mul (XLenVT GPR:$r), C9LeftShift:$i),
-          (SLLI (XLenVT (SH3ADD GPR:$r, GPR:$r)),
-                (TrailingZeros C9LeftShift:$i))>;
-
 } // Predicates = [HasStdExtZba]
 
 let Predicates = [HasStdExtZba, IsRV64] in {
@@ -780,15 +715,6 @@ def : Pat<(i64 (add_like_non_imm12 (and GPR:$rs1, 0x3FFFFFFFC), (XLenVT GPR:$rs2
 def : Pat<(i64 (add_like_non_imm12 (and GPR:$rs1, 0x7FFFFFFF8), (XLenVT GPR:$rs2))),
           (SH3ADD_UW (XLenVT (SRLI GPR:$rs1, 3)), GPR:$rs2)>;
 
-def : Pat<(i64 (mul (and_oneuse GPR:$r, 0xFFFFFFFF), C3LeftShiftUW:$i)),
-          (SH1ADD (XLenVT (SLLI_UW GPR:$r, (TrailingZeros C3LeftShiftUW:$i))),
-                  (XLenVT (SLLI_UW GPR:$r, (TrailingZeros C3LeftShiftUW:$i))))>;
-def : Pat<(i64 (mul (and_oneuse GPR:$r, 0xFFFFFFFF), C5LeftShiftUW:$i)),
-          (SH2ADD (XLenVT (SLLI_UW GPR:$r, (TrailingZeros C5LeftShiftUW:$i))),
-                  (XLenVT (SLLI_UW GPR:$r, (TrailingZeros C5LeftShiftUW:$i))))>;
-def : Pat<(i64 (mul (and_oneuse GPR:$r, 0xFFFFFFFF), C9LeftShiftUW:$i)),
-          (SH3ADD (XLenVT (SLLI_UW GPR:$r, (TrailingZeros C9LeftShiftUW:$i))),
-                  (XLenVT (SLLI_UW GPR:$r, (TrailingZeros C9LeftShiftUW:$i))))>;
 } // Predicates = [HasStdExtZba, IsRV64]
 
 let Predicates = [HasStdExtZbcOrZbkc] in {
diff --git a/llvm/test/CodeGen/RISCV/addimm-mulimm.ll b/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
index 8fb251a75bd1..e2f7be2e6d7f 100644
--- a/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
+++ b/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
@@ -600,8 +600,9 @@ define i64 @add_mul_combine_infinite_loop(i64 %x) {
 ; RV32IMB-NEXT:    sh3add a1, a1, a2
 ; RV32IMB-NEXT:    sh1add a0, a0, a0
 ; RV32IMB-NEXT:    slli a2, a0, 3
-; RV32IMB-NEXT:    addi a0, a2, 2047
-; RV32IMB-NEXT:    addi a0, a0, 1
+; RV32IMB-NEXT:    li a3, 1
+; RV32IMB-NEXT:    slli a3, a3, 11
+; RV32IMB-NEXT:    sh3add a0, a0, a3
 ; RV32IMB-NEXT:    sltu a2, a0, a2
 ; RV32IMB-NEXT:    add a1, a1, a2
 ; RV32IMB-NEXT:    ret
@@ -610,8 +611,8 @@ define i64 @add_mul_combine_infinite_loop(i64 %x) {
 ; RV64IMB:       # %bb.0:
 ; RV64IMB-NEXT:    addi a0, a0, 86
 ; RV64IMB-NEXT:    sh1add a0, a0, a0
-; RV64IMB-NEXT:    li a1, -16
-; RV64IMB-NEXT:    sh3add a0, a0, a1
+; RV64IMB-NEXT:    slli a0, a0, 3
+; RV64IMB-NEXT:    addi a0, a0, -16
 ; RV64IMB-NEXT:    ret
   %tmp0 = mul i64 %x, 24
   %tmp1 = add i64 %tmp0, 2048
diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zba.ll
index c3ae40124ba0..2db0d40b0ce5 100644
--- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zba.ll
@@ -646,8 +646,8 @@ define i64 @zext_mul12884901888(i32 signext %a) {
 ;
 ; RV64ZBA-LABEL: zext_mul12884901888:
 ; RV64ZBA:       # %bb.0:
-; RV64ZBA-NEXT:    sh1add a0, a0, a0
 ; RV64ZBA-NEXT:    slli a0, a0, 32
+; RV64ZBA-NEXT:    sh1add a0, a0, a0
 ; RV64ZBA-NEXT:    ret
   %b = zext i32 %a to i64
   %c = mul i64 %b, 12884901888
@@ -667,8 +667,8 @@ define i64 @zext_mul21474836480(i32 signext %a) {
 ;
 ; RV64ZBA-LABEL: zext_mul21474836480:
 ; RV64ZBA:       # %bb.0:
-; RV64ZBA-NEXT:    sh2add a0, a0, a0
 ; RV64ZBA-NEXT:    slli a0, a0, 32
+; RV64ZBA-NEXT:    sh2add a0, a0, a0
 ; RV64ZBA-NEXT:    ret
   %b = zext i32 %a to i64
   %c = mul i64 %b, 21474836480
@@ -688,8 +688,8 @@ define i64 @zext_mul38654705664(i32 signext %a) {
 ;
 ; RV64ZBA-LABEL: zext_mul38654705664:
 ; RV64ZBA:       # %bb.0:
-; RV64ZBA-NEXT:    sh3add a0, a0, a0
 ; RV64ZBA-NEXT:    slli a0, a0, 32
+; RV64ZBA-NEXT:    sh3add a0, a0, a0
 ; RV64ZBA-NEXT:    ret
   %b = zext i32 %a to i64
   %c = mul i64 %b, 38654705664
diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll
index 817e2b7d0bd9..5931e0982a4a 100644
--- a/llvm/test/CodeGen/RISCV/rv64zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zba.ll
@@ -865,8 +865,8 @@ define i64 @zext_mul12884901888(i32 signext %a) {
 ;
 ; RV64ZBA-LABEL: zext_mul12884901888:
 ; RV64ZBA:       # %bb.0:
-; RV64ZBA-NEXT:    sh1add a0, a0, a0
 ; RV64ZBA-NEXT:    slli a0, a0, 32
+; RV64ZBA-NEXT:    sh1add a0, a0, a0
 ; RV64ZBA-NEXT:    ret
   %b = zext i32 %a to i64
   %c = mul i64 %b, 12884901888
@@ -886,8 +886,8 @@ define i64 @zext_mul21474836480(i32 signext %a) {
 ;
 ; RV64ZBA-LABEL: zext_mul21474836480:
 ; RV64ZBA:       # %bb.0:
-; RV64ZBA-NEXT:    sh2add a0, a0, a0
 ; RV64ZBA-NEXT:    slli a0, a0, 32
+; RV64ZBA-NEXT:    sh2add a0, a0, a0
 ; RV64ZBA-NEXT:    ret
   %b = zext i32 %a to i64
   %c = mul i64 %b, 21474836480
@@ -907,8 +907,8 @@ define i64 @zext_mul38654705664(i32 signext %a) {
 ;
 ; RV64ZBA-LABEL: zext_mul38654705664:
 ; RV64ZBA:       # %bb.0:
-; RV64ZBA-NEXT:    sh3add a0, a0, a0
 ; RV64ZBA-NEXT:    slli a0, a0, 32
+; RV64ZBA-NEXT:    sh3add a0, a0, a0
 ; RV64ZBA-NEXT:    ret
   %b = zext i32 %a to i64
   %c = mul i64 %b, 38654705664
-- 
GitLab


From a620697340671aea2b0c65449fcddf3c2e4d1917 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Wed, 8 May 2024 10:14:51 -0700
Subject: [PATCH 0197/1206] [IR] Check callee param attributes as well in
 CallBase::getParamAttr() (#91394)

These methods aren't used yet, but may be in the future. This keeps them
in line with other methods like getFnAttr().
---
 llvm/include/llvm/IR/InstrTypes.h    | 12 ++++++--
 llvm/lib/IR/Instructions.cpp         | 16 ++++++++++
 llvm/unittests/IR/AttributesTest.cpp | 46 ++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index eaade9ce4755..9dd1bb455a71 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -1997,13 +1997,19 @@ public:
   /// Get the attribute of a given kind from a given arg
   Attribute getParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
     assert(ArgNo < arg_size() && "Out of bounds");
-    return getAttributes().getParamAttr(ArgNo, Kind);
+    Attribute A = getAttributes().getParamAttr(ArgNo, Kind);
+    if (A.isValid())
+      return A;
+    return getParamAttrOnCalledFunction(ArgNo, Kind);
   }
 
   /// Get the attribute of a given kind from a given arg
   Attribute getParamAttr(unsigned ArgNo, StringRef Kind) const {
     assert(ArgNo < arg_size() && "Out of bounds");
-    return getAttributes().getParamAttr(ArgNo, Kind);
+    Attribute A = getAttributes().getParamAttr(ArgNo, Kind);
+    if (A.isValid())
+      return A;
+    return getParamAttrOnCalledFunction(ArgNo, Kind);
   }
 
   /// Return true if the data operand at index \p i has the attribute \p
@@ -2652,6 +2658,8 @@ private:
     return hasFnAttrOnCalledFunction(Kind);
   }
   template <typename AK> Attribute getFnAttrOnCalledFunction(AK Kind) const;
+  template <typename AK>
+  Attribute getParamAttrOnCalledFunction(unsigned ArgNo, AK Kind) const;
 
   /// Determine whether the return value has the given attribute. Supports
   /// Attribute::AttrKind and StringRef as \p AttrKind types.
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 7ad1ad4cddb7..32af58a43b68 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -500,6 +500,22 @@ template Attribute
 CallBase::getFnAttrOnCalledFunction(Attribute::AttrKind Kind) const;
 template Attribute CallBase::getFnAttrOnCalledFunction(StringRef Kind) const;
 
+template <typename AK>
+Attribute CallBase::getParamAttrOnCalledFunction(unsigned ArgNo,
+                                                 AK Kind) const {
+  Value *V = getCalledOperand();
+
+  if (auto *F = dyn_cast<Function>(V))
+    return F->getAttributes().getParamAttr(ArgNo, Kind);
+
+  return Attribute();
+}
+template Attribute
+CallBase::getParamAttrOnCalledFunction(unsigned ArgNo,
+                                       Attribute::AttrKind Kind) const;
+template Attribute CallBase::getParamAttrOnCalledFunction(unsigned ArgNo,
+                                                          StringRef Kind) const;
+
 void CallBase::getOperandBundlesAsDefs(
     SmallVectorImpl<OperandBundleDef> &Defs) const {
   for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i)
diff --git a/llvm/unittests/IR/AttributesTest.cpp b/llvm/unittests/IR/AttributesTest.cpp
index a7967593c2f9..da72fa14510c 100644
--- a/llvm/unittests/IR/AttributesTest.cpp
+++ b/llvm/unittests/IR/AttributesTest.cpp
@@ -340,4 +340,50 @@ TEST(Attributes, ConstantRangeAttributeCAPI) {
   }
 }
 
+TEST(Attributes, CalleeAttributes) {
+  const char *IRString = R"IR(
+    declare void @f1(i32 %i)
+    declare void @f2(i32 range(i32 1, 2) %i)
+
+    define void @g1(i32 %i) {
+      call void @f1(i32 %i)
+      ret void
+    }
+    define void @g2(i32 %i) {
+      call void @f2(i32 %i)
+      ret void
+    }
+    define void @g3(i32 %i) {
+      call void @f1(i32 range(i32 3, 4) %i)
+      ret void
+    }
+    define void @g4(i32 %i) {
+      call void @f2(i32 range(i32 3, 4) %i)
+      ret void
+    }
+  )IR";
+
+  SMDiagnostic Err;
+  LLVMContext Context;
+  std::unique_ptr<Module> M = parseAssemblyString(IRString, Err, Context);
+  ASSERT_TRUE(M);
+
+  {
+    auto *I = cast<CallBase>(&M->getFunction("g1")->getEntryBlock().front());
+    ASSERT_FALSE(I->getParamAttr(0, Attribute::Range).isValid());
+  }
+  {
+    auto *I = cast<CallBase>(&M->getFunction("g2")->getEntryBlock().front());
+    ASSERT_TRUE(I->getParamAttr(0, Attribute::Range).isValid());
+  }
+  {
+    auto *I = cast<CallBase>(&M->getFunction("g3")->getEntryBlock().front());
+    ASSERT_TRUE(I->getParamAttr(0, Attribute::Range).isValid());
+  }
+  {
+    auto *I = cast<CallBase>(&M->getFunction("g4")->getEntryBlock().front());
+    ASSERT_TRUE(I->getParamAttr(0, Attribute::Range).isValid());
+  }
+}
+
 } // end anonymous namespace
-- 
GitLab


From 576838301d23bb779aa9c3f0cc3d086c46add44b Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Wed, 8 May 2024 10:15:27 -0700
Subject: [PATCH 0198/1206] [IR] Remove check for bitcast of called function in
 CallBase::has/getFnAttrOnCalledFunction (#91392)

With opaque pointers, we shouldn't have bitcasts between function
pointer types.
---
 llvm/lib/IR/Instructions.cpp | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 32af58a43b68..4b725610081c 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -454,24 +454,14 @@ bool CallBase::paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
 }
 
 bool CallBase::hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const {
-  Value *V = getCalledOperand();
-  if (auto *CE = dyn_cast<ConstantExpr>(V))
-    if (CE->getOpcode() == BitCast)
-      V = CE->getOperand(0);
-
-  if (auto *F = dyn_cast<Function>(V))
+  if (auto *F = dyn_cast<Function>(getCalledOperand()))
     return F->getAttributes().hasFnAttr(Kind);
 
   return false;
 }
 
 bool CallBase::hasFnAttrOnCalledFunction(StringRef Kind) const {
-  Value *V = getCalledOperand();
-  if (auto *CE = dyn_cast<ConstantExpr>(V))
-    if (CE->getOpcode() == BitCast)
-      V = CE->getOperand(0);
-
-  if (auto *F = dyn_cast<Function>(V))
+  if (auto *F = dyn_cast<Function>(getCalledOperand()))
     return F->getAttributes().hasFnAttr(Kind);
 
   return false;
@@ -485,12 +475,7 @@ Attribute CallBase::getFnAttrOnCalledFunction(AK Kind) const {
     assert(Kind != Attribute::Memory && "Use getMemoryEffects() instead");
   }
 
-  Value *V = getCalledOperand();
-  if (auto *CE = dyn_cast<ConstantExpr>(V))
-    if (CE->getOpcode() == BitCast)
-      V = CE->getOperand(0);
-
-  if (auto *F = dyn_cast<Function>(V))
+  if (auto *F = dyn_cast<Function>(getCalledOperand()))
     return F->getAttributes().getFnAttr(Kind);
 
   return Attribute();
-- 
GitLab


From 08011cf8453c7c9e87d135f063356b6764a91cbc Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy@outlook.com>
Date: Thu, 9 May 2024 01:15:49 +0800
Subject: [PATCH 0199/1206] [Docs][NFC] Use opaque ptr in the example (#91502)

---
 llvm/docs/MIRLangRef.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/docs/MIRLangRef.rst b/llvm/docs/MIRLangRef.rst
index 52ff24daa7fb..e248a14636a8 100644
--- a/llvm/docs/MIRLangRef.rst
+++ b/llvm/docs/MIRLangRef.rst
@@ -168,11 +168,11 @@ Here is an example of a YAML document that contains an LLVM module:
 
 .. code-block:: llvm
 
-       define i32 @inc(i32* %x) {
+       define i32 @inc(ptr %x) {
        entry:
-         %0 = load i32, i32* %x
+         %0 = load i32, ptr %x
          %1 = add i32 %0, 1
-         store i32 %1, i32* %x
+         store i32 %1, ptr %x
          ret i32 %1
        }
 
-- 
GitLab


From 46435ac19e09039fb146fa6c12da0e640a66d435 Mon Sep 17 00:00:00 2001
From: Kristof Beyls <kristof.beyls@arm.com>
Date: Wed, 8 May 2024 19:20:11 +0200
Subject: [PATCH 0200/1206] [NFC][BOLT] Remove dead code (SPTAllocatorsId)
 (#91477)

It seems that SPTAllocatorsId is no longer used in FrameAnalysis, so
let's remove it.

It seems the use of SPTAllocatorsId was removed back in 2019, in commit
cc8415406c7.
---
 bolt/include/bolt/Passes/FrameAnalysis.h | 4 ----
 bolt/lib/Passes/FrameAnalysis.cpp        | 5 -----
 2 files changed, 9 deletions(-)

diff --git a/bolt/include/bolt/Passes/FrameAnalysis.h b/bolt/include/bolt/Passes/FrameAnalysis.h
index 66246bd6647b..44b54d4ed45d 100644
--- a/bolt/include/bolt/Passes/FrameAnalysis.h
+++ b/bolt/include/bolt/Passes/FrameAnalysis.h
@@ -170,10 +170,6 @@ class FrameAnalysis {
                      std::unique_ptr<StackPointerTracking>>
       SPTMap;
 
-  /// A vector that stores ids of the allocators that are used in SPT
-  /// computation
-  std::vector<MCPlusBuilder::AllocatorIdTy> SPTAllocatorsId;
-
 public:
   explicit FrameAnalysis(BinaryContext &BC, BinaryFunctionCallGraph &CG);
 
diff --git a/bolt/lib/Passes/FrameAnalysis.cpp b/bolt/lib/Passes/FrameAnalysis.cpp
index 7f1245e39f56..4ebfd8f158f7 100644
--- a/bolt/lib/Passes/FrameAnalysis.cpp
+++ b/bolt/lib/Passes/FrameAnalysis.cpp
@@ -561,11 +561,6 @@ FrameAnalysis::FrameAnalysis(BinaryContext &BC, BinaryFunctionCallGraph &CG)
     NamedRegionTimer T1("clearspt", "clear spt", "FA", "FA breakdown",
                         opts::TimeFA);
     clearSPTMap();
-
-    // Clean up memory allocated for annotation values
-    if (!opts::NoThreads)
-      for (MCPlusBuilder::AllocatorIdTy Id : SPTAllocatorsId)
-        BC.MIB->freeValuesAllocator(Id);
   }
 }
 
-- 
GitLab


From bb6df0804ba0a0b0581aec4156138f5144dbcee2 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 8 May 2024 10:33:53 -0700
Subject: [PATCH 0201/1206] [llvm] Use StringRef::operator== instead of
 StringRef::equals (NFC) (#91441)

I'm planning to remove StringRef::equals in favor of
StringRef::operator==.

- StringRef::operator==/!= outnumber StringRef::equals by a factor of
  70 under llvm/ in terms of their usage.

- The elimination of StringRef::equals brings StringRef closer to
  std::string_view, which has operator== but not equals.

- S == "foo" is more readable than S.equals("foo"), especially for
  !Long.Expression.equals("str") vs Long.Expression != "str".
---
 llvm/include/llvm/ADT/SmallString.h           |  2 +-
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp     |  2 +-
 .../RuntimeDyld/RuntimeDyldELF.cpp            |  2 +-
 llvm/lib/FileCheck/FileCheck.cpp              |  2 +-
 llvm/lib/FuzzMutate/FuzzerCLI.cpp             |  6 ++--
 llvm/lib/LTO/LTOModule.cpp                    |  2 +-
 llvm/lib/MC/MCAsmStreamer.cpp                 |  2 +-
 llvm/lib/MC/MCParser/AsmParser.cpp            |  2 +-
 llvm/lib/MC/MCParser/DarwinAsmParser.cpp      |  2 +-
 llvm/lib/MC/MCSymbolXCOFF.cpp                 |  4 +--
 llvm/lib/Object/Archive.cpp                   |  4 +--
 llvm/lib/Object/MachOObjectFile.cpp           |  4 +--
 llvm/lib/Object/OffloadBinary.cpp             |  2 +-
 llvm/lib/ObjectYAML/COFFEmitter.cpp           |  4 +--
 llvm/lib/Passes/StandardInstrumentations.cpp  |  3 +-
 llvm/lib/ProfileData/GCOV.cpp                 |  2 +-
 llvm/lib/ProfileData/InstrProf.cpp            |  2 +-
 llvm/lib/ProfileData/MemProfReader.cpp        |  6 ++--
 llvm/lib/Support/VirtualFileSystem.cpp        |  4 +--
 llvm/lib/TargetParser/ARMTargetParser.cpp     |  2 +-
 llvm/lib/TargetParser/Triple.cpp              |  6 ++--
 llvm/tools/dsymutil/DwarfLinkerForBinary.cpp  |  2 +-
 llvm/tools/llvm-dwarfdump/Statistics.cpp      |  2 +-
 llvm/tools/llvm-extract/llvm-extract.cpp      |  2 +-
 llvm/tools/llvm-objdump/MachODump.cpp         |  2 +-
 llvm/tools/llvm-xray/xray-graph-diff.cpp      |  4 +--
 .../yaml-numeric-parser-fuzzer.cpp            |  2 +-
 llvm/unittests/ADT/StringRefTest.cpp          |  6 ++--
 llvm/unittests/IR/VerifierTest.cpp            | 36 +++++++++----------
 llvm/unittests/Support/MemoryBufferTest.cpp   | 22 ++++++------
 llvm/unittests/Support/YAMLIOTest.cpp         |  8 ++---
 .../TargetParser/CSKYTargetParserTest.cpp     |  2 +-
 .../TargetParser/TargetParserTest.cpp         |  4 +--
 llvm/utils/TableGen/AsmMatcherEmitter.cpp     |  2 +-
 34 files changed, 79 insertions(+), 80 deletions(-)

diff --git a/llvm/include/llvm/ADT/SmallString.h b/llvm/include/llvm/ADT/SmallString.h
index a5b9eec50c82..be3193c6ef9b 100644
--- a/llvm/include/llvm/ADT/SmallString.h
+++ b/llvm/include/llvm/ADT/SmallString.h
@@ -89,7 +89,7 @@ public:
 
   /// Check for string equality.  This is more efficient than compare() when
   /// the relative ordering of inequal strings isn't needed.
-  [[nodiscard]] bool equals(StringRef RHS) const { return str().equals(RHS); }
+  [[nodiscard]] bool equals(StringRef RHS) const { return str() == RHS; }
 
   /// Check for string equality, ignoring case.
   [[nodiscard]] bool equals_insensitive(StringRef RHS) const {
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 73fe63b5b8f6..be2381cd7d77 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6896,7 +6896,7 @@ Error BitcodeReader::materialize(GlobalValue *GV) {
         MDString *MDS = cast<MDString>(MD->getOperand(0));
         StringRef ProfName = MDS->getString();
         // Check consistency of !prof branch_weights metadata.
-        if (!ProfName.equals("branch_weights"))
+        if (ProfName != "branch_weights")
           continue;
         unsigned ExpectedNumOperands = 0;
         if (BranchInst *BI = dyn_cast<BranchInst>(&I))
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index edeb563076fd..eaf8c35142de 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -659,7 +659,7 @@ void RuntimeDyldELF::setMipsABI(const ObjectFile &Obj) {
     IsMipsO32ABI = AbiVariant & ELF::EF_MIPS_ABI_O32;
     IsMipsN32ABI = AbiVariant & ELF::EF_MIPS_ABI2;
   }
-  IsMipsN64ABI = Obj.getFileFormatName().equals("elf64-mips");
+  IsMipsN64ABI = Obj.getFileFormatName() == "elf64-mips";
 }
 
 // Return the .TOC. section and offset.
diff --git a/llvm/lib/FileCheck/FileCheck.cpp b/llvm/lib/FileCheck/FileCheck.cpp
index 8f80a69c4abd..1719f8ef2b43 100644
--- a/llvm/lib/FileCheck/FileCheck.cpp
+++ b/llvm/lib/FileCheck/FileCheck.cpp
@@ -374,7 +374,7 @@ Expected<NumericVariable *> Pattern::parseNumericVariableDefinition(
 Expected<std::unique_ptr<NumericVariableUse>> Pattern::parseNumericVariableUse(
     StringRef Name, bool IsPseudo, std::optional<size_t> LineNumber,
     FileCheckPatternContext *Context, const SourceMgr &SM) {
-  if (IsPseudo && !Name.equals("@LINE"))
+  if (IsPseudo && Name != "@LINE")
     return ErrorDiagnostic::get(
         SM, Name, "invalid pseudo numeric variable '" + Name + "'");
 
diff --git a/llvm/lib/FuzzMutate/FuzzerCLI.cpp b/llvm/lib/FuzzMutate/FuzzerCLI.cpp
index 58e4b74f4b22..504532865440 100644
--- a/llvm/lib/FuzzMutate/FuzzerCLI.cpp
+++ b/llvm/lib/FuzzMutate/FuzzerCLI.cpp
@@ -21,7 +21,7 @@ void llvm::parseFuzzerCLOpts(int ArgC, char *ArgV[]) {
 
   int I = 1;
   while (I < ArgC)
-    if (StringRef(ArgV[I++]).equals("-ignore_remaining_args=1"))
+    if (StringRef(ArgV[I++]) == "-ignore_remaining_args=1")
       break;
   while (I < ArgC)
     CLArgs.push_back(ArgV[I++]);
@@ -39,7 +39,7 @@ void llvm::handleExecNameEncodedBEOpts(StringRef ExecName) {
   SmallVector<StringRef, 4> Opts;
   NameAndArgs.second.split(Opts, '-');
   for (StringRef Opt : Opts) {
-    if (Opt.equals("gisel")) {
+    if (Opt == "gisel") {
       Args.push_back("-global-isel");
       // For now we default GlobalISel to -O0
       Args.push_back("-O0");
@@ -151,7 +151,7 @@ int llvm::runFuzzerOnInputs(int ArgC, char *ArgV[], FuzzerTestFun TestOne,
   for (int I = 1; I < ArgC; ++I) {
     StringRef Arg(ArgV[I]);
     if (Arg.starts_with("-")) {
-      if (Arg.equals("-ignore_remaining_args=1"))
+      if (Arg == "-ignore_remaining_args=1")
         break;
       continue;
     }
diff --git a/llvm/lib/LTO/LTOModule.cpp b/llvm/lib/LTO/LTOModule.cpp
index f839fe944e18..eac78069f4d2 100644
--- a/llvm/lib/LTO/LTOModule.cpp
+++ b/llvm/lib/LTO/LTOModule.cpp
@@ -694,7 +694,7 @@ bool LTOModule::hasCtorDtor() const {
     if (auto *GV = dyn_cast_if_present<GlobalValue *>(Sym)) {
       StringRef Name = GV->getName();
       if (Name.consume_front("llvm.global_")) {
-        if (Name.equals("ctors") || Name.equals("dtors"))
+        if (Name == "ctors" || Name == "dtors")
           return true;
       }
     }
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 3dc70a401589..f257d0d9e83f 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -468,7 +468,7 @@ void MCAsmStreamer::emitRawComment(const Twine &T, bool TabPrefix) {
 
 void MCAsmStreamer::addExplicitComment(const Twine &T) {
   StringRef c = T.getSingleStringRef();
-  if (c.equals(StringRef(MAI->getSeparatorString())))
+  if (c == MAI->getSeparatorString())
     return;
   if (c.starts_with(StringRef("//"))) {
     ExplicitCommentToEmit.append("\t");
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 76a3e501f459..8d9acd54e879 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -4543,7 +4543,7 @@ bool AsmParser::parseDirectiveMacro(SMLoc DirectiveLoc) {
 
     // Emit an error if two (or more) named parameters share the same name
     for (const MCAsmMacroParameter& CurrParam : Parameters)
-      if (CurrParam.Name.equals(Parameter.Name))
+      if (CurrParam.Name == Parameter.Name)
         return TokError("macro '" + Name + "' has multiple parameters"
                         " named '" + Parameter.Name + "'");
 
diff --git a/llvm/lib/MC/MCParser/DarwinAsmParser.cpp b/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
index 3cd44e7195be..a97b72997ae3 100644
--- a/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
@@ -705,7 +705,7 @@ bool DarwinAsmParser::parseDirectiveSection(StringRef, SMLoc) {
                                    .Case("__datacoal_nt", "__data")
                                    .Default(Section);
 
-    if (!Section.equals(NonCoalSection)) {
+    if (Section != NonCoalSection) {
       StringRef SectionVal(Loc.getPointer());
       size_t B = SectionVal.find(',') + 1, E = SectionVal.find(',', B);
       SMLoc BLoc = SMLoc::getFromPointer(SectionVal.data() + B);
diff --git a/llvm/lib/MC/MCSymbolXCOFF.cpp b/llvm/lib/MC/MCSymbolXCOFF.cpp
index b4c96a1ffa23..599a3946a1ed 100644
--- a/llvm/lib/MC/MCSymbolXCOFF.cpp
+++ b/llvm/lib/MC/MCSymbolXCOFF.cpp
@@ -13,7 +13,7 @@ using namespace llvm;
 MCSectionXCOFF *MCSymbolXCOFF::getRepresentedCsect() const {
   assert(RepresentedCsect &&
          "Trying to get csect representation of this symbol but none was set.");
-  assert(getSymbolTableName().equals(RepresentedCsect->getSymbolTableName()) &&
+  assert(getSymbolTableName() == RepresentedCsect->getSymbolTableName() &&
          "SymbolTableNames need to be the same for this symbol and its csect "
          "representation.");
   return RepresentedCsect;
@@ -24,7 +24,7 @@ void MCSymbolXCOFF::setRepresentedCsect(MCSectionXCOFF *C) {
   assert((!RepresentedCsect || RepresentedCsect == C) &&
          "Trying to set a csect that doesn't match the one that this symbol is "
          "already mapped to.");
-  assert(getSymbolTableName().equals(C->getSymbolTableName()) &&
+  assert(getSymbolTableName() == C->getSymbolTableName() &&
          "SymbolTableNames need to be the same for this symbol and its csect "
          "representation.");
   RepresentedCsect = C;
diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp
index 6139d9996bda..e798bbdd16f1 100644
--- a/llvm/lib/Object/Archive.cpp
+++ b/llvm/lib/Object/Archive.cpp
@@ -269,11 +269,11 @@ Expected<StringRef> ArchiveMemberHeader::getName(uint64_t Size) const {
       return Name;
     // System libraries from the Windows SDK for Windows 11 contain this symbol.
     // It looks like a CFG guard: we just skip it for now.
-    if (Name.equals("/<XFGHASHMAP>/"))
+    if (Name == "/<XFGHASHMAP>/")
       return Name;
     // Some libraries (e.g., arm64rt.lib) from the Windows WDK
     // (version 10.0.22000.0) contain this undocumented special member.
-    if (Name.equals("/<ECSYMBOLS>/"))
+    if (Name == "/<ECSYMBOLS>/")
       return Name;
     // It's a long name.
     // Get the string table offset.
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index 1cfd0a069463..06186ad362aa 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -399,7 +399,7 @@ static Error parseSegmentLoadCommand(
       return malformedError("load command " + Twine(LoadCommandIndex) +
                             " filesize field in " + CmdName +
                             " greater than vmsize field");
-    IsPageZeroSegment |= StringRef("__PAGEZERO").equals(S.segname);
+    IsPageZeroSegment |= StringRef("__PAGEZERO") == S.segname;
   } else
     return SegOrErr.takeError();
 
@@ -4364,7 +4364,7 @@ BindRebaseSegInfo::BindRebaseSegInfo(const object::MachOObjectFile *Obj) {
     Info.Size = Section.getSize();
     Info.SegmentName =
         Obj->getSectionFinalSegmentName(Section.getRawDataRefImpl());
-    if (!Info.SegmentName.equals(CurSegName)) {
+    if (Info.SegmentName != CurSegName) {
       ++CurSegIndex;
       CurSegName = Info.SegmentName;
       CurSegAddress = Info.Address;
diff --git a/llvm/lib/Object/OffloadBinary.cpp b/llvm/lib/Object/OffloadBinary.cpp
index 6e9f8bed513c..89dc12551494 100644
--- a/llvm/lib/Object/OffloadBinary.cpp
+++ b/llvm/lib/Object/OffloadBinary.cpp
@@ -359,7 +359,7 @@ bool object::areTargetsCompatible(const OffloadFile::TargetID &LHS,
     return false;
 
   // If the architecture is "all" we assume it is always compatible.
-  if (LHS.second.equals("generic") || RHS.second.equals("generic"))
+  if (LHS.second == "generic" || RHS.second == "generic")
     return true;
 
   // Only The AMDGPU target requires additional checks.
diff --git a/llvm/lib/ObjectYAML/COFFEmitter.cpp b/llvm/lib/ObjectYAML/COFFEmitter.cpp
index 7088223b9b67..bb46de4c6f57 100644
--- a/llvm/lib/ObjectYAML/COFFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/COFFEmitter.cpp
@@ -359,9 +359,9 @@ static uint32_t initializeOptionalHeader(COFFParser &CP, uint16_t Magic,
       SizeOfInitializedData += S.Header.SizeOfRawData;
     if (S.Header.Characteristics & COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA)
       SizeOfUninitializedData += S.Header.SizeOfRawData;
-    if (S.Name.equals(".text"))
+    if (S.Name == ".text")
       Header->BaseOfCode = S.Header.VirtualAddress; // RVA
-    else if (S.Name.equals(".data"))
+    else if (S.Name == ".data")
       BaseOfData = S.Header.VirtualAddress; // RVA
     if (S.Header.VirtualAddress)
       SizeOfImage += alignTo(S.Header.VirtualSize, Header->SectionAlignment);
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 79aff096fb08..c7adc7668b9a 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -822,8 +822,7 @@ PrintIRInstrumentation::PassRunDescriptor
 PrintIRInstrumentation::popPassRunDescriptor(StringRef PassID) {
   assert(!PassRunDescriptorStack.empty() && "empty PassRunDescriptorStack");
   PassRunDescriptor Descriptor = PassRunDescriptorStack.pop_back_val();
-  assert(Descriptor.PassID.equals(PassID) &&
-         "malformed PassRunDescriptorStack");
+  assert(Descriptor.PassID == PassID && "malformed PassRunDescriptorStack");
   return Descriptor;
 }
 
diff --git a/llvm/lib/ProfileData/GCOV.cpp b/llvm/lib/ProfileData/GCOV.cpp
index ee61784abade..ecb12c045b5b 100644
--- a/llvm/lib/ProfileData/GCOV.cpp
+++ b/llvm/lib/ProfileData/GCOV.cpp
@@ -678,7 +678,7 @@ std::string Context::getCoveragePath(StringRef filename,
     return std::string(filename);
 
   std::string CoveragePath;
-  if (options.LongFileNames && !filename.equals(mainFilename))
+  if (options.LongFileNames && filename != mainFilename)
     CoveragePath =
         mangleCoveragePath(mainFilename, options.PreservePaths) + "##";
   CoveragePath += mangleCoveragePath(filename, options.PreservePaths);
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index f9ba80bd99c8..1e3ca47b3d5a 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -1283,7 +1283,7 @@ MDNode *mayHaveValueProfileOfKind(const Instruction &Inst,
     return nullptr;
 
   MDString *Tag = cast<MDString>(MD->getOperand(0));
-  if (!Tag || !Tag->getString().equals("VP"))
+  if (!Tag || Tag->getString() != "VP")
     return nullptr;
 
   // Now check kind:
diff --git a/llvm/lib/ProfileData/MemProfReader.cpp b/llvm/lib/ProfileData/MemProfReader.cpp
index b4d2c6f043f6..c25babac844a 100644
--- a/llvm/lib/ProfileData/MemProfReader.cpp
+++ b/llvm/lib/ProfileData/MemProfReader.cpp
@@ -164,9 +164,9 @@ bool isRuntimePath(const StringRef Path) {
   const StringRef Filename = llvm::sys::path::filename(Path);
   // This list should be updated in case new files with additional interceptors
   // are added to the memprof runtime.
-  return Filename.equals("memprof_malloc_linux.cpp") ||
-         Filename.equals("memprof_interceptors.cpp") ||
-         Filename.equals("memprof_new_delete.cpp");
+  return Filename == "memprof_malloc_linux.cpp" ||
+         Filename == "memprof_interceptors.cpp" ||
+         Filename == "memprof_new_delete.cpp";
 }
 
 std::string getBuildIdString(const SegmentEntry &Entry) {
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 54b9c38f7609..fcefdef992be 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -1725,7 +1725,7 @@ public:
                       RedirectingFileSystem::Entry *ParentEntry = nullptr) {
     if (!ParentEntry) { // Look for a existent root
       for (const auto &Root : FS->Roots) {
-        if (Name.equals(Root->getName())) {
+        if (Name == Root->getName()) {
           ParentEntry = Root.get();
           return ParentEntry;
         }
@@ -1736,7 +1736,7 @@ public:
            llvm::make_range(DE->contents_begin(), DE->contents_end())) {
         auto *DirContent =
             dyn_cast<RedirectingFileSystem::DirectoryEntry>(Content.get());
-        if (DirContent && Name.equals(Content->getName()))
+        if (DirContent && Name == Content->getName())
           return DirContent;
       }
     }
diff --git a/llvm/lib/TargetParser/ARMTargetParser.cpp b/llvm/lib/TargetParser/ARMTargetParser.cpp
index 67f937ebc33f..9d9917d86a36 100644
--- a/llvm/lib/TargetParser/ARMTargetParser.cpp
+++ b/llvm/lib/TargetParser/ARMTargetParser.cpp
@@ -610,7 +610,7 @@ StringRef ARM::getARMCPUForArch(const llvm::Triple &Triple, StringRef MArch) {
     return StringRef();
 
   StringRef CPU = llvm::ARM::getDefaultCPU(MArch);
-  if (!CPU.empty() && !CPU.equals("invalid"))
+  if (!CPU.empty() && CPU != "invalid")
     return CPU;
 
   // If no specific architecture version is requested, return the minimum CPU
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index ef40ccf36806..f8269a51dc0b 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -373,14 +373,14 @@ StringRef Triple::getObjectFormatTypeName(ObjectFormatType Kind) {
 }
 
 static Triple::ArchType parseBPFArch(StringRef ArchName) {
-  if (ArchName.equals("bpf")) {
+  if (ArchName == "bpf") {
     if (sys::IsLittleEndianHost)
       return Triple::bpfel;
     else
       return Triple::bpfeb;
-  } else if (ArchName.equals("bpf_be") || ArchName.equals("bpfeb")) {
+  } else if (ArchName == "bpf_be" || ArchName == "bpfeb") {
     return Triple::bpfeb;
-  } else if (ArchName.equals("bpf_le") || ArchName.equals("bpfel")) {
+  } else if (ArchName == "bpf_le" || ArchName == "bpfel") {
     return Triple::bpfel;
   } else {
     return Triple::UnknownArch;
diff --git a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
index 7246ba45d5af..83473704398d 100644
--- a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
+++ b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
@@ -705,7 +705,7 @@ bool DwarfLinkerForBinary::linkImpl(
     } else {
       // Try and emit more helpful warnings by applying some heuristics.
       StringRef ObjFile = ContainerName;
-      bool IsClangModule = sys::path::extension(Path).equals(".pcm");
+      bool IsClangModule = sys::path::extension(Path) == ".pcm";
       bool IsArchive = ObjFile.ends_with(")");
 
       if (IsClangModule) {
diff --git a/llvm/tools/llvm-dwarfdump/Statistics.cpp b/llvm/tools/llvm-dwarfdump/Statistics.cpp
index 96841c3c387b..1846f9265c75 100644
--- a/llvm/tools/llvm-dwarfdump/Statistics.cpp
+++ b/llvm/tools/llvm-dwarfdump/Statistics.cpp
@@ -229,7 +229,7 @@ static std::string constructDieID(DWARFDie Die,
      << Die.getName(DINameKind::LinkageName);
 
   // Prefix + Name is enough for local variables and parameters.
-  if (!Prefix.empty() && !Prefix.equals("g"))
+  if (!Prefix.empty() && Prefix != "g")
     return ID.str();
 
   auto DeclFile = Die.findRecursively(dwarf::DW_AT_decl_file);
diff --git a/llvm/tools/llvm-extract/llvm-extract.cpp b/llvm/tools/llvm-extract/llvm-extract.cpp
index a879c203fc37..5915f92ea05c 100644
--- a/llvm/tools/llvm-extract/llvm-extract.cpp
+++ b/llvm/tools/llvm-extract/llvm-extract.cpp
@@ -357,7 +357,7 @@ int main(int argc, char **argv) {
         // The function has been materialized, so add its matching basic blocks
         // to the block extractor list, or fail if a name is not found.
         auto Res = llvm::find_if(*P.first, [&](const BasicBlock &BB) {
-          return BB.getName().equals(BBName);
+          return BB.getName() == BBName;
         });
         if (Res == P.first->end()) {
           errs() << argv[0] << ": function " << P.first->getName()
diff --git a/llvm/tools/llvm-objdump/MachODump.cpp b/llvm/tools/llvm-objdump/MachODump.cpp
index 5e0d69a68d69..749f98820175 100644
--- a/llvm/tools/llvm-objdump/MachODump.cpp
+++ b/llvm/tools/llvm-objdump/MachODump.cpp
@@ -2148,7 +2148,7 @@ static void ProcessMachO(StringRef Name, MachOObjectFile *MachOOF,
       else
         consumeError(NameOrErr.takeError());
 
-      if (SectName.equals("__text")) {
+      if (SectName == "__text") {
         DataRefImpl Ref = Section.getRawDataRefImpl();
         StringRef SegName = MachOOF->getSectionFinalSegmentName(Ref);
         DisassembleMachO(FileName, MachOOF, SegName, SectName);
diff --git a/llvm/tools/llvm-xray/xray-graph-diff.cpp b/llvm/tools/llvm-xray/xray-graph-diff.cpp
index 899a6725a5d3..b5c63ab0a918 100644
--- a/llvm/tools/llvm-xray/xray-graph-diff.cpp
+++ b/llvm/tools/llvm-xray/xray-graph-diff.cpp
@@ -381,14 +381,14 @@ void GraphDiffRenderer::exportGraphAsDOT(raw_ostream &OS, StatType EdgeLabel,
                   R"(color="{5}" labelfontcolor="{5}" penwidth={6}])"
                   "\n",
                   VertexNo[HeadId], VertexNo[TailId],
-                  (HeadId.equals("")) ? static_cast<StringRef>("F0") : HeadId,
+                  HeadId.empty() ? static_cast<StringRef>("F0") : HeadId,
                   TailId, getLabel(E, EdgeLabel), getColor(E, G, H, EdgeColor),
                   getLineWidth(E, EdgeColor));
   }
 
   for (const auto &V : G.vertices()) {
     const auto &VertexId = V.first;
-    if (VertexId.equals("")) {
+    if (VertexId.empty()) {
       OS << formatv(R"(F{0} [label="F0"])"
                     "\n",
                     VertexNo[VertexId]);
diff --git a/llvm/tools/llvm-yaml-numeric-parser-fuzzer/yaml-numeric-parser-fuzzer.cpp b/llvm/tools/llvm-yaml-numeric-parser-fuzzer/yaml-numeric-parser-fuzzer.cpp
index c8370289963d..9a572c1e0600 100644
--- a/llvm/tools/llvm-yaml-numeric-parser-fuzzer/yaml-numeric-parser-fuzzer.cpp
+++ b/llvm/tools/llvm-yaml-numeric-parser-fuzzer/yaml-numeric-parser-fuzzer.cpp
@@ -18,7 +18,7 @@ inline bool isNumericRegex(llvm::StringRef S) {
   static llvm::Regex Float(
       "^[-+]?(\\.[0-9]+|[0-9]+(\\.[0-9]*)?)([eE][-+]?[0-9]+)?$");
 
-  if (S.equals(".nan") || S.equals(".NaN") || S.equals(".NAN"))
+  if (S == ".nan" || S == ".NaN" || S == ".NAN")
     return true;
 
   if (Infinity.match(S))
diff --git a/llvm/unittests/ADT/StringRefTest.cpp b/llvm/unittests/ADT/StringRefTest.cpp
index fa537e816fc8..b3c206a33696 100644
--- a/llvm/unittests/ADT/StringRefTest.cpp
+++ b/llvm/unittests/ADT/StringRefTest.cpp
@@ -998,7 +998,7 @@ TEST(StringRefTest, AllocatorCopy) {
   // allocator.
   StringRef StrEmpty = "";
   StringRef StrEmptyc = StrEmpty.copy(Alloc);
-  EXPECT_TRUE(StrEmpty.equals(StrEmptyc));
+  EXPECT_TRUE(StrEmpty == StrEmptyc);
   EXPECT_EQ(StrEmptyc.data(), nullptr);
   EXPECT_EQ(StrEmptyc.size(), 0u);
   EXPECT_EQ(Alloc.getTotalMemory(), 0u);
@@ -1007,9 +1007,9 @@ TEST(StringRefTest, AllocatorCopy) {
   StringRef Str2 = "bye";
   StringRef Str1c = Str1.copy(Alloc);
   StringRef Str2c = Str2.copy(Alloc);
-  EXPECT_TRUE(Str1.equals(Str1c));
+  EXPECT_TRUE(Str1 == Str1c);
   EXPECT_NE(Str1.data(), Str1c.data());
-  EXPECT_TRUE(Str2.equals(Str2c));
+  EXPECT_TRUE(Str2 == Str2c);
   EXPECT_NE(Str2.data(), Str2c.data());
 }
 
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index c8db7fb7ab84..d79b4f3d8a44 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -173,27 +173,27 @@ TEST(VerifierTest, CrossModuleRef) {
   std::string Error;
   raw_string_ostream ErrorOS(Error);
   EXPECT_TRUE(verifyModule(M2, &ErrorOS));
-  EXPECT_TRUE(StringRef(ErrorOS.str())
-                  .equals("Global is referenced in a different module!\n"
-                          "ptr @foo2\n"
-                          "; ModuleID = 'M2'\n"
-                          "  %call = call i32 @foo2()\n"
-                          "ptr @foo1\n"
-                          "; ModuleID = 'M1'\n"
-                          "Global is used by function in a different module\n"
-                          "ptr @foo2\n"
-                          "; ModuleID = 'M2'\n"
-                          "ptr @foo3\n"
-                          "; ModuleID = 'M3'\n"));
+  EXPECT_TRUE(StringRef(ErrorOS.str()) ==
+              "Global is referenced in a different module!\n"
+              "ptr @foo2\n"
+              "; ModuleID = 'M2'\n"
+              "  %call = call i32 @foo2()\n"
+              "ptr @foo1\n"
+              "; ModuleID = 'M1'\n"
+              "Global is used by function in a different module\n"
+              "ptr @foo2\n"
+              "; ModuleID = 'M2'\n"
+              "ptr @foo3\n"
+              "; ModuleID = 'M3'\n");
 
   Error.clear();
   EXPECT_TRUE(verifyModule(M1, &ErrorOS));
-  EXPECT_TRUE(StringRef(ErrorOS.str()).equals(
-      "Referencing function in another module!\n"
-      "  %call = call i32 @foo2()\n"
-      "; ModuleID = 'M1'\n"
-      "ptr @foo2\n"
-      "; ModuleID = 'M2'\n"));
+  EXPECT_TRUE(StringRef(ErrorOS.str()) ==
+              "Referencing function in another module!\n"
+              "  %call = call i32 @foo2()\n"
+              "; ModuleID = 'M1'\n"
+              "ptr @foo2\n"
+              "; ModuleID = 'M2'\n");
 
   Error.clear();
   EXPECT_TRUE(verifyModule(M3, &ErrorOS));
diff --git a/llvm/unittests/Support/MemoryBufferTest.cpp b/llvm/unittests/Support/MemoryBufferTest.cpp
index cfee3e477d2e..4815e65c968d 100644
--- a/llvm/unittests/Support/MemoryBufferTest.cpp
+++ b/llvm/unittests/Support/MemoryBufferTest.cpp
@@ -317,13 +317,13 @@ TEST_F(MemoryBufferTest, slice) {
   EXPECT_EQ(0x4000UL, MB.get()->getBufferSize());
  
   StringRef BufData = MB.get()->getBuffer();
-  EXPECT_TRUE(BufData.substr(0x0000,8).equals("12345678"));
-  EXPECT_TRUE(BufData.substr(0x0FF8,8).equals("12345678"));
-  EXPECT_TRUE(BufData.substr(0x1000,8).equals("abcdefgh"));
-  EXPECT_TRUE(BufData.substr(0x2FF8,8).equals("abcdefgh"));
-  EXPECT_TRUE(BufData.substr(0x3000,8).equals("ABCDEFGH"));
-  EXPECT_TRUE(BufData.substr(0x3FF8,8).equals("ABCDEFGH"));
-   
+  EXPECT_TRUE(BufData.substr(0x0000, 8) == "12345678");
+  EXPECT_TRUE(BufData.substr(0x0FF8, 8) == "12345678");
+  EXPECT_TRUE(BufData.substr(0x1000, 8) == "abcdefgh");
+  EXPECT_TRUE(BufData.substr(0x2FF8, 8) == "abcdefgh");
+  EXPECT_TRUE(BufData.substr(0x3000, 8) == "ABCDEFGH");
+  EXPECT_TRUE(BufData.substr(0x3FF8, 8) == "ABCDEFGH");
+
   // Try non-page aligned.
   ErrorOr<OwningBuffer> MB2 = MemoryBuffer::getFileSlice(TestPath.str(),
                                                          0x3000, 0x0800);
@@ -332,10 +332,10 @@ TEST_F(MemoryBufferTest, slice) {
   EXPECT_EQ(0x3000UL, MB2.get()->getBufferSize());
   
   StringRef BufData2 = MB2.get()->getBuffer();
-  EXPECT_TRUE(BufData2.substr(0x0000,8).equals("12345678"));
-  EXPECT_TRUE(BufData2.substr(0x17F8,8).equals("12345678"));
-  EXPECT_TRUE(BufData2.substr(0x1800,8).equals("abcdefgh"));
-  EXPECT_TRUE(BufData2.substr(0x2FF8,8).equals("abcdefgh"));
+  EXPECT_TRUE(BufData2.substr(0x0000, 8) == "12345678");
+  EXPECT_TRUE(BufData2.substr(0x17F8, 8) == "12345678");
+  EXPECT_TRUE(BufData2.substr(0x1800, 8) == "abcdefgh");
+  EXPECT_TRUE(BufData2.substr(0x2FF8, 8) == "abcdefgh");
 }
 
 TEST_F(MemoryBufferTest, writableSlice) {
diff --git a/llvm/unittests/Support/YAMLIOTest.cpp b/llvm/unittests/Support/YAMLIOTest.cpp
index 6ac0d1b412f0..9d40b62115a6 100644
--- a/llvm/unittests/Support/YAMLIOTest.cpp
+++ b/llvm/unittests/Support/YAMLIOTest.cpp
@@ -1389,10 +1389,10 @@ TEST(YAMLIO, TestReadWriteMyFlowSequence) {
     yin >> map2;
 
     EXPECT_FALSE(yin.error());
-    EXPECT_TRUE(map2.name.equals("hello"));
+    EXPECT_TRUE(map2.name == "hello");
     EXPECT_EQ(map2.strings.size(), 2UL);
-    EXPECT_TRUE(map2.strings[0].value.equals("one"));
-    EXPECT_TRUE(map2.strings[1].value.equals("two"));
+    EXPECT_TRUE(map2.strings[0].value == "one");
+    EXPECT_TRUE(map2.strings[1].value == "two");
     EXPECT_EQ(map2.single.size(), 1UL);
     EXPECT_EQ(1,       map2.single[0]);
     EXPECT_EQ(map2.numbers.size(), 3UL);
@@ -1436,7 +1436,7 @@ TEST(YAMLIO, TestReadWriteSequenceOfMyFlowSequence) {
     yin >> map2;
 
     EXPECT_FALSE(yin.error());
-    EXPECT_TRUE(map2.name.equals("hello"));
+    EXPECT_TRUE(map2.name == "hello");
     EXPECT_EQ(map2.sequenceOfNumbers.size(), 3UL);
     EXPECT_EQ(map2.sequenceOfNumbers[0].size(), 1UL);
     EXPECT_EQ(0,    map2.sequenceOfNumbers[0][0]);
diff --git a/llvm/unittests/TargetParser/CSKYTargetParserTest.cpp b/llvm/unittests/TargetParser/CSKYTargetParserTest.cpp
index f28a2a33eb90..50e825c5f99f 100644
--- a/llvm/unittests/TargetParser/CSKYTargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/CSKYTargetParserTest.cpp
@@ -1020,7 +1020,7 @@ TEST(TargetParserTest, testInvalidCSKYArch) {
 bool testCSKYArch(StringRef Arch, StringRef DefaultCPU) {
   CSKY::ArchKind AK = CSKY::parseArch(Arch);
   bool Result = (AK != CSKY::ArchKind::INVALID);
-  Result &= CSKY::getDefaultCPU(Arch).equals(DefaultCPU);
+  Result &= CSKY::getDefaultCPU(Arch) == DefaultCPU;
   return Result;
 }
 
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index cc098c264065..b61928bd8f98 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -572,8 +572,8 @@ bool testARMArch(StringRef Arch, StringRef DefaultCPU, StringRef SubArch,
                  unsigned ArchAttr) {
   ARM::ArchKind AK = ARM::parseArch(Arch);
   bool Result = (AK != ARM::ArchKind::INVALID);
-  Result &= ARM::getDefaultCPU(Arch).equals(DefaultCPU);
-  Result &= ARM::getSubArch(AK).equals(SubArch);
+  Result &= ARM::getDefaultCPU(Arch) == DefaultCPU;
+  Result &= ARM::getSubArch(AK) == SubArch;
   Result &= (ARM::getArchAttr(AK) == ArchAttr);
   return Result;
 }
diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index 53d49a2900a1..8e475f9153b0 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -3121,7 +3121,7 @@ static void emitMnemonicSpellChecker(raw_ostream &OS, CodeGenTarget &Target,
     OS << "\n";
     OS << "    StringRef T = I->getMnemonic();\n";
     OS << "    // Avoid recomputing the edit distance for the same string.\n";
-    OS << "    if (T.equals(Prev))\n";
+    OS << "    if (T == Prev)\n";
     OS << "      continue;\n";
     OS << "\n";
     OS << "    Prev = T;\n";
-- 
GitLab


From dabdec1001dc368373dd581cf72f37a440873ce3 Mon Sep 17 00:00:00 2001
From: Benoit Jacob <jacob.benoit.1@gmail.com>
Date: Wed, 8 May 2024 13:37:05 -0400
Subject: [PATCH 0202/1206] Fix `memref.expand_shape` verifier (#91501)

Torch-mlir integration is currently blocked on `memref.expand_shape`
verifier errors of the form

```
'memref.expand_shape' op invalid output shape provided at pos 1
```

The verifier code generating these errors was introduced in
https://github.com/llvm/llvm-project/pull/91245. I have commented there
why I believe it's incorrect. This PR has my suggested fix.

Unfortunately, this does not seem to be directly testable on `memref`
IR, because `static_output_shape` is not directly exposed in the custom
assembly format.
---
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp            | 11 +++++------
 mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir |  2 +-
 mlir/test/Dialect/MemRef/ops.mlir                   |  7 ++++++-
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index 78201ae29cd9..c9a85919ec79 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -2356,12 +2356,11 @@ LogicalResult ExpandShapeOp::verify() {
   // Verify if provided output shapes are in agreement with output type.
   DenseI64ArrayAttr staticOutputShapes = getStaticOutputShapeAttr();
   ArrayRef<int64_t> resShape = getResult().getType().getShape();
-  unsigned staticShapeNum = 0;
-
-  for (auto [pos, shape] : llvm::enumerate(resShape))
-    if (!ShapedType::isDynamic(shape) &&
-        shape != staticOutputShapes[staticShapeNum++])
-      emitOpError("invalid output shape provided at pos ") << pos;
+  for (auto [pos, shape] : llvm::enumerate(resShape)) {
+    if (!ShapedType::isDynamic(shape) && shape != staticOutputShapes[pos]) {
+      return emitOpError("invalid output shape provided at pos ") << pos;
+    }
+  }
 
   return success();
 }
diff --git a/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir b/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir
index 99b5f78b03fb..e49dff44ae0d 100644
--- a/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir
+++ b/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir
@@ -502,7 +502,7 @@ func.func @fold_dynamic_subview_with_memref_store_expand_shape(%arg0 : memref<16
 // CHECK-SAME: (%[[ARG0:.*]]: memref<2048x16xf32>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index)
 func.func @fold_memref_alias_expand_shape_subview_load_store_dynamic_dim(%alloc: memref<2048x16xf32>, %c10: index, %c5: index, %c0: index, %sz0: index) {
   %subview = memref.subview %alloc[%c5, 0] [%c10, 16] [1, 1] : memref<2048x16xf32> to memref<?x16xf32, strided<[16, 1], offset: ?>>
-  %expand_shape = memref.expand_shape %subview [[0], [1, 2, 3]] output_shape [1, 16, %sz0, 1] : memref<?x16xf32, strided<[16, 1], offset: ?>> into memref<?x1x8x2xf32, strided<[16, 16, 2, 1], offset: ?>>
+  %expand_shape = memref.expand_shape %subview [[0], [1, 2, 3]] output_shape [%sz0, 1, 8, 2] : memref<?x16xf32, strided<[16, 1], offset: ?>> into memref<?x1x8x2xf32, strided<[16, 16, 2, 1], offset: ?>>
   %dim = memref.dim %expand_shape, %c0 : memref<?x1x8x2xf32, strided<[16, 16, 2, 1], offset: ?>>
 
   affine.for %arg6 = 0 to %dim step 64 {
diff --git a/mlir/test/Dialect/MemRef/ops.mlir b/mlir/test/Dialect/MemRef/ops.mlir
index 60fb0ffeee24..b60894377f22 100644
--- a/mlir/test/Dialect/MemRef/ops.mlir
+++ b/mlir/test/Dialect/MemRef/ops.mlir
@@ -203,7 +203,8 @@ func.func @expand_collapse_shape_dynamic(%arg0: memref<?x?x?xf32>,
          %arg3: memref<?x42xf32, strided<[42, 1], offset: 0>>,
          %arg4: index,
          %arg5: index,
-         %arg6: index) {
+         %arg6: index,
+         %arg7: memref<4x?x4xf32>) {
 //       CHECK:   memref.collapse_shape {{.*}} {{\[}}[0, 1], [2]]
 //  CHECK-SAME:     memref<?x?x?xf32> into memref<?x?xf32>
   %0 = memref.collapse_shape %arg0 [[0, 1], [2]] :
@@ -248,6 +249,10 @@ func.func @expand_collapse_shape_dynamic(%arg0: memref<?x?x?xf32>,
 //  CHECK-SAME:     memref<?xf32, strided<[1]>> into memref<?x42xf32>
   %r3 = memref.expand_shape %3 [[0, 1]] output_shape [%arg6, 42] :
     memref<?xf32, strided<[1]>> into memref<?x42xf32>
+
+//       CHECK:   memref.expand_shape {{.*}} {{\[}}[0, 1], [2], [3, 4]]
+  %4 = memref.expand_shape %arg7 [[0, 1], [2], [3, 4]] output_shape [2, 2, %arg4, 2, 2]
+        : memref<4x?x4xf32> into memref<2x2x?x2x2xf32>
   return
 }
 
-- 
GitLab


From fcfc15b7052a311b7a045e2c6bd26fb5d0b7122c Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Wed, 8 May 2024 10:38:09 -0700
Subject: [PATCH 0203/1206] =?UTF-8?q?Add=20a=20dependency=20from=20lldb-sb?=
 =?UTF-8?q?api-dwarf-enums=20as=20a=20dependency=20of=20libll=E2=80=A6=20(?=
 =?UTF-8?q?#91511)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…db-resource-headers

The Xcode build otherwise fails with
```
CMake Error in source/API/CMakeLists.txt:
  The custom command generating

    /Users/ec2-user/jenkins/workspace/llvm.org/lldb-cmake-standalone/lldb-xcode-build/include/lldb/API/SBLanguages.h

  is attached to multiple targets:

    lldb-sbapi-dwarf-enums
    liblldb-resource-headers

  but none of these is a common dependency of the other(s).  This is not
  allowed by the Xcode "new build system".

CMake Generate step failed.  Build files cannot be regenerated correctly.
```
---
 lldb/cmake/modules/LLDBFramework.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/cmake/modules/LLDBFramework.cmake b/lldb/cmake/modules/LLDBFramework.cmake
index df2f8ddf54a3..dd8c36bba0e9 100644
--- a/lldb/cmake/modules/LLDBFramework.cmake
+++ b/lldb/cmake/modules/LLDBFramework.cmake
@@ -105,7 +105,7 @@ foreach(header
 endforeach()
 
 # Wrap output in a target, so lldb-framework can depend on it.
-add_custom_target(liblldb-resource-headers DEPENDS ${lldb_staged_headers})
+add_custom_target(liblldb-resource-headers DEPENDS lldb-sbapi-dwarf-enums ${lldb_staged_headers})
 set_target_properties(liblldb-resource-headers PROPERTIES FOLDER "lldb misc")
 add_dependencies(liblldb liblldb-resource-headers)
 
-- 
GitLab


From dbcfa2957d9f99de62fc86db12a857caf929583c Mon Sep 17 00:00:00 2001
From: Nicklas Boman <smurfd@gmail.com>
Date: Wed, 8 May 2024 19:57:16 +0200
Subject: [PATCH 0204/1206] =?UTF-8?q?lldb=20create=20API=20folder=20if=20i?=
 =?UTF-8?q?t=20does=20not=20exist,=20before=20creating=20SBLangua=E2=80=A6?=
 =?UTF-8?q?=20(#91128)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Create API folder if it does not exist, before creating SBLanguages.h
---
 lldb/scripts/generate-sbapi-dwarf-enum.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lldb/scripts/generate-sbapi-dwarf-enum.py b/lldb/scripts/generate-sbapi-dwarf-enum.py
index 464eb2afff7d..f7a13e5efffe 100755
--- a/lldb/scripts/generate-sbapi-dwarf-enum.py
+++ b/lldb/scripts/generate-sbapi-dwarf-enum.py
@@ -2,6 +2,7 @@
 
 import argparse
 import re
+import os
 
 HEADER = """\
 //===-- SBLanguages.h -----------------------------------------*- C++ -*-===//
@@ -37,6 +38,9 @@ def emit_enum(input, output):
     with open(input, "r") as f:
         lines = f.readlines()
 
+    # Create output folder if it does not exist
+    os.makedirs(os.path.dirname(output), exist_ok=True)
+
     # Write the output.
     with open(output, "w") as f:
         # Emit the header.
-- 
GitLab


From 42d99013bd6b7ed4a085e39c94ab86938d633f8a Mon Sep 17 00:00:00 2001
From: AdityaK <hiraditya@msn.com>
Date: Wed, 8 May 2024 11:03:46 -0700
Subject: [PATCH 0205/1206] NFC: Add a comment indicating
 UpdateAnalysisInformation invalidates DFS Numbering (#91252)

---
 llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index b9ed077b660d..462283c0bfe0 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -1141,6 +1141,7 @@ BasicBlock *llvm::splitBlockBefore(BasicBlock *Old, BasicBlock::iterator SplitPt
 }
 
 /// Update DominatorTree, LoopInfo, and LCCSA analysis information.
+/// Invalidates DFS Numbering when DTU or DT is provided.
 static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
                                       ArrayRef<BasicBlock *> Preds,
                                       DomTreeUpdater *DTU, DominatorTree *DT,
-- 
GitLab


From 10bdcf6b4cd37d017753b3821fbf8eb2ad924a1a Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Wed, 8 May 2024 20:10:47 +0200
Subject: [PATCH 0206/1206] [clang-tidy] Handle implicit casts in
 hicpp-signed-bitwise for IgnorePositiveIntegerLiterals (#90621)

Improved hicpp-signed-bitwise check by ignoring false positives
involving positive integer literals behind implicit casts when
IgnorePositiveIntegerLiterals is enabled.

Closes #89367
---
 clang-tools-extra/clang-tidy/hicpp/SignedBitwiseCheck.cpp    | 5 +++--
 clang-tools-extra/docs/ReleaseNotes.rst                      | 4 ++++
 .../checkers/hicpp/signed-bitwise-integer-literals.cpp       | 3 +++
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/hicpp/SignedBitwiseCheck.cpp b/clang-tools-extra/clang-tidy/hicpp/SignedBitwiseCheck.cpp
index 51cc26400f7f..bf09a6662d95 100644
--- a/clang-tools-extra/clang-tidy/hicpp/SignedBitwiseCheck.cpp
+++ b/clang-tools-extra/clang-tidy/hicpp/SignedBitwiseCheck.cpp
@@ -9,6 +9,7 @@
 #include "SignedBitwiseCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
 
 using namespace clang::ast_matchers;
 using namespace clang::ast_matchers::internal;
@@ -29,8 +30,8 @@ void SignedBitwiseCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 void SignedBitwiseCheck::registerMatchers(MatchFinder *Finder) {
   const auto SignedIntegerOperand =
       (IgnorePositiveIntegerLiterals
-           ? expr(ignoringImpCasts(hasType(isSignedInteger())),
-                  unless(integerLiteral()))
+           ? expr(ignoringImpCasts(
+                 allOf(hasType(isSignedInteger()), unless(integerLiteral()))))
            : expr(ignoringImpCasts(hasType(isSignedInteger()))))
           .bind("signed-operand");
 
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 5d6d0351362e..c4c9df27ee1e 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -259,6 +259,10 @@ Changes in existing checks
 - Improved :doc:`google-runtime-int <clang-tidy/checks/google/runtime-int>`
   check performance through optimizations.
 
+- Improved :doc:`hicpp-signed-bitwise <clang-tidy/checks/hicpp/signed-bitwise>`
+  check by ignoring false positives involving positive integer literals behind
+  implicit casts when `IgnorePositiveIntegerLiterals` is enabled.
+
 - Improved :doc:`hicpp-ignored-remove-result <clang-tidy/checks/hicpp/ignored-remove-result>`
   check by ignoring other functions with same prefixes as the target specific
   functions.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/hicpp/signed-bitwise-integer-literals.cpp b/clang-tools-extra/test/clang-tidy/checkers/hicpp/signed-bitwise-integer-literals.cpp
index edbb56f90cb0..aca7ae1fd76f 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/hicpp/signed-bitwise-integer-literals.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/hicpp/signed-bitwise-integer-literals.cpp
@@ -11,6 +11,7 @@ void examples() {
   // CHECK-MESSAGES: :[[@LINE-1]]:19: warning: use of a signed integer operand with a binary bitwise operator
 
   unsigned URes2 = URes << 1; //Ok
+  unsigned URes3 = URes & 1; //Ok
 
   int IResult;
   IResult = 10 & 2; //Ok
@@ -21,6 +22,8 @@ void examples() {
   IResult = Int << 1;
   // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use of a signed integer operand with a binary bitwise operator
   IResult = ~0; //Ok
+  IResult = -1 & 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use of a signed integer operand with a binary bitwise operator [hicpp-signed-bitwise]
 }
 
 enum EnumConstruction {
-- 
GitLab


From 37b6ba96dea54e7af3772d78c90bfb3fd61140f6 Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Wed, 8 May 2024 20:11:10 +0200
Subject: [PATCH 0207/1206] [clang-tidy] Handle expr with side-effects in
 readability-static-accessed-through-instance (#90736)

Improved readability-static-accessed-through-instance check to
support expressions with side-effects.

Originally calls to overloaded operator were
ignored by check, in fear of possible side-effects.

This change remove that restriction, and enables
fix-its for expressions with side-effect via
--fix-notes.

Closes #75163
---
 .../StaticAccessedThroughInstanceCheck.cpp    | 37 +++++++++++-------
 clang-tools-extra/docs/ReleaseNotes.rst       |  5 +++
 .../static-accessed-through-instance.rst      |  3 ++
 .../static-accessed-through-instance.cpp      | 38 +++++++++++++++----
 4 files changed, 62 insertions(+), 21 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
index 65356cc3929c..08adc7134cfe 100644
--- a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
@@ -59,10 +59,6 @@ void StaticAccessedThroughInstanceCheck::check(
 
   const Expr *BaseExpr = MemberExpression->getBase();
 
-  // Do not warn for overloaded -> operators.
-  if (isa<CXXOperatorCallExpr>(BaseExpr))
-    return;
-
   const QualType BaseType =
       BaseExpr->getType()->isPointerType()
           ? BaseExpr->getType()->getPointeeType().getUnqualifiedType()
@@ -89,17 +85,30 @@ void StaticAccessedThroughInstanceCheck::check(
     return;
 
   SourceLocation MemberExprStartLoc = MemberExpression->getBeginLoc();
-  auto Diag =
-      diag(MemberExprStartLoc, "static member accessed through instance");
-
-  if (BaseExpr->HasSideEffects(*AstContext) ||
-      getNameSpecifierNestingLevel(BaseType) > NameSpecifierNestingThreshold)
-    return;
+  auto CreateFix = [&] {
+    return FixItHint::CreateReplacement(
+        CharSourceRange::getCharRange(MemberExprStartLoc,
+                                      MemberExpression->getMemberLoc()),
+        BaseTypeName + "::");
+  };
+
+  {
+    auto Diag =
+        diag(MemberExprStartLoc, "static member accessed through instance");
+
+    if (getNameSpecifierNestingLevel(BaseType) > NameSpecifierNestingThreshold)
+      return;
+
+    if (!BaseExpr->HasSideEffects(*AstContext,
+                                  /* IncludePossibleEffects =*/true)) {
+      Diag << CreateFix();
+      return;
+    }
+  }
 
-  Diag << FixItHint::CreateReplacement(
-      CharSourceRange::getCharRange(MemberExprStartLoc,
-                                    MemberExpression->getMemberLoc()),
-      BaseTypeName + "::");
+  diag(MemberExprStartLoc, "member base expression may carry some side effects",
+       DiagnosticIDs::Level::Note)
+      << BaseExpr->getSourceRange() << CreateFix();
 }
 
 } // namespace clang::tidy::readability
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index c4c9df27ee1e..5b7ea42c63d6 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -352,6 +352,11 @@ Changes in existing checks
   <clang-tidy/checks/readability/redundant-inline-specifier>` check to properly
   emit warnings for static data member with an in-class initializer.
 
+- Improved :doc:`readability-static-accessed-through-instance
+  <clang-tidy/checks/readability/static-accessed-through-instance>` check to
+  support calls to overloaded operators as base expression and provide fixes to
+  expressions with side-effects.
+
 - Improved :doc:`readability-static-definition-in-anonymous-namespace
   <clang-tidy/checks/readability/static-definition-in-anonymous-namespace>`
   check by resolving fix-it overlaps in template code by disregarding implicit
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/static-accessed-through-instance.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/static-accessed-through-instance.rst
index 23d12f418366..ffb3738bf72c 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability/static-accessed-through-instance.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability/static-accessed-through-instance.rst
@@ -35,3 +35,6 @@ is changed to:
   C::E1;
   C::E2;
 
+The `--fix` commandline option provides default support for safe fixes, whereas
+`--fix-notes` enables fixes that may replace expressions with side effects,
+potentially altering the program's behavior.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/static-accessed-through-instance.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/static-accessed-through-instance.cpp
index 81c1cecf607f..202fe9be6d00 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/static-accessed-through-instance.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/static-accessed-through-instance.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s readability-static-accessed-through-instance %t -- -- -isystem %S/Inputs/static-accessed-through-instance
+// RUN: %check_clang_tidy %s readability-static-accessed-through-instance %t -- --fix-notes -- -isystem %S/Inputs/static-accessed-through-instance
 #include <__clang_cuda_builtin_vars.h>
 
 enum OutEnum {
@@ -47,7 +47,8 @@ C &f(int, int, int, int);
 void g() {
   f(1, 2, 3, 4).x;
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: static member accessed through instance  [readability-static-accessed-through-instance]
-  // CHECK-FIXES: {{^}}  f(1, 2, 3, 4).x;{{$}}
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: member base expression may carry some side effects
+  // CHECK-FIXES: {{^}}  C::x;{{$}}
 }
 
 int i(int &);
@@ -59,12 +60,14 @@ int k(bool);
 void f(C c) {
   j(i(h().x));
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: static member
-  // CHECK-FIXES: {{^}}  j(i(h().x));{{$}}
+  // CHECK-MESSAGES: :[[@LINE-2]]:7: note: member base expression may carry some side effects
+  // CHECK-FIXES: {{^}}  j(i(C::x));{{$}}
 
   // The execution of h() depends on the return value of a().
   j(k(a() && h().x));
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: static member
-  // CHECK-FIXES: {{^}}  j(k(a() && h().x));{{$}}
+  // CHECK-MESSAGES: :[[@LINE-2]]:14: note: member base expression may carry some side effects
+  // CHECK-FIXES: {{^}}  j(k(a() && C::x));{{$}}
 
   if ([c]() {
         c.ns();
@@ -72,7 +75,8 @@ void f(C c) {
       }().x == 15)
     ;
   // CHECK-MESSAGES: :[[@LINE-5]]:7: warning: static member
-  // CHECK-FIXES: {{^}}  if ([c]() {{{$}}
+  // CHECK-MESSAGES: :[[@LINE-6]]:7: note: member base expression may carry some side effects
+  // CHECK-FIXES: {{^}}  if (C::x == 15){{$}}
 }
 
 // Nested specifiers
@@ -261,8 +265,11 @@ struct Qptr {
 };
 
 int func(Qptr qp) {
-  qp->y = 10; // OK, the overloaded operator might have side-effects.
-  qp->K = 10; //
+  qp->y = 10;
+  qp->K = 10;
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: static member accessed through instance [readability-static-accessed-through-instance]
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: member base expression may carry some side effects
+  // CHECK-FIXES: {{^}}  Q::K = 10;
 }
 
 namespace {
@@ -380,3 +387,20 @@ namespace PR51861 {
     // CHECK-FIXES: {{^}}    PR51861::Foo::getBar();{{$}}
   }
 }
+
+namespace PR75163 {
+  struct Static {
+    static void call();
+  };
+
+  struct Ptr {
+    Static* operator->();
+  };
+
+  void test(Ptr& ptr) {
+    ptr->call();
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: static member accessed through instance [readability-static-accessed-through-instance]
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: note: member base expression may carry some side effects
+    // CHECK-FIXES: {{^}}    PR75163::Static::call();{{$}}
+  }
+}
-- 
GitLab


From 79921fbd5c6223ff7e6c75ed75974b4d16cad529 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Wed, 8 May 2024 20:17:59 +0200
Subject: [PATCH 0208/1206] [libc++][CI] Reenables clang-tidy. (#90077)

The patch does several things:
- fixes module exports
- disables clang-tidy with Clang-17 due to known issues
- disabled clang-tidy on older libstdc++ versions since it lacks C++20
features used
- fixes the CMake dependency

The issue why clang-tidy was not used in the CI was the last issue; the
plugin was not a
dependency of the tests. Without a plugin the tests disable clang-tidy.

This was noticed while investigating
https://github.com/llvm/llvm-project/issues/89898
---
 libcxx/modules/std/chrono.inc                 | 18 +++++++++-----
 libcxx/modules/std/ranges.inc                 |  7 +++---
 libcxx/test/CMakeLists.txt                    |  6 +++++
 libcxx/test/libcxx/clang_tidy.gen.py          |  3 +++
 .../tools/clang_tidy_checks/CMakeLists.txt    | 24 +++++++++++++++++--
 5 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/libcxx/modules/std/chrono.inc b/libcxx/modules/std/chrono.inc
index 1265e21dc54e..813322a1797f 100644
--- a/libcxx/modules/std/chrono.inc
+++ b/libcxx/modules/std/chrono.inc
@@ -190,10 +190,11 @@ export namespace std {
     using std::chrono::make12;
     using std::chrono::make24;
 
-#if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                              \
-    !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#ifdef _LIBCPP_ENABLE_EXPERIMENTAL
+
+#  if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                            \
+      !defined(_LIBCPP_HAS_NO_LOCALIZATION)
 
-#  ifdef _LIBCPP_ENABLE_EXPERIMENTAL
     // [time.zone.db], time zone database
     using std::chrono::tzdb;
     using std::chrono::tzdb_list;
@@ -213,11 +214,16 @@ export namespace std {
     using std::chrono::ambiguous_local_time;
     using std::chrono::nonexistent_local_time;
 #    endif // if 0
+#  endif   //  !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&
+           //  !defined(_LIBCPP_HAS_NO_LOCALIZATION)
 
     // [time.zone.info], information classes
     using std::chrono::local_info;
     using std::chrono::sys_info;
 
+#  if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                            \
+      !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+
 #    if 0
     // [time.zone.timezone], class time_zone
     using std::chrono::choose;
@@ -246,9 +252,9 @@ export namespace std {
     // [time.format], formatting
     using std::chrono::local_time_format;
 #    endif
-#  endif // _LIBCPP_ENABLE_EXPERIMENTAL
-#endif   //  !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&
-         //    !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#  endif //  !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&
+         //  !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#endif   // _LIBCPP_ENABLE_EXPERIMENTAL
 
   } // namespace chrono
 
diff --git a/libcxx/modules/std/ranges.inc b/libcxx/modules/std/ranges.inc
index 80f31c79a1a4..f71efe948ede 100644
--- a/libcxx/modules/std/ranges.inc
+++ b/libcxx/modules/std/ranges.inc
@@ -138,9 +138,6 @@ export namespace std {
     }
 #endif // _LIBCPP_HAS_NO_LOCALIZATION
 
-#if _LIBCPP_STD_VER >= 23
-    // [range.adaptor.object], range adaptor objects
-    using std::ranges::range_adaptor_closure;
     // Note: This declaration not in the synopsis or explicitly in the wording.
     // However it is needed for the range adaptors.
     // [range.adaptor.object]/3
@@ -151,7 +148,11 @@ export namespace std {
     //   involving an object of type cv D as an operand to the | operator is
     //   undefined if overload resolution selects a program-defined operator|
     //   function.
+    // This is used internally in C++20 mode.
     using std::ranges::operator|;
+#if _LIBCPP_STD_VER >= 23
+    // [range.adaptor.object], range adaptor objects
+    using std::ranges::range_adaptor_closure;
 #endif
 
     // [range.all], all view
diff --git a/libcxx/test/CMakeLists.txt b/libcxx/test/CMakeLists.txt
index e0d3a0dbc400..fd57aa9fe8b3 100644
--- a/libcxx/test/CMakeLists.txt
+++ b/libcxx/test/CMakeLists.txt
@@ -1,5 +1,11 @@
 include(HandleLitArguments)
 add_subdirectory(tools)
+# When the tools add clang-tidy support, the dependencies need to be updated.
+# This cannot be done in the tools CMakeLists.txt since that does not update
+# the status in this (a parent) directory.
+if(TARGET cxx-tidy)
+  list(APPEND LIBCXX_TEST_DEPS cxx-tidy)
+endif()
 
 # By default, libcxx and libcxxabi share a library directory.
 if (NOT LIBCXX_CXX_ABI_LIBRARY_PATH)
diff --git a/libcxx/test/libcxx/clang_tidy.gen.py b/libcxx/test/libcxx/clang_tidy.gen.py
index f29447d00655..76b9db2d5cb8 100644
--- a/libcxx/test/libcxx/clang_tidy.gen.py
+++ b/libcxx/test/libcxx/clang_tidy.gen.py
@@ -26,6 +26,9 @@ for header in public_headers:
 // The GCC compiler flags are not always compatible with clang-tidy.
 // UNSUPPORTED: gcc
 
+// Clang 17 has false positives.
+// UNSUPPORTED: clang-17
+
 {lit_header_restrictions.get(header, '')}
 
 // TODO: run clang-tidy with modules enabled once they are supported
diff --git a/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt b/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
index 28eed6144583..28c1dbf8aca3 100644
--- a/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
+++ b/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
@@ -64,6 +64,28 @@ if(NOT HAS_CLANG_TIDY_HEADERS)
                  "clang-tidy headers are not present.")
   return()
 endif()
+
+# The clangTidy plugin uses C++20, so ensure that we support C++20 when using libstdc++.
+# This is required because some versions of libstdc++ used as a system library on build platforms
+# we support do not support C++20 yet.
+# Note it has not been tested whether version 11 works.
+file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/test.cpp" "
+#include <version>
+#if defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12
+  # error The libstdc++ version is too old.
+#endif
+int main(){}
+")
+try_compile(HAS_NEWER_STANDARD_LIBRARY
+  "${CMAKE_CURRENT_BINARY_DIR}"
+  "${CMAKE_CURRENT_BINARY_DIR}/test.cpp"
+   LINK_LIBRARIES clangTidy)
+
+if(NOT HAS_NEWER_STANDARD_LIBRARY)
+  message(STATUS "Clang-tidy tests are disabled due to using "
+                 "stdlibc++ older than version 12")
+  return()
+endif()
 message(STATUS "Clang-tidy tests are enabled.")
 
 set(SOURCES
@@ -88,5 +110,3 @@ set_target_properties(cxx-tidy PROPERTIES
 
 set_target_properties(cxx-tidy PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 set(CMAKE_SHARED_MODULE_SUFFIX_CXX .plugin) # Use a portable suffix to simplify how we can find it from Lit
-
-list(APPEND LIBCXX_TEST_DEPS cxx-tidy)
-- 
GitLab


From a6b623705b13b0f69c302ee7b36fe87f833ff193 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Wed, 8 May 2024 15:01:58 -0400
Subject: [PATCH 0209/1206] [gn] port 2868e26d0a6f (PERL_EXECUTABLE)

---
 llvm/utils/gn/secondary/clang/test/BUILD.gn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index 4ed9352da9c9..11454e68ec91 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -116,11 +116,13 @@ write_lit_config("lit_site_cfg") {
       "CMAKE_LIBRARY_OUTPUT_DIRECTORY=" + rebase_path("$root_out_dir/bin", dir),
       "LLVM_LIT_ERRC_MESSAGES=no such file or directory;is a directory;" +
           "invalid argument;permission denied",
+      "PERL_EXECUTABLE="
     ]
   } else {
     extra_values += [
       "CMAKE_LIBRARY_OUTPUT_DIRECTORY=" + rebase_path("$root_out_dir/lib", dir),
       "LLVM_LIT_ERRC_MESSAGES=",
+      "PERL_EXECUTABLE=/usr/bin/perl"
     ]
   }
 
-- 
GitLab


From db29f20fdd4f715553f663f21021330cb4497e00 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Wed, 8 May 2024 12:02:18 -0700
Subject: [PATCH 0210/1206] [BOLT] Ignore returns in DataAggregator

Returns are ignored in perf/pre-aggregated/fdata profile reader (see
DataReader::convertBranchData). They are also omitted in
YAMLProfileWriter by virtue of not having the profile attached to them
in the reader, and YAMLProfileWriter converting the profile attached to
BinaryFunctions. Thus, return profile is universally ignored across all
profile types except BAT YAML.

To make returns ignored for YAML produced in BAT mode, we can:
1) ignore them in YAMLProfileReader,
2) omit them from YAML profile in profile conversion/writing.

The first option is prone to profile staleness issue, where the profiled
binary doesn't match the one to be optimized, and thus returns in the
profile can no longer be reliably detected (as we don't distinguish them
from calls in the profile).

The second option is robust to staleness but requires disassembling the
branch source instruction.

Test Plan: Updated bolt-address-translation-yaml.test

Reviewers: rafaelauler, dcci, ayermolo, maksfb

Reviewed By: maksfb

Pull Request: https://github.com/llvm/llvm-project/pull/90807
---
 bolt/include/bolt/Core/BinaryFunction.h          |  2 ++
 bolt/lib/Core/BinaryFunction.cpp                 | 15 +++++++++++++++
 bolt/lib/Profile/DataAggregator.cpp              | 13 +++++++++++++
 bolt/test/X86/bolt-address-translation-yaml.test |  5 +++--
 4 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 26d2d01f8626..3c641581e247 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -930,6 +930,8 @@ public:
     return const_cast<BinaryFunction *>(this)->getInstructionAtOffset(Offset);
   }
 
+  std::optional<MCInst> disassembleInstructionAtOffset(uint64_t Offset) const;
+
   /// Return offset for the first instruction. If there is data at the
   /// beginning of a function then offset of the first instruction could
   /// be different from 0
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 1fa96dfaabde..de34421ebeb0 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -1167,6 +1167,21 @@ void BinaryFunction::handleAArch64IndirectCall(MCInst &Instruction,
   }
 }
 
+std::optional<MCInst>
+BinaryFunction::disassembleInstructionAtOffset(uint64_t Offset) const {
+  assert(CurrentState == State::Empty && "Function should not be disassembled");
+  assert(Offset < MaxSize && "Invalid offset");
+  ErrorOr<ArrayRef<unsigned char>> FunctionData = getData();
+  assert(FunctionData && "Cannot get function as data");
+  MCInst Instr;
+  uint64_t InstrSize = 0;
+  const uint64_t InstrAddress = getAddress() + Offset;
+  if (BC.DisAsm->getInstruction(Instr, InstrSize, FunctionData->slice(Offset),
+                                InstrAddress, nulls()))
+    return Instr;
+  return std::nullopt;
+}
+
 Error BinaryFunction::disassemble() {
   NamedRegionTimer T("disassemble", "Disassemble function", "buildfuncs",
                      "Build Binary Functions", opts::TimeBuild);
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 5108392c824c..d02e4499014e 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -773,9 +773,19 @@ bool DataAggregator::doInterBranch(BinaryFunction *FromFunc,
 
 bool DataAggregator::doBranch(uint64_t From, uint64_t To, uint64_t Count,
                               uint64_t Mispreds) {
+  bool IsReturn = false;
   auto handleAddress = [&](uint64_t &Addr, bool IsFrom) -> BinaryFunction * {
     if (BinaryFunction *Func = getBinaryFunctionContainingAddress(Addr)) {
       Addr -= Func->getAddress();
+      if (IsFrom) {
+        auto checkReturn = [&](auto MaybeInst) {
+          IsReturn = MaybeInst && BC->MIB->isReturn(*MaybeInst);
+        };
+        if (Func->hasInstructions())
+          checkReturn(Func->getInstructionAtOffset(Addr));
+        else
+          checkReturn(Func->disassembleInstructionAtOffset(Addr));
+      }
 
       if (BAT)
         Addr = BAT->translate(Func->getAddress(), Addr, IsFrom);
@@ -792,6 +802,9 @@ bool DataAggregator::doBranch(uint64_t From, uint64_t To, uint64_t Count,
   };
 
   BinaryFunction *FromFunc = handleAddress(From, /*IsFrom=*/true);
+  // Ignore returns.
+  if (IsReturn)
+    return true;
   BinaryFunction *ToFunc = handleAddress(To, /*IsFrom=*/false);
   if (!FromFunc && !ToFunc)
     return false;
diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test
index af24c3d84a0f..b3d8a8839450 100644
--- a/bolt/test/X86/bolt-address-translation-yaml.test
+++ b/bolt/test/X86/bolt-address-translation-yaml.test
@@ -13,7 +13,7 @@ RUN: llvm-bolt %t.exe -data %t.fdata -w %t.yaml-fdata -o /dev/null
 RUN: FileCheck --input-file %t.yaml-fdata --check-prefix YAML-BAT-CHECK %s
 
 # Test resulting YAML profile with the original binary (no-stale mode)
-RUN: llvm-bolt %t.exe -data %t.yaml -o %t.null -dyno-stats \
+RUN: llvm-bolt %t.exe -data %t.yaml -o %t.null -dyno-stats 2>&1 \
 RUN:   | FileCheck --check-prefix CHECK-BOLT-YAML %s
 
 WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps
@@ -63,7 +63,8 @@ YAML-BAT-CHECK-NEXT:   blocks:
 YAML-BAT-CHECK:        - bid:   1
 YAML-BAT-CHECK-NEXT:       insns: [[#]]
 YAML-BAT-CHECK-NEXT:       hash:  0xD70DC695320E0010
-YAML-BAT-CHECK-NEXT:       succ:  {{.*}} { bid: 2, cnt: [[#]] }
+YAML-BAT-CHECK-NEXT:       succ:  {{.*}} { bid: 2, cnt: [[#]]
 
 CHECK-BOLT-YAML:      pre-processing profile using YAML profile reader
 CHECK-BOLT-YAML-NEXT: 5 out of 16 functions in the binary (31.2%) have non-empty execution profile
+CHECK-BOLT-YAML-NOT: invalid (possibly stale) profile
-- 
GitLab


From 2f956a35edb61d250a52c4d883f368d060fae57c Mon Sep 17 00:00:00 2001
From: Artem Belevich <tra@google.com>
Date: Wed, 8 May 2024 12:02:57 -0700
Subject: [PATCH 0211/1206] [CUDA] Mark CUDA-12.4 as supported and introduce
 ptx 8.4. (#91516)

---
 clang/docs/ReleaseNotes.rst                 | 1 +
 clang/include/clang/Basic/BuiltinsNVPTX.def | 5 ++++-
 clang/include/clang/Basic/Cuda.h            | 3 ++-
 clang/lib/Basic/Cuda.cpp                    | 5 +++--
 clang/lib/Driver/ToolChains/Cuda.cpp        | 3 +++
 llvm/lib/Target/NVPTX/NVPTX.td              | 2 +-
 6 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 0f9728c00e64..a3c8e4141ca5 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -798,6 +798,7 @@ CUDA/HIP Language Changes
 
 CUDA Support
 ^^^^^^^^^^^^
+- Clang now supports CUDA SDK up to 12.4
 
 AIX Support
 ^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.def b/clang/include/clang/Basic/BuiltinsNVPTX.def
index 8d3c5e69d55c..9e243d740ed7 100644
--- a/clang/include/clang/Basic/BuiltinsNVPTX.def
+++ b/clang/include/clang/Basic/BuiltinsNVPTX.def
@@ -61,7 +61,9 @@
 #pragma push_macro("PTX81")
 #pragma push_macro("PTX82")
 #pragma push_macro("PTX83")
-#define PTX83 "ptx83"
+#pragma push_macro("PTX84")
+#define PTX84 "ptx84"
+#define PTX83 "ptx83|" PTX84
 #define PTX82 "ptx82|" PTX83
 #define PTX81 "ptx81|" PTX82
 #define PTX80 "ptx80|" PTX81
@@ -1091,3 +1093,4 @@ TARGET_BUILTIN(__nvvm_getctarank_shared_cluster, "iv*3", "", AND(SM_90,PTX78))
 #pragma pop_macro("PTX81")
 #pragma pop_macro("PTX82")
 #pragma pop_macro("PTX83")
+#pragma pop_macro("PTX84")
diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h
index ba0e4465a0f5..2d67c4181d12 100644
--- a/clang/include/clang/Basic/Cuda.h
+++ b/clang/include/clang/Basic/Cuda.h
@@ -41,9 +41,10 @@ enum class CudaVersion {
   CUDA_121,
   CUDA_122,
   CUDA_123,
+  CUDA_124,
   FULLY_SUPPORTED = CUDA_123,
   PARTIALLY_SUPPORTED =
-      CUDA_123, // Partially supported. Proceed with a warning.
+      CUDA_124, // Partially supported. Proceed with a warning.
   NEW = 10000,  // Too new. Issue a warning, but allow using it.
 };
 const char *CudaVersionToString(CudaVersion V);
diff --git a/clang/lib/Basic/Cuda.cpp b/clang/lib/Basic/Cuda.cpp
index 113483db5729..e8ce15eb0dec 100644
--- a/clang/lib/Basic/Cuda.cpp
+++ b/clang/lib/Basic/Cuda.cpp
@@ -14,7 +14,7 @@ struct CudaVersionMapEntry {
 };
 #define CUDA_ENTRY(major, minor)                                               \
   {                                                                            \
-#major "." #minor, CudaVersion::CUDA_##major##minor,                       \
+    #major "." #minor, CudaVersion::CUDA_##major##minor,                       \
         llvm::VersionTuple(major, minor)                                       \
   }
 
@@ -41,6 +41,7 @@ static const CudaVersionMapEntry CudaNameVersionMap[] = {
     CUDA_ENTRY(12, 1),
     CUDA_ENTRY(12, 2),
     CUDA_ENTRY(12, 3),
+    CUDA_ENTRY(12, 4),
     {"", CudaVersion::NEW, llvm::VersionTuple(std::numeric_limits<int>::max())},
     {"unknown", CudaVersion::UNKNOWN, {}} // End of list tombstone.
 };
@@ -241,7 +242,7 @@ CudaVersion MaxVersionForCudaArch(CudaArch A) {
   }
 }
 
-bool CudaFeatureEnabled(llvm::VersionTuple  Version, CudaFeature Feature) {
+bool CudaFeatureEnabled(llvm::VersionTuple Version, CudaFeature Feature) {
   return CudaFeatureEnabled(ToCudaVersion(Version), Feature);
 }
 
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index 6634e6d818b3..d5f93c9c830f 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -82,6 +82,8 @@ CudaVersion getCudaVersion(uint32_t raw_version) {
     return CudaVersion::CUDA_122;
   if (raw_version < 12040)
     return CudaVersion::CUDA_123;
+  if (raw_version < 12050)
+    return CudaVersion::CUDA_124;
   return CudaVersion::NEW;
 }
 
@@ -688,6 +690,7 @@ void NVPTX::getNVPTXTargetFeatures(const Driver &D, const llvm::Triple &Triple,
   case CudaVersion::CUDA_##CUDA_VER:                                           \
     PtxFeature = "+ptx" #PTX_VER;                                              \
     break;
+    CASE_CUDA_VERSION(124, 84);
     CASE_CUDA_VERSION(123, 83);
     CASE_CUDA_VERSION(122, 82);
     CASE_CUDA_VERSION(121, 81);
diff --git a/llvm/lib/Target/NVPTX/NVPTX.td b/llvm/lib/Target/NVPTX/NVPTX.td
index 6aa98543e5e2..05457c71cd39 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.td
+++ b/llvm/lib/Target/NVPTX/NVPTX.td
@@ -41,7 +41,7 @@ foreach sm = [20, 21, 30, 32, 35, 37, 50, 52, 53,
 def SM90a: FeatureSM<"90a", 901>;
 
 foreach version = [32, 40, 41, 42, 43, 50, 60, 61, 62, 63, 64, 65,
-                   70, 71, 72, 73, 74, 75, 76, 77, 78, 80, 81, 82, 83] in
+                   70, 71, 72, 73, 74, 75, 76, 77, 78, 80, 81, 82, 83, 84] in
   def PTX#version: FeaturePTX<version>;
 
 //===----------------------------------------------------------------------===//
-- 
GitLab


From 7e35a9a0e77aad673a3054855761ee4afd2605c1 Mon Sep 17 00:00:00 2001
From: Max191 <44243577+Max191@users.noreply.github.com>
Date: Wed, 8 May 2024 12:05:53 -0700
Subject: [PATCH 0212/1206] [mlir] Replace dynamic sizes in insert_slice of
 tensor.cast canonicalization (#91352)

In some cases this pattern may ignore static information due to dynamic
operands in the insert_slice sizes operands, e.g.:
```
%0 = tensor.cast %arg0 : tensor<1x?xf32> to tensor<?x?xf32>
%1 = tensor.insert_slice %0 into %arg1[...] [%s0, %s1] [...]
    : tensor<?x?xf32> into tensor<?x?xf32>
```
Can be rewritten into:
```
%1 = tensor.insert_slice %arg0 into %arg1[...] [1, %s1] [...]
    : tensor<1x?xf32> into tensor<?x?xf32>
```
This PR updates the matching in the pattern to allow rewrites like this.
---
 mlir/include/mlir/IR/BuiltinTypes.h        |  8 +++-
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp   | 29 +++++++++++++--
 mlir/lib/IR/BuiltinTypes.cpp               | 24 ++++++------
 mlir/test/Dialect/Tensor/canonicalize.mlir | 43 ++++++++++++++--------
 4 files changed, 73 insertions(+), 31 deletions(-)

diff --git a/mlir/include/mlir/IR/BuiltinTypes.h b/mlir/include/mlir/IR/BuiltinTypes.h
index 2361cf137123..5579b138668d 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.h
+++ b/mlir/include/mlir/IR/BuiltinTypes.h
@@ -360,9 +360,15 @@ private:
 /// which dimensions must be kept when e.g. compute MemRef strides under
 /// rank-reducing operations. Return std::nullopt if reducedShape cannot be
 /// obtained by dropping only `1` entries in `originalShape`.
+/// If `matchDynamic` is true, then dynamic dims in `originalShape` and
+/// `reducedShape` will be considered matching with non-dynamic dims, unless
+/// the non-dynamic dim is from `originalShape` and equal to 1. For example,
+/// in ([1, 3, ?], [?, 5]), the mask would be {1, 0, 0}, since 3 and 5 will
+/// match with the corresponding dynamic dims.
 std::optional<llvm::SmallDenseSet<unsigned>>
 computeRankReductionMask(ArrayRef<int64_t> originalShape,
-                         ArrayRef<int64_t> reducedShape);
+                         ArrayRef<int64_t> reducedShape,
+                         bool matchDynamic = false);
 
 /// Enum that captures information related to verifier error conditions on
 /// slice insert/extract type of ops.
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 7a13f7a7d135..1f94397e823f 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -2713,15 +2713,38 @@ struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
     auto dstType = llvm::dyn_cast<RankedTensorType>(dst.getType());
     if (!srcType || !dstType)
       return failure();
+
+    // The tensor.cast source could have additional static information not seen
+    // in the insert slice op static sizes, so we ignore dynamic dims when
+    // computing the rank reduction mask.
+    SmallVector<int64_t> staticSizes(insertSliceOp.getStaticSizes());
+    auto rankReductionMask = computeRankReductionMask(
+        staticSizes, srcType.getShape(), /*matchDynamic=*/true);
+    if (!rankReductionMask.has_value())
+      return failure();
+    // Replace dimensions in the insert slice op with corresponding static dims
+    // from the cast source type. If the insert slice sizes have static dims
+    // that are not static in the tensor.cast source (i.e., when the cast op
+    // casts a dynamic dim to static), the dim should not be replaced, and the
+    // pattern will fail later in `verifyInsertSliceOp`.
+    SmallVector<OpFoldResult> mixedSizes(insertSliceOp.getMixedSizes());
+    int64_t rankReducedIdx = 0;
+    for (auto [idx, size] : enumerate(staticSizes)) {
+      if (!rankReductionMask.value().contains(idx) &&
+          !srcType.isDynamicDim(rankReducedIdx)) {
+        mixedSizes[idx] = getAsIndexOpFoldResult(
+            rewriter.getContext(), srcType.getDimSize(rankReducedIdx));
+        size = srcType.getDimSize(rankReducedIdx++);
+      }
+    }
     if (verifyInsertSliceOp(srcType, dstType, insertSliceOp.getStaticOffsets(),
-                            insertSliceOp.getStaticSizes(),
-                            insertSliceOp.getStaticStrides()) !=
+                            staticSizes, insertSliceOp.getStaticStrides()) !=
         SliceVerificationResult::Success)
       return failure();
 
     Operation *replacement = rewriter.create<InsertOpTy>(
         insertSliceOp.getLoc(), src, dst, insertSliceOp.getMixedOffsets(),
-        insertSliceOp.getMixedSizes(), insertSliceOp.getMixedStrides());
+        mixedSizes, insertSliceOp.getMixedStrides());
 
     // In the parallel case there is no result and so nothing to cast.
     bool isParallelInsert =
diff --git a/mlir/lib/IR/BuiltinTypes.cpp b/mlir/lib/IR/BuiltinTypes.cpp
index a2738946de41..179797cb943a 100644
--- a/mlir/lib/IR/BuiltinTypes.cpp
+++ b/mlir/lib/IR/BuiltinTypes.cpp
@@ -408,24 +408,24 @@ unsigned BaseMemRefType::getMemorySpaceAsInt() const {
 // MemRefType
 //===----------------------------------------------------------------------===//
 
-/// Given an `originalShape` and a `reducedShape` assumed to be a subset of
-/// `originalShape` with some `1` entries erased, return the set of indices
-/// that specifies which of the entries of `originalShape` are dropped to obtain
-/// `reducedShape`. The returned mask can be applied as a projection to
-/// `originalShape` to obtain the `reducedShape`. This mask is useful to track
-/// which dimensions must be kept when e.g. compute MemRef strides under
-/// rank-reducing operations. Return std::nullopt if reducedShape cannot be
-/// obtained by dropping only `1` entries in `originalShape`.
 std::optional<llvm::SmallDenseSet<unsigned>>
 mlir::computeRankReductionMask(ArrayRef<int64_t> originalShape,
-                               ArrayRef<int64_t> reducedShape) {
+                               ArrayRef<int64_t> reducedShape,
+                               bool matchDynamic) {
   size_t originalRank = originalShape.size(), reducedRank = reducedShape.size();
   llvm::SmallDenseSet<unsigned> unusedDims;
   unsigned reducedIdx = 0;
   for (unsigned originalIdx = 0; originalIdx < originalRank; ++originalIdx) {
     // Greedily insert `originalIdx` if match.
-    if (reducedIdx < reducedRank &&
-        originalShape[originalIdx] == reducedShape[reducedIdx]) {
+    int64_t origSize = originalShape[originalIdx];
+    // if `matchDynamic`, count dynamic dims as a match, unless `origSize` is 1.
+    if (matchDynamic && reducedIdx < reducedRank && origSize != 1 &&
+        (ShapedType::isDynamic(reducedShape[reducedIdx]) ||
+         ShapedType::isDynamic(origSize))) {
+      reducedIdx++;
+      continue;
+    }
+    if (reducedIdx < reducedRank && origSize == reducedShape[reducedIdx]) {
       reducedIdx++;
       continue;
     }
@@ -433,7 +433,7 @@ mlir::computeRankReductionMask(ArrayRef<int64_t> originalShape,
     unusedDims.insert(originalIdx);
     // If no match on `originalIdx`, the `originalShape` at this dimension
     // must be 1, otherwise we bail.
-    if (originalShape[originalIdx] != 1)
+    if (origSize != 1)
       return std::nullopt;
   }
   // The whole reducedShape must be scanned, otherwise we bail.
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
index 6177fe3c752c..8036d996d232 100644
--- a/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -755,6 +755,34 @@ func.func @fold_dim_of_tensor.cast(%arg0 : tensor<4x?xf32>) -> (index, index) {
 
 // -----
 
+// CHECK-LABEL: func @insert_slice_cast
+func.func @insert_slice_cast(%arg0 : tensor<1x?xf32>, %arg1 : tensor<?x?xf32>, %arg2 : index, %arg3 : index, %arg4 : index, %arg5 : index, %arg6 : index, %arg7 : index) -> tensor<?x?xf32> {
+  // CHECK-SAME: %[[ARG0:.*]]: tensor<1x?xf32>
+  %0 = tensor.cast %arg0 : tensor<1x?xf32> to tensor<?x?xf32>
+  // CHECK: %[[RES:.*]] = tensor.insert_slice %[[ARG0]]
+  // CHECK-SAME: [{{.*}}, {{.*}}] [1, {{.*}}] [{{.*}}, {{.*}}]
+  // CHECK-SAME: : tensor<1x?xf32> into tensor<?x?xf32>
+  %1 = tensor.insert_slice %0 into %arg1[%arg2, %arg3] [%arg4, %arg5] [%arg6, %arg7] : tensor<?x?xf32> into tensor<?x?xf32>
+  // CHECK: return %[[RES]] : tensor<?x?xf32>
+  return %1 : tensor<?x?xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @insert_slice_cast_no_fold
+func.func @insert_slice_cast_no_fold(%arg0 : tensor<1x?xf32>, %arg1 : tensor<?x?xf32>, %arg2 : index, %arg3 : index, %arg4 : index, %arg5 : index, %arg6 : index, %arg7 : index) -> tensor<?x?xf32> {
+  %0 = tensor.cast %arg0 : tensor<1x?xf32> to tensor<?x5xf32>
+  // CHECK: %[[CAST:.*]] = tensor.cast
+  // CHECK: %[[RES:.*]] = tensor.insert_slice %[[CAST]]
+  // CHECK-SAME: [{{.*}}, {{.*}}] [{{.*}}, 5] [{{.*}}, {{.*}}]
+  // CHECK-SAME: : tensor<?x5xf32> into tensor<?x?xf32>
+  %1 = tensor.insert_slice %0 into %arg1[%arg2, %arg3] [%arg4, 5] [%arg6, %arg7] : tensor<?x5xf32> into tensor<?x?xf32>
+  // CHECK: return %[[RES]] : tensor<?x?xf32>
+  return %1 : tensor<?x?xf32>
+}
+
+// -----
+
 // CHECK-LABEL: func @insert_tensor_cast_on_insert_slice_src(
 // CHECK-SAME:      %[[arg0:.*]]: tensor<?x5x?xf32>, %[[arg1:.*]]: tensor<?x?x?xf32>
 //      CHECK:    %[[cast:.*]] = tensor.cast %[[arg0]] : tensor<?x5x?xf32> to tensor<64x5x64xf32>
@@ -1890,21 +1918,6 @@ func.func @splat_dynamic_no_fold(%m: index) -> tensor<4x?xf32> {
 
 // -----
 
-// There was an issue in cast + insert_slice folding generating invalid ir.
-// https://github.com/llvm/llvm-project/issues/53099
-// CHECK-LABEL: func @insert_slice_cast
-func.func @insert_slice_cast(%arg0 : tensor<1x?xf32>, %arg1 : tensor<?x?xf32>, %arg2 : index, %arg3 : index, %arg4 : index, %arg5 : index, %arg6 : index, %arg7 : index) -> tensor<?x?xf32> {
-  // CHECK: %[[CAST:.*]] = tensor.cast %{{.*}} : tensor<1x?xf32> to tensor<?x?xf32>
-  %0 = tensor.cast %arg0 : tensor<1x?xf32> to tensor<?x?xf32>
-  // CHECK: %[[RES:.*]] = tensor.insert_slice %[[CAST]]
-  // CHECK-SAME: : tensor<?x?xf32> into tensor<?x?xf32>
-  %1 = tensor.insert_slice %0 into %arg1[%arg2, %arg3] [%arg4, %arg5] [%arg6, %arg7] : tensor<?x?xf32> into tensor<?x?xf32>
-  // CHECK: return %[[RES]] : tensor<?x?xf32>
-  return %1 : tensor<?x?xf32>
-}
-
-// -----
-
 // CHECK-LABEL: func @cast_extract_slice
 func.func @cast_extract_slice(%arg0 : tensor<128x512xf32>, %s : index, %o : index)
     -> tensor<16x512xf32> {
-- 
GitLab


From 878c141adcd3a1ea47c4cc8429af5c8522678536 Mon Sep 17 00:00:00 2001
From: Lily Brown <lily@lily.fyi>
Date: Wed, 8 May 2024 12:07:37 -0700
Subject: [PATCH 0213/1206] [mlir-lsp] Add DiagnosticTag from LSP spec (#91396)

Adds the [DiagnosticTag][diagtag] LSP construct to the LSP support
headers. I also added a unit test file to validate that the `tags` array
is omitted entirely if it's empty.

The LSP spec requires that `Diagnostic::tags` be an array; in order to
conform to that I used `std::vector`, as `SmallVector` doesn't have JSON
decoding support (you can encode it to JSON, but not decode it from
JSON).

[diagtag]:
https://microsoft.github.io/language-server-protocol/specifications/lsp/3.18/specification/#diagnosticTag
---
 .../mlir/Tools/lsp-server-support/Protocol.h  | 13 +++++
 .../lib/Tools/lsp-server-support/Protocol.cpp | 19 ++++++-
 .../Tools/lsp-server-support/CMakeLists.txt   |  1 +
 .../Tools/lsp-server-support/Protocol.cpp     | 51 +++++++++++++++++++
 4 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 mlir/unittests/Tools/lsp-server-support/Protocol.cpp

diff --git a/mlir/include/mlir/Tools/lsp-server-support/Protocol.h b/mlir/include/mlir/Tools/lsp-server-support/Protocol.h
index 839d82bb02b8..1d22b8a66774 100644
--- a/mlir/include/mlir/Tools/lsp-server-support/Protocol.h
+++ b/mlir/include/mlir/Tools/lsp-server-support/Protocol.h
@@ -677,6 +677,16 @@ enum class DiagnosticSeverity {
   Hint = 4
 };
 
+enum class DiagnosticTag {
+  Unnecessary = 1,
+  Deprecated = 2,
+};
+
+/// Add support for JSON serialization.
+llvm::json::Value toJSON(DiagnosticTag tag);
+bool fromJSON(const llvm::json::Value &value, DiagnosticTag &result,
+              llvm::json::Path path);
+
 struct Diagnostic {
   /// The source range where the message applies.
   Range range;
@@ -696,6 +706,9 @@ struct Diagnostic {
   /// a scope collide all definitions can be marked via this property.
   std::optional<std::vector<DiagnosticRelatedInformation>> relatedInformation;
 
+  /// Additional metadata about the diagnostic.
+  std::vector<DiagnosticTag> tags;
+
   /// The diagnostic's category. Can be omitted.
   /// An LSP extension that's used to send the name of the category over to the
   /// client. The category typically describes the compilation stage during
diff --git a/mlir/lib/Tools/lsp-server-support/Protocol.cpp b/mlir/lib/Tools/lsp-server-support/Protocol.cpp
index e110fdd97a38..188f5253c95c 100644
--- a/mlir/lib/Tools/lsp-server-support/Protocol.cpp
+++ b/mlir/lib/Tools/lsp-server-support/Protocol.cpp
@@ -646,6 +646,20 @@ llvm::json::Value mlir::lsp::toJSON(const DiagnosticRelatedInformation &info) {
 // Diagnostic
 //===----------------------------------------------------------------------===//
 
+llvm::json::Value mlir::lsp::toJSON(DiagnosticTag tag) {
+  return static_cast<int>(tag);
+}
+
+bool mlir::lsp::fromJSON(const llvm::json::Value &value, DiagnosticTag &result,
+                         llvm::json::Path path) {
+  if (std::optional<int64_t> i = value.getAsInteger()) {
+    result = (DiagnosticTag)*i;
+    return true;
+  }
+
+  return false;
+}
+
 llvm::json::Value mlir::lsp::toJSON(const Diagnostic &diag) {
   llvm::json::Object result{
       {"range", diag.range},
@@ -658,6 +672,8 @@ llvm::json::Value mlir::lsp::toJSON(const Diagnostic &diag) {
     result["source"] = diag.source;
   if (diag.relatedInformation)
     result["relatedInformation"] = *diag.relatedInformation;
+  if (!diag.tags.empty())
+    result["tags"] = diag.tags;
   return std::move(result);
 }
 
@@ -675,7 +691,8 @@ bool mlir::lsp::fromJSON(const llvm::json::Value &value, Diagnostic &result,
          mapOptOrNull(value, "category", result.category, path) &&
          mapOptOrNull(value, "source", result.source, path) &&
          mapOptOrNull(value, "relatedInformation", result.relatedInformation,
-                      path);
+                      path) &&
+         mapOptOrNull(value, "tags", result.tags, path);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/unittests/Tools/lsp-server-support/CMakeLists.txt b/mlir/unittests/Tools/lsp-server-support/CMakeLists.txt
index 3aa8b9c4bc77..f777873ff7c6 100644
--- a/mlir/unittests/Tools/lsp-server-support/CMakeLists.txt
+++ b/mlir/unittests/Tools/lsp-server-support/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_mlir_unittest(MLIRLspServerSupportTests
+  Protocol.cpp
   Transport.cpp
 )
 target_link_libraries(MLIRLspServerSupportTests
diff --git a/mlir/unittests/Tools/lsp-server-support/Protocol.cpp b/mlir/unittests/Tools/lsp-server-support/Protocol.cpp
new file mode 100644
index 000000000000..04d7b2fbb440
--- /dev/null
+++ b/mlir/unittests/Tools/lsp-server-support/Protocol.cpp
@@ -0,0 +1,51 @@
+//===- Protocol.cpp - LSP JSON protocol unit tests ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Tools/lsp-server-support/Protocol.h"
+
+#include "gtest/gtest.h"
+
+using namespace mlir;
+using namespace mlir::lsp;
+using namespace testing;
+
+namespace {
+
+TEST(ProtocolTest, DiagnosticTagPresent) {
+  Diagnostic diagnostic;
+  diagnostic.tags.push_back(DiagnosticTag::Unnecessary);
+
+  llvm::json::Value json = toJSON(diagnostic);
+  const llvm::json::Object *o = json.getAsObject();
+  const llvm::json::Array *v = o->get("tags")->getAsArray();
+  EXPECT_EQ(*v, llvm::json::Array{1});
+
+  Diagnostic parsed;
+  llvm::json::Path::Root root = llvm::json::Path::Root();
+  bool success = fromJSON(json, parsed, llvm::json::Path(root));
+  EXPECT_TRUE(success);
+  ASSERT_EQ(parsed.tags.size(), (size_t)1);
+  EXPECT_EQ(parsed.tags.at(0), DiagnosticTag::Unnecessary);
+}
+
+TEST(ProtocolTest, DiagnosticTagNotPresent) {
+  Diagnostic diagnostic;
+
+  llvm::json::Value json = toJSON(diagnostic);
+  const llvm::json::Object *o = json.getAsObject();
+  const llvm::json::Value *v = o->get("tags");
+  EXPECT_EQ(v, nullptr);
+
+  Diagnostic parsed;
+  llvm::json::Path::Root root = llvm::json::Path::Root();
+  bool success = fromJSON(json, parsed, llvm::json::Path(root));
+  EXPECT_TRUE(success);
+  EXPECT_TRUE(parsed.tags.empty());
+}
+
+} // namespace
-- 
GitLab


From 1464aee3767bf516633ce595ccd89a9cb50ae763 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 8 May 2024 20:16:44 +0100
Subject: [PATCH 0214/1206] [LAA] Add tests with non-constant backward deps
 with known min value.

Add a set of tests with non-constant backward dependences, where the
minimum value is known (via the start value of the outer AddRec).
---
 .../non-constant-distance-backward.ll         | 258 ++++++++++++++++++
 1 file changed, 258 insertions(+)
 create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/non-constant-distance-backward.ll

diff --git a/llvm/test/Analysis/LoopAccessAnalysis/non-constant-distance-backward.ll b/llvm/test/Analysis/LoopAccessAnalysis/non-constant-distance-backward.ll
new file mode 100644
index 000000000000..5a95dcca1050
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/non-constant-distance-backward.ll
@@ -0,0 +1,258 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -passes='print<access-info>' -disable-output -mtriple=arm64-apple-macosx %s 2>&1 | FileCheck %s
+; RUN: opt -passes='print<access-info>' -disable-output -mtriple=arm64-apple-macosx -mattr=+sve %s 2>&1 | FileCheck %s
+
+; REQUIRES: aarch64-registered-target
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+define void @backward_min_distance_8(ptr %A, i64 %N) {
+; CHECK-LABEL: 'backward_min_distance_8'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP1:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+; CHECK-NEXT:        Against group ([[GRP2:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP1]]:
+; CHECK-NEXT:          (Low: {(1 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(257 + %A),+,1}<nw><%outer.header>)
+; CHECK-NEXT:            Member: {{\{\{}}(1 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP2]]:
+; CHECK-NEXT:          (Low: %A High: (256 + %A))
+; CHECK-NEXT:            Member: {%A,+,1}<nuw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+; CHECK-NEXT:    outer.header:
+; CHECK-NEXT:      Report: loop is not the innermost loop
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %outer.iv = phi i64 [ 1, %entry ], [ %outer.iv.next, %outer.latch ]
+  %gep.off = getelementptr inbounds i8, ptr %A, i64 %outer.iv
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %outer.header ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+  %l = load i8 , ptr %gep, align 4
+  %add = add nsw i8 %l, 5
+  %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+  store i8 %add, ptr %gep.off.iv, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 256
+  br i1 %exitcond.not, label %outer.latch, label %loop
+
+outer.latch:
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %ec.2 = icmp eq i64 %outer.iv.next, %N
+  br i1 %ec.2, label %exit, label %outer.header
+
+exit:
+  ret void
+}
+
+define void @backward_min_distance_120(ptr %A, i64 %N) {
+; CHECK-LABEL: 'backward_min_distance_120'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP3:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+; CHECK-NEXT:        Against group ([[GRP4:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP3]]:
+; CHECK-NEXT:          (Low: {(15 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(271 + %A),+,1}<nw><%outer.header>)
+; CHECK-NEXT:            Member: {{\{\{}}(15 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP4]]:
+; CHECK-NEXT:          (Low: %A High: (256 + %A))
+; CHECK-NEXT:            Member: {%A,+,1}<nuw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+; CHECK-NEXT:    outer.header:
+; CHECK-NEXT:      Report: loop is not the innermost loop
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %outer.iv = phi i64 [ 15, %entry ], [ %outer.iv.next, %outer.latch ]
+  %gep.off = getelementptr inbounds i8, ptr %A, i64 %outer.iv
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %outer.header ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+  %l = load i8 , ptr %gep, align 4
+  %add = add nsw i8 %l, 5
+  %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+  store i8 %add, ptr %gep.off.iv, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 256
+  br i1 %exitcond.not, label %outer.latch, label %loop
+
+outer.latch:
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %ec.2 = icmp eq i64 %outer.iv.next, %N
+  br i1 %ec.2, label %exit, label %outer.header
+
+exit:
+  ret void
+}
+
+
+declare void @llvm.assume(i1)
+define void @backward_min_distance_128(ptr %A, i64 %N) {
+; CHECK-LABEL: 'backward_min_distance_128'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP5:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+; CHECK-NEXT:        Against group ([[GRP6:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP5]]:
+; CHECK-NEXT:          (Low: {(16 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(272 + %A),+,1}<nw><%outer.header>)
+; CHECK-NEXT:            Member: {{\{\{}}(16 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP6]]:
+; CHECK-NEXT:          (Low: %A High: (256 + %A))
+; CHECK-NEXT:            Member: {%A,+,1}<nuw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+; CHECK-NEXT:    outer.header:
+; CHECK-NEXT:      Report: loop is not the innermost loop
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %outer.iv = phi i64 [ 16, %entry ], [ %outer.iv.next, %outer.latch ]
+  %gep.off = getelementptr inbounds i8, ptr %A, i64 %outer.iv
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %outer.header ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+  %l = load i8 , ptr %gep, align 4
+  %add = add nsw i8 %l, 5
+  %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+  store i8 %add, ptr %gep.off.iv, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 256
+  br i1 %exitcond.not, label %outer.latch, label %loop
+
+outer.latch:
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %ec.2 = icmp eq i64 %outer.iv.next, %N
+  br i1 %ec.2, label %exit, label %outer.header
+
+exit:
+  ret void
+}
+
+define void @backward_min_distance_256(ptr %A, i64 %N) {
+; CHECK-LABEL: 'backward_min_distance_256'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP7:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+; CHECK-NEXT:        Against group ([[GRP8:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP7]]:
+; CHECK-NEXT:          (Low: {(32 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(288 + %A),+,1}<nw><%outer.header>)
+; CHECK-NEXT:            Member: {{\{\{}}(32 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP8]]:
+; CHECK-NEXT:          (Low: %A High: (256 + %A))
+; CHECK-NEXT:            Member: {%A,+,1}<nuw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+; CHECK-NEXT:    outer.header:
+; CHECK-NEXT:      Report: loop is not the innermost loop
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %outer.iv = phi i64 [ 32, %entry ], [ %outer.iv.next, %outer.latch ]
+  %gep.off = getelementptr inbounds i8, ptr %A, i64 %outer.iv
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %outer.header ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+  %l = load i8 , ptr %gep, align 4
+  %add = add nsw i8 %l, 5
+  %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+  store i8 %add, ptr %gep.off.iv, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 256
+  br i1 %exitcond.not, label %outer.latch, label %loop
+
+outer.latch:
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %ec.2 = icmp eq i64 %outer.iv.next, %N
+  br i1 %ec.2, label %exit, label %outer.header
+
+exit:
+  ret void
+}
-- 
GitLab


From 63c38ba64ebe079439e29acf43f24c33ecf44f4c Mon Sep 17 00:00:00 2001
From: Mariusz Borsa <wrotki@msn.com>
Date: Wed, 8 May 2024 12:23:25 -0700
Subject: [PATCH 0215/1206] [Sanitizers] Fix fake_test_gc not working on
 devices (#91284)

The way the LIT RUN command is currently constructed ( %run not --crash
%t ) causes the test failure on devices - since 'not' is a LLVM built
command, not available on devices.

Changing the command to read 'not --crash %run %t' fixes it, as 'not'
now executes on the host running the test.

rdar://115914588

Co-authored-by: Mariusz Borsa <m_borsa@apple.com>
---
 compiler-rt/test/asan/TestCases/Posix/fake_stack_gc.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/test/asan/TestCases/Posix/fake_stack_gc.cpp b/compiler-rt/test/asan/TestCases/Posix/fake_stack_gc.cpp
index 524ca29f2fc5..8c368b9b1b94 100644
--- a/compiler-rt/test/asan/TestCases/Posix/fake_stack_gc.cpp
+++ b/compiler-rt/test/asan/TestCases/Posix/fake_stack_gc.cpp
@@ -1,4 +1,4 @@
-// RUN: %clangxx_asan -O0 -pthread %s -o %t && %env_asan_opts=use_sigaltstack=0 %run not --crash %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan -O0 -pthread %s -o %t && %env_asan_opts=use_sigaltstack=0 not --crash %run %t 2>&1 | FileCheck %s
 
 // Check that fake stack does not discard frames on the main stack, when GC is
 // triggered from high alt stack.
-- 
GitLab


From 9047331f1b4a623332966d888f05bcd3381c8abe Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Wed, 8 May 2024 20:34:37 +0100
Subject: [PATCH 0216/1206] [AArch64] Add some additional add mul imm tests
 with multiple uses. NFC

---
 llvm/test/CodeGen/AArch64/addimm-mulimm.ll | 299 +++++++++++++++++++--
 1 file changed, 279 insertions(+), 20 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/addimm-mulimm.ll b/llvm/test/CodeGen/AArch64/addimm-mulimm.ll
index cc6523d1bb1d..3618b14aa921 100644
--- a/llvm/test/CodeGen/AArch64/addimm-mulimm.ll
+++ b/llvm/test/CodeGen/AArch64/addimm-mulimm.ll
@@ -4,8 +4,8 @@
 define i64 @addimm_mulimm_accept_00(i64 %a) {
 ; CHECK-LABEL: addimm_mulimm_accept_00:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #37
-; CHECK-NEXT:    mov x9, #1147
+; CHECK-NEXT:    mov w8, #37 // =0x25
+; CHECK-NEXT:    mov x9, #1147 // =0x47b
 ; CHECK-NEXT:    madd x0, x0, x8, x9
 ; CHECK-NEXT:    ret
   %tmp0 = add i64 %a, 31
@@ -16,8 +16,8 @@ define i64 @addimm_mulimm_accept_00(i64 %a) {
 define i64 @addimm_mulimm_accept_01(i64 %a) {
 ; CHECK-LABEL: addimm_mulimm_accept_01:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #37
-; CHECK-NEXT:    mov x9, #-1147
+; CHECK-NEXT:    mov w8, #37 // =0x25
+; CHECK-NEXT:    mov x9, #-1147 // =0xfffffffffffffb85
 ; CHECK-NEXT:    madd x0, x0, x8, x9
 ; CHECK-NEXT:    ret
   %tmp0 = add i64 %a, -31
@@ -28,8 +28,8 @@ define i64 @addimm_mulimm_accept_01(i64 %a) {
 define signext i32 @addimm_mulimm_accept_02(i32 signext %a) {
 ; CHECK-LABEL: addimm_mulimm_accept_02:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #37
-; CHECK-NEXT:    mov w9, #1147
+; CHECK-NEXT:    mov w8, #37 // =0x25
+; CHECK-NEXT:    mov w9, #1147 // =0x47b
 ; CHECK-NEXT:    madd w0, w0, w8, w9
 ; CHECK-NEXT:    ret
   %tmp0 = add i32 %a, 31
@@ -40,8 +40,8 @@ define signext i32 @addimm_mulimm_accept_02(i32 signext %a) {
 define signext i32 @addimm_mulimm_accept_03(i32 signext %a) {
 ; CHECK-LABEL: addimm_mulimm_accept_03:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #37
-; CHECK-NEXT:    mov w9, #-1147
+; CHECK-NEXT:    mov w8, #37 // =0x25
+; CHECK-NEXT:    mov w9, #-1147 // =0xfffffb85
 ; CHECK-NEXT:    madd w0, w0, w8, w9
 ; CHECK-NEXT:    ret
   %tmp0 = add i32 %a, -31
@@ -52,8 +52,8 @@ define signext i32 @addimm_mulimm_accept_03(i32 signext %a) {
 define i64 @addimm_mulimm_accept_10(i64 %a) {
 ; CHECK-LABEL: addimm_mulimm_accept_10:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #37
-; CHECK-NEXT:    mov w9, #32888
+; CHECK-NEXT:    mov w8, #37 // =0x25
+; CHECK-NEXT:    mov w9, #32888 // =0x8078
 ; CHECK-NEXT:    movk w9, #17, lsl #16
 ; CHECK-NEXT:    madd x0, x0, x8, x9
 ; CHECK-NEXT:    ret
@@ -65,8 +65,8 @@ define i64 @addimm_mulimm_accept_10(i64 %a) {
 define i64 @addimm_mulimm_accept_11(i64 %a) {
 ; CHECK-LABEL: addimm_mulimm_accept_11:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #37
-; CHECK-NEXT:    mov x9, #-32888
+; CHECK-NEXT:    mov w8, #37 // =0x25
+; CHECK-NEXT:    mov x9, #-32888 // =0xffffffffffff7f88
 ; CHECK-NEXT:    movk x9, #65518, lsl #16
 ; CHECK-NEXT:    madd x0, x0, x8, x9
 ; CHECK-NEXT:    ret
@@ -78,8 +78,8 @@ define i64 @addimm_mulimm_accept_11(i64 %a) {
 define signext i32 @addimm_mulimm_accept_12(i32 signext %a) {
 ; CHECK-LABEL: addimm_mulimm_accept_12:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #37
-; CHECK-NEXT:    mov w9, #32888
+; CHECK-NEXT:    mov w8, #37 // =0x25
+; CHECK-NEXT:    mov w9, #32888 // =0x8078
 ; CHECK-NEXT:    movk w9, #17, lsl #16
 ; CHECK-NEXT:    madd w0, w0, w8, w9
 ; CHECK-NEXT:    ret
@@ -91,8 +91,8 @@ define signext i32 @addimm_mulimm_accept_12(i32 signext %a) {
 define signext i32 @addimm_mulimm_accept_13(i32 signext %a) {
 ; CHECK-LABEL: addimm_mulimm_accept_13:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #37
-; CHECK-NEXT:    mov w9, #32648
+; CHECK-NEXT:    mov w8, #37 // =0x25
+; CHECK-NEXT:    mov w9, #32648 // =0x7f88
 ; CHECK-NEXT:    movk w9, #65518, lsl #16
 ; CHECK-NEXT:    madd w0, w0, w8, w9
 ; CHECK-NEXT:    ret
@@ -104,7 +104,7 @@ define signext i32 @addimm_mulimm_accept_13(i32 signext %a) {
 define i64 @addimm_mulimm_reject_00(i64 %a) {
 ; CHECK-LABEL: addimm_mulimm_reject_00:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3700
+; CHECK-NEXT:    mov w8, #3700 // =0xe74
 ; CHECK-NEXT:    add x9, x0, #3100
 ; CHECK-NEXT:    mul x0, x9, x8
 ; CHECK-NEXT:    ret
@@ -116,7 +116,7 @@ define i64 @addimm_mulimm_reject_00(i64 %a) {
 define i64 @addimm_mulimm_reject_01(i64 %a) {
 ; CHECK-LABEL: addimm_mulimm_reject_01:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3700
+; CHECK-NEXT:    mov w8, #3700 // =0xe74
 ; CHECK-NEXT:    sub x9, x0, #3100
 ; CHECK-NEXT:    mul x0, x9, x8
 ; CHECK-NEXT:    ret
@@ -128,7 +128,7 @@ define i64 @addimm_mulimm_reject_01(i64 %a) {
 define signext i32 @addimm_mulimm_reject_02(i32 signext %a) {
 ; CHECK-LABEL: addimm_mulimm_reject_02:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3700
+; CHECK-NEXT:    mov w8, #3700 // =0xe74
 ; CHECK-NEXT:    add w9, w0, #3100
 ; CHECK-NEXT:    mul w0, w9, w8
 ; CHECK-NEXT:    ret
@@ -140,7 +140,7 @@ define signext i32 @addimm_mulimm_reject_02(i32 signext %a) {
 define signext i32 @addimm_mulimm_reject_03(i32 signext %a) {
 ; CHECK-LABEL: addimm_mulimm_reject_03:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3700
+; CHECK-NEXT:    mov w8, #3700 // =0xe74
 ; CHECK-NEXT:    sub w9, w0, #3100
 ; CHECK-NEXT:    mul w0, w9, w8
 ; CHECK-NEXT:    ret
@@ -148,3 +148,262 @@ define signext i32 @addimm_mulimm_reject_03(i32 signext %a) {
   %tmp1 = mul i32 %tmp0, 3700
   ret i32 %tmp1
 }
+
+define signext i32 @addmuladd(i32 signext %a) {
+; CHECK-LABEL: addmuladd:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    mov w9, #1300 // =0x514
+; CHECK-NEXT:    madd w0, w0, w8, w9
+; CHECK-NEXT:    ret
+  %tmp0 = add i32 %a, 4
+  %tmp1 = mul i32 %tmp0, 324
+  %tmp2 = add i32 %tmp1, 4
+  ret i32 %tmp2
+}
+
+define signext i32 @addmuladd_multiuse(i32 signext %a) {
+; CHECK-LABEL: addmuladd_multiuse:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    add w9, w0, #4
+; CHECK-NEXT:    mov w10, #4 // =0x4
+; CHECK-NEXT:    madd w8, w9, w8, w10
+; CHECK-NEXT:    eor w0, w9, w8
+; CHECK-NEXT:    ret
+  %tmp0 = add i32 %a, 4
+  %tmp1 = mul i32 %tmp0, 324
+  %tmp2 = add i32 %tmp1, 4
+  %tmp3 = xor i32 %tmp0, %tmp2
+  ret i32 %tmp3
+}
+
+define signext i32 @addmuladd_multiusemul(i32 signext %a) {
+; CHECK-LABEL: addmuladd_multiusemul:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    mul w8, w0, w8
+; CHECK-NEXT:    add w9, w8, #1296
+; CHECK-NEXT:    add w8, w8, #1300
+; CHECK-NEXT:    eor w0, w9, w8
+; CHECK-NEXT:    ret
+  %tmp0 = add i32 %a, 4
+  %tmp1 = mul i32 %tmp0, 324
+  %tmp2 = add i32 %tmp1, 4
+  %tmp3 = xor i32 %tmp1, %tmp2
+  ret i32 %tmp3
+}
+
+define signext i32 @addmuladd_multiuse2(i32 signext %a) {
+; CHECK-LABEL: addmuladd_multiuse2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    add w9, w0, #4
+; CHECK-NEXT:    mov w11, #4 // =0x4
+; CHECK-NEXT:    lsl w10, w9, #2
+; CHECK-NEXT:    madd w8, w9, w8, w11
+; CHECK-NEXT:    add w9, w10, #4
+; CHECK-NEXT:    eor w0, w8, w9
+; CHECK-NEXT:    ret
+  %tmp0 = add i32 %a, 4
+  %tmp1 = mul i32 %tmp0, 4
+  %tmp2 = add i32 %tmp1, 4
+  %tmp3 = mul i32 %tmp0, 324
+  %tmp4 = add i32 %tmp3, 4
+  %tmp5 = xor i32 %tmp4, %tmp2
+  ret i32 %tmp5
+}
+
+define signext i32 @addaddmuladd(i32 signext %a, i32 %b) {
+; CHECK-LABEL: addaddmuladd:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    madd w8, w0, w8, w1
+; CHECK-NEXT:    add w0, w8, #1300
+; CHECK-NEXT:    ret
+  %tmp0 = add i32 %a, 4
+  %tmp1 = mul i32 %tmp0, 324
+  %tmp2 = add i32 %tmp1, %b
+  %tmp3 = add i32 %tmp2, 4
+  ret i32 %tmp3
+}
+
+define signext i32 @addaddmuladd_multiuse(i32 signext %a, i32 %b) {
+; CHECK-LABEL: addaddmuladd_multiuse:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    add w9, w0, #4
+; CHECK-NEXT:    madd w8, w9, w8, w1
+; CHECK-NEXT:    add w8, w8, #4
+; CHECK-NEXT:    eor w0, w9, w8
+; CHECK-NEXT:    ret
+  %tmp0 = add i32 %a, 4
+  %tmp1 = mul i32 %tmp0, 324
+  %tmp2 = add i32 %tmp1, %b
+  %tmp3 = add i32 %tmp2, 4
+  %tmp4 = xor i32 %tmp0, %tmp3
+  ret i32 %tmp4
+}
+
+define signext i32 @addaddmuladd_multiuse2(i32 signext %a, i32 %b) {
+; CHECK-LABEL: addaddmuladd_multiuse2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    add w9, w0, #4
+; CHECK-NEXT:    mov w10, #162 // =0xa2
+; CHECK-NEXT:    madd w8, w9, w8, w1
+; CHECK-NEXT:    madd w9, w9, w10, w1
+; CHECK-NEXT:    add w8, w8, #4
+; CHECK-NEXT:    add w9, w9, #4
+; CHECK-NEXT:    eor w0, w9, w8
+; CHECK-NEXT:    ret
+  %tmp0 = add i32 %a, 4
+  %tmp1 = mul i32 %tmp0, 324
+  %tmp2 = add i32 %tmp1, %b
+  %tmp3 = add i32 %tmp2, 4
+  %tmp1b = mul i32 %tmp0, 162
+  %tmp2b = add i32 %tmp1b, %b
+  %tmp3b = add i32 %tmp2b, 4
+  %tmp4 = xor i32 %tmp3b, %tmp3
+  ret i32 %tmp4
+}
+
+define <4 x i32> @addmuladd_vec(<4 x i32> %a) {
+; CHECK-LABEL: addmuladd_vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    mov w9, #1300 // =0x514
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    dup v1.4s, w9
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %tmp0 = add <4 x i32> %a, <i32 4, i32 4, i32 4, i32 4>
+  %tmp1 = mul <4 x i32> %tmp0, <i32 324, i32 324, i32 324, i32 324>
+  %tmp2 = add <4 x i32> %tmp1, <i32 4, i32 4, i32 4, i32 4>
+  ret <4 x i32> %tmp2
+}
+
+define <4 x i32> @addmuladd_vec_multiuse(<4 x i32> %a) {
+; CHECK-LABEL: addmuladd_vec_multiuse:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.4s, #4
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    dup v2.4s, w8
+; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mla v1.4s, v0.4s, v2.4s
+; CHECK-NEXT:    eor v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %tmp0 = add <4 x i32> %a, <i32 4, i32 4, i32 4, i32 4>
+  %tmp1 = mul <4 x i32> %tmp0, <i32 324, i32 324, i32 324, i32 324>
+  %tmp2 = add <4 x i32> %tmp1, <i32 4, i32 4, i32 4, i32 4>
+  %tmp3 = xor <4 x i32> %tmp0, %tmp2
+  ret <4 x i32> %tmp3
+}
+
+define void @addmuladd_gep(ptr %p, i64 %a) {
+; CHECK-LABEL: addmuladd_gep:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #40 // =0x28
+; CHECK-NEXT:    str wzr, [x0, #10]!
+; CHECK-NEXT:    madd x8, x1, x8, x0
+; CHECK-NEXT:    str wzr, [x8, #20]
+; CHECK-NEXT:    ret
+  %q = getelementptr i8, ptr %p, i64 10
+  %r = getelementptr [10 x [10 x i32]], ptr %q, i64 0, i64 %a, i64 5
+  store i32 0, ptr %q
+  store i32 0, ptr %r
+  ret void
+}
+
+define i32 @addmuladd_gep2(ptr %p, i32 %a) {
+; CHECK-LABEL: addmuladd_gep2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    sxtw x8, w1
+; CHECK-NEXT:    mov w9, #3240 // =0xca8
+; CHECK-NEXT:    add x8, x8, #1
+; CHECK-NEXT:    madd x9, x8, x9, x0
+; CHECK-NEXT:    ldr w9, [x9, #20]
+; CHECK-NEXT:    tbnz w9, #31, .LBB22_2
+; CHECK-NEXT:  // %bb.1:
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB22_2: // %then
+; CHECK-NEXT:    str x8, [x0]
+; CHECK-NEXT:    mov w0, #1 // =0x1
+; CHECK-NEXT:    ret
+  %b = sext i32 %a to i64
+  %c = add nsw i64 %b, 1
+  %d = mul nsw i64 %c, 81
+  %g = getelementptr [10 x [10 x i32]], ptr %p, i64 0, i64 %d, i64 5
+  %l = load i32, ptr %g, align 4
+  %cc = icmp slt i32 %l, 0
+  br i1 %cc, label %then, label %else
+then:
+  store i64 %c, ptr %p
+  ret i32 1
+else:
+  ret i32 0
+}
+
+define signext i32 @addmuladd_multiuse2_nsw(i32 signext %a) {
+; CHECK-LABEL: addmuladd_multiuse2_nsw:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    add w9, w0, #4
+; CHECK-NEXT:    mov w11, #4 // =0x4
+; CHECK-NEXT:    lsl w10, w9, #2
+; CHECK-NEXT:    madd w8, w9, w8, w11
+; CHECK-NEXT:    add w9, w10, #4
+; CHECK-NEXT:    eor w0, w8, w9
+; CHECK-NEXT:    ret
+  %tmp0 = add nsw i32 %a, 4
+  %tmp1 = mul nsw i32 %tmp0, 4
+  %tmp2 = add nsw i32 %tmp1, 4
+  %tmp3 = mul nsw i32 %tmp0, 324
+  %tmp4 = add nsw i32 %tmp3, 4
+  %tmp5 = xor i32 %tmp4, %tmp2
+  ret i32 %tmp5
+}
+
+define signext i32 @addmuladd_multiuse2_nuw(i32 signext %a) {
+; CHECK-LABEL: addmuladd_multiuse2_nuw:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    add w9, w0, #4
+; CHECK-NEXT:    mov w11, #4 // =0x4
+; CHECK-NEXT:    lsl w10, w9, #2
+; CHECK-NEXT:    madd w8, w9, w8, w11
+; CHECK-NEXT:    add w9, w10, #4
+; CHECK-NEXT:    eor w0, w8, w9
+; CHECK-NEXT:    ret
+  %tmp0 = add nuw i32 %a, 4
+  %tmp1 = mul nuw i32 %tmp0, 4
+  %tmp2 = add nuw i32 %tmp1, 4
+  %tmp3 = mul nuw i32 %tmp0, 324
+  %tmp4 = add nuw i32 %tmp3, 4
+  %tmp5 = xor i32 %tmp4, %tmp2
+  ret i32 %tmp5
+}
+
+define signext i32 @addmuladd_multiuse2_nswnuw(i32 signext %a) {
+; CHECK-LABEL: addmuladd_multiuse2_nswnuw:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    add w9, w0, #4
+; CHECK-NEXT:    mov w11, #4 // =0x4
+; CHECK-NEXT:    lsl w10, w9, #2
+; CHECK-NEXT:    madd w8, w9, w8, w11
+; CHECK-NEXT:    add w9, w10, #4
+; CHECK-NEXT:    eor w0, w8, w9
+; CHECK-NEXT:    ret
+  %tmp0 = add nsw nuw i32 %a, 4
+  %tmp1 = mul nsw nuw i32 %tmp0, 4
+  %tmp2 = add nsw nuw i32 %tmp1, 4
+  %tmp3 = mul nsw nuw i32 %tmp0, 324
+  %tmp4 = add nsw nuw i32 %tmp3, 4
+  %tmp5 = xor i32 %tmp4, %tmp2
+  ret i32 %tmp5
+}
+
-- 
GitLab


From 5526c8a7425350cff2cd9cafa1bf5f20753e7848 Mon Sep 17 00:00:00 2001
From: Thomas Raoux <thomas.raoux@openai.com>
Date: Wed, 8 May 2024 12:35:15 -0700
Subject: [PATCH 0217/1206] [MLIR] Model llvm.inline_asm side_effects (#91507)

Allow more cleanups on inline_asm ops modeling side effects based on the
side_effect attributed.
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td |  2 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp  | 13 +++++++++++++
 mlir/test/Dialect/LLVMIR/canonicalize.mlir  | 11 +++++++++++
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 6655ce6f123e..4b91708ea1aa 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -1795,7 +1795,7 @@ def LLVM_FenceOp : LLVM_Op<"fence">, LLVM_MemOpPatterns {
   let hasVerifier = 1;
 }
 
-def LLVM_InlineAsmOp : LLVM_Op<"inline_asm", []> {
+def LLVM_InlineAsmOp : LLVM_Op<"inline_asm", [DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
   let description = [{
     The InlineAsmOp mirrors the underlying LLVM semantics with a notable
     exception: the embedded `asm_string` is not allowed to define or reference
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 7be493d5992c..7d33d05feb65 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3034,6 +3034,19 @@ LogicalResult LinkerOptionsOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// InlineAsmOp
+//===----------------------------------------------------------------------===//
+
+void InlineAsmOp::getEffects(
+    SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
+        &effects) {
+  if (getHasSideEffects()) {
+    effects.emplace_back(MemoryEffects::Write::get());
+    effects.emplace_back(MemoryEffects::Read::get());
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // LLVMDialect initialization, type parsing, and registration.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/LLVMIR/canonicalize.mlir b/mlir/test/Dialect/LLVMIR/canonicalize.mlir
index 6b265bbbdbfb..15f960167cb5 100644
--- a/mlir/test/Dialect/LLVMIR/canonicalize.mlir
+++ b/mlir/test/Dialect/LLVMIR/canonicalize.mlir
@@ -248,3 +248,14 @@ llvm.func @volatile_load(%x : !llvm.ptr) {
   %3 = llvm.load %x  atomic unordered { alignment = 1 } : !llvm.ptr -> i8
   llvm.return
 }
+
+// -----
+
+// CHECK-LABEL: func @inline_asm_side_effects
+llvm.func @inline_asm_side_effects(%x : i32) {
+  // CHECK-NOT: llvm.inline_asm "pure inline asm"
+  llvm.inline_asm "pure inline asm", "r" %x : (i32) -> ()
+  // CHECK: llvm.inline_asm has_side_effects "inline asm with side effects"
+  llvm.inline_asm has_side_effects "inline asm with side effects", "r" %x : (i32) -> ()
+  llvm.return
+}
-- 
GitLab


From e3938f4d71493673033f6190454e7e19d5411ea7 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Wed, 8 May 2024 14:36:58 -0500
Subject: [PATCH 0218/1206] [Offload] Detect native ELF machine from
 preprocessor (#91282)

Summary:
This gets the target's corresponding ELF value from the preprocessor.
We use this to detect if a given ELF is compatible with the CPU
offloading impolementation for OpenMP. Previously we used defitions from
CMake, but this is easier for people to understand as there may be new
users of this in the future.
---
 .../plugins-nextgen/common/include/Utils/ELF.h    |  3 +++
 offload/plugins-nextgen/common/src/Utils/ELF.cpp  | 15 +++++++++++++++
 offload/plugins-nextgen/host/CMakeLists.txt       |  5 -----
 offload/plugins-nextgen/host/src/rtl.cpp          | 10 ++++------
 4 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/offload/plugins-nextgen/common/include/Utils/ELF.h b/offload/plugins-nextgen/common/include/Utils/ELF.h
index 88c83d39b68c..f87e0a5ed02b 100644
--- a/offload/plugins-nextgen/common/include/Utils/ELF.h
+++ b/offload/plugins-nextgen/common/include/Utils/ELF.h
@@ -24,6 +24,9 @@ namespace elf {
 /// Returns true or false if the \p Buffer is an ELF file.
 bool isELF(llvm::StringRef Buffer);
 
+/// Returns the ELF e_machine value of the current compilation target.
+uint16_t getTargetMachine();
+
 /// Checks if the given \p Object is a valid ELF matching the e_machine value.
 llvm::Expected<bool> checkMachine(llvm::StringRef Object, uint16_t EMachine);
 
diff --git a/offload/plugins-nextgen/common/src/Utils/ELF.cpp b/offload/plugins-nextgen/common/src/Utils/ELF.cpp
index 2ae97f0f2589..90d6950b83e5 100644
--- a/offload/plugins-nextgen/common/src/Utils/ELF.cpp
+++ b/offload/plugins-nextgen/common/src/Utils/ELF.cpp
@@ -36,6 +36,21 @@ bool utils::elf::isELF(StringRef Buffer) {
   }
 }
 
+uint16_t utils::elf::getTargetMachine() {
+#if defined(__x86_64__)
+  return EM_X86_64;
+#elif defined(__s390x__)
+  return EM_S390;
+#elif defined(__aarch64__)
+  return EM_AARCH64;
+#elif defined(__powerpc64__)
+  return EM_PPC64;
+#else
+#warning "Unknown ELF compilation target architecture"
+  return EM_NONE;
+#endif
+}
+
 template <class ELFT>
 static Expected<bool>
 checkMachineImpl(const object::ELFObjectFile<ELFT> &ELFObj, uint16_t EMachine) {
diff --git a/offload/plugins-nextgen/host/CMakeLists.txt b/offload/plugins-nextgen/host/CMakeLists.txt
index 48e591bc894e..1d000442c84d 100644
--- a/offload/plugins-nextgen/host/CMakeLists.txt
+++ b/offload/plugins-nextgen/host/CMakeLists.txt
@@ -52,27 +52,22 @@ endif()
 
 # Define the target specific triples and ELF machine values.
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$")
-  target_compile_definitions(omptarget.rtl.host PRIVATE TARGET_ELF_ID=EM_PPC64)
   list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
        "powerpc64le-ibm-linux-gnu" "powerpc64le-ibm-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64$")
-  target_compile_definitions(omptarget.rtl.host PRIVATE TARGET_ELF_ID=EM_PPC64)
   list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
        "powerpc64-ibm-linux-gnu" "powerpc64-ibm-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64$")
-  target_compile_definitions(omptarget.rtl.host PRIVATE TARGET_ELF_ID=EM_X86_64)
   list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
        "x86_64-pc-linux-gnu" "x86_64-pc-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64$")
-  target_compile_definitions(omptarget.rtl.host PRIVATE TARGET_ELF_ID=EM_AARCH64)
   list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
        "aarch64-unknown-linux-gnu" "aarch64-unknown-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x$")
-  target_compile_definitions(omptarget.rtl.host PRIVATE TARGET_ELF_ID=EM_S390)
   list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
        "s390x-ibm-linux-gnu" "s390x-ibm-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index c929db6c22d8..4bdcae3dd6a1 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -18,6 +18,7 @@
 
 #include "Shared/Debug.h"
 #include "Shared/Environment.h"
+#include "Utils/ELF.h"
 
 #include "GlobalHandler.h"
 #include "OpenMP/OMPT/Callback.h"
@@ -44,11 +45,6 @@
 // The number of devices in this plugin.
 #define NUM_DEVICES 4
 
-// The ELF ID should be defined at compile-time by the build system.
-#ifndef TARGET_ELF_ID
-#define TARGET_ELF_ID EM_NONE
-#endif
-
 namespace llvm {
 namespace omp {
 namespace target {
@@ -416,7 +412,9 @@ struct GenELF64PluginTy final : public GenericPluginTy {
   }
 
   /// Get the ELF code to recognize the compatible binary images.
-  uint16_t getMagicElfBits() const override { return ELF::TARGET_ELF_ID; }
+  uint16_t getMagicElfBits() const override {
+    return utils::elf::getTargetMachine();
+  }
 
   /// This plugin does not support exchanging data between two devices.
   bool isDataExchangable(int32_t SrcDeviceId, int32_t DstDeviceId) override {
-- 
GitLab


From 559accf365a6eb885c24cf15e14aea2eb8e66596 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Wed, 8 May 2024 12:53:31 -0700
Subject: [PATCH 0219/1206] [AMDGPU] Add VOP3_PACKED to V_PK_{MIN|MAX}IMUM_F16
 profile (#91512)

NFCI as far as I understand, added for consitency.
---
 llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index a7d63fdb2e04..71ce36647e45 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -110,8 +110,8 @@ defm V_PK_MAX_I16 : VOP3PInst<"v_pk_max_i16", VOP3P_Profile<VOP_V2I16_V2I16_V2I1
 defm V_PK_MAX_U16 : VOP3PInst<"v_pk_max_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, umax>;
 
 let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
-defm V_PK_MAXIMUM_F16 : VOP3PInst<"v_pk_maximum_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16>, fmaximum>;
-defm V_PK_MINIMUM_F16 : VOP3PInst<"v_pk_minimum_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16>, fminimum>;
+defm V_PK_MAXIMUM_F16 : VOP3PInst<"v_pk_maximum_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16, VOP3_PACKED>, fmaximum>;
+defm V_PK_MINIMUM_F16 : VOP3PInst<"v_pk_minimum_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16, VOP3_PACKED>, fminimum>;
 } // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
 }
 
-- 
GitLab


From b1da82ae3dba0982b3a9668ca895ddf4164fb3d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Wed, 8 May 2024 21:06:03 +0100
Subject: [PATCH 0220/1206] [mlir][arith] Fix overflow bug in
 arith::CeilDivSIOp::fold (#90947)

The folder for arith::CeilDivSIOp should only be applied when it can be
guaranteed that no overflow would happen. The current implementation
works fine when both dividends are positive and the only arithmetic
operation is the division itself.

However, in cases where either the dividend or divisor is negative (or
both),
the division is split into multiple arith operations, e.g.: `- ( -a /
b)`. That's
additional 2 operations on top of the actual division that can overflow
- the folder should check all 3 ops for overflow.

The current logic doesn't do that - it effectively only checks the last
operation
(i.e. the division). It breaks when using e.g. MININT values (e.g. -128
for
8-bit integers) - negating such values overflows.

This PR makes sure that no folding happens if any of the intermediate
arithmetic operations overflows.

Fixes https://github.com/llvm/llvm-project/issues/89382
---
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp  | 34 ++++++++++++++++------
 mlir/test/Transforms/constant-fold.mlir | 38 +++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 9 deletions(-)

diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 6f995b93bc3e..a1568d0ebba3 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -683,6 +683,8 @@ OpFoldResult arith::CeilDivSIOp::fold(FoldAdaptor adaptor) {
     return getLhs();
 
   // Don't fold if it would overflow or if it requires a division by zero.
+  // TODO: This hook won't fold operations where a = MININT, because
+  // negating MININT overflows. This can be improved.
   bool overflowOrDiv0 = false;
   auto result = constFoldBinaryOp<IntegerAttr>(
       adaptor.getOperands(), [&](APInt a, const APInt &b) {
@@ -701,22 +703,36 @@ OpFoldResult arith::CeilDivSIOp::fold(FoldAdaptor adaptor) {
           // Both positive, return ceil(a, b).
           return signedCeilNonnegInputs(a, b, overflowOrDiv0);
         }
+
+        // No folding happens if any of the intermediate arithmetic operations
+        // overflows.
+        bool overflowNegA = false;
+        bool overflowNegB = false;
+        bool overflowDiv = false;
+        bool overflowNegRes = false;
         if (!aGtZero && !bGtZero) {
           // Both negative, return ceil(-a, -b).
-          APInt posA = zero.ssub_ov(a, overflowOrDiv0);
-          APInt posB = zero.ssub_ov(b, overflowOrDiv0);
-          return signedCeilNonnegInputs(posA, posB, overflowOrDiv0);
+          APInt posA = zero.ssub_ov(a, overflowNegA);
+          APInt posB = zero.ssub_ov(b, overflowNegB);
+          APInt res = signedCeilNonnegInputs(posA, posB, overflowDiv);
+          overflowOrDiv0 = (overflowNegA || overflowNegB || overflowDiv);
+          return res;
         }
         if (!aGtZero && bGtZero) {
           // A is negative, b is positive, return - ( -a / b).
-          APInt posA = zero.ssub_ov(a, overflowOrDiv0);
-          APInt div = posA.sdiv_ov(b, overflowOrDiv0);
-          return zero.ssub_ov(div, overflowOrDiv0);
+          APInt posA = zero.ssub_ov(a, overflowNegA);
+          APInt div = posA.sdiv_ov(b, overflowDiv);
+          APInt res = zero.ssub_ov(div, overflowNegRes);
+          overflowOrDiv0 = (overflowNegA || overflowDiv || overflowNegRes);
+          return res;
         }
         // A is positive, b is negative, return - (a / -b).
-        APInt posB = zero.ssub_ov(b, overflowOrDiv0);
-        APInt div = a.sdiv_ov(posB, overflowOrDiv0);
-        return zero.ssub_ov(div, overflowOrDiv0);
+        APInt posB = zero.ssub_ov(b, overflowNegB);
+        APInt div = a.sdiv_ov(posB, overflowDiv);
+        APInt res = zero.ssub_ov(div, overflowNegRes);
+
+        overflowOrDiv0 = (overflowNegB || overflowDiv || overflowNegRes);
+        return res;
       });
 
   return overflowOrDiv0 ? Attribute() : result;
diff --git a/mlir/test/Transforms/constant-fold.mlir b/mlir/test/Transforms/constant-fold.mlir
index 253163f2af91..981757aed9b1 100644
--- a/mlir/test/Transforms/constant-fold.mlir
+++ b/mlir/test/Transforms/constant-fold.mlir
@@ -478,6 +478,44 @@ func.func @simple_arith.ceildivsi() -> (i32, i32, i32, i32, i32) {
 
 // -----
 
+// CHECK-LABEL: func @simple_arith.ceildivsi_overflow
+func.func @simple_arith.ceildivsi_overflow() -> (i8, i16, i32) {
+  // The negative values below are MININTs for the corresponding bit-width. The
+  // folder will try to negate them (so that the division operates on two
+  // positive numbers), but that would cause overflow (negating MININT
+  // overflows). Hence folding should not happen and the original ceildivsi is
+  // preserved.
+
+  // TODO: The folder should be able to fold the following by avoiding
+  // intermediate operations that overflow.
+
+  // CHECK-DAG: %[[C_1:.*]] = arith.constant 7 : i8
+  // CHECK-DAG: %[[MIN_I8:.*]] = arith.constant -128 : i8
+  // CHECK-DAG: %[[C_2:.*]] = arith.constant 7 : i16
+  // CHECK-DAG: %[[MIN_I16:.*]] = arith.constant -32768 : i16
+  // CHECK-DAG: %[[C_3:.*]] = arith.constant 7 : i32
+  // CHECK-DAG: %[[MIN_I32:.*]] = arith.constant -2147483648 : i32
+
+  // CHECK-NEXT: %[[CEILDIV_1:.*]] = arith.ceildivsi %[[MIN_I8]], %[[C_1]]  : i8
+  %0 = arith.constant 7 : i8
+  %min_int_i8 = arith.constant -128 : i8
+  %2 = arith.ceildivsi %min_int_i8, %0 : i8
+
+  // CHECK-NEXT: %[[CEILDIV_2:.*]] = arith.ceildivsi %[[MIN_I16]], %[[C_2]]  : i16
+  %3 = arith.constant 7 : i16
+  %min_int_i16 = arith.constant -32768 : i16
+  %5 = arith.ceildivsi %min_int_i16, %3 : i16
+
+  // CHECK-NEXT: %[[CEILDIV_2:.*]] = arith.ceildivsi %[[MIN_I32]], %[[C_3]]  : i32
+  %6 = arith.constant 7 : i32
+  %min_int_i32 = arith.constant -2147483648 : i32
+  %8 = arith.ceildivsi %min_int_i32, %6 : i32
+
+  return %2, %5, %8 : i8, i16, i32
+}
+
+// -----
+
 // CHECK-LABEL: func @simple_arith.ceildivui
 func.func @simple_arith.ceildivui() -> (i32, i32, i32, i32, i32) {
   // CHECK-DAG: [[C0:%.+]] = arith.constant 0
-- 
GitLab


From b52160dbae268cc87cb8f6cdf75553ca095e26a9 Mon Sep 17 00:00:00 2001
From: Dave Lee <davelee.com@gmail.com>
Date: Wed, 8 May 2024 13:07:07 -0700
Subject: [PATCH 0221/1206] [lldb] Consult Language plugin in
 GetDisplayDemangledName (#90294)

Give language plugins the opportunity to provide a language specific
display name.

This will be used in a follow up commit. The purpose of this change is
to ultimately display breakpoint locations with a more human friendly
demangling of Swift symbols.
---
 lldb/include/lldb/Target/Language.h | 4 ++++
 lldb/source/Core/Mangled.cpp        | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/lldb/include/lldb/Target/Language.h b/lldb/include/lldb/Target/Language.h
index 67714e6fdf94..ff7c60bf68bf 100644
--- a/lldb/include/lldb/Target/Language.h
+++ b/lldb/include/lldb/Target/Language.h
@@ -281,6 +281,10 @@ public:
     return mangled.GetMangledName();
   }
 
+  virtual ConstString GetDisplayDemangledName(Mangled mangled) const {
+    return mangled.GetDemangledName();
+  }
+
   virtual void GetExceptionResolverDescription(bool catch_on, bool throw_on,
                                                Stream &s);
 
diff --git a/lldb/source/Core/Mangled.cpp b/lldb/source/Core/Mangled.cpp
index b167c51fdce2..8efc4c639cca 100644
--- a/lldb/source/Core/Mangled.cpp
+++ b/lldb/source/Core/Mangled.cpp
@@ -310,6 +310,8 @@ ConstString Mangled::GetDemangledName() const {
 }
 
 ConstString Mangled::GetDisplayDemangledName() const {
+  if (Language *lang = Language::FindPlugin(GuessLanguage()))
+    return lang->GetDisplayDemangledName(*this);
   return GetDemangledName();
 }
 
-- 
GitLab


From cec6665f2b7583223eb20519dfc3289011d1d2d7 Mon Sep 17 00:00:00 2001
From: Teresa Johnson <tejohnson@google.com>
Date: Wed, 8 May 2024 13:41:29 -0700
Subject: [PATCH 0222/1206] [MemProf] Optionally update hints on existing
 hot/cold new calls (#91047)

If directed by an option, update hints on calls to new that already
provide a hot/cold hint.
---
 .../lib/Transforms/Utils/SimplifyLibCalls.cpp | 137 +++++++++++---
 .../InstCombine/simplify-libcalls-new.ll      | 176 +++++++++++++++++-
 2 files changed, 288 insertions(+), 25 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 2e68a9c01898..174cc7a3c778 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -52,6 +52,10 @@ static cl::opt<bool>
 static cl::opt<bool>
     OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(false),
                        cl::desc("Enable hot/cold operator new library calls"));
+static cl::opt<bool> OptimizeExistingHotColdNew(
+    "optimize-existing-hot-cold-new", cl::Hidden, cl::init(false),
+    cl::desc(
+        "Enable optimization of existing hot/cold operator new library calls"));
 
 namespace {
 
@@ -81,6 +85,10 @@ struct HotColdHintParser : public cl::parser<unsigned> {
 static cl::opt<unsigned, false, HotColdHintParser> ColdNewHintValue(
     "cold-new-hint-value", cl::Hidden, cl::init(1),
     cl::desc("Value to pass to hot/cold operator new for cold allocation"));
+static cl::opt<unsigned, false, HotColdHintParser>
+    NotColdNewHintValue("notcold-new-hint-value", cl::Hidden, cl::init(128),
+                        cl::desc("Value to pass to hot/cold operator new for "
+                                 "notcold (warm) allocation"));
 static cl::opt<unsigned, false, HotColdHintParser> HotNewHintValue(
     "hot-new-hint-value", cl::Hidden, cl::init(254),
     cl::desc("Value to pass to hot/cold operator new for hot allocation"));
@@ -1722,45 +1730,122 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
   uint8_t HotCold;
   if (CI->getAttributes().getFnAttr("memprof").getValueAsString() == "cold")
     HotCold = ColdNewHintValue;
+  else if (CI->getAttributes().getFnAttr("memprof").getValueAsString() ==
+           "notcold")
+    HotCold = NotColdNewHintValue;
   else if (CI->getAttributes().getFnAttr("memprof").getValueAsString() == "hot")
     HotCold = HotNewHintValue;
   else
     return nullptr;
 
+  // For calls that already pass a hot/cold hint, only update the hint if
+  // directed by OptimizeExistingHotColdNew. For other calls to new, add a hint
+  // if cold or hot, and leave as-is for default handling if "notcold" aka warm.
+  // Note that in cases where we decide it is "notcold", it might be slightly
+  // better to replace the hinted call with a non hinted call, to avoid the
+  // extra paramter and the if condition check of the hint value in the
+  // allocator. This can be considered in the future.
   switch (Func) {
+  case LibFunc_Znwm12__hot_cold_t:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdNew(CI->getArgOperand(0), B, TLI,
+                            LibFunc_Znwm12__hot_cold_t, HotCold);
+    break;
   case LibFunc_Znwm:
-    return emitHotColdNew(CI->getArgOperand(0), B, TLI,
-                          LibFunc_Znwm12__hot_cold_t, HotCold);
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdNew(CI->getArgOperand(0), B, TLI,
+                            LibFunc_Znwm12__hot_cold_t, HotCold);
+    break;
+  case LibFunc_Znam12__hot_cold_t:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdNew(CI->getArgOperand(0), B, TLI,
+                            LibFunc_Znam12__hot_cold_t, HotCold);
+    break;
   case LibFunc_Znam:
-    return emitHotColdNew(CI->getArgOperand(0), B, TLI,
-                          LibFunc_Znam12__hot_cold_t, HotCold);
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdNew(CI->getArgOperand(0), B, TLI,
+                            LibFunc_Znam12__hot_cold_t, HotCold);
+    break;
+  case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdNewNoThrow(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold);
+    break;
   case LibFunc_ZnwmRKSt9nothrow_t:
-    return emitHotColdNewNoThrow(CI->getArgOperand(0), CI->getArgOperand(1), B,
-                                 TLI, LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,
-                                 HotCold);
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdNewNoThrow(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold);
+    break;
+  case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdNewNoThrow(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold);
+    break;
   case LibFunc_ZnamRKSt9nothrow_t:
-    return emitHotColdNewNoThrow(CI->getArgOperand(0), CI->getArgOperand(1), B,
-                                 TLI, LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,
-                                 HotCold);
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdNewNoThrow(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold);
+    break;
+  case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdNewAligned(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold);
+    break;
   case LibFunc_ZnwmSt11align_val_t:
-    return emitHotColdNewAligned(CI->getArgOperand(0), CI->getArgOperand(1), B,
-                                 TLI, LibFunc_ZnwmSt11align_val_t12__hot_cold_t,
-                                 HotCold);
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdNewAligned(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold);
+    break;
+  case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdNewAligned(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold);
+    break;
   case LibFunc_ZnamSt11align_val_t:
-    return emitHotColdNewAligned(CI->getArgOperand(0), CI->getArgOperand(1), B,
-                                 TLI, LibFunc_ZnamSt11align_val_t12__hot_cold_t,
-                                 HotCold);
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdNewAligned(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold);
+    break;
+  case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdNewAlignedNoThrow(
+          CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
+          TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
+          HotCold);
+    break;
   case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
-    return emitHotColdNewAlignedNoThrow(
-        CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
-        TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdNewAlignedNoThrow(
+          CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
+          TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
+          HotCold);
+    break;
+  case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdNewAlignedNoThrow(
+          CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
+          TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
+          HotCold);
+    break;
   case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
-    return emitHotColdNewAlignedNoThrow(
-        CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
-        TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdNewAlignedNoThrow(
+          CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
+          TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
+          HotCold);
+    break;
   default:
     return nullptr;
   }
+  return nullptr;
 }
 
 //===----------------------------------------------------------------------===//
@@ -3675,6 +3760,14 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
     case LibFunc_ZnamRKSt9nothrow_t:
     case LibFunc_ZnamSt11align_val_t:
     case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
+    case LibFunc_Znwm12__hot_cold_t:
+    case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
+    case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
+    case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
+    case LibFunc_Znam12__hot_cold_t:
+    case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
+    case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
+    case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
       return optimizeNew(CI, Builder, Func);
     default:
       break;
diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
index 51debdf6643e..ecfafbc69797 100644
--- a/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
@@ -1,13 +1,19 @@
 ;; Test behavior of -optimize-hot-cold-new and related options.
 
 ;; Check that we don't get hot/cold new calls without enabling it explicitly.
-; RUN: opt < %s -passes=instcombine -S | FileCheck %s --implicit-check-not=hot_cold_t
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s --check-prefix=OFF
+; OFF-NOT: hot_cold_t
+; OFF-LABEL: @new_hot_cold()
 
 ;; First check with the default cold and hot hint values (255 = -2).
-; RUN: opt < %s -passes=instcombine -optimize-hot-cold-new -S | FileCheck %s --check-prefix=HOTCOLD -DCOLD=1 -DHOT=-2
+; RUN: opt < %s -passes=instcombine -optimize-hot-cold-new -S | FileCheck %s --check-prefix=HOTCOLD -DCOLD=1 -DHOT=-2 -DPREVHINTCOLD=7 -DPREVHINTNOTCOLD=7 -DPREVHINTHOT=7
 
 ;; Next check with the non-default cold and hot hint values (200 =-56).
-; RUN: opt < %s -passes=instcombine -optimize-hot-cold-new -cold-new-hint-value=5 -hot-new-hint-value=200 -S | FileCheck %s --check-prefix=HOTCOLD -DCOLD=5 -DHOT=-56
+; RUN: opt < %s -passes=instcombine -optimize-hot-cold-new -cold-new-hint-value=5 -hot-new-hint-value=200 -S | FileCheck %s --check-prefix=HOTCOLD -DCOLD=5 -DHOT=-56 -DPREVHINTCOLD=7 -DPREVHINTNOTCOLD=7 -DPREVHINTHOT=7
+
+;; Try again with the non-default cold and hot hint values (200 =-56), and this
+;; time specify that existing hints should be updated.
+; RUN: opt < %s -passes=instcombine -optimize-hot-cold-new -cold-new-hint-value=5 -notcold-new-hint-value=100 -hot-new-hint-value=200 -optimize-existing-hot-cold-new -S | FileCheck %s --check-prefix=HOTCOLD -DCOLD=5 -DHOT=-56 -DPREVHINTCOLD=5 -DPREVHINTNOTCOLD=100 -DPREVHINTHOT=-56
 
 ;; Make sure that values not in 0..255 are flagged with an error
 ; RUN: not opt < %s -passes=instcombine -optimize-hot-cold-new -cold-new-hint-value=256 -S 2>&1 | FileCheck %s --check-prefix=ERROR
@@ -178,6 +184,162 @@ define void @array_new_align_nothrow() {
   ret void
 }
 
+;; Check that operator new(unsigned long, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @new_hot_cold()
+define void @new_hot_cold() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @_Znwm12__hot_cold_t(i64 10, i8 [[PREVHINTCOLD]])
+  %call = call ptr @_Znwm12__hot_cold_t(i64 10, i8 7) #0
+  call void @dummy(ptr %call)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @_Znwm12__hot_cold_t(i64 10, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call ptr @_Znwm12__hot_cold_t(i64 10, i8 7) #1
+  call void @dummy(ptr %call1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @_Znwm12__hot_cold_t(i64 10, i8 [[PREVHINTHOT]])
+  %call2 = call ptr @_Znwm12__hot_cold_t(i64 10, i8 7) #2
+  call void @dummy(ptr %call2)
+  ret void
+}
+
+;; Check that operator new(unsigned long, std::align_val_t, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @new_align_hot_cold()
+define void @new_align_hot_cold() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @_ZnwmSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 [[PREVHINTCOLD]])
+  %call = call ptr @_ZnwmSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 7) #0
+  call void @dummy(ptr %call)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @_ZnwmSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call ptr @_ZnwmSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 7) #1
+  call void @dummy(ptr %call1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @_ZnwmSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 [[PREVHINTHOT]])
+  %call2 = call ptr @_ZnwmSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 7) #2
+  call void @dummy(ptr %call2)
+  ret void
+}
+
+;; Check that operator new(unsigned long, const std::nothrow_t&, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @new_nothrow_hot_cold()
+define void @new_nothrow_hot_cold() {
+  %nt = alloca i8
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64 10, ptr nonnull %nt, i8 [[PREVHINTCOLD]])
+  %call = call ptr @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64 10, ptr %nt, i8 7) #0
+  call void @dummy(ptr %call)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64 10, ptr nonnull %nt, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call ptr @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64 10, ptr %nt, i8 7) #1
+  call void @dummy(ptr %call1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64 10, ptr nonnull %nt, i8 [[PREVHINTHOT]])
+  %call2 = call ptr @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64 10, ptr %nt, i8 7) #2
+  call void @dummy(ptr %call2)
+  ret void
+}
+
+;; Check that operator new(unsigned long, std::align_val_t, const std::nothrow_t&, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @new_align_nothrow_hot_cold()
+define void @new_align_nothrow_hot_cold() {
+  %nt = alloca i8
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 [[PREVHINTCOLD]])
+  %call = call ptr @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr %nt, i8 7) #0
+  call void @dummy(ptr %call)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call ptr @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr %nt, i8 7) #1
+  call void @dummy(ptr %call1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 [[PREVHINTHOT]])
+  %call2 = call ptr @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr %nt, i8 7) #2
+  call void @dummy(ptr %call2)
+  ret void
+}
+
+;; Check that operator new[](unsigned long, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @array_new_hot_cold()
+define void @array_new_hot_cold() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @_Znam12__hot_cold_t(i64 10, i8 [[PREVHINTCOLD]])
+  %call = call ptr @_Znam12__hot_cold_t(i64 10, i8 7) #0
+  call void @dummy(ptr %call)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @_Znam12__hot_cold_t(i64 10, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call ptr @_Znam12__hot_cold_t(i64 10, i8 7) #1
+  call void @dummy(ptr %call1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @_Znam12__hot_cold_t(i64 10, i8 [[PREVHINTHOT]])
+  %call2 = call ptr @_Znam12__hot_cold_t(i64 10, i8 7) #2
+  call void @dummy(ptr %call2)
+  ret void
+}
+
+;; Check that operator new[](unsigned long, std::align_val_t, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @array_new_align_hot_cold()
+define void @array_new_align_hot_cold() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @_ZnamSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 [[PREVHINTCOLD]])
+  %call = call ptr @_ZnamSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 7) #0
+  call void @dummy(ptr %call)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @_ZnamSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call ptr @_ZnamSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 7) #1
+  call void @dummy(ptr %call1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @_ZnamSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 [[PREVHINTHOT]])
+  %call2 = call ptr @_ZnamSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 7) #2
+  call void @dummy(ptr %call2)
+  ret void
+}
+
+;; Check that operator new[](unsigned long, const std::nothrow_t&, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @array_new_nothrow_hot_cold()
+define void @array_new_nothrow_hot_cold() {
+  %nt = alloca i8
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @_ZnamRKSt9nothrow_t12__hot_cold_t(i64 10, ptr nonnull %nt, i8 [[PREVHINTCOLD]])
+  %call = call ptr @_ZnamRKSt9nothrow_t12__hot_cold_t(i64 10, ptr %nt, i8 7) #0
+  call void @dummy(ptr %call)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @_ZnamRKSt9nothrow_t12__hot_cold_t(i64 10, ptr nonnull %nt, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call ptr @_ZnamRKSt9nothrow_t12__hot_cold_t(i64 10, ptr %nt, i8 7) #1
+  call void @dummy(ptr %call1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @_ZnamRKSt9nothrow_t12__hot_cold_t(i64 10, ptr nonnull %nt, i8 [[PREVHINTHOT]])
+  %call2 = call ptr @_ZnamRKSt9nothrow_t12__hot_cold_t(i64 10, ptr %nt, i8 7) #2
+  call void @dummy(ptr %call2)
+  ret void
+}
+
+;; Check that operator new[](unsigned long, std::align_val_t, const std::nothrow_t&, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @array_new_align_nothrow_hot_cold()
+define void @array_new_align_nothrow_hot_cold() {
+  %nt = alloca i8
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 [[PREVHINTCOLD]])
+  %call = call ptr @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr %nt, i8 7) #0
+  call void @dummy(ptr %call)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call ptr @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr %nt, i8 7) #1
+  call void @dummy(ptr %call1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 [[PREVHINTHOT]])
+  %call2 = call ptr @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr %nt, i8 7) #2
+  call void @dummy(ptr %call2)
+  ret void
+}
+
 ;; So that instcombine doesn't optimize out the call.
 declare void @dummy(ptr)
 
@@ -189,6 +351,14 @@ declare ptr @_Znam(i64)
 declare ptr @_ZnamSt11align_val_t(i64, i64)
 declare ptr @_ZnamRKSt9nothrow_t(i64, ptr)
 declare ptr @_ZnamSt11align_val_tRKSt9nothrow_t(i64, i64, ptr)
+declare ptr @_Znwm12__hot_cold_t(i64, i8)
+declare ptr @_ZnwmSt11align_val_t12__hot_cold_t(i64, i64, i8)
+declare ptr @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64, ptr, i8)
+declare ptr @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64, i64, ptr, i8)
+declare ptr @_Znam12__hot_cold_t(i64, i8)
+declare ptr @_ZnamSt11align_val_t12__hot_cold_t(i64, i64, i8)
+declare ptr @_ZnamRKSt9nothrow_t12__hot_cold_t(i64, ptr, i8)
+declare ptr @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64, i64, ptr, i8)
 
 attributes #0 = { builtin allocsize(0) "memprof"="cold" }
 attributes #1 = { builtin allocsize(0) "memprof"="notcold" }
-- 
GitLab


From 965f3ca3dc5464892e283e176bf058ae04d8b654 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 8 May 2024 21:58:59 +0100
Subject: [PATCH 0223/1206] [GISel] Fold bitreverse(shl/srl(bitreverse(x),y))
 -> srl/shl(x,y) (#91355)

Sibling patch to #89897
---
 .../include/llvm/Target/GlobalISel/Combine.td |  26 +++-
 .../GlobalISel/combine-bitreverse-shift.ll    | 132 +++++-------------
 2 files changed, 59 insertions(+), 99 deletions(-)

diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index d0e125390347..98d266c8c0b4 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -325,6 +325,28 @@ def reduce_shl_of_extend : GICombineRule<
          [{ return Helper.matchCombineShlOfExtend(*${mi}, ${matchinfo}); }]),
   (apply [{ Helper.applyCombineShlOfExtend(*${mi}, ${matchinfo}); }])>;
 
+// Combine bitreverse(shl (bitreverse x), y)) -> (lshr x, y)
+def bitreverse_shl : GICombineRule<
+  (defs root:$d),
+  (match (G_BITREVERSE $rev, $val),
+         (G_SHL $src, $rev, $amt):$mi,
+         (G_BITREVERSE $d, $src),
+         [{ return Helper.isLegalOrBeforeLegalizer({TargetOpcode::G_LSHR,
+                                                   {MRI.getType(${val}.getReg()),
+                                                    MRI.getType(${amt}.getReg())}}); }]),
+  (apply (G_LSHR $d, $val, $amt))>;
+
+// Combine bitreverse(lshr (bitreverse x), y)) -> (shl x, y)
+def bitreverse_lshr : GICombineRule<
+  (defs root:$d, build_fn_matchinfo:$matchinfo),
+  (match (G_BITREVERSE $rev, $val),
+         (G_LSHR $src, $rev, $amt):$mi,
+         (G_BITREVERSE $d, $src),
+         [{ return Helper.isLegalOrBeforeLegalizer({TargetOpcode::G_SHL,
+                                                   {MRI.getType(${val}.getReg()),
+                                                    MRI.getType(${amt}.getReg())}}); }]),
+  (apply (G_SHL $d, $val, $amt))>;
+
 // Combine (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
 // Combine (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
 def commute_shift : GICombineRule<
@@ -1645,6 +1667,8 @@ def width_reduction_combines : GICombineGroup<[reduce_shl_of_extend,
 
 def phi_combines : GICombineGroup<[extend_through_phis]>;
 
+def bitreverse_shift : GICombineGroup<[bitreverse_shl, bitreverse_lshr]>;
+
 def select_combines : GICombineGroup<[select_undef_cmp, select_constant_cmp,
                                       match_selects]>;
 
@@ -1674,7 +1698,7 @@ def all_combines : GICombineGroup<[trivial_combines, vector_ops_combines,
     unmerge_zext_to_zext, merge_unmerge, trunc_ext_fold, trunc_shift,
     const_combines, xor_of_and_with_same_reg, ptr_add_with_zero,
     shift_immed_chain, shift_of_shifted_logic_chain, load_or_combine,
-    div_rem_to_divrem, funnel_shift_combines, commute_shift,
+    div_rem_to_divrem, funnel_shift_combines, bitreverse_shift, commute_shift,
     form_bitfield_extract, constant_fold_binops, constant_fold_fma,
     constant_fold_cast_op, fabs_fneg_fold,
     intdiv_combines, mulh_combines, redundant_neg_operands,
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-bitreverse-shift.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combine-bitreverse-shift.ll
index 3ce94e2c40a9..b9fbe2379a42 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-bitreverse-shift.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-bitreverse-shift.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefixes=SDAG
-; RUN: llc < %s -mtriple=aarch64-unknown-unknown -global-isel | FileCheck %s --check-prefixes=GISEL
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown -global-isel | FileCheck %s
 
 ; These tests can be optimised
 ;       fold (bitreverse(srl (bitreverse c), x)) -> (shl c, x)
@@ -12,19 +12,10 @@ declare i32 @llvm.bitreverse.i32(i32)
 declare i64 @llvm.bitreverse.i64(i64)
 
 define i8 @test_bitreverse_srli_bitreverse_i8(i8 %a) nounwind {
-; SDAG-LABEL: test_bitreverse_srli_bitreverse_i8:
-; SDAG:       // %bb.0:
-; SDAG-NEXT:    lsl w0, w0, #3
-; SDAG-NEXT:    ret
-;
-; GISEL-LABEL: test_bitreverse_srli_bitreverse_i8:
-; GISEL:       // %bb.0:
-; GISEL-NEXT:    rbit w8, w0
-; GISEL-NEXT:    lsr w8, w8, #24
-; GISEL-NEXT:    lsr w8, w8, #3
-; GISEL-NEXT:    rbit w8, w8
-; GISEL-NEXT:    lsr w0, w8, #24
-; GISEL-NEXT:    ret
+; CHECK-LABEL: test_bitreverse_srli_bitreverse_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl w0, w0, #3
+; CHECK-NEXT:    ret
   %1 = call i8 @llvm.bitreverse.i8(i8 %a)
   %2 = lshr i8 %1, 3
   %3 = call i8 @llvm.bitreverse.i8(i8 %2)
@@ -32,19 +23,10 @@ define i8 @test_bitreverse_srli_bitreverse_i8(i8 %a) nounwind {
 }
 
 define i16 @test_bitreverse_srli_bitreverse_i16(i16 %a) nounwind {
-; SDAG-LABEL: test_bitreverse_srli_bitreverse_i16:
-; SDAG:       // %bb.0:
-; SDAG-NEXT:    lsl w0, w0, #7
-; SDAG-NEXT:    ret
-;
-; GISEL-LABEL: test_bitreverse_srli_bitreverse_i16:
-; GISEL:       // %bb.0:
-; GISEL-NEXT:    rbit w8, w0
-; GISEL-NEXT:    lsr w8, w8, #16
-; GISEL-NEXT:    lsr w8, w8, #7
-; GISEL-NEXT:    rbit w8, w8
-; GISEL-NEXT:    lsr w0, w8, #16
-; GISEL-NEXT:    ret
+; CHECK-LABEL: test_bitreverse_srli_bitreverse_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl w0, w0, #7
+; CHECK-NEXT:    ret
   %1 = call i16 @llvm.bitreverse.i16(i16 %a)
   %2 = lshr i16 %1, 7
   %3 = call i16 @llvm.bitreverse.i16(i16 %2)
@@ -52,17 +34,10 @@ define i16 @test_bitreverse_srli_bitreverse_i16(i16 %a) nounwind {
 }
 
 define i32 @test_bitreverse_srli_bitreverse_i32(i32 %a) nounwind {
-; SDAG-LABEL: test_bitreverse_srli_bitreverse_i32:
-; SDAG:       // %bb.0:
-; SDAG-NEXT:    lsl w0, w0, #15
-; SDAG-NEXT:    ret
-;
-; GISEL-LABEL: test_bitreverse_srli_bitreverse_i32:
-; GISEL:       // %bb.0:
-; GISEL-NEXT:    rbit w8, w0
-; GISEL-NEXT:    lsr w8, w8, #15
-; GISEL-NEXT:    rbit w0, w8
-; GISEL-NEXT:    ret
+; CHECK-LABEL: test_bitreverse_srli_bitreverse_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl w0, w0, #15
+; CHECK-NEXT:    ret
   %1 = call i32 @llvm.bitreverse.i32(i32 %a)
   %2 = lshr i32 %1, 15
   %3 = call i32 @llvm.bitreverse.i32(i32 %2)
@@ -70,17 +45,10 @@ define i32 @test_bitreverse_srli_bitreverse_i32(i32 %a) nounwind {
 }
 
 define i64 @test_bitreverse_srli_bitreverse_i64(i64 %a) nounwind {
-; SDAG-LABEL: test_bitreverse_srli_bitreverse_i64:
-; SDAG:       // %bb.0:
-; SDAG-NEXT:    lsl x0, x0, #33
-; SDAG-NEXT:    ret
-;
-; GISEL-LABEL: test_bitreverse_srli_bitreverse_i64:
-; GISEL:       // %bb.0:
-; GISEL-NEXT:    rbit x8, x0
-; GISEL-NEXT:    lsr x8, x8, #33
-; GISEL-NEXT:    rbit x0, x8
-; GISEL-NEXT:    ret
+; CHECK-LABEL: test_bitreverse_srli_bitreverse_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl x0, x0, #33
+; CHECK-NEXT:    ret
   %1 = call i64 @llvm.bitreverse.i64(i64 %a)
   %2 = lshr i64 %1, 33
   %3 = call i64 @llvm.bitreverse.i64(i64 %2)
@@ -88,19 +56,10 @@ define i64 @test_bitreverse_srli_bitreverse_i64(i64 %a) nounwind {
 }
 
 define i8 @test_bitreverse_shli_bitreverse_i8(i8 %a) nounwind {
-; SDAG-LABEL: test_bitreverse_shli_bitreverse_i8:
-; SDAG:       // %bb.0:
-; SDAG-NEXT:    ubfx w0, w0, #3, #5
-; SDAG-NEXT:    ret
-;
-; GISEL-LABEL: test_bitreverse_shli_bitreverse_i8:
-; GISEL:       // %bb.0:
-; GISEL-NEXT:    rbit w8, w0
-; GISEL-NEXT:    lsr w8, w8, #24
-; GISEL-NEXT:    lsl w8, w8, #3
-; GISEL-NEXT:    rbit w8, w8
-; GISEL-NEXT:    lsr w0, w8, #24
-; GISEL-NEXT:    ret
+; CHECK-LABEL: test_bitreverse_shli_bitreverse_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ubfx w0, w0, #3, #5
+; CHECK-NEXT:    ret
   %1 = call i8 @llvm.bitreverse.i8(i8 %a)
   %2 = shl i8 %1, 3
   %3 = call i8 @llvm.bitreverse.i8(i8 %2)
@@ -108,19 +67,10 @@ define i8 @test_bitreverse_shli_bitreverse_i8(i8 %a) nounwind {
 }
 
 define i16 @test_bitreverse_shli_bitreverse_i16(i16 %a) nounwind {
-; SDAG-LABEL: test_bitreverse_shli_bitreverse_i16:
-; SDAG:       // %bb.0:
-; SDAG-NEXT:    ubfx w0, w0, #7, #9
-; SDAG-NEXT:    ret
-;
-; GISEL-LABEL: test_bitreverse_shli_bitreverse_i16:
-; GISEL:       // %bb.0:
-; GISEL-NEXT:    rbit w8, w0
-; GISEL-NEXT:    lsr w8, w8, #16
-; GISEL-NEXT:    lsl w8, w8, #7
-; GISEL-NEXT:    rbit w8, w8
-; GISEL-NEXT:    lsr w0, w8, #16
-; GISEL-NEXT:    ret
+; CHECK-LABEL: test_bitreverse_shli_bitreverse_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ubfx w0, w0, #7, #9
+; CHECK-NEXT:    ret
   %1 = call i16 @llvm.bitreverse.i16(i16 %a)
   %2 = shl i16 %1, 7
   %3 = call i16 @llvm.bitreverse.i16(i16 %2)
@@ -128,17 +78,10 @@ define i16 @test_bitreverse_shli_bitreverse_i16(i16 %a) nounwind {
 }
 
 define i32 @test_bitreverse_shli_bitreverse_i32(i32 %a) nounwind {
-; SDAG-LABEL: test_bitreverse_shli_bitreverse_i32:
-; SDAG:       // %bb.0:
-; SDAG-NEXT:    lsr w0, w0, #15
-; SDAG-NEXT:    ret
-;
-; GISEL-LABEL: test_bitreverse_shli_bitreverse_i32:
-; GISEL:       // %bb.0:
-; GISEL-NEXT:    rbit w8, w0
-; GISEL-NEXT:    lsl w8, w8, #15
-; GISEL-NEXT:    rbit w0, w8
-; GISEL-NEXT:    ret
+; CHECK-LABEL: test_bitreverse_shli_bitreverse_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsr w0, w0, #15
+; CHECK-NEXT:    ret
   %1 = call i32 @llvm.bitreverse.i32(i32 %a)
   %2 = shl i32 %1, 15
   %3 = call i32 @llvm.bitreverse.i32(i32 %2)
@@ -146,17 +89,10 @@ define i32 @test_bitreverse_shli_bitreverse_i32(i32 %a) nounwind {
 }
 
 define i64 @test_bitreverse_shli_bitreverse_i64(i64 %a) nounwind {
-; SDAG-LABEL: test_bitreverse_shli_bitreverse_i64:
-; SDAG:       // %bb.0:
-; SDAG-NEXT:    lsr x0, x0, #33
-; SDAG-NEXT:    ret
-;
-; GISEL-LABEL: test_bitreverse_shli_bitreverse_i64:
-; GISEL:       // %bb.0:
-; GISEL-NEXT:    rbit x8, x0
-; GISEL-NEXT:    lsl x8, x8, #33
-; GISEL-NEXT:    rbit x0, x8
-; GISEL-NEXT:    ret
+; CHECK-LABEL: test_bitreverse_shli_bitreverse_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsr x0, x0, #33
+; CHECK-NEXT:    ret
   %1 = call i64 @llvm.bitreverse.i64(i64 %a)
   %2 = shl i64 %1, 33
   %3 = call i64 @llvm.bitreverse.i64(i64 %2)
-- 
GitLab


From fcf945f4edbad1f2d82df067c2826baa6165dd3e Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Wed, 8 May 2024 22:11:18 +0100
Subject: [PATCH 0224/1206] [DAG] Fold add(mul(add(A, CA), CM), CB) ->
 add(mul(A, CM), CM*CA+CB) (#90860)

This is useful when the inner add has multiple uses, and so cannot be
canonicalized by pushing the constants down through the mul. This patch
adds patterns for both `add(mul(add(A, CA), CM), CB)` and with an extra add
`add(add(mul(add(A, CA), CM), B) CB)` as the second can come up when
lowering geps.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 60 +++++++++++++++++
 llvm/test/CodeGen/AArch64/addimm-mulimm.ll    | 67 +++++++++----------
 2 files changed, 91 insertions(+), 36 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e835bd950a7b..4589d201d620 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -2838,6 +2838,66 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) {
     return DAG.getNode(ISD::ADD, DL, VT, Not, N0.getOperand(0));
   }
 
+  // Fold add(mul(add(A, CA), CM), CB) -> add(mul(A, CM), CM*CA+CB).
+  // This can help if the inner add has multiple uses.
+  APInt CM, CA;
+  if (ConstantSDNode *CB = dyn_cast<ConstantSDNode>(N1)) {
+    if (VT.getScalarSizeInBits() <= 64) {
+      if (sd_match(N0, m_OneUse(m_Mul(m_Add(m_Value(A), m_ConstInt(CA)),
+                                      m_ConstInt(CM)))) &&
+          TLI.isLegalAddImmediate(
+              (CA * CM + CB->getAPIntValue()).getSExtValue())) {
+        SDNodeFlags Flags;
+        // If all the inputs are nuw, the outputs can be nuw. If all the input
+        // are _also_ nsw the outputs can be too.
+        if (N->getFlags().hasNoUnsignedWrap() &&
+            N0->getFlags().hasNoUnsignedWrap() &&
+            N0.getOperand(0)->getFlags().hasNoUnsignedWrap()) {
+          Flags.setNoUnsignedWrap(true);
+          if (N->getFlags().hasNoSignedWrap() &&
+              N0->getFlags().hasNoSignedWrap() &&
+              N0.getOperand(0)->getFlags().hasNoSignedWrap())
+            Flags.setNoSignedWrap(true);
+        }
+        SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N1), VT, A,
+                                  DAG.getConstant(CM, DL, VT), Flags);
+        return DAG.getNode(
+            ISD::ADD, DL, VT, Mul,
+            DAG.getConstant(CA * CM + CB->getAPIntValue(), DL, VT), Flags);
+      }
+      // Also look in case there is an intermediate add.
+      if (sd_match(N0, m_OneUse(m_Add(
+                           m_OneUse(m_Mul(m_Add(m_Value(A), m_ConstInt(CA)),
+                                          m_ConstInt(CM))),
+                           m_Value(B)))) &&
+          TLI.isLegalAddImmediate(
+              (CA * CM + CB->getAPIntValue()).getSExtValue())) {
+        SDNodeFlags Flags;
+        // If all the inputs are nuw, the outputs can be nuw. If all the input
+        // are _also_ nsw the outputs can be too.
+        SDValue OMul =
+            N0.getOperand(0) == B ? N0.getOperand(1) : N0.getOperand(0);
+        if (N->getFlags().hasNoUnsignedWrap() &&
+            N0->getFlags().hasNoUnsignedWrap() &&
+            OMul->getFlags().hasNoUnsignedWrap() &&
+            OMul.getOperand(0)->getFlags().hasNoUnsignedWrap()) {
+          Flags.setNoUnsignedWrap(true);
+          if (N->getFlags().hasNoSignedWrap() &&
+              N0->getFlags().hasNoSignedWrap() &&
+              OMul->getFlags().hasNoSignedWrap() &&
+              OMul.getOperand(0)->getFlags().hasNoSignedWrap())
+            Flags.setNoSignedWrap(true);
+        }
+        SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N1), VT, A,
+                                  DAG.getConstant(CM, DL, VT), Flags);
+        SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N1), VT, Mul, B, Flags);
+        return DAG.getNode(
+            ISD::ADD, DL, VT, Add,
+            DAG.getConstant(CA * CM + CB->getAPIntValue(), DL, VT), Flags);
+      }
+    }
+  }
+
   if (SDValue Combined = visitADDLikeCommutative(N0, N1, N))
     return Combined;
 
diff --git a/llvm/test/CodeGen/AArch64/addimm-mulimm.ll b/llvm/test/CodeGen/AArch64/addimm-mulimm.ll
index 3618b14aa921..6636813eb250 100644
--- a/llvm/test/CodeGen/AArch64/addimm-mulimm.ll
+++ b/llvm/test/CodeGen/AArch64/addimm-mulimm.ll
@@ -166,9 +166,9 @@ define signext i32 @addmuladd_multiuse(i32 signext %a) {
 ; CHECK-LABEL: addmuladd_multiuse:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #324 // =0x144
+; CHECK-NEXT:    mov w9, #1300 // =0x514
+; CHECK-NEXT:    madd w8, w0, w8, w9
 ; CHECK-NEXT:    add w9, w0, #4
-; CHECK-NEXT:    mov w10, #4 // =0x4
-; CHECK-NEXT:    madd w8, w9, w8, w10
 ; CHECK-NEXT:    eor w0, w9, w8
 ; CHECK-NEXT:    ret
   %tmp0 = add i32 %a, 4
@@ -198,11 +198,10 @@ define signext i32 @addmuladd_multiuse2(i32 signext %a) {
 ; CHECK-LABEL: addmuladd_multiuse2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #324 // =0x144
-; CHECK-NEXT:    add w9, w0, #4
-; CHECK-NEXT:    mov w11, #4 // =0x4
-; CHECK-NEXT:    lsl w10, w9, #2
-; CHECK-NEXT:    madd w8, w9, w8, w11
-; CHECK-NEXT:    add w9, w10, #4
+; CHECK-NEXT:    lsl w9, w0, #2
+; CHECK-NEXT:    mov w10, #1300 // =0x514
+; CHECK-NEXT:    madd w8, w0, w8, w10
+; CHECK-NEXT:    add w9, w9, #20
 ; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
   %tmp0 = add i32 %a, 4
@@ -233,8 +232,8 @@ define signext i32 @addaddmuladd_multiuse(i32 signext %a, i32 %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #324 // =0x144
 ; CHECK-NEXT:    add w9, w0, #4
-; CHECK-NEXT:    madd w8, w9, w8, w1
-; CHECK-NEXT:    add w8, w8, #4
+; CHECK-NEXT:    madd w8, w0, w8, w1
+; CHECK-NEXT:    add w8, w8, #1300
 ; CHECK-NEXT:    eor w0, w9, w8
 ; CHECK-NEXT:    ret
   %tmp0 = add i32 %a, 4
@@ -249,12 +248,11 @@ define signext i32 @addaddmuladd_multiuse2(i32 signext %a, i32 %b) {
 ; CHECK-LABEL: addaddmuladd_multiuse2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #324 // =0x144
-; CHECK-NEXT:    add w9, w0, #4
-; CHECK-NEXT:    mov w10, #162 // =0xa2
-; CHECK-NEXT:    madd w8, w9, w8, w1
-; CHECK-NEXT:    madd w9, w9, w10, w1
-; CHECK-NEXT:    add w8, w8, #4
-; CHECK-NEXT:    add w9, w9, #4
+; CHECK-NEXT:    mov w9, #162 // =0xa2
+; CHECK-NEXT:    madd w8, w0, w8, w1
+; CHECK-NEXT:    madd w9, w0, w9, w1
+; CHECK-NEXT:    add w8, w8, #1300
+; CHECK-NEXT:    add w9, w9, #652
 ; CHECK-NEXT:    eor w0, w9, w8
 ; CHECK-NEXT:    ret
   %tmp0 = add i32 %a, 4
@@ -319,17 +317,17 @@ define void @addmuladd_gep(ptr %p, i64 %a) {
 define i32 @addmuladd_gep2(ptr %p, i32 %a) {
 ; CHECK-LABEL: addmuladd_gep2:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #3240 // =0xca8
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    sxtw x8, w1
-; CHECK-NEXT:    mov w9, #3240 // =0xca8
-; CHECK-NEXT:    add x8, x8, #1
-; CHECK-NEXT:    madd x9, x8, x9, x0
-; CHECK-NEXT:    ldr w9, [x9, #20]
-; CHECK-NEXT:    tbnz w9, #31, .LBB22_2
+; CHECK-NEXT:    smaddl x8, w1, w8, x0
+; CHECK-NEXT:    ldr w8, [x8, #3260]
+; CHECK-NEXT:    tbnz w8, #31, .LBB22_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB22_2: // %then
+; CHECK-NEXT:    sxtw x8, w1
+; CHECK-NEXT:    add x8, x8, #1
 ; CHECK-NEXT:    str x8, [x0]
 ; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
@@ -351,11 +349,10 @@ define signext i32 @addmuladd_multiuse2_nsw(i32 signext %a) {
 ; CHECK-LABEL: addmuladd_multiuse2_nsw:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #324 // =0x144
-; CHECK-NEXT:    add w9, w0, #4
-; CHECK-NEXT:    mov w11, #4 // =0x4
-; CHECK-NEXT:    lsl w10, w9, #2
-; CHECK-NEXT:    madd w8, w9, w8, w11
-; CHECK-NEXT:    add w9, w10, #4
+; CHECK-NEXT:    lsl w9, w0, #2
+; CHECK-NEXT:    mov w10, #1300 // =0x514
+; CHECK-NEXT:    madd w8, w0, w8, w10
+; CHECK-NEXT:    add w9, w9, #20
 ; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
   %tmp0 = add nsw i32 %a, 4
@@ -371,11 +368,10 @@ define signext i32 @addmuladd_multiuse2_nuw(i32 signext %a) {
 ; CHECK-LABEL: addmuladd_multiuse2_nuw:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #324 // =0x144
-; CHECK-NEXT:    add w9, w0, #4
-; CHECK-NEXT:    mov w11, #4 // =0x4
-; CHECK-NEXT:    lsl w10, w9, #2
-; CHECK-NEXT:    madd w8, w9, w8, w11
-; CHECK-NEXT:    add w9, w10, #4
+; CHECK-NEXT:    lsl w9, w0, #2
+; CHECK-NEXT:    mov w10, #1300 // =0x514
+; CHECK-NEXT:    madd w8, w0, w8, w10
+; CHECK-NEXT:    add w9, w9, #20
 ; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
   %tmp0 = add nuw i32 %a, 4
@@ -391,11 +387,10 @@ define signext i32 @addmuladd_multiuse2_nswnuw(i32 signext %a) {
 ; CHECK-LABEL: addmuladd_multiuse2_nswnuw:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #324 // =0x144
-; CHECK-NEXT:    add w9, w0, #4
-; CHECK-NEXT:    mov w11, #4 // =0x4
-; CHECK-NEXT:    lsl w10, w9, #2
-; CHECK-NEXT:    madd w8, w9, w8, w11
-; CHECK-NEXT:    add w9, w10, #4
+; CHECK-NEXT:    lsl w9, w0, #2
+; CHECK-NEXT:    mov w10, #1300 // =0x514
+; CHECK-NEXT:    madd w8, w0, w8, w10
+; CHECK-NEXT:    add w9, w9, #20
 ; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
   %tmp0 = add nsw nuw i32 %a, 4
-- 
GitLab


From e37bd6c68b50a556ff0e9261cf9eba64afa06bf9 Mon Sep 17 00:00:00 2001
From: lntue <35648136+lntue@users.noreply.github.com>
Date: Wed, 8 May 2024 17:32:05 -0400
Subject: [PATCH 0225/1206] [libc][fenv] Add missing FE_* definitions for some
 environment. (#91519)

---
 libc/hdr/fenv_macros.h | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/libc/hdr/fenv_macros.h b/libc/hdr/fenv_macros.h
index 041fca5f224b..a2e4462ef02d 100644
--- a/libc/hdr/fenv_macros.h
+++ b/libc/hdr/fenv_macros.h
@@ -17,10 +17,51 @@
 
 #include <fenv.h>
 
+// In some environment, FE_ALL_EXCEPT is set to 0 and the remaining exceptions
+// FE_* are missing.
+#if (FE_ALL_EXCEPT == 0)
+#ifndef FE_DIVBYZERO
+#define FE_DIVBYZERO 0
+#endif // FE_DIVBYZERO
+
+#ifndef FE_INEXACT
+#define FE_INEXACT 0
+#endif // FE_INEXACT
+
+#ifndef FE_INVALID
+#define FE_INVALID 0
+#endif // FE_INVALID
+
+#ifndef FE_OVERFLOW
+#define FE_OVERFLOW 0
+#endif // FE_OVERFLOW
+
+#ifndef FE_UNDERFLOW
+#define FE_UNDERFLOW 0
+#endif // FE_UNDERFLOW
+#else
 // If this is not provided by the system, define it for use internally.
 #ifndef __FE_DENORM
 #define __FE_DENORM (1 << 6)
 #endif
+#endif
+
+// Rounding mode macros might be missing.
+#ifndef FE_DOWNWARD
+#define FE_DOWNWARD 0x400
+#endif // FE_DOWNWARD
+
+#ifndef FE_TONEAREST
+#define FE_TONEAREST 0
+#endif // FE_TONEAREST
+
+#ifndef FE_TOWARDZERO
+#define FE_TOWARDZERO 0xC00
+#endif // FE_TOWARDZERO
+
+#ifndef FE_UPWARD
+#define FE_UPWARD 0x800
+#endif // FE_UPWARD
 
 #endif // LLVM_LIBC_FULL_BUILD
 
-- 
GitLab


From 7ec8a333b5fdf1ee78426fe3557c330aa920aa5f Mon Sep 17 00:00:00 2001
From: Dave Lee <davelee.com@gmail.com>
Date: Wed, 8 May 2024 15:07:14 -0700
Subject: [PATCH 0226/1206] [lldb] Display breakpoint locations using display
 name (#90297)

Adds a `show_function_display_name` parameter to
`SymbolContext::DumpStopContext`. This
parameter defaults to false, but `BreakpointLocation::GetDescription`
sets it to true.

This is NFC in mainline lldb, and will be used to modify how Swift
breakpoint locations are printed.
---
 lldb/include/lldb/Symbol/SymbolContext.h      |  1 +
 lldb/source/Breakpoint/BreakpointLocation.cpp |  2 +-
 lldb/source/Core/Address.cpp                  |  5 +++--
 lldb/source/Symbol/SymbolContext.cpp          | 13 +++++++++++--
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/lldb/include/lldb/Symbol/SymbolContext.h b/lldb/include/lldb/Symbol/SymbolContext.h
index bd33a71b46ca..0bc707070f85 100644
--- a/lldb/include/lldb/Symbol/SymbolContext.h
+++ b/lldb/include/lldb/Symbol/SymbolContext.h
@@ -158,6 +158,7 @@ public:
       Stream *s, ExecutionContextScope *exe_scope, const Address &so_addr,
       bool show_fullpaths, bool show_module, bool show_inlined_frames,
       bool show_function_arguments, bool show_function_name,
+      bool show_function_display_name = false,
       std::optional<Stream::HighlightSettings> settings = std::nullopt) const;
 
   /// Get the address range contained within a symbol context.
diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp
index b48ec1398d63..41911fad41c6 100644
--- a/lldb/source/Breakpoint/BreakpointLocation.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocation.cpp
@@ -507,7 +507,7 @@ void BreakpointLocation::GetDescription(Stream *s,
       else
         s->PutCString("where = ");
       sc.DumpStopContext(s, m_owner.GetTarget().GetProcessSP().get(), m_address,
-                         false, true, false, true, true);
+                         false, true, false, true, true, true);
     } else {
       if (sc.module_sp) {
         s->EOL();
diff --git a/lldb/source/Core/Address.cpp b/lldb/source/Core/Address.cpp
index b23398883fa5..5a4751bd5256 100644
--- a/lldb/source/Core/Address.cpp
+++ b/lldb/source/Core/Address.cpp
@@ -645,7 +645,8 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style,
                     pointer_sc.symbol != nullptr) {
                   s->PutCString(": ");
                   pointer_sc.DumpStopContext(s, exe_scope, so_addr, true, false,
-                                             false, true, true, settings);
+                                             false, true, true, false,
+                                             settings);
                 }
               }
             }
@@ -685,7 +686,7 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style,
               sc.DumpStopContext(s, exe_scope, *this, show_fullpaths,
                                  show_module, show_inlined_frames,
                                  show_function_arguments, show_function_name,
-                                 settings);
+                                 false, settings);
             } else {
               // We found a symbol but it was in a different section so it
               // isn't the symbol we should be showing, just show the section
diff --git a/lldb/source/Symbol/SymbolContext.cpp b/lldb/source/Symbol/SymbolContext.cpp
index f368896fbad4..8f26e41d1920 100644
--- a/lldb/source/Symbol/SymbolContext.cpp
+++ b/lldb/source/Symbol/SymbolContext.cpp
@@ -73,6 +73,7 @@ bool SymbolContext::DumpStopContext(
     Stream *s, ExecutionContextScope *exe_scope, const Address &addr,
     bool show_fullpaths, bool show_module, bool show_inlined_frames,
     bool show_function_arguments, bool show_function_name,
+    bool show_function_display_name,
     std::optional<Stream::HighlightSettings> settings) const {
   bool dumped_something = false;
   if (show_module && module_sp) {
@@ -93,6 +94,8 @@ bool SymbolContext::DumpStopContext(
       ConstString name;
       if (!show_function_arguments)
         name = function->GetNameNoArguments();
+      if (!name && show_function_display_name)
+        name = function->GetDisplayName();
       if (!name)
         name = function->GetName();
       if (name)
@@ -146,7 +149,8 @@ bool SymbolContext::DumpStopContext(
         const bool show_function_name = true;
         return inline_parent_sc.DumpStopContext(
             s, exe_scope, inline_parent_addr, show_fullpaths, show_module,
-            show_inlined_frames, show_function_arguments, show_function_name);
+            show_inlined_frames, show_function_arguments, show_function_name,
+            show_function_display_name);
       }
     } else {
       if (line_entry.IsValid()) {
@@ -164,7 +168,12 @@ bool SymbolContext::DumpStopContext(
       dumped_something = true;
       if (symbol->GetType() == eSymbolTypeTrampoline)
         s->PutCString("symbol stub for: ");
-      s->PutCStringColorHighlighted(symbol->GetName().GetStringRef(), settings);
+      ConstString name;
+      if (show_function_display_name)
+        name = symbol->GetDisplayName();
+      if (!name)
+        name = symbol->GetName();
+      s->PutCStringColorHighlighted(name.GetStringRef(), settings);
     }
 
     if (addr.IsValid() && symbol->ValueIsAddress()) {
-- 
GitLab


From 1610eaad39ad882f006f32c29771862a610f8314 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 8 May 2024 15:12:48 -0700
Subject: [PATCH 0227/1206] [memprof] Make Version2 officially available
 (#91541)

---
 llvm/include/llvm/ProfileData/MemProf.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index 4274f2a6849b..3ef6ca8586fb 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -26,7 +26,7 @@ enum IndexedVersion : uint64_t {
   Version0 = 0,
   // Version 1: Added a version field to the header.
   Version1 = 1,
-  // Version 2: Added a call stack table.  Under development.
+  // Version 2: Added a call stack table.
   Version2 = 2,
 };
 
-- 
GitLab


From 64f4ceb09ec3559368dd775330184b5259531cd3 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Wed, 8 May 2024 15:48:40 -0700
Subject: [PATCH 0228/1206] [Inline][PGO] After inline, update InvokeInst
 profile counts in caller and cloned callee (#83809)

A related change is https://reviews.llvm.org/D133121, which correctly
preserves both branch weights and value profiles for invoke instruction.
* If the branch weight of the `invokeinst` specifies taken / not-taken branches, there is no scale.
---
 llvm/include/llvm/IR/Instructions.h           |  3 ++
 llvm/lib/IR/Instructions.cpp                  | 12 ++++++
 llvm/lib/IR/ProfDataUtils.cpp                 | 20 ++++++++++
 llvm/lib/Transforms/Utils/InlineFunction.cpp  | 11 +++++-
 .../Transforms/Inline/update_invoke_prof.ll   | 38 ++++++++++++++-----
 5 files changed, 72 insertions(+), 12 deletions(-)

diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index d7ec3c16bec2..0f7b215b80fd 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -4370,6 +4370,9 @@ public:
 
   unsigned getNumSuccessors() const { return 2; }
 
+  /// Updates profile metadata by scaling it by \p S / \p T.
+  void updateProfWeight(uint64_t S, uint64_t T);
+
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
     return (I->getOpcode() == Instruction::Invoke);
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 4b725610081c..c31d399b01d1 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -927,6 +927,18 @@ LandingPadInst *InvokeInst::getLandingPadInst() const {
   return cast<LandingPadInst>(getUnwindDest()->getFirstNonPHI());
 }
 
+void InvokeInst::updateProfWeight(uint64_t S, uint64_t T) {
+  if (T == 0) {
+    LLVM_DEBUG(dbgs() << "Attempting to update profile weights will result in "
+                         "div by 0. Ignoring. Likely the function "
+                      << getParent()->getParent()->getName()
+                      << " has 0 entry count, and contains call instructions "
+                         "with non-zero prof info.");
+    return;
+  }
+  scaleProfData(*this, S, T);
+}
+
 //===----------------------------------------------------------------------===//
 //                        CallBrInst Implementation
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp
index 3d72418593a7..51e78dc5e6c0 100644
--- a/llvm/lib/IR/ProfDataUtils.cpp
+++ b/llvm/lib/IR/ProfDataUtils.cpp
@@ -46,6 +46,9 @@ constexpr unsigned WeightsIdx = 1;
 // the minimum number of operands for MD_prof nodes with branch weights
 constexpr unsigned MinBWOps = 3;
 
+// the minimum number of operands for MD_prof nodes with value profiles
+constexpr unsigned MinVPOps = 5;
+
 // We may want to add support for other MD_prof types, so provide an abstraction
 // for checking the metadata type.
 bool isTargetMD(const MDNode *ProfData, const char *Name, unsigned MinOps) {
@@ -97,11 +100,25 @@ bool isBranchWeightMD(const MDNode *ProfileData) {
   return isTargetMD(ProfileData, "branch_weights", MinBWOps);
 }
 
+bool isValueProfileMD(const MDNode *ProfileData) {
+  return isTargetMD(ProfileData, "VP", MinVPOps);
+}
+
 bool hasBranchWeightMD(const Instruction &I) {
   auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
   return isBranchWeightMD(ProfileData);
 }
 
+bool hasCountTypeMD(const Instruction &I) {
+  auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
+  // Value profiles record count-type information.
+  if (isValueProfileMD(ProfileData))
+    return true;
+  // Conservatively assume non CallBase instruction only get taken/not-taken
+  // branch probability, so not interpret them as count.
+  return isa<CallBase>(I) && !isBranchWeightMD(ProfileData);
+}
+
 bool hasValidBranchWeightMD(const Instruction &I) {
   return getValidBranchWeightMDNode(I);
 }
@@ -212,6 +229,9 @@ void scaleProfData(Instruction &I, uint64_t S, uint64_t T) {
                         ProfDataName->getString() != "VP"))
     return;
 
+  if (!hasCountTypeMD(I))
+    return;
+
   LLVMContext &C = I.getContext();
 
   MDBuilder MDB(C);
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 1aae561d8817..48bb76eb85e3 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1982,10 +1982,14 @@ void llvm::updateProfileCallee(
   // During inlining ?
   if (VMap) {
     uint64_t CloneEntryCount = PriorEntryCount - NewEntryCount;
-    for (auto Entry : *VMap)
+    for (auto Entry : *VMap) {
       if (isa<CallInst>(Entry.first))
         if (auto *CI = dyn_cast_or_null<CallInst>(Entry.second))
           CI->updateProfWeight(CloneEntryCount, PriorEntryCount);
+      if (isa<InvokeInst>(Entry.first))
+        if (auto *II = dyn_cast_or_null<InvokeInst>(Entry.second))
+          II->updateProfWeight(CloneEntryCount, PriorEntryCount);
+    }
   }
 
   if (EntryDelta) {
@@ -1994,9 +1998,12 @@ void llvm::updateProfileCallee(
     for (BasicBlock &BB : *Callee)
       // No need to update the callsite if it is pruned during inlining.
       if (!VMap || VMap->count(&BB))
-        for (Instruction &I : BB)
+        for (Instruction &I : BB) {
           if (CallInst *CI = dyn_cast<CallInst>(&I))
             CI->updateProfWeight(NewEntryCount, PriorEntryCount);
+          if (InvokeInst *II = dyn_cast<InvokeInst>(&I))
+            II->updateProfWeight(NewEntryCount, PriorEntryCount);
+        }
   }
 }
 
diff --git a/llvm/test/Transforms/Inline/update_invoke_prof.ll b/llvm/test/Transforms/Inline/update_invoke_prof.ll
index 5f09c7cf8fe0..f6b86dfe5bb1 100644
--- a/llvm/test/Transforms/Inline/update_invoke_prof.ll
+++ b/llvm/test/Transforms/Inline/update_invoke_prof.ll
@@ -1,22 +1,31 @@
-; A pre-commit test to show that branch weights and value profiles associated with invoke are not updated.
+; Test that branch weights and value profiles associated with invoke are updated
+; in both caller and callee after inline, but invoke instructions with taken or
+; not taken branch probabilities are not updated.
 ; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -S | FileCheck %s
 
 declare i32 @__gxx_personality_v0(...)
 
 define void @caller(ptr %func) personality ptr @__gxx_personality_v0 !prof !15 {
   call void @callee(ptr %func), !prof !16
+
   ret void
 }
 
-declare void @inner_callee(ptr %func)
+declare void @callee1(ptr %func)
+
+declare void @callee2(ptr %func)
 
 define void @callee(ptr %func) personality ptr @__gxx_personality_v0 !prof !17 {
   invoke void %func()
           to label %next unwind label %lpad, !prof !18
 
 next:
-  invoke void @inner_callee(ptr %func)
-          to label %ret unwind label %lpad, !prof !19
+  invoke void @callee1(ptr %func)
+          to label %cont unwind label %lpad, !prof !19
+
+cont:
+  invoke void @callee2(ptr %func)
+          to label %ret unwind label %lpad, !prof !20
 
 lpad:
   %exn = landingpad {ptr, i32}
@@ -47,18 +56,27 @@ ret:
 !17 = !{!"function_entry_count", i32 1500}
 !18 = !{!"VP", i32 0, i64 1500, i64 123, i64 900, i64 456, i64 600}
 !19 = !{!"branch_weights", i32 1500}
+!20 = !{!"branch_weights", i32 1234, i32 5678}
 
 ; CHECK-LABEL: @caller(
 ; CHECK:  invoke void %func(
 ; CHECK-NEXT: {{.*}} !prof ![[PROF1:[0-9]+]]
-; CHECK:  invoke void @inner_callee(
+; CHECK:  invoke void @callee1(
 ; CHECK-NEXT: {{.*}} !prof ![[PROF2:[0-9]+]]
+; CHECK:  invoke void @callee2(
+; CHECK-NEXT: {{.*}} !prof ![[PROF3:[0-9]+]]
 
 ; CHECK-LABL: @callee(
 ; CHECK:  invoke void %func(
-; CHECK-NEXT: {{.*}} !prof ![[PROF1]] 
-; CHECK:  invoke void @inner_callee(
-; CHECK-NEXT: {{.*}} !prof ![[PROF2]]
+; CHECK-NEXT: {{.*}} !prof ![[PROF4:[0-9]+]]
+; CHECK:  invoke void @callee1(
+; CHECK-NEXT: {{.*}} !prof ![[PROF5:[0-9]+]]
+; CHECK:  invoke void @callee2(
+; CHECK-NEXT: {{.*}} !prof ![[PROF3]]
+
 
-; CHECK: ![[PROF1]] = !{!"VP", i32 0, i64 1500, i64 123, i64 900, i64 456, i64 600}
-; CHECK: ![[PROF2]] = !{!"branch_weights", i32 1500}
+; CHECK: ![[PROF1]] = !{!"VP", i32 0, i64 1000, i64 123, i64 600, i64 456, i64 400}
+; CHECK: ![[PROF2]] = !{!"branch_weights", i32 1000}
+; CHECK: ![[PROF3]] = !{!"branch_weights", i32 1234, i32 5678}
+; CHECK: ![[PROF4]] = !{!"VP", i32 0, i64 500, i64 123, i64 300, i64 456, i64 200}
+; CHECK: ![[PROF5]] = !{!"branch_weights", i32 500}
-- 
GitLab


From 99052c4bdf9593a2e648f2c99cabaab36580898c Mon Sep 17 00:00:00 2001
From: Augusto Noronha <augusto2112@me.com>
Date: Wed, 8 May 2024 15:51:46 -0700
Subject: [PATCH 0229/1206] [gardening][DebugInfo][NFC] Improve comment on
 HashingDISubprogram test (#91543)

---
 llvm/unittests/IR/DebugInfoTest.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index 8847a5759ad9..cac8acbe15a7 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -1195,8 +1195,9 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
   UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
-// Test that the hashing function for DISubprograms produce the same result
-// after replacing the temporary scope.
+// Test that the hashing function for DISubprograms representing methods produce
+// the same result after replacing their scope (the type containing the
+// subprogram) from a temporary DIType with the permanent one.
 TEST(DIBuilder, HashingDISubprogram) {
   LLVMContext Ctx;
   std::unique_ptr<Module> M = std::make_unique<Module>("MyModule", Ctx);
-- 
GitLab


From 2fb377432134b12c3522b1ba8fa35ac4d0f14e1d Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Wed, 8 May 2024 22:55:37 +0000
Subject: [PATCH 0230/1206] Revert "[SLP]Fix PR91467: Look through scalar cast,
 when trying to cast to another type."

This reverts commit 2475efa91d8b4fa8f1a2d16052cb6d14be7d5dc6.

Causes crashes, see comments on https://github.com/llvm/llvm-project/commit/2475efa91d8b4fa8f1a2d16052cb6d14be7d5dc6.
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp          | 6 +-----
 .../SLPVectorizer/AArch64/gather-with-minbith-user.ll    | 9 ++++++++-
 .../SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll  | 7 ++++++-
 .../SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll      | 3 ++-
 4 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index cc9219ca02cf..98561f9ca044 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11419,12 +11419,8 @@ Value *BoUpSLP::gather(ArrayRef<Value *> VL, Value *Root, Type *ScalarTy) {
     if (Scalar->getType() != Ty) {
       assert(Scalar->getType()->isIntegerTy() && Ty->isIntegerTy() &&
              "Expected integer types only.");
-      Value *V = Scalar;
-      if (auto *CI = dyn_cast<CastInst>(Scalar);
-          isa_and_nonnull<SExtInst, ZExtInst>(CI))
-        V = CI->getOperand(0);
       Scalar = Builder.CreateIntCast(
-          V, Ty, !isKnownNonNegative(Scalar, SimplifyQuery(*DL)));
+          Scalar, Ty, !isKnownNonNegative(Scalar, SimplifyQuery(*DL)));
     }
 
     Vec = Builder.CreateInsertElement(Vec, Scalar, Builder.getInt32(Pos));
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll
index 3ebe920d1734..76bb882171b1 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll
@@ -5,7 +5,14 @@ define void @h() {
 ; CHECK-LABEL: define void @h() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, ptr null, i64 16
-; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr [[ARRAYIDX2]], align 2
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 0 to i1
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 poison, i1 false, i1 false, i1 false>, i1 [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <8 x i1> [[TMP0]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = add <8 x i1> [[TMP0]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i1> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i1> [[TMP5]] to <8 x i16>
+; CHECK-NEXT:    store <8 x i16> [[TMP4]], ptr [[ARRAYIDX2]], align 2
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll
index 6404cf4a2cd1..2ab6e919c23b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll
@@ -5,7 +5,12 @@ define void @h() {
 ; CHECK-LABEL: define void @h() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, ptr null, i64 16
-; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr [[ARRAYIDX2]], align 2
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 0 to i1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 poison, i1 false, i1 false, i1 false>, i1 [[TMP0]], i32 4
+; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i1> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i1> zeroinitializer, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i1> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    store <8 x i16> [[TMP4]], ptr [[ARRAYIDX2]], align 2
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll
index 3c8e98485ffc..1bb87bf6205f 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll
@@ -4,9 +4,10 @@
 define void @test(ptr %a, i8 %0, i16 %b.promoted.i) {
 ; CHECK-LABEL: define void @test(
 ; CHECK-SAME: ptr [[A:%.*]], i8 [[TMP0:%.*]], i16 [[B_PROMOTED_I:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP0]] to i128
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i16> poison, i16 [[B_PROMOTED_I]], i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = zext i8 [[TMP0]] to i16
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP2]] to i16
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i16> poison, i16 [[TMP5]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[TMP6]], <4 x i16> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i16> [[TMP4]], [[TMP7]]
-- 
GitLab


From c0b5a96ac44a70bdc0b138e117e3fea1b49189f5 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 8 May 2024 15:55:10 -0700
Subject: [PATCH 0231/1206] [RISCV] Add tests where we could use Zbs
 instructions in constant materialization. NFC

---
 llvm/test/CodeGen/RISCV/imm.ll                | 116 +++++++++++++++
 llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll | 137 ++++++++++++++++++
 2 files changed, 253 insertions(+)

diff --git a/llvm/test/CodeGen/RISCV/imm.ll b/llvm/test/CodeGen/RISCV/imm.ll
index 9e356a93526c..6456401dbb86 100644
--- a/llvm/test/CodeGen/RISCV/imm.ll
+++ b/llvm/test/CodeGen/RISCV/imm.ll
@@ -3994,3 +3994,119 @@ define i64 @imm64_same_lo_hi_negative() nounwind {
 ; RV64-REMAT-NEXT:    ret
   ret i64 9259542123273814144 ; 0x8080808080808080
 }
+
+define i64 @imm64_0x8000080000000() {
+; RV32I-LABEL: imm64_0x8000080000000:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a0, 524288
+; RV32I-NEXT:    lui a1, 128
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: imm64_0x8000080000000:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a0, 256
+; RV64I-NEXT:    addiw a0, a0, 1
+; RV64I-NEXT:    slli a0, a0, 31
+; RV64I-NEXT:    ret
+;
+; RV64IZBA-LABEL: imm64_0x8000080000000:
+; RV64IZBA:       # %bb.0:
+; RV64IZBA-NEXT:    lui a0, 256
+; RV64IZBA-NEXT:    addiw a0, a0, 1
+; RV64IZBA-NEXT:    slli a0, a0, 31
+; RV64IZBA-NEXT:    ret
+;
+; RV64IZBB-LABEL: imm64_0x8000080000000:
+; RV64IZBB:       # %bb.0:
+; RV64IZBB-NEXT:    lui a0, 256
+; RV64IZBB-NEXT:    addiw a0, a0, 1
+; RV64IZBB-NEXT:    slli a0, a0, 31
+; RV64IZBB-NEXT:    ret
+;
+; RV64IZBS-LABEL: imm64_0x8000080000000:
+; RV64IZBS:       # %bb.0:
+; RV64IZBS-NEXT:    lui a0, 256
+; RV64IZBS-NEXT:    addiw a0, a0, 1
+; RV64IZBS-NEXT:    slli a0, a0, 31
+; RV64IZBS-NEXT:    ret
+;
+; RV64IXTHEADBB-LABEL: imm64_0x8000080000000:
+; RV64IXTHEADBB:       # %bb.0:
+; RV64IXTHEADBB-NEXT:    lui a0, 256
+; RV64IXTHEADBB-NEXT:    addiw a0, a0, 1
+; RV64IXTHEADBB-NEXT:    slli a0, a0, 31
+; RV64IXTHEADBB-NEXT:    ret
+;
+; RV32-REMAT-LABEL: imm64_0x8000080000000:
+; RV32-REMAT:       # %bb.0:
+; RV32-REMAT-NEXT:    lui a0, 524288
+; RV32-REMAT-NEXT:    lui a1, 128
+; RV32-REMAT-NEXT:    ret
+;
+; RV64-REMAT-LABEL: imm64_0x8000080000000:
+; RV64-REMAT:       # %bb.0:
+; RV64-REMAT-NEXT:    lui a0, 256
+; RV64-REMAT-NEXT:    addiw a0, a0, 1
+; RV64-REMAT-NEXT:    slli a0, a0, 31
+; RV64-REMAT-NEXT:    ret
+  ret i64 2251801961168896 ; 0x8000080000000
+}
+
+define i64 @imm64_0x10000100000000() {
+; RV32I-LABEL: imm64_0x10000100000000:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 256
+; RV32I-NEXT:    addi a1, a1, 1
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: imm64_0x10000100000000:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a0, 256
+; RV64I-NEXT:    addi a0, a0, 1
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    ret
+;
+; RV64IZBA-LABEL: imm64_0x10000100000000:
+; RV64IZBA:       # %bb.0:
+; RV64IZBA-NEXT:    lui a0, 256
+; RV64IZBA-NEXT:    addi a0, a0, 1
+; RV64IZBA-NEXT:    slli a0, a0, 32
+; RV64IZBA-NEXT:    ret
+;
+; RV64IZBB-LABEL: imm64_0x10000100000000:
+; RV64IZBB:       # %bb.0:
+; RV64IZBB-NEXT:    lui a0, 256
+; RV64IZBB-NEXT:    addi a0, a0, 1
+; RV64IZBB-NEXT:    slli a0, a0, 32
+; RV64IZBB-NEXT:    ret
+;
+; RV64IZBS-LABEL: imm64_0x10000100000000:
+; RV64IZBS:       # %bb.0:
+; RV64IZBS-NEXT:    lui a0, 256
+; RV64IZBS-NEXT:    addi a0, a0, 1
+; RV64IZBS-NEXT:    slli a0, a0, 32
+; RV64IZBS-NEXT:    ret
+;
+; RV64IXTHEADBB-LABEL: imm64_0x10000100000000:
+; RV64IXTHEADBB:       # %bb.0:
+; RV64IXTHEADBB-NEXT:    lui a0, 256
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 1
+; RV64IXTHEADBB-NEXT:    slli a0, a0, 32
+; RV64IXTHEADBB-NEXT:    ret
+;
+; RV32-REMAT-LABEL: imm64_0x10000100000000:
+; RV32-REMAT:       # %bb.0:
+; RV32-REMAT-NEXT:    lui a1, 256
+; RV32-REMAT-NEXT:    addi a1, a1, 1
+; RV32-REMAT-NEXT:    li a0, 0
+; RV32-REMAT-NEXT:    ret
+;
+; RV64-REMAT-LABEL: imm64_0x10000100000000:
+; RV64-REMAT:       # %bb.0:
+; RV64-REMAT-NEXT:    lui a0, 256
+; RV64-REMAT-NEXT:    addi a0, a0, 1
+; RV64-REMAT-NEXT:    slli a0, a0, 32
+; RV64-REMAT-NEXT:    ret
+  ret i64 4503603922337792 ; 0x10000100000000
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll
index 0ef17ca964db..c5bb7289e448 100644
--- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll
@@ -2562,3 +2562,140 @@ define i64 @imm64_same_lo_hi_optsize() nounwind optsize {
 ; RV64IXTHEADBB-NEXT:    ret
   ret i64 1157442765409226768 ; 0x0101010101010101
 }
+; Hi and lo are the same and also negative.
+define i64 @imm64_same_lo_hi_negative() nounwind {
+; RV64-NOPOOL-LABEL: imm64_same_lo_hi_negative:
+; RV64-NOPOOL:       # %bb.0:
+; RV64-NOPOOL-NEXT:    lui a0, 983297
+; RV64-NOPOOL-NEXT:    slli a0, a0, 4
+; RV64-NOPOOL-NEXT:    addi a0, a0, 257
+; RV64-NOPOOL-NEXT:    slli a0, a0, 16
+; RV64-NOPOOL-NEXT:    addi a0, a0, 257
+; RV64-NOPOOL-NEXT:    slli a0, a0, 15
+; RV64-NOPOOL-NEXT:    addi a0, a0, 128
+; RV64-NOPOOL-NEXT:    ret
+;
+; RV64I-POOL-LABEL: imm64_same_lo_hi_negative:
+; RV64I-POOL:       # %bb.0:
+; RV64I-POOL-NEXT:    lui a0, %hi(.LCPI65_0)
+; RV64I-POOL-NEXT:    ld a0, %lo(.LCPI65_0)(a0)
+; RV64I-POOL-NEXT:    ret
+;
+; RV64IZBA-LABEL: imm64_same_lo_hi_negative:
+; RV64IZBA:       # %bb.0:
+; RV64IZBA-NEXT:    lui a0, 526344
+; RV64IZBA-NEXT:    addi a0, a0, 128
+; RV64IZBA-NEXT:    slli a1, a0, 32
+; RV64IZBA-NEXT:    add.uw a0, a0, a1
+; RV64IZBA-NEXT:    ret
+;
+; RV64IZBB-LABEL: imm64_same_lo_hi_negative:
+; RV64IZBB:       # %bb.0:
+; RV64IZBB-NEXT:    lui a0, 983297
+; RV64IZBB-NEXT:    slli a0, a0, 4
+; RV64IZBB-NEXT:    addi a0, a0, 257
+; RV64IZBB-NEXT:    slli a0, a0, 16
+; RV64IZBB-NEXT:    addi a0, a0, 257
+; RV64IZBB-NEXT:    slli a0, a0, 15
+; RV64IZBB-NEXT:    addi a0, a0, 128
+; RV64IZBB-NEXT:    ret
+;
+; RV64IZBS-LABEL: imm64_same_lo_hi_negative:
+; RV64IZBS:       # %bb.0:
+; RV64IZBS-NEXT:    lui a0, 983297
+; RV64IZBS-NEXT:    slli a0, a0, 4
+; RV64IZBS-NEXT:    addi a0, a0, 257
+; RV64IZBS-NEXT:    slli a0, a0, 16
+; RV64IZBS-NEXT:    addi a0, a0, 257
+; RV64IZBS-NEXT:    slli a0, a0, 15
+; RV64IZBS-NEXT:    addi a0, a0, 128
+; RV64IZBS-NEXT:    ret
+;
+; RV64IXTHEADBB-LABEL: imm64_same_lo_hi_negative:
+; RV64IXTHEADBB:       # %bb.0:
+; RV64IXTHEADBB-NEXT:    lui a0, 983297
+; RV64IXTHEADBB-NEXT:    slli a0, a0, 4
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 257
+; RV64IXTHEADBB-NEXT:    slli a0, a0, 16
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 257
+; RV64IXTHEADBB-NEXT:    slli a0, a0, 15
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 128
+; RV64IXTHEADBB-NEXT:    ret
+  ret i64 9259542123273814144 ; 0x8080808080808080
+}
+
+define i64 @imm64_0x8000080000000() {
+; RV64I-LABEL: imm64_0x8000080000000:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a0, 256
+; RV64I-NEXT:    addiw a0, a0, 1
+; RV64I-NEXT:    slli a0, a0, 31
+; RV64I-NEXT:    ret
+;
+; RV64IZBA-LABEL: imm64_0x8000080000000:
+; RV64IZBA:       # %bb.0:
+; RV64IZBA-NEXT:    lui a0, 256
+; RV64IZBA-NEXT:    addiw a0, a0, 1
+; RV64IZBA-NEXT:    slli a0, a0, 31
+; RV64IZBA-NEXT:    ret
+;
+; RV64IZBB-LABEL: imm64_0x8000080000000:
+; RV64IZBB:       # %bb.0:
+; RV64IZBB-NEXT:    lui a0, 256
+; RV64IZBB-NEXT:    addiw a0, a0, 1
+; RV64IZBB-NEXT:    slli a0, a0, 31
+; RV64IZBB-NEXT:    ret
+;
+; RV64IZBS-LABEL: imm64_0x8000080000000:
+; RV64IZBS:       # %bb.0:
+; RV64IZBS-NEXT:    lui a0, 256
+; RV64IZBS-NEXT:    addiw a0, a0, 1
+; RV64IZBS-NEXT:    slli a0, a0, 31
+; RV64IZBS-NEXT:    ret
+;
+; RV64IXTHEADBB-LABEL: imm64_0x8000080000000:
+; RV64IXTHEADBB:       # %bb.0:
+; RV64IXTHEADBB-NEXT:    lui a0, 256
+; RV64IXTHEADBB-NEXT:    addiw a0, a0, 1
+; RV64IXTHEADBB-NEXT:    slli a0, a0, 31
+; RV64IXTHEADBB-NEXT:    ret
+  ret i64 2251801961168896 ; 0x8000080000000
+}
+
+define i64 @imm64_0x10000100000000() {
+; RV64I-LABEL: imm64_0x10000100000000:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a0, 256
+; RV64I-NEXT:    addi a0, a0, 1
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    ret
+;
+; RV64IZBA-LABEL: imm64_0x10000100000000:
+; RV64IZBA:       # %bb.0:
+; RV64IZBA-NEXT:    lui a0, 256
+; RV64IZBA-NEXT:    addi a0, a0, 1
+; RV64IZBA-NEXT:    slli a0, a0, 32
+; RV64IZBA-NEXT:    ret
+;
+; RV64IZBB-LABEL: imm64_0x10000100000000:
+; RV64IZBB:       # %bb.0:
+; RV64IZBB-NEXT:    lui a0, 256
+; RV64IZBB-NEXT:    addi a0, a0, 1
+; RV64IZBB-NEXT:    slli a0, a0, 32
+; RV64IZBB-NEXT:    ret
+;
+; RV64IZBS-LABEL: imm64_0x10000100000000:
+; RV64IZBS:       # %bb.0:
+; RV64IZBS-NEXT:    lui a0, 256
+; RV64IZBS-NEXT:    addi a0, a0, 1
+; RV64IZBS-NEXT:    slli a0, a0, 32
+; RV64IZBS-NEXT:    ret
+;
+; RV64IXTHEADBB-LABEL: imm64_0x10000100000000:
+; RV64IXTHEADBB:       # %bb.0:
+; RV64IXTHEADBB-NEXT:    lui a0, 256
+; RV64IXTHEADBB-NEXT:    addi a0, a0, 1
+; RV64IXTHEADBB-NEXT:    slli a0, a0, 32
+; RV64IXTHEADBB-NEXT:    ret
+  ret i64 4503603922337792 ; 0x10000100000000
+}
-- 
GitLab


From 36d8b37dfaa95b8b4e21cb8269fefb62e1f59c2f Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 8 May 2024 16:20:05 -0700
Subject: [PATCH 0232/1206] [RISCV] Add another missed Zbs constant
 materialization test. NFC

This can be LI+BCLRI+BCLRI.
---
 llvm/test/CodeGen/RISCV/imm.ll                | 67 +++++++++++++++++++
 llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll | 43 ++++++++++++
 2 files changed, 110 insertions(+)

diff --git a/llvm/test/CodeGen/RISCV/imm.ll b/llvm/test/CodeGen/RISCV/imm.ll
index 6456401dbb86..0dc5c7ceb500 100644
--- a/llvm/test/CodeGen/RISCV/imm.ll
+++ b/llvm/test/CodeGen/RISCV/imm.ll
@@ -4110,3 +4110,70 @@ define i64 @imm64_0x10000100000000() {
 ; RV64-REMAT-NEXT:    ret
   ret i64 4503603922337792 ; 0x10000100000000
 }
+
+define i64 @imm64_0xFF7FFFFF7FFFFFFE() {
+; RV32I-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a0, 524288
+; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    lui a1, 1046528
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a0, 1044480
+; RV64I-NEXT:    addiw a0, a0, -1
+; RV64I-NEXT:    slli a0, a0, 31
+; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    ret
+;
+; RV64IZBA-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV64IZBA:       # %bb.0:
+; RV64IZBA-NEXT:    lui a0, 1044480
+; RV64IZBA-NEXT:    addiw a0, a0, -1
+; RV64IZBA-NEXT:    slli a0, a0, 31
+; RV64IZBA-NEXT:    addi a0, a0, -1
+; RV64IZBA-NEXT:    ret
+;
+; RV64IZBB-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV64IZBB:       # %bb.0:
+; RV64IZBB-NEXT:    lui a0, 1044480
+; RV64IZBB-NEXT:    addiw a0, a0, -1
+; RV64IZBB-NEXT:    slli a0, a0, 31
+; RV64IZBB-NEXT:    addi a0, a0, -1
+; RV64IZBB-NEXT:    ret
+;
+; RV64IZBS-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV64IZBS:       # %bb.0:
+; RV64IZBS-NEXT:    lui a0, 1044480
+; RV64IZBS-NEXT:    addiw a0, a0, -1
+; RV64IZBS-NEXT:    slli a0, a0, 31
+; RV64IZBS-NEXT:    addi a0, a0, -1
+; RV64IZBS-NEXT:    ret
+;
+; RV64IXTHEADBB-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV64IXTHEADBB:       # %bb.0:
+; RV64IXTHEADBB-NEXT:    lui a0, 1044480
+; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1
+; RV64IXTHEADBB-NEXT:    slli a0, a0, 31
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1
+; RV64IXTHEADBB-NEXT:    ret
+;
+; RV32-REMAT-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV32-REMAT:       # %bb.0:
+; RV32-REMAT-NEXT:    lui a0, 524288
+; RV32-REMAT-NEXT:    addi a0, a0, -1
+; RV32-REMAT-NEXT:    lui a1, 1046528
+; RV32-REMAT-NEXT:    addi a1, a1, -1
+; RV32-REMAT-NEXT:    ret
+;
+; RV64-REMAT-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV64-REMAT:       # %bb.0:
+; RV64-REMAT-NEXT:    lui a0, 1044480
+; RV64-REMAT-NEXT:    addiw a0, a0, -1
+; RV64-REMAT-NEXT:    slli a0, a0, 31
+; RV64-REMAT-NEXT:    addi a0, a0, -1
+; RV64-REMAT-NEXT:    ret
+  ret i64 -36028799166447617 ; 0xFF7FFFFF7FFFFFFE
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll
index c5bb7289e448..bac4bb9ce6f1 100644
--- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll
@@ -2699,3 +2699,46 @@ define i64 @imm64_0x10000100000000() {
 ; RV64IXTHEADBB-NEXT:    ret
   ret i64 4503603922337792 ; 0x10000100000000
 }
+
+define i64 @imm64_0xFF7FFFFF7FFFFFFE() {
+; RV64I-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a0, 1044480
+; RV64I-NEXT:    addiw a0, a0, -1
+; RV64I-NEXT:    slli a0, a0, 31
+; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    ret
+;
+; RV64IZBA-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV64IZBA:       # %bb.0:
+; RV64IZBA-NEXT:    lui a0, 1044480
+; RV64IZBA-NEXT:    addiw a0, a0, -1
+; RV64IZBA-NEXT:    slli a0, a0, 31
+; RV64IZBA-NEXT:    addi a0, a0, -1
+; RV64IZBA-NEXT:    ret
+;
+; RV64IZBB-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV64IZBB:       # %bb.0:
+; RV64IZBB-NEXT:    lui a0, 1044480
+; RV64IZBB-NEXT:    addiw a0, a0, -1
+; RV64IZBB-NEXT:    slli a0, a0, 31
+; RV64IZBB-NEXT:    addi a0, a0, -1
+; RV64IZBB-NEXT:    ret
+;
+; RV64IZBS-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV64IZBS:       # %bb.0:
+; RV64IZBS-NEXT:    lui a0, 1044480
+; RV64IZBS-NEXT:    addiw a0, a0, -1
+; RV64IZBS-NEXT:    slli a0, a0, 31
+; RV64IZBS-NEXT:    addi a0, a0, -1
+; RV64IZBS-NEXT:    ret
+;
+; RV64IXTHEADBB-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
+; RV64IXTHEADBB:       # %bb.0:
+; RV64IXTHEADBB-NEXT:    lui a0, 1044480
+; RV64IXTHEADBB-NEXT:    addiw a0, a0, -1
+; RV64IXTHEADBB-NEXT:    slli a0, a0, 31
+; RV64IXTHEADBB-NEXT:    addi a0, a0, -1
+; RV64IXTHEADBB-NEXT:    ret
+  ret i64 -36028799166447617 ; 0xFF7FFFFF7FFFFFFE
+}
-- 
GitLab


From 1710c8cf0f8def4984893e9dd646579de5528d95 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Wed, 8 May 2024 16:48:14 -0700
Subject: [PATCH 0233/1206] [flang] Lowering changes for assigning dummy_scope
 to hlfir.declare. (#90989)

The lowering produces fir.dummy_scope operation if the current
function has dummy arguments. Each hlfir.declare generated
for a dummy argument is then using the result of fir.dummy_scope
as its dummy_scope operand. This is only done for HLFIR.

I was not able to find a reliable way to identify dummy symbols
in `genDeclareSymbol`, so I added a set of registered dummy symbols
that is alive during the variables instantiation for the current
function. The set is initialized during the mapping of the dummy
argument symbols to their MLIR values. It is reset right after
all variables are instantiated - this is done to avoid generating
hlfir.declare operations with dummy_scope for the clones of
the dummy symbols (e.g. this happens with OpenMP privatization).

If this can be done in a cleaner way, please advise.
---
 flang/include/flang/Lower/AbstractConverter.h |  12 ++
 .../flang/Optimizer/Builder/HLFIRTools.h      |   1 +
 .../include/flang/Optimizer/HLFIR/HLFIROps.td |   1 +
 flang/lib/Lower/Bridge.cpp                    |  73 +++++++++++--
 flang/lib/Lower/ConvertArrayConstructor.cpp   |   2 +-
 flang/lib/Lower/ConvertExprToHLFIR.cpp        |   3 +-
 flang/lib/Lower/ConvertVariable.cpp           |  20 +++-
 flang/lib/Lower/OpenACC.cpp                   |  22 ++--
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  12 +-
 flang/lib/Optimizer/Builder/HLFIRTools.cpp    |  19 ++--
 .../Optimizer/Builder/TemporaryStorage.cpp    |   3 +-
 flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp     |   4 +-
 .../HLFIR/Transforms/BufferizeHLFIR.cpp       |  15 +--
 flang/test/Fir/dispatch.f90                   |   2 +-
 flang/test/HLFIR/assumed-type-actual-args.f90 |  30 +++--
 .../assumed_shape_with_value_keyword.f90      |  20 ++--
 flang/test/HLFIR/boxchar_emboxing.f90         |   4 +-
 flang/test/HLFIR/c_ptr_byvalue.f90            |   3 +-
 flang/test/HLFIR/call_with_poly_dummy.f90     |   3 +-
 flang/test/HLFIR/optional_dummy.f90           |   2 +-
 .../order_assignments/where-scheduling.f90    |  10 +-
 flang/test/Lower/CUDA/cuda-data-attribute.cuf |   8 +-
 .../HLFIR/actual_target_for_dummy_pointer.f90 |  22 ++--
 .../allocatable-and-pointer-status-change.f90 |  12 +-
 .../Lower/HLFIR/allocatables-and-pointers.f90 |  18 +--
 .../HLFIR/array-ctor-as-elemental-nested.f90  |   4 +-
 .../Lower/HLFIR/array-ctor-as-elemental.f90   |  11 +-
 .../HLFIR/array-ctor-as-inlined-temp.f90      |  12 +-
 flang/test/Lower/HLFIR/array-ctor-index.f90   |   8 +-
 .../Lower/HLFIR/assignment-intrinsics.f90     |  46 ++++----
 .../HLFIR/assumed-rank-iface-alloc-ptr.f90    |  10 +-
 flang/test/Lower/HLFIR/assumed-rank-iface.f90 |  18 +--
 flang/test/Lower/HLFIR/binary-ops.f90         |  84 +++++++-------
 .../test/Lower/HLFIR/bindc-value-derived.f90  |   4 +-
 .../call-sequence-associated-descriptors.f90  |  16 +--
 .../test/Lower/HLFIR/calls-assumed-shape.f90  |  14 +--
 .../Lower/HLFIR/calls-constant-expr-arg.f90   |   4 +-
 flang/test/Lower/HLFIR/calls-f77.f90          |  14 +--
 flang/test/Lower/HLFIR/calls-optional.f90     |  14 +--
 .../Lower/HLFIR/calls-percent-val-ref.f90     |  12 +-
 .../HLFIR/calls-poly-to-assumed-type.f90      |   2 +-
 flang/test/Lower/HLFIR/char_extremum.f03      |  40 +++----
 flang/test/Lower/HLFIR/charconvert.f90        |   8 +-
 .../Lower/HLFIR/convert-mbox-to-value.f90     |  16 +--
 .../Lower/HLFIR/convert-variable-block.f90    |   2 +-
 flang/test/Lower/HLFIR/convert-variable.f90   |  20 ++--
 flang/test/Lower/HLFIR/cray-pointers.f90      |  10 +-
 flang/test/Lower/HLFIR/custom-intrinsic.f90   | 103 +++++++++---------
 .../Lower/HLFIR/designators-component-ref.f90 |   8 +-
 flang/test/Lower/HLFIR/designators.f90        |  42 +++----
 flang/test/Lower/HLFIR/dot_product.f90        |   4 +-
 .../test/Lower/HLFIR/elemental-array-ops.f90  |  12 +-
 .../HLFIR/elemental-polymorphic-merge.f90     |   8 +-
 .../HLFIR/elemental-user-procedure-ref.f90    |   6 +-
 flang/test/Lower/HLFIR/expr-addr.f90          |   2 +-
 flang/test/Lower/HLFIR/expr-box.f90           |   2 +-
 flang/test/Lower/HLFIR/expr-value.f90         |   2 +-
 .../ignore-rank-unlimited-polymorphic.f90     |  10 +-
 .../Lower/HLFIR/implicit-type-conversion.f90  |  28 ++---
 .../intentout-allocatable-components.f90      |   4 +-
 .../test/Lower/HLFIR/internal-procedures.f90  |   2 +-
 .../HLFIR/intrinsic-dynamically-optional.f90  |   4 +-
 flang/test/Lower/HLFIR/issue80884.f90         |   4 +-
 flang/test/Lower/HLFIR/maxloc.f90             |  11 +-
 flang/test/Lower/HLFIR/minloc.f90             |  11 +-
 flang/test/Lower/HLFIR/procedure-pointer.f90  |   6 +-
 .../test/Lower/HLFIR/statement-functions.f90  |   2 +-
 .../Lower/HLFIR/structure-constructor.f90     |  16 +--
 flang/test/Lower/HLFIR/transformational.f90   |   2 +-
 flang/test/Lower/HLFIR/transpose.f90          |  10 +-
 flang/test/Lower/HLFIR/unary-ops.f90          |  12 +-
 .../Lower/HLFIR/user-defined-assignment.f90   |  38 +++----
 .../Lower/HLFIR/vector-subscript-as-value.f90 |   6 +-
 .../Intrinsics/associated-proc-pointers.f90   |  12 +-
 .../test/Lower/Intrinsics/c_f_procpointer.f90 |   8 +-
 .../Intrinsics/c_funloc-proc-pointers.f90     |   4 +-
 flang/test/Lower/Intrinsics/c_ptr_eq_ne.f90   |   8 +-
 .../execute_command_line-optional.f90         |  11 +-
 .../Lower/Intrinsics/execute_command_line.f90 |  16 +--
 flang/test/Lower/Intrinsics/ieee_logb.f90     |   2 +-
 flang/test/Lower/Intrinsics/product.f90       |   2 +-
 flang/test/Lower/Intrinsics/signal.f90        |   2 +-
 flang/test/Lower/Intrinsics/sizeof.f90        |   4 +-
 flang/test/Lower/Intrinsics/sum.f90           |   2 +-
 .../test/Lower/Intrinsics/system-optional.f90 |   5 +-
 flang/test/Lower/Intrinsics/system.f90        |  10 +-
 .../Lower/OpenACC/acc-atomic-update-array.f90 |  18 +--
 flang/test/Lower/OpenACC/acc-bounds.f90       |  12 +-
 flang/test/Lower/OpenACC/acc-declare.f90      |  18 +--
 flang/test/Lower/OpenACC/acc-loop-exit.f90    |   6 +-
 flang/test/Lower/OpenACC/acc-private.f90      |  10 +-
 flang/test/Lower/OpenACC/acc-reduction.f90    |  10 +-
 .../Lower/OpenMP/allocatable-array-bounds.f90 |   2 +-
 flang/test/Lower/OpenMP/array-bounds.f90      |   4 +-
 flang/test/Lower/OpenMP/flush.f90             |  14 +--
 .../parallel-firstprivate-clause-scalar.f90   |  42 +++----
 .../parallel-lastprivate-clause-scalar.f90    |  18 +--
 .../OpenMP/parallel-private-clause-fixes.f90  |   6 +-
 .../OpenMP/parallel-private-clause-str.f90    |   4 +-
 .../test/Lower/OpenMP/parallel-reduction3.f90 |   2 +-
 .../OpenMP/parallel-wsloop-firstpriv.f90      |   6 +-
 flang/test/Lower/OpenMP/parallel-wsloop.f90   |  22 ++--
 flang/test/Lower/OpenMP/sections.f90          |   2 +-
 flang/test/Lower/OpenMP/simd.f90              |  14 +--
 flang/test/Lower/OpenMP/single.f90            |  12 +-
 flang/test/Lower/OpenMP/target.f90            |   4 +-
 .../wsloop-reduction-array-assumed-shape.f90  |   2 +-
 .../OpenMP/wsloop-reduction-iand-byref.f90    |   2 +-
 .../Lower/OpenMP/wsloop-reduction-iand.f90    |   2 +-
 .../OpenMP/wsloop-reduction-ieor-byref.f90    |   2 +-
 .../Lower/OpenMP/wsloop-reduction-ieor.f90    |   2 +-
 .../OpenMP/wsloop-reduction-ior-byref.f90     |   2 +-
 .../Lower/OpenMP/wsloop-reduction-ior.f90     |   2 +-
 .../wsloop-reduction-logical-and-byref.f90    |   6 +-
 .../OpenMP/wsloop-reduction-logical-and.f90   |   6 +-
 .../wsloop-reduction-logical-eqv-byref.f90    |   6 +-
 .../OpenMP/wsloop-reduction-logical-eqv.f90   |   6 +-
 .../wsloop-reduction-logical-neqv-byref.f90   |   6 +-
 .../OpenMP/wsloop-reduction-logical-neqv.f90  |   6 +-
 .../wsloop-reduction-logical-or-byref.f90     |   6 +-
 .../OpenMP/wsloop-reduction-logical-or.f90    |   6 +-
 .../OpenMP/wsloop-reduction-max-byref.f90     |   4 +-
 .../wsloop-reduction-max-hlfir-byref.f90      |   2 +-
 .../OpenMP/wsloop-reduction-max-hlfir.f90     |   2 +-
 .../Lower/OpenMP/wsloop-reduction-max.f90     |   4 +-
 .../OpenMP/wsloop-reduction-min-byref.f90     |   4 +-
 .../Lower/OpenMP/wsloop-reduction-min.f90     |   4 +-
 flang/test/Lower/allocatable-polymorphic.f90  |   6 +-
 flang/test/Lower/array-expression.f90         |   2 +-
 flang/test/Lower/character-substrings.f90     |   4 +-
 flang/test/Lower/charconvert.f90              |   8 +-
 flang/test/Lower/dispatch.f90                 |  30 ++---
 flang/test/Lower/do_loop.f90                  |   1 +
 flang/test/Lower/pointer-references.f90       |   2 +-
 flang/test/Lower/polymorphic.f90              |   2 +-
 flang/test/Lower/select-type.f90              |   2 +-
 .../structure-constructors-alloc-comp.f90     |   6 +-
 137 files changed, 806 insertions(+), 696 deletions(-)

diff --git a/flang/include/flang/Lower/AbstractConverter.h b/flang/include/flang/Lower/AbstractConverter.h
index 1cb6bcb1f5d2..0bc68de6938d 100644
--- a/flang/include/flang/Lower/AbstractConverter.h
+++ b/flang/include/flang/Lower/AbstractConverter.h
@@ -219,6 +219,18 @@ public:
   /// function.
   virtual void bindHostAssocTuple(mlir::Value val) = 0;
 
+  /// Returns fir.dummy_scope operation's result value to be used
+  /// as dummy_scope operand of hlfir.declare operations for the dummy
+  /// arguments of this function.
+  virtual mlir::Value dummyArgsScopeValue() const = 0;
+
+  /// Returns true if the given symbol is a dummy argument of this function.
+  /// Note that it returns false for all the symbols after all the variables
+  /// are instantiated for this function, i.e. it can only be used reliably
+  /// during the instatiation of the variables.
+  virtual bool
+  isRegisteredDummySymbol(Fortran::semantics::SymbolRef symRef) const = 0;
+
   //===--------------------------------------------------------------------===//
   // Types
   //===--------------------------------------------------------------------===//
diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index cf7df38b1cdf..6cc8e71b3b18 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -238,6 +238,7 @@ fir::FortranVariableOpInterface
 genDeclare(mlir::Location loc, fir::FirOpBuilder &builder,
            const fir::ExtendedValue &exv, llvm::StringRef name,
            fir::FortranVariableFlagsAttr flags,
+           mlir::Value dummyScope = nullptr,
            fir::CUDADataAttributeAttr cudaAttr = {});
 
 /// Generate an hlfir.associate to build a variable from an expression value.
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
index ee3c26800ae3..9558a6832972 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
@@ -104,6 +104,7 @@ def hlfir_DeclareOp : hlfir_Op<"declare", [AttrSizedOperandSegments,
   let builders = [
     OpBuilder<(ins "mlir::Value":$memref, "llvm::StringRef":$uniq_name,
       CArg<"mlir::Value", "{}">:$shape, CArg<"mlir::ValueRange", "{}">:$typeparams,
+      CArg<"mlir::Value", "{}">:$dummy_scope,
       CArg<"fir::FortranVariableFlagsAttr", "{}">:$fortran_attrs,
       CArg<"fir::CUDADataAttributeAttr", "{}">:$cuda_attr)>];
 
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index b0fc26332651..4902886712e9 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -900,6 +900,16 @@ public:
     hostAssocTuple = val;
   }
 
+  mlir::Value dummyArgsScopeValue() const override final {
+    return dummyArgsScope;
+  }
+
+  bool isRegisteredDummySymbol(
+      Fortran::semantics::SymbolRef symRef) const override final {
+    auto *sym = &*symRef;
+    return registeredDummySymbols.contains(sym);
+  }
+
   void registerTypeInfo(mlir::Location loc,
                         Fortran::lower::SymbolRef typeInfoSym,
                         const Fortran::semantics::DerivedTypeSpec &typeSpec,
@@ -1145,10 +1155,11 @@ private:
   /// yet. The final mapping will be done using this pre-mapping in
   /// Fortran::lower::mapSymbolAttributes.
   bool mapBlockArgToDummyOrResult(const Fortran::semantics::SymbolRef sym,
-                                  mlir::Value val, bool forced = false) {
-    if (!forced && lookupSymbol(sym))
-      return false;
-    localSymbols.addSymbol(sym, val, forced);
+                                  mlir::Value val, bool isResult) {
+    localSymbols.addSymbol(sym, val);
+    if (!isResult)
+      registerDummySymbol(sym);
+
     return true;
   }
 
@@ -4559,7 +4570,7 @@ private:
                             const Fortran::lower::CalleeInterface &callee) {
     assert(builder && "require a builder object at this point");
     using PassBy = Fortran::lower::CalleeInterface::PassEntityBy;
-    auto mapPassedEntity = [&](const auto arg) {
+    auto mapPassedEntity = [&](const auto arg, bool isResult = false) {
       if (arg.passBy == PassBy::AddressAndLength) {
         if (callee.characterize().IsBindC())
           return;
@@ -4569,10 +4580,11 @@ private:
         fir::factory::CharacterExprHelper charHelp{*builder, loc};
         mlir::Value box =
             charHelp.createEmboxChar(arg.firArgument, arg.firLength);
-        mapBlockArgToDummyOrResult(arg.entity->get(), box);
+        mapBlockArgToDummyOrResult(arg.entity->get(), box, isResult);
       } else {
         if (arg.entity.has_value()) {
-          mapBlockArgToDummyOrResult(arg.entity->get(), arg.firArgument);
+          mapBlockArgToDummyOrResult(arg.entity->get(), arg.firArgument,
+                                     isResult);
         } else {
           assert(funit.parentHasTupleHostAssoc() && "expect tuple argument");
         }
@@ -4581,15 +4593,19 @@ private:
     for (const Fortran::lower::CalleeInterface::PassedEntity &arg :
          callee.getPassedArguments())
       mapPassedEntity(arg);
+    if (lowerToHighLevelFIR() && !callee.getPassedArguments().empty()) {
+      mlir::Value scopeOp = builder->create<fir::DummyScopeOp>(toLocation());
+      setDummyArgsScope(scopeOp);
+    }
     if (std::optional<Fortran::lower::CalleeInterface::PassedEntity>
             passedResult = callee.getPassedResult()) {
-      mapPassedEntity(*passedResult);
+      mapPassedEntity(*passedResult, /*isResult=*/true);
       // FIXME: need to make sure things are OK here. addSymbol may not be OK
       if (funit.primaryResult &&
           passedResult->entity->get() != *funit.primaryResult)
         mapBlockArgToDummyOrResult(
-            *funit.primaryResult,
-            getSymbolAddress(passedResult->entity->get()));
+            *funit.primaryResult, getSymbolAddress(passedResult->entity->get()),
+            /*isResult=*/true);
     }
   }
 
@@ -4766,7 +4782,8 @@ private:
       Fortran::lower::StatementContext stmtCtx;
       if (std::optional<Fortran::lower::CalleeInterface::PassedEntity>
               passedResult = callee.getPassedResult()) {
-        mapBlockArgToDummyOrResult(altResult.getSymbol(), resultArg.getAddr());
+        mapBlockArgToDummyOrResult(altResult.getSymbol(), resultArg.getAddr(),
+                                   /*isResult=*/true);
         Fortran::lower::mapSymbolAttributes(*this, altResult, localSymbols,
                                             stmtCtx);
       } else {
@@ -4810,6 +4827,11 @@ private:
     if (!funit.getHostAssoc().empty())
       funit.getHostAssoc().hostProcedureBindings(*this, localSymbols);
 
+    // Unregister all dummy symbols, so that their cloning (e.g. for OpenMP
+    // privatization) does not create the cloned hlfir.declare operations
+    // with dummy_scope operands.
+    resetRegisteredDummySymbols();
+
     // Create most function blocks in advance.
     createEmptyBlocks(funit.evaluationList);
 
@@ -4929,6 +4951,8 @@ private:
     hostAssocTuple = mlir::Value{};
     localSymbols.clear();
     blockId = 0;
+    dummyArgsScope = mlir::Value{};
+    resetRegisteredDummySymbols();
   }
 
   /// Helper to generate GlobalOps when the builder is not positioned in any
@@ -4957,6 +4981,7 @@ private:
     delete builder;
     builder = nullptr;
     localSymbols.clear();
+    resetRegisteredDummySymbols();
   }
 
   /// Instantiate the data from a BLOCK DATA unit.
@@ -5374,6 +5399,23 @@ private:
                                         globalOmpRequiresSymbol);
   }
 
+  /// Record fir.dummy_scope operation for this function.
+  /// It will be used to set dummy_scope operand of the hlfir.declare
+  /// operations.
+  void setDummyArgsScope(mlir::Value val) {
+    assert(!dummyArgsScope && val);
+    dummyArgsScope = val;
+  }
+
+  /// Record the given symbol as a dummy argument of this function.
+  void registerDummySymbol(Fortran::semantics::SymbolRef symRef) {
+    auto *sym = &*symRef;
+    registeredDummySymbols.insert(sym);
+  }
+
+  /// Reset all registered dummy symbols.
+  void resetRegisteredDummySymbols() { registeredDummySymbols.clear(); }
+
   //===--------------------------------------------------------------------===//
 
   Fortran::lower::LoweringBridge &bridge;
@@ -5400,6 +5442,15 @@ private:
   /// Tuple of host associated variables
   mlir::Value hostAssocTuple;
 
+  /// Value of fir.dummy_scope operation for this function.
+  mlir::Value dummyArgsScope;
+
+  /// A set of dummy argument symbols for this function.
+  /// The set is only preserved during the instatiation
+  /// of variables for this function.
+  llvm::SmallPtrSet<const Fortran::semantics::Symbol *, 16>
+      registeredDummySymbols;
+
   /// A map of unique names for constant expressions.
   /// The names are used for representing the constant expressions
   /// with global constant initialized objects.
diff --git a/flang/lib/Lower/ConvertArrayConstructor.cpp b/flang/lib/Lower/ConvertArrayConstructor.cpp
index a5b5838fe6b6..341fad9a5e43 100644
--- a/flang/lib/Lower/ConvertArrayConstructor.cpp
+++ b/flang/lib/Lower/ConvertArrayConstructor.cpp
@@ -318,7 +318,7 @@ public:
       mlir::Value shape = builder.genShape(loc, extents);
       declare = builder.create<hlfir::DeclareOp>(
           loc, tempStorage, tempName, shape, lengths,
-          fir::FortranVariableFlagsAttr{});
+          /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
       initialBoxValue =
           builder.createBox(loc, boxType, declare->getOriginalBase(), shape,
                             /*slice=*/mlir::Value{}, lengths, /*tdesc=*/{});
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index 93bdf650f9ff..3c305955520e 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1676,7 +1676,8 @@ private:
     mlir::Value storagePtr = builder.createTemporary(loc, recTy);
     auto varOp = hlfir::EntityWithAttributes{builder.create<hlfir::DeclareOp>(
         loc, storagePtr, "ctor.temp", /*shape=*/nullptr,
-        /*typeparams=*/mlir::ValueRange{}, fir::FortranVariableFlagsAttr{})};
+        /*typeparams=*/mlir::ValueRange{}, /*dummy_scope=*/nullptr,
+        fir::FortranVariableFlagsAttr{})};
 
     // Initialize any components that need initialization.
     mlir::Value box = builder.createBox(loc, fir::ExtendedValue{varOp});
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index f31fbab41028..5ddd8a6a9d41 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -1683,7 +1683,8 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
 
       // Declare a local pointer variable.
       auto newBase = builder.create<hlfir::DeclareOp>(
-          loc, boxAlloc, name, /*shape=*/nullptr, lenParams, attributes);
+          loc, boxAlloc, name, /*shape=*/nullptr, lenParams,
+          /*dummy_scope=*/nullptr, attributes);
       mlir::Value nullAddr = builder.createNullConstant(
           loc, llvm::cast<fir::BaseBoxType>(ptrBoxType).getEleTy());
 
@@ -1710,8 +1711,12 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
       symMap.addVariableDefinition(sym, newBase, force);
       return;
     }
+    mlir::Value dummyScope;
+    if (converter.isRegisteredDummySymbol(sym))
+      dummyScope = converter.dummyArgsScopeValue();
     auto newBase = builder.create<hlfir::DeclareOp>(
-        loc, base, name, shapeOrShift, lenParams, attributes, cudaAttr);
+        loc, base, name, shapeOrShift, lenParams, dummyScope, attributes,
+        cudaAttr);
     symMap.addVariableDefinition(sym, newBase, force);
     return;
   }
@@ -1761,8 +1766,11 @@ void Fortran::lower::genDeclareSymbol(
         Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(),
                                                          sym.GetUltimate());
     auto name = converter.mangleName(sym);
-    hlfir::EntityWithAttributes declare =
-        hlfir::genDeclare(loc, builder, exv, name, attributes, cudaAttr);
+    mlir::Value dummyScope;
+    if (converter.isRegisteredDummySymbol(sym))
+      dummyScope = converter.dummyArgsScopeValue();
+    hlfir::EntityWithAttributes declare = hlfir::genDeclare(
+        loc, builder, exv, name, attributes, dummyScope, cudaAttr);
     symMap.addVariableDefinition(sym, declare.getIfVariableInterface(), force);
     return;
   }
@@ -2022,7 +2030,9 @@ void Fortran::lower::mapSymbolAttributes(
           fir::factory::genMutableBoxRead(
               builder, loc,
               fir::factory::createTempMutableBox(builder, loc, ty, {}, {},
-                                                 isPolymorphic)));
+                                                 isPolymorphic)),
+          fir::FortranVariableFlagsEnum::None,
+          converter.isRegisteredDummySymbol(sym));
       return true;
     }
     return false;
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index eae2afc760e6..b02e7be75d20 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -425,7 +425,8 @@ static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe,
       auto alloca = builder.create<fir::AllocaOp>(loc, refTy.getEleTy());
       auto declareOp = builder.create<hlfir::DeclareOp>(
           loc, alloca, accPrivateInitName, /*shape=*/nullptr,
-          llvm::ArrayRef<mlir::Value>{}, fir::FortranVariableFlagsAttr{});
+          llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
+          fir::FortranVariableFlagsAttr{});
       retVal = declareOp.getBase();
     } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(
                    refTy.getEleTy())) {
@@ -446,7 +447,8 @@ static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe,
             loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
         auto declareOp = builder.create<hlfir::DeclareOp>(
             loc, alloca, accPrivateInitName, shape,
-            llvm::ArrayRef<mlir::Value>{}, fir::FortranVariableFlagsAttr{});
+            llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
+            fir::FortranVariableFlagsAttr{});
         retVal = declareOp.getBase();
       }
     }
@@ -666,10 +668,12 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
 
     auto leftDeclOp = builder.create<hlfir::DeclareOp>(
         loc, recipe.getCopyRegion().getArgument(0), llvm::StringRef{}, shape,
-        llvm::ArrayRef<mlir::Value>{}, fir::FortranVariableFlagsAttr{});
+        llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
+        fir::FortranVariableFlagsAttr{});
     auto rightDeclOp = builder.create<hlfir::DeclareOp>(
         loc, recipe.getCopyRegion().getArgument(1), llvm::StringRef{}, shape,
-        llvm::ArrayRef<mlir::Value>{}, fir::FortranVariableFlagsAttr{});
+        llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
+        fir::FortranVariableFlagsAttr{});
 
     hlfir::DesignateOp::Subscripts triplets =
         getSubscriptsFromArgs(recipe.getCopyRegion().getArguments());
@@ -975,7 +979,8 @@ static mlir::Value genReductionInitRegion(fir::FirOpBuilder &builder,
     mlir::Value alloca = builder.create<fir::AllocaOp>(loc, ty);
     auto declareOp = builder.create<hlfir::DeclareOp>(
         loc, alloca, accReductionInitName, /*shape=*/nullptr,
-        llvm::ArrayRef<mlir::Value>{}, fir::FortranVariableFlagsAttr{});
+        llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
+        fir::FortranVariableFlagsAttr{});
     builder.create<fir::StoreOp>(loc, builder.createConvert(loc, ty, initValue),
                                  declareOp.getBase());
     return declareOp.getBase();
@@ -991,7 +996,8 @@ static mlir::Value genReductionInitRegion(fir::FirOpBuilder &builder,
           loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
       auto declareOp = builder.create<hlfir::DeclareOp>(
           loc, alloca, accReductionInitName, shape,
-          llvm::ArrayRef<mlir::Value>{}, fir::FortranVariableFlagsAttr{});
+          llvm::ArrayRef<mlir::Value>{}, /*dummy_scope=*/nullptr,
+          fir::FortranVariableFlagsAttr{});
       mlir::Type idxTy = builder.getIndexType();
       mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
       llvm::SmallVector<fir::DoLoopOp> loops;
@@ -1143,10 +1149,10 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
                                    recipe.getCombinerRegion().getArguments());
       auto v1DeclareOp = builder.create<hlfir::DeclareOp>(
           loc, value1, llvm::StringRef{}, shape, llvm::ArrayRef<mlir::Value>{},
-          fir::FortranVariableFlagsAttr{});
+          /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
       auto v2DeclareOp = builder.create<hlfir::DeclareOp>(
           loc, value2, llvm::StringRef{}, shape, llvm::ArrayRef<mlir::Value>{},
-          fir::FortranVariableFlagsAttr{});
+          /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
       hlfir::DesignateOp::Subscripts triplets = getTripletsFromArgs(recipe);
 
       llvm::SmallVector<mlir::Value> lenParamsLeft;
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index bb83e8d5e788..b7198c951c8f 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -657,12 +657,12 @@ createCopyFunc(mlir::Location loc, Fortran::lower::AbstractConverter &converter,
           builder.createIntegerConstant(loc, builder.getIndexType(), extent));
     shape = builder.create<fir::ShapeOp>(loc, extents);
   }
-  auto declDst = builder.create<hlfir::DeclareOp>(loc, funcOp.getArgument(0),
-                                                  copyFuncName + "_dst", shape,
-                                                  typeparams, attrs);
-  auto declSrc = builder.create<hlfir::DeclareOp>(loc, funcOp.getArgument(1),
-                                                  copyFuncName + "_src", shape,
-                                                  typeparams, attrs);
+  auto declDst = builder.create<hlfir::DeclareOp>(
+      loc, funcOp.getArgument(0), copyFuncName + "_dst", shape, typeparams,
+      /*dummy_scope=*/nullptr, attrs);
+  auto declSrc = builder.create<hlfir::DeclareOp>(
+      loc, funcOp.getArgument(1), copyFuncName + "_src", shape, typeparams,
+      /*dummy_scope=*/nullptr, attrs);
   converter.copyVar(loc, declDst.getBase(), declSrc.getBase());
   builder.create<mlir::func::ReturnOp>(loc);
   return funcOp;
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index b32c3e50647e..8fdab2a57181 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -198,7 +198,7 @@ mlir::Value hlfir::Entity::getFirBase() const {
 fir::FortranVariableOpInterface
 hlfir::genDeclare(mlir::Location loc, fir::FirOpBuilder &builder,
                   const fir::ExtendedValue &exv, llvm::StringRef name,
-                  fir::FortranVariableFlagsAttr flags,
+                  fir::FortranVariableFlagsAttr flags, mlir::Value dummyScope,
                   fir::CUDADataAttributeAttr cudaAttr) {
 
   mlir::Value base = fir::getBase(exv);
@@ -229,7 +229,7 @@ hlfir::genDeclare(mlir::Location loc, fir::FirOpBuilder &builder,
       },
       [](const auto &) {});
   auto declareOp = builder.create<hlfir::DeclareOp>(
-      loc, base, name, shapeOrShift, lenParams, flags, cudaAttr);
+      loc, base, name, shapeOrShift, lenParams, dummyScope, flags, cudaAttr);
   return mlir::cast<fir::FortranVariableOpInterface>(declareOp.getOperation());
 }
 
@@ -1096,8 +1096,9 @@ hlfir::createTempFromMold(mlir::Location loc, fir::FirOpBuilder &builder,
                                     /*shape=*/std::nullopt, lenParams);
     isHeapAlloc = builder.createBool(loc, false);
   }
-  auto declareOp = builder.create<hlfir::DeclareOp>(loc, alloc, tmpName, shape,
-                                                    lenParams, declAttrs);
+  auto declareOp =
+      builder.create<hlfir::DeclareOp>(loc, alloc, tmpName, shape, lenParams,
+                                       /*dummy_scope=*/nullptr, declAttrs);
   if (mold.isPolymorphic()) {
     int rank = mold.getRank();
     // TODO: should probably read rank from the mold.
@@ -1134,8 +1135,9 @@ hlfir::Entity hlfir::createStackTempFromMold(mlir::Location loc,
     alloc = builder.createTemporary(loc, mold.getFortranElementType(), tmpName,
                                     /*shape=*/std::nullopt, lenParams);
   }
-  auto declareOp = builder.create<hlfir::DeclareOp>(loc, alloc, tmpName, shape,
-                                                    lenParams, declAttrs);
+  auto declareOp =
+      builder.create<hlfir::DeclareOp>(loc, alloc, tmpName, shape, lenParams,
+                                       /*dummy_scope=*/nullptr, declAttrs);
   return hlfir::Entity{declareOp.getBase()};
 }
 
@@ -1153,7 +1155,7 @@ hlfir::convertCharacterKind(mlir::Location loc, fir::FirOpBuilder &builder,
   return hlfir::EntityWithAttributes{builder.create<hlfir::DeclareOp>(
       loc, res.getAddr(), ".temp.kindconvert", /*shape=*/nullptr,
       /*typeparams=*/mlir::ValueRange{res.getLen()},
-      fir::FortranVariableFlagsAttr{})};
+      /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{})};
 }
 
 std::pair<hlfir::Entity, std::optional<hlfir::CleanupFunction>>
@@ -1225,7 +1227,8 @@ hlfir::genTypeAndKindConvert(mlir::Location loc, fir::FirOpBuilder &builder,
         builder.create<fir::ShapeShiftOp>(loc, shapeShiftType, lbAndExtents);
     auto declareOp = builder.create<hlfir::DeclareOp>(
         loc, associate.getFirBase(), *associate.getUniqName(), shapeShift,
-        associate.getTypeparams(), /*flags=*/fir::FortranVariableFlagsAttr{});
+        associate.getTypeparams(), /*dummy_scope=*/nullptr,
+        /*flags=*/fir::FortranVariableFlagsAttr{});
     hlfir::Entity castWithLbounds =
         mlir::cast<fir::FortranVariableOpInterface>(declareOp.getOperation());
     fir::FirOpBuilder *bldr = &builder;
diff --git a/flang/lib/Optimizer/Builder/TemporaryStorage.cpp b/flang/lib/Optimizer/Builder/TemporaryStorage.cpp
index dbc285ce9e22..d34dad52c28b 100644
--- a/flang/lib/Optimizer/Builder/TemporaryStorage.cpp
+++ b/flang/lib/Optimizer/Builder/TemporaryStorage.cpp
@@ -83,7 +83,8 @@ fir::factory::HomogeneousScalarStack::HomogeneousScalarStack(
   mlir::Value shape = builder.genShape(loc, extents);
   temp = builder
              .create<hlfir::DeclareOp>(loc, tempStorage, tempName, shape,
-                                       lengths, fir::FortranVariableFlagsAttr{})
+                                       lengths, /*dummy_scope=*/nullptr,
+                                       fir::FortranVariableFlagsAttr{})
              .getBase();
 }
 
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index 4b586ad1d3a4..c232ae165d4c 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -125,6 +125,7 @@ void hlfir::DeclareOp::build(mlir::OpBuilder &builder,
                              mlir::OperationState &result, mlir::Value memref,
                              llvm::StringRef uniq_name, mlir::Value shape,
                              mlir::ValueRange typeparams,
+                             mlir::Value dummy_scope,
                              fir::FortranVariableFlagsAttr fortran_attrs,
                              fir::CUDADataAttributeAttr cuda_attr) {
   auto nameAttr = builder.getStringAttr(uniq_name);
@@ -133,8 +134,7 @@ void hlfir::DeclareOp::build(mlir::OpBuilder &builder,
   mlir::Type hlfirVariableType =
       getHLFIRVariableType(inputType, hasExplicitLbs);
   build(builder, result, {hlfirVariableType, inputType}, memref, shape,
-        typeparams, /*dummy_scope=*/nullptr, nameAttr, fortran_attrs,
-        cuda_attr);
+        typeparams, dummy_scope, nameAttr, fortran_attrs, cuda_attr);
 }
 
 mlir::LogicalResult hlfir::DeclareOp::verify() {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index d4e4835ee726..76b42c57277b 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -122,9 +122,10 @@ createArrayTemp(mlir::Location loc, fir::FirOpBuilder &builder,
         fir::FortranVariableFlagsAttr::get(
             builder.getContext(), fir::FortranVariableFlagsEnum::allocatable);
 
-    auto declareOp = builder.create<hlfir::DeclareOp>(loc, alloc, tmpName,
-                                                      /*shape=*/nullptr,
-                                                      lenParams, declAttrs);
+    auto declareOp =
+        builder.create<hlfir::DeclareOp>(loc, alloc, tmpName,
+                                         /*shape=*/nullptr, lenParams,
+                                         /*dummy_scope=*/nullptr, declAttrs);
 
     int rank = extents.size();
     fir::runtime::genAllocatableApplyMold(builder, loc, alloc,
@@ -152,9 +153,9 @@ createArrayTemp(mlir::Location loc, fir::FirOpBuilder &builder,
 
   mlir::Value allocmem = builder.createHeapTemporary(loc, sequenceType, tmpName,
                                                      extents, lenParams);
-  auto declareOp =
-      builder.create<hlfir::DeclareOp>(loc, allocmem, tmpName, shape, lenParams,
-                                       fir::FortranVariableFlagsAttr{});
+  auto declareOp = builder.create<hlfir::DeclareOp>(
+      loc, allocmem, tmpName, shape, lenParams,
+      /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
   mlir::Value trueVal = builder.createBool(loc, true);
   return {hlfir::Entity{declareOp.getBase()}, trueVal};
 }
@@ -331,7 +332,7 @@ struct SetLengthOpConversion
                                           /*shape=*/std::nullopt, lenParams);
     auto declareOp = builder.create<hlfir::DeclareOp>(
         loc, alloca, tmpName, /*shape=*/mlir::Value{}, lenParams,
-        fir::FortranVariableFlagsAttr{});
+        /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
     hlfir::Entity temp{declareOp.getBase()};
     // Assign string value to the created temp.
     builder.create<hlfir::AssignOp>(loc, string, temp,
diff --git a/flang/test/Fir/dispatch.f90 b/flang/test/Fir/dispatch.f90
index 1479d611b986..fc935217defa 100644
--- a/flang/test/Fir/dispatch.f90
+++ b/flang/test/Fir/dispatch.f90
@@ -184,7 +184,7 @@ end
 
 ! CHECK-LABEL: func.func @_QMdispatch1Pdisplay_class(
 ! CHECK-SAME: %[[ARG:.*]]: [[CLASS:!fir.class<.*>>]]
-! CHECK: %[[ARG_DECL:.*]]:2 = hlfir.declare %[[ARG]] {uniq_name = "_QMdispatch1Fdisplay_classEp"} : (!fir.class<!fir.type<_QMdispatch1Tp1{a:i32,b:i32}>>) -> (!fir.class<!fir.type<_QMdispatch1Tp1{a:i32,b:i32}>>, !fir.class<!fir.type<_QMdispatch1Tp1{a:i32,b:i32}>>)
+! CHECK: %[[ARG_DECL:.*]]:2 = hlfir.declare %[[ARG]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMdispatch1Fdisplay_classEp"} : (!fir.class<!fir.type<_QMdispatch1Tp1{a:i32,b:i32}>>, !fir.dscope) -> (!fir.class<!fir.type<_QMdispatch1Tp1{a:i32,b:i32}>>, !fir.class<!fir.type<_QMdispatch1Tp1{a:i32,b:i32}>>)
 
 ! Check dynamic dispatch equal to `call p%display2()` with binding index = 2.
 ! CHECK: %[[BOXDESC:.*]] = fir.box_tdesc %[[ARG_DECL]]#0 : ([[CLASS]]) -> !fir.tdesc<none>
diff --git a/flang/test/HLFIR/assumed-type-actual-args.f90 b/flang/test/HLFIR/assumed-type-actual-args.f90
index dbdfc1785ce9..7ce1067d7acd 100644
--- a/flang/test/HLFIR/assumed-type-actual-args.f90
+++ b/flang/test/HLFIR/assumed-type-actual-args.f90
@@ -104,30 +104,34 @@ end subroutine
 
 ! CHECK-LABEL:   func.func @_QPtest1(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<none> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest1Ex"} : (!fir.ref<none>) -> (!fir.ref<none>, !fir.ref<none>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtest1Ex"} : (!fir.ref<none>, !fir.dscope) -> (!fir.ref<none>, !fir.ref<none>)
 ! CHECK:           fir.call @_QPs1(%[[VAL_1]]#1) fastmath<contract> : (!fir.ref<none>) -> ()
 ! CHECK:           return
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QPtest2(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.array<?xnone>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
 ! CHECK:           %[[VAL_1:.*]] = arith.constant -1 : index
 ! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFtest2Ex"} : (!fir.ref<!fir.array<?xnone>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xnone>>, !fir.ref<!fir.array<?xnone>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) dummy_scope %[[DSCOPE]] {uniq_name = "_QFtest2Ex"} : (!fir.ref<!fir.array<?xnone>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xnone>>, !fir.ref<!fir.array<?xnone>>)
 ! CHECK:           fir.call @_QPs2(%[[VAL_3]]#1) fastmath<contract> : (!fir.ref<!fir.array<?xnone>>) -> ()
 ! CHECK:           return
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QPtest3(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest3Ex"} : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtest3Ex"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
 ! CHECK:           fir.call @_QPs3(%[[VAL_1]]#0) fastmath<contract> : (!fir.box<!fir.array<?xnone>>) -> ()
 ! CHECK:           return
 ! CHECK:         }
 
 ! CHECK-LABEL:   func.func @_QPtest4(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest4Ex"} : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtest4Ex"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, i1)
 ! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]]#0 : (!fir.box<!fir.array<?xnone>>) -> !fir.ref<!fir.array<?xnone>>
 ! CHECK:           fir.call @_QPs4(%[[VAL_3]]) fastmath<contract> : (!fir.ref<!fir.array<?xnone>>) -> ()
@@ -137,7 +141,8 @@ end subroutine
 
 ! CHECK-LABEL:   func.func @_QPtest3b(
 ! CHECK-SAME:                         %[[VAL_0:.*]]: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x", fir.optional}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest3bEx"} : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest3bEx"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#0 : (!fir.box<!fir.array<?xnone>>) -> i1
 ! CHECK:           %[[VAL_3:.*]]:4 = fir.if %[[VAL_2]] -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>, i1, !fir.box<!fir.array<?xnone>>) {
 ! CHECK:             %[[VAL_4:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, i1)
@@ -156,7 +161,8 @@ end subroutine
 
 ! CHECK-LABEL:   func.func @_QPtest4b(
 ! CHECK-SAME:                         %[[VAL_0:.*]]: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x", fir.optional}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest4bEx"} : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest4bEx"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#0 : (!fir.box<!fir.array<?xnone>>) -> i1
 ! CHECK:           %[[VAL_3:.*]]:4 = fir.if %[[VAL_2]] -> (!fir.ref<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>, i1, !fir.box<!fir.array<?xnone>>) {
 ! CHECK:             %[[VAL_4:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, i1)
@@ -176,7 +182,8 @@ end subroutine
 
 ! CHECK-LABEL:   func.func @_QPtest4c(
 ! CHECK-SAME:                         %[[VAL_0:.*]]: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x", fir.contiguous, fir.optional}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<contiguous, optional>, uniq_name = "_QFtest4cEx"} : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<contiguous, optional>, uniq_name = "_QFtest4cEx"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#0 : (!fir.box<!fir.array<?xnone>>) -> i1
 ! CHECK:           %[[VAL_3:.*]] = fir.if %[[VAL_2]] -> (!fir.ref<!fir.array<?xnone>>) {
 ! CHECK:             %[[VAL_4:.*]] = fir.box_addr %[[VAL_1]]#1 : (!fir.box<!fir.array<?xnone>>) -> !fir.ref<!fir.array<?xnone>>
@@ -191,7 +198,8 @@ end subroutine
 
 ! CHECK-LABEL:   func.func @_QPtest4d(
 ! CHECK-SAME:                         %[[VAL_0:.*]]: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x", fir.contiguous}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<contiguous>, uniq_name = "_QFtest4dEx"} : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<contiguous>, uniq_name = "_QFtest4dEx"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#1 : (!fir.box<!fir.array<?xnone>>) -> !fir.ref<!fir.array<?xnone>>
 ! CHECK:           fir.call @_QPs4d(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?xnone>>) -> ()
 ! CHECK:           return
@@ -199,7 +207,8 @@ end subroutine
 
 ! CHECK-LABEL:   func.func @_QPtest5(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest5Ex"} : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtest5Ex"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.box<!fir.array<?xnone>>) -> !fir.box<!fir.array<*:none>>
 ! CHECK:           fir.call @_QPs5(%[[VAL_2]]) fastmath<contract> : (!fir.box<!fir.array<*:none>>) -> ()
 ! CHECK:           return
@@ -207,7 +216,8 @@ end subroutine
 
 ! CHECK-LABEL:   func.func @_QPtest5b(
 ! CHECK-SAME:                         %[[VAL_0:.*]]: !fir.box<!fir.array<?xnone>> {fir.bindc_name = "x", fir.optional}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest5bEx"} : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest5bEx"} : (!fir.box<!fir.array<?xnone>>, !fir.dscope) -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#0 : (!fir.box<!fir.array<?xnone>>) -> i1
 ! CHECK:           %[[VAL_3:.*]]:4 = fir.if %[[VAL_2]] -> (!fir.box<!fir.array<?xnone>>, !fir.box<!fir.array<?xnone>>, i1, !fir.box<!fir.array<?xnone>>) {
 ! CHECK:             %[[VAL_4:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.box<!fir.array<?xnone>>) -> (!fir.box<!fir.array<?xnone>>, i1)
diff --git a/flang/test/HLFIR/assumed_shape_with_value_keyword.f90 b/flang/test/HLFIR/assumed_shape_with_value_keyword.f90
index b5080d9bedca..da3dff16382c 100644
--- a/flang/test/HLFIR/assumed_shape_with_value_keyword.f90
+++ b/flang/test/HLFIR/assumed_shape_with_value_keyword.f90
@@ -9,7 +9,7 @@ end
 
 ! CHECK-LABEL:  func.func @_QPtest_integer_value1(
 ! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"}) {
-! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_integer_value1Ex"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_integer_value1Ex"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:          %[[VAL_1:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, i1)
 ! CHECK:          %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
 ! CHECK:          fir.call @_QPinternal_call1(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?xi32>>) -> ()
@@ -23,7 +23,7 @@ subroutine test_integer_value2(x)
 end
 ! CHECK-LABEL:  func.func @_QPtest_integer_value2(
 ! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "x"}) {
-! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_integer_value2Ex"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_integer_value2Ex"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
 ! CHECK:          %[[VAL_1:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, i1)
 ! CHECK:          %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#0 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.ref<!fir.array<?x?xi32>>
 ! CHECK:          fir.call @_QPinternal_call2(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?x?xi32>>) -> ()
@@ -37,7 +37,7 @@ subroutine test_real_value1(x)
 end
 ! CHECK-LABEL:  func.func @_QPtest_real_value1(
 ! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
-! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_real_value1Ex"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_real_value1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:          %[[VAL_1:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, i1)
 ! CHECK:          %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#0 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
 ! CHECK:          fir.call @_QPinternal_call3(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?xf32>>) -> ()
@@ -51,7 +51,7 @@ subroutine test_real_value2(x)
 end
 ! CHECK-LABEL:  func.func @_QPtest_real_value2(
 ! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "x"}) {
-! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_real_value2Ex"} : (!fir.box<!fir.array<?x?xf32>>) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
+! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_real_value2Ex"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
 ! CHECK:          %[[VAL_1:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 : (!fir.box<!fir.array<?x?xf32>>) -> (!fir.box<!fir.array<?x?xf32>>, i1)
 ! CHECK:          %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#0 : (!fir.box<!fir.array<?x?xf32>>) -> !fir.ref<!fir.array<?x?xf32>>
 ! CHECK:          fir.call @_QPinternal_call4(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?x?xf32>>) -> ()
@@ -65,7 +65,7 @@ subroutine test_complex_value1(x)
 end
 ! CHECK-LABEL:  func.func @_QPtest_complex_value1(
 ! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?x!fir.complex<4>>> {fir.bindc_name = "x"}) {
-! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_complex_value1Ex"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_complex_value1Ex"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
 ! CHECK:          %[[VAL_1:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, i1)
 ! CHECK:          %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#0 : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> !fir.ref<!fir.array<?x!fir.complex<4>>>
 ! CHECK:          fir.call @_QPinternal_call5(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?x!fir.complex<4>>>) -> ()
@@ -79,7 +79,7 @@ subroutine test_complex_value2(x)
 end
 ! CHECK-LABEL:  func.func @_QPtest_complex_value2(
 ! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?x?x!fir.complex<4>>> {fir.bindc_name = "x"}) {
-! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_complex_value2Ex"} : (!fir.box<!fir.array<?x?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x?x!fir.complex<4>>>, !fir.box<!fir.array<?x?x!fir.complex<4>>>)
+! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFtest_complex_value2Ex"} : (!fir.box<!fir.array<?x?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.complex<4>>>, !fir.box<!fir.array<?x?x!fir.complex<4>>>)
 ! CHECK:          %[[VAL_1:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 : (!fir.box<!fir.array<?x?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x?x!fir.complex<4>>>, i1)
 ! CHECK:          %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#0 : (!fir.box<!fir.array<?x?x!fir.complex<4>>>) -> !fir.ref<!fir.array<?x?x!fir.complex<4>>>
 ! CHECK:          fir.call @_QPinternal_call6(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?x?x!fir.complex<4>>>) -> ()
@@ -95,7 +95,7 @@ subroutine test_optional1(x)
 end
 ! CHECK-LABEL:  func.func @_QPtest_optional1(
 ! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x", fir.optional}) {
-! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<optional, value>, uniq_name = "_QFtest_optional1Ex"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional, value>, uniq_name = "_QFtest_optional1Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:          %[[VAL_1:.*]] = fir.is_present %[[VAL_0]]#1 : (!fir.box<!fir.array<?xf32>>) -> i1
 ! CHECK:          fir.if %[[VAL_1:.*]] {
 ! CHECK:            %[[VAL_2:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, i1)
@@ -115,7 +115,7 @@ subroutine test_optional2(x)
 end
 ! CHECK-LABEL:  func.func @_QPtest_optional2(
 ! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "x", fir.optional}) {
-! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<optional, value>, uniq_name = "_QFtest_optional2Ex"} : (!fir.box<!fir.array<?x?xf32>>) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
+! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional, value>, uniq_name = "_QFtest_optional2Ex"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
 ! CHECK:          %[[VAL_1:.*]] = fir.is_present %[[VAL_0]]#1 : (!fir.box<!fir.array<?x?xf32>>) -> i1
 ! CHECK:          fir.if %[[VAL_1:.*]] {
 ! CHECK:            %[[VAL_2:.*]]:2 = hlfir.copy_in %[[VAL_0]]#0 : (!fir.box<!fir.array<?x?xf32>>) -> (!fir.box<!fir.array<?x?xf32>>, i1)
@@ -135,7 +135,7 @@ subroutine test_optional3(x)
 end
 ! CHECK-LABEL:  func.func @_QPtest_optional3(
 ! CHECK-SAME:     %[[ARG0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x", fir.optional}) {
-! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<optional, value>, uniq_name = "_QFtest_optional3Ex"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:          %[[VAL_0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional, value>, uniq_name = "_QFtest_optional3Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:          %[[VAL_1:.*]] = fir.is_present %[[VAL_0]]#1 : (!fir.box<!fir.array<?xf32>>) -> i1
 ! CHECK:          cf.cond_br %[[VAL_1]], ^bb1, ^bb2
 ! CHECK:          b1:  // pred: ^bb0
@@ -146,4 +146,4 @@ end
 ! CHECK:          fir.unreachable
 ! CHECK:          b2:  // pred: ^bb0
 ! CHECK:          return
-! CHECK:        }
\ No newline at end of file
+! CHECK:        }
diff --git a/flang/test/HLFIR/boxchar_emboxing.f90 b/flang/test/HLFIR/boxchar_emboxing.f90
index fbc41bbea72d..c25a5c283e36 100644
--- a/flang/test/HLFIR/boxchar_emboxing.f90
+++ b/flang/test/HLFIR/boxchar_emboxing.f90
@@ -2,7 +2,7 @@
 
 ! CHECK-LABEL:   func.func @_QPtest1(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.class<none> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest1Ex"} : (!fir.class<none>) -> (!fir.class<none>, !fir.class<none>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest1Ex"} : (!fir.class<none>, !fir.dscope) -> (!fir.class<none>, !fir.class<none>)
 ! CHECK:           fir.select_type %[[VAL_1]]#1 : !fir.class<none> [#fir.type_is<!fir.char<1,?>>, ^bb1, unit, ^bb2]
 ! CHECK:         ^bb1:
 ! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#1 : (!fir.class<none>) -> !fir.ref<!fir.char<1,?>>
@@ -44,7 +44,7 @@ end subroutine test1
 
 ! CHECK-LABEL:   func.func @_QPtest2(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.class<!fir.array<10xnone>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest2Ex"} : (!fir.class<!fir.array<10xnone>>) -> (!fir.class<!fir.array<10xnone>>, !fir.class<!fir.array<10xnone>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest2Ex"} : (!fir.class<!fir.array<10xnone>>, !fir.dscope) -> (!fir.class<!fir.array<10xnone>>, !fir.class<!fir.array<10xnone>>)
 ! CHECK:           fir.select_type %[[VAL_1]]#1 : !fir.class<!fir.array<10xnone>> [#fir.type_is<!fir.char<1,?>>, ^bb1, unit, ^bb2]
 ! CHECK:         ^bb1:
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.class<!fir.array<10xnone>>) -> !fir.box<!fir.array<10x!fir.char<1,?>>>
diff --git a/flang/test/HLFIR/c_ptr_byvalue.f90 b/flang/test/HLFIR/c_ptr_byvalue.f90
index 45e17c0ff630..377c9fccbee3 100644
--- a/flang/test/HLFIR/c_ptr_byvalue.f90
+++ b/flang/test/HLFIR/c_ptr_byvalue.f90
@@ -22,7 +22,8 @@ end
 
 ! CHECK-LABEL:   func.func @_QPtest2(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "cptr"}) {
-! CHECK:           %[[VAL_97:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest2Ecptr"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_97:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtest2Ecptr"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
 ! CHECK:           %[[VAL_98:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
 ! CHECK:           %[[VAL_99:.*]] = fir.coordinate_of %[[VAL_97]]#0, %[[VAL_98]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
 ! CHECK:           %[[VAL_100:.*]] = fir.load %[[VAL_99]] : !fir.ref<i64>
diff --git a/flang/test/HLFIR/call_with_poly_dummy.f90 b/flang/test/HLFIR/call_with_poly_dummy.f90
index 00a795c5b1fb..93cd410428f7 100644
--- a/flang/test/HLFIR/call_with_poly_dummy.f90
+++ b/flang/test/HLFIR/call_with_poly_dummy.f90
@@ -22,7 +22,8 @@ end subroutine test1
 
 ! CHECK-LABEL:   func.func @_QPtest2(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<f32> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest2Ex"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtest2Ex"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<f32>
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           %[[VAL_4:.*]] = arith.cmpf oeq, %[[VAL_2]], %[[VAL_3]] {{.*}} : f32
diff --git a/flang/test/HLFIR/optional_dummy.f90 b/flang/test/HLFIR/optional_dummy.f90
index 0f1a8d5b9c39..8534a414eaaf 100644
--- a/flang/test/HLFIR/optional_dummy.f90
+++ b/flang/test/HLFIR/optional_dummy.f90
@@ -5,7 +5,7 @@
 
 ! CHECK-LABEL:   func.func @_QPtest(
 ! CHECK-SAME:        %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "ext_buf", fir.contiguous, fir.optional}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<contiguous, optional>, uniq_name = "_QFtestEext_buf"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<contiguous, optional>, uniq_name = "_QFtestEext_buf"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#1 : (!fir.box<!fir.array<?xi32>>) -> i1
 ! CHECK:           cf.cond_br %[[VAL_2]], ^bb1, ^bb2
 ! CHECK:         ^bb1:
diff --git a/flang/test/HLFIR/order_assignments/where-scheduling.f90 b/flang/test/HLFIR/order_assignments/where-scheduling.f90
index 0f79058a6ae9..d3665d234a71 100644
--- a/flang/test/HLFIR/order_assignments/where-scheduling.f90
+++ b/flang/test/HLFIR/order_assignments/where-scheduling.f90
@@ -134,7 +134,7 @@ end subroutine
 !CHECK-NEXT: run 1 save    : where/mask
 !CHECK-NEXT: run 2 evaluate: where/region_assign1
 !CHECK-LABEL: ------------ scheduling where in _QPonly_once ------------
-!CHECK-NEXT: unknown effect: %9 = fir.call @llvm.stacksave.p0() fastmath<contract> : () -> !fir.ref<i8>
+!CHECK-NEXT: unknown effect: %{{[0-9]+}} = fir.call @llvm.stacksave.p0() fastmath<contract> : () -> !fir.ref<i8>
 !CHECK-NEXT: run 1 save  (w): where/mask
 !CHECK-NEXT: run 2 evaluate: where/region_assign1
 !CHECK-NEXT: run 3 evaluate: where/region_assign2
@@ -172,12 +172,12 @@ end subroutine
 !CHECK-NEXT: run 1 save    : forall/where1/region_assign1/rhs
 !CHECK-NEXT: run 2 evaluate: forall/where1/region_assign1
 !CHECK-LABEL: ------------ scheduling where in _QFno_need_to_make_lhs_tempPinternal ------------
-!CHECK-NEXT: conflict: R/W: %7 = fir.load %6 : !fir.llvm_ptr<!fir.ref<i32>> W:%13 = fir.load %12 : !fir.ref<!fir.box<!fir.array<?x?xi32>>>
+!CHECK-NEXT: conflict: R/W: %{{[0-9]+}} = fir.load %{{[0-9]+}} : !fir.llvm_ptr<!fir.ref<i32>> W:%{{[0-9]+}} = fir.load %{{[0-9]+}} : !fir.ref<!fir.box<!fir.array<?x?xi32>>>
 !CHECK-NEXT: run 1 save    : where/mask
 !CHECK-NEXT: run 2 evaluate: where/region_assign1
 !CHECK-NEXT: ------------ scheduling where in _QPwhere_construct_unknown_conflict ------------
 !CHECK-NEXT: unknown effect: %{{.*}} = fir.call @_QPf() fastmath<contract> : () -> f32
-!CHECK-NEXT: conflict: R/W: %{{.*}} = hlfir.declare %{{.*}} {uniq_name = "_QFwhere_construct_unknown_conflictEmask"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>) W:<unknown>
+!CHECK-NEXT: conflict: R/W: %{{.*}} = hlfir.declare %{{.*}} {uniq_name = "_QFwhere_construct_unknown_conflictEmask"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>) W:<unknown>
 !CHECK-NEXT: run 1 save    : where/mask
 !CHECK-NEXT: unknown effect: %{{.*}} = fir.call @_QPf() fastmath<contract> : () -> f32
 !CHECK-NEXT: run 2 save  (w): where/region_assign1/rhs
@@ -185,9 +185,9 @@ end subroutine
 !CHECK-NEXT: ------------ scheduling where in _QPelsewhere_construct_unknown_conflict ------------
 !CHECK-NEXT: run 1 evaluate: where/region_assign1
 !CHECK-NEXT: unknown effect: %{{.*}} = fir.call @_QPf() fastmath<contract> : () -> f32
-!CHECK-NEXT: conflict: R/W: %{{.*}} = hlfir.declare %{{.*}} {uniq_name = "_QFelsewhere_construct_unknown_conflictEmask1"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>) W:<unknown>
+!CHECK-NEXT: conflict: R/W: %{{.*}} = hlfir.declare %{{.*}} {uniq_name = "_QFelsewhere_construct_unknown_conflictEmask1"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>) W:<unknown>
 !CHECK-NEXT: run 2 save    : where/mask
-!CHECK-NEXT: conflict: R/W: %{{.*}} = hlfir.declare %{{.*}} {uniq_name = "_QFelsewhere_construct_unknown_conflictEmask2"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>) W:<unknown>
+!CHECK-NEXT: conflict: R/W: %{{.*}} = hlfir.declare %{{.*}} {uniq_name = "_QFelsewhere_construct_unknown_conflictEmask2"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>) W:<unknown>
 !CHECK-NEXT: run 2 save    : where/elsewhere1/mask
 !CHECK-NEXT: unknown effect: %{{.*}} = fir.call @_QPf() fastmath<contract> : () -> f32
 !CHECK-NEXT: run 3 save  (w): where/elsewhere1/region_assign1/rhs
diff --git a/flang/test/Lower/CUDA/cuda-data-attribute.cuf b/flang/test/Lower/CUDA/cuda-data-attribute.cuf
index 2688d220d8bb..3eb42a6a5d40 100644
--- a/flang/test/Lower/CUDA/cuda-data-attribute.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-attribute.cuf
@@ -39,28 +39,28 @@ subroutine dummy_arg_device(dd)
 end subroutine
 ! CHECK-LABEL: func.func @_QMcuda_varPdummy_arg_device(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<f32> {fir.bindc_name = "dd", fir.cuda_attr = #fir.cuda<device>}) {
-! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] {cuda_attr = #fir.cuda<device>, uniq_name = "_QMcuda_varFdummy_arg_deviceEdd"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {cuda_attr = #fir.cuda<device>, uniq_name = "_QMcuda_varFdummy_arg_deviceEdd"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 
 subroutine dummy_arg_managed(dm)
   real, allocatable, managed :: dm
 end subroutine
 ! CHECK-LABEL: func.func @_QMcuda_varPdummy_arg_managed(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<f32>>> {fir.bindc_name = "dm", fir.cuda_attr = #fir.cuda<managed>}) {
-! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] {cuda_attr = #fir.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFdummy_arg_managedEdm"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
+! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {cuda_attr = #fir.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFdummy_arg_managedEdm"} : (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
 
 subroutine dummy_arg_pinned(dp)
   real, allocatable, pinned :: dp
 end subroutine
 ! CHECK-LABEL: func.func @_QMcuda_varPdummy_arg_pinned(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<f32>>> {fir.bindc_name = "dp", fir.cuda_attr = #fir.cuda<pinned>}) {
-! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] {cuda_attr = #fir.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFdummy_arg_pinnedEdp"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
+! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {cuda_attr = #fir.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFdummy_arg_pinnedEdp"} : (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
 
 subroutine dummy_arg_unified(du)
   real, unified :: du
 end subroutine
 ! CHECK-LABEL: func.func @_QMcuda_varPdummy_arg_unified(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<f32> {fir.bindc_name = "du", fir.cuda_attr = #fir.cuda<unified>})
-! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] {cuda_attr = #fir.cuda<unified>, uniq_name = "_QMcuda_varFdummy_arg_unifiedEdu"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {cuda_attr = #fir.cuda<unified>, uniq_name = "_QMcuda_varFdummy_arg_unifiedEdu"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 
 subroutine cuda_alloc_free(n)
   integer :: n
diff --git a/flang/test/Lower/HLFIR/actual_target_for_dummy_pointer.f90 b/flang/test/Lower/HLFIR/actual_target_for_dummy_pointer.f90
index 129aa49b811d..e6c247205c39 100644
--- a/flang/test/Lower/HLFIR/actual_target_for_dummy_pointer.f90
+++ b/flang/test/Lower/HLFIR/actual_target_for_dummy_pointer.f90
@@ -50,7 +50,7 @@ end subroutine integer_assumed_shape_array
 ! CHECK-SAME:                                              %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "i", fir.target}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.ptr<!fir.array<?xnone>>>
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xi32>>>
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFinteger_assumed_shape_arrayEi"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFinteger_assumed_shape_arrayEi"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.rebox %[[VAL_3]]#1 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.ptr<!fir.array<?xi32>>>
 ! CHECK:           fir.store %[[VAL_4]] to %[[VAL_2]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
 ! CHECK:           fir.call @_QPinteger_assumed_shape_array_callee(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> ()
@@ -159,8 +159,8 @@ end subroutine char_assumed_shape_array
 ! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>
 ! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,2>>>>
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 2 : index
-! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_8]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFchar_assumed_shape_arrayEa1"} : (!fir.box<!fir.array<?x!fir.char<1,2>>>, index) -> (!fir.box<!fir.array<?x!fir.char<1,2>>>, !fir.box<!fir.array<?x!fir.char<1,2>>>)
-! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFchar_assumed_shape_arrayEa2"} : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_8]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFchar_assumed_shape_arrayEa1"} : (!fir.box<!fir.array<?x!fir.char<1,2>>>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,2>>>, !fir.box<!fir.array<?x!fir.char<1,2>>>)
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFchar_assumed_shape_arrayEa2"} : (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>)
 ! CHECK:           %[[VAL_11:.*]] = fir.rebox %[[VAL_9]]#1 : (!fir.box<!fir.array<?x!fir.char<1,2>>>) -> !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,2>>>>
 ! CHECK:           fir.store %[[VAL_11]] to %[[VAL_7]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,2>>>>>
 ! CHECK:           fir.call @_QPchar_assumed_shape_array_explicit_len_callee(%[[VAL_7]]) fastmath<contract> : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,2>>>>>) -> ()
@@ -220,7 +220,7 @@ end subroutine char_explicit_shape_array
 ! CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<100x!fir.char<1,?>>>
 ! CHECK:           %[[VAL_14:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_15:.*]] = fir.shape %[[VAL_14]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_13]](%[[VAL_15]]) typeparams %[[VAL_12]]#1 {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFchar_explicit_shape_arrayEa2"} : (!fir.ref<!fir.array<100x!fir.char<1,?>>>, !fir.shape<1>, index) -> (!fir.box<!fir.array<100x!fir.char<1,?>>>, !fir.ref<!fir.array<100x!fir.char<1,?>>>)
+! CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_13]](%[[VAL_15]]) typeparams %[[VAL_12]]#1 dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFchar_explicit_shape_arrayEa2"} : (!fir.ref<!fir.array<100x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<100x!fir.char<1,?>>>, !fir.ref<!fir.array<100x!fir.char<1,?>>>)
 ! CHECK:           %[[VAL_17:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
 ! CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_11]]#1 : (!fir.ref<!fir.array<100x!fir.char<1,2>>>) -> !fir.ref<!fir.array<?x!fir.char<1,2>>>
 ! CHECK:           %[[VAL_19:.*]] = fir.embox %[[VAL_18]](%[[VAL_17]]) : (!fir.ref<!fir.array<?x!fir.char<1,2>>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,2>>>>
@@ -317,7 +317,7 @@ end subroutine type_assumed_shape_array
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.ptr<!fir.array<?xnone>>>
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFtype_assumed_shape_arrayEt"} : (!fir.box<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>) -> (!fir.box<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>, !fir.box<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFtype_assumed_shape_arrayEt"} : (!fir.box<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>, !fir.box<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>)
 ! CHECK:           %[[VAL_5:.*]] = fir.rebox %[[VAL_4]]#1 : (!fir.box<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>) -> !fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>
 ! CHECK:           fir.store %[[VAL_5]] to %[[VAL_3]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>>
 ! CHECK:           fir.call @_QPtype_assumed_shape_array_callee(%[[VAL_3]]) fastmath<contract> : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>>) -> ()
@@ -400,7 +400,7 @@ end subroutine class_scalar
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.ptr<none>>
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.class<!fir.ptr<!fir.type<_QMtarget_to_pointer_typesTt1>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.type<_QMtarget_to_pointer_typesTt1>>>
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFclass_scalarEt"} : (!fir.class<!fir.type<_QMtarget_to_pointer_typesTt1>>) -> (!fir.class<!fir.type<_QMtarget_to_pointer_typesTt1>>, !fir.class<!fir.type<_QMtarget_to_pointer_typesTt1>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFclass_scalarEt"} : (!fir.class<!fir.type<_QMtarget_to_pointer_typesTt1>>, !fir.dscope) -> (!fir.class<!fir.type<_QMtarget_to_pointer_typesTt1>>, !fir.class<!fir.type<_QMtarget_to_pointer_typesTt1>>)
 ! CHECK:           %[[VAL_5:.*]] = fir.rebox %[[VAL_4]]#1 : (!fir.class<!fir.type<_QMtarget_to_pointer_typesTt1>>) -> !fir.box<!fir.ptr<!fir.type<_QMtarget_to_pointer_typesTt1>>>
 ! CHECK:           fir.store %[[VAL_5]] to %[[VAL_3]] : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMtarget_to_pointer_typesTt1>>>>
 ! CHECK:           fir.call @_QPclass_scalar_callee(%[[VAL_3]]) fastmath<contract> : (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMtarget_to_pointer_typesTt1>>>>) -> ()
@@ -439,7 +439,7 @@ end subroutine class_assumed_shape_array
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.ptr<!fir.array<?xnone>>>
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFclass_assumed_shape_arrayEt"} : (!fir.class<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>) -> (!fir.class<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>, !fir.class<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFclass_assumed_shape_arrayEt"} : (!fir.class<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>, !fir.dscope) -> (!fir.class<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>, !fir.class<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>)
 ! CHECK:           %[[VAL_5:.*]] = fir.rebox %[[VAL_4]]#1 : (!fir.class<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>) -> !fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>
 ! CHECK:           fir.store %[[VAL_5]] to %[[VAL_3]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>>
 ! CHECK:           fir.call @_QPclass_assumed_shape_array_callee(%[[VAL_3]]) fastmath<contract> : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>>) -> ()
@@ -478,7 +478,7 @@ end subroutine class_explicit_shape_array
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.ptr<!fir.array<?xnone>>>
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFclass_explicit_shape_arrayEt"} : (!fir.class<!fir.array<100x!fir.type<_QMtarget_to_pointer_typesTt1>>>) -> (!fir.class<!fir.array<100x!fir.type<_QMtarget_to_pointer_typesTt1>>>, !fir.class<!fir.array<100x!fir.type<_QMtarget_to_pointer_typesTt1>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFclass_explicit_shape_arrayEt"} : (!fir.class<!fir.array<100x!fir.type<_QMtarget_to_pointer_typesTt1>>>, !fir.dscope) -> (!fir.class<!fir.array<100x!fir.type<_QMtarget_to_pointer_typesTt1>>>, !fir.class<!fir.array<100x!fir.type<_QMtarget_to_pointer_typesTt1>>>)
 ! CHECK:           %[[VAL_5:.*]] = fir.rebox %[[VAL_4]]#1 : (!fir.class<!fir.array<100x!fir.type<_QMtarget_to_pointer_typesTt1>>>) -> !fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>
 ! CHECK:           fir.store %[[VAL_5]] to %[[VAL_3]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>>
 ! CHECK:           fir.call @_QPclass_explicit_shape_array_callee(%[[VAL_3]]) fastmath<contract> : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMtarget_to_pointer_typesTt1>>>>>) -> ()
@@ -505,7 +505,7 @@ end subroutine uclass_scalar
 ! CHECK-LABEL:   func.func @_QPuclass_scalar(
 ! CHECK-SAME:                                %[[VAL_0:.*]]: !fir.class<none> {fir.bindc_name = "t", fir.target}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.ptr<none>>
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFuclass_scalarEt"} : (!fir.class<none>) -> (!fir.class<none>, !fir.class<none>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFuclass_scalarEt"} : (!fir.class<none>, !fir.dscope) -> (!fir.class<none>, !fir.class<none>)
 ! CHECK:           %[[VAL_3:.*]] = fir.rebox %[[VAL_2]]#1 : (!fir.class<none>) -> !fir.class<!fir.ptr<none>>
 ! CHECK:           fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<!fir.class<!fir.ptr<none>>>
 ! CHECK:           fir.call @_QPuclass_scalar_uclass_callee(%[[VAL_1]]) fastmath<contract> : (!fir.ref<!fir.class<!fir.ptr<none>>>) -> ()
@@ -526,7 +526,7 @@ end subroutine uclass_assumed_shape_array
 ! CHECK-LABEL:   func.func @_QPuclass_assumed_shape_array(
 ! CHECK-SAME:                                             %[[VAL_0:.*]]: !fir.class<!fir.array<?xnone>> {fir.bindc_name = "t", fir.target}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.ptr<!fir.array<?xnone>>>
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFuclass_assumed_shape_arrayEt"} : (!fir.class<!fir.array<?xnone>>) -> (!fir.class<!fir.array<?xnone>>, !fir.class<!fir.array<?xnone>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFuclass_assumed_shape_arrayEt"} : (!fir.class<!fir.array<?xnone>>, !fir.dscope) -> (!fir.class<!fir.array<?xnone>>, !fir.class<!fir.array<?xnone>>)
 ! CHECK:           %[[VAL_3:.*]] = fir.rebox %[[VAL_2]]#1 : (!fir.class<!fir.array<?xnone>>) -> !fir.class<!fir.ptr<!fir.array<?xnone>>>
 ! CHECK:           fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>>
 ! CHECK:           fir.call @_QPuclass_assumed_shape_array_uclass_callee(%[[VAL_1]]) fastmath<contract> : (!fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>>) -> ()
@@ -547,7 +547,7 @@ end subroutine uclass_explicit_shape_array
 ! CHECK-LABEL:   func.func @_QPuclass_explicit_shape_array(
 ! CHECK-SAME:                                              %[[VAL_0:.*]]: !fir.class<!fir.array<100xnone>> {fir.bindc_name = "t", fir.target}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.class<!fir.ptr<!fir.array<?xnone>>>
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFuclass_explicit_shape_arrayEt"} : (!fir.class<!fir.array<100xnone>>) -> (!fir.class<!fir.array<100xnone>>, !fir.class<!fir.array<100xnone>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFuclass_explicit_shape_arrayEt"} : (!fir.class<!fir.array<100xnone>>, !fir.dscope) -> (!fir.class<!fir.array<100xnone>>, !fir.class<!fir.array<100xnone>>)
 ! CHECK:           %[[VAL_3:.*]] = fir.rebox %[[VAL_2]]#1 : (!fir.class<!fir.array<100xnone>>) -> !fir.class<!fir.ptr<!fir.array<?xnone>>>
 ! CHECK:           fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>>
 ! CHECK:           fir.call @_QPuclass_explicit_shape_array_uclass_callee(%[[VAL_1]]) fastmath<contract> : (!fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>>) -> ()
diff --git a/flang/test/Lower/HLFIR/allocatable-and-pointer-status-change.f90 b/flang/test/Lower/HLFIR/allocatable-and-pointer-status-change.f90
index f5ae6592faa4..328fb778eaf8 100644
--- a/flang/test/Lower/HLFIR/allocatable-and-pointer-status-change.f90
+++ b/flang/test/Lower/HLFIR/allocatable-and-pointer-status-change.f90
@@ -5,7 +5,7 @@
 subroutine allocation(x)
   character(*), allocatable :: x(:)
 ! CHECK-LABEL: func.func @_QPallocation(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] typeparams %[[VAL_2:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<allocatable>,  {{.*}}Ex
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] typeparams %[[VAL_2:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>,  {{.*}}Ex
   deallocate(x)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>
 ! CHECK:  %[[VAL_5:.*]] = fir.box_addr %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>) -> !fir.heap<!fir.array<?x!fir.char<1,?>>>
@@ -30,8 +30,8 @@ subroutine pointer_assignment(p, ziel)
   real, pointer :: p(:)
   real, target :: ziel(42:)
 ! CHECK-LABEL: func.func @_QPpointer_assignment(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>,  {{.*}}Ep
-! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]](%[[VAL_5:[a-z0-9]*]]) {fortran_attrs = #fir.var_attrs<target>,  {{.*}}Eziel
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>,  {{.*}}Ep
+! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]](%[[VAL_5:[a-z0-9]*]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>,  {{.*}}Eziel
   p => ziel
 ! CHECK:  %[[VAL_7:.*]] = fir.shift %[[VAL_4:.*]] : (index) -> !fir.shift<1>
 ! CHECK:  %[[VAL_8:.*]] = fir.rebox %[[VAL_6]]#1(%[[VAL_7]]) : (!fir.box<!fir.array<?xf32>>, !fir.shift<1>) -> !fir.box<!fir.ptr<!fir.array<?xf32>>>
@@ -46,8 +46,8 @@ subroutine pointer_remapping(p, ziel)
   real, pointer :: p(:, :)
   real, target :: ziel(10, 20, 30)
 ! CHECK-LABEL: func.func @_QPpointer_remapping(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>,  {{.*}}Ep
-! CHECK:  %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]](%[[VAL_6:[a-z0-9]*]]) {fortran_attrs = #fir.var_attrs<target>,  {{.*}}Eziel
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>,  {{.*}}Ep
+! CHECK:  %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]](%[[VAL_6:[a-z0-9]*]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>,  {{.*}}Eziel
   p(2:7, 3:102) => ziel
 ! CHECK:  %[[VAL_8:.*]] = arith.constant 2 : i64
 ! CHECK:  %[[VAL_9:.*]] = arith.constant 7 : i64
@@ -101,7 +101,7 @@ subroutine ptr_comp_assign(x, ziel)
   x(9_8)%p => ziel
 ! CHECK:  %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:  %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]](%[[VAL_6:[a-z0-9]*]]) {fortran_attrs = #fir.var_attrs<target>,  {{.*}}Eziel
+! CHECK:  %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]](%[[VAL_6:[a-z0-9]*]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>,  {{.*}}Eziel
 ! CHECK:  %[[VAL_8:.*]] = arith.constant 9 : index
 ! CHECK:  %[[VAL_9:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_8]])  : (!fir.ref<!fir.array<10x!fir.type<_QFptr_comp_assignTt{p:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>>, index) -> !fir.ref<!fir.type<_QFptr_comp_assignTt{p:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>
 ! CHECK:  %[[VAL_10:.*]] = hlfir.designate %[[VAL_9]]{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<_QFptr_comp_assignTt{p:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
diff --git a/flang/test/Lower/HLFIR/allocatables-and-pointers.f90 b/flang/test/Lower/HLFIR/allocatables-and-pointers.f90
index ad6b2cf932e3..eb278508eba2 100644
--- a/flang/test/Lower/HLFIR/allocatables-and-pointers.f90
+++ b/flang/test/Lower/HLFIR/allocatables-and-pointers.f90
@@ -15,7 +15,7 @@ subroutine passing_allocatable(x)
   call takes_array(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPpassing_allocatable(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name =  {{.*}}Ex"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name =  {{.*}}Ex"}
 ! CHECK:  fir.call @_QPtakes_allocatable(%[[VAL_1]]#0) {{.*}} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> ()
 ! CHECK:  %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK:  %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
@@ -34,7 +34,7 @@ subroutine passing_pointer(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPpassing_pointer(
 ! CHECK:  %[[VAL_1:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xf32>>>
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name =  {{.*}}Ex"}
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name =  {{.*}}Ex"}
 ! CHECK:  fir.call @_QPtakes_pointer(%[[VAL_2]]#0) {{.*}} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> ()
 ! CHECK:  %[[VAL_3:.*]] = fir.zero_bits !fir.ptr<!fir.array<?xf32>>
 ! CHECK:  %[[VAL_4:.*]] = arith.constant 0 : index
@@ -53,7 +53,7 @@ subroutine passing_contiguous_pointer(x)
   call takes_array(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPpassing_contiguous_pointer(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<contiguous, pointer>, uniq_name =  {{.*}}Ex"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<contiguous, pointer>, uniq_name =  {{.*}}Ex"}
 ! CHECK:  %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK:  %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.ptr<!fir.array<?xf32>>
 ! CHECK:  %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ptr<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
@@ -66,7 +66,7 @@ subroutine character_allocatable_cst_len(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPcharacter_allocatable_cst_len(
 ! CHECK:  %[[VAL_1:.*]] = arith.constant 10 : index
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] typeparams %[[VAL_1:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name =  {{.*}}Ex"}
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] typeparams %[[VAL_1:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name =  {{.*}}Ex"}
 ! CHECK:  %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.char<1,10>>>>
 ! CHECK:  %[[VAL_4:.*]] = fir.box_addr %[[VAL_3]] : (!fir.box<!fir.heap<!fir.char<1,10>>>) -> !fir.heap<!fir.char<1,10>>
 ! CHECK:  %[[VAL_5:.*]] = arith.constant 10 : index
@@ -87,12 +87,12 @@ subroutine character_allocatable_dyn_len(x, l)
   call takes_char(x//"hello")
 end subroutine
 ! CHECK-LABEL: func.func @_QPcharacter_allocatable_dyn_len(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]] {uniq_name =  {{.*}}El"}
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {uniq_name =  {{.*}}El"}
 ! CHECK:  %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i64>
 ! CHECK:  %[[VAL_4:.*]] = arith.constant 0 : i64
 ! CHECK:  %[[VAL_5:.*]] = arith.cmpi sgt, %[[VAL_3]], %[[VAL_4]] : i64
 ! CHECK:  %[[VAL_6:.*]] = arith.select %[[VAL_5]], %[[VAL_3]], %[[VAL_4]] : i64
-! CHECK:  %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] typeparams %[[VAL_6:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name =  {{.*}}Ex"}
+! CHECK:  %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] typeparams %[[VAL_6:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name =  {{.*}}Ex"}
 ! CHECK:  %[[VAL_8:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
 ! CHECK:  %[[VAL_9:.*]] = fir.box_addr %[[VAL_8]] : (!fir.box<!fir.heap<!fir.char<1,?>>>) -> !fir.heap<!fir.char<1,?>>
 ! CHECK:  %[[VAL_10:.*]] = fir.emboxchar %[[VAL_9]], %[[VAL_6]] : (!fir.heap<!fir.char<1,?>>, i64) -> !fir.boxchar<1>
@@ -110,7 +110,7 @@ subroutine print_allocatable(x)
   print *, x
 end subroutine
 ! CHECK-LABEL: func.func @_QPprint_allocatable(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name =  {{.*}}Ex"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name =  {{.*}}Ex"}
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK:  %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.box<none>
 ! CHECK:  %[[VAL_9:.*]] = fir.call @_FortranAioOutputDescriptor(%{{.*}}, %[[VAL_8]])
@@ -120,7 +120,7 @@ subroutine print_pointer(x)
   print *, x
 end subroutine
 ! CHECK-LABEL: func.func @_QPprint_pointer(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name =  {{.*}}Ex"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name =  {{.*}}Ex"}
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK:  %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.box<none>
 ! CHECK:  %[[VAL_9:.*]] = fir.call @_FortranAioOutputDescriptor(%{{.*}}, %[[VAL_8]])
@@ -130,7 +130,7 @@ subroutine elemental_expr(x)
   call takes_array_2(x+42)
 end subroutine
 ! CHECK-LABEL: func.func @_QPelemental_expr(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name =  {{.*}}Ex"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name =  {{.*}}Ex"}
 ! CHECK:  %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xi32>>>>
 ! CHECK:  %[[VAL_3:.*]] = arith.constant 42 : i32
 ! CHECK:  %[[VAL_4:.*]] = arith.constant 0 : index
diff --git a/flang/test/Lower/HLFIR/array-ctor-as-elemental-nested.f90 b/flang/test/Lower/HLFIR/array-ctor-as-elemental-nested.f90
index f3f5653a7c48..a30c6c6e4a22 100644
--- a/flang/test/Lower/HLFIR/array-ctor-as-elemental-nested.f90
+++ b/flang/test/Lower/HLFIR/array-ctor-as-elemental-nested.f90
@@ -9,14 +9,14 @@
 ! CHECK-SAME:                       %[[VAL_1:.*]]: !fir.ref<!fir.array<2xf32>> {fir.bindc_name = "h1"}) {
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_3]]) {uniq_name = "_QFtestEh1"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xf32>>, !fir.ref<!fir.array<2xf32>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_3]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFtestEh1"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<2xf32>>, !fir.ref<!fir.array<2xf32>>)
 ! CHECK:           %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "k", uniq_name = "_QFtestEk"}
 ! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFtestEk"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "l", uniq_name = "_QFtestEl"}
 ! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFtestEl"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_9:.*]] = fir.address_of(@_QFtestECn) : !fir.ref<i32>
 ! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_9]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QFtestECn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtestEpi"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtestEpi"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:           %[[VAL_12:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_12]] : (index) -> !fir.shape<1>
 ! CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_13]] unordered : (!fir.shape<1>) -> !hlfir.expr<2xf32> {
diff --git a/flang/test/Lower/HLFIR/array-ctor-as-elemental.f90 b/flang/test/Lower/HLFIR/array-ctor-as-elemental.f90
index 7cbc052ea709..277e2683c64f 100644
--- a/flang/test/Lower/HLFIR/array-ctor-as-elemental.f90
+++ b/flang/test/Lower/HLFIR/array-ctor-as-elemental.f90
@@ -7,7 +7,7 @@ subroutine test_as_simple_elemental(n)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_as_simple_elemental(
 ! CHECK-SAME:                                           %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_as_simple_elementalEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_as_simple_elementalEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 4 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i64
@@ -41,9 +41,10 @@ end subroutine
 ! CHECK-SAME:                                            %[[VAL_0:.*]]: !fir.ref<i64> {fir.bindc_name = "lb"},
 ! CHECK-SAME:                                            %[[VAL_1:.*]]: !fir.ref<i64> {fir.bindc_name = "ub"},
 ! CHECK-SAME:                                            %[[VAL_2:.*]]: !fir.ref<i64> {fir.bindc_name = "stride"}) {
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_as_strided_elementalElb"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtest_as_strided_elementalEstride"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest_as_strided_elementalEub"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtest_as_strided_elementalElb"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtest_as_strided_elementalEstride"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFtest_as_strided_elementalEub"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
 ! CHECK:           %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i64>
 ! CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i64>
@@ -91,7 +92,7 @@ subroutine test_as_elemental_with_pure_call(n)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_as_elemental_with_pure_call(
 ! CHECK-SAME:                                                   %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_as_elemental_with_pure_callEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_as_elemental_with_pure_callEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 4 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i64
diff --git a/flang/test/Lower/HLFIR/array-ctor-as-inlined-temp.f90 b/flang/test/Lower/HLFIR/array-ctor-as-inlined-temp.f90
index 1fc882015108..a7c2faa410fb 100644
--- a/flang/test/Lower/HLFIR/array-ctor-as-inlined-temp.f90
+++ b/flang/test/Lower/HLFIR/array-ctor-as-inlined-temp.f90
@@ -116,7 +116,7 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_implied_do(
 ! CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.ref<i64> {fir.bindc_name = "n"}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca index
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_implied_doEn"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_implied_doEn"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i64
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i64
 ! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i64>
@@ -178,9 +178,9 @@ end subroutine
 ! CHECK-SAME:                                          %[[VAL_1:.*]]: !fir.ref<i64> {fir.bindc_name = "ub"},
 ! CHECK-SAME:                                          %[[VAL_2:.*]]: !fir.ref<i64> {fir.bindc_name = "stride"}) {
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca index
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_strided_implied_doElb"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtest_strided_implied_doEstride"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest_strided_implied_doEub"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_strided_implied_doElb"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_strided_implied_doEstride"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_strided_implied_doEub"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:           %[[VAL_7:.*]] = arith.constant 0 : i64
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 2 : i64
 ! CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i64>
@@ -241,8 +241,8 @@ end subroutine
 ! CHECK-SAME:                                         %[[VAL_0:.*]]: !fir.ref<i64> {fir.bindc_name = "n"},
 ! CHECK-SAME:                                         %[[VAL_1:.*]]: !fir.ref<i64> {fir.bindc_name = "m"}) {
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca index
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest_nested_implied_doEm"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_nested_implied_doEn"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_nested_implied_doEm"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_nested_implied_doEn"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 0 : i64
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i64
 ! CHECK:           %[[VAL_7:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i64>
diff --git a/flang/test/Lower/HLFIR/array-ctor-index.f90 b/flang/test/Lower/HLFIR/array-ctor-index.f90
index 83eb2cd3a408..f0c7cf620e9a 100644
--- a/flang/test/Lower/HLFIR/array-ctor-index.f90
+++ b/flang/test/Lower/HLFIR/array-ctor-index.f90
@@ -8,7 +8,7 @@ function test1(k)
 end function test1
 ! CHECK-LABEL:   func.func @_QPtest1(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i8> {fir.bindc_name = "k"}) -> !fir.array<4xi8> {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest1Ek"} : (!fir.ref<i8>) -> (!fir.ref<i8>, !fir.ref<i8>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest1Ek"} : (!fir.ref<i8>, !fir.dscope) -> (!fir.ref<i8>, !fir.ref<i8>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 4 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.array<4xi8> {bindc_name = "test1", uniq_name = "_QFtest1Etest1"}
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
@@ -58,7 +58,7 @@ function test2(k)
 end function test2
 ! CHECK-LABEL:   func.func @_QPtest2(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i16> {fir.bindc_name = "k"}) -> !fir.array<4xi16> {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest2Ek"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest2Ek"} : (!fir.ref<i16>, !fir.dscope) -> (!fir.ref<i16>, !fir.ref<i16>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 4 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.array<4xi16> {bindc_name = "test2", uniq_name = "_QFtest2Etest2"}
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
@@ -108,7 +108,7 @@ function test3(k)
 end function test3
 ! CHECK-LABEL:   func.func @_QPtest3(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "k"}) -> !fir.array<4xi32> {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest3Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest3Ek"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 4 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.array<4xi32> {bindc_name = "test3", uniq_name = "_QFtest3Etest3"}
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
@@ -158,7 +158,7 @@ function test4(k)
 end function test4
 ! CHECK-LABEL:   func.func @_QPtest4(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i64> {fir.bindc_name = "k"}) -> !fir.array<4xi64> {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest4Ek"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest4Ek"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 4 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.array<4xi64> {bindc_name = "test4", uniq_name = "_QFtest4Etest4"}
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
diff --git a/flang/test/Lower/HLFIR/assignment-intrinsics.f90 b/flang/test/Lower/HLFIR/assignment-intrinsics.f90
index 984395d8f90d..544815e88140 100644
--- a/flang/test/Lower/HLFIR/assignment-intrinsics.f90
+++ b/flang/test/Lower/HLFIR/assignment-intrinsics.f90
@@ -10,8 +10,8 @@ subroutine scalar_int(x, y)
   x = y
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_int(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_intEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_intEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_intEy"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0
 ! CHECK:  hlfir.assign %[[VAL_4]] to %[[VAL_2]]#0 : i32, !fir.ref<i32>
 
@@ -20,8 +20,8 @@ subroutine scalar_logical(x, y)
   x = y
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_logical(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_logicalEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_logicalEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_logicalEx"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_logicalEy"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0
 ! CHECK:  hlfir.assign %[[VAL_4]] to %[[VAL_2]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 
@@ -30,8 +30,8 @@ subroutine scalar_real(x, y)
   x = y
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_real(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_realEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_realEx"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_realEy"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0
 ! CHECK:  hlfir.assign %[[VAL_4]] to %[[VAL_2]]#0 : f32, !fir.ref<f32>
 
@@ -40,8 +40,8 @@ subroutine scalar_complex(x, y)
   x = y
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_complex(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complexEx"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complexEy"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complexEx"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complexEy"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0
 ! CHECK:  hlfir.assign %[[VAL_4]] to %[[VAL_2]]#0 : !fir.complex<4>, !fir.ref<!fir.complex<4>>
 
@@ -50,8 +50,8 @@ subroutine scalar_character(x, y)
   x = y
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_character(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_characterEx"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_characterEy"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_characterEx"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_characterEy"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  hlfir.assign %[[VAL_5]]#0 to %[[VAL_3]]#0 : !fir.boxchar<1>, !fir.boxchar<1>
 
 ! -----------------------------------------------------------------------------
@@ -63,7 +63,7 @@ subroutine scalar_int_2(x)
   x = 42
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_int_2(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_int_2Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_int_2Ex"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 42 : i32
 ! CHECK:  hlfir.assign %[[VAL_2]] to %[[VAL_1]]#0 : i32, !fir.ref<i32>
 
@@ -72,7 +72,7 @@ subroutine scalar_logical_2(x)
   x = .true.
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_logical_2(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_logical_2Ex"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_logical_2Ex"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:  %[[VAL_2:.*]] = arith.constant true
 ! CHECK:  %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<4>
 ! CHECK:  hlfir.assign %[[VAL_3]] to %[[VAL_1]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -82,7 +82,7 @@ subroutine scalar_real_2(x)
   x = 3.14
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_real_2(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_real_2Ex"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_real_2Ex"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 3.140000e+00 : f32
 ! CHECK:  hlfir.assign %[[VAL_2]] to %[[VAL_1]]#0 : f32, !fir.ref<f32>
 
@@ -91,7 +91,7 @@ subroutine scalar_complex_2(x)
   x = (1., -1.)
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_complex_2(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complex_2Ex"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_complex_2Ex"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:  %[[VAL_3:.*]] = arith.constant -1.000000e+00 : f32
 ! CHECK:  %[[VAL_4:.*]] = fir.undefined !fir.complex<4>
@@ -104,7 +104,7 @@ subroutine scalar_character_2(x)
   x = "hello"
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_character_2(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}} {uniq_name = "_QFscalar_character_2Ex"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}} {uniq_name = "_QFscalar_character_2Ex"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare {{.*}} {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQclX68656C6C6F"} : (!fir.ref<!fir.char<1,5>>, index) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
 ! CHECK:  hlfir.assign %[[VAL_5]]#0 to %[[VAL_2]]#0 : !fir.ref<!fir.char<1,5>>, !fir.boxchar<1>
 
@@ -117,8 +117,8 @@ subroutine array(x, y)
   x = y
 end subroutine
 ! CHECK-LABEL: func.func @_QParray(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarrayEx"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarrayEy"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarrayEx"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarrayEy"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! CHECK:  hlfir.assign %[[VAL_5]]#0 to %[[VAL_2]]#0 : !fir.ref<!fir.array<100xi32>>, !fir.box<!fir.array<?xi32>>
 
 subroutine array_lbs(x, y)
@@ -126,8 +126,8 @@ subroutine array_lbs(x, y)
   x = y
 end subroutine
 ! CHECK-LABEL: func.func @_QParray_lbs(
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_lbsEx"} : (!fir.ref<!fir.array<20x!fir.logical<4>>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<20x!fir.logical<4>>>, !fir.ref<!fir.array<20x!fir.logical<4>>>)
-! CHECK:  %[[VAL_9:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_lbsEy"} : (!fir.ref<!fir.array<20x!fir.logical<4>>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<20x!fir.logical<4>>>, !fir.ref<!fir.array<20x!fir.logical<4>>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_lbsEx"} : (!fir.ref<!fir.array<20x!fir.logical<4>>>, !fir.shapeshift<1>, !fir.dscope) -> (!fir.box<!fir.array<20x!fir.logical<4>>>, !fir.ref<!fir.array<20x!fir.logical<4>>>)
+! CHECK:  %[[VAL_9:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_lbsEy"} : (!fir.ref<!fir.array<20x!fir.logical<4>>>, !fir.shapeshift<1>, !fir.dscope) -> (!fir.box<!fir.array<20x!fir.logical<4>>>, !fir.ref<!fir.array<20x!fir.logical<4>>>)
 ! CHECK:  hlfir.assign %[[VAL_9]]#0 to %[[VAL_5]]#0 : !fir.box<!fir.array<20x!fir.logical<4>>>, !fir.box<!fir.array<20x!fir.logical<4>>>
 
 
@@ -136,8 +136,8 @@ subroutine array_character(x, y)
   x = y
 end subroutine
 ! CHECK-LABEL: func.func @_QParray_character(
-! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_characterEx"} : (!fir.ref<!fir.array<10x!fir.char<1,?>>>, !fir.shape<1>, index) -> (!fir.box<!fir.array<10x!fir.char<1,?>>>, !fir.ref<!fir.array<10x!fir.char<1,?>>>)
-! CHECK:  %[[VAL_11:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_characterEy"} : (!fir.ref<!fir.array<10x!fir.char<1,?>>>, !fir.shape<1>, index) -> (!fir.box<!fir.array<10x!fir.char<1,?>>>, !fir.ref<!fir.array<10x!fir.char<1,?>>>)
+! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_characterEx"} : (!fir.ref<!fir.array<10x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<10x!fir.char<1,?>>>, !fir.ref<!fir.array<10x!fir.char<1,?>>>)
+! CHECK:  %[[VAL_11:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_characterEy"} : (!fir.ref<!fir.array<10x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<10x!fir.char<1,?>>>, !fir.ref<!fir.array<10x!fir.char<1,?>>>)
 ! CHECK:  hlfir.assign %[[VAL_11]]#0 to %[[VAL_6]]#0 : !fir.box<!fir.array<10x!fir.char<1,?>>>, !fir.box<!fir.array<10x!fir.char<1,?>>>
 
 subroutine array_pointer(x, y)
@@ -160,8 +160,8 @@ subroutine array_scalar(x, y)
   x = y
 end subroutine
 ! CHECK-LABEL: func.func @_QParray_scalar(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_scalarEx"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_scalarEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_scalarEx"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFarray_scalarEy"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_5]]#0
 ! CHECK:  hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<!fir.array<100xi32>>
 
diff --git a/flang/test/Lower/HLFIR/assumed-rank-iface-alloc-ptr.f90 b/flang/test/Lower/HLFIR/assumed-rank-iface-alloc-ptr.f90
index 1bb5c001ece8..fb1385f87f1b 100644
--- a/flang/test/Lower/HLFIR/assumed-rank-iface-alloc-ptr.f90
+++ b/flang/test/Lower/HLFIR/assumed-rank-iface-alloc-ptr.f90
@@ -23,7 +23,7 @@ subroutine scalar_alloc_to_assumed_rank(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPscalar_alloc_to_assumed_rank(
 ! CHECK-SAME:                                               %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<f32>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFscalar_alloc_to_assumed_rankEx"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFscalar_alloc_to_assumed_rankEx"} : (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>
 ! CHECK:           fir.call @_QPalloc_assumed_rank(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>) -> ()
 
@@ -34,7 +34,7 @@ subroutine r2_alloc_to_assumed_rank(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPr2_alloc_to_assumed_rank(
 ! CHECK-SAME:                                           %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFr2_alloc_to_assumed_rankEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFr2_alloc_to_assumed_rankEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>
 ! CHECK:           fir.call @_QPalloc_assumed_rank(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.box<!fir.heap<!fir.array<*:f32>>>>) -> ()
 
@@ -45,7 +45,7 @@ subroutine scalar_pointer_to_assumed_rank(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPscalar_pointer_to_assumed_rank(
 ! CHECK-SAME:                                                 %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<f32>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFscalar_pointer_to_assumed_rankEx"} : (!fir.ref<!fir.box<!fir.ptr<f32>>>) -> (!fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFscalar_pointer_to_assumed_rankEx"} : (!fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>
 ! CHECK:           fir.call @_QPpointer_assumed_rank(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>) -> ()
 
@@ -56,7 +56,7 @@ subroutine r2_pointer_to_assumed_rank(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPr2_pointer_to_assumed_rank(
 ! CHECK-SAME:                                             %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFr2_pointer_to_assumed_rankEx"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFr2_pointer_to_assumed_rankEx"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>
 ! CHECK:           fir.call @_QPpointer_assumed_rank(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>) -> ()
 
@@ -68,7 +68,7 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QPr2_target_to_pointer_assumed_rank(
 ! CHECK-SAME:                                                    %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "x", fir.target}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFr2_target_to_pointer_assumed_rankEx"} : (!fir.box<!fir.array<?x?xf32>>) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFr2_target_to_pointer_assumed_rankEx"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
 ! CHECK:           %[[VAL_3:.*]] = fir.rebox %[[VAL_2]]#1 : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<!fir.ptr<!fir.array<?x?xf32>>>
 ! CHECK:           fir.store %[[VAL_3]] to %[[VAL_1]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<*:f32>>>>
diff --git a/flang/test/Lower/HLFIR/assumed-rank-iface.f90 b/flang/test/Lower/HLFIR/assumed-rank-iface.f90
index 155ce8fb55f2..2d1d941238b1 100644
--- a/flang/test/Lower/HLFIR/assumed-rank-iface.f90
+++ b/flang/test/Lower/HLFIR/assumed-rank-iface.f90
@@ -23,7 +23,7 @@ subroutine int_scalar_to_assumed_rank(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPint_scalar_to_assumed_rank(
 ! CHECK-SAME:                                             %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFint_scalar_to_assumed_rankEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFint_scalar_to_assumed_rankEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.embox %[[VAL_1]]#0 : (!fir.ref<i32>) -> !fir.box<i32>
 ! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.box<i32>) -> !fir.box<!fir.array<*:i32>>
 ! CHECK:           fir.call @_QPint_assumed_rank(%[[VAL_3]]) fastmath<contract> : (!fir.box<!fir.array<*:i32>>) -> ()
@@ -35,7 +35,7 @@ subroutine int_scalar_to_assumed_rank_bindc(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPint_scalar_to_assumed_rank_bindc(
 ! CHECK-SAME:                                                   %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFint_scalar_to_assumed_rank_bindcEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFint_scalar_to_assumed_rank_bindcEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.embox %[[VAL_1]]#0 : (!fir.ref<i32>) -> !fir.box<i32>
 ! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.box<i32>) -> !fir.box<!fir.array<*:i32>>
 ! CHECK:           fir.call @int_assumed_rank_bindc(%[[VAL_3]]) fastmath<contract> : (!fir.box<!fir.array<*:i32>>) -> ()
@@ -49,7 +49,7 @@ end subroutine
 ! CHECK-SAME:                                         %[[VAL_0:.*]]: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "x"}) {
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFint_r1_to_assumed_rankEx"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFint_r1_to_assumed_rankEx"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<10xi32>>
 ! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.box<!fir.array<10xi32>>) -> !fir.box<!fir.array<*:i32>>
 ! CHECK:           fir.call @_QPint_assumed_rank(%[[VAL_5]]) fastmath<contract> : (!fir.box<!fir.array<*:i32>>) -> ()
@@ -66,7 +66,7 @@ end subroutine
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 4 : index
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 5 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_1]], %[[VAL_2]], %[[VAL_3]], %[[VAL_4]] : (index, index, index, index) -> !fir.shape<4>
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) {uniq_name = "_QFint_r4_to_assumed_rankEx"} : (!fir.ref<!fir.array<2x3x4x5xi32>>, !fir.shape<4>) -> (!fir.ref<!fir.array<2x3x4x5xi32>>, !fir.ref<!fir.array<2x3x4x5xi32>>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFint_r4_to_assumed_rankEx"} : (!fir.ref<!fir.array<2x3x4x5xi32>>, !fir.shape<4>, !fir.dscope) -> (!fir.ref<!fir.array<2x3x4x5xi32>>, !fir.ref<!fir.array<2x3x4x5xi32>>)
 ! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[VAL_5]]) : (!fir.ref<!fir.array<2x3x4x5xi32>>, !fir.shape<4>) -> !fir.box<!fir.array<2x3x4x5xi32>>
 ! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.array<2x3x4x5xi32>>) -> !fir.box<!fir.array<*:i32>>
 ! CHECK:           fir.call @_QPint_assumed_rank(%[[VAL_8]]) fastmath<contract> : (!fir.box<!fir.array<*:i32>>) -> ()
@@ -78,7 +78,7 @@ subroutine int_assumed_shape_to_assumed_rank(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPint_assumed_shape_to_assumed_rank(
 ! CHECK-SAME:                                                    %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFint_assumed_shape_to_assumed_rankEx"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFint_assumed_shape_to_assumed_rankEx"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<*:i32>>
 ! CHECK:           fir.call @_QPint_assumed_rank(%[[VAL_2]]) fastmath<contract> : (!fir.box<!fir.array<*:i32>>) -> ()
 
@@ -89,7 +89,7 @@ subroutine int_assumed_shape_to_assumed_rank_bindc(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPint_assumed_shape_to_assumed_rank_bindc(
 ! CHECK-SAME:                                                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFint_assumed_shape_to_assumed_rank_bindcEx"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFint_assumed_shape_to_assumed_rank_bindcEx"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.shift %[[VAL_2]], %[[VAL_2]] : (index, index) -> !fir.shift<2>
 ! CHECK:           %[[VAL_4:.*]] = fir.rebox %[[VAL_1]]#0(%[[VAL_3]]) : (!fir.box<!fir.array<?x?xi32>>, !fir.shift<2>) -> !fir.box<!fir.array<?x?xi32>>
@@ -103,7 +103,7 @@ subroutine int_allocatable_to_assumed_rank(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPint_allocatable_to_assumed_rank(
 ! CHECK-SAME:                                                  %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFint_allocatable_to_assumed_rankEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFint_allocatable_to_assumed_rankEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>) -> !fir.box<!fir.array<?x?xi32>>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<*:i32>>
@@ -116,7 +116,7 @@ subroutine int_allocatable_to_assumed_rank_opt(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPint_allocatable_to_assumed_rank_opt(
 ! CHECK-SAME:                                                      %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFint_allocatable_to_assumed_rank_optEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFint_allocatable_to_assumed_rank_optEx"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>) -> !fir.heap<!fir.array<?x?xi32>>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.heap<!fir.array<?x?xi32>>) -> i64
@@ -147,6 +147,6 @@ end subroutine
 ! CHECK:           %[[VAL_5:.*]] = arith.select %[[VAL_4]], %[[VAL_2]], %[[VAL_3]] : index
 ! CHECK:           %[[VAL_6:.*]] = arith.constant -1 : index
 ! CHECK:           %[[VAL_7:.*]] = fir.shape %[[VAL_5]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
-! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_7]]) {uniq_name = "_QFint_r2_assumed_size_to_assumed_rankEx"} : (!fir.ref<!fir.array<10x?xi32>>, !fir.shape<2>) -> (!fir.box<!fir.array<10x?xi32>>, !fir.ref<!fir.array<10x?xi32>>)
+! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_7]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFint_r2_assumed_size_to_assumed_rankEx"} : (!fir.ref<!fir.array<10x?xi32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<10x?xi32>>, !fir.ref<!fir.array<10x?xi32>>)
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]]#0 : (!fir.box<!fir.array<10x?xi32>>) -> !fir.box<!fir.array<*:i32>>
 ! CHECK:           fir.call @_QPint_assumed_rank(%[[VAL_9]]) fastmath<contract> : (!fir.box<!fir.array<*:i32>>) -> ()
diff --git a/flang/test/Lower/HLFIR/binary-ops.f90 b/flang/test/Lower/HLFIR/binary-ops.f90
index e0af9258cda3..912cea0f5e0e 100644
--- a/flang/test/Lower/HLFIR/binary-ops.f90
+++ b/flang/test/Lower/HLFIR/binary-ops.f90
@@ -6,8 +6,8 @@ subroutine int_add(x, y, z)
  x = y + z
 end subroutine
 ! CHECK-LABEL: func.func @_QPint_add(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_8:.*]] = arith.addi %[[VAL_6]], %[[VAL_7]] : i32
@@ -17,8 +17,8 @@ subroutine real_add(x, y, z)
  x = y + z
 end subroutine
 ! CHECK-LABEL: func.func @_QPreal_add(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_8:.*]] = arith.addf %[[VAL_6]], %[[VAL_7]] fastmath<contract> : f32
@@ -28,8 +28,8 @@ subroutine complex_add(x, y, z)
  x = y + z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_add(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_8:.*]] = fir.addc %[[VAL_6]], %[[VAL_7]] {fastmath = #arith.fastmath<contract>} : !fir.complex<4>
@@ -39,8 +39,8 @@ subroutine int_sub(x, y, z)
  x = y - z
 end subroutine
 ! CHECK-LABEL: func.func @_QPint_sub(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_8:.*]] = arith.subi %[[VAL_6]], %[[VAL_7]] : i32
@@ -50,8 +50,8 @@ subroutine real_sub(x, y, z)
  x = y - z
 end subroutine
 ! CHECK-LABEL: func.func @_QPreal_sub(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_8:.*]] = arith.subf %[[VAL_6]], %[[VAL_7]] fastmath<contract> : f32
@@ -61,8 +61,8 @@ subroutine complex_sub(x, y, z)
  x = y - z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_sub(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_8:.*]] = fir.subc %[[VAL_6]], %[[VAL_7]] {fastmath = #arith.fastmath<contract>} : !fir.complex<4>
@@ -72,8 +72,8 @@ subroutine int_mul(x, y, z)
  x = y * z
 end subroutine
 ! CHECK-LABEL: func.func @_QPint_mul(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_8:.*]] = arith.muli %[[VAL_6]], %[[VAL_7]] : i32
@@ -83,8 +83,8 @@ subroutine real_mul(x, y, z)
  x = y * z
 end subroutine
 ! CHECK-LABEL: func.func @_QPreal_mul(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_8:.*]] = arith.mulf %[[VAL_6]], %[[VAL_7]] fastmath<contract> : f32
@@ -94,8 +94,8 @@ subroutine complex_mul(x, y, z)
  x = y * z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_mul(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_8:.*]] = fir.mulc %[[VAL_6]], %[[VAL_7]] {fastmath = #arith.fastmath<contract>} : !fir.complex<4>
@@ -105,8 +105,8 @@ subroutine int_div(x, y, z)
  x = y / z
 end subroutine
 ! CHECK-LABEL: func.func @_QPint_div(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_8:.*]] = arith.divsi %[[VAL_6]], %[[VAL_7]] : i32
@@ -116,8 +116,8 @@ subroutine real_div(x, y, z)
  x = y / z
 end subroutine
 ! CHECK-LABEL: func.func @_QPreal_div(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_8:.*]] = arith.divf %[[VAL_6]], %[[VAL_7]] fastmath<contract> : f32
@@ -127,8 +127,8 @@ subroutine complex_div(x, y, z)
  x = y / z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_div(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_8:.*]] = fir.extract_value %[[VAL_6]], [0 : index] : (!fir.complex<4>) -> f32
@@ -142,8 +142,8 @@ subroutine int_power(x, y, z)
   x = y**z
 end subroutine
 ! CHECK-LABEL: func.func @_QPint_power(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_8:.*]] = math.ipowi %[[VAL_6]], %[[VAL_7]] : i32
@@ -153,8 +153,8 @@ subroutine real_power(x, y, z)
   x = y**z
 end subroutine
 ! CHECK-LABEL: func.func @_QPreal_power(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_8:.*]] = math.powf %[[VAL_6]], %[[VAL_7]] fastmath<contract> : f32
@@ -164,8 +164,8 @@ subroutine complex_power(x, y, z)
   x = y**z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_power(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_8:.*]] = fir.call @cpowf(%[[VAL_6]], %[[VAL_7]]) fastmath<contract> : (!fir.complex<4>, !fir.complex<4>) -> !fir.complex<4>
@@ -177,8 +177,8 @@ subroutine real_to_int_power(x, y, z)
   x = y**z
 end subroutine
 ! CHECK-LABEL: func.func @_QPreal_to_int_power(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_8:.*]] = math.fpowi %[[VAL_6]], %[[VAL_7]] fastmath<contract> : f32, i32
@@ -189,8 +189,8 @@ subroutine complex_to_int_power(x, y, z)
   x = y**z
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_to_int_power(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_8:.*]] = fir.call @_FortranAcpowi(%[[VAL_6]], %[[VAL_7]]) fastmath<contract> : (!fir.complex<4>, i32) -> !fir.complex<4>
@@ -203,7 +203,7 @@ subroutine extremum(c, n, l)
   n = len(c, 8)
 end subroutine
 ! CHECK-LABEL: func.func @_QPextremum(
-! CHECK:  hlfir.declare {{.*}}c
+! CHECK:  hlfir.declare {{.*}}c"}
 ! CHECK:  %[[VAL_11:.*]] = arith.constant 0 : i64
 ! CHECK:  %[[VAL_12:.*]] = fir.load %{{.*}} : !fir.ref<i64>
 ! CHECK:  %[[VAL_13:.*]] = arith.cmpi sgt, %[[VAL_11]], %[[VAL_12]] : i64
@@ -281,8 +281,8 @@ subroutine cmp_char(l, x, y)
   l = x .eq. y
 end subroutine
 ! CHECK-LABEL: func.func @_QPcmp_char(
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}} typeparams %[[VAL_4:.*]]#1 {uniq_name = "_QFcmp_charEx"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
-! CHECK:  %[[VAL_7:.*]]:2 = hlfir.declare %{{.*}} typeparams %[[VAL_6:.*]]#1 {uniq_name = "_QFcmp_charEy"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}} typeparams %[[VAL_4:.*]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFcmp_charEx"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_7:.*]]:2 = hlfir.declare %{{.*}} typeparams %[[VAL_6:.*]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFcmp_charEy"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_8:.*]] = fir.convert %[[VAL_5]]#1 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
 ! CHECK:  %[[VAL_9:.*]] = fir.convert %[[VAL_7]]#1 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
 ! CHECK:  %[[VAL_10:.*]] = fir.convert %[[VAL_4]]#1 : (index) -> i64
@@ -296,8 +296,8 @@ subroutine logical_and(x, y, z)
   x = y.and.z
 end subroutine
 ! CHECK-LABEL: func.func @_QPlogical_and(
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}z"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK:  %[[VAL_8:.*]] = fir.convert %[[VAL_6]] : (!fir.logical<4>) -> i1
@@ -331,8 +331,8 @@ subroutine cmplx_ctor(z, x, y)
   z = cmplx(x, y)
 end subroutine
 ! CHECK-LABEL: func.func @_QPcmplx_ctor(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}y"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_8:.*]] = fir.undefined !fir.complex<4>
@@ -345,7 +345,7 @@ subroutine cmplx_ctor_2(z, x)
   z = cmplx(x, 1._8, kind=8)
 end subroutine
 ! CHECK-LABEL: func.func @_QPcmplx_ctor_2(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<f64>, !fir.dscope) -> (!fir.ref<f64>, !fir.ref<f64>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<f64>
 ! CHECK:  %[[VAL_5:.*]] = arith.constant 1.000000e+00 : f64
 ! CHECK:  %[[VAL_6:.*]] = fir.undefined !fir.complex<8>
diff --git a/flang/test/Lower/HLFIR/bindc-value-derived.f90 b/flang/test/Lower/HLFIR/bindc-value-derived.f90
index 671e6d45b9a9..7103d54c3e3d 100644
--- a/flang/test/Lower/HLFIR/bindc-value-derived.f90
+++ b/flang/test/Lower/HLFIR/bindc-value-derived.f90
@@ -17,7 +17,7 @@ contains
 ! CHECK-SAME:                    %[[VAL_0:.*]]: !fir.type<_QMbindc_byvalTt{i:i32}> {fir.bindc_name = "x"}) attributes {fir.bindc_name = "test"} {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QMbindc_byvalTt{i:i32}>
 ! CHECK:           fir.store %[[VAL_0]] to %[[VAL_1]] : !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QMbindc_byvalFtestEx"} : (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>) -> (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>, !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QMbindc_byvalFtestEx"} : (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>, !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>)
 ! CHECK:           %[[VAL_3:.*]] = hlfir.designate %[[VAL_2]]#0{"i"}   : (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>) -> !fir.ref<i32>
 ! CHECK:           fir.call @_QPuse_it(%[[VAL_3]]) fastmath<contract> : (!fir.ref<i32>) -> ()
 ! CHECK:           return
@@ -29,7 +29,7 @@ contains
   end subroutine
 ! CHECK-LABEL:   func.func @_QMbindc_byvalPcall_it(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QMbindc_byvalFcall_itEx"} : (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>) -> (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>, !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMbindc_byvalFcall_itEx"} : (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>, !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>
 ! CHECK:           fir.call @test(%[[VAL_2]]) fastmath<contract> : (!fir.type<_QMbindc_byvalTt{i:i32}>) -> ()
 ! CHECK:           return
diff --git a/flang/test/Lower/HLFIR/call-sequence-associated-descriptors.f90 b/flang/test/Lower/HLFIR/call-sequence-associated-descriptors.f90
index 7a2ea5cc14b6..bad647bb3ac9 100644
--- a/flang/test/Lower/HLFIR/call-sequence-associated-descriptors.f90
+++ b/flang/test/Lower/HLFIR/call-sequence-associated-descriptors.f90
@@ -23,7 +23,7 @@ contains
     call takes_char(x, 100)
   end subroutine
 ! CHECK-LABEL:   func.func @_QMbindc_seq_assocPtest_char_1(
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2:.*]](%[[VAL_5:.*]]) typeparams %[[VAL_1:.*]]#1 {uniq_name = "_QMbindc_seq_assocFtest_char_1Ex"} : (!fir.ref<!fir.array<10x20x!fir.char<1,?>>>, !fir.shape<2>, index) -> (!fir.box<!fir.array<10x20x!fir.char<1,?>>>, !fir.ref<!fir.array<10x20x!fir.char<1,?>>>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2:.*]](%[[VAL_5:.*]]) typeparams %[[VAL_1:.*]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QMbindc_seq_assocFtest_char_1Ex"} : (!fir.ref<!fir.array<10x20x!fir.char<1,?>>>, !fir.shape<2>, index, !fir.dscope) -> (!fir.box<!fir.array<10x20x!fir.char<1,?>>>, !fir.ref<!fir.array<10x20x!fir.char<1,?>>>)
 ! CHECK:           %[[VAL_7:.*]] = arith.constant 100 : i32
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_9:.*]] = fir.shift %[[VAL_8]], %[[VAL_8]] : (index, index) -> !fir.shift<2>
@@ -56,7 +56,7 @@ contains
     call takes_char(x, 100)
   end subroutine
 ! CHECK-LABEL:   func.func @_QMbindc_seq_assocPtest_char_copy_in_copy_out(
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] {uniq_name = "_QMbindc_seq_assocFtest_char_copy_in_copy_outEx"} : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.box<!fir.array<?x?x!fir.char<1,?>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMbindc_seq_assocFtest_char_copy_in_copy_outEx"} : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.box<!fir.array<?x?x!fir.char<1,?>>>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 100 : i32
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, i1)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
@@ -91,7 +91,7 @@ contains
     call takes_char_assumed_size(x)
   end subroutine
 ! CHECK-LABEL:   func.func @_QMbindc_seq_assocPtest_char_assumed_size(
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] {uniq_name = "_QMbindc_seq_assocFtest_char_assumed_sizeEx"} : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.box<!fir.array<?x?x!fir.char<1,?>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMbindc_seq_assocFtest_char_assumed_sizeEx"} : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.box<!fir.array<?x?x!fir.char<1,?>>>)
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, i1)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_4:.*]] = fir.shift %[[VAL_3]], %[[VAL_3]] : (index, index) -> !fir.shift<2>
@@ -123,7 +123,7 @@ contains
     call takes_optional_char(x, 100)
   end subroutine
 ! CHECK-LABEL:   func.func @_QMbindc_seq_assocPtest_optional_char(
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2:.*]](%[[VAL_5:.*]]) typeparams %[[VAL_1:.*]]#1 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QMbindc_seq_assocFtest_optional_charEx"} : (!fir.ref<!fir.array<10x20x!fir.char<1,?>>>, !fir.shape<2>, index) -> (!fir.box<!fir.array<10x20x!fir.char<1,?>>>, !fir.ref<!fir.array<10x20x!fir.char<1,?>>>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2:.*]](%[[VAL_5:.*]]) typeparams %[[VAL_1:.*]]#1 dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QMbindc_seq_assocFtest_optional_charEx"} : (!fir.ref<!fir.array<10x20x!fir.char<1,?>>>, !fir.shape<2>, index, !fir.dscope) -> (!fir.box<!fir.array<10x20x!fir.char<1,?>>>, !fir.ref<!fir.array<10x20x!fir.char<1,?>>>)
 ! CHECK:           %[[VAL_7:.*]] = fir.is_present %[[VAL_6]]#0 : (!fir.box<!fir.array<10x20x!fir.char<1,?>>>) -> i1
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:           %[[VAL_9:.*]] = fir.if %[[VAL_7]] -> (!fir.box<!fir.array<10x20x!fir.char<1,?>>>) {
@@ -186,7 +186,7 @@ contains
   end subroutine
 ! CHECK-LABEL:   func.func @_QMpoly_seq_assocPtest_poly_1(
 ! CHECK-SAME:                                             %[[VAL_0:.*]]: !fir.class<!fir.array<10x20xnone>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] {uniq_name = "_QMpoly_seq_assocFtest_poly_1Ex"} : (!fir.class<!fir.array<10x20xnone>>) -> (!fir.class<!fir.array<10x20xnone>>, !fir.class<!fir.array<10x20xnone>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMpoly_seq_assocFtest_poly_1Ex"} : (!fir.class<!fir.array<10x20xnone>>, !fir.dscope) -> (!fir.class<!fir.array<10x20xnone>>, !fir.class<!fir.array<10x20xnone>>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 100 : i32
 ! CHECK:           %[[VAL_3:.*]]:3 = hlfir.associate %[[VAL_2]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]]#1 {uniq_name = "_QMpoly_seq_assocFtakes_polyEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -214,7 +214,7 @@ contains
     call takes_poly(x, 100)
   end subroutine
 ! CHECK-LABEL:   func.func @_QMpoly_seq_assocPtest_poly_copy_in_copy_out(
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] {uniq_name = "_QMpoly_seq_assocFtest_poly_copy_in_copy_outEx"} : (!fir.class<!fir.array<?x?xnone>>) -> (!fir.class<!fir.array<?x?xnone>>, !fir.class<!fir.array<?x?xnone>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMpoly_seq_assocFtest_poly_copy_in_copy_outEx"} : (!fir.class<!fir.array<?x?xnone>>, !fir.dscope) -> (!fir.class<!fir.array<?x?xnone>>, !fir.class<!fir.array<?x?xnone>>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 100 : i32
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.class<!fir.array<?x?xnone>>) -> (!fir.class<!fir.array<?x?xnone>>, i1)
 ! CHECK:           %[[VAL_4:.*]]:3 = hlfir.associate %[[VAL_2]] {adapt.valuebyref} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
@@ -244,7 +244,7 @@ contains
     call takes_poly_assumed_size(x)
   end subroutine
 ! CHECK-LABEL:   func.func @_QMpoly_seq_assocPtest_poly_assumed_size(
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] {uniq_name = "_QMpoly_seq_assocFtest_poly_assumed_sizeEx"} : (!fir.class<!fir.array<?x?xnone>>) -> (!fir.class<!fir.array<?x?xnone>>, !fir.class<!fir.array<?x?xnone>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMpoly_seq_assocFtest_poly_assumed_sizeEx"} : (!fir.class<!fir.array<?x?xnone>>, !fir.dscope) -> (!fir.class<!fir.array<?x?xnone>>, !fir.class<!fir.array<?x?xnone>>)
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.class<!fir.array<?x?xnone>>) -> (!fir.class<!fir.array<?x?xnone>>, i1)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 10 : i64
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i64
@@ -271,7 +271,7 @@ contains
     call takes_optional_poly(x, 100)
   end subroutine
 ! CHECK-LABEL:   func.func @_QMpoly_seq_assocPtest_optional_poly(
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QMpoly_seq_assocFtest_optional_polyEx"} : (!fir.class<!fir.array<10x20xnone>>) -> (!fir.class<!fir.array<10x20xnone>>, !fir.class<!fir.array<10x20xnone>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:.*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QMpoly_seq_assocFtest_optional_polyEx"} : (!fir.class<!fir.array<10x20xnone>>, !fir.dscope) -> (!fir.class<!fir.array<10x20xnone>>, !fir.class<!fir.array<10x20xnone>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#0 : (!fir.class<!fir.array<10x20xnone>>) -> i1
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 100 : i32
 ! CHECK:           %[[VAL_4:.*]] = fir.if %[[VAL_2]] -> (!fir.class<!fir.array<10x20xnone>>) {
diff --git a/flang/test/Lower/HLFIR/calls-assumed-shape.f90 b/flang/test/Lower/HLFIR/calls-assumed-shape.f90
index a2094f1f1f0e..cfe607a69102 100644
--- a/flang/test/Lower/HLFIR/calls-assumed-shape.f90
+++ b/flang/test/Lower/HLFIR/calls-assumed-shape.f90
@@ -12,7 +12,7 @@ subroutine test_assumed_to_assumed(x)
   call takes_assumed(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_assumed_to_assumed(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {uniq_name = "_QFtest_assumed_to_assumedEx"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_assumed_to_assumedEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:  fir.call @_QPtakes_assumed(%[[VAL_1]]#0) {{.*}} : (!fir.box<!fir.array<?xf32>>) -> ()
 
 subroutine test_ptr_to_assumed(p)
@@ -25,7 +25,7 @@ subroutine test_ptr_to_assumed(p)
   call takes_assumed(p)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_ptr_to_assumed(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ptr_to_assumedEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ptr_to_assumedEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
 ! CHECK:  %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK:  %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.box<!fir.array<?xf32>>
 ! CHECK:  fir.call @_QPtakes_assumed(%[[VAL_3]]) {{.*}} : (!fir.box<!fir.array<?xf32>>) -> ()
@@ -40,7 +40,7 @@ subroutine test_ptr_to_contiguous_assumed(p)
   call takes_contiguous_assumed(p)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_ptr_to_contiguous_assumed(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ptr_to_contiguous_assumedEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ptr_to_contiguous_assumedEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
 ! CHECK:  %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK:  %[[VAL_3:.*]]:2 = hlfir.copy_in %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> (!fir.box<!fir.ptr<!fir.array<?xf32>>>, i1)
 ! CHECK:  %[[VAL_4:.*]] = fir.rebox %[[VAL_3]]#0 : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.box<!fir.array<?xf32>>
@@ -57,7 +57,7 @@ subroutine test_ptr_to_contiguous_assumed_classstar(p)
   call takes_contiguous_assumed_classstar(p)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_ptr_to_contiguous_assumed_classstar(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ptr_to_contiguous_assumed_classstarEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ptr_to_contiguous_assumed_classstarEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
 ! CHECK:  %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK:  %[[VAL_3:.*]]:2 = hlfir.copy_in %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> (!fir.box<!fir.ptr<!fir.array<?xf32>>>, i1)
 ! CHECK:  %[[VAL_4:.*]] = fir.rebox %[[VAL_3]]#0 : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.class<!fir.array<?xnone>>
@@ -74,7 +74,7 @@ subroutine test_ptr_to_assumed_typestar(p)
   call takes_assumed_typestar(p)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_ptr_to_assumed_typestar(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ptr_to_assumed_typestarEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ptr_to_assumed_typestarEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
 ! CHECK:  %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK:  %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.box<!fir.array<?xnone>>
 ! CHECK:  fir.call @_QPtakes_assumed_typestar(%[[VAL_3]]) {{.*}} : (!fir.box<!fir.array<?xnone>>) -> ()
@@ -94,7 +94,7 @@ end subroutine
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 10 : index
 ! CHECK:  %[[VAL_4:.*]] = arith.constant 20 : index
 ! CHECK:  %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_3]](%[[VAL_5:[a-z0-9]*]]) typeparams %[[VAL_2:[a-z0-9]*]] {uniq_name = "_QFtest_explicit_char_to_boxEe"} : (!fir.ref<!fir.array<20x!fir.char<1,10>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<20x!fir.char<1,10>>>, !fir.ref<!fir.array<20x!fir.char<1,10>>>)
+! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_3]](%[[VAL_5:[a-z0-9]*]]) typeparams %[[VAL_2:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_explicit_char_to_boxEe"} : (!fir.ref<!fir.array<20x!fir.char<1,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.ref<!fir.array<20x!fir.char<1,10>>>, !fir.ref<!fir.array<20x!fir.char<1,10>>>)
 ! CHECK:  %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0(%[[VAL_5]]) : (!fir.ref<!fir.array<20x!fir.char<1,10>>>, !fir.shape<1>) -> !fir.box<!fir.array<20x!fir.char<1,10>>>
 ! CHECK:  %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.array<20x!fir.char<1,10>>>) -> !fir.box<!fir.array<?x!fir.char<1,?>>>
 ! CHECK:  fir.call @_QPtakes_assumed_character(%[[VAL_8]]) {{.*}} : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> ()
@@ -109,7 +109,7 @@ subroutine test_explicit_by_val(x)
   call takes_explicit_by_value(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_explicit_by_val(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]](%[[VAL_2:[a-z0-9]*]]) {uniq_name = "_QFtest_explicit_by_valEx"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]](%[[VAL_2:[a-z0-9]*]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_explicit_by_valEx"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 ! CHECK:  %[[VAL_4:.*]] = hlfir.as_expr %[[VAL_3]]#0 : (!fir.ref<!fir.array<10xf32>>) -> !hlfir.expr<10xf32>
 ! CHECK:  %[[VAL_5:.*]]:3 = hlfir.associate %[[VAL_4]](%[[VAL_2]]) {adapt.valuebyref} : (!hlfir.expr<10xf32>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>, i1)
 ! CHECK:  fir.call @_QPtakes_explicit_by_value(%[[VAL_5]]#1) {{.*}} : (!fir.ref<!fir.array<10xf32>>) -> ()
diff --git a/flang/test/Lower/HLFIR/calls-constant-expr-arg.f90 b/flang/test/Lower/HLFIR/calls-constant-expr-arg.f90
index cfe8cf726045..7c8faf4fca8f 100644
--- a/flang/test/Lower/HLFIR/calls-constant-expr-arg.f90
+++ b/flang/test/Lower/HLFIR/calls-constant-expr-arg.f90
@@ -18,7 +18,7 @@ end subroutine sub
 ! CHECK-LABEL:   func.func @_QPsub(
 ! CHECK-SAME:                      %[[VAL_0:.*]]: !fir.ref<!fir.array<?xi32>> {fir.bindc_name = "i"},
 ! CHECK-SAME:                      %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFsubEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFsubEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i32>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i32) -> i64
 ! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i64) -> index
@@ -26,7 +26,7 @@ end subroutine sub
 ! CHECK:           %[[VAL_7:.*]] = arith.cmpi sgt, %[[VAL_5]], %[[VAL_6]] : index
 ! CHECK:           %[[VAL_8:.*]] = arith.select %[[VAL_7]], %[[VAL_5]], %[[VAL_6]] : index
 ! CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_9]]) {uniq_name = "_QFsubEi"} : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_9]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsubEi"} : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_11:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_12:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 0 : index
diff --git a/flang/test/Lower/HLFIR/calls-f77.f90 b/flang/test/Lower/HLFIR/calls-f77.f90
index cefe379a45d3..a970deb056f5 100644
--- a/flang/test/Lower/HLFIR/calls-f77.f90
+++ b/flang/test/Lower/HLFIR/calls-f77.f90
@@ -18,7 +18,7 @@ subroutine call_int_arg_var(n)
 end subroutine
 ! CHECK-LABEL: func.func @_QPcall_int_arg_var(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<i32>
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFcall_int_arg_varEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFcall_int_arg_varEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  fir.call @_QPtake_i4(%[[VAL_1]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
 
 subroutine call_int_arg_expr()
@@ -45,7 +45,7 @@ subroutine call_real_arg_var(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPcall_real_arg_var(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<f32>
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFcall_real_arg_varEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFcall_real_arg_varEx"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:  fir.call @_QPtake_r4(%[[VAL_1]]#1) fastmath<contract> : (!fir.ref<f32>) -> ()
 
 subroutine call_logical_arg_var(x)
@@ -54,7 +54,7 @@ subroutine call_logical_arg_var(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPcall_logical_arg_var(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.logical<4>>
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFcall_logical_arg_varEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFcall_logical_arg_varEx"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:  fir.call @_QPtake_l4(%[[VAL_1]]#1) fastmath<contract> : (!fir.ref<!fir.logical<4>>) -> ()
 
 subroutine call_logical_arg_expr()
@@ -84,7 +84,7 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPcall_char_arg_var(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.boxchar<1>
 ! CHECK:  %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]]#0 typeparams %[[VAL_1]]#1 {uniq_name = "_QFcall_char_arg_varEx"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]]#0 typeparams %[[VAL_1]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFcall_char_arg_varEx"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  fir.call @_QPtake_c(%[[VAL_2]]#0) fastmath<contract> : (!fir.boxchar<1>) -> ()
 
 subroutine call_char_arg_var_expr(x)
@@ -94,7 +94,7 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPcall_char_arg_var_expr(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.boxchar<1>
 ! CHECK:  %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]]#0 typeparams %[[VAL_1]]#1 {uniq_name = "_QFcall_char_arg_var_exprEx"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]]#0 typeparams %[[VAL_1]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFcall_char_arg_var_exprEx"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_3:.*]] = arith.addi %[[VAL_1]]#1, %[[VAL_1]]#1 : index
 ! CHECK:  %[[VAL_4:.*]] = hlfir.concat %[[VAL_2]]#0, %[[VAL_2]]#0 len %[[VAL_3]] : (!fir.boxchar<1>, !fir.boxchar<1>, index) -> !hlfir.expr<!fir.char<1,?>>
 ! CHECK:  %[[VAL_5:.*]]:3 = hlfir.associate %[[VAL_4]] typeparams %[[VAL_3]] {adapt.valuebyref} : (!hlfir.expr<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>, i1)
@@ -110,7 +110,7 @@ end subroutine
 ! CHECK:  %[[VAL_1:.*]] = arith.constant 10 : index
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 20 : index
 ! CHECK:  %[[VAL_3:.*]] = fir.shape %[[VAL_1]], %[[VAL_2]] : (index, index) -> !fir.shape<2>
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) {uniq_name = "_QFcall_arg_array_varEn"} : (!fir.ref<!fir.array<10x20xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<10x20xi32>>, !fir.ref<!fir.array<10x20xi32>>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFcall_arg_array_varEn"} : (!fir.ref<!fir.array<10x20xi32>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<10x20xi32>>, !fir.ref<!fir.array<10x20xi32>>)
 ! CHECK:  fir.call @_QPtake_arr(%[[VAL_4]]#1) fastmath<contract> : (!fir.ref<!fir.array<10x20xi32>>) -> ()
 
 subroutine call_arg_array_2(n)
@@ -119,7 +119,7 @@ subroutine call_arg_array_2(n)
 end subroutine
 ! CHECK-LABEL: func.func @_QPcall_arg_array_2(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>>
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<contiguous, optional>, uniq_name = "_QFcall_arg_array_2En"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<contiguous, optional>, uniq_name = "_QFcall_arg_array_2En"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
 ! CHECK:  %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#1 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.ref<!fir.array<?x?xi32>>
 ! CHECK:  fir.call @_QPtake_arr_2(%[[VAL_2]]) fastmath<contract> : (!fir.ref<!fir.array<?x?xi32>>) -> ()
 
diff --git a/flang/test/Lower/HLFIR/calls-optional.f90 b/flang/test/Lower/HLFIR/calls-optional.f90
index df9519a24fb7..1ada5b198aed 100644
--- a/flang/test/Lower/HLFIR/calls-optional.f90
+++ b/flang/test/Lower/HLFIR/calls-optional.f90
@@ -14,7 +14,7 @@ subroutine optional_copy_in_out(x)
   call  takes_optional_explicit(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPoptional_copy_in_out(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_copy_in_outEx"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_copy_in_outEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:  %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#0 : (!fir.box<!fir.array<?xf32>>) -> i1
 ! CHECK:  %[[VAL_3:.*]]:4 = fir.if %[[VAL_2]] -> (!fir.ref<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>, i1, !fir.box<!fir.array<?xf32>>) {
 ! CHECK:    %[[VAL_4:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, i1)
@@ -40,7 +40,7 @@ subroutine optional_value_copy(x)
   call  takes_optional_explicit_value(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPoptional_value_copy(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]](%[[VAL_2:[a-z0-9]*]]) {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_value_copyEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]](%[[VAL_2:[a-z0-9]*]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_value_copyEx"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.is_present %[[VAL_3]]#0 : (!fir.ref<!fir.array<100xf32>>) -> i1
 ! CHECK:  %[[VAL_5:.*]]:3 = fir.if %[[VAL_4]] -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>, i1) {
 ! CHECK:    %[[VAL_6:.*]] = hlfir.as_expr %[[VAL_3]]#0 : (!fir.ref<!fir.array<100xf32>>) -> !hlfir.expr<100xf32>
@@ -66,8 +66,8 @@ subroutine elem_pointer_to_optional(x, y)
   call elem_takes_two_optional(x, y)
 end subroutine
 ! CHECK-LABEL: func.func @_QPelem_pointer_to_optional(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {uniq_name = "_QFelem_pointer_to_optionalEx"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFelem_pointer_to_optionalEy"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFelem_pointer_to_optionalEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFelem_pointer_to_optionalEy"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK:  %[[VAL_5:.*]] = fir.box_addr %[[VAL_4]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.ptr<!fir.array<?xf32>>
 ! CHECK:  %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.ptr<!fir.array<?xf32>>) -> i64
@@ -105,7 +105,7 @@ subroutine optional_cannot_be_absent_optional(x)
   call elem_takes_one_optional(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPoptional_cannot_be_absent_optional(
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_cannot_be_absent_optionalEx"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_cannot_be_absent_optionalEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 0 : index
 ! CHECK:  %[[VAL_3:.*]]:3 = fir.box_dims %[[VAL_1]]#0, %[[VAL_2]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 ! CHECK:  %[[VAL_4:.*]] = arith.constant 1 : index
@@ -125,8 +125,8 @@ subroutine optional_elem_poly(x, y)
   call elem_optional_poly(x, y)
 end subroutine
 ! CHECK-LABEL: func.func @_QPoptional_elem_poly(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {uniq_name = "_QFoptional_elem_polyEx"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_elem_polyEy"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFoptional_elem_polyEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFoptional_elem_polyEy"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.is_present %[[VAL_3]]#0 : (!fir.box<!fir.array<?xf32>>) -> i1
 ! CHECK:  %[[VAL_5:.*]] = arith.constant 0 : index
 ! CHECK:  %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_2]]#0, %[[VAL_5]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
diff --git a/flang/test/Lower/HLFIR/calls-percent-val-ref.f90 b/flang/test/Lower/HLFIR/calls-percent-val-ref.f90
index c6acc42455f1..c8724e6d7bee 100644
--- a/flang/test/Lower/HLFIR/calls-percent-val-ref.f90
+++ b/flang/test/Lower/HLFIR/calls-percent-val-ref.f90
@@ -7,7 +7,7 @@ subroutine test_val_1(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_val_1(
 ! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_val_1Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_val_1Ex"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<i32>
 ! CHECK:           fir.call @_QPval1(%[[VAL_2]]) fastmath<contract> : (i32) -> ()
 
@@ -17,7 +17,7 @@ subroutine test_val_2(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_val_2(
 ! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_val_2Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>>, !fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_val_2Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>>, !fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.complex<4>>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.heap<!fir.complex<4>>>) -> !fir.heap<!fir.complex<4>>
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.heap<!fir.complex<4>>
@@ -32,7 +32,7 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_ref_char(
 ! CHECK-SAME:                                %[[VAL_0:.*]]: !fir.boxchar<1> {fir.bindc_name = "x"}) {
 ! CHECK:           %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]]#0 typeparams %[[VAL_1]]#1 {uniq_name = "_QFtest_ref_charEx"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]]#0 typeparams %[[VAL_1]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_ref_charEx"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:           %[[VAL_3:.*]]:2 = fir.unboxchar %[[VAL_2]]#0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK:           fir.call @_QPref_char(%[[VAL_3]]#0) fastmath<contract> : (!fir.ref<!fir.char<1,?>>) -> ()
 
@@ -42,7 +42,7 @@ subroutine test_ref_1(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_ref_1(
 ! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_ref_1Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_ref_1Ex"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           fir.call @_QPref1(%[[VAL_1]]#1) fastmath<contract> : (!fir.ref<i32>) -> ()
 
 subroutine test_ref_2(x)
@@ -51,7 +51,7 @@ subroutine test_ref_2(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_ref_2(
 ! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ref_2Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_ref_2Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.complex<4>>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.complex<4>>>) -> !fir.ptr<!fir.complex<4>>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ptr<!fir.complex<4>>) -> !fir.ref<!fir.complex<4>>
@@ -63,7 +63,7 @@ subroutine test_skip_copy_in_out(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_skip_copy_in_out(
 ! CHECK-SAME:                                        %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_skip_copy_in_outEx"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_skip_copy_in_outEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#1 : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
 ! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<?xf32>>) -> i64
 ! CHECK:           fir.call @_QPval3(%[[VAL_3]]) fastmath<contract> : (i64) -> ()
diff --git a/flang/test/Lower/HLFIR/calls-poly-to-assumed-type.f90 b/flang/test/Lower/HLFIR/calls-poly-to-assumed-type.f90
index b14f1bb1f443..05885e729f93 100644
--- a/flang/test/Lower/HLFIR/calls-poly-to-assumed-type.f90
+++ b/flang/test/Lower/HLFIR/calls-poly-to-assumed-type.f90
@@ -12,7 +12,7 @@ subroutine pass_poly_to_assumed_type_assumed_size(x)
   call assumed_type_assumed_size(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPpass_poly_to_assumed_type_assumed_size(
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFpass_poly_to_assumed_type_assumed_sizeEx"} : (!fir.class<!fir.array<?x?xnone>>) -> (!fir.class<!fir.array<?x?xnone>>, !fir.class<!fir.array<?x?xnone>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFpass_poly_to_assumed_type_assumed_sizeEx"} : (!fir.class<!fir.array<?x?xnone>>, !fir.dscope) -> (!fir.class<!fir.array<?x?xnone>>, !fir.class<!fir.array<?x?xnone>>)
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.class<!fir.array<?x?xnone>>) -> (!fir.class<!fir.array<?x?xnone>>, i1)
 ! CHECK:           %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]]#0 : (!fir.class<!fir.array<?x?xnone>>) -> !fir.ref<!fir.array<?x?xnone>>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.array<?x?xnone>>) -> !fir.ref<!fir.array<?xnone>>
diff --git a/flang/test/Lower/HLFIR/char_extremum.f03 b/flang/test/Lower/HLFIR/char_extremum.f03
index cc7b80184935..4996128a3753 100644
--- a/flang/test/Lower/HLFIR/char_extremum.f03
+++ b/flang/test/Lower/HLFIR/char_extremum.f03
@@ -8,11 +8,11 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPmax1
 ! CHECK:  %[[VAL_0:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %{{.*}} : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_1:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_1:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_2:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %{{.*}} : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_4:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %{{.*}} : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_5:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_4]]#0 typeparams %[[VAL_4]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_5:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_4]]#0 typeparams %[[VAL_4]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_6:[a-zA-Z0-9_]*]] = hlfir.char_extremum max, %[[VAL_3]]#0, %[[VAL_5]]#0 : (!fir.boxchar<1>, !fir.boxchar<1>) -> !hlfir.expr<!fir.char<1,?>>
 ! CHECK:  hlfir.assign %[[VAL_6]] to %[[VAL_1]]#0 : !hlfir.expr<!fir.char<1,?>>, !fir.boxchar<1>
 ! CHECK:  hlfir.destroy %[[VAL_6]] : !hlfir.expr<!fir.char<1,?>>
@@ -23,11 +23,11 @@ subroutine min1(c1, c2, c3)
 end subroutine
 ! CHECK-LABEL: func.func @_QPmin1
 ! CHECK:  %[[VAL_0:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %{{.*}} : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_1:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_1:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_2:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %{{.*}} : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_4:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %{{.*}} : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_5:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_4]]#0 typeparams %[[VAL_4]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_5:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_4]]#0 typeparams %[[VAL_4]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_6:[a-zA-Z0-9_]*]] = hlfir.char_extremum min, %[[VAL_3]]#0, %[[VAL_5]]#0 : (!fir.boxchar<1>, !fir.boxchar<1>) -> !hlfir.expr<!fir.char<1,?>>
 ! CHECK:  hlfir.assign %[[VAL_6]] to %[[VAL_1]]#0 : !hlfir.expr<!fir.char<1,?>>, !fir.boxchar<1>
 ! CHECK:  hlfir.destroy %[[VAL_6]] : !hlfir.expr<!fir.char<1,?>>
@@ -43,19 +43,19 @@ end subroutine
 ! CHECK:  %[[VAL_1:[a-zA-Z0-9_]*]] = fir.convert %[[VAL_0]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<100x!fir.char<1,?>>>
 ! CHECK:  %[[VAL_C100:[a-zA-Z0-9_]*]] = arith.constant 100 : index
 ! CHECK:  %[[VAL_2:[a-zA-Z0-9_]*]]  = fir.shape %[[VAL_C100]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_2]]) typeparams %[[VAL_0]]#1 {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,?>>>, !fir.shape<1>, index) -> (!fir.box<!fir.array<100x!fir.char<1,?>>>, !fir.ref<!fir.array<100x!fir.char<1,?>>>)
+! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_2]]) typeparams %[[VAL_0]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<100x!fir.char<1,?>>>, !fir.ref<!fir.array<100x!fir.char<1,?>>>)
 ! CHECK:  %[[VAL_4:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %{{.*}} : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK:  %[[VAL_5:[a-zA-Z0-9_]*]] = fir.convert %[[VAL_4]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<100x!fir.char<1,10>>>
 ! CHECK:  %[[VAL_C10:[a-zA-Z0-9_]*]] = arith.constant 10 : index
 ! CHECK:  %[[VAL_C100_0:[a-zA-Z0-9_]*]] = arith.constant 100 : index
 ! CHECK:  %[[VAL_6:[a-zA-Z0-9_]*]] = fir.shape %[[VAL_C100_0]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_7:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_5]](%[[VAL_6]]) typeparams %[[VAL_C10]] {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,10>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<100x!fir.char<1,10>>>, !fir.ref<!fir.array<100x!fir.char<1,10>>>)
+! CHECK:  %[[VAL_7:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_5]](%[[VAL_6]]) typeparams %[[VAL_C10]] dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.char<1,10>>>, !fir.ref<!fir.array<100x!fir.char<1,10>>>)
 ! CHECK:  %[[VAL_8:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %{{.*}} : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK:  %[[VAL_9:[a-zA-Z0-9_]*]] = fir.convert %[[VAL_8]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<100x!fir.char<1,20>>>
 ! CHECK:  %[[VAL_C20:[a-zA-Z0-9_]*]] = arith.constant 20 : index
 ! CHECK:  %[[VAL_C100_1:[a-zA-Z0-9_]*]] = arith.constant 100 : index
 ! CHECK:  %[[VAL_10:[a-zA-Z0-9_]*]] = fir.shape %[[VAL_C100_1]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_11:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_9]](%[[VAL_10]]) typeparams %[[VAL_C20]] {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,20>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<100x!fir.char<1,20>>>, !fir.ref<!fir.array<100x!fir.char<1,20>>>)
+! CHECK:  %[[VAL_11:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_9]](%[[VAL_10]]) typeparams %[[VAL_C20]] dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,20>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.char<1,20>>>, !fir.ref<!fir.array<100x!fir.char<1,20>>>)
 ! CHECK:  %[[VAL_C1:[a-zA-Z0-9_]*]] = arith.constant 1 : index
 ! CHECK:  %[[VAL_12:[a-zA-Z0-9_]*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_C1]])  typeparams %[[VAL_C10]] : (!fir.ref<!fir.array<100x!fir.char<1,10>>>, index, index) -> !fir.ref<!fir.char<1,10>>
 ! CHECK:  %[[VAL_C1_2:[a-zA-Z0-9_]*]] = arith.constant 1 : index
@@ -76,19 +76,19 @@ end subroutine
 ! CHECK:  %[[VAL_1:[a-zA-Z0-9_]*]] = fir.convert %[[VAL_0]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<100x!fir.char<1,?>>>
 ! CHECK:  %[[VAL_C100:[a-zA-Z0-9_]*]] = arith.constant 100 : index
 ! CHECK:  %[[VAL_2:[a-zA-Z0-9_]*]] = fir.shape %[[VAL_C100]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_2]]) typeparams %[[VAL_0]]#1 {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,?>>>, !fir.shape<1>, index) -> (!fir.box<!fir.array<100x!fir.char<1,?>>>, !fir.ref<!fir.array<100x!fir.char<1,?>>>)
+! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_2]]) typeparams %[[VAL_0]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<100x!fir.char<1,?>>>, !fir.ref<!fir.array<100x!fir.char<1,?>>>)
 ! CHECK:  %[[VAL_4:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK:  %[[VAL_5:[a-zA-Z0-9_]*]] = fir.convert %[[VAL_4]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<100x!fir.char<1,10>>>
 ! CHECK:  %[[VAL_C10:[a-zA-Z0-9_]*]] = arith.constant 10 : index
 ! CHECK:  %[[VAL_C100_0:[a-zA-Z0-9_]*]] = arith.constant 100 : index
 ! CHECK:  %[[VAL_6:[a-zA-Z0-9_]*]] = fir.shape %[[VAL_C100_0]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_7:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_5]](%[[VAL_6]]) typeparams %[[VAL_C10]] {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,10>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<100x!fir.char<1,10>>>, !fir.ref<!fir.array<100x!fir.char<1,10>>>)
+! CHECK:  %[[VAL_7:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_5]](%[[VAL_6]]) typeparams %[[VAL_C10]] dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.char<1,10>>>, !fir.ref<!fir.array<100x!fir.char<1,10>>>)
 ! CHECK:  %[[VAL_8:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %arg2 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK:  %[[VAL_C9:[a-zA-Z0-9_]*]] = fir.convert %[[VAL_8]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<100x!fir.char<1,20>>>
 ! CHECK:  %[[VAL_C20:[a-zA-Z0-9_]*]] = arith.constant 20 : index
 ! CHECK:  %[[VAL_C100_1:[a-zA-Z0-9_]*]] = arith.constant 100 : index
 ! CHECK:  %[[VAL_10:[a-zA-Z0-9_]*]] = fir.shape %[[VAL_C100_1]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_11:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_C9]](%[[VAL_10]]) typeparams %[[VAL_C20]] {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,20>>>, !fir.shape<1>, index) -> (!fir.ref<!fir.array<100x!fir.char<1,20>>>, !fir.ref<!fir.array<100x!fir.char<1,20>>>)
+! CHECK:  %[[VAL_11:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_C9]](%[[VAL_10]]) typeparams %[[VAL_C20]] dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.array<100x!fir.char<1,20>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.char<1,20>>>, !fir.ref<!fir.array<100x!fir.char<1,20>>>)
 ! CHECK:  %[[VAL_C1:[a-zA-Z0-9_]*]] = arith.constant 1 : index
 ! CHECK:  %[[VAL_12:[a-zA-Z0-9_]*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_C1]])  typeparams %[[VAL_C10]] : (!fir.ref<!fir.array<100x!fir.char<1,10>>>, index, index) -> !fir.ref<!fir.char<1,10>>
 ! CHECK:  %[[VAL_C1_2:[a-zA-Z0-9_]*]] = arith.constant 1 : index
@@ -105,13 +105,13 @@ subroutine max3(c1, c2, c3, c4)
 end subroutine
 ! CHECK-LABEL: func.func @_QPmax3
 ! CHECK:  %[[VAL_0:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %arg0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_1:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_1:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_2:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_4:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %arg2 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_5:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_4]]#0 typeparams %[[VAL_4]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_5:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_4]]#0 typeparams %[[VAL_4]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_6:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %arg3 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_7:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_6]]#0 typeparams %[[VAL_6]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_7:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_6]]#0 typeparams %[[VAL_6]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_8:[a-zA-Z0-9_]*]] = hlfir.char_extremum max, %[[VAL_3]]#0, %[[VAL_5]]#0, %[[VAL_7]]#0 : (!fir.boxchar<1>, !fir.boxchar<1>, !fir.boxchar<1>) -> !hlfir.expr<!fir.char<1,?>>
 ! CHECK:  hlfir.assign %[[VAL_8]] to %[[VAL_1]]#0 : !hlfir.expr<!fir.char<1,?>>, !fir.boxchar<1>
 ! CHECK:  hlfir.destroy %[[VAL_8]] : !hlfir.expr<!fir.char<1,?>>
@@ -122,13 +122,13 @@ subroutine min3(c1, c2, c3, c4)
 end subroutine
 ! CHECK-LABEL: func.func @_QPmin3
 ! CHECK:  %[[VAL_0:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %arg0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_1:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_1:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_2:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_3:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_4:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %arg2 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_5:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_4]]#0 typeparams %[[VAL_4]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_5:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_4]]#0 typeparams %[[VAL_4]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_6:[a-zA-Z0-9_]*]]:2 = fir.unboxchar %arg3 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_7:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_6]]#0 typeparams %[[VAL_6]]#1 {{.*}} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_7:[a-zA-Z0-9_]*]]:2 = hlfir.declare %[[VAL_6]]#0 typeparams %[[VAL_6]]#1 dummy_scope %{{[0-9]+}} {{.*}} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:  %[[VAL_8:[a-zA-Z0-9_]*]] = hlfir.char_extremum min, %[[VAL_3]]#0, %[[VAL_5]]#0, %[[VAL_7]]#0 : (!fir.boxchar<1>, !fir.boxchar<1>, !fir.boxchar<1>) -> !hlfir.expr<!fir.char<1,?>>
 ! CHECK:  hlfir.assign %[[VAL_8]] to %[[VAL_1]]#0 : !hlfir.expr<!fir.char<1,?>>, !fir.boxchar<1>
 ! CHECK:  hlfir.destroy %[[VAL_8]] : !hlfir.expr<!fir.char<1,?>>
diff --git a/flang/test/Lower/HLFIR/charconvert.f90 b/flang/test/Lower/HLFIR/charconvert.f90
index 117fdc0d3ad4..45b0f356617a 100644
--- a/flang/test/Lower/HLFIR/charconvert.f90
+++ b/flang/test/Lower/HLFIR/charconvert.f90
@@ -13,7 +13,7 @@ subroutine charconvert1(c,n)
 end subroutine charconvert1
 
 ! CHECK-LABEL: func.func @_QPcharconvert1
-! CHECK:   %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFcharconvert1Ec"} : (!fir.box<!fir.array<?x!fir.char<4,?>>>) -> (!fir.box<!fir.array<?x!fir.char<4,?>>>, !fir.box<!fir.array<?x!fir.char<4,?>>>)
+! CHECK:   %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFcharconvert1Ec"} : (!fir.box<!fir.array<?x!fir.char<4,?>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<4,?>>>, !fir.box<!fir.array<?x!fir.char<4,?>>>)
 ! CHECK:   ^bb0(%[[ARG2:.*]]: index):
 ! CHECK:     %[[VAL_37:.*]] = fir.box_elesize %[[VAL_2]]#1 : (!fir.box<!fir.array<?x!fir.char<4,?>>>) -> index
 ! CHECK:     %[[C4_4:.*]] = arith.constant 4 : index
@@ -36,7 +36,7 @@ end subroutine charconvert2
 ! CHECK:   %[[C1:.*]] = arith.constant 1 : index
 ! CHECK:   %[[VAL_1:.*]] = fir.alloca !fir.char<4> {bindc_name = "cx", uniq_name = "_QFcharconvert2Ecx"}
 ! CHECK:   %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] typeparams %[[C1]] {uniq_name = "_QFcharconvert2Ecx"} : (!fir.ref<!fir.char<4>>, index) -> (!fir.ref<!fir.char<4>>, !fir.ref<!fir.char<4>>)
-! CHECK:   %[[VAL_3:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFcharconvert2Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:   %[[VAL_3:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFcharconvert2Ex"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:   %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
 ! CHECK:   %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i32) -> i64
 ! CHECK:   %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i64) -> i8
@@ -58,9 +58,9 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPcharconvert3
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.boxchar<1> {{.*}}, %[[ARG1:.*]]: !fir.boxchar<4> 
 ! CHECK:   %[[VAL_0:.*]]:2 = fir.unboxchar %[[ARG0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:   %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 {uniq_name = "_QFcharconvert3Ec"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:   %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFcharconvert3Ec"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:   %[[VAL_2:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<4>) -> (!fir.ref<!fir.char<4,?>>, index)
-! CHECK:   %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 {uniq_name = "_QFcharconvert3Ec4"} : (!fir.ref<!fir.char<4,?>>, index) -> (!fir.boxchar<4>, !fir.ref<!fir.char<4,?>>)
+! CHECK:   %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFcharconvert3Ec4"} : (!fir.ref<!fir.char<4,?>>, index, !fir.dscope) -> (!fir.boxchar<4>, !fir.ref<!fir.char<4,?>>)
 ! CHECK:   %[[VAL_4:.*]] = arith.addi %[[VAL_0]]#1, %[[VAL_0]]#1 : index
 ! CHECK:   %[[VAL_5:.*]] = hlfir.concat %[[VAL_1]]#0, %[[VAL_1]]#0 len %[[VAL_4]] : (!fir.boxchar<1>, !fir.boxchar<1>, index) -> !hlfir.expr<!fir.char<1,?>>
 ! CHECK:   %[[VAL_7:.*]]:3 = hlfir.associate %[[VAL_5]] typeparams %[[VAL_4]] {adapt.valuebyref} : (!hlfir.expr<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>, i1)
diff --git a/flang/test/Lower/HLFIR/convert-mbox-to-value.f90 b/flang/test/Lower/HLFIR/convert-mbox-to-value.f90
index b9d55d3fde4f..ef9c12102a56 100644
--- a/flang/test/Lower/HLFIR/convert-mbox-to-value.f90
+++ b/flang/test/Lower/HLFIR/convert-mbox-to-value.f90
@@ -7,7 +7,7 @@ subroutine test_int_allocatable(a)
 end subroutine test_int_allocatable
 ! CHECK-LABEL:   func.func @_QPtest_int_allocatable(
 ! CHECK-SAME:                                       %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<i32>>> {fir.bindc_name = "a"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_int_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_int_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 6 : i32
 ! CHECK:           %[[VAL_3:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
@@ -27,7 +27,7 @@ subroutine test_int_pointer(p)
 end subroutine test_int_pointer
 ! CHECK-LABEL:   func.func @_QPtest_int_pointer(
 ! CHECK-SAME:                                   %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "p"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_int_pointerEp"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_int_pointerEp"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 6 : i32
 ! CHECK:           %[[VAL_3:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
@@ -49,7 +49,7 @@ end subroutine test_char_allocatable
 ! CHECK-LABEL:   func.func @_QPtest_char_allocatable(
 ! CHECK-SAME:                                        %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.char<1,11>>>> {fir.bindc_name = "a"}) {
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 11 : index
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_1]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_char_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.char<1,11>>>>, index) -> (!fir.ref<!fir.box<!fir.heap<!fir.char<1,11>>>>, !fir.ref<!fir.box<!fir.heap<!fir.char<1,11>>>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_char_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.char<1,11>>>>, index, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.char<1,11>>>>, !fir.ref<!fir.box<!fir.heap<!fir.char<1,11>>>>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtest_char_allocatableEi"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFtest_char_allocatableEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.char<1,11>>>>
@@ -86,7 +86,7 @@ end subroutine test_char_pointer
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtest_char_pointerEi"}
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest_char_pointerEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 11 : index
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_3]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_char_pointerEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,11>>>>, index) -> (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,11>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.char<1,11>>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_3]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_char_pointerEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,11>>>>, index, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,11>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.char<1,11>>>>)
 ! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,11>>>>
 ! CHECK:           %[[VAL_6:.*]] = fir.box_addr %[[VAL_5]] : (!fir.box<!fir.ptr<!fir.char<1,11>>>) -> !fir.ptr<!fir.char<1,11>>
 ! CHECK:           %[[VAL_3B:.*]] = arith.constant 11 : index
@@ -120,7 +120,7 @@ end subroutine test_dyn_char_allocatable
 ! CHECK-SAME:                                            %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>> {fir.bindc_name = "a"}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
 ! CHECK:           %[[VAL_2:.*]] = fir.box_elesize %[[VAL_1]] : (!fir.box<!fir.heap<!fir.char<1,?>>>) -> index
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_2]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_dyn_char_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, index) -> (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_2]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_dyn_char_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, index, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtest_dyn_char_allocatableEi"}
 ! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFtest_dyn_char_allocatableEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
@@ -157,7 +157,7 @@ end subroutine test_dyn_char_pointer
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest_dyn_char_pointerEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
 ! CHECK:           %[[VAL_4:.*]] = fir.box_elesize %[[VAL_3]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> index
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_4]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_dyn_char_pointerEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, index) -> (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_4]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_dyn_char_pointerEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, index, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
 ! CHECK:           %[[VAL_7:.*]] = fir.box_addr %[[VAL_6]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> !fir.ptr<!fir.char<1,?>>
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 1 : index
@@ -201,7 +201,7 @@ end subroutine test_derived_allocatable
 ! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]] : (!fir.heap<!fir.type<_QFtest_derived_allocatableTt>>) -> !fir.class<!fir.heap<!fir.type<_QFtest_derived_allocatableTt>>>
 ! CHECK:           fir.store %[[VAL_7]] to %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtest_derived_allocatableTt>>>>
 ! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_5]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_derived_allocatableEa2"} : (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtest_derived_allocatableTt>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtest_derived_allocatableTt>>>>, !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtest_derived_allocatableTt>>>>)
-! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_derived_allocatableEl"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_derived_allocatableEl"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_10:.*]] = fir.alloca !fir.class<!fir.heap<!fir.type<_QFtest_derived_allocatableTt>>> {bindc_name = "r", uniq_name = "_QFtest_derived_allocatableEr"}
 ! CHECK:           %[[VAL_11:.*]] = fir.zero_bits !fir.heap<!fir.type<_QFtest_derived_allocatableTt>>
 ! CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_11]] : (!fir.heap<!fir.type<_QFtest_derived_allocatableTt>>) -> !fir.class<!fir.heap<!fir.type<_QFtest_derived_allocatableTt>>>
@@ -241,7 +241,7 @@ end subroutine test_derived_pointer
 ! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]] : (!fir.heap<!fir.type<_QFtest_derived_pointerTt>>) -> !fir.class<!fir.heap<!fir.type<_QFtest_derived_pointerTt>>>
 ! CHECK:           fir.store %[[VAL_7]] to %[[VAL_5]] : !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtest_derived_pointerTt>>>>
 ! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_5]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_derived_pointerEa2"} : (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtest_derived_pointerTt>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.type<_QFtest_derived_pointerTt>>>>, !fir.ref<!fir.class<!fir.heap<!fir.type<_QFtest_derived_pointerTt>>>>)
-! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_derived_pointerEl"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_derived_pointerEl"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_10:.*]] = fir.alloca !fir.class<!fir.heap<!fir.type<_QFtest_derived_pointerTt>>> {bindc_name = "r", uniq_name = "_QFtest_derived_pointerEr"}
 ! CHECK:           %[[VAL_11:.*]] = fir.zero_bits !fir.heap<!fir.type<_QFtest_derived_pointerTt>>
 ! CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_11]] : (!fir.heap<!fir.type<_QFtest_derived_pointerTt>>) -> !fir.class<!fir.heap<!fir.type<_QFtest_derived_pointerTt>>>
diff --git a/flang/test/Lower/HLFIR/convert-variable-block.f90 b/flang/test/Lower/HLFIR/convert-variable-block.f90
index 30f8eacaaed1..dad6bc14fbdb 100644
--- a/flang/test/Lower/HLFIR/convert-variable-block.f90
+++ b/flang/test/Lower/HLFIR/convert-variable-block.f90
@@ -12,7 +12,7 @@ subroutine test(n)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest(
 ! CHECK-SAME:                       %[[VAL_0:.*]]: !fir.ref<i64> {fir.bindc_name = "n"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtestEn"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtestEn"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:           fir.call @_QPbefore_block() {{.*}}: () -> ()
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<i64>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i64) -> index
diff --git a/flang/test/Lower/HLFIR/convert-variable.f90 b/flang/test/Lower/HLFIR/convert-variable.f90
index e7487ef870d1..7acb1be578b9 100644
--- a/flang/test/Lower/HLFIR/convert-variable.f90
+++ b/flang/test/Lower/HLFIR/convert-variable.f90
@@ -6,7 +6,7 @@ subroutine scalar_numeric(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_numeric(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<i32>
-! CHECK:  %[[VAL_1:.*]] = hlfir.declare %[[VAL_0]] {uniq_name = "_QFscalar_numericEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_1:.*]] = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFscalar_numericEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 
 subroutine scalar_character(c)
   character(*) :: c
@@ -14,7 +14,7 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_character(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.boxchar<1>
 ! CHECK:  %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:  %[[VAL_2:.*]] = hlfir.declare %[[VAL_1]]#0 typeparams %[[VAL_1]]#1 {uniq_name = "_QFscalar_characterEc"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_2:.*]] = hlfir.declare %[[VAL_1]]#0 typeparams %[[VAL_1]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFscalar_characterEc"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 
 subroutine scalar_character_cst_len(c)
   character(10) :: c
@@ -24,7 +24,7 @@ end subroutine
 ! CHECK:  %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK:  %[[VAL_3:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,10>>
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 10 : index
-! CHECK:  %[[VAL_4:.*]] = hlfir.declare %[[VAL_3]] typeparams %[[VAL_2]] {uniq_name = "_QFscalar_character_cst_lenEc"} : (!fir.ref<!fir.char<1,10>>, index) -> (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>)
+! CHECK:  %[[VAL_4:.*]] = hlfir.declare %[[VAL_3]] typeparams %[[VAL_2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFscalar_character_cst_lenEc"} : (!fir.ref<!fir.char<1,10>>, index, !fir.dscope) -> (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>)
 
 subroutine array_numeric(x)
   integer :: x(10, 20)
@@ -34,7 +34,7 @@ end subroutine
 ! CHECK:  %[[VAL_1:.*]] = arith.constant 10 : index
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 20 : index
 ! CHECK:  %[[VAL_3:.*]] = fir.shape %[[VAL_1]], %[[VAL_2]] : (index, index) -> !fir.shape<2>
-! CHECK:  %[[VAL_4:.*]] = hlfir.declare %[[VAL_0]](%[[VAL_3]]) {uniq_name = "_QFarray_numericEx"} : (!fir.ref<!fir.array<10x20xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<10x20xi32>>, !fir.ref<!fir.array<10x20xi32>>)
+! CHECK:  %[[VAL_4:.*]] = hlfir.declare %[[VAL_0]](%[[VAL_3]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFarray_numericEx"} : (!fir.ref<!fir.array<10x20xi32>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<10x20xi32>>, !fir.ref<!fir.array<10x20xi32>>)
 
 
 subroutine array_numeric_lbounds(x)
@@ -47,7 +47,7 @@ end subroutine
 ! CHECK:  %[[VAL_3:.*]] = arith.constant -2 : index
 ! CHECK:  %[[VAL_4:.*]] = arith.constant 23 : index
 ! CHECK:  %[[VAL_5:.*]] = fir.shape_shift %[[VAL_1]], %[[VAL_2]], %[[VAL_3]], %[[VAL_4]] : (index, index, index, index) -> !fir.shapeshift<2>
-! CHECK:  %[[VAL_6:.*]] = hlfir.declare %[[VAL_0]](%[[VAL_5]]) {uniq_name = "_QFarray_numeric_lboundsEx"} : (!fir.ref<!fir.array<12x23xi32>>, !fir.shapeshift<2>) -> (!fir.box<!fir.array<12x23xi32>>, !fir.ref<!fir.array<12x23xi32>>)
+! CHECK:  %[[VAL_6:.*]] = hlfir.declare %[[VAL_0]](%[[VAL_5]]) dummy_scope %{{[0-9]+}}  {uniq_name = "_QFarray_numeric_lboundsEx"} : (!fir.ref<!fir.array<12x23xi32>>, !fir.shapeshift<2>, !fir.dscope) -> (!fir.box<!fir.array<12x23xi32>>, !fir.ref<!fir.array<12x23xi32>>)
 
 subroutine array_character(c)
   character(*) :: c(50)
@@ -58,14 +58,14 @@ end subroutine
 ! CHECK:  %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<50x!fir.char<1,?>>>
 ! CHECK:  %[[VAL_3:.*]] = arith.constant 50 : index
 ! CHECK:  %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_5:.*]] = hlfir.declare %[[VAL_2]](%[[VAL_4]]) typeparams %[[VAL_1]]#1 {uniq_name = "_QFarray_characterEc"} : (!fir.ref<!fir.array<50x!fir.char<1,?>>>, !fir.shape<1>, index) -> (!fir.box<!fir.array<50x!fir.char<1,?>>>, !fir.ref<!fir.array<50x!fir.char<1,?>>>)
+! CHECK:  %[[VAL_5:.*]] = hlfir.declare %[[VAL_2]](%[[VAL_4]]) typeparams %[[VAL_1]]#1 dummy_scope %{{[0-9]+}}  {uniq_name = "_QFarray_characterEc"} : (!fir.ref<!fir.array<50x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<50x!fir.char<1,?>>>, !fir.ref<!fir.array<50x!fir.char<1,?>>>)
 
 subroutine scalar_numeric_attributes(x)
   integer, optional, target, intent(in) :: x
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_numeric_attributes(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<i32>
-! CHECK:  %[[VAL_1:.*]] = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_in, optional, target>, uniq_name = "_QFscalar_numeric_attributesEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_1:.*]] = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in, optional, target>, uniq_name = "_QFscalar_numeric_attributesEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 
 subroutine scalar_numeric_attributes_2(x)
   real(16), value :: x(100)
@@ -74,21 +74,21 @@ end subroutine
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.array<100xf128>>
 ! CHECK:  %[[VAL_1:.*]] = arith.constant 100 : index
 ! CHECK:  %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_3:.*]] = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFscalar_numeric_attributes_2Ex"} : (!fir.ref<!fir.array<100xf128>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf128>>, !fir.ref<!fir.array<100xf128>>)
+! CHECK:  %[[VAL_3:.*]] = hlfir.declare %[[VAL_0]](%[[VAL_2]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QFscalar_numeric_attributes_2Ex"} : (!fir.ref<!fir.array<100xf128>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xf128>>, !fir.ref<!fir.array<100xf128>>)
 
 subroutine scalar_numeric_attributes_3(x)
   real, intent(in) :: x
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_numeric_attributes_3(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<f32>
-! CHECK:  %[[VAL_1:.*]] = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFscalar_numeric_attributes_3Ex"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_1:.*]] = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFscalar_numeric_attributes_3Ex"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 
 subroutine scalar_numeric_attributes_4(x)
   logical(8), intent(out) :: x
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_numeric_attributes_4(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.logical<8>>
-! CHECK:  %[[VAL_1:.*]] = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "_QFscalar_numeric_attributes_4Ex"} : (!fir.ref<!fir.logical<8>>) -> (!fir.ref<!fir.logical<8>>, !fir.ref<!fir.logical<8>>)
+! CHECK:  %[[VAL_1:.*]] = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "_QFscalar_numeric_attributes_4Ex"} : (!fir.ref<!fir.logical<8>>, !fir.dscope) -> (!fir.ref<!fir.logical<8>>, !fir.ref<!fir.logical<8>>)
 
 subroutine scalar_numeric_parameter()
   integer, parameter :: p = 42
diff --git a/flang/test/Lower/HLFIR/cray-pointers.f90 b/flang/test/Lower/HLFIR/cray-pointers.f90
index d969aa5d747a..ae903c8b44be 100644
--- a/flang/test/Lower/HLFIR/cray-pointers.f90
+++ b/flang/test/Lower/HLFIR/cray-pointers.f90
@@ -62,8 +62,8 @@ end subroutine test3
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i64> {fir.bindc_name = "cp"},
 ! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,11>>>>
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest3En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest3Ecp"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest3En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest3Ecp"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 11 : index
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 11 : index
 ! CHECK:           %[[VAL_24:.*]] = fir.shape_shift %{{.*}}, %{{.*}} : (index, index) -> !fir.shapeshift<1>
@@ -88,7 +88,7 @@ end subroutine test4
 ! CHECK-LABEL:   func.func @_QPtest4(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.char<1,?>>>
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest4En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest4En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i64 {bindc_name = "cp", uniq_name = "_QFtest4Ecp"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFtest4Ecp"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i32>
@@ -153,7 +153,7 @@ end subroutine test6
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xf32>>>
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest6En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest6En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_4:.*]] = fir.alloca i64 {bindc_name = "cp", uniq_name = "_QFtest6Ecp"}
 ! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFtest6Ecp"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
@@ -379,7 +379,7 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_craypointer_capture(
 ! CHECK-SAME:                                           %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.char<1,?>>>
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_craypointer_captureEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_craypointer_captureEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i64 {bindc_name = "cray_pointer", uniq_name = "_QFtest_craypointer_captureEcray_pointer"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFtest_craypointer_captureEcray_pointer"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/HLFIR/custom-intrinsic.f90 b/flang/test/Lower/HLFIR/custom-intrinsic.f90
index cf91ea332cdc..f4af94cfee2f 100644
--- a/flang/test/Lower/HLFIR/custom-intrinsic.f90
+++ b/flang/test/Lower/HLFIR/custom-intrinsic.f90
@@ -7,8 +7,9 @@ end function
 ! CHECK-LABEL: func.func @_QPmax_simple(
 ! CHECK-SAME:      %[[A_ARG:.*]]: !fir.ref<i32> {fir.bindc_name = "a"}
 ! CHECK-SAME:      %[[B_ARG:.*]]: !fir.ref<i32> {fir.bindc_name = "b"}
-! CHECK-NEXT:    %[[A_DECL:.*]]:2 = hlfir.declare %[[A_ARG]] {uniq_name = "_QFmax_simpleEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK-NEXT:    %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ARG]] {uniq_name = "_QFmax_simpleEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK-NEXT:    %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK-NEXT:    %[[A_DECL:.*]]:2 = hlfir.declare %[[A_ARG]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFmax_simpleEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK-NEXT:    %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ARG]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFmax_simpleEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK-NEXT:    %[[RES_ALLOC:.*]] = fir.alloca i32 {bindc_name = "max_simple", uniq_name = "_QFmax_simpleEmax_simple"}
 ! CHECK-NEXT:    %[[RES_DECL:.*]]:2 = hlfir.declare %[[RES_ALLOC]] {uniq_name = "_QFmax_simpleEmax_simple"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK-NEXT:    %[[A_LD:.*]] = fir.load %[[A_DECL]]#0 : !fir.ref<i32>
@@ -29,9 +30,9 @@ end function
 ! CHECK-SAME:                                              %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"},
 ! CHECK-SAME:                                              %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "b"},
 ! CHECK-SAME:                                              %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "c", fir.optional}) -> i32 {
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmax_dynamic_optional_scalarEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmax_dynamic_optional_scalarEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmax_dynamic_optional_scalarEc"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmax_dynamic_optional_scalarEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmax_dynamic_optional_scalarEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %{{[0-9]+}}  {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmax_dynamic_optional_scalarEc"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "max_dynamic_optional_scalar", uniq_name = "_QFmax_dynamic_optional_scalarEmax_dynamic_optional_scalar"}
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmax_dynamic_optional_scalarEmax_dynamic_optional_scalar"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
@@ -62,10 +63,10 @@ end function
 ! CHECK-SAME:                                               %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "b"},
 ! CHECK-SAME:                                               %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "c", fir.optional},
 ! CHECK-SAME:                                               %[[VAL_3:.*]]: !fir.ref<i32> {fir.bindc_name = "d", fir.optional}) -> i32 {
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmax_dynamic_optional_scalar2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmax_dynamic_optional_scalar2Eb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmax_dynamic_optional_scalar2Ec"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_3]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmax_dynamic_optional_scalar2Ed"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmax_dynamic_optional_scalar2Ea"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmax_dynamic_optional_scalar2Eb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmax_dynamic_optional_scalar2Ec"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_3]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmax_dynamic_optional_scalar2Ed"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_8:.*]] = fir.alloca i32 {bindc_name = "max_dynamic_optional_scalar2", uniq_name = "_QFmax_dynamic_optional_scalar2Emax_dynamic_optional_scalar2"}
 ! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFmax_dynamic_optional_scalar2Emax_dynamic_optional_scalar2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_10:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<i32>
@@ -104,10 +105,10 @@ end function
 ! CHECK-SAME:                            %[[VAL_1:.*]]: !fir.ref<!fir.array<42xi32>> {fir.bindc_name = "b"}) -> !fir.array<42xi32> {
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) {uniq_name = "_QFmax_arrayEa"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmax_arrayEa"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_6]]) {uniq_name = "_QFmax_arrayEb"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmax_arrayEb"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<42xi32> {bindc_name = "max_array", uniq_name = "_QFmax_arrayEmax_array"}
 ! CHECK:           %[[VAL_10:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
@@ -137,13 +138,13 @@ end function
 ! CHECK-SAME:                                             %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"},
 ! CHECK-SAME:                                             %[[VAL_1:.*]]: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "b"},
 ! CHECK-SAME:                                             %[[VAL_2:.*]]: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "c", fir.optional}) -> !fir.array<10xi32> {
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmax_dynamic_optional_arrayEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmax_dynamic_optional_arrayEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = "_QFmax_dynamic_optional_arrayEb"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmax_dynamic_optional_arrayEb"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK:           %[[VAL_7:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]](%[[VAL_8]]) {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmax_dynamic_optional_arrayEc"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]](%[[VAL_8]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmax_dynamic_optional_arrayEc"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.array<10xi32> {bindc_name = "max_dynamic_optional_array", uniq_name = "_QFmax_dynamic_optional_arrayEmax_dynamic_optional_array"}
 ! CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_10]] : (index) -> !fir.shape<1>
@@ -180,8 +181,8 @@ end function
 ! CHECK-LABEL:   func.func @_QPmin_simple(
 ! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"},
 ! CHECK-SAME:                             %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "b"}) -> i32 {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmin_simpleEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmin_simpleEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmin_simpleEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmin_simpleEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "min_simple", uniq_name = "_QFmin_simpleEmin_simple"}
 ! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFmin_simpleEmin_simple"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i32>
@@ -202,9 +203,9 @@ end function
 ! CHECK-SAME:                                              %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"},
 ! CHECK-SAME:                                              %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "b"},
 ! CHECK-SAME:                                              %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "c", fir.optional}) -> i32 {
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmin_dynamic_optional_scalarEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmin_dynamic_optional_scalarEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmin_dynamic_optional_scalarEc"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmin_dynamic_optional_scalarEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmin_dynamic_optional_scalarEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %{{[0-9]+}}  {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmin_dynamic_optional_scalarEc"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "min_dynamic_optional_scalar", uniq_name = "_QFmin_dynamic_optional_scalarEmin_dynamic_optional_scalar"}
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmin_dynamic_optional_scalarEmin_dynamic_optional_scalar"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
@@ -235,10 +236,10 @@ end function
 ! CHECK-SAME:                                               %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "b"},
 ! CHECK-SAME:                                               %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "c", fir.optional},
 ! CHECK-SAME:                                               %[[VAL_3:.*]]: !fir.ref<i32> {fir.bindc_name = "d", fir.optional}) -> i32 {
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmin_dynamic_optional_scalar2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmin_dynamic_optional_scalar2Eb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmin_dynamic_optional_scalar2Ec"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_3]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmin_dynamic_optional_scalar2Ed"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmin_dynamic_optional_scalar2Ea"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmin_dynamic_optional_scalar2Eb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmin_dynamic_optional_scalar2Ec"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_3]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmin_dynamic_optional_scalar2Ed"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_8:.*]] = fir.alloca i32 {bindc_name = "min_dynamic_optional_scalar2", uniq_name = "_QFmin_dynamic_optional_scalar2Emin_dynamic_optional_scalar2"}
 ! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFmin_dynamic_optional_scalar2Emin_dynamic_optional_scalar2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_10:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<i32>
@@ -277,10 +278,10 @@ end function
 ! CHECK-SAME:                            %[[VAL_1:.*]]: !fir.ref<!fir.array<42xi32>> {fir.bindc_name = "b"}) -> !fir.array<42xi32> {
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) {uniq_name = "_QFmin_arrayEa"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmin_arrayEa"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_6]]) {uniq_name = "_QFmin_arrayEb"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmin_arrayEb"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<42xi32> {bindc_name = "min_array", uniq_name = "_QFmin_arrayEmin_array"}
 ! CHECK:           %[[VAL_10:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
@@ -310,13 +311,13 @@ end function
 ! CHECK-SAME:                                             %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"},
 ! CHECK-SAME:                                             %[[VAL_1:.*]]: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "b"},
 ! CHECK-SAME:                                             %[[VAL_2:.*]]: !fir.ref<!fir.array<10xi32>> {fir.bindc_name = "c", fir.optional}) -> !fir.array<10xi32> {
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmin_dynamic_optional_arrayEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmin_dynamic_optional_arrayEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) {uniq_name = "_QFmin_dynamic_optional_arrayEb"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_5]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmin_dynamic_optional_arrayEb"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK:           %[[VAL_7:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]](%[[VAL_8]]) {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmin_dynamic_optional_arrayEc"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_2]](%[[VAL_8]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFmin_dynamic_optional_arrayEc"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.array<10xi32> {bindc_name = "min_dynamic_optional_array", uniq_name = "_QFmin_dynamic_optional_arrayEmin_dynamic_optional_array"}
 ! CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_10]] : (index) -> !fir.shape<1>
@@ -355,7 +356,7 @@ end function
 ! CHECK-SAME:                                    %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "pointer"}) -> !fir.logical<4> {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.logical<4> {bindc_name = "associated_simple", uniq_name = "_QFassociated_simpleEassociated_simple"}
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFassociated_simpleEassociated_simple"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_simpleEpointer"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_simpleEpointer"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>
 ! CHECK:           %[[VAL_5:.*]] = fir.box_addr %[[VAL_4]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
 ! CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.ptr<i32>) -> i64
@@ -378,8 +379,8 @@ end function
 ! CHECK-SAME:                                    %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "target", fir.target}) -> !fir.logical<4> {
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.logical<4> {bindc_name = "associated_target", uniq_name = "_QFassociated_targetEassociated_target"}
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFassociated_targetEassociated_target"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_targetEpointer"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFassociated_targetEtarget"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_targetEpointer"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFassociated_targetEtarget"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_6:.*]] = fir.embox %[[VAL_5]]#1 : (!fir.ref<i32>) -> !fir.box<i32>
 ! CHECK:           %[[VAL_7:.*]] = fir.load %[[VAL_4]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>
 ! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.ptr<i32>>) -> !fir.box<none>
@@ -402,8 +403,8 @@ end function
 ! CHECK-SAME:                                     %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "target"}) -> !fir.logical<4> {
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.logical<4> {bindc_name = "associated_pointer", uniq_name = "_QFassociated_pointerEassociated_pointer"}
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFassociated_pointerEassociated_pointer"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_pointerEpointer"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_pointerEtarget"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_pointerEpointer"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_pointerEtarget"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>
 ! CHECK:           %[[VAL_7:.*]] = fir.load %[[VAL_4]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>
 ! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.ptr<i32>>) -> !fir.box<none>
@@ -426,8 +427,8 @@ end function
 ! CHECK-SAME:                                   %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "target"}) -> !fir.logical<4> {
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.logical<4> {bindc_name = "associated_array", uniq_name = "_QFassociated_arrayEassociated_array"}
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFassociated_arrayEassociated_array"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_arrayEpointer"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_arrayEtarget"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_arrayEpointer"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFassociated_arrayEtarget"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
 ! CHECK:           %[[VAL_7:.*]] = fir.load %[[VAL_4]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
 ! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !fir.box<none>
@@ -447,11 +448,11 @@ end function
 ! CHECK-SAME:                                %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "i"},
 ! CHECK-SAME:                                %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "shift"},
 ! CHECK-SAME:                                %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "size"}) -> i32 {
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFishftc_simpleEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFishftc_simpleEi"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "ishftc_simple", uniq_name = "_QFishftc_simpleEishftc_simple"}
 ! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFishftc_simpleEishftc_simple"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFishftc_simpleEshift"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFishftc_simpleEsize"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFishftc_simpleEshift"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFishftc_simpleEsize"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
 ! CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
 ! CHECK:           %[[VAL_10:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<i32>
@@ -498,11 +499,11 @@ end function
 ! CHECK-SAME:                                                     %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "i"},
 ! CHECK-SAME:                                                     %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "shift"},
 ! CHECK-SAME:                                                     %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "size", fir.optional}) -> i32 {
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFishftc_dynamically_optional_scalarEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFishftc_dynamically_optional_scalarEi"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "ishftc_dynamically_optional_scalar", uniq_name = "_QFishftc_dynamically_optional_scalarEishftc_dynamically_optional_scalar"}
 ! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFishftc_dynamically_optional_scalarEishftc_dynamically_optional_scalar"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFishftc_dynamically_optional_scalarEshift"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_2]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFishftc_dynamically_optional_scalarEsize"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFishftc_dynamically_optional_scalarEshift"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFishftc_dynamically_optional_scalarEsize"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
 ! CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
 ! CHECK:           %[[VAL_10:.*]] = fir.is_present %[[VAL_7]]#0 : (!fir.ref<i32>) -> i1
@@ -557,17 +558,17 @@ end function
 ! CHECK-SAME:                               %[[VAL_2:.*]]: !fir.ref<!fir.array<42xi32>> {fir.bindc_name = "size"}) -> !fir.array<42xi32> {
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFishftc_arrayEi"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFishftc_arrayEi"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<42xi32> {bindc_name = "ishftc_array", uniq_name = "_QFishftc_arrayEishftc_array"}
 ! CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
 ! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_8]]) {uniq_name = "_QFishftc_arrayEishftc_array"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_10]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_11]]) {uniq_name = "_QFishftc_arrayEshift"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
+! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_11]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFishftc_arrayEshift"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_14:.*]] = fir.shape %[[VAL_13]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_2]](%[[VAL_14]]) {uniq_name = "_QFishftc_arrayEsize"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
+! CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_2]](%[[VAL_14]]) dummy_scope %{{[0-9]+}}  {uniq_name = "_QFishftc_arrayEsize"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
 ! CHECK:           %[[VAL_16:.*]] = hlfir.elemental %[[VAL_4]] unordered : (!fir.shape<1>) -> !hlfir.expr<42xi32> {
 ! CHECK:           ^bb0(%[[VAL_17:.*]]: index):
 ! CHECK:             %[[VAL_18:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_17]])  : (!fir.ref<!fir.array<42xi32>>, index) -> !fir.ref<i32>
@@ -624,13 +625,13 @@ end function
 ! CHECK-SAME:                                                    %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "size", fir.optional}) -> !fir.array<42xi32> {
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFishftc_dynamically_optional_arrayEi"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFishftc_dynamically_optional_arrayEi"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 42 : index
 ! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.array<42xi32> {bindc_name = "ishftc_dynamically_optional_array", uniq_name = "_QFishftc_dynamically_optional_arrayEishftc_dynamically_optional_array"}
 ! CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
 ! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_8]]) {uniq_name = "_QFishftc_dynamically_optional_arrayEishftc_dynamically_optional_array"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
-! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFishftc_dynamically_optional_arrayEshift"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_2]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFishftc_dynamically_optional_arrayEsize"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFishftc_dynamically_optional_arrayEshift"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFishftc_dynamically_optional_arrayEsize"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_12:.*]] = fir.is_present %[[VAL_11]]#0 : (!fir.ref<i32>) -> i1
 ! CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_10]]#0 : !fir.ref<i32>
 ! CHECK:           %[[VAL_14:.*]] = hlfir.elemental %[[VAL_4]] unordered : (!fir.shape<1>) -> !hlfir.expr<42xi32> {
@@ -698,9 +699,9 @@ end subroutine
 ! CHECK-SAME:                                    %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>> {fir.bindc_name = "a"},
 ! CHECK-SAME:                                    %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>> {fir.bindc_name = "b"},
 ! CHECK-SAME:                                    %[[VAL_2:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>> {fir.bindc_name = "c"}) {
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFallocatables_testEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>)
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFallocatables_testEb"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFallocatables_testEc"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFallocatables_testEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFallocatables_testEb"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFallocatables_testEc"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.address_of(@_QFallocatables_testECnx) : !fir.ref<i32>
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QFallocatables_testECnx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_8:.*]] = fir.address_of(@_QFallocatables_testECny) : !fir.ref<i32>
@@ -840,4 +841,4 @@ end subroutine
 ! CHECK:           hlfir.assign %[[VAL_136:.*]] to %[[VAL_5]]#0 realloc : !hlfir.expr<?x?x?xi32>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xi32>>>>
 ! CHECK:           hlfir.destroy %[[VAL_136]] : !hlfir.expr<?x?x?xi32>
 ! CHECK:           return
-! CHECK:         }
\ No newline at end of file
+! CHECK:         }
diff --git a/flang/test/Lower/HLFIR/designators-component-ref.f90 b/flang/test/Lower/HLFIR/designators-component-ref.f90
index 392eda66fd03..69cc7d2e5aa6 100644
--- a/flang/test/Lower/HLFIR/designators-component-ref.f90
+++ b/flang/test/Lower/HLFIR/designators-component-ref.f90
@@ -340,7 +340,7 @@ subroutine test_scalar_array_complex_chain(a)
   type(t_complex) :: a
   print *, a%array_comp%im
 ! CHECK-LABEL:   func.func @_QPtest_scalar_array_complex_chain(
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_scalar_array_complex_chainEa"} : (!fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20x!fir.complex<4>>}>>) -> (!fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20x!fir.complex<4>>}>>, !fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20x!fir.complex<4>>}>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_scalar_array_complex_chainEa"} : (!fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20x!fir.complex<4>>}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20x!fir.complex<4>>}>>, !fir.ref<!fir.type<_QMcomp_refTt_complex{array_comp:!fir.array<10x20x!fir.complex<4>>}>>)
 ! CHECK:           %[[VAL_7:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 20 : index
 ! CHECK:           %[[VAL_9:.*]] = arith.constant 2 : index
@@ -379,13 +379,13 @@ end subroutine test_poly_array_vector_subscript
 ! CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcomp_refTt1{scalar_i:i32,scalar_x:f32}>>>>> {fir.bindc_name = "p"},
 ! CHECK-SAME:      %[[VAL_1:.*]]: !fir.ref<!fir.array<3xi32>> {fir.bindc_name = "v"},
 ! CHECK-SAME:      %[[VAL_2:.*]]: !fir.ref<!fir.array<3xi32>> {fir.bindc_name = "r"}) {
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_poly_array_vector_subscriptEp"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcomp_refTt1{scalar_i:i32,scalar_x:f32}>>>>>) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcomp_refTt1{scalar_i:i32,scalar_x:f32}>>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcomp_refTt1{scalar_i:i32,scalar_x:f32}>>>>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_poly_array_vector_subscriptEp"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcomp_refTt1{scalar_i:i32,scalar_x:f32}>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcomp_refTt1{scalar_i:i32,scalar_x:f32}>>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcomp_refTt1{scalar_i:i32,scalar_x:f32}>>>>>)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2]](%[[VAL_5]]) {uniq_name = "_QFtest_poly_array_vector_subscriptEr"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_2]](%[[VAL_5]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_poly_array_vector_subscriptEr"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
 ! CHECK:           %[[VAL_7:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_8]]) {uniq_name = "_QFtest_poly_array_vector_subscriptEv"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_8]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_poly_array_vector_subscriptEv"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
 ! CHECK:           %[[VAL_10:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcomp_refTt1{scalar_i:i32,scalar_x:f32}>>>>>
 ! CHECK:           %[[VAL_11:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<3xi64> {
 ! CHECK:           ^bb0(%[[VAL_12:.*]]: index):
diff --git a/flang/test/Lower/HLFIR/designators.f90 b/flang/test/Lower/HLFIR/designators.f90
index de1ec6e5b3cf..09753d06cc27 100644
--- a/flang/test/Lower/HLFIR/designators.f90
+++ b/flang/test/Lower/HLFIR/designators.f90
@@ -7,8 +7,8 @@ subroutine array_ref(x, n)
   print *, x(n)
 end subroutine
 ! CHECK-LABEL: func.func @_QParray_ref(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFarray_refEn"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFarray_refEx"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFarray_refEn"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFarray_refEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:  %[[VAL_9:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i64>
 ! CHECK:  %[[VAL_10:.*]] = hlfir.designate %[[VAL_3]]#0 (%[[VAL_9]])  : (!fir.box<!fir.array<?xf32>>, i64) -> !fir.ref<f32>
 
@@ -17,8 +17,8 @@ subroutine char_array_ref(x, n)
   print *, x(10)
 end subroutine
 ! CHECK-LABEL: func.func @_QPchar_array_ref(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFchar_array_refEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFchar_array_refEx"} : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFchar_array_refEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFchar_array_refEx"} : (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>)
 ! CHECK:  %[[VAL_9:.*]] = fir.box_elesize %[[VAL_3]]#1 : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> index
 ! CHECK:  %[[VAL_10:.*]] = arith.constant 10 : index
 ! CHECK:  %[[VAL_11:.*]] = hlfir.designate %[[VAL_3]]#0 (%[[VAL_10]])  typeparams %[[VAL_9]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index, index) -> !fir.boxchar<1>
@@ -28,9 +28,9 @@ subroutine char_array_ref_cst_len(x, n)
   print *, x(10)
 end subroutine
 ! CHECK-LABEL: func.func @_QPchar_array_ref_cst_len(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFchar_array_ref_cst_lenEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFchar_array_ref_cst_lenEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_3:.*]] = arith.constant 5 : index
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}} typeparams %[[VAL_3]] {uniq_name = "_QFchar_array_ref_cst_lenEx"} : (!fir.box<!fir.array<?x!fir.char<1,5>>>, index) -> (!fir.box<!fir.array<?x!fir.char<1,5>>>, !fir.box<!fir.array<?x!fir.char<1,5>>>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}} typeparams %[[VAL_3]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFchar_array_ref_cst_lenEx"} : (!fir.box<!fir.array<?x!fir.char<1,5>>>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,5>>>, !fir.box<!fir.array<?x!fir.char<1,5>>>)
 ! CHECK:  %[[VAL_10:.*]] = arith.constant 10 : index
 ! CHECK:  %[[VAL_11:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_10]])  typeparams %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.char<1,5>>>, index, index) -> !fir.ref<!fir.char<1,5>>
 
@@ -41,7 +41,7 @@ end subroutine
 ! CHECK-LABEL: func.func @_QParray_section(
 ! CHECK:  %[[VAL_1:.*]] = arith.constant 10 : index
 ! CHECK:  %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}(%[[VAL_2]]) {uniq_name = "_QFarray_sectionEx"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}(%[[VAL_2]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFarray_sectionEx"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 ! CHECK:  %[[VAL_9:.*]] = arith.constant 2 : index
 ! CHECK:  %[[VAL_10:.*]] = arith.constant 8 : index
 ! CHECK:  %[[VAL_11:.*]] = arith.constant 3 : index
@@ -55,8 +55,8 @@ subroutine array_section_2(x, n)
   print *, x(n::3)
 end subroutine
 ! CHECK-LABEL: func.func @_QParray_section_2(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFarray_section_2En"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFarray_section_2Ex"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFarray_section_2En"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFarray_section_2Ex"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:  %[[VAL_9:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i64>
 ! CHECK:  %[[VAL_10:.*]] = arith.constant 0 : index
 ! CHECK:  %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_3]]#1, %[[VAL_10]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
@@ -76,8 +76,8 @@ subroutine char_array_section(x, n)
   print *, x(::3)
 end subroutine
 ! CHECK-LABEL: func.func @_QPchar_array_section(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFchar_array_sectionEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFchar_array_sectionEx"} : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFchar_array_sectionEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFchar_array_sectionEx"} : (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>)
 ! CHECK:  %[[VAL_9:.*]] = fir.box_elesize %[[VAL_3]]#1 : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> index
 ! CHECK:  %[[VAL_10:.*]] = arith.constant 1 : index
 ! CHECK:  %[[VAL_11:.*]] = arith.constant 0 : index
@@ -97,9 +97,9 @@ subroutine char_array_section_cst_len(x, n)
   print *, x(::3)
 end subroutine
 ! CHECK-LABEL: func.func @_QPchar_array_section_cst_len(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFchar_array_section_cst_lenEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFchar_array_section_cst_lenEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_3:.*]] = arith.constant 5 : index
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}} typeparams %[[VAL_3]] {uniq_name = "_QFchar_array_section_cst_lenEx"} : (!fir.box<!fir.array<?x!fir.char<1,5>>>, index) -> (!fir.box<!fir.array<?x!fir.char<1,5>>>, !fir.box<!fir.array<?x!fir.char<1,5>>>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}} typeparams %[[VAL_3]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFchar_array_section_cst_lenEx"} : (!fir.box<!fir.array<?x!fir.char<1,5>>>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,5>>>, !fir.box<!fir.array<?x!fir.char<1,5>>>)
 ! CHECK:  %[[VAL_10:.*]] = arith.constant 1 : index
 ! CHECK:  %[[VAL_11:.*]] = arith.constant 0 : index
 ! CHECK:  %[[VAL_12:.*]]:3 = fir.box_dims %[[VAL_4]]#1, %[[VAL_11]] : (!fir.box<!fir.array<?x!fir.char<1,5>>>, index) -> (index, index, index)
@@ -120,7 +120,7 @@ subroutine complex_imag_ref(x)
   print *, x%im
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_imag_ref(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFcomplex_imag_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_imag_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
 ! CHECK:  %[[VAL_3:.*]] = fir.shape %[[VAL_4:.*]]#1 : (index) -> !fir.shape<1>
 ! CHECK:  %[[VAL_5:.*]] = hlfir.designate %[[VAL_2]]#0  imag shape %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 
@@ -129,7 +129,7 @@ subroutine complex_real_ref(x)
   print *, x%re
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_real_ref(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFcomplex_real_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_real_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
 ! CHECK:  %[[VAL_3:.*]] = fir.shape %[[VAL_4:.*]]#1 : (index) -> !fir.shape<1>
 ! CHECK:  %[[VAL_5:.*]] = hlfir.designate %[[VAL_2]]#0  real shape %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
 
@@ -139,11 +139,11 @@ subroutine complex_individual_ref(x, n)
   print *, x(n)%im
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_individual_ref(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFcomplex_individual_refEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFcomplex_individual_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_individual_refEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_individual_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i32) -> i64
-! CHECK:  %[[VAL_6:.*]] = hlfir.designate %1#0 (%[[VAL_5]]) imag : (!fir.box<!fir.array<?x!fir.complex<4>>>, i64) -> !fir.ref<f32>
+! CHECK:  %[[VAL_6:.*]] = hlfir.designate %{{[0-9]+}}#0 (%[[VAL_5]]) imag : (!fir.box<!fir.array<?x!fir.complex<4>>>, i64) -> !fir.ref<f32>
 
 subroutine complex_slice_ref(x, start, end)
   complex :: x(:)
@@ -151,9 +151,9 @@ subroutine complex_slice_ref(x, start, end)
   print *, x(start:end)%re
 end subroutine
 ! CHECK-LABEL: func.func @_QPcomplex_slice_ref(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFcomplex_slice_refEend"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFcomplex_slice_refEstart"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFcomplex_slice_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_slice_refEend"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_slice_refEstart"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %arg0 dummy_scope %{{[0-9]+}} {uniq_name = "_QFcomplex_slice_refEx"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
 ! CHECK:  %[[VAL_5:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i32) -> i64
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/HLFIR/dot_product.f90 b/flang/test/Lower/HLFIR/dot_product.f90
index 890dc4abca49..2d3ee97b7e40 100644
--- a/flang/test/Lower/HLFIR/dot_product.f90
+++ b/flang/test/Lower/HLFIR/dot_product.f90
@@ -72,10 +72,10 @@ endsubroutine
 ! CHECK-NEXT:   }
 
 ! CHECK-LABEL: func.func @_QPdot_product5
-! CHECK:    %[[LHS:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFdot_product5Elhs"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:    %[[LHS:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFdot_product5Elhs"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:    %[[C3:.*]] = arith.constant 3 : index
 ! CHECK:    %[[RHS_SHAPE:.*]] = fir.shape %[[C3]] : (index) -> !fir.shape<1>
-! CHECK:    %[[RHS:.*]]:2 = hlfir.declare %{{.*}}(%[[RHS_SHAPE]]) {uniq_name = "_QFdot_product5Erhs"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
+! CHECK:    %[[RHS:.*]]:2 = hlfir.declare %{{.*}}(%[[RHS_SHAPE]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFdot_product5Erhs"} : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<3xi32>>, !fir.ref<!fir.array<3xi32>>)
 ! CHECK:    {{.*}} = hlfir.dot_product %[[LHS]]#0 %[[RHS]]#0 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<3xi32>>) -> i32
 subroutine dot_product5(lhs, rhs, res)
   integer :: lhs(:), rhs(3)
diff --git a/flang/test/Lower/HLFIR/elemental-array-ops.f90 b/flang/test/Lower/HLFIR/elemental-array-ops.f90
index 9778adeb6179..80801fdde0d7 100644
--- a/flang/test/Lower/HLFIR/elemental-array-ops.f90
+++ b/flang/test/Lower/HLFIR/elemental-array-ops.f90
@@ -166,9 +166,9 @@ end subroutine char_return
 ! CHECK:           fir.store %[[VAL_7]] to %[[VAL_3]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.logical<4>>>>>
 ! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_3]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFchar_returnEl"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.logical<4>>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.logical<4>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.logical<4>>>>>)
 ! CHECK:           %[[VAL_9:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_9]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFchar_returnEx"} : (!fir.box<!fir.array<?x!fir.char<1,3>>>, index) -> (!fir.box<!fir.array<?x!fir.char<1,3>>>, !fir.box<!fir.array<?x!fir.char<1,3>>>)
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_9]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFchar_returnEx"} : (!fir.box<!fir.array<?x!fir.char<1,3>>>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,3>>>, !fir.box<!fir.array<?x!fir.char<1,3>>>)
 ! CHECK:           %[[VAL_11:.*]] = arith.constant 3 : index
-! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_1]] typeparams %[[VAL_11]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFchar_returnEy"} : (!fir.box<!fir.array<?x!fir.char<1,3>>>, index) -> (!fir.box<!fir.array<?x!fir.char<1,3>>>, !fir.box<!fir.array<?x!fir.char<1,3>>>)
+! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_1]] typeparams %[[VAL_11]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFchar_returnEy"} : (!fir.box<!fir.array<?x!fir.char<1,3>>>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,3>>>, !fir.box<!fir.array<?x!fir.char<1,3>>>)
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_12]]#0, %[[VAL_13]] : (!fir.box<!fir.array<?x!fir.char<1,3>>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_15:.*]] = fir.shape %[[VAL_14]]#1 : (index) -> !fir.shape<1>
@@ -225,8 +225,8 @@ end subroutine polymorphic_parenthesis
 ! CHECK-LABEL:   func.func @_QPpolymorphic_parenthesis(
 ! CHECK-SAME:        %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>>> {fir.bindc_name = "x"},
 ! CHECK-SAME:        %[[VAL_1:.*]]: !fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>> {fir.bindc_name = "y"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFpolymorphic_parenthesisEx"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFpolymorphic_parenthesisEy"} : (!fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>) -> (!fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>, !fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFpolymorphic_parenthesisEx"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFpolymorphic_parenthesisEy"} : (!fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>, !fir.dscope) -> (!fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>, !fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_3]]#0, %[[VAL_4]] : (!fir.class<!fir.array<?x!fir.type<_QFpolymorphic_parenthesisTt>>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]]#1 : (index) -> !fir.shape<1>
@@ -249,8 +249,8 @@ end subroutine unlimited_polymorphic_parenthesis
 ! CHECK-LABEL:   func.func @_QPunlimited_polymorphic_parenthesis(
 ! CHECK-SAME:        %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>> {fir.bindc_name = "x"},
 ! CHECK-SAME:        %[[VAL_1:.*]]: !fir.class<!fir.array<?xnone>> {fir.bindc_name = "y"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFunlimited_polymorphic_parenthesisEx"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFunlimited_polymorphic_parenthesisEy"} : (!fir.class<!fir.array<?xnone>>) -> (!fir.class<!fir.array<?xnone>>, !fir.class<!fir.array<?xnone>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFunlimited_polymorphic_parenthesisEx"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFunlimited_polymorphic_parenthesisEy"} : (!fir.class<!fir.array<?xnone>>, !fir.dscope) -> (!fir.class<!fir.array<?xnone>>, !fir.class<!fir.array<?xnone>>)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_3]]#0, %[[VAL_4]] : (!fir.class<!fir.array<?xnone>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]]#1 : (index) -> !fir.shape<1>
diff --git a/flang/test/Lower/HLFIR/elemental-polymorphic-merge.f90 b/flang/test/Lower/HLFIR/elemental-polymorphic-merge.f90
index eb93099b3890..36762d47100c 100644
--- a/flang/test/Lower/HLFIR/elemental-polymorphic-merge.f90
+++ b/flang/test/Lower/HLFIR/elemental-polymorphic-merge.f90
@@ -14,10 +14,10 @@ end subroutine test_polymorphic_merge
 ! CHECK-SAME:        %[[VAL_1:.*]]: !fir.class<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>> {fir.bindc_name = "y"},
 ! CHECK-SAME:        %[[VAL_2:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>>> {fir.bindc_name = "r"},
 ! CHECK-SAME:        %[[VAL_3:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "m"}) {
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFtest_polymorphic_mergeEm"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_polymorphic_mergeEr"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>>>)
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_polymorphic_mergeEx"} : (!fir.class<!fir.type<_QFtest_polymorphic_mergeTt>>) -> (!fir.class<!fir.type<_QFtest_polymorphic_mergeTt>>, !fir.class<!fir.type<_QFtest_polymorphic_mergeTt>>)
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_polymorphic_mergeEy"} : (!fir.class<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>) -> (!fir.class<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>, !fir.class<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_polymorphic_mergeEm"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_polymorphic_mergeEr"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>>>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_polymorphic_mergeEx"} : (!fir.class<!fir.type<_QFtest_polymorphic_mergeTt>>, !fir.dscope) -> (!fir.class<!fir.type<_QFtest_polymorphic_mergeTt>>, !fir.class<!fir.type<_QFtest_polymorphic_mergeTt>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_polymorphic_mergeEy"} : (!fir.class<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>, !fir.dscope) -> (!fir.class<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>, !fir.class<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_9:.*]]:3 = fir.box_dims %[[VAL_7]]#0, %[[VAL_8]] : (!fir.class<!fir.array<?x!fir.type<_QFtest_polymorphic_mergeTt>>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_10:.*]] = fir.shape %[[VAL_9]]#1 : (index) -> !fir.shape<1>
diff --git a/flang/test/Lower/HLFIR/elemental-user-procedure-ref.f90 b/flang/test/Lower/HLFIR/elemental-user-procedure-ref.f90
index d015ba3b0707..aea23d8d9467 100644
--- a/flang/test/Lower/HLFIR/elemental-user-procedure-ref.f90
+++ b/flang/test/Lower/HLFIR/elemental-user-procedure-ref.f90
@@ -111,7 +111,7 @@ end subroutine
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 20 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_1]], %[[VAL_2]] : (index, index) -> !fir.shape<2>
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) {uniq_name = "_QFimpure_elementalEx"} : (!fir.ref<!fir.array<10x20xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<10x20xf32>>, !fir.ref<!fir.array<10x20xf32>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFimpure_elementalEx"} : (!fir.ref<!fir.array<10x20xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<10x20xf32>>, !fir.ref<!fir.array<10x20xf32>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
 ! CHECK:           fir.do_loop %[[VAL_6:.*]] = %[[VAL_5]] to %[[VAL_2]] step %[[VAL_5]] {
 ! CHECK:             fir.do_loop %[[VAL_7:.*]] = %[[VAL_5]] to %[[VAL_1]] step %[[VAL_5]] {
@@ -136,7 +136,7 @@ end subroutine
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 20 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_1]], %[[VAL_2]] : (index, index) -> !fir.shape<2>
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) {uniq_name = "_QFordered_elementalEx"} : (!fir.ref<!fir.array<10x20xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<10x20xf32>>, !fir.ref<!fir.array<10x20xf32>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFordered_elementalEx"} : (!fir.ref<!fir.array<10x20xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<10x20xf32>>, !fir.ref<!fir.array<10x20xf32>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
 ! CHECK:           fir.do_loop %[[VAL_6:.*]] = %[[VAL_5]] to %[[VAL_2]] step %[[VAL_5]] {
 ! CHECK:             fir.do_loop %[[VAL_7:.*]] = %[[VAL_5]] to %[[VAL_1]] step %[[VAL_5]] {
@@ -161,7 +161,7 @@ end subroutine
 ! CHECK:           %[[VAL_1:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 20 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_1]], %[[VAL_2]] : (index, index) -> !fir.shape<2>
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) {uniq_name = "_QFimpure_elemental_arg_evalEx"} : (!fir.ref<!fir.array<10x20xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<10x20xf32>>, !fir.ref<!fir.array<10x20xf32>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFimpure_elemental_arg_evalEx"} : (!fir.ref<!fir.array<10x20xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<10x20xf32>>, !fir.ref<!fir.array<10x20xf32>>)
 ! CHECK:           %[[VAL_5:.*]] = hlfir.elemental %[[VAL_3]] unordered : (!fir.shape<2>) -> !hlfir.expr<10x20xf32> {
 ! CHECK:           ^bb0(%[[VAL_6:.*]]: index, %[[VAL_7:.*]]: index):
 ! CHECK:             %[[VAL_8:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_6]], %[[VAL_7]])  : (!fir.ref<!fir.array<10x20xf32>>, index, index) -> !fir.ref<f32>
diff --git a/flang/test/Lower/HLFIR/expr-addr.f90 b/flang/test/Lower/HLFIR/expr-addr.f90
index 876aad8925d7..917a68d59910 100644
--- a/flang/test/Lower/HLFIR/expr-addr.f90
+++ b/flang/test/Lower/HLFIR/expr-addr.f90
@@ -6,7 +6,7 @@
 subroutine foo(x)
   integer :: x
   read (*,*) x
-  ! CHECK: %[[x:.]]:2 = hlfir.declare %[[arg0]] {uniq_name = "_QFfooEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[x:.]]:2 = hlfir.declare %[[arg0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfooEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK: %[[x_cast:.*]] = fir.convert %[[x]]#1 : (!fir.ref<i32>) -> !fir.ref<i64>
   ! CHECK: fir.call @_FortranAioInputInteger(%{{.*}}, %[[x_cast]], %{{.*}}) {{.*}}: (!fir.ref<i8>, !fir.ref<i64>, i32) -> i1
 end subroutine
diff --git a/flang/test/Lower/HLFIR/expr-box.f90 b/flang/test/Lower/HLFIR/expr-box.f90
index e7ab006751a0..f0de381c7457 100644
--- a/flang/test/Lower/HLFIR/expr-box.f90
+++ b/flang/test/Lower/HLFIR/expr-box.f90
@@ -9,7 +9,7 @@ subroutine foo(x)
 ! CHECK-DAG:  %[[VAL_3:.*]] = arith.constant 21 : index
 ! CHECK-DAG:  %[[VAL_4:.*]] = arith.constant 10 : index
 ! CHECK:  %[[VAL_5:.*]] = fir.shape_shift %[[VAL_3]], %[[VAL_4]] : (index, index) -> !fir.shapeshift<1>
-! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) {uniq_name = "_QFfooEx"} : (!fir.ref<!fir.array<10xi32>>, !fir.shapeshift<1>) -> (!fir.box<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFfooEx"} : (!fir.ref<!fir.array<10xi32>>, !fir.shapeshift<1>, !fir.dscope) -> (!fir.box<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK:  fir.embox %[[VAL_6]]#1(%[[VAL_5]]) : (!fir.ref<!fir.array<10xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.array<10xi32>>
 end subroutine
 
diff --git a/flang/test/Lower/HLFIR/expr-value.f90 b/flang/test/Lower/HLFIR/expr-value.f90
index cd2f42533c27..c692ec72bf7e 100644
--- a/flang/test/Lower/HLFIR/expr-value.f90
+++ b/flang/test/Lower/HLFIR/expr-value.f90
@@ -11,7 +11,7 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPfoo_designator(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.ref<i32>
 subroutine foo_designator(n)
-  !CHECK:  %[[n:.*]]:2 = hlfir.declare %[[arg0]] {uniq_name = "_QFfoo_designatorEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  !CHECK:  %[[n:.*]]:2 = hlfir.declare %[[arg0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfoo_designatorEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   print *, n
   ! CHECK: %[[nval:.*]] = fir.load %[[n]]#0 : !fir.ref<i32>
   ! CHECK: fir.call @_FortranAioOutputInteger32(%{{.*}}, %[[nval]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
diff --git a/flang/test/Lower/HLFIR/ignore-rank-unlimited-polymorphic.f90 b/flang/test/Lower/HLFIR/ignore-rank-unlimited-polymorphic.f90
index 43986c8198b9..c2118432a981 100644
--- a/flang/test/Lower/HLFIR/ignore-rank-unlimited-polymorphic.f90
+++ b/flang/test/Lower/HLFIR/ignore-rank-unlimited-polymorphic.f90
@@ -49,7 +49,7 @@ subroutine test_logical_assumed_shape_array(x)
 end subroutine test_logical_assumed_shape_array
 ! CHECK-LABEL:   func.func @_QPtest_logical_assumed_shape_array(
 ! CHECK-SAME:                                                   %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_logical_assumed_shape_arrayEx"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_logical_assumed_shape_arrayEx"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.rebox %[[VAL_1]]#0 : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> !fir.class<!fir.array<?xnone>>
 ! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.class<!fir.array<?xnone>>) -> !fir.class<none>
 ! CHECK:           fir.call @_QPcallee(%[[VAL_3]]) fastmath<contract> : (!fir.class<none>) -> ()
@@ -63,7 +63,7 @@ subroutine test_real_2d_pointer(x)
 end subroutine test_real_2d_pointer
 ! CHECK-LABEL:   func.func @_QPtest_real_2d_pointer(
 ! CHECK-SAME:                                       %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_real_2d_pointerEx"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_real_2d_pointerEx"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?xf32>>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.array<?x?xf32>>>) -> !fir.class<!fir.array<?x?xnone>>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.class<none>
@@ -78,7 +78,7 @@ subroutine test_up_assumed_shape_1d_array(x)
 end subroutine test_up_assumed_shape_1d_array
 ! CHECK-LABEL:   func.func @_QPtest_up_assumed_shape_1d_array(
 ! CHECK-SAME:                                                 %[[VAL_0:.*]]: !fir.class<!fir.array<?xnone>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_up_assumed_shape_1d_arrayEx"} : (!fir.class<!fir.array<?xnone>>) -> (!fir.class<!fir.array<?xnone>>, !fir.class<!fir.array<?xnone>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_up_assumed_shape_1d_arrayEx"} : (!fir.class<!fir.array<?xnone>>, !fir.dscope) -> (!fir.class<!fir.array<?xnone>>, !fir.class<!fir.array<?xnone>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.class<!fir.array<?xnone>>) -> !fir.class<none>
 ! CHECK:           fir.call @_QPcallee(%[[VAL_2]]) fastmath<contract> : (!fir.class<none>) -> ()
 ! CHECK:           return
@@ -115,7 +115,7 @@ subroutine test_up_allocatable_2d_array(x)
 end subroutine test_up_allocatable_2d_array
 ! CHECK-LABEL:   func.func @_QPtest_up_allocatable_2d_array(
 ! CHECK-SAME:                                               %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_up_allocatable_2d_arrayEx"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_up_allocatable_2d_arrayEx"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !fir.class<!fir.array<?x?xnone>>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.class<none>
@@ -130,7 +130,7 @@ subroutine test_up_pointer_1d_array(x)
 end subroutine test_up_pointer_1d_array
 ! CHECK-LABEL:   func.func @_QPtest_up_pointer_1d_array(
 ! CHECK-SAME:                                           %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_up_pointer_1d_arrayEx"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>>) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_up_pointer_1d_arrayEx"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<?xnone>>>>
 ! CHECK:           %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.class<!fir.ptr<!fir.array<?xnone>>>) -> !fir.class<!fir.array<?xnone>>
 ! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.class<!fir.array<?xnone>>) -> !fir.class<none>
diff --git a/flang/test/Lower/HLFIR/implicit-type-conversion.f90 b/flang/test/Lower/HLFIR/implicit-type-conversion.f90
index ec0fb6e3bb12..dc2d111a8f7f 100644
--- a/flang/test/Lower/HLFIR/implicit-type-conversion.f90
+++ b/flang/test/Lower/HLFIR/implicit-type-conversion.f90
@@ -3,8 +3,8 @@
 ! CHECK-LABEL:   func.func @_QPtest1(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"},
 ! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "y"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest1Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest1Ey"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest1Ex"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest1Ey"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.logical<4>) -> i32
 ! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_2]]#0 : i32, !fir.ref<i32>
@@ -19,8 +19,8 @@ end subroutine test1
 ! CHECK-LABEL:   func.func @_QPtest2(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"},
 ! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "y"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest2Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest2Ey"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest2Ex"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest2Ey"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i32>
 ! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i32) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_3]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -35,8 +35,8 @@ end subroutine test2
 ! CHECK-LABEL:   func.func @_QPtest3(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "x"},
 ! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "y"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest3Ex"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest3Ey"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest3Ex"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest3Ey"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
 ! CHECK:           %[[VAL_6:.*]] = arith.cmpi eq, %[[VAL_4]], %[[VAL_5]] : i32
@@ -54,8 +54,8 @@ end subroutine test3
 ! CHECK-LABEL:   func.func @_QPtest4(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"},
 ! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "y"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest4Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest4Ey"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest4Ex"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest4Ey"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
 ! CHECK:           %[[VAL_6:.*]] = arith.cmpi eq, %[[VAL_4]], %[[VAL_5]] : i32
@@ -73,8 +73,8 @@ end subroutine test4
 ! CHECK-LABEL:   func.func @_QPtest5(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"},
 ! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "y"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest5Ex"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest5Ey"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest5Ex"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest5Ey"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.logical<4>) -> i32
 ! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_2]]#0 : i32, !fir.box<!fir.array<?xi32>>
@@ -89,8 +89,8 @@ end subroutine test5
 ! CHECK-LABEL:   func.func @_QPtest6(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"},
 ! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "y"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest6Ex"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest6Ey"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest6Ex"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest6Ey"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_3]]#0, %[[VAL_4]] : (!fir.box<!fir.array<?x!fir.logical<4>>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]]#1 : (index) -> !fir.shape<1>
@@ -114,8 +114,8 @@ end subroutine test6
 ! CHECK-LABEL:   func.func @_QPtest7(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "x"},
 ! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "y"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest7Ex"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest7Ey"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest7Ex"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest7Ey"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_3]]#0, %[[VAL_4]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]]#1 : (index) -> !fir.shape<1>
diff --git a/flang/test/Lower/HLFIR/intentout-allocatable-components.f90 b/flang/test/Lower/HLFIR/intentout-allocatable-components.f90
index 797e4c89ae23..9d4bedbd9be6 100644
--- a/flang/test/Lower/HLFIR/intentout-allocatable-components.f90
+++ b/flang/test/Lower/HLFIR/intentout-allocatable-components.f90
@@ -10,7 +10,7 @@ subroutine test_intentout_component_deallocate(a)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_intentout_component_deallocate(
 ! CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<!fir.type<_QFtest_intentout_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "_QFtest_intentout_component_deallocateEa"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "_QFtest_intentout_component_deallocateEa"}
 ! CHECK:  %[[VAL_2:.*]] = fir.embox %[[VAL_1]]#1 : (!fir.ref<!fir.type<_QFtest_intentout_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<!fir.type<_QFtest_intentout_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>
 ! CHECK:  %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.box<!fir.type<_QFtest_intentout_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<none>
 ! CHECK:  %[[VAL_4:.*]] = fir.call @_FortranADestroy(%[[VAL_3]]) fastmath<contract> : (!fir.box<none>) -> none
@@ -23,7 +23,7 @@ subroutine test_intentout_optional_component_deallocate(a)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_intentout_optional_component_deallocate(
 ! CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<!fir.type<_QFtest_intentout_optional_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_out, optional>, uniq_name = "_QFtest_intentout_optional_component_deallocateEa"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_out, optional>, uniq_name = "_QFtest_intentout_optional_component_deallocateEa"}
 ! CHECK:  %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#1 : (!fir.ref<!fir.type<_QFtest_intentout_optional_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>) -> i1
 ! CHECK:  fir.if %[[VAL_2]] {
 ! CHECK:    %[[VAL_3:.*]] = fir.embox %[[VAL_1]]#1 : (!fir.ref<!fir.type<_QFtest_intentout_optional_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<!fir.type<_QFtest_intentout_optional_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>
diff --git a/flang/test/Lower/HLFIR/internal-procedures.f90 b/flang/test/Lower/HLFIR/internal-procedures.f90
index 3c4439911809..f0df1a7f6e64 100644
--- a/flang/test/Lower/HLFIR/internal-procedures.f90
+++ b/flang/test/Lower/HLFIR/internal-procedures.f90
@@ -64,7 +64,7 @@ contains
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_proc_pointer(
 ! CHECK-SAME:                                    %[[VAL_0:.*]]: !fir.ref<!fir.boxproc<() -> ()>>) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointerEp"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointerEp"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca tuple<!fir.ref<!fir.boxproc<() -> ()>>>
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
 ! CHECK:           %[[VAL_4:.*]] = fir.coordinate_of %[[VAL_2]], %[[VAL_3]] : (!fir.ref<tuple<!fir.ref<!fir.boxproc<() -> ()>>>>, i32) -> !fir.llvm_ptr<!fir.ref<!fir.boxproc<() -> ()>>>
diff --git a/flang/test/Lower/HLFIR/intrinsic-dynamically-optional.f90 b/flang/test/Lower/HLFIR/intrinsic-dynamically-optional.f90
index 39671d7931a1..d1969049828c 100644
--- a/flang/test/Lower/HLFIR/intrinsic-dynamically-optional.f90
+++ b/flang/test/Lower/HLFIR/intrinsic-dynamically-optional.f90
@@ -166,10 +166,10 @@ end function
 ! CHECK-SAME:                                                   %[[VAL_1:.*]]: !fir.ref<!fir.array<3xf32>> {fir.bindc_name = "imaginary", fir.optional}) -> !fir.array<3x!fir.complex<4>> {
 ! CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_3]]) {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest_elemental_optional_as_valueEimaginary"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xf32>>, !fir.ref<!fir.array<3xf32>>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_3]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFtest_elemental_optional_as_valueEimaginary"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<3xf32>>, !fir.ref<!fir.array<3xf32>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFtest_elemental_optional_as_valueEreal"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<3xf32>>, !fir.ref<!fir.array<3xf32>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_elemental_optional_as_valueEreal"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<3xf32>>, !fir.ref<!fir.array<3xf32>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 3 : index
 ! CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.array<3x!fir.complex<4>> {bindc_name = "test_elemental_optional_as_value", uniq_name = "_QFtest_elemental_optional_as_valueEtest_elemental_optional_as_value"}
 ! CHECK:           %[[VAL_10:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
diff --git a/flang/test/Lower/HLFIR/issue80884.f90 b/flang/test/Lower/HLFIR/issue80884.f90
index 2a7792b6004c..725ed1982975 100644
--- a/flang/test/Lower/HLFIR/issue80884.f90
+++ b/flang/test/Lower/HLFIR/issue80884.f90
@@ -12,8 +12,8 @@ subroutine issue80884(p, targ)
   p(1:100) => targ%array
 end subroutine
 ! CHECK-LABEL:   func.func @_QPissue80884(
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFissue80884Ep"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFissue80884Etarg"} : (!fir.ref<!fir.type<_QFissue80884Tt{t0:!fir.type<_QFissue80884Tt0{array:!fir.array<10x10xf32>}>}>>) -> (!fir.ref<!fir.type<_QFissue80884Tt{t0:!fir.type<_QFissue80884Tt0{array:!fir.array<10x10xf32>}>}>>, !fir.ref<!fir.type<_QFissue80884Tt{t0:!fir.type<_QFissue80884Tt0{array:!fir.array<10x10xf32>}>}>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFissue80884Ep"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFissue80884Etarg"} : (!fir.ref<!fir.type<_QFissue80884Tt{t0:!fir.type<_QFissue80884Tt0{array:!fir.array<10x10xf32>}>}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QFissue80884Tt{t0:!fir.type<_QFissue80884Tt0{array:!fir.array<10x10xf32>}>}>>, !fir.ref<!fir.type<_QFissue80884Tt{t0:!fir.type<_QFissue80884Tt0{array:!fir.array<10x10xf32>}>}>>)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i64
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : i64
 ! CHECK:           %[[VAL_6:.*]] = hlfir.designate %[[VAL_3]]#0{"t0"}   : (!fir.ref<!fir.type<_QFissue80884Tt{t0:!fir.type<_QFissue80884Tt0{array:!fir.array<10x10xf32>}>}>>) -> !fir.ref<!fir.type<_QFissue80884Tt0{array:!fir.array<10x10xf32>}>>
diff --git a/flang/test/Lower/HLFIR/maxloc.f90 b/flang/test/Lower/HLFIR/maxloc.f90
index ea3cce92ae90..166a1b9db172 100644
--- a/flang/test/Lower/HLFIR/maxloc.f90
+++ b/flang/test/Lower/HLFIR/maxloc.f90
@@ -357,11 +357,12 @@ subroutine scalar_dim1(a, d, m, b, s)
 end subroutine
 ! CHECK-LABEL:  func.func @_QPscalar_dim1(
 ! CHECK:            %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "d"}, %[[ARG2:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "m"}, %[[ARG3:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "b"}, %[[ARG4:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "s"}) {
-! CHECK-NEXT:    %[[V0:.*]]:2 = hlfir.declare %[[ARG0]]
-! CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %[[ARG3]]
-! CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[ARG2]]
-! CHECK-NEXT:    %[[V4:.*]]:2 = hlfir.declare %[[ARG4]]
+! CHECK-NEXT:    %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK-NEXT:    %[[V0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[DSCOPE]]
+! CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %[[ARG3]] dummy_scope %[[DSCOPE]]
+! CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %[[DSCOPE]]
+! CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %[[DSCOPE]]
+! CHECK-NEXT:    %[[V4:.*]]:2 = hlfir.declare %[[ARG4]] dummy_scope %[[DSCOPE]]
 ! CHECK-NEXT:    %[[V5:.*]] = fir.load %[[V1]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK-NEXT:    %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
 ! CHECK-NEXT:    %[[V7:.*]] = hlfir.maxloc %[[V0]]#0 dim %[[V6]] mask %[[V3]]#0 back %[[V5]] {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, i32, !fir.box<!fir.array<?x!fir.logical<4>>>, !fir.logical<4>) -> i16
diff --git a/flang/test/Lower/HLFIR/minloc.f90 b/flang/test/Lower/HLFIR/minloc.f90
index c27430689ee0..f835cf54b2a7 100644
--- a/flang/test/Lower/HLFIR/minloc.f90
+++ b/flang/test/Lower/HLFIR/minloc.f90
@@ -357,11 +357,12 @@ subroutine scalar_dim1(a, d, m, b, s)
 end subroutine
 ! CHECK-LABEL:  func.func @_QPscalar_dim1(
 ! CHECK:            %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "d"}, %[[ARG2:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "m"}, %[[ARG3:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "b"}, %[[ARG4:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "s"}) {
-! CHECK-NEXT:    %[[V0:.*]]:2 = hlfir.declare %[[ARG0]]
-! CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %[[ARG3]]
-! CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %[[ARG1]]
-! CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[ARG2]]
-! CHECK-NEXT:    %[[V4:.*]]:2 = hlfir.declare %[[ARG4]]
+! CHECK-NEXT:    %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK-NEXT:    %[[V0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[DSCOPE]]
+! CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %[[ARG3]] dummy_scope %[[DSCOPE]]
+! CHECK-NEXT:    %[[V2:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %[[DSCOPE]]
+! CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %[[DSCOPE]]
+! CHECK-NEXT:    %[[V4:.*]]:2 = hlfir.declare %[[ARG4]] dummy_scope %[[DSCOPE]]
 ! CHECK-NEXT:    %[[V5:.*]] = fir.load %[[V1]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK-NEXT:    %[[V6:.*]] = fir.load %[[V2]]#0 : !fir.ref<i32>
 ! CHECK-NEXT:    %[[V7:.*]] = hlfir.minloc %[[V0]]#0 dim %[[V6]] mask %[[V3]]#0 back %[[V5]] {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, i32, !fir.box<!fir.array<?x!fir.logical<4>>>, !fir.logical<4>) -> i16
diff --git a/flang/test/Lower/HLFIR/procedure-pointer.f90 b/flang/test/Lower/HLFIR/procedure-pointer.f90
index 28965b22de97..ce20f19322b4 100644
--- a/flang/test/Lower/HLFIR/procedure-pointer.f90
+++ b/flang/test/Lower/HLFIR/procedure-pointer.f90
@@ -186,10 +186,10 @@ end subroutine
 subroutine sub7(p1, p2)
 use m
   procedure(real_func), pointer :: p1
-! CHECK: %[[VAL_0:.*]]:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub7Ep1"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK: %[[VAL_0:.*]]:2 = hlfir.declare %arg0 dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub7Ep1"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 
   procedure(char_func), pointer :: p2
-! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %arg1 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub7Ep2"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %arg1 dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFsub7Ep2"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 
   call foo1(p1)
 ! CHECK: %[[VAL_2:.*]] = fir.load %[[VAL_0]]#0 : !fir.ref<!fir.boxproc<() -> ()>>
@@ -265,7 +265,7 @@ contains
   function reffunc(arg) result(pp)
     integer :: arg
     procedure(real_func), pointer :: pp
-! CHECK: %[[VAL_0:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFsub10FreffuncEarg"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[VAL_0:.*]]:2 = hlfir.declare %arg0 dummy_scope %{{[0-9]+}} {uniq_name = "_QFsub10FreffuncEarg"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.boxproc<(!fir.ref<f32>) -> f32> {bindc_name = "pp", uniq_name = "_QFsub10FreffuncEpp"}
 ! CHECK: %[[VAL_2:.*]] = fir.zero_bits (!fir.ref<f32>) -> f32
 ! CHECK: %[[VAL_3:.*]] = fir.emboxproc %[[VAL_2]] : ((!fir.ref<f32>) -> f32) -> !fir.boxproc<(!fir.ref<f32>) -> f32>
diff --git a/flang/test/Lower/HLFIR/statement-functions.f90 b/flang/test/Lower/HLFIR/statement-functions.f90
index d19b912e0fe2..4f91c947690c 100644
--- a/flang/test/Lower/HLFIR/statement-functions.f90
+++ b/flang/test/Lower/HLFIR/statement-functions.f90
@@ -43,7 +43,7 @@ subroutine char_test2(c)
   call test(stmt_func(c))
 end subroutine
 ! CHECK-LABEL:  func.func @_QPchar_test2(
-! CHECK:    %[[C:.*]]:2 = hlfir.declare %1 typeparams %c10 {uniq_name = "_QFchar_test2Ec"} : (!fir.ref<!fir.char<1,10>>, index) -> (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>)
+! CHECK:    %[[C:.*]]:2 = hlfir.declare %{{.*}} typeparams %c10 dummy_scope %{{[0-9]+}} {uniq_name = "_QFchar_test2Ec"} : (!fir.ref<!fir.char<1,10>>, index, !fir.dscope) -> (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>)
 ! CHECK:    %[[CAST:.*]] = fir.convert %[[C]]#0 : (!fir.ref<!fir.char<1,10>>) -> !fir.ref<!fir.char<1,5>>
 ! CHECK:    %[[C_STMT_FUNC:.*]]:2 = hlfir.declare %[[CAST]] typeparams %c5{{.*}} {uniq_name = "_QFchar_test2Fstmt_funcEc_stmt_func"} : (!fir.ref<!fir.char<1,5>>, index) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
 ! CHECK:    hlfir.concat %[[C_STMT_FUNC]]#0, %{{.*}} len %{{.*}} : (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,7>>, index) -> !hlfir.expr<!fir.char<1,12>>
diff --git a/flang/test/Lower/HLFIR/structure-constructor.f90 b/flang/test/Lower/HLFIR/structure-constructor.f90
index d02427d2ff67..41d08c14f5fa 100644
--- a/flang/test/Lower/HLFIR/structure-constructor.f90
+++ b/flang/test/Lower/HLFIR/structure-constructor.f90
@@ -43,7 +43,7 @@ end subroutine test1
 ! CHECK:           %[[VAL_4:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,4>>
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 4 : index
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_5]] typeparams %[[VAL_6]] {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.char<1,4>>, index) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_5]] typeparams %[[VAL_6]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.char<1,4>>, index, !fir.dscope) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
 ! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>) -> (!fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>, !fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>)
 ! CHECK:           %[[VAL_9:.*]] = fir.embox %[[VAL_8]]#0 : (!fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>) -> !fir.box<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>
 ! CHECK:           %[[VAL_10:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
@@ -71,7 +71,7 @@ end subroutine test2
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtest2Eres"} : (!fir.ref<!fir.type<_QMtypesTt2{i:!fir.array<10xi32>}>>) -> (!fir.ref<!fir.type<_QMtypesTt2{i:!fir.array<10xi32>}>>, !fir.ref<!fir.type<_QMtypesTt2{i:!fir.array<10xi32>}>>)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 10 : index
 ! CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) {uniq_name = "_QFtest2Ex"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest2Ex"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt2{i:!fir.array<10xi32>}>>) -> (!fir.ref<!fir.type<_QMtypesTt2{i:!fir.array<10xi32>}>>, !fir.ref<!fir.type<_QMtypesTt2{i:!fir.array<10xi32>}>>)
 ! CHECK:           %[[VAL_8:.*]] = fir.embox %[[VAL_7]]#0 : (!fir.ref<!fir.type<_QMtypesTt2{i:!fir.array<10xi32>}>>) -> !fir.box<!fir.type<_QMtypesTt2{i:!fir.array<10xi32>}>>
 ! CHECK:           %[[VAL_9:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
@@ -104,7 +104,7 @@ end subroutine test3
 ! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_4]] : (!fir.box<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>) -> !fir.box<none>
 ! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
 ! CHECK:           %[[VAL_9:.*]] = fir.call @_FortranAInitialize(%[[VAL_7]], %[[VAL_8]], %[[VAL_6]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> none
-! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest3Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest3Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
 ! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>, !fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>)
 ! CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_11]]#0 : (!fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>) -> !fir.box<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>
 ! CHECK:           %[[VAL_13:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
@@ -141,7 +141,7 @@ end subroutine test4
 ! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
 ! CHECK:           %[[VAL_9:.*]] = fir.call @_FortranAInitialize(%[[VAL_7]], %[[VAL_8]], %[[VAL_6]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> none
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 2 : index
-! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_10]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest4Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>>, index) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>>)
+! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_10]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest4Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>>, index, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>>)
 ! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>, !fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>)
 ! CHECK:           %[[VAL_13:.*]] = fir.embox %[[VAL_12]]#0 : (!fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>) -> !fir.box<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>
 ! CHECK:           %[[VAL_14:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
@@ -184,7 +184,7 @@ end subroutine test5
 ! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_4]] : (!fir.box<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>) -> !fir.box<none>
 ! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
 ! CHECK:           %[[VAL_9:.*]] = fir.call @_FortranAInitialize(%[[VAL_7]], %[[VAL_8]], %[[VAL_6]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> none
-! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest5Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>)
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest5Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>)
 ! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>, !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>)
 ! CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_11]]#0 : (!fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>) -> !fir.box<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>
 ! CHECK:           %[[VAL_13:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
@@ -226,7 +226,7 @@ end subroutine test6
 ! CHECK:           %[[VAL_7:.*]]:2 = fir.unboxchar %[[VAL_1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,4>>
 ! CHECK:           %[[VAL_9:.*]] = arith.constant 4 : index
-! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_8]] typeparams %[[VAL_9]] {uniq_name = "_QFtest6Ec"} : (!fir.ref<!fir.char<1,4>>, index) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_8]] typeparams %[[VAL_9]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest6Ec"} : (!fir.ref<!fir.char<1,4>>, index, !fir.dscope) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
 ! CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}> {bindc_name = "res", uniq_name = "_QFtest6Eres"}
 ! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFtest6Eres"} : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>)
 ! CHECK:           %[[VAL_13:.*]] = fir.embox %[[VAL_12]]#1 : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
@@ -235,7 +235,7 @@ end subroutine test6
 ! CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_13]] : (!fir.box<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<none>
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_14]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
 ! CHECK:           %[[VAL_18:.*]] = fir.call @_FortranAInitialize(%[[VAL_16]], %[[VAL_17]], %[[VAL_15]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> none
-! CHECK:           %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>)
+! CHECK:           %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>)
 ! CHECK:           %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>)
 ! CHECK:           %[[VAL_21:.*]] = fir.embox %[[VAL_20]]#0 : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
 ! CHECK:           %[[VAL_22:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
@@ -316,7 +316,7 @@ end subroutine test7
 ! CHECK-LABEL:   func.func @_QPtest7(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest7En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest7En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}> {bindc_name = "x", uniq_name = "_QFtest7Ex"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFtest7Ex"} : (!fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>, !fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>)
 ! CHECK:           %[[VAL_5:.*]] = fir.embox %[[VAL_4]]#1 : (!fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>) -> !fir.box<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
diff --git a/flang/test/Lower/HLFIR/transformational.f90 b/flang/test/Lower/HLFIR/transformational.f90
index 5f1137277336..96cda5daaacb 100644
--- a/flang/test/Lower/HLFIR/transformational.f90
+++ b/flang/test/Lower/HLFIR/transformational.f90
@@ -16,7 +16,7 @@ subroutine test_transformational_implemented_with_runtime_allocation(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_transformational_implemented_with_runtime_allocation(
 ! CHECK-SAME:                                                                          %[[ARG0:.*]]: !fir.ref<!fir.array<10x10xf32>> {fir.bindc_name = "x"}) {
-! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QFtest_transformational_implemented_with_runtime_allocationEx"}
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_transformational_implemented_with_runtime_allocationEx"}
 ! CHECK:  %[[VAL_2:.*]] = hlfir.minloc %[[VAL_1]]#0
 ! CHECK:  %[[VAL_3:.*]] = hlfir.shape_of %[[VAL_2]]
 ! CHECK:  %[[VAL_4:.*]]:3 = hlfir.associate %[[VAL_2]](%[[VAL_3]]) {adapt.valuebyref}
diff --git a/flang/test/Lower/HLFIR/transpose.f90 b/flang/test/Lower/HLFIR/transpose.f90
index e37e83c7a501..6d8e337f1ac8 100644
--- a/flang/test/Lower/HLFIR/transpose.f90
+++ b/flang/test/Lower/HLFIR/transpose.f90
@@ -8,8 +8,8 @@ endsubroutine
 ! CHECK-LABEL: func.func @_QPtranspose1
 ! CHECK:           %[[M_ARG:.*]]: !fir.ref<!fir.array<1x2xi32>>
 ! CHECK:           %[[RES_ARG:.*]]: !fir.ref<!fir.array<2x1xi32>>
-! CHECK-DAG:     %[[ARG:.*]]:2 = hlfir.declare %[[M_ARG]](%[[M_SHAPE:.*]]) {[[NAME:.*]]} : (!fir.ref<!fir.array<1x2xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<1x2xi32>>, !fir.ref<!fir.array<1x2xi32>>)
-! CHECK-DAG:     %[[RES:.*]]:2 = hlfir.declare %[[RES_ARG]](%[[RES_SHAPE:.*]]) {[[NAME2:.*]]} : (!fir.ref<!fir.array<2x1xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<2x1xi32>>, !fir.ref<!fir.array<2x1xi32>>)
+! CHECK-DAG:     %[[ARG:.*]]:2 = hlfir.declare %[[M_ARG]](%[[M_SHAPE:.*]]) dummy_scope %{{[0-9]+}} {[[NAME:.*]]} : (!fir.ref<!fir.array<1x2xi32>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<1x2xi32>>, !fir.ref<!fir.array<1x2xi32>>)
+! CHECK-DAG:     %[[RES:.*]]:2 = hlfir.declare %[[RES_ARG]](%[[RES_SHAPE:.*]]) dummy_scope %{{[0-9]+}} {[[NAME2:.*]]} : (!fir.ref<!fir.array<2x1xi32>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<2x1xi32>>, !fir.ref<!fir.array<2x1xi32>>)
 ! CHECK:         %[[EXPR:.*]] = hlfir.transpose %[[ARG]]#0 : (!fir.ref<!fir.array<1x2xi32>>) -> !hlfir.expr<2x1xi32>
 ! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[RES]]#0
 ! CHECK-NEXT:    hlfir.destroy %[[EXPR]]
@@ -38,7 +38,7 @@ endsubroutine
 ! CHECK:           %[[M_ARG:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
 ! CHECK:           %[[RES_ARG:.*]]: !fir.ref<!fir.array<2x1xi32>>
 ! CHECK-DAG:     %[[ARG:.*]]:2 = hlfir.declare %[[M_ARG]]
-! CHECK-DAG:     %[[RES:.*]]:2 = hlfir.declare %[[RES_ARG]](%[[RES_SHAPE:.*]]) {[[NAME2:.*]]} : (!fir.ref<!fir.array<2x1xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<2x1xi32>>, !fir.ref<!fir.array<2x1xi32>>)
+! CHECK-DAG:     %[[RES:.*]]:2 = hlfir.declare %[[RES_ARG]](%[[RES_SHAPE:.*]]) dummy_scope %{{[0-9]+}} {[[NAME2:.*]]} : (!fir.ref<!fir.array<2x1xi32>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<2x1xi32>>, !fir.ref<!fir.array<2x1xi32>>)
 ! CHECK:         %[[ARG_LOADED:.*]] = fir.load %[[ARG]]#0
 ! CHECK:         %[[EXPR:.*]] = hlfir.transpose %[[ARG_LOADED]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>) -> !hlfir.expr<?x?xi32>
 ! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[RES]]#0
@@ -54,8 +54,8 @@ end subroutine test_polymorphic_result
 ! CHECK-LABEL:   func.func @_QPtest_polymorphic_result(
 ! CHECK-SAME:        %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>> {fir.bindc_name = "m"},
 ! CHECK-SAME:        %[[VAL_1:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>> {fir.bindc_name = "res"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_polymorphic_resultEm"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_polymorphic_resultEres"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_polymorphic_resultEm"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest_polymorphic_resultEres"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
 ! CHECK:           %[[VAL_5:.*]] = hlfir.transpose %[[VAL_4]] : (!fir.class<!fir.heap<!fir.array<?x?xnone>>>) -> !hlfir.expr<?x?xnone?>
 ! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_3]]#0 realloc : !hlfir.expr<?x?xnone?>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x?xnone>>>>
diff --git a/flang/test/Lower/HLFIR/unary-ops.f90 b/flang/test/Lower/HLFIR/unary-ops.f90
index db2c1ceefaa9..b04d6b4cf949 100644
--- a/flang/test/Lower/HLFIR/unary-ops.f90
+++ b/flang/test/Lower/HLFIR/unary-ops.f90
@@ -39,7 +39,7 @@ subroutine test_not(l, x)
   l = .not.x
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_not(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK:  %[[VAL_5:.*]] = arith.constant true
 ! CHECK:  %[[VAL_6:.*]] = fir.convert %[[VAL_4]] : (!fir.logical<4>) -> i1
@@ -50,7 +50,7 @@ subroutine test_negate_int(res, x)
   res = -x
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_negate_int(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
 ! CHECK:  %[[VAL_5:.*]] = arith.constant 0 : i32
 ! CHECK:  %[[VAL_6:.*]] = arith.subi %[[VAL_5]], %[[VAL_4]] : i32
@@ -60,7 +60,7 @@ subroutine test_negate_real(res, x)
   res = -x
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_negate_real(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<f32>
 ! CHECK:  %[[VAL_5:.*]] = arith.negf %[[VAL_4]] fastmath<contract> : f32
 
@@ -69,7 +69,7 @@ subroutine test_negate_complex(res, x)
   res = -x
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_negate_complex(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_5:.*]] = fir.negc %[[VAL_4]] : !fir.complex<4>
 
@@ -79,7 +79,7 @@ subroutine test_complex_component_real(res, x)
   res = real(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_complex_component_real(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_5:.*]] = fir.extract_value %[[VAL_4]], [0 : index] : (!fir.complex<4>) -> f32
 
@@ -89,6 +89,6 @@ subroutine test_complex_component_imag(res, x)
   res = aimag(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_complex_component_imag(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}x"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
 ! CHECK:  %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.complex<4>>
 ! CHECK:  %[[VAL_5:.*]] = fir.extract_value %[[VAL_4]], [1 : index] : (!fir.complex<4>) -> f32
diff --git a/flang/test/Lower/HLFIR/user-defined-assignment.f90 b/flang/test/Lower/HLFIR/user-defined-assignment.f90
index 6f887cb00de3..f0e24f11c5ab 100644
--- a/flang/test/Lower/HLFIR/user-defined-assignment.f90
+++ b/flang/test/Lower/HLFIR/user-defined-assignment.f90
@@ -35,8 +35,8 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QMuser_defPtest_user_defined_elemental_array(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "i"},
 ! CHECK-SAME:    %[[VAL_1:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "l"}) {
-! CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QMuser_defFtest_user_defined_elemental_arrayEi"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QMuser_defFtest_user_defined_elemental_arrayEl"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+! CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_user_defined_elemental_arrayEi"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_user_defined_elemental_arrayEl"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
 ! CHECK:    hlfir.region_assign {
 ! CHECK:      hlfir.yield %[[VAL_3]]#0 : !fir.box<!fir.array<?x!fir.logical<4>>>
 ! CHECK:    } to {
@@ -53,8 +53,8 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QMuser_defPtest_user_defined_elemental_array_value(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.complex<4>>> {fir.bindc_name = "z"},
 ! CHECK-SAME:    %[[VAL_1:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "l"}) {
-! CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QMuser_defFtest_user_defined_elemental_array_valueEl"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
-! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QMuser_defFtest_user_defined_elemental_array_valueEz"} : (!fir.box<!fir.array<?x!fir.complex<4>>>) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
+! CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_user_defined_elemental_array_valueEl"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_user_defined_elemental_array_valueEz"} : (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.complex<4>>>, !fir.box<!fir.array<?x!fir.complex<4>>>)
 ! CHECK:    hlfir.region_assign {
 ! CHECK:      hlfir.yield %[[VAL_2]]#0 : !fir.box<!fir.array<?x!fir.logical<4>>>
 ! CHECK:    } to {
@@ -72,8 +72,8 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QMuser_defPtest_user_defined_scalar(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "i"},
 ! CHECK-SAME:    %[[VAL_1:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "l"}) {
-! CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QMuser_defFtest_user_defined_scalarEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QMuser_defFtest_user_defined_scalarEl"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_user_defined_scalarEi"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_user_defined_scalarEl"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:    hlfir.region_assign {
 ! CHECK:      %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.logical<4>>
 ! CHECK:      hlfir.yield %[[VAL_4]] : !fir.logical<4>
@@ -91,7 +91,7 @@ subroutine test_non_elemental_array(x)
 end subroutine
 ! CHECK-LABEL:   func.func @_QMuser_defPtest_non_elemental_array(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
-! CHECK:    %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QMuser_defFtest_non_elemental_arrayEx"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:    %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_non_elemental_arrayEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:    hlfir.region_assign {
 ! CHECK:      %[[VAL_2:.*]] = arith.constant 4.200000e+01 : f32
 ! CHECK:      %[[VAL_3:.*]] = arith.constant 0 : index
@@ -126,9 +126,9 @@ end subroutine
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "i"},
 ! CHECK-SAME:    %[[VAL_1:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "l"},
 ! CHECK-SAME:    %[[VAL_2:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "l2"}) {
-! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QMuser_defFtest_where_user_def_assignmentEi"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-! CHECK:    %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QMuser_defFtest_where_user_def_assignmentEl"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
-! CHECK:    %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QMuser_defFtest_where_user_def_assignmentEl2"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_where_user_def_assignmentEi"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:    %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_where_user_def_assignmentEl"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+! CHECK:    %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_where_user_def_assignmentEl2"} : (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
 ! CHECK:    hlfir.where {
 ! CHECK:      hlfir.yield %[[VAL_4]]#0 : !fir.box<!fir.array<?x!fir.logical<4>>>
 ! CHECK:    } do {
@@ -171,11 +171,11 @@ end subroutine
 ! CHECK:    %[[VAL_2:.*]] = arith.constant 20 : index
 ! CHECK:    %[[VAL_3:.*]] = arith.constant 10 : index
 ! CHECK:    %[[VAL_4:.*]] = fir.shape %[[VAL_2]], %[[VAL_3]] : (index, index) -> !fir.shape<2>
-! CHECK:    %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QMuser_defFtest_forall_user_def_assignmentEi"} : (!fir.ref<!fir.array<20x10xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<20x10xi32>>, !fir.ref<!fir.array<20x10xi32>>)
+! CHECK:    %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_forall_user_def_assignmentEi"} : (!fir.ref<!fir.array<20x10xi32>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<20x10xi32>>, !fir.ref<!fir.array<20x10xi32>>)
 ! CHECK:    %[[VAL_6:.*]] = arith.constant 20 : index
 ! CHECK:    %[[VAL_7:.*]] = arith.constant 10 : index
 ! CHECK:    %[[VAL_8:.*]] = fir.shape %[[VAL_6]], %[[VAL_7]] : (index, index) -> !fir.shape<2>
-! CHECK:    %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_8]]) {uniq_name = "_QMuser_defFtest_forall_user_def_assignmentEl"} : (!fir.ref<!fir.array<20x10x!fir.logical<4>>>, !fir.shape<2>) -> (!fir.ref<!fir.array<20x10x!fir.logical<4>>>, !fir.ref<!fir.array<20x10x!fir.logical<4>>>)
+! CHECK:    %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_8]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_forall_user_def_assignmentEl"} : (!fir.ref<!fir.array<20x10x!fir.logical<4>>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<20x10x!fir.logical<4>>>, !fir.ref<!fir.array<20x10x!fir.logical<4>>>)
 ! CHECK:    %[[VAL_10:.*]] = arith.constant 1 : i32
 ! CHECK:    %[[VAL_11:.*]] = arith.constant 10 : i32
 ! CHECK:    hlfir.forall lb {
@@ -218,11 +218,11 @@ end subroutine
 ! CHECK:    %[[VAL_2:.*]] = arith.constant 20 : index
 ! CHECK:    %[[VAL_3:.*]] = arith.constant 10 : index
 ! CHECK:    %[[VAL_4:.*]] = fir.shape %[[VAL_2]], %[[VAL_3]] : (index, index) -> !fir.shape<2>
-! CHECK:    %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_4]]) {uniq_name = "_QMuser_defFtest_forall_user_def_assignment_non_elemental_arrayEl"} : (!fir.ref<!fir.array<20x10x!fir.logical<4>>>, !fir.shape<2>) -> (!fir.ref<!fir.array<20x10x!fir.logical<4>>>, !fir.ref<!fir.array<20x10x!fir.logical<4>>>)
+! CHECK:    %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_forall_user_def_assignment_non_elemental_arrayEl"} : (!fir.ref<!fir.array<20x10x!fir.logical<4>>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<20x10x!fir.logical<4>>>, !fir.ref<!fir.array<20x10x!fir.logical<4>>>)
 ! CHECK:    %[[VAL_6:.*]] = arith.constant 20 : index
 ! CHECK:    %[[VAL_7:.*]] = arith.constant 10 : index
 ! CHECK:    %[[VAL_8:.*]] = fir.shape %[[VAL_6]], %[[VAL_7]] : (index, index) -> !fir.shape<2>
-! CHECK:    %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_8]]) {uniq_name = "_QMuser_defFtest_forall_user_def_assignment_non_elemental_arrayEx"} : (!fir.ref<!fir.array<20x10xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<20x10xf32>>, !fir.ref<!fir.array<20x10xf32>>)
+! CHECK:    %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_8]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_forall_user_def_assignment_non_elemental_arrayEx"} : (!fir.ref<!fir.array<20x10xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.ref<!fir.array<20x10xf32>>, !fir.ref<!fir.array<20x10xf32>>)
 ! CHECK:    %[[VAL_10:.*]] = arith.constant 1 : i32
 ! CHECK:    %[[VAL_11:.*]] = arith.constant 10 : i32
 ! CHECK:    hlfir.forall lb {
@@ -269,8 +269,8 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QMuser_defPtest_pointer(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "p"},
 ! CHECK-SAME:    %[[VAL_1:.*]]: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "x"}) {
-! CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMuser_defFtest_pointerEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
-! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QMuser_defFtest_pointerEx"} : (!fir.box<!fir.array<?x?xf32>>) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
+! CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMuser_defFtest_pointerEp"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_pointerEx"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
 ! CHECK:    hlfir.region_assign {
 ! CHECK:      hlfir.yield %[[VAL_3]]#0 : !fir.box<!fir.array<?x?xf32>>
 ! CHECK:    } to {
@@ -287,8 +287,8 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QMuser_defPtest_allocatable(
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {fir.bindc_name = "a"},
 ! CHECK-SAME:    %[[VAL_1:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
-! CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMuser_defFtest_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
-! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QMuser_defFtest_allocatableEx"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:    %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMuser_defFtest_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+! CHECK:    %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMuser_defFtest_allocatableEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:    hlfir.region_assign {
 ! CHECK:      hlfir.yield %[[VAL_3]]#0 : !fir.box<!fir.array<?xf32>>
 ! CHECK:    } to {
@@ -313,7 +313,7 @@ end subroutine test_char_get_length
 ! CHECK-LABEL:   func.func @_QPtest_char_get_length(
 ! CHECK-SAME:                                       %[[VAL_0:.*]]: !fir.boxchar<1> {fir.bindc_name = "ch"}) {
 ! CHECK:           %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]]#0 typeparams %[[VAL_1]]#1 {uniq_name = "_QFtest_char_get_lengthEch"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]]#0 typeparams %[[VAL_1]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_char_get_lengthEch"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtest_char_get_lengthEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFtest_char_get_lengthEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           hlfir.region_assign {
diff --git a/flang/test/Lower/HLFIR/vector-subscript-as-value.f90 b/flang/test/Lower/HLFIR/vector-subscript-as-value.f90
index ee8ded197c95..7161ee088b57 100644
--- a/flang/test/Lower/HLFIR/vector-subscript-as-value.f90
+++ b/flang/test/Lower/HLFIR/vector-subscript-as-value.f90
@@ -68,7 +68,7 @@ subroutine foo3(x, y)
   call bar2(x(1:8:2, 5, y))
 end subroutine
 ! CHECK-LABEL:   func.func @_QPfoo3(
-! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFfoo3Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>)
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFfoo3Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>)
 ! CHECK:  %[[VAL_3:.*]] = arith.constant 20 : index
 ! CHECK:  %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
 ! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]](%[[VAL_4:[a-z0-9]*]])  {{.*}}Ey
@@ -196,8 +196,8 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_passing_subscripted_poly(
 ! CHECK-SAME:                                                %[[VAL_0:.*]]: !fir.class<!fir.array<?x?xnone>>
 ! CHECK-SAME:                                                %[[VAL_1:.*]]: !fir.box<!fir.array<?xi64>>
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest_passing_subscripted_polyEvector"} : (!fir.box<!fir.array<?xi64>>) -> (!fir.box<!fir.array<?xi64>>, !fir.box<!fir.array<?xi64>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_passing_subscripted_polyEx"} : (!fir.class<!fir.array<?x?xnone>>) -> (!fir.class<!fir.array<?x?xnone>>, !fir.class<!fir.array<?x?xnone>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_passing_subscripted_polyEvector"} : (!fir.box<!fir.array<?xi64>>, !fir.dscope) -> (!fir.box<!fir.array<?xi64>>, !fir.box<!fir.array<?xi64>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_passing_subscripted_polyEx"} : (!fir.class<!fir.array<?x?xnone>>, !fir.dscope) -> (!fir.class<!fir.array<?x?xnone>>, !fir.class<!fir.array<?x?xnone>>)
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 314 : index
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
 ! CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_2]]#0, %[[VAL_5]] : (!fir.box<!fir.array<?xi64>>, index) -> (index, index, index)
diff --git a/flang/test/Lower/Intrinsics/associated-proc-pointers.f90 b/flang/test/Lower/Intrinsics/associated-proc-pointers.f90
index 1772b9afdfc0..e07e61b7e597 100644
--- a/flang/test/Lower/Intrinsics/associated-proc-pointers.f90
+++ b/flang/test/Lower/Intrinsics/associated-proc-pointers.f90
@@ -9,7 +9,7 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_proc_pointer_1(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.boxproc<() -> ()>>,
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.boxproc<() -> ()>) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_1Ep"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_1Ep"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref<!fir.boxproc<() -> ()>>
 ! CHECK:           %[[VAL_4:.*]] = fir.box_addr %[[VAL_3]] : (!fir.boxproc<() -> ()>) -> (() -> ())
 ! CHECK:           %[[VAL_5:.*]] = fir.box_addr %[[VAL_1]] : (!fir.boxproc<() -> ()>) -> (() -> ())
@@ -28,8 +28,8 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_proc_pointer_2(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.boxproc<() -> ()>>,
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.ref<!fir.boxproc<() -> ()>>) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_2Ep"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_2Ep_target"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_2Ep"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_2Ep_target"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref<!fir.boxproc<() -> ()>>
 ! CHECK:           %[[VAL_5:.*]] = fir.box_addr %[[VAL_4]] : (!fir.boxproc<() -> ()>) -> (() -> ())
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref<!fir.boxproc<() -> ()>>
@@ -50,7 +50,7 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_proc_pointer_3(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.boxproc<() -> ()>>,
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.boxproc<() -> ()>) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_3Ep"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_3Ep"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]]#1 : !fir.ref<!fir.boxproc<() -> ()>>
 ! CHECK:           %[[VAL_4:.*]] = fir.box_addr %[[VAL_3]] : (!fir.boxproc<() -> ()>) -> (() -> ())
 ! CHECK:           %[[VAL_5:.*]] = fir.box_addr %[[VAL_1]] : (!fir.boxproc<() -> ()>) -> (() -> ())
@@ -69,7 +69,7 @@ subroutine test_proc_pointer_4(p)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_proc_pointer_4(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.boxproc<() -> ()>>) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_4Ep"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_4Ep"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QPsome_external) : () -> ()
 ! CHECK:           %[[VAL_3:.*]] = fir.emboxproc %[[VAL_2]] : (() -> ()) -> !fir.boxproc<() -> ()>
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<!fir.boxproc<() -> ()>>
@@ -95,7 +95,7 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_proc_pointer_5(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.boxproc<() -> ()>>,
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: tuple<!fir.boxproc<() -> ()>, i64> {fir.char_proc}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_5Ep"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_proc_pointer_5Ep"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 ! CHECK:           %[[VAL_3:.*]] = fir.extract_value %[[VAL_1]], [0 : index] : (tuple<!fir.boxproc<() -> ()>, i64>) -> !fir.boxproc<() -> ()>
 ! CHECK:           %[[VAL_4:.*]] = fir.box_addr %[[VAL_3]] : (!fir.boxproc<() -> ()>) -> (() -> ())
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 10 : i64
diff --git a/flang/test/Lower/Intrinsics/c_f_procpointer.f90 b/flang/test/Lower/Intrinsics/c_f_procpointer.f90
index f70a56c91b91..f8792e4c1be0 100644
--- a/flang/test/Lower/Intrinsics/c_f_procpointer.f90
+++ b/flang/test/Lower/Intrinsics/c_f_procpointer.f90
@@ -10,8 +10,8 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_c_funloc(
 ! CHECK-SAME:                                %[[VAL_0:.*]]: !fir.ref<!fir.boxproc<() -> ()>>,
 ! CHECK-SAME:                                %[[VAL_1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>> {fir.bindc_name = "cptr"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest_c_funlocEcptr"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_c_funlocEfptr"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_c_funlocEcptr"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_c_funlocEfptr"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>
 ! CHECK:           %[[VAL_5:.*]] = fir.coordinate_of %[[VAL_2]]#1, %[[VAL_4]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<i64>
@@ -32,8 +32,8 @@ end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_c_funloc_char(
 ! CHECK-SAME:                                     %[[VAL_0:.*]]: !fir.ref<!fir.boxproc<() -> ()>>,
 ! CHECK-SAME:                                     %[[VAL_1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>> {fir.bindc_name = "cptr"}) {
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest_c_funloc_charEcptr"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_c_funloc_charEfptr"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_c_funloc_charEcptr"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_c_funloc_charEfptr"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>
 ! CHECK:           %[[VAL_5:.*]] = fir.coordinate_of %[[VAL_2]]#1, %[[VAL_4]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
 ! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<i64>
diff --git a/flang/test/Lower/Intrinsics/c_funloc-proc-pointers.f90 b/flang/test/Lower/Intrinsics/c_funloc-proc-pointers.f90
index c9578b17ac52..0f398a346d45 100644
--- a/flang/test/Lower/Intrinsics/c_funloc-proc-pointers.f90
+++ b/flang/test/Lower/Intrinsics/c_funloc-proc-pointers.f90
@@ -8,7 +8,7 @@ subroutine test_c_funloc(p)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_c_funloc(
 ! CHECK-SAME:                                %[[VAL_0:.*]]: !fir.ref<!fir.boxproc<() -> ()>>) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_c_funlocEp"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_c_funlocEp"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.boxproc<() -> ()>>
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>
 ! CHECK:           %[[VAL_4:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>
@@ -28,7 +28,7 @@ subroutine test_c_funloc_char(p)
 end subroutine
 ! CHECK-LABEL:   func.func @_QPtest_c_funloc_char(
 ! CHECK-SAME:                                     %[[VAL_0:.*]]: !fir.ref<!fir.boxproc<() -> ()>>) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_c_funloc_charEp"} : (!fir.ref<!fir.boxproc<() -> ()>>) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_c_funloc_charEp"} : (!fir.ref<!fir.boxproc<() -> ()>>, !fir.dscope) -> (!fir.ref<!fir.boxproc<() -> ()>>, !fir.ref<!fir.boxproc<() -> ()>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.boxproc<() -> ()>>
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>
 ! CHECK:           %[[VAL_4:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>
diff --git a/flang/test/Lower/Intrinsics/c_ptr_eq_ne.f90 b/flang/test/Lower/Intrinsics/c_ptr_eq_ne.f90
index 38468739ead5..c6a2f186e4c1 100644
--- a/flang/test/Lower/Intrinsics/c_ptr_eq_ne.f90
+++ b/flang/test/Lower/Intrinsics/c_ptr_eq_ne.f90
@@ -10,8 +10,8 @@ end
 
 ! CHECK-LABEL: func.func @_QPtest_c_ptr_eq(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr1"}, %[[ARG1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr2"}) -> !fir.logical<4> {
-! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_c_ptr_eqEptr1"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
-! CHECK: %[[DECL_ARG1:.*]]:2 = hlfir.declare %[[ARG1]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_c_ptr_eqEptr2"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
+! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_c_ptr_eqEptr1"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
+! CHECK: %[[DECL_ARG1:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_c_ptr_eqEptr2"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
 ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.logical<4> {bindc_name = "test_c_ptr_eq", uniq_name = "_QFtest_c_ptr_eqEtest_c_ptr_eq"}
 ! CHECK: %[[DECL_RET:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFtest_c_ptr_eqEtest_c_ptr_eq"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK: %[[FIELD_ADDRESS:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
@@ -37,8 +37,8 @@ end
 
 ! CHECK-LABEL: func.func @_QPtest_c_ptr_ne(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr1"}, %[[ARG1:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "ptr2"}) -> !fir.logical<4> {
-! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_c_ptr_neEptr1"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
-! CHECK: %[[DECL_ARG1:.*]]:2 = hlfir.declare %[[ARG1]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_c_ptr_neEptr2"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
+! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_c_ptr_neEptr1"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
+! CHECK: %[[DECL_ARG1:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest_c_ptr_neEptr2"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
 ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.logical<4> {bindc_name = "test_c_ptr_ne", uniq_name = "_QFtest_c_ptr_neEtest_c_ptr_ne"}
 ! CHECK: %[[DECL_RET:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFtest_c_ptr_neEtest_c_ptr_ne"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK: %[[FIELD_ADDRESS:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
diff --git a/flang/test/Lower/Intrinsics/execute_command_line-optional.f90 b/flang/test/Lower/Intrinsics/execute_command_line-optional.f90
index 0b75a20216af..e4f9a241197c 100644
--- a/flang/test/Lower/Intrinsics/execute_command_line-optional.f90
+++ b/flang/test/Lower/Intrinsics/execute_command_line-optional.f90
@@ -15,14 +15,15 @@ subroutine all_args_optional(command, isWait, exitVal, cmdVal, msg)
 ! CHECK-NEXT:    %[[c14:.*]] = arith.constant 14 : i32 
 ! CHECK-NEXT:    %true = arith.constant true 
 ! CHECK-NEXT:    %[[c0:.*]] = arith.constant 0 : i64 
-! CHECK-NEXT:    %[[cmdstatDeclare:.*]] = fir.declare %[[cmdstatArg]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_args_optionalEcmdval"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK-NEXT:    %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK-NEXT:    %[[cmdstatDeclare:.*]] = fir.declare %[[cmdstatArg]] dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_args_optionalEcmdval"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
 ! CHECK-NEXT:    %[[commandUnbox:.*]]:2 = fir.unboxchar %[[commandArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT:    %[[commandDeclare:.*]] = fir.declare %[[commandUnbox]]#0 typeparams %[[commandUnbox]]#1 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_args_optionalEcommand"} : (!fir.ref<!fir.char<1,?>>, index) -> !fir.ref<!fir.char<1,?>>
+! CHECK-NEXT:    %[[commandDeclare:.*]] = fir.declare %[[commandUnbox]]#0 typeparams %[[commandUnbox]]#1 dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_args_optionalEcommand"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> !fir.ref<!fir.char<1,?>>
 ! CHECK-NEXT:    %[[commandBoxTemp:.*]] = fir.emboxchar %[[commandDeclare]], %[[commandUnbox]]#1 : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
-! CHECK-NEXT:    %[[exitstatDeclare:.*]] = fir.declare %[[exitstatArg]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_args_optionalEexitval"} : (!fir.ref<i32>) -> !fir.ref<i32>
-! CHECK-NEXT:    %[[waitDeclare:.*]] = fir.declare %[[waitArg]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_args_optionalEiswait"} : (!fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>>
+! CHECK-NEXT:    %[[exitstatDeclare:.*]] = fir.declare %[[exitstatArg]] dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_args_optionalEexitval"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK-NEXT:    %[[waitDeclare:.*]] = fir.declare %[[waitArg]] dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_args_optionalEiswait"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> !fir.ref<!fir.logical<4>>
 ! CHECK-NEXT:    %[[cmdmsgUnbox:.*]]:2 = fir.unboxchar %[[cmdmsgArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT:    %[[cmdmsgDeclare:.*]] = fir.declare %[[cmdmsgUnbox]]#0 typeparams %[[cmdmsgUnbox]]#1 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_args_optionalEmsg"} : (!fir.ref<!fir.char<1,?>>, index) -> !fir.ref<!fir.char<1,?>>
+! CHECK-NEXT:    %[[cmdmsgDeclare:.*]] = fir.declare %[[cmdmsgUnbox]]#0 typeparams %[[cmdmsgUnbox]]#1 dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_args_optionalEmsg"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> !fir.ref<!fir.char<1,?>>
 ! CHECK-NEXT:    %[[cmdmsgBoxTemp:.*]] = fir.emboxchar %[[cmdmsgDeclare]], %[[cmdmsgUnbox]]#1 : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
 ! CHECK-NEXT:    %[[exitstatIsPresent:.*]] = fir.is_present %[[exitstatDeclare]] : (!fir.ref<i32>) -> i1
 ! CHECK-NEXT:    %[[cmdstatIsPresent:.*]] = fir.is_present %[[cmdstatDeclare]] : (!fir.ref<i32>) -> i1
diff --git a/flang/test/Lower/Intrinsics/execute_command_line.f90 b/flang/test/Lower/Intrinsics/execute_command_line.f90
index 8aacd34346b4..6bde50e807b2 100644
--- a/flang/test/Lower/Intrinsics/execute_command_line.f90
+++ b/flang/test/Lower/Intrinsics/execute_command_line.f90
@@ -15,15 +15,16 @@ call execute_command_line(command, isWait, exitVal, cmdVal, msg)
 ! CHECK-NEXT:        %true = arith.constant true 
 ! CHECK-NEXT:        %[[c0:.*]] = arith.constant 0 : i64 
 ! CHECK-NEXT:        %[[c30:.*]] = arith.constant 30 : index
-! CHECK-NEXT:        %[[cmdstatsDeclare:.*]] = fir.declare %[[cmdstatArg]] {uniq_name = "_QFall_argsEcmdval"} : (!fir.ref<i32>) -> !fir.ref<i32>
+! CHECK-NEXT:        %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK-NEXT:        %[[cmdstatsDeclare:.*]] = fir.declare %[[cmdstatArg]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFall_argsEcmdval"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
 ! CHECK-NEXT:        %[[commandUnbox:.*]]:2 = fir.unboxchar %[[commandArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK-NEXT:        %[[commandCast:.*]] = fir.convert %[[commandUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,30>>
-! CHECK-NEXT:        %[[commandDeclare:.*]] = fir.declare %[[commandCast]] typeparams %[[c30]] {uniq_name = "_QFall_argsEcommand"} : (!fir.ref<!fir.char<1,30>>, index) -> !fir.ref<!fir.char<1,30>>
-! CHECK-NEXT:        %[[exitstatDeclare:.*]] = fir.declare %[[exitstatArg]] {uniq_name = "_QFall_argsEexitval"} : (!fir.ref<i32>) -> !fir.ref<i32>
-! CHECK-NEXT:        %[[waitDeclare:.*]] = fir.declare %[[waitArg]] {uniq_name = "_QFall_argsEiswait"} : (!fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>>
+! CHECK-NEXT:        %[[commandDeclare:.*]] = fir.declare %[[commandCast]] typeparams %[[c30]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFall_argsEcommand"} : (!fir.ref<!fir.char<1,30>>, index, !fir.dscope) -> !fir.ref<!fir.char<1,30>>
+! CHECK-NEXT:        %[[exitstatDeclare:.*]] = fir.declare %[[exitstatArg]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFall_argsEexitval"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
+! CHECK-NEXT:        %[[waitDeclare:.*]] = fir.declare %[[waitArg]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFall_argsEiswait"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> !fir.ref<!fir.logical<4>>
 ! CHECK-NEXT:        %[[cmdmsgUnbox:.*]]:2 = fir.unboxchar %[[cmdmsgArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK-NEXT:        %[[cmdmsgCast:.*]] = fir.convert %[[cmdmsgUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,30>>
-! CHECK-NEXT:        %[[cmdmsgDeclare:.*]] = fir.declare %[[cmdmsgCast]] typeparams %[[c30]] {uniq_name = "_QFall_argsEmsg"} : (!fir.ref<!fir.char<1,30>>, index) -> !fir.ref<!fir.char<1,30>>
+! CHECK-NEXT:        %[[cmdmsgDeclare:.*]] = fir.declare %[[cmdmsgCast]] typeparams %[[c30]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFall_argsEmsg"} : (!fir.ref<!fir.char<1,30>>, index, !fir.dscope) -> !fir.ref<!fir.char<1,30>>
 ! CHECK-NEXT:        %[[commandBox:.*]] = fir.embox %[[commandDeclare]] : (!fir.ref<!fir.char<1,30>>) -> !fir.box<!fir.char<1,30>>
 ! CHECK-NEXT:        %[[exitstatBox:.*]] = fir.embox %[[exitstatDeclare]] : (!fir.ref<i32>) -> !fir.box<i32>
 ! CHECK-NEXT:        %[[cmdstatBox:.*]] = fir.embox %[[cmdstatsDeclare]] : (!fir.ref<i32>) -> !fir.box<i32>
@@ -50,12 +51,13 @@ end subroutine all_args
 subroutine only_command_default_wait_true(command)
 CHARACTER(30) :: command
 call execute_command_line(command)
-! CHECK-NEXT:     %[[c52:.*]] = arith.constant 52 : i32 
+! CHECK-NEXT:     %[[c52:.*]] = arith.constant 53 : i32 
 ! CHECK-NEXT:     %true = arith.constant true 
 ! CHECK-NEXT:     %[[c30:.*]] = arith.constant 30 : index
+! CHECK-NEXT:        %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
 ! CHECK-NEXT:     %[[commandUnbox:.*]]:2 = fir.unboxchar %[[cmdArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK-NEXT:     %[[commandCast:.*]] = fir.convert %[[commandUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,30>>
-! CHECK-NEXT:     %[[commandDeclare:.*]] = fir.declare %[[commandCast]] typeparams %[[c30]] {uniq_name = "_QFonly_command_default_wait_trueEcommand"} : (!fir.ref<!fir.char<1,30>>, index) -> !fir.ref<!fir.char<1,30>>
+! CHECK-NEXT:     %[[commandDeclare:.*]] = fir.declare %[[commandCast]] typeparams %[[c30]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFonly_command_default_wait_trueEcommand"} : (!fir.ref<!fir.char<1,30>>, index, !fir.dscope) -> !fir.ref<!fir.char<1,30>>
 ! CHECK-NEXT:     %[[commandBox:.*]] = fir.embox %[[commandDeclare]] : (!fir.ref<!fir.char<1,30>>) -> !fir.box<!fir.char<1,30>>
 ! CHECK-NEXT:     %[[absent:.*]] = fir.absent !fir.box<none>
 ! CHECK:          %[[command:.*]] = fir.convert %[[commandBox]] : (!fir.box<!fir.char<1,30>>) -> !fir.box<none> 
diff --git a/flang/test/Lower/Intrinsics/ieee_logb.f90 b/flang/test/Lower/Intrinsics/ieee_logb.f90
index df15661d51b2..4195ac7af245 100644
--- a/flang/test/Lower/Intrinsics/ieee_logb.f90
+++ b/flang/test/Lower/Intrinsics/ieee_logb.f90
@@ -9,7 +9,7 @@ subroutine out(x)
   ! CHECK:     %[[V_61:[0-9]+]] = fir.declare %[[V_60]] {uniq_name = "_QFoutEl"} : (!fir.ref<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>>
   ! CHECK:     %[[V_62:[0-9]+]] = fir.alloca f64 {bindc_name = "r", uniq_name = "_QFoutEr"}
   ! CHECK:     %[[V_63:[0-9]+]] = fir.declare %[[V_62]] {uniq_name = "_QFoutEr"} : (!fir.ref<f64>) -> !fir.ref<f64>
-  ! CHECK:     %[[V_64:[0-9]+]] = fir.declare %arg0 {uniq_name = "_QFoutEx"} : (!fir.ref<f64>) -> !fir.ref<f64>
+  ! CHECK:     %[[V_64:[0-9]+]] = fir.declare %arg0 dummy_scope %{{[0-9]+}} {uniq_name = "_QFoutEx"} : (!fir.ref<f64>, !fir.dscope) -> !fir.ref<f64>
   real(k) :: x, r
   logical :: L
 
diff --git a/flang/test/Lower/Intrinsics/product.f90 b/flang/test/Lower/Intrinsics/product.f90
index e7f7c0d39ee0..ddefa7a37184 100644
--- a/flang/test/Lower/Intrinsics/product.f90
+++ b/flang/test/Lower/Intrinsics/product.f90
@@ -58,7 +58,7 @@ product_test4 = product(x)
 ! CHECK-DAG: %[[a5:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x!fir.complex<10>>>) -> !fir.box<none>
 ! CHECK-DAG:  %[[a7:.*]] = fir.convert %[[c0]] : (index) -> i32
 ! CHECK-DAG:  %[[a8:.*]] = fir.convert %[[a2]] : (!fir.box<i1>) -> !fir.box<none>
-! CHECK: fir.call @_FortranACppProductComplex10(%[[a4]], %[[a5]], %{{.*}}, %{{.*}}, %[[a7]], %8) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> ()
+! CHECK: fir.call @_FortranACppProductComplex10(%[[a4]], %[[a5]], %{{.*}}, %{{.*}}, %[[a7]], %{{[0-9]+}}) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> ()
 end
 
 ! CHECK-LABEL: func @_QPproduct_test_optional(
diff --git a/flang/test/Lower/Intrinsics/signal.f90 b/flang/test/Lower/Intrinsics/signal.f90
index d6678000677e..5d20bb5c5c07 100644
--- a/flang/test/Lower/Intrinsics/signal.f90
+++ b/flang/test/Lower/Intrinsics/signal.f90
@@ -23,7 +23,7 @@ contains
     integer, optional, intent(out) :: optional_status
 
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca i32
-! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_out, optional>, uniq_name = "_QMmFsetup_signalsEoptional_status"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_out, optional>, uniq_name = "_QMmFsetup_signalsEoptional_status"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QMmFsetup_signalsEstat"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
     call signal(SIGFPE, handler)
diff --git a/flang/test/Lower/Intrinsics/sizeof.f90 b/flang/test/Lower/Intrinsics/sizeof.f90
index e10cb79981a6..7e749f096112 100644
--- a/flang/test/Lower/Intrinsics/sizeof.f90
+++ b/flang/test/Lower/Intrinsics/sizeof.f90
@@ -6,7 +6,7 @@ integer(8) function test1(x)
   test1 = sizeof(x)
 end function
 ! CHECK-LABEL:   func.func @_QPtest1(
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest1Ex"} : (!fir.class<none>) -> (!fir.class<none>, !fir.class<none>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest1Ex"} : (!fir.class<none>, !fir.dscope) -> (!fir.class<none>, !fir.class<none>)
 ! CHECK:           %[[VAL_4:.*]] = fir.box_elesize %[[VAL_3]]#1 : (!fir.class<none>) -> i64
 ! CHECK:           hlfir.assign %[[VAL_4]] to %{{.*}} : i64, !fir.ref<i64>
 
@@ -15,7 +15,7 @@ integer(8) function test2(x)
   test2 = sizeof(x)
 end function
 ! CHECK-LABEL:   func.func @_QPtest2(
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtest2Ex"} : (!fir.class<!fir.array<?x?xnone>>) -> (!fir.class<!fir.array<?x?xnone>>, !fir.class<!fir.array<?x?xnone>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest2Ex"} : (!fir.class<!fir.array<?x?xnone>>, !fir.dscope) -> (!fir.class<!fir.array<?x?xnone>>, !fir.class<!fir.array<?x?xnone>>)
 ! CHECK:           %[[VAL_4:.*]] = fir.box_elesize %[[VAL_3]]#1 : (!fir.class<!fir.array<?x?xnone>>) -> i64
 ! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_3]]#1 : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
 ! CHECK:           %[[VAL_9:.*]] = fir.call @_FortranASize(%[[VAL_7]], %{{.*}}, %{{.*}}) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> i64
diff --git a/flang/test/Lower/Intrinsics/sum.f90 b/flang/test/Lower/Intrinsics/sum.f90
index cafcc0828df8..696892d29126 100644
--- a/flang/test/Lower/Intrinsics/sum.f90
+++ b/flang/test/Lower/Intrinsics/sum.f90
@@ -58,7 +58,7 @@ sum_test4 = sum(x)
 ! CHECK-DAG: %[[a5:.*]] = fir.convert %[[arg0]] : (!fir.box<!fir.array<?x!fir.complex<10>>>) -> !fir.box<none>
 ! CHECK-DAG:  %[[a7:.*]] = fir.convert %[[c0]] : (index) -> i32
 ! CHECK-DAG:  %[[a8:.*]] = fir.convert %[[a2]] : (!fir.box<i1>) -> !fir.box<none>
-! CHECK: fir.call @_FortranACppSumComplex10(%[[a4]], %[[a5]], %{{.*}}, %{{.*}}, %[[a7]], %8) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> ()
+! CHECK: fir.call @_FortranACppSumComplex10(%[[a4]], %[[a5]], %{{.*}}, %{{.*}}, %[[a7]], %{{[0-9]+}}) {{.*}}: (!fir.ref<complex<f80>>, !fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> ()
 end
 
 ! CHECK-LABEL: func @_QPsum_test_optional(
diff --git a/flang/test/Lower/Intrinsics/system-optional.f90 b/flang/test/Lower/Intrinsics/system-optional.f90
index 5047437c5c3c..8a2db132d672 100644
--- a/flang/test/Lower/Intrinsics/system-optional.f90
+++ b/flang/test/Lower/Intrinsics/system-optional.f90
@@ -9,9 +9,10 @@ INTEGER, OPTIONAL :: exitstat
 call system(command, exitstat)
 
 ! CHECK-NEXT:    %[[cmdstatVal:.*]] = fir.alloca i16
+! CHECK-NEXT:    %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
 ! CHECK-NEXT:    %[[commandUnbox:.*]]:2 = fir.unboxchar %[[commandArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT:    %[[commandDeclare:.*]]:2 = hlfir.declare %[[commandUnbox]]#0 typeparams %[[commandUnbox]]#1 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_argsEcommand"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
-! CHECK-NEXT:    %[[exitstatDeclare:.*]]:2 = hlfir.declare %[[exitstatArg]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_argsEexitstat"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK-NEXT:    %[[commandDeclare:.*]]:2 = hlfir.declare %[[commandUnbox]]#0 typeparams %[[commandUnbox]]#1 dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_argsEcommand"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK-NEXT:    %[[exitstatDeclare:.*]]:2 = hlfir.declare %[[exitstatArg]] dummy_scope %[[DSCOPE]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFall_argsEexitstat"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK-NEXT:    %[[exitstatIsPresent:.*]] = fir.is_present %[[exitstatDeclare]]#0 : (!fir.ref<i32>) -> i1
 ! CHECK-NEXT:    %[[commandBox:.*]] = fir.embox %[[commandDeclare]]#1 typeparams %[[commandUnbox]]#1 : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
 ! CHECK-NEXT:    %[[exitstatBox:.*]] = fir.embox %[[exitstatDeclare]]#1 : (!fir.ref<i32>) -> !fir.box<i32>
diff --git a/flang/test/Lower/Intrinsics/system.f90 b/flang/test/Lower/Intrinsics/system.f90
index 0cafc0b2a9cf..4ce3c553891d 100644
--- a/flang/test/Lower/Intrinsics/system.f90
+++ b/flang/test/Lower/Intrinsics/system.f90
@@ -8,9 +8,10 @@ CHARACTER(*) :: command
 INTEGER :: exitstat
 call system(command, exitstat)
 ! CHECK-NEXT:   %[[cmdstatVal:.*]] = fir.alloca i16
+! CHECK-NEXT:   %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
 ! CHECK-NEXT:   %[[commandUnbox:.*]]:2 = fir.unboxchar %[[commandArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT:   %[[commandDeclare:.*]]:2 = hlfir.declare %[[commandUnbox]]#0 typeparams %[[commandUnbox]]#1 {uniq_name = "_QFall_argsEcommand"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
-! CHECK-NEXT:   %[[exitstatDeclare:.*]]:2 = hlfir.declare %[[exitstatArg]] {uniq_name = "_QFall_argsEexitstat"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK-NEXT:   %[[commandDeclare:.*]]:2 = hlfir.declare %[[commandUnbox]]#0 typeparams %[[commandUnbox]]#1 dummy_scope %[[DSCOPE]] {uniq_name = "_QFall_argsEcommand"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK-NEXT:   %[[exitstatDeclare:.*]]:2 = hlfir.declare %[[exitstatArg]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFall_argsEexitstat"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK-NEXT:   %[[commandBox:.*]] = fir.embox %[[commandDeclare]]#1 typeparams %[[commandUnbox]]#1 : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
 ! CHECK-NEXT:   %[[exitstatBox:.*]] = fir.embox %[[exitstatDeclare]]#1 : (!fir.ref<i32>) -> !fir.box<i32>
 ! CHECK-NEXT:   %[[true:.*]] = arith.constant true
@@ -34,8 +35,9 @@ subroutine only_command(command)
 CHARACTER(*) :: command
 call system(command)
 ! CHECK-NEXT:   %[[cmdstatVal:.*]] = fir.alloca i16
+! CHECK-NEXT:   %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
 ! CHECK-NEXT:   %[[commandUnbox:.*]]:2 = fir.unboxchar %arg0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT:   %[[commandDeclare:.*]]:2 = hlfir.declare %[[commandUnbox]]#0 typeparams %[[commandUnbox]]#1 {uniq_name = "_QFonly_commandEcommand"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK-NEXT:   %[[commandDeclare:.*]]:2 = hlfir.declare %[[commandUnbox]]#0 typeparams %[[commandUnbox]]#1 dummy_scope %[[DSCOPE]] {uniq_name = "_QFonly_commandEcommand"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK-NEXT:   %[[commandBox:.*]] = fir.embox %[[commandDeclare]]#1 typeparams %[[commandUnbox]]#1 : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
 ! CHECK-NEXT:   %[[true:.*]] = arith.constant true
 ! CHECK-NEXT:   %[[absentBox:.*]] = fir.absent !fir.box<none>
@@ -44,7 +46,7 @@ call system(command)
 ! CHECK-NEXT:   fir.store %[[c0_i16]] to %[[cmdstatVal]] : !fir.ref<i16>
 ! CHECK-NEXT:   %[[cmdstatBox:.*]] = fir.embox %[[cmdstatVal]] : (!fir.ref<i16>) -> !fir.box<i16>
 ! CHECK-NEXT:   %[[absentBox2:.*]] = fir.absent !fir.box<none>
-! CHECK:        %[[c35_i32:.*]] = arith.constant 35 : i32
+! CHECK:        %[[c35_i32:.*]] = arith.constant {{[0-9]+}} : i32
 ! CHECK-NEXT:   %[[command:.*]] = fir.convert %[[commandBox]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
 ! CHECK-NEXT:   %[[cmdstat:.*]] = fir.convert %[[cmdstatBox]] : (!fir.box<i16>) -> !fir.box<none>
 ! CHECK:        %[[VAL_12:.*]] = fir.call @_FortranAExecuteCommandLine(%[[command]], %[[true]], %[[absentBox]], %[[cmdstat]], %[[absentBox2]], %[[VAL_11:.*]], %[[c35_i32]]) fastmath<contract> : (!fir.box<none>, i1, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
diff --git a/flang/test/Lower/OpenACC/acc-atomic-update-array.f90 b/flang/test/Lower/OpenACC/acc-atomic-update-array.f90
index e36c39c830ec..eeb7ea299408 100644
--- a/flang/test/Lower/OpenACC/acc-atomic-update-array.f90
+++ b/flang/test/Lower/OpenACC/acc-atomic-update-array.f90
@@ -20,8 +20,8 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPatomic_update_array1(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "r"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}, %[[ARG2:.*]]: !fir.ref<f32> {fir.bindc_name = "x"}) {
-! CHECK: %[[DECL_ARG2:.*]]:2 = hlfir.declare %[[ARG2]] {uniq_name = "_QFatomic_update_array1Ex"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QFatomic_update_array1Er"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+! CHECK: %[[DECL_ARG2:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFatomic_update_array1Ex"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFatomic_update_array1Er"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
 ! CHECK: %[[ARRAY_REF:.*]] = hlfir.designate %[[DECL_ARG0]]#0 (%{{.*}})  : (!fir.box<!fir.array<?xf32>>, i64) -> !fir.ref<f32>
 ! CHECK: %[[LOAD_X:.*]] = fir.load %[[DECL_ARG2]]#0 : !fir.ref<f32>
 ! CHECK: acc.atomic.update %[[ARRAY_REF]] : !fir.ref<f32> {
@@ -42,8 +42,8 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPatomic_read_array1(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "r"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}, %[[ARG2:.*]]: !fir.ref<f32> {fir.bindc_name = "x"}) {
-! CHECK: %[[DECL_X:.*]]:2 = hlfir.declare %[[ARG2]] {uniq_name = "_QFatomic_read_array1Ex"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK: %[[DECL_R:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QFatomic_read_array1Er"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+! CHECK: %[[DECL_X:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFatomic_read_array1Ex"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %[[DECL_R:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFatomic_read_array1Er"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
 ! CHECK: %[[DES:.*]] = hlfir.designate %[[DECL_R]]#0 (%{{.*}})  : (!fir.box<!fir.array<?xf32>>, i64) -> !fir.ref<f32>
 ! CHECK: acc.atomic.read %[[DECL_X]]#1 = %[[DES]] : !fir.ref<f32>, f32
 
@@ -58,8 +58,8 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPatomic_write_array1(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "r"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}, %[[ARG2:.*]]: !fir.ref<f32> {fir.bindc_name = "x"}) {
-! CHECK: %[[DECL_X:.*]]:2 = hlfir.declare %[[ARG2]] {uniq_name = "_QFatomic_write_array1Ex"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK: %[[DECL_R:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QFatomic_write_array1Er"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+! CHECK: %[[DECL_X:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFatomic_write_array1Ex"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %[[DECL_R:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFatomic_write_array1Er"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
 ! CHECK: %[[DES:.*]] = hlfir.designate %[[DECL_R]]#0 (%{{.*}})  : (!fir.box<!fir.array<?xf32>>, i64) -> !fir.ref<f32>
 ! CHECK: %[[LOAD:.*]] = fir.load %[[DES]] : !fir.ref<f32> 
 ! CHECK: acc.atomic.write %[[DECL_X]]#1 = %[[LOAD]] : !fir.ref<f32>, f32
@@ -77,9 +77,9 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPatomic_capture_array1(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "r"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}, %[[ARG2:.*]]: !fir.ref<f32> {fir.bindc_name = "x"}, %[[ARG3:.*]]: !fir.ref<f32> {fir.bindc_name = "y"}) {
-! CHECK: %[[DECL_X:.*]]:2 = hlfir.declare %[[ARG2]] {uniq_name = "_QFatomic_capture_array1Ex"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK: %[[DECL_Y:.*]]:2 = hlfir.declare %[[ARG3]] {uniq_name = "_QFatomic_capture_array1Ey"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK: %[[DECL_R:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QFatomic_capture_array1Er"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+! CHECK: %[[DECL_X:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFatomic_capture_array1Ex"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %[[DECL_Y:.*]]:2 = hlfir.declare %[[ARG3]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFatomic_capture_array1Ey"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %[[DECL_R:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFatomic_capture_array1Er"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
 ! CHECK: %[[R_I:.*]] = hlfir.designate %[[DECL_R]]#0 (%{{.*}})  : (!fir.box<!fir.array<?xf32>>, i64) -> !fir.ref<f32>
 ! CHECK: %[[LOAD:.*]] = fir.load %[[DECL_X]]#0 : !fir.ref<f32>
 ! CHECK: acc.atomic.capture {
diff --git a/flang/test/Lower/OpenACC/acc-bounds.f90 b/flang/test/Lower/OpenACC/acc-bounds.f90
index c275d4f1b1d5..a83de91a67ae 100644
--- a/flang/test/Lower/OpenACC/acc-bounds.f90
+++ b/flang/test/Lower/OpenACC/acc-bounds.f90
@@ -88,7 +88,7 @@ contains
 
 ! CHECK-LABEL: func.func @_QMopenacc_boundsPacc_undefined_extent(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "a"}) {
-! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QMopenacc_boundsFacc_undefined_extentEa"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QMopenacc_boundsFacc_undefined_extentEa"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
 ! CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECL_ARG0]]#0, %c0{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
 ! CHECK: %[[UB:.*]] = arith.subi %[[DIMS0]]#1, %c1{{.*}} : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%[[UB]] : index) extent(%[[DIMS0]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%c1{{.*}} : index) {strideInBytes = true}
@@ -105,7 +105,7 @@ contains
 
 ! CHECK-LABEL: func.func @_QMopenacc_boundsPacc_multi_strides(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?x?x?xf32>> {fir.bindc_name = "a"})
-! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QMopenacc_boundsFacc_multi_stridesEa"} : (!fir.box<!fir.array<?x?x?xf32>>) -> (!fir.box<!fir.array<?x?x?xf32>>, !fir.box<!fir.array<?x?x?xf32>>)
+! CHECK: %[[DECL_ARG0:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMopenacc_boundsFacc_multi_stridesEa"} : (!fir.box<!fir.array<?x?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?x?xf32>>, !fir.box<!fir.array<?x?x?xf32>>)
 ! CHECK: %[[BOX_DIMS0:.*]]:3 = fir.box_dims %[[DECL_ARG0]]#0, %c0{{.*}} : (!fir.box<!fir.array<?x?x?xf32>>, index) -> (index, index, index)
 ! CHECK: %[[BOUNDS0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%[[BOX_DIMS0]]#1 : index) stride(%[[BOX_DIMS0]]#2 : index) startIdx(%{{.*}} : index) {strideInBytes = true}
 ! CHECK: %[[STRIDE1:.*]] = arith.muli %[[BOX_DIMS0]]#2, %[[BOX_DIMS0]]#1 : index
@@ -126,7 +126,7 @@ contains
   
 ! CHECK-LABEL: func.func @_QMopenacc_boundsPacc_optional_data(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {fir.bindc_name = "a", fir.optional}) {
-! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<optional, pointer>, uniq_name = "_QMopenacc_boundsFacc_optional_dataEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional, pointer>, uniq_name = "_QMopenacc_boundsFacc_optional_dataEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
 ! CHECK: %[[IS_PRESENT:.*]] = fir.is_present %[[ARG0_DECL]]#1 : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> i1
 ! CHECK: %[[BOX:.*]] = fir.if %[[IS_PRESENT]] -> (!fir.box<!fir.ptr<!fir.array<?xf32>>>) {
 ! CHECK:   %[[LOAD:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
@@ -162,8 +162,8 @@ contains
 
 ! CHECK-LABEL: func.func @_QMopenacc_boundsPacc_optional_data2(
 ! CHECK-SAME: %[[A:.*]]: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "a", fir.optional}, %[[N:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
-! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[A]](%{{.*}}) {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QMopenacc_boundsFacc_optional_data2Ea"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
-! CHECK: %[[NO_CREATE:.*]] = acc.nocreate varPtr(%[[DECL_A]]#1 : !fir.ref<!fir.array<?xf32>>) bounds(%10) -> !fir.ref<!fir.array<?xf32>> {name = "a"}
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[A]](%{{.*}}) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QMopenacc_boundsFacc_optional_data2Ea"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+! CHECK: %[[NO_CREATE:.*]] = acc.nocreate varPtr(%[[DECL_A]]#1 : !fir.ref<!fir.array<?xf32>>) bounds(%{{[0-9]+}}) -> !fir.ref<!fir.array<?xf32>> {name = "a"}
 ! CHECK: acc.data dataOperands(%[[NO_CREATE]] : !fir.ref<!fir.array<?xf32>>) {
 
   subroutine acc_optional_data3(a, n)
@@ -175,7 +175,7 @@ contains
 
 ! CHECK-LABEL: func.func @_QMopenacc_boundsPacc_optional_data3(
 ! CHECK-SAME: %[[A:.*]]: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "a", fir.optional}, %[[N:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
-! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[A]](%{{.*}}) {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QMopenacc_boundsFacc_optional_data3Ea"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[A]](%{{.*}}) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QMopenacc_boundsFacc_optional_data3Ea"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
 ! CHECK: %[[PRES:.*]] = fir.is_present %[[DECL_A]]#1 : (!fir.ref<!fir.array<?xf32>>) -> i1
 ! CHECK: %[[STRIDE:.*]] = fir.if %[[PRES]] -> (index) {
 ! CHECK:   %[[DIMS:.*]]:3 = fir.box_dims %[[DECL_A]]#0, %c0{{.*}} : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
diff --git a/flang/test/Lower/OpenACC/acc-declare.f90 b/flang/test/Lower/OpenACC/acc-declare.f90
index 5d3f9e3fe97e..ff1e756c20e1 100644
--- a/flang/test/Lower/OpenACC/acc-declare.f90
+++ b/flang/test/Lower/OpenACC/acc-declare.f90
@@ -62,7 +62,7 @@ module acc_declare
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_present(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"})
 ! CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
-! CHECK-DAG: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_present>, uniq_name = "_QMacc_declareFacc_declare_presentEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK-DAG: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_present>, uniq_name = "_QMacc_declareFacc_declare_presentEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) startIdx(%[[C1]] : index)
 ! CHECK: %[[PRESENT:.*]] = acc.present varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
 ! CHECK: acc.declare_enter dataOperands(%[[PRESENT]] : !fir.ref<!fir.array<100xi32>>)
@@ -119,7 +119,7 @@ module acc_declare
 
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_deviceptr(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}) {
-! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_deviceptr>, uniq_name = "_QMacc_declareFacc_declare_deviceptrEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_deviceptr>, uniq_name = "_QMacc_declareFacc_declare_deviceptrEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %[[DEVICEPTR:.*]] = acc.deviceptr varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
 ! CHECK: acc.declare_enter dataOperands(%[[DEVICEPTR]] : !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
@@ -135,7 +135,7 @@ module acc_declare
 
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_link(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"})
-! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_declare_link>, uniq_name = "_QMacc_declareFacc_declare_linkEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_declare_link>, uniq_name = "_QMacc_declareFacc_declare_linkEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %[[LINK:.*]] = acc.declare_link varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
 ! CHECK: acc.declare_enter dataOperands(%[[LINK]] : !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
@@ -151,7 +151,7 @@ module acc_declare
 
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_device_resident(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"})
-! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_declare_device_resident>, uniq_name = "_QMacc_declareFacc_declare_device_residentEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_declare_device_resident>, uniq_name = "_QMacc_declareFacc_declare_device_residentEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %[[DEVICERES:.*]] = acc.declare_device_resident varPtr(%[[DECL]]#0 : !fir.ref<!fir.array<100xi32>>)   bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {name = "a"}
 ! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[DEVICERES]] : !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %{{.*}}:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%arg{{.*}} = %{{.*}}) -> (index, i32)
@@ -220,12 +220,12 @@ module acc_declare
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_in_func2(%arg0: !fir.ref<i32> {fir.bindc_name = "i"}) -> f32 {
 ! CHECK: %[[ALLOCA_A:.*]] = fir.alloca !fir.array<1024xf32> {bindc_name = "a", uniq_name = "_QMacc_declareFacc_declare_in_func2Ea"}
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ALLOCA_A]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_create>, uniq_name = "_QMacc_declareFacc_declare_in_func2Ea"} : (!fir.ref<!fir.array<1024xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1024xf32>>, !fir.ref<!fir.array<1024xf32>>)
-! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<1024xf32>>) bounds(%7) -> !fir.ref<!fir.array<1024xf32>> {name = "a"}
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<1024xf32>>) bounds(%{{[0-9]+}}) -> !fir.ref<!fir.array<1024xf32>> {name = "a"}
 ! CHECK: %[[TOKEN:.*]] = acc.declare_enter dataOperands(%[[CREATE]] : !fir.ref<!fir.array<1024xf32>>)
 ! CHECK:   cf.br ^bb1
 ! CHECK: ^bb1:
 ! CHECK: acc.declare_exit token(%[[TOKEN]]) dataOperands(%[[CREATE]] : !fir.ref<!fir.array<1024xf32>>)
-! CHECK: acc.delete accPtr(%[[CREATE]] : !fir.ref<!fir.array<1024xf32>>) bounds(%7) {dataClause = #acc<data_clause acc_create>, name = "a"}
+! CHECK: acc.delete accPtr(%[[CREATE]] : !fir.ref<!fir.array<1024xf32>>) bounds(%{{[0-9]+}}) {dataClause = #acc<data_clause acc_create>, name = "a"}
 ! CHECK:   return %{{.*}} : f32
 ! CHECK: }
 
@@ -294,8 +294,8 @@ module acc_declare
 
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_multiple_directive(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<!fir.array<100xi32>> {fir.bindc_name = "b"}) {
-! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_copy>, uniq_name = "_QMacc_declareFacc_declare_multiple_directiveEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
-! CHECK: %[[DECL_B:.*]]:2 = hlfir.declare %[[ARG1]](%{{.*}}) {acc.declare = #acc.declare<dataClause =  acc_copyout>, uniq_name = "_QMacc_declareFacc_declare_multiple_directiveEb"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_copy>, uniq_name = "_QMacc_declareFacc_declare_multiple_directiveEa"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
+! CHECK: %[[DECL_B:.*]]:2 = hlfir.declare %[[ARG1]](%{{.*}}) dummy_scope %{{[0-9]+}} {acc.declare = #acc.declare<dataClause =  acc_copyout>, uniq_name = "_QMacc_declareFacc_declare_multiple_directiveEb"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[DECL_A]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DECL_B]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<100xi32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
 ! CHECK: acc.declare_enter dataOperands(%[[COPYIN]], %[[CREATE]] : !fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
@@ -316,7 +316,7 @@ module acc_declare
 
 ! CHECK-LABEL: func.func @_QMacc_declarePacc_declare_array_section(
 ! CHECK-SAME:    %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}) {
-! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QMacc_declareFacc_declare_array_sectionEa"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMacc_declareFacc_declare_array_sectionEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 {acc.declare = #acc.declare<dataClause =  acc_copy>} : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
 ! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {dataClause = #acc<data_clause acc_copy>, name = "a(1:10)"}
 ! CHECK: acc.declare_enter dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<?xi32>>)
diff --git a/flang/test/Lower/OpenACC/acc-loop-exit.f90 b/flang/test/Lower/OpenACC/acc-loop-exit.f90
index c1ea057af667..85394e4a5b74 100644
--- a/flang/test/Lower/OpenACC/acc-loop-exit.f90
+++ b/flang/test/Lower/OpenACC/acc-loop-exit.f90
@@ -14,9 +14,9 @@ subroutine sub1(x, a)
 end 
 
 ! CHECK-LABEL: func.func @_QPsub1
-! CHECK: %[[A:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFsub1Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK: %[[I:.*]]:2 = hlfir.declare %2 {uniq_name = "_QFsub1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK: %[[I:.*]]:2 = hlfir.declare %6 {uniq_name = "_QFsub1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[A:.*]]:2 = hlfir.declare %arg1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFsub1Ea"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[I:.*]]:2 = hlfir.declare %{{[0-9]+}} {uniq_name = "_QFsub1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[I:.*]]:2 = hlfir.declare %{{[0-9]+}} {uniq_name = "_QFsub1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: %[[EXIT_COND:.*]] = acc.loop
 ! CHECK: ^bb{{.*}}:
 ! CHECK: ^bb{{.*}}:
diff --git a/flang/test/Lower/OpenACC/acc-private.f90 b/flang/test/Lower/OpenACC/acc-private.f90
index 4d9f84b1fa74..a299a7486c3b 100644
--- a/flang/test/Lower/OpenACC/acc-private.f90
+++ b/flang/test/Lower/OpenACC/acc-private.f90
@@ -271,7 +271,7 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_private_assumed_shape(
 ! CHECK-SAME:    %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}
-! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QFacc_private_assumed_shapeEa"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_assumed_shapeEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
 ! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
@@ -293,7 +293,7 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_private_allocatable_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a"}
-! CHECK: %[[DECLA_A:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_private_allocatable_arrayEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+! CHECK: %[[DECLA_A:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_private_allocatable_arrayEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
@@ -313,7 +313,7 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_private_pointer_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}) {
-! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_private_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %arg0 dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_private_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !fir.ptr<!fir.array<?xi32>>
@@ -332,8 +332,8 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_private_dynamic_extent(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?x?x2xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
-! CHECK: %[[DECL_N:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFacc_private_dynamic_extentEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QFacc_private_dynamic_extentEa"} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
+! CHECK: %[[DECL_N:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_dynamic_extentEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_private_dynamic_extentEa"} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>, !fir.dscope) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
 ! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?x?x2xi32>>) -> !fir.ref<!fir.array<?x?x2xi32>>
 ! CHECK: %[[PRIV:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?x?x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?x2xi32>> {name = "a"}
diff --git a/flang/test/Lower/OpenACC/acc-reduction.f90 b/flang/test/Lower/OpenACC/acc-reduction.f90
index 6918bc1ec7d6..545c4f217577 100644
--- a/flang/test/Lower/OpenACC/acc-reduction.f90
+++ b/flang/test/Lower/OpenACC/acc-reduction.f90
@@ -1162,7 +1162,7 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_dynamic_extent_add_with_section(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"})
-! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QFacc_reduction_add_dynamic_extent_add_with_sectionEa"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_reduction_add_dynamic_extent_add_with_sectionEa"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c1{{.*}} : index) upperbound(%c3{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}} : index) {strideInBytes = true}
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL]]#0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
 ! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xi32>> {name = "a(2:4)"}
@@ -1176,11 +1176,11 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_allocatable(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {fir.bindc_name = "a"})
-! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_reduction_add_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_reduction_add_allocatableEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true}
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
-! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>)   bounds(%6) -> !fir.heap<!fir.array<?xf32>> {name = "a"}
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>)   bounds(%{{[0-9]+}}) -> !fir.heap<!fir.array<?xf32>> {name = "a"}
 ! CHECK: acc.parallel reduction(@reduction_max_box_heap_Uxf32 -> %[[RED]] : !fir.heap<!fir.array<?xf32>>)
 
 subroutine acc_reduction_add_pointer_array(a)
@@ -1191,7 +1191,7 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_add_pointer_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> {fir.bindc_name = "a"})
-! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_reduction_add_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_reduction_add_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true}
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.ptr<!fir.array<?xf32>>
@@ -1207,7 +1207,7 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_reduction_max_dynamic_extent_max(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "a"}, %{{.*}}: !fir.ref<i32> {fir.bindc_name = "n"})
-! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QFacc_reduction_max_dynamic_extent_maxEa"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QFacc_reduction_max_dynamic_extent_maxEa"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
 ! CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?x?xf32>>) -> !fir.ref<!fir.array<?x?xf32>>
 ! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[ADDR]] : !fir.ref<!fir.array<?x?xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?xf32>> {name = "a"}
 ! CHECK: acc.parallel reduction(@reduction_max_box_UxUxf32 -> %[[RED]] : !fir.ref<!fir.array<?x?xf32>>)
diff --git a/flang/test/Lower/OpenMP/allocatable-array-bounds.f90 b/flang/test/Lower/OpenMP/allocatable-array-bounds.f90
index aeb56a0427e3..14a50692bc60 100644
--- a/flang/test/Lower/OpenMP/allocatable-array-bounds.f90
+++ b/flang/test/Lower/OpenMP/allocatable-array-bounds.f90
@@ -64,7 +64,7 @@ module assumed_allocatable_array_routines
 
 !HOST-LABEL: func.func @_QMassumed_allocatable_array_routinesPassumed_shape_array(
 
-!HOST: %[[DECLARE:.*]]:2 = hlfir.declare %[[ARG:.*]] {fortran_attrs = #fir.var_attrs<allocatable, intent_inout>, uniq_name = "_QMassumed_allocatable_array_routinesFassumed_shape_arrayEarr_read_write"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+!HOST: %[[DECLARE:.*]]:2 = hlfir.declare %[[ARG:.*]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable, intent_inout>, uniq_name = "_QMassumed_allocatable_array_routinesFassumed_shape_arrayEarr_read_write"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 !HOST: %[[LOAD_1:.*]] = fir.load %[[DECLARE]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 !HOST: %[[LOAD_2:.*]] = fir.load %[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 !HOST: %[[CONSTANT_1:.*]] = arith.constant 0 : index
diff --git a/flang/test/Lower/OpenMP/array-bounds.f90 b/flang/test/Lower/OpenMP/array-bounds.f90
index 2c8a8999a2cc..8c9d8944bf33 100644
--- a/flang/test/Lower/OpenMP/array-bounds.f90
+++ b/flang/test/Lower/OpenMP/array-bounds.f90
@@ -41,7 +41,7 @@ module assumed_array_routines
 !HOST-LABEL: func.func @_QMassumed_array_routinesPassumed_shape_array(
 !HOST-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "arr_read_write"}) {
 !HOST: %[[INTERMEDIATE_ALLOCA:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
-!HOST: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QMassumed_array_routinesFassumed_shape_arrayEarr_read_write"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+!HOST: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QMassumed_array_routinesFassumed_shape_arrayEarr_read_write"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 !HOST: %[[C0:.*]] = arith.constant 1 : index
 !HOST: %[[C1:.*]] = arith.constant 0 : index
 !HOST: %[[DIMS0:.*]]:3 = fir.box_dims %[[ARG0_DECL]]#0, %[[C1]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
@@ -69,7 +69,7 @@ module assumed_array_routines
 !HOST-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?xi32>> {fir.bindc_name = "arr_read_write"}) {
 !HOST: %[[INTERMEDIATE_ALLOCA:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
 !HOST: %[[ARG0_SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
-!HOST: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]](%[[ARG0_SHAPE]]) {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QMassumed_array_routinesFassumed_size_arrayEarr_read_write"} : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
+!HOST: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]](%[[ARG0_SHAPE]]) dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QMassumed_array_routinesFassumed_size_arrayEarr_read_write"} : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
 !HOST: %[[ALLOCA:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QMassumed_array_routinesFassumed_size_arrayEi"}
 !HOST: %[[DIMS0:.*]]:3 = fir.box_dims %[[ARG0_DECL]]#0, %c0{{.*}} : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
 !HOST: %[[C4_1:.*]] = arith.subi %c4, %c1{{.*}} : index
diff --git a/flang/test/Lower/OpenMP/flush.f90 b/flang/test/Lower/OpenMP/flush.f90
index ad2e3609ebef..8438fdba4ee4 100644
--- a/flang/test/Lower/OpenMP/flush.f90
+++ b/flang/test/Lower/OpenMP/flush.f90
@@ -7,9 +7,9 @@
 subroutine flush_standalone(a, b, c)
     integer, intent(inout) :: a, b, c
 
-!CHECK:    %[[A:.*]]:2 = hlfir.declare %[[ARG_A]] {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_standaloneEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:    %[[B:.*]]:2 = hlfir.declare %[[ARG_B]] {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_standaloneEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:    %[[C:.*]]:2 = hlfir.declare %[[ARG_C]] {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_standaloneEc"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[A:.*]]:2 = hlfir.declare %[[ARG_A]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_standaloneEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[B:.*]]:2 = hlfir.declare %[[ARG_B]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_standaloneEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[C:.*]]:2 = hlfir.declare %[[ARG_C]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_standaloneEc"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    omp.flush(%[[A]]#1, %[[B]]#1, %[[C]]#1 : !fir.ref<i32>, !fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    omp.flush
 !$omp flush(a,b,c)
@@ -21,9 +21,9 @@ end subroutine flush_standalone
 !CHECK-SAME: %[[ARG_A:.*]]: !fir.ref<i32> {fir.bindc_name = "a"}, %[[ARG_B:.*]]: !fir.ref<i32> {fir.bindc_name = "b"}, %[[ARG_C:.*]]: !fir.ref<i32> {fir.bindc_name = "c"})
 subroutine flush_parallel(a, b, c)
     integer, intent(inout) :: a, b, c
-!CHECK:    %[[A:.*]]:2 = hlfir.declare %[[ARG_A]] {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_parallelEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:    %[[B:.*]]:2 = hlfir.declare %[[ARG_B]] {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_parallelEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:    %[[C:.*]]:2 = hlfir.declare %[[ARG_C]] {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_parallelEc"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[A:.*]]:2 = hlfir.declare %[[ARG_A]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_parallelEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[B:.*]]:2 = hlfir.declare %[[ARG_B]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_parallelEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[C:.*]]:2 = hlfir.declare %[[ARG_C]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFflush_parallelEc"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 
 !$omp parallel
 !CHECK:    omp.parallel
@@ -34,7 +34,7 @@ subroutine flush_parallel(a, b, c)
 
 !CHECK:      %[[A_VAL:.*]] = fir.load %[[A]]#0 : !fir.ref<i32>
 !CHECK:      %[[B_VAL:.*]] = fir.load %[[B]]#0 : !fir.ref<i32>
-!CHECK:      %[[C_VAL:.*]] = arith.addi %3, %4 : i32
+!CHECK:      %[[C_VAL:.*]] = arith.addi %[[A_VAL]], %[[B_VAL]] : i32
 !CHECK:      hlfir.assign %[[C_VAL]] to %[[C]]#0 : i32, !fir.ref<i32>
     c = a + b
 
diff --git a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
index 6402f98a2add..93dcd4b74b00 100644
--- a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
@@ -4,8 +4,8 @@
 ! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s --check-prefix=CHECK
 
 !CHECK-DAG: func @_QPfirstprivate_complex(%[[ARG1:.*]]: !fir.ref<!fir.complex<4>>{{.*}}, %[[ARG2:.*]]: !fir.ref<!fir.complex<8>>{{.*}}) {
-!CHECK:    %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
-!CHECK:    %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<!fir.complex<8>>) -> (!fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>)
+!CHECK:    %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<!fir.complex<4>>, !fir.dscope) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+!CHECK:    %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<!fir.complex<8>>, !fir.dscope) -> (!fir.ref<!fir.complex<8>>, !fir.ref<!fir.complex<8>>)
 !CHECK:   omp.parallel {
 !CHECK:     %[[ARG1_PVT:.*]] = fir.alloca !fir.complex<4> {bindc_name = "arg1", pinned, uniq_name = "_QFfirstprivate_complexEarg1"}
 !CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
@@ -30,12 +30,12 @@ subroutine firstprivate_complex(arg1, arg2)
 end subroutine
 
 !CHECK-DAG: func @_QPfirstprivate_integer(%[[ARG1:.*]]: !fir.ref<i32>{{.*}}, %[[ARG2:.*]]: !fir.ref<i8>{{.*}}, %[[ARG3:.*]]: !fir.ref<i16>{{.*}}, %[[ARG4:.*]]: !fir.ref<i32>{{.*}}, %[[ARG5:.*]]: !fir.ref<i64>{{.*}}, %[[ARG6:.*]]: !fir.ref<i128>{{.*}}) {
-!CHECK:  %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFfirstprivate_integerEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:  %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] {uniq_name = "_QFfirstprivate_integerEarg2"} : (!fir.ref<i8>) -> (!fir.ref<i8>, !fir.ref<i8>)
-!CHECK:  %[[ARG3_DECL:.*]]:2 = hlfir.declare %[[ARG3]] {uniq_name = "_QFfirstprivate_integerEarg3"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
-!CHECK:  %[[ARG4_DECL:.*]]:2 = hlfir.declare %[[ARG4]] {uniq_name = "_QFfirstprivate_integerEarg4"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:  %[[ARG5_DECL:.*]]:2 = hlfir.declare %[[ARG5]] {uniq_name = "_QFfirstprivate_integerEarg5"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
-!CHECK:  %[[ARG6_DECL:.*]]:2 = hlfir.declare %[[ARG6]] {uniq_name = "_QFfirstprivate_integerEarg6"} : (!fir.ref<i128>) -> (!fir.ref<i128>, !fir.ref<i128>)
+!CHECK:  %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_integerEarg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:  %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_integerEarg2"} : (!fir.ref<i8>, !fir.dscope) -> (!fir.ref<i8>, !fir.ref<i8>)
+!CHECK:  %[[ARG3_DECL:.*]]:2 = hlfir.declare %[[ARG3]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_integerEarg3"} : (!fir.ref<i16>, !fir.dscope) -> (!fir.ref<i16>, !fir.ref<i16>)
+!CHECK:  %[[ARG4_DECL:.*]]:2 = hlfir.declare %[[ARG4]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_integerEarg4"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:  %[[ARG5_DECL:.*]]:2 = hlfir.declare %[[ARG5]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_integerEarg5"} : (!fir.ref<i64>, !fir.dscope) -> (!fir.ref<i64>, !fir.ref<i64>)
+!CHECK:  %[[ARG6_DECL:.*]]:2 = hlfir.declare %[[ARG6]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_integerEarg6"} : (!fir.ref<i128>, !fir.dscope) -> (!fir.ref<i128>, !fir.ref<i128>)
 !CHECK:  omp.parallel {
 !CHECK:    %[[ARG1_PVT:.*]] = fir.alloca i32 {bindc_name = "arg1", pinned, uniq_name = "_QFfirstprivate_integerEarg1"}
 !CHECK:    %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_integerEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -81,11 +81,11 @@ subroutine firstprivate_integer(arg1, arg2, arg3, arg4, arg5, arg6)
 end subroutine
 
 !CHECK-DAG: func @_QPfirstprivate_logical(%[[ARG1:.*]]: !fir.ref<!fir.logical<4>>{{.*}}, %[[ARG2:.*]]: !fir.ref<!fir.logical<1>>{{.*}}, %[[ARG3:.*]]: !fir.ref<!fir.logical<2>>{{.*}}, %[[ARG4:.*]]: !fir.ref<!fir.logical<4>>{{.*}}, %[[ARG5:.*]]: !fir.ref<!fir.logical<8>>{{.*}}) {
-!CHECK:    %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFfirstprivate_logicalEarg1"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-!CHECK:    %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] {uniq_name = "_QFfirstprivate_logicalEarg2"} : (!fir.ref<!fir.logical<1>>) -> (!fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<1>>)
-!CHECK:    %[[ARG3_DECL:.*]]:2 = hlfir.declare %[[ARG3]] {uniq_name = "_QFfirstprivate_logicalEarg3"} : (!fir.ref<!fir.logical<2>>) -> (!fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<2>>)
-!CHECK:    %[[ARG4_DECL:.*]]:2 = hlfir.declare %[[ARG4]] {uniq_name = "_QFfirstprivate_logicalEarg4"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-!CHECK:    %[[ARG5_DECL:.*]]:2 = hlfir.declare %[[ARG5]] {uniq_name = "_QFfirstprivate_logicalEarg5"} : (!fir.ref<!fir.logical<8>>) -> (!fir.ref<!fir.logical<8>>, !fir.ref<!fir.logical<8>>)
+!CHECK:    %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_logicalEarg1"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+!CHECK:    %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_logicalEarg2"} : (!fir.ref<!fir.logical<1>>, !fir.dscope) -> (!fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<1>>)
+!CHECK:    %[[ARG3_DECL:.*]]:2 = hlfir.declare %[[ARG3]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_logicalEarg3"} : (!fir.ref<!fir.logical<2>>, !fir.dscope) -> (!fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<2>>)
+!CHECK:    %[[ARG4_DECL:.*]]:2 = hlfir.declare %[[ARG4]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_logicalEarg4"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+!CHECK:    %[[ARG5_DECL:.*]]:2 = hlfir.declare %[[ARG5]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_logicalEarg5"} : (!fir.ref<!fir.logical<8>>, !fir.dscope) -> (!fir.ref<!fir.logical<8>>, !fir.ref<!fir.logical<8>>)
 !CHECK:   omp.parallel {
 !CHECK:     %[[ARG1_PVT:.*]] = fir.alloca !fir.logical<4> {bindc_name = "arg1", pinned, uniq_name = "_QFfirstprivate_logicalEarg1"}
 !CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg1"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -125,12 +125,12 @@ subroutine firstprivate_logical(arg1, arg2, arg3, arg4, arg5)
 end subroutine
 
 !CHECK-DAG: func @_QPfirstprivate_real(%[[ARG1:.*]]: !fir.ref<f32>{{.*}}, %[[ARG2:.*]]: !fir.ref<f16>{{.*}}, %[[ARG3:.*]]: !fir.ref<f32>{{.*}}, %[[ARG4:.*]]: !fir.ref<f64>{{.*}}, %[[ARG5:.*]]: !fir.ref<f80>{{.*}}, %[[ARG6:.*]]: !fir.ref<f128>{{.*}}) {
-!CHECK:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFfirstprivate_realEarg1"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-!CHECK:   %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] {uniq_name = "_QFfirstprivate_realEarg2"} : (!fir.ref<f16>) -> (!fir.ref<f16>, !fir.ref<f16>)
-!CHECK:   %[[ARG3_DECL:.*]]:2 = hlfir.declare %[[ARG3]] {uniq_name = "_QFfirstprivate_realEarg3"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-!CHECK:   %[[ARG4_DECL:.*]]:2 = hlfir.declare %[[ARG4]] {uniq_name = "_QFfirstprivate_realEarg4"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
-!CHECK:   %[[ARG5_DECL:.*]]:2 = hlfir.declare %[[ARG5]] {uniq_name = "_QFfirstprivate_realEarg5"} : (!fir.ref<f80>) -> (!fir.ref<f80>, !fir.ref<f80>)
-!CHECK:   %[[ARG6_DECL:.*]]:2 = hlfir.declare %[[ARG6]] {uniq_name = "_QFfirstprivate_realEarg6"} : (!fir.ref<f128>) -> (!fir.ref<f128>, !fir.ref<f128>)
+!CHECK:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg1"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+!CHECK:   %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg2"} : (!fir.ref<f16>, !fir.dscope) -> (!fir.ref<f16>, !fir.ref<f16>)
+!CHECK:   %[[ARG3_DECL:.*]]:2 = hlfir.declare %[[ARG3]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg3"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+!CHECK:   %[[ARG4_DECL:.*]]:2 = hlfir.declare %[[ARG4]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg4"} : (!fir.ref<f64>, !fir.dscope) -> (!fir.ref<f64>, !fir.ref<f64>)
+!CHECK:   %[[ARG5_DECL:.*]]:2 = hlfir.declare %[[ARG5]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg5"} : (!fir.ref<f80>, !fir.dscope) -> (!fir.ref<f80>, !fir.ref<f80>)
+!CHECK:   %[[ARG6_DECL:.*]]:2 = hlfir.declare %[[ARG6]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_realEarg6"} : (!fir.ref<f128>, !fir.dscope) -> (!fir.ref<f128>, !fir.ref<f128>)
 !CHECK:   omp.parallel {
 !CHECK:     %[[ARG1_PVT:.*]] = fir.alloca f32 {bindc_name = "arg1", pinned, uniq_name = "_QFfirstprivate_realEarg1"}
 !CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_realEarg1"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
@@ -177,8 +177,8 @@ end subroutine
 !CHECK-LABEL:   func.func @_QPmultiple_firstprivate(
 !CHECK-SAME:                                        %[[A_ADDR:.*]]: !fir.ref<i32> {fir.bindc_name = "a"},
 !CHECK-SAME:                                        %[[B_ADDR:.*]]: !fir.ref<i32> {fir.bindc_name = "b"}) {
-!CHECK:           %[[A_DECL:.*]]:2 = hlfir.declare %[[A_ADDR]] {uniq_name = "_QFmultiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:           %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ADDR]] {uniq_name = "_QFmultiple_firstprivateEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[A_DECL:.*]]:2 = hlfir.declare %[[A_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmultiple_firstprivateEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:           %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmultiple_firstprivateEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:           omp.parallel   {
 !CHECK:             %[[A_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFmultiple_firstprivateEa"}
 !CHECK:             %[[A_PRIV_DECL:.*]]:2 = hlfir.declare %[[A_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
index bb81e5eac62f..b7f11c8c722f 100644
--- a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
@@ -7,7 +7,7 @@
 !CHECK-DAG: %[[ARG1_UNBOX:.*]]:2 = fir.unboxchar
 !CHECK-DAG: %[[FIVE:.*]] = arith.constant 5 : index
 !CHECK-DAG: %[[ARG1_REF:.*]] = fir.convert %[[ARG1_UNBOX]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,5>>
-!CHECK-DAG: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1_REF]] typeparams %[[FIVE]] {uniq_name = "_QFlastprivate_characterEarg1"} : (!fir.ref<!fir.char<1,5>>, index) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
+!CHECK-DAG: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1_REF]] typeparams %[[FIVE]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFlastprivate_characterEarg1"} : (!fir.ref<!fir.char<1,5>>, index, !fir.dscope) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
 
 !CHECK: omp.parallel {
 !CHECK-DAG: %[[ARG1_PVT:.*]] = fir.alloca !fir.char<1,5> {bindc_name = "arg1",
@@ -57,7 +57,7 @@ end do
 end subroutine
 
 !CHECK: func @_QPlastprivate_int(%[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "arg1"}) {
-!CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFlastprivate_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFlastprivate_intEarg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK-DAG: omp.parallel  {
 !CHECK-DAG: %[[CLONE:.*]] = fir.alloca i32 {bindc_name = "arg1"
 !CHECK-DAG: %[[CLONE_DECL:.*]]:2 = hlfir.declare %[[CLONE]] {uniq_name = "_QFlastprivate_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -97,8 +97,8 @@ print *, arg1
 end subroutine
 
 !CHECK: func.func @_QPmult_lastprivate_int(%[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "arg1"}, %[[ARG2:.*]]: !fir.ref<i32> {fir.bindc_name = "arg2"}) {
-!CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFmult_lastprivate_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] {uniq_name = "_QFmult_lastprivate_intEarg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_intEarg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_intEarg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
 !CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
 !CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -142,8 +142,8 @@ print *, arg1, arg2
 end subroutine
 
 !CHECK: func.func @_QPmult_lastprivate_int2(%[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "arg1"}, %[[ARG2:.*]]: !fir.ref<i32> {fir.bindc_name = "arg2"}) {
-!CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %arg0 {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[ARG2_DECL:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
 !CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
 !CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -187,8 +187,8 @@ print *, arg1, arg2
 end subroutine
 
 !CHECK: func.func @_QPfirstpriv_lastpriv_int(%[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "arg1"}, %[[ARG2:.*]]: !fir.ref<i32> {fir.bindc_name = "arg2"}) {
-!CHECK:    %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFfirstpriv_lastpriv_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:    %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] {uniq_name = "_QFfirstpriv_lastpriv_intEarg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstpriv_lastpriv_intEarg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstpriv_lastpriv_intEarg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
 ! Firstprivate update
 !CHECK: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
@@ -235,7 +235,7 @@ print *, arg1, arg2
 end subroutine
 
 !CHECK: func.func @_QPfirstpriv_lastpriv_int2(%[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "arg1"}) {
-!CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFfirstpriv_lastpriv_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstpriv_lastpriv_int2Earg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
 ! Firstprivate update
 !CHECK: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
index f8343338112c..d3843c8e241a 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
@@ -4,7 +4,7 @@
 
 ! CHECK-LABEL: multiple_private_fix
 ! CHECK-SAME:  %[[GAMA:.*]]: !fir.ref<i32> {fir.bindc_name = "gama"}
-! CHECK-DAG:         %[[GAMA_DECL:.*]]:2 = hlfir.declare %[[GAMA]] {uniq_name = "_QFmultiple_private_fixEgama"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK-DAG:         %[[GAMA_DECL:.*]]:2 = hlfir.declare %[[GAMA]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmultiple_private_fixEgama"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK-DAG:         %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_private_fixEi"}
 ! CHECK-DAG:         %[[I_DECL:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmultiple_private_fixEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK-DAG:         %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFmultiple_private_fixEj"}
@@ -99,7 +99,7 @@ end subroutine
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>> {fir.bindc_name = "aaa"}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
 ! CHECK:           %[[VAL_2:.*]] = fir.box_elesize %[[VAL_1]] : (!fir.box<!fir.heap<!fir.char<1,?>>>) -> index
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_2]] {fortran_attrs = #{{.*}}<allocatable>, uniq_name = "_QFsub01Eaaa"} : (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, index) -> (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_2]] dummy_scope %{{[0-9]+}} {fortran_attrs = #{{.*}}<allocatable>, uniq_name = "_QFsub01Eaaa"} : (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, index, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_4:.*]] = fir.alloca !fir.box<!fir.heap<!fir.char<1,?>>> {bindc_name = "aaa", pinned, uniq_name = "_QFsub01Eaaa"}
 ! CHECK:             %[[VAL_5:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
@@ -148,7 +148,7 @@ end subroutine
 
 ! CHECK-LABEL:   func.func @_QPsub02(
 ! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>> {fir.bindc_name = "bbb"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #{{.*}}<allocatable>, uniq_name = "_QFsub02Ebbb"} : (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #{{.*}}<allocatable>, uniq_name = "_QFsub02Ebbb"} : (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.heap<!fir.char<1,?>>> {bindc_name = "bbb", pinned, uniq_name = "_QFsub02Ebbb"}
 ! CHECK:             %[[VAL_3:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.char<1,?>>>>
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause-str.f90 b/flang/test/Lower/OpenMP/parallel-private-clause-str.f90
index 025e51e06617..19ea37c5339b 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause-str.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause-str.f90
@@ -30,8 +30,8 @@ subroutine test_allocatable_string(n)
   !$omp end parallel
 end subroutine
 
-!CHECK:  func.func @_QPtest_allocatable_string_array(%{{.*}}: !fir.ref<i32> {fir.bindc_name = "n"}) {
-!CHECK:    %0:2 = hlfir.declare %arg0 {uniq_name = "_QFtest_allocatable_string_arrayEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:  func.func @_QPtest_allocatable_string_array(%[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
+!CHECK:    %{{.*}} = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_allocatable_string_arrayEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[C_BOX_REF:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>> {bindc_name = "c", uniq_name = "_QFtest_allocatable_string_arrayEc"}
 !CHECK:    %[[C_BOX:.*]] = fir.embox %{{.*}}(%{{.*}}) typeparams %{{.*}} : (!fir.heap<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, i32) -> !fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>
 !CHECK:    fir.store %[[C_BOX]] to %[[C_BOX_REF]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,?>>>>>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction3.f90 b/flang/test/Lower/OpenMP/parallel-reduction3.f90
index 2a4e338f255e..17d805c0d142 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction3.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction3.f90
@@ -52,7 +52,7 @@
 
 ! CHECK-LABEL:   func.func @_QPs(
 ! CHECK-SAME:                    %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"}) {
-! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFsEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFsEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsEi"}
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
index ac8b9f50f54e..c32eb2400a34 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
@@ -5,7 +5,7 @@
 
 ! CHECK: func @_QPomp_do_firstprivate(%[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"}) 
 subroutine omp_do_firstprivate(a)
-  ! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QFomp_do_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_do_firstprivateEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   integer::a
   integer::n
   n = a+1
@@ -38,8 +38,8 @@ end subroutine omp_do_firstprivate
 
 ! CHECK: func @_QPomp_do_firstprivate2(%[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) 
 subroutine omp_do_firstprivate2(a, n)
-  ! CHECK:  %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QFomp_do_firstprivate2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  ! CHECK:  %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFomp_do_firstprivate2En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:  %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_do_firstprivate2Ea"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:  %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_do_firstprivate2En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   integer::a
   integer::n
   n = a+1
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop.f90 b/flang/test/Lower/OpenMP/parallel-wsloop.f90
index 602b3d1c05f0..5fa42da2269f 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop.f90
@@ -27,8 +27,8 @@ end subroutine
 ! CHECK-LABEL: func @_QPparallel_do_with_parallel_clauses
 ! CHECK-SAME: %[[COND_REF:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "cond"}, %[[NT_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "nt"}
 subroutine parallel_do_with_parallel_clauses(cond, nt)
-  ! CHECK: %[[COND_DECL:.*]]:2 = hlfir.declare %[[COND_REF]] {uniq_name = "_QFparallel_do_with_parallel_clausesEcond"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  ! CHECK: %[[NT_DECL:.*]]:2 = hlfir.declare %[[NT_REF]] {uniq_name = "_QFparallel_do_with_parallel_clausesEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[COND_DECL:.*]]:2 = hlfir.declare %[[COND_REF]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFparallel_do_with_parallel_clausesEcond"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  ! CHECK: %[[NT_DECL:.*]]:2 = hlfir.declare %[[NT_REF]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFparallel_do_with_parallel_clausesEnt"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   logical :: cond
   integer :: nt
   integer :: i
@@ -57,7 +57,7 @@ end subroutine
 ! CHECK-LABEL: func @_QPparallel_do_with_clauses
 ! CHECK-SAME: %[[NT_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "nt"}
 subroutine parallel_do_with_clauses(nt)
-  ! CHECK:  %[[NT_DECL:.*]]:2 = hlfir.declare %[[NT_REF]] {uniq_name = "_QFparallel_do_with_clausesEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:  %[[NT_DECL:.*]]:2 = hlfir.declare %[[NT_REF]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFparallel_do_with_clausesEnt"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   integer :: nt
   integer :: i
   ! CHECK:  %[[NT:.*]] = fir.load %[[NT_DECL]]#0 : !fir.ref<i32>
@@ -88,8 +88,8 @@ end subroutine
 ! CHECK-LABEL: func @_QPparallel_do_with_privatisation_clauses
 ! CHECK-SAME: %[[COND_REF:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "cond"}, %[[NT_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "nt"}
 subroutine parallel_do_with_privatisation_clauses(cond,nt)
-  ! CHECK: %[[COND_DECL:.*]]:2 = hlfir.declare %[[COND_REF]] {uniq_name = "_QFparallel_do_with_privatisation_clausesEcond"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
-  ! CHECK: %[[NT_DECL:.*]]:2 = hlfir.declare %[[NT_REF]] {uniq_name = "_QFparallel_do_with_privatisation_clausesEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[COND_DECL:.*]]:2 = hlfir.declare %[[COND_REF]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFparallel_do_with_privatisation_clausesEcond"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  ! CHECK: %[[NT_DECL:.*]]:2 = hlfir.declare %[[NT_REF]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFparallel_do_with_privatisation_clausesEnt"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   logical :: cond
   integer :: nt
   integer :: i
@@ -145,7 +145,7 @@ end subroutine parallel_private_do
 ! CHECK-LABEL:   func.func @_QPparallel_private_do(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "cond"},
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "nt"}) {
-! CHECK:           %[[NT_DECL:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFparallel_private_doEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[NT_DECL:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFparallel_private_doEnt"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           omp.parallel   {
 ! CHECK:             %[[I_PRIV:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
 ! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV]] {uniq_name = "_QFparallel_private_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -191,8 +191,8 @@ end subroutine omp_parallel_multiple_firstprivate_do
 ! CHECK-LABEL:   func.func @_QPomp_parallel_multiple_firstprivate_do(
 ! CHECK-SAME:                                                        %[[A_ADDR:.*]]: !fir.ref<i32> {fir.bindc_name = "a"},
 ! CHECK-SAME:                                                        %[[B_ADDR:.*]]: !fir.ref<i32> {fir.bindc_name = "b"}) {
-! CHECK:            %[[A_DECL:.*]]:2 = hlfir.declare %[[A_ADDR]] {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:            %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ADDR]] {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:            %[[A_DECL:.*]]:2 = hlfir.declare %[[A_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:            %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           omp.parallel   {
 ! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
 ! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -242,7 +242,7 @@ end subroutine parallel_do_private
 ! CHECK-LABEL:   func.func @_QPparallel_do_private(
 ! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "cond"},
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "nt"}) {
-! CHECK:           %[[NT_DECL:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFparallel_do_privateEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[NT_DECL:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFparallel_do_privateEnt"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           omp.parallel   {
 ! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
 ! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFparallel_do_privateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -288,8 +288,8 @@ end subroutine omp_parallel_do_multiple_firstprivate
 ! CHECK-LABEL:   func.func @_QPomp_parallel_do_multiple_firstprivate(
 ! CHECK-SAME:                                                        %[[A_ADDR:.*]]: !fir.ref<i32> {fir.bindc_name = "a"},
 ! CHECK-SAME:                                                        %[[B_ADDR:.*]]: !fir.ref<i32> {fir.bindc_name = "b"}) {
-! CHECK:           %[[A_DECL:.*]]:2 = hlfir.declare %[[A_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>
+! CHECK:           %[[A_DECL:.*]]:2 = hlfir.declare %[[A_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
 ! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/sections.f90 b/flang/test/Lower/OpenMP/sections.f90
index a93d6e34e374..bd76bd53e5a0 100644
--- a/flang/test/Lower/OpenMP/sections.f90
+++ b/flang/test/Lower/OpenMP/sections.f90
@@ -79,7 +79,7 @@ program sample
 end program sample
 
 !CHECK: func @_QPfirstprivate(%[[ARG:.*]]: !fir.ref<f32> {fir.bindc_name = "alpha"}) {
-!CHECK:   %[[ARG_DECL:.*]]:2 = hlfir.declare %[[ARG]] {uniq_name = "_QFfirstprivateEalpha"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>) 
+!CHECK:   %[[ARG_DECL:.*]]:2 = hlfir.declare %[[ARG]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivateEalpha"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>) 
 !CHECK:   %[[PRIVATE_ALPHA:.*]] = fir.alloca f32 {bindc_name = "alpha", pinned, uniq_name = "_QFfirstprivateEalpha"}
 !CHECK:   %[[PRIVATE_ALPHA_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_ALPHA]] {uniq_name = "_QFfirstprivateEalpha"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:   %[[TEMP:.*]] = fir.load %[[ARG_DECL]]#0 : !fir.ref<f32>
diff --git a/flang/test/Lower/OpenMP/simd.f90 b/flang/test/Lower/OpenMP/simd.f90
index 8ec1a3cefb4a..223b248b7934 100644
--- a/flang/test/Lower/OpenMP/simd.f90
+++ b/flang/test/Lower/OpenMP/simd.f90
@@ -24,7 +24,7 @@ end subroutine
 
 !CHECK-LABEL: func @_QPsimd_with_if_clause
 subroutine simd_with_if_clause(n, threshold)
-  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_if_clauseEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimd_with_if_clauseEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   integer :: i, n, threshold
   !$OMP SIMD IF( n .GE. threshold )
   ! CHECK: %[[LB:.*]] = arith.constant 1 : i32
@@ -44,7 +44,7 @@ end subroutine
 
 !CHECK-LABEL: func @_QPsimd_with_simdlen_clause
 subroutine simd_with_simdlen_clause(n, threshold)
-  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_simdlen_clauseEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimd_with_simdlen_clauseEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   integer :: i, n, threshold
   !$OMP SIMD SIMDLEN(2)
   ! CHECK: %[[LB:.*]] = arith.constant 1 : i32
@@ -63,7 +63,7 @@ end subroutine
 
 !CHECK-LABEL: func @_QPsimd_with_simdlen_clause_from_param
 subroutine simd_with_simdlen_clause_from_param(n, threshold)
-  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_simdlen_clause_from_paramEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimd_with_simdlen_clause_from_paramEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   integer :: i, n, threshold
   integer, parameter :: simdlen = 2;
   !$OMP SIMD SIMDLEN(simdlen)
@@ -83,7 +83,7 @@ end subroutine
 
 !CHECK-LABEL: func @_QPsimd_with_simdlen_clause_from_expr_from_param
 subroutine simd_with_simdlen_clause_from_expr_from_param(n, threshold)
-  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_simdlen_clause_from_expr_from_paramEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimd_with_simdlen_clause_from_expr_from_paramEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   integer :: i, n, threshold
   integer, parameter :: simdlen = 2;
   !$OMP SIMD SIMDLEN(simdlen*2 + 2)
@@ -103,7 +103,7 @@ end subroutine
 
 !CHECK-LABEL: func @_QPsimd_with_safelen_clause
 subroutine simd_with_safelen_clause(n, threshold)
-  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_safelen_clauseEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimd_with_safelen_clauseEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   integer :: i, n, threshold
   !$OMP SIMD SAFELEN(2)
   ! CHECK: %[[LB:.*]] = arith.constant 1 : i32
@@ -122,7 +122,7 @@ end subroutine
 
 !CHECK-LABEL: func @_QPsimd_with_safelen_clause_from_expr_from_param
 subroutine simd_with_safelen_clause_from_expr_from_param(n, threshold)
-  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_safelen_clause_from_expr_from_paramEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimd_with_safelen_clause_from_expr_from_paramEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   integer :: i, n, threshold
   integer, parameter :: safelen = 2;
   !$OMP SIMD SAFELEN(safelen*2 + 2)
@@ -142,7 +142,7 @@ end subroutine
 
 !CHECK-LABEL: func @_QPsimd_with_simdlen_safelen_clause
 subroutine simd_with_simdlen_safelen_clause(n, threshold)
-  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsimd_with_simdlen_safelen_clauseEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[ARG_N:.*]]:2 = hlfir.declare %{{.*}} dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimd_with_simdlen_safelen_clauseEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   integer :: i, n, threshold
   !$OMP SIMD SIMDLEN(1) SAFELEN(2)
   ! CHECK: %[[LB:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/OpenMP/single.f90 b/flang/test/Lower/OpenMP/single.f90
index 10d537a0e18b..91f8a592909a 100644
--- a/flang/test/Lower/OpenMP/single.f90
+++ b/flang/test/Lower/OpenMP/single.f90
@@ -11,7 +11,7 @@
 !CHECK-SAME: (%[[X:.*]]: !fir.ref<i32> {fir.bindc_name = "x"})
 subroutine omp_single(x)
   integer, intent(inout) :: x
-  !CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFomp_singleEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  !CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFomp_singleEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   !CHECK: omp.parallel
   !$omp parallel
   !CHECK: omp.single
@@ -34,7 +34,7 @@ end subroutine omp_single
 !CHECK-SAME: (%[[X:.*]]: !fir.ref<i32> {fir.bindc_name = "x"})
 subroutine omp_single_nowait(x)
   integer, intent(inout) :: x
-  !CHECK:   %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFomp_single_nowaitEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  !CHECK:   %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFomp_single_nowaitEx"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
   !CHECK: omp.parallel
   !$omp parallel
   !CHECK: omp.single nowait
@@ -76,8 +76,8 @@ end subroutine single_allocate
 ! CHECK-LABEL: func.func @_QPsingle_privatization(
 ! CHECK-SAME:                                     %[[X:.*]]: !fir.ref<f32> {fir.bindc_name = "x"}, 
 ! CHECK-SAME:                                     %[[Y:.*]]: !fir.ref<f64> {fir.bindc_name = "y"}) {
-! CHECK:           %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] {uniq_name = "_QFsingle_privatizationEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:           %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y]] {uniq_name = "_QFsingle_privatizationEy"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
+! CHECK:           %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFsingle_privatizationEx"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:           %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFsingle_privatizationEy"} : (!fir.ref<f64>, !fir.dscope) -> (!fir.ref<f64>, !fir.ref<f64>)
 ! CHECK:           omp.single   {
 ! CHECK:             %[[X_PVT:.*]] = fir.alloca f32 {bindc_name = "x", pinned, uniq_name = "_QFsingle_privatizationEx"}
 ! CHECK:             %[[X_PVT_DECL:.*]]:2 = hlfir.declare %[[X_PVT]] {uniq_name = "_QFsingle_privatizationEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
@@ -103,8 +103,8 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPsingle_privatization2(
 ! CHECK-SAME:                                      %[[X:.*]]: !fir.ref<f32> {fir.bindc_name = "x"},
 ! CHECK-SAME:                                      %[[Y:.*]]: !fir.ref<f64> {fir.bindc_name = "y"}) {
-! CHECK:         %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] {uniq_name = "_QFsingle_privatization2Ex"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:         %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y]] {uniq_name = "_QFsingle_privatization2Ey"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
+! CHECK:         %[[X_DECL:.*]]:2 = hlfir.declare %[[X]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFsingle_privatization2Ex"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:         %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFsingle_privatization2Ey"} : (!fir.ref<f64>, !fir.dscope) -> (!fir.ref<f64>, !fir.ref<f64>)
 ! CHECK:         omp.parallel   {
 ! CHECK:           omp.single   {
 ! CHECK:             %[[X_PVT:.*]] = fir.alloca f32 {bindc_name = "x", pinned, uniq_name = "_QFsingle_privatization2Ex"}
diff --git a/flang/test/Lower/OpenMP/target.f90 b/flang/test/Lower/OpenMP/target.f90
index 44f77b5c3360..e3b3799e5e7a 100644
--- a/flang/test/Lower/OpenMP/target.f90
+++ b/flang/test/Lower/OpenMP/target.f90
@@ -444,7 +444,7 @@ end subroutine omp_target_implicit_nested
 !CHECK: %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
 subroutine omp_target_implicit_bounds(n)
    !CHECK: %[[VAL_COPY:.*]] = fir.alloca i32
-   !CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFomp_target_implicit_boundsEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+   !CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_target_implicit_boundsEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
    !CHECK: %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<i32>
    !CHECK: fir.store %[[VAL_2]] to %[[VAL_COPY]] : !fir.ref<i32>
    !CHECK: %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (i32) -> i64
@@ -455,7 +455,7 @@ subroutine omp_target_implicit_bounds(n)
    !CHECK: %[[VAL_8:.*]] = fir.alloca !fir.array<?xi32>, %[[VAL_7]] {bindc_name = "a", uniq_name = "_QFomp_target_implicit_boundsEa"}
    !CHECK: %[[VAL_9:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>
    !CHECK: %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_8]](%[[VAL_9]]) {uniq_name = "_QFomp_target_implicit_boundsEa"} : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>)
-   !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %9#0, %c0{{.*}} : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+   !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %{{[0-9]+}}#0, %c0{{.*}} : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
    !CHECK: %[[UB:.*]] = arith.subi %[[DIMS0]]#1, %c1{{.*}} : index
 
    integer :: n
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
index 6c9bc75b81d7..197800486c39 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
@@ -75,7 +75,7 @@ end program
 ! CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.box<!fir.array<?xf64>> {fir.bindc_name = "r"}) attributes {{.*}} {
 ! CHECK:           %[[VAL_1:.*]] = fir.address_of(@_QFFreduceEi) : !fir.ref<i32>
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFFreduceEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = {{.*}}, uniq_name = "_QFFreduceEr"} : (!fir.box<!fir.array<?xf64>>) -> (!fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = {{.*}}, uniq_name = "_QFFreduceEr"} : (!fir.box<!fir.array<?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>)
 ! CHECK:           omp.parallel {
 ! CHECK:             %[[VAL_4:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
 ! CHECK:             %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFFreduceEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
index 40280c56dad6..df07a9065331 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
@@ -26,7 +26,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_iandEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iandEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_iandEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_iandEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_iandEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
index 986892d3584f..ae771c692b98 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
@@ -20,7 +20,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_iandEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iandEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_iandEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_iandEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_iandEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
index ee33ce2f348d..50cec61b602b 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
@@ -22,7 +22,7 @@
 !CHECK-SAME: %[[Y_BOX:.*]]: !fir.box<!fir.array<?xi32>>
 !CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_ieorEx"}
 !CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {uniq_name = "_QFreduction_ieorEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_BOX]] {uniq_name = "_QFreduction_ieorEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+!CHECK: %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_BOX]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_ieorEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 
 
 !CHECK: omp.parallel
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
index b362731b3371..d50f6b854f48 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
@@ -13,7 +13,7 @@
 !CHECK-SAME: %[[Y_BOX:.*]]: !fir.box<!fir.array<?xi32>>
 !CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_ieorEx"}
 !CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {uniq_name = "_QFreduction_ieorEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_BOX]] {uniq_name = "_QFreduction_ieorEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+!CHECK: %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_BOX]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_ieorEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 
 
 !CHECK: omp.parallel
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
index 0052773bb5ad..d847bba89782 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
@@ -24,7 +24,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_iorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iorEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_iorEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_iorEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_iorEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
index f32be43b9b71..182f1eaeeeb7 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
@@ -20,7 +20,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_iorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iorEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_iorEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_iorEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_iorEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
index dfc018ed7c5a..69789e4c751e 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
@@ -32,7 +32,7 @@
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -82,7 +82,7 @@ end subroutine simple_reduction
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -129,7 +129,7 @@ end subroutine
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"}
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
index c529bd4755b6..078a463919e9 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
@@ -26,7 +26,7 @@
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -76,7 +76,7 @@ end subroutine simple_reduction
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -123,7 +123,7 @@ end subroutine
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"}
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
index a54795a4446f..54175994ecd8 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
@@ -32,7 +32,7 @@
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -81,7 +81,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -128,7 +128,7 @@ end subroutine
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"}
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
index 1021b5926b91..8204e88815f3 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
@@ -26,7 +26,7 @@
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -75,7 +75,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -122,7 +122,7 @@ end subroutine
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"}
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
index 854cb19ecd75..c0a82476c7b1 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
@@ -32,7 +32,7 @@
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -82,7 +82,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -131,7 +131,7 @@ end subroutine
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"}
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
index f5c84aaaf485..957de9b6741a 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
@@ -26,7 +26,7 @@
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -76,7 +76,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -125,7 +125,7 @@ end subroutine
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"}
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
index e268c6ff6cf5..0af9e0d5c9fd 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
@@ -31,7 +31,7 @@
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -80,7 +80,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -127,7 +127,7 @@ end subroutine
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"}
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
index 26dc0c327aad..d77566b109e5 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
@@ -26,7 +26,7 @@
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reductionEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -75,7 +75,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_6]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFsimple_reduction_switch_orderEy"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_8:.*]] = arith.constant true
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
@@ -122,7 +122,7 @@ end subroutine
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = arith.constant 100 : index
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_4]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QFmultiple_reductionsEw"} : (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<100x!fir.logical<4>>>, !fir.ref<!fir.array<100x!fir.logical<4>>>)
 ! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.logical<4> {bindc_name = "x", uniq_name = "_QFmultiple_reductionsEx"}
 ! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.logical<4> {bindc_name = "y", uniq_name = "_QFmultiple_reductionsEy"}
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
index 95bdc98f18c2..11d039f9226c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
@@ -37,7 +37,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_max_intEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_max_intEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_max_intEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
@@ -68,7 +68,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFreduction_max_realEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_max_realEy"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_max_realEy"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90
index 352888bb94f5..a352cb195c25 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90
@@ -24,7 +24,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_max_intEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_max_intEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_max_intEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90
index f4caea5a269a..71631fb14592 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90
@@ -20,7 +20,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_max_intEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_max_intEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_max_intEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
index ff005f32487e..d4e827f3b7e2 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
@@ -31,7 +31,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_max_intEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_max_intEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_max_intEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
@@ -62,7 +62,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFreduction_max_realEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_max_realEy"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_max_realEy"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
index 9787512ab078..d168b2a89295 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
@@ -37,7 +37,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_min_intEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_min_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_min_intEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_min_intEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
@@ -68,7 +68,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFreduction_min_realEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_min_realEy"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_min_realEy"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
index 801ef99480a2..80c056b5e8c5 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
@@ -31,7 +31,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_min_intEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_min_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_min_intEy"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_min_intEy"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
@@ -62,7 +62,7 @@
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFreduction_min_realEx"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFreduction_min_realEy"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFreduction_min_realEy"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
diff --git a/flang/test/Lower/allocatable-polymorphic.f90 b/flang/test/Lower/allocatable-polymorphic.f90
index 10d7d957a257..e96945ef89e5 100644
--- a/flang/test/Lower/allocatable-polymorphic.f90
+++ b/flang/test/Lower/allocatable-polymorphic.f90
@@ -520,8 +520,8 @@ contains
 
 ! CHECK-LABEL: func.func @_QMpolyPtest_allocatable_up_from_up_mold(
 ! CHECK-SAME: %[[A:.*]]: !fir.ref<!fir.class<!fir.heap<none>>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.class<!fir.ptr<none>>> {fir.bindc_name = "b"}) {
-! CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMpolyFtest_allocatable_up_from_up_moldEa"} : (!fir.ref<!fir.class<!fir.heap<none>>>) -> (!fir.ref<!fir.class<!fir.heap<none>>>, !fir.ref<!fir.class<!fir.heap<none>>>)
-! CHECK: %[[B_DECL:.*]]:2 = hlfir.declare %[[B]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMpolyFtest_allocatable_up_from_up_moldEb"} : (!fir.ref<!fir.class<!fir.ptr<none>>>) -> (!fir.ref<!fir.class<!fir.ptr<none>>>, !fir.ref<!fir.class<!fir.ptr<none>>>)
+! CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMpolyFtest_allocatable_up_from_up_moldEa"} : (!fir.ref<!fir.class<!fir.heap<none>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<none>>>, !fir.ref<!fir.class<!fir.heap<none>>>)
+! CHECK: %[[B_DECL:.*]]:2 = hlfir.declare %[[B]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMpolyFtest_allocatable_up_from_up_moldEb"} : (!fir.ref<!fir.class<!fir.ptr<none>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<none>>>, !fir.ref<!fir.class<!fir.ptr<none>>>)
 ! CHECK: %[[LOAD_B:.*]] = fir.load %[[B_DECL]]#1 : !fir.ref<!fir.class<!fir.ptr<none>>>
 ! CHECK: %[[RANK:.*]] = arith.constant 0 : i32
 ! CHECK: %[[A_BOX_NONE:.*]] = fir.convert %[[A_DECL]]#1 : (!fir.ref<!fir.class<!fir.heap<none>>>) -> !fir.ref<!fir.box<none>>
@@ -539,7 +539,7 @@ contains
 ! CHECK-LABEL: func.func @_QMpolyPtest_allocatable_up_from_mold_rank(
 ! CHECK-SAME: %[[A:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>> {fir.bindc_name = "a"}) {
 ! CHECK: %[[VALUE_10:.*]] = fir.alloca i32
-! CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMpolyFtest_allocatable_up_from_mold_rankEa"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>)
+! CHECK: %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMpolyFtest_allocatable_up_from_mold_rankEa"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?xnone>>>>)
 ! CHECK: %[[C10:.*]] = arith.constant 10 : i32
 ! CHECK: fir.store %[[C10]] to %[[VALUE_10]] : !fir.ref<i32>
 ! CHECK: %[[EMBOX_10:.*]] = fir.embox %[[VALUE_10]] : (!fir.ref<i32>) -> !fir.box<i32>
diff --git a/flang/test/Lower/array-expression.f90 b/flang/test/Lower/array-expression.f90
index 75789cd6952a..bdfbe6dd3509 100644
--- a/flang/test/Lower/array-expression.f90
+++ b/flang/test/Lower/array-expression.f90
@@ -991,7 +991,7 @@ end subroutine test19f
 ! CHECK:         %[[VAL_24:.*]] = fir.array_load %[[VAL_5]](%[[VAL_22]]) {{\[}}%[[VAL_23]]] : (!fir.ref<!fir.array<140x!fir.char<2,13>>>, !fir.shape<1>, !fir.slice<1>) -> !fir.array<140x!fir.char<2,13>>
 ! CHECK:         %[[VAL_25:.*]] = fir.load %[[VAL_2]] : !fir.ref<i32>
 ! CHECK:         %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i32) -> i64
-! CHECK:         %[[char_temp:.*]] = fir.alloca !fir.char<4,?>(%16 : i64) {bindc_name = ".chrtmp"}
+! CHECK:         %[[char_temp:.*]] = fir.alloca !fir.char<4,?>(%{{[0-9]+}} : i64) {bindc_name = ".chrtmp"}
 ! CHECK:         %[[VAL_27:.*]] = arith.constant 1 : index
 ! CHECK:         %[[VAL_28:.*]] = arith.constant 0 : index
 ! CHECK:         %[[VAL_29:.*]] = arith.subi %[[VAL_13]], %[[VAL_27]] : index
diff --git a/flang/test/Lower/character-substrings.f90 b/flang/test/Lower/character-substrings.f90
index 8e1a91a247d4..874f2944cff1 100644
--- a/flang/test/Lower/character-substrings.f90
+++ b/flang/test/Lower/character-substrings.f90
@@ -231,7 +231,7 @@ end subroutine array_substring_assignment
 ! CHECK:         %[[c0:.*]] = arith.constant 0 : index
 ! CHECK:         %[[sub:.*]] = arith.subi %[[VAL_1]], %[[VAL_4]] : index
 ! CHECK:         %[[add:.*]] = arith.addi %[[sub]], %[[VAL_4]] : index
-! CHECK:         %[[div:.*]] = arith.divsi %4, %[[VAL_4]] : index
+! CHECK:         %[[div:.*]] = arith.divsi %{{[0-9]+}}, %[[VAL_4]] : index
 ! CHECK:         %[[cmp:.*]] = arith.cmpi sgt, %[[div]], %[[c0]] : index
 ! CHECK:         %[[select:.*]] = arith.select %[[cmp]], %[[div]], %[[c0]] : index
 ! CHECK:         %[[VAL_6:.*]] = fir.array_load %[[VAL_0]](%[[VAL_3]]) {{\[}}%[[VAL_5]]] : (!fir.ref<!fir.array<8x!fir.type<_QFarray_substring_assignment2Tt{ch:!fir.char<1,7>}>>>, !fir.shape<1>, !fir.slice<1>) -> !fir.array<8x!fir.char<1,7>>
@@ -323,7 +323,7 @@ end subroutine array_substring_assignment2
 ! CHECK:         %[[c0:.*]] = arith.constant 0 : index
 ! CHECK:         %[[sub:.*]] = arith.subi %[[VAL_2]], %[[VAL_6]] : index
 ! CHECK:         %[[add:.*]] = arith.addi %[[sub]], %[[VAL_6]] : index
-! CHECK:         %[[div:.*]] = arith.divsi %4, %[[VAL_6]] : index
+! CHECK:         %[[div:.*]] = arith.divsi %[[add]], %[[VAL_6]] : index
 ! CHECK:         %[[cmp:.*]] = arith.cmpi sgt, %[[div]], %[[c0]] : index
 ! CHECK:         %[[select:.*]] = arith.select %[[cmp]], %[[div]], %[[c0]] : index
 ! CHECK:         %[[VAL_8:.*]] = fir.array_load %[[VAL_0]](%[[VAL_5]]) {{\[}}%[[VAL_7]]] : (!fir.ref<!fir.array<8x!fir.type<_QFarray_substring_assignment3Tt{ch:!fir.char<1,7>}>>>, !fir.shape<1>, !fir.slice<1>) -> !fir.array<8x!fir.char<1,7>>
diff --git a/flang/test/Lower/charconvert.f90 b/flang/test/Lower/charconvert.f90
index c8ec254b6a54..e3f7f66b8476 100644
--- a/flang/test/Lower/charconvert.f90
+++ b/flang/test/Lower/charconvert.f90
@@ -14,17 +14,17 @@ end subroutine
                                                 
 ! CHECK: func.func @_QPtest_c1_to_c4(%[[ARG0:.*]]: !fir.boxchar<4> {fir.bindc_name = "c4"}, %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "c1"}) {
 ! CHECK:   %[[VAL_0:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:   %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 {uniq_name = "_QFtest_c1_to_c4Ec1"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:   %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_c1_to_c4Ec1"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:   %[[VAL_2:.*]]:2 = fir.unboxchar %[[ARG0]] : (!fir.boxchar<4>) -> (!fir.ref<!fir.char<4,?>>, index)
-! CHECK:   %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 {uniq_name = "_QFtest_c1_to_c4Ec4"} : (!fir.ref<!fir.char<4,?>>, index) -> (!fir.boxchar<4>, !fir.ref<!fir.char<4,?>>)
+! CHECK:   %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_c1_to_c4Ec4"} : (!fir.ref<!fir.char<4,?>>, index, !fir.dscope) -> (!fir.boxchar<4>, !fir.ref<!fir.char<4,?>>)
 ! CHECK:   %[[VAL_4:.*]] = fir.alloca !fir.char<4,?>(%[[VAL_0]]#1 : index)
 ! CHECK:   fir.char_convert %[[VAL_1]]#1 for %[[VAL_0]]#1 to %[[VAL_4:.*]] : !fir.ref<!fir.char<1,?>>, index, !fir.ref<!fir.char<4,?>>
 
 ! CHECK: func.func @_QPtest_c4_to_c1(%[[ARG0:.*]]: !fir.boxchar<4> {fir.bindc_name = "c4"}, %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "c1"}) {
 ! CHECK:   %[[VAL_0:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:   %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 {uniq_name = "_QFtest_c4_to_c1Ec1"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:   %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]]#0 typeparams %[[VAL_0]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_c4_to_c1Ec1"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
 ! CHECK:   %[[VAL_2:.*]]:2 = fir.unboxchar %[[ARG0]] : (!fir.boxchar<4>) -> (!fir.ref<!fir.char<4,?>>, index)
-! CHECK:   %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 {uniq_name = "_QFtest_c4_to_c1Ec4"} : (!fir.ref<!fir.char<4,?>>, index) -> (!fir.boxchar<4>, !fir.ref<!fir.char<4,?>>)
+! CHECK:   %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]]#0 typeparams %[[VAL_2]]#1 dummy_scope %{{[0-9]+}} {uniq_name = "_QFtest_c4_to_c1Ec4"} : (!fir.ref<!fir.char<4,?>>, index, !fir.dscope) -> (!fir.boxchar<4>, !fir.ref<!fir.char<4,?>>)
 ! CHECK:   %[[C4:.*]] = arith.constant 4 : index
 ! CHECK:   %[[VAL_4:.*]] = arith.muli %[[VAL_2]]#1, %[[C4]] : index
 ! CHECK:   %[[VAL_5:.*]] = fir.alloca !fir.char<1,?>(%[[VAL_4]] : index)
diff --git a/flang/test/Lower/dispatch.f90 b/flang/test/Lower/dispatch.f90
index 60364076e633..02338065548d 100644
--- a/flang/test/Lower/dispatch.f90
+++ b/flang/test/Lower/dispatch.f90
@@ -151,7 +151,7 @@ module call_dispatch
 
 ! CHECK-LABEL: func.func @_QMcall_dispatchPcheck_dispatch(
 ! CHECK-SAME:  %[[P:.*]]: !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>> {fir.bindc_name = "p"}) {
-! CHECK:       %[[P_DECL:.*]]:2 = hlfir.declare %[[P]] {uniq_name = "_QMcall_dispatchFcheck_dispatchEp"} : (!fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) -> (!fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>, !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>)
+! CHECK:       %[[P_DECL:.*]]:2 = hlfir.declare %[[P]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMcall_dispatchFcheck_dispatchEp"} : (!fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>, !fir.dscope) -> (!fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>, !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>)
 ! CHECK:       fir.dispatch "tbp_nopass"(%[[P_DECL]]#1 : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>){{$}}
 ! CHECK:       fir.dispatch "tbp_pass"(%[[P_DECL]]#0 : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) (%[[P_DECL]]#0 : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 ! CHECK:       fir.dispatch "tbp_pass_arg0"(%[[P_DECL]]#0 : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) (%[[P_DECL]]#0 : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
@@ -176,8 +176,8 @@ module call_dispatch
 ! CHECK-LABEL: func.func @_QMcall_dispatchPcheck_dispatch_deferred(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMcall_dispatchTa1{a:f32,b:f32}>> {fir.bindc_name = "a"}, 
 ! CHECK-SAME: %[[ARG1:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "x"}) {
-! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QMcall_dispatchFcheck_dispatch_deferredEa"} : (!fir.class<!fir.type<_QMcall_dispatchTa1{a:f32,b:f32}>>) -> (!fir.class<!fir.type<_QMcall_dispatchTa1{a:f32,b:f32}>>, !fir.class<!fir.type<_QMcall_dispatchTa1{a:f32,b:f32}>>)
-! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QMcall_dispatchFcheck_dispatch_deferredEx"} : (!fir.box<!fir.array<?xf32>>) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMcall_dispatchFcheck_dispatch_deferredEa"} : (!fir.class<!fir.type<_QMcall_dispatchTa1{a:f32,b:f32}>>, !fir.dscope) -> (!fir.class<!fir.type<_QMcall_dispatchTa1{a:f32,b:f32}>>, !fir.class<!fir.type<_QMcall_dispatchTa1{a:f32,b:f32}>>)
+! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMcall_dispatchFcheck_dispatch_deferredEx"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
 ! CHECK: fir.dispatch "nopassd"(%[[ARG0_DECL]]#1 : !fir.class<!fir.type<_QMcall_dispatchTa1{a:f32,b:f32}>>) (%[[ARG1_DECL]]#0 : !fir.box<!fir.array<?xf32>>)
 
     subroutine check_dispatch_scalar_allocatable(p)
@@ -187,7 +187,7 @@ module call_dispatch
 
 ! CHECK-LABEL: func.func @_QMcall_dispatchPcheck_dispatch_scalar_allocatable(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>> {fir.bindc_name = "p"}) {
-! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcall_dispatchFcheck_dispatch_scalar_allocatableEp"} : (!fir.ref<!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>, !fir.ref<!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>)
+! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcall_dispatchFcheck_dispatch_scalar_allocatableEp"} : (!fir.ref<!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>, !fir.ref<!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>)
 ! CHECK: %[[LOAD:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[REBOX:.*]] = fir.rebox %[[LOAD]] : (!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>
 ! CHECK: fir.dispatch "tbp_pass"(%[[REBOX]] : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) (%[[REBOX]] : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
@@ -199,7 +199,7 @@ module call_dispatch
 
 ! CHECK-LABEL: func.func @_QMcall_dispatchPcheck_dispatch_scalar_pointer(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>> {fir.bindc_name = "p"}) {
-! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMcall_dispatchFcheck_dispatch_scalar_pointerEp"} : (!fir.ref<!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>) -> (!fir.ref<!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>)
+! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMcall_dispatchFcheck_dispatch_scalar_pointerEp"} : (!fir.ref<!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>)
 ! CHECK: %[[LOAD:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[REBOX:.*]] = fir.rebox %[[LOAD]] : (!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>
 ! CHECK: fir.dispatch "tbp_pass"(%[[REBOX]] : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) (%[[REBOX]] : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
@@ -220,8 +220,8 @@ module call_dispatch
 ! CHECK-LABEL: func.func @_QMcall_dispatchPcheck_dispatch_static_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>> {fir.bindc_name = "p"}, 
 ! CHECK-SAME: %[[ARG1:.*]]: !fir.ref<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>> {fir.bindc_name = "t"}) {
-! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QMcall_dispatchFcheck_dispatch_static_arrayEp"} : (!fir.class<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>) -> (!fir.class<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.class<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
-! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]](%{{.*}}) {uniq_name = "_QMcall_dispatchFcheck_dispatch_static_arrayEt"} : (!fir.ref<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.ref<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
+! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMcall_dispatchFcheck_dispatch_static_arrayEp"} : (!fir.class<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.dscope) -> (!fir.class<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.class<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
+! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]](%{{.*}}) dummy_scope %{{[0-9]+}} {uniq_name = "_QMcall_dispatchFcheck_dispatch_static_arrayEt"} : (!fir.ref<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.ref<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
 ! CHECK: fir.do_loop {{.*}} {
 ! CHECK: %[[DESIGNATE:.*]] = hlfir.designate %[[ARG0_DECL]]#0 (%{{.*}})  : (!fir.class<!fir.array<10x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, i64) -> !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>
 ! CHECK: fir.dispatch "tbp_pass"(%[[DESIGNATE]] : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) (%[[DESIGNATE]] : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
@@ -248,8 +248,8 @@ module call_dispatch
 ! CHECK-LABEL: func.func @_QMcall_dispatchPcheck_dispatch_dynamic_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>> {fir.bindc_name = "p"}, 
 ! CHECK-SAME: %[[ARG1:.*]]: !fir.box<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>> {fir.bindc_name = "t"}) {
-! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QMcall_dispatchFcheck_dispatch_dynamic_arrayEp"} : (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>) -> (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
-! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QMcall_dispatchFcheck_dispatch_dynamic_arrayEt"} : (!fir.box<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>) -> (!fir.box<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.box<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
+! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMcall_dispatchFcheck_dispatch_dynamic_arrayEp"} : (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.dscope) -> (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
+! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMcall_dispatchFcheck_dispatch_dynamic_arrayEt"} : (!fir.box<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.box<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
 ! CHECK: %{{.*}} = fir.do_loop {{.*}} {
 ! CHECK: %[[DESIGNATE:.*]] = hlfir.designate %[[ARG0_DECL]]#0 (%{{.*}})  : (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, i64) -> !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>
 ! CHECK: fir.dispatch "tbp_pass"(%[[DESIGNATE]] : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) (%[[DESIGNATE]] : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
@@ -276,8 +276,8 @@ module call_dispatch
 ! CHECK-LABEL: func.func @_QMcall_dispatchPcheck_dispatch_allocatable_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>> {fir.bindc_name = "p"}, 
 ! CHECK-SAME: %[[ARG1:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>> {fir.bindc_name = "t"}) {
-! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcall_dispatchFcheck_dispatch_allocatable_arrayEp"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>)
-! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcall_dispatchFcheck_dispatch_allocatable_arrayEt"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>)
+! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcall_dispatchFcheck_dispatch_allocatable_arrayEp"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>)
+! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcall_dispatchFcheck_dispatch_allocatable_arrayEt"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>)
 ! CHECK: %{{.*}} = fir.do_loop {{.*}} {
 ! CHECK: %[[LOAD_ARG0:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>
 ! CHECK: %[[DESIGNATE:.*]] = hlfir.designate %[[LOAD_ARG0]] (%{{.*}})  : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>, i64) -> !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>
@@ -306,8 +306,8 @@ module call_dispatch
 ! CHECK-LABEL: func.func @_QMcall_dispatchPcheck_dispatch_pointer_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>> {fir.bindc_name = "p"}, 
 ! CHECK-SAME: %[[ARG1:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>> {fir.bindc_name = "t"}) {
-! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMcall_dispatchFcheck_dispatch_pointer_arrayEp"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>)
-! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMcall_dispatchFcheck_dispatch_pointer_arrayEt"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>)
+! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMcall_dispatchFcheck_dispatch_pointer_arrayEp"} : (!fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>)
+! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMcall_dispatchFcheck_dispatch_pointer_arrayEt"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>)
 
 ! CHECK: %{{.*}} = fir.do_loop {{.*}} {
 ! CHECK: %[[LOAD_ARG0:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>>
@@ -334,8 +334,8 @@ module call_dispatch
 ! CHECK-LABEL: func.func @_QMcall_dispatchPcheck_dispatch_dynamic_array_copy(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>> {fir.bindc_name = "p"}, 
 ! CHECK-SAME: %[[ARG1:.*]]: !fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>> {fir.bindc_name = "o"}) {
-! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QMcall_dispatchFcheck_dispatch_dynamic_array_copyEo"} : (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>) -> (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
-! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QMcall_dispatchFcheck_dispatch_dynamic_array_copyEp"} : (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>) -> (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
+! CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMcall_dispatchFcheck_dispatch_dynamic_array_copyEo"} : (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.dscope) -> (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
+! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMcall_dispatchFcheck_dispatch_dynamic_array_copyEp"} : (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.dscope) -> (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, !fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>)
 
 ! CHECK: %{{.*}} = fir.do_loop {{.*}} {
 ! CHECK: %[[DESIGNATE0:.*]] = hlfir.designate %[[ARG0_DECL]]#0 (%{{.*}})  : (!fir.class<!fir.array<?x!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>, i64) -> !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>
diff --git a/flang/test/Lower/do_loop.f90 b/flang/test/Lower/do_loop.f90
index 4ace17342ade..d9c83658ee25 100644
--- a/flang/test/Lower/do_loop.f90
+++ b/flang/test/Lower/do_loop.f90
@@ -132,6 +132,7 @@ end subroutine
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}, %[[E_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "e"}, %[[ST_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "st"}) {
 subroutine loop_with_variable_step(s,e,st)
   integer :: s, e, st
+  ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFloop_with_variable_stepEi"}
   ! CHECK: %[[S:.*]] = fir.load %[[S_REF]] : !fir.ref<i32>
   ! CHECK: %[[S_CVT:.*]] = fir.convert %[[S]] : (i32) -> index
   ! CHECK: %[[E:.*]] = fir.load %[[E_REF]] : !fir.ref<i32>
diff --git a/flang/test/Lower/pointer-references.f90 b/flang/test/Lower/pointer-references.f90
index ace64f9ec7ef..02394e7ec76b 100644
--- a/flang/test/Lower/pointer-references.f90
+++ b/flang/test/Lower/pointer-references.f90
@@ -34,7 +34,7 @@ subroutine char_ptr(p)
   ! CHECK: %[[count:.*]] = arith.muli %[[one]], %[[size]] : i64
   ! CHECK: %[[dst:.*]] = fir.convert %[[addr]] : (!fir.ptr<!fir.char<1,12>>) -> !fir.ref<i8>
   ! CHECK: %[[src:.*]] = fir.convert %[[str]] : (!fir.ref<!fir.char<1,12>>) -> !fir.ref<i8>
-  ! CHECK: fir.call @llvm.memmove.p0.p0.i64(%[[dst]], %[[src]], %5, %false) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64, i1) -> ()
+  ! CHECK: fir.call @llvm.memmove.p0.p0.i64(%[[dst]], %[[src]], %{{[0-9]+}}, %false) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64, i1) -> ()
   p = "hello world!"
 
   ! CHECK: %[[boxload2:.*]] = fir.load %[[arg0]]
diff --git a/flang/test/Lower/polymorphic.f90 b/flang/test/Lower/polymorphic.f90
index 70c1f768e389..14ec8a06a964 100644
--- a/flang/test/Lower/polymorphic.f90
+++ b/flang/test/Lower/polymorphic.f90
@@ -298,7 +298,7 @@ module polymorphic_test
 ! CHECK: %[[ZERO:.*]] = fir.zero_bits !fir.ptr<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
 ! CHECK: fir.store %[[ZERO]] to %[[PTR]] : !fir.ref<!fir.ptr<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> !fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
-! CHECK: %[[CONVERT:.*]] = fir.convert %3 : (!fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> !fir.ptr<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
+! CHECK: %[[CONVERT:.*]] = fir.convert %{{[0-9]+}} : (!fir.ref<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>) -> !fir.ptr<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>
 ! CHECK: fir.store %[[CONVERT]] to %[[PTR]] : !fir.ref<!fir.ptr<!fir.type<_QMpolymorphic_testTp1{a:i32,b:i32}>>>
 
   subroutine nullify_pointer_array(a)
diff --git a/flang/test/Lower/select-type.f90 b/flang/test/Lower/select-type.f90
index 3243a813e9d5..e4ff2fef0efd 100644
--- a/flang/test/Lower/select-type.f90
+++ b/flang/test/Lower/select-type.f90
@@ -498,7 +498,7 @@ contains
 ! CHECK:  fir.array_merge_store %[[ARRAY_LOAD]], %[[LOOP_RES]] to %[[BOX]] : !fir.array<?xf32>, !fir.array<?xf32>, !fir.box<!fir.array<?xf32>>
 ! CHECK:  cf.br ^{{.*}}
 ! CHECK: ^bb{{.*}}:
-! CHECK:  %[[BOX:.*]] = fir.convert %0 : (!fir.class<!fir.array<?xnone>>) -> !fir.box<!fir.array<?x!fir.char<1,?>>> 
+! CHECK:  %[[BOX:.*]] = fir.convert %{{[0-9]+}} : (!fir.class<!fir.array<?xnone>>) -> !fir.box<!fir.array<?x!fir.char<1,?>>> 
 ! CHECK:  cf.br ^bb{{.*}}
 ! CHECK: ^bb{{.*}}:
 ! CHECK:  %[[EXACT_BOX:.*]] = fir.convert %[[SELECTOR]] : (!fir.class<!fir.array<?xnone>>) -> !fir.box<!fir.array<?x!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>
diff --git a/flang/test/Lower/structure-constructors-alloc-comp.f90 b/flang/test/Lower/structure-constructors-alloc-comp.f90
index 5b56463303ba..5b1bca317c94 100644
--- a/flang/test/Lower/structure-constructors-alloc-comp.f90
+++ b/flang/test/Lower/structure-constructors-alloc-comp.f90
@@ -24,7 +24,7 @@ contains
 ! HLFIR-LABEL:  func.func @_QMm_struct_ctorPtest_alloc1(
 ! HLFIR-SAME:      %[[ARG_0:.*]]: !fir.ref<f32> {fir.bindc_name = "y"}) {
 ! HLFIR:    %[[VAL_0:.*]] = fir.alloca !fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>
-! HLFIR:    %[[VAL_12:.*]]:2 = hlfir.declare %[[ARG_0]] {uniq_name = "_QMm_struct_ctorFtest_alloc1Ey"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! HLFIR:    %[[VAL_12:.*]]:2 = hlfir.declare %[[ARG_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMm_struct_ctorFtest_alloc1Ey"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! HLFIR:    %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> (!fir.ref<!fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>, !fir.ref<!fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>)
 ! HLFIR:    %[[VAL_14:.*]] = fir.embox %[[VAL_13]]#0 : (!fir.ref<!fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.box<!fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
 ! HLFIR:    %[[VAL_15:.*]] = fir.address_of(@_QQ{{.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
@@ -49,8 +49,8 @@ contains
 ! HLFIR:    %[[VAL_0:.*]] = fir.alloca !fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>
 ! HLFIR:    %[[CONS_6:.*]] = arith.constant 5 : index
 ! HLFIR:    %[[VAL_12:.*]] = fir.shape %[[CONS_6]] : (index) -> !fir.shape<1>
-! HLFIR:    %[[VAL_13:.*]]:2 = hlfir.declare %[[ARG_1]](%[[VAL_12]]) {uniq_name = "_QMm_struct_ctorFtest_alloc2Eb"} : (!fir.ref<!fir.array<5xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<5xi32>>, !fir.ref<!fir.array<5xi32>>)
-! HLFIR:    %[[VAL_14:.*]]:2 = hlfir.declare %[[ARG_0]] {uniq_name = "_QMm_struct_ctorFtest_alloc2Ey"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! HLFIR:    %[[VAL_13:.*]]:2 = hlfir.declare %[[ARG_1]](%[[VAL_12]]) dummy_scope %{{[0-9]+}} {uniq_name = "_QMm_struct_ctorFtest_alloc2Eb"} : (!fir.ref<!fir.array<5xi32>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<5xi32>>, !fir.ref<!fir.array<5xi32>>)
+! HLFIR:    %[[VAL_14:.*]]:2 = hlfir.declare %[[ARG_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMm_struct_ctorFtest_alloc2Ey"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! HLFIR:    %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> (!fir.ref<!fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>, !fir.ref<!fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>)
 ! HLFIR:    %[[VAL_16:.*]] = fir.embox %[[VAL_15]]#0 : (!fir.ref<!fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.box<!fir.type<_QMm_struct_ctorTt_alloc{x:f32,a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
 ! HLFIR:    %[[VAL_17:.*]] = fir.address_of(@_QQ{{.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
-- 
GitLab


From 96568f3539d8a72432a03257a7a8ed2f36014b59 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Wed, 8 May 2024 16:49:08 -0700
Subject: [PATCH 0234/1206] [llvm][ctx_profile] Add instrumentation lowering
 (#90821)

This adds the instrumentation lowering pass.

(Tracking Issue: #89287, RFC referenced there)
---
 llvm/docs/LangRef.rst                         |  48 ++-
 .../Instrumentation/PGOCtxProfLowering.h      |   5 +-
 llvm/lib/Passes/PassBuilder.cpp               |   1 +
 llvm/lib/Passes/PassBuilderPipelines.cpp      |   5 +
 llvm/lib/Passes/PassRegistry.def              |   1 +
 .../Instrumentation/PGOCtxProfLowering.cpp    | 326 ++++++++++++++++++
 .../ctx-instrumentation-invalid-roots.ll      |  17 +
 .../PGOProfile/ctx-instrumentation.ll         | 229 ++++++++++++
 8 files changed, 626 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index cc7094116b8b..6f5a4644ffc2 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -14111,6 +14111,25 @@ structures and the code to increment the appropriate value, in a
 format that can be written out by a compiler runtime and consumed via
 the ``llvm-profdata`` tool.
 
+.. FIXME: write complete doc on contextual instrumentation and link from here
+.. and from llvm.instrprof.callsite.
+
+The intrinsic is lowered differently for contextual profiling by the
+``-ctx-instr-lower`` pass. Here:
+
+* the entry basic block increment counter is lowered as a call to compiler-rt,
+  to either ``__llvm_ctx_profile_start_context`` or
+  ``__llvm_ctx_profile_get_context``. Either returns a pointer to a context object
+  which contains a buffer into which counter increments can happen. Note that the
+  pointer value returned by compiler-rt may have its LSB set - counter increments
+  happen offset from the address with the LSB cleared.
+
+* all the other lowerings of ``llvm.instrprof.increment[.step]`` happen within
+  that context.
+
+* the context is assumed to be a local value to the function, and no concurrency
+  concerns need to be handled by LLVM.
+
 '``llvm.instrprof.increment.step``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -14156,10 +14175,10 @@ Syntax:
 Overview:
 """""""""
 
-.. FIXME: detail when it's emitted once the support is added
-
 The '``llvm.instrprof.callsite``' intrinsic should be emitted before a callsite
-that's not to a "fake" callee (like another intrinsic or asm).
+that's not to a "fake" callee (like another intrinsic or asm). It is used by
+contextual profiling and has side-effects. Its lowering happens in IR, and
+target-specific backends should never encounter it.
 
 Arguments:
 """"""""""
@@ -14172,9 +14191,28 @@ The last argument is the called value of the callsite this intrinsic precedes.
 
 Semantics:
 """"""""""
-.. FIXME: detail how when the lowering pass is added.
 
-This is lowered by contextual profiling.
+This is lowered by contextual profiling. In contextual profiling, functions get,
+from compiler-rt, a pointer to a context object. The context object consists of
+a buffer LLVM can use to perform counter increments (i.e. the lowering of
+``llvm.instrprof.increment[.step]``. The address range following the counter
+buffer, ``<num-counters>`` x ``sizeof(ptr)`` - sized, is expected to contain
+pointers to contexts of functions called from this function ("subcontexts").
+LLVM does not dereference into that memory region, just calculates GEPs. 
+
+The lowering of ``llvm.instrprof.callsite`` consists of:
+
+* write to ``__llvm_ctx_profile_expected_callee`` the ``<callsite>`` value;
+
+* write to ``__llvm_ctx_profile_callsite`` the address into this function's
+  context of the ``<index>`` position into the subcontexts region.
+
+
+``__llvm_ctx_profile_{expected_callee|callsite}`` are initialized by compiler-rt
+and are TLS. They are both vectors of pointers of size 2. The index into each is
+determined when the current function obtains the pointer to its context from
+compiler-rt. The pointer's LSB gives the index.
+
 
 '``llvm.instrprof.timestamp``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h
index 38afa0c6fd32..5256aff56205 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h
@@ -12,13 +12,16 @@
 #ifndef LLVM_TRANSFORMS_INSTRUMENTATION_PGOCTXPROFLOWERING_H
 #define LLVM_TRANSFORMS_INSTRUMENTATION_PGOCTXPROFLOWERING_H
 
+#include "llvm/IR/PassManager.h"
 namespace llvm {
 class Type;
 
-class PGOCtxProfLoweringPass {
+class PGOCtxProfLoweringPass : public PassInfoMixin<PGOCtxProfLoweringPass> {
 public:
   explicit PGOCtxProfLoweringPass() = default;
   static bool isContextualIRPGOEnabled();
+
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 };
 } // namespace llvm
 #endif
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 51ddb73943b1..e4131706aba0 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -177,6 +177,7 @@
 #include "llvm/Transforms/Instrumentation/LowerAllowCheckPass.h"
 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
+#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
 #include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 100889c0845b..1d7f0510450c 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -74,6 +74,7 @@
 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
 #include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h"
 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
 #include "llvm/Transforms/Scalar/ADCE.h"
@@ -834,6 +835,10 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
         PTO.EagerlyInvalidateAnalyses));
   }
 
+  if (PGOCtxProfLoweringPass::isContextualIRPGOEnabled()) {
+    MPM.addPass(PGOCtxProfLoweringPass());
+    return;
+  }
   // Add the profile lowering pass.
   InstrProfOptions Options;
   if (!ProfileFile.empty())
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 6864f307e56c..e5ce6cb7da64 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -77,6 +77,7 @@ MODULE_PASS("inliner-wrapper-no-mandatory-first",
 MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass())
 MODULE_PASS("instrorderfile", InstrOrderFilePass())
 MODULE_PASS("instrprof", InstrProfilingLoweringPass())
+MODULE_PASS("ctx-instr-lower", PGOCtxProfLoweringPass())
 MODULE_PASS("internalize", InternalizePass())
 MODULE_PASS("invalidate<all>", InvalidateAllAnalysesPass())
 MODULE_PASS("iroutliner", IROutlinerPass())
diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
index 9d6dd5ccb38b..76afa2f22461 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
@@ -8,10 +8,20 @@
 //
 
 #include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/IR/Analysis.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/Support/CommandLine.h"
+#include <utility>
 
 using namespace llvm;
 
+#define DEBUG_TYPE "ctx-instr-lower"
+
 static cl::list<std::string> ContextRoots(
     "profile-context-root", cl::Hidden,
     cl::desc(
@@ -22,3 +32,319 @@ static cl::list<std::string> ContextRoots(
 bool PGOCtxProfLoweringPass::isContextualIRPGOEnabled() {
   return !ContextRoots.empty();
 }
+
+// the names of symbols we expect in compiler-rt. Using a namespace for
+// readability.
+namespace CompilerRtAPINames {
+static auto StartCtx = "__llvm_ctx_profile_start_context";
+static auto ReleaseCtx = "__llvm_ctx_profile_release_context";
+static auto GetCtx = "__llvm_ctx_profile_get_context";
+static auto ExpectedCalleeTLS = "__llvm_ctx_profile_expected_callee";
+static auto CallsiteTLS = "__llvm_ctx_profile_callsite";
+} // namespace CompilerRtAPINames
+
+namespace {
+// The lowering logic and state.
+class CtxInstrumentationLowerer final {
+  Module &M;
+  ModuleAnalysisManager &MAM;
+  Type *ContextNodeTy = nullptr;
+  Type *ContextRootTy = nullptr;
+
+  DenseMap<const Function *, Constant *> ContextRootMap;
+  Function *StartCtx = nullptr;
+  Function *GetCtx = nullptr;
+  Function *ReleaseCtx = nullptr;
+  GlobalVariable *ExpectedCalleeTLS = nullptr;
+  GlobalVariable *CallsiteInfoTLS = nullptr;
+
+public:
+  CtxInstrumentationLowerer(Module &M, ModuleAnalysisManager &MAM);
+  // return true if lowering happened (i.e. a change was made)
+  bool lowerFunction(Function &F);
+};
+
+// llvm.instrprof.increment[.step] captures the total number of counters as one
+// of its parameters, and llvm.instrprof.callsite captures the total number of
+// callsites. Those values are the same for instances of those intrinsics in
+// this function. Find the first instance of each and return them.
+std::pair<uint32_t, uint32_t> getNrCountersAndCallsites(const Function &F) {
+  uint32_t NrCounters = 0;
+  uint32_t NrCallsites = 0;
+  for (const auto &BB : F) {
+    for (const auto &I : BB) {
+      if (const auto *Incr = dyn_cast<InstrProfIncrementInst>(&I)) {
+        uint32_t V =
+            static_cast<uint32_t>(Incr->getNumCounters()->getZExtValue());
+        assert((!NrCounters || V == NrCounters) &&
+               "expected all llvm.instrprof.increment[.step] intrinsics to "
+               "have the same total nr of counters parameter");
+        NrCounters = V;
+      } else if (const auto *CSIntr = dyn_cast<InstrProfCallsite>(&I)) {
+        uint32_t V =
+            static_cast<uint32_t>(CSIntr->getNumCounters()->getZExtValue());
+        assert((!NrCallsites || V == NrCallsites) &&
+               "expected all llvm.instrprof.callsite intrinsics to have the "
+               "same total nr of callsites parameter");
+        NrCallsites = V;
+      }
+#if NDEBUG
+      if (NrCounters && NrCallsites)
+        return std::make_pair(NrCounters, NrCallsites);
+#endif
+    }
+  }
+  return {NrCounters, NrCallsites};
+}
+} // namespace
+
+// set up tie-in with compiler-rt.
+// NOTE!!!
+// These have to match compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
+CtxInstrumentationLowerer::CtxInstrumentationLowerer(Module &M,
+                                                     ModuleAnalysisManager &MAM)
+    : M(M), MAM(MAM) {
+  auto *PointerTy = PointerType::get(M.getContext(), 0);
+  auto *SanitizerMutexType = Type::getInt8Ty(M.getContext());
+  auto *I32Ty = Type::getInt32Ty(M.getContext());
+  auto *I64Ty = Type::getInt64Ty(M.getContext());
+
+  // The ContextRoot type
+  ContextRootTy =
+      StructType::get(M.getContext(), {
+                                          PointerTy,          /*FirstNode*/
+                                          PointerTy,          /*FirstMemBlock*/
+                                          PointerTy,          /*CurrentMem*/
+                                          SanitizerMutexType, /*Taken*/
+                                      });
+  // The Context header.
+  ContextNodeTy = StructType::get(M.getContext(), {
+                                                      I64Ty,     /*Guid*/
+                                                      PointerTy, /*Next*/
+                                                      I32Ty,     /*NrCounters*/
+                                                      I32Ty,     /*NrCallsites*/
+                                                  });
+
+  // Define a global for each entrypoint. We'll reuse the entrypoint's name as
+  // prefix. We assume the entrypoint names to be unique.
+  for (const auto &Fname : ContextRoots) {
+    if (const auto *F = M.getFunction(Fname)) {
+      if (F->isDeclaration())
+        continue;
+      auto *G = M.getOrInsertGlobal(Fname + "_ctx_root", ContextRootTy);
+      cast<GlobalVariable>(G)->setInitializer(
+          Constant::getNullValue(ContextRootTy));
+      ContextRootMap.insert(std::make_pair(F, G));
+      for (const auto &BB : *F)
+        for (const auto &I : BB)
+          if (const auto *CB = dyn_cast<CallBase>(&I))
+            if (CB->isMustTailCall()) {
+              M.getContext().emitError(
+                  "The function " + Fname +
+                  " was indicated as a context root, but it features musttail "
+                  "calls, which is not supported.");
+            }
+    }
+  }
+
+  // Declare the functions we will call.
+  StartCtx = cast<Function>(
+      M.getOrInsertFunction(
+           CompilerRtAPINames::StartCtx,
+           FunctionType::get(ContextNodeTy->getPointerTo(),
+                             {ContextRootTy->getPointerTo(), /*ContextRoot*/
+                              I64Ty, /*Guid*/ I32Ty,
+                              /*NrCounters*/ I32Ty /*NrCallsites*/},
+                             false))
+          .getCallee());
+  GetCtx = cast<Function>(
+      M.getOrInsertFunction(CompilerRtAPINames::GetCtx,
+                            FunctionType::get(ContextNodeTy->getPointerTo(),
+                                              {PointerTy, /*Callee*/
+                                               I64Ty,     /*Guid*/
+                                               I32Ty,     /*NrCounters*/
+                                               I32Ty},    /*NrCallsites*/
+                                              false))
+          .getCallee());
+  ReleaseCtx = cast<Function>(
+      M.getOrInsertFunction(
+           CompilerRtAPINames::ReleaseCtx,
+           FunctionType::get(Type::getVoidTy(M.getContext()),
+                             {
+                                 ContextRootTy->getPointerTo(), /*ContextRoot*/
+                             },
+                             false))
+          .getCallee());
+
+  // Declare the TLSes we will need to use.
+  CallsiteInfoTLS =
+      new GlobalVariable(M, PointerTy, false, GlobalValue::ExternalLinkage,
+                         nullptr, CompilerRtAPINames::CallsiteTLS);
+  CallsiteInfoTLS->setThreadLocal(true);
+  CallsiteInfoTLS->setVisibility(llvm::GlobalValue::HiddenVisibility);
+  ExpectedCalleeTLS =
+      new GlobalVariable(M, PointerTy, false, GlobalValue::ExternalLinkage,
+                         nullptr, CompilerRtAPINames::ExpectedCalleeTLS);
+  ExpectedCalleeTLS->setThreadLocal(true);
+  ExpectedCalleeTLS->setVisibility(llvm::GlobalValue::HiddenVisibility);
+}
+
+PreservedAnalyses PGOCtxProfLoweringPass::run(Module &M,
+                                              ModuleAnalysisManager &MAM) {
+  CtxInstrumentationLowerer Lowerer(M, MAM);
+  bool Changed = false;
+  for (auto &F : M)
+    Changed |= Lowerer.lowerFunction(F);
+  return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
+}
+
+bool CtxInstrumentationLowerer::lowerFunction(Function &F) {
+  if (F.isDeclaration())
+    return false;
+  auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+  auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+
+  Value *Guid = nullptr;
+  auto [NrCounters, NrCallsites] = getNrCountersAndCallsites(F);
+
+  Value *Context = nullptr;
+  Value *RealContext = nullptr;
+
+  StructType *ThisContextType = nullptr;
+  Value *TheRootContext = nullptr;
+  Value *ExpectedCalleeTLSAddr = nullptr;
+  Value *CallsiteInfoTLSAddr = nullptr;
+
+  auto &Head = F.getEntryBlock();
+  for (auto &I : Head) {
+    // Find the increment intrinsic in the entry basic block.
+    if (auto *Mark = dyn_cast<InstrProfIncrementInst>(&I)) {
+      assert(Mark->getIndex()->isZero());
+
+      IRBuilder<> Builder(Mark);
+      // FIXME(mtrofin): use InstrProfSymtab::getCanonicalName
+      Guid = Builder.getInt64(F.getGUID());
+      // The type of the context of this function is now knowable since we have
+      // NrCallsites and NrCounters. We delcare it here because it's more
+      // convenient - we have the Builder.
+      ThisContextType = StructType::get(
+          F.getContext(),
+          {ContextNodeTy, ArrayType::get(Builder.getInt64Ty(), NrCounters),
+           ArrayType::get(Builder.getPtrTy(), NrCallsites)});
+      // Figure out which way we obtain the context object for this function -
+      // if it's an entrypoint, then we call StartCtx, otherwise GetCtx. In the
+      // former case, we also set TheRootContext since we need to release it
+      // at the end (plus it can be used to know if we have an entrypoint or a
+      // regular function)
+      auto Iter = ContextRootMap.find(&F);
+      if (Iter != ContextRootMap.end()) {
+        TheRootContext = Iter->second;
+        Context = Builder.CreateCall(StartCtx, {TheRootContext, Guid,
+                                                Builder.getInt32(NrCounters),
+                                                Builder.getInt32(NrCallsites)});
+        ORE.emit(
+            [&] { return OptimizationRemark(DEBUG_TYPE, "Entrypoint", &F); });
+      } else {
+        Context =
+            Builder.CreateCall(GetCtx, {&F, Guid, Builder.getInt32(NrCounters),
+                                        Builder.getInt32(NrCallsites)});
+        ORE.emit([&] {
+          return OptimizationRemark(DEBUG_TYPE, "RegularFunction", &F);
+        });
+      }
+      // The context could be scratch.
+      auto *CtxAsInt = Builder.CreatePtrToInt(Context, Builder.getInt64Ty());
+      if (NrCallsites > 0) {
+        // Figure out which index of the TLS 2-element buffers to use.
+        // Scratch context => we use index == 1. Real contexts => index == 0.
+        auto *Index = Builder.CreateAnd(CtxAsInt, Builder.getInt64(1));
+        // The GEPs corresponding to that index, in the respective TLS.
+        ExpectedCalleeTLSAddr = Builder.CreateGEP(
+            Builder.getInt8Ty()->getPointerTo(),
+            Builder.CreateThreadLocalAddress(ExpectedCalleeTLS), {Index});
+        CallsiteInfoTLSAddr = Builder.CreateGEP(
+            Builder.getInt32Ty(),
+            Builder.CreateThreadLocalAddress(CallsiteInfoTLS), {Index});
+      }
+      // Because the context pointer may have LSB set (to indicate scratch),
+      // clear it for the value we use as base address for the counter vector.
+      // This way, if later we want to have "real" (not clobbered) buffers
+      // acting as scratch, the lowering (at least this part of it that deals
+      // with counters) stays the same.
+      RealContext = Builder.CreateIntToPtr(
+          Builder.CreateAnd(CtxAsInt, Builder.getInt64(-2)),
+          ThisContextType->getPointerTo());
+      I.eraseFromParent();
+      break;
+    }
+  }
+  if (!Context) {
+    ORE.emit([&] {
+      return OptimizationRemarkMissed(DEBUG_TYPE, "Skip", &F)
+             << "Function doesn't have instrumentation, skipping";
+    });
+    return false;
+  }
+
+  bool ContextWasReleased = false;
+  for (auto &BB : F) {
+    for (auto &I : llvm::make_early_inc_range(BB)) {
+      if (auto *Instr = dyn_cast<InstrProfCntrInstBase>(&I)) {
+        IRBuilder<> Builder(Instr);
+        switch (Instr->getIntrinsicID()) {
+        case llvm::Intrinsic::instrprof_increment:
+        case llvm::Intrinsic::instrprof_increment_step: {
+          // Increments (or increment-steps) are just a typical load - increment
+          // - store in the RealContext.
+          auto *AsStep = cast<InstrProfIncrementInst>(Instr);
+          auto *GEP = Builder.CreateGEP(
+              ThisContextType, RealContext,
+              {Builder.getInt32(0), Builder.getInt32(1), AsStep->getIndex()});
+          Builder.CreateStore(
+              Builder.CreateAdd(Builder.CreateLoad(Builder.getInt64Ty(), GEP),
+                                AsStep->getStep()),
+              GEP);
+        } break;
+        case llvm::Intrinsic::instrprof_callsite:
+          // callsite lowering: write the called value in the expected callee
+          // TLS we treat the TLS as volatile because of signal handlers and to
+          // avoid these being moved away from the callsite they decorate.
+          auto *CSIntrinsic = dyn_cast<InstrProfCallsite>(Instr);
+          Builder.CreateStore(CSIntrinsic->getCallee(), ExpectedCalleeTLSAddr,
+                              true);
+          // write the GEP of the slot in the sub-contexts portion of the
+          // context in TLS. Now, here, we use the actual Context value - as
+          // returned from compiler-rt - which may have the LSB set if the
+          // Context was scratch. Since the header of the context object and
+          // then the values are all 8-aligned (or, really, insofar as we care,
+          // they are even) - if the context is scratch (meaning, an odd value),
+          // so will the GEP. This is important because this is then visible to
+          // compiler-rt which will produce scratch contexts for callers that
+          // have a scratch context.
+          Builder.CreateStore(
+              Builder.CreateGEP(ThisContextType, Context,
+                                {Builder.getInt32(0), Builder.getInt32(2),
+                                 CSIntrinsic->getIndex()}),
+              CallsiteInfoTLSAddr, true);
+          break;
+        }
+        I.eraseFromParent();
+      } else if (TheRootContext && isa<ReturnInst>(I)) {
+        // Remember to release the context if we are an entrypoint.
+        IRBuilder<> Builder(&I);
+        Builder.CreateCall(ReleaseCtx, {TheRootContext});
+        ContextWasReleased = true;
+      }
+    }
+  }
+  // FIXME: This would happen if the entrypoint tailcalls. A way to fix would be
+  // to disallow this, (so this then stays as an error), another is to detect
+  // that and then do a wrapper or disallow the tail call. This only affects
+  // instrumentation, when we want to detect the call graph.
+  if (TheRootContext && !ContextWasReleased)
+    F.getContext().emitError(
+        "[ctx_prof] An entrypoint was instrumented but it has no `ret` "
+        "instructions above which to release the context: " +
+        F.getName());
+  return true;
+}
diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll
new file mode 100644
index 000000000000..99c7762a67df
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll
@@ -0,0 +1,17 @@
+; RUN: not opt -passes=pgo-instr-gen,ctx-instr-lower -profile-context-root=good \
+; RUN:   -profile-context-root=bad \
+; RUN:   -S < %s 2>&1 | FileCheck %s
+
+declare void @foo()
+
+define void @good() {
+  call void @foo()
+  ret void
+}
+
+define void @bad() {
+  musttail call void @foo()
+  ret void
+}
+
+; CHECK: error: The function bad was indicated as a context root, but it features musttail calls, which is not supported.
diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
index 2ad95ab51cc6..56c7c7519f69 100644
--- a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
+++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
@@ -1,11 +1,31 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 4
 ; RUN: opt -passes=pgo-instr-gen -profile-context-root=an_entrypoint \
 ; RUN:   -S < %s | FileCheck --check-prefix=INSTRUMENT %s
+; RUN: opt -passes=pgo-instr-gen,ctx-instr-lower -profile-context-root=an_entrypoint \
+; RUN:   -profile-context-root=another_entrypoint_no_callees \
+; RUN:   -S < %s | FileCheck --check-prefix=LOWERING %s
+
 
 declare void @bar()
 
 ;.
 ; INSTRUMENT: @__profn_foo = private constant [3 x i8] c"foo"
+; INSTRUMENT: @__profn_an_entrypoint = private constant [13 x i8] c"an_entrypoint"
+; INSTRUMENT: @__profn_another_entrypoint_no_callees = private constant [29 x i8] c"another_entrypoint_no_callees"
+; INSTRUMENT: @__profn_simple = private constant [6 x i8] c"simple"
+; INSTRUMENT: @__profn_no_callsites = private constant [12 x i8] c"no_callsites"
+; INSTRUMENT: @__profn_no_counters = private constant [11 x i8] c"no_counters"
+;.
+; LOWERING: @__profn_foo = private constant [3 x i8] c"foo"
+; LOWERING: @__profn_an_entrypoint = private constant [13 x i8] c"an_entrypoint"
+; LOWERING: @__profn_another_entrypoint_no_callees = private constant [29 x i8] c"another_entrypoint_no_callees"
+; LOWERING: @__profn_simple = private constant [6 x i8] c"simple"
+; LOWERING: @__profn_no_callsites = private constant [12 x i8] c"no_callsites"
+; LOWERING: @__profn_no_counters = private constant [11 x i8] c"no_counters"
+; LOWERING: @an_entrypoint_ctx_root = global { ptr, ptr, ptr, i8 } zeroinitializer
+; LOWERING: @another_entrypoint_no_callees_ctx_root = global { ptr, ptr, ptr, i8 } zeroinitializer
+; LOWERING: @__llvm_ctx_profile_callsite = external hidden thread_local global ptr
+; LOWERING: @__llvm_ctx_profile_expected_callee = external hidden thread_local global ptr
 ;.
 define void @foo(i32 %a, ptr %fct) {
 ; INSTRUMENT-LABEL: define void @foo(
@@ -24,6 +44,38 @@ define void @foo(i32 %a, ptr %fct) {
 ; INSTRUMENT-NEXT:    br label [[EXIT]]
 ; INSTRUMENT:       exit:
 ; INSTRUMENT-NEXT:    ret void
+;
+; LOWERING-LABEL: define void @foo(
+; LOWERING-SAME: i32 [[A:%.*]], ptr [[FCT:%.*]]) {
+; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @foo, i64 6699318081062747564, i32 2, i32 2)
+; LOWERING-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; LOWERING-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], 1
+; LOWERING-NEXT:    [[TMP4:%.*]] = call ptr @llvm.threadlocal.address.p0(ptr @__llvm_ctx_profile_expected_callee)
+; LOWERING-NEXT:    [[TMP5:%.*]] = getelementptr ptr, ptr [[TMP4]], i64 [[TMP3]]
+; LOWERING-NEXT:    [[TMP6:%.*]] = call ptr @llvm.threadlocal.address.p0(ptr @__llvm_ctx_profile_callsite)
+; LOWERING-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i64 [[TMP3]]
+; LOWERING-NEXT:    [[TMP8:%.*]] = and i64 [[TMP2]], -2
+; LOWERING-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; LOWERING-NEXT:    [[T:%.*]] = icmp eq i32 [[A]], 0
+; LOWERING-NEXT:    br i1 [[T]], label [[YES:%.*]], label [[NO:%.*]]
+; LOWERING:       yes:
+; LOWERING-NEXT:    [[TMP10:%.*]] = getelementptr { { i64, ptr, i32, i32 }, [2 x i64], [2 x ptr] }, ptr [[TMP9]], i32 0, i32 1, i32 1
+; LOWERING-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 4
+; LOWERING-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], 1
+; LOWERING-NEXT:    store i64 [[TMP12]], ptr [[TMP10]], align 4
+; LOWERING-NEXT:    store volatile ptr [[FCT]], ptr [[TMP5]], align 8
+; LOWERING-NEXT:    [[TMP13:%.*]] = getelementptr { { i64, ptr, i32, i32 }, [2 x i64], [2 x ptr] }, ptr [[TMP1]], i32 0, i32 2, i32 0
+; LOWERING-NEXT:    store volatile ptr [[TMP13]], ptr [[TMP7]], align 8
+; LOWERING-NEXT:    call void [[FCT]](i32 [[A]])
+; LOWERING-NEXT:    br label [[EXIT:%.*]]
+; LOWERING:       no:
+; LOWERING-NEXT:    store volatile ptr @bar, ptr [[TMP5]], align 8
+; LOWERING-NEXT:    [[TMP14:%.*]] = getelementptr { { i64, ptr, i32, i32 }, [2 x i64], [2 x ptr] }, ptr [[TMP1]], i32 0, i32 2, i32 1
+; LOWERING-NEXT:    store volatile ptr [[TMP14]], ptr [[TMP7]], align 8
+; LOWERING-NEXT:    call void @bar()
+; LOWERING-NEXT:    br label [[EXIT]]
+; LOWERING:       exit:
+; LOWERING-NEXT:    ret void
 ;
   %t = icmp eq i32 %a, 0
   br i1 %t, label %yes, label %no
@@ -36,6 +88,183 @@ no:
 exit:
   ret void
 }
+
+define void @an_entrypoint(i32 %a) {
+; INSTRUMENT-LABEL: define void @an_entrypoint(
+; INSTRUMENT-SAME: i32 [[A:%.*]]) {
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.increment(ptr @__profn_an_entrypoint, i64 784007058953177093, i32 2, i32 0)
+; INSTRUMENT-NEXT:    [[T:%.*]] = icmp eq i32 [[A]], 0
+; INSTRUMENT-NEXT:    br i1 [[T]], label [[YES:%.*]], label [[NO:%.*]]
+; INSTRUMENT:       yes:
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.increment(ptr @__profn_an_entrypoint, i64 784007058953177093, i32 2, i32 1)
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.callsite(ptr @__profn_an_entrypoint, i64 784007058953177093, i32 1, i32 0, ptr @foo)
+; INSTRUMENT-NEXT:    call void @foo(i32 1, ptr null)
+; INSTRUMENT-NEXT:    ret void
+; INSTRUMENT:       no:
+; INSTRUMENT-NEXT:    ret void
+;
+; LOWERING-LABEL: define void @an_entrypoint(
+; LOWERING-SAME: i32 [[A:%.*]]) {
+; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_start_context(ptr @an_entrypoint_ctx_root, i64 4909520559318251808, i32 2, i32 1)
+; LOWERING-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; LOWERING-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], 1
+; LOWERING-NEXT:    [[TMP4:%.*]] = call ptr @llvm.threadlocal.address.p0(ptr @__llvm_ctx_profile_expected_callee)
+; LOWERING-NEXT:    [[TMP5:%.*]] = getelementptr ptr, ptr [[TMP4]], i64 [[TMP3]]
+; LOWERING-NEXT:    [[TMP6:%.*]] = call ptr @llvm.threadlocal.address.p0(ptr @__llvm_ctx_profile_callsite)
+; LOWERING-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i64 [[TMP3]]
+; LOWERING-NEXT:    [[TMP8:%.*]] = and i64 [[TMP2]], -2
+; LOWERING-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; LOWERING-NEXT:    [[T:%.*]] = icmp eq i32 [[A]], 0
+; LOWERING-NEXT:    br i1 [[T]], label [[YES:%.*]], label [[NO:%.*]]
+; LOWERING:       yes:
+; LOWERING-NEXT:    [[TMP10:%.*]] = getelementptr { { i64, ptr, i32, i32 }, [2 x i64], [1 x ptr] }, ptr [[TMP9]], i32 0, i32 1, i32 1
+; LOWERING-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 4
+; LOWERING-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], 1
+; LOWERING-NEXT:    store i64 [[TMP12]], ptr [[TMP10]], align 4
+; LOWERING-NEXT:    store volatile ptr @foo, ptr [[TMP5]], align 8
+; LOWERING-NEXT:    [[TMP13:%.*]] = getelementptr { { i64, ptr, i32, i32 }, [2 x i64], [1 x ptr] }, ptr [[TMP1]], i32 0, i32 2, i32 0
+; LOWERING-NEXT:    store volatile ptr [[TMP13]], ptr [[TMP7]], align 8
+; LOWERING-NEXT:    call void @foo(i32 1, ptr null)
+; LOWERING-NEXT:    call void @__llvm_ctx_profile_release_context(ptr @an_entrypoint_ctx_root)
+; LOWERING-NEXT:    ret void
+; LOWERING:       no:
+; LOWERING-NEXT:    call void @__llvm_ctx_profile_release_context(ptr @an_entrypoint_ctx_root)
+; LOWERING-NEXT:    ret void
+;
+  %t = icmp eq i32 %a, 0
+  br i1 %t, label %yes, label %no
+
+yes:
+  call void @foo(i32 1, ptr null)
+  ret void
+no:
+  ret void
+}
+
+define void @another_entrypoint_no_callees(i32 %a) {
+; INSTRUMENT-LABEL: define void @another_entrypoint_no_callees(
+; INSTRUMENT-SAME: i32 [[A:%.*]]) {
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.increment(ptr @__profn_another_entrypoint_no_callees, i64 784007058953177093, i32 2, i32 0)
+; INSTRUMENT-NEXT:    [[T:%.*]] = icmp eq i32 [[A]], 0
+; INSTRUMENT-NEXT:    br i1 [[T]], label [[YES:%.*]], label [[NO:%.*]]
+; INSTRUMENT:       yes:
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.increment(ptr @__profn_another_entrypoint_no_callees, i64 784007058953177093, i32 2, i32 1)
+; INSTRUMENT-NEXT:    ret void
+; INSTRUMENT:       no:
+; INSTRUMENT-NEXT:    ret void
+;
+; LOWERING-LABEL: define void @another_entrypoint_no_callees(
+; LOWERING-SAME: i32 [[A:%.*]]) {
+; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_start_context(ptr @another_entrypoint_no_callees_ctx_root, i64 -6371873725078000974, i32 2, i32 0)
+; LOWERING-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; LOWERING-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], -2
+; LOWERING-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; LOWERING-NEXT:    [[T:%.*]] = icmp eq i32 [[A]], 0
+; LOWERING-NEXT:    br i1 [[T]], label [[YES:%.*]], label [[NO:%.*]]
+; LOWERING:       yes:
+; LOWERING-NEXT:    [[TMP5:%.*]] = getelementptr { { i64, ptr, i32, i32 }, [2 x i64], [0 x ptr] }, ptr [[TMP4]], i32 0, i32 1, i32 1
+; LOWERING-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP5]], align 4
+; LOWERING-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 1
+; LOWERING-NEXT:    store i64 [[TMP7]], ptr [[TMP5]], align 4
+; LOWERING-NEXT:    call void @__llvm_ctx_profile_release_context(ptr @another_entrypoint_no_callees_ctx_root)
+; LOWERING-NEXT:    ret void
+; LOWERING:       no:
+; LOWERING-NEXT:    call void @__llvm_ctx_profile_release_context(ptr @another_entrypoint_no_callees_ctx_root)
+; LOWERING-NEXT:    ret void
+;
+  %t = icmp eq i32 %a, 0
+  br i1 %t, label %yes, label %no
+
+yes:
+  ret void
+no:
+  ret void
+}
+
+define void @simple(i32 %a) {
+; INSTRUMENT-LABEL: define void @simple(
+; INSTRUMENT-SAME: i32 [[A:%.*]]) {
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.increment(ptr @__profn_simple, i64 742261418966908927, i32 1, i32 0)
+; INSTRUMENT-NEXT:    ret void
+;
+; LOWERING-LABEL: define void @simple(
+; LOWERING-SAME: i32 [[A:%.*]]) {
+; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @simple, i64 -3006003237940970099, i32 1, i32 0)
+; LOWERING-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; LOWERING-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], -2
+; LOWERING-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; LOWERING-NEXT:    ret void
+;
+  ret void
+}
+
+
+define i32 @no_callsites(i32 %a) {
+; INSTRUMENT-LABEL: define i32 @no_callsites(
+; INSTRUMENT-SAME: i32 [[A:%.*]]) {
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.increment(ptr @__profn_no_callsites, i64 784007058953177093, i32 2, i32 0)
+; INSTRUMENT-NEXT:    [[C:%.*]] = icmp eq i32 [[A]], 0
+; INSTRUMENT-NEXT:    br i1 [[C]], label [[YES:%.*]], label [[NO:%.*]]
+; INSTRUMENT:       yes:
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.increment(ptr @__profn_no_callsites, i64 784007058953177093, i32 2, i32 1)
+; INSTRUMENT-NEXT:    ret i32 1
+; INSTRUMENT:       no:
+; INSTRUMENT-NEXT:    ret i32 0
+;
+; LOWERING-LABEL: define i32 @no_callsites(
+; LOWERING-SAME: i32 [[A:%.*]]) {
+; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @no_callsites, i64 5679753335911435902, i32 2, i32 0)
+; LOWERING-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; LOWERING-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], -2
+; LOWERING-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; LOWERING-NEXT:    [[C:%.*]] = icmp eq i32 [[A]], 0
+; LOWERING-NEXT:    br i1 [[C]], label [[YES:%.*]], label [[NO:%.*]]
+; LOWERING:       yes:
+; LOWERING-NEXT:    [[TMP5:%.*]] = getelementptr { { i64, ptr, i32, i32 }, [2 x i64], [0 x ptr] }, ptr [[TMP4]], i32 0, i32 1, i32 1
+; LOWERING-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP5]], align 4
+; LOWERING-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 1
+; LOWERING-NEXT:    store i64 [[TMP7]], ptr [[TMP5]], align 4
+; LOWERING-NEXT:    ret i32 1
+; LOWERING:       no:
+; LOWERING-NEXT:    ret i32 0
+;
+  %c = icmp eq i32 %a, 0
+  br i1 %c, label %yes, label %no
+yes:
+  ret i32 1
+no:
+  ret i32 0
+}
+
+define void @no_counters() {
+; INSTRUMENT-LABEL: define void @no_counters() {
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.increment(ptr @__profn_no_counters, i64 742261418966908927, i32 1, i32 0)
+; INSTRUMENT-NEXT:    call void @llvm.instrprof.callsite(ptr @__profn_no_counters, i64 742261418966908927, i32 1, i32 0, ptr @bar)
+; INSTRUMENT-NEXT:    call void @bar()
+; INSTRUMENT-NEXT:    ret void
+;
+; LOWERING-LABEL: define void @no_counters() {
+; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @no_counters, i64 5458232184388660970, i32 1, i32 1)
+; LOWERING-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; LOWERING-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], 1
+; LOWERING-NEXT:    [[TMP4:%.*]] = call ptr @llvm.threadlocal.address.p0(ptr @__llvm_ctx_profile_expected_callee)
+; LOWERING-NEXT:    [[TMP5:%.*]] = getelementptr ptr, ptr [[TMP4]], i64 [[TMP3]]
+; LOWERING-NEXT:    [[TMP6:%.*]] = call ptr @llvm.threadlocal.address.p0(ptr @__llvm_ctx_profile_callsite)
+; LOWERING-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i64 [[TMP3]]
+; LOWERING-NEXT:    [[TMP8:%.*]] = and i64 [[TMP2]], -2
+; LOWERING-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; LOWERING-NEXT:    store volatile ptr @bar, ptr [[TMP5]], align 8
+; LOWERING-NEXT:    [[TMP10:%.*]] = getelementptr { { i64, ptr, i32, i32 }, [1 x i64], [1 x ptr] }, ptr [[TMP1]], i32 0, i32 2, i32 0
+; LOWERING-NEXT:    store volatile ptr [[TMP10]], ptr [[TMP7]], align 8
+; LOWERING-NEXT:    call void @bar()
+; LOWERING-NEXT:    ret void
+;
+  call void @bar()
+  ret void
+}
 ;.
 ; INSTRUMENT: attributes #[[ATTR0:[0-9]+]] = { nounwind }
 ;.
+; LOWERING: attributes #[[ATTR0:[0-9]+]] = { nounwind }
+; LOWERING: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+;.
-- 
GitLab


From 1aaab334c53d5c52ae337939e9c853e6e1061128 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 8 May 2024 17:22:18 -0700
Subject: [PATCH 0235/1206] [RISCV] Don't use std::vector<std::string> for
 split extensions in RISCVISAInfo::parseArchString. NFC (#91538)

We can use a SmallVector<StringRef>.

Adjust the code so we check for empty strings in the loop instead of
making a copy of the vector returned from StringRef::split.

This overlaps with #91532 which also removed the std::vector, but
that PR may be more controversial.
---
 llvm/lib/TargetParser/RISCVISAInfo.cpp | 49 ++++++++++----------------
 1 file changed, 18 insertions(+), 31 deletions(-)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 96590745b2eb..c553e330a878 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -500,24 +500,6 @@ RISCVISAInfo::parseNormalizedArchString(StringRef Arch) {
   return std::move(ISAInfo);
 }
 
-static Error splitExtsByUnderscore(StringRef Exts,
-                                   std::vector<std::string> &SplitExts) {
-  SmallVector<StringRef, 8> Split;
-  if (Exts.empty())
-    return Error::success();
-
-  Exts.split(Split, "_");
-
-  for (auto Ext : Split) {
-    if (Ext.empty())
-      return createStringError(errc::invalid_argument,
-                               "extension name missing after separator '_'");
-
-    SplitExts.push_back(Ext.str());
-  }
-  return Error::success();
-}
-
 static Error processMultiLetterExtension(
     StringRef RawExt,
     MapVector<std::string, RISCVISAUtils::ExtensionVersion,
@@ -714,20 +696,25 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
   Exts = Exts.drop_front(ConsumeLength);
   Exts.consume_front("_");
 
-  std::vector<std::string> SplitExts;
-  if (auto E = splitExtsByUnderscore(Exts, SplitExts))
-    return std::move(E);
+  SmallVector<StringRef, 8> SplitExts;
+  // Only split if the string is not empty. Otherwise the split will push an
+  // empty string into the vector.
+  if (!Exts.empty())
+    Exts.split(SplitExts, '_');
+
+  for (auto Ext : SplitExts) {
+    if (Ext.empty())
+      return createStringError(errc::invalid_argument,
+                               "extension name missing after separator '_'");
 
-  for (auto &Ext : SplitExts) {
-    StringRef CurrExt = Ext;
-    while (!CurrExt.empty()) {
-      if (RISCVISAUtils::AllStdExts.contains(CurrExt.front())) {
+    do {
+      if (RISCVISAUtils::AllStdExts.contains(Ext.front())) {
         if (auto E = processSingleLetterExtension(
-                CurrExt, SeenExtMap, IgnoreUnknown, EnableExperimentalExtension,
+                Ext, SeenExtMap, IgnoreUnknown, EnableExperimentalExtension,
                 ExperimentalExtensionVersionCheck))
           return std::move(E);
-      } else if (CurrExt.front() == 'z' || CurrExt.front() == 's' ||
-                 CurrExt.front() == 'x') {
+      } else if (Ext.front() == 'z' || Ext.front() == 's' ||
+                 Ext.front() == 'x') {
         // Handle other types of extensions other than the standard
         // general purpose and standard user-level extensions.
         // Parse the ISA string containing non-standard user-level
@@ -737,7 +724,7 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
         // version number (major, minor) and are separated by a single
         // underscore '_'. We do not enforce a canonical order for them.
         if (auto E = processMultiLetterExtension(
-                CurrExt, SeenExtMap, IgnoreUnknown, EnableExperimentalExtension,
+                Ext, SeenExtMap, IgnoreUnknown, EnableExperimentalExtension,
                 ExperimentalExtensionVersionCheck))
           return std::move(E);
         // Multi-letter extension must be seperate following extension with
@@ -747,9 +734,9 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
         // FIXME: Could it be ignored by IgnoreUnknown?
         return createStringError(errc::invalid_argument,
                                  "invalid standard user-level extension '" +
-                                     Twine(CurrExt.front()) + "'");
+                                     Twine(Ext.front()) + "'");
       }
-    }
+    } while (!Ext.empty());
   }
 
   // Check all Extensions are supported.
-- 
GitLab


From 409ff97aac00e5a677c90353b8b413c2bf46e28f Mon Sep 17 00:00:00 2001
From: AtariDreams <gfunni234@gmail.com>
Date: Wed, 8 May 2024 20:26:36 -0400
Subject: [PATCH 0236/1206] [InstCombine] Fix comment from #88193 (NFC)
 (#91427)

It is inaccurate and needs to be corrected.
---
 llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 8847de366713..ba297111d945 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -1259,7 +1259,7 @@ Instruction *InstCombinerImpl::visitLShr(BinaryOperator &I) {
       match(Op1, m_SpecificIntAllowPoison(BitWidth - 1)))
     return new ZExtInst(Builder.CreateIsNotNeg(X, "isnotneg"), Ty);
 
-  // ((X << nuw Z) sub nuw Y) >>u exact Z --> X sub nuw (Y >>u exact Z),
+  // ((X << nuw Z) sub nuw Y) >>u exact Z --> X sub nuw (Y >>u exact Z)
   Value *Y;
   if (I.isExact() &&
       match(Op0, m_OneUse(m_NUWSub(m_NUWShl(m_Value(X), m_Specific(Op1)),
@@ -1279,7 +1279,7 @@ Instruction *InstCombinerImpl::visitLShr(BinaryOperator &I) {
     case Instruction::And:
     case Instruction::Or:
     case Instruction::Xor:
-      // And does not work here, and sub is handled separately.
+      // Sub is handled separately.
       return true;
     }
   };
-- 
GitLab


From ba5170f430b027c6d290f57d7a5d7ba6ee2b265b Mon Sep 17 00:00:00 2001
From: AtariDreams <gfunni234@gmail.com>
Date: Wed, 8 May 2024 20:29:14 -0400
Subject: [PATCH 0237/1206] [InstCombine] Thwart complexity-based
 canonicalization in shl-add test (NFC) (#91413)

Fixed test for #88193
---
 llvm/test/Transforms/InstCombine/lshr.ll | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/lshr.ll b/llvm/test/Transforms/InstCombine/lshr.ll
index 563e669f9035..fa92c1c4b3be 100644
--- a/llvm/test/Transforms/InstCombine/lshr.ll
+++ b/llvm/test/Transforms/InstCombine/lshr.ll
@@ -397,12 +397,14 @@ define i32 @shl_add_lshr(i32 %x, i32 %c, i32 %y) {
 
 define i32 @shl_add_lshr_comm(i32 %x, i32 %c, i32 %y) {
 ; CHECK-LABEL: @shl_add_lshr_comm(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[Y2:%.*]] = mul i32 [[Y:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[Y2]], [[C:%.*]]
 ; CHECK-NEXT:    [[LSHR:%.*]] = add nuw i32 [[TMP1]], [[X:%.*]]
 ; CHECK-NEXT:    ret i32 [[LSHR]]
 ;
   %shl = shl nuw i32 %x, %c
-  %add = add nuw i32 %y, %shl
+  %y2 = mul i32 %y, %y ; thwart complexity-based canonicalization
+  %add = add nuw i32 %y2, %shl
   %lshr = lshr i32 %add, %c
   ret i32 %lshr
 }
-- 
GitLab


From 62b5b61f436add042d8729dc9837d055613180d9 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Wed, 8 May 2024 20:49:59 -0400
Subject: [PATCH 0238/1206] [Clang][Sema] Fix lookup of dependent operator=
 outside of complete-class contexts (#91498)

Fixes a crash caused by #90152.
---
 clang/lib/Sema/SemaLookup.cpp                 | 35 +++++++++++--------
 clang/lib/Sema/SemaTemplate.cpp               |  7 ++--
 .../temp.res/temp.dep/temp.dep.type/p4.cpp    | 13 +++++++
 3 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index e63da5875d2c..e20de338ebb1 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -1267,6 +1267,20 @@ struct FindLocalExternScope {
   LookupResult &R;
   bool OldFindLocalExtern;
 };
+
+/// Returns true if 'operator=' should be treated as a dependent name.
+bool isDependentAssignmentOperator(DeclarationName Name,
+                                   DeclContext *LookupContext) {
+  const auto *LookupRecord = dyn_cast_if_present<CXXRecordDecl>(LookupContext);
+  // If the lookup context is the current instantiation but we are outside a
+  // complete-class context, we will never find the implicitly declared
+  // copy/move assignment operators because they are declared at the closing '}'
+  // of the class specifier. In such cases, we treat 'operator=' like any other
+  // unqualified name because the results of name lookup in the template
+  // definition/instantiation context will always be the same.
+  return Name.getCXXOverloadedOperator() == OO_Equal && LookupRecord &&
+         !LookupRecord->isBeingDefined() && LookupRecord->isDependentContext();
+}
 } // end anonymous namespace
 
 bool Sema::CppLookupName(LookupResult &R, Scope *S) {
@@ -1275,13 +1289,6 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
   DeclarationName Name = R.getLookupName();
   Sema::LookupNameKind NameKind = R.getLookupKind();
 
-  // If this is the name of an implicitly-declared special member function,
-  // go through the scope stack to implicitly declare
-  if (isImplicitlyDeclaredMemberFunctionName(Name)) {
-    for (Scope *PreS = S; PreS; PreS = PreS->getParent())
-      if (DeclContext *DC = PreS->getEntity())
-        DeclareImplicitMemberFunctionsWithName(*this, Name, R.getNameLoc(), DC);
-  }
   // C++23 [temp.dep.general]p2:
   //   The component name of an unqualified-id is dependent if
   //   - it is a conversion-function-id whose conversion-type-id
@@ -1299,9 +1306,8 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
   if (isImplicitlyDeclaredMemberFunctionName(Name)) {
     for (Scope *PreS = S; PreS; PreS = PreS->getParent())
       if (DeclContext *DC = PreS->getEntity()) {
-        if (DC->isDependentContext() && isa<CXXRecordDecl>(DC) &&
-            Name.getCXXOverloadedOperator() == OO_Equal &&
-            !R.isTemplateNameLookup()) {
+        if (!R.isTemplateNameLookup() &&
+            isDependentAssignmentOperator(Name, DC)) {
           R.setNotFoundInCurrentInstantiation();
           return false;
         }
@@ -2472,8 +2478,6 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
     }
   } QL(LookupCtx);
 
-  bool TemplateNameLookup = R.isTemplateNameLookup();
-  CXXRecordDecl *LookupRec = dyn_cast<CXXRecordDecl>(LookupCtx);
   if (!InUnqualifiedLookup && !R.isForRedeclaration()) {
     // C++23 [temp.dep.type]p5:
     //   A qualified name is dependent if
@@ -2486,13 +2490,14 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
     if (DeclarationName Name = R.getLookupName();
         (Name.getNameKind() == DeclarationName::CXXConversionFunctionName &&
          Name.getCXXNameType()->isDependentType()) ||
-        (Name.getCXXOverloadedOperator() == OO_Equal && LookupRec &&
-         LookupRec->isDependentContext() && !TemplateNameLookup)) {
+        (!R.isTemplateNameLookup() &&
+         isDependentAssignmentOperator(Name, LookupCtx))) {
       R.setNotFoundInCurrentInstantiation();
       return false;
     }
   }
 
+  CXXRecordDecl *LookupRec = dyn_cast<CXXRecordDecl>(LookupCtx);
   if (LookupDirect(*this, R, LookupCtx)) {
     R.resolveKind();
     if (LookupRec)
@@ -2604,7 +2609,7 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
         //   template, and if the name is used as a template-name, the
         //   reference refers to the class template itself and not a
         //   specialization thereof, and is not ambiguous.
-        if (TemplateNameLookup)
+        if (R.isTemplateNameLookup())
           if (auto *TD = getAsTemplateNameDecl(ND))
             ND = TD;
 
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 7e57fa069672..480bc74c2001 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -726,7 +726,7 @@ Sema::ActOnDependentIdExpression(const CXXScopeSpec &SS,
                                  const DeclarationNameInfo &NameInfo,
                                  bool isAddressOfOperand,
                            const TemplateArgumentListInfo *TemplateArgs) {
-  DeclContext *DC = getFunctionLevelDeclContext();
+  QualType ThisType = getCurrentThisType();
 
   // C++11 [expr.prim.general]p12:
   //   An id-expression that denotes a non-static data member or non-static
@@ -748,10 +748,7 @@ Sema::ActOnDependentIdExpression(const CXXScopeSpec &SS,
     IsEnum = isa_and_nonnull<EnumType>(NNS->getAsType());
 
   if (!MightBeCxx11UnevalField && !isAddressOfOperand && !IsEnum &&
-      isa<CXXMethodDecl>(DC) &&
-      cast<CXXMethodDecl>(DC)->isImplicitObjectMemberFunction()) {
-    QualType ThisType = cast<CXXMethodDecl>(DC)->getThisType().getNonReferenceType();
-
+      !ThisType.isNull()) {
     // Since the 'this' expression is synthesized, we don't need to
     // perform the double-lookup check.
     NamedDecl *FirstQualifierInScope = nullptr;
diff --git a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
index 46dd52f8c4c1..43053c18c507 100644
--- a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
@@ -471,6 +471,19 @@ namespace N3 {
       this->C::operator=(*this);
     }
   };
+
+  template<typename T>
+  struct D {
+    auto not_instantiated() -> decltype(operator=(0)); // expected-error {{use of undeclared 'operator='}}
+  };
+
+  template<typename T>
+  struct E {
+    auto instantiated(E& e) -> decltype(operator=(e)); // expected-error {{use of undeclared 'operator='}}
+  };
+
+  template struct E<int>; // expected-note {{in instantiation of template class 'N3::E<int>' requested here}}
+
 } // namespace N3
 
 namespace N4 {
-- 
GitLab


From 73a01448c733bf08443435927677d0ebd133615b Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks@fb.com>
Date: Wed, 8 May 2024 17:56:44 -0700
Subject: [PATCH 0239/1206] [BOLT] Add test case for PIC fixed indirect jump
 (#91547)

A compiler can generate a redundant indirection for a jump via a fixed
jump table target. Add a test case that covers such pattern that covers
PIC case. We already have non-PIC case detection.

Currently XFAIL.
---
 .../X86/Inputs/jump-table-fixed-ref-pic.s     | 35 +++++++++++++++++++
 bolt/test/X86/jump-table-fixed-ref-pic.test   |  9 +++++
 2 files changed, 44 insertions(+)
 create mode 100644 bolt/test/X86/Inputs/jump-table-fixed-ref-pic.s
 create mode 100644 bolt/test/X86/jump-table-fixed-ref-pic.test

diff --git a/bolt/test/X86/Inputs/jump-table-fixed-ref-pic.s b/bolt/test/X86/Inputs/jump-table-fixed-ref-pic.s
new file mode 100644
index 000000000000..66629a4880e6
--- /dev/null
+++ b/bolt/test/X86/Inputs/jump-table-fixed-ref-pic.s
@@ -0,0 +1,35 @@
+  .globl main
+  .type main, %function
+main:
+  .cfi_startproc
+  cmpq $0x3, %rdi
+  jae .L4
+  cmpq $0x1, %rdi
+  jne .L4
+  mov .Ljt_pic+8(%rip), %rax
+  lea .Ljt_pic(%rip), %rdx
+  add %rdx, %rax
+  jmpq *%rax
+.L1:
+  movq $0x1, %rax
+  jmp .L5
+.L2:
+  movq $0x0, %rax
+  jmp .L5
+.L3:
+  movq $0x2, %rax
+  jmp .L5
+.L4:
+  mov $0x3, %rax
+.L5:
+  retq
+  .cfi_endproc
+
+  .section .rodata
+  .align 16
+.Ljt_pic:
+  .long .L1 - .Ljt_pic
+  .long .L2 - .Ljt_pic
+  .long .L3 - .Ljt_pic
+  .long .L4 - .Ljt_pic
+
diff --git a/bolt/test/X86/jump-table-fixed-ref-pic.test b/bolt/test/X86/jump-table-fixed-ref-pic.test
new file mode 100644
index 000000000000..4195b97aac50
--- /dev/null
+++ b/bolt/test/X86/jump-table-fixed-ref-pic.test
@@ -0,0 +1,9 @@
+# Verify that BOLT detects fixed destination of indirect jump for PIC
+# case.
+
+XFAIL: *
+
+RUN: %clang %cflags -no-pie %S/Inputs/jump-table-fixed-ref-pic.s -Wl,-q -o %t
+RUN: llvm-bolt %t --relocs -o %t.null 2>&1 | FileCheck %s
+
+CHECK: BOLT-INFO: fixed indirect branch detected in main
-- 
GitLab


From 51f178d909d477bd269e0b434af1a7f9373d4e61 Mon Sep 17 00:00:00 2001
From: Artem Dergachev <adergachev@apple.com>
Date: Wed, 8 May 2024 18:00:59 -0700
Subject: [PATCH 0240/1206] [analyzer] MallocChecker: Recognize std::atomics in
 smart pointer suppression. (#90918)

Fixes #90498.

Same as 5337efc69cdd5 for atomic builtins, but for `std::atomic` this
time. This is useful because even though the actual builtin atomic is
still there, it may be buried beyond the inlining depth limit.

Also add one popular custom smart pointer class name to the name-based
heuristics, which isn't necessary to fix the bug but arguably a good
idea regardless.
---
 .../StaticAnalyzer/Checkers/MallocChecker.cpp |  19 ++-
 .../Inputs/system-header-simulator-cxx.h      |   7 ++
 clang/test/Analysis/NewDelete-atomics.cpp     | 116 ++++++++++++++++--
 3 files changed, 130 insertions(+), 12 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index dd204b62dcc0..ab89fb14046b 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -3451,7 +3451,7 @@ static bool isReferenceCountingPointerDestructor(const CXXDestructorDecl *DD) {
     if (N.contains_insensitive("ptr") || N.contains_insensitive("pointer")) {
       if (N.contains_insensitive("ref") || N.contains_insensitive("cnt") ||
           N.contains_insensitive("intrusive") ||
-          N.contains_insensitive("shared")) {
+          N.contains_insensitive("shared") || N.ends_with_insensitive("rc")) {
         return true;
       }
     }
@@ -3483,13 +3483,24 @@ PathDiagnosticPieceRef MallocBugVisitor::VisitNode(const ExplodedNode *N,
   // original reference count is positive, we should not report use-after-frees
   // on objects deleted in such destructors. This can probably be improved
   // through better shared pointer modeling.
-  if (ReleaseDestructorLC) {
+  if (ReleaseDestructorLC && (ReleaseDestructorLC == CurrentLC ||
+                              ReleaseDestructorLC->isParentOf(CurrentLC))) {
     if (const auto *AE = dyn_cast<AtomicExpr>(S)) {
+      // Check for manual use of atomic builtins.
       AtomicExpr::AtomicOp Op = AE->getOp();
       if (Op == AtomicExpr::AO__c11_atomic_fetch_add ||
           Op == AtomicExpr::AO__c11_atomic_fetch_sub) {
-        if (ReleaseDestructorLC == CurrentLC ||
-            ReleaseDestructorLC->isParentOf(CurrentLC)) {
+        BR.markInvalid(getTag(), S);
+      }
+    } else if (const auto *CE = dyn_cast<CallExpr>(S)) {
+      // Check for `std::atomic` and such. This covers both regular method calls
+      // and operator calls.
+      if (const auto *MD =
+              dyn_cast_or_null<CXXMethodDecl>(CE->getDirectCallee())) {
+        const CXXRecordDecl *RD = MD->getParent();
+        // A bit wobbly with ".contains()" because it may be like
+        // "__atomic_base" or something.
+        if (StringRef(RD->getNameAsString()).contains("atomic")) {
           BR.markInvalid(getTag(), S);
         }
       }
diff --git a/clang/test/Analysis/Inputs/system-header-simulator-cxx.h b/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
index 1c2be322f83c..29326ec1f928 100644
--- a/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
+++ b/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
@@ -1260,6 +1260,13 @@ template<
     iterator end() const { return iterator(val + 1); }
 };
 
+template <typename T>
+class atomic {
+public:
+  T operator++();
+  T operator--();
+};
+
 namespace execution {
 class sequenced_policy {};
 }
diff --git a/clang/test/Analysis/NewDelete-atomics.cpp b/clang/test/Analysis/NewDelete-atomics.cpp
index 54fce17ea7bd..1425acab7489 100644
--- a/clang/test/Analysis/NewDelete-atomics.cpp
+++ b/clang/test/Analysis/NewDelete-atomics.cpp
@@ -20,7 +20,7 @@ typedef enum memory_order {
   memory_order_seq_cst = __ATOMIC_SEQ_CST
 } memory_order;
 
-class Obj {
+class RawObj {
   int RefCnt;
 
 public:
@@ -37,11 +37,27 @@ public:
   void foo();
 };
 
+class StdAtomicObj {
+  std::atomic<int> RefCnt;
+
+public:
+  int incRef() {
+    return ++RefCnt;
+  }
+
+  int decRef() {
+    return --RefCnt;
+  }
+
+  void foo();
+};
+
+template <typename T>
 class IntrusivePtr {
-  Obj *Ptr;
+  T *Ptr;
 
 public:
-  IntrusivePtr(Obj *Ptr) : Ptr(Ptr) {
+  IntrusivePtr(T *Ptr) : Ptr(Ptr) {
     Ptr->incRef();
   }
 
@@ -55,22 +71,106 @@ public:
       delete Ptr;
   }
 
-  Obj *getPtr() const { return Ptr; } // no-warning
+  T *getPtr() const { return Ptr; } // no-warning
+};
+
+// Also IntrusivePtr but let's dodge name-based heuristics.
+template <typename T>
+class DifferentlyNamed {
+  T *Ptr;
+
+public:
+  DifferentlyNamed(T *Ptr) : Ptr(Ptr) {
+    Ptr->incRef();
+  }
+
+  DifferentlyNamed(const DifferentlyNamed &Other) : Ptr(Other.Ptr) {
+    Ptr->incRef();
+  }
+
+  ~DifferentlyNamed() {
+  // We should not take the path on which the object is deleted.
+    if (Ptr->decRef() == 1)
+      delete Ptr;
+  }
+
+  T *getPtr() const { return Ptr; } // no-warning
 };
 
 void testDestroyLocalRefPtr() {
-  IntrusivePtr p1(new Obj());
+  IntrusivePtr<RawObj> p1(new RawObj());
+  {
+    IntrusivePtr<RawObj> p2(p1);
+  }
+
+  // p1 still maintains ownership. The object is not deleted.
+  p1.getPtr()->foo(); // no-warning
+}
+
+void testDestroySymbolicRefPtr(const IntrusivePtr<RawObj> &p1) {
+  {
+    IntrusivePtr<RawObj> p2(p1);
+  }
+
+  // p1 still maintains ownership. The object is not deleted.
+  p1.getPtr()->foo(); // no-warning
+}
+
+void testDestroyLocalRefPtrWithAtomics() {
+  IntrusivePtr<StdAtomicObj> p1(new StdAtomicObj());
+  {
+    IntrusivePtr<StdAtomicObj> p2(p1);
+  }
+
+  // p1 still maintains ownership. The object is not deleted.
+  p1.getPtr()->foo(); // no-warning
+}
+
+
+void testDestroyLocalRefPtrWithAtomics(const IntrusivePtr<StdAtomicObj> &p1) {
   {
-    IntrusivePtr p2(p1);
+    IntrusivePtr<StdAtomicObj> p2(p1);
   }
 
   // p1 still maintains ownership. The object is not deleted.
   p1.getPtr()->foo(); // no-warning
 }
 
-void testDestroySymbolicRefPtr(const IntrusivePtr &p1) {
+void testDestroyLocalRefPtrDifferentlyNamed() {
+  DifferentlyNamed<RawObj> p1(new RawObj());
+  {
+    DifferentlyNamed<RawObj> p2(p1);
+  }
+
+  // p1 still maintains ownership. The object is not deleted.
+  p1.getPtr()->foo(); // no-warning
+}
+
+void testDestroySymbolicRefPtrDifferentlyNamed(
+    const DifferentlyNamed<RawObj> &p1) {
+  {
+    DifferentlyNamed<RawObj> p2(p1);
+  }
+
+  // p1 still maintains ownership. The object is not deleted.
+  p1.getPtr()->foo(); // no-warning
+}
+
+void testDestroyLocalRefPtrWithAtomicsDifferentlyNamed() {
+  DifferentlyNamed<StdAtomicObj> p1(new StdAtomicObj());
+  {
+    DifferentlyNamed<StdAtomicObj> p2(p1);
+  }
+
+  // p1 still maintains ownership. The object is not deleted.
+  p1.getPtr()->foo(); // no-warning
+}
+
+
+void testDestroyLocalRefPtrWithAtomicsDifferentlyNamed(
+    const DifferentlyNamed<StdAtomicObj> &p1) {
   {
-    IntrusivePtr p2(p1);
+    DifferentlyNamed<StdAtomicObj> p2(p1);
   }
 
   // p1 still maintains ownership. The object is not deleted.
-- 
GitLab


From ea126aebdc9d8205016f355d85dbf1c15f2f4b28 Mon Sep 17 00:00:00 2001
From: "Felix (Ting Wang)" <Ting.Wang.SH@ibm.com>
Date: Thu, 9 May 2024 09:50:36 +0800
Subject: [PATCH 0241/1206] [PowerPC] Tune AIX shared library TLS model at
 function level (#84132)

Under some circumstance (library loaded with the main program), TLS
initial-exec model can be applied to local-dynamic access(es). We
could use some simple heuristic to decide the update at function level:
* If there is equal or less than a number of TLS local-dynamic access(es)
in the function, use TLS initial-exec model. (the threshold which default to
1 is controlled by hidden option)
---
 clang/include/clang/Driver/Options.td         |   4 +
 clang/lib/Basic/Targets/PPC.cpp               |   6 +
 clang/lib/Basic/Targets/PPC.h                 |   1 +
 llvm/lib/Target/PowerPC/PPC.td                |   6 +
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp     |  15 +-
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |  58 ++
 llvm/lib/Target/PowerPC/PPCMCInstLower.cpp    |  11 +-
 .../Target/PowerPC/PPCMachineFunctionInfo.h   |  12 +
 llvm/lib/Target/PowerPC/PPCSubtarget.cpp      |   5 +
 ...b-tls-model-opt-small-local-dynamic-tls.ll |  74 +++
 .../PowerPC/aix-shared-lib-tls-model-opt.ll   | 627 ++++++++++++++++++
 ...ix-shared-lib-tls-model-opt-IRattribute.ll |  21 +
 ...eck-aix-shared-lib-tls-model-opt-Option.ll |  22 +
 13 files changed, 859 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-shared-lib-tls-model-opt-small-local-dynamic-tls.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-shared-lib-tls-model-opt.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/check-aix-shared-lib-tls-model-opt-IRattribute.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/check-aix-shared-lib-tls-model-opt-Option.ll

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 322cc12af34a..142952897585 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5077,6 +5077,10 @@ def maix_small_local_dynamic_tls : Flag<["-"], "maix-small-local-dynamic-tls">,
            "where the offset from the TLS base is encoded as an "
            "immediate operand (AIX 64-bit only). "
            "This access sequence is not used for variables larger than 32KB.">;
+def maix_shared_lib_tls_model_opt : Flag<["-"], "maix-shared-lib-tls-model-opt">,
+  Group<m_ppc_Features_Group>,
+  HelpText<"For shared library loaded with the main program, change local-dynamic access(es) "
+           "to initial-exec access(es) at the function level (AIX 64-bit only).">;
 def maix_struct_return : Flag<["-"], "maix-struct-return">,
   Group<m_Group>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Return all structs in memory (PPC32 only)">,
diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index bad5259958a8..a1e5f20f7dbe 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -91,6 +91,8 @@ bool PPCTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       IsISA3_1 = true;
     } else if (Feature == "+quadword-atomics") {
       HasQuadwordAtomics = true;
+    } else if (Feature == "+aix-shared-lib-tls-model-opt") {
+      HasAIXShLibTLSModelOpt = true;
     }
     // TODO: Finish this list and add an assert that we've handled them
     // all.
@@ -580,6 +582,9 @@ bool PPCTargetInfo::initFeatureMap(
   Features["aix-small-local-exec-tls"] = false;
   Features["aix-small-local-dynamic-tls"] = false;
 
+  // Turn off TLS model opt by default.
+  Features["aix-shared-lib-tls-model-opt"] = false;
+
   Features["spe"] = llvm::StringSwitch<bool>(CPU)
                         .Case("8548", true)
                         .Case("e500", true)
@@ -722,6 +727,7 @@ bool PPCTargetInfo::hasFeature(StringRef Feature) const {
       .Case("isa-v30-instructions", IsISA3_0)
       .Case("isa-v31-instructions", IsISA3_1)
       .Case("quadword-atomics", HasQuadwordAtomics)
+      .Case("aix-shared-lib-tls-model-opt", HasAIXShLibTLSModelOpt)
       .Default(false);
 }
 
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index 30059e418e69..496b6131d09b 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -81,6 +81,7 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
   bool IsISA3_0 = false;
   bool IsISA3_1 = false;
   bool HasQuadwordAtomics = false;
+  bool HasAIXShLibTLSModelOpt = false;
 
 protected:
   std::string ABI;
diff --git a/llvm/lib/Target/PowerPC/PPC.td b/llvm/lib/Target/PowerPC/PPC.td
index b962ed28d720..639771ab9eab 100644
--- a/llvm/lib/Target/PowerPC/PPC.td
+++ b/llvm/lib/Target/PowerPC/PPC.td
@@ -338,6 +338,12 @@ def FeatureAIXLocalDynamicTLS :
                    "true", "Produce a faster local-dynamic TLS sequence for this "
                    "function for 64-bit AIX">;
 
+def FeatureAIXSharedLibTLSModelOpt :
+  SubtargetFeature<"aix-shared-lib-tls-model-opt",
+                   "HasAIXShLibTLSModelOpt", "true",
+                   "Tune TLS model at function level in shared library loaded "
+                   "with the main program (for 64-bit AIX only)">;
+
 def FeaturePredictableSelectIsExpensive :
   SubtargetFeature<"predictable-select-expensive",
                    "PredictableSelectIsExpensive",
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index a63824735490..ac48dc5af9d5 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -878,6 +878,15 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
         return MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSLE;
       if (Model == TLSModel::InitialExec)
         return MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSIE;
+      // On AIX, TLS model opt may have turned local-dynamic accesses into
+      // initial-exec accesses.
+      PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
+      if (Model == TLSModel::LocalDynamic &&
+          FuncInfo->isAIXFuncUseTLSIEForLD()) {
+        LLVM_DEBUG(
+            dbgs() << "Current function uses IE access for default LD vars.\n");
+        return MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSIE;
+      }
       llvm_unreachable("Only expecting local-exec or initial-exec accesses!");
     }
     // For GD TLS access on AIX, we have two TOC entries for the symbol (one for
@@ -2950,7 +2959,11 @@ void PPCAIXAsmPrinter::emitEndOfAsmFile(Module &M) {
     // Setup the csect for the current TC entry. If the variant kind is
     // VK_PPC_AIX_TLSGDM the entry represents the region handle, we create a
     // new symbol to prefix the name with a dot.
-    if (I.first.second == MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSGDM) {
+    // If TLS model opt is turned on, create a new symbol to prefix the name
+    // with a dot.
+    if (I.first.second == MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSGDM ||
+        (Subtarget->hasAIXShLibTLSModelOpt() &&
+         I.first.second == MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSLD)) {
       SmallString<128> Name;
       StringRef Prefix = ".";
       Name += Prefix;
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index d27932f2915f..0a7483fc45b2 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -141,6 +141,11 @@ static cl::opt<unsigned> PPCGatherAllAliasesMaxDepth(
     "ppc-gather-alias-max-depth", cl::init(18), cl::Hidden,
     cl::desc("max depth when checking alias info in GatherAllAliases()"));
 
+static cl::opt<unsigned> PPCAIXTLSModelOptUseIEForLDLimit(
+    "ppc-aix-shared-lib-tls-model-opt-limit", cl::init(1), cl::Hidden,
+    cl::desc("Set inclusive limit count of TLS local-dynamic access(es) in a "
+             "function to use initial-exec"));
+
 STATISTIC(NumTailCalls, "Number of tail calls");
 STATISTIC(NumSiblingCalls, "Number of sibling calls");
 STATISTIC(ShufflesHandledWithVPERM,
@@ -3362,6 +3367,54 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op,
   return LowerGlobalTLSAddressLinux(Op, DAG);
 }
 
+/// updateForAIXShLibTLSModelOpt - Helper to initialize TLS model opt settings,
+/// and then apply the update.
+static void updateForAIXShLibTLSModelOpt(TLSModel::Model &Model,
+                                         SelectionDAG &DAG,
+                                         const TargetMachine &TM) {
+  // Initialize TLS model opt setting lazily:
+  // (1) Use initial-exec for single TLS var references within current function.
+  // (2) Use local-dynamic for multiple TLS var references within current
+  // function.
+  PPCFunctionInfo *FuncInfo =
+      DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
+  if (!FuncInfo->isAIXFuncTLSModelOptInitDone()) {
+    SmallPtrSet<const GlobalValue *, 8> TLSGV;
+    // Iterate over all instructions within current function, collect all TLS
+    // global variables (global variables taken as the first parameter to
+    // Intrinsic::threadlocal_address).
+    const Function &Func = DAG.getMachineFunction().getFunction();
+    for (Function::const_iterator BI = Func.begin(), BE = Func.end(); BI != BE;
+         ++BI)
+      for (BasicBlock::const_iterator II = BI->begin(), IE = BI->end();
+           II != IE; ++II)
+        if (II->getOpcode() == Instruction::Call)
+          if (const CallInst *CI = dyn_cast<const CallInst>(&*II))
+            if (Function *CF = CI->getCalledFunction())
+              if (CF->isDeclaration() &&
+                  CF->getIntrinsicID() == Intrinsic::threadlocal_address)
+                if (const GlobalValue *GV =
+                        dyn_cast<GlobalValue>(II->getOperand(0))) {
+                  TLSModel::Model GVModel = TM.getTLSModel(GV);
+                  if (GVModel == TLSModel::LocalDynamic)
+                    TLSGV.insert(GV);
+                }
+
+    unsigned TLSGVCnt = TLSGV.size();
+    LLVM_DEBUG(dbgs() << format("LocalDynamic TLSGV count:%d\n", TLSGVCnt));
+    if (TLSGVCnt <= PPCAIXTLSModelOptUseIEForLDLimit)
+      FuncInfo->setAIXFuncUseTLSIEForLD();
+    FuncInfo->setAIXFuncTLSModelOptInitDone();
+  }
+
+  if (FuncInfo->isAIXFuncUseTLSIEForLD()) {
+    LLVM_DEBUG(
+        dbgs() << DAG.getMachineFunction().getName()
+               << " function is using the TLS-IE model for TLS-LD access.\n");
+    Model = TLSModel::InitialExec;
+  }
+}
+
 SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op,
                                                     SelectionDAG &DAG) const {
   GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
@@ -3374,6 +3427,11 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op,
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
   bool Is64Bit = Subtarget.isPPC64();
   TLSModel::Model Model = getTargetMachine().getTLSModel(GV);
+
+  // Apply update to the TLS model.
+  if (Subtarget.hasAIXShLibTLSModelOpt())
+    updateForAIXShLibTLSModelOpt(Model, DAG, getTargetMachine());
+
   bool IsTLSLocalExecModel = Model == TLSModel::LocalExec;
 
   if (IsTLSLocalExecModel || Model == TLSModel::InitialExec) {
diff --git a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
index c05bb37e58bf..31a261482358 100644
--- a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
@@ -13,6 +13,7 @@
 
 #include "MCTargetDesc/PPCMCExpr.h"
 #include "PPC.h"
+#include "PPCMachineFunctionInfo.h"
 #include "PPCSubtarget.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
@@ -81,6 +82,8 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol,
   }
 
   const TargetMachine &TM = Printer.TM;
+  const MachineInstr *MI = MO.getParent();
+  const MachineFunction *MF = MI->getMF();
 
   if (MO.getTargetFlags() == PPCII::MO_PLT)
     RefKind = MCSymbolRefExpr::VK_PLT;
@@ -100,18 +103,22 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol,
            MO.getTargetFlags() == PPCII::MO_TLSLD_FLAG) {
     assert(MO.isGlobal() && "Only expecting a global MachineOperand here!");
     TLSModel::Model Model = TM.getTLSModel(MO.getGlobal());
+    const PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
     // For the local-[exec|dynamic] TLS model, we may generate the offset from
     // the TLS base as an immediate operand (instead of using a TOC entry). Set
     // the relocation type in case the result is used for purposes other than a
     // TOC reference. In TOC reference cases, this result is discarded.
     if (Model == TLSModel::LocalExec)
       RefKind = MCSymbolRefExpr::VK_PPC_AIX_TLSLE;
+    else if (Model == TLSModel::LocalDynamic &&
+             FuncInfo->isAIXFuncUseTLSIEForLD())
+      // On AIX, TLS model opt may have turned local-dynamic accesses into
+      // initial-exec accesses.
+      RefKind = MCSymbolRefExpr::VK_PPC_AIX_TLSIE;
     else if (Model == TLSModel::LocalDynamic)
       RefKind = MCSymbolRefExpr::VK_PPC_AIX_TLSLD;
   }
 
-  const MachineInstr *MI = MO.getParent();
-  const MachineFunction *MF = MI->getMF();
   const Module *M = MF->getFunction().getParent();
   const PPCSubtarget *Subtarget = &(MF->getSubtarget<PPCSubtarget>());
 
diff --git a/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h b/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h
index df655a3be951..b7d14da05ee2 100644
--- a/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h
@@ -150,6 +150,11 @@ private:
   /// to use SExt/ZExt flags in later optimization.
   std::vector<std::pair<Register, ISD::ArgFlagsTy>> LiveInAttrs;
 
+  /// Flags for aix-shared-lib-tls-model-opt, will be lazily initialized for
+  /// each function.
+  bool AIXFuncUseTLSIEForLD = false;
+  bool AIXFuncTLSModelOptInitDone = false;
+
 public:
   explicit PPCFunctionInfo(const Function &F, const TargetSubtargetInfo *STI);
 
@@ -221,6 +226,13 @@ public:
   void setHasFastCall() { HasFastCall = true; }
   bool hasFastCall() const { return HasFastCall;}
 
+  void setAIXFuncTLSModelOptInitDone() { AIXFuncTLSModelOptInitDone = true; }
+  bool isAIXFuncTLSModelOptInitDone() const {
+    return AIXFuncTLSModelOptInitDone;
+  }
+  void setAIXFuncUseTLSIEForLD() { AIXFuncUseTLSIEForLD = true; }
+  bool isAIXFuncUseTLSIEForLD() const { return AIXFuncUseTLSIEForLD; }
+
   int getVarArgsFrameIndex() const { return VarArgsFrameIndex; }
   void setVarArgsFrameIndex(int Index) { VarArgsFrameIndex = Index; }
 
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index d1722555f1fc..0628fbb26245 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -141,6 +141,11 @@ void PPCSubtarget::initSubtargetFeatures(StringRef CPU, StringRef TuneCPU,
                          "-data-sections.\n",
                          false);
   }
+
+  if (HasAIXShLibTLSModelOpt && (!TargetTriple.isOSAIX() || !IsPPC64))
+    report_fatal_error("The aix-shared-lib-tls-model-opt attribute "
+                       "is only supported on AIX in 64-bit mode.\n",
+                       false);
 }
 
 bool PPCSubtarget::enableMachineScheduler() const { return true; }
diff --git a/llvm/test/CodeGen/PowerPC/aix-shared-lib-tls-model-opt-small-local-dynamic-tls.ll b/llvm/test/CodeGen/PowerPC/aix-shared-lib-tls-model-opt-small-local-dynamic-tls.ll
new file mode 100644
index 000000000000..cfb652ceeb8a
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-shared-lib-tls-model-opt-small-local-dynamic-tls.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      -mattr=+aix-shared-lib-tls-model-opt --code-model=large < %s | FileCheck %s --check-prefixes=OPT
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      -mattr=+aix-small-local-dynamic-tls --code-model=large < %s | FileCheck %s --check-prefixes=SMALL
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      -mattr=+aix-shared-lib-tls-model-opt -mattr=+aix-small-local-dynamic-tls \
+; RUN:      --code-model=large < %s | FileCheck %s --check-prefixes=BOTH
+
+@VarTLSLD1 = internal thread_local(localdynamic) global i32 42, align 4
+
+define i32 @Single_LD(i32 %P, i32 %Q) {
+; OPT-LABEL: Single_LD:
+; OPT:       # %bb.0: # %entry
+; OPT-NEXT:    and 4, 3, 4
+; OPT-NEXT:    addis 3, L..C0@u(2)
+; OPT-NEXT:    ld 3, L..C0@l(3)
+; OPT-NEXT:    cmpwi 4, -1
+; OPT-NEXT:    lwzx 3, 13, 3
+; OPT-NEXT:    blr
+;
+; SMALL-LABEL: Single_LD:
+; SMALL:       # %bb.0: # %entry
+; SMALL-NEXT:    mflr 0
+; SMALL-NEXT:    stdu 1, -48(1)
+; SMALL-NEXT:    and 6, 3, 4
+; SMALL-NEXT:    addis 3, L..C0@u(2)
+; SMALL-NEXT:    std 0, 64(1)
+; SMALL-NEXT:    ld 3, L..C0@l(3)
+; SMALL-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-NEXT:    cmpwi 6, -1
+; SMALL-NEXT:    lwz 3, VarTLSLD1[TL]@ld(3)
+; SMALL-NEXT:    addi 1, 1, 48
+; SMALL-NEXT:    ld 0, 16(1)
+; SMALL-NEXT:    mtlr 0
+; SMALL-NEXT:    blr
+;
+; BOTH-LABEL: Single_LD:
+; BOTH:       # %bb.0: # %entry
+; BOTH-NEXT:    and 4, 3, 4
+; BOTH-NEXT:    addis 3, L..C0@u(2)
+; BOTH-NEXT:    ld 3, L..C0@l(3)
+; BOTH-NEXT:    cmpwi 4, -1
+; BOTH-NEXT:    lwzx 3, 13, 3
+; BOTH-NEXT:    blr
+entry:
+  %a = icmp slt i32 %P, 0
+  %b = icmp slt i32 %Q, 0
+  %c = and i1 %a, %b
+  %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @VarTLSLD1)
+  %load1 = load i32, ptr %tls1, align 4
+  br i1 %c, label %bb1, label %return
+
+bb1:
+  %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @VarTLSLD1)
+  %load2 = load i32, ptr %tls2, align 4
+  ret i32 %load2
+
+return:
+  ret i32 %load1
+}
+
+; OPT-LABEL: .toc
+; OPT-LABEL: L..C0:
+; OPT-NEXT: .tc VarTLSLD1[TE],VarTLSLD1[TL]@ie
+
+; SMALL-LABEL: .toc
+; SMALL-LABEL: L..C0:
+; SMALL-NEXT: .tc _Renamed..5f24__TLSML[TC],_Renamed..5f24__TLSML[TC]@ml
+; SMALL-NEXT: .rename _Renamed..5f24__TLSML[TC],"_$TLSML"
+
+; BOTH-LABEL: .toc
+; BOTH-LABEL: L..C0:
+; BOTH-NEXT: .tc VarTLSLD1[TE],VarTLSLD1[TL]@ie
diff --git a/llvm/test/CodeGen/PowerPC/aix-shared-lib-tls-model-opt.ll b/llvm/test/CodeGen/PowerPC/aix-shared-lib-tls-model-opt.ll
new file mode 100644
index 000000000000..140377270d6d
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-shared-lib-tls-model-opt.ll
@@ -0,0 +1,627 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      --code-model=small < %s | FileCheck %s --check-prefixes=DEFAULT_SMALL64
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      --code-model=large < %s | FileCheck %s --check-prefixes=DEFAULT_LARGE64
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      -mattr=+aix-shared-lib-tls-model-opt --code-model=small < %s | FileCheck %s --check-prefixes=TLS_MODEL_OPT_SMALL64
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      -mattr=+aix-shared-lib-tls-model-opt --code-model=large < %s | FileCheck %s --check-prefixes=TLS_MODEL_OPT_LARGE64
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      -mattr=+aix-shared-lib-tls-model-opt -ppc-aix-shared-lib-tls-model-opt-limit=2 \
+; RUN:      --code-model=small < %s | FileCheck %s --check-prefixes=TLS_MODEL_OPT_LIMIT2_SMALL64
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      -mattr=+aix-shared-lib-tls-model-opt -ppc-aix-shared-lib-tls-model-opt-limit=2 \
+; RUN:      --code-model=large < %s | FileCheck %s --check-prefixes=TLS_MODEL_OPT_LIMIT2_LARGE64
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      -mattr=+aix-shared-lib-tls-model-opt -ppc-aix-shared-lib-tls-model-opt-limit=3 \
+; RUN:      --code-model=small < %s | FileCheck %s --check-prefixes=TLS_MODEL_OPT_LIMIT3_SMALL64
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      -mattr=+aix-shared-lib-tls-model-opt -ppc-aix-shared-lib-tls-model-opt-limit=3 \
+; RUN:      --code-model=large < %s | FileCheck %s --check-prefixes=TLS_MODEL_OPT_LIMIT3_LARGE64
+
+@VarTLSLD1 = internal thread_local(localdynamic) global i32 42, align 4
+@VarTLSLD2 = internal thread_local(localdynamic) global i32 0, align 4
+@VarTLSLD3 = internal thread_local(localdynamic) global i32 0, align 4
+
+; Tune function level TLS model settings:
+; Use initial-exec when we have a function accessing only one TLS variable.
+; Use local-dynamic when we have a function accessing a handful or more different TLS variables.
+
+define i32 @Single_LD(i32 %P, i32 %Q) {
+; DEFAULT_SMALL64-LABEL: Single_LD:
+; DEFAULT_SMALL64:       # %bb.0: # %entry
+; DEFAULT_SMALL64-NEXT:    mflr 0
+; DEFAULT_SMALL64-NEXT:    stdu 1, -48(1)
+; DEFAULT_SMALL64-NEXT:    and 6, 3, 4
+; DEFAULT_SMALL64-NEXT:    ld 3, L..C0(2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT_SMALL64-NEXT:    std 0, 64(1)
+; DEFAULT_SMALL64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT_SMALL64-NEXT:    ld 4, L..C1(2) # target-flags(ppc-tlsld) @VarTLSLD1
+; DEFAULT_SMALL64-NEXT:    cmpwi 6, -1
+; DEFAULT_SMALL64-NEXT:    lwzx 3, 3, 4
+; DEFAULT_SMALL64-NEXT:    addi 1, 1, 48
+; DEFAULT_SMALL64-NEXT:    ld 0, 16(1)
+; DEFAULT_SMALL64-NEXT:    mtlr 0
+; DEFAULT_SMALL64-NEXT:    blr
+;
+; DEFAULT_LARGE64-LABEL: Single_LD:
+; DEFAULT_LARGE64:       # %bb.0: # %entry
+; DEFAULT_LARGE64-NEXT:    mflr 0
+; DEFAULT_LARGE64-NEXT:    stdu 1, -48(1)
+; DEFAULT_LARGE64-NEXT:    and 6, 3, 4
+; DEFAULT_LARGE64-NEXT:    addis 3, L..C0@u(2)
+; DEFAULT_LARGE64-NEXT:    addis 7, L..C1@u(2)
+; DEFAULT_LARGE64-NEXT:    ld 3, L..C0@l(3)
+; DEFAULT_LARGE64-NEXT:    std 0, 64(1)
+; DEFAULT_LARGE64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT_LARGE64-NEXT:    ld 4, L..C1@l(7)
+; DEFAULT_LARGE64-NEXT:    lwzx 3, 3, 4
+; DEFAULT_LARGE64-NEXT:    cmpwi 6, -1
+; DEFAULT_LARGE64-NEXT:    addi 1, 1, 48
+; DEFAULT_LARGE64-NEXT:    ld 0, 16(1)
+; DEFAULT_LARGE64-NEXT:    mtlr 0
+; DEFAULT_LARGE64-NEXT:    blr
+;
+; TLS_MODEL_OPT_SMALL64-LABEL: Single_LD:
+; TLS_MODEL_OPT_SMALL64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_SMALL64-NEXT:    and 4, 3, 4
+; TLS_MODEL_OPT_SMALL64-NEXT:    ld 3, L..C0(2) # target-flags(ppc-tprel) @VarTLSLD1
+; TLS_MODEL_OPT_SMALL64-NEXT:    cmpwi 4, -1
+; TLS_MODEL_OPT_SMALL64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_SMALL64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LARGE64-LABEL: Single_LD:
+; TLS_MODEL_OPT_LARGE64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LARGE64-NEXT:    and 4, 3, 4
+; TLS_MODEL_OPT_LARGE64-NEXT:    addis 3, L..C0@u(2)
+; TLS_MODEL_OPT_LARGE64-NEXT:    ld 3, L..C0@l(3)
+; TLS_MODEL_OPT_LARGE64-NEXT:    cmpwi 4, -1
+; TLS_MODEL_OPT_LARGE64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LARGE64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT2_SMALL64-LABEL: Single_LD:
+; TLS_MODEL_OPT_LIMIT2_SMALL64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    and 4, 3, 4
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    ld 3, L..C0(2) # target-flags(ppc-tprel) @VarTLSLD1
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    cmpwi 4, -1
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT2_LARGE64-LABEL: Single_LD:
+; TLS_MODEL_OPT_LIMIT2_LARGE64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    and 4, 3, 4
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    addis 3, L..C0@u(2)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    ld 3, L..C0@l(3)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    cmpwi 4, -1
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT3_SMALL64-LABEL: Single_LD:
+; TLS_MODEL_OPT_LIMIT3_SMALL64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    and 4, 3, 4
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    ld 3, L..C0(2) # target-flags(ppc-tprel) @VarTLSLD1
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    cmpwi 4, -1
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT3_LARGE64-LABEL: Single_LD:
+; TLS_MODEL_OPT_LIMIT3_LARGE64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    and 4, 3, 4
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    addis 3, L..C0@u(2)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    ld 3, L..C0@l(3)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    cmpwi 4, -1
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    blr
+entry:
+  %a = icmp slt i32 %P, 0
+  %b = icmp slt i32 %Q, 0
+  %c = and i1 %a, %b
+  %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @VarTLSLD1)
+  %load1 = load i32, ptr %tls1, align 4
+  br i1 %c, label %bb1, label %return
+
+bb1:
+  %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @VarTLSLD1)
+  %load2 = load i32, ptr %tls2, align 4
+  ret i32 %load2
+
+return:
+  ret i32 %load1
+}
+
+define i32 @Two_LDs(i32 %P, i32 %Q) {
+; DEFAULT_SMALL64-LABEL: Two_LDs:
+; DEFAULT_SMALL64:       # %bb.0: # %entry
+; DEFAULT_SMALL64-NEXT:    mflr 0
+; DEFAULT_SMALL64-NEXT:    stdu 1, -48(1)
+; DEFAULT_SMALL64-NEXT:    and 6, 3, 4
+; DEFAULT_SMALL64-NEXT:    ld 3, L..C0(2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT_SMALL64-NEXT:    std 0, 64(1)
+; DEFAULT_SMALL64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT_SMALL64-NEXT:    cmpwi 6, -1
+; DEFAULT_SMALL64-NEXT:    bgt 0, L..BB1_2
+; DEFAULT_SMALL64-NEXT:  # %bb.1: # %bb1
+; DEFAULT_SMALL64-NEXT:    ld 4, L..C2(2) # target-flags(ppc-tlsld) @VarTLSLD2
+; DEFAULT_SMALL64-NEXT:    lwzx 3, 3, 4
+; DEFAULT_SMALL64-NEXT:    b L..BB1_3
+; DEFAULT_SMALL64-NEXT:  L..BB1_2: # %return
+; DEFAULT_SMALL64-NEXT:    ld 4, L..C1(2) # target-flags(ppc-tlsld) @VarTLSLD1
+; DEFAULT_SMALL64-NEXT:    lwzx 3, 3, 4
+; DEFAULT_SMALL64-NEXT:  L..BB1_3: # %bb1
+; DEFAULT_SMALL64-NEXT:    addi 1, 1, 48
+; DEFAULT_SMALL64-NEXT:    ld 0, 16(1)
+; DEFAULT_SMALL64-NEXT:    mtlr 0
+; DEFAULT_SMALL64-NEXT:    blr
+;
+; DEFAULT_LARGE64-LABEL: Two_LDs:
+; DEFAULT_LARGE64:       # %bb.0: # %entry
+; DEFAULT_LARGE64-NEXT:    mflr 0
+; DEFAULT_LARGE64-NEXT:    stdu 1, -48(1)
+; DEFAULT_LARGE64-NEXT:    and 6, 3, 4
+; DEFAULT_LARGE64-NEXT:    addis 3, L..C0@u(2)
+; DEFAULT_LARGE64-NEXT:    std 0, 64(1)
+; DEFAULT_LARGE64-NEXT:    ld 3, L..C0@l(3)
+; DEFAULT_LARGE64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT_LARGE64-NEXT:    cmpwi 6, -1
+; DEFAULT_LARGE64-NEXT:    bgt 0, L..BB1_2
+; DEFAULT_LARGE64-NEXT:  # %bb.1: # %bb1
+; DEFAULT_LARGE64-NEXT:    addis 4, L..C2@u(2)
+; DEFAULT_LARGE64-NEXT:    ld 4, L..C2@l(4)
+; DEFAULT_LARGE64-NEXT:    lwzx 3, 3, 4
+; DEFAULT_LARGE64-NEXT:    b L..BB1_3
+; DEFAULT_LARGE64-NEXT:  L..BB1_2: # %return
+; DEFAULT_LARGE64-NEXT:    addis 4, L..C1@u(2)
+; DEFAULT_LARGE64-NEXT:    ld 4, L..C1@l(4)
+; DEFAULT_LARGE64-NEXT:    lwzx 3, 3, 4
+; DEFAULT_LARGE64-NEXT:  L..BB1_3: # %bb1
+; DEFAULT_LARGE64-NEXT:    addi 1, 1, 48
+; DEFAULT_LARGE64-NEXT:    ld 0, 16(1)
+; DEFAULT_LARGE64-NEXT:    mtlr 0
+; DEFAULT_LARGE64-NEXT:    blr
+;
+; TLS_MODEL_OPT_SMALL64-LABEL: Two_LDs:
+; TLS_MODEL_OPT_SMALL64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_SMALL64-NEXT:    mflr 0
+; TLS_MODEL_OPT_SMALL64-NEXT:    stdu 1, -48(1)
+; TLS_MODEL_OPT_SMALL64-NEXT:    and 6, 3, 4
+; TLS_MODEL_OPT_SMALL64-NEXT:    ld 3, L..C1(2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; TLS_MODEL_OPT_SMALL64-NEXT:    std 0, 64(1)
+; TLS_MODEL_OPT_SMALL64-NEXT:    bla .__tls_get_mod[PR]
+; TLS_MODEL_OPT_SMALL64-NEXT:    cmpwi 6, -1
+; TLS_MODEL_OPT_SMALL64-NEXT:    bgt 0, L..BB1_2
+; TLS_MODEL_OPT_SMALL64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_SMALL64-NEXT:    ld 4, L..C2(2) # target-flags(ppc-tlsld) @VarTLSLD2
+; TLS_MODEL_OPT_SMALL64-NEXT:    lwzx 3, 3, 4
+; TLS_MODEL_OPT_SMALL64-NEXT:    b L..BB1_3
+; TLS_MODEL_OPT_SMALL64-NEXT:  L..BB1_2: # %return
+; TLS_MODEL_OPT_SMALL64-NEXT:    ld 4, L..C3(2) # target-flags(ppc-tlsld) @VarTLSLD1
+; TLS_MODEL_OPT_SMALL64-NEXT:    lwzx 3, 3, 4
+; TLS_MODEL_OPT_SMALL64-NEXT:  L..BB1_3: # %bb1
+; TLS_MODEL_OPT_SMALL64-NEXT:    addi 1, 1, 48
+; TLS_MODEL_OPT_SMALL64-NEXT:    ld 0, 16(1)
+; TLS_MODEL_OPT_SMALL64-NEXT:    mtlr 0
+; TLS_MODEL_OPT_SMALL64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LARGE64-LABEL: Two_LDs:
+; TLS_MODEL_OPT_LARGE64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LARGE64-NEXT:    mflr 0
+; TLS_MODEL_OPT_LARGE64-NEXT:    stdu 1, -48(1)
+; TLS_MODEL_OPT_LARGE64-NEXT:    and 6, 3, 4
+; TLS_MODEL_OPT_LARGE64-NEXT:    addis 3, L..C1@u(2)
+; TLS_MODEL_OPT_LARGE64-NEXT:    std 0, 64(1)
+; TLS_MODEL_OPT_LARGE64-NEXT:    ld 3, L..C1@l(3)
+; TLS_MODEL_OPT_LARGE64-NEXT:    bla .__tls_get_mod[PR]
+; TLS_MODEL_OPT_LARGE64-NEXT:    cmpwi 6, -1
+; TLS_MODEL_OPT_LARGE64-NEXT:    bgt 0, L..BB1_2
+; TLS_MODEL_OPT_LARGE64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_LARGE64-NEXT:    addis 4, L..C2@u(2)
+; TLS_MODEL_OPT_LARGE64-NEXT:    ld 4, L..C2@l(4)
+; TLS_MODEL_OPT_LARGE64-NEXT:    lwzx 3, 3, 4
+; TLS_MODEL_OPT_LARGE64-NEXT:    b L..BB1_3
+; TLS_MODEL_OPT_LARGE64-NEXT:  L..BB1_2: # %return
+; TLS_MODEL_OPT_LARGE64-NEXT:    addis 4, L..C3@u(2)
+; TLS_MODEL_OPT_LARGE64-NEXT:    ld 4, L..C3@l(4)
+; TLS_MODEL_OPT_LARGE64-NEXT:    lwzx 3, 3, 4
+; TLS_MODEL_OPT_LARGE64-NEXT:  L..BB1_3: # %bb1
+; TLS_MODEL_OPT_LARGE64-NEXT:    addi 1, 1, 48
+; TLS_MODEL_OPT_LARGE64-NEXT:    ld 0, 16(1)
+; TLS_MODEL_OPT_LARGE64-NEXT:    mtlr 0
+; TLS_MODEL_OPT_LARGE64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT2_SMALL64-LABEL: Two_LDs:
+; TLS_MODEL_OPT_LIMIT2_SMALL64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    and 3, 3, 4
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    cmpwi 3, -1
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    bgt 0, L..BB1_2
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    ld 3, L..C1(2) # target-flags(ppc-tprel) @VarTLSLD2
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    blr
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:  L..BB1_2: # %return
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    ld 3, L..C0(2) # target-flags(ppc-tprel) @VarTLSLD1
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT2_LARGE64-LABEL: Two_LDs:
+; TLS_MODEL_OPT_LIMIT2_LARGE64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    and 3, 3, 4
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    cmpwi 3, -1
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    bgt 0, L..BB1_2
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    addis 3, L..C1@u(2)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    ld 3, L..C1@l(3)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    blr
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:  L..BB1_2: # %return
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    addis 3, L..C0@u(2)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    ld 3, L..C0@l(3)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT3_SMALL64-LABEL: Two_LDs:
+; TLS_MODEL_OPT_LIMIT3_SMALL64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    and 3, 3, 4
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    cmpwi 3, -1
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    bgt 0, L..BB1_2
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    ld 3, L..C1(2) # target-flags(ppc-tprel) @VarTLSLD2
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    blr
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:  L..BB1_2: # %return
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    ld 3, L..C0(2) # target-flags(ppc-tprel) @VarTLSLD1
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT3_LARGE64-LABEL: Two_LDs:
+; TLS_MODEL_OPT_LIMIT3_LARGE64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    and 3, 3, 4
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    cmpwi 3, -1
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    bgt 0, L..BB1_2
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    addis 3, L..C1@u(2)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    ld 3, L..C1@l(3)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    blr
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:  L..BB1_2: # %return
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    addis 3, L..C0@u(2)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    ld 3, L..C0@l(3)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    blr
+entry:
+  %a = icmp slt i32 %P, 0
+  %b = icmp slt i32 %Q, 0
+  %c = and i1 %a, %b
+  %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @VarTLSLD1)
+  %load1 = load i32, ptr %tls1, align 4
+  br i1 %c, label %bb1, label %return
+
+bb1:
+  %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @VarTLSLD2)
+  %load2 = load i32, ptr %tls2, align 4
+  ret i32 %load2
+
+return:
+  ret i32 %load1
+}
+
+define i32 @Three_LDs(i32 %P, i32 %Q) {
+; DEFAULT_SMALL64-LABEL: Three_LDs:
+; DEFAULT_SMALL64:       # %bb.0: # %entry
+; DEFAULT_SMALL64-NEXT:    mflr 0
+; DEFAULT_SMALL64-NEXT:    stdu 1, -48(1)
+; DEFAULT_SMALL64-NEXT:    and 6, 3, 4
+; DEFAULT_SMALL64-NEXT:    ld 3, L..C0(2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT_SMALL64-NEXT:    std 0, 64(1)
+; DEFAULT_SMALL64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT_SMALL64-NEXT:    cmpwi 6, -1
+; DEFAULT_SMALL64-NEXT:    bgt 0, L..BB2_2
+; DEFAULT_SMALL64-NEXT:  # %bb.1: # %bb1
+; DEFAULT_SMALL64-NEXT:    ld 4, L..C2(2) # target-flags(ppc-tlsld) @VarTLSLD2
+; DEFAULT_SMALL64-NEXT:    ld 5, L..C3(2) # target-flags(ppc-tlsld) @VarTLSLD3
+; DEFAULT_SMALL64-NEXT:    lwzx 4, 3, 4
+; DEFAULT_SMALL64-NEXT:    lwzx 3, 3, 5
+; DEFAULT_SMALL64-NEXT:    add 3, 4, 3
+; DEFAULT_SMALL64-NEXT:    b L..BB2_3
+; DEFAULT_SMALL64-NEXT:  L..BB2_2: # %return
+; DEFAULT_SMALL64-NEXT:    ld 4, L..C1(2) # target-flags(ppc-tlsld) @VarTLSLD1
+; DEFAULT_SMALL64-NEXT:    lwzx 3, 3, 4
+; DEFAULT_SMALL64-NEXT:  L..BB2_3: # %return
+; DEFAULT_SMALL64-NEXT:    addi 1, 1, 48
+; DEFAULT_SMALL64-NEXT:    ld 0, 16(1)
+; DEFAULT_SMALL64-NEXT:    mtlr 0
+; DEFAULT_SMALL64-NEXT:    blr
+;
+; DEFAULT_LARGE64-LABEL: Three_LDs:
+; DEFAULT_LARGE64:       # %bb.0: # %entry
+; DEFAULT_LARGE64-NEXT:    mflr 0
+; DEFAULT_LARGE64-NEXT:    stdu 1, -48(1)
+; DEFAULT_LARGE64-NEXT:    and 6, 3, 4
+; DEFAULT_LARGE64-NEXT:    addis 3, L..C0@u(2)
+; DEFAULT_LARGE64-NEXT:    std 0, 64(1)
+; DEFAULT_LARGE64-NEXT:    ld 3, L..C0@l(3)
+; DEFAULT_LARGE64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT_LARGE64-NEXT:    cmpwi 6, -1
+; DEFAULT_LARGE64-NEXT:    bgt 0, L..BB2_2
+; DEFAULT_LARGE64-NEXT:  # %bb.1: # %bb1
+; DEFAULT_LARGE64-NEXT:    addis 4, L..C2@u(2)
+; DEFAULT_LARGE64-NEXT:    addis 5, L..C3@u(2)
+; DEFAULT_LARGE64-NEXT:    ld 4, L..C2@l(4)
+; DEFAULT_LARGE64-NEXT:    ld 5, L..C3@l(5)
+; DEFAULT_LARGE64-NEXT:    lwzx 4, 3, 4
+; DEFAULT_LARGE64-NEXT:    lwzx 3, 3, 5
+; DEFAULT_LARGE64-NEXT:    add 3, 4, 3
+; DEFAULT_LARGE64-NEXT:    b L..BB2_3
+; DEFAULT_LARGE64-NEXT:  L..BB2_2: # %return
+; DEFAULT_LARGE64-NEXT:    addis 4, L..C1@u(2)
+; DEFAULT_LARGE64-NEXT:    ld 4, L..C1@l(4)
+; DEFAULT_LARGE64-NEXT:    lwzx 3, 3, 4
+; DEFAULT_LARGE64-NEXT:  L..BB2_3: # %return
+; DEFAULT_LARGE64-NEXT:    addi 1, 1, 48
+; DEFAULT_LARGE64-NEXT:    ld 0, 16(1)
+; DEFAULT_LARGE64-NEXT:    mtlr 0
+; DEFAULT_LARGE64-NEXT:    blr
+;
+; TLS_MODEL_OPT_SMALL64-LABEL: Three_LDs:
+; TLS_MODEL_OPT_SMALL64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_SMALL64-NEXT:    mflr 0
+; TLS_MODEL_OPT_SMALL64-NEXT:    stdu 1, -48(1)
+; TLS_MODEL_OPT_SMALL64-NEXT:    and 6, 3, 4
+; TLS_MODEL_OPT_SMALL64-NEXT:    ld 3, L..C1(2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; TLS_MODEL_OPT_SMALL64-NEXT:    std 0, 64(1)
+; TLS_MODEL_OPT_SMALL64-NEXT:    bla .__tls_get_mod[PR]
+; TLS_MODEL_OPT_SMALL64-NEXT:    cmpwi 6, -1
+; TLS_MODEL_OPT_SMALL64-NEXT:    bgt 0, L..BB2_2
+; TLS_MODEL_OPT_SMALL64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_SMALL64-NEXT:    ld 4, L..C2(2) # target-flags(ppc-tlsld) @VarTLSLD2
+; TLS_MODEL_OPT_SMALL64-NEXT:    ld 5, L..C4(2) # target-flags(ppc-tlsld) @VarTLSLD3
+; TLS_MODEL_OPT_SMALL64-NEXT:    lwzx 4, 3, 4
+; TLS_MODEL_OPT_SMALL64-NEXT:    lwzx 3, 3, 5
+; TLS_MODEL_OPT_SMALL64-NEXT:    add 3, 4, 3
+; TLS_MODEL_OPT_SMALL64-NEXT:    b L..BB2_3
+; TLS_MODEL_OPT_SMALL64-NEXT:  L..BB2_2: # %return
+; TLS_MODEL_OPT_SMALL64-NEXT:    ld 4, L..C3(2) # target-flags(ppc-tlsld) @VarTLSLD1
+; TLS_MODEL_OPT_SMALL64-NEXT:    lwzx 3, 3, 4
+; TLS_MODEL_OPT_SMALL64-NEXT:  L..BB2_3: # %return
+; TLS_MODEL_OPT_SMALL64-NEXT:    addi 1, 1, 48
+; TLS_MODEL_OPT_SMALL64-NEXT:    ld 0, 16(1)
+; TLS_MODEL_OPT_SMALL64-NEXT:    mtlr 0
+; TLS_MODEL_OPT_SMALL64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LARGE64-LABEL: Three_LDs:
+; TLS_MODEL_OPT_LARGE64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LARGE64-NEXT:    mflr 0
+; TLS_MODEL_OPT_LARGE64-NEXT:    stdu 1, -48(1)
+; TLS_MODEL_OPT_LARGE64-NEXT:    and 6, 3, 4
+; TLS_MODEL_OPT_LARGE64-NEXT:    addis 3, L..C1@u(2)
+; TLS_MODEL_OPT_LARGE64-NEXT:    std 0, 64(1)
+; TLS_MODEL_OPT_LARGE64-NEXT:    ld 3, L..C1@l(3)
+; TLS_MODEL_OPT_LARGE64-NEXT:    bla .__tls_get_mod[PR]
+; TLS_MODEL_OPT_LARGE64-NEXT:    cmpwi 6, -1
+; TLS_MODEL_OPT_LARGE64-NEXT:    bgt 0, L..BB2_2
+; TLS_MODEL_OPT_LARGE64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_LARGE64-NEXT:    addis 4, L..C2@u(2)
+; TLS_MODEL_OPT_LARGE64-NEXT:    addis 5, L..C4@u(2)
+; TLS_MODEL_OPT_LARGE64-NEXT:    ld 4, L..C2@l(4)
+; TLS_MODEL_OPT_LARGE64-NEXT:    ld 5, L..C4@l(5)
+; TLS_MODEL_OPT_LARGE64-NEXT:    lwzx 4, 3, 4
+; TLS_MODEL_OPT_LARGE64-NEXT:    lwzx 3, 3, 5
+; TLS_MODEL_OPT_LARGE64-NEXT:    add 3, 4, 3
+; TLS_MODEL_OPT_LARGE64-NEXT:    b L..BB2_3
+; TLS_MODEL_OPT_LARGE64-NEXT:  L..BB2_2: # %return
+; TLS_MODEL_OPT_LARGE64-NEXT:    addis 4, L..C3@u(2)
+; TLS_MODEL_OPT_LARGE64-NEXT:    ld 4, L..C3@l(4)
+; TLS_MODEL_OPT_LARGE64-NEXT:    lwzx 3, 3, 4
+; TLS_MODEL_OPT_LARGE64-NEXT:  L..BB2_3: # %return
+; TLS_MODEL_OPT_LARGE64-NEXT:    addi 1, 1, 48
+; TLS_MODEL_OPT_LARGE64-NEXT:    ld 0, 16(1)
+; TLS_MODEL_OPT_LARGE64-NEXT:    mtlr 0
+; TLS_MODEL_OPT_LARGE64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT2_SMALL64-LABEL: Three_LDs:
+; TLS_MODEL_OPT_LIMIT2_SMALL64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    mflr 0
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    stdu 1, -48(1)
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    and 6, 3, 4
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    ld 3, L..C2(2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    std 0, 64(1)
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    bla .__tls_get_mod[PR]
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    cmpwi 6, -1
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    bgt 0, L..BB2_2
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    ld 4, L..C3(2) # target-flags(ppc-tlsld) @VarTLSLD2
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    ld 5, L..C4(2) # target-flags(ppc-tlsld) @VarTLSLD3
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    lwzx 4, 3, 4
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    lwzx 3, 3, 5
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    add 3, 4, 3
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    b L..BB2_3
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:  L..BB2_2: # %return
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    ld 4, L..C5(2) # target-flags(ppc-tlsld) @VarTLSLD1
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    lwzx 3, 3, 4
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:  L..BB2_3: # %return
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    addi 1, 1, 48
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    ld 0, 16(1)
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    mtlr 0
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT2_LARGE64-LABEL: Three_LDs:
+; TLS_MODEL_OPT_LIMIT2_LARGE64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    mflr 0
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    stdu 1, -48(1)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    and 6, 3, 4
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    addis 3, L..C2@u(2)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    std 0, 64(1)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    ld 3, L..C2@l(3)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    bla .__tls_get_mod[PR]
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    cmpwi 6, -1
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    bgt 0, L..BB2_2
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    addis 4, L..C3@u(2)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    addis 5, L..C4@u(2)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    ld 4, L..C3@l(4)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    ld 5, L..C4@l(5)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    lwzx 4, 3, 4
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    lwzx 3, 3, 5
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    add 3, 4, 3
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    b L..BB2_3
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:  L..BB2_2: # %return
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    addis 4, L..C5@u(2)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    ld 4, L..C5@l(4)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    lwzx 3, 3, 4
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:  L..BB2_3: # %return
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    addi 1, 1, 48
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    ld 0, 16(1)
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    mtlr 0
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT3_SMALL64-LABEL: Three_LDs:
+; TLS_MODEL_OPT_LIMIT3_SMALL64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    and 3, 3, 4
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    cmpwi 3, -1
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    bgt 0, L..BB2_2
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    ld 3, L..C1(2) # target-flags(ppc-tprel) @VarTLSLD2
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    ld 4, L..C2(2) # target-flags(ppc-tprel) @VarTLSLD3
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    lwzx 4, 13, 4
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    add 3, 3, 4
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    blr
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:  L..BB2_2: # %return
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    ld 3, L..C0(2) # target-flags(ppc-tprel) @VarTLSLD1
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT:    blr
+;
+; TLS_MODEL_OPT_LIMIT3_LARGE64-LABEL: Three_LDs:
+; TLS_MODEL_OPT_LIMIT3_LARGE64:       # %bb.0: # %entry
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    and 3, 3, 4
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    cmpwi 3, -1
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    bgt 0, L..BB2_2
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:  # %bb.1: # %bb1
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    addis 3, L..C1@u(2)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    addis 4, L..C2@u(2)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    ld 3, L..C1@l(3)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    ld 4, L..C2@l(4)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    lwzx 4, 13, 4
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    add 3, 3, 4
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    blr
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:  L..BB2_2: # %return
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    addis 3, L..C0@u(2)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    ld 3, L..C0@l(3)
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    lwzx 3, 13, 3
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT:    blr
+entry:
+  %a = icmp slt i32 %P, 0
+  %b = icmp slt i32 %Q, 0
+  %c = and i1 %a, %b
+  %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @VarTLSLD1)
+  %load1 = load i32, ptr %tls1, align 4
+  br i1 %c, label %bb1, label %return
+
+bb1:
+  %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @VarTLSLD2)
+  %load2 = load i32, ptr %tls2, align 4
+  %tls3 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @VarTLSLD3)
+  %load3 = load i32, ptr %tls3, align 4
+  %sum = add i32 %load2, %load3
+  ret i32 %sum
+
+return:
+  ret i32 %load1
+}
+
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull)
+
+; DEFAULT_SMALL64-LABEL: .toc
+; DEFAULT_SMALL64-LABEL: L..C0:
+; DEFAULT_SMALL64-NEXT: .tc _Renamed..5f24__TLSML[TC],_Renamed..5f24__TLSML[TC]@ml
+; DEFAULT_SMALL64-NEXT: .rename _Renamed..5f24__TLSML[TC],"_$TLSML"
+; DEFAULT_SMALL64-LABEL: L..C1:
+; DEFAULT_SMALL64-NEXT: .tc VarTLSLD1[TC],VarTLSLD1[TL]@ld
+; DEFAULT_SMALL64-LABEL: L..C2:
+; DEFAULT_SMALL64-NEXT: .tc VarTLSLD2[TC],VarTLSLD2[UL]@ld
+; DEFAULT_SMALL64-LABEL: L..C3:
+; DEFAULT_SMALL64-NEXT: .tc VarTLSLD3[TC],VarTLSLD3[UL]@ld
+
+; DEFAULT_LARGE64-LABEL: .toc
+; DEFAULT_LARGE64-LABEL: L..C0:
+; DEFAULT_LARGE64-NEXT: .tc _Renamed..5f24__TLSML[TC],_Renamed..5f24__TLSML[TC]@ml
+; DEFAULT_LARGE64-NEXT: .rename _Renamed..5f24__TLSML[TC],"_$TLSML"
+; DEFAULT_LARGE64-LABEL: L..C1:
+; DEFAULT_LARGE64-NEXT: .tc VarTLSLD1[TE],VarTLSLD1[TL]@ld
+; DEFAULT_LARGE64-LABEL: L..C2:
+; DEFAULT_LARGE64-NEXT: .tc VarTLSLD2[TE],VarTLSLD2[UL]@ld
+; DEFAULT_LARGE64-LABEL: L..C3:
+; DEFAULT_LARGE64-NEXT: .tc VarTLSLD3[TE],VarTLSLD3[UL]@ld
+
+; TLS_MODEL_OPT_SMALL64-LABEL: .toc
+; TLS_MODEL_OPT_SMALL64-LABEL: L..C0:
+; TLS_MODEL_OPT_SMALL64-NEXT: .tc VarTLSLD1[TC],VarTLSLD1[TL]@ie
+; TLS_MODEL_OPT_SMALL64-LABEL: L..C1:
+; TLS_MODEL_OPT_SMALL64-NEXT: .tc _Renamed..5f24__TLSML[TC],_Renamed..5f24__TLSML[TC]@ml
+; TLS_MODEL_OPT_SMALL64-NEXT: .rename _Renamed..5f24__TLSML[TC],"_$TLSML"
+; TLS_MODEL_OPT_SMALL64-LABEL: L..C2:
+; TLS_MODEL_OPT_SMALL64-NEXT: .tc .VarTLSLD2[TC],VarTLSLD2[UL]@ld
+; TLS_MODEL_OPT_SMALL64-LABEL: L..C3:
+; TLS_MODEL_OPT_SMALL64-NEXT: .tc .VarTLSLD1[TC],VarTLSLD1[TL]@ld
+; TLS_MODEL_OPT_SMALL64-LABEL: L..C4:
+; TLS_MODEL_OPT_SMALL64-NEXT: .tc .VarTLSLD3[TC],VarTLSLD3[UL]@ld
+
+; TLS_MODEL_OPT_LARGE64-LABEL: .toc
+; TLS_MODEL_OPT_LARGE64-LABEL: L..C0:
+; TLS_MODEL_OPT_LARGE64-NEXT: .tc VarTLSLD1[TE],VarTLSLD1[TL]@ie
+; TLS_MODEL_OPT_LARGE64-LABEL: L..C1:
+; TLS_MODEL_OPT_LARGE64-NEXT: .tc _Renamed..5f24__TLSML[TC],_Renamed..5f24__TLSML[TC]@ml
+; TLS_MODEL_OPT_LARGE64-NEXT: .rename _Renamed..5f24__TLSML[TC],"_$TLSML"
+; TLS_MODEL_OPT_LARGE64-LABEL: L..C2:
+; TLS_MODEL_OPT_LARGE64-NEXT: .tc .VarTLSLD2[TE],VarTLSLD2[UL]@ld
+; TLS_MODEL_OPT_LARGE64-LABEL: L..C3:
+; TLS_MODEL_OPT_LARGE64-NEXT: .tc .VarTLSLD1[TE],VarTLSLD1[TL]@ld
+; TLS_MODEL_OPT_LARGE64-LABEL: L..C4:
+; TLS_MODEL_OPT_LARGE64-NEXT: .tc .VarTLSLD3[TE],VarTLSLD3[UL]@ld
+
+; TLS_MODEL_OPT_LIMIT2_SMALL64-LABEL: .toc
+; TLS_MODEL_OPT_LIMIT2_SMALL64-LABEL: L..C0:
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT: .tc VarTLSLD1[TC],VarTLSLD1[TL]@ie
+; TLS_MODEL_OPT_LIMIT2_SMALL64-LABEL: L..C1:
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT: .tc VarTLSLD2[TC],VarTLSLD2[UL]@ie
+; TLS_MODEL_OPT_LIMIT2_SMALL64-LABEL: L..C2:
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT: .tc _Renamed..5f24__TLSML[TC],_Renamed..5f24__TLSML[TC]@ml
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT: .rename _Renamed..5f24__TLSML[TC],"_$TLSML"
+; TLS_MODEL_OPT_LIMIT2_SMALL64-LABEL: L..C3:
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT: .tc .VarTLSLD2[TC],VarTLSLD2[UL]@ld
+; TLS_MODEL_OPT_LIMIT2_SMALL64-LABEL: L..C4:
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT: .tc .VarTLSLD3[TC],VarTLSLD3[UL]@ld
+; TLS_MODEL_OPT_LIMIT2_SMALL64-LABEL: L..C5:
+; TLS_MODEL_OPT_LIMIT2_SMALL64-NEXT: .tc .VarTLSLD1[TC],VarTLSLD1[TL]@ld
+
+; TLS_MODEL_OPT_LIMIT2_LARGE64-LABEL: .toc
+; TLS_MODEL_OPT_LIMIT2_LARGE64-LABEL: L..C0:
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT: .tc VarTLSLD1[TE],VarTLSLD1[TL]@ie
+; TLS_MODEL_OPT_LIMIT2_LARGE64-LABEL: L..C1:
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT: .tc VarTLSLD2[TE],VarTLSLD2[UL]@ie
+; TLS_MODEL_OPT_LIMIT2_LARGE64-LABEL: L..C2:
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT: .tc _Renamed..5f24__TLSML[TC],_Renamed..5f24__TLSML[TC]@ml
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT: .rename _Renamed..5f24__TLSML[TC],"_$TLSML"
+; TLS_MODEL_OPT_LIMIT2_LARGE64-LABEL: L..C3:
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT: .tc .VarTLSLD2[TE],VarTLSLD2[UL]@ld
+; TLS_MODEL_OPT_LIMIT2_LARGE64-LABEL: L..C4:
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT: .tc .VarTLSLD3[TE],VarTLSLD3[UL]@ld
+; TLS_MODEL_OPT_LIMIT2_LARGE64-LABEL: L..C5:
+; TLS_MODEL_OPT_LIMIT2_LARGE64-NEXT: .tc .VarTLSLD1[TE],VarTLSLD1[TL]@ld
+
+; TLS_MODEL_OPT_LIMIT3_SMALL64-LABEL: .toc
+; TLS_MODEL_OPT_LIMIT3_SMALL64-LABEL: L..C0:
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT: .tc VarTLSLD1[TC],VarTLSLD1[TL]@ie
+; TLS_MODEL_OPT_LIMIT3_SMALL64-LABEL: L..C1:
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT: .tc VarTLSLD2[TC],VarTLSLD2[UL]@ie
+; TLS_MODEL_OPT_LIMIT3_SMALL64-LABEL: L..C2:
+; TLS_MODEL_OPT_LIMIT3_SMALL64-NEXT: .tc VarTLSLD3[TC],VarTLSLD3[UL]@ie
+
+; TLS_MODEL_OPT_LIMIT3_LARGE64-LABEL: .toc
+; TLS_MODEL_OPT_LIMIT3_LARGE64-LABEL: L..C0:
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT: .tc VarTLSLD1[TE],VarTLSLD1[TL]@ie
+; TLS_MODEL_OPT_LIMIT3_LARGE64-LABEL: L..C1:
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT: .tc VarTLSLD2[TE],VarTLSLD2[UL]@ie
+; TLS_MODEL_OPT_LIMIT3_LARGE64-LABEL: L..C2:
+; TLS_MODEL_OPT_LIMIT3_LARGE64-NEXT: .tc VarTLSLD3[TE],VarTLSLD3[UL]@ie
diff --git a/llvm/test/CodeGen/PowerPC/check-aix-shared-lib-tls-model-opt-IRattribute.ll b/llvm/test/CodeGen/PowerPC/check-aix-shared-lib-tls-model-opt-IRattribute.ll
new file mode 100644
index 000000000000..15fac2d0c0ad
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/check-aix-shared-lib-tls-model-opt-IRattribute.ll
@@ -0,0 +1,21 @@
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -ppc-asm-full-reg-names \
+; RUN:   < %s | FileCheck %s
+; RUN: not llc -mtriple powerpc-ibm-aix-xcoff -ppc-asm-full-reg-names \
+; RUN:   < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NOT-SUPPORTED
+; RUN: not llc -mtriple powerpc64le-unknown-linux-gnu -ppc-asm-full-reg-names \
+; RUN:   < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NOT-SUPPORTED
+
+define dso_local signext i32 @testWithIRAttr() #0 {
+entry:
+  ret i32 0
+}
+; Check that the aix-shared-lib-tls-model-opt attribute is not supported on Linux and AIX (32-bit).
+; CHECK-NOT-SUPPORTED: The aix-shared-lib-tls-model-opt attribute is only supported on AIX in 64-bit mode.
+
+; Make sure that the test was actually compiled successfully after using the
+; aix-shared-lib-tls-model-opt attribute.
+; CHECK-LABEL: testWithIRAttr:
+; CHECK:        li r3, 0
+; CHECK-NEXT:   blr
+
+attributes #0 = { "target-features"="+aix-shared-lib-tls-model-opt" }
diff --git a/llvm/test/CodeGen/PowerPC/check-aix-shared-lib-tls-model-opt-Option.ll b/llvm/test/CodeGen/PowerPC/check-aix-shared-lib-tls-model-opt-Option.ll
new file mode 100644
index 000000000000..36f8bc78c77a
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/check-aix-shared-lib-tls-model-opt-Option.ll
@@ -0,0 +1,22 @@
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -mattr=+aix-shared-lib-tls-model-opt \
+; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN: not llc -mtriple powerpc-ibm-aix-xcoff -mattr=+aix-shared-lib-tls-model-opt \
+; RUN:   -ppc-asm-full-reg-names < %s 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=CHECK-NOT-SUPPORTED
+; RUN: not llc -mtriple powerpc64le-unknown-linux-gnu -mattr=+aix-shared-lib-tls-model-opt \
+; RUN:   -ppc-asm-full-reg-names < %s 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=CHECK-NOT-SUPPORTED
+
+define dso_local signext i32 @testNoIRAttr() {
+entry:
+  ret i32 0
+}
+
+; Check that the aix-shared-lib-tls-model-opt attribute is not supported on Linux and AIX (32-bit).
+; CHECK-NOT-SUPPORTED: The aix-shared-lib-tls-model-opt attribute is only supported on AIX in 64-bit mode.
+
+; Make sure that the test was actually compiled successfully after using the
+; aix-shared-lib-tls-model-opt attribute.
+; CHECK-LABEL: testNoIRAttr:
+; CHECK:        li r3, 0
+; CHECK-NEXT:   blr
-- 
GitLab


From b910bebc300dafb30569cecc3017b446ea8eafa0 Mon Sep 17 00:00:00 2001
From: Zixu Wang <9819235+zixu-w@users.noreply.github.com>
Date: Wed, 8 May 2024 18:53:15 -0700
Subject: [PATCH 0242/1206] [llvm][MachO] Fix integer truncation in rebase/bind
 parsing (#89337)

`Count` and `Skip` should use `uint64_t` as they are encoded/decoded
using 64-bit ULEB128.

In `*_OPCODE_DO_*_ULEB_TIMES_SKIPPING_ULEB`, `Skip` could be encoded as
a two's complement for moving `SegmentOffset` backwards. Having a 32-bit
`Skip` truncates the encoded value and leads to a malformed
`AdvanceAmount`
and invalid `SegmentOffset` that extends past valid sections.
---
 llvm/include/llvm/Object/MachO.h              |  15 +-
 llvm/lib/Object/MachOObjectFile.cpp           |  20 +-
 .../Inputs/MachO/bind-negative-skip.yaml      | 499 ++++++++++++++++++
 .../test/Object/macho-bind-negative-skip.test |  17 +
 4 files changed, 534 insertions(+), 17 deletions(-)
 create mode 100644 llvm/test/Object/Inputs/MachO/bind-negative-skip.yaml
 create mode 100644 llvm/test/Object/macho-bind-negative-skip.test

diff --git a/llvm/include/llvm/Object/MachO.h b/llvm/include/llvm/Object/MachO.h
index 24f9954584ed..35350df78f8d 100644
--- a/llvm/include/llvm/Object/MachO.h
+++ b/llvm/include/llvm/Object/MachO.h
@@ -134,9 +134,9 @@ public:
   BindRebaseSegInfo(const MachOObjectFile *Obj);
 
   // Used to check a Mach-O Bind or Rebase entry for errors when iterating.
-  const char* checkSegAndOffsets(int32_t SegIndex, uint64_t SegOffset,
-                                 uint8_t PointerSize, uint32_t Count=1,
-                                 uint32_t Skip=0);
+  const char *checkSegAndOffsets(int32_t SegIndex, uint64_t SegOffset,
+                                 uint8_t PointerSize, uint64_t Count = 1,
+                                 uint64_t Skip = 0);
   // Used with valid SegIndex/SegOffset values from checked entries.
   StringRef segmentName(int32_t SegIndex);
   StringRef sectionName(int32_t SegIndex, uint64_t SegOffset);
@@ -576,8 +576,9 @@ public:
   //
   // This is used by MachOBindEntry::moveNext() to validate a MachOBindEntry.
   const char *BindEntryCheckSegAndOffsets(int32_t SegIndex, uint64_t SegOffset,
-                                         uint8_t PointerSize, uint32_t Count=1,
-                                          uint32_t Skip=0) const {
+                                          uint8_t PointerSize,
+                                          uint64_t Count = 1,
+                                          uint64_t Skip = 0) const {
     return BindRebaseSectionTable->checkSegAndOffsets(SegIndex, SegOffset,
                                                      PointerSize, Count, Skip);
   }
@@ -591,8 +592,8 @@ public:
   const char *RebaseEntryCheckSegAndOffsets(int32_t SegIndex,
                                             uint64_t SegOffset,
                                             uint8_t PointerSize,
-                                            uint32_t Count=1,
-                                            uint32_t Skip=0) const {
+                                            uint64_t Count = 1,
+                                            uint64_t Skip = 0) const {
     return BindRebaseSectionTable->checkSegAndOffsets(SegIndex, SegOffset,
                                                       PointerSize, Count, Skip);
   }
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index 06186ad362aa..ef390ceca218 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -3515,7 +3515,7 @@ void MachORebaseEntry::moveNext() {
     uint8_t Byte = *Ptr++;
     uint8_t ImmValue = Byte & MachO::REBASE_IMMEDIATE_MASK;
     uint8_t Opcode = Byte & MachO::REBASE_OPCODE_MASK;
-    uint32_t Count, Skip;
+    uint64_t Count, Skip;
     const char *error = nullptr;
     switch (Opcode) {
     case MachO::REBASE_OPCODE_DONE:
@@ -3854,7 +3854,7 @@ void MachOBindEntry::moveNext() {
     uint8_t Opcode = Byte & MachO::BIND_OPCODE_MASK;
     int8_t SignExtended;
     const uint8_t *SymStart;
-    uint32_t Count, Skip;
+    uint64_t Count, Skip;
     const char *error = nullptr;
     switch (Opcode) {
     case MachO::BIND_OPCODE_DONE:
@@ -4384,18 +4384,18 @@ BindRebaseSegInfo::BindRebaseSegInfo(const object::MachOObjectFile *Obj) {
 // that fully contains a pointer at that location. Multiple fixups in a bind
 // (such as with the BIND_OPCODE_DO_BIND_ULEB_TIMES_SKIPPING_ULEB opcode) can
 // be tested via the Count and Skip parameters.
-const char * BindRebaseSegInfo::checkSegAndOffsets(int32_t SegIndex,
-                                                   uint64_t SegOffset,
-                                                   uint8_t PointerSize,
-                                                   uint32_t Count,
-                                                   uint32_t Skip) {
+const char *BindRebaseSegInfo::checkSegAndOffsets(int32_t SegIndex,
+                                                  uint64_t SegOffset,
+                                                  uint8_t PointerSize,
+                                                  uint64_t Count,
+                                                  uint64_t Skip) {
   if (SegIndex == -1)
     return "missing preceding *_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB";
   if (SegIndex >= MaxSegIndex)
     return "bad segIndex (too large)";
-  for (uint32_t i = 0; i < Count; ++i) {
-    uint32_t Start = SegOffset + i * (PointerSize + Skip);
-    uint32_t End = Start + PointerSize;
+  for (uint64_t i = 0; i < Count; ++i) {
+    uint64_t Start = SegOffset + i * (PointerSize + Skip);
+    uint64_t End = Start + PointerSize;
     bool Found = false;
     for (const SectionInfo &SI : Sections) {
       if (SI.SegmentIndex != SegIndex)
diff --git a/llvm/test/Object/Inputs/MachO/bind-negative-skip.yaml b/llvm/test/Object/Inputs/MachO/bind-negative-skip.yaml
new file mode 100644
index 000000000000..aef5664a798f
--- /dev/null
+++ b/llvm/test/Object/Inputs/MachO/bind-negative-skip.yaml
@@ -0,0 +1,499 @@
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x100000C
+  cpusubtype:      0x0
+  filetype:        0x2
+  ncmds:           17
+  sizeofcmds:      1384
+  flags:           0x200085
+  reserved:        0x0
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __PAGEZERO
+    vmaddr:          0
+    vmsize:          4294967296
+    fileoff:         0
+    filesize:        0
+    maxprot:         0
+    initprot:        0
+    nsects:          0
+    flags:           0
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         472
+    segname:         __TEXT
+    vmaddr:          4294967296
+    vmsize:          16384
+    fileoff:         0
+    filesize:        16384
+    maxprot:         5
+    initprot:        5
+    nsects:          5
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x100003E58
+        size:            228
+        offset:          0x3E58
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         FF8300D1FD7B01A9FD430091E9030091080000B0080540F9280100F90000009000B03D9130000094E9030091080000B0080140F9280100F90000009000E03D9129000094280000B0080940F9E9030091280100F90000009000083E9122000094280000B0081140F9E9030091280100F90000009000243E911B000094280000B0081940F9E9030091280100F90000009000403E9114000094280000B0E80700F9082140F9E9030091280100F900000090005C3E910C000094E80740F9082140F9E9030091280100F90000009000783E910500009400008052FD7B41A9FF830091C0035FD6
+      - sectname:        __stubs
+        segname:         __TEXT
+        addr:            0x100003F3C
+        size:            12
+        offset:          0x3F3C
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000408
+        reserved1:       0x0
+        reserved2:       0xC
+        reserved3:       0x0
+        content:         300000B0100240F900021FD6
+      - sectname:        __stub_helper
+        segname:         __TEXT
+        addr:            0x100003F48
+        size:            36
+        offset:          0x3F48
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         310000B031220091F047BFA9100000B0100A40F900021FD650000018F9FFFF1700000000
+      - sectname:        __cstring
+        segname:         __TEXT
+        addr:            0x100003F6C
+        size:            57
+        offset:          0x3F6C
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         6D616C6C6F633A2025700A00667265653A2025700A00613A2025700A00623A2025700A00633A2025700A00643A2025700A00653A2025700A00
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x100003FA8
+        size:            88
+        offset:          0x3FA8
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         010000001C000000000000001C000000000000001C00000002000000583E000040000000400000003C3F00000000000040000000000000000000000000000000030000000C00010010000100000000000000000400000000
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         152
+    segname:         __DATA_CONST
+    vmaddr:          4294983680
+    vmsize:          16384
+    fileoff:         16384
+    filesize:        16384
+    maxprot:         3
+    initprot:        3
+    nsects:          1
+    flags:           16
+    Sections:
+      - sectname:        __got
+        segname:         __DATA_CONST
+        addr:            0x100004000
+        size:            24
+        offset:          0x4000
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x6
+        reserved1:       0x1
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         '000000000000000000000000000000000000000000000000'
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         232
+    segname:         __DATA
+    vmaddr:          4295000064
+    vmsize:          16384
+    fileoff:         32768
+    filesize:        16384
+    maxprot:         3
+    initprot:        3
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __la_symbol_ptr
+        segname:         __DATA
+        addr:            0x100008000
+        size:            8
+        offset:          0x8000
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x7
+        reserved1:       0x4
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         603F000001000000
+      - sectname:        __data
+        segname:         __DATA
+        addr:            0x100008008
+        size:            88
+        offset:          0x8008
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         '00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          4295016448
+    vmsize:          32768
+    fileoff:         49152
+    filesize:        19184
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      49152
+    rebase_size:     8
+    bind_off:        49160
+    bind_size:       72
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   49232
+    lazy_bind_size:  16
+    export_off:      49248
+    export_size:     96
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          49352
+    nsyms:           13
+    stroff:          49584
+    strsize:         104
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       1
+    iextdefsym:      1
+    nextdefsym:      7
+    iundefsym:       8
+    nundefsym:       5
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  49560
+    nindirectsyms:   5
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_LOAD_DYLINKER
+    cmdsize:         32
+    name:            12
+    Content:         '/usr/lib/dyld'
+    ZeroPadBytes:    7
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            2018719F-D4DC-3EE9-B8C3-3B790A01EAF7
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        1
+    minos:           917504
+    sdk:             918784
+    ntools:          1
+    Tools:
+      - tool:            3
+        version:         0
+  - cmd:             LC_SOURCE_VERSION
+    cmdsize:         16
+    version:         0
+  - cmd:             LC_MAIN
+    cmdsize:         24
+    entryoff:        15960
+    stacksize:       0
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       2
+      current_version: 88176642
+      compatibility_version: 65536
+    Content:         '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    6
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         49344
+    datasize:        8
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         49352
+    datasize:        0
+  - cmd:             LC_CODE_SIGNATURE
+    cmdsize:         16
+    dataoff:         49696
+    datasize:        18640
+LinkEditData:
+  RebaseOpcodes:
+    - Opcode:          REBASE_OPCODE_SET_TYPE_IMM
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             3
+      ExtraData:       [ 0x0 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DONE
+      Imm:             0
+  BindOpcodes:
+    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          _free
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             2
+      ULEBExtraData:   [ 0x0 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             3
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          _malloc
+    - Opcode:          BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             2
+      ULEBExtraData:   [ 0x8 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             3
+      ULEBExtraData:   [ 0x30 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x2, 0xFFFFFFFFFFFFFFF0 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          dyld_stub_binder
+    - Opcode:          BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             2
+      ULEBExtraData:   [ 0x10 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+  LazyBindOpcodes:
+    - Opcode:          BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             3
+      ULEBExtraData:   [ 0x0 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          _printf
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0
+    Address:         0x0
+    Other:           0x0
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      48
+        Name:            _
+        Flags:           0x0
+        Address:         0x0
+        Other:           0x0
+        ImportName:      ''
+        Children:
+          - TerminalSize:    2
+            NodeOffset:      9
+            Name:            _mh_execute_header
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    4
+            NodeOffset:      13
+            Name:            a
+            Flags:           0x0
+            Address:         0x8010
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    4
+            NodeOffset:      19
+            Name:            b
+            Flags:           0x0
+            Address:         0x8020
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    4
+            NodeOffset:      25
+            Name:            c
+            Flags:           0x0
+            Address:         0x8030
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    4
+            NodeOffset:      31
+            Name:            d
+            Flags:           0x0
+            Address:         0x8040
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    4
+            NodeOffset:      37
+            Name:            e
+            Flags:           0x0
+            Address:         0x8050
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    3
+            NodeOffset:      43
+            Name:            main
+            Flags:           0x0
+            Address:         0x3E58
+            Other:           0x0
+            ImportName:      ''
+  NameList:
+    - n_strx:          88
+      n_type:          0xE
+      n_sect:          8
+      n_desc:          0
+      n_value:         4295000072
+    - n_strx:          2
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          16
+      n_value:         4294967296
+    - n_strx:          22
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         4295000080
+    - n_strx:          25
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         4295000096
+    - n_strx:          28
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         4295000112
+    - n_strx:          31
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         4295000128
+    - n_strx:          34
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         4295000144
+    - n_strx:          37
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294983256
+    - n_strx:          43
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          256
+      n_value:         0
+    - n_strx:          49
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          256
+      n_value:         0
+    - n_strx:          57
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          256
+      n_value:         0
+    - n_strx:          65
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          256
+      n_value:         0
+    - n_strx:          71
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          256
+      n_value:         0
+  StringTable:
+    - ' '
+    - __mh_execute_header
+    - _a
+    - _b
+    - _c
+    - _d
+    - _e
+    - _main
+    - _free
+    - _malloc
+    - _printf
+    - _read
+    - dyld_stub_binder
+    - __dyld_private
+    - ''
+  IndirectSymbols: [ 0xA, 0x8, 0x9, 0xC, 0xA ]
+  FunctionStarts:  [ 0x3E58 ]
+...
diff --git a/llvm/test/Object/macho-bind-negative-skip.test b/llvm/test/Object/macho-bind-negative-skip.test
new file mode 100644
index 000000000000..26884a28ea46
--- /dev/null
+++ b/llvm/test/Object/macho-bind-negative-skip.test
@@ -0,0 +1,17 @@
+// A valid MachO object with a bind table containing an opcode
+// `BIND_OPCODE_DO_BIND_ULEB_TIMES_SKIPPING_ULEB` with negative skip value
+// (0xFFFFFFFFFFFFFFF0).
+
+RUN: yaml2obj %p/Inputs/MachO/bind-negative-skip.yaml | \
+RUN: llvm-objdump --bind --macho - | \
+RUN: FileCheck %s
+
+CHECK:      Bind table:
+CHECK-NEXT: segment      section            address     type       addend dylib            symbol
+CHECK-NEXT: __DATA_CONST __got              0x100004000 pointer         0 libSystem        _free
+CHECK-NEXT: __DATA       __data             0x100008040 pointer         0 libSystem        _free
+CHECK-NEXT: __DATA_CONST __got              0x100004008 pointer         0 libSystem        _malloc
+CHECK-NEXT: __DATA       __data             0x100008030 pointer         0 libSystem        _malloc
+CHECK-NEXT: __DATA       __data             0x100008028 pointer         0 libSystem        _malloc
+CHECK-NEXT: __DATA       __data             0x100008020 pointer         0 libSystem        _malloc
+CHECK-NEXT: __DATA_CONST __got              0x100004010 pointer         0 libSystem        dyld_stub_binder
-- 
GitLab


From a39a382755c8cf27ecd9a646e720610f48dc09ad Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 9 May 2024 12:29:01 +0900
Subject: [PATCH 0243/1206] [InstCombine] Thwart complexity-based
 canonicalization (NFC)

These tests did not test what they were supposed to. The transform
fails to actually handle the commuted cases.
---
 llvm/test/Transforms/InstCombine/add.ll | 59 +++++++++++++++++--------
 1 file changed, 40 insertions(+), 19 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll
index 56ee54d351e7..42e901ea2d5a 100644
--- a/llvm/test/Transforms/InstCombine/add.ll
+++ b/llvm/test/Transforms/InstCombine/add.ll
@@ -3284,12 +3284,17 @@ define i32 @add_reduce_sqr_sum_flipped(i32 %a, i32 %b) {
   ret i32 %add
 }
 
-define i32 @add_reduce_sqr_sum_flipped2(i32 %a, i32 %b) {
+define i32 @add_reduce_sqr_sum_flipped2(i32 %a, i32 %bx) {
 ; CHECK-LABEL: @add_reduce_sqr_sum_flipped2(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[ADD:%.*]] = mul i32 [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[A_SQ:%.*]] = mul nsw i32 [[A:%.*]], [[A]]
+; CHECK-NEXT:    [[TWO_A:%.*]] = shl i32 [[A]], 1
+; CHECK-NEXT:    [[TWO_A_PLUS_B:%.*]] = add i32 [[TWO_A]], [[B]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[B]], [[TWO_A_PLUS_B]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[MUL]], [[A_SQ]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
+  %b = xor i32 %bx, 42 ; thwart complexity-based canonicalization
   %a_sq = mul nsw i32 %a, %a
   %two_a = shl i32 %a, 1
   %two_a_plus_b = add i32 %two_a, %b
@@ -3342,12 +3347,17 @@ define i32 @add_reduce_sqr_sum_order2_flipped(i32 %a, i32 %b) {
   ret i32 %ab2
 }
 
-define i32 @add_reduce_sqr_sum_order2_flipped2(i32 %a, i32 %b) {
+define i32 @add_reduce_sqr_sum_order2_flipped2(i32 %a, i32 %bx) {
 ; CHECK-LABEL: @add_reduce_sqr_sum_order2_flipped2(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[AB2:%.*]] = mul i32 [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[A_SQ:%.*]] = mul nsw i32 [[A:%.*]], [[A]]
+; CHECK-NEXT:    [[TWOA:%.*]] = shl i32 [[A]], 1
+; CHECK-NEXT:    [[TWOAB1:%.*]] = add i32 [[B]], [[TWOA]]
+; CHECK-NEXT:    [[TWOAB_B2:%.*]] = mul i32 [[B]], [[TWOAB1]]
+; CHECK-NEXT:    [[AB2:%.*]] = add i32 [[A_SQ]], [[TWOAB_B2]]
 ; CHECK-NEXT:    ret i32 [[AB2]]
 ;
+  %b = xor i32 %bx, 42 ; thwart complexity-based canonicalization
   %a_sq = mul nsw i32 %a, %a
   %twoa = mul i32 %a, 2
   %twoab = mul i32 %twoa, %b
@@ -3357,12 +3367,17 @@ define i32 @add_reduce_sqr_sum_order2_flipped2(i32 %a, i32 %b) {
   ret i32 %ab2
 }
 
-define i32 @add_reduce_sqr_sum_order2_flipped3(i32 %a, i32 %b) {
+define i32 @add_reduce_sqr_sum_order2_flipped3(i32 %a, i32 %bx) {
 ; CHECK-LABEL: @add_reduce_sqr_sum_order2_flipped3(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[AB2:%.*]] = mul i32 [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[A_SQ:%.*]] = mul nsw i32 [[A:%.*]], [[A]]
+; CHECK-NEXT:    [[TWOA:%.*]] = shl i32 [[A]], 1
+; CHECK-NEXT:    [[B_SQ1:%.*]] = add i32 [[TWOA]], [[B]]
+; CHECK-NEXT:    [[TWOAB_B2:%.*]] = mul i32 [[B]], [[B_SQ1]]
+; CHECK-NEXT:    [[AB2:%.*]] = add i32 [[A_SQ]], [[TWOAB_B2]]
 ; CHECK-NEXT:    ret i32 [[AB2]]
 ;
+  %b = xor i32 %bx, 42 ; thwart complexity-based canonicalization
   %a_sq = mul nsw i32 %a, %a
   %twoa = mul i32 %a, 2
   %twoab = mul i32 %b, %twoa
@@ -3552,12 +3567,18 @@ define i32 @add_reduce_sqr_sum_order5_flipped2(i32 %a, i32 %b) {
   ret i32 %ab2
 }
 
-define i32 @add_reduce_sqr_sum_order5_flipped3(i32 %a, i32 %b) {
+define i32 @add_reduce_sqr_sum_order5_flipped3(i32 %ax, i32 %b) {
 ; CHECK-LABEL: @add_reduce_sqr_sum_order5_flipped3(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[B:%.*]], [[A:%.*]]
-; CHECK-NEXT:    [[AB2:%.*]] = mul i32 [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[A:%.*]] = xor i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[A_SQ:%.*]] = mul nsw i32 [[A]], [[A]]
+; CHECK-NEXT:    [[TWOB:%.*]] = shl i32 [[B:%.*]], 1
+; CHECK-NEXT:    [[TWOAB:%.*]] = mul i32 [[A]], [[TWOB]]
+; CHECK-NEXT:    [[B_SQ:%.*]] = mul i32 [[B]], [[B]]
+; CHECK-NEXT:    [[A2_B2:%.*]] = add i32 [[A_SQ]], [[B_SQ]]
+; CHECK-NEXT:    [[AB2:%.*]] = add i32 [[TWOAB]], [[A2_B2]]
 ; CHECK-NEXT:    ret i32 [[AB2]]
 ;
+  %a = xor i32 %ax, 42 ; thwart complexity-based canonicalization
   %a_sq = mul nsw i32 %a, %a
   %twob = mul i32 %b, 2
   %twoab = mul i32 %a, %twob
@@ -4018,8 +4039,8 @@ define i32 @add_reduce_sqr_sum_varC_invalid2(i32 %a, i32 %b) {
 
 define i32 @fold_sext_addition_or_disjoint(i8 %x) {
 ; CHECK-LABEL: @fold_sext_addition_or_disjoint(
-; CHECK-NEXT:    [[SE:%.*]] = sext i8 [[XX:%.*]] to i32
-; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[SE]], 1246
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i8 [[X:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[TMP1]], 1246
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %xx = or disjoint i8 %x, 12
@@ -4043,8 +4064,8 @@ define i32 @fold_sext_addition_fail(i8 %x) {
 
 define i32 @fold_zext_addition_or_disjoint(i8 %x) {
 ; CHECK-LABEL: @fold_zext_addition_or_disjoint(
-; CHECK-NEXT:    [[SE:%.*]] = zext i8 [[XX:%.*]] to i32
-; CHECK-NEXT:    [[R:%.*]] = add nuw nsw i32 [[SE]], 1246
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[X:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = add nuw nsw i32 [[TMP1]], 1246
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %xx = or disjoint i8 %x, 12
@@ -4055,9 +4076,9 @@ define i32 @fold_zext_addition_or_disjoint(i8 %x) {
 
 define i32 @fold_zext_addition_or_disjoint2(i8 %x) {
 ; CHECK-LABEL: @fold_zext_addition_or_disjoint2(
-; CHECK-NEXT:    [[XX:%.*]] = add nuw i8 [[X:%.*]], 4
-; CHECK-NEXT:    [[SE:%.*]] = zext i8 [[XX]] to i32
-; CHECK-NEXT:    ret i32 [[SE]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i8 [[X:%.*]], 4
+; CHECK-NEXT:    [[R:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
 ;
   %xx = or disjoint i8 %x, 18
   %se = zext i8 %xx to i32
-- 
GitLab


From 0d335f78e45341db53d9f956adcebbb2d2616c9a Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 9 May 2024 12:35:16 +0900
Subject: [PATCH 0244/1206] [InstCombine] Handle more commuted cases in
 matchesSquareSum()

---
 .../InstCombine/InstCombineAddSub.cpp         | 20 ++++++-------
 llvm/test/Transforms/InstCombine/add.ll       | 29 +++++--------------
 2 files changed, 18 insertions(+), 31 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 51ac77348ed9..bff09f567668 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1014,7 +1014,7 @@ static bool matchesSquareSum(BinaryOperator &I, Mul2Rhs M2Rhs, Value *&A,
   // (a * a) + (((a * 2) + b) * b)
   if (match(&I, m_c_BinOp(
                     AddOp, m_OneUse(m_BinOp(MulOp, m_Value(A), m_Deferred(A))),
-                    m_OneUse(m_BinOp(
+                    m_OneUse(m_c_BinOp(
                         MulOp,
                         m_c_BinOp(AddOp, m_BinOp(Mul2Op, m_Deferred(A), M2Rhs),
                                   m_Value(B)),
@@ -1025,16 +1025,16 @@ static bool matchesSquareSum(BinaryOperator &I, Mul2Rhs M2Rhs, Value *&A,
   // +
   // (a * a + b * b) or (b * b + a * a)
   return match(
-      &I,
-      m_c_BinOp(AddOp,
-                m_CombineOr(
-                    m_OneUse(m_BinOp(
-                        Mul2Op, m_BinOp(MulOp, m_Value(A), m_Value(B)), M2Rhs)),
-                    m_OneUse(m_BinOp(MulOp, m_BinOp(Mul2Op, m_Value(A), M2Rhs),
+      &I, m_c_BinOp(
+              AddOp,
+              m_CombineOr(
+                  m_OneUse(m_BinOp(
+                      Mul2Op, m_BinOp(MulOp, m_Value(A), m_Value(B)), M2Rhs)),
+                  m_OneUse(m_c_BinOp(MulOp, m_BinOp(Mul2Op, m_Value(A), M2Rhs),
                                      m_Value(B)))),
-                m_OneUse(m_c_BinOp(
-                    AddOp, m_BinOp(MulOp, m_Deferred(A), m_Deferred(A)),
-                    m_BinOp(MulOp, m_Deferred(B), m_Deferred(B))))));
+              m_OneUse(
+                  m_c_BinOp(AddOp, m_BinOp(MulOp, m_Deferred(A), m_Deferred(A)),
+                            m_BinOp(MulOp, m_Deferred(B), m_Deferred(B))))));
 }
 
 // Fold integer variations of a^2 + 2*a*b + b^2 -> (a + b)^2
diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll
index 42e901ea2d5a..25087fef68a1 100644
--- a/llvm/test/Transforms/InstCombine/add.ll
+++ b/llvm/test/Transforms/InstCombine/add.ll
@@ -3287,11 +3287,8 @@ define i32 @add_reduce_sqr_sum_flipped(i32 %a, i32 %b) {
 define i32 @add_reduce_sqr_sum_flipped2(i32 %a, i32 %bx) {
 ; CHECK-LABEL: @add_reduce_sqr_sum_flipped2(
 ; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BX:%.*]], 42
-; CHECK-NEXT:    [[A_SQ:%.*]] = mul nsw i32 [[A:%.*]], [[A]]
-; CHECK-NEXT:    [[TWO_A:%.*]] = shl i32 [[A]], 1
-; CHECK-NEXT:    [[TWO_A_PLUS_B:%.*]] = add i32 [[TWO_A]], [[B]]
-; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[B]], [[TWO_A_PLUS_B]]
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[MUL]], [[A_SQ]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = mul i32 [[TMP1]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %b = xor i32 %bx, 42 ; thwart complexity-based canonicalization
@@ -3350,11 +3347,8 @@ define i32 @add_reduce_sqr_sum_order2_flipped(i32 %a, i32 %b) {
 define i32 @add_reduce_sqr_sum_order2_flipped2(i32 %a, i32 %bx) {
 ; CHECK-LABEL: @add_reduce_sqr_sum_order2_flipped2(
 ; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BX:%.*]], 42
-; CHECK-NEXT:    [[A_SQ:%.*]] = mul nsw i32 [[A:%.*]], [[A]]
-; CHECK-NEXT:    [[TWOA:%.*]] = shl i32 [[A]], 1
-; CHECK-NEXT:    [[TWOAB1:%.*]] = add i32 [[B]], [[TWOA]]
-; CHECK-NEXT:    [[TWOAB_B2:%.*]] = mul i32 [[B]], [[TWOAB1]]
-; CHECK-NEXT:    [[AB2:%.*]] = add i32 [[A_SQ]], [[TWOAB_B2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    [[AB2:%.*]] = mul i32 [[TMP1]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[AB2]]
 ;
   %b = xor i32 %bx, 42 ; thwart complexity-based canonicalization
@@ -3370,11 +3364,8 @@ define i32 @add_reduce_sqr_sum_order2_flipped2(i32 %a, i32 %bx) {
 define i32 @add_reduce_sqr_sum_order2_flipped3(i32 %a, i32 %bx) {
 ; CHECK-LABEL: @add_reduce_sqr_sum_order2_flipped3(
 ; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BX:%.*]], 42
-; CHECK-NEXT:    [[A_SQ:%.*]] = mul nsw i32 [[A:%.*]], [[A]]
-; CHECK-NEXT:    [[TWOA:%.*]] = shl i32 [[A]], 1
-; CHECK-NEXT:    [[B_SQ1:%.*]] = add i32 [[TWOA]], [[B]]
-; CHECK-NEXT:    [[TWOAB_B2:%.*]] = mul i32 [[B]], [[B_SQ1]]
-; CHECK-NEXT:    [[AB2:%.*]] = add i32 [[A_SQ]], [[TWOAB_B2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    [[AB2:%.*]] = mul i32 [[TMP1]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[AB2]]
 ;
   %b = xor i32 %bx, 42 ; thwart complexity-based canonicalization
@@ -3570,12 +3561,8 @@ define i32 @add_reduce_sqr_sum_order5_flipped2(i32 %a, i32 %b) {
 define i32 @add_reduce_sqr_sum_order5_flipped3(i32 %ax, i32 %b) {
 ; CHECK-LABEL: @add_reduce_sqr_sum_order5_flipped3(
 ; CHECK-NEXT:    [[A:%.*]] = xor i32 [[AX:%.*]], 42
-; CHECK-NEXT:    [[A_SQ:%.*]] = mul nsw i32 [[A]], [[A]]
-; CHECK-NEXT:    [[TWOB:%.*]] = shl i32 [[B:%.*]], 1
-; CHECK-NEXT:    [[TWOAB:%.*]] = mul i32 [[A]], [[TWOB]]
-; CHECK-NEXT:    [[B_SQ:%.*]] = mul i32 [[B]], [[B]]
-; CHECK-NEXT:    [[A2_B2:%.*]] = add i32 [[A_SQ]], [[B_SQ]]
-; CHECK-NEXT:    [[AB2:%.*]] = add i32 [[TWOAB]], [[A2_B2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[A]], [[B:%.*]]
+; CHECK-NEXT:    [[AB2:%.*]] = mul i32 [[TMP1]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[AB2]]
 ;
   %a = xor i32 %ax, 42 ; thwart complexity-based canonicalization
-- 
GitLab


From f958a7348fcb27c3c6b07f1c8bdb902c7525b845 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 9 May 2024 12:43:00 +0900
Subject: [PATCH 0245/1206] [InstCombine] Fix name clashes in check lines (NFC)

These used both lower and upper case variants of the same name,
resulting in malformed check lines when regenerated.
---
 llvm/test/Transforms/InstCombine/call-guard.ll | 16 ++++++++--------
 llvm/test/Transforms/InstCombine/fast-math.ll  |  8 ++++----
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/call-guard.ll b/llvm/test/Transforms/InstCombine/call-guard.ll
index 6d9308bbbd81..358518b9bd1c 100644
--- a/llvm/test/Transforms/InstCombine/call-guard.ll
+++ b/llvm/test/Transforms/InstCombine/call-guard.ll
@@ -114,22 +114,22 @@ define void @negative_div(i32 %V1, i32 %D) {
 ; Highlight the limit of the window in a case which would otherwise be mergable
 define void @negative_window(i32 %V1, i32 %a, i32 %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: @negative_window(
-; CHECK-NEXT:    [[A:%.*]] = icmp slt i32 [[V1:%.*]], 0
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[A]], i32 123) [ "deopt"() ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[V1:%.*]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP1]], i32 123) [ "deopt"() ]
 ; CHECK-NEXT:    [[V2:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[V3:%.*]] = add i32 [[V2]], [[C:%.*]]
 ; CHECK-NEXT:    [[V4:%.*]] = add i32 [[V3]], [[D:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[V4]], 0
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[B]], i32 456) [ "deopt"() ]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[V4]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP2]], i32 456) [ "deopt"() ]
 ; CHECK-NEXT:    ret void
 ;
-  %A = icmp slt i32 %V1, 0
-  call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
+  %cmp1 = icmp slt i32 %V1, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %cmp1, i32 123 )[ "deopt"() ]
   %V2 = add i32 %a, %b
   %V3 = add i32 %V2, %c
   %V4 = add i32 %V3, %d
-  %B = icmp slt i32 %V4, 0
-  call void(i1, ...) @llvm.experimental.guard( i1 %B, i32 456 )[ "deopt"() ]
+  %cmp2 = icmp slt i32 %V4, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %cmp2, i32 456 )[ "deopt"() ]
   ret void
 }
 
diff --git a/llvm/test/Transforms/InstCombine/fast-math.ll b/llvm/test/Transforms/InstCombine/fast-math.ll
index 83f2091244e5..da403555ebe2 100644
--- a/llvm/test/Transforms/InstCombine/fast-math.ll
+++ b/llvm/test/Transforms/InstCombine/fast-math.ll
@@ -922,8 +922,8 @@ define float @test55(i1 %which, float %a) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = fadd float [[A:%.*]], 1.000000e+00
 ; CHECK-NEXT:    br label [[FINAL]]
 ; CHECK:       final:
-; CHECK-NEXT:    [[A:%.*]] = phi float [ 3.000000e+00, [[ENTRY:%.*]] ], [ [[TMP0]], [[DELAY]] ]
-; CHECK-NEXT:    ret float [[A]]
+; CHECK-NEXT:    [[PHI:%.*]] = phi float [ 3.000000e+00, [[ENTRY:%.*]] ], [ [[TMP0]], [[DELAY]] ]
+; CHECK-NEXT:    ret float [[PHI]]
 ;
 entry:
   br i1 %which, label %final, label %delay
@@ -932,7 +932,7 @@ delay:
   br label %final
 
 final:
-  %A = phi float [ 2.0, %entry ], [ %a, %delay ]
-  %value = fadd float %A, 1.0
+  %phi = phi float [ 2.0, %entry ], [ %a, %delay ]
+  %value = fadd float %phi, 1.0
   ret float %value
 }
-- 
GitLab


From 8f4f34f10345806b25b892d3df15951ee820de82 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Thu, 9 May 2024 11:46:34 +0800
Subject: [PATCH 0246/1206] [RISCV] Add test for vmerge.vvm that could have
 splat sunk. NFC

---
 .../CodeGen/RISCV/rvv/sink-splat-operands.ll  | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
index 9046c861c336..6e902e79896b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
@@ -5423,3 +5423,41 @@ vector.body:                                      ; preds = %vector.body, %entry
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
+
+define void @sink_splat_select(ptr nocapture %a, i32 signext %x) {
+; CHECK-LABEL: sink_splat_select:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.x v8, a1
+; CHECK-NEXT:    lui a1, 1
+; CHECK-NEXT:    add a1, a0, a1
+; CHECK-NEXT:    li a2, 42
+; CHECK-NEXT:  .LBB117_1: # %vector.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vle32.v v9, (a0)
+; CHECK-NEXT:    vmseq.vx v0, v9, a2
+; CHECK-NEXT:    vmerge.vvm v9, v9, v8, v0
+; CHECK-NEXT:    vse32.v v9, (a0)
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    bne a0, a1, .LBB117_1
+; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %x, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, ptr %a, i64 %index
+  %load = load <4 x i32>, ptr %0, align 4
+  %cond = icmp eq <4 x i32> %load, splat (i32 42)
+  %1 = select <4 x i1> %cond, <4 x i32> %broadcast.splat, <4 x i32> %load
+  store <4 x i32> %1, ptr %0, align 4
+  %index.next = add nuw i64 %index, 4
+  %2 = icmp eq i64 %index.next, 1024
+  br i1 %2, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body
+  ret void
+}
-- 
GitLab


From 73d423319c0957a9b16ed8d5fb7c8336729b9c38 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 9 May 2024 12:48:37 +0900
Subject: [PATCH 0247/1206] [InstCombine] Regenerate test checks (NFC)

---
 .../InstCombine/apint-and-xor-merge.ll        |   8 +-
 llvm/test/Transforms/InstCombine/apint-or.ll  |  90 +++----
 .../Transforms/InstCombine/trunc-binop-ext.ll | 228 ++++++++++--------
 3 files changed, 182 insertions(+), 144 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/apint-and-xor-merge.ll b/llvm/test/Transforms/InstCombine/apint-and-xor-merge.ll
index c904035f41ca..9810e5057d8a 100644
--- a/llvm/test/Transforms/InstCombine/apint-and-xor-merge.ll
+++ b/llvm/test/Transforms/InstCombine/apint-and-xor-merge.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; This test case checks that the merge of and/xor can work on arbitrary
 ; precision integers.
 
@@ -7,8 +7,8 @@
 ; (x &z ) ^ (y & z) -> (x ^ y) & z
 define i57 @test1(i57 %x, i57 %y, i57 %z) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[TMP61:%.*]] = xor i57 %x, %y
-; CHECK-NEXT:    [[TMP7:%.*]] = and i57 [[TMP61]], %z
+; CHECK-NEXT:    [[TMP61:%.*]] = xor i57 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i57 [[TMP61]], [[Z:%.*]]
 ; CHECK-NEXT:    ret i57 [[TMP7]]
 ;
   %tmp3 = and i57 %z, %x
@@ -20,7 +20,7 @@ define i57 @test1(i57 %x, i57 %y, i57 %z) {
 ; (x & y) ^ (x | y) -> x ^ y
 define i23 @test2(i23 %x, i23 %y, i23 %z) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[TMP7:%.*]] = xor i23 %y, %x
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i23 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    ret i23 [[TMP7]]
 ;
   %tmp3 = and i23 %y, %x
diff --git a/llvm/test/Transforms/InstCombine/apint-or.ll b/llvm/test/Transforms/InstCombine/apint-or.ll
index 939d151c21d2..38bffdf35a36 100644
--- a/llvm/test/Transforms/InstCombine/apint-or.ll
+++ b/llvm/test/Transforms/InstCombine/apint-or.ll
@@ -1,56 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 ; These tests are for Integer BitWidth <= 64 && BitWidth % 2 != 0.
+;; A | ~A == -1
 define i23 @test1(i23 %A) {
-    ;; A | ~A == -1
-    %NotA = xor i23 -1, %A
-    %B = or i23 %A, %NotA
-    ret i23 %B
-; CHECK-LABEL: @test1
-; CHECK-NEXT: ret i23 -1
+; CHECK-LABEL: define i23 @test1(
+; CHECK-SAME: i23 [[A:%.*]]) {
+; CHECK-NEXT:    ret i23 -1
+;
+  %NotA = xor i23 -1, %A
+  %B = or i23 %A, %NotA
+  ret i23 %B
 }
 
+;; If we have: ((V + N) & C1) | (V & C2)
+;; .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0
+;; replace with V+N.
 define i39 @test2(i39 %V, i39 %M) {
-    ;; If we have: ((V + N) & C1) | (V & C2)
-    ;; .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0
-    ;; replace with V+N.
-    %C1 = xor i39 274877906943, -1 ;; C2 = 274877906943
-    %N = and i39 %M, 274877906944
-    %A = add i39 %V, %N
-    %B = and i39 %A, %C1
-    %D = and i39 %V, 274877906943
-    %R = or i39 %B, %D
-    ret i39 %R
-; CHECK-LABEL: @test2
-; CHECK-NEXT: %N = and i39 %M, -274877906944
-; CHECK-NEXT: %A = add i39 %N, %V
-; CHECK-NEXT: ret i39 %A
+; CHECK-LABEL: define i39 @test2(
+; CHECK-SAME: i39 [[V:%.*]], i39 [[M:%.*]]) {
+; CHECK-NEXT:    [[N:%.*]] = and i39 [[M]], -274877906944
+; CHECK-NEXT:    [[A:%.*]] = add i39 [[N]], [[V]]
+; CHECK-NEXT:    ret i39 [[A]]
+;
+  %C1 = xor i39 274877906943, -1 ;; C2 = 274877906943
+  %N = and i39 %M, 274877906944
+  %A = add i39 %V, %N
+  %B = and i39 %A, %C1
+  %D = and i39 %V, 274877906943
+  %R = or i39 %B, %D
+  ret i39 %R
 }
 
 ; These tests are for Integer BitWidth > 64 && BitWidth <= 1024.
+;; A | ~A == -1
 define i1023 @test4(i1023 %A) {
-    ;; A | ~A == -1
-    %NotA = xor i1023 -1, %A
-    %B = or i1023 %A, %NotA
-    ret i1023 %B
-; CHECK-LABEL: @test4
-; CHECK-NEXT: ret i1023 -1
+; CHECK-LABEL: define i1023 @test4(
+; CHECK-SAME: i1023 [[A:%.*]]) {
+; CHECK-NEXT:    ret i1023 -1
+;
+  %NotA = xor i1023 -1, %A
+  %B = or i1023 %A, %NotA
+  ret i1023 %B
 }
 
+;; If we have: ((V + N) & C1) | (V & C2)
+;; .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0
+;; replace with V+N.
 define i399 @test5(i399 %V, i399 %M) {
-    ;; If we have: ((V + N) & C1) | (V & C2)
-    ;; .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0
-    ;; replace with V+N.
-    %C1 = xor i399 274877906943, -1 ;; C2 = 274877906943
-    %N = and i399 %M, 18446742974197923840
-    %A = add i399 %V, %N
-    %B = and i399 %A, %C1
-    %D = and i399 %V, 274877906943
-    %R = or i399 %B, %D
-    ret i399 %R
-; CHECK-LABEL: @test5
-; CHECK-NEXT: %N = and i399 %M, 18446742974197923840
-; CHECK-NEXT: %A = add i399 %N, %V
-; CHECK-NEXT: ret i399 %A
+; CHECK-LABEL: define i399 @test5(
+; CHECK-SAME: i399 [[V:%.*]], i399 [[M:%.*]]) {
+; CHECK-NEXT:    [[N:%.*]] = and i399 [[M]], 18446742974197923840
+; CHECK-NEXT:    [[A:%.*]] = add i399 [[N]], [[V]]
+; CHECK-NEXT:    ret i399 [[A]]
+;
+  %C1 = xor i399 274877906943, -1 ;; C2 = 274877906943
+  %N = and i399 %M, 18446742974197923840
+  %A = add i399 %V, %N
+  %B = and i399 %A, %C1
+  %D = and i399 %V, 274877906943
+  %R = or i399 %B, %D
+  ret i399 %R
 }
-
diff --git a/llvm/test/Transforms/InstCombine/trunc-binop-ext.ll b/llvm/test/Transforms/InstCombine/trunc-binop-ext.ll
index 787df081eef2..e3103906911a 100644
--- a/llvm/test/Transforms/InstCombine/trunc-binop-ext.ll
+++ b/llvm/test/Transforms/InstCombine/trunc-binop-ext.ll
@@ -1,9 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 define i16 @narrow_sext_and(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_sext_and(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = and i16 [[TMP1]], %x16
+; CHECK-LABEL: define i16 @narrow_sext_and(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = and i16 [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = sext i16 %x16 to i32
@@ -13,9 +15,10 @@ define i16 @narrow_sext_and(i16 %x16, i32 %y32) {
 }
 
 define i16 @narrow_zext_and(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_zext_and(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = and i16 [[TMP1]], %x16
+; CHECK-LABEL: define i16 @narrow_zext_and(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = and i16 [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = zext i16 %x16 to i32
@@ -25,9 +28,10 @@ define i16 @narrow_zext_and(i16 %x16, i32 %y32) {
 }
 
 define i16 @narrow_sext_or(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_sext_or(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = or i16 [[TMP1]], %x16
+; CHECK-LABEL: define i16 @narrow_sext_or(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = or i16 [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = sext i16 %x16 to i32
@@ -37,9 +41,10 @@ define i16 @narrow_sext_or(i16 %x16, i32 %y32) {
 }
 
 define i16 @narrow_zext_or(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_zext_or(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = or i16 [[TMP1]], %x16
+; CHECK-LABEL: define i16 @narrow_zext_or(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = or i16 [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = zext i16 %x16 to i32
@@ -49,9 +54,10 @@ define i16 @narrow_zext_or(i16 %x16, i32 %y32) {
 }
 
 define i16 @narrow_sext_xor(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_sext_xor(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = xor i16 [[TMP1]], %x16
+; CHECK-LABEL: define i16 @narrow_sext_xor(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = xor i16 [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = sext i16 %x16 to i32
@@ -61,9 +67,10 @@ define i16 @narrow_sext_xor(i16 %x16, i32 %y32) {
 }
 
 define i16 @narrow_zext_xor(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_zext_xor(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = xor i16 [[TMP1]], %x16
+; CHECK-LABEL: define i16 @narrow_zext_xor(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = xor i16 [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = zext i16 %x16 to i32
@@ -73,9 +80,10 @@ define i16 @narrow_zext_xor(i16 %x16, i32 %y32) {
 }
 
 define i16 @narrow_sext_add(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_sext_add(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = add i16 [[TMP1]], %x16
+; CHECK-LABEL: define i16 @narrow_sext_add(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = add i16 [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = sext i16 %x16 to i32
@@ -85,9 +93,10 @@ define i16 @narrow_sext_add(i16 %x16, i32 %y32) {
 }
 
 define i16 @narrow_zext_add(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_zext_add(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = add i16 [[TMP1]], %x16
+; CHECK-LABEL: define i16 @narrow_zext_add(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = add i16 [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = zext i16 %x16 to i32
@@ -97,9 +106,10 @@ define i16 @narrow_zext_add(i16 %x16, i32 %y32) {
 }
 
 define i16 @narrow_sext_sub(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_sext_sub(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = sub i16 %x16, [[TMP1]]
+; CHECK-LABEL: define i16 @narrow_sext_sub(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = sub i16 [[X16]], [[TMP1]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = sext i16 %x16 to i32
@@ -109,9 +119,10 @@ define i16 @narrow_sext_sub(i16 %x16, i32 %y32) {
 }
 
 define i16 @narrow_zext_sub(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_zext_sub(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = sub i16 %x16, [[TMP1]]
+; CHECK-LABEL: define i16 @narrow_zext_sub(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = sub i16 [[X16]], [[TMP1]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = zext i16 %x16 to i32
@@ -121,9 +132,10 @@ define i16 @narrow_zext_sub(i16 %x16, i32 %y32) {
 }
 
 define i16 @narrow_sext_mul(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_sext_mul(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = mul i16 [[TMP1]], %x16
+; CHECK-LABEL: define i16 @narrow_sext_mul(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = mul i16 [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = sext i16 %x16 to i32
@@ -133,9 +145,10 @@ define i16 @narrow_sext_mul(i16 %x16, i32 %y32) {
 }
 
 define i16 @narrow_zext_mul(i16 %x16, i32 %y32) {
-; CHECK-LABEL: @narrow_zext_mul(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
-; CHECK-NEXT:    [[R:%.*]] = mul i16 [[TMP1]], %x16
+; CHECK-LABEL: define i16 @narrow_zext_mul(
+; CHECK-SAME: i16 [[X16:%.*]], i32 [[Y32:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[Y32]] to i16
+; CHECK-NEXT:    [[R:%.*]] = mul i16 [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x32 = zext i16 %x16 to i32
@@ -148,10 +161,11 @@ define i16 @narrow_zext_mul(i16 %x16, i32 %y32) {
 ; canonicalization doesn't swap the binop operands. Use vector types to show those work too.
 
 define <2 x i16> @narrow_sext_and_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_sext_and_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_sext_and_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = and <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -162,10 +176,11 @@ define <2 x i16> @narrow_sext_and_commute(<2 x i16> %x16, <2 x i32> %y32) {
 }
 
 define <2 x i16> @narrow_zext_and_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_zext_and_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_zext_and_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = and <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -176,10 +191,11 @@ define <2 x i16> @narrow_zext_and_commute(<2 x i16> %x16, <2 x i32> %y32) {
 }
 
 define <2 x i16> @narrow_sext_or_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_sext_or_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_sext_or_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = or <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -190,10 +206,11 @@ define <2 x i16> @narrow_sext_or_commute(<2 x i16> %x16, <2 x i32> %y32) {
 }
 
 define <2 x i16> @narrow_zext_or_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_zext_or_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_zext_or_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = or <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -204,10 +221,11 @@ define <2 x i16> @narrow_zext_or_commute(<2 x i16> %x16, <2 x i32> %y32) {
 }
 
 define <2 x i16> @narrow_sext_xor_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_sext_xor_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_sext_xor_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = xor <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -218,10 +236,11 @@ define <2 x i16> @narrow_sext_xor_commute(<2 x i16> %x16, <2 x i32> %y32) {
 }
 
 define <2 x i16> @narrow_zext_xor_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_zext_xor_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_zext_xor_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = xor <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -232,10 +251,11 @@ define <2 x i16> @narrow_zext_xor_commute(<2 x i16> %x16, <2 x i32> %y32) {
 }
 
 define <2 x i16> @narrow_sext_add_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_sext_add_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_sext_add_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = add <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -246,10 +266,11 @@ define <2 x i16> @narrow_sext_add_commute(<2 x i16> %x16, <2 x i32> %y32) {
 }
 
 define <2 x i16> @narrow_zext_add_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_zext_add_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_zext_add_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = add <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -260,10 +281,11 @@ define <2 x i16> @narrow_zext_add_commute(<2 x i16> %x16, <2 x i32> %y32) {
 }
 
 define <2 x i16> @narrow_sext_sub_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_sext_sub_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_sext_sub_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = sub <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -274,10 +296,11 @@ define <2 x i16> @narrow_sext_sub_commute(<2 x i16> %x16, <2 x i32> %y32) {
 }
 
 define <2 x i16> @narrow_zext_sub_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_zext_sub_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_zext_sub_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = sub <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -288,10 +311,11 @@ define <2 x i16> @narrow_zext_sub_commute(<2 x i16> %x16, <2 x i32> %y32) {
 }
 
 define <2 x i16> @narrow_sext_mul_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_sext_mul_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_sext_mul_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = mul <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = mul <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -302,10 +326,11 @@ define <2 x i16> @narrow_sext_mul_commute(<2 x i16> %x16, <2 x i32> %y32) {
 }
 
 define <2 x i16> @narrow_zext_mul_commute(<2 x i16> %x16, <2 x i32> %y32) {
-; CHECK-LABEL: @narrow_zext_mul_commute(
-; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-LABEL: define <2 x i16> @narrow_zext_mul_commute(
+; CHECK-SAME: <2 x i16> [[X16:%.*]], <2 x i32> [[Y32:%.*]]) {
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> [[Y32]], <i32 7, i32 -17>
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = mul <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    [[R:%.*]] = mul <2 x i16> [[TMP1]], [[X16]]
 ; CHECK-NEXT:    ret <2 x i16> [[R]]
 ;
   %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
@@ -317,12 +342,13 @@ define <2 x i16> @narrow_zext_mul_commute(<2 x i16> %x16, <2 x i32> %y32) {
 
 ; Test cases for PR43580
 define i8 @narrow_zext_ashr_keep_trunc(i8 %i1, i8 %i2) {
-; CHECK-LABEL: @narrow_zext_ashr_keep_trunc(
-; CHECK-NEXT:    [[I1_EXT:%.*]] = sext i8 [[I1:%.*]] to i16
-; CHECK-NEXT:    [[I2_EXT:%.*]] = sext i8 [[I2:%.*]] to i16
+; CHECK-LABEL: define i8 @narrow_zext_ashr_keep_trunc(
+; CHECK-SAME: i8 [[I1:%.*]], i8 [[I2:%.*]]) {
+; CHECK-NEXT:    [[I1_EXT:%.*]] = sext i8 [[I1]] to i16
+; CHECK-NEXT:    [[I2_EXT:%.*]] = sext i8 [[I2]] to i16
 ; CHECK-NEXT:    [[SUB:%.*]] = add nsw i16 [[I1_EXT]], [[I2_EXT]]
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[SUB]], 1
-; CHECK-NEXT:    [[T:%.*]] = trunc i16 [[TMP1]] to i8
+; CHECK-NEXT:    [[SHIFT:%.*]] = lshr i16 [[SUB]], 1
+; CHECK-NEXT:    [[T:%.*]] = trunc i16 [[SHIFT]] to i8
 ; CHECK-NEXT:    ret i8 [[T]]
 ;
   %i1.ext = sext i8 %i1 to i32
@@ -334,12 +360,13 @@ define i8 @narrow_zext_ashr_keep_trunc(i8 %i1, i8 %i2) {
 }
 
 define i8 @narrow_zext_ashr_keep_trunc2(i9 %i1, i9 %i2) {
-; CHECK-LABEL: @narrow_zext_ashr_keep_trunc2(
-; CHECK-NEXT:    [[I1_EXT1:%.*]] = zext i9 [[I1:%.*]] to i16
-; CHECK-NEXT:    [[I2_EXT2:%.*]] = zext i9 [[I2:%.*]] to i16
-; CHECK-NEXT:    [[SUB:%.*]] = add nuw nsw i16 [[I1_EXT1]], [[I2_EXT2]]
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[SUB]], 1
-; CHECK-NEXT:    [[T:%.*]] = trunc i16 [[TMP1]] to i8
+; CHECK-LABEL: define i8 @narrow_zext_ashr_keep_trunc2(
+; CHECK-SAME: i9 [[I1:%.*]], i9 [[I2:%.*]]) {
+; CHECK-NEXT:    [[I1_EXT:%.*]] = zext i9 [[I1]] to i16
+; CHECK-NEXT:    [[I2_EXT:%.*]] = zext i9 [[I2]] to i16
+; CHECK-NEXT:    [[SUB:%.*]] = add nuw nsw i16 [[I1_EXT]], [[I2_EXT]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = lshr i16 [[SUB]], 1
+; CHECK-NEXT:    [[T:%.*]] = trunc i16 [[SHIFT]] to i8
 ; CHECK-NEXT:    ret i8 [[T]]
 ;
   %i1.ext = sext i9 %i1 to i64
@@ -351,12 +378,13 @@ define i8 @narrow_zext_ashr_keep_trunc2(i9 %i1, i9 %i2) {
 }
 
 define i7 @narrow_zext_ashr_keep_trunc3(i8 %i1, i8 %i2) {
-; CHECK-LABEL: @narrow_zext_ashr_keep_trunc3(
-; CHECK-NEXT:    [[I1_EXT1:%.*]] = zext i8 [[I1:%.*]] to i14
-; CHECK-NEXT:    [[I2_EXT2:%.*]] = zext i8 [[I2:%.*]] to i14
-; CHECK-NEXT:    [[SUB:%.*]] = add nuw nsw i14 [[I1_EXT1]], [[I2_EXT2]]
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i14 [[SUB]], 1
-; CHECK-NEXT:    [[T:%.*]] = trunc i14 [[TMP1]] to i7
+; CHECK-LABEL: define i7 @narrow_zext_ashr_keep_trunc3(
+; CHECK-SAME: i8 [[I1:%.*]], i8 [[I2:%.*]]) {
+; CHECK-NEXT:    [[I1_EXT:%.*]] = zext i8 [[I1]] to i14
+; CHECK-NEXT:    [[I2_EXT:%.*]] = zext i8 [[I2]] to i14
+; CHECK-NEXT:    [[SUB:%.*]] = add nuw nsw i14 [[I1_EXT]], [[I2_EXT]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = lshr i14 [[SUB]], 1
+; CHECK-NEXT:    [[T:%.*]] = trunc i14 [[SHIFT]] to i7
 ; CHECK-NEXT:    ret i7 [[T]]
 ;
   %i1.ext = sext i8 %i1 to i64
@@ -368,12 +396,13 @@ define i7 @narrow_zext_ashr_keep_trunc3(i8 %i1, i8 %i2) {
 }
 
 define <8 x i8> @narrow_zext_ashr_keep_trunc_vector(<8 x i8> %i1, <8 x i8> %i2) {
-; CHECK-LABEL: @narrow_zext_ashr_keep_trunc_vector(
-; CHECK-NEXT:    [[I1_EXT:%.*]] = sext <8 x i8> [[I1:%.*]] to <8 x i32>
-; CHECK-NEXT:    [[I2_EXT:%.*]] = sext <8 x i8> [[I2:%.*]] to <8 x i32>
+; CHECK-LABEL: define <8 x i8> @narrow_zext_ashr_keep_trunc_vector(
+; CHECK-SAME: <8 x i8> [[I1:%.*]], <8 x i8> [[I2:%.*]]) {
+; CHECK-NEXT:    [[I1_EXT:%.*]] = sext <8 x i8> [[I1]] to <8 x i32>
+; CHECK-NEXT:    [[I2_EXT:%.*]] = sext <8 x i8> [[I2]] to <8 x i32>
 ; CHECK-NEXT:    [[SUB:%.*]] = add nsw <8 x i32> [[I1_EXT]], [[I2_EXT]]
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> [[SUB]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; CHECK-NEXT:    [[T:%.*]] = trunc <8 x i32> [[TMP1]] to <8 x i8>
+; CHECK-NEXT:    [[SHIFT:%.*]] = lshr <8 x i32> [[SUB]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[T:%.*]] = trunc <8 x i32> [[SHIFT]] to <8 x i8>
 ; CHECK-NEXT:    ret <8 x i8> [[T]]
 ;
   %i1.ext = sext <8 x i8> %i1 to <8 x i32>
@@ -385,12 +414,13 @@ define <8 x i8> @narrow_zext_ashr_keep_trunc_vector(<8 x i8> %i1, <8 x i8> %i2)
 }
 
 define i8 @dont_narrow_zext_ashr_keep_trunc(i8 %i1, i8 %i2) {
-; CHECK-LABEL: @dont_narrow_zext_ashr_keep_trunc(
-; CHECK-NEXT:    [[I1_EXT:%.*]] = sext i8 [[I1:%.*]] to i16
-; CHECK-NEXT:    [[I2_EXT:%.*]] = sext i8 [[I2:%.*]] to i16
+; CHECK-LABEL: define i8 @dont_narrow_zext_ashr_keep_trunc(
+; CHECK-SAME: i8 [[I1:%.*]], i8 [[I2:%.*]]) {
+; CHECK-NEXT:    [[I1_EXT:%.*]] = sext i8 [[I1]] to i16
+; CHECK-NEXT:    [[I2_EXT:%.*]] = sext i8 [[I2]] to i16
 ; CHECK-NEXT:    [[SUB:%.*]] = add nsw i16 [[I1_EXT]], [[I2_EXT]]
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[SUB]], 1
-; CHECK-NEXT:    [[T:%.*]] = trunc i16 [[TMP1]] to i8
+; CHECK-NEXT:    [[SHIFT:%.*]] = lshr i16 [[SUB]], 1
+; CHECK-NEXT:    [[T:%.*]] = trunc i16 [[SHIFT]] to i8
 ; CHECK-NEXT:    ret i8 [[T]]
 ;
   %i1.ext = sext i8 %i1 to i16
-- 
GitLab


From 3a3aeb8eba40e981d3a9ff92175f949c2f3d4434 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 9 May 2024 13:27:20 +0900
Subject: [PATCH 0248/1206] [PPCMergeStringPool] Avoid replacing constant with
 instruction (#88846)

String pool merging currently, for a reason that's not entirely clear to
me, tries to create GEP instructions instead of GEP constant expressions
when replacing constant references. It only uses constant expressions in
cases where this is required. However, it does not catch all cases where
such a requirement exists. For example, the landingpad catch clause has
to be a constant.

Fix this by always using the constant expression variant, which also
makes the implementation simpler.

Additionally, there are some edge cases where even replacement with a
constant GEP is not legal. The one I am aware of is the
llvm.eh.typeid.for intrinsic, so add a special case to forbid
replacements for it.

Fixes https://github.com/llvm/llvm-project/issues/88844.
---
 .../lib/Target/PowerPC/PPCMergeStringPool.cpp | 57 ++++++-------------
 .../PowerPC/merge-string-used-by-metadata.mir |  6 +-
 .../mergeable-string-pool-exceptions.ll       | 47 +++++++++++++++
 .../mergeable-string-pool-pass-only.mir       | 18 +++---
 4 files changed, 75 insertions(+), 53 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/mergeable-string-pool-exceptions.ll

diff --git a/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp b/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp
index 76d60c28f1e4..abc5353e4a5e 100644
--- a/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp
@@ -23,6 +23,7 @@
 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ValueSymbolTable.h"
 #include "llvm/Pass.h"
@@ -117,9 +118,20 @@ private:
 // sure that they can be replaced.
 static bool hasReplaceableUsers(GlobalVariable &GV) {
   for (User *CurrentUser : GV.users()) {
-    // Instruction users are always valid.
-    if (isa<Instruction>(CurrentUser))
+    if (auto *I = dyn_cast<Instruction>(CurrentUser)) {
+      // Do not merge globals in exception pads.
+      if (I->isEHPad())
+        return false;
+
+      if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+        // Some intrinsics require a plain global.
+        if (II->getIntrinsicID() == Intrinsic::eh_typeid_for)
+          return false;
+      }
+
+      // Other instruction users are always valid.
       continue;
+    }
 
     // We cannot replace GlobalValue users because they are not just nodes
     // in IR. To replace a user like this we would need to create a new
@@ -314,14 +326,6 @@ void PPCMergeStringPool::replaceUsesWithGEP(GlobalVariable *GlobalToReplace,
     Users.push_back(CurrentUser);
 
   for (User *CurrentUser : Users) {
-    Instruction *UserInstruction = dyn_cast<Instruction>(CurrentUser);
-    Constant *UserConstant = dyn_cast<Constant>(CurrentUser);
-
-    // At this point we expect that the user is either an instruction or a
-    // constant.
-    assert((UserConstant || UserInstruction) &&
-           "Expected the user to be an instruction or a constant.");
-
     // The user was not found so it must have been replaced earlier.
     if (!userHasOperand(CurrentUser, GlobalToReplace))
       continue;
@@ -330,38 +334,13 @@ void PPCMergeStringPool::replaceUsesWithGEP(GlobalVariable *GlobalToReplace,
     if (isa<GlobalValue>(CurrentUser))
       continue;
 
-    if (!UserInstruction) {
-      // User is a constant type.
-      Constant *ConstGEP = ConstantExpr::getInBoundsGetElementPtr(
-          PooledStructType, GPool, Indices);
-      UserConstant->handleOperandChange(GlobalToReplace, ConstGEP);
-      continue;
-    }
-
-    if (PHINode *UserPHI = dyn_cast<PHINode>(UserInstruction)) {
-      // GEP instructions cannot be added before PHI nodes.
-      // With getInBoundsGetElementPtr we create the GEP and then replace it
-      // inline into the PHI.
-      Constant *ConstGEP = ConstantExpr::getInBoundsGetElementPtr(
-          PooledStructType, GPool, Indices);
-      UserPHI->replaceUsesOfWith(GlobalToReplace, ConstGEP);
-      continue;
-    }
-    // The user is a valid instruction that is not a PHINode.
-    GetElementPtrInst *GEPInst =
-        GetElementPtrInst::Create(PooledStructType, GPool, Indices);
-    GEPInst->insertBefore(UserInstruction);
-
-    LLVM_DEBUG(dbgs() << "Inserting GEP before:\n");
-    LLVM_DEBUG(UserInstruction->dump());
-
+    Constant *ConstGEP = ConstantExpr::getInBoundsGetElementPtr(
+        PooledStructType, GPool, Indices);
     LLVM_DEBUG(dbgs() << "Replacing this global:\n");
     LLVM_DEBUG(GlobalToReplace->dump());
     LLVM_DEBUG(dbgs() << "with this:\n");
-    LLVM_DEBUG(GEPInst->dump());
-
-    // After the GEP is inserted the GV can be replaced.
-    CurrentUser->replaceUsesOfWith(GlobalToReplace, GEPInst);
+    LLVM_DEBUG(ConstGEP->dump());
+    GlobalToReplace->replaceAllUsesWith(ConstGEP);
   }
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/merge-string-used-by-metadata.mir b/llvm/test/CodeGen/PowerPC/merge-string-used-by-metadata.mir
index 2a791966be4e..4a40974a2a22 100644
--- a/llvm/test/CodeGen/PowerPC/merge-string-used-by-metadata.mir
+++ b/llvm/test/CodeGen/PowerPC/merge-string-used-by-metadata.mir
@@ -14,16 +14,14 @@
 
   define noundef ptr @func1(ptr noundef nonnull align 8 dereferenceable(8) %this) #0 !dbg !6 {
   ; CHECK-LABEL: func1
-  ; CHECK:       %0 = getelementptr { [7 x i8], [7 x i8] }, ptr @__ModuleStringPool, i32 0, i32 1
-  ; CHECK-NEXT:  ret ptr %0, !dbg !14
+  ; CHECK:       ret ptr getelementptr inbounds ({ [7 x i8], [7 x i8] }, ptr @__ModuleStringPool, i32 0, i32 1), !dbg !14
   entry:
     ret ptr @const.2, !dbg !14
   }
 
   define noundef ptr @func2(ptr noundef nonnull align 8 dereferenceable(8) %this) #0 {
   ; CHECK-LABEL: func2
-  ; CHECK:       %0 = getelementptr { [7 x i8], [7 x i8] }, ptr @__ModuleStringPool, i32 0, i32 0
-  ; CHECK-NEXT:  ret ptr %0
+  ; CHECK:       ret ptr @__ModuleStringPool
   entry:
     ret ptr @const.1
   }
diff --git a/llvm/test/CodeGen/PowerPC/mergeable-string-pool-exceptions.ll b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-exceptions.ll
new file mode 100644
index 000000000000..03a830e087d2
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-exceptions.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=ppc64le-unknown-linux-gnu < %s | FileCheck %s
+
+@id = private unnamed_addr constant [4 x i8] c"@id\00", align 1
+@id2 = private unnamed_addr constant [5 x i8] c"@id2\00", align 1
+
+; Higher-aligned dummy to make sure it is first in the string pool.
+@dummy = private unnamed_addr constant [1 x i32] [i32 42], align 4
+
+define ptr @test1() personality ptr @__gnu_objc_personality_v0 {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    stdu 1, -32(1)
+; CHECK-NEXT:    std 0, 48(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    addis 3, 2, .Ldummy@toc@ha
+; CHECK-NEXT:    addi 3, 3, .Ldummy@toc@l
+; CHECK-NEXT:    bl foo
+; CHECK-NEXT:    nop
+  invoke void @foo(ptr @dummy)
+          to label %cont unwind label %unwind
+
+cont:
+  unreachable
+
+unwind:
+  %lp = landingpad { ptr, i32 }
+          catch ptr @id
+  resume { ptr, i32 } %lp
+}
+
+define i32 @test2() personality ptr @__gnu_objc_personality_v0 {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    blr
+  %id = tail call i32 @llvm.eh.typeid.for(ptr @id2)
+  ret i32 %id
+}
+
+declare i32 @__gnu_objc_personality_v0(...)
+
+declare i32 @llvm.eh.typeid.for(ptr)
+
+declare void @foo()
diff --git a/llvm/test/CodeGen/PowerPC/mergeable-string-pool-pass-only.mir b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-pass-only.mir
index e2fb0ced8f34..3d8afb604fd3 100644
--- a/llvm/test/CodeGen/PowerPC/mergeable-string-pool-pass-only.mir
+++ b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-pass-only.mir
@@ -35,8 +35,7 @@
     ret i32 %call
 
   ; CHECK-LABEL: test1
-  ; CHECK:         %0 = getelementptr { [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 6
-  ; CHECK:         tail call signext i32 @calleeStr
+  ; CHECK:         %call = tail call signext i32 @calleeStr(ptr noundef nonnull getelementptr inbounds ({ [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 6))
   }
 
   define dso_local signext i32 @test2() local_unnamed_addr #0 {
@@ -49,7 +48,7 @@
     ret i32 %call
 
   ; CHECK-LABEL: test2
-  ; CHECK:         %0 = getelementptr { [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 2
+  ; CHECK:         call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) %A, ptr noundef nonnull align 4 dereferenceable(24) getelementptr inbounds ({ [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 2), i64 24, i1 false)
   ; CHECK:         call signext i32 @calleeInt
   }
 
@@ -62,7 +61,7 @@
     call void @llvm.lifetime.end.p0(i64 28, ptr nonnull %A) #0
     ret i32 %call
   ; CHECK-LABEL: test3
-  ; CHECK:         %0 = getelementptr { [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 4
+  ; CHECK:         call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(28) %A, ptr noundef nonnull align 4 dereferenceable(28) getelementptr inbounds ({ [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 4), i64 28, i1 false)
   ; CHECK:         call signext i32 @calleeFloat
   }
 
@@ -75,7 +74,7 @@
     call void @llvm.lifetime.end.p0(i64 56, ptr nonnull %A) #0
     ret i32 %call
   ; CHECK-LABEL: test4
-  ; CHECK:         %0 = getelementptr { [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 0
+  ; CHECK:         call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(56) %A, ptr noundef nonnull align 8 dereferenceable(56) @__ModuleStringPool, i64 56, i1 false)
   ; CHECK:         call signext i32 @calleeDouble
   }
 
@@ -102,11 +101,10 @@
     call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %B) #0
     ret i32 %add7
   ; CHECK-LABEL: test5
-  ; CHECK:         %0 = getelementptr { [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 3
-  ; CHECK:         %1 = getelementptr { [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 5
-  ; CHECK:         %2 = getelementptr { [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 1
-  ; CHECK:         %3 = getelementptr { [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 7
-  ; CHECK:         call signext i32 @calleeStr
+  ; CHECK:         call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) %B, ptr noundef nonnull align 4 dereferenceable(24) getelementptr inbounds ({ [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 3), i64 24, i1 false)
+  ; CHECK:         call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(28) %C, ptr noundef nonnull align 4 dereferenceable(28) getelementptr inbounds ({ [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 5), i64 28, i1 false)
+  ; CHECK:         call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(56) %D, ptr noundef nonnull align 8 dereferenceable(56) getelementptr inbounds ({ [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 1), i64 56, i1 false)
+  ; CHECK:         call signext i32 @calleeStr(ptr noundef nonnull getelementptr inbounds ({ [7 x double], [7 x double], [6 x i32], [6 x i32], [7 x float], [7 x float], [8 x i8], [16 x i8] }, ptr @__ModuleStringPool, i32 0, i32 7))
   ; CHECK:         call signext i32 @calleeInt
   ; CHECK:         call signext i32 @calleeFloat
   ; CHECK:         call signext i32 @calleeDouble
-- 
GitLab


From 5baf58b628a4488de3f1a6af7c0df180358ba5dc Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 8 May 2024 21:53:15 -0700
Subject: [PATCH 0249/1206] [RISCV] Improve use of BSETI/BCLRI in constant
 materialization. (#91546)

We failed to use BSETI when bit 31 was set and a few bits above bit 31
were set. We also failed to use multiple BSETI when the low 32 bits were
zero.

I've removed the special cases for constants 0x80000000-0xffffffff and
wrote a more generic algorithm for BSETI.

I've rewritten the BCLRI handling to be similar to the new BSETI
algorithm. This picks up cases where bit 31 is 0 and only a few high
bits are 0.
---
 .../Target/RISCV/MCTargetDesc/RISCVMatInt.cpp | 77 ++++++++-----------
 llvm/test/CodeGen/RISCV/imm.ll                | 17 ++--
 llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll | 17 ++--
 3 files changed, 47 insertions(+), 64 deletions(-)

diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
index c3bae152993e..0a304d4cb7d9 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
@@ -310,56 +310,45 @@ InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI) {
     }
   }
 
-  // Perform optimization with BCLRI/BSETI in the Zbs extension.
+  // Perform optimization with BSETI in the Zbs extension.
   if (Res.size() > 2 && STI.hasFeature(RISCV::FeatureStdExtZbs)) {
-    // 1. For values in range 0xffffffff 7fffffff ~ 0xffffffff 00000000,
-    //    call generateInstSeqImpl with Val|0x80000000 (which is expected be
-    //    an int32), then emit (BCLRI r, 31).
-    // 2. For values in range 0x80000000 ~ 0xffffffff, call generateInstSeqImpl
-    //    with Val&~0x80000000 (which is expected to be an int32), then
-    //    emit (BSETI r, 31).
-    int64_t NewVal;
-    unsigned Opc;
-    if (Val < 0) {
-      Opc = RISCV::BCLRI;
-      NewVal = Val | 0x80000000ll;
-    } else {
-      Opc = RISCV::BSETI;
-      NewVal = Val & ~0x80000000ll;
-    }
-    if (isInt<32>(NewVal)) {
-      RISCVMatInt::InstSeq TmpSeq;
-      generateInstSeqImpl(NewVal, STI, TmpSeq);
-      if ((TmpSeq.size() + 1) < Res.size()) {
-        TmpSeq.emplace_back(Opc, 31);
-        Res = TmpSeq;
-      }
+    // Create a simm32 value for LUI+ADDIW by forcing the upper 33 bits to zero.
+    // Xor that with original value to get which bits should be set by BSETI.
+    uint64_t Lo = Val & 0x7fffffff;
+    uint64_t Hi = Val ^ Lo;
+    assert(Hi != 0);
+    RISCVMatInt::InstSeq TmpSeq;
+
+    if (Lo != 0)
+      generateInstSeqImpl(Lo, STI, TmpSeq);
+
+    if (TmpSeq.size() + llvm::popcount(Hi) < Res.size()) {
+      do {
+        TmpSeq.emplace_back(RISCV::BSETI, llvm::countr_zero(Hi));
+        Hi &= (Hi - 1); // Clear lowest set bit.
+      } while (Hi != 0);
+      Res = TmpSeq;
     }
+  }
+
+  // Perform optimization with BCLRI in the Zbs extension.
+  if (Res.size() > 2 && STI.hasFeature(RISCV::FeatureStdExtZbs)) {
+    // Create a simm32 value for LUI+ADDIW by forcing the upper 33 bits to one.
+    // Xor that with original value to get which bits should be cleared by
+    // BCLRI.
+    uint64_t Lo = Val | 0xffffffff80000000;
+    uint64_t Hi = Val ^ Lo;
+    assert(Hi != 0);
 
-    // Try to use BCLRI for upper 32 bits if the original lower 32 bits are
-    // negative int32, or use BSETI for upper 32 bits if the original lower
-    // 32 bits are positive int32.
-    int32_t Lo = Lo_32(Val);
-    uint32_t Hi = Hi_32(Val);
-    Opc = 0;
     RISCVMatInt::InstSeq TmpSeq;
     generateInstSeqImpl(Lo, STI, TmpSeq);
-    // Check if it is profitable to use BCLRI/BSETI.
-    if (Lo > 0 && TmpSeq.size() + llvm::popcount(Hi) < Res.size()) {
-      Opc = RISCV::BSETI;
-    } else if (Lo < 0 && TmpSeq.size() + llvm::popcount(~Hi) < Res.size()) {
-      Opc = RISCV::BCLRI;
-      Hi = ~Hi;
-    }
-    // Search for each bit and build corresponding BCLRI/BSETI.
-    if (Opc > 0) {
-      while (Hi != 0) {
-        unsigned Bit = llvm::countr_zero(Hi);
-        TmpSeq.emplace_back(Opc, Bit + 32);
+
+    if (TmpSeq.size() + llvm::popcount(Hi) < Res.size()) {
+      do {
+        TmpSeq.emplace_back(RISCV::BCLRI, llvm::countr_zero(Hi));
         Hi &= (Hi - 1); // Clear lowest set bit.
-      }
-      if (TmpSeq.size() < Res.size())
-        Res = TmpSeq;
+      } while (Hi != 0);
+      Res = TmpSeq;
     }
   }
 
diff --git a/llvm/test/CodeGen/RISCV/imm.ll b/llvm/test/CodeGen/RISCV/imm.ll
index 0dc5c7ceb500..c5c1657b526a 100644
--- a/llvm/test/CodeGen/RISCV/imm.ll
+++ b/llvm/test/CodeGen/RISCV/imm.ll
@@ -4025,9 +4025,8 @@ define i64 @imm64_0x8000080000000() {
 ;
 ; RV64IZBS-LABEL: imm64_0x8000080000000:
 ; RV64IZBS:       # %bb.0:
-; RV64IZBS-NEXT:    lui a0, 256
-; RV64IZBS-NEXT:    addiw a0, a0, 1
-; RV64IZBS-NEXT:    slli a0, a0, 31
+; RV64IZBS-NEXT:    bseti a0, zero, 31
+; RV64IZBS-NEXT:    bseti a0, a0, 51
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm64_0x8000080000000:
@@ -4083,9 +4082,8 @@ define i64 @imm64_0x10000100000000() {
 ;
 ; RV64IZBS-LABEL: imm64_0x10000100000000:
 ; RV64IZBS:       # %bb.0:
-; RV64IZBS-NEXT:    lui a0, 256
-; RV64IZBS-NEXT:    addi a0, a0, 1
-; RV64IZBS-NEXT:    slli a0, a0, 32
+; RV64IZBS-NEXT:    bseti a0, zero, 32
+; RV64IZBS-NEXT:    bseti a0, a0, 52
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm64_0x10000100000000:
@@ -4146,10 +4144,9 @@ define i64 @imm64_0xFF7FFFFF7FFFFFFE() {
 ;
 ; RV64IZBS-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
 ; RV64IZBS:       # %bb.0:
-; RV64IZBS-NEXT:    lui a0, 1044480
-; RV64IZBS-NEXT:    addiw a0, a0, -1
-; RV64IZBS-NEXT:    slli a0, a0, 31
-; RV64IZBS-NEXT:    addi a0, a0, -1
+; RV64IZBS-NEXT:    li a0, -1
+; RV64IZBS-NEXT:    bclri a0, a0, 31
+; RV64IZBS-NEXT:    bclri a0, a0, 55
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll
index bac4bb9ce6f1..561686374a9b 100644
--- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/imm.ll
@@ -2648,9 +2648,8 @@ define i64 @imm64_0x8000080000000() {
 ;
 ; RV64IZBS-LABEL: imm64_0x8000080000000:
 ; RV64IZBS:       # %bb.0:
-; RV64IZBS-NEXT:    lui a0, 256
-; RV64IZBS-NEXT:    addiw a0, a0, 1
-; RV64IZBS-NEXT:    slli a0, a0, 31
+; RV64IZBS-NEXT:    bseti a0, zero, 31
+; RV64IZBS-NEXT:    bseti a0, a0, 51
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm64_0x8000080000000:
@@ -2686,9 +2685,8 @@ define i64 @imm64_0x10000100000000() {
 ;
 ; RV64IZBS-LABEL: imm64_0x10000100000000:
 ; RV64IZBS:       # %bb.0:
-; RV64IZBS-NEXT:    lui a0, 256
-; RV64IZBS-NEXT:    addi a0, a0, 1
-; RV64IZBS-NEXT:    slli a0, a0, 32
+; RV64IZBS-NEXT:    bseti a0, zero, 32
+; RV64IZBS-NEXT:    bseti a0, a0, 52
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm64_0x10000100000000:
@@ -2727,10 +2725,9 @@ define i64 @imm64_0xFF7FFFFF7FFFFFFE() {
 ;
 ; RV64IZBS-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
 ; RV64IZBS:       # %bb.0:
-; RV64IZBS-NEXT:    lui a0, 1044480
-; RV64IZBS-NEXT:    addiw a0, a0, -1
-; RV64IZBS-NEXT:    slli a0, a0, 31
-; RV64IZBS-NEXT:    addi a0, a0, -1
+; RV64IZBS-NEXT:    li a0, -1
+; RV64IZBS-NEXT:    bclri a0, a0, 31
+; RV64IZBS-NEXT:    bclri a0, a0, 55
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm64_0xFF7FFFFF7FFFFFFE:
-- 
GitLab


From 666970cab2e80055cf1e5b5e9025c8f88e0d0732 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 8 May 2024 22:37:17 -0700
Subject: [PATCH 0250/1206] [RISCV] Remove unnecessary initialization from
 RISCVPostRAExpandPseudo pass constructor.

It is already initialized in RISCVTargetMachine.cpp
---
 llvm/lib/Target/RISCV/RISCVPostRAExpandPseudoInsts.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVPostRAExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVPostRAExpandPseudoInsts.cpp
index 57b473645ae7..52f2ce27164d 100644
--- a/llvm/lib/Target/RISCV/RISCVPostRAExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVPostRAExpandPseudoInsts.cpp
@@ -31,9 +31,7 @@ public:
   const RISCVInstrInfo *TII;
   static char ID;
 
-  RISCVPostRAExpandPseudo() : MachineFunctionPass(ID) {
-    initializeRISCVPostRAExpandPseudoPass(*PassRegistry::getPassRegistry());
-  }
+  RISCVPostRAExpandPseudo() : MachineFunctionPass(ID) {}
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
-- 
GitLab


From 2a57657d5571b097eb0070e6f26ad4954c0fd990 Mon Sep 17 00:00:00 2001
From: chandan singh <36783761+chandankds@users.noreply.github.com>
Date: Thu, 9 May 2024 11:11:04 +0530
Subject: [PATCH 0251/1206] [OpenMP] [Flang] Resolved Issue llvm#76121:
 Implemented Check for Unhandled Arguments in __kmpc_fork_call_if (#82221)

Root cause: Segmentation fault is caused by null pointer dereference
inside the __kmpc_fork_call_if function at
https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/z_Linux_asm.S#L1186
. __kmpc_fork_call_if is missing case to handle argc=0 .

Fix: Added a check inside the __kmp_invoke_microtask function to handle
the case when argc is 0.

---------

Co-authored-by: Singh <chasingh@amd.com>
---
 openmp/runtime/src/z_Linux_asm.S              |  4 +++
 .../test/misc_bugs/omp__kmpc_fork_call_if.c   | 36 +++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 openmp/runtime/test/misc_bugs/omp__kmpc_fork_call_if.c

diff --git a/openmp/runtime/src/z_Linux_asm.S b/openmp/runtime/src/z_Linux_asm.S
index 201949003c01..5b614e26a833 100644
--- a/openmp/runtime/src/z_Linux_asm.S
+++ b/openmp/runtime/src/z_Linux_asm.S
@@ -1150,6 +1150,9 @@ KMP_LABEL(kmp_invoke_pass_parms):	// put 1st - 6th parms to pkfn in registers.
 	movq	%rdi, %rbx	// pkfn -> %rbx
 	leaq	__gtid(%rbp), %rdi // &gtid -> %rdi (store 1st parm to pkfn)
 	leaq	__tid(%rbp), %rsi  // &tid -> %rsi (store 2nd parm to pkfn)
+	// Check if argc is 0
+	cmpq $0, %rax
+	je KMP_LABEL(kmp_no_args) // Jump ahead
 
 	movq	%r8, %r11	// p_argv -> %r11
 
@@ -1195,6 +1198,7 @@ KMP_LABEL(kmp_1_exit):
 	cmovnsq	(%r11), %rdx	// p_argv[0] -> %rdx (store 3rd parm to pkfn)
 #endif // KMP_MIC
 
+KMP_LABEL(kmp_no_args):
 	call	*%rbx		// call (*pkfn)();
 	movq	$1, %rax	// move 1 into return register;
 
diff --git a/openmp/runtime/test/misc_bugs/omp__kmpc_fork_call_if.c b/openmp/runtime/test/misc_bugs/omp__kmpc_fork_call_if.c
new file mode 100644
index 000000000000..60d4bff96787
--- /dev/null
+++ b/openmp/runtime/test/misc_bugs/omp__kmpc_fork_call_if.c
@@ -0,0 +1,36 @@
+// RUN: %libomp-compile && %t | FileCheck %s
+
+#include <stdio.h>
+#include <omp.h>
+
+typedef int32_t kmp_int32;
+typedef void *ident_t;
+typedef void *kmpc_micro;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern void __kmpc_fork_call_if(ident_t *loc, kmp_int32 argc,
+                                kmpc_micro microtask, kmp_int32 cond,
+                                void *args);
+#ifdef __cplusplus
+}
+#endif
+
+// Microtask function for parallel region
+void microtask(int *global_tid, int *bound_tid) {
+  // CHECK: PASS
+  if (omp_in_parallel()) {
+    printf("FAIL\n");
+  } else {
+    printf("PASS\n");
+  }
+}
+
+int main() {
+  // Condition for parallelization (false in this case)
+  int cond = 0;
+  // Call __kmpc_fork_call_if
+  __kmpc_fork_call_if(NULL, 0, microtask, cond, NULL);
+  return 0;
+}
-- 
GitLab


From 5adcfd4c17826b2b8f023881baa1c7f79cb23920 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm@amd.com>
Date: Thu, 9 May 2024 08:08:55 +0200
Subject: [PATCH 0252/1206] [flang][CMake] Add missing dependency to generate
 Fortran module files (#91517)

Fixes bug https://github.com/llvm/llvm-project/issues/90769. Many thanks
to @Meinersbur for providing the initial thought and solution to this.
---
 llvm/runtimes/CMakeLists.txt | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 3020ba72f4a6..8a3ec1e3300d 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -431,8 +431,10 @@ if(runtimes)
       set(LIBOMP_MODULES_INSTALL_PATH "${CMAKE_INSTALL_INCLUDEDIR}/flang")
       # TODO: This is a workaround until flang becomes a first-class project
       # in llvm/CMakeList.txt.  Until then, this line ensures that flang-new is
-      # built before "openmp" is built as a runtime project.
-      list(APPEND extra_deps "flang-new")
+      # built before "openmp" is built as a runtime project.  Besides "flang-new"
+      # to build the compiler, we also need to add "module_files" to make sure
+      # that all .mod files are also properly build.
+      list(APPEND extra_deps "flang-new" "module_files")
     endif()
     foreach(dep opt llvm-link llvm-extract clang clang-offload-packager)
       if(TARGET ${dep})
-- 
GitLab


From 90ffaa6ccc6dc7351c72979da217bd3eb7fd4491 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 9 May 2024 15:04:39 +0900
Subject: [PATCH 0253/1206] [InstCombine] Add proper test coverage for or of
 xors pattern (NFC)

Test all commuted variants of the pattern, most of which currently
fail to fold.
---
 llvm/test/Transforms/InstCombine/or-xor.ll | 370 ++++++++++++++++++++-
 1 file changed, 352 insertions(+), 18 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/or-xor.ll b/llvm/test/Transforms/InstCombine/or-xor.ll
index 0a322d6aa023..32198c0a81dd 100644
--- a/llvm/test/Transforms/InstCombine/or-xor.ll
+++ b/llvm/test/Transforms/InstCombine/or-xor.ll
@@ -7,8 +7,8 @@ declare void @use(i8)
 
 define i32 @test1(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i32 [[Y:%.*]], -1
-; CHECK-NEXT:    [[Z:%.*]] = or i32 [[Y_NOT]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[TMP1]], [[X:%.*]]
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %or = or i32 %x, %y
@@ -22,8 +22,8 @@ define i32 @test1(i32 %x, i32 %y) {
 
 define i32 @test2(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[X_NOT:%.*]] = xor i32 [[X:%.*]], -1
-; CHECK-NEXT:    [[Z:%.*]] = or i32 [[X_NOT]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %or = or i32 %x, %y
@@ -36,8 +36,8 @@ define i32 @test2(i32 %x, i32 %y) {
 
 define i32 @test3(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i32 [[Y:%.*]], -1
-; CHECK-NEXT:    [[Z:%.*]] = or i32 [[Y_NOT]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[TMP1]], [[X:%.*]]
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %xor = xor i32 %x, %y
@@ -51,8 +51,8 @@ define i32 @test3(i32 %x, i32 %y) {
 
 define i32 @test4(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[X_NOT:%.*]] = xor i32 [[X:%.*]], -1
-; CHECK-NEXT:    [[Z:%.*]] = or i32 [[X_NOT]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %xor = xor i32 %x, %y
@@ -205,8 +205,8 @@ define i8 @xor_common_op_commute3(i8 %p, i8 %q) {
 
 define i32 @test8(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[X_NOT:%.*]] = xor i32 [[X:%.*]], -1
-; CHECK-NEXT:    [[Z:%.*]] = or i32 [[X_NOT]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %not = xor i32 %y, -1
@@ -217,8 +217,8 @@ define i32 @test8(i32 %x, i32 %y) {
 
 define i32 @test9(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i32 [[Y:%.*]], -1
-; CHECK-NEXT:    [[Z:%.*]] = or i32 [[Y_NOT]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[TMP1]], [[X:%.*]]
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %not = xor i32 %x, -1
@@ -1097,8 +1097,8 @@ define i32 @PR75692_3(i32 %x, i32 %y) {
 
 define i32 @or_xor_not(i32 %x, i32 %y) {
 ; CHECK-LABEL: @or_xor_not(
-; CHECK-NEXT:    [[X_NOT:%.*]] = xor i32 [[X:%.*]], -1
-; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[X_NOT]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i32 [[OR1]]
 ;
   %not = xor i32 %y, -1
@@ -1140,8 +1140,8 @@ define i32 @or_xor_not_uses2(i32 %x, i32 %y) {
 define i32 @or_xor_and_commuted1(i32 %x, i32 %y) {
 ; CHECK-LABEL: @or_xor_and_commuted1(
 ; CHECK-NEXT:    [[YY:%.*]] = mul i32 [[Y:%.*]], [[Y]]
-; CHECK-NEXT:    [[X_NOT:%.*]] = xor i32 [[X:%.*]], -1
-; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[YY]], [[X_NOT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[YY]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[OR1]]
 ;
   %yy = mul i32 %y, %y ; thwart complexity-based ordering
@@ -1155,8 +1155,8 @@ define i32 @or_xor_and_commuted2(i32 %x, i32 %y) {
 ; CHECK-LABEL: @or_xor_and_commuted2(
 ; CHECK-NEXT:    [[YY:%.*]] = mul i32 [[Y:%.*]], [[Y]]
 ; CHECK-NEXT:    [[XX:%.*]] = mul i32 [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[XX_NOT:%.*]] = xor i32 [[XX]], -1
-; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[YY]], [[XX_NOT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[XX]], -1
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[YY]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[OR1]]
 ;
   %yy = mul i32 %y, %y ; thwart complexity-based ordering
@@ -1166,3 +1166,337 @@ define i32 @or_xor_and_commuted2(i32 %x, i32 %y) {
   %or1 = or i32 %xor, %yy
   ret i32 %or1
 }
+
+; (A ^ B) | ((B ^ C) ^ A) -> (A ^ B) | C and commuted variants.
+
+define i32 @or_xor_tree_0000(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_0000(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %a, %b
+  %xor2 = xor i32 %b, %c
+  %xor3 = xor i32 %xor2, %a
+  %or = or i32 %xor1, %xor3
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_0001(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_0001(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %b, %a
+  %xor2 = xor i32 %b, %c
+  %xor3 = xor i32 %xor2, %a
+  %or = or i32 %xor1, %xor3
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_0010(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_0010(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %a, %b
+  %xor2 = xor i32 %c, %b
+  %xor3 = xor i32 %xor2, %a
+  %or = or i32 %xor1, %xor3
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_0011(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_0011(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %b, %a
+  %xor2 = xor i32 %c, %b
+  %xor3 = xor i32 %xor2, %a
+  %or = or i32 %xor1, %xor3
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_0100(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_0100(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %a, %b
+  %xor2 = xor i32 %b, %c
+  %xor3 = xor i32 %a, %xor2
+  %or = or i32 %xor1, %xor3
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_0101(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_0101(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %b, %a
+  %xor2 = xor i32 %b, %c
+  %xor3 = xor i32 %a, %xor2
+  %or = or i32 %xor1, %xor3
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_0110(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_0110(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %a, %b
+  %xor2 = xor i32 %c, %b
+  %xor3 = xor i32 %a, %xor2
+  %or = or i32 %xor1, %xor3
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_0111(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_0111(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %b, %a
+  %xor2 = xor i32 %c, %b
+  %xor3 = xor i32 %a, %xor2
+  %or = or i32 %xor1, %xor3
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_1000(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_1000(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %a, %b
+  %xor2 = xor i32 %b, %c
+  %xor3 = xor i32 %xor2, %a
+  %or = or i32 %xor3, %xor1
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_1001(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_1001(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %b, %a
+  %xor2 = xor i32 %b, %c
+  %xor3 = xor i32 %xor2, %a
+  %or = or i32 %xor3, %xor1
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_1010(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_1010(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %a, %b
+  %xor2 = xor i32 %c, %b
+  %xor3 = xor i32 %xor2, %a
+  %or = or i32 %xor3, %xor1
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_1011(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_1011(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %b, %a
+  %xor2 = xor i32 %c, %b
+  %xor3 = xor i32 %xor2, %a
+  %or = or i32 %xor3, %xor1
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_1100(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_1100(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %a, %b
+  %xor2 = xor i32 %b, %c
+  %xor3 = xor i32 %a, %xor2
+  %or = or i32 %xor3, %xor1
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_1101(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_1101(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %b, %a
+  %xor2 = xor i32 %b, %c
+  %xor3 = xor i32 %a, %xor2
+  %or = or i32 %xor3, %xor1
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_1110(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_1110(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %a, %b
+  %xor2 = xor i32 %c, %b
+  %xor3 = xor i32 %a, %xor2
+  %or = or i32 %xor3, %xor1
+  ret i32 %or
+}
+
+define i32 @or_xor_tree_1111(i32 %ax, i32 %bx, i32 %cx) {
+; CHECK-LABEL: @or_xor_tree_1111(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[AX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
+; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = mul i32 %ax, 42
+  %b = mul i32 %bx, 42
+  %c = mul i32 %cx, 42
+  %xor1 = xor i32 %b, %a
+  %xor2 = xor i32 %c, %b
+  %xor3 = xor i32 %a, %xor2
+  %or = or i32 %xor3, %xor1
+  ret i32 %or
+}
-- 
GitLab


From 534701d5f93369e822f3afc9670e3a42b08dfc6f Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 9 May 2024 15:06:36 +0900
Subject: [PATCH 0254/1206] [InstCombine] Handle commuted variants in or of xor
 pattern

This pattern only handled commutation in the "or", while all
involved operations are commutative. Make sure we handle all
sixteen patterns.
---
 .../InstCombine/InstCombineAndOrXor.cpp       | 12 ++--
 llvm/test/Transforms/InstCombine/or-xor.ll    | 56 +++++--------------
 2 files changed, 22 insertions(+), 46 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index ed9a89b14efc..a52c70dbdf3f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -3599,12 +3599,16 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
 
   // (A ^ B) | ((B ^ C) ^ A) -> (A ^ B) | C
   if (match(Op0, m_Xor(m_Value(A), m_Value(B))))
-    if (match(Op1, m_Xor(m_Xor(m_Specific(B), m_Value(C)), m_Specific(A))))
+    if (match(Op1,
+              m_c_Xor(m_c_Xor(m_Specific(B), m_Value(C)), m_Specific(A))) ||
+        match(Op1, m_c_Xor(m_c_Xor(m_Specific(A), m_Value(C)), m_Specific(B))))
       return BinaryOperator::CreateOr(Op0, C);
 
-  // ((A ^ C) ^ B) | (B ^ A) -> (B ^ A) | C
-  if (match(Op0, m_Xor(m_Xor(m_Value(A), m_Value(C)), m_Value(B))))
-    if (match(Op1, m_Xor(m_Specific(B), m_Specific(A))))
+  // ((B ^ C) ^ A) | (A ^ B) -> (A ^ B) | C
+  if (match(Op1, m_Xor(m_Value(A), m_Value(B))))
+    if (match(Op0,
+              m_c_Xor(m_c_Xor(m_Specific(B), m_Value(C)), m_Specific(A))) ||
+        match(Op0, m_c_Xor(m_c_Xor(m_Specific(A), m_Value(C)), m_Specific(B))))
       return BinaryOperator::CreateOr(Op1, C);
 
   if (Instruction *DeMorgan = matchDeMorgansLaws(I, *this))
diff --git a/llvm/test/Transforms/InstCombine/or-xor.ll b/llvm/test/Transforms/InstCombine/or-xor.ll
index 32198c0a81dd..cf6b9000182d 100644
--- a/llvm/test/Transforms/InstCombine/or-xor.ll
+++ b/llvm/test/Transforms/InstCombine/or-xor.ll
@@ -1194,9 +1194,7 @@ define i32 @or_xor_tree_0001(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1215,9 +1213,7 @@ define i32 @or_xor_tree_0010(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1236,9 +1232,7 @@ define i32 @or_xor_tree_0011(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1257,9 +1251,7 @@ define i32 @or_xor_tree_0100(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1278,9 +1270,7 @@ define i32 @or_xor_tree_0101(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1299,9 +1289,7 @@ define i32 @or_xor_tree_0110(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1320,9 +1308,7 @@ define i32 @or_xor_tree_0111(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[XOR3]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1360,9 +1346,7 @@ define i32 @or_xor_tree_1001(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1381,9 +1365,7 @@ define i32 @or_xor_tree_1010(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1402,9 +1384,7 @@ define i32 @or_xor_tree_1011(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[XOR2]], [[A]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1423,9 +1403,7 @@ define i32 @or_xor_tree_1100(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1444,9 +1422,7 @@ define i32 @or_xor_tree_1101(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[B]], [[C]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1465,9 +1441,7 @@ define i32 @or_xor_tree_1110(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A]], [[B]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
@@ -1486,9 +1460,7 @@ define i32 @or_xor_tree_1111(i32 %ax, i32 %bx, i32 %cx) {
 ; CHECK-NEXT:    [[B:%.*]] = mul i32 [[BX:%.*]], 42
 ; CHECK-NEXT:    [[C:%.*]] = mul i32 [[CX:%.*]], 42
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[B]], [[A]]
-; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[C]], [[B]]
-; CHECK-NEXT:    [[XOR3:%.*]] = xor i32 [[A]], [[XOR2]]
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR3]], [[XOR1]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR1]], [[C]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %a = mul i32 %ax, 42
-- 
GitLab


From 97be79ca126c1a0e174fdbc345a28868edc7cdc7 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 9 May 2024 15:19:29 +0900
Subject: [PATCH 0255/1206] [Reassociate] Generate test checks (NFC)

---
 .../Reassociate/fast-ArrayOutOfBounds.ll      | 45 ++++++++++---------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/llvm/test/Transforms/Reassociate/fast-ArrayOutOfBounds.ll b/llvm/test/Transforms/Reassociate/fast-ArrayOutOfBounds.ll
index faabd8d7815b..6dc7b89a9b18 100644
--- a/llvm/test/Transforms/Reassociate/fast-ArrayOutOfBounds.ll
+++ b/llvm/test/Transforms/Reassociate/fast-ArrayOutOfBounds.ll
@@ -1,25 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; RUN: opt < %s -passes=reassociate,instcombine -S | FileCheck %s
 
 ; Not marked as fast, so must not change.
 define float @test1(float %a0, float %a1, float %a2, float %a3, float %a4) {
-; CHECK-LABEL: test1
-; CHECK-NEXT: %tmp.2 = fadd float %a3, %a4
-; CHECK-NEXT: %tmp.4 = fadd float %tmp.2, %a2
-; CHECK-NEXT: %tmp.6 = fadd float %tmp.4, %a1
-; CHECK-NEXT: %tmp.8 = fadd float %tmp.6, %a0
-; CHECK-NEXT: %tmp.11 = fadd float %a2, %a3
-; CHECK-NEXT: %tmp.13 = fadd float %tmp.11, %a1
-; CHECK-NEXT: %tmp.15 = fadd float %tmp.13, %a0
-; CHECK-NEXT: %tmp.18 = fadd float %a1, %a2
-; CHECK-NEXT: %tmp.20 = fadd float %tmp.18, %a0
-; CHECK-NEXT: %tmp.23 = fadd float %a0, %a1
-; CHECK-NEXT: %tmp.26 = fsub float %tmp.8, %tmp.15
-; CHECK-NEXT: %tmp.28 = fadd float %tmp.20, %tmp.26
-; CHECK-NEXT: %tmp.30 = fsub float %tmp.28, %tmp.23
-; CHECK-NEXT: %tmp.32 = fsub float %tmp.30, %a4
-; CHECK-NEXT: %tmp.34 = fsub float %tmp.32, %a2
-; CHECK-NEXT: %T = fmul float %tmp.34, %tmp.34
-; CHECK-NEXT: ret float %T
+; CHECK-LABEL: define float @test1(
+; CHECK-SAME: float [[A0:%.*]], float [[A1:%.*]], float [[A2:%.*]], float [[A3:%.*]], float [[A4:%.*]]) {
+; CHECK-NEXT:    [[TMP_2:%.*]] = fadd float [[A3]], [[A4]]
+; CHECK-NEXT:    [[TMP_4:%.*]] = fadd float [[TMP_2]], [[A2]]
+; CHECK-NEXT:    [[TMP_6:%.*]] = fadd float [[TMP_4]], [[A1]]
+; CHECK-NEXT:    [[TMP_8:%.*]] = fadd float [[TMP_6]], [[A0]]
+; CHECK-NEXT:    [[TMP_11:%.*]] = fadd float [[A2]], [[A3]]
+; CHECK-NEXT:    [[TMP_13:%.*]] = fadd float [[TMP_11]], [[A1]]
+; CHECK-NEXT:    [[TMP_15:%.*]] = fadd float [[TMP_13]], [[A0]]
+; CHECK-NEXT:    [[TMP_18:%.*]] = fadd float [[A1]], [[A2]]
+; CHECK-NEXT:    [[TMP_20:%.*]] = fadd float [[TMP_18]], [[A0]]
+; CHECK-NEXT:    [[TMP_23:%.*]] = fadd float [[A0]], [[A1]]
+; CHECK-NEXT:    [[TMP_26:%.*]] = fsub float [[TMP_8]], [[TMP_15]]
+; CHECK-NEXT:    [[TMP_28:%.*]] = fadd float [[TMP_20]], [[TMP_26]]
+; CHECK-NEXT:    [[TMP_30:%.*]] = fsub float [[TMP_28]], [[TMP_23]]
+; CHECK-NEXT:    [[TMP_32:%.*]] = fsub float [[TMP_30]], [[A4]]
+; CHECK-NEXT:    [[TMP_34:%.*]] = fsub float [[TMP_32]], [[A2]]
+; CHECK-NEXT:    [[T:%.*]] = fmul float [[TMP_34]], [[TMP_34]]
+; CHECK-NEXT:    ret float [[T]]
+;
 
   %tmp.2 = fadd float %a4, %a3
   %tmp.4 = fadd float %tmp.2, %a2
@@ -42,8 +45,10 @@ define float @test1(float %a0, float %a1, float %a2, float %a3, float %a4) {
 
 ; Should be able to eliminate everything.
 define float @test2(float %a0, float %a1, float %a2, float %a3, float %a4) {
-; CHECK-LABEL: test2
-; CHECK: ret float 0.000000e+00
+; CHECK-LABEL: define float @test2(
+; CHECK-SAME: float [[A0:%.*]], float [[A1:%.*]], float [[A2:%.*]], float [[A3:%.*]], float [[A4:%.*]]) {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
 
   %tmp.2 = fadd fast float %a4, %a3
   %tmp.4 = fadd fast float %tmp.2, %a2
-- 
GitLab


From 042a0b000dfe602ee0432be5ff88c67f531791bc Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Thu, 9 May 2024 08:37:48 +0200
Subject: [PATCH 0256/1206] [lldb] Make SBType::GetDirectNestedType (mostly)
 work with typedefs (#91189)

The implementation is straight-forward, but comes with a big disclaimer.
See #91186 for details.
---
 .../Plugins/TypeSystem/Clang/TypeSystemClang.cpp |  2 ++
 lldb/test/API/python_api/type/TestTypeList.py    | 16 ++++++++++++++++
 lldb/test/API/python_api/type/main.cpp           |  5 +++++
 3 files changed, 23 insertions(+)

diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index a771016039e8..d0033fcd9cdf 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -7106,6 +7106,8 @@ TypeSystemClang::GetDirectNestedTypeWithName(lldb::opaque_compiler_type_t type,
     for (NamedDecl *decl : record_decl->lookup(decl_name)) {
       if (auto *tag_decl = dyn_cast<clang::TagDecl>(decl))
         return GetType(getASTContext().getTagDeclType(tag_decl));
+      if (auto *typedef_decl = dyn_cast<clang::TypedefNameDecl>(decl))
+        return GetType(getASTContext().getTypedefType(typedef_decl));
     }
     break;
   }
diff --git a/lldb/test/API/python_api/type/TestTypeList.py b/lldb/test/API/python_api/type/TestTypeList.py
index 0498396903dc..b028929eea44 100644
--- a/lldb/test/API/python_api/type/TestTypeList.py
+++ b/lldb/test/API/python_api/type/TestTypeList.py
@@ -273,6 +273,22 @@ class TypeAndTypeListTestCase(TestBase):
             self.DebugSBType(int_enum_uchar)
             self.assertEqual(int_enum_uchar.GetName(), "unsigned char")
 
+    def test_nested_typedef(self):
+        """Exercise FindDirectNestedType for typedefs."""
+        self.build()
+        target = self.dbg.CreateTarget(self.getBuildArtifact())
+        self.assertTrue(target)
+
+        with_nested_typedef = target.FindFirstType("WithNestedTypedef")
+        self.assertTrue(with_nested_typedef)
+
+        # This is necessary to work around #91186
+        self.assertTrue(target.FindFirstGlobalVariable("typedefed_value").GetType())
+
+        the_typedef = with_nested_typedef.FindDirectNestedType("TheTypedef")
+        self.assertTrue(the_typedef)
+        self.assertEqual(the_typedef.GetTypedefedType().GetName(), "int")
+
     def test_GetByteAlign(self):
         """Exercise SBType::GetByteAlign"""
         self.build()
diff --git a/lldb/test/API/python_api/type/main.cpp b/lldb/test/API/python_api/type/main.cpp
index 986ed3009a15..6acde5bb666a 100644
--- a/lldb/test/API/python_api/type/main.cpp
+++ b/lldb/test/API/python_api/type/main.cpp
@@ -53,6 +53,11 @@ enum class EnumUChar : unsigned char {};
 struct alignas(128) OverAlignedStruct {};
 OverAlignedStruct over_aligned_struct;
 
+struct WithNestedTypedef {
+  typedef int TheTypedef;
+};
+WithNestedTypedef::TheTypedef typedefed_value;
+
 int main (int argc, char const *argv[])
 {
     Task *task_head = new Task(-1, NULL);
-- 
GitLab


From fd1bd53ba5a06f344698a55578f6a5d79c457e30 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Thu, 9 May 2024 08:47:12 +0200
Subject: [PATCH 0257/1206] [lldb/aarch64] Fix unwinding when signal interrupts
 a leaf function (#91321)

A leaf function may not store the link register to stack, but we it can
still end up being a non-zero frame if it gets interrupted by a signal.
Currently, we were unable to unwind past this function because we could
not read the link register value.

To make this work, this patch:
- changes the function-entry unwind plan to include the `fp|lr = <same>`
rules. This in turn necessitated an adjustment in the generic
instruction emulation logic to ensure that `lr=[sp-X]` can override the
`<same>` rule.
- allows the `<same>` rule for pc and lr in all
`m_all_registers_available` frames (and not just frame zero).

The test verifies that we can unwind in a situation like this, and that
the backtrace matches the one we computed before getting a signal.
---
 .../ARM64/EmulateInstructionARM64.cpp         |  2 ++
 .../UnwindAssemblyInstEmulation.cpp           |  4 +---
 lldb/source/Target/RegisterContextUnwind.cpp  |  6 ++---
 .../Inputs/signal-in-leaf-function-aarch64.c  | 15 ++++++++++++
 .../signal-in-leaf-function-aarch64.test      | 24 +++++++++++++++++++
 .../ARM64/TestArm64InstEmulation.cpp          | 24 +++++++++++++++----
 6 files changed, 65 insertions(+), 10 deletions(-)
 create mode 100644 lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
 create mode 100644 lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test

diff --git a/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp b/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
index 6ca4fb052457..62ecac3e0831 100644
--- a/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
+++ b/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
@@ -444,6 +444,8 @@ bool EmulateInstructionARM64::CreateFunctionEntryUnwind(
 
   // Our previous Call Frame Address is the stack pointer
   row->GetCFAValue().SetIsRegisterPlusOffset(gpr_sp_arm64, 0);
+  row->SetRegisterLocationToSame(gpr_lr_arm64, /*must_replace=*/false);
+  row->SetRegisterLocationToSame(gpr_fp_arm64, /*must_replace=*/false);
 
   unwind_plan.AppendRow(row);
   unwind_plan.SetSourceName("EmulateInstructionARM64");
diff --git a/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp b/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
index c4a171ec7d01..49edd40544e3 100644
--- a/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
+++ b/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
@@ -424,8 +424,6 @@ size_t UnwindAssemblyInstEmulation::WriteMemory(
     log->PutString(strm.GetString());
   }
 
-  const bool cant_replace = false;
-
   switch (context.type) {
   default:
   case EmulateInstruction::eContextInvalid:
@@ -467,7 +465,7 @@ size_t UnwindAssemblyInstEmulation::WriteMemory(
         m_pushed_regs[reg_num] = addr;
         const int32_t offset = addr - m_initial_sp;
         m_curr_row->SetRegisterLocationToAtCFAPlusOffset(reg_num, offset,
-                                                         cant_replace);
+                                                         /*can_replace=*/true);
         m_curr_row_modified = true;
       }
     }
diff --git a/lldb/source/Target/RegisterContextUnwind.cpp b/lldb/source/Target/RegisterContextUnwind.cpp
index 13e101413a47..e2d712cb72ea 100644
--- a/lldb/source/Target/RegisterContextUnwind.cpp
+++ b/lldb/source/Target/RegisterContextUnwind.cpp
@@ -1555,12 +1555,12 @@ RegisterContextUnwind::SavedLocationForRegister(
   }
 
   if (unwindplan_regloc.IsSame()) {
-    if (!IsFrameZero() &&
+    if (!m_all_registers_available &&
         (regnum.GetAsKind(eRegisterKindGeneric) == LLDB_REGNUM_GENERIC_PC ||
          regnum.GetAsKind(eRegisterKindGeneric) == LLDB_REGNUM_GENERIC_RA)) {
       UnwindLogMsg("register %s (%d) is marked as 'IsSame' - it is a pc or "
-                   "return address reg on a non-zero frame -- treat as if we "
-                   "have no information",
+                   "return address reg on a frame which does not have all "
+                   "registers available -- treat as if we have no information",
                    regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
       return UnwindLLDB::RegisterSearchResult::eRegisterNotFound;
     } else {
diff --git a/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c b/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
new file mode 100644
index 000000000000..9a751330623f
--- /dev/null
+++ b/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
@@ -0,0 +1,15 @@
+#include <signal.h>
+#include <unistd.h>
+
+int __attribute__((naked)) signal_generating_add(int a, int b) {
+  asm("add w0, w1, w0\n\t"
+      "udf #0xdead\n\t"
+      "ret");
+}
+
+void sigill_handler(int) { _exit(0); }
+
+int main() {
+  signal(SIGILL, sigill_handler);
+  return signal_generating_add(42, 47);
+}
diff --git a/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test b/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test
new file mode 100644
index 000000000000..0580d0cf734a
--- /dev/null
+++ b/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test
@@ -0,0 +1,24 @@
+# REQUIRES: target-aarch64 && native
+# UNSUPPORTED: system-windows
+
+# RUN: %clang_host %S/Inputs/signal-in-leaf-function-aarch64.c -o %t
+# RUN: %lldb -s %s -o exit %t | FileCheck %s
+
+breakpoint set -n sigill_handler
+# CHECK: Breakpoint 1: where = {{.*}}`sigill_handler
+
+run
+# CHECK: thread #1, {{.*}} stop reason = signal SIGILL
+
+thread backtrace
+# CHECK: frame #0: [[ADD:0x[0-9a-fA-F]*]] {{.*}}`signal_generating_add
+# CHECK: frame #1: [[MAIN:0x[0-9a-fA-F]*]] {{.*}}`main
+
+continue
+# CHECK: thread #1, {{.*}} stop reason = breakpoint 1
+
+thread backtrace
+# CHECK: frame #0: {{.*}}`sigill_handler
+# Unknown number of signal trampoline frames
+# CHECK: frame #{{[0-9]+}}: [[ADD]] {{.*}}`signal_generating_add
+# CHECK: frame #{{[0-9]+}}: [[MAIN]] {{.*}}`main
diff --git a/lldb/unittests/UnwindAssembly/ARM64/TestArm64InstEmulation.cpp b/lldb/unittests/UnwindAssembly/ARM64/TestArm64InstEmulation.cpp
index 80abeb8fae9e..9303d6f5f3c6 100644
--- a/lldb/unittests/UnwindAssembly/ARM64/TestArm64InstEmulation.cpp
+++ b/lldb/unittests/UnwindAssembly/ARM64/TestArm64InstEmulation.cpp
@@ -77,7 +77,7 @@ TEST_F(TestArm64InstEmulation, TestSimpleDarwinFunction) {
 
   // UnwindPlan we expect:
 
-  // row[0]:    0: CFA=sp +0 =>
+  // row[0]:    0: CFA=sp +0 => fp= <same> lr= <same>
   // row[1]:    4: CFA=sp+16 => fp=[CFA-16] lr=[CFA-8]
   // row[2]:    8: CFA=fp+16 => fp=[CFA-16] lr=[CFA-8]
   // row[2]:   16: CFA=sp+16 => fp=[CFA-16] lr=[CFA-8]
@@ -88,13 +88,19 @@ TEST_F(TestArm64InstEmulation, TestSimpleDarwinFunction) {
   EXPECT_TRUE(engine->GetNonCallSiteUnwindPlanFromAssembly(
       sample_range, data, sizeof(data), unwind_plan));
 
-  // CFA=sp +0
+  // CFA=sp +0 => fp= <same> lr= <same>
   row_sp = unwind_plan.GetRowForFunctionOffset(0);
   EXPECT_EQ(0ull, row_sp->GetOffset());
   EXPECT_TRUE(row_sp->GetCFAValue().GetRegisterNumber() == gpr_sp_arm64);
   EXPECT_TRUE(row_sp->GetCFAValue().IsRegisterPlusOffset() == true);
   EXPECT_EQ(0, row_sp->GetCFAValue().GetOffset());
 
+  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_fp_arm64, regloc));
+  EXPECT_TRUE(regloc.IsSame());
+
+  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_lr_arm64, regloc));
+  EXPECT_TRUE(regloc.IsSame());
+
   // CFA=sp+16 => fp=[CFA-16] lr=[CFA-8]
   row_sp = unwind_plan.GetRowForFunctionOffset(4);
   EXPECT_EQ(4ull, row_sp->GetOffset());
@@ -146,6 +152,12 @@ TEST_F(TestArm64InstEmulation, TestSimpleDarwinFunction) {
   EXPECT_TRUE(row_sp->GetCFAValue().GetRegisterNumber() == gpr_sp_arm64);
   EXPECT_TRUE(row_sp->GetCFAValue().IsRegisterPlusOffset() == true);
   EXPECT_EQ(0, row_sp->GetCFAValue().GetOffset());
+
+  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_fp_arm64, regloc));
+  EXPECT_TRUE(regloc.IsSame());
+
+  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_lr_arm64, regloc));
+  EXPECT_TRUE(regloc.IsSame());
 }
 
 TEST_F(TestArm64InstEmulation, TestMediumDarwinFunction) {
@@ -381,8 +393,12 @@ TEST_F(TestArm64InstEmulation, TestFramelessThreeEpilogueFunction) {
   EXPECT_FALSE(row_sp->GetRegisterInfo(gpr_x26_arm64, regloc));
   EXPECT_FALSE(row_sp->GetRegisterInfo(gpr_x27_arm64, regloc));
   EXPECT_FALSE(row_sp->GetRegisterInfo(gpr_x28_arm64, regloc));
-  EXPECT_FALSE(row_sp->GetRegisterInfo(gpr_fp_arm64, regloc));
-  EXPECT_FALSE(row_sp->GetRegisterInfo(gpr_lr_arm64, regloc));
+
+  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_fp_arm64, regloc));
+  EXPECT_TRUE(regloc.IsSame());
+
+  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_lr_arm64, regloc));
+  EXPECT_TRUE(regloc.IsSame());
 
   row_sp = unwind_plan.GetRowForFunctionOffset(36);
   EXPECT_TRUE(row_sp->GetCFAValue().GetRegisterNumber() == gpr_sp_arm64);
-- 
GitLab


From dec8055a1e71fe25d4b85416ede742e8fdfaf3f0 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 8 May 2024 23:52:22 -0700
Subject: [PATCH 0258/1206] [mlir] Use StringRef::operator== instead of
 StringRef::equals (NFC) (#91560)

I'm planning to remove StringRef::equals in favor of
StringRef::operator==.

- StringRef::operator==/!= outnumber StringRef::equals by a factor of
  10 under mlir/ in terms of their usage.

- The elimination of StringRef::equals brings StringRef closer to
  std::string_view, which has operator== but not equals.

- S == "foo" is more readable than S.equals("foo"), especially for
  !Long.Expression.equals("str") vs Long.Expression != "str".
---
 .../Conversion/GPUCommon/GPUToLLVMConversion.cpp |  2 +-
 mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp  | 16 ++++++++--------
 mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp  |  2 +-
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp         | 14 ++++++--------
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp           | 13 +++++--------
 mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h          |  3 +--
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp         | 14 ++++++--------
 .../SparseTensor/IR/Detail/LvlTypeParser.cpp     |  6 +++---
 mlir/lib/IR/AttributeDetail.h                    |  2 +-
 mlir/lib/TableGen/Builder.cpp                    |  2 +-
 .../Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp   |  4 ++--
 mlir/test/lib/Dialect/Test/TestOpsSyntax.cpp     |  2 +-
 mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp      |  2 +-
 mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp    |  4 ++--
 14 files changed, 39 insertions(+), 47 deletions(-)

diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 3a4fc7d8063f..82bfa9514a88 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -926,7 +926,7 @@ LogicalResult ConvertAsyncYieldToGpuRuntimeCallPattern::matchAndRewrite(
 static bool isDefinedByCallTo(Value value, StringRef functionName) {
   assert(isa<LLVM::LLVMPointerType>(value.getType()));
   if (auto defOp = value.getDefiningOp<LLVM::CallOp>())
-    return defOp.getCallee()->equals(functionName);
+    return *defOp.getCallee() == functionName;
   return false;
 }
 
diff --git a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
index 775dd1e60903..b7fd454c6090 100644
--- a/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/WmmaOpsToNvvm.cpp
@@ -42,11 +42,11 @@ static LogicalResult areAllLLVMTypes(Operation *op, ValueRange operands,
 static constexpr StringRef kInvalidCaseStr = "Unsupported WMMA variant.";
 
 static NVVM::MMAFrag convertOperand(StringRef operandName) {
-  if (operandName.equals("AOp"))
+  if (operandName == "AOp")
     return NVVM::MMAFrag::a;
-  if (operandName.equals("BOp"))
+  if (operandName == "BOp")
     return NVVM::MMAFrag::b;
-  if (operandName.equals("COp"))
+  if (operandName == "COp")
     return NVVM::MMAFrag::c;
   llvm_unreachable("Unknown operand name");
 }
@@ -55,8 +55,8 @@ static NVVM::MMATypes getElementType(gpu::MMAMatrixType type) {
   if (type.getElementType().isF16())
     return NVVM::MMATypes::f16;
   if (type.getElementType().isF32())
-    return type.getOperand().equals("COp") ? NVVM::MMATypes::f32
-                                           : NVVM::MMATypes::tf32;
+    return type.getOperand() == "COp" ? NVVM::MMATypes::f32
+                                      : NVVM::MMATypes::tf32;
 
   if (type.getElementType().isSignedInteger(8))
     return NVVM::MMATypes::s8;
@@ -99,15 +99,15 @@ struct WmmaLoadOpToNVVMLowering
     NVVM::MMATypes eltype = getElementType(retType);
     // NVVM intrinsics require to give mxnxk dimensions, infer the missing
     // dimension based on the valid intrinsics available.
-    if (retType.getOperand().equals("AOp")) {
+    if (retType.getOperand() == "AOp") {
       m = retTypeShape[0];
       k = retTypeShape[1];
       n = NVVM::WMMALoadOp::inferNDimension(m, k, eltype);
-    } else if (retType.getOperand().equals("BOp")) {
+    } else if (retType.getOperand() == "BOp") {
       k = retTypeShape[0];
       n = retTypeShape[1];
       m = NVVM::WMMALoadOp::inferMDimension(k, n, eltype);
-    } else if (retType.getOperand().equals("COp")) {
+    } else if (retType.getOperand() == "COp") {
       m = retTypeShape[0];
       n = retTypeShape[1];
       k = NVVM::WMMALoadOp::inferKDimension(m, n, eltype);
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index f8485e02a220..19f02297bfbb 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -261,7 +261,7 @@ static void maybeApplyPassLabel(OpBuilder &b, OpTy newXferOp,
 template <typename OpTy>
 static bool isTensorOp(OpTy xferOp) {
   if (isa<RankedTensorType>(xferOp.getShapedType())) {
-    if (xferOp.getOperationName().equals(TransferWriteOp::getOperationName())) {
+    if (xferOp.getOperationName() == TransferWriteOp::getOperationName()) {
       // TransferWriteOps on tensors have a result.
       assert(xferOp->getNumResults() > 0);
     }
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index c9c0a7b4cc68..2e31487bd55a 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -3585,20 +3585,18 @@ ParseResult AffinePrefetchOp::parse(OpAsmParser &parser,
       parser.resolveOperands(mapOperands, indexTy, result.operands))
     return failure();
 
-  if (!readOrWrite.equals("read") && !readOrWrite.equals("write"))
+  if (readOrWrite != "read" && readOrWrite != "write")
     return parser.emitError(parser.getNameLoc(),
                             "rw specifier has to be 'read' or 'write'");
-  result.addAttribute(
-      AffinePrefetchOp::getIsWriteAttrStrName(),
-      parser.getBuilder().getBoolAttr(readOrWrite.equals("write")));
+  result.addAttribute(AffinePrefetchOp::getIsWriteAttrStrName(),
+                      parser.getBuilder().getBoolAttr(readOrWrite == "write"));
 
-  if (!cacheType.equals("data") && !cacheType.equals("instr"))
+  if (cacheType != "data" && cacheType != "instr")
     return parser.emitError(parser.getNameLoc(),
                             "cache type has to be 'data' or 'instr'");
 
-  result.addAttribute(
-      AffinePrefetchOp::getIsDataCacheAttrStrName(),
-      parser.getBuilder().getBoolAttr(cacheType.equals("data")));
+  result.addAttribute(AffinePrefetchOp::getIsDataCacheAttrStrName(),
+                      parser.getBuilder().getBoolAttr(cacheType == "data"));
 
   return success();
 }
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index f1b9ca5c5002..0c2590d71130 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -152,8 +152,7 @@ LogicalResult
 MMAMatrixType::verify(function_ref<InFlightDiagnostic()> emitError,
                       ArrayRef<int64_t> shape, Type elementType,
                       StringRef operand) {
-  if (!operand.equals("AOp") && !operand.equals("BOp") &&
-      !operand.equals("COp"))
+  if (operand != "AOp" && operand != "BOp" && operand != "COp")
     return emitError() << "operand expected to be one of AOp, BOp or COp";
 
   if (shape.size() != 2)
@@ -1941,8 +1940,7 @@ LogicalResult SubgroupMmaLoadMatrixOp::verify() {
     return emitError(
         "expected source memref most minor dim must have unit stride");
 
-  if (!operand.equals("AOp") && !operand.equals("BOp") &&
-      !operand.equals("COp"))
+  if (operand != "AOp" && operand != "BOp" && operand != "COp")
     return emitError("only AOp, BOp and COp can be loaded");
 
   return success();
@@ -1962,7 +1960,7 @@ LogicalResult SubgroupMmaStoreMatrixOp::verify() {
     return emitError(
         "expected destination memref most minor dim must have unit stride");
 
-  if (!srcMatrixType.getOperand().equals("COp"))
+  if (srcMatrixType.getOperand() != "COp")
     return emitError(
         "expected the operand matrix being stored to have 'COp' operand type");
 
@@ -1980,9 +1978,8 @@ LogicalResult SubgroupMmaComputeOp::verify() {
   opTypes.push_back(llvm::cast<MMAMatrixType>(getOpB().getType()));
   opTypes.push_back(llvm::cast<MMAMatrixType>(getOpC().getType()));
 
-  if (!opTypes[A].getOperand().equals("AOp") ||
-      !opTypes[B].getOperand().equals("BOp") ||
-      !opTypes[C].getOperand().equals("COp"))
+  if (opTypes[A].getOperand() != "AOp" || opTypes[B].getOperand() != "BOp" ||
+      opTypes[C].getOperand() != "COp")
     return emitError("operands must be in the order AOp, BOp, COp");
 
   ArrayRef<int64_t> aShape, bShape, cShape;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h b/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
index 2040d0a06b2e..8767b1c3ffc5 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
+++ b/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
@@ -131,8 +131,7 @@ public:
     /// Compares two keys.
     bool operator==(const Key &other) const {
       if (isIdentified())
-        return other.isIdentified() &&
-               other.getIdentifier().equals(getIdentifier());
+        return other.isIdentified() && other.getIdentifier() == getIdentifier();
 
       return !other.isIdentified() && other.isPacked() == isPacked() &&
              other.getTypeList() == getTypeList();
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index c9a85919ec79..199e7330a233 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -1742,20 +1742,18 @@ ParseResult PrefetchOp::parse(OpAsmParser &parser, OperationState &result) {
       parser.resolveOperands(indexInfo, indexTy, result.operands))
     return failure();
 
-  if (!readOrWrite.equals("read") && !readOrWrite.equals("write"))
+  if (readOrWrite != "read" && readOrWrite != "write")
     return parser.emitError(parser.getNameLoc(),
                             "rw specifier has to be 'read' or 'write'");
-  result.addAttribute(
-      PrefetchOp::getIsWriteAttrStrName(),
-      parser.getBuilder().getBoolAttr(readOrWrite.equals("write")));
+  result.addAttribute(PrefetchOp::getIsWriteAttrStrName(),
+                      parser.getBuilder().getBoolAttr(readOrWrite == "write"));
 
-  if (!cacheType.equals("data") && !cacheType.equals("instr"))
+  if (cacheType != "data" && cacheType != "instr")
     return parser.emitError(parser.getNameLoc(),
                             "cache type has to be 'data' or 'instr'");
 
-  result.addAttribute(
-      PrefetchOp::getIsDataCacheAttrStrName(),
-      parser.getBuilder().getBoolAttr(cacheType.equals("data")));
+  result.addAttribute(PrefetchOp::getIsDataCacheAttrStrName(),
+                      parser.getBuilder().getBoolAttr(cacheType == "data"));
 
   return success();
 }
diff --git a/mlir/lib/Dialect/SparseTensor/IR/Detail/LvlTypeParser.cpp b/mlir/lib/Dialect/SparseTensor/IR/Detail/LvlTypeParser.cpp
index 92e5efaa8104..39f5cf1a7508 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/Detail/LvlTypeParser.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/Detail/LvlTypeParser.cpp
@@ -89,11 +89,11 @@ ParseResult LvlTypeParser::parseProperty(AsmParser &parser,
   auto loc = parser.getCurrentLocation();
   ERROR_IF(failed(parser.parseOptionalKeyword(&strVal)),
            "expected valid level property (e.g. nonordered, nonunique or high)")
-  if (strVal.equals(toPropString(LevelPropNonDefault::Nonunique))) {
+  if (strVal == toPropString(LevelPropNonDefault::Nonunique)) {
     *properties |= static_cast<uint64_t>(LevelPropNonDefault::Nonunique);
-  } else if (strVal.equals(toPropString(LevelPropNonDefault::Nonordered))) {
+  } else if (strVal == toPropString(LevelPropNonDefault::Nonordered)) {
     *properties |= static_cast<uint64_t>(LevelPropNonDefault::Nonordered);
-  } else if (strVal.equals(toPropString(LevelPropNonDefault::SoA))) {
+  } else if (strVal == toPropString(LevelPropNonDefault::SoA)) {
     *properties |= static_cast<uint64_t>(LevelPropNonDefault::SoA);
   } else {
     parser.emitError(loc, "unknown level property: ") << strVal;
diff --git a/mlir/lib/IR/AttributeDetail.h b/mlir/lib/IR/AttributeDetail.h
index dcd24af0107d..26d40ac3a38f 100644
--- a/mlir/lib/IR/AttributeDetail.h
+++ b/mlir/lib/IR/AttributeDetail.h
@@ -261,7 +261,7 @@ struct DenseStringElementsAttrStorage : public DenseElementsAttributeStorage {
     // Check to see if this storage represents a splat. If it doesn't then
     // combine the hash for the data starting with the first non splat element.
     for (size_t i = 1, e = data.size(); i != e; i++)
-      if (!firstElt.equals(data[i]))
+      if (firstElt != data[i])
         return KeyTy(ty, data, llvm::hash_combine(hashVal, data.drop_front(i)));
 
     // Otherwise, this is a splat so just return the hash of the first element.
diff --git a/mlir/lib/TableGen/Builder.cpp b/mlir/lib/TableGen/Builder.cpp
index 47a2f6cc4456..044765c72601 100644
--- a/mlir/lib/TableGen/Builder.cpp
+++ b/mlir/lib/TableGen/Builder.cpp
@@ -52,7 +52,7 @@ Builder::Builder(const llvm::Record *record, ArrayRef<SMLoc> loc)
   // Initialize the parameters of the builder.
   const llvm::DagInit *dag = def->getValueAsDag("dagParams");
   auto *defInit = dyn_cast<llvm::DefInit>(dag->getOperator());
-  if (!defInit || !defInit->getDef()->getName().equals("ins"))
+  if (!defInit || defInit->getDef()->getName() != "ins")
     PrintFatalError(def->getLoc(), "expected 'ins' in builders");
 
   bool seenDefaultValue = false;
diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
index 40d8253d822f..06673965245c 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
@@ -93,7 +93,7 @@ static LogicalResult setProfilingAttr(OpBuilder &builder, llvm::MDNode *node,
     return failure();
 
   // Handle function entry count metadata.
-  if (name->getString().equals("function_entry_count")) {
+  if (name->getString() == "function_entry_count") {
 
     // TODO support function entry count metadata with GUID fields.
     if (node->getNumOperands() != 2)
@@ -111,7 +111,7 @@ static LogicalResult setProfilingAttr(OpBuilder &builder, llvm::MDNode *node,
            << "expected function_entry_count to be attached to a function";
   }
 
-  if (!name->getString().equals("branch_weights"))
+  if (name->getString() != "branch_weights")
     return failure();
 
   // Handle branch weights metadata.
diff --git a/mlir/test/lib/Dialect/Test/TestOpsSyntax.cpp b/mlir/test/lib/Dialect/Test/TestOpsSyntax.cpp
index c376d6c73c64..ebaced57a24a 100644
--- a/mlir/test/lib/Dialect/Test/TestOpsSyntax.cpp
+++ b/mlir/test/lib/Dialect/Test/TestOpsSyntax.cpp
@@ -413,7 +413,7 @@ void PrettyPrintedRegionOp::print(OpAsmPrinter &p) {
   // of inner-op), then we can print the entire region in a succinct way.
   // Here we assume that the prototype of "test.special.op" can be trivially
   // derived while parsing it back.
-  if (innerOp.getName().getStringRef().equals("test.special.op")) {
+  if (innerOp.getName().getStringRef() == "test.special.op") {
     p << " start test.special.op end";
   } else {
     p << " (";
diff --git a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
index b9a72119790e..55bc0714c20e 100644
--- a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
+++ b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
@@ -50,7 +50,7 @@ static void collectAllDefs(StringRef selectedDialect,
   } else {
     // Otherwise, generate the defs that belong to the selected dialect.
     auto dialectDefs = llvm::make_filter_range(defs, [&](const auto &def) {
-      return def.getDialect().getName().equals(selectedDialect);
+      return def.getDialect().getName() == selectedDialect;
     });
     resultDefs.assign(dialectDefs.begin(), dialectDefs.end());
   }
diff --git a/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp b/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
index 814008c25451..052020acdcb7 100644
--- a/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
@@ -457,7 +457,7 @@ static void emitAttributeAccessors(const Operator &op, raw_ostream &os) {
     std::string sanitizedName = sanitizeName(namedAttr.name);
 
     // Unit attributes are handled specially.
-    if (namedAttr.attr.getStorageType().trim().equals("::mlir::UnitAttr")) {
+    if (namedAttr.attr.getStorageType().trim() == "::mlir::UnitAttr") {
       os << llvm::formatv(unitAttributeGetterTemplate, sanitizedName,
                           namedAttr.name);
       os << llvm::formatv(unitAttributeSetterTemplate, sanitizedName,
@@ -668,7 +668,7 @@ populateBuilderLinesAttr(const Operator &op,
       continue;
 
     // Unit attributes are handled specially.
-    if (attribute->attr.getStorageType().trim().equals("::mlir::UnitAttr")) {
+    if (attribute->attr.getStorageType().trim() == "::mlir::UnitAttr") {
       builderLines.push_back(llvm::formatv(initUnitAttributeTemplate,
                                            attribute->name, argNames[i]));
       continue;
-- 
GitLab


From aacea0d0f67401f5a0b74947f3ff179ade9cbf6d Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Wed, 8 May 2024 23:58:55 -0700
Subject: [PATCH 0259/1206] [utils] Add script to generate elaborated IR and
 assembly tests (#89026)

Generally, IR and assembly test files benefit from being cleaned to
remove unnecessary details. However, for tests requiring elaborate
IR or assembly files where cleanup is less practical (e.g., large amount
of debug information output from Clang), the current practice is to
include the C/C++ source file and the generation instructions as
comments.

This is inconvenient when regeneration is needed. This patch adds
`llvm/utils/update_test_body.py` to allow easier regeneration.

`ld.lld --debug-names` tests (#86508) utilize this script for
Clang-generated assembly tests.

Note: `-o pipefail` is standard (since
https://www.austingroupbugs.net/view.php?id=789) but not supported by
dash.

Link:
https://discourse.llvm.org/t/utility-to-generate-elaborated-assembly-ir-tests/78408
---
 llvm/docs/TestingGuide.rst                    |  81 +++++++++++++
 .../test/tools/UpdateTestChecks/lit.local.cfg |   7 +-
 .../Inputs/basic-asm.test.expected            |  13 +++
 .../Inputs/basic.test.expected                |  16 +++
 .../update_test_body/basic-asm.test           |  11 ++
 .../update_test_body/basic.test               |  13 +++
 .../update_test_body/empty-stdout.test        |  13 +++
 .../update_test_body/gen-absent.test          |   7 ++
 .../update_test_body/gen-fail.test            |  11 ++
 .../update_test_body/gen-unterminated.test    |   8 ++
 .../update_test_body/lit.local.cfg            |   4 +
 .../tools/llvm-dwarfdump/X86/formclass4.s     |  26 +++--
 .../X86/prettyprint_type_units_split_v5.s     |  14 +--
 llvm/utils/update_test_body.py                | 110 ++++++++++++++++++
 14 files changed, 314 insertions(+), 20 deletions(-)
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_body/Inputs/basic-asm.test.expected
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_body/Inputs/basic.test.expected
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_body/basic-asm.test
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_body/basic.test
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_body/empty-stdout.test
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_body/gen-absent.test
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_body/gen-fail.test
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_body/gen-unterminated.test
 create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_body/lit.local.cfg
 create mode 100755 llvm/utils/update_test_body.py

diff --git a/llvm/docs/TestingGuide.rst b/llvm/docs/TestingGuide.rst
index e32e4d1e535a..e24feb3bf5fa 100644
--- a/llvm/docs/TestingGuide.rst
+++ b/llvm/docs/TestingGuide.rst
@@ -433,6 +433,87 @@ actually participate in the test besides holding the ``RUN:`` lines.
   putting the extra files in an ``Inputs/`` directory. This pattern is
   deprecated.
 
+Elaborated tests
+----------------
+
+Generally, IR and assembly test files benefit from being cleaned to remove
+unnecessary details. However, for tests requiring elaborate IR or assembly
+files where cleanup is less practical (e.g., large amount of debug information
+output from Clang), you can include generation instructions within
+``split-file`` part called ``gen``. Then, run
+``llvm/utils/update_test_body.py`` on the test file to generate the needed
+content.
+
+.. code-block:: none
+
+    ; RUN: rm -rf %t && split-file %s %t && cd %t
+    ; RUN: opt -S a.ll ... | FileCheck %s
+
+    ; CHECK: hello
+
+    ;--- a.cc
+    int va;
+    ;--- gen
+    clang --target=x86_64-linux -S -emit-llvm -g a.cc -o -
+
+    ;--- a.ll
+    # content generated by the script 'gen'
+
+.. code-block:: bash
+
+   PATH=/path/to/clang_build/bin:$PATH llvm/utils/update_test_body.py path/to/test.ll
+
+The script will prepare extra files with ``split-file``, invoke ``gen``, and
+then rewrite the part after ``gen`` with its stdout.
+
+For convenience, if the test needs one single assembly file, you can also wrap
+``gen`` and its required files with ``.ifdef`` and ``.endif``. Then you can
+skip ``split-file`` in RUN lines.
+
+.. code-block:: none
+
+    # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o a.o
+    # RUN: ... | FileCheck %s
+
+    # CHECK: hello
+
+    .ifdef GEN
+    #--- a.cc
+    int va;
+    #--- gen
+    clang --target=x86_64-linux -S -g a.cc -o -
+    .endif
+    # content generated by the script 'gen'
+
+.. note::
+
+  Consider specifying an explicit target triple to avoid differences when
+  regeneration is needed on another machine.
+
+  ``gen`` is invoked with ``PWD`` set to ``/proc/self/cwd``. Clang commands
+  don't need ``-fdebug-compilation-dir=`` since its default value is ``PWD``.
+
+  Check prefixes should be placed before ``.endif`` since the part after
+  ``.endif`` is replaced.
+
+If the test body contains multiple files, you can print ``---`` separators and
+utilize ``split-file`` in ``RUN`` lines.
+
+.. code-block:: none
+
+    # RUN: rm -rf %t && split-file %s %t && cd %t
+    ...
+
+    #--- a.cc
+    int va;
+    #--- b.cc
+    int vb;
+    #--- gen
+    clang --target=x86_64-linux -S -O1 -g a.cc -o -
+    echo '#--- b.s'
+    clang --target=x86_64-linux -S -O1 -g b.cc -o -
+    #--- a.s
+
 Fragile tests
 -------------
 
diff --git a/llvm/test/tools/UpdateTestChecks/lit.local.cfg b/llvm/test/tools/UpdateTestChecks/lit.local.cfg
index f8ab6b82cde7..2e695490b005 100644
--- a/llvm/test/tools/UpdateTestChecks/lit.local.cfg
+++ b/llvm/test/tools/UpdateTestChecks/lit.local.cfg
@@ -19,7 +19,8 @@ def add_update_script_substition(
     # Specify an explicit default version in UTC tests, so that the --version
     # embedded in UTC_ARGS does not change in all test expectations every time
     # the default is bumped.
-    extra_args += " --version=1"
+    if name != "%update_test_body":
+        extra_args += " --version=1"
     config.substitutions.append(
         (name, "'%s' %s %s" % (python_exe, script_path, extra_args))
     )
@@ -47,3 +48,7 @@ if os.path.isfile(llvm_mca_path):
     config.available_features.add("llvm-mca-binary")
     mca_arg = "--llvm-mca-binary " + shell_quote(llvm_mca_path)
     add_update_script_substition("%update_test_checks", extra_args=mca_arg)
+
+split_file_path = os.path.join(config.llvm_tools_dir, "split-file")
+if os.path.isfile(split_file_path):
+    add_update_script_substition("%update_test_body")
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_body/Inputs/basic-asm.test.expected b/llvm/test/tools/UpdateTestChecks/update_test_body/Inputs/basic-asm.test.expected
new file mode 100644
index 000000000000..05024d8799cd
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_body/Inputs/basic-asm.test.expected
@@ -0,0 +1,13 @@
+# RUN: cp %s %t && %update_test_body %t 2>&1 | count 0
+# RUN: diff -u %S/Inputs/basic-asm.test.expected %t
+
+.ifdef GEN
+#--- a.txt
+.long 0
+#--- b.txt
+.long 1
+#--- gen
+cat a.txt b.txt
+.endif
+.long 0
+.long 1
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_body/Inputs/basic.test.expected b/llvm/test/tools/UpdateTestChecks/update_test_body/Inputs/basic.test.expected
new file mode 100644
index 000000000000..80a2676d0a75
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_body/Inputs/basic.test.expected
@@ -0,0 +1,16 @@
+; RUN: cp %s %t && %update_test_body %t 2>&1 | count 0
+; RUN: diff -u %S/Inputs/basic.test.expected %t
+
+;--- a.txt
+@a = global i32 0
+;--- b.txt
+@b = global i32 0
+;--- gen
+cat a.txt
+echo ';--- b.ll'
+cat b.txt
+
+;--- a.ll
+@a = global i32 0
+;--- b.ll
+@b = global i32 0
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_body/basic-asm.test b/llvm/test/tools/UpdateTestChecks/update_test_body/basic-asm.test
new file mode 100644
index 000000000000..3e82a3ffab9a
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_body/basic-asm.test
@@ -0,0 +1,11 @@
+# RUN: cp %s %t && %update_test_body %t 2>&1 | count 0
+# RUN: diff -u %S/Inputs/basic-asm.test.expected %t
+
+.ifdef GEN
+#--- a.txt
+.long 0
+#--- b.txt
+.long 1
+#--- gen
+cat a.txt b.txt
+.endif
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_body/basic.test b/llvm/test/tools/UpdateTestChecks/update_test_body/basic.test
new file mode 100644
index 000000000000..d99946e2bd92
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_body/basic.test
@@ -0,0 +1,13 @@
+; RUN: cp %s %t && %update_test_body %t 2>&1 | count 0
+; RUN: diff -u %S/Inputs/basic.test.expected %t
+
+;--- a.txt
+@a = global i32 0
+;--- b.txt
+@b = global i32 0
+;--- gen
+cat a.txt
+echo ';--- b.ll'
+cat b.txt
+
+;--- a.ll
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_body/empty-stdout.test b/llvm/test/tools/UpdateTestChecks/update_test_body/empty-stdout.test
new file mode 100644
index 000000000000..9ea9c7bc7ac9
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_body/empty-stdout.test
@@ -0,0 +1,13 @@
+# RUN: cp %s %t && not %update_test_body %t 2>&1 | FileCheck %s
+# RUN: diff -u %t %s
+
+# CHECK: stdout is empty; forgot -o - ?
+
+.ifdef GEN
+#--- a.txt
+.long 0
+#--- b.txt
+.long 1
+#--- gen
+true
+.endif
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_body/gen-absent.test b/llvm/test/tools/UpdateTestChecks/update_test_body/gen-absent.test
new file mode 100644
index 000000000000..c12f22adceb2
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_body/gen-absent.test
@@ -0,0 +1,7 @@
+# RUN: cp %s %t && not %update_test_body %t 2>&1 | FileCheck %s
+
+# CHECK: 'gen' does not exist
+
+.ifdef GEN
+#--- a.txt
+.endif
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_body/gen-fail.test b/llvm/test/tools/UpdateTestChecks/update_test_body/gen-fail.test
new file mode 100644
index 000000000000..7e1a9365df14
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_body/gen-fail.test
@@ -0,0 +1,11 @@
+# RUN: cp %s %t && not %update_test_body %t 2>&1 | FileCheck %s
+
+# CHECK:      log
+# CHECK-NEXT: 'gen' failed
+
+.ifdef GEN
+#--- gen
+echo log >&2
+false  # gen fails due to sh -e
+true
+.endif
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_body/gen-unterminated.test b/llvm/test/tools/UpdateTestChecks/update_test_body/gen-unterminated.test
new file mode 100644
index 000000000000..c0026939e414
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_body/gen-unterminated.test
@@ -0,0 +1,8 @@
+# RUN: cp %s %t && not %update_test_body %t 2>&1 | FileCheck %s
+
+# CHECK: 'gen' should be followed by another part (---) or .endif
+
+#--- a.txt
+.long 0
+#--- gen
+cat a.txt
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_body/lit.local.cfg b/llvm/test/tools/UpdateTestChecks/update_test_body/lit.local.cfg
new file mode 100644
index 000000000000..1bb2464ad957
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_body/lit.local.cfg
@@ -0,0 +1,4 @@
+import platform
+
+if platform.system() == "Windows":
+    config.unsupported = True
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/formclass4.s b/llvm/test/tools/llvm-dwarfdump/X86/formclass4.s
index d0f8857c638f..5b3cdfc97790 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/formclass4.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/formclass4.s
@@ -1,15 +1,3 @@
-# Source:
-#   struct e {
-#     enum {} f[16384];
-#     short g;
-#   };
-#   e foo() {
-#     auto E = new e;
-#     return *E;
-#   }
-# Compile with:
-#   clang -O2 -gdwarf-4 -S a.cpp -o a4.s
-
 # RUN: llvm-mc %s -filetype obj -triple x86_64-apple-darwin -o %t.o
 # RUN: llvm-dwarfdump -debug-info -name g %t.o | FileCheck %s
 
@@ -17,6 +5,20 @@
 # CHECK: DW_AT_name ("g")
 # CHECK: DW_AT_data_member_location    (0x4000)
 
+.ifdef GEN
+#--- a.cpp
+struct e {
+  enum {} f[16384];
+  short g;
+};
+e foo() {
+  auto E = new e;
+  return *E;
+}
+#--- gen
+clang --target=x86_64-apple-macosx -O2 -gdwarf-4 -S a.cpp -o -
+.endif
+
 	.section	__TEXT,__text,regular,pure_instructions
 	.macosx_version_min 10, 14
 	.globl	__Z3foov                ## -- Begin function _Z3foov
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/prettyprint_type_units_split_v5.s b/llvm/test/tools/llvm-dwarfdump/X86/prettyprint_type_units_split_v5.s
index e8bb95175087..81d15cd2be22 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/prettyprint_type_units_split_v5.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/prettyprint_type_units_split_v5.s
@@ -1,16 +1,16 @@
 # RUN: llvm-mc < %s -filetype obj -triple x86_64 -o - \
 # RUN:   | llvm-dwarfdump - | FileCheck %s
 
-# Generated from:
-#
-#   struct t1 { };
-#   t1 v1;
-#
-# $ clang++ -S -g -fdebug-types-section -gsplit-dwarf -o test.5.split.s -gdwarf-5 -g
-
 # CHECK: DW_TAG_variable
 # CHECK:   DW_AT_type ({{.*}} "t1")
 
+.ifdef GEN
+#--- test.cpp
+struct t1 { };
+t1 v1;
+#--- gen
+clang++ --target=x86_64-linux -S -g -fdebug-types-section -gsplit-dwarf -gdwarf-5 test.cpp -o -
+.endif
 	.text
 	.file	"test.cpp"
 	.section	.debug_types.dwo,"e",@progbits
diff --git a/llvm/utils/update_test_body.py b/llvm/utils/update_test_body.py
new file mode 100755
index 000000000000..661b0270d783
--- /dev/null
+++ b/llvm/utils/update_test_body.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Generate test body using split-file and a custom script.
+
+The script will prepare extra files with `split-file`, invoke `gen`, and then
+rewrite the part after `gen` with its stdout.
+
+https://llvm.org/docs/TestingGuide.html#elaborated-tests
+
+Example:
+PATH=/path/to/clang_build/bin:$PATH llvm/utils/update_test_body.py path/to/test.s
+"""
+import argparse
+import contextlib
+import os
+import re
+import subprocess
+import sys
+import tempfile
+
+
+@contextlib.contextmanager
+def cd(directory):
+    cwd = os.getcwd()
+    os.chdir(directory)
+    try:
+        yield
+    finally:
+        os.chdir(cwd)
+
+
+def process(args, path):
+    prolog = []
+    seen_gen = False
+    with open(path) as f:
+        for line in f.readlines():
+            line = line.rstrip()
+            prolog.append(line)
+            if (seen_gen and re.match(r"(.|//)---", line)) or line.startswith(".endif"):
+                break
+            if re.match(r"(.|//)--- gen", line):
+                seen_gen = True
+        else:
+            print(
+                "'gen' should be followed by another part (---) or .endif",
+                file=sys.stderr,
+            )
+            return 1
+
+    if not seen_gen:
+        print("'gen' does not exist", file=sys.stderr)
+        return 1
+    with tempfile.TemporaryDirectory(prefix="update_test_body_") as dir:
+        try:
+            # If the last line starts with ".endif", remove it.
+            sub = subprocess.run(
+                ["split-file", "-", dir],
+                input="\n".join(
+                    prolog[:-1] if prolog[-1].startswith(".endif") else prolog
+                ).encode(),
+                capture_output=True,
+                check=True,
+            )
+        except subprocess.CalledProcessError as ex:
+            sys.stderr.write(ex.stderr.decode())
+            return 1
+        with cd(dir):
+            if args.shell:
+                print(f"invoke shell in the temporary directory '{dir}'")
+                subprocess.run([os.environ.get("SHELL", "sh")])
+                return 0
+
+            sub = subprocess.run(
+                ["sh", "-eu", "gen"],
+                capture_output=True,
+                # Don't encode the directory information to the Clang output.
+                # Remove unneeded details (.ident) as well.
+                env=dict(
+                    os.environ,
+                    CCC_OVERRIDE_OPTIONS="#^-fno-ident",
+                    PWD="/proc/self/cwd",
+                ),
+            )
+            sys.stderr.write(sub.stderr.decode())
+            if sub.returncode != 0:
+                print("'gen' failed", file=sys.stderr)
+                return sub.returncode
+            if not sub.stdout:
+                print("stdout is empty; forgot -o - ?", file=sys.stderr)
+                return 1
+            content = sub.stdout.decode()
+
+    with open(path, "w") as f:
+        # Print lines up to '.endif'.
+        print("\n".join(prolog), file=f)
+        # Then print the stdout of 'gen'.
+        f.write(content)
+
+
+parser = argparse.ArgumentParser(
+    description="Generate test body using split-file and a custom script"
+)
+parser.add_argument("files", nargs="+")
+parser.add_argument(
+    "--shell", action="store_true", help="invoke shell instead of 'gen'"
+)
+args = parser.parse_args()
+for path in args.files:
+    retcode = process(args, path)
+    if retcode != 0:
+        sys.exit(retcode)
-- 
GitLab


From c4a3d184db5fdffe798208b8281dfe944616f9ed Mon Sep 17 00:00:00 2001
From: Vlad Mishel <43666597+vmishelcs@users.noreply.github.com>
Date: Thu, 9 May 2024 00:06:18 -0700
Subject: [PATCH 0260/1206] [libc] Replace `MutexLock` with `cpp::lock_guard`
 (#89340)

This PR address issue #89002.

#### Changes in this PR

* Added a simple implementation of `cpp::lock_guard` (an equivalent of
`std::lock_guard`) in libc/src/__support/CPP inspired by the libstdc++
implementation
* Added tests for `cpp::lock_guard` in
/libc/test/src/__support/CPP/mutex_test.cpp
* Replaced all references to `MutexLock` with `cpp::lock_guard`

---------

Co-authored-by: Guillaume Chatelet <gchatelet@google.com>
---
 libc/src/__support/CPP/CMakeLists.txt         |  6 ++
 libc/src/__support/CPP/mutex.h                | 46 +++++++++++
 libc/src/__support/File/CMakeLists.txt        |  1 +
 libc/src/__support/File/dir.cpp               |  5 +-
 libc/src/__support/threads/CMakeLists.txt     |  2 +
 libc/src/__support/threads/fork_callbacks.cpp |  9 ++-
 libc/src/__support/threads/thread.cpp         | 11 +--
 libc/src/stdlib/CMakeLists.txt                |  1 +
 libc/src/stdlib/atexit.cpp                    |  3 +-
 libc/src/threads/linux/CMakeLists.txt         |  1 +
 libc/src/threads/linux/CndVar.h               |  5 +-
 libc/test/src/__support/CPP/CMakeLists.txt    | 10 +++
 libc/test/src/__support/CPP/mutex_test.cpp    | 79 +++++++++++++++++++
 13 files changed, 165 insertions(+), 14 deletions(-)
 create mode 100644 libc/src/__support/CPP/mutex.h
 create mode 100644 libc/test/src/__support/CPP/mutex_test.cpp

diff --git a/libc/src/__support/CPP/CMakeLists.txt b/libc/src/__support/CPP/CMakeLists.txt
index 84d01fe04516..08661aba5b6b 100644
--- a/libc/src/__support/CPP/CMakeLists.txt
+++ b/libc/src/__support/CPP/CMakeLists.txt
@@ -51,6 +51,12 @@ add_header_library(
     libc.src.__support.macros.properties.types
 )
 
+add_header_library(
+  mutex
+  HDRS
+    mutex.h
+)
+
 add_header_library(
   span
   HDRS
diff --git a/libc/src/__support/CPP/mutex.h b/libc/src/__support/CPP/mutex.h
new file mode 100644
index 000000000000..c25c1155b766
--- /dev/null
+++ b/libc/src/__support/CPP/mutex.h
@@ -0,0 +1,46 @@
+//===--- A self contained equivalent of std::mutex --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_MUTEX_H
+#define LLVM_LIBC_SRC___SUPPORT_CPP_MUTEX_H
+
+namespace LIBC_NAMESPACE {
+namespace cpp {
+
+// Assume the calling thread has already obtained mutex ownership.
+struct adopt_lock_t {
+  explicit adopt_lock_t() = default;
+};
+
+// Tag used to make a scoped lock take ownership of a locked mutex.
+constexpr adopt_lock_t adopt_lock{};
+
+// An RAII class for easy locking and unlocking of mutexes.
+template <typename MutexType> class lock_guard {
+  MutexType &mutex;
+
+public:
+  // Calls `m.lock()` upon resource acquisition.
+  explicit lock_guard(MutexType &m) : mutex(m) { mutex.lock(); }
+
+  // Acquires ownership of the mutex object `m` without attempting to lock
+  // it. The behavior is undefined if the current thread does not hold the
+  // lock on `m`. Does not call `m.lock()` upon resource acquisition.
+  lock_guard(MutexType &m, adopt_lock_t t) : mutex(m) {}
+
+  ~lock_guard() { mutex.unlock(); }
+
+  // non-copyable
+  lock_guard &operator=(const lock_guard &) = delete;
+  lock_guard(const lock_guard &) = delete;
+};
+
+} // namespace cpp
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_CPP_MUTEX_H
diff --git a/libc/src/__support/File/CMakeLists.txt b/libc/src/__support/File/CMakeLists.txt
index b7c0612096aa..0416ac2cc902 100644
--- a/libc/src/__support/File/CMakeLists.txt
+++ b/libc/src/__support/File/CMakeLists.txt
@@ -25,6 +25,7 @@ add_object_library(
   HDRS
     dir.h
   DEPENDS
+    libc.src.__support.CPP.mutex
     libc.src.__support.CPP.new
     libc.src.__support.CPP.span
     libc.src.__support.threads.mutex
diff --git a/libc/src/__support/File/dir.cpp b/libc/src/__support/File/dir.cpp
index 9ff639a777e2..e0f7695b3932 100644
--- a/libc/src/__support/File/dir.cpp
+++ b/libc/src/__support/File/dir.cpp
@@ -8,6 +8,7 @@
 
 #include "dir.h"
 
+#include "src/__support/CPP/mutex.h" // lock_guard
 #include "src/__support/CPP/new.h"
 #include "src/__support/error_or.h"
 #include "src/errno/libc_errno.h" // For error macros
@@ -27,7 +28,7 @@ ErrorOr<Dir *> Dir::open(const char *path) {
 }
 
 ErrorOr<struct ::dirent *> Dir::read() {
-  MutexLock lock(&mutex);
+  cpp::lock_guard lock(mutex);
   if (readptr >= fillsize) {
     auto readsize = platform_fetch_dirents(fd, buffer);
     if (!readsize)
@@ -51,7 +52,7 @@ ErrorOr<struct ::dirent *> Dir::read() {
 
 int Dir::close() {
   {
-    MutexLock lock(&mutex);
+    cpp::lock_guard lock(mutex);
     int retval = platform_closedir(fd);
     if (retval != 0)
       return retval;
diff --git a/libc/src/__support/threads/CMakeLists.txt b/libc/src/__support/threads/CMakeLists.txt
index 731adf6f9c8e..34412be4dfed 100644
--- a/libc/src/__support/threads/CMakeLists.txt
+++ b/libc/src/__support/threads/CMakeLists.txt
@@ -31,6 +31,7 @@ if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.mutex)
       fork_callbacks.h
     DEPENDS
       .mutex
+      libc.src.__support.CPP.mutex
   )
 endif()
 
@@ -57,6 +58,7 @@ if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.thread)
       libc.src.__support.common
       libc.src.__support.fixedvector
       libc.src.__support.CPP.array
+      libc.src.__support.CPP.mutex
       libc.src.__support.CPP.optional
   )
 endif()
diff --git a/libc/src/__support/threads/fork_callbacks.cpp b/libc/src/__support/threads/fork_callbacks.cpp
index 54fda676f281..6efaf62f135a 100644
--- a/libc/src/__support/threads/fork_callbacks.cpp
+++ b/libc/src/__support/threads/fork_callbacks.cpp
@@ -8,6 +8,7 @@
 
 #include "fork_callbacks.h"
 
+#include "src/__support/CPP/mutex.h" // lock_guard
 #include "src/__support/threads/mutex.h"
 
 #include <stddef.h> // For size_t
@@ -35,7 +36,7 @@ public:
   constexpr AtForkCallbackManager() : mtx(false, false, false), next_index(0) {}
 
   bool register_triple(const ForkCallbackTriple &triple) {
-    MutexLock lock(&mtx);
+    cpp::lock_guard lock(mtx);
     if (next_index >= CALLBACK_SIZE)
       return false;
     list[next_index] = triple;
@@ -44,7 +45,7 @@ public:
   }
 
   void invoke_prepare() {
-    MutexLock lock(&mtx);
+    cpp::lock_guard lock(mtx);
     for (size_t i = 0; i < next_index; ++i) {
       auto prepare = list[i].prepare;
       if (prepare)
@@ -53,7 +54,7 @@ public:
   }
 
   void invoke_parent() {
-    MutexLock lock(&mtx);
+    cpp::lock_guard lock(mtx);
     for (size_t i = 0; i < next_index; ++i) {
       auto parent = list[i].parent;
       if (parent)
@@ -62,7 +63,7 @@ public:
   }
 
   void invoke_child() {
-    MutexLock lock(&mtx);
+    cpp::lock_guard lock(mtx);
     for (size_t i = 0; i < next_index; ++i) {
       auto child = list[i].child;
       if (child)
diff --git a/libc/src/__support/threads/thread.cpp b/libc/src/__support/threads/thread.cpp
index c1785343671c..7b02f8246e24 100644
--- a/libc/src/__support/threads/thread.cpp
+++ b/libc/src/__support/threads/thread.cpp
@@ -10,6 +10,7 @@
 #include "src/__support/threads/mutex.h"
 
 #include "src/__support/CPP/array.h"
+#include "src/__support/CPP/mutex.h" // lock_guard
 #include "src/__support/CPP/optional.h"
 #include "src/__support/fixedvector.h"
 #include "src/__support/macros/attributes.h"
@@ -56,7 +57,7 @@ public:
   constexpr TSSKeyMgr() : mtx(false, false, false) {}
 
   cpp::optional<unsigned int> new_key(TSSDtor *dtor) {
-    MutexLock lock(&mtx);
+    cpp::lock_guard lock(mtx);
     for (unsigned int i = 0; i < TSS_KEY_COUNT; ++i) {
       TSSKeyUnit &u = units[i];
       if (!u.active) {
@@ -70,20 +71,20 @@ public:
   TSSDtor *get_dtor(unsigned int key) {
     if (key >= TSS_KEY_COUNT)
       return nullptr;
-    MutexLock lock(&mtx);
+    cpp::lock_guard lock(mtx);
     return units[key].dtor;
   }
 
   bool remove_key(unsigned int key) {
     if (key >= TSS_KEY_COUNT)
       return false;
-    MutexLock lock(&mtx);
+    cpp::lock_guard lock(mtx);
     units[key].reset();
     return true;
   }
 
   bool is_valid_key(unsigned int key) {
-    MutexLock lock(&mtx);
+    cpp::lock_guard lock(mtx);
     return units[key].active;
   }
 };
@@ -113,7 +114,7 @@ public:
   constexpr ThreadAtExitCallbackMgr() : mtx(false, false, false) {}
 
   int add_callback(AtExitCallback *callback, void *obj) {
-    MutexLock lock(&mtx);
+    cpp::lock_guard lock(mtx);
     return callback_list.push_back({callback, obj});
   }
 
diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt
index e526ba040bef..9b76a6a0f857 100644
--- a/libc/src/stdlib/CMakeLists.txt
+++ b/libc/src/stdlib/CMakeLists.txt
@@ -414,6 +414,7 @@ add_entrypoint_object(
   CXX_STANDARD
     20 # For constinit of the atexit callback list.
   DEPENDS
+    libc.src.__support.CPP.mutex
     libc.src.__support.CPP.new
     libc.src.__support.OSUtil.osutil
     libc.src.__support.blockstore
diff --git a/libc/src/stdlib/atexit.cpp b/libc/src/stdlib/atexit.cpp
index fa072b2fdf8d..4f0497444773 100644
--- a/libc/src/stdlib/atexit.cpp
+++ b/libc/src/stdlib/atexit.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdlib/atexit.h"
+#include "src/__support/CPP/mutex.h" // lock_guard
 #include "src/__support/blockstore.h"
 #include "src/__support/common.h"
 #include "src/__support/fixedvector.h"
@@ -68,7 +69,7 @@ void call_exit_callbacks() {
 }
 
 int add_atexit_unit(const AtExitUnit &unit) {
-  MutexLock lock(&handler_list_mtx);
+  cpp::lock_guard lock(handler_list_mtx);
   if (exit_callbacks.push_back(unit))
     return 0;
   return -1;
diff --git a/libc/src/threads/linux/CMakeLists.txt b/libc/src/threads/linux/CMakeLists.txt
index d372bd9e18c4..68b7106c2052 100644
--- a/libc/src/threads/linux/CMakeLists.txt
+++ b/libc/src/threads/linux/CMakeLists.txt
@@ -7,6 +7,7 @@ add_header_library(
     libc.include.sys_syscall
     libc.include.threads
     libc.src.__support.CPP.atomic
+    libc.src.__support.CPP.mutex
     libc.src.__support.OSUtil.osutil
     libc.src.__support.threads.mutex
     libc.src.__support.threads.linux.futex_utils
diff --git a/libc/src/threads/linux/CndVar.h b/libc/src/threads/linux/CndVar.h
index 525a8f0f2b53..c08ffa393856 100644
--- a/libc/src/threads/linux/CndVar.h
+++ b/libc/src/threads/linux/CndVar.h
@@ -10,6 +10,7 @@
 #define LLVM_LIBC_SRC_THREADS_LINUX_CNDVAR_H
 
 #include "src/__support/CPP/atomic.h"
+#include "src/__support/CPP/mutex.h" // lock_guard
 #include "src/__support/CPP/optional.h"
 #include "src/__support/OSUtil/syscall.h" // For syscall functions.
 #include "src/__support/threads/linux/futex_utils.h"
@@ -59,7 +60,7 @@ struct CndVar {
 
     CndWaiter waiter;
     {
-      MutexLock ml(&qmtx);
+      cpp::lock_guard ml(qmtx);
       CndWaiter *old_back = nullptr;
       if (waitq_front == nullptr) {
         waitq_front = waitq_back = &waiter;
@@ -118,7 +119,7 @@ struct CndVar {
   }
 
   int broadcast() {
-    MutexLock ml(&qmtx);
+    cpp::lock_guard ml(qmtx);
     uint32_t dummy_futex_word;
     CndWaiter *waiter = waitq_front;
     waitq_front = waitq_back = nullptr;
diff --git a/libc/test/src/__support/CPP/CMakeLists.txt b/libc/test/src/__support/CPP/CMakeLists.txt
index 708548f812c6..cec13afc8dd1 100644
--- a/libc/test/src/__support/CPP/CMakeLists.txt
+++ b/libc/test/src/__support/CPP/CMakeLists.txt
@@ -64,6 +64,16 @@ add_libc_test(
     libc.src.__support.macros.properties.types
 )
 
+add_libc_test(
+  mutex_test
+  SUITE
+    libc-cpp-utils-tests
+  SRCS
+    mutex_test.cpp
+  DEPENDS
+    libc.src.__support.CPP.mutex
+)
+
 add_libc_test(
   int_seq_test
   SUITE
diff --git a/libc/test/src/__support/CPP/mutex_test.cpp b/libc/test/src/__support/CPP/mutex_test.cpp
new file mode 100644
index 000000000000..a68c84cfc78a
--- /dev/null
+++ b/libc/test/src/__support/CPP/mutex_test.cpp
@@ -0,0 +1,79 @@
+//===-- Unittests for mutex -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/mutex.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::cpp::adopt_lock;
+using LIBC_NAMESPACE::cpp::lock_guard;
+
+// Simple struct for testing cpp::lock_guard. It defines methods 'lock' and
+// 'unlock' which are required for the cpp::lock_guard class template.
+struct Mutex {
+  // Flag to show whether this mutex is locked.
+  bool locked = false;
+
+  // Flag to show if this mutex has been double locked.
+  bool double_locked = false;
+
+  // Flag to show if this mutex has been double unlocked.
+  bool double_unlocked = false;
+
+  Mutex() {}
+
+  void lock() {
+    if (locked)
+      double_locked = true;
+
+    locked = true;
+  }
+
+  void unlock() {
+    if (!locked)
+      double_unlocked = true;
+
+    locked = false;
+  }
+};
+
+TEST(LlvmLibcMutexTest, Basic) {
+  Mutex m;
+  ASSERT_FALSE(m.locked);
+  ASSERT_FALSE(m.double_locked);
+  ASSERT_FALSE(m.double_unlocked);
+
+  {
+    lock_guard lg(m);
+    ASSERT_TRUE(m.locked);
+    ASSERT_FALSE(m.double_locked);
+  }
+
+  ASSERT_FALSE(m.locked);
+  ASSERT_FALSE(m.double_unlocked);
+}
+
+TEST(LlvmLibcMutexTest, AcquireLocked) {
+  Mutex m;
+  ASSERT_FALSE(m.locked);
+  ASSERT_FALSE(m.double_locked);
+  ASSERT_FALSE(m.double_unlocked);
+
+  // Lock the mutex before placing a lock guard on it.
+  m.lock();
+  ASSERT_TRUE(m.locked);
+  ASSERT_FALSE(m.double_locked);
+
+  {
+    lock_guard lg(m, adopt_lock);
+    ASSERT_TRUE(m.locked);
+    ASSERT_FALSE(m.double_locked);
+  }
+
+  ASSERT_FALSE(m.locked);
+  ASSERT_FALSE(m.double_unlocked);
+}
-- 
GitLab


From 443377a9d1a8d4a69a317a1a892184c59dd0aec6 Mon Sep 17 00:00:00 2001
From: "Daniel M. Katz" <katzdm@gmail.com>
Date: Thu, 9 May 2024 03:22:11 -0400
Subject: [PATCH 0261/1206] [Clang] Fix P2564 handling of variable initializers
 (#89565)

The following program produces a diagnostic in Clang and EDG, but
compiles correctly in GCC and MSVC:
```cpp
#include <vector>

consteval std::vector<int> fn() { return {1,2,3}; }
constexpr int a = fn()[1];
```

Clang's diagnostic is as follows:
```cpp
<source>:6:19: error: call to consteval function 'fn' is not a constant expression
    6 | constexpr int a = fn()[1];
      |                   ^
<source>:6:19: note: pointer to subobject of heap-allocated object is not a constant expression
/opt/compiler-explorer/gcc-snapshot/lib/gcc/x86_64-linux-gnu/14.0.1/../../../../include/c++/14.0.1/bits/allocator.h:193:31: note: heap allocation performed here
  193 |             return static_cast<_Tp*>(::operator new(__n));
      |                                      ^
1 error generated.
Compiler returned: 1
```

Based on my understanding of
[`[dcl.constexpr]/6`](https://eel.is/c++draft/dcl.constexpr#6):
> In any constexpr variable declaration, the full-expression of the
initialization shall be a constant expression

It seems to me that GCC and MSVC are correct: the initializer `fn()[1]`
does not evaluate to an lvalue referencing a heap-allocated value within
the `vector` returned by `fn()`; it evaluates to an lvalue-to-rvalue
conversion _from_ that heap-allocated value.

This PR turns out to be a bug fix on the implementation of
[P2564R3](https://wg21.link/p2564r3); as such, it only applies to C++23
and later. The core problem is that the definition of a
constant-initialized variable
([`[expr.const/2]`](https://eel.is/c++draft/expr.const#2)) is contingent
on whether the initializer can be evaluated as a constant expression:

> A variable or temporary object o is _constant-initialized_ if [...]
the full-expression of its initialization is a constant expression when
interpreted as a _constant-expression_, [...]

That can't be known until we've finished parsing the initializer, by
which time we've already added immediate invocations and consteval
references to the current expression evaluation context. This will have
the effect of evaluating said invocations as full expressions when the
context is popped, even if they're subexpressions of a larger constant
expression initializer. If, however, the variable _is_
constant-initialized, then its initializer is [manifestly
constant-evaluated](https://eel.is/c++draft/expr.const#20):

> An expression or conversion is _manifestly constant-evaluated_ if it
is [...] **the initializer of a variable that is usable in constant
expressions or has constant initialization** [...]

which in turn means that any subexpressions naming an immediate function
are in an [immediate function
context](https://eel.is/c++draft/expr.const#16):

> An expression or conversion is in an immediate function context if it
is potentially evaluated and either [...] it is a **subexpression of a
manifestly constant-evaluated expression** or conversion

and therefore _are not to be considered [immediate
invocations](https://eel.is/c++draft/expr.const#16) or
[immediate-escalating
expressions](https://eel.is/c++draft/expr.const#17) in the first place_:

> An invocation is an _immediate invocation_ if it is a
potentially-evaluated explicit or implicit invocation of an immediate
function and **is not in an immediate function context**.

> An expression or conversion is _immediate-escalating_ if **it is not
initially in an immediate function context** and [...]


The approach that I'm therefore proposing is:
1. Create a new expression evaluation context for _every_ variable
initializer (rather than only nonlocal ones).
2. Attach initializers to `VarDecl`s _prior_ to popping the expression
evaluation context / scope / etc. This sequences the determination of
whether the initializer is in an immediate function context _before_ any
contained immediate invocations are evaluated.
3. When popping an expression evaluation context, elide all evaluations
of constant invocations, and all checks for consteval references, if the
context is an immediate function context. Note that if it could be
ascertained that this was an immediate function context at parse-time,
we [would never have
registered](https://github.com/llvm/llvm-project/blob/760910ddb918d77e7632be1678f69909384d69ae/clang/lib/Sema/SemaExpr.cpp#L17799)
these immediate invocations or consteval references in the first place.

Most of the test changes previously made for this PR are now reverted
and passing as-is. The only test updates needed are now as follows:
- A few diagnostics in `consteval-cxx2a.cpp` are updated to reflect that
it is the `consteval tester::tester` constructor, not the more narrow
`make_name` function call, which fails to be evaluated as a constant
expression.
- The reclassification of `warn_impcast_integer_precision_constant` as a
compile-time diagnostic adds a (somewhat duplicative) warning when
attempting to define an enum constant using a narrowing conversion. It
also, however, retains the existing diagnostics which @erichkeane
(rightly) objected to being lost from an earlier revision of this PR.

---------

Co-authored-by: cor3ntin <corentinjabot@gmail.com>
---
 clang/docs/ReleaseNotes.rst                   |  4 ++
 clang/include/clang/Sema/Sema.h               |  4 +-
 clang/lib/Parse/ParseDecl.cpp                 | 21 ++++----
 clang/lib/Sema/SemaChecking.cpp               |  9 ++--
 clang/lib/Sema/SemaDeclCXX.cpp                | 53 ++++++++++---------
 clang/lib/Sema/SemaExpr.cpp                   |  2 +-
 clang/test/SemaCXX/cxx2a-consteval.cpp        | 16 ++++--
 .../SemaCXX/cxx2b-consteval-propagate.cpp     | 26 +++++++++
 clang/test/SemaCXX/enum-scoped.cpp            |  1 +
 9 files changed, 90 insertions(+), 46 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index a3c8e4141ca5..4547636318a7 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -699,6 +699,10 @@ Bug Fixes to C++ Support
   performed incorrectly when checking constraints. Fixes (#GH90349).
 - Clang now allows constrained member functions to be explicitly specialized for an implicit instantiation
   of a class template.
+- Fix a C++23 bug in implementation of P2564R3 which evaluates immediate invocations in place
+  within initializers for variables that are usable in constant expressions or are constant
+  initialized, rather than evaluating them as a part of the larger manifestly constant evaluated
+  expression.
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index ddb3de2b6602..4efd3878e861 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -10202,7 +10202,9 @@ public:
         S.ExprEvalContexts.back().InImmediateFunctionContext =
             FD->isImmediateFunction() ||
             S.ExprEvalContexts[S.ExprEvalContexts.size() - 2]
-                .isConstantEvaluated();
+                .isConstantEvaluated() ||
+            S.ExprEvalContexts[S.ExprEvalContexts.size() - 2]
+                .isImmediateFunctionContext();
         S.ExprEvalContexts.back().InImmediateEscalatingFunctionContext =
             S.getLangOpts().CPlusPlus20 && FD->isImmediateEscalating();
       } else
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 4e4b05b21383..2c11ae693c35 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -2587,25 +2587,30 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
     Parser &P;
     Declarator &D;
     Decl *ThisDecl;
+    bool Entered;
 
     InitializerScopeRAII(Parser &P, Declarator &D, Decl *ThisDecl)
-        : P(P), D(D), ThisDecl(ThisDecl) {
+        : P(P), D(D), ThisDecl(ThisDecl), Entered(false) {
       if (ThisDecl && P.getLangOpts().CPlusPlus) {
         Scope *S = nullptr;
         if (D.getCXXScopeSpec().isSet()) {
           P.EnterScope(0);
           S = P.getCurScope();
         }
-        P.Actions.ActOnCXXEnterDeclInitializer(S, ThisDecl);
+        if (ThisDecl && !ThisDecl->isInvalidDecl()) {
+          P.Actions.ActOnCXXEnterDeclInitializer(S, ThisDecl);
+          Entered = true;
+        }
       }
     }
-    ~InitializerScopeRAII() { pop(); }
-    void pop() {
+    ~InitializerScopeRAII() {
       if (ThisDecl && P.getLangOpts().CPlusPlus) {
         Scope *S = nullptr;
         if (D.getCXXScopeSpec().isSet())
           S = P.getCurScope();
-        P.Actions.ActOnCXXExitDeclInitializer(S, ThisDecl);
+
+        if (Entered)
+          P.Actions.ActOnCXXExitDeclInitializer(S, ThisDecl);
         if (S)
           P.ExitScope();
       }
@@ -2736,8 +2741,6 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
         FRI->RangeExpr = Init;
       }
 
-      InitScope.pop();
-
       if (Init.isInvalid()) {
         SmallVector<tok::TokenKind, 2> StopTokens;
         StopTokens.push_back(tok::comma);
@@ -2785,8 +2788,6 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
 
     bool SawError = ParseExpressionList(Exprs, ExpressionStarts);
 
-    InitScope.pop();
-
     if (SawError) {
       if (ThisVarDecl && PP.isCodeCompletionReached() && !CalledSignatureHelp) {
         Actions.ProduceConstructorSignatureHelp(
@@ -2818,8 +2819,6 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
     PreferredType.enterVariableInit(Tok.getLocation(), ThisDecl);
     ExprResult Init(ParseBraceInitializer());
 
-    InitScope.pop();
-
     if (Init.isInvalid()) {
       Actions.ActOnInitializerError(ThisDecl);
     } else
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index e8e74467208c..54789dde5069 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -16574,11 +16574,10 @@ static void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
         std::string PrettySourceValue = toString(Value, 10);
         std::string PrettyTargetValue = PrettyPrintInRange(Value, TargetRange);
 
-        S.DiagRuntimeBehavior(
-            E->getExprLoc(), E,
-            S.PDiag(diag::warn_impcast_integer_precision_constant)
-                << PrettySourceValue << PrettyTargetValue << E->getType() << T
-                << E->getSourceRange() << SourceRange(CC));
+        S.Diag(E->getExprLoc(),
+               S.PDiag(diag::warn_impcast_integer_precision_constant)
+                   << PrettySourceValue << PrettyTargetValue << E->getType()
+                   << T << E->getSourceRange() << SourceRange(CC));
         return;
       }
     }
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 157d42c09cfc..d77b9507066b 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -18553,15 +18553,6 @@ void Sema::ActOnPureSpecifier(Decl *D, SourceLocation ZeroLoc) {
     Diag(D->getLocation(), diag::err_illegal_initializer);
 }
 
-/// Determine whether the given declaration is a global variable or
-/// static data member.
-static bool isNonlocalVariable(const Decl *D) {
-  if (const VarDecl *Var = dyn_cast_or_null<VarDecl>(D))
-    return Var->hasGlobalStorage();
-
-  return false;
-}
-
 /// Invoked when we are about to parse an initializer for the declaration
 /// 'Dcl'.
 ///
@@ -18570,9 +18561,7 @@ static bool isNonlocalVariable(const Decl *D) {
 /// class X. If the declaration had a scope specifier, a scope will have
 /// been created and passed in for this purpose. Otherwise, S will be null.
 void Sema::ActOnCXXEnterDeclInitializer(Scope *S, Decl *D) {
-  // If there is no declaration, there was an error parsing it.
-  if (!D || D->isInvalidDecl())
-    return;
+  assert(D && !D->isInvalidDecl());
 
   // We will always have a nested name specifier here, but this declaration
   // might not be out of line if the specifier names the current namespace:
@@ -18581,25 +18570,41 @@ void Sema::ActOnCXXEnterDeclInitializer(Scope *S, Decl *D) {
   if (S && D->isOutOfLine())
     EnterDeclaratorContext(S, D->getDeclContext());
 
-  // If we are parsing the initializer for a static data member, push a
-  // new expression evaluation context that is associated with this static
-  // data member.
-  if (isNonlocalVariable(D))
-    PushExpressionEvaluationContext(
-        ExpressionEvaluationContext::PotentiallyEvaluated, D);
+  PushExpressionEvaluationContext(
+      ExpressionEvaluationContext::PotentiallyEvaluated, D);
 }
 
 /// Invoked after we are finished parsing an initializer for the declaration D.
 void Sema::ActOnCXXExitDeclInitializer(Scope *S, Decl *D) {
-  // If there is no declaration, there was an error parsing it.
-  if (!D || D->isInvalidDecl())
-    return;
-
-  if (isNonlocalVariable(D))
-    PopExpressionEvaluationContext();
+  assert(D);
 
   if (S && D->isOutOfLine())
     ExitDeclaratorContext(S);
+
+  if (getLangOpts().CPlusPlus23) {
+    // An expression or conversion is 'manifestly constant-evaluated' if it is:
+    // [...]
+    // - the initializer of a variable that is usable in constant expressions or
+    //   has constant initialization.
+    if (auto *VD = dyn_cast<VarDecl>(D);
+        VD && (VD->isUsableInConstantExpressions(Context) ||
+               VD->hasConstantInitialization())) {
+      // An expression or conversion is in an 'immediate function context' if it
+      // is potentially evaluated and either:
+      // [...]
+      // - it is a subexpression of a manifestly constant-evaluated expression
+      //   or conversion.
+      ExprEvalContexts.back().InImmediateFunctionContext = true;
+    }
+  }
+
+  // Unless the initializer is in an immediate function context (as determined
+  // above), this will evaluate all contained immediate function calls as
+  // constant expressions. If the initializer IS an immediate function context,
+  // the initializer has been determined to be a constant expression, and all
+  // such evaluations will be elided (i.e., as if we "knew the whole time" that
+  // it was a constant expression).
+  PopExpressionEvaluationContext();
 }
 
 /// ActOnCXXConditionDeclarationExpr - Parsed a condition declaration of a
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 94f99a423f0e..c688cb21f236 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -18039,7 +18039,7 @@ HandleImmediateInvocations(Sema &SemaRef,
                            Sema::ExpressionEvaluationContextRecord &Rec) {
   if ((Rec.ImmediateInvocationCandidates.size() == 0 &&
        Rec.ReferenceToConsteval.size() == 0) ||
-      SemaRef.RebuildingImmediateInvocation)
+      Rec.isImmediateFunctionContext() || SemaRef.RebuildingImmediateInvocation)
     return;
 
   /// When we have more than 1 ImmediateInvocationCandidates or previously
diff --git a/clang/test/SemaCXX/cxx2a-consteval.cpp b/clang/test/SemaCXX/cxx2a-consteval.cpp
index e19807437207..622ec31c459d 100644
--- a/clang/test/SemaCXX/cxx2a-consteval.cpp
+++ b/clang/test/SemaCXX/cxx2a-consteval.cpp
@@ -1068,6 +1068,14 @@ void test() {
   constexpr int (*f2)(void) = lstatic; // expected-error {{constexpr variable 'f2' must be initialized by a constant expression}} \
                                        // expected-note  {{pointer to a consteval declaration is not a constant expression}}
 
+  int (*f3)(void) = []() consteval { return 3; };  // expected-error {{cannot take address of consteval call operator of '(lambda at}} \
+                                                   // expected-note {{declared here}}
+}
+
+consteval void consteval_test() {
+  constexpr auto l1 = []() consteval { return 3; };
+
+  int (*f1)(void) = l1;  // ok
 }
 }
 
@@ -1098,11 +1106,11 @@ int bad = 10; // expected-note 6{{declared here}}
 tester glob1(make_name("glob1"));
 tester glob2(make_name("glob2"));
 constexpr tester cglob(make_name("cglob"));
-tester paddedglob(make_name(pad(bad))); // expected-error {{call to consteval function 'GH58207::make_name' is not a constant expression}} \
+tester paddedglob(make_name(pad(bad))); // expected-error {{call to consteval function 'GH58207::tester::tester' is not a constant expression}} \
                                         // expected-note {{read of non-const variable 'bad' is not allowed in a constant expression}}
 
 constexpr tester glob3 = { make_name("glob3") };
-constexpr tester glob4 = { make_name(pad(bad)) }; // expected-error {{call to consteval function 'GH58207::make_name' is not a constant expression}} \
+constexpr tester glob4 = { make_name(pad(bad)) }; // expected-error {{call to consteval function 'GH58207::tester::tester' is not a constant expression}} \
                                                   // expected-error {{constexpr variable 'glob4' must be initialized by a constant expression}} \
                                                   // expected-note 2{{read of non-const variable 'bad' is not allowed in a constant expression}}
 
@@ -1114,12 +1122,12 @@ auto V1 = make_name(pad(bad)); // expected-error {{call to consteval function 'G
 void foo() {
   static tester loc1(make_name("loc1"));
   static constexpr tester loc2(make_name("loc2"));
-  static tester paddedloc(make_name(pad(bad))); // expected-error {{call to consteval function 'GH58207::make_name' is not a constant expression}} \
+  static tester paddedloc(make_name(pad(bad))); // expected-error {{call to consteval function 'GH58207::tester::tester' is not a constant expression}} \
                                                 // expected-note {{read of non-const variable 'bad' is not allowed in a constant expression}}
 }
 
 void bar() {
-  static tester paddedloc(make_name(pad(bad))); // expected-error {{call to consteval function 'GH58207::make_name' is not a constant expression}} \
+  static tester paddedloc(make_name(pad(bad))); // expected-error {{call to consteval function 'GH58207::tester::tester' is not a constant expression}} \
                                                 // expected-note {{read of non-const variable 'bad' is not allowed in a constant expression}}
 }
 }
diff --git a/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp b/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
index 4a75392045d0..37fa1f1bdf59 100644
--- a/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
+++ b/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
@@ -394,3 +394,29 @@ static_assert(none_of(
 ));
 
 }
+
+#if __cplusplus >= 202302L
+namespace lvalue_to_rvalue_init_from_heap {
+
+struct S {
+    int *value;
+    constexpr S(int v) : value(new int {v}) {}  // expected-note 2 {{heap allocation performed here}}
+    constexpr ~S() { delete value; }
+};
+consteval S fn() { return S(5); }
+int fn2() { return 2; }  // expected-note {{declared here}}
+
+constexpr int a = *fn().value;
+constinit int b = *fn().value;
+const int c = *fn().value;
+int d = *fn().value;
+
+constexpr int e = *fn().value + fn2(); // expected-error {{must be initialized by a constant expression}} \
+                                       // expected-error {{call to consteval function 'lvalue_to_rvalue_init_from_heap::fn' is not a constant expression}} \
+                                       // expected-note {{non-constexpr function 'fn2'}} \
+                                       // expected-note {{pointer to heap-allocated object}}
+
+int f = *fn().value + fn2();  // expected-error {{call to consteval function 'lvalue_to_rvalue_init_from_heap::fn' is not a constant expression}} \
+                              // expected-note {{pointer to heap-allocated object}}
+}
+#endif
diff --git a/clang/test/SemaCXX/enum-scoped.cpp b/clang/test/SemaCXX/enum-scoped.cpp
index b1d9a215c437..d7b7923430af 100644
--- a/clang/test/SemaCXX/enum-scoped.cpp
+++ b/clang/test/SemaCXX/enum-scoped.cpp
@@ -53,6 +53,7 @@ enum class E4 {
   e1 = -2147483648, // ok
   e2 = 2147483647, // ok
   e3 = 2147483648 // expected-error{{enumerator value evaluates to 2147483648, which cannot be narrowed to type 'int'}}
+                  // expected-warning@-1{{changes value}}
 };
 
 enum class E5 {
-- 
GitLab


From febd89cafea11e6603f593e41be1a21ca9d009ac Mon Sep 17 00:00:00 2001
From: Vadim D <36827317+vvd170501@users.noreply.github.com>
Date: Thu, 9 May 2024 11:25:26 +0300
Subject: [PATCH 0262/1206] [clang-tidy] check `std::string_view` and custom
 string-like classes in `readability-string-compare` (#88636)

This PR aims to expand the list of classes that are considered to be
"strings" by `readability-string-compare` check.

1. Currently only `std::string;:compare` is checked, but
`std::string_view` has a similar `compare` method. This PR enables
checking of `std::string_view::compare` by default.
2. Some codebases use custom string-like classes that have public
interfaces similar to `std::string` or `std::string_view`. Example:
[TStringBase](https://github.com/yandex/yatool/blob/main/util/generic/strbase.h#L38),
A new option, `readability-string-compare.StringClassNames`, is added to
allow specifying a custom list of string-like classes.

Related to, but does not solve #28396 (only adds support for custom
string-like classes, not custom functions)
---
 .../readability/StringCompareCheck.cpp        | 27 ++++++++++++--
 .../readability/StringCompareCheck.h          | 10 ++++-
 clang-tools-extra/docs/ReleaseNotes.rst       |  5 +++
 .../checks/readability/string-compare.rst     | 37 ++++++++++++++++++-
 .../clang-tidy/checkers/Inputs/Headers/string | 10 +++++
 .../string-compare-custom-string-classes.cpp  | 35 ++++++++++++++++++
 .../checkers/readability/string-compare.cpp   | 23 ++++++++++++
 7 files changed, 139 insertions(+), 8 deletions(-)
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability/string-compare-custom-string-classes.cpp

diff --git a/clang-tools-extra/clang-tidy/readability/StringCompareCheck.cpp b/clang-tools-extra/clang-tidy/readability/StringCompareCheck.cpp
index 3b5d89c8c647..7c0bbef3ca08 100644
--- a/clang-tools-extra/clang-tidy/readability/StringCompareCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/StringCompareCheck.cpp
@@ -7,12 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "StringCompareCheck.h"
-#include "../utils/FixItHintUtils.h"
+#include "../utils/OptionsUtils.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Tooling/FixIt.h"
+#include "llvm/ADT/StringRef.h"
 
 using namespace clang::ast_matchers;
+namespace optutils = clang::tidy::utils::options;
 
 namespace clang::tidy::readability {
 
@@ -20,11 +23,27 @@ static const StringRef CompareMessage = "do not use 'compare' to test equality "
                                         "of strings; use the string equality "
                                         "operator instead";
 
+static const StringRef DefaultStringLikeClasses = "::std::basic_string;"
+                                                  "::std::basic_string_view";
+
+StringCompareCheck::StringCompareCheck(StringRef Name,
+                                       ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      StringLikeClasses(optutils::parseStringList(
+          Options.get("StringLikeClasses", DefaultStringLikeClasses))) {}
+
+void StringCompareCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "StringLikeClasses",
+                optutils::serializeStringList(StringLikeClasses));
+}
+
 void StringCompareCheck::registerMatchers(MatchFinder *Finder) {
+  if (StringLikeClasses.empty()) {
+    return;
+  }
   const auto StrCompare = cxxMemberCallExpr(
-      callee(cxxMethodDecl(hasName("compare"),
-                           ofClass(classTemplateSpecializationDecl(
-                               hasName("::std::basic_string"))))),
+      callee(cxxMethodDecl(hasName("compare"), ofClass(cxxRecordDecl(hasAnyName(
+                                                   StringLikeClasses))))),
       hasArgument(0, expr().bind("str2")), argumentCountIs(1),
       callee(memberExpr().bind("str1")));
 
diff --git a/clang-tools-extra/clang-tidy/readability/StringCompareCheck.h b/clang-tools-extra/clang-tidy/readability/StringCompareCheck.h
index 812736d806b7..150090901a6e 100644
--- a/clang-tools-extra/clang-tidy/readability/StringCompareCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/StringCompareCheck.h
@@ -10,6 +10,7 @@
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_STRINGCOMPARECHECK_H
 
 #include "../ClangTidyCheck.h"
+#include <vector>
 
 namespace clang::tidy::readability {
 
@@ -20,13 +21,18 @@ namespace clang::tidy::readability {
 /// http://clang.llvm.org/extra/clang-tidy/checks/readability/string-compare.html
 class StringCompareCheck : public ClangTidyCheck {
 public:
-  StringCompareCheck(StringRef Name, ClangTidyContext *Context)
-      : ClangTidyCheck(Name, Context) {}
+  StringCompareCheck(StringRef Name, ClangTidyContext *Context);
+
   bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
     return LangOpts.CPlusPlus;
   }
+
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+
+private:
+  const std::vector<StringRef> StringLikeClasses;
 };
 
 } // namespace clang::tidy::readability
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 5b7ea42c63d6..3f0d25ec8c75 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -362,6 +362,11 @@ Changes in existing checks
   check by resolving fix-it overlaps in template code by disregarding implicit
   instances.
 
+- Improved :doc:`readability-string-compare
+  <clang-tidy/checks/readability/string-compare>` check to also detect
+  usages of ``std::string_view::compare``. Added a `StringLikeClasses` option
+  to detect usages of ``compare`` method in custom string-like classes.
+
 Removed checks
 ^^^^^^^^^^^^^^
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/string-compare.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/string-compare.rst
index 268632eee61a..4be2473bed2d 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability/string-compare.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability/string-compare.rst
@@ -14,10 +14,12 @@ recommended to avoid the risk of incorrect interpretation of the return value
 and to simplify the code. The string equality and inequality operators can
 also be faster than the ``compare`` method due to early termination.
 
-Examples:
+Example
+-------
 
 .. code-block:: c++
 
+  // The same rules apply to std::string_view.
   std::string str1{"a"};
   std::string str2{"b"};
 
@@ -50,5 +52,36 @@ Examples:
   }
 
 The above code examples show the list of if-statements that this check will
-give a warning for. All of them uses ``compare`` to check if equality or
+give a warning for. All of them use ``compare`` to check equality or
 inequality of two strings instead of using the correct operators.
+
+Options
+-------
+
+.. option:: StringLikeClasses
+
+   A string containing semicolon-separated names of string-like classes.
+   By default contains only ``::std::basic_string``
+   and ``::std::basic_string_view``. If a class from this list has
+   a ``compare`` method similar to that of ``std::string``, it will be checked
+   in the same way.
+
+Example
+^^^^^^^
+
+.. code-block:: c++
+
+  struct CustomString {
+  public:
+    int compare (const CustomString& other) const;
+  }
+
+  CustomString str1;
+  CustomString str2;
+
+  // use str1 != str2 instead.
+  if (str1.compare(str2)) {
+  }
+
+If `StringLikeClasses` contains ``CustomString``, the check will suggest
+replacing ``compare`` with equality operator.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string
index d031f27beb9d..0c160bc182b6 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/string
@@ -108,6 +108,8 @@ struct basic_string_view {
   constexpr bool starts_with(C ch) const noexcept;
   constexpr bool starts_with(const C* s) const;
 
+  constexpr int compare(basic_string_view sv) const noexcept;
+
   static constexpr size_t npos = -1;
 };
 
@@ -132,6 +134,14 @@ bool operator==(const std::wstring&, const std::wstring&);
 bool operator==(const std::wstring&, const wchar_t*);
 bool operator==(const wchar_t*, const std::wstring&);
 
+bool operator==(const std::string_view&, const std::string_view&);
+bool operator==(const std::string_view&, const char*);
+bool operator==(const char*, const std::string_view&);
+
+bool operator!=(const std::string_view&, const std::string_view&);
+bool operator!=(const std::string_view&, const char*);
+bool operator!=(const char*, const std::string_view&);
+
 size_t strlen(const char* str);
 }
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/string-compare-custom-string-classes.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/string-compare-custom-string-classes.cpp
new file mode 100644
index 000000000000..faf135833ee1
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/string-compare-custom-string-classes.cpp
@@ -0,0 +1,35 @@
+// RUN: %check_clang_tidy %s readability-string-compare %t -- -config='{CheckOptions: {readability-string-compare.StringLikeClasses: "CustomStringTemplateBase;CustomStringNonTemplateBase"}}' -- -isystem %clang_tidy_headers
+#include <string>
+
+struct CustomStringNonTemplateBase {
+  int compare(const CustomStringNonTemplateBase& Other) const {
+    return 123;  // value is not important for check
+  }
+};
+
+template <typename T>
+struct CustomStringTemplateBase {
+  int compare(const CustomStringTemplateBase& Other) const {
+    return 123;
+  }
+};
+
+struct CustomString1 : CustomStringNonTemplateBase {};
+struct CustomString2 : CustomStringTemplateBase<char> {};
+
+void CustomStringClasses() {
+  std::string_view sv1("a");
+  std::string_view sv2("b");
+  if (sv1.compare(sv2)) {  // No warning - if a std class is not listed in StringLikeClasses, it won't be checked.
+  }
+
+  CustomString1 custom1;
+  if (custom1.compare(custom1)) {
+  }
+  // CHECK-MESSAGES: [[@LINE-2]]:7: warning: do not use 'compare' to test equality of strings; use the string equality operator instead [readability-string-compare]
+
+  CustomString2 custom2;
+  if (custom2.compare(custom2)) {
+  }
+  // CHECK-MESSAGES: [[@LINE-2]]:7: warning: do not use 'compare' to test equality of strings; use the string equality operator instead [readability-string-compare]
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/string-compare.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/string-compare.cpp
index 2c08b86cf72f..c4fea4341617 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/string-compare.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/string-compare.cpp
@@ -67,11 +67,27 @@ void Test() {
   if (str1.compare(comp())) {
   }
   // CHECK-MESSAGES: [[@LINE-2]]:7: warning: do not use 'compare' to test equality of strings;
+
+  std::string_view sv1("a");
+  std::string_view sv2("b");
+  if (sv1.compare(sv2)) {
+  }
+  // CHECK-MESSAGES: [[@LINE-2]]:7: warning: do not use 'compare' to test equality of strings; use the string equality operator instead [readability-string-compare]
+}
+
+struct DerivedFromStdString : std::string {};
+
+void TestDerivedClass() {
+  DerivedFromStdString derived;
+  if (derived.compare(derived)) {
+  }
+  // CHECK-MESSAGES: [[@LINE-2]]:7: warning: do not use 'compare' to test equality of strings; use the string equality operator instead [readability-string-compare]
 }
 
 void Valid() {
   std::string str1("a", 1);
   std::string str2("b", 1);
+
   if (str1 == str2) {
   }
   if (str1 != str2) {
@@ -96,4 +112,11 @@ void Valid() {
   }
   if (str1.compare(str2) == -1) {
   }
+
+  std::string_view sv1("a");
+  std::string_view sv2("b");
+  if (sv1 == sv2) {
+  }
+  if (sv1.compare(sv2) > 0) {
+  }
 }
-- 
GitLab


From b52fa9461ab73eaf2d04f32c806a1715b2595830 Mon Sep 17 00:00:00 2001
From: David Sherwood <57997763+david-arm@users.noreply.github.com>
Date: Thu, 9 May 2024 09:40:33 +0100
Subject: [PATCH 0263/1206] [Analysis] Add cost model for
 experimental.cttz.elts intrinsic (#90720)

In PR #88385 I've added support for auto-vectorisation of some early
exit loops, which requires using the experimental.cttz.elts to calculate
final indices in the early exit block. We need a more accurate cost
model for this intrinsic to better reflect the cost of work required in
the early exit block. I've tried to accurately represent the expansion
code for the intrinsic when the target does not have efficient lowering
for it. It's quite tricky to model because you need to first figure out
what types will actually be used in the expansion. The type used can
have a significant effect on the cost if you end up using illegal vector
types.

Tests added here:

  Analysis/CostModel/AArch64/cttz_elts.ll
  Analysis/CostModel/RISCV/cttz_elts.ll
---
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      |  48 ++++
 llvm/include/llvm/CodeGen/TargetLowering.h    |   6 +
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  19 +-
 llvm/lib/CodeGen/TargetLoweringBase.cpp       |  18 ++
 .../Analysis/CostModel/AArch64/cttz_elts.ll   | 209 ++++++++++++++++++
 .../Analysis/CostModel/RISCV/cttz_elts.ll     | 149 +++++++++++++
 6 files changed, 437 insertions(+), 12 deletions(-)
 create mode 100644 llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll
 create mode 100644 llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll

diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index c6e90e57e46e..bcb60c656296 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -25,6 +25,7 @@
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/TargetTransformInfoImpl.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
@@ -1758,6 +1759,53 @@ public:
                                           CmpInst::ICMP_ULT, CostKind);
       return Cost;
     }
+    case Intrinsic::experimental_cttz_elts: {
+      EVT ArgType = getTLI()->getValueType(DL, ICA.getArgTypes()[0], true);
+
+      // If we're not expanding the intrinsic then we assume this is cheap
+      // to implement.
+      if (!getTLI()->shouldExpandCttzElements(ArgType))
+        return getTypeLegalizationCost(RetTy).first;
+
+      // TODO: The costs below reflect the expansion code in
+      // SelectionDAGBuilder, but we may want to sacrifice some accuracy in
+      // favour of compile time.
+
+      // Find the smallest "sensible" element type to use for the expansion.
+      bool ZeroIsPoison = !cast<ConstantInt>(Args[1])->isZero();
+      ConstantRange VScaleRange(APInt(64, 1), APInt::getZero(64));
+      if (isa<ScalableVectorType>(ICA.getArgTypes()[0]) && I && I->getCaller())
+        VScaleRange = getVScaleRange(I->getCaller(), 64);
+
+      unsigned EltWidth = getTLI()->getBitWidthForCttzElements(
+          RetTy, ArgType.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
+      Type *NewEltTy = IntegerType::getIntNTy(RetTy->getContext(), EltWidth);
+
+      // Create the new vector type & get the vector length
+      Type *NewVecTy = VectorType::get(
+          NewEltTy, cast<VectorType>(Args[0]->getType())->getElementCount());
+
+      IntrinsicCostAttributes StepVecAttrs(Intrinsic::experimental_stepvector,
+                                           NewVecTy, {}, FMF);
+      InstructionCost Cost =
+          thisT()->getIntrinsicInstrCost(StepVecAttrs, CostKind);
+
+      Cost +=
+          thisT()->getArithmeticInstrCost(Instruction::Sub, NewVecTy, CostKind);
+      Cost += thisT()->getCastInstrCost(Instruction::SExt, NewVecTy,
+                                        Args[0]->getType(),
+                                        TTI::CastContextHint::None, CostKind);
+      Cost +=
+          thisT()->getArithmeticInstrCost(Instruction::And, NewVecTy, CostKind);
+
+      IntrinsicCostAttributes ReducAttrs(Intrinsic::vector_reduce_umax,
+                                         NewEltTy, NewVecTy, FMF, I, 1);
+      Cost += thisT()->getTypeBasedIntrinsicInstrCost(ReducAttrs, CostKind);
+      Cost +=
+          thisT()->getArithmeticInstrCost(Instruction::Sub, NewEltTy, CostKind);
+
+      return Cost;
+    }
     }
 
     // VP Intrinsics should have the same cost as their non-vp counterpart.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 7ed08cfa8a20..50a8c7eb75af 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -470,6 +470,12 @@ public:
   /// expanded using generic code in SelectionDAGBuilder.
   virtual bool shouldExpandCttzElements(EVT VT) const { return true; }
 
+  /// Return the minimum number of bits required to hold the maximum possible
+  /// number of trailing zero vector elements.
+  unsigned getBitWidthForCttzElements(Type *RetTy, ElementCount EC,
+                                      bool ZeroIsPoison,
+                                      const ConstantRange *VScaleRange) const;
+
   // Return true if op(vecreduce(x), vecreduce(y)) should be reassociated to
   // vecreduce(op(x, y)) for the reduction opcode RedOpc.
   virtual bool shouldReassociateReduction(unsigned RedOpc, EVT VT) const {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 55aed43070df..9ef02a792fd0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7861,20 +7861,15 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
       Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE);
     }
 
-    // Find the smallest "sensible" element type to use for the expansion.
-    ConstantRange CR(
-        APInt(64, OpVT.getVectorElementCount().getKnownMinValue()));
-    if (OpVT.isScalableVT())
-      CR = CR.umul_sat(getVScaleRange(I.getCaller(), 64));
-
     // If the zero-is-poison flag is set, we can assume the upper limit
     // of the result is VF-1.
-    if (!cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero())
-      CR = CR.subtract(APInt(64, 1));
-
-    unsigned EltWidth = I.getType()->getScalarSizeInBits();
-    EltWidth = std::min(EltWidth, (unsigned)CR.getActiveBits());
-    EltWidth = std::max(llvm::bit_ceil(EltWidth), (unsigned)8);
+    bool ZeroIsPoison =
+        !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();
+    ConstantRange VScaleRange(1, true); // Dummy value.
+    if (isa<ScalableVectorType>(I.getOperand(0)->getType()))
+      VScaleRange = getVScaleRange(I.getCaller(), 64);
+    unsigned EltWidth = TLI.getBitWidthForCttzElements(
+        I.getType(), OpVT.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
 
     MVT NewEltTy = MVT::getIntegerVT(EltWidth);
 
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 75b3f14e9622..09b70cfb7227 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1048,6 +1048,24 @@ bool TargetLoweringBase::isFreeAddrSpaceCast(unsigned SrcAS,
   return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
 }
 
+unsigned TargetLoweringBase::getBitWidthForCttzElements(
+    Type *RetTy, ElementCount EC, bool ZeroIsPoison,
+    const ConstantRange *VScaleRange) const {
+  // Find the smallest "sensible" element type to use for the expansion.
+  ConstantRange CR(APInt(64, EC.getKnownMinValue()));
+  if (EC.isScalable())
+    CR = CR.umul_sat(*VScaleRange);
+
+  if (ZeroIsPoison)
+    CR = CR.subtract(APInt(64, 1));
+
+  unsigned EltWidth = RetTy->getScalarSizeInBits();
+  EltWidth = std::min(EltWidth, (unsigned)CR.getActiveBits());
+  EltWidth = std::max(llvm::bit_ceil(EltWidth), (unsigned)8);
+
+  return EltWidth;
+}
+
 void TargetLoweringBase::setJumpIsExpensive(bool isExpensive) {
   // If the command-line option was specified, ignore this request.
   if (!JumpIsExpensiveOverride.getNumOccurrences())
diff --git a/llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll b/llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll
new file mode 100644
index 000000000000..01dc086d9385
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll
@@ -0,0 +1,209 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck %s
+
+define void @foo_no_vscale_range() {
+; CHECK-LABEL: 'foo_no_vscale_range'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.v2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.v4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.v8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.v16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res.i64.v32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v32i1(<32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.v2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.v4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.v8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.v16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res.i32.v32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v32i1(<32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.v2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.v4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.v8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.v16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res.i64.v32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v32i1(<32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.v2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.v4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.v8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.v16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res.i32.v32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v32i1(<32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+  %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+  %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+  %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv161(<vscale x 16 x i1> undef, i1 true)
+  %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+  %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+  %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+  %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+  %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv161(<vscale x 16 x i1> undef, i1 true)
+  %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+
+  %res.i64.v2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> undef, i1 true)
+  %res.i64.v4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> undef, i1 true)
+  %res.i64.v8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> undef, i1 true)
+  %res.i64.v16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v161(<16 x i1> undef, i1 true)
+  %res.i64.v32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v32i1(<32 x i1> undef, i1 true)
+  %res.i32.v2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> undef, i1 true)
+  %res.i32.v4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> undef, i1 true)
+  %res.i32.v8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> undef, i1 true)
+  %res.i32.v16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v161(<16 x i1> undef, i1 true)
+  %res.i32.v32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v32i1(<32 x i1> undef, i1 true)
+
+  %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+  %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+  %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+  %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv161(<vscale x 16 x i1> undef, i1 false)
+  %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+  %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+  %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+  %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+  %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv161(<vscale x 16 x i1> undef, i1 false)
+  %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+
+  %res.i64.v2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> undef, i1 false)
+  %res.i64.v4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> undef, i1 false)
+  %res.i64.v8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> undef, i1 false)
+  %res.i64.v16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v161(<16 x i1> undef, i1 false)
+  %res.i64.v32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v32i1(<32 x i1> undef, i1 false)
+  %res.i32.v2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> undef, i1 false)
+  %res.i32.v4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> undef, i1 false)
+  %res.i32.v8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> undef, i1 false)
+  %res.i32.v16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v161(<16 x i1> undef, i1 false)
+  %res.i32.v32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.v32i1(<32 x i1> undef, i1 false)
+
+  ret void
+}
+
+
+define void @foo_vscale_range_1_16() vscale_range(1,16) {
+; CHECK-LABEL: 'foo_vscale_range_1_16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+  %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+  %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+  %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv161(<vscale x 16 x i1> undef, i1 true)
+  %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+  %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+  %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+  %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+  %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv161(<vscale x 16 x i1> undef, i1 true)
+  %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+
+  %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+  %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+  %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+  %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv161(<vscale x 16 x i1> undef, i1 false)
+  %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+  %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+  %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+  %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+  %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv161(<vscale x 16 x i1> undef, i1 false)
+  %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+
+  ret void
+}
+
+define void @foo_vscale_range_1_16384() vscale_range(1,16384) {
+; CHECK-LABEL: 'foo_vscale_range_1_16384'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+  %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+  %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+  %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv161(<vscale x 16 x i1> undef, i1 true)
+  %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+  %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+  %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+  %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+  %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv161(<vscale x 16 x i1> undef, i1 true)
+  %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+
+  %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+  %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+  %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+  %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv161(<vscale x 16 x i1> undef, i1 false)
+  %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+  %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+  %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+  %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+  %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv161(<vscale x 16 x i1> undef, i1 false)
+  %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+
+  ret void
+}
+
+declare i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.v32i1(<32 x i1>, i1)
+
+declare i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1>, i1)
+declare i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1>, i1)
+declare i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1>, i1)
+declare i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1>, i1)
+declare i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1>, i1)
diff --git a/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll b/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
new file mode 100644
index 000000000000..ca09d027b547
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
@@ -0,0 +1,149 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv64 -mattr=+v | FileCheck %s
+
+define void @foo_no_vscale_range() {
+; CHECK-LABEL: 'foo_no_vscale_range'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv64i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 781 for instruction: %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 390 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 781 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 390 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+  %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+  %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+  %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv161(<vscale x 16 x i1> undef, i1 true)
+  %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+  %res.i64.nxv64i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 true)
+  %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+  %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+  %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+  %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+  %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv161(<vscale x 16 x i1> undef, i1 true)
+  %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+  %res.i32.nxv64i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 true)
+  %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+
+  %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+  %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+  %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+  %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv161(<vscale x 16 x i1> undef, i1 false)
+  %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+  %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+  %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+  %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+  %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+  %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+  %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv161(<vscale x 16 x i1> undef, i1 false)
+  %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+  %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+  %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+
+  ret void
+}
+
+
+define void @foo_vscale_range_2_16() vscale_range(2,16) {
+; CHECK-LABEL: 'foo_vscale_range_2_16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv64i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+  %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+  %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+  %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv161(<vscale x 16 x i1> undef, i1 true)
+  %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+  %res.i64.nxv64i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 true)
+  %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+  %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+  %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+  %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+  %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv161(<vscale x 16 x i1> undef, i1 true)
+  %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
+  %res.i32.nxv64i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 true)
+  %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+
+  %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+  %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+  %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+  %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv161(<vscale x 16 x i1> undef, i1 false)
+  %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+  %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+  %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+  %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+  %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+  %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+  %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv161(<vscale x 16 x i1> undef, i1 false)
+  %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+  %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+  %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+
+  ret void
+}
+
+declare i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1>, i1)
+declare i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1>, i1)
+
+declare i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1>, i1)
+declare i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1>, i1)
+declare i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1>, i1)
+declare i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1>, i1)
+declare i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1>, i1)
+declare i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1>, i1)
+declare i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1>, i1)
-- 
GitLab


From 105dd60fc86a20404bd97ea7132e2c746ade300a Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Thu, 9 May 2024 10:45:19 +0100
Subject: [PATCH 0264/1206] [Clang][AArch64] Fixed incorrect _BitInt alignment
 (#90602)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This patch makes determining alignment and width of BitInt to be target
ABI specific and makes it consistent with [Procedure Call Standard for
the Arm® 64-bit Architecture
(AArch64)](https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst)
for AArch64 targets.
---
 clang/include/clang/Basic/TargetInfo.h | 20 ++++++++
 clang/lib/AST/ASTContext.cpp           |  5 +-
 clang/lib/Basic/Targets/AArch64.cpp    |  1 +
 clang/test/CodeGen/aapcs64-align.cpp   | 64 +++++++++++++++++++++++++-
 4 files changed, 86 insertions(+), 4 deletions(-)

diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index 3ced2e7397a7..8a6511b9ced8 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -97,6 +97,10 @@ struct TransferrableTargetInfo {
   unsigned char LongLongWidth, LongLongAlign;
   unsigned char Int128Align;
 
+  // This is an optional parameter for targets that
+  // don't use 'LongLongAlign' for '_BitInt' max alignment
+  std::optional<unsigned> BitIntMaxAlign;
+
   // Fixed point bit widths
   unsigned char ShortAccumWidth, ShortAccumAlign;
   unsigned char AccumWidth, AccumAlign;
@@ -518,6 +522,22 @@ public:
   /// getInt128Align() - Returns the alignment of Int128.
   unsigned getInt128Align() const { return Int128Align; }
 
+  /// getBitIntMaxAlign() - Returns the maximum possible alignment of
+  /// '_BitInt' and 'unsigned _BitInt'.
+  unsigned getBitIntMaxAlign() const {
+    return BitIntMaxAlign.value_or(LongLongAlign);
+  }
+
+  /// getBitIntAlign/Width - Return aligned size of '_BitInt' and
+  /// 'unsigned _BitInt' for this target, in bits.
+  unsigned getBitIntWidth(unsigned NumBits) const {
+    return llvm::alignTo(NumBits, getBitIntAlign(NumBits));
+  }
+  unsigned getBitIntAlign(unsigned NumBits) const {
+    return std::clamp<unsigned>(llvm::PowerOf2Ceil(NumBits), getCharWidth(),
+                                getBitIntMaxAlign());
+  }
+
   /// getShortAccumWidth/Align - Return the size of 'signed short _Accum' and
   /// 'unsigned short _Accum' for this target, in bits.
   unsigned getShortAccumWidth() const { return ShortAccumWidth; }
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 91e7a5f67a93..4475f399a120 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2258,9 +2258,8 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
   }
   case Type::BitInt: {
     const auto *EIT = cast<BitIntType>(T);
-    Align = std::clamp<unsigned>(llvm::PowerOf2Ceil(EIT->getNumBits()),
-                                 getCharWidth(), Target->getLongLongAlign());
-    Width = llvm::alignTo(EIT->getNumBits(), Align);
+    Align = Target->getBitIntAlign(EIT->getNumBits());
+    Width = Target->getBitIntWidth(EIT->getNumBits());
     break;
   }
   case Type::Record:
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 1a02520d7bd1..4b1545339f69 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -154,6 +154,7 @@ AArch64TargetInfo::AArch64TargetInfo(const llvm::Triple &Triple,
   else
     LongWidth = LongAlign = PointerWidth = PointerAlign = 32;
 
+  BitIntMaxAlign = 128;
   MaxVectorAlign = 128;
   MaxAtomicInlineWidth = 128;
   MaxAtomicPromoteWidth = 128;
diff --git a/clang/test/CodeGen/aapcs64-align.cpp b/clang/test/CodeGen/aapcs64-align.cpp
index de231f2123b9..7a8151022852 100644
--- a/clang/test/CodeGen/aapcs64-align.cpp
+++ b/clang/test/CodeGen/aapcs64-align.cpp
@@ -1,7 +1,7 @@
 // REQUIRES: arm-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-elf \
 // RUN:   -O2 \
-// RUN:   -emit-llvm -o - %s | FileCheck %s
+// RUN:   -emit-llvm -fexperimental-max-bitint-width=1024 -o - %s | FileCheck %s
 
 extern "C" {
 
@@ -100,4 +100,66 @@ void f5m(int, int, int, int, int, P16);
 // CHECK: declare void @f5(i32 noundef, [2 x i64])
 // CHECK: declare void @f5m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [2 x i64])
 
+//BitInt alignment
+struct BITINT129 {
+    char ch;
+    unsigned _BitInt(129) v;
+};
+
+int test_bitint129(){
+  return __builtin_offsetof(struct BITINT129, v);
 }
+// CHECK:  ret i32 16 
+
+struct BITINT127 {
+    char ch;
+    _BitInt(127) v;
+};
+
+int test_bitint127(){
+  return __builtin_offsetof(struct BITINT127, v);
+}
+// CHECK:  ret i32 16 
+
+struct BITINT63 {
+    char ch;
+    _BitInt(63) v;
+};
+
+int test_bitint63(){
+  return __builtin_offsetof(struct BITINT63, v);
+}
+// CHECK:  ret i32 8 
+
+struct BITINT32 {
+    char ch;
+    unsigned _BitInt(32) v;
+};
+
+int test_bitint32(){
+  return __builtin_offsetof(struct BITINT32, v);
+}
+// CHECK:  ret i32 4
+
+struct BITINT9 {
+    char ch;
+    unsigned _BitInt(9) v;
+};
+
+int test_bitint9(){
+  return __builtin_offsetof(struct BITINT9, v);
+}
+// CHECK:  ret i32 2
+
+struct BITINT8 {
+    char ch;
+    unsigned _BitInt(8) v;
+};
+
+int test_bitint8(){
+  return __builtin_offsetof(struct BITINT8, v);
+}
+// CHECK:  ret i32 1
+
+}
+
-- 
GitLab


From 8afa6cf510608079e24d07423782c4db20de7498 Mon Sep 17 00:00:00 2001
From: Hristo Hristov <hghristov.rmm@gmail.com>
Date: Thu, 9 May 2024 12:48:37 +0300
Subject: [PATCH 0265/1206] [libc++][functional] P2944R3 (partial): Comparisons
 for `reference_wrapper` (`reference_wrapper` operators only) (#88384)

Implements https://wg21.link/P2944R3 (partially)
Implements https://wg21.link/LWG4071 /
https://cplusplus.github.io/LWG/issue4071 (fixes build failures in the
test suite)
- https://eel.is/c++draft/refwrap.comparisons
---
 libcxx/docs/FeatureTestMacroTable.rst         |  2 +-
 libcxx/docs/ReleaseNotes/19.rst               |  1 +
 libcxx/docs/Status/Cxx2c.rst                  |  1 +
 libcxx/docs/Status/Cxx2cIssues.csv            |  1 +
 libcxx/docs/Status/Cxx2cPapers.csv            |  2 +-
 .../include/__functional/reference_wrapper.h  | 51 ++++++++++
 libcxx/include/functional                     |  9 ++
 libcxx/include/version                        |  2 +-
 .../functional.version.compile.pass.cpp       | 16 +---
 .../version.version.compile.pass.cpp          | 16 +---
 ...mpare.three_way.refwrap.const_ref.pass.cpp | 89 +++++++++++++++++
 ...compare.three_way.refwrap.refwrap.pass.cpp | 93 ++++++++++++++++++
 ...e.three_way.refwrap.refwrap_const.pass.cpp | 95 +++++++++++++++++++
 .../equal.refwrap.const_ref.pass.cpp          | 62 ++++++++++++
 .../equal.refwrap.refwrap.pass.cpp            | 64 +++++++++++++
 .../equal.refwrap.refwrap_const.pass.cpp      | 67 +++++++++++++
 .../refwrap.comparissons/helper_concepts.h    | 38 ++++++++
 .../refwrap.comparissons/helper_types.h       | 30 ++++++
 .../generate_feature_test_macro_components.py |  1 -
 19 files changed, 614 insertions(+), 26 deletions(-)
 create mode 100644 libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.const_ref.pass.cpp
 create mode 100644 libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap.pass.cpp
 create mode 100644 libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap_const.pass.cpp
 create mode 100644 libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.const_ref.pass.cpp
 create mode 100644 libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap.pass.cpp
 create mode 100644 libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap_const.pass.cpp
 create mode 100644 libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_concepts.h
 create mode 100644 libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_types.h

diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 1032a9c338f4..17b3476d2c86 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -446,7 +446,7 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_rcu``                                          *unimplemented*
     ---------------------------------------------------------- -----------------
-    ``__cpp_lib_reference_wrapper``                            *unimplemented*
+    ``__cpp_lib_reference_wrapper``                            ``202403L``
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_saturation_arithmetic``                        ``202311L``
     ---------------------------------------------------------- -----------------
diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index 5a07b11cbcd5..3fc007f45985 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -46,6 +46,7 @@ Implemented Papers
 - P2869R4 - Remove Deprecated ``shared_ptr`` Atomic Access APIs from C++26
 - P2872R3 - Remove ``wstring_convert`` From C++26
 - P3142R0 - Printing Blank Lines with ``println`` (as DR against C++23)
+- P2944R3 - Comparisons for ``reference_wrapper`` (comparison operators for ``reference_wrapper`` only)
 - P2302R4 - ``std::ranges::contains``
 - P1659R3 - ``std::ranges::starts_with`` and ``std::ranges::ends_with``
 - P3029R1 - Better ``mdspan``'s CTAD
diff --git a/libcxx/docs/Status/Cxx2c.rst b/libcxx/docs/Status/Cxx2c.rst
index e3d9cbb551ff..5f459b4b3e4e 100644
--- a/libcxx/docs/Status/Cxx2c.rst
+++ b/libcxx/docs/Status/Cxx2c.rst
@@ -41,6 +41,7 @@ Paper Status
 
    .. [#note-P2510R3] This paper is applied as DR against C++20. (MSVC STL and libstdc++ will do the same.)
    .. [#note-P3142R0] This paper is applied as DR against C++23. (MSVC STL and libstdc++ will do the same.)
+   .. [#note-P2944R3] Implemented comparisons for ``reference_wrapper`` only.
 
 .. _issues-status-cxx2c:
 
diff --git a/libcxx/docs/Status/Cxx2cIssues.csv b/libcxx/docs/Status/Cxx2cIssues.csv
index 30a059f8a3df..76717e1d3448 100644
--- a/libcxx/docs/Status/Cxx2cIssues.csv
+++ b/libcxx/docs/Status/Cxx2cIssues.csv
@@ -64,4 +64,5 @@
 "","","","","",""
 "`3343 <https://wg21.link/LWG3343>`__","Ordering of calls to ``unlock()`` and ``notify_all()`` in Effects element of ``notify_all_at_thread_exit()`` should be reversed","Not Yet Adopted","|Complete|","16.0",""
 "XXXX","","The sys_info range should be affected by save","Not Yet Adopted","|Complete|","19.0"
+"`4071 <https://wg21.link/LWG4071>`__","","``reference_wrapper`` comparisons are not SFINAE-friendly","Not Yet Adopted","|Complete|","19.0"
 "","","","","",""
diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv
index 409278db1e87..30a601858b63 100644
--- a/libcxx/docs/Status/Cxx2cPapers.csv
+++ b/libcxx/docs/Status/Cxx2cPapers.csv
@@ -59,7 +59,7 @@
 "`P2248R8 <https://wg21.link/P2248R8>`__","LWG","Enabling list-initialization for algorithms","Tokyo March 2024","","",""
 "`P2810R4 <https://wg21.link/P2810R4>`__","LWG","``is_debugger_present`` ``is_replaceable``","Tokyo March 2024","","",""
 "`P1068R11 <https://wg21.link/P1068R11>`__","LWG","Vector API for random number generation","Tokyo March 2024","","",""
-"`P2944R3 <https://wg21.link/P2944R3>`__","LWG","Comparisons for ``reference_wrapper``","Tokyo March 2024","","",""
+"`P2944R3 <https://wg21.link/P2944R3>`__","LWG","Comparisons for ``reference_wrapper``","Tokyo March 2024","|Partial| [#note-P2944R3]_","19.0",""
 "`P2642R6 <https://wg21.link/P2642R6>`__","LWG","Padded ``mdspan`` layouts","Tokyo March 2024","","",""
 "`P3029R1 <https://wg21.link/P3029R1>`__","LWG","Better ``mdspan``'s CTAD","Tokyo March 2024","|Complete|","19.0",""
 "","","","","","",""
diff --git a/libcxx/include/__functional/reference_wrapper.h b/libcxx/include/__functional/reference_wrapper.h
index 94b39e3bc786..ab5d7c7cee11 100644
--- a/libcxx/include/__functional/reference_wrapper.h
+++ b/libcxx/include/__functional/reference_wrapper.h
@@ -10,11 +10,14 @@
 #ifndef _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
 #define _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
 
+#include <__compare/synth_three_way.h>
+#include <__concepts/boolean_testable.h>
 #include <__config>
 #include <__functional/invoke.h>
 #include <__functional/weak_result_type.h>
 #include <__memory/addressof.h>
 #include <__type_traits/enable_if.h>
+#include <__type_traits/is_const.h>
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
@@ -64,6 +67,54 @@ public:
   {
     return std::__invoke(get(), std::forward<_ArgTypes>(__args)...);
   }
+
+#if _LIBCPP_STD_VER >= 26
+
+  // [refwrap.comparisons], comparisons
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(reference_wrapper __x, reference_wrapper __y)
+    requires requires {
+      { __x.get() == __y.get() } -> __boolean_testable;
+    }
+  {
+    return __x.get() == __y.get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(reference_wrapper __x, const _Tp& __y)
+    requires requires {
+      { __x.get() == __y } -> __boolean_testable;
+    }
+  {
+    return __x.get() == __y;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(reference_wrapper __x, reference_wrapper<const _Tp> __y)
+    requires(!is_const_v<_Tp>) && requires {
+      { __x.get() == __y.get() } -> __boolean_testable;
+    }
+  {
+    return __x.get() == __y.get();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(reference_wrapper __x, reference_wrapper __y)
+    requires requires { std::__synth_three_way(__x.get(), __y.get()); }
+  {
+    return std::__synth_three_way(__x.get(), __y.get());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(reference_wrapper __x, const _Tp& __y)
+    requires requires { std::__synth_three_way(__x.get(), __y); }
+  {
+    return std::__synth_three_way(__x.get(), __y);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(reference_wrapper __x, reference_wrapper<const _Tp> __y)
+    requires(!is_const_v<_Tp>) && requires { std::__synth_three_way(__x.get(), __y.get()); }
+  {
+    return std::__synth_three_way(__x.get(), __y.get());
+  }
+
+#endif // _LIBCPP_STD_VER >= 26
 };
 
 #if _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/functional b/libcxx/include/functional
index a2476c93ad1b..27cf21e1a4c8 100644
--- a/libcxx/include/functional
+++ b/libcxx/include/functional
@@ -77,6 +77,15 @@ template <class T> struct unwrap_ref_decay : unwrap_reference<decay_t<T>> { };
 template <class T> using unwrap_reference_t = typename unwrap_reference<T>::type; // since C++20
 template <class T> using unwrap_ref_decay_t = typename unwrap_ref_decay<T>::type; // since C++20
 
+// [refwrap.comparisons], comparisons
+friend constexpr bool operator==(reference_wrapper, reference_wrapper);           // Since C++26
+friend constexpr bool operator==(reference_wrapper, const T&);                    // Since C++26
+friend constexpr bool operator==(reference_wrapper, reference_wrapper<const T>);  // Since C++26
+
+friend constexpr auto operator<=>(reference_wrapper, reference_wrapper);          // Since C++26
+friend constexpr auto operator<=>(reference_wrapper, const T&);                   // Since C++26
+friend constexpr auto operator<=>(reference_wrapper, reference_wrapper<const T>); // Since C++26
+
 template <class T> // <class T=void> in C++14
 struct plus {
     T operator()(const T& x, const T& y) const;
diff --git a/libcxx/include/version b/libcxx/include/version
index eb5fd5c80578..ba116957b033 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -526,7 +526,7 @@ __cpp_lib_within_lifetime                               202306L <type_traits>
 // # define __cpp_lib_ranges_concat                        202403L
 # define __cpp_lib_ratio                                202306L
 // # define __cpp_lib_rcu                                  202306L
-// # define __cpp_lib_reference_wrapper                    202403L
+# define __cpp_lib_reference_wrapper                    202403L
 # define __cpp_lib_saturation_arithmetic                202311L
 // # define __cpp_lib_smart_ptr_owner_equality             202306L
 # define __cpp_lib_span_at                              202311L
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
index aeb09a30b425..27e76e5b2b05 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
@@ -535,17 +535,11 @@
 #   error "__cpp_lib_ranges should have the value 202207L in c++26"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_reference_wrapper
-#     error "__cpp_lib_reference_wrapper should be defined in c++26"
-#   endif
-#   if __cpp_lib_reference_wrapper != 202403L
-#     error "__cpp_lib_reference_wrapper should have the value 202403L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_reference_wrapper
-#     error "__cpp_lib_reference_wrapper should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_reference_wrapper
+#   error "__cpp_lib_reference_wrapper should be defined in c++26"
+# endif
+# if __cpp_lib_reference_wrapper != 202403L
+#   error "__cpp_lib_reference_wrapper should have the value 202403L in c++26"
 # endif
 
 # ifndef __cpp_lib_result_of_sfinae
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index da7a780528c7..d7035d7e5e3a 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -7464,17 +7464,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_reference_wrapper
-#     error "__cpp_lib_reference_wrapper should be defined in c++26"
-#   endif
-#   if __cpp_lib_reference_wrapper != 202403L
-#     error "__cpp_lib_reference_wrapper should have the value 202403L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_reference_wrapper
-#     error "__cpp_lib_reference_wrapper should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_reference_wrapper
+#   error "__cpp_lib_reference_wrapper should be defined in c++26"
+# endif
+# if __cpp_lib_reference_wrapper != 202403L
+#   error "__cpp_lib_reference_wrapper should have the value 202403L in c++26"
 # endif
 
 # ifndef __cpp_lib_remove_cvref
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.const_ref.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.const_ref.pass.cpp
new file mode 100644
index 000000000000..85106c18ec35
--- /dev/null
+++ b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.const_ref.pass.cpp
@@ -0,0 +1,89 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+
+// <functional>
+
+// class reference_wrapper
+
+// [refwrap.comparisons], comparisons
+
+// friend constexpr auto operator<=>(reference_wrapper, const T&);                   // Since C++26
+
+#include <cassert>
+#include <concepts>
+#include <functional>
+
+#include "test_comparisons.h"
+#include "test_macros.h"
+
+#include "helper_concepts.h"
+#include "helper_types.h"
+
+// Test SFINAE.
+
+static_assert(HasSpaceshipOperatorWithInt<std::reference_wrapper<StrongOrder>>);
+static_assert(HasSpaceshipOperatorWithInt<std::reference_wrapper<WeakOrder>>);
+static_assert(HasSpaceshipOperatorWithInt<std::reference_wrapper<PartialOrder>>);
+
+static_assert(!HasSpaceshipOperatorWithInt<std::reference_wrapper<NonComparable>>);
+
+// Test comparisons.
+
+template <typename T, typename Order>
+constexpr void test() {
+  T t{47};
+
+  T bigger{94};
+  T smaller{82};
+
+  T unordered{std::numeric_limits<int>::min()};
+
+  // Identical contents
+  {
+    std::reference_wrapper<T> rw1{t};
+    assert(testOrder(rw1, t, Order::equivalent));
+  }
+  // Less
+  {
+    std::reference_wrapper<T> rw1{smaller};
+    assert(testOrder(rw1, bigger, Order::less));
+  }
+  // Greater
+  {
+    std::reference_wrapper<T> rw1{bigger};
+    assert(testOrder(rw1, smaller, Order::greater));
+  }
+  // Unordered
+  if constexpr (std::same_as<T, PartialOrder>) {
+    std::reference_wrapper<T> rw1{bigger};
+    assert(testOrder(rw1, unordered, Order::unordered));
+  }
+}
+
+constexpr bool test() {
+  test<int, std::strong_ordering>();
+  test<StrongOrder, std::strong_ordering>();
+  test<int, std::weak_ordering>();
+  test<WeakOrder, std::weak_ordering>();
+  test<int, std::partial_ordering>();
+  test<PartialOrder, std::partial_ordering>();
+
+  // `LessAndEqComp` does not have `operator<=>`. Ordering is synthesized based on `operator<`
+  test<LessAndEqComp, std::weak_ordering>();
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap.pass.cpp
new file mode 100644
index 000000000000..794fac00de8a
--- /dev/null
+++ b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap.pass.cpp
@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+
+// <functional>
+
+// class reference_wrapper
+
+// [refwrap.comparisons], comparisons
+
+// friend constexpr auto operator<=>(reference_wrapper, reference_wrapper);          // Since C++26
+
+#include <cassert>
+#include <concepts>
+#include <functional>
+
+#include "test_comparisons.h"
+#include "test_macros.h"
+
+#include "helper_concepts.h"
+#include "helper_types.h"
+
+// Test SFINAE.
+
+static_assert(std::three_way_comparable<std::reference_wrapper<StrongOrder>>);
+static_assert(std::three_way_comparable<std::reference_wrapper<WeakOrder>>);
+static_assert(std::three_way_comparable<std::reference_wrapper<PartialOrder>>);
+
+static_assert(!std::three_way_comparable<std::reference_wrapper<NonComparable>>);
+
+// Test comparisons.
+
+template <typename T, typename Order>
+constexpr void test() {
+  T t{47};
+
+  T bigger{94};
+  T smaller{82};
+
+  T unordered{std::numeric_limits<int>::min()};
+
+  // Identical contents
+  {
+    std::reference_wrapper<T> rw1{t};
+    std::reference_wrapper<T> rw2{t};
+    assert(testOrder(rw1, rw2, Order::equivalent));
+  }
+  // Less
+  {
+    std::reference_wrapper<T> rw1{smaller};
+    std::reference_wrapper<T> rw2{bigger};
+    assert(testOrder(rw1, rw2, Order::less));
+  }
+  // Greater
+  {
+    std::reference_wrapper<T> rw1{bigger};
+    std::reference_wrapper<T> rw2{smaller};
+    assert(testOrder(rw1, rw2, Order::greater));
+  }
+  // Unordered
+  if constexpr (std::same_as<T, PartialOrder>) {
+    std::reference_wrapper<T> rw1{bigger};
+    std::reference_wrapper<T> rw2{unordered};
+    assert(testOrder(rw1, rw2, Order::unordered));
+  }
+}
+
+constexpr bool test() {
+  test<int, std::strong_ordering>();
+  test<StrongOrder, std::strong_ordering>();
+  test<int, std::weak_ordering>();
+  test<WeakOrder, std::weak_ordering>();
+  test<int, std::partial_ordering>();
+  test<PartialOrder, std::partial_ordering>();
+
+  // `LessAndEqComp` does not have `operator<=>`. Ordering is synthesized based on `operator<`
+  test<LessAndEqComp, std::weak_ordering>();
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap_const.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap_const.pass.cpp
new file mode 100644
index 000000000000..9b1302affa85
--- /dev/null
+++ b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap_const.pass.cpp
@@ -0,0 +1,95 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+
+// <functional>
+
+// class reference_wrapper
+
+// [refwrap.comparisons], comparisons
+
+// friend constexpr auto operator<=>(reference_wrapper, reference_wrapper<const T>); // Since C++26
+
+#include <cassert>
+#include <concepts>
+#include <functional>
+
+#include "test_comparisons.h"
+#include "test_macros.h"
+
+#include "helper_concepts.h"
+#include "helper_types.h"
+
+// Test SFINAE.
+
+static_assert(std::three_way_comparable_with<std::reference_wrapper<StrongOrder>, const StrongOrder>);
+static_assert(std::three_way_comparable_with<std::reference_wrapper<WeakOrder>, const WeakOrder>);
+static_assert(std::three_way_comparable_with<std::reference_wrapper<PartialOrder>, const PartialOrder>);
+
+static_assert(!std::three_way_comparable_with<std::reference_wrapper<StrongOrder>, const NonComparable>);
+static_assert(!std::three_way_comparable_with<std::reference_wrapper<WeakOrder>, const NonComparable>);
+static_assert(!std::three_way_comparable_with<std::reference_wrapper<PartialOrder>, const NonComparable>);
+
+// Test comparisons.
+
+template <typename T, typename Order>
+constexpr void test() {
+  T t{47};
+
+  T bigger{94};
+  T smaller{82};
+
+  T unordered{std::numeric_limits<int>::min()};
+
+  // Identical contents
+  {
+    std::reference_wrapper<T> rw1{t};
+    std::reference_wrapper<const T> rw2{t};
+    assert(testOrder(rw1, rw2, Order::equivalent));
+  }
+  // Less
+  {
+    std::reference_wrapper<T> rw1{smaller};
+    std::reference_wrapper<const T> rw2{bigger};
+    assert(testOrder(rw1, rw2, Order::less));
+  }
+  // Greater
+  {
+    std::reference_wrapper<T> rw1{bigger};
+    std::reference_wrapper<const T> rw2{smaller};
+    assert(testOrder(rw1, rw2, Order::greater));
+  }
+  // Unordered
+  if constexpr (std::same_as<T, PartialOrder>) {
+    std::reference_wrapper<T> rw1{bigger};
+    std::reference_wrapper<const T> rw2{unordered};
+    assert(testOrder(rw1, rw2, Order::unordered));
+  }
+}
+
+constexpr bool test() {
+  test<int, std::strong_ordering>();
+  test<StrongOrder, std::strong_ordering>();
+  test<int, std::weak_ordering>();
+  test<WeakOrder, std::weak_ordering>();
+  test<int, std::partial_ordering>();
+  test<PartialOrder, std::partial_ordering>();
+
+  // `LessAndEqComp` does not have `operator<=>`. Ordering is synthesized based on `operator<`
+  test<LessAndEqComp, std::weak_ordering>();
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.const_ref.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.const_ref.pass.cpp
new file mode 100644
index 000000000000..465326818f17
--- /dev/null
+++ b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.const_ref.pass.cpp
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+
+// <functional>
+
+// class reference_wrapper
+
+// [refwrap.comparisons], comparisons
+
+// friend constexpr bool operator==(reference_wrapper, const T&);                                         // Since C++26
+
+#include <cassert>
+#include <concepts>
+#include <functional>
+
+#include "test_comparisons.h"
+#include "test_macros.h"
+
+#include "helper_concepts.h"
+#include "helper_types.h"
+
+// Test SFINAE.
+
+static_assert(HasEqualityOperatorWithInt<std::reference_wrapper<EqualityComparable>>);
+
+static_assert(!HasEqualityOperatorWithInt<std::reference_wrapper<NonComparable>>);
+
+// Test equality.
+
+template <typename T>
+constexpr void test() {
+  T i{92};
+  T j{84};
+
+  std::reference_wrapper<T> rw1{i};
+
+  // refwrap, const&
+  AssertEqualityReturnBool<decltype(rw1), decltype(i)>();
+  assert(testEquality(rw1, i, true));
+  assert(testEquality(rw1, j, false));
+}
+
+constexpr bool test() {
+  test<int>();
+  test<EqualityComparable>();
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap.pass.cpp
new file mode 100644
index 000000000000..a50b530bbc6e
--- /dev/null
+++ b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap.pass.cpp
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+
+// <functional>
+
+// class reference_wrapper
+
+// [refwrap.comparisons], comparisons
+// friend constexpr bool operator==(reference_wrapper, reference_wrapper);                                // Since C++26
+
+#include <cassert>
+#include <concepts>
+#include <functional>
+
+#include "test_comparisons.h"
+#include "test_macros.h"
+
+#include "helper_concepts.h"
+#include "helper_types.h"
+
+// Test SFINAE.
+
+static_assert(std::equality_comparable<std::reference_wrapper<EqualityComparable>>);
+
+static_assert(!std::equality_comparable<std::reference_wrapper<NonComparable>>);
+
+// Test equality.
+
+template <typename T>
+constexpr void test() {
+  T i{92};
+  T j{84};
+
+  std::reference_wrapper<T> rw1{i};
+  std::reference_wrapper<T> rw2 = rw1;
+  std::reference_wrapper<T> rw3{j};
+  std::reference_wrapper<const T> crw1{i};
+  std::reference_wrapper<const T> crw3{j};
+
+  AssertEqualityReturnBool<decltype(rw1), decltype(rw2)>();
+  assert(testEquality(rw1, rw2, true));
+  assert(testEquality(rw1, rw3, false));
+}
+
+constexpr bool test() {
+  test<int>();
+  test<EqualityComparable>();
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap_const.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap_const.pass.cpp
new file mode 100644
index 000000000000..10f017742a87
--- /dev/null
+++ b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap_const.pass.cpp
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+
+// <functional>
+
+// class reference_wrapper
+
+// [refwrap.comparisons], comparisons
+
+// friend constexpr bool operator==(reference_wrapper, reference_wrapper<const T>);                       // Since C++26
+
+#include <cassert>
+#include <concepts>
+#include <functional>
+
+#include "test_comparisons.h"
+#include "test_macros.h"
+
+#include "helper_concepts.h"
+#include "helper_types.h"
+
+// Test SFINAE.
+
+static_assert(std::equality_comparable_with<std::reference_wrapper<EqualityComparable>,
+                                            std::reference_wrapper<const EqualityComparable>>);
+
+static_assert(!std::equality_comparable_with<std::reference_wrapper<EqualityComparable>,
+                                             std::reference_wrapper<const NonComparable>>);
+
+// Test equality.
+
+template <typename T>
+constexpr void test() {
+  T i{92};
+  T j{84};
+
+  std::reference_wrapper<T> rw1{i};
+
+  std::reference_wrapper<T> rw3{j};
+  std::reference_wrapper<const T> crw1{i};
+  std::reference_wrapper<const T> crw3{j};
+
+  AssertEqualityReturnBool<decltype(rw1), decltype(crw1)>();
+  assert(testEquality(rw1, crw1, true));
+  assert(testEquality(rw1, crw3, false));
+}
+
+constexpr bool test() {
+  test<int>();
+  test<EqualityComparable>();
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_concepts.h b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_concepts.h
new file mode 100644
index 000000000000..2dbb304f8af6
--- /dev/null
+++ b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_concepts.h
@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_CONCEPTS_H
+#define TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_CONCEPTS_H
+
+#include <concepts>
+#include <utility>
+
+// Equality
+
+template <typename T>
+concept HasEqualityOperatorWithInt = requires(T t, int i) {
+  { t.get() == i } -> std::convertible_to<bool>;
+};
+
+// Spaceship
+
+template <class T>
+concept BooleanTestableImpl = std::convertible_to<T, bool>;
+
+template <class T>
+concept BooleanTestable = BooleanTestableImpl<T> && requires(T&& t) {
+  { !std::forward<T>(t) } -> BooleanTestableImpl;
+};
+
+template <typename T>
+concept HasSpaceshipOperatorWithInt = requires(T t, int i) {
+  { t < i } -> BooleanTestable;
+  { i < t } -> BooleanTestable;
+};
+
+#endif // TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_CONCEPTS_H
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_types.h b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_types.h
new file mode 100644
index 000000000000..cf5e568dbf93
--- /dev/null
+++ b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/helper_types.h
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_TYPES_H
+#define TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_TYPES_H
+
+#include <concepts>
+
+struct EqualityComparable {
+  constexpr EqualityComparable(int value) : value_{value} {};
+
+  friend constexpr bool operator==(const EqualityComparable&, const EqualityComparable&) noexcept = default;
+
+  int value_;
+};
+
+static_assert(std::equality_comparable<EqualityComparable>);
+static_assert(EqualityComparable{94} == EqualityComparable{94});
+static_assert(EqualityComparable{94} != EqualityComparable{82});
+
+struct NonComparable {};
+
+static_assert(!std::three_way_comparable<NonComparable>);
+
+#endif // TEST_STD_FUNCTIONOBJECTS_REFWRAP_HELPER_TYPES_H
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index 29c8ceba0c7e..d0171c84acbc 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -1044,7 +1044,6 @@ feature_test_macros = [
             "name": "__cpp_lib_reference_wrapper",
             "values": {"c++26": 202403}, # P2944R3: Comparisons for reference_wrapper
             "headers": ["functional"],
-            "unimplemented": True,
         },
         {
             "name": "__cpp_lib_remove_cvref",
-- 
GitLab


From 2cbfe4a823020b2efe53d32ad7eccbc5a037943f Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Thu, 9 May 2024 10:59:25 +0100
Subject: [PATCH 0266/1206] [AMDGPU] Remove duplicate -mtriple options in tests
 (#91576)

---
 .../GlobalISel/llvm.amdgcn.workitem.id.ll      |  6 +++---
 llvm/test/CodeGen/AMDGPU/combine_vloads.ll     |  2 +-
 llvm/test/CodeGen/AMDGPU/dead_bundle.mir       |  2 +-
 llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll |  2 +-
 llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll   | 18 +++++++++---------
 .../CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll  |  4 ++--
 llvm/test/CodeGen/AMDGPU/load-constant-i1.ll   |  2 +-
 llvm/test/CodeGen/AMDGPU/load-global-i1.ll     |  2 +-
 llvm/test/CodeGen/AMDGPU/load-local-i1.ll      |  2 +-
 llvm/test/CodeGen/AMDGPU/load-local-i8.ll      |  2 +-
 .../AMDGPU/nullptr-long-address-spaces.ll      |  2 +-
 llvm/test/CodeGen/AMDGPU/nullptr.ll            |  2 +-
 llvm/test/CodeGen/AMDGPU/setcc.ll              |  2 +-
 llvm/test/CodeGen/AMDGPU/sext-in-reg.ll        |  2 +-
 llvm/test/CodeGen/AMDGPU/shl.ll                |  2 +-
 llvm/test/CodeGen/AMDGPU/sra.ll                |  2 +-
 llvm/test/CodeGen/AMDGPU/store-global.ll       |  4 ++--
 llvm/test/CodeGen/AMDGPU/store-local.ll        |  4 ++--
 .../trunc-vector-store-assertion-failure.ll    |  2 +-
 llvm/test/CodeGen/AMDGPU/unknown-processor.ll  |  2 +-
 llvm/test/CodeGen/AMDGPU/unsupported-calls.ll  |  2 +-
 llvm/test/CodeGen/AMDGPU/vector-alloca.ll      |  2 +-
 .../CodeGen/AMDGPU/wrong-transalu-pos-fix.ll   |  2 +-
 23 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll
index 2e62d13f1e69..09882c446fc0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll
@@ -4,9 +4,9 @@
 ; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -global-isel -mtriple=amdgcn-- -mcpu=tonga -mattr=+flat-for-global -verify-machineinstrs | FileCheck --check-prefixes=ALL,MESA,UNPACKED %s
 ; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -global-isel -mtriple=amdgcn-unknown-mesa3d -mattr=+flat-for-global -mcpu=hawaii -verify-machineinstrs | FileCheck -check-prefixes=ALL,MESA3D,UNPACKED %s
 ; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -global-isel -mtriple=amdgcn-unknown-mesa3d -mcpu=tonga -verify-machineinstrs | FileCheck -check-prefixes=ALL,MESA3D,UNPACKED %s
-; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -global-isel -mtriple=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx90a -verify-machineinstrs | FileCheck -check-prefixes=ALL,PACKED-TID %s
-; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -global-isel -mtriple=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-vopd=0 | FileCheck -check-prefixes=ALL,PACKED-TID %s
-; RUN: sed 's/CODE_OBJECT_VERSION/600/g' %s | llc -global-isel -mtriple=amdgcn -mtriple=amdgcn-unknown-amdhsa --amdhsa-code-object-version=6 -mcpu=gfx11-generic -verify-machineinstrs -amdgpu-enable-vopd=0 | FileCheck -check-prefixes=ALL,PACKED-TID %s
+; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -global-isel -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx90a -verify-machineinstrs | FileCheck -check-prefixes=ALL,PACKED-TID %s
+; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -global-isel -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-vopd=0 | FileCheck -check-prefixes=ALL,PACKED-TID %s
+; RUN: sed 's/CODE_OBJECT_VERSION/600/g' %s | llc -global-isel -mtriple=amdgcn-unknown-amdhsa --amdhsa-code-object-version=6 -mcpu=gfx11-generic -verify-machineinstrs -amdgpu-enable-vopd=0 | FileCheck -check-prefixes=ALL,PACKED-TID %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 declare i32 @llvm.amdgcn.workitem.id.y() #0
diff --git a/llvm/test/CodeGen/AMDGPU/combine_vloads.ll b/llvm/test/CodeGen/AMDGPU/combine_vloads.ll
index 10b7d62e275d..42a9b80b134c 100644
--- a/llvm/test/CodeGen/AMDGPU/combine_vloads.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine_vloads.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=cypress < %s | FileCheck -check-prefix=EG %s
+; RUN: llc -mtriple=r600-- -mcpu=cypress < %s | FileCheck -check-prefix=EG %s
 
 ;
 ; kernel void combine_vloads(global char8 addrspace(5)* src, global char8 addrspace(5)* result) {
diff --git a/llvm/test/CodeGen/AMDGPU/dead_bundle.mir b/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
index dd9d6a1c788e..af656ea1c719 100644
--- a/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/dead_bundle.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn--amdpal -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs=1 -start-before=greedy,0 -stop-after=virtregrewriter,0 -stress-regalloc=5 %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1100 -verify-machineinstrs=1 -start-before=greedy,0 -stop-after=virtregrewriter,0 -stress-regalloc=5 %s -o - | FileCheck %s
 
 # This test checks that dead bundles are handled correctly.
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index 7c5bdd691fc4..1c093bf31ea7 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -1,6 +1,6 @@
 ; RUN: not llc -mtriple=amdgcn-- -mcpu=tahiti -mattr=+promote-alloca -verify-machineinstrs < %s 2>&1 | FileCheck %s
 ; RUN: not llc -mtriple=amdgcn-- -mcpu=tahiti -mattr=-promote-alloca -verify-machineinstrs < %s 2>&1 | FileCheck %s
-; RUN: not llc -mtriple=r600 -mtriple=r600-- -mcpu=cypress < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=r600-- -mcpu=cypress < %s 2>&1 | FileCheck %s
 target datalayout = "A5"
 
 ; CHECK: in function test_dynamic_stackalloc{{.*}}: unsupported dynamic alloca
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
index 1633d21c41d5..e4ffedd686ac 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll
@@ -7,17 +7,17 @@
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=carrizo -mattr=+xnack -verify-machineinstrs | FileCheck -check-prefix=VI-XNACK  -check-prefix=GCN %s
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=stoney -mattr=+xnack -verify-machineinstrs | FileCheck -check-prefix=VI-XNACK  -check-prefix=GCN %s
 
-; RUN: llc < %s -mtriple=amdgcn -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs | FileCheck -check-prefixes=GCN %s
-; RUN: llc < %s -mtriple=amdgcn -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=-xnack -verify-machineinstrs | FileCheck -check-prefixes=VI-NOXNACK,HSA-VI-NOXNACK,GCN %s
-; RUN: llc < %s -mtriple=amdgcn -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=+xnack -verify-machineinstrs | FileCheck -check-prefixes=VI-XNACK,HSA-VI-XNACK,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs | FileCheck -check-prefixes=GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=-xnack -verify-machineinstrs | FileCheck -check-prefixes=VI-NOXNACK,HSA-VI-NOXNACK,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=+xnack -verify-machineinstrs | FileCheck -check-prefixes=VI-XNACK,HSA-VI-XNACK,GCN %s
 
-; RUN: llc < %s -mtriple=amdgcn -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch -verify-machineinstrs | FileCheck -check-prefixes=GCN %s
-; RUN: llc < %s -mtriple=amdgcn -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch,-xnack -verify-machineinstrs | FileCheck -check-prefixes=HSA-VI-NOXNACK,GFX9-ARCH-FLAT,GCN %s
-; RUN: llc < %s -mtriple=amdgcn -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch,+xnack -verify-machineinstrs | FileCheck -check-prefixes=HSA-VI-XNACK,GFX9-ARCH-FLAT,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch -verify-machineinstrs | FileCheck -check-prefixes=GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch,-xnack -verify-machineinstrs | FileCheck -check-prefixes=HSA-VI-NOXNACK,GFX9-ARCH-FLAT,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=+architected-flat-scratch,+xnack -verify-machineinstrs | FileCheck -check-prefixes=HSA-VI-XNACK,GFX9-ARCH-FLAT,GCN %s
 
-; RUN: llc < %s -mtriple=amdgcn -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+architected-flat-scratch -verify-machineinstrs | FileCheck -check-prefixes=GCN %s
-; RUN: llc < %s -mtriple=amdgcn -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+architected-flat-scratch,-xnack -verify-machineinstrs | FileCheck -check-prefixes=HSA-VI-NOXNACK,GFX10-ARCH-FLAT,GCN %s
-; RUN: llc < %s -mtriple=amdgcn -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+architected-flat-scratch,+xnack -verify-machineinstrs | FileCheck -check-prefixes=HSA-VI-XNACK,GFX10-ARCH-FLAT,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+architected-flat-scratch -verify-machineinstrs | FileCheck -check-prefixes=GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+architected-flat-scratch,-xnack -verify-machineinstrs | FileCheck -check-prefixes=HSA-VI-NOXNACK,GFX10-ARCH-FLAT,GCN %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+architected-flat-scratch,+xnack -verify-machineinstrs | FileCheck -check-prefixes=HSA-VI-XNACK,GFX10-ARCH-FLAT,GCN %s
 
 ; GCN-LABEL: {{^}}no_vcc_no_flat:
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll
index a1835ea176d5..47f988fc17d2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll
@@ -2,8 +2,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck --check-prefixes=ALL,MESA,UNPACKED %s
 ; RUN: llc -mtriple=amdgcn-unknown-mesa3d -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,MESA3D,UNPACKED %s
 ; RUN: llc -mtriple=amdgcn-unknown-mesa3d -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,MESA3D,UNPACKED %s
-; RUN: llc -mtriple=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,PACKED-TID %s
-; RUN: llc -mtriple=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=ALL,PACKED-TID %s
+; RUN: llc -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,PACKED-TID %s
+; RUN: llc -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=ALL,PACKED-TID %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 declare i32 @llvm.amdgcn.workitem.id.y() #0
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 88b18232ef9c..502cd14284e1 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -check-prefix=GFX6 %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=cypress < %s | FileCheck -check-prefix=EG %s
+; RUN: llc -mtriple=r600-- -mcpu=cypress < %s | FileCheck -check-prefix=EG %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s
 
 define amdgpu_kernel void @constant_load_i1(ptr addrspace(1) %out, ptr addrspace(4) nocapture %in) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i1.ll b/llvm/test/CodeGen/AMDGPU/load-global-i1.ll
index 5ab1f3d972b0..dac928d70c65 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i1.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -mtriple=r600-- -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}global_load_i1:
 ; GCN: buffer_load_ubyte
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i1.ll b/llvm/test/CodeGen/AMDGPU/load-local-i1.ll
index ea858fb67443..578170941efa 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i1.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=cypress < %s | FileCheck -check-prefixes=EG,FUNC %s
+; RUN: llc -mtriple=r600-- -mcpu=cypress < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_i1:
 ; SICIVI: s_mov_b32 m0
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i8.ll b/llvm/test/CodeGen/AMDGPU/load-local-i8.ll
index 9b1b32a65f23..a2e55ce06b52 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i8.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,SI,SICIVI,FUNC %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,VI,SICIVI,FUNC %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GFX9,FUNC %s
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=redwood -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -mtriple=r600-- -mcpu=redwood -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=EG -check-prefix=FUNC %s
 
 ; Testing for ds_read/write_b128
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=+enable-ds128 < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=CIVI,FUNC %s
diff --git a/llvm/test/CodeGen/AMDGPU/nullptr-long-address-spaces.ll b/llvm/test/CodeGen/AMDGPU/nullptr-long-address-spaces.ll
index 98c869f23d47..6556f07c7350 100644
--- a/llvm/test/CodeGen/AMDGPU/nullptr-long-address-spaces.ll
+++ b/llvm/test/CodeGen/AMDGPU/nullptr-long-address-spaces.ll
@@ -1,6 +1,6 @@
 ; XFAIL: *
 ; RUN: llc < %s -mtriple=amdgcn-- -verify-machineinstrs | FileCheck -check-prefixes=CHECK,GCN %s
-; RUN: llc < %s -mtriple=r600 -mtriple=r600-- -verify-machineinstrs | FileCheck -check-prefixes=CHECK,R600 %s
+; RUN: llc < %s -mtriple=r600-- -verify-machineinstrs | FileCheck -check-prefixes=CHECK,R600 %s
 
 ; This is a temporary xfail, as the assembly printer is broken when dealing with
 ; lowerConstant() trying to return a value of size greater than 8 bytes.
diff --git a/llvm/test/CodeGen/AMDGPU/nullptr.ll b/llvm/test/CodeGen/AMDGPU/nullptr.ll
index b7a15f97e103..5a736aabd4ee 100644
--- a/llvm/test/CodeGen/AMDGPU/nullptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/nullptr.ll
@@ -1,5 +1,5 @@
 ;RUN: llc < %s -mtriple=amdgcn-- -verify-machineinstrs | FileCheck -check-prefixes=CHECK,GCN %s
-;RUN: llc < %s -mtriple=r600 -mtriple=r600-- -verify-machineinstrs | FileCheck -check-prefixes=CHECK,R600 %s
+;RUN: llc < %s -mtriple=r600-- -verify-machineinstrs | FileCheck -check-prefixes=CHECK,R600 %s
 
 %struct.S = type { ptr addrspace(5), ptr addrspace(1), ptr addrspace(4), ptr addrspace(3), ptr, ptr addrspace(2)}
 
diff --git a/llvm/test/CodeGen/AMDGPU/setcc.ll b/llvm/test/CodeGen/AMDGPU/setcc.ll
index 6ab49382b904..c00cd763992d 100644
--- a/llvm/test/CodeGen/AMDGPU/setcc.ll
+++ b/llvm/test/CodeGen/AMDGPU/setcc.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=redwood -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=R600 -check-prefix=FUNC %s
+; RUN: llc -mtriple=r600-- -mcpu=redwood -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=R600 -check-prefix=FUNC %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
diff --git a/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll b/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll
index 38672da3c647..4e3dccb975fe 100644
--- a/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll
@@ -1,7 +1,7 @@
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,SI,FUNC %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX89,FUNC %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX9,GFX89,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=r600 -mtriple=r600-- -mcpu=cypress < %s | FileCheck -enable-var-scope --check-prefixes=EG,FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=r600-- -mcpu=cypress < %s | FileCheck -enable-var-scope --check-prefixes=EG,FUNC %s
 
 ; FIXME: i16 promotion pass ruins the scalar cases when legal.
 ; FIXME: r600 fails verifier
diff --git a/llvm/test/CodeGen/AMDGPU/shl.ll b/llvm/test/CodeGen/AMDGPU/shl.ll
index c440392153ad..b1a82daa8e7d 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn-- -mcpu=verde -verify-machineinstrs | FileCheck %s --check-prefixes=SI
 ; RUN: llc < %s -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s -check-prefixes=VI
-; RUN: llc < %s -amdgpu-scalarize-global-loads=false  -mtriple=r600 -mtriple=r600-- -mcpu=redwood -verify-machineinstrs | FileCheck %s --check-prefixes=EG
+; RUN: llc < %s -amdgpu-scalarize-global-loads=false  -mtriple=r600-- -mcpu=redwood -verify-machineinstrs | FileCheck %s --check-prefixes=EG
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 
diff --git a/llvm/test/CodeGen/AMDGPU/sra.ll b/llvm/test/CodeGen/AMDGPU/sra.ll
index ae0221b8b32b..b8cf69237206 100644
--- a/llvm/test/CodeGen/AMDGPU/sra.ll
+++ b/llvm/test/CodeGen/AMDGPU/sra.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn-- -mcpu=verde -verify-machineinstrs < %s | FileCheck %s -check-prefixes=SI
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck %s -check-prefixes=VI
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=r600 -mtriple=r600-- -mcpu=redwood -verify-machineinstrs < %s | FileCheck %s -check-prefixes=EG
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=r600-- -mcpu=redwood -verify-machineinstrs < %s | FileCheck %s -check-prefixes=EG
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 
diff --git a/llvm/test/CodeGen/AMDGPU/store-global.ll b/llvm/test/CodeGen/AMDGPU/store-global.ll
index f068b1481aa9..1ff9b117237f 100644
--- a/llvm/test/CodeGen/AMDGPU/store-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-global.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -mtriple=amdgcn-- -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=VI -check-prefix=FUNC %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=FUNC %s
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=cayman -verify-machineinstrs < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
+; RUN: llc -mtriple=r600-- -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -mtriple=r600-- -mcpu=cayman -verify-machineinstrs < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}store_i1:
 ; EG: MEM_RAT MSKOR
diff --git a/llvm/test/CodeGen/AMDGPU/store-local.ll b/llvm/test/CodeGen/AMDGPU/store-local.ll
index 479f881cd40c..76e2d4366e3e 100644
--- a/llvm/test/CodeGen/AMDGPU/store-local.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-local.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -mtriple=amdgcn-- -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,VI,FUNC %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=cayman < %s | FileCheck -check-prefixes=CM,FUNC %s
+; RUN: llc -mtriple=r600-- -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
+; RUN: llc -mtriple=r600-- -mcpu=cayman < %s | FileCheck -check-prefixes=CM,FUNC %s
 
 ; FUNC-LABEL: {{^}}store_local_i1:
 ; SICIVI: s_mov_b32 m0
diff --git a/llvm/test/CodeGen/AMDGPU/trunc-vector-store-assertion-failure.ll b/llvm/test/CodeGen/AMDGPU/trunc-vector-store-assertion-failure.ll
index 8ccd3b9dd124..8de059d1d7b8 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc-vector-store-assertion-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc-vector-store-assertion-failure.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=r600 -mtriple=r600-- -mcpu=redwood | FileCheck %s
+; RUN: llc < %s -mtriple=r600-- -mcpu=redwood | FileCheck %s
 
 ; This tests for a bug in the SelectionDAG where custom lowered truncated
 ; vector stores at the end of a basic block were not being added to the
diff --git a/llvm/test/CodeGen/AMDGPU/unknown-processor.ll b/llvm/test/CodeGen/AMDGPU/unknown-processor.ll
index f1f1c92bcbed..9cfba8b2e5c0 100644
--- a/llvm/test/CodeGen/AMDGPU/unknown-processor.ll
+++ b/llvm/test/CodeGen/AMDGPU/unknown-processor.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=amdgcn-- -mcpu=unknown -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR -check-prefix=GCN %s
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=unknown -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR -check-prefix=R600 %s
+; RUN: llc -mtriple=r600-- -mcpu=unknown -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR -check-prefix=R600 %s
 target datalayout = "A5"
 
 ; Should not crash when the processor is not recognized and the
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll b/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll
index 694f444b7747..fc00937e6c8a 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-calls.ll
@@ -1,6 +1,6 @@
 ; RUN: not llc -mtriple=amdgcn-mesa-mesa3d -tailcallopt < %s 2>&1 | FileCheck --check-prefix=GCN %s
 ; RUN: not llc -mtriple=amdgcn--amdpal -tailcallopt < %s 2>&1 | FileCheck --check-prefix=GCN %s
-; RUN: not llc -mtriple=r600 -mtriple=r600-- -mcpu=cypress -tailcallopt < %s 2>&1 | FileCheck -check-prefix=R600 %s
+; RUN: not llc -mtriple=r600-- -mcpu=cypress -tailcallopt < %s 2>&1 | FileCheck -check-prefix=R600 %s
 
 declare i32 @external_function(i32) nounwind
 
diff --git a/llvm/test/CodeGen/AMDGPU/vector-alloca.ll b/llvm/test/CodeGen/AMDGPU/vector-alloca.ll
index 5ef794b64c0b..2c87680284e2 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-alloca.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-alloca.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -mtriple=amdgcn-- -mcpu=verde -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC %s
 ; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC %s
-; RUN: llc -mtriple=r600 -mtriple=r600-- -mcpu=redwood < %s | FileCheck --check-prefixes=EG,FUNC %s
+; RUN: llc -mtriple=r600-- -mcpu=redwood < %s | FileCheck --check-prefixes=EG,FUNC %s
 ; RUN: opt -S -mtriple=amdgcn-- -passes='amdgpu-promote-alloca,sroa,instcombine' < %s | FileCheck -check-prefix=OPT %s
 target datalayout = "A5"
 
diff --git a/llvm/test/CodeGen/AMDGPU/wrong-transalu-pos-fix.ll b/llvm/test/CodeGen/AMDGPU/wrong-transalu-pos-fix.ll
index bdfa89d9f304..6db7fe80c3cc 100644
--- a/llvm/test/CodeGen/AMDGPU/wrong-transalu-pos-fix.ll
+++ b/llvm/test/CodeGen/AMDGPU/wrong-transalu-pos-fix.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=r600 -mcpu=redwood -mtriple=r600-- < %s | FileCheck %s
+; RUN: llc -mtriple=r600-- -mcpu=redwood < %s | FileCheck %s
 
 ; We want all MULLO_INT inst to be last in their instruction group
 ;CHECK: {{^}}fill3d:
-- 
GitLab


From 643c38333fc1b1e7e705e6e1035c595bbd95bc74 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Thu, 9 May 2024 11:10:02 +0100
Subject: [PATCH 0267/1206] [AArch64] Remove EXT instr before UZP when
 extracting elements from vector (#91328)

Assembly generated for getting odd/even elements from vector contained
extra EXT instruction. This was due to way llvm constructs DAGs when
vector_shuffling from larger type to smaller. This patch optimises DAG
in these situations, allowing for correct assembly to be emitted.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    | 24 ++++++++++
 llvm/test/CodeGen/AArch64/aarch64-vuzp.ll     | 10 ++--
 ...complex-deinterleaving-f16-add-scalable.ll | 18 +++----
 ...complex-deinterleaving-f16-mul-scalable.ll | 29 ++++++-----
 .../AArch64/fixed-vector-deinterleave.ll      | 22 +++------
 llvm/test/CodeGen/AArch64/neon-perm.ll        |  7 +--
 .../AArch64/sve-vector-deinterleave.ll        | 48 +++++++++----------
 7 files changed, 87 insertions(+), 71 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c1ca78af5cda..7344387ffe55 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -21448,6 +21448,29 @@ static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG,
   SDValue Op1 = N->getOperand(1);
   EVT ResVT = N->getValueType(0);
 
+  // uzp(extract_lo(x), extract_hi(x)) -> extract_lo(uzp x, x)
+  if (Op0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+      Op1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+      Op0.getOperand(0) == Op1.getOperand(0)) {
+
+    SDValue SourceVec = Op0.getOperand(0);
+    uint64_t ExtIdx0 = Op0.getConstantOperandVal(1);
+    uint64_t ExtIdx1 = Op1.getConstantOperandVal(1);
+    uint64_t NumElements = SourceVec.getValueType().getVectorMinNumElements();
+    if (ExtIdx0 == 0 && ExtIdx1 == NumElements / 2) {
+      EVT OpVT = Op0.getOperand(1).getValueType();
+      EVT WidenedResVT = ResVT.getDoubleNumVectorElementsVT(*DAG.getContext());
+      SDValue Uzp = DAG.getNode(N->getOpcode(), DL, WidenedResVT, SourceVec,
+                                DAG.getUNDEF(WidenedResVT));
+      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT, Uzp,
+                         DAG.getConstant(0, DL, OpVT));
+    }
+  }
+
+  // Following optimizations only work with uzp1.
+  if (N->getOpcode() == AArch64ISD::UZP2)
+    return SDValue();
+
   // uzp1(x, undef) -> concat(truncate(x), undef)
   if (Op1.getOpcode() == ISD::UNDEF) {
     EVT BCVT = MVT::Other, HalfVT = MVT::Other;
@@ -24665,6 +24688,7 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
   case AArch64ISD::UUNPKHI:
     return performUnpackCombine(N, DAG, Subtarget);
   case AArch64ISD::UZP1:
+  case AArch64ISD::UZP2:
     return performUzpCombine(N, DAG, Subtarget);
   case AArch64ISD::SETCC_MERGE_ZERO:
     return performSetccMergeZeroCombine(N, DCI);
diff --git a/llvm/test/CodeGen/AArch64/aarch64-vuzp.ll b/llvm/test/CodeGen/AArch64/aarch64-vuzp.ll
index ba1ad9ba989c..1debf0256467 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-vuzp.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-vuzp.ll
@@ -3,18 +3,18 @@
 declare <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8>, <16 x i8>)
 
 ; CHECK-LABEL: fun1:
-; CHECK: uzp1 {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+; CHECK: uzp1 {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b
 define i32 @fun1() {
 entry:
   %vtbl1.i.1 = tail call <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8> <i8 0, i8 16, i8 19, i8 4, i8 -65, i8 -65, i8 -71, i8 -71, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i8> undef)
-  %vuzp.i212.1 = shufflevector <16 x i8> %vtbl1.i.1, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-  %scevgep = getelementptr <8 x i8>, ptr undef, i64 1
-  store <8 x i8> %vuzp.i212.1, ptr %scevgep, align 1
+  %vuzp.i212.1 = shufflevector <16 x i8> %vtbl1.i.1, <16 x i8> %vtbl1.i.1, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  %scevgep = getelementptr <16 x i8>, ptr undef, i64 1
+  store <16 x i8> %vuzp.i212.1, ptr %scevgep, align 1
   ret i32 undef
 }
 
 ; CHECK-LABEL: fun2:
-; CHECK: uzp2 {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+; CHECK: uzp2 {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b
 define i32 @fun2() {
 entry:
   %vtbl1.i.1 = tail call <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8> <i8 0, i8 16, i8 19, i8 4, i8 -65, i8 -65, i8 -71, i8 -71, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i8> undef)
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-add-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-add-scalable.ll
index dae8d9f89e99..c2fc959d8e10 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-add-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-add-scalable.ll
@@ -7,18 +7,18 @@ target triple = "aarch64"
 define <vscale x 4 x half> @complex_add_v4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
 ; CHECK-LABEL: complex_add_v4f16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uunpkhi z2.d, z0.s
+; CHECK-NEXT:    uzp1 z2.s, z0.s, z0.s
+; CHECK-NEXT:    uzp2 z0.s, z0.s, z0.s
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uzp2 z3.s, z1.s, z0.s
+; CHECK-NEXT:    uzp1 z1.s, z1.s, z0.s
+; CHECK-NEXT:    uunpklo z2.d, z2.s
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    uunpkhi z3.d, z1.s
+; CHECK-NEXT:    uunpklo z3.d, z3.s
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    uzp1 z4.d, z0.d, z2.d
-; CHECK-NEXT:    uzp2 z0.d, z0.d, z2.d
-; CHECK-NEXT:    uzp2 z2.d, z1.d, z3.d
-; CHECK-NEXT:    uzp1 z1.d, z1.d, z3.d
 ; CHECK-NEXT:    fsubr z0.h, p0/m, z0.h, z1.h
-; CHECK-NEXT:    movprfx z1, z2
-; CHECK-NEXT:    fadd z1.h, p0/m, z1.h, z4.h
+; CHECK-NEXT:    movprfx z1, z3
+; CHECK-NEXT:    fadd z1.h, p0/m, z1.h, z2.h
 ; CHECK-NEXT:    zip2 z2.d, z0.d, z1.d
 ; CHECK-NEXT:    zip1 z0.d, z0.d, z1.d
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z2.s
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll
index c09ec616b015..b42d484ea74c 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll
@@ -7,23 +7,22 @@ target triple = "aarch64"
 define <vscale x 4 x half> @complex_mul_v4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
 ; CHECK-LABEL: complex_mul_v4f16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uunpkhi z2.d, z0.s
+; CHECK-NEXT:    uzp2 z2.s, z0.s, z0.s
+; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uzp2 z3.s, z1.s, z0.s
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    uunpkhi z3.d, z1.s
+; CHECK-NEXT:    uunpklo z2.d, z2.s
+; CHECK-NEXT:    uunpklo z3.d, z3.s
+; CHECK-NEXT:    uzp1 z1.s, z1.s, z0.s
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    uzp2 z4.d, z0.d, z2.d
-; CHECK-NEXT:    uzp1 z0.d, z0.d, z2.d
-; CHECK-NEXT:    uzp2 z2.d, z1.d, z3.d
-; CHECK-NEXT:    uzp1 z1.d, z1.d, z3.d
-; CHECK-NEXT:    movprfx z5, z2
-; CHECK-NEXT:    fmul z5.h, p0/m, z5.h, z0.h
-; CHECK-NEXT:    fmul z2.h, p0/m, z2.h, z4.h
-; CHECK-NEXT:    movprfx z3, z5
-; CHECK-NEXT:    fmla z3.h, p0/m, z1.h, z4.h
-; CHECK-NEXT:    fnmsb z0.h, p0/m, z1.h, z2.h
-; CHECK-NEXT:    zip2 z1.d, z0.d, z3.d
-; CHECK-NEXT:    zip1 z0.d, z0.d, z3.d
+; CHECK-NEXT:    movprfx z4, z3
+; CHECK-NEXT:    fmul z4.h, p0/m, z4.h, z0.h
+; CHECK-NEXT:    fmul z3.h, p0/m, z3.h, z2.h
+; CHECK-NEXT:    fmad z2.h, p0/m, z1.h, z4.h
+; CHECK-NEXT:    fnmsb z0.h, p0/m, z1.h, z3.h
+; CHECK-NEXT:    zip2 z1.d, z0.d, z2.d
+; CHECK-NEXT:    zip1 z0.d, z0.d, z2.d
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z1.s
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
index c58db8290c87..5bd680ed4893 100644
--- a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
@@ -30,21 +30,13 @@ define {<2 x half>, <2 x half>} @vector_deinterleave_v2f16_v4f16(<4 x half> %vec
 }
 
 define {<4 x half>, <4 x half>} @vector_deinterleave_v4f16_v8f16(<8 x half> %vec) {
-; CHECK-SD-LABEL: vector_deinterleave_v4f16_v8f16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    uzp1 v2.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT:    uzp2 v1.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT:    fmov d0, d2
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: vector_deinterleave_v4f16_v8f16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    uzp1 v2.8h, v0.8h, v0.8h
-; CHECK-GI-NEXT:    uzp2 v1.8h, v0.8h, v0.8h
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-GI-NEXT:    fmov d0, d2
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: vector_deinterleave_v4f16_v8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uzp1 v2.8h, v0.8h, v0.8h
+; CHECK-NEXT:    uzp2 v1.8h, v0.8h, v0.8h
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    fmov d0, d2
+; CHECK-NEXT:    ret
   %retval = call {<4 x half>, <4 x half>} @llvm.vector.deinterleave2.v8f16(<8 x half> %vec)
   ret {<4 x half>, <4 x half>}   %retval
 }
diff --git a/llvm/test/CodeGen/AArch64/neon-perm.ll b/llvm/test/CodeGen/AArch64/neon-perm.ll
index 037451eb803c..15763543113e 100644
--- a/llvm/test/CodeGen/AArch64/neon-perm.ll
+++ b/llvm/test/CodeGen/AArch64/neon-perm.ll
@@ -4092,9 +4092,9 @@ entry:
 define %struct.uint8x8x2_t @test_uzp(<16 x i8> %y) {
 ; CHECK-SD-LABEL: test_uzp:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    uzp1 v2.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    uzp2 v1.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    xtn v2.8b, v0.8h
+; CHECK-SD-NEXT:    uzp2 v1.16b, v0.16b, v0.16b
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
 ; CHECK-SD-NEXT:    fmov d0, d2
 ; CHECK-SD-NEXT:    ret
 ;
@@ -4106,6 +4106,7 @@ define %struct.uint8x8x2_t @test_uzp(<16 x i8> %y) {
 ; CHECK-GI-NEXT:    fmov d0, d2
 ; CHECK-GI-NEXT:    ret
 
+
   %vuzp.i = shufflevector <16 x i8> %y, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %vuzp1.i = shufflevector <16 x i8> %y, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   %.fca.0.0.insert = insertvalue %struct.uint8x8x2_t undef, <8 x i8> %vuzp.i, 0, 0
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
index 478f4a689d3c..fd1365d56fee 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
@@ -4,10 +4,10 @@
 define {<vscale x 2 x half>, <vscale x 2 x half>} @vector_deinterleave_nxv2f16_nxv4f16(<vscale x 4 x half> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv2f16_nxv4f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpkhi z1.d, z0.s
-; CHECK-NEXT:    uunpklo z2.d, z0.s
-; CHECK-NEXT:    uzp1 z0.d, z2.d, z1.d
-; CHECK-NEXT:    uzp2 z1.d, z2.d, z1.d
+; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT:    uzp2 z2.s, z0.s, z0.s
+; CHECK-NEXT:    uunpklo z0.d, z1.s
+; CHECK-NEXT:    uunpklo z1.d, z2.s
 ; CHECK-NEXT:    ret
   %retval = call {<vscale x 2 x half>, <vscale x 2 x half>} @llvm.vector.deinterleave2.nxv4f16(<vscale x 4 x half> %vec)
   ret {<vscale x 2 x half>, <vscale x 2 x half>}   %retval
@@ -16,10 +16,10 @@ define {<vscale x 2 x half>, <vscale x 2 x half>} @vector_deinterleave_nxv2f16_n
 define {<vscale x 4 x half>, <vscale x 4 x half>} @vector_deinterleave_nxv4f16_nxv8f16(<vscale x 8 x half> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv4f16_nxv8f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpkhi z1.s, z0.h
-; CHECK-NEXT:    uunpklo z2.s, z0.h
-; CHECK-NEXT:    uzp1 z0.s, z2.s, z1.s
-; CHECK-NEXT:    uzp2 z1.s, z2.s, z1.s
+; CHECK-NEXT:    uzp1 z1.h, z0.h, z0.h
+; CHECK-NEXT:    uzp2 z2.h, z0.h, z0.h
+; CHECK-NEXT:    uunpklo z0.s, z1.h
+; CHECK-NEXT:    uunpklo z1.s, z2.h
 ; CHECK-NEXT:    ret
   %retval = call {<vscale x 4 x half>, <vscale x 4 x half>} @llvm.vector.deinterleave2.nxv8f16(<vscale x 8 x half> %vec)
   ret {<vscale x 4 x half>, <vscale x 4 x half>}   %retval
@@ -39,10 +39,10 @@ define {<vscale x 8 x half>, <vscale x 8 x half>} @vector_deinterleave_nxv8f16_n
 define {<vscale x 2 x float>, <vscale x 2 x float>} @vector_deinterleave_nxv2f32_nxv4f32(<vscale x 4 x float> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv2f32_nxv4f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpkhi z1.d, z0.s
-; CHECK-NEXT:    uunpklo z2.d, z0.s
-; CHECK-NEXT:    uzp1 z0.d, z2.d, z1.d
-; CHECK-NEXT:    uzp2 z1.d, z2.d, z1.d
+; CHECK-NEXT:    uzp1 z1.s, z0.s, z0.s
+; CHECK-NEXT:    uzp2 z2.s, z0.s, z0.s
+; CHECK-NEXT:    uunpklo z0.d, z1.s
+; CHECK-NEXT:    uunpklo z1.d, z2.s
 ; CHECK-NEXT:    ret
   %retval = call {<vscale x 2 x float>, <vscale x 2 x float>} @llvm.vector.deinterleave2.nxv4f32(<vscale x 4 x float> %vec)
   ret {<vscale x 2 x float>, <vscale x 2 x float>}   %retval
@@ -131,10 +131,10 @@ define {<vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_nxv16i1_nxv
 define {<vscale x 8 x i1>, <vscale x 8 x i1>} @vector_deinterleave_nxv8i1_nxv16i1(<vscale x 16 x i1> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv8i1_nxv16i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    punpkhi p1.h, p0.b
-; CHECK-NEXT:    punpklo p2.h, p0.b
-; CHECK-NEXT:    uzp1 p0.h, p2.h, p1.h
-; CHECK-NEXT:    uzp2 p1.h, p2.h, p1.h
+; CHECK-NEXT:    uzp1 p1.b, p0.b, p0.b
+; CHECK-NEXT:    uzp2 p2.b, p0.b, p0.b
+; CHECK-NEXT:    punpklo p0.h, p1.b
+; CHECK-NEXT:    punpklo p1.h, p2.b
 ; CHECK-NEXT:    ret
   %retval = call {<vscale x 8 x i1>, <vscale x 8 x i1>} @llvm.vector.deinterleave2.nxv16i1(<vscale x 16 x i1> %vec)
   ret {<vscale x 8 x i1>, <vscale x 8 x i1>}   %retval
@@ -143,10 +143,10 @@ define {<vscale x 8 x i1>, <vscale x 8 x i1>} @vector_deinterleave_nxv8i1_nxv16i
 define {<vscale x 4 x i1>, <vscale x 4 x i1>} @vector_deinterleave_nxv4i1_nxv8i1(<vscale x 8 x i1> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv4i1_nxv8i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    punpkhi p1.h, p0.b
-; CHECK-NEXT:    punpklo p2.h, p0.b
-; CHECK-NEXT:    uzp1 p0.s, p2.s, p1.s
-; CHECK-NEXT:    uzp2 p1.s, p2.s, p1.s
+; CHECK-NEXT:    uzp1 p1.h, p0.h, p0.h
+; CHECK-NEXT:    uzp2 p2.h, p0.h, p0.h
+; CHECK-NEXT:    punpklo p0.h, p1.b
+; CHECK-NEXT:    punpklo p1.h, p2.b
 ; CHECK-NEXT:    ret
   %retval = call {<vscale x 4 x i1>, <vscale x 4 x i1>} @llvm.vector.deinterleave2.nxv8i1(<vscale x 8 x i1> %vec)
   ret {<vscale x 4 x i1>, <vscale x 4 x i1>}   %retval
@@ -155,10 +155,10 @@ define {<vscale x 4 x i1>, <vscale x 4 x i1>} @vector_deinterleave_nxv4i1_nxv8i1
 define {<vscale x 2 x i1>, <vscale x 2 x i1>} @vector_deinterleave_nxv2i1_nxv4i1(<vscale x 4 x i1> %vec) {
 ; CHECK-LABEL: vector_deinterleave_nxv2i1_nxv4i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    punpkhi p1.h, p0.b
-; CHECK-NEXT:    punpklo p2.h, p0.b
-; CHECK-NEXT:    uzp1 p0.d, p2.d, p1.d
-; CHECK-NEXT:    uzp2 p1.d, p2.d, p1.d
+; CHECK-NEXT:    uzp1 p1.s, p0.s, p0.s
+; CHECK-NEXT:    uzp2 p2.s, p0.s, p0.s
+; CHECK-NEXT:    punpklo p0.h, p1.b
+; CHECK-NEXT:    punpklo p1.h, p2.b
 ; CHECK-NEXT:    ret
   %retval = call {<vscale x 2 x i1>, <vscale x 2 x i1>} @llvm.vector.deinterleave2.nxv4i1(<vscale x 4 x i1> %vec)
   ret {<vscale x 2 x i1>, <vscale x 2 x i1>}   %retval
-- 
GitLab


From 1494d8849fa2aef575dabd431b0060639f4a57c1 Mon Sep 17 00:00:00 2001
From: jofrn <jofernau@amd.com>
Date: Thu, 9 May 2024 06:17:01 -0400
Subject: [PATCH 0268/1206] [AMDGPU] Always Inline preserved analyses (#91198)

When replacing all uses, the structural-hash of the IR can change, so
keep track of changes using Changed variable and return it to pass
manager.
---
 llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
index b53def912ab6..f55f656ff922 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
@@ -42,7 +42,7 @@ public:
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesAll();
- }
+  }
 };
 
 } // End anonymous namespace
@@ -89,6 +89,7 @@ recursivelyVisitUsers(GlobalValue &GV,
 static bool alwaysInlineImpl(Module &M, bool GlobalOpt) {
   std::vector<GlobalAlias*> AliasesToRemove;
 
+  bool Changed = false;
   SmallPtrSet<Function *, 8> FuncsToAlwaysInline;
   SmallPtrSet<Function *, 8> FuncsToNoInline;
   Triple TT(M.getTargetTriple());
@@ -98,6 +99,7 @@ static bool alwaysInlineImpl(Module &M, bool GlobalOpt) {
       if (TT.getArch() == Triple::amdgcn &&
           A.getLinkage() != GlobalValue::InternalLinkage)
         continue;
+      Changed = true;
       A.replaceAllUsesWith(F);
       AliasesToRemove.push_back(&A);
     }
@@ -153,7 +155,7 @@ static bool alwaysInlineImpl(Module &M, bool GlobalOpt) {
   for (Function *F : FuncsToNoInline)
     F->addFnAttr(Attribute::NoInline);
 
-  return !FuncsToAlwaysInline.empty() || !FuncsToNoInline.empty();
+  return Changed || !FuncsToAlwaysInline.empty() || !FuncsToNoInline.empty();
 }
 
 bool AMDGPUAlwaysInline::runOnModule(Module &M) {
@@ -166,6 +168,6 @@ ModulePass *llvm::createAMDGPUAlwaysInlinePass(bool GlobalOpt) {
 
 PreservedAnalyses AMDGPUAlwaysInlinePass::run(Module &M,
                                               ModuleAnalysisManager &AM) {
-  alwaysInlineImpl(M, GlobalOpt);
-  return PreservedAnalyses::all();
+  const bool Changed = alwaysInlineImpl(M, GlobalOpt);
+  return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
 }
-- 
GitLab


From 6eb9e214b3cb06fc31f2547275e746761c3e41de Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Thu, 9 May 2024 11:37:28 +0100
Subject: [PATCH 0269/1206] RFC: [AMDGPU] Check subtarget features for
 consistency (#86957)

Implement GCNSubtarget::checkSubtargetFeatures as a canonical place to
check subtarget features for consistency and diagnose any
inconsistencies. To start with, the implementation just checks that
either wavefrontsize32 or wavefrontsize64 is selected.

checkSubtargetFeatures is called at the start of instruction selection.
This is pretty arbitrary. It is just a convenient point at which we have
access to the subtarget that we're going to use for codegenning a
particular function.
---
 llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp          |  1 +
 llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp   |  1 +
 llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp             | 10 ++++++++++
 llvm/lib/Target/AMDGPU/GCNSubtarget.h                  |  4 ++++
 llvm/test/CodeGen/AMDGPU/check-subtarget-features.ll   | 10 ++++++++++
 .../AMDGPU/remove-incompatible-wave32-feature.ll       |  8 ++++----
 llvm/test/CodeGen/AMDGPU/unknown-processor.ll          |  2 +-
 7 files changed, 31 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/check-subtarget-features.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index bba7682cd7a0..c11c7a57e059 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -132,6 +132,7 @@ bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
   }
 #endif
   Subtarget = &MF.getSubtarget<GCNSubtarget>();
+  Subtarget->checkSubtargetFeatures(MF.getFunction());
   Mode = SIModeRegisterDefaults(MF.getFunction(), *Subtarget);
   return SelectionDAGISel::runOnMachineFunction(MF);
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index e13c13913d4e..b48a09489653 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -63,6 +63,7 @@ void AMDGPUInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB,
                                         BlockFrequencyInfo *BFI) {
   MRI = &MF.getRegInfo();
   Subtarget = &MF.getSubtarget<GCNSubtarget>();
+  Subtarget->checkSubtargetFeatures(MF.getFunction());
   InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 36e453f04426..9cfe81e5288e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -25,6 +25,7 @@
 #include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
 #include "llvm/CodeGen/MachineScheduler.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
+#include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/IntrinsicsR600.h"
 #include "llvm/IR/MDBuilder.h"
@@ -165,6 +166,15 @@ GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
   return *this;
 }
 
+void GCNSubtarget::checkSubtargetFeatures(const Function &F) const {
+  LLVMContext &Ctx = F.getContext();
+  if (hasFeature(AMDGPU::FeatureWavefrontSize32) ==
+      hasFeature(AMDGPU::FeatureWavefrontSize64)) {
+    Ctx.diagnose(DiagnosticInfoUnsupported(
+        F, "must specify exactly one of wavefrontsize32 and wavefrontsize64"));
+  }
+}
+
 AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT) : TargetTriple(TT) {}
 
 bool AMDGPUSubtarget::useRealTrue16Insts() const {
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index be337e0b2192..b7548671f2c5 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -250,6 +250,10 @@ public:
   GCNSubtarget &initializeSubtargetDependencies(const Triple &TT,
                                                    StringRef GPU, StringRef FS);
 
+  /// Diagnose inconsistent subtarget features before attempting to codegen
+  /// function \p F.
+  void checkSubtargetFeatures(const Function &F) const;
+
   const SIInstrInfo *getInstrInfo() const override {
     return &InstrInfo;
   }
diff --git a/llvm/test/CodeGen/AMDGPU/check-subtarget-features.ll b/llvm/test/CodeGen/AMDGPU/check-subtarget-features.ll
new file mode 100644
index 000000000000..c24693981104
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/check-subtarget-features.ll
@@ -0,0 +1,10 @@
+; RUN: not llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,-wavefrontsize64 < %s 2>&1 | FileCheck %s -check-prefix=ERR -implicit-check-not=error:
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,-wavefrontsize64 < %s 2>&1 | FileCheck %s -check-prefix=ERR -implicit-check-not=error:
+; RUN: not llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+wavefrontsize64 < %s 2>&1 | FileCheck %s -check-prefix=ERR -implicit-check-not=error:
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,+wavefrontsize64 < %s 2>&1 | FileCheck %s -check-prefix=ERR -implicit-check-not=error:
+
+; ERR: error: {{.*}} in function f void (): must specify exactly one of wavefrontsize32 and wavefrontsize64
+
+define void @f() {
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/remove-incompatible-wave32-feature.ll b/llvm/test/CodeGen/AMDGPU/remove-incompatible-wave32-feature.ll
index 8ef1d3ff27e5..406c953a06d9 100644
--- a/llvm/test/CodeGen/AMDGPU/remove-incompatible-wave32-feature.ll
+++ b/llvm/test/CodeGen/AMDGPU/remove-incompatible-wave32-feature.ll
@@ -8,13 +8,13 @@
 ; RUN: FileCheck --check-prefix=WARN-GFX90A %s < %t
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -mattr=+wavefrontsize64 -verify-machineinstrs < %s
 
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -mattr=+wavefrontsize64 -stop-after=amdgpu-remove-incompatible-functions\
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -mattr=-wavefrontsize32,+wavefrontsize64 -stop-after=amdgpu-remove-incompatible-functions\
 ; RUN:   -pass-remarks=amdgpu-remove-incompatible-functions < %s 2>%t | FileCheck -check-prefixes=GFX10 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -mattr=+wavefrontsize64 -verify-machineinstrs < %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s
 
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -stop-after=amdgpu-remove-incompatible-functions\
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,+wavefrontsize64 -stop-after=amdgpu-remove-incompatible-functions\
 ; RUN:   -pass-remarks=amdgpu-remove-incompatible-functions < %s 2>%t | FileCheck -check-prefixes=GFX11 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -verify-machineinstrs < %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s
 
 ; WARN-GFX906: removing function 'needs_wavefrontsize32': +wavefrontsize32 is not supported on the current target
 ; WARN-GFX906-NOT: not supported
diff --git a/llvm/test/CodeGen/AMDGPU/unknown-processor.ll b/llvm/test/CodeGen/AMDGPU/unknown-processor.ll
index 9cfba8b2e5c0..683ba98e52cf 100644
--- a/llvm/test/CodeGen/AMDGPU/unknown-processor.ll
+++ b/llvm/test/CodeGen/AMDGPU/unknown-processor.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-- -mcpu=unknown -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR -check-prefix=GCN %s
+; RUN: not llc -mtriple=amdgcn-- -mcpu=unknown -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR -check-prefix=GCN %s
 ; RUN: llc -mtriple=r600-- -mcpu=unknown -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR -check-prefix=R600 %s
 target datalayout = "A5"
 
-- 
GitLab


From 58a94b1d0ad8df85bc6b1edb22c74ffb718ca1a1 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Wed, 8 May 2024 06:53:12 -0700
Subject: [PATCH 0270/1206] [SLP]Fix PR91467: Look through scalar cast, when
 trying to cast to another type.

Need to look through the SExt/ZExt scalars to be gathered, when trying
to reduce their width after minbitwidth analysis to prevent permanent
attempts to revectorize such gathered instructions.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 10 ++-
 .../AArch64/gather-with-minbith-user.ll       |  9 +--
 .../AArch64/user-node-not-in-bitwidths.ll     |  7 +-
 .../SystemZ/minbitwidth-root-trunc.ll         |  3 +-
 .../X86/extended-vectorized-gathered-inst.ll  | 65 +++++++++++++++++++
 5 files changed, 77 insertions(+), 17 deletions(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/extended-vectorized-gathered-inst.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 98561f9ca044..2e0a39c4b4fd 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11419,8 +11419,16 @@ Value *BoUpSLP::gather(ArrayRef<Value *> VL, Value *Root, Type *ScalarTy) {
     if (Scalar->getType() != Ty) {
       assert(Scalar->getType()->isIntegerTy() && Ty->isIntegerTy() &&
              "Expected integer types only.");
+      Value *V = Scalar;
+      if (auto *CI = dyn_cast<CastInst>(Scalar);
+          isa_and_nonnull<SExtInst, ZExtInst>(CI)) {
+        Value *Op = CI->getOperand(0);
+        if (auto *IOp = dyn_cast<Instruction>(Op);
+            !IOp || !(isDeleted(IOp) || getTreeEntry(IOp)))
+          V = Op;
+      }
       Scalar = Builder.CreateIntCast(
-          Scalar, Ty, !isKnownNonNegative(Scalar, SimplifyQuery(*DL)));
+          V, Ty, !isKnownNonNegative(Scalar, SimplifyQuery(*DL)));
     }
 
     Vec = Builder.CreateInsertElement(Vec, Scalar, Builder.getInt32(Pos));
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll
index 76bb882171b1..3ebe920d1734 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-with-minbith-user.ll
@@ -5,14 +5,7 @@ define void @h() {
 ; CHECK-LABEL: define void @h() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, ptr null, i64 16
-; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 0 to i1
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 poison, i1 false, i1 false, i1 false>, i1 [[TMP6]], i32 4
-; CHECK-NEXT:    [[TMP1:%.*]] = sub <8 x i1> [[TMP0]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = add <8 x i1> [[TMP0]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP5:%.*]] = or <8 x i1> [[TMP3]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i1> [[TMP5]] to <8 x i16>
-; CHECK-NEXT:    store <8 x i16> [[TMP4]], ptr [[ARRAYIDX2]], align 2
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr [[ARRAYIDX2]], align 2
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll
index 2ab6e919c23b..6404cf4a2cd1 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/user-node-not-in-bitwidths.ll
@@ -5,12 +5,7 @@ define void @h() {
 ; CHECK-LABEL: define void @h() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, ptr null, i64 16
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 0 to i1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 poison, i1 false, i1 false, i1 false>, i1 [[TMP0]], i32 4
-; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i1> zeroinitializer, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i1> zeroinitializer, [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i1> [[TMP3]] to <8 x i16>
-; CHECK-NEXT:    store <8 x i16> [[TMP4]], ptr [[ARRAYIDX2]], align 2
+; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr [[ARRAYIDX2]], align 2
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll
index 1bb87bf6205f..3c8e98485ffc 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll
@@ -4,10 +4,9 @@
 define void @test(ptr %a, i8 %0, i16 %b.promoted.i) {
 ; CHECK-LABEL: define void @test(
 ; CHECK-SAME: ptr [[A:%.*]], i8 [[TMP0:%.*]], i16 [[B_PROMOTED_I:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP0]] to i128
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i16> poison, i16 [[B_PROMOTED_I]], i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i128 [[TMP2]] to i16
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i8 [[TMP0]] to i16
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i16> poison, i16 [[TMP5]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[TMP6]], <4 x i16> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i16> [[TMP4]], [[TMP7]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extended-vectorized-gathered-inst.ll b/llvm/test/Transforms/SLPVectorizer/X86/extended-vectorized-gathered-inst.ll
new file mode 100644
index 000000000000..2d028060f491
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extended-vectorized-gathered-inst.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s | FileCheck %s
+
+define void @test(ptr %top) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[TOP:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[TOP]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[TMP0]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i8> [[TMP0]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[TMP3]] to i8
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i8> <i8 0, i8 0, i8 0, i8 poison>, i8 [[TMP4]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i8> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <4 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr <4 x i8> [[TMP7]], <i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEXT:    br label [[FOR_COND_I:%.*]]
+; CHECK:       for.cond.i:
+; CHECK-NEXT:    store <4 x i8> [[TMP8]], ptr null, align 1
+; CHECK-NEXT:    br label [[FOR_COND_I]]
+;
+entry:
+  %0 = load i8, ptr %top, align 1
+  %conv2.i = zext i8 %0 to i32
+  %mul.i = mul i32 %conv2.i, 0
+  %add.i = or i32 %mul.i, 0
+  %arrayidx3.i = getelementptr i8, ptr %top, i64 1
+  %1 = load i8, ptr %arrayidx3.i, align 1
+  %conv4.i = zext i8 %1 to i32
+  %add5.i = or i32 %add.i, 0
+  %shr.i = lshr i32 %add5.i, 2
+  %conv7.i = trunc i32 %shr.i to i8
+  %mul12.i = mul i32 %conv4.i, 0
+  %arrayidx14.i = getelementptr i8, ptr %top, i64 2
+  %2 = load i8, ptr %arrayidx14.i, align 1
+  %conv15.i = zext i8 %2 to i32
+  %add16.i = or i32 %mul12.i, 0
+  %add17.i = or i32 %add16.i, 0
+  %shr18.i = lshr i32 %add17.i, 2
+  %conv19.i = trunc i32 %shr18.i to i8
+  %mul25.i = mul i32 %conv15.i, 0
+  %arrayidx27.i = getelementptr i8, ptr %top, i64 3
+  %3 = load i8, ptr %arrayidx27.i, align 1
+  %conv28.i = zext i8 %3 to i32
+  %add29.i = or i32 %mul25.i, 0
+  %add30.i = or i32 %add29.i, 0
+  %shr31.i = lshr i32 %add30.i, 2
+  %conv32.i = trunc i32 %shr31.i to i8
+  %mul38.i = mul i32 %conv28.i, 0
+  %add39.i = or i32 %mul38.i, %conv15.i
+  %add42.i = or i32 %add39.i, 0
+  %shr44.i = lshr i32 %add42.i, 2
+  %conv45.i = trunc i32 %shr44.i to i8
+  br label %for.cond.i
+
+for.cond.i:
+  store i8 %conv7.i, ptr null, align 1
+  %vals.sroa.5.0.add.ptr.sroa_idx.i = getelementptr i8, ptr null, i64 1
+  store i8 %conv19.i, ptr %vals.sroa.5.0.add.ptr.sroa_idx.i, align 1
+  %vals.sroa.7.0.add.ptr.sroa_idx.i = getelementptr i8, ptr null, i64 2
+  store i8 %conv32.i, ptr %vals.sroa.7.0.add.ptr.sroa_idx.i, align 1
+  %vals.sroa.9.0.add.ptr.sroa_idx.i = getelementptr i8, ptr null, i64 3
+  store i8 %conv45.i, ptr %vals.sroa.9.0.add.ptr.sroa_idx.i, align 1
+  br label %for.cond.i
+}
-- 
GitLab


From aa16de6399a42421076ed642c3b4f7fb12c6d44b Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 9 May 2024 06:35:18 -0500
Subject: [PATCH 0271/1206] [Linker] Propagate `nobuiltin` attributes when
 linking known libcalls (#89431)

Summary:
As discussed in
https://discourse.llvm.org/t/rfc-libc-ffreestanding-fno-builtin.

LLVM ascribes special semantics to several functions that are known to
be `libcalls`. These are functions that middle-end optimizations may
transforms calls into or perform optimizations based off of known
semantics. However, these assumptions require an opaque function call to
be known valid. In situations like LTO or IR linking it is possible to
bring a libcall definition into the current module. Once this happens,
we can no longer make any guarantees about the semantics of these
functions.

We currently attempt to solve this by preventing all inlining if the
called function has `no-builtin` https://reviews.llvm.org/D74162.
However, this is overly pessimistic as it prevents all inlining even for
non-libcall functions.

This patch modifies the IRMover class to track known libcalls enabled
for the given target. If we encounter a known libcall during IR linking,
we then need to append the `nobuiltin` attribute to the destination
module. Afterwards, all new definitions we link in will be applied as
well.
---
 llvm/include/llvm/Linker/IRMover.h | 33 ++++++++++++++-
 llvm/lib/Linker/CMakeLists.txt     |  1 +
 llvm/lib/Linker/IRMover.cpp        | 67 ++++++++++++++++++++++++++++--
 llvm/test/Linker/Inputs/strlen.ll  | 21 ++++++++++
 llvm/test/Linker/libcalls.ll       | 39 +++++++++++++++++
 5 files changed, 156 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/Linker/Inputs/strlen.ll
 create mode 100644 llvm/test/Linker/libcalls.ll

diff --git a/llvm/include/llvm/Linker/IRMover.h b/llvm/include/llvm/Linker/IRMover.h
index 1e3c5394ffa2..8e71c6080dff 100644
--- a/llvm/include/llvm/Linker/IRMover.h
+++ b/llvm/include/llvm/Linker/IRMover.h
@@ -12,11 +12,14 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/FunctionExtras.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/Support/StringSaver.h"
+#include "llvm/TargetParser/Triple.h"
 #include <functional>
 
 namespace llvm {
 class Error;
-class GlobalValue;
 class Metadata;
 class Module;
 class StructType;
@@ -60,6 +63,33 @@ public:
     bool hasType(StructType *Ty);
   };
 
+  /// Utility for handling linking of known libcall functions. If a merged
+  /// module contains a recognized library call we can no longer perform any
+  /// libcall related transformations.
+  class LibcallHandler {
+    bool HasLibcalls = false;
+
+    StringSet<> Libcalls;
+    StringSet<> Triples;
+
+    BumpPtrAllocator Alloc;
+    StringSaver Saver;
+
+  public:
+    LibcallHandler() : Saver(Alloc) {}
+
+    void updateLibcalls(const Triple &TheTriple);
+
+    bool checkLibcalls(GlobalValue &GV) {
+      if (HasLibcalls)
+        return false;
+      return HasLibcalls = isa<Function>(&GV) && !GV.isDeclaration() &&
+                           Libcalls.count(GV.getName());
+    }
+
+    bool hasLibcalls() const { return HasLibcalls; }
+  };
+
   IRMover(Module &M);
 
   typedef std::function<void(GlobalValue &)> ValueAdder;
@@ -84,6 +114,7 @@ private:
   Module &Composite;
   IdentifiedStructTypeSet IdentifiedStructTypes;
   MDMapT SharedMDs; ///< A Metadata map to use for all calls to \a move().
+  LibcallHandler Libcalls;
 };
 
 } // End llvm namespace
diff --git a/llvm/lib/Linker/CMakeLists.txt b/llvm/lib/Linker/CMakeLists.txt
index 5afb40f8b588..25001c09a62d 100644
--- a/llvm/lib/Linker/CMakeLists.txt
+++ b/llvm/lib/Linker/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_component_library(LLVMLinker
   intrinsics_gen
 
   LINK_COMPONENTS
+  Analysis
   Core
   Object
   Support
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index 7a5aa0c80478..fe2b53183589 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -12,6 +12,7 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/AutoUpgrade.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfoMetadata.h"
@@ -399,6 +400,9 @@ class IRLinker {
   /// A metadata map that's shared between IRLinker instances.
   MDMapT &SharedMDs;
 
+  /// A list of libcalls that the current target may call.
+  IRMover::LibcallHandler &Libcalls;
+
   /// Mapping of values from what they used to be in Src, to what they are now
   /// in DstM.  ValueToValueMapTy is a ValueMap, which involves some overhead
   /// due to the use of Value handles which the Linker doesn't actually need,
@@ -540,10 +544,12 @@ public:
   IRLinker(Module &DstM, MDMapT &SharedMDs,
            IRMover::IdentifiedStructTypeSet &Set, std::unique_ptr<Module> SrcM,
            ArrayRef<GlobalValue *> ValuesToLink,
-           IRMover::LazyCallback AddLazyFor, bool IsPerformingImport)
+           IRMover::LibcallHandler &Libcalls, IRMover::LazyCallback AddLazyFor,
+           bool IsPerformingImport)
       : DstM(DstM), SrcM(std::move(SrcM)), AddLazyFor(std::move(AddLazyFor)),
         TypeMap(Set), GValMaterializer(*this), LValMaterializer(*this),
-        SharedMDs(SharedMDs), IsPerformingImport(IsPerformingImport),
+        SharedMDs(SharedMDs), Libcalls(Libcalls),
+        IsPerformingImport(IsPerformingImport),
         Mapper(ValueMap, RF_ReuseAndMutateDistinctMDs | RF_IgnoreMissingLocals,
                &TypeMap, &GValMaterializer),
         IndirectSymbolMCID(Mapper.registerAlternateMappingContext(
@@ -561,6 +567,13 @@ public:
 };
 }
 
+static void addNoBuiltinAttributes(Function &F) {
+  F.setAttributes(
+      F.getAttributes().addFnAttribute(F.getContext(), "no-builtins"));
+  F.setAttributes(
+      F.getAttributes().addFnAttribute(F.getContext(), Attribute::NoBuiltin));
+}
+
 /// The LLVM SymbolTable class autorenames globals that conflict in the symbol
 /// table. This is good for all clients except for us. Go through the trouble
 /// to force this back.
@@ -1605,14 +1618,26 @@ Error IRLinker::run() {
 
   DstM.setTargetTriple(SrcTriple.merge(DstTriple));
 
+  // Update the target triple's libcall information if it was changed.
+  Libcalls.updateLibcalls(Triple(DstM.getTargetTriple()));
+
   // Loop over all of the linked values to compute type mappings.
   computeTypeMapping();
 
+  bool AddsLibcalls = false;
   std::reverse(Worklist.begin(), Worklist.end());
   while (!Worklist.empty()) {
     GlobalValue *GV = Worklist.back();
     Worklist.pop_back();
 
+    // If the module already contains libcall functions we need every function
+    // linked in to have `nobuiltin` attributes. Otherwise check if this is a
+    // libcall definition.
+    if (Function *F = dyn_cast<Function>(GV); F && Libcalls.hasLibcalls())
+      addNoBuiltinAttributes(*F);
+    else
+      AddsLibcalls = Libcalls.checkLibcalls(*GV);
+
     // Already mapped.
     if (ValueMap.find(GV) != ValueMap.end() ||
         IndirectSymbolValueMap.find(GV) != IndirectSymbolValueMap.end())
@@ -1675,6 +1700,13 @@ Error IRLinker::run() {
     }
   }
 
+  // If we have imported a recognized libcall function we can no longer make any
+  // reasonable optimizations based off of its semantics. Add the 'nobuiltin'
+  // attribute to every function to suppress libcall detection.
+  if (AddsLibcalls)
+    for (Function &F : DstM.functions())
+      addNoBuiltinAttributes(F);
+
   // Merge the module flags into the DstM module.
   return linkModuleFlagsMetadata();
 }
@@ -1757,6 +1789,22 @@ bool IRMover::IdentifiedStructTypeSet::hasType(StructType *Ty) {
   return I == NonOpaqueStructTypes.end() ? false : *I == Ty;
 }
 
+void IRMover::LibcallHandler::updateLibcalls(const Triple &TheTriple) {
+  if (Triples.count(TheTriple.getTriple()))
+    return;
+  Triples.insert(Saver.save(TheTriple.getTriple()));
+
+  // Collect the names of runtime functions that the target may want to call.
+  TargetLibraryInfoImpl TLII(TheTriple);
+  TargetLibraryInfo TLI(TLII);
+  for (unsigned I = 0, E = static_cast<unsigned>(LibFunc::NumLibFuncs); I != E;
+       ++I) {
+    LibFunc F = static_cast<LibFunc>(I);
+    if (TLI.has(F))
+      Libcalls.insert(TLI.getName(F));
+  }
+}
+
 IRMover::IRMover(Module &M) : Composite(M) {
   TypeFinder StructTypes;
   StructTypes.run(M, /* OnlyNamed */ false);
@@ -1772,14 +1820,25 @@ IRMover::IRMover(Module &M) : Composite(M) {
   for (const auto *MD : StructTypes.getVisitedMetadata()) {
     SharedMDs[MD].reset(const_cast<MDNode *>(MD));
   }
+
+  // Check the composite module for any already present libcalls. If we define
+  // these then it is important to mark any imported functions as 'nobuiltin'.
+  Libcalls.updateLibcalls(Triple(Composite.getTargetTriple()));
+  for (Function &F : Composite.functions())
+    if (Libcalls.checkLibcalls(F))
+      break;
+
+  if (Libcalls.hasLibcalls())
+    for (Function &F : Composite.functions())
+      addNoBuiltinAttributes(F);
 }
 
 Error IRMover::move(std::unique_ptr<Module> Src,
                     ArrayRef<GlobalValue *> ValuesToLink,
                     LazyCallback AddLazyFor, bool IsPerformingImport) {
   IRLinker TheIRLinker(Composite, SharedMDs, IdentifiedStructTypes,
-                       std::move(Src), ValuesToLink, std::move(AddLazyFor),
-                       IsPerformingImport);
+                       std::move(Src), ValuesToLink, Libcalls,
+                       std::move(AddLazyFor), IsPerformingImport);
   Error E = TheIRLinker.run();
   Composite.dropTriviallyDeadConstantArrays();
   return E;
diff --git a/llvm/test/Linker/Inputs/strlen.ll b/llvm/test/Linker/Inputs/strlen.ll
new file mode 100644
index 000000000000..bc54aaf41e0c
--- /dev/null
+++ b/llvm/test/Linker/Inputs/strlen.ll
@@ -0,0 +1,21 @@
+target triple = "x86_64-unknown-linux-gnu"
+
+define i64 @strlen(ptr %s) #0 {
+entry:
+  br label %for.cond
+
+for.cond:
+  %s.addr.0 = phi ptr [ %s, %entry ], [ %incdec.ptr, %for.cond ]
+  %0 = load i8, ptr %s.addr.0, align 1
+  %tobool.not = icmp eq i8 %0, 0
+  %incdec.ptr = getelementptr inbounds i8, ptr %s.addr.0, i64 1
+  br i1 %tobool.not, label %for.end, label %for.cond
+
+for.end:
+  %sub.ptr.lhs.cast = ptrtoint ptr %s.addr.0 to i64
+  %sub.ptr.rhs.cast = ptrtoint ptr %s to i64
+  %sub.ptr.sub = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
+  ret i64 %sub.ptr.sub
+}
+
+attributes #0 = { noinline }
diff --git a/llvm/test/Linker/libcalls.ll b/llvm/test/Linker/libcalls.ll
new file mode 100644
index 000000000000..ddc0d35e91d9
--- /dev/null
+++ b/llvm/test/Linker/libcalls.ll
@@ -0,0 +1,39 @@
+; RUN: llvm-link %s %S/Inputs/strlen.ll -S -o - 2>%t.a.err | FileCheck %s --check-prefix=CHECK1
+; RUN: llvm-link %S/Inputs/strlen.ll %s -S -o - 2>%t.a.err | FileCheck %s --check-prefix=CHECK2
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [7 x i8] c"string\00", align 1
+@str = dso_local global ptr @.str, align 8
+
+define void @foo() #0 {
+  ret void
+}
+
+declare i64 @strlen(ptr)
+
+define void @bar() #0 {
+  ret void
+}
+
+define i64 @baz() #0 {
+entry:
+  %0 = load ptr, ptr @str, align 8
+  %call = call i64 @strlen(ptr noundef %0)
+  ret i64 %call
+}
+
+attributes #0 = { noinline }
+
+; CHECK1: define void @foo() #[[ATTR0:[0-9]+]]
+; CHECK1: define void @bar() #[[ATTR0:[0-9]+]]
+; CHECK1: define i64 @baz() #[[ATTR0:[0-9]+]]
+; CHECK1: define i64 @strlen(ptr [[S:%.*]]) #[[ATTR0]]
+
+; CHECK2: define i64 @strlen(ptr [[S:%.*]]) #[[ATTR0:[0-9]+]]
+; CHECK2: define void @foo() #[[ATTR0:[0-9]+]]
+; CHECK2: define void @bar() #[[ATTR0:[0-9]+]]
+; CHECK2: define i64 @baz() #[[ATTR0]]
+
+; CHECK1: attributes #[[ATTR0]] = { nobuiltin noinline "no-builtins" }
+; CHECK2: attributes #[[ATTR0]] = { nobuiltin noinline "no-builtins" }
-- 
GitLab


From 80f9e814ec896fdc57ee84afad8ac4cb1f8e4627 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 9 May 2024 06:35:54 -0500
Subject: [PATCH 0272/1206] [Libomptarget] Statically link all plugin runtimes
 (#87009)

This patch overhauls the `libomptarget` and plugin interface. Currently,
we define a C API and compile each plugin as a separate shared library.
Then, `libomptarget` loads these API functions and forwards its internal
calls to them. This was originally designed to allow multiple
implementations of a library to be live. However, since then no one has
used this functionality and it prevents us from using much nicer
interfaces. If the old behavior is desired it should instead be
implemented as a separate plugin.

This patch replaces the `PluginAdaptorTy` interface with the
`GenericPluginTy` that is used by the plugins. Each plugin exports a
`createPlugin_<name>` function that is used to get the specific
implementation. This code is now shared with `libomptarget`.

There are some notable improvements to this.
1. Massively improved lifetimes of life runtime objects
2. The plugins can use a C++ interface
3. Global state does not need to be duplicated for each plugin +
   libomptarget
4. Easier to use and add features and improve error handling
5. Less function call overhead / Improved LTO performance.

Additional changes in this plugin are related to contending with the
fact that state is now shared. Initialization and deinitialization is
now handled correctly and in phase with the underlying runtime, allowing
us to actually know when something is getting deallocated.

Depends on https://github.com/llvm/llvm-project/pull/86971
https://github.com/llvm/llvm-project/pull/86875
https://github.com/llvm/llvm-project/pull/86868
---
 clang/test/Driver/linker-wrapper-image.c      |   2 +-
 .../Frontend/Offloading/OffloadWrapper.cpp    |   7 +-
 offload/include/PluginManager.h               |  61 ++----
 offload/include/device.h                      |   8 +-
 offload/plugins-nextgen/CMakeLists.txt        |  19 +-
 offload/plugins-nextgen/amdgpu/CMakeLists.txt |   5 -
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    |  14 +-
 offload/plugins-nextgen/common/CMakeLists.txt |   5 +-
 .../common/include/PluginInterface.h          |  94 +-------
 .../common/include/Utils/ELF.h                |   2 -
 offload/plugins-nextgen/common/src/JIT.cpp    |  40 ++--
 .../common/src/PluginInterface.cpp            | 205 ------------------
 offload/plugins-nextgen/cuda/CMakeLists.txt   |   5 -
 offload/plugins-nextgen/cuda/src/rtl.cpp      |  14 +-
 offload/plugins-nextgen/host/CMakeLists.txt   |   8 -
 offload/plugins-nextgen/host/src/rtl.cpp      |  14 +-
 offload/src/CMakeLists.txt                    |   4 +
 offload/src/OffloadRTL.cpp                    |   1 +
 offload/src/OpenMP/InteropAPI.cpp             |   4 +-
 offload/src/PluginManager.cpp                 | 129 ++++-------
 offload/src/device.cpp                        |   3 +-
 offload/src/interface.cpp                     |   2 -
 .../kernelreplay/llvm-omp-kernel-replay.cpp   |   2 -
 .../unittests/Plugins/NextgenPluginsTest.cpp  |   1 -
 24 files changed, 125 insertions(+), 524 deletions(-)

diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c
index d01445e3aed0..5d5d62805e17 100644
--- a/clang/test/Driver/linker-wrapper-image.c
+++ b/clang/test/Driver/linker-wrapper-image.c
@@ -30,8 +30,8 @@
 
 //      OPENMP: define internal void @.omp_offloading.descriptor_reg() section ".text.startup" {
 // OPENMP-NEXT: entry:
-// OPENMP-NEXT:   %0 = call i32 @atexit(ptr @.omp_offloading.descriptor_unreg)
 // OPENMP-NEXT:   call void @__tgt_register_lib(ptr @.omp_offloading.descriptor)
+// OPENMP-NEXT:   %0 = call i32 @atexit(ptr @.omp_offloading.descriptor_unreg)
 // OPENMP-NEXT:   ret void
 // OPENMP-NEXT: }
 
diff --git a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
index 7241d15ed1c6..8b6f9ea1f4cc 100644
--- a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
+++ b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
@@ -232,12 +232,13 @@ void createRegisterFunction(Module &M, GlobalVariable *BinDesc,
   // Construct function body
   IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));
 
+  Builder.CreateCall(RegFuncC, BinDesc);
+
   // Register the destructors with 'atexit'. This is expected by the CUDA
   // runtime and ensures that we clean up before dynamic objects are destroyed.
-  // This needs to be done before the runtime is called and registers its own.
+  // This needs to be done after plugin initialization to ensure that it is
+  // called before the plugin runtime is destroyed.
   Builder.CreateCall(AtExit, UnregFunc);
-
-  Builder.CreateCall(RegFuncC, BinDesc);
   Builder.CreateRetVoid();
 
   // Add this function to constructors.
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index eece7525e25e..1d6804da75d9 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -13,10 +13,11 @@
 #ifndef OMPTARGET_PLUGIN_MANAGER_H
 #define OMPTARGET_PLUGIN_MANAGER_H
 
+#include "PluginInterface.h"
+
 #include "DeviceImage.h"
 #include "ExclusiveAccess.h"
 #include "Shared/APITypes.h"
-#include "Shared/PluginAPI.h"
 #include "Shared/Requirements.h"
 
 #include "device.h"
@@ -34,38 +35,7 @@
 #include <mutex>
 #include <string>
 
-struct PluginManager;
-
-/// Plugin adaptors should be created via `PluginAdaptorTy::create` which will
-/// invoke the constructor and call `PluginAdaptorTy::init`. Eventual errors are
-/// reported back to the caller, otherwise a valid and initialized adaptor is
-/// returned.
-struct PluginAdaptorTy {
-  /// Try to create a plugin adaptor from a filename.
-  static llvm::Expected<std::unique_ptr<PluginAdaptorTy>>
-  create(const std::string &Name);
-
-  /// Name of the shared object file representing the plugin.
-  std::string Name;
-
-  /// Access to the shared object file representing the plugin.
-  std::unique_ptr<llvm::sys::DynamicLibrary> LibraryHandler;
-
-#define PLUGIN_API_HANDLE(NAME)                                                \
-  using NAME##_ty = decltype(__tgt_rtl_##NAME);                                \
-  NAME##_ty *NAME = nullptr;
-
-#include "Shared/PluginAPI.inc"
-#undef PLUGIN_API_HANDLE
-
-  /// Create a plugin adaptor for filename \p Name with a dynamic library \p DL.
-  PluginAdaptorTy(const std::string &Name,
-                  std::unique_ptr<llvm::sys::DynamicLibrary> DL);
-
-  /// Initialize the plugin adaptor, this can fail in which case the adaptor is
-  /// useless.
-  llvm::Error init();
-};
+using GenericPluginTy = llvm::omp::target::plugin::GenericPluginTy;
 
 /// Struct for the data required to handle plugins
 struct PluginManager {
@@ -80,6 +50,8 @@ struct PluginManager {
 
   void init();
 
+  void deinit();
+
   // Register a shared library with all (compatible) RTLs.
   void registerLib(__tgt_bin_desc *Desc);
 
@@ -92,10 +64,9 @@ struct PluginManager {
         std::make_unique<DeviceImageTy>(TgtBinDesc, TgtDeviceImage));
   }
 
-  /// Initialize as many devices as possible for this plugin adaptor. Devices
-  /// that fail to initialize are ignored. Returns the offset the devices were
-  /// registered at.
-  void initDevices(PluginAdaptorTy &RTL);
+  /// Initialize as many devices as possible for this plugin. Devices that fail
+  /// to initialize are ignored.
+  void initDevices(GenericPluginTy &RTL);
 
   /// Return the device presented to the user as device \p DeviceNo if it is
   /// initialized and ready. Otherwise return an error explaining the problem.
@@ -151,8 +122,8 @@ struct PluginManager {
   // Initialize all plugins.
   void initAllPlugins();
 
-  /// Iterator range for all plugin adaptors (in use or not, but always valid).
-  auto pluginAdaptors() { return llvm::make_pointee_range(PluginAdaptors); }
+  /// Iterator range for all plugins (in use or not, but always valid).
+  auto plugins() { return llvm::make_pointee_range(Plugins); }
 
   /// Return the user provided requirements.
   int64_t getRequirements() const { return Requirements.getRequirements(); }
@@ -164,14 +135,14 @@ private:
   bool RTLsLoaded = false;
   llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
 
-  // List of all plugin adaptors, in use or not.
-  llvm::SmallVector<std::unique_ptr<PluginAdaptorTy>> PluginAdaptors;
+  // List of all plugins, in use or not.
+  llvm::SmallVector<std::unique_ptr<GenericPluginTy>> Plugins;
 
-  // Mapping of plugin adaptors to offsets in the device table.
-  llvm::DenseMap<const PluginAdaptorTy *, int32_t> DeviceOffsets;
+  // Mapping of plugins to offsets in the device table.
+  llvm::DenseMap<const GenericPluginTy *, int32_t> DeviceOffsets;
 
-  // Mapping of plugin adaptors to the number of used devices.
-  llvm::DenseMap<const PluginAdaptorTy *, int32_t> DeviceUsed;
+  // Mapping of plugins to the number of used devices.
+  llvm::DenseMap<const GenericPluginTy *, int32_t> DeviceUsed;
 
   // Set of all device images currently in use.
   llvm::DenseSet<const __tgt_device_image *> UsedImages;
diff --git a/offload/include/device.h b/offload/include/device.h
index bd2829722bb3..fd6e5fba5fc5 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -33,17 +33,19 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 
+#include "PluginInterface.h"
+using GenericPluginTy = llvm::omp::target::plugin::GenericPluginTy;
+
 // Forward declarations.
-struct PluginAdaptorTy;
 struct __tgt_bin_desc;
 struct __tgt_target_table;
 
 struct DeviceTy {
   int32_t DeviceID;
-  PluginAdaptorTy *RTL;
+  GenericPluginTy *RTL;
   int32_t RTLDeviceID;
 
-  DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID);
+  DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID);
   // DeviceTy is not copyable
   DeviceTy(const DeviceTy &D) = delete;
   DeviceTy &operator=(const DeviceTy &D) = delete;
diff --git a/offload/plugins-nextgen/CMakeLists.txt b/offload/plugins-nextgen/CMakeLists.txt
index df625e97c7eb..d1079f8a3e9c 100644
--- a/offload/plugins-nextgen/CMakeLists.txt
+++ b/offload/plugins-nextgen/CMakeLists.txt
@@ -14,7 +14,7 @@
 set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common)
 add_subdirectory(common)
 function(add_target_library target_name lib_name)
-  add_llvm_library(${target_name} SHARED
+  add_llvm_library(${target_name} STATIC
     LINK_COMPONENTS
       ${LLVM_TARGETS_TO_BUILD}
       AggressiveInstCombine
@@ -46,27 +46,14 @@ function(add_target_library target_name lib_name)
   )
 
   llvm_update_compile_flags(${target_name})
+  target_include_directories(${target_name} PUBLIC ${common_dir}/include)
   target_link_libraries(${target_name} PRIVATE
                         PluginCommon ${OPENMP_PTHREAD_LIB})
 
   target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name})
   target_compile_definitions(${target_name} PRIVATE 
                              DEBUG_PREFIX="TARGET ${lib_name} RTL")
-
-  if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
-    # On FreeBSD, the 'environ' symbol is undefined at link time, but resolved by
-    # the dynamic linker at runtime. Therefore, allow the symbol to be undefined
-    # when creating a shared library.
-    target_link_libraries(${target_name} PRIVATE "-Wl,--allow-shlib-undefined")
-  else()
-    target_link_libraries(${target_name} PRIVATE "-Wl,-z,defs")
-  endif()
-
-  if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
-    target_link_libraries(${target_name} PRIVATE
-    "-Wl,--version-script=${common_dir}/../exports")
-  endif()
-  set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET protected)
+  set_target_properties(${target_name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
 
 foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD)
diff --git a/offload/plugins-nextgen/amdgpu/CMakeLists.txt b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
index f5f7096137c2..738183f8945e 100644
--- a/offload/plugins-nextgen/amdgpu/CMakeLists.txt
+++ b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
@@ -57,8 +57,3 @@ else()
   libomptarget_say("Not generating AMDGPU tests, no supported devices detected."
                    " Use 'LIBOMPTARGET_FORCE_AMDGPU_TESTS' to override.")
 endif()
-
-# Install plugin under the lib destination folder.
-install(TARGETS omptarget.rtl.amdgpu LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
-set_target_properties(omptarget.rtl.amdgpu PROPERTIES
-  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 00650b801b42..295685fceaa4 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -3064,10 +3064,6 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
     // HSA functions from now on, e.g., hsa_shut_down.
     Initialized = true;
 
-#ifdef OMPT_SUPPORT
-    ompt::connectLibrary();
-#endif
-
     // Register event handler to detect memory errors on the devices.
     Status = hsa_amd_register_system_event_handler(eventHandler, nullptr);
     if (auto Err = Plugin::check(
@@ -3155,6 +3151,8 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
 
   Triple::ArchType getTripleArch() const override { return Triple::amdgcn; }
 
+  const char *getName() const override { return GETNAME(TARGET_NAME); }
+
   /// Get the ELF code for recognizing the compatible image binary.
   uint16_t getMagicElfBits() const override { return ELF::EM_AMDGPU; }
 
@@ -3387,8 +3385,6 @@ Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
   return Plugin::success();
 }
 
-GenericPluginTy *PluginTy::createPlugin() { return new AMDGPUPluginTy(); }
-
 template <typename... ArgsTy>
 static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
   hsa_status_t ResultCode = static_cast<hsa_status_t>(Code);
@@ -3476,3 +3472,9 @@ void *AMDGPUDeviceTy::allocate(size_t Size, void *, TargetAllocTy Kind) {
 } // namespace target
 } // namespace omp
 } // namespace llvm
+
+extern "C" {
+llvm::omp::target::plugin::GenericPluginTy *createPlugin_amdgpu() {
+  return new llvm::omp::target::plugin::AMDGPUPluginTy();
+}
+}
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index acf0af63f050..a6bbb7e9454b 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -46,7 +46,6 @@ endif()
 
 # If we have OMPT enabled include it in the list of sources.
 if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
-  target_sources(PluginCommon PRIVATE OMPT/OmptCallback.cpp)
   target_include_directories(PluginCommon PRIVATE OMPT)
 endif()
 
@@ -66,6 +65,4 @@ target_include_directories(PluginCommon PUBLIC
   ${LIBOMPTARGET_INCLUDE_DIR}
 )
 
-set_target_properties(PluginCommon PROPERTIES
-  POSITION_INDEPENDENT_CODE ON
-  CXX_VISIBILITY_PRESET protected)
+set_target_properties(PluginCommon PROPERTIES POSITION_INDEPENDENT_CODE ON)
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 79e8464bfda5..e7a008f3a857 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -1010,6 +1010,9 @@ struct GenericPluginTy {
   /// Get the target triple of this plugin.
   virtual Triple::ArchType getTripleArch() const = 0;
 
+  /// Get the constant name identifier for this plugin.
+  virtual const char *getName() const = 0;
+
   /// Allocate a structure using the internal allocator.
   template <typename Ty> Ty *allocate() {
     return reinterpret_cast<Ty *>(Allocator.Allocate(sizeof(Ty), alignof(Ty)));
@@ -1226,7 +1229,7 @@ namespace Plugin {
 /// Create a success error. This is the same as calling Error::success(), but
 /// it is recommended to use this one for consistency with Plugin::error() and
 /// Plugin::check().
-static Error success() { return Error::success(); }
+static inline Error success() { return Error::success(); }
 
 /// Create a string error.
 template <typename... ArgsTy>
@@ -1246,95 +1249,6 @@ template <typename... ArgsTy>
 static Error check(int32_t ErrorCode, const char *ErrFmt, ArgsTy... Args);
 } // namespace Plugin
 
-/// Class for simplifying the getter operation of the plugin. Anywhere on the
-/// code, the current plugin can be retrieved by Plugin::get(). The class also
-/// declares functions to create plugin-specific object instances. The check(),
-/// createPlugin(), createDevice() and createGlobalHandler() functions should be
-/// defined by each plugin implementation.
-class PluginTy {
-  // Reference to the plugin instance.
-  static GenericPluginTy *SpecificPlugin;
-
-  PluginTy() {
-    if (auto Err = init())
-      REPORT("Failed to initialize plugin: %s\n",
-             toString(std::move(Err)).data());
-  }
-
-  ~PluginTy() {
-    if (auto Err = deinit())
-      REPORT("Failed to deinitialize plugin: %s\n",
-             toString(std::move(Err)).data());
-  }
-
-  PluginTy(const PluginTy &) = delete;
-  void operator=(const PluginTy &) = delete;
-
-  /// Create and intialize the plugin instance.
-  static Error init() {
-    assert(!SpecificPlugin && "Plugin already created");
-
-    // Create the specific plugin.
-    SpecificPlugin = createPlugin();
-    assert(SpecificPlugin && "Plugin was not created");
-
-    // Initialize the plugin.
-    return SpecificPlugin->init();
-  }
-
-  // Deinitialize and destroy the plugin instance.
-  static Error deinit() {
-    assert(SpecificPlugin && "Plugin no longer valid");
-
-    for (int32_t DevNo = 0, NumDev = SpecificPlugin->getNumDevices();
-         DevNo < NumDev; ++DevNo)
-      if (auto Err = SpecificPlugin->deinitDevice(DevNo))
-        return Err;
-
-    // Deinitialize the plugin.
-    if (auto Err = SpecificPlugin->deinit())
-      return Err;
-
-    // Delete the plugin instance.
-    delete SpecificPlugin;
-
-    // Invalidate the plugin reference.
-    SpecificPlugin = nullptr;
-
-    return Plugin::success();
-  }
-
-public:
-  /// Initialize the plugin if needed. The plugin could have been initialized by
-  /// a previous call to Plugin::get().
-  static Error initIfNeeded() {
-    // Trigger the initialization if needed.
-    get();
-
-    return Error::success();
-  }
-
-  /// Get a reference (or create if it was not created) to the plugin instance.
-  static GenericPluginTy &get() {
-    // This static variable will initialize the underlying plugin instance in
-    // case there was no previous explicit initialization. The initialization is
-    // thread safe.
-    static PluginTy Plugin;
-
-    assert(SpecificPlugin && "Plugin is not active");
-    return *SpecificPlugin;
-  }
-
-  /// Get a reference to the plugin with a specific plugin-specific type.
-  template <typename Ty> static Ty &get() { return static_cast<Ty &>(get()); }
-
-  /// Indicate whether the plugin is active.
-  static bool isActive() { return SpecificPlugin != nullptr; }
-
-  /// Create a plugin instance.
-  static GenericPluginTy *createPlugin();
-};
-
 /// Auxiliary interface class for GenericDeviceResourceManagerTy. This class
 /// acts as a reference to a device resource, such as a stream, and requires
 /// some basic functions to be implemented. The derived class should define an
diff --git a/offload/plugins-nextgen/common/include/Utils/ELF.h b/offload/plugins-nextgen/common/include/Utils/ELF.h
index f87e0a5ed02b..dcfdb5bd7b03 100644
--- a/offload/plugins-nextgen/common/include/Utils/ELF.h
+++ b/offload/plugins-nextgen/common/include/Utils/ELF.h
@@ -13,8 +13,6 @@
 #ifndef LLVM_OPENMP_LIBOMPTARGET_PLUGINS_ELF_UTILS_H
 #define LLVM_OPENMP_LIBOMPTARGET_PLUGINS_ELF_UTILS_H
 
-#include "Shared/PluginAPI.h"
-
 #include "llvm/Object/ELF.h"
 #include "llvm/Object/ELFObjectFile.h"
 
diff --git a/offload/plugins-nextgen/common/src/JIT.cpp b/offload/plugins-nextgen/common/src/JIT.cpp
index 9eb610cab4de..9d58e6060646 100644
--- a/offload/plugins-nextgen/common/src/JIT.cpp
+++ b/offload/plugins-nextgen/common/src/JIT.cpp
@@ -56,28 +56,6 @@ bool isImageBitcode(const __tgt_device_image &Image) {
   return identify_magic(Binary) == file_magic::bitcode;
 }
 
-std::once_flag InitFlag;
-
-void init(Triple TT) {
-  codegen::RegisterCodeGenFlags();
-#ifdef LIBOMPTARGET_JIT_NVPTX
-  if (TT.isNVPTX()) {
-    LLVMInitializeNVPTXTargetInfo();
-    LLVMInitializeNVPTXTarget();
-    LLVMInitializeNVPTXTargetMC();
-    LLVMInitializeNVPTXAsmPrinter();
-  }
-#endif
-#ifdef LIBOMPTARGET_JIT_AMDGPU
-  if (TT.isAMDGPU()) {
-    LLVMInitializeAMDGPUTargetInfo();
-    LLVMInitializeAMDGPUTarget();
-    LLVMInitializeAMDGPUTargetMC();
-    LLVMInitializeAMDGPUAsmPrinter();
-  }
-#endif
-}
-
 Expected<std::unique_ptr<Module>>
 createModuleFromMemoryBuffer(std::unique_ptr<MemoryBuffer> &MB,
                              LLVMContext &Context) {
@@ -148,7 +126,23 @@ createTargetMachine(Module &M, std::string CPU, unsigned OptLevel) {
 } // namespace
 
 JITEngine::JITEngine(Triple::ArchType TA) : TT(Triple::getArchTypeName(TA)) {
-  std::call_once(InitFlag, init, TT);
+  codegen::RegisterCodeGenFlags();
+#ifdef LIBOMPTARGET_JIT_NVPTX
+  if (TT.isNVPTX()) {
+    LLVMInitializeNVPTXTargetInfo();
+    LLVMInitializeNVPTXTarget();
+    LLVMInitializeNVPTXTargetMC();
+    LLVMInitializeNVPTXAsmPrinter();
+  }
+#endif
+#ifdef LIBOMPTARGET_JIT_AMDGPU
+  if (TT.isAMDGPU()) {
+    LLVMInitializeAMDGPUTargetInfo();
+    LLVMInitializeAMDGPUTarget();
+    LLVMInitializeAMDGPUTargetMC();
+    LLVMInitializeAMDGPUAsmPrinter();
+  }
+#endif
 }
 
 void JITEngine::opt(TargetMachine *TM, TargetLibraryInfoImpl *TLII, Module &M,
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 8de93ba17a56..fae197527850 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -13,7 +13,6 @@
 #include "Shared/APITypes.h"
 #include "Shared/Debug.h"
 #include "Shared/Environment.h"
-#include "Shared/PluginAPI.h"
 
 #include "GlobalHandler.h"
 #include "JIT.h"
@@ -39,8 +38,6 @@ using namespace omp;
 using namespace target;
 using namespace plugin;
 
-GenericPluginTy *PluginTy::SpecificPlugin = nullptr;
-
 // TODO: Fix any thread safety issues for multi-threaded kernel recording.
 struct RecordReplayTy {
 
@@ -2035,205 +2032,3 @@ bool llvm::omp::target::plugin::libomptargetSupportsRPC() {
   return false;
 #endif
 }
-
-/// Exposed library API function, basically wrappers around the GenericDeviceTy
-/// functionality with the same name. All non-async functions are redirected
-/// to the async versions right away with a NULL AsyncInfoPtr.
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-int32_t __tgt_rtl_init_plugin() {
-  auto Err = PluginTy::initIfNeeded();
-  if (Err) {
-    [[maybe_unused]] std::string ErrStr = toString(std::move(Err));
-    DP("Failed to init plugin: %s", ErrStr.c_str());
-    return OFFLOAD_FAIL;
-  }
-
-  return OFFLOAD_SUCCESS;
-}
-
-int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) {
-  if (!PluginTy::isActive())
-    return false;
-
-  return PluginTy::get().is_valid_binary(Image);
-}
-
-int32_t __tgt_rtl_init_device(int32_t DeviceId) {
-  return PluginTy::get().init_device(DeviceId);
-}
-
-int32_t __tgt_rtl_number_of_devices() {
-  return PluginTy::get().number_of_devices();
-}
-
-int64_t __tgt_rtl_init_requires(int64_t RequiresFlags) {
-  return PluginTy::get().init_requires(RequiresFlags);
-}
-
-int32_t __tgt_rtl_is_data_exchangable(int32_t SrcDeviceId,
-                                      int32_t DstDeviceId) {
-  return PluginTy::get().is_data_exchangable(SrcDeviceId, DstDeviceId);
-}
-
-int32_t __tgt_rtl_initialize_record_replay(int32_t DeviceId, int64_t MemorySize,
-                                           void *VAddr, bool isRecord,
-                                           bool SaveOutput,
-                                           uint64_t &ReqPtrArgOffset) {
-  return PluginTy::get().initialize_record_replay(
-      DeviceId, MemorySize, VAddr, isRecord, SaveOutput, ReqPtrArgOffset);
-}
-
-int32_t __tgt_rtl_load_binary(int32_t DeviceId, __tgt_device_image *TgtImage,
-                              __tgt_device_binary *Binary) {
-  return PluginTy::get().load_binary(DeviceId, TgtImage, Binary);
-}
-
-void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr,
-                           int32_t Kind) {
-  return PluginTy::get().data_alloc(DeviceId, Size, HostPtr, Kind);
-}
-
-int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind) {
-  return PluginTy::get().data_delete(DeviceId, TgtPtr, Kind);
-}
-
-int32_t __tgt_rtl_data_lock(int32_t DeviceId, void *Ptr, int64_t Size,
-                            void **LockedPtr) {
-  return PluginTy::get().data_lock(DeviceId, Ptr, Size, LockedPtr);
-}
-
-int32_t __tgt_rtl_data_unlock(int32_t DeviceId, void *Ptr) {
-  return PluginTy::get().data_unlock(DeviceId, Ptr);
-}
-
-int32_t __tgt_rtl_data_notify_mapped(int32_t DeviceId, void *HstPtr,
-                                     int64_t Size) {
-  return PluginTy::get().data_notify_mapped(DeviceId, HstPtr, Size);
-}
-
-int32_t __tgt_rtl_data_notify_unmapped(int32_t DeviceId, void *HstPtr) {
-  return PluginTy::get().data_notify_unmapped(DeviceId, HstPtr);
-}
-
-int32_t __tgt_rtl_data_submit(int32_t DeviceId, void *TgtPtr, void *HstPtr,
-                              int64_t Size) {
-  return PluginTy::get().data_submit(DeviceId, TgtPtr, HstPtr, Size);
-}
-
-int32_t __tgt_rtl_data_submit_async(int32_t DeviceId, void *TgtPtr,
-                                    void *HstPtr, int64_t Size,
-                                    __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().data_submit_async(DeviceId, TgtPtr, HstPtr, Size,
-                                           AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr,
-                                int64_t Size) {
-  return PluginTy::get().data_retrieve(DeviceId, HstPtr, TgtPtr, Size);
-}
-
-int32_t __tgt_rtl_data_retrieve_async(int32_t DeviceId, void *HstPtr,
-                                      void *TgtPtr, int64_t Size,
-                                      __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().data_retrieve_async(DeviceId, HstPtr, TgtPtr, Size,
-                                             AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_data_exchange(int32_t SrcDeviceId, void *SrcPtr,
-                                int32_t DstDeviceId, void *DstPtr,
-                                int64_t Size) {
-  return PluginTy::get().data_exchange(SrcDeviceId, SrcPtr, DstDeviceId, DstPtr,
-                                       Size);
-}
-
-int32_t __tgt_rtl_data_exchange_async(int32_t SrcDeviceId, void *SrcPtr,
-                                      int DstDeviceId, void *DstPtr,
-                                      int64_t Size,
-                                      __tgt_async_info *AsyncInfo) {
-  return PluginTy::get().data_exchange_async(SrcDeviceId, SrcPtr, DstDeviceId,
-                                             DstPtr, Size, AsyncInfo);
-}
-
-int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
-                                void **TgtArgs, ptrdiff_t *TgtOffsets,
-                                KernelArgsTy *KernelArgs,
-                                __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().launch_kernel(DeviceId, TgtEntryPtr, TgtArgs,
-                                       TgtOffsets, KernelArgs, AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_synchronize(int32_t DeviceId,
-                              __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().synchronize(DeviceId, AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_query_async(int32_t DeviceId,
-                              __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().query_async(DeviceId, AsyncInfoPtr);
-}
-
-void __tgt_rtl_print_device_info(int32_t DeviceId) {
-  PluginTy::get().print_device_info(DeviceId);
-}
-
-int32_t __tgt_rtl_create_event(int32_t DeviceId, void **EventPtr) {
-  return PluginTy::get().create_event(DeviceId, EventPtr);
-}
-
-int32_t __tgt_rtl_record_event(int32_t DeviceId, void *EventPtr,
-                               __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().record_event(DeviceId, EventPtr, AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_wait_event(int32_t DeviceId, void *EventPtr,
-                             __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().wait_event(DeviceId, EventPtr, AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_sync_event(int32_t DeviceId, void *EventPtr) {
-  return PluginTy::get().sync_event(DeviceId, EventPtr);
-}
-
-int32_t __tgt_rtl_destroy_event(int32_t DeviceId, void *EventPtr) {
-  return PluginTy::get().destroy_event(DeviceId, EventPtr);
-}
-
-void __tgt_rtl_set_info_flag(uint32_t NewInfoLevel) {
-  return PluginTy::get().set_info_flag(NewInfoLevel);
-}
-
-int32_t __tgt_rtl_init_async_info(int32_t DeviceId,
-                                  __tgt_async_info **AsyncInfoPtr) {
-  return PluginTy::get().init_async_info(DeviceId, AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_init_device_info(int32_t DeviceId,
-                                   __tgt_device_info *DeviceInfo,
-                                   const char **ErrStr) {
-  return PluginTy::get().init_device_info(DeviceId, DeviceInfo, ErrStr);
-}
-
-int32_t __tgt_rtl_set_device_offset(int32_t DeviceIdOffset) {
-  return PluginTy::get().set_device_offset(DeviceIdOffset);
-}
-
-int32_t __tgt_rtl_use_auto_zero_copy(int32_t DeviceId) {
-  return PluginTy::get().use_auto_zero_copy(DeviceId);
-}
-
-int32_t __tgt_rtl_get_global(__tgt_device_binary Binary, uint64_t Size,
-                             const char *Name, void **DevicePtr) {
-  return PluginTy::get().get_global(Binary, Size, Name, DevicePtr);
-}
-
-int32_t __tgt_rtl_get_function(__tgt_device_binary Binary, const char *Name,
-                               void **KernelPtr) {
-  return PluginTy::get().get_function(Binary, Name, KernelPtr);
-}
-
-#ifdef __cplusplus
-}
-#endif
diff --git a/offload/plugins-nextgen/cuda/CMakeLists.txt b/offload/plugins-nextgen/cuda/CMakeLists.txt
index 0284bd22d2a4..dd684bb22343 100644
--- a/offload/plugins-nextgen/cuda/CMakeLists.txt
+++ b/offload/plugins-nextgen/cuda/CMakeLists.txt
@@ -51,8 +51,3 @@ else()
   libomptarget_say("Not generating NVIDIA tests, no supported devices detected."
                    " Use 'LIBOMPTARGET_FORCE_NVIDIA_TESTS' to override.")
 endif()
-
-# Install plugin under the lib destination folder.
-install(TARGETS omptarget.rtl.cuda LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
-set_target_properties(omptarget.rtl.cuda PROPERTIES
-  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index fc74c6aa23fd..b260334baa18 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -1342,10 +1342,6 @@ struct CUDAPluginTy final : public GenericPluginTy {
       return 0;
     }
 
-#ifdef OMPT_SUPPORT
-    ompt::connectLibrary();
-#endif
-
     if (Res == CUDA_ERROR_NO_DEVICE) {
       // Do not initialize if there are no devices.
       DP("There are no devices supporting CUDA.\n");
@@ -1390,6 +1386,8 @@ struct CUDAPluginTy final : public GenericPluginTy {
     return Triple::nvptx64;
   }
 
+  const char *getName() const override { return GETNAME(TARGET_NAME); }
+
   /// Check whether the image is compatible with the available CUDA devices.
   Expected<bool> isELFCompatible(StringRef Image) const override {
     auto ElfOrErr =
@@ -1495,8 +1493,6 @@ Error CUDADeviceTy::dataExchangeImpl(const void *SrcPtr,
   return Plugin::check(Res, "Error in cuMemcpyDtoDAsync: %s");
 }
 
-GenericPluginTy *PluginTy::createPlugin() { return new CUDAPluginTy(); }
-
 template <typename... ArgsTy>
 static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
   CUresult ResultCode = static_cast<CUresult>(Code);
@@ -1516,3 +1512,9 @@ static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
 } // namespace target
 } // namespace omp
 } // namespace llvm
+
+extern "C" {
+llvm::omp::target::plugin::GenericPluginTy *createPlugin_cuda() {
+  return new llvm::omp::target::plugin::CUDAPluginTy();
+}
+}
diff --git a/offload/plugins-nextgen/host/CMakeLists.txt b/offload/plugins-nextgen/host/CMakeLists.txt
index 1d000442c84d..72b5681283fe 100644
--- a/offload/plugins-nextgen/host/CMakeLists.txt
+++ b/offload/plugins-nextgen/host/CMakeLists.txt
@@ -31,14 +31,6 @@ else()
   target_include_directories(omptarget.rtl.host PRIVATE dynamic_ffi)
 endif()
 
-# Install plugin under the lib destination folder.
-install(TARGETS omptarget.rtl.host
-        LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
-set_target_properties(omptarget.rtl.host PROPERTIES
-  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.."
-  POSITION_INDEPENDENT_CODE ON
-  CXX_VISIBILITY_PRESET protected)
-
 target_include_directories(omptarget.rtl.host PRIVATE
                            ${LIBOMPTARGET_INCLUDE_DIR})
 
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index 4bdcae3dd6a1..409b44b1640a 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -385,10 +385,6 @@ struct GenELF64PluginTy final : public GenericPluginTy {
 
   /// Initialize the plugin and return the number of devices.
   Expected<int32_t> initImpl() override {
-#ifdef OMPT_SUPPORT
-    ompt::connectLibrary();
-#endif
-
 #ifdef USES_DYNAMIC_FFI
     if (auto Err = Plugin::check(ffi_init(), "Failed to initialize libffi"))
       return std::move(Err);
@@ -445,9 +441,9 @@ struct GenELF64PluginTy final : public GenericPluginTy {
     return llvm::Triple::UnknownArch;
 #endif
   }
-};
 
-GenericPluginTy *PluginTy::createPlugin() { return new GenELF64PluginTy(); }
+  const char *getName() const override { return GETNAME(TARGET_NAME); }
+};
 
 template <typename... ArgsTy>
 static Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) {
@@ -462,3 +458,9 @@ static Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) {
 } // namespace target
 } // namespace omp
 } // namespace llvm
+
+extern "C" {
+llvm::omp::target::plugin::GenericPluginTy *createPlugin_host() {
+  return new llvm::omp::target::plugin::GenELF64PluginTy();
+}
+}
diff --git a/offload/src/CMakeLists.txt b/offload/src/CMakeLists.txt
index eda5a85ff1ab..8fe6d19d83eb 100644
--- a/offload/src/CMakeLists.txt
+++ b/offload/src/CMakeLists.txt
@@ -65,6 +65,10 @@ target_compile_definitions(omptarget PRIVATE
   DEBUG_PREFIX="omptarget"
 )
 
+foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD)
+  target_link_libraries(omptarget PRIVATE omptarget.rtl.${plugin})
+endforeach()
+
 target_compile_options(omptarget PUBLIC ${offload_compile_flags})
 target_link_options(omptarget PUBLIC ${offload_link_flags})
 
diff --git a/offload/src/OffloadRTL.cpp b/offload/src/OffloadRTL.cpp
index dd75b1b18150..29b573a27d08 100644
--- a/offload/src/OffloadRTL.cpp
+++ b/offload/src/OffloadRTL.cpp
@@ -50,6 +50,7 @@ void deinitRuntime() {
 
   if (RefCount == 1) {
     DP("Deinit offload library!\n");
+    PM->deinit();
     delete PM;
     PM = nullptr;
   }
diff --git a/offload/src/OpenMP/InteropAPI.cpp b/offload/src/OpenMP/InteropAPI.cpp
index 1a995cde7816..bdbc440c64a2 100644
--- a/offload/src/OpenMP/InteropAPI.cpp
+++ b/offload/src/OpenMP/InteropAPI.cpp
@@ -230,14 +230,14 @@ void __tgt_interop_init(ident_t *LocRef, int32_t Gtid,
   }
 
   DeviceTy &Device = *DeviceOrErr;
-  if (!Device.RTL || !Device.RTL->init_device_info ||
+  if (!Device.RTL ||
       Device.RTL->init_device_info(DeviceId, &(InteropPtr)->device_info,
                                    &(InteropPtr)->err_str)) {
     delete InteropPtr;
     InteropPtr = omp_interop_none;
   }
   if (InteropType == kmp_interop_type_tasksync) {
-    if (!Device.RTL || !Device.RTL->init_async_info ||
+    if (!Device.RTL ||
         Device.RTL->init_async_info(DeviceId, &(InteropPtr)->async_info)) {
       delete InteropPtr;
       InteropPtr = omp_interop_none;
diff --git a/offload/src/PluginManager.cpp b/offload/src/PluginManager.cpp
index dbb556c179e5..191afa345641 100644
--- a/offload/src/PluginManager.cpp
+++ b/offload/src/PluginManager.cpp
@@ -23,85 +23,25 @@ using namespace llvm::sys;
 
 PluginManager *PM = nullptr;
 
-Expected<std::unique_ptr<PluginAdaptorTy>>
-PluginAdaptorTy::create(const std::string &Name) {
-  DP("Attempting to load library '%s'...\n", Name.c_str());
-  TIMESCOPE_WITH_NAME_AND_IDENT(Name, (const ident_t *)nullptr);
-
-  std::string ErrMsg;
-  auto LibraryHandler = std::make_unique<DynamicLibrary>(
-      DynamicLibrary::getPermanentLibrary(Name.c_str(), &ErrMsg));
-
-  if (!LibraryHandler->isValid()) {
-    // Library does not exist or cannot be found.
-    return createStringError(inconvertibleErrorCode(),
-                             "Unable to load library '%s': %s!\n", Name.c_str(),
-                             ErrMsg.c_str());
-  }
-
-  DP("Successfully loaded library '%s'!\n", Name.c_str());
-  auto PluginAdaptor = std::unique_ptr<PluginAdaptorTy>(
-      new PluginAdaptorTy(Name, std::move(LibraryHandler)));
-  if (auto Err = PluginAdaptor->init())
-    return Err;
-  return std::move(PluginAdaptor);
-}
-
-PluginAdaptorTy::PluginAdaptorTy(const std::string &Name,
-                                 std::unique_ptr<llvm::sys::DynamicLibrary> DL)
-    : Name(Name), LibraryHandler(std::move(DL)) {}
-
-Error PluginAdaptorTy::init() {
-
-#define PLUGIN_API_HANDLE(NAME)                                                \
-  NAME = reinterpret_cast<decltype(NAME)>(                                     \
-      LibraryHandler->getAddressOfSymbol(GETNAME(__tgt_rtl_##NAME)));          \
-  if (!NAME) {                                                                 \
-    return createStringError(inconvertibleErrorCode(),                         \
-                             "Invalid plugin as necessary interface function " \
-                             "(%s) was not found.\n",                          \
-                             std::string(#NAME).c_str());                      \
-  }
-
-#include "Shared/PluginAPI.inc"
-#undef PLUGIN_API_HANDLE
-
-  // Remove plugin on failure to call optional init_plugin
-  int32_t Rc = init_plugin();
-  if (Rc != OFFLOAD_SUCCESS) {
-    return createStringError(inconvertibleErrorCode(),
-                             "Unable to initialize library '%s': %u!\n",
-                             Name.c_str(), Rc);
-  }
-
-  // No devices are supported by this RTL?
-  int32_t NumberOfPluginDevices = number_of_devices();
-  if (!NumberOfPluginDevices) {
-    return createStringError(inconvertibleErrorCode(),
-                             "No devices supported in this RTL\n");
-  }
-
-  DP("Registered '%s' with %d plugin visible devices!\n", Name.c_str(),
-     NumberOfPluginDevices);
-  return Error::success();
-}
+// Every plugin exports this method to create an instance of the plugin type.
+#define PLUGIN_TARGET(Name) extern "C" GenericPluginTy *createPlugin_##Name();
+#include "Shared/Targets.def"
 
 void PluginManager::init() {
   TIMESCOPE();
   DP("Loading RTLs...\n");
 
-  // Attempt to open all the plugins and, if they exist, check if the interface
-  // is correct and if they are supporting any devices.
+  // Attempt to create an instance of each supported plugin.
 #define PLUGIN_TARGET(Name)                                                    \
   do {                                                                         \
-    auto PluginAdaptorOrErr =                                                  \
-        PluginAdaptorTy::create("libomptarget.rtl." #Name ".so");              \
-    if (!PluginAdaptorOrErr) {                                                 \
-      [[maybe_unused]] std::string InfoMsg =                                   \
-          toString(PluginAdaptorOrErr.takeError());                            \
-      DP("%s", InfoMsg.c_str());                                               \
+    auto Plugin = std::unique_ptr<GenericPluginTy>(createPlugin_##Name());     \
+    if (auto Err = Plugin->init()) {                                           \
+      [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));         \
+      DP("Failed to init plugin: %s\n", InfoMsg.c_str());                      \
     } else {                                                                   \
-      PluginAdaptors.push_back(std::move(*PluginAdaptorOrErr));                \
+      DP("Registered plugin %s with %d visible device(s)\n",                   \
+         Plugin->getName(), Plugin->number_of_devices());                      \
+      Plugins.emplace_back(std::move(Plugin));                                 \
     }                                                                          \
   } while (false);
 #include "Shared/Targets.def"
@@ -109,15 +49,29 @@ void PluginManager::init() {
   DP("RTLs loaded!\n");
 }
 
-void PluginManager::initDevices(PluginAdaptorTy &RTL) {
+void PluginManager::deinit() {
+  TIMESCOPE();
+  DP("Unloading RTLs...\n");
+
+  for (auto &Plugin : Plugins) {
+    if (auto Err = Plugin->deinit()) {
+      [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));
+      DP("Failed to deinit plugin: %s\n", InfoMsg.c_str());
+    }
+    Plugin.release();
+  }
+
+  DP("RTLs unloaded!\n");
+}
+
+void PluginManager::initDevices(GenericPluginTy &RTL) {
   // If this RTL has already been initialized.
   if (PM->DeviceOffsets.contains(&RTL))
     return;
   TIMESCOPE();
 
   // If this RTL is not already in use, initialize it.
-  assert(RTL.number_of_devices() > 0 &&
-         "Tried to initialize useless plugin adaptor");
+  assert(RTL.number_of_devices() > 0 && "Tried to initialize useless plugin!");
 
   // Initialize the device information for the RTL we are about to use.
   auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
@@ -157,13 +111,12 @@ void PluginManager::initDevices(PluginAdaptorTy &RTL) {
 
   DeviceOffsets[&RTL] = DeviceOffset;
   DeviceUsed[&RTL] = NumberOfUserDevices;
-  DP("Plugin adaptor " DPxMOD " has index %d, exposes %d out of %d devices!\n",
-     DPxPTR(RTL.LibraryHandler.get()), DeviceOffset, NumberOfUserDevices,
-     RTL.number_of_devices());
+  DP("Plugin has index %d, exposes %d out of %d devices!\n", DeviceOffset,
+     NumberOfUserDevices, RTL.number_of_devices());
 }
 
 void PluginManager::initAllPlugins() {
-  for (auto &R : PluginAdaptors)
+  for (auto &R : Plugins)
     initDevices(*R);
 }
 
@@ -216,19 +169,22 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) {
     // Obtain the image and information that was previously extracted.
     __tgt_device_image *Img = &DI.getExecutableImage();
 
-    PluginAdaptorTy *FoundRTL = nullptr;
+    GenericPluginTy *FoundRTL = nullptr;
 
     // Scan the RTLs that have associated images until we find one that supports
     // the current image.
-    for (auto &R : PM->pluginAdaptors()) {
+    for (auto &R : PM->plugins()) {
+      if (!R.number_of_devices())
+        continue;
+
       if (!R.is_valid_binary(Img)) {
         DP("Image " DPxMOD " is NOT compatible with RTL %s!\n",
-           DPxPTR(Img->ImageStart), R.Name.c_str());
+           DPxPTR(Img->ImageStart), R.getName());
         continue;
       }
 
       DP("Image " DPxMOD " is compatible with RTL %s!\n",
-         DPxPTR(Img->ImageStart), R.Name.c_str());
+         DPxPTR(Img->ImageStart), R.getName());
 
       PM->initDevices(R);
 
@@ -247,7 +203,7 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) {
           (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin];
 
       DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(Img->ImageStart),
-         R.Name.c_str());
+         R.getName());
 
       registerImageIntoTranslationTable(TransTable, PM->DeviceOffsets[&R],
                                         PM->DeviceUsed[&R], Img);
@@ -282,11 +238,11 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
     // Obtain the image and information that was previously extracted.
     __tgt_device_image *Img = &DI.getExecutableImage();
 
-    PluginAdaptorTy *FoundRTL = NULL;
+    GenericPluginTy *FoundRTL = NULL;
 
     // Scan the RTLs that have associated images until we find one that supports
     // the current image. We only need to scan RTLs that are already being used.
-    for (auto &R : PM->pluginAdaptors()) {
+    for (auto &R : PM->plugins()) {
       if (!DeviceOffsets.contains(&R))
         continue;
 
@@ -296,8 +252,7 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
 
       FoundRTL = &R;
 
-      DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n",
-         DPxPTR(Img->ImageStart), DPxPTR(R.LibraryHandler.get()));
+      DP("Unregistered image " DPxMOD " from RTL\n", DPxPTR(Img->ImageStart));
 
       break;
     }
diff --git a/offload/src/device.cpp b/offload/src/device.cpp
index 44a2facc8d3d..749b4c567f8e 100644
--- a/offload/src/device.cpp
+++ b/offload/src/device.cpp
@@ -64,7 +64,7 @@ int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,
   return OFFLOAD_SUCCESS;
 }
 
-DeviceTy::DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID)
+DeviceTy::DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID)
     : DeviceID(DeviceID), RTL(RTL), RTLDeviceID(RTLDeviceID),
       MappingInfo(*this) {}
 
@@ -192,7 +192,6 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
           RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
   if (!AsyncInfo) {
-    assert(RTL->data_exchange && "RTL->data_exchange is nullptr");
     return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr,
                               Size);
   }
diff --git a/offload/src/interface.cpp b/offload/src/interface.cpp
index 557703632c62..763b051cc6d7 100644
--- a/offload/src/interface.cpp
+++ b/offload/src/interface.cpp
@@ -456,8 +456,6 @@ EXTERN void __tgt_set_info_flag(uint32_t NewInfoLevel) {
   assert(PM && "Runtime not initialized");
   std::atomic<uint32_t> &InfoLevel = getInfoLevelInternal();
   InfoLevel.store(NewInfoLevel);
-  for (auto &R : PM->pluginAdaptors())
-    R.set_info_flag(NewInfoLevel);
 }
 
 EXTERN int __tgt_print_device_info(int64_t DeviceId) {
diff --git a/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp b/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
index 761e04e4c7bb..1e9a6a84d805 100644
--- a/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
+++ b/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
@@ -13,8 +13,6 @@
 
 #include "omptarget.h"
 
-#include "Shared/PluginAPI.h"
-
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/MemoryBuffer.h"
diff --git a/offload/unittests/Plugins/NextgenPluginsTest.cpp b/offload/unittests/Plugins/NextgenPluginsTest.cpp
index 635bd1637c90..479b3f614aed 100644
--- a/offload/unittests/Plugins/NextgenPluginsTest.cpp
+++ b/offload/unittests/Plugins/NextgenPluginsTest.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Shared/PluginAPI.h"
 #include "omptarget.h"
 #include "gtest/gtest.h"
 
-- 
GitLab


From b903badd73a2467fdd4e363231f2bf9b0704b546 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Thu, 9 May 2024 11:29:07 +0000
Subject: [PATCH 0273/1206] [lldb] Attempt to fix
 signal-in-leaf-function-aarch64 on darwin

Convert settings set EXC_BAD_INSTRUCTION to SIGILL so we get uniform
behavior (and can resume the inferior).

Fix a "omitting the parameter name in a function definition is a C23
extension" warning as a drive-by.
---
 .../test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c | 2 +-
 lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test    | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c b/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
index 9a751330623f..fe020affcad0 100644
--- a/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
+++ b/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
@@ -7,7 +7,7 @@ int __attribute__((naked)) signal_generating_add(int a, int b) {
       "ret");
 }
 
-void sigill_handler(int) { _exit(0); }
+void sigill_handler(int signo) { _exit(0); }
 
 int main() {
   signal(SIGILL, sigill_handler);
diff --git a/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test b/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test
index 0580d0cf734a..09f17c174bbf 100644
--- a/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test
+++ b/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test
@@ -4,6 +4,9 @@
 # RUN: %clang_host %S/Inputs/signal-in-leaf-function-aarch64.c -o %t
 # RUN: %lldb -s %s -o exit %t | FileCheck %s
 
+# Convert EXC_BAD_INSTRUCTION to SIGILL on darwin
+settings set platform.plugin.darwin.ignored-exceptions EXC_BAD_INSTRUCTION
+
 breakpoint set -n sigill_handler
 # CHECK: Breakpoint 1: where = {{.*}}`sigill_handler
 
-- 
GitLab


From b452b34932a3f0450026c40fb797698a6671f9a7 Mon Sep 17 00:00:00 2001
From: Shan Huang <52285902006@stu.ecnu.edu.cn>
Date: Thu, 9 May 2024 19:58:53 +0800
Subject: [PATCH 0274/1206] [DebugInfo][IndVarSimplify] Fix missing debug
 location updates (#91443)

Adds debug location updates for the newly created `phi`, `add`, `icmp` and `sitofp` instructions in `IndVarSimplify`.

Fixes #91436
---
 llvm/lib/Transforms/Scalar/IndVarSimplify.cpp | 10 ++-
 .../IndVarSimplify/preserving-debugloc.ll     | 61 +++++++++++++++++++
 2 files changed, 68 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/Transforms/IndVarSimplify/preserving-debugloc.ll

diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index ba392e187b8b..dd7c89034ca0 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -359,15 +359,18 @@ bool IndVarSimplify::handleFloatingPointIV(Loop *L, PHINode *PN) {
       PHINode::Create(Int32Ty, 2, PN->getName() + ".int", PN->getIterator());
   NewPHI->addIncoming(ConstantInt::get(Int32Ty, InitValue),
                       PN->getIncomingBlock(IncomingEdge));
+  NewPHI->setDebugLoc(PN->getDebugLoc());
 
-  Value *NewAdd =
+  Instruction *NewAdd =
       BinaryOperator::CreateAdd(NewPHI, ConstantInt::get(Int32Ty, IncValue),
                                 Incr->getName() + ".int", Incr->getIterator());
+  NewAdd->setDebugLoc(Incr->getDebugLoc());
   NewPHI->addIncoming(NewAdd, PN->getIncomingBlock(BackEdge));
 
   ICmpInst *NewCompare =
       new ICmpInst(TheBr->getIterator(), NewPred, NewAdd,
                    ConstantInt::get(Int32Ty, ExitValue), Compare->getName());
+  NewCompare->setDebugLoc(Compare->getDebugLoc());
 
   // In the following deletions, PN may become dead and may be deleted.
   // Use a WeakTrackingVH to observe whether this happens.
@@ -391,8 +394,9 @@ bool IndVarSimplify::handleFloatingPointIV(Loop *L, PHINode *PN) {
   // We give preference to sitofp over uitofp because it is faster on most
   // platforms.
   if (WeakPH) {
-    Value *Conv = new SIToFPInst(NewPHI, PN->getType(), "indvar.conv",
-                                 PN->getParent()->getFirstInsertionPt());
+    Instruction *Conv = new SIToFPInst(NewPHI, PN->getType(), "indvar.conv",
+                                       PN->getParent()->getFirstInsertionPt());
+    Conv->setDebugLoc(PN->getDebugLoc());
     PN->replaceAllUsesWith(Conv);
     RecursivelyDeleteTriviallyDeadInstructions(PN, TLI, MSSAU.get());
   }
diff --git a/llvm/test/Transforms/IndVarSimplify/preserving-debugloc.ll b/llvm/test/Transforms/IndVarSimplify/preserving-debugloc.ll
new file mode 100644
index 000000000000..7d23c8697efa
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/preserving-debugloc.ll
@@ -0,0 +1,61 @@
+; RUN: opt < %s -passes=indvars -S | FileCheck %s
+
+; This testcase checks the preservation of debug locations of newly created 
+; phi, sitofp, add and icmp instructions in IndVarSimplify Pass.
+
+define void @test1() !dbg !5 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB:%.*]], !dbg
+; CHECK:  bb:
+; CHECK:    [[IV_INT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[DOTINT:%.*]], [[BB]] ], !dbg ![[DBG1:[0-9]+]]
+; CHECK:    [[INDVAR_CONV:%.*]] = sitofp i32 [[IV_INT]] to double, !dbg ![[DBG1]]
+; CHECK:    [[DOTINT]] = add nuw nsw i32 [[IV_INT]], 1, !dbg ![[DBG2:[0-9]+]]
+; CHECK:    [[TMP1:%.*]] = icmp ult i32 [[DOTINT]], 10000, !dbg ![[DBG3:[0-9]+]]
+; CHECK: ![[DBG1]] = !DILocation(line: 2
+; CHECK: ![[DBG2]] = !DILocation(line: 4
+; CHECK: ![[DBG3]] = !DILocation(line: 5
+;
+entry:
+  br label %bb, !dbg !16
+
+bb:                                               ; preds = %bb, %entry
+  %iv = phi double [ 0.000000e+00, %entry ], [ %1, %bb ], !dbg !17
+  %0 = tail call i32 @foo(double %iv), !dbg !18
+  %1 = fadd double %iv, 1.000000e+00, !dbg !19
+  %2 = fcmp olt double %1, 1.000000e+04, !dbg !20
+  br i1 %2, label %bb, label %return, !dbg !21
+
+return:                                           ; preds = %bb
+  ret void, !dbg !22
+}
+
+declare i32 @foo(double)
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!2, !3}
+!llvm.module.flags = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "indvars-preserving.ll", directory: "/")
+!2 = !{i32 7}
+!3 = !{i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !{!9, !11, !13, !14}
+!9 = !DILocalVariable(name: "1", scope: !5, file: !1, line: 2, type: !10)
+!10 = !DIBasicType(name: "ty64", size: 64, encoding: DW_ATE_unsigned)
+!11 = !DILocalVariable(name: "2", scope: !5, file: !1, line: 3, type: !12)
+!12 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+!13 = !DILocalVariable(name: "3", scope: !5, file: !1, line: 4, type: !10)
+!14 = !DILocalVariable(name: "4", scope: !5, file: !1, line: 5, type: !15)
+!15 = !DIBasicType(name: "ty8", size: 8, encoding: DW_ATE_unsigned)
+!16 = !DILocation(line: 1, column: 1, scope: !5)
+!17 = !DILocation(line: 2, column: 1, scope: !5)
+!18 = !DILocation(line: 3, column: 1, scope: !5)
+!19 = !DILocation(line: 4, column: 1, scope: !5)
+!20 = !DILocation(line: 5, column: 1, scope: !5)
+!21 = !DILocation(line: 6, column: 1, scope: !5)
+!22 = !DILocation(line: 7, column: 1, scope: !5)
-- 
GitLab


From a7ee81e8279e0bf6e05617a4a638e5f2f8e45022 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 9 May 2024 06:59:45 -0500
Subject: [PATCH 0275/1206] [libc] Remove unused variable causing build errors

---
 libc/src/__support/CPP/mutex.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/src/__support/CPP/mutex.h b/libc/src/__support/CPP/mutex.h
index c25c1155b766..345816fae233 100644
--- a/libc/src/__support/CPP/mutex.h
+++ b/libc/src/__support/CPP/mutex.h
@@ -31,7 +31,7 @@ public:
   // Acquires ownership of the mutex object `m` without attempting to lock
   // it. The behavior is undefined if the current thread does not hold the
   // lock on `m`. Does not call `m.lock()` upon resource acquisition.
-  lock_guard(MutexType &m, adopt_lock_t t) : mutex(m) {}
+  lock_guard(MutexType &m, adopt_lock_t /* t */) : mutex(m) {}
 
   ~lock_guard() { mutex.unlock(); }
 
-- 
GitLab


From d86b68afd7f0d7684adc312bcdc87f9027d0d896 Mon Sep 17 00:00:00 2001
From: Janek van Oirschot <5994977+JanekvO@users.noreply.github.com>
Date: Thu, 9 May 2024 13:02:32 +0100
Subject: [PATCH 0276/1206] MCExpr-ify SIProgramInfo (#88257)

Convert members in SIProgramInfo affected by variables provided by AMDGPUResourceUsageAnalysis into MCExprs.
---
 llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp   | 421 ++++++++++++------
 llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h     |   2 +
 .../AMDGPU/AMDGPUHSAMetadataStreamer.cpp      |  30 +-
 llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp    |  25 +-
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      |   6 +-
 .../AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp      | 201 +++++++++
 .../Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h |  42 +-
 llvm/lib/Target/AMDGPU/SIProgramInfo.cpp      | 207 +++++++--
 llvm/lib/Target/AMDGPU/SIProgramInfo.h        |  49 +-
 .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp    |  39 +-
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h |  12 +
 llvm/test/MC/AMDGPU/alignto_mcexpr.s          |  15 +
 llvm/test/MC/AMDGPU/extrasgprs_mcexpr.s       |  31 ++
 llvm/test/MC/AMDGPU/occupancy_mcexpr.s        |  61 +++
 llvm/test/MC/AMDGPU/totalnumvgpr_mcexpr.s     |  26 ++
 llvm/unittests/MC/AMDGPU/CMakeLists.txt       |  10 +-
 .../MC/AMDGPU/SIProgramInfoMCExprs.cpp        |  81 ++++
 17 files changed, 1040 insertions(+), 218 deletions(-)
 create mode 100644 llvm/test/MC/AMDGPU/alignto_mcexpr.s
 create mode 100644 llvm/test/MC/AMDGPU/extrasgprs_mcexpr.s
 create mode 100644 llvm/test/MC/AMDGPU/occupancy_mcexpr.s
 create mode 100644 llvm/test/MC/AMDGPU/totalnumvgpr_mcexpr.s
 create mode 100644 llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 89a5ceac629b..de81904143b7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -22,6 +22,7 @@
 #include "AMDKernelCodeT.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUInstPrinter.h"
+#include "MCTargetDesc/AMDGPUMCExpr.h"
 #include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
 #include "R600AsmPrinter.h"
@@ -134,6 +135,15 @@ void AMDGPUAsmPrinter::initTargetStreamer(Module &M) {
     getTargetStreamer()->getPALMetadata()->readFromIR(M);
 }
 
+uint64_t AMDGPUAsmPrinter::getMCExprValue(const MCExpr *Value, MCContext &Ctx) {
+  int64_t Val;
+  if (!Value->evaluateAsAbsolute(Val)) {
+    Ctx.reportError(SMLoc(), "could not resolve expression when required.");
+    return 0;
+  }
+  return static_cast<uint64_t>(Val);
+}
+
 void AMDGPUAsmPrinter::emitEndOfAsmFile(Module &M) {
   // Init target streamer if it has not yet happened
   if (!IsTargetStreamerInitialized)
@@ -237,12 +247,14 @@ void AMDGPUAsmPrinter::emitFunctionBodyEnd() {
   getNameWithPrefix(KernelName, &MF->getFunction());
   getTargetStreamer()->EmitAmdhsaKernelDescriptor(
       STM, KernelName, getAmdhsaKernelDescriptor(*MF, CurrentProgramInfo),
-      CurrentProgramInfo.NumVGPRsForWavesPerEU,
-      CurrentProgramInfo.NumSGPRsForWavesPerEU -
+      getMCExprValue(CurrentProgramInfo.NumVGPRsForWavesPerEU, Context),
+      getMCExprValue(CurrentProgramInfo.NumSGPRsForWavesPerEU, Context) -
           IsaInfo::getNumExtraSGPRs(
-              &STM, CurrentProgramInfo.VCCUsed, CurrentProgramInfo.FlatUsed,
+              &STM, getMCExprValue(CurrentProgramInfo.VCCUsed, Context),
+              getMCExprValue(CurrentProgramInfo.FlatUsed, Context),
               getTargetStreamer()->getTargetID()->isXnackOnOrAny()),
-      CurrentProgramInfo.VCCUsed, CurrentProgramInfo.FlatUsed);
+      getMCExprValue(CurrentProgramInfo.VCCUsed, Context),
+      getMCExprValue(CurrentProgramInfo.FlatUsed, Context));
 
   Streamer.popSection();
 }
@@ -422,7 +434,7 @@ uint16_t AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties(
         amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
   }
 
-  if (CurrentProgramInfo.DynamicCallStack &&
+  if (getMCExprValue(CurrentProgramInfo.DynamicCallStack, MF.getContext()) &&
       CodeObjectVersion >= AMDGPU::AMDHSA_COV5)
     KernelCodeProperties |= amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK;
 
@@ -439,29 +451,22 @@ AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF,
 
   MCKernelDescriptor KernelDescriptor;
 
-  assert(isUInt<32>(PI.ScratchSize));
-  assert(isUInt<32>(PI.getComputePGMRSrc1(STM)));
-  assert(isUInt<32>(PI.getComputePGMRSrc2()));
-
   KernelDescriptor.group_segment_fixed_size =
       MCConstantExpr::create(PI.LDSSize, Ctx);
-  KernelDescriptor.private_segment_fixed_size =
-      MCConstantExpr::create(PI.ScratchSize, Ctx);
+  KernelDescriptor.private_segment_fixed_size = PI.ScratchSize;
 
   Align MaxKernArgAlign;
   KernelDescriptor.kernarg_size = MCConstantExpr::create(
       STM.getKernArgSegmentSize(F, MaxKernArgAlign), Ctx);
 
-  KernelDescriptor.compute_pgm_rsrc1 =
-      MCConstantExpr::create(PI.getComputePGMRSrc1(STM), Ctx);
-  KernelDescriptor.compute_pgm_rsrc2 =
-      MCConstantExpr::create(PI.getComputePGMRSrc2(), Ctx);
+  KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(STM, Ctx);
+  KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2(Ctx);
   KernelDescriptor.kernel_code_properties =
       MCConstantExpr::create(getAmdhsaKernelCodeProperties(MF), Ctx);
 
-  assert(STM.hasGFX90AInsts() || CurrentProgramInfo.ComputePGMRSrc3GFX90A == 0);
-  KernelDescriptor.compute_pgm_rsrc3 = MCConstantExpr::create(
-      STM.hasGFX90AInsts() ? CurrentProgramInfo.ComputePGMRSrc3GFX90A : 0, Ctx);
+  assert(STM.hasGFX90AInsts() ||
+         getMCExprValue(CurrentProgramInfo.ComputePGMRSrc3GFX90A, Ctx) == 0);
+  KernelDescriptor.compute_pgm_rsrc3 = CurrentProgramInfo.ComputePGMRSrc3GFX90A;
 
   KernelDescriptor.kernarg_preload = MCConstantExpr::create(
       AMDGPU::hasKernargPreload(STM) ? Info->getNumKernargPreloadedSGPRs() : 0,
@@ -477,9 +482,10 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
     initTargetStreamer(*MF.getFunction().getParent());
 
   ResourceUsage = &getAnalysis<AMDGPUResourceUsageAnalysis>();
-  CurrentProgramInfo = SIProgramInfo();
+  CurrentProgramInfo.reset(MF);
 
   const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>();
+  MCContext &Ctx = MF.getContext();
 
   // The starting address of all shader programs must be 256 bytes aligned.
   // Regular functions just need the basic required instruction alignment.
@@ -550,11 +556,13 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
 
     OutStreamer->emitRawComment(" Kernel info:", false);
     emitCommonFunctionComments(
-        CurrentProgramInfo.NumArchVGPR,
-        STM.hasMAIInsts() ? CurrentProgramInfo.NumAccVGPR
+        getMCExprValue(CurrentProgramInfo.NumArchVGPR, Ctx),
+        STM.hasMAIInsts() ? getMCExprValue(CurrentProgramInfo.NumAccVGPR, Ctx)
                           : std::optional<uint32_t>(),
-        CurrentProgramInfo.NumVGPR, CurrentProgramInfo.NumSGPR,
-        CurrentProgramInfo.ScratchSize, getFunctionCodeSize(MF), MFI);
+        getMCExprValue(CurrentProgramInfo.NumVGPR, Ctx),
+        getMCExprValue(CurrentProgramInfo.NumSGPR, Ctx),
+        getMCExprValue(CurrentProgramInfo.ScratchSize, Ctx),
+        getFunctionCodeSize(MF), MFI);
 
     OutStreamer->emitRawComment(
       " FloatMode: " + Twine(CurrentProgramInfo.FloatMode), false);
@@ -565,32 +573,44 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
       " bytes/workgroup (compile time only)", false);
 
     OutStreamer->emitRawComment(
-      " SGPRBlocks: " + Twine(CurrentProgramInfo.SGPRBlocks), false);
+        " SGPRBlocks: " +
+            Twine(getMCExprValue(CurrentProgramInfo.SGPRBlocks, Ctx)),
+        false);
     OutStreamer->emitRawComment(
-      " VGPRBlocks: " + Twine(CurrentProgramInfo.VGPRBlocks), false);
+        " VGPRBlocks: " +
+            Twine(getMCExprValue(CurrentProgramInfo.VGPRBlocks, Ctx)),
+        false);
 
     OutStreamer->emitRawComment(
-      " NumSGPRsForWavesPerEU: " +
-      Twine(CurrentProgramInfo.NumSGPRsForWavesPerEU), false);
+        " NumSGPRsForWavesPerEU: " +
+            Twine(
+                getMCExprValue(CurrentProgramInfo.NumSGPRsForWavesPerEU, Ctx)),
+        false);
     OutStreamer->emitRawComment(
-      " NumVGPRsForWavesPerEU: " +
-      Twine(CurrentProgramInfo.NumVGPRsForWavesPerEU), false);
+        " NumVGPRsForWavesPerEU: " +
+            Twine(
+                getMCExprValue(CurrentProgramInfo.NumVGPRsForWavesPerEU, Ctx)),
+        false);
 
     if (STM.hasGFX90AInsts())
       OutStreamer->emitRawComment(
-        " AccumOffset: " +
-        Twine((CurrentProgramInfo.AccumOffset + 1) * 4), false);
+          " AccumOffset: " +
+              Twine((getMCExprValue(CurrentProgramInfo.AccumOffset, Ctx) + 1) *
+                    4),
+          false);
 
     OutStreamer->emitRawComment(
-      " Occupancy: " +
-      Twine(CurrentProgramInfo.Occupancy), false);
+        " Occupancy: " +
+            Twine(getMCExprValue(CurrentProgramInfo.Occupancy, Ctx)),
+        false);
 
     OutStreamer->emitRawComment(
       " WaveLimiterHint : " + Twine(MFI->needsWaveLimiter()), false);
 
-    OutStreamer->emitRawComment(" COMPUTE_PGM_RSRC2:SCRATCH_EN: " +
-                                    Twine(CurrentProgramInfo.ScratchEnable),
-                                false);
+    OutStreamer->emitRawComment(
+        " COMPUTE_PGM_RSRC2:SCRATCH_EN: " +
+            Twine(getMCExprValue(CurrentProgramInfo.ScratchEnable, Ctx)),
+        false);
     OutStreamer->emitRawComment(" COMPUTE_PGM_RSRC2:USER_SGPR: " +
                                     Twine(CurrentProgramInfo.UserSGPR),
                                 false);
@@ -611,18 +631,20 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
                                 false);
 
     assert(STM.hasGFX90AInsts() ||
-           CurrentProgramInfo.ComputePGMRSrc3GFX90A == 0);
+           getMCExprValue(CurrentProgramInfo.ComputePGMRSrc3GFX90A, Ctx) == 0);
     if (STM.hasGFX90AInsts()) {
       OutStreamer->emitRawComment(
-        " COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: " +
-        Twine((AMDHSA_BITS_GET(CurrentProgramInfo.ComputePGMRSrc3GFX90A,
-                               amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET))),
-                               false);
+          " COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: " +
+              Twine((AMDHSA_BITS_GET(
+                  getMCExprValue(CurrentProgramInfo.ComputePGMRSrc3GFX90A, Ctx),
+                  amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET))),
+          false);
       OutStreamer->emitRawComment(
-        " COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: " +
-        Twine((AMDHSA_BITS_GET(CurrentProgramInfo.ComputePGMRSrc3GFX90A,
-                               amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT))),
-                               false);
+          " COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: " +
+              Twine((AMDHSA_BITS_GET(
+                  getMCExprValue(CurrentProgramInfo.ComputePGMRSrc3GFX90A, Ctx),
+                  amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT))),
+          false);
     }
   }
 
@@ -702,23 +724,40 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
   const AMDGPUResourceUsageAnalysis::SIFunctionResourceInfo &Info =
       ResourceUsage->getResourceInfo(&MF.getFunction());
   const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
+  MCContext &Ctx = MF.getContext();
+
+  auto CreateExpr = [&Ctx](int64_t Value) {
+    return MCConstantExpr::create(Value, Ctx);
+  };
 
-  ProgInfo.NumArchVGPR = Info.NumVGPR;
-  ProgInfo.NumAccVGPR = Info.NumAGPR;
-  ProgInfo.NumVGPR = Info.getTotalNumVGPRs(STM);
-  ProgInfo.AccumOffset = alignTo(std::max(1, Info.NumVGPR), 4) / 4 - 1;
+  auto TryGetMCExprValue = [&Ctx](const MCExpr *Value, uint64_t &Res) -> bool {
+    int64_t Val;
+    if (Value->evaluateAsAbsolute(Val)) {
+      Res = Val;
+      return true;
+    }
+    return false;
+  };
+
+  ProgInfo.NumArchVGPR = CreateExpr(Info.NumVGPR);
+  ProgInfo.NumAccVGPR = CreateExpr(Info.NumAGPR);
+  ProgInfo.NumVGPR = CreateExpr(Info.getTotalNumVGPRs(STM));
+  ProgInfo.AccumOffset =
+      CreateExpr(alignTo(std::max(1, Info.NumVGPR), 4) / 4 - 1);
   ProgInfo.TgSplit = STM.isTgSplitEnabled();
-  ProgInfo.NumSGPR = Info.NumExplicitSGPR;
-  ProgInfo.ScratchSize = Info.PrivateSegmentSize;
-  ProgInfo.VCCUsed = Info.UsesVCC;
-  ProgInfo.FlatUsed = Info.UsesFlatScratch;
-  ProgInfo.DynamicCallStack = Info.HasDynamicallySizedStack || Info.HasRecursion;
+  ProgInfo.NumSGPR = CreateExpr(Info.NumExplicitSGPR);
+  ProgInfo.ScratchSize = CreateExpr(Info.PrivateSegmentSize);
+  ProgInfo.VCCUsed = CreateExpr(Info.UsesVCC);
+  ProgInfo.FlatUsed = CreateExpr(Info.UsesFlatScratch);
+  ProgInfo.DynamicCallStack =
+      CreateExpr(Info.HasDynamicallySizedStack || Info.HasRecursion);
 
   const uint64_t MaxScratchPerWorkitem =
       STM.getMaxWaveScratchSize() / STM.getWavefrontSize();
-  if (ProgInfo.ScratchSize > MaxScratchPerWorkitem) {
-    DiagnosticInfoStackSize DiagStackSize(MF.getFunction(),
-                                          ProgInfo.ScratchSize,
+  uint64_t ScratchSize;
+  if (TryGetMCExprValue(ProgInfo.ScratchSize, ScratchSize) &&
+      ScratchSize > MaxScratchPerWorkitem) {
+    DiagnosticInfoStackSize DiagStackSize(MF.getFunction(), ScratchSize,
                                           MaxScratchPerWorkitem, DS_Error);
     MF.getFunction().getContext().diagnose(DiagStackSize);
   }
@@ -728,27 +767,29 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
   // The calculations related to SGPR/VGPR blocks are
   // duplicated in part in AMDGPUAsmParser::calculateGPRBlocks, and could be
   // unified.
-  unsigned ExtraSGPRs = IsaInfo::getNumExtraSGPRs(
-      &STM, ProgInfo.VCCUsed, ProgInfo.FlatUsed,
-      getTargetStreamer()->getTargetID()->isXnackOnOrAny());
+  const MCExpr *ExtraSGPRs = AMDGPUVariadicMCExpr::createExtraSGPRs(
+      ProgInfo.VCCUsed, ProgInfo.FlatUsed,
+      getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx);
 
   // Check the addressable register limit before we add ExtraSGPRs.
   if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
       !STM.hasSGPRInitBug()) {
     unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
-    if (ProgInfo.NumSGPR > MaxAddressableNumSGPRs) {
+    uint64_t NumSgpr;
+    if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
+        NumSgpr > MaxAddressableNumSGPRs) {
       // This can happen due to a compiler bug or when using inline asm.
       LLVMContext &Ctx = MF.getFunction().getContext();
       DiagnosticInfoResourceLimit Diag(
-          MF.getFunction(), "addressable scalar registers", ProgInfo.NumSGPR,
+          MF.getFunction(), "addressable scalar registers", NumSgpr,
           MaxAddressableNumSGPRs, DS_Error, DK_ResourceLimit);
       Ctx.diagnose(Diag);
-      ProgInfo.NumSGPR = MaxAddressableNumSGPRs - 1;
+      ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs - 1);
     }
   }
 
   // Account for extra SGPRs and VGPRs reserved for debugger use.
-  ProgInfo.NumSGPR += ExtraSGPRs;
+  ProgInfo.NumSGPR = MCBinaryExpr::createAdd(ProgInfo.NumSGPR, ExtraSGPRs, Ctx);
 
   const Function &F = MF.getFunction();
 
@@ -819,40 +860,51 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
         }
       }
     }
-    ProgInfo.NumSGPR = std::max(ProgInfo.NumSGPR, WaveDispatchNumSGPR);
-    ProgInfo.NumArchVGPR = std::max(ProgInfo.NumVGPR, WaveDispatchNumVGPR);
-    ProgInfo.NumVGPR =
-        Info.getTotalNumVGPRs(STM, Info.NumAGPR, ProgInfo.NumArchVGPR);
+    ProgInfo.NumSGPR = AMDGPUVariadicMCExpr::createMax(
+        {ProgInfo.NumSGPR, CreateExpr(WaveDispatchNumSGPR)}, Ctx);
+
+    ProgInfo.NumArchVGPR = AMDGPUVariadicMCExpr::createMax(
+        {ProgInfo.NumVGPR, CreateExpr(WaveDispatchNumVGPR)}, Ctx);
+
+    ProgInfo.NumVGPR = AMDGPUVariadicMCExpr::createTotalNumVGPR(
+        ProgInfo.NumAccVGPR, ProgInfo.NumArchVGPR, Ctx);
   }
 
   // Adjust number of registers used to meet default/requested minimum/maximum
   // number of waves per execution unit request.
-  ProgInfo.NumSGPRsForWavesPerEU = std::max(
-    std::max(ProgInfo.NumSGPR, 1u), STM.getMinNumSGPRs(MFI->getMaxWavesPerEU()));
-  ProgInfo.NumVGPRsForWavesPerEU = std::max(
-    std::max(ProgInfo.NumVGPR, 1u), STM.getMinNumVGPRs(MFI->getMaxWavesPerEU()));
+  unsigned MaxWaves = MFI->getMaxWavesPerEU();
+  ProgInfo.NumSGPRsForWavesPerEU = AMDGPUVariadicMCExpr::createMax(
+      {ProgInfo.NumSGPR, CreateExpr(1ul),
+       CreateExpr(STM.getMinNumSGPRs(MaxWaves))},
+      Ctx);
+  ProgInfo.NumVGPRsForWavesPerEU = AMDGPUVariadicMCExpr::createMax(
+      {ProgInfo.NumVGPR, CreateExpr(1ul),
+       CreateExpr(STM.getMinNumVGPRs(MaxWaves))},
+      Ctx);
 
   if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
       STM.hasSGPRInitBug()) {
     unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
-    if (ProgInfo.NumSGPR > MaxAddressableNumSGPRs) {
+    uint64_t NumSgpr;
+    if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
+        NumSgpr > MaxAddressableNumSGPRs) {
       // This can happen due to a compiler bug or when using inline asm to use
       // the registers which are usually reserved for vcc etc.
       LLVMContext &Ctx = MF.getFunction().getContext();
       DiagnosticInfoResourceLimit Diag(MF.getFunction(), "scalar registers",
-                                       ProgInfo.NumSGPR, MaxAddressableNumSGPRs,
+                                       NumSgpr, MaxAddressableNumSGPRs,
                                        DS_Error, DK_ResourceLimit);
       Ctx.diagnose(Diag);
-      ProgInfo.NumSGPR = MaxAddressableNumSGPRs;
-      ProgInfo.NumSGPRsForWavesPerEU = MaxAddressableNumSGPRs;
+      ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs);
+      ProgInfo.NumSGPRsForWavesPerEU = CreateExpr(MaxAddressableNumSGPRs);
     }
   }
 
   if (STM.hasSGPRInitBug()) {
     ProgInfo.NumSGPR =
-        AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
+        CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
     ProgInfo.NumSGPRsForWavesPerEU =
-        AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
+        CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
   }
 
   if (MFI->getNumUserSGPRs() > STM.getMaxNumUserSGPRs()) {
@@ -871,11 +923,26 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
         STM.getAddressableLocalMemorySize(), DS_Error);
     Ctx.diagnose(Diag);
   }
+  // The MCExpr equivalent of getNumSGPRBlocks/getNumVGPRBlocks:
+  // (alignTo(max(1u, NumGPR), GPREncodingGranule) / GPREncodingGranule) - 1
+  auto GetNumGPRBlocks = [&CreateExpr, &Ctx](const MCExpr *NumGPR,
+                                             unsigned Granule) {
+    const MCExpr *OneConst = CreateExpr(1ul);
+    const MCExpr *GranuleConst = CreateExpr(Granule);
+    const MCExpr *MaxNumGPR =
+        AMDGPUVariadicMCExpr::createMax({NumGPR, OneConst}, Ctx);
+    const MCExpr *AlignToGPR =
+        AMDGPUVariadicMCExpr::createAlignTo(MaxNumGPR, GranuleConst, Ctx);
+    const MCExpr *DivGPR =
+        MCBinaryExpr::createDiv(AlignToGPR, GranuleConst, Ctx);
+    const MCExpr *SubGPR = MCBinaryExpr::createSub(DivGPR, OneConst, Ctx);
+    return SubGPR;
+  };
 
-  ProgInfo.SGPRBlocks = IsaInfo::getNumSGPRBlocks(
-      &STM, ProgInfo.NumSGPRsForWavesPerEU);
-  ProgInfo.VGPRBlocks =
-      IsaInfo::getEncodedNumVGPRBlocks(&STM, ProgInfo.NumVGPRsForWavesPerEU);
+  ProgInfo.SGPRBlocks = GetNumGPRBlocks(ProgInfo.NumSGPRsForWavesPerEU,
+                                        IsaInfo::getSGPREncodingGranule(&STM));
+  ProgInfo.VGPRBlocks = GetNumGPRBlocks(ProgInfo.NumVGPRsForWavesPerEU,
+                                        IsaInfo::getVGPREncodingGranule(&STM));
 
   const SIModeRegisterDefaults Mode = MFI->getMode();
 
@@ -904,14 +971,23 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
   ProgInfo.LDSBlocks =
       alignTo(ProgInfo.LDSSize, 1ULL << LDSAlignShift) >> LDSAlignShift;
 
+  // The MCExpr equivalent of divideCeil.
+  auto DivideCeil = [&Ctx](const MCExpr *Numerator, const MCExpr *Denominator) {
+    const MCExpr *Ceil =
+        AMDGPUVariadicMCExpr::createAlignTo(Numerator, Denominator, Ctx);
+    return MCBinaryExpr::createDiv(Ceil, Denominator, Ctx);
+  };
+
   // Scratch is allocated in 64-dword or 256-dword blocks.
   unsigned ScratchAlignShift =
       STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 8 : 10;
   // We need to program the hardware with the amount of scratch memory that
   // is used by the entire wave.  ProgInfo.ScratchSize is the amount of
   // scratch memory used per thread.
-  ProgInfo.ScratchBlocks = divideCeil(
-      ProgInfo.ScratchSize * STM.getWavefrontSize(), 1ULL << ScratchAlignShift);
+  ProgInfo.ScratchBlocks = DivideCeil(
+      MCBinaryExpr::createMul(ProgInfo.ScratchSize,
+                              CreateExpr(STM.getWavefrontSize()), Ctx),
+      CreateExpr(1ULL << ScratchAlignShift));
 
   if (getIsaVersion(getGlobalSTI()->getCPU()).Major >= 10) {
     ProgInfo.WgpMode = STM.isCuModeEnabled() ? 0 : 1;
@@ -930,8 +1006,11 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
   // anything to disable it if we know the stack isn't used here. We may still
   // have emitted code reading it to initialize scratch, but if that's unused
   // reading garbage should be OK.
-  ProgInfo.ScratchEnable =
-      ProgInfo.ScratchBlocks > 0 || ProgInfo.DynamicCallStack;
+  ProgInfo.ScratchEnable = MCBinaryExpr::createLOr(
+      MCBinaryExpr::createGT(ProgInfo.ScratchBlocks,
+                             MCConstantExpr::create(0, Ctx), Ctx),
+      ProgInfo.DynamicCallStack, Ctx);
+
   ProgInfo.UserSGPR = MFI->getNumUserSGPRs();
   // For AMDHSA, TRAP_HANDLER must be zero, as it is populated by the CP.
   ProgInfo.TrapHandlerEnable =
@@ -947,26 +1026,41 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
   ProgInfo.EXCPEnable = 0;
 
   if (STM.hasGFX90AInsts()) {
-    AMDHSA_BITS_SET(ProgInfo.ComputePGMRSrc3GFX90A,
-                    amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
-                    ProgInfo.AccumOffset);
-    AMDHSA_BITS_SET(ProgInfo.ComputePGMRSrc3GFX90A,
-                    amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
-                    ProgInfo.TgSplit);
+    // return ((Dst & ~Mask) | (Value << Shift))
+    auto SetBits = [&Ctx](const MCExpr *Dst, const MCExpr *Value, uint32_t Mask,
+                          uint32_t Shift) {
+      auto Shft = MCConstantExpr::create(Shift, Ctx);
+      auto Msk = MCConstantExpr::create(Mask, Ctx);
+      Dst = MCBinaryExpr::createAnd(Dst, MCUnaryExpr::createNot(Msk, Ctx), Ctx);
+      Dst = MCBinaryExpr::createOr(
+          Dst, MCBinaryExpr::createShl(Value, Shft, Ctx), Ctx);
+      return Dst;
+    };
+
+    ProgInfo.ComputePGMRSrc3GFX90A =
+        SetBits(ProgInfo.ComputePGMRSrc3GFX90A, ProgInfo.AccumOffset,
+                amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
+                amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT);
+    ProgInfo.ComputePGMRSrc3GFX90A =
+        SetBits(ProgInfo.ComputePGMRSrc3GFX90A, CreateExpr(ProgInfo.TgSplit),
+                amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
+                amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT);
   }
 
-  ProgInfo.Occupancy = STM.computeOccupancy(MF.getFunction(), ProgInfo.LDSSize,
-                                            ProgInfo.NumSGPRsForWavesPerEU,
-                                            ProgInfo.NumVGPRsForWavesPerEU);
+  ProgInfo.Occupancy = AMDGPUVariadicMCExpr::createOccupancy(
+      STM.computeOccupancy(F, ProgInfo.LDSSize), ProgInfo.NumSGPRsForWavesPerEU,
+      ProgInfo.NumVGPRsForWavesPerEU, STM, Ctx);
+
   const auto [MinWEU, MaxWEU] =
       AMDGPU::getIntegerPairAttribute(F, "amdgpu-waves-per-eu", {0, 0}, true);
-  if (ProgInfo.Occupancy < MinWEU) {
+  uint64_t Occupancy;
+  if (TryGetMCExprValue(ProgInfo.Occupancy, Occupancy) && Occupancy < MinWEU) {
     DiagnosticInfoOptimizationFailure Diag(
         F, F.getSubprogram(),
         "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in "
         "'" +
             F.getName() + "': desired occupancy was " + Twine(MinWEU) +
-            ", final occupancy is " + Twine(ProgInfo.Occupancy));
+            ", final occupancy is " + Twine(Occupancy));
     F.getContext().diagnose(Diag);
   }
 }
@@ -989,36 +1083,78 @@ void AMDGPUAsmPrinter::EmitProgramInfoSI(const MachineFunction &MF,
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
   unsigned RsrcReg = getRsrcReg(MF.getFunction().getCallingConv());
+  MCContext &Ctx = MF.getContext();
+
+  // (((Value) & Mask) << Shift)
+  auto SetBits = [&Ctx](const MCExpr *Value, uint32_t Mask, uint32_t Shift) {
+    const MCExpr *msk = MCConstantExpr::create(Mask, Ctx);
+    const MCExpr *shft = MCConstantExpr::create(Shift, Ctx);
+    return MCBinaryExpr::createShl(MCBinaryExpr::createAnd(Value, msk, Ctx),
+                                   shft, Ctx);
+  };
+
+  auto EmitResolvedOrExpr = [this](const MCExpr *Value, unsigned Size) {
+    int64_t Val;
+    if (Value->evaluateAsAbsolute(Val))
+      OutStreamer->emitIntValue(static_cast<uint64_t>(Val), Size);
+    else
+      OutStreamer->emitValue(Value, Size);
+  };
 
   if (AMDGPU::isCompute(MF.getFunction().getCallingConv())) {
     OutStreamer->emitInt32(R_00B848_COMPUTE_PGM_RSRC1);
 
-    OutStreamer->emitInt32(CurrentProgramInfo.getComputePGMRSrc1(STM));
+    EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc1(STM, Ctx),
+                       /*Size=*/4);
 
     OutStreamer->emitInt32(R_00B84C_COMPUTE_PGM_RSRC2);
-    OutStreamer->emitInt32(CurrentProgramInfo.getComputePGMRSrc2());
+    EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc2(Ctx), /*Size=*/4);
 
     OutStreamer->emitInt32(R_00B860_COMPUTE_TMPRING_SIZE);
-    OutStreamer->emitInt32(
-        STM.getGeneration() >= AMDGPUSubtarget::GFX12
-            ? S_00B860_WAVESIZE_GFX12Plus(CurrentProgramInfo.ScratchBlocks)
-        : STM.getGeneration() == AMDGPUSubtarget::GFX11
-            ? S_00B860_WAVESIZE_GFX11(CurrentProgramInfo.ScratchBlocks)
-            : S_00B860_WAVESIZE_PreGFX11(CurrentProgramInfo.ScratchBlocks));
+
+    // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
+    // appropriate generation.
+    if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
+      EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
+                                 /*Mask=*/0x3FFFF, /*Shift=*/12),
+                         /*Size=*/4);
+    } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
+      EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
+                                 /*Mask=*/0x7FFF, /*Shift=*/12),
+                         /*Size=*/4);
+    } else {
+      EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
+                                 /*Mask=*/0x1FFF, /*Shift=*/12),
+                         /*Size=*/4);
+    }
 
     // TODO: Should probably note flat usage somewhere. SC emits a "FlatPtr32 =
     // 0" comment but I don't see a corresponding field in the register spec.
   } else {
     OutStreamer->emitInt32(RsrcReg);
-    OutStreamer->emitIntValue(S_00B028_VGPRS(CurrentProgramInfo.VGPRBlocks) |
-                              S_00B028_SGPRS(CurrentProgramInfo.SGPRBlocks), 4);
+
+    const MCExpr *GPRBlocks = MCBinaryExpr::createOr(
+        SetBits(CurrentProgramInfo.VGPRBlocks, /*Mask=*/0x3F, /*Shift=*/0),
+        SetBits(CurrentProgramInfo.SGPRBlocks, /*Mask=*/0x0F, /*Shift=*/6),
+        MF.getContext());
+    EmitResolvedOrExpr(GPRBlocks, /*Size=*/4);
     OutStreamer->emitInt32(R_0286E8_SPI_TMPRING_SIZE);
-    OutStreamer->emitInt32(
-        STM.getGeneration() >= AMDGPUSubtarget::GFX12
-            ? S_0286E8_WAVESIZE_GFX12Plus(CurrentProgramInfo.ScratchBlocks)
-        : STM.getGeneration() == AMDGPUSubtarget::GFX11
-            ? S_0286E8_WAVESIZE_GFX11(CurrentProgramInfo.ScratchBlocks)
-            : S_0286E8_WAVESIZE_PreGFX11(CurrentProgramInfo.ScratchBlocks));
+
+    // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
+    // appropriate generation.
+    if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
+      EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
+                                 /*Mask=*/0x3FFFF, /*Shift=*/12),
+                         /*Size=*/4);
+    } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
+      EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
+                                 /*Mask=*/0x7FFF, /*Shift=*/12),
+                         /*Size=*/4);
+    } else {
+      EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
+                                 /*Mask=*/0x1FFF, /*Shift=*/12),
+                         /*Size=*/4);
+    }
   }
 
   if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
@@ -1070,33 +1206,38 @@ void AMDGPUAsmPrinter::EmitPALMetadata(const MachineFunction &MF,
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   auto CC = MF.getFunction().getCallingConv();
   auto MD = getTargetStreamer()->getPALMetadata();
+  auto &Ctx = MF.getContext();
 
   MD->setEntryPoint(CC, MF.getFunction().getName());
-  MD->setNumUsedVgprs(CC, CurrentProgramInfo.NumVGPRsForWavesPerEU);
+  MD->setNumUsedVgprs(
+      CC, getMCExprValue(CurrentProgramInfo.NumVGPRsForWavesPerEU, Ctx));
 
   // Only set AGPRs for supported devices
   const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
   if (STM.hasMAIInsts()) {
-    MD->setNumUsedAgprs(CC, CurrentProgramInfo.NumAccVGPR);
+    MD->setNumUsedAgprs(CC, getMCExprValue(CurrentProgramInfo.NumAccVGPR, Ctx));
   }
 
-  MD->setNumUsedSgprs(CC, CurrentProgramInfo.NumSGPRsForWavesPerEU);
+  MD->setNumUsedSgprs(
+      CC, getMCExprValue(CurrentProgramInfo.NumSGPRsForWavesPerEU, Ctx));
   if (MD->getPALMajorVersion() < 3) {
     MD->setRsrc1(CC, CurrentProgramInfo.getPGMRSrc1(CC, STM));
     if (AMDGPU::isCompute(CC)) {
       MD->setRsrc2(CC, CurrentProgramInfo.getComputePGMRSrc2());
     } else {
-      if (CurrentProgramInfo.ScratchBlocks > 0)
+      if (getMCExprValue(CurrentProgramInfo.ScratchBlocks, Ctx) > 0)
         MD->setRsrc2(CC, S_00B84C_SCRATCH_EN(1));
     }
   } else {
     MD->setHwStage(CC, ".debug_mode", (bool)CurrentProgramInfo.DebugMode);
-    MD->setHwStage(CC, ".scratch_en", (bool)CurrentProgramInfo.ScratchEnable);
+    MD->setHwStage(CC, ".scratch_en",
+                   (bool)getMCExprValue(CurrentProgramInfo.ScratchEnable, Ctx));
     EmitPALMetadataCommon(MD, CurrentProgramInfo, CC, STM);
   }
 
   // ScratchSize is in bytes, 16 aligned.
-  MD->setScratchSize(CC, alignTo(CurrentProgramInfo.ScratchSize, 16));
+  MD->setScratchSize(
+      CC, alignTo(getMCExprValue(CurrentProgramInfo.ScratchSize, Ctx), 16));
   if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
     unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
                                 ? divideCeil(CurrentProgramInfo.LDSBlocks, 2)
@@ -1145,6 +1286,7 @@ void AMDGPUAsmPrinter::emitPALFunctionMetadata(const MachineFunction &MF) {
   StringRef FnName = MF.getFunction().getName();
   MD->setFunctionScratchSize(FnName, MFI.getStackSize());
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  MCContext &Ctx = MF.getContext();
 
   if (MD->getPALMajorVersion() < 3) {
     // Set compute registers
@@ -1158,8 +1300,10 @@ void AMDGPUAsmPrinter::emitPALFunctionMetadata(const MachineFunction &MF) {
 
   // Set optional info
   MD->setFunctionLdsSize(FnName, CurrentProgramInfo.LDSSize);
-  MD->setFunctionNumUsedVgprs(FnName, CurrentProgramInfo.NumVGPRsForWavesPerEU);
-  MD->setFunctionNumUsedSgprs(FnName, CurrentProgramInfo.NumSGPRsForWavesPerEU);
+  MD->setFunctionNumUsedVgprs(
+      FnName, getMCExprValue(CurrentProgramInfo.NumVGPRsForWavesPerEU, Ctx));
+  MD->setFunctionNumUsedSgprs(
+      FnName, getMCExprValue(CurrentProgramInfo.NumSGPRsForWavesPerEU, Ctx));
 }
 
 // This is supposed to be log2(Size)
@@ -1185,6 +1329,7 @@ void AMDGPUAsmPrinter::getAmdKernelCode(amd_kernel_code_t &Out,
 
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
+  MCContext &Ctx = MF.getContext();
 
   AMDGPU::initDefaultAMDKernelCodeT(Out, &STM);
 
@@ -1193,7 +1338,7 @@ void AMDGPUAsmPrinter::getAmdKernelCode(amd_kernel_code_t &Out,
       (CurrentProgramInfo.getComputePGMRSrc2() << 32);
   Out.code_properties |= AMD_CODE_PROPERTY_IS_PTR64;
 
-  if (CurrentProgramInfo.DynamicCallStack)
+  if (getMCExprValue(CurrentProgramInfo.DynamicCallStack, Ctx))
     Out.code_properties |= AMD_CODE_PROPERTY_IS_DYNAMIC_CALLSTACK;
 
   AMD_HSA_BITS_SET(Out.code_properties,
@@ -1229,9 +1374,10 @@ void AMDGPUAsmPrinter::getAmdKernelCode(amd_kernel_code_t &Out,
 
   Align MaxKernArgAlign;
   Out.kernarg_segment_byte_size = STM.getKernArgSegmentSize(F, MaxKernArgAlign);
-  Out.wavefront_sgpr_count = CurrentProgramInfo.NumSGPR;
-  Out.workitem_vgpr_count = CurrentProgramInfo.NumVGPR;
-  Out.workitem_private_segment_byte_size = CurrentProgramInfo.ScratchSize;
+  Out.wavefront_sgpr_count = getMCExprValue(CurrentProgramInfo.NumSGPR, Ctx);
+  Out.workitem_vgpr_count = getMCExprValue(CurrentProgramInfo.NumVGPR, Ctx);
+  Out.workitem_private_segment_byte_size =
+      getMCExprValue(CurrentProgramInfo.ScratchSize, Ctx);
   Out.workgroup_group_segment_byte_size = CurrentProgramInfo.LDSSize;
 
   // kernarg_segment_alignment is specified as log of the alignment.
@@ -1322,19 +1468,28 @@ void AMDGPUAsmPrinter::emitResourceUsageRemarks(
   // remarks to simulate newlines. If and when clang does accept newlines, this
   // formatting should be aggregated into one remark with newlines to avoid
   // printing multiple diagnostic location and diag opts.
+  MCContext &MCCtx = MF.getContext();
   EmitResourceUsageRemark("FunctionName", "Function Name",
                           MF.getFunction().getName());
-  EmitResourceUsageRemark("NumSGPR", "SGPRs", CurrentProgramInfo.NumSGPR);
-  EmitResourceUsageRemark("NumVGPR", "VGPRs", CurrentProgramInfo.NumArchVGPR);
-  if (hasMAIInsts)
-    EmitResourceUsageRemark("NumAGPR", "AGPRs", CurrentProgramInfo.NumAccVGPR);
-  EmitResourceUsageRemark("ScratchSize", "ScratchSize [bytes/lane]",
-                          CurrentProgramInfo.ScratchSize);
+  EmitResourceUsageRemark("NumSGPR", "SGPRs",
+                          getMCExprValue(CurrentProgramInfo.NumSGPR, MCCtx));
+  EmitResourceUsageRemark(
+      "NumVGPR", "VGPRs",
+      getMCExprValue(CurrentProgramInfo.NumArchVGPR, MCCtx));
+  if (hasMAIInsts) {
+    EmitResourceUsageRemark(
+        "NumAGPR", "AGPRs",
+        getMCExprValue(CurrentProgramInfo.NumAccVGPR, MCCtx));
+  }
+  EmitResourceUsageRemark(
+      "ScratchSize", "ScratchSize [bytes/lane]",
+      getMCExprValue(CurrentProgramInfo.ScratchSize, MCCtx));
   StringRef DynamicStackStr =
-      CurrentProgramInfo.DynamicCallStack ? "True" : "False";
+      getMCExprValue(CurrentProgramInfo.DynamicCallStack, MCCtx) ? "True"
+                                                                 : "False";
   EmitResourceUsageRemark("DynamicStack", "Dynamic Stack", DynamicStackStr);
   EmitResourceUsageRemark("Occupancy", "Occupancy [waves/SIMD]",
-                          CurrentProgramInfo.Occupancy);
+                          getMCExprValue(CurrentProgramInfo.Occupancy, MCCtx));
   EmitResourceUsageRemark("SGPRSpill", "SGPRs Spill",
                           CurrentProgramInfo.SGPRSpill);
   EmitResourceUsageRemark("VGPRSpill", "VGPRs Spill",
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
index b8b2718d293e..16d8952a533e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
@@ -78,6 +78,8 @@ private:
 
   void initTargetStreamer(Module &M);
 
+  static uint64_t getMCExprValue(const MCExpr *Value, MCContext &Ctx);
+
 public:
   explicit AMDGPUAsmPrinter(TargetMachine &TM,
                             std::unique_ptr<MCStreamer> Streamer);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
index 9e288ab50e17..7ab9ba285133 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
@@ -19,6 +19,8 @@
 #include "SIMachineFunctionInfo.h"
 #include "SIProgramInfo.h"
 #include "llvm/IR/Module.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
 using namespace llvm;
 
 static std::pair<Type *, Align> getArgumentTypeAlign(const Argument &Arg,
@@ -462,6 +464,16 @@ MetadataStreamerMsgPackV4::getHSAKernelProps(const MachineFunction &MF,
   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
   const Function &F = MF.getFunction();
 
+  auto GetMCExprValue = [&MF](const MCExpr *Value) {
+    int64_t Val;
+    if (!Value->evaluateAsAbsolute(Val)) {
+      MCContext &Ctx = MF.getContext();
+      Ctx.reportError(SMLoc(), "could not resolve expression when required.");
+      Val = 0;
+    }
+    return static_cast<uint64_t>(Val);
+  };
+
   auto Kern = HSAMetadataDoc->getMapNode();
 
   Align MaxKernArgAlign;
@@ -470,10 +482,11 @@ MetadataStreamerMsgPackV4::getHSAKernelProps(const MachineFunction &MF,
   Kern[".group_segment_fixed_size"] =
       Kern.getDocument()->getNode(ProgramInfo.LDSSize);
   Kern[".private_segment_fixed_size"] =
-      Kern.getDocument()->getNode(ProgramInfo.ScratchSize);
-  if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5)
-    Kern[".uses_dynamic_stack"] =
-        Kern.getDocument()->getNode(ProgramInfo.DynamicCallStack);
+      Kern.getDocument()->getNode(GetMCExprValue(ProgramInfo.ScratchSize));
+  if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5) {
+    Kern[".uses_dynamic_stack"] = Kern.getDocument()->getNode(
+        static_cast<bool>(GetMCExprValue(ProgramInfo.DynamicCallStack)));
+  }
 
   if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5 && STM.supportsWGP())
     Kern[".workgroup_processor_mode"] =
@@ -484,12 +497,15 @@ MetadataStreamerMsgPackV4::getHSAKernelProps(const MachineFunction &MF,
       Kern.getDocument()->getNode(std::max(Align(4), MaxKernArgAlign).value());
   Kern[".wavefront_size"] =
       Kern.getDocument()->getNode(STM.getWavefrontSize());
-  Kern[".sgpr_count"] = Kern.getDocument()->getNode(ProgramInfo.NumSGPR);
-  Kern[".vgpr_count"] = Kern.getDocument()->getNode(ProgramInfo.NumVGPR);
+  Kern[".sgpr_count"] =
+      Kern.getDocument()->getNode(GetMCExprValue(ProgramInfo.NumSGPR));
+  Kern[".vgpr_count"] =
+      Kern.getDocument()->getNode(GetMCExprValue(ProgramInfo.NumVGPR));
 
   // Only add AGPR count to metadata for supported devices
   if (STM.hasMAIInsts()) {
-    Kern[".agpr_count"] = Kern.getDocument()->getNode(ProgramInfo.NumAccVGPR);
+    Kern[".agpr_count"] =
+        Kern.getDocument()->getNode(GetMCExprValue(ProgramInfo.NumAccVGPR));
   }
 
   Kern[".max_flat_workgroup_size"] =
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 9cfe81e5288e..94ee4ac78142 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -674,29 +674,8 @@ bool GCNSubtarget::useVGPRIndexMode() const {
 bool GCNSubtarget::useAA() const { return UseAA; }
 
 unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
-  if (getGeneration() >= AMDGPUSubtarget::GFX10)
-    return getMaxWavesPerEU();
-
-  if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
-    if (SGPRs <= 80)
-      return 10;
-    if (SGPRs <= 88)
-      return 9;
-    if (SGPRs <= 100)
-      return 8;
-    return 7;
-  }
-  if (SGPRs <= 48)
-    return 10;
-  if (SGPRs <= 56)
-    return 9;
-  if (SGPRs <= 64)
-    return 8;
-  if (SGPRs <= 72)
-    return 7;
-  if (SGPRs <= 80)
-    return 6;
-  return 5;
+  return AMDGPU::IsaInfo::getOccupancyWithNumSGPRs(SGPRs, getMaxWavesPerEU(),
+                                                   getGeneration());
 }
 
 unsigned GCNSubtarget::getOccupancyWithNumVGPRs(unsigned NumVGPRs) const {
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 5ac245ac3b63..d47a5f8ebb81 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -8399,12 +8399,16 @@ bool AMDGPUAsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
     AGVK VK = StringSwitch<AGVK>(TokenId)
                   .Case("max", AGVK::AGVK_Max)
                   .Case("or", AGVK::AGVK_Or)
+                  .Case("extrasgprs", AGVK::AGVK_ExtraSGPRs)
+                  .Case("totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
+                  .Case("alignto", AGVK::AGVK_AlignTo)
+                  .Case("occupancy", AGVK::AGVK_Occupancy)
                   .Default(AGVK::AGVK_None);
 
     if (VK != AGVK::AGVK_None && peekToken().is(AsmToken::LParen)) {
       SmallVector<const MCExpr *, 4> Exprs;
       uint64_t CommaCount = 0;
-      lex(); // Eat 'max'/'or'
+      lex(); // Eat Arg ('or', 'max', 'occupancy', etc.)
       lex(); // Eat '('
       while (true) {
         if (trySkipToken(AsmToken::RParen)) {
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
index 4578c33d92dc..159664faf983 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
@@ -7,6 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPUMCExpr.h"
+#include "GCNSubtarget.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/IR/Function.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
@@ -16,6 +19,7 @@
 #include <optional>
 
 using namespace llvm;
+using namespace llvm::AMDGPU;
 
 AMDGPUVariadicMCExpr::AMDGPUVariadicMCExpr(VariadicKind Kind,
                                            ArrayRef<const MCExpr *> Args,
@@ -61,6 +65,18 @@ void AMDGPUVariadicMCExpr::printImpl(raw_ostream &OS,
   case AGVK_Max:
     OS << "max(";
     break;
+  case AGVK_ExtraSGPRs:
+    OS << "extrasgprs(";
+    break;
+  case AGVK_TotalNumVGPRs:
+    OS << "totalnumvgprs(";
+    break;
+  case AGVK_AlignTo:
+    OS << "alignto(";
+    break;
+  case AGVK_Occupancy:
+    OS << "occupancy(";
+    break;
   }
   for (auto It = Args.begin(); It != Args.end(); ++It) {
     (*It)->print(OS, MAI, /*InParens=*/false);
@@ -82,10 +98,151 @@ static int64_t op(AMDGPUVariadicMCExpr::VariadicKind Kind, int64_t Arg1,
   }
 }
 
+bool AMDGPUVariadicMCExpr::evaluateExtraSGPRs(MCValue &Res,
+                                              const MCAsmLayout *Layout,
+                                              const MCFixup *Fixup) const {
+  auto TryGetMCExprValue = [&](const MCExpr *Arg, uint64_t &ConstantValue) {
+    MCValue MCVal;
+    if (!Arg->evaluateAsRelocatable(MCVal, Layout, Fixup) ||
+        !MCVal.isAbsolute())
+      return false;
+
+    ConstantValue = MCVal.getConstant();
+    return true;
+  };
+
+  assert(Args.size() == 3 &&
+         "AMDGPUVariadic Argument count incorrect for ExtraSGPRs");
+  const MCSubtargetInfo *STI = Ctx.getSubtargetInfo();
+  uint64_t VCCUsed = 0, FlatScrUsed = 0, XNACKUsed = 0;
+
+  bool Success = TryGetMCExprValue(Args[2], XNACKUsed);
+
+  assert(Success && "Arguments 3 for ExtraSGPRs should be a known constant");
+  if (!Success || !TryGetMCExprValue(Args[0], VCCUsed) ||
+      !TryGetMCExprValue(Args[1], FlatScrUsed))
+    return false;
+
+  uint64_t ExtraSGPRs = IsaInfo::getNumExtraSGPRs(
+      STI, (bool)VCCUsed, (bool)FlatScrUsed, (bool)XNACKUsed);
+  Res = MCValue::get(ExtraSGPRs);
+  return true;
+}
+
+bool AMDGPUVariadicMCExpr::evaluateTotalNumVGPR(MCValue &Res,
+                                                const MCAsmLayout *Layout,
+                                                const MCFixup *Fixup) const {
+  auto TryGetMCExprValue = [&](const MCExpr *Arg, uint64_t &ConstantValue) {
+    MCValue MCVal;
+    if (!Arg->evaluateAsRelocatable(MCVal, Layout, Fixup) ||
+        !MCVal.isAbsolute())
+      return false;
+
+    ConstantValue = MCVal.getConstant();
+    return true;
+  };
+  assert(Args.size() == 2 &&
+         "AMDGPUVariadic Argument count incorrect for TotalNumVGPRs");
+  const MCSubtargetInfo *STI = Ctx.getSubtargetInfo();
+  uint64_t NumAGPR = 0, NumVGPR = 0;
+
+  bool Has90AInsts = AMDGPU::isGFX90A(*STI);
+
+  if (!TryGetMCExprValue(Args[0], NumAGPR) ||
+      !TryGetMCExprValue(Args[1], NumVGPR))
+    return false;
+
+  uint64_t TotalNum = Has90AInsts && NumAGPR ? alignTo(NumVGPR, 4) + NumAGPR
+                                             : std::max(NumVGPR, NumAGPR);
+  Res = MCValue::get(TotalNum);
+  return true;
+}
+
+bool AMDGPUVariadicMCExpr::evaluateAlignTo(MCValue &Res,
+                                           const MCAsmLayout *Layout,
+                                           const MCFixup *Fixup) const {
+  auto TryGetMCExprValue = [&](const MCExpr *Arg, uint64_t &ConstantValue) {
+    MCValue MCVal;
+    if (!Arg->evaluateAsRelocatable(MCVal, Layout, Fixup) ||
+        !MCVal.isAbsolute())
+      return false;
+
+    ConstantValue = MCVal.getConstant();
+    return true;
+  };
+
+  assert(Args.size() == 2 &&
+         "AMDGPUVariadic Argument count incorrect for AlignTo");
+  uint64_t Value = 0, Align = 0;
+  if (!TryGetMCExprValue(Args[0], Value) || !TryGetMCExprValue(Args[1], Align))
+    return false;
+
+  Res = MCValue::get(alignTo(Value, Align));
+  return true;
+}
+
+bool AMDGPUVariadicMCExpr::evaluateOccupancy(MCValue &Res,
+                                             const MCAsmLayout *Layout,
+                                             const MCFixup *Fixup) const {
+  auto TryGetMCExprValue = [&](const MCExpr *Arg, uint64_t &ConstantValue) {
+    MCValue MCVal;
+    if (!Arg->evaluateAsRelocatable(MCVal, Layout, Fixup) ||
+        !MCVal.isAbsolute())
+      return false;
+
+    ConstantValue = MCVal.getConstant();
+    return true;
+  };
+  assert(Args.size() == 7 &&
+         "AMDGPUVariadic Argument count incorrect for Occupancy");
+  uint64_t InitOccupancy, MaxWaves, Granule, TargetTotalNumVGPRs, Generation,
+      NumSGPRs, NumVGPRs;
+
+  bool Success = true;
+  Success &= TryGetMCExprValue(Args[0], MaxWaves);
+  Success &= TryGetMCExprValue(Args[1], Granule);
+  Success &= TryGetMCExprValue(Args[2], TargetTotalNumVGPRs);
+  Success &= TryGetMCExprValue(Args[3], Generation);
+  Success &= TryGetMCExprValue(Args[4], InitOccupancy);
+
+  assert(Success && "Arguments 1 to 5 for Occupancy should be known constants");
+
+  if (!Success || !TryGetMCExprValue(Args[5], NumSGPRs) ||
+      !TryGetMCExprValue(Args[6], NumVGPRs))
+    return false;
+
+  unsigned Occupancy = InitOccupancy;
+  if (NumSGPRs)
+    Occupancy = std::min(
+        Occupancy, IsaInfo::getOccupancyWithNumSGPRs(
+                       NumSGPRs, MaxWaves,
+                       static_cast<AMDGPUSubtarget::Generation>(Generation)));
+  if (NumVGPRs)
+    Occupancy = std::min(Occupancy,
+                         IsaInfo::getNumWavesPerEUWithNumVGPRs(
+                             NumVGPRs, Granule, MaxWaves, TargetTotalNumVGPRs));
+
+  Res = MCValue::get(Occupancy);
+  return true;
+}
+
 bool AMDGPUVariadicMCExpr::evaluateAsRelocatableImpl(
     MCValue &Res, const MCAsmLayout *Layout, const MCFixup *Fixup) const {
   std::optional<int64_t> Total;
 
+  switch (Kind) {
+  default:
+    break;
+  case AGVK_ExtraSGPRs:
+    return evaluateExtraSGPRs(Res, Layout, Fixup);
+  case AGVK_AlignTo:
+    return evaluateAlignTo(Res, Layout, Fixup);
+  case AGVK_TotalNumVGPRs:
+    return evaluateTotalNumVGPR(Res, Layout, Fixup);
+  case AGVK_Occupancy:
+    return evaluateOccupancy(Res, Layout, Fixup);
+  }
+
   for (const MCExpr *Arg : Args) {
     MCValue ArgRes;
     if (!Arg->evaluateAsRelocatable(ArgRes, Layout, Fixup) ||
@@ -113,3 +270,47 @@ MCFragment *AMDGPUVariadicMCExpr::findAssociatedFragment() const {
   }
   return nullptr;
 }
+
+/// Allow delayed MCExpr resolve of ExtraSGPRs (in case VCCUsed or FlatScrUsed
+/// are unresolvable but needed for further MCExprs). Derived from
+/// implementation of IsaInfo::getNumExtraSGPRs in AMDGPUBaseInfo.cpp.
+///
+const AMDGPUVariadicMCExpr *
+AMDGPUVariadicMCExpr::createExtraSGPRs(const MCExpr *VCCUsed,
+                                       const MCExpr *FlatScrUsed,
+                                       bool XNACKUsed, MCContext &Ctx) {
+
+  return create(AGVK_ExtraSGPRs,
+                {VCCUsed, FlatScrUsed, MCConstantExpr::create(XNACKUsed, Ctx)},
+                Ctx);
+}
+
+const AMDGPUVariadicMCExpr *AMDGPUVariadicMCExpr::createTotalNumVGPR(
+    const MCExpr *NumAGPR, const MCExpr *NumVGPR, MCContext &Ctx) {
+  return create(AGVK_TotalNumVGPRs, {NumAGPR, NumVGPR}, Ctx);
+}
+
+/// Mimics GCNSubtarget::computeOccupancy for MCExpr.
+///
+/// Remove dependency on GCNSubtarget and depend only only the necessary values
+/// for said occupancy computation. Should match computeOccupancy implementation
+/// without passing \p STM on.
+const AMDGPUVariadicMCExpr *
+AMDGPUVariadicMCExpr::createOccupancy(unsigned InitOcc, const MCExpr *NumSGPRs,
+                                      const MCExpr *NumVGPRs,
+                                      const GCNSubtarget &STM, MCContext &Ctx) {
+  unsigned MaxWaves = IsaInfo::getMaxWavesPerEU(&STM);
+  unsigned Granule = IsaInfo::getVGPRAllocGranule(&STM);
+  unsigned TargetTotalNumVGPRs = IsaInfo::getTotalNumVGPRs(&STM);
+  unsigned Generation = STM.getGeneration();
+
+  auto CreateExpr = [&Ctx](unsigned Value) {
+    return MCConstantExpr::create(Value, Ctx);
+  };
+
+  return create(AGVK_Occupancy,
+                {CreateExpr(MaxWaves), CreateExpr(Granule),
+                 CreateExpr(TargetTotalNumVGPRs), CreateExpr(Generation),
+                 CreateExpr(InitOcc), NumSGPRs, NumVGPRs},
+                Ctx);
+}
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
index 238e0dea791b..f92350b59235 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
@@ -14,6 +14,9 @@
 
 namespace llvm {
 
+class Function;
+class GCNSubtarget;
+
 /// AMDGPU target specific variadic MCExpr operations.
 ///
 /// Takes in a minimum of 1 argument to be used with an operation. The supported
@@ -26,7 +29,15 @@ namespace llvm {
 ///
 class AMDGPUVariadicMCExpr : public MCTargetExpr {
 public:
-  enum VariadicKind { AGVK_None, AGVK_Or, AGVK_Max };
+  enum VariadicKind {
+    AGVK_None,
+    AGVK_Or,
+    AGVK_Max,
+    AGVK_ExtraSGPRs,
+    AGVK_TotalNumVGPRs,
+    AGVK_AlignTo,
+    AGVK_Occupancy
+  };
 
 private:
   VariadicKind Kind;
@@ -38,6 +49,15 @@ private:
                        MCContext &Ctx);
   ~AMDGPUVariadicMCExpr();
 
+  bool evaluateExtraSGPRs(MCValue &Res, const MCAsmLayout *Layout,
+                          const MCFixup *Fixup) const;
+  bool evaluateTotalNumVGPR(MCValue &Res, const MCAsmLayout *Layout,
+                            const MCFixup *Fixup) const;
+  bool evaluateAlignTo(MCValue &Res, const MCAsmLayout *Layout,
+                       const MCFixup *Fixup) const;
+  bool evaluateOccupancy(MCValue &Res, const MCAsmLayout *Layout,
+                         const MCFixup *Fixup) const;
+
 public:
   static const AMDGPUVariadicMCExpr *
   create(VariadicKind Kind, ArrayRef<const MCExpr *> Args, MCContext &Ctx);
@@ -52,6 +72,26 @@ public:
     return create(VariadicKind::AGVK_Max, Args, Ctx);
   }
 
+  static const AMDGPUVariadicMCExpr *createExtraSGPRs(const MCExpr *VCCUsed,
+                                                      const MCExpr *FlatScrUsed,
+                                                      bool XNACKUsed,
+                                                      MCContext &Ctx);
+
+  static const AMDGPUVariadicMCExpr *createTotalNumVGPR(const MCExpr *NumAGPR,
+                                                        const MCExpr *NumVGPR,
+                                                        MCContext &Ctx);
+
+  static const AMDGPUVariadicMCExpr *
+  createAlignTo(const MCExpr *Value, const MCExpr *Align, MCContext &Ctx) {
+    return create(VariadicKind::AGVK_AlignTo, {Value, Align}, Ctx);
+  }
+
+  static const AMDGPUVariadicMCExpr *createOccupancy(unsigned InitOcc,
+                                                     const MCExpr *NumSGPRs,
+                                                     const MCExpr *NumVGPRs,
+                                                     const GCNSubtarget &STM,
+                                                     MCContext &Ctx);
+
   VariadicKind getKind() const { return Kind; }
   const MCExpr *getSubExpr(size_t Index) const;
 
diff --git a/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp b/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
index 9ed7aacc0538..0d40816cdd4b 100644
--- a/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
@@ -18,57 +18,114 @@
 #include "GCNSubtarget.h"
 #include "SIDefines.h"
 #include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/MC/MCExpr.h"
 
 using namespace llvm;
 
-uint64_t SIProgramInfo::getComputePGMRSrc1(const GCNSubtarget &ST) const {
-  uint64_t Reg = S_00B848_VGPRS(VGPRBlocks) | S_00B848_SGPRS(SGPRBlocks) |
-                 S_00B848_PRIORITY(Priority) | S_00B848_FLOAT_MODE(FloatMode) |
-                 S_00B848_PRIV(Priv) | S_00B848_DEBUG_MODE(DebugMode) |
-                 S_00B848_WGP_MODE(WgpMode) | S_00B848_MEM_ORDERED(MemOrdered);
+void SIProgramInfo::reset(const MachineFunction &MF) {
+  MCContext &Ctx = MF.getContext();
+
+  const MCExpr *ZeroExpr = MCConstantExpr::create(0, Ctx);
+
+  VGPRBlocks = ZeroExpr;
+  SGPRBlocks = ZeroExpr;
+  Priority = 0;
+  FloatMode = 0;
+  Priv = 0;
+  DX10Clamp = 0;
+  DebugMode = 0;
+  IEEEMode = 0;
+  WgpMode = 0;
+  MemOrdered = 0;
+  RrWgMode = 0;
+  ScratchSize = ZeroExpr;
+
+  LDSBlocks = 0;
+  ScratchBlocks = ZeroExpr;
+
+  ScratchEnable = ZeroExpr;
+  UserSGPR = 0;
+  TrapHandlerEnable = 0;
+  TGIdXEnable = 0;
+  TGIdYEnable = 0;
+  TGIdZEnable = 0;
+  TGSizeEnable = 0;
+  TIdIGCompCount = 0;
+  EXCPEnMSB = 0;
+  LdsSize = 0;
+  EXCPEnable = 0;
+
+  ComputePGMRSrc3GFX90A = ZeroExpr;
+
+  NumVGPR = ZeroExpr;
+  NumArchVGPR = ZeroExpr;
+  NumAccVGPR = ZeroExpr;
+  AccumOffset = ZeroExpr;
+  TgSplit = 0;
+  NumSGPR = ZeroExpr;
+  SGPRSpill = 0;
+  VGPRSpill = 0;
+  LDSSize = 0;
+  FlatUsed = ZeroExpr;
+
+  NumSGPRsForWavesPerEU = ZeroExpr;
+  NumVGPRsForWavesPerEU = ZeroExpr;
+  Occupancy = ZeroExpr;
+  DynamicCallStack = ZeroExpr;
+  VCCUsed = ZeroExpr;
+}
+
+static uint64_t getComputePGMRSrc1Reg(const SIProgramInfo &ProgInfo,
+                                      const GCNSubtarget &ST) {
+  uint64_t Reg = S_00B848_PRIORITY(ProgInfo.Priority) |
+                 S_00B848_FLOAT_MODE(ProgInfo.FloatMode) |
+                 S_00B848_PRIV(ProgInfo.Priv) |
+                 S_00B848_DEBUG_MODE(ProgInfo.DebugMode) |
+                 S_00B848_WGP_MODE(ProgInfo.WgpMode) |
+                 S_00B848_MEM_ORDERED(ProgInfo.MemOrdered);
 
   if (ST.hasDX10ClampMode())
-    Reg |= S_00B848_DX10_CLAMP(DX10Clamp);
+    Reg |= S_00B848_DX10_CLAMP(ProgInfo.DX10Clamp);
 
   if (ST.hasIEEEMode())
-    Reg |= S_00B848_IEEE_MODE(IEEEMode);
+    Reg |= S_00B848_IEEE_MODE(ProgInfo.IEEEMode);
 
   if (ST.hasRrWGMode())
-    Reg |= S_00B848_RR_WG_MODE(RrWgMode);
+    Reg |= S_00B848_RR_WG_MODE(ProgInfo.RrWgMode);
 
   return Reg;
 }
 
-uint64_t SIProgramInfo::getPGMRSrc1(CallingConv::ID CC,
-                                    const GCNSubtarget &ST) const {
-  if (AMDGPU::isCompute(CC)) {
-    return getComputePGMRSrc1(ST);
-  }
-  uint64_t Reg = S_00B848_VGPRS(VGPRBlocks) | S_00B848_SGPRS(SGPRBlocks) |
-                 S_00B848_PRIORITY(Priority) | S_00B848_FLOAT_MODE(FloatMode) |
-                 S_00B848_PRIV(Priv) | S_00B848_DEBUG_MODE(DebugMode);
+static uint64_t getPGMRSrc1Reg(const SIProgramInfo &ProgInfo,
+                               CallingConv::ID CC, const GCNSubtarget &ST) {
+  uint64_t Reg = S_00B848_PRIORITY(ProgInfo.Priority) |
+                 S_00B848_FLOAT_MODE(ProgInfo.FloatMode) |
+                 S_00B848_PRIV(ProgInfo.Priv) |
+                 S_00B848_DEBUG_MODE(ProgInfo.DebugMode);
 
   if (ST.hasDX10ClampMode())
-    Reg |= S_00B848_DX10_CLAMP(DX10Clamp);
+    Reg |= S_00B848_DX10_CLAMP(ProgInfo.DX10Clamp);
 
   if (ST.hasIEEEMode())
-    Reg |= S_00B848_IEEE_MODE(IEEEMode);
+    Reg |= S_00B848_IEEE_MODE(ProgInfo.IEEEMode);
 
   if (ST.hasRrWGMode())
-    Reg |= S_00B848_RR_WG_MODE(RrWgMode);
+    Reg |= S_00B848_RR_WG_MODE(ProgInfo.RrWgMode);
 
   switch (CC) {
   case CallingConv::AMDGPU_PS:
-    Reg |= S_00B028_MEM_ORDERED(MemOrdered);
+    Reg |= S_00B028_MEM_ORDERED(ProgInfo.MemOrdered);
     break;
   case CallingConv::AMDGPU_VS:
-    Reg |= S_00B128_MEM_ORDERED(MemOrdered);
+    Reg |= S_00B128_MEM_ORDERED(ProgInfo.MemOrdered);
     break;
   case CallingConv::AMDGPU_GS:
-    Reg |= S_00B228_WGP_MODE(WgpMode) | S_00B228_MEM_ORDERED(MemOrdered);
+    Reg |= S_00B228_WGP_MODE(ProgInfo.WgpMode) |
+           S_00B228_MEM_ORDERED(ProgInfo.MemOrdered);
     break;
   case CallingConv::AMDGPU_HS:
-    Reg |= S_00B428_WGP_MODE(WgpMode) | S_00B428_MEM_ORDERED(MemOrdered);
+    Reg |= S_00B428_WGP_MODE(ProgInfo.WgpMode) |
+           S_00B428_MEM_ORDERED(ProgInfo.MemOrdered);
     break;
   default:
     break;
@@ -76,22 +133,108 @@ uint64_t SIProgramInfo::getPGMRSrc1(CallingConv::ID CC,
   return Reg;
 }
 
-uint64_t SIProgramInfo::getComputePGMRSrc2() const {
-  uint64_t Reg =
-      S_00B84C_SCRATCH_EN(ScratchEnable) | S_00B84C_USER_SGPR(UserSGPR) |
-      S_00B84C_TRAP_HANDLER(TrapHandlerEnable) |
-      S_00B84C_TGID_X_EN(TGIdXEnable) | S_00B84C_TGID_Y_EN(TGIdYEnable) |
-      S_00B84C_TGID_Z_EN(TGIdZEnable) | S_00B84C_TG_SIZE_EN(TGSizeEnable) |
-      S_00B84C_TIDIG_COMP_CNT(TIdIGCompCount) |
-      S_00B84C_EXCP_EN_MSB(EXCPEnMSB) | S_00B84C_LDS_SIZE(LdsSize) |
-      S_00B84C_EXCP_EN(EXCPEnable);
+static uint64_t getComputePGMRSrc2Reg(const SIProgramInfo &ProgInfo) {
+  uint64_t Reg = S_00B84C_USER_SGPR(ProgInfo.UserSGPR) |
+                 S_00B84C_TRAP_HANDLER(ProgInfo.TrapHandlerEnable) |
+                 S_00B84C_TGID_X_EN(ProgInfo.TGIdXEnable) |
+                 S_00B84C_TGID_Y_EN(ProgInfo.TGIdYEnable) |
+                 S_00B84C_TGID_Z_EN(ProgInfo.TGIdZEnable) |
+                 S_00B84C_TG_SIZE_EN(ProgInfo.TGSizeEnable) |
+                 S_00B84C_TIDIG_COMP_CNT(ProgInfo.TIdIGCompCount) |
+                 S_00B84C_EXCP_EN_MSB(ProgInfo.EXCPEnMSB) |
+                 S_00B84C_LDS_SIZE(ProgInfo.LdsSize) |
+                 S_00B84C_EXCP_EN(ProgInfo.EXCPEnable);
+
+  return Reg;
+}
+
+static const MCExpr *MaskShift(const MCExpr *Val, uint32_t Mask, uint32_t Shift,
+                               MCContext &Ctx) {
+  if (Mask) {
+    const MCExpr *MaskExpr = MCConstantExpr::create(Mask, Ctx);
+    Val = MCBinaryExpr::createAnd(Val, MaskExpr, Ctx);
+  }
+  if (Shift) {
+    const MCExpr *ShiftExpr = MCConstantExpr::create(Shift, Ctx);
+    Val = MCBinaryExpr::createShl(Val, ShiftExpr, Ctx);
+  }
+  return Val;
+}
+
+uint64_t SIProgramInfo::getComputePGMRSrc1(const GCNSubtarget &ST) const {
+  int64_t VBlocks, SBlocks;
+  VGPRBlocks->evaluateAsAbsolute(VBlocks);
+  SGPRBlocks->evaluateAsAbsolute(SBlocks);
+
+  uint64_t Reg = S_00B848_VGPRS(static_cast<uint64_t>(VBlocks)) |
+                 S_00B848_SGPRS(static_cast<uint64_t>(SBlocks)) |
+                 getComputePGMRSrc1Reg(*this, ST);
 
   return Reg;
 }
 
+uint64_t SIProgramInfo::getPGMRSrc1(CallingConv::ID CC,
+                                    const GCNSubtarget &ST) const {
+  if (AMDGPU::isCompute(CC)) {
+    return getComputePGMRSrc1(ST);
+  }
+  int64_t VBlocks, SBlocks;
+  VGPRBlocks->evaluateAsAbsolute(VBlocks);
+  SGPRBlocks->evaluateAsAbsolute(SBlocks);
+
+  return getPGMRSrc1Reg(*this, CC, ST) |
+         S_00B848_VGPRS(static_cast<uint64_t>(VBlocks)) |
+         S_00B848_SGPRS(static_cast<uint64_t>(SBlocks));
+}
+
+uint64_t SIProgramInfo::getComputePGMRSrc2() const {
+  int64_t ScratchEn;
+  ScratchEnable->evaluateAsAbsolute(ScratchEn);
+  return ScratchEn | getComputePGMRSrc2Reg(*this);
+}
+
 uint64_t SIProgramInfo::getPGMRSrc2(CallingConv::ID CC) const {
   if (AMDGPU::isCompute(CC))
     return getComputePGMRSrc2();
 
   return 0;
 }
+
+const MCExpr *SIProgramInfo::getComputePGMRSrc1(const GCNSubtarget &ST,
+                                                MCContext &Ctx) const {
+  uint64_t Reg = getComputePGMRSrc1Reg(*this, ST);
+  const MCExpr *RegExpr = MCConstantExpr::create(Reg, Ctx);
+  const MCExpr *Res = MCBinaryExpr::createOr(
+      MaskShift(VGPRBlocks, /*Mask=*/0x3F, /*Shift=*/0, Ctx),
+      MaskShift(SGPRBlocks, /*Mask=*/0xF, /*Shift=*/6, Ctx), Ctx);
+  return MCBinaryExpr::createOr(RegExpr, Res, Ctx);
+}
+
+const MCExpr *SIProgramInfo::getPGMRSrc1(CallingConv::ID CC,
+                                         const GCNSubtarget &ST,
+                                         MCContext &Ctx) const {
+  if (AMDGPU::isCompute(CC)) {
+    return getComputePGMRSrc1(ST, Ctx);
+  }
+
+  uint64_t Reg = getPGMRSrc1Reg(*this, CC, ST);
+  const MCExpr *RegExpr = MCConstantExpr::create(Reg, Ctx);
+  const MCExpr *Res = MCBinaryExpr::createOr(
+      MaskShift(VGPRBlocks, /*Mask=*/0x3F, /*Shift=*/0, Ctx),
+      MaskShift(SGPRBlocks, /*Mask=*/0xF, /*Shift=*/6, Ctx), Ctx);
+  return MCBinaryExpr::createOr(RegExpr, Res, Ctx);
+}
+
+const MCExpr *SIProgramInfo::getComputePGMRSrc2(MCContext &Ctx) const {
+  uint64_t Reg = getComputePGMRSrc2Reg(*this);
+  const MCExpr *RegExpr = MCConstantExpr::create(Reg, Ctx);
+  return MCBinaryExpr::createOr(ScratchEnable, RegExpr, Ctx);
+}
+
+const MCExpr *SIProgramInfo::getPGMRSrc2(CallingConv::ID CC,
+                                         MCContext &Ctx) const {
+  if (AMDGPU::isCompute(CC))
+    return getComputePGMRSrc2(Ctx);
+
+  return MCConstantExpr::create(0, Ctx);
+}
diff --git a/llvm/lib/Target/AMDGPU/SIProgramInfo.h b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
index 8c26789f936c..c0a353033c3c 100644
--- a/llvm/lib/Target/AMDGPU/SIProgramInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
@@ -22,12 +22,15 @@
 namespace llvm {
 
 class GCNSubtarget;
+class MCContext;
+class MCExpr;
+class MachineFunction;
 
 /// Track resource usage for kernels / entry functions.
 struct SIProgramInfo {
     // Fields set in PGM_RSRC1 pm4 packet.
-    uint32_t VGPRBlocks = 0;
-    uint32_t SGPRBlocks = 0;
+    const MCExpr *VGPRBlocks = nullptr;
+    const MCExpr *SGPRBlocks = nullptr;
     uint32_t Priority = 0;
     uint32_t FloatMode = 0;
     uint32_t Priv = 0;
@@ -37,14 +40,14 @@ struct SIProgramInfo {
     uint32_t WgpMode = 0; // GFX10+
     uint32_t MemOrdered = 0; // GFX10+
     uint32_t RrWgMode = 0;   // GFX12+
-    uint64_t ScratchSize = 0;
+    const MCExpr *ScratchSize = nullptr;
 
     // State used to calculate fields set in PGM_RSRC2 pm4 packet.
     uint32_t LDSBlocks = 0;
-    uint32_t ScratchBlocks = 0;
+    const MCExpr *ScratchBlocks = nullptr;
 
     // Fields set in PGM_RSRC2 pm4 packet
-    uint32_t ScratchEnable = 0;
+    const MCExpr *ScratchEnable = nullptr;
     uint32_t UserSGPR = 0;
     uint32_t TrapHandlerEnable = 0;
     uint32_t TGIdXEnable = 0;
@@ -56,44 +59,56 @@ struct SIProgramInfo {
     uint32_t LdsSize = 0;
     uint32_t EXCPEnable = 0;
 
-    uint64_t ComputePGMRSrc3GFX90A = 0;
+    const MCExpr *ComputePGMRSrc3GFX90A = nullptr;
 
-    uint32_t NumVGPR = 0;
-    uint32_t NumArchVGPR = 0;
-    uint32_t NumAccVGPR = 0;
-    uint32_t AccumOffset = 0;
+    const MCExpr *NumVGPR = nullptr;
+    const MCExpr *NumArchVGPR = nullptr;
+    const MCExpr *NumAccVGPR = nullptr;
+    const MCExpr *AccumOffset = nullptr;
     uint32_t TgSplit = 0;
-    uint32_t NumSGPR = 0;
+    const MCExpr *NumSGPR = nullptr;
     unsigned SGPRSpill = 0;
     unsigned VGPRSpill = 0;
     uint32_t LDSSize = 0;
-    bool FlatUsed = false;
+    const MCExpr *FlatUsed = nullptr;
 
     // Number of SGPRs that meets number of waves per execution unit request.
-    uint32_t NumSGPRsForWavesPerEU = 0;
+    const MCExpr *NumSGPRsForWavesPerEU = nullptr;
 
     // Number of VGPRs that meets number of waves per execution unit request.
-    uint32_t NumVGPRsForWavesPerEU = 0;
+    const MCExpr *NumVGPRsForWavesPerEU = nullptr;
 
     // Final occupancy.
-    uint32_t Occupancy = 0;
+    const MCExpr *Occupancy = nullptr;
 
     // Whether there is recursion, dynamic allocas, indirect calls or some other
     // reason there may be statically unknown stack usage.
-    bool DynamicCallStack = false;
+    const MCExpr *DynamicCallStack = nullptr;
 
     // Bonus information for debugging.
-    bool VCCUsed = false;
+    const MCExpr *VCCUsed = nullptr;
 
     SIProgramInfo() = default;
 
+    // The constructor sets the values for each member as shown in the struct.
+    // However, setting the MCExpr members to their zero value equivalent
+    // happens in reset together with (duplicated) value re-set for the
+    // non-MCExpr members.
+    void reset(const MachineFunction &MF);
+
     /// Compute the value of the ComputePGMRsrc1 register.
     uint64_t getComputePGMRSrc1(const GCNSubtarget &ST) const;
     uint64_t getPGMRSrc1(CallingConv::ID CC, const GCNSubtarget &ST) const;
+    const MCExpr *getComputePGMRSrc1(const GCNSubtarget &ST,
+                                     MCContext &Ctx) const;
+    const MCExpr *getPGMRSrc1(CallingConv::ID CC, const GCNSubtarget &ST,
+                              MCContext &Ctx) const;
 
     /// Compute the value of the ComputePGMRsrc2 register.
     uint64_t getComputePGMRSrc2() const;
     uint64_t getPGMRSrc2(CallingConv::ID CC) const;
+    const MCExpr *getComputePGMRSrc2(MCContext &Ctx) const;
+    const MCExpr *getPGMRSrc2(CallingConv::ID CC, MCContext &Ctx) const;
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 2fae7a31d70b..2beaf903542b 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1129,12 +1129,45 @@ unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) {
 
 unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo *STI,
                                       unsigned NumVGPRs) {
-  unsigned MaxWaves = getMaxWavesPerEU(STI);
-  unsigned Granule = getVGPRAllocGranule(STI);
+  return getNumWavesPerEUWithNumVGPRs(NumVGPRs, getVGPRAllocGranule(STI),
+                                      getMaxWavesPerEU(STI),
+                                      getTotalNumVGPRs(STI));
+}
+
+unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
+                                      unsigned MaxWaves,
+                                      unsigned TotalNumVGPRs) {
   if (NumVGPRs < Granule)
     return MaxWaves;
   unsigned RoundedRegs = alignTo(NumVGPRs, Granule);
-  return std::min(std::max(getTotalNumVGPRs(STI) / RoundedRegs, 1u), MaxWaves);
+  return std::min(std::max(TotalNumVGPRs / RoundedRegs, 1u), MaxWaves);
+}
+
+unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
+                                  AMDGPUSubtarget::Generation Gen) {
+  if (Gen >= AMDGPUSubtarget::GFX10)
+    return MaxWaves;
+
+  if (Gen >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
+    if (SGPRs <= 80)
+      return 10;
+    if (SGPRs <= 88)
+      return 9;
+    if (SGPRs <= 100)
+      return 8;
+    return 7;
+  }
+  if (SGPRs <= 48)
+    return 10;
+  if (SGPRs <= 56)
+    return 9;
+  if (SGPRs <= 64)
+    return 8;
+  if (SGPRs <= 72)
+    return 7;
+  if (SGPRs <= 80)
+    return 6;
+  return 5;
 }
 
 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 12d1b3a55ccc..fc4147df76e3 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUBASEINFO_H
 #define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUBASEINFO_H
 
+#include "AMDGPUSubtarget.h"
 #include "SIDefines.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/InstrTypes.h"
@@ -311,6 +312,17 @@ unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU);
 unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo *STI,
                                       unsigned NumVGPRs);
 
+/// \returns Number of waves reachable for a given \p NumVGPRs usage, \p Granule
+/// size, \p MaxWaves possible, and \p TotalNumVGPRs available.
+unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
+                                      unsigned MaxWaves,
+                                      unsigned TotalNumVGPRs);
+
+/// \returns Occupancy for a given \p SGPRs usage, \p MaxWaves possible, and \p
+/// Gen.
+unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
+                                  AMDGPUSubtarget::Generation Gen);
+
 /// \returns Number of VGPR blocks needed for given subtarget \p STI when
 /// \p NumVGPRs are used. We actually return the number of blocks -1, since
 /// that's what we encode.
diff --git a/llvm/test/MC/AMDGPU/alignto_mcexpr.s b/llvm/test/MC/AMDGPU/alignto_mcexpr.s
new file mode 100644
index 000000000000..e864f3736828
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/alignto_mcexpr.s
@@ -0,0 +1,15 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa < %s | FileCheck --check-prefix=ASM %s
+
+// ASM: .set alignto_zero_eight, 0
+// ASM: .set alignto_one_eight, 8
+// ASM: .set alignto_five_eight, 8
+// ASM: .set alignto_seven_eight, 8
+// ASM: .set alignto_eight_eight, 8
+// ASM: .set alignto_ten_eight, 16
+
+.set alignto_zero_eight, alignto(0, 8)
+.set alignto_one_eight, alignto(1, 8)
+.set alignto_five_eight, alignto(5, 8)
+.set alignto_seven_eight, alignto(7, 8)
+.set alignto_eight_eight, alignto(8, 8)
+.set alignto_ten_eight, alignto(10, 8)
diff --git a/llvm/test/MC/AMDGPU/extrasgprs_mcexpr.s b/llvm/test/MC/AMDGPU/extrasgprs_mcexpr.s
new file mode 100644
index 000000000000..e88b23bb34d4
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/extrasgprs_mcexpr.s
@@ -0,0 +1,31 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=bonaire < %s | FileCheck --check-prefix=GFX7 %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=GFX90A %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx940 < %s | FileCheck --check-prefix=GFX940 %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck --check-prefix=GFX10 %s
+
+// gfx940 has architected flat scratch enabled.
+
+// GFX7: .set extrasgpr_none, 0
+// GFX7: .set extrasgpr_vcc, 2
+// GFX7: .set extrasgpr_flatscr, 4
+// GFX7: .set extrasgpr_xnack, 0
+
+// GFX90A: .set extrasgpr_none, 0
+// GFX90A: .set extrasgpr_vcc, 2
+// GFX90A: .set extrasgpr_flatscr, 6
+// GFX90A: .set extrasgpr_xnack, 4
+
+// GFX940: .set extrasgpr_none, 6
+// GFX940: .set extrasgpr_vcc, 6
+// GFX940: .set extrasgpr_flatscr, 6
+// GFX940: .set extrasgpr_xnack, 6
+
+// GFX10: .set extrasgpr_none, 0
+// GFX10: .set extrasgpr_vcc, 2
+// GFX10: .set extrasgpr_flatscr, 0
+// GFX10: .set extrasgpr_xnack, 0
+
+.set extrasgpr_none, extrasgprs(0, 0, 0)
+.set extrasgpr_vcc, extrasgprs(1, 0, 0)
+.set extrasgpr_flatscr, extrasgprs(0, 1, 0)
+.set extrasgpr_xnack, extrasgprs(0, 0, 1)
diff --git a/llvm/test/MC/AMDGPU/occupancy_mcexpr.s b/llvm/test/MC/AMDGPU/occupancy_mcexpr.s
new file mode 100644
index 000000000000..06bec8c538da
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/occupancy_mcexpr.s
@@ -0,0 +1,61 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa < %s | FileCheck --check-prefix=ASM %s
+
+// ASM: .set occupancy_init_one, 1
+// ASM: .set occupancy_init_seven, 7
+// ASM: .set occupancy_init_eight, 8
+
+.set occupancy_init_one, occupancy(0, 0, 0, 0, 1, 0, 0)
+.set occupancy_init_seven, occupancy(0, 0, 0, 0, 7, 0, 0)
+.set occupancy_init_eight, occupancy(0, 0, 0, 0, 8, 0, 0)
+
+// ASM: .set occupancy_numsgpr_seaisle_ten, 10
+// ASM: .set occupancy_numsgpr_seaisle_nine, 9
+// ASM: .set occupancy_numsgpr_seaisle_eight, 8
+// ASM: .set occupancy_numsgpr_seaisle_seven, 7
+// ASM: .set occupancy_numsgpr_seaisle_six, 6
+// ASM: .set occupancy_numsgpr_seaisle_five, 5
+
+.set occupancy_numsgpr_seaisle_ten, occupancy(0, 0, 0, 6, 11, 1, 0)
+.set occupancy_numsgpr_seaisle_nine, occupancy(0, 0, 0, 6, 11, 49, 0)
+.set occupancy_numsgpr_seaisle_eight, occupancy(0, 0, 0, 6, 11, 57, 0)
+.set occupancy_numsgpr_seaisle_seven, occupancy(0, 0, 0, 6, 11, 65, 0)
+.set occupancy_numsgpr_seaisle_six, occupancy(0, 0, 0, 6, 11, 73, 0)
+.set occupancy_numsgpr_seaisle_five, occupancy(0, 0, 0, 6, 11, 81, 0)
+
+// ASM: .set occupancy_numsgpr_gfx9_ten, 10
+// ASM: .set occupancy_numsgpr_gfx9_nine, 9
+// ASM: .set occupancy_numsgpr_gfx9_eight, 8
+// ASM: .set occupancy_numsgpr_gfx9_seven, 7
+
+.set occupancy_numsgpr_gfx9_ten, occupancy(0, 0, 0, 8, 11, 1, 0)
+.set occupancy_numsgpr_gfx9_nine, occupancy(0, 0, 0, 8, 11, 81, 0)
+.set occupancy_numsgpr_gfx9_eight, occupancy(0, 0, 0, 8, 11, 89, 0)
+.set occupancy_numsgpr_gfx9_seven, occupancy(0, 0, 0, 8, 11, 101, 0)
+
+// ASM: .set occupancy_numsgpr_gfx10_one, 1
+// ASM: .set occupancy_numsgpr_gfx10_seven, 7
+// ASM: .set occupancy_numsgpr_gfx10_eight, 8
+
+.set occupancy_numsgpr_gfx10_one, occupancy(1, 0, 0, 9, 11, 1, 0)
+.set occupancy_numsgpr_gfx10_seven, occupancy(7, 0, 0, 9, 11, 1, 0)
+.set occupancy_numsgpr_gfx10_eight, occupancy(8, 0, 0, 9, 11, 1, 0)
+
+// ASM: .set occupancy_numvgpr_high_granule_one, 1
+// ASM: .set occupancy_numvgpr_high_granule_seven, 7
+// ASM: .set occupancy_numvgpr_high_granule_eight, 8
+
+.set occupancy_numvgpr_high_granule_one, occupancy(1, 2, 0, 0, 11, 0, 1)
+.set occupancy_numvgpr_high_granule_seven, occupancy(7, 2, 0, 0, 11, 0, 1)
+.set occupancy_numvgpr_high_granule_eight, occupancy(8, 2, 0, 0, 11, 0, 1)
+
+// ASM: .set occupancy_numvgpr_low_total_one, 1
+// ASM: .set occupancy_numvgpr_one, 1
+// ASM: .set occupancy_numvgpr_seven, 7
+// ASM: .set occupancy_numvgpr_eight, 8
+// ASM: .set occupancy_numvgpr_ten, 10
+
+.set occupancy_numvgpr_low_total_one, occupancy(11, 4, 2, 0, 11, 0, 4)
+.set occupancy_numvgpr_one, occupancy(11, 4, 4, 0, 11, 0, 4)
+.set occupancy_numvgpr_seven, occupancy(11, 4, 28, 0, 11, 0, 4)
+.set occupancy_numvgpr_eight, occupancy(11, 4, 32, 0, 11, 0, 4)
+.set occupancy_numvgpr_ten, occupancy(11, 4, 40, 0, 11, 0, 4)
diff --git a/llvm/test/MC/AMDGPU/totalnumvgpr_mcexpr.s b/llvm/test/MC/AMDGPU/totalnumvgpr_mcexpr.s
new file mode 100644
index 000000000000..29bb885b2080
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/totalnumvgpr_mcexpr.s
@@ -0,0 +1,26 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=GFX90A %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck --check-prefix=GFX10 %s
+
+// GFX10: .set totalvgpr_none, 0
+// GFX10: .set totalvgpr_one, 1
+// GFX10: .set totalvgpr_two, 2
+
+.set totalvgpr_none, totalnumvgprs(0, 0)
+.set totalvgpr_one, totalnumvgprs(1, 0)
+.set totalvgpr_two, totalnumvgprs(1, 2)
+
+// GFX90A: .set totalvgpr90a_none, 0
+// GFX90A: .set totalvgpr90a_one, 1
+// GFX90A: .set totalvgpr90a_two, 2
+
+.set totalvgpr90a_none, totalnumvgprs(0, 0)
+.set totalvgpr90a_one, totalnumvgprs(0, 1)
+.set totalvgpr90a_two, totalnumvgprs(0, 2)
+
+// GFX90A: .set totalvgpr90a_agpr_minimal, 1
+// GFX90A: .set totalvgpr90a_agpr_rounded_eight, 8
+// GFX90A: .set totalvgpr90a_agpr_exact_eight, 8
+
+.set totalvgpr90a_agpr_minimal, totalnumvgprs(1, 0)
+.set totalvgpr90a_agpr_rounded_eight, totalnumvgprs(4, 2)
+.set totalvgpr90a_agpr_exact_eight, totalnumvgprs(4, 4)
diff --git a/llvm/unittests/MC/AMDGPU/CMakeLists.txt b/llvm/unittests/MC/AMDGPU/CMakeLists.txt
index 06ca89a72a7c..be8ff572e6f7 100644
--- a/llvm/unittests/MC/AMDGPU/CMakeLists.txt
+++ b/llvm/unittests/MC/AMDGPU/CMakeLists.txt
@@ -1,12 +1,20 @@
+include_directories(
+  ${PROJECT_SOURCE_DIR}/lib/Target/AMDGPU
+  ${PROJECT_BINARY_DIR}/lib/Target/AMDGPU
+  )
+
 set(LLVM_LINK_COMPONENTS
   AMDGPUCodeGen
   AMDGPUDesc
   AMDGPUInfo
+  CodeGen
+  Core
   MC
   Support
   TargetParser
   )
 
-add_llvm_unittest(AMDGPUDwarfTests
+add_llvm_unittest(AMDGPUMCTests
   DwarfRegMappings.cpp
+  SIProgramInfoMCExprs.cpp
   )
diff --git a/llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp b/llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp
new file mode 100644
index 000000000000..f2161f71e6e9
--- /dev/null
+++ b/llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp
@@ -0,0 +1,81 @@
+//===- llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUHSAMetadataStreamer.h"
+#include "AMDGPUTargetMachine.h"
+#include "GCNSubtarget.h"
+#include "SIProgramInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCTargetOptions.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Target/TargetMachine.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+class SIProgramInfoMCExprsTest : public testing::Test {
+protected:
+  std::unique_ptr<GCNTargetMachine> TM;
+  std::unique_ptr<LLVMContext> Ctx;
+  std::unique_ptr<GCNSubtarget> ST;
+  std::unique_ptr<MachineModuleInfo> MMI;
+  std::unique_ptr<MachineFunction> MF;
+  std::unique_ptr<Module> M;
+
+  SIProgramInfo PI;
+
+  static void SetUpTestSuite() {
+    LLVMInitializeAMDGPUTargetInfo();
+    LLVMInitializeAMDGPUTarget();
+    LLVMInitializeAMDGPUTargetMC();
+  }
+
+  SIProgramInfoMCExprsTest() {
+    std::string Triple = "amdgcn-amd-amdhsa";
+    std::string CPU = "gfx1010";
+    std::string FS = "";
+
+    std::string Error;
+    const Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error);
+    TargetOptions Options;
+
+    TM.reset(static_cast<GCNTargetMachine *>(TheTarget->createTargetMachine(
+        Triple, CPU, FS, Options, std::nullopt, std::nullopt)));
+
+    Ctx = std::make_unique<LLVMContext>();
+    M = std::make_unique<Module>("Module", *Ctx);
+    M->setDataLayout(TM->createDataLayout());
+    auto *FType = FunctionType::get(Type::getVoidTy(*Ctx), false);
+    auto *F = Function::Create(FType, GlobalValue::ExternalLinkage, "Test", *M);
+    MMI = std::make_unique<MachineModuleInfo>(TM.get());
+
+    ST = std::make_unique<GCNSubtarget>(TM->getTargetTriple(),
+                                        TM->getTargetCPU(),
+                                        TM->getTargetFeatureString(), *TM);
+
+    MF = std::make_unique<MachineFunction>(*F, *TM, *ST, 1, *MMI);
+    PI.reset(*MF.get());
+  }
+};
+
+TEST_F(SIProgramInfoMCExprsTest, TestDeathHSAKernelEmit) {
+  MCContext &Ctx = MF->getContext();
+  MCSymbol *Sym = Ctx.getOrCreateSymbol("Unknown");
+  PI.ScratchSize = MCSymbolRefExpr::create(Sym, Ctx);
+
+  auto &Func = MF->getFunction();
+  Func.setCallingConv(CallingConv::AMDGPU_KERNEL);
+  AMDGPU::HSAMD::MetadataStreamerMsgPackV4 MD;
+  EXPECT_DEATH(MD.emitKernel(*MF, PI),
+               "could not resolve expression when required.");
+}
-- 
GitLab


From e5e66073c3d404f4dedf1b0be160b7815ccf8903 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 9 May 2024 07:04:48 -0500
Subject: [PATCH 0277/1206] Revert "[Libomptarget] Statically link all plugin
 runtimes (#87009)"

Caused failures on build-bots, reverting to investigate.

This reverts commit 80f9e814ec896fdc57ee84afad8ac4cb1f8e4627.
---
 clang/test/Driver/linker-wrapper-image.c      |   2 +-
 .../Frontend/Offloading/OffloadWrapper.cpp    |   7 +-
 offload/include/PluginManager.h               |  61 ++++--
 offload/include/device.h                      |   8 +-
 offload/plugins-nextgen/CMakeLists.txt        |  19 +-
 offload/plugins-nextgen/amdgpu/CMakeLists.txt |   5 +
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    |  14 +-
 offload/plugins-nextgen/common/CMakeLists.txt |   5 +-
 .../common/include/PluginInterface.h          |  94 +++++++-
 .../common/include/Utils/ELF.h                |   2 +
 offload/plugins-nextgen/common/src/JIT.cpp    |  40 ++--
 .../common/src/PluginInterface.cpp            | 205 ++++++++++++++++++
 offload/plugins-nextgen/cuda/CMakeLists.txt   |   5 +
 offload/plugins-nextgen/cuda/src/rtl.cpp      |  14 +-
 offload/plugins-nextgen/host/CMakeLists.txt   |   8 +
 offload/plugins-nextgen/host/src/rtl.cpp      |  14 +-
 offload/src/CMakeLists.txt                    |   4 -
 offload/src/OffloadRTL.cpp                    |   1 -
 offload/src/OpenMP/InteropAPI.cpp             |   4 +-
 offload/src/PluginManager.cpp                 | 129 +++++++----
 offload/src/device.cpp                        |   3 +-
 offload/src/interface.cpp                     |   2 +
 .../kernelreplay/llvm-omp-kernel-replay.cpp   |   2 +
 .../unittests/Plugins/NextgenPluginsTest.cpp  |   1 +
 24 files changed, 524 insertions(+), 125 deletions(-)

diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c
index 5d5d62805e17..d01445e3aed0 100644
--- a/clang/test/Driver/linker-wrapper-image.c
+++ b/clang/test/Driver/linker-wrapper-image.c
@@ -30,8 +30,8 @@
 
 //      OPENMP: define internal void @.omp_offloading.descriptor_reg() section ".text.startup" {
 // OPENMP-NEXT: entry:
-// OPENMP-NEXT:   call void @__tgt_register_lib(ptr @.omp_offloading.descriptor)
 // OPENMP-NEXT:   %0 = call i32 @atexit(ptr @.omp_offloading.descriptor_unreg)
+// OPENMP-NEXT:   call void @__tgt_register_lib(ptr @.omp_offloading.descriptor)
 // OPENMP-NEXT:   ret void
 // OPENMP-NEXT: }
 
diff --git a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
index 8b6f9ea1f4cc..7241d15ed1c6 100644
--- a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
+++ b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
@@ -232,13 +232,12 @@ void createRegisterFunction(Module &M, GlobalVariable *BinDesc,
   // Construct function body
   IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));
 
-  Builder.CreateCall(RegFuncC, BinDesc);
-
   // Register the destructors with 'atexit'. This is expected by the CUDA
   // runtime and ensures that we clean up before dynamic objects are destroyed.
-  // This needs to be done after plugin initialization to ensure that it is
-  // called before the plugin runtime is destroyed.
+  // This needs to be done before the runtime is called and registers its own.
   Builder.CreateCall(AtExit, UnregFunc);
+
+  Builder.CreateCall(RegFuncC, BinDesc);
   Builder.CreateRetVoid();
 
   // Add this function to constructors.
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index 1d6804da75d9..eece7525e25e 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -13,11 +13,10 @@
 #ifndef OMPTARGET_PLUGIN_MANAGER_H
 #define OMPTARGET_PLUGIN_MANAGER_H
 
-#include "PluginInterface.h"
-
 #include "DeviceImage.h"
 #include "ExclusiveAccess.h"
 #include "Shared/APITypes.h"
+#include "Shared/PluginAPI.h"
 #include "Shared/Requirements.h"
 
 #include "device.h"
@@ -35,7 +34,38 @@
 #include <mutex>
 #include <string>
 
-using GenericPluginTy = llvm::omp::target::plugin::GenericPluginTy;
+struct PluginManager;
+
+/// Plugin adaptors should be created via `PluginAdaptorTy::create` which will
+/// invoke the constructor and call `PluginAdaptorTy::init`. Eventual errors are
+/// reported back to the caller, otherwise a valid and initialized adaptor is
+/// returned.
+struct PluginAdaptorTy {
+  /// Try to create a plugin adaptor from a filename.
+  static llvm::Expected<std::unique_ptr<PluginAdaptorTy>>
+  create(const std::string &Name);
+
+  /// Name of the shared object file representing the plugin.
+  std::string Name;
+
+  /// Access to the shared object file representing the plugin.
+  std::unique_ptr<llvm::sys::DynamicLibrary> LibraryHandler;
+
+#define PLUGIN_API_HANDLE(NAME)                                                \
+  using NAME##_ty = decltype(__tgt_rtl_##NAME);                                \
+  NAME##_ty *NAME = nullptr;
+
+#include "Shared/PluginAPI.inc"
+#undef PLUGIN_API_HANDLE
+
+  /// Create a plugin adaptor for filename \p Name with a dynamic library \p DL.
+  PluginAdaptorTy(const std::string &Name,
+                  std::unique_ptr<llvm::sys::DynamicLibrary> DL);
+
+  /// Initialize the plugin adaptor, this can fail in which case the adaptor is
+  /// useless.
+  llvm::Error init();
+};
 
 /// Struct for the data required to handle plugins
 struct PluginManager {
@@ -50,8 +80,6 @@ struct PluginManager {
 
   void init();
 
-  void deinit();
-
   // Register a shared library with all (compatible) RTLs.
   void registerLib(__tgt_bin_desc *Desc);
 
@@ -64,9 +92,10 @@ struct PluginManager {
         std::make_unique<DeviceImageTy>(TgtBinDesc, TgtDeviceImage));
   }
 
-  /// Initialize as many devices as possible for this plugin. Devices that fail
-  /// to initialize are ignored.
-  void initDevices(GenericPluginTy &RTL);
+  /// Initialize as many devices as possible for this plugin adaptor. Devices
+  /// that fail to initialize are ignored. Returns the offset the devices were
+  /// registered at.
+  void initDevices(PluginAdaptorTy &RTL);
 
   /// Return the device presented to the user as device \p DeviceNo if it is
   /// initialized and ready. Otherwise return an error explaining the problem.
@@ -122,8 +151,8 @@ struct PluginManager {
   // Initialize all plugins.
   void initAllPlugins();
 
-  /// Iterator range for all plugins (in use or not, but always valid).
-  auto plugins() { return llvm::make_pointee_range(Plugins); }
+  /// Iterator range for all plugin adaptors (in use or not, but always valid).
+  auto pluginAdaptors() { return llvm::make_pointee_range(PluginAdaptors); }
 
   /// Return the user provided requirements.
   int64_t getRequirements() const { return Requirements.getRequirements(); }
@@ -135,14 +164,14 @@ private:
   bool RTLsLoaded = false;
   llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
 
-  // List of all plugins, in use or not.
-  llvm::SmallVector<std::unique_ptr<GenericPluginTy>> Plugins;
+  // List of all plugin adaptors, in use or not.
+  llvm::SmallVector<std::unique_ptr<PluginAdaptorTy>> PluginAdaptors;
 
-  // Mapping of plugins to offsets in the device table.
-  llvm::DenseMap<const GenericPluginTy *, int32_t> DeviceOffsets;
+  // Mapping of plugin adaptors to offsets in the device table.
+  llvm::DenseMap<const PluginAdaptorTy *, int32_t> DeviceOffsets;
 
-  // Mapping of plugins to the number of used devices.
-  llvm::DenseMap<const GenericPluginTy *, int32_t> DeviceUsed;
+  // Mapping of plugin adaptors to the number of used devices.
+  llvm::DenseMap<const PluginAdaptorTy *, int32_t> DeviceUsed;
 
   // Set of all device images currently in use.
   llvm::DenseSet<const __tgt_device_image *> UsedImages;
diff --git a/offload/include/device.h b/offload/include/device.h
index fd6e5fba5fc5..bd2829722bb3 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -33,19 +33,17 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 
-#include "PluginInterface.h"
-using GenericPluginTy = llvm::omp::target::plugin::GenericPluginTy;
-
 // Forward declarations.
+struct PluginAdaptorTy;
 struct __tgt_bin_desc;
 struct __tgt_target_table;
 
 struct DeviceTy {
   int32_t DeviceID;
-  GenericPluginTy *RTL;
+  PluginAdaptorTy *RTL;
   int32_t RTLDeviceID;
 
-  DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID);
+  DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID);
   // DeviceTy is not copyable
   DeviceTy(const DeviceTy &D) = delete;
   DeviceTy &operator=(const DeviceTy &D) = delete;
diff --git a/offload/plugins-nextgen/CMakeLists.txt b/offload/plugins-nextgen/CMakeLists.txt
index d1079f8a3e9c..df625e97c7eb 100644
--- a/offload/plugins-nextgen/CMakeLists.txt
+++ b/offload/plugins-nextgen/CMakeLists.txt
@@ -14,7 +14,7 @@
 set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common)
 add_subdirectory(common)
 function(add_target_library target_name lib_name)
-  add_llvm_library(${target_name} STATIC
+  add_llvm_library(${target_name} SHARED
     LINK_COMPONENTS
       ${LLVM_TARGETS_TO_BUILD}
       AggressiveInstCombine
@@ -46,14 +46,27 @@ function(add_target_library target_name lib_name)
   )
 
   llvm_update_compile_flags(${target_name})
-  target_include_directories(${target_name} PUBLIC ${common_dir}/include)
   target_link_libraries(${target_name} PRIVATE
                         PluginCommon ${OPENMP_PTHREAD_LIB})
 
   target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name})
   target_compile_definitions(${target_name} PRIVATE 
                              DEBUG_PREFIX="TARGET ${lib_name} RTL")
-  set_target_properties(${target_name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
+  if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+    # On FreeBSD, the 'environ' symbol is undefined at link time, but resolved by
+    # the dynamic linker at runtime. Therefore, allow the symbol to be undefined
+    # when creating a shared library.
+    target_link_libraries(${target_name} PRIVATE "-Wl,--allow-shlib-undefined")
+  else()
+    target_link_libraries(${target_name} PRIVATE "-Wl,-z,defs")
+  endif()
+
+  if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
+    target_link_libraries(${target_name} PRIVATE
+    "-Wl,--version-script=${common_dir}/../exports")
+  endif()
+  set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET protected)
 endfunction()
 
 foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD)
diff --git a/offload/plugins-nextgen/amdgpu/CMakeLists.txt b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
index 738183f8945e..f5f7096137c2 100644
--- a/offload/plugins-nextgen/amdgpu/CMakeLists.txt
+++ b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
@@ -57,3 +57,8 @@ else()
   libomptarget_say("Not generating AMDGPU tests, no supported devices detected."
                    " Use 'LIBOMPTARGET_FORCE_AMDGPU_TESTS' to override.")
 endif()
+
+# Install plugin under the lib destination folder.
+install(TARGETS omptarget.rtl.amdgpu LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
+set_target_properties(omptarget.rtl.amdgpu PROPERTIES
+  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 295685fceaa4..00650b801b42 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -3064,6 +3064,10 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
     // HSA functions from now on, e.g., hsa_shut_down.
     Initialized = true;
 
+#ifdef OMPT_SUPPORT
+    ompt::connectLibrary();
+#endif
+
     // Register event handler to detect memory errors on the devices.
     Status = hsa_amd_register_system_event_handler(eventHandler, nullptr);
     if (auto Err = Plugin::check(
@@ -3151,8 +3155,6 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
 
   Triple::ArchType getTripleArch() const override { return Triple::amdgcn; }
 
-  const char *getName() const override { return GETNAME(TARGET_NAME); }
-
   /// Get the ELF code for recognizing the compatible image binary.
   uint16_t getMagicElfBits() const override { return ELF::EM_AMDGPU; }
 
@@ -3385,6 +3387,8 @@ Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
   return Plugin::success();
 }
 
+GenericPluginTy *PluginTy::createPlugin() { return new AMDGPUPluginTy(); }
+
 template <typename... ArgsTy>
 static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
   hsa_status_t ResultCode = static_cast<hsa_status_t>(Code);
@@ -3472,9 +3476,3 @@ void *AMDGPUDeviceTy::allocate(size_t Size, void *, TargetAllocTy Kind) {
 } // namespace target
 } // namespace omp
 } // namespace llvm
-
-extern "C" {
-llvm::omp::target::plugin::GenericPluginTy *createPlugin_amdgpu() {
-  return new llvm::omp::target::plugin::AMDGPUPluginTy();
-}
-}
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index a6bbb7e9454b..acf0af63f050 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -46,6 +46,7 @@ endif()
 
 # If we have OMPT enabled include it in the list of sources.
 if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
+  target_sources(PluginCommon PRIVATE OMPT/OmptCallback.cpp)
   target_include_directories(PluginCommon PRIVATE OMPT)
 endif()
 
@@ -65,4 +66,6 @@ target_include_directories(PluginCommon PUBLIC
   ${LIBOMPTARGET_INCLUDE_DIR}
 )
 
-set_target_properties(PluginCommon PROPERTIES POSITION_INDEPENDENT_CODE ON)
+set_target_properties(PluginCommon PROPERTIES
+  POSITION_INDEPENDENT_CODE ON
+  CXX_VISIBILITY_PRESET protected)
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index e7a008f3a857..79e8464bfda5 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -1010,9 +1010,6 @@ struct GenericPluginTy {
   /// Get the target triple of this plugin.
   virtual Triple::ArchType getTripleArch() const = 0;
 
-  /// Get the constant name identifier for this plugin.
-  virtual const char *getName() const = 0;
-
   /// Allocate a structure using the internal allocator.
   template <typename Ty> Ty *allocate() {
     return reinterpret_cast<Ty *>(Allocator.Allocate(sizeof(Ty), alignof(Ty)));
@@ -1229,7 +1226,7 @@ namespace Plugin {
 /// Create a success error. This is the same as calling Error::success(), but
 /// it is recommended to use this one for consistency with Plugin::error() and
 /// Plugin::check().
-static inline Error success() { return Error::success(); }
+static Error success() { return Error::success(); }
 
 /// Create a string error.
 template <typename... ArgsTy>
@@ -1249,6 +1246,95 @@ template <typename... ArgsTy>
 static Error check(int32_t ErrorCode, const char *ErrFmt, ArgsTy... Args);
 } // namespace Plugin
 
+/// Class for simplifying the getter operation of the plugin. Anywhere on the
+/// code, the current plugin can be retrieved by Plugin::get(). The class also
+/// declares functions to create plugin-specific object instances. The check(),
+/// createPlugin(), createDevice() and createGlobalHandler() functions should be
+/// defined by each plugin implementation.
+class PluginTy {
+  // Reference to the plugin instance.
+  static GenericPluginTy *SpecificPlugin;
+
+  PluginTy() {
+    if (auto Err = init())
+      REPORT("Failed to initialize plugin: %s\n",
+             toString(std::move(Err)).data());
+  }
+
+  ~PluginTy() {
+    if (auto Err = deinit())
+      REPORT("Failed to deinitialize plugin: %s\n",
+             toString(std::move(Err)).data());
+  }
+
+  PluginTy(const PluginTy &) = delete;
+  void operator=(const PluginTy &) = delete;
+
+  /// Create and intialize the plugin instance.
+  static Error init() {
+    assert(!SpecificPlugin && "Plugin already created");
+
+    // Create the specific plugin.
+    SpecificPlugin = createPlugin();
+    assert(SpecificPlugin && "Plugin was not created");
+
+    // Initialize the plugin.
+    return SpecificPlugin->init();
+  }
+
+  // Deinitialize and destroy the plugin instance.
+  static Error deinit() {
+    assert(SpecificPlugin && "Plugin no longer valid");
+
+    for (int32_t DevNo = 0, NumDev = SpecificPlugin->getNumDevices();
+         DevNo < NumDev; ++DevNo)
+      if (auto Err = SpecificPlugin->deinitDevice(DevNo))
+        return Err;
+
+    // Deinitialize the plugin.
+    if (auto Err = SpecificPlugin->deinit())
+      return Err;
+
+    // Delete the plugin instance.
+    delete SpecificPlugin;
+
+    // Invalidate the plugin reference.
+    SpecificPlugin = nullptr;
+
+    return Plugin::success();
+  }
+
+public:
+  /// Initialize the plugin if needed. The plugin could have been initialized by
+  /// a previous call to Plugin::get().
+  static Error initIfNeeded() {
+    // Trigger the initialization if needed.
+    get();
+
+    return Error::success();
+  }
+
+  /// Get a reference (or create if it was not created) to the plugin instance.
+  static GenericPluginTy &get() {
+    // This static variable will initialize the underlying plugin instance in
+    // case there was no previous explicit initialization. The initialization is
+    // thread safe.
+    static PluginTy Plugin;
+
+    assert(SpecificPlugin && "Plugin is not active");
+    return *SpecificPlugin;
+  }
+
+  /// Get a reference to the plugin with a specific plugin-specific type.
+  template <typename Ty> static Ty &get() { return static_cast<Ty &>(get()); }
+
+  /// Indicate whether the plugin is active.
+  static bool isActive() { return SpecificPlugin != nullptr; }
+
+  /// Create a plugin instance.
+  static GenericPluginTy *createPlugin();
+};
+
 /// Auxiliary interface class for GenericDeviceResourceManagerTy. This class
 /// acts as a reference to a device resource, such as a stream, and requires
 /// some basic functions to be implemented. The derived class should define an
diff --git a/offload/plugins-nextgen/common/include/Utils/ELF.h b/offload/plugins-nextgen/common/include/Utils/ELF.h
index dcfdb5bd7b03..f87e0a5ed02b 100644
--- a/offload/plugins-nextgen/common/include/Utils/ELF.h
+++ b/offload/plugins-nextgen/common/include/Utils/ELF.h
@@ -13,6 +13,8 @@
 #ifndef LLVM_OPENMP_LIBOMPTARGET_PLUGINS_ELF_UTILS_H
 #define LLVM_OPENMP_LIBOMPTARGET_PLUGINS_ELF_UTILS_H
 
+#include "Shared/PluginAPI.h"
+
 #include "llvm/Object/ELF.h"
 #include "llvm/Object/ELFObjectFile.h"
 
diff --git a/offload/plugins-nextgen/common/src/JIT.cpp b/offload/plugins-nextgen/common/src/JIT.cpp
index 9d58e6060646..9eb610cab4de 100644
--- a/offload/plugins-nextgen/common/src/JIT.cpp
+++ b/offload/plugins-nextgen/common/src/JIT.cpp
@@ -56,6 +56,28 @@ bool isImageBitcode(const __tgt_device_image &Image) {
   return identify_magic(Binary) == file_magic::bitcode;
 }
 
+std::once_flag InitFlag;
+
+void init(Triple TT) {
+  codegen::RegisterCodeGenFlags();
+#ifdef LIBOMPTARGET_JIT_NVPTX
+  if (TT.isNVPTX()) {
+    LLVMInitializeNVPTXTargetInfo();
+    LLVMInitializeNVPTXTarget();
+    LLVMInitializeNVPTXTargetMC();
+    LLVMInitializeNVPTXAsmPrinter();
+  }
+#endif
+#ifdef LIBOMPTARGET_JIT_AMDGPU
+  if (TT.isAMDGPU()) {
+    LLVMInitializeAMDGPUTargetInfo();
+    LLVMInitializeAMDGPUTarget();
+    LLVMInitializeAMDGPUTargetMC();
+    LLVMInitializeAMDGPUAsmPrinter();
+  }
+#endif
+}
+
 Expected<std::unique_ptr<Module>>
 createModuleFromMemoryBuffer(std::unique_ptr<MemoryBuffer> &MB,
                              LLVMContext &Context) {
@@ -126,23 +148,7 @@ createTargetMachine(Module &M, std::string CPU, unsigned OptLevel) {
 } // namespace
 
 JITEngine::JITEngine(Triple::ArchType TA) : TT(Triple::getArchTypeName(TA)) {
-  codegen::RegisterCodeGenFlags();
-#ifdef LIBOMPTARGET_JIT_NVPTX
-  if (TT.isNVPTX()) {
-    LLVMInitializeNVPTXTargetInfo();
-    LLVMInitializeNVPTXTarget();
-    LLVMInitializeNVPTXTargetMC();
-    LLVMInitializeNVPTXAsmPrinter();
-  }
-#endif
-#ifdef LIBOMPTARGET_JIT_AMDGPU
-  if (TT.isAMDGPU()) {
-    LLVMInitializeAMDGPUTargetInfo();
-    LLVMInitializeAMDGPUTarget();
-    LLVMInitializeAMDGPUTargetMC();
-    LLVMInitializeAMDGPUAsmPrinter();
-  }
-#endif
+  std::call_once(InitFlag, init, TT);
 }
 
 void JITEngine::opt(TargetMachine *TM, TargetLibraryInfoImpl *TLII, Module &M,
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index fae197527850..8de93ba17a56 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -13,6 +13,7 @@
 #include "Shared/APITypes.h"
 #include "Shared/Debug.h"
 #include "Shared/Environment.h"
+#include "Shared/PluginAPI.h"
 
 #include "GlobalHandler.h"
 #include "JIT.h"
@@ -38,6 +39,8 @@ using namespace omp;
 using namespace target;
 using namespace plugin;
 
+GenericPluginTy *PluginTy::SpecificPlugin = nullptr;
+
 // TODO: Fix any thread safety issues for multi-threaded kernel recording.
 struct RecordReplayTy {
 
@@ -2032,3 +2035,205 @@ bool llvm::omp::target::plugin::libomptargetSupportsRPC() {
   return false;
 #endif
 }
+
+/// Exposed library API function, basically wrappers around the GenericDeviceTy
+/// functionality with the same name. All non-async functions are redirected
+/// to the async versions right away with a NULL AsyncInfoPtr.
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t __tgt_rtl_init_plugin() {
+  auto Err = PluginTy::initIfNeeded();
+  if (Err) {
+    [[maybe_unused]] std::string ErrStr = toString(std::move(Err));
+    DP("Failed to init plugin: %s", ErrStr.c_str());
+    return OFFLOAD_FAIL;
+  }
+
+  return OFFLOAD_SUCCESS;
+}
+
+int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) {
+  if (!PluginTy::isActive())
+    return false;
+
+  return PluginTy::get().is_valid_binary(Image);
+}
+
+int32_t __tgt_rtl_init_device(int32_t DeviceId) {
+  return PluginTy::get().init_device(DeviceId);
+}
+
+int32_t __tgt_rtl_number_of_devices() {
+  return PluginTy::get().number_of_devices();
+}
+
+int64_t __tgt_rtl_init_requires(int64_t RequiresFlags) {
+  return PluginTy::get().init_requires(RequiresFlags);
+}
+
+int32_t __tgt_rtl_is_data_exchangable(int32_t SrcDeviceId,
+                                      int32_t DstDeviceId) {
+  return PluginTy::get().is_data_exchangable(SrcDeviceId, DstDeviceId);
+}
+
+int32_t __tgt_rtl_initialize_record_replay(int32_t DeviceId, int64_t MemorySize,
+                                           void *VAddr, bool isRecord,
+                                           bool SaveOutput,
+                                           uint64_t &ReqPtrArgOffset) {
+  return PluginTy::get().initialize_record_replay(
+      DeviceId, MemorySize, VAddr, isRecord, SaveOutput, ReqPtrArgOffset);
+}
+
+int32_t __tgt_rtl_load_binary(int32_t DeviceId, __tgt_device_image *TgtImage,
+                              __tgt_device_binary *Binary) {
+  return PluginTy::get().load_binary(DeviceId, TgtImage, Binary);
+}
+
+void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr,
+                           int32_t Kind) {
+  return PluginTy::get().data_alloc(DeviceId, Size, HostPtr, Kind);
+}
+
+int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind) {
+  return PluginTy::get().data_delete(DeviceId, TgtPtr, Kind);
+}
+
+int32_t __tgt_rtl_data_lock(int32_t DeviceId, void *Ptr, int64_t Size,
+                            void **LockedPtr) {
+  return PluginTy::get().data_lock(DeviceId, Ptr, Size, LockedPtr);
+}
+
+int32_t __tgt_rtl_data_unlock(int32_t DeviceId, void *Ptr) {
+  return PluginTy::get().data_unlock(DeviceId, Ptr);
+}
+
+int32_t __tgt_rtl_data_notify_mapped(int32_t DeviceId, void *HstPtr,
+                                     int64_t Size) {
+  return PluginTy::get().data_notify_mapped(DeviceId, HstPtr, Size);
+}
+
+int32_t __tgt_rtl_data_notify_unmapped(int32_t DeviceId, void *HstPtr) {
+  return PluginTy::get().data_notify_unmapped(DeviceId, HstPtr);
+}
+
+int32_t __tgt_rtl_data_submit(int32_t DeviceId, void *TgtPtr, void *HstPtr,
+                              int64_t Size) {
+  return PluginTy::get().data_submit(DeviceId, TgtPtr, HstPtr, Size);
+}
+
+int32_t __tgt_rtl_data_submit_async(int32_t DeviceId, void *TgtPtr,
+                                    void *HstPtr, int64_t Size,
+                                    __tgt_async_info *AsyncInfoPtr) {
+  return PluginTy::get().data_submit_async(DeviceId, TgtPtr, HstPtr, Size,
+                                           AsyncInfoPtr);
+}
+
+int32_t __tgt_rtl_data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr,
+                                int64_t Size) {
+  return PluginTy::get().data_retrieve(DeviceId, HstPtr, TgtPtr, Size);
+}
+
+int32_t __tgt_rtl_data_retrieve_async(int32_t DeviceId, void *HstPtr,
+                                      void *TgtPtr, int64_t Size,
+                                      __tgt_async_info *AsyncInfoPtr) {
+  return PluginTy::get().data_retrieve_async(DeviceId, HstPtr, TgtPtr, Size,
+                                             AsyncInfoPtr);
+}
+
+int32_t __tgt_rtl_data_exchange(int32_t SrcDeviceId, void *SrcPtr,
+                                int32_t DstDeviceId, void *DstPtr,
+                                int64_t Size) {
+  return PluginTy::get().data_exchange(SrcDeviceId, SrcPtr, DstDeviceId, DstPtr,
+                                       Size);
+}
+
+int32_t __tgt_rtl_data_exchange_async(int32_t SrcDeviceId, void *SrcPtr,
+                                      int DstDeviceId, void *DstPtr,
+                                      int64_t Size,
+                                      __tgt_async_info *AsyncInfo) {
+  return PluginTy::get().data_exchange_async(SrcDeviceId, SrcPtr, DstDeviceId,
+                                             DstPtr, Size, AsyncInfo);
+}
+
+int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
+                                void **TgtArgs, ptrdiff_t *TgtOffsets,
+                                KernelArgsTy *KernelArgs,
+                                __tgt_async_info *AsyncInfoPtr) {
+  return PluginTy::get().launch_kernel(DeviceId, TgtEntryPtr, TgtArgs,
+                                       TgtOffsets, KernelArgs, AsyncInfoPtr);
+}
+
+int32_t __tgt_rtl_synchronize(int32_t DeviceId,
+                              __tgt_async_info *AsyncInfoPtr) {
+  return PluginTy::get().synchronize(DeviceId, AsyncInfoPtr);
+}
+
+int32_t __tgt_rtl_query_async(int32_t DeviceId,
+                              __tgt_async_info *AsyncInfoPtr) {
+  return PluginTy::get().query_async(DeviceId, AsyncInfoPtr);
+}
+
+void __tgt_rtl_print_device_info(int32_t DeviceId) {
+  PluginTy::get().print_device_info(DeviceId);
+}
+
+int32_t __tgt_rtl_create_event(int32_t DeviceId, void **EventPtr) {
+  return PluginTy::get().create_event(DeviceId, EventPtr);
+}
+
+int32_t __tgt_rtl_record_event(int32_t DeviceId, void *EventPtr,
+                               __tgt_async_info *AsyncInfoPtr) {
+  return PluginTy::get().record_event(DeviceId, EventPtr, AsyncInfoPtr);
+}
+
+int32_t __tgt_rtl_wait_event(int32_t DeviceId, void *EventPtr,
+                             __tgt_async_info *AsyncInfoPtr) {
+  return PluginTy::get().wait_event(DeviceId, EventPtr, AsyncInfoPtr);
+}
+
+int32_t __tgt_rtl_sync_event(int32_t DeviceId, void *EventPtr) {
+  return PluginTy::get().sync_event(DeviceId, EventPtr);
+}
+
+int32_t __tgt_rtl_destroy_event(int32_t DeviceId, void *EventPtr) {
+  return PluginTy::get().destroy_event(DeviceId, EventPtr);
+}
+
+void __tgt_rtl_set_info_flag(uint32_t NewInfoLevel) {
+  return PluginTy::get().set_info_flag(NewInfoLevel);
+}
+
+int32_t __tgt_rtl_init_async_info(int32_t DeviceId,
+                                  __tgt_async_info **AsyncInfoPtr) {
+  return PluginTy::get().init_async_info(DeviceId, AsyncInfoPtr);
+}
+
+int32_t __tgt_rtl_init_device_info(int32_t DeviceId,
+                                   __tgt_device_info *DeviceInfo,
+                                   const char **ErrStr) {
+  return PluginTy::get().init_device_info(DeviceId, DeviceInfo, ErrStr);
+}
+
+int32_t __tgt_rtl_set_device_offset(int32_t DeviceIdOffset) {
+  return PluginTy::get().set_device_offset(DeviceIdOffset);
+}
+
+int32_t __tgt_rtl_use_auto_zero_copy(int32_t DeviceId) {
+  return PluginTy::get().use_auto_zero_copy(DeviceId);
+}
+
+int32_t __tgt_rtl_get_global(__tgt_device_binary Binary, uint64_t Size,
+                             const char *Name, void **DevicePtr) {
+  return PluginTy::get().get_global(Binary, Size, Name, DevicePtr);
+}
+
+int32_t __tgt_rtl_get_function(__tgt_device_binary Binary, const char *Name,
+                               void **KernelPtr) {
+  return PluginTy::get().get_function(Binary, Name, KernelPtr);
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/plugins-nextgen/cuda/CMakeLists.txt b/offload/plugins-nextgen/cuda/CMakeLists.txt
index dd684bb22343..0284bd22d2a4 100644
--- a/offload/plugins-nextgen/cuda/CMakeLists.txt
+++ b/offload/plugins-nextgen/cuda/CMakeLists.txt
@@ -51,3 +51,8 @@ else()
   libomptarget_say("Not generating NVIDIA tests, no supported devices detected."
                    " Use 'LIBOMPTARGET_FORCE_NVIDIA_TESTS' to override.")
 endif()
+
+# Install plugin under the lib destination folder.
+install(TARGETS omptarget.rtl.cuda LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
+set_target_properties(omptarget.rtl.cuda PROPERTIES
+  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index b260334baa18..fc74c6aa23fd 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -1342,6 +1342,10 @@ struct CUDAPluginTy final : public GenericPluginTy {
       return 0;
     }
 
+#ifdef OMPT_SUPPORT
+    ompt::connectLibrary();
+#endif
+
     if (Res == CUDA_ERROR_NO_DEVICE) {
       // Do not initialize if there are no devices.
       DP("There are no devices supporting CUDA.\n");
@@ -1386,8 +1390,6 @@ struct CUDAPluginTy final : public GenericPluginTy {
     return Triple::nvptx64;
   }
 
-  const char *getName() const override { return GETNAME(TARGET_NAME); }
-
   /// Check whether the image is compatible with the available CUDA devices.
   Expected<bool> isELFCompatible(StringRef Image) const override {
     auto ElfOrErr =
@@ -1493,6 +1495,8 @@ Error CUDADeviceTy::dataExchangeImpl(const void *SrcPtr,
   return Plugin::check(Res, "Error in cuMemcpyDtoDAsync: %s");
 }
 
+GenericPluginTy *PluginTy::createPlugin() { return new CUDAPluginTy(); }
+
 template <typename... ArgsTy>
 static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
   CUresult ResultCode = static_cast<CUresult>(Code);
@@ -1512,9 +1516,3 @@ static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
 } // namespace target
 } // namespace omp
 } // namespace llvm
-
-extern "C" {
-llvm::omp::target::plugin::GenericPluginTy *createPlugin_cuda() {
-  return new llvm::omp::target::plugin::CUDAPluginTy();
-}
-}
diff --git a/offload/plugins-nextgen/host/CMakeLists.txt b/offload/plugins-nextgen/host/CMakeLists.txt
index 72b5681283fe..1d000442c84d 100644
--- a/offload/plugins-nextgen/host/CMakeLists.txt
+++ b/offload/plugins-nextgen/host/CMakeLists.txt
@@ -31,6 +31,14 @@ else()
   target_include_directories(omptarget.rtl.host PRIVATE dynamic_ffi)
 endif()
 
+# Install plugin under the lib destination folder.
+install(TARGETS omptarget.rtl.host
+        LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
+set_target_properties(omptarget.rtl.host PROPERTIES
+  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.."
+  POSITION_INDEPENDENT_CODE ON
+  CXX_VISIBILITY_PRESET protected)
+
 target_include_directories(omptarget.rtl.host PRIVATE
                            ${LIBOMPTARGET_INCLUDE_DIR})
 
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index 409b44b1640a..4bdcae3dd6a1 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -385,6 +385,10 @@ struct GenELF64PluginTy final : public GenericPluginTy {
 
   /// Initialize the plugin and return the number of devices.
   Expected<int32_t> initImpl() override {
+#ifdef OMPT_SUPPORT
+    ompt::connectLibrary();
+#endif
+
 #ifdef USES_DYNAMIC_FFI
     if (auto Err = Plugin::check(ffi_init(), "Failed to initialize libffi"))
       return std::move(Err);
@@ -441,10 +445,10 @@ struct GenELF64PluginTy final : public GenericPluginTy {
     return llvm::Triple::UnknownArch;
 #endif
   }
-
-  const char *getName() const override { return GETNAME(TARGET_NAME); }
 };
 
+GenericPluginTy *PluginTy::createPlugin() { return new GenELF64PluginTy(); }
+
 template <typename... ArgsTy>
 static Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) {
   if (Code == 0)
@@ -458,9 +462,3 @@ static Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) {
 } // namespace target
 } // namespace omp
 } // namespace llvm
-
-extern "C" {
-llvm::omp::target::plugin::GenericPluginTy *createPlugin_host() {
-  return new llvm::omp::target::plugin::GenELF64PluginTy();
-}
-}
diff --git a/offload/src/CMakeLists.txt b/offload/src/CMakeLists.txt
index 8fe6d19d83eb..eda5a85ff1ab 100644
--- a/offload/src/CMakeLists.txt
+++ b/offload/src/CMakeLists.txt
@@ -65,10 +65,6 @@ target_compile_definitions(omptarget PRIVATE
   DEBUG_PREFIX="omptarget"
 )
 
-foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD)
-  target_link_libraries(omptarget PRIVATE omptarget.rtl.${plugin})
-endforeach()
-
 target_compile_options(omptarget PUBLIC ${offload_compile_flags})
 target_link_options(omptarget PUBLIC ${offload_link_flags})
 
diff --git a/offload/src/OffloadRTL.cpp b/offload/src/OffloadRTL.cpp
index 29b573a27d08..dd75b1b18150 100644
--- a/offload/src/OffloadRTL.cpp
+++ b/offload/src/OffloadRTL.cpp
@@ -50,7 +50,6 @@ void deinitRuntime() {
 
   if (RefCount == 1) {
     DP("Deinit offload library!\n");
-    PM->deinit();
     delete PM;
     PM = nullptr;
   }
diff --git a/offload/src/OpenMP/InteropAPI.cpp b/offload/src/OpenMP/InteropAPI.cpp
index bdbc440c64a2..1a995cde7816 100644
--- a/offload/src/OpenMP/InteropAPI.cpp
+++ b/offload/src/OpenMP/InteropAPI.cpp
@@ -230,14 +230,14 @@ void __tgt_interop_init(ident_t *LocRef, int32_t Gtid,
   }
 
   DeviceTy &Device = *DeviceOrErr;
-  if (!Device.RTL ||
+  if (!Device.RTL || !Device.RTL->init_device_info ||
       Device.RTL->init_device_info(DeviceId, &(InteropPtr)->device_info,
                                    &(InteropPtr)->err_str)) {
     delete InteropPtr;
     InteropPtr = omp_interop_none;
   }
   if (InteropType == kmp_interop_type_tasksync) {
-    if (!Device.RTL ||
+    if (!Device.RTL || !Device.RTL->init_async_info ||
         Device.RTL->init_async_info(DeviceId, &(InteropPtr)->async_info)) {
       delete InteropPtr;
       InteropPtr = omp_interop_none;
diff --git a/offload/src/PluginManager.cpp b/offload/src/PluginManager.cpp
index 191afa345641..dbb556c179e5 100644
--- a/offload/src/PluginManager.cpp
+++ b/offload/src/PluginManager.cpp
@@ -23,25 +23,85 @@ using namespace llvm::sys;
 
 PluginManager *PM = nullptr;
 
-// Every plugin exports this method to create an instance of the plugin type.
-#define PLUGIN_TARGET(Name) extern "C" GenericPluginTy *createPlugin_##Name();
-#include "Shared/Targets.def"
+Expected<std::unique_ptr<PluginAdaptorTy>>
+PluginAdaptorTy::create(const std::string &Name) {
+  DP("Attempting to load library '%s'...\n", Name.c_str());
+  TIMESCOPE_WITH_NAME_AND_IDENT(Name, (const ident_t *)nullptr);
+
+  std::string ErrMsg;
+  auto LibraryHandler = std::make_unique<DynamicLibrary>(
+      DynamicLibrary::getPermanentLibrary(Name.c_str(), &ErrMsg));
+
+  if (!LibraryHandler->isValid()) {
+    // Library does not exist or cannot be found.
+    return createStringError(inconvertibleErrorCode(),
+                             "Unable to load library '%s': %s!\n", Name.c_str(),
+                             ErrMsg.c_str());
+  }
+
+  DP("Successfully loaded library '%s'!\n", Name.c_str());
+  auto PluginAdaptor = std::unique_ptr<PluginAdaptorTy>(
+      new PluginAdaptorTy(Name, std::move(LibraryHandler)));
+  if (auto Err = PluginAdaptor->init())
+    return Err;
+  return std::move(PluginAdaptor);
+}
+
+PluginAdaptorTy::PluginAdaptorTy(const std::string &Name,
+                                 std::unique_ptr<llvm::sys::DynamicLibrary> DL)
+    : Name(Name), LibraryHandler(std::move(DL)) {}
+
+Error PluginAdaptorTy::init() {
+
+#define PLUGIN_API_HANDLE(NAME)                                                \
+  NAME = reinterpret_cast<decltype(NAME)>(                                     \
+      LibraryHandler->getAddressOfSymbol(GETNAME(__tgt_rtl_##NAME)));          \
+  if (!NAME) {                                                                 \
+    return createStringError(inconvertibleErrorCode(),                         \
+                             "Invalid plugin as necessary interface function " \
+                             "(%s) was not found.\n",                          \
+                             std::string(#NAME).c_str());                      \
+  }
+
+#include "Shared/PluginAPI.inc"
+#undef PLUGIN_API_HANDLE
+
+  // Remove plugin on failure to call optional init_plugin
+  int32_t Rc = init_plugin();
+  if (Rc != OFFLOAD_SUCCESS) {
+    return createStringError(inconvertibleErrorCode(),
+                             "Unable to initialize library '%s': %u!\n",
+                             Name.c_str(), Rc);
+  }
+
+  // No devices are supported by this RTL?
+  int32_t NumberOfPluginDevices = number_of_devices();
+  if (!NumberOfPluginDevices) {
+    return createStringError(inconvertibleErrorCode(),
+                             "No devices supported in this RTL\n");
+  }
+
+  DP("Registered '%s' with %d plugin visible devices!\n", Name.c_str(),
+     NumberOfPluginDevices);
+  return Error::success();
+}
 
 void PluginManager::init() {
   TIMESCOPE();
   DP("Loading RTLs...\n");
 
-  // Attempt to create an instance of each supported plugin.
+  // Attempt to open all the plugins and, if they exist, check if the interface
+  // is correct and if they are supporting any devices.
 #define PLUGIN_TARGET(Name)                                                    \
   do {                                                                         \
-    auto Plugin = std::unique_ptr<GenericPluginTy>(createPlugin_##Name());     \
-    if (auto Err = Plugin->init()) {                                           \
-      [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));         \
-      DP("Failed to init plugin: %s\n", InfoMsg.c_str());                      \
+    auto PluginAdaptorOrErr =                                                  \
+        PluginAdaptorTy::create("libomptarget.rtl." #Name ".so");              \
+    if (!PluginAdaptorOrErr) {                                                 \
+      [[maybe_unused]] std::string InfoMsg =                                   \
+          toString(PluginAdaptorOrErr.takeError());                            \
+      DP("%s", InfoMsg.c_str());                                               \
     } else {                                                                   \
-      DP("Registered plugin %s with %d visible device(s)\n",                   \
-         Plugin->getName(), Plugin->number_of_devices());                      \
-      Plugins.emplace_back(std::move(Plugin));                                 \
+      PluginAdaptors.push_back(std::move(*PluginAdaptorOrErr));                \
     }                                                                          \
   } while (false);
 #include "Shared/Targets.def"
@@ -49,29 +109,15 @@ void PluginManager::init() {
   DP("RTLs loaded!\n");
 }
 
-void PluginManager::deinit() {
-  TIMESCOPE();
-  DP("Unloading RTLs...\n");
-
-  for (auto &Plugin : Plugins) {
-    if (auto Err = Plugin->deinit()) {
-      [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));
-      DP("Failed to deinit plugin: %s\n", InfoMsg.c_str());
-    }
-    Plugin.release();
-  }
-
-  DP("RTLs unloaded!\n");
-}
-
-void PluginManager::initDevices(GenericPluginTy &RTL) {
+void PluginManager::initDevices(PluginAdaptorTy &RTL) {
   // If this RTL has already been initialized.
   if (PM->DeviceOffsets.contains(&RTL))
     return;
   TIMESCOPE();
 
   // If this RTL is not already in use, initialize it.
-  assert(RTL.number_of_devices() > 0 && "Tried to initialize useless plugin!");
+  assert(RTL.number_of_devices() > 0 &&
+         "Tried to initialize useless plugin adaptor");
 
   // Initialize the device information for the RTL we are about to use.
   auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
@@ -111,12 +157,13 @@ void PluginManager::initDevices(GenericPluginTy &RTL) {
 
   DeviceOffsets[&RTL] = DeviceOffset;
   DeviceUsed[&RTL] = NumberOfUserDevices;
-  DP("Plugin has index %d, exposes %d out of %d devices!\n", DeviceOffset,
-     NumberOfUserDevices, RTL.number_of_devices());
+  DP("Plugin adaptor " DPxMOD " has index %d, exposes %d out of %d devices!\n",
+     DPxPTR(RTL.LibraryHandler.get()), DeviceOffset, NumberOfUserDevices,
+     RTL.number_of_devices());
 }
 
 void PluginManager::initAllPlugins() {
-  for (auto &R : Plugins)
+  for (auto &R : PluginAdaptors)
     initDevices(*R);
 }
 
@@ -169,22 +216,19 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) {
     // Obtain the image and information that was previously extracted.
     __tgt_device_image *Img = &DI.getExecutableImage();
 
-    GenericPluginTy *FoundRTL = nullptr;
+    PluginAdaptorTy *FoundRTL = nullptr;
 
     // Scan the RTLs that have associated images until we find one that supports
     // the current image.
-    for (auto &R : PM->plugins()) {
-      if (!R.number_of_devices())
-        continue;
-
+    for (auto &R : PM->pluginAdaptors()) {
       if (!R.is_valid_binary(Img)) {
         DP("Image " DPxMOD " is NOT compatible with RTL %s!\n",
-           DPxPTR(Img->ImageStart), R.getName());
+           DPxPTR(Img->ImageStart), R.Name.c_str());
         continue;
       }
 
       DP("Image " DPxMOD " is compatible with RTL %s!\n",
-         DPxPTR(Img->ImageStart), R.getName());
+         DPxPTR(Img->ImageStart), R.Name.c_str());
 
       PM->initDevices(R);
 
@@ -203,7 +247,7 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) {
           (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin];
 
       DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(Img->ImageStart),
-         R.getName());
+         R.Name.c_str());
 
       registerImageIntoTranslationTable(TransTable, PM->DeviceOffsets[&R],
                                         PM->DeviceUsed[&R], Img);
@@ -238,11 +282,11 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
     // Obtain the image and information that was previously extracted.
     __tgt_device_image *Img = &DI.getExecutableImage();
 
-    GenericPluginTy *FoundRTL = NULL;
+    PluginAdaptorTy *FoundRTL = NULL;
 
     // Scan the RTLs that have associated images until we find one that supports
     // the current image. We only need to scan RTLs that are already being used.
-    for (auto &R : PM->plugins()) {
+    for (auto &R : PM->pluginAdaptors()) {
       if (!DeviceOffsets.contains(&R))
         continue;
 
@@ -252,7 +296,8 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
 
       FoundRTL = &R;
 
-      DP("Unregistered image " DPxMOD " from RTL\n", DPxPTR(Img->ImageStart));
+      DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n",
+         DPxPTR(Img->ImageStart), DPxPTR(R.LibraryHandler.get()));
 
       break;
     }
diff --git a/offload/src/device.cpp b/offload/src/device.cpp
index 749b4c567f8e..44a2facc8d3d 100644
--- a/offload/src/device.cpp
+++ b/offload/src/device.cpp
@@ -64,7 +64,7 @@ int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,
   return OFFLOAD_SUCCESS;
 }
 
-DeviceTy::DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID)
+DeviceTy::DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID)
     : DeviceID(DeviceID), RTL(RTL), RTLDeviceID(RTLDeviceID),
       MappingInfo(*this) {}
 
@@ -192,6 +192,7 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
           RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
   if (!AsyncInfo) {
+    assert(RTL->data_exchange && "RTL->data_exchange is nullptr");
     return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr,
                               Size);
   }
diff --git a/offload/src/interface.cpp b/offload/src/interface.cpp
index 763b051cc6d7..557703632c62 100644
--- a/offload/src/interface.cpp
+++ b/offload/src/interface.cpp
@@ -456,6 +456,8 @@ EXTERN void __tgt_set_info_flag(uint32_t NewInfoLevel) {
   assert(PM && "Runtime not initialized");
   std::atomic<uint32_t> &InfoLevel = getInfoLevelInternal();
   InfoLevel.store(NewInfoLevel);
+  for (auto &R : PM->pluginAdaptors())
+    R.set_info_flag(NewInfoLevel);
 }
 
 EXTERN int __tgt_print_device_info(int64_t DeviceId) {
diff --git a/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp b/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
index 1e9a6a84d805..761e04e4c7bb 100644
--- a/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
+++ b/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
@@ -13,6 +13,8 @@
 
 #include "omptarget.h"
 
+#include "Shared/PluginAPI.h"
+
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/MemoryBuffer.h"
diff --git a/offload/unittests/Plugins/NextgenPluginsTest.cpp b/offload/unittests/Plugins/NextgenPluginsTest.cpp
index 479b3f614aed..635bd1637c90 100644
--- a/offload/unittests/Plugins/NextgenPluginsTest.cpp
+++ b/offload/unittests/Plugins/NextgenPluginsTest.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "Shared/PluginAPI.h"
 #include "omptarget.h"
 #include "gtest/gtest.h"
 
-- 
GitLab


From e0d8dbc1dc203983243860c1fcc4698f60de37c0 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 9 May 2024 13:13:08 +0100
Subject: [PATCH 0278/1206] [X86] opt-shuff-tstore.ll - regenerate checks

---
 llvm/test/CodeGen/X86/opt-shuff-tstore.ll | 33 ++++++++++++++---------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/llvm/test/CodeGen/X86/opt-shuff-tstore.ll b/llvm/test/CodeGen/X86/opt-shuff-tstore.ll
index 0a2d4e9ba9fe..c331f8ffb369 100644
--- a/llvm/test/CodeGen/X86/opt-shuff-tstore.ll
+++ b/llvm/test/CodeGen/X86/opt-shuff-tstore.ll
@@ -1,37 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mcpu=corei7 -mtriple=x86_64-linux < %s  -mattr=+sse2,+sse4.1 | FileCheck %s
 
-; CHECK: func_4_8
 ; A single memory write
-; CHECK: movd
-; CHECK-NEXT: ret
 define void @func_4_8(<4 x i8> %param, ptr %p) {
+; CHECK-LABEL: func_4_8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    paddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT:    movd %xmm0, (%rdi)
+; CHECK-NEXT:    retq
   %r = add <4 x i8> %param, <i8 1, i8 2, i8 3, i8 4>
   store <4 x i8> %r, ptr %p
   ret void
 }
 
-; CHECK: func_4_16
-; CHECK: movq
-; CHECK-NEXT: ret
 define void @func_4_16(<4 x i16> %param, ptr %p) {
+; CHECK-LABEL: func_4_16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    paddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT:    movq %xmm0, (%rdi)
+; CHECK-NEXT:    retq
   %r = add <4 x i16> %param, <i16 1, i16 2, i16 3, i16 4>
   store <4 x i16> %r, ptr %p
   ret void
 }
 
-; CHECK: func_8_8
-; CHECK: movq
-; CHECK-NEXT: ret
 define void @func_8_8(<8 x i8> %param, ptr %p) {
+; CHECK-LABEL: func_8_8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    paddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT:    movq %xmm0, (%rdi)
+; CHECK-NEXT:    retq
   %r = add <8 x i8> %param, <i8 1, i8 2, i8 3, i8 4, i8 1, i8 2, i8 3, i8 4>
   store <8 x i8> %r, ptr %p
   ret void
 }
 
-; CHECK: func_2_32
-; CHECK: movq
-; CHECK-NEXT: ret
 define void @func_2_32(<2 x i32> %param, ptr %p) {
+; CHECK-LABEL: func_2_32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT:    movq %xmm0, (%rdi)
+; CHECK-NEXT:    retq
   %r = add <2 x i32> %param, <i32 1, i32 2>
   store <2 x i32> %r, ptr %p
   ret void
-- 
GitLab


From 8b400de79eff2a4fb95f06b6e4d167e65abbf448 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 9 May 2024 13:15:01 +0100
Subject: [PATCH 0279/1206] [X86] Enable TuningSlowDivide64 on
 Barcelona/Bobcat/Bulldozer/Ryzen Families (#91277)

Despite most AMD cpus having a lower latency for i64 divisions that converge early, we are still better off testing for values representable as i32 and performing a i32 division if possible.

All AMD cpus appear to have been missed when we added the "idivq-to-divl" attribute - this patch now matches Intel cpu behaviour (and the x86-64/v2/3/4 levels).

Unfortunately the difference in code scheduling means I've had to stop using the update_llc_test_checks script and just use old-fashioned CHECK-DAG checks for divl/divq pairs.

Fixes #90985
---
 llvm/lib/Target/X86/X86.td                    |  5 +
 .../CodeGen/X86/bypass-slow-division-64.ll    | 95 +++++++++----------
 2 files changed, 52 insertions(+), 48 deletions(-)

diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 25ab08187cf1..9f5b58d78fcc 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -1350,6 +1350,7 @@ def ProcessorFeatures {
                                               FeatureCMOV,
                                               FeatureX86_64];
   list<SubtargetFeature> BarcelonaTuning = [TuningFastScalarShiftMasks,
+                                            TuningSlowDivide64,
                                             TuningSlowSHLD,
                                             TuningSBBDepBreaking,
                                             TuningInsertVZEROUPPER];
@@ -1372,6 +1373,7 @@ def ProcessorFeatures {
   list<SubtargetFeature> BtVer1Tuning = [TuningFast15ByteNOP,
                                          TuningFastScalarShiftMasks,
                                          TuningFastVectorShiftMasks,
+                                         TuningSlowDivide64,
                                          TuningSlowSHLD,
                                          TuningFastImm16,
                                          TuningSBBDepBreaking,
@@ -1396,6 +1398,7 @@ def ProcessorFeatures {
                                          TuningFastMOVBE,
                                          TuningFastImm16,
                                          TuningSBBDepBreaking,
+                                         TuningSlowDivide64,
                                          TuningSlowSHLD];
   list<SubtargetFeature> BtVer2Features =
     !listconcat(BtVer1Features, BtVer2AdditionalFeatures);
@@ -1420,6 +1423,7 @@ def ProcessorFeatures {
                                            FeatureLWP,
                                            FeatureLAHFSAHF64];
   list<SubtargetFeature> BdVer1Tuning = [TuningSlowSHLD,
+                                         TuningSlowDivide64,
                                          TuningFast11ByteNOP,
                                          TuningFastScalarShiftMasks,
                                          TuningBranchFusion,
@@ -1500,6 +1504,7 @@ def ProcessorFeatures {
                                      TuningFastVariablePerLaneShuffle,
                                      TuningFastMOVBE,
                                      TuningFastImm16,
+                                     TuningSlowDivide64,
                                      TuningSlowSHLD,
                                      TuningSBBDepBreaking,
                                      TuningInsertVZEROUPPER,
diff --git a/llvm/test/CodeGen/X86/bypass-slow-division-64.ll b/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
index 66d7082d9b7c..6e0cfdd26a78 100644
--- a/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
+++ b/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
@@ -1,4 +1,3 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; Check that 64-bit division is bypassed correctly.
 ; RUN: llc < %s -mtriple=x86_64-- -mattr=-idivq-to-divl | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
 ; RUN: llc < %s -mtriple=x86_64-- -mattr=+idivq-to-divl | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
@@ -13,17 +12,17 @@
 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=skylake         | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=alderlake       | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
 ; AMD
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=barcelona       | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=btver1          | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=btver2          | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver1          | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver2          | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver3          | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver4          | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver1          | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver2          | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3          | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4          | FileCheck %s --check-prefixes=CHECK,FAST-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=barcelona       | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=btver1          | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=btver2          | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver1          | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver2          | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver3          | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver4          | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver1          | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver2          | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3          | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4          | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
 
 ; Additional tests for 64-bit divide bypass
 
@@ -41,18 +40,18 @@ define i64 @sdiv_quotient(i64 %a, i64 %b) nounwind {
 ;
 ; SLOW-DIVQ-LABEL: sdiv_quotient:
 ; SLOW-DIVQ:       # %bb.0:
-; SLOW-DIVQ-NEXT:    movq %rdi, %rax
-; SLOW-DIVQ-NEXT:    movq %rdi, %rcx
-; SLOW-DIVQ-NEXT:    orq %rsi, %rcx
-; SLOW-DIVQ-NEXT:    shrq $32, %rcx
+; SLOW-DIVQ-DAG:     movq %rdi, %rax
+; SLOW-DIVQ-DAG:     movq %rdi, %rcx
+; SLOW-DIVQ-DAG:     orq %rsi, %rcx
+; SLOW-DIVQ-DAG:     shrq $32, %rcx
 ; SLOW-DIVQ-NEXT:    je .LBB0_1
 ; SLOW-DIVQ-NEXT:  # %bb.2:
 ; SLOW-DIVQ-NEXT:    cqto
 ; SLOW-DIVQ-NEXT:    idivq %rsi
 ; SLOW-DIVQ-NEXT:    retq
 ; SLOW-DIVQ-NEXT:  .LBB0_1:
-; SLOW-DIVQ-NEXT:    # kill: def $eax killed $eax killed $rax
-; SLOW-DIVQ-NEXT:    xorl %edx, %edx
+; SLOW-DIVQ-DAG:     # kill: def $eax killed $eax killed $rax
+; SLOW-DIVQ-DAG:     xorl %edx, %edx
 ; SLOW-DIVQ-NEXT:    divl %esi
 ; SLOW-DIVQ-NEXT:    # kill: def $eax killed $eax def $rax
 ; SLOW-DIVQ-NEXT:    retq
@@ -93,10 +92,10 @@ define i64 @sdiv_remainder(i64 %a, i64 %b) nounwind {
 ;
 ; SLOW-DIVQ-LABEL: sdiv_remainder:
 ; SLOW-DIVQ:       # %bb.0:
-; SLOW-DIVQ-NEXT:    movq %rdi, %rax
-; SLOW-DIVQ-NEXT:    movq %rdi, %rcx
-; SLOW-DIVQ-NEXT:    orq %rsi, %rcx
-; SLOW-DIVQ-NEXT:    shrq $32, %rcx
+; SLOW-DIVQ-DAG:     movq %rdi, %rax
+; SLOW-DIVQ-DAG:     movq %rdi, %rcx
+; SLOW-DIVQ-DAG:     orq %rsi, %rcx
+; SLOW-DIVQ-DAG:     shrq $32, %rcx
 ; SLOW-DIVQ-NEXT:    je .LBB3_1
 ; SLOW-DIVQ-NEXT:  # %bb.2:
 ; SLOW-DIVQ-NEXT:    cqto
@@ -104,8 +103,8 @@ define i64 @sdiv_remainder(i64 %a, i64 %b) nounwind {
 ; SLOW-DIVQ-NEXT:    movq %rdx, %rax
 ; SLOW-DIVQ-NEXT:    retq
 ; SLOW-DIVQ-NEXT:  .LBB3_1:
-; SLOW-DIVQ-NEXT:    # kill: def $eax killed $eax killed $rax
-; SLOW-DIVQ-NEXT:    xorl %edx, %edx
+; SLOW-DIVQ-DAG:     # kill: def $eax killed $eax killed $rax
+; SLOW-DIVQ-DAG:     xorl %edx, %edx
 ; SLOW-DIVQ-NEXT:    divl %esi
 ; SLOW-DIVQ-NEXT:    movl %edx, %eax
 ; SLOW-DIVQ-NEXT:    retq
@@ -148,10 +147,10 @@ define i64 @sdiv_quotient_and_remainder(i64 %a, i64 %b) nounwind {
 ;
 ; SLOW-DIVQ-LABEL: sdiv_quotient_and_remainder:
 ; SLOW-DIVQ:       # %bb.0:
-; SLOW-DIVQ-NEXT:    movq %rdi, %rax
-; SLOW-DIVQ-NEXT:    movq %rdi, %rcx
-; SLOW-DIVQ-NEXT:    orq %rsi, %rcx
-; SLOW-DIVQ-NEXT:    shrq $32, %rcx
+; SLOW-DIVQ-DAG:     movq %rdi, %rax
+; SLOW-DIVQ-DAG:     movq %rdi, %rcx
+; SLOW-DIVQ-DAG:     orq %rsi, %rcx
+; SLOW-DIVQ-DAG:     shrq $32, %rcx
 ; SLOW-DIVQ-NEXT:    je .LBB6_1
 ; SLOW-DIVQ-NEXT:  # %bb.2:
 ; SLOW-DIVQ-NEXT:    cqto
@@ -159,8 +158,8 @@ define i64 @sdiv_quotient_and_remainder(i64 %a, i64 %b) nounwind {
 ; SLOW-DIVQ-NEXT:    addq %rdx, %rax
 ; SLOW-DIVQ-NEXT:    retq
 ; SLOW-DIVQ-NEXT:  .LBB6_1:
-; SLOW-DIVQ-NEXT:    # kill: def $eax killed $eax killed $rax
-; SLOW-DIVQ-NEXT:    xorl %edx, %edx
+; SLOW-DIVQ-DAG:     # kill: def $eax killed $eax killed $rax
+; SLOW-DIVQ-DAG:     xorl %edx, %edx
 ; SLOW-DIVQ-NEXT:    divl %esi
 ; SLOW-DIVQ-NEXT:    # kill: def $edx killed $edx def $rdx
 ; SLOW-DIVQ-NEXT:    # kill: def $eax killed $eax def $rax
@@ -214,18 +213,18 @@ define i64 @udiv_quotient(i64 %a, i64 %b) nounwind {
 ;
 ; SLOW-DIVQ-LABEL: udiv_quotient:
 ; SLOW-DIVQ:       # %bb.0:
-; SLOW-DIVQ-NEXT:    movq %rdi, %rax
-; SLOW-DIVQ-NEXT:    movq %rdi, %rcx
-; SLOW-DIVQ-NEXT:    orq %rsi, %rcx
-; SLOW-DIVQ-NEXT:    shrq $32, %rcx
+; SLOW-DIVQ-DAG:     movq %rdi, %rax
+; SLOW-DIVQ-DAG:     movq %rdi, %rcx
+; SLOW-DIVQ-DAG:     orq %rsi, %rcx
+; SLOW-DIVQ-DAG:     shrq $32, %rcx
 ; SLOW-DIVQ-NEXT:    je .LBB9_1
 ; SLOW-DIVQ-NEXT:  # %bb.2:
 ; SLOW-DIVQ-NEXT:    xorl %edx, %edx
 ; SLOW-DIVQ-NEXT:    divq %rsi
 ; SLOW-DIVQ-NEXT:    retq
 ; SLOW-DIVQ-NEXT:  .LBB9_1:
-; SLOW-DIVQ-NEXT:    # kill: def $eax killed $eax killed $rax
-; SLOW-DIVQ-NEXT:    xorl %edx, %edx
+; SLOW-DIVQ-DAG:     # kill: def $eax killed $eax killed $rax
+; SLOW-DIVQ-DAG:     xorl %edx, %edx
 ; SLOW-DIVQ-NEXT:    divl %esi
 ; SLOW-DIVQ-NEXT:    # kill: def $eax killed $eax def $rax
 ; SLOW-DIVQ-NEXT:    retq
@@ -266,10 +265,10 @@ define i64 @udiv_remainder(i64 %a, i64 %b) nounwind {
 ;
 ; SLOW-DIVQ-LABEL: udiv_remainder:
 ; SLOW-DIVQ:       # %bb.0:
-; SLOW-DIVQ-NEXT:    movq %rdi, %rax
-; SLOW-DIVQ-NEXT:    movq %rdi, %rcx
-; SLOW-DIVQ-NEXT:    orq %rsi, %rcx
-; SLOW-DIVQ-NEXT:    shrq $32, %rcx
+; SLOW-DIVQ-DAG:     movq %rdi, %rax
+; SLOW-DIVQ-DAG:     movq %rdi, %rcx
+; SLOW-DIVQ-DAG:     orq %rsi, %rcx
+; SLOW-DIVQ-DAG:     shrq $32, %rcx
 ; SLOW-DIVQ-NEXT:    je .LBB12_1
 ; SLOW-DIVQ-NEXT:  # %bb.2:
 ; SLOW-DIVQ-NEXT:    xorl %edx, %edx
@@ -277,8 +276,8 @@ define i64 @udiv_remainder(i64 %a, i64 %b) nounwind {
 ; SLOW-DIVQ-NEXT:    movq %rdx, %rax
 ; SLOW-DIVQ-NEXT:    retq
 ; SLOW-DIVQ-NEXT:  .LBB12_1:
-; SLOW-DIVQ-NEXT:    # kill: def $eax killed $eax killed $rax
-; SLOW-DIVQ-NEXT:    xorl %edx, %edx
+; SLOW-DIVQ-DAG:     # kill: def $eax killed $eax killed $rax
+; SLOW-DIVQ-DAG:     xorl %edx, %edx
 ; SLOW-DIVQ-NEXT:    divl %esi
 ; SLOW-DIVQ-NEXT:    movl %edx, %eax
 ; SLOW-DIVQ-NEXT:    retq
@@ -321,10 +320,10 @@ define i64 @udiv_quotient_and_remainder(i64 %a, i64 %b) nounwind {
 ;
 ; SLOW-DIVQ-LABEL: udiv_quotient_and_remainder:
 ; SLOW-DIVQ:       # %bb.0:
-; SLOW-DIVQ-NEXT:    movq %rdi, %rax
-; SLOW-DIVQ-NEXT:    movq %rdi, %rcx
-; SLOW-DIVQ-NEXT:    orq %rsi, %rcx
-; SLOW-DIVQ-NEXT:    shrq $32, %rcx
+; SLOW-DIVQ-DAG:     movq %rdi, %rax
+; SLOW-DIVQ-DAG:     movq %rdi, %rcx
+; SLOW-DIVQ-DAG:     orq %rsi, %rcx
+; SLOW-DIVQ-DAG:     shrq $32, %rcx
 ; SLOW-DIVQ-NEXT:    je .LBB15_1
 ; SLOW-DIVQ-NEXT:  # %bb.2:
 ; SLOW-DIVQ-NEXT:    xorl %edx, %edx
@@ -332,8 +331,8 @@ define i64 @udiv_quotient_and_remainder(i64 %a, i64 %b) nounwind {
 ; SLOW-DIVQ-NEXT:    addq %rdx, %rax
 ; SLOW-DIVQ-NEXT:    retq
 ; SLOW-DIVQ-NEXT:  .LBB15_1:
-; SLOW-DIVQ-NEXT:    # kill: def $eax killed $eax killed $rax
-; SLOW-DIVQ-NEXT:    xorl %edx, %edx
+; SLOW-DIVQ-DAG:     # kill: def $eax killed $eax killed $rax
+; SLOW-DIVQ-DAG:     xorl %edx, %edx
 ; SLOW-DIVQ-NEXT:    divl %esi
 ; SLOW-DIVQ-NEXT:    # kill: def $edx killed $edx def $rdx
 ; SLOW-DIVQ-NEXT:    # kill: def $eax killed $eax def $rax
-- 
GitLab


From c2a87d7e032f8e6c8cbe6ab4c7cfbb7f7996ca9f Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Thu, 9 May 2024 20:14:30 +0800
Subject: [PATCH 0280/1206] [AMDGPU] Remove unused lambda capture (NFC)

/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp:733:30:
error: lambda capture 'Ctx' is not used [-Werror,-Wunused-lambda-capture]
  auto TryGetMCExprValue = [&Ctx](const MCExpr *Value, uint64_t &Res) -> bool {
                            ~^~~
1 error generated.
---
 llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index de81904143b7..b7388ed9e85a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -730,7 +730,7 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
     return MCConstantExpr::create(Value, Ctx);
   };
 
-  auto TryGetMCExprValue = [&Ctx](const MCExpr *Value, uint64_t &Res) -> bool {
+  auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
     int64_t Val;
     if (Value->evaluateAsAbsolute(Val)) {
       Res = Val;
-- 
GitLab


From ad652efa1f65e16f5380acfba1bb132145984805 Mon Sep 17 00:00:00 2001
From: Daniil Kovalev <dkovalev@accesssoftek.com>
Date: Thu, 9 May 2024 15:32:18 +0300
Subject: [PATCH 0281/1206] [AArch64][PAC][clang][ELF] Support PAuth ABI core
 info (#85235)

Depends on #87545

Emit PAuth ABI compatibility tag values as llvm module flags:
- `aarch64-elf-pauthabi-platform`
- `aarch64-elf-pauthabi-version`

For platform 0x10000002 (llvm_linux), the version value bits correspond
to the following LangOptions defined in #85232:

- bit 0: `PointerAuthIntrinsics`;
- bit 1: `PointerAuthCalls`;
- bit 2: `PointerAuthReturns`;
- bit 3: `PointerAuthAuthTraps`;
- bit 4: `PointerAuthVTPtrAddressDiscrimination`;
- bit 5: `PointerAuthVTPtrTypeDiscrimination`;
- bit 6: `PointerAuthInitFini`.

---------

Co-authored-by: Ahmed Bougacha <ahmed@bougacha.org>
---
 clang/include/clang/Basic/Features.def    |   6 ++
 clang/include/clang/Basic/LangOptions.def |   6 ++
 clang/include/clang/Driver/Options.td     |   8 ++
 clang/lib/CodeGen/CodeGenModule.cpp       |  32 +++++++
 clang/lib/Driver/ToolChains/Clang.cpp     |  14 +++
 clang/lib/Frontend/CompilerInvocation.cpp |  20 ++++
 clang/test/CodeGen/aarch64-elf-pauthabi.c |  59 ++++++++++++
 clang/test/Driver/aarch64-ptrauth.c       |  28 +++++-
 clang/test/Preprocessor/ptrauth_feature.c | 107 +++++++++++++++++++++-
 9 files changed, 274 insertions(+), 6 deletions(-)
 create mode 100644 clang/test/CodeGen/aarch64-elf-pauthabi.c

diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def
index fe4d1c4afcca..b762e44e755e 100644
--- a/clang/include/clang/Basic/Features.def
+++ b/clang/include/clang/Basic/Features.def
@@ -103,6 +103,12 @@ FEATURE(thread_sanitizer, LangOpts.Sanitize.has(SanitizerKind::Thread))
 FEATURE(dataflow_sanitizer, LangOpts.Sanitize.has(SanitizerKind::DataFlow))
 FEATURE(scudo, LangOpts.Sanitize.hasOneOf(SanitizerKind::Scudo))
 FEATURE(ptrauth_intrinsics, LangOpts.PointerAuthIntrinsics)
+FEATURE(ptrauth_calls, LangOpts.PointerAuthCalls)
+FEATURE(ptrauth_returns, LangOpts.PointerAuthReturns)
+FEATURE(ptrauth_vtable_pointer_address_discrimination, LangOpts.PointerAuthVTPtrAddressDiscrimination)
+FEATURE(ptrauth_vtable_pointer_type_discrimination, LangOpts.PointerAuthVTPtrTypeDiscrimination)
+FEATURE(ptrauth_member_function_pointer_type_discrimination, LangOpts.PointerAuthCalls)
+FEATURE(ptrauth_init_fini, LangOpts.PointerAuthInitFini)
 EXTENSION(swiftcc,
   PP.getTargetInfo().checkCallingConvention(CC_Swift) ==
   clang::TargetInfo::CCCR_OK)
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index c967d8b22292..09eb92d6f10d 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -162,6 +162,12 @@ LANGOPT(RelaxedTemplateTemplateArgs, 1, 1, "C++17 relaxed matching of template t
 LANGOPT(ExperimentalLibrary, 1, 0, "enable unstable and experimental library features")
 
 LANGOPT(PointerAuthIntrinsics, 1, 0, "pointer authentication intrinsics")
+LANGOPT(PointerAuthCalls  , 1, 0, "function pointer authentication")
+LANGOPT(PointerAuthReturns, 1, 0, "return pointer authentication")
+LANGOPT(PointerAuthAuthTraps, 1, 0, "pointer authentication failure traps")
+LANGOPT(PointerAuthVTPtrAddressDiscrimination, 1, 0, "incorporate address discrimination in authenticated vtable pointers")
+LANGOPT(PointerAuthVTPtrTypeDiscrimination, 1, 0, "incorporate type discrimination in authenticated vtable pointers")
+LANGOPT(PointerAuthInitFini, 1, 0, "sign function pointers in init/fini arrays")
 
 LANGOPT(DoubleSquareBracketAttributes, 1, 0, "'[[]]' attributes extension for all language standard modes")
 LANGOPT(ExperimentalLateParseAttributes, 1, 0, "experimental late parsing of attributes")
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 142952897585..73a2518480e9 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -4180,6 +4180,14 @@ defm strict_return : BoolFOption<"strict-return",
 
 let Flags = [TargetSpecific] in {
 defm ptrauth_intrinsics : OptInCC1FFlag<"ptrauth-intrinsics", "Enable pointer authentication intrinsics">;
+defm ptrauth_calls : OptInCC1FFlag<"ptrauth-calls", "Enable signing and authentication of all indirect calls">;
+defm ptrauth_returns : OptInCC1FFlag<"ptrauth-returns", "Enable signing and authentication of return addresses">;
+defm ptrauth_auth_traps : OptInCC1FFlag<"ptrauth-auth-traps", "Enable traps on authentication failures">;
+defm ptrauth_vtable_pointer_address_discrimination :
+  OptInCC1FFlag<"ptrauth-vtable-pointer-address-discrimination", "Enable address discrimination of vtable pointers">;
+defm ptrauth_vtable_pointer_type_discrimination :
+  OptInCC1FFlag<"ptrauth-vtable-pointer-type-discrimination", "Enable type discrimination of vtable pointers">;
+defm ptrauth_init_fini : OptInCC1FFlag<"ptrauth-init-fini", "Enable signing of function pointers in init/fini arrays">;
 }
 
 def fenable_matrix : Flag<["-"], "fenable-matrix">, Group<f_Group>,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index c8898ce196c1..489c08a4d481 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -53,6 +53,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/IR/AttributeMask.h"
 #include "llvm/IR/CallingConv.h"
@@ -1190,6 +1191,37 @@ void CodeGenModule::Release() {
     if (!LangOpts.isSignReturnAddressWithAKey())
       getModule().addModuleFlag(llvm::Module::Min,
                                 "sign-return-address-with-bkey", 1);
+
+    if (getTriple().isOSLinux()) {
+      assert(getTriple().isOSBinFormatELF());
+      using namespace llvm::ELF;
+      uint64_t PAuthABIVersion =
+          (LangOpts.PointerAuthIntrinsics
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INTRINSICS) |
+          (LangOpts.PointerAuthCalls
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_CALLS) |
+          (LangOpts.PointerAuthReturns
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_RETURNS) |
+          (LangOpts.PointerAuthAuthTraps
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_AUTHTRAPS) |
+          (LangOpts.PointerAuthVTPtrAddressDiscrimination
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_VPTRADDRDISCR) |
+          (LangOpts.PointerAuthVTPtrTypeDiscrimination
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_VPTRTYPEDISCR) |
+          (LangOpts.PointerAuthInitFini
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINI);
+      static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINI ==
+                        AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_LAST,
+                    "Update when new enum items are defined");
+      if (PAuthABIVersion != 0) {
+        getModule().addModuleFlag(llvm::Module::Error,
+                                  "aarch64-elf-pauthabi-platform",
+                                  AARCH64_PAUTH_PLATFORM_LLVM_LINUX);
+        getModule().addModuleFlag(llvm::Module::Error,
+                                  "aarch64-elf-pauthabi-version",
+                                  PAuthABIVersion);
+      }
+    }
   }
 
   if (CodeGenOpts.StackClashProtector)
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 0a2ea96de738..775dc249999e 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1756,6 +1756,20 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
 
   Args.addOptInFlag(CmdArgs, options::OPT_fptrauth_intrinsics,
                     options::OPT_fno_ptrauth_intrinsics);
+  Args.addOptInFlag(CmdArgs, options::OPT_fptrauth_calls,
+                    options::OPT_fno_ptrauth_calls);
+  Args.addOptInFlag(CmdArgs, options::OPT_fptrauth_returns,
+                    options::OPT_fno_ptrauth_returns);
+  Args.addOptInFlag(CmdArgs, options::OPT_fptrauth_auth_traps,
+                    options::OPT_fno_ptrauth_auth_traps);
+  Args.addOptInFlag(
+      CmdArgs, options::OPT_fptrauth_vtable_pointer_address_discrimination,
+      options::OPT_fno_ptrauth_vtable_pointer_address_discrimination);
+  Args.addOptInFlag(
+      CmdArgs, options::OPT_fptrauth_vtable_pointer_type_discrimination,
+      options::OPT_fno_ptrauth_vtable_pointer_type_discrimination);
+  Args.addOptInFlag(CmdArgs, options::OPT_fptrauth_init_fini,
+                    options::OPT_fno_ptrauth_init_fini);
 }
 
 void Clang::AddLoongArchTargetArgs(const ArgList &Args,
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 948fe08c863a..dbb5f5662ebf 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -3346,11 +3346,31 @@ static void GeneratePointerAuthArgs(const LangOptions &Opts,
                                     ArgumentConsumer Consumer) {
   if (Opts.PointerAuthIntrinsics)
     GenerateArg(Consumer, OPT_fptrauth_intrinsics);
+  if (Opts.PointerAuthCalls)
+    GenerateArg(Consumer, OPT_fptrauth_calls);
+  if (Opts.PointerAuthReturns)
+    GenerateArg(Consumer, OPT_fptrauth_returns);
+  if (Opts.PointerAuthAuthTraps)
+    GenerateArg(Consumer, OPT_fptrauth_auth_traps);
+  if (Opts.PointerAuthVTPtrAddressDiscrimination)
+    GenerateArg(Consumer, OPT_fptrauth_vtable_pointer_address_discrimination);
+  if (Opts.PointerAuthVTPtrTypeDiscrimination)
+    GenerateArg(Consumer, OPT_fptrauth_vtable_pointer_type_discrimination);
+  if (Opts.PointerAuthInitFini)
+    GenerateArg(Consumer, OPT_fptrauth_init_fini);
 }
 
 static void ParsePointerAuthArgs(LangOptions &Opts, ArgList &Args,
                                  DiagnosticsEngine &Diags) {
   Opts.PointerAuthIntrinsics = Args.hasArg(OPT_fptrauth_intrinsics);
+  Opts.PointerAuthCalls = Args.hasArg(OPT_fptrauth_calls);
+  Opts.PointerAuthReturns = Args.hasArg(OPT_fptrauth_returns);
+  Opts.PointerAuthAuthTraps = Args.hasArg(OPT_fptrauth_auth_traps);
+  Opts.PointerAuthVTPtrAddressDiscrimination =
+      Args.hasArg(OPT_fptrauth_vtable_pointer_address_discrimination);
+  Opts.PointerAuthVTPtrTypeDiscrimination =
+      Args.hasArg(OPT_fptrauth_vtable_pointer_type_discrimination);
+  Opts.PointerAuthInitFini = Args.hasArg(OPT_fptrauth_init_fini);
 }
 
 /// Check if input file kind and language standard are compatible.
diff --git a/clang/test/CodeGen/aarch64-elf-pauthabi.c b/clang/test/CodeGen/aarch64-elf-pauthabi.c
new file mode 100644
index 000000000000..aa83ee3e0d7b
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-elf-pauthabi.c
@@ -0,0 +1,59 @@
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-intrinsics \
+// RUN:   -fptrauth-calls \
+// RUN:   -fptrauth-returns \
+// RUN:   -fptrauth-auth-traps \
+// RUN:   -fptrauth-vtable-pointer-address-discrimination \
+// RUN:   -fptrauth-vtable-pointer-type-discrimination \
+// RUN:   -fptrauth-init-fini %s | \
+// RUN:   FileCheck %s --check-prefix=ALL
+
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-intrinsics %s | FileCheck %s --check-prefix=INTRIN
+
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-calls %s | FileCheck %s --check-prefix=CALL
+
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-returns %s | FileCheck %s --check-prefix=RET
+
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-auth-traps %s | FileCheck %s --check-prefix=TRAP
+
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-calls -fptrauth-vtable-pointer-address-discrimination %s | \
+// RUN:   FileCheck %s --check-prefix=VPTRADDR
+
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-calls -fptrauth-vtable-pointer-type-discrimination %s | \
+// RUN:   FileCheck %s --check-prefix=VPTRTYPE
+
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-calls -fptrauth-init-fini %s | \
+// RUN:   FileCheck %s --check-prefix=INITFINI
+
+// ALL: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// ALL: !{i32 1, !"aarch64-elf-pauthabi-version", i32 127}
+
+// INTRIN: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// INTRIN: !{i32 1, !"aarch64-elf-pauthabi-version", i32 1}
+
+// CALL: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// CALL: !{i32 1, !"aarch64-elf-pauthabi-version", i32 2}
+
+// RET: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// RET: !{i32 1, !"aarch64-elf-pauthabi-version", i32 4}
+
+// TRAP: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// TRAP: !{i32 1, !"aarch64-elf-pauthabi-version", i32 8}
+
+// VPTRADDR: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// VPTRADDR: !{i32 1, !"aarch64-elf-pauthabi-version", i32 18}
+
+// VPTRTYPE: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// VPTRTYPE: !{i32 1, !"aarch64-elf-pauthabi-version", i32 34}
+
+// INITFINI: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// INITFINI: !{i32 1, !"aarch64-elf-pauthabi-version", i32 66}
+
+void foo() {}
diff --git a/clang/test/Driver/aarch64-ptrauth.c b/clang/test/Driver/aarch64-ptrauth.c
index 1a69b2c6edfb..fa0125f4b22a 100644
--- a/clang/test/Driver/aarch64-ptrauth.c
+++ b/clang/test/Driver/aarch64-ptrauth.c
@@ -1,5 +1,25 @@
-// RUN: %clang -### -c --target=aarch64 -fno-ptrauth-intrinsics -fptrauth-intrinsics %s 2>&1 | FileCheck %s --check-prefix=INTRIN
-// INTRIN: "-cc1"{{.*}} "-fptrauth-intrinsics"
+// RUN: %clang -### -c --target=aarch64 %s 2>&1 | FileCheck %s --check-prefix NONE
+// NONE: "-cc1"
+// NONE-NOT: "-fptrauth-
 
-// RUN: not %clang -### -c --target=x86_64 -fptrauth-intrinsics %s 2>&1 | FileCheck %s --check-prefix=ERR
-// ERR: error: unsupported option '-fptrauth-intrinsics' for target '{{.*}}'
+// RUN: %clang -### -c --target=aarch64 \
+// RUN:   -fno-ptrauth-intrinsics -fptrauth-intrinsics \
+// RUN:   -fno-ptrauth-calls -fptrauth-calls \
+// RUN:   -fno-ptrauth-returns -fptrauth-returns \
+// RUN:   -fno-ptrauth-auth-traps -fptrauth-auth-traps \
+// RUN:   -fno-ptrauth-vtable-pointer-address-discrimination -fptrauth-vtable-pointer-address-discrimination \
+// RUN:   -fno-ptrauth-vtable-pointer-type-discrimination -fptrauth-vtable-pointer-type-discrimination \
+// RUN:   -fno-ptrauth-init-fini -fptrauth-init-fini \
+// RUN:   %s 2>&1 | FileCheck %s --check-prefix=ALL
+// ALL: "-cc1"{{.*}} "-fptrauth-intrinsics" "-fptrauth-calls" "-fptrauth-returns" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-init-fini"
+
+// RUN: not %clang -### -c --target=x86_64 -fptrauth-intrinsics -fptrauth-calls -fptrauth-returns -fptrauth-auth-traps \
+// RUN:   -fptrauth-vtable-pointer-address-discrimination -fptrauth-vtable-pointer-type-discrimination \
+// RUN:   -fptrauth-init-fini %s 2>&1 | FileCheck %s --check-prefix=ERR
+// ERR:      error: unsupported option '-fptrauth-intrinsics' for target '{{.*}}'
+// ERR-NEXT: error: unsupported option '-fptrauth-calls' for target '{{.*}}'
+// ERR-NEXT: error: unsupported option '-fptrauth-returns' for target '{{.*}}'
+// ERR-NEXT: error: unsupported option '-fptrauth-auth-traps' for target '{{.*}}'
+// ERR-NEXT: error: unsupported option '-fptrauth-vtable-pointer-address-discrimination' for target '{{.*}}'
+// ERR-NEXT: error: unsupported option '-fptrauth-vtable-pointer-type-discrimination' for target '{{.*}}'
+// ERR-NEXT: error: unsupported option '-fptrauth-init-fini' for target '{{.*}}'
diff --git a/clang/test/Preprocessor/ptrauth_feature.c b/clang/test/Preprocessor/ptrauth_feature.c
index e45c6ea90fd1..80e239110ffc 100644
--- a/clang/test/Preprocessor/ptrauth_feature.c
+++ b/clang/test/Preprocessor/ptrauth_feature.c
@@ -1,5 +1,59 @@
-// RUN: %clang_cc1 %s -E -triple=arm64-- | FileCheck %s --check-prefixes=NOINTRIN
-// RUN: %clang_cc1 %s -E -triple=arm64-- -fptrauth-intrinsics | FileCheck %s --check-prefixes=INTRIN
+// RUN: %clang_cc1 -E %s -triple=aarch64 \
+// RUN:   -fptrauth-intrinsics \
+// RUN:   -fptrauth-calls \
+// RUN:   -fptrauth-returns \
+// RUN:   -fptrauth-vtable-pointer-address-discrimination \
+// RUN:   -fptrauth-vtable-pointer-type-discrimination \
+// RUN:   -fptrauth-init-fini | \
+// RUN:   FileCheck %s --check-prefixes=INTRIN,CALLS,RETS,VPTR_ADDR_DISCR,VPTR_TYPE_DISCR,INITFINI
+
+// RUN: %clang_cc1 -E %s -triple=aarch64 \
+// RUN:   -fptrauth-calls \
+// RUN:   -fptrauth-returns \
+// RUN:   -fptrauth-vtable-pointer-address-discrimination \
+// RUN:   -fptrauth-vtable-pointer-type-discrimination \
+// RUN:   -fptrauth-init-fini | \
+// RUN:   FileCheck %s --check-prefixes=NOINTRIN,CALLS,RETS,VPTR_ADDR_DISCR,VPTR_TYPE_DISCR,INITFINI
+
+// RUN: %clang_cc1 -E %s -triple=aarch64 \
+// RUN:   -fptrauth-intrinsics \
+// RUN:   -fptrauth-returns \
+// RUN:   -fptrauth-vtable-pointer-address-discrimination \
+// RUN:   -fptrauth-vtable-pointer-type-discrimination \
+// RUN:   -fptrauth-init-fini | \
+// RUN:   FileCheck %s --check-prefixes=INTRIN,NOCALLS,RETS,VPTR_ADDR_DISCR,VPTR_TYPE_DISCR,INITFINI
+
+// RUN: %clang_cc1 -E %s -triple=aarch64 \
+// RUN:   -fptrauth-intrinsics \
+// RUN:   -fptrauth-calls \
+// RUN:   -fptrauth-vtable-pointer-address-discrimination \
+// RUN:   -fptrauth-vtable-pointer-type-discrimination \
+// RUN:   -fptrauth-init-fini | \
+// RUN:   FileCheck %s --check-prefixes=INTRIN,CALLS,NORETS,VPTR_ADDR_DISCR,VPTR_TYPE_DISCR,INITFINI
+
+// RUN: %clang_cc1 -E %s -triple=aarch64 \
+// RUN:   -fptrauth-intrinsics \
+// RUN:   -fptrauth-calls \
+// RUN:   -fptrauth-returns \
+// RUN:   -fptrauth-vtable-pointer-type-discrimination \
+// RUN:   -fptrauth-init-fini | \
+// RUN:   FileCheck %s --check-prefixes=INTRIN,CALLS,RETS,NOVPTR_ADDR_DISCR,VPTR_TYPE_DISCR,INITFINI
+
+// RUN: %clang_cc1 -E %s -triple=aarch64 \
+// RUN:   -fptrauth-intrinsics \
+// RUN:   -fptrauth-calls \
+// RUN:   -fptrauth-returns \
+// RUN:   -fptrauth-vtable-pointer-address-discrimination \
+// RUN:   -fptrauth-init-fini | \
+// RUN:   FileCheck %s --check-prefixes=INTRIN,CALLS,RETS,VPTR_ADDR_DISCR,NOVPTR_TYPE_DISCR,INITFINI
+
+// RUN: %clang_cc1 -E %s -triple=aarch64 \
+// RUN:   -fptrauth-intrinsics \
+// RUN:   -fptrauth-calls \
+// RUN:   -fptrauth-returns \
+// RUN:   -fptrauth-vtable-pointer-address-discrimination \
+// RUN:   -fptrauth-vtable-pointer-type-discrimination | \
+// RUN:   FileCheck %s --check-prefixes=INTRIN,CALLS,RETS,VPTR_ADDR_DISCR,VPTR_TYPE_DISCR,NOINITFINI
 
 #if __has_feature(ptrauth_intrinsics)
 // INTRIN: has_ptrauth_intrinsics
@@ -8,3 +62,52 @@ void has_ptrauth_intrinsics() {}
 // NOINTRIN: no_ptrauth_intrinsics
 void no_ptrauth_intrinsics() {}
 #endif
+
+#if __has_feature(ptrauth_calls)
+// CALLS: has_ptrauth_calls
+void has_ptrauth_calls() {}
+#else
+// NOCALLS: no_ptrauth_calls
+void no_ptrauth_calls() {}
+#endif
+
+// This is always enabled when ptrauth_calls is enabled
+#if __has_feature(ptrauth_member_function_pointer_type_discrimination)
+// CALLS: has_ptrauth_member_function_pointer_type_discrimination
+void has_ptrauth_member_function_pointer_type_discrimination() {}
+#else
+// NOCALLS: no_ptrauth_member_function_pointer_type_discrimination
+void no_ptrauth_member_function_pointer_type_discrimination() {}
+#endif
+
+#if __has_feature(ptrauth_returns)
+// RETS: has_ptrauth_returns
+void has_ptrauth_returns() {}
+#else
+// NORETS: no_ptrauth_returns
+void no_ptrauth_returns() {}
+#endif
+
+#if __has_feature(ptrauth_vtable_pointer_address_discrimination)
+// VPTR_ADDR_DISCR: has_ptrauth_vtable_pointer_address_discrimination
+void has_ptrauth_vtable_pointer_address_discrimination() {}
+#else
+// NOVPTR_ADDR_DISCR: no_ptrauth_vtable_pointer_address_discrimination
+void no_ptrauth_vtable_pointer_address_discrimination() {}
+#endif
+
+#if __has_feature(ptrauth_vtable_pointer_type_discrimination)
+// VPTR_TYPE_DISCR: has_ptrauth_vtable_pointer_type_discrimination
+void has_ptrauth_vtable_pointer_type_discrimination() {}
+#else
+// NOVPTR_TYPE_DISCR: no_ptrauth_vtable_pointer_type_discrimination
+void no_ptrauth_vtable_pointer_type_discrimination() {}
+#endif
+
+#if __has_feature(ptrauth_init_fini)
+// INITFINI: has_ptrauth_init_fini
+void has_ptrauth_init_fini() {}
+#else
+// NOINITFINI: no_ptrauth_init_fini
+void no_ptrauth_init_fini() {}
+#endif
-- 
GitLab


From 6a8d30b1c1be5e46deb9b3a17915d488e1dbc470 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 9 May 2024 14:41:13 +0200
Subject: [PATCH 0282/1206] DAG: Skip 0 sign handling in minimum/maximum
 lowering for _ieee case (#91326)

dc9664a8adae17f2083fbcc8e96cfce606c56d57 changed the documentation to
assume these order -0 as less than +0.
---
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |   10 +-
 llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll  | 3438 +++--------------
 llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll  |  267 +-
 llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll  |  484 +--
 llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll  | 2600 ++-----------
 llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll  |  267 +-
 llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll  |  484 +--
 .../test/CodeGen/PowerPC/fminimum-fmaximum.ll |  240 +-
 8 files changed, 1286 insertions(+), 6504 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 9ec3ac4f9991..7beaeb9b7a17 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -8401,8 +8401,14 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
   SDValue MinMax;
   unsigned CompOpcIeee = IsMax ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
   unsigned CompOpc = IsMax ? ISD::FMAXNUM : ISD::FMINNUM;
+
+  // FIXME: We should probably define fminnum/fmaxnum variants with correct
+  // signed zero behavior.
+  bool MinMaxMustRespectOrderedZero = false;
+
   if (isOperationLegalOrCustom(CompOpcIeee, VT)) {
     MinMax = DAG.getNode(CompOpcIeee, DL, VT, LHS, RHS);
+    MinMaxMustRespectOrderedZero = true;
   } else if (isOperationLegalOrCustom(CompOpc, VT)) {
     MinMax = DAG.getNode(CompOpc, DL, VT, LHS, RHS);
   } else {
@@ -8422,8 +8428,8 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
   }
 
   // fminimum/fmaximum requires -0.0 less than +0.0
-  if (!N->getFlags().hasNoSignedZeros() && !DAG.isKnownNeverZeroFloat(RHS) &&
-      !DAG.isKnownNeverZeroFloat(LHS)) {
+  if (!MinMaxMustRespectOrderedZero && !N->getFlags().hasNoSignedZeros() &&
+      !DAG.isKnownNeverZeroFloat(RHS) && !DAG.isKnownNeverZeroFloat(LHS)) {
     SDValue IsZero = DAG.getSetCC(DL, CCVT, MinMax,
                                   DAG.getConstantFP(0.0, DL, VT), ISD::SETEQ);
     SDValue TestZero =
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
index c476208ed8f4..7d7a46259710 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
@@ -18,13 +18,7 @@ define half @v_maximum_f16(half %src0, half %src1) {
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_max_f32_e32 v3, v0, v1
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f16:
@@ -33,13 +27,7 @@ define half @v_maximum_f16(half %src0, half %src1) {
 ; GFX8-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f16:
@@ -48,13 +36,7 @@ define half @v_maximum_f16(half %src0, half %src1) {
 ; GFX9-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f16:
@@ -64,16 +46,7 @@ define half @v_maximum_f16(half %src0, half %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f16:
@@ -81,13 +54,7 @@ define half @v_maximum_f16(half %src0, half %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f16:
@@ -95,15 +62,8 @@ define half @v_maximum_f16(half %src0, half %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f16:
@@ -127,78 +87,37 @@ define half @v_maximum_f16__nnan(half %src0, half %src1) {
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_max_f32_e32 v2, v0, v1
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f16__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_max_f16_e32 v2, v0, v1
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_max_f16_e32 v0, v0, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f16__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_max_f16_e32 v2, v0, v1
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_max_f16_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f16__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_max_f16_e32 v2, v0, v1
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_max_f16_e32 v0, v0, v1
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f16__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_max_f16_e32 v2, v0, v1
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_max_f16_e32 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f16__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_max_f16_e32 v2, v0, v1
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_max_f16_e32 v0, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f16__nnan:
@@ -352,13 +271,7 @@ define half @v_maximum_f16__nnan_src0(half %arg0, half %src1) {
 ; GFX7-NEXT:    v_add_f32_e32 v0, 1.0, v0
 ; GFX7-NEXT:    v_max_f32_e32 v3, v0, v1
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f16__nnan_src0:
@@ -368,13 +281,7 @@ define half @v_maximum_f16__nnan_src0(half %arg0, half %src1) {
 ; GFX8-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f16__nnan_src0:
@@ -384,13 +291,7 @@ define half @v_maximum_f16__nnan_src0(half %arg0, half %src1) {
 ; GFX9-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f16__nnan_src0:
@@ -401,16 +302,7 @@ define half @v_maximum_f16__nnan_src0(half %arg0, half %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f16__nnan_src0:
@@ -419,13 +311,7 @@ define half @v_maximum_f16__nnan_src0(half %arg0, half %src1) {
 ; GFX10-NEXT:    v_add_f16_e32 v0, 1.0, v0
 ; GFX10-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f16__nnan_src0:
@@ -435,15 +321,7 @@ define half @v_maximum_f16__nnan_src0(half %arg0, half %src1) {
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f16__nnan_src0:
@@ -474,13 +352,7 @@ define half @v_maximum_f16__nnan_src1(half %src0, half %arg1) {
 ; GFX7-NEXT:    v_add_f32_e32 v1, 1.0, v1
 ; GFX7-NEXT:    v_max_f32_e32 v3, v0, v1
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f16__nnan_src1:
@@ -490,13 +362,7 @@ define half @v_maximum_f16__nnan_src1(half %src0, half %arg1) {
 ; GFX8-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f16__nnan_src1:
@@ -506,13 +372,7 @@ define half @v_maximum_f16__nnan_src1(half %src0, half %arg1) {
 ; GFX9-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f16__nnan_src1:
@@ -523,16 +383,7 @@ define half @v_maximum_f16__nnan_src1(half %src0, half %arg1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f16__nnan_src1:
@@ -541,13 +392,7 @@ define half @v_maximum_f16__nnan_src1(half %src0, half %arg1) {
 ; GFX10-NEXT:    v_add_f16_e32 v1, 1.0, v1
 ; GFX10-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f16__nnan_src1:
@@ -557,15 +402,7 @@ define half @v_maximum_f16__nnan_src1(half %src0, half %arg1) {
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_max_f16_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f16__nnan_src1:
@@ -595,13 +432,7 @@ define void @s_maximum_f16(half inreg %src0, half inreg %src1) {
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; GFX7-NEXT:    v_max_f32_e32 v3, v1, v0
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    ;;#ASMSTART
 ; GFX7-NEXT:    ; use v0
@@ -615,14 +446,7 @@ define void @s_maximum_f16(half inreg %src0, half inreg %src1) {
 ; GFX8-NEXT:    v_max_f16_e32 v1, s4, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use v0
@@ -636,14 +460,7 @@ define void @s_maximum_f16(half inreg %src0, half inreg %src1) {
 ; GFX9-NEXT:    v_max_f16_e32 v1, s4, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    ;;#ASMSTART
 ; GFX9-NEXT:    ; use v0
@@ -658,17 +475,7 @@ define void @s_maximum_f16(half inreg %src0, half inreg %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, s0, v0
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v2, s0
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v1
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX940-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; use v0
@@ -680,13 +487,7 @@ define void @s_maximum_f16(half inreg %src0, half inreg %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_max_f16_e64 v0, s4, s5
 ; GFX10-NEXT:    v_cmp_o_f16_e64 vcc_lo, s4, s5
-; GFX10-NEXT:    v_cmp_class_f16_e64 s6, s4, 64
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v0, s4, s6
-; GFX10-NEXT:    v_cmp_class_f16_e64 s4, s5, 64
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s5, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; use v0
@@ -698,16 +499,8 @@ define void @s_maximum_f16(half inreg %src0, half inreg %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_max_f16_e64 v0, s0, s1
 ; GFX11-NEXT:    v_cmp_o_f16_e64 vcc_lo, s0, s1
-; GFX11-NEXT:    v_cmp_class_f16_e64 s2, s0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v0, s0, s2
-; GFX11-NEXT:    v_cmp_class_f16_e64 s0, s1, 64
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s1, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use v0
@@ -750,22 +543,10 @@ define <2 x half> @v_maximum_v2f16(<2 x half> %src0, <2 x half> %src1) {
 ; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX7-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v2, v1, v3
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f16:
@@ -805,23 +586,9 @@ define <2 x half> @v_maximum_v2f16(<2 x half> %src0, <2 x half> %src1) {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    v_perm_b32 v0, v0, v4, s4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -835,30 +602,10 @@ define <2 x half> @v_maximum_v2f16(<2 x half> %src0, <2 x half> %src1) {
 ; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    v_perm_b32 v0, v0, v4, s0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -866,26 +613,12 @@ define <2 x half> @v_maximum_v2f16(<2 x half> %src0, <2 x half> %src1) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_max_f16 v2, v0, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7e00, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v3, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v0, v0, v2, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v2f16:
@@ -897,25 +630,10 @@ define <2 x half> @v_maximum_v2f16(<2 x half> %src0, <2 x half> %src1) {
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, 0x7e00, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v5, vcc_lo
 ; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -947,22 +665,10 @@ define <2 x half> @v_maximum_v2f16__nnan(<2 x half> %src0, <2 x half> %src1) {
 ; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX7-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v2, v1, v3
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f16__nnan:
@@ -993,100 +699,25 @@ define <2 x half> @v_maximum_v2f16__nnan(<2 x half> %src0, <2 x half> %src1) {
 ; GFX9-LABEL: v_maximum_v2f16__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX9-NEXT:    v_pk_max_f16 v3, v0, v1
-; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
-; GFX9-NEXT:    v_perm_b32 v0, v2, v0, s4
+; GFX9-NEXT:    v_pk_max_f16 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v2f16__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX940-NEXT:    v_pk_max_f16 v3, v0, v1
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX940-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX940-NEXT:    v_perm_b32 v0, v2, v0, s0
+; GFX940-NEXT:    v_pk_max_f16 v0, v0, v1
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v2f16__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_pk_max_f16 v2, v0, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    v_pk_max_f16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v2f16__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_pk_max_f16 v2, v0, v1
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-NEXT:    v_pk_max_f16 v0, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v2f16__nnan:
@@ -1117,22 +748,10 @@ define <2 x half> @v_maximum_v2f16__nsz(<2 x half> %src0, <2 x half> %src1) {
 ; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX7-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v2, v1, v3
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f16__nsz:
@@ -1239,22 +858,10 @@ define <2 x half> @v_maximum_v2f16__nnan_nsz(<2 x half> %src0, <2 x half> %src1)
 ; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX7-NEXT:    v_max_f32_e32 v4, v0, v2
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v2, v1, v3
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v2f16__nnan_nsz:
@@ -1322,23 +929,11 @@ define void @s_maximum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
 ; GFX7-NEXT:    v_mov_b32_e32 v5, 0x7fc00000
 ; GFX7-NEXT:    v_max_f32_e32 v4, v1, v0
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v5, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v1, v3, v2
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX7-NEXT:    v_or_b32_e32 v0, v1, v0
@@ -1389,30 +984,16 @@ define void @s_maximum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s5
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s5, 16
 ; GFX9-NEXT:    v_pk_max_f16 v1, s4, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX9-NEXT:    s_lshr_b32 s5, s5, 16
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX9-NEXT:    s_lshr_b32 s4, s4, 16
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, s4, v3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX9-NEXT:    ;;#ASMSTART
@@ -1425,38 +1006,18 @@ define void @s_maximum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX940-NEXT:    v_mov_b32_e32 v1, s1
+; GFX940-NEXT:    s_lshr_b32 s1, s1, 16
 ; GFX940-NEXT:    v_pk_max_f16 v1, s0, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, s0, v0
-; GFX940-NEXT:    v_mov_b32_e32 v4, s0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v2, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s0, 64
 ; GFX940-NEXT:    s_lshr_b32 s0, s0, 16
-; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s1, 64
-; GFX940-NEXT:    s_lshr_b32 s1, s1, 16
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX940-NEXT:    v_mov_b32_e32 v3, s1
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, s0, v3
 ; GFX940-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v2, s0
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v1
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; GFX940-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; use v0
@@ -1469,24 +1030,12 @@ define void @s_maximum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
 ; GFX10-NEXT:    v_pk_max_f16 v0, s4, s5
 ; GFX10-NEXT:    v_cmp_o_f16_e64 vcc_lo, s4, s5
 ; GFX10-NEXT:    s_lshr_b32 s6, s5, 16
-; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX10-NEXT:    v_cmp_class_f16_e64 s8, s4, 64
+; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e64 vcc_lo, s7, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v0, s4, s8
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 s4, s7, 64
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v1, s7, s4
-; GFX10-NEXT:    v_cmp_class_f16_e64 s4, s5, 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s5, s4
-; GFX10-NEXT:    v_cmp_class_f16_e64 s4, s6, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, s6, s4
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v1
+; GFX10-NEXT:    v_cmp_o_f16_e64 vcc_lo, s4, s6
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v1, vcc_lo
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; use v0
@@ -1499,27 +1048,14 @@ define void @s_maximum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
 ; GFX11-NEXT:    v_pk_max_f16 v0, s0, s1
 ; GFX11-NEXT:    v_cmp_o_f16_e64 vcc_lo, s0, s1
 ; GFX11-NEXT:    s_lshr_b32 s2, s1, 16
-; GFX11-NEXT:    s_lshr_b32 s3, s0, 16
-; GFX11-NEXT:    v_cmp_class_f16_e64 s4, s0, 64
+; GFX11-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e64 vcc_lo, s3, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v0, s0, s4
+; GFX11-NEXT:    v_cmp_o_f16_e64 vcc_lo, s0, s2
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 s0, s3, 64
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v1, s3, s0
-; GFX11-NEXT:    v_cmp_class_f16_e64 s0, s1, 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s1, s0
-; GFX11-NEXT:    v_cmp_class_f16_e64 s0, s2, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, s2, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_dual_cndmask_b32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff, v0
 ; GFX11-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use v0
@@ -1552,42 +1088,24 @@ define <3 x half> @v_maximum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; GFX7-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v3, v1, v4
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f16:
@@ -1598,31 +1116,13 @@ define <3 x half> @v_maximum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX8-NEXT:    v_max_f16_e32 v6, v5, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v5, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v6, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v5, v1, v3
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v5, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v3, v0, v2
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -1633,33 +1133,13 @@ define <3 x half> @v_maximum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX9-NEXT:    v_pk_max_f16 v4, v1, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
 ; GFX9-NEXT:    v_pk_max_f16 v3, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
 ; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
 ; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    v_perm_b32 v0, v0, v4, s4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -1670,46 +1150,16 @@ define <3 x half> @v_maximum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX940-NEXT:    v_pk_max_f16 v4, v1, v3
 ; GFX940-NEXT:    v_mov_b32_e32 v5, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
 ; GFX940-NEXT:    v_pk_max_f16 v3, v0, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
 ; GFX940-NEXT:    v_perm_b32 v0, v0, v4, s0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1717,35 +1167,15 @@ define <3 x half> @v_maximum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_max_f16 v4, v0, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_pk_max_f16 v8, v1, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0x7e00, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_pk_max_f16 v2, v1, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v6, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v0, v0, v4, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v3f16:
@@ -1755,35 +1185,17 @@ define <3 x half> @v_maximum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_pk_max_f16 v8, v1, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v4, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v4, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, 0x7e00, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc_lo
+; GFX11-NEXT:    v_pk_max_f16 v4, v1, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v7, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v6, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX11-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v4, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v3f16:
@@ -1808,201 +1220,61 @@ define <3 x half> @v_maximum_v3f16__nnan(<3 x half> %src0, <3 x half> %src1) {
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; GFX7-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v3, v1, v4
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f16__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT:    v_max_f16_e32 v6, v5, v4
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX8-NEXT:    v_max_f16_e32 v5, v1, v3
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_max_f16_e32 v3, v0, v2
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_e32 v0, v0, v2
+; GFX8-NEXT:    v_max_f16_e32 v1, v1, v3
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v3f16__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
-; GFX9-NEXT:    v_pk_max_f16 v5, v0, v2
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT:    v_pk_max_f16 v6, v1, v3
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
-; GFX9-NEXT:    v_perm_b32 v0, v4, v0, s4
+; GFX9-NEXT:    v_pk_max_f16 v0, v0, v2
+; GFX9-NEXT:    v_pk_max_f16 v1, v1, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v3f16__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
-; GFX940-NEXT:    v_pk_max_f16 v5, v0, v2
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX940-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_pk_max_f16 v6, v1, v3
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX940-NEXT:    v_perm_b32 v0, v4, v0, s0
+; GFX940-NEXT:    v_pk_max_f16 v0, v0, v2
+; GFX940-NEXT:    v_pk_max_f16 v1, v1, v3
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v3f16__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_pk_max_f16 v4, v0, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
-; GFX10-NEXT:    v_pk_max_f16 v8, v1, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX10-NEXT:    v_pk_max_f16 v0, v0, v2
+; GFX10-NEXT:    v_pk_max_f16 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v3f16__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_pk_max_f16 v4, v0, v2
-; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
-; GFX11-NEXT:    v_pk_max_f16 v8, v1, v3
-; GFX11-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX11-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
+; GFX11-NEXT:    v_pk_max_f16 v0, v0, v2
+; GFX11-NEXT:    v_pk_max_f16 v1, v1, v3
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v3f16__nnan:
@@ -2027,42 +1299,24 @@ define <3 x half> @v_maximum_v3f16__nsz(<3 x half> %src0, <3 x half> %src1) {
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; GFX7-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v3, v1, v4
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f16__nsz:
@@ -2177,42 +1431,24 @@ define <3 x half> @v_maximum_v3f16__nnan_nsz(<3 x half> %src0, <3 x half> %src1)
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; GFX7-NEXT:    v_max_f32_e32 v6, v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v7, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v6
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v6, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v3, v1, v4
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v7, v3, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v3, v2, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v7, v3, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v3f16__nnan_nsz:
@@ -2274,55 +1510,31 @@ define <4 x half> @v_maximum_v4f16(<4 x half> %src0, <4 x half> %src1) {
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX7-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v1, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f16:
@@ -2333,42 +1545,18 @@ define <4 x half> @v_maximum_v4f16(<4 x half> %src0, <4 x half> %src1) {
 ; GFX8-NEXT:    v_max_f16_e32 v6, v5, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v5, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v6, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
 ; GFX8-NEXT:    v_max_f16_e32 v8, v6, v5
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v6, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v7, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v6, v1, v3
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v3, v0, v2
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
@@ -2382,43 +1570,15 @@ define <4 x half> @v_maximum_v4f16(<4 x half> %src0, <4 x half> %src1) {
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
 ; GFX9-NEXT:    v_pk_max_f16 v3, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
 ; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
 ; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    v_perm_b32 v0, v0, v4, s4
 ; GFX9-NEXT:    v_perm_b32 v1, v1, v6, s4
@@ -2433,59 +1593,19 @@ define <4 x half> @v_maximum_v4f16(<4 x half> %src0, <4 x half> %src1) {
 ; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v6, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_max_f16 v3, v0, v2
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
 ; GFX940-NEXT:    v_perm_b32 v1, v1, v6, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
 ; GFX940-NEXT:    v_perm_b32 v0, v0, v4, s0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2494,46 +1614,18 @@ define <4 x half> @v_maximum_v4f16(<4 x half> %src0, <4 x half> %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_max_f16 v4, v1, v3
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX10-NEXT:    v_pk_max_f16 v7, v0, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX10-NEXT:    v_pk_max_f16 v5, v0, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7e00, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v10, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v4, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v7, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v1, v3, v1, 0x5040100
+; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
+; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7e00, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_perm_b32 v0, v0, v5, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v4, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v1, v1, v6, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v4f16:
@@ -2541,47 +1633,23 @@ define <4 x half> @v_maximum_v4f16(<4 x half> %src0, <4 x half> %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_pk_max_f16 v4, v1, v3
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
 ; GFX11-NEXT:    v_pk_max_f16 v7, v0, v2
-; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX11-NEXT:    v_lshrrev_b32_e32 v9, 16, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
-; GFX11-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v4, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v7, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v8
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v9, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v10, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v4, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v3, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v7, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v4, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_perm_b32 v1, v3, v1, 0x5040100
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2607,266 +1675,70 @@ define <4 x half> @v_maximum_v4f16__nnan(<4 x half> %src0, <4 x half> %src1) {
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX7-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v1, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f16__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX8-NEXT:    v_max_f16_e32 v6, v5, v4
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX8-NEXT:    v_max_f16_e32 v7, v6, v5
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT:    v_max_f16_e32 v6, v1, v3
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX8-NEXT:    v_max_f16_e32 v3, v0, v2
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
-; GFX8-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_max_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_max_f16_e32 v1, v1, v3
+; GFX8-NEXT:    v_max_f16_e32 v0, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v5
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_v4f16__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
-; GFX9-NEXT:    v_pk_max_f16 v5, v1, v3
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX9-NEXT:    v_pk_max_f16 v7, v0, v2
-; GFX9-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v9, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
-; GFX9-NEXT:    v_perm_b32 v0, v6, v0, s4
-; GFX9-NEXT:    v_perm_b32 v1, v4, v1, s4
+; GFX9-NEXT:    v_pk_max_f16 v0, v0, v2
+; GFX9-NEXT:    v_pk_max_f16 v1, v1, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v4f16__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
-; GFX940-NEXT:    v_pk_max_f16 v5, v1, v3
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX940-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v9, 16, v2
-; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX940-NEXT:    v_pk_max_f16 v7, v0, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX940-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v9, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX940-NEXT:    v_perm_b32 v1, v4, v1, s0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX940-NEXT:    v_perm_b32 v0, v6, v0, s0
+; GFX940-NEXT:    v_pk_max_f16 v0, v0, v2
+; GFX940-NEXT:    v_pk_max_f16 v1, v1, v3
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_v4f16__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_pk_max_f16 v4, v1, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX10-NEXT:    v_pk_max_f16 v6, v0, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v4
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v9, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v11, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v1, v5, v1, 0x5040100
+; GFX10-NEXT:    v_pk_max_f16 v0, v0, v2
+; GFX10-NEXT:    v_pk_max_f16 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v4f16__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_pk_max_f16 v4, v1, v3
-; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX11-NEXT:    v_pk_max_f16 v6, v0, v2
-; GFX11-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
-; GFX11-NEXT:    v_lshrrev_b32_e32 v8, 16, v4
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
-; GFX11-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v9, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v11, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11-NEXT:    v_perm_b32 v1, v5, v1, 0x5040100
+; GFX11-NEXT:    v_pk_max_f16 v0, v0, v2
+; GFX11-NEXT:    v_pk_max_f16 v1, v1, v3
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v4f16__nnan:
@@ -2891,55 +1763,31 @@ define <4 x half> @v_maximum_v4f16__nsz(<4 x half> %src0, <4 x half> %src1) {
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX7-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v1, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f16__nsz:
@@ -3080,55 +1928,31 @@ define <4 x half> @v_maximum_v4f16__nnan_nsz(<4 x half> %src0, <4 x half> %src1)
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; GFX7-NEXT:    v_max_f32_e32 v8, v0, v4
 ; GFX7-NEXT:    v_mov_b32_e32 v9, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v9, v8, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v1, v5
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v9, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v2, v6
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v9, v4, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v4, v3, v7
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v7
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v9, v4, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v4f16__nnan_nsz:
@@ -3192,107 +2016,59 @@ define <8 x half> @v_maximum_v8f16(<8 x half> %src0, <8 x half> %src1) {
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v9, v9
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v8, v8
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v10, v10
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v11, v11
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v12, v12
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v13, v13
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v9, v9
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v11, v11
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v12, v12
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
 ; GFX7-NEXT:    v_max_f32_e32 v16, v0, v8
 ; GFX7-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v10, v10
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v13, v13
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v17, v16, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v8, v1, v9
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v9
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v9, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v11, v11
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v8, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v8, v2, v10
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v10
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v10, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v15, v15
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v17, v8, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v8, v3, v11
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v11
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v11, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, v17, v8, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v8, v4, v12
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v4, v12
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v12, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v15, v15
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v17, v8, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v8, v5, v13
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v5, v13
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v13, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, v17, v8, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v8, v6, v14
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v6, v14
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v14, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v17, v8, vcc
 ; GFX7-NEXT:    v_max_f32_e32 v8, v7, v15
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v7, v15
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v15, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v17, v8, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v8f16:
@@ -3303,82 +2079,34 @@ define <8 x half> @v_maximum_v8f16(<8 x half> %src0, <8 x half> %src1) {
 ; GFX8-NEXT:    v_max_f16_e32 v10, v9, v8
 ; GFX8-NEXT:    v_mov_b32_e32 v11, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v9, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v11, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v9, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v9, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v8, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v10, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v6
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
 ; GFX8-NEXT:    v_max_f16_e32 v12, v10, v9
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v10, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v11, v12, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v10, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v9, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v9, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v12, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v11, v12, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v5
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v1
 ; GFX8-NEXT:    v_max_f16_e32 v13, v12, v10
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v12, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v11, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v12, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v13, v12, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v10, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v11, v13, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v13, 16, v0
 ; GFX8-NEXT:    v_max_f16_e32 v14, v13, v12
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v13, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v11, v14, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v13, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v14, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v12, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v13, v12, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v12, v11, v14, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v13, v3, v7
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v11, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v13, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v7, v2, v6
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v7, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v6, v1, v5
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v11, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v11, v6, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v5, v0, v4
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v11, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v11, v5, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v12
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v10
@@ -3396,83 +2124,27 @@ define <8 x half> @v_maximum_v8f16(<8 x half> %src0, <8 x half> %src1) {
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v3, v7
 ; GFX9-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v8, vcc
 ; GFX9-NEXT:    v_pk_max_f16 v7, v2, v6
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v2, v6
 ; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v8, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v6, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v8, v11, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v7, vcc
 ; GFX9-NEXT:    v_pk_max_f16 v6, v1, v5
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v5
 ; GFX9-NEXT:    v_cndmask_b32_e32 v7, v9, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v7, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v6, vcc
 ; GFX9-NEXT:    v_pk_max_f16 v5, v0, v4
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v4
 ; GFX9-NEXT:    v_cndmask_b32_e32 v6, v9, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v11, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v9, v5, vcc
 ; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    v_perm_b32 v0, v0, v6, s4
 ; GFX9-NEXT:    v_perm_b32 v1, v1, v7, s4
@@ -3489,117 +2161,37 @@ define <8 x half> @v_maximum_v8f16(<8 x half> %src0, <8 x half> %src1) {
 ; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v10, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX940-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v3, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_max_f16 v7, v2, v6
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v8, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v2, v6
 ; GFX940-NEXT:    v_perm_b32 v3, v3, v10, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v8, v9, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v8, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v6, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v8, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v2, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_max_f16 v6, v1, v5
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v9, v7, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v5
 ; GFX940-NEXT:    v_perm_b32 v2, v2, v8, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v7, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX940-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_max_f16 v5, v0, v4
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v6, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v4
 ; GFX940-NEXT:    v_perm_b32 v1, v1, v7, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v6, v9, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v9, v5, vcc
 ; GFX940-NEXT:    v_perm_b32 v0, v0, v6, s0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3608,88 +2200,32 @@ define <8 x half> @v_maximum_v8f16(<8 x half> %src0, <8 x half> %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_max_f16 v8, v3, v7
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
-; GFX10-NEXT:    v_pk_max_f16 v13, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7e00, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v11, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v8, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_pk_max_f16 v11, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v10, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
+; GFX10-NEXT:    v_pk_max_f16 v9, v2, v6
+; GFX10-NEXT:    v_pk_max_f16 v12, v1, v5
+; GFX10-NEXT:    v_pk_max_f16 v13, v0, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, 0x7e00, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v10, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, 0x7e00, v12, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v9
+; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7e00, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v11, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, 0x7e00, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v14, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v10, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_pk_max_f16 v10, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v15, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v12, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v14
-; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
-; GFX10-NEXT:    v_perm_b32 v2, v6, v2, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v14, v9, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v13
+; GFX10-NEXT:    v_perm_b32 v2, v2, v9, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7e00, v12, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, 0x7e00, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v12, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, 0x7e00, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v5
+; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
 ; GFX10-NEXT:    v_cndmask_b32_e32 v13, 0x7e00, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v12, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v13
-; GFX10-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_perm_b32 v1, v1, v9, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v7, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v3, v5, v3, 0x5040100
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_perm_b32 v0, v0, v13, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_perm_b32 v1, v1, v6, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v8, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v3, v3, v10, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v8f16:
@@ -3697,94 +2233,42 @@ define <8 x half> @v_maximum_v8f16(<8 x half> %src0, <8 x half> %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_pk_max_f16 v8, v3, v7
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_lshrrev_b32_e32 v10, 16, v7
-; GFX11-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
-; GFX11-NEXT:    v_pk_max_f16 v13, v1, v5
+; GFX11-NEXT:    v_pk_max_f16 v10, v2, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v2
+; GFX11-NEXT:    v_pk_max_f16 v14, v1, v5
 ; GFX11-NEXT:    v_cndmask_b32_e32 v9, 0x7e00, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v11, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v8, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v9
-; GFX11-NEXT:    v_pk_max_f16 v11, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v9, 16, v6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v7, v10, vcc_lo
-; GFX11-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v10, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, 0x7e00, v12, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v13, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v10, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v12, v11
+; GFX11-NEXT:    v_pk_max_f16 v11, v0, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
+; GFX11-NEXT:    v_cndmask_b32_e32 v6, 0x7e00, v13, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, 0x7e00, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v14, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v10, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_pk_max_f16 v10, v0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v15, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v13, 16, v0
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v12
-; GFX11-NEXT:    v_lshrrev_b32_e32 v11, 16, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v12, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v14
-; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_perm_b32 v2, v6, v2, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v14, v9, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v15, 16, v11
+; GFX11-NEXT:    v_cndmask_b32_e32 v10, 0x7e00, v14, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, 0x7e00, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v12, v11
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, 0x7e00, v14, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
+; GFX11-NEXT:    v_perm_b32 v2, v6, v2, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v11, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v13, v12
+; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v15, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, 0x7e00, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v12, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v14
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_perm_b32 v1, v1, v9, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v8, v7, vcc_lo
 ; GFX11-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v3, v5, v3, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v14, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v7
+; GFX11-NEXT:    v_perm_b32 v1, v1, v10, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v8, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_perm_b32 v3, v3, v9, 0x5040100
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v8f16:
@@ -3809,381 +2293,189 @@ define <16 x half> @v_maximum_v16f16(<16 x half> %src0, <16 x half> %src1) {
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v16, v16
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v8, v8
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[12:13], v0, v16
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v16
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v16, v22
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v8, v8
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v9, v9
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v10, v10
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v17
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v9, v9
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v10, v10
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[14:15], v6, v16
+; GFX7-NEXT:    v_max_f32_e32 v6, v6, v16
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v16, v23
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v31, v16
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT:    v_mov_b32_e32 v16, 0x7fc00000
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v17, v17
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT:    v_max_f32_e32 v32, v0, v31
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v31
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v18, v18
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v2, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v32, v16, v32, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v31, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v31, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v32, v0, vcc
-; GFX7-NEXT:    v_max_f32_e32 v31, v1, v17
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v18, v18
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v31, v16, v31, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v31, v1, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v17, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v31
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v31, v1, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v2, v18
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v2, v18
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v2, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v17, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v18, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v18, vcc
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v18, v20
-; GFX7-NEXT:    buffer_load_dword v20, off, s[0:3], s32
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v19, v19
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v3, v3
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[16:17], v7, v16
+; GFX7-NEXT:    v_max_f32_e32 v7, v7, v16
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v16, v24
+; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
+; GFX7-NEXT:    v_max_f32_e32 v1, v1, v17
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v18
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v17, v2, vcc
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v4, v4
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[18:19], v8, v16
+; GFX7-NEXT:    v_max_f32_e32 v8, v8, v16
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v16, v25
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v17
+; GFX7-NEXT:    v_max_f32_e32 v2, v2, v17
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v19
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v4, v4
-; GFX7-NEXT:    v_max_f32_e32 v17, v3, v19
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v3, v19
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v3, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v19, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v3, v19, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v19, v21
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v5, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v4, v18
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v4, v18
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v17, v4, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v18, 64
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[20:21], v9, v16
+; GFX7-NEXT:    v_max_f32_e32 v9, v9, v16
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v16, v26
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v17
+; GFX7-NEXT:    v_max_f32_e32 v3, v3, v17
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v20
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v16, v16
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v18, vcc
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v18, v22
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v6, v6
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v17, v4, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v5, v19
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v5, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v18, v18
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v6, v6
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v19, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v19, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v19, v23
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v7, v7
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v6, v18
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v6, v18
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v6, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v17, v6, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v18, 64
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v7, v7
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v6, v18, vcc
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v18, v24
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v8, v8
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v17, v6, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v7, v19
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v7, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v18, v18
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v8, v8
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v7, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v17, v7, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v19, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v7, v19, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v19, v25
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v9, v9
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v17, v7, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v8, v18
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v8, v18
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v8, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v18, 64
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v9, v9
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v8, v18, vcc
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v18, v26
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v10, v10
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
-; GFX7-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v9, v19
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v9, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v18, v18
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v10, v10
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v9, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v19, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v9, v19, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v19, v27
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v11, v11
-; GFX7-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v10, v18
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v10, v18
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v10, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v17, v10, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v18, 64
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v11, v11
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v10, v18, vcc
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v18, v28
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[22:23], v10, v16
+; GFX7-NEXT:    v_max_f32_e32 v10, v10, v16
+; GFX7-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v17
+; GFX7-NEXT:    v_max_f32_e32 v4, v4, v17
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v21
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v20, v28
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v12, v12
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
-; GFX7-NEXT:    v_cndmask_b32_e32 v10, v17, v10, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v11, v19
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v11, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v18, v18
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v12, v12
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v11, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v19, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v11, v19, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
-; GFX7-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v12, v18
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v12, v18
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v12, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v18, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v12, v18, vcc
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v18, v20
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v20, v29
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v19, v29
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v17, v17
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v13, v13
-; GFX7-NEXT:    v_cvt_f16_f32_e32 v19, v30
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v18, v30
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v14, v14
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v20, v20
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v13, v13
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v17
+; GFX7-NEXT:    v_max_f32_e32 v5, v5, v17
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v17, v27
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v11, v11
 ; GFX7-NEXT:    v_cvt_f16_f32_e32 v15, v15
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v20, v20
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v17, v17
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v12, v12
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v19, v19
-; GFX7-NEXT:    v_cvt_f32_f16_e32 v14, v14
-; GFX7-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v13, v20
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v13, v20
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v13, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v20, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v13, v20, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v13, v13
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v18, v18
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v14, v14
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[24:25], v11, v17
+; GFX7-NEXT:    v_max_f32_e32 v11, v11, v17
+; GFX7-NEXT:    v_mov_b32_e32 v17, 0x7fc00000
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v15, v15
-; GFX7-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v14, v19
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v14, v19
-; GFX7-NEXT:    v_cndmask_b32_e32 v17, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v14, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v17, v14, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v19, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v14, v19, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v17
-; GFX7-NEXT:    v_cndmask_b32_e32 v14, v17, v14, vcc
-; GFX7-NEXT:    v_max_f32_e32 v17, v15, v18
-; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v15, v18
-; GFX7-NEXT:    v_cndmask_b32_e32 v16, v16, v17, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v15, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v18, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v15, v18, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v16
-; GFX7-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v12, v20
+; GFX7-NEXT:    v_max_f32_e32 v12, v12, v20
+; GFX7-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc
+; GFX7-NEXT:    v_max_f32_e32 v20, v13, v19
+; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v13, v19
+; GFX7-NEXT:    v_cndmask_b32_e32 v13, v17, v20, vcc
+; GFX7-NEXT:    v_max_f32_e32 v19, v14, v18
+; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v14, v18
+; GFX7-NEXT:    v_cndmask_b32_e32 v14, v17, v19, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v17, v0, s[12:13]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, v17, v2, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, v17, v3, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v4, v17, v4, s[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e64 v5, v17, v5, s[10:11]
+; GFX7-NEXT:    v_cndmask_b32_e64 v6, v17, v6, s[14:15]
+; GFX7-NEXT:    v_cndmask_b32_e64 v7, v17, v7, s[16:17]
+; GFX7-NEXT:    v_cndmask_b32_e64 v8, v17, v8, s[18:19]
+; GFX7-NEXT:    v_cndmask_b32_e64 v9, v17, v9, s[20:21]
+; GFX7-NEXT:    v_cndmask_b32_e64 v10, v17, v10, s[22:23]
+; GFX7-NEXT:    v_cndmask_b32_e64 v11, v17, v11, s[24:25]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v16, v16
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v16, v16
+; GFX7-NEXT:    v_max_f32_e32 v18, v15, v16
+; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v15, v16
+; GFX7-NEXT:    v_cndmask_b32_e32 v15, v17, v18, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v16f16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v16, 16, v15
-; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v7
-; GFX8-NEXT:    v_max_f16_e32 v19, v18, v16
-; GFX8-NEXT:    v_mov_b32_e32 v17, 0x7e00
-; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v18, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v17, v19, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v18, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v16, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v19, v16, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v14
-; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v6
-; GFX8-NEXT:    v_max_f16_e32 v20, v19, v18
-; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v19, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v17, v20, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v19, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v18, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v13
+; GFX8-NEXT:    v_lshrrev_b32_e32 v17, 16, v7
+; GFX8-NEXT:    v_max_f16_e32 v18, v17, v16
+; GFX8-NEXT:    v_mov_b32_e32 v19, 0x7e00
+; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v17, v16
+; GFX8-NEXT:    v_cndmask_b32_e32 v16, v19, v18, vcc
+; GFX8-NEXT:    v_lshrrev_b32_e32 v17, 16, v14
+; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
+; GFX8-NEXT:    v_max_f16_e32 v20, v18, v17
+; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v18, v17
+; GFX8-NEXT:    v_cndmask_b32_e32 v17, v19, v20, vcc
+; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v13
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v5
-; GFX8-NEXT:    v_max_f16_e32 v21, v20, v19
-; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v20, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v17, v21, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v20, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v19, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
+; GFX8-NEXT:    v_max_f16_e32 v21, v20, v18
+; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v20, v18
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v21, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v12
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v4
 ; GFX8-NEXT:    v_max_f16_e32 v22, v21, v20
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v21, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v17, v22, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v21, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v20, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v20, v19, v22, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v11
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v3
 ; GFX8-NEXT:    v_max_f16_e32 v23, v22, v21
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v22, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v17, v23, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v22, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v21, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v21, v19, v23, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v10
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v2
 ; GFX8-NEXT:    v_max_f16_e32 v24, v23, v22
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v23, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v17, v24, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v23, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v22, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v22, v19, v24, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v9
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v1
 ; GFX8-NEXT:    v_max_f16_e32 v25, v24, v23
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v24, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v17, v25, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v24, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v23, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v23, v19, v25, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v8
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v25, 16, v0
 ; GFX8-NEXT:    v_max_f16_e32 v26, v25, v24
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v25, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v26, v17, v26, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v25, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v26, v25, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v24, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v26
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v26, v24, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v24, v19, v26, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v25, v7, v15
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v7, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v17, v25, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v15, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v19, v25, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v15, v6, v14
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v6, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v17, v15, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v14, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v19, v15, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v14, v5, v13
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v5, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v17, v14, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v13, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v19, v14, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v13, v4, v12
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v4, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v12, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v19, v13, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v12, v3, v11
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v3, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v11, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v19, v12, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v11, v2, v10
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v2, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v10, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v19, v11, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v10, v1, v9
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v1, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v17, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v9, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v19, v10, vcc
 ; GFX8-NEXT:    v_max_f16_e32 v9, v0, v8
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v8, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v19, v9, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v24
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v23
@@ -4194,9 +2486,9 @@ define <16 x half> @v_maximum_v16f16(<16 x half> %src0, <16 x half> %src1) {
 ; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v20
 ; GFX8-NEXT:    v_or_b32_sdwa v4, v4, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v19
-; GFX8-NEXT:    v_or_b32_sdwa v5, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v18
+; GFX8-NEXT:    v_or_b32_sdwa v5, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v17
 ; GFX8-NEXT:    v_or_b32_sdwa v6, v6, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v16
 ; GFX8-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -4205,414 +2497,142 @@ define <16 x half> @v_maximum_v16f16(<16 x half> %src0, <16 x half> %src1) {
 ; GFX9-LABEL: v_maximum_v16f16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_max_f16 v18, v7, v15
+; GFX9-NEXT:    v_pk_max_f16 v16, v7, v15
 ; GFX9-NEXT:    v_mov_b32_e32 v17, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v7, v15
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v16, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v15, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v15, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v16
-; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
-; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v16, v19, vcc
-; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v7, v15
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v18, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v15, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v18, v7, vcc
-; GFX9-NEXT:    v_pk_max_f16 v18, v6, v14
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v6, v14
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v15, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v14, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v14, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v15
-; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v15, v19, vcc
-; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
+; GFX9-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v7, v15 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v17, v16, vcc
+; GFX9-NEXT:    v_pk_max_f16 v15, v6, v14
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v6, v14
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v14, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX9-NEXT:    v_pk_max_f16 v18, v5, v13
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v5, v13
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v14, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v13, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v13, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v14
-; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
-; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v14, v19, vcc
-; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
+; GFX9-NEXT:    v_cndmask_b32_e32 v16, v17, v15, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v6, v14 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v17, v15, vcc
+; GFX9-NEXT:    v_pk_max_f16 v14, v5, v13
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v5, v13
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v18, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v13, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v18, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v15, v17, v14, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v5, v13 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v17, v14, vcc
 ; GFX9-NEXT:    v_pk_max_f16 v13, v4, v12
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v4, v12
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v17, v13, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v18, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v12, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v12, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v18, v19, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v14, v17, v13, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v4, v12
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v12, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v4, v12 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v17, v13, vcc
 ; GFX9-NEXT:    v_pk_max_f16 v12, v3, v11
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v3, v11
 ; GFX9-NEXT:    v_cndmask_b32_e32 v13, v17, v12, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v13, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v11, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v11, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
-; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v13, v19, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v3, v11
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v11, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v3, v11 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v17, v12, vcc
 ; GFX9-NEXT:    v_pk_max_f16 v11, v2, v10
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v2, v10
 ; GFX9-NEXT:    v_cndmask_b32_e32 v12, v17, v11, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v12, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v10, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v10, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
-; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
-; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v12, v19, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v2, v10
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v10, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v11
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v2, v10 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v17, v11, vcc
 ; GFX9-NEXT:    v_pk_max_f16 v10, v1, v9
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v9
 ; GFX9-NEXT:    v_cndmask_b32_e32 v11, v17, v10, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v11, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v9, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v9, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v11
-; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v19, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v17, v10, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v9, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v9 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v10, vcc
 ; GFX9-NEXT:    v_pk_max_f16 v9, v0, v8
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v8
 ; GFX9-NEXT:    v_cndmask_b32_e32 v10, v17, v9, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v10, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v8, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX9-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v10, v19, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v8, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v8 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v17, v9, vcc
 ; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    v_perm_b32 v0, v0, v10, s4
 ; GFX9-NEXT:    v_perm_b32 v1, v1, v11, s4
 ; GFX9-NEXT:    v_perm_b32 v2, v2, v12, s4
 ; GFX9-NEXT:    v_perm_b32 v3, v3, v13, s4
-; GFX9-NEXT:    v_perm_b32 v4, v4, v18, s4
-; GFX9-NEXT:    v_perm_b32 v5, v5, v14, s4
-; GFX9-NEXT:    v_perm_b32 v6, v6, v15, s4
-; GFX9-NEXT:    v_perm_b32 v7, v7, v16, s4
+; GFX9-NEXT:    v_perm_b32 v4, v4, v14, s4
+; GFX9-NEXT:    v_perm_b32 v5, v5, v15, s4
+; GFX9-NEXT:    v_perm_b32 v6, v6, v16, s4
+; GFX9-NEXT:    v_perm_b32 v7, v7, v18, s4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_v16f16:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_pk_max_f16 v18, v7, v15
+; GFX940-NEXT:    v_pk_max_f16 v16, v7, v15
 ; GFX940-NEXT:    v_mov_b32_e32 v17, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v7, v15
 ; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v16, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v15, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GFX940-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc
+; GFX940-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v7, v15 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX940-NEXT:    v_pk_max_f16 v15, v6, v14
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v15, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v16
-; GFX940-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v16, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v7, v15
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v18, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v15, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v18, v7, vcc
-; GFX940-NEXT:    v_pk_max_f16 v18, v6, v14
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v17, v16, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v6, v14
-; GFX940-NEXT:    v_perm_b32 v7, v7, v16, s0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
+; GFX940-NEXT:    v_perm_b32 v7, v7, v18, s0
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v15, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v14, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v14, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v15
-; GFX940-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
+; GFX940-NEXT:    v_cndmask_b32_e32 v16, v17, v15, vcc
+; GFX940-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v6, v14 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX940-NEXT:    v_pk_max_f16 v14, v5, v13
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v15, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v6, v14
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v6, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v14, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX940-NEXT:    v_pk_max_f16 v18, v5, v13
+; GFX940-NEXT:    v_cndmask_b32_e32 v6, v17, v15, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v5, v13
-; GFX940-NEXT:    v_perm_b32 v6, v6, v15, s0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v14, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v13, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v13, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v14
-; GFX940-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
+; GFX940-NEXT:    v_perm_b32 v6, v6, v16, s0
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v14, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v5, v13
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v5, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v18, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v13, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
+; GFX940-NEXT:    v_cndmask_b32_e32 v15, v17, v14, vcc
+; GFX940-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v5, v13 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_max_f16 v13, v4, v12
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v18, v5, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v17, v14, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v4, v12
-; GFX940-NEXT:    v_perm_b32 v5, v5, v14, s0
+; GFX940-NEXT:    v_perm_b32 v5, v5, v15, s0
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v17, v13, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
+; GFX940-NEXT:    v_cndmask_b32_e32 v14, v17, v13, vcc
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v18, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v12, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v12, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX940-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v18, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v4, v12
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v12, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v4, v12 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_max_f16 v12, v3, v11
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v4, v17, v13, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v3, v11
-; GFX940-NEXT:    v_perm_b32 v4, v4, v18, s0
+; GFX940-NEXT:    v_perm_b32 v4, v4, v14, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v13, v17, v12, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v13, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v11, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v11, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX940-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v13, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v3, v11
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v11, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v3, v11 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_max_f16 v11, v2, v10
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v17, v12, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v2, v10
 ; GFX940-NEXT:    v_perm_b32 v3, v3, v13, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v12, v17, v11, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v12, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v10, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v10, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
-; GFX940-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v12, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v2, v10
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v10, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v11
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v2, v10 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_max_f16 v10, v1, v9
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v17, v11, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v9
 ; GFX940-NEXT:    v_perm_b32 v2, v2, v12, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v11, v17, v10, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v11, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v9, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v9, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v11
-; GFX940-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v9
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v17, v10, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v9, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v9 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_max_f16 v9, v0, v8
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v10, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v8
 ; GFX940-NEXT:    v_perm_b32 v1, v1, v11, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v10, v17, v9, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v10, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v8, 64
-; GFX940-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v8, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX940-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v10, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v8, 64
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v8 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v9
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v17, v9, vcc
 ; GFX940-NEXT:    v_perm_b32 v0, v0, v10, s0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4621,353 +2641,145 @@ define <16 x half> @v_maximum_v16f16(<16 x half> %src0, <16 x half> %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_max_f16 v16, v7, v15
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v7, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, 0x7e00, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v17, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v15, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v18, v15, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v7, v15
 ; GFX10-NEXT:    v_pk_max_f16 v18, v6, v14
+; GFX10-NEXT:    v_pk_max_f16 v19, v3, v11
+; GFX10-NEXT:    v_pk_max_f16 v20, v2, v10
+; GFX10-NEXT:    v_lshrrev_b32_e32 v17, 16, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v16, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v7, v15 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v18
+; GFX10-NEXT:    v_pk_max_f16 v21, v0, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0x7e00, v17, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v14
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX10-NEXT:    v_pk_max_f16 v17, v5, v13
+; GFX10-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; GFX10-NEXT:    v_perm_b32 v7, v7, v16, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v18, 0x7e00, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v20, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v15, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v21, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v14, 64
-; GFX10-NEXT:    v_pk_max_f16 v20, v4, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v19, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v16, 16, v13
-; GFX10-NEXT:    v_perm_b32 v7, v7, v17, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v15, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v18
-; GFX10-NEXT:    v_pk_max_f16 v15, v5, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v21, v14, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v6, v14 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v17
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7e00, v15, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v5, v13
-; GFX10-NEXT:    v_perm_b32 v6, v14, v6, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, 0x7e00, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v18, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, 0x7e00, v19, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v6, v6, v18, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, 0x7e00, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v5, v13 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_pk_max_f16 v17, v4, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7e00, v14, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v18, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v21, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v13, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v16, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v18, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v22, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v19, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v11
-; GFX10-NEXT:    v_perm_b32 v5, v13, v5, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v21, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v12
-; GFX10-NEXT:    v_pk_max_f16 v16, v3, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, 0x7e00, v18, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v17
+; GFX10-NEXT:    v_perm_b32 v5, v5, v15, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, 0x7e00, v17, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v11
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v20, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v18, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 64
-; GFX10-NEXT:    v_pk_max_f16 v12, v2, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v19, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v20, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_pk_max_f16 v19, v1, v9
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v21, v11, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v17, 16, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v19, 0x7e00, v19, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v3, v11 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_pk_max_f16 v11, v1, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v17, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v2, v10
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v2
-; GFX10-NEXT:    v_perm_b32 v3, v11, v3, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v9
-; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, 0x7e00, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v21, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, 0x7e00, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v22, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 64
-; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v21, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v12, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v23, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v22
-; GFX10-NEXT:    v_pk_max_f16 v20, v0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v22, v21, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v11
+; GFX10-NEXT:    v_perm_b32 v3, v3, v19, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v17, 0x7e00, v20, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v9
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, 0x7e00, v19, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v1, v9 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v22, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, 0x7e00, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v22, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, 0x7e00, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v20, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v22, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v8, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v21, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v22, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v20
-; GFX10-NEXT:    v_perm_b32 v1, v1, v16, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v20, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v23, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_perm_b32 v0, v8, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v12, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v18
-; GFX10-NEXT:    v_perm_b32 v2, v9, v2, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v18, v4, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v4, v4, v15, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v1, v1, v11, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7e00, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v0, v8 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v2, v10 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_perm_b32 v0, v0, v9, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v20, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v4, v12 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_perm_b32 v2, v2, v17, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v14, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v4, v4, v13, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_v16f16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_pk_max_f16 v16, v7, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v17, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v7
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v7, v15
-; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v6
-; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v17, 0x7e00, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v18, v17, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v15, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v18, v18, v15, vcc_lo
-; GFX11-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v17
-; GFX11-NEXT:    v_cndmask_b32_e32 v17, v17, v18, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v7, v15
-; GFX11-NEXT:    v_pk_max_f16 v18, v6, v14
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v14
-; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
-; GFX11-NEXT:    v_cndmask_b32_e32 v18, 0x7e00, v18, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v20, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v21, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v15, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v21, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v14, 64
+; GFX11-NEXT:    v_pk_max_f16 v15, v6, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
 ; GFX11-NEXT:    v_pk_max_f16 v20, v4, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v19, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 16, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, v15, v19, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v18
-; GFX11-NEXT:    v_pk_max_f16 v15, v5, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_pk_max_f16 v22, v2, v10
+; GFX11-NEXT:    v_cndmask_b32_e32 v7, 0x7e00, v16, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v18, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v17, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v19, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v14
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v15
-; GFX11-NEXT:    v_perm_b32 v7, v7, v17, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, v21, v14, vcc_lo
+; GFX11-NEXT:    v_pk_max_f16 v14, v5, v13
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v6, 0x7e00, v15, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v18, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v17, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v5
+; GFX11-NEXT:    v_cndmask_b32_e32 v15, 0x7e00, v19, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v5, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, 0x7e00, v15, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v18, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v19, 0x7e00, v19, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v6, v15, v6, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v5, 0x7e00, v14, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v18, v17
+; GFX11-NEXT:    v_pk_max_f16 v17, v3, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
+; GFX11-NEXT:    v_cndmask_b32_e32 v13, 0x7e00, v19, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v18, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v22, v21, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v13, 64
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v16, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, v18, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
+; GFX11-NEXT:    v_cndmask_b32_e32 v14, 0x7e00, v20, vcc_lo
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v3
-; GFX11-NEXT:    v_perm_b32 v6, v14, v6, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v22, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v15
-; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, v19, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v11
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_perm_b32 v5, v13, v5, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v21, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v12
-; GFX11-NEXT:    v_pk_max_f16 v16, v3, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v18, 0x7e00, v18, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v11
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v16, vcc_lo
+; GFX11-NEXT:    v_perm_b32 v5, v13, v5, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v17, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v20, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v21, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v18, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 64
-; GFX11-NEXT:    v_pk_max_f16 v12, v2, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v19, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, v20, v19, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v10
 ; GFX11-NEXT:    v_pk_max_f16 v19, v1, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, v21, v11, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v22
+; GFX11-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v21, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v2, v10
-; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v12, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_perm_b32 v3, v11, v3, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v17, 0x7e00, v22, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v9
-; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v22, 0x7e00, v19, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v21, v20
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, 0x7e00, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v23, v22, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 64
-; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v21, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v12, v21, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v21, v23, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v10, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v22
-; GFX11-NEXT:    v_pk_max_f16 v20, v0, v8
-; GFX11-NEXT:    v_perm_b32 v3, v11, v3, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v22, v21, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    v_pk_max_f16 v22, v0, v8
+; GFX11-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v19, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v9
-; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v8
-; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v0
-; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
-; GFX11-NEXT:    v_cndmask_b32_e32 v19, 0x7e00, v19, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v19, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v20, 0x7e00, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v22, v21
-; GFX11-NEXT:    v_cndmask_b32_e32 v23, 0x7e00, v23, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v20, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v22, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v8, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v21, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v22, v21, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v20
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v20, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v23
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v23, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v12
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v1, v1, v21, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v22, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v24, v23
+; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v25, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v2, v10
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_perm_b32 v0, v8, v0, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v12, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v18
-; GFX11-NEXT:    v_perm_b32 v1, v1, v16, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v2, v9, v2, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v18, v4, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v20, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v12
+; GFX11-NEXT:    v_perm_b32 v2, v2, v17, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v18, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_perm_b32 v4, v4, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v4, v14, 0x5040100
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_v16f16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll
index 920c57c90b71..7c5bc7da4df2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll
@@ -14,13 +14,7 @@ define float @v_maximum_f32(float %src0, float %src1) {
 ; GFX7-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f32:
@@ -29,13 +23,7 @@ define float @v_maximum_f32(float %src0, float %src1) {
 ; GFX8-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f32:
@@ -44,13 +32,7 @@ define float @v_maximum_f32(float %src0, float %src1) {
 ; GFX9-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f32:
@@ -60,16 +42,7 @@ define float @v_maximum_f32(float %src0, float %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f32:
@@ -77,13 +50,7 @@ define float @v_maximum_f32(float %src0, float %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f32:
@@ -91,15 +58,8 @@ define float @v_maximum_f32(float %src0, float %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f32:
@@ -119,78 +79,37 @@ define float @v_maximum_f32__nnan(float %src0, float %src1) {
 ; GFX7-LABEL: v_maximum_f32__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_f32_e32 v2, v0, v1
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_max_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f32__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_max_f32_e32 v2, v0, v1
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_max_f32_e32 v0, v0, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f32__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_max_f32_e32 v2, v0, v1
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_max_f32_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f32__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_max_f32_e32 v2, v0, v1
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_max_f32_e32 v0, v0, v1
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f32__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_max_f32_e32 v2, v0, v1
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_max_f32_e32 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f32__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_max_f32_e32 v2, v0, v1
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_max_f32_e32 v0, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f32__nnan:
@@ -332,13 +251,7 @@ define float @v_maximum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX7-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f32__nnan_src0:
@@ -348,13 +261,7 @@ define float @v_maximum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX8-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f32__nnan_src0:
@@ -364,13 +271,7 @@ define float @v_maximum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX9-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f32__nnan_src0:
@@ -381,16 +282,7 @@ define float @v_maximum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f32__nnan_src0:
@@ -399,13 +291,7 @@ define float @v_maximum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX10-NEXT:    v_add_f32_e32 v0, 1.0, v0
 ; GFX10-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f32__nnan_src0:
@@ -415,15 +301,7 @@ define float @v_maximum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f32__nnan_src0:
@@ -450,13 +328,7 @@ define float @v_maximum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX7-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f32__nnan_src1:
@@ -466,13 +338,7 @@ define float @v_maximum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX8-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f32__nnan_src1:
@@ -482,13 +348,7 @@ define float @v_maximum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX9-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f32__nnan_src1:
@@ -499,16 +359,7 @@ define float @v_maximum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f32__nnan_src1:
@@ -517,13 +368,7 @@ define float @v_maximum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX10-NEXT:    v_add_f32_e32 v1, 1.0, v1
 ; GFX10-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f32__nnan_src1:
@@ -533,15 +378,7 @@ define float @v_maximum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_max_f32_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 64
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f32__nnan_src1:
@@ -568,14 +405,7 @@ define void @s_maximum_f32(float inreg %src0, float inreg %src1) {
 ; GFX7-NEXT:    v_max_f32_e32 v1, s4, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s4, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s5, 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX7-NEXT:    ;;#ASMSTART
 ; GFX7-NEXT:    ; use v0
 ; GFX7-NEXT:    ;;#ASMEND
@@ -588,14 +418,7 @@ define void @s_maximum_f32(float inreg %src0, float inreg %src1) {
 ; GFX8-NEXT:    v_max_f32_e32 v1, s4, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s4, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s5, 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use v0
 ; GFX8-NEXT:    ;;#ASMEND
@@ -608,14 +431,7 @@ define void @s_maximum_f32(float inreg %src0, float inreg %src1) {
 ; GFX9-NEXT:    v_max_f32_e32 v1, s4, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s4, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s5, 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX9-NEXT:    ;;#ASMSTART
 ; GFX9-NEXT:    ; use v0
 ; GFX9-NEXT:    ;;#ASMEND
@@ -629,17 +445,7 @@ define void @s_maximum_f32(float inreg %src0, float inreg %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, s0, v0
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v2, s0
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s0, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s1, 64
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; use v0
 ; GFX940-NEXT:    ;;#ASMEND
@@ -650,13 +456,7 @@ define void @s_maximum_f32(float inreg %src0, float inreg %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_max_f32_e64 v0, s4, s5
 ; GFX10-NEXT:    v_cmp_o_f32_e64 vcc_lo, s4, s5
-; GFX10-NEXT:    v_cmp_class_f32_e64 s6, s4, 64
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v0, s4, s6
-; GFX10-NEXT:    v_cmp_class_f32_e64 s4, s5, 64
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s5, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; use v0
 ; GFX10-NEXT:    ;;#ASMEND
@@ -667,15 +467,8 @@ define void @s_maximum_f32(float inreg %src0, float inreg %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_max_f32_e64 v0, s0, s1
 ; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s0, s1
-; GFX11-NEXT:    v_cmp_class_f32_e64 s2, s0, 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v0, s0, s2
-; GFX11-NEXT:    v_cmp_class_f32_e64 s0, s1, 64
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s1, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use v0
 ; GFX11-NEXT:    ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
index e6b62b18bc37..d60a28e74043 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
@@ -13,18 +13,9 @@ define double @v_maximum_f64(double %src0, double %src1) {
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f64:
@@ -32,18 +23,9 @@ define double @v_maximum_f64(double %src0, double %src1) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f64:
@@ -51,39 +33,20 @@ define double @v_maximum_f64(double %src0, double %src1) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[2:3], 64
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f64:
@@ -91,17 +54,8 @@ define double @v_maximum_f64(double %src0, double %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[4:5]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f64:
@@ -109,20 +63,9 @@ define double @v_maximum_f64(double %src0, double %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s1, 0, v[4:5]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f64:
@@ -142,93 +85,37 @@ define double @v_maximum_f64__nnan(double %src0, double %src1) {
 ; GFX7-LABEL: v_maximum_f64__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
+; GFX7-NEXT:    v_max_f64 v[0:1], v[0:1], v[2:3]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f64__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
+; GFX8-NEXT:    v_max_f64 v[0:1], v[0:1], v[2:3]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f64__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[2:3]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f64__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[2:3], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
+; GFX940-NEXT:    v_max_f64 v[0:1], v[0:1], v[2:3]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f64__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[4:5]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s5
+; GFX10-NEXT:    v_max_f64 v[0:1], v[0:1], v[2:3]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f64__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s1, 0, v[4:5]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s1
+; GFX11-NEXT:    v_max_f64 v[0:1], v[0:1], v[2:3]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f64__nnan:
@@ -373,60 +260,33 @@ define double @v_maximum_f64__nnan_src0(double %arg0, double %src1) {
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX7-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
 ; GFX7-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f64__nnan_src0:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX8-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
 ; GFX8-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f64__nnan_src0:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
 ; GFX9-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f64__nnan_src0:
@@ -434,62 +294,33 @@ define double @v_maximum_f64__nnan_src0(double %arg0, double %src1) {
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
 ; GFX940-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[2:3], 64
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f64__nnan_src0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 64
 ; GFX10-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[4:5]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f64__nnan_src0:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s1, 0, v[4:5]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f64__nnan_src0:
@@ -513,60 +344,33 @@ define double @v_maximum_f64__nnan_src1(double %src0, double %arg1) {
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX7-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX7-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_f64__nnan_src1:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX8-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX8-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_maximum_f64__nnan_src1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX9-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 64
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_maximum_f64__nnan_src1:
@@ -574,21 +378,11 @@ define double @v_maximum_f64__nnan_src1(double %src0, double %arg1) {
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX940-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[2:3], 64
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_maximum_f64__nnan_src1:
@@ -597,39 +391,20 @@ define double @v_maximum_f64__nnan_src1(double %src0, double %arg1) {
 ; GFX10-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX10-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 64
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[4:5]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_maximum_f64__nnan_src1:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_max_f64 v[4:5], v[0:1], v[2:3]
 ; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 64
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 64
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s1, 0, v[4:5]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_maximum_f64__nnan_src1:
@@ -654,30 +429,13 @@ define void @s_maximum_f64(double inreg %src0, double inreg %src1) {
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
 ; GFX7-NEXT:    v_max_f64 v[2:3], s[4:5], v[0:1]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], s[4:5], 64
-; GFX7-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX7-NEXT:    v_readfirstlane_b32 s12, v3
-; GFX7-NEXT:    v_readfirstlane_b32 s10, v2
-; GFX7-NEXT:    s_cselect_b32 s11, 0x7ff80000, s12
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[12:13], s[6:7], 64
-; GFX7-NEXT:    s_cselect_b32 s10, 0, s10
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[14:15], s[10:11], 0
-; GFX7-NEXT:    s_and_b64 s[16:17], s[8:9], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX7-NEXT:    s_and_b64 s[16:17], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX7-NEXT:    s_and_b64 s[16:17], s[14:15], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX7-NEXT:    s_and_b64 s[8:9], s[8:9], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX7-NEXT:    s_and_b64 s[8:9], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX7-NEXT:    s_and_b64 s[6:7], s[14:15], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
 ; GFX7-NEXT:    ;;#ASMSTART
-; GFX7-NEXT:    ; use s[4:5]
+; GFX7-NEXT:    ; use v[0:1]
 ; GFX7-NEXT:    ;;#ASMEND
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -686,30 +444,13 @@ define void @s_maximum_f64(double inreg %src0, double inreg %src1) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
 ; GFX8-NEXT:    v_max_f64 v[2:3], s[4:5], v[0:1]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], s[4:5], 64
-; GFX8-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX8-NEXT:    v_readfirstlane_b32 s12, v3
-; GFX8-NEXT:    v_readfirstlane_b32 s10, v2
-; GFX8-NEXT:    s_cselect_b32 s11, 0x7ff80000, s12
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[12:13], s[6:7], 64
-; GFX8-NEXT:    s_cselect_b32 s10, 0, s10
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[14:15], s[10:11], 0
-; GFX8-NEXT:    s_and_b64 s[16:17], s[8:9], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX8-NEXT:    s_and_b64 s[16:17], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX8-NEXT:    s_and_b64 s[16:17], s[14:15], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX8-NEXT:    s_and_b64 s[8:9], s[8:9], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX8-NEXT:    s_and_b64 s[8:9], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX8-NEXT:    s_and_b64 s[6:7], s[14:15], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
 ; GFX8-NEXT:    ;;#ASMSTART
-; GFX8-NEXT:    ; use s[4:5]
+; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -718,30 +459,13 @@ define void @s_maximum_f64(double inreg %src0, double inreg %src1) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
 ; GFX9-NEXT:    v_max_f64 v[2:3], s[4:5], v[0:1]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], s[4:5], 64
-; GFX9-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX9-NEXT:    v_readfirstlane_b32 s12, v3
-; GFX9-NEXT:    v_readfirstlane_b32 s10, v2
-; GFX9-NEXT:    s_cselect_b32 s11, 0x7ff80000, s12
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[12:13], s[6:7], 64
-; GFX9-NEXT:    s_cselect_b32 s10, 0, s10
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[14:15], s[10:11], 0
-; GFX9-NEXT:    s_and_b64 s[16:17], s[8:9], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX9-NEXT:    s_and_b64 s[16:17], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX9-NEXT:    s_and_b64 s[16:17], s[14:15], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX9-NEXT:    s_and_b64 s[8:9], s[8:9], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX9-NEXT:    s_and_b64 s[8:9], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX9-NEXT:    s_and_b64 s[6:7], s[14:15], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
 ; GFX9-NEXT:    ;;#ASMSTART
-; GFX9-NEXT:    ; use s[4:5]
+; GFX9-NEXT:    ; use v[0:1]
 ; GFX9-NEXT:    ;;#ASMEND
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -749,30 +473,14 @@ define void @s_maximum_f64(double inreg %src0, double inreg %src1) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_max_f64 v[2:3], s[0:1], v[0:1]
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
 ; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, s[0:1], v[0:1]
-; GFX940-NEXT:    v_max_f64 v[0:1], s[0:1], v[0:1]
-; GFX940-NEXT:    s_and_b64 s[4:5], vcc, exec
-; GFX940-NEXT:    v_readfirstlane_b32 s6, v1
-; GFX940-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[8:9], s[0:1], 64
-; GFX940-NEXT:    s_cselect_b32 s5, 0x7ff80000, s6
-; GFX940-NEXT:    s_cselect_b32 s4, 0, s4
-; GFX940-NEXT:    s_and_b64 s[10:11], s[8:9], exec
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[10:11], s[2:3], 64
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[6:7], s[4:5], 0
-; GFX940-NEXT:    s_cselect_b32 s1, s1, s5
-; GFX940-NEXT:    s_and_b64 s[12:13], s[10:11], exec
-; GFX940-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX940-NEXT:    s_and_b64 s[12:13], s[6:7], exec
-; GFX940-NEXT:    s_cselect_b32 s1, s1, s5
-; GFX940-NEXT:    s_and_b64 s[8:9], s[8:9], exec
-; GFX940-NEXT:    s_cselect_b32 s0, s0, s4
-; GFX940-NEXT:    s_and_b64 s[8:9], s[10:11], exec
-; GFX940-NEXT:    s_cselect_b32 s0, s2, s0
-; GFX940-NEXT:    s_and_b64 s[2:3], s[6:7], exec
-; GFX940-NEXT:    s_cselect_b32 s0, s0, s4
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; use s[0:1]
+; GFX940-NEXT:    ; use v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -780,29 +488,11 @@ define void @s_maximum_f64(double inreg %src0, double inreg %src1) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_max_f64 v[0:1], s[4:5], s[6:7]
-; GFX10-NEXT:    v_cmp_u_f64_e64 s8, s[4:5], s[6:7]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s11, s[4:5], 64
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, s[6:7], 64
-; GFX10-NEXT:    v_readfirstlane_b32 s9, v1
-; GFX10-NEXT:    v_readfirstlane_b32 s10, v0
-; GFX10-NEXT:    s_and_b32 s8, s8, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s9, 0x7ff80000, s9
-; GFX10-NEXT:    s_cselect_b32 s8, 0, s10
-; GFX10-NEXT:    s_and_b32 s13, s11, exec_lo
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s10, s[8:9], 0
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s9
-; GFX10-NEXT:    s_and_b32 s13, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX10-NEXT:    s_and_b32 s7, s10, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s9
-; GFX10-NEXT:    s_and_b32 s7, s11, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s8
-; GFX10-NEXT:    s_and_b32 s7, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX10-NEXT:    s_and_b32 s6, s10, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s8
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, s[4:5], s[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s4
 ; GFX10-NEXT:    ;;#ASMSTART
-; GFX10-NEXT:    ; use s[4:5]
+; GFX10-NEXT:    ; use v[0:1]
 ; GFX10-NEXT:    ;;#ASMEND
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -810,32 +500,12 @@ define void @s_maximum_f64(double inreg %src0, double inreg %src1) {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_max_f64 v[0:1], s[0:1], s[2:3]
-; GFX11-NEXT:    v_cmp_u_f64_e64 s4, s[0:1], s[2:3]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s7, s[0:1], 64
-; GFX11-NEXT:    v_cmp_class_f64_e64 s8, s[2:3], 64
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX11-NEXT:    v_readfirstlane_b32 s6, v0
-; GFX11-NEXT:    s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s5, 0x7ff80000, s5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    s_cselect_b32 s4, 0, s6
-; GFX11-NEXT:    s_and_b32 s9, s7, exec_lo
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s6, s[4:5], 0
-; GFX11-NEXT:    s_cselect_b32 s1, s1, s5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    s_and_b32 s9, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX11-NEXT:    s_and_b32 s3, s6, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s1, s1, s5
-; GFX11-NEXT:    s_and_b32 s3, s7, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s0, s4
-; GFX11-NEXT:    s_and_b32 s3, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s2, s0
-; GFX11-NEXT:    s_and_b32 s2, s6, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s0, s4
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, s[0:1], s[2:3]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s0
 ; GFX11-NEXT:    ;;#ASMSTART
-; GFX11-NEXT:    ; use s[0:1]
+; GFX11-NEXT:    ; use v[0:1]
 ; GFX11-NEXT:    ;;#ASMEND
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
index 66f3a48b13ee..95d351e8f1fa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
@@ -14,13 +14,7 @@ define half @v_minimum_f16(half %src0, half %src1) {
 ; GFX8-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f16:
@@ -29,13 +23,7 @@ define half @v_minimum_f16(half %src0, half %src1) {
 ; GFX9-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f16:
@@ -45,16 +33,7 @@ define half @v_minimum_f16(half %src0, half %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f16:
@@ -62,13 +41,7 @@ define half @v_minimum_f16(half %src0, half %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f16:
@@ -76,15 +49,8 @@ define half @v_minimum_f16(half %src0, half %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f16:
@@ -104,66 +70,31 @@ define half @v_minimum_f16__nnan(half %src0, half %src1) {
 ; GFX8-LABEL: v_minimum_f16__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_min_f16_e32 v2, v0, v1
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_min_f16_e32 v0, v0, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f16__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_f16_e32 v2, v0, v1
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_min_f16_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f16__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_min_f16_e32 v2, v0, v1
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_min_f16_e32 v0, v0, v1
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f16__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_min_f16_e32 v2, v0, v1
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_min_f16_e32 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f16__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_min_f16_e32 v2, v0, v1
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_min_f16_e32 v0, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f16__nnan:
@@ -290,13 +221,7 @@ define half @v_minimum_f16__nnan_src0(half %arg0, half %src1) {
 ; GFX8-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f16__nnan_src0:
@@ -306,13 +231,7 @@ define half @v_minimum_f16__nnan_src0(half %arg0, half %src1) {
 ; GFX9-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f16__nnan_src0:
@@ -323,16 +242,7 @@ define half @v_minimum_f16__nnan_src0(half %arg0, half %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f16__nnan_src0:
@@ -341,13 +251,7 @@ define half @v_minimum_f16__nnan_src0(half %arg0, half %src1) {
 ; GFX10-NEXT:    v_add_f16_e32 v0, 1.0, v0
 ; GFX10-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f16__nnan_src0:
@@ -357,15 +261,7 @@ define half @v_minimum_f16__nnan_src0(half %arg0, half %src1) {
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f16__nnan_src0:
@@ -392,13 +288,7 @@ define half @v_minimum_f16__nnan_src1(half %src0, half %arg1) {
 ; GFX8-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f16__nnan_src1:
@@ -408,13 +298,7 @@ define half @v_minimum_f16__nnan_src1(half %src0, half %arg1) {
 ; GFX9-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f16__nnan_src1:
@@ -425,16 +309,7 @@ define half @v_minimum_f16__nnan_src1(half %src0, half %arg1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f16__nnan_src1:
@@ -443,13 +318,7 @@ define half @v_minimum_f16__nnan_src1(half %src0, half %arg1) {
 ; GFX10-NEXT:    v_add_f16_e32 v1, 1.0, v1
 ; GFX10-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f16__nnan_src1:
@@ -459,15 +328,7 @@ define half @v_minimum_f16__nnan_src1(half %src0, half %arg1) {
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_min_f16_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f16__nnan_src1:
@@ -494,14 +355,7 @@ define void @s_minimum_f16(half inreg %src0, half inreg %src1) {
 ; GFX8-NEXT:    v_min_f16_e32 v1, s4, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, s5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use v0
@@ -515,14 +369,7 @@ define void @s_minimum_f16(half inreg %src0, half inreg %src1) {
 ; GFX9-NEXT:    v_min_f16_e32 v1, s4, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    ;;#ASMSTART
 ; GFX9-NEXT:    ; use v0
@@ -537,17 +384,7 @@ define void @s_minimum_f16(half inreg %src0, half inreg %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, s0, v0
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v2, s0
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v1
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX940-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; use v0
@@ -559,13 +396,7 @@ define void @s_minimum_f16(half inreg %src0, half inreg %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_min_f16_e64 v0, s4, s5
 ; GFX10-NEXT:    v_cmp_o_f16_e64 vcc_lo, s4, s5
-; GFX10-NEXT:    v_cmp_class_f16_e64 s6, s4, 32
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v0, s4, s6
-; GFX10-NEXT:    v_cmp_class_f16_e64 s4, s5, 32
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s5, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; use v0
@@ -577,16 +408,8 @@ define void @s_minimum_f16(half inreg %src0, half inreg %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_min_f16_e64 v0, s0, s1
 ; GFX11-NEXT:    v_cmp_o_f16_e64 vcc_lo, s0, s1
-; GFX11-NEXT:    v_cmp_class_f16_e64 s2, s0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v0, s0, s2
-; GFX11-NEXT:    v_cmp_class_f16_e64 s0, s1, 32
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s1, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use v0
@@ -652,23 +475,9 @@ define <2 x half> @v_minimum_v2f16(<2 x half> %src0, <2 x half> %src1) {
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    v_perm_b32 v0, v0, v4, s4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -682,30 +491,10 @@ define <2 x half> @v_minimum_v2f16(<2 x half> %src0, <2 x half> %src1) {
 ; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v1
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v2
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    v_perm_b32 v0, v0, v4, s0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -713,26 +502,12 @@ define <2 x half> @v_minimum_v2f16(<2 x half> %src0, <2 x half> %src1) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_min_f16 v2, v0, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7e00, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v0, v1 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v3, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v0, v0, v2, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v2f16:
@@ -744,25 +519,10 @@ define <2 x half> @v_minimum_v2f16(<2 x half> %src0, <2 x half> %src1) {
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, 0x7e00, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v5, vcc_lo
 ; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -808,100 +568,25 @@ define <2 x half> @v_minimum_v2f16__nnan(<2 x half> %src0, <2 x half> %src1) {
 ; GFX9-LABEL: v_minimum_v2f16__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX9-NEXT:    v_pk_min_f16 v3, v0, v1
-; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
-; GFX9-NEXT:    v_perm_b32 v0, v2, v0, s4
+; GFX9-NEXT:    v_pk_min_f16 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v2f16__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
-; GFX940-NEXT:    v_pk_min_f16 v3, v0, v1
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX940-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX940-NEXT:    v_perm_b32 v0, v2, v0, s0
+; GFX940-NEXT:    v_pk_min_f16 v0, v0, v1
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v2f16__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_pk_min_f16 v2, v0, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    v_pk_min_f16 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v2f16__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_pk_min_f16 v2, v0, v1
-; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX11-NEXT:    v_pk_min_f16 v0, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v2f16__nnan:
@@ -1101,30 +786,16 @@ define void @s_minimum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s5
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s5, 16
 ; GFX9-NEXT:    v_pk_min_f16 v1, s4, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v2, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX9-NEXT:    s_lshr_b32 s5, s5, 16
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX9-NEXT:    s_lshr_b32 s4, s4, 16
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, s4, v3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, s5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX9-NEXT:    ;;#ASMSTART
@@ -1137,38 +808,18 @@ define void @s_minimum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX940-NEXT:    v_mov_b32_e32 v1, s1
+; GFX940-NEXT:    s_lshr_b32 s1, s1, 16
 ; GFX940-NEXT:    v_pk_min_f16 v1, s0, v1
 ; GFX940-NEXT:    v_mov_b32_e32 v2, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, s0, v0
-; GFX940-NEXT:    v_mov_b32_e32 v4, s0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v2, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s0, 32
 ; GFX940-NEXT:    s_lshr_b32 s0, s0, 16
-; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v3, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s1, 32
-; GFX940-NEXT:    s_lshr_b32 s1, s1, 16
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
 ; GFX940-NEXT:    v_mov_b32_e32 v3, s1
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, s0, v3
 ; GFX940-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v2, s0
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, s1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v1
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; GFX940-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; use v0
@@ -1181,24 +832,12 @@ define void @s_minimum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
 ; GFX10-NEXT:    v_pk_min_f16 v0, s4, s5
 ; GFX10-NEXT:    v_cmp_o_f16_e64 vcc_lo, s4, s5
 ; GFX10-NEXT:    s_lshr_b32 s6, s5, 16
-; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX10-NEXT:    v_cmp_class_f16_e64 s8, s4, 32
+; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e64 vcc_lo, s7, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v0, s4, s8
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 s4, s7, 32
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v1, s7, s4
-; GFX10-NEXT:    v_cmp_class_f16_e64 s4, s5, 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s5, s4
-; GFX10-NEXT:    v_cmp_class_f16_e64 s4, s6, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, s6, s4
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v1
+; GFX10-NEXT:    v_cmp_o_f16_e64 vcc_lo, s4, s6
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v1, vcc_lo
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; use v0
@@ -1211,27 +850,14 @@ define void @s_minimum_v2f16(<2 x half> inreg %src0, <2 x half> inreg %src1) {
 ; GFX11-NEXT:    v_pk_min_f16 v0, s0, s1
 ; GFX11-NEXT:    v_cmp_o_f16_e64 vcc_lo, s0, s1
 ; GFX11-NEXT:    s_lshr_b32 s2, s1, 16
-; GFX11-NEXT:    s_lshr_b32 s3, s0, 16
-; GFX11-NEXT:    v_cmp_class_f16_e64 s4, s0, 32
+; GFX11-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e64 vcc_lo, s3, s2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v0, s0, s4
+; GFX11-NEXT:    v_cmp_o_f16_e64 vcc_lo, s0, s2
+; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 s0, s3, 32
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v1, s3, s0
-; GFX11-NEXT:    v_cmp_class_f16_e64 s0, s1, 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s1, s0
-; GFX11-NEXT:    v_cmp_class_f16_e64 s0, s2, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, s2, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_dual_cndmask_b32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff, v0
 ; GFX11-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use v0
@@ -1265,31 +891,13 @@ define <3 x half> @v_minimum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX8-NEXT:    v_min_f16_e32 v6, v5, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v5, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v6, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v5, v1, v3
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v5, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v3, v0, v2
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -1300,33 +908,13 @@ define <3 x half> @v_minimum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX9-NEXT:    v_pk_min_f16 v4, v1, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
 ; GFX9-NEXT:    v_pk_min_f16 v3, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
 ; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
 ; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    v_perm_b32 v0, v0, v4, s4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -1337,46 +925,16 @@ define <3 x half> @v_minimum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX940-NEXT:    v_pk_min_f16 v4, v1, v3
 ; GFX940-NEXT:    v_mov_b32_e32 v5, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
 ; GFX940-NEXT:    v_pk_min_f16 v3, v0, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
 ; GFX940-NEXT:    s_nop 1
 ; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
 ; GFX940-NEXT:    v_perm_b32 v0, v0, v4, s0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1384,35 +942,15 @@ define <3 x half> @v_minimum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_min_f16 v4, v0, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_pk_min_f16 v8, v1, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0x7e00, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_pk_min_f16 v2, v1, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v6, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v0, v0, v4, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v3f16:
@@ -1422,35 +960,17 @@ define <3 x half> @v_minimum_v3f16(<3 x half> %src0, <3 x half> %src1) {
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_pk_min_f16 v8, v1, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v4, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v4, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, 0x7e00, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc_lo
+; GFX11-NEXT:    v_pk_min_f16 v4, v1, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v7, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v6, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
 ; GFX11-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v4, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v3f16:
@@ -1471,160 +991,38 @@ define <3 x half> @v_minimum_v3f16__nnan(<3 x half> %src0, <3 x half> %src1) {
 ; GFX8-LABEL: v_minimum_v3f16__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT:    v_min_f16_e32 v6, v5, v4
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX8-NEXT:    v_min_f16_e32 v5, v1, v3
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_min_f16_e32 v3, v0, v2
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_min_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_e32 v0, v0, v2
+; GFX8-NEXT:    v_min_f16_e32 v1, v1, v3
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v3f16__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
-; GFX9-NEXT:    v_pk_min_f16 v5, v0, v2
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT:    v_pk_min_f16 v6, v1, v3
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
-; GFX9-NEXT:    v_perm_b32 v0, v4, v0, s4
+; GFX9-NEXT:    v_pk_min_f16 v0, v0, v2
+; GFX9-NEXT:    v_pk_min_f16 v1, v1, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v3f16__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
-; GFX940-NEXT:    v_pk_min_f16 v5, v0, v2
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX940-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_pk_min_f16 v6, v1, v3
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX940-NEXT:    v_perm_b32 v0, v4, v0, s0
+; GFX940-NEXT:    v_pk_min_f16 v0, v0, v2
+; GFX940-NEXT:    v_pk_min_f16 v1, v1, v3
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v3f16__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_pk_min_f16 v4, v0, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
-; GFX10-NEXT:    v_pk_min_f16 v8, v1, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX10-NEXT:    v_pk_min_f16 v0, v0, v2
+; GFX10-NEXT:    v_pk_min_f16 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v3f16__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_pk_min_f16 v4, v0, v2
-; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
-; GFX11-NEXT:    v_pk_min_f16 v8, v1, v3
-; GFX11-NEXT:    v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX11-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
+; GFX11-NEXT:    v_pk_min_f16 v0, v0, v2
+; GFX11-NEXT:    v_pk_min_f16 v1, v1, v3
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v3f16__nnan:
@@ -1807,42 +1205,18 @@ define <4 x half> @v_minimum_v4f16(<4 x half> %src0, <4 x half> %src1) {
 ; GFX8-NEXT:    v_min_f16_e32 v6, v5, v4
 ; GFX8-NEXT:    v_mov_b32_e32 v7, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v5, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v7, v6, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
 ; GFX8-NEXT:    v_min_f16_e32 v8, v6, v5
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v6, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v7, v8, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v8, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v6, v1, v3
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v7, v6, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v3, v0, v2
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v3, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
@@ -1856,44 +1230,16 @@ define <4 x half> @v_minimum_v4f16(<4 x half> %src0, <4 x half> %src1) {
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
 ; GFX9-NEXT:    v_pk_min_f16 v3, v0, v2
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
 ; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v4, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    v_perm_b32 v0, v0, v4, s4
 ; GFX9-NEXT:    v_perm_b32 v1, v1, v6, s4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -1907,59 +1253,19 @@ define <4 x half> @v_minimum_v4f16(<4 x half> %src0, <4 x half> %src1) {
 ; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v6, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v7, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v3
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_min_f16 v3, v0, v2
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
 ; GFX940-NEXT:    v_perm_b32 v1, v1, v6, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v4, v5, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v4, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v4
-; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v3, vcc
 ; GFX940-NEXT:    v_perm_b32 v0, v0, v4, s0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1968,46 +1274,18 @@ define <4 x half> @v_minimum_v4f16(<4 x half> %src0, <4 x half> %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_min_f16 v4, v1, v3
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX10-NEXT:    v_pk_min_f16 v7, v0, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX10-NEXT:    v_pk_min_f16 v5, v0, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7e00, v4, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v10, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v4, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v3, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v7, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v1, v3, v1, 0x5040100
+; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v5
+; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7e00, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v0, v2 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_perm_b32 v0, v0, v5, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v4, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v1, v1, v6, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v4f16:
@@ -2015,47 +1293,23 @@ define <4 x half> @v_minimum_v4f16(<4 x half> %src0, <4 x half> %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_pk_min_f16 v4, v1, v3
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
 ; GFX11-NEXT:    v_pk_min_f16 v7, v0, v2
-; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX11-NEXT:    v_lshrrev_b32_e32 v9, 16, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
-; GFX11-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v4, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v9, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v7, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v8
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v9, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v10, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v4, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v3, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v7, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v11, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v3
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v4, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_perm_b32 v1, v3, v1, 0x5040100
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2077,212 +1331,40 @@ define <4 x half> @v_minimum_v4f16__nnan(<4 x half> %src0, <4 x half> %src1) {
 ; GFX8-LABEL: v_minimum_v4f16__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX8-NEXT:    v_min_f16_e32 v6, v5, v4
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX8-NEXT:    v_min_f16_e32 v7, v6, v5
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX8-NEXT:    v_min_f16_e32 v6, v1, v3
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX8-NEXT:    v_min_f16_e32 v3, v0, v2
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
-; GFX8-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_min_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_min_f16_e32 v1, v1, v3
+; GFX8-NEXT:    v_min_f16_e32 v0, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v5
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_v4f16__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
-; GFX9-NEXT:    v_pk_min_f16 v5, v1, v3
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX9-NEXT:    v_pk_min_f16 v7, v0, v2
-; GFX9-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v9, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
-; GFX9-NEXT:    v_perm_b32 v0, v6, v0, s4
-; GFX9-NEXT:    v_perm_b32 v1, v4, v1, s4
+; GFX9-NEXT:    v_pk_min_f16 v0, v0, v2
+; GFX9-NEXT:    v_pk_min_f16 v1, v1, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v4f16__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
-; GFX940-NEXT:    v_pk_min_f16 v5, v1, v3
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v5
-; GFX940-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v9, 16, v2
-; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX940-NEXT:    v_pk_min_f16 v7, v0, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
-; GFX940-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v9, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    v_perm_b32 v1, v4, v1, s0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX940-NEXT:    v_perm_b32 v0, v6, v0, s0
+; GFX940-NEXT:    v_pk_min_f16 v0, v0, v2
+; GFX940-NEXT:    v_pk_min_f16 v1, v1, v3
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_v4f16__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_pk_min_f16 v4, v1, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX10-NEXT:    v_pk_min_f16 v6, v0, v2
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v4
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v9, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v11, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v1, v5, v1, 0x5040100
+; GFX10-NEXT:    v_pk_min_f16 v0, v0, v2
+; GFX10-NEXT:    v_pk_min_f16 v1, v1, v3
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v4f16__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_pk_min_f16 v4, v1, v3
-; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX11-NEXT:    v_pk_min_f16 v6, v0, v2
-; GFX11-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
-; GFX11-NEXT:    v_lshrrev_b32_e32 v8, 16, v4
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
-; GFX11-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v9, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v11, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_perm_b32 v0, v2, v0, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11-NEXT:    v_perm_b32 v1, v5, v1, 0x5040100
+; GFX11-NEXT:    v_pk_min_f16 v0, v0, v2
+; GFX11-NEXT:    v_pk_min_f16 v1, v1, v3
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v4f16__nnan:
@@ -2493,82 +1575,34 @@ define <8 x half> @v_minimum_v8f16(<8 x half> %src0, <8 x half> %src1) {
 ; GFX8-NEXT:    v_min_f16_e32 v10, v9, v8
 ; GFX8-NEXT:    v_mov_b32_e32 v11, 0x7e00
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v9, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v11, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v9, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v9, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v8, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v11, v10, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v6
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
 ; GFX8-NEXT:    v_min_f16_e32 v12, v10, v9
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v10, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v11, v12, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v10, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v9, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v10, v9, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v12, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v9, v11, v12, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v5
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v1
 ; GFX8-NEXT:    v_min_f16_e32 v13, v12, v10
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v12, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v11, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v12, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v13, v12, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v10, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v13, v10, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v10, v11, v13, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v13, 16, v0
 ; GFX8-NEXT:    v_min_f16_e32 v14, v13, v12
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v13, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v11, v14, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v13, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v14, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v12, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v13, v12, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v12, v11, v14, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v13, v3, v7
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v3, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v11, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v13, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v13, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v7, v2, v6
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v2, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v7, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v6, v1, v5
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v1, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v11, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v11, v6, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v5, v0, v4
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v4
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v11, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v11, v5, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v12
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v10
@@ -2586,83 +1620,27 @@ define <8 x half> @v_minimum_v8f16(<8 x half> %src0, <8 x half> %src1) {
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v3, v7
 ; GFX9-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v10, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v3, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v8, vcc
 ; GFX9-NEXT:    v_pk_min_f16 v7, v2, v6
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v2, v6
 ; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v8, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v6, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v8, v8, v11, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v2, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v9, v7, vcc
 ; GFX9-NEXT:    v_pk_min_f16 v6, v1, v5
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v5
 ; GFX9-NEXT:    v_cndmask_b32_e32 v7, v9, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v7, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v6, vcc
 ; GFX9-NEXT:    v_pk_min_f16 v5, v0, v4
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v4
 ; GFX9-NEXT:    v_cndmask_b32_e32 v6, v9, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v6, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v11, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v9, v5, vcc
 ; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    v_perm_b32 v0, v0, v6, s4
 ; GFX9-NEXT:    v_perm_b32 v1, v1, v7, s4
@@ -2679,117 +1657,37 @@ define <8 x half> @v_minimum_v8f16(<8 x half> %src0, <8 x half> %src1) {
 ; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v10, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX940-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v3, v7
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_min_f16 v7, v2, v6
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v9, v8, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v2, v6
 ; GFX940-NEXT:    v_perm_b32 v3, v3, v10, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v8, v9, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v8, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v6, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v8
-; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v8, v8, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v2, v6
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_min_f16 v6, v1, v5
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v9, v7, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v5
 ; GFX940-NEXT:    v_perm_b32 v2, v2, v8, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v7, v9, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v7, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v7
-; GFX940-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v9, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_min_f16 v5, v0, v4
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v9, v6, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v4
 ; GFX940-NEXT:    v_perm_b32 v1, v1, v7, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v6, v9, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v6, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v6
-; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v11, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v4
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v5
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v9, v5, vcc
 ; GFX940-NEXT:    v_perm_b32 v0, v0, v6, s0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -2798,88 +1696,32 @@ define <8 x half> @v_minimum_v8f16(<8 x half> %src0, <8 x half> %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_min_f16 v8, v3, v7
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v7
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
-; GFX10-NEXT:    v_pk_min_f16 v13, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7e00, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v11, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v8, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    v_pk_min_f16 v11, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v6
-; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v10, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
+; GFX10-NEXT:    v_pk_min_f16 v9, v2, v6
+; GFX10-NEXT:    v_pk_min_f16 v12, v1, v5
+; GFX10-NEXT:    v_pk_min_f16 v13, v0, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, 0x7e00, v8, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v2, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v10, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, 0x7e00, v12, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v9
+; GFX10-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7e00, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v2, v6 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v11, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, 0x7e00, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v14, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v10, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_pk_min_f16 v10, v0, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v15, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v12, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v14
-; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
-; GFX10-NEXT:    v_perm_b32 v2, v6, v2, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v14, v9, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v11, 16, v13
+; GFX10-NEXT:    v_perm_b32 v2, v2, v9, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7e00, v12, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v4
-; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, 0x7e00, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v12, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, 0x7e00, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v5
+; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
 ; GFX10-NEXT:    v_cndmask_b32_e32 v13, 0x7e00, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v12, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v14
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v13
-; GFX10-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    v_perm_b32 v1, v1, v9, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v7, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v3, v5, v3, 0x5040100
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v0, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v1, v5 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_perm_b32 v0, v0, v13, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v3, v7 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_perm_b32 v1, v1, v6, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v8, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v3, v3, v10, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v8f16:
@@ -2887,94 +1729,42 @@ define <8 x half> @v_minimum_v8f16(<8 x half> %src0, <8 x half> %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_pk_min_f16 v8, v3, v7
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v7
-; GFX11-NEXT:    v_lshrrev_b32_e32 v10, 16, v7
-; GFX11-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
-; GFX11-NEXT:    v_pk_min_f16 v13, v1, v5
+; GFX11-NEXT:    v_pk_min_f16 v10, v2, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v11, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v2
+; GFX11-NEXT:    v_pk_min_f16 v14, v1, v5
 ; GFX11-NEXT:    v_cndmask_b32_e32 v9, 0x7e00, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v11, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v8, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v9
-; GFX11-NEXT:    v_pk_min_f16 v11, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v9, 16, v6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v7, v10, vcc_lo
-; GFX11-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v2, v6
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v10, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, 0x7e00, v12, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v13, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v10, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v12, v11
+; GFX11-NEXT:    v_pk_min_f16 v11, v0, v4
+; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
+; GFX11-NEXT:    v_cndmask_b32_e32 v6, 0x7e00, v13, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, 0x7e00, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v14, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v10, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_pk_min_f16 v10, v0, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v15, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v13, 16, v0
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v12
-; GFX11-NEXT:    v_lshrrev_b32_e32 v11, 16, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v12, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v14
-; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_perm_b32 v2, v6, v2, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v14, v9, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v15, 16, v11
+; GFX11-NEXT:    v_cndmask_b32_e32 v10, 0x7e00, v14, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v4
-; GFX11-NEXT:    v_lshrrev_b32_e32 v14, 16, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, 0x7e00, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v12, v11
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, 0x7e00, v14, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
+; GFX11-NEXT:    v_perm_b32 v2, v6, v2, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v11, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v13, v12
+; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v15, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v5
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, 0x7e00, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v12, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v14
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v14, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v13, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v8
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_perm_b32 v1, v1, v9, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v8, v7, vcc_lo
 ; GFX11-NEXT:    v_perm_b32 v0, v4, v0, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v3, v5, v3, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v14, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v7
+; GFX11-NEXT:    v_perm_b32 v1, v1, v10, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v8, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_perm_b32 v3, v3, v9, 0x5040100
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v8f16:
@@ -2998,166 +1788,70 @@ define <16 x half> @v_minimum_v16f16(<16 x half> %src0, <16 x half> %src1) {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v16, 16, v15
-; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v7
-; GFX8-NEXT:    v_min_f16_e32 v19, v18, v16
-; GFX8-NEXT:    v_mov_b32_e32 v17, 0x7e00
-; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v18, v16
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v17, v19, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v18, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v16, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v18, v16, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v16, v19, v16, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v14
-; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v6
-; GFX8-NEXT:    v_min_f16_e32 v20, v19, v18
-; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v19, v18
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v17, v20, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v19, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v18, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v18, v20, v18, vcc
-; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v13
+; GFX8-NEXT:    v_lshrrev_b32_e32 v17, 16, v7
+; GFX8-NEXT:    v_min_f16_e32 v18, v17, v16
+; GFX8-NEXT:    v_mov_b32_e32 v19, 0x7e00
+; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v17, v16
+; GFX8-NEXT:    v_cndmask_b32_e32 v16, v19, v18, vcc
+; GFX8-NEXT:    v_lshrrev_b32_e32 v17, 16, v14
+; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
+; GFX8-NEXT:    v_min_f16_e32 v20, v18, v17
+; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v18, v17
+; GFX8-NEXT:    v_cndmask_b32_e32 v17, v19, v20, vcc
+; GFX8-NEXT:    v_lshrrev_b32_e32 v18, 16, v13
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v5
-; GFX8-NEXT:    v_min_f16_e32 v21, v20, v19
-; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v20, v19
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v17, v21, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v20, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v19, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v20, v19, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v19, v21, v19, vcc
+; GFX8-NEXT:    v_min_f16_e32 v21, v20, v18
+; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v20, v18
+; GFX8-NEXT:    v_cndmask_b32_e32 v18, v19, v21, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v20, 16, v12
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v4
 ; GFX8-NEXT:    v_min_f16_e32 v22, v21, v20
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v21, v20
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v17, v22, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v21, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v20, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v20, v22, v20, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v20, v19, v22, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v11
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v3
 ; GFX8-NEXT:    v_min_f16_e32 v23, v22, v21
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v22, v21
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v17, v23, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v22, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v21, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v22, v21, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v21, v23, v21, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v21, v19, v23, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 16, v10
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v2
 ; GFX8-NEXT:    v_min_f16_e32 v24, v23, v22
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v23, v22
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v17, v24, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v23, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v22, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v22, v24, v22, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v22, v19, v24, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v9
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v1
 ; GFX8-NEXT:    v_min_f16_e32 v25, v24, v23
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v24, v23
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v17, v25, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v24, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v23, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v24, v23, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v23, v25, v23, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v23, v19, v25, vcc
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v24, 16, v8
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v25, 16, v0
 ; GFX8-NEXT:    v_min_f16_e32 v26, v25, v24
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v25, v24
-; GFX8-NEXT:    v_cndmask_b32_e32 v26, v17, v26, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v25, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v26, v25, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v24, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v25, v24, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v26
-; GFX8-NEXT:    v_cndmask_b32_e32 v24, v26, v24, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v24, v19, v26, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v25, v7, v15
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v7, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v25, v17, v25, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v15, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v25
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v25, v7, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v19, v25, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v15, v6, v14
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v6, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v17, v15, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v14, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v15
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v19, v15, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v14, v5, v13
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v5, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v17, v14, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v13, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v14
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v14, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v19, v14, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v13, v4, v12
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v4, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v12, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v19, v13, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v12, v3, v11
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v3, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v11, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v19, v12, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v11, v2, v10
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v2, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v10, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v11
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v19, v11, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v10, v1, v9
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v1, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v17, v10, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v9, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v19, v10, vcc
 ; GFX8-NEXT:    v_min_f16_e32 v9, v0, v8
 ; GFX8-NEXT:    v_cmp_o_f16_e32 vcc, v0, v8
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f16_e64 vcc, v8, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX8-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v9
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v19, v9, vcc
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v24
 ; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v23
@@ -3168,9 +1862,9 @@ define <16 x half> @v_minimum_v16f16(<16 x half> %src0, <16 x half> %src1) {
 ; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v20
 ; GFX8-NEXT:    v_or_b32_sdwa v4, v4, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v19
-; GFX8-NEXT:    v_or_b32_sdwa v5, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v18
+; GFX8-NEXT:    v_or_b32_sdwa v5, v5, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v17
 ; GFX8-NEXT:    v_or_b32_sdwa v6, v6, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v16
 ; GFX8-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
@@ -3179,414 +1873,142 @@ define <16 x half> @v_minimum_v16f16(<16 x half> %src0, <16 x half> %src1) {
 ; GFX9-LABEL: v_minimum_v16f16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_pk_min_f16 v18, v7, v15
+; GFX9-NEXT:    v_pk_min_f16 v16, v7, v15
 ; GFX9-NEXT:    v_mov_b32_e32 v17, 0x7e00
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v7, v15
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v16, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v15, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v15, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v16
-; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
-; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NEXT:    v_cndmask_b32_e32 v16, v16, v19, vcc
-; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v7, v15
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v18, v7, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v15, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX9-NEXT:    v_cndmask_b32_e32 v7, v18, v7, vcc
-; GFX9-NEXT:    v_pk_min_f16 v18, v6, v14
+; GFX9-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v7, v15 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v17, v16, vcc
+; GFX9-NEXT:    v_pk_min_f16 v15, v6, v14
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v6, v14
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v15, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v14, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v14, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v15
-; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
-; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX9-NEXT:    v_cndmask_b32_e32 v15, v15, v19, vcc
-; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v6, v14
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v14, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX9-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX9-NEXT:    v_pk_min_f16 v18, v5, v13
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v5, v13
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v14, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v13, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v13, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v14
-; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
-; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX9-NEXT:    v_cndmask_b32_e32 v14, v14, v19, vcc
-; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
+; GFX9-NEXT:    v_cndmask_b32_e32 v16, v17, v15, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v6, v14 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v17, v15, vcc
+; GFX9-NEXT:    v_pk_min_f16 v14, v5, v13
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v5, v13
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v18, v5, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v13, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v18, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v15, v17, v14, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v5, v13 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v17, v14, vcc
 ; GFX9-NEXT:    v_pk_min_f16 v13, v4, v12
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v4, v12
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v17, v13, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v18, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v12, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v12, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-NEXT:    v_cndmask_b32_e32 v18, v18, v19, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v14, v17, v13, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v4, v12
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v12, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v4, v12 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v17, v13, vcc
 ; GFX9-NEXT:    v_pk_min_f16 v12, v3, v11
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v3, v11
 ; GFX9-NEXT:    v_cndmask_b32_e32 v13, v17, v12, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v13, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v11, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v11, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
-; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v13, v19, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v3, v11
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v11, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v3, v11 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v17, v12, vcc
 ; GFX9-NEXT:    v_pk_min_f16 v11, v2, v10
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v2, v10
 ; GFX9-NEXT:    v_cndmask_b32_e32 v12, v17, v11, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v12, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v10, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v10, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
-; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
-; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v12, v12, v19, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v2, v10
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v10, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v11
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v2, v10 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v17, v11, vcc
 ; GFX9-NEXT:    v_pk_min_f16 v10, v1, v9
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v9
 ; GFX9-NEXT:    v_cndmask_b32_e32 v11, v17, v10, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v11, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v9, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v9, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v11
-; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v11, v11, v19, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v1, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v17, v10, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v9, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v9 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v10, vcc
 ; GFX9-NEXT:    v_pk_min_f16 v9, v0, v8
 ; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v8
 ; GFX9-NEXT:    v_cndmask_b32_e32 v10, v17, v9, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v10, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v8, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v19, v19, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX9-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v10, v10, v19, vcc
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX9-NEXT:    v_cmp_o_f16_e32 vcc, v0, v8
-; GFX9-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f16_e64 vcc, v8, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
+; GFX9-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v8 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v17, v9, vcc
 ; GFX9-NEXT:    s_mov_b32 s4, 0x5040100
 ; GFX9-NEXT:    v_perm_b32 v0, v0, v10, s4
 ; GFX9-NEXT:    v_perm_b32 v1, v1, v11, s4
 ; GFX9-NEXT:    v_perm_b32 v2, v2, v12, s4
 ; GFX9-NEXT:    v_perm_b32 v3, v3, v13, s4
-; GFX9-NEXT:    v_perm_b32 v4, v4, v18, s4
-; GFX9-NEXT:    v_perm_b32 v5, v5, v14, s4
-; GFX9-NEXT:    v_perm_b32 v6, v6, v15, s4
-; GFX9-NEXT:    v_perm_b32 v7, v7, v16, s4
+; GFX9-NEXT:    v_perm_b32 v4, v4, v14, s4
+; GFX9-NEXT:    v_perm_b32 v5, v5, v15, s4
+; GFX9-NEXT:    v_perm_b32 v6, v6, v16, s4
+; GFX9-NEXT:    v_perm_b32 v7, v7, v18, s4
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_v16f16:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_pk_min_f16 v18, v7, v15
+; GFX940-NEXT:    v_pk_min_f16 v16, v7, v15
 ; GFX940-NEXT:    v_mov_b32_e32 v17, 0x7e00
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v7, v15
 ; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v16, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v15, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v15, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v16
-; GFX940-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GFX940-NEXT:    v_cndmask_b32_e32 v18, v17, v16, vcc
+; GFX940-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v7, v15 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX940-NEXT:    v_pk_min_f16 v15, v6, v14
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v16, v16, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v7, v15
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v7, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v18, v7, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v15, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v7, v18, v7, vcc
-; GFX940-NEXT:    v_pk_min_f16 v18, v6, v14
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v17, v16, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v6, v14
-; GFX940-NEXT:    v_perm_b32 v7, v7, v16, s0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
+; GFX940-NEXT:    v_perm_b32 v7, v7, v18, s0
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v15, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v14, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v14, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v15
-; GFX940-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
+; GFX940-NEXT:    v_cndmask_b32_e32 v16, v17, v15, vcc
+; GFX940-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v6, v14 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX940-NEXT:    v_pk_min_f16 v14, v5, v13
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v15, v15, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v6, v14
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v6, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v14, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
-; GFX940-NEXT:    v_pk_min_f16 v18, v5, v13
+; GFX940-NEXT:    v_cndmask_b32_e32 v6, v17, v15, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v5, v13
-; GFX940-NEXT:    v_perm_b32 v6, v6, v15, s0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v14, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v13, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v13, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v14
-; GFX940-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
+; GFX940-NEXT:    v_perm_b32 v6, v6, v16, s0
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v14, v14, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v5, v13
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v17, v18, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v5, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v18, v5, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v13, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
+; GFX940-NEXT:    v_cndmask_b32_e32 v15, v17, v14, vcc
+; GFX940-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v5, v13 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_min_f16 v13, v4, v12
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v18, v5, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v17, v14, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v4, v12
-; GFX940-NEXT:    v_perm_b32 v5, v5, v14, s0
+; GFX940-NEXT:    v_perm_b32 v5, v5, v15, s0
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v17, v13, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
+; GFX940-NEXT:    v_cndmask_b32_e32 v14, v17, v13, vcc
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v18, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v12, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v12, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v18
-; GFX940-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v18, v18, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v4, v12
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v4, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v12, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v4, v12 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_min_f16 v12, v3, v11
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v4, v13, v4, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v4, v17, v13, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v3, v11
-; GFX940-NEXT:    v_perm_b32 v4, v4, v18, s0
+; GFX940-NEXT:    v_perm_b32 v4, v4, v14, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v13, v17, v12, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v13, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v11, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v11, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v13
-; GFX940-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v13, v13, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v3, v11
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v3, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v11, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v3, v11 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_min_f16 v11, v2, v10
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v3, v12, v3, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v17, v12, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v2, v10
 ; GFX940-NEXT:    v_perm_b32 v3, v3, v13, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v12, v17, v11, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v11, 16, v11
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v12, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v10, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v10, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v12
-; GFX940-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v12, v12, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v2, v10
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v2, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v10, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v11
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v2, v10 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_min_f16 v10, v1, v9
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v17, v11, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v9
 ; GFX940-NEXT:    v_perm_b32 v2, v2, v12, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v11, v17, v10, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v11, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v9, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v9, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v11
-; GFX940-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v11, v11, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v1, v9
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v17, v10, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v9, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v1, v9 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    v_pk_min_f16 v9, v0, v8
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v10, v1, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v17, v10, vcc
 ; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v8
 ; GFX940-NEXT:    v_perm_b32 v1, v1, v11, s0
 ; GFX940-NEXT:    s_nop 0
 ; GFX940-NEXT:    v_cndmask_b32_e32 v10, v17, v9, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
 ; GFX940-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v10, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v8, 32
-; GFX940-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v19, v19, v8, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v10
-; GFX940-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v10, v10, v19, vcc
-; GFX940-NEXT:    v_cmp_o_f16_e32 vcc, v0, v8
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f16_e64 vcc, v8, 32
+; GFX940-NEXT:    v_cmp_o_f16_sdwa vcc, v0, v8 src0_sel:WORD_1 src1_sel:WORD_1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX940-NEXT:    v_cmp_eq_f16_e32 vcc, 0, v9
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v17, v9, vcc
 ; GFX940-NEXT:    v_perm_b32 v0, v0, v10, s0
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3595,353 +2017,145 @@ define <16 x half> @v_minimum_v16f16(<16 x half> %src0, <16 x half> %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_min_f16 v16, v7, v15
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v7, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v6
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, 0x7e00, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v17, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v15, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v18, v15, vcc_lo
-; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v17
-; GFX10-NEXT:    v_cndmask_b32_e32 v17, v17, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v7, v15
 ; GFX10-NEXT:    v_pk_min_f16 v18, v6, v14
+; GFX10-NEXT:    v_pk_min_f16 v19, v3, v11
+; GFX10-NEXT:    v_pk_min_f16 v20, v2, v10
+; GFX10-NEXT:    v_lshrrev_b32_e32 v17, 16, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v16, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v7, v15 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v15, 16, v18
+; GFX10-NEXT:    v_pk_min_f16 v21, v0, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, 0x7e00, v17, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v14
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
+; GFX10-NEXT:    v_pk_min_f16 v17, v5, v13
+; GFX10-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
+; GFX10-NEXT:    v_perm_b32 v7, v7, v16, 0x5040100
 ; GFX10-NEXT:    v_cndmask_b32_e32 v18, 0x7e00, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v20, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v15, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v21, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v14, 32
-; GFX10-NEXT:    v_pk_min_f16 v20, v4, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v19, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v16, 16, v13
-; GFX10-NEXT:    v_perm_b32 v7, v7, v17, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v15, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v18
-; GFX10-NEXT:    v_pk_min_f16 v15, v5, v13
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v5
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v15
-; GFX10-NEXT:    v_cndmask_b32_e32 v14, v21, v14, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v6, v14 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v17
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, 0x7e00, v15, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v5, v13
-; GFX10-NEXT:    v_perm_b32 v6, v14, v6, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, 0x7e00, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v18, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, 0x7e00, v19, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v6, v6, v18, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, 0x7e00, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v5, v13 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_pk_min_f16 v17, v4, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, 0x7e00, v14, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v18, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v21, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v13, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v16, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v18, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v22, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v15
-; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v19, v13, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v11
-; GFX10-NEXT:    v_perm_b32 v5, v13, v5, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v15, v21, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v12
-; GFX10-NEXT:    v_pk_min_f16 v16, v3, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v18, 0x7e00, v18, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v17
+; GFX10-NEXT:    v_perm_b32 v5, v5, v15, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, 0x7e00, v17, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v11
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v16
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v16, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v20, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v18, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 32
-; GFX10-NEXT:    v_pk_min_f16 v12, v2, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v19, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v20, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_pk_min_f16 v19, v1, v9
-; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v10
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v21, v11, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v17, 16, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v19, 0x7e00, v19, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v3, v11 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_pk_min_f16 v11, v1, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v17, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v2, v10
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v2
-; GFX10-NEXT:    v_perm_b32 v3, v11, v3, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v12, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v11
+; GFX10-NEXT:    v_perm_b32 v3, v3, v19, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v17, 0x7e00, v20, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v9
-; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, 0x7e00, v19, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v21, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, 0x7e00, v12, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, v22, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 32
-; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v21, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v12, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v21, v23, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v22
-; GFX10-NEXT:    v_pk_min_f16 v20, v0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v16, v22, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v9
-; GFX10-NEXT:    v_lshrrev_b32_e32 v21, 16, v8
-; GFX10-NEXT:    v_lshrrev_b32_e32 v22, 16, v0
-; GFX10-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
-; GFX10-NEXT:    v_cndmask_b32_e32 v19, 0x7e00, v19, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v20, 16, v20
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v1, v9 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v22, vcc_lo
 ; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v8
-; GFX10-NEXT:    v_cndmask_b32_e32 v20, 0x7e00, v20, vcc_lo
-; GFX10-NEXT:    v_cmp_o_f16_e32 vcc_lo, v22, v21
-; GFX10-NEXT:    v_cndmask_b32_e32 v23, 0x7e00, v23, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v20, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v22, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v8, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f16_e64 vcc_lo, v21, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v22, v21, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v19
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v20
-; GFX10-NEXT:    v_perm_b32 v1, v1, v16, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v20, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v23
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v23, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_perm_b32 v0, v8, v0, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v12, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v18
-; GFX10-NEXT:    v_perm_b32 v2, v9, v2, 0x5040100
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v18, v4, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v4, v4, v15, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v1, v1, v11, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0x7e00, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v0, v8 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v23, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v2, v10 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_perm_b32 v0, v0, v9, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v20, vcc_lo
+; GFX10-NEXT:    v_cmp_o_f16_sdwa vcc_lo, v4, v12 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX10-NEXT:    v_perm_b32 v2, v2, v17, 0x5040100
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v14, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v4, v4, v13, 0x5040100
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_v16f16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_pk_min_f16 v16, v7, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v17, 16, v15
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v7
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v7, v15
-; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v6
-; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v17, 0x7e00, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v18, v17, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v15, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v18, v18, v15, vcc_lo
-; GFX11-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v17
-; GFX11-NEXT:    v_cndmask_b32_e32 v17, v17, v18, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v7, v15
-; GFX11-NEXT:    v_pk_min_f16 v18, v6, v14
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v14
-; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v18
-; GFX11-NEXT:    v_cndmask_b32_e32 v18, 0x7e00, v18, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v7, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v20, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v21, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v6, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v15, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v21, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v14, 32
+; GFX11-NEXT:    v_pk_min_f16 v15, v6, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v16
 ; GFX11-NEXT:    v_pk_min_f16 v20, v4, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v7, v16, v7, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v19, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v16, 16, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, v15, v19, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v18
-; GFX11-NEXT:    v_pk_min_f16 v15, v5, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_pk_min_f16 v22, v2, v10
+; GFX11-NEXT:    v_cndmask_b32_e32 v7, 0x7e00, v16, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v18, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v17, 16, v14
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
+; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v8
+; GFX11-NEXT:    v_lshrrev_b32_e32 v24, 16, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v19, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v6, v14
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v15
-; GFX11-NEXT:    v_perm_b32 v7, v7, v17, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v14, v21, v14, vcc_lo
+; GFX11-NEXT:    v_pk_min_f16 v14, v5, v13
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_perm_b32 v7, v16, v7, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v6, 0x7e00, v15, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v18, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v17, 16, v13
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v5
+; GFX11-NEXT:    v_cndmask_b32_e32 v15, 0x7e00, v19, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v5, v13
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, 0x7e00, v15, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v18, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v19, 0x7e00, v19, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v6, v15, v6, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v5, 0x7e00, v14, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v18, v17
+; GFX11-NEXT:    v_pk_min_f16 v17, v3, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
+; GFX11-NEXT:    v_cndmask_b32_e32 v13, 0x7e00, v19, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v5, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v18, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v18, v19, v18, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v22, v21, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v13, 32
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v11
+; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v17
+; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v16, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, v18, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v18, 16, v20
+; GFX11-NEXT:    v_cndmask_b32_e32 v14, 0x7e00, v20, vcc_lo
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v3
-; GFX11-NEXT:    v_perm_b32 v6, v14, v6, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v22, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v15
-; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v15, v5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v13, v19, v13, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v11
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT:    v_perm_b32 v5, v13, v5, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v15, v21, v16, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v12
-; GFX11-NEXT:    v_pk_min_f16 v16, v3, v11
-; GFX11-NEXT:    v_cndmask_b32_e32 v18, 0x7e00, v18, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v3, v11
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v16
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v16, vcc_lo
+; GFX11-NEXT:    v_perm_b32 v5, v13, v5, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, 0x7e00, v17, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v20, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v21, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v4, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v18, v4, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v3, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v20, v21, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v12, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v11, 32
-; GFX11-NEXT:    v_pk_min_f16 v12, v2, v10
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v19, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, v20, v19, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v10
 ; GFX11-NEXT:    v_pk_min_f16 v19, v1, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v21
-; GFX11-NEXT:    v_cndmask_b32_e32 v11, v21, v11, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v20, 16, v22
+; GFX11-NEXT:    v_cndmask_b32_e32 v11, 0x7e00, v21, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v2, v10
-; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, 0x7e00, v12, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
+; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_perm_b32 v3, v11, v3, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v17, 0x7e00, v22, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v9
-; GFX11-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
-; GFX11-NEXT:    v_cndmask_b32_e32 v22, 0x7e00, v19, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v2, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v21, v20
-; GFX11-NEXT:    v_cndmask_b32_e32 v12, 0x7e00, v12, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v23, v22, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v10, 32
-; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v21, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v12, v21, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v21, v23, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v16
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v20, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v10, v10, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v22
-; GFX11-NEXT:    v_pk_min_f16 v20, v0, v8
-; GFX11-NEXT:    v_perm_b32 v3, v11, v3, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v16, v22, v21, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    v_pk_min_f16 v22, v0, v8
+; GFX11-NEXT:    v_cndmask_b32_e32 v21, 0x7e00, v19, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v1, v9
-; GFX11-NEXT:    v_lshrrev_b32_e32 v21, 16, v8
-; GFX11-NEXT:    v_lshrrev_b32_e32 v22, 16, v0
-; GFX11-NEXT:    v_lshrrev_b32_e32 v23, 16, v20
-; GFX11-NEXT:    v_cndmask_b32_e32 v19, 0x7e00, v19, vcc_lo
+; GFX11-NEXT:    v_lshrrev_b32_e32 v25, 16, v22
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, 0x7e00, v19, vcc_lo
 ; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v0, v8
-; GFX11-NEXT:    v_cndmask_b32_e32 v20, 0x7e00, v20, vcc_lo
-; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v22, v21
-; GFX11-NEXT:    v_cndmask_b32_e32 v23, 0x7e00, v23, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v20, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v22, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v22, v23, v22, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v9, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v8, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f16_e64 vcc_lo, v21, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v22, v21, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v19
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v19, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v20
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v20, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v23
-; GFX11-NEXT:    v_cndmask_b32_e32 v8, v23, v8, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v12
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_perm_b32 v1, v1, v21, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7e00, v22, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v24, v23
+; GFX11-NEXT:    v_cndmask_b32_e32 v8, 0x7e00, v25, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v2, v10
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_perm_b32 v0, v8, v0, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v9, v12, v10, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f16_e32 vcc_lo, 0, v18
-; GFX11-NEXT:    v_perm_b32 v1, v1, v16, 0x5040100
-; GFX11-NEXT:    v_perm_b32 v2, v9, v2, 0x5040100
-; GFX11-NEXT:    v_cndmask_b32_e32 v4, v18, v4, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7e00, v20, vcc_lo
+; GFX11-NEXT:    v_cmp_o_f16_e32 vcc_lo, v4, v12
+; GFX11-NEXT:    v_perm_b32 v2, v2, v17, 0x5040100
+; GFX11-NEXT:    v_cndmask_b32_e32 v4, 0x7e00, v18, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_perm_b32 v4, v4, v15, 0x5040100
+; GFX11-NEXT:    v_perm_b32 v4, v4, v14, 0x5040100
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_v16f16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll
index 0a9dc3d05676..1da2647fbd60 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll
@@ -14,13 +14,7 @@ define float @v_minimum_f32(float %src0, float %src1) {
 ; GFX7-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_f32:
@@ -29,13 +23,7 @@ define float @v_minimum_f32(float %src0, float %src1) {
 ; GFX8-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f32:
@@ -44,13 +32,7 @@ define float @v_minimum_f32(float %src0, float %src1) {
 ; GFX9-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f32:
@@ -60,16 +42,7 @@ define float @v_minimum_f32(float %src0, float %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f32:
@@ -77,13 +50,7 @@ define float @v_minimum_f32(float %src0, float %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f32:
@@ -91,15 +58,8 @@ define float @v_minimum_f32(float %src0, float %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f32:
@@ -119,78 +79,37 @@ define float @v_minimum_f32__nnan(float %src0, float %src1) {
 ; GFX7-LABEL: v_minimum_f32__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_f32_e32 v2, v0, v1
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_min_f32_e32 v0, v0, v1
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_f32__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_min_f32_e32 v2, v0, v1
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_min_f32_e32 v0, v0, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f32__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_f32_e32 v2, v0, v1
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_min_f32_e32 v0, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f32__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_min_f32_e32 v2, v0, v1
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_min_f32_e32 v0, v0, v1
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f32__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_min_f32_e32 v2, v0, v1
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_min_f32_e32 v0, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f32__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_min_f32_e32 v2, v0, v1
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_min_f32_e32 v0, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f32__nnan:
@@ -332,13 +251,7 @@ define float @v_minimum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX7-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_f32__nnan_src0:
@@ -348,13 +261,7 @@ define float @v_minimum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX8-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f32__nnan_src0:
@@ -364,13 +271,7 @@ define float @v_minimum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX9-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f32__nnan_src0:
@@ -381,16 +282,7 @@ define float @v_minimum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f32__nnan_src0:
@@ -399,13 +291,7 @@ define float @v_minimum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX10-NEXT:    v_add_f32_e32 v0, 1.0, v0
 ; GFX10-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f32__nnan_src0:
@@ -415,15 +301,7 @@ define float @v_minimum_f32__nnan_src0(float %arg0, float %src1) {
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f32__nnan_src0:
@@ -450,13 +328,7 @@ define float @v_minimum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX7-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX7-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_f32__nnan_src1:
@@ -466,13 +338,7 @@ define float @v_minimum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX8-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX8-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f32__nnan_src1:
@@ -482,13 +348,7 @@ define float @v_minimum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX9-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f32__nnan_src1:
@@ -499,16 +359,7 @@ define float @v_minimum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v3, 0x7fc00000
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, v0, v1
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, v1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f32__nnan_src1:
@@ -517,13 +368,7 @@ define float @v_minimum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX10-NEXT:    v_add_f32_e32 v1, 1.0, v1
 ; GFX10-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX10-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f32__nnan_src1:
@@ -533,15 +378,7 @@ define float @v_minimum_f32__nnan_src1(float %src0, float %arg1) {
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_min_f32_e32 v2, v0, v1
 ; GFX11-NEXT:    v_cmp_o_f32_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, 0x7fc00000, v2, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f32_e64 vcc_lo, v1, 32
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f32__nnan_src1:
@@ -568,14 +405,7 @@ define void @s_minimum_f32(float inreg %src0, float inreg %src1) {
 ; GFX7-NEXT:    v_min_f32_e32 v1, s4, v0
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s4, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX7-NEXT:    v_cmp_class_f32_e64 vcc, s5, 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX7-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX7-NEXT:    ;;#ASMSTART
 ; GFX7-NEXT:    ; use v0
 ; GFX7-NEXT:    ;;#ASMEND
@@ -588,14 +418,7 @@ define void @s_minimum_f32(float inreg %src0, float inreg %src1) {
 ; GFX8-NEXT:    v_min_f32_e32 v1, s4, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s4, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX8-NEXT:    v_cmp_class_f32_e64 vcc, s5, 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use v0
 ; GFX8-NEXT:    ;;#ASMEND
@@ -608,14 +431,7 @@ define void @s_minimum_f32(float inreg %src0, float inreg %src1) {
 ; GFX9-NEXT:    v_min_f32_e32 v1, s4, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX9-NEXT:    v_cmp_o_f32_e32 vcc, s4, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s4, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX9-NEXT:    v_cmp_class_f32_e64 vcc, s5, 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX9-NEXT:    ;;#ASMSTART
 ; GFX9-NEXT:    ; use v0
 ; GFX9-NEXT:    ;;#ASMEND
@@ -629,17 +445,7 @@ define void @s_minimum_f32(float inreg %src0, float inreg %src1) {
 ; GFX940-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
 ; GFX940-NEXT:    v_cmp_o_f32_e32 vcc, s0, v0
 ; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX940-NEXT:    v_mov_b32_e32 v2, s0
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s0, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX940-NEXT:    v_cmp_class_f32_e64 vcc, s1, 32
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX940-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v1
-; GFX940-NEXT:    s_nop 1
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX940-NEXT:    ;;#ASMSTART
 ; GFX940-NEXT:    ; use v0
 ; GFX940-NEXT:    ;;#ASMEND
@@ -650,13 +456,7 @@ define void @s_minimum_f32(float inreg %src0, float inreg %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_min_f32_e64 v0, s4, s5
 ; GFX10-NEXT:    v_cmp_o_f32_e64 vcc_lo, s4, s5
-; GFX10-NEXT:    v_cmp_class_f32_e64 s6, s4, 32
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v0, s4, s6
-; GFX10-NEXT:    v_cmp_class_f32_e64 s4, s5, 32
-; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s5, s4
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; use v0
 ; GFX10-NEXT:    ;;#ASMEND
@@ -667,15 +467,8 @@ define void @s_minimum_f32(float inreg %src0, float inreg %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_min_f32_e64 v0, s0, s1
 ; GFX11-NEXT:    v_cmp_o_f32_e64 vcc_lo, s0, s1
-; GFX11-NEXT:    v_cmp_class_f32_e64 s2, s0, 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, 0x7fc00000, v0, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v0, s0, s2
-; GFX11-NEXT:    v_cmp_class_f32_e64 s0, s1, 32
-; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s1, s0
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use v0
 ; GFX11-NEXT:    ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
index 2387cd9bc7a9..7013c60bada5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
@@ -13,18 +13,9 @@ define double @v_minimum_f64(double %src0, double %src1) {
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_f64:
@@ -32,18 +23,9 @@ define double @v_minimum_f64(double %src0, double %src1) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f64:
@@ -51,39 +33,20 @@ define double @v_minimum_f64(double %src0, double %src1) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f64:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[2:3], 32
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f64:
@@ -91,17 +54,8 @@ define double @v_minimum_f64(double %src0, double %src1) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[4:5]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f64:
@@ -109,20 +63,9 @@ define double @v_minimum_f64(double %src0, double %src1) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s1, 0, v[4:5]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f64:
@@ -142,93 +85,37 @@ define double @v_minimum_f64__nnan(double %src0, double %src1) {
 ; GFX7-LABEL: v_minimum_f64__nnan:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
+; GFX7-NEXT:    v_min_f64 v[0:1], v[0:1], v[2:3]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_f64__nnan:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
+; GFX8-NEXT:    v_min_f64 v[0:1], v[0:1], v[2:3]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f64__nnan:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[2:3]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f64__nnan:
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX940-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[2:3], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
+; GFX940-NEXT:    v_min_f64 v[0:1], v[0:1], v[2:3]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f64__nnan:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[4:5]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s5
+; GFX10-NEXT:    v_min_f64 v[0:1], v[0:1], v[2:3]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f64__nnan:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s1, 0, v[4:5]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s1
+; GFX11-NEXT:    v_min_f64 v[0:1], v[0:1], v[2:3]
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f64__nnan:
@@ -373,60 +260,33 @@ define double @v_minimum_f64__nnan_src0(double %arg0, double %src1) {
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX7-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
 ; GFX7-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_f64__nnan_src0:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX8-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
 ; GFX8-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f64__nnan_src0:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
 ; GFX9-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f64__nnan_src0:
@@ -434,62 +294,33 @@ define double @v_minimum_f64__nnan_src0(double %arg0, double %src1) {
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
 ; GFX940-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[2:3], 32
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f64__nnan_src0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 32
 ; GFX10-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[4:5]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f64__nnan_src0:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_add_f64 v[0:1], v[0:1], 1.0
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s1, 0, v[4:5]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f64__nnan_src0:
@@ -513,60 +344,33 @@ define double @v_minimum_f64__nnan_src1(double %src0, double %arg1) {
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX7-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX7-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX7-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX7-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX7-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX7-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_f64__nnan_src1:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX8-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX8-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX8-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_minimum_f64__nnan_src1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX9-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], 32
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX9-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[6:7], 0, v[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX940-LABEL: v_minimum_f64__nnan_src1:
@@ -574,21 +378,11 @@ define double @v_minimum_f64__nnan_src1(double %src0, double %arg1) {
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX940-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
-; GFX940-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
 ; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[2:3]
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[0:1], v[2:3], 32
-; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v6, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
-; GFX940-NEXT:    v_cmp_class_f64_e64 vcc, v[0:1], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[2:3], 0, v[4:5]
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0x7ff80000
 ; GFX940-NEXT:    s_nop 0
-; GFX940-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
 ; GFX940-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
-; GFX940-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[2:3]
-; GFX940-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[2:3]
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_minimum_f64__nnan_src1:
@@ -597,39 +391,20 @@ define double @v_minimum_f64__nnan_src1(double %src0, double %arg1) {
 ; GFX10-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
 ; GFX10-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX10-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s4, v[2:3], 32
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX10-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s5, 0, v[4:5]
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_minimum_f64__nnan_src1:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_add_f64 v[2:3], v[2:3], 1.0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_min_f64 v[4:5], v[0:1], v[2:3]
 ; GFX11-NEXT:    v_cmp_u_f64_e32 vcc_lo, v[0:1], v[2:3]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s0, v[2:3], 32
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, 0x7ff80000, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc_lo
-; GFX11-NEXT:    v_cmp_class_f64_e64 vcc_lo, v[0:1], 32
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s1, 0, v[4:5]
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, 0x7ff80000, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: v_minimum_f64__nnan_src1:
@@ -654,30 +429,13 @@ define void @s_minimum_f64(double inreg %src0, double inreg %src1) {
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
 ; GFX7-NEXT:    v_min_f64 v[2:3], s[4:5], v[0:1]
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[8:9], s[4:5], 32
-; GFX7-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX7-NEXT:    v_readfirstlane_b32 s12, v3
-; GFX7-NEXT:    v_readfirstlane_b32 s10, v2
-; GFX7-NEXT:    s_cselect_b32 s11, 0x7ff80000, s12
-; GFX7-NEXT:    v_cmp_class_f64_e64 s[12:13], s[6:7], 32
-; GFX7-NEXT:    s_cselect_b32 s10, 0, s10
-; GFX7-NEXT:    v_cmp_eq_f64_e64 s[14:15], s[10:11], 0
-; GFX7-NEXT:    s_and_b64 s[16:17], s[8:9], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX7-NEXT:    s_and_b64 s[16:17], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX7-NEXT:    s_and_b64 s[16:17], s[14:15], exec
-; GFX7-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX7-NEXT:    s_and_b64 s[8:9], s[8:9], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX7-NEXT:    s_and_b64 s[8:9], s[12:13], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX7-NEXT:    s_and_b64 s[6:7], s[14:15], exec
-; GFX7-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
 ; GFX7-NEXT:    ;;#ASMSTART
-; GFX7-NEXT:    ; use s[4:5]
+; GFX7-NEXT:    ; use v[0:1]
 ; GFX7-NEXT:    ;;#ASMEND
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -686,30 +444,13 @@ define void @s_minimum_f64(double inreg %src0, double inreg %src1) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
 ; GFX8-NEXT:    v_min_f64 v[2:3], s[4:5], v[0:1]
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[8:9], s[4:5], 32
-; GFX8-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX8-NEXT:    v_readfirstlane_b32 s12, v3
-; GFX8-NEXT:    v_readfirstlane_b32 s10, v2
-; GFX8-NEXT:    s_cselect_b32 s11, 0x7ff80000, s12
-; GFX8-NEXT:    v_cmp_class_f64_e64 s[12:13], s[6:7], 32
-; GFX8-NEXT:    s_cselect_b32 s10, 0, s10
-; GFX8-NEXT:    v_cmp_eq_f64_e64 s[14:15], s[10:11], 0
-; GFX8-NEXT:    s_and_b64 s[16:17], s[8:9], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX8-NEXT:    s_and_b64 s[16:17], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX8-NEXT:    s_and_b64 s[16:17], s[14:15], exec
-; GFX8-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX8-NEXT:    s_and_b64 s[8:9], s[8:9], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX8-NEXT:    s_and_b64 s[8:9], s[12:13], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX8-NEXT:    s_and_b64 s[6:7], s[14:15], exec
-; GFX8-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
 ; GFX8-NEXT:    ;;#ASMSTART
-; GFX8-NEXT:    ; use s[4:5]
+; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -718,30 +459,13 @@ define void @s_minimum_f64(double inreg %src0, double inreg %src1) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
 ; GFX9-NEXT:    v_min_f64 v[2:3], s[4:5], v[0:1]
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[8:9], s[4:5], 32
-; GFX9-NEXT:    s_and_b64 s[10:11], vcc, exec
-; GFX9-NEXT:    v_readfirstlane_b32 s12, v3
-; GFX9-NEXT:    v_readfirstlane_b32 s10, v2
-; GFX9-NEXT:    s_cselect_b32 s11, 0x7ff80000, s12
-; GFX9-NEXT:    v_cmp_class_f64_e64 s[12:13], s[6:7], 32
-; GFX9-NEXT:    s_cselect_b32 s10, 0, s10
-; GFX9-NEXT:    v_cmp_eq_f64_e64 s[14:15], s[10:11], 0
-; GFX9-NEXT:    s_and_b64 s[16:17], s[8:9], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX9-NEXT:    s_and_b64 s[16:17], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX9-NEXT:    s_and_b64 s[16:17], s[14:15], exec
-; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
-; GFX9-NEXT:    s_and_b64 s[8:9], s[8:9], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
-; GFX9-NEXT:    s_and_b64 s[8:9], s[12:13], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX9-NEXT:    s_and_b64 s[6:7], s[14:15], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX9-NEXT:    v_cmp_u_f64_e32 vcc, s[4:5], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
 ; GFX9-NEXT:    ;;#ASMSTART
-; GFX9-NEXT:    ; use s[4:5]
+; GFX9-NEXT:    ; use v[0:1]
 ; GFX9-NEXT:    ;;#ASMEND
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -749,30 +473,14 @@ define void @s_minimum_f64(double inreg %src0, double inreg %src1) {
 ; GFX940:       ; %bb.0:
 ; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_min_f64 v[2:3], s[0:1], v[0:1]
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
 ; GFX940-NEXT:    v_cmp_u_f64_e32 vcc, s[0:1], v[0:1]
-; GFX940-NEXT:    v_min_f64 v[0:1], s[0:1], v[0:1]
-; GFX940-NEXT:    s_and_b64 s[4:5], vcc, exec
-; GFX940-NEXT:    v_readfirstlane_b32 s6, v1
-; GFX940-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[8:9], s[0:1], 32
-; GFX940-NEXT:    s_cselect_b32 s5, 0x7ff80000, s6
-; GFX940-NEXT:    s_cselect_b32 s4, 0, s4
-; GFX940-NEXT:    s_and_b64 s[10:11], s[8:9], exec
-; GFX940-NEXT:    v_cmp_class_f64_e64 s[10:11], s[2:3], 32
-; GFX940-NEXT:    v_cmp_eq_f64_e64 s[6:7], s[4:5], 0
-; GFX940-NEXT:    s_cselect_b32 s1, s1, s5
-; GFX940-NEXT:    s_and_b64 s[12:13], s[10:11], exec
-; GFX940-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX940-NEXT:    s_and_b64 s[12:13], s[6:7], exec
-; GFX940-NEXT:    s_cselect_b32 s1, s1, s5
-; GFX940-NEXT:    s_and_b64 s[8:9], s[8:9], exec
-; GFX940-NEXT:    s_cselect_b32 s0, s0, s4
-; GFX940-NEXT:    s_and_b64 s[8:9], s[10:11], exec
-; GFX940-NEXT:    s_cselect_b32 s0, s2, s0
-; GFX940-NEXT:    s_and_b64 s[2:3], s[6:7], exec
-; GFX940-NEXT:    s_cselect_b32 s0, s0, s4
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX940-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
 ; GFX940-NEXT:    ;;#ASMSTART
-; GFX940-NEXT:    ; use s[0:1]
+; GFX940-NEXT:    ; use v[0:1]
 ; GFX940-NEXT:    ;;#ASMEND
 ; GFX940-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -780,29 +488,11 @@ define void @s_minimum_f64(double inreg %src0, double inreg %src1) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_min_f64 v[0:1], s[4:5], s[6:7]
-; GFX10-NEXT:    v_cmp_u_f64_e64 s8, s[4:5], s[6:7]
-; GFX10-NEXT:    v_cmp_class_f64_e64 s11, s[4:5], 32
-; GFX10-NEXT:    v_cmp_class_f64_e64 s12, s[6:7], 32
-; GFX10-NEXT:    v_readfirstlane_b32 s9, v1
-; GFX10-NEXT:    v_readfirstlane_b32 s10, v0
-; GFX10-NEXT:    s_and_b32 s8, s8, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s9, 0x7ff80000, s9
-; GFX10-NEXT:    s_cselect_b32 s8, 0, s10
-; GFX10-NEXT:    s_and_b32 s13, s11, exec_lo
-; GFX10-NEXT:    v_cmp_eq_f64_e64 s10, s[8:9], 0
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s9
-; GFX10-NEXT:    s_and_b32 s13, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX10-NEXT:    s_and_b32 s7, s10, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s5, s5, s9
-; GFX10-NEXT:    s_and_b32 s7, s11, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s8
-; GFX10-NEXT:    s_and_b32 s7, s12, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s6, s4
-; GFX10-NEXT:    s_and_b32 s6, s10, exec_lo
-; GFX10-NEXT:    s_cselect_b32 s4, s4, s8
+; GFX10-NEXT:    v_cmp_u_f64_e64 s4, s[4:5], s[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x7ff80000, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s4
 ; GFX10-NEXT:    ;;#ASMSTART
-; GFX10-NEXT:    ; use s[4:5]
+; GFX10-NEXT:    ; use v[0:1]
 ; GFX10-NEXT:    ;;#ASMEND
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -810,32 +500,12 @@ define void @s_minimum_f64(double inreg %src0, double inreg %src1) {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_min_f64 v[0:1], s[0:1], s[2:3]
-; GFX11-NEXT:    v_cmp_u_f64_e64 s4, s[0:1], s[2:3]
-; GFX11-NEXT:    v_cmp_class_f64_e64 s7, s[0:1], 32
-; GFX11-NEXT:    v_cmp_class_f64_e64 s8, s[2:3], 32
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX11-NEXT:    v_readfirstlane_b32 s6, v0
-; GFX11-NEXT:    s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s5, 0x7ff80000, s5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    s_cselect_b32 s4, 0, s6
-; GFX11-NEXT:    s_and_b32 s9, s7, exec_lo
-; GFX11-NEXT:    v_cmp_eq_f64_e64 s6, s[4:5], 0
-; GFX11-NEXT:    s_cselect_b32 s1, s1, s5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    s_and_b32 s9, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s1, s3, s1
-; GFX11-NEXT:    s_and_b32 s3, s6, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s1, s1, s5
-; GFX11-NEXT:    s_and_b32 s3, s7, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s0, s4
-; GFX11-NEXT:    s_and_b32 s3, s8, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s2, s0
-; GFX11-NEXT:    s_and_b32 s2, s6, exec_lo
-; GFX11-NEXT:    s_cselect_b32 s0, s0, s4
+; GFX11-NEXT:    v_cmp_u_f64_e64 s0, s[0:1], s[2:3]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x7ff80000, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s0
 ; GFX11-NEXT:    ;;#ASMSTART
-; GFX11-NEXT:    ; use s[0:1]
+; GFX11-NEXT:    ; use v[0:1]
 ; GFX11-NEXT:    ;;#ASMEND
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/PowerPC/fminimum-fmaximum.ll b/llvm/test/CodeGen/PowerPC/fminimum-fmaximum.ll
index c33875dbfee4..a99c25a4e447 100644
--- a/llvm/test/CodeGen/PowerPC/fminimum-fmaximum.ll
+++ b/llvm/test/CodeGen/PowerPC/fminimum-fmaximum.ll
@@ -45,74 +45,26 @@ define float @f32_minimum(float %a, float %b) {
 ;
 ; VSX-LABEL: f32_minimum:
 ; VSX:       # %bb.0: # %entry
-; VSX-NEXT:    xscvdpspn 0, 1
 ; VSX-NEXT:    fcmpu 0, 1, 2
-; VSX-NEXT:    xscvdpspn 3, 2
-; VSX-NEXT:    mffprwz 3, 0
 ; VSX-NEXT:    bc 12, 3, .LBB0_2
 ; VSX-NEXT:  # %bb.1: # %entry
-; VSX-NEXT:    xsmindp 0, 1, 2
-; VSX-NEXT:    b .LBB0_3
+; VSX-NEXT:    xsmindp 1, 1, 2
+; VSX-NEXT:    blr
 ; VSX-NEXT:  .LBB0_2:
-; VSX-NEXT:    addis 4, 2, .LCPI0_0@toc@ha
-; VSX-NEXT:    lfs 0, .LCPI0_0@toc@l(4)
-; VSX-NEXT:  .LBB0_3: # %entry
-; VSX-NEXT:    xoris 3, 3, 32768
-; VSX-NEXT:    mffprwz 4, 3
-; VSX-NEXT:    cmplwi 3, 0
-; VSX-NEXT:    bc 12, 2, .LBB0_5
-; VSX-NEXT:  # %bb.4: # %entry
-; VSX-NEXT:    fmr 1, 0
-; VSX-NEXT:  .LBB0_5: # %entry
-; VSX-NEXT:    xoris 3, 4, 32768
-; VSX-NEXT:    cmplwi 3, 0
-; VSX-NEXT:    bc 12, 2, .LBB0_7
-; VSX-NEXT:  # %bb.6: # %entry
-; VSX-NEXT:    fmr 2, 1
-; VSX-NEXT:  .LBB0_7: # %entry
-; VSX-NEXT:    xxlxor 1, 1, 1
-; VSX-NEXT:    fcmpu 0, 0, 1
-; VSX-NEXT:    bc 12, 2, .LBB0_9
-; VSX-NEXT:  # %bb.8: # %entry
-; VSX-NEXT:    fmr 2, 0
-; VSX-NEXT:  .LBB0_9: # %entry
-; VSX-NEXT:    fmr 1, 2
+; VSX-NEXT:    addis 3, 2, .LCPI0_0@toc@ha
+; VSX-NEXT:    lfs 1, .LCPI0_0@toc@l(3)
 ; VSX-NEXT:    blr
 ;
 ; AIX-LABEL: f32_minimum:
 ; AIX:       # %bb.0: # %entry
-; AIX-NEXT:    xscvdpspn 0, 1
 ; AIX-NEXT:    fcmpu 0, 1, 2
-; AIX-NEXT:    xscvdpspn 3, 2
-; AIX-NEXT:    mffprwz 3, 0
 ; AIX-NEXT:    bc 12, 3, L..BB0_2
 ; AIX-NEXT:  # %bb.1: # %entry
-; AIX-NEXT:    xsmindp 0, 1, 2
-; AIX-NEXT:    b L..BB0_3
+; AIX-NEXT:    xsmindp 1, 1, 2
+; AIX-NEXT:    blr
 ; AIX-NEXT:  L..BB0_2:
-; AIX-NEXT:    ld 4, L..C0(2) # %const.0
-; AIX-NEXT:    lfs 0, 0(4)
-; AIX-NEXT:  L..BB0_3: # %entry
-; AIX-NEXT:    xoris 3, 3, 32768
-; AIX-NEXT:    mffprwz 4, 3
-; AIX-NEXT:    cmplwi 3, 0
-; AIX-NEXT:    bc 12, 2, L..BB0_5
-; AIX-NEXT:  # %bb.4: # %entry
-; AIX-NEXT:    fmr 1, 0
-; AIX-NEXT:  L..BB0_5: # %entry
-; AIX-NEXT:    xoris 3, 4, 32768
-; AIX-NEXT:    cmplwi 3, 0
-; AIX-NEXT:    bc 12, 2, L..BB0_7
-; AIX-NEXT:  # %bb.6: # %entry
-; AIX-NEXT:    fmr 2, 1
-; AIX-NEXT:  L..BB0_7: # %entry
-; AIX-NEXT:    xxlxor 1, 1, 1
-; AIX-NEXT:    fcmpu 0, 0, 1
-; AIX-NEXT:    bc 12, 2, L..BB0_9
-; AIX-NEXT:  # %bb.8: # %entry
-; AIX-NEXT:    fmr 2, 0
-; AIX-NEXT:  L..BB0_9: # %entry
-; AIX-NEXT:    fmr 1, 2
+; AIX-NEXT:    ld 3, L..C0(2) # %const.0
+; AIX-NEXT:    lfs 1, 0(3)
 ; AIX-NEXT:    blr
 entry:
   %m = call float @llvm.minimum.f32(float %a, float %b)
@@ -159,70 +111,26 @@ define float @f32_maximum(float %a, float %b) {
 ;
 ; VSX-LABEL: f32_maximum:
 ; VSX:       # %bb.0: # %entry
-; VSX-NEXT:    xscvdpspn 0, 1
 ; VSX-NEXT:    fcmpu 0, 1, 2
-; VSX-NEXT:    xscvdpspn 3, 2
-; VSX-NEXT:    mffprwz 3, 0
 ; VSX-NEXT:    bc 12, 3, .LBB1_2
 ; VSX-NEXT:  # %bb.1: # %entry
-; VSX-NEXT:    xsmaxdp 0, 1, 2
-; VSX-NEXT:    b .LBB1_3
+; VSX-NEXT:    xsmaxdp 1, 1, 2
+; VSX-NEXT:    blr
 ; VSX-NEXT:  .LBB1_2:
-; VSX-NEXT:    addis 4, 2, .LCPI1_0@toc@ha
-; VSX-NEXT:    lfs 0, .LCPI1_0@toc@l(4)
-; VSX-NEXT:  .LBB1_3: # %entry
-; VSX-NEXT:    mffprwz 4, 3
-; VSX-NEXT:    cmpwi 3, 0
-; VSX-NEXT:    bc 12, 2, .LBB1_5
-; VSX-NEXT:  # %bb.4: # %entry
-; VSX-NEXT:    fmr 1, 0
-; VSX-NEXT:  .LBB1_5: # %entry
-; VSX-NEXT:    cmpwi 4, 0
-; VSX-NEXT:    bc 12, 2, .LBB1_7
-; VSX-NEXT:  # %bb.6: # %entry
-; VSX-NEXT:    fmr 2, 1
-; VSX-NEXT:  .LBB1_7: # %entry
-; VSX-NEXT:    xxlxor 1, 1, 1
-; VSX-NEXT:    fcmpu 0, 0, 1
-; VSX-NEXT:    bc 12, 2, .LBB1_9
-; VSX-NEXT:  # %bb.8: # %entry
-; VSX-NEXT:    fmr 2, 0
-; VSX-NEXT:  .LBB1_9: # %entry
-; VSX-NEXT:    fmr 1, 2
+; VSX-NEXT:    addis 3, 2, .LCPI1_0@toc@ha
+; VSX-NEXT:    lfs 1, .LCPI1_0@toc@l(3)
 ; VSX-NEXT:    blr
 ;
 ; AIX-LABEL: f32_maximum:
 ; AIX:       # %bb.0: # %entry
-; AIX-NEXT:    xscvdpspn 0, 1
 ; AIX-NEXT:    fcmpu 0, 1, 2
-; AIX-NEXT:    xscvdpspn 3, 2
-; AIX-NEXT:    mffprwz 3, 0
 ; AIX-NEXT:    bc 12, 3, L..BB1_2
 ; AIX-NEXT:  # %bb.1: # %entry
-; AIX-NEXT:    xsmaxdp 0, 1, 2
-; AIX-NEXT:    b L..BB1_3
+; AIX-NEXT:    xsmaxdp 1, 1, 2
+; AIX-NEXT:    blr
 ; AIX-NEXT:  L..BB1_2:
-; AIX-NEXT:    ld 4, L..C1(2) # %const.0
-; AIX-NEXT:    lfs 0, 0(4)
-; AIX-NEXT:  L..BB1_3: # %entry
-; AIX-NEXT:    mffprwz 4, 3
-; AIX-NEXT:    cmpwi 3, 0
-; AIX-NEXT:    bc 12, 2, L..BB1_5
-; AIX-NEXT:  # %bb.4: # %entry
-; AIX-NEXT:    fmr 1, 0
-; AIX-NEXT:  L..BB1_5: # %entry
-; AIX-NEXT:    cmpwi 4, 0
-; AIX-NEXT:    bc 12, 2, L..BB1_7
-; AIX-NEXT:  # %bb.6: # %entry
-; AIX-NEXT:    fmr 2, 1
-; AIX-NEXT:  L..BB1_7: # %entry
-; AIX-NEXT:    xxlxor 1, 1, 1
-; AIX-NEXT:    fcmpu 0, 0, 1
-; AIX-NEXT:    bc 12, 2, L..BB1_9
-; AIX-NEXT:  # %bb.8: # %entry
-; AIX-NEXT:    fmr 2, 0
-; AIX-NEXT:  L..BB1_9: # %entry
-; AIX-NEXT:    fmr 1, 2
+; AIX-NEXT:    ld 3, L..C1(2) # %const.0
+; AIX-NEXT:    lfs 1, 0(3)
 ; AIX-NEXT:    blr
 entry:
   %m = call float @llvm.maximum.f32(float %a, float %b)
@@ -272,69 +180,25 @@ define double @f64_minimum(double %a, double %b) {
 ; VSX-LABEL: f64_minimum:
 ; VSX:       # %bb.0: # %entry
 ; VSX-NEXT:    fcmpu 0, 1, 2
-; VSX-NEXT:    mffprd 3, 1
 ; VSX-NEXT:    bc 12, 3, .LBB2_2
 ; VSX-NEXT:  # %bb.1: # %entry
-; VSX-NEXT:    xsmindp 0, 1, 2
-; VSX-NEXT:    b .LBB2_3
+; VSX-NEXT:    xsmindp 1, 1, 2
+; VSX-NEXT:    blr
 ; VSX-NEXT:  .LBB2_2:
-; VSX-NEXT:    addis 4, 2, .LCPI2_0@toc@ha
-; VSX-NEXT:    lfs 0, .LCPI2_0@toc@l(4)
-; VSX-NEXT:  .LBB2_3: # %entry
-; VSX-NEXT:    li 5, 1
-; VSX-NEXT:    mffprd 4, 2
-; VSX-NEXT:    rldic 5, 5, 63, 0
-; VSX-NEXT:    cmpd 3, 5
-; VSX-NEXT:    bc 12, 2, .LBB2_5
-; VSX-NEXT:  # %bb.4: # %entry
-; VSX-NEXT:    fmr 1, 0
-; VSX-NEXT:  .LBB2_5: # %entry
-; VSX-NEXT:    cmpd 4, 5
-; VSX-NEXT:    bc 12, 2, .LBB2_7
-; VSX-NEXT:  # %bb.6: # %entry
-; VSX-NEXT:    fmr 2, 1
-; VSX-NEXT:  .LBB2_7: # %entry
-; VSX-NEXT:    xxlxor 1, 1, 1
-; VSX-NEXT:    fcmpu 0, 0, 1
-; VSX-NEXT:    bc 12, 2, .LBB2_9
-; VSX-NEXT:  # %bb.8: # %entry
-; VSX-NEXT:    fmr 2, 0
-; VSX-NEXT:  .LBB2_9: # %entry
-; VSX-NEXT:    fmr 1, 2
+; VSX-NEXT:    addis 3, 2, .LCPI2_0@toc@ha
+; VSX-NEXT:    lfs 1, .LCPI2_0@toc@l(3)
 ; VSX-NEXT:    blr
 ;
 ; AIX-LABEL: f64_minimum:
 ; AIX:       # %bb.0: # %entry
 ; AIX-NEXT:    fcmpu 0, 1, 2
-; AIX-NEXT:    mffprd 3, 1
 ; AIX-NEXT:    bc 12, 3, L..BB2_2
 ; AIX-NEXT:  # %bb.1: # %entry
-; AIX-NEXT:    xsmindp 0, 1, 2
-; AIX-NEXT:    b L..BB2_3
+; AIX-NEXT:    xsmindp 1, 1, 2
+; AIX-NEXT:    blr
 ; AIX-NEXT:  L..BB2_2:
-; AIX-NEXT:    ld 4, L..C2(2) # %const.0
-; AIX-NEXT:    lfs 0, 0(4)
-; AIX-NEXT:  L..BB2_3: # %entry
-; AIX-NEXT:    li 5, 1
-; AIX-NEXT:    mffprd 4, 2
-; AIX-NEXT:    rldic 5, 5, 63, 0
-; AIX-NEXT:    cmpd 3, 5
-; AIX-NEXT:    bc 12, 2, L..BB2_5
-; AIX-NEXT:  # %bb.4: # %entry
-; AIX-NEXT:    fmr 1, 0
-; AIX-NEXT:  L..BB2_5: # %entry
-; AIX-NEXT:    cmpd 4, 5
-; AIX-NEXT:    bc 12, 2, L..BB2_7
-; AIX-NEXT:  # %bb.6: # %entry
-; AIX-NEXT:    fmr 2, 1
-; AIX-NEXT:  L..BB2_7: # %entry
-; AIX-NEXT:    xxlxor 1, 1, 1
-; AIX-NEXT:    fcmpu 0, 0, 1
-; AIX-NEXT:    bc 12, 2, L..BB2_9
-; AIX-NEXT:  # %bb.8: # %entry
-; AIX-NEXT:    fmr 2, 0
-; AIX-NEXT:  L..BB2_9: # %entry
-; AIX-NEXT:    fmr 1, 2
+; AIX-NEXT:    ld 3, L..C2(2) # %const.0
+; AIX-NEXT:    lfs 1, 0(3)
 ; AIX-NEXT:    blr
 entry:
   %m = call double @llvm.minimum.f64(double %a, double %b)
@@ -382,65 +246,25 @@ define double @f64_maximum(double %a, double %b) {
 ; VSX-LABEL: f64_maximum:
 ; VSX:       # %bb.0: # %entry
 ; VSX-NEXT:    fcmpu 0, 1, 2
-; VSX-NEXT:    mffprd 3, 1
 ; VSX-NEXT:    bc 12, 3, .LBB3_2
 ; VSX-NEXT:  # %bb.1: # %entry
-; VSX-NEXT:    xsmaxdp 0, 1, 2
-; VSX-NEXT:    b .LBB3_3
+; VSX-NEXT:    xsmaxdp 1, 1, 2
+; VSX-NEXT:    blr
 ; VSX-NEXT:  .LBB3_2:
-; VSX-NEXT:    addis 4, 2, .LCPI3_0@toc@ha
-; VSX-NEXT:    lfs 0, .LCPI3_0@toc@l(4)
-; VSX-NEXT:  .LBB3_3: # %entry
-; VSX-NEXT:    mffprd 4, 2
-; VSX-NEXT:    cmpdi 3, 0
-; VSX-NEXT:    bc 12, 2, .LBB3_5
-; VSX-NEXT:  # %bb.4: # %entry
-; VSX-NEXT:    fmr 1, 0
-; VSX-NEXT:  .LBB3_5: # %entry
-; VSX-NEXT:    cmpdi 4, 0
-; VSX-NEXT:    bc 12, 2, .LBB3_7
-; VSX-NEXT:  # %bb.6: # %entry
-; VSX-NEXT:    fmr 2, 1
-; VSX-NEXT:  .LBB3_7: # %entry
-; VSX-NEXT:    xxlxor 1, 1, 1
-; VSX-NEXT:    fcmpu 0, 0, 1
-; VSX-NEXT:    bc 12, 2, .LBB3_9
-; VSX-NEXT:  # %bb.8: # %entry
-; VSX-NEXT:    fmr 2, 0
-; VSX-NEXT:  .LBB3_9: # %entry
-; VSX-NEXT:    fmr 1, 2
+; VSX-NEXT:    addis 3, 2, .LCPI3_0@toc@ha
+; VSX-NEXT:    lfs 1, .LCPI3_0@toc@l(3)
 ; VSX-NEXT:    blr
 ;
 ; AIX-LABEL: f64_maximum:
 ; AIX:       # %bb.0: # %entry
 ; AIX-NEXT:    fcmpu 0, 1, 2
-; AIX-NEXT:    mffprd 3, 1
 ; AIX-NEXT:    bc 12, 3, L..BB3_2
 ; AIX-NEXT:  # %bb.1: # %entry
-; AIX-NEXT:    xsmaxdp 0, 1, 2
-; AIX-NEXT:    b L..BB3_3
+; AIX-NEXT:    xsmaxdp 1, 1, 2
+; AIX-NEXT:    blr
 ; AIX-NEXT:  L..BB3_2:
-; AIX-NEXT:    ld 4, L..C3(2) # %const.0
-; AIX-NEXT:    lfs 0, 0(4)
-; AIX-NEXT:  L..BB3_3: # %entry
-; AIX-NEXT:    mffprd 4, 2
-; AIX-NEXT:    cmpdi 3, 0
-; AIX-NEXT:    bc 12, 2, L..BB3_5
-; AIX-NEXT:  # %bb.4: # %entry
-; AIX-NEXT:    fmr 1, 0
-; AIX-NEXT:  L..BB3_5: # %entry
-; AIX-NEXT:    cmpdi 4, 0
-; AIX-NEXT:    bc 12, 2, L..BB3_7
-; AIX-NEXT:  # %bb.6: # %entry
-; AIX-NEXT:    fmr 2, 1
-; AIX-NEXT:  L..BB3_7: # %entry
-; AIX-NEXT:    xxlxor 1, 1, 1
-; AIX-NEXT:    fcmpu 0, 0, 1
-; AIX-NEXT:    bc 12, 2, L..BB3_9
-; AIX-NEXT:  # %bb.8: # %entry
-; AIX-NEXT:    fmr 2, 0
-; AIX-NEXT:  L..BB3_9: # %entry
-; AIX-NEXT:    fmr 1, 2
+; AIX-NEXT:    ld 3, L..C3(2) # %const.0
+; AIX-NEXT:    lfs 1, 0(3)
 ; AIX-NEXT:    blr
 entry:
   %m = call double @llvm.maximum.f64(double %a, double %b)
-- 
GitLab


From e2d17a053edff68c9761aceb8ff8303e3e37172c Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Thu, 9 May 2024 13:42:40 +0100
Subject: [PATCH 0283/1206] [AMDGPU] Build lane intrinsics in a
 mangling-agnostic way. NFC. (#91583)

Use the form of CreateIntrinsic that takes an explicit return type and
works out the mangling based on that and the types of the arguments. The
advantage is that this still works if intrinsics are changed to have
type mangling, e.g. if readlane/readfirstlane/writelane are changed to
work on any type.
---
 .../Target/AMDGPU/AMDGPUAtomicOptimizer.cpp   | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index ad98f4f743ae..1d645002b1fe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -493,8 +493,8 @@ Value *AMDGPUAtomicOptimizerImpl::buildScan(IRBuilder<> &B,
     if (!ST->isWave32()) {
       // Combine lane 31 into lanes 32..63.
       V = B.CreateBitCast(V, IntNTy);
-      Value *const Lane31 = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {},
-                                              {V, B.getInt32(31)});
+      Value *const Lane31 = B.CreateIntrinsic(
+          V->getType(), Intrinsic::amdgcn_readlane, {V, B.getInt32(31)});
 
       Value *UpdateDPPCall = B.CreateCall(
           UpdateDPP, {Identity, Lane31, B.getInt32(DPP::QUAD_PERM_ID),
@@ -598,8 +598,8 @@ std::pair<Value *, Value *> AMDGPUAtomicOptimizerImpl::buildScanIteratively(
 
   // Get the value required for atomic operation
   V = B.CreateBitCast(V, IntNTy);
-  Value *LaneValue =
-      B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {}, {V, LaneIdxInt});
+  Value *LaneValue = B.CreateIntrinsic(V->getType(), Intrinsic::amdgcn_readlane,
+                                       {V, LaneIdxInt});
   LaneValue = B.CreateBitCast(LaneValue, Ty);
 
   // Perform writelane if intermediate scan results are required later in the
@@ -607,7 +607,7 @@ std::pair<Value *, Value *> AMDGPUAtomicOptimizerImpl::buildScanIteratively(
   Value *OldValue = nullptr;
   if (NeedResult) {
     OldValue =
-        B.CreateIntrinsic(Intrinsic::amdgcn_writelane, {},
+        B.CreateIntrinsic(IntNTy, Intrinsic::amdgcn_writelane,
                           {B.CreateBitCast(Accumulator, IntNTy), LaneIdxInt,
                            B.CreateBitCast(OldValuePhi, IntNTy)});
     OldValue = B.CreateBitCast(OldValue, Ty);
@@ -789,7 +789,7 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
         Value *const LastLaneIdx = B.getInt32(ST->getWavefrontSize() - 1);
         assert(TyBitWidth == 32);
         NewV = B.CreateBitCast(NewV, IntNTy);
-        NewV = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {},
+        NewV = B.CreateIntrinsic(IntNTy, Intrinsic::amdgcn_readlane,
                                  {NewV, LastLaneIdx});
         NewV = B.CreateBitCast(NewV, Ty);
       }
@@ -936,10 +936,10 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
       Value *const ExtractLo = B.CreateTrunc(CastedPhi, Int32Ty);
       Value *const ExtractHi =
           B.CreateTrunc(B.CreateLShr(CastedPhi, 32), Int32Ty);
-      CallInst *const ReadFirstLaneLo =
-          B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractLo);
-      CallInst *const ReadFirstLaneHi =
-          B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractHi);
+      CallInst *const ReadFirstLaneLo = B.CreateIntrinsic(
+          Int32Ty, Intrinsic::amdgcn_readfirstlane, ExtractLo);
+      CallInst *const ReadFirstLaneHi = B.CreateIntrinsic(
+          Int32Ty, Intrinsic::amdgcn_readfirstlane, ExtractHi);
       Value *const PartialInsert = B.CreateInsertElement(
           PoisonValue::get(VecTy), ReadFirstLaneLo, B.getInt32(0));
       Value *const Insert =
@@ -948,7 +948,7 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
     } else if (TyBitWidth == 32) {
       Value *CastedPhi = B.CreateBitCast(PHI, IntNTy);
       BroadcastI =
-          B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, CastedPhi);
+          B.CreateIntrinsic(IntNTy, Intrinsic::amdgcn_readfirstlane, CastedPhi);
       BroadcastI = B.CreateBitCast(BroadcastI, Ty);
 
     } else {
-- 
GitLab


From d98e3d43e7943f0440013c9f491f323bcc864aa3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Danny=20M=C3=B6sch?= <danny.moesch@icloud.com>
Date: Thu, 9 May 2024 14:52:03 +0200
Subject: [PATCH 0284/1206] [BOLT][NFC] Apply absorption rule to boolean
 expression (#91540)

Fixes #91197.
---
 bolt/include/bolt/Passes/IndirectCallPromotion.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bolt/include/bolt/Passes/IndirectCallPromotion.h b/bolt/include/bolt/Passes/IndirectCallPromotion.h
index adc58d70ec0f..8ec160b867cf 100644
--- a/bolt/include/bolt/Passes/IndirectCallPromotion.h
+++ b/bolt/include/bolt/Passes/IndirectCallPromotion.h
@@ -104,7 +104,7 @@ class IndirectCallPromotion : public BinaryFunctionPass {
   struct Location {
     MCSymbol *Sym{nullptr};
     uint64_t Addr{0};
-    bool isValid() const { return Sym || (!Sym && Addr != 0); }
+    bool isValid() const { return Sym || Addr != 0; }
     Location() {}
     explicit Location(MCSymbol *Sym) : Sym(Sym) {}
     explicit Location(uint64_t Addr) : Addr(Addr) {}
-- 
GitLab


From ed3a60c796c2ed3c9fc59efc83856913ab5860f5 Mon Sep 17 00:00:00 2001
From: Hongbin Jin <octopus.busts_0w@icloud.com>
Date: Thu, 9 May 2024 21:18:10 +0800
Subject: [PATCH 0285/1206] [RISCV][GlobalISel] Fix selectShiftMask when shift
 mask is created from G_AND (#89602)

This patch fixes cases where G_AND creating the shift mask is eliminated
if one of its source operands is a constant, resulting from an incorrect
predicate.
---
 .../RISCV/GISel/RISCVInstructionSelector.cpp  | 16 ++++-
 .../instruction-select/shift-rv32.mir         | 56 +++++++++++++++++
 .../instruction-select/shift-rv64.mir         | 60 +++++++++++++++++++
 3 files changed, 131 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 3103992a86c0..791d364655e5 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -177,6 +177,20 @@ RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
 
   APInt AndMask;
   Register AndSrcReg;
+  // Try to combine the following pattern (applicable to other shift
+  // instructions as well as 32-bit ones):
+  //
+  //   %4:gprb(s64) = G_AND %3, %2
+  //   %5:gprb(s64) = G_LSHR %1, %4(s64)
+  //
+  // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
+  // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
+  // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
+  // then it can be eliminated. Given register rs1 or rs2 holding a constant
+  // (the and mask), there are two cases G_AND can be erased:
+  //
+  // 1. the lowest log2(XLEN) bits of the and mask are all set
+  // 2. the bits of the register being masked are already unset (zero set)
   if (mi_match(ShAmtReg, MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
     APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
     if (ShMask.isSubsetOf(AndMask)) {
@@ -184,7 +198,7 @@ RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
     } else {
       // SimplifyDemandedBits may have optimized the mask so try restoring any
       // bits that are known zero.
-      KnownBits Known = KB->getKnownBits(ShAmtReg);
+      KnownBits Known = KB->getKnownBits(AndSrcReg);
       if (ShMask.isSubsetOf(AndMask | Known.Zero))
         ShAmtReg = AndSrcReg;
     }
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/shift-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/shift-rv32.mir
index 7d6c228c8086..4d0b5c2a2c86 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/shift-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/shift-rv32.mir
@@ -188,3 +188,59 @@ body:             |
     $x10 = COPY %4(s32)
     PseudoRET implicit $x10
 ...
+
+---
+name:            srl_and_needed
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: srl_and_needed
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[ANDI:%[0-9]+]]:gpr = ANDI [[COPY]], 15
+    ; CHECK-NEXT: [[SRL:%[0-9]+]]:gpr = SRL [[COPY1]], [[ANDI]]
+    ; CHECK-NEXT: $x10 = COPY [[SRL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = COPY $x10
+    %1:gprb(s32) = COPY $x11
+    %2:gprb(s32) = G_CONSTANT i32 15
+    %3:gprb(s32) = G_AND %0, %2
+    %4:gprb(s32) = G_LSHR %1, %3(s32)
+    $x10 = COPY %4(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            srl_and_eliminated
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: srl_and_eliminated
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[ANDI:%[0-9]+]]:gpr = ANDI [[COPY]], 47
+    ; CHECK-NEXT: [[SRL:%[0-9]+]]:gpr = SRL [[COPY1]], [[ANDI]]
+    ; CHECK-NEXT: $x10 = COPY [[SRL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = COPY $x10
+    %1:gprb(s32) = COPY $x11
+    %2:gprb(s32) = G_CONSTANT i32 15
+    %3:gprb(s32) = G_CONSTANT i32 47
+    %4:gprb(s32) = G_AND %0, %3
+    %5:gprb(s32) = G_AND %4, %2
+    %6:gprb(s32) = G_LSHR %1, %5(s32)
+    $x10 = COPY %6(s32)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/shift-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/shift-rv64.mir
index 1e6890098498..5e2c60323fcb 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/shift-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/shift-rv64.mir
@@ -241,3 +241,63 @@ body:             |
     $x10 = COPY %6(s64)
     PseudoRET implicit $x10
 ...
+
+---
+name:            srl_and_needed
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: srl_and_needed
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[ANDI:%[0-9]+]]:gpr = ANDI [[COPY]], 15
+    ; CHECK-NEXT: [[SRL:%[0-9]+]]:gpr = SRL [[COPY1]], [[ANDI]]
+    ; CHECK-NEXT: $x10 = COPY [[SRL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s64) = COPY $x11
+    %2:gprb(s32) = G_CONSTANT i32 15
+    %3:gprb(s32) = G_TRUNC %0(s64)
+    %4:gprb(s32) = G_AND %3, %2
+    %5:gprb(s64) = nneg G_ZEXT %4(s32)
+    %6:gprb(s64) = G_LSHR %1, %5(s64)
+    $x10 = COPY %6(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            srl_and_eliminated
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: srl_and_eliminated
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[ANDI:%[0-9]+]]:gpr = ANDI [[COPY]], 79
+    ; CHECK-NEXT: [[SRL:%[0-9]+]]:gpr = SRL [[COPY1]], [[ANDI]]
+    ; CHECK-NEXT: $x10 = COPY [[SRL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s64) = COPY $x11
+    %2:gprb(s32) = G_CONSTANT i32 15
+    %3:gprb(s32) = G_TRUNC %0(s64)
+    %7:gprb(s32) = G_CONSTANT i32 79
+    %8:gprb(s32) = G_AND %3, %7
+    %4:gprb(s32) = G_AND %8, %2
+    %5:gprb(s64) = nneg G_ZEXT %4(s32)
+    %6:gprb(s64) = G_LSHR %1, %5(s64)
+    $x10 = COPY %6(s64)
+    PseudoRET implicit $x10
+...
-- 
GitLab


From 0c0fc9a7c6e298269871b4f1bf5fea7fa9048209 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Thu, 9 May 2024 09:02:30 -0400
Subject: [PATCH 0286/1206] [gn build] Port d86b68afd7f0 (AMDGPUMCTests
 dependency mess)

---
 .../secondary/llvm/lib/Target/AMDGPU/BUILD.gn |  1 +
 .../llvm/lib/Target/AMDGPU/Utils/BUILD.gn     |  5 ++++-
 .../gn/secondary/llvm/unittests/BUILD.gn      |  2 +-
 .../llvm/unittests/MC/AMDGPU/BUILD.gn         | 20 +++++++++++++++++--
 4 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index e1b867bf70ba..edd8d4f1840d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -61,6 +61,7 @@ tablegen("AMDGPUGenRegisterBank") {
   visibility = [
     ":LLVMAMDGPUCodeGen",
     "Utils",
+    "//llvm/unittests/MC/AMDGPU:AMDGPUMCTests",
     "//llvm/unittests/Target/AMDGPU:AMDGPUTests",
   ]
   args = [ "-gen-register-bank" ]
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Utils/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Utils/BUILD.gn
index 631d1ef5c7b0..ec0d5fc767f7 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Utils/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Utils/BUILD.gn
@@ -1,7 +1,10 @@
 import("//llvm/utils/TableGen/tablegen.gni")
 
 tablegen("AMDGPUGenSearchableTables") {
-  visibility = [ ":Utils" ]
+  visibility = [
+    ":Utils",
+    "//llvm/unittests/MC/AMDGPU:AMDGPUMCTests",
+  ]
   args = [ "-gen-searchable-tables" ]
   td_file = "../AMDGPU.td"
 }
diff --git a/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
index 6cc3848c1114..2db5b9603f21 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
@@ -77,7 +77,7 @@ group("unittests") {
   }
   if (llvm_build_AMDGPU) {
     deps += [
-      "MC/AMDGPU:AMDGPUDwarfTests",
+      "MC/AMDGPU:AMDGPUMCTests",
       "Target/AMDGPU:AMDGPUTests",
     ]
   }
diff --git a/llvm/utils/gn/secondary/llvm/unittests/MC/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/MC/AMDGPU/BUILD.gn
index 603753abec35..4a7f829d6d8e 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/MC/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/MC/AMDGPU/BUILD.gn
@@ -1,13 +1,29 @@
 import("//third-party/unittest/unittest.gni")
 
-unittest("AMDGPUDwarfTests") {
+unittest("AMDGPUMCTests") {
   deps = [
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/IR",
     "//llvm/lib/MC",
     "//llvm/lib/Support",
     "//llvm/lib/Target/AMDGPU:LLVMAMDGPUCodeGen",
     "//llvm/lib/Target/AMDGPU/MCTargetDesc",
     "//llvm/lib/Target/AMDGPU/TargetInfo",
     "//llvm/lib/TargetParser",
+
+    # SIProgramInfoMCExprs.cpp includes AMDGPUTargetMachine.h, which includes
+    # the generated AMDGPUGenRegisterBank.inc file :/
+    "//llvm/lib/Target/AMDGPU:AMDGPUGenRegisterBank",
+
+    # SIProgramInfoMCExprs.cpp includes AMDGPUTargetMachine.h, which includes
+    # the generated AMDGPUGenSearchableTables.inc file :/
+    "//llvm/lib/Target/AMDGPU/Utils:AMDGPUGenSearchableTables",
+  ]
+
+  # AMDGPUMCTests heavily reaches into lib/Target/AMDGPU internals.
+  include_dirs = [ "//llvm/lib/Target/AMDGPU" ]
+  sources = [
+    "DwarfRegMappings.cpp",
+    "SIProgramInfoMCExprs.cpp",
   ]
-  sources = [ "DwarfRegMappings.cpp" ]
 }
-- 
GitLab


From c73516af10f800d0d5641651eefce128b866a155 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Thu, 9 May 2024 23:18:19 +1000
Subject: [PATCH 0287/1206] [ORC] Add tests for error handling paths in
 suspended generators.

Test that (1) errors returned from a manually suspended generator are
propagated as expected, and (2) automatic suspension does not interfere with
our ability to resume (and return errors from) a generator.
---
 .../ExecutionEngine/Orc/CoreAPIsTest.cpp      | 65 +++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
index 3b24e29e1ed3..a6fa69f97fcb 100644
--- a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
@@ -1132,6 +1132,71 @@ TEST_F(CoreAPIsStandardTest, SimpleAsynchronousGeneratorTest) {
   EXPECT_TRUE(LookupCompleted);
 }
 
+TEST_F(CoreAPIsStandardTest, ErrorFromSuspendedAsynchronousGeneratorTest) {
+
+  auto &G = JD.addGenerator(std::make_unique<SimpleAsyncGenerator>());
+
+  bool LookupCompleted = false;
+
+  ES.lookup(
+      LookupKind::Static, makeJITDylibSearchOrder(&JD), SymbolLookupSet(Foo),
+      SymbolState::Ready,
+      [&](Expected<SymbolMap> Result) {
+        LookupCompleted = true;
+        EXPECT_THAT_EXPECTED(Result, Failed());
+      },
+      NoDependenciesToRegister);
+
+  EXPECT_FALSE(LookupCompleted);
+
+  G.takeLookup().LS.continueLookup(
+      make_error<StringError>("boom", inconvertibleErrorCode()));
+
+  EXPECT_TRUE(LookupCompleted);
+}
+
+TEST_F(CoreAPIsStandardTest, ErrorFromAutoSuspendedAsynchronousGeneratorTest) {
+
+  auto &G = JD.addGenerator(std::make_unique<SimpleAsyncGenerator>());
+
+  std::atomic_size_t LookupsCompleted = 0;
+
+  ES.lookup(
+      LookupKind::Static, makeJITDylibSearchOrder(&JD), SymbolLookupSet(Foo),
+      SymbolState::Ready,
+      [&](Expected<SymbolMap> Result) {
+        ++LookupsCompleted;
+        EXPECT_THAT_EXPECTED(Result, Failed());
+      },
+      NoDependenciesToRegister);
+
+  EXPECT_EQ(LookupsCompleted, 0);
+
+  // Suspend the first lookup.
+  auto LS1 = std::move(G.takeLookup().LS);
+
+  // Start a second lookup that should be auto-suspended.
+  ES.lookup(
+      LookupKind::Static, makeJITDylibSearchOrder(&JD), SymbolLookupSet(Foo),
+      SymbolState::Ready,
+      [&](Expected<SymbolMap> Result) {
+        ++LookupsCompleted;
+        EXPECT_THAT_EXPECTED(Result, Failed());
+      },
+      NoDependenciesToRegister);
+
+  EXPECT_EQ(LookupsCompleted, 0);
+
+  // Unsuspend the first lookup.
+  LS1.continueLookup(make_error<StringError>("boom", inconvertibleErrorCode()));
+
+  // Unsuspend the second.
+  G.takeLookup().LS.continueLookup(
+      make_error<StringError>("boom", inconvertibleErrorCode()));
+
+  EXPECT_EQ(LookupsCompleted, 2);
+}
+
 TEST_F(CoreAPIsStandardTest, BlockedGeneratorAutoSuspensionTest) {
   // Test that repeated lookups while a generator is in use cause automatic
   // lookup suspension / resumption.
-- 
GitLab


From cbf1535cc813b2f226498d974c43675832abc233 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Thu, 9 May 2024 23:20:31 +1000
Subject: [PATCH 0288/1206] [ORC] Fix fall-through in error case in
 EPCGenericDylibManager::lookupAsync.

In the event of a serialization error (e.g. due to a network dropout) we should
only run the Complete handler once, passing the serialization error value.

No test-case: This would require a deliberately injected failure in a
remote-JIT test and we don't have the infrastructure for that at the moment.

rdar://126772381
---
 llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp b/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp
index 6a7cab4a5510..7c0d89012922 100644
--- a/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp
@@ -109,6 +109,7 @@ void EPCGenericDylibManager::lookupAsync(tpctypes::DylibHandle H,
         if (SerializationErr) {
           cantFail(Result.takeError());
           Complete(std::move(SerializationErr));
+          return;
         }
         Complete(std::move(Result));
       },
-- 
GitLab


From e88ba6d975d887ca001cae30bfa0c53d91165148 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Thu, 9 May 2024 14:30:53 +0100
Subject: [PATCH 0289/1206] [AArch64] Add intrinsics for multi-vector to ZA
 array vector accumulators (#88266)

According to the specification in
https://github.com/ARM-software/acle/pull/309 this adds the intrinsics

void_svadd_za16_vg1x2_f16(uint32_t slice, svfloat16x2_t zn)
__arm_streaming __arm_inout("za");
void_svadd_za16_vg1x4_f16(uint32_t slice, svfloat16x4_t zn)
__arm_streaming __arm_inout("za");
void_svsub_za16_vg1x2_f16(uint32_t slice, svfloat16x2_t zn)
__arm_streaming __arm_inout("za");
void_svsub_za16_vg1x4_f16(uint32_t slice, svfloat16x4_t zn)
__arm_streaming __arm_inout("za");

as well as the corresponding `bf16` variants.
---
 clang/include/clang/Basic/arm_sme.td          |  10 +
 .../acle_sme2_add_sub_za16.c                  | 193 ++++++++++++++++++
 .../acle_sme2_add_sub_za16.c                  |  29 +++
 llvm/include/llvm/IR/IntrinsicsAArch64.td     |   2 +-
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |  16 +-
 .../AArch64/sme2-intrinsics-add-sub-za16.ll   | 148 ++++++++++++++
 6 files changed, 389 insertions(+), 9 deletions(-)
 create mode 100644 clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
 create mode 100644 clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
 create mode 100644 llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll

diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index 1ac6d5170ea2..000bd97a4b25 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -298,6 +298,16 @@ multiclass ZAAddSub<string n_suffix> {
     def NAME # _ZA64_VG1X2_F64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x2", "vm2", "d", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x2", [IsStreaming, IsInOutZA], []>;
     def NAME # _ZA64_VG1X4_F64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x4", "vm4", "d", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x4", [IsStreaming, IsInOutZA], []>;
   }
+
+  let TargetGuard = "sme-f16f16|sme-f8f16" in {
+    def NAME # _ZA16_VG1X2_F16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x2", "vm2", "h", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x2", [IsStreaming, IsInOutZA], []>;
+    def NAME # _ZA16_VG1X4_F16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x4", "vm4", "h", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x4", [IsStreaming, IsInOutZA], []>;
+  }
+
+  let TargetGuard = "sme2,b16b16" in {
+    def NAME # _ZA16_VG1X2_BF16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x2", "vm2", "b", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x2", [IsStreaming, IsInOutZA], []>;
+    def NAME # _ZA16_VG1X4_BF16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x4", "vm4", "b", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x4", [IsStreaming, IsInOutZA], []>;
+  }
 }
 
 defm SVADD : ZAAddSub<"add">;
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
new file mode 100644
index 000000000000..d98427fac610
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
@@ -0,0 +1,193 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
+// RUN: %clang_cc1                               -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -S -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1                        -x c++ -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature  +sme-f8f16 -target-feature +b16b16 -O2 -S -Werror -Wall -emit-llvm -o - %s | FileCheck %s -check-prefix CHECK-CXX
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS        -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature  +sme-f8f16 -target-feature +b16b16 -O2 -S -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -x c++ -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -S -Werror -Wall -emit-llvm -o - %s | FileCheck %s -check-prefix CHECK-CXX
+
+// RUN: %clang_cc1                               -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -S -Werror -Wall -o /dev/null
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3) A1##A2##A3
+#endif
+
+// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x2_f16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svadd_za16_vg1x2_f16j13svfloat16x2_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svadd_za16_vg1x2_f16(uint32_t slice, svfloat16x2_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svadd_za16,_f16,_vg1x2)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x4_f16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svadd_za16_vg1x4_f16j13svfloat16x4_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svadd_za16_vg1x4_f16(uint32_t slice, svfloat16x4_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svadd_za16,_f16,_vg1x4)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x2_f16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svsub_za16_vg1x2_f16j13svfloat16x2_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svsub_za16_vg1x2_f16(uint32_t slice, svfloat16x2_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svsub_za16,_f16,_vg1x2)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x4_f16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svsub_za16_vg1x4_f16j13svfloat16x4_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svsub_za16_vg1x4_f16(uint32_t slice, svfloat16x4_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svsub_za16,_f16,_vg1x4)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x2_bf16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svadd_za16_vg1x2_bf16j14svbfloat16x2_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svadd_za16_vg1x2_bf16(uint32_t slice, svbfloat16x2_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svadd_za16,_bf16,_vg1x2)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x4_bf16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svadd_za16_vg1x4_bf16j14svbfloat16x4_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svadd_za16_vg1x4_bf16(uint32_t slice, svbfloat16x4_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svadd_za16,_bf16,_vg1x4)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x2_bf16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svsub_za16_vg1x2_bf16j14svbfloat16x2_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svsub_za16_vg1x2_bf16(uint32_t slice, svbfloat16x2_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svsub_za16,_bf16,_vg1x2)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x4_bf16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svsub_za16_vg1x4_bf16j14svbfloat16x4_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svsub_za16_vg1x4_bf16(uint32_t slice, svbfloat16x4_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svsub_za16,_bf16,_vg1x4)(slice, zn);
+}
diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
new file mode 100644
index 000000000000..0eeeac5a5046
--- /dev/null
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -fsyntax-only -verify -emit-llvm %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+void test_features(uint32_t slice, svfloat16x2_t zn2, svfloat16x4_t zn4,
+                   svbfloat16x2_t bzn2, svbfloat16x4_t bzn4) __arm_streaming __arm_inout("za") {
+  // expected-error@+1 {{'svadd_za16_f16_vg1x2' needs target feature sme-f16f16|sme-f8f16}}
+  svadd_za16_f16_vg1x2(slice, zn2);
+  // expected-error@+1 {{'svadd_za16_f16_vg1x4' needs target feature sme-f16f16|sme-f8f16}}
+  svadd_za16_f16_vg1x4(slice, zn4);
+  // expected-error@+1 {{'svsub_za16_f16_vg1x2' needs target feature sme-f16f16|sme-f8f16}}
+  svsub_za16_f16_vg1x2(slice, zn2);
+  // expected-error@+1 {{'svsub_za16_f16_vg1x4' needs target feature sme-f16f16|sme-f8f16}}
+  svsub_za16_f16_vg1x4(slice, zn4);
+
+  // expected-error@+1 {{'svadd_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
+  svadd_za16_bf16_vg1x2(slice, bzn2);
+  // expected-error@+1 {{'svadd_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
+  svadd_za16_bf16_vg1x4(slice, bzn4);
+  // expected-error@+1 {{'svsub_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
+  svsub_za16_bf16_vg1x2(slice, bzn2);
+  // expected-error@+1 {{'svsub_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
+  svsub_za16_bf16_vg1x4(slice, bzn4);
+}
+
+
+
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index e31e00a9c76f..04571faf1306 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -3481,7 +3481,7 @@ let TargetPrefix = "aarch64" in {
   // Multi-vector add/sub and accumulate into ZA
   //
   foreach intr = ["add", "sub"] in {
-    foreach za = ["za32", "za64"] in {
+    foreach za = ["za16","za32", "za64"] in {
       def int_aarch64_sme_ # intr # _ # za # _vg1x2 : SME2_ZA_Write_VG2_Intrinsic;
       def int_aarch64_sme_ # intr # _ # za # _vg1x4 : SME2_ZA_Write_VG4_Intrinsic;
     }
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index 574178c8d524..5102c602d54e 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -793,10 +793,10 @@ defm LUTI4_S_4ZTZI : sme2p1_luti4_vector_vg4_index<"luti4">;
 }
 
 let Predicates = [HasSMEF16F16orSMEF8F16] in {
-defm FADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fadd", 0b0100, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
-defm FADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fadd", 0b0100, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
-defm FSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fsub", 0b0101, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
-defm FSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fsub", 0b0101, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
+defm FADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fadd", 0b0100, MatrixOp16, ZZ_h_mul_r, nxv8f16, int_aarch64_sme_add_za16_vg1x2>;
+defm FADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fadd", 0b0100, MatrixOp16, ZZZZ_h_mul_r, nxv8f16,  int_aarch64_sme_add_za16_vg1x4>;
+defm FSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fsub", 0b0101, MatrixOp16, ZZ_h_mul_r, nxv8f16,  int_aarch64_sme_sub_za16_vg1x2>;
+defm FSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fsub", 0b0101, MatrixOp16, ZZZZ_h_mul_r, nxv8f16,  int_aarch64_sme_sub_za16_vg1x4>;
 }
 
 let Predicates = [HasSMEF16F16] in {
@@ -822,10 +822,10 @@ defm FMOPS_MPPZZ_H : sme2p1_fmop_tile_fp16<"fmops", 0b0, 0b1, 0b11, ZPR16>;
 }
 
 let Predicates = [HasSME2, HasB16B16] in {
-defm BFADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfadd", 0b1100, MatrixOp16, ZZ_h_mul_r, nxv8bf16, null_frag>;
-defm BFADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfadd", 0b1100, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16, null_frag>;
-defm BFSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfsub", 0b1101, MatrixOp16, ZZ_h_mul_r,  nxv8bf16, null_frag>;
-defm BFSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfsub", 0b1101, MatrixOp16, ZZZZ_h_mul_r,  nxv8bf16, null_frag>;
+defm BFADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfadd", 0b1100, MatrixOp16, ZZ_h_mul_r, nxv8bf16,    int_aarch64_sme_add_za16_vg1x2>;
+defm BFADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfadd", 0b1100, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16,  int_aarch64_sme_add_za16_vg1x4>;
+defm BFSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfsub", 0b1101, MatrixOp16, ZZ_h_mul_r,  nxv8bf16,   int_aarch64_sme_sub_za16_vg1x2>;
+defm BFSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfsub", 0b1101, MatrixOp16, ZZZZ_h_mul_r,  nxv8bf16, int_aarch64_sme_sub_za16_vg1x4>;
 
 defm BFMLA_VG2_M2ZZI : sme2p1_multi_vec_array_vg2_index_16b<"bfmla", 0b00, 0b110, ZZ_h_mul_r, ZPR4b16>;
 defm BFMLA_VG4_M4ZZI : sme2p1_multi_vec_array_vg4_index_16b<"bfmla", 0b010, ZZZZ_h_mul_r, ZPR4b16>;
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll
new file mode 100644
index 000000000000..e7a6c0d6c549
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll
@@ -0,0 +1,148 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+
+target triple = "aarch64-linux"
+
+define void @add_f16_vg1x2(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1) #0 {
+; CHECK-LABEL: add_f16_vg1x2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    fadd za.h[w8, 0, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    fadd za.h[w8, 7, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
+  ret void
+}
+
+define void @add_f16_vg1x4(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+; CHECK-LABEL: add_f16_vg1x4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    fadd za.h[w8, 0, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    fadd za.h[w8, 7, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    ret
+                            <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3) #1 {
+  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
+  ret void
+}
+
+define void @sub_f16_vg1x2(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1) #1 {
+; CHECK-LABEL: sub_f16_vg1x2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    fsub za.h[w8, 0, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    fsub za.h[w8, 7, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
+  ret void
+}
+
+define void @sub_f16_vg1x4(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+; CHECK-LABEL: sub_f16_vg1x4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    fsub za.h[w8, 0, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    fsub za.h[w8, 7, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    ret
+                            <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3) #0 {
+  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
+  ret void
+}
+
+define void @add_bf16_vg1x2(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1) #2 {
+; CHECK-LABEL: add_bf16_vg1x2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    bfadd za.h[w8, 0, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    bfadd za.h[w8, 7, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
+  ret void
+}
+
+define void @add_bf16_vg1x4(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+; CHECK-LABEL: add_bf16_vg1x4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    bfadd za.h[w8, 0, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    bfadd za.h[w8, 7, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    ret
+                            <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3) #2 {
+  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
+  ret void
+}
+
+define void @sub_bf16_vg1x2(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1) #2 {
+; CHECK-LABEL: sub_bf16_vg1x2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    bfsub za.h[w8, 0, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    bfsub za.h[w8, 7, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
+  ret void
+}
+
+define void @sub_bf16_vg1x4(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+; CHECK-LABEL: sub_bf16_vg1x4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    bfsub za.h[w8, 0, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    bfsub za.h[w8, 7, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    ret
+                            <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3) #2 {
+  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
+  ret void
+}
+
+attributes #0 = { nounwind "target-features"="+sme-f16f16" }
+attributes #1 = { nounwind "target-features"="+sme-f8f16" }
+attributes #2 = { nounwind "target-features"="+sme2,+bf16,+b16b16" }
-- 
GitLab


From 6f1013a5b3f92d3ae6e378d6706584a2a44e6964 Mon Sep 17 00:00:00 2001
From: lntue <35648136+lntue@users.noreply.github.com>
Date: Thu, 9 May 2024 09:31:06 -0400
Subject: [PATCH 0290/1206] [libc] Add template deduction guide for
 cpp::lock_guard. (#91589)

Fix ctad-maybe-unsupported warnings for `cpp::lock_guard`.
---
 libc/src/__support/CPP/mutex.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libc/src/__support/CPP/mutex.h b/libc/src/__support/CPP/mutex.h
index 345816fae233..ff9c9f43a43c 100644
--- a/libc/src/__support/CPP/mutex.h
+++ b/libc/src/__support/CPP/mutex.h
@@ -40,6 +40,9 @@ public:
   lock_guard(const lock_guard &) = delete;
 };
 
+// Deduction guide for lock_guard to suppress CTAD warnings.
+template <typename T> lock_guard(T &) -> lock_guard<T>;
+
 } // namespace cpp
 } // namespace LIBC_NAMESPACE
 
-- 
GitLab


From 8d2ab2a0ec168673696930ba3e3c403656cdfe55 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Wed, 8 May 2024 11:38:00 -0700
Subject: [PATCH 0291/1206] [OpenACC][NFC] Fix EndLoc behavior of optional
 clause params

It was discovered while writing the 'wait' clause ast tests that the
'endloc' wasn't set correctly when there was no arguments.  This patch
ensures we set it right, and adds an assert to prevent us from messing
it up in the future.
---
 clang/include/clang/AST/OpenACCClause.h | 5 ++++-
 clang/lib/Parse/ParseOpenACC.cpp        | 4 ++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index e7b0b411b654..125c2af79330 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -26,7 +26,10 @@ class OpenACCClause {
 protected:
   OpenACCClause(OpenACCClauseKind K, SourceLocation BeginLoc,
                 SourceLocation EndLoc)
-      : Kind(K), Location(BeginLoc, EndLoc) {}
+      : Kind(K), Location(BeginLoc, EndLoc) {
+    assert(!BeginLoc.isInvalid() && !EndLoc.isInvalid() &&
+           "Begin and end location must be valid for OpenACCClause");
+      }
 
 public:
   OpenACCClauseKind getClauseKind() const { return Kind; }
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 8c8330a5fad7..727854db9be1 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -1112,6 +1112,10 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
       ParsedClause.setEndLoc(getCurToken().getLocation());
       if (Parens.consumeClose())
         return OpenACCCannotContinue();
+    } else {
+      // If we have optional parens, make sure we set the end-location to the
+      // clause, as we are a 'single token' clause.
+      ParsedClause.setEndLoc(ClauseLoc);
     }
   }
   return OpenACCSuccess(
-- 
GitLab


From b1b465218d1fd3e1abe9332ed9ec535c49061425 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Tue, 7 May 2024 13:30:07 -0700
Subject: [PATCH 0292/1206] [OpenACC] 'wait' clause for compute construct sema

'wait' takes a few int-exprs (well, a series of async-arguments, but
    those are effectively just an int-expr), plus a pair of tags. This
patch adds the support for this to the AST, and does the appropriate
semantic analysis for them.
---
 clang/include/clang/AST/OpenACCClause.h       |  40 +++
 clang/include/clang/Basic/OpenACCClauses.def  |   1 +
 clang/include/clang/Parse/Parser.h            |  10 +-
 clang/include/clang/Sema/SemaOpenACC.h        |  52 +++-
 .../clang/Serialization/ASTRecordReader.h     |   3 +
 .../clang/Serialization/ASTRecordWriter.h     |   2 +
 clang/lib/AST/OpenACCClause.cpp               |  31 +++
 clang/lib/AST/StmtProfile.cpp                 |   7 +
 clang/lib/AST/TextNodeDumper.cpp              |   7 +
 clang/lib/Parse/ParseOpenACC.cpp              |  56 +++--
 clang/lib/Sema/SemaOpenACC.cpp                |  28 +++
 clang/lib/Sema/TreeTransform.h                |  45 ++++
 clang/lib/Serialization/ASTReader.cpp         |  18 +-
 clang/lib/Serialization/ASTWriter.cpp         |  18 +-
 .../ast-print-openacc-compute-construct.cpp   |  24 ++
 clang/test/ParserOpenACC/parse-wait-clause.c  |  48 ++--
 .../compute-construct-intexpr-clause-ast.cpp  | 229 ++++++++++++++++++
 .../compute-construct-wait-clause.c           |  38 +++
 .../compute-construct-wait-clause.cpp         | 104 ++++++++
 clang/tools/libclang/CIndex.cpp               |   6 +
 20 files changed, 711 insertions(+), 56 deletions(-)
 create mode 100644 clang/test/SemaOpenACC/compute-construct-wait-clause.c
 create mode 100644 clang/test/SemaOpenACC/compute-construct-wait-clause.cpp

diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 125c2af79330..d8332816a499 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -192,6 +192,46 @@ public:
   }
 };
 
+// Represents the 'devnum' and expressions lists for the 'wait' clause.
+class OpenACCWaitClause final
+    : public OpenACCClauseWithExprs,
+      public llvm::TrailingObjects<OpenACCWaitClause, Expr *> {
+  SourceLocation QueuesLoc;
+  OpenACCWaitClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
+                    Expr *DevNumExpr, SourceLocation QueuesLoc,
+                    ArrayRef<Expr *> QueueIdExprs, SourceLocation EndLoc)
+      : OpenACCClauseWithExprs(OpenACCClauseKind::Wait, BeginLoc, LParenLoc,
+                               EndLoc),
+        QueuesLoc(QueuesLoc) {
+    // The first element of the trailing storage is always the devnum expr,
+    // whether it is used or not.
+    std::uninitialized_copy(&DevNumExpr, &DevNumExpr + 1,
+                            getTrailingObjects<Expr *>());
+    std::uninitialized_copy(QueueIdExprs.begin(), QueueIdExprs.end(),
+                            getTrailingObjects<Expr *>() + 1);
+    setExprs(
+        MutableArrayRef(getTrailingObjects<Expr *>(), QueueIdExprs.size() + 1));
+  }
+
+public:
+  static OpenACCWaitClause *Create(const ASTContext &C, SourceLocation BeginLoc,
+                                   SourceLocation LParenLoc, Expr *DevNumExpr,
+                                   SourceLocation QueuesLoc,
+                                   ArrayRef<Expr *> QueueIdExprs,
+                                   SourceLocation EndLoc);
+
+  bool hasQueuesTag() const { return !QueuesLoc.isInvalid(); }
+  SourceLocation getQueuesLoc() const { return QueuesLoc; }
+  bool hasDevNumExpr() const { return getExprs()[0]; }
+  Expr *getDevNumExpr() const { return getExprs()[0]; }
+  llvm::ArrayRef<Expr *> getQueueIdExprs() {
+    return OpenACCClauseWithExprs::getExprs().drop_front();
+  }
+  llvm::ArrayRef<Expr *> getQueueIdExprs() const {
+    return OpenACCClauseWithExprs::getExprs().drop_front();
+  }
+};
+
 class OpenACCNumGangsClause final
     : public OpenACCClauseWithExprs,
       public llvm::TrailingObjects<OpenACCNumGangsClause, Expr *> {
diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def
index 8933e09b44f9..afb7b30b7465 100644
--- a/clang/include/clang/Basic/OpenACCClauses.def
+++ b/clang/include/clang/Basic/OpenACCClauses.def
@@ -46,6 +46,7 @@ VISIT_CLAUSE(Present)
 VISIT_CLAUSE(Private)
 VISIT_CLAUSE(Self)
 VISIT_CLAUSE(VectorLength)
+VISIT_CLAUSE(Wait)
 
 #undef VISIT_CLAUSE
 #undef CLAUSE_ALIAS
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 532b5c125ef5..60d59732269b 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3632,6 +3632,13 @@ private:
     // Wait constructs, we likely want to put that information in here as well.
   };
 
+  struct OpenACCWaitParseInfo {
+    bool Failed = false;
+    Expr *DevNumExpr = nullptr;
+    SourceLocation QueuesLoc;
+    SmallVector<Expr *> QueueIdExprs;
+  };
+
   /// Represents the 'error' state of parsing an OpenACC Clause, and stores
   /// whether we can continue parsing, or should give up on the directive.
   enum class OpenACCParseCanContinue { Cannot = 0, Can = 1 };
@@ -3674,7 +3681,8 @@ private:
   /// Parses the clause-list for an OpenACC directive.
   SmallVector<OpenACCClause *>
   ParseOpenACCClauseList(OpenACCDirectiveKind DirKind);
-  bool ParseOpenACCWaitArgument(SourceLocation Loc, bool IsDirective);
+  OpenACCWaitParseInfo ParseOpenACCWaitArgument(SourceLocation Loc,
+                                                bool IsDirective);
   /// Parses the clause of the 'bind' argument, which can be a string literal or
   /// an ID expression.
   ExprResult ParseOpenACCBindClauseArgument();
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 2cec2b73e918..e684ee6b2be1 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -54,8 +54,14 @@ public:
       bool IsZero;
     };
 
+    struct WaitDetails {
+      Expr *DevNumExpr;
+      SourceLocation QueuesLoc;
+      SmallVector<Expr *> QueueIdExprs;
+    };
+
     std::variant<std::monostate, DefaultDetails, ConditionDetails,
-                 IntExprDetails, VarListDetails>
+                 IntExprDetails, VarListDetails, WaitDetails>
         Details = std::monostate{};
 
   public:
@@ -104,14 +110,45 @@ public:
               ClauseKind == OpenACCClauseKind::Async ||
               ClauseKind == OpenACCClauseKind::VectorLength) &&
              "Parsed clause kind does not have a int exprs");
-      //
-      // 'async' has an optional IntExpr, so be tolerant of that.
-      if (ClauseKind == OpenACCClauseKind::Async &&
+
+      // 'async' and 'wait' have an optional IntExpr, so be tolerant of that.
+      if ((ClauseKind == OpenACCClauseKind::Async ||
+           ClauseKind == OpenACCClauseKind::Wait) &&
           std::holds_alternative<std::monostate>(Details))
         return 0;
       return std::get<IntExprDetails>(Details).IntExprs.size();
     }
 
+    SourceLocation getQueuesLoc() const {
+      assert(ClauseKind == OpenACCClauseKind::Wait &&
+             "Parsed clause kind does not have a queues location");
+
+      if (std::holds_alternative<std::monostate>(Details))
+        return SourceLocation{};
+
+      return std::get<WaitDetails>(Details).QueuesLoc;
+    }
+
+    Expr *getDevNumExpr() const {
+      assert(ClauseKind == OpenACCClauseKind::Wait &&
+             "Parsed clause kind does not have a device number expr");
+
+      if (std::holds_alternative<std::monostate>(Details))
+        return nullptr;
+
+      return std::get<WaitDetails>(Details).DevNumExpr;
+    }
+
+    ArrayRef<Expr *> getQueueIdExprs() const {
+      assert(ClauseKind == OpenACCClauseKind::Wait &&
+             "Parsed clause kind does not have a queue id expr list");
+
+      if (std::holds_alternative<std::monostate>(Details))
+        return ArrayRef<Expr *>{std::nullopt};
+
+      return std::get<WaitDetails>(Details).QueueIdExprs;
+    }
+
     ArrayRef<Expr *> getIntExprs() {
       assert((ClauseKind == OpenACCClauseKind::NumGangs ||
               ClauseKind == OpenACCClauseKind::NumWorkers ||
@@ -282,6 +319,13 @@ public:
              "zero: tag only valid on copyout/create");
       Details = VarListDetails{std::move(VarList), IsReadOnly, IsZero};
     }
+
+    void setWaitDetails(Expr *DevNum, SourceLocation QueuesLoc,
+                        llvm::SmallVector<Expr *> &&IntExprs) {
+      assert(ClauseKind == OpenACCClauseKind::Wait &&
+             "Parsed clause kind does not have a wait-details");
+      Details = WaitDetails{DevNum, QueuesLoc, std::move(IntExprs)};
+    }
   };
 
   SemaOpenACC(Sema &S);
diff --git a/clang/include/clang/Serialization/ASTRecordReader.h b/clang/include/clang/Serialization/ASTRecordReader.h
index 1e11d2d5e42f..d00fb182f05f 100644
--- a/clang/include/clang/Serialization/ASTRecordReader.h
+++ b/clang/include/clang/Serialization/ASTRecordReader.h
@@ -272,6 +272,9 @@ public:
   /// Read a list of Exprs used for a var-list.
   llvm::SmallVector<Expr *> readOpenACCVarList();
 
+  /// Read a list of Exprs used for a int-expr-list.
+  llvm::SmallVector<Expr *> readOpenACCIntExprList();
+
   /// Read an OpenACC clause, advancing Idx.
   OpenACCClause *readOpenACCClause();
 
diff --git a/clang/include/clang/Serialization/ASTRecordWriter.h b/clang/include/clang/Serialization/ASTRecordWriter.h
index 8b1da49bd4c5..0c8ac75fc40f 100644
--- a/clang/include/clang/Serialization/ASTRecordWriter.h
+++ b/clang/include/clang/Serialization/ASTRecordWriter.h
@@ -296,6 +296,8 @@ public:
 
   void writeOpenACCVarList(const OpenACCClauseWithVarList *C);
 
+  void writeOpenACCIntExprList(ArrayRef<Expr *> Exprs);
+
   /// Writes out a single OpenACC Clause.
   void writeOpenACCClause(const OpenACCClause *C);
 
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index ffa90884cef5..be079556a87a 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -147,6 +147,18 @@ OpenACCAsyncClause *OpenACCAsyncClause::Create(const ASTContext &C,
   return new (Mem) OpenACCAsyncClause(BeginLoc, LParenLoc, IntExpr, EndLoc);
 }
 
+OpenACCWaitClause *OpenACCWaitClause::Create(
+    const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
+    Expr *DevNumExpr, SourceLocation QueuesLoc, ArrayRef<Expr *> QueueIdExprs,
+    SourceLocation EndLoc) {
+  // Allocates enough room in trailing storage for all the int-exprs, plus a
+  // placeholder for the devnum.
+  void *Mem = C.Allocate(
+      OpenACCWaitClause::totalSizeToAlloc<Expr *>(QueueIdExprs.size() + 1));
+  return new (Mem) OpenACCWaitClause(BeginLoc, LParenLoc, DevNumExpr, QueuesLoc,
+                                     QueueIdExprs, EndLoc);
+}
+
 OpenACCNumGangsClause *OpenACCNumGangsClause::Create(const ASTContext &C,
                                                      SourceLocation BeginLoc,
                                                      SourceLocation LParenLoc,
@@ -393,3 +405,22 @@ void OpenACCClausePrinter::VisitCreateClause(const OpenACCCreateClause &C) {
                         [&](const Expr *E) { printExpr(E); });
   OS << ")";
 }
+
+void OpenACCClausePrinter::VisitWaitClause(const OpenACCWaitClause &C) {
+  OS << "wait";
+  if (!C.getLParenLoc().isInvalid()) {
+    OS << "(";
+    if (C.hasDevNumExpr()) {
+      OS << "devnum: ";
+      printExpr(C.getDevNumExpr());
+      OS << " : ";
+    }
+
+    if (C.hasQueuesTag())
+      OS << "queues: ";
+
+    llvm::interleaveComma(C.getQueueIdExprs(), OS,
+                          [&](const Expr *E) { printExpr(E); });
+    OS << ")";
+  }
+}
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 0910471098c9..8fb8940142eb 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2578,6 +2578,13 @@ void OpenACCClauseProfiler::VisitAsyncClause(const OpenACCAsyncClause &Clause) {
   if (Clause.hasIntExpr())
     Profiler.VisitStmt(Clause.getIntExpr());
 }
+
+void OpenACCClauseProfiler::VisitWaitClause(const OpenACCWaitClause &Clause) {
+  if (Clause.hasDevNumExpr())
+    Profiler.VisitStmt(Clause.getDevNumExpr());
+  for (auto *E : Clause.getQueueIdExprs())
+    Profiler.VisitStmt(E);
+}
 } // namespace
 
 void StmtProfiler::VisitOpenACCComputeConstruct(
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index bf02d9545f84..12aa5858b798 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -437,6 +437,13 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
       if (cast<OpenACCCreateClause>(C)->isZero())
         OS << " : zero";
       break;
+    case OpenACCClauseKind::Wait:
+      OS << " clause";
+      if (cast<OpenACCWaitClause>(C)->hasDevNumExpr())
+        OS << " has devnum";
+      if (cast<OpenACCWaitClause>(C)->hasQueuesTag())
+        OS << " has queues tag";
+      break;
     default:
       // Nothing to do here.
       break;
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 727854db9be1..90dfea6f5f5c 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -867,7 +867,6 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
   SemaOpenACC::OpenACCParsedClause ParsedClause(DirKind, ClauseKind, ClauseLoc);
 
   if (ClauseHasRequiredParens(DirKind, ClauseKind)) {
-    ParsedClause.setLParenLoc(getCurToken().getLocation());
     if (Parens.expectAndConsume()) {
       // We are missing a paren, so assume that the person just forgot the
       // parameter.  Return 'false' so we try to continue on and parse the next
@@ -876,6 +875,7 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
                 Parser::StopBeforeMatch);
       return OpenACCCanContinue();
     }
+    ParsedClause.setLParenLoc(Parens.getOpenLocation());
 
     switch (ClauseKind) {
     case OpenACCClauseKind::Default: {
@@ -1048,8 +1048,8 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
       return OpenACCCannotContinue();
 
   } else if (ClauseHasOptionalParens(DirKind, ClauseKind)) {
-    ParsedClause.setLParenLoc(getCurToken().getLocation());
     if (!Parens.consumeOpen()) {
+      ParsedClause.setLParenLoc(Parens.getOpenLocation());
       switch (ClauseKind) {
       case OpenACCClauseKind::Self: {
         assert(DirKind != OpenACCDirectiveKind::Update);
@@ -1099,13 +1099,19 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
           return OpenACCCanContinue();
         }
         break;
-      case OpenACCClauseKind::Wait:
-        if (ParseOpenACCWaitArgument(ClauseLoc,
-                                     /*IsDirective=*/false)) {
+      case OpenACCClauseKind::Wait: {
+        OpenACCWaitParseInfo Info =
+            ParseOpenACCWaitArgument(ClauseLoc,
+                                     /*IsDirective=*/false);
+        if (Info.Failed) {
           Parens.skipToEnd();
           return OpenACCCanContinue();
         }
+
+        ParsedClause.setWaitDetails(Info.DevNumExpr, Info.QueuesLoc,
+                                    std::move(Info.QueueIdExprs));
         break;
+      }
       default:
         llvm_unreachable("Not an optional parens type?");
       }
@@ -1139,7 +1145,9 @@ Parser::ParseOpenACCAsyncArgument(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
 /// In this section and throughout the specification, the term wait-argument
 /// means:
 /// [ devnum : int-expr : ] [ queues : ] async-argument-list
-bool Parser::ParseOpenACCWaitArgument(SourceLocation Loc, bool IsDirective) {
+Parser::OpenACCWaitParseInfo
+Parser::ParseOpenACCWaitArgument(SourceLocation Loc, bool IsDirective) {
+  OpenACCWaitParseInfo Result;
   // [devnum : int-expr : ]
   if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::DevNum, Tok) &&
       NextToken().is(tok::colon)) {
@@ -1153,18 +1161,25 @@ bool Parser::ParseOpenACCWaitArgument(SourceLocation Loc, bool IsDirective) {
                     : OpenACCDirectiveKind::Invalid,
         IsDirective ? OpenACCClauseKind::Invalid : OpenACCClauseKind::Wait,
         Loc);
-    if (Res.first.isInvalid() && Res.second == OpenACCParseCanContinue::Cannot)
-      return true;
+    if (Res.first.isInvalid() &&
+        Res.second == OpenACCParseCanContinue::Cannot) {
+      Result.Failed = true;
+      return Result;
+    }
 
-    if (ExpectAndConsume(tok::colon))
-      return true;
+    if (ExpectAndConsume(tok::colon)) {
+      Result.Failed = true;
+      return Result;
+    }
+
+    Result.DevNumExpr = Res.first.get();
   }
 
   // [ queues : ]
   if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::Queues, Tok) &&
       NextToken().is(tok::colon)) {
     // Consume queues.
-    ConsumeToken();
+    Result.QueuesLoc = ConsumeToken();
     // Consume colon.
     ConsumeToken();
   }
@@ -1176,8 +1191,10 @@ bool Parser::ParseOpenACCWaitArgument(SourceLocation Loc, bool IsDirective) {
   bool FirstArg = true;
   while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) {
     if (!FirstArg) {
-      if (ExpectAndConsume(tok::comma))
-        return true;
+      if (ExpectAndConsume(tok::comma)) {
+        Result.Failed = true;
+        return Result;
+      }
     }
     FirstArg = false;
 
@@ -1187,11 +1204,16 @@ bool Parser::ParseOpenACCWaitArgument(SourceLocation Loc, bool IsDirective) {
         IsDirective ? OpenACCClauseKind::Invalid : OpenACCClauseKind::Wait,
         Loc);
 
-    if (Res.first.isInvalid() && Res.second == OpenACCParseCanContinue::Cannot)
-      return true;
+    if (Res.first.isInvalid() &&
+        Res.second == OpenACCParseCanContinue::Cannot) {
+      Result.Failed = true;
+      return Result;
+    }
+
+    Result.QueueIdExprs.push_back(Res.first.get());
   }
 
-  return false;
+  return Result;
 }
 
 ExprResult Parser::ParseOpenACCIDExpression() {
@@ -1360,7 +1382,7 @@ Parser::OpenACCDirectiveParseInfo Parser::ParseOpenACCDirective() {
       break;
     case OpenACCDirectiveKind::Wait:
       // OpenACC has an optional paren-wrapped 'wait-argument'.
-      if (ParseOpenACCWaitArgument(StartLoc, /*IsDirective=*/true))
+      if (ParseOpenACCWaitArgument(StartLoc, /*IsDirective=*/true).Failed)
         T.skipToEnd();
       else
         T.consumeClose();
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index b1086baa3ae2..656d30947a8d 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -216,6 +216,22 @@ bool doesClauseApplyToDirective(OpenACCDirectiveKind DirectiveKind,
     default:
       return false;
     }
+  case OpenACCClauseKind::Wait:
+    switch (DirectiveKind) {
+    case OpenACCDirectiveKind::Parallel:
+    case OpenACCDirectiveKind::Serial:
+    case OpenACCDirectiveKind::Kernels:
+    case OpenACCDirectiveKind::Data:
+    case OpenACCDirectiveKind::EnterData:
+    case OpenACCDirectiveKind::ExitData:
+    case OpenACCDirectiveKind::Update:
+    case OpenACCDirectiveKind::ParallelLoop:
+    case OpenACCDirectiveKind::SerialLoop:
+    case OpenACCDirectiveKind::KernelsLoop:
+      return true;
+    default:
+      return false;
+    }
 
   default:
     // Do nothing so we can go to the 'unimplemented' diagnostic instead.
@@ -623,6 +639,18 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
         getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
         Clause.getVarList(), Clause.getEndLoc());
   }
+  case OpenACCClauseKind::Wait: {
+    // Restrictions only properly implemented on 'compute' constructs, and
+    // 'compute' constructs are the only construct that can do anything with
+    // this yet, so skip/treat as unimplemented in this case.
+    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+      break;
+
+    return OpenACCWaitClause::Create(
+        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
+        Clause.getDevNumExpr(), Clause.getQueuesLoc(), Clause.getQueueIdExprs(),
+        Clause.getEndLoc());
+  }
   default:
     break;
   }
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 2d6d6dae680c..0b3cf566e3a7 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -11425,6 +11425,51 @@ void OpenACCClauseTransform<Derived>::VisitAsyncClause(
                                          : nullptr,
       ParsedClause.getEndLoc());
 }
+template <typename Derived>
+void OpenACCClauseTransform<Derived>::VisitWaitClause(
+    const OpenACCWaitClause &C) {
+  if (!C.getLParenLoc().isInvalid()) {
+    Expr *DevNumExpr = nullptr;
+    llvm::SmallVector<Expr *> InstantiatedQueueIdExprs;
+
+    // Instantiate devnum expr if it exists.
+    if (C.getDevNumExpr()) {
+      ExprResult Res = Self.TransformExpr(C.getDevNumExpr());
+      if (!Res.isUsable())
+        return;
+      Res = Self.getSema().OpenACC().ActOnIntExpr(OpenACCDirectiveKind::Invalid,
+                                                  C.getClauseKind(),
+                                                  C.getBeginLoc(), Res.get());
+      if (!Res.isUsable())
+        return;
+
+      DevNumExpr = Res.get();
+    }
+
+    // Instantiate queue ids.
+    for (Expr *CurQueueIdExpr : C.getQueueIdExprs()) {
+      ExprResult Res = Self.TransformExpr(CurQueueIdExpr);
+      if (!Res.isUsable())
+        return;
+      Res = Self.getSema().OpenACC().ActOnIntExpr(OpenACCDirectiveKind::Invalid,
+                                                  C.getClauseKind(),
+                                                  C.getBeginLoc(), Res.get());
+      if (!Res.isUsable())
+        return;
+
+      InstantiatedQueueIdExprs.push_back(Res.get());
+    }
+
+    ParsedClause.setWaitDetails(DevNumExpr, C.getQueuesLoc(),
+                                std::move(InstantiatedQueueIdExprs));
+  }
+
+  NewClause = OpenACCWaitClause::Create(
+      Self.getSema().getASTContext(), ParsedClause.getBeginLoc(),
+      ParsedClause.getLParenLoc(), ParsedClause.getDevNumExpr(),
+      ParsedClause.getQueuesLoc(), ParsedClause.getQueueIdExprs(),
+      ParsedClause.getEndLoc());
+}
 } // namespace
 template <typename Derived>
 OpenACCClause *TreeTransform<Derived>::TransformOpenACCClause(
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 856c743086c5..78e4df440641 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11766,6 +11766,14 @@ SmallVector<Expr *> ASTRecordReader::readOpenACCVarList() {
   return VarList;
 }
 
+SmallVector<Expr *> ASTRecordReader::readOpenACCIntExprList() {
+  unsigned NumExprs = readInt();
+  llvm::SmallVector<Expr *> ExprList;
+  for (unsigned I = 0; I < NumExprs; ++I)
+    ExprList.push_back(readSubExpr());
+  return ExprList;
+}
+
 OpenACCClause *ASTRecordReader::readOpenACCClause() {
   OpenACCClauseKind ClauseKind = readEnum<OpenACCClauseKind>();
   SourceLocation BeginLoc = readSourceLocation();
@@ -11888,6 +11896,15 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
     return OpenACCAsyncClause::Create(getContext(), BeginLoc, LParenLoc,
                                       AsyncExpr, EndLoc);
   }
+  case OpenACCClauseKind::Wait: {
+    SourceLocation LParenLoc = readSourceLocation();
+    Expr *DevNumExpr = readBool() ? readSubExpr() : nullptr;
+    SourceLocation QueuesLoc = readSourceLocation();
+    llvm::SmallVector<Expr *> QueueIdExprs = readOpenACCIntExprList();
+    return OpenACCWaitClause::Create(getContext(), BeginLoc, LParenLoc,
+                                     DevNumExpr, QueuesLoc, QueueIdExprs,
+                                     EndLoc);
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -11913,7 +11930,6 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
   case OpenACCClauseKind::DType:
   case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
-  case OpenACCClauseKind::Wait:
   case OpenACCClauseKind::Invalid:
     llvm_unreachable("Clause serialization not yet implemented");
   }
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index ce2ea4e3d614..e7b9050165bb 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -7791,6 +7791,12 @@ void ASTRecordWriter::writeOpenACCVarList(const OpenACCClauseWithVarList *C) {
     AddStmt(E);
 }
 
+void ASTRecordWriter::writeOpenACCIntExprList(ArrayRef<Expr *> Exprs) {
+  writeUInt32(Exprs.size());
+  for (Expr *E : Exprs)
+    AddStmt(E);
+}
+
 void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   writeEnum(C->getClauseKind());
   writeSourceLocation(C->getBeginLoc());
@@ -7916,6 +7922,17 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
       AddStmt(const_cast<Expr *>(AC->getIntExpr()));
     return;
   }
+  case OpenACCClauseKind::Wait: {
+    const auto *WC = cast<OpenACCWaitClause>(C);
+    writeSourceLocation(WC->getLParenLoc());
+    writeBool(WC->getDevNumExpr());
+    if (const Expr *DNE = WC->getDevNumExpr())
+      AddStmt(const_cast<Expr *>(DNE));
+    writeSourceLocation(WC->getQueuesLoc());
+
+    writeOpenACCIntExprList(WC->getQueueIdExprs());
+    return;
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -7941,7 +7958,6 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   case OpenACCClauseKind::DType:
   case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
-  case OpenACCClauseKind::Wait:
   case OpenACCClauseKind::Invalid:
     llvm_unreachable("Clause serialization not yet implemented");
   }
diff --git a/clang/test/AST/ast-print-openacc-compute-construct.cpp b/clang/test/AST/ast-print-openacc-compute-construct.cpp
index 13597543e9b6..0bfb90bcb587 100644
--- a/clang/test/AST/ast-print-openacc-compute-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-compute-construct.cpp
@@ -83,5 +83,29 @@ void foo() {
   // CHECK: #pragma acc kernels async
 #pragma acc kernels async
   while(true);
+
+// CHECK: #pragma acc parallel wait
+#pragma acc parallel wait
+  while(true);
+
+// CHECK: #pragma acc parallel wait()
+#pragma acc parallel wait()
+  while(true);
+
+// CHECK: #pragma acc parallel wait(*iPtr, i)
+#pragma acc parallel wait(*iPtr, i)
+  while(true);
+
+// CHECK: #pragma acc parallel wait(queues: *iPtr, i)
+#pragma acc parallel wait(queues:*iPtr, i)
+  while(true);
+
+// CHECK: #pragma acc parallel wait(devnum: i : *iPtr, i)
+#pragma acc parallel wait(devnum:i:*iPtr, i)
+  while(true);
+
+// CHECK: #pragma acc parallel wait(devnum: i : queues: *iPtr, i)
+#pragma acc parallel wait(devnum:i:queues:*iPtr, i)
+  while(true);
 }
 
diff --git a/clang/test/ParserOpenACC/parse-wait-clause.c b/clang/test/ParserOpenACC/parse-wait-clause.c
index 64f5b9c8fd73..9c7faa5c02eb 100644
--- a/clang/test/ParserOpenACC/parse-wait-clause.c
+++ b/clang/test/ParserOpenACC/parse-wait-clause.c
@@ -3,12 +3,10 @@
 void func() {
   int i, j;
 
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait
   {}
 
-  // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc parallel wait clause-list
   {}
 
@@ -17,12 +15,10 @@ void func() {
   #pragma acc parallel wait (
       {}
 
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait ()
       {}
 
-  // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc parallel wait () clause-list
       {}
 
@@ -61,12 +57,10 @@ void func() {
   #pragma acc parallel wait (queues:
     {}
 
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait (queues:)
     {}
 
-  // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc parallel wait (queues:) clause-list
     {}
 
@@ -75,12 +69,10 @@ void func() {
   #pragma acc parallel wait (devnum: i + j:queues:
     {}
 
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
   #pragma acc parallel wait (devnum: i + j:queues:)
     {}
 
-  // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc parallel wait (devnum: i + j:queues:) clause-list
     {}
 
@@ -108,13 +100,11 @@ void func() {
   #pragma acc parallel wait(i, j, 1+1, 3.3
     {}
 
-  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+1{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   #pragma acc parallel wait(i, j, 1+1, 3.3)
     {}
-  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
-  // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
+  // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc parallel wait(i, j, 1+1, 3.3) clause-list
     {}
 
@@ -146,14 +136,12 @@ void func() {
   #pragma acc parallel wait(queues:i, j, 1+1, 3.3,
     {}
 
-  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+1{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   #pragma acc parallel wait(queues:i, j, 1+1, 3.3)
     {}
 
-  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
-  // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
+  // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc parallel wait(queues:i, j, 1+1, 3.3) clause-list
     {}
 
@@ -162,13 +150,11 @@ void func() {
   // expected-note@+1{{to match this '('}}
   #pragma acc parallel wait(devnum:3:i, j, 1+1, 3.3
     {}
-  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+1{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   #pragma acc parallel wait(devnum:3:i, j, 1+1, 3.3)
     {}
-  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
-  // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
+  // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc parallel wait(devnum:3:i, j, 1+1, 3.3) clause-list
     {}
 
@@ -177,13 +163,11 @@ void func() {
   // expected-note@+1{{to match this '('}}
   #pragma acc parallel wait(devnum:3:queues:i, j, 1+1, 3.3
     {}
-  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+1{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
   #pragma acc parallel wait(devnum:3:queues:i, j, 1+1, 3.3)
     {}
-  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
-  // expected-error@+2{{invalid OpenACC clause 'clause'}}
-  // expected-warning@+1{{OpenACC clause 'wait' not yet implemented, clause ignored}}
+  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('double' invalid)}}
+  // expected-error@+1{{invalid OpenACC clause 'clause'}}
   #pragma acc parallel wait(devnum:3:queues:i, j, 1+1, 3.3) clause-list
     {}
 }
diff --git a/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp b/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp
index b85de56c7ae9..56c3512dec3b 100644
--- a/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp
+++ b/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp
@@ -135,6 +135,93 @@ void NormalUses() {
   // CHECK-NEXT: WhileStmt
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel wait
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel wait()
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel wait(some_int(), some_long())
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel wait(queues:some_int(), some_long())
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause has queues tag
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel wait(devnum: some_int() :some_int(), some_long())
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause has devnum
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel wait(devnum: some_int() : queues :some_int(), some_long()) wait(devnum: some_int() : queues :some_int(), some_long())
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause has devnum has queues tag
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: wait clause has devnum has queues tag
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'int (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'int ()' lvalue Function{{.*}} 'some_int' 'int ()'
+  // CHECK-NEXT: CallExpr{{.*}}'long'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'long (*)()' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr{{.*}}'long ()' lvalue Function{{.*}} 'some_long' 'long ()'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
 }
 
 
@@ -282,6 +369,72 @@ void TemplUses(T t, U u) {
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: CompoundStmt
 
+#pragma acc parallel wait
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel wait()
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel wait(U::value, u)
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel wait(queues: U::value, u)
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause has queues tag
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel wait(devnum:u:queues: U::value, u)
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause has devnum has queues tag
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+#pragma acc parallel wait(devnum:u: U::value, u)
+  while (true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause has devnum
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'U'
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'U' lvalue ParmVar{{.*}} 'u' 'U'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
 
   // CHECK-NEXT: DeclStmt
   // CHECK-NEXT: VarDecl{{.*}}EndMarker
@@ -437,6 +590,82 @@ void TemplUses(T t, U u) {
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: CompoundStmt
 
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause has queues tag
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause has devnum has queues tag
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: wait clause has devnum
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const int' lvalue Var{{.*}} 'value' 'const int'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'HasInt'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'char' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}}'char'
+  // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator char
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'HasInt' lvalue ParmVar
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
   // CHECK-NEXT: DeclStmt
   // CHECK-NEXT: VarDecl{{.*}}EndMarker
 }
diff --git a/clang/test/SemaOpenACC/compute-construct-wait-clause.c b/clang/test/SemaOpenACC/compute-construct-wait-clause.c
new file mode 100644
index 000000000000..254aba8442fe
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-wait-clause.c
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct NotConvertible{} NC;
+short getS();
+int getI();
+
+void uses() {
+  int arr[5];
+
+#pragma acc parallel wait
+  while(1);
+
+#pragma acc serial wait()
+  while(1);
+
+#pragma acc kernels wait(getS(), getI())
+  while(1);
+
+#pragma acc parallel wait(devnum:getS(): getI())
+  while(1);
+
+#pragma acc parallel wait(devnum:getS(): queues: getI()) wait(devnum:getI(): queues: getS(), getI(), 5)
+  while(1);
+
+  // expected-error@+1{{OpenACC clause 'wait' requires expression of integer type ('struct NotConvertible' invalid)}}
+#pragma acc parallel wait(devnum:NC : 5)
+  while(1);
+
+  // expected-error@+1{{OpenACC clause 'wait' requires expression of integer type ('struct NotConvertible' invalid)}}
+#pragma acc parallel wait(devnum:5 : NC)
+  while(1);
+
+  // expected-error@+3{{OpenACC clause 'wait' requires expression of integer type ('int[5]' invalid)}}
+  // expected-error@+2{{OpenACC clause 'wait' requires expression of integer type ('int[5]' invalid)}}
+  // expected-error@+1{{OpenACC clause 'wait' requires expression of integer type ('struct NotConvertible' invalid)}}
+#pragma acc parallel wait(devnum:arr : queues: arr, NC, 5)
+  while(1);
+}
diff --git a/clang/test/SemaOpenACC/compute-construct-wait-clause.cpp b/clang/test/SemaOpenACC/compute-construct-wait-clause.cpp
new file mode 100644
index 000000000000..94f669be0f67
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-wait-clause.cpp
@@ -0,0 +1,104 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct ExplicitConvertOnly {
+  explicit operator int() const; // #EXPL_CONV
+} Explicit;
+
+struct AmbiguousConvert{
+  operator int(); // #AMBIG_INT
+  operator short(); // #AMBIG_SHORT
+  operator float();
+} Ambiguous;
+
+void Test() {
+
+  // expected-error@+3{{multiple conversions from expression type 'struct AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel wait(Ambiguous)
+  while (true);
+
+  // expected-error@+2{{OpenACC integer expression type 'struct ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+#pragma acc parallel wait(4, Explicit, 5)
+  while (true);
+
+  // expected-error@+3{{multiple conversions from expression type 'struct AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel wait(queues: Ambiguous, 5)
+  while (true);
+
+  // expected-error@+2{{OpenACC integer expression type 'struct ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+#pragma acc parallel wait(devnum: Explicit: 5)
+  while (true);
+
+  // expected-error@+2{{OpenACC integer expression type 'struct ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+#pragma acc parallel wait(devnum: Explicit:queues:  5)
+  while (true);
+
+  // expected-error@+1{{use of undeclared identifier 'queues'}}
+#pragma acc parallel wait(devnum: queues:  5)
+  while (true);
+}
+
+struct HasInt {
+  using IntTy = int;
+  using ShortTy = short;
+  static constexpr int value = 1;
+  static constexpr AmbiguousConvert ACValue;
+  static constexpr ExplicitConvertOnly EXValue;
+
+  operator char();
+};
+
+template<typename T>
+void TestInst() {
+
+#pragma acc parallel wait(T{})
+  while (true);
+
+#pragma acc parallel wait(devnum:typename T::ShortTy{}:queues:typename T::IntTy{})
+  while (true);
+
+  // expected-error@+4{{multiple conversions from expression type 'const AmbiguousConvert' to an integral type}}
+  // expected-note@#INST{{in instantiation of function template specialization}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel wait(devnum:T::value :queues:T::ACValue)
+  while (true);
+
+  // expected-error@+5{{OpenACC integer expression type 'const ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+  // expected-error@+3{{multiple conversions from expression type 'const AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel wait(devnum:T::EXValue :queues:T::ACValue)
+  while (true);
+
+  // expected-error@+5{{OpenACC integer expression type 'const ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+  // expected-error@+3{{multiple conversions from expression type 'const AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel wait(T::EXValue, T::ACValue)
+  while (true);
+
+  // expected-error@+5{{OpenACC integer expression type 'const ExplicitConvertOnly' requires explicit conversion to 'int'}}
+  // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
+  // expected-error@+3{{multiple conversions from expression type 'const AmbiguousConvert' to an integral type}}
+  // expected-note@#AMBIG_INT{{conversion to integral type 'int'}}
+  // expected-note@#AMBIG_SHORT{{conversion to integral type 'short'}}
+#pragma acc parallel wait(queues: T::EXValue, T::ACValue)
+  while (true);
+
+  // expected-error@+1{{no member named 'Invalid' in 'HasInt'}}
+#pragma acc parallel wait(queues: T::Invalid, T::Invalid2)
+  while (true);
+}
+
+void Inst() {
+  TestInst<HasInt>(); // #INST
+}
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index b845a381d63b..ae6659fe95e8 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2851,6 +2851,12 @@ void OpenACCClauseEnqueue::VisitAsyncClause(const OpenACCAsyncClause &C) {
   if (C.hasIntExpr())
     Visitor.AddStmt(C.getIntExpr());
 }
+void OpenACCClauseEnqueue::VisitWaitClause(const OpenACCWaitClause &C) {
+  if (const Expr *DevNumExpr = C.getDevNumExpr())
+    Visitor.AddStmt(DevNumExpr);
+  for (Expr *QE : C.getQueueIdExprs())
+    Visitor.AddStmt(QE);
+}
 } // namespace
 
 void EnqueueVisitor::EnqueueChildren(const OpenACCClause *C) {
-- 
GitLab


From c416e43571cca78dea3ee75b2a9fec944c0c65a1 Mon Sep 17 00:00:00 2001
From: alx32 <103613512+alx32@users.noreply.github.com>
Date: Thu, 9 May 2024 06:47:37 -0700
Subject: [PATCH 0293/1206] [lld-macho] Add support for non-lazy categories to
 ObjC category merger (#91548)

In ObjC we can have categories that define a `+load` method that is
called when the category is loaded. In such cases, we shouldn't optimize
the category. These categories are present in the `__objc_nlcatlist`
section. So we scan these section for such categories and ignore them
from optimization.
---
 lld/MachO/ObjC.cpp                            | 24 ++++++++++
 .../objc-category-merging-complete-test.s     | 47 +++++++++++++++++++
 2 files changed, 71 insertions(+)

diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp
index 15b89a808b05..96ec646095be 100644
--- a/lld/MachO/ObjC.cpp
+++ b/lld/MachO/ObjC.cpp
@@ -428,6 +428,7 @@ public:
   static void doCleanup();
 
 private:
+  DenseSet<const Symbol *> collectNlCategories();
   void collectAndValidateCategoriesData();
   void
   mergeCategoriesIntoSingleCategory(std::vector<InfoInputCategory> &categories);
@@ -1060,7 +1061,27 @@ void ObjcCategoryMerger::createSymbolReference(Defined *refFrom,
   refFrom->isec()->relocs.push_back(r);
 }
 
+// Get the list of categories in the '__objc_nlcatlist' section. We can't
+// optimize these as they have a '+load' method that has to be called at
+// runtime.
+DenseSet<const Symbol *> ObjcCategoryMerger::collectNlCategories() {
+  DenseSet<const Symbol *> nlCategories;
+
+  for (InputSection *sec : allInputSections) {
+    if (sec->getName() != section_names::objcNonLazyCatList)
+      continue;
+
+    for (auto &r : sec->relocs) {
+      const Symbol *sym = r.referent.dyn_cast<Symbol *>();
+      nlCategories.insert(sym);
+    }
+  }
+  return nlCategories;
+}
+
 void ObjcCategoryMerger::collectAndValidateCategoriesData() {
+  auto nlCategories = collectNlCategories();
+
   for (InputSection *sec : allInputSections) {
     if (sec->getName() != section_names::objcCatList)
       continue;
@@ -1074,6 +1095,9 @@ void ObjcCategoryMerger::collectAndValidateCategoriesData() {
       assert(categorySym &&
              "Failed to get a valid category at __objc_catlit offset");
 
+      if (nlCategories.count(categorySym))
+        continue;
+
       // We only support ObjC categories (no swift + @objc)
       // TODO: Support swift + @objc categories also
       if (!categorySym->getName().starts_with(objc::symbol_names::category))
diff --git a/lld/test/MachO/objc-category-merging-complete-test.s b/lld/test/MachO/objc-category-merging-complete-test.s
index d2d264a3f26c..74400177b550 100644
--- a/lld/test/MachO/objc-category-merging-complete-test.s
+++ b/lld/test/MachO/objc-category-merging-complete-test.s
@@ -88,6 +88,7 @@ MERGE_CATS-NEXT:                 name {{.*}} MyProtocol02Prop
 MERGE_CATS-NEXT:            attributes {{.*}} Ti,R,D
 MERGE_CATS-NEXT:                 name {{.*}} MyProtocol03Prop
 MERGE_CATS-NEXT:            attributes {{.*}} Ti,R,D
+MERGE_CATS:        __OBJC_$_CATEGORY_MyBaseClass_$_Category04
 
 
 NO_MERGE_CATS-NOT: __OBJC_$_CATEGORY_MyBaseClass(Category02|Category03)
@@ -431,6 +432,15 @@ L_OBJC_IMAGE_INFO:
 ## @dynamic MyProtocol03Prop;
 ## @end
 ##
+## // This category shouldn't be merged
+## @interface MyBaseClass(Category04)
+## + (void)load;
+## @end
+##
+## @implementation MyBaseClass(Category04)
+## + (void)load {}
+## @end
+##
 ## int main() {
 ##     return 0;
 ## }
@@ -493,6 +503,12 @@ L_OBJC_IMAGE_INFO:
 	b	_OUTLINED_FUNCTION_0
 	.cfi_endproc
                                         ; -- End function
+	.p2align	2
+"+[MyBaseClass(Category04) load]":
+	.cfi_startproc
+; %bb.0:
+	ret
+	.cfi_endproc
 	.globl	_main                           ; -- Begin function main
 	.p2align	2
 _main:                                  ; @main
@@ -746,11 +762,42 @@ __OBJC_$_CATEGORY_MyBaseClass_$_Category03:
 	.quad	0
 	.long	64                              ; 0x40
 	.space	4
+	.section	__TEXT,__objc_classname,cstring_literals
+l_OBJC_CLASS_NAME_.15:
+	.asciz	"Category04"
+	.section	__TEXT,__objc_methname,cstring_literals
+l_OBJC_METH_VAR_NAME_.16:
+	.asciz	"load"
+	.section	__DATA,__objc_const
+	.p2align	3, 0x0
+__OBJC_$_CATEGORY_CLASS_METHODS_MyBaseClass_$_Category04:
+	.long	24
+	.long	1
+	.quad	l_OBJC_METH_VAR_NAME_.16
+	.quad	l_OBJC_METH_VAR_TYPE_
+	.quad	"+[MyBaseClass(Category04) load]"
+	.p2align	3, 0x0
+__OBJC_$_CATEGORY_MyBaseClass_$_Category04:
+	.quad	l_OBJC_CLASS_NAME_.15
+	.quad	_OBJC_CLASS_$_MyBaseClass
+	.quad	0
+	.quad	__OBJC_$_CATEGORY_CLASS_METHODS_MyBaseClass_$_Category04
+	.quad	0
+	.quad	0
+	.quad	0
+	.long	64
+	.space	4
 	.section	__DATA,__objc_catlist,regular,no_dead_strip
 	.p2align	3, 0x0                          ; @"OBJC_LABEL_CATEGORY_$"
 l_OBJC_LABEL_CATEGORY_$:
 	.quad	__OBJC_$_CATEGORY_MyBaseClass_$_Category02
 	.quad	__OBJC_$_CATEGORY_MyBaseClass_$_Category03
+	.quad	__OBJC_$_CATEGORY_MyBaseClass_$_Category04
+	.section	__DATA,__objc_nlcatlist,regular,no_dead_strip
+	.p2align	3, 0x0
+l_OBJC_LABEL_NONLAZY_CATEGORY_$:
+	.quad	__OBJC_$_CATEGORY_MyBaseClass_$_Category04
+
 	.no_dead_strip	__OBJC_LABEL_PROTOCOL_$_MyProtocol02
 	.no_dead_strip	__OBJC_LABEL_PROTOCOL_$_MyProtocol03
 	.no_dead_strip	__OBJC_PROTOCOL_$_MyProtocol02
-- 
GitLab


From 4ad3de901e8acfafdeb59406064f24a3c2ea27e8 Mon Sep 17 00:00:00 2001
From: Jake Egan <Jake.egan@ibm.com>
Date: Thu, 9 May 2024 09:53:57 -0400
Subject: [PATCH 0294/1206] [driver][test] Only check for unused plugin options
 (#91522)

This fixes matching `clang: error: argument unused during compilation:
'-Werror' [-Werror,-Wunused-command-line-argument]` on AIX.

---------

Co-authored-by: Hubert Tong <hubert-reinterpretcast@users.noreply.github.com>
---
 clang/test/Driver/plugin-driver-args.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/test/Driver/plugin-driver-args.cpp b/clang/test/Driver/plugin-driver-args.cpp
index 6f0e6e2ba752..6efd859f9d08 100644
--- a/clang/test/Driver/plugin-driver-args.cpp
+++ b/clang/test/Driver/plugin-driver-args.cpp
@@ -23,5 +23,5 @@
 
 // Plugins are only relevant for the -cc1 phase. No warning should be raised
 // when only using the assembler. See GH #88173.
-// RUN: %clang -c -fpass-plugin=bar.so -fplugin=bar.so -fplugin-arg-bar-option -Werror -x assembler %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-PLUGIN-ASM
-// CHECK-PLUGIN-ASM-NOT: argument unused during compilation
+// RUN: %clang -c -fpass-plugin=bar.so -fplugin=bar.so -fplugin-arg-bar-option -Wunused-command-line-argument -x assembler %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-PLUGIN-ASM
+// CHECK-PLUGIN-ASM-NOT: argument unused during compilation: '-f{{[a-z-]*plugin[^']*}}'
-- 
GitLab


From b54a5e7271e34530893ae374876e284a65d785d4 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Thu, 9 May 2024 09:57:28 -0400
Subject: [PATCH 0295/1206] [gn] port d86b68afd7f0 more

See my comments on https://github.com/llvm/llvm-project/pull/88257.
(The AMDGPU target internal depencencies were among the messiest
among all targets even before that.)
---
 llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn       | 1 +
 .../gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index edd8d4f1840d..dad4f028236d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -60,6 +60,7 @@ tablegen("AMDGPUGenMCPseudoLowering") {
 tablegen("AMDGPUGenRegisterBank") {
   visibility = [
     ":LLVMAMDGPUCodeGen",
+    "MCTargetDesc",
     "Utils",
     "//llvm/unittests/MC/AMDGPU:AMDGPUMCTests",
     "//llvm/unittests/Target/AMDGPU:AMDGPUTests",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn
index 5ba91fcec83a..0df55cbc0826 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn
@@ -94,6 +94,10 @@ static_library("MCTargetDesc") {
     "//llvm/lib/Target/AMDGPU/TargetInfo",
     "//llvm/lib/Target/AMDGPU/Utils",
     "//llvm/lib/TargetParser",
+
+    # AMDGPUMCExpr.cpp includes GCNSubtarget.h which after 490e348e679
+    # includes the generated AMDGPUGenRegisterBank.inc file :/
+    "//llvm/lib/Target/AMDGPU/:AMDGPUGenRegisterBank",
   ]
   include_dirs = [ ".." ]
   sources = [
-- 
GitLab


From 139e0aa68dc23d2aeec05de1ae05ebf2aa5fa11e Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Thu, 9 May 2024 15:03:52 +0100
Subject: [PATCH 0296/1206] Revert "[AArch64] Add intrinsics for multi-vector
 to ZA array vector accumulators" (#91597)

Reverts llvm/llvm-project#88266  due to test failures

error: 'expected-error' diagnostics seen but not expected:
(frontend): '-fsyntax-only' action ignored; '-emit-llvm' action
specified previously
---
 clang/include/clang/Basic/arm_sme.td          |  10 -
 .../acle_sme2_add_sub_za16.c                  | 193 ------------------
 .../acle_sme2_add_sub_za16.c                  |  29 ---
 llvm/include/llvm/IR/IntrinsicsAArch64.td     |   2 +-
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |  16 +-
 .../AArch64/sme2-intrinsics-add-sub-za16.ll   | 148 --------------
 6 files changed, 9 insertions(+), 389 deletions(-)
 delete mode 100644 clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
 delete mode 100644 clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
 delete mode 100644 llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll

diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index 000bd97a4b25..1ac6d5170ea2 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -298,16 +298,6 @@ multiclass ZAAddSub<string n_suffix> {
     def NAME # _ZA64_VG1X2_F64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x2", "vm2", "d", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x2", [IsStreaming, IsInOutZA], []>;
     def NAME # _ZA64_VG1X4_F64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x4", "vm4", "d", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x4", [IsStreaming, IsInOutZA], []>;
   }
-
-  let TargetGuard = "sme-f16f16|sme-f8f16" in {
-    def NAME # _ZA16_VG1X2_F16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x2", "vm2", "h", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x2", [IsStreaming, IsInOutZA], []>;
-    def NAME # _ZA16_VG1X4_F16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x4", "vm4", "h", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x4", [IsStreaming, IsInOutZA], []>;
-  }
-
-  let TargetGuard = "sme2,b16b16" in {
-    def NAME # _ZA16_VG1X2_BF16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x2", "vm2", "b", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x2", [IsStreaming, IsInOutZA], []>;
-    def NAME # _ZA16_VG1X4_BF16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x4", "vm4", "b", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x4", [IsStreaming, IsInOutZA], []>;
-  }
 }
 
 defm SVADD : ZAAddSub<"add">;
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
deleted file mode 100644
index d98427fac610..000000000000
--- a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
+++ /dev/null
@@ -1,193 +0,0 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
-// RUN: %clang_cc1                               -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -S -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1                        -x c++ -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature  +sme-f8f16 -target-feature +b16b16 -O2 -S -Werror -Wall -emit-llvm -o - %s | FileCheck %s -check-prefix CHECK-CXX
-// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS        -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature  +sme-f8f16 -target-feature +b16b16 -O2 -S -Werror -Wall -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -x c++ -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -S -Werror -Wall -emit-llvm -o - %s | FileCheck %s -check-prefix CHECK-CXX
-
-// RUN: %clang_cc1                               -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -S -Werror -Wall -o /dev/null
-
-// REQUIRES: aarch64-registered-target
-
-#include <arm_sme.h>
-
-#ifdef SVE_OVERLOADED_FORMS
-#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
-#else
-#define SVE_ACLE_FUNC(A1,A2,A3) A1##A2##A3
-#endif
-
-// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x2_f16(
-// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
-// CHECK-NEXT:    ret void
-//
-// CHECK-CXX-LABEL: define dso_local void @_Z25test_svadd_za16_vg1x2_f16j13svfloat16x2_t(
-// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
-// CHECK-CXX-NEXT:    ret void
-//
-void test_svadd_za16_vg1x2_f16(uint32_t slice, svfloat16x2_t zn) __arm_streaming __arm_inout("za") {
-  SVE_ACLE_FUNC(svadd_za16,_f16,_vg1x2)(slice, zn);
-}
-
-// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x4_f16(
-// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
-// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
-// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
-// CHECK-NEXT:    ret void
-//
-// CHECK-CXX-LABEL: define dso_local void @_Z25test_svadd_za16_vg1x4_f16j13svfloat16x4_t(
-// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
-// CHECK-CXX-NEXT:    ret void
-//
-void test_svadd_za16_vg1x4_f16(uint32_t slice, svfloat16x4_t zn) __arm_streaming __arm_inout("za") {
-  SVE_ACLE_FUNC(svadd_za16,_f16,_vg1x4)(slice, zn);
-}
-
-// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x2_f16(
-// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
-// CHECK-NEXT:    ret void
-//
-// CHECK-CXX-LABEL: define dso_local void @_Z25test_svsub_za16_vg1x2_f16j13svfloat16x2_t(
-// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
-// CHECK-CXX-NEXT:    ret void
-//
-void test_svsub_za16_vg1x2_f16(uint32_t slice, svfloat16x2_t zn) __arm_streaming __arm_inout("za") {
-  SVE_ACLE_FUNC(svsub_za16,_f16,_vg1x2)(slice, zn);
-}
-
-// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x4_f16(
-// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
-// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
-// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
-// CHECK-NEXT:    ret void
-//
-// CHECK-CXX-LABEL: define dso_local void @_Z25test_svsub_za16_vg1x4_f16j13svfloat16x4_t(
-// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
-// CHECK-CXX-NEXT:    ret void
-//
-void test_svsub_za16_vg1x4_f16(uint32_t slice, svfloat16x4_t zn) __arm_streaming __arm_inout("za") {
-  SVE_ACLE_FUNC(svsub_za16,_f16,_vg1x4)(slice, zn);
-}
-
-// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x2_bf16(
-// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
-// CHECK-NEXT:    ret void
-//
-// CHECK-CXX-LABEL: define dso_local void @_Z26test_svadd_za16_vg1x2_bf16j14svbfloat16x2_t(
-// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
-// CHECK-CXX-NEXT:    ret void
-//
-void test_svadd_za16_vg1x2_bf16(uint32_t slice, svbfloat16x2_t zn) __arm_streaming __arm_inout("za") {
-  SVE_ACLE_FUNC(svadd_za16,_bf16,_vg1x2)(slice, zn);
-}
-
-// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x4_bf16(
-// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
-// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
-// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
-// CHECK-NEXT:    ret void
-//
-// CHECK-CXX-LABEL: define dso_local void @_Z26test_svadd_za16_vg1x4_bf16j14svbfloat16x4_t(
-// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
-// CHECK-CXX-NEXT:    ret void
-//
-void test_svadd_za16_vg1x4_bf16(uint32_t slice, svbfloat16x4_t zn) __arm_streaming __arm_inout("za") {
-  SVE_ACLE_FUNC(svadd_za16,_bf16,_vg1x4)(slice, zn);
-}
-
-// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x2_bf16(
-// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
-// CHECK-NEXT:    ret void
-//
-// CHECK-CXX-LABEL: define dso_local void @_Z26test_svsub_za16_vg1x2_bf16j14svbfloat16x2_t(
-// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
-// CHECK-CXX-NEXT:    ret void
-//
-void test_svsub_za16_vg1x2_bf16(uint32_t slice, svbfloat16x2_t zn) __arm_streaming __arm_inout("za") {
-  SVE_ACLE_FUNC(svsub_za16,_bf16,_vg1x2)(slice, zn);
-}
-
-// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x4_bf16(
-// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
-// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
-// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
-// CHECK-NEXT:    ret void
-//
-// CHECK-CXX-LABEL: define dso_local void @_Z26test_svsub_za16_vg1x4_bf16j14svbfloat16x4_t(
-// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
-// CHECK-CXX-NEXT:    ret void
-//
-void test_svsub_za16_vg1x4_bf16(uint32_t slice, svbfloat16x4_t zn) __arm_streaming __arm_inout("za") {
-  SVE_ACLE_FUNC(svsub_za16,_bf16,_vg1x4)(slice, zn);
-}
diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
deleted file mode 100644
index 0eeeac5a5046..000000000000
--- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
+++ /dev/null
@@ -1,29 +0,0 @@
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -fsyntax-only -verify -emit-llvm %s
-
-// REQUIRES: aarch64-registered-target
-
-#include <arm_sme.h>
-
-void test_features(uint32_t slice, svfloat16x2_t zn2, svfloat16x4_t zn4,
-                   svbfloat16x2_t bzn2, svbfloat16x4_t bzn4) __arm_streaming __arm_inout("za") {
-  // expected-error@+1 {{'svadd_za16_f16_vg1x2' needs target feature sme-f16f16|sme-f8f16}}
-  svadd_za16_f16_vg1x2(slice, zn2);
-  // expected-error@+1 {{'svadd_za16_f16_vg1x4' needs target feature sme-f16f16|sme-f8f16}}
-  svadd_za16_f16_vg1x4(slice, zn4);
-  // expected-error@+1 {{'svsub_za16_f16_vg1x2' needs target feature sme-f16f16|sme-f8f16}}
-  svsub_za16_f16_vg1x2(slice, zn2);
-  // expected-error@+1 {{'svsub_za16_f16_vg1x4' needs target feature sme-f16f16|sme-f8f16}}
-  svsub_za16_f16_vg1x4(slice, zn4);
-
-  // expected-error@+1 {{'svadd_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
-  svadd_za16_bf16_vg1x2(slice, bzn2);
-  // expected-error@+1 {{'svadd_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
-  svadd_za16_bf16_vg1x4(slice, bzn4);
-  // expected-error@+1 {{'svsub_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
-  svsub_za16_bf16_vg1x2(slice, bzn2);
-  // expected-error@+1 {{'svsub_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
-  svsub_za16_bf16_vg1x4(slice, bzn4);
-}
-
-
-
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 04571faf1306..e31e00a9c76f 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -3481,7 +3481,7 @@ let TargetPrefix = "aarch64" in {
   // Multi-vector add/sub and accumulate into ZA
   //
   foreach intr = ["add", "sub"] in {
-    foreach za = ["za16","za32", "za64"] in {
+    foreach za = ["za32", "za64"] in {
       def int_aarch64_sme_ # intr # _ # za # _vg1x2 : SME2_ZA_Write_VG2_Intrinsic;
       def int_aarch64_sme_ # intr # _ # za # _vg1x4 : SME2_ZA_Write_VG4_Intrinsic;
     }
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index 5102c602d54e..574178c8d524 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -793,10 +793,10 @@ defm LUTI4_S_4ZTZI : sme2p1_luti4_vector_vg4_index<"luti4">;
 }
 
 let Predicates = [HasSMEF16F16orSMEF8F16] in {
-defm FADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fadd", 0b0100, MatrixOp16, ZZ_h_mul_r, nxv8f16, int_aarch64_sme_add_za16_vg1x2>;
-defm FADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fadd", 0b0100, MatrixOp16, ZZZZ_h_mul_r, nxv8f16,  int_aarch64_sme_add_za16_vg1x4>;
-defm FSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fsub", 0b0101, MatrixOp16, ZZ_h_mul_r, nxv8f16,  int_aarch64_sme_sub_za16_vg1x2>;
-defm FSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fsub", 0b0101, MatrixOp16, ZZZZ_h_mul_r, nxv8f16,  int_aarch64_sme_sub_za16_vg1x4>;
+defm FADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fadd", 0b0100, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
+defm FADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fadd", 0b0100, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
+defm FSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fsub", 0b0101, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
+defm FSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fsub", 0b0101, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
 }
 
 let Predicates = [HasSMEF16F16] in {
@@ -822,10 +822,10 @@ defm FMOPS_MPPZZ_H : sme2p1_fmop_tile_fp16<"fmops", 0b0, 0b1, 0b11, ZPR16>;
 }
 
 let Predicates = [HasSME2, HasB16B16] in {
-defm BFADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfadd", 0b1100, MatrixOp16, ZZ_h_mul_r, nxv8bf16,    int_aarch64_sme_add_za16_vg1x2>;
-defm BFADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfadd", 0b1100, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16,  int_aarch64_sme_add_za16_vg1x4>;
-defm BFSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfsub", 0b1101, MatrixOp16, ZZ_h_mul_r,  nxv8bf16,   int_aarch64_sme_sub_za16_vg1x2>;
-defm BFSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfsub", 0b1101, MatrixOp16, ZZZZ_h_mul_r,  nxv8bf16, int_aarch64_sme_sub_za16_vg1x4>;
+defm BFADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfadd", 0b1100, MatrixOp16, ZZ_h_mul_r, nxv8bf16, null_frag>;
+defm BFADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfadd", 0b1100, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16, null_frag>;
+defm BFSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfsub", 0b1101, MatrixOp16, ZZ_h_mul_r,  nxv8bf16, null_frag>;
+defm BFSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfsub", 0b1101, MatrixOp16, ZZZZ_h_mul_r,  nxv8bf16, null_frag>;
 
 defm BFMLA_VG2_M2ZZI : sme2p1_multi_vec_array_vg2_index_16b<"bfmla", 0b00, 0b110, ZZ_h_mul_r, ZPR4b16>;
 defm BFMLA_VG4_M4ZZI : sme2p1_multi_vec_array_vg4_index_16b<"bfmla", 0b010, ZZZZ_h_mul_r, ZPR4b16>;
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll
deleted file mode 100644
index e7a6c0d6c549..000000000000
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll
+++ /dev/null
@@ -1,148 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
-
-target triple = "aarch64-linux"
-
-define void @add_f16_vg1x2(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1) #0 {
-; CHECK-LABEL: add_f16_vg1x2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    fadd za.h[w8, 0, vgx2], { z0.h, z1.h }
-; CHECK-NEXT:    fadd za.h[w8, 7, vgx2], { z0.h, z1.h }
-; CHECK-NEXT:    ret
-  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
-  %slice.7 = add i32 %slice, 7
-  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
-  ret void
-}
-
-define void @add_f16_vg1x4(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
-; CHECK-LABEL: add_f16_vg1x4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    fadd za.h[w8, 0, vgx4], { z0.h - z3.h }
-; CHECK-NEXT:    fadd za.h[w8, 7, vgx4], { z0.h - z3.h }
-; CHECK-NEXT:    ret
-                            <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3) #1 {
-  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
-                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
-  %slice.7 = add i32 %slice, 7
-  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
-                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
-  ret void
-}
-
-define void @sub_f16_vg1x2(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1) #1 {
-; CHECK-LABEL: sub_f16_vg1x2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    fsub za.h[w8, 0, vgx2], { z0.h, z1.h }
-; CHECK-NEXT:    fsub za.h[w8, 7, vgx2], { z0.h, z1.h }
-; CHECK-NEXT:    ret
-  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
-  %slice.7 = add i32 %slice, 7
-  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
-  ret void
-}
-
-define void @sub_f16_vg1x4(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
-; CHECK-LABEL: sub_f16_vg1x4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    fsub za.h[w8, 0, vgx4], { z0.h - z3.h }
-; CHECK-NEXT:    fsub za.h[w8, 7, vgx4], { z0.h - z3.h }
-; CHECK-NEXT:    ret
-                            <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3) #0 {
-  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
-                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
-  %slice.7 = add i32 %slice, 7
-  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
-                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
-  ret void
-}
-
-define void @add_bf16_vg1x2(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1) #2 {
-; CHECK-LABEL: add_bf16_vg1x2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    bfadd za.h[w8, 0, vgx2], { z0.h, z1.h }
-; CHECK-NEXT:    bfadd za.h[w8, 7, vgx2], { z0.h, z1.h }
-; CHECK-NEXT:    ret
-  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
-  %slice.7 = add i32 %slice, 7
-  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
-  ret void
-}
-
-define void @add_bf16_vg1x4(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
-; CHECK-LABEL: add_bf16_vg1x4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    bfadd za.h[w8, 0, vgx4], { z0.h - z3.h }
-; CHECK-NEXT:    bfadd za.h[w8, 7, vgx4], { z0.h - z3.h }
-; CHECK-NEXT:    ret
-                            <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3) #2 {
-  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
-                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
-  %slice.7 = add i32 %slice, 7
-  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
-                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
-  ret void
-}
-
-define void @sub_bf16_vg1x2(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1) #2 {
-; CHECK-LABEL: sub_bf16_vg1x2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    bfsub za.h[w8, 0, vgx2], { z0.h, z1.h }
-; CHECK-NEXT:    bfsub za.h[w8, 7, vgx2], { z0.h, z1.h }
-; CHECK-NEXT:    ret
-  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
-  %slice.7 = add i32 %slice, 7
-  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
-  ret void
-}
-
-define void @sub_bf16_vg1x4(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
-; CHECK-LABEL: sub_bf16_vg1x4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    bfsub za.h[w8, 0, vgx4], { z0.h - z3.h }
-; CHECK-NEXT:    bfsub za.h[w8, 7, vgx4], { z0.h - z3.h }
-; CHECK-NEXT:    ret
-                            <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3) #2 {
-  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
-                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
-  %slice.7 = add i32 %slice, 7
-  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
-                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
-  ret void
-}
-
-attributes #0 = { nounwind "target-features"="+sme-f16f16" }
-attributes #1 = { nounwind "target-features"="+sme-f8f16" }
-attributes #2 = { nounwind "target-features"="+sme2,+bf16,+b16b16" }
-- 
GitLab


From 673cfcd03b7b938b422fee07d8ca4a127d480b1f Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 9 May 2024 09:23:27 -0500
Subject: [PATCH 0297/1206] Revert "[Linker] Propagate `nobuiltin` attributes
 when linking known libcalls (#89431)"

This apparently breaks AMDGPU offloading for unknown reasons. Reverting
for now.

This reverts commit aa16de6399a42421076ed642c3b4f7fb12c6d44b.
---
 llvm/include/llvm/Linker/IRMover.h | 33 +--------------
 llvm/lib/Linker/CMakeLists.txt     |  1 -
 llvm/lib/Linker/IRMover.cpp        | 67 ++----------------------------
 llvm/test/Linker/Inputs/strlen.ll  | 21 ----------
 llvm/test/Linker/libcalls.ll       | 39 -----------------
 5 files changed, 5 insertions(+), 156 deletions(-)
 delete mode 100644 llvm/test/Linker/Inputs/strlen.ll
 delete mode 100644 llvm/test/Linker/libcalls.ll

diff --git a/llvm/include/llvm/Linker/IRMover.h b/llvm/include/llvm/Linker/IRMover.h
index 8e71c6080dff..1e3c5394ffa2 100644
--- a/llvm/include/llvm/Linker/IRMover.h
+++ b/llvm/include/llvm/Linker/IRMover.h
@@ -12,14 +12,11 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/FunctionExtras.h"
-#include "llvm/ADT/StringSet.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/Support/StringSaver.h"
-#include "llvm/TargetParser/Triple.h"
 #include <functional>
 
 namespace llvm {
 class Error;
+class GlobalValue;
 class Metadata;
 class Module;
 class StructType;
@@ -63,33 +60,6 @@ public:
     bool hasType(StructType *Ty);
   };
 
-  /// Utility for handling linking of known libcall functions. If a merged
-  /// module contains a recognized library call we can no longer perform any
-  /// libcall related transformations.
-  class LibcallHandler {
-    bool HasLibcalls = false;
-
-    StringSet<> Libcalls;
-    StringSet<> Triples;
-
-    BumpPtrAllocator Alloc;
-    StringSaver Saver;
-
-  public:
-    LibcallHandler() : Saver(Alloc) {}
-
-    void updateLibcalls(const Triple &TheTriple);
-
-    bool checkLibcalls(GlobalValue &GV) {
-      if (HasLibcalls)
-        return false;
-      return HasLibcalls = isa<Function>(&GV) && !GV.isDeclaration() &&
-                           Libcalls.count(GV.getName());
-    }
-
-    bool hasLibcalls() const { return HasLibcalls; }
-  };
-
   IRMover(Module &M);
 
   typedef std::function<void(GlobalValue &)> ValueAdder;
@@ -114,7 +84,6 @@ private:
   Module &Composite;
   IdentifiedStructTypeSet IdentifiedStructTypes;
   MDMapT SharedMDs; ///< A Metadata map to use for all calls to \a move().
-  LibcallHandler Libcalls;
 };
 
 } // End llvm namespace
diff --git a/llvm/lib/Linker/CMakeLists.txt b/llvm/lib/Linker/CMakeLists.txt
index 25001c09a62d..5afb40f8b588 100644
--- a/llvm/lib/Linker/CMakeLists.txt
+++ b/llvm/lib/Linker/CMakeLists.txt
@@ -9,7 +9,6 @@ add_llvm_component_library(LLVMLinker
   intrinsics_gen
 
   LINK_COMPONENTS
-  Analysis
   Core
   Object
   Support
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index fe2b53183589..7a5aa0c80478 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -12,7 +12,6 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/AutoUpgrade.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfoMetadata.h"
@@ -400,9 +399,6 @@ class IRLinker {
   /// A metadata map that's shared between IRLinker instances.
   MDMapT &SharedMDs;
 
-  /// A list of libcalls that the current target may call.
-  IRMover::LibcallHandler &Libcalls;
-
   /// Mapping of values from what they used to be in Src, to what they are now
   /// in DstM.  ValueToValueMapTy is a ValueMap, which involves some overhead
   /// due to the use of Value handles which the Linker doesn't actually need,
@@ -544,12 +540,10 @@ public:
   IRLinker(Module &DstM, MDMapT &SharedMDs,
            IRMover::IdentifiedStructTypeSet &Set, std::unique_ptr<Module> SrcM,
            ArrayRef<GlobalValue *> ValuesToLink,
-           IRMover::LibcallHandler &Libcalls, IRMover::LazyCallback AddLazyFor,
-           bool IsPerformingImport)
+           IRMover::LazyCallback AddLazyFor, bool IsPerformingImport)
       : DstM(DstM), SrcM(std::move(SrcM)), AddLazyFor(std::move(AddLazyFor)),
         TypeMap(Set), GValMaterializer(*this), LValMaterializer(*this),
-        SharedMDs(SharedMDs), Libcalls(Libcalls),
-        IsPerformingImport(IsPerformingImport),
+        SharedMDs(SharedMDs), IsPerformingImport(IsPerformingImport),
         Mapper(ValueMap, RF_ReuseAndMutateDistinctMDs | RF_IgnoreMissingLocals,
                &TypeMap, &GValMaterializer),
         IndirectSymbolMCID(Mapper.registerAlternateMappingContext(
@@ -567,13 +561,6 @@ public:
 };
 }
 
-static void addNoBuiltinAttributes(Function &F) {
-  F.setAttributes(
-      F.getAttributes().addFnAttribute(F.getContext(), "no-builtins"));
-  F.setAttributes(
-      F.getAttributes().addFnAttribute(F.getContext(), Attribute::NoBuiltin));
-}
-
 /// The LLVM SymbolTable class autorenames globals that conflict in the symbol
 /// table. This is good for all clients except for us. Go through the trouble
 /// to force this back.
@@ -1618,26 +1605,14 @@ Error IRLinker::run() {
 
   DstM.setTargetTriple(SrcTriple.merge(DstTriple));
 
-  // Update the target triple's libcall information if it was changed.
-  Libcalls.updateLibcalls(Triple(DstM.getTargetTriple()));
-
   // Loop over all of the linked values to compute type mappings.
   computeTypeMapping();
 
-  bool AddsLibcalls = false;
   std::reverse(Worklist.begin(), Worklist.end());
   while (!Worklist.empty()) {
     GlobalValue *GV = Worklist.back();
     Worklist.pop_back();
 
-    // If the module already contains libcall functions we need every function
-    // linked in to have `nobuiltin` attributes. Otherwise check if this is a
-    // libcall definition.
-    if (Function *F = dyn_cast<Function>(GV); F && Libcalls.hasLibcalls())
-      addNoBuiltinAttributes(*F);
-    else
-      AddsLibcalls = Libcalls.checkLibcalls(*GV);
-
     // Already mapped.
     if (ValueMap.find(GV) != ValueMap.end() ||
         IndirectSymbolValueMap.find(GV) != IndirectSymbolValueMap.end())
@@ -1700,13 +1675,6 @@ Error IRLinker::run() {
     }
   }
 
-  // If we have imported a recognized libcall function we can no longer make any
-  // reasonable optimizations based off of its semantics. Add the 'nobuiltin'
-  // attribute to every function to suppress libcall detection.
-  if (AddsLibcalls)
-    for (Function &F : DstM.functions())
-      addNoBuiltinAttributes(F);
-
   // Merge the module flags into the DstM module.
   return linkModuleFlagsMetadata();
 }
@@ -1789,22 +1757,6 @@ bool IRMover::IdentifiedStructTypeSet::hasType(StructType *Ty) {
   return I == NonOpaqueStructTypes.end() ? false : *I == Ty;
 }
 
-void IRMover::LibcallHandler::updateLibcalls(const Triple &TheTriple) {
-  if (Triples.count(TheTriple.getTriple()))
-    return;
-  Triples.insert(Saver.save(TheTriple.getTriple()));
-
-  // Collect the names of runtime functions that the target may want to call.
-  TargetLibraryInfoImpl TLII(TheTriple);
-  TargetLibraryInfo TLI(TLII);
-  for (unsigned I = 0, E = static_cast<unsigned>(LibFunc::NumLibFuncs); I != E;
-       ++I) {
-    LibFunc F = static_cast<LibFunc>(I);
-    if (TLI.has(F))
-      Libcalls.insert(TLI.getName(F));
-  }
-}
-
 IRMover::IRMover(Module &M) : Composite(M) {
   TypeFinder StructTypes;
   StructTypes.run(M, /* OnlyNamed */ false);
@@ -1820,25 +1772,14 @@ IRMover::IRMover(Module &M) : Composite(M) {
   for (const auto *MD : StructTypes.getVisitedMetadata()) {
     SharedMDs[MD].reset(const_cast<MDNode *>(MD));
   }
-
-  // Check the composite module for any already present libcalls. If we define
-  // these then it is important to mark any imported functions as 'nobuiltin'.
-  Libcalls.updateLibcalls(Triple(Composite.getTargetTriple()));
-  for (Function &F : Composite.functions())
-    if (Libcalls.checkLibcalls(F))
-      break;
-
-  if (Libcalls.hasLibcalls())
-    for (Function &F : Composite.functions())
-      addNoBuiltinAttributes(F);
 }
 
 Error IRMover::move(std::unique_ptr<Module> Src,
                     ArrayRef<GlobalValue *> ValuesToLink,
                     LazyCallback AddLazyFor, bool IsPerformingImport) {
   IRLinker TheIRLinker(Composite, SharedMDs, IdentifiedStructTypes,
-                       std::move(Src), ValuesToLink, Libcalls,
-                       std::move(AddLazyFor), IsPerformingImport);
+                       std::move(Src), ValuesToLink, std::move(AddLazyFor),
+                       IsPerformingImport);
   Error E = TheIRLinker.run();
   Composite.dropTriviallyDeadConstantArrays();
   return E;
diff --git a/llvm/test/Linker/Inputs/strlen.ll b/llvm/test/Linker/Inputs/strlen.ll
deleted file mode 100644
index bc54aaf41e0c..000000000000
--- a/llvm/test/Linker/Inputs/strlen.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-target triple = "x86_64-unknown-linux-gnu"
-
-define i64 @strlen(ptr %s) #0 {
-entry:
-  br label %for.cond
-
-for.cond:
-  %s.addr.0 = phi ptr [ %s, %entry ], [ %incdec.ptr, %for.cond ]
-  %0 = load i8, ptr %s.addr.0, align 1
-  %tobool.not = icmp eq i8 %0, 0
-  %incdec.ptr = getelementptr inbounds i8, ptr %s.addr.0, i64 1
-  br i1 %tobool.not, label %for.end, label %for.cond
-
-for.end:
-  %sub.ptr.lhs.cast = ptrtoint ptr %s.addr.0 to i64
-  %sub.ptr.rhs.cast = ptrtoint ptr %s to i64
-  %sub.ptr.sub = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
-  ret i64 %sub.ptr.sub
-}
-
-attributes #0 = { noinline }
diff --git a/llvm/test/Linker/libcalls.ll b/llvm/test/Linker/libcalls.ll
deleted file mode 100644
index ddc0d35e91d9..000000000000
--- a/llvm/test/Linker/libcalls.ll
+++ /dev/null
@@ -1,39 +0,0 @@
-; RUN: llvm-link %s %S/Inputs/strlen.ll -S -o - 2>%t.a.err | FileCheck %s --check-prefix=CHECK1
-; RUN: llvm-link %S/Inputs/strlen.ll %s -S -o - 2>%t.a.err | FileCheck %s --check-prefix=CHECK2
-
-target triple = "x86_64-unknown-linux-gnu"
-
-@.str = private unnamed_addr constant [7 x i8] c"string\00", align 1
-@str = dso_local global ptr @.str, align 8
-
-define void @foo() #0 {
-  ret void
-}
-
-declare i64 @strlen(ptr)
-
-define void @bar() #0 {
-  ret void
-}
-
-define i64 @baz() #0 {
-entry:
-  %0 = load ptr, ptr @str, align 8
-  %call = call i64 @strlen(ptr noundef %0)
-  ret i64 %call
-}
-
-attributes #0 = { noinline }
-
-; CHECK1: define void @foo() #[[ATTR0:[0-9]+]]
-; CHECK1: define void @bar() #[[ATTR0:[0-9]+]]
-; CHECK1: define i64 @baz() #[[ATTR0:[0-9]+]]
-; CHECK1: define i64 @strlen(ptr [[S:%.*]]) #[[ATTR0]]
-
-; CHECK2: define i64 @strlen(ptr [[S:%.*]]) #[[ATTR0:[0-9]+]]
-; CHECK2: define void @foo() #[[ATTR0:[0-9]+]]
-; CHECK2: define void @bar() #[[ATTR0:[0-9]+]]
-; CHECK2: define i64 @baz() #[[ATTR0]]
-
-; CHECK1: attributes #[[ATTR0]] = { nobuiltin noinline "no-builtins" }
-; CHECK2: attributes #[[ATTR0]] = { nobuiltin noinline "no-builtins" }
-- 
GitLab


From 846ffc7ac1a43dc83fc0ee1280a793988fae7ab0 Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Thu, 9 May 2024 22:32:54 +0800
Subject: [PATCH 0298/1206] Disable flaky test: dfsan/release_shadow_space.c
 (#91493)

The current pass rate on the bot is ~50%.

https://github.com/llvm/llvm-project/issues/91287
---
 compiler-rt/test/dfsan/release_shadow_space.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/compiler-rt/test/dfsan/release_shadow_space.c b/compiler-rt/test/dfsan/release_shadow_space.c
index 675640a1c296..60dec98ebec4 100644
--- a/compiler-rt/test/dfsan/release_shadow_space.c
+++ b/compiler-rt/test/dfsan/release_shadow_space.c
@@ -3,6 +3,9 @@
 // DFSAN_OPTIONS=no_huge_pages_for_shadow=false RUN: %clang_dfsan %s -DORIGIN_TRACKING -mllvm -dfsan-track-origins=1 -o %t && %run %t
 // DFSAN_OPTIONS=no_huge_pages_for_shadow=true RUN: %clang_dfsan %s -DORIGIN_TRACKING -mllvm -dfsan-track-origins=1 -o %t && %run %t
 
+// This test is flaky right now: https://github.com/llvm/llvm-project/issues/91287
+// UNSUPPORTED:  target={{.*}}
+
 #include <assert.h>
 #include <sanitizer/dfsan_interface.h>
 #include <stdbool.h>
-- 
GitLab


From fa9e90f5d23312587b3a17920941334e0d1a58a1 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 9 May 2024 06:35:54 -0500
Subject: [PATCH 0299/1206] [Reland][Libomptarget] Statically link all plugin
 runtimes (#87009)

This patch overhauls the `libomptarget` and plugin interface. Currently,
we define a C API and compile each plugin as a separate shared library.
Then, `libomptarget` loads these API functions and forwards its internal
calls to them. This was originally designed to allow multiple
implementations of a library to be live. However, since then no one has
used this functionality and it prevents us from using much nicer
interfaces. If the old behavior is desired it should instead be
implemented as a separate plugin.

This patch replaces the `PluginAdaptorTy` interface with the
`GenericPluginTy` that is used by the plugins. Each plugin exports a
`createPlugin_<name>` function that is used to get the specific
implementation. This code is now shared with `libomptarget`.

There are some notable improvements to this.
1. Massively improved lifetimes of life runtime objects
2. The plugins can use a C++ interface
3. Global state does not need to be duplicated for each plugin +
   libomptarget
4. Easier to use and add features and improve error handling
5. Less function call overhead / Improved LTO performance.

Additional changes in this plugin are related to contending with the
fact that state is now shared. Initialization and deinitialization is
now handled correctly and in phase with the underlying runtime, allowing
us to actually know when something is getting deallocated.

Depends on https://github.com/llvm/llvm-project/pull/86971
https://github.com/llvm/llvm-project/pull/86875
https://github.com/llvm/llvm-project/pull/86868
---
 clang/test/Driver/linker-wrapper-image.c      |   2 +-
 .../Frontend/Offloading/OffloadWrapper.cpp    |   7 +-
 offload/include/PluginManager.h               |  61 ++----
 offload/include/device.h                      |   8 +-
 offload/plugins-nextgen/CMakeLists.txt        |  19 +-
 offload/plugins-nextgen/amdgpu/CMakeLists.txt |   5 -
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    |  14 +-
 offload/plugins-nextgen/common/CMakeLists.txt |   4 +-
 .../common/include/PluginInterface.h          |  94 +-------
 .../common/include/Utils/ELF.h                |   2 -
 offload/plugins-nextgen/common/src/JIT.cpp    |  40 ++--
 .../common/src/PluginInterface.cpp            | 205 ------------------
 offload/plugins-nextgen/cuda/CMakeLists.txt   |   5 -
 offload/plugins-nextgen/cuda/src/rtl.cpp      |  14 +-
 offload/plugins-nextgen/host/CMakeLists.txt   |   8 -
 offload/plugins-nextgen/host/src/rtl.cpp      |  14 +-
 offload/src/CMakeLists.txt                    |   4 +
 offload/src/OffloadRTL.cpp                    |   1 +
 offload/src/OpenMP/InteropAPI.cpp             |   4 +-
 offload/src/PluginManager.cpp                 | 129 ++++-------
 offload/src/device.cpp                        |   3 +-
 offload/src/interface.cpp                     |   2 -
 .../kernelreplay/llvm-omp-kernel-replay.cpp   |   2 -
 .../unittests/Plugins/NextgenPluginsTest.cpp  |   1 -
 24 files changed, 125 insertions(+), 523 deletions(-)

diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c
index d01445e3aed0..5d5d62805e17 100644
--- a/clang/test/Driver/linker-wrapper-image.c
+++ b/clang/test/Driver/linker-wrapper-image.c
@@ -30,8 +30,8 @@
 
 //      OPENMP: define internal void @.omp_offloading.descriptor_reg() section ".text.startup" {
 // OPENMP-NEXT: entry:
-// OPENMP-NEXT:   %0 = call i32 @atexit(ptr @.omp_offloading.descriptor_unreg)
 // OPENMP-NEXT:   call void @__tgt_register_lib(ptr @.omp_offloading.descriptor)
+// OPENMP-NEXT:   %0 = call i32 @atexit(ptr @.omp_offloading.descriptor_unreg)
 // OPENMP-NEXT:   ret void
 // OPENMP-NEXT: }
 
diff --git a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
index 7241d15ed1c6..8b6f9ea1f4cc 100644
--- a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
+++ b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp
@@ -232,12 +232,13 @@ void createRegisterFunction(Module &M, GlobalVariable *BinDesc,
   // Construct function body
   IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));
 
+  Builder.CreateCall(RegFuncC, BinDesc);
+
   // Register the destructors with 'atexit'. This is expected by the CUDA
   // runtime and ensures that we clean up before dynamic objects are destroyed.
-  // This needs to be done before the runtime is called and registers its own.
+  // This needs to be done after plugin initialization to ensure that it is
+  // called before the plugin runtime is destroyed.
   Builder.CreateCall(AtExit, UnregFunc);
-
-  Builder.CreateCall(RegFuncC, BinDesc);
   Builder.CreateRetVoid();
 
   // Add this function to constructors.
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index eece7525e25e..1d6804da75d9 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -13,10 +13,11 @@
 #ifndef OMPTARGET_PLUGIN_MANAGER_H
 #define OMPTARGET_PLUGIN_MANAGER_H
 
+#include "PluginInterface.h"
+
 #include "DeviceImage.h"
 #include "ExclusiveAccess.h"
 #include "Shared/APITypes.h"
-#include "Shared/PluginAPI.h"
 #include "Shared/Requirements.h"
 
 #include "device.h"
@@ -34,38 +35,7 @@
 #include <mutex>
 #include <string>
 
-struct PluginManager;
-
-/// Plugin adaptors should be created via `PluginAdaptorTy::create` which will
-/// invoke the constructor and call `PluginAdaptorTy::init`. Eventual errors are
-/// reported back to the caller, otherwise a valid and initialized adaptor is
-/// returned.
-struct PluginAdaptorTy {
-  /// Try to create a plugin adaptor from a filename.
-  static llvm::Expected<std::unique_ptr<PluginAdaptorTy>>
-  create(const std::string &Name);
-
-  /// Name of the shared object file representing the plugin.
-  std::string Name;
-
-  /// Access to the shared object file representing the plugin.
-  std::unique_ptr<llvm::sys::DynamicLibrary> LibraryHandler;
-
-#define PLUGIN_API_HANDLE(NAME)                                                \
-  using NAME##_ty = decltype(__tgt_rtl_##NAME);                                \
-  NAME##_ty *NAME = nullptr;
-
-#include "Shared/PluginAPI.inc"
-#undef PLUGIN_API_HANDLE
-
-  /// Create a plugin adaptor for filename \p Name with a dynamic library \p DL.
-  PluginAdaptorTy(const std::string &Name,
-                  std::unique_ptr<llvm::sys::DynamicLibrary> DL);
-
-  /// Initialize the plugin adaptor, this can fail in which case the adaptor is
-  /// useless.
-  llvm::Error init();
-};
+using GenericPluginTy = llvm::omp::target::plugin::GenericPluginTy;
 
 /// Struct for the data required to handle plugins
 struct PluginManager {
@@ -80,6 +50,8 @@ struct PluginManager {
 
   void init();
 
+  void deinit();
+
   // Register a shared library with all (compatible) RTLs.
   void registerLib(__tgt_bin_desc *Desc);
 
@@ -92,10 +64,9 @@ struct PluginManager {
         std::make_unique<DeviceImageTy>(TgtBinDesc, TgtDeviceImage));
   }
 
-  /// Initialize as many devices as possible for this plugin adaptor. Devices
-  /// that fail to initialize are ignored. Returns the offset the devices were
-  /// registered at.
-  void initDevices(PluginAdaptorTy &RTL);
+  /// Initialize as many devices as possible for this plugin. Devices that fail
+  /// to initialize are ignored.
+  void initDevices(GenericPluginTy &RTL);
 
   /// Return the device presented to the user as device \p DeviceNo if it is
   /// initialized and ready. Otherwise return an error explaining the problem.
@@ -151,8 +122,8 @@ struct PluginManager {
   // Initialize all plugins.
   void initAllPlugins();
 
-  /// Iterator range for all plugin adaptors (in use or not, but always valid).
-  auto pluginAdaptors() { return llvm::make_pointee_range(PluginAdaptors); }
+  /// Iterator range for all plugins (in use or not, but always valid).
+  auto plugins() { return llvm::make_pointee_range(Plugins); }
 
   /// Return the user provided requirements.
   int64_t getRequirements() const { return Requirements.getRequirements(); }
@@ -164,14 +135,14 @@ private:
   bool RTLsLoaded = false;
   llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
 
-  // List of all plugin adaptors, in use or not.
-  llvm::SmallVector<std::unique_ptr<PluginAdaptorTy>> PluginAdaptors;
+  // List of all plugins, in use or not.
+  llvm::SmallVector<std::unique_ptr<GenericPluginTy>> Plugins;
 
-  // Mapping of plugin adaptors to offsets in the device table.
-  llvm::DenseMap<const PluginAdaptorTy *, int32_t> DeviceOffsets;
+  // Mapping of plugins to offsets in the device table.
+  llvm::DenseMap<const GenericPluginTy *, int32_t> DeviceOffsets;
 
-  // Mapping of plugin adaptors to the number of used devices.
-  llvm::DenseMap<const PluginAdaptorTy *, int32_t> DeviceUsed;
+  // Mapping of plugins to the number of used devices.
+  llvm::DenseMap<const GenericPluginTy *, int32_t> DeviceUsed;
 
   // Set of all device images currently in use.
   llvm::DenseSet<const __tgt_device_image *> UsedImages;
diff --git a/offload/include/device.h b/offload/include/device.h
index bd2829722bb3..fd6e5fba5fc5 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -33,17 +33,19 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 
+#include "PluginInterface.h"
+using GenericPluginTy = llvm::omp::target::plugin::GenericPluginTy;
+
 // Forward declarations.
-struct PluginAdaptorTy;
 struct __tgt_bin_desc;
 struct __tgt_target_table;
 
 struct DeviceTy {
   int32_t DeviceID;
-  PluginAdaptorTy *RTL;
+  GenericPluginTy *RTL;
   int32_t RTLDeviceID;
 
-  DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID);
+  DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID);
   // DeviceTy is not copyable
   DeviceTy(const DeviceTy &D) = delete;
   DeviceTy &operator=(const DeviceTy &D) = delete;
diff --git a/offload/plugins-nextgen/CMakeLists.txt b/offload/plugins-nextgen/CMakeLists.txt
index df625e97c7eb..d1079f8a3e9c 100644
--- a/offload/plugins-nextgen/CMakeLists.txt
+++ b/offload/plugins-nextgen/CMakeLists.txt
@@ -14,7 +14,7 @@
 set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common)
 add_subdirectory(common)
 function(add_target_library target_name lib_name)
-  add_llvm_library(${target_name} SHARED
+  add_llvm_library(${target_name} STATIC
     LINK_COMPONENTS
       ${LLVM_TARGETS_TO_BUILD}
       AggressiveInstCombine
@@ -46,27 +46,14 @@ function(add_target_library target_name lib_name)
   )
 
   llvm_update_compile_flags(${target_name})
+  target_include_directories(${target_name} PUBLIC ${common_dir}/include)
   target_link_libraries(${target_name} PRIVATE
                         PluginCommon ${OPENMP_PTHREAD_LIB})
 
   target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name})
   target_compile_definitions(${target_name} PRIVATE 
                              DEBUG_PREFIX="TARGET ${lib_name} RTL")
-
-  if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
-    # On FreeBSD, the 'environ' symbol is undefined at link time, but resolved by
-    # the dynamic linker at runtime. Therefore, allow the symbol to be undefined
-    # when creating a shared library.
-    target_link_libraries(${target_name} PRIVATE "-Wl,--allow-shlib-undefined")
-  else()
-    target_link_libraries(${target_name} PRIVATE "-Wl,-z,defs")
-  endif()
-
-  if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
-    target_link_libraries(${target_name} PRIVATE
-    "-Wl,--version-script=${common_dir}/../exports")
-  endif()
-  set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET protected)
+  set_target_properties(${target_name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
 
 foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD)
diff --git a/offload/plugins-nextgen/amdgpu/CMakeLists.txt b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
index f5f7096137c2..738183f8945e 100644
--- a/offload/plugins-nextgen/amdgpu/CMakeLists.txt
+++ b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
@@ -57,8 +57,3 @@ else()
   libomptarget_say("Not generating AMDGPU tests, no supported devices detected."
                    " Use 'LIBOMPTARGET_FORCE_AMDGPU_TESTS' to override.")
 endif()
-
-# Install plugin under the lib destination folder.
-install(TARGETS omptarget.rtl.amdgpu LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
-set_target_properties(omptarget.rtl.amdgpu PROPERTIES
-  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 00650b801b42..295685fceaa4 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -3064,10 +3064,6 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
     // HSA functions from now on, e.g., hsa_shut_down.
     Initialized = true;
 
-#ifdef OMPT_SUPPORT
-    ompt::connectLibrary();
-#endif
-
     // Register event handler to detect memory errors on the devices.
     Status = hsa_amd_register_system_event_handler(eventHandler, nullptr);
     if (auto Err = Plugin::check(
@@ -3155,6 +3151,8 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
 
   Triple::ArchType getTripleArch() const override { return Triple::amdgcn; }
 
+  const char *getName() const override { return GETNAME(TARGET_NAME); }
+
   /// Get the ELF code for recognizing the compatible image binary.
   uint16_t getMagicElfBits() const override { return ELF::EM_AMDGPU; }
 
@@ -3387,8 +3385,6 @@ Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
   return Plugin::success();
 }
 
-GenericPluginTy *PluginTy::createPlugin() { return new AMDGPUPluginTy(); }
-
 template <typename... ArgsTy>
 static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
   hsa_status_t ResultCode = static_cast<hsa_status_t>(Code);
@@ -3476,3 +3472,9 @@ void *AMDGPUDeviceTy::allocate(size_t Size, void *, TargetAllocTy Kind) {
 } // namespace target
 } // namespace omp
 } // namespace llvm
+
+extern "C" {
+llvm::omp::target::plugin::GenericPluginTy *createPlugin_amdgpu() {
+  return new llvm::omp::target::plugin::AMDGPUPluginTy();
+}
+}
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index acf0af63f050..a470dcee6d85 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -66,6 +66,4 @@ target_include_directories(PluginCommon PUBLIC
   ${LIBOMPTARGET_INCLUDE_DIR}
 )
 
-set_target_properties(PluginCommon PROPERTIES
-  POSITION_INDEPENDENT_CODE ON
-  CXX_VISIBILITY_PRESET protected)
+set_target_properties(PluginCommon PROPERTIES POSITION_INDEPENDENT_CODE ON)
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 79e8464bfda5..e7a008f3a857 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -1010,6 +1010,9 @@ struct GenericPluginTy {
   /// Get the target triple of this plugin.
   virtual Triple::ArchType getTripleArch() const = 0;
 
+  /// Get the constant name identifier for this plugin.
+  virtual const char *getName() const = 0;
+
   /// Allocate a structure using the internal allocator.
   template <typename Ty> Ty *allocate() {
     return reinterpret_cast<Ty *>(Allocator.Allocate(sizeof(Ty), alignof(Ty)));
@@ -1226,7 +1229,7 @@ namespace Plugin {
 /// Create a success error. This is the same as calling Error::success(), but
 /// it is recommended to use this one for consistency with Plugin::error() and
 /// Plugin::check().
-static Error success() { return Error::success(); }
+static inline Error success() { return Error::success(); }
 
 /// Create a string error.
 template <typename... ArgsTy>
@@ -1246,95 +1249,6 @@ template <typename... ArgsTy>
 static Error check(int32_t ErrorCode, const char *ErrFmt, ArgsTy... Args);
 } // namespace Plugin
 
-/// Class for simplifying the getter operation of the plugin. Anywhere on the
-/// code, the current plugin can be retrieved by Plugin::get(). The class also
-/// declares functions to create plugin-specific object instances. The check(),
-/// createPlugin(), createDevice() and createGlobalHandler() functions should be
-/// defined by each plugin implementation.
-class PluginTy {
-  // Reference to the plugin instance.
-  static GenericPluginTy *SpecificPlugin;
-
-  PluginTy() {
-    if (auto Err = init())
-      REPORT("Failed to initialize plugin: %s\n",
-             toString(std::move(Err)).data());
-  }
-
-  ~PluginTy() {
-    if (auto Err = deinit())
-      REPORT("Failed to deinitialize plugin: %s\n",
-             toString(std::move(Err)).data());
-  }
-
-  PluginTy(const PluginTy &) = delete;
-  void operator=(const PluginTy &) = delete;
-
-  /// Create and intialize the plugin instance.
-  static Error init() {
-    assert(!SpecificPlugin && "Plugin already created");
-
-    // Create the specific plugin.
-    SpecificPlugin = createPlugin();
-    assert(SpecificPlugin && "Plugin was not created");
-
-    // Initialize the plugin.
-    return SpecificPlugin->init();
-  }
-
-  // Deinitialize and destroy the plugin instance.
-  static Error deinit() {
-    assert(SpecificPlugin && "Plugin no longer valid");
-
-    for (int32_t DevNo = 0, NumDev = SpecificPlugin->getNumDevices();
-         DevNo < NumDev; ++DevNo)
-      if (auto Err = SpecificPlugin->deinitDevice(DevNo))
-        return Err;
-
-    // Deinitialize the plugin.
-    if (auto Err = SpecificPlugin->deinit())
-      return Err;
-
-    // Delete the plugin instance.
-    delete SpecificPlugin;
-
-    // Invalidate the plugin reference.
-    SpecificPlugin = nullptr;
-
-    return Plugin::success();
-  }
-
-public:
-  /// Initialize the plugin if needed. The plugin could have been initialized by
-  /// a previous call to Plugin::get().
-  static Error initIfNeeded() {
-    // Trigger the initialization if needed.
-    get();
-
-    return Error::success();
-  }
-
-  /// Get a reference (or create if it was not created) to the plugin instance.
-  static GenericPluginTy &get() {
-    // This static variable will initialize the underlying plugin instance in
-    // case there was no previous explicit initialization. The initialization is
-    // thread safe.
-    static PluginTy Plugin;
-
-    assert(SpecificPlugin && "Plugin is not active");
-    return *SpecificPlugin;
-  }
-
-  /// Get a reference to the plugin with a specific plugin-specific type.
-  template <typename Ty> static Ty &get() { return static_cast<Ty &>(get()); }
-
-  /// Indicate whether the plugin is active.
-  static bool isActive() { return SpecificPlugin != nullptr; }
-
-  /// Create a plugin instance.
-  static GenericPluginTy *createPlugin();
-};
-
 /// Auxiliary interface class for GenericDeviceResourceManagerTy. This class
 /// acts as a reference to a device resource, such as a stream, and requires
 /// some basic functions to be implemented. The derived class should define an
diff --git a/offload/plugins-nextgen/common/include/Utils/ELF.h b/offload/plugins-nextgen/common/include/Utils/ELF.h
index f87e0a5ed02b..dcfdb5bd7b03 100644
--- a/offload/plugins-nextgen/common/include/Utils/ELF.h
+++ b/offload/plugins-nextgen/common/include/Utils/ELF.h
@@ -13,8 +13,6 @@
 #ifndef LLVM_OPENMP_LIBOMPTARGET_PLUGINS_ELF_UTILS_H
 #define LLVM_OPENMP_LIBOMPTARGET_PLUGINS_ELF_UTILS_H
 
-#include "Shared/PluginAPI.h"
-
 #include "llvm/Object/ELF.h"
 #include "llvm/Object/ELFObjectFile.h"
 
diff --git a/offload/plugins-nextgen/common/src/JIT.cpp b/offload/plugins-nextgen/common/src/JIT.cpp
index 9eb610cab4de..9d58e6060646 100644
--- a/offload/plugins-nextgen/common/src/JIT.cpp
+++ b/offload/plugins-nextgen/common/src/JIT.cpp
@@ -56,28 +56,6 @@ bool isImageBitcode(const __tgt_device_image &Image) {
   return identify_magic(Binary) == file_magic::bitcode;
 }
 
-std::once_flag InitFlag;
-
-void init(Triple TT) {
-  codegen::RegisterCodeGenFlags();
-#ifdef LIBOMPTARGET_JIT_NVPTX
-  if (TT.isNVPTX()) {
-    LLVMInitializeNVPTXTargetInfo();
-    LLVMInitializeNVPTXTarget();
-    LLVMInitializeNVPTXTargetMC();
-    LLVMInitializeNVPTXAsmPrinter();
-  }
-#endif
-#ifdef LIBOMPTARGET_JIT_AMDGPU
-  if (TT.isAMDGPU()) {
-    LLVMInitializeAMDGPUTargetInfo();
-    LLVMInitializeAMDGPUTarget();
-    LLVMInitializeAMDGPUTargetMC();
-    LLVMInitializeAMDGPUAsmPrinter();
-  }
-#endif
-}
-
 Expected<std::unique_ptr<Module>>
 createModuleFromMemoryBuffer(std::unique_ptr<MemoryBuffer> &MB,
                              LLVMContext &Context) {
@@ -148,7 +126,23 @@ createTargetMachine(Module &M, std::string CPU, unsigned OptLevel) {
 } // namespace
 
 JITEngine::JITEngine(Triple::ArchType TA) : TT(Triple::getArchTypeName(TA)) {
-  std::call_once(InitFlag, init, TT);
+  codegen::RegisterCodeGenFlags();
+#ifdef LIBOMPTARGET_JIT_NVPTX
+  if (TT.isNVPTX()) {
+    LLVMInitializeNVPTXTargetInfo();
+    LLVMInitializeNVPTXTarget();
+    LLVMInitializeNVPTXTargetMC();
+    LLVMInitializeNVPTXAsmPrinter();
+  }
+#endif
+#ifdef LIBOMPTARGET_JIT_AMDGPU
+  if (TT.isAMDGPU()) {
+    LLVMInitializeAMDGPUTargetInfo();
+    LLVMInitializeAMDGPUTarget();
+    LLVMInitializeAMDGPUTargetMC();
+    LLVMInitializeAMDGPUAsmPrinter();
+  }
+#endif
 }
 
 void JITEngine::opt(TargetMachine *TM, TargetLibraryInfoImpl *TLII, Module &M,
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 8de93ba17a56..fae197527850 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -13,7 +13,6 @@
 #include "Shared/APITypes.h"
 #include "Shared/Debug.h"
 #include "Shared/Environment.h"
-#include "Shared/PluginAPI.h"
 
 #include "GlobalHandler.h"
 #include "JIT.h"
@@ -39,8 +38,6 @@ using namespace omp;
 using namespace target;
 using namespace plugin;
 
-GenericPluginTy *PluginTy::SpecificPlugin = nullptr;
-
 // TODO: Fix any thread safety issues for multi-threaded kernel recording.
 struct RecordReplayTy {
 
@@ -2035,205 +2032,3 @@ bool llvm::omp::target::plugin::libomptargetSupportsRPC() {
   return false;
 #endif
 }
-
-/// Exposed library API function, basically wrappers around the GenericDeviceTy
-/// functionality with the same name. All non-async functions are redirected
-/// to the async versions right away with a NULL AsyncInfoPtr.
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-int32_t __tgt_rtl_init_plugin() {
-  auto Err = PluginTy::initIfNeeded();
-  if (Err) {
-    [[maybe_unused]] std::string ErrStr = toString(std::move(Err));
-    DP("Failed to init plugin: %s", ErrStr.c_str());
-    return OFFLOAD_FAIL;
-  }
-
-  return OFFLOAD_SUCCESS;
-}
-
-int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) {
-  if (!PluginTy::isActive())
-    return false;
-
-  return PluginTy::get().is_valid_binary(Image);
-}
-
-int32_t __tgt_rtl_init_device(int32_t DeviceId) {
-  return PluginTy::get().init_device(DeviceId);
-}
-
-int32_t __tgt_rtl_number_of_devices() {
-  return PluginTy::get().number_of_devices();
-}
-
-int64_t __tgt_rtl_init_requires(int64_t RequiresFlags) {
-  return PluginTy::get().init_requires(RequiresFlags);
-}
-
-int32_t __tgt_rtl_is_data_exchangable(int32_t SrcDeviceId,
-                                      int32_t DstDeviceId) {
-  return PluginTy::get().is_data_exchangable(SrcDeviceId, DstDeviceId);
-}
-
-int32_t __tgt_rtl_initialize_record_replay(int32_t DeviceId, int64_t MemorySize,
-                                           void *VAddr, bool isRecord,
-                                           bool SaveOutput,
-                                           uint64_t &ReqPtrArgOffset) {
-  return PluginTy::get().initialize_record_replay(
-      DeviceId, MemorySize, VAddr, isRecord, SaveOutput, ReqPtrArgOffset);
-}
-
-int32_t __tgt_rtl_load_binary(int32_t DeviceId, __tgt_device_image *TgtImage,
-                              __tgt_device_binary *Binary) {
-  return PluginTy::get().load_binary(DeviceId, TgtImage, Binary);
-}
-
-void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr,
-                           int32_t Kind) {
-  return PluginTy::get().data_alloc(DeviceId, Size, HostPtr, Kind);
-}
-
-int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind) {
-  return PluginTy::get().data_delete(DeviceId, TgtPtr, Kind);
-}
-
-int32_t __tgt_rtl_data_lock(int32_t DeviceId, void *Ptr, int64_t Size,
-                            void **LockedPtr) {
-  return PluginTy::get().data_lock(DeviceId, Ptr, Size, LockedPtr);
-}
-
-int32_t __tgt_rtl_data_unlock(int32_t DeviceId, void *Ptr) {
-  return PluginTy::get().data_unlock(DeviceId, Ptr);
-}
-
-int32_t __tgt_rtl_data_notify_mapped(int32_t DeviceId, void *HstPtr,
-                                     int64_t Size) {
-  return PluginTy::get().data_notify_mapped(DeviceId, HstPtr, Size);
-}
-
-int32_t __tgt_rtl_data_notify_unmapped(int32_t DeviceId, void *HstPtr) {
-  return PluginTy::get().data_notify_unmapped(DeviceId, HstPtr);
-}
-
-int32_t __tgt_rtl_data_submit(int32_t DeviceId, void *TgtPtr, void *HstPtr,
-                              int64_t Size) {
-  return PluginTy::get().data_submit(DeviceId, TgtPtr, HstPtr, Size);
-}
-
-int32_t __tgt_rtl_data_submit_async(int32_t DeviceId, void *TgtPtr,
-                                    void *HstPtr, int64_t Size,
-                                    __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().data_submit_async(DeviceId, TgtPtr, HstPtr, Size,
-                                           AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr,
-                                int64_t Size) {
-  return PluginTy::get().data_retrieve(DeviceId, HstPtr, TgtPtr, Size);
-}
-
-int32_t __tgt_rtl_data_retrieve_async(int32_t DeviceId, void *HstPtr,
-                                      void *TgtPtr, int64_t Size,
-                                      __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().data_retrieve_async(DeviceId, HstPtr, TgtPtr, Size,
-                                             AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_data_exchange(int32_t SrcDeviceId, void *SrcPtr,
-                                int32_t DstDeviceId, void *DstPtr,
-                                int64_t Size) {
-  return PluginTy::get().data_exchange(SrcDeviceId, SrcPtr, DstDeviceId, DstPtr,
-                                       Size);
-}
-
-int32_t __tgt_rtl_data_exchange_async(int32_t SrcDeviceId, void *SrcPtr,
-                                      int DstDeviceId, void *DstPtr,
-                                      int64_t Size,
-                                      __tgt_async_info *AsyncInfo) {
-  return PluginTy::get().data_exchange_async(SrcDeviceId, SrcPtr, DstDeviceId,
-                                             DstPtr, Size, AsyncInfo);
-}
-
-int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
-                                void **TgtArgs, ptrdiff_t *TgtOffsets,
-                                KernelArgsTy *KernelArgs,
-                                __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().launch_kernel(DeviceId, TgtEntryPtr, TgtArgs,
-                                       TgtOffsets, KernelArgs, AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_synchronize(int32_t DeviceId,
-                              __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().synchronize(DeviceId, AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_query_async(int32_t DeviceId,
-                              __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().query_async(DeviceId, AsyncInfoPtr);
-}
-
-void __tgt_rtl_print_device_info(int32_t DeviceId) {
-  PluginTy::get().print_device_info(DeviceId);
-}
-
-int32_t __tgt_rtl_create_event(int32_t DeviceId, void **EventPtr) {
-  return PluginTy::get().create_event(DeviceId, EventPtr);
-}
-
-int32_t __tgt_rtl_record_event(int32_t DeviceId, void *EventPtr,
-                               __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().record_event(DeviceId, EventPtr, AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_wait_event(int32_t DeviceId, void *EventPtr,
-                             __tgt_async_info *AsyncInfoPtr) {
-  return PluginTy::get().wait_event(DeviceId, EventPtr, AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_sync_event(int32_t DeviceId, void *EventPtr) {
-  return PluginTy::get().sync_event(DeviceId, EventPtr);
-}
-
-int32_t __tgt_rtl_destroy_event(int32_t DeviceId, void *EventPtr) {
-  return PluginTy::get().destroy_event(DeviceId, EventPtr);
-}
-
-void __tgt_rtl_set_info_flag(uint32_t NewInfoLevel) {
-  return PluginTy::get().set_info_flag(NewInfoLevel);
-}
-
-int32_t __tgt_rtl_init_async_info(int32_t DeviceId,
-                                  __tgt_async_info **AsyncInfoPtr) {
-  return PluginTy::get().init_async_info(DeviceId, AsyncInfoPtr);
-}
-
-int32_t __tgt_rtl_init_device_info(int32_t DeviceId,
-                                   __tgt_device_info *DeviceInfo,
-                                   const char **ErrStr) {
-  return PluginTy::get().init_device_info(DeviceId, DeviceInfo, ErrStr);
-}
-
-int32_t __tgt_rtl_set_device_offset(int32_t DeviceIdOffset) {
-  return PluginTy::get().set_device_offset(DeviceIdOffset);
-}
-
-int32_t __tgt_rtl_use_auto_zero_copy(int32_t DeviceId) {
-  return PluginTy::get().use_auto_zero_copy(DeviceId);
-}
-
-int32_t __tgt_rtl_get_global(__tgt_device_binary Binary, uint64_t Size,
-                             const char *Name, void **DevicePtr) {
-  return PluginTy::get().get_global(Binary, Size, Name, DevicePtr);
-}
-
-int32_t __tgt_rtl_get_function(__tgt_device_binary Binary, const char *Name,
-                               void **KernelPtr) {
-  return PluginTy::get().get_function(Binary, Name, KernelPtr);
-}
-
-#ifdef __cplusplus
-}
-#endif
diff --git a/offload/plugins-nextgen/cuda/CMakeLists.txt b/offload/plugins-nextgen/cuda/CMakeLists.txt
index 0284bd22d2a4..dd684bb22343 100644
--- a/offload/plugins-nextgen/cuda/CMakeLists.txt
+++ b/offload/plugins-nextgen/cuda/CMakeLists.txt
@@ -51,8 +51,3 @@ else()
   libomptarget_say("Not generating NVIDIA tests, no supported devices detected."
                    " Use 'LIBOMPTARGET_FORCE_NVIDIA_TESTS' to override.")
 endif()
-
-# Install plugin under the lib destination folder.
-install(TARGETS omptarget.rtl.cuda LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
-set_target_properties(omptarget.rtl.cuda PROPERTIES
-  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index fc74c6aa23fd..b260334baa18 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -1342,10 +1342,6 @@ struct CUDAPluginTy final : public GenericPluginTy {
       return 0;
     }
 
-#ifdef OMPT_SUPPORT
-    ompt::connectLibrary();
-#endif
-
     if (Res == CUDA_ERROR_NO_DEVICE) {
       // Do not initialize if there are no devices.
       DP("There are no devices supporting CUDA.\n");
@@ -1390,6 +1386,8 @@ struct CUDAPluginTy final : public GenericPluginTy {
     return Triple::nvptx64;
   }
 
+  const char *getName() const override { return GETNAME(TARGET_NAME); }
+
   /// Check whether the image is compatible with the available CUDA devices.
   Expected<bool> isELFCompatible(StringRef Image) const override {
     auto ElfOrErr =
@@ -1495,8 +1493,6 @@ Error CUDADeviceTy::dataExchangeImpl(const void *SrcPtr,
   return Plugin::check(Res, "Error in cuMemcpyDtoDAsync: %s");
 }
 
-GenericPluginTy *PluginTy::createPlugin() { return new CUDAPluginTy(); }
-
 template <typename... ArgsTy>
 static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
   CUresult ResultCode = static_cast<CUresult>(Code);
@@ -1516,3 +1512,9 @@ static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
 } // namespace target
 } // namespace omp
 } // namespace llvm
+
+extern "C" {
+llvm::omp::target::plugin::GenericPluginTy *createPlugin_cuda() {
+  return new llvm::omp::target::plugin::CUDAPluginTy();
+}
+}
diff --git a/offload/plugins-nextgen/host/CMakeLists.txt b/offload/plugins-nextgen/host/CMakeLists.txt
index 1d000442c84d..72b5681283fe 100644
--- a/offload/plugins-nextgen/host/CMakeLists.txt
+++ b/offload/plugins-nextgen/host/CMakeLists.txt
@@ -31,14 +31,6 @@ else()
   target_include_directories(omptarget.rtl.host PRIVATE dynamic_ffi)
 endif()
 
-# Install plugin under the lib destination folder.
-install(TARGETS omptarget.rtl.host
-        LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")
-set_target_properties(omptarget.rtl.host PROPERTIES
-  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.."
-  POSITION_INDEPENDENT_CODE ON
-  CXX_VISIBILITY_PRESET protected)
-
 target_include_directories(omptarget.rtl.host PRIVATE
                            ${LIBOMPTARGET_INCLUDE_DIR})
 
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index 4bdcae3dd6a1..409b44b1640a 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -385,10 +385,6 @@ struct GenELF64PluginTy final : public GenericPluginTy {
 
   /// Initialize the plugin and return the number of devices.
   Expected<int32_t> initImpl() override {
-#ifdef OMPT_SUPPORT
-    ompt::connectLibrary();
-#endif
-
 #ifdef USES_DYNAMIC_FFI
     if (auto Err = Plugin::check(ffi_init(), "Failed to initialize libffi"))
       return std::move(Err);
@@ -445,9 +441,9 @@ struct GenELF64PluginTy final : public GenericPluginTy {
     return llvm::Triple::UnknownArch;
 #endif
   }
-};
 
-GenericPluginTy *PluginTy::createPlugin() { return new GenELF64PluginTy(); }
+  const char *getName() const override { return GETNAME(TARGET_NAME); }
+};
 
 template <typename... ArgsTy>
 static Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) {
@@ -462,3 +458,9 @@ static Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) {
 } // namespace target
 } // namespace omp
 } // namespace llvm
+
+extern "C" {
+llvm::omp::target::plugin::GenericPluginTy *createPlugin_host() {
+  return new llvm::omp::target::plugin::GenELF64PluginTy();
+}
+}
diff --git a/offload/src/CMakeLists.txt b/offload/src/CMakeLists.txt
index eda5a85ff1ab..8fe6d19d83eb 100644
--- a/offload/src/CMakeLists.txt
+++ b/offload/src/CMakeLists.txt
@@ -65,6 +65,10 @@ target_compile_definitions(omptarget PRIVATE
   DEBUG_PREFIX="omptarget"
 )
 
+foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD)
+  target_link_libraries(omptarget PRIVATE omptarget.rtl.${plugin})
+endforeach()
+
 target_compile_options(omptarget PUBLIC ${offload_compile_flags})
 target_link_options(omptarget PUBLIC ${offload_link_flags})
 
diff --git a/offload/src/OffloadRTL.cpp b/offload/src/OffloadRTL.cpp
index dd75b1b18150..29b573a27d08 100644
--- a/offload/src/OffloadRTL.cpp
+++ b/offload/src/OffloadRTL.cpp
@@ -50,6 +50,7 @@ void deinitRuntime() {
 
   if (RefCount == 1) {
     DP("Deinit offload library!\n");
+    PM->deinit();
     delete PM;
     PM = nullptr;
   }
diff --git a/offload/src/OpenMP/InteropAPI.cpp b/offload/src/OpenMP/InteropAPI.cpp
index 1a995cde7816..bdbc440c64a2 100644
--- a/offload/src/OpenMP/InteropAPI.cpp
+++ b/offload/src/OpenMP/InteropAPI.cpp
@@ -230,14 +230,14 @@ void __tgt_interop_init(ident_t *LocRef, int32_t Gtid,
   }
 
   DeviceTy &Device = *DeviceOrErr;
-  if (!Device.RTL || !Device.RTL->init_device_info ||
+  if (!Device.RTL ||
       Device.RTL->init_device_info(DeviceId, &(InteropPtr)->device_info,
                                    &(InteropPtr)->err_str)) {
     delete InteropPtr;
     InteropPtr = omp_interop_none;
   }
   if (InteropType == kmp_interop_type_tasksync) {
-    if (!Device.RTL || !Device.RTL->init_async_info ||
+    if (!Device.RTL ||
         Device.RTL->init_async_info(DeviceId, &(InteropPtr)->async_info)) {
       delete InteropPtr;
       InteropPtr = omp_interop_none;
diff --git a/offload/src/PluginManager.cpp b/offload/src/PluginManager.cpp
index dbb556c179e5..191afa345641 100644
--- a/offload/src/PluginManager.cpp
+++ b/offload/src/PluginManager.cpp
@@ -23,85 +23,25 @@ using namespace llvm::sys;
 
 PluginManager *PM = nullptr;
 
-Expected<std::unique_ptr<PluginAdaptorTy>>
-PluginAdaptorTy::create(const std::string &Name) {
-  DP("Attempting to load library '%s'...\n", Name.c_str());
-  TIMESCOPE_WITH_NAME_AND_IDENT(Name, (const ident_t *)nullptr);
-
-  std::string ErrMsg;
-  auto LibraryHandler = std::make_unique<DynamicLibrary>(
-      DynamicLibrary::getPermanentLibrary(Name.c_str(), &ErrMsg));
-
-  if (!LibraryHandler->isValid()) {
-    // Library does not exist or cannot be found.
-    return createStringError(inconvertibleErrorCode(),
-                             "Unable to load library '%s': %s!\n", Name.c_str(),
-                             ErrMsg.c_str());
-  }
-
-  DP("Successfully loaded library '%s'!\n", Name.c_str());
-  auto PluginAdaptor = std::unique_ptr<PluginAdaptorTy>(
-      new PluginAdaptorTy(Name, std::move(LibraryHandler)));
-  if (auto Err = PluginAdaptor->init())
-    return Err;
-  return std::move(PluginAdaptor);
-}
-
-PluginAdaptorTy::PluginAdaptorTy(const std::string &Name,
-                                 std::unique_ptr<llvm::sys::DynamicLibrary> DL)
-    : Name(Name), LibraryHandler(std::move(DL)) {}
-
-Error PluginAdaptorTy::init() {
-
-#define PLUGIN_API_HANDLE(NAME)                                                \
-  NAME = reinterpret_cast<decltype(NAME)>(                                     \
-      LibraryHandler->getAddressOfSymbol(GETNAME(__tgt_rtl_##NAME)));          \
-  if (!NAME) {                                                                 \
-    return createStringError(inconvertibleErrorCode(),                         \
-                             "Invalid plugin as necessary interface function " \
-                             "(%s) was not found.\n",                          \
-                             std::string(#NAME).c_str());                      \
-  }
-
-#include "Shared/PluginAPI.inc"
-#undef PLUGIN_API_HANDLE
-
-  // Remove plugin on failure to call optional init_plugin
-  int32_t Rc = init_plugin();
-  if (Rc != OFFLOAD_SUCCESS) {
-    return createStringError(inconvertibleErrorCode(),
-                             "Unable to initialize library '%s': %u!\n",
-                             Name.c_str(), Rc);
-  }
-
-  // No devices are supported by this RTL?
-  int32_t NumberOfPluginDevices = number_of_devices();
-  if (!NumberOfPluginDevices) {
-    return createStringError(inconvertibleErrorCode(),
-                             "No devices supported in this RTL\n");
-  }
-
-  DP("Registered '%s' with %d plugin visible devices!\n", Name.c_str(),
-     NumberOfPluginDevices);
-  return Error::success();
-}
+// Every plugin exports this method to create an instance of the plugin type.
+#define PLUGIN_TARGET(Name) extern "C" GenericPluginTy *createPlugin_##Name();
+#include "Shared/Targets.def"
 
 void PluginManager::init() {
   TIMESCOPE();
   DP("Loading RTLs...\n");
 
-  // Attempt to open all the plugins and, if they exist, check if the interface
-  // is correct and if they are supporting any devices.
+  // Attempt to create an instance of each supported plugin.
 #define PLUGIN_TARGET(Name)                                                    \
   do {                                                                         \
-    auto PluginAdaptorOrErr =                                                  \
-        PluginAdaptorTy::create("libomptarget.rtl." #Name ".so");              \
-    if (!PluginAdaptorOrErr) {                                                 \
-      [[maybe_unused]] std::string InfoMsg =                                   \
-          toString(PluginAdaptorOrErr.takeError());                            \
-      DP("%s", InfoMsg.c_str());                                               \
+    auto Plugin = std::unique_ptr<GenericPluginTy>(createPlugin_##Name());     \
+    if (auto Err = Plugin->init()) {                                           \
+      [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));         \
+      DP("Failed to init plugin: %s\n", InfoMsg.c_str());                      \
     } else {                                                                   \
-      PluginAdaptors.push_back(std::move(*PluginAdaptorOrErr));                \
+      DP("Registered plugin %s with %d visible device(s)\n",                   \
+         Plugin->getName(), Plugin->number_of_devices());                      \
+      Plugins.emplace_back(std::move(Plugin));                                 \
     }                                                                          \
   } while (false);
 #include "Shared/Targets.def"
@@ -109,15 +49,29 @@ void PluginManager::init() {
   DP("RTLs loaded!\n");
 }
 
-void PluginManager::initDevices(PluginAdaptorTy &RTL) {
+void PluginManager::deinit() {
+  TIMESCOPE();
+  DP("Unloading RTLs...\n");
+
+  for (auto &Plugin : Plugins) {
+    if (auto Err = Plugin->deinit()) {
+      [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));
+      DP("Failed to deinit plugin: %s\n", InfoMsg.c_str());
+    }
+    Plugin.release();
+  }
+
+  DP("RTLs unloaded!\n");
+}
+
+void PluginManager::initDevices(GenericPluginTy &RTL) {
   // If this RTL has already been initialized.
   if (PM->DeviceOffsets.contains(&RTL))
     return;
   TIMESCOPE();
 
   // If this RTL is not already in use, initialize it.
-  assert(RTL.number_of_devices() > 0 &&
-         "Tried to initialize useless plugin adaptor");
+  assert(RTL.number_of_devices() > 0 && "Tried to initialize useless plugin!");
 
   // Initialize the device information for the RTL we are about to use.
   auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();
@@ -157,13 +111,12 @@ void PluginManager::initDevices(PluginAdaptorTy &RTL) {
 
   DeviceOffsets[&RTL] = DeviceOffset;
   DeviceUsed[&RTL] = NumberOfUserDevices;
-  DP("Plugin adaptor " DPxMOD " has index %d, exposes %d out of %d devices!\n",
-     DPxPTR(RTL.LibraryHandler.get()), DeviceOffset, NumberOfUserDevices,
-     RTL.number_of_devices());
+  DP("Plugin has index %d, exposes %d out of %d devices!\n", DeviceOffset,
+     NumberOfUserDevices, RTL.number_of_devices());
 }
 
 void PluginManager::initAllPlugins() {
-  for (auto &R : PluginAdaptors)
+  for (auto &R : Plugins)
     initDevices(*R);
 }
 
@@ -216,19 +169,22 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) {
     // Obtain the image and information that was previously extracted.
     __tgt_device_image *Img = &DI.getExecutableImage();
 
-    PluginAdaptorTy *FoundRTL = nullptr;
+    GenericPluginTy *FoundRTL = nullptr;
 
     // Scan the RTLs that have associated images until we find one that supports
     // the current image.
-    for (auto &R : PM->pluginAdaptors()) {
+    for (auto &R : PM->plugins()) {
+      if (!R.number_of_devices())
+        continue;
+
       if (!R.is_valid_binary(Img)) {
         DP("Image " DPxMOD " is NOT compatible with RTL %s!\n",
-           DPxPTR(Img->ImageStart), R.Name.c_str());
+           DPxPTR(Img->ImageStart), R.getName());
         continue;
       }
 
       DP("Image " DPxMOD " is compatible with RTL %s!\n",
-         DPxPTR(Img->ImageStart), R.Name.c_str());
+         DPxPTR(Img->ImageStart), R.getName());
 
       PM->initDevices(R);
 
@@ -247,7 +203,7 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) {
           (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin];
 
       DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(Img->ImageStart),
-         R.Name.c_str());
+         R.getName());
 
       registerImageIntoTranslationTable(TransTable, PM->DeviceOffsets[&R],
                                         PM->DeviceUsed[&R], Img);
@@ -282,11 +238,11 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
     // Obtain the image and information that was previously extracted.
     __tgt_device_image *Img = &DI.getExecutableImage();
 
-    PluginAdaptorTy *FoundRTL = NULL;
+    GenericPluginTy *FoundRTL = NULL;
 
     // Scan the RTLs that have associated images until we find one that supports
     // the current image. We only need to scan RTLs that are already being used.
-    for (auto &R : PM->pluginAdaptors()) {
+    for (auto &R : PM->plugins()) {
       if (!DeviceOffsets.contains(&R))
         continue;
 
@@ -296,8 +252,7 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
 
       FoundRTL = &R;
 
-      DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n",
-         DPxPTR(Img->ImageStart), DPxPTR(R.LibraryHandler.get()));
+      DP("Unregistered image " DPxMOD " from RTL\n", DPxPTR(Img->ImageStart));
 
       break;
     }
diff --git a/offload/src/device.cpp b/offload/src/device.cpp
index 44a2facc8d3d..749b4c567f8e 100644
--- a/offload/src/device.cpp
+++ b/offload/src/device.cpp
@@ -64,7 +64,7 @@ int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,
   return OFFLOAD_SUCCESS;
 }
 
-DeviceTy::DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID)
+DeviceTy::DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID)
     : DeviceID(DeviceID), RTL(RTL), RTLDeviceID(RTLDeviceID),
       MappingInfo(*this) {}
 
@@ -192,7 +192,6 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
           RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
   if (!AsyncInfo) {
-    assert(RTL->data_exchange && "RTL->data_exchange is nullptr");
     return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr,
                               Size);
   }
diff --git a/offload/src/interface.cpp b/offload/src/interface.cpp
index 557703632c62..763b051cc6d7 100644
--- a/offload/src/interface.cpp
+++ b/offload/src/interface.cpp
@@ -456,8 +456,6 @@ EXTERN void __tgt_set_info_flag(uint32_t NewInfoLevel) {
   assert(PM && "Runtime not initialized");
   std::atomic<uint32_t> &InfoLevel = getInfoLevelInternal();
   InfoLevel.store(NewInfoLevel);
-  for (auto &R : PM->pluginAdaptors())
-    R.set_info_flag(NewInfoLevel);
 }
 
 EXTERN int __tgt_print_device_info(int64_t DeviceId) {
diff --git a/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp b/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
index 761e04e4c7bb..1e9a6a84d805 100644
--- a/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
+++ b/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp
@@ -13,8 +13,6 @@
 
 #include "omptarget.h"
 
-#include "Shared/PluginAPI.h"
-
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/MemoryBuffer.h"
diff --git a/offload/unittests/Plugins/NextgenPluginsTest.cpp b/offload/unittests/Plugins/NextgenPluginsTest.cpp
index 635bd1637c90..479b3f614aed 100644
--- a/offload/unittests/Plugins/NextgenPluginsTest.cpp
+++ b/offload/unittests/Plugins/NextgenPluginsTest.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Shared/PluginAPI.h"
 #include "omptarget.h"
 #include "gtest/gtest.h"
 
-- 
GitLab


From fdede92d435068f31e7ea3a1dddb46d50343dd8c Mon Sep 17 00:00:00 2001
From: Zequan Wu <zequanwu@google.com>
Date: Thu, 9 May 2024 10:42:53 -0400
Subject: [PATCH 0300/1206] [lldb][DWARF] Sort ranges list in dwarf 5. (#91343)

Dwarf 5 says "There is no requirement that the entries be ordered in any
particular way" in 2.17.3 Non-Contiguous Address Ranges for rnglist.
Some places assume the ranges are already sorted but it's not.

For example, when [parsing function
info](https://github.com/llvm/llvm-project/blob/bc8a42762057d7036f6871211e62b1c3efb2738a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp#L922-L927),
it validates low and hi address of the function: GetMinRangeBase returns
the first range entry base and GetMaxRangeEnd returns the last range
end. If low >= hi, it stops parsing this function. This causes missing
inline stack frames for those functions.

This change fixes it and updates the test
`lldb/test/Shell/SymbolFile/DWARF/x86/debug_rnglists.s` so that two
ranges in `.debug_rnglists` are out of order and `image lookup -v -s
lookup_rnglists` is still able to produce sorted ranges for the inner
block.
---
 lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp    | 1 +
 lldb/test/Shell/SymbolFile/DWARF/x86/debug_rnglists.s | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
index dabc595427df..3a57ec970b07 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
@@ -1062,6 +1062,7 @@ DWARFUnit::FindRnglistFromOffset(dw_offset_t offset) {
     ranges.Append(DWARFRangeList::Entry(llvm_range.LowPC,
                                         llvm_range.HighPC - llvm_range.LowPC));
   }
+  ranges.Sort();
   return ranges;
 }
 
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/debug_rnglists.s b/lldb/test/Shell/SymbolFile/DWARF/x86/debug_rnglists.s
index 89b5d94c68c3..af8a1796f3ab 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/x86/debug_rnglists.s
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/debug_rnglists.s
@@ -124,12 +124,12 @@ lookup_rnglists2:
 .Lrnglists_table_base0:
         .long   .Ldebug_ranges0-.Lrnglists_table_base0
 .Ldebug_ranges0:
-        .byte   4                       # DW_RLE_offset_pair
-        .uleb128 .Lblock1_begin-rnglists  #   starting offset
-        .uleb128 .Lblock1_end-rnglists    #   ending offset
         .byte   4                       # DW_RLE_offset_pair
         .uleb128 .Lblock2_begin-rnglists  #   starting offset
         .uleb128 .Lblock2_end-rnglists    #   ending offset
+        .byte   4                       # DW_RLE_offset_pair
+        .uleb128 .Lblock1_begin-rnglists  #   starting offset
+        .uleb128 .Lblock1_end-rnglists    #   ending offset
         .byte   0                       # DW_RLE_end_of_list
 .Ldebug_rnglist_table_end0:
 
-- 
GitLab


From aa9d467abaeb440dc70b64c0f35b8d5e731f3a19 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Thu, 9 May 2024 16:03:16 +0100
Subject: [PATCH 0301/1206] Revert "[AArch64] NFC: Add RUN lines for
 streaming-compatible code." (#91599)

Reverts llvm/llvm-project#90617
---
 ...streaming-mode-fixed-length-and-combine.ll |   83 -
 ...treaming-mode-fixed-length-bit-counting.ll |  457 ---
 ...sve-streaming-mode-fixed-length-bitcast.ll |   97 -
 ...e-streaming-mode-fixed-length-bitselect.ll |   12 -
 ...treaming-mode-fixed-length-build-vector.ll |   88 -
 .../sve-streaming-mode-fixed-length-concat.ll |  228 --
 ...e-streaming-mode-fixed-length-ext-loads.ll |  138 -
 ...ing-mode-fixed-length-extract-subvector.ll |  136 -
 ...ng-mode-fixed-length-extract-vector-elt.ll |   53 -
 ...e-streaming-mode-fixed-length-fcopysign.ll |  171 --
 ...ve-streaming-mode-fixed-length-fp-arith.ll |  989 -------
 ...streaming-mode-fixed-length-fp-compares.ll | 2486 -----------------
 ...-streaming-mode-fixed-length-fp-convert.ll |   12 -
 ...aming-mode-fixed-length-fp-extend-trunc.ll |  270 --
 .../sve-streaming-mode-fixed-length-fp-fma.ll |  116 -
 ...e-streaming-mode-fixed-length-fp-minmax.ll |  965 -------
 ...eaming-mode-fixed-length-fp-reduce-fa64.ll |   25 -
 ...e-streaming-mode-fixed-length-fp-reduce.ll | 1058 -------
 ...streaming-mode-fixed-length-fp-rounding.ll |  547 ----
 ...e-streaming-mode-fixed-length-fp-select.ll |   99 -
 ...e-streaming-mode-fixed-length-fp-to-int.ll |  925 ------
 ...-streaming-mode-fixed-length-fp-vselect.ll |  199 --
 ...ing-mode-fixed-length-insert-vector-elt.ll |  172 --
 ...e-streaming-mode-fixed-length-int-arith.ll |  371 ---
 ...treaming-mode-fixed-length-int-compares.ll |  154 -
 ...sve-streaming-mode-fixed-length-int-div.ll | 1145 --------
 ...streaming-mode-fixed-length-int-extends.ll |  763 -----
 ...eaming-mode-fixed-length-int-immediates.ll |  546 ----
 ...sve-streaming-mode-fixed-length-int-log.ll |  229 --
 ...-streaming-mode-fixed-length-int-minmax.ll |  325 ---
 ...ing-mode-fixed-length-int-mla-neon-fa64.ll |    7 -
 ...ve-streaming-mode-fixed-length-int-mulh.ll |  291 --
 ...-streaming-mode-fixed-length-int-reduce.ll |  415 ---
 ...sve-streaming-mode-fixed-length-int-rem.ll | 1631 -----------
 ...-streaming-mode-fixed-length-int-select.ll |  137 -
 ...-streaming-mode-fixed-length-int-shifts.ll |  313 ---
 ...e-streaming-mode-fixed-length-int-to-fp.ll |  822 ------
 ...streaming-mode-fixed-length-int-vselect.ll |  123 -
 ...reaming-mode-fixed-length-limit-duplane.ll |   27 -
 .../sve-streaming-mode-fixed-length-loads.ll  |  127 -
 ...-streaming-mode-fixed-length-log-reduce.ll |  436 ---
 ...streaming-mode-fixed-length-masked-load.ll |  954 -------
 ...treaming-mode-fixed-length-masked-store.ll |  774 -----
 ...eaming-mode-fixed-length-optimize-ptrue.ll |  216 --
 ...streaming-mode-fixed-length-permute-rev.ll |  127 -
 ...g-mode-fixed-length-permute-zip-uzp-trn.ll |  320 ---
 .../sve-streaming-mode-fixed-length-ptest.ll  |   72 -
 .../sve-streaming-mode-fixed-length-rev.ll    |  159 --
 ...e-streaming-mode-fixed-length-sdiv-pow2.ll |  132 -
 ...treaming-mode-fixed-length-splat-vector.ll |  182 --
 .../sve-streaming-mode-fixed-length-stores.ll |  136 -
 ...e-streaming-mode-fixed-length-subvector.ll |  133 -
 ...treaming-mode-fixed-length-trunc-stores.ll |   38 -
 .../sve-streaming-mode-fixed-length-trunc.ll  |  389 ---
 ...eaming-mode-fixed-length-vector-shuffle.ll |  151 -
 .../sve-streaming-mode-test-register-mov.ll   |   21 -
 56 files changed, 20992 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-and-combine.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-and-combine.ll
index fd9259048df5..d81f725eaefc 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-and-combine.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-and-combine.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -15,12 +14,6 @@ define <4 x i8> @vls_sve_and_4xi8(<4 x i8> %b) nounwind {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_4xi8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d1, #0xff000000ff0000
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
  %c = and <4 x i8> %b, <i8 0, i8 255, i8 0, i8 255>
  ret <4 x i8> %c
 }
@@ -34,12 +27,6 @@ define <8 x i8> @vls_sve_and_8xi8(<8 x i8> %b) nounwind {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_8xi8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d1, #0xff00ff00ff00ff00
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
  %c = and <8 x i8> %b, <i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255>
  ret <8 x i8> %c
 }
@@ -53,12 +40,6 @@ define <16 x i8> @vls_sve_and_16xi8(<16 x i8> %b) nounwind {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_16xi8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v1.2d, #0xff00ff00ff00ff00
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
  %c = and <16 x i8> %b, <i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255>
  ret <16 x i8> %c
 }
@@ -75,13 +56,6 @@ define <32 x i8> @vls_sve_and_32xi8(<32 x i8> %ap) nounwind {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_32xi8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v2.2d, #0xff00ff00ff00ff00
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    and v1.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
  %b = and <32 x i8> %ap, <i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255,
                          i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255, i8 0, i8 255>
  ret <32 x i8> %b
@@ -99,13 +73,6 @@ define <2 x i16> @vls_sve_and_2xi16(<2 x i16> %b) nounwind {
 ; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_2xi16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov v0.s[0], wzr
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
  %c = and <2 x i16> %b, <i16 0, i16 65535>
  ret <2 x i16> %c
 }
@@ -119,12 +86,6 @@ define <4 x i16> @vls_sve_and_4xi16(<4 x i16> %b) nounwind {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_4xi16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d1, #0xffff0000ffff0000
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
  %c = and <4 x i16> %b, <i16 0, i16 65535, i16 0, i16 65535>
  ret <4 x i16> %c
 }
@@ -138,12 +99,6 @@ define <8 x i16> @vls_sve_and_8xi16(<8 x i16> %b) nounwind {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_8xi16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v1.2d, #0xffff0000ffff0000
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
  %c = and <8 x i16> %b, <i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535>
  ret <8 x i16> %c
 }
@@ -160,13 +115,6 @@ define <16 x i16> @vls_sve_and_16xi16(<16 x i16> %b) nounwind {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_16xi16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v2.2d, #0xffff0000ffff0000
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    and v1.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
  %c = and <16 x i16> %b, <i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535, i16 0, i16 65535>
  ret <16 x i16> %c
 }
@@ -180,13 +128,6 @@ define <2 x i32> @vls_sve_and_2xi32(<2 x i32> %b) nounwind {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_2xi32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov v0.s[0], wzr
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
  %c = and <2 x i32> %b, <i32 0, i32 4294967295>
  ret <2 x i32> %c
 }
@@ -200,12 +141,6 @@ define <4 x i32> @vls_sve_and_4xi32(<4 x i32> %b) nounwind {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_4xi32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v1.2d, #0xffffffff00000000
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
  %c = and <4 x i32> %b, <i32 0, i32 4294967295, i32 0, i32 4294967295>
  ret <4 x i32> %c
 }
@@ -222,13 +157,6 @@ define <8 x i32> @vls_sve_and_8xi32(<8 x i32> %b) nounwind {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_8xi32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v2.2d, #0xffffffff00000000
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    and v1.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
  %c = and <8 x i32> %b, <i32 0, i32 4294967295, i32 0, i32 4294967295, i32 0, i32 4294967295, i32 0, i32 4294967295>
  ret <8 x i32> %c
 }
@@ -242,11 +170,6 @@ define <2 x i64> @vls_sve_and_2xi64(<2 x i64> %b) nounwind {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_2xi64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov v0.d[0], xzr
-; NONEON-NOSVE-NEXT:    ret
  %c = and <2 x i64> %b, <i64 0, i64 18446744073709551615>
  ret <2 x i64> %c
 }
@@ -262,12 +185,6 @@ define <4 x i64> @vls_sve_and_4xi64(<4 x i64> %b) nounwind {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: vls_sve_and_4xi64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov v0.d[0], xzr
-; NONEON-NOSVE-NEXT:    mov v1.d[0], xzr
-; NONEON-NOSVE-NEXT:    ret
  %c = and <4 x i64> %b, <i64 0, i64 18446744073709551615, i64 0, i64 18446744073709551615>
  ret <4 x i64> %c
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll
index 8f0378252a54..d547f99a0230 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -19,16 +18,6 @@ define <4 x i8> @ctlz_v4i8(<4 x i8> %op) {
 ; CHECK-NEXT:    sub z0.h, z0.h, #8 // =0x8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d1, #0xff00ff00ff00ff
-; NONEON-NOSVE-NEXT:    mov w8, #8 // =0x8
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    dup v1.4h, w8
-; NONEON-NOSVE-NEXT:    clz v0.4h, v0.4h
-; NONEON-NOSVE-NEXT:    sub v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i8> @llvm.ctlz.v4i8(<4 x i8> %op)
   ret <4 x i8> %res
 }
@@ -41,11 +30,6 @@ define <8 x i8> @ctlz_v8i8(<8 x i8> %op) {
 ; CHECK-NEXT:    clz z0.b, p0/m, z0.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    clz v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %op)
   ret <8 x i8> %res
 }
@@ -58,11 +42,6 @@ define <16 x i8> @ctlz_v16i8(<16 x i8> %op) {
 ; CHECK-NEXT:    clz z0.b, p0/m, z0.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    clz v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %op)
   ret <16 x i8> %res
 }
@@ -76,14 +55,6 @@ define void @ctlz_v32i8(ptr %a) {
 ; CHECK-NEXT:    clz z1.b, p0/m, z1.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    clz v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    clz v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %op)
   store <32 x i8> %res, ptr %a
@@ -100,16 +71,6 @@ define <2 x i16> @ctlz_v2i16(<2 x i16> %op) {
 ; CHECK-NEXT:    sub z0.s, z0.s, #16 // =0x10
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d1, #0x00ffff0000ffff
-; NONEON-NOSVE-NEXT:    mov w8, #16 // =0x10
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    dup v1.2s, w8
-; NONEON-NOSVE-NEXT:    clz v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    sub v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i16> @llvm.ctlz.v2i16(<2 x i16> %op)
   ret <2 x i16> %res
 }
@@ -122,11 +83,6 @@ define <4 x i16> @ctlz_v4i16(<4 x i16> %op) {
 ; CHECK-NEXT:    clz z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    clz v0.4h, v0.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %op)
   ret <4 x i16> %res
 }
@@ -139,11 +95,6 @@ define <8 x i16> @ctlz_v8i16(<8 x i16> %op) {
 ; CHECK-NEXT:    clz z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    clz v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %op)
   ret <8 x i16> %res
 }
@@ -157,14 +108,6 @@ define void @ctlz_v16i16(ptr %a) {
 ; CHECK-NEXT:    clz z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    clz v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    clz v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %op)
   store <16 x i16> %res, ptr %a
@@ -179,11 +122,6 @@ define <2 x i32> @ctlz_v2i32(<2 x i32> %op) {
 ; CHECK-NEXT:    clz z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    clz v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %op)
   ret <2 x i32> %res
 }
@@ -196,11 +134,6 @@ define <4 x i32> @ctlz_v4i32(<4 x i32> %op) {
 ; CHECK-NEXT:    clz z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    clz v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %op)
   ret <4 x i32> %res
 }
@@ -214,14 +147,6 @@ define void @ctlz_v8i32(ptr %a) {
 ; CHECK-NEXT:    clz z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    clz v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    clz v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %op)
   store <8 x i32> %res, ptr %a
@@ -236,27 +161,6 @@ define <1 x i64> @ctlz_v1i64(<1 x i64> %op) {
 ; CHECK-NEXT:    clz z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushr d1, d0, #1
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ushr d1, d0, #2
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ushr d1, d0, #4
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ushr d1, d0, #8
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ushr d1, d0, #16
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ushr d1, d0, #32
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    mvn v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    cnt v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4h, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.2s, v0.4h
-; NONEON-NOSVE-NEXT:    uaddlp v0.1d, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x i64> @llvm.ctlz.v1i64(<1 x i64> %op)
   ret <1 x i64> %res
 }
@@ -269,27 +173,6 @@ define <2 x i64> @ctlz_v2i64(<2 x i64> %op) {
 ; CHECK-NEXT:    clz z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushr v1.2d, v0.2d, #1
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ushr v1.2d, v0.2d, #2
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ushr v1.2d, v0.2d, #4
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ushr v1.2d, v0.2d, #8
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ushr v1.2d, v0.2d, #16
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ushr v1.2d, v0.2d, #32
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    mvn v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.8h, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    uaddlp v0.2d, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %op)
   ret <2 x i64> %res
 }
@@ -303,46 +186,6 @@ define void @ctlz_v4i64(ptr %a) {
 ; CHECK-NEXT:    clz z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctlz_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ushr v2.2d, v0.2d, #1
-; NONEON-NOSVE-NEXT:    ushr v3.2d, v1.2d, #1
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    orr v1.16b, v1.16b, v3.16b
-; NONEON-NOSVE-NEXT:    ushr v2.2d, v0.2d, #2
-; NONEON-NOSVE-NEXT:    ushr v3.2d, v1.2d, #2
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    orr v1.16b, v1.16b, v3.16b
-; NONEON-NOSVE-NEXT:    ushr v2.2d, v0.2d, #4
-; NONEON-NOSVE-NEXT:    ushr v3.2d, v1.2d, #4
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    orr v1.16b, v1.16b, v3.16b
-; NONEON-NOSVE-NEXT:    ushr v2.2d, v0.2d, #8
-; NONEON-NOSVE-NEXT:    ushr v3.2d, v1.2d, #8
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    orr v1.16b, v1.16b, v3.16b
-; NONEON-NOSVE-NEXT:    ushr v2.2d, v0.2d, #16
-; NONEON-NOSVE-NEXT:    ushr v3.2d, v1.2d, #16
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    orr v1.16b, v1.16b, v3.16b
-; NONEON-NOSVE-NEXT:    ushr v2.2d, v0.2d, #32
-; NONEON-NOSVE-NEXT:    ushr v3.2d, v1.2d, #32
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    orr v1.16b, v1.16b, v3.16b
-; NONEON-NOSVE-NEXT:    mvn v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    mvn v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    cnt v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.8h, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v1.8h, v1.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    uaddlp v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    uaddlp v0.2d, v0.4s
-; NONEON-NOSVE-NEXT:    uaddlp v1.2d, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %op)
   store <4 x i64> %res, ptr %a
@@ -362,14 +205,6 @@ define <4 x i8> @ctpop_v4i8(<4 x i8> %op) {
 ; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d1, #0xff00ff00ff00ff
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    cnt v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4h, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i8> @llvm.ctpop.v4i8(<4 x i8> %op)
   ret <4 x i8> %res
 }
@@ -382,11 +217,6 @@ define <8 x i8> @ctpop_v8i8(<8 x i8> %op) {
 ; CHECK-NEXT:    cnt z0.b, p0/m, z0.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cnt v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> %op)
   ret <8 x i8> %res
 }
@@ -399,11 +229,6 @@ define <16 x i8> @ctpop_v16i8(<16 x i8> %op) {
 ; CHECK-NEXT:    cnt z0.b, p0/m, z0.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %op)
   ret <16 x i8> %res
 }
@@ -417,14 +242,6 @@ define void @ctpop_v32i8(ptr %a) {
 ; CHECK-NEXT:    cnt z1.b, p0/m, z1.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    cnt v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> %op)
   store <32 x i8> %res, ptr %a
@@ -440,15 +257,6 @@ define <2 x i16> @ctpop_v2i16(<2 x i16> %op) {
 ; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d1, #0x00ffff0000ffff
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    cnt v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4h, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.2s, v0.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i16> @llvm.ctpop.v2i16(<2 x i16> %op)
   ret <2 x i16> %res
 }
@@ -461,12 +269,6 @@ define <4 x i16> @ctpop_v4i16(<4 x i16> %op) {
 ; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cnt v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4h, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %op)
   ret <4 x i16> %res
 }
@@ -479,12 +281,6 @@ define <8 x i16> @ctpop_v8i16(<8 x i16> %op) {
 ; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.8h, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %op)
   ret <8 x i16> %res
 }
@@ -498,16 +294,6 @@ define void @ctpop_v16i16(ptr %a) {
 ; CHECK-NEXT:    cnt z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    cnt v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.8h, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v1.8h, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> %op)
   store <16 x i16> %res, ptr %a
@@ -522,13 +308,6 @@ define <2 x i32> @ctpop_v2i32(<2 x i32> %op) {
 ; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cnt v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4h, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.2s, v0.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %op)
   ret <2 x i32> %res
 }
@@ -541,13 +320,6 @@ define <4 x i32> @ctpop_v4i32(<4 x i32> %op) {
 ; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.8h, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %op)
   ret <4 x i32> %res
 }
@@ -561,18 +333,6 @@ define void @ctpop_v8i32(ptr %a) {
 ; CHECK-NEXT:    cnt z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    cnt v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.8h, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v1.8h, v1.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    uaddlp v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %op)
   store <8 x i32> %res, ptr %a
@@ -587,14 +347,6 @@ define <1 x i64> @ctpop_v1i64(<1 x i64> %op) {
 ; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cnt v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4h, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.2s, v0.4h
-; NONEON-NOSVE-NEXT:    uaddlp v0.1d, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %op)
   ret <1 x i64> %res
 }
@@ -607,14 +359,6 @@ define <2 x i64> @ctpop_v2i64(<2 x i64> %op) {
 ; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.8h, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    uaddlp v0.2d, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %op)
   ret <2 x i64> %res
 }
@@ -628,20 +372,6 @@ define void @ctpop_v4i64(ptr %a) {
 ; CHECK-NEXT:    cnt z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ctpop_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    cnt v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.8h, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v1.8h, v1.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    uaddlp v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    uaddlp v0.2d, v0.4s
-; NONEON-NOSVE-NEXT:    uaddlp v1.2d, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %op)
   store <4 x i64> %res, ptr %a
@@ -662,21 +392,6 @@ define <4 x i8> @cttz_v4i8(<4 x i8> %op) {
 ; CHECK-NEXT:    clz z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #256 // =0x100
-; NONEON-NOSVE-NEXT:    dup v1.4h, w8
-; NONEON-NOSVE-NEXT:    mov w8, #1 // =0x1
-; NONEON-NOSVE-NEXT:    dup v2.4h, w8
-; NONEON-NOSVE-NEXT:    mov w8, #16 // =0x10
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    sub v1.4h, v0.4h, v2.4h
-; NONEON-NOSVE-NEXT:    bic v0.8b, v1.8b, v0.8b
-; NONEON-NOSVE-NEXT:    dup v1.4h, w8
-; NONEON-NOSVE-NEXT:    clz v0.4h, v0.4h
-; NONEON-NOSVE-NEXT:    sub v0.4h, v1.4h, v0.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %op)
   ret <4 x i8> %res
 }
@@ -690,14 +405,6 @@ define <8 x i8> @cttz_v8i8(<8 x i8> %op) {
 ; CHECK-NEXT:    clz z0.b, p0/m, z0.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v1.8b, #1
-; NONEON-NOSVE-NEXT:    sub v1.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    bic v0.8b, v1.8b, v0.8b
-; NONEON-NOSVE-NEXT:    cnt v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %op)
   ret <8 x i8> %res
 }
@@ -711,14 +418,6 @@ define <16 x i8> @cttz_v16i8(<16 x i8> %op) {
 ; CHECK-NEXT:    clz z0.b, p0/m, z0.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v1.16b, #1
-; NONEON-NOSVE-NEXT:    sub v1.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    bic v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %op)
   ret <16 x i8> %res
 }
@@ -734,19 +433,6 @@ define void @cttz_v32i8(ptr %a) {
 ; CHECK-NEXT:    clz z1.b, p0/m, z1.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #1
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    sub v3.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    sub v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bic v1.16b, v3.16b, v1.16b
-; NONEON-NOSVE-NEXT:    bic v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    cnt v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %op)
   store <32 x i8> %res, ptr %a
@@ -763,21 +449,6 @@ define <2 x i16> @cttz_v2i16(<2 x i16> %op) {
 ; CHECK-NEXT:    clz z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #65536 // =0x10000
-; NONEON-NOSVE-NEXT:    dup v1.2s, w8
-; NONEON-NOSVE-NEXT:    mov w8, #1 // =0x1
-; NONEON-NOSVE-NEXT:    dup v2.2s, w8
-; NONEON-NOSVE-NEXT:    mov w8, #32 // =0x20
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    sub v1.2s, v0.2s, v2.2s
-; NONEON-NOSVE-NEXT:    bic v0.8b, v1.8b, v0.8b
-; NONEON-NOSVE-NEXT:    dup v1.2s, w8
-; NONEON-NOSVE-NEXT:    clz v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    sub v0.2s, v1.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %op)
   ret <2 x i16> %res
 }
@@ -791,18 +462,6 @@ define <4 x i16> @cttz_v4i16(<4 x i16> %op) {
 ; CHECK-NEXT:    clz z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #1 // =0x1
-; NONEON-NOSVE-NEXT:    dup v1.4h, w8
-; NONEON-NOSVE-NEXT:    mov w8, #16 // =0x10
-; NONEON-NOSVE-NEXT:    sub v1.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    bic v0.8b, v1.8b, v0.8b
-; NONEON-NOSVE-NEXT:    dup v1.4h, w8
-; NONEON-NOSVE-NEXT:    clz v0.4h, v0.4h
-; NONEON-NOSVE-NEXT:    sub v0.4h, v1.4h, v0.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %op)
   ret <4 x i16> %res
 }
@@ -816,18 +475,6 @@ define <8 x i16> @cttz_v8i16(<8 x i16> %op) {
 ; CHECK-NEXT:    clz z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #1 // =0x1
-; NONEON-NOSVE-NEXT:    dup v1.8h, w8
-; NONEON-NOSVE-NEXT:    mov w8, #16 // =0x10
-; NONEON-NOSVE-NEXT:    sub v1.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    bic v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    dup v1.8h, w8
-; NONEON-NOSVE-NEXT:    clz v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    sub v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %op)
   ret <8 x i16> %res
 }
@@ -843,24 +490,6 @@ define void @cttz_v16i16(ptr %a) {
 ; CHECK-NEXT:    clz z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #1 // =0x1
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    mov w8, #16 // =0x10
-; NONEON-NOSVE-NEXT:    sub v3.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    sub v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    bic v1.16b, v3.16b, v1.16b
-; NONEON-NOSVE-NEXT:    bic v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    dup v2.8h, w8
-; NONEON-NOSVE-NEXT:    clz v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    clz v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    sub v1.8h, v2.8h, v1.8h
-; NONEON-NOSVE-NEXT:    sub v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %op)
   store <16 x i16> %res, ptr %a
@@ -876,18 +505,6 @@ define <2 x i32> @cttz_v2i32(<2 x i32> %op) {
 ; CHECK-NEXT:    clz z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #1 // =0x1
-; NONEON-NOSVE-NEXT:    dup v1.2s, w8
-; NONEON-NOSVE-NEXT:    mov w8, #32 // =0x20
-; NONEON-NOSVE-NEXT:    sub v1.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    bic v0.8b, v1.8b, v0.8b
-; NONEON-NOSVE-NEXT:    dup v1.2s, w8
-; NONEON-NOSVE-NEXT:    clz v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    sub v0.2s, v1.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %op)
   ret <2 x i32> %res
 }
@@ -901,18 +518,6 @@ define <4 x i32> @cttz_v4i32(<4 x i32> %op) {
 ; CHECK-NEXT:    clz z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #1 // =0x1
-; NONEON-NOSVE-NEXT:    dup v1.4s, w8
-; NONEON-NOSVE-NEXT:    mov w8, #32 // =0x20
-; NONEON-NOSVE-NEXT:    sub v1.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    bic v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    dup v1.4s, w8
-; NONEON-NOSVE-NEXT:    clz v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    sub v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %op)
   ret <4 x i32> %res
 }
@@ -928,24 +533,6 @@ define void @cttz_v8i32(ptr %a) {
 ; CHECK-NEXT:    clz z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #1 // =0x1
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    mov w8, #32 // =0x20
-; NONEON-NOSVE-NEXT:    sub v3.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    sub v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    bic v1.16b, v3.16b, v1.16b
-; NONEON-NOSVE-NEXT:    bic v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    dup v2.4s, w8
-; NONEON-NOSVE-NEXT:    clz v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    clz v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    sub v1.4s, v2.4s, v1.4s
-; NONEON-NOSVE-NEXT:    sub v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %op)
   store <8 x i32> %res, ptr %a
@@ -961,18 +548,6 @@ define <1 x i64> @cttz_v1i64(<1 x i64> %op) {
 ; CHECK-NEXT:    clz z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #1 // =0x1
-; NONEON-NOSVE-NEXT:    fmov d1, x8
-; NONEON-NOSVE-NEXT:    sub d1, d0, d1
-; NONEON-NOSVE-NEXT:    bic v0.8b, v1.8b, v0.8b
-; NONEON-NOSVE-NEXT:    cnt v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4h, v0.8b
-; NONEON-NOSVE-NEXT:    uaddlp v0.2s, v0.4h
-; NONEON-NOSVE-NEXT:    uaddlp v0.1d, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %op)
   ret <1 x i64> %res
 }
@@ -986,18 +561,6 @@ define <2 x i64> @cttz_v2i64(<2 x i64> %op) {
 ; CHECK-NEXT:    clz z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #1 // =0x1
-; NONEON-NOSVE-NEXT:    dup v1.2d, x8
-; NONEON-NOSVE-NEXT:    sub v1.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    bic v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.8h, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    uaddlp v0.2d, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %op)
   ret <2 x i64> %res
 }
@@ -1013,26 +576,6 @@ define void @cttz_v4i64(ptr %a) {
 ; CHECK-NEXT:    clz z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: cttz_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #1 // =0x1
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    sub v3.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    sub v0.2d, v2.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bic v1.16b, v3.16b, v1.16b
-; NONEON-NOSVE-NEXT:    bic v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    cnt v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    cnt v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v1.8h, v1.16b
-; NONEON-NOSVE-NEXT:    uaddlp v0.8h, v0.16b
-; NONEON-NOSVE-NEXT:    uaddlp v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    uaddlp v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    uaddlp v1.2d, v1.4s
-; NONEON-NOSVE-NEXT:    uaddlp v0.2d, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %op)
   store <4 x i64> %res, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitcast.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitcast.ll
index 64dc7ae117d3..e3cc74f766ee 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitcast.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitcast.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -12,12 +11,6 @@ define void @bitcast_v4i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ld1b { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    st1b { z0.h }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr w8, [x0]
-; NONEON-NOSVE-NEXT:    str w8, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <4 x i8>, ptr %a
   %cast = bitcast <4 x i8> %load to <4 x i8>
   store volatile <4 x i8> %cast, ptr %b
@@ -30,12 +23,6 @@ define void @bitcast_v8i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <8 x i8>, ptr %a
   %cast = bitcast <8 x i8> %load to <8 x i8>
   store volatile <8 x i8> %cast, ptr %b
@@ -48,12 +35,6 @@ define void @bitcast_v16i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <16 x i8>, ptr %a
   %cast = bitcast <16 x i8> %load to <16 x i8>
   store volatile <16 x i8> %cast, ptr %b
@@ -68,14 +49,6 @@ define void @bitcast_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    str q1, [x1, #16]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <32 x i8>, ptr %a
   %cast = bitcast <32 x i8> %load to <32 x i8>
   store volatile <32 x i8> %cast, ptr %b
@@ -99,16 +72,6 @@ define void @bitcast_v2i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    str w8, [x1]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldrh w8, [x0]
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    add x8, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[2], [x8]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
-; NONEON-NOSVE-NEXT:    str s0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <2 x i16>, ptr %a
   %cast = bitcast <2 x i16> %load to <2 x half>
   store volatile <2 x half> %cast, ptr %b
@@ -121,12 +84,6 @@ define void @bitcast_v4i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <4 x i16>, ptr %a
   %cast = bitcast <4 x i16> %load to <4 x half>
   store volatile <4 x half> %cast, ptr %b
@@ -139,12 +96,6 @@ define void @bitcast_v8i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <8 x i16>, ptr %a
   %cast = bitcast <8 x i16> %load to <8 x half>
   store volatile <8 x half> %cast, ptr %b
@@ -159,14 +110,6 @@ define void @bitcast_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    str q1, [x1, #16]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <16 x i16>, ptr %a
   %cast = bitcast <16 x i16> %load to <16 x half>
   store volatile <16 x half> %cast, ptr %b
@@ -179,12 +122,6 @@ define void @bitcast_v2i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <2 x i32>, ptr %a
   %cast = bitcast <2 x i32> %load to <2 x float>
   store volatile <2 x float> %cast, ptr %b
@@ -197,12 +134,6 @@ define void @bitcast_v4i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <4 x i32>, ptr %a
   %cast = bitcast <4 x i32> %load to <4 x float>
   store volatile <4 x float> %cast, ptr %b
@@ -217,14 +148,6 @@ define void @bitcast_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    str q1, [x1, #16]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <8 x i32>, ptr %a
   %cast = bitcast <8 x i32> %load to <8 x float>
   store volatile <8 x float> %cast, ptr %b
@@ -237,12 +160,6 @@ define void @bitcast_v1i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <1 x i64>, ptr %a
   %cast = bitcast <1 x i64> %load to <1 x double>
   store volatile <1 x double> %cast, ptr %b
@@ -255,12 +172,6 @@ define void @bitcast_v2i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <2 x i64>, ptr %a
   %cast = bitcast <2 x i64> %load to <2 x double>
   store volatile <2 x double> %cast, ptr %b
@@ -275,14 +186,6 @@ define void @bitcast_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    str q1, [x1, #16]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitcast_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %load = load volatile <4 x i64>, ptr %a
   %cast = bitcast <4 x i64> %load to <4 x double>
   store volatile <4 x double> %cast, ptr %b
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitselect.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitselect.ll
index 5e06cd62118d..74a4aab15597 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitselect.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bitselect.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64"
 
@@ -31,17 +30,6 @@ define <8 x i32> @fixed_bitselect_v8i32(ptr %pre_cond_ptr, ptr %left_ptr, ptr %r
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fixed_bitselect_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x2]
-; NONEON-NOSVE-NEXT:    neg v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    neg v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v3.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    ret
   %pre_cond = load <8 x i32>, ptr %pre_cond_ptr
   %left = load <8 x i32>, ptr %left_ptr
   %right = load <8 x i32>, ptr %right_ptr
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-build-vector.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-build-vector.ll
index 7a24430a3385..0c490a662a79 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-build-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-build-vector.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -11,12 +10,6 @@ define void @build_vector_7_inc1_v4i1(ptr %a) {
 ; CHECK-NEXT:    mov w8, #5 // =0x5
 ; CHECK-NEXT:    strb w8, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: build_vector_7_inc1_v4i1:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    strb w8, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x i1> <i1 true, i1 false, i1 true, i1 false>, ptr %a, align 1
   ret void
 }
@@ -30,15 +23,6 @@ define void @build_vector_7_inc1_v32i8(ptr %a) {
 ; CHECK-NEXT:    add z1.b, z1.b, #23 // =0x17
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: build_vector_7_inc1_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI1_0
-; NONEON-NOSVE-NEXT:    adrp x9, .LCPI1_1
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI1_0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI1_1]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <32 x i8> <i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38>, ptr %a, align 1
   ret void
 }
@@ -51,15 +35,6 @@ define void @build_vector_0_inc2_v16i16(ptr %a) {
 ; CHECK-NEXT:    add z0.h, z0.h, #16 // =0x10
 ; CHECK-NEXT:    str q0, [x0, #16]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: build_vector_0_inc2_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI2_0
-; NONEON-NOSVE-NEXT:    adrp x9, .LCPI2_1
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI2_0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI2_1]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <16 x i16> <i16 0, i16 2, i16 4, i16 6, i16 8, i16 10, i16 12, i16 14, i16 16, i16 18, i16 20, i16 22, i16 24, i16 26, i16 28, i16 30>, ptr %a, align 2
   ret void
 }
@@ -73,15 +48,6 @@ define void @build_vector_0_dec3_v8i32(ptr %a) {
 ; CHECK-NEXT:    add z1.s, z0.s, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: build_vector_0_dec3_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI3_0
-; NONEON-NOSVE-NEXT:    adrp x9, .LCPI3_1
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI3_0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI3_1]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <8 x i32> <i32 0, i32 -3, i32 -6, i32 -9, i32 -12, i32 -15, i32 -18, i32 -21>, ptr %a, align 4
   ret void
 }
@@ -98,15 +64,6 @@ define void @build_vector_minus2_dec32_v4i64(ptr %a) {
 ; CHECK-NEXT:    add z0.d, z0.d, z2.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: build_vector_minus2_dec32_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI4_0
-; NONEON-NOSVE-NEXT:    adrp x9, .LCPI4_1
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI4_0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI4_1]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x i64> <i64 -2, i64 -34, i64 -66, i64 -98>, ptr %a, align 8
   ret void
 }
@@ -119,15 +76,6 @@ define void @build_vector_no_stride_v4i64(ptr %a) {
 ; CHECK-NEXT:    index z1.d, #0, #4
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: build_vector_no_stride_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI5_0
-; NONEON-NOSVE-NEXT:    adrp x9, .LCPI5_1
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI5_0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI5_1]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x i64> <i64 0, i64 4, i64 1, i64 8>, ptr %a, align 8
   ret void
 }
@@ -141,15 +89,6 @@ define void @build_vector_0_inc2_v16f16(ptr %a) {
 ; CHECK-NEXT:    ldr q1, [x9, :lo12:.LCPI6_1]
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: build_vector_0_inc2_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI6_0
-; NONEON-NOSVE-NEXT:    adrp x9, .LCPI6_1
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI6_0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI6_1]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <16 x half> <half 0.0, half 2.0, half 4.0, half 6.0, half 8.0, half 10.0, half 12.0, half 14.0, half 16.0, half 18.0, half 20.0, half 22.0, half 24.0, half 26.0, half 28.0, half 30.0>, ptr %a, align 2
   ret void
 }
@@ -164,15 +103,6 @@ define void @build_vector_0_dec3_v8f32(ptr %a) {
 ; CHECK-NEXT:    ldr q1, [x9, :lo12:.LCPI7_1]
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: build_vector_0_dec3_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI7_0
-; NONEON-NOSVE-NEXT:    adrp x9, .LCPI7_1
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI7_0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI7_1]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <8 x float> <float 0.0, float -3.0, float -6.0, float -9.0, float -12.0, float -15.0, float -18.0, float -21.0>, ptr %a, align 4
   ret void
 }
@@ -187,15 +117,6 @@ define void @build_vector_minus2_dec32_v4f64(ptr %a) {
 ; CHECK-NEXT:    ldr q1, [x9, :lo12:.LCPI8_1]
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: build_vector_minus2_dec32_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI8_0
-; NONEON-NOSVE-NEXT:    adrp x9, .LCPI8_1
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI8_0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI8_1]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x double> <double -2.0, double -34.0, double -66.0, double -98.0>, ptr %a, align 8
   ret void
 }
@@ -210,15 +131,6 @@ define void @build_vector_no_stride_v4f64(ptr %a) {
 ; CHECK-NEXT:    ldr q1, [x9, :lo12:.LCPI9_1]
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: build_vector_no_stride_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI9_0
-; NONEON-NOSVE-NEXT:    adrp x9, .LCPI9_1
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI9_0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI9_1]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x double> <double 0.0, double 4.0, double 1.0, double 8.0>, ptr %a, align 8
   ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
index ee997228e453..86494c4be501 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -41,11 +40,6 @@ define <8 x i8> @concat_v8i8(<4 x i8> %op1, <4 x i8> %op2)  {
 ; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    uzp1 v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <4 x i8> %op1, <4 x i8> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i8> %res
 }
@@ -59,13 +53,6 @@ define <16 x i8> @concat_v16i8(<8 x i8> %op1, <8 x i8> %op2)  {
 ; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <8 x i8> %op1, <8 x i8> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
                                                                  i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i8> %res
@@ -78,13 +65,6 @@ define void @concat_v32i8(ptr %a, ptr %b, ptr %c)  {
 ; CHECK-NEXT:    ldr q1, [x0]
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x1]
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i8>, ptr %a
   %op2 = load <16 x i8>, ptr %b
   %res = shufflevector <16 x i8> %op1, <16 x i8> %op2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -103,14 +83,6 @@ define void @concat_v64i8(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    stp q0, q1, [x2, #32]
 ; CHECK-NEXT:    stp q3, q2, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v64i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x2, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = shufflevector <32 x i8> %op1, <32 x i8> %op2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -149,11 +121,6 @@ define <4 x i16> @concat_v4i16(<2 x i16> %op1, <2 x i16> %op2)  {
 ; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <2 x i16> %op1, <2 x i16> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x i16> %res
 }
@@ -168,13 +135,6 @@ define <8 x i16> @concat_v8i16(<4 x i16> %op1, <4 x i16> %op2)  {
 ; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <4 x i16> %op1, <4 x i16> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i16> %res
 }
@@ -186,13 +146,6 @@ define void @concat_v16i16(ptr %a, ptr %b, ptr %c)  {
 ; CHECK-NEXT:    ldr q1, [x0]
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x1]
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %op2 = load <8 x i16>, ptr %b
   %res = shufflevector <8 x i16> %op1, <8 x i16> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -209,14 +162,6 @@ define void @concat_v32i16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    stp q0, q1, [x2, #32]
 ; CHECK-NEXT:    stp q3, q2, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v32i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x2, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = shufflevector <16 x i16> %op1, <16 x i16> %op2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -240,11 +185,6 @@ define <2 x i32> @concat_v2i32(<1 x i32> %op1, <1 x i32> %op2)  {
 ; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <1 x i32> %op1, <1 x i32> %op2, <2 x i32> <i32 0, i32 1>
   ret <2 x i32> %res
 }
@@ -259,13 +199,6 @@ define <4 x i32> @concat_v4i32(<2 x i32> %op1, <2 x i32> %op2)  {
 ; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <2 x i32> %op1, <2 x i32> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x i32> %res
 }
@@ -277,13 +210,6 @@ define void @concat_v8i32(ptr %a, ptr %b, ptr %c)  {
 ; CHECK-NEXT:    ldr q1, [x0]
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x1]
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i32>, ptr %a
   %op2 = load <4 x i32>, ptr %b
   %res = shufflevector <4 x i32> %op1, <4 x i32> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -299,14 +225,6 @@ define void @concat_v16i32(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    stp q0, q1, [x2, #32]
 ; CHECK-NEXT:    stp q3, q2, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v16i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x2, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = shufflevector <8 x i32> %op1, <8 x i32> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -329,13 +247,6 @@ define <2 x i64> @concat_v2i64(<1 x i64> %op1, <1 x i64> %op2)  {
 ; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <1 x i64> %op1, <1 x i64> %op2, <2 x i32> <i32 0, i32 1>
   ret <2 x i64> %res
 }
@@ -347,13 +258,6 @@ define void @concat_v4i64(ptr %a, ptr %b, ptr %c)  {
 ; CHECK-NEXT:    ldr q1, [x0]
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x1]
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x i64>, ptr %a
   %op2 = load <2 x i64>, ptr %b
   %res = shufflevector <2 x i64> %op1, <2 x i64> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -369,14 +273,6 @@ define void @concat_v8i64(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    stp q0, q1, [x2, #32]
 ; CHECK-NEXT:    stp q3, q2, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v8i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x2, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = shufflevector <4 x i64> %op1, <4 x i64> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -404,11 +300,6 @@ define <4 x half> @concat_v4f16(<2 x half> %op1, <2 x half> %op2)  {
 ; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <2 x half> %op1, <2 x half> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x half> %res
 }
@@ -422,13 +313,6 @@ define <8 x half> @concat_v8f16(<4 x half> %op1, <4 x half> %op2)  {
 ; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <4 x half> %op1, <4 x half> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x half> %res
 }
@@ -440,13 +324,6 @@ define void @concat_v16f16(ptr %a, ptr %b, ptr %c)  {
 ; CHECK-NEXT:    ldr q1, [x0]
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x1]
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x half>, ptr %a
   %op2 = load <8 x half>, ptr %b
   %res = shufflevector <8 x half> %op1, <8 x half> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -463,14 +340,6 @@ define void @concat_v32f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    stp q0, q1, [x2, #32]
 ; CHECK-NEXT:    stp q3, q2, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v32f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x2, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %res = shufflevector <16 x half> %op1, <16 x half> %op2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -494,11 +363,6 @@ define <2 x float> @concat_v2f32(<1 x float> %op1, <1 x float> %op2)  {
 ; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    zip1 v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <1 x float> %op1, <1 x float> %op2, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %res
 }
@@ -513,13 +377,6 @@ define <4 x float> @concat_v4f32(<2 x float> %op1, <2 x float> %op2)  {
 ; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <2 x float> %op1, <2 x float> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x float> %res
 }
@@ -531,13 +388,6 @@ define void @concat_v8f32(ptr %a, ptr %b, ptr %c)  {
 ; CHECK-NEXT:    ldr q1, [x0]
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x1]
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x float>, ptr %a
   %op2 = load <4 x float>, ptr %b
   %res = shufflevector <4 x float> %op1, <4 x float> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -553,14 +403,6 @@ define void @concat_v16f32(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    stp q0, q1, [x2, #32]
 ; CHECK-NEXT:    stp q3, q2, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v16f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x2, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %res = shufflevector <8 x float> %op1, <8 x float> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -583,13 +425,6 @@ define <2 x double> @concat_v2f64(<1 x double> %op1, <1 x double> %op2)  {
 ; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    ret
   %res = shufflevector <1 x double> %op1, <1 x double> %op2, <2 x i32> <i32 0, i32 1>
   ret <2 x double> %res
 }
@@ -601,13 +436,6 @@ define void @concat_v4f64(ptr %a, ptr %b, ptr %c)  {
 ; CHECK-NEXT:    ldr q1, [x0]
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x1]
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x double>, ptr %a
   %op2 = load <2 x double>, ptr %b
   %res = shufflevector <2 x double> %op1, <2 x double> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -623,14 +451,6 @@ define void @concat_v8f64(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    stp q0, q1, [x2, #32]
 ; CHECK-NEXT:    stp q3, q2, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v8f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x2, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %res = shufflevector <4 x double> %op1, <4 x double> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -648,12 +468,6 @@ define void @concat_v32i8_undef(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v32i8_undef:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i8>, ptr %a
   %res = shufflevector <16 x i8> %op1, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
                                                                     i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
@@ -669,12 +483,6 @@ define void @concat_v16i16_undef(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v16i16_undef:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %res = shufflevector <8 x i16> %op1, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
                                                                     i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -688,12 +496,6 @@ define void @concat_v8i32_undef(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v8i32_undef:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i32>, ptr %a
   %res = shufflevector <4 x i32> %op1, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   store <8 x i32> %res, ptr %b
@@ -706,12 +508,6 @@ define void @concat_v4i64_undef(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v4i64_undef:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x i64>, ptr %a
   %res = shufflevector <2 x i64> %op1, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   store <4 x i64> %res, ptr %b
@@ -728,12 +524,6 @@ define void @concat_v32i8_4op(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v32i8_4op:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i8>, ptr %a
   %shuffle = shufflevector <8 x i8> %op1, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
                                                                       i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -751,12 +541,6 @@ define void @concat_v16i16_4op(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v16i16_4op:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i16>, ptr %a
   %shuffle = shufflevector <4 x i16> %op1, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %res = shufflevector <8 x i16> %shuffle, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -771,12 +555,6 @@ define void @concat_v8i32_4op(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v8i32_4op:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x i32>, ptr %a
   %shuffle = shufflevector <2 x i32> %op1, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = shufflevector <4 x i32> %shuffle, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -790,12 +568,6 @@ define void @concat_v4i64_4op(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: concat_v4i64_4op:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <1 x i64>, ptr %a
   %shuffle = shufflevector <1 x i64> %op1, <1 x i64> undef, <2 x i32> <i32 0, i32 1>
   %res = shufflevector <2 x i64> %shuffle, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll
index 42aa67fb2ab8..0aefba2d4c6a 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -12,12 +11,6 @@ define <8 x i16> @load_zext_v8i8i16(ptr %ap)  {
 ; CHECK-NEXT:    ld1b { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_zext_v8i8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i8>, ptr %ap
   %val = zext <8 x i8> %a to <8 x i16>
   ret <8 x i16> %val
@@ -30,12 +23,6 @@ define <4 x i32> @load_zext_v4i16i32(ptr %ap)  {
 ; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_zext_v4i16i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i16>, ptr %ap
   %val = zext <4 x i16> %a to <4 x i32>
   ret <4 x i32> %val
@@ -48,12 +35,6 @@ define <2 x i64> @load_zext_v2i32i64(ptr %ap) {
 ; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_zext_v2i32i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x i32>, ptr %ap
   %val = zext <2 x i32> %a to <2 x i64>
   ret <2 x i64> %val
@@ -73,19 +54,6 @@ define <2 x i256> @load_zext_v2i64i256(ptr %ap) {
 ; CHECK-NEXT:    mov x7, xzr
 ; CHECK-NEXT:    fmov x4, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_zext_v2i64i256:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    mov x1, xzr
-; NONEON-NOSVE-NEXT:    mov x2, xzr
-; NONEON-NOSVE-NEXT:    mov x3, xzr
-; NONEON-NOSVE-NEXT:    mov x5, xzr
-; NONEON-NOSVE-NEXT:    mov x6, xzr
-; NONEON-NOSVE-NEXT:    mov x4, v0.d[1]
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    mov x7, xzr
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x i64>, ptr %ap
   %val = zext <2 x i64> %a to <2 x i256>
   ret <2 x i256> %val
@@ -107,24 +75,6 @@ define <16 x i32> @load_sext_v16i8i32(ptr %ap)  {
 ; CHECK-NEXT:    // kill: def $q2 killed $q2 killed $z2
 ; CHECK-NEXT:    // kill: def $q3 killed $q3 killed $z3
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_sext_v16i8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    sshll v1.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-48]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v2.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q2, q1, [sp, #16]
-; NONEON-NOSVE-NEXT:    sshll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #24]
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v3.4s, v4.4h, #0
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i8>, ptr %ap
   %val = sext <16 x i8> %a to <16 x i32>
   ret <16 x i32> %val
@@ -140,17 +90,6 @@ define <8 x i32> @load_sext_v8i16i32(ptr %ap)  {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_sext_v8i16i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i16>, ptr %ap
   %val = sext <8 x i16> %a to <8 x i32>
   ret <8 x i32> %val
@@ -182,39 +121,6 @@ define <4 x i256> @load_sext_v4i32i256(ptr %ap) {
 ; CHECK-NEXT:    stp x12, x12, [x8, #112]
 ; CHECK-NEXT:    stp x11, x12, [x8, #96]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_sext_v4i32i256:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    add x10, x8, #32
-; NONEON-NOSVE-NEXT:    add x11, x8, #96
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    mov x9, v0.d[1]
-; NONEON-NOSVE-NEXT:    st1 { v0.d }[1], [x10]
-; NONEON-NOSVE-NEXT:    fmov x10, d0
-; NONEON-NOSVE-NEXT:    st1 { v1.d }[1], [x11]
-; NONEON-NOSVE-NEXT:    mov x11, v1.d[1]
-; NONEON-NOSVE-NEXT:    asr x10, x10, #63
-; NONEON-NOSVE-NEXT:    str d0, [x8]
-; NONEON-NOSVE-NEXT:    asr x9, x9, #63
-; NONEON-NOSVE-NEXT:    str d1, [x8, #64]
-; NONEON-NOSVE-NEXT:    stp x10, x10, [x8, #16]
-; NONEON-NOSVE-NEXT:    stp x9, x9, [x8, #48]
-; NONEON-NOSVE-NEXT:    str x9, [x8, #40]
-; NONEON-NOSVE-NEXT:    fmov x9, d1
-; NONEON-NOSVE-NEXT:    str x10, [x8, #8]
-; NONEON-NOSVE-NEXT:    asr x10, x11, #63
-; NONEON-NOSVE-NEXT:    asr x9, x9, #63
-; NONEON-NOSVE-NEXT:    stp x10, x10, [x8, #112]
-; NONEON-NOSVE-NEXT:    str x10, [x8, #104]
-; NONEON-NOSVE-NEXT:    stp x9, x9, [x8, #80]
-; NONEON-NOSVE-NEXT:    str x9, [x8, #72]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i32>, ptr %ap
   %val = sext <4 x i32> %a to <4 x i256>
   ret <4 x i256> %val
@@ -248,22 +154,6 @@ define <2 x i256> @load_sext_v2i64i256(ptr %ap) {
 ; CHECK-NEXT:    fmov x1, d6
 ; CHECK-NEXT:    fmov x5, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_sext_v2i64i256:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    mov x8, v0.d[1]
-; NONEON-NOSVE-NEXT:    dup v1.2d, v0.d[1]
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    asr x1, x0, #63
-; NONEON-NOSVE-NEXT:    asr x5, x8, #63
-; NONEON-NOSVE-NEXT:    mov x2, x1
-; NONEON-NOSVE-NEXT:    mov x3, x1
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x5
-; NONEON-NOSVE-NEXT:    mov x6, x5
-; NONEON-NOSVE-NEXT:    mov x7, x5
-; NONEON-NOSVE-NEXT:    fmov x4, d1
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x i64>, ptr %ap
   %val = sext <2 x i64> %a to <2 x i256>
   ret <2 x i256> %val
@@ -297,34 +187,6 @@ define <16 x i64> @load_zext_v16i16i64(ptr %ap)  {
 ; CHECK-NEXT:    // kill: def $q6 killed $q6 killed $z6
 ; CHECK-NEXT:    // kill: def $q7 killed $q7 killed $z7
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_zext_v16i16i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-96]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v3.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
-; NONEON-NOSVE-NEXT:    ushll v4.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v5.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    stp q1, q2, [sp, #32]
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp q5, q3, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldr d16, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldr d17, [sp, #72]
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v6.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v3.2d, v7.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v6.2d, v5.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v5.2d, v16.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v7.2d, v17.2s, #0
-; NONEON-NOSVE-NEXT:    add sp, sp, #96
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %ap
   %val = zext <16 x i16> %a to <16 x i64>
   ret <16 x i64> %val
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll
index d050ddc77640..25ecd7a8d7e3 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-subvector.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -28,11 +27,6 @@ define <4 x i1> @extract_subvector_v8i1(<8 x i1> %op) {
 ; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v8i1:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    zip2 v0.8b, v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <4 x i1> @llvm.vector.extract.v4i1.v8i1(<8 x i1> %op, i64 4)
   ret <4 x i1> %ret
 }
@@ -60,11 +54,6 @@ define <4 x i8> @extract_subvector_v8i8(<8 x i8> %op) {
 ; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    zip2 v0.8b, v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <4 x i8> @llvm.vector.extract.v4i8.v8i8(<8 x i8> %op, i64 4)
   ret <4 x i8> %ret
 }
@@ -76,14 +65,6 @@ define <8 x i8> @extract_subvector_v16i8(<16 x i8> %op) {
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <8 x i8> @llvm.vector.extract.v8i8.v16i8(<16 x i8> %op, i64 8)
   ret <8 x i8> %ret
 }
@@ -94,12 +75,6 @@ define void @extract_subvector_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0, #16]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %ret = call <16 x i8> @llvm.vector.extract.v16i8.v32i8(<32 x i8> %op, i64 16)
   store <16 x i8> %ret, ptr %b
@@ -116,15 +91,6 @@ define <2 x i16> @extract_subvector_v4i16(<4 x i16> %op) {
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <2 x i16> @llvm.vector.extract.v2i16.v4i16(<4 x i16> %op, i64 2)
   ret <2 x i16> %ret
 }
@@ -136,14 +102,6 @@ define <4 x i16> @extract_subvector_v8i16(<8 x i16> %op) {
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <4 x i16> @llvm.vector.extract.v4i16.v8i16(<8 x i16> %op, i64 4)
   ret <4 x i16> %ret
 }
@@ -154,12 +112,6 @@ define void @extract_subvector_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0, #16]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %ret = call <8 x i16> @llvm.vector.extract.v8i16.v16i16(<16 x i16> %op, i64 8)
   store <8 x i16> %ret, ptr %b
@@ -175,12 +127,6 @@ define <1 x i32> @extract_subvector_v2i32(<2 x i32> %op) {
 ; CHECK-NEXT:    mov z0.s, z0.s[1]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.2s, v0.s[1]
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <1 x i32> @llvm.vector.extract.v1i32.v2i32(<2 x i32> %op, i64 1)
   ret <1 x i32> %ret
 }
@@ -192,14 +138,6 @@ define <2 x i32> @extract_subvector_v4i32(<4 x i32> %op) {
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <2 x i32> @llvm.vector.extract.v2i32.v4i32(<4 x i32> %op, i64 2)
   ret <2 x i32> %ret
 }
@@ -210,12 +148,6 @@ define void @extract_subvector_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0, #16]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %ret = call <4 x i32> @llvm.vector.extract.v4i32.v8i32(<8 x i32> %op, i64 4)
   store <4 x i32> %ret, ptr %b
@@ -231,14 +163,6 @@ define <1 x i64> @extract_subvector_v2i64(<2 x i64> %op) {
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <1 x i64> @llvm.vector.extract.v1i64.v2i64(<2 x i64> %op, i64 1)
   ret <1 x i64> %ret
 }
@@ -249,12 +173,6 @@ define void @extract_subvector_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0, #16]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %ret = call <2 x i64> @llvm.vector.extract.v2i64.v4i64(<4 x i64> %op, i64 2)
   store <2 x i64> %ret, ptr %b
@@ -272,12 +190,6 @@ define <2 x half> @extract_subvector_v4f16(<4 x half> %op) {
 ; CHECK-NEXT:    tbl z0.h, { z0.h }, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.2s, v0.s[1]
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <2 x half> @llvm.vector.extract.v2f16.v4f16(<4 x half> %op, i64 2)
   ret <2 x half> %ret
 }
@@ -289,14 +201,6 @@ define <4 x half> @extract_subvector_v8f16(<8 x half> %op) {
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <4 x half> @llvm.vector.extract.v4f16.v8f16(<8 x half> %op, i64 4)
   ret <4 x half> %ret
 }
@@ -307,12 +211,6 @@ define void @extract_subvector_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0, #16]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %ret = call <8 x half> @llvm.vector.extract.v8f16.v16f16(<16 x half> %op, i64 8)
   store <8 x half> %ret, ptr %b
@@ -328,12 +226,6 @@ define <1 x float> @extract_subvector_v2f32(<2 x float> %op) {
 ; CHECK-NEXT:    mov z0.s, z0.s[1]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.2s, v0.s[1]
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <1 x float> @llvm.vector.extract.v1f32.v2f32(<2 x float> %op, i64 1)
   ret <1 x float> %ret
 }
@@ -345,14 +237,6 @@ define <2 x float> @extract_subvector_v4f32(<4 x float> %op) {
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <2 x float> @llvm.vector.extract.v2f32.v4f32(<4 x float> %op, i64 2)
   ret <2 x float> %ret
 }
@@ -363,12 +247,6 @@ define void @extract_subvector_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0, #16]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %ret = call <4 x float> @llvm.vector.extract.v4f32.v8f32(<8 x float> %op, i64 4)
   store <4 x float> %ret, ptr %b
@@ -384,14 +262,6 @@ define <1 x double> @extract_subvector_v2f64(<2 x double> %op) {
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %ret = call <1 x double> @llvm.vector.extract.v1f64.v2f64(<2 x double> %op, i64 1)
   ret <1 x double> %ret
 }
@@ -402,12 +272,6 @@ define void @extract_subvector_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0, #16]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extract_subvector_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %ret = call <2 x double> @llvm.vector.extract.v2f64.v4f64(<4 x double> %op, i64 2)
   store <2 x double> %ret, ptr %b
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-vector-elt.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-vector-elt.ll
index b2cf818e6e3c..a752e119b2fb 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-vector-elt.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-extract-vector-elt.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -16,12 +15,6 @@ define half @extractelement_v2f16(<2 x half> %op1) {
 ; CHECK-NEXT:    mov z0.h, z0.h[1]
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extractelement_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[1]
-; NONEON-NOSVE-NEXT:    ret
   %r = extractelement <2 x half> %op1, i64 1
   ret half %r
 }
@@ -33,12 +26,6 @@ define half @extractelement_v4f16(<4 x half> %op1) {
 ; CHECK-NEXT:    mov z0.h, z0.h[3]
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extractelement_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[3]
-; NONEON-NOSVE-NEXT:    ret
   %r = extractelement <4 x half> %op1, i64 3
   ret half %r
 }
@@ -50,11 +37,6 @@ define half @extractelement_v8f16(<8 x half> %op1) {
 ; CHECK-NEXT:    mov z0.h, z0.h[7]
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extractelement_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    ret
   %r = extractelement <8 x half> %op1, i64 7
   ret half %r
 }
@@ -66,11 +48,6 @@ define half @extractelement_v16f16(ptr %a) {
 ; CHECK-NEXT:    mov z0.h, z0.h[7]
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extractelement_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr h0, [x0, #30]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %r = extractelement <16 x half> %op1, i64 15
   ret half %r
@@ -83,12 +60,6 @@ define float @extractelement_v2f32(<2 x float> %op1) {
 ; CHECK-NEXT:    mov z0.s, z0.s[1]
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extractelement_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov s0, v0.s[1]
-; NONEON-NOSVE-NEXT:    ret
   %r = extractelement <2 x float> %op1, i64 1
   ret float %r
 }
@@ -100,11 +71,6 @@ define float @extractelement_v4f32(<4 x float> %op1) {
 ; CHECK-NEXT:    mov z0.s, z0.s[3]
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extractelement_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov s0, v0.s[3]
-; NONEON-NOSVE-NEXT:    ret
   %r = extractelement <4 x float> %op1, i64 3
   ret float %r
 }
@@ -116,11 +82,6 @@ define float @extractelement_v8f32(ptr %a) {
 ; CHECK-NEXT:    mov z0.s, z0.s[3]
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extractelement_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr s0, [x0, #28]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %r = extractelement <8 x float> %op1, i64 7
   ret float %r
@@ -130,10 +91,6 @@ define double @extractelement_v1f64(<1 x double> %op1) {
 ; CHECK-LABEL: extractelement_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extractelement_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ret
   %r = extractelement <1 x double> %op1, i64 0
   ret double %r
 }
@@ -144,11 +101,6 @@ define double @extractelement_v2f64(<2 x double> %op1) {
 ; CHECK-NEXT:    mov z0.d, z0.d[1]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extractelement_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov d0, v0.d[1]
-; NONEON-NOSVE-NEXT:    ret
   %r = extractelement <2 x double> %op1, i64 1
   ret double %r
 }
@@ -160,11 +112,6 @@ define double @extractelement_v4f64(ptr %a) {
 ; CHECK-NEXT:    mov z0.d, z0.d[1]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extractelement_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0, #24]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %r = extractelement <4 x double> %op1, i64 3
   ret double %r
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
index bed5dd53c519..0d6675def8b5 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
@@ -2,7 +2,6 @@
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
@@ -29,16 +28,6 @@ define void @test_copysign_v4f16_v4f16(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z1.d, z1.d, z2.d, z0.d
 ; SVE2-NEXT:    str d1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v4f16_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #32767 // =0x7fff
-; NONEON-NOSVE-NEXT:    ldr d1, [x0]
-; NONEON-NOSVE-NEXT:    ldr d2, [x1]
-; NONEON-NOSVE-NEXT:    dup v0.4h, w8
-; NONEON-NOSVE-NEXT:    bsl v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x half>, ptr %ap
   %b = load <4 x half>, ptr %bp
   %r = call <4 x half> @llvm.copysign.v4f16(<4 x half> %a, <4 x half> %b)
@@ -65,16 +54,6 @@ define void @test_copysign_v8f16_v8f16(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z1.d, z1.d, z2.d, z0.d
 ; SVE2-NEXT:    str q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v8f16_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #32767 // =0x7fff
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x half>, ptr %ap
   %b = load <8 x half>, ptr %bp
   %r = call <8 x half> @llvm.copysign.v8f16(<8 x half> %a, <8 x half> %b)
@@ -105,17 +84,6 @@ define void @test_copysign_v16f16_v16f16(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z3.d, z3.d, z4.d, z0.d
 ; SVE2-NEXT:    stp q2, q3, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v16f16_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #32767 // =0x7fff
-; NONEON-NOSVE-NEXT:    ldp q1, q4, [x1]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    bit v1.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v3.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x half>, ptr %ap
   %b = load <16 x half>, ptr %bp
   %r = call <16 x half> @llvm.copysign.v16f16(<16 x half> %a, <16 x half> %b)
@@ -144,16 +112,6 @@ define void @test_copysign_v2f32_v2f32(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z1.d, z1.d, z2.d, z0.d
 ; SVE2-NEXT:    str d1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v2f32_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d0, #0xffffffffffffffff
-; NONEON-NOSVE-NEXT:    ldr d1, [x0]
-; NONEON-NOSVE-NEXT:    ldr d2, [x1]
-; NONEON-NOSVE-NEXT:    fneg v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    bsl v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x float>, ptr %ap
   %b = load <2 x float>, ptr %bp
   %r = call <2 x float> @llvm.copysign.v2f32(<2 x float> %a, <2 x float> %b)
@@ -180,16 +138,6 @@ define void @test_copysign_v4f32_v4f32(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z1.d, z1.d, z2.d, z0.d
 ; SVE2-NEXT:    str q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v4f32_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0xffffffffffffffff
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1]
-; NONEON-NOSVE-NEXT:    fneg v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x float>, ptr %ap
   %b = load <4 x float>, ptr %bp
   %r = call <4 x float> @llvm.copysign.v4f32(<4 x float> %a, <4 x float> %b)
@@ -220,17 +168,6 @@ define void @test_copysign_v8f32_v8f32(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z3.d, z3.d, z4.d, z0.d
 ; SVE2-NEXT:    stp q2, q3, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v8f32_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0xffffffffffffffff
-; NONEON-NOSVE-NEXT:    ldp q1, q4, [x1]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    fneg v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    bit v1.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v3.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x float>, ptr %ap
   %b = load <8 x float>, ptr %bp
   %r = call <8 x float> @llvm.copysign.v8f32(<8 x float> %a, <8 x float> %b)
@@ -259,16 +196,6 @@ define void @test_copysign_v2f64_v2f64(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z1.d, z1.d, z2.d, z0.d
 ; SVE2-NEXT:    str q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v2f64_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0xffffffffffffffff
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1]
-; NONEON-NOSVE-NEXT:    fneg v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x double>, ptr %ap
   %b = load <2 x double>, ptr %bp
   %r = call <2 x double> @llvm.copysign.v2f64(<2 x double> %a, <2 x double> %b)
@@ -299,17 +226,6 @@ define void @test_copysign_v4f64_v4f64(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z3.d, z3.d, z4.d, z0.d
 ; SVE2-NEXT:    stp q2, q3, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v4f64_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0xffffffffffffffff
-; NONEON-NOSVE-NEXT:    ldp q1, q4, [x1]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    fneg v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bit v1.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v3.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x double>, ptr %ap
   %b = load <4 x double>, ptr %bp
   %r = call <4 x double> @llvm.copysign.v4f64(<4 x double> %a, <4 x double> %b)
@@ -344,17 +260,6 @@ define void @test_copysign_v2f32_v2f64(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z2.d, z2.d, z0.d, z1.d
 ; SVE2-NEXT:    str d2, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v2f32_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d0, #0xffffffffffffffff
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    ldr d2, [x0]
-; NONEON-NOSVE-NEXT:    fcvtn v1.2s, v1.2d
-; NONEON-NOSVE-NEXT:    fneg v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    bsl v0.8b, v2.8b, v1.8b
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x float>, ptr %ap
   %b = load <2 x double>, ptr %bp
   %tmp0 = fptrunc <2 x double> %b to <2 x float>
@@ -399,18 +304,6 @@ define void @test_copysign_v4f32_v4f64(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z2.d, z2.d, z0.d, z1.d
 ; SVE2-NEXT:    str q2, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v4f32_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x1]
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0xffffffffffffffff
-; NONEON-NOSVE-NEXT:    fcvtn v1.2s, v1.2d
-; NONEON-NOSVE-NEXT:    fneg v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.4s, v2.2d
-; NONEON-NOSVE-NEXT:    ldr q2, [x0]
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v2.16b, v1.16b
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x float>, ptr %ap
   %b = load <4 x double>, ptr %bp
   %tmp0 = fptrunc <4 x double> %b to <4 x float>
@@ -444,17 +337,6 @@ define void @test_copysign_v2f64_v2f32(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z2.d, z2.d, z0.d, z1.d
 ; SVE2-NEXT:    str q2, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v2f64_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0xffffffffffffffff
-; NONEON-NOSVE-NEXT:    ldr d1, [x1]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v1.2s
-; NONEON-NOSVE-NEXT:    fneg v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v2.16b, v1.16b
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x double>, ptr %ap
   %b = load < 2 x float>, ptr %bp
   %tmp0 = fpext <2 x float> %b to <2 x double>
@@ -499,23 +381,6 @@ define void @test_copysign_v4f64_v4f32(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z4.d, z4.d, z1.d, z2.d
 ; SVE2-NEXT:    stp q3, q4, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v4f64_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0xffffffffffffffff
-; NONEON-NOSVE-NEXT:    str q1, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v1.2s
-; NONEON-NOSVE-NEXT:    fneg v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtl v4.2d, v4.2s
-; NONEON-NOSVE-NEXT:    bit v1.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v3.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x double>, ptr %ap
   %b = load <4 x float>, ptr %bp
   %tmp0 = fpext <4 x float> %b to <4 x double>
@@ -551,17 +416,6 @@ define void @test_copysign_v4f16_v4f32(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z2.d, z2.d, z0.d, z1.d
 ; SVE2-NEXT:    str d2, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v4f16_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x1]
-; NONEON-NOSVE-NEXT:    mov w8, #32767 // =0x7fff
-; NONEON-NOSVE-NEXT:    ldr d2, [x0]
-; NONEON-NOSVE-NEXT:    dup v1.4h, w8
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    bit v0.8b, v2.8b, v1.8b
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x half>, ptr %ap
   %b = load <4 x float>, ptr %bp
   %tmp0 = fptrunc <4 x float> %b to <4 x half>
@@ -603,19 +457,6 @@ define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z2.d, z2.d, z0.d, z1.d
 ; SVE2-NEXT:    str d2, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v4f16_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    mov w8, #32767 // =0x7fff
-; NONEON-NOSVE-NEXT:    ldr d2, [x0]
-; NONEON-NOSVE-NEXT:    fcvtxn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtxn2 v0.4s, v1.2d
-; NONEON-NOSVE-NEXT:    dup v1.4h, w8
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    bit v0.8b, v2.8b, v1.8b
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x half>, ptr %ap
   %b = load <4 x double>, ptr %bp
   %tmp0 = fptrunc <4 x double> %b to <4 x half>
@@ -659,18 +500,6 @@ define void @test_copysign_v8f16_v8f32(ptr %ap, ptr %bp) {
 ; SVE2-NEXT:    bsl z2.d, z2.d, z0.d, z1.d
 ; SVE2-NEXT:    str q2, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_copysign_v8f16_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    mov w8, #32767 // =0x7fff
-; NONEON-NOSVE-NEXT:    ldr q2, [x0]
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    dup v1.8h, w8
-; NONEON-NOSVE-NEXT:    bit v0.16b, v2.16b, v1.16b
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x half>, ptr %ap
   %b = load <8 x float>, ptr %bp
   %tmp0 = fptrunc <8 x float> %b to <8 x half>
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll
index 662a8f2b55fd..c2d6ed4e9ccf 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -18,14 +17,6 @@ define <2 x half> @fadd_v2f16(<2 x half> %op1, <2 x half> %op2) {
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fadd <2 x half> %op1, %op2
   ret <2 x half> %res
 }
@@ -39,14 +30,6 @@ define <4 x half> @fadd_v4f16(<4 x half> %op1, <4 x half> %op2) {
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fadd <4 x half> %op1, %op2
   ret <4 x half> %res
 }
@@ -60,18 +43,6 @@ define <8 x half> @fadd_v8f16(<8 x half> %op1, <8 x half> %op2) {
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fadd v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fadd v1.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fadd <8 x half> %op1, %op2
   ret <8 x half> %res
 }
@@ -87,29 +58,6 @@ define void @fadd_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v4.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v6.4s, v3.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl v5.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v7.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v3.4s, v3.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v2.8h
-; NONEON-NOSVE-NEXT:    fadd v4.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    fadd v5.4s, v7.4s, v6.4s
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fadd v2.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v4.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v5.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v2.4s
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %res = fadd <16 x half> %op1, %op2
@@ -126,11 +74,6 @@ define <2 x float> @fadd_v2f32(<2 x float> %op1, <2 x float> %op2) {
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fadd v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = fadd <2 x float> %op1, %op2
   ret <2 x float> %res
 }
@@ -144,11 +87,6 @@ define <4 x float> @fadd_v4f32(<4 x float> %op1, <4 x float> %op2) {
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fadd <4 x float> %op1, %op2
   ret <4 x float> %res
 }
@@ -164,15 +102,6 @@ define void @fadd_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fadd v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %res = fadd <8 x float> %op1, %op2
@@ -189,11 +118,6 @@ define <2 x double> @fadd_v2f64(<2 x double> %op1, <2 x double> %op2) {
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fadd v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fadd <2 x double> %op1, %op2
   ret <2 x double> %res
 }
@@ -209,15 +133,6 @@ define void @fadd_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fadd v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fadd v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %res = fadd <4 x double> %op1, %op2
@@ -238,14 +153,6 @@ define <2 x half> @fdiv_v2f16(<2 x half> %op1, <2 x half> %op2) {
 ; CHECK-NEXT:    fdiv z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fdiv_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fdiv v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fdiv <2 x half> %op1, %op2
   ret <2 x half> %res
 }
@@ -259,14 +166,6 @@ define <4 x half> @fdiv_v4f16(<4 x half> %op1, <4 x half> %op2) {
 ; CHECK-NEXT:    fdiv z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fdiv_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fdiv v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fdiv <4 x half> %op1, %op2
   ret <4 x half> %res
 }
@@ -280,18 +179,6 @@ define <8 x half> @fdiv_v8f16(<8 x half> %op1, <8 x half> %op2) {
 ; CHECK-NEXT:    fdiv z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fdiv_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fdiv v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fdiv v1.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fdiv <8 x half> %op1, %op2
   ret <8 x half> %res
 }
@@ -307,30 +194,6 @@ define void @fdiv_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fdiv z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fdiv_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q4, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v5.4s, v4.8h
-; NONEON-NOSVE-NEXT:    fcvtl v4.4s, v4.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fdiv v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    ldr q3, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl2 v6.4s, v3.8h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v3.4h
-; NONEON-NOSVE-NEXT:    fdiv v3.4s, v3.4s, v4.4s
-; NONEON-NOSVE-NEXT:    fcvtn v2.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fdiv v5.4s, v6.4s, v5.4s
-; NONEON-NOSVE-NEXT:    fdiv v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v5.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.8h, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %res = fdiv <16 x half> %op1, %op2
@@ -347,11 +210,6 @@ define <2 x float> @fdiv_v2f32(<2 x float> %op1, <2 x float> %op2) {
 ; CHECK-NEXT:    fdiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fdiv_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fdiv v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = fdiv <2 x float> %op1, %op2
   ret <2 x float> %res
 }
@@ -365,11 +223,6 @@ define <4 x float> @fdiv_v4f32(<4 x float> %op1, <4 x float> %op2) {
 ; CHECK-NEXT:    fdiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fdiv_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fdiv v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fdiv <4 x float> %op1, %op2
   ret <4 x float> %res
 }
@@ -385,15 +238,6 @@ define void @fdiv_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fdiv z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fdiv_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fdiv v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fdiv v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %res = fdiv <8 x float> %op1, %op2
@@ -410,11 +254,6 @@ define <2 x double> @fdiv_v2f64(<2 x double> %op1, <2 x double> %op2) {
 ; CHECK-NEXT:    fdiv z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fdiv_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fdiv v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fdiv <2 x double> %op1, %op2
   ret <2 x double> %res
 }
@@ -430,15 +269,6 @@ define void @fdiv_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fdiv z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fdiv_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fdiv v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fdiv v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %res = fdiv <4 x double> %op1, %op2
@@ -460,46 +290,6 @@ define <2 x half> @fma_v2f16(<2 x half> %op1, <2 x half> %op2, <2 x half> %op3)
 ; CHECK-NEXT:    fmad z0.h, p0/m, z1.h, z2.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d2 killed $d2 def $q2
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    fcvt s16, h0
-; NONEON-NOSVE-NEXT:    mov h17, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h18, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fmadd s6, s16, s7, s6
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s7, h19
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmadd s3, s5, s4, s3
-; NONEON-NOSVE-NEXT:    fcvt s4, h17
-; NONEON-NOSVE-NEXT:    fcvt s5, h18
-; NONEON-NOSVE-NEXT:    fcvt h0, s6
-; NONEON-NOSVE-NEXT:    fmadd s4, s7, s5, s4
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h16
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h3, s4
-; NONEON-NOSVE-NEXT:    fmadd s1, s5, s1, s2
-; NONEON-NOSVE-NEXT:    mov v0.h[2], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[3], v1.h[0]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x half> @llvm.fma.v2f16(<2 x half> %op1, <2 x half> %op2, <2 x half> %op3)
   ret <2 x half> %res
 }
@@ -514,46 +304,6 @@ define <4 x half> @fma_v4f16(<4 x half> %op1, <4 x half> %op2, <4 x half> %op3)
 ; CHECK-NEXT:    fmad z0.h, p0/m, z1.h, z2.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d2 killed $d2 def $q2
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    fcvt s16, h0
-; NONEON-NOSVE-NEXT:    mov h17, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h18, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fmadd s6, s16, s7, s6
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s7, h19
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmadd s3, s5, s4, s3
-; NONEON-NOSVE-NEXT:    fcvt s4, h17
-; NONEON-NOSVE-NEXT:    fcvt s5, h18
-; NONEON-NOSVE-NEXT:    fcvt h0, s6
-; NONEON-NOSVE-NEXT:    fmadd s4, s7, s5, s4
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h16
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h3, s4
-; NONEON-NOSVE-NEXT:    fmadd s1, s5, s1, s2
-; NONEON-NOSVE-NEXT:    mov v0.h[2], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[3], v1.h[0]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.fma.v4f16(<4 x half> %op1, <4 x half> %op2, <4 x half> %op3)
   ret <4 x half> %res
 }
@@ -568,79 +318,6 @@ define <8 x half> @fma_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3)
 ; CHECK-NEXT:    fmad z0.h, p0/m, z1.h, z2.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    fcvt s16, h0
-; NONEON-NOSVE-NEXT:    mov h17, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h18, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fmadd s6, s16, s7, s6
-; NONEON-NOSVE-NEXT:    fcvt s7, h17
-; NONEON-NOSVE-NEXT:    fcvt s16, h18
-; NONEON-NOSVE-NEXT:    fcvt s17, h19
-; NONEON-NOSVE-NEXT:    mov h18, v1.h[3]
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[3]
-; NONEON-NOSVE-NEXT:    fmadd s4, s5, s4, s3
-; NONEON-NOSVE-NEXT:    mov h5, v2.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h3, s6
-; NONEON-NOSVE-NEXT:    fmadd s6, s17, s16, s7
-; NONEON-NOSVE-NEXT:    mov h17, v2.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s7, h18
-; NONEON-NOSVE-NEXT:    fcvt s16, h19
-; NONEON-NOSVE-NEXT:    mov h18, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fcvt s18, h18
-; NONEON-NOSVE-NEXT:    mov v3.h[1], v4.h[0]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    fmadd s5, s16, s7, s5
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s19, h19
-; NONEON-NOSVE-NEXT:    mov v3.h[2], v6.h[0]
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fcvt h5, s5
-; NONEON-NOSVE-NEXT:    fmadd s17, s19, s18, s17
-; NONEON-NOSVE-NEXT:    mov h18, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fmadd s4, s16, s7, s4
-; NONEON-NOSVE-NEXT:    mov v3.h[3], v5.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h6
-; NONEON-NOSVE-NEXT:    fcvt s6, h18
-; NONEON-NOSVE-NEXT:    fcvt s7, h19
-; NONEON-NOSVE-NEXT:    fcvt h16, s17
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fmadd s5, s7, s6, s5
-; NONEON-NOSVE-NEXT:    mov v3.h[4], v16.h[0]
-; NONEON-NOSVE-NEXT:    fmadd s0, s0, s1, s2
-; NONEON-NOSVE-NEXT:    mov v3.h[5], v4.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h4, s5
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    mov v3.h[6], v4.h[0]
-; NONEON-NOSVE-NEXT:    mov v3.h[7], v0.h[0]
-; NONEON-NOSVE-NEXT:    mov v0.16b, v3.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.fma.v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3)
   ret <8 x half> %res
 }
@@ -657,150 +334,6 @@ define void @fma_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    fmla z1.h, p0/m, z3.h, z4.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q3, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q4, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q5, q2, [x2]
-; NONEON-NOSVE-NEXT:    mov h25, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s19, h0
-; NONEON-NOSVE-NEXT:    mov h24, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h17, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s18, h1
-; NONEON-NOSVE-NEXT:    mov h22, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h16, v2.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    mov h20, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h26, v5.h[1]
-; NONEON-NOSVE-NEXT:    mov h27, v4.h[1]
-; NONEON-NOSVE-NEXT:    mov h28, v3.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s25, h25
-; NONEON-NOSVE-NEXT:    mov h7, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h29, v4.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s23, h17
-; NONEON-NOSVE-NEXT:    mov h17, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h30, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s21, h16
-; NONEON-NOSVE-NEXT:    fmadd s6, s19, s18, s6
-; NONEON-NOSVE-NEXT:    fcvt s18, h20
-; NONEON-NOSVE-NEXT:    fcvt s19, h22
-; NONEON-NOSVE-NEXT:    fcvt s20, h24
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s22, h5
-; NONEON-NOSVE-NEXT:    fcvt s24, h4
-; NONEON-NOSVE-NEXT:    fcvt s26, h26
-; NONEON-NOSVE-NEXT:    fcvt s27, h27
-; NONEON-NOSVE-NEXT:    fcvt s28, h28
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fmadd s21, s25, s23, s21
-; NONEON-NOSVE-NEXT:    fcvt s23, h3
-; NONEON-NOSVE-NEXT:    mov h25, v5.h[2]
-; NONEON-NOSVE-NEXT:    fmadd s18, s20, s19, s18
-; NONEON-NOSVE-NEXT:    mov h19, v3.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    mov h31, v0.h[4]
-; NONEON-NOSVE-NEXT:    fmadd s26, s28, s27, s26
-; NONEON-NOSVE-NEXT:    mov h27, v4.h[3]
-; NONEON-NOSVE-NEXT:    mov h28, v3.h[3]
-; NONEON-NOSVE-NEXT:    fmadd s22, s23, s24, s22
-; NONEON-NOSVE-NEXT:    fcvt h20, s21
-; NONEON-NOSVE-NEXT:    mov h21, v2.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s23, h25
-; NONEON-NOSVE-NEXT:    fcvt s24, h29
-; NONEON-NOSVE-NEXT:    fcvt s19, h19
-; NONEON-NOSVE-NEXT:    fmadd s16, s17, s16, s7
-; NONEON-NOSVE-NEXT:    mov h25, v5.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h18, s18
-; NONEON-NOSVE-NEXT:    fcvt h26, s26
-; NONEON-NOSVE-NEXT:    mov h29, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov v6.h[1], v20.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s17, h21
-; NONEON-NOSVE-NEXT:    fcvt s20, h30
-; NONEON-NOSVE-NEXT:    fmadd s19, s19, s24, s23
-; NONEON-NOSVE-NEXT:    fcvt s21, h31
-; NONEON-NOSVE-NEXT:    fcvt h7, s22
-; NONEON-NOSVE-NEXT:    fcvt s22, h25
-; NONEON-NOSVE-NEXT:    fcvt s23, h27
-; NONEON-NOSVE-NEXT:    fcvt s24, h28
-; NONEON-NOSVE-NEXT:    mov h25, v5.h[4]
-; NONEON-NOSVE-NEXT:    mov h27, v4.h[4]
-; NONEON-NOSVE-NEXT:    mov h28, v3.h[4]
-; NONEON-NOSVE-NEXT:    mov h30, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h31, v0.h[5]
-; NONEON-NOSVE-NEXT:    mov v6.h[2], v18.h[0]
-; NONEON-NOSVE-NEXT:    fmadd s17, s21, s20, s17
-; NONEON-NOSVE-NEXT:    mov v7.h[1], v26.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h18, s19
-; NONEON-NOSVE-NEXT:    fmadd s19, s24, s23, s22
-; NONEON-NOSVE-NEXT:    mov h26, v5.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h16, s16
-; NONEON-NOSVE-NEXT:    fcvt s20, h25
-; NONEON-NOSVE-NEXT:    fcvt s21, h27
-; NONEON-NOSVE-NEXT:    fcvt s22, h28
-; NONEON-NOSVE-NEXT:    mov h27, v4.h[5]
-; NONEON-NOSVE-NEXT:    mov h28, v3.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s23, h29
-; NONEON-NOSVE-NEXT:    fcvt s24, h30
-; NONEON-NOSVE-NEXT:    fcvt s25, h31
-; NONEON-NOSVE-NEXT:    mov h29, v2.h[6]
-; NONEON-NOSVE-NEXT:    mov h30, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h31, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov v7.h[2], v18.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h18, s19
-; NONEON-NOSVE-NEXT:    fmadd s19, s22, s21, s20
-; NONEON-NOSVE-NEXT:    mov h20, v5.h[6]
-; NONEON-NOSVE-NEXT:    mov h21, v4.h[6]
-; NONEON-NOSVE-NEXT:    mov h22, v3.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s26, h26
-; NONEON-NOSVE-NEXT:    fmadd s23, s25, s24, s23
-; NONEON-NOSVE-NEXT:    fcvt s27, h27
-; NONEON-NOSVE-NEXT:    fcvt s28, h28
-; NONEON-NOSVE-NEXT:    mov v6.h[3], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h16, s17
-; NONEON-NOSVE-NEXT:    fcvt s17, h29
-; NONEON-NOSVE-NEXT:    fcvt s24, h30
-; NONEON-NOSVE-NEXT:    fcvt s25, h31
-; NONEON-NOSVE-NEXT:    fcvt s20, h20
-; NONEON-NOSVE-NEXT:    fcvt s21, h21
-; NONEON-NOSVE-NEXT:    fcvt s22, h22
-; NONEON-NOSVE-NEXT:    mov v7.h[3], v18.h[0]
-; NONEON-NOSVE-NEXT:    fmadd s26, s28, s27, s26
-; NONEON-NOSVE-NEXT:    fcvt h18, s19
-; NONEON-NOSVE-NEXT:    mov h5, v5.h[7]
-; NONEON-NOSVE-NEXT:    mov h4, v4.h[7]
-; NONEON-NOSVE-NEXT:    mov h3, v3.h[7]
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    fmadd s17, s25, s24, s17
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fmadd s19, s22, s21, s20
-; NONEON-NOSVE-NEXT:    mov v6.h[4], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h16, s23
-; NONEON-NOSVE-NEXT:    mov v7.h[4], v18.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h18, s26
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v6.h[5], v16.h[0]
-; NONEON-NOSVE-NEXT:    mov v7.h[5], v18.h[0]
-; NONEON-NOSVE-NEXT:    fmadd s3, s3, s4, s5
-; NONEON-NOSVE-NEXT:    fcvt h4, s19
-; NONEON-NOSVE-NEXT:    fcvt h5, s17
-; NONEON-NOSVE-NEXT:    fmadd s0, s0, s1, s2
-; NONEON-NOSVE-NEXT:    mov v7.h[6], v4.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s3
-; NONEON-NOSVE-NEXT:    mov v6.h[6], v5.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], v1.h[0]
-; NONEON-NOSVE-NEXT:    mov v6.h[7], v0.h[0]
-; NONEON-NOSVE-NEXT:    stp q7, q6, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %op3 = load <16 x half>, ptr %c
@@ -819,12 +352,6 @@ define <2 x float> @fma_v2f32(<2 x float> %op1, <2 x float> %op2, <2 x float> %o
 ; CHECK-NEXT:    fmad z0.s, p0/m, z1.s, z2.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmla v2.2s, v1.2s, v0.2s
-; NONEON-NOSVE-NEXT:    fmov d0, d2
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.fma.v2f32(<2 x float> %op1, <2 x float> %op2, <2 x float> %op3)
   ret <2 x float> %res
 }
@@ -839,12 +366,6 @@ define <4 x float> @fma_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %o
 ; CHECK-NEXT:    fmad z0.s, p0/m, z1.s, z2.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmla v2.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.fma.v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %op3)
   ret <4 x float> %res
 }
@@ -861,16 +382,6 @@ define void @fma_v8f32(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    fmla z1.s, p0/m, z3.s, z4.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q4, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q5, [x2]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    fmla v1.4s, v0.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fmla v5.4s, v4.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q1, q5, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %op3 = load <8 x float>, ptr %c
@@ -889,12 +400,6 @@ define <2 x double> @fma_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double
 ; CHECK-NEXT:    fmad z0.d, p0/m, z1.d, z2.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmla v2.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.fma.v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double> %op3)
   ret <2 x double> %res
 }
@@ -911,16 +416,6 @@ define void @fma_v4f64(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    fmla z1.d, p0/m, z3.d, z4.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q4, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q5, [x2]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    fmla v1.2d, v0.2d, v2.2d
-; NONEON-NOSVE-NEXT:    fmla v5.2d, v4.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q1, q5, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %op3 = load <4 x double>, ptr %c
@@ -942,14 +437,6 @@ define <2 x half> @fmul_v2f16(<2 x half> %op1, <2 x half> %op2) {
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmul_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fmul v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fmul <2 x half> %op1, %op2
   ret <2 x half> %res
 }
@@ -963,14 +450,6 @@ define <4 x half> @fmul_v4f16(<4 x half> %op1, <4 x half> %op2) {
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmul_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fmul v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fmul <4 x half> %op1, %op2
   ret <4 x half> %res
 }
@@ -984,18 +463,6 @@ define <8 x half> @fmul_v8f16(<8 x half> %op1, <8 x half> %op2) {
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmul_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fmul v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fmul v1.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fmul <8 x half> %op1, %op2
   ret <8 x half> %res
 }
@@ -1011,29 +478,6 @@ define void @fmul_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmul z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmul_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v4.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v6.4s, v3.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl v5.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v7.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v3.4s, v3.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v2.8h
-; NONEON-NOSVE-NEXT:    fmul v4.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    fmul v5.4s, v7.4s, v6.4s
-; NONEON-NOSVE-NEXT:    fmul v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fmul v2.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v4.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v5.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v2.4s
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %res = fmul <16 x half> %op1, %op2
@@ -1050,11 +494,6 @@ define <2 x float> @fmul_v2f32(<2 x float> %op1, <2 x float> %op2) {
 ; CHECK-NEXT:    fmul z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmul_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmul v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = fmul <2 x float> %op1, %op2
   ret <2 x float> %res
 }
@@ -1068,11 +507,6 @@ define <4 x float> @fmul_v4f32(<4 x float> %op1, <4 x float> %op2) {
 ; CHECK-NEXT:    fmul z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmul_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmul v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fmul <4 x float> %op1, %op2
   ret <4 x float> %res
 }
@@ -1088,15 +522,6 @@ define void @fmul_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmul z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmul_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fmul v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fmul v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %res = fmul <8 x float> %op1, %op2
@@ -1113,11 +538,6 @@ define <2 x double> @fmul_v2f64(<2 x double> %op1, <2 x double> %op2) {
 ; CHECK-NEXT:    fmul z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmul_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmul v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fmul <2 x double> %op1, %op2
   ret <2 x double> %res
 }
@@ -1133,15 +553,6 @@ define void @fmul_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmul z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmul_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fmul v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fmul v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %res = fmul <4 x double> %op1, %op2
@@ -1161,12 +572,6 @@ define <2 x half> @fneg_v2f16(<2 x half> %op) {
 ; CHECK-NEXT:    fneg z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fneg_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v1.4h, #128, lsl #8
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = fneg <2 x half> %op
   ret <2 x half> %res
 }
@@ -1179,12 +584,6 @@ define <4 x half> @fneg_v4f16(<4 x half> %op) {
 ; CHECK-NEXT:    fneg z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fneg_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v1.4h, #128, lsl #8
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = fneg <4 x half> %op
   ret <4 x half> %res
 }
@@ -1197,12 +596,6 @@ define <8 x half> @fneg_v8f16(<8 x half> %op) {
 ; CHECK-NEXT:    fneg z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fneg_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v1.8h, #128, lsl #8
-; NONEON-NOSVE-NEXT:    eor v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = fneg <8 x half> %op
   ret <8 x half> %res
 }
@@ -1216,15 +609,6 @@ define void @fneg_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fneg z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fneg_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.8h, #128, lsl #8
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    eor v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    eor v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = fneg <16 x half> %op
   store <16 x half> %res, ptr %a
@@ -1239,11 +623,6 @@ define <2 x float> @fneg_v2f32(<2 x float> %op) {
 ; CHECK-NEXT:    fneg z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fneg_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fneg v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = fneg <2 x float> %op
   ret <2 x float> %res
 }
@@ -1256,11 +635,6 @@ define <4 x float> @fneg_v4f32(<4 x float> %op) {
 ; CHECK-NEXT:    fneg z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fneg_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fneg v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fneg <4 x float> %op
   ret <4 x float> %res
 }
@@ -1274,14 +648,6 @@ define void @fneg_v8f32(ptr %a) {
 ; CHECK-NEXT:    fneg z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fneg_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fneg v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fneg v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = fneg <8 x float> %op
   store <8 x float> %res, ptr %a
@@ -1296,11 +662,6 @@ define <2 x double> @fneg_v2f64(<2 x double> %op) {
 ; CHECK-NEXT:    fneg z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fneg_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fneg v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fneg <2 x double> %op
   ret <2 x double> %res
 }
@@ -1314,14 +675,6 @@ define void @fneg_v4f64(ptr %a) {
 ; CHECK-NEXT:    fneg z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fneg_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fneg v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fneg v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = fneg <4 x double> %op
   store <4 x double> %res, ptr %a
@@ -1340,30 +693,6 @@ define <2 x half> @fsqrt_v2f16(<2 x half> %op) {
 ; CHECK-NEXT:    fsqrt z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsqrt_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fsqrt s2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fsqrt s1, s1
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fsqrt s3, s3
-; NONEON-NOSVE-NEXT:    fsqrt s4, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s2
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v1.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s3
-; NONEON-NOSVE-NEXT:    mov v0.h[2], v1.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s4
-; NONEON-NOSVE-NEXT:    mov v0.h[3], v1.h[0]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x half> @llvm.sqrt.v2f16(<2 x half> %op)
   ret <2 x half> %res
 }
@@ -1376,30 +705,6 @@ define <4 x half> @fsqrt_v4f16(<4 x half> %op) {
 ; CHECK-NEXT:    fsqrt z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsqrt_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fsqrt s2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fsqrt s1, s1
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fsqrt s3, s3
-; NONEON-NOSVE-NEXT:    fsqrt s4, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s2
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v1.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s3
-; NONEON-NOSVE-NEXT:    mov v0.h[2], v1.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s4
-; NONEON-NOSVE-NEXT:    mov v0.h[3], v1.h[0]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.sqrt.v4f16(<4 x half> %op)
   ret <4 x half> %res
 }
@@ -1412,48 +717,6 @@ define <8 x half> @fsqrt_v8f16(<8 x half> %op) {
 ; CHECK-NEXT:    fsqrt z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsqrt_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fsqrt s2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt s16, h0
-; NONEON-NOSVE-NEXT:    fcvt h0, s2
-; NONEON-NOSVE-NEXT:    fsqrt s1, s1
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v1.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s3, s3
-; NONEON-NOSVE-NEXT:    fcvt h1, s3
-; NONEON-NOSVE-NEXT:    mov v0.h[2], v1.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s4, s4
-; NONEON-NOSVE-NEXT:    fcvt h1, s4
-; NONEON-NOSVE-NEXT:    mov v0.h[3], v1.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s5, s5
-; NONEON-NOSVE-NEXT:    fcvt h1, s5
-; NONEON-NOSVE-NEXT:    mov v0.h[4], v1.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s6, s6
-; NONEON-NOSVE-NEXT:    fcvt h1, s6
-; NONEON-NOSVE-NEXT:    mov v0.h[5], v1.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s7, s7
-; NONEON-NOSVE-NEXT:    fcvt h1, s7
-; NONEON-NOSVE-NEXT:    mov v0.h[6], v1.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s2, s16
-; NONEON-NOSVE-NEXT:    fcvt h1, s2
-; NONEON-NOSVE-NEXT:    mov v0.h[7], v1.h[0]
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.sqrt.v8f16(<8 x half> %op)
   ret <8 x half> %res
 }
@@ -1467,89 +730,6 @@ define void @fsqrt_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fsqrt z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsqrt_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q16, [x0]
-; NONEON-NOSVE-NEXT:    mov h0, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h17, v16.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s18, h16
-; NONEON-NOSVE-NEXT:    mov h19, v16.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[3]
-; NONEON-NOSVE-NEXT:    mov h20, v16.h[3]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h21, v16.h[4]
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h22, v16.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fsqrt s2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s19, h19
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s20, h20
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s21, h21
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s22, h22
-; NONEON-NOSVE-NEXT:    mov h23, v16.h[6]
-; NONEON-NOSVE-NEXT:    mov h16, v16.h[7]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt s23, h23
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fsqrt s0, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    mov v2.h[1], v0.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s17, s17
-; NONEON-NOSVE-NEXT:    fcvt h17, s17
-; NONEON-NOSVE-NEXT:    fsqrt s18, s18
-; NONEON-NOSVE-NEXT:    fcvt h18, s18
-; NONEON-NOSVE-NEXT:    mov v18.h[1], v17.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s3, s3
-; NONEON-NOSVE-NEXT:    fcvt h0, s3
-; NONEON-NOSVE-NEXT:    mov v2.h[2], v0.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s19, s19
-; NONEON-NOSVE-NEXT:    fcvt h17, s19
-; NONEON-NOSVE-NEXT:    mov v18.h[2], v17.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s4, s4
-; NONEON-NOSVE-NEXT:    fcvt h0, s4
-; NONEON-NOSVE-NEXT:    mov v2.h[3], v0.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s20, s20
-; NONEON-NOSVE-NEXT:    fcvt h3, s20
-; NONEON-NOSVE-NEXT:    mov v18.h[3], v3.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s5, s5
-; NONEON-NOSVE-NEXT:    fcvt h0, s5
-; NONEON-NOSVE-NEXT:    mov v2.h[4], v0.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s21, s21
-; NONEON-NOSVE-NEXT:    fcvt h3, s21
-; NONEON-NOSVE-NEXT:    mov v18.h[4], v3.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s6, s6
-; NONEON-NOSVE-NEXT:    fcvt h0, s6
-; NONEON-NOSVE-NEXT:    mov v2.h[5], v0.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s22, s22
-; NONEON-NOSVE-NEXT:    fcvt h3, s22
-; NONEON-NOSVE-NEXT:    mov v18.h[5], v3.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s7, s7
-; NONEON-NOSVE-NEXT:    fcvt h0, s7
-; NONEON-NOSVE-NEXT:    mov v2.h[6], v0.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s23, s23
-; NONEON-NOSVE-NEXT:    fcvt h3, s23
-; NONEON-NOSVE-NEXT:    mov v18.h[6], v3.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s16, s16
-; NONEON-NOSVE-NEXT:    fcvt h3, s16
-; NONEON-NOSVE-NEXT:    mov v18.h[7], v3.h[0]
-; NONEON-NOSVE-NEXT:    fsqrt s1, s1
-; NONEON-NOSVE-NEXT:    fcvt h0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[7], v0.h[0]
-; NONEON-NOSVE-NEXT:    stp q18, q2, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call <16 x half> @llvm.sqrt.v16f16(<16 x half> %op)
   store <16 x half> %res, ptr %a
@@ -1564,11 +744,6 @@ define <2 x float> @fsqrt_v2f32(<2 x float> %op) {
 ; CHECK-NEXT:    fsqrt z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsqrt_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fsqrt v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.sqrt.v2f32(<2 x float> %op)
   ret <2 x float> %res
 }
@@ -1581,11 +756,6 @@ define <4 x float> @fsqrt_v4f32(<4 x float> %op) {
 ; CHECK-NEXT:    fsqrt z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsqrt_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fsqrt v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %op)
   ret <4 x float> %res
 }
@@ -1599,14 +769,6 @@ define void @fsqrt_v8f32(ptr %a) {
 ; CHECK-NEXT:    fsqrt z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsqrt_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fsqrt v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fsqrt v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call <8 x float> @llvm.sqrt.v8f32(<8 x float> %op)
   store <8 x float> %res, ptr %a
@@ -1621,11 +783,6 @@ define <2 x double> @fsqrt_v2f64(<2 x double> %op) {
 ; CHECK-NEXT:    fsqrt z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsqrt_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fsqrt v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.sqrt.v2f64(<2 x double> %op)
   ret <2 x double> %res
 }
@@ -1639,14 +796,6 @@ define void @fsqrt_v4f64(ptr %a) {
 ; CHECK-NEXT:    fsqrt z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsqrt_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fsqrt v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fsqrt v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %op)
   store <4 x double> %res, ptr %a
@@ -1666,14 +815,6 @@ define <2 x half> @fsub_v2f16(<2 x half> %op1, <2 x half> %op2) {
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsub_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fsub v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fsub <2 x half> %op1, %op2
   ret <2 x half> %res
 }
@@ -1687,14 +828,6 @@ define <4 x half> @fsub_v4f16(<4 x half> %op1, <4 x half> %op2) {
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsub_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fsub v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fsub <4 x half> %op1, %op2
   ret <4 x half> %res
 }
@@ -1708,18 +841,6 @@ define <8 x half> @fsub_v8f16(<8 x half> %op1, <8 x half> %op2) {
 ; CHECK-NEXT:    fsub z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsub_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fsub v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fsub v1.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fsub <8 x half> %op1, %op2
   ret <8 x half> %res
 }
@@ -1735,29 +856,6 @@ define void @fsub_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fsub z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsub_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v4.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v6.4s, v3.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl v5.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v7.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v3.4s, v3.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v2.8h
-; NONEON-NOSVE-NEXT:    fsub v4.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    fsub v5.4s, v7.4s, v6.4s
-; NONEON-NOSVE-NEXT:    fsub v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fsub v2.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v4.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v5.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v2.4s
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %res = fsub <16 x half> %op1, %op2
@@ -1774,11 +872,6 @@ define <2 x float> @fsub_v2f32(<2 x float> %op1, <2 x float> %op2) {
 ; CHECK-NEXT:    fsub z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsub_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fsub v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = fsub <2 x float> %op1, %op2
   ret <2 x float> %res
 }
@@ -1792,11 +885,6 @@ define <4 x float> @fsub_v4f32(<4 x float> %op1, <4 x float> %op2) {
 ; CHECK-NEXT:    fsub z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsub_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fsub v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fsub <4 x float> %op1, %op2
   ret <4 x float> %res
 }
@@ -1812,15 +900,6 @@ define void @fsub_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fsub z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsub_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fsub v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fsub v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %res = fsub <8 x float> %op1, %op2
@@ -1837,11 +916,6 @@ define <2 x double> @fsub_v2f64(<2 x double> %op1, <2 x double> %op2) {
 ; CHECK-NEXT:    fsub z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsub_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fsub v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fsub <2 x double> %op1, %op2
   ret <2 x double> %res
 }
@@ -1857,15 +931,6 @@ define void @fsub_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fsub z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fsub_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fsub v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fsub v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %res = fsub <4 x double> %op1, %op2
@@ -1885,11 +950,6 @@ define <2 x half> @fabs_v2f16(<2 x half> %op) {
 ; CHECK-NEXT:    fabs z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fabs_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    bic v0.4h, #128, lsl #8
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x half> @llvm.fabs.v2f16(<2 x half> %op)
   ret <2 x half> %res
 }
@@ -1902,11 +962,6 @@ define <4 x half> @fabs_v4f16(<4 x half> %op) {
 ; CHECK-NEXT:    fabs z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fabs_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    bic v0.4h, #128, lsl #8
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.fabs.v4f16(<4 x half> %op)
   ret <4 x half> %res
 }
@@ -1919,11 +974,6 @@ define <8 x half> @fabs_v8f16(<8 x half> %op) {
 ; CHECK-NEXT:    fabs z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fabs_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    bic v0.8h, #128, lsl #8
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.fabs.v8f16(<8 x half> %op)
   ret <8 x half> %res
 }
@@ -1937,14 +987,6 @@ define void @fabs_v16f16(ptr %a) {
 ; CHECK-NEXT:    fabs z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fabs_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    bic v0.8h, #128, lsl #8
-; NONEON-NOSVE-NEXT:    bic v1.8h, #128, lsl #8
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call <16 x half> @llvm.fabs.v16f16(<16 x half> %op)
   store <16 x half> %res, ptr %a
@@ -1959,11 +1001,6 @@ define <2 x float> @fabs_v2f32(<2 x float> %op) {
 ; CHECK-NEXT:    fabs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fabs_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fabs v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.fabs.v2f32(<2 x float> %op)
   ret <2 x float> %res
 }
@@ -1976,11 +1013,6 @@ define <4 x float> @fabs_v4f32(<4 x float> %op) {
 ; CHECK-NEXT:    fabs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fabs_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fabs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.fabs.v4f32(<4 x float> %op)
   ret <4 x float> %res
 }
@@ -1994,14 +1026,6 @@ define void @fabs_v8f32(ptr %a) {
 ; CHECK-NEXT:    fabs z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fabs_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fabs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fabs v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call <8 x float> @llvm.fabs.v8f32(<8 x float> %op)
   store <8 x float> %res, ptr %a
@@ -2016,11 +1040,6 @@ define <2 x double> @fabs_v2f64(<2 x double> %op) {
 ; CHECK-NEXT:    fabs z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fabs_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fabs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.fabs.v2f64(<2 x double> %op)
   ret <2 x double> %res
 }
@@ -2034,14 +1053,6 @@ define void @fabs_v4f64(ptr %a) {
 ; CHECK-NEXT:    fabs z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fabs_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fabs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fabs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call <4 x double> @llvm.fabs.v4f64(<4 x double> %op)
   store <4 x double> %res, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll
index d4810c78cb53..465cc179a3b9 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -20,14 +19,6 @@ define <2 x i16> @fcmp_oeq_v2f16(<2 x half> %op1, <2 x half> %op2) {
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_oeq_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %cmp = fcmp oeq <2 x half> %op1, %op2
   %sext = sext <2 x i1> %cmp to <2 x i16>
   ret <2 x i16> %sext
@@ -43,14 +34,6 @@ define <4 x i16> @fcmp_oeq_v4f16(<4 x half> %op1, <4 x half> %op2) {
 ; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_oeq_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %cmp = fcmp oeq <4 x half> %op1, %op2
   %sext = sext <4 x i1> %cmp to <4 x i16>
   ret <4 x i16> %sext
@@ -66,65 +49,6 @@ define <8 x i16> @fcmp_oeq_v8f16(<8 x half> %op1, <8 x half> %op2) {
 ; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_oeq_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcmp s3, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    fcvt s5, h6
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[4]
-; NONEON-NOSVE-NEXT:    csetm w9, eq
-; NONEON-NOSVE-NEXT:    fcmp s2, s5
-; NONEON-NOSVE-NEXT:    fmov s2, w9
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    fcvt s3, h5
-; NONEON-NOSVE-NEXT:    fcvt s4, h6
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %cmp = fcmp oeq <8 x half> %op1, %op2
   %sext = sext <8 x i1> %cmp to <8 x i16>
   ret <8 x i16> %sext
@@ -142,123 +66,6 @@ define void @fcmp_oeq_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_oeq_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, eq
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, eq
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, eq
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, eq
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, eq
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, eq
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, eq
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp oeq <16 x half> %op1, %op2
@@ -277,11 +84,6 @@ define <2 x i32> @fcmp_oeq_v2f32(<2 x float> %op1, <2 x float> %op2) {
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_oeq_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcmeq v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %cmp = fcmp oeq <2 x float> %op1, %op2
   %sext = sext <2 x i1> %cmp to <2 x i32>
   ret <2 x i32> %sext
@@ -297,11 +99,6 @@ define <4 x i32> @fcmp_oeq_v4f32(<4 x float> %op1, <4 x float> %op2) {
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_oeq_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %cmp = fcmp oeq <4 x float> %op1, %op2
   %sext = sext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %sext
@@ -319,15 +116,6 @@ define void @fcmp_oeq_v8f32(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_oeq_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fcmeq v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcmeq v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %cmp = fcmp oeq <8 x float> %op1, %op2
@@ -344,11 +132,6 @@ define <1 x i64> @fcmp_oeq_v1f64(<1 x double> %op1, <1 x double> %op2) {
 ; CHECK-NEXT:    mov z0.d, x8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_oeq_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcmeq d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %cmp = fcmp oeq <1 x double> %op1, %op2
   %sext = sext <1 x i1> %cmp to <1 x i64>
   ret <1 x i64> %sext
@@ -364,11 +147,6 @@ define <2 x i64> @fcmp_oeq_v2f64(<2 x double> %op1, <2 x double> %op2) {
 ; CHECK-NEXT:    mov z0.d, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_oeq_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcmeq v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %cmp = fcmp oeq <2 x double> %op1, %op2
   %sext = sext <2 x i1> %cmp to <2 x i64>
   ret <2 x i64> %sext
@@ -386,15 +164,6 @@ define void @fcmp_oeq_v4f64(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.d, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_oeq_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fcmeq v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcmeq v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %cmp = fcmp oeq <4 x double> %op1, %op2
@@ -423,139 +192,6 @@ define void @fcmp_ueq_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_ueq_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h2
-; NONEON-NOSVE-NEXT:    mov h5, v2.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h1
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s6, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w9, eq
-; NONEON-NOSVE-NEXT:    csinv w12, w9, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s7, s5
-; NONEON-NOSVE-NEXT:    mov h5, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w9, eq
-; NONEON-NOSVE-NEXT:    csinv w10, w9, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x1]
-; NONEON-NOSVE-NEXT:    csetm w9, eq
-; NONEON-NOSVE-NEXT:    csinv w11, w9, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s6, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s6, h16
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w9, eq
-; NONEON-NOSVE-NEXT:    csinv w9, w9, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s7, s5
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w13, eq
-; NONEON-NOSVE-NEXT:    csinv w13, w13, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s6, s3
-; NONEON-NOSVE-NEXT:    fcvt s3, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h7
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[2]
-; NONEON-NOSVE-NEXT:    csetm w14, eq
-; NONEON-NOSVE-NEXT:    csinv w14, w14, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s4, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w15, eq
-; NONEON-NOSVE-NEXT:    csinv w15, w15, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s5, s3
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w16, eq
-; NONEON-NOSVE-NEXT:    csinv w16, w16, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s4, s2
-; NONEON-NOSVE-NEXT:    fcvt s4, h3
-; NONEON-NOSVE-NEXT:    fmov s2, w12
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w17, eq
-; NONEON-NOSVE-NEXT:    csinv w17, w17, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[4]
-; NONEON-NOSVE-NEXT:    fmov s3, w17
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    mov v3.h[1], w16
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w10
-; NONEON-NOSVE-NEXT:    mov v3.h[2], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w11
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov v3.h[3], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    fcvt s4, h6
-; NONEON-NOSVE-NEXT:    fcvt s5, h7
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w9
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v3.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w13
-; NONEON-NOSVE-NEXT:    mov v3.h[5], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, vc
-; NONEON-NOSVE-NEXT:    fcmp s1, s0
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w14
-; NONEON-NOSVE-NEXT:    mov v3.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, vc
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w15
-; NONEON-NOSVE-NEXT:    mov v3.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp ueq <16 x half> %op1, %op2
@@ -584,139 +220,6 @@ define void @fcmp_one_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_one_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h2
-; NONEON-NOSVE-NEXT:    mov h5, v2.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h1
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s6, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w9, mi
-; NONEON-NOSVE-NEXT:    csinv w12, w9, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s7, s5
-; NONEON-NOSVE-NEXT:    mov h5, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w9, mi
-; NONEON-NOSVE-NEXT:    csinv w10, w9, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x1]
-; NONEON-NOSVE-NEXT:    csetm w9, mi
-; NONEON-NOSVE-NEXT:    csinv w11, w9, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s6, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s6, h16
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w9, mi
-; NONEON-NOSVE-NEXT:    csinv w9, w9, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s7, s5
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w13, mi
-; NONEON-NOSVE-NEXT:    csinv w13, w13, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s6, s3
-; NONEON-NOSVE-NEXT:    fcvt s3, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h7
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[2]
-; NONEON-NOSVE-NEXT:    csetm w14, mi
-; NONEON-NOSVE-NEXT:    csinv w14, w14, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s4, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w15, mi
-; NONEON-NOSVE-NEXT:    csinv w15, w15, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s5, s3
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w16, mi
-; NONEON-NOSVE-NEXT:    csinv w16, w16, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s4, s2
-; NONEON-NOSVE-NEXT:    fcvt s4, h3
-; NONEON-NOSVE-NEXT:    fmov s2, w12
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w17, mi
-; NONEON-NOSVE-NEXT:    csinv w17, w17, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[4]
-; NONEON-NOSVE-NEXT:    fmov s3, w17
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    mov v3.h[1], w16
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w10
-; NONEON-NOSVE-NEXT:    mov v3.h[2], w8
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w11
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov v3.h[3], w8
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    fcvt s4, h6
-; NONEON-NOSVE-NEXT:    fcvt s5, h7
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w9
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v3.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w13
-; NONEON-NOSVE-NEXT:    mov v3.h[5], w8
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, le
-; NONEON-NOSVE-NEXT:    fcmp s1, s0
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w14
-; NONEON-NOSVE-NEXT:    mov v3.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    csinv w8, w8, wzr, le
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w15
-; NONEON-NOSVE-NEXT:    mov v3.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp one <16 x half> %op1, %op2
@@ -741,123 +244,6 @@ define void @fcmp_une_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_une_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, ne
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, ne
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, ne
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, ne
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, ne
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, ne
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, ne
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, ne
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, ne
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp une <16 x half> %op1, %op2
@@ -882,123 +268,6 @@ define void @fcmp_ogt_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_ogt_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, gt
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, gt
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, gt
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, gt
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, gt
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, gt
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, gt
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, gt
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, gt
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp ogt <16 x half> %op1, %op2
@@ -1026,123 +295,6 @@ define void @fcmp_ugt_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    eor z0.d, z2.d, z0.d
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_ugt_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, hi
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, hi
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, hi
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, hi
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, hi
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, hi
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, hi
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, hi
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, hi
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, hi
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, hi
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, hi
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, hi
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, hi
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, hi
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, hi
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp ugt <16 x half> %op1, %op2
@@ -1167,123 +319,6 @@ define void @fcmp_olt_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_olt_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, mi
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, mi
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, mi
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, mi
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, mi
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, mi
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, mi
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, mi
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, mi
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, mi
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp olt <16 x half> %op1, %op2
@@ -1311,123 +346,6 @@ define void @fcmp_ult_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    eor z0.d, z2.d, z0.d
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_ult_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, lt
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, lt
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, lt
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, lt
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, lt
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, lt
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, lt
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, lt
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, lt
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp ult <16 x half> %op1, %op2
@@ -1452,123 +370,6 @@ define void @fcmp_oge_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_oge_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, ge
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, ge
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, ge
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, ge
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, ge
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, ge
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, ge
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, ge
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, ge
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp oge <16 x half> %op1, %op2
@@ -1596,123 +397,6 @@ define void @fcmp_uge_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    eor z0.d, z2.d, z0.d
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_uge_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, pl
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, pl
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, pl
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, pl
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, pl
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, pl
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, pl
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, pl
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, pl
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, pl
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, pl
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, pl
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, pl
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, pl
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, pl
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, pl
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp uge <16 x half> %op1, %op2
@@ -1737,123 +421,6 @@ define void @fcmp_ole_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_ole_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, ls
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, ls
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, ls
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, ls
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, ls
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, ls
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, ls
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, ls
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, ls
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, ls
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, ls
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, ls
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, ls
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, ls
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, ls
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, ls
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp ole <16 x half> %op1, %op2
@@ -1881,123 +448,6 @@ define void @fcmp_ule_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    eor z0.d, z2.d, z0.d
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_ule_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, le
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, le
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, le
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, le
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, le
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, le
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, le
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, le
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, le
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp ule <16 x half> %op1, %op2
@@ -2022,123 +472,6 @@ define void @fcmp_uno_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_uno_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, vs
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, vs
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, vs
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, vs
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, vs
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, vs
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, vs
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, vs
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, vs
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, vs
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, vs
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, vs
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, vs
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, vs
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, vs
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, vs
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp uno <16 x half> %op1, %op2
@@ -2166,123 +499,6 @@ define void @fcmp_ord_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    eor z0.d, z2.d, z0.d
 ; CHECK-NEXT:    stp q1, q0, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_ord_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, vc
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, vc
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, vc
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, vc
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, vc
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, vc
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, vc
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, vc
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, vc
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, vc
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, vc
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, vc
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, vc
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, vc
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, vc
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, vc
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp ord <16 x half> %op1, %op2
@@ -2307,123 +523,6 @@ define void @fcmp_eq_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_eq_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, eq
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, eq
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, eq
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, eq
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, eq
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, eq
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, eq
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp fast oeq <16 x half> %op1, %op2
@@ -2448,123 +547,6 @@ define void @fcmp_ne_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_ne_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, ne
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, ne
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, ne
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, ne
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, ne
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, ne
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, ne
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, ne
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, ne
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp fast one <16 x half> %op1, %op2
@@ -2589,123 +571,6 @@ define void @fcmp_gt_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_gt_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, gt
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, gt
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, gt
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, gt
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, gt
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, gt
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, gt
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, gt
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, gt
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, gt
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp fast ogt <16 x half> %op1, %op2
@@ -2730,123 +595,6 @@ define void @fcmp_lt_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_lt_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, lt
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, lt
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, lt
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, lt
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, lt
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, lt
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, lt
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, lt
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, lt
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, lt
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp fast olt <16 x half> %op1, %op2
@@ -2871,123 +619,6 @@ define void @fcmp_ge_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_ge_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, ge
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, ge
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, ge
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, ge
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, ge
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, ge
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, ge
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, ge
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, ge
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, ge
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp fast oge <16 x half> %op1, %op2
@@ -3012,123 +643,6 @@ define void @fcmp_le_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcmp_le_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x1, #16]
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h2
-; NONEON-NOSVE-NEXT:    fcvt s7, h1
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h0, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w12, le
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w11, le
-; NONEON-NOSVE-NEXT:    fcmp s3, s0
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w9, le
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    csetm w10, le
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    csetm w13, le
-; NONEON-NOSVE-NEXT:    fcmp s7, s3
-; NONEON-NOSVE-NEXT:    fmov s7, w12
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    csetm w14, le
-; NONEON-NOSVE-NEXT:    fcmp s6, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov v7.h[1], w8
-; NONEON-NOSVE-NEXT:    csetm w15, le
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    mov v7.h[2], w11
-; NONEON-NOSVE-NEXT:    csetm w16, le
-; NONEON-NOSVE-NEXT:    fcmp s5, s2
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    csetm w17, le
-; NONEON-NOSVE-NEXT:    mov v7.h[3], w9
-; NONEON-NOSVE-NEXT:    fmov s2, w17
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w16
-; NONEON-NOSVE-NEXT:    mov v7.h[4], w10
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    mov v7.h[5], w13
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v7.h[6], w14
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    fcmp s6, s5
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v7.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    fcmp s4, s3
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    fcmp s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    csetm w8, le
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    stp q2, q7, [x2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %cmp = fcmp fast ole <16 x half> %op1, %op2
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll
index ac0b6c0e0440..9bdde14e8d83 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -18,17 +17,6 @@ define void @fp_convert_combine_crash(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fp_convert_combine_crash:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov v0.4s, #8.00000000
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fmul v1.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fmul v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %f = load <8 x float>, ptr %a
   %mul.i = fmul <8 x float> %f, <float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00,
                                  float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00>
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
index 16f30adbd14e..244a40510173 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -18,12 +17,6 @@ define void @fcvt_v2f16_to_v2f32(<2 x half> %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.s, p0/m, z0.h
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v2f16_to_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %res = fpext <2 x half> %a to <2 x float>
   store <2 x float> %res, ptr %b
   ret void
@@ -38,12 +31,6 @@ define void @fcvt_v4f16_to_v4f32(<4 x half> %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.s, p0/m, z0.h
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v4f16_to_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %res = fpext <4 x half> %a to <4 x float>
   store <4 x float> %res, ptr %b
   ret void
@@ -61,17 +48,6 @@ define void @fcvt_v8f16_to_v8f32(<8 x half> %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.s, p0/m, z0.h
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v8f16_to_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = fpext <8 x half> %a to <8 x float>
   store <8 x float> %res, ptr %b
   ret void
@@ -96,21 +72,6 @@ define void @fcvt_v16f16_to_v16f32(<16 x half> %a, ptr %b) {
 ; CHECK-NEXT:    stp q3, q0, [x0]
 ; CHECK-NEXT:    stp q2, q1, [x0, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v16f16_to_v16f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v3.4h
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %res = fpext <16 x half> %a to <16 x float>
   store <16 x float> %res, ptr %b
   ret void
@@ -129,13 +90,6 @@ define void @fcvt_v2f16_v2f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.s, p0/m, z0.h
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v2f16_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr s0, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x half>, ptr %a
   %res = fpext <2 x half> %op1 to <2 x float>
   store <2 x float> %res, ptr %b
@@ -150,13 +104,6 @@ define void @fcvt_v4f16_v4f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.s, p0/m, z0.h
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v4f16_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x half>, ptr %a
   %res = fpext <4 x half> %op1 to <4 x float>
   store <4 x float> %res, ptr %b
@@ -174,18 +121,6 @@ define void @fcvt_v8f16_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z1.s, p0/m, z1.h
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v8f16_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x half>, ptr %a
   %res = fpext <8 x half> %op1 to <8 x float>
   store <8 x float> %res, ptr %b
@@ -210,22 +145,6 @@ define void @fcvt_v16f16_v16f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q0, q1, [x1, #32]
 ; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v16f16_v16f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v3.4h
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %res = fpext <16 x half> %op1 to <16 x float>
   store <16 x float> %res, ptr %b
@@ -243,13 +162,6 @@ define void @fcvt_v1f16_v1f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt d0, h0
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v1f16_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr h0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt d0, h0
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <1 x half>, ptr %a
   %res = fpext <1 x half> %op1 to <1 x double>
   store <1 x double> %res, ptr %b
@@ -264,14 +176,6 @@ define void @fcvt_v2f16_v2f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.d, p0/m, z0.h
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v2f16_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr s0, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x half>, ptr %a
   %res = fpext <2 x half> %op1 to <2 x double>
   store <2 x double> %res, ptr %b
@@ -289,19 +193,6 @@ define void @fcvt_v4f16_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z1.d, p0/m, z1.h
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v4f16_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v1.2s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x half>, ptr %a
   %res = fpext <4 x half> %op1 to <4 x double>
   store <4 x double> %res, ptr %b
@@ -326,26 +217,6 @@ define void @fcvt_v8f16_v8f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q0, q1, [x1, #32]
 ; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v8f16_v8f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-48]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v1.2s
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    fcvtl v2.2d, v2.2s
-; NONEON-NOSVE-NEXT:    fcvtl v3.2d, v3.2s
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x half>, ptr %a
   %res = fpext <8 x half> %op1 to <8 x double>
   store <8 x double> %res, ptr %b
@@ -387,38 +258,6 @@ define void @fcvt_v16f16_v16f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q4, q0, [x1, #32]
 ; CHECK-NEXT:    stp q1, q2, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v16f16_v16f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-96]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v3.4h
-; NONEON-NOSVE-NEXT:    stp q2, q0, [sp, #32]
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtl v2.2d, v2.2s
-; NONEON-NOSVE-NEXT:    stp q3, q1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldr d5, [sp, #56]
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v1.2s
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #40]
-; NONEON-NOSVE-NEXT:    fcvtl v5.2d, v5.2s
-; NONEON-NOSVE-NEXT:    fcvtl v3.2d, v3.2s
-; NONEON-NOSVE-NEXT:    fcvtl v4.2d, v4.2s
-; NONEON-NOSVE-NEXT:    stp q0, q5, [x1]
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v7.2s
-; NONEON-NOSVE-NEXT:    stp q1, q4, [x1, #64]
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v6.2s
-; NONEON-NOSVE-NEXT:    stp q2, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q1, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #96
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %res = fpext <16 x half> %op1 to <16 x double>
   store <16 x double> %res, ptr %b
@@ -436,13 +275,6 @@ define void @fcvt_v1f32_v1f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt d0, s0
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v1f32_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr s0, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <1 x float>, ptr %a
   %res = fpext <1 x float> %op1 to <1 x double>
   store <1 x double> %res, ptr %b
@@ -457,13 +289,6 @@ define void @fcvt_v2f32_v2f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.d, p0/m, z0.s
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v2f32_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x float>, ptr %a
   %res = fpext <2 x float> %op1 to <2 x double>
   store <2 x double> %res, ptr %b
@@ -481,18 +306,6 @@ define void @fcvt_v4f32_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z1.d, p0/m, z1.s
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v4f32_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v1.2s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x float>, ptr %a
   %res = fpext <4 x float> %op1 to <4 x double>
   store <4 x double> %res, ptr %b
@@ -517,22 +330,6 @@ define void @fcvt_v8f32_v8f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q0, q1, [x1, #32]
 ; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v8f32_v8f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v1.2s
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtl v2.2d, v2.2s
-; NONEON-NOSVE-NEXT:    fcvtl v3.2d, v3.2s
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %res = fpext <8 x float> %op1 to <8 x double>
   store <8 x double> %res, ptr %b
@@ -551,13 +348,6 @@ define void @fcvt_v2f32_v2f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.h, p0/m, z0.s
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v2f32_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    str s0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x float>, ptr %a
   %res = fptrunc <2 x float> %op1 to <2 x half>
   store <2 x half> %res, ptr %b
@@ -572,13 +362,6 @@ define void @fcvt_v4f32_v4f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.h, p0/m, z0.s
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v4f32_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x float>, ptr %a
   %res = fptrunc <4 x float> %op1 to <4 x half>
   store <4 x half> %res, ptr %b
@@ -596,14 +379,6 @@ define void @fcvt_v8f32_v8f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x1, x8, lsl #1]
 ; CHECK-NEXT:    st1h { z1.s }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v8f32_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %res = fptrunc <8 x float> %op1 to <8 x half>
   store <8 x half> %res, ptr %b
@@ -622,13 +397,6 @@ define void @fcvt_v1f64_v1f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.h, p0/m, z0.d
 ; CHECK-NEXT:    st1h { z0.d }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v1f64_v1f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    fcvt h0, d0
-; NONEON-NOSVE-NEXT:    str h0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <1 x double>, ptr %a
   %res = fptrunc <1 x double> %op1 to <1 x half>
   store <1 x half> %res, ptr %b
@@ -643,14 +411,6 @@ define void @fcvt_v2f64_v2f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.h, p0/m, z0.d
 ; CHECK-NEXT:    st1h { z0.d }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v2f64_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    fcvtxn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    str s0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x double>, ptr %a
   %res = fptrunc <2 x double> %op1 to <2 x half>
   store <2 x half> %res, ptr %b
@@ -668,15 +428,6 @@ define void @fcvt_v4f64_v4f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    st1h { z0.d }, p0, [x1, x8, lsl #1]
 ; CHECK-NEXT:    st1h { z1.d }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v4f64_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtxn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtxn2 v0.4s, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %res = fptrunc <4 x double> %op1 to <4 x half>
   store <4 x half> %res, ptr %b
@@ -695,13 +446,6 @@ define void @fcvt_v1f64_v1f32(<1 x double> %op1, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.s, p0/m, z0.d
 ; CHECK-NEXT:    st1w { z0.d }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v1f64_v1f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fcvtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    str s0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %res = fptrunc <1 x double> %op1 to <1 x float>
   store <1 x float> %res, ptr %b
   ret void
@@ -715,12 +459,6 @@ define void @fcvt_v2f64_v2f32(<2 x double> %op1, ptr %b) {
 ; CHECK-NEXT:    fcvt z0.s, p0/m, z0.d
 ; CHECK-NEXT:    st1w { z0.d }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v2f64_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %res = fptrunc <2 x double> %op1 to <2 x float>
   store <2 x float> %res, ptr %b
   ret void
@@ -737,14 +475,6 @@ define void @fcvt_v4f64_v4f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    st1w { z0.d }, p0, [x1, x8, lsl #2]
 ; CHECK-NEXT:    st1w { z1.d }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvt_v4f64_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.4s, v1.2d
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %res = fptrunc <4 x double> %op1 to <4 x float>
   store <4 x float> %res, ptr %b
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll
index 44d7116e5f87..cbe71d715a8f 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -18,18 +17,6 @@ define <4 x half> @fma_v4f16(<4 x half> %op1, <4 x half> %op2, <4 x half> %op3)
 ; CHECK-NEXT:    fmad z0.h, p0/m, z1.h, z2.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fmul v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %mul = fmul contract <4 x half> %op1, %op2
   %res = fadd contract <4 x half> %mul, %op3
   ret <4 x half> %res
@@ -45,26 +32,6 @@ define <8 x half> @fma_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3)
 ; CHECK-NEXT:    fmad z0.h, p0/m, z1.h, z2.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v4.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fmul v3.4s, v4.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fmul v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v2.8h
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v0.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fadd v1.4s, v1.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %mul = fmul contract <8 x half> %op1, %op2
   %res = fadd contract <8 x half> %mul, %op3
   ret <8 x half> %res
@@ -82,46 +49,6 @@ define void @fma_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    fmla z1.h, p0/m, z3.h, z4.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    fcvtl v5.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v7.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl v4.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v6.4s, v3.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v3.4s, v3.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v2.8h
-; NONEON-NOSVE-NEXT:    fmul v4.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    fmul v5.4s, v7.4s, v6.4s
-; NONEON-NOSVE-NEXT:    fmul v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fmul v2.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v4.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v5.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v2.4s
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x2]
-; NONEON-NOSVE-NEXT:    fcvtl v4.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v5.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v6.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl v7.4s, v3.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v2.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v3.4s, v3.8h
-; NONEON-NOSVE-NEXT:    fadd v4.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    fadd v5.4s, v7.4s, v6.4s
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fadd v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v4.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v5.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v2.4s
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %op3 = load <16 x half>, ptr %c
@@ -141,12 +68,6 @@ define <2 x float> @fma_v2f32(<2 x float> %op1, <2 x float> %op2, <2 x float> %o
 ; CHECK-NEXT:    fmad z0.s, p0/m, z1.s, z2.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmla v2.2s, v1.2s, v0.2s
-; NONEON-NOSVE-NEXT:    fmov d0, d2
-; NONEON-NOSVE-NEXT:    ret
   %mul = fmul contract <2 x float> %op1, %op2
   %res = fadd contract <2 x float> %mul, %op3
   ret <2 x float> %res
@@ -162,12 +83,6 @@ define <4 x float> @fma_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %o
 ; CHECK-NEXT:    fmad z0.s, p0/m, z1.s, z2.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmla v2.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %mul = fmul contract <4 x float> %op1, %op2
   %res = fadd contract <4 x float> %mul, %op3
   ret <4 x float> %res
@@ -185,16 +100,6 @@ define void @fma_v8f32(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    fmla z1.s, p0/m, z3.s, z4.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q4, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q5, [x2]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    fmla v1.4s, v0.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fmla v5.4s, v4.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q1, q5, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %op3 = load <8 x float>, ptr %c
@@ -209,11 +114,6 @@ define <1 x double> @fma_v1f64(<1 x double> %op1, <1 x double> %op2, <1 x double
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmadd d0, d0, d1, d2
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmadd d0, d0, d1, d2
-; NONEON-NOSVE-NEXT:    ret
   %mul = fmul contract <1 x double> %op1, %op2
   %res = fadd contract <1 x double> %mul, %op3
   ret <1 x double> %res
@@ -229,12 +129,6 @@ define <2 x double> @fma_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double
 ; CHECK-NEXT:    fmad z0.d, p0/m, z1.d, z2.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmla v2.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %mul = fmul contract <2 x double> %op1, %op2
   %res = fadd contract <2 x double> %mul, %op3
   ret <2 x double> %res
@@ -252,16 +146,6 @@ define void @fma_v4f64(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    fmla z1.d, p0/m, z3.d, z4.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fma_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q4, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q5, [x2]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    fmla v1.2d, v0.2d, v2.2d
-; NONEON-NOSVE-NEXT:    fmla v5.2d, v4.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q1, q5, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %op3 = load <4 x double>, ptr %c
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll
index bc7659c06ad0..94a74763aa0e 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -18,38 +17,6 @@ define <4 x half> @fmaxnm_v4f16(<4 x half> %op1, <4 x half> %op2) {
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxnm_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h1
-; NONEON-NOSVE-NEXT:    fcvt s7, h0
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s2, s3, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fmaxnm s5, s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    fmaxnm s3, s4, s3
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt h0, s5
-; NONEON-NOSVE-NEXT:    fcvt s4, h6
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v2.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h2, s3
-; NONEON-NOSVE-NEXT:    fmaxnm s1, s4, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[2], v2.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[3], v1.h[0]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.maxnum.v4f16(<4 x half> %op1, <4 x half> %op2)
   ret <4 x half> %res
 }
@@ -63,64 +30,6 @@ define <8 x half> @fmaxnm_v8f16(<8 x half> %op1, <8 x half> %op2) {
 ; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxnm_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fmaxnm s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fmaxnm s3, s3, s2
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s4
-; NONEON-NOSVE-NEXT:    fmaxnm s4, s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fmaxnm s5, s5, s16
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    mov v2.h[1], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s3, h6
-; NONEON-NOSVE-NEXT:    fcvt s6, h7
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h5, s5
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    mov v2.h[2], v4.h[0]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[6]
-; NONEON-NOSVE-NEXT:    fmaxnm s3, s6, s3
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], v5.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h6
-; NONEON-NOSVE-NEXT:    fmaxnm s6, s16, s7
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v2.h[4], v3.h[0]
-; NONEON-NOSVE-NEXT:    fmaxnm s4, s5, s4
-; NONEON-NOSVE-NEXT:    fcvt h3, s6
-; NONEON-NOSVE-NEXT:    fmaxnm s0, s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[5], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h3, s4
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    mov v2.h[6], v3.h[0]
-; NONEON-NOSVE-NEXT:    mov v2.h[7], v0.h[0]
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.maxnum.v8f16(<8 x half> %op1, <8 x half> %op2)
   ret <8 x half> %res
 }
@@ -136,119 +45,6 @@ define void @fmaxnm_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmaxnm z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxnm_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h18, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h17, v3.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s19, h0
-; NONEON-NOSVE-NEXT:    fcvt s20, h3
-; NONEON-NOSVE-NEXT:    fcvt s21, h2
-; NONEON-NOSVE-NEXT:    mov h22, v3.h[2]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fcvt s18, h18
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fmaxnm s4, s19, s4
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h24, v3.h[3]
-; NONEON-NOSVE-NEXT:    fmaxnm s20, s21, s20
-; NONEON-NOSVE-NEXT:    fcvt s21, h22
-; NONEON-NOSVE-NEXT:    fcvt s22, h23
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h25, v2.h[6]
-; NONEON-NOSVE-NEXT:    fmaxnm s5, s7, s5
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[3]
-; NONEON-NOSVE-NEXT:    fmaxnm s6, s16, s6
-; NONEON-NOSVE-NEXT:    fmaxnm s16, s18, s17
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s18, h19
-; NONEON-NOSVE-NEXT:    fcvt s19, h24
-; NONEON-NOSVE-NEXT:    mov h24, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h17, s5
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt h5, s20
-; NONEON-NOSVE-NEXT:    fmaxnm s20, s22, s21
-; NONEON-NOSVE-NEXT:    fcvt h16, s16
-; NONEON-NOSVE-NEXT:    fcvt s21, h23
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    mov h22, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov v4.h[1], v17.h[0]
-; NONEON-NOSVE-NEXT:    mov h17, v1.h[4]
-; NONEON-NOSVE-NEXT:    fmaxnm s7, s18, s7
-; NONEON-NOSVE-NEXT:    mov h18, v3.h[4]
-; NONEON-NOSVE-NEXT:    mov v5.h[1], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h16, s20
-; NONEON-NOSVE-NEXT:    fmaxnm s19, s21, s19
-; NONEON-NOSVE-NEXT:    fcvt s20, h23
-; NONEON-NOSVE-NEXT:    mov h21, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    mov v4.h[2], v6.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s6, h17
-; NONEON-NOSVE-NEXT:    fcvt s17, h22
-; NONEON-NOSVE-NEXT:    fcvt h7, s7
-; NONEON-NOSVE-NEXT:    fcvt s18, h18
-; NONEON-NOSVE-NEXT:    mov h22, v3.h[5]
-; NONEON-NOSVE-NEXT:    mov v5.h[2], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h16, s19
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmaxnm s6, s17, s6
-; NONEON-NOSVE-NEXT:    mov h17, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fmaxnm s18, s20, s18
-; NONEON-NOSVE-NEXT:    mov h20, v3.h[6]
-; NONEON-NOSVE-NEXT:    mov v4.h[3], v7.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s7, h22
-; NONEON-NOSVE-NEXT:    fcvt s22, h23
-; NONEON-NOSVE-NEXT:    mov v5.h[3], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s16, h21
-; NONEON-NOSVE-NEXT:    fcvt s21, h24
-; NONEON-NOSVE-NEXT:    fcvt s19, h19
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fcvt s23, h25
-; NONEON-NOSVE-NEXT:    fcvt h18, s18
-; NONEON-NOSVE-NEXT:    fcvt s20, h20
-; NONEON-NOSVE-NEXT:    mov h3, v3.h[7]
-; NONEON-NOSVE-NEXT:    fmaxnm s7, s22, s7
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fmaxnm s16, s21, s16
-; NONEON-NOSVE-NEXT:    mov v4.h[4], v6.h[0]
-; NONEON-NOSVE-NEXT:    fmaxnm s6, s19, s17
-; NONEON-NOSVE-NEXT:    mov v5.h[4], v18.h[0]
-; NONEON-NOSVE-NEXT:    fmaxnm s17, s23, s20
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt h7, s7
-; NONEON-NOSVE-NEXT:    fmaxnm s0, s0, s1
-; NONEON-NOSVE-NEXT:    fcvt h16, s16
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    fmaxnm s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h3, s17
-; NONEON-NOSVE-NEXT:    mov v5.h[5], v7.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    mov v4.h[5], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s2
-; NONEON-NOSVE-NEXT:    mov v5.h[6], v3.h[0]
-; NONEON-NOSVE-NEXT:    mov v4.h[6], v6.h[0]
-; NONEON-NOSVE-NEXT:    mov v5.h[7], v1.h[0]
-; NONEON-NOSVE-NEXT:    mov v4.h[7], v0.h[0]
-; NONEON-NOSVE-NEXT:    stp q5, q4, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %res = call <16 x half> @llvm.maxnum.v16f16(<16 x half> %op1, <16 x half> %op2)
@@ -265,11 +61,6 @@ define <2 x float> @fmaxnm_v2f32(<2 x float> %op1, <2 x float> %op2) {
 ; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxnm_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmaxnm v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %op1, <2 x float> %op2)
   ret <2 x float> %res
 }
@@ -283,11 +74,6 @@ define <4 x float> @fmaxnm_v4f32(<4 x float> %op1, <4 x float> %op2) {
 ; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxnm_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmaxnm v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.maxnum.v4f32(<4 x float> %op1, <4 x float> %op2)
   ret <4 x float> %res
 }
@@ -303,15 +89,6 @@ define void @fmaxnm_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmaxnm z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxnm_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fmaxnm v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fmaxnm v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %res = call <8 x float> @llvm.maxnum.v8f32(<8 x float> %op1, <8 x float> %op2)
@@ -324,11 +101,6 @@ define <1 x double> @fmaxnm_v1f64(<1 x double> %op1, <1 x double> %op2) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmaxnm d0, d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxnm_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmaxnm d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x double> @llvm.maxnum.v1f64(<1 x double> %op1, <1 x double> %op2)
   ret <1 x double> %res
 }
@@ -342,11 +114,6 @@ define <2 x double> @fmaxnm_v2f64(<2 x double> %op1, <2 x double> %op2) {
 ; CHECK-NEXT:    fmaxnm z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxnm_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmaxnm v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.maxnum.v2f64(<2 x double> %op1, <2 x double> %op2)
   ret <2 x double> %res
 }
@@ -362,15 +129,6 @@ define void @fmaxnm_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmaxnm z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxnm_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fmaxnm v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fmaxnm v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %res = call <4 x double> @llvm.maxnum.v4f64(<4 x double> %op1, <4 x double> %op2)
@@ -391,38 +149,6 @@ define <4 x half> @fminnm_v4f16(<4 x half> %op1, <4 x half> %op2) {
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminnm_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h1
-; NONEON-NOSVE-NEXT:    fcvt s7, h0
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s2, s3, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fminnm s5, s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    fminnm s3, s4, s3
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt h0, s5
-; NONEON-NOSVE-NEXT:    fcvt s4, h6
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v2.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h2, s3
-; NONEON-NOSVE-NEXT:    fminnm s1, s4, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[2], v2.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[3], v1.h[0]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.minnum.v4f16(<4 x half> %op1, <4 x half> %op2)
   ret <4 x half> %res
 }
@@ -436,64 +162,6 @@ define <8 x half> @fminnm_v8f16(<8 x half> %op1, <8 x half> %op2) {
 ; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminnm_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fminnm s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fminnm s3, s3, s2
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s4
-; NONEON-NOSVE-NEXT:    fminnm s4, s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fminnm s5, s5, s16
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    mov v2.h[1], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s3, h6
-; NONEON-NOSVE-NEXT:    fcvt s6, h7
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h5, s5
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    mov v2.h[2], v4.h[0]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[6]
-; NONEON-NOSVE-NEXT:    fminnm s3, s6, s3
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], v5.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h6
-; NONEON-NOSVE-NEXT:    fminnm s6, s16, s7
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v2.h[4], v3.h[0]
-; NONEON-NOSVE-NEXT:    fminnm s4, s5, s4
-; NONEON-NOSVE-NEXT:    fcvt h3, s6
-; NONEON-NOSVE-NEXT:    fminnm s0, s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[5], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h3, s4
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    mov v2.h[6], v3.h[0]
-; NONEON-NOSVE-NEXT:    mov v2.h[7], v0.h[0]
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.minnum.v8f16(<8 x half> %op1, <8 x half> %op2)
   ret <8 x half> %res
 }
@@ -509,119 +177,6 @@ define void @fminnm_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fminnm z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminnm_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h18, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h17, v3.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s19, h0
-; NONEON-NOSVE-NEXT:    fcvt s20, h3
-; NONEON-NOSVE-NEXT:    fcvt s21, h2
-; NONEON-NOSVE-NEXT:    mov h22, v3.h[2]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fcvt s18, h18
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fminnm s4, s19, s4
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h24, v3.h[3]
-; NONEON-NOSVE-NEXT:    fminnm s20, s21, s20
-; NONEON-NOSVE-NEXT:    fcvt s21, h22
-; NONEON-NOSVE-NEXT:    fcvt s22, h23
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h25, v2.h[6]
-; NONEON-NOSVE-NEXT:    fminnm s5, s7, s5
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[3]
-; NONEON-NOSVE-NEXT:    fminnm s6, s16, s6
-; NONEON-NOSVE-NEXT:    fminnm s16, s18, s17
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s18, h19
-; NONEON-NOSVE-NEXT:    fcvt s19, h24
-; NONEON-NOSVE-NEXT:    mov h24, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h17, s5
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt h5, s20
-; NONEON-NOSVE-NEXT:    fminnm s20, s22, s21
-; NONEON-NOSVE-NEXT:    fcvt h16, s16
-; NONEON-NOSVE-NEXT:    fcvt s21, h23
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    mov h22, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov v4.h[1], v17.h[0]
-; NONEON-NOSVE-NEXT:    mov h17, v1.h[4]
-; NONEON-NOSVE-NEXT:    fminnm s7, s18, s7
-; NONEON-NOSVE-NEXT:    mov h18, v3.h[4]
-; NONEON-NOSVE-NEXT:    mov v5.h[1], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h16, s20
-; NONEON-NOSVE-NEXT:    fminnm s19, s21, s19
-; NONEON-NOSVE-NEXT:    fcvt s20, h23
-; NONEON-NOSVE-NEXT:    mov h21, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    mov v4.h[2], v6.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s6, h17
-; NONEON-NOSVE-NEXT:    fcvt s17, h22
-; NONEON-NOSVE-NEXT:    fcvt h7, s7
-; NONEON-NOSVE-NEXT:    fcvt s18, h18
-; NONEON-NOSVE-NEXT:    mov h22, v3.h[5]
-; NONEON-NOSVE-NEXT:    mov v5.h[2], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h16, s19
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fminnm s6, s17, s6
-; NONEON-NOSVE-NEXT:    mov h17, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fminnm s18, s20, s18
-; NONEON-NOSVE-NEXT:    mov h20, v3.h[6]
-; NONEON-NOSVE-NEXT:    mov v4.h[3], v7.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s7, h22
-; NONEON-NOSVE-NEXT:    fcvt s22, h23
-; NONEON-NOSVE-NEXT:    mov v5.h[3], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s16, h21
-; NONEON-NOSVE-NEXT:    fcvt s21, h24
-; NONEON-NOSVE-NEXT:    fcvt s19, h19
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fcvt s23, h25
-; NONEON-NOSVE-NEXT:    fcvt h18, s18
-; NONEON-NOSVE-NEXT:    fcvt s20, h20
-; NONEON-NOSVE-NEXT:    mov h3, v3.h[7]
-; NONEON-NOSVE-NEXT:    fminnm s7, s22, s7
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fminnm s16, s21, s16
-; NONEON-NOSVE-NEXT:    mov v4.h[4], v6.h[0]
-; NONEON-NOSVE-NEXT:    fminnm s6, s19, s17
-; NONEON-NOSVE-NEXT:    mov v5.h[4], v18.h[0]
-; NONEON-NOSVE-NEXT:    fminnm s17, s23, s20
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt h7, s7
-; NONEON-NOSVE-NEXT:    fminnm s0, s0, s1
-; NONEON-NOSVE-NEXT:    fcvt h16, s16
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    fminnm s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h3, s17
-; NONEON-NOSVE-NEXT:    mov v5.h[5], v7.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    mov v4.h[5], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s2
-; NONEON-NOSVE-NEXT:    mov v5.h[6], v3.h[0]
-; NONEON-NOSVE-NEXT:    mov v4.h[6], v6.h[0]
-; NONEON-NOSVE-NEXT:    mov v5.h[7], v1.h[0]
-; NONEON-NOSVE-NEXT:    mov v4.h[7], v0.h[0]
-; NONEON-NOSVE-NEXT:    stp q5, q4, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %res = call <16 x half> @llvm.minnum.v16f16(<16 x half> %op1, <16 x half> %op2)
@@ -638,11 +193,6 @@ define <2 x float> @fminnm_v2f32(<2 x float> %op1, <2 x float> %op2) {
 ; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminnm_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fminnm v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.minnum.v2f32(<2 x float> %op1, <2 x float> %op2)
   ret <2 x float> %res
 }
@@ -656,11 +206,6 @@ define <4 x float> @fminnm_v4f32(<4 x float> %op1, <4 x float> %op2) {
 ; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminnm_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fminnm v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.minnum.v4f32(<4 x float> %op1, <4 x float> %op2)
   ret <4 x float> %res
 }
@@ -676,15 +221,6 @@ define void @fminnm_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fminnm z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminnm_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fminnm v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fminnm v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %res = call <8 x float> @llvm.minnum.v8f32(<8 x float> %op1, <8 x float> %op2)
@@ -697,11 +233,6 @@ define <1 x double> @fminnm_v1f64(<1 x double> %op1, <1 x double> %op2) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fminnm d0, d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminnm_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fminnm d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x double> @llvm.minnum.v1f64(<1 x double> %op1, <1 x double> %op2)
   ret <1 x double> %res
 }
@@ -715,11 +246,6 @@ define <2 x double> @fminnm_v2f64(<2 x double> %op1, <2 x double> %op2) {
 ; CHECK-NEXT:    fminnm z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminnm_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fminnm v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.minnum.v2f64(<2 x double> %op1, <2 x double> %op2)
   ret <2 x double> %res
 }
@@ -735,15 +261,6 @@ define void @fminnm_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fminnm z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminnm_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fminnm v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fminnm v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %res = call <4 x double> @llvm.minnum.v4f64(<4 x double> %op1, <4 x double> %op2)
@@ -764,38 +281,6 @@ define <4 x half> @fmax_v4f16(<4 x half> %op1, <4 x half> %op2) {
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmax_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h1
-; NONEON-NOSVE-NEXT:    fcvt s7, h0
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s2, s3, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fmax s5, s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    fmax s3, s4, s3
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt h0, s5
-; NONEON-NOSVE-NEXT:    fcvt s4, h6
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v2.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h2, s3
-; NONEON-NOSVE-NEXT:    fmax s1, s4, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[2], v2.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[3], v1.h[0]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.maximum.v4f16(<4 x half> %op1, <4 x half> %op2)
   ret <4 x half> %res
 }
@@ -809,64 +294,6 @@ define <8 x half> @fmax_v8f16(<8 x half> %op1, <8 x half> %op2) {
 ; CHECK-NEXT:    fmax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmax_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fmax s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fmax s3, s3, s2
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s4
-; NONEON-NOSVE-NEXT:    fmax s4, s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fmax s5, s5, s16
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    mov v2.h[1], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s3, h6
-; NONEON-NOSVE-NEXT:    fcvt s6, h7
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h5, s5
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    mov v2.h[2], v4.h[0]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[6]
-; NONEON-NOSVE-NEXT:    fmax s3, s6, s3
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], v5.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h6
-; NONEON-NOSVE-NEXT:    fmax s6, s16, s7
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v2.h[4], v3.h[0]
-; NONEON-NOSVE-NEXT:    fmax s4, s5, s4
-; NONEON-NOSVE-NEXT:    fcvt h3, s6
-; NONEON-NOSVE-NEXT:    fmax s0, s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[5], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h3, s4
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    mov v2.h[6], v3.h[0]
-; NONEON-NOSVE-NEXT:    mov v2.h[7], v0.h[0]
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.maximum.v8f16(<8 x half> %op1, <8 x half> %op2)
   ret <8 x half> %res
 }
@@ -882,119 +309,6 @@ define void @fmax_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmax z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmax_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h18, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h17, v3.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s19, h0
-; NONEON-NOSVE-NEXT:    fcvt s20, h3
-; NONEON-NOSVE-NEXT:    fcvt s21, h2
-; NONEON-NOSVE-NEXT:    mov h22, v3.h[2]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fcvt s18, h18
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fmax s4, s19, s4
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h24, v3.h[3]
-; NONEON-NOSVE-NEXT:    fmax s20, s21, s20
-; NONEON-NOSVE-NEXT:    fcvt s21, h22
-; NONEON-NOSVE-NEXT:    fcvt s22, h23
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h25, v2.h[6]
-; NONEON-NOSVE-NEXT:    fmax s5, s7, s5
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[3]
-; NONEON-NOSVE-NEXT:    fmax s6, s16, s6
-; NONEON-NOSVE-NEXT:    fmax s16, s18, s17
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s18, h19
-; NONEON-NOSVE-NEXT:    fcvt s19, h24
-; NONEON-NOSVE-NEXT:    mov h24, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h17, s5
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt h5, s20
-; NONEON-NOSVE-NEXT:    fmax s20, s22, s21
-; NONEON-NOSVE-NEXT:    fcvt h16, s16
-; NONEON-NOSVE-NEXT:    fcvt s21, h23
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    mov h22, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov v4.h[1], v17.h[0]
-; NONEON-NOSVE-NEXT:    mov h17, v1.h[4]
-; NONEON-NOSVE-NEXT:    fmax s7, s18, s7
-; NONEON-NOSVE-NEXT:    mov h18, v3.h[4]
-; NONEON-NOSVE-NEXT:    mov v5.h[1], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h16, s20
-; NONEON-NOSVE-NEXT:    fmax s19, s21, s19
-; NONEON-NOSVE-NEXT:    fcvt s20, h23
-; NONEON-NOSVE-NEXT:    mov h21, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    mov v4.h[2], v6.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s6, h17
-; NONEON-NOSVE-NEXT:    fcvt s17, h22
-; NONEON-NOSVE-NEXT:    fcvt h7, s7
-; NONEON-NOSVE-NEXT:    fcvt s18, h18
-; NONEON-NOSVE-NEXT:    mov h22, v3.h[5]
-; NONEON-NOSVE-NEXT:    mov v5.h[2], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h16, s19
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmax s6, s17, s6
-; NONEON-NOSVE-NEXT:    mov h17, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fmax s18, s20, s18
-; NONEON-NOSVE-NEXT:    mov h20, v3.h[6]
-; NONEON-NOSVE-NEXT:    mov v4.h[3], v7.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s7, h22
-; NONEON-NOSVE-NEXT:    fcvt s22, h23
-; NONEON-NOSVE-NEXT:    mov v5.h[3], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s16, h21
-; NONEON-NOSVE-NEXT:    fcvt s21, h24
-; NONEON-NOSVE-NEXT:    fcvt s19, h19
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fcvt s23, h25
-; NONEON-NOSVE-NEXT:    fcvt h18, s18
-; NONEON-NOSVE-NEXT:    fcvt s20, h20
-; NONEON-NOSVE-NEXT:    mov h3, v3.h[7]
-; NONEON-NOSVE-NEXT:    fmax s7, s22, s7
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fmax s16, s21, s16
-; NONEON-NOSVE-NEXT:    mov v4.h[4], v6.h[0]
-; NONEON-NOSVE-NEXT:    fmax s6, s19, s17
-; NONEON-NOSVE-NEXT:    mov v5.h[4], v18.h[0]
-; NONEON-NOSVE-NEXT:    fmax s17, s23, s20
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt h7, s7
-; NONEON-NOSVE-NEXT:    fmax s0, s0, s1
-; NONEON-NOSVE-NEXT:    fcvt h16, s16
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    fmax s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h3, s17
-; NONEON-NOSVE-NEXT:    mov v5.h[5], v7.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    mov v4.h[5], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s2
-; NONEON-NOSVE-NEXT:    mov v5.h[6], v3.h[0]
-; NONEON-NOSVE-NEXT:    mov v4.h[6], v6.h[0]
-; NONEON-NOSVE-NEXT:    mov v5.h[7], v1.h[0]
-; NONEON-NOSVE-NEXT:    mov v4.h[7], v0.h[0]
-; NONEON-NOSVE-NEXT:    stp q5, q4, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %res = call <16 x half> @llvm.maximum.v16f16(<16 x half> %op1, <16 x half> %op2)
@@ -1011,11 +325,6 @@ define <2 x float> @fmax_v2f32(<2 x float> %op1, <2 x float> %op2) {
 ; CHECK-NEXT:    fmax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmax_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmax v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.maximum.v2f32(<2 x float> %op1, <2 x float> %op2)
   ret <2 x float> %res
 }
@@ -1029,11 +338,6 @@ define <4 x float> @fmax_v4f32(<4 x float> %op1, <4 x float> %op2) {
 ; CHECK-NEXT:    fmax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmax_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmax v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.maximum.v4f32(<4 x float> %op1, <4 x float> %op2)
   ret <4 x float> %res
 }
@@ -1049,15 +353,6 @@ define void @fmax_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmax z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmax_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fmax v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fmax v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %res = call <8 x float> @llvm.maximum.v8f32(<8 x float> %op1, <8 x float> %op2)
@@ -1070,11 +365,6 @@ define <1 x double> @fmax_v1f64(<1 x double> %op1, <1 x double> %op2) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmax d0, d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmax_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmax d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x double> @llvm.maximum.v1f64(<1 x double> %op1, <1 x double> %op2)
   ret <1 x double> %res
 }
@@ -1088,11 +378,6 @@ define <2 x double> @fmax_v2f64(<2 x double> %op1, <2 x double> %op2) {
 ; CHECK-NEXT:    fmax z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmax_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmax v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.maximum.v2f64(<2 x double> %op1, <2 x double> %op2)
   ret <2 x double> %res
 }
@@ -1108,15 +393,6 @@ define void @fmax_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmax z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmax_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fmax v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fmax v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %res = call <4 x double> @llvm.maximum.v4f64(<4 x double> %op1, <4 x double> %op2)
@@ -1137,38 +413,6 @@ define <4 x half> @fmin_v4f16(<4 x half> %op1, <4 x half> %op2) {
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmin_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h1
-; NONEON-NOSVE-NEXT:    fcvt s7, h0
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s2, s3, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h4
-; NONEON-NOSVE-NEXT:    fcvt s4, h5
-; NONEON-NOSVE-NEXT:    fmin s5, s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[3]
-; NONEON-NOSVE-NEXT:    fmin s3, s4, s3
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt h0, s5
-; NONEON-NOSVE-NEXT:    fcvt s4, h6
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v2.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h2, s3
-; NONEON-NOSVE-NEXT:    fmin s1, s4, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[2], v2.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[3], v1.h[0]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.minimum.v4f16(<4 x half> %op1, <4 x half> %op2)
   ret <4 x half> %res
 }
@@ -1182,64 +426,6 @@ define <8 x half> @fmin_v8f16(<8 x half> %op1, <8 x half> %op2) {
 ; CHECK-NEXT:    fmin z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmin_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fmin s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fmin s3, s3, s2
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s4
-; NONEON-NOSVE-NEXT:    fmin s4, s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fmin s5, s5, s16
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    mov v2.h[1], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s3, h6
-; NONEON-NOSVE-NEXT:    fcvt s6, h7
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h5, s5
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    mov v2.h[2], v4.h[0]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[6]
-; NONEON-NOSVE-NEXT:    fmin s3, s6, s3
-; NONEON-NOSVE-NEXT:    mov h6, v0.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], v5.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h6
-; NONEON-NOSVE-NEXT:    fmin s6, s16, s7
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov v2.h[4], v3.h[0]
-; NONEON-NOSVE-NEXT:    fmin s4, s5, s4
-; NONEON-NOSVE-NEXT:    fcvt h3, s6
-; NONEON-NOSVE-NEXT:    fmin s0, s0, s1
-; NONEON-NOSVE-NEXT:    mov v2.h[5], v3.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h3, s4
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    mov v2.h[6], v3.h[0]
-; NONEON-NOSVE-NEXT:    mov v2.h[7], v0.h[0]
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.minimum.v8f16(<8 x half> %op1, <8 x half> %op2)
   ret <8 x half> %res
 }
@@ -1255,119 +441,6 @@ define void @fmin_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmin z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmin_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h18, v2.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h17, v3.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s19, h0
-; NONEON-NOSVE-NEXT:    fcvt s20, h3
-; NONEON-NOSVE-NEXT:    fcvt s21, h2
-; NONEON-NOSVE-NEXT:    mov h22, v3.h[2]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fcvt s18, h18
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fmin s4, s19, s4
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h24, v3.h[3]
-; NONEON-NOSVE-NEXT:    fmin s20, s21, s20
-; NONEON-NOSVE-NEXT:    fcvt s21, h22
-; NONEON-NOSVE-NEXT:    fcvt s22, h23
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h25, v2.h[6]
-; NONEON-NOSVE-NEXT:    fmin s5, s7, s5
-; NONEON-NOSVE-NEXT:    mov h7, v1.h[3]
-; NONEON-NOSVE-NEXT:    fmin s6, s16, s6
-; NONEON-NOSVE-NEXT:    fmin s16, s18, s17
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s18, h19
-; NONEON-NOSVE-NEXT:    fcvt s19, h24
-; NONEON-NOSVE-NEXT:    mov h24, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h17, s5
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcvt h5, s20
-; NONEON-NOSVE-NEXT:    fmin s20, s22, s21
-; NONEON-NOSVE-NEXT:    fcvt h16, s16
-; NONEON-NOSVE-NEXT:    fcvt s21, h23
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    mov h22, v0.h[4]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov v4.h[1], v17.h[0]
-; NONEON-NOSVE-NEXT:    mov h17, v1.h[4]
-; NONEON-NOSVE-NEXT:    fmin s7, s18, s7
-; NONEON-NOSVE-NEXT:    mov h18, v3.h[4]
-; NONEON-NOSVE-NEXT:    mov v5.h[1], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h16, s20
-; NONEON-NOSVE-NEXT:    fmin s19, s21, s19
-; NONEON-NOSVE-NEXT:    fcvt s20, h23
-; NONEON-NOSVE-NEXT:    mov h21, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h23, v2.h[5]
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    mov v4.h[2], v6.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s6, h17
-; NONEON-NOSVE-NEXT:    fcvt s17, h22
-; NONEON-NOSVE-NEXT:    fcvt h7, s7
-; NONEON-NOSVE-NEXT:    fcvt s18, h18
-; NONEON-NOSVE-NEXT:    mov h22, v3.h[5]
-; NONEON-NOSVE-NEXT:    mov v5.h[2], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h16, s19
-; NONEON-NOSVE-NEXT:    mov h19, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmin s6, s17, s6
-; NONEON-NOSVE-NEXT:    mov h17, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fmin s18, s20, s18
-; NONEON-NOSVE-NEXT:    mov h20, v3.h[6]
-; NONEON-NOSVE-NEXT:    mov v4.h[3], v7.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s7, h22
-; NONEON-NOSVE-NEXT:    fcvt s22, h23
-; NONEON-NOSVE-NEXT:    mov v5.h[3], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt s16, h21
-; NONEON-NOSVE-NEXT:    fcvt s21, h24
-; NONEON-NOSVE-NEXT:    fcvt s19, h19
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fcvt s23, h25
-; NONEON-NOSVE-NEXT:    fcvt h18, s18
-; NONEON-NOSVE-NEXT:    fcvt s20, h20
-; NONEON-NOSVE-NEXT:    mov h3, v3.h[7]
-; NONEON-NOSVE-NEXT:    fmin s7, s22, s7
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fmin s16, s21, s16
-; NONEON-NOSVE-NEXT:    mov v4.h[4], v6.h[0]
-; NONEON-NOSVE-NEXT:    fmin s6, s19, s17
-; NONEON-NOSVE-NEXT:    mov v5.h[4], v18.h[0]
-; NONEON-NOSVE-NEXT:    fmin s17, s23, s20
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt h7, s7
-; NONEON-NOSVE-NEXT:    fmin s0, s0, s1
-; NONEON-NOSVE-NEXT:    fcvt h16, s16
-; NONEON-NOSVE-NEXT:    fcvt h6, s6
-; NONEON-NOSVE-NEXT:    fmin s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h3, s17
-; NONEON-NOSVE-NEXT:    mov v5.h[5], v7.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    mov v4.h[5], v16.h[0]
-; NONEON-NOSVE-NEXT:    fcvt h1, s2
-; NONEON-NOSVE-NEXT:    mov v5.h[6], v3.h[0]
-; NONEON-NOSVE-NEXT:    mov v4.h[6], v6.h[0]
-; NONEON-NOSVE-NEXT:    mov v5.h[7], v1.h[0]
-; NONEON-NOSVE-NEXT:    mov v4.h[7], v0.h[0]
-; NONEON-NOSVE-NEXT:    stp q5, q4, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %res = call <16 x half> @llvm.minimum.v16f16(<16 x half> %op1, <16 x half> %op2)
@@ -1384,11 +457,6 @@ define <2 x float> @fmin_v2f32(<2 x float> %op1, <2 x float> %op2) {
 ; CHECK-NEXT:    fmin z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmin_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmin v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.minimum.v2f32(<2 x float> %op1, <2 x float> %op2)
   ret <2 x float> %res
 }
@@ -1402,11 +470,6 @@ define <4 x float> @fmin_v4f32(<4 x float> %op1, <4 x float> %op2) {
 ; CHECK-NEXT:    fmin z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmin_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmin v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.minimum.v4f32(<4 x float> %op1, <4 x float> %op2)
   ret <4 x float> %res
 }
@@ -1422,15 +485,6 @@ define void @fmin_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmin z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmin_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fmin v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fmin v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %res = call <8 x float> @llvm.minimum.v8f32(<8 x float> %op1, <8 x float> %op2)
@@ -1443,11 +497,6 @@ define <1 x double> @fmin_v1f64(<1 x double> %op1, <1 x double> %op2) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmin d0, d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmin_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmin d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x double> @llvm.minimum.v1f64(<1 x double> %op1, <1 x double> %op2)
   ret <1 x double> %res
 }
@@ -1461,11 +510,6 @@ define <2 x double> @fmin_v2f64(<2 x double> %op1, <2 x double> %op2) {
 ; CHECK-NEXT:    fmin z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmin_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmin v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.minimum.v2f64(<2 x double> %op1, <2 x double> %op2)
   ret <2 x double> %res
 }
@@ -1481,15 +525,6 @@ define void @fmin_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmin z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmin_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fmin v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fmin v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %res = call <4 x double> @llvm.minimum.v4f64(<4 x double> %op1, <4 x double> %op2)
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce-fa64.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce-fa64.ll
index fdb81b8e5fe1..b56e67d95ba0 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce-fa64.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce-fa64.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sme-fa64 -force-streaming-compatible-sve < %s | FileCheck %s -check-prefix=FA64
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s -check-prefix=NO-FA64
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -27,30 +26,6 @@ define half @fadda_v4f16(half %start, <4 x half> %a) {
 ; NO-FA64-NEXT:    fadd h0, h0, h2
 ; NO-FA64-NEXT:    fadd h0, h0, h1
 ; NO-FA64-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadda_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call half @llvm.vector.reduce.fadd.v4f16(half %start, <4 x half> %a)
   ret half %res
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll
index 74a5db4b38e0..df9613a30e40 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -20,30 +19,6 @@ define half @fadda_v4f16(half %start, <4 x half> %a) {
 ; CHECK-NEXT:    fadd h0, h0, h2
 ; CHECK-NEXT:    fadd h0, h0, h1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadda_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call half @llvm.vector.reduce.fadd.v4f16(half %start, <4 x half> %a)
   ret half %res
 }
@@ -68,49 +43,6 @@ define half @fadda_v8f16(half %start, <8 x half> %a) {
 ; CHECK-NEXT:    fadd h0, h0, h2
 ; CHECK-NEXT:    fadd h0, h0, h1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadda_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call half @llvm.vector.reduce.fadd.v8f16(half %start, <8 x half> %a)
   ret half %res
 }
@@ -151,90 +83,6 @@ define half @fadda_v16f16(half %start, ptr %a) {
 ; CHECK-NEXT:    fadd h0, h0, h2
 ; CHECK-NEXT:    fadd h0, h0, h1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadda_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    fcvt s2, h1
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call half @llvm.vector.reduce.fadd.v16f16(half %start, <16 x half> %op)
   ret half %res
@@ -248,14 +96,6 @@ define float @fadda_v2f32(float %start, <2 x float> %a) {
 ; CHECK-NEXT:    mov z1.s, z1.s[1]
 ; CHECK-NEXT:    fadd s0, s0, s1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadda_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    mov s2, v1.s[1]
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    ret
   %res = call float @llvm.vector.reduce.fadd.v2f32(float %start, <2 x float> %a)
   ret float %res
 }
@@ -272,17 +112,6 @@ define float @fadda_v4f32(float %start, <4 x float> %a) {
 ; CHECK-NEXT:    fadd s0, s0, s2
 ; CHECK-NEXT:    fadd s0, s0, s1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadda_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov s2, v1.s[1]
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    mov s3, v1.s[2]
-; NONEON-NOSVE-NEXT:    mov s1, v1.s[3]
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s3
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    ret
   %res = call float @llvm.vector.reduce.fadd.v4f32(float %start, <4 x float> %a)
   ret float %res
 }
@@ -307,26 +136,6 @@ define float @fadda_v8f32(float %start, ptr %a) {
 ; CHECK-NEXT:    fadd s0, s0, s2
 ; CHECK-NEXT:    fadd s0, s0, s1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadda_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    mov s2, v1.s[1]
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    mov s3, v1.s[2]
-; NONEON-NOSVE-NEXT:    mov s1, v1.s[3]
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s3
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    mov s2, v1.s[1]
-; NONEON-NOSVE-NEXT:    mov s3, v1.s[2]
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    mov s1, v1.s[3]
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s2
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s3
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call float @llvm.vector.reduce.fadd.v8f32(float %start, <8 x float> %op)
   ret float %res
@@ -337,11 +146,6 @@ define double @fadda_v1f64(double %start, <1 x double> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fadd d0, d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadda_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fadd d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = call double @llvm.vector.reduce.fadd.v1f64(double %start, <1 x double> %a)
   ret double %res
 }
@@ -354,13 +158,6 @@ define double @fadda_v2f64(double %start, <2 x double> %a) {
 ; CHECK-NEXT:    mov z1.d, z1.d[1]
 ; CHECK-NEXT:    fadd d0, d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadda_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov d2, v1.d[1]
-; NONEON-NOSVE-NEXT:    fadd d0, d0, d1
-; NONEON-NOSVE-NEXT:    fadd d0, d0, d2
-; NONEON-NOSVE-NEXT:    ret
   %res = call double @llvm.vector.reduce.fadd.v2f64(double %start, <2 x double> %a)
   ret double %res
 }
@@ -377,17 +174,6 @@ define double @fadda_v4f64(double %start, ptr %a) {
 ; CHECK-NEXT:    mov z1.d, z1.d[1]
 ; CHECK-NEXT:    fadd d0, d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadda_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x0]
-; NONEON-NOSVE-NEXT:    mov d2, v3.d[1]
-; NONEON-NOSVE-NEXT:    fadd d0, d0, d3
-; NONEON-NOSVE-NEXT:    fadd d0, d0, d2
-; NONEON-NOSVE-NEXT:    mov d2, v1.d[1]
-; NONEON-NOSVE-NEXT:    fadd d0, d0, d1
-; NONEON-NOSVE-NEXT:    fadd d0, d0, d2
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call double @llvm.vector.reduce.fadd.v4f64(double %start, <4 x double> %op)
   ret double %res
@@ -405,30 +191,6 @@ define half @faddv_v4f16(half %start, <4 x half> %a) {
 ; CHECK-NEXT:    faddv h1, p0, z1.h
 ; CHECK-NEXT:    fadd h0, h0, h1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: faddv_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fadd s2, s3, s2
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fadd s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fadd s1, s2, s1
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call fast half @llvm.vector.reduce.fadd.v4f16(half %start, <4 x half> %a)
   ret half %res
 }
@@ -441,49 +203,6 @@ define half @faddv_v8f16(half %start, <8 x half> %a) {
 ; CHECK-NEXT:    faddv h1, p0, z1.h
 ; CHECK-NEXT:    fadd h0, h0, h1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: faddv_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h1
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fadd s2, s3, s2
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fadd s2, s2, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fadd s2, s2, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fadd s2, s2, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fadd s2, s2, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fadd s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fadd s1, s2, s1
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call fast half @llvm.vector.reduce.fadd.v8f16(half %start, <8 x half> %a)
   ret half %res
 }
@@ -497,58 +216,6 @@ define half @faddv_v16f16(half %start, ptr %a) {
 ; CHECK-NEXT:    faddv h1, p0, z1.h
 ; CHECK-NEXT:    fadd h0, h0, h1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: faddv_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl v4.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v2.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fadd v3.4s, v4.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fadd v1.4s, v1.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v2.4h, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.8h, v1.4s
-; NONEON-NOSVE-NEXT:    mov h1, v2.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s3, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fadd s1, s3, s1
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fadd s1, s1, s3
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fadd s1, s1, s3
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fadd s1, s1, s3
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fadd s1, s1, s3
-; NONEON-NOSVE-NEXT:    mov h3, v2.h[6]
-; NONEON-NOSVE-NEXT:    mov h2, v2.h[7]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fadd s1, s1, s3
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fadd s1, s1, s2
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call fast half @llvm.vector.reduce.fadd.v16f16(half %start, <16 x half> %op)
   ret half %res
@@ -562,12 +229,6 @@ define float @faddv_v2f32(float %start, <2 x float> %a) {
 ; CHECK-NEXT:    faddv s1, p0, z1.s
 ; CHECK-NEXT:    fadd s0, s0, s1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: faddv_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    faddp s1, v1.2s
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    ret
   %res = call fast float @llvm.vector.reduce.fadd.v2f32(float %start, <2 x float> %a)
   ret float %res
 }
@@ -580,13 +241,6 @@ define float @faddv_v4f32(float %start, <4 x float> %a) {
 ; CHECK-NEXT:    faddv s1, p0, z1.s
 ; CHECK-NEXT:    fadd s0, s0, s1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: faddv_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    faddp v1.4s, v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    faddp s1, v1.2s
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    ret
   %res = call fast float @llvm.vector.reduce.fadd.v4f32(float %start, <4 x float> %a)
   ret float %res
 }
@@ -600,15 +254,6 @@ define float @faddv_v8f32(float %start, ptr %a) {
 ; CHECK-NEXT:    faddv s1, p0, z1.s
 ; CHECK-NEXT:    fadd s0, s0, s1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: faddv_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
-; NONEON-NOSVE-NEXT:    fadd v1.4s, v2.4s, v1.4s
-; NONEON-NOSVE-NEXT:    faddp v1.4s, v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    faddp s1, v1.2s
-; NONEON-NOSVE-NEXT:    fadd s0, s0, s1
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call fast float @llvm.vector.reduce.fadd.v8f32(float %start, <8 x float> %op)
   ret float %res
@@ -619,11 +264,6 @@ define double @faddv_v1f64(double %start, <1 x double> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fadd d0, d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: faddv_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fadd d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = call fast double @llvm.vector.reduce.fadd.v1f64(double %start, <1 x double> %a)
   ret double %res
 }
@@ -636,12 +276,6 @@ define double @faddv_v2f64(double %start, <2 x double> %a) {
 ; CHECK-NEXT:    faddv d1, p0, z1.d
 ; CHECK-NEXT:    fadd d0, d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: faddv_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    faddp d1, v1.2d
-; NONEON-NOSVE-NEXT:    fadd d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = call fast double @llvm.vector.reduce.fadd.v2f64(double %start, <2 x double> %a)
   ret double %res
 }
@@ -655,14 +289,6 @@ define double @faddv_v4f64(double %start, ptr %a) {
 ; CHECK-NEXT:    faddv d1, p0, z1.d
 ; CHECK-NEXT:    fadd d0, d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: faddv_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
-; NONEON-NOSVE-NEXT:    fadd v1.2d, v2.2d, v1.2d
-; NONEON-NOSVE-NEXT:    faddp d1, v1.2d
-; NONEON-NOSVE-NEXT:    fadd d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call fast double @llvm.vector.reduce.fadd.v4f64(double %start, <4 x double> %op)
   ret double %res
@@ -680,26 +306,6 @@ define half @fmaxv_v4f16(<4 x half> %a) {
 ; CHECK-NEXT:    fmaxnmv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxv_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s1, s2, s1
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call half @llvm.vector.reduce.fmax.v4f16(<4 x half> %a)
   ret half %res
 }
@@ -712,45 +318,6 @@ define half @fmaxv_v8f16(<8 x half> %a) {
 ; CHECK-NEXT:    fmaxnmv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxv_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s1, s2, s1
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call half @llvm.vector.reduce.fmax.v8f16(<8 x half> %a)
   ret half %res
 }
@@ -764,85 +331,6 @@ define half @fmaxv_v16f16(ptr %a) {
 ; CHECK-NEXT:    fmaxnmv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxv_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fmaxnm s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[2]
-; NONEON-NOSVE-NEXT:    fmaxnm s2, s3, s2
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmaxnm s3, s5, s3
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[3]
-; NONEON-NOSVE-NEXT:    fmaxnm s2, s4, s2
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmaxnm s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[4]
-; NONEON-NOSVE-NEXT:    fmaxnm s2, s2, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmaxnm s3, s5, s3
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[5]
-; NONEON-NOSVE-NEXT:    fmaxnm s2, s2, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmaxnm s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fmaxnm s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h3, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmaxnm s0, s0, s1
-; NONEON-NOSVE-NEXT:    fmaxnm s2, s2, s3
-; NONEON-NOSVE-NEXT:    fmaxnm s3, s5, s4
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fmaxnm s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h1, s2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmaxnm s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call half @llvm.vector.reduce.fmax.v16f16(<16 x half> %op)
   ret half %res
@@ -856,11 +344,6 @@ define float @fmaxv_v2f32(<2 x float> %a) {
 ; CHECK-NEXT:    fmaxnmv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxv_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmaxnmp s0, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call float @llvm.vector.reduce.fmax.v2f32(<2 x float> %a)
   ret float %res
 }
@@ -873,11 +356,6 @@ define float @fmaxv_v4f32(<4 x float> %a) {
 ; CHECK-NEXT:    fmaxnmv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxv_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmaxnmv s0, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> %a)
   ret float %res
 }
@@ -891,13 +369,6 @@ define float @fmaxv_v8f32(ptr %a) {
 ; CHECK-NEXT:    fmaxnmv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxv_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    fmaxnm v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fmaxnmv s0, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> %op)
   ret float %res
@@ -907,10 +378,6 @@ define double @fmaxv_v1f64(<1 x double> %a) {
 ; CHECK-LABEL: fmaxv_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxv_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ret
   %res = call double @llvm.vector.reduce.fmax.v1f64(<1 x double> %a)
   ret double %res
 }
@@ -923,11 +390,6 @@ define double @fmaxv_v2f64(<2 x double> %a) {
 ; CHECK-NEXT:    fmaxnmv d0, p0, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxv_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmaxnmp d0, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call double @llvm.vector.reduce.fmax.v2f64(<2 x double> %a)
   ret double %res
 }
@@ -941,13 +403,6 @@ define double @fmaxv_v4f64(ptr %a) {
 ; CHECK-NEXT:    fmaxnmv d0, p0, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaxv_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    fmaxnm v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fmaxnmp d0, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call double @llvm.vector.reduce.fmax.v4f64(<4 x double> %op)
   ret double %res
@@ -965,26 +420,6 @@ define half @fminv_v4f16(<4 x half> %a) {
 ; CHECK-NEXT:    fminnmv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminv_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s1, s2, s1
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call half @llvm.vector.reduce.fmin.v4f16(<4 x half> %a)
   ret half %res
 }
@@ -997,45 +432,6 @@ define half @fminv_v8f16(<8 x half> %a) {
 ; CHECK-NEXT:    fminnmv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminv_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s1, s2, s1
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s1, s1, s2
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call half @llvm.vector.reduce.fmin.v8f16(<8 x half> %a)
   ret half %res
 }
@@ -1049,85 +445,6 @@ define half @fminv_v16f16(ptr %a) {
 ; CHECK-NEXT:    fminnmv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminv_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fminnm s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[2]
-; NONEON-NOSVE-NEXT:    fminnm s2, s3, s2
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fminnm s3, s5, s3
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[3]
-; NONEON-NOSVE-NEXT:    fminnm s2, s4, s2
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fminnm s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[4]
-; NONEON-NOSVE-NEXT:    fminnm s2, s2, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fminnm s3, s5, s3
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[5]
-; NONEON-NOSVE-NEXT:    fminnm s2, s2, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fminnm s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fminnm s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h3, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fminnm s0, s0, s1
-; NONEON-NOSVE-NEXT:    fminnm s2, s2, s3
-; NONEON-NOSVE-NEXT:    fminnm s3, s5, s4
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fminnm s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h1, s2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fminnm s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call half @llvm.vector.reduce.fmin.v16f16(<16 x half> %op)
   ret half %res
@@ -1141,11 +458,6 @@ define float @fminv_v2f32(<2 x float> %a) {
 ; CHECK-NEXT:    fminnmv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminv_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fminnmp s0, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call float @llvm.vector.reduce.fmin.v2f32(<2 x float> %a)
   ret float %res
 }
@@ -1158,11 +470,6 @@ define float @fminv_v4f32(<4 x float> %a) {
 ; CHECK-NEXT:    fminnmv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminv_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fminnmv s0, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> %a)
   ret float %res
 }
@@ -1176,13 +483,6 @@ define float @fminv_v8f32(ptr %a) {
 ; CHECK-NEXT:    fminnmv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminv_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    fminnm v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fminnmv s0, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call float @llvm.vector.reduce.fmin.v8f32(<8 x float> %op)
   ret float %res
@@ -1192,10 +492,6 @@ define double @fminv_v1f64(<1 x double> %a) {
 ; CHECK-LABEL: fminv_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminv_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ret
   %res = call double @llvm.vector.reduce.fmin.v1f64(<1 x double> %a)
   ret double %res
 }
@@ -1208,11 +504,6 @@ define double @fminv_v2f64(<2 x double> %a) {
 ; CHECK-NEXT:    fminnmv d0, p0, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminv_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fminnmp d0, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call double @llvm.vector.reduce.fmin.v2f64(<2 x double> %a)
   ret double %res
 }
@@ -1226,13 +517,6 @@ define double @fminv_v4f64(ptr %a) {
 ; CHECK-NEXT:    fminnmv d0, p0, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminv_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    fminnm v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fminnmp d0, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call double @llvm.vector.reduce.fmin.v4f64(<4 x double> %op)
   ret double %res
@@ -1250,26 +534,6 @@ define half @fmaximumv_v4f16(<4 x half> %a) {
 ; CHECK-NEXT:    fmaxv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaximumv_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s1, s2, s1
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s1, s1, s2
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call half @llvm.vector.reduce.fmaximum.v4f16(<4 x half> %a)
   ret half %res
 }
@@ -1282,45 +546,6 @@ define half @fmaximumv_v8f16(<8 x half> %a) {
 ; CHECK-NEXT:    fmaxv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaximumv_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s1, s2, s1
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s1, s1, s2
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call half @llvm.vector.reduce.fmaximum.v8f16(<8 x half> %a)
   ret half %res
 }
@@ -1334,85 +559,6 @@ define half @fmaximumv_v16f16(ptr %a) {
 ; CHECK-NEXT:    fmaxv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaximumv_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fmax s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[2]
-; NONEON-NOSVE-NEXT:    fmax s2, s3, s2
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmax s3, s5, s3
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[3]
-; NONEON-NOSVE-NEXT:    fmax s2, s4, s2
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmax s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[4]
-; NONEON-NOSVE-NEXT:    fmax s2, s2, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmax s3, s5, s3
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[5]
-; NONEON-NOSVE-NEXT:    fmax s2, s2, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmax s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fmax s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h3, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmax s0, s0, s1
-; NONEON-NOSVE-NEXT:    fmax s2, s2, s3
-; NONEON-NOSVE-NEXT:    fmax s3, s5, s4
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fmax s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h1, s2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmax s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call half @llvm.vector.reduce.fmaximum.v16f16(<16 x half> %op)
   ret half %res
@@ -1426,11 +572,6 @@ define float @fmaximumv_v2f32(<2 x float> %a) {
 ; CHECK-NEXT:    fmaxv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaximumv_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmaxp s0, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> %a)
   ret float %res
 }
@@ -1443,11 +584,6 @@ define float @fmaximumv_v4f32(<4 x float> %a) {
 ; CHECK-NEXT:    fmaxv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaximumv_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmaxv s0, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> %a)
   ret float %res
 }
@@ -1461,13 +597,6 @@ define float @fmaximumv_v8f32(ptr %a) {
 ; CHECK-NEXT:    fmaxv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaximumv_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    fmax v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fmaxv s0, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> %op)
   ret float %res
@@ -1477,10 +606,6 @@ define double @fmaximumv_v1f64(<1 x double> %a) {
 ; CHECK-LABEL: fmaximumv_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaximumv_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ret
   %res = call double @llvm.vector.reduce.fmaximum.v1f64(<1 x double> %a)
   ret double %res
 }
@@ -1493,11 +618,6 @@ define double @fmaximumv_v2f64(<2 x double> %a) {
 ; CHECK-NEXT:    fmaxv d0, p0, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaximumv_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmaxp d0, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> %a)
   ret double %res
 }
@@ -1511,13 +631,6 @@ define double @fmaximumv_v4f64(ptr %a) {
 ; CHECK-NEXT:    fmaxv d0, p0, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fmaximumv_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    fmax v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fmaxp d0, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> %op)
   ret double %res
@@ -1535,26 +648,6 @@ define half @fminimumv_v4f16(<4 x half> %a) {
 ; CHECK-NEXT:    fminv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminimumv_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s1, s2, s1
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s1, s1, s2
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call half @llvm.vector.reduce.fminimum.v4f16(<4 x half> %a)
   ret half %res
 }
@@ -1567,45 +660,6 @@ define half @fminimumv_v8f16(<8 x half> %a) {
 ; CHECK-NEXT:    fminv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminimumv_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s1, s2, s1
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s1, s1, s2
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s1, s1, s2
-; NONEON-NOSVE-NEXT:    fcvt h1, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call half @llvm.vector.reduce.fminimum.v8f16(<8 x half> %a)
   ret half %res
 }
@@ -1619,85 +673,6 @@ define half @fminimumv_v16f16(ptr %a) {
 ; CHECK-NEXT:    fminv h0, p0, z0.h
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminimumv_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s4, h1
-; NONEON-NOSVE-NEXT:    fcvt s5, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fmin s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[2]
-; NONEON-NOSVE-NEXT:    fmin s2, s3, s2
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmin s3, s5, s3
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[3]
-; NONEON-NOSVE-NEXT:    fmin s2, s4, s2
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[3]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmin s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[4]
-; NONEON-NOSVE-NEXT:    fmin s2, s2, s3
-; NONEON-NOSVE-NEXT:    mov h3, v1.h[4]
-; NONEON-NOSVE-NEXT:    fcvt h4, s4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmin s3, s5, s3
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[5]
-; NONEON-NOSVE-NEXT:    fmin s2, s2, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[5]
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmin s4, s5, s4
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[6]
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[7]
-; NONEON-NOSVE-NEXT:    fmin s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h3, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[7]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fmin s0, s0, s1
-; NONEON-NOSVE-NEXT:    fmin s2, s2, s3
-; NONEON-NOSVE-NEXT:    fmin s3, s5, s4
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    fcvt h2, s2
-; NONEON-NOSVE-NEXT:    fcvt h3, s3
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fmin s2, s2, s3
-; NONEON-NOSVE-NEXT:    fcvt h1, s2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fmin s0, s1, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call half @llvm.vector.reduce.fminimum.v16f16(<16 x half> %op)
   ret half %res
@@ -1711,11 +686,6 @@ define float @fminimumv_v2f32(<2 x float> %a) {
 ; CHECK-NEXT:    fminv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminimumv_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fminp s0, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> %a)
   ret float %res
 }
@@ -1728,11 +698,6 @@ define float @fminimumv_v4f32(<4 x float> %a) {
 ; CHECK-NEXT:    fminv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminimumv_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fminv s0, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> %a)
   ret float %res
 }
@@ -1746,13 +711,6 @@ define float @fminimumv_v8f32(ptr %a) {
 ; CHECK-NEXT:    fminv s0, p0, z0.s
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminimumv_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    fmin v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fminv s0, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> %op)
   ret float %res
@@ -1762,10 +720,6 @@ define double @fminimumv_v1f64(<1 x double> %a) {
 ; CHECK-LABEL: fminimumv_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminimumv_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ret
   %res = call double @llvm.vector.reduce.fminimum.v1f64(<1 x double> %a)
   ret double %res
 }
@@ -1778,11 +732,6 @@ define double @fminimumv_v2f64(<2 x double> %a) {
 ; CHECK-NEXT:    fminv d0, p0, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminimumv_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fminp d0, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> %a)
   ret double %res
 }
@@ -1796,13 +745,6 @@ define double @fminimumv_v4f64(ptr %a) {
 ; CHECK-NEXT:    fminv d0, p0, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fminimumv_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    fmin v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fminp d0, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> %op)
   ret double %res
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll
index 454683865eb9..7ddc641f366c 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -17,13 +16,6 @@ define <2 x half> @frintp_v2f16(<2 x half> %op) {
 ; CHECK-NEXT:    frintp z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintp_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frintp v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x half> @llvm.ceil.v2f16(<2 x half> %op)
   ret <2 x half> %res
 }
@@ -36,13 +28,6 @@ define <4 x half> @frintp_v4f16(<4 x half> %op) {
 ; CHECK-NEXT:    frintp z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintp_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frintp v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.ceil.v4f16(<4 x half> %op)
   ret <4 x half> %res
 }
@@ -55,16 +40,6 @@ define <8 x half> @frintp_v8f16(<8 x half> %op) {
 ; CHECK-NEXT:    frintp z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintp_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v0.8h
-; NONEON-NOSVE-NEXT:    frintp v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v1.4s
-; NONEON-NOSVE-NEXT:    frintp v1.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.ceil.v8f16(<8 x half> %op)
   ret <8 x half> %res
 }
@@ -78,24 +53,6 @@ define void @frintp_v16f16(ptr %a) {
 ; CHECK-NEXT:    frintp z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintp_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    frintp v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    frintp v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    frintp v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frintp v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v2.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v1.4s
-; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call <16 x half> @llvm.ceil.v16f16(<16 x half> %op)
   store <16 x half> %res, ptr %a
@@ -110,11 +67,6 @@ define <2 x float> @frintp_v2f32(<2 x float> %op) {
 ; CHECK-NEXT:    frintp z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintp_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintp v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.ceil.v2f32(<2 x float> %op)
   ret <2 x float> %res
 }
@@ -127,11 +79,6 @@ define <4 x float> @frintp_v4f32(<4 x float> %op) {
 ; CHECK-NEXT:    frintp z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintp_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintp v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.ceil.v4f32(<4 x float> %op)
   ret <4 x float> %res
 }
@@ -145,14 +92,6 @@ define void @frintp_v8f32(ptr %a) {
 ; CHECK-NEXT:    frintp z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintp_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frintp v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frintp v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call <8 x float> @llvm.ceil.v8f32(<8 x float> %op)
   store <8 x float> %res, ptr %a
@@ -164,11 +103,6 @@ define <1 x double> @frintp_v1f64(<1 x double> %op) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintp d0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintp_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintp d0, d0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x double> @llvm.ceil.v1f64(<1 x double> %op)
   ret <1 x double> %res
 }
@@ -181,11 +115,6 @@ define <2 x double> @frintp_v2f64(<2 x double> %op) {
 ; CHECK-NEXT:    frintp z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintp_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintp v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.ceil.v2f64(<2 x double> %op)
   ret <2 x double> %res
 }
@@ -199,14 +128,6 @@ define void @frintp_v4f64(ptr %a) {
 ; CHECK-NEXT:    frintp z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintp_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frintp v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    frintp v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call <4 x double> @llvm.ceil.v4f64(<4 x double> %op)
   store <4 x double> %res, ptr %a
@@ -225,13 +146,6 @@ define <2 x half> @frintm_v2f16(<2 x half> %op) {
 ; CHECK-NEXT:    frintm z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintm_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frintm v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x half> @llvm.floor.v2f16(<2 x half> %op)
   ret <2 x half> %res
 }
@@ -244,13 +158,6 @@ define <4 x half> @frintm_v4f16(<4 x half> %op) {
 ; CHECK-NEXT:    frintm z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintm_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frintm v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.floor.v4f16(<4 x half> %op)
   ret <4 x half> %res
 }
@@ -263,16 +170,6 @@ define <8 x half> @frintm_v8f16(<8 x half> %op) {
 ; CHECK-NEXT:    frintm z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintm_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v0.8h
-; NONEON-NOSVE-NEXT:    frintm v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v1.4s
-; NONEON-NOSVE-NEXT:    frintm v1.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.floor.v8f16(<8 x half> %op)
   ret <8 x half> %res
 }
@@ -286,24 +183,6 @@ define void @frintm_v16f16(ptr %a) {
 ; CHECK-NEXT:    frintm z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintm_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    frintm v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    frintm v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    frintm v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frintm v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v2.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v1.4s
-; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call <16 x half> @llvm.floor.v16f16(<16 x half> %op)
   store <16 x half> %res, ptr %a
@@ -318,11 +197,6 @@ define <2 x float> @frintm_v2f32(<2 x float> %op) {
 ; CHECK-NEXT:    frintm z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintm_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintm v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.floor.v2f32(<2 x float> %op)
   ret <2 x float> %res
 }
@@ -335,11 +209,6 @@ define <4 x float> @frintm_v4f32(<4 x float> %op) {
 ; CHECK-NEXT:    frintm z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintm_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintm v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.floor.v4f32(<4 x float> %op)
   ret <4 x float> %res
 }
@@ -353,14 +222,6 @@ define void @frintm_v8f32(ptr %a) {
 ; CHECK-NEXT:    frintm z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintm_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frintm v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frintm v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call <8 x float> @llvm.floor.v8f32(<8 x float> %op)
   store <8 x float> %res, ptr %a
@@ -372,11 +233,6 @@ define <1 x double> @frintm_v1f64(<1 x double> %op) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintm d0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintm_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintm d0, d0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x double> @llvm.floor.v1f64(<1 x double> %op)
   ret <1 x double> %res
 }
@@ -389,11 +245,6 @@ define <2 x double> @frintm_v2f64(<2 x double> %op) {
 ; CHECK-NEXT:    frintm z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintm_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintm v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.floor.v2f64(<2 x double> %op)
   ret <2 x double> %res
 }
@@ -407,14 +258,6 @@ define void @frintm_v4f64(ptr %a) {
 ; CHECK-NEXT:    frintm z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintm_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frintm v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    frintm v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call <4 x double> @llvm.floor.v4f64(<4 x double> %op)
   store <4 x double> %res, ptr %a
@@ -433,13 +276,6 @@ define <2 x half> @frinti_v2f16(<2 x half> %op) {
 ; CHECK-NEXT:    frinti z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinti_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frinti v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x half> @llvm.nearbyint.v2f16(<2 x half> %op)
   ret <2 x half> %res
 }
@@ -452,13 +288,6 @@ define <4 x half> @frinti_v4f16(<4 x half> %op) {
 ; CHECK-NEXT:    frinti z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinti_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frinti v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.nearbyint.v4f16(<4 x half> %op)
   ret <4 x half> %res
 }
@@ -471,16 +300,6 @@ define <8 x half> @frinti_v8f16(<8 x half> %op) {
 ; CHECK-NEXT:    frinti z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinti_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v0.8h
-; NONEON-NOSVE-NEXT:    frinti v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v1.4s
-; NONEON-NOSVE-NEXT:    frinti v1.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.nearbyint.v8f16(<8 x half> %op)
   ret <8 x half> %res
 }
@@ -494,24 +313,6 @@ define void @frinti_v16f16(ptr %a) {
 ; CHECK-NEXT:    frinti z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinti_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    frinti v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    frinti v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    frinti v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frinti v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v2.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v1.4s
-; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call <16 x half> @llvm.nearbyint.v16f16(<16 x half> %op)
   store <16 x half> %res, ptr %a
@@ -526,11 +327,6 @@ define <2 x float> @frinti_v2f32(<2 x float> %op) {
 ; CHECK-NEXT:    frinti z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinti_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frinti v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.nearbyint.v2f32(<2 x float> %op)
   ret <2 x float> %res
 }
@@ -543,11 +339,6 @@ define <4 x float> @frinti_v4f32(<4 x float> %op) {
 ; CHECK-NEXT:    frinti z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinti_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frinti v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> %op)
   ret <4 x float> %res
 }
@@ -561,14 +352,6 @@ define void @frinti_v8f32(ptr %a) {
 ; CHECK-NEXT:    frinti z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinti_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frinti v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frinti v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call <8 x float> @llvm.nearbyint.v8f32(<8 x float> %op)
   store <8 x float> %res, ptr %a
@@ -580,11 +363,6 @@ define <1 x double> @frinti_v1f64(<1 x double> %op) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frinti d0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinti_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frinti d0, d0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x double> @llvm.nearbyint.v1f64(<1 x double> %op)
   ret <1 x double> %res
 }
@@ -597,11 +375,6 @@ define <2 x double> @frinti_v2f64(<2 x double> %op) {
 ; CHECK-NEXT:    frinti z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinti_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frinti v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> %op)
   ret <2 x double> %res
 }
@@ -615,14 +388,6 @@ define void @frinti_v4f64(ptr %a) {
 ; CHECK-NEXT:    frinti z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinti_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frinti v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    frinti v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call <4 x double> @llvm.nearbyint.v4f64(<4 x double> %op)
   store <4 x double> %res, ptr %a
@@ -641,13 +406,6 @@ define <2 x half> @frintx_v2f16(<2 x half> %op) {
 ; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintx_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frintx v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x half> @llvm.rint.v2f16(<2 x half> %op)
   ret <2 x half> %res
 }
@@ -660,13 +418,6 @@ define <4 x half> @frintx_v4f16(<4 x half> %op) {
 ; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintx_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frintx v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.rint.v4f16(<4 x half> %op)
   ret <4 x half> %res
 }
@@ -679,16 +430,6 @@ define <8 x half> @frintx_v8f16(<8 x half> %op) {
 ; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintx_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v0.8h
-; NONEON-NOSVE-NEXT:    frintx v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v1.4s
-; NONEON-NOSVE-NEXT:    frintx v1.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.rint.v8f16(<8 x half> %op)
   ret <8 x half> %res
 }
@@ -702,24 +443,6 @@ define void @frintx_v16f16(ptr %a) {
 ; CHECK-NEXT:    frintx z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintx_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    frintx v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    frintx v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    frintx v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frintx v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v2.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v1.4s
-; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call <16 x half> @llvm.rint.v16f16(<16 x half> %op)
   store <16 x half> %res, ptr %a
@@ -734,11 +457,6 @@ define <2 x float> @frintx_v2f32(<2 x float> %op) {
 ; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintx_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintx v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.rint.v2f32(<2 x float> %op)
   ret <2 x float> %res
 }
@@ -751,11 +469,6 @@ define <4 x float> @frintx_v4f32(<4 x float> %op) {
 ; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintx_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintx v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.rint.v4f32(<4 x float> %op)
   ret <4 x float> %res
 }
@@ -769,14 +482,6 @@ define void @frintx_v8f32(ptr %a) {
 ; CHECK-NEXT:    frintx z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintx_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frintx v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frintx v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call <8 x float> @llvm.rint.v8f32(<8 x float> %op)
   store <8 x float> %res, ptr %a
@@ -788,11 +493,6 @@ define <1 x double> @frintx_v1f64(<1 x double> %op) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx d0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintx_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintx d0, d0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x double> @llvm.rint.v1f64(<1 x double> %op)
   ret <1 x double> %res
 }
@@ -805,11 +505,6 @@ define <2 x double> @frintx_v2f64(<2 x double> %op) {
 ; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintx_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintx v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.rint.v2f64(<2 x double> %op)
   ret <2 x double> %res
 }
@@ -823,14 +518,6 @@ define void @frintx_v4f64(ptr %a) {
 ; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintx_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frintx v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    frintx v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call <4 x double> @llvm.rint.v4f64(<4 x double> %op)
   store <4 x double> %res, ptr %a
@@ -849,13 +536,6 @@ define <2 x half> @frinta_v2f16(<2 x half> %op) {
 ; CHECK-NEXT:    frinta z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinta_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frinta v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x half> @llvm.round.v2f16(<2 x half> %op)
   ret <2 x half> %res
 }
@@ -868,13 +548,6 @@ define <4 x half> @frinta_v4f16(<4 x half> %op) {
 ; CHECK-NEXT:    frinta z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinta_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frinta v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.round.v4f16(<4 x half> %op)
   ret <4 x half> %res
 }
@@ -887,16 +560,6 @@ define <8 x half> @frinta_v8f16(<8 x half> %op) {
 ; CHECK-NEXT:    frinta z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinta_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v0.8h
-; NONEON-NOSVE-NEXT:    frinta v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v1.4s
-; NONEON-NOSVE-NEXT:    frinta v1.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.round.v8f16(<8 x half> %op)
   ret <8 x half> %res
 }
@@ -910,24 +573,6 @@ define void @frinta_v16f16(ptr %a) {
 ; CHECK-NEXT:    frinta z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinta_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    frinta v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    frinta v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    frinta v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frinta v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v2.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v1.4s
-; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call <16 x half> @llvm.round.v16f16(<16 x half> %op)
   store <16 x half> %res, ptr %a
@@ -942,11 +587,6 @@ define <2 x float> @frinta_v2f32(<2 x float> %op) {
 ; CHECK-NEXT:    frinta z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinta_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frinta v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.round.v2f32(<2 x float> %op)
   ret <2 x float> %res
 }
@@ -959,11 +599,6 @@ define <4 x float> @frinta_v4f32(<4 x float> %op) {
 ; CHECK-NEXT:    frinta z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinta_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frinta v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.round.v4f32(<4 x float> %op)
   ret <4 x float> %res
 }
@@ -977,14 +612,6 @@ define void @frinta_v8f32(ptr %a) {
 ; CHECK-NEXT:    frinta z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinta_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frinta v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frinta v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call <8 x float> @llvm.round.v8f32(<8 x float> %op)
   store <8 x float> %res, ptr %a
@@ -996,11 +623,6 @@ define <1 x double> @frinta_v1f64(<1 x double> %op) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frinta d0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinta_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frinta d0, d0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x double> @llvm.round.v1f64(<1 x double> %op)
   ret <1 x double> %res
 }
@@ -1013,11 +635,6 @@ define <2 x double> @frinta_v2f64(<2 x double> %op) {
 ; CHECK-NEXT:    frinta z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinta_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frinta v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.round.v2f64(<2 x double> %op)
   ret <2 x double> %res
 }
@@ -1031,14 +648,6 @@ define void @frinta_v4f64(ptr %a) {
 ; CHECK-NEXT:    frinta z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frinta_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frinta v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    frinta v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call <4 x double> @llvm.round.v4f64(<4 x double> %op)
   store <4 x double> %res, ptr %a
@@ -1057,13 +666,6 @@ define <2 x half> @frintn_v2f16(<2 x half> %op) {
 ; CHECK-NEXT:    frintn z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintn_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frintn v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x half> @llvm.roundeven.v2f16(<2 x half> %op)
   ret <2 x half> %res
 }
@@ -1076,13 +678,6 @@ define <4 x half> @frintn_v4f16(<4 x half> %op) {
 ; CHECK-NEXT:    frintn z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintn_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frintn v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.roundeven.v4f16(<4 x half> %op)
   ret <4 x half> %res
 }
@@ -1095,16 +690,6 @@ define <8 x half> @frintn_v8f16(<8 x half> %op) {
 ; CHECK-NEXT:    frintn z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintn_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v0.8h
-; NONEON-NOSVE-NEXT:    frintn v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v1.4s
-; NONEON-NOSVE-NEXT:    frintn v1.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.roundeven.v8f16(<8 x half> %op)
   ret <8 x half> %res
 }
@@ -1118,24 +703,6 @@ define void @frintn_v16f16(ptr %a) {
 ; CHECK-NEXT:    frintn z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintn_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    frintn v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    frintn v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    frintn v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frintn v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v2.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v1.4s
-; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call <16 x half> @llvm.roundeven.v16f16(<16 x half> %op)
   store <16 x half> %res, ptr %a
@@ -1150,11 +717,6 @@ define <2 x float> @frintn_v2f32(<2 x float> %op) {
 ; CHECK-NEXT:    frintn z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintn_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintn v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.roundeven.v2f32(<2 x float> %op)
   ret <2 x float> %res
 }
@@ -1167,11 +729,6 @@ define <4 x float> @frintn_v4f32(<4 x float> %op) {
 ; CHECK-NEXT:    frintn z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintn_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintn v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %op)
   ret <4 x float> %res
 }
@@ -1185,14 +742,6 @@ define void @frintn_v8f32(ptr %a) {
 ; CHECK-NEXT:    frintn z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintn_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frintn v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frintn v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call <8 x float> @llvm.roundeven.v8f32(<8 x float> %op)
   store <8 x float> %res, ptr %a
@@ -1204,11 +753,6 @@ define <1 x double> @frintn_v1f64(<1 x double> %op) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintn d0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintn_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintn d0, d0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x double> @llvm.roundeven.v1f64(<1 x double> %op)
   ret <1 x double> %res
 }
@@ -1221,11 +765,6 @@ define <2 x double> @frintn_v2f64(<2 x double> %op) {
 ; CHECK-NEXT:    frintn z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintn_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintn v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.roundeven.v2f64(<2 x double> %op)
   ret <2 x double> %res
 }
@@ -1239,14 +778,6 @@ define void @frintn_v4f64(ptr %a) {
 ; CHECK-NEXT:    frintn z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintn_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frintn v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    frintn v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call <4 x double> @llvm.roundeven.v4f64(<4 x double> %op)
   store <4 x double> %res, ptr %a
@@ -1265,13 +796,6 @@ define <2 x half> @frintz_v2f16(<2 x half> %op) {
 ; CHECK-NEXT:    frintz z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintz_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frintz v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x half> @llvm.trunc.v2f16(<2 x half> %op)
   ret <2 x half> %res
 }
@@ -1284,13 +808,6 @@ define <4 x half> @frintz_v4f16(<4 x half> %op) {
 ; CHECK-NEXT:    frintz z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintz_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    frintz v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x half> @llvm.trunc.v4f16(<4 x half> %op)
   ret <4 x half> %res
 }
@@ -1303,16 +820,6 @@ define <8 x half> @frintz_v8f16(<8 x half> %op) {
 ; CHECK-NEXT:    frintz z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintz_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v0.8h
-; NONEON-NOSVE-NEXT:    frintz v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v1.4s
-; NONEON-NOSVE-NEXT:    frintz v1.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x half> @llvm.trunc.v8f16(<8 x half> %op)
   ret <8 x half> %res
 }
@@ -1326,24 +833,6 @@ define void @frintz_v16f16(ptr %a) {
 ; CHECK-NEXT:    frintz z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintz_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    frintz v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    frintz v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    frintz v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frintz v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v2.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v1.4s
-; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x half>, ptr %a
   %res = call <16 x half> @llvm.trunc.v16f16(<16 x half> %op)
   store <16 x half> %res, ptr %a
@@ -1358,11 +847,6 @@ define <2 x float> @frintz_v2f32(<2 x float> %op) {
 ; CHECK-NEXT:    frintz z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintz_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintz v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x float> @llvm.trunc.v2f32(<2 x float> %op)
   ret <2 x float> %res
 }
@@ -1375,11 +859,6 @@ define <4 x float> @frintz_v4f32(<4 x float> %op) {
 ; CHECK-NEXT:    frintz z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintz_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintz v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x float> @llvm.trunc.v4f32(<4 x float> %op)
   ret <4 x float> %res
 }
@@ -1393,14 +872,6 @@ define void @frintz_v8f32(ptr %a) {
 ; CHECK-NEXT:    frintz z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintz_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frintz v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    frintz v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x float>, ptr %a
   %res = call <8 x float> @llvm.trunc.v8f32(<8 x float> %op)
   store <8 x float> %res, ptr %a
@@ -1412,11 +883,6 @@ define <1 x double> @frintz_v1f64(<1 x double> %op) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintz d0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintz_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintz d0, d0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x double> @llvm.trunc.v1f64(<1 x double> %op)
   ret <1 x double> %res
 }
@@ -1429,11 +895,6 @@ define <2 x double> @frintz_v2f64(<2 x double> %op) {
 ; CHECK-NEXT:    frintz z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintz_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    frintz v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x double> @llvm.trunc.v2f64(<2 x double> %op)
   ret <2 x double> %res
 }
@@ -1447,14 +908,6 @@ define void @frintz_v4f64(ptr %a) {
 ; CHECK-NEXT:    frintz z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: frintz_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    frintz v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    frintz v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x double>, ptr %a
   %res = call <4 x double> @llvm.trunc.v4f64(<4 x double> %op)
   store <4 x double> %res, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll
index 0268dd1b5d31..7d36925fdc57 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -17,14 +16,6 @@ define <2 x half> @select_v2f16(<2 x half> %op1, <2 x half> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.4h, w8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <2 x half> %op1, <2 x half> %op2
   ret <2 x half> %sel
 }
@@ -41,14 +32,6 @@ define <4 x half> @select_v4f16(<4 x half> %op1, <4 x half> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.4h, w8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <4 x half> %op1, <4 x half> %op2
   ret <4 x half> %sel
 }
@@ -65,14 +48,6 @@ define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.8h, w8
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <8 x half> %op1, <8 x half> %op2
   ret <8 x half> %sel
 }
@@ -92,20 +67,6 @@ define void @select_v16f16(ptr %a, ptr %b, i1 %mask) {
 ; CHECK-NEXT:    sel z1.h, p0, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w2, #0x1
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    ldr q4, [x1, #16]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    bif v1.16b, v3.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load volatile <16 x half>, ptr %a
   %op2 = load volatile <16 x half>, ptr %b
   %sel = select i1 %mask, <16 x half> %op1, <16 x half> %op2
@@ -125,14 +86,6 @@ define <2 x float> @select_v2f32(<2 x float> %op1, <2 x float> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.2s, w8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <2 x float> %op1, <2 x float> %op2
   ret <2 x float> %sel
 }
@@ -149,14 +102,6 @@ define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.4s, w8
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <4 x float> %op1, <4 x float> %op2
   ret <4 x float> %sel
 }
@@ -176,20 +121,6 @@ define void @select_v8f32(ptr %a, ptr %b, i1 %mask) {
 ; CHECK-NEXT:    sel z1.s, p0, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w2, #0x1
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    ldr q4, [x1, #16]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    bif v1.16b, v3.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load volatile <8 x float>, ptr %a
   %op2 = load volatile <8 x float>, ptr %b
   %sel = select i1 %mask, <8 x float> %op1, <8 x float> %op2
@@ -203,14 +134,6 @@ define <1 x double> @select_v1f64(<1 x double> %op1, <1 x double> %op2, i1 %mask
 ; CHECK-NEXT:    tst w0, #0x1
 ; CHECK-NEXT:    fcsel d0, d0, d1, ne
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm x8, ne
-; NONEON-NOSVE-NEXT:    fmov d2, x8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <1 x double> %op1, <1 x double> %op2
   ret <1 x double> %sel
 }
@@ -228,14 +151,6 @@ define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, i1 %mask
 ; CHECK-NEXT:    sel z0.d, p0, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm x8, ne
-; NONEON-NOSVE-NEXT:    dup v2.2d, x8
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <2 x double> %op1, <2 x double> %op2
   ret <2 x double> %sel
 }
@@ -256,20 +171,6 @@ define void @select_v4f64(ptr %a, ptr %b, i1 %mask) {
 ; CHECK-NEXT:    sel z1.d, p0, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w2, #0x1
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    csetm x8, ne
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    ldr q4, [x1, #16]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    bif v1.16b, v3.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load volatile <4 x double>, ptr %a
   %op2 = load volatile <4 x double>, ptr %b
   %sel = select i1 %mask, <4 x double> %op1, <4 x double> %op2
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
index 1c63a3870d68..bf8a335a8503 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -16,13 +15,6 @@ define <4 x i16> @fcvtzu_v4f16_v4i16(<4 x half> %op1) {
 ; CHECK-NEXT:    fcvtzu z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v4f16_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <4 x half> %op1 to <4 x i16>
   ret <4 x i16> %res
 }
@@ -35,21 +27,6 @@ define void @fcvtzu_v8f16_v8i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzu z0.h, p0/m, z0.h
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v8f16_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x half>, ptr %a
   %res = fptoui <8 x half> %op1 to <8 x i16>
   store <8 x i16> %res, ptr %b
@@ -65,27 +42,6 @@ define void @fcvtzu_v16f16_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzu z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v16f16_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v3.4h
-; NONEON-NOSVE-NEXT:    fcvtzu v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %res = fptoui <16 x half> %op1 to <16 x i16>
   store <16 x i16> %res, ptr %b
@@ -105,13 +61,6 @@ define <2 x i32> @fcvtzu_v2f16_v2i32(<2 x half> %op1) {
 ; CHECK-NEXT:    fcvtzu z0.s, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v2f16_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <2 x half> %op1 to <2 x i32>
   ret <2 x i32> %res
 }
@@ -125,12 +74,6 @@ define <4 x i32> @fcvtzu_v4f16_v4i32(<4 x half> %op1) {
 ; CHECK-NEXT:    fcvtzu z0.s, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v4f16_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <4 x half> %op1 to <4 x i32>
   ret <4 x i32> %res
 }
@@ -147,20 +90,6 @@ define void @fcvtzu_v8f16_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzu z0.s, p0/m, z0.h
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v8f16_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x half>, ptr %a
   %res = fptoui <8 x half> %op1 to <8 x i32>
   store <8 x i32> %res, ptr %b
@@ -185,26 +114,6 @@ define void @fcvtzu_v16f16_v16i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v16f16_v16i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v3.4h
-; NONEON-NOSVE-NEXT:    fcvtzu v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %res = fptoui <16 x half> %op1 to <16 x i32>
   store <16 x i32> %res, ptr %b
@@ -221,13 +130,6 @@ define <1 x i64> @fcvtzu_v1f16_v1i64(<1 x half> %op1) {
 ; CHECK-NEXT:    fcvtzu x8, h0
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v1f16_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvtzu x8, s0
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <1 x half> %op1 to <1 x i64>
   ret <1 x i64> %res
 }
@@ -243,18 +145,6 @@ define <2 x i64> @fcvtzu_v2f16_v2i64(<2 x half> %op1) {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    ldr q0, [sp], #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v2f16_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvtzu x8, s0
-; NONEON-NOSVE-NEXT:    fcvtzu x9, s1
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x9
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <2 x half> %op1 to <2 x i64>
   ret <2 x i64> %res
 }
@@ -277,27 +167,6 @@ define void @fcvtzu_v4f16_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v4f16_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvtzu x9, s0
-; NONEON-NOSVE-NEXT:    fcvtzu x8, s1
-; NONEON-NOSVE-NEXT:    fcvtzu x10, s2
-; NONEON-NOSVE-NEXT:    fcvtzu x11, s3
-; NONEON-NOSVE-NEXT:    fmov d1, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x11
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x10
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x half>, ptr %a
   %res = fptoui <4 x half> %op1 to <4 x i64>
   store <4 x i64> %res, ptr %b
@@ -335,47 +204,6 @@ define void @fcvtzu_v8f16_v8i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q1, q0, [x1, #32]
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v8f16_v8i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov h5, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h7, v2.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvtzu x9, s0
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvtzu x13, s2
-; NONEON-NOSVE-NEXT:    fcvtzu x8, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h7
-; NONEON-NOSVE-NEXT:    fcvtzu x10, s3
-; NONEON-NOSVE-NEXT:    fcvtzu x11, s4
-; NONEON-NOSVE-NEXT:    fcvtzu x12, s5
-; NONEON-NOSVE-NEXT:    fcvtzu x14, s6
-; NONEON-NOSVE-NEXT:    fmov d3, x13
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    fcvtzu x8, s1
-; NONEON-NOSVE-NEXT:    fmov d1, x9
-; NONEON-NOSVE-NEXT:    fmov d2, x12
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x10
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x11
-; NONEON-NOSVE-NEXT:    mov v3.d[1], x8
-; NONEON-NOSVE-NEXT:    mov v2.d[1], x14
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x half>, ptr %a
   %res = fptoui <8 x half> %op1 to <8 x i64>
   store <8 x i64> %res, ptr %b
@@ -436,80 +264,6 @@ define void @fcvtzu_v16f16_v16i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q5, q2, [x1, #96]
 ; CHECK-NEXT:    add sp, sp, #128
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v16f16_v16i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s3, h1
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #24]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s6, h0
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s17, h4
-; NONEON-NOSVE-NEXT:    mov h18, v4.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvtzu x8, s3
-; NONEON-NOSVE-NEXT:    fcvt s3, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h7
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    mov h16, v4.h[3]
-; NONEON-NOSVE-NEXT:    fcvtzu x9, s6
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    mov h4, v4.h[1]
-; NONEON-NOSVE-NEXT:    fcvtzu x11, s2
-; NONEON-NOSVE-NEXT:    mov h2, v6.h[2]
-; NONEON-NOSVE-NEXT:    fcvtzu x10, s17
-; NONEON-NOSVE-NEXT:    fcvtzu x13, s5
-; NONEON-NOSVE-NEXT:    fcvtzu x12, s3
-; NONEON-NOSVE-NEXT:    mov h3, v6.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    mov h5, v6.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s17, h18
-; NONEON-NOSVE-NEXT:    fcvtzu x14, s7
-; NONEON-NOSVE-NEXT:    fmov d7, x8
-; NONEON-NOSVE-NEXT:    fcvtzu x8, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fmov d0, x11
-; NONEON-NOSVE-NEXT:    fcvtzu x11, s1
-; NONEON-NOSVE-NEXT:    fmov d1, x13
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvtzu x13, s16
-; NONEON-NOSVE-NEXT:    fmov d16, x9
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvtzu x15, s17
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x12
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x14
-; NONEON-NOSVE-NEXT:    fcvtzu x9, s2
-; NONEON-NOSVE-NEXT:    mov v16.d[1], x8
-; NONEON-NOSVE-NEXT:    fcvtzu x8, s6
-; NONEON-NOSVE-NEXT:    fcvtzu x14, s4
-; NONEON-NOSVE-NEXT:    fcvtzu x12, s3
-; NONEON-NOSVE-NEXT:    mov v7.d[1], x11
-; NONEON-NOSVE-NEXT:    fmov d3, x10
-; NONEON-NOSVE-NEXT:    fcvtzu x11, s5
-; NONEON-NOSVE-NEXT:    fmov d2, x15
-; NONEON-NOSVE-NEXT:    stp q16, q1, [x1, #64]
-; NONEON-NOSVE-NEXT:    fmov d1, x9
-; NONEON-NOSVE-NEXT:    fmov d4, x8
-; NONEON-NOSVE-NEXT:    stp q7, q0, [x1]
-; NONEON-NOSVE-NEXT:    mov v2.d[1], x13
-; NONEON-NOSVE-NEXT:    mov v3.d[1], x14
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x12
-; NONEON-NOSVE-NEXT:    mov v4.d[1], x11
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x1, #96]
-; NONEON-NOSVE-NEXT:    stp q4, q1, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %res = fptoui <16 x half> %op1 to <16 x i64>
   store <16 x i64> %res, ptr %b
@@ -528,11 +282,6 @@ define <2 x i16> @fcvtzu_v2f32_v2i16(<2 x float> %op1) {
 ; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v2f32_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <2 x float> %op1 to <2 x i16>
   ret <2 x i16> %res
 }
@@ -546,12 +295,6 @@ define <4 x i16> @fcvtzu_v4f32_v4i16(<4 x float> %op1) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v4f32_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <4 x float> %op1 to <4 x i16>
   ret <4 x i16> %res
 }
@@ -569,14 +312,6 @@ define <8 x i16> @fcvtzu_v8f32_v8i16(ptr %a) {
 ; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v8f32_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzu v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %res = fptoui <8 x float> %op1 to <8 x i16>
   ret <8 x i16> %res
@@ -601,19 +336,6 @@ define void @fcvtzu_v16f32_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
 ; CHECK-NEXT:    stp q2, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v16f32_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzu v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x float>, ptr %a
   %res = fptoui <16 x float> %op1 to <16 x i16>
   store <16 x i16> %res, ptr %b
@@ -632,11 +354,6 @@ define <2 x i32> @fcvtzu_v2f32_v2i32(<2 x float> %op1) {
 ; CHECK-NEXT:    fcvtzu z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v2f32_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzu v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <2 x float> %op1 to <2 x i32>
   ret <2 x i32> %res
 }
@@ -649,11 +366,6 @@ define <4 x i32> @fcvtzu_v4f32_v4i32(<4 x float> %op1) {
 ; CHECK-NEXT:    fcvtzu z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v4f32_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <4 x float> %op1 to <4 x i32>
   ret <4 x i32> %res
 }
@@ -667,14 +379,6 @@ define void @fcvtzu_v8f32_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzu z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v8f32_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzu v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzu v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %res = fptoui <8 x float> %op1 to <8 x i32>
   store <8 x i32> %res, ptr %b
@@ -694,13 +398,6 @@ define <1 x i64> @fcvtzu_v1f32_v1i64(<1 x float> %op1) {
 ; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v1f32_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtzu v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <1 x float> %op1 to <1 x i64>
   ret <1 x i64> %res
 }
@@ -714,12 +411,6 @@ define <2 x i64> @fcvtzu_v2f32_v2i64(<2 x float> %op1) {
 ; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v2f32_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtzu v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <2 x float> %op1 to <2 x i64>
   ret <2 x i64> %res
 }
@@ -736,20 +427,6 @@ define void @fcvtzu_v4f32_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.s
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v4f32_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v1.2s
-; NONEON-NOSVE-NEXT:    fcvtzu v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzu v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x float>, ptr %a
   %res = fptoui <4 x float> %op1 to <4 x i64>
   store <4 x i64> %res, ptr %b
@@ -774,26 +451,6 @@ define void @fcvtzu_v8f32_v8i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v8f32_v8i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v1.2s
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtl v2.2d, v2.2s
-; NONEON-NOSVE-NEXT:    fcvtl v3.2d, v3.2s
-; NONEON-NOSVE-NEXT:    fcvtzu v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzu v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzu v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    fcvtzu v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %res = fptoui <8 x float> %op1 to <8 x i64>
   store <8 x i64> %res, ptr %b
@@ -811,12 +468,6 @@ define <1 x i16> @fcvtzu_v1f64_v1i16(<1 x double> %op1) {
 ; CHECK-NEXT:    mov z0.h, w8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v1f64_v1i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs w8, d0
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <1 x double> %op1 to <1 x i16>
   ret <1 x i16> %res
 }
@@ -830,12 +481,6 @@ define <2 x i16> @fcvtzu_v2f64_v2i16(<2 x double> %op1) {
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v2f64_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    xtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <2 x double> %op1 to <2 x i16>
   ret <2 x i16> %res
 }
@@ -864,15 +509,6 @@ define <4 x i16> @fcvtzu_v4f64_v4i16(ptr %a) {
 ; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v4f64_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %res = fptoui <4 x double> %op1 to <4 x i16>
   ret <4 x i16> %res
@@ -916,23 +552,6 @@ define <8 x i16> @fcvtzu_v8f64_v8i16(ptr %a) {
 ; CHECK-NEXT:    strh w8, [sp, #2]
 ; CHECK-NEXT:    ldr q0, [sp], #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v8f64_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI26_0
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    xtn v7.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI26_0]
-; NONEON-NOSVE-NEXT:    xtn v6.2s, v1.2d
-; NONEON-NOSVE-NEXT:    xtn v5.2s, v2.2d
-; NONEON-NOSVE-NEXT:    xtn v4.2s, v3.2d
-; NONEON-NOSVE-NEXT:    tbl v0.16b, { v4.16b, v5.16b, v6.16b, v7.16b }, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x double>, ptr %a
   %res = fptoui <8 x double> %op1 to <8 x i16>
   ret <8 x i16> %res
@@ -1009,35 +628,6 @@ define void @fcvtzu_v16f64_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v16f64_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #96]
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI27_0
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q4, q5, [x0, #64]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v5.2d, v5.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v4.2d, v4.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v6.2d, v6.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v7.2d, v7.2d
-; NONEON-NOSVE-NEXT:    xtn v19.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI27_0]
-; NONEON-NOSVE-NEXT:    xtn v23.2s, v3.2d
-; NONEON-NOSVE-NEXT:    xtn v18.2s, v1.2d
-; NONEON-NOSVE-NEXT:    xtn v22.2s, v2.2d
-; NONEON-NOSVE-NEXT:    xtn v17.2s, v5.2d
-; NONEON-NOSVE-NEXT:    xtn v21.2s, v6.2d
-; NONEON-NOSVE-NEXT:    xtn v16.2s, v4.2d
-; NONEON-NOSVE-NEXT:    xtn v20.2s, v7.2d
-; NONEON-NOSVE-NEXT:    tbl v1.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v0.16b
-; NONEON-NOSVE-NEXT:    tbl v0.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v0.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x double>, ptr %a
   %res = fptoui <16 x double> %op1 to <16 x i16>
   store <16 x i16> %res, ptr %b
@@ -1057,13 +647,6 @@ define <1 x i32> @fcvtzu_v1f64_v1i32(<1 x double> %op1) {
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v1f64_v1i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fcvtzu v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    xtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <1 x double> %op1 to <1 x i32>
   ret <1 x i32> %res
 }
@@ -1077,12 +660,6 @@ define <2 x i32> @fcvtzu_v2f64_v2i32(<2 x double> %op1) {
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v2f64_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzu v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    xtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <2 x double> %op1 to <2 x i32>
   ret <2 x i32> %res
 }
@@ -1100,14 +677,6 @@ define <4 x i32> @fcvtzu_v4f64_v4i32(ptr %a) {
 ; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v4f64_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzu v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzu v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %res = fptoui <4 x double> %op1 to <4 x i32>
   ret <4 x i32> %res
@@ -1132,19 +701,6 @@ define void @fcvtzu_v8f64_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
 ; CHECK-NEXT:    stp q2, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v8f64_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzu v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzu v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzu v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    fcvtzu v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x double>, ptr %a
   %res = fptoui <8 x double> %op1 to <8 x i32>
   store <8 x i32> %res, ptr %b
@@ -1163,12 +719,6 @@ define <1 x i64> @fcvtzu_v1f64_v1i64(<1 x double> %op1) {
 ; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v1f64_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzu x8, d0
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <1 x double> %op1 to <1 x i64>
   ret <1 x i64> %res
 }
@@ -1181,11 +731,6 @@ define <2 x i64> @fcvtzu_v2f64_v2i64(<2 x double> %op1) {
 ; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v2f64_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzu v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fptoui <2 x double> %op1 to <2 x i64>
   ret <2 x i64> %res
 }
@@ -1199,14 +744,6 @@ define void @fcvtzu_v4f64_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzu z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzu_v4f64_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzu v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzu v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %res = fptoui <4 x double> %op1 to <4 x i64>
   store <4 x i64> %res, ptr %b
@@ -1225,13 +762,6 @@ define <4 x i16> @fcvtzs_v4f16_v4i16(<4 x half> %op1) {
 ; CHECK-NEXT:    fcvtzs z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v4f16_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <4 x half> %op1 to <4 x i16>
   ret <4 x i16> %res
 }
@@ -1244,21 +774,6 @@ define void @fcvtzs_v8f16_v8i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzs z0.h, p0/m, z0.h
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v8f16_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x half>, ptr %a
   %res = fptosi <8 x half> %op1 to <8 x i16>
   store <8 x i16> %res, ptr %b
@@ -1274,27 +789,6 @@ define void @fcvtzs_v16f16_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzs z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v16f16_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v3.4h
-; NONEON-NOSVE-NEXT:    fcvtzs v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %res = fptosi <16 x half> %op1 to <16 x i16>
   store <16 x i16> %res, ptr %b
@@ -1314,13 +808,6 @@ define <2 x i32> @fcvtzs_v2f16_v2i32(<2 x half> %op1) {
 ; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v2f16_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <2 x half> %op1 to <2 x i32>
   ret <2 x i32> %res
 }
@@ -1334,12 +821,6 @@ define <4 x i32> @fcvtzs_v4f16_v4i32(<4 x half> %op1) {
 ; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v4f16_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <4 x half> %op1 to <4 x i32>
   ret <4 x i32> %res
 }
@@ -1356,20 +837,6 @@ define void @fcvtzs_v8f16_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.h
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v8f16_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x half>, ptr %a
   %res = fptosi <8 x half> %op1 to <8 x i32>
   store <8 x i32> %res, ptr %b
@@ -1394,26 +861,6 @@ define void @fcvtzs_v16f16_v16i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v16f16_v16i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v3.4h
-; NONEON-NOSVE-NEXT:    fcvtzs v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %res = fptosi <16 x half> %op1 to <16 x i32>
   store <16 x i32> %res, ptr %b
@@ -1430,13 +877,6 @@ define <1 x i64> @fcvtzs_v1f16_v1i64(<1 x half> %op1) {
 ; CHECK-NEXT:    fcvtzs x8, h0
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v1f16_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvtzs x8, s0
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <1 x half> %op1 to <1 x i64>
   ret <1 x i64> %res
 }
@@ -1453,18 +893,6 @@ define <2 x i64> @fcvtzs_v2f16_v2i64(<2 x half> %op1) {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    ldr q0, [sp], #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v2f16_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvtzs x8, s0
-; NONEON-NOSVE-NEXT:    fcvtzs x9, s1
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x9
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <2 x half> %op1 to <2 x i64>
   ret <2 x i64> %res
 }
@@ -1487,27 +915,6 @@ define void @fcvtzs_v4f16_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v4f16_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h2, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvtzs x9, s0
-; NONEON-NOSVE-NEXT:    fcvtzs x8, s1
-; NONEON-NOSVE-NEXT:    fcvtzs x10, s2
-; NONEON-NOSVE-NEXT:    fcvtzs x11, s3
-; NONEON-NOSVE-NEXT:    fmov d1, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x11
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x10
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x half>, ptr %a
   %res = fptosi <4 x half> %op1 to <4 x i64>
   store <4 x i64> %res, ptr %b
@@ -1545,47 +952,6 @@ define void @fcvtzs_v8f16_v8i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q1, q0, [x1, #32]
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v8f16_v8i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    mov h1, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[3]
-; NONEON-NOSVE-NEXT:    mov h4, v0.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    mov h5, v2.h[2]
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[3]
-; NONEON-NOSVE-NEXT:    mov h7, v2.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvtzs x9, s0
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvtzs x13, s2
-; NONEON-NOSVE-NEXT:    fcvtzs x8, s1
-; NONEON-NOSVE-NEXT:    fcvt s1, h7
-; NONEON-NOSVE-NEXT:    fcvtzs x10, s3
-; NONEON-NOSVE-NEXT:    fcvtzs x11, s4
-; NONEON-NOSVE-NEXT:    fcvtzs x12, s5
-; NONEON-NOSVE-NEXT:    fcvtzs x14, s6
-; NONEON-NOSVE-NEXT:    fmov d3, x13
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    fcvtzs x8, s1
-; NONEON-NOSVE-NEXT:    fmov d1, x9
-; NONEON-NOSVE-NEXT:    fmov d2, x12
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x10
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x11
-; NONEON-NOSVE-NEXT:    mov v3.d[1], x8
-; NONEON-NOSVE-NEXT:    mov v2.d[1], x14
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x half>, ptr %a
   %res = fptosi <8 x half> %op1 to <8 x i64>
   store <8 x i64> %res, ptr %b
@@ -1646,80 +1012,6 @@ define void @fcvtzs_v16f16_v16i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q5, q2, [x1, #96]
 ; CHECK-NEXT:    add sp, sp, #128
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v16f16_v16i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s3, h1
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #24]
-; NONEON-NOSVE-NEXT:    mov h5, v1.h[3]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[2]
-; NONEON-NOSVE-NEXT:    mov h16, v0.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s6, h0
-; NONEON-NOSVE-NEXT:    mov h0, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h1, v1.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s17, h4
-; NONEON-NOSVE-NEXT:    mov h18, v4.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvtzs x8, s3
-; NONEON-NOSVE-NEXT:    fcvt s3, h5
-; NONEON-NOSVE-NEXT:    fcvt s5, h7
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    mov h16, v4.h[3]
-; NONEON-NOSVE-NEXT:    fcvtzs x9, s6
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvt s0, h0
-; NONEON-NOSVE-NEXT:    fcvt s1, h1
-; NONEON-NOSVE-NEXT:    mov h4, v4.h[1]
-; NONEON-NOSVE-NEXT:    fcvtzs x11, s2
-; NONEON-NOSVE-NEXT:    mov h2, v6.h[2]
-; NONEON-NOSVE-NEXT:    fcvtzs x10, s17
-; NONEON-NOSVE-NEXT:    fcvtzs x13, s5
-; NONEON-NOSVE-NEXT:    fcvtzs x12, s3
-; NONEON-NOSVE-NEXT:    mov h3, v6.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    mov h5, v6.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s17, h18
-; NONEON-NOSVE-NEXT:    fcvtzs x14, s7
-; NONEON-NOSVE-NEXT:    fmov d7, x8
-; NONEON-NOSVE-NEXT:    fcvtzs x8, s0
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fmov d0, x11
-; NONEON-NOSVE-NEXT:    fcvtzs x11, s1
-; NONEON-NOSVE-NEXT:    fmov d1, x13
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvtzs x13, s16
-; NONEON-NOSVE-NEXT:    fmov d16, x9
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvtzs x15, s17
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x12
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x14
-; NONEON-NOSVE-NEXT:    fcvtzs x9, s2
-; NONEON-NOSVE-NEXT:    mov v16.d[1], x8
-; NONEON-NOSVE-NEXT:    fcvtzs x8, s6
-; NONEON-NOSVE-NEXT:    fcvtzs x14, s4
-; NONEON-NOSVE-NEXT:    fcvtzs x12, s3
-; NONEON-NOSVE-NEXT:    mov v7.d[1], x11
-; NONEON-NOSVE-NEXT:    fmov d3, x10
-; NONEON-NOSVE-NEXT:    fcvtzs x11, s5
-; NONEON-NOSVE-NEXT:    fmov d2, x15
-; NONEON-NOSVE-NEXT:    stp q16, q1, [x1, #64]
-; NONEON-NOSVE-NEXT:    fmov d1, x9
-; NONEON-NOSVE-NEXT:    fmov d4, x8
-; NONEON-NOSVE-NEXT:    stp q7, q0, [x1]
-; NONEON-NOSVE-NEXT:    mov v2.d[1], x13
-; NONEON-NOSVE-NEXT:    mov v3.d[1], x14
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x12
-; NONEON-NOSVE-NEXT:    mov v4.d[1], x11
-; NONEON-NOSVE-NEXT:    stp q3, q2, [x1, #96]
-; NONEON-NOSVE-NEXT:    stp q4, q1, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %res = fptosi <16 x half> %op1 to <16 x i64>
   store <16 x i64> %res, ptr %b
@@ -1738,11 +1030,6 @@ define <2 x i16> @fcvtzs_v2f32_v2i16(<2 x float> %op1) {
 ; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v2f32_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <2 x float> %op1 to <2 x i16>
   ret <2 x i16> %res
 }
@@ -1756,12 +1043,6 @@ define <4 x i16> @fcvtzs_v4f32_v4i16(<4 x float> %op1) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v4f32_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <4 x float> %op1 to <4 x i16>
   ret <4 x i16> %res
 }
@@ -1779,14 +1060,6 @@ define <8 x i16> @fcvtzs_v8f32_v8i16(ptr %a) {
 ; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v8f32_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %res = fptosi <8 x float> %op1 to <8 x i16>
   ret <8 x i16> %res
@@ -1811,19 +1084,6 @@ define void @fcvtzs_v16f32_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
 ; CHECK-NEXT:    stp q2, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v16f32_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x float>, ptr %a
   %res = fptosi <16 x float> %op1 to <16 x i16>
   store <16 x i16> %res, ptr %b
@@ -1842,11 +1102,6 @@ define <2 x i32> @fcvtzs_v2f32_v2i32(<2 x float> %op1) {
 ; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v2f32_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <2 x float> %op1 to <2 x i32>
   ret <2 x i32> %res
 }
@@ -1859,11 +1114,6 @@ define <4 x i32> @fcvtzs_v4f32_v4i32(<4 x float> %op1) {
 ; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v4f32_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <4 x float> %op1 to <4 x i32>
   ret <4 x i32> %res
 }
@@ -1877,14 +1127,6 @@ define void @fcvtzs_v8f32_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzs z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v8f32_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtzs v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %res = fptosi <8 x float> %op1 to <8 x i32>
   store <8 x i32> %res, ptr %b
@@ -1904,13 +1146,6 @@ define <1 x i64> @fcvtzs_v1f32_v1i64(<1 x float> %op1) {
 ; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v1f32_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <1 x float> %op1 to <1 x i64>
   ret <1 x i64> %res
 }
@@ -1924,12 +1159,6 @@ define <2 x i64> @fcvtzs_v2f32_v2i64(<2 x float> %op1) {
 ; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v2f32_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <2 x float> %op1 to <2 x i64>
   ret <2 x i64> %res
 }
@@ -1946,20 +1175,6 @@ define void @fcvtzs_v4f32_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.s
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v4f32_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v1.2s
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x float>, ptr %a
   %res = fptosi <4 x float> %op1 to <4 x i64>
   store <4 x i64> %res, ptr %b
@@ -1984,26 +1199,6 @@ define void @fcvtzs_v8f32_v8i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v8f32_v8i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    fcvtl v1.2d, v1.2s
-; NONEON-NOSVE-NEXT:    fcvtl v0.2d, v0.2s
-; NONEON-NOSVE-NEXT:    fcvtl v2.2d, v2.2s
-; NONEON-NOSVE-NEXT:    fcvtl v3.2d, v3.2s
-; NONEON-NOSVE-NEXT:    fcvtzs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %res = fptosi <8 x float> %op1 to <8 x i64>
   store <8 x i64> %res, ptr %b
@@ -2023,12 +1218,6 @@ define <1 x i16> @fcvtzs_v1f64_v1i16(<1 x double> %op1) {
 ; CHECK-NEXT:    mov z0.h, w8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v1f64_v1i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs w8, d0
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <1 x double> %op1 to <1 x i16>
   ret <1 x i16> %res
 }
@@ -2042,12 +1231,6 @@ define <2 x i16> @fcvtzs_v2f64_v2i16(<2 x double> %op1) {
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v2f64_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    xtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <2 x double> %op1 to <2 x i16>
   ret <2 x i16> %res
 }
@@ -2076,15 +1259,6 @@ define <4 x i16> @fcvtzs_v4f64_v4i16(ptr %a) {
 ; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v4f64_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %res = fptosi <4 x double> %op1 to <4 x i16>
   ret <4 x i16> %res
@@ -2128,23 +1302,6 @@ define <8 x i16> @fcvtzs_v8f64_v8i16(ptr %a) {
 ; CHECK-NEXT:    strh w8, [sp, #2]
 ; CHECK-NEXT:    ldr q0, [sp], #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v8f64_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI61_0
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    xtn v7.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI61_0]
-; NONEON-NOSVE-NEXT:    xtn v6.2s, v1.2d
-; NONEON-NOSVE-NEXT:    xtn v5.2s, v2.2d
-; NONEON-NOSVE-NEXT:    xtn v4.2s, v3.2d
-; NONEON-NOSVE-NEXT:    tbl v0.16b, { v4.16b, v5.16b, v6.16b, v7.16b }, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x double>, ptr %a
   %res = fptosi <8 x double> %op1 to <8 x i16>
   ret <8 x i16> %res
@@ -2221,35 +1378,6 @@ define void @fcvtzs_v16f64_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v16f64_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #96]
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI62_0
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q4, q5, [x0, #64]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v5.2d, v5.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v4.2d, v4.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v6.2d, v6.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v7.2d, v7.2d
-; NONEON-NOSVE-NEXT:    xtn v19.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI62_0]
-; NONEON-NOSVE-NEXT:    xtn v23.2s, v3.2d
-; NONEON-NOSVE-NEXT:    xtn v18.2s, v1.2d
-; NONEON-NOSVE-NEXT:    xtn v22.2s, v2.2d
-; NONEON-NOSVE-NEXT:    xtn v17.2s, v5.2d
-; NONEON-NOSVE-NEXT:    xtn v21.2s, v6.2d
-; NONEON-NOSVE-NEXT:    xtn v16.2s, v4.2d
-; NONEON-NOSVE-NEXT:    xtn v20.2s, v7.2d
-; NONEON-NOSVE-NEXT:    tbl v1.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v0.16b
-; NONEON-NOSVE-NEXT:    tbl v0.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v0.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x double>, ptr %a
   %res = fptosi <16 x double> %op1 to <16 x i16>
   store <16 x i16> %res, ptr %b
@@ -2269,13 +1397,6 @@ define <1 x i32> @fcvtzs_v1f64_v1i32(<1 x double> %op1) {
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v1f64_v1i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    xtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <1 x double> %op1 to <1 x i32>
   ret <1 x i32> %res
 }
@@ -2289,12 +1410,6 @@ define <2 x i32> @fcvtzs_v2f64_v2i32(<2 x double> %op1) {
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v2f64_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    xtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <2 x double> %op1 to <2 x i32>
   ret <2 x i32> %res
 }
@@ -2312,14 +1427,6 @@ define <4 x i32> @fcvtzs_v4f64_v4i32(ptr %a) {
 ; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v4f64_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %res = fptosi <4 x double> %op1 to <4 x i32>
   ret <4 x i32> %res
@@ -2344,19 +1451,6 @@ define void @fcvtzs_v8f64_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
 ; CHECK-NEXT:    stp q2, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v8f64_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x double>, ptr %a
   %res = fptosi <8 x double> %op1 to <8 x i32>
   store <8 x i32> %res, ptr %b
@@ -2375,12 +1469,6 @@ define <1 x i64> @fcvtzs_v1f64_v1i64(<1 x double> %op1) {
 ; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v1f64_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs x8, d0
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <1 x double> %op1 to <1 x i64>
   ret <1 x i64> %res
 }
@@ -2393,11 +1481,6 @@ define <2 x i64> @fcvtzs_v2f64_v2i64(<2 x double> %op1) {
 ; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v2f64_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = fptosi <2 x double> %op1 to <2 x i64>
   ret <2 x i64> %res
 }
@@ -2411,14 +1494,6 @@ define void @fcvtzs_v4f64_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fcvtzs z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fcvtzs_v4f64_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fcvtzs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtzs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %res = fptosi <4 x double> %op1 to <4 x i64>
   store <4 x i64> %res, ptr %b
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll
index 32fe74bbb65f..30a4f04a3d2b 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -28,14 +27,6 @@ define <2 x half> @select_v2f16(<2 x half> %op1, <2 x half> %op2, <2 x i1> %mask
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    uzp1 v2.4h, v2.4h, v0.4h
-; NONEON-NOSVE-NEXT:    shl v2.4h, v2.4h, #15
-; NONEON-NOSVE-NEXT:    cmlt v2.4h, v2.4h, #0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <2 x i1> %mask, <2 x half> %op1, <2 x half> %op2
   ret <2 x half> %sel
 }
@@ -54,13 +45,6 @@ define <4 x half> @select_v4f16(<4 x half> %op1, <4 x half> %op2, <4 x i1> %mask
 ; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v2.4h, v2.4h, #15
-; NONEON-NOSVE-NEXT:    cmlt v2.4h, v2.4h, #0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <4 x i1> %mask, <4 x half> %op1, <4 x half> %op2
   ret <4 x half> %sel
 }
@@ -80,14 +64,6 @@ define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x i1> %mask
 ; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v2.8h, v2.8b, #0
-; NONEON-NOSVE-NEXT:    shl v2.8h, v2.8h, #15
-; NONEON-NOSVE-NEXT:    cmlt v2.8h, v2.8h, #0
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <8 x i1> %mask, <8 x half> %op1, <8 x half> %op2
   ret <8 x half> %sel
 }
@@ -104,126 +80,6 @@ define void @select_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sel z1.h, p0, z2.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[1]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[1]
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[2]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s6, h1
-; NONEON-NOSVE-NEXT:    fcvt s7, h0
-; NONEON-NOSVE-NEXT:    mov h16, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov h17, v0.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fcmp s3, s2
-; NONEON-NOSVE-NEXT:    mov h2, v1.h[3]
-; NONEON-NOSVE-NEXT:    mov h3, v0.h[3]
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[4]
-; NONEON-NOSVE-NEXT:    fcvt s2, h2
-; NONEON-NOSVE-NEXT:    fcvt s3, h3
-; NONEON-NOSVE-NEXT:    csetm w14, eq
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov h5, v0.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w12, eq
-; NONEON-NOSVE-NEXT:    fcmp s3, s2
-; NONEON-NOSVE-NEXT:    ldr q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    ldr q3, [x1, #16]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w11, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov h7, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov h18, v3.h[3]
-; NONEON-NOSVE-NEXT:    csetm w13, eq
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    mov h4, v3.h[1]
-; NONEON-NOSVE-NEXT:    mov h5, v2.h[1]
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    csetm w9, eq
-; NONEON-NOSVE-NEXT:    fcmp s17, s16
-; NONEON-NOSVE-NEXT:    mov h16, v3.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s4, h4
-; NONEON-NOSVE-NEXT:    mov h17, v2.h[2]
-; NONEON-NOSVE-NEXT:    fcvt s5, h5
-; NONEON-NOSVE-NEXT:    csetm w10, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    fcvt s6, h3
-; NONEON-NOSVE-NEXT:    fcvt s7, h2
-; NONEON-NOSVE-NEXT:    csetm w15, eq
-; NONEON-NOSVE-NEXT:    fcmp s5, s4
-; NONEON-NOSVE-NEXT:    fmov s4, w14
-; NONEON-NOSVE-NEXT:    csetm w16, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v2.h[3]
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    fcvt s16, h17
-; NONEON-NOSVE-NEXT:    mov v4.h[1], w8
-; NONEON-NOSVE-NEXT:    fcvt s17, h18
-; NONEON-NOSVE-NEXT:    csetm w14, eq
-; NONEON-NOSVE-NEXT:    fmov s5, w14
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcmp s16, s7
-; NONEON-NOSVE-NEXT:    mov h7, v3.h[4]
-; NONEON-NOSVE-NEXT:    mov h16, v2.h[4]
-; NONEON-NOSVE-NEXT:    mov v4.h[2], w12
-; NONEON-NOSVE-NEXT:    mov v5.h[1], w16
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s6, s17
-; NONEON-NOSVE-NEXT:    mov h17, v2.h[5]
-; NONEON-NOSVE-NEXT:    fcvt s6, h7
-; NONEON-NOSVE-NEXT:    fcvt s7, h16
-; NONEON-NOSVE-NEXT:    mov h16, v3.h[5]
-; NONEON-NOSVE-NEXT:    mov v4.h[3], w11
-; NONEON-NOSVE-NEXT:    mov v5.h[2], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcvt s17, h17
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov h6, v3.h[6]
-; NONEON-NOSVE-NEXT:    mov h7, v2.h[6]
-; NONEON-NOSVE-NEXT:    fcvt s16, h16
-; NONEON-NOSVE-NEXT:    mov v4.h[4], w13
-; NONEON-NOSVE-NEXT:    mov v5.h[3], w8
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcvt s6, h6
-; NONEON-NOSVE-NEXT:    fcvt s7, h7
-; NONEON-NOSVE-NEXT:    fcmp s17, s16
-; NONEON-NOSVE-NEXT:    mov h16, v3.h[7]
-; NONEON-NOSVE-NEXT:    mov h17, v2.h[7]
-; NONEON-NOSVE-NEXT:    mov v5.h[4], w8
-; NONEON-NOSVE-NEXT:    mov v4.h[5], w9
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    fcvt s6, h16
-; NONEON-NOSVE-NEXT:    fcvt s7, h17
-; NONEON-NOSVE-NEXT:    mov v5.h[5], w8
-; NONEON-NOSVE-NEXT:    mov v4.h[6], w10
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    fcmp s7, s6
-; NONEON-NOSVE-NEXT:    mov v5.h[6], w8
-; NONEON-NOSVE-NEXT:    mov v4.h[7], w15
-; NONEON-NOSVE-NEXT:    csetm w8, eq
-; NONEON-NOSVE-NEXT:    mov v5.h[7], w8
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v4.16b
-; NONEON-NOSVE-NEXT:    mov v1.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %mask = fcmp oeq <16 x half> %op1, %op2
@@ -246,13 +102,6 @@ define <2 x float> @select_v2f32(<2 x float> %op1, <2 x float> %op2, <2 x i1> %m
 ; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v2.2s, v2.2s, #31
-; NONEON-NOSVE-NEXT:    cmlt v2.2s, v2.2s, #0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <2 x i1> %mask, <2 x float> %op1, <2 x float> %op2
   ret <2 x float> %sel
 }
@@ -272,14 +121,6 @@ define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x i1> %m
 ; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    shl v2.4s, v2.4s, #31
-; NONEON-NOSVE-NEXT:    cmlt v2.4s, v2.4s, #0
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <4 x i1> %mask, <4 x float> %op1, <4 x float> %op2
   ret <4 x float> %sel
 }
@@ -296,18 +137,6 @@ define void @select_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sel z1.s, p0, z2.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    fcmeq v4.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcmeq v5.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v4.16b
-; NONEON-NOSVE-NEXT:    mov v1.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %mask = fcmp oeq <8 x float> %op1, %op2
@@ -322,14 +151,6 @@ define <1 x double> @select_v1f64(<1 x double> %op1, <1 x double> %op2, <1 x i1>
 ; CHECK-NEXT:    tst w0, #0x1
 ; CHECK-NEXT:    fcsel d0, d0, d1, ne
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm x8, ne
-; NONEON-NOSVE-NEXT:    fmov d2, x8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <1 x i1> %mask, <1 x double> %op1, <1 x double> %op2
   ret <1 x double> %sel
 }
@@ -349,14 +170,6 @@ define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x i1>
 ; CHECK-NEXT:    sel z0.d, p0, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    shl v2.2d, v2.2d, #63
-; NONEON-NOSVE-NEXT:    cmlt v2.2d, v2.2d, #0
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <2 x i1> %mask, <2 x double> %op1, <2 x double> %op2
   ret <2 x double> %sel
 }
@@ -373,18 +186,6 @@ define void @select_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sel z1.d, p0, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    fcmeq v4.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcmeq v5.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v4.16b
-; NONEON-NOSVE-NEXT:    mov v1.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %mask = fcmp oeq <4 x double> %op1, %op2
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll
index c85048ab72e0..4aa965777c74 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -22,14 +21,6 @@ define <4 x i8> @insertelement_v4i8(<4 x i8> %op1) {
 ; CHECK-NEXT:    mov z0.h, p0/m, w8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v0.h[3], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <4 x i8> %op1, i8 5, i64 3
     ret <4 x i8> %r
 }
@@ -47,14 +38,6 @@ define <8 x i8> @insertelement_v8i8(<8 x i8> %op1) {
 ; CHECK-NEXT:    mov z0.b, p0/m, w8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v0.b[7], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <8 x i8> %op1, i8 5, i64 7
     ret <8 x i8> %r
 }
@@ -72,12 +55,6 @@ define <16 x i8> @insertelement_v16i8(<16 x i8> %op1) {
 ; CHECK-NEXT:    mov z0.b, p0/m, w8
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v0.b[15], w8
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <16 x i8> %op1, i8 5, i64 15
     ret <16 x i8> %r
 }
@@ -95,12 +72,6 @@ define <32 x i8> @insertelement_v32i8(<32 x i8> %op1) {
 ; CHECK-NEXT:    mov z1.b, p0/m, w8
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v1.b[15], w8
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <32 x i8> %op1, i8 5, i64 31
     ret <32 x i8> %r
 }
@@ -119,14 +90,6 @@ define <2 x i16> @insertelement_v2i16(<2 x i16> %op1) {
 ; CHECK-NEXT:    mov z0.s, p0/m, w8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <2 x i16> %op1, i16 5, i64 1
     ret <2 x i16> %r
 }
@@ -144,14 +107,6 @@ define <4 x i16> @insertelement_v4i16(<4 x i16> %op1) {
 ; CHECK-NEXT:    mov z0.h, p0/m, w8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v0.h[3], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <4 x i16> %op1, i16 5, i64 3
     ret <4 x i16> %r
 }
@@ -169,12 +124,6 @@ define <8 x i16> @insertelement_v8i16(<8 x i16> %op1) {
 ; CHECK-NEXT:    mov z0.h, p0/m, w8
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v0.h[7], w8
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <8 x i16> %op1, i16 5, i64 7
     ret <8 x i16> %r
 }
@@ -192,12 +141,6 @@ define <16 x i16> @insertelement_v16i16(<16 x i16> %op1) {
 ; CHECK-NEXT:    mov z1.h, p0/m, w8
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v1.h[7], w8
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <16 x i16> %op1, i16 5, i64 15
     ret <16 x i16> %r
 }
@@ -216,14 +159,6 @@ define <2 x i32> @insertelement_v2i32(<2 x i32> %op1) {
 ; CHECK-NEXT:    mov z0.s, p0/m, w8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <2 x i32> %op1, i32 5, i64 1
     ret <2 x i32> %r
 }
@@ -241,12 +176,6 @@ define <4 x i32> @insertelement_v4i32(<4 x i32> %op1) {
 ; CHECK-NEXT:    mov z0.s, p0/m, w8
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v0.s[3], w8
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <4 x i32> %op1, i32 5, i64 3
     ret <4 x i32> %r
 }
@@ -264,13 +193,6 @@ define <8 x i32> @insertelement_v8i32(ptr %a) {
 ; CHECK-NEXT:    mov z1.s, p0/m, w8
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v1.s[3], w8
-; NONEON-NOSVE-NEXT:    ret
     %op1 = load <8 x i32>, ptr %a
     %r = insertelement <8 x i32> %op1, i32 5, i64 7
     ret <8 x i32> %r
@@ -283,12 +205,6 @@ define <1 x i64> @insertelement_v1i64(<1 x i64> %op1) {
 ; CHECK-NEXT:    mov z0.d, #5 // =0x5
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <1 x i64> %op1, i64 5, i64 0
     ret <1 x i64> %r
 }
@@ -306,12 +222,6 @@ define <2 x i64> @insertelement_v2i64(<2 x i64> %op1) {
 ; CHECK-NEXT:    mov z0.d, p0/m, x8
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x8
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <2 x i64> %op1, i64 5, i64 1
     ret <2 x i64> %r
 }
@@ -329,13 +239,6 @@ define <4 x i64> @insertelement_v4i64(ptr %a) {
 ; CHECK-NEXT:    mov z1.d, p0/m, x8
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    mov w8, #5 // =0x5
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x8
-; NONEON-NOSVE-NEXT:    ret
     %op1 = load <4 x i64>, ptr %a
     %r = insertelement <4 x i64> %op1, i64 5, i64 3
     ret <4 x i64> %r
@@ -354,16 +257,6 @@ define <2 x half> @insertelement_v2f16(<2 x half> %op1) {
 ; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI14_0
-; NONEON-NOSVE-NEXT:    add x8, x8, :lo12:.LCPI14_0
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    ld1r { v1.4h }, [x8]
-; NONEON-NOSVE-NEXT:    mov v1.h[0], v0.h[0]
-; NONEON-NOSVE-NEXT:    fmov d0, d1
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <2 x half> %op1, half 5.0, i64 1
     ret <2 x half> %r
 }
@@ -381,15 +274,6 @@ define <4 x half> @insertelement_v4f16(<4 x half> %op1) {
 ; CHECK-NEXT:    mov z0.h, p0/m, h1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI15_0
-; NONEON-NOSVE-NEXT:    add x8, x8, :lo12:.LCPI15_0
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[3], [x8]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <4 x half> %op1, half 5.0, i64 3
     ret <4 x half> %r
 }
@@ -407,13 +291,6 @@ define <8 x half> @insertelement_v8f16(<8 x half> %op1) {
 ; CHECK-NEXT:    mov z0.h, p0/m, h1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI16_0
-; NONEON-NOSVE-NEXT:    add x8, x8, :lo12:.LCPI16_0
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[7], [x8]
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <8 x half> %op1, half 5.0, i64 7
     ret <8 x half> %r
 }
@@ -431,14 +308,6 @@ define <16 x half> @insertelement_v16f16(ptr %a) {
 ; CHECK-NEXT:    mov z1.h, p0/m, h2
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI17_0
-; NONEON-NOSVE-NEXT:    add x8, x8, :lo12:.LCPI17_0
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[7], [x8]
-; NONEON-NOSVE-NEXT:    ret
     %op1 = load <16 x half>, ptr %a
     %r = insertelement <16 x half> %op1, half 5.0, i64 15
     ret <16 x half> %r
@@ -458,14 +327,6 @@ define <2 x float> @insertelement_v2f32(<2 x float> %op1) {
 ; CHECK-NEXT:    mov z0.s, p0/m, s1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov s1, #5.00000000
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    mov v0.s[1], v1.s[0]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <2 x float> %op1, float 5.0, i64 1
     ret <2 x float> %r
 }
@@ -483,12 +344,6 @@ define <4 x float> @insertelement_v4f32(<4 x float> %op1) {
 ; CHECK-NEXT:    mov z0.s, p0/m, s1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov s1, #5.00000000
-; NONEON-NOSVE-NEXT:    mov v0.s[3], v1.s[0]
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <4 x float> %op1, float 5.0, i64 3
     ret <4 x float> %r
 }
@@ -506,13 +361,6 @@ define <8 x float> @insertelement_v8f32(ptr %a) {
 ; CHECK-NEXT:    mov z1.s, p0/m, s2
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov s2, #5.00000000
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    mov v1.s[3], v2.s[0]
-; NONEON-NOSVE-NEXT:    ret
     %op1 = load <8 x float>, ptr %a
     %r = insertelement <8 x float> %op1, float 5.0, i64 7
     ret <8 x float> %r
@@ -524,12 +372,6 @@ define <1 x double> @insertelement_v1f64(<1 x double> %op1) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov d0, #5.00000000
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov x8, #4617315517961601024 // =0x4014000000000000
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <1 x double> %op1, double 5.0, i64 0
     ret <1 x double> %r
 }
@@ -547,12 +389,6 @@ define <2 x double> @insertelement_v2f64(<2 x double> %op1) {
 ; CHECK-NEXT:    mov z0.d, p0/m, d1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov d1, #5.00000000
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    ret
     %r = insertelement <2 x double> %op1, double 5.0, i64 1
     ret <2 x double> %r
 }
@@ -570,14 +406,6 @@ define <4 x double> @insertelement_v4f64(ptr %a) {
 ; CHECK-NEXT:    mov z1.d, p0/m, d2
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: insertelement_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov d0, #5.00000000
-; NONEON-NOSVE-NEXT:    ldr q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    mov v1.d[1], v0.d[0]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
     %op1 = load <4 x double>, ptr %a
     %r = insertelement <4 x double> %op1, double 5.0, i64 3
     ret <4 x double> %r
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll
index da408a11e784..8baa87c6d686 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll
@@ -2,7 +2,6 @@
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -17,11 +16,6 @@ define <4 x i8> @add_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    add z0.h, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    add v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = add <4 x i8> %op1, %op2
   ret <4 x i8> %res
 }
@@ -34,11 +28,6 @@ define <8 x i8> @add_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    add z0.b, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    add v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = add <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -51,11 +40,6 @@ define <16 x i8> @add_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    add z0.b, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = add <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -69,15 +53,6 @@ define void @add_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z1.b, z2.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    add v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = add <32 x i8> %op1, %op2
@@ -93,11 +68,6 @@ define <2 x i16> @add_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; CHECK-NEXT:    add z0.s, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    add v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = add <2 x i16> %op1, %op2
   ret <2 x i16> %res
 }
@@ -110,11 +80,6 @@ define <4 x i16> @add_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    add z0.h, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    add v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = add <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -127,11 +92,6 @@ define <8 x i16> @add_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    add z0.h, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = add <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -145,15 +105,6 @@ define void @add_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z1.h, z2.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    add v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = add <16 x i16> %op1, %op2
@@ -169,11 +120,6 @@ define <2 x i32> @add_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    add z0.s, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    add v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = add <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -186,11 +132,6 @@ define <4 x i32> @add_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    add z0.s, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    add v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = add <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -204,15 +145,6 @@ define void @add_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z1.s, z2.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    add v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    add v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = add <8 x i32> %op1, %op2
@@ -228,11 +160,6 @@ define <1 x i64> @add_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    add z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    add d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = add <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -245,11 +172,6 @@ define <2 x i64> @add_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    add z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    add v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = add <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -263,15 +185,6 @@ define void @add_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    add v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    add v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = add <4 x i64> %op1, %op2
@@ -300,11 +213,6 @@ define <4 x i8> @mul_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; SVE2-NEXT:    mul z0.h, z0.h, z1.h
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mul v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = mul <4 x i8> %op1, %op2
   ret <4 x i8> %res
 }
@@ -326,11 +234,6 @@ define <8 x i8> @mul_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; SVE2-NEXT:    mul z0.b, z0.b, z1.b
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mul v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = mul <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -352,11 +255,6 @@ define <16 x i8> @mul_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; SVE2-NEXT:    mul z0.b, z0.b, z1.b
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mul v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = mul <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -381,15 +279,6 @@ define void @mul_v32i8(ptr %a, ptr %b) {
 ; SVE2-NEXT:    mul z1.b, z2.b, z3.b
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    mul v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    mul v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = mul <32 x i8> %op1, %op2
@@ -414,11 +303,6 @@ define <2 x i16> @mul_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; SVE2-NEXT:    mul z0.s, z0.s, z1.s
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mul v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = mul <2 x i16> %op1, %op2
   ret <2 x i16> %res
 }
@@ -440,11 +324,6 @@ define <4 x i16> @mul_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; SVE2-NEXT:    mul z0.h, z0.h, z1.h
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mul v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = mul <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -466,11 +345,6 @@ define <8 x i16> @mul_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; SVE2-NEXT:    mul z0.h, z0.h, z1.h
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mul v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = mul <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -495,15 +369,6 @@ define void @mul_v16i16(ptr %a, ptr %b) {
 ; SVE2-NEXT:    mul z1.h, z2.h, z3.h
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    mul v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    mul v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = mul <16 x i16> %op1, %op2
@@ -528,11 +393,6 @@ define <2 x i32> @mul_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; SVE2-NEXT:    mul z0.s, z0.s, z1.s
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mul v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = mul <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -554,11 +414,6 @@ define <4 x i32> @mul_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; SVE2-NEXT:    mul z0.s, z0.s, z1.s
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mul v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = mul <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -583,15 +438,6 @@ define void @mul_v8i32(ptr %a, ptr %b) {
 ; SVE2-NEXT:    mul z1.s, z2.s, z3.s
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    mul v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    mul v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = mul <8 x i32> %op1, %op2
@@ -616,16 +462,6 @@ define <1 x i64> @mul_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; SVE2-NEXT:    mul z0.d, z0.d, z1.d
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    mul x8, x9, x8
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %res = mul <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -647,18 +483,6 @@ define <2 x i64> @mul_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; SVE2-NEXT:    mul z0.d, z0.d, z1.d
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x10, d1
-; NONEON-NOSVE-NEXT:    fmov x11, d0
-; NONEON-NOSVE-NEXT:    mov x8, v1.d[1]
-; NONEON-NOSVE-NEXT:    mov x9, v0.d[1]
-; NONEON-NOSVE-NEXT:    mul x10, x11, x10
-; NONEON-NOSVE-NEXT:    mul x8, x9, x8
-; NONEON-NOSVE-NEXT:    fmov d0, x10
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x8
-; NONEON-NOSVE-NEXT:    ret
   %res = mul <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -683,29 +507,6 @@ define void @mul_v4i64(ptr %a, ptr %b) {
 ; SVE2-NEXT:    mul z1.d, z2.d, z3.d
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    fmov x12, d2
-; NONEON-NOSVE-NEXT:    mov x11, v2.d[1]
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    mov x10, v3.d[1]
-; NONEON-NOSVE-NEXT:    mov x13, v1.d[1]
-; NONEON-NOSVE-NEXT:    mov x14, v0.d[1]
-; NONEON-NOSVE-NEXT:    mul x8, x9, x8
-; NONEON-NOSVE-NEXT:    fmov x9, d3
-; NONEON-NOSVE-NEXT:    mul x10, x11, x10
-; NONEON-NOSVE-NEXT:    mul x9, x12, x9
-; NONEON-NOSVE-NEXT:    fmov d1, x8
-; NONEON-NOSVE-NEXT:    mul x11, x14, x13
-; NONEON-NOSVE-NEXT:    fmov d0, x9
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x11
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x10
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = mul <4 x i64> %op1, %op2
@@ -725,11 +526,6 @@ define <4 x i8> @sub_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    sub z0.h, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = sub <4 x i8> %op1, %op2
   ret <4 x i8> %res
 }
@@ -742,11 +538,6 @@ define <8 x i8> @sub_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    sub z0.b, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = sub <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -759,11 +550,6 @@ define <16 x i8> @sub_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    sub z0.b, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = sub <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -777,15 +563,6 @@ define void @sub_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sub z1.b, z2.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    sub v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    sub v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = sub <32 x i8> %op1, %op2
@@ -801,11 +578,6 @@ define <2 x i16> @sub_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; CHECK-NEXT:    sub z0.s, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = sub <2 x i16> %op1, %op2
   ret <2 x i16> %res
 }
@@ -818,11 +590,6 @@ define <4 x i16> @sub_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    sub z0.h, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = sub <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -835,11 +602,6 @@ define <8 x i16> @sub_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    sub z0.h, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = sub <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -853,15 +615,6 @@ define void @sub_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sub z1.h, z2.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    sub v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    sub v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = sub <16 x i16> %op1, %op2
@@ -877,11 +630,6 @@ define <2 x i32> @sub_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    sub z0.s, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = sub <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -894,11 +642,6 @@ define <4 x i32> @sub_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    sub z0.s, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = sub <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -912,15 +655,6 @@ define void @sub_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sub z1.s, z2.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    sub v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    sub v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = sub <8 x i32> %op1, %op2
@@ -936,11 +670,6 @@ define <1 x i64> @sub_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    sub z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = sub <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -953,11 +682,6 @@ define <2 x i64> @sub_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    sub z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = sub <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -971,15 +695,6 @@ define void @sub_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sub z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    sub v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    sub v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = sub <4 x i64> %op1, %op2
@@ -1000,13 +715,6 @@ define <4 x i8> @abs_v4i8(<4 x i8> %op1) {
 ; CHECK-NEXT:    abs z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    sshr v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    abs v0.4h, v0.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i8> @llvm.abs.v4i8(<4 x i8> %op1, i1 false)
   ret <4 x i8> %res
 }
@@ -1019,11 +727,6 @@ define <8 x i8> @abs_v8i8(<8 x i8> %op1) {
 ; CHECK-NEXT:    abs z0.b, p0/m, z0.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    abs v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i8> @llvm.abs.v8i8(<8 x i8> %op1, i1 false)
   ret <8 x i8> %res
 }
@@ -1036,11 +739,6 @@ define <16 x i8> @abs_v16i8(<16 x i8> %op1) {
 ; CHECK-NEXT:    abs z0.b, p0/m, z0.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    abs v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %op1, i1 false)
   ret <16 x i8> %res
 }
@@ -1054,14 +752,6 @@ define void @abs_v32i8(ptr %a) {
 ; CHECK-NEXT:    abs z1.b, p0/m, z1.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    abs v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    abs v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %res = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %op1, i1 false)
   store <32 x i8> %res, ptr %a
@@ -1077,13 +767,6 @@ define <2 x i16> @abs_v2i16(<2 x i16> %op1) {
 ; CHECK-NEXT:    abs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    sshr v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    abs v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i16> @llvm.abs.v2i16(<2 x i16> %op1, i1 false)
   ret <2 x i16> %res
 }
@@ -1096,11 +779,6 @@ define <4 x i16> @abs_v4i16(<4 x i16> %op1) {
 ; CHECK-NEXT:    abs z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    abs v0.4h, v0.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i16> @llvm.abs.v4i16(<4 x i16> %op1, i1 false)
   ret <4 x i16> %res
 }
@@ -1113,11 +791,6 @@ define <8 x i16> @abs_v8i16(<8 x i16> %op1) {
 ; CHECK-NEXT:    abs z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    abs v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %op1, i1 false)
   ret <8 x i16> %res
 }
@@ -1131,14 +804,6 @@ define void @abs_v16i16(ptr %a) {
 ; CHECK-NEXT:    abs z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    abs v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    abs v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %res = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %op1, i1 false)
   store <16 x i16> %res, ptr %a
@@ -1153,11 +818,6 @@ define <2 x i32> @abs_v2i32(<2 x i32> %op1) {
 ; CHECK-NEXT:    abs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    abs v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %op1, i1 false)
   ret <2 x i32> %res
 }
@@ -1170,11 +830,6 @@ define <4 x i32> @abs_v4i32(<4 x i32> %op1) {
 ; CHECK-NEXT:    abs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    abs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %op1, i1 false)
   ret <4 x i32> %res
 }
@@ -1188,14 +843,6 @@ define void @abs_v8i32(ptr %a) {
 ; CHECK-NEXT:    abs z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    abs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    abs v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %res = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %op1, i1 false)
   store <8 x i32> %res, ptr %a
@@ -1210,11 +857,6 @@ define <1 x i64> @abs_v1i64(<1 x i64> %op1) {
 ; CHECK-NEXT:    abs z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    abs d0, d0
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x i64> @llvm.abs.v1i64(<1 x i64> %op1, i1 false)
   ret <1 x i64> %res
 }
@@ -1227,11 +869,6 @@ define <2 x i64> @abs_v2i64(<2 x i64> %op1) {
 ; CHECK-NEXT:    abs z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    abs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %op1, i1 false)
   ret <2 x i64> %res
 }
@@ -1245,14 +882,6 @@ define void @abs_v4i64(ptr %a) {
 ; CHECK-NEXT:    abs z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    abs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    abs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %res = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %op1, i1 false)
   store <4 x i64> %res, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll
index 3148d4f1677c..73c1eac99dd3 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -19,11 +18,6 @@ define <8 x i8> @icmp_eq_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmeq v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %cmp = icmp eq <8 x i8> %op1, %op2
   %sext = sext <8 x i1> %cmp to <8 x i8>
   ret <8 x i8> %sext
@@ -39,11 +33,6 @@ define <16 x i8> @icmp_eq_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    mov z0.b, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmeq v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %cmp = icmp eq <16 x i8> %op1, %op2
   %sext = sext <16 x i1> %cmp to <16 x i8>
   ret <16 x i8> %sext
@@ -61,15 +50,6 @@ define void @icmp_eq_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z1.b, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmeq v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    cmeq v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %cmp = icmp eq <32 x i8> %op1, %op2
@@ -88,11 +68,6 @@ define <4 x i16> @icmp_eq_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmeq v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %cmp = icmp eq <4 x i16> %op1, %op2
   %sext = sext <4 x i1> %cmp to <4 x i16>
   ret <4 x i16> %sext
@@ -108,11 +83,6 @@ define <8 x i16> @icmp_eq_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmeq v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %cmp = icmp eq <8 x i16> %op1, %op2
   %sext = sext <8 x i1> %cmp to <8 x i16>
   ret <8 x i16> %sext
@@ -130,15 +100,6 @@ define void @icmp_eq_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmeq v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    cmeq v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %cmp = icmp eq <16 x i16> %op1, %op2
@@ -157,11 +118,6 @@ define <2 x i32> @icmp_eq_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmeq v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %cmp = icmp eq <2 x i32> %op1, %op2
   %sext = sext <2 x i1> %cmp to <2 x i32>
   ret <2 x i32> %sext
@@ -177,11 +133,6 @@ define <4 x i32> @icmp_eq_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmeq v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %cmp = icmp eq <4 x i32> %op1, %op2
   %sext = sext <4 x i1> %cmp to <4 x i32>
   ret <4 x i32> %sext
@@ -199,15 +150,6 @@ define void @icmp_eq_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z1.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmeq v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    cmeq v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %cmp = icmp eq <8 x i32> %op1, %op2
@@ -226,11 +168,6 @@ define <1 x i64> @icmp_eq_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    mov z0.d, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmeq d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %cmp = icmp eq <1 x i64> %op1, %op2
   %sext = sext <1 x i1> %cmp to <1 x i64>
   ret <1 x i64> %sext
@@ -246,11 +183,6 @@ define <2 x i64> @icmp_eq_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    mov z0.d, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmeq v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %cmp = icmp eq <2 x i64> %op1, %op2
   %sext = sext <2 x i1> %cmp to <2 x i64>
   ret <2 x i64> %sext
@@ -268,15 +200,6 @@ define void @icmp_eq_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z1.d, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmeq v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    cmeq v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %cmp = icmp eq <4 x i64> %op1, %op2
@@ -301,17 +224,6 @@ define void @icmp_ne_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z1.b, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_ne_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmeq v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    cmeq v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    mvn v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    mvn v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %cmp = icmp ne <32 x i8> %op1, %op2
@@ -334,14 +246,6 @@ define void @icmp_sge_v8i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_sge_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    cmge v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %op2 = load <8 x i16>, ptr %b
   %cmp = icmp sge <8 x i16> %op1, %op2
@@ -366,15 +270,6 @@ define void @icmp_sgt_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_sgt_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmgt v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    cmgt v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %cmp = icmp sgt <16 x i16> %op1, %op2
@@ -397,14 +292,6 @@ define void @icmp_sle_v4i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_sle_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    cmge v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i32>, ptr %a
   %op2 = load <4 x i32>, ptr %b
   %cmp = icmp sle <4 x i32> %op1, %op2
@@ -429,15 +316,6 @@ define void @icmp_slt_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z1.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_slt_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmgt v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    cmgt v1.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %cmp = icmp slt <8 x i32> %op1, %op2
@@ -460,14 +338,6 @@ define void @icmp_uge_v2i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.d, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_uge_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    cmhs v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x i64>, ptr %a
   %op2 = load <2 x i64>, ptr %b
   %cmp = icmp uge <2 x i64> %op1, %op2
@@ -490,14 +360,6 @@ define void @icmp_ugt_v2i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.d, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_ugt_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    cmhi v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x i64>, ptr %a
   %op2 = load <2 x i64>, ptr %b
   %cmp = icmp ugt <2 x i64> %op1, %op2
@@ -520,14 +382,6 @@ define void @icmp_ule_v2i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.d, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_ule_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    cmhs v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x i64>, ptr %a
   %op2 = load <2 x i64>, ptr %b
   %cmp = icmp ule <2 x i64> %op1, %op2
@@ -550,14 +404,6 @@ define void @icmp_ult_v2i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.d, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_ult_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    cmhi v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x i64>, ptr %a
   %op2 = load <2 x i64>, ptr %b
   %cmp = icmp ult <2 x i64> %op1, %op2
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll
index 27a4924ea367..5158dda37a8b 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll
@@ -2,7 +2,6 @@
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -25,31 +24,6 @@ define <4 x i8> @sdiv_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    shl v1.4h, v1.4h, #8
-; NONEON-NOSVE-NEXT:    sshr v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    sshr v1.4h, v1.4h, #8
-; NONEON-NOSVE-NEXT:    smov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    smov w10, v0.h[0]
-; NONEON-NOSVE-NEXT:    smov w11, v0.h[2]
-; NONEON-NOSVE-NEXT:    smov w12, v0.h[3]
-; NONEON-NOSVE-NEXT:    sdiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    smov w9, v1.h[0]
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.h[2]
-; NONEON-NOSVE-NEXT:    sdiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    smov w11, v1.h[3]
-; NONEON-NOSVE-NEXT:    fmov s0, w9
-; NONEON-NOSVE-NEXT:    mov v0.h[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w8, w12, w11
-; NONEON-NOSVE-NEXT:    mov v0.h[2], w10
-; NONEON-NOSVE-NEXT:    mov v0.h[3], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <4 x i8> %op1, %op2
   ret <4 x i8> %res
 }
@@ -77,45 +51,6 @@ define <8 x i8> @sdiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    uzp1 z0.b, z1.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    smov w10, v0.b[0]
-; NONEON-NOSVE-NEXT:    smov w11, v0.b[2]
-; NONEON-NOSVE-NEXT:    smov w12, v0.b[3]
-; NONEON-NOSVE-NEXT:    smov w13, v0.b[4]
-; NONEON-NOSVE-NEXT:    smov w14, v0.b[5]
-; NONEON-NOSVE-NEXT:    sdiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    smov w9, v1.b[0]
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.b[2]
-; NONEON-NOSVE-NEXT:    sdiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    smov w11, v1.b[3]
-; NONEON-NOSVE-NEXT:    fmov s2, w9
-; NONEON-NOSVE-NEXT:    smov w9, v1.b[6]
-; NONEON-NOSVE-NEXT:    mov v2.b[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    smov w12, v1.b[4]
-; NONEON-NOSVE-NEXT:    mov v2.b[2], w10
-; NONEON-NOSVE-NEXT:    smov w10, v0.b[6]
-; NONEON-NOSVE-NEXT:    sdiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    smov w13, v1.b[5]
-; NONEON-NOSVE-NEXT:    mov v2.b[3], w11
-; NONEON-NOSVE-NEXT:    smov w11, v0.b[7]
-; NONEON-NOSVE-NEXT:    sdiv w8, w14, w13
-; NONEON-NOSVE-NEXT:    mov v2.b[4], w12
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.b[7]
-; NONEON-NOSVE-NEXT:    mov v2.b[5], w8
-; NONEON-NOSVE-NEXT:    sdiv w8, w11, w10
-; NONEON-NOSVE-NEXT:    mov v2.b[6], w9
-; NONEON-NOSVE-NEXT:    mov v2.b[7], w8
-; NONEON-NOSVE-NEXT:    fmov d0, d2
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -163,75 +98,6 @@ define <16 x i8> @sdiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    smov w10, v0.b[0]
-; NONEON-NOSVE-NEXT:    smov w11, v0.b[2]
-; NONEON-NOSVE-NEXT:    smov w12, v0.b[3]
-; NONEON-NOSVE-NEXT:    smov w13, v0.b[4]
-; NONEON-NOSVE-NEXT:    smov w14, v0.b[5]
-; NONEON-NOSVE-NEXT:    smov w15, v0.b[6]
-; NONEON-NOSVE-NEXT:    smov w16, v0.b[7]
-; NONEON-NOSVE-NEXT:    smov w17, v0.b[8]
-; NONEON-NOSVE-NEXT:    smov w18, v0.b[9]
-; NONEON-NOSVE-NEXT:    sdiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    smov w9, v1.b[0]
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.b[2]
-; NONEON-NOSVE-NEXT:    sdiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    smov w11, v1.b[3]
-; NONEON-NOSVE-NEXT:    fmov s2, w9
-; NONEON-NOSVE-NEXT:    smov w9, v1.b[10]
-; NONEON-NOSVE-NEXT:    mov v2.b[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    smov w12, v1.b[4]
-; NONEON-NOSVE-NEXT:    mov v2.b[2], w10
-; NONEON-NOSVE-NEXT:    smov w10, v0.b[10]
-; NONEON-NOSVE-NEXT:    sdiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    smov w13, v1.b[5]
-; NONEON-NOSVE-NEXT:    mov v2.b[3], w11
-; NONEON-NOSVE-NEXT:    smov w11, v0.b[11]
-; NONEON-NOSVE-NEXT:    sdiv w13, w14, w13
-; NONEON-NOSVE-NEXT:    smov w14, v1.b[6]
-; NONEON-NOSVE-NEXT:    mov v2.b[4], w12
-; NONEON-NOSVE-NEXT:    smov w12, v0.b[12]
-; NONEON-NOSVE-NEXT:    sdiv w14, w15, w14
-; NONEON-NOSVE-NEXT:    smov w15, v1.b[7]
-; NONEON-NOSVE-NEXT:    mov v2.b[5], w13
-; NONEON-NOSVE-NEXT:    smov w13, v0.b[13]
-; NONEON-NOSVE-NEXT:    sdiv w15, w16, w15
-; NONEON-NOSVE-NEXT:    smov w16, v1.b[8]
-; NONEON-NOSVE-NEXT:    mov v2.b[6], w14
-; NONEON-NOSVE-NEXT:    sdiv w16, w17, w16
-; NONEON-NOSVE-NEXT:    smov w17, v1.b[9]
-; NONEON-NOSVE-NEXT:    mov v2.b[7], w15
-; NONEON-NOSVE-NEXT:    sdiv w8, w18, w17
-; NONEON-NOSVE-NEXT:    mov v2.b[8], w16
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.b[11]
-; NONEON-NOSVE-NEXT:    mov v2.b[9], w8
-; NONEON-NOSVE-NEXT:    sdiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    smov w11, v1.b[12]
-; NONEON-NOSVE-NEXT:    mov v2.b[10], w9
-; NONEON-NOSVE-NEXT:    smov w9, v1.b[14]
-; NONEON-NOSVE-NEXT:    sdiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    smov w12, v1.b[13]
-; NONEON-NOSVE-NEXT:    mov v2.b[11], w10
-; NONEON-NOSVE-NEXT:    smov w10, v1.b[15]
-; NONEON-NOSVE-NEXT:    sdiv w8, w13, w12
-; NONEON-NOSVE-NEXT:    smov w12, v0.b[14]
-; NONEON-NOSVE-NEXT:    mov v2.b[12], w11
-; NONEON-NOSVE-NEXT:    smov w11, v0.b[15]
-; NONEON-NOSVE-NEXT:    sdiv w9, w12, w9
-; NONEON-NOSVE-NEXT:    mov v2.b[13], w8
-; NONEON-NOSVE-NEXT:    sdiv w8, w11, w10
-; NONEON-NOSVE-NEXT:    mov v2.b[14], w9
-; NONEON-NOSVE-NEXT:    mov v2.b[15], w8
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -312,163 +178,6 @@ define void @sdiv_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    splice z3.b, p0, z3.b, z1.b
 ; CHECK-NEXT:    stp q3, q2, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str x27, [sp, #-80]! // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #16] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #32] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
-; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
-; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
-; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
-; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
-; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
-; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
-; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
-; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
-; NONEON-NOSVE-NEXT:    .cfi_offset w27, -80
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0]
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    smov w10, v0.b[0]
-; NONEON-NOSVE-NEXT:    smov w11, v0.b[2]
-; NONEON-NOSVE-NEXT:    smov w12, v0.b[3]
-; NONEON-NOSVE-NEXT:    smov w13, v0.b[4]
-; NONEON-NOSVE-NEXT:    smov w14, v0.b[5]
-; NONEON-NOSVE-NEXT:    smov w15, v0.b[6]
-; NONEON-NOSVE-NEXT:    smov w17, v0.b[8]
-; NONEON-NOSVE-NEXT:    smov w2, v0.b[10]
-; NONEON-NOSVE-NEXT:    smov w3, v0.b[11]
-; NONEON-NOSVE-NEXT:    smov w4, v0.b[12]
-; NONEON-NOSVE-NEXT:    sdiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    smov w9, v1.b[0]
-; NONEON-NOSVE-NEXT:    smov w5, v0.b[13]
-; NONEON-NOSVE-NEXT:    smov w6, v0.b[14]
-; NONEON-NOSVE-NEXT:    smov w1, v3.b[1]
-; NONEON-NOSVE-NEXT:    smov w7, v2.b[0]
-; NONEON-NOSVE-NEXT:    smov w19, v2.b[2]
-; NONEON-NOSVE-NEXT:    smov w20, v2.b[3]
-; NONEON-NOSVE-NEXT:    smov w21, v2.b[4]
-; NONEON-NOSVE-NEXT:    smov w22, v2.b[5]
-; NONEON-NOSVE-NEXT:    smov w23, v2.b[6]
-; NONEON-NOSVE-NEXT:    smov w24, v2.b[7]
-; NONEON-NOSVE-NEXT:    smov w25, v2.b[8]
-; NONEON-NOSVE-NEXT:    smov w26, v2.b[9]
-; NONEON-NOSVE-NEXT:    smov w27, v2.b[10]
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.b[2]
-; NONEON-NOSVE-NEXT:    sdiv w11, w11, w10
-; NONEON-NOSVE-NEXT:    smov w10, v1.b[3]
-; NONEON-NOSVE-NEXT:    fmov s5, w9
-; NONEON-NOSVE-NEXT:    smov w9, v3.b[11]
-; NONEON-NOSVE-NEXT:    mov v5.b[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w10, w12, w10
-; NONEON-NOSVE-NEXT:    smov w12, v1.b[4]
-; NONEON-NOSVE-NEXT:    mov v5.b[2], w11
-; NONEON-NOSVE-NEXT:    smov w11, v2.b[11]
-; NONEON-NOSVE-NEXT:    sdiv w13, w13, w12
-; NONEON-NOSVE-NEXT:    smov w12, v1.b[5]
-; NONEON-NOSVE-NEXT:    mov v5.b[3], w10
-; NONEON-NOSVE-NEXT:    smov w10, v3.b[12]
-; NONEON-NOSVE-NEXT:    sdiv w12, w14, w12
-; NONEON-NOSVE-NEXT:    smov w14, v1.b[6]
-; NONEON-NOSVE-NEXT:    mov v5.b[4], w13
-; NONEON-NOSVE-NEXT:    smov w13, v2.b[14]
-; NONEON-NOSVE-NEXT:    sdiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    smov w14, v1.b[7]
-; NONEON-NOSVE-NEXT:    smov w15, v0.b[7]
-; NONEON-NOSVE-NEXT:    mov v5.b[5], w12
-; NONEON-NOSVE-NEXT:    smov w12, v2.b[13]
-; NONEON-NOSVE-NEXT:    sdiv w14, w15, w14
-; NONEON-NOSVE-NEXT:    smov w15, v1.b[8]
-; NONEON-NOSVE-NEXT:    mov v5.b[6], w16
-; NONEON-NOSVE-NEXT:    sdiv w18, w17, w15
-; NONEON-NOSVE-NEXT:    smov w15, v1.b[9]
-; NONEON-NOSVE-NEXT:    smov w17, v0.b[9]
-; NONEON-NOSVE-NEXT:    mov v5.b[7], w14
-; NONEON-NOSVE-NEXT:    sdiv w17, w17, w15
-; NONEON-NOSVE-NEXT:    smov w15, v1.b[10]
-; NONEON-NOSVE-NEXT:    mov v5.b[8], w18
-; NONEON-NOSVE-NEXT:    sdiv w15, w2, w15
-; NONEON-NOSVE-NEXT:    smov w2, v1.b[11]
-; NONEON-NOSVE-NEXT:    mov v5.b[9], w17
-; NONEON-NOSVE-NEXT:    sdiv w2, w3, w2
-; NONEON-NOSVE-NEXT:    smov w3, v1.b[12]
-; NONEON-NOSVE-NEXT:    mov v5.b[10], w15
-; NONEON-NOSVE-NEXT:    sdiv w3, w4, w3
-; NONEON-NOSVE-NEXT:    smov w4, v1.b[13]
-; NONEON-NOSVE-NEXT:    mov v5.b[11], w2
-; NONEON-NOSVE-NEXT:    sdiv w4, w5, w4
-; NONEON-NOSVE-NEXT:    smov w5, v1.b[14]
-; NONEON-NOSVE-NEXT:    mov v5.b[12], w3
-; NONEON-NOSVE-NEXT:    sdiv w5, w6, w5
-; NONEON-NOSVE-NEXT:    smov w6, v2.b[1]
-; NONEON-NOSVE-NEXT:    mov v5.b[13], w4
-; NONEON-NOSVE-NEXT:    sdiv w1, w6, w1
-; NONEON-NOSVE-NEXT:    smov w6, v3.b[0]
-; NONEON-NOSVE-NEXT:    mov v5.b[14], w5
-; NONEON-NOSVE-NEXT:    sdiv w6, w7, w6
-; NONEON-NOSVE-NEXT:    smov w7, v3.b[2]
-; NONEON-NOSVE-NEXT:    sdiv w7, w19, w7
-; NONEON-NOSVE-NEXT:    smov w19, v3.b[3]
-; NONEON-NOSVE-NEXT:    fmov s4, w6
-; NONEON-NOSVE-NEXT:    mov v4.b[1], w1
-; NONEON-NOSVE-NEXT:    sdiv w19, w20, w19
-; NONEON-NOSVE-NEXT:    smov w20, v3.b[4]
-; NONEON-NOSVE-NEXT:    mov v4.b[2], w7
-; NONEON-NOSVE-NEXT:    sdiv w20, w21, w20
-; NONEON-NOSVE-NEXT:    smov w21, v3.b[5]
-; NONEON-NOSVE-NEXT:    mov v4.b[3], w19
-; NONEON-NOSVE-NEXT:    sdiv w21, w22, w21
-; NONEON-NOSVE-NEXT:    smov w22, v3.b[6]
-; NONEON-NOSVE-NEXT:    mov v4.b[4], w20
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w22, w23, w22
-; NONEON-NOSVE-NEXT:    smov w23, v3.b[7]
-; NONEON-NOSVE-NEXT:    mov v4.b[5], w21
-; NONEON-NOSVE-NEXT:    sdiv w23, w24, w23
-; NONEON-NOSVE-NEXT:    smov w24, v3.b[8]
-; NONEON-NOSVE-NEXT:    mov v4.b[6], w22
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w24, w25, w24
-; NONEON-NOSVE-NEXT:    smov w25, v3.b[9]
-; NONEON-NOSVE-NEXT:    mov v4.b[7], w23
-; NONEON-NOSVE-NEXT:    sdiv w25, w26, w25
-; NONEON-NOSVE-NEXT:    smov w26, v3.b[10]
-; NONEON-NOSVE-NEXT:    mov v4.b[8], w24
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #32] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w8, w27, w26
-; NONEON-NOSVE-NEXT:    mov v4.b[9], w25
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #16] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w9, w11, w9
-; NONEON-NOSVE-NEXT:    smov w11, v2.b[12]
-; NONEON-NOSVE-NEXT:    mov v4.b[10], w8
-; NONEON-NOSVE-NEXT:    smov w8, v3.b[15]
-; NONEON-NOSVE-NEXT:    sdiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    smov w11, v3.b[13]
-; NONEON-NOSVE-NEXT:    mov v4.b[11], w9
-; NONEON-NOSVE-NEXT:    smov w9, v1.b[15]
-; NONEON-NOSVE-NEXT:    sdiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    smov w12, v3.b[14]
-; NONEON-NOSVE-NEXT:    mov v4.b[12], w10
-; NONEON-NOSVE-NEXT:    smov w10, v0.b[15]
-; NONEON-NOSVE-NEXT:    sdiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    smov w13, v2.b[15]
-; NONEON-NOSVE-NEXT:    mov v4.b[13], w11
-; NONEON-NOSVE-NEXT:    sdiv w8, w13, w8
-; NONEON-NOSVE-NEXT:    mov v4.b[14], w12
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    mov v4.b[15], w8
-; NONEON-NOSVE-NEXT:    mov v5.b[15], w9
-; NONEON-NOSVE-NEXT:    stp q4, q5, [x0]
-; NONEON-NOSVE-NEXT:    ldr x27, [sp], #80 // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = sdiv <32 x i8> %op1, %op2
@@ -487,23 +196,6 @@ define <2 x i16> @sdiv_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; CHECK-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    shl v1.2s, v1.2s, #16
-; NONEON-NOSVE-NEXT:    sshr v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    sshr v1.2s, v1.2s, #16
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    fmov w9, s0
-; NONEON-NOSVE-NEXT:    mov w10, v0.s[1]
-; NONEON-NOSVE-NEXT:    sdiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    mov w9, v1.s[1]
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w9
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <2 x i16> %op1, %op2
   ret <2 x i16> %res
 }
@@ -520,29 +212,6 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    smov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    smov w10, v0.h[0]
-; NONEON-NOSVE-NEXT:    smov w11, v0.h[2]
-; NONEON-NOSVE-NEXT:    smov w12, v0.h[3]
-; NONEON-NOSVE-NEXT:    sdiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    smov w9, v1.h[0]
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.h[2]
-; NONEON-NOSVE-NEXT:    sdiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    smov w11, v1.h[3]
-; NONEON-NOSVE-NEXT:    fmov s0, w9
-; NONEON-NOSVE-NEXT:    mov v0.h[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w8, w12, w11
-; NONEON-NOSVE-NEXT:    mov v0.h[2], w10
-; NONEON-NOSVE-NEXT:    mov v0.h[3], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -569,43 +238,6 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    smov w10, v0.h[0]
-; NONEON-NOSVE-NEXT:    smov w11, v0.h[2]
-; NONEON-NOSVE-NEXT:    smov w12, v0.h[3]
-; NONEON-NOSVE-NEXT:    smov w13, v0.h[4]
-; NONEON-NOSVE-NEXT:    smov w14, v0.h[5]
-; NONEON-NOSVE-NEXT:    sdiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    smov w9, v1.h[0]
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.h[2]
-; NONEON-NOSVE-NEXT:    sdiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    smov w11, v1.h[3]
-; NONEON-NOSVE-NEXT:    fmov s2, w9
-; NONEON-NOSVE-NEXT:    smov w9, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    smov w12, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w10
-; NONEON-NOSVE-NEXT:    smov w10, v0.h[6]
-; NONEON-NOSVE-NEXT:    sdiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    smov w13, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w11
-; NONEON-NOSVE-NEXT:    smov w11, v0.h[7]
-; NONEON-NOSVE-NEXT:    sdiv w8, w14, w13
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w12
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    sdiv w8, w11, w10
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w9
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -646,79 +278,6 @@ define void @sdiv_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    splice z3.h, p0, z3.h, z1.h
 ; CHECK-NEXT:    stp q3, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0]
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    smov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    smov w10, v0.h[0]
-; NONEON-NOSVE-NEXT:    smov w11, v0.h[2]
-; NONEON-NOSVE-NEXT:    smov w12, v0.h[3]
-; NONEON-NOSVE-NEXT:    smov w13, v0.h[4]
-; NONEON-NOSVE-NEXT:    smov w14, v0.h[5]
-; NONEON-NOSVE-NEXT:    smov w15, v0.h[6]
-; NONEON-NOSVE-NEXT:    smov w16, v2.h[1]
-; NONEON-NOSVE-NEXT:    smov w17, v2.h[0]
-; NONEON-NOSVE-NEXT:    smov w18, v2.h[2]
-; NONEON-NOSVE-NEXT:    smov w1, v2.h[3]
-; NONEON-NOSVE-NEXT:    sdiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    smov w9, v1.h[0]
-; NONEON-NOSVE-NEXT:    smov w2, v2.h[4]
-; NONEON-NOSVE-NEXT:    smov w3, v2.h[5]
-; NONEON-NOSVE-NEXT:    smov w4, v2.h[6]
-; NONEON-NOSVE-NEXT:    sdiv w10, w10, w9
-; NONEON-NOSVE-NEXT:    smov w9, v1.h[2]
-; NONEON-NOSVE-NEXT:    sdiv w9, w11, w9
-; NONEON-NOSVE-NEXT:    smov w11, v1.h[3]
-; NONEON-NOSVE-NEXT:    fmov s5, w10
-; NONEON-NOSVE-NEXT:    smov w10, v3.h[7]
-; NONEON-NOSVE-NEXT:    mov v5.h[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    smov w12, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov v5.h[2], w9
-; NONEON-NOSVE-NEXT:    smov w9, v2.h[7]
-; NONEON-NOSVE-NEXT:    sdiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    smov w13, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov v5.h[3], w11
-; NONEON-NOSVE-NEXT:    smov w11, v0.h[7]
-; NONEON-NOSVE-NEXT:    sdiv w13, w14, w13
-; NONEON-NOSVE-NEXT:    smov w14, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v5.h[4], w12
-; NONEON-NOSVE-NEXT:    sdiv w14, w15, w14
-; NONEON-NOSVE-NEXT:    smov w15, v3.h[1]
-; NONEON-NOSVE-NEXT:    mov v5.h[5], w13
-; NONEON-NOSVE-NEXT:    sdiv w15, w16, w15
-; NONEON-NOSVE-NEXT:    smov w16, v3.h[0]
-; NONEON-NOSVE-NEXT:    mov v5.h[6], w14
-; NONEON-NOSVE-NEXT:    sdiv w16, w17, w16
-; NONEON-NOSVE-NEXT:    smov w17, v3.h[2]
-; NONEON-NOSVE-NEXT:    sdiv w17, w18, w17
-; NONEON-NOSVE-NEXT:    smov w18, v3.h[3]
-; NONEON-NOSVE-NEXT:    fmov s4, w16
-; NONEON-NOSVE-NEXT:    mov v4.h[1], w15
-; NONEON-NOSVE-NEXT:    sdiv w18, w1, w18
-; NONEON-NOSVE-NEXT:    smov w1, v3.h[4]
-; NONEON-NOSVE-NEXT:    mov v4.h[2], w17
-; NONEON-NOSVE-NEXT:    sdiv w1, w2, w1
-; NONEON-NOSVE-NEXT:    smov w2, v3.h[5]
-; NONEON-NOSVE-NEXT:    mov v4.h[3], w18
-; NONEON-NOSVE-NEXT:    sdiv w2, w3, w2
-; NONEON-NOSVE-NEXT:    smov w3, v3.h[6]
-; NONEON-NOSVE-NEXT:    mov v4.h[4], w1
-; NONEON-NOSVE-NEXT:    sdiv w8, w4, w3
-; NONEON-NOSVE-NEXT:    mov v4.h[5], w2
-; NONEON-NOSVE-NEXT:    sdiv w9, w9, w10
-; NONEON-NOSVE-NEXT:    smov w10, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov v4.h[6], w8
-; NONEON-NOSVE-NEXT:    sdiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    mov v4.h[7], w9
-; NONEON-NOSVE-NEXT:    mov v5.h[7], w10
-; NONEON-NOSVE-NEXT:    stp q4, q5, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = sdiv <16 x i16> %op1, %op2
@@ -735,21 +294,6 @@ define <2 x i32> @sdiv_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    fmov w9, s0
-; NONEON-NOSVE-NEXT:    mov w10, v0.s[1]
-; NONEON-NOSVE-NEXT:    sdiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    mov w9, v1.s[1]
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w9
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -763,26 +307,6 @@ define <4 x i32> @sdiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    sdiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, v1.s[1]
-; NONEON-NOSVE-NEXT:    mov w9, v0.s[1]
-; NONEON-NOSVE-NEXT:    fmov w10, s0
-; NONEON-NOSVE-NEXT:    mov w11, v0.s[2]
-; NONEON-NOSVE-NEXT:    mov w12, v0.s[3]
-; NONEON-NOSVE-NEXT:    sdiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    fmov w9, s1
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    mov w10, v1.s[2]
-; NONEON-NOSVE-NEXT:    sdiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    mov w11, v1.s[3]
-; NONEON-NOSVE-NEXT:    fmov s0, w9
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w8, w12, w11
-; NONEON-NOSVE-NEXT:    mov v0.s[2], w10
-; NONEON-NOSVE-NEXT:    mov v0.s[3], w8
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -798,45 +322,6 @@ define void @sdiv_v8i32(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    sdiv z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    mov w9, v0.s[1]
-; NONEON-NOSVE-NEXT:    fmov w10, s0
-; NONEON-NOSVE-NEXT:    mov w11, v0.s[2]
-; NONEON-NOSVE-NEXT:    mov w8, v1.s[1]
-; NONEON-NOSVE-NEXT:    mov w12, v2.s[1]
-; NONEON-NOSVE-NEXT:    fmov w13, s2
-; NONEON-NOSVE-NEXT:    mov w14, v2.s[2]
-; NONEON-NOSVE-NEXT:    mov w15, v2.s[3]
-; NONEON-NOSVE-NEXT:    mov w16, v0.s[3]
-; NONEON-NOSVE-NEXT:    sdiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    fmov w9, s1
-; NONEON-NOSVE-NEXT:    sdiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    mov w10, v1.s[2]
-; NONEON-NOSVE-NEXT:    sdiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    mov w11, v3.s[1]
-; NONEON-NOSVE-NEXT:    sdiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    fmov w12, s3
-; NONEON-NOSVE-NEXT:    sdiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    mov w13, v3.s[2]
-; NONEON-NOSVE-NEXT:    sdiv w13, w14, w13
-; NONEON-NOSVE-NEXT:    mov w14, v3.s[3]
-; NONEON-NOSVE-NEXT:    fmov s0, w12
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w11
-; NONEON-NOSVE-NEXT:    sdiv w14, w15, w14
-; NONEON-NOSVE-NEXT:    mov w15, v1.s[3]
-; NONEON-NOSVE-NEXT:    fmov s1, w9
-; NONEON-NOSVE-NEXT:    mov v0.s[2], w13
-; NONEON-NOSVE-NEXT:    mov v1.s[1], w8
-; NONEON-NOSVE-NEXT:    mov v1.s[2], w10
-; NONEON-NOSVE-NEXT:    sdiv w8, w16, w15
-; NONEON-NOSVE-NEXT:    mov v0.s[3], w14
-; NONEON-NOSVE-NEXT:    mov v1.s[3], w8
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = sdiv <8 x i32> %op1, %op2
@@ -853,16 +338,6 @@ define <1 x i64> @sdiv_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    sdiv z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    sdiv x8, x9, x8
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -876,18 +351,6 @@ define <2 x i64> @sdiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    sdiv z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    mov x10, v0.d[1]
-; NONEON-NOSVE-NEXT:    sdiv x8, x9, x8
-; NONEON-NOSVE-NEXT:    mov x9, v1.d[1]
-; NONEON-NOSVE-NEXT:    sdiv x9, x10, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x9
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -903,29 +366,6 @@ define void @sdiv_v4i64(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    sdiv z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    mov x10, v2.d[1]
-; NONEON-NOSVE-NEXT:    fmov x11, d2
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    mov x12, v0.d[1]
-; NONEON-NOSVE-NEXT:    sdiv x8, x9, x8
-; NONEON-NOSVE-NEXT:    mov x9, v3.d[1]
-; NONEON-NOSVE-NEXT:    sdiv x9, x10, x9
-; NONEON-NOSVE-NEXT:    fmov x10, d3
-; NONEON-NOSVE-NEXT:    sdiv x10, x11, x10
-; NONEON-NOSVE-NEXT:    mov x11, v1.d[1]
-; NONEON-NOSVE-NEXT:    fmov d1, x8
-; NONEON-NOSVE-NEXT:    sdiv x11, x12, x11
-; NONEON-NOSVE-NEXT:    fmov d0, x10
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x9
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x11
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = sdiv <4 x i64> %op1, %op2
@@ -951,37 +391,6 @@ define <4 x i8> @udiv_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    umov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    umov w10, v0.h[0]
-; NONEON-NOSVE-NEXT:    umov w11, v0.h[2]
-; NONEON-NOSVE-NEXT:    umov w12, v0.h[3]
-; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
-; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
-; NONEON-NOSVE-NEXT:    and w10, w10, #0xff
-; NONEON-NOSVE-NEXT:    udiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    umov w9, v1.h[0]
-; NONEON-NOSVE-NEXT:    and w11, w11, #0xff
-; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.h[2]
-; NONEON-NOSVE-NEXT:    and w10, w10, #0xff
-; NONEON-NOSVE-NEXT:    udiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    umov w11, v1.h[3]
-; NONEON-NOSVE-NEXT:    fmov s0, w9
-; NONEON-NOSVE-NEXT:    mov v0.h[1], w8
-; NONEON-NOSVE-NEXT:    and w9, w11, #0xff
-; NONEON-NOSVE-NEXT:    and w11, w12, #0xff
-; NONEON-NOSVE-NEXT:    udiv w8, w11, w9
-; NONEON-NOSVE-NEXT:    mov v0.h[2], w10
-; NONEON-NOSVE-NEXT:    mov v0.h[3], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = udiv <4 x i8> %op1, %op2
   ret <4 x i8> %res
 }
@@ -1009,45 +418,6 @@ define <8 x i8> @udiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    uzp1 z0.b, z1.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    umov w10, v0.b[0]
-; NONEON-NOSVE-NEXT:    umov w11, v0.b[2]
-; NONEON-NOSVE-NEXT:    umov w12, v0.b[3]
-; NONEON-NOSVE-NEXT:    umov w13, v0.b[4]
-; NONEON-NOSVE-NEXT:    umov w14, v0.b[5]
-; NONEON-NOSVE-NEXT:    udiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    umov w9, v1.b[0]
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.b[2]
-; NONEON-NOSVE-NEXT:    udiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    umov w11, v1.b[3]
-; NONEON-NOSVE-NEXT:    fmov s2, w9
-; NONEON-NOSVE-NEXT:    umov w9, v1.b[6]
-; NONEON-NOSVE-NEXT:    mov v2.b[1], w8
-; NONEON-NOSVE-NEXT:    udiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    umov w12, v1.b[4]
-; NONEON-NOSVE-NEXT:    mov v2.b[2], w10
-; NONEON-NOSVE-NEXT:    umov w10, v0.b[6]
-; NONEON-NOSVE-NEXT:    udiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    umov w13, v1.b[5]
-; NONEON-NOSVE-NEXT:    mov v2.b[3], w11
-; NONEON-NOSVE-NEXT:    umov w11, v0.b[7]
-; NONEON-NOSVE-NEXT:    udiv w8, w14, w13
-; NONEON-NOSVE-NEXT:    mov v2.b[4], w12
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.b[7]
-; NONEON-NOSVE-NEXT:    mov v2.b[5], w8
-; NONEON-NOSVE-NEXT:    udiv w8, w11, w10
-; NONEON-NOSVE-NEXT:    mov v2.b[6], w9
-; NONEON-NOSVE-NEXT:    mov v2.b[7], w8
-; NONEON-NOSVE-NEXT:    fmov d0, d2
-; NONEON-NOSVE-NEXT:    ret
   %res = udiv <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -1095,75 +465,6 @@ define <16 x i8> @udiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    umov w10, v0.b[0]
-; NONEON-NOSVE-NEXT:    umov w11, v0.b[2]
-; NONEON-NOSVE-NEXT:    umov w12, v0.b[3]
-; NONEON-NOSVE-NEXT:    umov w13, v0.b[4]
-; NONEON-NOSVE-NEXT:    umov w14, v0.b[5]
-; NONEON-NOSVE-NEXT:    umov w15, v0.b[6]
-; NONEON-NOSVE-NEXT:    umov w16, v0.b[7]
-; NONEON-NOSVE-NEXT:    umov w17, v0.b[8]
-; NONEON-NOSVE-NEXT:    umov w18, v0.b[9]
-; NONEON-NOSVE-NEXT:    udiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    umov w9, v1.b[0]
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.b[2]
-; NONEON-NOSVE-NEXT:    udiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    umov w11, v1.b[3]
-; NONEON-NOSVE-NEXT:    fmov s2, w9
-; NONEON-NOSVE-NEXT:    umov w9, v1.b[10]
-; NONEON-NOSVE-NEXT:    mov v2.b[1], w8
-; NONEON-NOSVE-NEXT:    udiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    umov w12, v1.b[4]
-; NONEON-NOSVE-NEXT:    mov v2.b[2], w10
-; NONEON-NOSVE-NEXT:    umov w10, v0.b[10]
-; NONEON-NOSVE-NEXT:    udiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    umov w13, v1.b[5]
-; NONEON-NOSVE-NEXT:    mov v2.b[3], w11
-; NONEON-NOSVE-NEXT:    umov w11, v0.b[11]
-; NONEON-NOSVE-NEXT:    udiv w13, w14, w13
-; NONEON-NOSVE-NEXT:    umov w14, v1.b[6]
-; NONEON-NOSVE-NEXT:    mov v2.b[4], w12
-; NONEON-NOSVE-NEXT:    umov w12, v0.b[12]
-; NONEON-NOSVE-NEXT:    udiv w14, w15, w14
-; NONEON-NOSVE-NEXT:    umov w15, v1.b[7]
-; NONEON-NOSVE-NEXT:    mov v2.b[5], w13
-; NONEON-NOSVE-NEXT:    umov w13, v0.b[13]
-; NONEON-NOSVE-NEXT:    udiv w15, w16, w15
-; NONEON-NOSVE-NEXT:    umov w16, v1.b[8]
-; NONEON-NOSVE-NEXT:    mov v2.b[6], w14
-; NONEON-NOSVE-NEXT:    udiv w16, w17, w16
-; NONEON-NOSVE-NEXT:    umov w17, v1.b[9]
-; NONEON-NOSVE-NEXT:    mov v2.b[7], w15
-; NONEON-NOSVE-NEXT:    udiv w8, w18, w17
-; NONEON-NOSVE-NEXT:    mov v2.b[8], w16
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.b[11]
-; NONEON-NOSVE-NEXT:    mov v2.b[9], w8
-; NONEON-NOSVE-NEXT:    udiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    umov w11, v1.b[12]
-; NONEON-NOSVE-NEXT:    mov v2.b[10], w9
-; NONEON-NOSVE-NEXT:    umov w9, v1.b[14]
-; NONEON-NOSVE-NEXT:    udiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    umov w12, v1.b[13]
-; NONEON-NOSVE-NEXT:    mov v2.b[11], w10
-; NONEON-NOSVE-NEXT:    umov w10, v1.b[15]
-; NONEON-NOSVE-NEXT:    udiv w8, w13, w12
-; NONEON-NOSVE-NEXT:    umov w12, v0.b[14]
-; NONEON-NOSVE-NEXT:    mov v2.b[12], w11
-; NONEON-NOSVE-NEXT:    umov w11, v0.b[15]
-; NONEON-NOSVE-NEXT:    udiv w9, w12, w9
-; NONEON-NOSVE-NEXT:    mov v2.b[13], w8
-; NONEON-NOSVE-NEXT:    udiv w8, w11, w10
-; NONEON-NOSVE-NEXT:    mov v2.b[14], w9
-; NONEON-NOSVE-NEXT:    mov v2.b[15], w8
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = udiv <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -1244,163 +545,6 @@ define void @udiv_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    splice z3.b, p0, z3.b, z1.b
 ; CHECK-NEXT:    stp q3, q2, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str x27, [sp, #-80]! // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #16] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #32] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
-; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
-; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
-; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
-; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
-; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
-; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
-; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
-; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
-; NONEON-NOSVE-NEXT:    .cfi_offset w27, -80
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0]
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    umov w10, v0.b[0]
-; NONEON-NOSVE-NEXT:    umov w11, v0.b[2]
-; NONEON-NOSVE-NEXT:    umov w12, v0.b[3]
-; NONEON-NOSVE-NEXT:    umov w13, v0.b[4]
-; NONEON-NOSVE-NEXT:    umov w14, v0.b[5]
-; NONEON-NOSVE-NEXT:    umov w15, v0.b[6]
-; NONEON-NOSVE-NEXT:    umov w17, v0.b[8]
-; NONEON-NOSVE-NEXT:    umov w2, v0.b[10]
-; NONEON-NOSVE-NEXT:    umov w3, v0.b[11]
-; NONEON-NOSVE-NEXT:    umov w4, v0.b[12]
-; NONEON-NOSVE-NEXT:    udiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    umov w9, v1.b[0]
-; NONEON-NOSVE-NEXT:    umov w5, v0.b[13]
-; NONEON-NOSVE-NEXT:    umov w6, v0.b[14]
-; NONEON-NOSVE-NEXT:    umov w1, v3.b[1]
-; NONEON-NOSVE-NEXT:    umov w7, v2.b[0]
-; NONEON-NOSVE-NEXT:    umov w19, v2.b[2]
-; NONEON-NOSVE-NEXT:    umov w20, v2.b[3]
-; NONEON-NOSVE-NEXT:    umov w21, v2.b[4]
-; NONEON-NOSVE-NEXT:    umov w22, v2.b[5]
-; NONEON-NOSVE-NEXT:    umov w23, v2.b[6]
-; NONEON-NOSVE-NEXT:    umov w24, v2.b[7]
-; NONEON-NOSVE-NEXT:    umov w25, v2.b[8]
-; NONEON-NOSVE-NEXT:    umov w26, v2.b[9]
-; NONEON-NOSVE-NEXT:    umov w27, v2.b[10]
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.b[2]
-; NONEON-NOSVE-NEXT:    udiv w11, w11, w10
-; NONEON-NOSVE-NEXT:    umov w10, v1.b[3]
-; NONEON-NOSVE-NEXT:    fmov s5, w9
-; NONEON-NOSVE-NEXT:    umov w9, v3.b[11]
-; NONEON-NOSVE-NEXT:    mov v5.b[1], w8
-; NONEON-NOSVE-NEXT:    udiv w10, w12, w10
-; NONEON-NOSVE-NEXT:    umov w12, v1.b[4]
-; NONEON-NOSVE-NEXT:    mov v5.b[2], w11
-; NONEON-NOSVE-NEXT:    umov w11, v2.b[11]
-; NONEON-NOSVE-NEXT:    udiv w13, w13, w12
-; NONEON-NOSVE-NEXT:    umov w12, v1.b[5]
-; NONEON-NOSVE-NEXT:    mov v5.b[3], w10
-; NONEON-NOSVE-NEXT:    umov w10, v3.b[12]
-; NONEON-NOSVE-NEXT:    udiv w12, w14, w12
-; NONEON-NOSVE-NEXT:    umov w14, v1.b[6]
-; NONEON-NOSVE-NEXT:    mov v5.b[4], w13
-; NONEON-NOSVE-NEXT:    umov w13, v2.b[14]
-; NONEON-NOSVE-NEXT:    udiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    umov w14, v1.b[7]
-; NONEON-NOSVE-NEXT:    umov w15, v0.b[7]
-; NONEON-NOSVE-NEXT:    mov v5.b[5], w12
-; NONEON-NOSVE-NEXT:    umov w12, v2.b[13]
-; NONEON-NOSVE-NEXT:    udiv w14, w15, w14
-; NONEON-NOSVE-NEXT:    umov w15, v1.b[8]
-; NONEON-NOSVE-NEXT:    mov v5.b[6], w16
-; NONEON-NOSVE-NEXT:    udiv w18, w17, w15
-; NONEON-NOSVE-NEXT:    umov w15, v1.b[9]
-; NONEON-NOSVE-NEXT:    umov w17, v0.b[9]
-; NONEON-NOSVE-NEXT:    mov v5.b[7], w14
-; NONEON-NOSVE-NEXT:    udiv w17, w17, w15
-; NONEON-NOSVE-NEXT:    umov w15, v1.b[10]
-; NONEON-NOSVE-NEXT:    mov v5.b[8], w18
-; NONEON-NOSVE-NEXT:    udiv w15, w2, w15
-; NONEON-NOSVE-NEXT:    umov w2, v1.b[11]
-; NONEON-NOSVE-NEXT:    mov v5.b[9], w17
-; NONEON-NOSVE-NEXT:    udiv w2, w3, w2
-; NONEON-NOSVE-NEXT:    umov w3, v1.b[12]
-; NONEON-NOSVE-NEXT:    mov v5.b[10], w15
-; NONEON-NOSVE-NEXT:    udiv w3, w4, w3
-; NONEON-NOSVE-NEXT:    umov w4, v1.b[13]
-; NONEON-NOSVE-NEXT:    mov v5.b[11], w2
-; NONEON-NOSVE-NEXT:    udiv w4, w5, w4
-; NONEON-NOSVE-NEXT:    umov w5, v1.b[14]
-; NONEON-NOSVE-NEXT:    mov v5.b[12], w3
-; NONEON-NOSVE-NEXT:    udiv w5, w6, w5
-; NONEON-NOSVE-NEXT:    umov w6, v2.b[1]
-; NONEON-NOSVE-NEXT:    mov v5.b[13], w4
-; NONEON-NOSVE-NEXT:    udiv w1, w6, w1
-; NONEON-NOSVE-NEXT:    umov w6, v3.b[0]
-; NONEON-NOSVE-NEXT:    mov v5.b[14], w5
-; NONEON-NOSVE-NEXT:    udiv w6, w7, w6
-; NONEON-NOSVE-NEXT:    umov w7, v3.b[2]
-; NONEON-NOSVE-NEXT:    udiv w7, w19, w7
-; NONEON-NOSVE-NEXT:    umov w19, v3.b[3]
-; NONEON-NOSVE-NEXT:    fmov s4, w6
-; NONEON-NOSVE-NEXT:    mov v4.b[1], w1
-; NONEON-NOSVE-NEXT:    udiv w19, w20, w19
-; NONEON-NOSVE-NEXT:    umov w20, v3.b[4]
-; NONEON-NOSVE-NEXT:    mov v4.b[2], w7
-; NONEON-NOSVE-NEXT:    udiv w20, w21, w20
-; NONEON-NOSVE-NEXT:    umov w21, v3.b[5]
-; NONEON-NOSVE-NEXT:    mov v4.b[3], w19
-; NONEON-NOSVE-NEXT:    udiv w21, w22, w21
-; NONEON-NOSVE-NEXT:    umov w22, v3.b[6]
-; NONEON-NOSVE-NEXT:    mov v4.b[4], w20
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w22, w23, w22
-; NONEON-NOSVE-NEXT:    umov w23, v3.b[7]
-; NONEON-NOSVE-NEXT:    mov v4.b[5], w21
-; NONEON-NOSVE-NEXT:    udiv w23, w24, w23
-; NONEON-NOSVE-NEXT:    umov w24, v3.b[8]
-; NONEON-NOSVE-NEXT:    mov v4.b[6], w22
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w24, w25, w24
-; NONEON-NOSVE-NEXT:    umov w25, v3.b[9]
-; NONEON-NOSVE-NEXT:    mov v4.b[7], w23
-; NONEON-NOSVE-NEXT:    udiv w25, w26, w25
-; NONEON-NOSVE-NEXT:    umov w26, v3.b[10]
-; NONEON-NOSVE-NEXT:    mov v4.b[8], w24
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #32] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w8, w27, w26
-; NONEON-NOSVE-NEXT:    mov v4.b[9], w25
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #16] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w9, w11, w9
-; NONEON-NOSVE-NEXT:    umov w11, v2.b[12]
-; NONEON-NOSVE-NEXT:    mov v4.b[10], w8
-; NONEON-NOSVE-NEXT:    umov w8, v3.b[15]
-; NONEON-NOSVE-NEXT:    udiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    umov w11, v3.b[13]
-; NONEON-NOSVE-NEXT:    mov v4.b[11], w9
-; NONEON-NOSVE-NEXT:    umov w9, v1.b[15]
-; NONEON-NOSVE-NEXT:    udiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    umov w12, v3.b[14]
-; NONEON-NOSVE-NEXT:    mov v4.b[12], w10
-; NONEON-NOSVE-NEXT:    umov w10, v0.b[15]
-; NONEON-NOSVE-NEXT:    udiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    umov w13, v2.b[15]
-; NONEON-NOSVE-NEXT:    mov v4.b[13], w11
-; NONEON-NOSVE-NEXT:    udiv w8, w13, w8
-; NONEON-NOSVE-NEXT:    mov v4.b[14], w12
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    mov v4.b[15], w8
-; NONEON-NOSVE-NEXT:    mov v5.b[15], w9
-; NONEON-NOSVE-NEXT:    stp q4, q5, [x0]
-; NONEON-NOSVE-NEXT:    ldr x27, [sp], #80 // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = udiv <32 x i8> %op1, %op2
@@ -1419,22 +563,6 @@ define <2 x i16> @udiv_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; CHECK-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d2, #0x00ffff0000ffff
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v2.8b
-; NONEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    fmov w9, s0
-; NONEON-NOSVE-NEXT:    mov w10, v0.s[1]
-; NONEON-NOSVE-NEXT:    udiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    mov w9, v1.s[1]
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w9
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = udiv <2 x i16> %op1, %op2
   ret <2 x i16> %res
 }
@@ -1451,29 +579,6 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    umov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    umov w10, v0.h[0]
-; NONEON-NOSVE-NEXT:    umov w11, v0.h[2]
-; NONEON-NOSVE-NEXT:    umov w12, v0.h[3]
-; NONEON-NOSVE-NEXT:    udiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    umov w9, v1.h[0]
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.h[2]
-; NONEON-NOSVE-NEXT:    udiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    umov w11, v1.h[3]
-; NONEON-NOSVE-NEXT:    fmov s0, w9
-; NONEON-NOSVE-NEXT:    mov v0.h[1], w8
-; NONEON-NOSVE-NEXT:    udiv w8, w12, w11
-; NONEON-NOSVE-NEXT:    mov v0.h[2], w10
-; NONEON-NOSVE-NEXT:    mov v0.h[3], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = udiv <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -1500,43 +605,6 @@ define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    umov w10, v0.h[0]
-; NONEON-NOSVE-NEXT:    umov w11, v0.h[2]
-; NONEON-NOSVE-NEXT:    umov w12, v0.h[3]
-; NONEON-NOSVE-NEXT:    umov w13, v0.h[4]
-; NONEON-NOSVE-NEXT:    umov w14, v0.h[5]
-; NONEON-NOSVE-NEXT:    udiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    umov w9, v1.h[0]
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.h[2]
-; NONEON-NOSVE-NEXT:    udiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    umov w11, v1.h[3]
-; NONEON-NOSVE-NEXT:    fmov s2, w9
-; NONEON-NOSVE-NEXT:    umov w9, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w8
-; NONEON-NOSVE-NEXT:    udiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    umov w12, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w10
-; NONEON-NOSVE-NEXT:    umov w10, v0.h[6]
-; NONEON-NOSVE-NEXT:    udiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    umov w13, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w11
-; NONEON-NOSVE-NEXT:    umov w11, v0.h[7]
-; NONEON-NOSVE-NEXT:    udiv w8, w14, w13
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w12
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    udiv w8, w11, w10
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w9
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = udiv <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -1577,79 +645,6 @@ define void @udiv_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    splice z3.h, p0, z3.h, z1.h
 ; CHECK-NEXT:    stp q3, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0]
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    umov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    umov w10, v0.h[0]
-; NONEON-NOSVE-NEXT:    umov w11, v0.h[2]
-; NONEON-NOSVE-NEXT:    umov w12, v0.h[3]
-; NONEON-NOSVE-NEXT:    umov w13, v0.h[4]
-; NONEON-NOSVE-NEXT:    umov w14, v0.h[5]
-; NONEON-NOSVE-NEXT:    umov w15, v0.h[6]
-; NONEON-NOSVE-NEXT:    umov w16, v2.h[1]
-; NONEON-NOSVE-NEXT:    umov w17, v2.h[0]
-; NONEON-NOSVE-NEXT:    umov w18, v2.h[2]
-; NONEON-NOSVE-NEXT:    umov w1, v2.h[3]
-; NONEON-NOSVE-NEXT:    udiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    umov w9, v1.h[0]
-; NONEON-NOSVE-NEXT:    umov w2, v2.h[4]
-; NONEON-NOSVE-NEXT:    umov w3, v2.h[5]
-; NONEON-NOSVE-NEXT:    umov w4, v2.h[6]
-; NONEON-NOSVE-NEXT:    udiv w10, w10, w9
-; NONEON-NOSVE-NEXT:    umov w9, v1.h[2]
-; NONEON-NOSVE-NEXT:    udiv w9, w11, w9
-; NONEON-NOSVE-NEXT:    umov w11, v1.h[3]
-; NONEON-NOSVE-NEXT:    fmov s5, w10
-; NONEON-NOSVE-NEXT:    umov w10, v3.h[7]
-; NONEON-NOSVE-NEXT:    mov v5.h[1], w8
-; NONEON-NOSVE-NEXT:    udiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    umov w12, v1.h[4]
-; NONEON-NOSVE-NEXT:    mov v5.h[2], w9
-; NONEON-NOSVE-NEXT:    umov w9, v2.h[7]
-; NONEON-NOSVE-NEXT:    udiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    umov w13, v1.h[5]
-; NONEON-NOSVE-NEXT:    mov v5.h[3], w11
-; NONEON-NOSVE-NEXT:    umov w11, v0.h[7]
-; NONEON-NOSVE-NEXT:    udiv w13, w14, w13
-; NONEON-NOSVE-NEXT:    umov w14, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v5.h[4], w12
-; NONEON-NOSVE-NEXT:    udiv w14, w15, w14
-; NONEON-NOSVE-NEXT:    umov w15, v3.h[1]
-; NONEON-NOSVE-NEXT:    mov v5.h[5], w13
-; NONEON-NOSVE-NEXT:    udiv w15, w16, w15
-; NONEON-NOSVE-NEXT:    umov w16, v3.h[0]
-; NONEON-NOSVE-NEXT:    mov v5.h[6], w14
-; NONEON-NOSVE-NEXT:    udiv w16, w17, w16
-; NONEON-NOSVE-NEXT:    umov w17, v3.h[2]
-; NONEON-NOSVE-NEXT:    udiv w17, w18, w17
-; NONEON-NOSVE-NEXT:    umov w18, v3.h[3]
-; NONEON-NOSVE-NEXT:    fmov s4, w16
-; NONEON-NOSVE-NEXT:    mov v4.h[1], w15
-; NONEON-NOSVE-NEXT:    udiv w18, w1, w18
-; NONEON-NOSVE-NEXT:    umov w1, v3.h[4]
-; NONEON-NOSVE-NEXT:    mov v4.h[2], w17
-; NONEON-NOSVE-NEXT:    udiv w1, w2, w1
-; NONEON-NOSVE-NEXT:    umov w2, v3.h[5]
-; NONEON-NOSVE-NEXT:    mov v4.h[3], w18
-; NONEON-NOSVE-NEXT:    udiv w2, w3, w2
-; NONEON-NOSVE-NEXT:    umov w3, v3.h[6]
-; NONEON-NOSVE-NEXT:    mov v4.h[4], w1
-; NONEON-NOSVE-NEXT:    udiv w8, w4, w3
-; NONEON-NOSVE-NEXT:    mov v4.h[5], w2
-; NONEON-NOSVE-NEXT:    udiv w9, w9, w10
-; NONEON-NOSVE-NEXT:    umov w10, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov v4.h[6], w8
-; NONEON-NOSVE-NEXT:    udiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    mov v4.h[7], w9
-; NONEON-NOSVE-NEXT:    mov v5.h[7], w10
-; NONEON-NOSVE-NEXT:    stp q4, q5, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = udiv <16 x i16> %op1, %op2
@@ -1666,21 +661,6 @@ define <2 x i32> @udiv_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    fmov w9, s0
-; NONEON-NOSVE-NEXT:    mov w10, v0.s[1]
-; NONEON-NOSVE-NEXT:    udiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    mov w9, v1.s[1]
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w9
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = udiv <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -1694,26 +674,6 @@ define <4 x i32> @udiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    udiv z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, v1.s[1]
-; NONEON-NOSVE-NEXT:    mov w9, v0.s[1]
-; NONEON-NOSVE-NEXT:    fmov w10, s0
-; NONEON-NOSVE-NEXT:    mov w11, v0.s[2]
-; NONEON-NOSVE-NEXT:    mov w12, v0.s[3]
-; NONEON-NOSVE-NEXT:    udiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    fmov w9, s1
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    mov w10, v1.s[2]
-; NONEON-NOSVE-NEXT:    udiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    mov w11, v1.s[3]
-; NONEON-NOSVE-NEXT:    fmov s0, w9
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w8
-; NONEON-NOSVE-NEXT:    udiv w8, w12, w11
-; NONEON-NOSVE-NEXT:    mov v0.s[2], w10
-; NONEON-NOSVE-NEXT:    mov v0.s[3], w8
-; NONEON-NOSVE-NEXT:    ret
   %res = udiv <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -1729,45 +689,6 @@ define void @udiv_v8i32(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    udiv z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    mov w9, v0.s[1]
-; NONEON-NOSVE-NEXT:    fmov w10, s0
-; NONEON-NOSVE-NEXT:    mov w11, v0.s[2]
-; NONEON-NOSVE-NEXT:    mov w8, v1.s[1]
-; NONEON-NOSVE-NEXT:    mov w12, v2.s[1]
-; NONEON-NOSVE-NEXT:    fmov w13, s2
-; NONEON-NOSVE-NEXT:    mov w14, v2.s[2]
-; NONEON-NOSVE-NEXT:    mov w15, v2.s[3]
-; NONEON-NOSVE-NEXT:    mov w16, v0.s[3]
-; NONEON-NOSVE-NEXT:    udiv w8, w9, w8
-; NONEON-NOSVE-NEXT:    fmov w9, s1
-; NONEON-NOSVE-NEXT:    udiv w9, w10, w9
-; NONEON-NOSVE-NEXT:    mov w10, v1.s[2]
-; NONEON-NOSVE-NEXT:    udiv w10, w11, w10
-; NONEON-NOSVE-NEXT:    mov w11, v3.s[1]
-; NONEON-NOSVE-NEXT:    udiv w11, w12, w11
-; NONEON-NOSVE-NEXT:    fmov w12, s3
-; NONEON-NOSVE-NEXT:    udiv w12, w13, w12
-; NONEON-NOSVE-NEXT:    mov w13, v3.s[2]
-; NONEON-NOSVE-NEXT:    udiv w13, w14, w13
-; NONEON-NOSVE-NEXT:    mov w14, v3.s[3]
-; NONEON-NOSVE-NEXT:    fmov s0, w12
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w11
-; NONEON-NOSVE-NEXT:    udiv w14, w15, w14
-; NONEON-NOSVE-NEXT:    mov w15, v1.s[3]
-; NONEON-NOSVE-NEXT:    fmov s1, w9
-; NONEON-NOSVE-NEXT:    mov v0.s[2], w13
-; NONEON-NOSVE-NEXT:    mov v1.s[1], w8
-; NONEON-NOSVE-NEXT:    mov v1.s[2], w10
-; NONEON-NOSVE-NEXT:    udiv w8, w16, w15
-; NONEON-NOSVE-NEXT:    mov v0.s[3], w14
-; NONEON-NOSVE-NEXT:    mov v1.s[3], w8
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = udiv <8 x i32> %op1, %op2
@@ -1784,16 +705,6 @@ define <1 x i64> @udiv_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    udiv z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    udiv x8, x9, x8
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %res = udiv <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -1807,18 +718,6 @@ define <2 x i64> @udiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    udiv z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    mov x10, v0.d[1]
-; NONEON-NOSVE-NEXT:    udiv x8, x9, x8
-; NONEON-NOSVE-NEXT:    mov x9, v1.d[1]
-; NONEON-NOSVE-NEXT:    udiv x9, x10, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x9
-; NONEON-NOSVE-NEXT:    ret
   %res = udiv <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -1834,29 +733,6 @@ define void @udiv_v4i64(ptr %a, ptr %b)  {
 ; CHECK-NEXT:    udiv z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    mov x10, v2.d[1]
-; NONEON-NOSVE-NEXT:    fmov x11, d2
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    mov x12, v0.d[1]
-; NONEON-NOSVE-NEXT:    udiv x8, x9, x8
-; NONEON-NOSVE-NEXT:    mov x9, v3.d[1]
-; NONEON-NOSVE-NEXT:    udiv x9, x10, x9
-; NONEON-NOSVE-NEXT:    fmov x10, d3
-; NONEON-NOSVE-NEXT:    udiv x10, x11, x10
-; NONEON-NOSVE-NEXT:    mov x11, v1.d[1]
-; NONEON-NOSVE-NEXT:    fmov d1, x8
-; NONEON-NOSVE-NEXT:    udiv x11, x12, x11
-; NONEON-NOSVE-NEXT:    fmov d0, x10
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x9
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x11
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = udiv <4 x i64> %op1, %op2
@@ -1902,27 +778,6 @@ define void @udiv_constantsplat_v8i32(ptr %a)  {
 ; SVE2-NEXT:    lsr z0.s, z0.s, #6
 ; SVE2-NEXT:    stp q1, q0, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: udiv_constantsplat_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #8969 // =0x2309
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    movk w8, #22765, lsl #16
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    umull2 v3.2d, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    umull v4.2d, v1.2s, v0.2s
-; NONEON-NOSVE-NEXT:    umull2 v5.2d, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    umull v0.2d, v2.2s, v0.2s
-; NONEON-NOSVE-NEXT:    uzp2 v3.4s, v4.4s, v3.4s
-; NONEON-NOSVE-NEXT:    uzp2 v0.4s, v0.4s, v5.4s
-; NONEON-NOSVE-NEXT:    sub v1.4s, v1.4s, v3.4s
-; NONEON-NOSVE-NEXT:    sub v2.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    usra v3.4s, v1.4s, #1
-; NONEON-NOSVE-NEXT:    usra v0.4s, v2.4s, #1
-; NONEON-NOSVE-NEXT:    ushr v1.4s, v3.4s, #6
-; NONEON-NOSVE-NEXT:    ushr v0.4s, v0.4s, #6
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %res = udiv <8 x i32> %op1, <i32 95, i32 95, i32 95, i32 95, i32 95, i32 95, i32 95, i32 95>
   store <8 x i32> %res, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
index e320fed2a498..c7a89612d278 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
@@ -2,7 +2,6 @@
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -27,22 +26,6 @@ define void @sext_v8i1_v8i32(<8 x i1> %a, ptr %out) {
 ; CHECK-NEXT:    asr z0.s, z0.s, #31
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v8i1_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    shl v0.4s, v0.4s, #31
-; NONEON-NOSVE-NEXT:    shl v1.4s, v1.4s, #31
-; NONEON-NOSVE-NEXT:    cmlt v0.4s, v0.4s, #0
-; NONEON-NOSVE-NEXT:    cmlt v1.4s, v1.4s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <8 x i1> %a to <8 x i32>
   store <8 x i32> %b, ptr %out
   ret void
@@ -69,22 +52,6 @@ define void @sext_v4i3_v4i64(<4 x i3> %a, ptr %out) {
 ; CHECK-NEXT:    asr z0.d, z0.d, #61
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v4i3_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    shl v0.2d, v0.2d, #61
-; NONEON-NOSVE-NEXT:    shl v1.2d, v1.2d, #61
-; NONEON-NOSVE-NEXT:    sshr v0.2d, v0.2d, #61
-; NONEON-NOSVE-NEXT:    sshr v1.2d, v1.2d, #61
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <4 x i3> %a to <4 x i64>
   store <4 x i64> %b, ptr %out
   ret void
@@ -103,17 +70,6 @@ define void @sext_v16i8_v16i16(<16 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    sunpklo z0.h, z0.b
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v16i8_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    sshll v1.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <16 x i8> %a to <16 x i16>
   store <16 x i16>%b, ptr %out
   ret void
@@ -135,24 +91,6 @@ define void @sext_v32i8_v32i16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v32i8_v32i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v1.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    sshll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    sshll v2.8h, v2.8b, #0
-; NONEON-NOSVE-NEXT:    sshll v3.8h, v3.8b, #0
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
   %c = sext <32 x i8> %b to <32 x i16>
@@ -174,18 +112,6 @@ define void @sext_v8i8_v8i32(<8 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v8i8_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sshll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <8 x i8> %a to <8 x i32>
   store <8 x i32>%b, ptr %out
   ret void
@@ -207,25 +133,6 @@ define void @sext_v16i8_v16i32(<16 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    stp q2, q1, [x0]
 ; CHECK-NEXT:    stp q3, q0, [x0, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v16i8_v16i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-48]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    sshll v1.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    sshll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <16 x i8> %a to <16 x i32>
   store <16 x i32> %b, ptr %out
   ret void
@@ -260,40 +167,6 @@ define void @sext_v32i8_v32i32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q6, q0, [x1, #96]
 ; CHECK-NEXT:    stp q7, q1, [x1, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v32i8_v32i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-96]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    sshll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    sshll v1.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    sshll v2.8h, v2.8b, #0
-; NONEON-NOSVE-NEXT:    sshll v3.8h, v3.8b, #0
-; NONEON-NOSVE-NEXT:    stp q2, q0, [sp, #32]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    stp q3, q1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldr d5, [sp, #56]
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #72]
-; NONEON-NOSVE-NEXT:    sshll v5.4s, v5.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v4.4s, v4.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q5, [x1]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v2.4s, v6.4h, #0
-; NONEON-NOSVE-NEXT:    stp q1, q4, [x1, #64]
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v3.4s, v7.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #96
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
   %c = sext <32 x i8> %b to <32 x i32>
@@ -321,22 +194,6 @@ define void @sext_v4i8_v4i64(<4 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    sxtb z0.d, p0/m, z0.d
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v4i8_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    shl v0.2d, v0.2d, #56
-; NONEON-NOSVE-NEXT:    shl v1.2d, v1.2d, #56
-; NONEON-NOSVE-NEXT:    sshr v0.2d, v0.2d, #56
-; NONEON-NOSVE-NEXT:    sshr v1.2d, v1.2d, #56
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <4 x i8> %a to <4 x i64>
   store <4 x i64>%b, ptr %out
   ret void
@@ -359,26 +216,6 @@ define void @sext_v8i8_v8i64(<8 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    stp q2, q1, [x0]
 ; CHECK-NEXT:    stp q3, q0, [x0, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v8i8_v8i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sshll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-48]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    sshll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <8 x i8> %a to <8 x i64>
   store <8 x i64>%b, ptr %out
   ret void
@@ -416,41 +253,6 @@ define void @sext_v16i8_v16i64(<16 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    stp q1, q4, [x0, #32]
 ; CHECK-NEXT:    stp q0, q2, [x0, #96]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v16i8_v16i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-112]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 112
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    sshll v1.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #40]
-; NONEON-NOSVE-NEXT:    sshll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    stp q2, q1, [sp, #48]
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    stp q3, q0, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldr d5, [sp, #72]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #88]
-; NONEON-NOSVE-NEXT:    sshll v5.2d, v5.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v4.2d, v4.2s, #0
-; NONEON-NOSVE-NEXT:    stp q1, q5, [x0, #64]
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v2.2d, v6.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q4, [x0]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v3.2d, v7.2s, #0
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x0, #96]
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #112
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <16 x i8> %a to <16 x i64>
   store <16 x i64> %b, ptr %out
   ret void
@@ -519,73 +321,6 @@ define void @sext_v32i8_v32i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q0, q2, [x1, #224]
 ; CHECK-NEXT:    stp q3, q1, [x1, #96]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v32i8_v32i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #224
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 224
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp]
-; NONEON-NOSVE-NEXT:    sshll v5.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    sshll v6.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v3.8h, v2.8b, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    sshll v4.8h, v2.8b, #0
-; NONEON-NOSVE-NEXT:    stp q3, q5, [sp, #32]
-; NONEON-NOSVE-NEXT:    sshll v5.4s, v5.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp q4, q6, [sp, #64]
-; NONEON-NOSVE-NEXT:    sshll v6.4s, v6.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v4.4s, v4.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #88]
-; NONEON-NOSVE-NEXT:    sshll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #72]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v7.4s, v7.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q2, q5, [sp, #128]
-; NONEON-NOSVE-NEXT:    sshll v5.2d, v5.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d19, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp q0, q3, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldr d20, [sp, #136]
-; NONEON-NOSVE-NEXT:    stp q1, q4, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldr d17, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldr d21, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp q7, q6, [sp, #192]
-; NONEON-NOSVE-NEXT:    sshll v6.2d, v6.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v19.2d, v19.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d16, [sp, #216]
-; NONEON-NOSVE-NEXT:    ldr d22, [sp, #200]
-; NONEON-NOSVE-NEXT:    ldr d23, [sp, #184]
-; NONEON-NOSVE-NEXT:    ldr d18, [sp, #168]
-; NONEON-NOSVE-NEXT:    sshll v4.2d, v4.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v16.2d, v16.2s, #0
-; NONEON-NOSVE-NEXT:    stp q5, q19, [x1]
-; NONEON-NOSVE-NEXT:    sshll v5.2d, v7.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v7.2d, v22.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    stp q6, q16, [x1, #128]
-; NONEON-NOSVE-NEXT:    sshll v6.2d, v23.2s, #0
-; NONEON-NOSVE-NEXT:    stp q5, q7, [x1, #160]
-; NONEON-NOSVE-NEXT:    sshll v5.2d, v20.2s, #0
-; NONEON-NOSVE-NEXT:    stp q4, q6, [x1, #192]
-; NONEON-NOSVE-NEXT:    sshll v4.2d, v21.2s, #0
-; NONEON-NOSVE-NEXT:    stp q2, q5, [x1, #32]
-; NONEON-NOSVE-NEXT:    sshll v2.2d, v17.2s, #0
-; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #64]
-; NONEON-NOSVE-NEXT:    sshll v3.2d, v18.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1, #96]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x1, #224]
-; NONEON-NOSVE-NEXT:    add sp, sp, #224
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
   %c = sext <32 x i8> %b to <32 x i64>
@@ -606,17 +341,6 @@ define void @sext_v8i16_v8i32(<8 x i16> %a, ptr %out) {
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v8i16_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <8 x i16> %a to <8 x i32>
   store <8 x i32>%b, ptr %out
   ret void
@@ -637,24 +361,6 @@ define void @sext_v16i16_v16i32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v16i16_v16i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %in
   %b = add <16 x i16> %a, %a
   %c = sext <16 x i16> %b to <16 x i32>
@@ -676,18 +382,6 @@ define void @sext_v4i16_v4i64(<4 x i16> %a, ptr %out) {
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v4i16_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <4 x i16> %a to <4 x i64>
   store <4 x i64>%b, ptr %out
   ret void
@@ -709,25 +403,6 @@ define void @sext_v8i16_v8i64(<8 x i16> %a, ptr %out) {
 ; CHECK-NEXT:    stp q2, q1, [x0]
 ; CHECK-NEXT:    stp q3, q0, [x0, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v8i16_v8i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-48]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    sshll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <8 x i16> %a to <8 x i64>
   store <8 x i64>%b, ptr %out
   ret void
@@ -762,40 +437,6 @@ define void @sext_v16i16_v16i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q6, q0, [x1, #96]
 ; CHECK-NEXT:    stp q7, q1, [x1, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v16i16_v16i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-96]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    stp q2, q0, [sp, #32]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    stp q3, q1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldr d5, [sp, #56]
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #72]
-; NONEON-NOSVE-NEXT:    sshll v5.2d, v5.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v4.2d, v4.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q5, [x1]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v2.2d, v6.2s, #0
-; NONEON-NOSVE-NEXT:    stp q1, q4, [x1, #64]
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v3.2d, v7.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #96
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %in
   %b = add <16 x i16> %a, %a
   %c = sext <16 x i16> %b to <16 x i64>
@@ -816,17 +457,6 @@ define void @sext_v4i32_v4i64(<4 x i32> %a, ptr %out) {
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v4i32_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = sext <4 x i32> %a to <4 x i64>
   store <4 x i64>%b, ptr %out
   ret void
@@ -847,24 +477,6 @@ define void @sext_v8i32_v8i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sext_v8i32_v8i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    add v1.4s, v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i32>, ptr %in
   %b = add <8 x i32> %a, %a
   %c = sext <8 x i32> %b to <8 x i64>
@@ -885,17 +497,6 @@ define void @zext_v16i8_v16i16(<16 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    uunpklo z0.h, z0.b
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v16i8_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    ushll v1.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = zext <16 x i8> %a to <16 x i16>
   store <16 x i16>%b, ptr %out
   ret void
@@ -917,24 +518,6 @@ define void @zext_v32i8_v32i16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v32i8_v32i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v1.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    ushll v2.8h, v2.8b, #0
-; NONEON-NOSVE-NEXT:    ushll v3.8h, v3.8b, #0
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
   %c = zext <32 x i8> %b to <32 x i16>
@@ -956,18 +539,6 @@ define void @zext_v8i8_v8i32(<8 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v8i8_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = zext <8 x i8> %a to <8 x i32>
   store <8 x i32>%b, ptr %out
   ret void
@@ -989,25 +560,6 @@ define void @zext_v16i8_v16i32(<16 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    stp q2, q1, [x0]
 ; CHECK-NEXT:    stp q3, q0, [x0, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v16i8_v16i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-48]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    ushll v1.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
-; NONEON-NOSVE-NEXT:    ret
   %b = zext <16 x i8> %a to <16 x i32>
   store <16 x i32> %b, ptr %out
   ret void
@@ -1042,40 +594,6 @@ define void @zext_v32i8_v32i32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q6, q0, [x1, #96]
 ; CHECK-NEXT:    stp q7, q1, [x1, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v32i8_v32i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-96]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    ushll v1.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    ushll v2.8h, v2.8b, #0
-; NONEON-NOSVE-NEXT:    ushll v3.8h, v3.8b, #0
-; NONEON-NOSVE-NEXT:    stp q2, q0, [sp, #32]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    stp q3, q1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldr d5, [sp, #56]
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #72]
-; NONEON-NOSVE-NEXT:    ushll v5.4s, v5.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v4.4s, v4.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q5, [x1]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v6.4h, #0
-; NONEON-NOSVE-NEXT:    stp q1, q4, [x1, #64]
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v3.4s, v7.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #96
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
   %c = zext <32 x i8> %b to <32 x i32>
@@ -1101,20 +619,6 @@ define void @zext_v4i8_v4i64(<4 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v4i8_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d1, #0xff00ff00ff00ff
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = zext <4 x i8> %a to <4 x i64>
   store <4 x i64>%b, ptr %out
   ret void
@@ -1137,26 +641,6 @@ define void @zext_v8i8_v8i64(<8 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    stp q2, q1, [x0]
 ; CHECK-NEXT:    stp q3, q0, [x0, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v8i8_v8i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-48]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
-; NONEON-NOSVE-NEXT:    ret
   %b = zext <8 x i8> %a to <8 x i64>
   store <8 x i64>%b, ptr %out
   ret void
@@ -1194,41 +678,6 @@ define void @zext_v16i8_v16i64(<16 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    stp q1, q4, [x0, #32]
 ; CHECK-NEXT:    stp q0, q2, [x0, #96]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v16i8_v16i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-112]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 112
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    ushll v1.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #40]
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    stp q2, q1, [sp, #48]
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    stp q3, q0, [sp, #80]
-; NONEON-NOSVE-NEXT:    ldr d5, [sp, #72]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #88]
-; NONEON-NOSVE-NEXT:    ushll v5.2d, v5.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v4.2d, v4.2s, #0
-; NONEON-NOSVE-NEXT:    stp q1, q5, [x0, #64]
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v6.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q4, [x0]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v3.2d, v7.2s, #0
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x0, #96]
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #112
-; NONEON-NOSVE-NEXT:    ret
   %b = zext <16 x i8> %a to <16 x i64>
   store <16 x i64> %b, ptr %out
   ret void
@@ -1297,73 +746,6 @@ define void @zext_v32i8_v32i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q0, q2, [x1, #224]
 ; CHECK-NEXT:    stp q3, q1, [x1, #96]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v32i8_v32i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #224
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 224
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp]
-; NONEON-NOSVE-NEXT:    ushll v5.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    ushll v6.8h, v1.8b, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v3.8h, v2.8b, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ushll v4.8h, v2.8b, #0
-; NONEON-NOSVE-NEXT:    stp q3, q5, [sp, #32]
-; NONEON-NOSVE-NEXT:    ushll v5.4s, v5.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #56]
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #40]
-; NONEON-NOSVE-NEXT:    stp q4, q6, [sp, #64]
-; NONEON-NOSVE-NEXT:    ushll v6.4s, v6.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v4.4s, v4.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #88]
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #72]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v7.4s, v7.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q2, q5, [sp, #128]
-; NONEON-NOSVE-NEXT:    ushll v5.2d, v5.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d19, [sp, #152]
-; NONEON-NOSVE-NEXT:    stp q0, q3, [sp, #96]
-; NONEON-NOSVE-NEXT:    ldr d20, [sp, #136]
-; NONEON-NOSVE-NEXT:    stp q1, q4, [sp, #160]
-; NONEON-NOSVE-NEXT:    ldr d17, [sp, #104]
-; NONEON-NOSVE-NEXT:    ldr d21, [sp, #120]
-; NONEON-NOSVE-NEXT:    stp q7, q6, [sp, #192]
-; NONEON-NOSVE-NEXT:    ushll v6.2d, v6.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v19.2d, v19.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d16, [sp, #216]
-; NONEON-NOSVE-NEXT:    ldr d22, [sp, #200]
-; NONEON-NOSVE-NEXT:    ldr d23, [sp, #184]
-; NONEON-NOSVE-NEXT:    ldr d18, [sp, #168]
-; NONEON-NOSVE-NEXT:    ushll v4.2d, v4.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v16.2d, v16.2s, #0
-; NONEON-NOSVE-NEXT:    stp q5, q19, [x1]
-; NONEON-NOSVE-NEXT:    ushll v5.2d, v7.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v7.2d, v22.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    stp q6, q16, [x1, #128]
-; NONEON-NOSVE-NEXT:    ushll v6.2d, v23.2s, #0
-; NONEON-NOSVE-NEXT:    stp q5, q7, [x1, #160]
-; NONEON-NOSVE-NEXT:    ushll v5.2d, v20.2s, #0
-; NONEON-NOSVE-NEXT:    stp q4, q6, [x1, #192]
-; NONEON-NOSVE-NEXT:    ushll v4.2d, v21.2s, #0
-; NONEON-NOSVE-NEXT:    stp q2, q5, [x1, #32]
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v17.2s, #0
-; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #64]
-; NONEON-NOSVE-NEXT:    ushll v3.2d, v18.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1, #96]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x1, #224]
-; NONEON-NOSVE-NEXT:    add sp, sp, #224
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   %b = add <32 x i8> %a, %a
   %c = zext <32 x i8> %b to <32 x i64>
@@ -1384,17 +766,6 @@ define void @zext_v8i16_v8i32(<8 x i16> %a, ptr %out) {
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v8i16_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = zext <8 x i16> %a to <8 x i32>
   store <8 x i32>%b, ptr %out
   ret void
@@ -1415,24 +786,6 @@ define void @zext_v16i16_v16i32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v16i16_v16i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %in
   %b = add <16 x i16> %a, %a
   %c = zext <16 x i16> %b to <16 x i32>
@@ -1454,18 +807,6 @@ define void @zext_v4i16_v4i64(<4 x i16> %a, ptr %out) {
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v4i16_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = zext <4 x i16> %a to <4 x i64>
   store <4 x i64>%b, ptr %out
   ret void
@@ -1487,25 +828,6 @@ define void @zext_v8i16_v8i64(<8 x i16> %a, ptr %out) {
 ; CHECK-NEXT:    stp q2, q1, [x0]
 ; CHECK-NEXT:    stp q3, q0, [x0, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v8i16_v8i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-48]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
-; NONEON-NOSVE-NEXT:    ret
   %b = zext <8 x i16> %a to <8 x i64>
   store <8 x i64>%b, ptr %out
   ret void
@@ -1540,40 +862,6 @@ define void @zext_v16i16_v16i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q6, q0, [x1, #96]
 ; CHECK-NEXT:    stp q7, q1, [x1, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v16i16_v16i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-96]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    stp q2, q0, [sp, #32]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    stp q3, q1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldr d5, [sp, #56]
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #72]
-; NONEON-NOSVE-NEXT:    ushll v5.2d, v5.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v4.2d, v4.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q5, [x1]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v6.2s, #0
-; NONEON-NOSVE-NEXT:    stp q1, q4, [x1, #64]
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v3.2d, v7.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #96
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %in
   %b = add <16 x i16> %a, %a
   %c = zext <16 x i16> %b to <16 x i64>
@@ -1594,17 +882,6 @@ define void @zext_v4i32_v4i64(<4 x i32> %a, ptr %out) {
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v4i32_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %b = zext <4 x i32> %a to <4 x i64>
   store <4 x i64>%b, ptr %out
   ret void
@@ -1625,24 +902,6 @@ define void @zext_v8i32_v8i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zext_v8i32_v8i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    add v0.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    add v1.4s, v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i32>, ptr %in
   %b = add <8 x i32> %a, %a
   %c = zext <8 x i32> %b to <8 x i64>
@@ -1669,21 +928,6 @@ define void @extend_and_mul(i32 %0, <2 x i64> %1, ptr %2) {
 ; SVE2-NEXT:    mul z0.d, z1.d, z0.d
 ; SVE2-NEXT:    str q0, [x1]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extend_and_mul:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v1.2s, w0
-; NONEON-NOSVE-NEXT:    fmov x10, d0
-; NONEON-NOSVE-NEXT:    mov x8, v0.d[1]
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    fmov x11, d1
-; NONEON-NOSVE-NEXT:    mov x9, v1.d[1]
-; NONEON-NOSVE-NEXT:    mul x10, x11, x10
-; NONEON-NOSVE-NEXT:    mul x8, x9, x8
-; NONEON-NOSVE-NEXT:    fmov d0, x10
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x8
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %broadcast.splatinsert2 = insertelement <2 x i32> poison, i32 %0, i64 0
   %broadcast.splat3 = shufflevector <2 x i32> %broadcast.splatinsert2, <2 x i32> poison, <2 x i32> zeroinitializer
   %4 = zext <2 x i32> %broadcast.splat3 to <2 x i64>
@@ -1699,13 +943,6 @@ define void @extend_no_mul(i32 %0, <2 x i64> %1, ptr %2) {
 ; CHECK-NEXT:    mov z0.d, x8
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: extend_no_mul:
-; NONEON-NOSVE:       // %bb.0: // %entry
-; NONEON-NOSVE-NEXT:    dup v0.2s, w0
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
 entry:
   %broadcast.splatinsert2 = insertelement <2 x i32> poison, i32 %0, i64 0
   %broadcast.splat3 = shufflevector <2 x i32> %broadcast.splatinsert2, <2 x i32> poison, <2 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll
index d86cfcbfb4f6..f028b3eeca25 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -23,15 +22,6 @@ define void @add_v32i8(ptr %a) {
 ; CHECK-NEXT:    add z1.b, z1.b, #7 // =0x7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #7
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i32 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -48,16 +38,6 @@ define void @add_v16i16(ptr %a) {
 ; CHECK-NEXT:    add z1.h, z1.h, #15 // =0xf
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #15 // =0xf
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -74,16 +54,6 @@ define void @add_v8i32(ptr %a) {
 ; CHECK-NEXT:    add z1.s, z1.s, #31 // =0x1f
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #31 // =0x1f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    add v1.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    add v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -100,16 +70,6 @@ define void @add_v4i64(ptr %a) {
 ; CHECK-NEXT:    add z1.d, z1.d, #63 // =0x3f
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #63 // =0x3f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    add v1.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    add v0.2d, v2.2d, v0.2d
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -130,15 +90,6 @@ define void @and_v32i8(ptr %a) {
 ; CHECK-NEXT:    and z1.b, z1.b, #0x7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #7
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    and v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    and v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i32 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -155,16 +106,6 @@ define void @and_v16i16(ptr %a) {
 ; CHECK-NEXT:    and z1.h, z1.h, #0xf
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #15 // =0xf
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    and v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    and v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -181,16 +122,6 @@ define void @and_v8i32(ptr %a) {
 ; CHECK-NEXT:    and z1.s, z1.s, #0x1f
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #31 // =0x1f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    and v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    and v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -207,16 +138,6 @@ define void @and_v4i64(ptr %a) {
 ; CHECK-NEXT:    and z1.d, z1.d, #0x3f
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #63 // =0x3f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    and v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    and v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -237,14 +158,6 @@ define void @ashr_v32i8(ptr %a) {
 ; CHECK-NEXT:    asr z1.b, z1.b, #7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cmlt v0.16b, v0.16b, #0
-; NONEON-NOSVE-NEXT:    cmlt v1.16b, v1.16b, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i32 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -261,14 +174,6 @@ define void @ashr_v16i16(ptr %a) {
 ; CHECK-NEXT:    asr z1.h, z1.h, #15
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cmlt v0.8h, v0.8h, #0
-; NONEON-NOSVE-NEXT:    cmlt v1.8h, v1.8h, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -285,14 +190,6 @@ define void @ashr_v8i32(ptr %a) {
 ; CHECK-NEXT:    asr z1.s, z1.s, #31
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cmlt v0.4s, v0.4s, #0
-; NONEON-NOSVE-NEXT:    cmlt v1.4s, v1.4s, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -309,14 +206,6 @@ define void @ashr_v4i64(ptr %a) {
 ; CHECK-NEXT:    asr z1.d, z1.d, #63
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cmlt v0.2d, v0.2d, #0
-; NONEON-NOSVE-NEXT:    cmlt v1.2d, v1.2d, #0
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -340,15 +229,6 @@ define void @icmp_eq_v32i8(ptr %a) {
 ; CHECK-NEXT:    mov z1.b, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_eq_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #7
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmeq v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    cmeq v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -369,16 +249,6 @@ define void @icmp_sge_v16i16(ptr %a) {
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_sge_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #15 // =0xf
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    cmge v1.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    cmge v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -399,16 +269,6 @@ define void @icmp_sgt_v8i32(ptr %a) {
 ; CHECK-NEXT:    mov z1.s, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_sgt_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #-8 // =0xfffffff8
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    cmgt v1.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    cmgt v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 -8, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -429,16 +289,6 @@ define void @icmp_ult_v4i64(ptr %a) {
 ; CHECK-NEXT:    mov z1.d, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: icmp_ult_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #63 // =0x3f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    cmhi v1.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    cmhi v0.2d, v0.2d, v2.2d
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -460,14 +310,6 @@ define void @lshr_v32i8(ptr %a) {
 ; CHECK-NEXT:    lsr z1.b, z1.b, #7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ushr v0.16b, v0.16b, #7
-; NONEON-NOSVE-NEXT:    ushr v1.16b, v1.16b, #7
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -484,14 +326,6 @@ define void @lshr_v16i16(ptr %a) {
 ; CHECK-NEXT:    lsr z1.h, z1.h, #15
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ushr v0.8h, v0.8h, #15
-; NONEON-NOSVE-NEXT:    ushr v1.8h, v1.8h, #15
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -508,14 +342,6 @@ define void @lshr_v8i32(ptr %a) {
 ; CHECK-NEXT:    lsr z1.s, z1.s, #31
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ushr v0.4s, v0.4s, #31
-; NONEON-NOSVE-NEXT:    ushr v1.4s, v1.4s, #31
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -532,14 +358,6 @@ define void @lshr_v4i64(ptr %a) {
 ; CHECK-NEXT:    lsr z1.d, z1.d, #63
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ushr v0.2d, v0.2d, #63
-; NONEON-NOSVE-NEXT:    ushr v1.2d, v1.2d, #63
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -560,15 +378,6 @@ define void @mul_v32i8(ptr %a) {
 ; CHECK-NEXT:    mul z1.b, z1.b, #7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #7
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    mul v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    mul v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -585,16 +394,6 @@ define void @mul_v16i16(ptr %a) {
 ; CHECK-NEXT:    mul z1.h, z1.h, #15
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #15 // =0xf
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    mul v1.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    mul v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -611,16 +410,6 @@ define void @mul_v8i32(ptr %a) {
 ; CHECK-NEXT:    mul z1.s, z1.s, #31
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #31 // =0x1f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    mul v1.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    mul v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -637,28 +426,6 @@ define void @mul_v4i64(ptr %a) {
 ; CHECK-NEXT:    mul z1.d, z1.d, #63
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mul_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    fmov x10, d0
-; NONEON-NOSVE-NEXT:    fmov x11, d1
-; NONEON-NOSVE-NEXT:    mov x8, v0.d[1]
-; NONEON-NOSVE-NEXT:    mov x9, v1.d[1]
-; NONEON-NOSVE-NEXT:    lsl x12, x10, #6
-; NONEON-NOSVE-NEXT:    lsl x13, x11, #6
-; NONEON-NOSVE-NEXT:    lsl x14, x8, #6
-; NONEON-NOSVE-NEXT:    sub x10, x12, x10
-; NONEON-NOSVE-NEXT:    sub x11, x13, x11
-; NONEON-NOSVE-NEXT:    lsl x12, x9, #6
-; NONEON-NOSVE-NEXT:    fmov d0, x10
-; NONEON-NOSVE-NEXT:    fmov d1, x11
-; NONEON-NOSVE-NEXT:    sub x8, x14, x8
-; NONEON-NOSVE-NEXT:    sub x9, x12, x9
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x8
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x9
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -679,15 +446,6 @@ define void @or_v32i8(ptr %a) {
 ; CHECK-NEXT:    orr z1.b, z1.b, #0x7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #7
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    orr v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    orr v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -704,16 +462,6 @@ define void @or_v16i16(ptr %a) {
 ; CHECK-NEXT:    orr z1.h, z1.h, #0xf
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #15 // =0xf
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    orr v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    orr v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -730,16 +478,6 @@ define void @or_v8i32(ptr %a) {
 ; CHECK-NEXT:    orr z1.s, z1.s, #0x1f
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #31 // =0x1f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    orr v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    orr v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -756,16 +494,6 @@ define void @or_v4i64(ptr %a) {
 ; CHECK-NEXT:    orr z1.d, z1.d, #0x3f
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #63 // =0x3f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    orr v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    orr v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -786,14 +514,6 @@ define void @shl_v32i8(ptr %a) {
 ; CHECK-NEXT:    lsl z1.b, z1.b, #7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    shl v0.16b, v0.16b, #7
-; NONEON-NOSVE-NEXT:    shl v1.16b, v1.16b, #7
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -810,14 +530,6 @@ define void @shl_v16i16(ptr %a) {
 ; CHECK-NEXT:    lsl z1.h, z1.h, #15
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    shl v0.8h, v0.8h, #15
-; NONEON-NOSVE-NEXT:    shl v1.8h, v1.8h, #15
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -834,14 +546,6 @@ define void @shl_v8i32(ptr %a) {
 ; CHECK-NEXT:    lsl z1.s, z1.s, #31
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    shl v0.4s, v0.4s, #31
-; NONEON-NOSVE-NEXT:    shl v1.4s, v1.4s, #31
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -858,14 +562,6 @@ define void @shl_v4i64(ptr %a) {
 ; CHECK-NEXT:    lsl z1.d, z1.d, #63
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    shl v0.2d, v0.2d, #63
-; NONEON-NOSVE-NEXT:    shl v1.2d, v1.2d, #63
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -886,15 +582,6 @@ define void @smax_v32i8(ptr %a) {
 ; CHECK-NEXT:    smax z1.b, z1.b, #7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #7
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    smax v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    smax v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -911,16 +598,6 @@ define void @smax_v16i16(ptr %a) {
 ; CHECK-NEXT:    smax z1.h, z1.h, #15
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #15 // =0xf
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    smax v1.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    smax v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -937,16 +614,6 @@ define void @smax_v8i32(ptr %a) {
 ; CHECK-NEXT:    smax z1.s, z1.s, #31
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #31 // =0x1f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    smax v1.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    smax v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -963,18 +630,6 @@ define void @smax_v4i64(ptr %a) {
 ; CHECK-NEXT:    smax z1.d, z1.d, #63
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #63 // =0x3f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    cmgt v3.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    cmgt v4.2d, v2.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bif v1.16b, v0.16b, v3.16b
-; NONEON-NOSVE-NEXT:    bit v0.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -995,15 +650,6 @@ define void @smin_v32i8(ptr %a) {
 ; CHECK-NEXT:    smin z1.b, z1.b, #7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #7
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    smin v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    smin v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -1020,16 +666,6 @@ define void @smin_v16i16(ptr %a) {
 ; CHECK-NEXT:    smin z1.h, z1.h, #15
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #15 // =0xf
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    smin v1.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    smin v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -1046,16 +682,6 @@ define void @smin_v8i32(ptr %a) {
 ; CHECK-NEXT:    smin z1.s, z1.s, #31
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #31 // =0x1f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    smin v1.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    smin v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -1072,18 +698,6 @@ define void @smin_v4i64(ptr %a) {
 ; CHECK-NEXT:    smin z1.d, z1.d, #63
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #63 // =0x3f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    cmgt v3.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    cmgt v4.2d, v0.2d, v2.2d
-; NONEON-NOSVE-NEXT:    bif v1.16b, v0.16b, v3.16b
-; NONEON-NOSVE-NEXT:    bit v0.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -1104,15 +718,6 @@ define void @sub_v32i8(ptr %a) {
 ; CHECK-NEXT:    sub z1.b, z1.b, #7 // =0x7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #7
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    sub v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    sub v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -1129,16 +734,6 @@ define void @sub_v16i16(ptr %a) {
 ; CHECK-NEXT:    sub z1.h, z1.h, #15 // =0xf
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #15 // =0xf
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    sub v1.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    sub v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -1155,16 +750,6 @@ define void @sub_v8i32(ptr %a) {
 ; CHECK-NEXT:    sub z1.s, z1.s, #31 // =0x1f
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #31 // =0x1f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    sub v1.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    sub v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -1181,16 +766,6 @@ define void @sub_v4i64(ptr %a) {
 ; CHECK-NEXT:    sub z1.d, z1.d, #63 // =0x3f
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sub_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #63 // =0x3f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    sub v1.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    sub v0.2d, v2.2d, v0.2d
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -1211,15 +786,6 @@ define void @umax_v32i8(ptr %a) {
 ; CHECK-NEXT:    umax z1.b, z1.b, #7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #7
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    umax v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    umax v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -1236,16 +802,6 @@ define void @umax_v16i16(ptr %a) {
 ; CHECK-NEXT:    umax z1.h, z1.h, #15
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #15 // =0xf
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    umax v1.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    umax v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -1262,16 +818,6 @@ define void @umax_v8i32(ptr %a) {
 ; CHECK-NEXT:    umax z1.s, z1.s, #31
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #31 // =0x1f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    umax v1.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    umax v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -1288,18 +834,6 @@ define void @umax_v4i64(ptr %a) {
 ; CHECK-NEXT:    umax z1.d, z1.d, #63
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #63 // =0x3f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    cmhi v3.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    cmhi v4.2d, v2.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bif v1.16b, v0.16b, v3.16b
-; NONEON-NOSVE-NEXT:    bit v0.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -1320,15 +854,6 @@ define void @umin_v32i8(ptr %a) {
 ; CHECK-NEXT:    umin z1.b, z1.b, #7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #7
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    umin v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    umin v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -1345,16 +870,6 @@ define void @umin_v16i16(ptr %a) {
 ; CHECK-NEXT:    umin z1.h, z1.h, #15
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #15 // =0xf
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    umin v1.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    umin v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -1371,16 +886,6 @@ define void @umin_v8i32(ptr %a) {
 ; CHECK-NEXT:    umin z1.s, z1.s, #31
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #31 // =0x1f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    umin v1.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    umin v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -1397,18 +902,6 @@ define void @umin_v4i64(ptr %a) {
 ; CHECK-NEXT:    umin z1.d, z1.d, #63
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #63 // =0x3f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    cmhi v3.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    cmhi v4.2d, v0.2d, v2.2d
-; NONEON-NOSVE-NEXT:    bif v1.16b, v0.16b, v3.16b
-; NONEON-NOSVE-NEXT:    bit v0.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -1429,15 +922,6 @@ define void @xor_v32i8(ptr %a) {
 ; CHECK-NEXT:    eor z1.b, z1.b, #0x7
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #7
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    eor v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    eor v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %ins = insertelement <32 x i8> undef, i8 7, i64 0
   %op2 = shufflevector <32 x i8> %ins, <32 x i8> undef, <32 x i32> zeroinitializer
@@ -1454,16 +938,6 @@ define void @xor_v16i16(ptr %a) {
 ; CHECK-NEXT:    eor z1.h, z1.h, #0xf
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #15 // =0xf
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    eor v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    eor v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %ins = insertelement <16 x i16> undef, i16 15, i64 0
   %op2 = shufflevector <16 x i16> %ins, <16 x i16> undef, <16 x i32> zeroinitializer
@@ -1480,16 +954,6 @@ define void @xor_v8i32(ptr %a) {
 ; CHECK-NEXT:    eor z1.s, z1.s, #0x1f
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #31 // =0x1f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    eor v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    eor v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %ins = insertelement <8 x i32> undef, i32 31, i64 0
   %op2 = shufflevector <8 x i32> %ins, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -1506,16 +970,6 @@ define void @xor_v4i64(ptr %a) {
 ; CHECK-NEXT:    eor z1.d, z1.d, #0x3f
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #63 // =0x3f
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    eor v1.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    eor v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %ins = insertelement <4 x i64> undef, i64 63, i64 0
   %op2 = shufflevector <4 x i64> %ins, <4 x i64> undef, <4 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-log.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-log.ll
index f0b39b275614..4d70c1dd1c91 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-log.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-log.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -17,11 +16,6 @@ define <8 x i8> @and_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = and <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -34,11 +28,6 @@ define <16 x i8> @and_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = and <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -52,15 +41,6 @@ define void @and_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    and z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    and v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    and v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = and <32 x i8> %op1, %op2
@@ -76,11 +56,6 @@ define <4 x i16> @and_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = and <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -93,11 +68,6 @@ define <8 x i16> @and_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = and <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -111,15 +81,6 @@ define void @and_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    and z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    and v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    and v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = and <16 x i16> %op1, %op2
@@ -135,11 +96,6 @@ define <2 x i32> @and_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = and <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -152,11 +108,6 @@ define <4 x i32> @and_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = and <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -170,15 +121,6 @@ define void @and_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    and z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    and v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    and v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = and <8 x i32> %op1, %op2
@@ -194,11 +136,6 @@ define <1 x i64> @and_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = and <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -211,11 +148,6 @@ define <2 x i64> @and_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    and z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = and <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -229,15 +161,6 @@ define void @and_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    and z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: and_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    and v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    and v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = and <4 x i64> %op1, %op2
@@ -257,11 +180,6 @@ define <8 x i8> @or_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = or <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -274,11 +192,6 @@ define <16 x i8> @or_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = or <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -292,15 +205,6 @@ define void @or_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    orr z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    orr v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    orr v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = or <32 x i8> %op1, %op2
@@ -316,11 +220,6 @@ define <4 x i16> @or_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = or <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -333,11 +232,6 @@ define <8 x i16> @or_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = or <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -351,15 +245,6 @@ define void @or_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    orr z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    orr v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    orr v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = or <16 x i16> %op1, %op2
@@ -375,11 +260,6 @@ define <2 x i32> @or_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = or <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -392,11 +272,6 @@ define <4 x i32> @or_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = or <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -410,15 +285,6 @@ define void @or_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    orr z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    orr v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    orr v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = or <8 x i32> %op1, %op2
@@ -434,11 +300,6 @@ define <1 x i64> @or_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = or <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -451,11 +312,6 @@ define <2 x i64> @or_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    orr z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    orr v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = or <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -469,15 +325,6 @@ define void @or_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    orr z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: or_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    orr v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    orr v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = or <4 x i64> %op1, %op2
@@ -497,11 +344,6 @@ define <8 x i8> @xor_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = xor <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -514,11 +356,6 @@ define <16 x i8> @xor_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    eor v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = xor <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -532,15 +369,6 @@ define void @xor_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    eor z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    eor v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    eor v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = xor <32 x i8> %op1, %op2
@@ -556,11 +384,6 @@ define <4 x i16> @xor_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = xor <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -573,11 +396,6 @@ define <8 x i16> @xor_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    eor v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = xor <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -591,15 +409,6 @@ define void @xor_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    eor z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    eor v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    eor v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = xor <16 x i16> %op1, %op2
@@ -615,11 +424,6 @@ define <2 x i32> @xor_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = xor <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -632,11 +436,6 @@ define <4 x i32> @xor_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    eor v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = xor <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -650,15 +449,6 @@ define void @xor_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    eor z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    eor v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    eor v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = xor <8 x i32> %op1, %op2
@@ -674,11 +464,6 @@ define <1 x i64> @xor_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = xor <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -691,11 +476,6 @@ define <2 x i64> @xor_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    eor z0.d, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    eor v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = xor <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -709,15 +489,6 @@ define void @xor_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    eor z1.d, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: xor_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    eor v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    eor v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = xor <4 x i64> %op1, %op2
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll
index 51c404ece6cd..50cf9b73d9a7 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -18,11 +17,6 @@ define <8 x i8> @smax_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    smax z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smax v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i8> @llvm.smax.v8i8(<8 x i8> %op1, <8 x i8> %op2)
   ret <8 x i8> %res
 }
@@ -36,11 +30,6 @@ define <16 x i8> @smax_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    smax z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smax v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <16 x i8> @llvm.smax.v16i8(<16 x i8> %op1, <16 x i8> %op2)
   ret <16 x i8> %res
 }
@@ -56,15 +45,6 @@ define void @smax_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    smax z1.b, p0/m, z1.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    smax v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    smax v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = call <32 x i8> @llvm.smax.v32i8(<32 x i8> %op1, <32 x i8> %op2)
@@ -81,11 +61,6 @@ define <4 x i16> @smax_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    smax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smax v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i16> @llvm.smax.v4i16(<4 x i16> %op1, <4 x i16> %op2)
   ret <4 x i16> %res
 }
@@ -99,11 +74,6 @@ define <8 x i16> @smax_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    smax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smax v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i16> @llvm.smax.v8i16(<8 x i16> %op1, <8 x i16> %op2)
   ret <8 x i16> %res
 }
@@ -119,15 +89,6 @@ define void @smax_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    smax z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    smax v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    smax v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = call <16 x i16> @llvm.smax.v16i16(<16 x i16> %op1, <16 x i16> %op2)
@@ -144,11 +105,6 @@ define <2 x i32> @smax_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    smax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smax v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i32> @llvm.smax.v2i32(<2 x i32> %op1, <2 x i32> %op2)
   ret <2 x i32> %res
 }
@@ -162,11 +118,6 @@ define <4 x i32> @smax_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    smax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smax v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %op1, <4 x i32> %op2)
   ret <4 x i32> %res
 }
@@ -182,15 +133,6 @@ define void @smax_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    smax z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    smax v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    smax v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = call <8 x i32> @llvm.smax.v8i32(<8 x i32> %op1, <8 x i32> %op2)
@@ -208,12 +150,6 @@ define <1 x i64> @smax_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    smax z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmgt d2, d0, d1
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x i64> @llvm.smax.v1i64(<1 x i64> %op1, <1 x i64> %op2)
   ret <1 x i64> %res
 }
@@ -228,12 +164,6 @@ define <2 x i64> @smax_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    smax z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmgt v2.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %op1, <2 x i64> %op2)
   ret <2 x i64> %res
 }
@@ -249,18 +179,6 @@ define void @smax_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    smax z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smax_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmgt v4.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    cmgt v5.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    bit v0.16b, v1.16b, v4.16b
-; NONEON-NOSVE-NEXT:    mov v1.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %op1, <4 x i64> %op2)
@@ -281,11 +199,6 @@ define <8 x i8> @smin_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    smin z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smin v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i8> @llvm.smin.v8i8(<8 x i8> %op1, <8 x i8> %op2)
   ret <8 x i8> %res
 }
@@ -299,11 +212,6 @@ define <16 x i8> @smin_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    smin z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smin v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <16 x i8> @llvm.smin.v16i8(<16 x i8> %op1, <16 x i8> %op2)
   ret <16 x i8> %res
 }
@@ -319,15 +227,6 @@ define void @smin_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    smin z1.b, p0/m, z1.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    smin v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    smin v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = call <32 x i8> @llvm.smin.v32i8(<32 x i8> %op1, <32 x i8> %op2)
@@ -344,11 +243,6 @@ define <4 x i16> @smin_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    smin z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smin v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i16> @llvm.smin.v4i16(<4 x i16> %op1, <4 x i16> %op2)
   ret <4 x i16> %res
 }
@@ -362,11 +256,6 @@ define <8 x i16> @smin_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    smin z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smin v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i16> @llvm.smin.v8i16(<8 x i16> %op1, <8 x i16> %op2)
   ret <8 x i16> %res
 }
@@ -382,15 +271,6 @@ define void @smin_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    smin z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    smin v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    smin v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = call <16 x i16> @llvm.smin.v16i16(<16 x i16> %op1, <16 x i16> %op2)
@@ -407,11 +287,6 @@ define <2 x i32> @smin_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    smin z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smin v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i32> @llvm.smin.v2i32(<2 x i32> %op1, <2 x i32> %op2)
   ret <2 x i32> %res
 }
@@ -425,11 +300,6 @@ define <4 x i32> @smin_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    smin z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smin v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %op1, <4 x i32> %op2)
   ret <4 x i32> %res
 }
@@ -445,15 +315,6 @@ define void @smin_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    smin z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    smin v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    smin v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = call <8 x i32> @llvm.smin.v8i32(<8 x i32> %op1, <8 x i32> %op2)
@@ -471,12 +332,6 @@ define <1 x i64> @smin_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    smin z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmgt d2, d1, d0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x i64> @llvm.smin.v1i64(<1 x i64> %op1, <1 x i64> %op2)
   ret <1 x i64> %res
 }
@@ -491,12 +346,6 @@ define <2 x i64> @smin_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    smin z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmgt v2.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i64> @llvm.smin.v2i64(<2 x i64> %op1, <2 x i64> %op2)
   ret <2 x i64> %res
 }
@@ -512,18 +361,6 @@ define void @smin_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    smin z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smin_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmgt v4.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    cmgt v5.2d, v3.2d, v2.2d
-; NONEON-NOSVE-NEXT:    bit v0.16b, v1.16b, v4.16b
-; NONEON-NOSVE-NEXT:    mov v1.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %op1, <4 x i64> %op2)
@@ -544,11 +381,6 @@ define <8 x i8> @umax_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    umax z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umax v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i8> @llvm.umax.v8i8(<8 x i8> %op1, <8 x i8> %op2)
   ret <8 x i8> %res
 }
@@ -562,11 +394,6 @@ define <16 x i8> @umax_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    umax z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umax v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <16 x i8> @llvm.umax.v16i8(<16 x i8> %op1, <16 x i8> %op2)
   ret <16 x i8> %res
 }
@@ -582,15 +409,6 @@ define void @umax_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    umax z1.b, p0/m, z1.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    umax v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    umax v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = call <32 x i8> @llvm.umax.v32i8(<32 x i8> %op1, <32 x i8> %op2)
@@ -607,11 +425,6 @@ define <4 x i16> @umax_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    umax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umax v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i16> @llvm.umax.v4i16(<4 x i16> %op1, <4 x i16> %op2)
   ret <4 x i16> %res
 }
@@ -625,11 +438,6 @@ define <8 x i16> @umax_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    umax z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umax v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i16> @llvm.umax.v8i16(<8 x i16> %op1, <8 x i16> %op2)
   ret <8 x i16> %res
 }
@@ -645,15 +453,6 @@ define void @umax_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    umax z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    umax v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    umax v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = call <16 x i16> @llvm.umax.v16i16(<16 x i16> %op1, <16 x i16> %op2)
@@ -670,11 +469,6 @@ define <2 x i32> @umax_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    umax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umax v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i32> @llvm.umax.v2i32(<2 x i32> %op1, <2 x i32> %op2)
   ret <2 x i32> %res
 }
@@ -688,11 +482,6 @@ define <4 x i32> @umax_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    umax z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umax v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %op1, <4 x i32> %op2)
   ret <4 x i32> %res
 }
@@ -708,15 +497,6 @@ define void @umax_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    umax z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    umax v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    umax v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = call <8 x i32> @llvm.umax.v8i32(<8 x i32> %op1, <8 x i32> %op2)
@@ -734,12 +514,6 @@ define <1 x i64> @umax_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    umax z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmhi d2, d0, d1
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x i64> @llvm.umax.v1i64(<1 x i64> %op1, <1 x i64> %op2)
   ret <1 x i64> %res
 }
@@ -754,12 +528,6 @@ define <2 x i64> @umax_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    umax z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmhi v2.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i64> @llvm.umax.v2i64(<2 x i64> %op1, <2 x i64> %op2)
   ret <2 x i64> %res
 }
@@ -775,18 +543,6 @@ define void @umax_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    umax z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umax_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmhi v4.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    cmhi v5.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    bit v0.16b, v1.16b, v4.16b
-; NONEON-NOSVE-NEXT:    mov v1.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = call <4 x i64> @llvm.umax.v4i64(<4 x i64> %op1, <4 x i64> %op2)
@@ -807,11 +563,6 @@ define <8 x i8> @umin_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    umin z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umin v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i8> @llvm.umin.v8i8(<8 x i8> %op1, <8 x i8> %op2)
   ret <8 x i8> %res
 }
@@ -825,11 +576,6 @@ define <16 x i8> @umin_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    umin z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umin v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <16 x i8> @llvm.umin.v16i8(<16 x i8> %op1, <16 x i8> %op2)
   ret <16 x i8> %res
 }
@@ -845,15 +591,6 @@ define void @umin_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    umin z1.b, p0/m, z1.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    umin v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    umin v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = call <32 x i8> @llvm.umin.v32i8(<32 x i8> %op1, <32 x i8> %op2)
@@ -870,11 +607,6 @@ define <4 x i16> @umin_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    umin z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umin v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i16> @llvm.umin.v4i16(<4 x i16> %op1, <4 x i16> %op2)
   ret <4 x i16> %res
 }
@@ -888,11 +620,6 @@ define <8 x i16> @umin_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    umin z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umin v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i16> @llvm.umin.v8i16(<8 x i16> %op1, <8 x i16> %op2)
   ret <8 x i16> %res
 }
@@ -908,15 +635,6 @@ define void @umin_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    umin z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    umin v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    umin v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = call <16 x i16> @llvm.umin.v16i16(<16 x i16> %op1, <16 x i16> %op2)
@@ -933,11 +651,6 @@ define <2 x i32> @umin_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    umin z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umin v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i32> @llvm.umin.v2i32(<2 x i32> %op1, <2 x i32> %op2)
   ret <2 x i32> %res
 }
@@ -951,11 +664,6 @@ define <4 x i32> @umin_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    umin z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umin v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %op1, <4 x i32> %op2)
   ret <4 x i32> %res
 }
@@ -971,15 +679,6 @@ define void @umin_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    umin z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    umin v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    umin v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %op1, <8 x i32> %op2)
@@ -997,12 +696,6 @@ define <1 x i64> @umin_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    umin z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmhi d2, d1, d0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x i64> @llvm.umin.v1i64(<1 x i64> %op1, <1 x i64> %op2)
   ret <1 x i64> %res
 }
@@ -1017,12 +710,6 @@ define <2 x i64> @umin_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    umin z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmhi v2.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i64> @llvm.umin.v2i64(<2 x i64> %op1, <2 x i64> %op2)
   ret <2 x i64> %res
 }
@@ -1038,18 +725,6 @@ define void @umin_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    umin z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umin_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    cmhi v4.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    cmhi v5.2d, v3.2d, v2.2d
-; NONEON-NOSVE-NEXT:    bit v0.16b, v1.16b, v4.16b
-; NONEON-NOSVE-NEXT:    mov v1.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %op1, <4 x i64> %op2)
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mla-neon-fa64.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mla-neon-fa64.ll
index 83714152c173..149ad6d1e267 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mla-neon-fa64.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mla-neon-fa64.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sme-fa64 -force-streaming-compatible-sve < %s | FileCheck %s -check-prefix=FA64
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s -check-prefix=NO-FA64
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -21,12 +20,6 @@ define <8 x i8> @mla8xi8(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C) {
 ; NO-FA64-NEXT:    mad z0.b, p0/m, z1.b, z2.b
 ; NO-FA64-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; NO-FA64-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: mla8xi8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mla v2.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov d0, d2
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = mul <8 x i8> %A, %B;
   %tmp2 = add <8 x i8> %C, %tmp1;
   ret <8 x i8> %tmp2
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
index 6e6d40e2ea04..cb7fa53eac51 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
@@ -2,7 +2,6 @@
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2 -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 ; This test only tests the legal types for a given vector width, as mulh nodes
 ; do not get generated for non-legal types.
@@ -37,16 +36,6 @@ define <4 x i8> @smulh_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; SVE2-NEXT:    lsr z0.h, z0.h, #4
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    shl v1.4h, v1.4h, #8
-; NONEON-NOSVE-NEXT:    sshr v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    sshr v1.4h, v1.4h, #8
-; NONEON-NOSVE-NEXT:    mul v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ushr v0.4h, v0.4h, #4
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <4 x i16> undef, i16 4, i64 0
   %splat = shufflevector <4 x i16> %insert, <4 x i16> undef, <4 x i32> zeroinitializer
   %1 = sext <4 x i8> %op1 to <4 x i16>
@@ -74,12 +63,6 @@ define <8 x i8> @smulh_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; SVE2-NEXT:    smulh z0.b, z0.b, z1.b
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smull v0.8h, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    shrn v0.8b, v0.8h, #8
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <8 x i16> undef, i16 8, i64 0
   %splat = shufflevector <8 x i16> %insert, <8 x i16> undef, <8 x i32> zeroinitializer
   %1 = sext <8 x i8> %op1 to <8 x i16>
@@ -107,13 +90,6 @@ define <16 x i8> @smulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; SVE2-NEXT:    smulh z0.b, z0.b, z1.b
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smull2 v2.8h, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    smull v0.8h, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    uzp2 v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %1 = sext <16 x i8> %op1 to <16 x i16>
   %2 = sext <16 x i8> %op2 to <16 x i16>
   %mul = mul <16 x i16> %1, %2
@@ -142,19 +118,6 @@ define void @smulh_v32i8(ptr %a, ptr %b) {
 ; SVE2-NEXT:    smulh z1.b, z2.b, z3.b
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    smull2 v4.8h, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    smull v0.8h, v1.8b, v0.8b
-; NONEON-NOSVE-NEXT:    smull2 v1.8h, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    smull v2.8h, v2.8b, v3.8b
-; NONEON-NOSVE-NEXT:    uzp2 v0.16b, v0.16b, v4.16b
-; NONEON-NOSVE-NEXT:    uzp2 v1.16b, v2.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %1 = sext <32 x i8> %op1 to <32 x i16>
@@ -190,16 +153,6 @@ define <2 x i16> @smulh_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; SVE2-NEXT:    lsr z0.s, z0.s, #16
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    shl v1.2s, v1.2s, #16
-; NONEON-NOSVE-NEXT:    sshr v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    sshr v1.2s, v1.2s, #16
-; NONEON-NOSVE-NEXT:    mul v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ushr v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    ret
   %1 = sext <2 x i16> %op1 to <2 x i32>
   %2 = sext <2 x i16> %op2 to <2 x i32>
   %mul = mul <2 x i32> %1, %2
@@ -225,12 +178,6 @@ define <4 x i16> @smulh_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; SVE2-NEXT:    smulh z0.h, z0.h, z1.h
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smull v0.4s, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    shrn v0.4h, v0.4s, #16
-; NONEON-NOSVE-NEXT:    ret
   %1 = sext <4 x i16> %op1 to <4 x i32>
   %2 = sext <4 x i16> %op2 to <4 x i32>
   %mul = mul <4 x i32> %1, %2
@@ -256,13 +203,6 @@ define <8 x i16> @smulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; SVE2-NEXT:    smulh z0.h, z0.h, z1.h
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smull2 v2.4s, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    smull v0.4s, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
-; NONEON-NOSVE-NEXT:    ret
   %1 = sext <8 x i16> %op1 to <8 x i32>
   %2 = sext <8 x i16> %op2 to <8 x i32>
   %mul = mul <8 x i32> %1, %2
@@ -291,19 +231,6 @@ define void @smulh_v16i16(ptr %a, ptr %b) {
 ; SVE2-NEXT:    smulh z1.h, z2.h, z3.h
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    smull2 v4.4s, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    smull v0.4s, v1.4h, v0.4h
-; NONEON-NOSVE-NEXT:    smull2 v1.4s, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    smull v2.4s, v2.4h, v3.4h
-; NONEON-NOSVE-NEXT:    uzp2 v0.8h, v0.8h, v4.8h
-; NONEON-NOSVE-NEXT:    uzp2 v1.8h, v2.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %1 = sext <16 x i16> %op1 to <16 x i32>
@@ -332,12 +259,6 @@ define <2 x i32> @smulh_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; SVE2-NEXT:    smulh z0.s, z0.s, z1.s
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smull v0.2d, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    shrn v0.2s, v0.2d, #32
-; NONEON-NOSVE-NEXT:    ret
   %1 = sext <2 x i32> %op1 to <2 x i64>
   %2 = sext <2 x i32> %op2 to <2 x i64>
   %mul = mul <2 x i64> %1, %2
@@ -363,13 +284,6 @@ define <4 x i32> @smulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; SVE2-NEXT:    smulh z0.s, z0.s, z1.s
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smull2 v2.2d, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    smull v0.2d, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    uzp2 v0.4s, v0.4s, v2.4s
-; NONEON-NOSVE-NEXT:    ret
   %1 = sext <4 x i32> %op1 to <4 x i64>
   %2 = sext <4 x i32> %op2 to <4 x i64>
   %mul = mul <4 x i64> %1, %2
@@ -398,19 +312,6 @@ define void @smulh_v8i32(ptr %a, ptr %b) {
 ; SVE2-NEXT:    smulh z1.s, z2.s, z3.s
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    smull2 v4.2d, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    smull v0.2d, v1.2s, v0.2s
-; NONEON-NOSVE-NEXT:    smull2 v1.2d, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    smull v2.2d, v2.2s, v3.2s
-; NONEON-NOSVE-NEXT:    uzp2 v0.4s, v0.4s, v4.4s
-; NONEON-NOSVE-NEXT:    uzp2 v1.4s, v2.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %1 = sext <8 x i32> %op1 to <8 x i64>
@@ -439,16 +340,6 @@ define <1 x i64> @smulh_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; SVE2-NEXT:    smulh z0.d, z0.d, z1.d
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    fmov x9, d1
-; NONEON-NOSVE-NEXT:    smulh x8, x8, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <1 x i128> undef, i128 64, i128 0
   %splat = shufflevector <1 x i128> %insert, <1 x i128> undef, <1 x i32> zeroinitializer
   %1 = sext <1 x i64> %op1 to <1 x i128>
@@ -476,19 +367,6 @@ define <2 x i64> @smulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; SVE2-NEXT:    smulh z0.d, z0.d, z1.d
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov x8, v0.d[1]
-; NONEON-NOSVE-NEXT:    mov x9, v1.d[1]
-; NONEON-NOSVE-NEXT:    fmov x10, d0
-; NONEON-NOSVE-NEXT:    fmov x11, d1
-; NONEON-NOSVE-NEXT:    smulh x10, x10, x11
-; NONEON-NOSVE-NEXT:    smulh x8, x8, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x10
-; NONEON-NOSVE-NEXT:    fmov d1, x8
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    ret
   %1 = sext <2 x i64> %op1 to <2 x i128>
   %2 = sext <2 x i64> %op2 to <2 x i128>
   %mul = mul <2 x i128> %1, %2
@@ -517,31 +395,6 @@ define void @smulh_v4i64(ptr %a, ptr %b) {
 ; SVE2-NEXT:    smulh z1.d, z2.d, z3.d
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smulh_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    mov x11, v0.d[1]
-; NONEON-NOSVE-NEXT:    mov x14, v3.d[1]
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    mov x10, v1.d[1]
-; NONEON-NOSVE-NEXT:    mov x13, v2.d[1]
-; NONEON-NOSVE-NEXT:    fmov x12, d3
-; NONEON-NOSVE-NEXT:    smulh x8, x8, x9
-; NONEON-NOSVE-NEXT:    fmov x9, d2
-; NONEON-NOSVE-NEXT:    smulh x10, x10, x11
-; NONEON-NOSVE-NEXT:    smulh x9, x9, x12
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    smulh x11, x13, x14
-; NONEON-NOSVE-NEXT:    fmov d1, x10
-; NONEON-NOSVE-NEXT:    fmov d2, x9
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    fmov d3, x11
-; NONEON-NOSVE-NEXT:    mov v2.d[1], v3.d[0]
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %1 = sext <4 x i64> %op1 to <4 x i128>
@@ -580,15 +433,6 @@ define <4 x i8> @umulh_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; SVE2-NEXT:    lsr z0.h, z0.h, #4
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d2, #0xff00ff00ff00ff
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v2.8b
-; NONEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    mul v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ushr v0.4h, v0.4h, #4
-; NONEON-NOSVE-NEXT:    ret
   %1 = zext <4 x i8> %op1 to <4 x i16>
   %2 = zext <4 x i8> %op2 to <4 x i16>
   %mul = mul <4 x i16> %1, %2
@@ -614,12 +458,6 @@ define <8 x i8> @umulh_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; SVE2-NEXT:    umulh z0.b, z0.b, z1.b
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umull v0.8h, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    shrn v0.8b, v0.8h, #8
-; NONEON-NOSVE-NEXT:    ret
   %1 = zext <8 x i8> %op1 to <8 x i16>
   %2 = zext <8 x i8> %op2 to <8 x i16>
   %mul = mul <8 x i16> %1, %2
@@ -645,13 +483,6 @@ define <16 x i8> @umulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; SVE2-NEXT:    umulh z0.b, z0.b, z1.b
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umull2 v2.8h, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    umull v0.8h, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    uzp2 v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %1 = zext <16 x i8> %op1 to <16 x i16>
   %2 = zext <16 x i8> %op2 to <16 x i16>
   %mul = mul <16 x i16> %1, %2
@@ -680,19 +511,6 @@ define void @umulh_v32i8(ptr %a, ptr %b) {
 ; SVE2-NEXT:    umulh z1.b, z2.b, z3.b
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    umull2 v4.8h, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    umull v0.8h, v1.8b, v0.8b
-; NONEON-NOSVE-NEXT:    umull2 v1.8h, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    umull v2.8h, v2.8b, v3.8b
-; NONEON-NOSVE-NEXT:    uzp2 v0.16b, v0.16b, v4.16b
-; NONEON-NOSVE-NEXT:    uzp2 v1.16b, v2.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %1 = zext <32 x i8> %op1 to <32 x i16>
@@ -727,15 +545,6 @@ define <2 x i16> @umulh_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; SVE2-NEXT:    lsr z0.s, z0.s, #16
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d2, #0x00ffff0000ffff
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v2.8b
-; NONEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    mul v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ushr v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    ret
   %1 = zext <2 x i16> %op1 to <2 x i32>
   %2 = zext <2 x i16> %op2 to <2 x i32>
   %mul = mul <2 x i32> %1, %2
@@ -761,12 +570,6 @@ define <4 x i16> @umulh_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; SVE2-NEXT:    umulh z0.h, z0.h, z1.h
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umull v0.4s, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    shrn v0.4h, v0.4s, #16
-; NONEON-NOSVE-NEXT:    ret
   %1 = zext <4 x i16> %op1 to <4 x i32>
   %2 = zext <4 x i16> %op2 to <4 x i32>
   %mul = mul <4 x i32> %1, %2
@@ -792,13 +595,6 @@ define <8 x i16> @umulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; SVE2-NEXT:    umulh z0.h, z0.h, z1.h
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umull2 v2.4s, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    umull v0.4s, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
-; NONEON-NOSVE-NEXT:    ret
   %1 = zext <8 x i16> %op1 to <8 x i32>
   %2 = zext <8 x i16> %op2 to <8 x i32>
   %mul = mul <8 x i32> %1, %2
@@ -827,19 +623,6 @@ define void @umulh_v16i16(ptr %a, ptr %b) {
 ; SVE2-NEXT:    umulh z1.h, z2.h, z3.h
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    umull2 v4.4s, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    umull v0.4s, v1.4h, v0.4h
-; NONEON-NOSVE-NEXT:    umull2 v1.4s, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    umull v2.4s, v2.4h, v3.4h
-; NONEON-NOSVE-NEXT:    uzp2 v0.8h, v0.8h, v4.8h
-; NONEON-NOSVE-NEXT:    uzp2 v1.8h, v2.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %1 = zext <16 x i16> %op1 to <16 x i32>
@@ -868,12 +651,6 @@ define <2 x i32> @umulh_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; SVE2-NEXT:    umulh z0.s, z0.s, z1.s
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umull v0.2d, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    shrn v0.2s, v0.2d, #32
-; NONEON-NOSVE-NEXT:    ret
   %1 = zext <2 x i32> %op1 to <2 x i64>
   %2 = zext <2 x i32> %op2 to <2 x i64>
   %mul = mul <2 x i64> %1, %2
@@ -899,13 +676,6 @@ define <4 x i32> @umulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; SVE2-NEXT:    umulh z0.s, z0.s, z1.s
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umull2 v2.2d, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    umull v0.2d, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    uzp2 v0.4s, v0.4s, v2.4s
-; NONEON-NOSVE-NEXT:    ret
   %1 = zext <4 x i32> %op1 to <4 x i64>
   %2 = zext <4 x i32> %op2 to <4 x i64>
   %mul = mul <4 x i64> %1, %2
@@ -934,19 +704,6 @@ define void @umulh_v8i32(ptr %a, ptr %b) {
 ; SVE2-NEXT:    umulh z1.s, z2.s, z3.s
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    umull2 v4.2d, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    umull v0.2d, v1.2s, v0.2s
-; NONEON-NOSVE-NEXT:    umull2 v1.2d, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    umull v2.2d, v2.2s, v3.2s
-; NONEON-NOSVE-NEXT:    uzp2 v0.4s, v0.4s, v4.4s
-; NONEON-NOSVE-NEXT:    uzp2 v1.4s, v2.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %insert = insertelement <8 x i64> undef, i64 32, i64 0
@@ -977,16 +734,6 @@ define <1 x i64> @umulh_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; SVE2-NEXT:    umulh z0.d, z0.d, z1.d
 ; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    fmov x9, d1
-; NONEON-NOSVE-NEXT:    umulh x8, x8, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %1 = zext <1 x i64> %op1 to <1 x i128>
   %2 = zext <1 x i64> %op2 to <1 x i128>
   %mul = mul <1 x i128> %1, %2
@@ -1012,19 +759,6 @@ define <2 x i64> @umulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; SVE2-NEXT:    umulh z0.d, z0.d, z1.d
 ; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov x8, v0.d[1]
-; NONEON-NOSVE-NEXT:    mov x9, v1.d[1]
-; NONEON-NOSVE-NEXT:    fmov x10, d0
-; NONEON-NOSVE-NEXT:    fmov x11, d1
-; NONEON-NOSVE-NEXT:    umulh x10, x10, x11
-; NONEON-NOSVE-NEXT:    umulh x8, x8, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x10
-; NONEON-NOSVE-NEXT:    fmov d1, x8
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    ret
   %1 = zext <2 x i64> %op1 to <2 x i128>
   %2 = zext <2 x i64> %op2 to <2 x i128>
   %mul = mul <2 x i128> %1, %2
@@ -1053,31 +787,6 @@ define void @umulh_v4i64(ptr %a, ptr %b) {
 ; SVE2-NEXT:    umulh z1.d, z2.d, z3.d
 ; SVE2-NEXT:    stp q0, q1, [x0]
 ; SVE2-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umulh_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    mov x11, v0.d[1]
-; NONEON-NOSVE-NEXT:    mov x14, v3.d[1]
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    mov x10, v1.d[1]
-; NONEON-NOSVE-NEXT:    mov x13, v2.d[1]
-; NONEON-NOSVE-NEXT:    fmov x12, d3
-; NONEON-NOSVE-NEXT:    umulh x8, x8, x9
-; NONEON-NOSVE-NEXT:    fmov x9, d2
-; NONEON-NOSVE-NEXT:    umulh x10, x10, x11
-; NONEON-NOSVE-NEXT:    umulh x9, x9, x12
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    umulh x11, x13, x14
-; NONEON-NOSVE-NEXT:    fmov d1, x10
-; NONEON-NOSVE-NEXT:    fmov d2, x9
-; NONEON-NOSVE-NEXT:    mov v0.d[1], v1.d[0]
-; NONEON-NOSVE-NEXT:    fmov d3, x11
-; NONEON-NOSVE-NEXT:    mov v2.d[1], v3.d[0]
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %1 = zext <4 x i64> %op1 to <4 x i128>
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll
index 50eaa6c12d71..751f43768a51 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -18,12 +17,6 @@ define i8 @uaddv_v8i8(<8 x i8> %a) {
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uaddv_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    addv b0, v0.8b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %a)
   ret i8 %res
 }
@@ -37,12 +30,6 @@ define i8 @uaddv_v16i8(<16 x i8> %a) {
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uaddv_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    addv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %a)
   ret i8 %res
 }
@@ -57,14 +44,6 @@ define i8 @uaddv_v32i8(ptr %a) {
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uaddv_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    add v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    addv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %op)
   ret i8 %res
@@ -79,12 +58,6 @@ define i16 @uaddv_v4i16(<4 x i16> %a) {
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uaddv_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    addv h0, v0.4h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a)
   ret i16 %res
 }
@@ -98,12 +71,6 @@ define i16 @uaddv_v8i16(<8 x i16> %a) {
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uaddv_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    addv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %a)
   ret i16 %res
 }
@@ -118,14 +85,6 @@ define i16 @uaddv_v16i16(ptr %a) {
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uaddv_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    add v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    addv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %op)
   ret i16 %res
@@ -140,12 +99,6 @@ define i32 @uaddv_v2i32(<2 x i32> %a) {
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uaddv_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    addp v0.2s, v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a)
   ret i32 %res
 }
@@ -159,12 +112,6 @@ define i32 @uaddv_v4i32(<4 x i32> %a) {
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uaddv_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    addv s0, v0.4s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
   ret i32 %res
 }
@@ -179,14 +126,6 @@ define i32 @uaddv_v8i32(ptr %a) {
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uaddv_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    add v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    addv s0, v0.4s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %op)
   ret i32 %res
@@ -200,12 +139,6 @@ define i64 @uaddv_v2i64(<2 x i64> %a) {
 ; CHECK-NEXT:    uaddv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uaddv_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    addp d0, v0.2d
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a)
   ret i64 %res
 }
@@ -219,14 +152,6 @@ define i64 @uaddv_v4i64(ptr %a) {
 ; CHECK-NEXT:    uaddv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uaddv_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    add v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    addp d0, v0.2d
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %op)
   ret i64 %res
@@ -244,12 +169,6 @@ define i8 @smaxv_v8i8(<8 x i8> %a) {
 ; CHECK-NEXT:    smaxv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smaxv_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smaxv b0, v0.8b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> %a)
   ret i8 %res
 }
@@ -262,12 +181,6 @@ define i8 @smaxv_v16i8(<16 x i8> %a) {
 ; CHECK-NEXT:    smaxv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smaxv_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smaxv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> %a)
   ret i8 %res
 }
@@ -281,14 +194,6 @@ define i8 @smaxv_v32i8(ptr %a) {
 ; CHECK-NEXT:    smaxv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smaxv_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    smax v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    smaxv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> %op)
   ret i8 %res
@@ -302,12 +207,6 @@ define i16 @smaxv_v4i16(<4 x i16> %a) {
 ; CHECK-NEXT:    smaxv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smaxv_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smaxv h0, v0.4h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> %a)
   ret i16 %res
 }
@@ -320,12 +219,6 @@ define i16 @smaxv_v8i16(<8 x i16> %a) {
 ; CHECK-NEXT:    smaxv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smaxv_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smaxv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> %a)
   ret i16 %res
 }
@@ -339,14 +232,6 @@ define i16 @smaxv_v16i16(ptr %a) {
 ; CHECK-NEXT:    smaxv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smaxv_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    smax v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    smaxv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> %op)
   ret i16 %res
@@ -360,12 +245,6 @@ define i32 @smaxv_v2i32(<2 x i32> %a) {
 ; CHECK-NEXT:    smaxv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smaxv_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smaxp v0.2s, v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> %a)
   ret i32 %res
 }
@@ -378,12 +257,6 @@ define i32 @smaxv_v4i32(<4 x i32> %a) {
 ; CHECK-NEXT:    smaxv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smaxv_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smaxv s0, v0.4s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %a)
   ret i32 %res
 }
@@ -397,14 +270,6 @@ define i32 @smaxv_v8i32(ptr %a) {
 ; CHECK-NEXT:    smaxv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smaxv_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    smax v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    smaxv s0, v0.4s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> %op)
   ret i32 %res
@@ -419,17 +284,6 @@ define i64 @smaxv_v2i64(<2 x i64> %a) {
 ; CHECK-NEXT:    smaxv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smaxv_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    cmgt d2, d0, d1
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> %a)
   ret i64 %res
 }
@@ -443,20 +297,6 @@ define i64 @smaxv_v4i64(ptr %a) {
 ; CHECK-NEXT:    smaxv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: smaxv_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    cmgt v2.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bit v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    cmgt d2, d0, d1
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> %op)
   ret i64 %res
@@ -474,12 +314,6 @@ define i8 @sminv_v8i8(<8 x i8> %a) {
 ; CHECK-NEXT:    sminv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sminv_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sminv b0, v0.8b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> %a)
   ret i8 %res
 }
@@ -492,12 +326,6 @@ define i8 @sminv_v16i8(<16 x i8> %a) {
 ; CHECK-NEXT:    sminv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sminv_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sminv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> %a)
   ret i8 %res
 }
@@ -511,14 +339,6 @@ define i8 @sminv_v32i8(ptr %a) {
 ; CHECK-NEXT:    sminv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sminv_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    smin v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    sminv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> %op)
   ret i8 %res
@@ -532,12 +352,6 @@ define i16 @sminv_v4i16(<4 x i16> %a) {
 ; CHECK-NEXT:    sminv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sminv_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sminv h0, v0.4h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> %a)
   ret i16 %res
 }
@@ -550,12 +364,6 @@ define i16 @sminv_v8i16(<8 x i16> %a) {
 ; CHECK-NEXT:    sminv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sminv_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sminv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> %a)
   ret i16 %res
 }
@@ -569,14 +377,6 @@ define i16 @sminv_v16i16(ptr %a) {
 ; CHECK-NEXT:    sminv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sminv_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    smin v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    sminv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> %op)
   ret i16 %res
@@ -590,12 +390,6 @@ define i32 @sminv_v2i32(<2 x i32> %a) {
 ; CHECK-NEXT:    sminv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sminv_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sminp v0.2s, v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> %a)
   ret i32 %res
 }
@@ -608,12 +402,6 @@ define i32 @sminv_v4i32(<4 x i32> %a) {
 ; CHECK-NEXT:    sminv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sminv_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sminv s0, v0.4s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> %a)
   ret i32 %res
 }
@@ -627,14 +415,6 @@ define i32 @sminv_v8i32(ptr %a) {
 ; CHECK-NEXT:    sminv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sminv_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    smin v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    sminv s0, v0.4s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> %op)
   ret i32 %res
@@ -649,17 +429,6 @@ define i64 @sminv_v2i64(<2 x i64> %a) {
 ; CHECK-NEXT:    sminv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sminv_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    cmgt d2, d1, d0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> %a)
   ret i64 %res
 }
@@ -673,20 +442,6 @@ define i64 @sminv_v4i64(ptr %a) {
 ; CHECK-NEXT:    sminv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sminv_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cmgt v2.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    cmgt d2, d1, d0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> %op)
   ret i64 %res
@@ -704,12 +459,6 @@ define i8 @umaxv_v8i8(<8 x i8> %a) {
 ; CHECK-NEXT:    umaxv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umaxv_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umaxv b0, v0.8b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> %a)
   ret i8 %res
 }
@@ -722,12 +471,6 @@ define i8 @umaxv_v16i8(<16 x i8> %a) {
 ; CHECK-NEXT:    umaxv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umaxv_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umaxv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %a)
   ret i8 %res
 }
@@ -741,14 +484,6 @@ define i8 @umaxv_v32i8(ptr %a) {
 ; CHECK-NEXT:    umaxv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umaxv_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    umax v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    umaxv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> %op)
   ret i8 %res
@@ -762,12 +497,6 @@ define i16 @umaxv_v4i16(<4 x i16> %a) {
 ; CHECK-NEXT:    umaxv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umaxv_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umaxv h0, v0.4h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> %a)
   ret i16 %res
 }
@@ -780,12 +509,6 @@ define i16 @umaxv_v8i16(<8 x i16> %a) {
 ; CHECK-NEXT:    umaxv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umaxv_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umaxv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> %a)
   ret i16 %res
 }
@@ -799,14 +522,6 @@ define i16 @umaxv_v16i16(ptr %a) {
 ; CHECK-NEXT:    umaxv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umaxv_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    umax v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    umaxv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> %op)
   ret i16 %res
@@ -820,12 +535,6 @@ define i32 @umaxv_v2i32(<2 x i32> %a) {
 ; CHECK-NEXT:    umaxv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umaxv_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umaxp v0.2s, v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> %a)
   ret i32 %res
 }
@@ -838,12 +547,6 @@ define i32 @umaxv_v4i32(<4 x i32> %a) {
 ; CHECK-NEXT:    umaxv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umaxv_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umaxv s0, v0.4s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> %a)
   ret i32 %res
 }
@@ -857,14 +560,6 @@ define i32 @umaxv_v8i32(ptr %a) {
 ; CHECK-NEXT:    umaxv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umaxv_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    umax v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    umaxv s0, v0.4s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> %op)
   ret i32 %res
@@ -879,17 +574,6 @@ define i64 @umaxv_v2i64(<2 x i64> %a) {
 ; CHECK-NEXT:    umaxv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umaxv_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    cmhi d2, d0, d1
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> %a)
   ret i64 %res
 }
@@ -903,20 +587,6 @@ define i64 @umaxv_v4i64(ptr %a) {
 ; CHECK-NEXT:    umaxv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: umaxv_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    cmhi v2.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bit v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    cmhi d2, d0, d1
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> %op)
   ret i64 %res
@@ -934,12 +604,6 @@ define i8 @uminv_v8i8(<8 x i8> %a) {
 ; CHECK-NEXT:    uminv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uminv_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    uminv b0, v0.8b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> %a)
   ret i8 %res
 }
@@ -952,12 +616,6 @@ define i8 @uminv_v16i8(<16 x i8> %a) {
 ; CHECK-NEXT:    uminv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uminv_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    uminv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> %a)
   ret i8 %res
 }
@@ -971,14 +629,6 @@ define i8 @uminv_v32i8(ptr %a) {
 ; CHECK-NEXT:    uminv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uminv_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    umin v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uminv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> %op)
   ret i8 %res
@@ -992,12 +642,6 @@ define i16 @uminv_v4i16(<4 x i16> %a) {
 ; CHECK-NEXT:    uminv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uminv_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    uminv h0, v0.4h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> %a)
   ret i16 %res
 }
@@ -1010,12 +654,6 @@ define i16 @uminv_v8i16(<8 x i16> %a) {
 ; CHECK-NEXT:    uminv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uminv_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    uminv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> %a)
   ret i16 %res
 }
@@ -1029,14 +667,6 @@ define i16 @uminv_v16i16(ptr %a) {
 ; CHECK-NEXT:    uminv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uminv_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    umin v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    uminv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> %op)
   ret i16 %res
@@ -1050,12 +680,6 @@ define i32 @uminv_v2i32(<2 x i32> %a) {
 ; CHECK-NEXT:    uminv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uminv_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    uminp v0.2s, v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> %a)
   ret i32 %res
 }
@@ -1068,12 +692,6 @@ define i32 @uminv_v4i32(<4 x i32> %a) {
 ; CHECK-NEXT:    uminv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uminv_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    uminv s0, v0.4s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> %a)
   ret i32 %res
 }
@@ -1087,14 +705,6 @@ define i32 @uminv_v8i32(ptr %a) {
 ; CHECK-NEXT:    uminv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uminv_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    umin v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uminv s0, v0.4s
-; NONEON-NOSVE-NEXT:    fmov w0, s0
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> %op)
   ret i32 %res
@@ -1109,17 +719,6 @@ define i64 @uminv_v2i64(<2 x i64> %a) {
 ; CHECK-NEXT:    uminv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uminv_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    cmhi d2, d1, d0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> %a)
   ret i64 %res
 }
@@ -1133,20 +732,6 @@ define i64 @uminv_v4i64(ptr %a) {
 ; CHECK-NEXT:    uminv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uminv_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cmhi v2.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    cmhi d2, d1, d0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> %op)
   ret i64 %res
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
index 97bd76311b61..d373a9063f85 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -25,35 +24,6 @@ define <4 x i8> @srem_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    mls z0.h, p0/m, z2.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    shl v1.4h, v1.4h, #8
-; NONEON-NOSVE-NEXT:    sshr v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    sshr v1.4h, v1.4h, #8
-; NONEON-NOSVE-NEXT:    smov w11, v1.h[0]
-; NONEON-NOSVE-NEXT:    smov w12, v0.h[0]
-; NONEON-NOSVE-NEXT:    smov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    smov w14, v1.h[2]
-; NONEON-NOSVE-NEXT:    smov w15, v0.h[2]
-; NONEON-NOSVE-NEXT:    smov w17, v1.h[3]
-; NONEON-NOSVE-NEXT:    smov w18, v0.h[3]
-; NONEON-NOSVE-NEXT:    sdiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    fmov s0, w11
-; NONEON-NOSVE-NEXT:    sdiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    mov v0.h[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w9, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    mov v0.h[2], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w17, w18
-; NONEON-NOSVE-NEXT:    mov v0.h[3], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = srem <4 x i8> %op1, %op2
   ret <4 x i8> %res
 }
@@ -83,53 +53,6 @@ define <8 x i8> @srem_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    mls z0.b, p0/m, z2.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    smov w11, v1.b[0]
-; NONEON-NOSVE-NEXT:    smov w12, v0.b[0]
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    smov w14, v1.b[2]
-; NONEON-NOSVE-NEXT:    smov w15, v0.b[2]
-; NONEON-NOSVE-NEXT:    smov w17, v1.b[3]
-; NONEON-NOSVE-NEXT:    smov w18, v0.b[3]
-; NONEON-NOSVE-NEXT:    smov w1, v1.b[4]
-; NONEON-NOSVE-NEXT:    smov w2, v0.b[4]
-; NONEON-NOSVE-NEXT:    smov w4, v1.b[5]
-; NONEON-NOSVE-NEXT:    smov w5, v0.b[5]
-; NONEON-NOSVE-NEXT:    sdiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    smov w13, v1.b[7]
-; NONEON-NOSVE-NEXT:    fmov s2, w11
-; NONEON-NOSVE-NEXT:    smov w11, v0.b[6]
-; NONEON-NOSVE-NEXT:    sdiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.b[6]
-; NONEON-NOSVE-NEXT:    mov v2.b[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w0, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    smov w14, v0.b[7]
-; NONEON-NOSVE-NEXT:    mov v2.b[2], w8
-; NONEON-NOSVE-NEXT:    sdiv w3, w2, w1
-; NONEON-NOSVE-NEXT:    msub w8, w0, w17, w18
-; NONEON-NOSVE-NEXT:    mov v2.b[3], w8
-; NONEON-NOSVE-NEXT:    sdiv w9, w5, w4
-; NONEON-NOSVE-NEXT:    msub w8, w3, w1, w2
-; NONEON-NOSVE-NEXT:    mov v2.b[4], w8
-; NONEON-NOSVE-NEXT:    sdiv w12, w11, w10
-; NONEON-NOSVE-NEXT:    msub w8, w9, w4, w5
-; NONEON-NOSVE-NEXT:    mov v2.b[5], w8
-; NONEON-NOSVE-NEXT:    sdiv w9, w14, w13
-; NONEON-NOSVE-NEXT:    msub w8, w12, w10, w11
-; NONEON-NOSVE-NEXT:    mov v2.b[6], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w13, w14
-; NONEON-NOSVE-NEXT:    mov v2.b[7], w8
-; NONEON-NOSVE-NEXT:    fmov d0, d2
-; NONEON-NOSVE-NEXT:    ret
   %res = srem <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -179,112 +102,6 @@ define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    mls z0.b, p0/m, z3.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #-80]! // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #16] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #32] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
-; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
-; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
-; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
-; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
-; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
-; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
-; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
-; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
-; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
-; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
-; NONEON-NOSVE-NEXT:    smov w11, v1.b[0]
-; NONEON-NOSVE-NEXT:    smov w12, v0.b[0]
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    smov w14, v1.b[2]
-; NONEON-NOSVE-NEXT:    smov w15, v0.b[2]
-; NONEON-NOSVE-NEXT:    smov w17, v1.b[3]
-; NONEON-NOSVE-NEXT:    smov w18, v0.b[3]
-; NONEON-NOSVE-NEXT:    smov w1, v1.b[4]
-; NONEON-NOSVE-NEXT:    smov w2, v0.b[4]
-; NONEON-NOSVE-NEXT:    smov w4, v1.b[5]
-; NONEON-NOSVE-NEXT:    smov w5, v0.b[5]
-; NONEON-NOSVE-NEXT:    sdiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    smov w7, v1.b[6]
-; NONEON-NOSVE-NEXT:    smov w19, v0.b[6]
-; NONEON-NOSVE-NEXT:    smov w21, v1.b[7]
-; NONEON-NOSVE-NEXT:    smov w22, v0.b[7]
-; NONEON-NOSVE-NEXT:    smov w24, v1.b[8]
-; NONEON-NOSVE-NEXT:    smov w25, v0.b[8]
-; NONEON-NOSVE-NEXT:    smov w27, v1.b[9]
-; NONEON-NOSVE-NEXT:    smov w28, v0.b[9]
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    smov w13, v1.b[11]
-; NONEON-NOSVE-NEXT:    fmov s2, w11
-; NONEON-NOSVE-NEXT:    smov w11, v0.b[10]
-; NONEON-NOSVE-NEXT:    sdiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.b[10]
-; NONEON-NOSVE-NEXT:    mov v2.b[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w0, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    smov w14, v0.b[11]
-; NONEON-NOSVE-NEXT:    smov w16, v1.b[12]
-; NONEON-NOSVE-NEXT:    mov v2.b[2], w8
-; NONEON-NOSVE-NEXT:    sdiv w3, w2, w1
-; NONEON-NOSVE-NEXT:    msub w8, w0, w17, w18
-; NONEON-NOSVE-NEXT:    smov w17, v0.b[12]
-; NONEON-NOSVE-NEXT:    smov w0, v1.b[13]
-; NONEON-NOSVE-NEXT:    mov v2.b[3], w8
-; NONEON-NOSVE-NEXT:    sdiv w6, w5, w4
-; NONEON-NOSVE-NEXT:    msub w8, w3, w1, w2
-; NONEON-NOSVE-NEXT:    smov w1, v0.b[13]
-; NONEON-NOSVE-NEXT:    mov v2.b[4], w8
-; NONEON-NOSVE-NEXT:    sdiv w20, w19, w7
-; NONEON-NOSVE-NEXT:    msub w8, w6, w4, w5
-; NONEON-NOSVE-NEXT:    mov v2.b[5], w8
-; NONEON-NOSVE-NEXT:    sdiv w23, w22, w21
-; NONEON-NOSVE-NEXT:    msub w8, w20, w7, w19
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v2.b[6], w8
-; NONEON-NOSVE-NEXT:    sdiv w26, w25, w24
-; NONEON-NOSVE-NEXT:    msub w8, w23, w21, w22
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v2.b[7], w8
-; NONEON-NOSVE-NEXT:    sdiv w9, w28, w27
-; NONEON-NOSVE-NEXT:    msub w8, w26, w24, w25
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #32] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #16] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v2.b[8], w8
-; NONEON-NOSVE-NEXT:    sdiv w12, w11, w10
-; NONEON-NOSVE-NEXT:    msub w8, w9, w27, w28
-; NONEON-NOSVE-NEXT:    mov v2.b[9], w8
-; NONEON-NOSVE-NEXT:    sdiv w15, w14, w13
-; NONEON-NOSVE-NEXT:    msub w8, w12, w10, w11
-; NONEON-NOSVE-NEXT:    smov w10, v1.b[14]
-; NONEON-NOSVE-NEXT:    smov w11, v0.b[14]
-; NONEON-NOSVE-NEXT:    mov v2.b[10], w8
-; NONEON-NOSVE-NEXT:    sdiv w18, w17, w16
-; NONEON-NOSVE-NEXT:    msub w8, w15, w13, w14
-; NONEON-NOSVE-NEXT:    smov w13, v1.b[15]
-; NONEON-NOSVE-NEXT:    smov w14, v0.b[15]
-; NONEON-NOSVE-NEXT:    mov v2.b[11], w8
-; NONEON-NOSVE-NEXT:    sdiv w9, w1, w0
-; NONEON-NOSVE-NEXT:    msub w8, w18, w16, w17
-; NONEON-NOSVE-NEXT:    mov v2.b[12], w8
-; NONEON-NOSVE-NEXT:    sdiv w12, w11, w10
-; NONEON-NOSVE-NEXT:    msub w8, w9, w0, w1
-; NONEON-NOSVE-NEXT:    mov v2.b[13], w8
-; NONEON-NOSVE-NEXT:    sdiv w9, w14, w13
-; NONEON-NOSVE-NEXT:    msub w8, w12, w10, w11
-; NONEON-NOSVE-NEXT:    mov v2.b[14], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w13, w14
-; NONEON-NOSVE-NEXT:    mov v2.b[15], w8
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp], #80 // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ret
   %res = srem <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -372,279 +189,6 @@ define void @srem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mls z2.b, p0/m, z7.b, z4.b
 ; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #320
-; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #224] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #240] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #256] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #272] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #288] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #304] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 320
-; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
-; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
-; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
-; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
-; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
-; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
-; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
-; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
-; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
-; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
-; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0]
-; NONEON-NOSVE-NEXT:    str x0, [sp, #216] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    smov w4, v3.b[1]
-; NONEON-NOSVE-NEXT:    smov w1, v2.b[1]
-; NONEON-NOSVE-NEXT:    smov w7, v3.b[7]
-; NONEON-NOSVE-NEXT:    smov w5, v2.b[7]
-; NONEON-NOSVE-NEXT:    smov w6, v3.b[8]
-; NONEON-NOSVE-NEXT:    smov w3, v2.b[8]
-; NONEON-NOSVE-NEXT:    smov w22, v3.b[9]
-; NONEON-NOSVE-NEXT:    smov w20, v2.b[9]
-; NONEON-NOSVE-NEXT:    smov w13, v3.b[0]
-; NONEON-NOSVE-NEXT:    smov w17, v3.b[3]
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    str w8, [sp, #100] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[0]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #108] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[0]
-; NONEON-NOSVE-NEXT:    smov w14, v2.b[3]
-; NONEON-NOSVE-NEXT:    smov w15, v3.b[4]
-; NONEON-NOSVE-NEXT:    smov w12, v2.b[4]
-; NONEON-NOSVE-NEXT:    smov w2, v3.b[5]
-; NONEON-NOSVE-NEXT:    smov w18, v2.b[5]
-; NONEON-NOSVE-NEXT:    smov w0, v3.b[6]
-; NONEON-NOSVE-NEXT:    smov w16, v2.b[6]
-; NONEON-NOSVE-NEXT:    smov w21, v3.b[10]
-; NONEON-NOSVE-NEXT:    smov w19, v2.b[10]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #36] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    ldr w30, [sp, #36] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    str w10, [sp, #116] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[2]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[2]
-; NONEON-NOSVE-NEXT:    stp w10, w8, [sp, #44] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[3]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #52] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[3]
-; NONEON-NOSVE-NEXT:    sdiv w26, w14, w17
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w11, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[4]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[4]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #60] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[5]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[5]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #96] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w9, [sp, #104] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #68] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[6]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[6]
-; NONEON-NOSVE-NEXT:    stp w11, w8, [sp, #80] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #112] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[7]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #88] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[7]
-; NONEON-NOSVE-NEXT:    sdiv w25, w12, w15
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #132] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[8]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[8]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #140] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[9]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[9]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #148] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w9, [sp, #156] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w11, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[10]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[10]
-; NONEON-NOSVE-NEXT:    str w10, [sp, #128] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #204] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[11]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[11]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #192] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #212] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[12]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[12]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #172] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w9, [sp, #180] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #200] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[13]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[13]
-; NONEON-NOSVE-NEXT:    stp w11, w8, [sp, #164] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    smov w11, v3.b[2]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #176] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #188] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.b[14]
-; NONEON-NOSVE-NEXT:    smov w9, v0.b[14]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #144] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w9, [sp, #152] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #184] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w9, v2.b[2]
-; NONEON-NOSVE-NEXT:    sdiv w8, w1, w4
-; NONEON-NOSVE-NEXT:    str w10, [sp, #160] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    smov w10, v2.b[0]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #24] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w8, w5, w7
-; NONEON-NOSVE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w8, w3, w6
-; NONEON-NOSVE-NEXT:    str w8, [sp, #20] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w8, w20, w22
-; NONEON-NOSVE-NEXT:    sdiv w24, w10, w13
-; NONEON-NOSVE-NEXT:    str w8, [sp, #32] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    ldp w29, w8, [sp, #40] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w8, w8, w30, w29
-; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #224] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    fmov s4, w8
-; NONEON-NOSVE-NEXT:    sdiv w23, w9, w11
-; NONEON-NOSVE-NEXT:    msub w10, w24, w13, w10
-; NONEON-NOSVE-NEXT:    ldr w13, [sp, #24] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldr w24, [sp, #100] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w13, w13, w4, w1
-; NONEON-NOSVE-NEXT:    ldr w1, [sp, #116] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldr w4, [sp, #108] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    fmov s5, w10
-; NONEON-NOSVE-NEXT:    msub w1, w1, w24, w4
-; NONEON-NOSVE-NEXT:    mov v5.b[1], w13
-; NONEON-NOSVE-NEXT:    mov v4.b[1], w1
-; NONEON-NOSVE-NEXT:    ldr w1, [sp, #120] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w8, w23, w11, w9
-; NONEON-NOSVE-NEXT:    ldr w11, [sp, #48] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w28, w18, w2
-; NONEON-NOSVE-NEXT:    ldp w10, w9, [sp, #52] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #272] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[2], w8
-; NONEON-NOSVE-NEXT:    msub w8, w26, w17, w14
-; NONEON-NOSVE-NEXT:    ldr w14, [sp, #72] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w11, w10
-; NONEON-NOSVE-NEXT:    ldr w17, [sp, #96] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    smov w10, v3.b[11]
-; NONEON-NOSVE-NEXT:    smov w11, v2.b[11]
-; NONEON-NOSVE-NEXT:    mov v4.b[2], w9
-; NONEON-NOSVE-NEXT:    mov v5.b[3], w8
-; NONEON-NOSVE-NEXT:    msub w8, w25, w15, w12
-; NONEON-NOSVE-NEXT:    ldp w13, w9, [sp, #76] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w27, w16, w0
-; NONEON-NOSVE-NEXT:    ldr w15, [sp, #104] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #256] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w14, w13
-; NONEON-NOSVE-NEXT:    ldr w14, [sp, #60] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[4], w8
-; NONEON-NOSVE-NEXT:    msub w8, w28, w2, w18
-; NONEON-NOSVE-NEXT:    ldr w2, [sp, #156] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[3], w9
-; NONEON-NOSVE-NEXT:    ldp w12, w9, [sp, #64] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[5], w8
-; NONEON-NOSVE-NEXT:    msub w8, w27, w0, w16
-; NONEON-NOSVE-NEXT:    ldr w0, [sp, #132] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w4, w19, w21
-; NONEON-NOSVE-NEXT:    msub w9, w9, w14, w12
-; NONEON-NOSVE-NEXT:    smov w12, v3.b[12]
-; NONEON-NOSVE-NEXT:    smov w14, v2.b[12]
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #240] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[6], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[4], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #112] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w8, w8, w7, w5
-; NONEON-NOSVE-NEXT:    ldr w5, [sp, #204] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w17, w15
-; NONEON-NOSVE-NEXT:    ldr w17, [sp, #84] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[7], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w13, w11, w10
-; NONEON-NOSVE-NEXT:    mov v4.b[5], w9
-; NONEON-NOSVE-NEXT:    ldp w16, w9, [sp, #88] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w8, w8, w6, w3
-; NONEON-NOSVE-NEXT:    ldr w3, [sp, #148] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w17, w16
-; NONEON-NOSVE-NEXT:    smov w16, v3.b[13]
-; NONEON-NOSVE-NEXT:    smov w17, v2.b[13]
-; NONEON-NOSVE-NEXT:    mov v5.b[8], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #32] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[6], w9
-; NONEON-NOSVE-NEXT:    msub w8, w8, w22, w20
-; NONEON-NOSVE-NEXT:    sdiv w15, w14, w12
-; NONEON-NOSVE-NEXT:    ldp w18, w9, [sp, #136] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[9], w8
-; NONEON-NOSVE-NEXT:    msub w8, w4, w21, w19
-; NONEON-NOSVE-NEXT:    msub w9, w9, w0, w18
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #304] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #288] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[7], w9
-; NONEON-NOSVE-NEXT:    mov v5.b[10], w8
-; NONEON-NOSVE-NEXT:    msub w8, w13, w10, w11
-; NONEON-NOSVE-NEXT:    ldp w0, w9, [sp, #124] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp w11, w10, [sp, #196] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldr w13, [sp, #192] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w18, w17, w16
-; NONEON-NOSVE-NEXT:    msub w9, w9, w1, w0
-; NONEON-NOSVE-NEXT:    mov v5.b[11], w8
-; NONEON-NOSVE-NEXT:    smov w0, v3.b[14]
-; NONEON-NOSVE-NEXT:    msub w10, w10, w13, w11
-; NONEON-NOSVE-NEXT:    smov w1, v2.b[14]
-; NONEON-NOSVE-NEXT:    msub w8, w15, w12, w14
-; NONEON-NOSVE-NEXT:    mov v4.b[8], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #164] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp w15, w13, [sp, #168] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w3, w2
-; NONEON-NOSVE-NEXT:    mov v5.b[12], w8
-; NONEON-NOSVE-NEXT:    ldp w4, w3, [sp, #208] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp w14, w12, [sp, #176] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[9], w9
-; NONEON-NOSVE-NEXT:    sdiv w2, w1, w0
-; NONEON-NOSVE-NEXT:    smov w9, v3.b[15]
-; NONEON-NOSVE-NEXT:    msub w3, w3, w5, w4
-; NONEON-NOSVE-NEXT:    smov w4, v2.b[15]
-; NONEON-NOSVE-NEXT:    msub w8, w18, w16, w17
-; NONEON-NOSVE-NEXT:    ldr w16, [sp, #144] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[10], w3
-; NONEON-NOSVE-NEXT:    mov v5.b[13], w8
-; NONEON-NOSVE-NEXT:    mov v4.b[11], w10
-; NONEON-NOSVE-NEXT:    ldr w10, [sp, #188] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w11, w4, w9
-; NONEON-NOSVE-NEXT:    msub w8, w2, w0, w1
-; NONEON-NOSVE-NEXT:    msub w10, w10, w13, w12
-; NONEON-NOSVE-NEXT:    smov w12, v1.b[15]
-; NONEON-NOSVE-NEXT:    smov w13, v0.b[15]
-; NONEON-NOSVE-NEXT:    mov v5.b[14], w8
-; NONEON-NOSVE-NEXT:    mov v4.b[12], w10
-; NONEON-NOSVE-NEXT:    ldr w10, [sp, #184] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w10, w10, w15, w14
-; NONEON-NOSVE-NEXT:    ldr w15, [sp, #152] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w14, w13, w12
-; NONEON-NOSVE-NEXT:    msub w8, w11, w9, w4
-; NONEON-NOSVE-NEXT:    mov v4.b[13], w10
-; NONEON-NOSVE-NEXT:    ldr w10, [sp, #160] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[15], w8
-; NONEON-NOSVE-NEXT:    ldr x8, [sp, #216] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w10, w10, w16, w15
-; NONEON-NOSVE-NEXT:    mov v4.b[14], w10
-; NONEON-NOSVE-NEXT:    msub w9, w14, w12, w13
-; NONEON-NOSVE-NEXT:    mov v4.b[15], w9
-; NONEON-NOSVE-NEXT:    stp q5, q4, [x8]
-; NONEON-NOSVE-NEXT:    add sp, sp, #320
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = srem <32 x i8> %op1, %op2
@@ -666,33 +210,6 @@ define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    mls z0.h, p0/m, z2.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    smov w11, v1.h[0]
-; NONEON-NOSVE-NEXT:    smov w12, v0.h[0]
-; NONEON-NOSVE-NEXT:    smov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    smov w14, v1.h[2]
-; NONEON-NOSVE-NEXT:    smov w15, v0.h[2]
-; NONEON-NOSVE-NEXT:    smov w17, v1.h[3]
-; NONEON-NOSVE-NEXT:    smov w18, v0.h[3]
-; NONEON-NOSVE-NEXT:    sdiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    fmov s0, w11
-; NONEON-NOSVE-NEXT:    sdiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    mov v0.h[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w9, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    mov v0.h[2], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w17, w18
-; NONEON-NOSVE-NEXT:    mov v0.h[3], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = srem <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -721,51 +238,6 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    mls z0.h, p0/m, z3.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    smov w11, v1.h[0]
-; NONEON-NOSVE-NEXT:    smov w12, v0.h[0]
-; NONEON-NOSVE-NEXT:    smov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    smov w14, v1.h[2]
-; NONEON-NOSVE-NEXT:    smov w15, v0.h[2]
-; NONEON-NOSVE-NEXT:    smov w17, v1.h[3]
-; NONEON-NOSVE-NEXT:    smov w18, v0.h[3]
-; NONEON-NOSVE-NEXT:    smov w1, v1.h[4]
-; NONEON-NOSVE-NEXT:    smov w2, v0.h[4]
-; NONEON-NOSVE-NEXT:    smov w4, v1.h[5]
-; NONEON-NOSVE-NEXT:    smov w5, v0.h[5]
-; NONEON-NOSVE-NEXT:    sdiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    smov w13, v1.h[7]
-; NONEON-NOSVE-NEXT:    fmov s2, w11
-; NONEON-NOSVE-NEXT:    smov w11, v0.h[6]
-; NONEON-NOSVE-NEXT:    sdiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    smov w10, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w0, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    smov w14, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    sdiv w3, w2, w1
-; NONEON-NOSVE-NEXT:    msub w8, w0, w17, w18
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    sdiv w9, w5, w4
-; NONEON-NOSVE-NEXT:    msub w8, w3, w1, w2
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    sdiv w12, w11, w10
-; NONEON-NOSVE-NEXT:    msub w8, w9, w4, w5
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    sdiv w9, w14, w13
-; NONEON-NOSVE-NEXT:    msub w8, w12, w10, w11
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w13, w14
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = srem <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -810,139 +282,6 @@ define void @srem_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mls z0.h, p0/m, z7.h, z1.h
 ; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #144
-; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #48] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #64] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #80] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #96] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #112] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #128] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 144
-; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
-; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
-; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
-; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
-; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
-; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
-; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
-; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
-; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
-; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
-; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0]
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    smov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    smov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    smov w20, v1.h[0]
-; NONEON-NOSVE-NEXT:    smov w21, v0.h[0]
-; NONEON-NOSVE-NEXT:    smov w19, v0.h[3]
-; NONEON-NOSVE-NEXT:    smov w5, v1.h[4]
-; NONEON-NOSVE-NEXT:    smov w2, v0.h[4]
-; NONEON-NOSVE-NEXT:    smov w1, v3.h[1]
-; NONEON-NOSVE-NEXT:    smov w23, v2.h[1]
-; NONEON-NOSVE-NEXT:    smov w25, v3.h[0]
-; NONEON-NOSVE-NEXT:    smov w26, v2.h[0]
-; NONEON-NOSVE-NEXT:    smov w6, v1.h[5]
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #36] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    smov w8, v1.h[2]
-; NONEON-NOSVE-NEXT:    smov w9, v0.h[2]
-; NONEON-NOSVE-NEXT:    smov w3, v0.h[5]
-; NONEON-NOSVE-NEXT:    smov w4, v1.h[6]
-; NONEON-NOSVE-NEXT:    smov w7, v0.h[6]
-; NONEON-NOSVE-NEXT:    smov w28, v3.h[2]
-; NONEON-NOSVE-NEXT:    smov w29, v2.h[2]
-; NONEON-NOSVE-NEXT:    smov w15, v3.h[3]
-; NONEON-NOSVE-NEXT:    smov w13, v2.h[3]
-; NONEON-NOSVE-NEXT:    smov w12, v3.h[4]
-; NONEON-NOSVE-NEXT:    smov w14, v3.h[5]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w11, w21, w20
-; NONEON-NOSVE-NEXT:    str w10, [sp, #44] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    smov w8, v1.h[3]
-; NONEON-NOSVE-NEXT:    stp w8, w11, [sp] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    smov w11, v2.h[4]
-; NONEON-NOSVE-NEXT:    ldr w22, [sp, #4] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w20, w22, w20, w21
-; NONEON-NOSVE-NEXT:    sdiv w9, w19, w8
-; NONEON-NOSVE-NEXT:    str w10, [sp, #32] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    smov w10, v3.h[6]
-; NONEON-NOSVE-NEXT:    fmov s5, w20
-; NONEON-NOSVE-NEXT:    smov w20, v3.h[7]
-; NONEON-NOSVE-NEXT:    sdiv w8, w2, w5
-; NONEON-NOSVE-NEXT:    sdiv w24, w23, w1
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    sdiv w27, w26, w25
-; NONEON-NOSVE-NEXT:    msub w1, w24, w1, w23
-; NONEON-NOSVE-NEXT:    ldp w24, w23, [sp, #40] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w9, w3, w6
-; NONEON-NOSVE-NEXT:    msub w21, w27, w25, w26
-; NONEON-NOSVE-NEXT:    ldr w25, [sp, #36] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w23, w23, w25, w24
-; NONEON-NOSVE-NEXT:    ldr w25, [sp, #24] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    fmov s4, w21
-; NONEON-NOSVE-NEXT:    mov v5.h[1], w23
-; NONEON-NOSVE-NEXT:    ldp w23, w21, [sp, #28] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.h[1], w1
-; NONEON-NOSVE-NEXT:    sdiv w8, w7, w4
-; NONEON-NOSVE-NEXT:    msub w21, w21, w25, w23
-; NONEON-NOSVE-NEXT:    smov w23, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #80] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.h[2], w21
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #112] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    sdiv w30, w29, w28
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    smov w9, v2.h[5]
-; NONEON-NOSVE-NEXT:    smov w8, v2.h[6]
-; NONEON-NOSVE-NEXT:    sdiv w18, w13, w15
-; NONEON-NOSVE-NEXT:    msub w1, w30, w28, w29
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #64] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #48] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.h[2], w1
-; NONEON-NOSVE-NEXT:    sdiv w16, w11, w12
-; NONEON-NOSVE-NEXT:    msub w13, w18, w15, w13
-; NONEON-NOSVE-NEXT:    ldr w15, [sp, #20] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldr w18, [sp] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w15, w15, w18, w19
-; NONEON-NOSVE-NEXT:    mov v4.h[3], w13
-; NONEON-NOSVE-NEXT:    smov w13, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov v5.h[3], w15
-; NONEON-NOSVE-NEXT:    smov w15, v0.h[7]
-; NONEON-NOSVE-NEXT:    sdiv w17, w9, w14
-; NONEON-NOSVE-NEXT:    msub w11, w16, w12, w11
-; NONEON-NOSVE-NEXT:    ldr w12, [sp, #16] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w12, w12, w5, w2
-; NONEON-NOSVE-NEXT:    mov v4.h[4], w11
-; NONEON-NOSVE-NEXT:    ldr w11, [sp, #12] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.h[4], w12
-; NONEON-NOSVE-NEXT:    msub w11, w11, w6, w3
-; NONEON-NOSVE-NEXT:    sdiv w24, w8, w10
-; NONEON-NOSVE-NEXT:    msub w9, w17, w14, w9
-; NONEON-NOSVE-NEXT:    mov v5.h[5], w11
-; NONEON-NOSVE-NEXT:    mov v4.h[5], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w4, w7
-; NONEON-NOSVE-NEXT:    sdiv w18, w23, w20
-; NONEON-NOSVE-NEXT:    msub w8, w24, w10, w8
-; NONEON-NOSVE-NEXT:    mov v5.h[6], w9
-; NONEON-NOSVE-NEXT:    mov v4.h[6], w8
-; NONEON-NOSVE-NEXT:    sdiv w12, w15, w13
-; NONEON-NOSVE-NEXT:    msub w8, w18, w20, w23
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #128] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #96] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.h[7], w8
-; NONEON-NOSVE-NEXT:    msub w9, w12, w13, w15
-; NONEON-NOSVE-NEXT:    mov v5.h[7], w9
-; NONEON-NOSVE-NEXT:    stp q4, q5, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #144
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = srem <16 x i16> %op1, %op2
@@ -961,23 +300,6 @@ define <2 x i32> @srem_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    mls z0.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    fmov w9, s0
-; NONEON-NOSVE-NEXT:    mov w11, v1.s[1]
-; NONEON-NOSVE-NEXT:    mov w12, v0.s[1]
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    sdiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    msub w9, w13, w11, w12
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w9
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = srem <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -993,30 +315,6 @@ define <4 x i32> @srem_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    mls z0.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov w11, s1
-; NONEON-NOSVE-NEXT:    fmov w12, s0
-; NONEON-NOSVE-NEXT:    mov w8, v1.s[1]
-; NONEON-NOSVE-NEXT:    mov w9, v0.s[1]
-; NONEON-NOSVE-NEXT:    mov w14, v1.s[2]
-; NONEON-NOSVE-NEXT:    mov w15, v0.s[2]
-; NONEON-NOSVE-NEXT:    mov w17, v1.s[3]
-; NONEON-NOSVE-NEXT:    mov w18, v0.s[3]
-; NONEON-NOSVE-NEXT:    sdiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    fmov s0, w11
-; NONEON-NOSVE-NEXT:    sdiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w9, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    mov v0.s[2], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w17, w18
-; NONEON-NOSVE-NEXT:    mov v0.s[3], w8
-; NONEON-NOSVE-NEXT:    ret
   %res = srem <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -1036,65 +334,6 @@ define void @srem_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mls z1.s, p0/m, z5.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str x23, [sp, #-48]! // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
-; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
-; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
-; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
-; NONEON-NOSVE-NEXT:    .cfi_offset w23, -48
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    fmov w12, s0
-; NONEON-NOSVE-NEXT:    fmov w3, s2
-; NONEON-NOSVE-NEXT:    mov w9, v0.s[1]
-; NONEON-NOSVE-NEXT:    fmov w11, s1
-; NONEON-NOSVE-NEXT:    fmov w2, s3
-; NONEON-NOSVE-NEXT:    mov w8, v1.s[1]
-; NONEON-NOSVE-NEXT:    mov w17, v3.s[1]
-; NONEON-NOSVE-NEXT:    mov w18, v2.s[1]
-; NONEON-NOSVE-NEXT:    mov w14, v1.s[2]
-; NONEON-NOSVE-NEXT:    mov w15, v0.s[2]
-; NONEON-NOSVE-NEXT:    mov w5, v3.s[2]
-; NONEON-NOSVE-NEXT:    mov w6, v2.s[2]
-; NONEON-NOSVE-NEXT:    sdiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    mov w19, v3.s[3]
-; NONEON-NOSVE-NEXT:    mov w20, v2.s[3]
-; NONEON-NOSVE-NEXT:    mov w22, v1.s[3]
-; NONEON-NOSVE-NEXT:    mov w23, v0.s[3]
-; NONEON-NOSVE-NEXT:    sdiv w4, w3, w2
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    fmov s1, w11
-; NONEON-NOSVE-NEXT:    sdiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w12, w4, w2, w3
-; NONEON-NOSVE-NEXT:    fmov s0, w12
-; NONEON-NOSVE-NEXT:    sdiv w1, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    mov v1.s[1], w8
-; NONEON-NOSVE-NEXT:    sdiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w13, w1, w17, w18
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w13
-; NONEON-NOSVE-NEXT:    sdiv w7, w6, w5
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    mov v1.s[2], w8
-; NONEON-NOSVE-NEXT:    sdiv w21, w20, w19
-; NONEON-NOSVE-NEXT:    msub w10, w7, w5, w6
-; NONEON-NOSVE-NEXT:    mov v0.s[2], w10
-; NONEON-NOSVE-NEXT:    sdiv w9, w23, w22
-; NONEON-NOSVE-NEXT:    msub w10, w21, w19, w20
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v0.s[3], w10
-; NONEON-NOSVE-NEXT:    msub w8, w9, w22, w23
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v1.s[3], w8
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr x23, [sp], #48 // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = srem <8 x i32> %op1, %op2
@@ -1113,17 +352,6 @@ define <1 x i64> @srem_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    sdiv x10, x9, x8
-; NONEON-NOSVE-NEXT:    msub x8, x10, x8, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %res = srem <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -1139,20 +367,6 @@ define <2 x i64> @srem_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    mov x11, v1.d[1]
-; NONEON-NOSVE-NEXT:    mov x12, v0.d[1]
-; NONEON-NOSVE-NEXT:    sdiv x10, x9, x8
-; NONEON-NOSVE-NEXT:    sdiv x13, x12, x11
-; NONEON-NOSVE-NEXT:    msub x8, x10, x8, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    msub x9, x13, x11, x12
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x9
-; NONEON-NOSVE-NEXT:    ret
   %res = srem <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -1172,33 +386,6 @@ define void @srem_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mls z1.d, p0/m, z5.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: srem_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    fmov x15, d2
-; NONEON-NOSVE-NEXT:    mov x12, v2.d[1]
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    fmov x14, d3
-; NONEON-NOSVE-NEXT:    mov x11, v3.d[1]
-; NONEON-NOSVE-NEXT:    mov x17, v1.d[1]
-; NONEON-NOSVE-NEXT:    mov x18, v0.d[1]
-; NONEON-NOSVE-NEXT:    sdiv x10, x9, x8
-; NONEON-NOSVE-NEXT:    sdiv x16, x15, x14
-; NONEON-NOSVE-NEXT:    msub x8, x10, x8, x9
-; NONEON-NOSVE-NEXT:    fmov d1, x8
-; NONEON-NOSVE-NEXT:    sdiv x13, x12, x11
-; NONEON-NOSVE-NEXT:    msub x10, x16, x14, x15
-; NONEON-NOSVE-NEXT:    fmov d0, x10
-; NONEON-NOSVE-NEXT:    sdiv x1, x18, x17
-; NONEON-NOSVE-NEXT:    msub x9, x13, x11, x12
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x9
-; NONEON-NOSVE-NEXT:    msub x11, x1, x17, x18
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x11
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = srem <4 x i64> %op1, %op2
@@ -1226,41 +413,6 @@ define <4 x i8> @urem_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    mls z0.h, p0/m, z2.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    umov w11, v1.h[0]
-; NONEON-NOSVE-NEXT:    umov w12, v0.h[0]
-; NONEON-NOSVE-NEXT:    umov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    umov w14, v1.h[2]
-; NONEON-NOSVE-NEXT:    umov w15, v0.h[2]
-; NONEON-NOSVE-NEXT:    umov w17, v1.h[3]
-; NONEON-NOSVE-NEXT:    umov w18, v0.h[3]
-; NONEON-NOSVE-NEXT:    and w11, w11, #0xff
-; NONEON-NOSVE-NEXT:    and w12, w12, #0xff
-; NONEON-NOSVE-NEXT:    and w8, w8, #0xff
-; NONEON-NOSVE-NEXT:    udiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    and w9, w9, #0xff
-; NONEON-NOSVE-NEXT:    and w14, w14, #0xff
-; NONEON-NOSVE-NEXT:    and w15, w15, #0xff
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    and w12, w17, #0xff
-; NONEON-NOSVE-NEXT:    and w13, w18, #0xff
-; NONEON-NOSVE-NEXT:    fmov s0, w11
-; NONEON-NOSVE-NEXT:    udiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    mov v0.h[1], w8
-; NONEON-NOSVE-NEXT:    udiv w9, w13, w12
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    mov v0.h[2], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w12, w13
-; NONEON-NOSVE-NEXT:    mov v0.h[3], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = urem <4 x i8> %op1, %op2
   ret <4 x i8> %res
 }
@@ -1290,53 +442,6 @@ define <8 x i8> @urem_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    mls z0.b, p0/m, z2.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    umov w11, v1.b[0]
-; NONEON-NOSVE-NEXT:    umov w12, v0.b[0]
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    umov w14, v1.b[2]
-; NONEON-NOSVE-NEXT:    umov w15, v0.b[2]
-; NONEON-NOSVE-NEXT:    umov w17, v1.b[3]
-; NONEON-NOSVE-NEXT:    umov w18, v0.b[3]
-; NONEON-NOSVE-NEXT:    umov w1, v1.b[4]
-; NONEON-NOSVE-NEXT:    umov w2, v0.b[4]
-; NONEON-NOSVE-NEXT:    umov w4, v1.b[5]
-; NONEON-NOSVE-NEXT:    umov w5, v0.b[5]
-; NONEON-NOSVE-NEXT:    udiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    umov w13, v1.b[7]
-; NONEON-NOSVE-NEXT:    fmov s2, w11
-; NONEON-NOSVE-NEXT:    umov w11, v0.b[6]
-; NONEON-NOSVE-NEXT:    udiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.b[6]
-; NONEON-NOSVE-NEXT:    mov v2.b[1], w8
-; NONEON-NOSVE-NEXT:    udiv w0, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    umov w14, v0.b[7]
-; NONEON-NOSVE-NEXT:    mov v2.b[2], w8
-; NONEON-NOSVE-NEXT:    udiv w3, w2, w1
-; NONEON-NOSVE-NEXT:    msub w8, w0, w17, w18
-; NONEON-NOSVE-NEXT:    mov v2.b[3], w8
-; NONEON-NOSVE-NEXT:    udiv w9, w5, w4
-; NONEON-NOSVE-NEXT:    msub w8, w3, w1, w2
-; NONEON-NOSVE-NEXT:    mov v2.b[4], w8
-; NONEON-NOSVE-NEXT:    udiv w12, w11, w10
-; NONEON-NOSVE-NEXT:    msub w8, w9, w4, w5
-; NONEON-NOSVE-NEXT:    mov v2.b[5], w8
-; NONEON-NOSVE-NEXT:    udiv w9, w14, w13
-; NONEON-NOSVE-NEXT:    msub w8, w12, w10, w11
-; NONEON-NOSVE-NEXT:    mov v2.b[6], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w13, w14
-; NONEON-NOSVE-NEXT:    mov v2.b[7], w8
-; NONEON-NOSVE-NEXT:    fmov d0, d2
-; NONEON-NOSVE-NEXT:    ret
   %res = urem <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -1386,112 +491,6 @@ define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    mls z0.b, p0/m, z3.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #-80]! // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #16] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #32] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
-; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
-; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
-; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
-; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
-; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
-; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
-; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
-; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
-; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
-; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
-; NONEON-NOSVE-NEXT:    umov w11, v1.b[0]
-; NONEON-NOSVE-NEXT:    umov w12, v0.b[0]
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    umov w14, v1.b[2]
-; NONEON-NOSVE-NEXT:    umov w15, v0.b[2]
-; NONEON-NOSVE-NEXT:    umov w17, v1.b[3]
-; NONEON-NOSVE-NEXT:    umov w18, v0.b[3]
-; NONEON-NOSVE-NEXT:    umov w1, v1.b[4]
-; NONEON-NOSVE-NEXT:    umov w2, v0.b[4]
-; NONEON-NOSVE-NEXT:    umov w4, v1.b[5]
-; NONEON-NOSVE-NEXT:    umov w5, v0.b[5]
-; NONEON-NOSVE-NEXT:    udiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    umov w7, v1.b[6]
-; NONEON-NOSVE-NEXT:    umov w19, v0.b[6]
-; NONEON-NOSVE-NEXT:    umov w21, v1.b[7]
-; NONEON-NOSVE-NEXT:    umov w22, v0.b[7]
-; NONEON-NOSVE-NEXT:    umov w24, v1.b[8]
-; NONEON-NOSVE-NEXT:    umov w25, v0.b[8]
-; NONEON-NOSVE-NEXT:    umov w27, v1.b[9]
-; NONEON-NOSVE-NEXT:    umov w28, v0.b[9]
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    umov w13, v1.b[11]
-; NONEON-NOSVE-NEXT:    fmov s2, w11
-; NONEON-NOSVE-NEXT:    umov w11, v0.b[10]
-; NONEON-NOSVE-NEXT:    udiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.b[10]
-; NONEON-NOSVE-NEXT:    mov v2.b[1], w8
-; NONEON-NOSVE-NEXT:    udiv w0, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    umov w14, v0.b[11]
-; NONEON-NOSVE-NEXT:    umov w16, v1.b[12]
-; NONEON-NOSVE-NEXT:    mov v2.b[2], w8
-; NONEON-NOSVE-NEXT:    udiv w3, w2, w1
-; NONEON-NOSVE-NEXT:    msub w8, w0, w17, w18
-; NONEON-NOSVE-NEXT:    umov w17, v0.b[12]
-; NONEON-NOSVE-NEXT:    umov w0, v1.b[13]
-; NONEON-NOSVE-NEXT:    mov v2.b[3], w8
-; NONEON-NOSVE-NEXT:    udiv w6, w5, w4
-; NONEON-NOSVE-NEXT:    msub w8, w3, w1, w2
-; NONEON-NOSVE-NEXT:    umov w1, v0.b[13]
-; NONEON-NOSVE-NEXT:    mov v2.b[4], w8
-; NONEON-NOSVE-NEXT:    udiv w20, w19, w7
-; NONEON-NOSVE-NEXT:    msub w8, w6, w4, w5
-; NONEON-NOSVE-NEXT:    mov v2.b[5], w8
-; NONEON-NOSVE-NEXT:    udiv w23, w22, w21
-; NONEON-NOSVE-NEXT:    msub w8, w20, w7, w19
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v2.b[6], w8
-; NONEON-NOSVE-NEXT:    udiv w26, w25, w24
-; NONEON-NOSVE-NEXT:    msub w8, w23, w21, w22
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v2.b[7], w8
-; NONEON-NOSVE-NEXT:    udiv w9, w28, w27
-; NONEON-NOSVE-NEXT:    msub w8, w26, w24, w25
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #32] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #16] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v2.b[8], w8
-; NONEON-NOSVE-NEXT:    udiv w12, w11, w10
-; NONEON-NOSVE-NEXT:    msub w8, w9, w27, w28
-; NONEON-NOSVE-NEXT:    mov v2.b[9], w8
-; NONEON-NOSVE-NEXT:    udiv w15, w14, w13
-; NONEON-NOSVE-NEXT:    msub w8, w12, w10, w11
-; NONEON-NOSVE-NEXT:    umov w10, v1.b[14]
-; NONEON-NOSVE-NEXT:    umov w11, v0.b[14]
-; NONEON-NOSVE-NEXT:    mov v2.b[10], w8
-; NONEON-NOSVE-NEXT:    udiv w18, w17, w16
-; NONEON-NOSVE-NEXT:    msub w8, w15, w13, w14
-; NONEON-NOSVE-NEXT:    umov w13, v1.b[15]
-; NONEON-NOSVE-NEXT:    umov w14, v0.b[15]
-; NONEON-NOSVE-NEXT:    mov v2.b[11], w8
-; NONEON-NOSVE-NEXT:    udiv w9, w1, w0
-; NONEON-NOSVE-NEXT:    msub w8, w18, w16, w17
-; NONEON-NOSVE-NEXT:    mov v2.b[12], w8
-; NONEON-NOSVE-NEXT:    udiv w12, w11, w10
-; NONEON-NOSVE-NEXT:    msub w8, w9, w0, w1
-; NONEON-NOSVE-NEXT:    mov v2.b[13], w8
-; NONEON-NOSVE-NEXT:    udiv w9, w14, w13
-; NONEON-NOSVE-NEXT:    msub w8, w12, w10, w11
-; NONEON-NOSVE-NEXT:    mov v2.b[14], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w13, w14
-; NONEON-NOSVE-NEXT:    mov v2.b[15], w8
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp], #80 // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ret
   %res = urem <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -1579,279 +578,6 @@ define void @urem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mls z2.b, p0/m, z7.b, z4.b
 ; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #320
-; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #224] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #240] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #256] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #272] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #288] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #304] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 320
-; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
-; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
-; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
-; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
-; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
-; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
-; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
-; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
-; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
-; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
-; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0]
-; NONEON-NOSVE-NEXT:    str x0, [sp, #216] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[1]
-; NONEON-NOSVE-NEXT:    umov w4, v3.b[1]
-; NONEON-NOSVE-NEXT:    umov w1, v2.b[1]
-; NONEON-NOSVE-NEXT:    umov w7, v3.b[7]
-; NONEON-NOSVE-NEXT:    umov w5, v2.b[7]
-; NONEON-NOSVE-NEXT:    umov w6, v3.b[8]
-; NONEON-NOSVE-NEXT:    umov w3, v2.b[8]
-; NONEON-NOSVE-NEXT:    umov w22, v3.b[9]
-; NONEON-NOSVE-NEXT:    umov w20, v2.b[9]
-; NONEON-NOSVE-NEXT:    umov w13, v3.b[0]
-; NONEON-NOSVE-NEXT:    umov w17, v3.b[3]
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    str w8, [sp, #100] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[0]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #108] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[0]
-; NONEON-NOSVE-NEXT:    umov w14, v2.b[3]
-; NONEON-NOSVE-NEXT:    umov w15, v3.b[4]
-; NONEON-NOSVE-NEXT:    umov w12, v2.b[4]
-; NONEON-NOSVE-NEXT:    umov w2, v3.b[5]
-; NONEON-NOSVE-NEXT:    umov w18, v2.b[5]
-; NONEON-NOSVE-NEXT:    umov w0, v3.b[6]
-; NONEON-NOSVE-NEXT:    umov w16, v2.b[6]
-; NONEON-NOSVE-NEXT:    umov w21, v3.b[10]
-; NONEON-NOSVE-NEXT:    umov w19, v2.b[10]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #36] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    ldr w30, [sp, #36] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    str w10, [sp, #116] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[2]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[2]
-; NONEON-NOSVE-NEXT:    stp w10, w8, [sp, #44] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[3]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #52] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[3]
-; NONEON-NOSVE-NEXT:    udiv w26, w14, w17
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w11, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[4]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[4]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #60] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[5]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[5]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #96] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w9, [sp, #104] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #68] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[6]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[6]
-; NONEON-NOSVE-NEXT:    stp w11, w8, [sp, #80] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #112] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[7]
-; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #88] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[7]
-; NONEON-NOSVE-NEXT:    udiv w25, w12, w15
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #132] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[8]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[8]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #140] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[9]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[9]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #148] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w9, [sp, #156] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w11, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[10]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[10]
-; NONEON-NOSVE-NEXT:    str w10, [sp, #128] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #204] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[11]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[11]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #192] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #212] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[12]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[12]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #172] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w9, [sp, #180] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #200] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[13]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[13]
-; NONEON-NOSVE-NEXT:    stp w11, w8, [sp, #164] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    umov w11, v3.b[2]
-; NONEON-NOSVE-NEXT:    str w9, [sp, #176] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #188] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.b[14]
-; NONEON-NOSVE-NEXT:    umov w9, v0.b[14]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #144] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w9, [sp, #152] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    str w10, [sp, #184] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w9, v2.b[2]
-; NONEON-NOSVE-NEXT:    udiv w8, w1, w4
-; NONEON-NOSVE-NEXT:    str w10, [sp, #160] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    umov w10, v2.b[0]
-; NONEON-NOSVE-NEXT:    str w8, [sp, #24] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w8, w5, w7
-; NONEON-NOSVE-NEXT:    str w8, [sp, #28] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w8, w3, w6
-; NONEON-NOSVE-NEXT:    str w8, [sp, #20] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w8, w20, w22
-; NONEON-NOSVE-NEXT:    udiv w24, w10, w13
-; NONEON-NOSVE-NEXT:    str w8, [sp, #32] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    ldp w29, w8, [sp, #40] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w8, w8, w30, w29
-; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #224] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    fmov s4, w8
-; NONEON-NOSVE-NEXT:    udiv w23, w9, w11
-; NONEON-NOSVE-NEXT:    msub w10, w24, w13, w10
-; NONEON-NOSVE-NEXT:    ldr w13, [sp, #24] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldr w24, [sp, #100] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w13, w13, w4, w1
-; NONEON-NOSVE-NEXT:    ldr w1, [sp, #116] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldr w4, [sp, #108] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    fmov s5, w10
-; NONEON-NOSVE-NEXT:    msub w1, w1, w24, w4
-; NONEON-NOSVE-NEXT:    mov v5.b[1], w13
-; NONEON-NOSVE-NEXT:    mov v4.b[1], w1
-; NONEON-NOSVE-NEXT:    ldr w1, [sp, #120] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w8, w23, w11, w9
-; NONEON-NOSVE-NEXT:    ldr w11, [sp, #48] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w28, w18, w2
-; NONEON-NOSVE-NEXT:    ldp w10, w9, [sp, #52] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #272] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[2], w8
-; NONEON-NOSVE-NEXT:    msub w8, w26, w17, w14
-; NONEON-NOSVE-NEXT:    ldr w14, [sp, #72] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w11, w10
-; NONEON-NOSVE-NEXT:    ldr w17, [sp, #96] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    umov w10, v3.b[11]
-; NONEON-NOSVE-NEXT:    umov w11, v2.b[11]
-; NONEON-NOSVE-NEXT:    mov v4.b[2], w9
-; NONEON-NOSVE-NEXT:    mov v5.b[3], w8
-; NONEON-NOSVE-NEXT:    msub w8, w25, w15, w12
-; NONEON-NOSVE-NEXT:    ldp w13, w9, [sp, #76] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w27, w16, w0
-; NONEON-NOSVE-NEXT:    ldr w15, [sp, #104] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #256] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w14, w13
-; NONEON-NOSVE-NEXT:    ldr w14, [sp, #60] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[4], w8
-; NONEON-NOSVE-NEXT:    msub w8, w28, w2, w18
-; NONEON-NOSVE-NEXT:    ldr w2, [sp, #156] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[3], w9
-; NONEON-NOSVE-NEXT:    ldp w12, w9, [sp, #64] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[5], w8
-; NONEON-NOSVE-NEXT:    msub w8, w27, w0, w16
-; NONEON-NOSVE-NEXT:    ldr w0, [sp, #132] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w4, w19, w21
-; NONEON-NOSVE-NEXT:    msub w9, w9, w14, w12
-; NONEON-NOSVE-NEXT:    umov w12, v3.b[12]
-; NONEON-NOSVE-NEXT:    umov w14, v2.b[12]
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #240] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[6], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[4], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #112] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w8, w8, w7, w5
-; NONEON-NOSVE-NEXT:    ldr w5, [sp, #204] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w17, w15
-; NONEON-NOSVE-NEXT:    ldr w17, [sp, #84] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[7], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w13, w11, w10
-; NONEON-NOSVE-NEXT:    mov v4.b[5], w9
-; NONEON-NOSVE-NEXT:    ldp w16, w9, [sp, #88] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w8, w8, w6, w3
-; NONEON-NOSVE-NEXT:    ldr w3, [sp, #148] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w17, w16
-; NONEON-NOSVE-NEXT:    umov w16, v3.b[13]
-; NONEON-NOSVE-NEXT:    umov w17, v2.b[13]
-; NONEON-NOSVE-NEXT:    mov v5.b[8], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #32] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[6], w9
-; NONEON-NOSVE-NEXT:    msub w8, w8, w22, w20
-; NONEON-NOSVE-NEXT:    udiv w15, w14, w12
-; NONEON-NOSVE-NEXT:    ldp w18, w9, [sp, #136] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[9], w8
-; NONEON-NOSVE-NEXT:    msub w8, w4, w21, w19
-; NONEON-NOSVE-NEXT:    msub w9, w9, w0, w18
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #304] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #288] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[7], w9
-; NONEON-NOSVE-NEXT:    mov v5.b[10], w8
-; NONEON-NOSVE-NEXT:    msub w8, w13, w10, w11
-; NONEON-NOSVE-NEXT:    ldp w0, w9, [sp, #124] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp w11, w10, [sp, #196] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldr w13, [sp, #192] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w18, w17, w16
-; NONEON-NOSVE-NEXT:    msub w9, w9, w1, w0
-; NONEON-NOSVE-NEXT:    mov v5.b[11], w8
-; NONEON-NOSVE-NEXT:    umov w0, v3.b[14]
-; NONEON-NOSVE-NEXT:    msub w10, w10, w13, w11
-; NONEON-NOSVE-NEXT:    umov w1, v2.b[14]
-; NONEON-NOSVE-NEXT:    msub w8, w15, w12, w14
-; NONEON-NOSVE-NEXT:    mov v4.b[8], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #164] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp w15, w13, [sp, #168] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w3, w2
-; NONEON-NOSVE-NEXT:    mov v5.b[12], w8
-; NONEON-NOSVE-NEXT:    ldp w4, w3, [sp, #208] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp w14, w12, [sp, #176] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[9], w9
-; NONEON-NOSVE-NEXT:    udiv w2, w1, w0
-; NONEON-NOSVE-NEXT:    umov w9, v3.b[15]
-; NONEON-NOSVE-NEXT:    msub w3, w3, w5, w4
-; NONEON-NOSVE-NEXT:    umov w4, v2.b[15]
-; NONEON-NOSVE-NEXT:    msub w8, w18, w16, w17
-; NONEON-NOSVE-NEXT:    ldr w16, [sp, #144] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.b[10], w3
-; NONEON-NOSVE-NEXT:    mov v5.b[13], w8
-; NONEON-NOSVE-NEXT:    mov v4.b[11], w10
-; NONEON-NOSVE-NEXT:    ldr w10, [sp, #188] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w11, w4, w9
-; NONEON-NOSVE-NEXT:    msub w8, w2, w0, w1
-; NONEON-NOSVE-NEXT:    msub w10, w10, w13, w12
-; NONEON-NOSVE-NEXT:    umov w12, v1.b[15]
-; NONEON-NOSVE-NEXT:    umov w13, v0.b[15]
-; NONEON-NOSVE-NEXT:    mov v5.b[14], w8
-; NONEON-NOSVE-NEXT:    mov v4.b[12], w10
-; NONEON-NOSVE-NEXT:    ldr w10, [sp, #184] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w10, w10, w15, w14
-; NONEON-NOSVE-NEXT:    ldr w15, [sp, #152] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w14, w13, w12
-; NONEON-NOSVE-NEXT:    msub w8, w11, w9, w4
-; NONEON-NOSVE-NEXT:    mov v4.b[13], w10
-; NONEON-NOSVE-NEXT:    ldr w10, [sp, #160] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.b[15], w8
-; NONEON-NOSVE-NEXT:    ldr x8, [sp, #216] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w10, w10, w16, w15
-; NONEON-NOSVE-NEXT:    mov v4.b[14], w10
-; NONEON-NOSVE-NEXT:    msub w9, w14, w12, w13
-; NONEON-NOSVE-NEXT:    mov v4.b[15], w9
-; NONEON-NOSVE-NEXT:    stp q5, q4, [x8]
-; NONEON-NOSVE-NEXT:    add sp, sp, #320
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = urem <32 x i8> %op1, %op2
@@ -1873,33 +599,6 @@ define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    mls z0.h, p0/m, z2.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    umov w11, v1.h[0]
-; NONEON-NOSVE-NEXT:    umov w12, v0.h[0]
-; NONEON-NOSVE-NEXT:    umov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    umov w14, v1.h[2]
-; NONEON-NOSVE-NEXT:    umov w15, v0.h[2]
-; NONEON-NOSVE-NEXT:    umov w17, v1.h[3]
-; NONEON-NOSVE-NEXT:    umov w18, v0.h[3]
-; NONEON-NOSVE-NEXT:    udiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    fmov s0, w11
-; NONEON-NOSVE-NEXT:    udiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    mov v0.h[1], w8
-; NONEON-NOSVE-NEXT:    udiv w9, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    mov v0.h[2], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w17, w18
-; NONEON-NOSVE-NEXT:    mov v0.h[3], w8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = urem <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -1928,51 +627,6 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    mls z0.h, p0/m, z3.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    umov w11, v1.h[0]
-; NONEON-NOSVE-NEXT:    umov w12, v0.h[0]
-; NONEON-NOSVE-NEXT:    umov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    umov w14, v1.h[2]
-; NONEON-NOSVE-NEXT:    umov w15, v0.h[2]
-; NONEON-NOSVE-NEXT:    umov w17, v1.h[3]
-; NONEON-NOSVE-NEXT:    umov w18, v0.h[3]
-; NONEON-NOSVE-NEXT:    umov w1, v1.h[4]
-; NONEON-NOSVE-NEXT:    umov w2, v0.h[4]
-; NONEON-NOSVE-NEXT:    umov w4, v1.h[5]
-; NONEON-NOSVE-NEXT:    umov w5, v0.h[5]
-; NONEON-NOSVE-NEXT:    udiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    umov w13, v1.h[7]
-; NONEON-NOSVE-NEXT:    fmov s2, w11
-; NONEON-NOSVE-NEXT:    umov w11, v0.h[6]
-; NONEON-NOSVE-NEXT:    udiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    umov w10, v1.h[6]
-; NONEON-NOSVE-NEXT:    mov v2.h[1], w8
-; NONEON-NOSVE-NEXT:    udiv w0, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    umov w14, v0.h[7]
-; NONEON-NOSVE-NEXT:    mov v2.h[2], w8
-; NONEON-NOSVE-NEXT:    udiv w3, w2, w1
-; NONEON-NOSVE-NEXT:    msub w8, w0, w17, w18
-; NONEON-NOSVE-NEXT:    mov v2.h[3], w8
-; NONEON-NOSVE-NEXT:    udiv w9, w5, w4
-; NONEON-NOSVE-NEXT:    msub w8, w3, w1, w2
-; NONEON-NOSVE-NEXT:    mov v2.h[4], w8
-; NONEON-NOSVE-NEXT:    udiv w12, w11, w10
-; NONEON-NOSVE-NEXT:    msub w8, w9, w4, w5
-; NONEON-NOSVE-NEXT:    mov v2.h[5], w8
-; NONEON-NOSVE-NEXT:    udiv w9, w14, w13
-; NONEON-NOSVE-NEXT:    msub w8, w12, w10, w11
-; NONEON-NOSVE-NEXT:    mov v2.h[6], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w13, w14
-; NONEON-NOSVE-NEXT:    mov v2.h[7], w8
-; NONEON-NOSVE-NEXT:    mov v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = urem <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -2017,139 +671,6 @@ define void @urem_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mls z0.h, p0/m, z7.h, z1.h
 ; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #144
-; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #48] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #64] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #80] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #96] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #112] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #128] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 144
-; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
-; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
-; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
-; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
-; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
-; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
-; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
-; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
-; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
-; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
-; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
-; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0]
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    umov w8, v1.h[1]
-; NONEON-NOSVE-NEXT:    umov w9, v0.h[1]
-; NONEON-NOSVE-NEXT:    umov w20, v1.h[0]
-; NONEON-NOSVE-NEXT:    umov w21, v0.h[0]
-; NONEON-NOSVE-NEXT:    umov w19, v0.h[3]
-; NONEON-NOSVE-NEXT:    umov w5, v1.h[4]
-; NONEON-NOSVE-NEXT:    umov w2, v0.h[4]
-; NONEON-NOSVE-NEXT:    umov w1, v3.h[1]
-; NONEON-NOSVE-NEXT:    umov w23, v2.h[1]
-; NONEON-NOSVE-NEXT:    umov w25, v3.h[0]
-; NONEON-NOSVE-NEXT:    umov w26, v2.h[0]
-; NONEON-NOSVE-NEXT:    umov w6, v1.h[5]
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #36] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    umov w8, v1.h[2]
-; NONEON-NOSVE-NEXT:    umov w9, v0.h[2]
-; NONEON-NOSVE-NEXT:    umov w3, v0.h[5]
-; NONEON-NOSVE-NEXT:    umov w4, v1.h[6]
-; NONEON-NOSVE-NEXT:    umov w7, v0.h[6]
-; NONEON-NOSVE-NEXT:    umov w28, v3.h[2]
-; NONEON-NOSVE-NEXT:    umov w29, v2.h[2]
-; NONEON-NOSVE-NEXT:    umov w15, v3.h[3]
-; NONEON-NOSVE-NEXT:    umov w13, v2.h[3]
-; NONEON-NOSVE-NEXT:    umov w12, v3.h[4]
-; NONEON-NOSVE-NEXT:    umov w14, v3.h[5]
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w11, w21, w20
-; NONEON-NOSVE-NEXT:    str w10, [sp, #44] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    umov w8, v1.h[3]
-; NONEON-NOSVE-NEXT:    stp w8, w11, [sp] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    umov w11, v2.h[4]
-; NONEON-NOSVE-NEXT:    ldr w22, [sp, #4] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w20, w22, w20, w21
-; NONEON-NOSVE-NEXT:    udiv w9, w19, w8
-; NONEON-NOSVE-NEXT:    str w10, [sp, #32] // 4-byte Folded Spill
-; NONEON-NOSVE-NEXT:    umov w10, v3.h[6]
-; NONEON-NOSVE-NEXT:    fmov s5, w20
-; NONEON-NOSVE-NEXT:    umov w20, v3.h[7]
-; NONEON-NOSVE-NEXT:    udiv w8, w2, w5
-; NONEON-NOSVE-NEXT:    udiv w24, w23, w1
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    udiv w27, w26, w25
-; NONEON-NOSVE-NEXT:    msub w1, w24, w1, w23
-; NONEON-NOSVE-NEXT:    ldp w24, w23, [sp, #40] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w9, w3, w6
-; NONEON-NOSVE-NEXT:    msub w21, w27, w25, w26
-; NONEON-NOSVE-NEXT:    ldr w25, [sp, #36] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w23, w23, w25, w24
-; NONEON-NOSVE-NEXT:    ldr w25, [sp, #24] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    fmov s4, w21
-; NONEON-NOSVE-NEXT:    mov v5.h[1], w23
-; NONEON-NOSVE-NEXT:    ldp w23, w21, [sp, #28] // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.h[1], w1
-; NONEON-NOSVE-NEXT:    udiv w8, w7, w4
-; NONEON-NOSVE-NEXT:    msub w21, w21, w25, w23
-; NONEON-NOSVE-NEXT:    umov w23, v2.h[7]
-; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #80] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.h[2], w21
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #112] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    udiv w30, w29, w28
-; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    umov w9, v2.h[5]
-; NONEON-NOSVE-NEXT:    umov w8, v2.h[6]
-; NONEON-NOSVE-NEXT:    udiv w18, w13, w15
-; NONEON-NOSVE-NEXT:    msub w1, w30, w28, w29
-; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #64] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #48] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.h[2], w1
-; NONEON-NOSVE-NEXT:    udiv w16, w11, w12
-; NONEON-NOSVE-NEXT:    msub w13, w18, w15, w13
-; NONEON-NOSVE-NEXT:    ldr w15, [sp, #20] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldr w18, [sp] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w15, w15, w18, w19
-; NONEON-NOSVE-NEXT:    mov v4.h[3], w13
-; NONEON-NOSVE-NEXT:    umov w13, v1.h[7]
-; NONEON-NOSVE-NEXT:    mov v5.h[3], w15
-; NONEON-NOSVE-NEXT:    umov w15, v0.h[7]
-; NONEON-NOSVE-NEXT:    udiv w17, w9, w14
-; NONEON-NOSVE-NEXT:    msub w11, w16, w12, w11
-; NONEON-NOSVE-NEXT:    ldr w12, [sp, #16] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w12, w12, w5, w2
-; NONEON-NOSVE-NEXT:    mov v4.h[4], w11
-; NONEON-NOSVE-NEXT:    ldr w11, [sp, #12] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v5.h[4], w12
-; NONEON-NOSVE-NEXT:    msub w11, w11, w6, w3
-; NONEON-NOSVE-NEXT:    udiv w24, w8, w10
-; NONEON-NOSVE-NEXT:    msub w9, w17, w14, w9
-; NONEON-NOSVE-NEXT:    mov v5.h[5], w11
-; NONEON-NOSVE-NEXT:    mov v4.h[5], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
-; NONEON-NOSVE-NEXT:    msub w9, w9, w4, w7
-; NONEON-NOSVE-NEXT:    udiv w18, w23, w20
-; NONEON-NOSVE-NEXT:    msub w8, w24, w10, w8
-; NONEON-NOSVE-NEXT:    mov v5.h[6], w9
-; NONEON-NOSVE-NEXT:    mov v4.h[6], w8
-; NONEON-NOSVE-NEXT:    udiv w12, w15, w13
-; NONEON-NOSVE-NEXT:    msub w8, w18, w20, w23
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #128] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #96] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v4.h[7], w8
-; NONEON-NOSVE-NEXT:    msub w9, w12, w13, w15
-; NONEON-NOSVE-NEXT:    mov v5.h[7], w9
-; NONEON-NOSVE-NEXT:    stp q4, q5, [x0]
-; NONEON-NOSVE-NEXT:    add sp, sp, #144
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = urem <16 x i16> %op1, %op2
@@ -2168,23 +689,6 @@ define <2 x i32> @urem_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    mls z0.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    fmov w9, s0
-; NONEON-NOSVE-NEXT:    mov w11, v1.s[1]
-; NONEON-NOSVE-NEXT:    mov w12, v0.s[1]
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    udiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    msub w9, w13, w11, w12
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w9
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = urem <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -2200,30 +704,6 @@ define <4 x i32> @urem_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    mls z0.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov w11, s1
-; NONEON-NOSVE-NEXT:    fmov w12, s0
-; NONEON-NOSVE-NEXT:    mov w8, v1.s[1]
-; NONEON-NOSVE-NEXT:    mov w9, v0.s[1]
-; NONEON-NOSVE-NEXT:    mov w14, v1.s[2]
-; NONEON-NOSVE-NEXT:    mov w15, v0.s[2]
-; NONEON-NOSVE-NEXT:    mov w17, v1.s[3]
-; NONEON-NOSVE-NEXT:    mov w18, v0.s[3]
-; NONEON-NOSVE-NEXT:    udiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    fmov s0, w11
-; NONEON-NOSVE-NEXT:    udiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w8
-; NONEON-NOSVE-NEXT:    udiv w9, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    mov v0.s[2], w8
-; NONEON-NOSVE-NEXT:    msub w8, w9, w17, w18
-; NONEON-NOSVE-NEXT:    mov v0.s[3], w8
-; NONEON-NOSVE-NEXT:    ret
   %res = urem <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -2243,65 +723,6 @@ define void @urem_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mls z1.s, p0/m, z5.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str x23, [sp, #-48]! // 8-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
-; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
-; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
-; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
-; NONEON-NOSVE-NEXT:    .cfi_offset w23, -48
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    fmov w12, s0
-; NONEON-NOSVE-NEXT:    fmov w3, s2
-; NONEON-NOSVE-NEXT:    mov w9, v0.s[1]
-; NONEON-NOSVE-NEXT:    fmov w11, s1
-; NONEON-NOSVE-NEXT:    fmov w2, s3
-; NONEON-NOSVE-NEXT:    mov w8, v1.s[1]
-; NONEON-NOSVE-NEXT:    mov w17, v3.s[1]
-; NONEON-NOSVE-NEXT:    mov w18, v2.s[1]
-; NONEON-NOSVE-NEXT:    mov w14, v1.s[2]
-; NONEON-NOSVE-NEXT:    mov w15, v0.s[2]
-; NONEON-NOSVE-NEXT:    mov w5, v3.s[2]
-; NONEON-NOSVE-NEXT:    mov w6, v2.s[2]
-; NONEON-NOSVE-NEXT:    udiv w13, w12, w11
-; NONEON-NOSVE-NEXT:    mov w19, v3.s[3]
-; NONEON-NOSVE-NEXT:    mov w20, v2.s[3]
-; NONEON-NOSVE-NEXT:    mov w22, v1.s[3]
-; NONEON-NOSVE-NEXT:    mov w23, v0.s[3]
-; NONEON-NOSVE-NEXT:    udiv w4, w3, w2
-; NONEON-NOSVE-NEXT:    msub w11, w13, w11, w12
-; NONEON-NOSVE-NEXT:    fmov s1, w11
-; NONEON-NOSVE-NEXT:    udiv w10, w9, w8
-; NONEON-NOSVE-NEXT:    msub w12, w4, w2, w3
-; NONEON-NOSVE-NEXT:    fmov s0, w12
-; NONEON-NOSVE-NEXT:    udiv w1, w18, w17
-; NONEON-NOSVE-NEXT:    msub w8, w10, w8, w9
-; NONEON-NOSVE-NEXT:    mov v1.s[1], w8
-; NONEON-NOSVE-NEXT:    udiv w16, w15, w14
-; NONEON-NOSVE-NEXT:    msub w13, w1, w17, w18
-; NONEON-NOSVE-NEXT:    mov v0.s[1], w13
-; NONEON-NOSVE-NEXT:    udiv w7, w6, w5
-; NONEON-NOSVE-NEXT:    msub w8, w16, w14, w15
-; NONEON-NOSVE-NEXT:    mov v1.s[2], w8
-; NONEON-NOSVE-NEXT:    udiv w21, w20, w19
-; NONEON-NOSVE-NEXT:    msub w10, w7, w5, w6
-; NONEON-NOSVE-NEXT:    mov v0.s[2], w10
-; NONEON-NOSVE-NEXT:    udiv w9, w23, w22
-; NONEON-NOSVE-NEXT:    msub w10, w21, w19, w20
-; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v0.s[3], w10
-; NONEON-NOSVE-NEXT:    msub w8, w9, w22, w23
-; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
-; NONEON-NOSVE-NEXT:    mov v1.s[3], w8
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr x23, [sp], #48 // 8-byte Folded Reload
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = urem <8 x i32> %op1, %op2
@@ -2320,17 +741,6 @@ define <1 x i64> @urem_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d1 killed $d1 def $q1
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    udiv x10, x9, x8
-; NONEON-NOSVE-NEXT:    msub x8, x10, x8, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %res = urem <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -2346,20 +756,6 @@ define <2 x i64> @urem_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    mov x11, v1.d[1]
-; NONEON-NOSVE-NEXT:    mov x12, v0.d[1]
-; NONEON-NOSVE-NEXT:    udiv x10, x9, x8
-; NONEON-NOSVE-NEXT:    udiv x13, x12, x11
-; NONEON-NOSVE-NEXT:    msub x8, x10, x8, x9
-; NONEON-NOSVE-NEXT:    fmov d0, x8
-; NONEON-NOSVE-NEXT:    msub x9, x13, x11, x12
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x9
-; NONEON-NOSVE-NEXT:    ret
   %res = urem <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -2379,33 +775,6 @@ define void @urem_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mls z1.d, p0/m, z5.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: urem_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    fmov x15, d2
-; NONEON-NOSVE-NEXT:    mov x12, v2.d[1]
-; NONEON-NOSVE-NEXT:    fmov x8, d1
-; NONEON-NOSVE-NEXT:    fmov x14, d3
-; NONEON-NOSVE-NEXT:    mov x11, v3.d[1]
-; NONEON-NOSVE-NEXT:    mov x17, v1.d[1]
-; NONEON-NOSVE-NEXT:    mov x18, v0.d[1]
-; NONEON-NOSVE-NEXT:    udiv x10, x9, x8
-; NONEON-NOSVE-NEXT:    udiv x16, x15, x14
-; NONEON-NOSVE-NEXT:    msub x8, x10, x8, x9
-; NONEON-NOSVE-NEXT:    fmov d1, x8
-; NONEON-NOSVE-NEXT:    udiv x13, x12, x11
-; NONEON-NOSVE-NEXT:    msub x10, x16, x14, x15
-; NONEON-NOSVE-NEXT:    fmov d0, x10
-; NONEON-NOSVE-NEXT:    udiv x1, x18, x17
-; NONEON-NOSVE-NEXT:    msub x9, x13, x11, x12
-; NONEON-NOSVE-NEXT:    mov v0.d[1], x9
-; NONEON-NOSVE-NEXT:    msub x11, x1, x17, x18
-; NONEON-NOSVE-NEXT:    mov v1.d[1], x11
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = urem <4 x i64> %op1, %op2
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll
index b3adf4720ece..906112f7ac39 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -17,14 +16,6 @@ define <4 x i8> @select_v4i8(<4 x i8> %op1, <4 x i8> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.4h, w8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <4 x i8> %op1, <4 x i8> %op2
   ret <4 x i8> %sel
 }
@@ -40,14 +31,6 @@ define <8 x i8> @select_v8i8(<8 x i8> %op1, <8 x i8> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.b, p0, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.8b, w8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <8 x i8> %op1, <8 x i8> %op2
   ret <8 x i8> %sel
 }
@@ -63,14 +46,6 @@ define <16 x i8> @select_v16i8(<16 x i8> %op1, <16 x i8> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.b, p0, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.16b, w8
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <16 x i8> %op1, <16 x i8> %op2
   ret <16 x i8> %sel
 }
@@ -89,20 +64,6 @@ define void @select_v32i8(ptr %a, ptr %b, i1 %mask) {
 ; CHECK-NEXT:    sel z1.b, p0, z1.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w2, #0x1
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    ldr q4, [x1, #16]
-; NONEON-NOSVE-NEXT:    dup v0.16b, w8
-; NONEON-NOSVE-NEXT:    bif v1.16b, v3.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load volatile <32 x i8>, ptr %a
   %op2 = load volatile <32 x i8>, ptr %b
   %sel = select i1 %mask, <32 x i8> %op1, <32 x i8> %op2
@@ -122,14 +83,6 @@ define <2 x i16> @select_v2i16(<2 x i16> %op1, <2 x i16> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.2s, w8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <2 x i16> %op1, <2 x i16> %op2
   ret <2 x i16> %sel
 }
@@ -146,14 +99,6 @@ define <4 x i16> @select_v4i16(<4 x i16> %op1, <4 x i16> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.4h, w8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <4 x i16> %op1, <4 x i16> %op2
   ret <4 x i16> %sel
 }
@@ -170,14 +115,6 @@ define <8 x i16> @select_v8i16(<8 x i16> %op1, <8 x i16> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.8h, w8
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <8 x i16> %op1, <8 x i16> %op2
   ret <8 x i16> %sel
 }
@@ -197,20 +134,6 @@ define void @select_v16i16(ptr %a, ptr %b, i1 %mask) {
 ; CHECK-NEXT:    sel z1.h, p0, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w2, #0x1
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    ldr q4, [x1, #16]
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    bif v1.16b, v3.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load volatile <16 x i16>, ptr %a
   %op2 = load volatile <16 x i16>, ptr %b
   %sel = select i1 %mask, <16 x i16> %op1, <16 x i16> %op2
@@ -230,14 +153,6 @@ define <2 x i32> @select_v2i32(<2 x i32> %op1, <2 x i32> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.2s, w8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <2 x i32> %op1, <2 x i32> %op2
   ret <2 x i32> %sel
 }
@@ -254,14 +169,6 @@ define <4 x i32> @select_v4i32(<4 x i32> %op1, <4 x i32> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    dup v2.4s, w8
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <4 x i32> %op1, <4 x i32> %op2
   ret <4 x i32> %sel
 }
@@ -281,20 +188,6 @@ define void @select_v8i32(ptr %a, ptr %b, i1 %mask) {
 ; CHECK-NEXT:    sel z1.s, p0, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w2, #0x1
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    csetm w8, ne
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    ldr q4, [x1, #16]
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    bif v1.16b, v3.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load volatile <8 x i32>, ptr %a
   %op2 = load volatile <8 x i32>, ptr %b
   %sel = select i1 %mask, <8 x i32> %op1, <8 x i32> %op2
@@ -315,14 +208,6 @@ define <1 x i64> @select_v1i64(<1 x i64> %op1, <1 x i64> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.d, p0, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm x8, ne
-; NONEON-NOSVE-NEXT:    fmov d2, x8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <1 x i64> %op1, <1 x i64> %op2
   ret <1 x i64> %sel
 }
@@ -340,14 +225,6 @@ define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, i1 %mask) {
 ; CHECK-NEXT:    sel z0.d, p0, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm x8, ne
-; NONEON-NOSVE-NEXT:    dup v2.2d, x8
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select i1 %mask, <2 x i64> %op1, <2 x i64> %op2
   ret <2 x i64> %sel
 }
@@ -368,20 +245,6 @@ define void @select_v4i64(ptr %a, ptr %b, i1 %mask) {
 ; CHECK-NEXT:    sel z1.d, p0, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w2, #0x1
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    csetm x8, ne
-; NONEON-NOSVE-NEXT:    ldr q3, [x1]
-; NONEON-NOSVE-NEXT:    ldr q4, [x1, #16]
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    bif v1.16b, v3.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bsl v0.16b, v2.16b, v4.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load volatile <4 x i64>, ptr %a
   %op2 = load volatile <4 x i64>, ptr %b
   %sel = select i1 %mask, <4 x i64> %op1, <4 x i64> %op2
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll
index a429cd82a449..9ed52e321d9a 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -20,16 +19,6 @@ define <4 x i8> @ashr_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    asr z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d2, #0xff00ff00ff00ff
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    sshr v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    neg v1.4h, v1.4h
-; NONEON-NOSVE-NEXT:    sshl v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = ashr <4 x i8> %op1, %op2
   ret <4 x i8> %res
 }
@@ -43,12 +32,6 @@ define <8 x i8> @ashr_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    asr z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.8b, v1.8b
-; NONEON-NOSVE-NEXT:    sshl v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = ashr <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -62,12 +45,6 @@ define <16 x i8> @ashr_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    asr z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    sshl v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = ashr <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -83,17 +60,6 @@ define void @ashr_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    asr z1.b, p0/m, z1.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    neg v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    neg v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    sshl v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    sshl v1.16b, v3.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = ashr <32 x i8> %op1, %op2
@@ -112,16 +78,6 @@ define <2 x i16> @ashr_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; CHECK-NEXT:    asr z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d2, #0x00ffff0000ffff
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    sshr v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    neg v1.2s, v1.2s
-; NONEON-NOSVE-NEXT:    sshl v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = ashr <2 x i16> %op1, %op2
   ret <2 x i16> %res
 }
@@ -135,12 +91,6 @@ define <4 x i16> @ashr_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    asr z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.4h, v1.4h
-; NONEON-NOSVE-NEXT:    sshl v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = ashr <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -154,12 +104,6 @@ define <8 x i16> @ashr_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    asr z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    sshl v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = ashr <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -175,17 +119,6 @@ define void @ashr_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    asr z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    neg v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    neg v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    sshl v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    sshl v1.8h, v3.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = ashr <16 x i16> %op1, %op2
@@ -202,12 +135,6 @@ define <2 x i32> @ashr_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    asr z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.2s, v1.2s
-; NONEON-NOSVE-NEXT:    sshl v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = ashr <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -221,12 +148,6 @@ define <4 x i32> @ashr_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    asr z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    sshl v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = ashr <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -242,17 +163,6 @@ define void @ashr_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    asr z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    neg v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    neg v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    sshl v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    sshl v1.4s, v3.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = ashr <8 x i32> %op1, %op2
@@ -269,12 +179,6 @@ define <1 x i64> @ashr_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    asr z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg d1, d1
-; NONEON-NOSVE-NEXT:    sshl d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = ashr <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -288,12 +192,6 @@ define <2 x i64> @ashr_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    asr z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    sshl v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = ashr <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -309,17 +207,6 @@ define void @ashr_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    asr z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ashr_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    neg v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    neg v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    sshl v0.2d, v2.2d, v0.2d
-; NONEON-NOSVE-NEXT:    sshl v1.2d, v3.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = ashr <4 x i64> %op1, %op2
@@ -342,15 +229,6 @@ define <4 x i8> @lshr_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    lsr z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d2, #0xff00ff00ff00ff
-; NONEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v2.8b
-; NONEON-NOSVE-NEXT:    neg v1.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ushl v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = lshr <4 x i8> %op1, %op2
   ret <4 x i8> %res
 }
@@ -364,12 +242,6 @@ define <8 x i8> @lshr_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    lsr z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ushl v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = lshr <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -383,12 +255,6 @@ define <16 x i8> @lshr_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    lsr z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ushl v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = lshr <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -404,17 +270,6 @@ define void @lshr_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    lsr z1.b, p0/m, z1.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    neg v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    neg v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ushl v0.16b, v2.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ushl v1.16b, v3.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = lshr <32 x i8> %op1, %op2
@@ -433,15 +288,6 @@ define <2 x i16> @lshr_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; CHECK-NEXT:    lsr z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d2, #0x00ffff0000ffff
-; NONEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v2.8b
-; NONEON-NOSVE-NEXT:    neg v1.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ushl v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = lshr <2 x i16> %op1, %op2
   ret <2 x i16> %res
 }
@@ -455,12 +301,6 @@ define <4 x i16> @lshr_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    lsr z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ushl v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = lshr <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -474,12 +314,6 @@ define <8 x i16> @lshr_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    lsr z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ushl v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = lshr <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -495,17 +329,6 @@ define void @lshr_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    lsr z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    neg v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    neg v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ushl v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    ushl v1.8h, v3.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = lshr <16 x i16> %op1, %op2
@@ -522,12 +345,6 @@ define <2 x i32> @lshr_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    lsr z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ushl v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = lshr <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -541,12 +358,6 @@ define <4 x i32> @lshr_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    lsr z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ushl v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = lshr <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -562,17 +373,6 @@ define void @lshr_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    lsr z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    neg v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    neg v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ushl v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ushl v1.4s, v3.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = lshr <8 x i32> %op1, %op2
@@ -589,12 +389,6 @@ define <1 x i64> @lshr_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    lsr z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg d1, d1
-; NONEON-NOSVE-NEXT:    ushl d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = lshr <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -608,12 +402,6 @@ define <2 x i64> @lshr_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    lsr z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    neg v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ushl v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = lshr <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -629,17 +417,6 @@ define void @lshr_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    lsr z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: lshr_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    neg v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    neg v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ushl v0.2d, v2.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ushl v1.2d, v3.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = lshr <4 x i64> %op1, %op2
@@ -661,13 +438,6 @@ define <2 x i8> @shl_v2i8(<2 x i8> %op1, <2 x i8> %op2) {
 ; CHECK-NEXT:    lsl z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v2i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d2, #0x0000ff000000ff
-; NONEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ushl v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = shl <2 x i8> %op1, %op2
   ret <2 x i8> %res
 }
@@ -682,13 +452,6 @@ define <4 x i8> @shl_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    lsl z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d2, #0xff00ff00ff00ff
-; NONEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ushl v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = shl <4 x i8> %op1, %op2
   ret <4 x i8> %res
 }
@@ -702,11 +465,6 @@ define <8 x i8> @shl_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    lsl z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushl v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = shl <8 x i8> %op1, %op2
   ret <8 x i8> %res
 }
@@ -720,11 +478,6 @@ define <16 x i8> @shl_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    lsl z0.b, p0/m, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushl v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = shl <16 x i8> %op1, %op2
   ret <16 x i8> %res
 }
@@ -740,15 +493,6 @@ define void @shl_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    lsl z1.b, p0/m, z1.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    ushl v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ushl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = shl <32 x i8> %op1, %op2
@@ -765,11 +509,6 @@ define <4 x i16> @shl_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    lsl z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushl v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %res = shl <4 x i16> %op1, %op2
   ret <4 x i16> %res
 }
@@ -783,11 +522,6 @@ define <8 x i16> @shl_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    lsl z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushl v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    ret
   %res = shl <8 x i16> %op1, %op2
   ret <8 x i16> %res
 }
@@ -803,15 +537,6 @@ define void @shl_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    lsl z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    ushl v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    ushl v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = shl <16 x i16> %op1, %op2
@@ -828,11 +553,6 @@ define <2 x i32> @shl_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    lsl z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushl v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = shl <2 x i32> %op1, %op2
   ret <2 x i32> %res
 }
@@ -846,11 +566,6 @@ define <4 x i32> @shl_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    lsl z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushl v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = shl <4 x i32> %op1, %op2
   ret <4 x i32> %res
 }
@@ -866,15 +581,6 @@ define void @shl_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    lsl z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    ushl v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ushl v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %res = shl <8 x i32> %op1, %op2
@@ -891,11 +597,6 @@ define <1 x i64> @shl_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-NEXT:    lsl z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushl d0, d0, d1
-; NONEON-NOSVE-NEXT:    ret
   %res = shl <1 x i64> %op1, %op2
   ret <1 x i64> %res
 }
@@ -909,11 +610,6 @@ define <2 x i64> @shl_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    lsl z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushl v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = shl <2 x i64> %op1, %op2
   ret <2 x i64> %res
 }
@@ -929,15 +625,6 @@ define void @shl_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    lsl z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shl_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    ushl v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ushl v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %res = shl <4 x i64> %op1, %op2
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
index d9ca19baea7d..b285659258f3 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -16,13 +15,6 @@ define <4 x half> @ucvtf_v4i16_v4f16(<4 x i16> %op1) {
 ; CHECK-NEXT:    ucvtf z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v4i16_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <4 x i16> %op1 to <4 x half>
   ret <4 x half> %res
 }
@@ -35,22 +27,6 @@ define void @ucvtf_v8i16_v8f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ucvtf z0.h, p0/m, z0.h
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v8i16_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    ucvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v1.4s
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v0.4s
-; NONEON-NOSVE-NEXT:    str q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %res = uitofp <8 x i16> %op1 to <8 x half>
   store <8 x half> %res, ptr %b
@@ -66,29 +42,6 @@ define void @ucvtf_v16i16_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ucvtf z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v16i16_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ucvtf v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ushll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    ucvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ucvtf v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn v2.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.8h, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v3.4s
-; NONEON-NOSVE-NEXT:    stp q2, q0, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %res = uitofp <16 x i16> %op1 to <16 x half>
   store <16 x half> %res, ptr %b
@@ -108,13 +61,6 @@ define <2 x float> @ucvtf_v2i16_v2f32(<2 x i16> %op1) {
 ; CHECK-NEXT:    ucvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v2i16_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d1, #0x00ffff0000ffff
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ucvtf v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <2 x i16> %op1 to <2 x float>
   ret <2 x float> %res
 }
@@ -128,12 +74,6 @@ define <4 x float> @ucvtf_v4i16_v4f32(<4 x i16> %op1) {
 ; CHECK-NEXT:    ucvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v4i16_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <4 x i16> %op1 to <4 x float>
   ret <4 x float> %res
 }
@@ -150,20 +90,6 @@ define void @ucvtf_v8i16_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ucvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v8i16_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ucvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %res = uitofp <8 x i16> %op1 to <8 x float>
   store <8 x float> %res, ptr %b
@@ -188,26 +114,6 @@ define void @ucvtf_v16i16_v16f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v16i16_v16f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    ucvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ucvtf v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    ucvtf v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %res = uitofp <16 x i16> %op1 to <16 x float>
   store <16 x float> %res, ptr %b
@@ -226,13 +132,6 @@ define <1 x double> @ucvtf_v1i16_v1f64(<1 x i16> %op1) {
 ; CHECK-NEXT:    and w8, w8, #0xffff
 ; CHECK-NEXT:    ucvtf d0, w8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v1i16_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    umov w8, v0.h[0]
-; NONEON-NOSVE-NEXT:    ucvtf d0, w8
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <1 x i16> %op1 to <1 x double>
   ret <1 x double> %res
 }
@@ -247,14 +146,6 @@ define <2 x double> @ucvtf_v2i16_v2f64(<2 x i16> %op1) {
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v2i16_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d1, #0x00ffff0000ffff
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <2 x i16> %op1 to <2 x double>
   ret <2 x double> %res
 }
@@ -272,21 +163,6 @@ define void @ucvtf_v4i16_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v4i16_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ucvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i16>, ptr %a
   %res = uitofp <4 x i16> %op1 to <4 x double>
   store <4 x double> %res, ptr %b
@@ -314,30 +190,6 @@ define void @ucvtf_v8i16_v8f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q1, [x1]
 ; CHECK-NEXT:    stp q3, q0, [x1, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v8i16_v8f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-48]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ucvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ucvtf v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    ucvtf v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %res = uitofp <8 x i16> %op1 to <8 x double>
   store <8 x double> %res, ptr %b
@@ -386,46 +238,6 @@ define void @ucvtf_v16i16_v16f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q1, q2, [x1, #32]
 ; CHECK-NEXT:    stp q3, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v16i16_v16f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-96]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    ushll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    stp q2, q0, [sp, #32]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    stp q3, q1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldr d5, [sp, #56]
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #40]
-; NONEON-NOSVE-NEXT:    ushll v5.2d, v5.2s, #0
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ushll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v4.2d, v4.2s, #0
-; NONEON-NOSVE-NEXT:    ucvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ushll v6.2d, v6.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v7.2d, v7.2s, #0
-; NONEON-NOSVE-NEXT:    ucvtf v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    ucvtf v5.2d, v5.2d
-; NONEON-NOSVE-NEXT:    ucvtf v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    ucvtf v4.2d, v4.2d
-; NONEON-NOSVE-NEXT:    stp q0, q5, [x1]
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v7.2d
-; NONEON-NOSVE-NEXT:    stp q1, q4, [x1, #64]
-; NONEON-NOSVE-NEXT:    ucvtf v1.2d, v6.2d
-; NONEON-NOSVE-NEXT:    stp q2, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q1, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #96
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %res = uitofp <16 x i16> %op1 to <16 x double>
   store <16 x double> %res, ptr %b
@@ -445,13 +257,6 @@ define <2 x half> @ucvtf_v2i32_v2f16(<2 x i32> %op1) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v2i32_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <2 x i32> %op1 to <2 x half>
   ret <2 x half> %res
 }
@@ -465,12 +270,6 @@ define <4 x half> @ucvtf_v4i32_v4f16(<4 x i32> %op1) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v4i32_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <4 x i32> %op1 to <4 x half>
   ret <4 x half> %res
 }
@@ -488,15 +287,6 @@ define <8 x half> @ucvtf_v8i32_v8f16(ptr %a) {
 ; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v8i32_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ucvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %res = uitofp <8 x i32> %op1 to <8 x half>
   ret <8 x half> %res
@@ -521,21 +311,6 @@ define void @ucvtf_v16i32_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    splice z2.h, p0, z2.h, z3.h
 ; CHECK-NEXT:    stp q2, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v16i32_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ucvtf v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    ucvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ucvtf v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v3.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i32>, ptr %a
   %res = uitofp <16 x i32> %op1 to <16 x half>
   store <16 x half> %res, ptr %b
@@ -554,11 +329,6 @@ define <2 x float> @ucvtf_v2i32_v2f32(<2 x i32> %op1) {
 ; CHECK-NEXT:    ucvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v2i32_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ucvtf v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <2 x i32> %op1 to <2 x float>
   ret <2 x float> %res
 }
@@ -571,11 +341,6 @@ define <4 x float> @ucvtf_v4i32_v4f32(<4 x i32> %op1) {
 ; CHECK-NEXT:    ucvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v4i32_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <4 x i32> %op1 to <4 x float>
   ret <4 x float> %res
 }
@@ -589,14 +354,6 @@ define void @ucvtf_v8i32_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ucvtf z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v8i32_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ucvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %res = uitofp <8 x i32> %op1 to <8 x float>
   store <8 x float> %res, ptr %b
@@ -616,12 +373,6 @@ define <2 x double> @ucvtf_v2i32_v2f64(<2 x i32> %op1) {
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v2i32_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <2 x i32> %op1 to <2 x double>
   ret <2 x double> %res
 }
@@ -638,20 +389,6 @@ define void @ucvtf_v4i32_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v4i32_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ucvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i32>, ptr %a
   %res = uitofp <4 x i32> %op1 to <4 x double>
   store <4 x double> %res, ptr %b
@@ -676,26 +413,6 @@ define void @ucvtf_v8i32_v8f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v8i32_v8f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    ushll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    ushll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    ucvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ucvtf v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    ucvtf v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %res = uitofp <8 x i32> %op1 to <8 x double>
   store <8 x double> %res, ptr %b
@@ -722,18 +439,6 @@ define <2 x half> @ucvtf_v2i64_v2f16(<2 x i64> %op1) {
 ; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v2i64_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov x8, v0.d[1]
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    ucvtf s1, x9
-; NONEON-NOSVE-NEXT:    ucvtf s0, x8
-; NONEON-NOSVE-NEXT:    fcvt h2, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v2.h[0]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <2 x i64> %op1 to <2 x half>
   ret <2 x half> %res
 }
@@ -754,16 +459,6 @@ define <4 x half> @ucvtf_v4i64_v4f16(ptr %a) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v4i64_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ucvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.4s, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %res = uitofp <4 x i64> %op1 to <4 x half>
   ret <4 x half> %res
@@ -797,22 +492,6 @@ define <8 x half> @ucvtf_v8i64_v8f16(ptr %a) {
 ; CHECK-NEXT:    splice z0.h, p0, z0.h, z2.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v8i64_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0, #32]
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ucvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ucvtf v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    ucvtf v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtn v2.2s, v2.2d
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.4s, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.4s, v3.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v2.4s
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i64>, ptr %a
   %res = uitofp <8 x i64> %op1 to <8 x half>
   ret <8 x half> %res
@@ -831,12 +510,6 @@ define <2 x float> @ucvtf_v2i64_v2f32(<2 x i64> %op1) {
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v2i64_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <2 x i64> %op1 to <2 x float>
   ret <2 x float> %res
 }
@@ -854,15 +527,6 @@ define <4 x float> @ucvtf_v4i64_v4f32(ptr %a) {
 ; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v4i64_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ucvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.4s, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %res = uitofp <4 x i64> %op1 to <4 x float>
   ret <4 x float> %res
@@ -887,21 +551,6 @@ define void @ucvtf_v8i64_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
 ; CHECK-NEXT:    stp q2, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v8i64_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ucvtf v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    ucvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    ucvtf v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtn v1.2s, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.4s, v2.2d
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.4s, v3.2d
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i64>, ptr %a
   %res = uitofp <8 x i64> %op1 to <8 x float>
   store <8 x float> %res, ptr %b
@@ -920,11 +569,6 @@ define <2 x double> @ucvtf_v2i64_v2f64(<2 x i64> %op1) {
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v2i64_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = uitofp <2 x i64> %op1 to <2 x double>
   ret <2 x double> %res
 }
@@ -938,14 +582,6 @@ define void @ucvtf_v4i64_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ucvtf z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_v4i64_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ucvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %res = uitofp <4 x i64> %op1 to <4 x double>
   store <4 x double> %res, ptr %b
@@ -964,13 +600,6 @@ define <4 x half> @scvtf_v4i16_v4f16(<4 x i16> %op1) {
 ; CHECK-NEXT:    scvtf z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v4i16_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    scvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <4 x i16> %op1 to <4 x half>
   ret <4 x half> %res
 }
@@ -983,22 +612,6 @@ define void @scvtf_v8i16_v8f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    scvtf z0.h, p0/m, z0.h
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v8i16_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
-; NONEON-NOSVE-NEXT:    scvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v1.4s
-; NONEON-NOSVE-NEXT:    scvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v0.4s
-; NONEON-NOSVE-NEXT:    str q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %res = sitofp <8 x i16> %op1 to <8 x half>
   store <8 x half> %res, ptr %b
@@ -1014,29 +627,6 @@ define void @scvtf_v16i16_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    scvtf z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v16i16_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    sshll v2.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    scvtf v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    scvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    sshll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    scvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    scvtf v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn v2.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v2.8h, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v3.4s
-; NONEON-NOSVE-NEXT:    stp q2, q0, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %res = sitofp <16 x i16> %op1 to <16 x half>
   store <16 x half> %res, ptr %b
@@ -1055,13 +645,6 @@ define <2 x float> @scvtf_v2i16_v2f32(<2 x i16> %op1) {
 ; CHECK-NEXT:    scvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v2i16_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    sshr v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    scvtf v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <2 x i16> %op1 to <2 x float>
   ret <2 x float> %res
 }
@@ -1075,12 +658,6 @@ define <4 x float> @scvtf_v4i16_v4f32(<4 x i16> %op1) {
 ; CHECK-NEXT:    scvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v4i16_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    scvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <4 x i16> %op1 to <4 x float>
   ret <4 x float> %res
 }
@@ -1097,20 +674,6 @@ define void @scvtf_v8i16_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    scvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v8i16_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    scvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    scvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %res = sitofp <8 x i16> %op1 to <8 x float>
   store <8 x float> %res, ptr %b
@@ -1135,26 +698,6 @@ define void @scvtf_v16i16_v16f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v16i16_v16f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    scvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    scvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    scvtf v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    scvtf v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %res = sitofp <16 x i16> %op1 to <16 x float>
   store <16 x float> %res, ptr %b
@@ -1176,14 +719,6 @@ define <2 x double> @scvtf_v2i16_v2f64(<2 x i16> %op1) {
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v2i16_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    sshr v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <2 x i16> %op1 to <2 x double>
   ret <2 x double> %res
 }
@@ -1201,21 +736,6 @@ define void @scvtf_v4i16_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v4i16_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    scvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i16>, ptr %a
   %res = sitofp <4 x i16> %op1 to <4 x double>
   store <4 x double> %res, ptr %b
@@ -1243,30 +763,6 @@ define void @scvtf_v8i16_v8f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q1, [x1]
 ; CHECK-NEXT:    stp q3, q0, [x1, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v8i16_v8f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-48]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #40]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    sshll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    scvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    scvtf v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    scvtf v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #48
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %res = sitofp <8 x i16> %op1 to <8 x double>
   store <8 x double> %res, ptr %b
@@ -1315,46 +811,6 @@ define void @scvtf_v16i16_v16f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q1, q2, [x1, #32]
 ; CHECK-NEXT:    stp q3, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v16i16_v16f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-96]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #8]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #24]
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v1.4s, v1.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    sshll v3.4s, v3.4h, #0
-; NONEON-NOSVE-NEXT:    stp q2, q0, [sp, #32]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    stp q3, q1, [sp, #64]
-; NONEON-NOSVE-NEXT:    ldr d5, [sp, #56]
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #88]
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #72]
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #40]
-; NONEON-NOSVE-NEXT:    sshll v5.2d, v5.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    sshll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v4.2d, v4.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    sshll v6.2d, v6.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v7.2d, v7.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    scvtf v5.2d, v5.2d
-; NONEON-NOSVE-NEXT:    scvtf v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    scvtf v4.2d, v4.2d
-; NONEON-NOSVE-NEXT:    stp q0, q5, [x1]
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v7.2d
-; NONEON-NOSVE-NEXT:    stp q1, q4, [x1, #64]
-; NONEON-NOSVE-NEXT:    scvtf v1.2d, v6.2d
-; NONEON-NOSVE-NEXT:    stp q2, q0, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q1, [x1, #96]
-; NONEON-NOSVE-NEXT:    add sp, sp, #96
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %res = sitofp <16 x i16> %op1 to <16 x double>
   store <16 x double> %res, ptr %b
@@ -1374,13 +830,6 @@ define <2 x half> @scvtf_v2i32_v2f16(<2 x i32> %op1) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v2i32_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    scvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <2 x i32> %op1 to <2 x half>
   ret <2 x half> %res
 }
@@ -1394,12 +843,6 @@ define <4 x half> @scvtf_v4i32_v4f16(<4 x i32> %op1) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v4i32_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    scvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <4 x i32> %op1 to <4 x half>
   ret <4 x half> %res
 }
@@ -1417,15 +860,6 @@ define <8 x half> @scvtf_v8i32_v8f16(ptr %a) {
 ; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v8i32_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    scvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    scvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.8h, v1.4s
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %res = sitofp <8 x i32> %op1 to <8 x half>
   ret <8 x half> %res
@@ -1443,11 +877,6 @@ define <2 x float> @scvtf_v2i32_v2f32(<2 x i32> %op1) {
 ; CHECK-NEXT:    scvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v2i32_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    scvtf v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <2 x i32> %op1 to <2 x float>
   ret <2 x float> %res
 }
@@ -1460,11 +889,6 @@ define <4 x float> @scvtf_v4i32_v4f32(<4 x i32> %op1) {
 ; CHECK-NEXT:    scvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v4i32_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    scvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <4 x i32> %op1 to <4 x float>
   ret <4 x float> %res
 }
@@ -1478,14 +902,6 @@ define void @scvtf_v8i32_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    scvtf z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v8i32_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    scvtf v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    scvtf v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %res = sitofp <8 x i32> %op1 to <8 x float>
   store <8 x float> %res, ptr %b
@@ -1505,12 +921,6 @@ define <2 x double> @scvtf_v2i32_v2f64(<2 x i32> %op1) {
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v2i32_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <2 x i32> %op1 to <2 x double>
   ret <2 x double> %res
 }
@@ -1527,20 +937,6 @@ define void @scvtf_v4i32_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v4i32_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    scvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i32>, ptr %a
   %res = sitofp <4 x i32> %op1 to <4 x double>
   store <4 x double> %res, ptr %b
@@ -1565,26 +961,6 @@ define void @scvtf_v8i32_v8f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    stp q3, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v8i32_v8f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-32]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT:    ldr d2, [sp, #24]
-; NONEON-NOSVE-NEXT:    ldr d3, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    scvtf v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    scvtf v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add sp, sp, #32
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %res = sitofp <8 x i32> %op1 to <8 x double>
   store <8 x double> %res, ptr %b
@@ -1629,40 +1005,6 @@ define void @scvtf_v16i32_v16f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q1, [x1]
 ; CHECK-NEXT:    stp q4, q0, [x1, #32]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v16i32_v16f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #32]
-; NONEON-NOSVE-NEXT:    stp q0, q2, [sp, #-64]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
-; NONEON-NOSVE-NEXT:    stp q1, q3, [sp, #32]
-; NONEON-NOSVE-NEXT:    ldr d4, [sp, #24]
-; NONEON-NOSVE-NEXT:    sshll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d5, [sp, #56]
-; NONEON-NOSVE-NEXT:    sshll v3.2d, v3.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d6, [sp, #40]
-; NONEON-NOSVE-NEXT:    sshll v4.2d, v4.2s, #0
-; NONEON-NOSVE-NEXT:    ldr d7, [sp, #8]
-; NONEON-NOSVE-NEXT:    sshll v1.2d, v1.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v5.2d, v5.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v2.2d, v2.2d
-; NONEON-NOSVE-NEXT:    sshll v6.2d, v6.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v3.2d, v3.2d
-; NONEON-NOSVE-NEXT:    sshll v0.2d, v0.2s, #0
-; NONEON-NOSVE-NEXT:    sshll v7.2d, v7.2s, #0
-; NONEON-NOSVE-NEXT:    scvtf v4.2d, v4.2d
-; NONEON-NOSVE-NEXT:    scvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    scvtf v5.2d, v5.2d
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    stp q2, q4, [x1, #96]
-; NONEON-NOSVE-NEXT:    scvtf v2.2d, v6.2d
-; NONEON-NOSVE-NEXT:    stp q3, q5, [x1, #64]
-; NONEON-NOSVE-NEXT:    scvtf v3.2d, v7.2d
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    add sp, sp, #64
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i32>, ptr %a
   %res = sitofp <16 x i32> %op1 to <16 x double>
   store <16 x double> %res, ptr %b
@@ -1689,18 +1031,6 @@ define <2 x half> @scvtf_v2i64_v2f16(<2 x i64> %op1) {
 ; CHECK-NEXT:    ldr d0, [sp, #8]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v2i64_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov x8, v0.d[1]
-; NONEON-NOSVE-NEXT:    fmov x9, d0
-; NONEON-NOSVE-NEXT:    scvtf s1, x9
-; NONEON-NOSVE-NEXT:    scvtf s0, x8
-; NONEON-NOSVE-NEXT:    fcvt h2, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s1
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v2.h[0]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <2 x i64> %op1 to <2 x half>
   ret <2 x half> %res
 }
@@ -1721,16 +1051,6 @@ define <4 x half> @scvtf_v4i64_v4f16(ptr %a) {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v4i64_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    scvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.4s, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %res = sitofp <4 x i64> %op1 to <4 x half>
   ret <4 x half> %res
@@ -1749,12 +1069,6 @@ define <2 x float> @scvtf_v2i64_v2f32(<2 x i64> %op1) {
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v2i64_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <2 x i64> %op1 to <2 x float>
   ret <2 x float> %res
 }
@@ -1772,15 +1086,6 @@ define <4 x float> @scvtf_v4i64_v4f32(ptr %a) {
 ; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v4i64_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    scvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    fcvtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    fcvtn2 v0.4s, v1.2d
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %res = sitofp <4 x i64> %op1 to <4 x float>
   ret <4 x float> %res
@@ -1798,11 +1103,6 @@ define <2 x double> @scvtf_v2i64_v2f64(<2 x i64> %op1) {
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v2i64_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    ret
   %res = sitofp <2 x i64> %op1 to <2 x double>
   ret <2 x double> %res
 }
@@ -1816,14 +1116,6 @@ define void @scvtf_v4i64_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_v4i64_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    scvtf v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    scvtf v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %res = sitofp <4 x i64> %op1 to <4 x double>
   store <4 x double> %res, ptr %b
@@ -1836,13 +1128,6 @@ define half @scvtf_i16_f16(ptr %0) {
 ; CHECK-NEXT:    ldrsh w8, [x0]
 ; CHECK-NEXT:    scvtf h0, w8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_i16_f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldrsh w8, [x0]
-; NONEON-NOSVE-NEXT:    scvtf s0, w8
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i16, ptr %0, align 64
   %3 = sitofp i16 %2 to half
   ret half %3
@@ -1854,12 +1139,6 @@ define float @scvtf_i16_f32(ptr %0) {
 ; CHECK-NEXT:    ldrsh w8, [x0]
 ; CHECK-NEXT:    scvtf s0, w8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_i16_f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldrsh w8, [x0]
-; NONEON-NOSVE-NEXT:    scvtf s0, w8
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i16, ptr %0, align 64
   %3 = sitofp i16 %2 to float
   ret float %3
@@ -1871,12 +1150,6 @@ define double @scvtf_i16_f64(ptr %0) {
 ; CHECK-NEXT:    ldrsh w8, [x0]
 ; CHECK-NEXT:    scvtf d0, w8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_i16_f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldrsh w8, [x0]
-; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i16, ptr %0, align 64
   %3 = sitofp i16 %2 to double
   ret double %3
@@ -1888,13 +1161,6 @@ define half @scvtf_i32_f16(ptr %0) {
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    scvtf h0, w8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_i32_f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr w8, [x0]
-; NONEON-NOSVE-NEXT:    scvtf s0, w8
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i32, ptr %0, align 64
   %3 = sitofp i32 %2 to half
   ret half %3
@@ -1906,12 +1172,6 @@ define float @scvtf_i32_f32(ptr %0) {
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    scvtf s0, w8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_i32_f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr w8, [x0]
-; NONEON-NOSVE-NEXT:    scvtf s0, w8
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i32, ptr %0, align 64
   %3 = sitofp i32 %2 to float
   ret float %3
@@ -1923,12 +1183,6 @@ define double @scvtf_i32_f64(ptr %0) {
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    scvtf d0, w8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_i32_f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr w8, [x0]
-; NONEON-NOSVE-NEXT:    scvtf d0, w8
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i32, ptr %0, align 64
   %3 = sitofp i32 %2 to double
   ret double %3
@@ -1940,13 +1194,6 @@ define half @scvtf_i64_f16(ptr %0) {
 ; CHECK-NEXT:    ldr x8, [x0]
 ; CHECK-NEXT:    scvtf h0, x8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_i64_f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr x8, [x0]
-; NONEON-NOSVE-NEXT:    scvtf s0, x8
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i64, ptr %0, align 64
   %3 = sitofp i64 %2 to half
   ret half %3
@@ -1958,12 +1205,6 @@ define float @scvtf_i64_f32(ptr %0) {
 ; CHECK-NEXT:    ldr x8, [x0]
 ; CHECK-NEXT:    scvtf s0, x8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_i64_f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr x8, [x0]
-; NONEON-NOSVE-NEXT:    scvtf s0, x8
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i64, ptr %0, align 64
   %3 = sitofp i64 %2 to float
   ret float %3
@@ -1975,12 +1216,6 @@ define double @scvtf_i64_f64(ptr %0) {
 ; CHECK-NEXT:    ldr x8, [x0]
 ; CHECK-NEXT:    scvtf d0, x8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: scvtf_i64_f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr x8, [x0]
-; NONEON-NOSVE-NEXT:    scvtf d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i64, ptr %0, align 64
   %3 = sitofp i64 %2 to double
   ret double %3
@@ -1992,13 +1227,6 @@ define half @ucvtf_i16_f16(ptr %0) {
 ; CHECK-NEXT:    ldrh w8, [x0]
 ; CHECK-NEXT:    ucvtf h0, w8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_i16_f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr h0, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf s0, s0
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i16, ptr %0, align 64
   %3 = uitofp i16 %2 to half
   ret half %3
@@ -2010,12 +1238,6 @@ define float @ucvtf_i16_f32(ptr %0) {
 ; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ucvtf s0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_i16_f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr h0, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf s0, s0
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i16, ptr %0, align 64
   %3 = uitofp i16 %2 to float
   ret float %3
@@ -2027,12 +1249,6 @@ define double @ucvtf_i16_f64(ptr %0) {
 ; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    ucvtf d0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_i16_f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr h0, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf d0, d0
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i16, ptr %0, align 64
   %3 = uitofp i16 %2 to double
   ret double %3
@@ -2044,13 +1260,6 @@ define half @ucvtf_i32_f16(ptr %0) {
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    ucvtf h0, w8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_i32_f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr w8, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf s0, w8
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i32, ptr %0, align 64
   %3 = uitofp i32 %2 to half
   ret half %3
@@ -2062,12 +1271,6 @@ define float @ucvtf_i32_f32(ptr %0) {
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    ucvtf s0, w8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_i32_f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr w8, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf s0, w8
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i32, ptr %0, align 64
   %3 = uitofp i32 %2 to float
   ret float %3
@@ -2079,12 +1282,6 @@ define double @ucvtf_i32_f64(ptr %0) {
 ; CHECK-NEXT:    ldr s0, [x0]
 ; CHECK-NEXT:    ucvtf d0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_i32_f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr s0, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf d0, d0
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i32, ptr %0, align 64
   %3 = uitofp i32 %2 to double
   ret double %3
@@ -2096,13 +1293,6 @@ define half @ucvtf_i64_f16(ptr %0) {
 ; CHECK-NEXT:    ldr x8, [x0]
 ; CHECK-NEXT:    ucvtf h0, x8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_i64_f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr x8, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf s0, x8
-; NONEON-NOSVE-NEXT:    fcvt h0, s0
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i64, ptr %0, align 64
   %3 = uitofp i64 %2 to half
   ret half %3
@@ -2114,12 +1304,6 @@ define float @ucvtf_i64_f32(ptr %0) {
 ; CHECK-NEXT:    ldr x8, [x0]
 ; CHECK-NEXT:    ucvtf s0, x8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_i64_f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr x8, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf s0, x8
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i64, ptr %0, align 64
   %3 = uitofp i64 %2 to float
   ret float %3
@@ -2131,12 +1315,6 @@ define double @ucvtf_i64_f64(ptr %0) {
 ; CHECK-NEXT:    ldr x8, [x0]
 ; CHECK-NEXT:    ucvtf d0, x8
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ucvtf_i64_f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr x8, [x0]
-; NONEON-NOSVE-NEXT:    ucvtf d0, x8
-; NONEON-NOSVE-NEXT:    ret
   %2 = load i64, ptr %0, align 64
   %3 = uitofp i64 %2 to double
   ret double %3
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll
index 42daa4fedc94..81bbaa92d4b4 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -19,13 +18,6 @@ define <4 x i8> @select_v4i8(<4 x i8> %op1, <4 x i8> %op2, <4 x i1> %mask) {
 ; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v2.4h, v2.4h, #15
-; NONEON-NOSVE-NEXT:    cmlt v2.4h, v2.4h, #0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <4 x i1> %mask, <4 x i8> %op1, <4 x i8> %op2
   ret <4 x i8> %sel
 }
@@ -44,13 +36,6 @@ define <8 x i8> @select_v8i8(<8 x i8> %op1, <8 x i8> %op2, <8 x i1> %mask) {
 ; CHECK-NEXT:    sel z0.b, p0, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v2.8b, v2.8b, #7
-; NONEON-NOSVE-NEXT:    cmlt v2.8b, v2.8b, #0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <8 x i1> %mask, <8 x i8> %op1, <8 x i8> %op2
   ret <8 x i8> %sel
 }
@@ -69,13 +54,6 @@ define <16 x i8> @select_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %mask)
 ; CHECK-NEXT:    sel z0.b, p0, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v2.16b, v2.16b, #7
-; NONEON-NOSVE-NEXT:    cmlt v2.16b, v2.16b, #0
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <16 x i1> %mask, <16 x i8> %op1, <16 x i8> %op2
   ret <16 x i8> %sel
 }
@@ -92,18 +70,6 @@ define void @select_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sel z1.b, p0, z2.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    cmeq v4.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    cmeq v5.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v4.16b
-; NONEON-NOSVE-NEXT:    mov v1.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %mask = icmp eq <32 x i8> %op1, %op2
@@ -126,13 +92,6 @@ define <2 x i16> @select_v2i16(<2 x i16> %op1, <2 x i16> %op2, <2 x i1> %mask) {
 ; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v2.2s, v2.2s, #31
-; NONEON-NOSVE-NEXT:    cmlt v2.2s, v2.2s, #0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <2 x i1> %mask, <2 x i16> %op1, <2 x i16> %op2
   ret <2 x i16> %sel
 }
@@ -151,13 +110,6 @@ define <4 x i16> @select_v4i16(<4 x i16> %op1, <4 x i16> %op2, <4 x i1> %mask) {
 ; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v2.4h, v2.4h, #15
-; NONEON-NOSVE-NEXT:    cmlt v2.4h, v2.4h, #0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <4 x i1> %mask, <4 x i16> %op1, <4 x i16> %op2
   ret <4 x i16> %sel
 }
@@ -177,14 +129,6 @@ define <8 x i16> @select_v8i16(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask) {
 ; CHECK-NEXT:    sel z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v2.8h, v2.8b, #0
-; NONEON-NOSVE-NEXT:    shl v2.8h, v2.8h, #15
-; NONEON-NOSVE-NEXT:    cmlt v2.8h, v2.8h, #0
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <8 x i1> %mask, <8 x i16> %op1, <8 x i16> %op2
   ret <8 x i16> %sel
 }
@@ -201,18 +145,6 @@ define void @select_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sel z1.h, p0, z2.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    cmeq v4.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    cmeq v5.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v4.16b
-; NONEON-NOSVE-NEXT:    mov v1.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %mask = icmp eq <16 x i16> %op1, %op2
@@ -235,13 +167,6 @@ define <2 x i32> @select_v2i32(<2 x i32> %op1, <2 x i32> %op2, <2 x i1> %mask) {
 ; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v2.2s, v2.2s, #31
-; NONEON-NOSVE-NEXT:    cmlt v2.2s, v2.2s, #0
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <2 x i1> %mask, <2 x i32> %op1, <2 x i32> %op2
   ret <2 x i32> %sel
 }
@@ -261,14 +186,6 @@ define <4 x i32> @select_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask) {
 ; CHECK-NEXT:    sel z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v2.4s, v2.4h, #0
-; NONEON-NOSVE-NEXT:    shl v2.4s, v2.4s, #31
-; NONEON-NOSVE-NEXT:    cmlt v2.4s, v2.4s, #0
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <4 x i1> %mask, <4 x i32> %op1, <4 x i32> %op2
   ret <4 x i32> %sel
 }
@@ -285,18 +202,6 @@ define void @select_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sel z1.s, p0, z2.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    cmeq v4.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    cmeq v5.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v4.16b
-; NONEON-NOSVE-NEXT:    mov v1.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %mask = icmp eq <8 x i32> %op1, %op2
@@ -318,14 +223,6 @@ define <1 x i64> @select_v1i64(<1 x i64> %op1, <1 x i64> %op2, <1 x i1> %mask) {
 ; CHECK-NEXT:    sel z0.d, p0, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    tst w0, #0x1
-; NONEON-NOSVE-NEXT:    csetm x8, ne
-; NONEON-NOSVE-NEXT:    fmov d2, x8
-; NONEON-NOSVE-NEXT:    bif v0.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <1 x i1> %mask, <1 x i64> %op1, <1 x i64> %op2
   ret <1 x i64> %sel
 }
@@ -345,14 +242,6 @@ define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) {
 ; CHECK-NEXT:    sel z0.d, p0, z0.d, z1.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ushll v2.2d, v2.2s, #0
-; NONEON-NOSVE-NEXT:    shl v2.2d, v2.2d, #63
-; NONEON-NOSVE-NEXT:    cmlt v2.2d, v2.2d, #0
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %sel = select <2 x i1> %mask, <2 x i64> %op1, <2 x i64> %op2
   ret <2 x i64> %sel
 }
@@ -369,18 +258,6 @@ define void @select_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sel z1.d, p0, z2.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: select_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    cmeq v4.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    cmeq v5.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    bif v0.16b, v1.16b, v4.16b
-; NONEON-NOSVE-NEXT:    mov v1.16b, v5.16b
-; NONEON-NOSVE-NEXT:    bsl v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %mask = icmp eq <4 x i64> %op1, %op2
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-limit-duplane.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-limit-duplane.ll
index 01a7a5cafd26..885030861469 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-limit-duplane.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-limit-duplane.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve  < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve  < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -19,19 +18,6 @@ define <4 x i32> @test(ptr %arg1, ptr %arg2) {
 ; CHECK-NEXT:    stp q2, q5, [x0, #32]
 ; CHECK-NEXT:    stp q1, q3, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test:
-; NONEON-NOSVE:       // %bb.0: // %entry
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q3, q4, [x0]
-; NONEON-NOSVE-NEXT:    add v2.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    add v5.4s, v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    dup v0.4s, v1.s[2]
-; NONEON-NOSVE-NEXT:    add v1.4s, v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    add v3.4s, v4.4s, v4.4s
-; NONEON-NOSVE-NEXT:    stp q2, q5, [x0, #32]
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
 entry:
   %0 = load <16 x i32>, ptr %arg1, align 256
   %1 = load <16 x i32>, ptr %arg2, align 256
@@ -56,19 +42,6 @@ define <2 x i32> @test2(ptr %arg1, ptr %arg2) {
 ; CHECK-NEXT:    stp q3, q4, [x0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test2:
-; NONEON-NOSVE:       // %bb.0: // %entry
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q3, q4, [x0]
-; NONEON-NOSVE-NEXT:    add v2.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    dup v0.2s, v1.s[2]
-; NONEON-NOSVE-NEXT:    add v1.4s, v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    add v3.4s, v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    add v4.4s, v4.4s, v4.4s
-; NONEON-NOSVE-NEXT:    stp q2, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q4, [x0]
-; NONEON-NOSVE-NEXT:    ret
 entry:
   %0 = load <16 x i32>, ptr %arg1, align 256
   %1 = load <16 x i32>, ptr %arg2, align 256
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-loads.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-loads.ll
index c57f3af0d4b6..8ca8e6980913 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-loads.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -12,13 +11,6 @@ define <4 x i8> @load_v4i8(ptr %a) {
 ; CHECK-NEXT:    ld1b { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr s0, [x0]
-; NONEON-NOSVE-NEXT:    ushll v0.8h, v0.8b, #0
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %load = load <4 x i8>, ptr %a
   ret <4 x i8> %load
 }
@@ -28,11 +20,6 @@ define <8 x i8> @load_v8i8(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <8 x i8>, ptr %a
   ret <8 x i8> %load
 }
@@ -42,11 +29,6 @@ define <16 x i8> @load_v16i8(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <16 x i8>, ptr %a
   ret <16 x i8> %load
 }
@@ -56,11 +38,6 @@ define <32 x i8> @load_v32i8(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <32 x i8>, ptr %a
   ret <32 x i8> %load
 }
@@ -72,15 +49,6 @@ define <2 x i16> @load_v2i16(ptr %a) {
 ; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldrh w8, [x0]
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    add x8, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[2], [x8]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %load = load <2 x i16>, ptr %a
   ret <2 x i16> %load
 }
@@ -90,11 +58,6 @@ define <2 x half> @load_v2f16(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr s0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr s0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <2 x half>, ptr %a
   ret <2 x half> %load
 }
@@ -104,11 +67,6 @@ define <4 x i16> @load_v4i16(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <4 x i16>, ptr %a
   ret <4 x i16> %load
 }
@@ -118,11 +76,6 @@ define <4 x half> @load_v4f16(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <4 x half>, ptr %a
   ret <4 x half> %load
 }
@@ -132,11 +85,6 @@ define <8 x i16> @load_v8i16(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <8 x i16>, ptr %a
   ret <8 x i16> %load
 }
@@ -146,11 +94,6 @@ define <8 x half> @load_v8f16(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <8 x half>, ptr %a
   ret <8 x half> %load
 }
@@ -160,11 +103,6 @@ define <16 x i16> @load_v16i16(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <16 x i16>, ptr %a
   ret <16 x i16> %load
 }
@@ -174,11 +112,6 @@ define <16 x half> @load_v16f16(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <16 x half>, ptr %a
   ret <16 x half> %load
 }
@@ -188,11 +121,6 @@ define <2 x i32> @load_v2i32(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <2 x i32>, ptr %a
   ret <2 x i32> %load
 }
@@ -202,11 +130,6 @@ define <2 x float> @load_v2f32(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <2 x float>, ptr %a
   ret <2 x float> %load
 }
@@ -216,11 +139,6 @@ define <4 x i32> @load_v4i32(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <4 x i32>, ptr %a
   ret <4 x i32> %load
 }
@@ -230,11 +148,6 @@ define <4 x float> @load_v4f32(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <4 x float>, ptr %a
   ret <4 x float> %load
 }
@@ -244,11 +157,6 @@ define <8 x i32> @load_v8i32(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <8 x i32>, ptr %a
   ret <8 x i32> %load
 }
@@ -258,11 +166,6 @@ define <8 x float> @load_v8f32(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <8 x float>, ptr %a
   ret <8 x float> %load
 }
@@ -272,11 +175,6 @@ define <1 x i64> @load_v1i64(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <1 x i64>, ptr %a
   ret <1 x i64> %load
 }
@@ -286,11 +184,6 @@ define <1 x double> @load_v1f64(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <1 x double>, ptr %a
   ret <1 x double> %load
 }
@@ -300,11 +193,6 @@ define <2 x i64> @load_v2i64(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <2 x i64>, ptr %a
   ret <2 x i64> %load
 }
@@ -314,11 +202,6 @@ define <2 x double> @load_v2f64(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <2 x double>, ptr %a
   ret <2 x double> %load
 }
@@ -328,11 +211,6 @@ define <4 x i64> @load_v4i64(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <4 x i64>, ptr %a
   ret <4 x i64> %load
 }
@@ -342,11 +220,6 @@ define <4 x double> @load_v4f64(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: load_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %load = load <4 x double>, ptr %a
   ret <4 x double> %load
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll
index 65c45587e120..c4aeb4465c53 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -18,14 +17,6 @@ define i8 @andv_v4i8(<4 x i8> %a) {
 ; CHECK-NEXT:    andv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    and x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    and w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> %a)
   ret i8 %res
 }
@@ -38,15 +29,6 @@ define i8 @andv_v8i8(<8 x i8> %a) {
 ; CHECK-NEXT:    andv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    and x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    and x8, x8, x8, lsr #16
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #8
-; NONEON-NOSVE-NEXT:    and w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> %a)
   ret i8 %res
 }
@@ -59,20 +41,6 @@ define i8 @andv_v16i8(<16 x i8> %a) {
 ; CHECK-NEXT:    andv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    and x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    and x8, x8, x8, lsr #16
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #8
-; NONEON-NOSVE-NEXT:    and w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> %a)
   ret i8 %res
 }
@@ -86,22 +54,6 @@ define i8 @andv_v32i8(ptr %a) {
 ; CHECK-NEXT:    andv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    and v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    and x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    and x8, x8, x8, lsr #16
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #8
-; NONEON-NOSVE-NEXT:    and w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> %op)
   ret i8 %res
@@ -115,13 +67,6 @@ define i16 @andv_v2i16(<2 x i16> %a) {
 ; CHECK-NEXT:    andv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    and w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> %a)
   ret i16 %res
 }
@@ -134,14 +79,6 @@ define i16 @andv_v4i16(<4 x i16> %a) {
 ; CHECK-NEXT:    andv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    and x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    and w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> %a)
   ret i16 %res
 }
@@ -154,19 +91,6 @@ define i16 @andv_v8i16(<8 x i16> %a) {
 ; CHECK-NEXT:    andv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    and x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    and w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> %a)
   ret i16 %res
 }
@@ -180,21 +104,6 @@ define i16 @andv_v16i16(ptr %a) {
 ; CHECK-NEXT:    andv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    and v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    and x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    and w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> %op)
   ret i16 %res
@@ -208,13 +117,6 @@ define i32 @andv_v2i32(<2 x i32> %a) {
 ; CHECK-NEXT:    andv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    and w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> %a)
   ret i32 %res
 }
@@ -227,18 +129,6 @@ define i32 @andv_v4i32(<4 x i32> %a) {
 ; CHECK-NEXT:    andv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    and w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %a)
   ret i32 %res
 }
@@ -252,20 +142,6 @@ define i32 @andv_v8i32(ptr %a) {
 ; CHECK-NEXT:    andv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    and v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    and w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> %op)
   ret i32 %res
@@ -279,16 +155,6 @@ define i64 @andv_v2i64(<2 x i64> %a) {
 ; CHECK-NEXT:    andv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> %a)
   ret i64 %res
 }
@@ -302,18 +168,6 @@ define i64 @andv_v4i64(ptr %a) {
 ; CHECK-NEXT:    andv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: andv_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    and v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> %op)
   ret i64 %res
@@ -331,14 +185,6 @@ define i8 @eorv_v4i8(<4 x i8> %a) {
 ; CHECK-NEXT:    eorv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    eor x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    eor w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> %a)
   ret i8 %res
 }
@@ -351,15 +197,6 @@ define i8 @eorv_v8i8(<8 x i8> %a) {
 ; CHECK-NEXT:    eorv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    eor x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    eor x8, x8, x8, lsr #16
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #8
-; NONEON-NOSVE-NEXT:    eor w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> %a)
   ret i8 %res
 }
@@ -372,20 +209,6 @@ define i8 @eorv_v16i8(<16 x i8> %a) {
 ; CHECK-NEXT:    eorv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    eor x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    eor x8, x8, x8, lsr #16
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #8
-; NONEON-NOSVE-NEXT:    eor w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> %a)
   ret i8 %res
 }
@@ -399,22 +222,6 @@ define i8 @eorv_v32i8(ptr %a) {
 ; CHECK-NEXT:    eorv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    eor v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    eor x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    eor x8, x8, x8, lsr #16
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #8
-; NONEON-NOSVE-NEXT:    eor w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> %op)
   ret i8 %res
@@ -428,13 +235,6 @@ define i16 @eorv_v2i16(<2 x i16> %a) {
 ; CHECK-NEXT:    eorv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    eor w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.xor.v2i16(<2 x i16> %a)
   ret i16 %res
 }
@@ -447,14 +247,6 @@ define i16 @eorv_v4i16(<4 x i16> %a) {
 ; CHECK-NEXT:    eorv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    eor x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    eor w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> %a)
   ret i16 %res
 }
@@ -467,19 +259,6 @@ define i16 @eorv_v8i16(<8 x i16> %a) {
 ; CHECK-NEXT:    eorv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    eor x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    eor w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> %a)
   ret i16 %res
 }
@@ -493,21 +272,6 @@ define i16 @eorv_v16i16(ptr %a) {
 ; CHECK-NEXT:    eorv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    eor v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    eor x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    eor w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> %op)
   ret i16 %res
@@ -521,13 +285,6 @@ define i32 @eorv_v2i32(<2 x i32> %a) {
 ; CHECK-NEXT:    eorv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    eor w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> %a)
   ret i32 %res
 }
@@ -540,18 +297,6 @@ define i32 @eorv_v4i32(<4 x i32> %a) {
 ; CHECK-NEXT:    eorv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    eor w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> %a)
   ret i32 %res
 }
@@ -565,20 +310,6 @@ define i32 @eorv_v8i32(ptr %a) {
 ; CHECK-NEXT:    eorv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    eor v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    eor w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> %op)
   ret i32 %res
@@ -592,16 +323,6 @@ define i64 @eorv_v2i64(<2 x i64> %a) {
 ; CHECK-NEXT:    eorv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> %a)
   ret i64 %res
 }
@@ -615,18 +336,6 @@ define i64 @eorv_v4i64(ptr %a) {
 ; CHECK-NEXT:    eorv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: eorv_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    eor v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    eor v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> %op)
   ret i64 %res
@@ -644,14 +353,6 @@ define i8 @orv_v4i8(<4 x i8> %a) {
 ; CHECK-NEXT:    orv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    orr x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    orr w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> %a)
   ret i8 %res
 }
@@ -664,15 +365,6 @@ define i8 @orv_v8i8(<8 x i8> %a) {
 ; CHECK-NEXT:    orv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    orr x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    orr x8, x8, x8, lsr #16
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #8
-; NONEON-NOSVE-NEXT:    orr w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> %a)
   ret i8 %res
 }
@@ -685,20 +377,6 @@ define i8 @orv_v16i8(<16 x i8> %a) {
 ; CHECK-NEXT:    orv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    orr x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    orr x8, x8, x8, lsr #16
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #8
-; NONEON-NOSVE-NEXT:    orr w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %a)
   ret i8 %res
 }
@@ -712,22 +390,6 @@ define i8 @orv_v32i8(ptr %a) {
 ; CHECK-NEXT:    orv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    orr v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    orr x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    orr x8, x8, x8, lsr #16
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #8
-; NONEON-NOSVE-NEXT:    orr w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> %op)
   ret i8 %res
@@ -741,13 +403,6 @@ define i16 @orv_v2i16(<2 x i16> %a) {
 ; CHECK-NEXT:    orv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    orr w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> %a)
   ret i16 %res
 }
@@ -760,14 +415,6 @@ define i16 @orv_v4i16(<4 x i16> %a) {
 ; CHECK-NEXT:    orv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    orr x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    orr w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> %a)
   ret i16 %res
 }
@@ -780,19 +427,6 @@ define i16 @orv_v8i16(<8 x i16> %a) {
 ; CHECK-NEXT:    orv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    orr x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    orr w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> %a)
   ret i16 %res
 }
@@ -806,21 +440,6 @@ define i16 @orv_v16i16(ptr %a) {
 ; CHECK-NEXT:    orv h0, p0, z0.h
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    orr v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    orr x8, x8, x8, lsr #32
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #16
-; NONEON-NOSVE-NEXT:    orr w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> %op)
   ret i16 %res
@@ -834,13 +453,6 @@ define i32 @orv_v2i32(<2 x i32> %a) {
 ; CHECK-NEXT:    orv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    orr w0, w8, w9
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> %a)
   ret i32 %res
 }
@@ -853,18 +465,6 @@ define i32 @orv_v4i32(<4 x i32> %a) {
 ; CHECK-NEXT:    orv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    orr w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> %a)
   ret i32 %res
 }
@@ -878,20 +478,6 @@ define i32 @orv_v8i32(ptr %a) {
 ; CHECK-NEXT:    orv s0, p0, z0.s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    orr v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x8, d0
-; NONEON-NOSVE-NEXT:    lsr x9, x8, #32
-; NONEON-NOSVE-NEXT:    orr w0, w8, w9
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %op)
   ret i32 %res
@@ -905,16 +491,6 @@ define i64 @orv_v2i64(<2 x i64> %a) {
 ; CHECK-NEXT:    orv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> %a)
   ret i64 %res
 }
@@ -928,18 +504,6 @@ define i64 @orv_v4i64(ptr %a) {
 ; CHECK-NEXT:    orv d0, p0, z0.d
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: orv_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    orr v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    ldr d1, [sp, #8]
-; NONEON-NOSVE-NEXT:    orr v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    fmov x0, d0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> %op)
   ret i64 %res
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
index 886f97ed988d..ca58099244cf 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -20,44 +19,6 @@ define <4 x i8> @masked_load_v4i8(ptr %src, <4 x i1> %mask) {
 ; CHECK-NEXT:    ld1b { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #15
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI0_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI0_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.4h, v0.4h, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv h0, v0.4h
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB0_2
-; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[0], [x0]
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB0_3
-; NONEON-NOSVE-NEXT:    b .LBB0_4
-; NONEON-NOSVE-NEXT:  .LBB0_2:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB0_4
-; NONEON-NOSVE-NEXT:  .LBB0_3: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x9, x0, #1
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[2], [x9]
-; NONEON-NOSVE-NEXT:  .LBB0_4: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB0_7
-; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB0_8
-; NONEON-NOSVE-NEXT:  .LBB0_6: // %else8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB0_7: // %cond.load4
-; NONEON-NOSVE-NEXT:    add x9, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[4], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB0_6
-; NONEON-NOSVE-NEXT:  .LBB0_8: // %cond.load7
-; NONEON-NOSVE-NEXT:    add x8, x0, #3
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[6], [x8]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %load = call <4 x i8> @llvm.masked.load.v4i8(ptr %src, i32 8, <4 x i1> %mask, <4 x i8> zeroinitializer)
   ret <4 x i8> %load
 }
@@ -73,67 +34,6 @@ define <8 x i8> @masked_load_v8i8(ptr %src, <8 x i1> %mask) {
 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.8b, v0.8b, #7
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI1_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI1_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.8b, v0.8b, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv b0, v0.8b
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB1_2
-; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr b0, [x0]
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB1_3
-; NONEON-NOSVE-NEXT:    b .LBB1_4
-; NONEON-NOSVE-NEXT:  .LBB1_2:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB1_4
-; NONEON-NOSVE-NEXT:  .LBB1_3: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x9, x0, #1
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[1], [x9]
-; NONEON-NOSVE-NEXT:  .LBB1_4: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB1_11
-; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB1_12
-; NONEON-NOSVE-NEXT:  .LBB1_6: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB1_13
-; NONEON-NOSVE-NEXT:  .LBB1_7: // %else11
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB1_14
-; NONEON-NOSVE-NEXT:  .LBB1_8: // %else14
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB1_15
-; NONEON-NOSVE-NEXT:  .LBB1_9: // %else17
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB1_16
-; NONEON-NOSVE-NEXT:  .LBB1_10: // %else20
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB1_11: // %cond.load4
-; NONEON-NOSVE-NEXT:    add x9, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[2], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB1_6
-; NONEON-NOSVE-NEXT:  .LBB1_12: // %cond.load7
-; NONEON-NOSVE-NEXT:    add x9, x0, #3
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[3], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB1_7
-; NONEON-NOSVE-NEXT:  .LBB1_13: // %cond.load10
-; NONEON-NOSVE-NEXT:    add x9, x0, #4
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[4], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB1_8
-; NONEON-NOSVE-NEXT:  .LBB1_14: // %cond.load13
-; NONEON-NOSVE-NEXT:    add x9, x0, #5
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[5], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB1_9
-; NONEON-NOSVE-NEXT:  .LBB1_15: // %cond.load16
-; NONEON-NOSVE-NEXT:    add x9, x0, #6
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[6], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB1_10
-; NONEON-NOSVE-NEXT:  .LBB1_16: // %cond.load19
-; NONEON-NOSVE-NEXT:    add x8, x0, #7
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[7], [x8]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %load = call <8 x i8> @llvm.masked.load.v8i8(ptr %src, i32 8, <8 x i1> %mask, <8 x i8> zeroinitializer)
   ret <8 x i8> %load
 }
@@ -149,115 +49,6 @@ define <16 x i8> @masked_load_v16i8(ptr %src, <16 x i1> %mask) {
 ; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.16b, v0.16b, #7
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI2_0
-; NONEON-NOSVE-NEXT:    ldr q1, [x8, :lo12:.LCPI2_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.16b, v0.16b, #0
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; NONEON-NOSVE-NEXT:    zip1 v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    addv h1, v0.8h
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB2_17
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB2_18
-; NONEON-NOSVE-NEXT:  .LBB2_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB2_19
-; NONEON-NOSVE-NEXT:  .LBB2_3: // %else5
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB2_20
-; NONEON-NOSVE-NEXT:  .LBB2_4: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB2_21
-; NONEON-NOSVE-NEXT:  .LBB2_5: // %else11
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB2_22
-; NONEON-NOSVE-NEXT:  .LBB2_6: // %else14
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB2_23
-; NONEON-NOSVE-NEXT:  .LBB2_7: // %else17
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB2_24
-; NONEON-NOSVE-NEXT:  .LBB2_8: // %else20
-; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB2_25
-; NONEON-NOSVE-NEXT:  .LBB2_9: // %else23
-; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB2_26
-; NONEON-NOSVE-NEXT:  .LBB2_10: // %else26
-; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB2_27
-; NONEON-NOSVE-NEXT:  .LBB2_11: // %else29
-; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB2_28
-; NONEON-NOSVE-NEXT:  .LBB2_12: // %else32
-; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB2_29
-; NONEON-NOSVE-NEXT:  .LBB2_13: // %else35
-; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB2_30
-; NONEON-NOSVE-NEXT:  .LBB2_14: // %else38
-; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB2_31
-; NONEON-NOSVE-NEXT:  .LBB2_15: // %else41
-; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB2_32
-; NONEON-NOSVE-NEXT:  .LBB2_16: // %else44
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB2_17: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr b0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB2_2
-; NONEON-NOSVE-NEXT:  .LBB2_18: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x9, x0, #1
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[1], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB2_3
-; NONEON-NOSVE-NEXT:  .LBB2_19: // %cond.load4
-; NONEON-NOSVE-NEXT:    add x9, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[2], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB2_4
-; NONEON-NOSVE-NEXT:  .LBB2_20: // %cond.load7
-; NONEON-NOSVE-NEXT:    add x9, x0, #3
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[3], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB2_5
-; NONEON-NOSVE-NEXT:  .LBB2_21: // %cond.load10
-; NONEON-NOSVE-NEXT:    add x9, x0, #4
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[4], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB2_6
-; NONEON-NOSVE-NEXT:  .LBB2_22: // %cond.load13
-; NONEON-NOSVE-NEXT:    add x9, x0, #5
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[5], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB2_7
-; NONEON-NOSVE-NEXT:  .LBB2_23: // %cond.load16
-; NONEON-NOSVE-NEXT:    add x9, x0, #6
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[6], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB2_8
-; NONEON-NOSVE-NEXT:  .LBB2_24: // %cond.load19
-; NONEON-NOSVE-NEXT:    add x9, x0, #7
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[7], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB2_9
-; NONEON-NOSVE-NEXT:  .LBB2_25: // %cond.load22
-; NONEON-NOSVE-NEXT:    add x9, x0, #8
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[8], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB2_10
-; NONEON-NOSVE-NEXT:  .LBB2_26: // %cond.load25
-; NONEON-NOSVE-NEXT:    add x9, x0, #9
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[9], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB2_11
-; NONEON-NOSVE-NEXT:  .LBB2_27: // %cond.load28
-; NONEON-NOSVE-NEXT:    add x9, x0, #10
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[10], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB2_12
-; NONEON-NOSVE-NEXT:  .LBB2_28: // %cond.load31
-; NONEON-NOSVE-NEXT:    add x9, x0, #11
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[11], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB2_13
-; NONEON-NOSVE-NEXT:  .LBB2_29: // %cond.load34
-; NONEON-NOSVE-NEXT:    add x9, x0, #12
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[12], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB2_14
-; NONEON-NOSVE-NEXT:  .LBB2_30: // %cond.load37
-; NONEON-NOSVE-NEXT:    add x9, x0, #13
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[13], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB2_15
-; NONEON-NOSVE-NEXT:  .LBB2_31: // %cond.load40
-; NONEON-NOSVE-NEXT:    add x9, x0, #14
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[14], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB2_16
-; NONEON-NOSVE-NEXT:  .LBB2_32: // %cond.load43
-; NONEON-NOSVE-NEXT:    add x8, x0, #15
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[15], [x8]
-; NONEON-NOSVE-NEXT:    ret
   %load = call <16 x i8> @llvm.masked.load.v16i8(ptr %src, i32 8, <16 x i1> %mask, <16 x i8> zeroinitializer)
   ret <16 x i8> %load
 }
@@ -339,277 +130,6 @@ define <32 x i8> @masked_load_v32i8(ptr %src, <32 x i1> %mask) {
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    fmov s1, w1
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #80]
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    mov v1.b[1], w2
-; NONEON-NOSVE-NEXT:    mov v0.b[1], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp]
-; NONEON-NOSVE-NEXT:    mov v1.b[2], w3
-; NONEON-NOSVE-NEXT:    mov v0.b[2], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    mov v1.b[3], w4
-; NONEON-NOSVE-NEXT:    mov v0.b[3], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #104]
-; NONEON-NOSVE-NEXT:    mov v1.b[4], w5
-; NONEON-NOSVE-NEXT:    mov v0.b[4], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #112]
-; NONEON-NOSVE-NEXT:    mov v1.b[5], w6
-; NONEON-NOSVE-NEXT:    mov v0.b[5], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #120]
-; NONEON-NOSVE-NEXT:    mov v1.b[6], w7
-; NONEON-NOSVE-NEXT:    mov v0.b[6], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #128]
-; NONEON-NOSVE-NEXT:    mov v1.b[7], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8]
-; NONEON-NOSVE-NEXT:    mov v0.b[7], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #136]
-; NONEON-NOSVE-NEXT:    mov v1.b[8], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #16]
-; NONEON-NOSVE-NEXT:    mov v0.b[8], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #144]
-; NONEON-NOSVE-NEXT:    mov v1.b[9], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #24]
-; NONEON-NOSVE-NEXT:    mov v0.b[9], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #152]
-; NONEON-NOSVE-NEXT:    mov v1.b[10], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    mov v0.b[10], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #160]
-; NONEON-NOSVE-NEXT:    mov v1.b[11], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    mov v0.b[11], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #168]
-; NONEON-NOSVE-NEXT:    mov v1.b[12], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #48]
-; NONEON-NOSVE-NEXT:    mov v0.b[12], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #176]
-; NONEON-NOSVE-NEXT:    mov v1.b[13], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #56]
-; NONEON-NOSVE-NEXT:    mov v0.b[13], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #184]
-; NONEON-NOSVE-NEXT:    mov v1.b[14], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #64]
-; NONEON-NOSVE-NEXT:    mov v0.b[14], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #192]
-; NONEON-NOSVE-NEXT:    mov v1.b[15], w9
-; NONEON-NOSVE-NEXT:    mov v0.b[15], w8
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI3_0
-; NONEON-NOSVE-NEXT:    ldr q2, [x8, :lo12:.LCPI3_0]
-; NONEON-NOSVE-NEXT:    shl v1.16b, v1.16b, #7
-; NONEON-NOSVE-NEXT:    shl v0.16b, v0.16b, #7
-; NONEON-NOSVE-NEXT:    cmlt v1.16b, v1.16b, #0
-; NONEON-NOSVE-NEXT:    cmlt v0.16b, v0.16b, #0
-; NONEON-NOSVE-NEXT:    and v1.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; NONEON-NOSVE-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; NONEON-NOSVE-NEXT:    zip1 v1.16b, v1.16b, v3.16b
-; NONEON-NOSVE-NEXT:    zip1 v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    addv h1, v1.8h
-; NONEON-NOSVE-NEXT:    addv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    movi v1.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    fmov w9, s0
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    bfi w8, w9, #16, #16
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB3_33
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB3_34
-; NONEON-NOSVE-NEXT:  .LBB3_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB3_35
-; NONEON-NOSVE-NEXT:  .LBB3_3: // %else5
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB3_36
-; NONEON-NOSVE-NEXT:  .LBB3_4: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB3_37
-; NONEON-NOSVE-NEXT:  .LBB3_5: // %else11
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB3_38
-; NONEON-NOSVE-NEXT:  .LBB3_6: // %else14
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB3_39
-; NONEON-NOSVE-NEXT:  .LBB3_7: // %else17
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB3_40
-; NONEON-NOSVE-NEXT:  .LBB3_8: // %else20
-; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB3_41
-; NONEON-NOSVE-NEXT:  .LBB3_9: // %else23
-; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB3_42
-; NONEON-NOSVE-NEXT:  .LBB3_10: // %else26
-; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB3_43
-; NONEON-NOSVE-NEXT:  .LBB3_11: // %else29
-; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB3_44
-; NONEON-NOSVE-NEXT:  .LBB3_12: // %else32
-; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB3_45
-; NONEON-NOSVE-NEXT:  .LBB3_13: // %else35
-; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB3_46
-; NONEON-NOSVE-NEXT:  .LBB3_14: // %else38
-; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB3_47
-; NONEON-NOSVE-NEXT:  .LBB3_15: // %else41
-; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB3_48
-; NONEON-NOSVE-NEXT:  .LBB3_16: // %else44
-; NONEON-NOSVE-NEXT:    tbnz w8, #16, .LBB3_49
-; NONEON-NOSVE-NEXT:  .LBB3_17: // %else47
-; NONEON-NOSVE-NEXT:    tbnz w8, #17, .LBB3_50
-; NONEON-NOSVE-NEXT:  .LBB3_18: // %else50
-; NONEON-NOSVE-NEXT:    tbnz w8, #18, .LBB3_51
-; NONEON-NOSVE-NEXT:  .LBB3_19: // %else53
-; NONEON-NOSVE-NEXT:    tbnz w8, #19, .LBB3_52
-; NONEON-NOSVE-NEXT:  .LBB3_20: // %else56
-; NONEON-NOSVE-NEXT:    tbnz w8, #20, .LBB3_53
-; NONEON-NOSVE-NEXT:  .LBB3_21: // %else59
-; NONEON-NOSVE-NEXT:    tbnz w8, #21, .LBB3_54
-; NONEON-NOSVE-NEXT:  .LBB3_22: // %else62
-; NONEON-NOSVE-NEXT:    tbnz w8, #22, .LBB3_55
-; NONEON-NOSVE-NEXT:  .LBB3_23: // %else65
-; NONEON-NOSVE-NEXT:    tbnz w8, #23, .LBB3_56
-; NONEON-NOSVE-NEXT:  .LBB3_24: // %else68
-; NONEON-NOSVE-NEXT:    tbnz w8, #24, .LBB3_57
-; NONEON-NOSVE-NEXT:  .LBB3_25: // %else71
-; NONEON-NOSVE-NEXT:    tbnz w8, #25, .LBB3_58
-; NONEON-NOSVE-NEXT:  .LBB3_26: // %else74
-; NONEON-NOSVE-NEXT:    tbnz w8, #26, .LBB3_59
-; NONEON-NOSVE-NEXT:  .LBB3_27: // %else77
-; NONEON-NOSVE-NEXT:    tbnz w8, #27, .LBB3_60
-; NONEON-NOSVE-NEXT:  .LBB3_28: // %else80
-; NONEON-NOSVE-NEXT:    tbnz w8, #28, .LBB3_61
-; NONEON-NOSVE-NEXT:  .LBB3_29: // %else83
-; NONEON-NOSVE-NEXT:    tbnz w8, #29, .LBB3_62
-; NONEON-NOSVE-NEXT:  .LBB3_30: // %else86
-; NONEON-NOSVE-NEXT:    tbnz w8, #30, .LBB3_63
-; NONEON-NOSVE-NEXT:  .LBB3_31: // %else89
-; NONEON-NOSVE-NEXT:    tbnz w8, #31, .LBB3_64
-; NONEON-NOSVE-NEXT:  .LBB3_32: // %else92
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB3_33: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr b0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB3_2
-; NONEON-NOSVE-NEXT:  .LBB3_34: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x9, x0, #1
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[1], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB3_3
-; NONEON-NOSVE-NEXT:  .LBB3_35: // %cond.load4
-; NONEON-NOSVE-NEXT:    add x9, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[2], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB3_4
-; NONEON-NOSVE-NEXT:  .LBB3_36: // %cond.load7
-; NONEON-NOSVE-NEXT:    add x9, x0, #3
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[3], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB3_5
-; NONEON-NOSVE-NEXT:  .LBB3_37: // %cond.load10
-; NONEON-NOSVE-NEXT:    add x9, x0, #4
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[4], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB3_6
-; NONEON-NOSVE-NEXT:  .LBB3_38: // %cond.load13
-; NONEON-NOSVE-NEXT:    add x9, x0, #5
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[5], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB3_7
-; NONEON-NOSVE-NEXT:  .LBB3_39: // %cond.load16
-; NONEON-NOSVE-NEXT:    add x9, x0, #6
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[6], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB3_8
-; NONEON-NOSVE-NEXT:  .LBB3_40: // %cond.load19
-; NONEON-NOSVE-NEXT:    add x9, x0, #7
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[7], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB3_9
-; NONEON-NOSVE-NEXT:  .LBB3_41: // %cond.load22
-; NONEON-NOSVE-NEXT:    add x9, x0, #8
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[8], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB3_10
-; NONEON-NOSVE-NEXT:  .LBB3_42: // %cond.load25
-; NONEON-NOSVE-NEXT:    add x9, x0, #9
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[9], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB3_11
-; NONEON-NOSVE-NEXT:  .LBB3_43: // %cond.load28
-; NONEON-NOSVE-NEXT:    add x9, x0, #10
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[10], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB3_12
-; NONEON-NOSVE-NEXT:  .LBB3_44: // %cond.load31
-; NONEON-NOSVE-NEXT:    add x9, x0, #11
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[11], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB3_13
-; NONEON-NOSVE-NEXT:  .LBB3_45: // %cond.load34
-; NONEON-NOSVE-NEXT:    add x9, x0, #12
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[12], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB3_14
-; NONEON-NOSVE-NEXT:  .LBB3_46: // %cond.load37
-; NONEON-NOSVE-NEXT:    add x9, x0, #13
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[13], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB3_15
-; NONEON-NOSVE-NEXT:  .LBB3_47: // %cond.load40
-; NONEON-NOSVE-NEXT:    add x9, x0, #14
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[14], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB3_16
-; NONEON-NOSVE-NEXT:  .LBB3_48: // %cond.load43
-; NONEON-NOSVE-NEXT:    add x9, x0, #15
-; NONEON-NOSVE-NEXT:    ld1 { v0.b }[15], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #16, .LBB3_17
-; NONEON-NOSVE-NEXT:  .LBB3_49: // %cond.load46
-; NONEON-NOSVE-NEXT:    add x9, x0, #16
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[0], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #17, .LBB3_18
-; NONEON-NOSVE-NEXT:  .LBB3_50: // %cond.load49
-; NONEON-NOSVE-NEXT:    add x9, x0, #17
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[1], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #18, .LBB3_19
-; NONEON-NOSVE-NEXT:  .LBB3_51: // %cond.load52
-; NONEON-NOSVE-NEXT:    add x9, x0, #18
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[2], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #19, .LBB3_20
-; NONEON-NOSVE-NEXT:  .LBB3_52: // %cond.load55
-; NONEON-NOSVE-NEXT:    add x9, x0, #19
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[3], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #20, .LBB3_21
-; NONEON-NOSVE-NEXT:  .LBB3_53: // %cond.load58
-; NONEON-NOSVE-NEXT:    add x9, x0, #20
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[4], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #21, .LBB3_22
-; NONEON-NOSVE-NEXT:  .LBB3_54: // %cond.load61
-; NONEON-NOSVE-NEXT:    add x9, x0, #21
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[5], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #22, .LBB3_23
-; NONEON-NOSVE-NEXT:  .LBB3_55: // %cond.load64
-; NONEON-NOSVE-NEXT:    add x9, x0, #22
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[6], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #23, .LBB3_24
-; NONEON-NOSVE-NEXT:  .LBB3_56: // %cond.load67
-; NONEON-NOSVE-NEXT:    add x9, x0, #23
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[7], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #24, .LBB3_25
-; NONEON-NOSVE-NEXT:  .LBB3_57: // %cond.load70
-; NONEON-NOSVE-NEXT:    add x9, x0, #24
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[8], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #25, .LBB3_26
-; NONEON-NOSVE-NEXT:  .LBB3_58: // %cond.load73
-; NONEON-NOSVE-NEXT:    add x9, x0, #25
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[9], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #26, .LBB3_27
-; NONEON-NOSVE-NEXT:  .LBB3_59: // %cond.load76
-; NONEON-NOSVE-NEXT:    add x9, x0, #26
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[10], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #27, .LBB3_28
-; NONEON-NOSVE-NEXT:  .LBB3_60: // %cond.load79
-; NONEON-NOSVE-NEXT:    add x9, x0, #27
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[11], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #28, .LBB3_29
-; NONEON-NOSVE-NEXT:  .LBB3_61: // %cond.load82
-; NONEON-NOSVE-NEXT:    add x9, x0, #28
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[12], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #29, .LBB3_30
-; NONEON-NOSVE-NEXT:  .LBB3_62: // %cond.load85
-; NONEON-NOSVE-NEXT:    add x9, x0, #29
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[13], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #30, .LBB3_31
-; NONEON-NOSVE-NEXT:  .LBB3_63: // %cond.load88
-; NONEON-NOSVE-NEXT:    add x9, x0, #30
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[14], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #31, .LBB3_32
-; NONEON-NOSVE-NEXT:  .LBB3_64: // %cond.load91
-; NONEON-NOSVE-NEXT:    add x8, x0, #31
-; NONEON-NOSVE-NEXT:    ld1 { v1.b }[15], [x8]
-; NONEON-NOSVE-NEXT:    ret
   %load = call <32 x i8> @llvm.masked.load.v32i8(ptr %src, i32 8, <32 x i1> %mask, <32 x i8> zeroinitializer)
   ret <32 x i8> %load
 }
@@ -635,31 +155,6 @@ define <2 x half> @masked_load_v2f16(ptr %src, <2 x i1> %mask) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #31
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI4_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI4_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.2s, v0.2s, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addp v1.2s, v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    movi d0, #0000000000000000
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB4_3
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB4_4
-; NONEON-NOSVE-NEXT:  .LBB4_2: // %else2
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB4_3: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr h0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB4_2
-; NONEON-NOSVE-NEXT:  .LBB4_4: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x8, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[1], [x8]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %load = call <2 x half> @llvm.masked.load.v2f16(ptr %src, i32 8, <2 x i1> %mask, <2 x half> zeroinitializer)
   ret <2 x half> %load
 }
@@ -675,43 +170,6 @@ define <4 x half> @masked_load_v4f16(ptr %src, <4 x i1> %mask) {
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #15
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI5_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI5_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.4h, v0.4h, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv h1, v0.4h
-; NONEON-NOSVE-NEXT:    movi d0, #0000000000000000
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB5_5
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB5_6
-; NONEON-NOSVE-NEXT:  .LBB5_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB5_7
-; NONEON-NOSVE-NEXT:  .LBB5_3: // %else5
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB5_8
-; NONEON-NOSVE-NEXT:  .LBB5_4: // %else8
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB5_5: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr h0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB5_2
-; NONEON-NOSVE-NEXT:  .LBB5_6: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x9, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[1], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB5_3
-; NONEON-NOSVE-NEXT:  .LBB5_7: // %cond.load4
-; NONEON-NOSVE-NEXT:    add x9, x0, #4
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[2], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB5_4
-; NONEON-NOSVE-NEXT:  .LBB5_8: // %cond.load7
-; NONEON-NOSVE-NEXT:    add x8, x0, #6
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[3], [x8]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %load = call <4 x half> @llvm.masked.load.v4f16(ptr %src, i32 8, <4 x i1> %mask, <4 x half> zeroinitializer)
   ret <4 x half> %load
 }
@@ -728,65 +186,6 @@ define <8 x half> @masked_load_v8f16(ptr %src, <8 x i1> %mask) {
 ; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.8b, v0.8b, #7
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI6_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI6_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.8b, v0.8b, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv b1, v0.8b
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB6_9
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB6_10
-; NONEON-NOSVE-NEXT:  .LBB6_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB6_11
-; NONEON-NOSVE-NEXT:  .LBB6_3: // %else5
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB6_12
-; NONEON-NOSVE-NEXT:  .LBB6_4: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB6_13
-; NONEON-NOSVE-NEXT:  .LBB6_5: // %else11
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB6_14
-; NONEON-NOSVE-NEXT:  .LBB6_6: // %else14
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB6_15
-; NONEON-NOSVE-NEXT:  .LBB6_7: // %else17
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB6_16
-; NONEON-NOSVE-NEXT:  .LBB6_8: // %else20
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB6_9: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr h0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB6_2
-; NONEON-NOSVE-NEXT:  .LBB6_10: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x9, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[1], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB6_3
-; NONEON-NOSVE-NEXT:  .LBB6_11: // %cond.load4
-; NONEON-NOSVE-NEXT:    add x9, x0, #4
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[2], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB6_4
-; NONEON-NOSVE-NEXT:  .LBB6_12: // %cond.load7
-; NONEON-NOSVE-NEXT:    add x9, x0, #6
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[3], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB6_5
-; NONEON-NOSVE-NEXT:  .LBB6_13: // %cond.load10
-; NONEON-NOSVE-NEXT:    add x9, x0, #8
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[4], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB6_6
-; NONEON-NOSVE-NEXT:  .LBB6_14: // %cond.load13
-; NONEON-NOSVE-NEXT:    add x9, x0, #10
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[5], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB6_7
-; NONEON-NOSVE-NEXT:  .LBB6_15: // %cond.load16
-; NONEON-NOSVE-NEXT:    add x9, x0, #12
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[6], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB6_8
-; NONEON-NOSVE-NEXT:  .LBB6_16: // %cond.load19
-; NONEON-NOSVE-NEXT:    add x8, x0, #14
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[7], [x8]
-; NONEON-NOSVE-NEXT:    ret
   %load = call <8 x half> @llvm.masked.load.v8f16(ptr %src, i32 8, <8 x i1> %mask, <8 x half> zeroinitializer)
   ret <8 x half> %load
 }
@@ -811,116 +210,6 @@ define <16 x half> @masked_load_v16f16(ptr %src, <16 x i1> %mask) {
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x0, x8, lsl #1]
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.16b, v0.16b, #7
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI7_0
-; NONEON-NOSVE-NEXT:    ldr q1, [x8, :lo12:.LCPI7_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.16b, v0.16b, #0
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; NONEON-NOSVE-NEXT:    zip1 v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    movi v1.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    addv h2, v0.8h
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    fmov w8, s2
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB7_17
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB7_18
-; NONEON-NOSVE-NEXT:  .LBB7_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB7_19
-; NONEON-NOSVE-NEXT:  .LBB7_3: // %else5
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB7_20
-; NONEON-NOSVE-NEXT:  .LBB7_4: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB7_21
-; NONEON-NOSVE-NEXT:  .LBB7_5: // %else11
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB7_22
-; NONEON-NOSVE-NEXT:  .LBB7_6: // %else14
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB7_23
-; NONEON-NOSVE-NEXT:  .LBB7_7: // %else17
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB7_24
-; NONEON-NOSVE-NEXT:  .LBB7_8: // %else20
-; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB7_25
-; NONEON-NOSVE-NEXT:  .LBB7_9: // %else23
-; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB7_26
-; NONEON-NOSVE-NEXT:  .LBB7_10: // %else26
-; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB7_27
-; NONEON-NOSVE-NEXT:  .LBB7_11: // %else29
-; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB7_28
-; NONEON-NOSVE-NEXT:  .LBB7_12: // %else32
-; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB7_29
-; NONEON-NOSVE-NEXT:  .LBB7_13: // %else35
-; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB7_30
-; NONEON-NOSVE-NEXT:  .LBB7_14: // %else38
-; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB7_31
-; NONEON-NOSVE-NEXT:  .LBB7_15: // %else41
-; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB7_32
-; NONEON-NOSVE-NEXT:  .LBB7_16: // %else44
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB7_17: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr h0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB7_2
-; NONEON-NOSVE-NEXT:  .LBB7_18: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x9, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[1], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB7_3
-; NONEON-NOSVE-NEXT:  .LBB7_19: // %cond.load4
-; NONEON-NOSVE-NEXT:    add x9, x0, #4
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[2], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB7_4
-; NONEON-NOSVE-NEXT:  .LBB7_20: // %cond.load7
-; NONEON-NOSVE-NEXT:    add x9, x0, #6
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[3], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB7_5
-; NONEON-NOSVE-NEXT:  .LBB7_21: // %cond.load10
-; NONEON-NOSVE-NEXT:    add x9, x0, #8
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[4], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB7_6
-; NONEON-NOSVE-NEXT:  .LBB7_22: // %cond.load13
-; NONEON-NOSVE-NEXT:    add x9, x0, #10
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[5], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB7_7
-; NONEON-NOSVE-NEXT:  .LBB7_23: // %cond.load16
-; NONEON-NOSVE-NEXT:    add x9, x0, #12
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[6], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB7_8
-; NONEON-NOSVE-NEXT:  .LBB7_24: // %cond.load19
-; NONEON-NOSVE-NEXT:    add x9, x0, #14
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[7], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB7_9
-; NONEON-NOSVE-NEXT:  .LBB7_25: // %cond.load22
-; NONEON-NOSVE-NEXT:    add x9, x0, #16
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[0], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB7_10
-; NONEON-NOSVE-NEXT:  .LBB7_26: // %cond.load25
-; NONEON-NOSVE-NEXT:    add x9, x0, #18
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[1], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB7_11
-; NONEON-NOSVE-NEXT:  .LBB7_27: // %cond.load28
-; NONEON-NOSVE-NEXT:    add x9, x0, #20
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[2], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB7_12
-; NONEON-NOSVE-NEXT:  .LBB7_28: // %cond.load31
-; NONEON-NOSVE-NEXT:    add x9, x0, #22
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[3], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB7_13
-; NONEON-NOSVE-NEXT:  .LBB7_29: // %cond.load34
-; NONEON-NOSVE-NEXT:    add x9, x0, #24
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[4], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB7_14
-; NONEON-NOSVE-NEXT:  .LBB7_30: // %cond.load37
-; NONEON-NOSVE-NEXT:    add x9, x0, #26
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[5], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB7_15
-; NONEON-NOSVE-NEXT:  .LBB7_31: // %cond.load40
-; NONEON-NOSVE-NEXT:    add x9, x0, #28
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[6], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB7_16
-; NONEON-NOSVE-NEXT:  .LBB7_32: // %cond.load43
-; NONEON-NOSVE-NEXT:    add x8, x0, #30
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[7], [x8]
-; NONEON-NOSVE-NEXT:    ret
   %load = call <16 x half> @llvm.masked.load.v16f16(ptr %src, i32 8, <16 x i1> %mask, <16 x half> zeroinitializer)
   ret <16 x half> %load
 }
@@ -936,31 +225,6 @@ define <2 x float> @masked_load_v2f32(ptr %src, <2 x i1> %mask) {
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #31
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI8_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI8_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.2s, v0.2s, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addp v1.2s, v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    movi d0, #0000000000000000
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB8_3
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB8_4
-; NONEON-NOSVE-NEXT:  .LBB8_2: // %else2
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB8_3: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr s0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB8_2
-; NONEON-NOSVE-NEXT:  .LBB8_4: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x8, x0, #4
-; NONEON-NOSVE-NEXT:    ld1 { v0.s }[1], [x8]
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NONEON-NOSVE-NEXT:    ret
   %load = call <2 x float> @llvm.masked.load.v2f32(ptr %src, i32 8, <2 x i1> %mask, <2 x float> zeroinitializer)
   ret <2 x float> %load
 }
@@ -977,41 +241,6 @@ define <4 x float> @masked_load_v4f32(ptr %src, <4 x i1> %mask) {
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #15
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI9_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI9_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.4h, v0.4h, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv h1, v0.4h
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB9_5
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB9_6
-; NONEON-NOSVE-NEXT:  .LBB9_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB9_7
-; NONEON-NOSVE-NEXT:  .LBB9_3: // %else5
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB9_8
-; NONEON-NOSVE-NEXT:  .LBB9_4: // %else8
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB9_5: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr s0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB9_2
-; NONEON-NOSVE-NEXT:  .LBB9_6: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x9, x0, #4
-; NONEON-NOSVE-NEXT:    ld1 { v0.s }[1], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB9_3
-; NONEON-NOSVE-NEXT:  .LBB9_7: // %cond.load4
-; NONEON-NOSVE-NEXT:    add x9, x0, #8
-; NONEON-NOSVE-NEXT:    ld1 { v0.s }[2], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB9_4
-; NONEON-NOSVE-NEXT:  .LBB9_8: // %cond.load7
-; NONEON-NOSVE-NEXT:    add x8, x0, #12
-; NONEON-NOSVE-NEXT:    ld1 { v0.s }[3], [x8]
-; NONEON-NOSVE-NEXT:    ret
   %load = call <4 x float> @llvm.masked.load.v4f32(ptr %src, i32 8, <4 x i1> %mask, <4 x float> zeroinitializer)
   ret <4 x float> %load
 }
@@ -1061,66 +290,6 @@ define <8 x float> @masked_load_v8f32(ptr %src, <8 x i1> %mask) {
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.8b, v0.8b, #7
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI10_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI10_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.8b, v0.8b, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    movi v1.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    addv b2, v0.8b
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    fmov w8, s2
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB10_9
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB10_10
-; NONEON-NOSVE-NEXT:  .LBB10_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB10_11
-; NONEON-NOSVE-NEXT:  .LBB10_3: // %else5
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB10_12
-; NONEON-NOSVE-NEXT:  .LBB10_4: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB10_13
-; NONEON-NOSVE-NEXT:  .LBB10_5: // %else11
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB10_14
-; NONEON-NOSVE-NEXT:  .LBB10_6: // %else14
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB10_15
-; NONEON-NOSVE-NEXT:  .LBB10_7: // %else17
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB10_16
-; NONEON-NOSVE-NEXT:  .LBB10_8: // %else20
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB10_9: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr s0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB10_2
-; NONEON-NOSVE-NEXT:  .LBB10_10: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x9, x0, #4
-; NONEON-NOSVE-NEXT:    ld1 { v0.s }[1], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB10_3
-; NONEON-NOSVE-NEXT:  .LBB10_11: // %cond.load4
-; NONEON-NOSVE-NEXT:    add x9, x0, #8
-; NONEON-NOSVE-NEXT:    ld1 { v0.s }[2], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB10_4
-; NONEON-NOSVE-NEXT:  .LBB10_12: // %cond.load7
-; NONEON-NOSVE-NEXT:    add x9, x0, #12
-; NONEON-NOSVE-NEXT:    ld1 { v0.s }[3], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB10_5
-; NONEON-NOSVE-NEXT:  .LBB10_13: // %cond.load10
-; NONEON-NOSVE-NEXT:    add x9, x0, #16
-; NONEON-NOSVE-NEXT:    ld1 { v1.s }[0], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB10_6
-; NONEON-NOSVE-NEXT:  .LBB10_14: // %cond.load13
-; NONEON-NOSVE-NEXT:    add x9, x0, #20
-; NONEON-NOSVE-NEXT:    ld1 { v1.s }[1], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB10_7
-; NONEON-NOSVE-NEXT:  .LBB10_15: // %cond.load16
-; NONEON-NOSVE-NEXT:    add x9, x0, #24
-; NONEON-NOSVE-NEXT:    ld1 { v1.s }[2], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB10_8
-; NONEON-NOSVE-NEXT:  .LBB10_16: // %cond.load19
-; NONEON-NOSVE-NEXT:    add x8, x0, #28
-; NONEON-NOSVE-NEXT:    ld1 { v1.s }[3], [x8]
-; NONEON-NOSVE-NEXT:    ret
   %load = call <8 x float> @llvm.masked.load.v8f32(ptr %src, i32 8, <8 x i1> %mask, <8 x float> zeroinitializer)
   ret <8 x float> %load
 }
@@ -1137,29 +306,6 @@ define <2 x double> @masked_load_v2f64(ptr %src, <2 x i1> %mask) {
 ; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #31
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI11_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI11_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.2s, v0.2s, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addp v1.2s, v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB11_3
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB11_4
-; NONEON-NOSVE-NEXT:  .LBB11_2: // %else2
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB11_3: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB11_2
-; NONEON-NOSVE-NEXT:  .LBB11_4: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x8, x0, #8
-; NONEON-NOSVE-NEXT:    ld1 { v0.d }[1], [x8]
-; NONEON-NOSVE-NEXT:    ret
   %load = call <2 x double> @llvm.masked.load.v2f64(ptr %src, i32 8, <2 x i1> %mask, <2 x double> zeroinitializer)
   ret <2 x double> %load
 }
@@ -1185,42 +331,6 @@ define <4 x double> @masked_load_v4f64(ptr %src, <4 x i1> %mask) {
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, x8, lsl #3]
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #15
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI12_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI12_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.4h, v0.4h, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    movi v1.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    addv h2, v0.4h
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    fmov w8, s2
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB12_5
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB12_6
-; NONEON-NOSVE-NEXT:  .LBB12_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB12_7
-; NONEON-NOSVE-NEXT:  .LBB12_3: // %else5
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB12_8
-; NONEON-NOSVE-NEXT:  .LBB12_4: // %else8
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB12_5: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB12_2
-; NONEON-NOSVE-NEXT:  .LBB12_6: // %cond.load1
-; NONEON-NOSVE-NEXT:    add x9, x0, #8
-; NONEON-NOSVE-NEXT:    ld1 { v0.d }[1], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB12_3
-; NONEON-NOSVE-NEXT:  .LBB12_7: // %cond.load4
-; NONEON-NOSVE-NEXT:    add x9, x0, #16
-; NONEON-NOSVE-NEXT:    ld1 { v1.d }[0], [x9]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB12_4
-; NONEON-NOSVE-NEXT:  .LBB12_8: // %cond.load7
-; NONEON-NOSVE-NEXT:    add x8, x0, #24
-; NONEON-NOSVE-NEXT:    ld1 { v1.d }[1], [x8]
-; NONEON-NOSVE-NEXT:    ret
   %load = call <4 x double> @llvm.masked.load.v4f64(ptr %src, i32 8, <4 x i1> %mask, <4 x double> zeroinitializer)
   ret <4 x double> %load
 }
@@ -1246,38 +356,6 @@ define <3 x i32> @masked_load_zext_v3i32(ptr %load_ptr, <3 x i1> %pm) {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_zext_v3i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #16
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    and w8, w1, #0x1
-; NONEON-NOSVE-NEXT:    bfi w8, w2, #1, #1
-; NONEON-NOSVE-NEXT:    bfi w8, w3, #2, #1
-; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB13_2
-; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr h0, [x0]
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB13_3
-; NONEON-NOSVE-NEXT:    b .LBB13_4
-; NONEON-NOSVE-NEXT:  .LBB13_2:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB13_4
-; NONEON-NOSVE-NEXT:  .LBB13_3: // %cond.load1
-; NONEON-NOSVE-NEXT:    mov v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add x9, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[1], [x9]
-; NONEON-NOSVE-NEXT:    mov v1.h[2], v0.h[2]
-; NONEON-NOSVE-NEXT:    fmov d0, d1
-; NONEON-NOSVE-NEXT:  .LBB13_4: // %else2
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB13_6
-; NONEON-NOSVE-NEXT:  // %bb.5: // %cond.load4
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v0.h[1]
-; NONEON-NOSVE-NEXT:    add x8, x0, #4
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[2], [x8]
-; NONEON-NOSVE-NEXT:  .LBB13_6: // %else5
-; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %load_value = tail call <3 x i16> @llvm.masked.load.v3i16.p0(ptr %load_ptr, i32 4, <3 x i1> %pm, <3 x i16> zeroinitializer)
   %extend = zext <3 x i16> %load_value to <3 x i32>
   ret <3 x i32> %extend;
@@ -1304,38 +382,6 @@ define <3 x i32> @masked_load_sext_v3i32(ptr %load_ptr, <3 x i1> %pm) {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_load_sext_v3i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    sub sp, sp, #16
-; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT:    and w8, w1, #0x1
-; NONEON-NOSVE-NEXT:    bfi w8, w2, #1, #1
-; NONEON-NOSVE-NEXT:    bfi w8, w3, #2, #1
-; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB14_2
-; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
-; NONEON-NOSVE-NEXT:    ldr h0, [x0]
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB14_3
-; NONEON-NOSVE-NEXT:    b .LBB14_4
-; NONEON-NOSVE-NEXT:  .LBB14_2:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB14_4
-; NONEON-NOSVE-NEXT:  .LBB14_3: // %cond.load1
-; NONEON-NOSVE-NEXT:    mov v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add x9, x0, #2
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[1], [x9]
-; NONEON-NOSVE-NEXT:    mov v1.h[2], v0.h[2]
-; NONEON-NOSVE-NEXT:    fmov d0, d1
-; NONEON-NOSVE-NEXT:  .LBB14_4: // %else2
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB14_6
-; NONEON-NOSVE-NEXT:  // %bb.5: // %cond.load4
-; NONEON-NOSVE-NEXT:    mov v0.h[1], v0.h[1]
-; NONEON-NOSVE-NEXT:    add x8, x0, #4
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[2], [x8]
-; NONEON-NOSVE-NEXT:  .LBB14_6: // %else5
-; NONEON-NOSVE-NEXT:    sshll v0.4s, v0.4h, #0
-; NONEON-NOSVE-NEXT:    add sp, sp, #16
-; NONEON-NOSVE-NEXT:    ret
   %load_value = tail call <3 x i16> @llvm.masked.load.v3i16.p0(ptr %load_ptr, i32 4, <3 x i1> %pm, <3 x i16> zeroinitializer)
   %extend = sext <3 x i16> %load_value to <3 x i32>
   ret <3 x i32> %extend;
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
index b175dcf3e9a0..f2b3f9b12ea7 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -20,37 +19,6 @@ define void @masked_store_v4i8(ptr %dst, <4 x i1> %mask) {
 ; CHECK-NEXT:    mov z0.h, #0 // =0x0
 ; CHECK-NEXT:    st1b { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #15
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI0_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI0_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.4h, v0.4h, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv h0, v0.4h
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB0_5
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB0_6
-; NONEON-NOSVE-NEXT:  .LBB0_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB0_7
-; NONEON-NOSVE-NEXT:  .LBB0_3: // %else4
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB0_8
-; NONEON-NOSVE-NEXT:  .LBB0_4: // %else6
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB0_5: // %cond.store
-; NONEON-NOSVE-NEXT:    strb wzr, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB0_2
-; NONEON-NOSVE-NEXT:  .LBB0_6: // %cond.store1
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #1]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB0_3
-; NONEON-NOSVE-NEXT:  .LBB0_7: // %cond.store3
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #2]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB0_4
-; NONEON-NOSVE-NEXT:  .LBB0_8: // %cond.store5
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #3]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v4i8(<4 x i8> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask)
   ret void
 }
@@ -66,57 +34,6 @@ define void @masked_store_v8i8(ptr %dst, <8 x i1> %mask) {
 ; CHECK-NEXT:    mov z0.b, #0 // =0x0
 ; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.8b, v0.8b, #7
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI1_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI1_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.8b, v0.8b, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv b0, v0.8b
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB1_9
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB1_10
-; NONEON-NOSVE-NEXT:  .LBB1_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB1_11
-; NONEON-NOSVE-NEXT:  .LBB1_3: // %else4
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB1_12
-; NONEON-NOSVE-NEXT:  .LBB1_4: // %else6
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB1_13
-; NONEON-NOSVE-NEXT:  .LBB1_5: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB1_14
-; NONEON-NOSVE-NEXT:  .LBB1_6: // %else10
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB1_15
-; NONEON-NOSVE-NEXT:  .LBB1_7: // %else12
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB1_16
-; NONEON-NOSVE-NEXT:  .LBB1_8: // %else14
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB1_9: // %cond.store
-; NONEON-NOSVE-NEXT:    strb wzr, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB1_2
-; NONEON-NOSVE-NEXT:  .LBB1_10: // %cond.store1
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #1]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB1_3
-; NONEON-NOSVE-NEXT:  .LBB1_11: // %cond.store3
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #2]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB1_4
-; NONEON-NOSVE-NEXT:  .LBB1_12: // %cond.store5
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #3]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB1_5
-; NONEON-NOSVE-NEXT:  .LBB1_13: // %cond.store7
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #4]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB1_6
-; NONEON-NOSVE-NEXT:  .LBB1_14: // %cond.store9
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #5]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB1_7
-; NONEON-NOSVE-NEXT:  .LBB1_15: // %cond.store11
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #6]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB1_8
-; NONEON-NOSVE-NEXT:  .LBB1_16: // %cond.store13
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #7]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v8i8(<8 x i8> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask)
   ret void
 }
@@ -132,99 +49,6 @@ define void @masked_store_v16i8(ptr %dst, <16 x i1> %mask) {
 ; CHECK-NEXT:    mov z0.b, #0 // =0x0
 ; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.16b, v0.16b, #7
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI2_0
-; NONEON-NOSVE-NEXT:    ldr q1, [x8, :lo12:.LCPI2_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.16b, v0.16b, #0
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; NONEON-NOSVE-NEXT:    zip1 v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    addv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB2_17
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB2_18
-; NONEON-NOSVE-NEXT:  .LBB2_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB2_19
-; NONEON-NOSVE-NEXT:  .LBB2_3: // %else4
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB2_20
-; NONEON-NOSVE-NEXT:  .LBB2_4: // %else6
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB2_21
-; NONEON-NOSVE-NEXT:  .LBB2_5: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB2_22
-; NONEON-NOSVE-NEXT:  .LBB2_6: // %else10
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB2_23
-; NONEON-NOSVE-NEXT:  .LBB2_7: // %else12
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB2_24
-; NONEON-NOSVE-NEXT:  .LBB2_8: // %else14
-; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB2_25
-; NONEON-NOSVE-NEXT:  .LBB2_9: // %else16
-; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB2_26
-; NONEON-NOSVE-NEXT:  .LBB2_10: // %else18
-; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB2_27
-; NONEON-NOSVE-NEXT:  .LBB2_11: // %else20
-; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB2_28
-; NONEON-NOSVE-NEXT:  .LBB2_12: // %else22
-; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB2_29
-; NONEON-NOSVE-NEXT:  .LBB2_13: // %else24
-; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB2_30
-; NONEON-NOSVE-NEXT:  .LBB2_14: // %else26
-; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB2_31
-; NONEON-NOSVE-NEXT:  .LBB2_15: // %else28
-; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB2_32
-; NONEON-NOSVE-NEXT:  .LBB2_16: // %else30
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB2_17: // %cond.store
-; NONEON-NOSVE-NEXT:    strb wzr, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB2_2
-; NONEON-NOSVE-NEXT:  .LBB2_18: // %cond.store1
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #1]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB2_3
-; NONEON-NOSVE-NEXT:  .LBB2_19: // %cond.store3
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #2]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB2_4
-; NONEON-NOSVE-NEXT:  .LBB2_20: // %cond.store5
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #3]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB2_5
-; NONEON-NOSVE-NEXT:  .LBB2_21: // %cond.store7
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #4]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB2_6
-; NONEON-NOSVE-NEXT:  .LBB2_22: // %cond.store9
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #5]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB2_7
-; NONEON-NOSVE-NEXT:  .LBB2_23: // %cond.store11
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #6]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB2_8
-; NONEON-NOSVE-NEXT:  .LBB2_24: // %cond.store13
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #7]
-; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB2_9
-; NONEON-NOSVE-NEXT:  .LBB2_25: // %cond.store15
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #8]
-; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB2_10
-; NONEON-NOSVE-NEXT:  .LBB2_26: // %cond.store17
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #9]
-; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB2_11
-; NONEON-NOSVE-NEXT:  .LBB2_27: // %cond.store19
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #10]
-; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB2_12
-; NONEON-NOSVE-NEXT:  .LBB2_28: // %cond.store21
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #11]
-; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB2_13
-; NONEON-NOSVE-NEXT:  .LBB2_29: // %cond.store23
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #12]
-; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB2_14
-; NONEON-NOSVE-NEXT:  .LBB2_30: // %cond.store25
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #13]
-; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB2_15
-; NONEON-NOSVE-NEXT:  .LBB2_31: // %cond.store27
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #14]
-; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB2_16
-; NONEON-NOSVE-NEXT:  .LBB2_32: // %cond.store29
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #15]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v16i8(<16 x i8> zeroinitializer, ptr %dst, i32 8, <16 x i1> %mask)
   ret void
 }
@@ -305,244 +129,6 @@ define void @masked_store_v32i8(ptr %dst, <32 x i1> %mask) {
 ; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #72]
-; NONEON-NOSVE-NEXT:    fmov s1, w1
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #80]
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #88]
-; NONEON-NOSVE-NEXT:    mov v1.b[1], w2
-; NONEON-NOSVE-NEXT:    mov v0.b[1], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp]
-; NONEON-NOSVE-NEXT:    mov v1.b[2], w3
-; NONEON-NOSVE-NEXT:    mov v0.b[2], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #96]
-; NONEON-NOSVE-NEXT:    mov v1.b[3], w4
-; NONEON-NOSVE-NEXT:    mov v0.b[3], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #104]
-; NONEON-NOSVE-NEXT:    mov v1.b[4], w5
-; NONEON-NOSVE-NEXT:    mov v0.b[4], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #112]
-; NONEON-NOSVE-NEXT:    mov v1.b[5], w6
-; NONEON-NOSVE-NEXT:    mov v0.b[5], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #120]
-; NONEON-NOSVE-NEXT:    mov v1.b[6], w7
-; NONEON-NOSVE-NEXT:    mov v0.b[6], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #128]
-; NONEON-NOSVE-NEXT:    mov v1.b[7], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8]
-; NONEON-NOSVE-NEXT:    mov v0.b[7], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #136]
-; NONEON-NOSVE-NEXT:    mov v1.b[8], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #16]
-; NONEON-NOSVE-NEXT:    mov v0.b[8], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #144]
-; NONEON-NOSVE-NEXT:    mov v1.b[9], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #24]
-; NONEON-NOSVE-NEXT:    mov v0.b[9], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #152]
-; NONEON-NOSVE-NEXT:    mov v1.b[10], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #32]
-; NONEON-NOSVE-NEXT:    mov v0.b[10], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #160]
-; NONEON-NOSVE-NEXT:    mov v1.b[11], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #40]
-; NONEON-NOSVE-NEXT:    mov v0.b[11], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #168]
-; NONEON-NOSVE-NEXT:    mov v1.b[12], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #48]
-; NONEON-NOSVE-NEXT:    mov v0.b[12], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #176]
-; NONEON-NOSVE-NEXT:    mov v1.b[13], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #56]
-; NONEON-NOSVE-NEXT:    mov v0.b[13], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #184]
-; NONEON-NOSVE-NEXT:    mov v1.b[14], w9
-; NONEON-NOSVE-NEXT:    ldr w9, [sp, #64]
-; NONEON-NOSVE-NEXT:    mov v0.b[14], w8
-; NONEON-NOSVE-NEXT:    ldr w8, [sp, #192]
-; NONEON-NOSVE-NEXT:    mov v1.b[15], w9
-; NONEON-NOSVE-NEXT:    mov v0.b[15], w8
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI3_0
-; NONEON-NOSVE-NEXT:    ldr q2, [x8, :lo12:.LCPI3_0]
-; NONEON-NOSVE-NEXT:    shl v1.16b, v1.16b, #7
-; NONEON-NOSVE-NEXT:    shl v0.16b, v0.16b, #7
-; NONEON-NOSVE-NEXT:    cmlt v1.16b, v1.16b, #0
-; NONEON-NOSVE-NEXT:    cmlt v0.16b, v0.16b, #0
-; NONEON-NOSVE-NEXT:    and v1.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; NONEON-NOSVE-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; NONEON-NOSVE-NEXT:    zip1 v1.16b, v1.16b, v3.16b
-; NONEON-NOSVE-NEXT:    zip1 v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    addv h1, v1.8h
-; NONEON-NOSVE-NEXT:    addv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w8, s1
-; NONEON-NOSVE-NEXT:    fmov w9, s0
-; NONEON-NOSVE-NEXT:    bfi w8, w9, #16, #16
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB3_33
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB3_34
-; NONEON-NOSVE-NEXT:  .LBB3_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB3_35
-; NONEON-NOSVE-NEXT:  .LBB3_3: // %else4
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB3_36
-; NONEON-NOSVE-NEXT:  .LBB3_4: // %else6
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB3_37
-; NONEON-NOSVE-NEXT:  .LBB3_5: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB3_38
-; NONEON-NOSVE-NEXT:  .LBB3_6: // %else10
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB3_39
-; NONEON-NOSVE-NEXT:  .LBB3_7: // %else12
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB3_40
-; NONEON-NOSVE-NEXT:  .LBB3_8: // %else14
-; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB3_41
-; NONEON-NOSVE-NEXT:  .LBB3_9: // %else16
-; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB3_42
-; NONEON-NOSVE-NEXT:  .LBB3_10: // %else18
-; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB3_43
-; NONEON-NOSVE-NEXT:  .LBB3_11: // %else20
-; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB3_44
-; NONEON-NOSVE-NEXT:  .LBB3_12: // %else22
-; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB3_45
-; NONEON-NOSVE-NEXT:  .LBB3_13: // %else24
-; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB3_46
-; NONEON-NOSVE-NEXT:  .LBB3_14: // %else26
-; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB3_47
-; NONEON-NOSVE-NEXT:  .LBB3_15: // %else28
-; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB3_48
-; NONEON-NOSVE-NEXT:  .LBB3_16: // %else30
-; NONEON-NOSVE-NEXT:    tbnz w8, #16, .LBB3_49
-; NONEON-NOSVE-NEXT:  .LBB3_17: // %else32
-; NONEON-NOSVE-NEXT:    tbnz w8, #17, .LBB3_50
-; NONEON-NOSVE-NEXT:  .LBB3_18: // %else34
-; NONEON-NOSVE-NEXT:    tbnz w8, #18, .LBB3_51
-; NONEON-NOSVE-NEXT:  .LBB3_19: // %else36
-; NONEON-NOSVE-NEXT:    tbnz w8, #19, .LBB3_52
-; NONEON-NOSVE-NEXT:  .LBB3_20: // %else38
-; NONEON-NOSVE-NEXT:    tbnz w8, #20, .LBB3_53
-; NONEON-NOSVE-NEXT:  .LBB3_21: // %else40
-; NONEON-NOSVE-NEXT:    tbnz w8, #21, .LBB3_54
-; NONEON-NOSVE-NEXT:  .LBB3_22: // %else42
-; NONEON-NOSVE-NEXT:    tbnz w8, #22, .LBB3_55
-; NONEON-NOSVE-NEXT:  .LBB3_23: // %else44
-; NONEON-NOSVE-NEXT:    tbnz w8, #23, .LBB3_56
-; NONEON-NOSVE-NEXT:  .LBB3_24: // %else46
-; NONEON-NOSVE-NEXT:    tbnz w8, #24, .LBB3_57
-; NONEON-NOSVE-NEXT:  .LBB3_25: // %else48
-; NONEON-NOSVE-NEXT:    tbnz w8, #25, .LBB3_58
-; NONEON-NOSVE-NEXT:  .LBB3_26: // %else50
-; NONEON-NOSVE-NEXT:    tbnz w8, #26, .LBB3_59
-; NONEON-NOSVE-NEXT:  .LBB3_27: // %else52
-; NONEON-NOSVE-NEXT:    tbnz w8, #27, .LBB3_60
-; NONEON-NOSVE-NEXT:  .LBB3_28: // %else54
-; NONEON-NOSVE-NEXT:    tbnz w8, #28, .LBB3_61
-; NONEON-NOSVE-NEXT:  .LBB3_29: // %else56
-; NONEON-NOSVE-NEXT:    tbnz w8, #29, .LBB3_62
-; NONEON-NOSVE-NEXT:  .LBB3_30: // %else58
-; NONEON-NOSVE-NEXT:    tbnz w8, #30, .LBB3_63
-; NONEON-NOSVE-NEXT:  .LBB3_31: // %else60
-; NONEON-NOSVE-NEXT:    tbnz w8, #31, .LBB3_64
-; NONEON-NOSVE-NEXT:  .LBB3_32: // %else62
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB3_33: // %cond.store
-; NONEON-NOSVE-NEXT:    strb wzr, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB3_2
-; NONEON-NOSVE-NEXT:  .LBB3_34: // %cond.store1
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #1]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB3_3
-; NONEON-NOSVE-NEXT:  .LBB3_35: // %cond.store3
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #2]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB3_4
-; NONEON-NOSVE-NEXT:  .LBB3_36: // %cond.store5
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #3]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB3_5
-; NONEON-NOSVE-NEXT:  .LBB3_37: // %cond.store7
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #4]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB3_6
-; NONEON-NOSVE-NEXT:  .LBB3_38: // %cond.store9
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #5]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB3_7
-; NONEON-NOSVE-NEXT:  .LBB3_39: // %cond.store11
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #6]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB3_8
-; NONEON-NOSVE-NEXT:  .LBB3_40: // %cond.store13
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #7]
-; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB3_9
-; NONEON-NOSVE-NEXT:  .LBB3_41: // %cond.store15
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #8]
-; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB3_10
-; NONEON-NOSVE-NEXT:  .LBB3_42: // %cond.store17
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #9]
-; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB3_11
-; NONEON-NOSVE-NEXT:  .LBB3_43: // %cond.store19
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #10]
-; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB3_12
-; NONEON-NOSVE-NEXT:  .LBB3_44: // %cond.store21
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #11]
-; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB3_13
-; NONEON-NOSVE-NEXT:  .LBB3_45: // %cond.store23
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #12]
-; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB3_14
-; NONEON-NOSVE-NEXT:  .LBB3_46: // %cond.store25
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #13]
-; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB3_15
-; NONEON-NOSVE-NEXT:  .LBB3_47: // %cond.store27
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #14]
-; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB3_16
-; NONEON-NOSVE-NEXT:  .LBB3_48: // %cond.store29
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #15]
-; NONEON-NOSVE-NEXT:    tbz w8, #16, .LBB3_17
-; NONEON-NOSVE-NEXT:  .LBB3_49: // %cond.store31
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #16]
-; NONEON-NOSVE-NEXT:    tbz w8, #17, .LBB3_18
-; NONEON-NOSVE-NEXT:  .LBB3_50: // %cond.store33
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #17]
-; NONEON-NOSVE-NEXT:    tbz w8, #18, .LBB3_19
-; NONEON-NOSVE-NEXT:  .LBB3_51: // %cond.store35
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #18]
-; NONEON-NOSVE-NEXT:    tbz w8, #19, .LBB3_20
-; NONEON-NOSVE-NEXT:  .LBB3_52: // %cond.store37
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #19]
-; NONEON-NOSVE-NEXT:    tbz w8, #20, .LBB3_21
-; NONEON-NOSVE-NEXT:  .LBB3_53: // %cond.store39
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #20]
-; NONEON-NOSVE-NEXT:    tbz w8, #21, .LBB3_22
-; NONEON-NOSVE-NEXT:  .LBB3_54: // %cond.store41
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #21]
-; NONEON-NOSVE-NEXT:    tbz w8, #22, .LBB3_23
-; NONEON-NOSVE-NEXT:  .LBB3_55: // %cond.store43
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #22]
-; NONEON-NOSVE-NEXT:    tbz w8, #23, .LBB3_24
-; NONEON-NOSVE-NEXT:  .LBB3_56: // %cond.store45
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #23]
-; NONEON-NOSVE-NEXT:    tbz w8, #24, .LBB3_25
-; NONEON-NOSVE-NEXT:  .LBB3_57: // %cond.store47
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #24]
-; NONEON-NOSVE-NEXT:    tbz w8, #25, .LBB3_26
-; NONEON-NOSVE-NEXT:  .LBB3_58: // %cond.store49
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #25]
-; NONEON-NOSVE-NEXT:    tbz w8, #26, .LBB3_27
-; NONEON-NOSVE-NEXT:  .LBB3_59: // %cond.store51
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #26]
-; NONEON-NOSVE-NEXT:    tbz w8, #27, .LBB3_28
-; NONEON-NOSVE-NEXT:  .LBB3_60: // %cond.store53
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #27]
-; NONEON-NOSVE-NEXT:    tbz w8, #28, .LBB3_29
-; NONEON-NOSVE-NEXT:  .LBB3_61: // %cond.store55
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #28]
-; NONEON-NOSVE-NEXT:    tbz w8, #29, .LBB3_30
-; NONEON-NOSVE-NEXT:  .LBB3_62: // %cond.store57
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #29]
-; NONEON-NOSVE-NEXT:    tbz w8, #30, .LBB3_31
-; NONEON-NOSVE-NEXT:  .LBB3_63: // %cond.store59
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #30]
-; NONEON-NOSVE-NEXT:    tbz w8, #31, .LBB3_32
-; NONEON-NOSVE-NEXT:  .LBB3_64: // %cond.store61
-; NONEON-NOSVE-NEXT:    strb wzr, [x0, #31]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v32i8(<32 x i8> zeroinitializer, ptr %dst, i32 8, <32 x i1> %mask)
   ret void
 }
@@ -568,29 +154,6 @@ define void @masked_store_v2f16(ptr %dst, <2 x i1> %mask) {
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #31
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI4_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI4_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.2s, v0.2s, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addp v0.2s, v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB4_3
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB4_4
-; NONEON-NOSVE-NEXT:  .LBB4_2: // %else2
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB4_3: // %cond.store
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB4_2
-; NONEON-NOSVE-NEXT:  .LBB4_4: // %cond.store1
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #2]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v2f16(<2 x half> zeroinitializer, ptr %dst, i32 8, <2 x i1> %mask)
   ret void
 }
@@ -606,41 +169,6 @@ define void @masked_store_v4f16(ptr %dst, <4 x i1> %mask) {
 ; CHECK-NEXT:    mov z0.h, #0 // =0x0
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #15
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI5_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI5_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.4h, v0.4h, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv h0, v0.4h
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB5_5
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB5_6
-; NONEON-NOSVE-NEXT:  .LBB5_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB5_7
-; NONEON-NOSVE-NEXT:  .LBB5_3: // %else4
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB5_8
-; NONEON-NOSVE-NEXT:  .LBB5_4: // %else6
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB5_5: // %cond.store
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB5_2
-; NONEON-NOSVE-NEXT:  .LBB5_6: // %cond.store1
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #2]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB5_3
-; NONEON-NOSVE-NEXT:  .LBB5_7: // %cond.store3
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #4]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB5_4
-; NONEON-NOSVE-NEXT:  .LBB5_8: // %cond.store5
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #6]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v4f16(<4 x half> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask)
   ret void
 }
@@ -657,65 +185,6 @@ define void @masked_store_v8f16(ptr %dst, <8 x i1> %mask) {
 ; CHECK-NEXT:    mov z0.h, #0 // =0x0
 ; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.8b, v0.8b, #7
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI6_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI6_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.8b, v0.8b, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv b0, v0.8b
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB6_9
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB6_10
-; NONEON-NOSVE-NEXT:  .LBB6_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB6_11
-; NONEON-NOSVE-NEXT:  .LBB6_3: // %else4
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB6_12
-; NONEON-NOSVE-NEXT:  .LBB6_4: // %else6
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB6_13
-; NONEON-NOSVE-NEXT:  .LBB6_5: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB6_14
-; NONEON-NOSVE-NEXT:  .LBB6_6: // %else10
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB6_15
-; NONEON-NOSVE-NEXT:  .LBB6_7: // %else12
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB6_16
-; NONEON-NOSVE-NEXT:  .LBB6_8: // %else14
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB6_9: // %cond.store
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB6_2
-; NONEON-NOSVE-NEXT:  .LBB6_10: // %cond.store1
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #2]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB6_3
-; NONEON-NOSVE-NEXT:  .LBB6_11: // %cond.store3
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #4]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB6_4
-; NONEON-NOSVE-NEXT:  .LBB6_12: // %cond.store5
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #6]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB6_5
-; NONEON-NOSVE-NEXT:  .LBB6_13: // %cond.store7
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #8]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB6_6
-; NONEON-NOSVE-NEXT:  .LBB6_14: // %cond.store9
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #10]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB6_7
-; NONEON-NOSVE-NEXT:  .LBB6_15: // %cond.store11
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #12]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB6_8
-; NONEON-NOSVE-NEXT:  .LBB6_16: // %cond.store13
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #14]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v8f16(<8 x half> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask)
   ret void
 }
@@ -740,115 +209,6 @@ define void @masked_store_v16f16(ptr %dst, <16 x i1> %mask) {
 ; CHECK-NEXT:    st1h { z1.h }, p1, [x0, x8, lsl #1]
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.16b, v0.16b, #7
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI7_0
-; NONEON-NOSVE-NEXT:    ldr q1, [x8, :lo12:.LCPI7_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.16b, v0.16b, #0
-; NONEON-NOSVE-NEXT:    and v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; NONEON-NOSVE-NEXT:    zip1 v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    addv h0, v0.8h
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB7_17
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB7_18
-; NONEON-NOSVE-NEXT:  .LBB7_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB7_19
-; NONEON-NOSVE-NEXT:  .LBB7_3: // %else4
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB7_20
-; NONEON-NOSVE-NEXT:  .LBB7_4: // %else6
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB7_21
-; NONEON-NOSVE-NEXT:  .LBB7_5: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB7_22
-; NONEON-NOSVE-NEXT:  .LBB7_6: // %else10
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB7_23
-; NONEON-NOSVE-NEXT:  .LBB7_7: // %else12
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB7_24
-; NONEON-NOSVE-NEXT:  .LBB7_8: // %else14
-; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB7_25
-; NONEON-NOSVE-NEXT:  .LBB7_9: // %else16
-; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB7_26
-; NONEON-NOSVE-NEXT:  .LBB7_10: // %else18
-; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB7_27
-; NONEON-NOSVE-NEXT:  .LBB7_11: // %else20
-; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB7_28
-; NONEON-NOSVE-NEXT:  .LBB7_12: // %else22
-; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB7_29
-; NONEON-NOSVE-NEXT:  .LBB7_13: // %else24
-; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB7_30
-; NONEON-NOSVE-NEXT:  .LBB7_14: // %else26
-; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB7_31
-; NONEON-NOSVE-NEXT:  .LBB7_15: // %else28
-; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB7_32
-; NONEON-NOSVE-NEXT:  .LBB7_16: // %else30
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB7_17: // %cond.store
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB7_2
-; NONEON-NOSVE-NEXT:  .LBB7_18: // %cond.store1
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #2]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB7_3
-; NONEON-NOSVE-NEXT:  .LBB7_19: // %cond.store3
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #4]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB7_4
-; NONEON-NOSVE-NEXT:  .LBB7_20: // %cond.store5
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #6]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB7_5
-; NONEON-NOSVE-NEXT:  .LBB7_21: // %cond.store7
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #8]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB7_6
-; NONEON-NOSVE-NEXT:  .LBB7_22: // %cond.store9
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #10]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB7_7
-; NONEON-NOSVE-NEXT:  .LBB7_23: // %cond.store11
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #12]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB7_8
-; NONEON-NOSVE-NEXT:  .LBB7_24: // %cond.store13
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #14]
-; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB7_9
-; NONEON-NOSVE-NEXT:  .LBB7_25: // %cond.store15
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #16]
-; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB7_10
-; NONEON-NOSVE-NEXT:  .LBB7_26: // %cond.store17
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #18]
-; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB7_11
-; NONEON-NOSVE-NEXT:  .LBB7_27: // %cond.store19
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #20]
-; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB7_12
-; NONEON-NOSVE-NEXT:  .LBB7_28: // %cond.store21
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #22]
-; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB7_13
-; NONEON-NOSVE-NEXT:  .LBB7_29: // %cond.store23
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #24]
-; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB7_14
-; NONEON-NOSVE-NEXT:  .LBB7_30: // %cond.store25
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #26]
-; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB7_15
-; NONEON-NOSVE-NEXT:  .LBB7_31: // %cond.store27
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #28]
-; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB7_16
-; NONEON-NOSVE-NEXT:  .LBB7_32: // %cond.store29
-; NONEON-NOSVE-NEXT:    fmov s0, wzr
-; NONEON-NOSVE-NEXT:    str h0, [x0, #30]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v16f16(<16 x half> zeroinitializer, ptr %dst, i32 8, <16 x i1> %mask)
   ret void
 }
@@ -865,37 +225,6 @@ define void @masked_store_v4f32(ptr %dst, <4 x i1> %mask) {
 ; CHECK-NEXT:    mov z0.s, #0 // =0x0
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #15
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI8_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI8_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.4h, v0.4h, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv h0, v0.4h
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB8_5
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB8_6
-; NONEON-NOSVE-NEXT:  .LBB8_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB8_7
-; NONEON-NOSVE-NEXT:  .LBB8_3: // %else4
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB8_8
-; NONEON-NOSVE-NEXT:  .LBB8_4: // %else6
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB8_5: // %cond.store
-; NONEON-NOSVE-NEXT:    str wzr, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB8_2
-; NONEON-NOSVE-NEXT:  .LBB8_6: // %cond.store1
-; NONEON-NOSVE-NEXT:    str wzr, [x0, #4]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB8_3
-; NONEON-NOSVE-NEXT:  .LBB8_7: // %cond.store3
-; NONEON-NOSVE-NEXT:    str wzr, [x0, #8]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB8_4
-; NONEON-NOSVE-NEXT:  .LBB8_8: // %cond.store5
-; NONEON-NOSVE-NEXT:    str wzr, [x0, #12]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v4f32(<4 x float> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask)
   ret void
 }
@@ -946,57 +275,6 @@ define void @masked_store_v8f32(ptr %dst, <8 x i1> %mask) {
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.8b, v0.8b, #7
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI9_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI9_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.8b, v0.8b, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv b0, v0.8b
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB9_9
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB9_10
-; NONEON-NOSVE-NEXT:  .LBB9_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB9_11
-; NONEON-NOSVE-NEXT:  .LBB9_3: // %else4
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB9_12
-; NONEON-NOSVE-NEXT:  .LBB9_4: // %else6
-; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB9_13
-; NONEON-NOSVE-NEXT:  .LBB9_5: // %else8
-; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB9_14
-; NONEON-NOSVE-NEXT:  .LBB9_6: // %else10
-; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB9_15
-; NONEON-NOSVE-NEXT:  .LBB9_7: // %else12
-; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB9_16
-; NONEON-NOSVE-NEXT:  .LBB9_8: // %else14
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB9_9: // %cond.store
-; NONEON-NOSVE-NEXT:    str wzr, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB9_2
-; NONEON-NOSVE-NEXT:  .LBB9_10: // %cond.store1
-; NONEON-NOSVE-NEXT:    str wzr, [x0, #4]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB9_3
-; NONEON-NOSVE-NEXT:  .LBB9_11: // %cond.store3
-; NONEON-NOSVE-NEXT:    str wzr, [x0, #8]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB9_4
-; NONEON-NOSVE-NEXT:  .LBB9_12: // %cond.store5
-; NONEON-NOSVE-NEXT:    str wzr, [x0, #12]
-; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB9_5
-; NONEON-NOSVE-NEXT:  .LBB9_13: // %cond.store7
-; NONEON-NOSVE-NEXT:    str wzr, [x0, #16]
-; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB9_6
-; NONEON-NOSVE-NEXT:  .LBB9_14: // %cond.store9
-; NONEON-NOSVE-NEXT:    str wzr, [x0, #20]
-; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB9_7
-; NONEON-NOSVE-NEXT:  .LBB9_15: // %cond.store11
-; NONEON-NOSVE-NEXT:    str wzr, [x0, #24]
-; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB9_8
-; NONEON-NOSVE-NEXT:  .LBB9_16: // %cond.store13
-; NONEON-NOSVE-NEXT:    str wzr, [x0, #28]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v8f32(<8 x float> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask)
   ret void
 }
@@ -1013,27 +291,6 @@ define void @masked_store_v2f64(ptr %dst, <2 x i1> %mask) {
 ; CHECK-NEXT:    mov z0.d, #0 // =0x0
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #31
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI10_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI10_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.2s, v0.2s, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addp v0.2s, v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB10_3
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB10_4
-; NONEON-NOSVE-NEXT:  .LBB10_2: // %else2
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB10_3: // %cond.store
-; NONEON-NOSVE-NEXT:    str xzr, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB10_2
-; NONEON-NOSVE-NEXT:  .LBB10_4: // %cond.store1
-; NONEON-NOSVE-NEXT:    str xzr, [x0, #8]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v2f64(<2 x double> zeroinitializer, ptr %dst, i32 8, <2 x i1> %mask)
   ret void
 }
@@ -1058,37 +315,6 @@ define void @masked_store_v4f64(ptr %dst, <4 x i1> %mask) {
 ; CHECK-NEXT:    st1d { z0.d }, p1, [x0, x8, lsl #3]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: masked_store_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #15
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI11_0
-; NONEON-NOSVE-NEXT:    ldr d1, [x8, :lo12:.LCPI11_0]
-; NONEON-NOSVE-NEXT:    cmlt v0.4h, v0.4h, #0
-; NONEON-NOSVE-NEXT:    and v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    addv h0, v0.4h
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB11_5
-; NONEON-NOSVE-NEXT:  // %bb.1: // %else
-; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB11_6
-; NONEON-NOSVE-NEXT:  .LBB11_2: // %else2
-; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB11_7
-; NONEON-NOSVE-NEXT:  .LBB11_3: // %else4
-; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB11_8
-; NONEON-NOSVE-NEXT:  .LBB11_4: // %else6
-; NONEON-NOSVE-NEXT:    ret
-; NONEON-NOSVE-NEXT:  .LBB11_5: // %cond.store
-; NONEON-NOSVE-NEXT:    str xzr, [x0]
-; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB11_2
-; NONEON-NOSVE-NEXT:  .LBB11_6: // %cond.store1
-; NONEON-NOSVE-NEXT:    str xzr, [x0, #8]
-; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB11_3
-; NONEON-NOSVE-NEXT:  .LBB11_7: // %cond.store3
-; NONEON-NOSVE-NEXT:    str xzr, [x0, #16]
-; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB11_4
-; NONEON-NOSVE-NEXT:  .LBB11_8: // %cond.store5
-; NONEON-NOSVE-NEXT:    str xzr, [x0, #24]
-; NONEON-NOSVE-NEXT:    ret
   call void @llvm.masked.store.v4f64(<4 x double> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask)
   ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll
index d7eaf766e7df..b5adea594242 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -15,15 +14,6 @@ define void @add_v4i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z0.h, z0.h, z1.h
 ; CHECK-NEXT:    st1b { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr s0, [x0]
-; NONEON-NOSVE-NEXT:    ldr s1, [x1]
-; NONEON-NOSVE-NEXT:    uaddl v0.8h, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    str s0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i8>, ptr %a
   %op2 = load <4 x i8>, ptr %b
   %res = add <4 x i8> %op1, %op2
@@ -39,14 +29,6 @@ define void @add_v8i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z0.b, z0.b, z1.b
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ldr d1, [x1]
-; NONEON-NOSVE-NEXT:    add v0.8b, v0.8b, v1.8b
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i8>, ptr %a
   %op2 = load <8 x i8>, ptr %b
   %res = add <8 x i8> %op1, %op2
@@ -62,14 +44,6 @@ define void @add_v16i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z0.b, z0.b, z1.b
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i8>, ptr %a
   %op2 = load <16 x i8>, ptr %b
   %res = add <16 x i8> %op1, %op2
@@ -86,15 +60,6 @@ define void @add_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z1.b, z2.b, z3.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    add v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %res = add <32 x i8> %op1, %op2
@@ -111,23 +76,6 @@ define void @add_v2i16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    add z0.s, z0.s, z1.s
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldrh w8, [x0]
-; NONEON-NOSVE-NEXT:    ldrh w9, [x1]
-; NONEON-NOSVE-NEXT:    fmov s0, w8
-; NONEON-NOSVE-NEXT:    fmov s1, w9
-; NONEON-NOSVE-NEXT:    add x8, x0, #2
-; NONEON-NOSVE-NEXT:    add x9, x1, #2
-; NONEON-NOSVE-NEXT:    ld1 { v0.h }[2], [x8]
-; NONEON-NOSVE-NEXT:    ld1 { v1.h }[2], [x9]
-; NONEON-NOSVE-NEXT:    add v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    mov w8, v0.s[1]
-; NONEON-NOSVE-NEXT:    fmov w9, s0
-; NONEON-NOSVE-NEXT:    strh w9, [x0]
-; NONEON-NOSVE-NEXT:    strh w8, [x0, #2]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x i16>, ptr %a
   %op2 = load <2 x i16>, ptr %b
   %res = add <2 x i16> %op1, %op2
@@ -143,14 +91,6 @@ define void @add_v4i16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    add z0.h, z0.h, z1.h
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ldr d1, [x1]
-; NONEON-NOSVE-NEXT:    add v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i16>, ptr %a
   %op2 = load <4 x i16>, ptr %b
   %res = add <4 x i16> %op1, %op2
@@ -166,14 +106,6 @@ define void @add_v8i16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    add z0.h, z0.h, z1.h
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %op2 = load <8 x i16>, ptr %b
   %res = add <8 x i16> %op1, %op2
@@ -190,15 +122,6 @@ define void @add_v16i16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    add z1.h, z2.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: add_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    add v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %res = add <16 x i16> %op1, %op2
@@ -214,13 +137,6 @@ define void @abs_v2i32(ptr %a) {
 ; CHECK-NEXT:    abs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    abs v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x i32>, ptr %a
   %res = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %op1, i1 false)
   store <2 x i32> %res, ptr %a
@@ -235,13 +151,6 @@ define void @abs_v4i32(ptr %a) {
 ; CHECK-NEXT:    abs z0.s, p0/m, z0.s
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    abs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i32>, ptr %a
   %res = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %op1, i1 false)
   store <4 x i32> %res, ptr %a
@@ -257,14 +166,6 @@ define void @abs_v8i32(ptr %a) {
 ; CHECK-NEXT:    abs z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    abs v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    abs v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %res = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %op1, i1 false)
   store <8 x i32> %res, ptr %a
@@ -279,13 +180,6 @@ define void @abs_v2i64(ptr %a) {
 ; CHECK-NEXT:    abs z0.d, p0/m, z0.d
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    abs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x i64>, ptr %a
   %res = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %op1, i1 false)
   store <2 x i64> %res, ptr %a
@@ -301,14 +195,6 @@ define void @abs_v4i64(ptr %a) {
 ; CHECK-NEXT:    abs z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: abs_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    abs v0.2d, v0.2d
-; NONEON-NOSVE-NEXT:    abs v1.2d, v1.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %res = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %op1, i1 false)
   store <4 x i64> %res, ptr %a
@@ -325,17 +211,6 @@ define void @fadd_v2f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    str w8, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr s0, [x0]
-; NONEON-NOSVE-NEXT:    ldr s1, [x1]
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    str s0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x half>, ptr %a
   %op2 = load <2 x half>, ptr %b
   %res = fadd <2 x half> %op1, %op2
@@ -352,17 +227,6 @@ define void @fadd_v4f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ldr d1, [x1]
-; NONEON-NOSVE-NEXT:    fcvtl v1.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v0.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x half>, ptr %a
   %op2 = load <4 x half>, ptr %b
   %res = fadd <4 x half> %op1, %op2
@@ -379,21 +243,6 @@ define void @fadd_v8f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fcvtl v2.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v3.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fadd v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v2.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v0.4s
-; NONEON-NOSVE-NEXT:    str q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x half>, ptr %a
   %op2 = load <8 x half>, ptr %b
   %res = fadd <8 x half> %op1, %op2
@@ -412,29 +261,6 @@ define void @fadd_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z1.h, p0/m, z1.h, z3.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fcvtl v4.4s, v0.4h
-; NONEON-NOSVE-NEXT:    fcvtl v6.4s, v3.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v0.4s, v0.8h
-; NONEON-NOSVE-NEXT:    fcvtl v5.4s, v1.4h
-; NONEON-NOSVE-NEXT:    fcvtl v7.4s, v2.4h
-; NONEON-NOSVE-NEXT:    fcvtl2 v1.4s, v1.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v3.4s, v3.8h
-; NONEON-NOSVE-NEXT:    fcvtl2 v2.4s, v2.8h
-; NONEON-NOSVE-NEXT:    fadd v4.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    fadd v5.4s, v7.4s, v6.4s
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fadd v2.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    fcvtn v1.4h, v4.4s
-; NONEON-NOSVE-NEXT:    fcvtn v3.4h, v5.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v1.8h, v0.4s
-; NONEON-NOSVE-NEXT:    fcvtn2 v3.8h, v2.4s
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %res = fadd <16 x half> %op1, %op2
@@ -451,14 +277,6 @@ define void @fadd_v2f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ldr d1, [x1]
-; NONEON-NOSVE-NEXT:    fadd v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x float>, ptr %a
   %op2 = load <2 x float>, ptr %b
   %res = fadd <2 x float> %op1, %op2
@@ -475,14 +293,6 @@ define void @fadd_v4f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x float>, ptr %a
   %op2 = load <4 x float>, ptr %b
   %res = fadd <4 x float> %op1, %op2
@@ -501,15 +311,6 @@ define void @fadd_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z1.s, p0/m, z1.s, z3.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fadd v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %res = fadd <8 x float> %op1, %op2
@@ -526,14 +327,6 @@ define void @fadd_v2f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    fadd v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <2 x double>, ptr %a
   %op2 = load <2 x double>, ptr %b
   %res = fadd <2 x double> %op1, %op2
@@ -552,15 +345,6 @@ define void @fadd_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fadd_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q3, [x1]
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    fadd v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fadd v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %res = fadd <4 x double> %op1, %op2
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll
index f595a4219cac..00413302798c 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -16,14 +15,6 @@ define void @test_revbv16i16(ptr %a) {
 ; CHECK-NEXT:    revb z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revbv16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev16 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rev16 v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <32 x i8>, ptr %a
   %tmp2 = shufflevector <32 x i8> %tmp1, <32 x i8> undef, <32 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 undef, i32 24, i32 27, i32 undef, i32 29, i32 28, i32 undef, i32 undef>
   store <32 x i8> %tmp2, ptr %a
@@ -40,14 +31,6 @@ define void @test_revbv8i32(ptr %a) {
 ; CHECK-NEXT:    revb z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revbv8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev32 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rev32 v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <32 x i8>, ptr %a
   %tmp2 = shufflevector <32 x i8> %tmp1, <32 x i8> undef, <32 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12, i32 19, i32 18, i32 17, i32 16, i32 23, i32 22, i32 21, i32 20, i32 27, i32 undef, i32 undef, i32 undef, i32 31, i32 30, i32 29, i32 undef>
   store <32 x i8> %tmp2, ptr %a
@@ -64,14 +47,6 @@ define void @test_revbv4i64(ptr %a) {
 ; CHECK-NEXT:    revb z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revbv4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev64 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rev64 v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <32 x i8>, ptr %a
   %tmp2 = shufflevector <32 x i8> %tmp1, <32 x i8> undef, <32 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 31, i32 30, i32 29, i32 undef, i32 27, i32 undef, i32 undef, i32 undef>
   store <32 x i8> %tmp2, ptr %a
@@ -88,14 +63,6 @@ define void @test_revhv8i32(ptr %a) {
 ; CHECK-NEXT:    revh z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revhv8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev32 v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    rev32 v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <16 x i16>, ptr %a
   %tmp2 = shufflevector <16 x i16> %tmp1, <16 x i16> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
   store <16 x i16> %tmp2, ptr %a
@@ -112,14 +79,6 @@ define void @test_revhv8f32(ptr %a) {
 ; CHECK-NEXT:    revh z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revhv8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev32 v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    rev32 v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <16 x half>, ptr %a
   %tmp2 = shufflevector <16 x half> %tmp1, <16 x half> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
   store <16 x half> %tmp2, ptr %a
@@ -136,14 +95,6 @@ define void @test_revhv4i64(ptr %a) {
 ; CHECK-NEXT:    revh z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revhv4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev64 v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    rev64 v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <16 x i16>, ptr %a
   %tmp2 = shufflevector <16 x i16> %tmp1, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
   store <16 x i16> %tmp2, ptr %a
@@ -160,14 +111,6 @@ define void @test_revwv4i64(ptr %a) {
 ; CHECK-NEXT:    revw z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revwv4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev64 v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    rev64 v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <8 x i32>, ptr %a
   %tmp2 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   store <8 x i32> %tmp2, ptr %a
@@ -184,14 +127,6 @@ define void @test_revwv4f64(ptr %a) {
 ; CHECK-NEXT:    revw z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revwv4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev64 v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    rev64 v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <8 x float>, ptr %a
   %tmp2 = shufflevector <8 x float> %tmp1, <8 x float> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   store <8 x float> %tmp2, ptr %a
@@ -206,12 +141,6 @@ define <16 x i8> @test_revv16i8(ptr %a) {
 ; CHECK-NEXT:    revb z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revv16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    rev64 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <16 x i8>, ptr %a
   %tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
   ret <16 x i8> %tmp2
@@ -227,14 +156,6 @@ define void @test_revwv8i32v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    revw z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revwv8i32v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    rev64 v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    rev64 v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <8 x i32>, ptr %a
   %tmp2 = load <8 x i32>, ptr %b
   %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> %tmp2, <8 x i32> <i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
@@ -255,18 +176,6 @@ define void @test_revhv32i16(ptr %a) {
 ; CHECK-NEXT:    stp q0, q1, [x0, #32]
 ; CHECK-NEXT:    stp q2, q3, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revhv32i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    rev64 v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    rev64 v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    rev64 v2.8h, v2.8h
-; NONEON-NOSVE-NEXT:    rev64 v3.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <32 x i16>, ptr %a
   %tmp2 = shufflevector <32 x i16> %tmp1, <32 x i16> undef, <32 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12, i32 19, i32 18, i32 17, i32 16, i32 23, i32 22, i32 21, i32 20, i32 27, i32 undef, i32 undef, i32 undef, i32 31, i32 30, i32 29, i32 undef>
   store <32 x i16> %tmp2, ptr %a
@@ -282,14 +191,6 @@ define void @test_rev_elts_fail(ptr %a) {
 ; CHECK-NEXT:    tbl z0.d, { z2.d }, z0.d
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_rev_elts_fail:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; NONEON-NOSVE-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <4 x i64>, ptr %a
   %tmp2 = shufflevector <4 x i64> %tmp1, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   store <4 x i64> %tmp2, ptr %a
@@ -307,15 +208,6 @@ define void @test_revdv4i64_sve2p1(ptr %a) #1 {
 ; CHECK-NEXT:    revd z1.q, p0/m, z1.q
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revdv4i64_sve2p1:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ptrue p0.d, vl2
-; NONEON-NOSVE-NEXT:    revd z0.q, p0/m, z0.q
-; NONEON-NOSVE-NEXT:    revd z1.q, p0/m, z1.q
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <4 x i64>, ptr %a
   %tmp2 = shufflevector <4 x i64> %tmp1, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   store <4 x i64> %tmp2, ptr %a
@@ -331,15 +223,6 @@ define void @test_revdv4f64_sve2p1(ptr %a) #1 {
 ; CHECK-NEXT:    revd z1.q, p0/m, z1.q
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revdv4f64_sve2p1:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ptrue p0.d
-; NONEON-NOSVE-NEXT:    revd z0.q, p0/m, z0.q
-; NONEON-NOSVE-NEXT:    revd z1.q, p0/m, z1.q
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <4 x double>, ptr %a
   %tmp2 = shufflevector <4 x double> %tmp1, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   store <4 x double> %tmp2, ptr %a
@@ -355,16 +238,6 @@ define void @test_revv8i32(ptr %a) {
 ; CHECK-NEXT:    tbl z0.s, { z2.s }, z0.s
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_revv8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    rev64 v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    rev64 v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; NONEON-NOSVE-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <8 x i32>, ptr %a
   %tmp2 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   store <8 x i32> %tmp2, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
index df786933da88..cb73030306b0 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -69,18 +68,6 @@ define void @zip1_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    str q1, [x0, #16]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip1_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    zip2 v2.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    zip1 v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    str q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load volatile <32 x i8>, ptr %a
   %tmp2 = load volatile <32 x i8>, ptr %b
   %tmp3 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47>
@@ -209,28 +196,6 @@ define void @zip_v32i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip_v32i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q4, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q5, q1, [x0]
-; NONEON-NOSVE-NEXT:    ldp q6, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    ldp q7, q3, [x1]
-; NONEON-NOSVE-NEXT:    zip1 v17.8h, v0.8h, v2.8h
-; NONEON-NOSVE-NEXT:    zip2 v0.8h, v0.8h, v2.8h
-; NONEON-NOSVE-NEXT:    zip1 v16.8h, v1.8h, v3.8h
-; NONEON-NOSVE-NEXT:    zip2 v1.8h, v1.8h, v3.8h
-; NONEON-NOSVE-NEXT:    zip1 v2.8h, v5.8h, v7.8h
-; NONEON-NOSVE-NEXT:    zip1 v3.8h, v4.8h, v6.8h
-; NONEON-NOSVE-NEXT:    zip2 v5.8h, v5.8h, v7.8h
-; NONEON-NOSVE-NEXT:    zip2 v4.8h, v4.8h, v6.8h
-; NONEON-NOSVE-NEXT:    add v6.8h, v16.8h, v17.8h
-; NONEON-NOSVE-NEXT:    add v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    add v2.8h, v5.8h, v4.8h
-; NONEON-NOSVE-NEXT:    stp q6, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    stp q1, q2, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <32 x i16>, ptr %a
   %tmp2 = load <32 x i16>, ptr %b
   %tmp3 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47>
@@ -279,18 +244,6 @@ define void @zip1_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    str q1, [x0, #16]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip1_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    zip2 v2.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    zip1 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    str q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load volatile <16 x i16>, ptr %a
   %tmp2 = load volatile <16 x i16>, ptr %b
   %tmp3 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
@@ -323,18 +276,6 @@ define void @zip1_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    str q1, [x0, #16]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip1_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    zip2 v2.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    zip1 v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    str q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load volatile <8 x i32>, ptr %a
   %tmp2 = load volatile <8 x i32>, ptr %b
   %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> %tmp2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
@@ -357,19 +298,6 @@ define void @zip_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
-; NONEON-NOSVE-NEXT:    zip1 v4.2d, v1.2d, v3.2d
-; NONEON-NOSVE-NEXT:    zip1 v5.2d, v0.2d, v2.2d
-; NONEON-NOSVE-NEXT:    zip2 v1.2d, v1.2d, v3.2d
-; NONEON-NOSVE-NEXT:    zip2 v0.2d, v0.2d, v2.2d
-; NONEON-NOSVE-NEXT:    fadd v2.2d, v4.2d, v5.2d
-; NONEON-NOSVE-NEXT:    fadd v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    stp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <4 x double>, ptr %a
   %tmp2 = load <4 x double>, ptr %b
   %tmp3 = shufflevector <4 x double> %tmp1, <4 x double> %tmp2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
@@ -402,16 +330,6 @@ define void @zip_v4i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    zip1 v2.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    zip2 v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    add v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <4 x i32>, ptr %a
   %tmp2 = load <4 x i32>, ptr %b
   %tmp3 = shufflevector <4 x i32> %tmp1, <4 x i32> %tmp2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
@@ -433,16 +351,6 @@ define void @zip1_v8i32_undef(ptr %a) {
 ; CHECK-NEXT:    str q1, [x0, #16]
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip1_v8i32_undef:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    zip2 v1.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    zip1 v0.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    str q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load  volatile <8 x i32>, ptr %a
   %tmp2 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3>
   store volatile <8 x i32> %tmp2, ptr %a
@@ -462,19 +370,6 @@ define void @trn_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z1.b, z1.b, z2.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trn_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    trn1 v4.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    trn2 v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    trn1 v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    trn2 v2.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    add v0.16b, v4.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <32 x i8>, ptr %a
   %tmp2 = load <32 x i8>, ptr %b
   %tmp3 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>
@@ -497,19 +392,6 @@ define void @trn_v8i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z0.h, z1.h, z0.h
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trn_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    adrp x8, .LCPI8_0
-; NONEON-NOSVE-NEXT:    adrp x9, .LCPI8_1
-; NONEON-NOSVE-NEXT:    ldr q1, [x0]
-; NONEON-NOSVE-NEXT:    ldr q0, [x8, :lo12:.LCPI8_0]
-; NONEON-NOSVE-NEXT:    ldr q2, [x9, :lo12:.LCPI8_1]
-; NONEON-NOSVE-NEXT:    tbl v0.16b, { v1.16b }, v0.16b
-; NONEON-NOSVE-NEXT:    tbl v1.16b, { v1.16b }, v2.16b
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <8 x i16>, ptr %a
   %tmp2 = load <8 x i16>, ptr %b
   %tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 0, i32 7, i32 2, i32 6, i32 4, i32 5, i32 1, i32 3>
@@ -532,19 +414,6 @@ define void @trn_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z1.h, z1.h, z2.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trn_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    trn1 v4.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    trn2 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    trn1 v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    trn2 v2.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    add v0.8h, v4.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v2.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <16 x i16>, ptr %a
   %tmp2 = load <16 x i16>, ptr %b
   %tmp3 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
@@ -567,19 +436,6 @@ define void @trn_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    add z1.s, z1.s, z2.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trn_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    zip1 v4.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    trn2 v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    trn1 v1.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    trn2 v2.4s, v2.4s, v3.4s
-; NONEON-NOSVE-NEXT:    add v0.4s, v4.4s, v0.4s
-; NONEON-NOSVE-NEXT:    add v1.4s, v1.4s, v2.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <8 x i32>, ptr %a
   %tmp2 = load <8 x i32>, ptr %b
   %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> %tmp2, <8 x i32> <i32 0, i32 8, i32 undef, i32 undef, i32 4, i32 12, i32 6, i32 14>
@@ -603,19 +459,6 @@ define void @trn_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z1.d, p0/m, z1.d, z2.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trn_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    zip1 v4.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    zip2 v0.2d, v0.2d, v1.2d
-; NONEON-NOSVE-NEXT:    zip1 v1.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    zip2 v2.2d, v2.2d, v3.2d
-; NONEON-NOSVE-NEXT:    fadd v0.2d, v4.2d, v0.2d
-; NONEON-NOSVE-NEXT:    fadd v1.2d, v1.2d, v2.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <4 x double>, ptr %a
   %tmp2 = load <4 x double>, ptr %b
   %tmp3 = shufflevector <4 x double> %tmp1, <4 x double> %tmp2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
@@ -636,16 +479,6 @@ define void @trn_v4f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z0.s, p0/m, z0.s, z2.s
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trn_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    trn1 v2.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    trn2 v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <4 x float>, ptr %a
   %tmp2 = load <4 x float>, ptr %b
   %tmp3 = shufflevector <4 x float> %tmp1, <4 x float> %tmp2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
@@ -667,18 +500,6 @@ define void @trn_v8i32_undef(ptr %a) {
 ; CHECK-NEXT:    add z1.s, z3.s, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trn_v8i32_undef:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    trn1 v2.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    trn2 v0.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    trn1 v3.4s, v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    trn2 v1.4s, v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    add v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    add v1.4s, v3.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <8 x i32>, ptr %a
   %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
   %tmp4 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
@@ -750,18 +571,6 @@ define void @zip2_v32i8(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    str q1, [x0, #16]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip2_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    zip2 v2.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    zip1 v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    str q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load volatile <32 x i8>, ptr %a
   %tmp2 = load volatile <32 x i8>, ptr %b
   %tmp3 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
@@ -808,18 +617,6 @@ define void @zip2_v16i16(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    str q1, [x0, #16]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip2_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    zip2 v2.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    zip1 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    str q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load volatile <16 x i16>, ptr %a
   %tmp2 = load volatile <16 x i16>, ptr %b
   %tmp3 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
@@ -852,18 +649,6 @@ define void @zip2_v8i32(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    str q1, [x0, #16]
 ; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip2_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    zip2 v2.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    zip1 v0.4s, v0.4s, v1.4s
-; NONEON-NOSVE-NEXT:    str q2, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load volatile <8 x i32>, ptr %a
   %tmp2 = load volatile <8 x i32>, ptr %b
   %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> %tmp2, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
@@ -883,16 +668,6 @@ define void @zip2_v8i32_undef(ptr %a) #0{
 ; CHECK-NEXT:    str q1, [x0, #16]
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip2_v8i32_undef:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    zip2 v1.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    zip1 v0.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    str q1, [x0, #16]
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load volatile <8 x i32>, ptr %a
   %tmp2 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
   store volatile <8 x i32> %tmp2, ptr %a
@@ -1094,19 +869,6 @@ define void @uzp_v32i8(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uzp_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
-; NONEON-NOSVE-NEXT:    uzp1 v4.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uzp2 v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uzp1 v1.16b, v3.16b, v2.16b
-; NONEON-NOSVE-NEXT:    uzp2 v2.16b, v3.16b, v2.16b
-; NONEON-NOSVE-NEXT:    add v0.16b, v4.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v2.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <32 x i8>, ptr %a
   %tmp2 = load <32 x i8>, ptr %b
   %tmp3 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>
@@ -1129,17 +891,6 @@ define void @uzp_v4i16(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    add z0.h, z1.h, z0.h
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uzp_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    ext v1.8b, v0.8b, v0.8b, #6
-; NONEON-NOSVE-NEXT:    ext v2.8b, v0.8b, v0.8b, #2
-; NONEON-NOSVE-NEXT:    trn1 v1.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    zip1 v0.4h, v2.4h, v0.4h
-; NONEON-NOSVE-NEXT:    add v0.4h, v1.4h, v0.4h
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <4 x i16>, ptr %a
   %tmp2 = load <4 x i16>, ptr %b
   %tmp3 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <4 x i32> <i32 0, i32 3, i32 2, i32 1>
@@ -1257,19 +1008,6 @@ define void @uzp_v16i16(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uzp_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
-; NONEON-NOSVE-NEXT:    uzp1 v4.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    uzp2 v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v3.8h, v2.8h
-; NONEON-NOSVE-NEXT:    uzp2 v2.8h, v3.8h, v2.8h
-; NONEON-NOSVE-NEXT:    add v0.8h, v4.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v2.8h
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <16 x i16>, ptr %a
   %tmp2 = load <16 x i16>, ptr %b
   %tmp3 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
@@ -1309,19 +1047,6 @@ define void @uzp_v8f32(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    stp q1, q0, [x0]
 ; CHECK-NEXT:    add sp, sp, #48
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uzp_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
-; NONEON-NOSVE-NEXT:    uzp1 v4.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp2 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp1 v1.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    uzp2 v2.4s, v3.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fadd v0.4s, v4.4s, v0.4s
-; NONEON-NOSVE-NEXT:    fadd v1.4s, v1.4s, v2.4s
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <8 x float>, ptr %a
   %tmp2 = load <8 x float>, ptr %b
   %tmp3 = shufflevector <8 x float> %tmp1, <8 x float> %tmp2, <8 x i32> <i32 0, i32 undef, i32 4, i32 6, i32 undef, i32 10, i32 12, i32 14>
@@ -1344,19 +1069,6 @@ define void @uzp_v4i64(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    add z1.d, z1.d, z2.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uzp_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
-; NONEON-NOSVE-NEXT:    zip1 v4.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    zip2 v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    zip1 v1.2d, v3.2d, v2.2d
-; NONEON-NOSVE-NEXT:    zip2 v2.2d, v3.2d, v2.2d
-; NONEON-NOSVE-NEXT:    add v0.2d, v4.2d, v0.2d
-; NONEON-NOSVE-NEXT:    add v1.2d, v1.2d, v2.2d
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <4 x i64>, ptr %a
   %tmp2 = load <4 x i64>, ptr %b
   %tmp3 = shufflevector <4 x i64> %tmp1, <4 x i64> %tmp2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -1424,16 +1136,6 @@ define void @uzp_v8i16(ptr %a, ptr %b) #0{
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uzp_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    uzp1 v2.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    uzp2 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    add v0.8h, v2.8h, v0.8h
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <8 x i16>, ptr %a
   %tmp2 = load <8 x i16>, ptr %b
   %tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -1472,15 +1174,6 @@ define void @uzp_v8i32_undef(ptr %a) #0{
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: uzp_v8i32_undef:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v2.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp2 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    add v0.4s, v2.4s, v0.4s
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <8 x i32>, ptr %a
   %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>
   %tmp4 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 1, i32 3, i32 5, i32 7>
@@ -1504,19 +1197,6 @@ define void @zip_vscale2_4(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fadd z0.d, p0/m, z0.d, z1.d
 ; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: zip_vscale2_4:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]
-; NONEON-NOSVE-NEXT:    zip1 v4.2d, v1.2d, v3.2d
-; NONEON-NOSVE-NEXT:    zip1 v5.2d, v0.2d, v2.2d
-; NONEON-NOSVE-NEXT:    zip2 v1.2d, v1.2d, v3.2d
-; NONEON-NOSVE-NEXT:    zip2 v0.2d, v0.2d, v2.2d
-; NONEON-NOSVE-NEXT:    fadd v2.2d, v4.2d, v5.2d
-; NONEON-NOSVE-NEXT:    fadd v0.2d, v1.2d, v0.2d
-; NONEON-NOSVE-NEXT:    stp q2, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %tmp1 = load <4 x double>, ptr %a
   %tmp2 = load <4 x double>, ptr %b
   %tmp3 = shufflevector <4 x double> %tmp1, <4 x double> %tmp2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
index 6b3c85f59357..ab7c42b3e9e3 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -36,23 +35,6 @@ define i1 @ptest_v16i1(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ptest_v16i1:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    fcmeq v0.4s, v0.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v1.4s, v1.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v3.4s, v3.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v2.4s, v2.4s, #0.0
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    mvn v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    umaxv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    and w0, w8, #0x1
-; NONEON-NOSVE-NEXT:    ret
   %v0 = bitcast ptr %a to ptr
   %v1 = load <16 x float>, ptr %v0, align 4
   %v2 = fcmp une <16 x float> %v1, zeroinitializer
@@ -110,33 +92,6 @@ define i1 @ptest_or_v16i1(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ptest_or_v16i1:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x1, #32]
-; NONEON-NOSVE-NEXT:    fcmeq v1.4s, v1.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v0.4s, v0.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v3.4s, v3.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v2.4s, v2.4s, #0.0
-; NONEON-NOSVE-NEXT:    ldp q6, q7, [x1]
-; NONEON-NOSVE-NEXT:    fcmeq v4.4s, v4.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v5.4s, v5.4s, #0.0
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    fcmeq v7.4s, v7.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v6.4s, v6.4s, #0.0
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    uzp1 v2.8h, v5.8h, v4.8h
-; NONEON-NOSVE-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uzp1 v1.16b, v3.16b, v2.16b
-; NONEON-NOSVE-NEXT:    mvn v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    orn v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    umaxv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    and w0, w8, #0x1
-; NONEON-NOSVE-NEXT:    ret
   %v0 = bitcast ptr %a to ptr
   %v1 = load <16 x float>, ptr %v0, align 4
   %v2 = fcmp une <16 x float> %v1, zeroinitializer
@@ -204,33 +159,6 @@ define i1 @ptest_and_v16i1(ptr %a, ptr %b) {
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: ptest_and_v16i1:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q2, q3, [x0]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x1, #32]
-; NONEON-NOSVE-NEXT:    fcmeq v1.4s, v1.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v0.4s, v0.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v3.4s, v3.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v2.4s, v2.4s, #0.0
-; NONEON-NOSVE-NEXT:    ldp q6, q7, [x1]
-; NONEON-NOSVE-NEXT:    fcmeq v4.4s, v4.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v5.4s, v5.4s, #0.0
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; NONEON-NOSVE-NEXT:    fcmeq v7.4s, v7.4s, #0.0
-; NONEON-NOSVE-NEXT:    fcmeq v6.4s, v6.4s, #0.0
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; NONEON-NOSVE-NEXT:    uzp1 v2.8h, v5.8h, v4.8h
-; NONEON-NOSVE-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uzp1 v1.16b, v3.16b, v2.16b
-; NONEON-NOSVE-NEXT:    mvn v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    bic v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    uminv b0, v0.16b
-; NONEON-NOSVE-NEXT:    fmov w8, s0
-; NONEON-NOSVE-NEXT:    and w0, w8, #0x1
-; NONEON-NOSVE-NEXT:    ret
   %v0 = bitcast ptr %a to ptr
   %v1 = load <16 x float>, ptr %v0, align 4
   %v2 = fcmp une <16 x float> %v1, zeroinitializer
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll
index 0a7352bf4944..bfa931044bc5 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -19,13 +18,6 @@ define <4 x i8> @bitreverse_v4i8(<4 x i8> %op) {
 ; CHECK-NEXT:    lsr z0.h, z0.h, #8
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev16 v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    rbit v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ushr v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i8> @llvm.bitreverse.v4i8(<4 x i8> %op)
   ret <4 x i8> %res
 }
@@ -38,11 +30,6 @@ define <8 x i8> @bitreverse_v8i8(<8 x i8> %op) {
 ; CHECK-NEXT:    rbit z0.b, p0/m, z0.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rbit v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i8> @llvm.bitreverse.v8i8(<8 x i8> %op)
   ret <8 x i8> %res
 }
@@ -55,11 +42,6 @@ define <16 x i8> @bitreverse_v16i8(<16 x i8> %op) {
 ; CHECK-NEXT:    rbit z0.b, p0/m, z0.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rbit v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %op)
   ret <16 x i8> %res
 }
@@ -73,14 +55,6 @@ define void @bitreverse_v32i8(ptr %a) {
 ; CHECK-NEXT:    rbit z1.b, p0/m, z1.b
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rbit v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rbit v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <32 x i8>, ptr %a
   %res = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %op)
   store <32 x i8> %res, ptr %a
@@ -96,13 +70,6 @@ define <2 x i16> @bitreverse_v2i16(<2 x i16> %op) {
 ; CHECK-NEXT:    lsr z0.s, z0.s, #16
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev32 v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    rbit v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ushr v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %op)
   ret <2 x i16> %res
 }
@@ -115,12 +82,6 @@ define <4 x i16> @bitreverse_v4i16(<4 x i16> %op) {
 ; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev16 v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    rbit v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i16> @llvm.bitreverse.v4i16(<4 x i16> %op)
   ret <4 x i16> %res
 }
@@ -133,12 +94,6 @@ define <8 x i16> @bitreverse_v8i16(<8 x i16> %op) {
 ; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev16 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rbit v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %op)
   ret <8 x i16> %res
 }
@@ -152,16 +107,6 @@ define void @bitreverse_v16i16(ptr %a) {
 ; CHECK-NEXT:    rbit z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev16 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rev16 v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    rbit v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rbit v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %op)
   store <16 x i16> %res, ptr %a
@@ -176,12 +121,6 @@ define <2 x i32> @bitreverse_v2i32(<2 x i32> %op) {
 ; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev32 v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    rbit v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i32> @llvm.bitreverse.v2i32(<2 x i32> %op)
   ret <2 x i32> %res
 }
@@ -194,12 +133,6 @@ define <4 x i32> @bitreverse_v4i32(<4 x i32> %op) {
 ; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev32 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rbit v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %op)
   ret <4 x i32> %res
 }
@@ -213,16 +146,6 @@ define void @bitreverse_v8i32(ptr %a) {
 ; CHECK-NEXT:    rbit z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev32 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rev32 v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    rbit v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rbit v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %op)
   store <8 x i32> %res, ptr %a
@@ -237,12 +160,6 @@ define <1 x i64> @bitreverse_v1i64(<1 x i64> %op) {
 ; CHECK-NEXT:    rbit z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev64 v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    rbit v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x i64> @llvm.bitreverse.v1i64(<1 x i64> %op)
   ret <1 x i64> %res
 }
@@ -255,12 +172,6 @@ define <2 x i64> @bitreverse_v2i64(<2 x i64> %op) {
 ; CHECK-NEXT:    rbit z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev64 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rbit v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %op)
   ret <2 x i64> %res
 }
@@ -274,16 +185,6 @@ define void @bitreverse_v4i64(ptr %a) {
 ; CHECK-NEXT:    rbit z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bitreverse_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev64 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rev64 v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    rbit v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rbit v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %op)
   store <4 x i64> %res, ptr %a
@@ -303,12 +204,6 @@ define <2 x i16> @bswap_v2i16(<2 x i16> %op) {
 ; CHECK-NEXT:    lsr z0.s, z0.s, #16
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bswap_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev32 v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ushr v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i16> @llvm.bswap.v2i16(<2 x i16> %op)
   ret <2 x i16> %res
 }
@@ -321,11 +216,6 @@ define <4 x i16> @bswap_v4i16(<4 x i16> %op) {
 ; CHECK-NEXT:    revb z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bswap_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev16 v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i16> @llvm.bswap.v4i16(<4 x i16> %op)
   ret <4 x i16> %res
 }
@@ -338,11 +228,6 @@ define <8 x i16> @bswap_v8i16(<8 x i16> %op) {
 ; CHECK-NEXT:    revb z0.h, p0/m, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bswap_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev16 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %op)
   ret <8 x i16> %res
 }
@@ -356,14 +241,6 @@ define void @bswap_v16i16(ptr %a) {
 ; CHECK-NEXT:    revb z1.h, p0/m, z1.h
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bswap_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev16 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rev16 v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <16 x i16>, ptr %a
   %res = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> %op)
   store <16 x i16> %res, ptr %a
@@ -378,11 +255,6 @@ define <2 x i32> @bswap_v2i32(<2 x i32> %op) {
 ; CHECK-NEXT:    revb z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bswap_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev32 v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i32> @llvm.bswap.v2i32(<2 x i32> %op)
   ret <2 x i32> %res
 }
@@ -395,11 +267,6 @@ define <4 x i32> @bswap_v4i32(<4 x i32> %op) {
 ; CHECK-NEXT:    revb z0.s, p0/m, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bswap_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev32 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %op)
   ret <4 x i32> %res
 }
@@ -413,14 +280,6 @@ define void @bswap_v8i32(ptr %a) {
 ; CHECK-NEXT:    revb z1.s, p0/m, z1.s
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bswap_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev32 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rev32 v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <8 x i32>, ptr %a
   %res = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> %op)
   store <8 x i32> %res, ptr %a
@@ -435,11 +294,6 @@ define <1 x i64> @bswap_v1i64(<1 x i64> %op) {
 ; CHECK-NEXT:    revb z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bswap_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev64 v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <1 x i64> @llvm.bswap.v1i64(<1 x i64> %op)
   ret <1 x i64> %res
 }
@@ -452,11 +306,6 @@ define <2 x i64> @bswap_v2i64(<2 x i64> %op) {
 ; CHECK-NEXT:    revb z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bswap_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev64 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %res = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %op)
   ret <2 x i64> %res
 }
@@ -470,14 +319,6 @@ define void @bswap_v4i64(ptr %a) {
 ; CHECK-NEXT:    revb z1.d, p0/m, z1.d
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: bswap_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    rev64 v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    rev64 v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op = load <4 x i64>, ptr %a
   %res = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> %op)
   store <4 x i64> %res, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll
index d86c7d36a104..9dd42e7831e0 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -15,19 +14,6 @@ define <4 x i8> @sdiv_v4i8(<4 x i8> %op1) {
 ; CHECK-NEXT:    asrd z0.h, p0/m, z0.h, #5
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v1.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    movi d2, #0xff00ff00ff00ff
-; NONEON-NOSVE-NEXT:    sshr v1.4h, v1.4h, #8
-; NONEON-NOSVE-NEXT:    sshr v1.4h, v1.4h, #7
-; NONEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    usra v0.4h, v1.4h, #3
-; NONEON-NOSVE-NEXT:    shl v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    sshr v0.4h, v0.4h, #8
-; NONEON-NOSVE-NEXT:    sshr v0.4h, v0.4h, #5
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <4 x i8> %op1, shufflevector (<4 x i8> insertelement (<4 x i8> poison, i8 32, i32 0), <4 x i8> poison, <4 x i32> zeroinitializer)
   ret <4 x i8> %res
 }
@@ -40,13 +26,6 @@ define <8 x i8> @sdiv_v8i8(<8 x i8> %op1) {
 ; CHECK-NEXT:    asrd z0.b, p0/m, z0.b, #5
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmlt v1.8b, v0.8b, #0
-; NONEON-NOSVE-NEXT:    usra v0.8b, v1.8b, #3
-; NONEON-NOSVE-NEXT:    sshr v0.8b, v0.8b, #5
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <8 x i8> %op1, shufflevector (<8 x i8> insertelement (<8 x i8> poison, i8 32, i32 0), <8 x i8> poison, <8 x i32> zeroinitializer)
   ret <8 x i8> %res
 }
@@ -59,13 +38,6 @@ define <16 x i8> @sdiv_v16i8(<16 x i8> %op1) {
 ; CHECK-NEXT:    asrd z0.b, p0/m, z0.b, #5
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmlt v1.16b, v0.16b, #0
-; NONEON-NOSVE-NEXT:    usra v0.16b, v1.16b, #3
-; NONEON-NOSVE-NEXT:    sshr v0.16b, v0.16b, #5
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <16 x i8> %op1, shufflevector (<16 x i8> insertelement (<16 x i8> poison, i8 32, i32 0), <16 x i8> poison, <16 x i32> zeroinitializer)
   ret <16 x i8> %res
 }
@@ -79,18 +51,6 @@ define void @sdiv_v32i8(ptr %a) {
 ; CHECK-NEXT:    asrd z1.b, p0/m, z1.b, #5
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cmlt v2.16b, v0.16b, #0
-; NONEON-NOSVE-NEXT:    cmlt v3.16b, v1.16b, #0
-; NONEON-NOSVE-NEXT:    usra v0.16b, v2.16b, #3
-; NONEON-NOSVE-NEXT:    usra v1.16b, v3.16b, #3
-; NONEON-NOSVE-NEXT:    sshr v0.16b, v0.16b, #5
-; NONEON-NOSVE-NEXT:    sshr v1.16b, v1.16b, #5
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %res = sdiv <32 x i8> %op1, shufflevector (<32 x i8> insertelement (<32 x i8> poison, i8 32, i32 0), <32 x i8> poison, <32 x i32> zeroinitializer)
   store <32 x i8> %res, ptr %a
@@ -106,20 +66,6 @@ define <2 x i16> @sdiv_v2i16(<2 x i16> %op1) {
 ; CHECK-NEXT:    asrd z0.s, p0/m, z0.s, #5
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    shl v1.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    mov w8, #31 // =0x1f
-; NONEON-NOSVE-NEXT:    dup v2.2s, w8
-; NONEON-NOSVE-NEXT:    sshr v1.2s, v1.2s, #16
-; NONEON-NOSVE-NEXT:    ushr v1.2s, v1.2s, #26
-; NONEON-NOSVE-NEXT:    and v1.8b, v1.8b, v2.8b
-; NONEON-NOSVE-NEXT:    add v0.2s, v0.2s, v1.2s
-; NONEON-NOSVE-NEXT:    shl v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    sshr v0.2s, v0.2s, #16
-; NONEON-NOSVE-NEXT:    sshr v0.2s, v0.2s, #5
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <2 x i16> %op1, shufflevector (<2 x i16> insertelement (<2 x i16> poison, i16 32, i32 0), <2 x i16> poison, <2 x i32> zeroinitializer)
   ret <2 x i16> %res
 }
@@ -132,13 +78,6 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %op1) {
 ; CHECK-NEXT:    asrd z0.h, p0/m, z0.h, #5
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmlt v1.4h, v0.4h, #0
-; NONEON-NOSVE-NEXT:    usra v0.4h, v1.4h, #11
-; NONEON-NOSVE-NEXT:    sshr v0.4h, v0.4h, #5
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <4 x i16> %op1, shufflevector (<4 x i16> insertelement (<4 x i16> poison, i16 32, i32 0), <4 x i16> poison, <4 x i32> zeroinitializer)
   ret <4 x i16> %res
 }
@@ -151,13 +90,6 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1) {
 ; CHECK-NEXT:    asrd z0.h, p0/m, z0.h, #5
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmlt v1.8h, v0.8h, #0
-; NONEON-NOSVE-NEXT:    usra v0.8h, v1.8h, #11
-; NONEON-NOSVE-NEXT:    sshr v0.8h, v0.8h, #5
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <8 x i16> %op1, shufflevector (<8 x i16> insertelement (<8 x i16> poison, i16 32, i32 0), <8 x i16> poison, <8 x i32> zeroinitializer)
   ret <8 x i16> %res
 }
@@ -171,18 +103,6 @@ define void @sdiv_v16i16(ptr %a) {
 ; CHECK-NEXT:    asrd z1.h, p0/m, z1.h, #5
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cmlt v2.8h, v0.8h, #0
-; NONEON-NOSVE-NEXT:    cmlt v3.8h, v1.8h, #0
-; NONEON-NOSVE-NEXT:    usra v0.8h, v2.8h, #11
-; NONEON-NOSVE-NEXT:    usra v1.8h, v3.8h, #11
-; NONEON-NOSVE-NEXT:    sshr v0.8h, v0.8h, #5
-; NONEON-NOSVE-NEXT:    sshr v1.8h, v1.8h, #5
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %res = sdiv <16 x i16> %op1, shufflevector (<16 x i16> insertelement (<16 x i16> poison, i16 32, i32 0), <16 x i16> poison, <16 x i32> zeroinitializer)
   store <16 x i16> %res, ptr %a
@@ -197,13 +117,6 @@ define <2 x i32> @sdiv_v2i32(<2 x i32> %op1) {
 ; CHECK-NEXT:    asrd z0.s, p0/m, z0.s, #5
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmlt v1.2s, v0.2s, #0
-; NONEON-NOSVE-NEXT:    usra v0.2s, v1.2s, #27
-; NONEON-NOSVE-NEXT:    sshr v0.2s, v0.2s, #5
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <2 x i32> %op1, shufflevector (<2 x i32> insertelement (<2 x i32> poison, i32 32, i32 0), <2 x i32> poison, <2 x i32> zeroinitializer)
   ret <2 x i32> %res
 }
@@ -216,13 +129,6 @@ define <4 x i32> @sdiv_v4i32(<4 x i32> %op1) {
 ; CHECK-NEXT:    asrd z0.s, p0/m, z0.s, #5
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmlt v1.4s, v0.4s, #0
-; NONEON-NOSVE-NEXT:    usra v0.4s, v1.4s, #27
-; NONEON-NOSVE-NEXT:    sshr v0.4s, v0.4s, #5
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <4 x i32> %op1, shufflevector (<4 x i32> insertelement (<4 x i32> poison, i32 32, i32 0), <4 x i32> poison, <4 x i32> zeroinitializer)
   ret <4 x i32> %res
 }
@@ -236,18 +142,6 @@ define void @sdiv_v8i32(ptr %a) {
 ; CHECK-NEXT:    asrd z1.s, p0/m, z1.s, #5
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cmlt v2.4s, v0.4s, #0
-; NONEON-NOSVE-NEXT:    cmlt v3.4s, v1.4s, #0
-; NONEON-NOSVE-NEXT:    usra v0.4s, v2.4s, #27
-; NONEON-NOSVE-NEXT:    usra v1.4s, v3.4s, #27
-; NONEON-NOSVE-NEXT:    sshr v0.4s, v0.4s, #5
-; NONEON-NOSVE-NEXT:    sshr v1.4s, v1.4s, #5
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %res = sdiv <8 x i32> %op1, shufflevector (<8 x i32> insertelement (<8 x i32> poison, i32 32, i32 0), <8 x i32> poison, <8 x i32> zeroinitializer)
   store <8 x i32> %res, ptr %a
@@ -262,13 +156,6 @@ define <1 x i64> @sdiv_v1i64(<1 x i64> %op1) {
 ; CHECK-NEXT:    asrd z0.d, p0/m, z0.d, #5
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmlt d1, d0, #0
-; NONEON-NOSVE-NEXT:    usra d0, d1, #59
-; NONEON-NOSVE-NEXT:    sshr d0, d0, #5
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <1 x i64> %op1, shufflevector (<1 x i64> insertelement (<1 x i64> poison, i64 32, i32 0), <1 x i64> poison, <1 x i32> zeroinitializer)
   ret <1 x i64> %res
 }
@@ -282,13 +169,6 @@ define <2 x i64> @sdiv_v2i64(<2 x i64> %op1) {
 ; CHECK-NEXT:    asrd z0.d, p0/m, z0.d, #5
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    cmlt v1.2d, v0.2d, #0
-; NONEON-NOSVE-NEXT:    usra v0.2d, v1.2d, #59
-; NONEON-NOSVE-NEXT:    sshr v0.2d, v0.2d, #5
-; NONEON-NOSVE-NEXT:    ret
   %res = sdiv <2 x i64> %op1, shufflevector (<2 x i64> insertelement (<2 x i64> poison, i64 32, i32 0), <2 x i64> poison, <2 x i32> zeroinitializer)
   ret <2 x i64> %res
 }
@@ -302,18 +182,6 @@ define void @sdiv_v4i64(ptr %a) {
 ; CHECK-NEXT:    asrd z1.d, p0/m, z1.d, #5
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: sdiv_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    cmlt v2.2d, v0.2d, #0
-; NONEON-NOSVE-NEXT:    cmlt v3.2d, v1.2d, #0
-; NONEON-NOSVE-NEXT:    usra v0.2d, v2.2d, #59
-; NONEON-NOSVE-NEXT:    usra v1.2d, v3.2d, #59
-; NONEON-NOSVE-NEXT:    sshr v0.2d, v0.2d, #5
-; NONEON-NOSVE-NEXT:    sshr v1.2d, v1.2d, #5
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %res = sdiv <4 x i64> %op1, shufflevector (<4 x i64> insertelement (<4 x i64> poison, i64 32, i32 0), <4 x i64> poison, <4 x i32> zeroinitializer)
   store <4 x i64> %res, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-splat-vector.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-splat-vector.ll
index 6489e8d94d31..323d5278592f 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-splat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-splat-vector.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 
@@ -16,11 +15,6 @@ define <4 x i8> @splat_v4i8(i8 %a) {
 ; CHECK-NEXT:    mov z0.h, w0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.4h, w0
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <4 x i8> undef, i8 %a, i64 0
   %splat = shufflevector <4 x i8> %insert, <4 x i8> undef, <4 x i32> zeroinitializer
   ret <4 x i8> %splat
@@ -32,11 +26,6 @@ define <8 x i8> @splat_v8i8(i8 %a) {
 ; CHECK-NEXT:    mov z0.b, w0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.8b, w0
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <8 x i8> undef, i8 %a, i64 0
   %splat = shufflevector <8 x i8> %insert, <8 x i8> undef, <8 x i32> zeroinitializer
   ret <8 x i8> %splat
@@ -48,11 +37,6 @@ define <16 x i8> @splat_v16i8(i8 %a) {
 ; CHECK-NEXT:    mov z0.b, w0
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.16b, w0
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <16 x i8> undef, i8 %a, i64 0
   %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer
   ret <16 x i8> %splat
@@ -64,12 +48,6 @@ define void @splat_v32i8(i8 %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.b, w0
 ; CHECK-NEXT:    stp q0, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.16b, w0
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <32 x i8> undef, i8 %a, i64 0
   %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer
   store <32 x i8> %splat, ptr %b
@@ -82,11 +60,6 @@ define <2 x i16> @splat_v2i16(i16 %a) {
 ; CHECK-NEXT:    mov z0.s, w0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.2s, w0
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <2 x i16> undef, i16 %a, i64 0
   %splat = shufflevector <2 x i16> %insert, <2 x i16> undef, <2 x i32> zeroinitializer
   ret <2 x i16> %splat
@@ -98,11 +71,6 @@ define <4 x i16> @splat_v4i16(i16 %a) {
 ; CHECK-NEXT:    mov z0.h, w0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.4h, w0
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <4 x i16> undef, i16 %a, i64 0
   %splat = shufflevector <4 x i16> %insert, <4 x i16> undef, <4 x i32> zeroinitializer
   ret <4 x i16> %splat
@@ -114,11 +82,6 @@ define <8 x i16> @splat_v8i16(i16 %a) {
 ; CHECK-NEXT:    mov z0.h, w0
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.8h, w0
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <8 x i16> undef, i16 %a, i64 0
   %splat = shufflevector <8 x i16> %insert, <8 x i16> undef, <8 x i32> zeroinitializer
   ret <8 x i16> %splat
@@ -130,12 +93,6 @@ define void @splat_v16i16(i16 %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.h, w0
 ; CHECK-NEXT:    stp q0, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.8h, w0
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <16 x i16> undef, i16 %a, i64 0
   %splat = shufflevector <16 x i16> %insert, <16 x i16> undef, <16 x i32> zeroinitializer
   store <16 x i16> %splat, ptr %b
@@ -148,11 +105,6 @@ define <2 x i32> @splat_v2i32(i32 %a) {
 ; CHECK-NEXT:    mov z0.s, w0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.2s, w0
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <2 x i32> undef, i32 %a, i64 0
   %splat = shufflevector <2 x i32> %insert, <2 x i32> undef, <2 x i32> zeroinitializer
   ret <2 x i32> %splat
@@ -164,11 +116,6 @@ define <4 x i32> @splat_v4i32(i32 %a) {
 ; CHECK-NEXT:    mov z0.s, w0
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.4s, w0
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <4 x i32> undef, i32 %a, i64 0
   %splat = shufflevector <4 x i32> %insert, <4 x i32> undef, <4 x i32> zeroinitializer
   ret <4 x i32> %splat
@@ -180,12 +127,6 @@ define void @splat_v8i32(i32 %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.s, w0
 ; CHECK-NEXT:    stp q0, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.4s, w0
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <8 x i32> undef, i32 %a, i64 0
   %splat = shufflevector <8 x i32> %insert, <8 x i32> undef, <8 x i32> zeroinitializer
   store <8 x i32> %splat, ptr %b
@@ -198,11 +139,6 @@ define <1 x i64> @splat_v1i64(i64 %a) {
 ; CHECK-NEXT:    mov z0.d, x0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov d0, x0
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <1 x i64> undef, i64 %a, i64 0
   %splat = shufflevector <1 x i64> %insert, <1 x i64> undef, <1 x i32> zeroinitializer
   ret <1 x i64> %splat
@@ -214,11 +150,6 @@ define <2 x i64> @splat_v2i64(i64 %a) {
 ; CHECK-NEXT:    mov z0.d, x0
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.2d, x0
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <2 x i64> undef, i64 %a, i64 0
   %splat = shufflevector <2 x i64> %insert, <2 x i64> undef, <2 x i32> zeroinitializer
   ret <2 x i64> %splat
@@ -230,12 +161,6 @@ define void @splat_v4i64(i64 %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.d, x0
 ; CHECK-NEXT:    stp q0, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    dup v0.2d, x0
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <4 x i64> undef, i64 %a, i64 0
   %splat = shufflevector <4 x i64> %insert, <4 x i64> undef, <4 x i32> zeroinitializer
   store <4 x i64> %splat, ptr %b
@@ -253,12 +178,6 @@ define <2 x half> @splat_v2f16(half %a) {
 ; CHECK-NEXT:    mov z0.h, h0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $h0 killed $h0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.4h, v0.h[0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <2 x half> undef, half %a, i64 0
   %splat = shufflevector <2 x half> %insert, <2 x half> undef, <2 x i32> zeroinitializer
   ret <2 x half> %splat
@@ -271,12 +190,6 @@ define <4 x half> @splat_v4f16(half %a) {
 ; CHECK-NEXT:    mov z0.h, h0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $h0 killed $h0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.4h, v0.h[0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <4 x half> undef, half %a, i64 0
   %splat = shufflevector <4 x half> %insert, <4 x half> undef, <4 x i32> zeroinitializer
   ret <4 x half> %splat
@@ -289,12 +202,6 @@ define <8 x half> @splat_v8f16(half %a) {
 ; CHECK-NEXT:    mov z0.h, h0
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $h0 killed $h0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.8h, v0.h[0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <8 x half> undef, half %a, i64 0
   %splat = shufflevector <8 x half> %insert, <8 x half> undef, <8 x i32> zeroinitializer
   ret <8 x half> %splat
@@ -307,13 +214,6 @@ define void @splat_v16f16(half %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.h, h0
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $h0 killed $h0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.8h, v0.h[0]
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <16 x half> undef, half %a, i64 0
   %splat = shufflevector <16 x half> %insert, <16 x half> undef, <16 x i32> zeroinitializer
   store <16 x half> %splat, ptr %b
@@ -327,12 +227,6 @@ define <2 x float> @splat_v2f32(float %a, <2 x float> %op2) {
 ; CHECK-NEXT:    mov z0.s, s0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $s0 killed $s0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.2s, v0.s[0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <2 x float> undef, float %a, i64 0
   %splat = shufflevector <2 x float> %insert, <2 x float> undef, <2 x i32> zeroinitializer
   ret <2 x float> %splat
@@ -345,12 +239,6 @@ define <4 x float> @splat_v4f32(float %a, <4 x float> %op2) {
 ; CHECK-NEXT:    mov z0.s, s0
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $s0 killed $s0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.4s, v0.s[0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <4 x float> undef, float %a, i64 0
   %splat = shufflevector <4 x float> %insert, <4 x float> undef, <4 x i32> zeroinitializer
   ret <4 x float> %splat
@@ -363,13 +251,6 @@ define void @splat_v8f32(float %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.s, s0
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $s0 killed $s0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.4s, v0.s[0]
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <8 x float> undef, float %a, i64 0
   %splat = shufflevector <8 x float> %insert, <8 x float> undef, <8 x i32> zeroinitializer
   store <8 x float> %splat, ptr %b
@@ -380,10 +261,6 @@ define <1 x double> @splat_v1f64(double %a, <1 x double> %op2) {
 ; CHECK-LABEL: splat_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <1 x double> undef, double %a, i64 0
   %splat = shufflevector <1 x double> %insert, <1 x double> undef, <1 x i32> zeroinitializer
   ret <1 x double> %splat
@@ -396,12 +273,6 @@ define <2 x double> @splat_v2f64(double %a, <2 x double> %op2) {
 ; CHECK-NEXT:    mov z0.d, d0
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.2d, v0.d[0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <2 x double> undef, double %a, i64 0
   %splat = shufflevector <2 x double> %insert, <2 x double> undef, <2 x i32> zeroinitializer
   ret <2 x double> %splat
@@ -414,13 +285,6 @@ define void @splat_v4f64(double %a, ptr %b) {
 ; CHECK-NEXT:    mov z0.d, d0
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    // kill: def $d0 killed $d0 def $q0
-; NONEON-NOSVE-NEXT:    dup v0.2d, v0.d[0]
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <4 x double> undef, double %a, i64 0
   %splat = shufflevector <4 x double> %insert, <4 x double> undef, <4 x i32> zeroinitializer
   store <4 x double> %splat, ptr %b
@@ -437,12 +301,6 @@ define void @splat_imm_v32i8(ptr %a) {
 ; CHECK-NEXT:    mov z0.b, #1 // =0x1
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_imm_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.16b, #1
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <32 x i8> undef, i8 1, i64 0
   %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer
   store <32 x i8> %splat, ptr %a
@@ -455,13 +313,6 @@ define void @splat_imm_v16i16(ptr %a) {
 ; CHECK-NEXT:    mov z0.h, #2 // =0x2
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_imm_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #2 // =0x2
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <16 x i16> undef, i16 2, i64 0
   %splat = shufflevector <16 x i16> %insert, <16 x i16> undef, <16 x i32> zeroinitializer
   store <16 x i16> %splat, ptr %a
@@ -474,13 +325,6 @@ define void @splat_imm_v8i32(ptr %a) {
 ; CHECK-NEXT:    mov z0.s, #3 // =0x3
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_imm_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #3 // =0x3
-; NONEON-NOSVE-NEXT:    dup v0.4s, w8
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <8 x i32> undef, i32 3, i64 0
   %splat = shufflevector <8 x i32> %insert, <8 x i32> undef, <8 x i32> zeroinitializer
   store <8 x i32> %splat, ptr %a
@@ -493,13 +337,6 @@ define void @splat_imm_v4i64(ptr %a) {
 ; CHECK-NEXT:    mov z0.d, #4 // =0x4
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_imm_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #4 // =0x4
-; NONEON-NOSVE-NEXT:    dup v0.2d, x8
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <4 x i64> undef, i64 4, i64 0
   %splat = shufflevector <4 x i64> %insert, <4 x i64> undef, <4 x i32> zeroinitializer
   store <4 x i64> %splat, ptr %a
@@ -516,13 +353,6 @@ define void @splat_imm_v16f16(ptr %a) {
 ; CHECK-NEXT:    fmov z0.h, #5.00000000
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_imm_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov w8, #17664 // =0x4500
-; NONEON-NOSVE-NEXT:    dup v0.8h, w8
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <16 x half> undef, half 5.0, i64 0
   %splat = shufflevector <16 x half> %insert, <16 x half> undef, <16 x i32> zeroinitializer
   store <16 x half> %splat, ptr %a
@@ -535,12 +365,6 @@ define void @splat_imm_v8f32(ptr %a) {
 ; CHECK-NEXT:    fmov z0.s, #6.00000000
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_imm_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov v0.4s, #6.00000000
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <8 x float> undef, float 6.0, i64 0
   %splat = shufflevector <8 x float> %insert, <8 x float> undef, <8 x i32> zeroinitializer
   store <8 x float> %splat, ptr %a
@@ -553,12 +377,6 @@ define void @splat_imm_v4f64(ptr %a) {
 ; CHECK-NEXT:    fmov z0.d, #7.00000000
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: splat_imm_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov v0.2d, #7.00000000
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %insert = insertelement <4 x double> undef, double 7.0, i64 0
   %splat = shufflevector <4 x double> %insert, <4 x double> undef, <4 x i32> zeroinitializer
   store <4 x double> %splat, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll
index 41449aa90ba0..06709ca3685c 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -13,11 +12,6 @@ define void @store_v4i8(ptr %a) {
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    st1b { z0.h }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str wzr, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x i8> zeroinitializer, ptr %a
   ret void
 }
@@ -28,12 +22,6 @@ define void @store_v8i8(ptr %a) {
 ; CHECK-NEXT:    mov z0.b, #0 // =0x0
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <8 x i8> zeroinitializer, ptr %a
   ret void
 }
@@ -44,12 +32,6 @@ define void @store_v16i8(ptr %a) {
 ; CHECK-NEXT:    mov z0.b, #0 // =0x0
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <16 x i8> zeroinitializer, ptr %a
   ret void
 }
@@ -60,12 +42,6 @@ define void @store_v32i8(ptr %a) {
 ; CHECK-NEXT:    mov z0.b, #0 // =0x0
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <32 x i8> zeroinitializer, ptr %a
   ret void
 }
@@ -77,11 +53,6 @@ define void @store_v2i16(ptr %a) {
 ; CHECK-NEXT:    ptrue p0.s, vl2
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str wzr, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <2 x i16> zeroinitializer, ptr %a
   ret void
 }
@@ -93,11 +64,6 @@ define void @store_v2f16(ptr %a) {
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    str w8, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v2f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str wzr, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <2 x half> zeroinitializer, ptr %a
   ret void
 }
@@ -108,12 +74,6 @@ define void @store_v4i16(ptr %a) {
 ; CHECK-NEXT:    mov z0.h, #0 // =0x0
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x i16> zeroinitializer, ptr %a
   ret void
 }
@@ -124,12 +84,6 @@ define void @store_v4f16(ptr %a) {
 ; CHECK-NEXT:    mov z0.h, #0 // =0x0
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d0, #0000000000000000
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x half> zeroinitializer, ptr %a
   ret void
 }
@@ -140,12 +94,6 @@ define void @store_v8i16(ptr %a) {
 ; CHECK-NEXT:    mov z0.h, #0 // =0x0
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <8 x i16> zeroinitializer, ptr %a
   ret void
 }
@@ -156,12 +104,6 @@ define void @store_v8f16(ptr %a) {
 ; CHECK-NEXT:    mov z0.h, #0 // =0x0
 ; CHECK-NEXT:    str q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    str q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <8 x half> zeroinitializer, ptr %a
   ret void
 }
@@ -172,12 +114,6 @@ define void @store_v16i16(ptr %a) {
 ; CHECK-NEXT:    mov z0.h, #0 // =0x0
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <16 x i16> zeroinitializer, ptr %a
   ret void
 }
@@ -188,12 +124,6 @@ define void @store_v16f16(ptr %a) {
 ; CHECK-NEXT:    mov z0.h, #0 // =0x0
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <16 x half> zeroinitializer, ptr %a
   ret void
 }
@@ -203,11 +133,6 @@ define void @store_v2i32(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str xzr, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str xzr, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <2 x i32> zeroinitializer, ptr %a
   ret void
 }
@@ -217,11 +142,6 @@ define void @store_v2f32(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str xzr, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    str xzr, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <2 x float> zeroinitializer, ptr %a
   ret void
 }
@@ -231,11 +151,6 @@ define void @store_v4i32(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp xzr, xzr, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    stp xzr, xzr, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x i32> zeroinitializer, ptr %a
   ret void
 }
@@ -245,11 +160,6 @@ define void @store_v4f32(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp xzr, xzr, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    stp xzr, xzr, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x float> zeroinitializer, ptr %a
   ret void
 }
@@ -260,12 +170,6 @@ define void @store_v8i32(ptr %a) {
 ; CHECK-NEXT:    mov z0.s, #0 // =0x0
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <8 x i32> zeroinitializer, ptr %a
   ret void
 }
@@ -276,12 +180,6 @@ define void @store_v8f32(ptr %a) {
 ; CHECK-NEXT:    mov z0.s, #0 // =0x0
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <8 x float> zeroinitializer, ptr %a
   ret void
 }
@@ -292,12 +190,6 @@ define void @store_v1i64(ptr %a) {
 ; CHECK-NEXT:    mov z0.d, #0 // =0x0
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v1i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <1 x i64> zeroinitializer, ptr %a
   ret void
 }
@@ -308,12 +200,6 @@ define void @store_v1f64(ptr %a) {
 ; CHECK-NEXT:    fmov d0, xzr
 ; CHECK-NEXT:    str d0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v1f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi d0, #0000000000000000
-; NONEON-NOSVE-NEXT:    str d0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <1 x double> zeroinitializer, ptr %a
   ret void
 }
@@ -323,11 +209,6 @@ define void @store_v2i64(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp xzr, xzr, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    stp xzr, xzr, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <2 x i64> zeroinitializer, ptr %a
   ret void
 }
@@ -337,11 +218,6 @@ define void @store_v2f64(ptr %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp xzr, xzr, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    stp xzr, xzr, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <2 x double> zeroinitializer, ptr %a
   ret void
 }
@@ -352,12 +228,6 @@ define void @store_v4i64(ptr %a) {
 ; CHECK-NEXT:    mov z0.d, #0 // =0x0
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x i64> zeroinitializer, ptr %a
   ret void
 }
@@ -368,12 +238,6 @@ define void @store_v4f64(ptr %a) {
 ; CHECK-NEXT:    mov z0.d, #0 // =0x0
 ; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    stp q0, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   store <4 x double> zeroinitializer, ptr %a
   ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-subvector.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-subvector.ll
index d1873f436815..838db0ce8185 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-subvector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-subvector.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 ; Test we can code generater patterns of the form:
@@ -24,12 +23,6 @@ define void @subvector_v4i8(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ld1b { z0.h }, p0/z, [x0]
 ; CHECK-NEXT:    st1b { z0.h }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v4i8:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr w8, [x0]
-; NONEON-NOSVE-NEXT:    str w8, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i8>, ptr %in
   br label %bb1
 
@@ -44,12 +37,6 @@ define void @subvector_v8i8(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v8i8:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i8>, ptr %in
   br label %bb1
 
@@ -64,12 +51,6 @@ define void @subvector_v16i8(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v16i8:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i8>, ptr %in
   br label %bb1
 
@@ -84,12 +65,6 @@ define void @subvector_v32i8(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v32i8:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i8>, ptr %in
   br label %bb1
 
@@ -106,12 +81,6 @@ define void @subvector_v2i16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v2i16:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr w8, [x0]
-; NONEON-NOSVE-NEXT:    str w8, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x i16>, ptr %in
   br label %bb1
 
@@ -126,12 +95,6 @@ define void @subvector_v4i16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v4i16:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i16>, ptr %in
   br label %bb1
 
@@ -146,12 +109,6 @@ define void @subvector_v8i16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v8i16:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i16>, ptr %in
   br label %bb1
 
@@ -166,12 +123,6 @@ define void @subvector_v16i16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v16i16:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %in
   br label %bb1
 
@@ -187,12 +138,6 @@ define void @subvector_v2i32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v2i32:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x i32>, ptr %in
   br label %bb1
 
@@ -207,12 +152,6 @@ define void @subvector_v4i32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v4i32:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i32>, ptr %in
   br label %bb1
 
@@ -227,12 +166,6 @@ define void @subvector_v8i32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v8i32:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i32>, ptr %in
   br label %bb1
 
@@ -248,12 +181,6 @@ define void @subvector_v2i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v2i64:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x i64>, ptr %in
   br label %bb1
 
@@ -268,12 +195,6 @@ define void @subvector_v4i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v4i64:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i64>, ptr %in
   br label %bb1
 
@@ -289,12 +210,6 @@ define void @subvector_v2f16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    str w8, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v2f16:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr w8, [x0]
-; NONEON-NOSVE-NEXT:    str w8, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x half>, ptr %in
   br label %bb1
 
@@ -309,12 +224,6 @@ define void @subvector_v4f16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v4f16:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x half>, ptr %in
   br label %bb1
 
@@ -329,12 +238,6 @@ define void @subvector_v8f16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v8f16:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x half>, ptr %in
   br label %bb1
 
@@ -349,12 +252,6 @@ define void @subvector_v16f16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v16f16:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x half>, ptr %in
   br label %bb1
 
@@ -370,12 +267,6 @@ define void @subvector_v2f32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    str d0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v2f32:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr d0, [x0]
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x float>, ptr %in
   br label %bb1
 
@@ -390,12 +281,6 @@ define void @subvector_v4f32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v4f32:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x float>, ptr %in
   br label %bb1
 
@@ -410,12 +295,6 @@ define void @subvector_v8f32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v8f32:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x float>,ptr %in
   br label %bb1
 
@@ -431,12 +310,6 @@ define void @subvector_v2f64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    str q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v2f64:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    str q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x double>, ptr %in
   br label %bb1
 
@@ -451,12 +324,6 @@ define void @subvector_v4f64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: subvector_v4f64:
-; NONEON-NOSVE:       // %bb.0: // %bb1
-; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x double>, ptr %in
   br label %bb1
 
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
index f0a4368da3ee..7e3a175c40d2 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -13,13 +12,6 @@ define void @store_trunc_v8i16i8(ptr %ap, ptr %dest) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    st1b { z0.h }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_trunc_v8i16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    xtn v0.8b, v0.8h
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i16>, ptr %ap
   %val = trunc <8 x i16> %a to <8 x i8>
   store <8 x i8> %val, ptr %dest
@@ -33,14 +25,6 @@ define void @store_trunc_v4i32i8(ptr %ap, ptr %dest) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    st1b { z0.s }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_trunc_v4i32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
-; NONEON-NOSVE-NEXT:    str s0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i32>, ptr %ap
   %val = trunc <4 x i32> %a to <4 x i8>
   store <4 x i8> %val, ptr %dest
@@ -54,13 +38,6 @@ define void @store_trunc_v4i32i16(ptr %ap, ptr %dest) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    st1h { z0.s }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_trunc_v4i32i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i32>, ptr %ap
   %val = trunc <4 x i32> %a to <4 x i16>
   store <4 x i16> %val, ptr %dest
@@ -74,13 +51,6 @@ define void @store_trunc_v2i64i8(ptr %ap, ptr %dest) {
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    st1w { z0.d }, p0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_trunc_v2i64i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0]
-; NONEON-NOSVE-NEXT:    xtn v0.2s, v0.2d
-; NONEON-NOSVE-NEXT:    str d0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x i64>, ptr %ap
   %val = trunc <2 x i64> %a to <2 x i32>
   store <2 x i32> %val, ptr %dest
@@ -96,14 +66,6 @@ define void @store_trunc_v2i256i64(ptr %ap, ptr %dest) {
 ; CHECK-NEXT:    splice z1.d, p0, z1.d, z0.d
 ; CHECK-NEXT:    str q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: store_trunc_v2i256i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr d0, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldr d1, [x0]
-; NONEON-NOSVE-NEXT:    mov v1.d[1], v0.d[0]
-; NONEON-NOSVE-NEXT:    str q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <2 x i256>, ptr %ap
   %val = trunc <2 x i256> %a to <2 x i64>
   store <2 x i64> %val, ptr %dest
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll
index 4895ffb6858e..70219dd30f76 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -20,12 +19,6 @@ define <16 x i8> @trunc_v16i16_v16i8(ptr %in) nounwind {
 ; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v16i16_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i16>, ptr %in
   %b = trunc <16 x i16> %a to <16 x i8>
   ret <16 x i8> %b
@@ -48,17 +41,6 @@ define void @trunc_v32i16_v32i8(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    add z1.b, z2.b, z2.b
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v32i16_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uzp1 v1.16b, v3.16b, v2.16b
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i16>, ptr %in
   %b = trunc <32 x i16> %a to <32 x i8>
   %c = add <32 x i8> %b, %b
@@ -94,24 +76,6 @@ define void @trunc_v64i16_v64i8(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    stp q0, q1, [x1, #32]
 ; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v64i16_v64i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #64]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #96]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ldp q6, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    uzp1 v2.16b, v3.16b, v2.16b
-; NONEON-NOSVE-NEXT:    uzp1 v3.16b, v5.16b, v4.16b
-; NONEON-NOSVE-NEXT:    uzp1 v1.16b, v6.16b, v1.16b
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v2.16b, v2.16b, v2.16b
-; NONEON-NOSVE-NEXT:    add v3.16b, v3.16b, v3.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <64 x i16>, ptr %in
   %b = trunc <64 x i16> %a to <64 x i8>
   %c = add <64 x i8> %b, %b
@@ -169,38 +133,6 @@ define void @trunc_v128i16_v128i8(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    stp q2, q3, [x1, #32]
 ; CHECK-NEXT:    stp q4, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v128i16_v128i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #192]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0, #224]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [x0, #128]
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    ldp q16, q1, [x0, #160]
-; NONEON-NOSVE-NEXT:    uzp1 v4.16b, v5.16b, v4.16b
-; NONEON-NOSVE-NEXT:    ldp q17, q5, [x0, #64]
-; NONEON-NOSVE-NEXT:    uzp1 v6.16b, v7.16b, v6.16b
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q18, q7, [x0, #96]
-; NONEON-NOSVE-NEXT:    uzp1 v1.16b, v16.16b, v1.16b
-; NONEON-NOSVE-NEXT:    uzp1 v5.16b, v17.16b, v5.16b
-; NONEON-NOSVE-NEXT:    ldp q17, q16, [x0, #32]
-; NONEON-NOSVE-NEXT:    uzp1 v2.16b, v3.16b, v2.16b
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v4.16b, v4.16b, v4.16b
-; NONEON-NOSVE-NEXT:    uzp1 v7.16b, v18.16b, v7.16b
-; NONEON-NOSVE-NEXT:    add v3.16b, v6.16b, v6.16b
-; NONEON-NOSVE-NEXT:    uzp1 v6.16b, v17.16b, v16.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q4, [x1, #96]
-; NONEON-NOSVE-NEXT:    add v0.16b, v5.16b, v5.16b
-; NONEON-NOSVE-NEXT:    add v2.16b, v2.16b, v2.16b
-; NONEON-NOSVE-NEXT:    add v4.16b, v7.16b, v7.16b
-; NONEON-NOSVE-NEXT:    stp q3, q1, [x1, #64]
-; NONEON-NOSVE-NEXT:    add v1.16b, v6.16b, v6.16b
-; NONEON-NOSVE-NEXT:    stp q0, q4, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q2, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <128 x i16>, ptr %in
   %b = trunc <128 x i16> %a to <128 x i8>
   %c = add <128 x i8> %b, %b
@@ -223,13 +155,6 @@ define <8 x i8> @trunc_v8i32_v8i8(ptr %in) nounwind {
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v8i32_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    xtn v0.8b, v0.8h
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i32>, ptr %in
   %b = trunc <8 x i32> %a to <8 x i8>
   ret <8 x i8> %b
@@ -253,15 +178,6 @@ define <16 x i8> @trunc_v16i32_v16i8(ptr %in) nounwind {
 ; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v16i32_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #32]
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    uzp1 v2.8h, v3.8h, v2.8h
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i32>, ptr %in
   %b = trunc <16 x i32> %a to <16 x i8>
   ret <16 x i8> %b
@@ -299,23 +215,6 @@ define void @trunc_v32i32_v32i8(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    add z1.b, z3.b, z3.b
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v32i32_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #64]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #96]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [x0, #32]
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    uzp1 v2.8h, v3.8h, v2.8h
-; NONEON-NOSVE-NEXT:    uzp1 v3.8h, v5.8h, v4.8h
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v7.8h, v6.8h
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    uzp1 v1.16b, v3.16b, v1.16b
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i32>, ptr %in
   %b = trunc <32 x i32> %a to <32 x i8>
   %c = add <32 x i8> %b, %b
@@ -380,36 +279,6 @@ define void @trunc_v64i32_v64i8(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    stp q1, q2, [x1, #32]
 ; CHECK-NEXT:    stp q3, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v64i32_v64i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #128]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #160]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0, #192]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [x0, #224]
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    uzp1 v2.8h, v3.8h, v2.8h
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v4.8h, v5.8h, v4.8h
-; NONEON-NOSVE-NEXT:    ldp q17, q5, [x0, #64]
-; NONEON-NOSVE-NEXT:    uzp1 v6.8h, v7.8h, v6.8h
-; NONEON-NOSVE-NEXT:    ldp q16, q7, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q19, q18, [x0, #96]
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
-; NONEON-NOSVE-NEXT:    uzp1 v5.8h, v17.8h, v5.8h
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
-; NONEON-NOSVE-NEXT:    uzp1 v7.8h, v16.8h, v7.8h
-; NONEON-NOSVE-NEXT:    uzp1 v3.8h, v19.8h, v18.8h
-; NONEON-NOSVE-NEXT:    uzp1 v2.16b, v4.16b, v6.16b
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uzp1 v1.16b, v1.16b, v7.16b
-; NONEON-NOSVE-NEXT:    uzp1 v3.16b, v5.16b, v3.16b
-; NONEON-NOSVE-NEXT:    add v2.16b, v2.16b, v2.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add v3.16b, v3.16b, v3.16b
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <64 x i32>, ptr %in
   %b = trunc <64 x i32> %a to <64 x i8>
   %c = add <64 x i8> %b, %b
@@ -431,12 +300,6 @@ define <8 x i16> @trunc_v8i32_v8i16(ptr %in) nounwind {
 ; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v8i32_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i32>, ptr %in
   %b = trunc <8 x i32> %a to <8 x i16>
   ret <8 x i16> %b
@@ -459,17 +322,6 @@ define void @trunc_v16i32_v16i16(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    add z1.h, z2.h, z2.h
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v16i32_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v3.8h, v2.8h
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i32>, ptr %in
   %b = trunc <16 x i32> %a to <16 x i16>
   %c = add <16 x i16> %b, %b
@@ -505,24 +357,6 @@ define void @trunc_v32i32_v32i16(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    stp q0, q1, [x1, #32]
 ; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v32i32_v32i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #64]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #96]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    ldp q6, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    uzp1 v2.8h, v3.8h, v2.8h
-; NONEON-NOSVE-NEXT:    uzp1 v3.8h, v5.8h, v4.8h
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v6.8h, v1.8h
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v2.8h, v2.8h, v2.8h
-; NONEON-NOSVE-NEXT:    add v3.8h, v3.8h, v3.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i32>, ptr %in
   %b = trunc <32 x i32> %a to <32 x i16>
   %c = add <32 x i16> %b, %b
@@ -580,38 +414,6 @@ define void @trunc_v64i32_v64i16(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    stp q2, q3, [x1, #32]
 ; CHECK-NEXT:    stp q4, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v64i32_v64i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #192]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0, #224]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [x0, #128]
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    ldp q16, q1, [x0, #160]
-; NONEON-NOSVE-NEXT:    uzp1 v4.8h, v5.8h, v4.8h
-; NONEON-NOSVE-NEXT:    ldp q17, q5, [x0, #64]
-; NONEON-NOSVE-NEXT:    uzp1 v6.8h, v7.8h, v6.8h
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q18, q7, [x0, #96]
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v16.8h, v1.8h
-; NONEON-NOSVE-NEXT:    uzp1 v5.8h, v17.8h, v5.8h
-; NONEON-NOSVE-NEXT:    ldp q17, q16, [x0, #32]
-; NONEON-NOSVE-NEXT:    uzp1 v2.8h, v3.8h, v2.8h
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v4.8h, v4.8h, v4.8h
-; NONEON-NOSVE-NEXT:    uzp1 v7.8h, v18.8h, v7.8h
-; NONEON-NOSVE-NEXT:    add v3.8h, v6.8h, v6.8h
-; NONEON-NOSVE-NEXT:    uzp1 v6.8h, v17.8h, v16.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q4, [x1, #96]
-; NONEON-NOSVE-NEXT:    add v0.8h, v5.8h, v5.8h
-; NONEON-NOSVE-NEXT:    add v2.8h, v2.8h, v2.8h
-; NONEON-NOSVE-NEXT:    add v4.8h, v7.8h, v7.8h
-; NONEON-NOSVE-NEXT:    stp q3, q1, [x1, #64]
-; NONEON-NOSVE-NEXT:    add v1.8h, v6.8h, v6.8h
-; NONEON-NOSVE-NEXT:    stp q0, q4, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q2, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <64 x i32>, ptr %in
   %b = trunc <64 x i32> %a to <64 x i16>
   %c = add <64 x i16> %b, %b
@@ -635,13 +437,6 @@ define <4 x i8> @trunc_v4i64_v4i8(ptr %in) nounwind {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v4i64_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i64>, ptr %in
   %b = trunc <4 x i64> %a to <4 x i8>
   ret <4 x i8> %b
@@ -666,16 +461,6 @@ define <8 x i8> @trunc_v8i64_v8i8(ptr %in) nounwind {
 ; CHECK-NEXT:    uzp1 z0.b, z1.b, z1.b
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v8i64_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #32]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp1 v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; NONEON-NOSVE-NEXT:    xtn v0.8b, v0.8h
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i64>, ptr %in
   %b = trunc <8 x i64> %a to <8 x i8>
   ret <8 x i8> %b
@@ -714,21 +499,6 @@ define <16 x i8> @trunc_v16i64_v16i8(ptr %in) nounwind {
 ; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v16i64_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #96]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [x0, #64]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp1 v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    uzp1 v4.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    uzp1 v3.4s, v7.4s, v6.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v3.8h, v2.8h
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i64>, ptr %in
   %b = trunc <16 x i64> %a to <16 x i8>
   ret <16 x i8> %b
@@ -795,35 +565,6 @@ define void @trunc_v32i64_v32i8(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    add z0.b, z0.b, z0.b
 ; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v32i64_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #224]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0, #192]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [x0, #96]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp1 v1.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0, #128]
-; NONEON-NOSVE-NEXT:    ldp q17, q16, [x0, #160]
-; NONEON-NOSVE-NEXT:    uzp1 v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    ldp q19, q18, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q21, q20, [x0, #64]
-; NONEON-NOSVE-NEXT:    uzp1 v4.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    uzp1 v16.4s, v17.4s, v16.4s
-; NONEON-NOSVE-NEXT:    uzp1 v5.4s, v7.4s, v6.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; NONEON-NOSVE-NEXT:    uzp1 v7.4s, v19.4s, v18.4s
-; NONEON-NOSVE-NEXT:    uzp1 v6.4s, v21.4s, v20.4s
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v4.8h, v16.8h
-; NONEON-NOSVE-NEXT:    uzp1 v2.8h, v2.8h, v7.8h
-; NONEON-NOSVE-NEXT:    uzp1 v3.8h, v6.8h, v5.8h
-; NONEON-NOSVE-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
-; NONEON-NOSVE-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; NONEON-NOSVE-NEXT:    add v0.16b, v0.16b, v0.16b
-; NONEON-NOSVE-NEXT:    add v1.16b, v1.16b, v1.16b
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i64>, ptr %in
   %b = trunc <32 x i64> %a to <32 x i8>
   %c = add <32 x i8> %b, %b
@@ -846,13 +587,6 @@ define <4 x i16> @trunc_v4i64_v4i16(ptr %in) nounwind {
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v4i64_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    xtn v0.4h, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i64>, ptr %in
   %b = trunc <4 x i64> %a to <4 x i16>
   ret <4 x i16> %b
@@ -876,15 +610,6 @@ define <8 x i16> @trunc_v8i64_v8i16(ptr %in) nounwind {
 ; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v8i64_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #32]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp1 v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i64>, ptr %in
   %b = trunc <8 x i64> %a to <8 x i16>
   ret <8 x i16> %b
@@ -922,23 +647,6 @@ define void @trunc_v16i64_v16i16(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    add z1.h, z3.h, z3.h
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v16i64_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #64]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #96]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [x0, #32]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp1 v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    uzp1 v3.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    uzp1 v1.4s, v7.4s, v6.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i64>, ptr %in
   %b = trunc <16 x i64> %a to <16 x i16>
   %c = add <16 x i16> %b, %b
@@ -1003,36 +711,6 @@ define void @trunc_v32i64_v32i16(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    stp q1, q2, [x1, #32]
 ; CHECK-NEXT:    stp q3, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v32i64_v32i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #128]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #160]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0, #192]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [x0, #224]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp1 v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    ldp q3, q1, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v4.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    ldp q17, q5, [x0, #64]
-; NONEON-NOSVE-NEXT:    uzp1 v6.4s, v7.4s, v6.4s
-; NONEON-NOSVE-NEXT:    ldp q16, q7, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q19, q18, [x0, #96]
-; NONEON-NOSVE-NEXT:    uzp1 v1.4s, v3.4s, v1.4s
-; NONEON-NOSVE-NEXT:    uzp1 v5.4s, v17.4s, v5.4s
-; NONEON-NOSVE-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; NONEON-NOSVE-NEXT:    uzp1 v7.4s, v16.4s, v7.4s
-; NONEON-NOSVE-NEXT:    uzp1 v3.4s, v19.4s, v18.4s
-; NONEON-NOSVE-NEXT:    uzp1 v2.8h, v4.8h, v6.8h
-; NONEON-NOSVE-NEXT:    add v0.8h, v0.8h, v0.8h
-; NONEON-NOSVE-NEXT:    uzp1 v1.8h, v1.8h, v7.8h
-; NONEON-NOSVE-NEXT:    uzp1 v3.8h, v5.8h, v3.8h
-; NONEON-NOSVE-NEXT:    add v2.8h, v2.8h, v2.8h
-; NONEON-NOSVE-NEXT:    add v1.8h, v1.8h, v1.8h
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    add v3.8h, v3.8h, v3.8h
-; NONEON-NOSVE-NEXT:    stp q1, q3, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i64>, ptr %in
   %b = trunc <32 x i64> %a to <32 x i16>
   %c = add <32 x i16> %b, %b
@@ -1054,12 +732,6 @@ define <4 x i32> @trunc_v4i64_v4i32(ptr %in) nounwind {
 ; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v4i64_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ret
   %a = load <4 x i64>, ptr %in
   %b = trunc <4 x i64> %a to <4 x i32>
   ret <4 x i32> %b
@@ -1082,17 +754,6 @@ define void @trunc_v8i64_v8i32(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    add z1.s, z2.s, z2.s
 ; CHECK-NEXT:    stp q1, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v8i64_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #32]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    uzp1 v1.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    add v0.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    add v1.4s, v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <8 x i64>, ptr %in
   %b = trunc <8 x i64> %a to <8 x i32>
   %c = add <8 x i32> %b, %b
@@ -1128,24 +789,6 @@ define void @trunc_v16i64_v16i32(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    stp q0, q1, [x1, #32]
 ; CHECK-NEXT:    stp q2, q3, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v16i64_v16i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #64]
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0, #96]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ldp q6, q1, [x0, #32]
-; NONEON-NOSVE-NEXT:    uzp1 v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    uzp1 v3.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    uzp1 v1.4s, v6.4s, v1.4s
-; NONEON-NOSVE-NEXT:    add v0.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    add v2.4s, v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    add v3.4s, v3.4s, v3.4s
-; NONEON-NOSVE-NEXT:    add v1.4s, v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q2, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q3, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <16 x i64>, ptr %in
   %b = trunc <16 x i64> %a to <16 x i32>
   %c = add <16 x i32> %b, %b
@@ -1203,38 +846,6 @@ define void @trunc_v32i64_v32i32(ptr %in, ptr %out) nounwind {
 ; CHECK-NEXT:    stp q2, q3, [x1, #32]
 ; CHECK-NEXT:    stp q4, q0, [x1]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: trunc_v32i64_v32i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0, #192]
-; NONEON-NOSVE-NEXT:    ldp q5, q4, [x0, #224]
-; NONEON-NOSVE-NEXT:    ldp q7, q6, [x0, #128]
-; NONEON-NOSVE-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
-; NONEON-NOSVE-NEXT:    ldp q16, q1, [x0, #160]
-; NONEON-NOSVE-NEXT:    uzp1 v4.4s, v5.4s, v4.4s
-; NONEON-NOSVE-NEXT:    ldp q17, q5, [x0, #64]
-; NONEON-NOSVE-NEXT:    uzp1 v6.4s, v7.4s, v6.4s
-; NONEON-NOSVE-NEXT:    ldp q3, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldp q18, q7, [x0, #96]
-; NONEON-NOSVE-NEXT:    uzp1 v1.4s, v16.4s, v1.4s
-; NONEON-NOSVE-NEXT:    uzp1 v5.4s, v17.4s, v5.4s
-; NONEON-NOSVE-NEXT:    ldp q17, q16, [x0, #32]
-; NONEON-NOSVE-NEXT:    uzp1 v2.4s, v3.4s, v2.4s
-; NONEON-NOSVE-NEXT:    add v0.4s, v0.4s, v0.4s
-; NONEON-NOSVE-NEXT:    add v4.4s, v4.4s, v4.4s
-; NONEON-NOSVE-NEXT:    uzp1 v7.4s, v18.4s, v7.4s
-; NONEON-NOSVE-NEXT:    add v3.4s, v6.4s, v6.4s
-; NONEON-NOSVE-NEXT:    uzp1 v6.4s, v17.4s, v16.4s
-; NONEON-NOSVE-NEXT:    add v1.4s, v1.4s, v1.4s
-; NONEON-NOSVE-NEXT:    stp q0, q4, [x1, #96]
-; NONEON-NOSVE-NEXT:    add v0.4s, v5.4s, v5.4s
-; NONEON-NOSVE-NEXT:    add v2.4s, v2.4s, v2.4s
-; NONEON-NOSVE-NEXT:    add v4.4s, v7.4s, v7.4s
-; NONEON-NOSVE-NEXT:    stp q3, q1, [x1, #64]
-; NONEON-NOSVE-NEXT:    add v1.4s, v6.4s, v6.4s
-; NONEON-NOSVE-NEXT:    stp q0, q4, [x1, #32]
-; NONEON-NOSVE-NEXT:    stp q2, q1, [x1]
-; NONEON-NOSVE-NEXT:    ret
   %a = load <32 x i64>, ptr %in
   %b = trunc <32 x i64> %a to <32 x i32>
   %c = add <32 x i32> %b, %b
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-vector-shuffle.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-vector-shuffle.ll
index dd308dfadd80..175731480407 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-vector-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-vector-shuffle.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -15,12 +14,6 @@ define <4 x i8> @shuffle_ext_byone_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
 ; CHECK-NEXT:    tbl z0.h, { z0.h }, z1.h
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v4i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v1.8b, v0.8b, v0.8b, #6
-; NONEON-NOSVE-NEXT:    trn1 v0.4h, v0.4h, v1.4h
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <4 x i8> %op1, <4 x i8> %op2, <4 x i32> <i32 0, i32 3, i32 2, i32 1>
   ret <4 x i8> %ret
 }
@@ -35,11 +28,6 @@ define <8 x i8> @shuffle_ext_byone_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
 ; CHECK-NEXT:    insr z1.b, w8
 ; CHECK-NEXT:    fmov d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v8i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.8b, v0.8b, v1.8b, #7
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <8 x i8> %op1, <8 x i8> %op2, <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
   ret <8 x i8> %ret
 }
@@ -54,11 +42,6 @@ define <16 x i8> @shuffle_ext_byone_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    insr z1.b, w8
 ; CHECK-NEXT:    mov z0.d, z1.d
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v16i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #15
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <16 x i8> %op1, <16 x i8> %op2, <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22,
                                                                    i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
   ret <16 x i8> %ret
@@ -77,15 +60,6 @@ define void @shuffle_ext_byone_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    insr z3.b, w8
 ; CHECK-NEXT:    stp q1, q3, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v32i8:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x1]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #15
-; NONEON-NOSVE-NEXT:    ext v1.16b, v1.16b, v2.16b, #15
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
   %op2 = load <32 x i8>, ptr %b
   %ret = shufflevector <32 x i8> %op1, <32 x i8> %op2, <32 x i32> <i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38,
@@ -104,11 +78,6 @@ define <2 x i16> @shuffle_ext_byone_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
 ; CHECK-NEXT:    revw z0.d, p0/m, z0.d
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v2i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    rev64 v0.2s, v0.2s
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <2 x i16> %op1, <2 x i16> %op2, <2 x i32> <i32 1, i32 0>
   ret <2 x i16> %ret
 }
@@ -123,11 +92,6 @@ define <4 x i16> @shuffle_ext_byone_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
 ; CHECK-NEXT:    insr z1.h, w8
 ; CHECK-NEXT:    fmov d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v4i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.8b, v0.8b, v1.8b, #6
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <4 x i16> %op1, <4 x i16> %op2, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
   ret <4 x i16> %ret
 }
@@ -142,11 +106,6 @@ define <8 x i16> @shuffle_ext_byone_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    insr z1.h, w8
 ; CHECK-NEXT:    mov z0.d, z1.d
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v8i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #14
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <8 x i16> %op1, <8 x i16> %op2, <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
   ret <8 x i16> %ret
 }
@@ -164,15 +123,6 @@ define void @shuffle_ext_byone_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    insr z3.h, w8
 ; CHECK-NEXT:    stp q1, q3, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v16i16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x1]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #14
-; NONEON-NOSVE-NEXT:    ext v1.16b, v1.16b, v2.16b, #14
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
   %op2 = load <16 x i16>, ptr %b
   %ret = shufflevector <16 x i16> %op1, <16 x i16> %op2, <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22,
@@ -191,11 +141,6 @@ define <2 x i32> @shuffle_ext_byone_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-NEXT:    insr z1.s, w8
 ; CHECK-NEXT:    fmov d0, d1
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v2i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.8b, v0.8b, v1.8b, #4
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <2 x i32> %op1, <2 x i32> %op2, <2 x i32> <i32 1, i32 2>
   ret <2 x i32> %ret
 }
@@ -210,11 +155,6 @@ define <4 x i32> @shuffle_ext_byone_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-NEXT:    insr z1.s, w8
 ; CHECK-NEXT:    mov z0.d, z1.d
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v4i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #12
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <4 x i32> %op1, <4 x i32> %op2, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
   ret <4 x i32> %ret
 }
@@ -232,15 +172,6 @@ define void @shuffle_ext_byone_v8i32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    insr z3.s, w8
 ; CHECK-NEXT:    stp q1, q3, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v8i32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x1]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #12
-; NONEON-NOSVE-NEXT:    ext v1.16b, v1.16b, v2.16b, #12
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x i32>, ptr %a
   %op2 = load <8 x i32>, ptr %b
   %ret = shufflevector <8 x i32> %op1, <8 x i32> %op2, <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
@@ -258,11 +189,6 @@ define <2 x i64> @shuffle_ext_byone_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-NEXT:    insr z1.d, x8
 ; CHECK-NEXT:    mov z0.d, z1.d
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v2i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #8
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <2 x i64> %op1, <2 x i64> %op2, <2 x i32> <i32 1, i32 2>
   ret <2 x i64> %ret
 }
@@ -280,15 +206,6 @@ define void @shuffle_ext_byone_v4i64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    insr z3.d, x8
 ; CHECK-NEXT:    stp q1, q3, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v4i64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x1]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #8
-; NONEON-NOSVE-NEXT:    ext v1.16b, v1.16b, v2.16b, #8
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x i64>, ptr %a
   %op2 = load <4 x i64>, ptr %b
   %ret = shufflevector <4 x i64> %op1, <4 x i64> %op2, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
@@ -306,11 +223,6 @@ define <4 x half> @shuffle_ext_byone_v4f16(<4 x half> %op1, <4 x half> %op2) {
 ; CHECK-NEXT:    insr z0.h, h2
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v4f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.8b, v0.8b, v1.8b, #6
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <4 x half> %op1, <4 x half> %op2, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
   ret <4 x half> %ret
 }
@@ -324,11 +236,6 @@ define <8 x half> @shuffle_ext_byone_v8f16(<8 x half> %op1, <8 x half> %op2) {
 ; CHECK-NEXT:    insr z0.h, h2
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v8f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #14
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <8 x half> %op1, <8 x half> %op2, <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
   ret <8 x half> %ret
 }
@@ -344,15 +251,6 @@ define void @shuffle_ext_byone_v16f16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    insr z3.h, h2
 ; CHECK-NEXT:    stp q1, q3, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v16f16:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x1]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #14
-; NONEON-NOSVE-NEXT:    ext v1.16b, v1.16b, v2.16b, #14
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %op2 = load <16 x half>, ptr %b
   %ret = shufflevector <16 x half> %op1, <16 x half> %op2, <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22,
@@ -370,11 +268,6 @@ define <2 x float> @shuffle_ext_byone_v2f32(<2 x float> %op1, <2 x float> %op2)
 ; CHECK-NEXT:    insr z0.s, s2
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v2f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.8b, v0.8b, v1.8b, #4
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <2 x float> %op1, <2 x float> %op2, <2 x i32> <i32 1, i32 2>
   ret <2 x float> %ret
 }
@@ -388,11 +281,6 @@ define <4 x float> @shuffle_ext_byone_v4f32(<4 x float> %op1, <4 x float> %op2)
 ; CHECK-NEXT:    insr z0.s, s2
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v4f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #12
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <4 x float> %op1, <4 x float> %op2, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
   ret <4 x float> %ret
 }
@@ -408,15 +296,6 @@ define void @shuffle_ext_byone_v8f32(ptr %a, ptr %b) {
 ; CHECK-NEXT:    insr z3.s, s2
 ; CHECK-NEXT:    stp q1, q3, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v8f32:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x1]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #12
-; NONEON-NOSVE-NEXT:    ext v1.16b, v1.16b, v2.16b, #12
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <8 x float>, ptr %a
   %op2 = load <8 x float>, ptr %b
   %ret = shufflevector <8 x float> %op1, <8 x float> %op2, <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
@@ -433,11 +312,6 @@ define <2 x double> @shuffle_ext_byone_v2f64(<2 x double> %op1, <2 x double> %op
 ; CHECK-NEXT:    insr z0.d, d2
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v2f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #8
-; NONEON-NOSVE-NEXT:    ret
   %ret = shufflevector <2 x double> %op1, <2 x double> %op2, <2 x i32> <i32 1, i32 2>
   ret <2 x double> %ret
 }
@@ -453,15 +327,6 @@ define void @shuffle_ext_byone_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    insr z3.d, d2
 ; CHECK-NEXT:    stp q1, q3, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_v4f64:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q1, q2, [x1]
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v1.16b, #8
-; NONEON-NOSVE-NEXT:    ext v1.16b, v1.16b, v2.16b, #8
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %ret = shufflevector <4 x double> %op1, <4 x double> %op2, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
@@ -480,15 +345,6 @@ define void @shuffle_ext_byone_reverse(ptr %a, ptr %b) {
 ; CHECK-NEXT:    insr z3.d, d2
 ; CHECK-NEXT:    stp q1, q3, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_byone_reverse:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldp q0, q2, [x0]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1, #16]
-; NONEON-NOSVE-NEXT:    ext v1.16b, v1.16b, v0.16b, #8
-; NONEON-NOSVE-NEXT:    ext v0.16b, v0.16b, v2.16b, #8
-; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %ret = shufflevector <4 x double> %op1, <4 x double> %op2, <4 x i32> <i32 7, i32 0, i32 1, i32 2>
@@ -503,13 +359,6 @@ define void @shuffle_ext_invalid(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q1, [x1]
 ; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: shuffle_ext_invalid:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    ldr q0, [x0, #16]
-; NONEON-NOSVE-NEXT:    ldr q1, [x1]
-; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
-; NONEON-NOSVE-NEXT:    ret
   %op1 = load <4 x double>, ptr %a
   %op2 = load <4 x double>, ptr %b
   %ret = shufflevector <4 x double> %op1, <4 x double> %op2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-test-register-mov.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-test-register-mov.ll
index 42f3f03a5ea0..337a2134de5b 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-test-register-mov.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-test-register-mov.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve -force-streaming-compatible-sve < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming-compatible-sve < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible-sve < %s | FileCheck %s --check-prefix=NONEON-NOSVE
 
 
 target triple = "aarch64-unknown-linux-gnu"
@@ -12,11 +11,6 @@ define fp128 @test_streaming_compatible_register_mov(fp128 %q0, fp128 %q1) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov z0.d, z1.d
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: test_streaming_compatible_register_mov:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    mov v0.16b, v1.16b
-; NONEON-NOSVE-NEXT:    ret
   ret fp128 %q1
 }
 
@@ -26,11 +20,6 @@ define double @fp_zero_constant() {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov d0, xzr
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fp_zero_constant:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    fmov d0, xzr
-; NONEON-NOSVE-NEXT:    ret
   ret double 0.0
 }
 
@@ -40,11 +29,6 @@ define <2 x i64> @fixed_vec_zero_constant() {
 ; CHECK-NEXT:    mov z0.d, #0 // =0x0
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fixed_vec_zero_constant:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    ret
   ret <2 x i64> zeroinitializer
 }
 
@@ -54,10 +38,5 @@ define <2 x double> @fixed_vec_fp_zero_constant() {
 ; CHECK-NEXT:    mov z0.d, #0 // =0x0
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
-;
-; NONEON-NOSVE-LABEL: fixed_vec_fp_zero_constant:
-; NONEON-NOSVE:       // %bb.0:
-; NONEON-NOSVE-NEXT:    movi v0.2d, #0000000000000000
-; NONEON-NOSVE-NEXT:    ret
   ret <2 x double> <double 0.0, double 0.0>
 }
-- 
GitLab


From 87235fa9af06b639d7b4b1eb7ac5840fb56cc569 Mon Sep 17 00:00:00 2001
From: Yuxuan Chen <yuxuanchen1997@outlook.com>
Date: Thu, 9 May 2024 08:21:40 -0700
Subject: [PATCH 0302/1206] [NFC] CoroElide: Refactor `Lowerer` into
 `CoroIdElider` (#91539)

This patch contains no functional changes.

The main goal of this patch is to get better clarity out of the code, to
make intentions and assumptions clear.

One major design problem I had in the past were `Lowerer`. It previously
inherited from `coro::LowererBase` but it doesn't use any of the fields
or methods from `LowererBase`. It might be an artifact leftover from
previous designs of this code.

Furthermore, we should clarify that although one such instance is bound
to the function, `Lowerer` was dedicated to one `CoroId` instruction at
a time. We rely on a sequence of fragile constructs like
`CoroBegins.clear(); DestroyAddr.clear()`. This doesn't help understand
the code.

What's worse is that we have confusing calls like
`elideHeapAllocations(CoroId->getFunction(), ...` and it might get
confused with `CoroId->getCoroutine()`.

The new structure intends to make it clear that we always operate on one
`CoroId` at a time, which may have multiple `CoroBegin`s. Such structure
doesn't rely on frequent `.clear()` that's prone to miss.
---
 llvm/lib/Transforms/Coroutines/CoroElide.cpp | 318 ++++++++++---------
 1 file changed, 169 insertions(+), 149 deletions(-)

diff --git a/llvm/lib/Transforms/Coroutines/CoroElide.cpp b/llvm/lib/Transforms/Coroutines/CoroElide.cpp
index d356a6d2e575..bb244489e4c2 100644
--- a/llvm/lib/Transforms/Coroutines/CoroElide.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroElide.cpp
@@ -33,24 +33,47 @@ static cl::opt<std::string> CoroElideInfoOutputFilename(
 
 namespace {
 // Created on demand if the coro-elide pass has work to do.
-struct Lowerer : coro::LowererBase {
+class FunctionElideInfo {
+public:
+  FunctionElideInfo(Function *F) : ContainingFunction(F) {
+    this->collectPostSplitCoroIds();
+  }
+
+  bool hasCoroIds() const { return !CoroIds.empty(); }
+
+  const SmallVectorImpl<CoroIdInst *> &getCoroIds() const { return CoroIds; }
+
+private:
+  Function *ContainingFunction;
   SmallVector<CoroIdInst *, 4> CoroIds;
+  // Used in canCoroBeginEscape to distinguish coro.suspend switchs.
+  SmallPtrSet<const SwitchInst *, 4> CoroSuspendSwitches;
+
+  void collectPostSplitCoroIds();
+  friend class CoroIdElider;
+};
+
+class CoroIdElider {
+public:
+  CoroIdElider(CoroIdInst *CoroId, FunctionElideInfo &FEI, AAResults &AA,
+               DominatorTree &DT, OptimizationRemarkEmitter &ORE);
+  void elideHeapAllocations(uint64_t FrameSize, Align FrameAlign);
+  bool lifetimeEligibleForElide() const;
+  bool attemptElide();
+  bool canCoroBeginEscape(const CoroBeginInst *,
+                          const SmallPtrSetImpl<BasicBlock *> &) const;
+
+private:
+  CoroIdInst *CoroId;
+  FunctionElideInfo &FEI;
+  AAResults &AA;
+  DominatorTree &DT;
+  OptimizationRemarkEmitter &ORE;
+
   SmallVector<CoroBeginInst *, 1> CoroBegins;
   SmallVector<CoroAllocInst *, 1> CoroAllocs;
   SmallVector<CoroSubFnInst *, 4> ResumeAddr;
   DenseMap<CoroBeginInst *, SmallVector<CoroSubFnInst *, 4>> DestroyAddr;
-  SmallPtrSet<const SwitchInst *, 4> CoroSuspendSwitches;
-
-  Lowerer(Module &M) : LowererBase(M) {}
-
-  void elideHeapAllocations(Function *F, uint64_t FrameSize, Align FrameAlign,
-                            AAResults &AA);
-  bool shouldElide(Function *F, DominatorTree &DT) const;
-  void collectPostSplitCoroIds(Function *F);
-  bool processCoroId(CoroIdInst *, AAResults &AA, DominatorTree &DT,
-                     OptimizationRemarkEmitter &ORE);
-  bool hasEscapePath(const CoroBeginInst *,
-                     const SmallPtrSetImpl<BasicBlock *> &) const;
 };
 } // end anonymous namespace
 
@@ -136,14 +159,66 @@ static std::unique_ptr<raw_fd_ostream> getOrCreateLogFile() {
 }
 #endif
 
+void FunctionElideInfo::collectPostSplitCoroIds() {
+  for (auto &I : instructions(this->ContainingFunction)) {
+    if (auto *CII = dyn_cast<CoroIdInst>(&I))
+      if (CII->getInfo().isPostSplit())
+        // If it is the coroutine itself, don't touch it.
+        if (CII->getCoroutine() != CII->getFunction())
+          CoroIds.push_back(CII);
+
+    // Consider case like:
+    // %0 = call i8 @llvm.coro.suspend(...)
+    // switch i8 %0, label %suspend [i8 0, label %resume
+    //                              i8 1, label %cleanup]
+    // and collect the SwitchInsts which are used by escape analysis later.
+    if (auto *CSI = dyn_cast<CoroSuspendInst>(&I))
+      if (CSI->hasOneUse() && isa<SwitchInst>(CSI->use_begin()->getUser())) {
+        SwitchInst *SWI = cast<SwitchInst>(CSI->use_begin()->getUser());
+        if (SWI->getNumCases() == 2)
+          CoroSuspendSwitches.insert(SWI);
+      }
+  }
+}
+
+CoroIdElider::CoroIdElider(CoroIdInst *CoroId, FunctionElideInfo &FEI,
+                           AAResults &AA, DominatorTree &DT,
+                           OptimizationRemarkEmitter &ORE)
+    : CoroId(CoroId), FEI(FEI), AA(AA), DT(DT), ORE(ORE) {
+  // Collect all coro.begin and coro.allocs associated with this coro.id.
+  for (User *U : CoroId->users()) {
+    if (auto *CB = dyn_cast<CoroBeginInst>(U))
+      CoroBegins.push_back(CB);
+    else if (auto *CA = dyn_cast<CoroAllocInst>(U))
+      CoroAllocs.push_back(CA);
+  }
+
+  // Collect all coro.subfn.addrs associated with coro.begin.
+  // Note, we only devirtualize the calls if their coro.subfn.addr refers to
+  // coro.begin directly. If we run into cases where this check is too
+  // conservative, we can consider relaxing the check.
+  for (CoroBeginInst *CB : CoroBegins) {
+    for (User *U : CB->users())
+      if (auto *II = dyn_cast<CoroSubFnInst>(U))
+        switch (II->getIndex()) {
+        case CoroSubFnInst::ResumeIndex:
+          ResumeAddr.push_back(II);
+          break;
+        case CoroSubFnInst::DestroyIndex:
+          DestroyAddr[CB].push_back(II);
+          break;
+        default:
+          llvm_unreachable("unexpected coro.subfn.addr constant");
+        }
+  }
+}
+
 // To elide heap allocations we need to suppress code blocks guarded by
 // llvm.coro.alloc and llvm.coro.free instructions.
-void Lowerer::elideHeapAllocations(Function *F, uint64_t FrameSize,
-                                   Align FrameAlign, AAResults &AA) {
-  LLVMContext &C = F->getContext();
+void CoroIdElider::elideHeapAllocations(uint64_t FrameSize, Align FrameAlign) {
+  LLVMContext &C = FEI.ContainingFunction->getContext();
   BasicBlock::iterator InsertPt =
-      getFirstNonAllocaInTheEntryBlock(CoroIds.front()->getFunction())
-          ->getIterator();
+      getFirstNonAllocaInTheEntryBlock(FEI.ContainingFunction)->getIterator();
 
   // Replacing llvm.coro.alloc with false will suppress dynamic
   // allocation as it is expected for the frontend to generate the code that
@@ -161,7 +236,7 @@ void Lowerer::elideHeapAllocations(Function *F, uint64_t FrameSize,
   // is spilled into the coroutine frame and recreate the alignment information
   // here. Possibly we will need to do a mini SROA here and break the coroutine
   // frame into individual AllocaInst recreating the original alignment.
-  const DataLayout &DL = F->getParent()->getDataLayout();
+  const DataLayout &DL = FEI.ContainingFunction->getParent()->getDataLayout();
   auto FrameTy = ArrayType::get(Type::getInt8Ty(C), FrameSize);
   auto *Frame = new AllocaInst(FrameTy, DL.getAllocaAddrSpace(), "", InsertPt);
   Frame->setAlignment(FrameAlign);
@@ -178,8 +253,8 @@ void Lowerer::elideHeapAllocations(Function *F, uint64_t FrameSize,
   removeTailCallAttribute(Frame, AA);
 }
 
-bool Lowerer::hasEscapePath(const CoroBeginInst *CB,
-                            const SmallPtrSetImpl<BasicBlock *> &TIs) const {
+bool CoroIdElider::canCoroBeginEscape(
+    const CoroBeginInst *CB, const SmallPtrSetImpl<BasicBlock *> &TIs) const {
   const auto &It = DestroyAddr.find(CB);
   assert(It != DestroyAddr.end());
 
@@ -248,7 +323,7 @@ bool Lowerer::hasEscapePath(const CoroBeginInst *CB,
     // which means a escape path to normal terminator, it is reasonable to skip
     // it since coroutine frame doesn't change outside the coroutine body.
     if (isa<SwitchInst>(TI) &&
-        CoroSuspendSwitches.count(cast<SwitchInst>(TI))) {
+        FEI.CoroSuspendSwitches.count(cast<SwitchInst>(TI))) {
       Worklist.push_back(cast<SwitchInst>(TI)->getSuccessor(1));
       Worklist.push_back(cast<SwitchInst>(TI)->getSuccessor(2));
     } else
@@ -261,7 +336,7 @@ bool Lowerer::hasEscapePath(const CoroBeginInst *CB,
   return false;
 }
 
-bool Lowerer::shouldElide(Function *F, DominatorTree &DT) const {
+bool CoroIdElider::lifetimeEligibleForElide() const {
   // If no CoroAllocs, we cannot suppress allocation, so elision is not
   // possible.
   if (CoroAllocs.empty())
@@ -270,6 +345,7 @@ bool Lowerer::shouldElide(Function *F, DominatorTree &DT) const {
   // Check that for every coro.begin there is at least one coro.destroy directly
   // referencing the SSA value of that coro.begin along each
   // non-exceptional path.
+  //
   // If the value escaped, then coro.destroy would have been referencing a
   // memory location storing that value and not the virtual register.
 
@@ -277,7 +353,7 @@ bool Lowerer::shouldElide(Function *F, DominatorTree &DT) const {
   // First gather all of the terminators for the function.
   // Consider the final coro.suspend as the real terminator when the current
   // function is a coroutine.
-  for (BasicBlock &B : *F) {
+  for (BasicBlock &B : *FEI.ContainingFunction) {
     auto *TI = B.getTerminator();
 
     if (TI->getNumSuccessors() != 0 || isa<UnreachableInst>(TI))
@@ -287,91 +363,43 @@ bool Lowerer::shouldElide(Function *F, DominatorTree &DT) const {
   }
 
   // Filter out the coro.destroy that lie along exceptional paths.
-  SmallPtrSet<CoroBeginInst *, 8> ReferencedCoroBegins;
-  for (const auto &It : DestroyAddr) {
+  for (const auto *CB : CoroBegins) {
+    auto It = DestroyAddr.find(CB);
+
+    // FIXME: If we have not found any destroys for this coro.begin, we
+    // disqualify this elide.
+    if (It == DestroyAddr.end())
+      return false;
+
+    const auto &CorrespondingDestroyAddrs = It->second;
+
     // If every terminators is dominated by coro.destroy, we could know the
     // corresponding coro.begin wouldn't escape.
-    //
-    // Otherwise hasEscapePath would decide whether there is any paths from
+    auto DominatesTerminator = [&](auto *TI) {
+      return llvm::any_of(CorrespondingDestroyAddrs, [&](auto *Destroy) {
+        return DT.dominates(Destroy, TI->getTerminator());
+      });
+    };
+
+    if (llvm::all_of(Terminators, DominatesTerminator))
+      continue;
+
+    // Otherwise canCoroBeginEscape would decide whether there is any paths from
     // coro.begin to Terminators which not pass through any of the
-    // coro.destroys.
+    // coro.destroys. This is a slower analysis.
     //
-    // hasEscapePath is relatively slow, so we avoid to run it as much as
+    // canCoroBeginEscape is relatively slow, so we avoid to run it as much as
     // possible.
-    if (llvm::all_of(Terminators,
-                     [&](auto *TI) {
-                       return llvm::any_of(It.second, [&](auto *DA) {
-                         return DT.dominates(DA, TI->getTerminator());
-                       });
-                     }) ||
-        !hasEscapePath(It.first, Terminators))
-      ReferencedCoroBegins.insert(It.first);
+    if (canCoroBeginEscape(CB, Terminators))
+      return false;
   }
 
-  // If size of the set is the same as total number of coro.begin, that means we
-  // found a coro.free or coro.destroy referencing each coro.begin, so we can
-  // perform heap elision.
-  return ReferencedCoroBegins.size() == CoroBegins.size();
-}
-
-void Lowerer::collectPostSplitCoroIds(Function *F) {
-  CoroIds.clear();
-  CoroSuspendSwitches.clear();
-  for (auto &I : instructions(F)) {
-    if (auto *CII = dyn_cast<CoroIdInst>(&I))
-      if (CII->getInfo().isPostSplit())
-        // If it is the coroutine itself, don't touch it.
-        if (CII->getCoroutine() != CII->getFunction())
-          CoroIds.push_back(CII);
-
-    // Consider case like:
-    // %0 = call i8 @llvm.coro.suspend(...)
-    // switch i8 %0, label %suspend [i8 0, label %resume
-    //                              i8 1, label %cleanup]
-    // and collect the SwitchInsts which are used by escape analysis later.
-    if (auto *CSI = dyn_cast<CoroSuspendInst>(&I))
-      if (CSI->hasOneUse() && isa<SwitchInst>(CSI->use_begin()->getUser())) {
-        SwitchInst *SWI = cast<SwitchInst>(CSI->use_begin()->getUser());
-        if (SWI->getNumCases() == 2)
-          CoroSuspendSwitches.insert(SWI);
-      }
-  }
+  // We have checked all CoroBegins and their paths to the terminators without
+  // finding disqualifying code patterns, so we can perform heap allocations.
+  return true;
 }
 
-bool Lowerer::processCoroId(CoroIdInst *CoroId, AAResults &AA,
-                            DominatorTree &DT, OptimizationRemarkEmitter &ORE) {
-  CoroBegins.clear();
-  CoroAllocs.clear();
-  ResumeAddr.clear();
-  DestroyAddr.clear();
-
-  // Collect all coro.begin and coro.allocs associated with this coro.id.
-  for (User *U : CoroId->users()) {
-    if (auto *CB = dyn_cast<CoroBeginInst>(U))
-      CoroBegins.push_back(CB);
-    else if (auto *CA = dyn_cast<CoroAllocInst>(U))
-      CoroAllocs.push_back(CA);
-  }
-
-  // Collect all coro.subfn.addrs associated with coro.begin.
-  // Note, we only devirtualize the calls if their coro.subfn.addr refers to
-  // coro.begin directly. If we run into cases where this check is too
-  // conservative, we can consider relaxing the check.
-  for (CoroBeginInst *CB : CoroBegins) {
-    for (User *U : CB->users())
-      if (auto *II = dyn_cast<CoroSubFnInst>(U))
-        switch (II->getIndex()) {
-        case CoroSubFnInst::ResumeIndex:
-          ResumeAddr.push_back(II);
-          break;
-        case CoroSubFnInst::DestroyIndex:
-          DestroyAddr[CB].push_back(II);
-          break;
-        default:
-          llvm_unreachable("unexpected coro.subfn.addr constant");
-        }
-  }
-
+bool CoroIdElider::attemptElide() {
   // PostSplit coro.id refers to an array of subfunctions in its Info
   // argument.
   ConstantArray *Resumers = CoroId->getInfo().Resumers;
@@ -382,63 +410,55 @@ bool Lowerer::processCoroId(CoroIdInst *CoroId, AAResults &AA,
 
   replaceWithConstant(ResumeAddrConstant, ResumeAddr);
 
-  bool ShouldElide = shouldElide(CoroId->getFunction(), DT);
-  if (!ShouldElide)
-    ORE.emit([&]() {
-      if (auto FrameSizeAndAlign =
-              getFrameLayout(cast<Function>(ResumeAddrConstant)))
-        return OptimizationRemarkMissed(DEBUG_TYPE, "CoroElide", CoroId)
-               << "'" << ore::NV("callee", CoroId->getCoroutine()->getName())
-               << "' not elided in '"
-               << ore::NV("caller", CoroId->getFunction()->getName())
-               << "' (frame_size="
-               << ore::NV("frame_size", FrameSizeAndAlign->first) << ", align="
-               << ore::NV("align", FrameSizeAndAlign->second.value()) << ")";
-      else
-        return OptimizationRemarkMissed(DEBUG_TYPE, "CoroElide", CoroId)
-               << "'" << ore::NV("callee", CoroId->getCoroutine()->getName())
-               << "' not elided in '"
-               << ore::NV("caller", CoroId->getFunction()->getName())
-               << "' (frame_size=unknown, align=unknown)";
-    });
+  bool EligibleForElide = lifetimeEligibleForElide();
 
   auto *DestroyAddrConstant = Resumers->getAggregateElement(
-      ShouldElide ? CoroSubFnInst::CleanupIndex : CoroSubFnInst::DestroyIndex);
+      EligibleForElide ? CoroSubFnInst::CleanupIndex
+                       : CoroSubFnInst::DestroyIndex);
 
   for (auto &It : DestroyAddr)
     replaceWithConstant(DestroyAddrConstant, It.second);
 
-  if (ShouldElide) {
-    if (auto FrameSizeAndAlign =
-            getFrameLayout(cast<Function>(ResumeAddrConstant))) {
-      elideHeapAllocations(CoroId->getFunction(), FrameSizeAndAlign->first,
-                           FrameSizeAndAlign->second, AA);
-      coro::replaceCoroFree(CoroId, /*Elide=*/true);
-      NumOfCoroElided++;
+  auto FrameSizeAndAlign = getFrameLayout(cast<Function>(ResumeAddrConstant));
+
+  auto CallerFunctionName = FEI.ContainingFunction->getName();
+  auto CalleeCoroutineName = CoroId->getCoroutine()->getName();
+
+  if (EligibleForElide && FrameSizeAndAlign) {
+    elideHeapAllocations(FrameSizeAndAlign->first, FrameSizeAndAlign->second);
+    coro::replaceCoroFree(CoroId, /*Elide=*/true);
+    NumOfCoroElided++;
+
 #ifndef NDEBUG
       if (!CoroElideInfoOutputFilename.empty())
-        *getOrCreateLogFile()
-            << "Elide " << CoroId->getCoroutine()->getName() << " in "
-            << CoroId->getFunction()->getName() << "\n";
+        *getOrCreateLogFile() << "Elide " << CalleeCoroutineName << " in "
+                              << FEI.ContainingFunction->getName() << "\n";
 #endif
+
       ORE.emit([&]() {
         return OptimizationRemark(DEBUG_TYPE, "CoroElide", CoroId)
-               << "'" << ore::NV("callee", CoroId->getCoroutine()->getName())
-               << "' elided in '"
-               << ore::NV("caller", CoroId->getFunction()->getName())
+               << "'" << ore::NV("callee", CalleeCoroutineName)
+               << "' elided in '" << ore::NV("caller", CallerFunctionName)
                << "' (frame_size="
                << ore::NV("frame_size", FrameSizeAndAlign->first) << ", align="
                << ore::NV("align", FrameSizeAndAlign->second.value()) << ")";
       });
-    } else {
-      ORE.emit([&]() {
-        return OptimizationRemarkMissed(DEBUG_TYPE, "CoroElide", CoroId)
-               << "'" << ore::NV("callee", CoroId->getCoroutine()->getName())
-               << "' not elided in '"
-               << ore::NV("caller", CoroId->getFunction()->getName())
-               << "' (frame_size=unknown, align=unknown)";
-      });
-    }
+  } else {
+    ORE.emit([&]() {
+      auto Remark = OptimizationRemarkMissed(DEBUG_TYPE, "CoroElide", CoroId)
+                    << "'" << ore::NV("callee", CalleeCoroutineName)
+                    << "' not elided in '"
+                    << ore::NV("caller", CallerFunctionName);
+
+      if (FrameSizeAndAlign)
+        return Remark << "' (frame_size="
+                      << ore::NV("frame_size", FrameSizeAndAlign->first)
+                      << ", align="
+                      << ore::NV("align", FrameSizeAndAlign->second.value())
+                      << ")";
+      else
+        return Remark << "' (frame_size=unknown, align=unknown)";
+    });
   }
 
   return true;
@@ -453,11 +473,9 @@ PreservedAnalyses CoroElidePass::run(Function &F, FunctionAnalysisManager &AM) {
   if (!declaresCoroElideIntrinsics(M))
     return PreservedAnalyses::all();
 
-  Lowerer L(M);
-  L.CoroIds.clear();
-  L.collectPostSplitCoroIds(&F);
-  // If we did not find any coro.id, there is nothing to do.
-  if (L.CoroIds.empty())
+  FunctionElideInfo FEI{&F};
+  // Elide is not necessary if there's no coro.id within the function.
+  if (!FEI.hasCoroIds())
     return PreservedAnalyses::all();
 
   AAResults &AA = AM.getResult<AAManager>(F);
@@ -465,8 +483,10 @@ PreservedAnalyses CoroElidePass::run(Function &F, FunctionAnalysisManager &AM) {
   auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
 
   bool Changed = false;
-  for (auto *CII : L.CoroIds)
-    Changed |= L.processCoroId(CII, AA, DT, ORE);
+  for (auto *CII : FEI.getCoroIds()) {
+    CoroIdElider CIE(CII, FEI, AA, DT, ORE);
+    Changed |= CIE.attemptElide();
+  }
 
   return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
 }
-- 
GitLab


From fe0b7983a2f510cdede22cdf6c9227e32ded6a15 Mon Sep 17 00:00:00 2001
From: Joe Nash <joseph.nash@amd.com>
Date: Thu, 9 May 2024 11:37:56 -0400
Subject: [PATCH 0303/1206] [AMDGPU] Create AMDGPUMnemonicAlias tablegen class
 (#89288)

AMDGPUMnemonicAlias is a MnemonicAlias that inherits from
GCNPredicateControl, so that we can set predicates on the alias the same
way as Instructions.
Use AssemblerPredicate instead of Requires on aliases

NFC.
---
 llvm/lib/Target/AMDGPU/AMDGPU.td            |  8 +++--
 llvm/lib/Target/AMDGPU/BUFInstructions.td   | 12 ++++++--
 llvm/lib/Target/AMDGPU/DSInstructions.td    | 34 ++++++++++++---------
 llvm/lib/Target/AMDGPU/EXPInstructions.td   |  6 +++-
 llvm/lib/Target/AMDGPU/FLATInstructions.td  | 14 ++++++---
 llvm/lib/Target/AMDGPU/MIMGInstructions.td  | 21 ++++++++-----
 llvm/lib/Target/AMDGPU/SIInstrInfo.td       |  3 ++
 llvm/lib/Target/AMDGPU/SMInstructions.td    |  5 +--
 llvm/lib/Target/AMDGPU/SOPInstructions.td   | 28 ++++++++++++-----
 llvm/lib/Target/AMDGPU/VOP1Instructions.td  | 13 ++++----
 llvm/lib/Target/AMDGPU/VOP2Instructions.td  | 20 +++++++++---
 llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 15 ++++++---
 llvm/lib/Target/AMDGPU/VOPCInstructions.td  | 26 +++++-----------
 llvm/lib/Target/AMDGPU/VOPInstructions.td   | 10 ++++--
 14 files changed, 137 insertions(+), 78 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 8abe9920c02c..35b0cb439bfa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1918,7 +1918,8 @@ def Has16BitInsts : Predicate<"Subtarget->has16BitInsts()">,
 
 def HasTrue16BitInsts : Predicate<"Subtarget->hasTrue16BitInsts()">,
   AssemblerPredicate<(all_of FeatureTrue16BitInsts)>;
-def NotHasTrue16BitInsts : True16PredicateClass<"!Subtarget->hasTrue16BitInsts()">;
+def NotHasTrue16BitInsts : True16PredicateClass<"!Subtarget->hasTrue16BitInsts()">,
+  AssemblerPredicate<(all_of (not FeatureTrue16BitInsts))>;
 
 // Control use of True16 instructions. The real True16 instructions are
 // True16 instructions as they are defined in the ISA. Fake True16
@@ -1927,7 +1928,10 @@ def NotHasTrue16BitInsts : True16PredicateClass<"!Subtarget->hasTrue16BitInsts()
 def UseRealTrue16Insts : True16PredicateClass<"Subtarget->useRealTrue16Insts()">,
   AssemblerPredicate<(all_of FeatureTrue16BitInsts, FeatureRealTrue16Insts)>;
 def UseFakeTrue16Insts : True16PredicateClass<"Subtarget->hasTrue16BitInsts() && "
-                                              "!Subtarget->useRealTrue16Insts()">;
+                                              "!Subtarget->useRealTrue16Insts()">,
+  AssemblerPredicate<(all_of FeatureTrue16BitInsts)>;
+  // FIXME When we default to RealTrue16 instead of Fake, change the line as follows.
+  // AssemblerPredicate<(all_of FeatureTrue16BitInsts, (not FeatureRealTrue16Insts))>;
 
 def HasVOP3PInsts : Predicate<"Subtarget->hasVOP3PInsts()">,
   AssemblerPredicate<(all_of FeatureVOP3P)>;
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 8053d89aeb0a..8eaa113ac181 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -2456,13 +2456,19 @@ class get_BUF_ps<string name> {
 
 // gfx11 instruction that accept both old and new assembler name.
 class Mnem_gfx11_gfx12 <string mnemonic, string real_name> :
-  MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Plus]>;
+    AMDGPUMnemonicAlias<mnemonic, real_name> {
+  let AssemblerPredicate = isGFX11Plus;
+}
 
 class Mnem_gfx11 <string mnemonic, string real_name> :
-  MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Only]>;
+    AMDGPUMnemonicAlias<mnemonic, real_name> {
+  let AssemblerPredicate = isGFX11Only;
+}
 
 class Mnem_gfx12 <string mnemonic, string real_name> :
-  MnemonicAlias<mnemonic, real_name>, Requires<[isGFX12Plus]>;
+    AMDGPUMnemonicAlias<mnemonic, real_name> {
+  let AssemblerPredicate = isGFX12Plus;
+}
 
 multiclass MUBUF_Real_AllAddr_gfx11_Impl2<bits<8> op, string real_name> {
   defm _BOTHEN : MUBUF_Real_gfx11<op, real_name>;
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index d63f04ab6d4c..f2825c48fcec 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -1213,12 +1213,14 @@ class Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<bits<8> op, DS_Pseudo ps, int ef,
 
 multiclass DS_Real_gfx12<bits<8> op, string name = !tolower(NAME), bit needAlias = true> {
   defvar ps = !cast<DS_Pseudo>(NAME);
-  let AssemblerPredicate = isGFX12Plus, DecoderNamespace = "GFX12" in
-    def _gfx12 :
-      Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, ps, SIEncodingFamily.GFX12,
+  let AssemblerPredicate = isGFX12Plus in {
+    let DecoderNamespace = "GFX12" in
+      def _gfx12 :
+        Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, ps, SIEncodingFamily.GFX12,
                                                name, /*hasGDS=*/false>;
-  if !and(needAlias, !ne(ps.Mnemonic, name)) then
-    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX12Plus]>;
+    if !and(needAlias, !ne(ps.Mnemonic, name)) then
+      def : AMDGPUMnemonicAlias<ps.Mnemonic, name>;
+  } // End AssemblerPredicate
 }
 
 defm DS_MIN_F32           : DS_Real_gfx12<0x012, "ds_min_num_f32">;
@@ -1239,10 +1241,12 @@ defm DS_PK_ADD_BF16       : DS_Real_gfx12<0x09b>;
 defm DS_PK_ADD_RTN_BF16   : DS_Real_gfx12<0x0ab>;
 
 // New aliases added in GFX12 without renaming the instructions.
-def : MnemonicAlias<"ds_subrev_u32", "ds_rsub_u32">, Requires<[isGFX12Plus]>;
-def : MnemonicAlias<"ds_subrev_rtn_u32", "ds_rsub_rtn_u32">, Requires<[isGFX12Plus]>;
-def : MnemonicAlias<"ds_subrev_u64", "ds_rsub_u64">, Requires<[isGFX12Plus]>;
-def : MnemonicAlias<"ds_subrev_rtn_u64", "ds_rsub_rtn_u64">, Requires<[isGFX12Plus]>;
+let AssemblerPredicate = isGFX12Plus in {
+  def : AMDGPUMnemonicAlias<"ds_subrev_u32", "ds_rsub_u32">;
+  def : AMDGPUMnemonicAlias<"ds_subrev_rtn_u32", "ds_rsub_rtn_u32">;
+  def : AMDGPUMnemonicAlias<"ds_subrev_u64", "ds_rsub_u64">;
+  def : AMDGPUMnemonicAlias<"ds_subrev_rtn_u64", "ds_rsub_rtn_u64">;
+}
 
 //===----------------------------------------------------------------------===//
 // GFX11.
@@ -1250,12 +1254,14 @@ def : MnemonicAlias<"ds_subrev_rtn_u64", "ds_rsub_rtn_u64">, Requires<[isGFX12Pl
 
 multiclass DS_Real_gfx11<bits<8> op, string name = !tolower(NAME)> {
   defvar ps = !cast<DS_Pseudo>(NAME);
-  let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in
-    def _gfx11 :
-      Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, ps, SIEncodingFamily.GFX11,
+  let AssemblerPredicate = isGFX11Only in {
+    let DecoderNamespace = "GFX11" in
+      def _gfx11 :
+        Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, ps, SIEncodingFamily.GFX11,
                                                name>;
-  if !ne(ps.Mnemonic, name) then
-    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX11Only]>;
+    if !ne(ps.Mnemonic, name) then
+      def : AMDGPUMnemonicAlias<ps.Mnemonic, name>;
+  } // End AssemblerPredicate
 }
 
 multiclass DS_Real_gfx11_gfx12<bits<8> op, string name = !tolower(NAME)>
diff --git a/llvm/lib/Target/AMDGPU/EXPInstructions.td b/llvm/lib/Target/AMDGPU/EXPInstructions.td
index b73b83031af0..e02652e62a57 100644
--- a/llvm/lib/Target/AMDGPU/EXPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/EXPInstructions.td
@@ -117,12 +117,16 @@ multiclass EXP_Real_gfx11 {
 multiclass VEXPORT_Real_gfx12 {
   defvar ps = !cast<EXP_Pseudo>(NAME);
   def _gfx12 : EXP_Real_Row<ps, SIEncodingFamily.GFX12, "export">,
-    EXPe_Row, MnemonicAlias<"exp", "export">, Requires<[isGFX12Plus, HasExportInsts]> {
+    EXPe_Row {
     let AssemblerPredicate = isGFX12Only;
     let DecoderNamespace = "GFX12";
     let row = ps.row;
     let done = ps.done;
   }
+  def : AMDGPUMnemonicAlias<"exp", "export"> {
+    let AssemblerPredicate = isGFX12Plus;
+    let OtherPredicates = [HasExportInsts];
+  }
 }
 
 defm EXP          : EXP_Real_gfx11, VEXPORT_Real_gfx12;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 27d5616565f2..377d48a48e9b 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2357,7 +2357,9 @@ multiclass FLAT_Real_gfx11 <bits<7> op,
 multiclass FLAT_Aliases_gfx11<string name> {
   defvar ps = get_FLAT_ps<NAME>;
   if !ne(ps.Mnemonic, name) then
-    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX11Only]>;
+    def : AMDGPUMnemonicAlias<ps.Mnemonic, name> {
+      let AssemblerPredicate = isGFX11Only;
+    }
 }
 
 multiclass FLAT_Real_Base_gfx11<bits<7> op,
@@ -2544,10 +2546,12 @@ multiclass VFLAT_Real_gfx12 <bits<8> op, string name = get_FLAT_ps<NAME>.Mnemoni
 
 multiclass VFLAT_Aliases_gfx12<string name, string alias = name> {
   defvar ps = get_FLAT_ps<NAME>;
-  if !ne(ps.Mnemonic, name) then
-    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX12Only]>;
-  if !ne(alias, name) then
-    def : MnemonicAlias<alias, name>, Requires<[isGFX12Only]>;
+  let AssemblerPredicate = isGFX12Only in {
+    if !ne(ps.Mnemonic, name) then
+      def : AMDGPUMnemonicAlias<ps.Mnemonic, name>;
+    if !ne(alias, name) then
+      def : AMDGPUMnemonicAlias<alias, name>;
+  }
 }
 
 multiclass VFLAT_Real_Base_gfx12<bits<8> op,
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 23e8be0d5e45..9f6277f1257f 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -963,11 +963,10 @@ class VIMAGE_Atomic_gfx12<mimgopc op, string opcode, RegisterClass DataRC,
   let AsmString = opcode#" $vdata, "#AddrAsm#", $rsrc$dmask$dim$cpol$r128$a16$tfe";
 }
 
-class VIMAGE_Atomic_gfx12_Renamed<mimgopc op, string opcode, string renamed,
+class VIMAGE_Atomic_gfx12_Renamed<mimgopc op, string renamed,
                                   RegisterClass DataRC, int num_addrs,
                                   bit enableDisasm = 0>
-   : VIMAGE_Atomic_gfx12<op, renamed, DataRC, num_addrs, enableDisasm>,
-     MnemonicAlias<opcode, renamed>, Requires<[isGFX12Plus, HasImageInsts]>;
+  : VIMAGE_Atomic_gfx12<op, renamed, DataRC, num_addrs, enableDisasm>;
 
 multiclass MIMG_Atomic_Addr_Helper_m <mimgopc op, string asm,
                                       RegisterClass data_rc,
@@ -998,7 +997,7 @@ multiclass MIMG_Atomic_Addr_Helper_m <mimgopc op, string asm,
         if !empty(renamed) then
           def _V1_gfx12 : VIMAGE_Atomic_gfx12 <op, asm, data_rc, 1, enableDasm>;
         else
-          def _V1_gfx12 : VIMAGE_Atomic_gfx12_Renamed <op, asm, renamed, data_rc, 1, enableDasm>;
+          def _V1_gfx12 : VIMAGE_Atomic_gfx12_Renamed <op, renamed, data_rc, 1, enableDasm>;
       }
     }
     let VAddrDwords = 2 in {
@@ -1023,7 +1022,7 @@ multiclass MIMG_Atomic_Addr_Helper_m <mimgopc op, string asm,
         if !empty(renamed) then
           def _V2_gfx12 : VIMAGE_Atomic_gfx12 <op, asm, data_rc, 2, 0>;
         else
-          def _V2_gfx12 : VIMAGE_Atomic_gfx12_Renamed <op, asm, renamed, data_rc, 2, 0>;
+          def _V2_gfx12 : VIMAGE_Atomic_gfx12_Renamed <op, renamed, data_rc, 2, 0>;
       }
     }
     let VAddrDwords = 3 in {
@@ -1048,7 +1047,7 @@ multiclass MIMG_Atomic_Addr_Helper_m <mimgopc op, string asm,
         if !empty(renamed) then
           def _V3_gfx12 : VIMAGE_Atomic_gfx12 <op, asm, data_rc, 3, 0>;
         else
-          def _V3_gfx12 : VIMAGE_Atomic_gfx12_Renamed <op, asm, renamed, data_rc, 3, 0>;
+          def _V3_gfx12 : VIMAGE_Atomic_gfx12_Renamed <op, renamed, data_rc, 3, 0>;
       }
     }
     let VAddrDwords = 4 in {
@@ -1073,10 +1072,18 @@ multiclass MIMG_Atomic_Addr_Helper_m <mimgopc op, string asm,
         if !empty(renamed) then
           def _V4_gfx12 : VIMAGE_Atomic_gfx12 <op, asm, data_rc, 4, enableDasm>;
         else
-          def _V4_gfx12 : VIMAGE_Atomic_gfx12_Renamed <op, asm, renamed, data_rc, 4, enableDasm>;
+          def _V4_gfx12 : VIMAGE_Atomic_gfx12_Renamed <op, renamed, data_rc, 4, enableDasm>;
       }
     }
   }
+  if !and(op.HAS_GFX12, !not(!empty(renamed))) then
+    def : AMDGPUMnemonicAlias<asm, renamed> {
+      let AssemblerPredicate = isGFX12Plus;
+      let OtherPredicates = [HasImageInsts];
+      bit IsAtomicRet; // Unused
+      MIMGBaseOpcode BaseOpcode; // Unused
+      int VDataDwords; // Unused
+    }
 }
 
 multiclass MIMG_Atomic <mimgopc op, string asm, bit isCmpSwap = 0, bit isFP = 0,
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 6c16811b7dd6..0ed2f60ea66a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -16,6 +16,9 @@ class GCNPredicateControl : PredicateControl {
   Predicate VIAssemblerPredicate = isGFX8GFX9;
 }
 
+class AMDGPUMnemonicAlias<string From, string To, string VariantName = "">
+    : MnemonicAlias<From, To, VariantName>, GCNPredicateControl;
+
 // Except for the NONE field, this must be kept in sync with the
 // SIEncodingFamily enum in SIInstrInfo.cpp and the columns of the
 // getMCOpcodeGen table.
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index afc9da07bc96..40ba47f88771 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -1332,8 +1332,9 @@ multiclass SM_Real_Loads_gfx11<bits<8> op, string ps> {
   def _IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_IMM", opName>;
   def _SGPR_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR", opName>;
   def _SGPR_IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR_IMM", opName>;
-  def : MnemonicAlias<!cast<SM_Pseudo>(ps#"_IMM").Mnemonic, opName>,
-                      Requires<[isGFX11Plus]>;
+  def : AMDGPUMnemonicAlias<!cast<SM_Pseudo>(ps#"_IMM").Mnemonic, opName> {
+    let AssemblerPredicate = isGFX11Plus;
+  }
 }
 
 defm S_LOAD_B32  : SM_Real_Loads_gfx11<0x000, "S_LOAD_DWORD">;
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index b05d0018201b..394a5ed991bc 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -1975,7 +1975,9 @@ multiclass SOP1_Real_gfx11<bits<8> op, string name = !tolower(NAME)> {
   def _gfx11 : SOP1_Real<op, ps, name>,
                Select_gfx11<ps.PseudoInstr>;
   if !ne(ps.Mnemonic, name) then
-    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX11Only]>;
+    def : AMDGPUMnemonicAlias<ps.Mnemonic, name> {
+      let AssemblerPredicate = isGFX11Only;
+    }
 }
 
 multiclass SOP1_Real_gfx12<bits<8> op, string name = !tolower(NAME)> {
@@ -1983,7 +1985,9 @@ multiclass SOP1_Real_gfx12<bits<8> op, string name = !tolower(NAME)> {
   def _gfx12 : SOP1_Real<op, ps, name>,
                Select_gfx12<ps.PseudoInstr>;
   if !ne(ps.Mnemonic, name) then
-    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX12Plus]>;
+    def : AMDGPUMnemonicAlias<ps.Mnemonic, name> {
+      let AssemblerPredicate = isGFX12Plus;
+    }
 }
 
 multiclass SOP1_M0_Real_gfx12<bits<8> op> {
@@ -2208,7 +2212,9 @@ multiclass SOP2_Real_gfx12<bits<7> op, string name = !tolower(NAME)> {
   def _gfx12 : SOP2_Real32<op, ps, name>,
                Select_gfx12<ps.PseudoInstr>;
   if !ne(ps.Mnemonic, name) then
-    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX12Plus]>;
+    def : AMDGPUMnemonicAlias<ps.Mnemonic, name> {
+      let AssemblerPredicate = isGFX12Plus;
+    }
 }
 
 defm S_MINIMUM_F32 : SOP2_Real_gfx12<0x04f>;
@@ -2225,7 +2231,9 @@ multiclass SOP2_Real_gfx11<bits<7> op, string name = !tolower(NAME)> {
   def _gfx11 : SOP2_Real32<op, ps, name>,
                Select_gfx11<ps.PseudoInstr>;
   if !ne(ps.Mnemonic, name) then
-    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX11Only]>;
+    def : AMDGPUMnemonicAlias<ps.Mnemonic, name> {
+      let AssemblerPredicate = isGFX11Only;
+    }
 }
 
 multiclass SOP2_Real_gfx11_gfx12<bits<7> op, string name = !tolower(NAME)> :
@@ -2413,7 +2421,9 @@ multiclass SOPK_Real32_gfx12<bits<5> op, string name = !tolower(NAME)> {
   def _gfx12 : SOPK_Real32<op, ps, name>,
                Select_gfx12<ps.PseudoInstr>;
   if !ne(ps.Mnemonic, name) then
-    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX12Plus]>;
+    def : AMDGPUMnemonicAlias<ps.Mnemonic, name> {
+      let AssemblerPredicate = isGFX12Plus;
+    }
 }
 
 multiclass SOPK_Real32_gfx11<bits<5> op> {
@@ -2542,7 +2552,9 @@ multiclass SOPP_Real_32_gfx12<bits<7> op, string name = !tolower(NAME)> {
   def _gfx12 : SOPP_Real_32<op, ps, name>,
                Select_gfx12<ps.PseudoInstr>;
   if !ne(ps.Mnemonic, name) then
-    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX12Plus]>;
+    def : AMDGPUMnemonicAlias<ps.Mnemonic, name> {
+      let AssemblerPredicate = isGFX12Plus;
+    }
 }
 
 defm S_BARRIER_WAIT         : SOPP_Real_32_gfx12<0x014>;
@@ -2568,7 +2580,9 @@ multiclass SOPP_Real_32_gfx11<bits<7> op, string name = !tolower(NAME)> {
                Select_gfx11<ps.PseudoInstr>,
                SOPPRelaxTable<0, ps.KeyName, "_gfx11">;
   if !ne(ps.Mnemonic, name) then
-    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX11Only]>;
+    def : AMDGPUMnemonicAlias<ps.Mnemonic, name> {
+      let AssemblerPredicate = isGFX11Only;
+    }
 }
 
 multiclass SOPP_Real_64_gfx12<bits<7> op> {
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 0efb95d7d15a..012dca22eb4f 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -143,14 +143,14 @@ multiclass VOP1Inst <string opName, VOPProfile P,
       def _e64_dpp  : VOP3_DPP_Pseudo <opName, P>;
   } // End SubtargetPredicate = isGFX11Plus
 
-  def : MnemonicAlias<opName#"_e32", opName>, LetDummies;
-  def : MnemonicAlias<opName#"_e64", opName>, LetDummies;
+  def : LetDummies, AMDGPUMnemonicAlias<opName#"_e32", opName>;
+  def : LetDummies, AMDGPUMnemonicAlias<opName#"_e64", opName>;
 
   if P.HasExtSDWA then
-    def : MnemonicAlias<opName#"_sdwa", opName>, LetDummies;
+    def : LetDummies, AMDGPUMnemonicAlias<opName#"_sdwa", opName>;
 
   if P.HasExtDPP then
-    def : MnemonicAlias<opName#"_dpp", opName, AMDGPUAsmVariants.DPP>, LetDummies;
+    def : LetDummies, AMDGPUMnemonicAlias<opName#"_dpp", opName, AMDGPUAsmVariants.DPP>;
 }
 
 multiclass VOP1Inst_t16<string opName,
@@ -860,8 +860,9 @@ multiclass VOP1_Real_NO_VOP3_with_name_gfx11<bits<9> op, string opName,
               VOP1_Real_dpp_with_name<GFX11Gen, op, opName, asmName>,
               VOP1_Real_dpp8_with_name<GFX11Gen, op, opName, asmName>;
   defvar ps = !cast<VOP1_Pseudo>(opName#"_e32");
-  def gfx11_alias : MnemonicAlias<ps.Mnemonic, asmName>,
-                    Requires<[isGFX11Plus]>;
+  def gfx11_alias : AMDGPUMnemonicAlias<ps.Mnemonic, asmName> {
+    let AssemblerPredicate = isGFX11Plus;
+  }
 }
 
 multiclass VOP1_Real_NO_VOP3_with_name_gfx12<bits<9> op, string opName,
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index c001c5de81e0..d2af1753d550 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -1510,7 +1510,9 @@ multiclass VOP2_Real_NO_VOP3_with_name<GFXGen Gen, bits<6> op, string opName,
               VOP2_Real_dpp_with_name<Gen, op, opName, asmName>,
               VOP2_Real_dpp8_with_name<Gen, op, opName, asmName>;
   defvar ps = !cast<VOP2_Pseudo>(opName#"_e32");
-  def Gen.Suffix#"_alias" : MnemonicAlias<ps.Mnemonic, asmName>, Requires<[Gen.AssemblerPredicate]>;
+  def Gen.Suffix#"_alias" : AMDGPUMnemonicAlias<ps.Mnemonic, asmName> {
+    let AssemblerPredicate = Gen.AssemblerPredicate;
+  }
 }
 
 multiclass VOP2_Real_FULL_with_name<GFXGen Gen, bits<6> op, string opName,
@@ -1523,13 +1525,17 @@ multiclass VOP2_Real_NO_DPP_with_name<GFXGen Gen, bits<6> op, string opName,
   defm NAME : VOP2_Real_e32_with_name<Gen, op, opName, asmName>,
               VOP2_Real_e64_with_name<Gen, op, opName, asmName>;
   defvar ps = !cast<VOP2_Pseudo>(opName#"_e32");
-  def Gen.Suffix#"_alias" : MnemonicAlias<ps.Mnemonic, asmName>, Requires<[Gen.AssemblerPredicate]>;
+  def Gen.Suffix#"_alias" : AMDGPUMnemonicAlias<ps.Mnemonic, asmName> {
+    let AssemblerPredicate = Gen.AssemblerPredicate;
+  }
 }
 
 multiclass VOP2_Real_NO_DPP_with_alias<GFXGen Gen, bits<6> op, string alias> {
   defm NAME : VOP2_Real_e32<Gen, op>,
               VOP2_Real_e64<Gen, op>;
-  def Gen.Suffix#"_alias" : MnemonicAlias<alias, NAME>, Requires<[Gen.AssemblerPredicate]>;
+  def Gen.Suffix#"_alias" : AMDGPUMnemonicAlias<alias, NAME> {
+    let AssemblerPredicate = Gen.AssemblerPredicate;
+  }
 }
 
 //===----------------------------------------------------------------------===//
@@ -1550,7 +1556,9 @@ multiclass VOP2_Real_FULL_with_name_gfx12<bits<6> op, string opName,
 multiclass VOP2_Real_FULL_t16_with_name_gfx12<bits<6> op, string opName,
                                               string asmName, string alias> {
   defm NAME : VOP2_Real_FULL_with_name<GFX12Gen, op, opName, asmName>;
-  def _gfx12_2nd_alias : MnemonicAlias<alias, asmName>, Requires<[isGFX12Only]>;
+  def _gfx12_2nd_alias : AMDGPUMnemonicAlias<alias, asmName> {
+    let AssemblerPredicate = isGFX12Only;
+  }
 }
 
 multiclass VOP2_Real_NO_DPP_with_name_gfx12<bits<6> op, string opName,
@@ -1609,7 +1617,9 @@ multiclass VOP2_Real_NO_VOP3_with_name_gfx11<bits<6> op, string opName,
               VOP2_Real_dpp_with_name<GFX11Gen, op, opName, asmName>,
               VOP2_Real_dpp8_with_name<GFX11Gen, op, opName, asmName>;
   defvar ps = !cast<VOP2_Pseudo>(opName#"_e32");
-  def _gfx11_alias : MnemonicAlias<ps.Mnemonic, asmName>, Requires<[isGFX11Only]>;
+  def _gfx11_alias : AMDGPUMnemonicAlias<ps.Mnemonic, asmName> {
+    let AssemblerPredicate = isGFX11Only;
+  }
 }
 
 multiclass VOP2_Real_NO_DPP_with_name_gfx11<bits<6> op, string opName,
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 71ce36647e45..c3bdbbfc3846 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -814,8 +814,8 @@ let isCommutable = 1, isReMaterializable = 1 in {
   defm V_PK_MOV_B32 : VOP3PInst<"v_pk_mov_b32", VOP3P_Profile<VOP_V2I32_V2I32_V2I32, VOP3_PACKED>>;
 } // End isCommutable = 1, isReMaterializable = 1
 
-def : MnemonicAlias<"v_accvgpr_read",  "v_accvgpr_read_b32">;
-def : MnemonicAlias<"v_accvgpr_write", "v_accvgpr_write_b32">;
+def : AMDGPUMnemonicAlias<"v_accvgpr_read",  "v_accvgpr_read_b32">;
+def : AMDGPUMnemonicAlias<"v_accvgpr_write", "v_accvgpr_write_b32">;
 
 class VOPProfileWMMA<VOPProfile P, string Suffix, RegisterOperand _Src01RC64, bit _HasClamp, bit _HasOpSel> : VOP3P_Profile<P> {
   let DstRC = !if(!eq(Suffix, "_w32"), VDst_256, VDst_128);
@@ -1481,8 +1481,11 @@ multiclass VOP3P_Real_with_name<GFXGen Gen, bits<7> op,
   let AsmString = asmName # ps.AsmOperands in
     def Gen.Suffix :
       VOP3P_Real_Gen<!cast<VOP3P_Pseudo>(backing_ps_name), Gen, asmName>,
-      VOP3Pe_gfx11_gfx12<op, !cast<VOP3P_Pseudo>(backing_ps_name).Pfl>,
-      MnemonicAlias<ps.Mnemonic, asmName>, Requires<[Gen.AssemblerPredicate]>;
+      VOP3Pe_gfx11_gfx12<op, !cast<VOP3P_Pseudo>(backing_ps_name).Pfl>;
+
+  def : AMDGPUMnemonicAlias<ps.Mnemonic, asmName> {
+    let AssemblerPredicate = Gen.AssemblerPredicate;
+  }
 }
 
 multiclass VOP3P_Real_dpp<GFXGen Gen, bits<7> op, string backing_ps_name = NAME,
@@ -1661,7 +1664,9 @@ multiclass VOP3P_Real_SMFMAC<bits<7> op, string alias> {
     let AssemblerPredicate = isGFX940Plus;
     let DecoderNamespace = "GFX8";
   }
-  def : MnemonicAlias<alias, !cast<VOP3_Pseudo>(NAME#"_e64").Mnemonic>;
+  def : AMDGPUMnemonicAlias<alias, !cast<VOP3_Pseudo>(NAME#"_e64").Mnemonic> {
+    let AssemblerPredicate = isGFX940Plus;
+  }
 }
 
 let SubtargetPredicate = isGFX8GFX9 in {
diff --git a/llvm/lib/Target/AMDGPU/VOPCInstructions.td b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
index a0d666b39b2b..ddd6d8b074aa 100644
--- a/llvm/lib/Target/AMDGPU/VOPCInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
@@ -1389,17 +1389,12 @@ multiclass VOPC_Real_with_name<GFXGen Gen, bits<9> op, string OpName,
   defvar ps32 = !cast<VOPC_Pseudo>(OpName#"_e32");
   defvar ps64 = !cast<VOP3_Pseudo>(OpName#"_e64");
   let AssemblerPredicate = Gen.AssemblerPredicate in {
-    // MnemonicAlias and GCNPredicateControl both define the field Predicates,
-    // so GCNPredicateControl must come after MnemonicAlias because it contains
-    // the predicates we actually want.
-    def : MnemonicAlias<!if(!empty(pseudo_mnemonic), ps32.Mnemonic,
+    def : AMDGPUMnemonicAlias<!if(!empty(pseudo_mnemonic), ps32.Mnemonic,
                                                      pseudo_mnemonic),
-                        asm_name, ps32.AsmVariantName>,
-          GCNPredicateControl;
-    def : MnemonicAlias<!if(!empty(pseudo_mnemonic), ps64.Mnemonic,
+                              asm_name, ps32.AsmVariantName>;
+    def : AMDGPUMnemonicAlias<!if(!empty(pseudo_mnemonic), ps64.Mnemonic,
                                                      pseudo_mnemonic),
-                        asm_name, ps64.AsmVariantName>,
-          GCNPredicateControl;
+                              asm_name, ps64.AsmVariantName>;
 
     let DecoderNamespace = Gen.DecoderNamespace in {
       def _e32#Gen.Suffix :
@@ -1523,17 +1518,12 @@ multiclass VOPCX_Real_with_name<GFXGen Gen, bits<9> op, string OpName,
   defvar ps32 = !cast<VOPC_Pseudo>(OpName#"_nosdst_e32");
   defvar ps64 = !cast<VOP3_Pseudo>(OpName#"_nosdst_e64");
   let AssemblerPredicate = Gen.AssemblerPredicate in {
-    // MnemonicAlias and GCNPredicateControl both define the field Predicates,
-    // so GCNPredicateControl must come after MnemonicAlias because it contains
-    // the predicates we actually want.
-    def : MnemonicAlias<!if(!empty(pseudo_mnemonic), !subst("_nosdst", "", ps32.Mnemonic),
+    def : AMDGPUMnemonicAlias<!if(!empty(pseudo_mnemonic), !subst("_nosdst", "", ps32.Mnemonic),
                                                      pseudo_mnemonic),
-                        asm_name, ps32.AsmVariantName>,
-          GCNPredicateControl;
-    def : MnemonicAlias<!if(!empty(pseudo_mnemonic), !subst("_nosdst", "", ps64.Mnemonic),
+                              asm_name, ps32.AsmVariantName>;
+    def : AMDGPUMnemonicAlias<!if(!empty(pseudo_mnemonic), !subst("_nosdst", "", ps64.Mnemonic),
                                                      pseudo_mnemonic),
-                        asm_name, ps64.AsmVariantName>,
-          GCNPredicateControl;
+                              asm_name, ps64.AsmVariantName>;
 
     let DecoderNamespace = Gen.DecoderNamespace in {
       def _e32#Gen.Suffix
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index d974aacd7d45..f45ab9bf46db 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -1455,7 +1455,9 @@ multiclass VOP3_Real_with_name<GFXGen Gen, bits<10> op, string opName,
         VOP3e_gfx11_gfx12<op, ps.Pfl>;
     }
   }
-  def Gen.Suffix#"_VOP3_alias" : MnemonicAlias<ps.Mnemonic, asmName>, Requires<[Gen.AssemblerPredicate]>, LetDummies;
+  def Gen.Suffix#"_VOP3_alias" : LetDummies, AMDGPUMnemonicAlias<ps.Mnemonic, asmName> {
+    let AssemblerPredicate = Gen.AssemblerPredicate;
+  }
 }
 
 // for READLANE/WRITELANE
@@ -1628,8 +1630,10 @@ multiclass VOP3be_Real_with_name_gfx12<bits<10> op, string opName,
       IsSingle = !or(isSingle, ps.Pfl.IsSingle) in
     def _e64_gfx12 :
       VOP3_Real_Gen<ps, GFX12Gen, asmName>,
-      VOP3be_gfx11_gfx12<op, ps.Pfl>,
-      MnemonicAlias<ps.Mnemonic, asmName>, Requires<[isGFX12Only]>;
+      VOP3be_gfx11_gfx12<op, ps.Pfl>;
+  def : AMDGPUMnemonicAlias<ps.Mnemonic, asmName> {
+    let AssemblerPredicate = GFX12Gen.AssemblerPredicate;
+  }
 }
 
 multiclass VOP3_Realtriple_with_name_gfx12<bits<10> op, string opName,
-- 
GitLab


From b1cbf4a7c3174471690e20360c2adae173608ae0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Thu, 9 May 2024 16:51:32 +0100
Subject: [PATCH 0304/1206] [mlir][ArmSME] Add comments in
 tile-spills-and-fills.mlir (#91450)

* adds comments in tile-spills-and-fills.mlir
* adds comments in ArmSMEIntrinsicOps.td
* updates test in tile-spills-and-fills.mlir not to return 2D scalable
  vectors (e.g. vector<[4]x[4]xf32>) - that's not supported and not
  needed for that test
---
 .../Dialect/ArmSME/IR/ArmSMEIntrinsicOps.td   |  4 +-
 .../ArmSMEToLLVM/tile-spills-and-fills.mlir   | 48 +++++++++++++++++--
 2 files changed, 45 insertions(+), 7 deletions(-)

diff --git a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEIntrinsicOps.td b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEIntrinsicOps.td
index f051e03efbcd..0e38325f9891 100644
--- a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEIntrinsicOps.td
@@ -115,7 +115,7 @@ class ArmSME_IntrLoadStoreOp<string mnemonic>
                     /*immArgPositions=*/[2],
                     /*immArgAttrNames=*/["tile_id"]>;
 
-// Loads
+// Loads (from memory to ZA tile slice)
 class ArmSME_IntrLoadOp<string mnemonic>
     : ArmSME_IntrLoadStoreOp<mnemonic>,
       Arguments<(ins Arg<SVEPredicate, "Vector predicate">:$predicate,
@@ -134,7 +134,7 @@ def LLVM_aarch64_sme_ld1w_vert : ArmSME_IntrLoadOp<"ld1w.vert">;
 def LLVM_aarch64_sme_ld1d_vert : ArmSME_IntrLoadOp<"ld1d.vert">;
 def LLVM_aarch64_sme_ld1q_vert : ArmSME_IntrLoadOp<"ld1q.vert">;
 
-// Stores
+// Stores (ZA tile slice to memory)
 class ArmSME_IntrStoreOp<string mnemonic>
     : ArmSME_IntrLoadStoreOp<mnemonic>,
       Arguments<(ins Arg<SVEPredicate, "Vector predicate">:$predicate,
diff --git a/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir b/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
index 7a9e6b421575..a9c1a65a296f 100644
--- a/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
+++ b/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
@@ -72,17 +72,32 @@ func.func @use_too_many_tiles() {
 //  AFTER-LLVM-LOWERING-DAG: %[[C8:.*]] = arith.constant 8 : index
 //  AFTER-LLVM-LOWERING-DAG: %[[VSCALE:.*]] = vector.vscale
 //  AFTER-LLVM-LOWERING-DAG: %[[SVL_H:.*]] = arith.muli %[[VSCALE]], %[[C8]] : index
+
+///     0. Create an in-memory-tile
+///        Note: 16 is an in-memory tile ID, that is a tile ID >= 16
+
 //  AFTER-LLVM-LOWERING-DAG: %[[TILE_ALLOCA:.*]] = memref.alloca(%[[SVL_H]], %[[SVL_H]])
 // AFTER-LLVM-LOWERING-SAME:   {arm_sme.in_memory_tile_id = 16 : i32} : memref<?x?xi16>
 //
 //  AFTER-LLVM-LOWERING-NOT: scf.for
-//                           Note: 17 is the mask for the 32-bit tile 0.
+
+///     1. The following instruciton corresponds to %0 after tile allocation
+///        Note: 17 is the mask for the 32-bit tile 0.
+
 //      AFTER-LLVM-LOWERING: "arm_sme.intr.zero"() <{tile_mask = 17 : i32}>
 //
 //  AFTER-LLVM-LOWERING-NOT: scf.for
-//                           Note: 34 is the mask for the 32-bit tile 1.
+
+///     2. The following instruciton corresponds to %1 after tile allocation
+///        Note: 34 is the mask for the 32-bit tile 1.
+
 //      AFTER-LLVM-LOWERING: "arm_sme.intr.zero"() <{tile_mask = 34 : i32}>
-//
+
+///     3. swap(<in-memory-tile>, tile 0).
+///        This can be interpreted as spilling %0 (the 32-bit tile 0), so that
+///        %2 can be allocated a tile (16 bit tile 0). Note that this is
+///        swapping vector<[8]x[8]xi16> rather than vector<[4]x[4]xi32>.
+
 //      AFTER-LLVM-LOWERING: scf.for
 // AFTER-LLVM-LOWERING-SAME: %[[C0]] to %[[SVL_H]] step %[[C1]] {
 //      AFTER-LLVM-LOWERING:   %[[MEM_DESC:.*]] = builtin.unrealized_conversion_cast %[[TILE_ALLOCA]]
@@ -92,8 +107,15 @@ func.func @use_too_many_tiles() {
 // AFTER-LLVM-LOWERING-NEXT:   "arm_sme.intr.ld1h.horiz"({{.*}}, %[[SLICE_PTR]], {{.*}}) <{tile_id = 0 : i32}>
 // AFTER-LLVM-LOWERING-NEXT:   vector.store %[[SLICE]], %[[TILE_ALLOCA]]
 // AFTER-LLVM-LOWERING-NEXT: }
-//                           Note: 85 is the mask for the 16-bit tile 0.
+
+///     4. The following instruciton corresponds to %3 after tile allocation
+///        Note: 85 is the mask for the 16-bit tile 0.
+
 //      AFTER-LLVM-LOWERING: "arm_sme.intr.zero"() <{tile_mask = 85 : i32}>
+
+///     5.  swap(<inMemoryTile>, tile 0)
+///         This can be interpreted as restoring %0.
+
 //      AFTER-LLVM-LOWERING: scf.for
 // AFTER-LLVM-LOWERING-SAME: %[[C0]] to %[[SVL_H]] step %[[C1]] {
 //      AFTER-LLVM-LOWERING:   %[[MEM_DESC:.*]] = builtin.unrealized_conversion_cast %[[TILE_ALLOCA]]
@@ -116,7 +138,7 @@ func.func @very_excessive_spills(%memref : memref<?x?xf32>) -> vector<[4]x[4]xf3
   %tile = arm_sme.get_tile : vector<[4]x[4]xf32>
   %mask = vector.constant_mask [4] : vector<[4]xi1>
   %loadSlice = arm_sme.load_tile_slice %memref[%c0, %c0], %mask, %tile, %c0 : memref<?x?xf32>, vector<[4]xi1>, vector<[4]x[4]xf32>
-  return %loadSlice : vector<[4]x[4]xf32>
+  "test.some_use"(%loadSlice) : (vector<[4]x[4]xf32>) -> ()
 }
 // AFTER-TILE-ALLOC-LABEL: @very_excessive_spills
 //      AFTER-TILE-ALLOC: arm_sme.get_tile
@@ -133,22 +155,38 @@ func.func @very_excessive_spills(%memref : memref<?x?xf32>) -> vector<[4]x[4]xf3
 //  AFTER-LLVM-LOWERING-DAG: %[[TILE_ALLOCA:.*]] = memref.alloca(%[[SVL_S]], %[[SVL_S]])
 // AFTER-LLVM-LOWERING-SAME:   {arm_sme.in_memory_tile_id = 16 : i32} : memref<?x?xf32>
 //
+
+/// 1. Swap %useAllTiles and %tile - note that this will only swap one 32-bit
+///    tile (vector<[4]x[4]xf32>)
+
 //      AFTER-LLVM-LOWERING: scf.for
 // AFTER-LLVM-LOWERING-SAME: %[[C0]] to %[[SVL_S]] step %[[C1]] {
 //      AFTER-LLVM-LOWERING:   %[[MEM_DESC:.*]] = builtin.unrealized_conversion_cast %[[TILE_ALLOCA]]
 //      AFTER-LLVM-LOWERING:   %[[BASE_PTR:.*]] = llvm.extractvalue %[[MEM_DESC]][1]
 //      AFTER-LLVM-LOWERING:   %[[SLICE_PTR:.*]] = llvm.getelementptr %[[BASE_PTR]]
+// Read ZA tile slice -> vector
 //      AFTER-LLVM-LOWERING:   %[[SLICE:.*]] = "arm_sme.intr.read.horiz"{{.*}} <{tile_id = 0 : i32}>
+/// Load vector from memory -> ZA tile
 // AFTER-LLVM-LOWERING-NEXT:   "arm_sme.intr.ld1w.horiz"({{.*}}, %[[SLICE_PTR]], {{.*}}) <{tile_id = 0 : i32}>
+/// Store ZA tile slice in memory
 // AFTER-LLVM-LOWERING-NEXT:   vector.store %[[SLICE]], %[[TILE_ALLOCA]]
 // AFTER-LLVM-LOWERING-NEXT: }
+
+/// 2. Load into %tile
 //      AFTER-LLVM-LOWERING: "arm_sme.intr.ld1w.horiz"{{.*}} <{tile_id = 0 : i32}>
+
+/// 3. Swap %useAllTiles and %tile - note that this will only swap one 32-bit
+///    tile (vector<[4]x[4]xf32>)
+
 //      AFTER-LLVM-LOWERING: scf.for
 // AFTER-LLVM-LOWERING-SAME: %[[C0]] to %[[SVL_S]] step %[[C1]] {
 //      AFTER-LLVM-LOWERING:   %[[MEM_DESC:.*]] = builtin.unrealized_conversion_cast %[[TILE_ALLOCA]]
 //      AFTER-LLVM-LOWERING:   %[[BASE_PTR:.*]] = llvm.extractvalue %[[MEM_DESC]][1]
 //      AFTER-LLVM-LOWERING:   %[[SLICE_PTR:.*]] = llvm.getelementptr %[[BASE_PTR]]
+/// Read ZA tile slice -> vector
 //      AFTER-LLVM-LOWERING:   %[[SLICE:.*]] = "arm_sme.intr.read.horiz"{{.*}} <{tile_id = 0 : i32}>
+/// Load vector from memory -> ZA tile
 // AFTER-LLVM-LOWERING-NEXT:   "arm_sme.intr.ld1w.horiz"({{.*}}, %[[SLICE_PTR]], {{.*}}) <{tile_id = 0 : i32}>
+/// Store ZA tile slice in memory
 // AFTER-LLVM-LOWERING-NEXT:   vector.store %[[SLICE]], %[[TILE_ALLOCA]]
 // AFTER-LLVM-LOWERING-NEXT: }
-- 
GitLab


From aeab44d3861dbaac1eb8d68d818f432c86322759 Mon Sep 17 00:00:00 2001
From: Congcong Cai <congcongcai0907@163.com>
Date: Thu, 9 May 2024 23:59:11 +0800
Subject: [PATCH 0305/1206] [NFC][clang-tidy] remove magic-numbers-todo.cpp
 (#91577)

This XFAIL test is written in 4 years ago and still todo.
4 is already in DefaultIgnoredIntegerValues so I do not think this XFAIL
case can be passed.
---
 .../checkers/readability/magic-numbers-todo.cpp   | 15 ---------------
 1 file changed, 15 deletions(-)
 delete mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability/magic-numbers-todo.cpp

diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/magic-numbers-todo.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/magic-numbers-todo.cpp
deleted file mode 100644
index 99d9be262a89..000000000000
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/magic-numbers-todo.cpp
+++ /dev/null
@@ -1,15 +0,0 @@
-// RUN: %check_clang_tidy %s readability-magic-numbers %t --
-// XFAIL: *
-
-int ProcessSomething(int input);
-
-int DoWork()
-{
-  if (((int)4) > ProcessSomething(10))
-  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: 4 is a magic number; consider replacing it with a named constant [readability-magic-numbers]
-    return 0;
-
-   return 0;
-}
-
-
-- 
GitLab


From df21ee4c62e97239560485abdcc42aa340de65f7 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 9 May 2024 17:02:17 +0100
Subject: [PATCH 0306/1206] [DAG] Add clang-format off/on wrappers around
 compact switch handlers. NFC.

Avoids a problem identified in #90503
---
 .../SelectionDAG/SelectionDAGBuilder.cpp      | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 9ef02a792fd0..eac4297b89b5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6700,22 +6700,24 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
   case Intrinsic::roundeven:
   case Intrinsic::canonicalize: {
     unsigned Opcode;
+    // clang-format off
     switch (Intrinsic) {
     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
-    case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
-    case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
-    case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
-    case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
-    case Intrinsic::exp10:     Opcode = ISD::FEXP10;     break;
-    case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
-    case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
-    case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
-    case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
-    case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
-    case Intrinsic::round:     Opcode = ISD::FROUND;     break;
-    case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
+    case Intrinsic::sqrt:         Opcode = ISD::FSQRT;         break;
+    case Intrinsic::fabs:         Opcode = ISD::FABS;          break;
+    case Intrinsic::sin:          Opcode = ISD::FSIN;          break;
+    case Intrinsic::cos:          Opcode = ISD::FCOS;          break;
+    case Intrinsic::exp10:        Opcode = ISD::FEXP10;        break;
+    case Intrinsic::floor:        Opcode = ISD::FFLOOR;        break;
+    case Intrinsic::ceil:         Opcode = ISD::FCEIL;         break;
+    case Intrinsic::trunc:        Opcode = ISD::FTRUNC;        break;
+    case Intrinsic::rint:         Opcode = ISD::FRINT;         break;
+    case Intrinsic::nearbyint:    Opcode = ISD::FNEARBYINT;    break;
+    case Intrinsic::round:        Opcode = ISD::FROUND;        break;
+    case Intrinsic::roundeven:    Opcode = ISD::FROUNDEVEN;    break;
     case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
     }
+    // clang-format on
 
     setValue(&I, DAG.getNode(Opcode, sdl,
                              getValue(I.getArgOperand(0)).getValueType(),
@@ -6727,6 +6729,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
   case Intrinsic::lrint:
   case Intrinsic::llrint: {
     unsigned Opcode;
+    // clang-format off
     switch (Intrinsic) {
     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
     case Intrinsic::lround:  Opcode = ISD::LROUND;  break;
@@ -6734,6 +6737,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     case Intrinsic::lrint:   Opcode = ISD::LRINT;   break;
     case Intrinsic::llrint:  Opcode = ISD::LLRINT;  break;
     }
+    // clang-format on
 
     EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
     setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
-- 
GitLab


From eb177803bfc31ea8ce784c6cf5a881a8bbce0bce Mon Sep 17 00:00:00 2001
From: Yinying Li <yinyingli@google.com>
Date: Thu, 9 May 2024 12:09:40 -0400
Subject: [PATCH 0307/1206] [mlir][sparse] Change sparse_tensor.print format
 (#91528)

1. Remove the trailing comma for the last element of memref and add
closing parenthesis.
2. Change integration tests to use the new format.
---
 .../Transforms/SparseTensorRewriting.cpp      |  12 +-
 .../Dialect/SparseTensor/CPU/block.mlir       |  18 +-
 .../SparseTensor/CPU/block_majors.mlir        |  24 +--
 .../SparseTensor/CPU/concatenate_dim_0.mlir   |  20 +--
 .../CPU/concatenate_dim_0_permute.mlir        |  20 +--
 .../SparseTensor/CPU/concatenate_dim_1.mlir   |  20 +--
 .../CPU/concatenate_dim_1_permute.mlir        |  20 +--
 .../SparseTensor/CPU/dense_output.mlir        |   2 +-
 .../SparseTensor/CPU/dense_output_bf16.mlir   |   2 +-
 .../SparseTensor/CPU/dense_output_f16.mlir    |   2 +-
 .../SparseTensor/CPU/dual_sparse_conv_2d.mlir |  28 +--
 .../Dialect/SparseTensor/CPU/sparse_abs.mlir  |  12 +-
 .../SparseTensor/CPU/sparse_binary.mlir       | 106 +++++------
 .../SparseTensor/CPU/sparse_block3d.mlir      |  24 +--
 .../Dialect/SparseTensor/CPU/sparse_cmp.mlir  |  20 +--
 .../CPU/sparse_collapse_shape.mlir            |  52 +++---
 .../SparseTensor/CPU/sparse_complex32.mlir    |  12 +-
 .../SparseTensor/CPU/sparse_complex64.mlir    |  12 +-
 .../SparseTensor/CPU/sparse_complex_ops.mlir  |  42 ++---
 .../CPU/sparse_constant_to_sparse_tensor.mlir |  10 +-
 .../CPU/sparse_conv_1d_nwc_wcf.mlir           |  24 +--
 .../SparseTensor/CPU/sparse_conv_2d.mlir      |  38 ++--
 .../CPU/sparse_conv_2d_nhwc_hwcf.mlir         |  38 ++--
 .../SparseTensor/CPU/sparse_conv_3d.mlir      |  40 ++---
 .../CPU/sparse_conv_3d_ndhwc_dhwcf.mlir       |  36 ++--
 .../SparseTensor/CPU/sparse_conversion.mlir   | 168 +++++++++---------
 .../CPU/sparse_conversion_block.mlir          |  24 +--
 .../CPU/sparse_conversion_dyn.mlir            |  60 +++----
 .../CPU/sparse_conversion_ptr.mlir            |  56 +++---
 .../SparseTensor/CPU/sparse_coo_test.mlir     |   8 +-
 .../Dialect/SparseTensor/CPU/sparse_dot.mlir  |  12 +-
 .../Dialect/SparseTensor/CPU/sparse_ds.mlir   |  20 +--
 .../SparseTensor/CPU/sparse_empty.mlir        |  24 +--
 .../SparseTensor/CPU/sparse_expand.mlir       |   6 +-
 .../SparseTensor/CPU/sparse_expand_shape.mlir |  76 ++++----
 .../CPU/sparse_filter_conv2d.mlir             |  10 +-
 .../SparseTensor/CPU/sparse_index.mlir        |  74 ++++----
 .../SparseTensor/CPU/sparse_insert_1d.mlir    |  12 +-
 .../SparseTensor/CPU/sparse_insert_2d.mlir    |  32 ++--
 .../SparseTensor/CPU/sparse_insert_3d.mlir    |  36 ++--
 .../SparseTensor/CPU/sparse_loose.mlir        |   9 +-
 .../SparseTensor/CPU/sparse_matmul.mlir       | 112 ++++++------
 .../SparseTensor/CPU/sparse_matmul_slice.mlir |  34 ++--
 .../SparseTensor/CPU/sparse_matrix_ops.mlir   |  60 +++----
 .../SparseTensor/CPU/sparse_out_mult_elt.mlir |  10 +-
 .../CPU/sparse_out_reduction.mlir             |  10 +-
 .../SparseTensor/CPU/sparse_out_simple.mlir   |  10 +-
 .../SparseTensor/CPU/sparse_pack_d.mlir       |  26 +--
 .../SparseTensor/CPU/sparse_pooling_nhwc.mlir |  18 +-
 .../SparseTensor/CPU/sparse_print.mlir        | 102 +++++------
 .../SparseTensor/CPU/sparse_print_3d.mlir     |  10 +-
 .../SparseTensor/CPU/sparse_re_im.mlir        |  12 +-
 .../CPU/sparse_reduce_custom.mlir             |  24 +--
 .../CPU/sparse_reduce_custom_prod.mlir        |  24 +--
 .../SparseTensor/CPU/sparse_reshape.mlir      |  30 ++--
 .../CPU/sparse_sampled_mm_fusion.mlir         |  20 +--
 .../SparseTensor/CPU/sparse_scale.mlir        |   6 +-
 .../SparseTensor/CPU/sparse_scf_nested.mlir   |  28 +--
 .../SparseTensor/CPU/sparse_select.mlir       |  24 +--
 .../CPU/sparse_semiring_select.mlir           |  10 +-
 .../Dialect/SparseTensor/CPU/sparse_sign.mlir |   6 +-
 .../SparseTensor/CPU/sparse_sorted_coo.mlir   |  52 +++---
 .../SparseTensor/CPU/sparse_storage.mlir      |  46 ++---
 .../Dialect/SparseTensor/CPU/sparse_tanh.mlir |   6 +-
 .../SparseTensor/CPU/sparse_tensor_mul.mlir   |  14 +-
 .../SparseTensor/CPU/sparse_tensor_ops.mlir   |  24 +--
 .../SparseTensor/CPU/sparse_transpose.mlir    |  20 +--
 .../CPU/sparse_transpose_coo.mlir             |  16 +-
 .../SparseTensor/CPU/sparse_unary.mlir        |  44 ++---
 .../SparseTensor/CPU/sparse_vector_ops.mlir   |  38 ++--
 .../GPU/CUDA/sparse-gemm-lib.mlir             |   6 +-
 .../GPU/CUDA/sparse-sampled-matmul-lib.mlir   |  12 +-
 .../GPU/CUDA/sparse-sddmm-lib.mlir            |  12 +-
 73 files changed, 1068 insertions(+), 1059 deletions(-)

diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index 025fd3331ba8..da635c257888 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -830,11 +830,17 @@ private:
                                          vector::PrintPunctuation::Comma);
         rewriter.create<vector::PrintOp>(loc, imag,
                                          vector::PrintPunctuation::Close);
-        rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Comma);
       } else {
-        rewriter.create<vector::PrintOp>(loc, val,
-                                         vector::PrintPunctuation::Comma);
+        rewriter.create<vector::PrintOp>(
+            loc, val, vector::PrintPunctuation::NoPunctuation);
       }
+      // Terminating comma (except at end).
+      auto bound = rewriter.create<arith::AddIOp>(loc, idxs.back(), step);
+      Value cond = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ne,
+                                                  bound, size);
+      scf::IfOp ifOp = rewriter.create<scf::IfOp>(loc, cond, /*else*/ false);
+      rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
+      rewriter.create<vector::PrintOp>(loc, vector::PrintPunctuation::Comma);
     }
     idxs.pop_back();
     rewriter.setInsertionPointAfter(forOp);
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
index f79e7e68f382..ab4fd0e30d65 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
@@ -93,9 +93,9 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 4, 6 )
     // CHECK-NEXT: lvl = ( 2, 3, 2, 2 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 1,
-    // CHECK-NEXT: values : ( 1, 2, 0, 3, 4, 0, 0, 5, 6, 7, 8, 0,
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 1 )
+    // CHECK-NEXT: values : ( 1, 2, 0, 3, 4, 0, 0, 5, 6, 7, 8, 0 )
     // CHECK-NEXT: ----
     sparse_tensor.print %A : tensor<?x?xf64, #BSR>
 
@@ -103,9 +103,9 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 2, 3, 2, 2 )
     // CHECK-NEXT: lvl = ( 2, 3, 2, 2 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 1
-    // CHECK-NEXT: values : ( 1, 2, 0, 3, 4, 0, 0, 5, 6, 7, 8, 0,
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 1 )
+    // CHECK-NEXT: values : ( 1, 2, 0, 3, 4, 0, 0, 5, 6, 7, 8, 0 )
     // CHECK-NEXT: ----
     %t1 = sparse_tensor.reinterpret_map %A : tensor<?x?xf64, #BSR>
                                           to tensor<?x?x2x2xf64, #DSDD>
@@ -115,9 +115,9 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 4, 6 )
     // CHECK-NEXT: lvl = ( 2, 3, 2, 2 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 1,
-    // CHECK-NEXT: values : ( 3, 6, 0, 9, 12, 0, 0, 15, 18, 21, 24, 0,
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 1 )
+    // CHECK-NEXT: values : ( 3, 6, 0, 9, 12, 0, 0, 15, 18, 21, 24, 0 )
     // CHECK-NEXT: ----
     %As = call @scale(%A) : (tensor<?x?xf64, #BSR>) -> (tensor<?x?xf64, #BSR>)
     sparse_tensor.print %As : tensor<?x?xf64, #BSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir
index 3534e7d15207..caa0d6a71ed3 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir
@@ -108,9 +108,9 @@ module {
   // CHECK-NEXT: nse = 24
   // CHECK-NEXT: dim = ( 6, 16 )
   // CHECK-NEXT: lvl = ( 2, 4, 3, 4 )
-  // CHECK-NEXT: pos[1] : ( 0, 1, 2,
-  // CHECK-NEXT: crd[1] : ( 0, 2,
-  // CHECK-NEXT: values : ( 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7,
+  // CHECK-NEXT: pos[1] : ( 0, 1, 2 )
+  // CHECK-NEXT: crd[1] : ( 0, 2 )
+  // CHECK-NEXT: values : ( 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7 )
   // CHECK-NEXT: ----
   //
   func.func @foo1() {
@@ -134,9 +134,9 @@ module {
   // CHECK-NEXT: nse = 24
   // CHECK-NEXT: dim = ( 6, 16 )
   // CHECK-NEXT: lvl = ( 2, 4, 4, 3 )
-  // CHECK-NEXT: pos[1] : ( 0, 1, 2,
-  // CHECK-NEXT: crd[1] : ( 0, 2,
-  // CHECK-NEXT: values : ( 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 7,
+  // CHECK-NEXT: pos[1] : ( 0, 1, 2 )
+  // CHECK-NEXT: crd[1] : ( 0, 2 )
+  // CHECK-NEXT: values : ( 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 7 )
   // CHECK-NEXT: ----
   //
   func.func @foo2() {
@@ -160,9 +160,9 @@ module {
   // CHECK-NEXT: nse = 24
   // CHECK-NEXT: dim = ( 6, 16 )
   // CHECK-NEXT: lvl = ( 4, 2, 3, 4 )
-  // CHECK-NEXT: pos[1] : ( 0, 1, 1, 2, 2,
-  // CHECK-NEXT: crd[1] : ( 0, 1,
-  // CHECK-NEXT: values : ( 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7,
+  // CHECK-NEXT: pos[1] : ( 0, 1, 1, 2, 2 )
+  // CHECK-NEXT: crd[1] : ( 0, 1 )
+  // CHECK-NEXT: values : ( 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7 )
   // CHECK-NEXT: ----
   //
   func.func @foo3() {
@@ -186,9 +186,9 @@ module {
   // CHECK-NEXT: nse = 24
   // CHECK-NEXT: dim = ( 6, 16 )
   // CHECK-NEXT: lvl = ( 4, 2, 4, 3 )
-  // CHECK-NEXT: pos[1] : ( 0, 1, 1, 2, 2,
-  // CHECK-NEXT: crd[1] : ( 0, 1,
-  // CHECK-NEXT: values : ( 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 7,
+  // CHECK-NEXT: pos[1] : ( 0, 1, 1, 2, 2 )
+  // CHECK-NEXT: crd[1] : ( 0, 1 )
+  // CHECK-NEXT: values : ( 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 7 )
   // CHECK-NEXT: ----
   //
   func.func @foo4() {
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
index 6a4902057362..7edb76cc8045 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
@@ -111,11 +111,11 @@ module {
     // CHECK-NEXT: nse = 18
     // CHECK-NEXT: dim = ( 9, 4 )
     // CHECK-NEXT: lvl = ( 9, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 9,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8,
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 7, 10, 12, 13, 16, 18,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 1, 0, 2, 3, 1, 0, 1, 2, 2, 3, 1, 0, 1, 2, 0, 1,
-    // CHECK-NEXT: values : ( 1, 3, 2, 1, 1, 1, 0.5, 1, 5, 2, 1.5, 1, 3.5, 1, 5, 2, 1, 0.5,
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 7, 10, 12, 13, 16, 18 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 1, 0, 2, 3, 1, 0, 1, 2, 2, 3, 1, 0, 1, 2, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 3, 2, 1, 1, 1, 0.5, 1, 5, 2, 1.5, 1, 3.5, 1, 5, 2, 1, 0.5 )
     // CHECK-NEXT: ----
     //
     %0 = call @concat_sparse_sparse(%sm24cc, %sm34cd, %sm44dc)
@@ -142,11 +142,11 @@ module {
     // CHECK-NEXT: nse = 18
     // CHECK-NEXT: dim = ( 9, 4 )
     // CHECK-NEXT: lvl = ( 9, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 9,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8,
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 7, 10, 12, 13, 16, 18,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 1, 0, 2, 3, 1, 0, 1, 2, 2, 3, 1, 0, 1, 2, 0, 1,
-    // CHECK-NEXT: values : ( 1, 3, 2, 1, 1, 1, 0.5, 1, 5, 2, 1.5, 1, 3.5, 1, 5, 2, 1, 0.5,
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 7, 10, 12, 13, 16, 18 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 1, 0, 2, 3, 1, 0, 1, 2, 2, 3, 1, 0, 1, 2, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 3, 2, 1, 1, 1, 0.5, 1, 5, 2, 1.5, 1, 3.5, 1, 5, 2, 1, 0.5 )
     // CHECK-NEXT: ----
     //
     %2 = call @concat_mix_sparse(%m24, %sm34cd, %sm44dc)
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
index 9c9b0e3330c9..d17e110e2c2d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
@@ -144,11 +144,11 @@ module {
     // CHECK-NEXT: nse = 18
     // CHECK-NEXT: dim = ( 9, 4 )
     // CHECK-NEXT: lvl = ( 4, 9 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 5, 11, 16, 18
-    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 7, 8, 1, 3, 4, 6, 7, 8, 0, 2, 4, 5, 7, 2, 5
-    // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 2, 0.5, 5, 3.5, 5, 0.5, 3, 1, 2, 1.5, 2, 1, 1
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 5, 11, 16, 18 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 7, 8, 1, 3, 4, 6, 7, 8, 0, 2, 4, 5, 7, 2, 5 )
+    // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 2, 0.5, 5, 3.5, 5, 0.5, 3, 1, 2, 1.5, 2, 1, 1 )
     // CHECK-NEXT: ----
     //
     %4 = call @concat_sparse_sparse_perm(%sm24ccp, %sm34cd, %sm44dc)
@@ -173,11 +173,11 @@ module {
     // CHECK-NEXT: nse = 18
     // CHECK-NEXT: dim = ( 9, 4 )
     // CHECK-NEXT: lvl = ( 9, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 9
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 7, 10, 12, 13, 16, 18
-    // CHECK-NEXT: crd[1] : ( 0, 2, 1, 0, 2, 3, 1, 0, 1, 2, 2, 3, 1, 0, 1, 2, 0, 1
-    // CHECK-NEXT: values : ( 1, 3, 2, 1, 1, 1, 0.5, 1, 5, 2, 1.5, 1, 3.5, 1, 5, 2, 1, 0.5
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 7, 10, 12, 13, 16, 18 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 1, 0, 2, 3, 1, 0, 1, 2, 2, 3, 1, 0, 1, 2, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 3, 2, 1, 1, 1, 0.5, 1, 5, 2, 1.5, 1, 3.5, 1, 5, 2, 1, 0.5 )
     // CHECK-NEXT: ----
     //
     %6 = call @concat_mix_sparse_perm(%m24, %sm34cdp, %sm44dc)
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
index ae067bf18527..c2a4e95e7922 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
@@ -116,11 +116,11 @@ module {
     // CHECK-NEXT: nse = 18
     // CHECK-NEXT: dim = ( 4, 9 )
     // CHECK-NEXT: lvl = ( 4, 9 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 5, 9, 14, 18
-    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 7, 8, 0, 2, 4, 6, 1, 4, 5, 6, 7, 2, 3, 5, 6
-    // CHECK-NEXT: values : ( 1, 1, 1, 1.5, 1, 3.1, 1, 0.5, 3.5, 2, 1, 1, 5, 2, 5, 2, 1, 0.5
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 5, 9, 14, 18 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 7, 8, 0, 2, 4, 6, 1, 4, 5, 6, 7, 2, 3, 5, 6 )
+    // CHECK-NEXT: values : ( 1, 1, 1, 1.5, 1, 3.1, 1, 0.5, 3.5, 2, 1, 1, 5, 2, 5, 2, 1, 0.5 )
     // CHECK-NEXT: ----
     //
     %8 = call @concat_sparse_sparse_dim1(%sm42cc, %sm43cd, %sm44dc)
@@ -140,11 +140,11 @@ module {
     // CHECK-NEXT: nse = 18
     // CHECK-NEXT: dim = ( 4, 9 )
     // CHECK-NEXT: lvl = ( 4, 9 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 5, 9, 14, 18
-    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 7, 8, 0, 2, 4, 6, 1, 4, 5, 6, 7, 2, 3, 5, 6
-    // CHECK-NEXT: values : ( 1, 1, 1, 1.5, 1, 3.1, 1, 0.5, 3.5, 2, 1, 1, 5, 2, 5, 2, 1, 0.5
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 5, 9, 14, 18 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 7, 8, 0, 2, 4, 6, 1, 4, 5, 6, 7, 2, 3, 5, 6 )
+    // CHECK-NEXT: values : ( 1, 1, 1, 1.5, 1, 3.1, 1, 0.5, 3.5, 2, 1, 1, 5, 2, 5, 2, 1, 0.5 )
     // CHECK-NEXT: ----
     //
     %10 = call @concat_mix_sparse_dim1(%m42, %sm43cd, %sm44dc)
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
index ce746f27c4d8..8fe7e08a66d3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
@@ -130,11 +130,11 @@ module {
     // CHECK-NEXT: nse = 18
     // CHECK-NEXT: dim = ( 4, 9 )
     // CHECK-NEXT: lvl = ( 9, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 9
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 7, 10, 12, 15, 17, 18
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 3, 3, 0, 1, 2, 2, 3, 1, 2, 3, 0, 2, 0
-    // CHECK-NEXT: values : ( 1, 3.1, 2, 1, 1, 5, 2, 1, 0.5, 1, 1, 1, 3.5, 5, 0.5, 1.5, 2, 1
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 7, 10, 12, 15, 17, 18 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 3, 3, 0, 1, 2, 2, 3, 1, 2, 3, 0, 2, 0 )
+    // CHECK-NEXT: values : ( 1, 3.1, 2, 1, 1, 5, 2, 1, 0.5, 1, 1, 1, 3.5, 5, 0.5, 1.5, 2, 1 )
     // CHECK-NEXT: ----
     //
     %12 = call @concat_sparse_sparse_perm_dim1(%sm42ccp, %sm43cd, %sm44dc)
@@ -154,11 +154,11 @@ module {
     // CHECK-NEXT: nse = 18
     // CHECK-NEXT: dim = ( 4, 9 )
     // CHECK-NEXT: lvl = ( 4, 9 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 5, 9, 14, 18
-    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 7, 8, 0, 2, 4, 6, 1, 4, 5, 6, 7, 2, 3, 5, 6
-    // CHECK-NEXT: values : ( 1, 1, 1, 1.5, 1, 3.1, 1, 0.5, 3.5, 2, 1, 1, 5, 2, 5, 2, 1, 0.5
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 5, 9, 14, 18 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 7, 8, 0, 2, 4, 6, 1, 4, 5, 6, 7, 2, 3, 5, 6 )
+    // CHECK-NEXT: values : ( 1, 1, 1, 1.5, 1, 3.1, 1, 0.5, 3.5, 2, 1, 1, 5, 2, 5, 2, 1, 0.5 )
     // CHECK-NEXT: ----
     //
     %14 = call @concat_mix_sparse_perm_dim1(%m42, %sm43cdp, %sm44dc)
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
index b2bbc64f1688..d00d4c87f9bd 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
@@ -108,7 +108,7 @@ module {
     // CHECK-NEXT: nse = 25
     // CHECK-NEXT: dim = ( 5, 5 )
     // CHECK-NEXT: lvl = ( 5, 5 )
-    // CHECK-NEXT: values : ( 2, 0, 0, 2.8, 0, 0, 4, 0, 0, 5, 0, 0, 6, 0, 0, 8.2, 0, 0, 8, 0, 0, 10.4, 0, 0, 10,
+    // CHECK-NEXT: values : ( 2, 0, 0, 2.8, 0, 0, 4, 0, 0, 5, 0, 0, 6, 0, 0, 8.2, 0, 0, 8, 0, 0, 10.4, 0, 0, 10 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?x?xf64, #DenseMatrix>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
index ca9df03c69ee..49f182ddb1d4 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
@@ -95,7 +95,7 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: values : ( 1, 11, 0, 2, 13, 0, 0, 0, 0, 0, 14, 3, 0, 0, 0, 0, 15, 4, 16, 0, 5, 6, 0, 0, 0, 0, 0, 0, 7, 8, 0, 9,
+    // CHECK-NEXT: values : ( 1, 11, 0, 2, 13, 0, 0, 0, 0, 0, 14, 3, 0, 0, 0, 0, 15, 4, 16, 0, 5, 6, 0, 0, 0, 0, 0, 0, 7, 8, 0, 9 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?xbf16, #DenseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
index 4f5e6ddd48d8..cc2a3733c863 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
@@ -96,7 +96,7 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: values : ( 1, 11, 0, 2, 13, 0, 0, 0, 0, 0, 14, 3, 0, 0, 0, 0, 15, 4, 16, 0, 5, 6, 0, 0, 0, 0, 0, 0, 7, 8, 0, 9,
+    // CHECK-NEXT: values : ( 1, 11, 0, 2, 13, 0, 0, 0, 0, 0, 14, 3, 0, 0, 0, 0, 15, 4, 16, 0, 5, 6, 0, 0, 0, 0, 0, 0, 7, 8, 0, 9 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?xf16, #DenseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir
index c645ca656720..f33a3abc7a5f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir
@@ -161,11 +161,11 @@ module {
     // CHECK-NEXT: nse = 36
     // CHECK-NEXT: dim = ( 6, 6 )
     // CHECK-NEXT: lvl = ( 6, 6 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %2 : tensor<6x6xi32, #DCSR>
@@ -177,9 +177,9 @@ module {
     // CHECK-NEXT: nse = 36
     // CHECK-NEXT: dim = ( 6, 6 )
     // CHECK-NEXT: lvl = ( 6, 6 )
-    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0
+    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %3 : tensor<6x6xi32, #CSR>
@@ -191,9 +191,9 @@ module {
     // CHECK-NEXT: nse = 36
     // CHECK-NEXT: dim = ( 6, 6 )
     // CHECK-NEXT: lvl = ( 6, 6 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %4 : tensor<6x6xi32, #CDR>
@@ -205,9 +205,9 @@ module {
     // CHECK-NEXT: nse = 36
     // CHECK-NEXT: dim = ( 6, 6 )
     // CHECK-NEXT: lvl = ( 6, 6 )
-    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: values : ( 0, -1, 0, -1, 0, 2, 0, 0, -1, 0, 0, -1, -1, 1, 1, 0, 3, 3, -6, 0, 0, 0, 6, 0, -1, 1, 0, 0, -3, -3, 6, 0, 0, 0, -6, 0
+    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: values : ( 0, -1, 0, -1, 0, 2, 0, 0, -1, 0, 0, -1, -1, 1, 1, 0, 3, 3, -6, 0, 0, 0, 6, 0, -1, 1, 0, 0, -3, -3, 6, 0, 0, 0, -6, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %5 : tensor<6x6xi32, #CSC>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
index 4228bcdb1c0d..707c6c34d8dc 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
@@ -120,18 +120,18 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 12,
-    // CHECK-NEXT: crd[0] : ( 0, 3, 5, 11, 13, 17, 18, 20, 21, 28, 29, 31,
-    // CHECK-NEXT: values : ( 1.5, 1.5, 10.2, 11.3, 1, 1, nan, nan, inf, inf, 0, 0,
+    // CHECK-NEXT: pos[0] : ( 0, 12 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 5, 11, 13, 17, 18, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 1.5, 1.5, 10.2, 11.3, 1, 1, nan, nan, inf, inf, 0, 0 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 9,
-    // CHECK-NEXT: crd[0] : ( 0, 3, 5, 11, 13, 17, 18, 21, 31,
-    // CHECK-NEXT: values : ( -2147483648, 2147483647, 1000, 1, 0, 1, 1000, 2147483646, 2147483647,
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 5, 11, 13, 17, 18, 21, 31 )
+    // CHECK-NEXT: values : ( -2147483648, 2147483647, 1000, 1, 0, 1, 1000, 2147483646, 2147483647 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?xf64, #SparseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
index 36701b4385a2..69be2ee75221 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
@@ -453,131 +453,131 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 9,
-    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9,
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 10
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 10,
-    // CHECK-NEXT: crd[0] : ( 1, 3, 4, 10, 16, 18, 21, 28, 29, 31,
-    // CHECK-NEXT: values : ( 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+    // CHECK-NEXT: pos[0] : ( 0, 10 )
+    // CHECK-NEXT: crd[0] : ( 1, 3, 4, 10, 16, 18, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 14
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 14,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 3, 4, 10, 11, 16, 17, 18, 20, 21, 28, 29, 31,
-    // CHECK-NEXT: values : ( 1, 11, 2, 13, 14, 3, 15, 4, 16, 5, 6, 7, 8, 9,
+    // CHECK-NEXT: pos[0] : ( 0, 14 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 3, 4, 10, 11, 16, 17, 18, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 1, 11, 2, 13, 14, 3, 15, 4, 16, 5, 6, 7, 8, 9 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 9,
-    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31,
-    // CHECK-NEXT: values : ( 0, 6, 3, 28, 0, 6, 56, 72, 9,
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 0, 6, 3, 28, 0, 6, 56, 72, 9 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,
-    // CHECK-NEXT: crd[0] : ( 0, 11, 17, 20,
-    // CHECK-NEXT: values : ( 1, 3, 4, 5,
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 11, 17, 20 )
+    // CHECK-NEXT: values : ( 1, 3, 4, 5 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 9,
-    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31,
-    // CHECK-NEXT: values : ( 0, 3, 11, 17, 20, 21, 28, 29, 31,
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2,
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6,
-    // CHECK-NEXT: crd[1] : ( 0, 7, 0, 6, 1, 7,
-    // CHECK-NEXT: values : ( 7, -5, -4, -3, -2, 7,
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 7, 0, 6, 1, 7 )
+    // CHECK-NEXT: values : ( 7, -5, -4, -3, -2, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 10
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3,
-    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1,
-    // CHECK-NEXT: values : ( 2, 4, 1, 2.5, 1, 5, 2, 4, 5, 4,
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1 )
+    // CHECK-NEXT: values : ( 2, 4, 1, 2.5, 1, 5, 2, 4, 5, 4 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 10
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3,
-    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1,
-    // CHECK-NEXT: values : ( 2, 4, 1, 2.5, 1, 5, 2, 4, 5, 4,
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1 )
+    // CHECK-NEXT: values : ( 2, 4, 1, 2.5, 1, 5, 2, 4, 5, 4 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 10
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3,
-    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1,
-    // CHECK-NEXT: values : ( 2, 4, 1, 2.5, -1, -5, 2, 4, 1, 4,
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1 )
+    // CHECK-NEXT: values : ( 2, 4, 1, 2.5, -1, -5, 2, 4, 1, 4 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 10
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3,
-    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1,
-    // CHECK-NEXT: values : ( 0, 1, -1, 1, -1, -2, -2, 2, 1, 2,
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1 )
+    // CHECK-NEXT: values : ( 0, 1, -1, 1, -1, -2, -2, 2, 1, 2 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 3,
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 1, 0,
-    // CHECK-NEXT: values : ( 1, 0, 0, 0,
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 1, 0 )
+    // CHECK-NEXT: values : ( 1, 0, 0, 0 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3,
-    // CHECK-NEXT: pos[1] : ( 0, 1, 5, 6,
-    // CHECK-NEXT: crd[1] : ( 3, 0, 1, 2, 3, 1,
-    // CHECK-NEXT: values : ( -1, -1, -5, -2, 4, 4,
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 5, 6 )
+    // CHECK-NEXT: crd[1] : ( 3, 0, 1, 2, 3, 1 )
+    // CHECK-NEXT: values : ( -1, -1, -5, -2, 4, 4 )
     //
     sparse_tensor.print %sv1 : tensor<?xf64, #SparseVector>
     sparse_tensor.print %sv2 : tensor<?xf64, #SparseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir
index 467b671500e1..ac5f773d6718 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir
@@ -98,11 +98,11 @@ module {
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 4, 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 3
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2
-    // CHECK-NEXT: crd[1] : ( 0, 2
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2 )
+    // CHECK-NEXT: crd[1] : ( 0, 2 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %a : tensor<4x4x4xi32, #Sparse1>
@@ -116,13 +116,13 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 4, 4, 4 )
     // CHECK-NEXT: lvl = ( 2, 2, 2, 2, 2, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 1
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1
-    // CHECK-NEXT: pos[2] : ( 0, 1, 2, 3, 4
-    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1
-    // CHECK-NEXT: values : ( 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 6, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8, 0
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1 )
+    // CHECK-NEXT: pos[2] : ( 0, 1, 2, 3, 4 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 6, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %b : tensor<4x4x4xi32, #Sparse2>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir
index 732bde55be91..edeffea21171 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir
@@ -132,22 +132,22 @@ module {
     // CHECK-NEXT: nse = 16
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 11
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 9, 11
-    // CHECK-NEXT: crd[1] : ( 1, 2, 3, 0, 1, 0, 1, 2, 3, 0, 1
-    // CHECK-NEXT: values : ( 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 9, 11 )
+    // CHECK-NEXT: crd[1] : ( 1, 2, 3, 0, 1, 0, 1, 2, 3, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0 )
     // CHECK-NEXT: ----
     //
     %v = vector.transfer_read %all_dn_out[%c0, %c0], %d0
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
index cae599fa30ae..12132155e7cb 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
@@ -162,18 +162,18 @@ module {
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 12 )
     // CHECK-NEXT: lvl = ( 12 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 2, 4, 6, 8, 10
-    // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 4, 6, 8, 10 )
+    // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 12 )
     // CHECK-NEXT: lvl = ( 12 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 2, 4, 6, 8, 10
-    // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 4, 6, 8, 10 )
+    // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ( ( 1, 0, 3, 0, 5, 0, 7, 0, 9, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ), ( 21, 0, 23, 0, 25, 0, 27, 0, 29, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ), ( 41, 0, 43, 0, 45, 0, 47, 0, 49, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) )
@@ -183,22 +183,22 @@ module {
     // CHECK-NEXT: nse = 15
     // CHECK-NEXT: dim = ( 6, 10 )
     // CHECK-NEXT: lvl = ( 6, 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 2, 4
-    // CHECK-NEXT: pos[1] : ( 0, 5, 10, 15
-    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 6, 8, 0, 2, 4, 6, 8, 0, 2, 4, 6, 8
-    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 21, 23, 25, 27, 29, 41, 43, 45, 47, 49
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 4 )
+    // CHECK-NEXT: pos[1] : ( 0, 5, 10, 15 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 6, 8, 0, 2, 4, 6, 8, 0, 2, 4, 6, 8 )
+    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 21, 23, 25, 27, 29, 41, 43, 45, 47, 49 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 15
     // CHECK-NEXT: dim = ( 6, 10 )
     // CHECK-NEXT: lvl = ( 6, 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 2, 4
-    // CHECK-NEXT: pos[1] : ( 0, 5, 10, 15
-    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 6, 8, 0, 2, 4, 6, 8, 0, 2, 4, 6, 8
-    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 21, 23, 25, 27, 29, 41, 43, 45, 47, 49
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 4 )
+    // CHECK-NEXT: pos[1] : ( 0, 5, 10, 15 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 6, 8, 0, 2, 4, 6, 8, 0, 2, 4, 6, 8 )
+    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 21, 23, 25, 27, 29, 41, 43, 45, 47, 49 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ( ( 1, 0, 3, 0, 5, 0, 7, 0, 9, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ), ( 21, 0, 23, 0, 25, 0, 27, 0, 29, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ), ( 41, 0, 43, 0, 45, 0, 47, 0, 49, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) )
@@ -208,22 +208,22 @@ module {
     // CHECK-NEXT: nse = 15
     // CHECK-NEXT: dim = ( 6, 10 )
     // CHECK-NEXT: lvl = ( 6, 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 2, 4
-    // CHECK-NEXT: pos[1] : ( 0, 5, 10, 15
-    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 6, 8, 0, 2, 4, 6, 8, 0, 2, 4, 6, 8
-    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 21, 23, 25, 27, 29, 41, 43, 45, 47, 49
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 4 )
+    // CHECK-NEXT: pos[1] : ( 0, 5, 10, 15 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 6, 8, 0, 2, 4, 6, 8, 0, 2, 4, 6, 8 )
+    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 21, 23, 25, 27, 29, 41, 43, 45, 47, 49 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 15
     // CHECK-NEXT: dim = ( 6, 10 )
     // CHECK-NEXT: lvl = ( 6, 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 2, 4
-    // CHECK-NEXT: pos[1] : ( 0, 5, 10, 15
-    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 6, 8, 0, 2, 4, 6, 8, 0, 2, 4, 6, 8
-    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 21, 23, 25, 27, 29, 41, 43, 45, 47, 49
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 4 )
+    // CHECK-NEXT: pos[1] : ( 0, 5, 10, 15 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 6, 8, 0, 2, 4, 6, 8, 0, 2, 4, 6, 8 )
+    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 21, 23, 25, 27, 29, 41, 43, 45, 47, 49 )
     // CHECK-NEXT: ----
     //
     %v0 = vector.transfer_read %collapse0[%c0], %df: tensor<12xf64>, vector<12xf64>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
index 9747da27f9e9..087360f7a1ce 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
@@ -104,18 +104,18 @@ module {
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 28, 31,
-    // CHECK-NEXT: values : ( ( 511.13, 2 ), ( 1, 0 ), ( 5, 4 ), ( 8, 6 ),
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 28, 31 )
+    // CHECK-NEXT: values : ( ( 511.13, 2 ), ( 1, 0 ), ( 5, 4 ), ( 8, 6 ) )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 2
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,
-    // CHECK-NEXT: crd[0] : ( 28, 31,
-    // CHECK-NEXT: values : ( ( 6, 8 ), ( 15, 18 ),
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 28, 31 )
+    // CHECK-NEXT: values : ( ( 6, 8 ), ( 15, 18 ) )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?xcomplex<f32>, #SparseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
index d4b43eb57676..3f748015c958 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
@@ -101,18 +101,18 @@ module {
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 28, 31,
-    // CHECK-NEXT: values : ( ( 511.13, 2 ), ( 1, 0 ), ( 5, 4 ), ( 8, 6 ),
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 28, 31 )
+    // CHECK-NEXT: values : ( ( 511.13, 2 ), ( 1, 0 ), ( 5, 4 ), ( 8, 6 ) )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 2
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,
-    // CHECK-NEXT: crd[0] : ( 28, 31,
-    // CHECK-NEXT: values : ( ( 6, 8 ), ( 15, 18 ),
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 28, 31 )
+    // CHECK-NEXT: values : ( ( 6, 8 ), ( 15, 18 ) )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?xcomplex<f64>, #SparseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
index c4fc8b080787..2326234bc06c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
@@ -198,63 +198,63 @@ module {
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 28, 31,
-    // CHECK-NEXT: values : ( ( -5.13, 2 ), ( 1, 0 ), ( 1, 4 ), ( 8, 6 ),
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 28, 31 )
+    // CHECK-NEXT: values : ( ( -5.13, 2 ), ( 1, 0 ), ( 1, 4 ), ( 8, 6 ) )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 28, 31,
-    // CHECK-NEXT: values : ( ( 3.43887, 1.47097 ), ( 3.85374, -27.0168 ), ( -193.43, 57.2184 ),
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 28, 31 )
+    // CHECK-NEXT: values : ( ( 3.43887, 1.47097 ), ( 3.85374, -27.0168 ), ( -193.43, 57.2184 ) )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 28, 31,
-    // CHECK-NEXT: values : ( ( 0.433635, 2.30609 ), ( 2, 1 ), ( 2.53083, 1.18538 ),
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 28, 31 )
+    // CHECK-NEXT: values : ( ( 0.433635, 2.30609 ), ( 2, 1 ), ( 2.53083, 1.18538 ) )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 1, 28, 31,
-    // CHECK-NEXT: values : ( ( 0.761594, 0 ), ( -0.964028, 0 ), ( 0.995055, 0 ),
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 1, 28, 31 )
+    // CHECK-NEXT: values : ( ( 0.761594, 0 ), ( -0.964028, 0 ), ( 0.995055, 0 ) )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 28, 31,
-    // CHECK-NEXT: values : ( ( -5.13, 2 ), ( 3, 4 ), ( 5, 6 ),
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 28, 31 )
+    // CHECK-NEXT: values : ( ( -5.13, 2 ), ( 3, 4 ), ( 5, 6 ) )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 28, 31,
-    // CHECK-NEXT: values : ( ( -2.565, 1 ), ( 1.5, 2 ), ( 2.5, 3 ),
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 28, 31 )
+    // CHECK-NEXT: values : ( ( -2.565, 1 ), ( 1.5, 2 ), ( 2.5, 3 ) )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 28, 31,
-    // CHECK-NEXT: values : ( 5.50608, 5, 7.81025,
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 28, 31 )
+    // CHECK-NEXT: values : ( 5.50608, 5, 7.81025 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?xcomplex<f64>, #SparseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
index abdbf80d0bc4..51c13085cf3e 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
@@ -56,11 +56,11 @@ module {
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 10, 8 )
     // CHECK-NEXT: lvl = ( 10, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 1, 4, 5, 6, 9
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4, 5, 7, 8
-    // CHECK-NEXT: crd[1] : ( 0, 7, 2, 2, 3, 4, 6, 7
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 4, 5, 6, 9 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4, 5, 7, 8 )
+    // CHECK-NEXT: crd[1] : ( 0, 7, 2, 2, 3, 4, 6, 7 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %ts : tensor<10x8xf64, #Tensor1>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
index 612e62bd34d2..3e46b6d65112 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
@@ -116,13 +116,13 @@ func.func @main() {
   // CHECK-NEXT: nse = 18
   // CHECK-NEXT: dim = ( 3, 6, 1 )
   // CHECK-NEXT: lvl = ( 3, 6, 1 )
-  // CHECK-NEXT: pos[0] : ( 0, 3
-  // CHECK-NEXT: crd[0] : ( 0, 1, 2
-  // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18
-  // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
-  // CHECK-NEXT: pos[2] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18
-  // CHECK-NEXT: crd[2] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-  // CHECK-NEXT: values : ( 12, 28, 28, 28, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12
+  // CHECK-NEXT: pos[0] : ( 0, 3 )
+  // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+  // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18 )
+  // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
+  // CHECK-NEXT: pos[2] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18 )
+  // CHECK-NEXT: crd[2] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 )
+  // CHECK-NEXT: values : ( 12, 28, 28, 28, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %CCC_ret : tensor<?x?x?xf32, #CCC>
@@ -132,11 +132,11 @@ func.func @main() {
   // CHECK-NEXT: nse = 18
   // CHECK-NEXT: dim = ( 3, 6, 1 )
   // CHECK-NEXT: lvl = ( 3, 6, 1 )
-  // CHECK-NEXT: pos[0] : ( 0, 3
-  // CHECK-NEXT: crd[0] : ( 0, 1, 2
-  // CHECK-NEXT: pos[2] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18
-  // CHECK-NEXT: crd[2] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-  // CHECK-NEXT: values : ( 12, 28, 28, 28, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12
+  // CHECK-NEXT: pos[0] : ( 0, 3 )
+  // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+  // CHECK-NEXT: pos[2] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18 )
+  // CHECK-NEXT: crd[2] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 )
+  // CHECK-NEXT: values : ( 12, 28, 28, 28, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %CDC_ret : tensor<?x?x?xf32, #CDC>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
index 55d4caeb7eb3..97e9d1783f67 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
@@ -187,11 +187,11 @@ module {
     // CHECK-NEXT: nse = 36
     // CHECK-NEXT: dim = ( 6, 6 )
     // CHECK-NEXT: lvl = ( 6, 6 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<6x6xi32, #DCSR>
@@ -203,11 +203,11 @@ module {
     // CHECK-NEXT: nse = 36
     // CHECK-NEXT: dim = ( 6, 6 )
     // CHECK-NEXT: lvl = ( 6, 6 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %2 : tensor<6x6xi32, #DCSR>
@@ -219,9 +219,9 @@ module {
     // CHECK-NEXT: nse = 36
     // CHECK-NEXT: dim = ( 6, 6 )
     // CHECK-NEXT: lvl = ( 6, 6 )
-    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0
+    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %3 : tensor<6x6xi32, #CSR>
@@ -233,9 +233,9 @@ module {
     // CHECK-NEXT: nse = 36
     // CHECK-NEXT: dim = ( 6, 6 )
     // CHECK-NEXT: lvl = ( 6, 6 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %4 : tensor<6x6xi32, #CDR>
@@ -247,9 +247,9 @@ module {
     // CHECK-NEXT: nse = 36
     // CHECK-NEXT: dim = ( 6, 6 )
     // CHECK-NEXT: lvl = ( 6, 6 )
-    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: values : ( 0, -1, 0, -1, 0, 2, 0, 0, -1, 0, 0, -1, -1, 1, 1, 0, 3, 3, -6, 0, 0, 0, 6, 0, -1, 1, 0, 0, -3, -3, 6, 0, 0, 0, -6, 0
+    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: values : ( 0, -1, 0, -1, 0, 2, 0, 0, -1, 0, 0, -1, -1, 1, 1, 0, 3, 3, -6, 0, 0, 0, 6, 0, -1, 1, 0, 0, -3, -3, 6, 0, 0, 0, -6, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %5 : tensor<6x6xi32, #CSC>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
index d04311e59baf..429175c1a164 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
@@ -147,27 +147,27 @@ func.func @main() {
   // CHECK-NEXT: nse = 108
   // CHECK-NEXT: dim = ( 3, 6, 6, 1 )
   // CHECK-NEXT: lvl = ( 3, 6, 6, 1 )
-  // CHECK-NEXT: pos[0] : ( 0, 3
-  // CHECK-NEXT: crd[0] : ( 0, 1, 2
-  // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18
-  // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
-  // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, 96, 102, 108
+  // CHECK-NEXT: pos[0] : ( 0, 3 )
+  // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+  // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18 )
+  // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
+  // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, 96, 102, 108 )
   // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0,
   // CHECK-SAME:            1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2,
   // CHECK-SAME:            3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
-  // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: pos[3] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
   // CHECK-SAME:            21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
   // CHECK-SAME:            40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58,
   // CHECK-SAME:            59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77,
   // CHECK-SAME:            78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96,
-  // CHECK-SAME:            97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108
+  // CHECK-SAME:            97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108 )
   // CHECK-NEXT: crd[3] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-  // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0
+  // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0 )
   // CHECK-NEXT: values : ( 108, 124, 124, 124, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
@@ -175,7 +175,7 @@ func.func @main() {
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
-  // CHECK-SAME:            108, 108, 108
+  // CHECK-SAME:            108, 108, 108 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %CCCC_ret : tensor<?x?x?x?xf32, #CCCC>
@@ -185,14 +185,14 @@ func.func @main() {
   // CHECK-NEXT: nse = 108
   // CHECK-NEXT: dim = ( 3, 6, 6, 1 )
   // CHECK-NEXT: lvl = ( 3, 6, 6, 1 )
-  // CHECK-NEXT: pos[0] : ( 0, 3
-  // CHECK-NEXT: crd[0] : ( 0, 1, 2
-  // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, 96, 102, 108
+  // CHECK-NEXT: pos[0] : ( 0, 3 )
+  // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+  // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, 96, 102, 108 )
   // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0,
   // CHECK-SAME:            1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2,
   // CHECK-SAME:            3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
-  // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: values : ( 108, 124, 124, 124, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
@@ -200,7 +200,7 @@ func.func @main() {
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
-  // CHECK-SAME:            108, 108, 108
+  // CHECK-SAME:            108, 108, 108 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %CDCD_ret : tensor<?x?x?x?xf32, #CDCD>
@@ -210,14 +210,14 @@ func.func @main() {
   // CHECK-NEXT: nse = 108
   // CHECK-NEXT: dim = ( 3, 6, 6, 1 )
   // CHECK-NEXT: lvl = ( 3, 6, 6, 1 )
-  // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18
-  // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
-  // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, 96, 102, 108
+  // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18 )
+  // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
+  // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, 96, 102, 108 )
   // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0,
   // CHECK-SAME:            1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2,
   // CHECK-SAME:            3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
-  // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: values : ( 108, 124, 124, 124, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
@@ -225,7 +225,7 @@ func.func @main() {
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
-  // CHECK-SAME:            108, 108, 108
+  // CHECK-SAME:            108, 108, 108 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %DCCD_ret : tensor<?x?x?x?xf32, #DCCD>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
index 5e2d1707a249..b23b2dcc173d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
@@ -171,14 +171,14 @@ func.func @main() {
   // CHECK-NEXT: nse = 216
   // CHECK-NEXT: dim = ( 6, 6, 6 )
   // CHECK-NEXT: lvl = ( 6, 6, 6 )
-  // CHECK-NEXT: pos[0] : ( 0, 6
-  // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5
-  // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36
+  // CHECK-NEXT: pos[0] : ( 0, 6 )
+  // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5 )
+  // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36 )
   // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
-  // CHECK-SAME:            0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78,
   // CHECK-SAME:            84, 90, 96, 102, 108, 114, 120, 126, 132, 138, 144, 150,
-  // CHECK-SAME:            156, 162, 168, 174, 180, 186, 192, 198, 204, 210, 216
+  // CHECK-SAME:            156, 162, 168, 174, 180, 186, 192, 198, 204, 210, 216 )
   // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0,
   // CHECK-SAME:            1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2,
@@ -190,7 +190,7 @@ func.func @main() {
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2,
   // CHECK-SAME:            3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
   // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4,
-  // CHECK-SAME:            5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: values : ( 108, 108, 108, 108, 108, 108, 124, 108, 108, 108, 108, 108,
   // CHECK-SAME:            124, 108, 108, 108, 108, 108, 124, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
@@ -208,7 +208,7 @@ func.func @main() {
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
-  // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108
+  // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %CCC_ret : tensor<?x?x?xf32, #CCC>
@@ -218,11 +218,11 @@ func.func @main() {
   // CHECK-NEXT: nse = 216
   // CHECK-NEXT: dim = ( 6, 6, 6 )
   // CHECK-NEXT: lvl = ( 6, 6, 6 )
-  // CHECK-NEXT: pos[0] : ( 0, 6
-  // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5
+  // CHECK-NEXT: pos[0] : ( 0, 6 )
+  // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84,
   // CHECK-SAME:            90, 96, 102, 108, 114, 120, 126, 132, 138, 144, 150, 156,
-  // CHECK-SAME:            162, 168, 174, 180, 186, 192, 198, 204, 210, 216
+  // CHECK-SAME:            162, 168, 174, 180, 186, 192, 198, 204, 210, 216 )
   // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
   // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
@@ -233,7 +233,7 @@ func.func @main() {
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
   // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
   // CHECK-SAME:            0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
-  // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: values : ( 108, 108, 108, 108, 108, 108, 124, 108, 108, 108, 108, 108,
   // CHECK-SAME:            124, 108, 108, 108, 108, 108, 124, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
@@ -251,7 +251,7 @@ func.func @main() {
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
-  // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108
+  // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %CDC_ret : tensor<?x?x?xf32, #CDC>
@@ -263,7 +263,7 @@ func.func @main() {
   // CHECK-NEXT: lvl = ( 6, 6, 6 )
   // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90,
   // CHECK-SAME:            96, 102, 108, 114, 120, 126, 132, 138, 144, 150, 156, 162,
-  // CHECK-SAME:            168, 174, 180, 186, 192, 198, 204, 210, 216
+  // CHECK-SAME:            168, 174, 180, 186, 192, 198, 204, 210, 216 )
   // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
   // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
@@ -274,7 +274,7 @@ func.func @main() {
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
   // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
   // CHECK-SAME:            0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
-  // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: values : ( 108, 108, 108, 108, 108, 108, 124, 108, 108, 108, 108, 108,
   // CHECK-SAME:            124, 108, 108, 108, 108, 108, 124, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
@@ -292,7 +292,7 @@ func.func @main() {
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
-  // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108
+  // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %DDC_ret : tensor<?x?x?xf32, #DDC>
@@ -302,12 +302,12 @@ func.func @main() {
   // CHECK-NEXT: nse = 216
   // CHECK-NEXT: dim = ( 6, 6, 6 )
   // CHECK-NEXT: lvl = ( 6, 6, 6 )
-  // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36
+  // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36 )
   // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
-  // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90,
   // CHECK-SAME:            96, 102, 108, 114, 120, 126, 132, 138, 144, 150, 156, 162,
-  // CHECK-SAME:            168, 174, 180, 186, 192, 198, 204, 210, 216
+  // CHECK-SAME:            168, 174, 180, 186, 192, 198, 204, 210, 216 )
   // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
   // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
@@ -318,7 +318,7 @@ func.func @main() {
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
   // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
   // CHECK-SAME:            0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
-  // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: values : ( 108, 108, 108, 108, 108, 108, 124, 108, 108, 108, 108, 108,
   // CHECK-SAME:            124, 108, 108, 108, 108, 108, 124, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
@@ -336,7 +336,7 @@ func.func @main() {
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
-  // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108
+  // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %DCC_ret : tensor<?x?x?xf32, #DCC>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
index f68e429a3c82..8fb6704c7f50 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
@@ -155,16 +155,16 @@ func.func @main() {
   // CHECK-NEXT: nse = 216
   // CHECK-NEXT: dim = ( 1, 6, 6, 6, 1 )
   // CHECK-NEXT: lvl = ( 1, 6, 6, 6, 1 )
-  // CHECK-NEXT: pos[0] : ( 0, 1
-  // CHECK-NEXT: crd[0] : ( 0
-  // CHECK-NEXT: pos[1] : ( 0, 6
-  // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5
-  // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36
+  // CHECK-NEXT: pos[0] : ( 0, 1 )
+  // CHECK-NEXT: crd[0] : ( 0 )
+  // CHECK-NEXT: pos[1] : ( 0, 6 )
+  // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5 )
+  // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36 )
   // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
-  // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: pos[3] : ( 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, 96,
   // CHECK-SAME:            102, 108, 114, 120, 126, 132, 138, 144, 150, 156, 162, 168, 174,
-  // CHECK-SAME:            180, 186, 192, 198, 204, 210, 216
+  // CHECK-SAME:            180, 186, 192, 198, 204, 210, 216 )
   // CHECK-NEXT: crd[3] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
   // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
@@ -174,7 +174,7 @@ func.func @main() {
   // CHECK-SAME:            0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
   // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1,
   // CHECK-SAME:            2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
-  // CHECK-SAME:            0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: pos[4] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
   // CHECK-SAME:            19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
   // CHECK-SAME:            36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
@@ -189,7 +189,7 @@ func.func @main() {
   // CHECK-SAME:            173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186,
   // CHECK-SAME:            187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200,
   // CHECK-SAME:            201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214,
-  // CHECK-SAME:            215, 216
+  // CHECK-SAME:            215, 216 )
   // CHECK-NEXT: crd[4] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -199,7 +199,7 @@ func.func @main() {
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-  // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0
+  // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0 )
   // CHECK-NEXT: values : ( 108, 124, 124, 124, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
@@ -215,7 +215,7 @@ func.func @main() {
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
-  // CHECK-SAME:            108, 108, 108, 108, 108, 108
+  // CHECK-SAME:            108, 108, 108, 108, 108, 108 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %CCCCC_ret : tensor<?x?x?x?x?xf32, #CCCCC>
@@ -229,11 +229,11 @@ func.func @main() {
   // CHECK-NEXT: nse = 216
   // CHECK-NEXT: dim = ( 1, 6, 6, 6, 1 )
   // CHECK-NEXT: lvl = ( 1, 6, 6, 6, 1 )
-  // CHECK-NEXT: pos[0] : ( 0, 1
-  // CHECK-NEXT: crd[0] : ( 0
-  // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36
+  // CHECK-NEXT: pos[0] : ( 0, 1 )
+  // CHECK-NEXT: crd[0] : ( 0 )
+  // CHECK-NEXT: pos[2] : ( 0, 6, 12, 18, 24, 30, 36 )
   // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
-  // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
+  // CHECK-SAME:            4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
   // CHECK-NEXT: pos[4] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
   // CHECK-SAME:            19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
   // CHECK-SAME:            36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52,
@@ -248,7 +248,7 @@ func.func @main() {
   // CHECK-SAME:            173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186,
   // CHECK-SAME:            187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200,
   // CHECK-SAME:            201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214,
-  // CHECK-SAME:            215, 216
+  // CHECK-SAME:            215, 216 )
   // CHECK-NEXT: crd[4] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -258,7 +258,7 @@ func.func @main() {
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-  // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0
+  // CHECK-SAME:            0, 0, 0, 0, 0, 0, 0, 0, 0 )
   // CHECK-NEXT: values : ( 108, 124, 124, 124, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
@@ -274,7 +274,7 @@ func.func @main() {
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
   // CHECK-SAME:            108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108,
-  // CHECK-SAME:            108, 108, 108, 108, 108, 108
+  // CHECK-SAME:            108, 108, 108, 108, 108, 108 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %CDCDC_ret : tensor<?x?x?x?x?xf32, #CDCDC>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
index 8024c1281895..5de3aa0a2e97 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
@@ -98,156 +98,156 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 2, 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 1
-    // CHECK-NEXT: pos[1] : ( 0, 3, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 3, 4, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 4, 2, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 2, 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 1
-    // CHECK-NEXT: pos[1] : ( 0, 3, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 2, 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 1
-    // CHECK-NEXT: pos[1] : ( 0, 3, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 2, 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 1
-    // CHECK-NEXT: pos[1] : ( 0, 3, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 3, 4, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 3, 4, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 3, 4, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23, 12, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 4, 2, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 4, 2, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 4, 2, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9, 12, 15, 18, 21, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: values : ( 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23, 4, 8, 12, 16, 20, 24 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<2x3x4xf64, #Tensor1>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir
index ff22283f43a7..66215a340a0b 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir
@@ -82,36 +82,36 @@ module {
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 2, 4 )
     // CHECK-NEXT: lvl = ( 1, 2, 2, 2 )
-    // CHECK-NEXT: pos[1] : ( 0, 2
-    // CHECK-NEXT: crd[1] : ( 0, 1
-    // CHECK-NEXT: values : ( 1, 2, 5, 6, 3, 4, 7, 8
+    // CHECK-NEXT: pos[1] : ( 0, 2 )
+    // CHECK-NEXT: crd[1] : ( 0, 1 )
+    // CHECK-NEXT: values : ( 1, 2, 5, 6, 3, 4, 7, 8 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 2, 4 )
     // CHECK-NEXT: lvl = ( 1, 2, 2, 2 )
-    // CHECK-NEXT: pos[1] : ( 0, 2
-    // CHECK-NEXT: crd[1] : ( 0, 1
-    // CHECK-NEXT: values : ( 1, 2, 5, 6, 3, 4, 7, 8
+    // CHECK-NEXT: pos[1] : ( 0, 2 )
+    // CHECK-NEXT: crd[1] : ( 0, 1 )
+    // CHECK-NEXT: values : ( 1, 2, 5, 6, 3, 4, 7, 8 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 2, 4 )
     // CHECK-NEXT: lvl = ( 2, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 2, 4 )
     // CHECK-NEXT: lvl = ( 4, 2 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: values : ( 1, 5, 2, 6, 3, 7, 4, 8
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 5, 2, 6, 3, 7, 4, 8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<2x4xf64, #BSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
index 11baf65e6350..0d9722cd37e6 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
@@ -67,66 +67,66 @@ module {
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 32, 64 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 31
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 31 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 64, 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 63
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31
-    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 63 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 )
+    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 32, 64 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 31
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 31 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 64, 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 63
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31
-    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 63 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 )
+    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 64, 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 63
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31
-    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 63 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 )
+    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 32, 64 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 31
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 31 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<?x?xf64, #DCSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
index 6005aa6cfeae..531efb4f7f37 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
@@ -78,64 +78,64 @@ module {
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 32, 64 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 31
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 31 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 64, 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 63
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31
-    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 63 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 )
+    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 64, 32 )
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31
-    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 )
+    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 64, 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 63
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31
-    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 63 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 31, 0, 1, 0, 31 )
+    // CHECK-NEXT: values : ( 1, 4, 6, 2, 5, 3, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 32, 64 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 31
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 31 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 32, 64 )
     // CHECK-NEXT: lvl = ( 32, 64 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 31
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7
-    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 31 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 63, 0, 1, 0, 63 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<32x64xf64, #DCSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
index 16813e0aa707..c16ae0de1820 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
@@ -209,21 +209,21 @@ module {
     // CHECK-NEXT: nse = 64
     // CHECK-NEXT: dim = ( 8, 8 )
     // CHECK-NEXT: lvl = ( 8, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 64
+    // CHECK-NEXT: pos[0] : ( 0, 64 )
     // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2,
     // CHECK-SAME:            2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4,
     // CHECK-SAME:            5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7,
-    // CHECK-SAME:            7, 7, 7, 7
+    // CHECK-SAME:            7, 7, 7, 7 )
     // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3,
     // CHECK-SAME:            4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
     // CHECK-SAME:            0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3,
-    // CHECK-SAME:            4, 5, 6, 7
+    // CHECK-SAME:            4, 5, 6, 7 )
     // CHECK-NEXT: values : ( 8.8, 4.8, 6.8, 4.8, 8.8, 6.1, 14.8, 16.8, 4.4, 4.4, 4.4, 8.4,
     // CHECK-SAME:            8.4, 12.4, 16.4, 16.4, 8.8, 4.8, 6.8, 8.8, 8.8, 12.8, 14.8,
     // CHECK-SAME:            15.8, 4.3, 5.3, 6.3, 8.3, 8.3, 12.3, 14.3, 16.3, 4.5, 4.5,
     // CHECK-SAME:            6.5, 8.5, 8.5, 12.5, 14.5, 16.5, 9.9, 4.9, 6.9, 8.9, 8.9,
     // CHECK-SAME:            12.9, 15.9, 16.9, 12.1, 6.1, 5.1, 9.1, 9.1, 13.1, 15.1, 17.1,
-    // CHECK-SAME:            15.4, 5.4, 7.4, 5.4, 11.4, 10.4, 11.4, 9.4
+    // CHECK-SAME:            15.4, 5.4, 7.4, 5.4, 11.4, 10.4, 11.4, 9.4 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %COO_RET : tensor<8x8xf32, #SortedCOOSoA>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
index 5451f2d957ad..b41fda19459e 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
@@ -67,18 +67,18 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 1024 )
     // CHECK-NEXT: lvl = ( 1024 )
-    // CHECK-NEXT: pos[0] : ( 0, 5
-    // CHECK-NEXT: crd[0] : ( 0, 1, 22, 23, 1022
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5
+    // CHECK-NEXT: pos[0] : ( 0, 5 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 22, 23, 1022 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 1024 )
     // CHECK-NEXT: lvl = ( 1024 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 22, 1022, 1023
-    // CHECK-NEXT: values : ( 6, 7, 8
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 22, 1022, 1023 )
+    // CHECK-NEXT: values : ( 6, 7, 8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %s1 : tensor<1024xf32, #SparseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
index f4ae33a42d06..17fab93b9b21 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
@@ -79,9 +79,9 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 8 )
     // CHECK-NEXT: lvl = ( 3, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12,  )
-    // CHECK-NEXT: crd[1] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5,  )
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,  )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 )
+    // CHECK-NEXT: crd[1] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %A1 : tensor<?x?xf64, #CSR>
@@ -93,9 +93,9 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 8 )
     // CHECK-NEXT: lvl = ( 3, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 4, 4, 8, 8, 12,
-    // CHECK-NEXT: crd[1] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5,  )
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,  )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 4, 8, 8, 12, {{.*}} )
+    // CHECK-NEXT: crd[1] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %A2 : tensor<?x?xf64, #CSR_hi>
@@ -107,8 +107,8 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 8 )
     // CHECK-NEXT: lvl = ( 3, 2, 4 )
-    // CHECK-NEXT: crd[2] : ( 2, 3, 1, 3, 1, 2, 0, 3, 0, 2, 0, 1,  )
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,  )
+    // CHECK-NEXT: crd[2] : ( 2, 3, 1, 3, 1, 2, 0, 3, 0, 2, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 )
     // CHECK-NEXT: ----
     // CHECK-NEXT: ---- Sparse Tensor ----
     //
@@ -120,8 +120,8 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 8 )
     // CHECK-NEXT: lvl = ( 3, 1, 8 )
-    // CHECK-NEXT: crd[2] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5,  )
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,  )
+    // CHECK-NEXT: crd[2] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %A4 : tensor<?x?xf64, #NV_58>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
index 7fc37eade720..7255649ccd42 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
@@ -98,36 +98,36 @@ module {
     // CHECK-NEXT: nse = 0
     // CHECK-NEXT: dim = ( 10 )
     // CHECK-NEXT: lvl = ( 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 0,  )
-    // CHECK-NEXT: crd[0] : (  )
-    // CHECK-NEXT: values : (  )
+    // CHECK-NEXT: pos[0] : ( 0, 0 )
+    // CHECK-NEXT: crd[0] : ( )
+    // CHECK-NEXT: values : ( )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 0
     // CHECK-NEXT: dim = ( 10 )
     // CHECK-NEXT: lvl = ( 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 0,
-    // CHECK-NEXT: crd[0] : (  )
-    // CHECK-NEXT: values : (  )
+    // CHECK-NEXT: pos[0] : ( 0, 0 )
+    // CHECK-NEXT: crd[0] : ( )
+    // CHECK-NEXT: values : ( )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 0
     // CHECK-NEXT: dim = ( 10 )
     // CHECK-NEXT: lvl = ( 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 0,  )
-    // CHECK-NEXT: crd[0] : (  )
-    // CHECK-NEXT: values : (  )
+    // CHECK-NEXT: pos[0] : ( 0, 0 )
+    // CHECK-NEXT: crd[0] : ( )
+    // CHECK-NEXT: values : ( )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 10
     // CHECK-NEXT: dim = ( 10 )
     // CHECK-NEXT: lvl = ( 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 10,  )
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,  )
-    // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  )
+    // CHECK-NEXT: pos[0] : ( 0, 10 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 )
+    // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<10xf32, #SV>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
index 451195b2185b..6e875de7481e 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
@@ -86,13 +86,13 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 8, 4 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 8, 16, 24, 32
+    // CHECK-NEXT: pos[1] : ( 0, 8, 16, 24, 32 )
     // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0,
-    // CHECK-SAME:            1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7
+    // CHECK-SAME:            1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 )
     // CHECK-NEXT: values : ( 32.53, 34.56, 36.59, 38.62, 40.65, 42.68, 44.71, 46.74,
     // CHECK-SAME:            35.73, 37.96, 40.19, 42.42, 44.65, 46.88, 49.11, 51.34,
     // CHECK-SAME:            38.93, 41.36, 43.79, 46.22, 48.65, 51.08, 53.51, 55.94,
-    // CHECK-SAME:            42.13, 44.76, 47.39, 50.02, 52.65, 55.28, 57.91, 60.54
+    // CHECK-SAME:            42.13, 44.76, 47.39, 50.02, 52.65, 55.28, 57.91, 60.54 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %x3 : tensor<8x4xf64, #CSC>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
index 393242484576..5e021596efea 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
@@ -200,74 +200,74 @@ module {
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 3, 4 )
     // CHECK-NEXT: lvl = ( 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6
-    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2
-    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 11
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2 )
+    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 11 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 3, 4 )
     // CHECK-NEXT: lvl = ( 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6
-    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2
-    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 11
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2 )
+    // CHECK-NEXT: values : ( 1, 3, 5, 7, 9, 11 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 2, 2 )
     // CHECK-NEXT: lvl = ( 3, 2, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12
-    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 2, 2 )
     // CHECK-NEXT: lvl = ( 3, 2, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12
-    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 2, 2 )
     // CHECK-NEXT: lvl = ( 3, 2, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12
-    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 2, 2 )
     // CHECK-NEXT: lvl = ( 3, 2, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12
-    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
-    // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 6, 8, 10, 12 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 )
+    // CHECK-NEXT: values : ( 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 3.1, 3.2, 3.3, 3.4 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %expand2 : tensor<3x4xf64, #SparseMatrix>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
index 37ff2e3ffd3f..93b8eda2c2ae 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
@@ -106,13 +106,13 @@ module {
     // CHECK-NEXT: nse = 36
     // CHECK-NEXT: dim = ( 6, 6 )
     // CHECK-NEXT: lvl = ( 6, 6 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5
-    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5 )
+    // CHECK-NEXT: pos[1] : ( 0, 6, 12, 18, 24, 30, 36 )
     // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4,
-    // CHECK-SAME:            5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5
+    // CHECK-SAME:            5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5 )
     // CHECK-NEXT: values : ( 0, 0, -1, -6, -1, 6, -1, 0, 1, 0, 1, 0, 0, -1, 1,
-    // CHECK-SAME:            0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0
+    // CHECK-SAME:            0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 3, 6, -3, -6, 2, -1, 3, 0, -3, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<6x6xi32, #DCSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
index 3ce45e5fd971..005398445828 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
@@ -212,80 +212,80 @@ module {
     // CHECK-NEXT: nse = 2
     // CHECK-NEXT: dim = ( 8 )
     // CHECK-NEXT: lvl = ( 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 2, 4
-    // CHECK-NEXT: values : ( 20, 80
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 2, 4 )
+    // CHECK-NEXT: values : ( 20, 80 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 8 )
     // CHECK-NEXT: lvl = ( 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 8
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7
-    // CHECK-NEXT: values : ( 0, 1, 12, 3, 24, 5, 6, 7
+    // CHECK-NEXT: pos[0] : ( 0, 8 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7 )
+    // CHECK-NEXT: values : ( 0, 1, 12, 3, 24, 5, 6, 7 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 8 )
     // CHECK-NEXT: lvl = ( 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 8
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7
-    // CHECK-NEXT: values : ( 0, 2, 8, 24, 64, 160, 384, 896
+    // CHECK-NEXT: pos[0] : ( 0, 8 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7 )
+    // CHECK-NEXT: values : ( 0, 2, 8, 24, 64, 160, 384, 896 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 8 )
     // CHECK-NEXT: lvl = ( 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 8
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7
-    // CHECK-NEXT: values : ( 1, 3, 6, 11, 20, 37, 70, 135
+    // CHECK-NEXT: pos[0] : ( 0, 8 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7 )
+    // CHECK-NEXT: values : ( 1, 3, 6, 11, 20, 37, 70, 135 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 2
     // CHECK-NEXT: dim = ( 3, 4 )
     // CHECK-NEXT: lvl = ( 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2
-    // CHECK-NEXT: crd[1] : ( 1, 3
-    // CHECK-NEXT: values : ( 10, 120
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2 )
+    // CHECK-NEXT: crd[1] : ( 1, 3 )
+    // CHECK-NEXT: values : ( 10, 120 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 4 )
     // CHECK-NEXT: lvl = ( 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 0, 1, 2, 3, 1, 12, 3, 4, 2, 3, 4, 25
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 0, 1, 2, 3, 1, 12, 3, 4, 2, 3, 4, 25 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 4 )
     // CHECK-NEXT: lvl = ( 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 0, 0, 0, 0, 0, 2, 2, 3, 0, 2, 12, 24
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 0, 0, 0, 0, 0, 2, 2, 3, 0, 2, 12, 24 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 3, 4 )
     // CHECK-NEXT: lvl = ( 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 2, 4, 4, 5, 3, 4, 7, 9
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 2, 4, 4, 5, 3, 4, 7, 9 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<8xi64, #SparseVector>
@@ -304,11 +304,11 @@ module {
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 2, 3 )
     // CHECK-NEXT: lvl = ( 2, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 1
-    // CHECK-NEXT: pos[1] : ( 0, 3, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: values : ( 0, 10, 0, 1, 1, 42
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: values : ( 0, 10, 0, 1, 1, 42 )
     // CHECK-NEXT: ----
     //
     %100 = call @add_outer_2d(%sf32) : (tensor<2x3xf32, #SparseMatrix>)
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
index 12e0d2267a26..a81ec172f599 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
@@ -65,9 +65,9 @@ module {
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 1024 )
     // CHECK-NEXT: lvl = ( 1024 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,
-    // CHECK-NEXT: crd[0] : ( 0, 1, 3, 1023,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4,
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 3, 1023 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %5 : tensor<1024xf32, #SparseVector>
@@ -86,9 +86,9 @@ module {
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 1024 )
     // CHECK-NEXT: lvl = ( 1024 )
-    // CHECK-NEXT: pos[0] : ( 0, 8,
-    // CHECK-NEXT: crd[0] : ( 0, 3, 6, 9, 12, 15, 18, 21,
-    // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 1, 1, 1,
+    // CHECK-NEXT: pos[0] : ( 0, 8 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 6, 9, 12, 15, 18, 21 )
+    // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 1, 1, 1 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %8 : tensor<1024xf32, #SparseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
index 883109150653..baab6e759886 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
@@ -68,7 +68,7 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 4, 3 )
     // CHECK-NEXT: lvl = ( 4, 3 )
-    // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4,
+    // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4 )
     // CHECK-NEXT: ----
     //
     %densea = tensor.empty() : tensor<4x3xf64, #Dense>
@@ -86,10 +86,10 @@ module {
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 4, 3 )
     // CHECK-NEXT: lvl = ( 4, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3, 3,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4,
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4 )
     // CHECK-NEXT: ----
     //
     %cooa = tensor.empty() : tensor<4x3xf64, #SortedCOO>
@@ -107,9 +107,9 @@ module {
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 4, 3 )
     // CHECK-NEXT: lvl = ( 4, 3 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 1, 2, 4,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4,
+    // CHECK-NEXT: pos[1] : ( 0, 1, 1, 2, 4 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4 )
     // CHECK-NEXT: ----
     //
     %csra = tensor.empty() : tensor<4x3xf64, #CSR>
@@ -127,11 +127,11 @@ module {
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 4, 3 )
     // CHECK-NEXT: lvl = ( 4, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3,
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4,
-    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2,
-    // CHECK-NEXT: values : ( 1, 2, 3, 4,
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4 )
     // CHECK-NEXT: ----
     //
     %dcsra = tensor.empty() : tensor<4x3xf64, #DCSR>
@@ -149,9 +149,9 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 4, 3 )
     // CHECK-NEXT: lvl = ( 4, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3,
-    // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 2, 3, 0, 4,
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 2, 3, 0, 4 )
     // CHECK-NEXT: ----
     //
     %rowa = tensor.empty() : tensor<4x3xf64, #Row>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
index db6612402357..12ef94fc2baa 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
@@ -64,11 +64,11 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 5, 4, 3 )
     // CHECK-NEXT: lvl = ( 5, 4, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 3, 4
-    // CHECK-NEXT: pos[2] : ( 0, 2, 2, 2, 3, 3, 3, 4, 5
-    // CHECK-NEXT: crd[2] : ( 1, 2, 1, 2, 2
-    // CHECK-NEXT: values : ( 1.1, 2.2, 3.3, 4.4, 5.5
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 3, 4 )
+    // CHECK-NEXT: pos[2] : ( 0, 2, 2, 2, 3, 3, 3, 4, 5 )
+    // CHECK-NEXT: crd[2] : ( 1, 2, 1, 2, 2 )
+    // CHECK-NEXT: values : ( 1.1, 2.2, 3.3, 4.4, 5.5 )
     // CHECK-NEXT: ----
     %tensora = tensor.empty() : tensor<5x4x3xf64, #TensorCSR>
     %tensor1 = tensor.insert %f1 into %tensora[%c3, %c0, %c1] : tensor<5x4x3xf64, #TensorCSR>
@@ -83,11 +83,11 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 5, 4, 3 )
     // CHECK-NEXT: lvl = ( 5, 4, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 3, 4
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4
-    // CHECK-NEXT: crd[1] : ( 0, 3, 2, 3
-    // CHECK-NEXT: values : ( 0, 1.1, 2.2, 0, 3.3, 0, 0, 0, 4.4, 0, 0, 5.5
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 3, 4 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 2, 3 )
+    // CHECK-NEXT: values : ( 0, 1.1, 2.2, 0, 3.3, 0, 0, 0, 4.4, 0, 0, 5.5 )
     // CHECK-NEXT: ----
     %rowa = tensor.empty() : tensor<5x4x3xf64, #TensorRow>
     %row1 = tensor.insert %f1 into %rowa[%c3, %c0, %c1] : tensor<5x4x3xf64, #TensorRow>
@@ -102,11 +102,11 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 5, 4, 3 )
     // CHECK-NEXT: lvl = ( 5, 4, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 3, 4
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 2, 3, 1, 2, 2, 3, 2
-    // CHECK-NEXT: values : ( 1.1, 2.2, 3.3, 4.4, 5.5
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 3, 4 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 2, 3, 1, 2, 2, 3, 2 )
+    // CHECK-NEXT: values : ( 1.1, 2.2, 3.3, 4.4, 5.5 )
     // CHECK-NEXT: ----
     %ccoo = tensor.empty() : tensor<5x4x3xf64, #CCoo>
     %ccoo1 = tensor.insert %f1 into %ccoo[%c3, %c0, %c1] : tensor<5x4x3xf64, #CCoo>
@@ -121,9 +121,9 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 5, 4, 3 )
     // CHECK-NEXT: lvl = ( 5, 4, 3 )
-    // CHECK-NEXT: pos[1] : ( 0, 0, 0, 0, 3, 5
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 2, 3, 1, 2, 2, 3, 2
-    // CHECK-NEXT: values : ( 1.1, 2.2, 3.3, 4.4, 5.5
+    // CHECK-NEXT: pos[1] : ( 0, 0, 0, 0, 3, 5 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 2, 3, 1, 2, 2, 3, 2 )
+    // CHECK-NEXT: values : ( 1.1, 2.2, 3.3, 4.4, 5.5 )
     // CHECK-NEXT: ----
     %dcoo = tensor.empty() : tensor<5x4x3xf64, #DCoo>
     %dcoo1 = tensor.insert %f1 into %dcoo[%c3, %c0, %c1] : tensor<5x4x3xf64, #DCoo>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir
index c05a9f574269..416c137a1dc3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir
@@ -38,14 +38,17 @@ module {
                                [13.0, 14.0, 15.0, 16.0 ]]> : tensor<5x4xf64>
     %s = sparse_tensor.convert %d : tensor<5x4xf64> to tensor<5x4xf64, #CSR_hi>
 
+    //
+    // Note: position for loose_compressed level can vary in the end,
+    // therefore we loosly check it with {{.*}}.
     //
     // CHECK:   ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 17
     // CHECK-NEXT: dim = ( 5, 4 )
     // CHECK-NEXT: lvl = ( 5, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 4, 4, 8, 8, 9, 9, 13
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 2, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 5.5, 9, 10, 11, 12, 13, 14, 15, 16
+    // CHECK-NEXT: pos[1] : ( 0, 4, 4, 8, 8, 9, 9, 13, {{.*}} )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 2, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 5.5, 9, 10, 11, 12, 13, 14, 15, 16 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %s : tensor<5x4xf64, #CSR_hi>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
index e505559037a9..14fa22a70134 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
@@ -146,9 +146,9 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 8, 16, 24, 32
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7
-    // CHECK-NEXT: values : ( 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 1.2, 2.2, 3.2, 4.2, 5.2, 6.2, 7.2, 8.2, 1.3, 2.3, 3.3, 4.3, 5.3, 6.3, 7.3, 8.3, 1.4, 2.4, 3.4, 4.4, 5.4, 6.4, 7.4, 8.4
+    // CHECK-NEXT: pos[1] : ( 0, 8, 16, 24, 32 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 )
+    // CHECK-NEXT: values : ( 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 1.2, 2.2, 3.2, 4.2, 5.2, 6.2, 7.2, 8.2, 1.3, 2.3, 3.3, 4.3, 5.3, 6.3, 7.3, 8.3, 1.4, 2.4, 3.4, 4.4, 5.4, 6.4, 7.4, 8.4 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %a1 : tensor<4x8xf64, #CSR>
@@ -158,11 +158,11 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 8, 16, 24, 32
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7
-    // CHECK-NEXT: values : ( 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 1.2, 2.2, 3.2, 4.2, 5.2, 6.2, 7.2, 8.2, 1.3, 2.3, 3.3, 4.3, 5.3, 6.3, 7.3, 8.3, 1.4, 2.4, 3.4, 4.4, 5.4, 6.4, 7.4, 8.4
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 8, 16, 24, 32 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 )
+    // CHECK-NEXT: values : ( 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 1.2, 2.2, 3.2, 4.2, 5.2, 6.2, 7.2, 8.2, 1.3, 2.3, 3.3, 4.3, 5.3, 6.3, 7.3, 8.3, 1.4, 2.4, 3.4, 4.4, 5.4, 6.4, 7.4, 8.4 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %a2 : tensor<4x8xf64, #DCSR>
@@ -172,9 +172,9 @@ module {
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 2, 3, 4
-    // CHECK-NEXT: crd[1] : ( 1, 5, 1, 7
-    // CHECK-NEXT: values : ( 2.1, 6.1, 2.3, 1
+    // CHECK-NEXT: pos[1] : ( 0, 2, 2, 3, 4 )
+    // CHECK-NEXT: crd[1] : ( 1, 5, 1, 7 )
+    // CHECK-NEXT: values : ( 2.1, 6.1, 2.3, 1 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %a3 : tensor<4x8xf64, #CSR>
@@ -184,11 +184,11 @@ module {
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4
-    // CHECK-NEXT: crd[1] : ( 1, 5, 1, 7
-    // CHECK-NEXT: values : ( 2.1, 6.1, 2.3, 1
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4 )
+    // CHECK-NEXT: crd[1] : ( 1, 5, 1, 7 )
+    // CHECK-NEXT: values : ( 2.1, 6.1, 2.3, 1 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %a4 : tensor<4x8xf64, #DCSR>
@@ -198,9 +198,9 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 8, 4 )
     // CHECK-NEXT: lvl = ( 8, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16, 20, 24, 28, 32
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 10.1, 11.1, 12.1, 13.1, 10.2, 11.2, 12.2, 13.2, 10.3, 11.3, 12.3, 13.3, 10.4, 11.4, 12.4, 13.4, 10.5, 11.5, 12.5, 13.5, 10.6, 11.6, 12.6, 13.6, 10.7, 11.7, 12.7, 13.7, 10.8, 11.8, 12.8, 13.8
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16, 20, 24, 28, 32 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 10.1, 11.1, 12.1, 13.1, 10.2, 11.2, 12.2, 13.2, 10.3, 11.3, 12.3, 13.3, 10.4, 11.4, 12.4, 13.4, 10.5, 11.5, 12.5, 13.5, 10.6, 11.6, 12.6, 13.6, 10.7, 11.7, 12.7, 13.7, 10.8, 11.8, 12.8, 13.8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %b1 : tensor<8x4xf64, #CSR>
@@ -210,11 +210,11 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 8, 4 )
     // CHECK-NEXT: lvl = ( 8, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 8
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16, 20, 24, 28, 32
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 10.1, 11.1, 12.1, 13.1, 10.2, 11.2, 12.2, 13.2, 10.3, 11.3, 12.3, 13.3, 10.4, 11.4, 12.4, 13.4, 10.5, 11.5, 12.5, 13.5, 10.6, 11.6, 12.6, 13.6, 10.7, 11.7, 12.7, 13.7, 10.8, 11.8, 12.8, 13.8
+    // CHECK-NEXT: pos[0] : ( 0, 8 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7 )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16, 20, 24, 28, 32 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 10.1, 11.1, 12.1, 13.1, 10.2, 11.2, 12.2, 13.2, 10.3, 11.3, 12.3, 13.3, 10.4, 11.4, 12.4, 13.4, 10.5, 11.5, 12.5, 13.5, 10.6, 11.6, 12.6, 13.6, 10.7, 11.7, 12.7, 13.7, 10.8, 11.8, 12.8, 13.8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %b2 : tensor<8x4xf64, #DCSR>
@@ -224,9 +224,9 @@ module {
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 8, 4 )
     // CHECK-NEXT: lvl = ( 8, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 3, 4, 4, 5, 6, 8
-    // CHECK-NEXT: crd[1] : ( 3, 2, 1, 0, 1, 2, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 3, 4, 4, 5, 6, 8 )
+    // CHECK-NEXT: crd[1] : ( 3, 2, 1, 0, 1, 2, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %b3 : tensor<8x4xf64, #CSR>
@@ -236,11 +236,11 @@ module {
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 8, 4 )
     // CHECK-NEXT: lvl = ( 8, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 7
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 5, 6, 7
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 3, 4, 5, 6, 8
-    // CHECK-NEXT: crd[1] : ( 3, 2, 1, 0, 1, 2, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8
+    // CHECK-NEXT: pos[0] : ( 0, 7 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 5, 6, 7 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 3, 4, 5, 6, 8 )
+    // CHECK-NEXT: crd[1] : ( 3, 2, 1, 0, 1, 2, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %b4 : tensor<8x4xf64, #DCSR>
@@ -289,9 +289,9 @@ module {
     // CHECK-NEXT: nse = 16
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 388.76, 425.56, 462.36, 499.16, 397.12, 434.72, 472.32, 509.92, 405.48, 443.88, 482.28, 520.68, 413.84, 453.04, 492.24, 531.44
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 388.76, 425.56, 462.36, 499.16, 397.12, 434.72, 472.32, 509.92, 405.48, 443.88, 482.28, 520.68, 413.84, 453.04, 492.24, 531.44 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<4x4xf64, #CSR>
@@ -301,11 +301,11 @@ module {
     // CHECK-NEXT: nse = 16
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 388.76, 425.56, 462.36, 499.16, 397.12, 434.72, 472.32, 509.92, 405.48, 443.88, 482.28, 520.68, 413.84, 453.04, 492.24, 531.44
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 388.76, 425.56, 462.36, 499.16, 397.12, 434.72, 472.32, 509.92, 405.48, 443.88, 482.28, 520.68, 413.84, 453.04, 492.24, 531.44 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %2 : tensor<4x4xf64, #DCSR>
@@ -324,9 +324,9 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 4, 4, 8, 12
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 86.08, 94.28, 102.48, 110.68, 23.46, 25.76, 28.06, 30.36, 10.8, 11.8, 12.8, 13.8
+    // CHECK-NEXT: pos[1] : ( 0, 4, 4, 8, 12 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 86.08, 94.28, 102.48, 110.68, 23.46, 25.76, 28.06, 30.36, 10.8, 11.8, 12.8, 13.8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %4 : tensor<4x4xf64, #CSR>
@@ -336,11 +336,11 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 86.08, 94.28, 102.48, 110.68, 23.46, 25.76, 28.06, 30.36, 10.8, 11.8, 12.8, 13.8
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 86.08, 94.28, 102.48, 110.68, 23.46, 25.76, 28.06, 30.36, 10.8, 11.8, 12.8, 13.8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %5 : tensor<4x4xf64, #DCSR>
@@ -359,9 +359,9 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 2, 3, 5
-    // CHECK-NEXT: crd[1] : ( 1, 2, 2, 2, 3
-    // CHECK-NEXT: values : ( 30.5, 4.2, 4.6, 7, 8
+    // CHECK-NEXT: pos[1] : ( 0, 2, 2, 3, 5 )
+    // CHECK-NEXT: crd[1] : ( 1, 2, 2, 2, 3 )
+    // CHECK-NEXT: values : ( 30.5, 4.2, 4.6, 7, 8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %7 : tensor<4x4xf64, #CSR>
@@ -371,11 +371,11 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 5
-    // CHECK-NEXT: crd[1] : ( 1, 2, 2, 2, 3
-    // CHECK-NEXT: values : ( 30.5, 4.2, 4.6, 7, 8
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 5 )
+    // CHECK-NEXT: crd[1] : ( 1, 2, 2, 2, 3 )
+    // CHECK-NEXT: values : ( 30.5, 4.2, 4.6, 7, 8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %8 : tensor<4x4xf64, #DCSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir
index 58e96d1fa51f..c76bf2ccfe35 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir
@@ -174,11 +174,11 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 5
-    // CHECK-NEXT: crd[1] : ( 1, 2, 2, 2, 3
-    // CHECK-NEXT: values : ( 30.5, 4.2, 4.6, 7, 8
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 5 )
+    // CHECK-NEXT: crd[1] : ( 1, 2, 2, 2, 3 )
+    // CHECK-NEXT: values : ( 30.5, 4.2, 4.6, 7, 8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %2 : tensor<4x4xf64, #DCSR>
@@ -196,9 +196,9 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 2, 3, 5
-    // CHECK-NEXT: crd[1] : ( 1, 2, 2, 2, 3
-    // CHECK-NEXT: values : ( 30.5, 4.2, 4.6, 7, 8
+    // CHECK-NEXT: pos[1] : ( 0, 2, 2, 3, 5 )
+    // CHECK-NEXT: crd[1] : ( 1, 2, 2, 2, 3 )
+    // CHECK-NEXT: values : ( 30.5, 4.2, 4.6, 7, 8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %3 : tensor<4x4xf64, #CSR>
@@ -210,9 +210,9 @@ module {
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 2, 3
-    // CHECK-NEXT: crd[1] : ( 0, 0, 0
-    // CHECK-NEXT: values : ( 2.3, 6.9, 12.6
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 2, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 0 )
+    // CHECK-NEXT: values : ( 2.3, 6.9, 12.6 )
     // CHECK-NEXT: ----
     //
     %s1 = tensor.extract_slice %tmp[0, 1][4, 4][2, 1] : tensor<8x8xf64, #DCSR> to tensor<4x4xf64, #DCSR_SLICE_1>
@@ -228,9 +228,9 @@ module {
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 0, 1, 0, 3, 0
-    // CHECK-NEXT: values : ( 2.3, 6.9, 12.6
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 0, 1, 0, 3, 0 )
+    // CHECK-NEXT: values : ( 2.3, 6.9, 12.6 )
     // CHECK-NEXT: ----
     //
     %t1_coo = sparse_tensor.convert %sa : tensor<8x8xf64> to tensor<8x8xf64, #COO>
@@ -246,9 +246,9 @@ module {
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 2, 3
-    // CHECK-NEXT: crd[1] : ( 0, 0, 0
-    // CHECK-NEXT: values : ( 2.3, 6.9, 12.6
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 2, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 0 )
+    // CHECK-NEXT: values : ( 2.3, 6.9, 12.6 )
     // CHECK-NEXT: ----
     //
     %s1_dyn = tensor.extract_slice %tmp[%c_0, %c_1][4, 4][%c_2, %c_1] : tensor<8x8xf64, #DCSR> to tensor<4x4xf64, #DCSR_SLICE_dyn>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
index 8ea26fa3efdf..770c4f55a280 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
@@ -163,11 +163,11 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9
-    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %sm1 : tensor<?x?xf64, #DCSR>
@@ -177,11 +177,11 @@ module {
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6
-    // CHECK-NEXT: crd[1] : ( 0, 7, 0, 6, 1, 7
-    // CHECK-NEXT: values : ( 6, 5, 4, 3, 2, 1
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 7, 0, 6, 1, 7 )
+    // CHECK-NEXT: values : ( 6, 5, 4, 3, 2, 1 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %sm2 : tensor<?x?xf64, #DCSR>
@@ -191,11 +191,11 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9
-    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3
-    // CHECK-NEXT: values : ( 2, 4, 6, 8, 10, 12, 14, 16, 18
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3 )
+    // CHECK-NEXT: values : ( 2, 4, 6, 8, 10, 12, 14, 16, 18 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?x?xf64, #DCSR>
@@ -205,11 +205,11 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9
-    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3
-    // CHECK-NEXT: values : ( 2, 4, 6, 8, 10, 12, 14, 16, 18
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3 )
+    // CHECK-NEXT: values : ( 2, 4, 6, 8, 10, 12, 14, 16, 18 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<?x?xf64, #DCSR>
@@ -219,11 +219,11 @@ module {
     // CHECK-NEXT: nse = 13
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 3, 6, 10, 13
-    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 0, 6, 7, 1, 2, 4, 7, 0, 2, 3
-    // CHECK-NEXT: values : ( 8, 4, 5, 4, 3, 6, 2, 8, 10, 13, 14, 16, 18
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 6, 10, 13 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 0, 6, 7, 1, 2, 4, 7, 0, 2, 3 )
+    // CHECK-NEXT: values : ( 8, 4, 5, 4, 3, 6, 2, 8, 10, 13, 14, 16, 18 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %2 : tensor<?x?xf64, #DCSR>
@@ -233,11 +233,11 @@ module {
     // CHECK-NEXT: nse = 2
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 2
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2
-    // CHECK-NEXT: crd[1] : ( 0, 7
-    // CHECK-NEXT: values : ( 12, 12
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2 )
+    // CHECK-NEXT: crd[1] : ( 0, 7 )
+    // CHECK-NEXT: values : ( 12, 12 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %3 : tensor<?x?xf64, #DCSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
index c30c6b9b5cc2..683be61be2a2 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
@@ -88,11 +88,11 @@ module {
     // CHECK-NEXT: nse = 2
     // CHECK-NEXT: dim = ( 32, 16 )
     // CHECK-NEXT: lvl = ( 32, 16 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 2, 31
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2
-    // CHECK-NEXT: crd[1] : ( 2, 0
-    // CHECK-NEXT: values : ( 14, 20
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 2, 31 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2 )
+    // CHECK-NEXT: crd[1] : ( 2, 0 )
+    // CHECK-NEXT: values : ( 14, 20 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<32x16xf32, #DCSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
index 74f0e7698bc1..8eadc348020f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
@@ -95,11 +95,11 @@ module {
     // CHECK-NEXT: nse = 2
     // CHECK-NEXT: dim = ( 3, 3 )
     // CHECK-NEXT: lvl = ( 3, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 1, 2
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2
-    // CHECK-NEXT: crd[1] : ( 1, 2
-    // CHECK-NEXT: values : ( 7, 69
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 1, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2 )
+    // CHECK-NEXT: crd[1] : ( 1, 2 )
+    // CHECK-NEXT: values : ( 7, 69 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?x?xi32, #SparseMatrix>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
index 88513c80219a..334ffe492952 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
@@ -87,11 +87,11 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 5, 5 )
     // CHECK-NEXT: lvl = ( 5, 5 )
-    // CHECK-NEXT: pos[0] : ( 0, 5
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5, 7, 9
-    // CHECK-NEXT: crd[1] : ( 0, 3, 1, 4, 2, 0, 3, 1, 4
-    // CHECK-NEXT: values : ( 1, 1.96, 4, 6.25, 9, 16.81, 16, 27.04, 25
+    // CHECK-NEXT: pos[0] : ( 0, 5 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5, 7, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 1, 4, 2, 0, 3, 1, 4 )
+    // CHECK-NEXT: values : ( 1, 1.96, 4, 6.25, 9, 16.81, 16, 27.04, 25 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?x?xf64, #DCSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
index 467a77f30777..b48ff9c9df74 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
@@ -111,29 +111,29 @@ module {
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 4, 3, 2 )
     // CHECK-NEXT: lvl = ( 4, 3, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5
-    // CHECK-NEXT: crd[1] : ( 0, 1, 1, 2, 1
-    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 5, 7, 8
-    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 0, 1, 0
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 1, 2, 1 )
+    // CHECK-NEXT: pos[2] : ( 0, 2, 4, 5, 7, 8 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 0, 1, 0 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 16
     // CHECK-NEXT: dim = ( 4, 3, 2 )
     // CHECK-NEXT: lvl = ( 4, 3, 2 )
-    // CHECK-NEXT: pos[2] : ( 0, 2, 3, 4, 6, 6, 7, 9, 11, 13, 14, 15, 16
-    // CHECK-NEXT: crd[2] : ( 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
+    // CHECK-NEXT: pos[2] : ( 0, 2, 3, 4, 6, 6, 7, 9, 11, 13, 14, 15, 16 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 22
     // CHECK-NEXT: dim = ( 4, 3, 2 )
     // CHECK-NEXT: lvl = ( 4, 3, 2 )
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 8, 11
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 2, 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: values : ( 1, 2, 0, 3, 4, 0, 5, 6, 0, 7, 8, 9, 10, 11, 12, 13, 14, 0, 0, 15, 0, 16
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 8, 11 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 2, 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: values : ( 1, 2, 0, 3, 4, 0, 5, 6, 0, 7, 8, 9, 10, 11, 12, 13, 14, 0, 0, 15, 0, 16 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %s0 : tensor<4x3x2xf32, #CCC>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir
index 39699fbdb14e..7c78bfc36200 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir
@@ -80,15 +80,15 @@ func.func @main() {
   // CHECK-NEXT: nse = 9
   // CHECK-NEXT: dim = ( 1, 3, 3, 1 )
   // CHECK-NEXT: lvl = ( 1, 3, 3, 1 )
-  // CHECK-NEXT: pos[0] : ( 0, 1
-  // CHECK-NEXT: crd[0] : ( 0
-  // CHECK-NEXT: pos[1] : ( 0, 3
-  // CHECK-NEXT: crd[1] : ( 0, 1, 2
-  // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9
-  // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2
-  // CHECK-NEXT: pos[3] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
-  // CHECK-NEXT: crd[3] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0
-  // CHECK-NEXT: values : ( 6, 6, 6, 6, 6, 6, 6, 6, 6
+  // CHECK-NEXT: pos[0] : ( 0, 1 )
+  // CHECK-NEXT: crd[0] : ( 0 )
+  // CHECK-NEXT: pos[1] : ( 0, 3 )
+  // CHECK-NEXT: crd[1] : ( 0, 1, 2 )
+  // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9 )
+  // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2 )
+  // CHECK-NEXT: pos[3] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 )
+  // CHECK-NEXT: crd[3] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0 )
+  // CHECK-NEXT: values : ( 6, 6, 6, 6, 6, 6, 6, 6, 6 )
   // CHECK-NEXT: ----
   //
   sparse_tensor.print %CCCC_ret : tensor<1x3x3x1xf32, #CCCC>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
index b664b7f99944..f3c721535e75 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
@@ -147,7 +147,7 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 5, 0, 0,  )
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 5, 0, 0 )
     // CHECK-NEXT: ----
     sparse_tensor.print %XO : tensor<4x8xi32, #AllDense>
 
@@ -155,7 +155,7 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 8, 4 )
-    // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0,  )
+    // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0 )
     // CHECK-NEXT: ----
     sparse_tensor.print %XT : tensor<4x8xi32, #AllDenseT>
 
@@ -176,9 +176,9 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 2, 2, 5,  )
-    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5,  )
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 2, 2, 5 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5 )
     // CHECK-NEXT: ----
     sparse_tensor.print %a : tensor<4x8xi32, #CSR>
 
@@ -186,11 +186,11 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,  )
-    // CHECK-NEXT: crd[0] : ( 0, 3,  )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 5,  )
-    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5,  )
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 5 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5 )
     // CHECK-NEXT: ----
     sparse_tensor.print %b : tensor<4x8xi32, #DCSR>
 
@@ -198,9 +198,9 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 8, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 1, 3, 4, 4, 5, 5, 5,  )
-    // CHECK-NEXT: crd[1] : ( 0, 0, 3, 3, 3,  )
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 1, 3, 4, 4, 5, 5, 5 )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 3, 3, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5 )
     // CHECK-NEXT: ----
     sparse_tensor.print %c : tensor<4x8xi32, #CSC>
 
@@ -208,11 +208,11 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 8, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 4,  )
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3, 5,  )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 3, 4, 5,  )
-    // CHECK-NEXT: crd[1] : ( 0, 0, 3, 3, 3,  )
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3, 5 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 3, 4, 5 )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 3, 3, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5 )
     // CHECK-NEXT: ----
     sparse_tensor.print %d : tensor<4x8xi32, #DCSC>
 
@@ -220,11 +220,11 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 2, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,  )
-    // CHECK-NEXT: crd[0] : ( 0, 1,  )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 3,  )
-    // CHECK-NEXT: crd[1] : ( 0, 0, 1,  )
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,  )
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0 )
     // CHECK-NEXT: ----
     sparse_tensor.print %e : tensor<4x8xi32, #BSR>
 
@@ -232,11 +232,11 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 4, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,  )
-    // CHECK-NEXT: crd[0] : ( 0, 1,  )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 3,  )
-    // CHECK-NEXT: crd[1] : ( 0, 0, 1,  )
-    // CHECK-NEXT: values : ( 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 5, 0, 0, 0, 0,  )
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 5, 0, 0, 0, 0 )
     // CHECK-NEXT: ----
     sparse_tensor.print %f : tensor<4x8xi32, #BSRC>
 
@@ -244,11 +244,11 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 2, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,  )
-    // CHECK-NEXT: crd[0] : ( 0, 1,  )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3,  )
-    // CHECK-NEXT: crd[1] : ( 0, 1, 1,  )
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,  )
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 1 )
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0 )
     // CHECK-NEXT: ----
     sparse_tensor.print %g : tensor<4x8xi32, #BSC>
 
@@ -256,11 +256,11 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 4, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 2,  )
-    // CHECK-NEXT: crd[0] : ( 0, 1,  )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3,  )
-    // CHECK-NEXT: crd[1] : ( 0, 1, 1,  )
-    // CHECK-NEXT: values : ( 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 5, 0, 0, 0, 0,  )
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 1 )
+    // CHECK-NEXT: values : ( 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 5, 0, 0, 0, 0 )
     // CHECK-NEXT: ----
     sparse_tensor.print %h : tensor<4x8xi32, #BSCC>
 
@@ -268,9 +268,9 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 2, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 3,  )
-    // CHECK-NEXT: crd[1] : ( 0, 0, 1,  )
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,  )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 1 )
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0 )
     // CHECK-NEXT: ----
     sparse_tensor.print %i : tensor<4x8xi32, #BSR0>
 
@@ -278,9 +278,9 @@ module {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 2, 2, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3,  )
-    // CHECK-NEXT: crd[1] : ( 0, 1, 1,  )
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0,  )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 1 )
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0 )
     // CHECK-NEXT: ----
     sparse_tensor.print %j : tensor<4x8xi32, #BSC0>
 
@@ -288,9 +288,9 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 5,  )
-    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 2, 3, 2, 3, 3, 3, 5,  )
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
+    // CHECK-NEXT: pos[0] : ( 0, 5 )
+    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 2, 3, 2, 3, 3, 3, 5 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5 )
     // CHECK-NEXT: ----
     sparse_tensor.print %AoS : tensor<4x8xi32, #COOAoS>
 
@@ -298,10 +298,10 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 5,  )
-    // CHECK-NEXT: crd[0] : ( 0, 0, 3, 3, 3,  )
-    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5,  )
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5,  )
+    // CHECK-NEXT: pos[0] : ( 0, 5 )
+    // CHECK-NEXT: crd[0] : ( 0, 0, 3, 3, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5 )
     // CHECK-NEXT: ----
     sparse_tensor.print %SoA : tensor<4x8xi32, #COOSoA>
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir
index 98dee304fa51..4f1e4312d7bc 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir
@@ -61,11 +61,11 @@ module {
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 2, 4, 8 )
     // CHECK-NEXT: lvl = ( 2, 4, 8 )
-    // CHECK-NEXT: pos[2] : ( ( 0, 8, 16, 24, 32,  )( 0, 8, 16, 24, 32,  ) )
-    // CHECK-NEXT: crd[2] : ( ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,  )
-    // CHECK-SAME:            ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,  ) )
-    // CHECK-NEXT: values : ( ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,  )
-    // CHECK-SAME:            ( 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64,  ) )
+    // CHECK-NEXT: pos[2] : ( ( 0, 8, 16, 24, 32 )( 0, 8, 16, 24, 32 ) )
+    // CHECK-NEXT: crd[2] : ( ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 )
+    // CHECK-SAME:            ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 ) )
+    // CHECK-NEXT: values : ( ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 )
+    // CHECK-SAME:            ( 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 ) )
     // CHECK-NEXT: ----
     sparse_tensor.print %X : tensor<2x4x8xf64, #BatchedCSR>
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
index 7bacbe3b87e4..fc23fe501fcf 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
@@ -93,18 +93,18 @@ module {
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 20, 31,
-    // CHECK-NEXT: values : ( 5.13, 3, 5,
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 20, 31 )
+    // CHECK-NEXT: values : ( 5.13, 3, 5 )
     // CHECK-NEXT: ----
     //
     // CHECK-NEXT: ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 3,
-    // CHECK-NEXT: crd[0] : ( 0, 20, 31,
-    // CHECK-NEXT: values : ( 2, 4, 6,
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 0, 20, 31 )
+    // CHECK-NEXT: values : ( 2, 4, 6 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?xf32, #SparseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
index a927a5dfb94b..5da028c3685c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
@@ -144,33 +144,33 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 4, 5 )
     // CHECK-NEXT: lvl = ( 4, 5 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 2, 3, 4, 0, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 2, 3, 4, 0, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 5, 4 )
     // CHECK-NEXT: lvl = ( 5, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4, 5, 6
-    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1
-    // CHECK-NEXT: values : ( 6, 5, 4, 3, 2, 11
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4, 5, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1 )
+    // CHECK-NEXT: values : ( 6, 5, 4, 3, 2, 11 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9
-    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 0, 1, 3, 0, 1, 3
-    // CHECK-NEXT: values : ( 7, 7, 9, 8, 7, 7, 12, 11, 11
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 0, 1, 3, 0, 1, 3 )
+    // CHECK-NEXT: values : ( 7, 7, 9, 8, 7, 7, 12, 11, 11 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 4, 4 )
     // CHECK-NEXT: lvl = ( 4, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9
-    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 0, 1, 3, 0, 1, 3
-    // CHECK-NEXT: values : ( 7, 7, 9, 8, 7, 7, 12, 11, 11
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 0, 1, 3, 0, 1, 3 )
+    // CHECK-NEXT: values : ( 7, 7, 9, 8, 7, 7, 12, 11, 11 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %sm1 : tensor<?x?xf64, #CSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir
index 18bf6a71c530..d32a92e337ba 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir
@@ -118,33 +118,33 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 4, 5 )
     // CHECK-NEXT: lvl = ( 4, 5 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 2, 3, 4, 0, 2, 3
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 2, 3, 4, 0, 2, 3 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 5, 4 )
     // CHECK-NEXT: lvl = ( 5, 4 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4, 5, 6
-    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1
-    // CHECK-NEXT: values : ( 6, 5, 4, 3, 2, 11
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4, 5, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1 )
+    // CHECK-NEXT: values : ( 6, 5, 4, 3, 2, 11 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 4 )
     // CHECK-NEXT: lvl = ( 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 2, 3, 120, 504
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 2, 3, 120, 504 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 5 )
     // CHECK-NEXT: lvl = ( 5 )
-    // CHECK-NEXT: pos[0] : ( 0, 5
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4
-    // CHECK-NEXT: values : ( 6, 5, 12, 2, 11
+    // CHECK-NEXT: pos[0] : ( 0, 5 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4 )
+    // CHECK-NEXT: values : ( 6, 5, 12, 2, 11 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %sm1 : tensor<?x?xf64, #CSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir
index 4c26ebe6e401..317fe0f225ee 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir
@@ -81,31 +81,31 @@ module {
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 2, 6 )
     // CHECK-NEXT: lvl = ( 2, 6 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 1
-    // CHECK-NEXT: pos[1] : ( 0, 3, 6
-    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 0, 2, 4
-    // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 4, 0, 2, 4 )
+    // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 12 )
     // CHECK-NEXT: lvl = ( 12 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 2, 4, 6, 8, 10
-    // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 4, 6, 8, 10 )
+    // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 2, 3, 2 )
     // CHECK-NEXT: lvl = ( 2, 3, 2 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 1
-    // CHECK-NEXT: pos[1] : ( 0, 3, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: pos[2] : ( 0, 1, 2, 3, 4, 5, 6
-    // CHECK-NEXT: crd[2] : ( 0, 0, 0, 0, 0, 0
-    // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: pos[2] : ( 0, 1, 2, 3, 4, 5, 6 )
+    // CHECK-NEXT: crd[2] : ( 0, 0, 0, 0, 0, 0 )
+    // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %reshaped0: tensor<2x6xf64, #SparseMatrix>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
index 20a8c5f812de..eecd970e01ac 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
@@ -211,22 +211,22 @@ module {
     // CHECK-NEXT: nse = 2
     // CHECK-NEXT: dim = ( 8, 8 )
     // CHECK-NEXT: lvl = ( 8, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 7
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2
-    // CHECK-NEXT: crd[1] : ( 0, 7
-    // CHECK-NEXT: values : ( 96, 192
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 7 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2 )
+    // CHECK-NEXT: crd[1] : ( 0, 7 )
+    // CHECK-NEXT: values : ( 96, 192 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 2
     // CHECK-NEXT: dim = ( 8, 8 )
     // CHECK-NEXT: lvl = ( 8, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 7
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2
-    // CHECK-NEXT: crd[1] : ( 0, 7
-    // CHECK-NEXT: values : ( 96, 192
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 7 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2 )
+    // CHECK-NEXT: crd[1] : ( 0, 7 )
+    // CHECK-NEXT: values : ( 96, 192 )
     // CHECK-NEXT: ----
     //
     %v0 = vector.transfer_read %0[%c0, %c0], %d0
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
index 4e9090ae201d..c62cdc900b83 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
@@ -92,9 +92,9 @@ module {
     // CHECK-NEXT: nse = 16
     // CHECK-NEXT: dim = ( 8, 8 )
     // CHECK-NEXT: lvl = ( 8, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 5, 6, 8, 11, 14, 16
-    // CHECK-NEXT: crd[1] : ( 0, 2, 7, 1, 2, 3, 1, 4, 1, 2, 5, 2, 6, 7, 2, 7
-    // CHECK-NEXT: values : ( 2, 2, 2, 4, 6, 8, 2, 10, 2, 2, 12, 2, 14, 2, 2, 16
+    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 5, 6, 8, 11, 14, 16 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 7, 1, 2, 3, 1, 4, 1, 2, 5, 2, 6, 7, 2, 7 )
+    // CHECK-NEXT: values : ( 2, 2, 2, 4, 6, 8, 2, 10, 2, 2, 12, 2, 14, 2, 2, 16 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %2 : tensor<8x8xf32, #CSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
index dd8396dc23b0..3f0cf70675ba 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
@@ -91,25 +91,25 @@ module @func_sparse.2 {
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 2, 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 1
-    // CHECK-NEXT: pos[1] : ( 0, 3, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 2, 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 1
-    // CHECK-NEXT: pos[1] : ( 0, 3, 6
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2
-    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24
-    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3
-    // CHECK-NEXT: values : ( 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 1 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 )
+    // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 )
+    // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 )
+    // CHECK-NEXT: values : ( 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %sm_t : tensor<2x3x4xf64, #SparseMatrix>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
index 68bc17175e3b..bd61563b4b2d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
@@ -124,33 +124,33 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 10 )
     // CHECK-NEXT: lvl = ( 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 5
-    // CHECK-NEXT: crd[0] : ( 1, 3, 5, 7, 9
-    // CHECK-NEXT: values : ( 1, 2, -4, 0, 5
+    // CHECK-NEXT: pos[0] : ( 0, 5 )
+    // CHECK-NEXT: crd[0] : ( 1, 3, 5, 7, 9 )
+    // CHECK-NEXT: values : ( 1, 2, -4, 0, 5 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 7
     // CHECK-NEXT: dim = ( 5, 5 )
     // CHECK-NEXT: lvl = ( 5, 5 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4, 6, 7
-    // CHECK-NEXT: crd[1] : ( 3, 4, 1, 3, 3, 4, 2
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4, 6, 7 )
+    // CHECK-NEXT: crd[1] : ( 3, 4, 1, 3, 3, 4, 2 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 3
     // CHECK-NEXT: dim = ( 10 )
     // CHECK-NEXT: lvl = ( 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 3
-    // CHECK-NEXT: crd[0] : ( 1, 3, 9
-    // CHECK-NEXT: values : ( 1, 2, 5
+    // CHECK-NEXT: pos[0] : ( 0, 3 )
+    // CHECK-NEXT: crd[0] : ( 1, 3, 9 )
+    // CHECK-NEXT: values : ( 1, 2, 5 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 5, 5 )
     // CHECK-NEXT: lvl = ( 5, 5 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 3, 4, 4
-    // CHECK-NEXT: crd[1] : ( 3, 4, 3, 4
-    // CHECK-NEXT: values : ( 1, 2, 4, 6
+    // CHECK-NEXT: pos[1] : ( 0, 1, 2, 3, 4, 4 )
+    // CHECK-NEXT: crd[1] : ( 3, 4, 3, 4 )
+    // CHECK-NEXT: values : ( 1, 2, 4, 6 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %sv1 : tensor<?xf64, #SparseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir
index f4435c81117b..d96b07a0db33 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir
@@ -91,11 +91,11 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 5, 5 )
     // CHECK-NEXT: lvl = ( 5, 5 )
-    // CHECK-NEXT: pos[0] : ( 0, 5
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8, 9
-    // CHECK-NEXT: crd[1] : ( 0, 1, 1, 2, 2, 3, 3, 4, 4
-    // CHECK-NEXT: values : ( 0.1, 1.1, 1.1, 2.2, 2.1, 3.3, 3.1, 4.4, 4.1
+    // CHECK-NEXT: pos[0] : ( 0, 5 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 1, 2, 2, 3, 3, 4, 4 )
+    // CHECK-NEXT: values : ( 0.1, 1.1, 1.1, 2.2, 2.1, 3.3, 3.1, 4.4, 4.1 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<5x5xf64, #DCSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
index c09374918b7d..11d23d681c82 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
@@ -114,9 +114,9 @@ module {
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 12
-    // CHECK-NEXT: crd[0] : ( 0, 3, 5, 11, 13, 17, 18, 20, 21, 28, 29, 31
-    // CHECK-NEXT: values : ( -1, 1, -1, 1, 1, -1, nan, -nan, 1, -1, -0, 0
+    // CHECK-NEXT: pos[0] : ( 0, 12 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 5, 11, 13, 17, 18, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( -1, 1, -1, 1, 1, -1, nan, -nan, 1, -1, -0, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?xf64, #SparseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
index 7b3f9a2ce0e0..3117d2539f17 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
@@ -107,10 +107,10 @@ module {
     // CHECK-NEXT: nse = 17
     // CHECK-NEXT: dim = ( 4, 256 )
     // CHECK-NEXT: lvl = ( 4, 256 )
-    // CHECK-NEXT: pos[0] : ( 0, 17
-    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 1, 1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3
-    // CHECK-NEXT: crd[1] : ( 0, 126, 127, 254, 1, 253, 2, 0, 1, 3, 98, 126, 127, 128, 249, 253, 255
-    // CHECK-NEXT: values : ( -1, 2, -3, 4, -5, 6, -7, 8, -9, 10, -11, 12, -13, 14, -15, 16, -17
+    // CHECK-NEXT: pos[0] : ( 0, 17 )
+    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 1, 1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 126, 127, 254, 1, 253, 2, 0, 1, 3, 98, 126, 127, 128, 249, 253, 255 )
+    // CHECK-NEXT: values : ( -1, 2, -3, 4, -5, 6, -7, 8, -9, 10, -11, 12, -13, 14, -15, 16, -17 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?x?xf64, #SortedCOO>
@@ -120,10 +120,10 @@ module {
     // CHECK-NEXT: nse = 17
     // CHECK-NEXT: dim = ( 4, 256 )
     // CHECK-NEXT: lvl = ( 256, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 17
-    // CHECK-NEXT: crd[0] : ( 0, 0, 1, 1, 2, 3, 98, 126, 126, 127, 127, 128, 249, 253, 253, 254, 255
-    // CHECK-NEXT: crd[1] : ( 0, 3, 1, 3, 2, 3, 3, 0, 3, 0, 3, 3, 3, 1, 3, 0, 3
-    // CHECK-NEXT: values : ( -1, 8, -5, -9, -7, 10, -11, 2, 12, -3, -13, 14, -15, 6, 16, 4, -17
+    // CHECK-NEXT: pos[0] : ( 0, 17 )
+    // CHECK-NEXT: crd[0] : ( 0, 0, 1, 1, 2, 3, 98, 126, 126, 127, 127, 128, 249, 253, 253, 254, 255 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 1, 3, 2, 3, 3, 0, 3, 0, 3, 3, 3, 1, 3, 0, 3 )
+    // CHECK-NEXT: values : ( -1, 8, -5, -9, -7, 10, -11, 2, 12, -3, -13, 14, -15, 6, 16, 4, -17 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<?x?xf64, #SortedCOOPermuted>
@@ -133,11 +133,11 @@ module {
     // CHECK-NEXT: nse = 17
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 2, 3, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 17
-    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1
-    // CHECK-NEXT: crd[1] : ( 0, 0, 1, 1, 2, 2, 2, 2, 0, 0, 0, 1, 1, 1, 1, 2, 2
-    // CHECK-NEXT: crd[2] : ( 2, 3, 1, 2, 0, 1, 2, 3, 0, 2, 3, 0, 1, 2, 3, 1, 2
-    // CHECK-NEXT: values : ( 3, 63, 11, 100, 66, 61, 13, 43, 77, 10, 46, 61, 53, 3, 75, 22, 18
+    // CHECK-NEXT: pos[0] : ( 0, 17 )
+    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
+    // CHECK-NEXT: crd[1] : ( 0, 0, 1, 1, 2, 2, 2, 2, 0, 0, 0, 1, 1, 1, 1, 2, 2 )
+    // CHECK-NEXT: crd[2] : ( 2, 3, 1, 2, 0, 1, 2, 3, 0, 2, 3, 0, 1, 2, 3, 1, 2 )
+    // CHECK-NEXT: values : ( 3, 63, 11, 100, 66, 61, 13, 43, 77, 10, 46, 61, 53, 3, 75, 22, 18 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %2 : tensor<?x?x?xf64, #SortedCOO3D>
@@ -147,11 +147,11 @@ module {
     // CHECK-NEXT: nse = 17
     // CHECK-NEXT: dim = ( 2, 3, 4 )
     // CHECK-NEXT: lvl = ( 4, 2, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 17
-    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3
-    // CHECK-NEXT: crd[1] : ( 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1
-    // CHECK-NEXT: crd[2] : ( 2, 0, 1, 1, 2, 1, 2, 0, 1, 2, 0, 1, 2, 0, 2, 0, 1
-    // CHECK-NEXT: values : ( 66, 77, 61, 11, 61, 53, 22, 3, 100, 13, 10, 3, 18, 63, 43, 46, 75
+    // CHECK-NEXT: pos[0] : ( 0, 17 )
+    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1 )
+    // CHECK-NEXT: crd[2] : ( 2, 0, 1, 1, 2, 1, 2, 0, 1, 2, 0, 1, 2, 0, 2, 0, 1 )
+    // CHECK-NEXT: values : ( 66, 77, 61, 11, 61, 53, 22, 3, 100, 13, 10, 3, 18, 63, 43, 46, 75 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %3 : tensor<?x?x?xf64, #SortedCOO3DPermuted>
@@ -161,10 +161,10 @@ module {
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 5, 4 )
     // CHECK-NEXT: lvl = ( 5, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 2, 3, 4
-    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1
-    // CHECK-NEXT: values : ( 6, 5, 4, 3, 2, 11
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 2, 3, 4 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1 )
+    // CHECK-NEXT: values : ( 6, 5, 4, 3, 2, 11 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %4 : tensor<?x?xf64, #SortedCOO>
@@ -178,10 +178,10 @@ module {
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 5, 4 )
     // CHECK-NEXT: lvl = ( 5, 4 )
-    // CHECK-NEXT: pos[0] : ( 0, 6
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 2, 3, 4
-    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1
-    // CHECK-NEXT: values : ( 12, 10, 8, 6, 4, 22
+    // CHECK-NEXT: pos[0] : ( 0, 6 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 2, 3, 4 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1 )
+    // CHECK-NEXT: values : ( 12, 10, 8, 6, 4, 22 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %5 : tensor<?x?xf64, #SortedCOO>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
index 2ee189de7906..da87b5cc3c6d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
@@ -111,7 +111,7 @@ module {
     // CHECK-NEXT: nse = 80
     // CHECK-NEXT: dim = ( 10, 8 )
     // CHECK-NEXT: lvl = ( 10, 8 )
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7, 8, 0, 0, 0, 0, 9, 0, 0, 10, 0, 0, 0, 11, 12, 0, 13, 14, 0, 0, 0, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7, 8, 0, 0, 0, 0, 9, 0, 0, 10, 0, 0, 0, 11, 12, 0, 13, 14, 0, 0, 0, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<10x8xf64, #Dense>
@@ -124,9 +124,9 @@ module {
     // CHECK-NEXT: nse = 17
     // CHECK-NEXT: dim = ( 10, 8 )
     // CHECK-NEXT: lvl = ( 10, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 3, 3, 4, 5, 6, 9, 12, 16, 16, 17
-    // CHECK-NEXT: crd[1] : ( 0, 2, 7, 2, 3, 4, 1, 2, 7, 2, 6, 7, 1, 2, 6, 7, 6
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17
+    // CHECK-NEXT: pos[1] : ( 0, 3, 3, 4, 5, 6, 9, 12, 16, 16, 17 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 7, 2, 3, 4, 1, 2, 7, 2, 6, 7, 1, 2, 6, 7, 6 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<10x8xf64, #CSR>
@@ -138,11 +138,11 @@ module {
     // CHECK-NEXT: nse = 17
     // CHECK-NEXT: dim = ( 10, 8 )
     // CHECK-NEXT: lvl = ( 10, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 8
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3, 4, 5, 6, 7, 9
-    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 5, 6, 9, 12, 16, 17
-    // CHECK-NEXT: crd[1] : ( 0, 2, 7, 2, 3, 4, 1, 2, 7, 2, 6, 7, 1, 2, 6, 7, 6
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17
+    // CHECK-NEXT: pos[0] : ( 0, 8 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3, 4, 5, 6, 7, 9 )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 4, 5, 6, 9, 12, 16, 17 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 7, 2, 3, 4, 1, 2, 7, 2, 6, 7, 1, 2, 6, 7, 6 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %2 : tensor<10x8xf64, #DCSR>
@@ -154,9 +154,9 @@ module {
     // CHECK-NEXT: nse = 17
     // CHECK-NEXT: dim = ( 10, 8 )
     // CHECK-NEXT: lvl = ( 8, 10 )
-    // CHECK-NEXT: pos[1] : ( 0, 1, 3, 8, 9, 10, 10, 13, 17
-    // CHECK-NEXT: crd[1] : ( 0, 5, 7, 0, 2, 5, 6, 7, 3, 4, 6, 7, 9, 0, 5, 6, 7
-    // CHECK-NEXT: values : ( 1, 7, 13, 2, 4, 8, 10, 14, 5, 6, 11, 15, 17, 3, 9, 12, 16
+    // CHECK-NEXT: pos[1] : ( 0, 1, 3, 8, 9, 10, 10, 13, 17 )
+    // CHECK-NEXT: crd[1] : ( 0, 5, 7, 0, 2, 5, 6, 7, 3, 4, 6, 7, 9, 0, 5, 6, 7 )
+    // CHECK-NEXT: values : ( 1, 7, 13, 2, 4, 8, 10, 14, 5, 6, 11, 15, 17, 3, 9, 12, 16 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %3 : tensor<10x8xf64, #CSC>
@@ -168,11 +168,11 @@ module {
     // CHECK-NEXT: nse = 17
     // CHECK-NEXT: dim = ( 10, 8 )
     // CHECK-NEXT: lvl = ( 8, 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 7
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 6, 7
-    // CHECK-NEXT: pos[1] : ( 0, 1, 3, 8, 9, 10, 13, 17
-    // CHECK-NEXT: crd[1] : ( 0, 5, 7, 0, 2, 5, 6, 7, 3, 4, 6, 7, 9, 0, 5, 6, 7
-    // CHECK-NEXT: values : ( 1, 7, 13, 2, 4, 8, 10, 14, 5, 6, 11, 15, 17, 3, 9, 12, 16
+    // CHECK-NEXT: pos[0] : ( 0, 7 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 6, 7 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 3, 8, 9, 10, 13, 17 )
+    // CHECK-NEXT: crd[1] : ( 0, 5, 7, 0, 2, 5, 6, 7, 3, 4, 6, 7, 9, 0, 5, 6, 7 )
+    // CHECK-NEXT: values : ( 1, 7, 13, 2, 4, 8, 10, 14, 5, 6, 11, 15, 17, 3, 9, 12, 16 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %4 : tensor<10x8xf64, #DCSC>
@@ -184,9 +184,9 @@ module {
     // CHECK-NEXT: nse = 64
     // CHECK-NEXT: dim = ( 10, 8 )
     // CHECK-NEXT: lvl = ( 10, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 8
-    // CHECK-NEXT: crd[0] : ( 0, 2, 3, 4, 5, 6, 7, 9
-    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7, 8, 0, 0, 0, 0, 9, 0, 0, 10, 0, 0, 0, 11, 12, 0, 13, 14, 0, 0, 0, 15, 16, 0, 0, 0, 0, 0, 0, 17, 0
+    // CHECK-NEXT: pos[0] : ( 0, 8 )
+    // CHECK-NEXT: crd[0] : ( 0, 2, 3, 4, 5, 6, 7, 9 )
+    // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7, 8, 0, 0, 0, 0, 9, 0, 0, 10, 0, 0, 0, 11, 12, 0, 13, 14, 0, 0, 0, 15, 16, 0, 0, 0, 0, 0, 0, 17, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %x : tensor<10x8xf64, #BlockRow>
@@ -198,9 +198,9 @@ module {
     // CHECK-NEXT: nse = 70
     // CHECK-NEXT: dim = ( 10, 8 )
     // CHECK-NEXT: lvl = ( 8, 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 7
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 6, 7
-    // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 13, 0, 0, 2, 0, 4, 0, 0, 8, 10, 14, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 15, 0, 17, 3, 0, 0, 0, 0, 9, 12, 16, 0, 0
+    // CHECK-NEXT: pos[0] : ( 0, 7 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 6, 7 )
+    // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 13, 0, 0, 2, 0, 4, 0, 0, 8, 10, 14, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 15, 0, 17, 3, 0, 0, 0, 0, 9, 12, 16, 0, 0 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %y : tensor<10x8xf64, #BlockCol>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
index 29bc744c9920..748fffc1e637 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
@@ -77,9 +77,9 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 9
-    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31
-    // CHECK-NEXT: values : ({{ -0.761[0-9]*, 0.761[0-9]*, 0.96[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 1}}
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ({{ -0.761[0-9]*, 0.761[0-9]*, 0.96[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 1}} )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?xf64, #SparseVector>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
index 67155201c584..fe2f2690e860 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
@@ -110,13 +110,13 @@ module {
     // CHECK-NEXT: nse = 4
     // CHECK-NEXT: dim = ( 3, 3, 5 )
     // CHECK-NEXT: lvl = ( 3, 3, 5 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 2
-    // CHECK-NEXT: pos[1] : ( 0, 1, 3
-    // CHECK-NEXT: crd[1] : ( 2, 0, 2
-    // CHECK-NEXT: pos[2] : ( 0, 2, 3, 4
-    // CHECK-NEXT: crd[2] : ( 0, 2, 0, 2
-    // CHECK-NEXT: values : ( 2.4, 3.5, 2, 8
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 1, 3 )
+    // CHECK-NEXT: crd[1] : ( 2, 0, 2 )
+    // CHECK-NEXT: pos[2] : ( 0, 2, 3, 4 )
+    // CHECK-NEXT: crd[2] : ( 0, 2, 0, 2 )
+    // CHECK-NEXT: values : ( 2.4, 3.5, 2, 8 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?x?x?xf64, #ST>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
index 356808ebee3f..a46c3a8d5ef6 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
@@ -97,23 +97,23 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 3, 4, 8 )
     // CHECK-NEXT: lvl = ( 3, 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 2
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3
-    // CHECK-NEXT: crd[1] : ( 0, 3, 2
-    // CHECK-NEXT: pos[2] : ( 0, 1, 2, 5
-    // CHECK-NEXT: crd[2] : ( 0, 7, 1, 2, 7
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 2 )
+    // CHECK-NEXT: pos[2] : ( 0, 1, 2, 5 )
+    // CHECK-NEXT: crd[2] : ( 0, 7, 1, 2, 7 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 24
     // CHECK-NEXT: dim = ( 3, 4, 8 )
     // CHECK-NEXT: lvl = ( 3, 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 2
-    // CHECK-NEXT: crd[0] : ( 0, 2
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3
-    // CHECK-NEXT: crd[1] : ( 0, 3, 2
-    // CHECK-NEXT: values : ( 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 8, 0, 0, 0, 0, 10
+    // CHECK-NEXT: pos[0] : ( 0, 2 )
+    // CHECK-NEXT: crd[0] : ( 0, 2 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 2 )
+    // CHECK-NEXT: values : ( 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 8, 0, 0, 0, 0, 10 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %st : tensor<?x?x?xf64, #ST1>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
index 549c2082fcb3..434cc9509464 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
@@ -119,21 +119,21 @@ module {
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 4, 3 )
     // CHECK-NEXT: lvl = ( 4, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4, 6
-    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2
-    // CHECK-NEXT: values : ( 1.1, 3.1, 1.2, 3.3, 1.4, 3.4
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2 )
+    // CHECK-NEXT: values : ( 1.1, 3.1, 1.2, 3.3, 1.4, 3.4 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 6
     // CHECK-NEXT: dim = ( 4, 3 )
     // CHECK-NEXT: lvl = ( 4, 3 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4, 6
-    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2
-    // CHECK-NEXT: values : ( 1.1, 3.1, 1.2, 3.3, 1.4, 3.4
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4, 6 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2 )
+    // CHECK-NEXT: values : ( 1.1, 3.1, 1.2, 3.3, 1.4, 3.4 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<4x3xf64, #DCSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
index cc6f6a068746..3b7760e5052c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
@@ -83,19 +83,19 @@ module {
     // CHECK-NEXT: nse = 50
     // CHECK-NEXT: dim = ( 10, 5 )
     // CHECK-NEXT: lvl = ( 10, 5 )
-    // CHECK-NEXT: pos[0] : ( 0, 50
-    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4
-    // CHECK-NEXT: values : ( 10, 20, 30, 40, 50, 11, 21, 31, 41, 51, 12, 22, 32, 42, 52, 13, 23, 33, 43, 53, 14, 24, 34, 44, 54, 15, 25, 35, 45, 55, 16, 26, 36, 46, 56, 17, 27, 37, 47, 57, 18, 28, 38, 48, 58, 19, 29, 39, 49, 59
+    // CHECK-NEXT: pos[0] : ( 0, 50 )
+    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4 )
+    // CHECK-NEXT: values : ( 10, 20, 30, 40, 50, 11, 21, 31, 41, 51, 12, 22, 32, 42, 52, 13, 23, 33, 43, 53, 14, 24, 34, 44, 54, 15, 25, 35, 45, 55, 16, 26, 36, 46, 56, 17, 27, 37, 47, 57, 18, 28, 38, 48, 58, 19, 29, 39, 49, 59 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 50
     // CHECK-NEXT: dim = ( 5, 10 )
     // CHECK-NEXT: lvl = ( 5, 10 )
-    // CHECK-NEXT: pos[0] : ( 0, 50
-    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
-    // CHECK-NEXT: values : ( 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59
+    // CHECK-NEXT: pos[0] : ( 0, 50 )
+    // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 )
+    // CHECK-NEXT: values : ( 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %SA : tensor<10x5xf32, #SortedCOO>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
index 3da1e35818cf..acb7a99a3418 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
@@ -247,53 +247,53 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 9
-    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 23
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 23
-    // CHECK-NEXT: crd[0] : ( 1, 2, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 22, 23, 24, 25, 26, 27, 30
-    // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
+    // CHECK-NEXT: pos[0] : ( 0, 23 )
+    // CHECK-NEXT: crd[0] : ( 1, 2, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 22, 23, 24, 25, 26, 27, 30 )
+    // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 32
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
-    // CHECK-NEXT: values : ( -1, 1, 1, -2, 1, 1, 1, 1, 1, 1, 1, -3, 1, 1, 1, 1, 1, -4, 1, 1, -5, -6, 1, 1, 1, 1, 1, 1, -7, -8, 1, -9
+    // CHECK-NEXT: pos[0] : ( 0, 32 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 )
+    // CHECK-NEXT: values : ( -1, 1, 1, -2, 1, 1, 1, 1, 1, 1, 1, -3, 1, 1, 1, 1, 1, -4, 1, 1, -5, -6, 1, 1, 1, 1, 1, 1, -7, -8, 1, -9 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 9
-    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31
-    // CHECK-NEXT: values : ( 0, 6, 33, 68, 100, 126, 196, 232, 279
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 0, 6, 33, 68, 100, 126, 196, 232, 279 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9
-    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3
-    // CHECK-NEXT: values : ( 3, 3, 3, 4, 5, 6, 7, 7, 7
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3 )
+    // CHECK-NEXT: values : ( 3, 3, 3, 4, 5, 6, 7, 7, 7 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 4, 8 )
     // CHECK-NEXT: lvl = ( 4, 8 )
-    // CHECK-NEXT: pos[0] : ( 0, 4
-    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9
-    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3
-    // CHECK-NEXT: values : ( 99, 99, 99, 99, 5, 6, 99, 99, 99
+    // CHECK-NEXT: pos[0] : ( 0, 4 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3 )
+    // CHECK-NEXT: values : ( 99, 99, 99, 99, 5, 6, 99, 99, 99 )
     // CHECK-NEXT: ----
     // CHECK-NEXT: ( 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0 )
     //
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
index 553323331641..10ccf47c3408 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
@@ -209,55 +209,55 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 9
-    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31
-    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 10
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 10
-    // CHECK-NEXT: crd[0] : ( 1, 3, 4, 10, 16, 18, 21, 28, 29, 31
-    // CHECK-NEXT: values : ( 11, 12, 13, 14, 15, 16, 17, 18, 19, 20
+    // CHECK-NEXT: pos[0] : ( 0, 10 )
+    // CHECK-NEXT: crd[0] : ( 1, 3, 4, 10, 16, 18, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 9
-    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31
-    // CHECK-NEXT: values : ( 2, 4, 6, 8, 10, 12, 14, 16, 18
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 2, 4, 6, 8, 10, 12, 14, 16, 18 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 9
-    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31
-    // CHECK-NEXT: values : ( 2, 4, 6, 8, 10, 12, 14, 16, 18
+    // CHECK-NEXT: pos[0] : ( 0, 9 )
+    // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 2, 4, 6, 8, 10, 12, 14, 16, 18 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 14
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 14
-    // CHECK-NEXT: crd[0] : ( 0, 1, 3, 4, 10, 11, 16, 17, 18, 20, 21, 28, 29, 31
-    // CHECK-NEXT: values : ( 2, 11, 16, 13, 14, 6, 15, 8, 16, 10, 29, 32, 35, 38
+    // CHECK-NEXT: pos[0] : ( 0, 14 )
+    // CHECK-NEXT: crd[0] : ( 0, 1, 3, 4, 10, 11, 16, 17, 18, 20, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 2, 11, 16, 13, 14, 6, 15, 8, 16, 10, 29, 32, 35, 38 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: pos[0] : ( 0, 5
-    // CHECK-NEXT: crd[0] : ( 3, 21, 28, 29, 31
-    // CHECK-NEXT: values : ( 48, 204, 252, 304, 360
+    // CHECK-NEXT: pos[0] : ( 0, 5 )
+    // CHECK-NEXT: crd[0] : ( 3, 21, 28, 29, 31 )
+    // CHECK-NEXT: values : ( 48, 204, 252, 304, 360 )
     // CHECK-NEXT: ----
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 32
     // CHECK-NEXT: dim = ( 32 )
     // CHECK-NEXT: lvl = ( 32 )
-    // CHECK-NEXT: values : ( 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 252, 304, 0, 360
+    // CHECK-NEXT: values : ( 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 252, 304, 0, 360 )
     // CHECK-NEXT: ----
     // CHECK-NEXT: 1169.1
     //
diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir
index 9413119509c6..bd71409892f4 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir
@@ -68,9 +68,9 @@ module {
     // CHECK-NEXT: nse = 20
     // CHECK-NEXT: dim = ( 8, 8 )
     // CHECK-NEXT: lvl = ( 8, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 5, 5, 6, 7, 8, 12, 16, 20,  )
-    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 6, 7, 2, 3, 4, 1, 2, 6, 7, 1, 2, 6, 7, 1, 2, 6, 7,  )
-    // CHECK-NEXT: values : ( 1, 39, 52, 45, 51, 16, 25, 36, 117, 158, 135, 144, 156, 318, 301, 324, 208, 430, 405, 436,  )
+    // CHECK-NEXT: pos[1] : ( 0, 5, 5, 6, 7, 8, 12, 16, 20 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 2, 6, 7, 2, 3, 4, 1, 2, 6, 7, 1, 2, 6, 7, 1, 2, 6, 7 )
+    // CHECK-NEXT: values : ( 1, 39, 52, 45, 51, 16, 25, 36, 117, 158, 135, 144, 156, 318, 301, 324, 208, 430, 405, 436 )
     // CHECK-NEXT: ----
     sparse_tensor.print %Ccsr : tensor<8x8xf32, #CSR>
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir
index 3b3d074f7e2a..64f289626c07 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir
@@ -117,9 +117,9 @@ module {
     // CHECK-NEXT: nse = 9
     // CHECK-NEXT: dim = ( 5, 5 )
     // CHECK-NEXT: lvl = ( 5, 5 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5, 7, 9,  )
-    // CHECK-NEXT: crd[1] : ( 0, 3, 1, 4, 2, 0, 3, 1, 4,  )
-    // CHECK-NEXT: values : ( 11, 41.4, 42, 102.5, 93, 44.1, 164, 105.2, 255,  )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5, 7, 9 )
+    // CHECK-NEXT: crd[1] : ( 0, 3, 1, 4, 2, 0, 3, 1, 4 )
+    // CHECK-NEXT: values : ( 11, 41.4, 42, 102.5, 93, 44.1, 164, 105.2, 255 )
     // CHECK-NEXT: ----
     sparse_tensor.print %0 : tensor<?x?xf32, #CSR>
 
@@ -145,9 +145,9 @@ module {
     // CHECK-NEXT: nse = 5
     // CHECK-NEXT: dim = ( 8, 8 )
     // CHECK-NEXT: lvl = ( 8, 8 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 3, 4, 4, 4, 4, 5,  )
-    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 4, 7,  )
-    // CHECK-NEXT: values : ( 17, 18, 19, 20, 21,  )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3, 3, 4, 4, 4, 4, 5 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 0, 4, 7 )
+    // CHECK-NEXT: values : ( 17, 18, 19, 20, 21 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %1 : tensor<?x?xf32, #CSR>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir
index 18f59f59a9f0..4b503ae0d110 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir
@@ -170,18 +170,18 @@ module {
     // CHECK-NEXT: nse = 8
     // CHECK-NEXT: dim = ( 4, 6 )
     // CHECK-NEXT: lvl = ( 4, 6 )
-    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7, 8,  )
-    // CHECK-NEXT: crd[1] : ( 0, 1, 4, 1, 5, 2, 3, 2,  )
-    // CHECK-NEXT: values : ( 5, 10, 24, 19, 53, 42, 55, 56,  )
+    // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7, 8 )
+    // CHECK-NEXT: crd[1] : ( 0, 1, 4, 1, 5, 2, 3, 2 )
+    // CHECK-NEXT: values : ( 5, 10, 24, 19, 53, 42, 55, 56 )
     // CHECK-NEXT: ----
     //
     // CHECK:      ---- Sparse Tensor ----
     // CHECK-NEXT: nse = 12
     // CHECK-NEXT: dim = ( 4, 6 )
     // CHECK-NEXT: lvl = ( 2, 3, 2, 2 )
-    // CHECK-NEXT: pos[1] : ( 0, 2, 3,  )
-    // CHECK-NEXT: crd[1] : ( 0, 2, 1,  )
-    // CHECK-NEXT: values : ( 5, 10, 8, 19, 24, 24, 40, 53, 42, 55, 56, 64,  )
+    // CHECK-NEXT: pos[1] : ( 0, 2, 3 )
+    // CHECK-NEXT: crd[1] : ( 0, 2, 1 )
+    // CHECK-NEXT: values : ( 5, 10, 8, 19, 24, 24, 40, 53, 42, 55, 56, 64 )
     // CHECK-NEXT: ----
     //
     sparse_tensor.print %0 : tensor<?x?xf32, #CSR>
-- 
GitLab


From 9e1a49cba71416d7e425eb420e86e1120199f4de Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Thu, 9 May 2024 17:15:35 +0100
Subject: [PATCH 0308/1206] [AArch64][ARM] Add tests for frem power2 lowering.
 NFC

---
 llvm/test/CodeGen/AArch64/frem-power2.ll | 479 +++++++++++++++++++++++
 llvm/test/CodeGen/ARM/frem-power2.ll     |  50 +++
 2 files changed, 529 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/frem-power2.ll
 create mode 100644 llvm/test/CodeGen/ARM/frem-power2.ll

diff --git a/llvm/test/CodeGen/AArch64/frem-power2.ll b/llvm/test/CodeGen/AArch64/frem-power2.ll
new file mode 100644
index 000000000000..5d627fcd6b65
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/frem-power2.ll
@@ -0,0 +1,479 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+define float @frem2(float %x) {
+; CHECK-LABEL: frem2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s1, #2.00000000
+; CHECK-NEXT:    b fmodf
+entry:
+  %fmod = frem float %x, 2.0
+  ret float %fmod
+}
+
+define float @frem2_nsz(float %x) {
+; CHECK-LABEL: frem2_nsz:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s1, #2.00000000
+; CHECK-NEXT:    b fmodf
+entry:
+  %fmod = frem nsz float %x, 2.0
+  ret float %fmod
+}
+
+define float @frem2_fast(float %x) {
+; CHECK-LABEL: frem2_fast:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s1, #2.00000000
+; CHECK-NEXT:    b fmodf
+entry:
+  %fmod = frem fast float %x, 2.0
+  ret float %fmod
+}
+
+define float @frem2_abs(float %x) {
+; CHECK-LABEL: frem2_abs:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fabs s0, s0
+; CHECK-NEXT:    fmov s1, #2.00000000
+; CHECK-NEXT:    b fmodf
+entry:
+  %a = tail call float @llvm.fabs.f32(float %x)
+  %fmod = frem float %a, 2.0
+  ret float %fmod
+}
+
+define half @hrem2_nsz(half %x) {
+; CHECK-SD-LABEL: hrem2_nsz:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-SD-NEXT:    fcvt s0, h0
+; CHECK-SD-NEXT:    fmov s1, #2.00000000
+; CHECK-SD-NEXT:    bl fmodf
+; CHECK-SD-NEXT:    fcvt h0, s0
+; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: hrem2_nsz:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-GI-NEXT:    fmov h1, #2.00000000
+; CHECK-GI-NEXT:    fcvt s0, h0
+; CHECK-GI-NEXT:    fcvt s1, h1
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    fcvt h0, s0
+; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
+entry:
+  %fmod = frem nsz half %x, 2.0
+  ret half %fmod
+}
+
+define double @drem2_nsz(double %x) {
+; CHECK-LABEL: drem2_nsz:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov d1, #2.00000000
+; CHECK-NEXT:    b fmod
+entry:
+  %fmod = frem nsz double %x, 2.0
+  ret double %fmod
+}
+
+define float @frem3_nsz(float %x) {
+; CHECK-LABEL: frem3_nsz:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s1, #3.00000000
+; CHECK-NEXT:    b fmodf
+entry:
+  %fmod = frem nsz float %x, 3.0
+  ret float %fmod
+}
+
+define float @frem05_nsz(float %x) {
+; CHECK-LABEL: frem05_nsz:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s1, #0.50000000
+; CHECK-NEXT:    b fmodf
+entry:
+  %fmod = frem nsz float %x, 0.5
+  ret float %fmod
+}
+
+define float @frem1_nsz(float %x) {
+; CHECK-LABEL: frem1_nsz:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s1, #1.00000000
+; CHECK-NEXT:    b fmodf
+entry:
+  %fmod = frem nsz float %x, 1.0
+  ret float %fmod
+}
+
+define float @frem0_nsz(float %x) {
+; CHECK-LABEL: frem0_nsz:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi d1, #0000000000000000
+; CHECK-NEXT:    b fmodf
+entry:
+  %fmod = frem nsz float %x, 0.0
+  ret float %fmod
+}
+
+define float @fremm2_nsz(float %x) {
+; CHECK-LABEL: fremm2_nsz:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s1, #-2.00000000
+; CHECK-NEXT:    b fmodf
+entry:
+  %fmod = frem nsz float %x, -2.0
+  ret float %fmod
+}
+
+define float @frem4_abs(float %x) {
+; CHECK-LABEL: frem4_abs:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fabs s0, s0
+; CHECK-NEXT:    fmov s1, #4.00000000
+; CHECK-NEXT:    b fmodf
+entry:
+  %a = tail call float @llvm.fabs.f32(float %x)
+  %fmod = frem float %a, 4.0
+  ret float %fmod
+}
+
+define float @frem16_abs(float %x) {
+; CHECK-LABEL: frem16_abs:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fabs s0, s0
+; CHECK-NEXT:    fmov s1, #16.00000000
+; CHECK-NEXT:    b fmodf
+entry:
+  %a = tail call float @llvm.fabs.f32(float %x)
+  %fmod = frem float %a, 16.0
+  ret float %fmod
+}
+
+define float @frem4294967296_abs(float %x) {
+; CHECK-LABEL: frem4294967296_abs:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fabs s0, s0
+; CHECK-NEXT:    mov w8, #1333788672 // =0x4f800000
+; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    b fmodf
+entry:
+  %a = tail call float @llvm.fabs.f32(float %x)
+  %fmod = frem float %a, 4294967296.0
+  ret float %fmod
+}
+
+define float @frem1152921504606846976_abs(float %x) {
+; CHECK-LABEL: frem1152921504606846976_abs:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fabs s0, s0
+; CHECK-NEXT:    mov w8, #1568669696 // =0x5d800000
+; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    b fmodf
+entry:
+  %a = tail call float @llvm.fabs.f32(float %x)
+  %fmod = frem float %a, 1152921504606846976.0
+  ret float %fmod
+}
+
+define float @frem4611686018427387904_abs(float %x) {
+; CHECK-LABEL: frem4611686018427387904_abs:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fabs s0, s0
+; CHECK-NEXT:    mov w8, #1585446912 // =0x5e800000
+; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    b fmodf
+entry:
+  %a = tail call float @llvm.fabs.f32(float %x)
+  %fmod = frem float %a, 4611686018427387904.0
+  ret float %fmod
+}
+
+define float @frem9223372036854775808_abs(float %x) {
+; CHECK-LABEL: frem9223372036854775808_abs:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fabs s0, s0
+; CHECK-NEXT:    movi v1.2s, #95, lsl #24
+; CHECK-NEXT:    b fmodf
+entry:
+  %a = tail call float @llvm.fabs.f32(float %x)
+  %fmod = frem float %a, 9223372036854775808.0
+  ret float %fmod
+}
+
+define <4 x float> @frem2_nsz_vec(<4 x float> %x) {
+; CHECK-SD-LABEL: frem2_nsz_vec:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sub sp, sp, #48
+; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-SD-NEXT:    fmov s1, #2.00000000
+; CHECK-SD-NEXT:    bl fmodf
+; CHECK-SD-NEXT:    fmov s1, #2.00000000
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-SD-NEXT:    bl fmodf
+; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-SD-NEXT:    fmov s1, #2.00000000
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    mov s0, v0.s[2]
+; CHECK-SD-NEXT:    bl fmodf
+; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    mov s0, v0.s[3]
+; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    fmov s1, #2.00000000
+; CHECK-SD-NEXT:    bl fmodf
+; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-SD-NEXT:    mov v1.s[3], v0.s[0]
+; CHECK-SD-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NEXT:    add sp, sp, #48
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem2_nsz_vec:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sub sp, sp, #80
+; CHECK-GI-NEXT:    str d10, [sp, #48] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #56] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    str x30, [sp, #72] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-GI-NEXT:    .cfi_offset w30, -8
+; CHECK-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-GI-NEXT:    .cfi_offset b9, -24
+; CHECK-GI-NEXT:    .cfi_offset b10, -32
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-GI-NEXT:    mov s9, v0.s[2]
+; CHECK-GI-NEXT:    mov s10, v0.s[3]
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    fmov s0, s8
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    fmov s0, s9
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    fmov s0, s10
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    ldr x30, [sp, #72] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #56] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldr d10, [sp, #48] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    mov v1.s[1], v2.s[0]
+; CHECK-GI-NEXT:    ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v1.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v0.s[0]
+; CHECK-GI-NEXT:    mov v0.16b, v1.16b
+; CHECK-GI-NEXT:    add sp, sp, #80
+; CHECK-GI-NEXT:    ret
+entry:
+  %fmod = frem nsz <4 x float> %x, <float 2.0, float 2.0, float 2.0, float 2.0>
+  ret <4 x float> %fmod
+}
+
+define <4 x float> @frem1152921504606846976_absv(<4 x float> %x) {
+; CHECK-SD-LABEL: frem1152921504606846976_absv:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sub sp, sp, #48
+; CHECK-SD-NEXT:    str d8, [sp, #32] // 8-byte Folded Spill
+; CHECK-SD-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT:    .cfi_offset w30, -8
+; CHECK-SD-NEXT:    .cfi_offset b8, -16
+; CHECK-SD-NEXT:    fabs v0.4s, v0.4s
+; CHECK-SD-NEXT:    mov w8, #1568669696 // =0x5d800000
+; CHECK-SD-NEXT:    fmov s8, w8
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-SD-NEXT:    fmov s1, s8
+; CHECK-SD-NEXT:    bl fmodf
+; CHECK-SD-NEXT:    fmov s1, s8
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-SD-NEXT:    bl fmodf
+; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-SD-NEXT:    fmov s1, s8
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    mov s0, v0.s[2]
+; CHECK-SD-NEXT:    bl fmodf
+; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    mov s0, v0.s[3]
+; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    fmov s1, s8
+; CHECK-SD-NEXT:    bl fmodf
+; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ldr d8, [sp, #32] // 8-byte Folded Reload
+; CHECK-SD-NEXT:    mov v1.s[3], v0.s[0]
+; CHECK-SD-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NEXT:    add sp, sp, #48
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem1152921504606846976_absv:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sub sp, sp, #96
+; CHECK-GI-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-GI-NEXT:    .cfi_offset b8, -24
+; CHECK-GI-NEXT:    .cfi_offset b9, -32
+; CHECK-GI-NEXT:    .cfi_offset b10, -40
+; CHECK-GI-NEXT:    .cfi_offset b11, -48
+; CHECK-GI-NEXT:    mov w8, #1568669696 // =0x5d800000
+; CHECK-GI-NEXT:    fabs v0.4s, v0.4s
+; CHECK-GI-NEXT:    fmov s11, w8
+; CHECK-GI-NEXT:    fmov s1, s11
+; CHECK-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-GI-NEXT:    mov s9, v0.s[2]
+; CHECK-GI-NEXT:    mov s10, v0.s[3]
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    fmov s1, s11
+; CHECK-GI-NEXT:    fmov s0, s8
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    fmov s1, s11
+; CHECK-GI-NEXT:    fmov s0, s9
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    fmov s1, s11
+; CHECK-GI-NEXT:    fmov s0, s10
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v1.s[1], v2.s[0]
+; CHECK-GI-NEXT:    ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v1.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v0.s[0]
+; CHECK-GI-NEXT:    mov v0.16b, v1.16b
+; CHECK-GI-NEXT:    add sp, sp, #96
+; CHECK-GI-NEXT:    ret
+entry:
+  %a = tail call <4 x float> @llvm.fabs.v4f32(<4 x float> %x)
+  %fmod = frem <4 x float> %a, <float 1152921504606846976.0, float 1152921504606846976.0, float 1152921504606846976.0, float 1152921504606846976.0>
+  ret <4 x float> %fmod
+}
+
+define float @frem2_nsz_sitofp(float %x, i32 %sa) {
+; CHECK-LABEL: frem2_nsz_sitofp:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #1 // =0x1
+; CHECK-NEXT:    lsl w8, w8, w0
+; CHECK-NEXT:    scvtf s1, w8
+; CHECK-NEXT:    b fmodf
+entry:
+  %s = shl i32 1, %sa
+  %y = sitofp i32 %s to float
+  %fmod = frem nsz float %x, %y
+  ret float %fmod
+}
+
+define float @frem2_nsz_uitofp(float %x, i32 %sa) {
+; CHECK-LABEL: frem2_nsz_uitofp:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #1 // =0x1
+; CHECK-NEXT:    lsl w8, w8, w0
+; CHECK-NEXT:    ucvtf s1, w8
+; CHECK-NEXT:    b fmodf
+entry:
+  %s = shl i32 1, %sa
+  %y = uitofp i32 %s to float
+  %fmod = frem nsz float %x, %y
+  ret float %fmod
+}
+
+define float @frem2_const_sitofp(float %x, i32 %sa) {
+; CHECK-SD-LABEL: frem2_const_sitofp:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #1 // =0x1
+; CHECK-SD-NEXT:    fmov s0, #12.50000000
+; CHECK-SD-NEXT:    lsl w8, w8, w0
+; CHECK-SD-NEXT:    scvtf s1, w8
+; CHECK-SD-NEXT:    b fmodf
+;
+; CHECK-GI-LABEL: frem2_const_sitofp:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    and w9, w0, #0x1f
+; CHECK-GI-NEXT:    fmov s0, #12.50000000
+; CHECK-GI-NEXT:    lsl w8, w8, w9
+; CHECK-GI-NEXT:    scvtf s1, w8
+; CHECK-GI-NEXT:    b fmodf
+entry:
+  %sa2 = and i32 %sa, 31
+  %s = shl i32 1, %sa2
+  %y = sitofp i32 %s to float
+  %fmod = frem float 12.50, %y
+  ret float %fmod
+}
+
+define float @frem2_constneg_sitofp(float %x, i32 %sa) {
+; CHECK-SD-LABEL: frem2_constneg_sitofp:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #1 // =0x1
+; CHECK-SD-NEXT:    fmov s0, #-12.50000000
+; CHECK-SD-NEXT:    lsl w8, w8, w0
+; CHECK-SD-NEXT:    scvtf s1, w8
+; CHECK-SD-NEXT:    b fmodf
+;
+; CHECK-GI-LABEL: frem2_constneg_sitofp:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    and w9, w0, #0x1f
+; CHECK-GI-NEXT:    fmov s0, #-12.50000000
+; CHECK-GI-NEXT:    lsl w8, w8, w9
+; CHECK-GI-NEXT:    scvtf s1, w8
+; CHECK-GI-NEXT:    b fmodf
+entry:
+  %sa2 = and i32 %sa, 31
+  %s = shl i32 1, %sa2
+  %y = sitofp i32 %s to float
+  %fmod = frem float -12.50, %y
+  ret float %fmod
+}
diff --git a/llvm/test/CodeGen/ARM/frem-power2.ll b/llvm/test/CodeGen/ARM/frem-power2.ll
new file mode 100644
index 000000000000..8052c8c35bcf
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/frem-power2.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=thumbv8m.main-none-eabi %s -o - | FileCheck %s --check-prefix=CHECK-SOFT
+; RUN: llc -mtriple=thumbv8m.main-none-eabi -mattr=+fp-armv8 %s -o - | FileCheck %s --check-prefix=CHECK-FP
+; RUN: llc -mtriple=thumbv8m.main-none-eabi -mattr=+fp-armv8,+slowfpvfmx %s -o - | FileCheck %s --check-prefix=CHECK-M33
+
+define float @frem4(float %x) {
+; CHECK-SOFT-LABEL: frem4:
+; CHECK-SOFT:       @ %bb.0: @ %entry
+; CHECK-SOFT-NEXT:    .save {r7, lr}
+; CHECK-SOFT-NEXT:    push {r7, lr}
+; CHECK-SOFT-NEXT:    mov.w r1, #1082130432
+; CHECK-SOFT-NEXT:    bl fmodf
+; CHECK-SOFT-NEXT:    pop {r7, pc}
+;
+; CHECK-FP-LABEL: frem4:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    mov.w r1, #1082130432
+; CHECK-FP-NEXT:    b fmodf
+;
+; CHECK-M33-LABEL: frem4:
+; CHECK-M33:       @ %bb.0: @ %entry
+; CHECK-M33-NEXT:    mov.w r1, #1082130432
+; CHECK-M33-NEXT:    b fmodf
+entry:
+  %fmod = frem float %x, 4.0
+  ret float %fmod
+}
+
+define float @frem4_nsz(float %x) {
+; CHECK-SOFT-LABEL: frem4_nsz:
+; CHECK-SOFT:       @ %bb.0: @ %entry
+; CHECK-SOFT-NEXT:    .save {r7, lr}
+; CHECK-SOFT-NEXT:    push {r7, lr}
+; CHECK-SOFT-NEXT:    mov.w r1, #1082130432
+; CHECK-SOFT-NEXT:    bl fmodf
+; CHECK-SOFT-NEXT:    pop {r7, pc}
+;
+; CHECK-FP-LABEL: frem4_nsz:
+; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK-FP-NEXT:    mov.w r1, #1082130432
+; CHECK-FP-NEXT:    b fmodf
+;
+; CHECK-M33-LABEL: frem4_nsz:
+; CHECK-M33:       @ %bb.0: @ %entry
+; CHECK-M33-NEXT:    mov.w r1, #1082130432
+; CHECK-M33-NEXT:    b fmodf
+entry:
+  %fmod = frem nsz float %x, 4.0
+  ret float %fmod
+}
-- 
GitLab


From 937643b8e1017ce6456de0c05b1673bd9ed0800d Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Thu, 9 May 2024 18:39:08 +0200
Subject: [PATCH 0309/1206] [libc++][test] Fixes constexpr char_traits.
 (#90981)

The issue in nasty_char_traits was discovered by @StephanTLavavej who
provided
the solution they use in MSVC STL. This solution is based on that
example.

The same issue affects the constexpr_char_traits which was discovered in
https://github.com/llvm/llvm-project/pull/88389. This uses the same fix.

Fixes: https://github.com/llvm/llvm-project/issues/74221
---
 libcxx/test/support/constexpr_char_traits.h | 57 +++++++++++++++------
 libcxx/test/support/nasty_string.h          | 12 +++--
 2 files changed, 48 insertions(+), 21 deletions(-)

diff --git a/libcxx/test/support/constexpr_char_traits.h b/libcxx/test/support/constexpr_char_traits.h
index 75380d5a7ffb..7c487c504af1 100644
--- a/libcxx/test/support/constexpr_char_traits.h
+++ b/libcxx/test/support/constexpr_char_traits.h
@@ -16,6 +16,31 @@
 
 #include "test_macros.h"
 
+// Tests whether the pointer p is in the range [first, last).
+//
+// Precondition: The range [first, last) is a valid range.
+//
+// Typically the pointers are compared with less than. This is not allowed when
+// the pointers belong to different ranges, which is UB. Typically, this is
+// benign at run-time, however since UB is not allowed during constant
+// evaluation this does not compile. This function does the validation without
+// UB.
+//
+// When p is in the range [first, last) the data can be copied from the
+// beginning to the end. Otherwise it needs to be copied from the end to the
+// beginning.
+template <class CharT>
+TEST_CONSTEXPR_CXX14 bool is_pointer_in_range(const CharT* first, const CharT* last, const CharT* p) {
+  if (first == p) // Needed when n == 0
+    return true;
+
+  for (; first != last; ++first)
+    if (first == p)
+      return true;
+
+  return false;
+}
+
 template <class CharT>
 struct constexpr_char_traits
 {
@@ -98,23 +123,21 @@ constexpr_char_traits<CharT>::find(const char_type* s, std::size_t n, const char
 }
 
 template <class CharT>
-TEST_CONSTEXPR_CXX14 CharT*
-constexpr_char_traits<CharT>::move(char_type* s1, const char_type* s2, std::size_t n)
-{
-    char_type* r = s1;
-    if (s1 < s2)
-    {
-        for (; n; --n, ++s1, ++s2)
-            assign(*s1, *s2);
-    }
-    else if (s2 < s1)
-    {
-        s1 += n;
-        s2 += n;
-        for (; n; --n)
-            assign(*--s1, *--s2);
-    }
-    return r;
+TEST_CONSTEXPR_CXX14 CharT* constexpr_char_traits<CharT>::move(char_type* s1, const char_type* s2, std::size_t n) {
+  if (s1 == s2)
+    return s1;
+
+  char_type* r = s1;
+  if (is_pointer_in_range(s1, s1 + n, s2)) {
+    for (; n; --n)
+      assign(*s1++, *s2++);
+  } else {
+    s1 += n;
+    s2 += n;
+    for (; n; --n)
+      assign(*--s1, *--s2);
+  }
+  return r;
 }
 
 template <class CharT>
diff --git a/libcxx/test/support/nasty_string.h b/libcxx/test/support/nasty_string.h
index 672c3cb4ed9e..ea9d83ccf282 100644
--- a/libcxx/test/support/nasty_string.h
+++ b/libcxx/test/support/nasty_string.h
@@ -16,6 +16,7 @@
 
 #include "make_string.h"
 #include "test_macros.h"
+#include "constexpr_char_traits.h" // is_pointer_in_range
 
 // This defines a nasty_string similar to nasty_containers. This string's
 // value_type does operator hijacking, which allows us to ensure that the
@@ -118,11 +119,14 @@ constexpr const nasty_char* nasty_char_traits::find(const nasty_char* s, std::si
 }
 
 constexpr nasty_char* nasty_char_traits::move(nasty_char* s1, const nasty_char* s2, std::size_t n) {
+  if (s1 == s2)
+    return s1;
+
   nasty_char* r = s1;
-  if (s1 < s2) {
-    for (; n; --n, ++s1, ++s2)
-      assign(*s1, *s2);
-  } else if (s2 < s1) {
+  if (is_pointer_in_range(s1, s1 + n, s2)) {
+    for (; n; --n)
+      assign(*s1++, *s2++);
+  } else {
     s1 += n;
     s2 += n;
     for (; n; --n)
-- 
GitLab


From 6c8356579b20a6522d39649bcaaaf77e8e324daf Mon Sep 17 00:00:00 2001
From: Tacet <advenam.tacet@trailofbits.com>
Date: Thu, 9 May 2024 18:40:15 +0200
Subject: [PATCH 0310/1206] [libc++][ASan] Fix std::basic_string trait type
 (#91590)

Addresses the comment:
https://github.com/llvm/llvm-project/pull/79536#discussion_r1593652240

Changes the type to `void` instead of `false_type`.

The value is used here:


https://github.com/llvm/llvm-project/blob/6f1013a5b3f92d3ae6e378d6706584a2a44e6964/libcxx/include/__type_traits/is_trivially_relocatable.h#L35-L38
---
 libcxx/include/string | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcxx/include/string b/libcxx/include/string
index 8f629d8bf13c..1db803e822d7 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -741,8 +741,8 @@ public:
   // is kept inside objects memory (short string optimization), instead of in allocated
   // external memory. In such cases, the destructor is responsible for unpoisoning
   // the memory to avoid triggering false positives.
-  // Therefore it's crucial to ensure the destructor is called
-  using __trivially_relocatable = false_type;
+  // Therefore it's crucial to ensure the destructor is called.
+  using __trivially_relocatable = void;
 #else
   using __trivially_relocatable = __conditional_t<
       __libcpp_is_trivially_relocatable<allocator_type>::value && __libcpp_is_trivially_relocatable<pointer>::value,
-- 
GitLab


From 2083e97e88ed13d39d9190d65696ad3866c23caa Mon Sep 17 00:00:00 2001
From: Benoit Jacob <jacob.benoit.1@gmail.com>
Date: Thu, 9 May 2024 12:42:45 -0400
Subject: [PATCH 0311/1206] Fix VectorEmulateNarrowType asserting on scalar
 type vs vector type. (#91613)

---
 .../Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
index a301b919dc52..6025c4ad7c14 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp
@@ -1119,8 +1119,9 @@ struct RewriteAlignedSubByteIntExt : OpRewritePattern<ConversionOpType> {
                                 PatternRewriter &rewriter) const override {
     // Verify the preconditions.
     Value srcValue = conversionOp.getIn();
-    auto srcVecType = cast<VectorType>(srcValue.getType());
-    auto dstVecType = cast<VectorType>(conversionOp.getType());
+    auto srcVecType = dyn_cast<VectorType>(srcValue.getType());
+    auto dstVecType = dyn_cast<VectorType>(conversionOp.getType());
+
     if (failed(
             commonConversionPrecondition(rewriter, dstVecType, conversionOp)))
       return failure();
-- 
GitLab


From 28c427e5c022634ef479a98dc46291067a8c6c96 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 9 May 2024 09:44:07 -0700
Subject: [PATCH 0312/1206] [flang] Ensure that DATA converter can cope with
 proc ptr error (#90973)

Multiple definitions of a procedure pointer with DATA statements should
elicit an error message, not a compiler crash.

Fixes https://github.com/llvm/llvm-project/issues/90944.
---
 flang/include/flang/Evaluate/tools.h   |  6 ++++--
 flang/lib/Evaluate/fold-designator.cpp | 28 ++++++++++++++++----------
 flang/lib/Evaluate/tools.cpp           |  6 +++---
 flang/test/Semantics/data01.f90        |  2 +-
 flang/test/Semantics/data23.f90        | 18 +++++++++++++++++
 5 files changed, 43 insertions(+), 17 deletions(-)
 create mode 100644 flang/test/Semantics/data23.f90

diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index ca14c144af2d..cb750d5e82d8 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -152,9 +152,11 @@ std::optional<Expr<SomeType>> AsGenericExpr(const Symbol &);
 // Propagate std::optional from input to output.
 template <typename A>
 std::optional<Expr<SomeType>> AsGenericExpr(std::optional<A> &&x) {
-  if (!x)
+  if (x) {
+    return AsGenericExpr(std::move(*x));
+  } else {
     return std::nullopt;
-  return AsGenericExpr(std::move(*x));
+  }
 }
 
 template <typename A>
diff --git a/flang/lib/Evaluate/fold-designator.cpp b/flang/lib/Evaluate/fold-designator.cpp
index 6952436681f7..0d8c22fb2977 100644
--- a/flang/lib/Evaluate/fold-designator.cpp
+++ b/flang/lib/Evaluate/fold-designator.cpp
@@ -273,9 +273,8 @@ static std::optional<DataRef> OffsetToDataRef(FoldingContext &context,
   if (IsAllocatableOrPointer(symbol)) {
     return entity.IsSymbol() ? DataRef{symbol}
                              : DataRef{std::move(entity.GetComponent())};
-  }
-  std::optional<DataRef> result;
-  if (std::optional<DynamicType> type{DynamicType::From(symbol)}) {
+  } else if (std::optional<DynamicType> type{DynamicType::From(symbol)}) {
+    std::optional<DataRef> result;
     if (!type->IsUnlimitedPolymorphic()) {
       if (std::optional<Shape> shape{GetShape(context, symbol)}) {
         if (GetRank(*shape) > 0) {
@@ -289,7 +288,7 @@ static std::optional<DataRef> OffsetToDataRef(FoldingContext &context,
               : DataRef{std::move(entity.GetComponent())};
         }
         if (result && type->category() == TypeCategory::Derived &&
-            size < result->GetLastSymbol().size()) {
+            size <= result->GetLastSymbol().size()) {
           if (const Symbol *
               component{OffsetToUniqueComponent(
                   type->GetDerivedTypeSpec(), offset)}) {
@@ -298,25 +297,32 @@ static std::optional<DataRef> OffsetToDataRef(FoldingContext &context,
                 NamedEntity{Component{std::move(*result), *component}}, offset,
                 size);
           }
-          result.reset();
         }
       }
     }
+    return result;
+  } else {
+    return std::nullopt;
   }
-  return result;
 }
 
 // Reconstructs a Designator from a symbol, an offset, and a size.
+// Returns a ProcedureDesignator in the case of a whole procedure pointer.
 std::optional<Expr<SomeType>> OffsetToDesignator(FoldingContext &context,
     const Symbol &baseSymbol, ConstantSubscript offset, std::size_t size) {
   if (offset < 0) {
     return std::nullopt;
-  }
-  if (std::optional<DataRef> dataRef{
-          OffsetToDataRef(context, NamedEntity{baseSymbol}, offset, size)}) {
+  } else if (std::optional<DataRef> dataRef{OffsetToDataRef(
+                 context, NamedEntity{baseSymbol}, offset, size)}) {
     const Symbol &symbol{dataRef->GetLastSymbol()};
-    if (std::optional<Expr<SomeType>> result{
-            AsGenericExpr(std::move(*dataRef))}) {
+    if (IsProcedurePointer(symbol)) {
+      if (std::holds_alternative<SymbolRef>(dataRef->u)) {
+        return Expr<SomeType>{ProcedureDesignator{symbol}};
+      } else if (auto *component{std::get_if<Component>(&dataRef->u)}) {
+        return Expr<SomeType>{ProcedureDesignator{std::move(*component)}};
+      }
+    } else if (std::optional<Expr<SomeType>> result{
+                   AsGenericExpr(std::move(*dataRef))}) {
       if (IsAllocatableOrPointer(symbol)) {
       } else if (auto type{DynamicType::From(symbol)}) {
         if (auto elementBytes{
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 9a5f9130632e..826b97b87bf3 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -28,11 +28,11 @@ namespace Fortran::evaluate {
 static constexpr bool allowOperandDuplication{false};
 
 std::optional<Expr<SomeType>> AsGenericExpr(DataRef &&ref) {
-  const Symbol &symbol{ref.GetLastSymbol()};
-  if (auto dyType{DynamicType::From(symbol)}) {
+  if (auto dyType{DynamicType::From(ref.GetLastSymbol())}) {
     return TypedWrapper<Designator, DataRef>(*dyType, std::move(ref));
+  } else {
+    return std::nullopt;
   }
-  return std::nullopt;
 }
 
 std::optional<Expr<SomeType>> AsGenericExpr(const Symbol &symbol) {
diff --git a/flang/test/Semantics/data01.f90 b/flang/test/Semantics/data01.f90
index 9046487fa176..fe2d16e95ee1 100644
--- a/flang/test/Semantics/data01.f90
+++ b/flang/test/Semantics/data01.f90
@@ -67,6 +67,6 @@ subroutine CheckValue
   !ERROR: DATA statement value 'b(1_8)' for 'z' is not a constant
   data z / b(1) /
   type(hasAlloc) ha
-  !ERROR: DATA statement value 'hasalloc(a=0_4)' for 'ha' is not a constant
+  !ERROR: DATA statement value 'hasalloc(a=0_4)' for 'ha%a' is not a constant
   data ha / hasAlloc(0) /
 end
diff --git a/flang/test/Semantics/data23.f90 b/flang/test/Semantics/data23.f90
new file mode 100644
index 000000000000..8210e9e62b81
--- /dev/null
+++ b/flang/test/Semantics/data23.f90
@@ -0,0 +1,18 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+program p
+  interface
+    subroutine s
+    end subroutine
+  end interface
+  !ERROR: DATA statement initializations affect 'p' more than once
+  procedure(s), pointer :: p
+  type t
+    procedure(s), pointer, nopass :: p
+  end type
+  !ERROR: DATA statement initializations affect 'x%p' more than once
+  type(t) x
+  data p /s/
+  data p /s/
+  data x%p /s/
+  data x%p /s/
+end
-- 
GitLab


From e9be1292873428a81524f200aaa5cc23b857b22e Mon Sep 17 00:00:00 2001
From: Tulio Magno Quites Machado Filho <tuliom@redhat.com>
Date: Thu, 9 May 2024 13:46:49 -0300
Subject: [PATCH 0313/1206] [Offload] Fixes typo in aarch64 triple (#91622)

Use llvm::Triple:aarch64 as the little-endian triple.

Fixes commit 3e54768d7a0e1cfa65e892b6602993192ecad91e.
---
 offload/plugins-nextgen/host/src/rtl.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index 409b44b1640a..ef84cbaf5458 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -427,7 +427,7 @@ struct GenELF64PluginTy final : public GenericPluginTy {
     return llvm::Triple::systemz;
 #elif defined(__aarch64__)
 #ifdef LITTLEENDIAN_CPU
-    return llvm::Triple::aarch64_le;
+    return llvm::Triple::aarch64;
 #else
     return llvm::Triple::aarch64_be;
 #endif
-- 
GitLab


From 41f0574c4654fb8a8cbb8c26d453f51a31cfd2a0 Mon Sep 17 00:00:00 2001
From: Xiang Li <python3kgae@outlook.com>
Date: Thu, 9 May 2024 09:47:07 -0700
Subject: [PATCH 0314/1206] [HLSL] reenable add packoffset in AST (#91474)

This reapplies
https://github.com/llvm/llvm-project/commit/c5509fedc5757fffece385d9d068e36b26793ade
"[HLSL] Support packoffset attribute in AST
(https://github.com/llvm/llvm-project/pull/89836)" with a fix for the
test failure caused by missing -fnative-half-type.

Since we have to parse the attribute manually in ParseHLSLAnnotations,
we could create the ParsedAttribute with an integer offset parameter
instead of string. This approach avoids parsing the string if the offset
is saved as a string in HLSLPackOffsetAttr.

For #57914
---
 clang/include/clang/Basic/Attr.td             |  12 ++
 clang/include/clang/Basic/AttrDocs.td         |  20 +++
 clang/include/clang/Basic/DiagnosticGroups.td |   3 +
 .../clang/Basic/DiagnosticParseKinds.td       |   2 +
 .../clang/Basic/DiagnosticSemaKinds.td        |   5 +
 clang/lib/Parse/ParseHLSL.cpp                 |  88 +++++++++++++
 clang/lib/Sema/SemaDeclAttr.cpp               |  52 ++++++++
 clang/lib/Sema/SemaHLSL.cpp                   |  80 ++++++++++++
 clang/test/AST/HLSL/packoffset.hlsl           | 100 ++++++++++++++
 clang/test/SemaHLSL/packoffset-invalid.hlsl   | 122 ++++++++++++++++++
 10 files changed, 484 insertions(+)
 create mode 100644 clang/test/AST/HLSL/packoffset.hlsl
 create mode 100644 clang/test/SemaHLSL/packoffset-invalid.hlsl

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 0225598cbbe8..52552ba48856 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4415,6 +4415,18 @@ def HLSLResourceBinding: InheritableAttr {
   let Documentation = [HLSLResourceBindingDocs];
 }
 
+def HLSLPackOffset: HLSLAnnotationAttr {
+  let Spellings = [HLSLAnnotation<"packoffset">];
+  let LangOpts = [HLSL];
+  let Args = [IntArgument<"Subcomponent">, IntArgument<"Component">];
+  let Documentation = [HLSLPackOffsetDocs];
+  let AdditionalMembers = [{
+      unsigned getOffset() {
+        return subcomponent * 4 + component;
+      }
+  }];
+}
+
 def HLSLSV_DispatchThreadID: HLSLAnnotationAttr {
   let Spellings = [HLSLAnnotation<"SV_DispatchThreadID">];
   let Subjects = SubjectList<[ParmVar, Field]>;
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 8e6faabfae64..f351822ac74b 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -7408,6 +7408,26 @@ The full documentation is available here: https://docs.microsoft.com/en-us/windo
   }];
 }
 
+def HLSLPackOffsetDocs : Documentation {
+  let Category = DocCatFunction;
+  let Content = [{
+The packoffset attribute is used to change the layout of a cbuffer.
+Attribute spelling in HLSL is: ``packoffset( c[Subcomponent][.component] )``.
+A subcomponent is a register number, which is an integer. A component is in the form of [.xyzw].
+
+Examples:
+
+.. code-block:: c++
+
+  cbuffer A {
+    float3 a : packoffset(c0.y);
+    float4 b : packoffset(c4);
+  }
+
+The full documentation is available here: https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-variable-packoffset
+  }];
+}
+
 def HLSLSV_DispatchThreadIDDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 60f87da2a738..2beb1d45124b 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -1507,6 +1507,9 @@ def BranchProtection : DiagGroup<"branch-protection">;
 // Warnings for HLSL Clang extensions
 def HLSLExtension : DiagGroup<"hlsl-extensions">;
 
+// Warning for mix packoffset and non-packoffset.
+def HLSLMixPackOffset : DiagGroup<"mix-packoffset">;
+
 // Warnings for DXIL validation
 def DXILValidation : DiagGroup<"dxil-validation">;
 
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index fdffb35ea0d9..bc9d7cacc50b 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1754,5 +1754,7 @@ def err_hlsl_separate_attr_arg_and_number : Error<"wrong argument format for hls
 def ext_hlsl_access_specifiers : ExtWarn<
   "access specifiers are a clang HLSL extension">,
   InGroup<HLSLExtension>;
+def err_hlsl_unsupported_component : Error<"invalid component '%0' used; expected 'x', 'y', 'z', or 'w'">;
+def err_hlsl_packoffset_invalid_reg : Error<"invalid resource class specifier '%0' for packoffset, expected 'c'">;
 
 } // end of Parser diagnostics
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 9317ae675c72..d6863f90edb6 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12184,6 +12184,11 @@ def err_hlsl_init_priority_unsupported : Error<
 def err_hlsl_unsupported_register_type : Error<"invalid resource class specifier '%0' used; expected 'b', 's', 't', or 'u'">;
 def err_hlsl_unsupported_register_number : Error<"register number should be an integer">;
 def err_hlsl_expected_space : Error<"invalid space specifier '%0' used; expected 'space' followed by an integer, like space1">;
+def warn_hlsl_packoffset_mix : Warning<"cannot mix packoffset elements with nonpackoffset elements in a cbuffer">,
+    InGroup<HLSLMixPackOffset>;
+def err_hlsl_packoffset_overlap : Error<"packoffset overlap between %0, %1">;
+def err_hlsl_packoffset_cross_reg_boundary : Error<"packoffset cannot cross register boundary">;
+def err_hlsl_packoffset_alignment_mismatch : Error<"packoffset at 'y' not match alignment %0 required by %1">;
 def err_hlsl_pointers_unsupported : Error<
   "%select{pointers|references}0 are unsupported in HLSL">;
 
diff --git a/clang/lib/Parse/ParseHLSL.cpp b/clang/lib/Parse/ParseHLSL.cpp
index f4cbece31f18..e9c8d6dca7bf 100644
--- a/clang/lib/Parse/ParseHLSL.cpp
+++ b/clang/lib/Parse/ParseHLSL.cpp
@@ -183,6 +183,94 @@ void Parser::ParseHLSLAnnotations(ParsedAttributes &Attrs,
       return;
     }
   } break;
+  case ParsedAttr::AT_HLSLPackOffset: {
+    // Parse 'packoffset( c[Subcomponent][.component] )'.
+    // Check '('.
+    if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after)) {
+      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+      return;
+    }
+    // Check c[Subcomponent] as an identifier.
+    if (!Tok.is(tok::identifier)) {
+      Diag(Tok.getLocation(), diag::err_expected) << tok::identifier;
+      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+      return;
+    }
+    StringRef OffsetStr = Tok.getIdentifierInfo()->getName();
+    SourceLocation SubComponentLoc = Tok.getLocation();
+    if (OffsetStr[0] != 'c') {
+      Diag(Tok.getLocation(), diag::err_hlsl_packoffset_invalid_reg)
+          << OffsetStr;
+      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+      return;
+    }
+    OffsetStr = OffsetStr.substr(1);
+    unsigned SubComponent = 0;
+    if (!OffsetStr.empty()) {
+      // Make sure SubComponent is a number.
+      if (OffsetStr.getAsInteger(10, SubComponent)) {
+        Diag(SubComponentLoc.getLocWithOffset(1),
+             diag::err_hlsl_unsupported_register_number);
+        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+        return;
+      }
+    }
+    unsigned Component = 0;
+    ConsumeToken(); // consume identifier.
+    SourceLocation ComponentLoc;
+    if (Tok.is(tok::period)) {
+      ConsumeToken(); // consume period.
+      if (!Tok.is(tok::identifier)) {
+        Diag(Tok.getLocation(), diag::err_expected) << tok::identifier;
+        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+        return;
+      }
+      StringRef ComponentStr = Tok.getIdentifierInfo()->getName();
+      ComponentLoc = Tok.getLocation();
+      ConsumeToken(); // consume identifier.
+      // Make sure Component is a single character.
+      if (ComponentStr.size() != 1) {
+        Diag(ComponentLoc, diag::err_hlsl_unsupported_component)
+            << ComponentStr;
+        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+        return;
+      }
+      switch (ComponentStr[0]) {
+      case 'x':
+      case 'r':
+        Component = 0;
+        break;
+      case 'y':
+      case 'g':
+        Component = 1;
+        break;
+      case 'z':
+      case 'b':
+        Component = 2;
+        break;
+      case 'w':
+      case 'a':
+        Component = 3;
+        break;
+      default:
+        Diag(ComponentLoc, diag::err_hlsl_unsupported_component)
+            << ComponentStr;
+        SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+        return;
+      }
+    }
+    ASTContext &Ctx = Actions.getASTContext();
+    QualType SizeTy = Ctx.getSizeType();
+    uint64_t SizeTySize = Ctx.getTypeSize(SizeTy);
+    ArgExprs.push_back(IntegerLiteral::Create(
+        Ctx, llvm::APInt(SizeTySize, SubComponent), SizeTy, SubComponentLoc));
+    ArgExprs.push_back(IntegerLiteral::Create(
+        Ctx, llvm::APInt(SizeTySize, Component), SizeTy, ComponentLoc));
+    if (ExpectAndConsume(tok::r_paren, diag::err_expected)) {
+      SkipUntil(tok::r_paren, StopAtSemi); // skip through )
+      return;
+    }
+  } break;
   case ParsedAttr::UnknownAttribute:
     Diag(Loc, diag::err_unknown_hlsl_semantic) << II;
     return;
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 6ca42856459f..6d957ac09e1c 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -7309,6 +7309,55 @@ static void handleHLSLSV_DispatchThreadIDAttr(Sema &S, Decl *D,
   D->addAttr(::new (S.Context) HLSLSV_DispatchThreadIDAttr(S.Context, AL));
 }
 
+static void handleHLSLPackOffsetAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
+  if (!isa<VarDecl>(D) || !isa<HLSLBufferDecl>(D->getDeclContext())) {
+    S.Diag(AL.getLoc(), diag::err_hlsl_attr_invalid_ast_node)
+        << AL << "shader constant in a constant buffer";
+    return;
+  }
+
+  uint32_t SubComponent;
+  if (!checkUInt32Argument(S, AL, AL.getArgAsExpr(0), SubComponent))
+    return;
+  uint32_t Component;
+  if (!checkUInt32Argument(S, AL, AL.getArgAsExpr(1), Component))
+    return;
+
+  QualType T = cast<VarDecl>(D)->getType().getCanonicalType();
+  // Check if T is an array or struct type.
+  // TODO: mark matrix type as aggregate type.
+  bool IsAggregateTy = (T->isArrayType() || T->isStructureType());
+
+  // Check Component is valid for T.
+  if (Component) {
+    unsigned Size = S.getASTContext().getTypeSize(T);
+    if (IsAggregateTy || Size > 128) {
+      S.Diag(AL.getLoc(), diag::err_hlsl_packoffset_cross_reg_boundary);
+      return;
+    } else {
+      // Make sure Component + sizeof(T) <= 4.
+      if ((Component * 32 + Size) > 128) {
+        S.Diag(AL.getLoc(), diag::err_hlsl_packoffset_cross_reg_boundary);
+        return;
+      }
+      QualType EltTy = T;
+      if (const auto *VT = T->getAs<VectorType>())
+        EltTy = VT->getElementType();
+      unsigned Align = S.getASTContext().getTypeAlign(EltTy);
+      if (Align > 32 && Component == 1) {
+        // NOTE: Component 3 will hit err_hlsl_packoffset_cross_reg_boundary.
+        // So we only need to check Component 1 here.
+        S.Diag(AL.getLoc(), diag::err_hlsl_packoffset_alignment_mismatch)
+            << Align << EltTy;
+        return;
+      }
+    }
+  }
+
+  D->addAttr(::new (S.Context)
+                 HLSLPackOffsetAttr(S.Context, AL, SubComponent, Component));
+}
+
 static void handleHLSLShaderAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   StringRef Str;
   SourceLocation ArgLoc;
@@ -9730,6 +9779,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_HLSLSV_DispatchThreadID:
     handleHLSLSV_DispatchThreadIDAttr(S, D, AL);
     break;
+  case ParsedAttr::AT_HLSLPackOffset:
+    handleHLSLPackOffsetAttr(S, D, AL);
+    break;
   case ParsedAttr::AT_HLSLShader:
     handleHLSLShaderAttr(S, D, AL);
     break;
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index bb9e37f18d37..6a12c417e2f3 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -39,9 +39,89 @@ Decl *SemaHLSL::ActOnStartBuffer(Scope *BufferScope, bool CBuffer,
   return Result;
 }
 
+// Calculate the size of a legacy cbuffer type based on
+// https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-packing-rules
+static unsigned calculateLegacyCbufferSize(const ASTContext &Context,
+                                           QualType T) {
+  unsigned Size = 0;
+  constexpr unsigned CBufferAlign = 128;
+  if (const RecordType *RT = T->getAs<RecordType>()) {
+    const RecordDecl *RD = RT->getDecl();
+    for (const FieldDecl *Field : RD->fields()) {
+      QualType Ty = Field->getType();
+      unsigned FieldSize = calculateLegacyCbufferSize(Context, Ty);
+      unsigned FieldAlign = 32;
+      if (Ty->isAggregateType())
+        FieldAlign = CBufferAlign;
+      Size = llvm::alignTo(Size, FieldAlign);
+      Size += FieldSize;
+    }
+  } else if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T)) {
+    if (unsigned ElementCount = AT->getSize().getZExtValue()) {
+      unsigned ElementSize =
+          calculateLegacyCbufferSize(Context, AT->getElementType());
+      unsigned AlignedElementSize = llvm::alignTo(ElementSize, CBufferAlign);
+      Size = AlignedElementSize * (ElementCount - 1) + ElementSize;
+    }
+  } else if (const VectorType *VT = T->getAs<VectorType>()) {
+    unsigned ElementCount = VT->getNumElements();
+    unsigned ElementSize =
+        calculateLegacyCbufferSize(Context, VT->getElementType());
+    Size = ElementSize * ElementCount;
+  } else {
+    Size = Context.getTypeSize(T);
+  }
+  return Size;
+}
+
 void SemaHLSL::ActOnFinishBuffer(Decl *Dcl, SourceLocation RBrace) {
   auto *BufDecl = cast<HLSLBufferDecl>(Dcl);
   BufDecl->setRBraceLoc(RBrace);
+
+  // Validate packoffset.
+  llvm::SmallVector<std::pair<VarDecl *, HLSLPackOffsetAttr *>> PackOffsetVec;
+  bool HasPackOffset = false;
+  bool HasNonPackOffset = false;
+  for (auto *Field : BufDecl->decls()) {
+    VarDecl *Var = dyn_cast<VarDecl>(Field);
+    if (!Var)
+      continue;
+    if (Field->hasAttr<HLSLPackOffsetAttr>()) {
+      PackOffsetVec.emplace_back(Var, Field->getAttr<HLSLPackOffsetAttr>());
+      HasPackOffset = true;
+    } else {
+      HasNonPackOffset = true;
+    }
+  }
+
+  if (HasPackOffset && HasNonPackOffset)
+    Diag(BufDecl->getLocation(), diag::warn_hlsl_packoffset_mix);
+
+  if (HasPackOffset) {
+    ASTContext &Context = getASTContext();
+    // Make sure no overlap in packoffset.
+    // Sort PackOffsetVec by offset.
+    std::sort(PackOffsetVec.begin(), PackOffsetVec.end(),
+              [](const std::pair<VarDecl *, HLSLPackOffsetAttr *> &LHS,
+                 const std::pair<VarDecl *, HLSLPackOffsetAttr *> &RHS) {
+                return LHS.second->getOffset() < RHS.second->getOffset();
+              });
+
+    for (unsigned i = 0; i < PackOffsetVec.size() - 1; i++) {
+      VarDecl *Var = PackOffsetVec[i].first;
+      HLSLPackOffsetAttr *Attr = PackOffsetVec[i].second;
+      unsigned Size = calculateLegacyCbufferSize(Context, Var->getType());
+      unsigned Begin = Attr->getOffset() * 32;
+      unsigned End = Begin + Size;
+      unsigned NextBegin = PackOffsetVec[i + 1].second->getOffset() * 32;
+      if (End > NextBegin) {
+        VarDecl *NextVar = PackOffsetVec[i + 1].first;
+        Diag(NextVar->getLocation(), diag::err_hlsl_packoffset_overlap)
+            << NextVar << Var;
+      }
+    }
+  }
+
   SemaRef.PopDeclContext();
 }
 
diff --git a/clang/test/AST/HLSL/packoffset.hlsl b/clang/test/AST/HLSL/packoffset.hlsl
new file mode 100644
index 000000000000..060288c2f7f7
--- /dev/null
+++ b/clang/test/AST/HLSL/packoffset.hlsl
@@ -0,0 +1,100 @@
+// RUN: %clang_cc1 -triple dxil-unknown-shadermodel6.3-library -S -finclude-default-header -fnative-half-type -ast-dump  -x hlsl %s | FileCheck %s
+
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer A
+cbuffer A
+{
+    // CHECK-NEXT: VarDecl {{.*}} A1 'float4'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 0
+    float4 A1 : packoffset(c);
+    // CHECK-NEXT: VarDecl {{.*}} col:11 A2 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 0
+    float A2 : packoffset(c1);
+    // CHECK-NEXT: VarDecl {{.*}} col:11 A3 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 1
+    float A3 : packoffset(c1.y);
+}
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer B
+cbuffer B
+{
+    // CHECK: VarDecl {{.*}} B0 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
+    float B0 : packoffset(c0.g);
+    // CHECK-NEXT: VarDecl {{.*}} B1 'double'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 2
+	double B1 : packoffset(c0.b);
+    // CHECK-NEXT: VarDecl {{.*}} B2 'half'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 0
+	half B2 : packoffset(c0.r);
+}
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer C
+cbuffer C
+{
+    // CHECK: VarDecl {{.*}} C0 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1
+    float C0 : packoffset(c0.y);
+    // CHECK-NEXT: VarDecl {{.*}} C1 'float2'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2
+	float2 C1 : packoffset(c0.z);
+    // CHECK-NEXT: VarDecl {{.*}} C2 'half'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0
+	half C2 : packoffset(c0.x);
+}
+
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer D
+cbuffer D
+{
+    // CHECK: VarDecl {{.*}} D0 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
+    float D0 : packoffset(c0.y);
+    // CHECK-NEXT: VarDecl {{.*}} D1 'float[2]'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 1 0
+	float D1[2] : packoffset(c1.x);
+    // CHECK-NEXT: VarDecl {{.*}} D2 'half3'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2 1
+	half3 D2 : packoffset(c2.y);
+    // CHECK-NEXT: VarDecl {{.*}} D3 'double'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 2
+	double D3 : packoffset(c0.z);
+}
+
+struct ST {
+  float a;
+  float2 b;
+  half c;
+};
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer S
+cbuffer S {
+    // CHECK: VarDecl {{.*}} S0 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 1
+  float S0 : packoffset(c0.y);
+    // CHECK: VarDecl {{.*}} S1 'ST'
+    // CHECK: HLSLPackOffsetAttr {{.*}} 1 0
+  ST S1 : packoffset(c1);
+    // CHECK: VarDecl {{.*}} S2 'double2'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 2 0
+  double2 S2 : packoffset(c2);
+}
+
+struct ST2 {
+  float s0;
+  ST s1;
+  half s2;
+};
+
+// CHECK: HLSLBufferDecl {{.*}} cbuffer S2
+cbuffer S2 {
+    // CHECK: VarDecl {{.*}} S20 'float'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 0 3
+  float S20 : packoffset(c0.a);
+    // CHECK: VarDecl {{.*}} S21 'ST2'
+    // CHECK: HLSLPackOffsetAttr {{.*}} 1 0
+  ST2 S21 : packoffset(c1);
+    // CHECK: VarDecl {{.*}} S22 'half'
+    // CHECK-NEXT: HLSLPackOffsetAttr {{.*}} 3 1
+  half S22 : packoffset(c3.y);
+}
diff --git a/clang/test/SemaHLSL/packoffset-invalid.hlsl b/clang/test/SemaHLSL/packoffset-invalid.hlsl
new file mode 100644
index 000000000000..526a511edf1f
--- /dev/null
+++ b/clang/test/SemaHLSL/packoffset-invalid.hlsl
@@ -0,0 +1,122 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library -fnative-half-type -verify %s
+
+// expected-warning@+1{{cannot mix packoffset elements with nonpackoffset elements in a cbuffer}}
+cbuffer Mix
+{
+    float4 M1 : packoffset(c0);
+    float M2;
+    float M3 : packoffset(c1.y);
+}
+
+// expected-warning@+1{{cannot mix packoffset elements with nonpackoffset elements in a cbuffer}}
+cbuffer Mix2
+{
+    float4 M4;
+    float M5 : packoffset(c1.y);
+    float M6 ;
+}
+
+// expected-error@+1{{attribute 'packoffset' only applies to shader constant in a constant buffer}}
+float4 g : packoffset(c0);
+
+cbuffer IllegalOffset
+{
+    // expected-error@+1{{invalid resource class specifier 't2' for packoffset, expected 'c'}}
+    float4 i1 : packoffset(t2);
+    // expected-error@+1{{invalid component 'm' used; expected 'x', 'y', 'z', or 'w'}}
+    float i2 : packoffset(c1.m);
+}
+
+cbuffer Overlap
+{
+    float4 o1 : packoffset(c0);
+    // expected-error@+1{{packoffset overlap between 'o2', 'o1'}}
+    float2 o2 : packoffset(c0.z);
+}
+
+cbuffer CrossReg
+{
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+    float4 c1 : packoffset(c0.y);
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+    float2 c2 : packoffset(c1.w);
+}
+
+struct ST {
+  float s;
+};
+
+cbuffer Aggregate
+{
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+    ST A1 : packoffset(c0.y);
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+    float A2[2] : packoffset(c1.w);
+}
+
+cbuffer Double {
+    // expected-error@+1{{packoffset at 'y' not match alignment 64 required by 'double'}}
+    double d : packoffset(c.y);
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+	double2 d2 : packoffset(c.z);
+    // expected-error@+1{{packoffset cannot cross register boundary}}
+	double3 d3 : packoffset(c.z);
+}
+
+cbuffer ParsingFail {
+// expected-error@+1{{expected identifier}}
+float pf0 : packoffset();
+// expected-error@+1{{expected identifier}}
+float pf1 : packoffset((c0));
+// expected-error@+1{{expected ')'}}
+float pf2 : packoffset(c0, x);
+// expected-error@+1{{invalid component 'X' used}}
+float pf3 : packoffset(c.X);
+// expected-error@+1{{expected '(' after ''}}
+float pf4 : packoffset;
+// expected-error@+1{{expected identifier}}
+float pf5 : packoffset(;
+// expected-error@+1{{expected '(' after '}}
+float pf6 : packoffset);
+// expected-error@+1{{expected '(' after '}}
+float pf7 : packoffset c0.x;
+
+// expected-error@+1{{invalid component 'xy' used}}
+float pf8 : packoffset(c0.xy);
+// expected-error@+1{{invalid component 'rg' used}}
+float pf9 : packoffset(c0.rg);
+// expected-error@+1{{invalid component 'yes' used}}
+float pf10 : packoffset(c0.yes);
+// expected-error@+1{{invalid component 'woo'}}
+float pf11 : packoffset(c0.woo);
+// expected-error@+1{{invalid component 'xr' used}}
+float pf12 : packoffset(c0.xr);
+}
+
+struct ST2 {
+  float a;
+  float2 b;
+};
+
+cbuffer S {
+  float S0 : packoffset(c0.y);
+  ST2 S1[2] : packoffset(c1);
+  // expected-error@+1{{packoffset overlap between 'S2', 'S1'}}
+  half2 S2 : packoffset(c1.w);
+  half2 S3 : packoffset(c2.w);
+}
+
+struct ST23 {
+  float s0;
+  ST2 s1;
+};
+
+cbuffer S2 {
+  float S20 : packoffset(c0.y);
+  ST2 S21 : packoffset(c1);
+  half2 S22 : packoffset(c2.w);
+  double S23[2] : packoffset(c3);
+  // expected-error@+1{{packoffset overlap between 'S24', 'S23'}}
+  float S24 : packoffset(c3.z);
+  float S25 : packoffset(c4.z);
+}
-- 
GitLab


From d0bafb5435d5ebd90cdf965a9b35bdfa05dde23b Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 9 May 2024 09:28:06 -0700
Subject: [PATCH 0315/1206] [RISCV] Add coverage for zext.w/h interaction with
 shift transforms

Two cases where folding the and (which could be a zext.w) through
shifts in generic DAG result in net worse code quality.  And one
negative case where keeping a zext.h would result in a longer
critical path.
---
 llvm/test/CodeGen/RISCV/rv64zba.ll | 63 ++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll
index 5931e0982a4a..8fe221f2a297 100644
--- a/llvm/test/CodeGen/RISCV/rv64zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zba.ll
@@ -2853,3 +2853,66 @@ entry:
   ret i64 %6
 }
 
+define ptr @gep_lshr_i32(ptr %0, i64 %1) {
+; RV64I-LABEL: gep_lshr_i32:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    srli a1, a1, 2
+; RV64I-NEXT:    li a2, 5
+; RV64I-NEXT:    slli a2, a2, 36
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    mulhu a1, a1, a2
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: gep_lshr_i32:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    slli a1, a1, 2
+; RV64ZBA-NEXT:    srli a1, a1, 4
+; RV64ZBA-NEXT:    slli.uw a1, a1, 4
+; RV64ZBA-NEXT:    sh2add a1, a1, a1
+; RV64ZBA-NEXT:    add a0, a0, a1
+; RV64ZBA-NEXT:    ret
+entry:
+  %2 = lshr exact i64 %1, 2
+  %3 = and i64 %2, 4294967295
+  %5 = getelementptr [80 x i8], ptr %0, i64 %3
+  ret ptr %5
+}
+
+define i64 @srli_slliw(i64 %1) {
+; RV64I-LABEL: srli_slliw:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    slli a0, a0, 2
+; RV64I-NEXT:    li a1, 1
+; RV64I-NEXT:    slli a1, a1, 36
+; RV64I-NEXT:    addi a1, a1, -16
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: srli_slliw:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    slli a0, a0, 2
+; RV64ZBA-NEXT:    srli a0, a0, 4
+; RV64ZBA-NEXT:    slli.uw a0, a0, 4
+; RV64ZBA-NEXT:    ret
+entry:
+  %2 = lshr exact i64 %1, 2
+  %3 = and i64 %2, 4294967295
+  %4 = shl i64 %3, 4
+  ret i64 %4
+}
+
+define i64 @srli_slli_i16(i64 %1) {
+; CHECK-LABEL: srli_slli_i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    lui a1, 256
+; CHECK-NEXT:    addiw a1, a1, -16
+; CHECK-NEXT:    and a0, a0, a1
+; CHECK-NEXT:    ret
+entry:
+  %2 = lshr exact i64 %1, 2
+  %3 = and i64 %2, 65535
+  %4 = shl i64 %3, 4
+  ret i64 %4
+}
-- 
GitLab


From 8ed7ea08962bb878d31052c15e811d1a6cda0f07 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 9 May 2024 09:56:01 -0700
Subject: [PATCH 0316/1206] [flang] Defer conversion of PDT default
 initializers (#91026)

As the kinds of the integer types of type parameters may well depend on
the values of other type parameters, defer the attempt to convert their
values to the point of type instantiation instead of doing it during
declaration processing.
---
 flang/lib/Semantics/resolve-names.cpp | 3 +--
 flang/test/Semantics/modfile12.f90    | 2 +-
 flang/test/Semantics/modfile17.f90    | 6 +++---
 flang/test/Semantics/pdt03.f90        | 9 +++++++++
 4 files changed, 14 insertions(+), 6 deletions(-)
 create mode 100644 flang/test/Semantics/pdt03.f90

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 61394b0f41de..2199e3f16e62 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -5550,8 +5550,7 @@ void DeclarationVisitor::Post(const parser::TypeParamDefStmt &x) {
       SetType(name, *type);
       if (auto &init{
               std::get<std::optional<parser::ScalarIntConstantExpr>>(decl.t)}) {
-        if (auto maybeExpr{EvaluateNonPointerInitializer(
-                *symbol, *init, init->thing.thing.thing.value().source)}) {
+        if (auto maybeExpr{AnalyzeExpr(context(), *init)}) {
           if (auto *intExpr{std::get_if<SomeIntExpr>(&maybeExpr->u)}) {
             symbol->get<TypeParamDetails>().set_init(std::move(*intExpr));
           }
diff --git a/flang/test/Semantics/modfile12.f90 b/flang/test/Semantics/modfile12.f90
index 17b6e95c4a56..41ab300e00f6 100644
--- a/flang/test/Semantics/modfile12.f90
+++ b/flang/test/Semantics/modfile12.f90
@@ -41,7 +41,7 @@ end
 !  real(4)::y(1_8:8_8)
 !  type::t(c,d)
 !    integer(4),kind::c=1_4
-!    integer(4),len::d=3_4
+!    integer(4),len::d=3_8
 !  end type
 !  type(t(c=4_4,d=:)),allocatable::z
 !  class(t(c=5_4,d=:)),allocatable::z2
diff --git a/flang/test/Semantics/modfile17.f90 b/flang/test/Semantics/modfile17.f90
index 189d8a83de8c..4ab5cc85db25 100644
--- a/flang/test/Semantics/modfile17.f90
+++ b/flang/test/Semantics/modfile17.f90
@@ -97,10 +97,10 @@ end module
 !integer(k8)::j8
 !end type
 !type::defaulted(n1,n2,n4,n8)
-!integer(1),kind::n1=1_1
-!integer(2),kind::n2=int(2_4*int(int(n1,kind=1),kind=4),kind=2)
+!integer(1),kind::n1=1_4
+!integer(2),kind::n2=2_4*int(int(n1,kind=1),kind=4)
 !integer(4),kind::n4=2_4*int(int(n2,kind=2),kind=4)
-!integer(8),kind::n8=int(12_4-int(n4,kind=4),kind=8)
+!integer(8),kind::n8=12_4-int(n4,kind=4)
 !type(capture(k1=int(n1,kind=1),k2=int(n2,kind=2),k4=int(n4,kind=4),k8=n8))::cap
 !end type
 !type,extends(defaulted)::extension(k5)
diff --git a/flang/test/Semantics/pdt03.f90 b/flang/test/Semantics/pdt03.f90
new file mode 100644
index 000000000000..2fb63d21540b
--- /dev/null
+++ b/flang/test/Semantics/pdt03.f90
@@ -0,0 +1,9 @@
+! RUN: %flang_fc1 -fdebug-unparse %s 2>&1 | FileCheck %s
+type t(kp1,kp2)
+  integer, kind :: kp1
+  integer(kp1), kind :: kp2 = kp1
+end type
+type(t(kp1=8_8)) x
+!CHECK: 4_4, 8_4, 8_4, 8_8
+print *, kind(x%kp1), x%kp1, kind(x%kp2), x%kp2
+end
-- 
GitLab


From 4c3db2588e8b38f75744def6e2dd17c556950e46 Mon Sep 17 00:00:00 2001
From: Adam Siemieniuk <adam.siemieniuk@intel.com>
Date: Thu, 9 May 2024 19:02:44 +0200
Subject: [PATCH 0317/1206] [mlir][linalg] Block pack matmul pass (#89782)

Pack a matmul MxNxK operation into 4D blocked layout. Any present batch
dimensions remain unchanged and the result is unpacked back to the
original layout.

Matmul block packing splits the operands into major blocks (outer
dimensions) and minor blocks (inner dimensions). The desired block
layout can be controlled through packing options.
---
 mlir/include/mlir/Dialect/Linalg/Passes.td    |  59 +++
 .../Dialect/Linalg/Transforms/Transforms.h    |  64 +++
 .../Linalg/Transforms/BlockPackMatmul.cpp     | 321 ++++++++++++
 .../Dialect/Linalg/Transforms/CMakeLists.txt  |   1 +
 .../Linalg/block-pack-matmul-layout.mlir      | 101 ++++
 .../Linalg/block-pack-matmul-padding.mlir     |  82 +++
 .../Dialect/Linalg/block-pack-matmul.mlir     | 478 ++++++++++++++++++
 7 files changed, 1106 insertions(+)
 create mode 100644 mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
 create mode 100644 mlir/test/Dialect/Linalg/block-pack-matmul-layout.mlir
 create mode 100644 mlir/test/Dialect/Linalg/block-pack-matmul-padding.mlir
 create mode 100644 mlir/test/Dialect/Linalg/block-pack-matmul.mlir

diff --git a/mlir/include/mlir/Dialect/Linalg/Passes.td b/mlir/include/mlir/Dialect/Linalg/Passes.td
index 85f11c66d29a..0a4ce8953136 100644
--- a/mlir/include/mlir/Dialect/Linalg/Passes.td
+++ b/mlir/include/mlir/Dialect/Linalg/Passes.td
@@ -141,4 +141,63 @@ def LinalgDetensorizePass : InterfacePass<"linalg-detensorize", "FunctionOpInter
   ];
 }
 
+def LinalgBlockPackMatmul : Pass<"linalg-block-pack-matmul"> {
+  let summary = "Convert linalg matmul ops to block layout and back";
+  let description = [{
+    Pack a matmul operation into blocked layout with two levels of subdivision:
+    - major 2D blocks - outer dimensions, consist of minor blocks
+    - minor 2D blocks - inner dimensions, consist of scalar elements
+
+    A 2D matmul MxNxK gets reshaped into blocked 4D representation
+    as: [MB][NB][mb][nb] += [MB][KB][mb][kb] * [NB][KB][nb][kb]
+    where the (MB, NB, KB) dimensions represent the major blocks,
+    and the (mb, nb, kb) are the minor blocks of their respective
+    original 2D dimensions (M, N, K).
+
+    Depending on the initial operands' data layout and the specified
+    packing options, the major blocks dimensions might get transposed
+    e.g., [MB][KB] -> [KB][MB]. The minor blocks can also be transposed
+    e.g., [mb][kb] -> [kb][mb].
+    Any present batch dimensions remain unchanged.
+    The final result is unpacked back to the original shape.
+
+    For example, given a matmul operation:
+    ```mlir
+      %res = linalg.matmul ins(%A, %B) outs(%C)
+    ```
+    the default transformation result can be represented as:
+    ```mlir
+      %A_packed = pack %A : 2D <MxK> -> 4D <MBxKBxmbxkb>
+      %B_packed = pack %B : 2D <KxN> -> 4D <NBxKBxnbxkb>
+      %C_packed = pack %C : 2D <MxN> -> 4D <MBxNBxmbxnb>
+      %res_packed = linalg.mmt4d ins(%A_packed, %B_packed) outs(%C_packed)
+      %res = unpack %res_packed : 4D <MBxNBxmbxnb> -> 2D <MxN>
+    ```
+  }];
+  let dependentDialects = ["linalg::LinalgDialect", "tensor::TensorDialect"];
+  let options = [
+    ListOption<"blockFactors", "block-factors", "int64_t",
+               "Block factors (mb, nb, kb) for relayout">,
+    Option<"allowPadding", "allow-padding", "bool",
+           /*default=*/"true",
+           "Allow packing padding">,
+    ListOption<"mnkPaddedSizesNextMultipleOf", "mnk-padded-multiples", "int64_t",
+               "Next multiples of the packing sizes">,
+    ListOption<"mnkOrder", "mnk-order", "int64_t",
+               "Permutation of matmul (M, N, K) dimensions order">,
+    Option<"lhsTransposeOuterBlocks", "lhs-transpose-outer-blocks", "bool",
+           /*default=*/"false",
+           "Transpose LHS outer block layout [MB][KB] -> [KB][MB]">,
+    Option<"lhsTransposeInnerBlocks", "lhs-transpose-inner-blocks", "bool",
+           /*default=*/"false",
+           "Transpose LHS inner block layout [mb][kb] -> [kb][mb]">,
+    Option<"rhsTransposeOuterBlocks", "rhs-transpose-outer-blocks", "bool",
+           /*default=*/"true",
+           "Transpose RHS outer block layout [KB][NB] -> [NB][KB]">,
+    Option<"rhsTransposeInnerBlocks", "rhs-transpose-inner-blocks", "bool",
+           /*default=*/"true",
+           "Transpose RHS inner block layout [kb][nb] -> [nb][kb]">
+  ];
+}
+
 #endif // MLIR_DIALECT_LINALG_PASSES
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 5ecf84fa9c70..f77c19ed0fcc 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -1162,6 +1162,66 @@ packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
                    ArrayRef<int64_t> mnkPaddedSizesNextMultipleOf,
                    ArrayRef<int64_t> mnkOrder);
 
+struct BlockPackMatmulOptions {
+  /// Minor block factors (mb, nb, kb) for packing relayout where mb, mn are
+  /// the parallel dimensions and kb is the reduction dimension.
+  SmallVector<int64_t, 3> blockFactors;
+
+  /// If true, allows packing of dimensions that only partially fit into the
+  /// block factors.
+  bool allowPadding = true;
+
+  /// Next multiples of the packing sizes.
+  SmallVector<int64_t, 3> mnkPaddedSizesNextMultipleOf;
+
+  /// Permutation of matmul (M, N, K) dimensions order.
+  SmallVector<int64_t, 3> mnkOrder = {0, 1, 2};
+
+  /// Transpose LHS outer block layout [MB][KB] -> [KB][MB].
+  bool lhsTransposeOuterBlocks = false;
+
+  /// Transpose LHS inner block layout [mb][kb] -> [kb][mb].
+  bool lhsTransposeInnerBlocks = false;
+
+  /// Transpose RHS outer block layout [KB][NB] -> [NB][KB].
+  bool rhsTransposeOuterBlocks = true;
+
+  /// Transpose RHS inner block layout [kb][nb] -> [nb][kb].
+  bool rhsTransposeInnerBlocks = true;
+};
+
+/// Function type which is used to control matmul packing.
+/// It is expected to return valid packing configuration for each operation.
+/// Lack of packing options indicates that no valid configuration could be
+/// assigned and the operation will not be packed.
+using ControlBlockPackMatmulFn =
+    std::function<std::optional<BlockPackMatmulOptions>(linalg::LinalgOp)>;
+
+/// Pack a matmul operation into blocked 4D layout.
+///
+/// Relayout a matmul operation into blocked layout with two levels of
+/// subdivision:
+///   - major 2D blocks - outer dimensions, consist of minor blocks
+///   - minor 2D blocks - inner dimensions, consist of scalar elements
+///
+/// A 2D matmul MxNxK gets reshaped into blocked 4D representation
+/// as: [MB][NB][mb][nb] += [MB][KB][mb][kb] * [NB][KB][nb][kb]
+/// where the (MB, NB, KB) dimensions represent the major blocks,
+/// and the (mb, nb, kb) are the minor blocks of their respective
+/// original 2D dimensions (M, N, K).
+///
+/// Depending on the initial operands' data layout and the specified
+/// packing options, the major blocks dimensions might get transposed
+/// e.g., [MB][KB] -> [KB][MB]. The minor blocks can also be transposed
+/// e.g., [mb][kb] -> [kb][mb].
+/// Any present batch dimensions remain unchanged.
+/// The final result is unpacked back to the original shape.
+///
+/// Return failure if no valid packing options are provided.
+FailureOr<PackResult>
+blockPackMatmul(RewriterBase &rewriter, linalg::LinalgOp linalgOp,
+                const ControlBlockPackMatmulFn &controlPackMatmul);
+
 /// Rewrite tensor.from_elements to linalg.generic.
 FailureOr<Operation *>
 rewriteInDestinationPassingStyle(RewriterBase &rewriter,
@@ -1628,6 +1688,10 @@ void populateSplitReductionPattern(
 void populateTransposeMatmulPatterns(RewritePatternSet &patterns,
                                      bool transposeLHS = true);
 
+/// Patterns to block pack Linalg matmul ops.
+void populateBlockPackMatmulPatterns(RewritePatternSet &patterns,
+                                     const ControlBlockPackMatmulFn &controlFn);
+
 } // namespace linalg
 } // namespace mlir
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
new file mode 100644
index 000000000000..c07d1387ec75
--- /dev/null
+++ b/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
@@ -0,0 +1,321 @@
+//===- BlockPackMatmul.cpp - Linalg matmul block packing ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Linalg/Passes.h"
+
+#include "mlir/Dialect/Linalg/IR/Linalg.h"
+#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
+#include "mlir/Dialect/Linalg/Utils/Utils.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/TypeSwitch.h"
+
+#include <optional>
+
+namespace mlir {
+#define GEN_PASS_DEF_LINALGBLOCKPACKMATMUL
+#include "mlir/Dialect/Linalg/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+using namespace mlir::linalg;
+
+/// Return constant range span or nullopt, otherwise.
+static std::optional<int64_t> getConstantRange(const Range &range) {
+  std::optional<int64_t> stride = getConstantIntValue(range.stride);
+  if (!stride || *stride != 1)
+    return std::nullopt;
+  std::optional<int64_t> offset = getConstantIntValue(range.offset);
+  if (!offset)
+    return std::nullopt;
+  std::optional<int64_t> size = getConstantIntValue(range.size);
+  if (!size)
+    return std::nullopt;
+  return (*size - *offset);
+}
+
+/// Return true if all dimensions are fully divisible by the respective tiles.
+static bool validateFullTilesOnDims(linalg::LinalgOp linalgOp,
+                                    ArrayRef<OpFoldResult> tiles,
+                                    ArrayRef<int64_t> dims) {
+  if (dims.size() != tiles.size() || tiles.empty())
+    return false;
+
+  FailureOr<ContractionDimensions> contractDims =
+      inferContractionDims(linalgOp);
+  if (failed(contractDims))
+    return false;
+  unsigned batchDimsOffset = contractDims->batch.size();
+
+  // Skip the batch dimension if present.
+  // Offset all dimensions accordingly.
+  SmallVector<int64_t, 3> offsetDims{dims};
+  for (size_t i = 0; i < offsetDims.size(); i++)
+    offsetDims[i] += batchDimsOffset;
+
+  auto tileOp = cast<TilingInterface>(linalgOp.getOperation());
+  OpBuilder builder(tileOp);
+  OpBuilder::InsertionGuard guard(builder);
+  SmallVector<Range> iterationDomain = tileOp.getIterationDomain(builder);
+
+  for (auto dim : llvm::enumerate(offsetDims)) {
+    if (dim.value() >= static_cast<int64_t>(iterationDomain.size()))
+      return false;
+
+    std::optional<int64_t> tileSize = getConstantIntValue(tiles[dim.index()]);
+    std::optional<int64_t> rangeOnDim =
+        getConstantRange(iterationDomain[dim.value()]);
+
+    // If the tile factor or the range are non-constant, the tile size is
+    // considered to be invalid.
+    if (!tileSize || !rangeOnDim)
+      return false;
+
+    // The dimension must be fully divisible by the tile.
+    if (*rangeOnDim % *tileSize != 0)
+      return false;
+  }
+
+  return true;
+}
+
+/// Return failure or packed matmul with one of its operands transposed.
+static FailureOr<PackTransposeResult>
+transposePackedMatmul(RewriterBase &rewriter, linalg::LinalgOp linalgOp,
+                      tensor::PackOp packOp, AffineMap operandMap,
+                      ArrayRef<unsigned> blocksStartDimPos,
+                      bool transposeOuterBlocks, bool transposeInnerBlocks) {
+  assert(operandMap.getNumDims() >= 4 &&
+         "expected at least 4D prepacked matmul");
+  assert(blocksStartDimPos.size() >= 2 &&
+         "expected starting outer and inner block positions");
+
+  // Bias toward innermost dimensions.
+  unsigned outerBlockPos = operandMap.getNumResults() - 4;
+  unsigned innerBlockPos = operandMap.getNumResults() - 2;
+
+  // Transpose control options define the desired block and element layout.
+  // Block transposition (outer dimensions) or element transposition (inner
+  // dimensions) may not be necessary depending on the original matmul data
+  // layout.
+  bool isOuterTransposed =
+      operandMap.getDimPosition(outerBlockPos) != blocksStartDimPos.end()[-2];
+  bool isInnerTransposed =
+      operandMap.getDimPosition(innerBlockPos) != blocksStartDimPos.back();
+
+  // Transpose only the dimensions that need that to conform to the provided
+  // transpotion settings.
+  SmallVector<int64_t> innerPerm{0, 1};
+  if (isInnerTransposed != transposeInnerBlocks)
+    innerPerm = {1, 0};
+  SmallVector<int64_t> outerPerm{0, 1};
+  if (isOuterTransposed != transposeOuterBlocks)
+    outerPerm = {1, 0};
+
+  // Leave the outer dimensions, like batch, unchanged by offsetting all
+  // outer dimensions permutations.
+  SmallVector<int64_t> offsetPerms;
+  for (auto i : llvm::seq(0u, outerBlockPos))
+    offsetPerms.push_back(i);
+  for (auto perm : outerPerm)
+    offsetPerms.push_back(perm + outerBlockPos);
+  outerPerm = offsetPerms;
+
+  FailureOr<PackTransposeResult> packTransposedMatmul =
+      packTranspose(rewriter, packOp, linalgOp,
+                    /*maybeUnPackOp=*/nullptr, outerPerm, innerPerm);
+
+  return packTransposedMatmul;
+}
+
+/// Pack a matmul operation into blocked 4D layout.
+FailureOr<PackResult>
+linalg::blockPackMatmul(RewriterBase &rewriter, linalg::LinalgOp linalgOp,
+                        const ControlBlockPackMatmulFn &controlPackMatmul) {
+  if (linalgOp.hasPureBufferSemantics())
+    return rewriter.notifyMatchFailure(linalgOp, "require tensor semantics");
+
+  std::optional<BlockPackMatmulOptions> options = controlPackMatmul(linalgOp);
+  if (!options)
+    return rewriter.notifyMatchFailure(linalgOp, "invalid packing options");
+
+  if (options->blockFactors.size() != 3)
+    return rewriter.notifyMatchFailure(linalgOp, "require 3 tile factors");
+
+  SmallVector<OpFoldResult> mnkTiles =
+      getAsOpFoldResult(rewriter.getI64ArrayAttr(options->blockFactors));
+
+  // If padding is disabled, make sure that dimensions can be packed cleanly.
+  if (!options->allowPadding &&
+      !validateFullTilesOnDims(linalgOp, mnkTiles, options->mnkOrder)) {
+    return rewriter.notifyMatchFailure(linalgOp,
+                                       "expect packing full tiles only");
+  }
+
+  OpBuilder::InsertionGuard guard(rewriter);
+  // The op is replaced, we need to set the insertion point after it.
+  rewriter.setInsertionPointAfter(linalgOp);
+
+  // Pack the matmul operation into blocked layout with two levels of
+  // subdivision:
+  //   - major 2D blocks - outer dimensions, consist of minor blocks
+  //   - minor 2D blocks - inner dimensions, consist of scalar elements
+  FailureOr<PackResult> packedMatmul = packMatmulGreedily(
+      rewriter, linalgOp, mnkTiles, options->mnkPaddedSizesNextMultipleOf,
+      options->mnkOrder);
+  if (failed(packedMatmul))
+    return failure();
+
+  assert(packedMatmul->packOps.size() == 3 &&
+         "invalid number of pack ops after matmul packing");
+  assert(packedMatmul->unPackOps.size() == 1 &&
+         "invalid number of unpack ops after matmul packing");
+
+  FailureOr<ContractionDimensions> contractDims =
+      inferContractionDims(packedMatmul->packedLinalgOp);
+  if (failed(contractDims))
+    return failure();
+
+  auto genericOp =
+      dyn_cast<linalg::GenericOp>(packedMatmul->packedLinalgOp.getOperation());
+  SmallVector<AffineMap> maps = genericOp.getIndexingMapsArray();
+
+  // Transpose LHS matrix according to the options.
+  FailureOr<PackTransposeResult> packedLhs = transposePackedMatmul(
+      rewriter, packedMatmul->packedLinalgOp, packedMatmul->packOps[0], maps[0],
+      contractDims->m, options->lhsTransposeOuterBlocks,
+      options->lhsTransposeInnerBlocks);
+  if (failed(packedLhs))
+    return failure();
+
+  // Update results.
+  packedMatmul->packOps[0] = packedLhs->transposedPackOp;
+  packedMatmul->packedLinalgOp = packedLhs->transposedLinalgOp;
+
+  // Transpose RHS matrix according to the options.
+  FailureOr<PackTransposeResult> packedRhs = transposePackedMatmul(
+      rewriter, packedMatmul->packedLinalgOp, packedMatmul->packOps[1], maps[1],
+      contractDims->k, options->rhsTransposeOuterBlocks,
+      options->rhsTransposeInnerBlocks);
+  if (failed(packedRhs))
+    return failure();
+
+  // Update results.
+  packedMatmul->packOps[1] = packedRhs->transposedPackOp;
+  packedMatmul->packedLinalgOp = packedRhs->transposedLinalgOp;
+
+  return packedMatmul;
+}
+
+namespace {
+template <typename OpTy>
+struct BlockPackMatmul : public OpRewritePattern<OpTy> {
+  BlockPackMatmul(MLIRContext *context, ControlBlockPackMatmulFn fun,
+                  PatternBenefit benefit = 1)
+      : OpRewritePattern<OpTy>(context, benefit), controlFn(std::move(fun)) {}
+
+  LogicalResult matchAndRewrite(OpTy linalgOp,
+                                PatternRewriter &rewriter) const override {
+    FailureOr<PackResult> packedMatmul =
+        blockPackMatmul(rewriter, linalgOp, controlFn);
+    if (failed(packedMatmul))
+      return failure();
+    return success();
+  }
+
+private:
+  ControlBlockPackMatmulFn controlFn;
+};
+
+template <>
+struct BlockPackMatmul<linalg::GenericOp>
+    : public OpRewritePattern<linalg::GenericOp> {
+  BlockPackMatmul(MLIRContext *context, ControlBlockPackMatmulFn fun,
+                  PatternBenefit benefit = 1)
+      : OpRewritePattern<linalg::GenericOp>(context, benefit),
+        controlFn(std::move(fun)) {}
+
+  LogicalResult matchAndRewrite(linalg::GenericOp linalgOp,
+                                PatternRewriter &rewriter) const override {
+    // Match suitable generics.
+    if (failed(linalg::detail::verifyContractionInterface(
+            linalgOp.getOperation()))) {
+      return rewriter.notifyMatchFailure(linalgOp, "not a contraction");
+    }
+
+    using MapList = ArrayRef<ArrayRef<AffineExpr>>;
+    auto infer = [&](MapList m) {
+      return AffineMap::inferFromExprList(m, linalgOp.getContext());
+    };
+
+    AffineExpr i, j, k;
+    bindDims(linalgOp->getContext(), i, j, k);
+    SmallVector<AffineMap> maps = linalgOp.getIndexingMapsArray();
+
+    // For now, only match simple matmuls.
+    if (!(maps == infer({{i, k}, {k, j}, {i, j}}) ||
+          maps == infer({{k, i}, {k, j}, {i, j}}) ||
+          maps == infer({{i, k}, {j, k}, {i, j}}))) {
+      return rewriter.notifyMatchFailure(linalgOp, "not a suitable matmul");
+    }
+
+    FailureOr<PackResult> packedMatmul =
+        blockPackMatmul(rewriter, linalgOp, controlFn);
+    if (failed(packedMatmul))
+      return failure();
+    return success();
+  }
+
+private:
+  ControlBlockPackMatmulFn controlFn;
+};
+
+/// Convert linalg matmul ops to block layout and back.
+struct LinalgBlockPackMatmul
+    : public impl::LinalgBlockPackMatmulBase<LinalgBlockPackMatmul> {
+  using LinalgBlockPackMatmulBase::LinalgBlockPackMatmulBase;
+
+  void runOnOperation() override {
+    Operation *op = getOperation();
+    RewritePatternSet patterns(&getContext());
+
+    ControlBlockPackMatmulFn controlFn =
+        [&](linalg::LinalgOp op) -> BlockPackMatmulOptions {
+      BlockPackMatmulOptions options;
+      options.blockFactors = SmallVector<int64_t>{*blockFactors};
+      options.allowPadding = allowPadding;
+      options.mnkPaddedSizesNextMultipleOf =
+          SmallVector<int64_t>{*mnkPaddedSizesNextMultipleOf};
+      if (!mnkOrder.empty())
+        options.mnkOrder = SmallVector<int64_t>{*mnkOrder};
+      options.lhsTransposeOuterBlocks = lhsTransposeOuterBlocks;
+      options.lhsTransposeInnerBlocks = lhsTransposeInnerBlocks;
+      options.rhsTransposeOuterBlocks = rhsTransposeOuterBlocks;
+      options.rhsTransposeInnerBlocks = rhsTransposeInnerBlocks;
+      return options;
+    };
+
+    linalg::populateBlockPackMatmulPatterns(patterns, controlFn);
+    if (failed(applyPatternsAndFoldGreedily(op, std::move(patterns))))
+      return signalPassFailure();
+  }
+};
+} // namespace
+
+void linalg::populateBlockPackMatmulPatterns(
+    RewritePatternSet &patterns, const ControlBlockPackMatmulFn &controlFn) {
+  patterns.add<BlockPackMatmul<linalg::GenericOp>,
+               BlockPackMatmul<linalg::MatmulOp>,
+               BlockPackMatmul<linalg::BatchMatmulOp>,
+               BlockPackMatmul<linalg::MatmulTransposeAOp>,
+               BlockPackMatmul<linalg::BatchMatmulTransposeAOp>,
+               BlockPackMatmul<linalg::MatmulTransposeBOp>,
+               BlockPackMatmul<linalg::BatchMatmulTransposeBOp>>(
+      patterns.getContext(), controlFn);
+}
diff --git a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
index 3b5282a09569..ed9f40089282 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
@@ -25,6 +25,7 @@ add_mlir_dialect_library(MLIRLinalgTransforms
   TransposeMatmul.cpp
   MeshShardingInterfaceImpl.cpp
   NamedOpConversions.cpp
+  BlockPackMatmul.cpp
   Padding.cpp
   Promotion.cpp
   RuntimeOpVerification.cpp
diff --git a/mlir/test/Dialect/Linalg/block-pack-matmul-layout.mlir b/mlir/test/Dialect/Linalg/block-pack-matmul-layout.mlir
new file mode 100644
index 000000000000..01ca4374da04
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/block-pack-matmul-layout.mlir
@@ -0,0 +1,101 @@
+// RUN: mlir-opt %s -linalg-block-pack-matmul="block-factors=32,16,64 \
+// RUN: lhs-transpose-outer-blocks=false lhs-transpose-inner-blocks=false \
+// RUN: rhs-transpose-outer-blocks=true rhs-transpose-inner-blocks=true" \
+// RUN: -canonicalize | FileCheck %s --check-prefix=MMT4D
+
+// RUN: mlir-opt %s -linalg-block-pack-matmul="block-factors=32,16,64 \
+// RUN: lhs-transpose-outer-blocks=false lhs-transpose-inner-blocks=false \
+// RUN: rhs-transpose-outer-blocks=false rhs-transpose-inner-blocks=false" \
+// RUN: -canonicalize | FileCheck %s --check-prefix=MM4D
+
+// RUN: mlir-opt %s -linalg-block-pack-matmul="block-factors=32,16,64 \
+// RUN: lhs-transpose-outer-blocks=true lhs-transpose-inner-blocks=true \
+// RUN: rhs-transpose-outer-blocks=false rhs-transpose-inner-blocks=false" \
+// RUN: -canonicalize | FileCheck %s --check-prefix=MTM4D
+
+func.func @block_matmul(
+    %A: tensor<64x128xf32>, %B: tensor<128x64xf32>, %C: tensor<64x64xf32>) -> tensor<64x64xf32> {
+  %0 = linalg.matmul  ins(%A, %B : tensor<64x128xf32>, tensor<128x64xf32>)
+                      outs(%C : tensor<64x64xf32>) -> tensor<64x64xf32>
+  return %0 : tensor<64x64xf32>
+}
+
+func.func @block_matmul_transpose_a(
+    %A: tensor<128x64xf32>, %B: tensor<128x64xf32>, %C: tensor<64x64xf32>) -> tensor<64x64xf32> {
+  %0 = linalg.matmul_transpose_a ins(%A, %B : tensor<128x64xf32>, tensor<128x64xf32>)
+                                 outs(%C : tensor<64x64xf32>) -> tensor<64x64xf32>
+  return %0 : tensor<64x64xf32>
+}
+
+func.func @block_matmul_transpose_b(
+    %A: tensor<64x128xf32>, %B: tensor<64x128xf32>, %C: tensor<64x64xf32>) -> tensor<64x64xf32> {
+  %0 = linalg.matmul_transpose_b ins(%A, %B : tensor<64x128xf32>, tensor<64x128xf32>)
+                                 outs(%C : tensor<64x64xf32>) -> tensor<64x64xf32>
+  return %0 : tensor<64x64xf32>
+}
+
+// MMT4D-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
+// MMT4D-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
+// MMT4D-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+// MMT4D-LABEL: func @block_matmul
+// MMT4D-COUNT-3: tensor.pack
+// MMT4D: linalg.generic
+// MMT4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// MMT4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// MMT4D-COUNT-1: tensor.unpack
+// MMT4D-LABEL: func @block_matmul_transpose_a
+// MMT4D-COUNT-3: tensor.pack
+// MMT4D: linalg.generic
+// MMT4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// MMT4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// MMT4D-COUNT-1: tensor.unpack
+// MMT4D-LABEL: func @block_matmul_transpose_b
+// MMT4D-COUNT-3: tensor.pack
+// MMT4D: linalg.generic
+// MMT4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// MMT4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// MMT4D-COUNT-1: tensor.unpack
+
+// MM4D-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
+// MM4D-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d2, d1, d5, d4)>
+// MM4D-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+// MM4D-LABEL: func @block_matmul
+// MM4D-COUNT-3: tensor.pack
+// MM4D: linalg.generic
+// MM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// MM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// MM4D-COUNT-1: tensor.unpack
+// MM4D-LABEL: func @block_matmul_transpose_a
+// MM4D-COUNT-3: tensor.pack
+// MM4D: linalg.generic
+// MM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// MM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// MM4D-COUNT-1: tensor.unpack
+// MM4D-LABEL: func @block_matmul_transpose_b
+// MM4D-COUNT-3: tensor.pack
+// MM4D: linalg.generic
+// MM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// MM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// MM4D-COUNT-1: tensor.unpack
+
+// MTM4D-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d2, d0, d5, d3)>
+// MTM4D-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d2, d1, d5, d4)>
+// MTM4D-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+// MTM4D-LABEL: func @block_matmul
+// MTM4D-COUNT-3: tensor.pack
+// MTM4D: linalg.generic
+// MTM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// MTM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// MTM4D-COUNT-1: tensor.unpack
+// MTM4D-LABEL: func @block_matmul_transpose_a
+// MTM4D-COUNT-3: tensor.pack
+// MTM4D: linalg.generic
+// MTM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// MTM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// MTM4D-COUNT-1: tensor.unpack
+// MTM4D-LABEL: func @block_matmul_transpose_b
+// MTM4D-COUNT-3: tensor.pack
+// MTM4D: linalg.generic
+// MTM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// MTM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// MTM4D-COUNT-1: tensor.unpack
diff --git a/mlir/test/Dialect/Linalg/block-pack-matmul-padding.mlir b/mlir/test/Dialect/Linalg/block-pack-matmul-padding.mlir
new file mode 100644
index 000000000000..9e396ba08d24
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/block-pack-matmul-padding.mlir
@@ -0,0 +1,82 @@
+// RUN: mlir-opt %s -linalg-block-pack-matmul="block-factors=32,16,64 allow-padding=1" \
+// RUN: -canonicalize | FileCheck %s
+
+// RUN: mlir-opt %s -linalg-block-pack-matmul="block-factors=32,16,64 allow-padding=0" \
+// RUN: -canonicalize | FileCheck %s --check-prefix=NOPAD
+
+// RUN: mlir-opt %s -linalg-block-pack-matmul="block-factors=32,16,64 allow-padding=1 mnk-padded-multiples=256,512,384" \
+// RUN: -canonicalize | FileCheck %s --check-prefix=PAD-MULT
+
+func.func @block_matmul_padding(
+    %A: tensor<123x125xf32>, %B: tensor<125x124xf32>, %C: tensor<123x124xf32>) -> tensor<123x124xf32> {
+  %0 = linalg.matmul  ins(%A, %B : tensor<123x125xf32>, tensor<125x124xf32>)
+                      outs(%C : tensor<123x124xf32>) -> tensor<123x124xf32>
+  return %0 : tensor<123x124xf32>
+}
+
+// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
+// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
+// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+// CHECK-LABEL: func @block_matmul_padding(
+// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<123x125xf32>, %[[B:[0-9a-z]+]]: tensor<125x124xf32>, %[[C:[0-9a-z]+]]: tensor<123x124xf32>
+// CHECK-DAG: %[[ZERO:.+]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<4x2x32x64xf32>
+// CHECK: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// CHECK-SAME:  padding_value(%[[ZERO]] : f32)
+// CHECK-SAME:  outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [32, 64]
+// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<123x125xf32> -> tensor<4x2x32x64xf32>
+// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<8x2x16x64xf32>
+// CHECK: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// CHECK-SAME:  padding_value(%[[ZERO]] : f32)
+// CHECK-SAME:  outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [16, 64]
+// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<125x124xf32> -> tensor<8x2x16x64xf32>
+// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<4x8x32x16xf32>
+// CHECK: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// CHECK-SAME:  padding_value(%[[ZERO]] : f32)
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<123x124xf32> -> tensor<4x8x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<4x2x32x64xf32>, tensor<8x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<4x8x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<4x8x32x16xf32> -> tensor<123x124xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<123x124xf32>
+
+// NOPAD-LABEL: func @block_matmul_padding(
+// NOPAD-SAME:    %[[A:[0-9a-z]+]]: tensor<123x125xf32>, %[[B:[0-9a-z]+]]: tensor<125x124xf32>, %[[C:[0-9a-z]+]]: tensor<123x124xf32>
+// NOPAD-NOT: tensor.pack
+// NOPAD: linalg.matmul ins(%[[A]], %[[B]] : tensor<123x125xf32>, tensor<125x124xf32>)
+// NOPAD-SAME: outs(%[[C]] : tensor<123x124xf32>) -> tensor<123x124xf32>
+// NOPAD-NOT: tensor.unpack
+
+// PAD-MULT-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
+// PAD-MULT-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
+// PAD-MULT-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+// PAD-MULT-LABEL: func @block_matmul_padding(
+// PAD-MULT-SAME:    %[[A:[0-9a-z]+]]: tensor<123x125xf32>, %[[B:[0-9a-z]+]]: tensor<125x124xf32>, %[[C:[0-9a-z]+]]: tensor<123x124xf32>
+// PAD-MULT-DAG: %[[ZERO:.+]] = arith.constant 0.000000e+00 : f32
+// PAD-MULT: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<1x1x256x384xf32>
+// PAD-MULT: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// PAD-MULT-SAME:  padding_value(%[[ZERO]] : f32)
+// PAD-MULT-SAME:  outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [256, 384]
+// PAD-MULT-SAME:  into %[[PACK_DST_0]] : tensor<123x125xf32> -> tensor<1x1x256x384xf32>
+// PAD-MULT: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<1x1x512x384xf32>
+// PAD-MULT: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// PAD-MULT-SAME:  padding_value(%[[ZERO]] : f32)
+// PAD-MULT-SAME:  outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [512, 384]
+// PAD-MULT-SAME:  into %[[PACK_DST_1]] : tensor<125x124xf32> -> tensor<1x1x512x384xf32>
+// PAD-MULT: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<1x1x256x512xf32>
+// PAD-MULT: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// PAD-MULT-SAME:  padding_value(%[[ZERO]] : f32)
+// PAD-MULT-SAME:  inner_dims_pos = [0, 1] inner_tiles = [256, 512]
+// PAD-MULT-SAME:  into %[[PACK_DST_2]] : tensor<123x124xf32> -> tensor<1x1x256x512xf32>
+// PAD-MULT: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// PAD-MULT-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// PAD-MULT-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// PAD-MULT-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<1x1x256x384xf32>, tensor<1x1x512x384xf32>) outs(%[[C_PACKED]] : tensor<1x1x256x512xf32>)
+// PAD-MULT: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// PAD-MULT-SAME:  inner_dims_pos = [0, 1] inner_tiles = [256, 512]
+// PAD-MULT-SAME:  into %[[C]] : tensor<1x1x256x512xf32> -> tensor<123x124xf32>
+// PAD-MULT: return %[[RES_UNPACKED]] : tensor<123x124xf32>
diff --git a/mlir/test/Dialect/Linalg/block-pack-matmul.mlir b/mlir/test/Dialect/Linalg/block-pack-matmul.mlir
new file mode 100644
index 000000000000..cc9af913ca15
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/block-pack-matmul.mlir
@@ -0,0 +1,478 @@
+// RUN: mlir-opt %s -linalg-block-pack-matmul=block-factors=32,16,64 -canonicalize -split-input-file | FileCheck %s
+
+func.func @block_matmul(
+    %A: tensor<128x128xf32>, %B: tensor<128x128xf32>, %C: tensor<128x128xf32>) -> tensor<128x128xf32> {
+  %0 = linalg.matmul  ins(%A, %B : tensor<128x128xf32>, tensor<128x128xf32>)
+                      outs(%C : tensor<128x128xf32>) -> tensor<128x128xf32>
+  return %0 : tensor<128x128xf32>
+}
+
+// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
+// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
+// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+
+// CHECK-LABEL: func @block_matmul(
+// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<128x128xf32>, %[[B:[0-9a-z]+]]: tensor<128x128xf32>, %[[C:[0-9a-z]+]]: tensor<128x128xf32>
+// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<4x2x32x64xf32>
+// CHECK: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// CHECK-SAME:  outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [32, 64]
+// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<128x128xf32> -> tensor<4x2x32x64xf32>
+// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<8x2x16x64xf32>
+// CHECK: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// CHECK-SAME:  outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [16, 64]
+// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<128x128xf32> -> tensor<8x2x16x64xf32>
+// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<4x8x32x16xf32>
+// CHECK: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<128x128xf32> -> tensor<4x8x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<4x2x32x64xf32>, tensor<8x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<4x8x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<4x8x32x16xf32> -> tensor<128x128xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<128x128xf32>
+
+// -----
+
+func.func @block_matmul_dynamic(
+    %A: tensor<?x?xf32>, %B: tensor<?x?xf32>, %C: tensor<?x?xf32>) -> tensor<?x?xf32> {
+  %0 = linalg.matmul  ins(%A, %B : tensor<?x?xf32>, tensor<?x?xf32>)
+                      outs(%C : tensor<?x?xf32>) -> tensor<?x?xf32>
+  return %0 : tensor<?x?xf32>
+}
+
+// CHECK-DAG: #[[$MAP_M:.+]] = affine_map<()[s0] -> (s0 ceildiv 32)>
+// CHECK-DAG: #[[$MAP_K:.+]] = affine_map<()[s0] -> (s0 ceildiv 64)>
+// CHECK-DAG: #[[$MAP_N:.+]] = affine_map<()[s0] -> (s0 ceildiv 16)>
+// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
+// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
+// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+
+// CHECK-LABEL: func @block_matmul_dynamic(
+// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<?x?xf32>, %[[B:[0-9a-z]+]]: tensor<?x?xf32>, %[[C:[0-9a-z]+]]: tensor<?x?xf32>
+// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG: %[[C1:.+]] = arith.constant 1 : index
+// CHECK-DAG: %[[ZERO:.+]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG: %[[A_M:.+]] = tensor.dim %[[A]], %[[C0]] : tensor<?x?xf32>
+// CHECK-DAG: %[[A_K:.+]] = tensor.dim %[[A]], %[[C1]] : tensor<?x?xf32>
+// CHECK-DAG: %[[A_OUTER_TILE_M:.+]] = affine.apply #[[$MAP_M]]()[%[[A_M]]]
+// CHECK-DAG: %[[A_OUTER_TILE_K:.+]] = affine.apply #[[$MAP_K]]()[%[[A_K]]]
+// CHECK: %[[PACK_DST_0:.+]] = tensor.empty(%[[A_OUTER_TILE_M]], %[[A_OUTER_TILE_K]]) : tensor<?x?x32x64xf32>
+// CHECK: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// CHECK-SAME:  padding_value(%[[ZERO]] : f32)
+// CHECK-SAME:  outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [32, 64]
+// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<?x?xf32> -> tensor<?x?x32x64xf32>
+// CHECK-DAG: %[[B_K:.+]] = tensor.dim %[[B]], %[[C0]] : tensor<?x?xf32>
+// CHECK-DAG: %[[B_N:.+]] = tensor.dim %[[B]], %[[C1]] : tensor<?x?xf32>
+// CHECK-DAG: %[[B_OUTER_TILE_K:.+]] = affine.apply #[[$MAP_K]]()[%[[B_K]]]
+// CHECK-DAG: %[[B_OUTER_TILE_N:.+]] = affine.apply #[[$MAP_N]]()[%[[B_N]]]
+// CHECK: %[[PACK_DST_1:.+]] = tensor.empty(%[[B_OUTER_TILE_N]], %[[B_OUTER_TILE_K]]) : tensor<?x?x16x64xf32>
+// CHECK: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// CHECK-SAME:  padding_value(%[[ZERO]] : f32)
+// CHECK-SAME:  outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [16, 64]
+// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<?x?xf32> -> tensor<?x?x16x64xf32>
+// CHECK-DAG: %[[C_M:.+]] = tensor.dim %[[C]], %[[C0]] : tensor<?x?xf32>
+// CHECK-DAG: %[[C_N:.+]] = tensor.dim %[[C]], %[[C1]] : tensor<?x?xf32>
+// CHECK-DAG: %[[C_OUTER_TILE_M:.+]] = affine.apply #[[$MAP_M]]()[%[[C_M]]]
+// CHECK-DAG: %[[C_OUTER_TILE_N:.+]] = affine.apply #[[$MAP_N]]()[%[[C_N]]]
+// CHECK: %[[PACK_DST_2:.+]] = tensor.empty(%[[C_OUTER_TILE_M]], %[[C_OUTER_TILE_N]]) : tensor<?x?x32x16xf32>
+// CHECK: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// CHECK-SAME:  padding_value(%[[ZERO]] : f32)
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<?x?xf32> -> tensor<?x?x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<?x?x32x64xf32>, tensor<?x?x16x64xf32>) outs(%[[C_PACKED]] : tensor<?x?x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<?x?x32x16xf32> -> tensor<?x?xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<?x?xf32>
+
+// -----
+
+func.func @block_matmul_with_constant(
+    %A: tensor<128x128xf32>, %B: tensor<128x128xf32>) -> tensor<128x128xf32> {
+  %cst_acc = arith.constant dense<0.0> : tensor<128x128xf32>
+  %0 = linalg.matmul ins(%A, %B : tensor<128x128xf32>, tensor<128x128xf32>)
+                      outs(%cst_acc : tensor<128x128xf32>) -> tensor<128x128xf32>
+  return %0 : tensor<128x128xf32>
+}
+
+// CHECK-LABEL: func @block_matmul_with_constant(
+// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<128x128xf32>, %[[B:[0-9a-z]+]]: tensor<128x128xf32>
+// CHECK-DAG: %[[CST_ACC_PACKED:.+]] = arith.constant dense<0.000000e+00> : tensor<4x8x32x16xf32>
+// CHECK-DAG: %[[RES_DST:.+]] = arith.constant dense<0.000000e+00> : tensor<128x128xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  ins({{.*}} : tensor<4x2x32x64xf32>, tensor<8x2x16x64xf32>) outs(%[[CST_ACC_PACKED]] : tensor<4x8x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[RES_DST]] : tensor<4x8x32x16xf32> -> tensor<128x128xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<128x128xf32>
+
+// -----
+
+func.func @block_matmul_with_producer(
+    %A: tensor<128x128xf32>, %B: tensor<128x128xf32>, %C: tensor<128x128xf32>) -> tensor<128x128xf32> {
+  %cst = arith.constant 0.0 : f32
+  %acc = linalg.fill ins(%cst : f32) outs(%C : tensor<128x128xf32>) -> tensor<128x128xf32>
+  %1 = linalg.matmul ins(%A, %B : tensor<128x128xf32>, tensor<128x128xf32>)
+                      outs(%acc : tensor<128x128xf32>) -> tensor<128x128xf32>
+  return %1 : tensor<128x128xf32>
+}
+
+// CHECK-LABEL: func @block_matmul_with_producer(
+// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<128x128xf32>, %[[B:[0-9a-z]+]]: tensor<128x128xf32>, %[[C:[0-9a-z]+]]: tensor<128x128xf32>
+// CHECK-DAG: %[[C0:.+]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[FILL_DST_PACKED:.+]] = tensor.empty() : tensor<4x8x32x16xf32>
+// CHECK: %[[ACC_PACKED:.+]] = linalg.fill ins(%[[C0]] : f32) outs(%[[FILL_DST_PACKED]] : tensor<4x8x32x16xf32>) -> tensor<4x8x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  ins({{.*}} : tensor<4x2x32x64xf32>, tensor<8x2x16x64xf32>) outs(%[[ACC_PACKED]] : tensor<4x8x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<4x8x32x16xf32> -> tensor<128x128xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<128x128xf32>
+
+// -----
+
+func.func @block_matmul_with_consumer(
+    %A: tensor<128x128xf32>, %B: tensor<128x128xf32>, %C: tensor<128x128xf32>, %D: tensor<128x128xf32>) -> tensor<128x128xf32> {
+  %0 = tensor.empty() : tensor<128x128xf32>
+  %1 = linalg.matmul ins(%A, %B : tensor<128x128xf32>, tensor<128x128xf32>)
+                     outs(%C : tensor<128x128xf32>) -> tensor<128x128xf32>
+  %2 = linalg.add ins(%1, %D : tensor<128x128xf32>, tensor<128x128xf32>)
+                  outs(%0 : tensor<128x128xf32>) -> tensor<128x128xf32>
+  return %2 : tensor<128x128xf32>
+}
+
+// CHECK-LABEL: func @block_matmul_with_consumer(
+// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<128x128xf32>, %[[B:[0-9a-z]+]]: tensor<128x128xf32>, %[[C:[0-9a-z]+]]: tensor<128x128xf32>, %[[D:[0-9a-z]+]]: tensor<128x128xf32>
+// CHECK-DAG: %[[RES_DST:.+]] = tensor.empty() : tensor<128x128xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  outs({{.*}} : tensor<4x8x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<4x8x32x16xf32> -> tensor<128x128xf32>
+// CHECK: %[[ADD_RES:.+]] = linalg.add
+// CHECK-SAME:  ins(%[[RES_UNPACKED]], %[[D]] : tensor<128x128xf32>, tensor<128x128xf32>) outs(%[[RES_DST]] : tensor<128x128xf32>)
+// CHECK: return %[[ADD_RES]] : tensor<128x128xf32>
+
+// -----
+
+func.func @block_batch_matmul(
+    %A: tensor<512x64x128xf32>, %B: tensor<512x128x64xf32>, %C: tensor<512x64x64xf32>) -> tensor<512x64x64xf32> {
+  %0 = linalg.batch_matmul ins(%A, %B : tensor<512x64x128xf32>, tensor<512x128x64xf32>)
+                           outs(%C : tensor<512x64x64xf32>) -> tensor<512x64x64xf32>
+  return %0 : tensor<512x64x64xf32>
+}
+
+// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d1, d3, d4, d6)>
+// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d2, d3, d5, d6)>
+// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d1, d2, d4, d5)>
+
+// CHECK-LABEL: func @block_batch_matmul(
+// CHECK-SAME:   %[[A:.+]]: tensor<512x64x128xf32>, %[[B:.+]]: tensor<512x128x64xf32>, %[[C:.+]]: tensor<512x64x64xf32>
+// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<512x2x2x32x64xf32>
+// CHECK: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// CHECK-SAME:  outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [32, 64]
+// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<512x64x128xf32> -> tensor<512x2x2x32x64xf32>
+// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<512x4x2x16x64xf32>
+// CHECK: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// CHECK-SAME:  outer_dims_perm = [0, 2, 1] inner_dims_pos = [2, 1] inner_tiles = [16, 64]
+// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<512x128x64xf32> -> tensor<512x4x2x16x64xf32>
+// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<512x2x4x32x16xf32>
+// CHECK: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// CHECK-SAME:  inner_dims_pos = [1, 2] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<512x64x64xf32> -> tensor<512x2x4x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:  iterator_types = ["parallel", "parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<512x2x2x32x64xf32>, tensor<512x4x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<512x2x4x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [1, 2] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<512x2x4x32x16xf32> -> tensor<512x64x64xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<512x64x64xf32>
+
+// -----
+
+func.func @block_matmul_transpose_a(
+    %A: tensor<128x64xf32>, %B: tensor<128x64xf32>, %C: tensor<64x64xf32>) -> tensor<64x64xf32> {
+  %0 = linalg.matmul_transpose_a ins(%A, %B : tensor<128x64xf32>, tensor<128x64xf32>)
+                                 outs(%C : tensor<64x64xf32>) -> tensor<64x64xf32>
+  return %0 : tensor<64x64xf32>
+}
+
+// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
+// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
+// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+
+// CHECK-LABEL: func @block_matmul_transpose_a(
+// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<128x64xf32>, %[[B:[0-9a-z]+]]: tensor<128x64xf32>, %[[C:[0-9a-z]+]]: tensor<64x64xf32>
+// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<2x2x32x64xf32>
+// CHECK: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// CHECK-SAME:  outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [32, 64]
+// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<128x64xf32> -> tensor<2x2x32x64xf32>
+// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<4x2x16x64xf32>
+// CHECK: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// CHECK-SAME:  outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [16, 64]
+// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<128x64xf32> -> tensor<4x2x16x64xf32>
+// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<2x4x32x16xf32>
+// CHECK: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<64x64xf32> -> tensor<2x4x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<2x2x32x64xf32>, tensor<4x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<2x4x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<2x4x32x16xf32> -> tensor<64x64xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<64x64xf32>
+
+// -----
+
+func.func @block_batch_matmul_transpose_a(
+    %A: tensor<512x128x64xf32>, %B: tensor<512x128x64xf32>, %C: tensor<512x64x64xf32>) -> tensor<512x64x64xf32> {
+  %0 = linalg.batch_matmul_transpose_a ins(%A, %B : tensor<512x128x64xf32>, tensor<512x128x64xf32>)
+                                       outs(%C : tensor<512x64x64xf32>) -> tensor<512x64x64xf32>
+  return %0 : tensor<512x64x64xf32>
+}
+
+// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d1, d3, d4, d6)>
+// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d2, d3, d5, d6)>
+// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d1, d2, d4, d5)>
+
+// CHECK-LABEL: func @block_batch_matmul_transpose_a(
+// CHECK-SAME:   %[[A:.+]]: tensor<512x128x64xf32>, %[[B:.+]]: tensor<512x128x64xf32>, %[[C:.+]]: tensor<512x64x64xf32>
+// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<512x2x2x32x64xf32>
+// CHECK: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// CHECK-SAME:  outer_dims_perm = [0, 2, 1] inner_dims_pos = [2, 1] inner_tiles = [32, 64]
+// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<512x128x64xf32> -> tensor<512x2x2x32x64xf32>
+// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<512x4x2x16x64xf32>
+// CHECK: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// CHECK-SAME:  outer_dims_perm = [0, 2, 1] inner_dims_pos = [2, 1] inner_tiles = [16, 64]
+// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<512x128x64xf32> -> tensor<512x4x2x16x64xf32>
+// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<512x2x4x32x16xf32>
+// CHECK: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// CHECK-SAME:  inner_dims_pos = [1, 2] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<512x64x64xf32> -> tensor<512x2x4x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:  iterator_types = ["parallel", "parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<512x2x2x32x64xf32>, tensor<512x4x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<512x2x4x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [1, 2] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<512x2x4x32x16xf32> -> tensor<512x64x64xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<512x64x64xf32>
+
+// -----
+
+func.func @block_matmul_transpose_b(
+    %A: tensor<64x128xf32>, %B: tensor<64x128xf32>, %C: tensor<64x64xf32>) -> tensor<64x64xf32> {
+  %0 = linalg.matmul_transpose_b ins(%A, %B : tensor<64x128xf32>, tensor<64x128xf32>)
+                                 outs(%C : tensor<64x64xf32>) -> tensor<64x64xf32>
+  return %0 : tensor<64x64xf32>
+}
+
+// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
+// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
+// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+
+// CHECK-LABEL: func @block_matmul_transpose_b(
+// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<64x128xf32>, %[[B:[0-9a-z]+]]: tensor<64x128xf32>, %[[C:[0-9a-z]+]]: tensor<64x64xf32>
+// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<2x2x32x64xf32>
+// CHECK: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// CHECK-SAME:  outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [32, 64]
+// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<64x128xf32> -> tensor<2x2x32x64xf32>
+// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<4x2x16x64xf32>
+// CHECK: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// CHECK-SAME:  outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [16, 64]
+// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<64x128xf32> -> tensor<4x2x16x64xf32>
+// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<2x4x32x16xf32>
+// CHECK: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<64x64xf32> -> tensor<2x4x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<2x2x32x64xf32>, tensor<4x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<2x4x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<2x4x32x16xf32> -> tensor<64x64xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<64x64xf32>
+
+// -----
+
+func.func @block_batch_matmul_transpose_b(
+    %A: tensor<512x64x128xf32>, %B: tensor<512x64x128xf32>, %C: tensor<512x64x64xf32>) -> tensor<512x64x64xf32> {
+  %0 = linalg.batch_matmul_transpose_b ins(%A, %B : tensor<512x64x128xf32>, tensor<512x64x128xf32>)
+                                       outs(%C : tensor<512x64x64xf32>) -> tensor<512x64x64xf32>
+  return %0 : tensor<512x64x64xf32>
+}
+
+// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d1, d3, d4, d6)>
+// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d2, d3, d5, d6)>
+// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d1, d2, d4, d5)>
+
+// CHECK-LABEL: func @block_batch_matmul_transpose_b(
+// CHECK-SAME:   %[[A:.+]]: tensor<512x64x128xf32>, %[[B:.+]]: tensor<512x64x128xf32>, %[[C:.+]]: tensor<512x64x64xf32>
+// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<512x2x2x32x64xf32>
+// CHECK: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// CHECK-SAME:  outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [32, 64]
+// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<512x64x128xf32> -> tensor<512x2x2x32x64xf32>
+// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<512x4x2x16x64xf32>
+// CHECK: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// CHECK-SAME:  outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 64]
+// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<512x64x128xf32> -> tensor<512x4x2x16x64xf32>
+// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<512x2x4x32x16xf32>
+// CHECK: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// CHECK-SAME:  inner_dims_pos = [1, 2] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<512x64x64xf32> -> tensor<512x2x4x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:  iterator_types = ["parallel", "parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<512x2x2x32x64xf32>, tensor<512x4x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<512x2x4x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [1, 2] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<512x2x4x32x16xf32> -> tensor<512x64x64xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<512x64x64xf32>
+
+// -----
+
+#map = affine_map<(d0, d1, d2) -> (d0, d2)>
+#map1 = affine_map<(d0, d1, d2) -> (d2, d1)>
+#map2 = affine_map<(d0, d1, d2) -> (d0, d1)>
+
+func.func @block_generic_matmul(
+    %A: tensor<128x128xf32>, %B: tensor<128x128xf32>, %C: tensor<128x128xf32>) -> tensor<128x128xf32> {
+  %0 = linalg.generic {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel", "reduction"]}
+    ins(%A, %B : tensor<128x128xf32>, tensor<128x128xf32>)
+    outs(%C : tensor<128x128xf32>) {
+  ^bb0(%in: f32, %in_0: f32, %out: f32):
+    %1 = arith.mulf %in, %in_0 : f32
+    %2 = arith.addf %out, %1 : f32
+    linalg.yield %2 : f32
+  } -> tensor<128x128xf32>
+  return %0 : tensor<128x128xf32>
+}
+
+// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
+// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
+// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+
+// CHECK-LABEL: func @block_generic_matmul(
+// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<128x128xf32>, %[[B:[0-9a-z]+]]: tensor<128x128xf32>, %[[C:[0-9a-z]+]]: tensor<128x128xf32>
+// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<4x2x32x64xf32>
+// CHECK: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// CHECK-SAME:  outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [32, 64]
+// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<128x128xf32> -> tensor<4x2x32x64xf32>
+// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<8x2x16x64xf32>
+// CHECK: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// CHECK-SAME:  outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [16, 64]
+// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<128x128xf32> -> tensor<8x2x16x64xf32>
+// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<4x8x32x16xf32>
+// CHECK: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<128x128xf32> -> tensor<4x8x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<4x2x32x64xf32>, tensor<8x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<4x8x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<4x8x32x16xf32> -> tensor<128x128xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<128x128xf32>
+
+// -----
+
+#map = affine_map<(d0, d1, d2) -> (d2, d0)>
+#map1 = affine_map<(d0, d1, d2) -> (d2, d1)>
+#map2 = affine_map<(d0, d1, d2) -> (d0, d1)>
+
+func.func @block_generic_matmul_transpose_a(
+    %A: tensor<128x64xf32>, %B: tensor<128x64xf32>, %C: tensor<64x64xf32>) -> tensor<64x64xf32> {
+  %0 = linalg.generic {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel", "reduction"]}
+    ins(%A, %B : tensor<128x64xf32>, tensor<128x64xf32>)
+    outs(%C : tensor<64x64xf32>) {
+  ^bb0(%in: f32, %in_0: f32, %out: f32):
+    %1 = arith.mulf %in, %in_0 : f32
+    %2 = arith.addf %out, %1 : f32
+    linalg.yield %2 : f32
+  } -> tensor<64x64xf32>
+  return %0 : tensor<64x64xf32>
+}
+
+// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
+// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
+// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+
+// CHECK-LABEL: func @block_generic_matmul_transpose_a(
+// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<128x64xf32>, %[[B:[0-9a-z]+]]: tensor<128x64xf32>, %[[C:[0-9a-z]+]]: tensor<64x64xf32>
+// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<2x2x32x64xf32>
+// CHECK: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// CHECK-SAME:  outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [32, 64]
+// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<128x64xf32> -> tensor<2x2x32x64xf32>
+// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<4x2x16x64xf32>
+// CHECK: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// CHECK-SAME:  outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [16, 64]
+// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<128x64xf32> -> tensor<4x2x16x64xf32>
+// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<2x4x32x16xf32>
+// CHECK: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<64x64xf32> -> tensor<2x4x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<2x2x32x64xf32>, tensor<4x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<2x4x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<2x4x32x16xf32> -> tensor<64x64xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<64x64xf32>
+
+// -----
+
+#map = affine_map<(d0, d1, d2) -> (d0, d2)>
+#map1 = affine_map<(d0, d1, d2) -> (d1, d2)>
+#map2 = affine_map<(d0, d1, d2) -> (d0, d1)>
+
+func.func @block_generic_matmul_transpose_b(
+    %A: tensor<64x128xf32>, %B: tensor<64x128xf32>, %C: tensor<64x64xf32>) -> tensor<64x64xf32> {
+  %0 = linalg.generic {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel", "reduction"]}
+    ins(%A, %B : tensor<64x128xf32>, tensor<64x128xf32>)
+    outs(%C : tensor<64x64xf32>) {
+  ^bb0(%in: f32, %in_0: f32, %out: f32):
+    %1 = arith.mulf %in, %in_0 : f32
+    %2 = arith.addf %out, %1 : f32
+    linalg.yield %2 : f32
+  } -> tensor<64x64xf32>
+  return %0 : tensor<64x64xf32>
+}
+
+// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
+// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
+// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
+
+// CHECK-LABEL: func @block_generic_matmul_transpose_b(
+// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<64x128xf32>, %[[B:[0-9a-z]+]]: tensor<64x128xf32>, %[[C:[0-9a-z]+]]: tensor<64x64xf32>
+// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<2x2x32x64xf32>
+// CHECK: %[[A_PACKED:.+]] = tensor.pack %[[A]]
+// CHECK-SAME:  outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [32, 64]
+// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<64x128xf32> -> tensor<2x2x32x64xf32>
+// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<4x2x16x64xf32>
+// CHECK: %[[B_PACKED:.+]] = tensor.pack %[[B]]
+// CHECK-SAME:  outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [16, 64]
+// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<64x128xf32> -> tensor<4x2x16x64xf32>
+// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<2x4x32x16xf32>
+// CHECK: %[[C_PACKED:.+]] = tensor.pack %[[C]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<64x64xf32> -> tensor<2x4x32x16xf32>
+// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
+// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
+// CHECK-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
+// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<2x2x32x64xf32>, tensor<4x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<2x4x32x16xf32>)
+// CHECK: %[[RES_UNPACKED:.+]] = tensor.unpack %[[GEMM_RES_PACKED]]
+// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:  into %[[C]] : tensor<2x4x32x16xf32> -> tensor<64x64xf32>
+// CHECK: return %[[RES_UNPACKED]] : tensor<64x64xf32>
-- 
GitLab


From 317e6ff6290c4c3065cb79c3eaf52f171e40cdfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Danny=20M=C3=B6sch?= <danny.moesch@icloud.com>
Date: Thu, 9 May 2024 19:07:43 +0200
Subject: [PATCH 0318/1206] [NFC] Move parameter into field (#91065)

Fixes #89194.
---
 llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
index 06f1396c06fe..d8e41fe92258 100644
--- a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
+++ b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
@@ -97,13 +97,15 @@ struct LTOCodeGenerator {
   void setFileType(CodeGenFileType FT) { Config.CGFileType = FT; }
 
   void setCpu(StringRef MCpu) { Config.CPU = std::string(MCpu); }
-  void setAttrs(std::vector<std::string> MAttrs) { Config.MAttrs = MAttrs; }
+  void setAttrs(std::vector<std::string> MAttrs) {
+    Config.MAttrs = std::move(MAttrs);
+  }
   void setOptLevel(unsigned OptLevel);
 
   void setShouldInternalize(bool Value) { ShouldInternalize = Value; }
   void setShouldEmbedUselists(bool Value) { ShouldEmbedUselists = Value; }
   void setSaveIRBeforeOptPath(std::string Value) {
-    SaveIRBeforeOptPath = Value;
+    SaveIRBeforeOptPath = std::move(Value);
   }
 
   /// Restore linkage of globals
-- 
GitLab


From e1f279e92ddda4e4fdd6fff165b950d0879fa41e Mon Sep 17 00:00:00 2001
From: lntue <35648136+lntue@users.noreply.github.com>
Date: Thu, 9 May 2024 13:10:43 -0400
Subject: [PATCH 0319/1206] [libc] Use __builtin_fma(f) by default if
 LIBC_TARGET_CPU_HAS_FMA is defined. (#91535)

---
 libc/src/__support/FPUtil/FMA.h               | 26 +++++----
 libc/src/__support/FPUtil/aarch64/FMA.h       | 50 -----------------
 libc/src/__support/FPUtil/gpu/FMA.h           | 36 ------------
 libc/src/__support/FPUtil/riscv/FMA.h         | 54 ------------------
 libc/src/__support/FPUtil/x86_64/FMA.h        | 55 -------------------
 .../llvm-project-overlay/libc/BUILD.bazel     |  8 ---
 6 files changed, 16 insertions(+), 213 deletions(-)
 delete mode 100644 libc/src/__support/FPUtil/aarch64/FMA.h
 delete mode 100644 libc/src/__support/FPUtil/gpu/FMA.h
 delete mode 100644 libc/src/__support/FPUtil/riscv/FMA.h
 delete mode 100644 libc/src/__support/FPUtil/x86_64/FMA.h

diff --git a/libc/src/__support/FPUtil/FMA.h b/libc/src/__support/FPUtil/FMA.h
index 0e1ede02d5cc..c277da49538b 100644
--- a/libc/src/__support/FPUtil/FMA.h
+++ b/libc/src/__support/FPUtil/FMA.h
@@ -9,25 +9,31 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_FMA_H
 #define LLVM_LIBC_SRC___SUPPORT_FPUTIL_FMA_H
 
+#include "src/__support/CPP/type_traits.h"
 #include "src/__support/macros/properties/architectures.h"
 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
 
 #if defined(LIBC_TARGET_CPU_HAS_FMA)
 
-#if defined(LIBC_TARGET_ARCH_IS_X86_64)
-#include "x86_64/FMA.h"
-#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
-#include "aarch64/FMA.h"
-#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
-#include "riscv/FMA.h"
-#elif defined(LIBC_TARGET_ARCH_IS_GPU)
-#include "gpu/FMA.h"
-#endif
+namespace LIBC_NAMESPACE {
+namespace fputil {
+
+template <typename T>
+LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, float>, T> fma(T x, T y, T z) {
+  return __builtin_fmaf(x, y, z);
+}
+
+template <typename T>
+LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, double>, T> fma(T x, T y, T z) {
+  return __builtin_fma(x, y, z);
+}
+
+} // namespace fputil
+} // namespace LIBC_NAMESPACE
 
 #else
 // FMA instructions are not available
 #include "generic/FMA.h"
-#include "src/__support/CPP/type_traits.h"
 
 namespace LIBC_NAMESPACE {
 namespace fputil {
diff --git a/libc/src/__support/FPUtil/aarch64/FMA.h b/libc/src/__support/FPUtil/aarch64/FMA.h
deleted file mode 100644
index 6254a0673ff4..000000000000
--- a/libc/src/__support/FPUtil/aarch64/FMA.h
+++ /dev/null
@@ -1,50 +0,0 @@
-//===-- Aarch64 implementations of the fma function -------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_AARCH64_FMA_H
-#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_AARCH64_FMA_H
-
-#include "src/__support/macros/attributes.h" // LIBC_INLINE
-#include "src/__support/macros/properties/architectures.h"
-#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-
-#if !defined(LIBC_TARGET_ARCH_IS_AARCH64)
-#error "Invalid include"
-#endif
-
-#if !defined(LIBC_TARGET_CPU_HAS_FMA)
-#error "FMA instructions are not supported"
-#endif
-
-#include "src/__support/CPP/type_traits.h"
-
-namespace LIBC_NAMESPACE {
-namespace fputil {
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, float>, T> fma(T x, T y, T z) {
-  float result;
-  LIBC_INLINE_ASM("fmadd %s0, %s1, %s2, %s3\n\t"
-                  : "=w"(result)
-                  : "w"(x), "w"(y), "w"(z));
-  return result;
-}
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, double>, T> fma(T x, T y, T z) {
-  double result;
-  LIBC_INLINE_ASM("fmadd %d0, %d1, %d2, %d3\n\t"
-                  : "=w"(result)
-                  : "w"(x), "w"(y), "w"(z));
-  return result;
-}
-
-} // namespace fputil
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_AARCH64_FMA_H
diff --git a/libc/src/__support/FPUtil/gpu/FMA.h b/libc/src/__support/FPUtil/gpu/FMA.h
deleted file mode 100644
index ef1cd26a72dd..000000000000
--- a/libc/src/__support/FPUtil/gpu/FMA.h
+++ /dev/null
@@ -1,36 +0,0 @@
-//===-- GPU implementations of the fma function -----------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_GPU_FMA_H
-#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_GPU_FMA_H
-
-#include "src/__support/CPP/type_traits.h"
-
-// These intrinsics map to the FMA instructions in the target ISA for the GPU.
-// The default rounding mode generated from these will be to the nearest even.
-#if !__has_builtin(__builtin_fma) || !__has_builtin(__builtin_fmaf)
-#error "FMA builtins must be defined");
-#endif
-
-namespace LIBC_NAMESPACE {
-namespace fputil {
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, float>, T> fma(T x, T y, T z) {
-  return __builtin_fmaf(x, y, z);
-}
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, double>, T> fma(T x, T y, T z) {
-  return __builtin_fma(x, y, z);
-}
-
-} // namespace fputil
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_GPU_FMA_H
diff --git a/libc/src/__support/FPUtil/riscv/FMA.h b/libc/src/__support/FPUtil/riscv/FMA.h
deleted file mode 100644
index f01962174f16..000000000000
--- a/libc/src/__support/FPUtil/riscv/FMA.h
+++ /dev/null
@@ -1,54 +0,0 @@
-//===-- RISCV implementations of the fma function ---------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_RISCV_FMA_H
-#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_RISCV_FMA_H
-
-#include "src/__support/macros/attributes.h" // LIBC_INLINE
-#include "src/__support/macros/properties/architectures.h"
-#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-
-#if !defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
-#error "Invalid include"
-#endif
-
-#if !defined(LIBC_TARGET_CPU_HAS_FMA)
-#error "FMA instructions are not supported"
-#endif
-
-#include "src/__support/CPP/type_traits.h"
-
-namespace LIBC_NAMESPACE {
-namespace fputil {
-
-#ifdef __riscv_flen
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, float>, T> fma(T x, T y, T z) {
-  float result;
-  LIBC_INLINE_ASM("fmadd.s %0, %1, %2, %3\n\t"
-                  : "=f"(result)
-                  : "f"(x), "f"(y), "f"(z));
-  return result;
-}
-
-#if __riscv_flen >= 64
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, double>, T> fma(T x, T y, T z) {
-  double result;
-  LIBC_INLINE_ASM("fmadd.d %0, %1, %2, %3\n\t"
-                  : "=f"(result)
-                  : "f"(x), "f"(y), "f"(z));
-  return result;
-}
-#endif // __riscv_flen >= 64
-#endif // __riscv_flen
-
-} // namespace fputil
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_RISCV_FMA_H
diff --git a/libc/src/__support/FPUtil/x86_64/FMA.h b/libc/src/__support/FPUtil/x86_64/FMA.h
deleted file mode 100644
index 91ef7f96ff4d..000000000000
--- a/libc/src/__support/FPUtil/x86_64/FMA.h
+++ /dev/null
@@ -1,55 +0,0 @@
-//===-- x86_64 implementations of the fma function --------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FMA_H
-#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FMA_H
-
-#include "src/__support/macros/attributes.h" // LIBC_INLINE
-#include "src/__support/macros/properties/architectures.h"
-#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-
-#if !defined(LIBC_TARGET_ARCH_IS_X86_64)
-#error "Invalid include"
-#endif
-
-#if !defined(LIBC_TARGET_CPU_HAS_FMA)
-#error "FMA instructions are not supported"
-#endif
-
-#include "src/__support/CPP/type_traits.h"
-#include <immintrin.h>
-
-namespace LIBC_NAMESPACE {
-namespace fputil {
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, float>, T> fma(T x, T y, T z) {
-  float result;
-  __m128 xmm = _mm_load_ss(&x);           // NOLINT
-  __m128 ymm = _mm_load_ss(&y);           // NOLINT
-  __m128 zmm = _mm_load_ss(&z);           // NOLINT
-  __m128 r = _mm_fmadd_ss(xmm, ymm, zmm); // NOLINT
-  _mm_store_ss(&result, r);               // NOLINT
-  return result;
-}
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, double>, T> fma(T x, T y, T z) {
-  double result;
-  __m128d xmm = _mm_load_sd(&x);           // NOLINT
-  __m128d ymm = _mm_load_sd(&y);           // NOLINT
-  __m128d zmm = _mm_load_sd(&z);           // NOLINT
-  __m128d r = _mm_fmadd_sd(xmm, ymm, zmm); // NOLINT
-  _mm_store_sd(&result, r);                // NOLINT
-  return result;
-}
-
-} // namespace fputil
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FMA_H
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 055630cb6a00..6255ac998db1 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -913,17 +913,9 @@ fma_common_hdrs = [
     "src/__support/FPUtil/generic/FMA.h",
 ]
 
-fma_platform_hdrs = [
-    "src/__support/FPUtil/x86_64/FMA.h",
-    "src/__support/FPUtil/aarch64/FMA.h",
-]
-
 libc_support_library(
     name = "__support_fputil_fma",
     hdrs = fma_common_hdrs,
-    # These are conditionally included and will #error out if the platform
-    # doesn't support FMA, so they can't be compiled on their own.
-    textual_hdrs = fma_platform_hdrs,
     deps = [
         ":__support_cpp_bit",
         ":__support_cpp_type_traits",
-- 
GitLab


From ecae3ed958481cba7d60868cf3504292f7f4fdf5 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 9 May 2024 18:12:54 +0100
Subject: [PATCH 0320/1206] [LAA] Apply loop guards to dependence distance.

After supporting non-constant dependence distances in 933f49248bf,
applying information from loop guards can help further disambiguate
dependencies.
---
 llvm/lib/Analysis/LoopAccessAnalysis.cpp      |  3 +
 .../offset-range-known-via-assume.ll          | 62 +++++++------------
 2 files changed, 27 insertions(+), 38 deletions(-)

diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index ae7f0373c4e8..6fc7da168b42 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -2005,6 +2005,9 @@ getDependenceDistanceStrideAndSize(
     return MemoryDepChecker::Dependence::Unknown;
   }
 
+  if (!isa<SCEVConstant, SCEVCouldNotCompute>(Dist))
+    Dist = SE.applyLoopGuards(Dist, InnermostLoop);
+
   uint64_t TypeByteSize = DL.getTypeAllocSize(ATy);
   bool HasSameSize =
       DL.getTypeStoreSizeInBits(ATy) == DL.getTypeStoreSizeInBits(BTy);
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/offset-range-known-via-assume.ll b/llvm/test/Analysis/LoopAccessAnalysis/offset-range-known-via-assume.ll
index 7e36da78d6aa..c358b00dad22 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/offset-range-known-via-assume.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/offset-range-known-via-assume.ll
@@ -7,26 +7,19 @@ declare void @llvm.assume(i1)
 
 declare void @use(ptr noundef)
 
-; TODO: %offset is known positive via assume, so we should be able to detect the
+; %offset is known positive via assume, so we should be able to detect the
 ; forward dependence.
 define void @offset_i8_known_positive_via_assume_forward_dep_1(ptr %A, i64 %offset, i64 %N) {
 ; CHECK-LABEL: 'offset_i8_known_positive_via_assume_forward_dep_1'
 ; CHECK-NEXT:    loop:
-; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Memory dependences are safe
 ; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Forward:
+; CHECK-NEXT:            %l = load i8, ptr %gep.off, align 4 ->
+; CHECK-NEXT:            store i8 %add, ptr %gep, align 4
+; CHECK-EMPTY:
 ; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Check 0:
-; CHECK-NEXT:        Comparing group ([[GRP1:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
-; CHECK-NEXT:        Against group ([[GRP2:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep.off = getelementptr inbounds i8, ptr %off, i64 %iv
 ; CHECK-NEXT:      Grouped accesses:
-; CHECK-NEXT:        Group [[GRP1]]:
-; CHECK-NEXT:          (Low: %A High: (%N + %A))
-; CHECK-NEXT:            Member: {%A,+,1}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP2]]:
-; CHECK-NEXT:          (Low: (%offset + %A) High: (%offset + %N + %A))
-; CHECK-NEXT:            Member: {(%offset + %A),+,1}<nw><%loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -62,15 +55,15 @@ define void @offset_i32_known_positive_via_assume_forward_dep_1(ptr %A, i64 %off
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Check 0:
-; CHECK-NEXT:        Comparing group ([[GRP3:0x[0-9a-f]+]]):
+; CHECK-NEXT:        Comparing group ([[GRP1:0x[0-9a-f]+]]):
 ; CHECK-NEXT:          %gep = getelementptr inbounds i32, ptr %A, i64 %iv
-; CHECK-NEXT:        Against group ([[GRP4:0x[0-9a-f]+]]):
+; CHECK-NEXT:        Against group ([[GRP2:0x[0-9a-f]+]]):
 ; CHECK-NEXT:          %gep.off = getelementptr inbounds i32, ptr %off, i64 %iv
 ; CHECK-NEXT:      Grouped accesses:
-; CHECK-NEXT:        Group [[GRP3]]:
+; CHECK-NEXT:        Group [[GRP1]]:
 ; CHECK-NEXT:          (Low: %A High: (-3 + (4 * %N) + %A))
 ; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP4]]:
+; CHECK-NEXT:        Group [[GRP2]]:
 ; CHECK-NEXT:          (Low: ((4 * %offset)<nsw> + %A) High: (-3 + (4 * %offset)<nsw> + (4 * %N) + %A))
 ; CHECK-NEXT:            Member: {((4 * %offset)<nsw> + %A),+,4}<nw><%loop>
 ; CHECK-EMPTY:
@@ -103,26 +96,19 @@ exit:
   ret void
 }
 
-; TODO: %offset is known positive via assume, so we should be able to detect the
+; %offset is known positive via assume, so we should be able to detect the
 ; forward dependence.
 define void @offset_known_positive_via_assume_forward_dep_2(ptr %A, i64 %offset, i64 %N) {
 ; CHECK-LABEL: 'offset_known_positive_via_assume_forward_dep_2'
 ; CHECK-NEXT:    loop:
-; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Memory dependences are safe
 ; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Forward:
+; CHECK-NEXT:            %l = load i32, ptr %gep.off, align 4 ->
+; CHECK-NEXT:            store i32 %add, ptr %gep, align 4
+; CHECK-EMPTY:
 ; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Check 0:
-; CHECK-NEXT:        Comparing group ([[GRP5:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep = getelementptr inbounds i32, ptr %A, i64 %iv
-; CHECK-NEXT:        Against group ([[GRP6:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep.off = getelementptr inbounds i32, ptr %off, i64 %iv
 ; CHECK-NEXT:      Grouped accesses:
-; CHECK-NEXT:        Group [[GRP5]]:
-; CHECK-NEXT:          (Low: %A High: ((4 * %N) + %A))
-; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP6]]:
-; CHECK-NEXT:          (Low: ((4 * %offset)<nsw> + %A) High: ((4 * %offset)<nsw> + (4 * %N) + %A))
-; CHECK-NEXT:            Member: {((4 * %offset)<nsw> + %A),+,4}<nw><%loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -160,15 +146,15 @@ define void @offset_may_be_negative_via_assume_unknown_dep(ptr %A, i64 %offset,
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Check 0:
-; CHECK-NEXT:        Comparing group ([[GRP7:0x[0-9a-f]+]]):
+; CHECK-NEXT:        Comparing group ([[GRP3:0x[0-9a-f]+]]):
 ; CHECK-NEXT:          %gep.mul.2 = getelementptr inbounds i32, ptr %A, i64 %iv
-; CHECK-NEXT:        Against group ([[GRP8:0x[0-9a-f]+]]):
+; CHECK-NEXT:        Against group ([[GRP4:0x[0-9a-f]+]]):
 ; CHECK-NEXT:          %gep = getelementptr inbounds i32, ptr %off, i64 %iv
 ; CHECK-NEXT:      Grouped accesses:
-; CHECK-NEXT:        Group [[GRP7]]:
+; CHECK-NEXT:        Group [[GRP3]]:
 ; CHECK-NEXT:          (Low: %A High: ((4 * %N) + %A))
 ; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP8]]:
+; CHECK-NEXT:        Group [[GRP4]]:
 ; CHECK-NEXT:          (Low: ((4 * %offset)<nsw> + %A) High: ((4 * %offset)<nsw> + (4 * %N) + %A))
 ; CHECK-NEXT:            Member: {((4 * %offset)<nsw> + %A),+,4}<nw><%loop>
 ; CHECK-EMPTY:
@@ -207,15 +193,15 @@ define void @offset_no_assumes(ptr %A, i64 %offset, i64 %N) {
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
 ; CHECK-NEXT:      Check 0:
-; CHECK-NEXT:        Comparing group ([[GRP9:0x[0-9a-f]+]]):
+; CHECK-NEXT:        Comparing group ([[GRP5:0x[0-9a-f]+]]):
 ; CHECK-NEXT:          %gep = getelementptr inbounds i32, ptr %A, i64 %iv
-; CHECK-NEXT:        Against group ([[GRP10:0x[0-9a-f]+]]):
+; CHECK-NEXT:        Against group ([[GRP6:0x[0-9a-f]+]]):
 ; CHECK-NEXT:          %gep.off = getelementptr inbounds i32, ptr %off, i64 %iv
 ; CHECK-NEXT:      Grouped accesses:
-; CHECK-NEXT:        Group [[GRP9]]:
+; CHECK-NEXT:        Group [[GRP5]]:
 ; CHECK-NEXT:          (Low: %A High: ((4 * %N) + %A))
 ; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP10]]:
+; CHECK-NEXT:        Group [[GRP6]]:
 ; CHECK-NEXT:          (Low: ((4 * %offset)<nsw> + %A) High: ((4 * %offset)<nsw> + (4 * %N) + %A))
 ; CHECK-NEXT:            Member: {((4 * %offset)<nsw> + %A),+,4}<nw><%loop>
 ; CHECK-EMPTY:
-- 
GitLab


From 785b143a402a282822c3d5e30bb4e2b1980c0b1e Mon Sep 17 00:00:00 2001
From: Med Ismail Bennani <ismail@bennani.ma>
Date: Thu, 9 May 2024 10:13:20 -0700
Subject: [PATCH 0321/1206] [lldb/crashlog] Enforce image loading policy
 (#91109)

In `27f27d1`, we changed the image loading logic to conform to the
various options (`-a|--load-all` & `-c|--crashed-only`) and loaded them
concurrently.

However, instead of the subset of images that matched the user option,
the thread pool would always run on all the crashlog images, causing
them to be all loaded in the target everytime.

This matches the `-a|--load-all` option behaviour but depending on the
report, it can cause lldb to load thousands of images, which can take a
very long time if the images are downloaded over the network.

This patch fixes that issue by keeping a list of `images_to_load` based
of
the user-provided option. This list will be used with our executor
thread pool to load the images according to the user selection, and
reinstates
the expected default behaviour, by only loading the crashed thread
images and
skipping all the others.

This patch also unifies the way we load images into a single method
that's shared by both the batch mode & the interactive scripted process.

rdar://123694062

Signed-off-by: Med Ismail Bennani <ismail@bennani.ma>
---
 lldb/examples/python/crashlog.py              | 82 +++++++++++--------
 .../python/crashlog_scripted_process.py       | 38 +++------
 2 files changed, 63 insertions(+), 57 deletions(-)

diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py
index c992348b24be..d147d0e322a3 100755
--- a/lldb/examples/python/crashlog.py
+++ b/lldb/examples/python/crashlog.py
@@ -252,7 +252,7 @@ class CrashLog(symbolication.Symbolicator):
                 self.idents.append(ident)
 
         def did_crash(self):
-            return self.reason is not None
+            return self.crashed
 
         def __str__(self):
             if self.app_specific_backtrace:
@@ -526,6 +526,49 @@ class CrashLog(symbolication.Symbolicator):
     def get_target(self):
         return self.target
 
+    def load_images(self, options, loaded_images=None):
+        if not loaded_images:
+            loaded_images = []
+        images_to_load = self.images
+        if options.load_all_images:
+            for image in self.images:
+                image.resolve = True
+        elif options.crashed_only:
+            for thread in self.threads:
+                if thread.did_crash():
+                    images_to_load = []
+                    for ident in thread.idents:
+                        for image in self.find_images_with_identifier(ident):
+                            image.resolve = True
+                            images_to_load.append(image)
+
+        futures = []
+        with tempfile.TemporaryDirectory() as obj_dir:
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+
+                def add_module(image, target, obj_dir):
+                    return image, image.add_module(target, obj_dir)
+
+                for image in images_to_load:
+                    if image not in loaded_images:
+                        if image.uuid == uuid.UUID(int=0):
+                            continue
+                        futures.append(
+                            executor.submit(
+                                add_module,
+                                image=image,
+                                target=self.target,
+                                obj_dir=obj_dir,
+                            )
+                        )
+
+                for future in concurrent.futures.as_completed(futures):
+                    image, err = future.result()
+                    if err:
+                        print(err)
+                    else:
+                        loaded_images.append(image)
+
 
 class CrashLogFormatException(Exception):
     pass
@@ -1408,36 +1451,7 @@ def SymbolicateCrashLog(crash_log, options):
     if not target:
         return
 
-    if options.load_all_images:
-        for image in crash_log.images:
-            image.resolve = True
-    elif options.crashed_only:
-        for thread in crash_log.threads:
-            if thread.did_crash():
-                for ident in thread.idents:
-                    for image in crash_log.find_images_with_identifier(ident):
-                        image.resolve = True
-
-    futures = []
-    loaded_images = []
-    with tempfile.TemporaryDirectory() as obj_dir:
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-
-            def add_module(image, target, obj_dir):
-                return image, image.add_module(target, obj_dir)
-
-            for image in crash_log.images:
-                futures.append(
-                    executor.submit(
-                        add_module, image=image, target=target, obj_dir=obj_dir
-                    )
-                )
-            for future in concurrent.futures.as_completed(futures):
-                image, err = future.result()
-                if err:
-                    print(err)
-                else:
-                    loaded_images.append(image)
+    crash_log.load_images(options)
 
     if crash_log.backtraces:
         for thread in crash_log.backtraces:
@@ -1498,7 +1512,11 @@ def load_crashlog_in_scripted_process(debugger, crashlog_path, options, result):
     structured_data = lldb.SBStructuredData()
     structured_data.SetFromJSON(
         json.dumps(
-            {"file_path": crashlog_path, "load_all_images": options.load_all_images}
+            {
+                "file_path": crashlog_path,
+                "load_all_images": options.load_all_images,
+                "crashed_only": options.crashed_only,
+            }
         )
     )
     launch_info = lldb.SBLaunchInfo(None)
diff --git a/lldb/examples/python/crashlog_scripted_process.py b/lldb/examples/python/crashlog_scripted_process.py
index c69985b1a072..26c5c37b7371 100644
--- a/lldb/examples/python/crashlog_scripted_process.py
+++ b/lldb/examples/python/crashlog_scripted_process.py
@@ -29,27 +29,7 @@ class CrashLogScriptedProcess(ScriptedProcess):
         if hasattr(self.crashlog, "asb"):
             self.extended_thread_info = self.crashlog.asb
 
-        if self.load_all_images:
-            for image in self.crashlog.images:
-                image.resolve = True
-        else:
-            for thread in self.crashlog.threads:
-                if thread.did_crash():
-                    for ident in thread.idents:
-                        for image in self.crashlog.find_images_with_identifier(ident):
-                            image.resolve = True
-
-        with tempfile.TemporaryDirectory() as obj_dir:
-            for image in self.crashlog.images:
-                if image not in self.loaded_images:
-                    if image.uuid == uuid.UUID(int=0):
-                        continue
-                    err = image.add_module(self.target, obj_dir)
-                    if err:
-                        # Append to SBCommandReturnObject
-                        print(err)
-                    else:
-                        self.loaded_images.append(image)
+        crashlog.load_images(self.options, self.loaded_images)
 
         for thread in self.crashlog.threads:
             if (
@@ -70,6 +50,10 @@ class CrashLogScriptedProcess(ScriptedProcess):
                 self.app_specific_thread, self.addr_mask, self.target
             )
 
+    class CrashLogOptions:
+        load_all_images = False
+        crashed_only = True
+
     def __init__(self, exe_ctx: lldb.SBExecutionContext, args: lldb.SBStructuredData):
         super().__init__(exe_ctx, args)
 
@@ -88,13 +72,17 @@ class CrashLogScriptedProcess(ScriptedProcess):
             # Return error
             return
 
+        self.options = self.CrashLogOptions()
+
         load_all_images = args.GetValueForKey("load_all_images")
         if load_all_images and load_all_images.IsValid():
             if load_all_images.GetType() == lldb.eStructuredDataTypeBoolean:
-                self.load_all_images = load_all_images.GetBooleanValue()
+                self.options.load_all_images = load_all_images.GetBooleanValue()
 
-        if not self.load_all_images:
-            self.load_all_images = False
+        crashed_only = args.GetValueForKey("crashed_only")
+        if crashed_only and crashed_only.IsValid():
+            if crashed_only.GetType() == lldb.eStructuredDataTypeBoolean:
+                self.options.crashed_only = crashed_only.GetBooleanValue()
 
         self.pid = super().get_process_id()
         self.crashed_thread_idx = 0
@@ -159,7 +147,7 @@ class CrashLogScriptedThread(ScriptedThread):
         return frames
 
     def create_stackframes(self):
-        if not (self.originating_process.load_all_images or self.has_crashed):
+        if not (self.originating_process.options.load_all_images or self.has_crashed):
             return None
 
         if not self.backing_thread or not len(self.backing_thread.frames):
-- 
GitLab


From 22c59e01cd2f87164301415c93b60fc3c204dfb8 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 9 May 2024 10:17:15 -0700
Subject: [PATCH 0322/1206] [flang] Don't crash on bad inherited implied DO
 type (#91073)

Fortran has an ambiguously defined rule about the typing of index
variables of implied DO loops in DATA statements and array constructors
that omit an explicit type specification. Such indices have the type
that they would have "if they were variables" in the innermost enclosing
scope. Although this could, and perhaps should, be read to mean that
implicit typing rules active in that innermost enclosing scope should be
applied, every other Fortran compiler interprets that language to mean
that if there is a type declaration for that name that is visible from
the enclosing scope, it is applied, and it is an error if that type is
not integer.

Fixes https://github.com/llvm/llvm-project/issues/91053.
---
 flang/docs/Extensions.md                      | 23 ++++++++++++++-----
 flang/lib/Evaluate/formatting.cpp             |  4 ++--
 flang/lib/Semantics/expression.cpp            |  9 +++++---
 flang/lib/Semantics/resolve-names.cpp         |  5 ++--
 flang/test/Semantics/array-constr-index01.f90 |  8 +++++++
 5 files changed, 35 insertions(+), 14 deletions(-)
 create mode 100644 flang/test/Semantics/array-constr-index01.f90

diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 9030207d9bda..0a5bcdc6ff3f 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -107,12 +107,6 @@ end
   These definitions yield fairly poor results due to floating-point
   cancellation, and every Fortran compiler (including this one)
   uses better algorithms.
-* When an index variable of a `FORALL` or `DO CONCURRENT` is present
-  in the enclosing scope, and the construct does not have an explicit
-  type specification for its index variables, some weird restrictions
-  in F'2023 subclause 19.4 paragraphs 6 & 8 should apply.  Since this
-  compiler properly scopes these names, violations of these restrictions
-  elicit only portability warnings by default.
 * The rules for pairwise distinguishing the specific procedures of a
   generic interface are inadequate, as admitted in note C.11.6 of F'2023.
   Generic interfaces whose specific procedures can be easily proven by
@@ -728,6 +722,23 @@ end
   array and structure constructors not to be finalized, so it also makes sense
   not to finalize their allocatable components when releasing their storage).
 
+* F'2023 19.4 paragraph 5: "If integer-type-spec appears in data-implied-do or
+  ac-implied-do-control it has the specified type and type parameters; otherwise
+  it has the type and type parameters that it would have if it were the name of
+  a variable in the innermost executable construct or scoping unit that includes
+  the DATA statement or array constructor, and this type shall be integer type."
+  Reading "would have if it were" as being the subjunctive, this would mean that
+  an untyped implied DO index variable should be implicitly typed according to
+  the rules active in the enclosing scope.  But all other Fortran compilers interpret
+  the "would have if it were" as meaning "has if it is" -- i.e., if the name
+  is visible in the enclosing scope, the type of that name is used as the
+  type of the implied DO index.  So this is an error, not a simple application
+  of the default implicit typing rule:
+```
+character j
+print *, [(j,j=1,10)]
+```
+
 ## De Facto Standard Features
 
 * `EXTENDS_TYPE_OF()` returns `.TRUE.` if both of its arguments have the
diff --git a/flang/lib/Evaluate/formatting.cpp b/flang/lib/Evaluate/formatting.cpp
index 5f822bbcbb04..20193b006bf2 100644
--- a/flang/lib/Evaluate/formatting.cpp
+++ b/flang/lib/Evaluate/formatting.cpp
@@ -539,10 +539,10 @@ std::string DynamicType::AsFortran() const {
       result += length->AsFortran();
     }
     return result + ')';
-  } else if (IsUnlimitedPolymorphic()) {
-    return "CLASS(*)";
   } else if (IsAssumedType()) {
     return "TYPE(*)";
+  } else if (IsUnlimitedPolymorphic()) {
+    return "CLASS(*)";
   } else if (IsTypelessIntrinsicArgument()) {
     return "(typeless intrinsic function argument)";
   } else {
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 8445be581fdd..ff30b3ab831c 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -1805,10 +1805,13 @@ void ArrayConstructorContext::Add(const parser::AcImpliedDo &impliedDo) {
   const auto &bounds{std::get<parser::AcImpliedDoControl::Bounds>(control.t)};
   exprAnalyzer_.Analyze(bounds.name);
   parser::CharBlock name{bounds.name.thing.thing.source};
-  const Symbol *symbol{bounds.name.thing.thing.symbol};
   int kind{ImpliedDoIntType::kind};
-  if (const auto dynamicType{DynamicType::From(symbol)}) {
-    kind = dynamicType->kind();
+  if (const Symbol * symbol{bounds.name.thing.thing.symbol}) {
+    if (auto dynamicType{DynamicType::From(symbol)}) {
+      if (dynamicType->category() == TypeCategory::Integer) {
+        kind = dynamicType->kind();
+      }
+    }
   }
   std::optional<Expr<ImpliedDoIntType>> lower{
       GetSpecificIntExpr<ImpliedDoIntType::kind>(bounds.lower)};
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 2199e3f16e62..4df347a878de 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -6555,6 +6555,7 @@ Symbol *DeclarationVisitor::DeclareStatementEntity(
       return nullptr;
     }
     name.symbol = nullptr;
+    // F'2023 19.4 p5 ambiguous rule about outer declarations
     declTypeSpec = prev->GetType();
   }
   Symbol &symbol{DeclareEntity<ObjectEntityDetails>(name, {})};
@@ -6573,9 +6574,7 @@ Symbol *DeclarationVisitor::DeclareStatementEntity(
   } else {
     ApplyImplicitRules(symbol);
   }
-  Symbol *result{Resolve(name, &symbol)};
-  AnalyzeExpr(context(), doVar); // enforce INTEGER type
-  return result;
+  return Resolve(name, &symbol);
 }
 
 // Set the type of an entity or report an error.
diff --git a/flang/test/Semantics/array-constr-index01.f90 b/flang/test/Semantics/array-constr-index01.f90
new file mode 100644
index 000000000000..560b6be83139
--- /dev/null
+++ b/flang/test/Semantics/array-constr-index01.f90
@@ -0,0 +1,8 @@
+!RUN: %python %S/test_errors.py %s %flang_fc1
+subroutine s(i)
+  type(*) :: i
+  !ERROR: TYPE(*) dummy argument may only be used as an actual argument
+  !ERROR: Assumed-type entity 'i' must be a dummy argument
+  !ERROR: Must have INTEGER type, but is TYPE(*)
+  print *, [(i, i = 1,1)]
+end
-- 
GitLab


From 8585bf7542f1098bd03a667a408d42d2a815d305 Mon Sep 17 00:00:00 2001
From: Med Ismail Bennani <ismail@bennani.ma>
Date: Thu, 9 May 2024 10:19:34 -0700
Subject: [PATCH 0323/1206] [lldb/crashlog] Update incorrect help message for
 `--no-crashed-only` option (#91162)

This patch rephrases the crashlog `--no-crashed-only` option help
message. This option is mainly used in batch mode to symbolicate and
dump all the threads backtraces, instead of only doing it for the
crashed thread which is the default behavior.

rdar://127391524

Signed-off-by: Med Ismail Bennani <ismail@bennani.ma>
---
 lldb/examples/python/crashlog.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py
index d147d0e322a3..eb9af6ed3d95 100755
--- a/lldb/examples/python/crashlog.py
+++ b/lldb/examples/python/crashlog.py
@@ -1649,7 +1649,8 @@ def CreateSymbolicateCrashLogOptions(
         "--no-crashed-only",
         action="store_false",
         dest="crashed_only",
-        help="do not symbolicate the crashed thread",
+        help="in batch mode, symbolicate all threads, not only the crashed one",
+        default=False,
     )
     arg_parser.add_argument(
         "--disasm-depth",
-- 
GitLab


From b9e3fa84d3fdfe718a4a3085f7adeda3d81f2568 Mon Sep 17 00:00:00 2001
From: Chelsea Cassanova <chelsea_cassanova@apple.com>
Date: Thu, 9 May 2024 10:28:23 -0700
Subject: [PATCH 0324/1206] [lldb][enums] Remove broadcast bits from debugger
 (#91618)

Removes the debugger broadcast bits from `Debugger.h` and instead uses
the enum from `lldb-enumerations.h` and adds the
`eBroadcastSymbolChange` bit to the enum in `lldb-enumerations.h`. This fixes a bug wherein the incorrect broadcast bit could be referenced due both of these enums previously existing and being out-of-sync with each other.
---
 lldb/include/lldb/Core/Debugger.h           | 11 +----------
 lldb/include/lldb/lldb-enumerations.h       |  3 ++-
 lldb/source/Core/Debugger.cpp               | 17 +++++++++--------
 lldb/source/Core/Progress.cpp               |  2 +-
 lldb/unittests/Core/DiagnosticEventTest.cpp | 17 +++++++----------
 lldb/unittests/Core/ProgressReportTest.cpp  |  8 ++++----
 6 files changed, 24 insertions(+), 34 deletions(-)

diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h
index c0f7c732ad2d..ea994bf8c28d 100644
--- a/lldb/include/lldb/Core/Debugger.h
+++ b/lldb/include/lldb/Core/Debugger.h
@@ -78,15 +78,6 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
                  public UserID,
                  public Properties {
 public:
-  /// Broadcaster event bits definitions.
-  enum {
-    eBroadcastBitProgress = (1 << 0),
-    eBroadcastBitWarning = (1 << 1),
-    eBroadcastBitError = (1 << 2),
-    eBroadcastSymbolChange = (1 << 3),
-    eBroadcastBitProgressCategory = (1 << 4),
-  };
-
   using DebuggerList = std::vector<lldb::DebuggerSP>;
 
   static llvm::StringRef GetStaticBroadcasterClass();
@@ -628,7 +619,7 @@ protected:
   ReportProgress(uint64_t progress_id, std::string title, std::string details,
                  uint64_t completed, uint64_t total,
                  std::optional<lldb::user_id_t> debugger_id,
-                 uint32_t progress_category_bit = eBroadcastBitProgress);
+                 uint32_t progress_category_bit = lldb::eBroadcastBitProgress);
 
   static void ReportDiagnosticImpl(lldb::Severity severity, std::string message,
                                    std::optional<lldb::user_id_t> debugger_id,
diff --git a/lldb/include/lldb/lldb-enumerations.h b/lldb/include/lldb/lldb-enumerations.h
index 437971b3364c..8e05f6ba9c87 100644
--- a/lldb/include/lldb/lldb-enumerations.h
+++ b/lldb/include/lldb/lldb-enumerations.h
@@ -1344,7 +1344,8 @@ enum DebuggerBroadcastBit {
   eBroadcastBitProgress = (1 << 0),
   eBroadcastBitWarning = (1 << 1),
   eBroadcastBitError = (1 << 2),
-  eBroadcastBitProgressCategory = (1 << 3),
+  eBroadcastSymbolChange = (1 << 3),
+  eBroadcastBitProgressCategory = (1 << 4),
 };
 
 /// Used for expressing severity in logs and diagnostics.
diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp
index 976420a43443..9951fbcd3e7c 100644
--- a/lldb/source/Core/Debugger.cpp
+++ b/lldb/source/Core/Debugger.cpp
@@ -1485,10 +1485,10 @@ static void PrivateReportDiagnostic(Debugger &debugger, Severity severity,
     assert(false && "eSeverityInfo should not be broadcast");
     return;
   case eSeverityWarning:
-    event_type = Debugger::eBroadcastBitWarning;
+    event_type = lldb::eBroadcastBitWarning;
     break;
   case eSeverityError:
-    event_type = Debugger::eBroadcastBitError;
+    event_type = lldb::eBroadcastBitError;
     break;
   }
 
@@ -1572,7 +1572,7 @@ void Debugger::ReportSymbolChange(const ModuleSpec &module_spec) {
     std::lock_guard<std::recursive_mutex> guard(*g_debugger_list_mutex_ptr);
     for (DebuggerSP debugger_sp : *g_debugger_list_ptr) {
       EventSP event_sp = std::make_shared<Event>(
-          Debugger::eBroadcastSymbolChange,
+          lldb::eBroadcastSymbolChange,
           new SymbolChangeEventData(debugger_sp, module_spec));
       debugger_sp->GetBroadcaster().BroadcastEvent(event_sp);
     }
@@ -1879,8 +1879,9 @@ lldb::thread_result_t Debugger::DefaultEventHandler() {
           CommandInterpreter::eBroadcastBitAsynchronousErrorData);
 
   listener_sp->StartListeningForEvents(
-      &m_broadcaster, eBroadcastBitProgress | eBroadcastBitWarning |
-                          eBroadcastBitError | eBroadcastSymbolChange);
+      &m_broadcaster, lldb::eBroadcastBitProgress | lldb::eBroadcastBitWarning |
+                          lldb::eBroadcastBitError |
+                          lldb::eBroadcastSymbolChange);
 
   // Let the thread that spawned us know that we have started up and that we
   // are now listening to all required events so no events get missed
@@ -1932,11 +1933,11 @@ lldb::thread_result_t Debugger::DefaultEventHandler() {
               }
             }
           } else if (broadcaster == &m_broadcaster) {
-            if (event_type & Debugger::eBroadcastBitProgress)
+            if (event_type & lldb::eBroadcastBitProgress)
               HandleProgressEvent(event_sp);
-            else if (event_type & Debugger::eBroadcastBitWarning)
+            else if (event_type & lldb::eBroadcastBitWarning)
               HandleDiagnosticEvent(event_sp);
-            else if (event_type & Debugger::eBroadcastBitError)
+            else if (event_type & lldb::eBroadcastBitError)
               HandleDiagnosticEvent(event_sp);
           }
         }
diff --git a/lldb/source/Core/Progress.cpp b/lldb/source/Core/Progress.cpp
index 161038284e21..1a779e2ddf92 100644
--- a/lldb/source/Core/Progress.cpp
+++ b/lldb/source/Core/Progress.cpp
@@ -172,7 +172,7 @@ void ProgressManager::ReportProgress(
   Debugger::ReportProgress(progress_data.progress_id, progress_data.title, "",
                            completed, Progress::kNonDeterministicTotal,
                            progress_data.debugger_id,
-                           Debugger::eBroadcastBitProgressCategory);
+                           lldb::eBroadcastBitProgressCategory);
 }
 
 void ProgressManager::Expire(llvm::StringRef key) {
diff --git a/lldb/unittests/Core/DiagnosticEventTest.cpp b/lldb/unittests/Core/DiagnosticEventTest.cpp
index d06f164e87e7..1423f76b8b52 100644
--- a/lldb/unittests/Core/DiagnosticEventTest.cpp
+++ b/lldb/unittests/Core/DiagnosticEventTest.cpp
@@ -55,9 +55,8 @@ TEST_F(DiagnosticEventTest, Warning) {
   ListenerSP listener_sp = Listener::MakeListener("test-listener");
 
   listener_sp->StartListeningForEvents(&broadcaster,
-                                       Debugger::eBroadcastBitWarning);
-  EXPECT_TRUE(
-      broadcaster.EventTypeHasListeners(Debugger::eBroadcastBitWarning));
+                                       lldb::eBroadcastBitWarning);
+  EXPECT_TRUE(broadcaster.EventTypeHasListeners(lldb::eBroadcastBitWarning));
 
   Debugger::ReportWarning("foo", debugger_sp->GetID());
 
@@ -80,9 +79,8 @@ TEST_F(DiagnosticEventTest, Error) {
   Broadcaster &broadcaster = debugger_sp->GetBroadcaster();
   ListenerSP listener_sp = Listener::MakeListener("test-listener");
 
-  listener_sp->StartListeningForEvents(&broadcaster,
-                                       Debugger::eBroadcastBitError);
-  EXPECT_TRUE(broadcaster.EventTypeHasListeners(Debugger::eBroadcastBitError));
+  listener_sp->StartListeningForEvents(&broadcaster, lldb::eBroadcastBitError);
+  EXPECT_TRUE(broadcaster.EventTypeHasListeners(lldb::eBroadcastBitError));
 
   Debugger::ReportError("bar", debugger_sp->GetID());
 
@@ -111,7 +109,7 @@ TEST_F(DiagnosticEventTest, MultipleDebuggers) {
     listeners.push_back(listener);
 
     listener->StartListeningForEvents(&debugger->GetBroadcaster(),
-                                      Debugger::eBroadcastBitError);
+                                      lldb::eBroadcastBitError);
   }
 
   Debugger::ReportError("baz");
@@ -140,9 +138,8 @@ TEST_F(DiagnosticEventTest, WarningOnce) {
   ListenerSP listener_sp = Listener::MakeListener("test-listener");
 
   listener_sp->StartListeningForEvents(&broadcaster,
-                                       Debugger::eBroadcastBitWarning);
-  EXPECT_TRUE(
-      broadcaster.EventTypeHasListeners(Debugger::eBroadcastBitWarning));
+                                       lldb::eBroadcastBitWarning);
+  EXPECT_TRUE(broadcaster.EventTypeHasListeners(lldb::eBroadcastBitWarning));
 
   std::once_flag once;
   Debugger::ReportWarning("foo", debugger_sp->GetID(), &once);
diff --git a/lldb/unittests/Core/ProgressReportTest.cpp b/lldb/unittests/Core/ProgressReportTest.cpp
index f0d253be9bf6..141244feb1f0 100644
--- a/lldb/unittests/Core/ProgressReportTest.cpp
+++ b/lldb/unittests/Core/ProgressReportTest.cpp
@@ -61,7 +61,7 @@ protected:
 };
 
 TEST_F(ProgressReportTest, TestReportCreation) {
-  ListenerSP listener_sp = CreateListenerFor(Debugger::eBroadcastBitProgress);
+  ListenerSP listener_sp = CreateListenerFor(lldb::eBroadcastBitProgress);
   EventSP event_sp;
   const ProgressEventData *data;
 
@@ -135,7 +135,7 @@ TEST_F(ProgressReportTest, TestReportCreation) {
 
 TEST_F(ProgressReportTest, TestProgressManager) {
   ListenerSP listener_sp =
-      CreateListenerFor(Debugger::eBroadcastBitProgressCategory);
+      CreateListenerFor(lldb::eBroadcastBitProgressCategory);
   EventSP event_sp;
   const ProgressEventData *data;
 
@@ -173,7 +173,7 @@ TEST_F(ProgressReportTest, TestProgressManager) {
 
 TEST_F(ProgressReportTest, TestOverlappingEvents) {
   ListenerSP listener_sp =
-      CreateListenerFor(Debugger::eBroadcastBitProgressCategory);
+      CreateListenerFor(lldb::eBroadcastBitProgressCategory);
   EventSP event_sp;
   const ProgressEventData *data;
 
@@ -214,7 +214,7 @@ TEST_F(ProgressReportTest, TestOverlappingEvents) {
 
 TEST_F(ProgressReportTest, TestProgressManagerDisjointReports) {
   ListenerSP listener_sp =
-      CreateListenerFor(Debugger::eBroadcastBitProgressCategory);
+      CreateListenerFor(lldb::eBroadcastBitProgressCategory);
   EventSP event_sp;
   const ProgressEventData *data;
   uint64_t expected_progress_id;
-- 
GitLab


From 7c1b2898302c9f84fa43952f746d79817e1ead40 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 9 May 2024 10:29:43 -0700
Subject: [PATCH 0325/1206] [flang] Accept compiler directives between module
 subprograms (#91230)

Parse and represent compiler directives in a modules
module-subprogram-part between the module subprograms.
---
 flang/include/flang/Parser/parse-tree.h | 3 ++-
 flang/lib/Parser/program-parsers.cpp    | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 4641f9d20d5b..c06354458379 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -2905,7 +2905,8 @@ struct ModuleSubprogram {
   UNION_CLASS_BOILERPLATE(ModuleSubprogram);
   std::variant<common::Indirection<FunctionSubprogram>,
       common::Indirection<SubroutineSubprogram>,
-      common::Indirection<SeparateModuleSubprogram>>
+      common::Indirection<SeparateModuleSubprogram>,
+      common::Indirection<CompilerDirective>>
       u;
 };
 
diff --git a/flang/lib/Parser/program-parsers.cpp b/flang/lib/Parser/program-parsers.cpp
index e24559bf14f7..ff5e58ebc721 100644
--- a/flang/lib/Parser/program-parsers.cpp
+++ b/flang/lib/Parser/program-parsers.cpp
@@ -247,7 +247,8 @@ TYPE_CONTEXT_PARSER("module subprogram part"_en_US,
 //         separate-module-subprogram
 TYPE_PARSER(construct<ModuleSubprogram>(indirect(functionSubprogram)) ||
     construct<ModuleSubprogram>(indirect(subroutineSubprogram)) ||
-    construct<ModuleSubprogram>(indirect(Parser<SeparateModuleSubprogram>{})))
+    construct<ModuleSubprogram>(indirect(Parser<SeparateModuleSubprogram>{})) ||
+    construct<ModuleSubprogram>(indirect(compilerDirective)))
 
 // R1410 module-nature -> INTRINSIC | NON_INTRINSIC
 constexpr auto moduleNature{
-- 
GitLab


From b3a835e129ed8a67cf393f9ee26989b36a3eff1c Mon Sep 17 00:00:00 2001
From: Med Ismail Bennani <ismail@bennani.ma>
Date: Thu, 9 May 2024 10:39:05 -0700
Subject: [PATCH 0326/1206] [lldb] Verify target stop-hooks support with
 scripted process (#91107)

This patch makes sure that scripted process are compatible with target
stop-hooks. This wasn't tested in the past, but it turned out to be
working out of the box.

rdar://124396534

Signed-off-by: Med Ismail Bennani <ismail@bennani.ma>
---
 .../scripted_process/TestScriptedProcess.py   |  7 +++++++
 .../dummy_scripted_process.py                 | 21 +++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/lldb/test/API/functionalities/scripted_process/TestScriptedProcess.py b/lldb/test/API/functionalities/scripted_process/TestScriptedProcess.py
index 5aaf68575623..9519c576689d 100644
--- a/lldb/test/API/functionalities/scripted_process/TestScriptedProcess.py
+++ b/lldb/test/API/functionalities/scripted_process/TestScriptedProcess.py
@@ -187,6 +187,10 @@ class ScriptedProcesTestCase(TestBase):
             + os.path.join(self.getSourceDir(), scripted_process_example_relpath)
         )
 
+        self.runCmd(
+            "target stop-hook add -k first -v 1 -k second -v 2 -P dummy_scripted_process.DummyStopHook"
+        )
+
         launch_info = lldb.SBLaunchInfo(None)
         launch_info.SetProcessPluginName("ScriptedProcess")
         launch_info.SetScriptedProcessClassName(
@@ -207,6 +211,9 @@ class ScriptedProcesTestCase(TestBase):
         self.assertTrue(hasattr(py_impl, "my_super_secret_member"))
         self.assertEqual(py_impl.my_super_secret_method(), 42)
 
+        self.assertTrue(hasattr(py_impl, "handled_stop"))
+        self.assertTrue(py_impl.handled_stop)
+
         # Try reading from target #0 process ...
         addr = 0x500000000
         message = "Hello, target 0"
diff --git a/lldb/test/API/functionalities/scripted_process/dummy_scripted_process.py b/lldb/test/API/functionalities/scripted_process/dummy_scripted_process.py
index 5aff3aa4bb55..cb07bf32c508 100644
--- a/lldb/test/API/functionalities/scripted_process/dummy_scripted_process.py
+++ b/lldb/test/API/functionalities/scripted_process/dummy_scripted_process.py
@@ -7,6 +7,16 @@ from lldb.plugins.scripted_process import ScriptedProcess
 from lldb.plugins.scripted_process import ScriptedThread
 
 
+class DummyStopHook:
+    def __init__(self, target, args, internal_dict):
+        self.target = target
+        self.args = args
+
+    def handle_stop(self, exe_ctx, stream):
+        print("My DummyStopHook triggered. Printing args: \n%s" % self.args)
+        sp = exe_ctx.process.GetScriptedImplementation()
+        sp.handled_stop = True
+
 class DummyScriptedProcess(ScriptedProcess):
     memory = None
 
@@ -18,6 +28,7 @@ class DummyScriptedProcess(ScriptedProcess):
         debugger = self.target.GetDebugger()
         index = debugger.GetIndexOfTarget(self.target)
         self.memory[addr] = "Hello, target " + str(index)
+        self.handled_stop = False
 
     def read_memory_at_address(
         self, addr: int, size: int, error: lldb.SBError
@@ -99,7 +110,13 @@ class DummyScriptedThread(ScriptedThread):
 
 
 def __lldb_init_module(debugger, dict):
+    # This is used when loading the script in an interactive debug session to
+    # automatically, register the stop-hook and launch the scripted process.
     if not "SKIP_SCRIPTED_PROCESS_LAUNCH" in os.environ:
+        debugger.HandleCommand(
+            "target stop-hook add -k first -v 1 -k second -v 2 -P %s.%s"
+            % (__name__, DummyStopHook.__name__)
+        )
         debugger.HandleCommand(
             "process launch -C %s.%s" % (__name__, DummyScriptedProcess.__name__)
         )
@@ -108,3 +125,7 @@ def __lldb_init_module(debugger, dict):
             "Name of the class that will manage the scripted process: '%s.%s'"
             % (__name__, DummyScriptedProcess.__name__)
         )
+        print(
+            "Name of the class that will manage the stop-hook: '%s.%s'"
+            % (__name__, DummyStopHook.__name__)
+        )
-- 
GitLab


From 98c1ba460a697110c64f6d1dd362dcf7088a13ca Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Thu, 9 May 2024 10:41:23 -0700
Subject: [PATCH 0327/1206] [InstrProf]  Add vtables with type metadata into
 symtab  (#81051)

The indirect-call-promotion pass will look up the vtable to find out
the virtual function [1],
and add vtable-derived information in icall
candidate [2] for cost-benefit analysis.

[1] https://github.com/llvm/llvm-project/pull/81442/files#diff-a95d1ac8a0da69713fcb3346135d4b219f0a73920318d2549495620ea215191bR395-R416
[2] https://github.com/llvm/llvm-project/pull/81442/files#diff-a95d1ac8a0da69713fcb3346135d4b219f0a73920318d2549495620ea215191bR195-R199
---
 llvm/include/llvm/ProfileData/InstrProf.h    | 28 +++++++++++-
 llvm/lib/ProfileData/InstrProf.cpp           | 32 +++++++++++++
 llvm/unittests/ProfileData/InstrProfTest.cpp | 47 ++++++++++++++++++++
 3 files changed, 105 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index d5c1ba62911f..88c7fe425b5a 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -17,6 +17,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/IntervalMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
@@ -470,6 +471,12 @@ private:
   // A map from MD5 keys to function define. We only populate this map
   // when build the Symtab from a Module.
   std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
+  // A map from MD5 to the global variable. This map is only populated when
+  // building the symtab from a module. Use separate container instances for
+  // `MD5FuncMap` and `MD5VTableMap`.
+  // TODO: Unify the container type and the lambda function 'mapName' inside
+  // add{Func,VTable}WithName.
+  DenseMap<uint64_t, GlobalVariable *> MD5VTableMap;
   // A map from function runtime address to function name MD5 hash.
   // This map is only populated and used by raw instr profile reader.
   AddrHashMap AddrToMD5Map;
@@ -488,12 +495,18 @@ private:
 
   // Add the function into the symbol table, by creating the following
   // map entries:
-  // name-set = {PGOFuncName} + {getCanonicalName(PGOFuncName)} if the canonical
-  // name is different from pgo name
+  // name-set = {PGOFuncName} union {getCanonicalName(PGOFuncName)}
   // - In MD5NameMap: <MD5Hash(name), name> for name in name-set
   // - In MD5FuncMap: <MD5Hash(name), &F> for name in name-set
   Error addFuncWithName(Function &F, StringRef PGOFuncName);
 
+  // Add the vtable into the symbol table, by creating the following
+  // map entries:
+  // name-set = {PGOName} union {getCanonicalName(PGOName)}
+  // - In MD5NameMap:  <MD5Hash(name), name> for name in name-set
+  // - In MD5VTableMap: <MD5Hash(name), name> for name in name-set
+  Error addVTableWithName(GlobalVariable &V, StringRef PGOVTableName);
+
   // If the symtab is created by a series of calls to \c addFuncName, \c
   // finalizeSymtab needs to be called before looking up function names.
   // This is required because the underlying map is a vector (for space
@@ -555,6 +568,7 @@ public:
   Error create(const FuncNameIterRange &FuncIterRange,
                const VTableNameIterRange &VTableIterRange);
 
+  // Map the MD5 of the symbol name to the name.
   Error addSymbolName(StringRef SymbolName) {
     if (SymbolName.empty())
       return make_error<InstrProfError>(instrprof_error::malformed,
@@ -630,6 +644,10 @@ public:
   /// Return function from the name's md5 hash. Return nullptr if not found.
   inline Function *getFunction(uint64_t FuncMD5Hash);
 
+  /// Return the global variable corresponding to md5 hash. Return nullptr if
+  /// not found.
+  inline GlobalVariable *getGlobalVariable(uint64_t MD5Hash);
+
   /// Return the name section data.
   inline StringRef getNameData() const { return Data; }
 
@@ -709,6 +727,12 @@ Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
   return nullptr;
 }
 
+GlobalVariable *InstrProfSymtab::getGlobalVariable(uint64_t MD5Hash) {
+  if (auto Iter = MD5VTableMap.find(MD5Hash); Iter != MD5VTableMap.end())
+    return Iter->second;
+  return nullptr;
+}
+
 // To store the sums of profile count values, or the percentage of
 // the sums of the total count values.
 struct CountSumOrPercent {
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 1e3ca47b3d5a..806d01de1ada 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -476,11 +476,43 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) {
       return E;
   }
 
+  SmallVector<MDNode *, 2> Types;
+  for (GlobalVariable &G : M.globals()) {
+    if (!G.hasName() || !G.hasMetadata(LLVMContext::MD_type))
+      continue;
+    if (Error E = addVTableWithName(
+            G, getIRPGOObjectName(G, InLTO, /* PGONameMetadata */ nullptr)))
+      return E;
+  }
+
   Sorted = false;
   finalizeSymtab();
   return Error::success();
 }
 
+Error InstrProfSymtab::addVTableWithName(GlobalVariable &VTable,
+                                         StringRef VTablePGOName) {
+  auto mapName = [&](StringRef Name) -> Error {
+    if (Error E = addSymbolName(Name))
+      return E;
+
+    bool Inserted = true;
+    std::tie(std::ignore, Inserted) =
+        MD5VTableMap.try_emplace(GlobalValue::getGUID(Name), &VTable);
+    if (!Inserted)
+      LLVM_DEBUG(dbgs() << "GUID conflict within one module");
+    return Error::success();
+  };
+  if (Error E = mapName(VTablePGOName))
+    return E;
+
+  StringRef CanonicalName = getCanonicalName(VTablePGOName);
+  if (CanonicalName != VTablePGOName)
+    return mapName(CanonicalName);
+
+  return Error::success();
+}
+
 /// \c NameStrings is a string composed of one of more possibly encoded
 /// sub-strings. The substrings are separated by 0 or more zero bytes. This
 /// method decodes the string and calls `NameCallback` for each substring.
diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index 402de64fe99b..8f2c5aee1819 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/LLVMContext.h"
@@ -1730,6 +1732,34 @@ TEST(SymtabTest, instr_prof_symtab_module_test) {
   Function::Create(FTy, Function::WeakODRLinkage, "Wblah", M.get());
   Function::Create(FTy, Function::WeakODRLinkage, "Wbar", M.get());
 
+  // [ptr, ptr, ptr]
+  ArrayType *VTableArrayType = ArrayType::get(
+      PointerType::get(Ctx, M->getDataLayout().getDefaultGlobalsAddressSpace()),
+      3);
+  Constant *Int32TyNull =
+      llvm::ConstantExpr::getNullValue(PointerType::getUnqual(Ctx));
+  SmallVector<llvm::Type *, 1> tys = {VTableArrayType};
+  StructType *VTableType = llvm::StructType::get(Ctx, tys);
+
+  // Create two vtables in the module, one with external linkage and the other
+  // with local linkage.
+  for (auto [Name, Linkage] :
+       {std::pair{"ExternalGV", GlobalValue::ExternalLinkage},
+        {"LocalGV", GlobalValue::InternalLinkage}}) {
+    llvm::Twine FuncName(Name, StringRef("VFunc"));
+    Function *VFunc = Function::Create(FTy, Linkage, FuncName, M.get());
+    GlobalVariable *GV = new llvm::GlobalVariable(
+        *M, VTableType, /* isConstant= */ true, Linkage,
+        llvm::ConstantStruct::get(
+            VTableType,
+            {llvm::ConstantArray::get(VTableArrayType,
+                                      {Int32TyNull, Int32TyNull, VFunc})}),
+        Name);
+    // Add type metadata for the test data, since vtables with type metadata
+    // are added to symtab.
+    GV->addTypeMetadata(16, MDString::get(Ctx, Name));
+  }
+
   InstrProfSymtab ProfSymtab;
   EXPECT_THAT_ERROR(ProfSymtab.create(*M), Succeeded());
 
@@ -1751,6 +1781,23 @@ TEST(SymtabTest, instr_prof_symtab_module_test) {
     EXPECT_EQ(PGOName, PGOFuncName);
     EXPECT_THAT(PGOFuncName.str(), EndsWith(Funcs[I].str()));
   }
+
+  StringRef VTables[] = {"ExternalGV", "LocalGV"};
+  for (auto [VTableName, PGOName] : {std::pair{"ExternalGV", "ExternalGV"},
+                                     {"LocalGV", "MyModule.cpp;LocalGV"}}) {
+    GlobalVariable *GV =
+        M->getGlobalVariable(VTableName, /* AllowInternal=*/true);
+
+    // Test that ProfSymtab returns the expected name given a hash.
+    std::string IRPGOName = getPGOName(*GV);
+    EXPECT_STREQ(IRPGOName.c_str(), PGOName);
+    uint64_t GUID = IndexedInstrProf::ComputeHash(IRPGOName);
+    EXPECT_EQ(IRPGOName, ProfSymtab.getFuncOrVarName(GUID));
+    EXPECT_EQ(VTableName, getParsedIRPGOName(IRPGOName).second);
+
+    // Test that ProfSymtab returns the expected global variable
+    EXPECT_EQ(GV, ProfSymtab.getGlobalVariable(GUID));
+  }
 }
 
 // Testing symtab serialization and creator/deserialization interface
-- 
GitLab


From 7b25ddc559fad078b605c7b3c0d9f4a35a973a52 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 9 May 2024 10:43:25 -0700
Subject: [PATCH 0328/1206] [flang] Don't crash in expression analysis after
 detecting error (#91234)

Avoid calling GetArguments() if a fatal error has been detected.

Fixes https://github.com/llvm/llvm-project/issues/91114.
---
 flang/lib/Semantics/expression.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index ff30b3ab831c..c503ea3f0246 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -4310,7 +4310,9 @@ MaybeExpr ArgumentAnalyzer::TryDefinedOp(
     if (Symbol *symbol{scope.FindSymbol(oprName)}) {
       anyPossibilities = true;
       parser::Name name{symbol->name(), symbol};
-      result = context_.AnalyzeDefinedOp(name, GetActuals());
+      if (!fatalErrors_) {
+        result = context_.AnalyzeDefinedOp(name, GetActuals());
+      }
       if (result) {
         inaccessible = CheckAccessibleSymbol(scope, *symbol);
         if (inaccessible) {
-- 
GitLab


From bce3132cd26c9546a7429da534aed332f4d05d27 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Thu, 9 May 2024 19:44:18 +0200
Subject: [PATCH 0329/1206] [libc++][doc] Updates Spaceship status page.

The completed chrono parts no longer need an implementation.
---
 libcxx/docs/Status/SpaceshipProjects.csv | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcxx/docs/Status/SpaceshipProjects.csv b/libcxx/docs/Status/SpaceshipProjects.csv
index 3d14f487d9a9..128b23b0c2c7 100644
--- a/libcxx/docs/Status/SpaceshipProjects.csv
+++ b/libcxx/docs/Status/SpaceshipProjects.csv
@@ -171,10 +171,10 @@ Section,Description,Dependencies,Assignee,Complete
 | `month_weekday_last <https://reviews.llvm.org/D152699>`_
 | `year_month_weekday <https://reviews.llvm.org/D152699>`_
 | `year_month_weekday_last <https://reviews.llvm.org/D152699>`_",None,Hristo Hristov,|Complete|
-`[time.zone.nonmembers] <https://wg21.link/time.zone.nonmembers>`_,"`chrono::time_zone`",A ``<chrono>`` implementation,Mark de Wever,|Complete|
+`[time.zone.nonmembers] <https://wg21.link/time.zone.nonmembers>`_,"`chrono::time_zone`",,Mark de Wever,|Complete|
 `[time.zone.zonedtime.nonmembers] <https://wg21.link/time.zone.zonedtime.nonmembers>`_,"`chrono::zoned_time`",A ``<chrono>`` implementation,Mark de Wever,|In Progress|
-`[time.zone.leap.nonmembers] <https://wg21.link/time.zone.leap.nonmembers>`_,"`chrono::time_leap_seconds`",A ``<chrono>`` implementation,Mark de Wever,|Complete|
-`[time.zone.link.nonmembers] <https://wg21.link/time.zone.link.nonmembers>`_,"`chrono::time_zone_link`",A ``<chrono>`` implementation,Mark de Wever,|Complete|
+`[time.zone.leap.nonmembers] <https://wg21.link/time.zone.leap.nonmembers>`_,"`chrono::time_leap_seconds`",,Mark de Wever,|Complete|
+`[time.zone.link.nonmembers] <https://wg21.link/time.zone.link.nonmembers>`_,"`chrono::time_zone_link`",,Mark de Wever,|Complete|
 - `5.13 Clause 28: Localization library <https://wg21.link/p1614r2#clause-28-localization-library>`_,,,,
 "| `[locale] <https://wg21.link/locale>`_
 | `[locale.operators] <https://wg21.link/locale.operators>`_",| remove ops `locale <https://reviews.llvm.org/D152654>`_,None,Hristo Hristov,|Complete|
-- 
GitLab


From d742c2aa25226c2b48f3917ed86a5a224cf25734 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 9 May 2024 10:53:29 -0700
Subject: [PATCH 0330/1206] [flang] Move EQUIVALENCE object checking to
 check-declarations.cpp (#91259)

Move EQUIVALENCE object checking from resolve-names-utils.cpp to
check-declarations.cpp, where it can work on fully resolved symbols and
reduce clutter in name resolution. Add a check for EQUIVALENCE objects
that are not ObjectEntityDetails symbols so that attempts to equivalence
a procedure are caught.
---
 flang/lib/Semantics/check-declarations.cpp  | 68 ++++++++++++++++++++-
 flang/lib/Semantics/compute-offsets.cpp     | 28 +++++----
 flang/lib/Semantics/resolve-names-utils.cpp | 68 +--------------------
 flang/test/Semantics/equivalence01.f90      |  9 +++
 4 files changed, 91 insertions(+), 82 deletions(-)

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index c1d9538e557f..f57020bbe707 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -87,6 +87,7 @@ private:
   bool CheckDefinedAssignmentArg(const Symbol &, const DummyArgument &, int);
   void CheckSpecifics(const Symbol &, const GenericDetails &);
   void CheckEquivalenceSet(const EquivalenceSet &);
+  void CheckEquivalenceObject(const EquivalenceObject &);
   void CheckBlockData(const Scope &);
   void CheckGenericOps(const Scope &);
   bool CheckConflicting(const Symbol &, Attr, Attr);
@@ -2558,14 +2559,77 @@ void CheckHelper::CheckEquivalenceSet(const EquivalenceSet &set) {
       }
     }
   }
-  // TODO: Move C8106 (&al.) checks here from resolve-names-utils.cpp
   for (const EquivalenceObject &object : set) {
-    if (object.symbol.test(Symbol::Flag::CrayPointee)) {
+    CheckEquivalenceObject(object);
+  }
+}
+
+static bool InCommonWithBind(const Symbol &symbol) {
+  if (const auto *details{symbol.detailsIf<ObjectEntityDetails>()}) {
+    const Symbol *commonBlock{details->commonBlock()};
+    return commonBlock && commonBlock->attrs().test(Attr::BIND_C);
+  } else {
+    return false;
+  }
+}
+
+void CheckHelper::CheckEquivalenceObject(const EquivalenceObject &object) {
+  parser::MessageFixedText msg;
+  const Symbol &symbol{object.symbol};
+  if (symbol.owner().IsDerivedType()) {
+    msg =
+        "Derived type component '%s' is not allowed in an equivalence set"_err_en_US;
+  } else if (IsDummy(symbol)) {
+    msg = "Dummy argument '%s' is not allowed in an equivalence set"_err_en_US;
+  } else if (symbol.IsFuncResult()) {
+    msg = "Function result '%s' is not allow in an equivalence set"_err_en_US;
+  } else if (IsPointer(symbol)) {
+    msg = "Pointer '%s' is not allowed in an equivalence set"_err_en_US;
+  } else if (IsAllocatable(symbol)) {
+    msg =
+        "Allocatable variable '%s' is not allowed in an equivalence set"_err_en_US;
+  } else if (symbol.Corank() > 0) {
+    msg = "Coarray '%s' is not allowed in an equivalence set"_err_en_US;
+  } else if (symbol.has<UseDetails>()) {
+    msg =
+        "Use-associated variable '%s' is not allowed in an equivalence set"_err_en_US;
+  } else if (symbol.attrs().test(Attr::BIND_C)) {
+    msg =
+        "Variable '%s' with BIND attribute is not allowed in an equivalence set"_err_en_US;
+  } else if (symbol.attrs().test(Attr::TARGET)) {
+    msg =
+        "Variable '%s' with TARGET attribute is not allowed in an equivalence set"_err_en_US;
+  } else if (IsNamedConstant(symbol)) {
+    msg = "Named constant '%s' is not allowed in an equivalence set"_err_en_US;
+  } else if (InCommonWithBind(symbol)) {
+    msg =
+        "Variable '%s' in common block with BIND attribute is not allowed in an equivalence set"_err_en_US;
+  } else if (!symbol.has<ObjectEntityDetails>()) {
+    msg = "'%s' in equivalence set is not a data object"_err_en_US;
+  } else if (const auto *type{symbol.GetType()}) {
+    const auto *derived{type->AsDerived()};
+    if (derived && !derived->IsVectorType()) {
+      if (const auto *comp{
+              FindUltimateComponent(*derived, IsAllocatableOrPointer)}) {
+        msg = IsPointer(*comp)
+            ? "Derived type object '%s' with pointer ultimate component is not allowed in an equivalence set"_err_en_US
+            : "Derived type object '%s' with allocatable ultimate component is not allowed in an equivalence set"_err_en_US;
+      } else if (!derived->typeSymbol().get<DerivedTypeDetails>().sequence()) {
+        msg =
+            "Nonsequence derived type object '%s' is not allowed in an equivalence set"_err_en_US;
+      }
+    } else if (IsAutomatic(symbol)) {
+      msg =
+          "Automatic object '%s' is not allowed in an equivalence set"_err_en_US;
+    } else if (symbol.test(Symbol::Flag::CrayPointee)) {
       messages_.Say(object.symbol.name(),
           "Cray pointee '%s' may not be a member of an EQUIVALENCE group"_err_en_US,
           object.symbol.name());
     }
   }
+  if (!msg.text().empty()) {
+    context_.Say(object.source, std::move(msg), symbol.name());
+  }
 }
 
 void CheckHelper::CheckBlockData(const Scope &scope) {
diff --git a/flang/lib/Semantics/compute-offsets.cpp b/flang/lib/Semantics/compute-offsets.cpp
index 2eb3a34ad806..d9a9576e9d67 100644
--- a/flang/lib/Semantics/compute-offsets.cpp
+++ b/flang/lib/Semantics/compute-offsets.cpp
@@ -277,20 +277,22 @@ std::size_t ComputeOffsetsHelper::ComputeOffset(
     const EquivalenceObject &object) {
   std::size_t offset{0};
   if (!object.subscripts.empty()) {
-    const ArraySpec &shape{object.symbol.get<ObjectEntityDetails>().shape()};
-    auto lbound{[&](std::size_t i) {
-      return *ToInt64(shape[i].lbound().GetExplicit());
-    }};
-    auto ubound{[&](std::size_t i) {
-      return *ToInt64(shape[i].ubound().GetExplicit());
-    }};
-    for (std::size_t i{object.subscripts.size() - 1};;) {
-      offset += object.subscripts[i] - lbound(i);
-      if (i == 0) {
-        break;
+    if (const auto *details{object.symbol.detailsIf<ObjectEntityDetails>()}) {
+      const ArraySpec &shape{details->shape()};
+      auto lbound{[&](std::size_t i) {
+        return *ToInt64(shape[i].lbound().GetExplicit());
+      }};
+      auto ubound{[&](std::size_t i) {
+        return *ToInt64(shape[i].ubound().GetExplicit());
+      }};
+      for (std::size_t i{object.subscripts.size() - 1};;) {
+        offset += object.subscripts[i] - lbound(i);
+        if (i == 0) {
+          break;
+        }
+        --i;
+        offset *= ubound(i) - lbound(i) + 1;
       }
-      --i;
-      offset *= ubound(i) - lbound(i) + 1;
     }
   }
   auto result{offset * GetSizeAndAlignment(object.symbol, false).size};
diff --git a/flang/lib/Semantics/resolve-names-utils.cpp b/flang/lib/Semantics/resolve-names-utils.cpp
index 801473876e7e..3ca460b8e46a 100644
--- a/flang/lib/Semantics/resolve-names-utils.cpp
+++ b/flang/lib/Semantics/resolve-names-utils.cpp
@@ -568,75 +568,9 @@ bool EquivalenceSets::CheckDataRef(
       x.u);
 }
 
-static bool InCommonWithBind(const Symbol &symbol) {
-  if (const auto *details{symbol.detailsIf<ObjectEntityDetails>()}) {
-    const Symbol *commonBlock{details->commonBlock()};
-    return commonBlock && commonBlock->attrs().test(Attr::BIND_C);
-  } else {
-    return false;
-  }
-}
-
-// If symbol can't be in equivalence set report error and return false;
 bool EquivalenceSets::CheckObject(const parser::Name &name) {
-  if (!name.symbol) {
-    return false; // an error has already occurred
-  }
   currObject_.symbol = name.symbol;
-  parser::MessageFixedText msg;
-  const Symbol &symbol{*name.symbol};
-  if (symbol.owner().IsDerivedType()) { // C8107
-    msg = "Derived type component '%s'"
-          " is not allowed in an equivalence set"_err_en_US;
-  } else if (IsDummy(symbol)) { // C8106
-    msg = "Dummy argument '%s' is not allowed in an equivalence set"_err_en_US;
-  } else if (symbol.IsFuncResult()) { // C8106
-    msg = "Function result '%s' is not allow in an equivalence set"_err_en_US;
-  } else if (IsPointer(symbol)) { // C8106
-    msg = "Pointer '%s' is not allowed in an equivalence set"_err_en_US;
-  } else if (IsAllocatable(symbol)) { // C8106
-    msg = "Allocatable variable '%s'"
-          " is not allowed in an equivalence set"_err_en_US;
-  } else if (symbol.Corank() > 0) { // C8106
-    msg = "Coarray '%s' is not allowed in an equivalence set"_err_en_US;
-  } else if (symbol.has<UseDetails>()) { // C8115
-    msg = "Use-associated variable '%s'"
-          " is not allowed in an equivalence set"_err_en_US;
-  } else if (symbol.attrs().test(Attr::BIND_C)) { // C8106
-    msg = "Variable '%s' with BIND attribute"
-          " is not allowed in an equivalence set"_err_en_US;
-  } else if (symbol.attrs().test(Attr::TARGET)) { // C8108
-    msg = "Variable '%s' with TARGET attribute"
-          " is not allowed in an equivalence set"_err_en_US;
-  } else if (IsNamedConstant(symbol)) { // C8106
-    msg = "Named constant '%s' is not allowed in an equivalence set"_err_en_US;
-  } else if (InCommonWithBind(symbol)) { // C8106
-    msg = "Variable '%s' in common block with BIND attribute"
-          " is not allowed in an equivalence set"_err_en_US;
-  } else if (const auto *type{symbol.GetType()}) {
-    const auto *derived{type->AsDerived()};
-    if (derived && !derived->IsVectorType()) {
-      if (const auto *comp{FindUltimateComponent(
-              *derived, IsAllocatableOrPointer)}) { // C8106
-        msg = IsPointer(*comp)
-            ? "Derived type object '%s' with pointer ultimate component"
-              " is not allowed in an equivalence set"_err_en_US
-            : "Derived type object '%s' with allocatable ultimate component"
-              " is not allowed in an equivalence set"_err_en_US;
-      } else if (!derived->typeSymbol().get<DerivedTypeDetails>().sequence()) {
-        msg = "Nonsequence derived type object '%s'"
-              " is not allowed in an equivalence set"_err_en_US;
-      }
-    } else if (IsAutomatic(symbol)) {
-      msg = "Automatic object '%s'"
-            " is not allowed in an equivalence set"_err_en_US;
-    }
-  }
-  if (!msg.text().empty()) {
-    context_.Say(name.source, std::move(msg), name.source);
-    return false;
-  }
-  return true;
+  return currObject_.symbol != nullptr;
 }
 
 bool EquivalenceSets::CheckArrayBound(const parser::Expr &bound) {
diff --git a/flang/test/Semantics/equivalence01.f90 b/flang/test/Semantics/equivalence01.f90
index 7ef47fb554b5..ec68e9066a29 100644
--- a/flang/test/Semantics/equivalence01.f90
+++ b/flang/test/Semantics/equivalence01.f90
@@ -244,3 +244,12 @@ module m18
   type(t1) x
   common x
 end
+
+subroutine s19
+  entry e19
+  !ERROR: 'e19' in equivalence set is not a data object
+  equivalence (e19, j)
+  !ERROR: 'e20' in equivalence set is not a data object
+  equivalence (e20, j)
+  entry e20
+end
-- 
GitLab


From 5ad418b55c167fbdce31b92467e90eb3a03d85ce Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Thu, 9 May 2024 10:57:52 -0700
Subject: [PATCH 0331/1206] Remove stale TODO comment

---
 clang/lib/Parse/ParseOpenACC.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 90dfea6f5f5c..0e10632c8317 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -956,7 +956,6 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
       break;
     case OpenACCClauseKind::Attach:
     case OpenACCClauseKind::DevicePtr:
-      // TODO: ERICH: Figure out how to limit to just ptrs?
       ParsedClause.setVarListDetails(ParseOpenACCVarList(),
                                      /*IsReadOnly=*/false, /*IsZero=*/false);
       break;
-- 
GitLab


From 90501be35b2c4ad314a45634062e0dfe878d8621 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 9 May 2024 11:04:50 -0700
Subject: [PATCH 0332/1206] [flang] Accept interoperable types without BIND(C)
 (#91363)

A derived type that meets (most of) the requirements of an interoperable
type but doesn't actually have the BIND(C) attribute can be accepted as
an interoperable type, with optional warnings.
---
 flang/docs/Extensions.md                      |   7 +
 flang/include/flang/Common/Fortran-features.h |   3 +-
 flang/lib/Semantics/check-declarations.cpp    | 265 ++++++++++++------
 flang/test/Semantics/bind-c03.f90             |   7 +-
 flang/test/Semantics/bind-c06.f90             |  28 +-
 flang/test/Semantics/bindings01.f90           |   4 +-
 flang/test/Semantics/resolve81.f90            |   8 +-
 flang/test/Semantics/resolve85.f90            |   2 +-
 8 files changed, 214 insertions(+), 110 deletions(-)

diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 0a5bcdc6ff3f..43ed35e36a6e 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -114,6 +114,10 @@ end
   appear in real applications, but are still non-conforming under the
   incomplete tests in F'2023 15.4.3.4.5.
   These cases are compiled with optional portability warnings.
+* `PROCEDURE(), BIND(C) :: PROC` is not conforming, as there is no
+  procedure interface.  This compiler accepts it, since there is otherwise
+  no way to declare an interoperable dummy procedure with an arbitrary
+  interface like `void (*)()`.
 
 ## Extensions, deletions, and legacy features supported by default
 
@@ -345,6 +349,9 @@ end
   when necessary to the type of the result.
   An `OPTIONAL`, `POINTER`, or `ALLOCATABLE` argument after
   the first two cannot be converted, as it may not be present.
+* A derived type that meets (most of) the requirements of an interoperable
+  derived type can be used as such where an interoperable type is
+  required, with warnings, even if it lacks the BIND(C) attribute.
 
 ### Extensions supported when enabled by options
 
diff --git a/flang/include/flang/Common/Fortran-features.h b/flang/include/flang/Common/Fortran-features.h
index 6b3e37cd9c25..07ed7f43c1e7 100644
--- a/flang/include/flang/Common/Fortran-features.h
+++ b/flang/include/flang/Common/Fortran-features.h
@@ -48,7 +48,8 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     ImpliedDoIndexScope, DistinctCommonSizes, OddIndexVariableRestrictions,
     IndistinguishableSpecifics, SubroutineAndFunctionSpecifics,
     EmptySequenceType, NonSequenceCrayPointee, BranchIntoConstruct,
-    BadBranchTarget, ConvertedArgument, HollerithPolymorphic, ListDirectedSize)
+    BadBranchTarget, ConvertedArgument, HollerithPolymorphic, ListDirectedSize,
+    NonBindCInteroperability)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index f57020bbe707..9ef34f0f81c0 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -116,11 +116,16 @@ private:
     }
     return msg;
   }
+  bool InModuleFile() const {
+    return FindModuleFileContaining(context_.FindScope(messages_.at())) !=
+        nullptr;
+  }
   template <typename... A> parser::Message *WarnIfNotInModuleFile(A &&...x) {
-    if (FindModuleFileContaining(context_.FindScope(messages_.at()))) {
+    if (InModuleFile()) {
       return nullptr;
+    } else {
+      return messages_.Say(std::forward<A>(x)...);
     }
-    return messages_.Say(std::forward<A>(x)...);
   }
   template <typename... A>
   parser::Message *WarnIfNotInModuleFile(parser::CharBlock source, A &&...x) {
@@ -133,6 +138,7 @@ private:
   void CheckGlobalName(const Symbol &);
   void CheckProcedureAssemblyName(const Symbol &symbol);
   void CheckExplicitSave(const Symbol &);
+  parser::Messages WhyNotInteroperableDerivedType(const Symbol &, bool isError);
   void CheckBindC(const Symbol &);
   void CheckBindCFunctionResult(const Symbol &);
   // Check functions for defined I/O procedures
@@ -183,6 +189,8 @@ private:
   // Collection of target dependent assembly names of external and BIND(C)
   // procedures.
   std::map<std::string, SymbolRef> procedureAssemblyNames_;
+  // Derived types that have been examined by WhyNotInteroperableDerivedType
+  UnorderedSymbolSet examinedByWhyNotInteroperableDerivedType_;
 };
 
 class DistinguishabilityHelper {
@@ -2822,11 +2830,129 @@ void CheckHelper::CheckProcedureAssemblyName(const Symbol &symbol) {
   }
 }
 
+parser::Messages CheckHelper::WhyNotInteroperableDerivedType(
+    const Symbol &symbol, bool isError) {
+  parser::Messages msgs;
+  if (examinedByWhyNotInteroperableDerivedType_.find(symbol) !=
+      examinedByWhyNotInteroperableDerivedType_.end()) {
+    return msgs;
+  }
+  isError |= symbol.attrs().test(Attr::BIND_C);
+  examinedByWhyNotInteroperableDerivedType_.insert(symbol);
+  if (const auto *derived{symbol.detailsIf<DerivedTypeDetails>()}) {
+    if (derived->sequence()) { // C1801
+      msgs.Say(symbol.name(),
+          "An interoperable derived type cannot have the SEQUENCE attribute"_err_en_US);
+    } else if (!derived->paramDecls().empty()) { // C1802
+      msgs.Say(symbol.name(),
+          "An interoperable derived type cannot have a type parameter"_err_en_US);
+    } else if (const auto *parent{
+                   symbol.scope()->GetDerivedTypeParent()}) { // C1803
+      if (isError) {
+        msgs.Say(symbol.name(),
+            "A derived type with the BIND attribute cannot be an extended derived type"_err_en_US);
+      } else {
+        bool interoperableParent{true};
+        if (parent->symbol()) {
+          auto bad{WhyNotInteroperableDerivedType(*parent->symbol(), false)};
+          if (bad.AnyFatalError()) {
+            auto &msg{msgs.Say(symbol.name(),
+                "The parent of an interoperable type is not interoperable"_err_en_US)};
+            bad.AttachTo(msg, parser::Severity::None);
+            interoperableParent = false;
+          }
+        }
+        if (interoperableParent) {
+          msgs.Say(symbol.name(),
+              "An interoperable type should not be an extended derived type"_warn_en_US);
+        }
+      }
+    }
+    const Symbol *parentComponent{symbol.scope()
+            ? derived->GetParentComponent(*symbol.scope())
+            : nullptr};
+    for (const auto &pair : *symbol.scope()) {
+      const Symbol &component{*pair.second};
+      if (&component == parentComponent) {
+        continue; // was checked above
+      }
+      if (IsProcedure(component)) { // C1804
+        msgs.Say(component.name(),
+            "An interoperable derived type cannot have a type bound procedure"_err_en_US);
+      } else if (IsAllocatableOrPointer(component)) { // C1806
+        msgs.Say(component.name(),
+            "An interoperable derived type cannot have a pointer or allocatable component"_err_en_US);
+      } else if (const auto *type{component.GetType()}) {
+        if (const auto *derived{type->AsDerived()}) {
+          auto bad{
+              WhyNotInteroperableDerivedType(derived->typeSymbol(), isError)};
+          if (bad.AnyFatalError()) {
+            auto &msg{msgs.Say(component.name(),
+                "Component '%s' of an interoperable derived type must have an interoperable type but does not"_err_en_US,
+                component.name())};
+            bad.AttachTo(msg, parser::Severity::None);
+          } else if (!derived->typeSymbol().GetUltimate().attrs().test(
+                         Attr::BIND_C)) {
+            auto &msg{
+                msgs.Say(component.name(),
+                        "Derived type of component '%s' of an interoperable derived type should have the BIND attribute"_warn_en_US,
+                        component.name())
+                    .Attach(derived->typeSymbol().name(),
+                        "Non-BIND(C) component type"_en_US)};
+            bad.AttachTo(msg, parser::Severity::None);
+          } else {
+            msgs.Annex(std::move(bad));
+          }
+        } else if (!IsInteroperableIntrinsicType(
+                       *type, context_.languageFeatures())) {
+          auto maybeDyType{evaluate::DynamicType::From(*type)};
+          if (type->category() == DeclTypeSpec::Logical) {
+            if (context_.ShouldWarn(common::UsageWarning::LogicalVsCBool)) {
+              msgs.Say(component.name(),
+                  "A LOGICAL component of an interoperable type should have the interoperable KIND=C_BOOL"_port_en_US);
+            }
+          } else if (type->category() == DeclTypeSpec::Character &&
+              maybeDyType && maybeDyType->kind() == 1) {
+            if (context_.ShouldWarn(common::UsageWarning::BindCCharLength)) {
+              msgs.Say(component.name(),
+                  "A CHARACTER component of an interoperable type should have length 1"_port_en_US);
+            }
+          } else {
+            msgs.Say(component.name(),
+                "Each component of an interoperable derived type must have an interoperable type"_err_en_US);
+          }
+        }
+      }
+      if (auto extents{
+              evaluate::GetConstantExtents(foldingContext_, &component)};
+          extents && evaluate::GetSize(*extents) == 0) {
+        msgs.Say(component.name(),
+            "An array component of an interoperable type must have at least one element"_err_en_US);
+      }
+    }
+    if (derived->componentNames().empty()) { // F'2023 C1805
+      if (context_.ShouldWarn(common::LanguageFeature::EmptyBindCDerivedType)) {
+        msgs.Say(symbol.name(),
+            "A derived type with the BIND attribute should not be empty"_port_en_US);
+      }
+    }
+  }
+  if (isError) {
+    for (auto &m : msgs.messages()) {
+      if (!m.IsFatal()) {
+        m.set_severity(parser::Severity::Error);
+      }
+    }
+  }
+  return msgs;
+}
+
 void CheckHelper::CheckBindC(const Symbol &symbol) {
   bool isExplicitBindC{symbol.attrs().test(Attr::BIND_C)};
   if (isExplicitBindC) {
-    CheckConflicting(symbol, Attr::BIND_C, Attr::PARAMETER);
     CheckConflicting(symbol, Attr::BIND_C, Attr::ELEMENTAL);
+    CheckConflicting(symbol, Attr::BIND_C, Attr::INTRINSIC);
+    CheckConflicting(symbol, Attr::BIND_C, Attr::PARAMETER);
   } else {
     // symbol must be interoperable (e.g., dummy argument of interoperable
     // procedure interface) but is not itself BIND(C).
@@ -2896,13 +3022,30 @@ void CheckHelper::CheckBindC(const Symbol &symbol) {
     }
     if (const auto *type{symbol.GetType()}) {
       const auto *derived{type->AsDerived()};
-      if (derived && !derived->typeSymbol().attrs().test(Attr::BIND_C)) {
-        if (auto *msg{messages_.Say(symbol.name(),
-                "The derived type of a BIND(C) object must also be BIND(C)"_err_en_US)}) {
-          msg->Attach(
-              derived->typeSymbol().name(), "Non-interoperable type"_en_US);
+      if (derived) {
+        if (derived->typeSymbol().attrs().test(Attr::BIND_C)) {
+        } else if (isExplicitBindC) {
+          if (auto *msg{messages_.Say(symbol.name(),
+                  "The derived type of a BIND(C) object must also be BIND(C)"_err_en_US)}) {
+            msg->Attach(derived->typeSymbol().name(), "Non-BIND(C) type"_en_US);
+          }
+          context_.SetError(symbol);
+        } else if (auto bad{WhyNotInteroperableDerivedType(
+                       derived->typeSymbol(), false)};
+                   !bad.empty()) {
+          if (auto *msg{messages_.Say(symbol.name(),
+                  "The derived type of an interoperable object must be interoperable, but is not"_err_en_US)}) {
+            msg->Attach(
+                derived->typeSymbol().name(), "Non-interoperable type"_en_US);
+            bad.AttachTo(*msg, parser::Severity::None);
+          }
+          context_.SetError(symbol);
+        } else {
+          if (auto *msg{messages_.Say(symbol.name(),
+                  "The derived type of an interoperable object should be BIND(C)"_warn_en_US)}) {
+            msg->Attach(derived->typeSymbol().name(), "Non-BIND(C) type"_en_US);
+          }
         }
-        context_.SetError(symbol);
       }
       if (type->IsAssumedType() || IsAssumedLengthCharacter(symbol)) {
         // ok
@@ -2945,17 +3088,20 @@ void CheckHelper::CheckBindC(const Symbol &symbol) {
           "An interoperable pointer must not be CONTIGUOUS"_err_en_US);
     }
   } else if (const auto *proc{symbol.detailsIf<ProcEntityDetails>()}) {
-    if (!proc->procInterface() ||
-        !proc->procInterface()->attrs().test(Attr::BIND_C)) {
-      if (proc->isDummy()) {
-        messages_.Say(symbol.name(),
-            "A dummy procedure to an interoperable procedure must also be interoperable"_err_en_US);
-        context_.SetError(symbol);
-      } else {
-        messages_.Say(symbol.name(),
-            "An interface name with BIND attribute must be specified if the BIND attribute is specified in a procedure declaration statement"_err_en_US);
-        context_.SetError(symbol);
+    if (!IsBindCProcedure(symbol) && proc->isDummy()) {
+      messages_.Say(symbol.name(),
+          "A dummy procedure to an interoperable procedure must also be interoperable"_err_en_US);
+      context_.SetError(symbol);
+    } else if (!proc->procInterface()) {
+      if (context_.ShouldWarn(
+              common::LanguageFeature::NonBindCInteroperability)) {
+        WarnIfNotInModuleFile(symbol.name(),
+            "An interface name with BIND attribute should be specified if the BIND attribute is specified in a procedure declaration statement"_warn_en_US);
       }
+    } else if (!proc->procInterface()->attrs().test(Attr::BIND_C)) {
+      messages_.Say(symbol.name(),
+          "An interface name with BIND attribute must be specified if the BIND attribute is specified in a procedure declaration statement"_err_en_US);
+      context_.SetError(symbol);
     }
   } else if (const auto *subp{symbol.detailsIf<SubprogramDetails>()}) {
     for (const Symbol *dummy : subp->dummyArgs()) {
@@ -2967,77 +3113,18 @@ void CheckHelper::CheckBindC(const Symbol &symbol) {
         context_.SetError(symbol);
       }
     }
-  } else if (const auto *derived{symbol.detailsIf<DerivedTypeDetails>()}) {
-    if (derived->sequence()) { // C1801
-      messages_.Say(symbol.name(),
-          "A derived type with the BIND attribute cannot have the SEQUENCE attribute"_err_en_US);
-      context_.SetError(symbol);
-    } else if (!derived->paramDecls().empty()) { // C1802
-      messages_.Say(symbol.name(),
-          "A derived type with the BIND attribute has type parameter(s)"_err_en_US);
-      context_.SetError(symbol);
-    } else if (symbol.scope()->GetDerivedTypeParent()) { // C1803
-      messages_.Say(symbol.name(),
-          "A derived type with the BIND attribute cannot extend from another derived type"_err_en_US);
-      context_.SetError(symbol);
-    } else {
-      for (const auto &pair : *symbol.scope()) {
-        const Symbol *component{&*pair.second};
-        if (IsProcedure(*component)) { // C1804
-          messages_.Say(component->name(),
-              "A derived type with the BIND attribute cannot have a type bound procedure"_err_en_US);
-          context_.SetError(symbol);
-        }
-        if (IsAllocatableOrPointer(*component)) { // C1806
-          messages_.Say(component->name(),
-              "A derived type with the BIND attribute cannot have a pointer or allocatable component"_err_en_US);
-          context_.SetError(symbol);
-        }
-        if (const auto *type{component->GetType()}) {
-          if (const auto *derived{type->AsDerived()}) {
-            if (!derived->typeSymbol().attrs().test(Attr::BIND_C)) {
-              if (auto *msg{messages_.Say(component->name(),
-                      "Component '%s' of an interoperable derived type must have the BIND attribute"_err_en_US,
-                      component->name())}) {
-                msg->Attach(derived->typeSymbol().name(),
-                    "Non-interoperable component type"_en_US);
-              }
-              context_.SetError(symbol);
-            }
-          } else if (!IsInteroperableIntrinsicType(
-                         *type, context_.languageFeatures())) {
-            auto maybeDyType{evaluate::DynamicType::From(*type)};
-            if (type->category() == DeclTypeSpec::Logical) {
-              if (context_.ShouldWarn(common::UsageWarning::LogicalVsCBool)) {
-                WarnIfNotInModuleFile(component->name(),
-                    "A LOGICAL component of a BIND(C) type should have the interoperable KIND=C_BOOL"_port_en_US);
-              }
-            } else if (type->category() == DeclTypeSpec::Character &&
-                maybeDyType && maybeDyType->kind() == 1) {
-              if (context_.ShouldWarn(common::UsageWarning::BindCCharLength)) {
-                WarnIfNotInModuleFile(component->name(),
-                    "A CHARACTER component of a BIND(C) type should have length 1"_port_en_US);
-              }
-            } else {
-              messages_.Say(component->name(),
-                  "Each component of an interoperable derived type must have an interoperable type"_err_en_US);
-              context_.SetError(symbol);
-            }
-          }
-        }
-        if (auto extents{
-                evaluate::GetConstantExtents(foldingContext_, component)};
-            extents && evaluate::GetSize(*extents) == 0) {
-          messages_.Say(component->name(),
-              "An array component of an interoperable type must have at least one element"_err_en_US);
-          context_.SetError(symbol);
-        }
+  } else if (symbol.has<DerivedTypeDetails>()) {
+    if (auto msgs{WhyNotInteroperableDerivedType(symbol, false)};
+        !msgs.empty()) {
+      bool anyFatal{msgs.AnyFatalError()};
+      if (msgs.AnyFatalError() ||
+          (!InModuleFile() &&
+              context_.ShouldWarn(
+                  common::LanguageFeature::NonBindCInteroperability))) {
+        context_.messages().Annex(std::move(msgs));
       }
-    }
-    if (derived->componentNames().empty()) { // F'2023 C1805
-      if (context_.ShouldWarn(common::LanguageFeature::EmptyBindCDerivedType)) {
-        WarnIfNotInModuleFile(symbol.name(),
-            "A derived type with the BIND attribute is empty"_port_en_US);
+      if (anyFatal) {
+        context_.SetError(symbol);
       }
     }
   }
diff --git a/flang/test/Semantics/bind-c03.f90 b/flang/test/Semantics/bind-c03.f90
index 65d52e964ca4..c37cb2bccb1f 100644
--- a/flang/test/Semantics/bind-c03.f90
+++ b/flang/test/Semantics/bind-c03.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/test_errors.py %s %flang_fc1
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
 ! Check for C1521
 ! If proc-language-binding-spec (bind(c)) is specified, the proc-interface
 ! shall appear, it shall be an interface-name, and interface-name shall be
@@ -24,7 +24,10 @@ module m
   !ERROR: An interface name with BIND attribute must be specified if the BIND attribute is specified in a procedure declaration statement
   procedure(proc2), bind(c) :: pc2
 
-  !ERROR: An interface name with BIND attribute must be specified if the BIND attribute is specified in a procedure declaration statement
+  !WARNING: An interface name with BIND attribute should be specified if the BIND attribute is specified in a procedure declaration statement
   procedure(integer), bind(c) :: pc3
 
+  !WARNING: An interface name with BIND attribute should be specified if the BIND attribute is specified in a procedure declaration statement
+  procedure(), bind(c) :: pc5
+
 end
diff --git a/flang/test/Semantics/bind-c06.f90 b/flang/test/Semantics/bind-c06.f90
index 4c25722cb775..3ad3078c4b4a 100644
--- a/flang/test/Semantics/bind-c06.f90
+++ b/flang/test/Semantics/bind-c06.f90
@@ -16,19 +16,19 @@ program main
     integer :: i
   end type
 
-  ! ERROR: A derived type with the BIND attribute cannot have the SEQUENCE attribute
+  ! ERROR: An interoperable derived type cannot have the SEQUENCE attribute
   type, bind(c) :: t1
     sequence
     integer :: x
   end type
 
-  ! ERROR: A derived type with the BIND attribute has type parameter(s)
+  ! ERROR: An interoperable derived type cannot have a type parameter
   type, bind(c) :: t2(k)
     integer, KIND :: k
     integer :: x
   end type
 
-  ! ERROR: A derived type with the BIND attribute cannot extend from another derived type
+  ! ERROR: A derived type with the BIND attribute cannot be an extended derived type
   type, bind(c), extends(v) :: t3
     integer :: x
   end type
@@ -36,21 +36,21 @@ program main
   type, bind(c) :: t4
     integer :: x
    contains
-    ! ERROR: A derived type with the BIND attribute cannot have a type bound procedure
+    ! ERROR: An interoperable derived type cannot have a type bound procedure
     procedure, nopass :: b => s
   end type
 
-  ! WARNING: A derived type with the BIND attribute is empty
+  ! WARNING: A derived type with the BIND attribute should not be empty
   type, bind(c) :: t5
   end type
 
   type, bind(c) :: t6
-    ! ERROR: A derived type with the BIND attribute cannot have a pointer or allocatable component
+    ! ERROR: An interoperable derived type cannot have a pointer or allocatable component
     integer, pointer :: x
   end type
 
   type, bind(c) :: t7
-    ! ERROR: A derived type with the BIND attribute cannot have a pointer or allocatable component
+    ! ERROR: An interoperable derived type cannot have a pointer or allocatable component
     integer, allocatable :: y
   end type
 
@@ -58,14 +58,20 @@ program main
     integer :: x
   end type
 
+  type :: t8a
+    integer, pointer :: x
+  end type
+
   type, bind(c) :: t9
-    !ERROR: Component 'y' of an interoperable derived type must have the BIND attribute
-    type(t8) :: y
+    !WARNING: Derived type of component 'x' of an interoperable derived type should have the BIND attribute
+    type(t8) :: x
+    !ERROR: Component 'y' of an interoperable derived type must have an interoperable type but does not
+    type(t8a) :: y
     integer :: z
   end type
 
   type, bind(c) :: t10
-    !WARNING: A CHARACTER component of a BIND(C) type should have length 1
+    !WARNING: A CHARACTER component of an interoperable type should have length 1
     character(len=2) x
   end type
   type, bind(c) :: t11
@@ -73,7 +79,7 @@ program main
     character(kind=2) x
   end type
   type, bind(c) :: t12
-    !PORTABILITY: A LOGICAL component of a BIND(C) type should have the interoperable KIND=C_BOOL
+    !PORTABILITY: A LOGICAL component of an interoperable type should have the interoperable KIND=C_BOOL
     logical(kind=8) x
   end type
   type, bind(c) :: t13
diff --git a/flang/test/Semantics/bindings01.f90 b/flang/test/Semantics/bindings01.f90
index 7f119d4e55bf..7c2dc6448bb3 100644
--- a/flang/test/Semantics/bindings01.f90
+++ b/flang/test/Semantics/bindings01.f90
@@ -4,7 +4,7 @@
 
 module m
   !ERROR: An ABSTRACT derived type must be extensible
-  !PORTABILITY: A derived type with the BIND attribute is empty
+  !PORTABILITY: A derived type with the BIND attribute should not be empty
   type, abstract, bind(c) :: badAbstract1
   end type
   !ERROR: An ABSTRACT derived type must be extensible
@@ -45,7 +45,7 @@ module m
   end type
   type, extends(intermediate) :: concrete2  ! ensure no false missing binding error
   end type
-  !WARNING: A derived type with the BIND attribute is empty
+  !WARNING: A derived type with the BIND attribute should not be empty
   type, bind(c) :: inextensible1
   end type
   !ERROR: The parent type is not extensible
diff --git a/flang/test/Semantics/resolve81.f90 b/flang/test/Semantics/resolve81.f90
index 87901fd7d2ef..5f0b66669423 100644
--- a/flang/test/Semantics/resolve81.f90
+++ b/flang/test/Semantics/resolve81.f90
@@ -5,9 +5,9 @@
 ! R801 type-declaration-stmt ->
 !        declaration-type-spec [[, attr-spec]... ::] entity-decl-list
 !  attr-spec values are:
-!    PUBLIC, PRIVATE, ALLOCATABLE, ASYNCHRONOUS, CODIMENSION, CONTIGUOUS, 
-!    DIMENSION (array-spec), EXTERNAL, INTENT (intent-spec), INTRINSIC, 
-!    BIND(C), OPTIONAL, PARAMETER, POINTER, PROTECTED, SAVE, TARGET, VALUE, 
+!    PUBLIC, PRIVATE, ALLOCATABLE, ASYNCHRONOUS, CODIMENSION, CONTIGUOUS,
+!    DIMENSION (array-spec), EXTERNAL, INTENT (intent-spec), INTRINSIC,
+!    BIND(C), OPTIONAL, PARAMETER, POINTER, PROTECTED, SAVE, TARGET, VALUE,
 !    VOLATILE
 module m
 
@@ -28,7 +28,7 @@ module m
   !WARNING: Attribute 'EXTERNAL' cannot be used more than once
   real, external, external :: externFunc
   !WARNING: Attribute 'INTRINSIC' cannot be used more than once
-  !ERROR: An interface name with BIND attribute must be specified if the BIND attribute is specified in a procedure declaration statement
+  !ERROR: 'cos' may not have both the BIND(C) and INTRINSIC attributes
   real, intrinsic, bind(c), intrinsic :: cos
   !WARNING: Attribute 'BIND(C)' cannot be used more than once
   integer, bind(c), volatile, bind(c) :: bindVar
diff --git a/flang/test/Semantics/resolve85.f90 b/flang/test/Semantics/resolve85.f90
index f598456f9830..9b9358ecf477 100644
--- a/flang/test/Semantics/resolve85.f90
+++ b/flang/test/Semantics/resolve85.f90
@@ -24,7 +24,7 @@ module m
   end type derived4
 
   !WARNING: Attribute 'BIND(C)' cannot be used more than once
-  !WARNING: A derived type with the BIND attribute is empty
+  !WARNING: A derived type with the BIND attribute should not be empty
   type, bind(c), public, bind(c) :: derived5
   end type derived5
 
-- 
GitLab


From 8fd838a8c499b4ce2822d51d1c661058ccc08c7d Mon Sep 17 00:00:00 2001
From: Harald van Dijk <harald.vandijk@codeplay.com>
Date: Thu, 9 May 2024 19:15:42 +0100
Subject: [PATCH 0333/1206] [RISC-V] Limit vscale interleaving to addrspace 0.
 (#91573)

The vlseg and vsseg intrinsic functions are not overloaded on pointer
type, so cannot handle non-default address spaces.

This fixes an error we see after #90583.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  5 +
 .../RISCV/interleaved-accesses.ll             | 97 +++++++++++++++++++
 2 files changed, 102 insertions(+)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 846768f6d631..00a97d15db3e 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -21048,6 +21048,11 @@ bool RISCVTargetLowering::isLegalInterleavedAccessType(
       return false;
 
     ContainerVT = getContainerForFixedLengthVector(VT.getSimpleVT());
+  } else {
+    // The intrinsics for scalable vectors are not overloaded on pointer type
+    // and can only handle the default address space.
+    if (AddrSpace)
+      return false;
   }
 
   // Need to make sure that EMUL * NFIELDS ≤ 8
diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll
index 9ae9245eb15d..66ece62bd74f 100644
--- a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll
@@ -23,6 +23,55 @@ define void @load_factor2(ptr %ptr) {
   ret void
 }
 
+define void @load_factor2_as(ptr addrspace(1) %ptr) {
+; RV32-LABEL: @load_factor2_as(
+; RV32-NEXT:    [[TMP1:%.*]] = call { <8 x i32>, <8 x i32> } @llvm.riscv.seg2.load.v8i32.p1.i32(ptr addrspace(1) [[PTR:%.*]], i32 8)
+; RV32-NEXT:    [[TMP2:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[TMP1]], 1
+; RV32-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[TMP1]], 0
+; RV32-NEXT:    ret void
+;
+; RV64-LABEL: @load_factor2_as(
+; RV64-NEXT:    [[TMP1:%.*]] = call { <8 x i32>, <8 x i32> } @llvm.riscv.seg2.load.v8i32.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 8)
+; RV64-NEXT:    [[TMP2:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[TMP1]], 1
+; RV64-NEXT:    [[TMP3:%.*]] = extractvalue { <8 x i32>, <8 x i32> } [[TMP1]], 0
+; RV64-NEXT:    ret void
+;
+  %interleaved.vec = load <16 x i32>, ptr addrspace(1) %ptr
+  %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  ret void
+}
+
+define void @load_factor2_vscale(ptr %ptr) {
+; RV32-LABEL: @load_factor2_vscale(
+; RV32-NEXT:    [[TMP1:%.*]] = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.riscv.vlseg2.nxv8i32.i32(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison, ptr [[PTR:%.*]], i32 -1)
+; RV32-NEXT:    ret void
+;
+; RV64-LABEL: @load_factor2_vscale(
+; RV64-NEXT:    [[TMP1:%.*]] = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.riscv.vlseg2.nxv8i32.i64(<vscale x 8 x i32> poison, <vscale x 8 x i32> poison, ptr [[PTR:%.*]], i64 -1)
+; RV64-NEXT:    ret void
+;
+  %interleaved.vec = load <vscale x 16 x i32>, ptr %ptr
+  %v = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %interleaved.vec)
+  ret void
+}
+
+define void @load_factor2_vscale_as(ptr addrspace(1) %ptr) {
+; RV32-LABEL: @load_factor2_vscale_as(
+; RV32-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <vscale x 16 x i32>, ptr addrspace(1) [[PTR:%.*]], align 64
+; RV32-NEXT:    [[V:%.*]] = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> [[INTERLEAVED_VEC]])
+; RV32-NEXT:    ret void
+;
+; RV64-LABEL: @load_factor2_vscale_as(
+; RV64-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <vscale x 16 x i32>, ptr addrspace(1) [[PTR:%.*]], align 64
+; RV64-NEXT:    [[V:%.*]] = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> [[INTERLEAVED_VEC]])
+; RV64-NEXT:    ret void
+;
+  %interleaved.vec = load <vscale x 16 x i32>, ptr addrspace(1) %ptr
+  %v = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %interleaved.vec)
+  ret void
+}
+
 define void @load_factor3(ptr %ptr) {
 ; RV32-LABEL: @load_factor3(
 ; RV32-NEXT:    [[TMP1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.riscv.seg3.load.v4i32.p0.i32(ptr [[PTR:%.*]], i32 4)
@@ -219,6 +268,54 @@ define void @store_factor2(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
   ret void
 }
 
+define void @store_factor2_as(ptr addrspace(1) %ptr, <8 x i8> %v0, <8 x i8> %v1) {
+; RV32-LABEL: @store_factor2_as(
+; RV32-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i8> [[V0:%.*]], <8 x i8> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; RV32-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[V0]], <8 x i8> [[V1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; RV32-NEXT:    call void @llvm.riscv.seg2.store.v8i8.p1.i32(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]], ptr addrspace(1) [[PTR:%.*]], i32 8)
+; RV32-NEXT:    ret void
+;
+; RV64-LABEL: @store_factor2_as(
+; RV64-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i8> [[V0:%.*]], <8 x i8> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; RV64-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[V0]], <8 x i8> [[V1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; RV64-NEXT:    call void @llvm.riscv.seg2.store.v8i8.p1.i64(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]], ptr addrspace(1) [[PTR:%.*]], i64 8)
+; RV64-NEXT:    ret void
+;
+  %interleaved.vec = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i8> %interleaved.vec, ptr addrspace(1) %ptr, align 4
+  ret void
+}
+
+define void @store_factor2_vscale(ptr %ptr, <vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1) {
+; RV32-LABEL: @store_factor2_vscale(
+; RV32-NEXT:    call void @llvm.riscv.vsseg2.nxv8i8.i32(<vscale x 8 x i8> [[V0:%.*]], <vscale x 8 x i8> [[V1:%.*]], ptr [[PTR:%.*]], i32 -1)
+; RV32-NEXT:    ret void
+;
+; RV64-LABEL: @store_factor2_vscale(
+; RV64-NEXT:    call void @llvm.riscv.vsseg2.nxv8i8.i64(<vscale x 8 x i8> [[V0:%.*]], <vscale x 8 x i8> [[V1:%.*]], ptr [[PTR:%.*]], i64 -1)
+; RV64-NEXT:    ret void
+;
+  %interleaved.vec = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv8i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1)
+  store <vscale x 16 x i8> %interleaved.vec, ptr %ptr, align 4
+  ret void
+}
+
+define void @store_factor2_vscale_as(ptr addrspace(1) %ptr, <vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1) {
+; RV32-LABEL: @store_factor2_vscale_as(
+; RV32-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> [[V0:%.*]], <vscale x 8 x i8> [[V1:%.*]])
+; RV32-NEXT:    store <vscale x 16 x i8> [[INTERLEAVED_VEC]], ptr addrspace(1) [[PTR:%.*]], align 4
+; RV32-NEXT:    ret void
+;
+; RV64-LABEL: @store_factor2_vscale_as(
+; RV64-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> [[V0:%.*]], <vscale x 8 x i8> [[V1:%.*]])
+; RV64-NEXT:    store <vscale x 16 x i8> [[INTERLEAVED_VEC]], ptr addrspace(1) [[PTR:%.*]], align 4
+; RV64-NEXT:    ret void
+;
+  %interleaved.vec = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv8i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1)
+  store <vscale x 16 x i8> %interleaved.vec, ptr addrspace(1) %ptr, align 4
+  ret void
+}
+
 define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
 ; RV32-LABEL: @store_factor3(
 ; RV32-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-- 
GitLab


From c3d2af0f4e180e67c4c5dd0f83bed1ea226f4565 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 9 May 2024 19:18:36 +0100
Subject: [PATCH 0334/1206] [VPlan] VPEVLBasedIVPHI is a VPSingleDefRecipe.

VPEVLBasedIVPHIRecipe inherits from VPSingleDefRecipe. Add
VPEVLBasedIVPHISC to VPSingleDefRecipe::classof to make isa/dyn_cast &
co work as expected.

Split off https://github.com/llvm/llvm-project/pull/67934.
---
 llvm/lib/Transforms/Vectorize/VPlan.h             | 1 +
 llvm/unittests/Transforms/Vectorize/VPlanTest.cpp | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 461d7ec59862..0784665efd14 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -841,6 +841,7 @@ public:
   static inline bool classof(const VPRecipeBase *R) {
     switch (R->getVPDefID()) {
     case VPRecipeBase::VPDerivedIVSC:
+    case VPRecipeBase::VPEVLBasedIVPHISC:
     case VPRecipeBase::VPExpandSCEVSC:
     case VPRecipeBase::VPInstructionSC:
     case VPRecipeBase::VPReductionSC:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index eda4723f67b2..5c45d86130bd 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1530,5 +1530,13 @@ TEST(VPDoubleValueDefTest, traverseUseLists) {
   EXPECT_EQ(&DoubleValueDef, I3.getOperand(0)->getDefiningRecipe());
 }
 
+TEST(VPRecipeTest, CastToVPSingleDefRecipe) {
+  VPValue Start;
+  VPEVLBasedIVPHIRecipe R(&Start, {});
+  VPRecipeBase *B = &R;
+  EXPECT_TRUE(isa<VPSingleDefRecipe>(B));
+  // TODO: check other VPSingleDefRecipes.
+}
+
 } // namespace
 } // namespace llvm
-- 
GitLab


From b942c24845a39e6161c8623b1efc4e2083d879e9 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 9 May 2024 11:19:03 -0700
Subject: [PATCH 0335/1206] [flang] Don't crash on not-yet-implemented feature
 (#91368)

A procedure pointer can be initialized in a DATA statement, but
semantics crashes if the initializer is the name of an intrinsic
function. This patch fixes that crash so that compilation survives to
the point where lowering admits that it doesn't yet support the feature.

Addresses https://github.com/llvm/llvm-project/issues/91295.
---
 flang/lib/Semantics/data-to-inits.cpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/data-to-inits.cpp b/flang/lib/Semantics/data-to-inits.cpp
index 64050874bcde..605a9f10712e 100644
--- a/flang/lib/Semantics/data-to-inits.cpp
+++ b/flang/lib/Semantics/data-to-inits.cpp
@@ -903,7 +903,13 @@ void ConstructInitializer(const Symbol &symbol,
       if (const auto *procDesignator{
               std::get_if<evaluate::ProcedureDesignator>(&expr->u)}) {
         CHECK(!procDesignator->GetComponent());
-        mutableProc.set_init(DEREF(procDesignator->GetSymbol()));
+        if (const auto *intrin{procDesignator->GetSpecificIntrinsic()}) {
+          const Symbol *intrinSymbol{
+              symbol.owner().FindSymbol(SourceName{intrin->name})};
+          mutableProc.set_init(DEREF(intrinSymbol));
+        } else {
+          mutableProc.set_init(DEREF(procDesignator->GetSymbol()));
+        }
       } else {
         CHECK(evaluate::IsNullProcedurePointer(*expr));
         mutableProc.set_init(nullptr);
-- 
GitLab


From 19b41f40a4b93a6243c816b80b6e664a4418f79f Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 9 May 2024 11:31:13 -0700
Subject: [PATCH 0336/1206] [flang] Complete RESULT() name constraint checking
 (#91476)

There are two constraints in the language that prohibit the use of an
ENTRY name being used as the RESULT() variable of the function or
another ENTRY name in the same function's scope; neither can the name of
the function be used as the RESULT() of an ENTRY.

Move most of the existing partial enforcement of these constraints from
name resolution into declaration checking, complete it, and add more
cases to the tests.
---
 flang/lib/Semantics/check-declarations.cpp | 23 +++++++++-
 flang/lib/Semantics/resolve-names.cpp      | 49 +++++++++-------------
 flang/test/Semantics/entry01.f90           | 15 ++++++-
 3 files changed, 54 insertions(+), 33 deletions(-)

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index 9ef34f0f81c0..26efa288b5ae 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -1361,7 +1361,7 @@ void CheckHelper::CheckSubprogram(
     SubprogramMatchHelper{*this}.Check(symbol, *iface);
   }
   if (const Scope *entryScope{details.entryScope()}) {
-    // ENTRY 15.6.2.6, esp. C1571
+    // ENTRY F'2023 15.6.2.6
     std::optional<parser::MessageFixedText> error;
     const Symbol *subprogram{entryScope->symbol()};
     const SubprogramDetails *subprogramDetails{nullptr};
@@ -1393,6 +1393,27 @@ void CheckHelper::CheckSubprogram(
       }
     }
   }
+  if (details.isFunction() &&
+      details.result().name() != symbol.name()) { // F'2023 C1569 & C1583
+    if (auto iter{symbol.owner().find(details.result().name())};
+        iter != symbol.owner().end()) {
+      const Symbol &resNameSym{*iter->second};
+      if (const auto *resNameSubp{resNameSym.detailsIf<SubprogramDetails>()}) {
+        if (const Scope * resNameEntryScope{resNameSubp->entryScope()}) {
+          const Scope *myScope{
+              details.entryScope() ? details.entryScope() : symbol.scope()};
+          if (resNameEntryScope == myScope) {
+            if (auto *msg{messages_.Say(symbol.name(),
+                    "Explicit RESULT('%s') of function '%s' cannot have the same name as a distinct ENTRY into the same scope"_err_en_US,
+                    details.result().name(), symbol.name())}) {
+              msg->Attach(
+                  resNameSym.name(), "ENTRY with conflicting name"_en_US);
+            }
+          }
+        }
+      }
+    }
+  }
   if (const MaybeExpr & stmtFunction{details.stmtFunction()}) {
     if (auto msg{evaluate::CheckStatementFunction(
             symbol, *stmtFunction, context_.foldingContext())}) {
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 4df347a878de..e2875081b732 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -4048,27 +4048,10 @@ void SubprogramVisitor::CreateEntry(
       attrs = extant->attrs();
     }
   }
-  bool badResultName{false};
   std::optional<SourceName> distinctResultName;
   if (suffix && suffix->resultName &&
       suffix->resultName->source != entryName.source) {
     distinctResultName = suffix->resultName->source;
-    const parser::Name &resultName{*suffix->resultName};
-    if (resultName.source == subprogram.name()) { // C1574
-      Say2(resultName.source,
-          "RESULT(%s) may not have the same name as the function"_err_en_US,
-          subprogram, "Containing function"_en_US);
-      badResultName = true;
-    } else if (const Symbol * extant{FindSymbol(outer, resultName)}) { // C1574
-      if (const auto *details{extant->detailsIf<SubprogramDetails>()}) {
-        if (details->entryScope() == &currScope()) {
-          Say2(resultName.source,
-              "RESULT(%s) may not have the same name as an ENTRY in the function"_err_en_US,
-              extant->name(), "Conflicting ENTRY"_en_US);
-          badResultName = true;
-        }
-      }
-    }
   }
   if (outer.IsModule() && !attrs.test(Attr::PRIVATE)) {
     attrs.set(Attr::PUBLIC);
@@ -4104,17 +4087,24 @@ void SubprogramVisitor::CreateEntry(
     EntityDetails resultDetails;
     resultDetails.set_funcResult(true);
     if (distinctResultName) {
-      if (!badResultName) {
-        // RESULT(x) can be the same explicitly-named RESULT(x) as
-        // the enclosing function or another ENTRY.
-        if (auto iter{currScope().find(suffix->resultName->source)};
-            iter != currScope().end()) {
-          result = &*iter->second;
-        }
-        if (!result) {
-          result = &MakeSymbol(
-              *distinctResultName, Attrs{}, std::move(resultDetails));
-        }
+      // An explicit RESULT() can also be an explicit RESULT()
+      // of the function or another ENTRY.
+      if (auto iter{currScope().find(suffix->resultName->source)};
+          iter != currScope().end()) {
+        result = &*iter->second;
+      }
+      if (!result) {
+        result =
+            &MakeSymbol(*distinctResultName, Attrs{}, std::move(resultDetails));
+      } else if (!result->has<EntityDetails>()) {
+        Say(*distinctResultName,
+            "ENTRY cannot have RESULT(%s) that is not a variable"_err_en_US,
+            *distinctResultName)
+            .Attach(result->name(), "Existing declaration of '%s'"_en_US,
+                result->name());
+        result = nullptr;
+      }
+      if (result) {
         Resolve(*suffix->resultName, *result);
       }
     } else {
@@ -4124,8 +4114,7 @@ void SubprogramVisitor::CreateEntry(
       entryDetails.set_result(*result);
     }
   }
-  if (subpFlag == Symbol::Flag::Subroutine ||
-      (distinctResultName && !badResultName)) {
+  if (subpFlag == Symbol::Flag::Subroutine || distinctResultName) {
     Symbol &assoc{MakeSymbol(entryName.source)};
     assoc.set_details(HostAssocDetails{*entrySymbol});
     assoc.set(Symbol::Flag::Subroutine);
diff --git a/flang/test/Semantics/entry01.f90 b/flang/test/Semantics/entry01.f90
index 64bd954f8ae0..970cd109921a 100644
--- a/flang/test/Semantics/entry01.f90
+++ b/flang/test/Semantics/entry01.f90
@@ -86,11 +86,12 @@ function ifunc()
   entry ibad2()
   !ERROR: ENTRY in a function may not have an alternate return dummy argument
   entry ibadalt(*) ! C1573
-  !ERROR: RESULT(ifunc) may not have the same name as the function
+  !ERROR: ENTRY cannot have RESULT(ifunc) that is not a variable
   entry isameres() result(ifunc) ! C1574
   entry iok()
-  !ERROR: RESULT(iok) may not have the same name as an ENTRY in the function
+  !ERROR: Explicit RESULT('iok') of function 'isameres2' cannot have the same name as a distinct ENTRY into the same scope
   entry isameres2() result(iok) ! C1574
+  !ERROR: Explicit RESULT('iok2') of function 'isameres3' cannot have the same name as a distinct ENTRY into the same scope
   entry isameres3() result(iok2) ! C1574
   !ERROR: 'iok2' is already declared in this scoping unit
   entry iok2()
@@ -255,3 +256,13 @@ subroutine s7(q,q)
   !ERROR: 'z' appears more than once as a dummy argument name in this ENTRY statement
   entry baz(z,z)
 end
+
+!ERROR: Explicit RESULT('f8e1') of function 'f8' cannot have the same name as a distinct ENTRY into the same scope
+function f8() result(f8e1)
+  entry f8e1()
+  entry f8e2() result(f8e2) ! ok
+  !ERROR: Explicit RESULT('f8e1') of function 'f8e3' cannot have the same name as a distinct ENTRY into the same scope
+  entry f8e3() result(f8e1)
+  !ERROR: ENTRY cannot have RESULT(f8) that is not a variable
+  entry f8e4() result(f8)
+end
-- 
GitLab


From a51d92a44740fd1b17930de183c9ad6d993029b1 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Thu, 9 May 2024 11:42:25 -0700
Subject: [PATCH 0337/1206] [flang] Fix crash in semantics on error case
 (#91482)

An erroneous statement function declaration exposed an unhandled
situation in a utility routine in semantics. Patch that hole and add a
test.

Fixes https://github.com/llvm/llvm-project/issues/91429.
---
 flang/lib/Semantics/tools.cpp        | 14 ++++++++------
 flang/test/Semantics/stmt-func01.f90 |  8 ++++++++
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 2d0caff82eb2..99381918fc63 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -256,15 +256,17 @@ static const Symbol &FollowHostAssoc(const Symbol &symbol) {
 }
 
 bool IsHostAssociated(const Symbol &symbol, const Scope &scope) {
-  return DoesScopeContain(
-      &GetProgramUnitOrBlockConstructContaining(FollowHostAssoc(symbol)),
-      GetProgramUnitOrBlockConstructContaining(scope));
+  const Symbol &base{FollowHostAssoc(symbol)};
+  return base.owner().IsTopLevel() ||
+      DoesScopeContain(&GetProgramUnitOrBlockConstructContaining(base),
+          GetProgramUnitOrBlockConstructContaining(scope));
 }
 
 bool IsHostAssociatedIntoSubprogram(const Symbol &symbol, const Scope &scope) {
-  return DoesScopeContain(
-      &GetProgramUnitOrBlockConstructContaining(FollowHostAssoc(symbol)),
-      GetProgramUnitContaining(scope));
+  const Symbol &base{FollowHostAssoc(symbol)};
+  return base.owner().IsTopLevel() ||
+      DoesScopeContain(&GetProgramUnitOrBlockConstructContaining(base),
+          GetProgramUnitContaining(scope));
 }
 
 bool IsInStmtFunction(const Symbol &symbol) {
diff --git a/flang/test/Semantics/stmt-func01.f90 b/flang/test/Semantics/stmt-func01.f90
index 733a7a56dfdb..3c9ffa565900 100644
--- a/flang/test/Semantics/stmt-func01.f90
+++ b/flang/test/Semantics/stmt-func01.f90
@@ -83,3 +83,11 @@ subroutine s4
   !ERROR: VOLATILE attribute may apply only to a variable
   sf(x) = 1.
 end
+
+subroutine s5
+  !ERROR: Invalid specification expression: reference to impure function 'k'
+  real x(k())
+  !WARNING: Name 'k' from host scope should have a type declaration before its local statement function definition
+  !ERROR: 'k' is already declared in this scoping unit
+  k() = 0.0
+end
-- 
GitLab


From dcf92a249233cab103f848dd12e96e0d642a8899 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Thu, 9 May 2024 23:03:42 +0400
Subject: [PATCH 0338/1206] [clang][NFC] Add examples from [dcl.init.aggr] to
 C++ conformance tests (#91435)

This patch adds examples from 2024-04-22 draft of
[[dcl.init.aggr]](http://eel.is/c++draft/dcl.init.aggr) to C++
conformance tests. Testing is done via constant evaluation and static
asserts. As far as I can see, the rest of the conformance suite is
typically testing the latest language mode at the time of writing (with
a notable exception of defect report tests), so I'm also testing in the
latest language mode.
---
 clang/test/CXX/dcl/dcl.init/aggr.cpp | 294 +++++++++++++++++++++++++++
 1 file changed, 294 insertions(+)
 create mode 100644 clang/test/CXX/dcl/dcl.init/aggr.cpp

diff --git a/clang/test/CXX/dcl/dcl.init/aggr.cpp b/clang/test/CXX/dcl/dcl.init/aggr.cpp
new file mode 100644
index 000000000000..3206d2e7f616
--- /dev/null
+++ b/clang/test/CXX/dcl/dcl.init/aggr.cpp
@@ -0,0 +1,294 @@
+// RUN:  %clang_cc1 -std=c++2c -verify %s
+
+namespace ex1 {
+struct C {
+  union {
+    int a;
+    const char* p;
+  };
+  int x;
+};
+
+constexpr C c = { .a = 1, .x = 3 };
+static_assert(c.a == 1);
+static_assert(c.x == 3);
+
+static constexpr C c2 = { .a = 1.0, .x = 3 };
+// expected-error@-1 {{type 'double' cannot be narrowed to 'int' in initializer list}}
+//   expected-note@-2 {{insert an explicit cast to silence this issue}}
+} // namespace ex1
+
+namespace ex2 {
+struct A {
+  int x;
+  struct B {
+    int i;
+    int j;
+  } b;
+};
+
+constexpr A a = { 1, { 2, 3 } };
+static_assert(a.x == 1);
+static_assert(a.b.i == 2);
+static_assert(a.b.j == 3);
+
+struct base1 { int b1, b2 = 42; };
+struct base2 {
+  constexpr base2() {
+    b3 = 43;
+  }
+  int b3;
+};
+struct derived : base1, base2 {
+  int d;
+};
+
+constexpr derived d1{{1, 2}, {}, 4};
+static_assert(d1.b1 == 1);
+static_assert(d1.b2 == 2);
+static_assert(d1.b3 == 43);
+static_assert(d1.d == 4);
+
+constexpr derived d2{{}, {}, 4};
+static_assert(d2.b1 == 0);
+static_assert(d2.b2 == 42);
+static_assert(d2.b3 == 43);
+static_assert(d2.d == 4);
+} // namespace ex2
+
+namespace ex3 {
+struct S {
+  int a;
+  const char* b;
+  int c;
+  int d = b[a];
+};
+
+constexpr S ss = { 1, "asdf" };
+static_assert(ss.a == 1);
+static_assert(__builtin_strcmp(ss.b, "asdf") == 0);
+static_assert(ss.c == int{});
+static_assert(ss.d == ss.b[ss.a]);
+
+struct string {
+  int d = 43;
+};
+
+struct A {
+  string a;
+  int b = 42;
+  int c = -1;
+};
+
+constexpr A a{.c = 21};
+static_assert(a.a.d == string{}.d);
+static_assert(a.b == 42);
+static_assert(a.c == 21);
+} // namespace ex3
+
+namespace ex4 {
+int x[] = { 1, 3, 5 };
+static_assert(sizeof(x) / sizeof(int) == 3);
+} // namespace ex4
+
+namespace ex5 {
+struct X { int i, j, k; };
+
+constexpr X a[] = { 1, 2, 3, 4, 5, 6 };
+constexpr X b[2] = { { 1, 2, 3 }, { 4, 5, 6 } };
+static_assert(sizeof(a) == sizeof(b));
+static_assert(a[0].i == b[0].i);
+static_assert(a[0].j == b[0].j);
+static_assert(a[0].k == b[0].k);
+static_assert(a[1].i == b[1].i);
+static_assert(a[1].j == b[1].j);
+static_assert(a[1].k == b[1].k);
+} // namespace ex5
+
+namespace ex6 {
+struct S {
+  int y[] = { 0 };
+  // expected-error@-1 {{array bound cannot be deduced from a default member initializer}}
+};
+} // namespace ex6
+
+namespace ex7 {
+struct A {
+  int i;
+  static int s;
+  int j;
+  int :17;
+  int k;
+};
+
+constexpr A a = { 1, 2, 3 };
+static_assert(a.i == 1);
+static_assert(a.j == 2);
+static_assert(a.k == 3);
+} // namespace ex7
+
+namespace ex8 {
+struct A;
+extern A a;
+struct A {
+  const A& a1 { A{a,a} };
+  const A& a2 { A{} };
+  // expected-error@-1 {{default member initializer for 'a2' needed within definition of enclosing class 'A' outside of member functions}}
+  //   expected-note@-2 {{default member initializer declared here}}
+};
+A a{a,a};
+
+struct B {
+  int n = B{}.n;
+  // expected-error@-1 {{default member initializer for 'n' needed within definition of enclosing class 'B' outside of member functions}}
+  //   expected-note@-2 {{default member initializer declared here}}
+};
+} // namespace ex8
+
+namespace ex9 {
+constexpr int x[2][2] = { 3, 1, 4, 2 };
+static_assert(x[0][0] == 3);
+static_assert(x[0][1] == 1);
+static_assert(x[1][0] == 4);
+static_assert(x[1][1] == 2);
+
+constexpr float y[4][3] = {
+  { 1 }, { 2 }, { 3 }, { 4 }
+};
+static_assert(y[0][0] == 1);
+static_assert(y[0][1] == 0);
+static_assert(y[0][2] == 0);
+static_assert(y[1][0] == 2);
+static_assert(y[1][1] == 0);
+static_assert(y[1][2] == 0);
+static_assert(y[2][0] == 3);
+static_assert(y[2][1] == 0);
+static_assert(y[2][2] == 0);
+static_assert(y[3][0] == 4);
+static_assert(y[3][1] == 0);
+static_assert(y[3][2] == 0);
+} // namespace ex9
+
+namespace ex10 {
+struct S1 { int a, b; };
+struct S2 { S1 s, t; };
+
+constexpr S2 x[2] = { 1, 2, 3, 4, 5, 6, 7, 8 };
+constexpr S2 y[2] = {
+  {
+    { 1, 2 },
+    { 3, 4 }
+  },
+  {
+    { 5, 6 },
+    { 7, 8 }
+  }
+};
+static_assert(x[0].s.a == 1);
+static_assert(x[0].s.b == 2);
+static_assert(x[0].t.a == 3);
+static_assert(x[0].t.b == 4);
+static_assert(x[1].s.a == 5);
+static_assert(x[1].s.b == 6);
+static_assert(x[1].t.a == 7);
+static_assert(x[1].t.b == 8);
+} // namespace ex10
+
+namespace ex11 {
+char cv[4] = { 'a', 's', 'd', 'f', 0 };
+// expected-error@-1 {{excess elements in array initializer}}
+} // namespace ex11
+
+namespace ex12 {
+constexpr float y[4][3] = {
+  { 1, 3, 5 },
+  { 2, 4, 6 },
+  { 3, 5, 7 },
+};
+static_assert(y[0][0] == 1);
+static_assert(y[0][1] == 3);
+static_assert(y[0][2] == 5);
+static_assert(y[1][0] == 2);
+static_assert(y[1][1] == 4);
+static_assert(y[1][2] == 6);
+static_assert(y[2][0] == 3);
+static_assert(y[2][1] == 5);
+static_assert(y[2][2] == 7);
+static_assert(y[3][0] == 0.0);
+static_assert(y[3][1] == 0.0);
+static_assert(y[3][2] == 0.0);
+
+constexpr float z[4][3] = {
+  1, 3, 5, 2, 4, 6, 3, 5, 7
+};
+static_assert(z[0][0] == 1);
+static_assert(z[0][1] == 3);
+static_assert(z[0][2] == 5);
+static_assert(z[1][0] == 2);
+static_assert(z[1][1] == 4);
+static_assert(z[1][2] == 6);
+static_assert(z[2][0] == 3);
+static_assert(z[2][1] == 5);
+static_assert(z[2][2] == 7);
+static_assert(z[3][0] == 0.0);
+static_assert(z[3][1] == 0.0);
+static_assert(z[3][2] == 0.0);
+} // namespace ex12
+
+namespace ex13 {
+struct S { } s;
+struct A {
+  S s1;
+  int i1;
+  S s2;
+  int i2;
+  S s3;
+  int i3;
+} a = {
+  { },              // Required initialization
+  0,
+  s,                // Required initialization
+  0
+};                  // Initialization not required for A​::​s3 because A​::​i3 is also not initialized
+} // namespace ex13
+
+namespace ex14 {
+struct A {
+  int i;
+  constexpr operator int() const { return 42; };
+};
+struct B {
+  A a1, a2;
+  int z;
+};
+constexpr A a{};
+constexpr B b = { 4, a, a };
+static_assert(b.a1.i == 4);
+static_assert(b.a2.i == a.i);
+static_assert(b.z == a.operator int());
+} // namespace ex14
+
+namespace ex15 {
+union u { // #ex15-u
+  int a;
+  const char* b;
+};
+
+u a = { 1 };
+u b = a;
+u c = 1;
+// expected-error@-1 {{no viable conversion from 'int' to 'u'}}
+//   expected-note@#ex15-u {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int' to 'const u &' for 1st argument}}
+//   expected-note@#ex15-u {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int' to 'u &&' for 1st argument}}
+u d = { 0, "asdf" };
+// expected-error@-1 {{excess elements in union initializer}}
+u e = { "asdf" };
+// expected-error@-1 {{cannot initialize a member subobject of type 'int' with an lvalue of type 'const char[5]'}}
+u f = { .b = "asdf" };
+u g = {
+  .a = 1, // #ex15-g-a
+  .b = "asdf"
+  // expected-error@-1 {{initializer partially overrides prior initialization of this subobject}}
+  //   expected-note@#ex15-g-a {{previous initialization is here}}
+};
+} // namespace ex15
-- 
GitLab


From d36b4abb51a9f84d436f184581b15021fdf22114 Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Thu, 9 May 2024 13:14:03 -0700
Subject: [PATCH 0339/1206] [bazel] Rework liblldb (#91549)

Previously we were linking liblldb as a shared library, but also linking
the contents into the lldb binary. This is invalid and results in subtle
runtime issues because of duplicate constants, like the global plugin
registry. This now links the dylib to lldb directly. This requires we
switch to cc_binary instead because cc_shared_library expects your
library to export all symbols in your transitive dependency tree, where
we only want to export lldb symbols.
---
 .../llvm-project-overlay/lldb/BUILD.bazel     | 69 +++++++++++++------
 1 file changed, 48 insertions(+), 21 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index 8fb90e850f00..b3a413c401cd 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -728,37 +728,64 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "liblldb.static",
-    deps = [
-        ":API",
-        ":Interpreter",
-    ],
+genrule(
+    name = "gen_exports_file_linux",
+    srcs = ["//lldb:source/API/liblldb-private.exports"],
+    outs = ["exports_linux.txt"],
+    cmd = """
+cat > $(OUTS) <<EOF
+{
+  global:
+    $$(sed 's/$$/;/g' $(SRCS))
+};
+EOF
+""",
 )
 
-cc_shared_library(
-    name = "liblldb",
-    # TODO: Remove once fixed https://github.com/bazelbuild/bazel/issues/21893
+genrule(
+    name = "gen_exports_file_macos",
+    srcs = ["//lldb:source/API/liblldb-private.exports"],
+    outs = ["exports_macos.txt"],
+    cmd = "sed 's/^/_/g' $(SRCS) > $(OUTS)",
+)
+
+# Create a shared library using linkshared=True for liblldb. This uses
+# cc_binary instead of cc_shared_library since the latter expects you to
+# re-export all transitive dependencies vs them being relinked into other
+# binaries.
+cc_binary(
+    name = "lldb{}".format(PACKAGE_VERSION),
     additional_linker_inputs = select({
+        "@platforms//os:linux": [
+            ":gen_exports_file_linux",
+        ],
         "@platforms//os:macos": [
-            ":HostMacOSXObjCXX",
-            "//lldb/source/Plugins:PluginPlatformMacOSXObjCXX",
+            ":gen_exports_file_macos",
         ],
         "//conditions:default": [],
     }),
-    shared_lib_name = select({
-        "@platforms//os:macos": "liblldb{}.dylib".format(PACKAGE_VERSION),
-        "@platforms//os:linux": "liblldb{}.so".format(PACKAGE_VERSION),
-    }),
-    # TODO: Remove once fixed https://github.com/bazelbuild/bazel/issues/21893
-    user_link_flags = select({
+    linkopts = select({
+        "@platforms//os:linux": [
+            "-Wl,--export-dynamic-symbol-list=$(location :gen_exports_file_linux)",
+        ],
         "@platforms//os:macos": [
-            "$(location :HostMacOSXObjCXX)",
-            "$(location //lldb/source/Plugins:PluginPlatformMacOSXObjCXX)",
+            "-Wl,-exported_symbols_list,$(location :gen_exports_file_macos)",
         ],
         "//conditions:default": [],
     }),
-    deps = [":liblldb.static"],
+    linkshared = True,
+    deps = [
+        ":API",
+        ":Interpreter",
+    ],
+)
+
+# cc_binary targets using linkshared=True to build a shared library cannot be
+# imported directly and instead need to be referenced indirectly through
+# cc_import
+cc_import(
+    name = "liblldb.wrapper",
+    shared_library = "lldb{}".format(PACKAGE_VERSION),
 )
 
 gentbl_cc_library(
@@ -794,7 +821,7 @@ cc_binary(
     deps = [
         ":APIHeaders",
         ":Host",
-        ":liblldb.static",
+        ":liblldb.wrapper",
         ":lldb_options_inc_gen",
         "//llvm:Option",
         "//llvm:Support",
-- 
GitLab


From 75ebcbf24abb2e5b045ec920bc86f667516e4cc6 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Thu, 9 May 2024 16:34:40 -0400
Subject: [PATCH 0340/1206] [Clang][Sema] Revert changes to operator= lookup in
 templated classes from #91498, #90999, and #90152 (#91620)

This reverts changes in #91498, #90999, and #90152 which make
`operator=` dependent whenever the current class is templated.
---
 clang/lib/Sema/SemaExprMember.cpp             |  5 +-
 clang/lib/Sema/SemaLookup.cpp                 | 51 +++++++------------
 clang/lib/Sema/SemaTemplate.cpp               |  8 ++-
 .../temp.res/temp.dep/temp.dep.type/p4.cpp    |  2 +-
 4 files changed, 28 insertions(+), 38 deletions(-)

diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp
index 5facb14a18b7..9fa69da4f968 100644
--- a/clang/lib/Sema/SemaExprMember.cpp
+++ b/clang/lib/Sema/SemaExprMember.cpp
@@ -996,7 +996,10 @@ Sema::BuildMemberReferenceExpr(Expr *BaseExpr, QualType BaseExprType,
   // build a CXXDependentScopeMemberExpr.
   if (R.wasNotFoundInCurrentInstantiation() ||
       (IsArrow && !BaseExprType->isPointerType() &&
-       BaseExprType->isDependentType()))
+       BaseExprType->isDependentType()) ||
+      (R.getLookupName().getCXXOverloadedOperator() == OO_Equal &&
+       (SS.isSet() ? SS.getScopeRep()->isDependent()
+                   : BaseExprType->isDependentType())))
     return ActOnDependentMemberExpr(BaseExpr, BaseExprType, IsArrow, OpLoc, SS,
                                     TemplateKWLoc, FirstQualifierInScope,
                                     R.getLookupNameInfo(), TemplateArgs);
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index e20de338ebb1..7251aabc6af2 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -1267,20 +1267,6 @@ struct FindLocalExternScope {
   LookupResult &R;
   bool OldFindLocalExtern;
 };
-
-/// Returns true if 'operator=' should be treated as a dependent name.
-bool isDependentAssignmentOperator(DeclarationName Name,
-                                   DeclContext *LookupContext) {
-  const auto *LookupRecord = dyn_cast_if_present<CXXRecordDecl>(LookupContext);
-  // If the lookup context is the current instantiation but we are outside a
-  // complete-class context, we will never find the implicitly declared
-  // copy/move assignment operators because they are declared at the closing '}'
-  // of the class specifier. In such cases, we treat 'operator=' like any other
-  // unqualified name because the results of name lookup in the template
-  // definition/instantiation context will always be the same.
-  return Name.getCXXOverloadedOperator() == OO_Equal && LookupRecord &&
-         !LookupRecord->isBeingDefined() && LookupRecord->isDependentContext();
-}
 } // end anonymous namespace
 
 bool Sema::CppLookupName(LookupResult &R, Scope *S) {
@@ -1289,6 +1275,14 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
   DeclarationName Name = R.getLookupName();
   Sema::LookupNameKind NameKind = R.getLookupKind();
 
+  // If this is the name of an implicitly-declared special member function,
+  // go through the scope stack to implicitly declare
+  if (isImplicitlyDeclaredMemberFunctionName(Name)) {
+    for (Scope *PreS = S; PreS; PreS = PreS->getParent())
+      if (DeclContext *DC = PreS->getEntity())
+        DeclareImplicitMemberFunctionsWithName(*this, Name, R.getNameLoc(), DC);
+  }
+
   // C++23 [temp.dep.general]p2:
   //   The component name of an unqualified-id is dependent if
   //   - it is a conversion-function-id whose conversion-type-id
@@ -1301,20 +1295,6 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
     return false;
   }
 
-  // If this is the name of an implicitly-declared special member function,
-  // go through the scope stack to implicitly declare
-  if (isImplicitlyDeclaredMemberFunctionName(Name)) {
-    for (Scope *PreS = S; PreS; PreS = PreS->getParent())
-      if (DeclContext *DC = PreS->getEntity()) {
-        if (!R.isTemplateNameLookup() &&
-            isDependentAssignmentOperator(Name, DC)) {
-          R.setNotFoundInCurrentInstantiation();
-          return false;
-        }
-        DeclareImplicitMemberFunctionsWithName(*this, Name, R.getNameLoc(), DC);
-      }
-  }
-
   // Implicitly declare member functions with the name we're looking for, if in
   // fact we are in a scope where it matters.
 
@@ -2478,6 +2458,10 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
     }
   } QL(LookupCtx);
 
+  CXXRecordDecl *LookupRec = dyn_cast<CXXRecordDecl>(LookupCtx);
+  // FIXME: Per [temp.dep.general]p2, an unqualified name is also dependent
+  // if it's a dependent conversion-function-id or operator= where the current
+  // class is a templated entity. This should be handled in LookupName.
   if (!InUnqualifiedLookup && !R.isForRedeclaration()) {
     // C++23 [temp.dep.type]p5:
     //   A qualified name is dependent if
@@ -2488,16 +2472,13 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
     //     is operator=, or
     //   - [...]
     if (DeclarationName Name = R.getLookupName();
-        (Name.getNameKind() == DeclarationName::CXXConversionFunctionName &&
-         Name.getCXXNameType()->isDependentType()) ||
-        (!R.isTemplateNameLookup() &&
-         isDependentAssignmentOperator(Name, LookupCtx))) {
+        Name.getNameKind() == DeclarationName::CXXConversionFunctionName &&
+        Name.getCXXNameType()->isDependentType()) {
       R.setNotFoundInCurrentInstantiation();
       return false;
     }
   }
 
-  CXXRecordDecl *LookupRec = dyn_cast<CXXRecordDecl>(LookupCtx);
   if (LookupDirect(*this, R, LookupCtx)) {
     R.resolveKind();
     if (LookupRec)
@@ -2588,6 +2569,8 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
     return true;
   };
 
+  bool TemplateNameLookup = R.isTemplateNameLookup();
+
   // Determine whether two sets of members contain the same members, as
   // required by C++ [class.member.lookup]p6.
   auto HasSameDeclarations = [&](DeclContext::lookup_iterator A,
@@ -2609,7 +2592,7 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
         //   template, and if the name is used as a template-name, the
         //   reference refers to the class template itself and not a
         //   specialization thereof, and is not ambiguous.
-        if (R.isTemplateNameLookup())
+        if (TemplateNameLookup)
           if (auto *TD = getAsTemplateNameDecl(ND))
             ND = TD;
 
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 480bc74c2001..2fce2238f9c1 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -726,7 +726,7 @@ Sema::ActOnDependentIdExpression(const CXXScopeSpec &SS,
                                  const DeclarationNameInfo &NameInfo,
                                  bool isAddressOfOperand,
                            const TemplateArgumentListInfo *TemplateArgs) {
-  QualType ThisType = getCurrentThisType();
+  DeclContext *DC = getFunctionLevelDeclContext();
 
   // C++11 [expr.prim.general]p12:
   //   An id-expression that denotes a non-static data member or non-static
@@ -748,7 +748,11 @@ Sema::ActOnDependentIdExpression(const CXXScopeSpec &SS,
     IsEnum = isa_and_nonnull<EnumType>(NNS->getAsType());
 
   if (!MightBeCxx11UnevalField && !isAddressOfOperand && !IsEnum &&
-      !ThisType.isNull()) {
+      isa<CXXMethodDecl>(DC) &&
+      cast<CXXMethodDecl>(DC)->isImplicitObjectMemberFunction()) {
+    QualType ThisType =
+        cast<CXXMethodDecl>(DC)->getThisType().getNonReferenceType();
+
     // Since the 'this' expression is synthesized, we don't need to
     // perform the double-lookup check.
     NamedDecl *FirstQualifierInScope = nullptr;
diff --git a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
index 43053c18c507..1adbc33a701c 100644
--- a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
@@ -483,7 +483,6 @@ namespace N3 {
   };
 
   template struct E<int>; // expected-note {{in instantiation of template class 'N3::E<int>' requested here}}
-
 } // namespace N3
 
 namespace N4 {
@@ -551,4 +550,5 @@ namespace N4 {
   };
 
   template void D<B>::instantiated(D); // expected-note {{in instantiation of}}
+
 } // namespace N4
-- 
GitLab


From 8ac928fea8d561261133e0337a2777fcda99f7ba Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Thu, 9 May 2024 16:34:45 -0400
Subject: [PATCH 0341/1206] [libc][NFC] adjust time related implementations
 (#91485)

---
 libc/hdr/CMakeLists.txt                       |  9 ++++
 libc/hdr/time_macros.h                        | 22 +++++++++
 libc/hdr/types/CMakeLists.txt                 | 45 +++++++++++++++++++
 libc/hdr/types/clock_t.h                      | 22 +++++++++
 libc/hdr/types/clockid_t.h                    | 22 +++++++++
 libc/hdr/types/struct_timeval.h               | 21 +++++++++
 libc/hdr/types/suseconds_t.h                  | 22 +++++++++
 libc/hdr/types/time_t.h                       | 22 +++++++++
 libc/src/__support/CMakeLists.txt             |  2 +
 libc/src/__support/time/CMakeLists.txt        | 19 ++++++++
 libc/src/__support/time/clock_gettime.h       | 23 ++++++++++
 libc/src/__support/time/linux/CMakeLists.txt  | 14 ++++++
 .../time/linux/clock_gettime.cpp}             | 25 ++++-------
 libc/src/__support/time/units.h               | 38 ++++++++++++++++
 libc/src/time/clock.h                         |  2 +-
 libc/src/time/clock_gettime.h                 |  5 ++-
 libc/src/time/gettimeofday.h                  |  2 +-
 libc/src/time/linux/CMakeLists.txt            | 30 +++++++------
 libc/src/time/linux/clock.cpp                 | 20 ++++-----
 libc/src/time/linux/clock_gettime.cpp         |  9 +---
 libc/src/time/linux/gettimeofday.cpp          | 14 +++---
 libc/src/time/linux/time.cpp                  | 12 ++---
 libc/src/time/nanosleep.h                     |  4 +-
 libc/src/time/time_func.h                     |  2 +-
 24 files changed, 335 insertions(+), 71 deletions(-)
 create mode 100644 libc/hdr/time_macros.h
 create mode 100644 libc/hdr/types/clock_t.h
 create mode 100644 libc/hdr/types/clockid_t.h
 create mode 100644 libc/hdr/types/struct_timeval.h
 create mode 100644 libc/hdr/types/suseconds_t.h
 create mode 100644 libc/hdr/types/time_t.h
 create mode 100644 libc/src/__support/time/CMakeLists.txt
 create mode 100644 libc/src/__support/time/clock_gettime.h
 create mode 100644 libc/src/__support/time/linux/CMakeLists.txt
 rename libc/src/{time/linux/clockGetTimeImpl.h => __support/time/linux/clock_gettime.cpp} (64%)
 create mode 100644 libc/src/__support/time/units.h

diff --git a/libc/hdr/CMakeLists.txt b/libc/hdr/CMakeLists.txt
index 179b05e6ee96..754934251430 100644
--- a/libc/hdr/CMakeLists.txt
+++ b/libc/hdr/CMakeLists.txt
@@ -68,4 +68,13 @@ add_proxy_header_library(
     libc.include.llvm-libc-macros.sys_epoll_macros
 )
 
+add_proxy_header_library(
+  time_macros
+  HDRS
+    time_macros.h
+  FULL_BUILD_DEPENDS
+    libc.include.time
+    libc.include.llvm-libc-macros.time_macros
+)
+
 add_subdirectory(types)
diff --git a/libc/hdr/time_macros.h b/libc/hdr/time_macros.h
new file mode 100644
index 000000000000..dc36fe66f7a8
--- /dev/null
+++ b/libc/hdr/time_macros.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from time.h ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TIME_MACROS_H
+#define LLVM_LIBC_HDR_TIME_MACROS_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-macros/time-macros.h"
+
+#else // Overlay mode
+
+#include <time.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TIME_MACROS_H
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 46a66ec59020..3a1bb2f3c340 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -63,3 +63,48 @@ add_proxy_header_library(
     libc.include.llvm-libc-types.fexcept_t
     libc.include.fenv
 )
+
+add_proxy_header_library(
+  time_t
+  HDRS
+    time_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.time_t
+    libc.include.time
+)
+
+add_proxy_header_library(
+  clockid_t
+  HDRS
+    clockid_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.clockid_t
+    libc.include.sys_types
+)
+
+add_proxy_header_library(
+  clock_t
+  HDRS
+    clock_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.clock_t
+    libc.include.time
+)
+
+add_proxy_header_library(
+  suseconds_t
+  HDRS
+    suseconds_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.suseconds_t
+    libc.include.sys_time
+)
+
+add_proxy_header_library(
+  struct_timeval
+  HDRS
+    struct_timeval.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.struct_timeval
+    libc.include.sys_time
+)
diff --git a/libc/hdr/types/clock_t.h b/libc/hdr/types/clock_t.h
new file mode 100644
index 000000000000..b0b658e96c3d
--- /dev/null
+++ b/libc/hdr/types/clock_t.h
@@ -0,0 +1,22 @@
+//===-- Proxy for clock_t -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_CLOCK_T_H
+#define LLVM_LIBC_HDR_TYPES_CLOCK_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/clock_t.h"
+
+#else // Overlay mode
+
+#include <sys/types.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_CLOCK_T_H
diff --git a/libc/hdr/types/clockid_t.h b/libc/hdr/types/clockid_t.h
new file mode 100644
index 000000000000..333342072a2f
--- /dev/null
+++ b/libc/hdr/types/clockid_t.h
@@ -0,0 +1,22 @@
+//===-- Proxy for clockid_t -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_CLOCKID_T_H
+#define LLVM_LIBC_HDR_TYPES_CLOCKID_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/clockid_t.h"
+
+#else // Overlay mode
+
+#include <sys/types.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_CLOCKID_T_H
diff --git a/libc/hdr/types/struct_timeval.h b/libc/hdr/types/struct_timeval.h
new file mode 100644
index 000000000000..8fc321a52d71
--- /dev/null
+++ b/libc/hdr/types/struct_timeval.h
@@ -0,0 +1,21 @@
+//===-- Proxy for struct timeval  ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_HDR_TYPES_STRUCT_TIMEVAL_H
+#define LLVM_LIBC_HDR_TYPES_STRUCT_TIMEVAL_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/struct_timeval.h"
+
+#else
+
+#include <sys/time.h>
+
+#endif // LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_STRUCT_TIMEVAL_H
diff --git a/libc/hdr/types/suseconds_t.h b/libc/hdr/types/suseconds_t.h
new file mode 100644
index 000000000000..72e54a965f75
--- /dev/null
+++ b/libc/hdr/types/suseconds_t.h
@@ -0,0 +1,22 @@
+//===-- Proxy for suseconds_t ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TIMES_SUSECONDS_T_H
+#define LLVM_LIBC_HDR_TIMES_SUSECONDS_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/suseconds_t.h"
+
+#else // Overlay mode
+
+#include <sys/types.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // #ifndef LLVM_LIBC_HDR_TIMES_SUSECONDS_T_H
diff --git a/libc/hdr/types/time_t.h b/libc/hdr/types/time_t.h
new file mode 100644
index 000000000000..fc9a1506a2cd
--- /dev/null
+++ b/libc/hdr/types/time_t.h
@@ -0,0 +1,22 @@
+//===-- Proxy for time_t --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_TIME_T_H
+#define LLVM_LIBC_HDR_TYPES_TIME_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/time_t.h"
+
+#else // Overlay mode
+
+#include <time.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_TIME_T_H
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index dcae55e050bf..32d693ec6a26 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -281,3 +281,5 @@ add_subdirectory(File)
 add_subdirectory(HashTable)
 
 add_subdirectory(fixed_point)
+
+add_subdirectory(time)
diff --git a/libc/src/__support/time/CMakeLists.txt b/libc/src/__support/time/CMakeLists.txt
new file mode 100644
index 000000000000..36ce4f9dadb2
--- /dev/null
+++ b/libc/src/__support/time/CMakeLists.txt
@@ -0,0 +1,19 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${LIBC_TARGET_OS})
+endif()
+
+add_object_library(
+  clock_gettime
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.clock_gettime
+)
+
+add_header_library(
+  units
+  HDRS
+    units.h
+  DEPENDS
+    libc.src.__support.common
+    libc.hdr.types.time_t
+)
diff --git a/libc/src/__support/time/clock_gettime.h b/libc/src/__support/time/clock_gettime.h
new file mode 100644
index 000000000000..0655ccdc0028
--- /dev/null
+++ b/libc/src/__support/time/clock_gettime.h
@@ -0,0 +1,23 @@
+//===--- clock_gettime internal implementation ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_GETTIME_H
+#define LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_GETTIME_H
+#include "hdr/types/clockid_t.h"
+#include "hdr/types/struct_timespec.h"
+#include "src/__support/common.h"
+
+#include "src/__support/error_or.h"
+
+namespace LIBC_NAMESPACE {
+namespace internal {
+ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts);
+}
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_GETTIME_H
diff --git a/libc/src/__support/time/linux/CMakeLists.txt b/libc/src/__support/time/linux/CMakeLists.txt
new file mode 100644
index 000000000000..034fa317ff6d
--- /dev/null
+++ b/libc/src/__support/time/linux/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_object_library(
+  clock_gettime
+  HDRS
+    ../clock_gettime.h
+  SRCS
+    clock_gettime.cpp
+  DEPENDS
+    libc.include.sys_syscall
+    libc.hdr.types.struct_timespec
+    libc.hdr.types.clockid_t
+    libc.src.__support.common
+    libc.src.__support.error_or
+    libc.src.__support.OSUtil.osutil
+)
diff --git a/libc/src/time/linux/clockGetTimeImpl.h b/libc/src/__support/time/linux/clock_gettime.cpp
similarity index 64%
rename from libc/src/time/linux/clockGetTimeImpl.h
rename to libc/src/__support/time/linux/clock_gettime.cpp
index 8c8c9fcf845c..6a131df9ba59 100644
--- a/libc/src/time/linux/clockGetTimeImpl.h
+++ b/libc/src/__support/time/linux/clock_gettime.cpp
@@ -1,4 +1,4 @@
-//===- Linux implementation of the POSIX clock_gettime function -*- C++ -*-===//
+//===--- clock_gettime linux implementation ---------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,23 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_TIME_LINUX_CLOCKGETTIMEIMPL_H
-#define LLVM_LIBC_SRC_TIME_LINUX_CLOCKGETTIMEIMPL_H
-
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
-#include "src/__support/common.h"
-#include "src/__support/error_or.h"
-#include "src/errno/libc_errno.h"
-
-#include <stdint.h>      // For int64_t.
-#include <sys/syscall.h> // For syscall numbers.
-#include <time.h>
-
+#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
+#define LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
+#include "src/__support/time/clock_gettime.h"
+#include "src/__support/OSUtil/syscall.h"
+#include <sys/syscall.h>
 namespace LIBC_NAMESPACE {
 namespace internal {
-
-LIBC_INLINE ErrorOr<int> clock_gettimeimpl(clockid_t clockid,
-                                           struct timespec *ts) {
+ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts) {
 #if SYS_clock_gettime
   int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_clock_gettime,
                                               static_cast<long>(clockid),
@@ -45,4 +36,4 @@ LIBC_INLINE ErrorOr<int> clock_gettimeimpl(clockid_t clockid,
 } // namespace internal
 } // namespace LIBC_NAMESPACE
 
-#endif // LLVM_LIBC_SRC_TIME_LINUX_CLOCKGETTIMEIMPL_H
+#endif // LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
diff --git a/libc/src/__support/time/units.h b/libc/src/__support/time/units.h
new file mode 100644
index 000000000000..f6bd19f9b139
--- /dev/null
+++ b/libc/src/__support/time/units.h
@@ -0,0 +1,38 @@
+//===--- Time units conversion ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_UNITS_H
+#define LLVM_LIBC_SRC___SUPPORT_TIME_UNITS_H
+
+#include "hdr/types/time_t.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+namespace time_units {
+LIBC_INLINE constexpr time_t operator""_s_ns(unsigned long long s) {
+  return s * 1'000'000'000;
+}
+LIBC_INLINE constexpr time_t operator""_s_us(unsigned long long s) {
+  return s * 1'000'000;
+}
+LIBC_INLINE constexpr time_t operator""_s_ms(unsigned long long s) {
+  return s * 1'000;
+}
+LIBC_INLINE constexpr time_t operator""_ms_ns(unsigned long long ms) {
+  return ms * 1'000'000;
+}
+LIBC_INLINE constexpr time_t operator""_ms_us(unsigned long long ms) {
+  return ms * 1'000;
+}
+LIBC_INLINE constexpr time_t operator""_us_ns(unsigned long long us) {
+  return us * 1'000;
+}
+} // namespace time_units
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_TIME_UNITS_H
diff --git a/libc/src/time/clock.h b/libc/src/time/clock.h
index d4af7656644a..f5d14d036e13 100644
--- a/libc/src/time/clock.h
+++ b/libc/src/time/clock.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_TIME_CLOCK_H
 #define LLVM_LIBC_SRC_TIME_CLOCK_H
 
-#include <time.h>
+#include "hdr/types/clock_t.h"
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/time/clock_gettime.h b/libc/src/time/clock_gettime.h
index 72e2e1949feb..48e81a355429 100644
--- a/libc/src/time/clock_gettime.h
+++ b/libc/src/time/clock_gettime.h
@@ -9,11 +9,12 @@
 #ifndef LLVM_LIBC_SRC_TIME_CLOCK_GETTIME_H
 #define LLVM_LIBC_SRC_TIME_CLOCK_GETTIME_H
 
-#include <time.h>
+#include "hdr/types/clockid_t.h"
+#include "hdr/types/struct_timespec.h"
 
 namespace LIBC_NAMESPACE {
 
-int clock_gettime(clockid_t clockid, struct timespec *tp);
+int clock_gettime(clockid_t clockid, timespec *tp);
 
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/src/time/gettimeofday.h b/libc/src/time/gettimeofday.h
index 880b94cee731..62ee31edcad6 100644
--- a/libc/src/time/gettimeofday.h
+++ b/libc/src/time/gettimeofday.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_TIME_GETTIMEOFDAY_H
 #define LLVM_LIBC_SRC_TIME_GETTIMEOFDAY_H
 
-#include <time.h>
+#include "hdr/types/struct_timeval.h"
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/time/linux/CMakeLists.txt b/libc/src/time/linux/CMakeLists.txt
index df79bf598626..8a0e6b04b66e 100644
--- a/libc/src/time/linux/CMakeLists.txt
+++ b/libc/src/time/linux/CMakeLists.txt
@@ -5,9 +5,9 @@ add_entrypoint_object(
   HDRS
     ../time_func.h
   DEPENDS
-    libc.include.time
-    libc.include.sys_syscall
-    libc.src.__support.OSUtil.osutil
+    libc.hdr.time_macros
+    libc.hdr.types.time_t
+    libc.src.__support.time.clock_gettime
     libc.src.errno.errno
 )
 
@@ -18,10 +18,11 @@ add_entrypoint_object(
   HDRS
     ../clock.h
   DEPENDS
-    libc.include.time
-    libc.include.sys_syscall
+    libc.hdr.time_macros
+    libc.hdr.types.clock_t
+    libc.src.__support.time.units
+    libc.src.__support.time.clock_gettime
     libc.src.__support.CPP.limits
-    libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
 
@@ -32,10 +33,10 @@ add_entrypoint_object(
   HDRS
     ../nanosleep.h
   DEPENDS
-    libc.include.time
+    libc.hdr.types.struct_timespec
     libc.include.sys_syscall
-    libc.src.__support.CPP.limits
     libc.src.__support.OSUtil.osutil
+    libc.src.__support.CPP.limits
     libc.src.errno.errno
 )
 
@@ -46,9 +47,9 @@ add_entrypoint_object(
   HDRS
     ../clock_gettime.h
   DEPENDS
-    libc.include.time
-    libc.include.sys_syscall
-    libc.src.__support.OSUtil.osutil
+    libc.hdr.types.clockid_t
+    libc.hdr.types.struct_timespec
+    libc.src.__support.time.clock_gettime
     libc.src.errno.errno
 )
 
@@ -59,8 +60,9 @@ add_entrypoint_object(
   HDRS
     ../gettimeofday.h
   DEPENDS
-    libc.include.time
-    libc.include.sys_syscall
-    libc.src.__support.OSUtil.osutil
+    libc.hdr.time_macros
+    libc.hdr.types.suseconds_t
+    libc.src.__support.time.clock_gettime
+    libc.src.__support.time.units
     libc.src.errno.errno
 )
diff --git a/libc/src/time/linux/clock.cpp b/libc/src/time/linux/clock.cpp
index 1e95f0526bc9..fc48e2792747 100644
--- a/libc/src/time/linux/clock.cpp
+++ b/libc/src/time/linux/clock.cpp
@@ -7,21 +7,19 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/time/clock.h"
-
+#include "hdr/time_macros.h"
 #include "src/__support/CPP/limits.h"
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/time/clock_gettime.h"
+#include "src/__support/time/units.h"
 #include "src/errno/libc_errno.h"
-#include "src/time/linux/clockGetTimeImpl.h"
-
-#include <sys/syscall.h> // For syscall numbers.
-#include <time.h>
 
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(clock_t, clock, ()) {
+  using namespace time_units;
   struct timespec ts;
-  auto result = internal::clock_gettimeimpl(CLOCK_PROCESS_CPUTIME_ID, &ts);
+  auto result = internal::clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
   if (!result.has_value()) {
     libc_errno = result.error();
     return -1;
@@ -34,15 +32,15 @@ LLVM_LIBC_FUNCTION(clock_t, clock, ()) {
       cpp::numeric_limits<clock_t>::max() / CLOCKS_PER_SEC;
   if (ts.tv_sec > CLOCK_SECS_MAX)
     return clock_t(-1);
-  if (ts.tv_nsec / 1000000000 > CLOCK_SECS_MAX - ts.tv_sec)
+  if (ts.tv_nsec / 1_s_ns > CLOCK_SECS_MAX - ts.tv_sec)
     return clock_t(-1);
 
   // For the integer computation converting tv_nsec to clocks to work
   // correctly, we want CLOCKS_PER_SEC to be less than 1000000000.
-  static_assert(1000000000 > CLOCKS_PER_SEC,
-                "Expected CLOCKS_PER_SEC to be less than 1000000000.");
+  static_assert(1_s_ns > CLOCKS_PER_SEC,
+                "Expected CLOCKS_PER_SEC to be less than 1'000'000'000.");
   return clock_t(ts.tv_sec * CLOCKS_PER_SEC +
-                 ts.tv_nsec / (1000000000 / CLOCKS_PER_SEC));
+                 ts.tv_nsec / (1_s_ns / CLOCKS_PER_SEC));
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/time/linux/clock_gettime.cpp b/libc/src/time/linux/clock_gettime.cpp
index 47e974a866c8..920363e85e06 100644
--- a/libc/src/time/linux/clock_gettime.cpp
+++ b/libc/src/time/linux/clock_gettime.cpp
@@ -7,21 +7,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/time/clock_gettime.h"
-
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/time/clock_gettime.h"
 #include "src/errno/libc_errno.h"
-#include "src/time/linux/clockGetTimeImpl.h"
-
-#include <sys/syscall.h> // For syscall numbers.
-#include <time.h>
 
 namespace LIBC_NAMESPACE {
 
 // TODO(michaelrj): Move this into time/linux with the other syscalls.
 LLVM_LIBC_FUNCTION(int, clock_gettime,
                    (clockid_t clockid, struct timespec *ts)) {
-  auto result = internal::clock_gettimeimpl(clockid, ts);
+  auto result = internal::clock_gettime(clockid, ts);
 
   // A negative return value indicates an error with the magnitude of the
   // value being the error code.
diff --git a/libc/src/time/linux/gettimeofday.cpp b/libc/src/time/linux/gettimeofday.cpp
index 07ab4d579176..c7bcd45e01fa 100644
--- a/libc/src/time/linux/gettimeofday.cpp
+++ b/libc/src/time/linux/gettimeofday.cpp
@@ -7,24 +7,24 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/time/gettimeofday.h"
-
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "hdr/time_macros.h"
+#include "hdr/types/suseconds_t.h"
 #include "src/__support/common.h"
+#include "src/__support/time/clock_gettime.h"
+#include "src/__support/time/units.h"
 #include "src/errno/libc_errno.h"
-#include "src/time/linux/clockGetTimeImpl.h"
-
-#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE {
 
 // TODO(michaelrj): Move this into time/linux with the other syscalls.
 LLVM_LIBC_FUNCTION(int, gettimeofday,
                    (struct timeval * tv, [[maybe_unused]] void *unused)) {
+  using namespace time_units;
   if (tv == nullptr)
     return 0;
 
   struct timespec ts;
-  auto result = internal::clock_gettimeimpl(CLOCK_REALTIME, &ts);
+  auto result = internal::clock_gettime(CLOCK_REALTIME, &ts);
 
   // A negative return value indicates an error with the magnitude of the
   // value being the error code.
@@ -34,7 +34,7 @@ LLVM_LIBC_FUNCTION(int, gettimeofday,
   }
 
   tv->tv_sec = ts.tv_sec;
-  tv->tv_usec = static_cast<suseconds_t>(ts.tv_nsec / 1000);
+  tv->tv_usec = static_cast<suseconds_t>(ts.tv_nsec / 1_us_ns);
   return 0;
 }
 
diff --git a/libc/src/time/linux/time.cpp b/libc/src/time/linux/time.cpp
index e286fae095b2..93d5d7362764 100644
--- a/libc/src/time/linux/time.cpp
+++ b/libc/src/time/linux/time.cpp
@@ -6,22 +6,18 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/time/time_func.h"
-
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "hdr/time_macros.h"
 #include "src/__support/common.h"
+#include "src/__support/time/clock_gettime.h"
 #include "src/errno/libc_errno.h"
-#include "src/time/linux/clockGetTimeImpl.h"
-
-#include <sys/syscall.h> // For syscall numbers.
-#include <time.h>
+#include "src/time/time_func.h"
 
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(time_t, time, (time_t * tp)) {
   // TODO: Use the Linux VDSO to fetch the time and avoid the syscall.
   struct timespec ts;
-  auto result = internal::clock_gettimeimpl(CLOCK_REALTIME, &ts);
+  auto result = internal::clock_gettime(CLOCK_REALTIME, &ts);
   if (!result.has_value()) {
     libc_errno = result.error();
     return -1;
diff --git a/libc/src/time/nanosleep.h b/libc/src/time/nanosleep.h
index 757394232c07..2309666b2304 100644
--- a/libc/src/time/nanosleep.h
+++ b/libc/src/time/nanosleep.h
@@ -9,11 +9,11 @@
 #ifndef LLVM_LIBC_SRC_TIME_NANOSLEEP_H
 #define LLVM_LIBC_SRC_TIME_NANOSLEEP_H
 
-#include <time.h>
+#include "hdr/types/struct_timespec.h"
 
 namespace LIBC_NAMESPACE {
 
-int nanosleep(const struct timespec *req, struct timespec *rem);
+int nanosleep(const timespec *req, timespec *rem);
 
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/src/time/time_func.h b/libc/src/time/time_func.h
index beb02020b575..2a5239220942 100644
--- a/libc/src/time/time_func.h
+++ b/libc/src/time/time_func.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_TIME_TIME_FUNC_H
 #define LLVM_LIBC_SRC_TIME_TIME_FUNC_H
 
-#include <time.h>
+#include "hdr/types/time_t.h"
 
 // Note this header file is named time_func.h to avoid conflicts with the
 // public header file time.h.
-- 
GitLab


From ddad7c3c84a8706ad539f1a69660a31a00b411a2 Mon Sep 17 00:00:00 2001
From: Tomas Matheson <Tomas.Matheson@arm.com>
Date: Thu, 9 May 2024 21:51:39 +0100
Subject: [PATCH 0342/1206] [AArch64] add some more tests for FMV (#91490)

Add a couple of tests to make it clear:
- when FMV should be enabled and disabled by the driver.
- which extensions are enabled/disabled based on the dependencies
specified in TargetParser.
---
 clang/test/CodeGen/aarch64-fmv-dependencies.c | 240 ++++++++++++++++++
 clang/test/Driver/aarch64-fmv.c               |  27 ++
 2 files changed, 267 insertions(+)
 create mode 100644 clang/test/CodeGen/aarch64-fmv-dependencies.c
 create mode 100644 clang/test/Driver/aarch64-fmv.c

diff --git a/clang/test/CodeGen/aarch64-fmv-dependencies.c b/clang/test/CodeGen/aarch64-fmv-dependencies.c
new file mode 100644
index 000000000000..ec599e1b3fa7
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-fmv-dependencies.c
@@ -0,0 +1,240 @@
+// Test/document all of the dependencies between possible AArch64 FMV extensions.
+// Also test the name mangling.
+
+// RUN: %clang --target=aarch64-linux-gnu --rtlib=compiler-rt -emit-llvm -S -o - %s | FileCheck %s
+
+// CHECK: define dso_local i32 @fmv._Maes() #[[ATTR0:[0-9]+]] {
+__attribute__((target_version("aes"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mbf16() #[[bf16_ebf16:[0-9]+]] {
+__attribute__((target_version("bf16"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mbti() #[[bti:[0-9]+]] {
+__attribute__((target_version("bti"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mcrc() #[[crc:[0-9]+]] {
+__attribute__((target_version("crc"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mdgh() #[[ATTR0:[0-9]+]] {
+__attribute__((target_version("dgh"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mdit() #[[dit:[0-9]+]] {
+__attribute__((target_version("dit"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mdotprod() #[[dotprod:[0-9]+]] {
+__attribute__((target_version("dotprod"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mdpb() #[[dpb:[0-9]+]] {
+__attribute__((target_version("dpb"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mdpb2() #[[dpb2:[0-9]+]] {
+__attribute__((target_version("dpb2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mebf16() #[[bf16_ebf16:[0-9]+]] {
+__attribute__((target_version("ebf16"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mf32mm() #[[f32mm:[0-9]+]] {
+__attribute__((target_version("f32mm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mf64mm() #[[f64mm:[0-9]+]] {
+__attribute__((target_version("f64mm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfcma() #[[fcma:[0-9]+]] {
+__attribute__((target_version("fcma"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mflagm() #[[flagm:[0-9]+]] {
+__attribute__((target_version("flagm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mflagm2() #[[flagm2:[0-9]+]] {
+__attribute__((target_version("flagm2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfp() #[[ATTR0:[0-9]+]] {
+__attribute__((target_version("fp"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfp16() #[[fp16:[0-9]+]] {
+__attribute__((target_version("fp16"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfp16fml() #[[fp16fml:[0-9]+]] {
+__attribute__((target_version("fp16fml"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfrintts() #[[frintts:[0-9]+]] {
+__attribute__((target_version("frintts"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mi8mm() #[[i8mm:[0-9]+]] {
+__attribute__((target_version("i8mm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mjscvt() #[[jscvt:[0-9]+]] {
+__attribute__((target_version("jscvt"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mls64() #[[ATTR0:[0-9]+]] {
+__attribute__((target_version("ls64"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mls64_accdata() #[[ls64_accdata:[0-9]+]] {
+__attribute__((target_version("ls64_accdata"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mls64_v() #[[ATTR0:[0-9]+]] {
+__attribute__((target_version("ls64_v"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mlse() #[[lse:[0-9]+]] {
+__attribute__((target_version("lse"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mmemtag() #[[ATTR0:[0-9]+]] {
+__attribute__((target_version("memtag"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mmemtag2() #[[memtag2:[0-9]+]] {
+__attribute__((target_version("memtag2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mmemtag3() #[[memtag2:[0-9]+]] {
+__attribute__((target_version("memtag3"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mmops() #[[mops:[0-9]+]] {
+__attribute__((target_version("mops"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mpmull() #[[pmull:[0-9]+]] {
+__attribute__((target_version("pmull"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mpredres() #[[predres:[0-9]+]] {
+__attribute__((target_version("predres"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mrcpc() #[[rcpc:[0-9]+]] {
+__attribute__((target_version("rcpc"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mrcpc2() #[[rcpc:[0-9]+]] {
+__attribute__((target_version("rcpc2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mrcpc3() #[[rcpc3:[0-9]+]] {
+__attribute__((target_version("rcpc3"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mrdm() #[[rdm:[0-9]+]] {
+__attribute__((target_version("rdm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mrng() #[[rng:[0-9]+]] {
+__attribute__((target_version("rng"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mrpres() #[[ATTR0:[0-9]+]] {
+__attribute__((target_version("rpres"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msb() #[[sb:[0-9]+]] {
+__attribute__((target_version("sb"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msha1() #[[ATTR0:[0-9]+]] {
+__attribute__((target_version("sha1"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msha2() #[[sha2:[0-9]+]] {
+__attribute__((target_version("sha2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msha3() #[[sha3:[0-9]+]] {
+__attribute__((target_version("sha3"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msimd() #[[ATTR0:[0-9]+]] {
+__attribute__((target_version("simd"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msm4() #[[sm4:[0-9]+]] {
+__attribute__((target_version("sm4"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme() #[[sme:[0-9]+]] {
+__attribute__((target_version("sme"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-f64f64() #[[sme_f64f64:[0-9]+]] {
+__attribute__((target_version("sme-f64f64"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-i16i64() #[[sme_i16i64:[0-9]+]] {
+__attribute__((target_version("sme-i16i64"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme2() #[[sme2:[0-9]+]] {
+__attribute__((target_version("sme2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssbs() #[[ATTR0:[0-9]+]] {
+__attribute__((target_version("ssbs"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssbs2() #[[ssbs2:[0-9]+]] {
+__attribute__((target_version("ssbs2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve() #[[sve:[0-9]+]] {
+__attribute__((target_version("sve"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve-bf16() #[[sve_bf16_ebf16:[0-9]+]] {
+__attribute__((target_version("sve-bf16"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve-ebf16() #[[sve_bf16_ebf16:[0-9]+]] {
+__attribute__((target_version("sve-ebf16"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve-i8mm() #[[sve_i8mm:[0-9]+]] {
+__attribute__((target_version("sve-i8mm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve2() #[[sve2:[0-9]+]] {
+__attribute__((target_version("sve2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve2-aes() #[[sve2_aes_sve2_pmull128:[0-9]+]] {
+__attribute__((target_version("sve2-aes"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve2-bitperm() #[[sve2_bitperm:[0-9]+]] {
+__attribute__((target_version("sve2-bitperm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve2-pmull128() #[[sve2_aes_sve2_pmull128:[0-9]+]] {
+__attribute__((target_version("sve2-pmull128"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve2-sha3() #[[sve2_sha3:[0-9]+]] {
+__attribute__((target_version("sve2-sha3"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve2-sm4() #[[sve2_sm4:[0-9]+]] {
+__attribute__((target_version("sve2-sm4"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mwfxt() #[[wfxt:[0-9]+]] {
+__attribute__((target_version("wfxt"))) int fmv(void) { return 0; }
+
+// CHECK-NOT: define dso_local i32 @fmv._M{{.*}}
+__attribute__((target_version("non_existent_extension"))) int fmv(void);
+
+__attribute__((target_version("default"))) int fmv(void);
+
+int caller() {
+  return fmv();
+}
+
+// CHECK: attributes #[[ATTR0:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[bf16_ebf16:[0-9]+]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[bti:[0-9]+]] = { {{.*}} "target-features"="+bti,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[crc:[0-9]+]] = { {{.*}} "target-features"="+crc,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[dit:[0-9]+]] = { {{.*}} "target-features"="+dit,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[dotprod:[0-9]+]] = { {{.*}} "target-features"="+dotprod,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[dpb:[0-9]+]] = { {{.*}} "target-features"="+ccpp,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[dpb2:[0-9]+]] = { {{.*}} "target-features"="+ccdp,+ccpp,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[f32mm:[0-9]+]] = { {{.*}} "target-features"="+f32mm,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+v8a"
+// CHECK: attributes #[[f64mm:[0-9]+]] = { {{.*}} "target-features"="+f64mm,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+v8a"
+// CHECK: attributes #[[fcma:[0-9]+]] = { {{.*}} "target-features"="+complxnum,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[flagm:[0-9]+]] = { {{.*}} "target-features"="+flagm,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[flagm2:[0-9]+]] = { {{.*}} "target-features"="+altnzcv,+flagm,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[fp16:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[fp16fml:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+fp16fml,+fullfp16,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[frintts:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+fptoint,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[i8mm:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+i8mm,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[jscvt:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+jsconv,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[ls64_accdata:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+ls64,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[lse:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+lse,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[memtag2:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+mte,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[mops:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+mops,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[pmull:[0-9]+]] = { {{.*}} "target-features"="+aes,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[predres:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+predres,+v8a"
+// CHECK: attributes #[[rcpc:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+rcpc,+v8a"
+// CHECK: attributes #[[rcpc3:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+rcpc,+rcpc3,+v8a"
+// CHECK: attributes #[[rdm:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+rdm,+v8a"
+// CHECK: attributes #[[rng:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+rand,+v8a"
+// CHECK: attributes #[[sb:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+sb,+v8a"
+// CHECK: attributes #[[sha2:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+sha2,+v8a"
+// CHECK: attributes #[[sha3:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+sha2,+sha3,+v8a"
+// CHECK: attributes #[[sm4:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+sm4,+v8a"
+// CHECK: attributes #[[sme:[0-9]+]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+neon,+outline-atomics,+sme,+v8a"
+// CHECK: attributes #[[sme_f64f64:[0-9]+]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+neon,+outline-atomics,+sme,+sme-f64f64,+v8a"
+// CHECK: attributes #[[sme_i16i64:[0-9]+]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+neon,+outline-atomics,+sme,+sme-i16i64,+v8a"
+// CHECK: attributes #[[sme2:[0-9]+]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+neon,+outline-atomics,+sme,+sme2,+v8a"
+// CHECK: attributes #[[ssbs2:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+ssbs,+v8a"
+// CHECK: attributes #[[sve:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+v8a"
+// CHECK: attributes #[[sve_bf16_ebf16:[0-9]+]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+v8a"
+// CHECK: attributes #[[sve_i8mm:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+i8mm,+neon,+outline-atomics,+sve,+v8a"
+// CHECK: attributes #[[sve2:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve2,+v8a"
+// CHECK: attributes #[[sve2_aes_sve2_pmull128:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve2,+sve2-aes,+v8a"
+// CHECK: attributes #[[sve2_bitperm:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve2,+sve2-bitperm,+v8a"
+// CHECK: attributes #[[sve2_sha3:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve2,+sve2-sha3,+v8a"
+// CHECK: attributes #[[sve2_sm4:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve2,+sve2-sm4,+v8a"
+// CHECK: attributes #[[wfxt:[0-9]+]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+v8a,+wfxt"
diff --git a/clang/test/Driver/aarch64-fmv.c b/clang/test/Driver/aarch64-fmv.c
new file mode 100644
index 000000000000..873a88964e9b
--- /dev/null
+++ b/clang/test/Driver/aarch64-fmv.c
@@ -0,0 +1,27 @@
+// Test which driver flags enable/disable Function Multiversioning on aarch64.
+
+// FMV is enabled for non-android aarch64 targets:
+// RUN: %clang --target=aarch64 --rtlib=compiler-rt -### -c %s 2>&1 | FileCheck -check-prefix=FMV-ENABLED %s
+// RUN: %clang --target=aarch64-linux-gnu --rtlib=compiler-rt -### -c %s 2>&1 | FileCheck -check-prefix=FMV-ENABLED %s
+// RUN: %clang --target=arm64-apple-ios --rtlib=compiler-rt -### -c %s 2>&1 | FileCheck -check-prefix=FMV-ENABLED %s
+// RUN: %clang --target=arm64-apple-macosx --rtlib=compiler-rt -### -c %s 2>&1 | FileCheck -check-prefix=FMV-ENABLED %s
+
+// android23 defaults to --rtlib=compiler-rt:
+// RUN: %clang --target=aarch64-linux-android23 -### -c %s 2>&1 | FileCheck -check-prefix=FMV-ENABLED %s
+// RUN: %clang --target=aarch64-linux-android23 --rtlib=compiler-rt  -### -c %s 2>&1 | FileCheck -check-prefix=FMV-ENABLED %s
+
+// FMV is disabled without compiler-rt:
+// RUN: %clang --target=aarch64 -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
+// RUN: %clang --target=aarch64-linux-gnu -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
+
+// Disabled for older android versions:
+// RUN: %clang --rtlib=compiler-rt --target=aarch64-linux-android               -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
+// RUN: %clang --rtlib=compiler-rt --target=aarch64-linux-android22             -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
+// RUN: %clang --rtlib=compiler-rt --target=aarch64-linux-android22  -mno-fmv   -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
+
+// Disabled explicitly:
+// RUN: %clang --rtlib=compiler-rt --target=aarch64 -mno-fmv -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
+// RUN: %clang --rtlib=compiler-rt --target=aarch64-linux-android23  -mno-fmv   -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
+
+// FMV-ENABLED-NOT: "-target-feature" "-fmv"
+// FMV-DISABLED: "-target-feature" "-fmv"
-- 
GitLab


From f4a7e1f9bac1f11e1db1c0a895f3f681838f89f2 Mon Sep 17 00:00:00 2001
From: Med Ismail Bennani <ismail@bennani.ma>
Date: Thu, 9 May 2024 13:51:52 -0700
Subject: [PATCH 0343/1206] [lldb/crashlog] Fix module binary resolution
 (#91631)

This patch fixes a bug in when resolving and loading modules from the
binary image list.

When loading a module, we would first use the UUID from the binary image
list with `dsymForUUID` to fetch the dSYM bundle from our remote build
records and copy the executable locally.

If we failed to find a matching dSYM bundle for that UUID on the build
record, let's say if that module was built locally, we use Spotlight
(`mdfind`) to find the dSYM bundle once again using the UUID.

Prior to this patch, we would set the image path to be the same as the
symbol file. This resulted in trying to load the dSYM as a module in
lldb, which isn't allowed.

This patch address that by looking for a binary matching the image
identifier, next to the dSYM bundle and try to load that instead.

rdar://127433616

Signed-off-by: Med Ismail Bennani <ismail@bennani.ma>
---
 lldb/examples/python/crashlog.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py
index eb9af6ed3d95..2919b9c76e68 100755
--- a/lldb/examples/python/crashlog.py
+++ b/lldb/examples/python/crashlog.py
@@ -418,9 +418,20 @@ class CrashLog(symbolication.Symbolicator):
                         with print_lock:
                             print('falling back to binary inside "%s"' % dsym)
                         self.symfile = dsym
-                        for filename in os.listdir(dwarf_dir):
-                            self.path = os.path.join(dwarf_dir, filename)
-                            if self.find_matching_slice():
+                        # Look for the executable next to the dSYM bundle.
+                        parent_dir = os.path.dirname(dsym)
+                        executables = []
+                        for root, _, files in os.walk(parent_dir):
+                            for file in files:
+                                abs_path = os.path.join(root, file)
+                                if os.path.isfile(abs_path) and os.access(
+                                    abs_path, os.X_OK
+                                ):
+                                    executables.append(abs_path)
+                        for binary in executables:
+                            basename = os.path.basename(binary)
+                            if basename == self.identifier:
+                                self.path = binary
                                 found_matching_slice = True
                                 break
                         if found_matching_slice:
-- 
GitLab


From 639a740035b732e9bc0f43f3f95d1ce3acf82e1b Mon Sep 17 00:00:00 2001
From: Tomas Matheson <Tomas.Matheson@arm.com>
Date: Thu, 9 May 2024 21:54:48 +0100
Subject: [PATCH 0344/1206] [AArch64] move extension information into tablgen
 (#90987)

Generate TargetParser extension information from tablegen. This includes FMV extension information. FMV only extensions are represented by a separate tablegen class.

Use MArchName/ArchKindEnumSpelling to avoid renamings.
Cases where there is simply a case difference are handled by
consistently uppercasing the AEK_ name in the emitted code.

Remove some Extensions which were not needed.
These had AEK entries but were never actually used for anything.
They are not present in Extensions[] data.
---
 .../Driver/aarch64-implied-sme-features.c     |  12 +-
 .../Driver/aarch64-implied-sve-features.c     |  22 +-
 .../command-disassemble-aarch64-extensions.s  |   2 +-
 .../llvm/TargetParser/AArch64TargetParser.h   | 139 +----------
 llvm/lib/Target/AArch64/AArch64Features.td    | 228 ++++++++++++++----
 .../TargetParser/TargetParserTest.cpp         |   8 +-
 llvm/utils/TableGen/ARMTargetDefEmitter.cpp   |  59 ++++-
 7 files changed, 264 insertions(+), 206 deletions(-)

diff --git a/clang/test/Driver/aarch64-implied-sme-features.c b/clang/test/Driver/aarch64-implied-sme-features.c
index 67836f42f2c0..eca62e2563b7 100644
--- a/clang/test/Driver/aarch64-implied-sme-features.c
+++ b/clang/test/Driver/aarch64-implied-sme-features.c
@@ -14,7 +14,7 @@
 // SME-CONFLICT: "-target-feature" "-bf16"{{.*}} "-target-feature" "-sme"
 
 // RUN: %clang -target aarch64-linux-gnu -march=armv8-a+sme-i16i64 %s -### 2>&1 | FileCheck %s --check-prefix=SME-I16I64
-// SME-I16I64: "-target-feature" "+bf16"{{.*}} "-target-feature" "+sme-i16i64" "-target-feature" "+sme"
+// SME-I16I64: "-target-feature" "+bf16"{{.*}} "-target-feature" "+sme" "-target-feature" "+sme-i16i64"
 
 // RUN: %clang -target aarch64-linux-gnu -march=armv8-a+nosme-i16i64 %s -### 2>&1 | FileCheck %s --check-prefix=NOSME-I16I64
 // NOSME-I16I64-NOT: "-target-feature" "+sme-i16i64"
@@ -23,7 +23,7 @@
 // NOSME-I16I64-NOT: sme-i16i64"
 
 // RUN: %clang -target aarch64-linux-gnu -march=armv8-a+sme-i16i64+nosme-i16i64 %s -### 2>&1 | FileCheck %s --check-prefix=SME-I16I64-REVERT
-// SME-I16I64-REVERT: "-target-feature" "+bf16"{{.*}} "-target-feature" "-sme-i16i64" "-target-feature" "+sme"
+// SME-I16I64-REVERT: "-target-feature" "+bf16"{{.*}} "-target-feature" "+sme" "-target-feature" "-sme-i16i64"
 
 // RUN: %clang -target aarch64-linux-gnu -march=armv8-a+nosme-f64f64 %s -### 2>&1 | FileCheck %s --check-prefix=NOSME-F64F64
 // NOSME-F64F64-NOT: "-target-feature" "+sme-f64f64"
@@ -32,15 +32,15 @@
 // NOSME-F64F64-NOT: sme-f64f64"
 
 // RUN: %clang -target aarch64-linux-gnu -march=armv8-a+sme-f64f64+nosme-f64f64 %s -### 2>&1 | FileCheck %s --check-prefix=SME-F64F64-REVERT
-// SME-F64F64-REVERT: "-target-feature" "+bf16"{{.*}} "-target-feature" "-sme-f64f64" "-target-feature" "+sme"
+// SME-F64F64-REVERT: "-target-feature" "+bf16"{{.*}} "-target-feature" "+sme" "-target-feature" "-sme-f64f64"
 
 // RUN: %clang -target aarch64-linux-gnu -march=armv8-a+sme-f64f64+nosme-i16i64 %s -### 2>&1 | FileCheck %s --check-prefix=SME-SUBFEATURE-MIX
 // SME-SUBFEATURE-MIX-NOT: "+sme-i16i64"
-// SME-SUBFEATURE-MIX: "-target-feature" "+bf16"{{.*}} "-target-feature" "+sme-f64f64" "-target-feature" "+sme"
+// SME-SUBFEATURE-MIX: "-target-feature" "+bf16"{{.*}} "-target-feature" "+sme" "-target-feature" "+sme-f64f64"
 // SME-SUBFEATURE-MIX-NOT: "+sme-i16i64"
 
 // RUN: %clang -target aarch64-linux-gnu -march=armv8-a+sme-i16i64+nosme %s -### 2>&1 | FileCheck %s --check-prefix=SME-SUBFEATURE-CONFLICT1
-// SME-SUBFEATURE-CONFLICT1: "-target-feature" "+bf16"{{.*}} "-target-feature" "-sme-i16i64" "-target-feature" "-sme"
+// SME-SUBFEATURE-CONFLICT1: "-target-feature" "+bf16"{{.*}} "-target-feature" "-sme" "-target-feature" "-sme-i16i64"
 
 // RUN: %clang -target aarch64-linux-gnu -march=armv8-a+sme-f64f64+nobf16 %s -### 2>&1 | FileCheck %s --check-prefix=SME-SUBFEATURE-CONFLICT2
 // SME-SUBFEATURE-CONFLICT2-NOT: "-target-feature" "+bf16"
@@ -48,4 +48,4 @@
 // SME-SUBFEATURE-CONFLICT2-NOT: "-target-feature" "+sme-f64f64"
 
 // RUN: %clang -target aarch64-linux-gnu -march=armv8-a+nosme+sme-i16i64 %s -### 2>&1 | FileCheck %s --check-prefix=SME-SUBFEATURE-CONFLICT-REV
-// SME-SUBFEATURE-CONFLICT-REV: "-target-feature" "+bf16"{{.*}} "-target-feature" "+sme-i16i64" "-target-feature" "+sme"
+// SME-SUBFEATURE-CONFLICT-REV: "-target-feature" "+bf16"{{.*}} "-target-feature" "+sme" "-target-feature" "+sme-i16i64"
diff --git a/clang/test/Driver/aarch64-implied-sve-features.c b/clang/test/Driver/aarch64-implied-sve-features.c
index 9227cd4981c2..f04e1a785673 100644
--- a/clang/test/Driver/aarch64-implied-sve-features.c
+++ b/clang/test/Driver/aarch64-implied-sve-features.c
@@ -24,7 +24,7 @@
 // SVE-SVE2: "-target-feature" "+sve" "-target-feature" "+sve2"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-bitperm %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-BITPERM
-// SVE2-BITPERM: "-target-feature" "+sve" "-target-feature" "+sve2-bitperm" "-target-feature" "+sve2"
+// SVE2-BITPERM: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "+sve2-bitperm"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+nosve2-bitperm %s -### 2>&1 | FileCheck %s --check-prefix=NOSVE2-BITPERM
 // NOSVE2-BITPERM-NOT: "-target-feature" "+sve2-bitperm"
@@ -33,32 +33,32 @@
 // NOSVE2-BITPERM-NOT: sve2-bitperm"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-bitperm+nosve2-bitperm %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-BITPERM-REVERT
-// SVE2-BITPERM-REVERT: "-target-feature" "+sve" "-target-feature" "-sve2-bitperm" "-target-feature" "+sve2"
+// SVE2-BITPERM-REVERT: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "-sve2-bitperm"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-aes+nosve2-aes %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-AES-REVERT
-// SVE2-AES-REVERT: "-target-feature" "+sve" "-target-feature" "-sve2-aes" "-target-feature" "+sve2"
+// SVE2-AES-REVERT: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "-sve2-aes"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sha3+nosve2-sha3 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SHA3-REVERT
-// SVE2-SHA3-REVERT: "-target-feature" "+sve" "-target-feature" "-sve2-sha3" "-target-feature" "+sve2"
+// SVE2-SHA3-REVERT: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "-sve2-sha3"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sm4+nosve2-sm4 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SM4-REVERT
-// SVE2-SM4-REVERT: "-target-feature" "+sve" "-target-feature" "-sve2-sm4" "-target-feature" "+sve2"
+// SVE2-SM4-REVERT: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "-sve2-sm4"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sha3 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SHA3
-// SVE2-SHA3: "-target-feature" "+sve" "-target-feature" "+sve2-sha3" "-target-feature" "+sve2"
+// SVE2-SHA3: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "+sve2-sha3"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-aes %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-AES
-// SVE2-AES: "-target-feature" "+sve" "-target-feature" "+sve2-aes" "-target-feature" "+sve2"
+// SVE2-AES: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "+sve2-aes"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sm4 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SM4
-// SVE2-SM4: "-target-feature" "+sve" "-target-feature" "+sve2-sm4" "-target-feature" "+sve2"
+// SVE2-SM4: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "+sve2-sm4"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-bitperm+nosve2-aes %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SUBFEATURE-MIX
-// SVE2-SUBFEATURE-MIX: "-target-feature" "+sve" "-target-feature" "+sve2-bitperm" "-target-feature" "+sve2"
+// SVE2-SUBFEATURE-MIX: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "+sve2-bitperm"
 // SVE2-SUBFEATURE-NOT: sve2-aes
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sm4+nosve2 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SUBFEATURE-CONFLICT
-// SVE2-SUBFEATURE-CONFLICT: "-target-feature" "+sve" "-target-feature" "-sve2-sm4" "-target-feature" "-sve2"
+// SVE2-SUBFEATURE-CONFLICT: "-target-feature" "+sve" "-target-feature" "-sve2" "-target-feature" "-sve2-sm4"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-aes+nosve %s -### 2>&1 | FileCheck %s --check-prefix=SVE-SUBFEATURE-CONFLICT
 // SVE-SUBFEATURE-CONFLICT-NOT: "-target-feature" "+sve2-aes"
@@ -66,7 +66,7 @@
 // SVE-SUBFEATURE-CONFLICT-NOT: "-target-feature" "+sve"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+nosve+sve2-aes %s -### 2>&1 | FileCheck %s --check-prefix=SVE-SUBFEATURE-CONFLICT-REV
-// SVE-SUBFEATURE-CONFLICT-REV: "-target-feature" "+sve" "-target-feature" "+sve2-aes" "-target-feature" "+sve2"
+// SVE-SUBFEATURE-CONFLICT-REV: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "+sve2-aes"
 
 // RUN: %clang --target=aarch64-linux-gnu -mcpu=neoverse-n2+nosve2 %s -### 2>&1 | FileCheck %s --check-prefix=SVE-MCPU-FEATURES
 // SVE-MCPU-FEATURES-NOT: "-target-feature" "+sve2-bitperm"
diff --git a/lldb/test/Shell/Commands/command-disassemble-aarch64-extensions.s b/lldb/test/Shell/Commands/command-disassemble-aarch64-extensions.s
index e154f544e7cc..685d0a84ec28 100644
--- a/lldb/test/Shell/Commands/command-disassemble-aarch64-extensions.s
+++ b/lldb/test/Shell/Commands/command-disassemble-aarch64-extensions.s
@@ -59,7 +59,7 @@ fn:
   bdep z0.b, z1.b, z31.b                // AEK_SVE2BITPERM
   rax1 z0.d, z0.d, z0.d                 // AEK_SVE2SHA3
   sm4e z0.s, z0.s, z0.s                 // AEK_SVE2SM4
-  addqv   v0.8h, p0, z0.h               // AEK_SVE2p1 / AEK_SME2p1
+  addqv   v0.8h, p0, z0.h               // AEK_SVE2P1 / AEK_SME2P1
   rcwswp x0, x1, [x2]                   // AEK_THE
   tcommit                               // AEK_TME
 lbl:
diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index 04fbaf07adfb..1124420daf8d 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -104,29 +104,9 @@ static_assert(FEAT_MAX < 62,
               "Number of features in CPUFeatures are limited to 62 entries");
 
 // Each ArchExtKind correponds directly to a possible -target-feature.
-enum ArchExtKind : unsigned {
-  AEK_NONE = 1,
-#define ARM_EXTENSION(NAME, ENUM) ENUM,
+#define EMIT_ARCHEXTKIND_ENUM
 #include "llvm/TargetParser/AArch64TargetParserDef.inc"
-  AEK_NUM_EXTENSIONS,
-
-  // FIXME temporary fixes for inconsistent naming.
-  AEK_F32MM = AEK_MATMULFP32,
-  AEK_F64MM = AEK_MATMULFP64,
-  AEK_FCMA = AEK_COMPLXNUM,
-  AEK_FP = AEK_FPARMV8,
-  AEK_FP16 = AEK_FULLFP16,
-  AEK_I8MM = AEK_MATMULINT8,
-  AEK_JSCVT = AEK_JS,
-  AEK_PROFILE = AEK_SPE,
-  AEK_RASv2 = AEK_RASV2,
-  AEK_RAND = AEK_RANDGEN,
-  AEK_SIMD = AEK_NEON,
-  AEK_SME2p1 = AEK_SME2P1,
-  AEK_SVE2p1 = AEK_SVE2P1,
-  AEK_SME_LUTv2 = AEK_SME_LUTV2,
 
-};
 using ExtensionBitset = Bitset<AEK_NUM_EXTENSIONS>;
 
 // Represents an extension that can be enabled with -march=<arch>+<extension>.
@@ -148,111 +128,8 @@ struct ExtensionInfo {
       1000; // Maximum priority for FMV feature
 };
 
-// NOTE: If adding a new extension here, consider adding it to ExtensionMap
-// in AArch64AsmParser too, if supported as an extension name by binutils.
-// clang-format off
-inline constexpr ExtensionInfo Extensions[] = {
-    {"aes", AArch64::AEK_AES, "+aes", "-aes", FEAT_AES, "+fp-armv8,+neon", 150},
-    {"b16b16", AArch64::AEK_B16B16, "+b16b16", "-b16b16", FEAT_INIT, "", 0},
-    {"bf16", AArch64::AEK_BF16, "+bf16", "-bf16", FEAT_BF16, "+bf16", 280},
-    {"brbe", AArch64::AEK_BRBE, "+brbe", "-brbe", FEAT_INIT, "", 0},
-    {"bti", AArch64::AEK_NONE, {}, {}, FEAT_BTI, "+bti", 510},
-    {"crc", AArch64::AEK_CRC, "+crc", "-crc", FEAT_CRC, "+crc", 110},
-    {"crypto", AArch64::AEK_CRYPTO, "+crypto", "-crypto", FEAT_INIT, "+aes,+sha2", 0},
-    {"cssc", AArch64::AEK_CSSC, "+cssc", "-cssc", FEAT_INIT, "", 0},
-    {"d128", AArch64::AEK_D128, "+d128", "-d128", FEAT_INIT, "", 0},
-    {"dgh", AArch64::AEK_NONE, {}, {}, FEAT_DGH, "", 260},
-    {"dit", AArch64::AEK_NONE, {}, {}, FEAT_DIT, "+dit", 180},
-    {"dotprod", AArch64::AEK_DOTPROD, "+dotprod", "-dotprod", FEAT_DOTPROD, "+dotprod,+fp-armv8,+neon", 104},
-    {"dpb", AArch64::AEK_NONE, {}, {}, FEAT_DPB, "+ccpp", 190},
-    {"dpb2", AArch64::AEK_NONE, {}, {}, FEAT_DPB2, "+ccpp,+ccdp", 200},
-    {"ebf16", AArch64::AEK_NONE, {}, {}, FEAT_EBF16, "+bf16", 290},
-    {"f32mm", AArch64::AEK_F32MM, "+f32mm", "-f32mm", FEAT_SVE_F32MM, "+sve,+f32mm,+fullfp16,+fp-armv8,+neon", 350},
-    {"f64mm", AArch64::AEK_F64MM, "+f64mm", "-f64mm", FEAT_SVE_F64MM, "+sve,+f64mm,+fullfp16,+fp-armv8,+neon", 360},
-    {"fcma", AArch64::AEK_FCMA, "+complxnum", "-complxnum", FEAT_FCMA, "+fp-armv8,+neon,+complxnum", 220},
-    {"flagm", AArch64::AEK_FLAGM, "+flagm", "-flagm", FEAT_FLAGM, "+flagm", 20},
-    {"flagm2", AArch64::AEK_NONE, {}, {}, FEAT_FLAGM2, "+flagm,+altnzcv", 30},
-    {"fp", AArch64::AEK_FP, "+fp-armv8", "-fp-armv8", FEAT_FP, "+fp-armv8,+neon", 90},
-    {"fp16", AArch64::AEK_FP16, "+fullfp16", "-fullfp16", FEAT_FP16, "+fullfp16,+fp-armv8,+neon", 170},
-    {"fp16fml", AArch64::AEK_FP16FML, "+fp16fml", "-fp16fml", FEAT_FP16FML, "+fp16fml,+fullfp16,+fp-armv8,+neon", 175},
-    {"frintts", AArch64::AEK_NONE, {}, {}, FEAT_FRINTTS, "+fptoint", 250},
-    {"hbc", AArch64::AEK_HBC, "+hbc", "-hbc", FEAT_INIT, "", 0},
-    {"i8mm", AArch64::AEK_I8MM, "+i8mm", "-i8mm", FEAT_I8MM, "+i8mm", 270},
-    {"ite", AArch64::AEK_ITE, "+ite", "-ite", FEAT_INIT, "", 0},
-    {"jscvt", AArch64::AEK_JSCVT, "+jsconv", "-jsconv", FEAT_JSCVT, "+fp-armv8,+neon,+jsconv", 210},
-    {"ls64_accdata", AArch64::AEK_NONE, {}, {}, FEAT_LS64_ACCDATA, "+ls64", 540},
-    {"ls64_v", AArch64::AEK_NONE, {}, {}, FEAT_LS64_V, "", 530},
-    {"ls64", AArch64::AEK_LS64, "+ls64", "-ls64", FEAT_LS64, "", 520},
-    {"lse", AArch64::AEK_LSE, "+lse", "-lse", FEAT_LSE, "+lse", 80},
-    {"lse128", AArch64::AEK_LSE128, "+lse128", "-lse128", FEAT_INIT, "", 0},
-    {"memtag", AArch64::AEK_MTE, "+mte", "-mte", FEAT_MEMTAG, "", 440},
-    {"memtag2", AArch64::AEK_NONE, {}, {}, FEAT_MEMTAG2, "+mte", 450},
-    {"memtag3", AArch64::AEK_NONE, {}, {}, FEAT_MEMTAG3, "+mte", 460},
-    {"mops", AArch64::AEK_MOPS, "+mops", "-mops", FEAT_MOPS, "+mops", 650},
-    {"pauth", AArch64::AEK_PAUTH, "+pauth", "-pauth", FEAT_INIT, "", 0},
-    {"pmull", AArch64::AEK_NONE, {}, {}, FEAT_PMULL, "+aes,+fp-armv8,+neon", 160},
-    {"pmuv3", AArch64::AEK_PERFMON, "+perfmon", "-perfmon", FEAT_INIT, "", 0},
-    {"predres", AArch64::AEK_PREDRES, "+predres", "-predres", FEAT_PREDRES, "+predres", 480},
-    {"predres2", AArch64::AEK_SPECRES2, "+specres2", "-specres2", FEAT_INIT, "", 0},
-    {"profile", AArch64::AEK_PROFILE, "+spe", "-spe", FEAT_INIT, "", 0},
-    {"ras", AArch64::AEK_RAS, "+ras", "-ras", FEAT_INIT, "", 0},
-    {"rasv2", AArch64::AEK_RASv2, "+rasv2", "-rasv2", FEAT_INIT, "", 0},
-    {"rcpc", AArch64::AEK_RCPC, "+rcpc", "-rcpc", FEAT_RCPC, "+rcpc", 230},
-    {"rcpc2", AArch64::AEK_NONE, {}, {}, FEAT_RCPC2, "+rcpc", 240},
-    {"rcpc3", AArch64::AEK_RCPC3, "+rcpc3", "-rcpc3", FEAT_RCPC3, "+rcpc,+rcpc3", 241},
-    {"rdm", AArch64::AEK_RDM, "+rdm", "-rdm", FEAT_RDM, "+rdm,+fp-armv8,+neon", 108},
-    {"rng", AArch64::AEK_RAND, "+rand", "-rand", FEAT_RNG, "+rand", 10},
-    {"rpres", AArch64::AEK_NONE, {}, {}, FEAT_RPRES, "", 300},
-    {"sb", AArch64::AEK_SB, "+sb", "-sb", FEAT_SB, "+sb", 470},
-    {"sha1", AArch64::AEK_NONE, {}, {}, FEAT_SHA1, "+fp-armv8,+neon", 120},
-    {"sha2", AArch64::AEK_SHA2, "+sha2", "-sha2", FEAT_SHA2, "+sha2,+fp-armv8,+neon", 130},
-    {"sha3", AArch64::AEK_SHA3, "+sha3", "-sha3", FEAT_SHA3, "+sha3,+sha2,+fp-armv8,+neon", 140},
-    {"simd", AArch64::AEK_SIMD, "+neon", "-neon", FEAT_SIMD, "+fp-armv8,+neon", 100},
-    {"sm4", AArch64::AEK_SM4, "+sm4", "-sm4", FEAT_SM4, "+sm4,+fp-armv8,+neon", 106},
-    {"sme-f16f16", AArch64::AEK_SMEF16F16, "+sme-f16f16", "-sme-f16f16", FEAT_INIT, "+sme2,+sme-f16f16", 0},
-    {"sme-f64f64", AArch64::AEK_SMEF64F64, "+sme-f64f64", "-sme-f64f64", FEAT_SME_F64, "+sme,+sme-f64f64,+bf16", 560},
-    {"sme-i16i64", AArch64::AEK_SMEI16I64, "+sme-i16i64", "-sme-i16i64", FEAT_SME_I64, "+sme,+sme-i16i64,+bf16", 570},
-    {"sme", AArch64::AEK_SME, "+sme", "-sme", FEAT_SME, "+sme,+bf16", 430},
-    {"sme2", AArch64::AEK_SME2, "+sme2", "-sme2", FEAT_SME2, "+sme2,+sme,+bf16", 580},
-    {"sme2p1", AArch64::AEK_SME2p1, "+sme2p1", "-sme2p1", FEAT_INIT, "+sme2p1,+sme2,+sme,+bf16", 0},
-    {"ssbs", AArch64::AEK_SSBS, "+ssbs", "-ssbs", FEAT_SSBS, "", 490},
-    {"ssbs2", AArch64::AEK_NONE, {}, {}, FEAT_SSBS2, "+ssbs", 500},
-    {"sve-bf16", AArch64::AEK_NONE, {}, {}, FEAT_SVE_BF16, "+sve,+bf16,+fullfp16,+fp-armv8,+neon", 320},
-    {"sve-ebf16", AArch64::AEK_NONE, {}, {}, FEAT_SVE_EBF16, "+sve,+bf16,+fullfp16,+fp-armv8,+neon", 330},
-    {"sve-i8mm", AArch64::AEK_NONE, {}, {}, FEAT_SVE_I8MM, "+sve,+i8mm,+fullfp16,+fp-armv8,+neon", 340},
-    {"sve", AArch64::AEK_SVE, "+sve", "-sve", FEAT_SVE, "+sve,+fullfp16,+fp-armv8,+neon", 310},
-    {"sve2-aes", AArch64::AEK_SVE2AES, "+sve2-aes", "-sve2-aes", FEAT_SVE_AES, "+sve2,+sve,+sve2-aes,+fullfp16,+fp-armv8,+neon", 380},
-    {"sve2-bitperm", AArch64::AEK_SVE2BITPERM, "+sve2-bitperm", "-sve2-bitperm", FEAT_SVE_BITPERM, "+sve2,+sve,+sve2-bitperm,+fullfp16,+fp-armv8,+neon", 400},
-    {"sve2-pmull128", AArch64::AEK_NONE, {}, {}, FEAT_SVE_PMULL128, "+sve2,+sve,+sve2-aes,+fullfp16,+fp-armv8,+neon", 390},
-    {"sve2-sha3", AArch64::AEK_SVE2SHA3, "+sve2-sha3", "-sve2-sha3", FEAT_SVE_SHA3, "+sve2,+sve,+sve2-sha3,+fullfp16,+fp-armv8,+neon", 410},
-    {"sve2-sm4", AArch64::AEK_SVE2SM4, "+sve2-sm4", "-sve2-sm4", FEAT_SVE_SM4, "+sve2,+sve,+sve2-sm4,+fullfp16,+fp-armv8,+neon", 420},
-    {"sve2", AArch64::AEK_SVE2, "+sve2", "-sve2", FEAT_SVE2, "+sve2,+sve,+fullfp16,+fp-armv8,+neon", 370},
-    {"sve2p1", AArch64::AEK_SVE2p1, "+sve2p1", "-sve2p1", FEAT_INIT, "+sve2p1,+sve2,+sve,+fullfp16,+fp-armv8,+neon", 0},
-    {"the", AArch64::AEK_THE, "+the", "-the", FEAT_INIT, "", 0},
-    {"tme", AArch64::AEK_TME, "+tme", "-tme", FEAT_INIT, "", 0},
-    {"wfxt", AArch64::AEK_NONE, {}, {}, FEAT_WFXT, "+wfxt", 550},
-    {"gcs", AArch64::AEK_GCS, "+gcs", "-gcs", FEAT_INIT, "", 0},
-    {"fpmr", AArch64::AEK_FPMR, "+fpmr", "-fpmr", FEAT_INIT, "", 0},
-    {"fp8", AArch64::AEK_FP8, "+fp8", "-fp8", FEAT_INIT, "+fpmr", 0},
-    {"faminmax", AArch64::AEK_FAMINMAX, "+faminmax", "-faminmax", FEAT_INIT, "", 0},
-    {"fp8fma", AArch64::AEK_FP8FMA, "+fp8fma", "-fp8fma", FEAT_INIT, "+fpmr", 0},
-    {"ssve-fp8fma", AArch64::AEK_SSVE_FP8FMA, "+ssve-fp8fma", "-ssve-fp8fma", FEAT_INIT, "+sme2", 0},
-    {"fp8dot2", AArch64::AEK_FP8DOT2, "+fp8dot2", "-fp8dot2", FEAT_INIT, "", 0},
-    {"ssve-fp8dot2", AArch64::AEK_SSVE_FP8DOT2, "+ssve-fp8dot2", "-ssve-fp8dot2", FEAT_INIT, "+sme2", 0},
-    {"fp8dot4", AArch64::AEK_FP8DOT4, "+fp8dot4", "-fp8dot4", FEAT_INIT, "", 0},
-    {"ssve-fp8dot4", AArch64::AEK_SSVE_FP8DOT4, "+ssve-fp8dot4", "-ssve-fp8dot4", FEAT_INIT, "+sme2", 0},
-    {"lut", AArch64::AEK_LUT, "+lut", "-lut", FEAT_INIT, "", 0},
-    {"sme-lutv2", AArch64::AEK_SME_LUTv2, "+sme-lutv2", "-sme-lutv2", FEAT_INIT, "", 0},
-    {"sme-f8f16", AArch64::AEK_SMEF8F16, "+sme-f8f16", "-sme-f8f16", FEAT_INIT, "+fp8,+sme2", 0},
-    {"sme-f8f32", AArch64::AEK_SMEF8F32, "+sme-f8f32", "-sme-f8f32", FEAT_INIT, "+sme2,+fp8", 0},
-    {"sme-fa64",  AArch64::AEK_SMEFA64,  "+sme-fa64", "-sme-fa64",  FEAT_INIT, "", 0},
-    {"cpa", AArch64::AEK_CPA, "+cpa", "-cpa", FEAT_INIT, "", 0},
-    {"pauth-lr", AArch64::AEK_PAUTHLR, "+pauth-lr", "-pauth-lr", FEAT_INIT, "", 0},
-    {"tlbiw", AArch64::AEK_TLBIW, "+tlbiw", "-tlbiw", FEAT_INIT, "", 0},
-    // Special cases
-    {"none", AArch64::AEK_NONE, {}, {}, FEAT_INIT, "", ExtensionInfo::MaxFMVPriority},
-};
-// clang-format on
+#define EMIT_EXTENSIONS
+#include "llvm/TargetParser/AArch64TargetParserDef.inc"
 
 struct ExtensionSet {
   // Set of extensions which are currently enabled.
@@ -328,7 +205,7 @@ inline constexpr ExtensionDependency ExtensionDependencies[] = {
   {AEK_SVE, AEK_SVE2},
   {AEK_SVE, AEK_F32MM},
   {AEK_SVE, AEK_F64MM},
-  {AEK_SVE2, AEK_SVE2p1},
+  {AEK_SVE2, AEK_SVE2P1},
   {AEK_SVE2, AEK_SVE2BITPERM},
   {AEK_SVE2, AEK_SVE2AES},
   {AEK_SVE2, AEK_SVE2SHA3},
@@ -340,7 +217,7 @@ inline constexpr ExtensionDependency ExtensionDependencies[] = {
   {AEK_SME, AEK_SMEF64F64},
   {AEK_SME, AEK_SMEI16I64},
   {AEK_SME, AEK_SMEFA64},
-  {AEK_SME2, AEK_SME2p1},
+  {AEK_SME2, AEK_SME2P1},
   {AEK_SME2, AEK_SSVE_FP8FMA},
   {AEK_SME2, AEK_SSVE_FP8DOT2},
   {AEK_SME2, AEK_SSVE_FP8DOT4},
@@ -350,7 +227,7 @@ inline constexpr ExtensionDependency ExtensionDependencies[] = {
   {AEK_FP8, AEK_SMEF8F32},
   {AEK_LSE, AEK_LSE128},
   {AEK_PREDRES, AEK_SPECRES2},
-  {AEK_RAS, AEK_RASv2},
+  {AEK_RAS, AEK_RASV2},
   {AEK_RCPC, AEK_RCPC3},
 };
 // clang-format on
@@ -429,7 +306,7 @@ inline constexpr ArchInfo ARMV8_7A  = { VersionTuple{8, 7}, AProfile, "armv8.7-a
 inline constexpr ArchInfo ARMV8_8A  = { VersionTuple{8, 8}, AProfile, "armv8.8-a", "+v8.8a", (ARMV8_7A.DefaultExts |
                                         AArch64::ExtensionBitset({AArch64::AEK_MOPS, AArch64::AEK_HBC}))};
 inline constexpr ArchInfo ARMV8_9A  = { VersionTuple{8, 9}, AProfile, "armv8.9-a", "+v8.9a", (ARMV8_8A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_SPECRES2, AArch64::AEK_CSSC, AArch64::AEK_RASv2}))};
+                                        AArch64::ExtensionBitset({AArch64::AEK_SPECRES2, AArch64::AEK_CSSC, AArch64::AEK_RASV2}))};
 inline constexpr ArchInfo ARMV9A    = { VersionTuple{9, 0}, AProfile, "armv9-a", "+v9a", (ARMV8_5A.DefaultExts |
                                         AArch64::ExtensionBitset({AArch64::AEK_FP16, AArch64::AEK_SVE, AArch64::AEK_SVE2}))};
 inline constexpr ArchInfo ARMV9_1A  = { VersionTuple{9, 1}, AProfile, "armv9.1-a", "+v9.1a", (ARMV9A.DefaultExts |
@@ -438,7 +315,7 @@ inline constexpr ArchInfo ARMV9_2A  = { VersionTuple{9, 2}, AProfile, "armv9.2-a
 inline constexpr ArchInfo ARMV9_3A  = { VersionTuple{9, 3}, AProfile, "armv9.3-a", "+v9.3a", (ARMV9_2A.DefaultExts |
                                         AArch64::ExtensionBitset({AArch64::AEK_MOPS, AArch64::AEK_HBC}))};
 inline constexpr ArchInfo ARMV9_4A  = { VersionTuple{9, 4}, AProfile, "armv9.4-a", "+v9.4a", (ARMV9_3A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_SPECRES2, AArch64::AEK_CSSC, AArch64::AEK_RASv2}))};
+                                        AArch64::ExtensionBitset({AArch64::AEK_SPECRES2, AArch64::AEK_CSSC, AArch64::AEK_RASV2}))};
 inline constexpr ArchInfo ARMV9_5A  = { VersionTuple{9, 5}, AProfile, "armv9.5-a", "+v9.5a", (ARMV9_4A.DefaultExts |
                                         AArch64::ExtensionBitset({AArch64::AEK_CPA}))};
 // For v8-R, we do not enable crypto and align with GCC that enables a more minimal set of optional architecture extensions.
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index b6c8e5f16089..ddc324ea14ed 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -11,42 +11,123 @@
 
 // A SubtargetFeature that can be toggled from the command line, and therefore
 // has an AEK_* entry in ArmExtKind.
+//
+// If Function MultiVersioning (FMV) properties are left at their defaults
+// (FEAT_INIT, no dependencies, priority 0) it indiates that this extension is
+// not an FMV feature, but can be enabled via the command line (-march, -mcpu,
+// etc).
+//
+// Conversely if the ArchExtKindSpelling is set to AEK_NONE, this indicates
+// that a feature is FMV-only, and can not be selected on the command line.
+// Such extensions should be added via FMVOnlyExtension.
 class Extension<
-  string TargetFeatureName,            // String used for -target-feature.
+  string TargetFeatureName,            // String used for -target-feature and -march, unless overridden.
   string Spelling,                     // The XYZ in HasXYZ and AEK_XYZ.
   string Desc,                         // Description.
-  list<SubtargetFeature> Implies = []  // List of dependent features.
+  list<SubtargetFeature> Implies = [], // List of dependent features.
+  // FMV properties
+  string _FMVBit = "FEAT_INIT",        // FEAT_INIT is repurposed to indicate "not an FMV feature"
+  string _FMVDependencies = "",
+  int _FMVPriority = 0
 > : SubtargetFeature<TargetFeatureName, "Has" # Spelling, "true", Desc, Implies>
 {
     string ArchExtKindSpelling = "AEK_" # Spelling; // ArchExtKind enum name.
+
+    // In general, the name written on the command line should match the name
+    // used for -target-feature. However, there are exceptions. Therefore we
+    // add a separate field for this, to allow overriding it. Strongly prefer
+    // not doing so.
+    string MArchName = TargetFeatureName;
+
+    // Function MultiVersioning (FMV) properties
+
+    // A C++ expression giving the number of the bit in the FMV ABI.
+    // Currently this is given as a value from the enum "CPUFeatures".
+    // If this is not set, it indicates that this is not an FMV extension.
+    string FMVBit = _FMVBit;
+
+    // List of features that this feature depends on.
+    // FIXME generate this from Implies.
+    string FMVDependencies = _FMVDependencies;
+
+    // The FMV priority
+    int FMVPriority = _FMVPriority;
+}
+
+// Some extensions are available for FMV but can not be controlled via the
+// command line. These entries:
+//  - are SubtargetFeatures, so they have (unused) FieldNames on the subtarget
+//    e.g. HasFMVOnlyFEAT_XYZ
+//  - have incorrect (empty) Implies fields, because the code that handles FMV
+//    ignores these dependencies and looks only at FMVDependencies.
+//  - have no description.
+// 
+// In the generated data structures for extensions (ExtensionInfo), AEK_NONE is
+// used to indicate that a feature is FMV only. Therefore ArchExtKindSpelling is
+// manually overridden here.
+class FMVOnlyExtension<string FMVBit, string Name, string Deps, int Priority>
+  : Extension<Name, "FMVOnly"#FMVBit, "", [], FMVBit, Deps, Priority> {
+    let ArchExtKindSpelling = "AEK_NONE"; // AEK_NONE indicates FMV-only feature
 }
 
+def : FMVOnlyExtension<"FEAT_DGH", "dgh", "", 260>;
+def : FMVOnlyExtension<"FEAT_DIT", "dit", "+dit", 180>;
+def : FMVOnlyExtension<"FEAT_DPB", "dpb", "+ccpp", 190>;
+def : FMVOnlyExtension<"FEAT_DPB2", "dpb2", "+ccpp,+ccdp", 200>;
+def : FMVOnlyExtension<"FEAT_EBF16", "ebf16", "+bf16", 290>;
+def : FMVOnlyExtension<"FEAT_FLAGM2", "flagm2", "+flagm,+altnzcv", 30>;
+def : FMVOnlyExtension<"FEAT_FRINTTS", "frintts", "+fptoint", 250>;
+def : FMVOnlyExtension<"FEAT_LS64_ACCDATA", "ls64_accdata", "+ls64", 540>;
+def : FMVOnlyExtension<"FEAT_LS64_V", "ls64_v", "", 530>;
+def : FMVOnlyExtension<"FEAT_MEMTAG2", "memtag2", "+mte", 450>;
+def : FMVOnlyExtension<"FEAT_MEMTAG3", "memtag3", "+mte", 460>;
+def : FMVOnlyExtension<"FEAT_PMULL", "pmull", "+aes,+fp-armv8,+neon", 160>;
+def : FMVOnlyExtension<"FEAT_RCPC2", "rcpc2", "+rcpc", 240>;
+def : FMVOnlyExtension<"FEAT_RPRES", "rpres", "", 300>;
+def : FMVOnlyExtension<"FEAT_SHA1", "sha1", "+fp-armv8,+neon", 120>;
+def : FMVOnlyExtension<"FEAT_SSBS2", "ssbs2", "+ssbs", 500>;
+def : FMVOnlyExtension<"FEAT_SVE_BF16", "sve-bf16", "+sve,+bf16,+fullfp16,+fp-armv8,+neon", 320>;
+def : FMVOnlyExtension<"FEAT_SVE_EBF16", "sve-ebf16", "+sve,+bf16,+fullfp16,+fp-armv8,+neon", 330>;
+def : FMVOnlyExtension<"FEAT_SVE_I8MM", "sve-i8mm", "+sve,+i8mm,+fullfp16,+fp-armv8,+neon", 340>;
+def : FMVOnlyExtension<"FEAT_SVE_PMULL128", "sve2-pmull128", "+sve2,+sve,+sve2-aes,+fullfp16,+fp-armv8,+neon", 390>;
+def : FMVOnlyExtension<"FEAT_WFXT", "wfxt", "+wfxt", 550>;
+
+
 // Each SubtargetFeature which corresponds to an Arm Architecture feature should
 // be annotated with the respective FEAT_ feature name from the Architecture
 // Reference Manual. If a SubtargetFeature enables instructions from multiple
 // Arm Architecture Features, it should list all the relevant features. Not all
 // FEAT_ features have a corresponding SubtargetFeature.
 
-def FeatureFPARMv8 : Extension<"fp-armv8", "FPARMv8", "Enable ARMv8 (FEAT_FP)">;
+let ArchExtKindSpelling = "AEK_FP", MArchName = "fp" in
+def FeatureFPARMv8 : Extension<"fp-armv8", "FPARMv8",
+  "Enable ARMv8 (FEAT_FP)", [],
+  "FEAT_FP", "+fp-armv8,+neon", 90>;
 
+let ArchExtKindSpelling = "AEK_SIMD", MArchName = "simd" in
 def FeatureNEON : Extension<"neon", "NEON",
-  "Enable Advanced SIMD instructions (FEAT_AdvSIMD)", [FeatureFPARMv8]>;
+  "Enable Advanced SIMD instructions (FEAT_AdvSIMD)", [FeatureFPARMv8],
+  "FEAT_SIMD", "+fp-armv8,+neon", 100>;
 
 def FeatureSM4 : Extension<
     "sm4", "SM4",
-    "Enable SM3 and SM4 support (FEAT_SM4, FEAT_SM3)", [FeatureNEON]>;
+    "Enable SM3 and SM4 support (FEAT_SM4, FEAT_SM3)", [FeatureNEON],
+    "FEAT_SM4", "+sm4,+fp-armv8,+neon", 106>;
 
 def FeatureSHA2 : Extension<
     "sha2", "SHA2",
-    "Enable SHA1 and SHA256 support (FEAT_SHA1, FEAT_SHA256)", [FeatureNEON]>;
+    "Enable SHA1 and SHA256 support (FEAT_SHA1, FEAT_SHA256)", [FeatureNEON],
+    "FEAT_SHA2", "+sha2,+fp-armv8,+neon", 130>;
 
 def FeatureSHA3 : Extension<
     "sha3", "SHA3",
-    "Enable SHA512 and SHA3 support (FEAT_SHA3, FEAT_SHA512)", [FeatureNEON, FeatureSHA2]>;
+    "Enable SHA512 and SHA3 support (FEAT_SHA3, FEAT_SHA512)", [FeatureNEON, FeatureSHA2],
+    "FEAT_SHA3", "+sha3,+sha2,+fp-armv8,+neon", 140>;
 
 def FeatureAES : Extension<
     "aes", "AES",
-    "Enable AES support (FEAT_AES, FEAT_PMULL)", [FeatureNEON]>;
+    "Enable AES support (FEAT_AES, FEAT_PMULL)", [FeatureNEON],
+    "FEAT_AES", "+fp-armv8,+neon", 150>;
 
 // Crypto has been split up and any combination is now valid (see the
 // crypto definitions above). Also, crypto is now context sensitive:
@@ -56,11 +137,13 @@ def FeatureAES : Extension<
 // meaning anymore. We kept the Crypto definition here for backward
 // compatibility, and now imply features SHA2 and AES, which was the
 // "traditional" meaning of Crypto.
+let FMVDependencies = "+aes,+sha2" in
 def FeatureCrypto : Extension<"crypto", "Crypto",
   "Enable cryptographic instructions", [FeatureNEON, FeatureSHA2, FeatureAES]>;
 
 def FeatureCRC : Extension<"crc", "CRC",
-  "Enable ARMv8 CRC-32 checksum instructions (FEAT_CRC32)">;
+  "Enable ARMv8 CRC-32 checksum instructions (FEAT_CRC32)", [],
+  "FEAT_CRC", "+crc", 110>;
 
 def FeatureRAS : Extension<"ras", "RAS",
   "Enable ARMv8 Reliability, Availability and Serviceability Extensions (FEAT_RAS, FEAT_RASv1p1)">;
@@ -70,7 +153,8 @@ def FeatureRASv2 : Extension<"rasv2", "RASv2",
   [FeatureRAS]>;
 
 def FeatureLSE : Extension<"lse", "LSE",
-  "Enable ARMv8.1 Large System Extension (LSE) atomic instructions (FEAT_LSE)">;
+  "Enable ARMv8.1 Large System Extension (LSE) atomic instructions (FEAT_LSE)", [],
+  "FEAT_LSE", "+lse", 80>;
 
 def FeatureLSE2 : SubtargetFeature<"lse2", "HasLSE2", "true",
   "Enable ARMv8.4 Large System Extension 2 (LSE2) atomicity rules (FEAT_LSE2)">;
@@ -83,7 +167,8 @@ def FeatureFMV : SubtargetFeature<"fmv", "HasFMV", "true",
 
 def FeatureRDM : Extension<"rdm", "RDM",
   "Enable ARMv8.1 Rounding Double Multiply Add/Subtract instructions (FEAT_RDM)",
-  [FeatureNEON]>;
+  [FeatureNEON],
+  "FEAT_RDM", "+rdm,+fp-armv8,+neon", 108>;
 
 def FeaturePAN : SubtargetFeature<
     "pan", "HasPAN", "true",
@@ -102,15 +187,20 @@ def FeatureVH : SubtargetFeature<"vh", "HasVH", "true",
 // This SubtargetFeature is special. It controls only whether codegen will turn
 // `llvm.readcyclecounter()` into an access to a PMUv3 System Register. The
 // `FEAT_PMUv3*` system registers are always available for assembly/disassembly.
+let MArchName = "pmuv3" in
 def FeaturePerfMon : Extension<"perfmon", "PerfMon",
   "Enable Code Generation for ARMv8 PMUv3 Performance Monitors extension (FEAT_PMUv3)">;
 
+let ArchExtKindSpelling = "AEK_FP16", MArchName = "fp16" in
 def FeatureFullFP16 : Extension<"fullfp16", "FullFP16",
-  "Full FP16 (FEAT_FP16)", [FeatureFPARMv8]>;
+  "Full FP16 (FEAT_FP16)", [FeatureFPARMv8],
+  "FEAT_FP16", "+fullfp16,+fp-armv8,+neon", 170>;
 
 def FeatureFP16FML : Extension<"fp16fml", "FP16FML",
-  "Enable FP16 FML instructions (FEAT_FHM)", [FeatureFullFP16]>;
+  "Enable FP16 FML instructions (FEAT_FHM)", [FeatureFullFP16],
+  "FEAT_FP16FML", "+fp16fml,+fullfp16,+fp-armv8,+neon", 175>;
 
+let ArchExtKindSpelling = "AEK_PROFILE", MArchName = "profile" in
 def FeatureSPE : Extension<"spe", "SPE",
   "Enable Statistical Profiling extension (FEAT_SPE)">;
 
@@ -127,11 +217,13 @@ def FeatureCCPP : SubtargetFeature<"ccpp", "HasCCPP",
     "true", "Enable v8.2 data Cache Clean to Point of Persistence (FEAT_DPB)" >;
 
 def FeatureSVE : Extension<"sve", "SVE",
-  "Enable Scalable Vector Extension (SVE) instructions (FEAT_SVE)", [FeatureFullFP16]>;
+  "Enable Scalable Vector Extension (SVE) instructions (FEAT_SVE)", [FeatureFullFP16],
+  "FEAT_SVE", "+sve,+fullfp16,+fp-armv8,+neon", 310>;
 
 def FeatureFPMR : Extension<"fpmr", "FPMR",
   "Enable FPMR Register (FEAT_FPMR)">;
 
+let FMVDependencies = "+fpmr" in
 def FeatureFP8 : Extension<"fp8", "FP8",
   "Enable FP8 instructions (FEAT_FP8)">;
 
@@ -157,28 +249,35 @@ def FeatureUseScalarIncVL : SubtargetFeature<"use-scalar-inc-vl",
   "UseScalarIncVL", "true", "Prefer inc/dec over add+cnt">;
 
 def FeatureBF16 : Extension<"bf16", "BF16",
-    "Enable BFloat16 Extension (FEAT_BF16)" >;
+    "Enable BFloat16 Extension (FEAT_BF16)", [],
+    "FEAT_BF16", "+bf16", 280>;
 
 def FeatureNoSVEFPLD1R : SubtargetFeature<"no-sve-fp-ld1r",
   "NoSVEFPLD1R", "true", "Avoid using LD1RX instructions for FP">;
 
 def FeatureSVE2 : Extension<"sve2", "SVE2",
   "Enable Scalable Vector Extension 2 (SVE2) instructions (FEAT_SVE2)",
-  [FeatureSVE, FeatureUseScalarIncVL]>;
+  [FeatureSVE, FeatureUseScalarIncVL],
+  "FEAT_SVE2", "+sve2,+sve,+fullfp16,+fp-armv8,+neon", 370>;
 
 def FeatureSVE2AES : Extension<"sve2-aes", "SVE2AES",
   "Enable AES SVE2 instructions (FEAT_SVE_AES, FEAT_SVE_PMULL128)",
-  [FeatureSVE2, FeatureAES]>;
+  [FeatureSVE2, FeatureAES],
+  "FEAT_SVE_AES", "+sve2,+sve,+sve2-aes,+fullfp16,+fp-armv8,+neon", 380>;
 
 def FeatureSVE2SM4 : Extension<"sve2-sm4", "SVE2SM4",
-  "Enable SM4 SVE2 instructions (FEAT_SVE_SM4)", [FeatureSVE2, FeatureSM4]>;
+  "Enable SM4 SVE2 instructions (FEAT_SVE_SM4)", [FeatureSVE2, FeatureSM4],
+  "FEAT_SVE_SM4", "+sve2,+sve,+sve2-sm4,+fullfp16,+fp-armv8,+neon", 420>;
 
 def FeatureSVE2SHA3 : Extension<"sve2-sha3", "SVE2SHA3",
-  "Enable SHA3 SVE2 instructions (FEAT_SVE_SHA3)", [FeatureSVE2, FeatureSHA3]>;
+  "Enable SHA3 SVE2 instructions (FEAT_SVE_SHA3)", [FeatureSVE2, FeatureSHA3],
+  "FEAT_SVE_SHA3", "+sve2,+sve,+sve2-sha3,+fullfp16,+fp-armv8,+neon", 410>;
 
 def FeatureSVE2BitPerm : Extension<"sve2-bitperm", "SVE2BitPerm",
-  "Enable bit permutation SVE2 instructions (FEAT_SVE_BitPerm)", [FeatureSVE2]>;
+  "Enable bit permutation SVE2 instructions (FEAT_SVE_BitPerm)", [FeatureSVE2],
+  "FEAT_SVE_BITPERM", "+sve2,+sve,+sve2-bitperm,+fullfp16,+fp-armv8,+neon", 400>;
 
+let FMVDependencies = "+sve2p1,+sve2,+sve,+fullfp16,+fp-armv8,+neon" in
 def FeatureSVE2p1: Extension<"sve2p1", "SVE2p1",
   "Enable Scalable Vector Extension 2.1 instructions", [FeatureSVE2]>;
 
@@ -315,7 +414,8 @@ def FeatureForce32BitJumpTables
                       "Force jump table entries to be 32-bits wide except at MinSize">;
 
 def FeatureRCPC : Extension<"rcpc", "RCPC",
-    "Enable support for RCPC extension (FEAT_LRCPC)">;
+    "Enable support for RCPC extension (FEAT_LRCPC)", [],
+    "FEAT_RCPC", "+rcpc", 230>;
 
 def FeatureUseRSqrt : SubtargetFeature<
     "use-reciprocal-square-root", "UseRSqrt", "true",
@@ -323,25 +423,30 @@ def FeatureUseRSqrt : SubtargetFeature<
 
 def FeatureDotProd : Extension<
     "dotprod", "DotProd",
-    "Enable dot product support (FEAT_DotProd)", [FeatureNEON]>;
+    "Enable dot product support (FEAT_DotProd)", [FeatureNEON],
+    "FEAT_DOTPROD", "+dotprod,+fp-armv8,+neon", 104>;
 
 def FeaturePAuth : Extension<
     "pauth", "PAuth",
     "Enable v8.3-A Pointer Authentication extension (FEAT_PAuth)">;
 
+let ArchExtKindSpelling = "AEK_JSCVT", MArchName = "jscvt" in
 def FeatureJS : Extension<
     "jsconv", "JS",
     "Enable v8.3-A JavaScript FP conversion instructions (FEAT_JSCVT)",
-    [FeatureFPARMv8]>;
+    [FeatureFPARMv8],
+    "FEAT_JSCVT", "+fp-armv8,+neon,+jsconv", 210>;
 
 def FeatureCCIDX : SubtargetFeature<
     "ccidx", "HasCCIDX", "true",
     "Enable v8.3-A Extend of the CCSIDR number of sets (FEAT_CCIDX)">;
 
+let ArchExtKindSpelling = "AEK_FCMA", MArchName = "fcma" in
 def FeatureComplxNum : Extension<
     "complxnum", "ComplxNum",
     "Enable v8.3-A Floating-point complex number support (FEAT_FCMA)",
-    [FeatureNEON]>;
+    [FeatureNEON],
+    "FEAT_FCMA", "+fp-armv8,+neon,+complxnum", 220>;
 
 def FeatureNV : SubtargetFeature<
     "nv", "HasNV", "true",
@@ -378,7 +483,8 @@ def FeatureTLB_RMI : SubtargetFeature<
 
 def FeatureFlagM : Extension<
     "flagm", "FlagM",
-    "Enable v8.4-A Flag Manipulation Instructions (FEAT_FlagM)">;
+    "Enable v8.4-A Flag Manipulation Instructions (FEAT_FlagM)", [],
+    "FEAT_FLAGM", "+flagm", 20>;
 
 // 8.4 RCPC enchancements: LDAPR & STLR instructions with Immediate Offset
 def FeatureRCPC_IMMO : SubtargetFeature<"rcpc-immo", "HasRCPC_IMMO", "true",
@@ -426,30 +532,41 @@ def FeatureSpecRestrict : SubtargetFeature<"specrestrict", "HasSpecRestrict",
   "true", "Enable architectural speculation restriction (FEAT_CSV2_2)">;
 
 def FeatureSB : Extension<"sb", "SB",
-  "Enable v8.5 Speculation Barrier (FEAT_SB)" >;
+  "Enable v8.5 Speculation Barrier (FEAT_SB)", [],
+  "FEAT_SB", "+sb", 470>;
 
 def FeatureSSBS : Extension<"ssbs", "SSBS",
-  "Enable Speculative Store Bypass Safe bit (FEAT_SSBS, FEAT_SSBS2)" >;
+  "Enable Speculative Store Bypass Safe bit (FEAT_SSBS, FEAT_SSBS2)", [],
+  "FEAT_SSBS", "", 490>;
 
 def FeaturePredRes : Extension<"predres", "PredRes",
-  "Enable v8.5a execution and data prediction invalidation instructions (FEAT_SPECRES)" >;
+  "Enable v8.5a execution and data prediction invalidation instructions (FEAT_SPECRES)", [],
+  "FEAT_PREDRES", "+predres", 480>;
 
-def FeatureCacheDeepPersist : Extension<"ccdp", "CCDP",
+def FeatureCacheDeepPersist : SubtargetFeature<"ccdp", "CCDP", "true",
     "Enable v8.5 Cache Clean to Point of Deep Persistence (FEAT_DPB2)" >;
 
+let ArchExtKindSpelling = "AEK_NONE" in
 def FeatureBranchTargetId : Extension<"bti", "BTI",
-    "Enable Branch Target Identification (FEAT_BTI)" >;
+    "Enable Branch Target Identification (FEAT_BTI)", [],
+    "FEAT_BTI", "+bti", 510>;
 
+let ArchExtKindSpelling = "AEK_RAND", MArchName = "rng" in
 def FeatureRandGen : Extension<"rand", "RandGen",
-    "Enable Random Number generation instructions (FEAT_RNG)" >;
+    "Enable Random Number generation instructions (FEAT_RNG)", [],
+    "FEAT_RNG", "+rand", 10>;
 
+// NOTE: "memtag" means FEAT_MTE + FEAT_MTE2 for -march or
+// __attribute((target(...))), but only FEAT_MTE for FMV.
+let MArchName = "memtag" in
 def FeatureMTE : Extension<"mte", "MTE",
-    "Enable Memory Tagging Extension (FEAT_MTE, FEAT_MTE2)" >;
+    "Enable Memory Tagging Extension (FEAT_MTE, FEAT_MTE2)", [],
+    "FEAT_MEMTAG", "", 440>;
 
-def FeatureTRBE : Extension<"trbe", "TRBE",
+def FeatureTRBE : SubtargetFeature<"trbe", "TRBE", "true",
     "Enable Trace Buffer Extension (FEAT_TRBE)">;
 
-def FeatureETE : Extension<"ete", "ETE",
+def FeatureETE : SubtargetFeature<"ete", "ETE", "true",
     "Enable Embedded Trace Extension (FEAT_ETE)",
     [FeatureTRBE]>;
 
@@ -461,18 +578,25 @@ def FeatureTaggedGlobals : SubtargetFeature<"tagged-globals",
     "true", "Use an instruction sequence for taking the address of a global "
     "that allows a memory tag in the upper address bits">;
 
+let ArchExtKindSpelling = "AEK_I8MM" in
 def FeatureMatMulInt8 : Extension<"i8mm", "MatMulInt8",
-    "Enable Matrix Multiply Int8 Extension (FEAT_I8MM)">;
+    "Enable Matrix Multiply Int8 Extension (FEAT_I8MM)", [],
+    "FEAT_I8MM", "+i8mm", 270>;
 
+let ArchExtKindSpelling = "AEK_F32MM" in
 def FeatureMatMulFP32 : Extension<"f32mm", "MatMulFP32",
-    "Enable Matrix Multiply FP32 Extension (FEAT_F32MM)", [FeatureSVE]>;
+    "Enable Matrix Multiply FP32 Extension (FEAT_F32MM)", [FeatureSVE],
+    "FEAT_SVE_F32MM", "+sve,+f32mm,+fullfp16,+fp-armv8,+neon", 350>;
 
+let ArchExtKindSpelling = "AEK_F64MM" in
 def FeatureMatMulFP64 : Extension<"f64mm", "MatMulFP64",
-    "Enable Matrix Multiply FP64 Extension (FEAT_F64MM)", [FeatureSVE]>;
+    "Enable Matrix Multiply FP64 Extension (FEAT_F64MM)", [FeatureSVE],
+    "FEAT_SVE_F64MM", "+sve,+f64mm,+fullfp16,+fp-armv8,+neon", 360>;
 
 def FeatureXS : SubtargetFeature<"xs", "HasXS",
     "true", "Enable Armv8.7-A limited-TLB-maintenance instruction (FEAT_XS)">;
 
+// FIXME link with FMVExtension?
 def FeatureWFxT : SubtargetFeature<"wfxt", "HasWFxT",
     "true", "Enable Armv8.7-A WFET and WFIT instruction (FEAT_WFxT)">;
 
@@ -480,13 +604,15 @@ def FeatureHCX : SubtargetFeature<
     "hcx", "HasHCX", "true", "Enable Armv8.7-A HCRX_EL2 system register (FEAT_HCX)">;
 
 def FeatureLS64 : Extension<"ls64", "LS64",
-    "Enable Armv8.7-A LD64B/ST64B Accelerator Extension (FEAT_LS64, FEAT_LS64_V, FEAT_LS64_ACCDATA)">;
+    "Enable Armv8.7-A LD64B/ST64B Accelerator Extension (FEAT_LS64, FEAT_LS64_V, FEAT_LS64_ACCDATA)", [],
+    "FEAT_LS64", "", 520>;
 
 def FeatureHBC : Extension<"hbc", "HBC",
     "Enable Armv8.8-A Hinted Conditional Branches Extension (FEAT_HBC)">;
 
 def FeatureMOPS : Extension<"mops", "MOPS",
-    "Enable Armv8.8-A memcpy and memset acceleration instructions (FEAT_MOPS)">;
+    "Enable Armv8.8-A memcpy and memset acceleration instructions (FEAT_MOPS)", [],
+    "FEAT_MOPS", "+mops", 650>;
 
 def FeatureNMI : SubtargetFeature<"nmi", "HasNMI",
     "true", "Enable Armv8.8-A Non-maskable Interrupts (FEAT_NMI, FEAT_GICv3_NMI)">;
@@ -508,44 +634,54 @@ def FeatureRME : SubtargetFeature<"rme", "HasRME",
     "true", "Enable Realm Management Extension (FEAT_RME)">;
 
 def FeatureSME : Extension<"sme", "SME",
-  "Enable Scalable Matrix Extension (SME) (FEAT_SME)", [FeatureBF16, FeatureUseScalarIncVL]>;
+  "Enable Scalable Matrix Extension (SME) (FEAT_SME)", [FeatureBF16, FeatureUseScalarIncVL],
+  "FEAT_SME", "+sme,+bf16", 430>;
 
 def FeatureSMEF64F64 : Extension<"sme-f64f64", "SMEF64F64",
-  "Enable Scalable Matrix Extension (SME) F64F64 instructions (FEAT_SME_F64F64)", [FeatureSME]>;
+  "Enable Scalable Matrix Extension (SME) F64F64 instructions (FEAT_SME_F64F64)", [FeatureSME],
+  "FEAT_SME_F64", "+sme,+sme-f64f64,+bf16", 560>;
 
 def FeatureSMEI16I64 : Extension<"sme-i16i64", "SMEI16I64",
-  "Enable Scalable Matrix Extension (SME) I16I64 instructions (FEAT_SME_I16I64)", [FeatureSME]>;
+  "Enable Scalable Matrix Extension (SME) I16I64 instructions (FEAT_SME_I16I64)", [FeatureSME],
+  "FEAT_SME_I64", "+sme,+sme-i16i64,+bf16", 570>;
 
 def FeatureSMEFA64 : Extension<"sme-fa64", "SMEFA64",
   "Enable the full A64 instruction set in streaming SVE mode (FEAT_SME_FA64)", [FeatureSME, FeatureSVE2]>;
 
 def FeatureSME2 : Extension<"sme2", "SME2",
-  "Enable Scalable Matrix Extension 2 (SME2) instructions", [FeatureSME]>;
+  "Enable Scalable Matrix Extension 2 (SME2) instructions", [FeatureSME],
+  "FEAT_SME2", "+sme2,+sme,+bf16", 580>;
 
+let FMVDependencies = "+sme2,+sme-f16f16" in
 def FeatureSMEF16F16 : Extension<"sme-f16f16", "SMEF16F16",
   "Enable SME non-widening Float16 instructions (FEAT_SME_F16F16)", [FeatureSME2]>;
 
+let FMVDependencies = "+sme2p1,+sme2,+sme,+bf16" in
 def FeatureSME2p1 : Extension<"sme2p1", "SME2p1",
   "Enable Scalable Matrix Extension 2.1 (FEAT_SME2p1) instructions", [FeatureSME2]>;
 
 def FeatureFAMINMAX: Extension<"faminmax", "FAMINMAX",
    "Enable FAMIN and FAMAX instructions (FEAT_FAMINMAX)">;
 
+let FMVDependencies = "+fpmr" in
 def FeatureFP8FMA : Extension<"fp8fma", "FP8FMA",
   "Enable fp8 multiply-add instructions (FEAT_FP8FMA)">;
 
+let FMVDependencies = "+sme2" in
 def FeatureSSVE_FP8FMA : Extension<"ssve-fp8fma", "SSVE_FP8FMA",
   "Enable SVE2 fp8 multiply-add instructions (FEAT_SSVE_FP8FMA)", [FeatureSME2]>;
 
 def FeatureFP8DOT2: Extension<"fp8dot2", "FP8DOT2",
    "Enable fp8 2-way dot instructions (FEAT_FP8DOT2)">;
 
+let FMVDependencies = "+sme2" in
 def FeatureSSVE_FP8DOT2 : Extension<"ssve-fp8dot2", "SSVE_FP8DOT2",
   "Enable SVE2 fp8 2-way dot product instructions (FEAT_SSVE_FP8DOT2)", [FeatureSME2]>;
 
 def FeatureFP8DOT4: Extension<"fp8dot4", "FP8DOT4",
    "Enable fp8 4-way dot instructions (FEAT_FP8DOT4)">;
 
+let FMVDependencies = "+sme2" in
 def FeatureSSVE_FP8DOT4 : Extension<"ssve-fp8dot4", "SSVE_FP8DOT4",
   "Enable SVE2 fp8 4-way dot product instructions (FEAT_SSVE_FP8DOT4)", [FeatureSME2]>;
 def FeatureLUT: Extension<"lut", "LUT",
@@ -554,9 +690,11 @@ def FeatureLUT: Extension<"lut", "LUT",
 def FeatureSME_LUTv2 : Extension<"sme-lutv2", "SME_LUTv2",
   "Enable Scalable Matrix Extension (SME) LUTv2 instructions (FEAT_SME_LUTv2)">;
 
+let FMVDependencies = "+fp8,+sme2" in
 def FeatureSMEF8F16 : Extension<"sme-f8f16", "SMEF8F16",
   "Enable Scalable Matrix Extension (SME) F8F16 instructions(FEAT_SME_F8F16)", [FeatureSME2, FeatureFP8]>;
 
+let FMVDependencies = "+sme2,+fp8" in
 def FeatureSMEF8F32 : Extension<"sme-f8f32", "SMEF8F32",
   "Enable Scalable Matrix Extension (SME) F8F32 instructions (FEAT_SME_F8F32)", [FeatureSME2, FeatureFP8]>;
 
@@ -592,6 +730,7 @@ def FeatureCLRBHB : SubtargetFeature<"clrbhb", "HasCLRBHB",
 def FeaturePRFM_SLC : SubtargetFeature<"prfm-slc-target", "HasPRFM_SLC",
     "true", "Enable SLC target for PRFM instruction">;
 
+let MArchName = "predres2" in
 def FeatureSPECRES2 : Extension<"specres2", "SPECRES2",
     "Enable Speculation Restriction Instruction (FEAT_SPECRES2)",
     [FeaturePredRes]>;
@@ -605,7 +744,8 @@ def FeatureITE : Extension<"ite", "ITE",
 
 def FeatureRCPC3 : Extension<"rcpc3", "RCPC3",
     "Enable Armv8.9-A RCPC instructions for A64 and Advanced SIMD and floating-point instruction set (FEAT_LRCPC3)",
-    [FeatureRCPC_IMMO]>;
+    [FeatureRCPC_IMMO],
+    "FEAT_RCPC3", "+rcpc,+rcpc3", 241>;
 
 def FeatureTHE : Extension<"the", "THE",
     "Enable Armv8.9-A Translation Hardening Extension (FEAT_THE)">;
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index b61928bd8f98..0455e061f0bf 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -1989,19 +1989,19 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
       AArch64::AEK_SME,          AArch64::AEK_SMEF64F64,
       AArch64::AEK_SMEI16I64,    AArch64::AEK_SME2,
       AArch64::AEK_HBC,          AArch64::AEK_MOPS,
-      AArch64::AEK_PERFMON,      AArch64::AEK_SVE2p1,
-      AArch64::AEK_SME2p1,       AArch64::AEK_B16B16,
+      AArch64::AEK_PERFMON,      AArch64::AEK_SVE2P1,
+      AArch64::AEK_SME2P1,       AArch64::AEK_B16B16,
       AArch64::AEK_SMEF16F16,    AArch64::AEK_CSSC,
       AArch64::AEK_RCPC3,        AArch64::AEK_THE,
       AArch64::AEK_D128,         AArch64::AEK_LSE128,
-      AArch64::AEK_SPECRES2,     AArch64::AEK_RASv2,
+      AArch64::AEK_SPECRES2,     AArch64::AEK_RASV2,
       AArch64::AEK_ITE,          AArch64::AEK_GCS,
       AArch64::AEK_FPMR,         AArch64::AEK_FP8,
       AArch64::AEK_FAMINMAX,     AArch64::AEK_FP8FMA,
       AArch64::AEK_SSVE_FP8FMA,  AArch64::AEK_FP8DOT2,
       AArch64::AEK_SSVE_FP8DOT2, AArch64::AEK_FP8DOT4,
       AArch64::AEK_SSVE_FP8DOT4, AArch64::AEK_LUT,
-      AArch64::AEK_SME_LUTv2,    AArch64::AEK_SMEF8F16,
+      AArch64::AEK_SME_LUTV2,    AArch64::AEK_SMEF8F16,
       AArch64::AEK_SMEF8F32,     AArch64::AEK_SMEFA64,
       AArch64::AEK_CPA,          AArch64::AEK_PAUTHLR,
       AArch64::AEK_TLBIW,        AArch64::AEK_JSCVT,
diff --git a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
index 05aa146b5715..4a46f2ea9586 100644
--- a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/StringSet.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
+#include <cstdint>
 
 using namespace llvm;
 
@@ -33,6 +34,16 @@ static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
     return Set;
   };
 
+  // Sort the extensions alphabetically, so they don't appear in tablegen order.
+  std::vector<Record *> SortedExtensions =
+      RK.getAllDerivedDefinitions("Extension");
+  auto Alphabetical = [](Record *A, Record *B) -> bool {
+    const auto MarchA = A->getValueAsString("MArchName");
+    const auto MarchB = B->getValueAsString("MArchName");
+    return MarchA.compare(MarchB) < 0; // A lexographically less than B
+  };
+  std::sort(SortedExtensions.begin(), SortedExtensions.end(), Alphabetical);
+
   // The ARMProcFamilyEnum values are initialised by SubtargetFeature defs
   // which set the ARMProcFamily field. We can generate the enum from these defs
   // which look like this:
@@ -57,16 +68,46 @@ static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
     OS << "ARM_ARCHITECTURE(" << Arch << ")\n";
   OS << "\n#undef ARM_ARCHITECTURE\n\n";
 
-  // Emit information for each defined Extension; used to build ArmExtKind.
-  OS << "#ifndef ARM_EXTENSION\n"
-     << "#define ARM_EXTENSION(NAME, ENUM)\n"
-     << "#endif\n\n";
-  for (const Record *Rec : RK.getAllDerivedDefinitions("Extension")) {
-    StringRef Name = Rec->getValueAsString("Name");
-    std::string Enum = Rec->getValueAsString("ArchExtKindSpelling").upper();
-    OS << "ARM_EXTENSION(" << Name << ", " << Enum << ")\n";
+  // Emit the ArchExtKind enum
+  OS << "#ifdef EMIT_ARCHEXTKIND_ENUM\n"
+     << "enum ArchExtKind : unsigned {\n"
+     << "  AEK_NONE = 1,\n";
+  for (const Record *Rec : SortedExtensions) {
+    auto AEK = Rec->getValueAsString("ArchExtKindSpelling").upper();
+    if (AEK != "AEK_NONE")
+      OS << "  " << AEK << ",\n";
   }
-  OS << "\n#undef ARM_EXTENSION\n\n";
+  OS << "  AEK_NUM_EXTENSIONS\n"
+     << "};\n"
+     << "#undef EMIT_ARCHEXTKIND_ENUM\n"
+     << "#endif // EMIT_ARCHEXTKIND_ENUM\n";
+
+  // Emit information for each defined Extension; used to build ArmExtKind.
+  OS << "#ifdef EMIT_EXTENSIONS\n"
+     << "inline constexpr ExtensionInfo Extensions[] = {\n";
+  for (const Record *Rec : SortedExtensions) {
+    auto AEK = Rec->getValueAsString("ArchExtKindSpelling").upper();
+    OS << "  ";
+    OS << "{\"" << Rec->getValueAsString("MArchName") << "\"";
+    OS << ", AArch64::" << AEK;
+    if (AEK == "AEK_NONE") {
+      // HACK: don't emit posfeat/negfeat strings for FMVOnlyExtensions.
+      OS << ", {}, {}";
+    } else {
+      OS << ", \"+" << Rec->getValueAsString("Name") << "\""; // posfeature
+      OS << ", \"-" << Rec->getValueAsString("Name") << "\""; // negfeature
+    }
+    OS << ", " << Rec->getValueAsString("FMVBit");
+    OS << ", \"" << Rec->getValueAsString("FMVDependencies") << "\"";
+    OS << ", " << (uint64_t)Rec->getValueAsInt("FMVPriority");
+    OS << "},\n";
+  };
+  OS << "  {\"none\", AArch64::AEK_NONE, {}, {}, FEAT_INIT, \"\", "
+        "ExtensionInfo::MaxFMVPriority},\n";
+  OS << "};\n"
+     << "#undef EMIT_EXTENSIONS\n"
+     << "#endif // EMIT_EXTENSIONS\n"
+     << "\n";
 }
 
 static TableGen::Emitter::Opt
-- 
GitLab


From 99f45b4c5b67cccb7845580a67b42776f49ef0e2 Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Thu, 9 May 2024 14:03:08 -0700
Subject: [PATCH 0345/1206] [bazel] Fix new CodeGen dep (#91654)

```
.../AMDGPUUtilsAndDesc/AMDGPUCallLowering.h:17:10: fatal error: 'llvm/CodeGen/GlobalISel/CallLowering.h' file not found
```

https://buildkite.com/llvm-project/upstream-bazel/builds/97166
---
 utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index c159204cede7..df5cd276b12f 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -636,8 +636,8 @@ cc_binary(
 cc_binary(
     name = "llvm-min-tblgen",
     srcs = [
-        "utils/TableGen/Attributes.cpp",
         "utils/TableGen/ARMTargetDefEmitter.cpp",
+        "utils/TableGen/Attributes.cpp",
         "utils/TableGen/Basic/CodeGenIntrinsics.cpp",
         "utils/TableGen/Basic/CodeGenIntrinsics.h",
         "utils/TableGen/Basic/SDNodeProperties.cpp",
@@ -2421,6 +2421,7 @@ gentbl(
         strip_include_prefix = "lib/Target/" + target["name"],
         deps = [
             ":BinaryFormat",
+            ":CodeGen",
             ":CodeGenTypes",
             ":Core",
             ":DebugInfoCodeView",
@@ -4049,7 +4050,7 @@ cc_binary(
 
 cc_binary(
     name = "llvm-mca",
-    srcs =[
+    srcs = [
         "tools/llvm-mca/llvm-mca.cpp",
     ],
     copts = llvm_copts,
-- 
GitLab


From db9421381980cdf3d6914f8898a77d3237325019 Mon Sep 17 00:00:00 2001
From: Med Ismail Bennani <ismail@bennani.ma>
Date: Thu, 9 May 2024 14:13:44 -0700
Subject: [PATCH 0346/1206] [lldb/crashlog] Fix test failure when creating a
 target using command options (#91653)

This should fix the various crashlog test failures on the bots:

```
  lldb-shell :: ScriptInterpreter/Python/Crashlog/app_specific_backtrace_crashlog.test
  lldb-shell :: ScriptInterpreter/Python/Crashlog/interactive_crashlog_json.test
  lldb-shell :: ScriptInterpreter/Python/Crashlog/interactive_crashlog_legacy.test
  lldb-shell :: ScriptInterpreter/Python/Crashlog/last_exception_backtrace_crashlog.test
  lldb-shell :: ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test
```

When we create a target by using the command option, we don't set it in
the crashlog object which later on cause us to fail loading the images.

rdar://127832961

Signed-off-by: Med Ismail Bennani <ismail@bennani.ma>
---
 lldb/examples/python/crashlog.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py
index 2919b9c76e68..641b2e64d53b 100755
--- a/lldb/examples/python/crashlog.py
+++ b/lldb/examples/python/crashlog.py
@@ -1494,6 +1494,7 @@ def load_crashlog_in_scripted_process(debugger, crashlog_path, options, result):
             raise InteractiveCrashLogException(
                 "couldn't create target provided by the user (%s)" % options.target_path
             )
+        crashlog.target = target
 
     # 2. If the user didn't provide a target, try to create a target using the symbolicator
     if not target or not target.IsValid():
-- 
GitLab


From ca3917538de1deeb0e51f11fbdbe295b6d3768d1 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu@amd.com>
Date: Thu, 9 May 2024 17:14:43 -0400
Subject: [PATCH 0347/1206] [ClangOffloadBundler] make hipv4 and hip compatible
 (#91637)

The distinction between the hip and hipv4 offload kinds is historically
based. Originally, these designations might have indicated different
versions of the code object ABI (Application Binary Interface). However,
as the system has evolved, the ABI version is now embedded directly
within the code object itself, making these historical distinctions
irrelevant during the unbundling process. Consequently, hip and hipv4
are treated as compatible in current implementations, facilitating
interchangeable handling of code objects without differentiation based
on offload kind. This change streamlines code management within the
ecosystem.
---
 clang/docs/ClangOffloadBundler.rst                   | 12 ++++++++++--
 clang/lib/Driver/OffloadBundler.cpp                  |  5 ++++-
 clang/test/Driver/clang-offload-bundler.c            | 11 +++++++++++
 clang/test/Driver/linker-wrapper.c                   |  4 ++--
 .../clang-linker-wrapper/ClangLinkerWrapper.cpp      |  2 +-
 5 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/clang/docs/ClangOffloadBundler.rst b/clang/docs/ClangOffloadBundler.rst
index 515e6c00a3b8..3c241027d405 100644
--- a/clang/docs/ClangOffloadBundler.rst
+++ b/clang/docs/ClangOffloadBundler.rst
@@ -245,7 +245,7 @@ Where:
                     object as a data section with the name ``.hip_fatbin``.
 
       hipv4         Offload code object for the HIP language. Used for AMD GPU
-                    code objects with at least ABI version V4 when the
+                    code objects with at least ABI version V4 and above when the
                     ``clang-offload-bundler`` is used to create a *fat binary*
                     to be loaded by the HIP runtime. The fat binary can be
                     loaded directly from a file, or be embedded in the host code
@@ -254,6 +254,14 @@ Where:
       openmp        Offload code object for the OpenMP language extension.
       ============= ==============================================================
 
+Note: The distinction between the `hip` and `hipv4` offload kinds is historically based.
+Originally, these designations might have indicated different versions of the
+code object ABI. However, as the system has evolved, the ABI version is now embedded
+directly within the code object itself, making these historical distinctions irrelevant
+during the unbundling process. Consequently, `hip` and `hipv4` are treated as compatible
+in current implementations, facilitating interchangeable handling of code objects
+without differentiation based on offload kind.
+
 **target-triple**
     The target triple of the code object. See `Target Triple
     <https://clang.llvm.org/docs/CrossCompilation.html#target-triple>`_.
@@ -295,7 +303,7 @@ Compatibility Rules for Bundle Entry ID
   A code object, specified using its Bundle Entry ID, can be loaded and
   executed on a target processor, if:
 
-  * Their offload kinds are the same.
+  * Their offload kinds are the same or comptible.
   * Their target triples are compatible.
   * Their Target IDs are compatible as defined in :ref:`compatibility-target-id`.
 
diff --git a/clang/lib/Driver/OffloadBundler.cpp b/clang/lib/Driver/OffloadBundler.cpp
index 8cc82a0ee716..191d108e9b73 100644
--- a/clang/lib/Driver/OffloadBundler.cpp
+++ b/clang/lib/Driver/OffloadBundler.cpp
@@ -113,8 +113,11 @@ bool OffloadTargetInfo::isOffloadKindValid() const {
 
 bool OffloadTargetInfo::isOffloadKindCompatible(
     const StringRef TargetOffloadKind) const {
-  if (OffloadKind == TargetOffloadKind)
+  if ((OffloadKind == TargetOffloadKind) ||
+      (OffloadKind == "hip" && TargetOffloadKind == "hipv4") ||
+      (OffloadKind == "hipv4" && TargetOffloadKind == "hip"))
     return true;
+
   if (BundlerConfig.HipOpenmpCompatible) {
     bool HIPCompatibleWithOpenMP = OffloadKind.starts_with_insensitive("hip") &&
                                    TargetOffloadKind == "openmp";
diff --git a/clang/test/Driver/clang-offload-bundler.c b/clang/test/Driver/clang-offload-bundler.c
index e492da31abb7..1909ff2d71d0 100644
--- a/clang/test/Driver/clang-offload-bundler.c
+++ b/clang/test/Driver/clang-offload-bundler.c
@@ -505,6 +505,17 @@
 // RUN:   -output=%t.res.tgt1 -input=%t.hip.bundle.bc -unbundle 2>&1 | FileCheck %s -check-prefix=NOGFX906
 // NOGFX906: error: Can't find bundles for hip-amdgcn-amd-amdhsa--gfx906
 
+//
+// Check hip and hipv4 are compatible as offload kind.
+//
+// RUN: clang-offload-bundler -type=o -targets=hip-amdgcn-amd-amdhsa--gfx90a -input=%t.tgt1 -output=%t.bundle3.o
+// RUN: clang-offload-bundler -type=o -targets=hipv4-amdgcn-amd-amdhsa--gfx90a:sramecc-:xnack+ -output=%t.res.tgt1 -input=%t.bundle3.o -unbundle
+// RUN: diff %t.tgt1 %t.res.tgt1
+
+// RUN: clang-offload-bundler -type=o -targets=hipv4-amdgcn-amd-amdhsa--gfx90a -input=%t.tgt1 -output=%t.bundle3.o
+// RUN: clang-offload-bundler -type=o -targets=hip-amdgcn-amd-amdhsa--gfx90a:sramecc-:xnack+ -output=%t.res.tgt1 -input=%t.bundle3.o -unbundle
+// RUN: diff %t.tgt1 %t.res.tgt1
+
 //
 // Check archive unbundling
 //
diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c
index cbf24d4ce3a8..51bf98b2ed39 100644
--- a/clang/test/Driver/linker-wrapper.c
+++ b/clang/test/Driver/linker-wrapper.c
@@ -120,7 +120,7 @@ __attribute__((visibility("protected"), used)) int x;
 
 // HIP: clang{{.*}} -o [[IMG_GFX908:.+]] --target=amdgcn-amd-amdhsa -mcpu=gfx908
 // HIP: clang{{.*}} -o [[IMG_GFX90A:.+]] --target=amdgcn-amd-amdhsa -mcpu=gfx90a
-// HIP: clang-offload-bundler{{.*}}-type=o -bundle-align=4096 -compress -compression-level=6 -targets=host-x86_64-unknown-linux,hipv4-amdgcn-amd-amdhsa--gfx90a,hipv4-amdgcn-amd-amdhsa--gfx908 -input=/dev/null -input=[[IMG_GFX90A]] -input=[[IMG_GFX908]] -output={{.*}}.hipfb
+// HIP: clang-offload-bundler{{.*}}-type=o -bundle-align=4096 -compress -compression-level=6 -targets=host-x86_64-unknown-linux,hip-amdgcn-amd-amdhsa--gfx90a,hip-amdgcn-amd-amdhsa--gfx908 -input=/dev/null -input=[[IMG_GFX90A]] -input=[[IMG_GFX908]] -output={{.*}}.hipfb
 
 // RUN: clang-offload-packager -o %t.out \
 // RUN:   --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx908 \
@@ -210,7 +210,7 @@ __attribute__((visibility("protected"), used)) int x;
 // RUN:   %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=RELOCATABLE-LINK-HIP
 
 // RELOCATABLE-LINK-HIP: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa
-// RELOCATABLE-LINK-HIP: clang-offload-bundler{{.*}} -type=o -bundle-align=4096 -targets=host-x86_64-unknown-linux,hipv4-amdgcn-amd-amdhsa--gfx90a -input=/dev/null -input={{.*}} -output={{.*}}
+// RELOCATABLE-LINK-HIP: clang-offload-bundler{{.*}} -type=o -bundle-align=4096 -targets=host-x86_64-unknown-linux,hip-amdgcn-amd-amdhsa--gfx90a -input=/dev/null -input={{.*}} -output={{.*}}
 // RELOCATABLE-LINK-HIP: /usr/bin/ld.lld{{.*}}-r
 // RELOCATABLE-LINK-HIP: llvm-objcopy{{.*}}a.out --remove-section .llvm.offloading
 
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index a1879fc7712d..69d8cb446fad 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -413,7 +413,7 @@ fatbinary(ArrayRef<std::pair<StringRef, StringRef>> InputFiles,
 
   SmallVector<StringRef> Targets = {"-targets=host-x86_64-unknown-linux"};
   for (const auto &[File, Arch] : InputFiles)
-    Targets.push_back(Saver.save("hipv4-amdgcn-amd-amdhsa--" + Arch));
+    Targets.push_back(Saver.save("hip-amdgcn-amd-amdhsa--" + Arch));
   CmdArgs.push_back(Saver.save(llvm::join(Targets, ",")));
 
 #ifdef _WIN32
-- 
GitLab


From 5a0e0b659fb5c652c66a083224bf300b4ae32452 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Thu, 9 May 2024 17:27:59 -0400
Subject: [PATCH 0348/1206] Revert "[libc][NFC] adjust time related
 implementations" (#91657)

Reverts llvm/llvm-project#91485. It breaks GPU and fuchisa.
---
 libc/hdr/CMakeLists.txt                       |  9 ----
 libc/hdr/time_macros.h                        | 22 ---------
 libc/hdr/types/CMakeLists.txt                 | 45 -------------------
 libc/hdr/types/clock_t.h                      | 22 ---------
 libc/hdr/types/clockid_t.h                    | 22 ---------
 libc/hdr/types/struct_timeval.h               | 21 ---------
 libc/hdr/types/suseconds_t.h                  | 22 ---------
 libc/hdr/types/time_t.h                       | 22 ---------
 libc/src/__support/CMakeLists.txt             |  2 -
 libc/src/__support/time/CMakeLists.txt        | 19 --------
 libc/src/__support/time/clock_gettime.h       | 23 ----------
 libc/src/__support/time/linux/CMakeLists.txt  | 14 ------
 libc/src/__support/time/units.h               | 38 ----------------
 libc/src/time/clock.h                         |  2 +-
 libc/src/time/clock_gettime.h                 |  5 +--
 libc/src/time/gettimeofday.h                  |  2 +-
 libc/src/time/linux/CMakeLists.txt            | 30 ++++++-------
 libc/src/time/linux/clock.cpp                 | 20 +++++----
 .../linux/clockGetTimeImpl.h}                 | 25 +++++++----
 libc/src/time/linux/clock_gettime.cpp         |  9 +++-
 libc/src/time/linux/gettimeofday.cpp          | 14 +++---
 libc/src/time/linux/time.cpp                  | 12 +++--
 libc/src/time/nanosleep.h                     |  4 +-
 libc/src/time/time_func.h                     |  2 +-
 24 files changed, 71 insertions(+), 335 deletions(-)
 delete mode 100644 libc/hdr/time_macros.h
 delete mode 100644 libc/hdr/types/clock_t.h
 delete mode 100644 libc/hdr/types/clockid_t.h
 delete mode 100644 libc/hdr/types/struct_timeval.h
 delete mode 100644 libc/hdr/types/suseconds_t.h
 delete mode 100644 libc/hdr/types/time_t.h
 delete mode 100644 libc/src/__support/time/CMakeLists.txt
 delete mode 100644 libc/src/__support/time/clock_gettime.h
 delete mode 100644 libc/src/__support/time/linux/CMakeLists.txt
 delete mode 100644 libc/src/__support/time/units.h
 rename libc/src/{__support/time/linux/clock_gettime.cpp => time/linux/clockGetTimeImpl.h} (64%)

diff --git a/libc/hdr/CMakeLists.txt b/libc/hdr/CMakeLists.txt
index 754934251430..179b05e6ee96 100644
--- a/libc/hdr/CMakeLists.txt
+++ b/libc/hdr/CMakeLists.txt
@@ -68,13 +68,4 @@ add_proxy_header_library(
     libc.include.llvm-libc-macros.sys_epoll_macros
 )
 
-add_proxy_header_library(
-  time_macros
-  HDRS
-    time_macros.h
-  FULL_BUILD_DEPENDS
-    libc.include.time
-    libc.include.llvm-libc-macros.time_macros
-)
-
 add_subdirectory(types)
diff --git a/libc/hdr/time_macros.h b/libc/hdr/time_macros.h
deleted file mode 100644
index dc36fe66f7a8..000000000000
--- a/libc/hdr/time_macros.h
+++ /dev/null
@@ -1,22 +0,0 @@
-//===-- Definition of macros from time.h ----------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_HDR_TIME_MACROS_H
-#define LLVM_LIBC_HDR_TIME_MACROS_H
-
-#ifdef LIBC_FULL_BUILD
-
-#include "include/llvm-libc-macros/time-macros.h"
-
-#else // Overlay mode
-
-#include <time.h>
-
-#endif // LLVM_LIBC_FULL_BUILD
-
-#endif // LLVM_LIBC_HDR_TIME_MACROS_H
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 3a1bb2f3c340..46a66ec59020 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -63,48 +63,3 @@ add_proxy_header_library(
     libc.include.llvm-libc-types.fexcept_t
     libc.include.fenv
 )
-
-add_proxy_header_library(
-  time_t
-  HDRS
-    time_t.h
-  FULL_BUILD_DEPENDS
-    libc.include.llvm-libc-types.time_t
-    libc.include.time
-)
-
-add_proxy_header_library(
-  clockid_t
-  HDRS
-    clockid_t.h
-  FULL_BUILD_DEPENDS
-    libc.include.llvm-libc-types.clockid_t
-    libc.include.sys_types
-)
-
-add_proxy_header_library(
-  clock_t
-  HDRS
-    clock_t.h
-  FULL_BUILD_DEPENDS
-    libc.include.llvm-libc-types.clock_t
-    libc.include.time
-)
-
-add_proxy_header_library(
-  suseconds_t
-  HDRS
-    suseconds_t.h
-  FULL_BUILD_DEPENDS
-    libc.include.llvm-libc-types.suseconds_t
-    libc.include.sys_time
-)
-
-add_proxy_header_library(
-  struct_timeval
-  HDRS
-    struct_timeval.h
-  FULL_BUILD_DEPENDS
-    libc.include.llvm-libc-types.struct_timeval
-    libc.include.sys_time
-)
diff --git a/libc/hdr/types/clock_t.h b/libc/hdr/types/clock_t.h
deleted file mode 100644
index b0b658e96c3d..000000000000
--- a/libc/hdr/types/clock_t.h
+++ /dev/null
@@ -1,22 +0,0 @@
-//===-- Proxy for clock_t -------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_HDR_TYPES_CLOCK_T_H
-#define LLVM_LIBC_HDR_TYPES_CLOCK_T_H
-
-#ifdef LIBC_FULL_BUILD
-
-#include "include/llvm-libc-types/clock_t.h"
-
-#else // Overlay mode
-
-#include <sys/types.h>
-
-#endif // LLVM_LIBC_FULL_BUILD
-
-#endif // LLVM_LIBC_HDR_TYPES_CLOCK_T_H
diff --git a/libc/hdr/types/clockid_t.h b/libc/hdr/types/clockid_t.h
deleted file mode 100644
index 333342072a2f..000000000000
--- a/libc/hdr/types/clockid_t.h
+++ /dev/null
@@ -1,22 +0,0 @@
-//===-- Proxy for clockid_t -----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_HDR_TYPES_CLOCKID_T_H
-#define LLVM_LIBC_HDR_TYPES_CLOCKID_T_H
-
-#ifdef LIBC_FULL_BUILD
-
-#include "include/llvm-libc-types/clockid_t.h"
-
-#else // Overlay mode
-
-#include <sys/types.h>
-
-#endif // LLVM_LIBC_FULL_BUILD
-
-#endif // LLVM_LIBC_HDR_TYPES_CLOCKID_T_H
diff --git a/libc/hdr/types/struct_timeval.h b/libc/hdr/types/struct_timeval.h
deleted file mode 100644
index 8fc321a52d71..000000000000
--- a/libc/hdr/types/struct_timeval.h
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Proxy for struct timeval  ----------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_LIBC_HDR_TYPES_STRUCT_TIMEVAL_H
-#define LLVM_LIBC_HDR_TYPES_STRUCT_TIMEVAL_H
-
-#ifdef LIBC_FULL_BUILD
-
-#include "include/llvm-libc-types/struct_timeval.h"
-
-#else
-
-#include <sys/time.h>
-
-#endif // LIBC_FULL_BUILD
-
-#endif // LLVM_LIBC_HDR_TYPES_STRUCT_TIMEVAL_H
diff --git a/libc/hdr/types/suseconds_t.h b/libc/hdr/types/suseconds_t.h
deleted file mode 100644
index 72e54a965f75..000000000000
--- a/libc/hdr/types/suseconds_t.h
+++ /dev/null
@@ -1,22 +0,0 @@
-//===-- Proxy for suseconds_t ---------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_HDR_TIMES_SUSECONDS_T_H
-#define LLVM_LIBC_HDR_TIMES_SUSECONDS_T_H
-
-#ifdef LIBC_FULL_BUILD
-
-#include "include/llvm-libc-types/suseconds_t.h"
-
-#else // Overlay mode
-
-#include <sys/types.h>
-
-#endif // LLVM_LIBC_FULL_BUILD
-
-#endif // #ifndef LLVM_LIBC_HDR_TIMES_SUSECONDS_T_H
diff --git a/libc/hdr/types/time_t.h b/libc/hdr/types/time_t.h
deleted file mode 100644
index fc9a1506a2cd..000000000000
--- a/libc/hdr/types/time_t.h
+++ /dev/null
@@ -1,22 +0,0 @@
-//===-- Proxy for time_t --------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_HDR_TYPES_TIME_T_H
-#define LLVM_LIBC_HDR_TYPES_TIME_T_H
-
-#ifdef LIBC_FULL_BUILD
-
-#include "include/llvm-libc-types/time_t.h"
-
-#else // Overlay mode
-
-#include <time.h>
-
-#endif // LLVM_LIBC_FULL_BUILD
-
-#endif // LLVM_LIBC_HDR_TYPES_TIME_T_H
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 32d693ec6a26..dcae55e050bf 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -281,5 +281,3 @@ add_subdirectory(File)
 add_subdirectory(HashTable)
 
 add_subdirectory(fixed_point)
-
-add_subdirectory(time)
diff --git a/libc/src/__support/time/CMakeLists.txt b/libc/src/__support/time/CMakeLists.txt
deleted file mode 100644
index 36ce4f9dadb2..000000000000
--- a/libc/src/__support/time/CMakeLists.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
-  add_subdirectory(${LIBC_TARGET_OS})
-endif()
-
-add_object_library(
-  clock_gettime
-  ALIAS
-  DEPENDS
-    .${LIBC_TARGET_OS}.clock_gettime
-)
-
-add_header_library(
-  units
-  HDRS
-    units.h
-  DEPENDS
-    libc.src.__support.common
-    libc.hdr.types.time_t
-)
diff --git a/libc/src/__support/time/clock_gettime.h b/libc/src/__support/time/clock_gettime.h
deleted file mode 100644
index 0655ccdc0028..000000000000
--- a/libc/src/__support/time/clock_gettime.h
+++ /dev/null
@@ -1,23 +0,0 @@
-//===--- clock_gettime internal implementation ------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_GETTIME_H
-#define LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_GETTIME_H
-#include "hdr/types/clockid_t.h"
-#include "hdr/types/struct_timespec.h"
-#include "src/__support/common.h"
-
-#include "src/__support/error_or.h"
-
-namespace LIBC_NAMESPACE {
-namespace internal {
-ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts);
-}
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_TIME_CLOCK_GETTIME_H
diff --git a/libc/src/__support/time/linux/CMakeLists.txt b/libc/src/__support/time/linux/CMakeLists.txt
deleted file mode 100644
index 034fa317ff6d..000000000000
--- a/libc/src/__support/time/linux/CMakeLists.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-add_object_library(
-  clock_gettime
-  HDRS
-    ../clock_gettime.h
-  SRCS
-    clock_gettime.cpp
-  DEPENDS
-    libc.include.sys_syscall
-    libc.hdr.types.struct_timespec
-    libc.hdr.types.clockid_t
-    libc.src.__support.common
-    libc.src.__support.error_or
-    libc.src.__support.OSUtil.osutil
-)
diff --git a/libc/src/__support/time/units.h b/libc/src/__support/time/units.h
deleted file mode 100644
index f6bd19f9b139..000000000000
--- a/libc/src/__support/time/units.h
+++ /dev/null
@@ -1,38 +0,0 @@
-//===--- Time units conversion ----------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_UNITS_H
-#define LLVM_LIBC_SRC___SUPPORT_TIME_UNITS_H
-
-#include "hdr/types/time_t.h"
-#include "src/__support/common.h"
-
-namespace LIBC_NAMESPACE {
-namespace time_units {
-LIBC_INLINE constexpr time_t operator""_s_ns(unsigned long long s) {
-  return s * 1'000'000'000;
-}
-LIBC_INLINE constexpr time_t operator""_s_us(unsigned long long s) {
-  return s * 1'000'000;
-}
-LIBC_INLINE constexpr time_t operator""_s_ms(unsigned long long s) {
-  return s * 1'000;
-}
-LIBC_INLINE constexpr time_t operator""_ms_ns(unsigned long long ms) {
-  return ms * 1'000'000;
-}
-LIBC_INLINE constexpr time_t operator""_ms_us(unsigned long long ms) {
-  return ms * 1'000;
-}
-LIBC_INLINE constexpr time_t operator""_us_ns(unsigned long long us) {
-  return us * 1'000;
-}
-} // namespace time_units
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_TIME_UNITS_H
diff --git a/libc/src/time/clock.h b/libc/src/time/clock.h
index f5d14d036e13..d4af7656644a 100644
--- a/libc/src/time/clock.h
+++ b/libc/src/time/clock.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_TIME_CLOCK_H
 #define LLVM_LIBC_SRC_TIME_CLOCK_H
 
-#include "hdr/types/clock_t.h"
+#include <time.h>
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/time/clock_gettime.h b/libc/src/time/clock_gettime.h
index 48e81a355429..72e2e1949feb 100644
--- a/libc/src/time/clock_gettime.h
+++ b/libc/src/time/clock_gettime.h
@@ -9,12 +9,11 @@
 #ifndef LLVM_LIBC_SRC_TIME_CLOCK_GETTIME_H
 #define LLVM_LIBC_SRC_TIME_CLOCK_GETTIME_H
 
-#include "hdr/types/clockid_t.h"
-#include "hdr/types/struct_timespec.h"
+#include <time.h>
 
 namespace LIBC_NAMESPACE {
 
-int clock_gettime(clockid_t clockid, timespec *tp);
+int clock_gettime(clockid_t clockid, struct timespec *tp);
 
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/src/time/gettimeofday.h b/libc/src/time/gettimeofday.h
index 62ee31edcad6..880b94cee731 100644
--- a/libc/src/time/gettimeofday.h
+++ b/libc/src/time/gettimeofday.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_TIME_GETTIMEOFDAY_H
 #define LLVM_LIBC_SRC_TIME_GETTIMEOFDAY_H
 
-#include "hdr/types/struct_timeval.h"
+#include <time.h>
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/time/linux/CMakeLists.txt b/libc/src/time/linux/CMakeLists.txt
index 8a0e6b04b66e..df79bf598626 100644
--- a/libc/src/time/linux/CMakeLists.txt
+++ b/libc/src/time/linux/CMakeLists.txt
@@ -5,9 +5,9 @@ add_entrypoint_object(
   HDRS
     ../time_func.h
   DEPENDS
-    libc.hdr.time_macros
-    libc.hdr.types.time_t
-    libc.src.__support.time.clock_gettime
+    libc.include.time
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
 
@@ -18,11 +18,10 @@ add_entrypoint_object(
   HDRS
     ../clock.h
   DEPENDS
-    libc.hdr.time_macros
-    libc.hdr.types.clock_t
-    libc.src.__support.time.units
-    libc.src.__support.time.clock_gettime
+    libc.include.time
+    libc.include.sys_syscall
     libc.src.__support.CPP.limits
+    libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
 
@@ -33,10 +32,10 @@ add_entrypoint_object(
   HDRS
     ../nanosleep.h
   DEPENDS
-    libc.hdr.types.struct_timespec
+    libc.include.time
     libc.include.sys_syscall
-    libc.src.__support.OSUtil.osutil
     libc.src.__support.CPP.limits
+    libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
 
@@ -47,9 +46,9 @@ add_entrypoint_object(
   HDRS
     ../clock_gettime.h
   DEPENDS
-    libc.hdr.types.clockid_t
-    libc.hdr.types.struct_timespec
-    libc.src.__support.time.clock_gettime
+    libc.include.time
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
 
@@ -60,9 +59,8 @@ add_entrypoint_object(
   HDRS
     ../gettimeofday.h
   DEPENDS
-    libc.hdr.time_macros
-    libc.hdr.types.suseconds_t
-    libc.src.__support.time.clock_gettime
-    libc.src.__support.time.units
+    libc.include.time
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
diff --git a/libc/src/time/linux/clock.cpp b/libc/src/time/linux/clock.cpp
index fc48e2792747..1e95f0526bc9 100644
--- a/libc/src/time/linux/clock.cpp
+++ b/libc/src/time/linux/clock.cpp
@@ -7,19 +7,21 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/time/clock.h"
-#include "hdr/time_macros.h"
+
 #include "src/__support/CPP/limits.h"
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
-#include "src/__support/time/clock_gettime.h"
-#include "src/__support/time/units.h"
 #include "src/errno/libc_errno.h"
+#include "src/time/linux/clockGetTimeImpl.h"
+
+#include <sys/syscall.h> // For syscall numbers.
+#include <time.h>
 
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(clock_t, clock, ()) {
-  using namespace time_units;
   struct timespec ts;
-  auto result = internal::clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
+  auto result = internal::clock_gettimeimpl(CLOCK_PROCESS_CPUTIME_ID, &ts);
   if (!result.has_value()) {
     libc_errno = result.error();
     return -1;
@@ -32,15 +34,15 @@ LLVM_LIBC_FUNCTION(clock_t, clock, ()) {
       cpp::numeric_limits<clock_t>::max() / CLOCKS_PER_SEC;
   if (ts.tv_sec > CLOCK_SECS_MAX)
     return clock_t(-1);
-  if (ts.tv_nsec / 1_s_ns > CLOCK_SECS_MAX - ts.tv_sec)
+  if (ts.tv_nsec / 1000000000 > CLOCK_SECS_MAX - ts.tv_sec)
     return clock_t(-1);
 
   // For the integer computation converting tv_nsec to clocks to work
   // correctly, we want CLOCKS_PER_SEC to be less than 1000000000.
-  static_assert(1_s_ns > CLOCKS_PER_SEC,
-                "Expected CLOCKS_PER_SEC to be less than 1'000'000'000.");
+  static_assert(1000000000 > CLOCKS_PER_SEC,
+                "Expected CLOCKS_PER_SEC to be less than 1000000000.");
   return clock_t(ts.tv_sec * CLOCKS_PER_SEC +
-                 ts.tv_nsec / (1_s_ns / CLOCKS_PER_SEC));
+                 ts.tv_nsec / (1000000000 / CLOCKS_PER_SEC));
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/__support/time/linux/clock_gettime.cpp b/libc/src/time/linux/clockGetTimeImpl.h
similarity index 64%
rename from libc/src/__support/time/linux/clock_gettime.cpp
rename to libc/src/time/linux/clockGetTimeImpl.h
index 6a131df9ba59..8c8c9fcf845c 100644
--- a/libc/src/__support/time/linux/clock_gettime.cpp
+++ b/libc/src/time/linux/clockGetTimeImpl.h
@@ -1,4 +1,4 @@
-//===--- clock_gettime linux implementation ---------------------*- C++ -*-===//
+//===- Linux implementation of the POSIX clock_gettime function -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,14 +6,23 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
-#define LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
-#include "src/__support/time/clock_gettime.h"
-#include "src/__support/OSUtil/syscall.h"
-#include <sys/syscall.h>
+#ifndef LLVM_LIBC_SRC_TIME_LINUX_CLOCKGETTIMEIMPL_H
+#define LLVM_LIBC_SRC_TIME_LINUX_CLOCKGETTIMEIMPL_H
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/common.h"
+#include "src/__support/error_or.h"
+#include "src/errno/libc_errno.h"
+
+#include <stdint.h>      // For int64_t.
+#include <sys/syscall.h> // For syscall numbers.
+#include <time.h>
+
 namespace LIBC_NAMESPACE {
 namespace internal {
-ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts) {
+
+LIBC_INLINE ErrorOr<int> clock_gettimeimpl(clockid_t clockid,
+                                           struct timespec *ts) {
 #if SYS_clock_gettime
   int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_clock_gettime,
                                               static_cast<long>(clockid),
@@ -36,4 +45,4 @@ ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts) {
 } // namespace internal
 } // namespace LIBC_NAMESPACE
 
-#endif // LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
+#endif // LLVM_LIBC_SRC_TIME_LINUX_CLOCKGETTIMEIMPL_H
diff --git a/libc/src/time/linux/clock_gettime.cpp b/libc/src/time/linux/clock_gettime.cpp
index 920363e85e06..47e974a866c8 100644
--- a/libc/src/time/linux/clock_gettime.cpp
+++ b/libc/src/time/linux/clock_gettime.cpp
@@ -7,16 +7,21 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/time/clock_gettime.h"
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
-#include "src/__support/time/clock_gettime.h"
 #include "src/errno/libc_errno.h"
+#include "src/time/linux/clockGetTimeImpl.h"
+
+#include <sys/syscall.h> // For syscall numbers.
+#include <time.h>
 
 namespace LIBC_NAMESPACE {
 
 // TODO(michaelrj): Move this into time/linux with the other syscalls.
 LLVM_LIBC_FUNCTION(int, clock_gettime,
                    (clockid_t clockid, struct timespec *ts)) {
-  auto result = internal::clock_gettime(clockid, ts);
+  auto result = internal::clock_gettimeimpl(clockid, ts);
 
   // A negative return value indicates an error with the magnitude of the
   // value being the error code.
diff --git a/libc/src/time/linux/gettimeofday.cpp b/libc/src/time/linux/gettimeofday.cpp
index c7bcd45e01fa..07ab4d579176 100644
--- a/libc/src/time/linux/gettimeofday.cpp
+++ b/libc/src/time/linux/gettimeofday.cpp
@@ -7,24 +7,24 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/time/gettimeofday.h"
-#include "hdr/time_macros.h"
-#include "hdr/types/suseconds_t.h"
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
-#include "src/__support/time/clock_gettime.h"
-#include "src/__support/time/units.h"
 #include "src/errno/libc_errno.h"
+#include "src/time/linux/clockGetTimeImpl.h"
+
+#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE {
 
 // TODO(michaelrj): Move this into time/linux with the other syscalls.
 LLVM_LIBC_FUNCTION(int, gettimeofday,
                    (struct timeval * tv, [[maybe_unused]] void *unused)) {
-  using namespace time_units;
   if (tv == nullptr)
     return 0;
 
   struct timespec ts;
-  auto result = internal::clock_gettime(CLOCK_REALTIME, &ts);
+  auto result = internal::clock_gettimeimpl(CLOCK_REALTIME, &ts);
 
   // A negative return value indicates an error with the magnitude of the
   // value being the error code.
@@ -34,7 +34,7 @@ LLVM_LIBC_FUNCTION(int, gettimeofday,
   }
 
   tv->tv_sec = ts.tv_sec;
-  tv->tv_usec = static_cast<suseconds_t>(ts.tv_nsec / 1_us_ns);
+  tv->tv_usec = static_cast<suseconds_t>(ts.tv_nsec / 1000);
   return 0;
 }
 
diff --git a/libc/src/time/linux/time.cpp b/libc/src/time/linux/time.cpp
index 93d5d7362764..e286fae095b2 100644
--- a/libc/src/time/linux/time.cpp
+++ b/libc/src/time/linux/time.cpp
@@ -6,18 +6,22 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "hdr/time_macros.h"
+#include "src/time/time_func.h"
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
-#include "src/__support/time/clock_gettime.h"
 #include "src/errno/libc_errno.h"
-#include "src/time/time_func.h"
+#include "src/time/linux/clockGetTimeImpl.h"
+
+#include <sys/syscall.h> // For syscall numbers.
+#include <time.h>
 
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(time_t, time, (time_t * tp)) {
   // TODO: Use the Linux VDSO to fetch the time and avoid the syscall.
   struct timespec ts;
-  auto result = internal::clock_gettime(CLOCK_REALTIME, &ts);
+  auto result = internal::clock_gettimeimpl(CLOCK_REALTIME, &ts);
   if (!result.has_value()) {
     libc_errno = result.error();
     return -1;
diff --git a/libc/src/time/nanosleep.h b/libc/src/time/nanosleep.h
index 2309666b2304..757394232c07 100644
--- a/libc/src/time/nanosleep.h
+++ b/libc/src/time/nanosleep.h
@@ -9,11 +9,11 @@
 #ifndef LLVM_LIBC_SRC_TIME_NANOSLEEP_H
 #define LLVM_LIBC_SRC_TIME_NANOSLEEP_H
 
-#include "hdr/types/struct_timespec.h"
+#include <time.h>
 
 namespace LIBC_NAMESPACE {
 
-int nanosleep(const timespec *req, timespec *rem);
+int nanosleep(const struct timespec *req, struct timespec *rem);
 
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/src/time/time_func.h b/libc/src/time/time_func.h
index 2a5239220942..beb02020b575 100644
--- a/libc/src/time/time_func.h
+++ b/libc/src/time/time_func.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_TIME_TIME_FUNC_H
 #define LLVM_LIBC_SRC_TIME_TIME_FUNC_H
 
-#include "hdr/types/time_t.h"
+#include <time.h>
 
 // Note this header file is named time_func.h to avoid conflicts with the
 // public header file time.h.
-- 
GitLab


From dfff57e751f6bae12172a1a246e1f8b33db042f8 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 9 May 2024 14:39:06 -0700
Subject: [PATCH 0349/1206] [RISCV] Add isel special case for (and (shl X, c2),
 c1) -> (slli_uw (srli x, c3-c2), c3). (#91638)

Where c1 is a shifted mask with 32 set bits and c3 trailing zeros.
---
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 13 +++++++
 llvm/test/CodeGen/RISCV/rv64zba.ll          | 40 ++++++++++++++++++---
 2 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index e73a3af92af6..3c4646b95715 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -1416,6 +1416,19 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
           ReplaceNode(Node, SLLI);
           return;
         }
+
+        // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
+        if (C2 < Trailing && Leading + Trailing == 32 && OneUseOrZExtW &&
+            Subtarget->hasStdExtZba()) {
+          SDNode *SRLI = CurDAG->getMachineNode(
+              RISCV::SRLI, DL, VT, X,
+              CurDAG->getTargetConstant(Trailing - C2, DL, VT));
+          SDNode *SLLI_UW = CurDAG->getMachineNode(
+              RISCV::SLLI_UW, DL, VT, SDValue(SRLI, 0),
+              CurDAG->getTargetConstant(Trailing, DL, VT));
+          ReplaceNode(Node, SLLI_UW);
+          return;
+        }
       }
     }
 
diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll
index 8fe221f2a297..dc93c0215a25 100644
--- a/llvm/test/CodeGen/RISCV/rv64zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zba.ll
@@ -2866,8 +2866,7 @@ define ptr @gep_lshr_i32(ptr %0, i64 %1) {
 ;
 ; RV64ZBA-LABEL: gep_lshr_i32:
 ; RV64ZBA:       # %bb.0: # %entry
-; RV64ZBA-NEXT:    slli a1, a1, 2
-; RV64ZBA-NEXT:    srli a1, a1, 4
+; RV64ZBA-NEXT:    srli a1, a1, 2
 ; RV64ZBA-NEXT:    slli.uw a1, a1, 4
 ; RV64ZBA-NEXT:    sh2add a1, a1, a1
 ; RV64ZBA-NEXT:    add a0, a0, a1
@@ -2891,8 +2890,7 @@ define i64 @srli_slliw(i64 %1) {
 ;
 ; RV64ZBA-LABEL: srli_slliw:
 ; RV64ZBA:       # %bb.0: # %entry
-; RV64ZBA-NEXT:    slli a0, a0, 2
-; RV64ZBA-NEXT:    srli a0, a0, 4
+; RV64ZBA-NEXT:    srli a0, a0, 2
 ; RV64ZBA-NEXT:    slli.uw a0, a0, 4
 ; RV64ZBA-NEXT:    ret
 entry:
@@ -2902,6 +2900,40 @@ entry:
   ret i64 %4
 }
 
+define i64 @srli_slliw_canonical(i64 %0) {
+; RV64I-LABEL: srli_slliw_canonical:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    slli a0, a0, 2
+; RV64I-NEXT:    li a1, 1
+; RV64I-NEXT:    slli a1, a1, 36
+; RV64I-NEXT:    addi a1, a1, -16
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: srli_slliw_canonical:
+; RV64ZBA:       # %bb.0: # %entry
+; RV64ZBA-NEXT:    srli a0, a0, 2
+; RV64ZBA-NEXT:    slli.uw a0, a0, 4
+; RV64ZBA-NEXT:    ret
+entry:
+  %1 = shl i64 %0, 2
+  %2 = and i64 %1, 68719476720
+  ret i64 %2
+}
+
+; Make sure we don't accidentally use slli.uw with a shift of 32.
+define i64 @srli_slliuw_negative_test(i64 %0) {
+; CHECK-LABEL: srli_slliuw_negative_test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    srli a0, a0, 6
+; CHECK-NEXT:    slli a0, a0, 32
+; CHECK-NEXT:    ret
+entry:
+  %1 = lshr i64 %0, 6
+  %2 = shl i64 %1, 32
+  ret i64 %2
+}
+
 define i64 @srli_slli_i16(i64 %1) {
 ; CHECK-LABEL: srli_slli_i16:
 ; CHECK:       # %bb.0: # %entry
-- 
GitLab


From a3457369cd12b093185a5bda3443e08a4390f3ed Mon Sep 17 00:00:00 2001
From: Min Hsu <min.hsu@sifive.com>
Date: Thu, 9 May 2024 14:38:28 -0700
Subject: [PATCH 0350/1206] [Orc] Fix `-Wsign-compare` warnings in unittest

Multiple compares against `LookupsCompleted`, which is effectively an
unsigned long, with constant signed integer were throwing -Wsign-compare
warnings.

This is effectively NFC.
---
 llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
index a6fa69f97fcb..53a74c833eb3 100644
--- a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
@@ -1170,7 +1170,7 @@ TEST_F(CoreAPIsStandardTest, ErrorFromAutoSuspendedAsynchronousGeneratorTest) {
       },
       NoDependenciesToRegister);
 
-  EXPECT_EQ(LookupsCompleted, 0);
+  EXPECT_EQ(LookupsCompleted, 0U);
 
   // Suspend the first lookup.
   auto LS1 = std::move(G.takeLookup().LS);
@@ -1185,7 +1185,7 @@ TEST_F(CoreAPIsStandardTest, ErrorFromAutoSuspendedAsynchronousGeneratorTest) {
       },
       NoDependenciesToRegister);
 
-  EXPECT_EQ(LookupsCompleted, 0);
+  EXPECT_EQ(LookupsCompleted, 0U);
 
   // Unsuspend the first lookup.
   LS1.continueLookup(make_error<StringError>("boom", inconvertibleErrorCode()));
@@ -1194,7 +1194,7 @@ TEST_F(CoreAPIsStandardTest, ErrorFromAutoSuspendedAsynchronousGeneratorTest) {
   G.takeLookup().LS.continueLookup(
       make_error<StringError>("boom", inconvertibleErrorCode()));
 
-  EXPECT_EQ(LookupsCompleted, 2);
+  EXPECT_EQ(LookupsCompleted, 2U);
 }
 
 TEST_F(CoreAPIsStandardTest, BlockedGeneratorAutoSuspensionTest) {
-- 
GitLab


From 8466480bdad9d1ef858329ec51cd910c419036a0 Mon Sep 17 00:00:00 2001
From: Min Hsu <min.hsu@sifive.com>
Date: Thu, 9 May 2024 14:42:52 -0700
Subject: [PATCH 0351/1206] [ProfData] Remove unused variable in unittest

Removed unused `VTables` in unittests/ProfileData/InstrProfTest.cpp.

NFC.
---
 llvm/unittests/ProfileData/InstrProfTest.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index 8f2c5aee1819..924d848176e7 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1782,7 +1782,6 @@ TEST(SymtabTest, instr_prof_symtab_module_test) {
     EXPECT_THAT(PGOFuncName.str(), EndsWith(Funcs[I].str()));
   }
 
-  StringRef VTables[] = {"ExternalGV", "LocalGV"};
   for (auto [VTableName, PGOName] : {std::pair{"ExternalGV", "ExternalGV"},
                                      {"LocalGV", "MyModule.cpp;LocalGV"}}) {
     GlobalVariable *GV =
-- 
GitLab


From 04ce10357b485e5e03480b1ca2e91e75c50b1fef Mon Sep 17 00:00:00 2001
From: MaheshRavishankar <1663364+MaheshRavishankar@users.noreply.github.com>
Date: Thu, 9 May 2024 14:54:38 -0700
Subject: [PATCH 0352/1206] [mlir][SCF] Avoid generating unnecessary div/rem
 operations during coalescing (#91562)

When coalescing is some of the loops are unit-trip we can avoid
generating div/rem instructions during delinearization. Ideally we could
use some thing like `affine.delinearize` to handle this but tthat causes
dependence issues.
---
 mlir/lib/Dialect/SCF/Utils/Utils.cpp          | 57 +++++++++++---
 .../Dialect/SCF/transform-op-coalesce.mlir    | 77 +++++++++++++++++++
 2 files changed, 124 insertions(+), 10 deletions(-)

diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
index 9279081cfd45..6658cca03eba 100644
--- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
@@ -544,11 +544,24 @@ static void denormalizeInductionVariable(RewriterBase &rewriter, Location loc,
 static Value getProductOfIntsOrIndexes(RewriterBase &rewriter, Location loc,
                                        ArrayRef<Value> values) {
   assert(!values.empty() && "unexpected empty list");
-  Value productOf = values.front();
-  for (auto v : values.drop_front()) {
-    productOf = rewriter.create<arith::MulIOp>(loc, productOf, v);
+  std::optional<Value> productOf;
+  for (auto v : values) {
+    auto vOne = getConstantIntValue(v);
+    if (vOne && vOne.value() == 1)
+      continue;
+    if (productOf)
+      productOf =
+          rewriter.create<arith::MulIOp>(loc, productOf.value(), v).getResult();
+    else
+      productOf = v;
   }
-  return productOf;
+  if (!productOf) {
+    productOf = rewriter
+                    .create<arith::ConstantOp>(
+                        loc, rewriter.getOneAttr(values.front().getType()))
+                    .getResult();
+  }
+  return productOf.value();
 }
 
 /// For each original loop, the value of the
@@ -562,19 +575,43 @@ static Value getProductOfIntsOrIndexes(RewriterBase &rewriter, Location loc,
 static std::pair<SmallVector<Value>, SmallPtrSet<Operation *, 2>>
 delinearizeInductionVariable(RewriterBase &rewriter, Location loc,
                              Value linearizedIv, ArrayRef<Value> ubs) {
-  Value previous = linearizedIv;
   SmallVector<Value> delinearizedIvs(ubs.size());
   SmallPtrSet<Operation *, 2> preservedUsers;
-  for (unsigned i = 0, e = ubs.size(); i < e; ++i) {
-    unsigned idx = ubs.size() - i - 1;
-    if (i != 0) {
+
+  llvm::BitVector isUbOne(ubs.size());
+  for (auto [index, ub] : llvm::enumerate(ubs)) {
+    auto ubCst = getConstantIntValue(ub);
+    if (ubCst && ubCst.value() == 1)
+      isUbOne.set(index);
+  }
+
+  // Prune the lead ubs that are all ones.
+  unsigned numLeadingOneUbs = 0;
+  for (auto [index, ub] : llvm::enumerate(ubs)) {
+    if (!isUbOne.test(index)) {
+      break;
+    }
+    delinearizedIvs[index] = rewriter.create<arith::ConstantOp>(
+        loc, rewriter.getZeroAttr(ub.getType()));
+    numLeadingOneUbs++;
+  }
+
+  Value previous = linearizedIv;
+  for (unsigned i = numLeadingOneUbs, e = ubs.size(); i < e; ++i) {
+    unsigned idx = ubs.size() - (i - numLeadingOneUbs) - 1;
+    if (i != numLeadingOneUbs && !isUbOne.test(idx + 1)) {
       previous = rewriter.create<arith::DivSIOp>(loc, previous, ubs[idx + 1]);
       preservedUsers.insert(previous.getDefiningOp());
     }
     Value iv = previous;
     if (i != e - 1) {
-      iv = rewriter.create<arith::RemSIOp>(loc, previous, ubs[idx]);
-      preservedUsers.insert(iv.getDefiningOp());
+      if (!isUbOne.test(idx)) {
+        iv = rewriter.create<arith::RemSIOp>(loc, previous, ubs[idx]);
+        preservedUsers.insert(iv.getDefiningOp());
+      } else {
+        iv = rewriter.create<arith::ConstantOp>(
+            loc, rewriter.getZeroAttr(ubs[idx].getType()));
+      }
     }
     delinearizedIvs[idx] = iv;
   }
diff --git a/mlir/test/Dialect/SCF/transform-op-coalesce.mlir b/mlir/test/Dialect/SCF/transform-op-coalesce.mlir
index 4dc3e4ea0ef4..6fcd727621ba 100644
--- a/mlir/test/Dialect/SCF/transform-op-coalesce.mlir
+++ b/mlir/test/Dialect/SCF/transform-op-coalesce.mlir
@@ -299,3 +299,80 @@ module attributes {transform.with_named_sequence} {
 //  CHECK-NOT:       scf.for
 //      CHECK:   transform.named_sequence
 
+// -----
+
+// Check avoiding generating unnecessary operations while collapsing trip-1 loops.
+func.func @trip_one_loops(%arg0 : tensor<?x?xf32>, %arg1 : index, %arg2 : index) -> tensor<?x?xf32> {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %0 = scf.for %iv0 = %c0 to %c1 step %c1 iter_args(%iter0 = %arg0) -> tensor<?x?xf32> {
+    %1 = scf.for %iv1 = %c0 to %c1 step %c1 iter_args(%iter1 = %iter0) -> tensor<?x?xf32> {
+      %2 = scf.for %iv2 = %c0 to %arg1 step %c1 iter_args(%iter2 = %iter1) -> tensor<?x?xf32> {
+        %3 = scf.for %iv3 = %c0 to %c1 step %c1 iter_args(%iter3 = %iter2) -> tensor<?x?xf32> {
+          %4 = scf.for %iv4 = %c0 to %arg2 step %c1 iter_args(%iter4 = %iter3) -> tensor<?x?xf32> {
+            %5 = "some_use"(%iter4, %iv0, %iv1, %iv2, %iv3, %iv4)
+              : (tensor<?x?xf32>, index, index, index, index, index) -> (tensor<?x?xf32>)
+            scf.yield %5 : tensor<?x?xf32>
+          }
+          scf.yield %4 : tensor<?x?xf32>
+        }
+        scf.yield %3 : tensor<?x?xf32>
+      }
+      scf.yield %2 : tensor<?x?xf32>
+    }
+    scf.yield %1 : tensor<?x?xf32>
+  } {coalesce}
+  return %0 : tensor<?x?xf32>
+}
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["scf.for"]} attributes {coalesce} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.cast %0 : !transform.any_op to !transform.op<"scf.for">
+    %2 = transform.loop.coalesce %1 : (!transform.op<"scf.for">) -> (!transform.op<"scf.for">)
+    transform.yield
+  }
+}
+// CHECK-LABEL: func @trip_one_loops
+//  CHECK-SAME:     , %[[ARG1:.+]]: index,
+//  CHECK-SAME:     %[[ARG2:.+]]: index)
+//   CHECK-DAG:   %[[C0:.+]] = arith.constant 0 : index
+//   CHECK-DAG:   %[[C1:.+]] = arith.constant 1 : index
+//       CHECK:   %[[UB:.+]] = arith.muli %[[ARG1]], %[[ARG2]]
+//       CHECK:   scf.for %[[IV:.+]] = %[[C0]] to %[[UB]] step %[[C1]]
+//       CHECK:     %[[IV1:.+]] = arith.remsi %[[IV]], %[[ARG2]]
+//       CHECK:     %[[IV2:.+]] = arith.divsi %[[IV]], %[[ARG2]]
+//       CHECK:     "some_use"(%{{[a-zA-Z0-9]+}}, %[[C0]], %[[C0]], %[[IV2]], %[[C0]], %[[IV1]])
+
+// -----
+
+// Check generating no instructions when all except one loops is non unit-trip.
+func.func @all_outer_trip_one(%arg0 : tensor<?x?xf32>, %arg1 : index) -> tensor<?x?xf32> {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %0 = scf.for %iv0 = %c0 to %c1 step %c1 iter_args(%iter0 = %arg0) -> tensor<?x?xf32> {
+    %1 = scf.for %iv1 = %c0 to %c1 step %c1 iter_args(%iter1 = %iter0) -> tensor<?x?xf32> {
+      %2 = scf.for %iv2 = %c0 to %arg1 step %c1 iter_args(%iter2 = %iter1) -> tensor<?x?xf32> {
+        %3 = "some_use"(%iter2, %iv0, %iv1, %iv2)
+          : (tensor<?x?xf32>, index, index, index) -> (tensor<?x?xf32>)
+        scf.yield %3 : tensor<?x?xf32>
+      }
+      scf.yield %2 : tensor<?x?xf32>
+    }
+    scf.yield %1 : tensor<?x?xf32>
+  } {coalesce}
+  return %0 : tensor<?x?xf32>
+}
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["scf.for"]} attributes {coalesce} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.cast %0 : !transform.any_op to !transform.op<"scf.for">
+    %2 = transform.loop.coalesce %1 : (!transform.op<"scf.for">) -> (!transform.op<"scf.for">)
+    transform.yield
+  }
+}
+// CHECK-LABEL: func @all_outer_trip_one
+//  CHECK-SAME:     , %[[ARG1:.+]]: index)
+//   CHECK-DAG:   %[[C0:.+]] = arith.constant 0 : index
+//   CHECK-DAG:   %[[C1:.+]] = arith.constant 1 : index
+//       CHECK:   scf.for %[[IV:.+]] = %[[C0]] to %[[ARG1]] step %[[C1]]
+//       CHECK:     "some_use"(%{{[a-zA-Z0-9]+}}, %[[C0]], %[[C0]], %[[IV]])
-- 
GitLab


From ba66dfb11bcaef5e0dc21358b3712b491d61d020 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Fri, 10 May 2024 01:56:14 +0400
Subject: [PATCH 0353/1206] [lldb] Fixed SyntaxWarning: invalid escape sequence
 \[ \d \s (#91146)

Reproduced with Python 3.12.3
---
 .../lldbsuite/test/tools/lldb-server/gdbremote_testcase.py    | 4 ++--
 .../lldbsuite/test/tools/lldb-server/lldbgdbserverutils.py    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
index 75522158b322..8c8e4abed0b4 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
@@ -130,9 +130,9 @@ class GdbRemoteTestCaseBase(Base, metaclass=GdbRemoteTestCaseFactory):
         self.stub_sends_two_stop_notifications_on_kill = False
         if configuration.lldb_platform_url:
             if configuration.lldb_platform_url.startswith("unix-"):
-                url_pattern = "(.+)://\[?(.+?)\]?/.*"
+                url_pattern = r"(.+)://\[?(.+?)\]?/.*"
             else:
-                url_pattern = "(.+)://(.+):\d+"
+                url_pattern = r"(.+)://(.+):\d+"
             scheme, host = re.match(
                 url_pattern, configuration.lldb_platform_url
             ).groups()
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/lldbgdbserverutils.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/lldbgdbserverutils.py
index 61c5c3a7c865..d1a4119bac78 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/lldbgdbserverutils.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/lldbgdbserverutils.py
@@ -50,7 +50,7 @@ def get_debugserver_exe():
 
 
 _LOG_LINE_REGEX = re.compile(
-    r"^(lldb-server|debugserver)\s+<\s*(\d+)>" + "\s+(read|send)\s+packet:\s+(.+)$"
+    r"^(lldb-server|debugserver)\s+<\s*(\d+)>\s+(read|send)\s+packet:\s+(.+)$"
 )
 
 
-- 
GitLab


From 5d51db75e46155d0f3d70cf3253d3b075cfcf93a Mon Sep 17 00:00:00 2001
From: asraa <asraa@google.com>
Date: Thu, 9 May 2024 16:57:59 -0500
Subject: [PATCH 0354/1206] [mlir][affine] Use alias analysis to redetermine
 intervening memory effects in affine-scalrep (#90859)

This fixes a TODO to use alias analysis to determine whether a read op
intervenes between two write operations to the same memref.

Signed-off-by: Asra <asraa@google.com>
---
 mlir/include/mlir/Dialect/Affine/Utils.h      |  7 ++-
 .../Transforms/AffineScalarReplacement.cpp    |  4 +-
 mlir/lib/Dialect/Affine/Utils/Utils.cpp       | 52 ++++++++++---------
 mlir/test/Dialect/Affine/scalrep.mlir         | 18 +++++++
 4 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Affine/Utils.h b/mlir/include/mlir/Dialect/Affine/Utils.h
index 67c7a964feef..7f25db029781 100644
--- a/mlir/include/mlir/Dialect/Affine/Utils.h
+++ b/mlir/include/mlir/Dialect/Affine/Utils.h
@@ -13,6 +13,7 @@
 #ifndef MLIR_DIALECT_AFFINE_UTILS_H
 #define MLIR_DIALECT_AFFINE_UTILS_H
 
+#include "mlir/Analysis/AliasAnalysis.h"
 #include "mlir/Dialect/Affine/Analysis/AffineAnalysis.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/IR/OpDefinition.h"
@@ -106,7 +107,8 @@ struct VectorizationStrategy {
 /// loads and eliminate invariant affine loads; consequently, eliminate dead
 /// allocs.
 void affineScalarReplace(func::FuncOp f, DominanceInfo &domInfo,
-                         PostDominanceInfo &postDomInfo);
+                         PostDominanceInfo &postDomInfo,
+                         AliasAnalysis &analysis);
 
 /// Vectorizes affine loops in 'loops' using the n-D vectorization factors in
 /// 'vectorSizes'. By default, each vectorization factor is applied
@@ -325,7 +327,8 @@ OpFoldResult linearizeIndex(ArrayRef<OpFoldResult> multiIndex,
 /// will check if there is no write to the memory between `start` and `memOp`
 /// that would change the read within `memOp`.
 template <typename EffectType, typename T>
-bool hasNoInterveningEffect(Operation *start, T memOp);
+bool hasNoInterveningEffect(Operation *start, T memOp,
+                            llvm::function_ref<bool(Value, Value)> mayAlias);
 
 struct AffineValueExpr {
   explicit AffineValueExpr(AffineExpr e) : e(e) {}
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineScalarReplacement.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineScalarReplacement.cpp
index ed94fb690af2..707bba2f1e6f 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineScalarReplacement.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineScalarReplacement.cpp
@@ -13,6 +13,7 @@
 
 #include "mlir/Dialect/Affine/Passes.h"
 
+#include "mlir/Analysis/AliasAnalysis.h"
 #include "mlir/Dialect/Affine/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/Dominance.h"
@@ -47,5 +48,6 @@ mlir::affine::createAffineScalarReplacementPass() {
 
 void AffineScalarReplacement::runOnOperation() {
   affineScalarReplace(getOperation(), getAnalysis<DominanceInfo>(),
-                      getAnalysis<PostDominanceInfo>());
+                      getAnalysis<PostDominanceInfo>(),
+                      getAnalysis<AliasAnalysis>());
 }
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 8b8ed2578ca5..f46381403bc5 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -678,12 +678,9 @@ static bool mayHaveEffect(Operation *srcMemOp, Operation *destMemOp,
 }
 
 template <typename EffectType, typename T>
-bool mlir::affine::hasNoInterveningEffect(Operation *start, T memOp) {
-  auto isLocallyAllocated = [](Value memref) {
-    auto *defOp = memref.getDefiningOp();
-    return defOp && hasSingleEffect<MemoryEffects::Allocate>(defOp, memref);
-  };
-
+bool mlir::affine::hasNoInterveningEffect(
+    Operation *start, T memOp,
+    llvm::function_ref<bool(Value, Value)> mayAlias) {
   // A boolean representing whether an intervening operation could have impacted
   // memOp.
   bool hasSideEffect = false;
@@ -704,11 +701,8 @@ bool mlir::affine::hasNoInterveningEffect(Operation *start, T memOp) {
         // If op causes EffectType on a potentially aliasing location for
         // memOp, mark as having the effect.
         if (isa<EffectType>(effect.getEffect())) {
-          // TODO: This should be replaced with a check for no aliasing.
-          // Aliasing information should be passed to this method.
           if (effect.getValue() && effect.getValue() != memref &&
-              isLocallyAllocated(memref) &&
-              isLocallyAllocated(effect.getValue()))
+              !mayAlias(effect.getValue(), memref))
             continue;
           opMayHaveEffect = true;
           break;
@@ -832,10 +826,10 @@ bool mlir::affine::hasNoInterveningEffect(Operation *start, T memOp) {
 /// other operations will overwrite the memory loaded between the given load
 /// and store.  If such a value exists, the replaced `loadOp` will be added to
 /// `loadOpsToErase` and its memref will be added to `memrefsToErase`.
-static void forwardStoreToLoad(AffineReadOpInterface loadOp,
-                               SmallVectorImpl<Operation *> &loadOpsToErase,
-                               SmallPtrSetImpl<Value> &memrefsToErase,
-                               DominanceInfo &domInfo) {
+static void forwardStoreToLoad(
+    AffineReadOpInterface loadOp, SmallVectorImpl<Operation *> &loadOpsToErase,
+    SmallPtrSetImpl<Value> &memrefsToErase, DominanceInfo &domInfo,
+    llvm::function_ref<bool(Value, Value)> mayAlias) {
 
   // The store op candidate for forwarding that satisfies all conditions
   // to replace the load, if any.
@@ -872,7 +866,8 @@ static void forwardStoreToLoad(AffineReadOpInterface loadOp,
 
     // 4. Ensure there is no intermediate operation which could replace the
     // value in memory.
-    if (!affine::hasNoInterveningEffect<MemoryEffects::Write>(storeOp, loadOp))
+    if (!affine::hasNoInterveningEffect<MemoryEffects::Write>(storeOp, loadOp,
+                                                              mayAlias))
       continue;
 
     // We now have a candidate for forwarding.
@@ -901,7 +896,8 @@ static void forwardStoreToLoad(AffineReadOpInterface loadOp,
 template bool
 mlir::affine::hasNoInterveningEffect<mlir::MemoryEffects::Read,
                                      affine::AffineReadOpInterface>(
-    mlir::Operation *, affine::AffineReadOpInterface);
+    mlir::Operation *, affine::AffineReadOpInterface,
+    llvm::function_ref<bool(Value, Value)>);
 
 // This attempts to find stores which have no impact on the final result.
 // A writing op writeA will be eliminated if there exists an op writeB if
@@ -910,7 +906,8 @@ mlir::affine::hasNoInterveningEffect<mlir::MemoryEffects::Read,
 // 3) There is no potential read between writeA and writeB.
 static void findUnusedStore(AffineWriteOpInterface writeA,
                             SmallVectorImpl<Operation *> &opsToErase,
-                            PostDominanceInfo &postDominanceInfo) {
+                            PostDominanceInfo &postDominanceInfo,
+                            llvm::function_ref<bool(Value, Value)> mayAlias) {
 
   for (Operation *user : writeA.getMemRef().getUsers()) {
     // Only consider writing operations.
@@ -939,7 +936,8 @@ static void findUnusedStore(AffineWriteOpInterface writeA,
 
     // There cannot be an operation which reads from memory between
     // the two writes.
-    if (!affine::hasNoInterveningEffect<MemoryEffects::Read>(writeA, writeB))
+    if (!affine::hasNoInterveningEffect<MemoryEffects::Read>(writeA, writeB,
+                                                             mayAlias))
       continue;
 
     opsToErase.push_back(writeA);
@@ -955,7 +953,8 @@ static void findUnusedStore(AffineWriteOpInterface writeA,
 // 3) There is no write between loadA and loadB.
 static void loadCSE(AffineReadOpInterface loadA,
                     SmallVectorImpl<Operation *> &loadOpsToErase,
-                    DominanceInfo &domInfo) {
+                    DominanceInfo &domInfo,
+                    llvm::function_ref<bool(Value, Value)> mayAlias) {
   SmallVector<AffineReadOpInterface, 4> loadCandidates;
   for (auto *user : loadA.getMemRef().getUsers()) {
     auto loadB = dyn_cast<AffineReadOpInterface>(user);
@@ -976,7 +975,7 @@ static void loadCSE(AffineReadOpInterface loadA,
 
     // 3. There should not be a write between loadA and loadB.
     if (!affine::hasNoInterveningEffect<MemoryEffects::Write>(
-            loadB.getOperation(), loadA))
+            loadB.getOperation(), loadA, mayAlias))
       continue;
 
     // Check if two values have the same shape. This is needed for affine vector
@@ -1034,16 +1033,21 @@ static void loadCSE(AffineReadOpInterface loadA,
 // than dealloc) remain.
 //
 void mlir::affine::affineScalarReplace(func::FuncOp f, DominanceInfo &domInfo,
-                                       PostDominanceInfo &postDomInfo) {
+                                       PostDominanceInfo &postDomInfo,
+                                       AliasAnalysis &aliasAnalysis) {
   // Load op's whose results were replaced by those forwarded from stores.
   SmallVector<Operation *, 8> opsToErase;
 
   // A list of memref's that are potentially dead / could be eliminated.
   SmallPtrSet<Value, 4> memrefsToErase;
 
+  auto mayAlias = [&](Value val1, Value val2) -> bool {
+    return !aliasAnalysis.alias(val1, val2).isNo();
+  };
+
   // Walk all load's and perform store to load forwarding.
   f.walk([&](AffineReadOpInterface loadOp) {
-    forwardStoreToLoad(loadOp, opsToErase, memrefsToErase, domInfo);
+    forwardStoreToLoad(loadOp, opsToErase, memrefsToErase, domInfo, mayAlias);
   });
   for (auto *op : opsToErase)
     op->erase();
@@ -1051,7 +1055,7 @@ void mlir::affine::affineScalarReplace(func::FuncOp f, DominanceInfo &domInfo,
 
   // Walk all store's and perform unused store elimination
   f.walk([&](AffineWriteOpInterface storeOp) {
-    findUnusedStore(storeOp, opsToErase, postDomInfo);
+    findUnusedStore(storeOp, opsToErase, postDomInfo, mayAlias);
   });
   for (auto *op : opsToErase)
     op->erase();
@@ -1084,7 +1088,7 @@ void mlir::affine::affineScalarReplace(func::FuncOp f, DominanceInfo &domInfo,
   // stores. Otherwise, some stores are wrongly seen as having an intervening
   // effect.
   f.walk([&](AffineReadOpInterface loadOp) {
-    loadCSE(loadOp, opsToErase, domInfo);
+    loadCSE(loadOp, opsToErase, domInfo, mayAlias);
   });
   for (auto *op : opsToErase)
     op->erase();
diff --git a/mlir/test/Dialect/Affine/scalrep.mlir b/mlir/test/Dialect/Affine/scalrep.mlir
index 22d394bfcf09..4a99dee50a28 100644
--- a/mlir/test/Dialect/Affine/scalrep.mlir
+++ b/mlir/test/Dialect/Affine/scalrep.mlir
@@ -682,6 +682,24 @@ func.func @redundant_store_elim(%out : memref<512xf32>) {
 // CHECK-NEXT:   affine.store
 // CHECK-NEXT: }
 
+// CHECK-LABEL: func @redundant_store_elim_nonintervening
+
+func.func @redundant_store_elim_nonintervening(%in : memref<512xf32>) {
+  %cf1 = arith.constant 1.0 : f32
+  %out = memref.alloc() :  memref<512xf32>
+  affine.for %i = 0 to 16 {
+    affine.store %cf1, %out[32*%i] : memref<512xf32>
+    %0 = affine.load %in[32*%i] : memref<512xf32>
+    affine.store %0, %out[32*%i] : memref<512xf32>
+  }
+  return
+}
+
+// CHECK: affine.for
+// CHECK-NEXT:   affine.load
+// CHECK-NEXT:   affine.store
+// CHECK-NEXT: }
+
 // CHECK-LABEL: func @redundant_store_elim_fail
 
 func.func @redundant_store_elim_fail(%out : memref<512xf32>) {
-- 
GitLab


From a99cb96dfa97c04c3313cb3770b876fee20eb131 Mon Sep 17 00:00:00 2001
From: Ryosuke Niwa <rniwa@webkit.org>
Date: Thu, 9 May 2024 15:34:27 -0700
Subject: [PATCH 0355/1206] [alpha.webkit.UncountedCallArgsChecker] Allow
 trivial operator++ (#91102)

This PR adds the support for trivial operator++ implementations. T&
operator++() and T operator++(int) are trivial if the callee is trivial.

Also allow incrementing and decrementing of a POD member variable.

Also treat any __builtin_ functions as trivial.
---
 .../Checkers/WebKit/PtrTypesSemantics.cpp     | 29 ++++++-------
 .../Checkers/WebKit/uncounted-obj-arg.cpp     | 43 +++++++++++++++++++
 2 files changed, 56 insertions(+), 16 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index 6901dbb415bf..3abfa4cbb295 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -309,21 +309,8 @@ public:
   bool VisitDefaultStmt(const DefaultStmt *DS) { return VisitChildren(DS); }
 
   bool VisitUnaryOperator(const UnaryOperator *UO) {
-    // Operator '*' and '!' are allowed as long as the operand is trivial.
-    auto op = UO->getOpcode();
-    if (op == UO_Deref || op == UO_AddrOf || op == UO_LNot || op == UO_Not)
-      return Visit(UO->getSubExpr());
-
-    if (UO->isIncrementOp() || UO->isDecrementOp()) {
-      // Allow increment or decrement of a POD type.
-      if (auto *RefExpr = dyn_cast<DeclRefExpr>(UO->getSubExpr())) {
-        if (auto *Decl = dyn_cast<VarDecl>(RefExpr->getDecl()))
-          return Decl->isLocalVarDeclOrParm() &&
-                 Decl->getType().isPODType(Decl->getASTContext());
-      }
-    }
-    // Other operators are non-trivial.
-    return false;
+    // Unary operators are trivial if its operand is trivial except co_await.
+    return UO->getOpcode() != UO_Coawait && Visit(UO->getSubExpr());
   }
 
   bool VisitBinaryOperator(const BinaryOperator *BO) {
@@ -364,7 +351,7 @@ public:
 
     if (Name == "WTFCrashWithInfo" || Name == "WTFBreakpointTrap" ||
         Name == "WTFReportAssertionFailure" ||
-        Name == "compilerFenceForCrash" || Name == "__builtin_unreachable")
+        Name == "compilerFenceForCrash" || Name.find("__builtin") == 0)
       return true;
 
     return TrivialFunctionAnalysis::isTrivialImpl(Callee, Cache);
@@ -405,6 +392,16 @@ public:
     return TrivialFunctionAnalysis::isTrivialImpl(Callee, Cache);
   }
 
+  bool VisitCXXOperatorCallExpr(const CXXOperatorCallExpr *OCE) {
+    if (!checkArguments(OCE))
+      return false;
+    auto *Callee = OCE->getCalleeDecl();
+    if (!Callee)
+      return false;
+    // Recursively descend into the callee to confirm that it's trivial as well.
+    return TrivialFunctionAnalysis::isTrivialImpl(Callee, Cache);
+  }
+
   bool VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *E) {
     if (auto *Expr = E->getExpr()) {
       if (!Visit(Expr))
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
index e75d42b9f149..6ca7677511d7 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
@@ -138,11 +138,29 @@ public:
   Number(int v) : v(v) { }
   Number(double);
   Number operator+(const Number&);
+  Number& operator++() { ++v; return *this; }
+  Number operator++(int) { Number returnValue(v); ++v; return returnValue; }
   const int& value() const { return v; }
+  void someMethod();
+
 private:
   int v;
 };
 
+class ComplexNumber {
+public:
+  ComplexNumber() : real(0), complex(0) { }
+  ComplexNumber(const ComplexNumber&);
+  ComplexNumber& operator++() { real.someMethod(); return *this; }
+  ComplexNumber operator++(int);
+  ComplexNumber& operator<<(int);
+  ComplexNumber& operator+();
+
+private:
+  Number real;
+  Number complex;
+};
+
 class RefCounted {
 public:
   void ref() const;
@@ -210,6 +228,12 @@ public:
   unsigned trivial32() { return sizeof(int); }
   unsigned trivial33() { return ~0xff; }
   template <unsigned v> unsigned trivial34() { return v; }
+  void trivial35() { v++; }
+  void trivial36() { ++(*number); }
+  void trivial37() { (*number)++; }
+  void trivial38() { v++; if (__builtin_expect(!!(number), 1)) (*number)++; }
+  int trivial39() { return -v; }
+  int trivial40() { return v << 2; }
 
   static RefCounted& singleton() {
     static RefCounted s_RefCounted;
@@ -284,9 +308,14 @@ public:
 
   int nonTrivial13() { return ~otherFunction(); }
   int nonTrivial14() { int r = 0xff; r |= otherFunction(); return r; }
+  void nonTrivial15() { ++complex; }
+  void nonTrivial16() { complex++; }
+  ComplexNumber nonTrivial17() { return complex << 2; }
+  ComplexNumber nonTrivial18() { return +complex; }
 
   unsigned v { 0 };
   Number* number { nullptr };
+  ComplexNumber complex;
   Enum enumValue { Enum::Value1 };
 };
 
@@ -342,6 +371,12 @@ public:
     getFieldTrivial().trivial32(); // no-warning
     getFieldTrivial().trivial33(); // no-warning
     getFieldTrivial().trivial34<7>(); // no-warning
+    getFieldTrivial().trivial35(); // no-warning
+    getFieldTrivial().trivial36(); // no-warning
+    getFieldTrivial().trivial37(); // no-warning
+    getFieldTrivial().trivial38(); // no-warning
+    getFieldTrivial().trivial39(); // no-warning
+    getFieldTrivial().trivial40(); // no-warning
 
     RefCounted::singleton().trivial18(); // no-warning
     RefCounted::singleton().someFunction(); // no-warning
@@ -376,6 +411,14 @@ public:
     // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
     getFieldTrivial().nonTrivial14();
     // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    getFieldTrivial().nonTrivial15();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    getFieldTrivial().nonTrivial16();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    getFieldTrivial().nonTrivial17();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    getFieldTrivial().nonTrivial18();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
   }
 };
 
-- 
GitLab


From 8a3277acbc7b7af917f570f7d6430dda41c4d9ee Mon Sep 17 00:00:00 2001
From: Brendan Dahl <brendan.dahl@gmail.com>
Date: Thu, 9 May 2024 15:38:13 -0700
Subject: [PATCH 0356/1206] [WebAssembly] Implement prototype f32.store_f16
 instruction. (#91545)

Adds a builtin and intrinsic for the f32.store_f16 instruction.

The instruction stores an f32 value as an f16 memory. Specified at:

https://github.com/WebAssembly/half-precision/blob/29a9b9462c9285d4ccc1a5dc39214ddfd1892658/proposals/half-precision/Overview.md

Note: the current spec has f32.store_f16 as opcode 0xFD0121, but this is
incorrect and will be changed to 0xFC31 soon.
---
 .../clang/Basic/BuiltinsWebAssembly.def       |  1 +
 clang/lib/CodeGen/CGBuiltin.cpp               |  6 +++++
 clang/test/CodeGen/builtins-wasm.c            |  6 +++++
 llvm/include/llvm/IR/IntrinsicsWebAssembly.td |  5 ++++
 .../MCTargetDesc/WebAssemblyMCTargetDesc.h    |  1 +
 .../WebAssembly/WebAssemblyISelLowering.cpp   |  8 ++++++
 .../WebAssembly/WebAssemblyInstrMemory.td     | 11 ++++++--
 .../CodeGen/WebAssembly/half-precision.ll     |  9 +++++++
 llvm/test/CodeGen/WebAssembly/offset.ll       | 27 +++++++++++++++++++
 llvm/test/MC/WebAssembly/simd-encodings.s     |  3 +++
 10 files changed, 75 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Basic/BuiltinsWebAssembly.def b/clang/include/clang/Basic/BuiltinsWebAssembly.def
index cf54f8f4422f..8645cff1e867 100644
--- a/clang/include/clang/Basic/BuiltinsWebAssembly.def
+++ b/clang/include/clang/Basic/BuiltinsWebAssembly.def
@@ -192,6 +192,7 @@ TARGET_BUILTIN(__builtin_wasm_relaxed_dot_bf16x8_add_f32_f32x4, "V4fV8UsV8UsV4f"
 
 // Half-Precision (fp16)
 TARGET_BUILTIN(__builtin_wasm_loadf16_f32, "fh*", "nU", "half-precision")
+TARGET_BUILTIN(__builtin_wasm_storef16_f32, "vfh*", "n", "half-precision")
 
 // Reference Types builtins
 // Some builtins are custom type-checked - see 't' as part of the third argument,
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 4b03b8b0e093..f9ee93049b12 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -21310,6 +21310,12 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_loadf16_f32);
     return Builder.CreateCall(Callee, {Addr});
   }
+  case WebAssembly::BI__builtin_wasm_storef16_f32: {
+    Value *Val = EmitScalarExpr(E->getArg(0));
+    Value *Addr = EmitScalarExpr(E->getArg(1));
+    Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_storef16_f32);
+    return Builder.CreateCall(Callee, {Val, Addr});
+  }
   case WebAssembly::BI__builtin_wasm_table_get: {
     assert(E->getArg(0)->getType()->isArrayType());
     Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
diff --git a/clang/test/CodeGen/builtins-wasm.c b/clang/test/CodeGen/builtins-wasm.c
index ab1c6cd494ae..bcb15969de1c 100644
--- a/clang/test/CodeGen/builtins-wasm.c
+++ b/clang/test/CodeGen/builtins-wasm.c
@@ -807,6 +807,12 @@ float load_f16_f32(__fp16 *addr) {
   // WEBASSEMBLY: call float @llvm.wasm.loadf16.f32(ptr %{{.*}})
 }
 
+void store_f16_f32(float val, __fp16 *addr) {
+  return __builtin_wasm_storef16_f32(val, addr);
+  // WEBASSEMBLY: tail call void @llvm.wasm.storef16.f32(float %val, ptr %{{.*}})
+  // WEBASSEMBLY-NEXT: ret
+}
+
 __externref_t externref_null() {
   return __builtin_wasm_ref_null_extern();
   // WEBASSEMBLY: tail call ptr addrspace(10) @llvm.wasm.ref.null.extern()
diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
index f8142a8ca9e9..572d334ac955 100644
--- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
+++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -332,6 +332,11 @@ def int_wasm_loadf16_f32:
             [llvm_ptr_ty],
             [IntrReadMem, IntrArgMemOnly],
              "", [SDNPMemOperand]>;
+def int_wasm_storef16_f32:
+  Intrinsic<[],
+            [llvm_float_ty, llvm_ptr_ty],
+            [IntrWriteMem, IntrArgMemOnly],
+             "", [SDNPMemOperand]>;
 
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
index d3b496ae5917..d4e9fb057c44 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -207,6 +207,7 @@ inline unsigned GetDefaultP2AlignAny(unsigned Opc) {
   WASM_LOAD_STORE(LOAD_LANE_I16x8)
   WASM_LOAD_STORE(STORE_LANE_I16x8)
   WASM_LOAD_STORE(LOAD_F16_F32)
+  WASM_LOAD_STORE(STORE_F16_F32)
   return 1;
   WASM_LOAD_STORE(LOAD_I32)
   WASM_LOAD_STORE(LOAD_F32)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index ed52fe53bc60..527bb4c9fbea 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -914,6 +914,14 @@ bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.align = Align(2);
     Info.flags = MachineMemOperand::MOLoad;
     return true;
+  case Intrinsic::wasm_storef16_f32:
+    Info.opc = ISD::INTRINSIC_VOID;
+    Info.memVT = MVT::f16;
+    Info.ptrVal = I.getArgOperand(1);
+    Info.offset = 0;
+    Info.align = Align(2);
+    Info.flags = MachineMemOperand::MOStore;
+    return true;
   default:
     return false;
   }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
index e4baf842462a..9d452879bbf8 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
@@ -72,8 +72,9 @@ defm LOAD16_U_I64 : WebAssemblyLoad<I64, "i64.load16_u", 0x33, []>;
 defm LOAD32_S_I64 : WebAssemblyLoad<I64, "i64.load32_s", 0x34, []>;
 defm LOAD32_U_I64 : WebAssemblyLoad<I64, "i64.load32_u", 0x35, []>;
 
-// Half Precision
-defm LOAD_F16_F32 : WebAssemblyLoad<F32, "f32.load_f16", 0xfc30, [HasHalfPrecision]>;
+// Half-precision load.
+defm LOAD_F16_F32 :
+  WebAssemblyLoad<F32, "f32.load_f16", 0xfc30, [HasHalfPrecision]>;
 
 // Pattern matching
 
@@ -171,12 +172,18 @@ defm STORE8_I64 : WebAssemblyStore<I64, "i64.store8", 0x3c>;
 defm STORE16_I64 : WebAssemblyStore<I64, "i64.store16", 0x3d>;
 defm STORE32_I64 : WebAssemblyStore<I64, "i64.store32", 0x3e>;
 
+// Half-precision store.
+defm STORE_F16_F32 :
+  WebAssemblyStore<F32, "f32.store_f16", 0xfc31, [HasHalfPrecision]>;
+
 defm : StorePat<i32, truncstorei8, "STORE8_I32">;
 defm : StorePat<i32, truncstorei16, "STORE16_I32">;
 defm : StorePat<i64, truncstorei8, "STORE8_I64">;
 defm : StorePat<i64, truncstorei16, "STORE16_I64">;
 defm : StorePat<i64, truncstorei32, "STORE32_I64">;
 
+defm : StorePat<f32, int_wasm_storef16_f32, "STORE_F16_F32">;
+
 multiclass MemoryOps<WebAssemblyRegClass rc, string B> {
 // Current memory size.
 defm MEMORY_SIZE_A#B : I<(outs rc:$dst), (ins i32imm:$flags),
diff --git a/llvm/test/CodeGen/WebAssembly/half-precision.ll b/llvm/test/CodeGen/WebAssembly/half-precision.ll
index 582771d3f95f..89e9c42637c1 100644
--- a/llvm/test/CodeGen/WebAssembly/half-precision.ll
+++ b/llvm/test/CodeGen/WebAssembly/half-precision.ll
@@ -2,6 +2,7 @@
 ; RUN: llc < %s --mtriple=wasm64-unknown-unknown -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+half-precision | FileCheck %s
 
 declare float @llvm.wasm.loadf32.f16(ptr)
+declare void @llvm.wasm.storef16.f32(float, ptr)
 
 ; CHECK-LABEL: ldf16_32:
 ; CHECK:      f32.load_f16 $push[[NUM0:[0-9]+]]=, 0($0){{$}}
@@ -10,3 +11,11 @@ define float @ldf16_32(ptr %p) {
   %v = call float @llvm.wasm.loadf16.f32(ptr %p)
   ret float %v
 }
+
+; CHECK-LABEL: stf16_32:
+; CHECK:       f32.store_f16 0($1), $0
+; CHECK-NEXT:  return
+define void @stf16_32(float %v, ptr %p) {
+  tail call void @llvm.wasm.storef16.f32(float %v, ptr %p)
+  ret void
+}
diff --git a/llvm/test/CodeGen/WebAssembly/offset.ll b/llvm/test/CodeGen/WebAssembly/offset.ll
index b497ddd7273a..65de341780e3 100644
--- a/llvm/test/CodeGen/WebAssembly/offset.ll
+++ b/llvm/test/CodeGen/WebAssembly/offset.ll
@@ -692,3 +692,30 @@ define float @load_f16_f32_with_folded_gep_offset(ptr %p) {
   %t = call float @llvm.wasm.loadf16.f32(ptr %s)
   ret float %t
 }
+
+;===----------------------------------------------------------------------------
+; Stores: Half Precision
+;===----------------------------------------------------------------------------
+
+; Basic store.
+
+; CHECK-LABEL: store_f16_f32_no_offset:
+; CHECK-NEXT: .functype store_f16_f32_no_offset (i32, f32) -> (){{$}}
+; CHECK-NEXT: f32.store_f16 0($0), $1{{$}}
+; CHECK-NEXT: return{{$}}
+define void @store_f16_f32_no_offset(ptr %p, float %v) {
+  call void @llvm.wasm.storef16.f32(float %v, ptr %p)
+  ret void
+}
+
+; Storing to a fixed address.
+
+; CHECK-LABEL: store_f16_f32_to_numeric_address:
+; CHECK:      i32.const $push1=, 0{{$}}
+; CHECK-NEXT: f32.const $push0=, 0x0p0{{$}}
+; CHECK-NEXT: f32.store_f16 42($pop1), $pop0{{$}}
+define void @store_f16_f32_to_numeric_address() {
+  %s = inttoptr i32 42 to ptr
+  call void @llvm.wasm.storef16.f32(float 0.0, ptr %s)
+  ret void
+}
diff --git a/llvm/test/MC/WebAssembly/simd-encodings.s b/llvm/test/MC/WebAssembly/simd-encodings.s
index e7c3761f381d..57fa71e74b8d 100644
--- a/llvm/test/MC/WebAssembly/simd-encodings.s
+++ b/llvm/test/MC/WebAssembly/simd-encodings.s
@@ -842,4 +842,7 @@ main:
     # CHECK: f32.load_f16 48 # encoding: [0xfc,0x30,0x01,0x30]
     f32.load_f16 48
 
+    # CHECK: f32.store_f16 32 # encoding: [0xfc,0x31,0x01,0x20]
+    f32.store_f16 32
+
     end_function
-- 
GitLab


From 1e97d114b5b2b522de7e0aa9c950199de0798d53 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 9 May 2024 15:55:36 -0700
Subject: [PATCH 0357/1206] [dsymutil] Add -q/--quiet flag to suppress warnings
 (#91658)

Add a -q/--quiet flag to suppress dsymutil output. For now the flag is
limited to dsymutil, though there might be other places in the DWARF
linker that could be conditionalized by this flag.

The motivation is having a way to silence the "no debug symbols in
executable" warning. This is useful when we want to generate a dSYM for
a binary not containing debug symbols, but still want a dSYM that can be
indexed by spotlight.

rdar://127843467
---
 llvm/docs/CommandGuide/dsymutil.rst         |  4 ++
 llvm/test/tools/dsymutil/ARM/empty-map.test |  2 +
 llvm/test/tools/dsymutil/cmdline.test       |  4 ++
 llvm/tools/dsymutil/LinkUtils.h             |  3 ++
 llvm/tools/dsymutil/Options.td              |  8 ++++
 llvm/tools/dsymutil/dsymutil.cpp            | 45 ++++++++++++++-------
 6 files changed, 51 insertions(+), 15 deletions(-)

diff --git a/llvm/docs/CommandGuide/dsymutil.rst b/llvm/docs/CommandGuide/dsymutil.rst
index e3f2f33224b0..6026e2f534ed 100644
--- a/llvm/docs/CommandGuide/dsymutil.rst
+++ b/llvm/docs/CommandGuide/dsymutil.rst
@@ -115,6 +115,10 @@ OPTIONS
  Specifies an alternate ``path`` to place the dSYM bundle. The default dSYM
  bundle path is created by appending ``.dSYM`` to the executable name.
 
+.. option:: -q, --quiet
+
+ Enable quiet mode and limit output.
+
 .. option:: --remarks-drop-without-debug
 
  Drop remarks without valid debug locations. Without this flags, all remarks are kept.
diff --git a/llvm/test/tools/dsymutil/ARM/empty-map.test b/llvm/test/tools/dsymutil/ARM/empty-map.test
index 40ffa8b1cc51..eeca28273a3f 100644
--- a/llvm/test/tools/dsymutil/ARM/empty-map.test
+++ b/llvm/test/tools/dsymutil/ARM/empty-map.test
@@ -1,4 +1,5 @@
 # RUN: dsymutil -f -oso-prepend-path=%p/../Inputs -y %s -o - 2>&1 | FileCheck %s
+# RUN: dsymutil -q -f -oso-prepend-path=%p/../Inputs -y %s -o - 2>&1 | FileCheck %s --check-prefix QUIET
 
 # RUN: dsymutil --linker parallel -f -oso-prepend-path=%p/../Inputs -y %s -o - 2>&1 | FileCheck %s
 
@@ -7,3 +8,4 @@ triple:          'thumbv7-apple-darwin'
 ...
 
 # CHECK: warning: no debug symbols in executable (-arch armv7)
+# QUIET-NOT: no debug symbols in executable
diff --git a/llvm/test/tools/dsymutil/cmdline.test b/llvm/test/tools/dsymutil/cmdline.test
index 814252b6e230..6c67ac7cd723 100644
--- a/llvm/test/tools/dsymutil/cmdline.test
+++ b/llvm/test/tools/dsymutil/cmdline.test
@@ -23,6 +23,7 @@ CHECK: -object-prefix-map <prefix=remapped>
 CHECK: -oso-prepend-path <path>
 CHECK: -out <filename>
 CHECK: {{-o <filename>}}
+CHECK: -quiet
 CHECK: -remarks-drop-without-debug
 CHECK: -remarks-output-format <format>
 CHECK: -remarks-prepend-path <path>
@@ -46,3 +47,6 @@ NOINPUT: error: no input files specified
 
 RUN: dsymutil -bogus -help 2>&1 | FileCheck --check-prefix=BOGUS %s
 BOGUS: warning: ignoring unknown option: -bogus
+
+RUN: not dsymutil --quiet --verbose 2>&1 | FileCheck --check-prefix=CONFLICT %s
+CONFLICT: error: --quiet and --verbose cannot be specified together
diff --git a/llvm/tools/dsymutil/LinkUtils.h b/llvm/tools/dsymutil/LinkUtils.h
index 6aa0b847eebd..ad5515a04333 100644
--- a/llvm/tools/dsymutil/LinkUtils.h
+++ b/llvm/tools/dsymutil/LinkUtils.h
@@ -38,6 +38,9 @@ struct LinkOptions {
   /// Verbosity
   bool Verbose = false;
 
+  /// Quiet
+  bool Quiet = false;
+
   /// Statistics
   bool Statistics = false;
 
diff --git a/llvm/tools/dsymutil/Options.td b/llvm/tools/dsymutil/Options.td
index d8cec0cb2c41..b72ae1909a72 100644
--- a/llvm/tools/dsymutil/Options.td
+++ b/llvm/tools/dsymutil/Options.td
@@ -24,6 +24,14 @@ def verbose: F<"verbose">,
   HelpText<"Enable verbose mode.">,
   Group<grp_general>;
 
+def quiet: F<"quiet">,
+  HelpText<"Enable quiet mode.">,
+  Group<grp_general>;
+def: Flag<["-"], "q">,
+  Alias<quiet>,
+  HelpText<"Alias for --quiet">,
+  Group<grp_general>;
+
 def keep_func_for_static: F<"keep-function-for-static">,
   HelpText<"Make a static variable keep the enclosing function even if it would have been omitted otherwise.">,
   Group<grp_general>;
diff --git a/llvm/tools/dsymutil/dsymutil.cpp b/llvm/tools/dsymutil/dsymutil.cpp
index bc968b6387b6..728f2ed3e62a 100644
--- a/llvm/tools/dsymutil/dsymutil.cpp
+++ b/llvm/tools/dsymutil/dsymutil.cpp
@@ -169,6 +169,12 @@ static Expected<std::vector<std::string>> getInputs(opt::InputArgList &Args,
 
 // Verify that the given combination of options makes sense.
 static Error verifyOptions(const DsymutilOptions &Options) {
+  if (Options.LinkOpts.Verbose && Options.LinkOpts.Quiet) {
+    return make_error<StringError>(
+        "--quiet and --verbose cannot be specified together",
+        errc::invalid_argument);
+  }
+
   if (Options.InputFiles.empty()) {
     return make_error<StringError>("no input files specified",
                                    errc::invalid_argument);
@@ -311,6 +317,7 @@ static Expected<DsymutilOptions> getOptions(opt::InputArgList &Args) {
   Options.LinkOpts.NoTimestamp = Args.hasArg(OPT_no_swiftmodule_timestamp);
   Options.LinkOpts.Update = Args.hasArg(OPT_update);
   Options.LinkOpts.Verbose = Args.hasArg(OPT_verbose);
+  Options.LinkOpts.Quiet = Args.hasArg(OPT_quiet);
   Options.LinkOpts.Statistics = Args.hasArg(OPT_statistics);
   Options.LinkOpts.Fat64 = Args.hasArg(OPT_fat64);
   Options.LinkOpts.KeepFunctionForStatic =
@@ -483,16 +490,20 @@ static bool verifyOutput(StringRef OutputFile, StringRef Arch,
                          DsymutilOptions Options, std::mutex &Mutex) {
 
   if (OutputFile == "-") {
-    std::lock_guard<std::mutex> Guard(Mutex);
-    WithColor::warning() << "verification skipped for " << Arch
-                         << " because writing to stdout.\n";
+    if (!Options.LinkOpts.Quiet) {
+      std::lock_guard<std::mutex> Guard(Mutex);
+      WithColor::warning() << "verification skipped for " << Arch
+                           << " because writing to stdout.\n";
+    }
     return true;
   }
 
   if (Options.LinkOpts.NoOutput) {
-    std::lock_guard<std::mutex> Guard(Mutex);
-    WithColor::warning() << "verification skipped for " << Arch
-                         << " because --no-output was passed.\n";
+    if (!Options.LinkOpts.Quiet) {
+      std::lock_guard<std::mutex> Guard(Mutex);
+      WithColor::warning() << "verification skipped for " << Arch
+                           << " because --no-output was passed.\n";
+    }
     return true;
   }
 
@@ -507,10 +518,12 @@ static bool verifyOutput(StringRef OutputFile, StringRef Arch,
   if (auto *Obj = dyn_cast<MachOObjectFile>(&Binary)) {
     std::unique_ptr<DWARFContext> DICtx = DWARFContext::create(*Obj);
     if (DICtx->getMaxVersion() > 5) {
-      std::lock_guard<std::mutex> Guard(Mutex);
-      WithColor::warning()
-          << "verification skipped for " << Arch
-          << " because DWARF standard greater than v5 is not supported yet.\n";
+      if (!Options.LinkOpts.Quiet) {
+        std::lock_guard<std::mutex> Guard(Mutex);
+        WithColor::warning() << "verification skipped for " << Arch
+                             << " because DWARF standard greater than v5 is "
+                                "not supported yet.\n";
+      }
       return true;
     }
 
@@ -751,11 +764,13 @@ int dsymutil_main(int argc, char **argv, const llvm::ToolContext &) {
           continue;
 
         if (Map->begin() == Map->end()) {
-          std::lock_guard<std::mutex> Guard(ErrorHandlerMutex);
-          WithColor::warning()
-              << "no debug symbols in executable (-arch "
-              << MachOUtils::getArchName(Map->getTriple().getArchName())
-              << ")\n";
+          if (!Options.LinkOpts.Quiet) {
+            std::lock_guard<std::mutex> Guard(ErrorHandlerMutex);
+            WithColor::warning()
+                << "no debug symbols in executable (-arch "
+                << MachOUtils::getArchName(Map->getTriple().getArchName())
+                << ")\n";
+          }
         }
 
         // Using a std::shared_ptr rather than std::unique_ptr because move-only
-- 
GitLab


From 95f208f97e709139c3ecbce552bcf1e34b9fcf12 Mon Sep 17 00:00:00 2001
From: Anthony Ha <anthonyha96@gmail.com>
Date: Thu, 9 May 2024 15:57:46 -0700
Subject: [PATCH 0358/1206] [lldb] Unify CalculateMD5 return types (#91029)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is a retake of https://github.com/llvm/llvm-project/pull/90921
which got reverted because I forgot to modify the CalculateMD5 unit test
I had added in https://github.com/llvm/llvm-project/pull/88812

The prior failing build is here:
https://lab.llvm.org/buildbot/#/builders/68/builds/73622
To make sure this error doesn't happen, I ran `ninja
ProcessGdbRemoteTests` and then executed the resulting test binary and
observed the `CalculateMD5` test passed.

# Overview
In my previous PR: https://github.com/llvm/llvm-project/pull/88812,
@JDevlieghere suggested to match return types of the various calculate
md5 functions.

This PR achieves that by changing the various calculate md5 functions to
return `llvm::ErrorOr<llvm::MD5::MD5Result>`.
 
The suggestion was to go for `std::optional<>` but I opted for
`llvm::ErrorOr<>` because local calculate md5 was already possibly
returning `ErrorOr`.

To make sure I didn't break the md5 calculation functionality, I ran
some tests for the gdb remote client, and things seem to work.

# Testing
1. Remote file doesn't exist

![image](https://github.com/llvm/llvm-project/assets/1326275/b26859e2-18c3-4685-be8f-c6b6a5a4bc77)

1. Remote file differs

![image](https://github.com/llvm/llvm-project/assets/1326275/cbdb3c58-555a-401b-9444-c5ff4c04c491)

1. Remote file matches

![image](https://github.com/llvm/llvm-project/assets/1326275/07561572-22d1-4e0a-988f-bc91b5c2ffce)

## Test gaps
Unfortunately, I had to modify
`lldb/source/Plugins/Platform/MacOSX/PlatformDarwinDevice.cpp` and I
can't test the changes there. Hopefully, the existing test suite / code
review from whomever is reading this will catch any issues.
---
 lldb/include/lldb/Target/Platform.h           |  4 +--
 .../include/lldb/Target/RemoteAwarePlatform.h |  4 +--
 .../Platform/MacOSX/PlatformDarwinDevice.cpp  | 16 +++++----
 .../gdb-server/PlatformRemoteGDBServer.cpp    |  8 ++---
 .../gdb-server/PlatformRemoteGDBServer.h      |  4 +--
 .../GDBRemoteCommunicationClient.cpp          | 30 ++++++++++------
 .../gdb-remote/GDBRemoteCommunicationClient.h |  2 +-
 lldb/source/Target/Platform.cpp               | 36 +++++++++----------
 lldb/source/Target/RemoteAwarePlatform.cpp    |  8 ++---
 .../GDBRemoteCommunicationClientTest.cpp      | 13 ++++---
 10 files changed, 66 insertions(+), 59 deletions(-)

diff --git a/lldb/include/lldb/Target/Platform.h b/lldb/include/lldb/Target/Platform.h
index ad9c9dcbe684..e05c79cb501b 100644
--- a/lldb/include/lldb/Target/Platform.h
+++ b/lldb/include/lldb/Target/Platform.h
@@ -649,8 +649,8 @@ public:
 
   virtual std::string GetPlatformSpecificConnectionInformation() { return ""; }
 
-  virtual bool CalculateMD5(const FileSpec &file_spec, uint64_t &low,
-                            uint64_t &high);
+  virtual llvm::ErrorOr<llvm::MD5::MD5Result>
+  CalculateMD5(const FileSpec &file_spec);
 
   virtual uint32_t GetResumeCountForLaunchInfo(ProcessLaunchInfo &launch_info) {
     return 1;
diff --git a/lldb/include/lldb/Target/RemoteAwarePlatform.h b/lldb/include/lldb/Target/RemoteAwarePlatform.h
index d183815e1c8b..0b9d79f9ff03 100644
--- a/lldb/include/lldb/Target/RemoteAwarePlatform.h
+++ b/lldb/include/lldb/Target/RemoteAwarePlatform.h
@@ -58,8 +58,8 @@ public:
   Status SetFilePermissions(const FileSpec &file_spec,
                             uint32_t file_permissions) override;
 
-  bool CalculateMD5(const FileSpec &file_spec, uint64_t &low,
-                    uint64_t &high) override;
+  llvm::ErrorOr<llvm::MD5::MD5Result>
+  CalculateMD5(const FileSpec &file_spec) override;
 
   Status GetFileWithUUID(const FileSpec &platform_file, const UUID *uuid,
                          FileSpec &local_file) override;
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinDevice.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinDevice.cpp
index 52777909a1f8..82156aca8cf1 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinDevice.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinDevice.cpp
@@ -405,17 +405,21 @@ lldb_private::Status PlatformDarwinDevice::GetSharedModuleWithLocalCache(
           // when going over the *slow* GDB remote transfer mechanism we first
           // check the hashes of the files - and only do the actual transfer if
           // they differ
-          uint64_t high_local, high_remote, low_local, low_remote;
           auto MD5 = llvm::sys::fs::md5_contents(module_cache_spec.GetPath());
           if (!MD5)
             return Status(MD5.getError());
-          std::tie(high_local, low_local) = MD5->words();
 
-          m_remote_platform_sp->CalculateMD5(module_spec.GetFileSpec(),
-                                             low_remote, high_remote);
-          if (low_local != low_remote || high_local != high_remote) {
+          Log *log = GetLog(LLDBLog::Platform);
+          bool requires_transfer = true;
+          llvm::ErrorOr<llvm::MD5::MD5Result> remote_md5 =
+              m_remote_platform_sp->CalculateMD5(module_spec.GetFileSpec());
+          if (std::error_code ec = remote_md5.getError())
+            LLDB_LOG(log, "couldn't get md5 sum from remote: {0}",
+                     ec.message());
+          else
+            requires_transfer = *MD5 != *remote_md5;
+          if (requires_transfer) {
             // bring in the remote file
-            Log *log = GetLog(LLDBLog::Platform);
             LLDB_LOGF(log,
                       "[%s] module %s/%s needs to be replaced from remote copy",
                       (IsHost() ? "host" : "remote"),
diff --git a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
index 0dce5add2e37..4684947ede20 100644
--- a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
+++ b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
@@ -684,12 +684,12 @@ Status PlatformRemoteGDBServer::RunShellCommand(
                                           signo_ptr, command_output, timeout);
 }
 
-bool PlatformRemoteGDBServer::CalculateMD5(const FileSpec &file_spec,
-                                           uint64_t &low, uint64_t &high) {
+llvm::ErrorOr<llvm::MD5::MD5Result>
+PlatformRemoteGDBServer::CalculateMD5(const FileSpec &file_spec) {
   if (!IsConnected())
-    return false;
+    return std::make_error_code(std::errc::not_connected);
 
-  return m_gdb_client_up->CalculateMD5(file_spec, low, high);
+  return m_gdb_client_up->CalculateMD5(file_spec);
 }
 
 void PlatformRemoteGDBServer::CalculateTrapHandlerSymbolNames() {
diff --git a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.h b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.h
index d83fc386f594..0ae1f3cb4199 100644
--- a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.h
+++ b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.h
@@ -146,8 +146,8 @@ public:
 
   void CalculateTrapHandlerSymbolNames() override;
 
-  bool CalculateMD5(const FileSpec &file_spec, uint64_t &low,
-                    uint64_t &high) override;
+  llvm::ErrorOr<llvm::MD5::MD5Result>
+  CalculateMD5(const FileSpec &file_spec) override;
 
   const lldb::UnixSignalsSP &GetRemoteUnixSignals() override;
 
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
index 7498a070c260..db9fb37a9a3c 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
@@ -3418,8 +3418,8 @@ bool GDBRemoteCommunicationClient::GetFileExists(
   return true;
 }
 
-bool GDBRemoteCommunicationClient::CalculateMD5(
-    const lldb_private::FileSpec &file_spec, uint64_t &low, uint64_t &high) {
+llvm::ErrorOr<llvm::MD5::MD5Result> GDBRemoteCommunicationClient::CalculateMD5(
+    const lldb_private::FileSpec &file_spec) {
   std::string path(file_spec.GetPath(false));
   lldb_private::StreamString stream;
   stream.PutCString("vFile:MD5:");
@@ -3428,11 +3428,11 @@ bool GDBRemoteCommunicationClient::CalculateMD5(
   if (SendPacketAndWaitForResponse(stream.GetString(), response) ==
       PacketResult::Success) {
     if (response.GetChar() != 'F')
-      return false;
+      return std::make_error_code(std::errc::illegal_byte_sequence);
     if (response.GetChar() != ',')
-      return false;
+      return std::make_error_code(std::errc::illegal_byte_sequence);
     if (response.Peek() && *response.Peek() == 'x')
-      return false;
+      return std::make_error_code(std::errc::no_such_file_or_directory);
 
     // GDBRemoteCommunicationServerCommon::Handle_vFile_MD5 concatenates low and
     // high hex strings. We can't use response.GetHexMaxU64 because that can't
@@ -3455,25 +3455,33 @@ bool GDBRemoteCommunicationClient::CalculateMD5(
     auto part =
         response.GetStringRef().substr(response.GetFilePos(), MD5_HALF_LENGTH);
     if (part.size() != MD5_HALF_LENGTH)
-      return false;
+      return std::make_error_code(std::errc::illegal_byte_sequence);
     response.SetFilePos(response.GetFilePos() + part.size());
 
+    uint64_t low;
     if (part.getAsInteger(/*radix=*/16, low))
-      return false;
+      return std::make_error_code(std::errc::illegal_byte_sequence);
 
     // Get high part
     part =
         response.GetStringRef().substr(response.GetFilePos(), MD5_HALF_LENGTH);
     if (part.size() != MD5_HALF_LENGTH)
-      return false;
+      return std::make_error_code(std::errc::illegal_byte_sequence);
     response.SetFilePos(response.GetFilePos() + part.size());
 
+    uint64_t high;
     if (part.getAsInteger(/*radix=*/16, high))
-      return false;
+      return std::make_error_code(std::errc::illegal_byte_sequence);
 
-    return true;
+    llvm::MD5::MD5Result result;
+    llvm::support::endian::write<uint64_t, llvm::endianness::little>(
+        result.data(), low);
+    llvm::support::endian::write<uint64_t, llvm::endianness::little>(
+        result.data() + 8, high);
+
+    return result;
   }
-  return false;
+  return std::make_error_code(std::errc::operation_canceled);
 }
 
 bool GDBRemoteCommunicationClient::AvoidGPackets(ProcessGDBRemote *process) {
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
index 4be7eb00f42b..898d176abc34 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
@@ -392,7 +392,7 @@ public:
           *command_output, // Pass nullptr if you don't want the command output
       const Timeout<std::micro> &timeout);
 
-  bool CalculateMD5(const FileSpec &file_spec, uint64_t &low, uint64_t &high);
+  llvm::ErrorOr<llvm::MD5::MD5Result> CalculateMD5(const FileSpec &file_spec);
 
   lldb::DataBufferSP ReadRegister(
       lldb::tid_t tid,
diff --git a/lldb/source/Target/Platform.cpp b/lldb/source/Target/Platform.cpp
index 91483ba008f4..4af4aa68ccd0 100644
--- a/lldb/source/Target/Platform.cpp
+++ b/lldb/source/Target/Platform.cpp
@@ -1199,22 +1199,22 @@ Status Platform::PutFile(const FileSpec &source, const FileSpec &destination,
   Status error;
 
   bool requires_upload = true;
-  uint64_t dest_md5_low, dest_md5_high;
-  bool success = CalculateMD5(destination, dest_md5_low, dest_md5_high);
-  if (!success) {
-    LLDB_LOGF(log, "[PutFile] couldn't get md5 sum of destination");
+  llvm::ErrorOr<llvm::MD5::MD5Result> remote_md5 = CalculateMD5(destination);
+  if (std::error_code ec = remote_md5.getError()) {
+    LLDB_LOG(log, "[PutFile] couldn't get md5 sum of destination: {0}",
+             ec.message());
   } else {
-    auto local_md5 = llvm::sys::fs::md5_contents(source.GetPath());
-    if (!local_md5) {
-      LLDB_LOGF(log, "[PutFile] couldn't get md5 sum of source");
+    llvm::ErrorOr<llvm::MD5::MD5Result> local_md5 =
+        llvm::sys::fs::md5_contents(source.GetPath());
+    if (std::error_code ec = local_md5.getError()) {
+      LLDB_LOG(log, "[PutFile] couldn't get md5 sum of source: {0}",
+               ec.message());
     } else {
-      const auto [local_md5_high, local_md5_low] = local_md5->words();
       LLDB_LOGF(log, "[PutFile] destination md5: %016" PRIx64 "%016" PRIx64,
-                dest_md5_high, dest_md5_low);
+                remote_md5->high(), remote_md5->low());
       LLDB_LOGF(log, "[PutFile]       local md5: %016" PRIx64 "%016" PRIx64,
-                local_md5_high, local_md5_low);
-      requires_upload =
-          local_md5_high != dest_md5_high || local_md5_low != dest_md5_low;
+                local_md5->high(), local_md5->low());
+      requires_upload = *remote_md5 != *local_md5;
     }
   }
 
@@ -1339,15 +1339,11 @@ lldb_private::Status Platform::RunShellCommand(
   return Status("unable to run a remote command without a platform");
 }
 
-bool Platform::CalculateMD5(const FileSpec &file_spec, uint64_t &low,
-                            uint64_t &high) {
+llvm::ErrorOr<llvm::MD5::MD5Result>
+Platform::CalculateMD5(const FileSpec &file_spec) {
   if (!IsHost())
-    return false;
-  auto Result = llvm::sys::fs::md5_contents(file_spec.GetPath());
-  if (!Result)
-    return false;
-  std::tie(high, low) = Result->words();
-  return true;
+    return std::make_error_code(std::errc::not_supported);
+  return llvm::sys::fs::md5_contents(file_spec.GetPath());
 }
 
 void Platform::SetLocalCacheDirectory(const char *local) {
diff --git a/lldb/source/Target/RemoteAwarePlatform.cpp b/lldb/source/Target/RemoteAwarePlatform.cpp
index 0bd6c9251c85..9a41a423cadd 100644
--- a/lldb/source/Target/RemoteAwarePlatform.cpp
+++ b/lldb/source/Target/RemoteAwarePlatform.cpp
@@ -266,11 +266,11 @@ Status RemoteAwarePlatform::Unlink(const FileSpec &file_spec) {
   return Platform::Unlink(file_spec);
 }
 
-bool RemoteAwarePlatform::CalculateMD5(const FileSpec &file_spec, uint64_t &low,
-                                       uint64_t &high) {
+llvm::ErrorOr<llvm::MD5::MD5Result>
+RemoteAwarePlatform::CalculateMD5(const FileSpec &file_spec) {
   if (m_remote_platform_sp)
-    return m_remote_platform_sp->CalculateMD5(file_spec, low, high);
-  return Platform::CalculateMD5(file_spec, low, high);
+    return m_remote_platform_sp->CalculateMD5(file_spec);
+  return Platform::CalculateMD5(file_spec);
 }
 
 FileSpec RemoteAwarePlatform::GetRemoteWorkingDirectory() {
diff --git a/lldb/unittests/Process/gdb-remote/GDBRemoteCommunicationClientTest.cpp b/lldb/unittests/Process/gdb-remote/GDBRemoteCommunicationClientTest.cpp
index 6b11ec43a65d..24111396b0ac 100644
--- a/lldb/unittests/Process/gdb-remote/GDBRemoteCommunicationClientTest.cpp
+++ b/lldb/unittests/Process/gdb-remote/GDBRemoteCommunicationClientTest.cpp
@@ -595,10 +595,8 @@ TEST_F(GDBRemoteCommunicationClientTest, WriteMemoryTags) {
 
 TEST_F(GDBRemoteCommunicationClientTest, CalculateMD5) {
   FileSpec file_spec("/foo/bar", FileSpec::Style::posix);
-  uint64_t low, high;
-  std::future<bool> async_result = std::async(std::launch::async, [&] {
-    return client.CalculateMD5(file_spec, low, high);
-  });
+  std::future<ErrorOr<MD5::MD5Result>> async_result = std::async(
+      std::launch::async, [&] { return client.CalculateMD5(file_spec); });
 
   lldb_private::StreamString stream;
   stream.PutCString("vFile:MD5:");
@@ -607,11 +605,12 @@ TEST_F(GDBRemoteCommunicationClientTest, CalculateMD5) {
                "F,"
                "deadbeef01020304"
                "05060708deadbeef");
-  ASSERT_TRUE(async_result.get());
+  auto result = async_result.get();
 
   // Server and client puts/parses low, and then high
   const uint64_t expected_low = 0xdeadbeef01020304;
   const uint64_t expected_high = 0x05060708deadbeef;
-  EXPECT_EQ(expected_low, low);
-  EXPECT_EQ(expected_high, high);
+  ASSERT_TRUE(result);
+  EXPECT_EQ(expected_low, result->low());
+  EXPECT_EQ(expected_high, result->high());
 }
-- 
GitLab


From f893dccbba372792e7e7095d741f98a234654875 Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma@quicinc.com>
Date: Thu, 9 May 2024 16:50:01 -0700
Subject: [PATCH 0359/1206] Replace uses of ConstantExpr::getCompare. (#91558)

Use ICmpInst::compare() where possible, ConstantFoldCompareInstOperands
in other places. This only changes places where the either the fold is
guaranteed to succeed, or the code doesn't use the resulting compare if
we fail to fold.
---
 .../llvm/Transforms/Scalar/JumpThreading.h     |  2 +-
 llvm/lib/Analysis/BranchProbabilityInfo.cpp    |  4 ++--
 llvm/lib/Analysis/ConstantFolding.cpp          |  8 ++++----
 llvm/lib/Analysis/InlineCost.cpp               | 12 +++++-------
 llvm/lib/Analysis/ScalarEvolution.cpp          |  4 +---
 llvm/lib/IR/Constants.cpp                      |  4 ++--
 .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp      |  5 +++--
 .../lib/Target/X86/X86InstCombineIntrinsic.cpp | 18 ++++++++++--------
 .../InstCombine/InstCombineAndOrXor.cpp        |  4 ++--
 .../InstCombine/InstCombineCalls.cpp           |  3 ++-
 .../InstCombine/InstCombineCompares.cpp        | 15 ++++++---------
 .../InstCombine/InstCombineSelect.cpp          | 17 +++++++++--------
 .../InstCombine/InstructionCombining.cpp       |  9 ++++++---
 llvm/lib/Transforms/Scalar/JumpThreading.cpp   | 16 ++++++++++------
 llvm/lib/Transforms/Utils/SimplifyCFG.cpp      |  9 +++++----
 .../Transforms/JumpThreading/thread-two-bbs.ll | 15 ++++++++-------
 16 files changed, 76 insertions(+), 69 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
index f7358ac9b1ee..65d43775bdc1 100644
--- a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
+++ b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
@@ -142,7 +142,7 @@ public:
   }
 
   Constant *evaluateOnPredecessorEdge(BasicBlock *BB, BasicBlock *PredPredBB,
-                                      Value *cond);
+                                      Value *cond, const DataLayout &DL);
   bool maybethreadThroughTwoBasicBlocks(BasicBlock *BB, Value *Cond);
   void threadThroughTwoBasicBlocks(BasicBlock *PredPredBB, BasicBlock *PredBB,
                                    BasicBlock *BB, BasicBlock *SuccBB);
diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index 36a2df645913..cd3e3a499132 100644
--- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -630,8 +630,8 @@ computeUnlikelySuccessors(const BasicBlock *BB, Loop *L,
       if (!CmpLHSConst)
         continue;
       // Now constant-evaluate the compare
-      Constant *Result = ConstantExpr::getCompare(CI->getPredicate(),
-                                                  CmpLHSConst, CmpConst, true);
+      Constant *Result = ConstantFoldCompareInstOperands(
+          CI->getPredicate(), CmpLHSConst, CmpConst, DL);
       // If the result means we don't branch to the block then that block is
       // unlikely.
       if (Result &&
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 749374a3aa48..046a76945380 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1268,10 +1268,10 @@ Constant *llvm::ConstantFoldCompareInstOperands(
       Value *Stripped1 =
           Ops1->stripAndAccumulateInBoundsConstantOffsets(DL, Offset1);
       if (Stripped0 == Stripped1)
-        return ConstantExpr::getCompare(
-            ICmpInst::getSignedPredicate(Predicate),
-            ConstantInt::get(CE0->getContext(), Offset0),
-            ConstantInt::get(CE0->getContext(), Offset1));
+        return ConstantInt::getBool(
+            Ops0->getContext(),
+            ICmpInst::compare(Offset0, Offset1,
+                              ICmpInst::getSignedPredicate(Predicate)));
     }
   } else if (isa<ConstantExpr>(Ops1)) {
     // If RHS is a constant expression, but the left side isn't, swap the
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index c75460f44c1d..a531064e304d 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -2046,13 +2046,11 @@ bool CallAnalyzer::visitCmpInst(CmpInst &I) {
     if (RHSBase && LHSBase == RHSBase) {
       // We have common bases, fold the icmp to a constant based on the
       // offsets.
-      Constant *CLHS = ConstantInt::get(LHS->getContext(), LHSOffset);
-      Constant *CRHS = ConstantInt::get(RHS->getContext(), RHSOffset);
-      if (Constant *C = ConstantExpr::getICmp(I.getPredicate(), CLHS, CRHS)) {
-        SimplifiedValues[&I] = C;
-        ++NumConstantPtrCmps;
-        return true;
-      }
+      SimplifiedValues[&I] = ConstantInt::getBool(
+          I.getType(),
+          ICmpInst::compare(LHSOffset, RHSOffset, I.getPredicate()));
+      ++NumConstantPtrCmps;
+      return true;
     }
   }
 
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 93f885c5d5ad..7dc5aa084f3c 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -10615,9 +10615,7 @@ bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred,
   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
     // Check for both operands constant.
     if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
-      if (ConstantExpr::getICmp(Pred,
-                                LHSC->getValue(),
-                                RHSC->getValue())->isNullValue())
+      if (!ICmpInst::compare(LHSC->getAPInt(), RHSC->getAPInt(), Pred))
         return TrivialCase(false);
       return TrivialCase(true);
     }
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index 5268eccf7014..db442c54125a 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -315,8 +315,8 @@ bool Constant::isElementWiseEqual(Value *Y) const {
   Type *IntTy = VectorType::getInteger(VTy);
   Constant *C0 = ConstantExpr::getBitCast(const_cast<Constant *>(this), IntTy);
   Constant *C1 = ConstantExpr::getBitCast(cast<Constant>(Y), IntTy);
-  Constant *CmpEq = ConstantExpr::getICmp(ICmpInst::ICMP_EQ, C0, C1);
-  return isa<PoisonValue>(CmpEq) || match(CmpEq, m_One());
+  Constant *CmpEq = ConstantFoldCompareInstruction(ICmpInst::ICMP_EQ, C0, C1);
+  return CmpEq && (isa<PoisonValue>(CmpEq) || match(CmpEq, m_One()));
 }
 
 static bool
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 5b7fa13f2e83..160a17584ca3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -854,8 +854,9 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
 
     if (auto *CSrc0 = dyn_cast<Constant>(Src0)) {
       if (auto *CSrc1 = dyn_cast<Constant>(Src1)) {
-        Constant *CCmp = ConstantExpr::getCompare(CCVal, CSrc0, CSrc1);
-        if (CCmp->isNullValue()) {
+        Constant *CCmp = ConstantFoldCompareInstOperands(
+            (ICmpInst::Predicate)CCVal, CSrc0, CSrc1, DL);
+        if (CCmp && CCmp->isNullValue()) {
           return IC.replaceInstUsesWith(
               II, IC.Builder.CreateSExt(CCmp, II.getType()));
         }
diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index e46fc034cc26..8e75e185f0f6 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -26,20 +26,21 @@ using namespace llvm;
 
 /// Return a constant boolean vector that has true elements in all positions
 /// where the input constant data vector has an element with the sign bit set.
-static Constant *getNegativeIsTrueBoolVec(Constant *V) {
+static Constant *getNegativeIsTrueBoolVec(Constant *V, const DataLayout &DL) {
   VectorType *IntTy = VectorType::getInteger(cast<VectorType>(V->getType()));
   V = ConstantExpr::getBitCast(V, IntTy);
-  V = ConstantExpr::getICmp(CmpInst::ICMP_SGT, Constant::getNullValue(IntTy),
-                            V);
+  V = ConstantFoldCompareInstOperands(CmpInst::ICMP_SGT,
+                                      Constant::getNullValue(IntTy), V, DL);
+  assert(V && "Vector must be foldable");
   return V;
 }
 
 /// Convert the x86 XMM integer vector mask to a vector of bools based on
 /// each element's most significant bit (the sign bit).
-static Value *getBoolVecFromMask(Value *Mask) {
+static Value *getBoolVecFromMask(Value *Mask, const DataLayout &DL) {
   // Fold Constant Mask.
   if (auto *ConstantMask = dyn_cast<ConstantDataVector>(Mask))
-    return getNegativeIsTrueBoolVec(ConstantMask);
+    return getNegativeIsTrueBoolVec(ConstantMask, DL);
 
   // Mask was extended from a boolean vector.
   Value *ExtMask;
@@ -65,7 +66,7 @@ static Instruction *simplifyX86MaskedLoad(IntrinsicInst &II, InstCombiner &IC) {
 
   // The mask is constant or extended from a bool vector. Convert this x86
   // intrinsic to the LLVM intrinsic to allow target-independent optimizations.
-  if (Value *BoolMask = getBoolVecFromMask(Mask)) {
+  if (Value *BoolMask = getBoolVecFromMask(Mask, IC.getDataLayout())) {
     // First, cast the x86 intrinsic scalar pointer to a vector pointer to match
     // the LLVM intrinsic definition for the pointer argument.
     unsigned AddrSpace = cast<PointerType>(Ptr->getType())->getAddressSpace();
@@ -102,7 +103,7 @@ static bool simplifyX86MaskedStore(IntrinsicInst &II, InstCombiner &IC) {
 
   // The mask is constant or extended from a bool vector. Convert this x86
   // intrinsic to the LLVM intrinsic to allow target-independent optimizations.
-  if (Value *BoolMask = getBoolVecFromMask(Mask)) {
+  if (Value *BoolMask = getBoolVecFromMask(Mask, IC.getDataLayout())) {
     unsigned AddrSpace = cast<PointerType>(Ptr->getType())->getAddressSpace();
     PointerType *VecPtrTy = PointerType::get(Vec->getType(), AddrSpace);
     Value *PtrCast = IC.Builder.CreateBitCast(Ptr, VecPtrTy, "castvec");
@@ -2688,7 +2689,8 @@ X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
 
     // Constant Mask - select 1st/2nd argument lane based on top bit of mask.
     if (auto *ConstantMask = dyn_cast<ConstantDataVector>(Mask)) {
-      Constant *NewSelector = getNegativeIsTrueBoolVec(ConstantMask);
+      Constant *NewSelector =
+          getNegativeIsTrueBoolVec(ConstantMask, IC.getDataLayout());
       return SelectInst::Create(NewSelector, Op1, Op0, "blendv");
     }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index a52c70dbdf3f..8695e9e69df2 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -2504,8 +2504,8 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
           match(C1, m_Power2())) {
         Constant *Log2C1 = ConstantExpr::getExactLogBase2(C1);
         Constant *Cmp =
-            ConstantExpr::getCompare(ICmpInst::ICMP_ULT, Log2C3, C2);
-        if (Cmp->isZeroValue()) {
+            ConstantFoldCompareInstOperands(ICmpInst::ICMP_ULT, Log2C3, C2, DL);
+        if (Cmp && Cmp->isZeroValue()) {
           // iff C1,C3 is pow2 and Log2(C3) >= C2:
           // ((C1 >> X) << C2) & C3 -> X == (cttz(C1)+C2-cttz(C3)) ? C3 : 0
           Constant *ShlC = ConstantExpr::getAdd(C2, Log2C1);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index d7433ad3599f..77534e0d3613 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1982,7 +1982,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       if (ModuloC != ShAmtC)
         return replaceOperand(*II, 2, ModuloC);
 
-      assert(match(ConstantExpr::getICmp(ICmpInst::ICMP_UGT, WidthC, ShAmtC),
+      assert(match(ConstantFoldCompareInstOperands(ICmpInst::ICMP_UGT, WidthC,
+                                                   ShAmtC, DL),
                    m_One()) &&
              "Shift amount expected to be modulo bitwidth");
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 7092fb5e509b..e1a3194a1beb 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -3176,15 +3176,12 @@ Instruction *InstCombinerImpl::foldICmpSelectConstant(ICmpInst &Cmp,
                               C3GreaterThan)) {
     assert(C1LessThan && C2Equal && C3GreaterThan);
 
-    bool TrueWhenLessThan =
-        ConstantExpr::getCompare(Cmp.getPredicate(), C1LessThan, C)
-            ->isAllOnesValue();
-    bool TrueWhenEqual =
-        ConstantExpr::getCompare(Cmp.getPredicate(), C2Equal, C)
-            ->isAllOnesValue();
-    bool TrueWhenGreaterThan =
-        ConstantExpr::getCompare(Cmp.getPredicate(), C3GreaterThan, C)
-            ->isAllOnesValue();
+    bool TrueWhenLessThan = ICmpInst::compare(
+        C1LessThan->getValue(), C->getValue(), Cmp.getPredicate());
+    bool TrueWhenEqual = ICmpInst::compare(C2Equal->getValue(), C->getValue(),
+                                           Cmp.getPredicate());
+    bool TrueWhenGreaterThan = ICmpInst::compare(
+        C3GreaterThan->getValue(), C->getValue(), Cmp.getPredicate());
 
     // This generates the new instruction that will replace the original Cmp
     // Instruction. Instead of enumerating the various combinations when
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 8818369e7945..ee090e012508 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1365,7 +1365,8 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel,
 // Also ULT predicate can also be UGT iff C0 != -1 (+invert result)
 //      SLT predicate can also be SGT iff C2 != INT_MAX (+invert res.)
 static Value *canonicalizeClampLike(SelectInst &Sel0, ICmpInst &Cmp0,
-                                    InstCombiner::BuilderTy &Builder) {
+                                    InstCombiner::BuilderTy &Builder,
+                                    InstCombiner &IC) {
   Value *X = Sel0.getTrueValue();
   Value *Sel1 = Sel0.getFalseValue();
 
@@ -1493,14 +1494,14 @@ static Value *canonicalizeClampLike(SelectInst &Sel0, ICmpInst &Cmp0,
     std::swap(ThresholdLowIncl, ThresholdHighExcl);
 
   // The fold has a precondition 1: C2 s>= ThresholdLow
-  auto *Precond1 = ConstantExpr::getICmp(ICmpInst::Predicate::ICMP_SGE, C2,
-                                         ThresholdLowIncl);
-  if (!match(Precond1, m_One()))
+  auto *Precond1 = ConstantFoldCompareInstOperands(
+      ICmpInst::Predicate::ICMP_SGE, C2, ThresholdLowIncl, IC.getDataLayout());
+  if (!Precond1 || !match(Precond1, m_One()))
     return nullptr;
   // The fold has a precondition 2: C2 s<= ThresholdHigh
-  auto *Precond2 = ConstantExpr::getICmp(ICmpInst::Predicate::ICMP_SLE, C2,
-                                         ThresholdHighExcl);
-  if (!match(Precond2, m_One()))
+  auto *Precond2 = ConstantFoldCompareInstOperands(
+      ICmpInst::Predicate::ICMP_SLE, C2, ThresholdHighExcl, IC.getDataLayout());
+  if (!Precond2 || !match(Precond2, m_One()))
     return nullptr;
 
   // If we are matching from a truncated input, we need to sext the
@@ -1803,7 +1804,7 @@ Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
   if (Value *V = foldSelectInstWithICmpConst(SI, ICI, Builder))
     return replaceInstUsesWith(SI, V);
 
-  if (Value *V = canonicalizeClampLike(SI, *ICI, Builder))
+  if (Value *V = canonicalizeClampLike(SI, *ICI, Builder, *this))
     return replaceInstUsesWith(SI, V);
 
   if (Instruction *NewSel =
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index b6f8b24f43b8..6c25ff215c37 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -808,9 +808,12 @@ Instruction *InstCombinerImpl::tryFoldInstWithCtpopWithNot(Instruction *I) {
   Constant *BitWidthC = ConstantInt::get(Ty, Ty->getScalarSizeInBits());
   // Need extra check for icmp. Note if this check is true, it generally means
   // the icmp will simplify to true/false.
-  if (Opc == Instruction::ICmp && !cast<ICmpInst>(I)->isEquality() &&
-      !ConstantExpr::getICmp(ICmpInst::ICMP_UGT, C, BitWidthC)->isZeroValue())
-    return nullptr;
+  if (Opc == Instruction::ICmp && !cast<ICmpInst>(I)->isEquality()) {
+    Constant *Cmp =
+        ConstantFoldCompareInstOperands(ICmpInst::ICMP_UGT, C, BitWidthC, DL);
+    if (!Cmp || !Cmp->isZeroValue())
+      return nullptr;
+  }
 
   // Check we can invert `(not x)` for free.
   bool Consumes = false;
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 08d82fa66da3..802467b5b183 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -868,7 +868,8 @@ bool JumpThreadingPass::computeValueKnownInPredecessorsImpl(
 
       for (const auto &LHSVal : LHSVals) {
         Constant *V = LHSVal.first;
-        Constant *Folded = ConstantExpr::getCompare(Pred, V, CmpConst);
+        Constant *Folded =
+            ConstantFoldCompareInstOperands(Pred, V, CmpConst, DL);
         if (Constant *KC = getKnownConstant(Folded, WantInteger))
           Result.emplace_back(KC, LHSVal.second);
       }
@@ -1509,7 +1510,8 @@ findMostPopularDest(BasicBlock *BB,
 // BB->getSinglePredecessor() and then on to BB.
 Constant *JumpThreadingPass::evaluateOnPredecessorEdge(BasicBlock *BB,
                                                        BasicBlock *PredPredBB,
-                                                       Value *V) {
+                                                       Value *V,
+                                                       const DataLayout &DL) {
   BasicBlock *PredBB = BB->getSinglePredecessor();
   assert(PredBB && "Expected a single predecessor");
 
@@ -1534,11 +1536,12 @@ Constant *JumpThreadingPass::evaluateOnPredecessorEdge(BasicBlock *BB,
   if (CmpInst *CondCmp = dyn_cast<CmpInst>(V)) {
     if (CondCmp->getParent() == BB) {
       Constant *Op0 =
-          evaluateOnPredecessorEdge(BB, PredPredBB, CondCmp->getOperand(0));
+          evaluateOnPredecessorEdge(BB, PredPredBB, CondCmp->getOperand(0), DL);
       Constant *Op1 =
-          evaluateOnPredecessorEdge(BB, PredPredBB, CondCmp->getOperand(1));
+          evaluateOnPredecessorEdge(BB, PredPredBB, CondCmp->getOperand(1), DL);
       if (Op0 && Op1) {
-        return ConstantExpr::getCompare(CondCmp->getPredicate(), Op0, Op1);
+        return ConstantFoldCompareInstOperands(CondCmp->getPredicate(), Op0,
+                                               Op1, DL);
       }
     }
     return nullptr;
@@ -2191,12 +2194,13 @@ bool JumpThreadingPass::maybethreadThroughTwoBasicBlocks(BasicBlock *BB,
   unsigned OneCount = 0;
   BasicBlock *ZeroPred = nullptr;
   BasicBlock *OnePred = nullptr;
+  const DataLayout &DL = BB->getModule()->getDataLayout();
   for (BasicBlock *P : predecessors(PredBB)) {
     // If PredPred ends with IndirectBrInst, we can't handle it.
     if (isa<IndirectBrInst>(P->getTerminator()))
       continue;
     if (ConstantInt *CI = dyn_cast_or_null<ConstantInt>(
-            evaluateOnPredecessorEdge(BB, P, Cond))) {
+            evaluateOnPredecessorEdge(BB, P, Cond, DL))) {
       if (CI->isZero()) {
         ZeroCount++;
         ZeroPred = P;
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 23a896c59bf6..93701b2a7791 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -6580,16 +6580,17 @@ static void reuseTableCompare(
   Constant *FalseConst = ConstantInt::getFalse(RangeCmp->getType());
 
   // Check if the compare with the default value is constant true or false.
-  Constant *DefaultConst = ConstantExpr::getICmp(CmpInst->getPredicate(),
-                                                 DefaultValue, CmpOp1, true);
+  const DataLayout &DL = PhiBlock->getModule()->getDataLayout();
+  Constant *DefaultConst = ConstantFoldCompareInstOperands(
+      CmpInst->getPredicate(), DefaultValue, CmpOp1, DL);
   if (DefaultConst != TrueConst && DefaultConst != FalseConst)
     return;
 
   // Check if the compare with the case values is distinct from the default
   // compare result.
   for (auto ValuePair : Values) {
-    Constant *CaseConst = ConstantExpr::getICmp(CmpInst->getPredicate(),
-                                                ValuePair.second, CmpOp1, true);
+    Constant *CaseConst = ConstantFoldCompareInstOperands(
+        CmpInst->getPredicate(), ValuePair.second, CmpOp1, DL);
     if (!CaseConst || CaseConst == DefaultConst ||
         (CaseConst != TrueConst && CaseConst != FalseConst))
       return;
diff --git a/llvm/test/Transforms/JumpThreading/thread-two-bbs.ll b/llvm/test/Transforms/JumpThreading/thread-two-bbs.ll
index f7e6b2189dc8..09394a946241 100644
--- a/llvm/test/Transforms/JumpThreading/thread-two-bbs.ll
+++ b/llvm/test/Transforms/JumpThreading/thread-two-bbs.ll
@@ -130,8 +130,8 @@ exit:
 }
 
 
-; Verify that we do *not* thread any edge.  We used to evaluate
-; constant expressions like:
+; Verify that we thread the edge correctly.  We used to evaluate constant
+; expressions like:
 ;
 ;   icmp ugt ptr null, inttoptr (i64 4 to ptr)
 ;
@@ -141,16 +141,17 @@ define void @icmp_ult_null_constexpr(ptr %arg1, ptr %arg2) {
 ; CHECK-LABEL: @icmp_ult_null_constexpr(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq ptr [[ARG1:%.*]], null
-; CHECK-NEXT:    br i1 [[CMP1]], label [[BB_BAR1:%.*]], label [[BB_END:%.*]]
-; CHECK:       bb_bar1:
-; CHECK-NEXT:    call void @bar(i32 1)
-; CHECK-NEXT:    br label [[BB_END]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[BB_END_THREAD:%.*]], label [[BB_END:%.*]]
 ; CHECK:       bb_end:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne ptr [[ARG2:%.*]], null
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[BB_CONT:%.*]], label [[BB_BAR2:%.*]]
+; CHECK:       bb_end.thread:
+; CHECK-NEXT:    call void @bar(i32 1)
+; CHECK-NEXT:    [[CMP21:%.*]] = icmp ne ptr [[ARG2]], null
+; CHECK-NEXT:    br i1 [[CMP21]], label [[BB_EXIT:%.*]], label [[BB_BAR2]]
 ; CHECK:       bb_bar2:
 ; CHECK-NEXT:    call void @bar(i32 2)
-; CHECK-NEXT:    br label [[BB_EXIT:%.*]]
+; CHECK-NEXT:    br label [[BB_EXIT]]
 ; CHECK:       bb_cont:
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult ptr [[ARG1]], inttoptr (i64 4 to ptr)
 ; CHECK-NEXT:    br i1 [[CMP3]], label [[BB_EXIT]], label [[BB_BAR3:%.*]]
-- 
GitLab


From f865dbff17ca516d605b053d5556c1498c300a42 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <jeffrey.byrnes@amd.com>
Date: Thu, 9 May 2024 16:57:36 -0700
Subject: [PATCH 0360/1206] [SeparateConstOffsetFromGEP] Support GEP reordering
 for different types (#90802)

This doesn't show up in existing lit tests, but has an impact on real
code -- especially after the canonicalization of GEPs to i8.

Alive2 tests for the inbounds handling:

Case 1: https://alive2.llvm.org/ce/z/6bfFY3
Case 2: https://alive2.llvm.org/ce/z/DkLMLF
---
 .../Scalar/SeparateConstOffsetFromGEP.cpp     |  28 +-
 .../AMDGPU/reorder-gep-inbounds.ll            | 297 +++++++++++-
 .../AMDGPU/reorder-gep.ll                     | 437 +++++++++++-------
 .../NVPTX/lower-gep-reorder.ll                |  12 +-
 .../SeparateConstOffsetFromGEP/reorder-gep.ll |  63 +++
 5 files changed, 629 insertions(+), 208 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index c54a956fc7e2..9f85396cde25 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -972,22 +972,13 @@ SeparateConstOffsetFromGEP::lowerToArithmetics(GetElementPtrInst *Variadic,
 
 bool SeparateConstOffsetFromGEP::reorderGEP(GetElementPtrInst *GEP,
                                             TargetTransformInfo &TTI) {
-  Type *GEPType = GEP->getResultElementType();
-  // TODO: support reordering for non-trivial GEP chains
-  if (GEPType->isAggregateType() || GEP->getNumIndices() != 1)
+  if (GEP->getNumIndices() != 1)
     return false;
 
   auto PtrGEP = dyn_cast<GetElementPtrInst>(GEP->getPointerOperand());
   if (!PtrGEP)
     return false;
-  Type *PtrGEPType = PtrGEP->getResultElementType();
-  // TODO: support reordering for non-trivial GEP chains
-  if (PtrGEPType->isAggregateType() || PtrGEP->getNumIndices() != 1)
-    return false;
-
-  // TODO: support reordering for non-trivial GEP chains
-  if (PtrGEPType != GEPType ||
-      PtrGEP->getSourceElementType() != GEP->getSourceElementType())
+  if (PtrGEP->getNumIndices() != 1)
     return false;
 
   bool NestedNeedsExtraction;
@@ -1002,8 +993,6 @@ bool SeparateConstOffsetFromGEP::reorderGEP(GetElementPtrInst *GEP,
                                  /*HasBaseReg=*/true, /*Scale=*/0, AddrSpace))
     return false;
 
-  IRBuilder<> Builder(GEP);
-  Builder.SetCurrentDebugLocation(GEP->getDebugLoc());
   bool GEPInBounds = GEP->isInBounds();
   bool PtrGEPInBounds = PtrGEP->isInBounds();
   bool IsChainInBounds = GEPInBounds && PtrGEPInBounds;
@@ -1018,13 +1007,14 @@ bool SeparateConstOffsetFromGEP::reorderGEP(GetElementPtrInst *GEP,
     }
   }
 
+  IRBuilder<> Builder(GEP);
   // For trivial GEP chains, we can swap the indicies.
-  auto NewSrc = Builder.CreateGEP(PtrGEPType, PtrGEP->getPointerOperand(),
-                                  SmallVector<Value *, 4>(GEP->indices()));
-  cast<GetElementPtrInst>(NewSrc)->setIsInBounds(IsChainInBounds);
-  auto NewGEP = Builder.CreateGEP(GEPType, NewSrc,
-                                  SmallVector<Value *, 4>(PtrGEP->indices()));
-  cast<GetElementPtrInst>(NewGEP)->setIsInBounds(IsChainInBounds);
+  Value *NewSrc = Builder.CreateGEP(
+      GEP->getSourceElementType(), PtrGEP->getPointerOperand(),
+      SmallVector<Value *, 4>(GEP->indices()), "", IsChainInBounds);
+  Value *NewGEP = Builder.CreateGEP(PtrGEP->getSourceElementType(), NewSrc,
+                                    SmallVector<Value *, 4>(PtrGEP->indices()),
+                                    "", IsChainInBounds);
   GEP->replaceAllUsesWith(NewGEP);
   RecursivelyDeleteTriviallyDeadInstructions(GEP);
   return true;
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep-inbounds.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep-inbounds.ll
index c24bbd5f658f..16e47f057bab 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep-inbounds.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep-inbounds.ll
@@ -1,28 +1,27 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -S -passes=separate-const-offset-from-gep < %s | FileCheck %s
 
-define void @inboundsPossiblyNegative(ptr %in.ptr, i32 %in.idx1) {
+define void @inboundsPossiblyNegative(ptr %in.ptr, i64 %in.idx1) {
 ; CHECK-LABEL: define void @inboundsPossiblyNegative(
-; CHECK-SAME: ptr [[IN_PTR:%.*]], i32 [[IN_IDX1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IN_IDX1]] to i64
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <2 x i8>, ptr [[IN_PTR]], i64 [[IDXPROM]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <2 x i8>, ptr [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <2 x i8>, ptr [[IN_PTR]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <2 x i8>, ptr [[TMP0]], i64 1
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  %const1 = getelementptr inbounds <2 x i8>, ptr %in.ptr, i32 1
-  %idx1 = getelementptr inbounds <2 x i8>, ptr %const1, i32 %in.idx1
+  %const1 = getelementptr inbounds <2 x i8>, ptr %in.ptr, i64 1
+  %idx1 = getelementptr inbounds <2 x i8>, ptr %const1, i64 %in.idx1
   ret void
 }
 
-define void @inboundsNonNegative(ptr %in.ptr, i32 %in.idx1) {
-; CHECK-LABEL: define void @inboundsNonNegative(
+define void @inboundsNonNegative_nonCanonical(ptr %in.ptr, i32 %in.idx1) {
+; CHECK-LABEL: define void @inboundsNonNegative_nonCanonical(
 ; CHECK-SAME: ptr [[IN_PTR:%.*]], i32 [[IN_IDX1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i32 [[IN_IDX1]], 2147483647
-; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IN_IDX1_NNEG]] to i64
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <2 x i8>, ptr [[IN_PTR]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[IN_IDX1_NNEG1:%.*]] = and i32 [[IN_IDX1]], 2147483647
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = sext i32 [[IN_IDX1_NNEG1]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <2 x i8>, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <2 x i8>, ptr [[TMP0]], i32 1
 ; CHECK-NEXT:    ret void
 ;
@@ -33,19 +32,277 @@ entry:
   ret void
 }
 
-define void @inboundsNonchained(ptr %in.ptr, i32 %in.idx1) {
+define void @inboundsNonNegative(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @inboundsNonNegative(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <2 x i8>, ptr [[IN_PTR]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <2 x i8>, ptr [[TMP0]], i64 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr inbounds <2 x i8>, ptr %in.ptr, i64 1
+  %idx1 = getelementptr inbounds <2 x i8>, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @inboundsNonchained(ptr %in.ptr, i64 %in.idx1) {
 ; CHECK-LABEL: define void @inboundsNonchained(
-; CHECK-SAME: ptr [[IN_PTR:%.*]], i32 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i32 [[IN_IDX1]], 2147483647
-; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[IN_IDX1_NNEG]] to i64
+; CHECK-NEXT:    [[IDXPROM:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <2 x i8>, ptr [[IN_PTR]], i64 [[IDXPROM]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <2 x i8>, ptr [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <2 x i8>, ptr [[TMP0]], i64 1
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  %in.idx1.nneg = and i32 %in.idx1, 2147483647
-  %const1 = getelementptr inbounds <2 x i8>, ptr %in.ptr, i32 1
-  %idx1 = getelementptr <2 x i8>, ptr %const1, i32 %in.idx1.nneg
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr inbounds <2 x i8>, ptr %in.ptr, i64 1
+  %idx1 = getelementptr <2 x i8>, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @inboundsNonNegativeType_i16i8(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @inboundsNonNegativeType_i16i8(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[IN_PTR]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i64 1024
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr inbounds i16, ptr %in.ptr, i64 1024
+  %idx1 = getelementptr inbounds i8, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @inboundsNonNegative_i8i16(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @inboundsNonNegative_i8i16(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[IN_PTR]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 1024
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr inbounds i8, ptr %in.ptr, i64 1024
+  %idx1 = getelementptr inbounds i16, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @inboundsNonchained_first(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @inboundsNonchained_first(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i32, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 1024
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr inbounds i8, ptr %in.ptr, i64 1024
+  %idx1 = getelementptr i32, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @inboundsNonchained_second(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @inboundsNonchained_second(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i64, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 1024
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr i8, ptr %in.ptr, i64 1024
+  %idx1 = getelementptr inbounds i64, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @notInbounds(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @notInbounds(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i128, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 1024
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr i8, ptr %in.ptr, i64 1024
+  %idx1 = getelementptr i128, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @vectorType1(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @vectorType1(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 2147483647
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <2 x i8>, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i8>, ptr [[TMP0]], i32 3
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 2147483647
+  %const1 = getelementptr inbounds <4 x i8>, ptr %in.ptr, i32 3
+  %idx1 = getelementptr inbounds <2 x i8>, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @vectorType2(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @vectorType2(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 2147483647
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <4 x half>, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i8>, ptr [[TMP0]], i32 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 2147483647
+  %const1 = getelementptr inbounds <4 x i8>, ptr %in.ptr, i32 1
+  %idx1 = getelementptr inbounds <4 x half>, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @vectorType3(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @vectorType3(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 2147483647
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds ptr, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x ptr>, ptr [[TMP0]], i32 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 2147483647
+  %const1 = getelementptr inbounds <4 x ptr>, ptr %in.ptr, i32 1
+  %idx1 = getelementptr inbounds ptr, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @vectorType4(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @vectorType4(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 2147483647
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <8 x ptr addrspace(1)>, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x ptr>, ptr [[TMP0]], i32 3
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 2147483647
+  %const1 = getelementptr inbounds <4 x ptr>, ptr %in.ptr, i32 3
+  %idx1 = getelementptr inbounds <8 x ptr addrspace(1)>, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+
+define void @ptrType(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @ptrType(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 2147483647
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds ptr, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds ptr addrspace(2), ptr [[TMP0]], i32 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 2147483647
+  %const1 = getelementptr inbounds ptr addrspace(2), ptr %in.ptr, i32 1
+  %idx1 = getelementptr inbounds ptr, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @ptrType2(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @ptrType2(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 2147483647
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds ptr addrspace(3), ptr [[TMP0]], i32 3
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 2147483647
+  %const1 = getelementptr inbounds ptr addrspace(3), ptr %in.ptr, i32 3
+  %idx1 = getelementptr inbounds i64, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @ptrType3(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @ptrType3(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 2147483647
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds ptr addrspace(7), ptr [[TMP0]], i32 3
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 2147483647
+  %const1 = getelementptr inbounds ptr addrspace(7), ptr %in.ptr, i32 3
+  %idx1 = getelementptr inbounds i16, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @addrspace1(ptr addrspace(1) %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @addrspace1(
+; CHECK-SAME: ptr addrspace(1) [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i128, ptr addrspace(1) [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP0]], i64 1024
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr inbounds i8, ptr addrspace(1) %in.ptr, i64 1024
+  %idx1 = getelementptr inbounds i128, ptr addrspace(1) %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @addrspace3(ptr addrspace(3) %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @addrspace3(
+; CHECK-SAME: ptr addrspace(3) [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX1_NNEG]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i128, ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP0]], i64 1024
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr inbounds i8, ptr addrspace(3) %in.ptr, i64 1024
+  %idx1 = getelementptr inbounds i128, ptr addrspace(3) %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @addrspace7(ptr addrspace(7) %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @addrspace7(
+; CHECK-SAME: ptr addrspace(7) [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX1_NNEG]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i128, ptr addrspace(7) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(7) [[TMP0]], i64 1024
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr inbounds i8, ptr addrspace(7) %in.ptr, i64 1024
+  %idx1 = getelementptr inbounds i128, ptr addrspace(7) %const1, i64 %in.idx1.nneg
   ret void
 }
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll
index 7137f0fb66fd..b4119f0b50b4 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll
@@ -1,175 +1,286 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a --start-before=separate-const-offset-from-gep < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a --passes=separate-const-offset-from-gep < %s | FileCheck %s
 
-define protected amdgpu_kernel void @sink_addr(ptr addrspace(3) %in.ptr, i32 %in.idx0, i32 %in.idx1) {
-; CHECK-LABEL: sink_addr:
-; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[6:7], 0x0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    s_lshl_b32 s3, s1, 1
-; CHECK-NEXT:    s_add_i32 s0, s0, s3
-; CHECK-NEXT:    s_lshl_b32 s2, s2, 1
-; CHECK-NEXT:    s_add_i32 s0, s0, s2
-; CHECK-NEXT:    s_cmp_lg_u32 s1, 0
-; CHECK-NEXT:    s_cbranch_scc1 .LBB0_2
-; CHECK-NEXT:  ; %bb.1: ; %bb.1
-; CHECK-NEXT:    v_mov_b32_e32 v12, s0
-; CHECK-NEXT:    ds_read_b128 v[0:3], v12
-; CHECK-NEXT:    ds_read_b128 v[4:7], v12 offset:512
-; CHECK-NEXT:    ds_read_b128 v[8:11], v12 offset:1024
-; CHECK-NEXT:    ds_read_b128 v[12:15], v12 offset:1536
-; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v[0:3]
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v[4:7]
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v[8:11]
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v[12:15]
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:  .LBB0_2: ; %end
-; CHECK-NEXT:    s_add_i32 s1, s0, 0x200
-; CHECK-NEXT:    v_mov_b32_e32 v0, s0
-; CHECK-NEXT:    s_add_i32 s2, s0, 0x400
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v0
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_mov_b32_e32 v0, s1
-; CHECK-NEXT:    s_add_i32 s3, s0, 0x600
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v0
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v0
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_mov_b32_e32 v0, s3
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v0
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    s_endpgm
+define void @sink_addr(ptr addrspace(3) %in.ptr, i64 %in.idx0, i64 %in.idx1) {
+; CHECK-LABEL: define void @sink_addr(
+; CHECK-SAME: ptr addrspace(3) [[IN_PTR:%.*]], i64 [[IN_IDX0:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX0]] to i32
+; CHECK-NEXT:    [[BASE:%.*]] = getelementptr half, ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM1]]
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr half, ptr addrspace(3) [[TMP0]], i64 256
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr half, ptr addrspace(3) [[TMP2]], i64 512
+; CHECK-NEXT:    [[IDXPROM4:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr half, ptr addrspace(3) [[TMP4]], i64 768
+; CHECK-NEXT:    ret void
+;
 entry:
-  %base = getelementptr half, ptr addrspace(3) %in.ptr, i32 %in.idx0
-  %idx0 = getelementptr half, ptr addrspace(3) %base, i32 %in.idx1
-  %const1 = getelementptr half, ptr addrspace(3) %base, i32 256
-  %idx1 = getelementptr half, ptr addrspace(3) %const1, i32 %in.idx1
-  %const2 = getelementptr half, ptr addrspace(3) %base, i32 512
-  %idx2 = getelementptr half, ptr addrspace(3) %const2, i32 %in.idx1
-  %const3 = getelementptr half, ptr addrspace(3) %base, i32 768
-  %idx3 = getelementptr half, ptr addrspace(3) %const3, i32 %in.idx1
-  %cmp0 = icmp eq i32 %in.idx0, 0
-  br i1 %cmp0, label %bb.1, label %end
+  %base = getelementptr half, ptr addrspace(3) %in.ptr, i64 %in.idx0
+  %idx0 = getelementptr half, ptr addrspace(3) %base, i64 %in.idx1
+  %const1 = getelementptr half, ptr addrspace(3) %base, i64 256
+  %idx1 = getelementptr half, ptr addrspace(3) %const1, i64 %in.idx1
+  %const2 = getelementptr half, ptr addrspace(3) %base, i64 512
+  %idx2 = getelementptr half, ptr addrspace(3) %const2, i64 %in.idx1
+  %const3 = getelementptr half, ptr addrspace(3) %base, i64 768
+  %idx3 = getelementptr half, ptr addrspace(3) %const3, i64 %in.idx1
+  ret void
+}
+
+define void @illegal_addr_mode(ptr addrspace(3) %in.ptr, i64 %in.idx0, i64 %in.idx1) {
+; CHECK-LABEL: define void @illegal_addr_mode(
+; CHECK-SAME: ptr addrspace(3) [[IN_PTR:%.*]], i64 [[IN_IDX0:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX0]] to i32
+; CHECK-NEXT:    [[BASE:%.*]] = getelementptr half, ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM1]]
+; CHECK-NEXT:    [[CONST1:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i64 38192
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr half, ptr addrspace(3) [[CONST1]], i32 [[IDXPROM2]]
+; CHECK-NEXT:    [[CONST2:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i64 38448
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr half, ptr addrspace(3) [[CONST2]], i32 [[IDXPROM3]]
+; CHECK-NEXT:    [[CONST3:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i64 38764
+; CHECK-NEXT:    [[IDXPROM4:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr half, ptr addrspace(3) [[CONST3]], i32 [[IDXPROM4]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %base = getelementptr half, ptr addrspace(3) %in.ptr, i64 %in.idx0
+  %idx0 = getelementptr half, ptr addrspace(3) %base, i64 %in.idx1
+  %const1 = getelementptr half, ptr addrspace(3) %base, i64 38192
+  %idx1 = getelementptr half, ptr addrspace(3) %const1, i64 %in.idx1
+  %const2 = getelementptr half, ptr addrspace(3) %base, i64 38448
+  %idx2 = getelementptr half, ptr addrspace(3) %const2, i64 %in.idx1
+  %const3 = getelementptr half, ptr addrspace(3) %base, i64 38764
+  %idx3 = getelementptr half, ptr addrspace(3) %const3, i64 %in.idx1
+  ret void
+}
+
+
+define void @reorder_i8half(ptr addrspace(3) %in.ptr, i64 %in.idx0, i64 %in.idx1) {
+; CHECK-LABEL: define void @reorder_i8half(
+; CHECK-SAME: ptr addrspace(3) [[IN_PTR:%.*]], i64 [[IN_IDX0:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX0]] to i32
+; CHECK-NEXT:    [[BASE:%.*]] = getelementptr i8, ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM1]]
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP0]], i64 256
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP2]], i64 512
+; CHECK-NEXT:    [[IDXPROM4:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i64 768
+; CHECK-NEXT:    ret void
+;
+entry:
+  %base = getelementptr i8, ptr addrspace(3) %in.ptr, i64 %in.idx0
+  %idx0 = getelementptr half, ptr addrspace(3) %base, i64 %in.idx1
+  %const1 = getelementptr i8, ptr addrspace(3) %base, i64 256
+  %idx1 = getelementptr half, ptr addrspace(3) %const1, i64 %in.idx1
+  %const2 = getelementptr i8, ptr addrspace(3) %base, i64 512
+  %idx2 = getelementptr half, ptr addrspace(3) %const2, i64 %in.idx1
+  %const3 = getelementptr i8, ptr addrspace(3) %base, i64 768
+  %idx3 = getelementptr half, ptr addrspace(3) %const3, i64 %in.idx1
+  ret void
+}
+
+define void @reorder_i64half(ptr addrspace(3) %in.ptr, i64 %in.idx0, i64 %in.idx1) {
+; CHECK-LABEL: define void @reorder_i64half(
+; CHECK-SAME: ptr addrspace(3) [[IN_PTR:%.*]], i64 [[IN_IDX0:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX0]] to i32
+; CHECK-NEXT:    [[BASE:%.*]] = getelementptr i64, ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM1]]
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr addrspace(3) [[TMP0]], i64 256
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr addrspace(3) [[TMP2]], i64 512
+; CHECK-NEXT:    [[IDXPROM4:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr addrspace(3) [[TMP4]], i64 768
+; CHECK-NEXT:    ret void
+;
+entry:
+  %base = getelementptr i64, ptr addrspace(3) %in.ptr, i64 %in.idx0
+  %idx0 = getelementptr half, ptr addrspace(3) %base, i64 %in.idx1
+  %const1 = getelementptr i64, ptr addrspace(3) %base, i64 256
+  %idx1 = getelementptr half, ptr addrspace(3) %const1, i64 %in.idx1
+  %const2 = getelementptr i64, ptr addrspace(3) %base, i64 512
+  %idx2 = getelementptr half, ptr addrspace(3) %const2, i64 %in.idx1
+  %const3 = getelementptr i64, ptr addrspace(3) %base, i64 768
+  %idx3 = getelementptr half, ptr addrspace(3) %const3, i64 %in.idx1
+  ret void
+}
 
-bb.1:
-  %val0 = load <8 x half>, ptr addrspace(3) %idx0, align 16
-  %val1 = load <8 x half>, ptr addrspace(3) %idx1, align 16
-  %val2 = load <8 x half>, ptr addrspace(3) %idx2, align 16
-  %val3 = load <8 x half>, ptr addrspace(3) %idx3, align 16
-  call void asm sideeffect "; use $0", "v"(<8 x half> %val0)
-  call void asm sideeffect "; use $0", "v"(<8 x half> %val1)
-  call void asm sideeffect "; use $0", "v"(<8 x half> %val2)
-  call void asm sideeffect "; use $0", "v"(<8 x half> %val3)
-  br label %end
+define void @reorder_halfi8(ptr addrspace(3) %in.ptr, i64 %in.idx0, i64 %in.idx1) {
+; CHECK-LABEL: define void @reorder_halfi8(
+; CHECK-SAME: ptr addrspace(3) [[IN_PTR:%.*]], i64 [[IN_IDX0:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX0]] to i32
+; CHECK-NEXT:    [[BASE:%.*]] = getelementptr half, ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr i8, ptr addrspace(3) [[BASE]], i32 [[IDXPROM1]]
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr addrspace(3) [[BASE]], i32 [[IDXPROM2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr half, ptr addrspace(3) [[TMP0]], i64 256
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr addrspace(3) [[BASE]], i32 [[IDXPROM3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr half, ptr addrspace(3) [[TMP2]], i64 512
+; CHECK-NEXT:    [[IDXPROM4:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr addrspace(3) [[BASE]], i32 [[IDXPROM4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr half, ptr addrspace(3) [[TMP4]], i64 768
+; CHECK-NEXT:    ret void
+;
+entry:
+  %base = getelementptr half, ptr addrspace(3) %in.ptr, i64 %in.idx0
+  %idx0 = getelementptr i8, ptr addrspace(3) %base, i64 %in.idx1
+  %const1 = getelementptr half, ptr addrspace(3) %base, i64 256
+  %idx1 = getelementptr i8, ptr addrspace(3) %const1, i64 %in.idx1
+  %const2 = getelementptr half, ptr addrspace(3) %base, i64 512
+  %idx2 = getelementptr i8, ptr addrspace(3) %const2, i64 %in.idx1
+  %const3 = getelementptr half, ptr addrspace(3) %base, i64 768
+  %idx3 = getelementptr i8, ptr addrspace(3) %const3, i64 %in.idx1
+  ret void
+}
 
-end:
-  call void asm sideeffect "; use $0", "v"(ptr addrspace(3) %idx0)
-  call void asm sideeffect "; use $0", "v"(ptr addrspace(3) %idx1)
-  call void asm sideeffect "; use $0", "v"(ptr addrspace(3) %idx2)
-  call void asm sideeffect "; use $0", "v"(ptr addrspace(3) %idx3)
+
+
+define void @bad_index(ptr addrspace(3) %in.ptr, i64 %in.idx0, i64 %in.idx1) {
+; CHECK-LABEL: define void @bad_index(
+; CHECK-SAME: ptr addrspace(3) [[IN_PTR:%.*]], i64 [[IN_IDX0:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX0]] to i32
+; CHECK-NEXT:    [[BASE:%.*]] = getelementptr half, ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM1]]
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i64 1
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP2]], i64 2
+; CHECK-NEXT:    [[IDXPROM4:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP0]], i64 3
+; CHECK-NEXT:    ret void
+;
+entry:
+  %base = getelementptr half, ptr addrspace(3) %in.ptr, i64 %in.idx0
+  %idx0 = getelementptr half, ptr addrspace(3) %base, i64 %in.idx1
+  %const1 = getelementptr i8, ptr addrspace(3) %base, i64 1
+  %idx1 = getelementptr half, ptr addrspace(3) %const1, i64 %in.idx1
+  %const2 = getelementptr i8, ptr addrspace(3) %base, i64 2
+  %idx2 = getelementptr half, ptr addrspace(3) %const2, i64 %in.idx1
+  %const3 = getelementptr i8, ptr addrspace(3) %base, i64 3
+  %idx3 = getelementptr half, ptr addrspace(3) %const3, i64 %in.idx1
   ret void
 }
 
-define protected amdgpu_kernel void @illegal_addr_mode(ptr addrspace(3) %in.ptr, i32 %in.idx0, i32 %in.idx1) {
-; CHECK-LABEL: illegal_addr_mode:
-; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    s_load_dwordx4 s[4:7], s[6:7], 0x0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    s_lshl_b32 s0, s5, 1
-; CHECK-NEXT:    s_lshl_b32 s1, s6, 1
-; CHECK-NEXT:    s_add_i32 s3, s4, s0
-; CHECK-NEXT:    s_add_i32 s3, s3, s1
-; CHECK-NEXT:    s_add_i32 s2, s3, 0x12a60
-; CHECK-NEXT:    s_add_i32 s1, s3, 0x12c60
-; CHECK-NEXT:    s_add_i32 s0, s3, 0x12ed8
-; CHECK-NEXT:    s_cmp_lg_u32 s5, 0
-; CHECK-NEXT:    s_cbranch_scc1 .LBB1_2
-; CHECK-NEXT:  ; %bb.1: ; %bb.1
-; CHECK-NEXT:    v_mov_b32_e32 v0, s3
-; CHECK-NEXT:    v_mov_b32_e32 v4, s2
-; CHECK-NEXT:    v_mov_b32_e32 v8, s1
-; CHECK-NEXT:    v_mov_b32_e32 v12, s0
-; CHECK-NEXT:    ds_read_b128 v[0:3], v0
-; CHECK-NEXT:    ds_read_b128 v[4:7], v4
-; CHECK-NEXT:    ds_read_b128 v[8:11], v8
-; CHECK-NEXT:    ds_read_b128 v[12:15], v12
-; CHECK-NEXT:    s_waitcnt lgkmcnt(3)
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v[0:3]
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    s_waitcnt lgkmcnt(2)
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v[4:7]
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    s_waitcnt lgkmcnt(1)
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v[8:11]
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v[12:15]
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:  .LBB1_2: ; %end
-; CHECK-NEXT:    v_mov_b32_e32 v0, s3
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v0
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_mov_b32_e32 v0, s2
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v0
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_mov_b32_e32 v0, s1
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v0
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_mov_b32_e32 v0, s0
-; CHECK-NEXT:    ;;#ASMSTART
-; CHECK-NEXT:    ; use v0
-; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    s_endpgm
+
+%struct.vec = type { [8 x i8], [4 x half] }
+define void @vector_struct_type(ptr addrspace(3) %in.ptr, i64 %in.idx0, i64 %in.idx1) {
+; CHECK-LABEL: define void @vector_struct_type(
+; CHECK-SAME: ptr addrspace(3) [[IN_PTR:%.*]], i64 [[IN_IDX0:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX0]] to i32
+; CHECK-NEXT:    [[BASE:%.*]] = getelementptr [1024 x %struct.vec], ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr i8, ptr addrspace(3) [[BASE]], i32 [[IDXPROM1]]
+; CHECK-NEXT:    [[CONST1:%.*]] = getelementptr [1024 x %struct.vec], ptr addrspace(3) [[BASE]], i64 256
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr i8, ptr addrspace(3) [[CONST1]], i32 [[IDXPROM2]]
+; CHECK-NEXT:    [[CONST2:%.*]] = getelementptr [1024 x %struct.vec], ptr addrspace(3) [[BASE]], i64 512
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr i8, ptr addrspace(3) [[CONST2]], i32 [[IDXPROM3]]
+; CHECK-NEXT:    [[CONST3:%.*]] = getelementptr [1024 x %struct.vec], ptr addrspace(3) [[BASE]], i64 768
+; CHECK-NEXT:    [[IDXPROM4:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr i8, ptr addrspace(3) [[CONST3]], i32 [[IDXPROM4]]
+; CHECK-NEXT:    ret void
+;
 entry:
-  %base = getelementptr half, ptr addrspace(3) %in.ptr, i32 %in.idx0
-  %idx0 = getelementptr half, ptr addrspace(3) %base, i32 %in.idx1
-  %const1 = getelementptr half, ptr addrspace(3) %base, i32 38192
-  %idx1 = getelementptr half, ptr addrspace(3) %const1, i32 %in.idx1
-  %const2 = getelementptr half, ptr addrspace(3) %base, i32 38448
-  %idx2 = getelementptr half, ptr addrspace(3) %const2, i32 %in.idx1
-  %const3 = getelementptr half, ptr addrspace(3) %base, i32 38764
-  %idx3 = getelementptr half, ptr addrspace(3) %const3, i32 %in.idx1
-  %cmp0 = icmp eq i32 %in.idx0, 0
-  br i1 %cmp0, label %bb.1, label %end
+  %base = getelementptr [1024 x %struct.vec], ptr addrspace(3) %in.ptr, i64 %in.idx0
+  %idx0 = getelementptr i8, ptr addrspace(3) %base, i64 %in.idx1
+  %const1 = getelementptr [1024 x %struct.vec], ptr addrspace(3) %base, i64 256
+  %idx1 = getelementptr i8, ptr addrspace(3) %const1, i64 %in.idx1
+  %const2 = getelementptr [1024 x %struct.vec], ptr addrspace(3) %base, i64 512
+  %idx2 = getelementptr i8, ptr addrspace(3) %const2, i64 %in.idx1
+  %const3 = getelementptr [1024 x %struct.vec], ptr addrspace(3) %base, i64 768
+  %idx3 = getelementptr i8, ptr addrspace(3) %const3, i64 %in.idx1
+  ret void
+}
 
-bb.1:
-  %val0 = load <8 x half>, ptr addrspace(3) %idx0, align 16
-  %val1 = load <8 x half>, ptr addrspace(3) %idx1, align 16
-  %val2 = load <8 x half>, ptr addrspace(3) %idx2, align 16
-  %val3 = load <8 x half>, ptr addrspace(3) %idx3, align 16
-  call void asm sideeffect "; use $0", "v"(<8 x half> %val0)
-  call void asm sideeffect "; use $0", "v"(<8 x half> %val1)
-  call void asm sideeffect "; use $0", "v"(<8 x half> %val2)
-  call void asm sideeffect "; use $0", "v"(<8 x half> %val3)
-  br label %end
+define void @struct_type(ptr addrspace(3) %in.ptr, i64 %in.idx0, i64 %in.idx1) {
+; CHECK-LABEL: define void @struct_type(
+; CHECK-SAME: ptr addrspace(3) [[IN_PTR:%.*]], i64 [[IN_IDX0:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX0]] to i32
+; CHECK-NEXT:    [[BASE:%.*]] = getelementptr [[STRUCT_VEC:%.*]], ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr i8, ptr addrspace(3) [[BASE]], i32 [[IDXPROM1]]
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr addrspace(3) [[BASE]], i32 [[IDXPROM2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_VEC]], ptr addrspace(3) [[TMP0]], i64 256
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr addrspace(3) [[BASE]], i32 [[IDXPROM3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr [[STRUCT_VEC]], ptr addrspace(3) [[TMP2]], i64 512
+; CHECK-NEXT:    [[IDXPROM4:%.*]] = trunc i64 [[IN_IDX1]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr addrspace(3) [[BASE]], i32 [[IDXPROM4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr [[STRUCT_VEC]], ptr addrspace(3) [[TMP4]], i64 768
+; CHECK-NEXT:    ret void
+;
+entry:
+  %base = getelementptr %struct.vec, ptr addrspace(3) %in.ptr, i64 %in.idx0
+  %idx0 = getelementptr i8, ptr addrspace(3) %base, i64 %in.idx1
+  %const1 = getelementptr %struct.vec, ptr addrspace(3) %base, i64 256
+  %idx1 = getelementptr i8, ptr addrspace(3) %const1, i64 %in.idx1
+  %const2 = getelementptr %struct.vec, ptr addrspace(3) %base, i64 512
+  %idx2 = getelementptr i8, ptr addrspace(3) %const2, i64 %in.idx1
+  %const3 = getelementptr %struct.vec, ptr addrspace(3) %base, i64 768
+  %idx3 = getelementptr i8, ptr addrspace(3) %const3, i64 %in.idx1
+  ret void
+}
 
-end:
-  call void asm sideeffect "; use $0", "v"(ptr addrspace(3) %idx0)
-  call void asm sideeffect "; use $0", "v"(ptr addrspace(3) %idx1)
-  call void asm sideeffect "; use $0", "v"(ptr addrspace(3) %idx2)
-  call void asm sideeffect "; use $0", "v"(ptr addrspace(3) %idx3)
+define void @struct_type_multiindex(ptr addrspace(3) %in.ptr, i64 %in.idx0, i32 %in.idx1, i64 %in.idx2) {
+; CHECK-LABEL: define void @struct_type_multiindex(
+; CHECK-SAME: ptr addrspace(3) [[IN_PTR:%.*]], i64 [[IN_IDX0:%.*]], i32 [[IN_IDX1:%.*]], i64 [[IN_IDX2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX0]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr [[STRUCT_VEC:%.*]], ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]], i32 0, i32 0
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = trunc i64 [[IN_IDX2]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP0]], i32 [[IDXPROM2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP1]], i32 2
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = trunc i64 [[IN_IDX0]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr [[STRUCT_VEC]], ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM3]], i32 0, i32 0
+; CHECK-NEXT:    [[IDXPROM5:%.*]] = trunc i64 [[IN_IDX2]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP3]], i32 [[IDXPROM5]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i32 4
+; CHECK-NEXT:    [[IDXPROM6:%.*]] = trunc i64 [[IN_IDX0]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr [[STRUCT_VEC]], ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM6]], i32 0, i32 0
+; CHECK-NEXT:    [[IDXPROM8:%.*]] = trunc i64 [[IN_IDX2]] to i32
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP6]], i32 [[IDXPROM8]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP7]], i32 6
+; CHECK-NEXT:    ret void
+;
+entry:
+  %const1 = getelementptr %struct.vec, ptr addrspace(3) %in.ptr, i64 %in.idx0, i32 0, i32 2
+  %idx1 = getelementptr i8, ptr addrspace(3) %const1, i64 %in.idx2
+  %const2 = getelementptr %struct.vec, ptr addrspace(3) %in.ptr, i64 %in.idx0, i32 0, i32 4
+  %idx2 = getelementptr i8, ptr addrspace(3) %const2, i64 %in.idx2
+  %const3 = getelementptr %struct.vec, ptr addrspace(3) %in.ptr, i64 %in.idx0, i32 0, i32 6
+  %idx3 = getelementptr i8, ptr addrspace(3) %const3, i64 %in.idx2
   ret void
 }
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll
index a91c8172177f..43dda1ae1517 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll
@@ -7,14 +7,14 @@ define protected amdgpu_kernel void @sink_addr(ptr %in.ptr, i64 %in.idx0, i64 %i
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr [8192 x i64], ptr [[IN_PTR]], i64 [[IN_IDX0]], i64 [[IN_IDX1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr [8192 x i64], ptr [[IN_PTR]], i64 [[IN_IDX0]], i64 0
-; CHECK-NEXT:    [[CONST11:%.*]] = getelementptr i8, ptr [[TMP0]], i64 2048
-; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr i64, ptr [[CONST11]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr i8, ptr [[TMP3]], i64 2048
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [8192 x i64], ptr [[IN_PTR]], i64 [[IN_IDX0]], i64 0
-; CHECK-NEXT:    [[CONST22:%.*]] = getelementptr i8, ptr [[TMP1]], i64 4096
-; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr i64, ptr [[CONST22]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[TMP1]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr i8, ptr [[TMP4]], i64 4096
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr [8192 x i64], ptr [[IN_PTR]], i64 [[IN_IDX0]], i64 0
-; CHECK-NEXT:    [[CONST33:%.*]] = getelementptr i8, ptr [[TMP2]], i64 6144
-; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr i64, ptr [[CONST33]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[TMP2]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr i8, ptr [[TMP7]], i64 6144
 ; CHECK-NEXT:    [[CMP0:%.*]] = icmp eq i64 [[IN_IDX0]], 0
 ; CHECK-NEXT:    br i1 [[CMP0]], label [[BB_1:%.*]], label [[END:%.*]]
 ; CHECK:       bb.1:
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/reorder-gep.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/reorder-gep.ll
index a15f11a634db..2e3b6ca3653f 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/reorder-gep.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/reorder-gep.ll
@@ -186,3 +186,66 @@ end:
   call void asm sideeffect "; use $0", "v"(ptr %idx3)
   ret void
 }
+
+
+define void @different_type_reorder2(ptr %in.ptr, i64 %in.idx0, i64 %in.idx1) {
+; CHECK-LABEL: define void @different_type_reorder2(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX0:%.*]], i64 [[IN_IDX1:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE:%.*]] = getelementptr i8, ptr [[IN_PTR]], i64 [[IN_IDX0]]
+; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[CONST1:%.*]] = getelementptr i64, ptr [[BASE]], i64 256
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr i8, ptr [[CONST1]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[CONST2:%.*]] = getelementptr i64, ptr [[BASE]], i64 512
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr i8, ptr [[CONST2]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[CONST3:%.*]] = getelementptr i64, ptr [[BASE]], i64 768
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr i8, ptr [[CONST3]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp eq i64 [[IN_IDX0]], 0
+; CHECK-NEXT:    br i1 [[CMP0]], label [[BB_1:%.*]], label [[END:%.*]]
+; CHECK:       bb.1:
+; CHECK-NEXT:    [[VAL0:%.*]] = load <8 x i64>, ptr [[IDX0]], align 16
+; CHECK-NEXT:    [[VAL1:%.*]] = load <8 x i64>, ptr [[IDX1]], align 16
+; CHECK-NEXT:    [[VAL2:%.*]] = load <8 x i64>, ptr [[IDX2]], align 16
+; CHECK-NEXT:    [[VAL3:%.*]] = load <8 x i64>, ptr [[IDX3]], align 16
+; CHECK-NEXT:    call void asm sideeffect "
+; CHECK-NEXT:    call void asm sideeffect "
+; CHECK-NEXT:    call void asm sideeffect "
+; CHECK-NEXT:    call void asm sideeffect "
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    call void asm sideeffect "
+; CHECK-NEXT:    call void asm sideeffect "
+; CHECK-NEXT:    call void asm sideeffect "
+; CHECK-NEXT:    call void asm sideeffect "
+; CHECK-NEXT:    ret void
+;
+entry:
+  %base = getelementptr i8, ptr %in.ptr, i64 %in.idx0
+  %idx0 = getelementptr i8, ptr %base, i64 %in.idx1
+  %const1 = getelementptr i64, ptr %base, i64 256
+  %idx1 = getelementptr i8, ptr %const1, i64 %in.idx1
+  %const2 = getelementptr i64, ptr %base, i64 512
+  %idx2 = getelementptr i8, ptr %const2, i64 %in.idx1
+  %const3 = getelementptr i64, ptr %base, i64 768
+  %idx3 = getelementptr i8, ptr %const3, i64 %in.idx1
+  %cmp0 = icmp eq i64 %in.idx0, 0
+  br i1 %cmp0, label %bb.1, label %end
+
+bb.1:
+  %val0 = load <8 x i64>, ptr %idx0, align 16
+  %val1 = load <8 x i64>, ptr %idx1, align 16
+  %val2 = load <8 x i64>, ptr %idx2, align 16
+  %val3 = load <8 x i64>, ptr %idx3, align 16
+  call void asm sideeffect "; use $0", "v"(<8 x i64> %val0)
+  call void asm sideeffect "; use $0", "v"(<8 x i64> %val1)
+  call void asm sideeffect "; use $0", "v"(<8 x i64> %val2)
+  call void asm sideeffect "; use $0", "v"(<8 x i64> %val3)
+  br label %end
+
+end:
+  call void asm sideeffect "; use $0", "v"(ptr %idx0)
+  call void asm sideeffect "; use $0", "v"(ptr %idx1)
+  call void asm sideeffect "; use $0", "v"(ptr %idx2)
+  call void asm sideeffect "; use $0", "v"(ptr %idx3)
+  ret void
+}
-- 
GitLab


From e4763ca83b90eed96be6fd83a9867e435f4b8ffe Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Thu, 9 May 2024 16:58:40 -0700
Subject: [PATCH 0361/1206] [ctx_profile] Pull `ContextNode` in a `.inc` file
 (#91669)

This pulls out `ContextNode` as we need to use it pretty much as-is to implement a writer. The writer will be implemented on the LLVM side because it takes a dependency on BitStreamWriter.

Since we can't reuse a header between compiler-rt and llvm, we use a header file which is copied on both sides, and test that the 2 copies are identical.

The changes adds the necessary other stuff for compiler-rt/ctx_profile testing.
---
 compiler-rt/lib/ctx_profile/CMakeLists.txt    |   1 +
 .../lib/ctx_profile/CtxInstrContextNode.h     | 116 +++++++++++++++++
 .../lib/ctx_profile/CtxInstrProfiling.cpp     |  58 ++++-----
 .../lib/ctx_profile/CtxInstrProfiling.h       | 118 ++----------------
 compiler-rt/test/ctx_profile/CMakeLists.txt   |  22 ++++
 .../TestCases/check-same-ctx-node.test        |   5 +
 compiler-rt/test/ctx_profile/lit.cfg.py       |  31 +++++
 .../test/ctx_profile/lit.site.cfg.py.in       |  14 +++
 compiler-rt/test/lit.common.cfg.py            |   6 +
 llvm/lib/ProfileData/CtxInstrContextNode.h    | 116 +++++++++++++++++
 10 files changed, 354 insertions(+), 133 deletions(-)
 create mode 100644 compiler-rt/lib/ctx_profile/CtxInstrContextNode.h
 create mode 100644 compiler-rt/test/ctx_profile/TestCases/check-same-ctx-node.test
 create mode 100644 compiler-rt/test/ctx_profile/lit.cfg.py
 create mode 100644 compiler-rt/test/ctx_profile/lit.site.cfg.py.in
 create mode 100644 llvm/lib/ProfileData/CtxInstrContextNode.h

diff --git a/compiler-rt/lib/ctx_profile/CMakeLists.txt b/compiler-rt/lib/ctx_profile/CMakeLists.txt
index 80e71acc38f8..1fa70594b28a 100644
--- a/compiler-rt/lib/ctx_profile/CMakeLists.txt
+++ b/compiler-rt/lib/ctx_profile/CMakeLists.txt
@@ -5,6 +5,7 @@ set(CTX_PROFILE_SOURCES
   )
 
 set(CTX_PROFILE_HEADERS
+  CtxInstrContextNode.h
   CtxInstrProfiling.h
   )
 
diff --git a/compiler-rt/lib/ctx_profile/CtxInstrContextNode.h b/compiler-rt/lib/ctx_profile/CtxInstrContextNode.h
new file mode 100644
index 000000000000..1627bdfffd08
--- /dev/null
+++ b/compiler-rt/lib/ctx_profile/CtxInstrContextNode.h
@@ -0,0 +1,116 @@
+//===--- CtxInstrContextNode.h - Contextual Profile Node --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//==============================================================================
+//
+// NOTE!
+// llvm/lib/ProfileData/CtxInstrContextNode.h and
+//   compiler-rt/lib/ctx_profile/CtxInstrContextNode.h
+// must be exact copies of eachother
+//
+// compiler-rt creates these objects as part of the instrumentation runtime for
+// contextual profiling. LLVM only consumes them to convert a contextual tree
+// to a bitstream.
+//
+//==============================================================================
+
+/// The contextual profile is a directed tree where each node has one parent. A
+/// node (ContextNode) corresponds to a function activation. The root of the
+/// tree is at a function that was marked as entrypoint to the compiler. A node
+/// stores counter values for edges and a vector of subcontexts. These are the
+/// contexts of callees. The index in the subcontext vector corresponds to the
+/// index of the callsite (as was instrumented via llvm.instrprof.callsite). At
+/// that index we find a linked list, potentially empty, of ContextNodes. Direct
+/// calls will have 0 or 1 values in the linked list, but indirect callsites may
+/// have more.
+///
+/// The ContextNode has a fixed sized header describing it - the GUID of the
+/// function, the size of the counter and callsite vectors. It is also an
+/// (intrusive) linked list for the purposes of the indirect call case above.
+///
+/// Allocation is expected to happen on an Arena. The allocation lays out inline
+/// the counter and subcontexts vectors. The class offers APIs to correctly
+/// reference the latter.
+///
+/// The layout is as follows:
+///
+/// [[declared fields][counters vector][vector of ptrs to subcontexts]]
+///
+/// See also documentation on the counters and subContexts members below.
+///
+/// The structure of the ContextNode is known to LLVM, because LLVM needs to:
+///   (1) increment counts, and
+///   (2) form a GEP for the position in the subcontext list of a callsite
+/// This means changes to LLVM contextual profile lowering and changes here
+/// must be coupled.
+/// Note: the header content isn't interesting to LLVM (other than its size)
+///
+/// Part of contextual collection is the notion of "scratch contexts". These are
+/// buffers that are "large enough" to allow for memory-safe acceses during
+/// counter increments - meaning the counter increment code in LLVM doesn't need
+/// to be concerned with memory safety. Their subcontexts never get populated,
+/// though. The runtime code here produces and recognizes them.
+
+#ifndef LLVM_LIB_PROFILEDATA_CTXINSTRCONTEXTNODE_H
+#define LLVM_LIB_PROFILEDATA_CTXINSTRCONTEXTNODE_H
+
+#include <stdint.h>
+#include <stdlib.h>
+
+namespace llvm {
+namespace ctx_profile {
+using GUID = uint64_t;
+
+class ContextNode final {
+  const GUID Guid;
+  ContextNode *const Next;
+  const uint32_t NrCounters;
+  const uint32_t NrCallsites;
+
+public:
+  ContextNode(GUID Guid, uint32_t NrCounters, uint32_t NrCallsites,
+              ContextNode *Next = nullptr)
+      : Guid(Guid), Next(Next), NrCounters(NrCounters),
+        NrCallsites(NrCallsites) {}
+
+  static inline size_t getAllocSize(uint32_t NrCounters, uint32_t NrCallsites) {
+    return sizeof(ContextNode) + sizeof(uint64_t) * NrCounters +
+           sizeof(ContextNode *) * NrCallsites;
+  }
+
+  // The counters vector starts right after the static header.
+  uint64_t *counters() {
+    ContextNode *addr_after = &(this[1]);
+    return reinterpret_cast<uint64_t *>(addr_after);
+  }
+
+  uint32_t counters_size() const { return NrCounters; }
+  uint32_t callsites_size() const { return NrCallsites; }
+
+  const uint64_t *counters() const {
+    return const_cast<ContextNode *>(this)->counters();
+  }
+
+  // The subcontexts vector starts right after the end of the counters vector.
+  ContextNode **subContexts() {
+    return reinterpret_cast<ContextNode **>(&(counters()[NrCounters]));
+  }
+
+  ContextNode *const *subContexts() const {
+    return const_cast<ContextNode *>(this)->subContexts();
+  }
+
+  GUID guid() const { return Guid; }
+  ContextNode *next() const { return Next; }
+
+  size_t size() const { return getAllocSize(NrCounters, NrCallsites); }
+
+  uint64_t entrycount() const { return counters()[0]; }
+};
+} // namespace ctx_profile
+} // namespace llvm
+#endif
\ No newline at end of file
diff --git a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
index 68bfe5c1ae61..c5d167bf996a 100644
--- a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
+++ b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
@@ -90,6 +90,26 @@ bool validate(const ContextRoot *Root) {
   }
   return true;
 }
+
+inline ContextNode *allocContextNode(char *Place, GUID Guid,
+                                     uint32_t NrCounters, uint32_t NrCallsites,
+                                     ContextNode *Next = nullptr) {
+  assert(reinterpret_cast<uint64_t>(Place) % ExpectedAlignment == 0);
+  return new (Place) ContextNode(Guid, NrCounters, NrCallsites, Next);
+}
+
+void resetContextNode(ContextNode &Node) {
+  // FIXME(mtrofin): this is std::memset, which we can probably use if we
+  // drop/reduce the dependency on sanitizer_common.
+  for (uint32_t I = 0; I < Node.counters_size(); ++I)
+    Node.counters()[I] = 0;
+  for (uint32_t I = 0; I < Node.callsites_size(); ++I)
+    for (auto *Next = Node.subContexts()[I]; Next; Next = Next->next())
+      resetContextNode(*Next);
+}
+
+void onContextEnter(ContextNode &Node) { ++Node.counters()[0]; }
+
 } // namespace
 
 // the scratch buffer - what we give when we can't produce a real context (the
@@ -134,27 +154,9 @@ void Arena::freeArenaList(Arena *&A) {
   A = nullptr;
 }
 
-inline ContextNode *ContextNode::alloc(char *Place, GUID Guid,
-                                       uint32_t NrCounters,
-                                       uint32_t NrCallsites,
-                                       ContextNode *Next) {
-  assert(reinterpret_cast<uint64_t>(Place) % ExpectedAlignment == 0);
-  return new (Place) ContextNode(Guid, NrCounters, NrCallsites, Next);
-}
-
-void ContextNode::reset() {
-  // FIXME(mtrofin): this is std::memset, which we can probably use if we
-  // drop/reduce the dependency on sanitizer_common.
-  for (uint32_t I = 0; I < NrCounters; ++I)
-    counters()[I] = 0;
-  for (uint32_t I = 0; I < NrCallsites; ++I)
-    for (auto *Next = subContexts()[I]; Next; Next = Next->Next)
-      Next->reset();
-}
-
 // If this is the first time we hit a callsite with this (Guid) particular
 // callee, we need to allocate.
-ContextNode *getCallsiteSlow(uint64_t Guid, ContextNode **InsertionPoint,
+ContextNode *getCallsiteSlow(GUID Guid, ContextNode **InsertionPoint,
                              uint32_t NrCounters, uint32_t NrCallsites) {
   auto AllocSize = ContextNode::getAllocSize(NrCounters, NrCallsites);
   auto *Mem = __llvm_ctx_profile_current_context_root->CurrentMem;
@@ -169,8 +171,8 @@ ContextNode *getCallsiteSlow(uint64_t Guid, ContextNode **InsertionPoint,
         Mem->allocateNewArena(getArenaAllocSize(AllocSize), Mem);
     AllocPlace = Mem->tryBumpAllocate(AllocSize);
   }
-  auto *Ret = ContextNode::alloc(AllocPlace, Guid, NrCounters, NrCallsites,
-                                 *InsertionPoint);
+  auto *Ret = allocContextNode(AllocPlace, Guid, NrCounters, NrCallsites,
+                               *InsertionPoint);
   *InsertionPoint = Ret;
   return Ret;
 }
@@ -224,7 +226,7 @@ ContextNode *__llvm_ctx_profile_get_context(void *Callee, GUID Guid,
                         "Context: %p, Asked: %lu %u %u, Got: %lu %u %u \n",
                         Ret, Guid, NrCallsites, NrCounters, Ret->guid(),
                         Ret->callsites_size(), Ret->counters_size());
-  Ret->onEntry();
+  onContextEnter(*Ret);
   return Ret;
 }
 
@@ -241,8 +243,8 @@ void setupContext(ContextRoot *Root, GUID Guid, uint32_t NrCounters,
   auto *M = Arena::allocateNewArena(getArenaAllocSize(Needed));
   Root->FirstMemBlock = M;
   Root->CurrentMem = M;
-  Root->FirstNode = ContextNode::alloc(M->tryBumpAllocate(Needed), Guid,
-                                       NrCounters, NrCallsites);
+  Root->FirstNode = allocContextNode(M->tryBumpAllocate(Needed), Guid,
+                                     NrCounters, NrCallsites);
   AllContextRoots.PushBack(Root);
 }
 
@@ -254,7 +256,7 @@ ContextNode *__llvm_ctx_profile_start_context(
   }
   if (Root->Taken.TryLock()) {
     __llvm_ctx_profile_current_context_root = Root;
-    Root->FirstNode->onEntry();
+    onContextEnter(*Root->FirstNode);
     return Root->FirstNode;
   }
   // If this thread couldn't take the lock, return scratch context.
@@ -281,13 +283,13 @@ void __llvm_ctx_profile_start_collection() {
     for (auto *Mem = Root->FirstMemBlock; Mem; Mem = Mem->next())
       ++NrMemUnits;
 
-    Root->FirstNode->reset();
+    resetContextNode(*Root->FirstNode);
   }
   __sanitizer::Printf("[ctxprof] Initial NrMemUnits: %zu \n", NrMemUnits);
 }
 
-bool __llvm_ctx_profile_fetch(
-    void *Data, bool (*Writer)(void *W, const __ctx_profile::ContextNode &)) {
+bool __llvm_ctx_profile_fetch(void *Data,
+                              bool (*Writer)(void *W, const ContextNode &)) {
   assert(Writer);
   __sanitizer::GenericScopedLock<__sanitizer::SpinMutex> Lock(
       &AllContextsMutex);
diff --git a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
index 8c4be5d8a23a..69ce796b71e3 100644
--- a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
+++ b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
@@ -9,11 +9,14 @@
 #ifndef CTX_PROFILE_CTXINSTRPROFILING_H_
 #define CTX_PROFILE_CTXINSTRPROFILING_H_
 
+#include "CtxInstrContextNode.h"
 #include "sanitizer_common/sanitizer_mutex.h"
 #include <sanitizer/common_interface_defs.h>
 
+using namespace llvm::ctx_profile;
+
 namespace __ctx_profile {
-using GUID = uint64_t;
+
 static constexpr size_t ExpectedAlignment = 8;
 // We really depend on this, see further below. We currently support x86_64.
 // When we want to support other archs, we need to trace the places Alignment is
@@ -62,99 +65,6 @@ private:
 // it to be thus aligned.
 static_assert(alignof(Arena) == ExpectedAlignment);
 
-/// The contextual profile is a directed tree where each node has one parent. A
-/// node (ContextNode) corresponds to a function activation. The root of the
-/// tree is at a function that was marked as entrypoint to the compiler. A node
-/// stores counter values for edges and a vector of subcontexts. These are the
-/// contexts of callees. The index in the subcontext vector corresponds to the
-/// index of the callsite (as was instrumented via llvm.instrprof.callsite). At
-/// that index we find a linked list, potentially empty, of ContextNodes. Direct
-/// calls will have 0 or 1 values in the linked list, but indirect callsites may
-/// have more.
-///
-/// The ContextNode has a fixed sized header describing it - the GUID of the
-/// function, the size of the counter and callsite vectors. It is also an
-/// (intrusive) linked list for the purposes of the indirect call case above.
-///
-/// Allocation is expected to happen on an Arena. The allocation lays out inline
-/// the counter and subcontexts vectors. The class offers APIs to correctly
-/// reference the latter.
-///
-/// The layout is as follows:
-///
-/// [[declared fields][counters vector][vector of ptrs to subcontexts]]
-///
-/// See also documentation on the counters and subContexts members below.
-///
-/// The structure of the ContextNode is known to LLVM, because LLVM needs to:
-///   (1) increment counts, and
-///   (2) form a GEP for the position in the subcontext list of a callsite
-/// This means changes to LLVM contextual profile lowering and changes here
-/// must be coupled.
-/// Note: the header content isn't interesting to LLVM (other than its size)
-///
-/// Part of contextual collection is the notion of "scratch contexts". These are
-/// buffers that are "large enough" to allow for memory-safe acceses during
-/// counter increments - meaning the counter increment code in LLVM doesn't need
-/// to be concerned with memory safety. Their subcontexts never get populated,
-/// though. The runtime code here produces and recognizes them.
-class ContextNode final {
-  const GUID Guid;
-  ContextNode *const Next;
-  const uint32_t NrCounters;
-  const uint32_t NrCallsites;
-
-public:
-  ContextNode(GUID Guid, uint32_t NrCounters, uint32_t NrCallsites,
-              ContextNode *Next = nullptr)
-      : Guid(Guid), Next(Next), NrCounters(NrCounters),
-        NrCallsites(NrCallsites) {}
-  static inline ContextNode *alloc(char *Place, GUID Guid, uint32_t NrCounters,
-                                   uint32_t NrCallsites,
-                                   ContextNode *Next = nullptr);
-
-  static inline size_t getAllocSize(uint32_t NrCounters, uint32_t NrCallsites) {
-    return sizeof(ContextNode) + sizeof(uint64_t) * NrCounters +
-           sizeof(ContextNode *) * NrCallsites;
-  }
-
-  // The counters vector starts right after the static header.
-  uint64_t *counters() {
-    ContextNode *addr_after = &(this[1]);
-    return reinterpret_cast<uint64_t *>(addr_after);
-  }
-
-  uint32_t counters_size() const { return NrCounters; }
-  uint32_t callsites_size() const { return NrCallsites; }
-
-  const uint64_t *counters() const {
-    return const_cast<ContextNode *>(this)->counters();
-  }
-
-  // The subcontexts vector starts right after the end of the counters vector.
-  ContextNode **subContexts() {
-    return reinterpret_cast<ContextNode **>(&(counters()[NrCounters]));
-  }
-
-  ContextNode *const *subContexts() const {
-    return const_cast<ContextNode *>(this)->subContexts();
-  }
-
-  GUID guid() const { return Guid; }
-  ContextNode *next() { return Next; }
-
-  size_t size() const { return getAllocSize(NrCounters, NrCallsites); }
-
-  void reset();
-
-  // since we go through the runtime to get a context back to LLVM, in the entry
-  // basic block, might as well handle incrementing the entry basic block
-  // counter.
-  void onEntry() { ++counters()[0]; }
-
-  uint64_t entrycount() const { return counters()[0]; }
-};
-
 // Verify maintenance to ContextNode doesn't change this invariant, which makes
 // sure the inlined vectors are appropriately aligned.
 static_assert(alignof(ContextNode) == ExpectedAlignment);
@@ -219,8 +129,7 @@ extern "C" {
 extern __thread void *volatile __llvm_ctx_profile_expected_callee[2];
 /// TLS where LLVM stores the pointer inside a caller's subcontexts vector that
 /// corresponds to the callsite being lowered.
-extern __thread __ctx_profile::ContextNode *
-    *volatile __llvm_ctx_profile_callsite[2];
+extern __thread ContextNode **volatile __llvm_ctx_profile_callsite[2];
 
 // __llvm_ctx_profile_current_context_root is exposed for unit testing,
 // othwerise it's only used internally by compiler-rt/ctx_profile.
@@ -229,10 +138,9 @@ extern __thread __ctx_profile::ContextRoot
 
 /// called by LLVM in the entry BB of a "entry point" function. The returned
 /// pointer may be "tainted" - its LSB set to 1 - to indicate it's scratch.
-__ctx_profile::ContextNode *
-__llvm_ctx_profile_start_context(__ctx_profile::ContextRoot *Root,
-                                 __ctx_profile::GUID Guid, uint32_t Counters,
-                                 uint32_t Callsites);
+ContextNode *__llvm_ctx_profile_start_context(__ctx_profile::ContextRoot *Root,
+                                              GUID Guid, uint32_t Counters,
+                                              uint32_t Callsites);
 
 /// paired with __llvm_ctx_profile_start_context, and called at the exit of the
 /// entry point function.
@@ -240,9 +148,9 @@ void __llvm_ctx_profile_release_context(__ctx_profile::ContextRoot *Root);
 
 /// called for any other function than entry points, in the entry BB of such
 /// function. Same consideration about LSB of returned value as .._start_context
-__ctx_profile::ContextNode *
-__llvm_ctx_profile_get_context(void *Callee, __ctx_profile::GUID Guid,
-                               uint32_t NrCounters, uint32_t NrCallsites);
+ContextNode *__llvm_ctx_profile_get_context(void *Callee, GUID Guid,
+                                            uint32_t NrCounters,
+                                            uint32_t NrCallsites);
 
 /// Prepares for collection. Currently this resets counter values but preserves
 /// internal context tree structure.
@@ -257,7 +165,7 @@ void __llvm_ctx_profile_free();
 /// The Writer's first parameter plays the role of closure for Writer, and is
 /// what the caller of __llvm_ctx_profile_fetch passes as the Data parameter.
 /// The second parameter is the root of a context tree.
-bool __llvm_ctx_profile_fetch(
-    void *Data, bool (*Writer)(void *, const __ctx_profile::ContextNode &));
+bool __llvm_ctx_profile_fetch(void *Data,
+                              bool (*Writer)(void *, const ContextNode &));
 }
 #endif // CTX_PROFILE_CTXINSTRPROFILING_H_
diff --git a/compiler-rt/test/ctx_profile/CMakeLists.txt b/compiler-rt/test/ctx_profile/CMakeLists.txt
index 23c6fb16ed1f..371f1a2dcbb0 100644
--- a/compiler-rt/test/ctx_profile/CMakeLists.txt
+++ b/compiler-rt/test/ctx_profile/CMakeLists.txt
@@ -2,6 +2,28 @@ set(CTX_PROFILE_LIT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 set(CTX_PROFILE_TESTSUITES)
 
+macro(get_bits_for_arch arch bits)
+  if (${arch} MATCHES "x86_64")
+    set(${bits} 64)
+  else()
+    message(FATAL_ERROR "Unexpected target architecture: ${arch}")
+  endif()
+endmacro()
+
+set(CTX_PROFILE_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS} ctx_profile)
+
+foreach(arch ${CTX_PROFILE_SUPPORTED_ARCH})
+  set(CTX_PROFILE_TEST_TARGET_ARCH ${arch})
+  string(TOLOWER "-${arch}-${OS_NAME}" CTX_PROFILE_TEST_CONFIG_SUFFIX)
+  string(TOUPPER ${arch} ARCH_UPPER_CASE)
+  set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config)
+  configure_lit_site_cfg(
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
+    )
+  list(APPEND CTX_PROFILE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
+endforeach()
+
 # Add unit tests.
 if(COMPILER_RT_INCLUDE_TESTS)
   foreach(arch ${CTX_PROFILE_SUPPORTED_ARCH})
diff --git a/compiler-rt/test/ctx_profile/TestCases/check-same-ctx-node.test b/compiler-rt/test/ctx_profile/TestCases/check-same-ctx-node.test
new file mode 100644
index 000000000000..37d36dbb9379
--- /dev/null
+++ b/compiler-rt/test/ctx_profile/TestCases/check-same-ctx-node.test
@@ -0,0 +1,5 @@
+;
+; NOTE: if this test fails, please make sure the two files are identical copies
+; of eachother.
+;
+; RUN: diff %crt_src/lib/ctx_profile/CtxInstrContextNode.h %llvm_src/lib/ProfileData/CtxInstrContextNode.h
diff --git a/compiler-rt/test/ctx_profile/lit.cfg.py b/compiler-rt/test/ctx_profile/lit.cfg.py
new file mode 100644
index 000000000000..a56dabb8ebeb
--- /dev/null
+++ b/compiler-rt/test/ctx_profile/lit.cfg.py
@@ -0,0 +1,31 @@
+# -*- Python -*-
+
+import os
+import platform
+import re
+
+import lit.formats
+
+# Only run the tests on supported OSs.
+if config.host_os not in ["Linux"]:
+    config.unsupported = True
+
+
+def get_required_attr(config, attr_name):
+    attr_value = getattr(config, attr_name, None)
+    if attr_value == None:
+        lit_config.fatal(
+            "No attribute %r in test configuration! You may need to run "
+            "tests from your build directory or add this attribute "
+            "to lit.site.cfg.py " % attr_name
+        )
+    return attr_value
+
+
+# Setup config name.
+config.name = "CtxProfile" + config.name_suffix
+
+# Setup source root.
+config.test_source_root = os.path.dirname(__file__)
+# Default test suffixes.
+config.suffixes = [".c", ".cpp", ".test"]
diff --git a/compiler-rt/test/ctx_profile/lit.site.cfg.py.in b/compiler-rt/test/ctx_profile/lit.site.cfg.py.in
new file mode 100644
index 000000000000..e8df42d097d8
--- /dev/null
+++ b/compiler-rt/test/ctx_profile/lit.site.cfg.py.in
@@ -0,0 +1,14 @@
+@LIT_SITE_CFG_IN_HEADER@
+
+# Tool-specific config options.
+config.name_suffix = "@CTX_PROFILE_TEST_CONFIG_SUFFIX@"
+config.target_cflags = "@CTX_PROFILE_TEST_TARGET_CFLAGS@"
+config.clang = "@CTX_PROFILE_TEST_TARGET_CC@"
+config.bits = "@CTX_PROFILE_TEST_BITS@"
+config.target_arch = "@CTX_PROFILE_TEST_TARGET_ARCH@"
+
+# Load common config for all compiler-rt lit tests.
+lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
+
+# Load tool-specific config that would do the real work.
+lit_config.load_config(config, "@CTX_PROFILE_LIT_SOURCE_DIR@/lit.cfg.py")
diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index 28f126a11b16..fae1d1686e56 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -987,3 +987,9 @@ if config.compiler_id == "GNU":
     gcc_dir = os.path.dirname(config.clang)
     libasan_dir = os.path.join(gcc_dir, "..", "lib" + config.bits)
     push_dynamic_library_lookup_path(config, libasan_dir)
+
+
+# Help tests that make sure certain files are in-sync between compiler-rt and
+# llvm.
+config.substitutions.append(("%crt_src", config.compiler_rt_src_root))
+config.substitutions.append(("%llvm_src", config.llvm_src_root))
diff --git a/llvm/lib/ProfileData/CtxInstrContextNode.h b/llvm/lib/ProfileData/CtxInstrContextNode.h
new file mode 100644
index 000000000000..1627bdfffd08
--- /dev/null
+++ b/llvm/lib/ProfileData/CtxInstrContextNode.h
@@ -0,0 +1,116 @@
+//===--- CtxInstrContextNode.h - Contextual Profile Node --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//==============================================================================
+//
+// NOTE!
+// llvm/lib/ProfileData/CtxInstrContextNode.h and
+//   compiler-rt/lib/ctx_profile/CtxInstrContextNode.h
+// must be exact copies of eachother
+//
+// compiler-rt creates these objects as part of the instrumentation runtime for
+// contextual profiling. LLVM only consumes them to convert a contextual tree
+// to a bitstream.
+//
+//==============================================================================
+
+/// The contextual profile is a directed tree where each node has one parent. A
+/// node (ContextNode) corresponds to a function activation. The root of the
+/// tree is at a function that was marked as entrypoint to the compiler. A node
+/// stores counter values for edges and a vector of subcontexts. These are the
+/// contexts of callees. The index in the subcontext vector corresponds to the
+/// index of the callsite (as was instrumented via llvm.instrprof.callsite). At
+/// that index we find a linked list, potentially empty, of ContextNodes. Direct
+/// calls will have 0 or 1 values in the linked list, but indirect callsites may
+/// have more.
+///
+/// The ContextNode has a fixed sized header describing it - the GUID of the
+/// function, the size of the counter and callsite vectors. It is also an
+/// (intrusive) linked list for the purposes of the indirect call case above.
+///
+/// Allocation is expected to happen on an Arena. The allocation lays out inline
+/// the counter and subcontexts vectors. The class offers APIs to correctly
+/// reference the latter.
+///
+/// The layout is as follows:
+///
+/// [[declared fields][counters vector][vector of ptrs to subcontexts]]
+///
+/// See also documentation on the counters and subContexts members below.
+///
+/// The structure of the ContextNode is known to LLVM, because LLVM needs to:
+///   (1) increment counts, and
+///   (2) form a GEP for the position in the subcontext list of a callsite
+/// This means changes to LLVM contextual profile lowering and changes here
+/// must be coupled.
+/// Note: the header content isn't interesting to LLVM (other than its size)
+///
+/// Part of contextual collection is the notion of "scratch contexts". These are
+/// buffers that are "large enough" to allow for memory-safe acceses during
+/// counter increments - meaning the counter increment code in LLVM doesn't need
+/// to be concerned with memory safety. Their subcontexts never get populated,
+/// though. The runtime code here produces and recognizes them.
+
+#ifndef LLVM_LIB_PROFILEDATA_CTXINSTRCONTEXTNODE_H
+#define LLVM_LIB_PROFILEDATA_CTXINSTRCONTEXTNODE_H
+
+#include <stdint.h>
+#include <stdlib.h>
+
+namespace llvm {
+namespace ctx_profile {
+using GUID = uint64_t;
+
+class ContextNode final {
+  const GUID Guid;
+  ContextNode *const Next;
+  const uint32_t NrCounters;
+  const uint32_t NrCallsites;
+
+public:
+  ContextNode(GUID Guid, uint32_t NrCounters, uint32_t NrCallsites,
+              ContextNode *Next = nullptr)
+      : Guid(Guid), Next(Next), NrCounters(NrCounters),
+        NrCallsites(NrCallsites) {}
+
+  static inline size_t getAllocSize(uint32_t NrCounters, uint32_t NrCallsites) {
+    return sizeof(ContextNode) + sizeof(uint64_t) * NrCounters +
+           sizeof(ContextNode *) * NrCallsites;
+  }
+
+  // The counters vector starts right after the static header.
+  uint64_t *counters() {
+    ContextNode *addr_after = &(this[1]);
+    return reinterpret_cast<uint64_t *>(addr_after);
+  }
+
+  uint32_t counters_size() const { return NrCounters; }
+  uint32_t callsites_size() const { return NrCallsites; }
+
+  const uint64_t *counters() const {
+    return const_cast<ContextNode *>(this)->counters();
+  }
+
+  // The subcontexts vector starts right after the end of the counters vector.
+  ContextNode **subContexts() {
+    return reinterpret_cast<ContextNode **>(&(counters()[NrCounters]));
+  }
+
+  ContextNode *const *subContexts() const {
+    return const_cast<ContextNode *>(this)->subContexts();
+  }
+
+  GUID guid() const { return Guid; }
+  ContextNode *next() const { return Next; }
+
+  size_t size() const { return getAllocSize(NrCounters, NrCallsites); }
+
+  uint64_t entrycount() const { return counters()[0]; }
+};
+} // namespace ctx_profile
+} // namespace llvm
+#endif
\ No newline at end of file
-- 
GitLab


From 0fd017ce43875283ecce55f18f721f47ba37a920 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Thu, 9 May 2024 17:30:46 -0700
Subject: [PATCH 0362/1206] [nfc][ctx_profile] Move `CtxInstrContextNode.h` in
 `include`

Follow-up from PR #91669.
---
 compiler-rt/lib/ctx_profile/CtxInstrContextNode.h             | 4 ++--
 .../test/ctx_profile/TestCases/check-same-ctx-node.test       | 2 +-
 llvm/{lib => include/llvm}/ProfileData/CtxInstrContextNode.h  | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)
 rename llvm/{lib => include/llvm}/ProfileData/CtxInstrContextNode.h (97%)

diff --git a/compiler-rt/lib/ctx_profile/CtxInstrContextNode.h b/compiler-rt/lib/ctx_profile/CtxInstrContextNode.h
index 1627bdfffd08..a916f197aa14 100644
--- a/compiler-rt/lib/ctx_profile/CtxInstrContextNode.h
+++ b/compiler-rt/lib/ctx_profile/CtxInstrContextNode.h
@@ -55,8 +55,8 @@
 /// to be concerned with memory safety. Their subcontexts never get populated,
 /// though. The runtime code here produces and recognizes them.
 
-#ifndef LLVM_LIB_PROFILEDATA_CTXINSTRCONTEXTNODE_H
-#define LLVM_LIB_PROFILEDATA_CTXINSTRCONTEXTNODE_H
+#ifndef LLVM_PROFILEDATA_CTXINSTRCONTEXTNODE_H
+#define LLVM_PROFILEDATA_CTXINSTRCONTEXTNODE_H
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/ctx_profile/TestCases/check-same-ctx-node.test b/compiler-rt/test/ctx_profile/TestCases/check-same-ctx-node.test
index 37d36dbb9379..4ad7b23d458f 100644
--- a/compiler-rt/test/ctx_profile/TestCases/check-same-ctx-node.test
+++ b/compiler-rt/test/ctx_profile/TestCases/check-same-ctx-node.test
@@ -2,4 +2,4 @@
 ; NOTE: if this test fails, please make sure the two files are identical copies
 ; of eachother.
 ;
-; RUN: diff %crt_src/lib/ctx_profile/CtxInstrContextNode.h %llvm_src/lib/ProfileData/CtxInstrContextNode.h
+; RUN: diff %crt_src/lib/ctx_profile/CtxInstrContextNode.h %llvm_src/include/llvm/ProfileData/CtxInstrContextNode.h
diff --git a/llvm/lib/ProfileData/CtxInstrContextNode.h b/llvm/include/llvm/ProfileData/CtxInstrContextNode.h
similarity index 97%
rename from llvm/lib/ProfileData/CtxInstrContextNode.h
rename to llvm/include/llvm/ProfileData/CtxInstrContextNode.h
index 1627bdfffd08..a916f197aa14 100644
--- a/llvm/lib/ProfileData/CtxInstrContextNode.h
+++ b/llvm/include/llvm/ProfileData/CtxInstrContextNode.h
@@ -55,8 +55,8 @@
 /// to be concerned with memory safety. Their subcontexts never get populated,
 /// though. The runtime code here produces and recognizes them.
 
-#ifndef LLVM_LIB_PROFILEDATA_CTXINSTRCONTEXTNODE_H
-#define LLVM_LIB_PROFILEDATA_CTXINSTRCONTEXTNODE_H
+#ifndef LLVM_PROFILEDATA_CTXINSTRCONTEXTNODE_H
+#define LLVM_PROFILEDATA_CTXINSTRCONTEXTNODE_H
 
 #include <stdint.h>
 #include <stdlib.h>
-- 
GitLab


From db78ee0cb82669302a5e0f18a15fd53346a73823 Mon Sep 17 00:00:00 2001
From: alx32 <103613512+alx32@users.noreply.github.com>
Date: Thu, 9 May 2024 17:56:46 -0700
Subject: [PATCH 0363/1206] [lld-macho] Fix address sanitizer for category
 merging (#91680)

FIxing the address sanitizer issue reported in
https://github.com/llvm/llvm-project/pull/91548 .
The problem comes from the assignment `auto bodyData = newSectionData`
which defaults to `SmallVector<uint8_t> data = newSectionData` - which
actually creates a copy of the data, placed on the stack.
By explicitly using `ArrayRef` instead, we make sure that the original
copy is used.
We also change the assignment in `ObjcCategoryMerger::newStringData`
from `auto` to `SmallVector<uint8_t> &` to make it explicit.
---
 lld/MachO/ObjC.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp
index 96ec646095be..9d1612beae87 100644
--- a/lld/MachO/ObjC.cpp
+++ b/lld/MachO/ObjC.cpp
@@ -1148,7 +1148,7 @@ void ObjcCategoryMerger::generateCatListForNonErasedCategories(
       assert(nonErasedCatBody && "Failed to relocate non-deleted category");
 
       // Allocate data for the new __objc_catlist slot
-      auto bodyData = newSectionData(target->wordSize);
+      llvm::ArrayRef<uint8_t> bodyData = newSectionData(target->wordSize);
 
       // We mark the __objc_catlist slot as belonging to the same file as the
       // category
@@ -1279,7 +1279,7 @@ void ObjcCategoryMerger::doCleanup() { generatedSectionData.clear(); }
 StringRef ObjcCategoryMerger::newStringData(const char *str) {
   uint32_t len = strlen(str);
   uint32_t bufSize = len + 1;
-  auto &data = newSectionData(bufSize);
+  SmallVector<uint8_t> &data = newSectionData(bufSize);
   char *strData = reinterpret_cast<char *>(data.data());
   // Copy the string chars and null-terminator
   memcpy(strData, str, bufSize);
-- 
GitLab


From d24eaef92525d03b4b64c7b4acd07197bdfb57cc Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Fri, 10 May 2024 10:01:23 +0800
Subject: [PATCH 0364/1206] [RISCV] Sink vector select splat operands (#91554)

vmerge.vxm allows us to splat the true operand of a select, so sink it
where possible to reduce vector register pressure.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp     |  1 +
 .../CodeGen/RISCV/rvv/sink-splat-operands.ll    | 17 ++++++++---------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 00a97d15db3e..e0937989a6a4 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -2019,6 +2019,7 @@ bool RISCVTargetLowering::canSplatOperand(unsigned Opcode, int Operand) const {
   case Instruction::SDiv:
   case Instruction::URem:
   case Instruction::SRem:
+  case Instruction::Select:
     return Operand == 1;
   default:
     return false;
diff --git a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
index 6e902e79896b..618672344fe7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
@@ -5427,19 +5427,18 @@ for.cond.cleanup:                                 ; preds = %vector.body
 define void @sink_splat_select(ptr nocapture %a, i32 signext %x) {
 ; CHECK-LABEL: sink_splat_select:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a2, 1
+; CHECK-NEXT:    add a2, a0, a2
+; CHECK-NEXT:    li a3, 42
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    lui a1, 1
-; CHECK-NEXT:    add a1, a0, a1
-; CHECK-NEXT:    li a2, 42
 ; CHECK-NEXT:  .LBB117_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vle32.v v9, (a0)
-; CHECK-NEXT:    vmseq.vx v0, v9, a2
-; CHECK-NEXT:    vmerge.vvm v9, v9, v8, v0
-; CHECK-NEXT:    vse32.v v9, (a0)
+; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    vmseq.vx v0, v8, a3
+; CHECK-NEXT:    vmerge.vxm v8, v8, a1, v0
+; CHECK-NEXT:    vse32.v v8, (a0)
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    bne a0, a1, .LBB117_1
+; CHECK-NEXT:    bne a0, a2, .LBB117_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
-- 
GitLab


From 427beff2ad274f38f9de682f48f550cdcf5fc505 Mon Sep 17 00:00:00 2001
From: Kareem Ergawy <kareem.ergawy@amd.com>
Date: Fri, 10 May 2024 04:20:43 +0200
Subject: [PATCH 0365/1206] [OpenMP][MLIR] Add `private` clause to `omp.target`
 (#91202)

---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  6 +-
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 58 +++++++++++++++++--
 mlir/test/Dialect/OpenMP/invalid.mlir         |  2 +-
 mlir/test/Dialect/OpenMP/ops.mlir             | 40 ++++++++++++-
 4 files changed, 98 insertions(+), 8 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index a40676d071e6..a641588eaa8d 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1787,7 +1787,10 @@ def TargetOp : OpenMP_Op<"target", [IsolatedFromAbove, MapClauseOwningOpInterfac
                        UnitAttr:$nowait,
                        Variadic<OpenMP_PointerLikeType>:$is_device_ptr,
                        Variadic<OpenMP_PointerLikeType>:$has_device_addr,
-                       Variadic<AnyType>:$map_operands);
+                       Variadic<AnyType>:$map_operands,
+                       Variadic<AnyType>:$private_vars,
+                       OptionalAttr<SymbolRefArrayAttr>:$privatizers);
+
   let regions = (region AnyRegion:$region);
 
   let builders = [
@@ -1802,6 +1805,7 @@ def TargetOp : OpenMP_Op<"target", [IsolatedFromAbove, MapClauseOwningOpInterfac
     | `is_device_ptr` `(` $is_device_ptr `:` type($is_device_ptr) `)`
     | `has_device_addr` `(` $has_device_addr `:` type($has_device_addr) `)`
     | `map_entries` `(` custom<MapEntries>($map_operands, type($map_operands)) `)`
+    | `private` `(` custom<PrivateList>($private_vars, type($private_vars), $privatizers) `)`
     | `depend` `(` custom<DependVarList>($depend_vars, type($depend_vars), $depends) `)`
     ) $region attr-dict
   }];
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 0799090cdea9..e016a326ecc7 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -470,13 +470,17 @@ static void printClauseWithRegionArgs(OpAsmPrinter &p, Operation *op,
                                       ValueRange argsSubrange,
                                       StringRef clauseName, ValueRange operands,
                                       TypeRange types, ArrayAttr symbols) {
-  p << clauseName << "(";
+  if (!clauseName.empty())
+    p << clauseName << "(";
+
   llvm::interleaveComma(
       llvm::zip_equal(symbols, operands, argsSubrange, types), p, [&p](auto t) {
         auto [sym, op, arg, type] = t;
         p << sym << " " << op << " -> " << arg << " : " << type;
       });
-  p << ") ";
+
+  if (!clauseName.empty())
+    p << ") ";
 }
 
 static ParseResult parseParallelRegion(
@@ -1048,6 +1052,49 @@ static void printMapEntries(OpAsmPrinter &p, Operation *op,
   }
 }
 
+static ParseResult parsePrivateList(
+    OpAsmParser &parser,
+    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateOperands,
+    SmallVectorImpl<Type> &privateOperandTypes, ArrayAttr &privatizerSymbols) {
+  SmallVector<SymbolRefAttr> privateSymRefs;
+  SmallVector<OpAsmParser::Argument> regionPrivateArgs;
+
+  if (failed(parser.parseCommaSeparatedList([&]() {
+        if (parser.parseAttribute(privateSymRefs.emplace_back()) ||
+            parser.parseOperand(privateOperands.emplace_back()) ||
+            parser.parseArrow() ||
+            parser.parseArgument(regionPrivateArgs.emplace_back()) ||
+            parser.parseColonType(privateOperandTypes.emplace_back()))
+          return failure();
+        return success();
+      })))
+    return failure();
+
+  SmallVector<Attribute> privateSymAttrs(privateSymRefs.begin(),
+                                         privateSymRefs.end());
+  privatizerSymbols = ArrayAttr::get(parser.getContext(), privateSymAttrs);
+
+  return success();
+}
+
+static void printPrivateList(OpAsmPrinter &p, Operation *op,
+                             ValueRange privateVarOperands,
+                             TypeRange privateVarTypes,
+                             ArrayAttr privatizerSymbols) {
+  // TODO: Remove target-specific logic from this function.
+  auto targetOp = mlir::dyn_cast<mlir::omp::TargetOp>(op);
+  assert(targetOp);
+
+  auto &region = op->getRegion(0);
+  auto *argsBegin = region.front().getArguments().begin();
+  MutableArrayRef argsSubrange(argsBegin + targetOp.getMapOperands().size(),
+                               argsBegin + targetOp.getMapOperands().size() +
+                                   privateVarTypes.size());
+  printClauseWithRegionArgs(
+      p, op, argsSubrange, /*clauseName=*/llvm::StringRef{}, privateVarOperands,
+      privateVarTypes, privatizerSymbols);
+}
+
 static void printCaptureType(OpAsmPrinter &p, Operation *op,
                              VariableCaptureKindAttr mapCaptureType) {
   std::string typeCapStr;
@@ -1256,13 +1303,14 @@ void TargetOp::build(OpBuilder &builder, OperationState &state,
                      const TargetClauseOps &clauses) {
   MLIRContext *ctx = builder.getContext();
   // TODO Store clauses in op: allocateVars, allocatorVars, inReductionVars,
-  // inReductionDeclSymbols, privateVars, privatizers, reductionVars,
-  // reductionByRefAttr, reductionDeclSymbols.
+  // inReductionDeclSymbols, reductionVars, reductionByRefAttr,
+  // reductionDeclSymbols.
   TargetOp::build(
       builder, state, clauses.ifVar, clauses.deviceVar, clauses.threadLimitVar,
       makeArrayAttr(ctx, clauses.dependTypeAttrs), clauses.dependVars,
       clauses.nowaitAttr, clauses.isDevicePtrVars, clauses.hasDeviceAddrVars,
-      clauses.mapVars);
+      clauses.mapVars, clauses.privateVars,
+      makeArrayAttr(ctx, clauses.privatizers));
 }
 
 LogicalResult TargetOp::verify() {
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 511e7d396c68..138c2c9d418d 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -2087,7 +2087,7 @@ func.func @omp_target_depend(%data_var: memref<i32>) {
   // expected-error @below {{op expected as many depend values as depend variables}}
     "omp.target"(%data_var) ({
       "omp.terminator"() : () -> ()
-    }) {depends = [], operandSegmentSizes = array<i32: 0, 0, 0, 1, 0, 0, 0>} : (memref<i32>) -> ()
+    }) {depends = [], operandSegmentSizes = array<i32: 0, 0, 0, 1, 0, 0, 0, 0>} : (memref<i32>) -> ()
    "func.return"() : () -> ()
 }
 
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 60fc10f9d64b..828c9d2c3b84 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -737,7 +737,7 @@ func.func @omp_target(%if_cond : i1, %device : si32,  %num_threads : i32, %devic
     "omp.target"(%if_cond, %device, %num_threads) ({
        // CHECK: omp.terminator
        omp.terminator
-    }) {nowait, operandSegmentSizes = array<i32: 1,1,1,0,0,0,0>} : ( i1, si32, i32 ) -> ()
+    }) {nowait, operandSegmentSizes = array<i32: 1,1,1,0,0,0,0,0>} : ( i1, si32, i32 ) -> ()
 
     // Test with optional map clause.
     // CHECK: %[[MAP_A:.*]] = omp.map.info var_ptr(%[[VAL_1:.*]] : memref<?xi32>, tensor<?xi32>)   map_clauses(tofrom) capture(ByRef) -> memref<?xi32> {name = ""}
@@ -2550,3 +2550,41 @@ func.func @parallel_op_reduction_and_private(%priv_var: !llvm.ptr, %priv_var2: !
   }
   return
 }
+
+// CHECK-LABEL: omp_target_private
+func.func @omp_target_private(%map1: memref<?xi32>, %map2: memref<?xi32>, %priv_var: !llvm.ptr) -> () {
+  %mapv1 = omp.map.info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>) map_clauses(tofrom) capture(ByRef) -> memref<?xi32> {name = ""}
+  %mapv2 = omp.map.info var_ptr(%map2 : memref<?xi32>, tensor<?xi32>) map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> memref<?xi32> {name = ""}
+
+  // CHECK: omp.target
+  // CHECK-SAME: private(
+  // CHECK-SAME:   @x.privatizer %{{[^[:space:]]+}} -> %[[PRIV_ARG:[^[:space:]]+]]
+  // CHECK-SAME:   : !llvm.ptr
+  // CHECK-SAME: )
+  omp.target private(@x.privatizer %priv_var -> %priv_arg : !llvm.ptr) {
+  // CHECK: ^bb0(%[[PRIV_ARG]]: !llvm.ptr):
+  ^bb0(%priv_arg: !llvm.ptr):
+    omp.terminator
+  }
+
+  // CHECK: omp.target
+
+  // CHECK-SAME: map_entries(
+  // CHECK-SAME:   %{{[^[:space:]]+}} -> %[[MAP1_ARG:[^[:space:]]+]],
+  // CHECK-SAME:   %{{[^[:space:]]+}} -> %[[MAP2_ARG:[^[:space:]]+]]
+  // CHECK-SAME:   : memref<?xi32>, memref<?xi32>
+  // CHECK-SAME: )
+
+  // CHECK-SAME: private(
+  // CHECK-SAME:   @x.privatizer %{{[^[:space:]]+}} -> %[[PRIV_ARG:[^[:space:]]+]]
+  // CHECK-SAME:   : !llvm.ptr
+  // CHECK-SAME: )
+  omp.target map_entries(%mapv1 -> %arg0, %mapv2 -> %arg1 : memref<?xi32>, memref<?xi32>) private(@x.privatizer %priv_var -> %priv_arg : !llvm.ptr) {
+  // CHECK: ^bb0(%[[MAP1_ARG]]: memref<?xi32>, %[[MAP2_ARG]]: memref<?xi32>
+  // CHECK-SAME: , %[[PRIV_ARG]]: !llvm.ptr):
+  ^bb0(%arg0: memref<?xi32>, %arg1: memref<?xi32>, %priv_arg: !llvm.ptr):
+    omp.terminator
+  }
+
+  return
+}
-- 
GitLab


From e069bb7fd85b69abded27b44f33bf40a522ab9b6 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 9 May 2024 20:12:29 -0700
Subject: [PATCH 0366/1206] [RISCV] Use map::count instead of hasExtension in
 RISCVISAInfo::updateCombination. NFC

hasExtension check isSupportedExtension before the map lookup. All
of the extensions we check for in updateCombination should be valid
extension names so we can bypass that to save some time.
---
 llvm/lib/TargetParser/RISCVISAInfo.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index c553e330a878..975cb5897b76 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -890,7 +890,7 @@ void RISCVISAInfo::updateCombination() {
   do {
     MadeChange = false;
     for (StringRef CombineExt : CombineIntoExts) {
-      if (hasExtension(CombineExt))
+      if (Exts.count(CombineExt.str()))
         continue;
 
       // Look up the extension in the ImpliesExt table to find everything it
@@ -899,7 +899,7 @@ void RISCVISAInfo::updateCombination() {
                                     std::end(ImpliedExts), CombineExt);
       bool HasAllRequiredFeatures = std::all_of(
           Range.first, Range.second, [&](const ImpliedExtsEntry &Implied) {
-            return hasExtension(Implied.ImpliedExt);
+            return Exts.count(Implied.ImpliedExt);
           });
       if (HasAllRequiredFeatures) {
         auto Version = findDefaultVersion(CombineExt);
-- 
GitLab


From 0d31ac8893c382117abfb238f52c9736b2a38a8d Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar@redhat.com>
Date: Thu, 9 May 2024 20:29:25 -0700
Subject: [PATCH 0367/1206] workflows: Remove top-level permissions from
 release-tasks.yml (#91088)

This is the recommend best practice and we also don't need write access
for all jobs.
---
 .github/workflows/release-tasks.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release-tasks.yml b/.github/workflows/release-tasks.yml
index 53da8662b020..29049ff01428 100644
--- a/.github/workflows/release-tasks.yml
+++ b/.github/workflows/release-tasks.yml
@@ -1,7 +1,7 @@
 name: Release Task
 
 permissions:
-  contents: write
+  contents: read
 
 on:
   push:
@@ -27,6 +27,8 @@ jobs:
   release-create:
     name: Create a New Release
     runs-on: ubuntu-latest
+    permissions:
+      contents: write # For creating the release.
     needs: validate-tag
 
     steps:
@@ -55,6 +57,8 @@ jobs:
 
   release-doxygen:
     name: Build and Upload Release Doxygen
+    permissions:
+      contents: write
     needs:
       - validate-tag
       - release-create
@@ -72,6 +76,8 @@ jobs:
 
   release-binaries:
     name: Build Release Binaries
+    permissions:
+      contents: write
     needs:
       - validate-tag
       - release-create
-- 
GitLab


From 720dfd94dfe83d8ce57ad3fe317563d7e1a9c602 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar@redhat.com>
Date: Thu, 9 May 2024 20:30:29 -0700
Subject: [PATCH 0368/1206] workflows: Fix missing GITHUB_TOKEN in
 release-doxygen.yml upload step (#91091)

We were accidentally setting the GITHUB_TOKEN environment variable in
the previous step.
---
 .github/workflows/release-doxygen.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release-doxygen.yml b/.github/workflows/release-doxygen.yml
index 5e322849a1d0..12c14bea52f6 100644
--- a/.github/workflows/release-doxygen.yml
+++ b/.github/workflows/release-doxygen.yml
@@ -56,12 +56,12 @@ jobs:
           pip3 install --user -r ./llvm/docs/requirements.txt
 
       - name: Build Doxygen
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
         run: |
           ./llvm/utils/release/build-docs.sh -release "${{ inputs.release-version }}" -no-sphinx
 
       - name: Upload Doxygen
         if: env.upload
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         run: |
           ./llvm/utils/release/github-upload-release.py --token "$GITHUB_TOKEN" --release "${{ inputs.release-version }}" --user "${{ github.actor }}" upload --files ./*doxygen*.tar.xz
-- 
GitLab


From 181e2e8fb9efe6e8f3f7fc094516f125659b687c Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Thu, 9 May 2024 20:47:10 -0700
Subject: [PATCH 0369/1206] [nfc][memprof] Add missing license to `MemProfTest`
 (#91695)

---
 llvm/unittests/ProfileData/MemProfTest.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp
index 40335d191ba7..8b97866e403f 100644
--- a/llvm/unittests/ProfileData/MemProfTest.cpp
+++ b/llvm/unittests/ProfileData/MemProfTest.cpp
@@ -1,3 +1,11 @@
+//===- unittests/Support/MemProfTest.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
 #include "llvm/ProfileData/MemProf.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/MapVector.h"
-- 
GitLab


From 21be8182239a9c87a50071d122d5532037fd8305 Mon Sep 17 00:00:00 2001
From: Ryosuke Niwa <rniwa@webkit.org>
Date: Thu, 9 May 2024 21:32:28 -0700
Subject: [PATCH 0370/1206] [analyzer] Support determining origins in a
 conditional operator in WebKit checkers. (#91143)

This PR adds the support for determining the origin of a pointer in a
conditional operator.

Because such an expression can have two distinct origins each of which
needs to be visited, this PR refactors tryToFindPtrOrigin to take a
callback instead of returning a pair.

The callback is called for the second operand and the third operand of
the conditioanl operator (i.e. E2 and E3 in E1 ? E2 : E3).

Also treat nullptr and integer literal as safe pointer origins in the
local variable checker.
---
 .../Checkers/WebKit/ASTUtils.cpp              | 23 ++++--
 .../StaticAnalyzer/Checkers/WebKit/ASTUtils.h | 11 ++-
 .../WebKit/UncountedCallArgsChecker.cpp       | 36 +++++----
 .../WebKit/UncountedLocalVarsChecker.cpp      | 73 +++++++++++--------
 .../Analysis/Checkers/WebKit/call-args.cpp    | 14 ++++
 .../Checkers/WebKit/uncounted-local-vars.cpp  | 18 +++++
 6 files changed, 113 insertions(+), 62 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
index 5c49eecacc0f..f81db0e67d83 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
@@ -16,8 +16,9 @@
 
 namespace clang {
 
-std::pair<const Expr *, bool>
-tryToFindPtrOrigin(const Expr *E, bool StopAtFirstRefCountedObj) {
+bool tryToFindPtrOrigin(
+    const Expr *E, bool StopAtFirstRefCountedObj,
+    std::function<bool(const clang::Expr *, bool)> callback) {
   while (E) {
     if (auto *tempExpr = dyn_cast<MaterializeTemporaryExpr>(E)) {
       E = tempExpr->getSubExpr();
@@ -31,12 +32,18 @@ tryToFindPtrOrigin(const Expr *E, bool StopAtFirstRefCountedObj) {
       E = tempExpr->getSubExpr();
       continue;
     }
+    if (auto *Expr = dyn_cast<ConditionalOperator>(E)) {
+      return tryToFindPtrOrigin(Expr->getTrueExpr(), StopAtFirstRefCountedObj,
+                                callback) &&
+             tryToFindPtrOrigin(Expr->getFalseExpr(), StopAtFirstRefCountedObj,
+                                callback);
+    }
     if (auto *cast = dyn_cast<CastExpr>(E)) {
       if (StopAtFirstRefCountedObj) {
         if (auto *ConversionFunc =
                 dyn_cast_or_null<FunctionDecl>(cast->getConversionFunction())) {
           if (isCtorOfRefCounted(ConversionFunc))
-            return {E, true};
+            return callback(E, true);
         }
       }
       // FIXME: This can give false "origin" that would lead to false negatives
@@ -51,7 +58,7 @@ tryToFindPtrOrigin(const Expr *E, bool StopAtFirstRefCountedObj) {
           if (IsGetterOfRefCt && *IsGetterOfRefCt) {
             E = memberCall->getImplicitObjectArgument();
             if (StopAtFirstRefCountedObj) {
-              return {E, true};
+              return callback(E, true);
             }
             continue;
           }
@@ -68,17 +75,17 @@ tryToFindPtrOrigin(const Expr *E, bool StopAtFirstRefCountedObj) {
       if (auto *callee = call->getDirectCallee()) {
         if (isCtorOfRefCounted(callee)) {
           if (StopAtFirstRefCountedObj)
-            return {E, true};
+            return callback(E, true);
 
           E = call->getArg(0);
           continue;
         }
 
         if (isReturnValueRefCounted(callee))
-          return {E, true};
+          return callback(E, true);
 
         if (isSingleton(callee))
-          return {E, true};
+          return callback(E, true);
 
         if (isPtrConversion(callee)) {
           E = call->getArg(0);
@@ -95,7 +102,7 @@ tryToFindPtrOrigin(const Expr *E, bool StopAtFirstRefCountedObj) {
     break;
   }
   // Some other expression.
-  return {E, false};
+  return callback(E, false);
 }
 
 bool isASafeCallArg(const Expr *E) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.h b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.h
index e35ea4ef05dd..e972924e0c52 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.h
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/APInt.h"
 #include "llvm/Support/Casting.h"
 
+#include <functional>
 #include <string>
 #include <utility>
 
@@ -48,10 +49,12 @@ class Expr;
 /// represents ref-counted object during the traversal we return relevant
 /// sub-expression and true.
 ///
-/// \returns subexpression that we traversed to and if \p
-/// StopAtFirstRefCountedObj is true we also return whether we stopped early.
-std::pair<const clang::Expr *, bool>
-tryToFindPtrOrigin(const clang::Expr *E, bool StopAtFirstRefCountedObj);
+/// Calls \p callback with the subexpression that we traversed to and if \p
+/// StopAtFirstRefCountedObj is true we also specify whether we stopped early.
+/// Returns false if any of calls to callbacks returned false. Otherwise true.
+bool tryToFindPtrOrigin(
+    const clang::Expr *E, bool StopAtFirstRefCountedObj,
+    std::function<bool(const clang::Expr *, bool)> callback);
 
 /// For \p E referring to a ref-countable/-counted pointer/reference we return
 /// whether it's a safe call argument. Examples: function parameter or
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
index 9a178a690ff2..704c082a4d1d 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
@@ -126,25 +126,23 @@ public:
   }
 
   bool isPtrOriginSafe(const Expr *Arg) const {
-    std::pair<const clang::Expr *, bool> ArgOrigin =
-        tryToFindPtrOrigin(Arg, true);
-
-    // Temporary ref-counted object created as part of the call argument
-    // would outlive the call.
-    if (ArgOrigin.second)
-      return true;
-
-    if (isa<CXXNullPtrLiteralExpr>(ArgOrigin.first)) {
-      // foo(nullptr)
-      return true;
-    }
-    if (isa<IntegerLiteral>(ArgOrigin.first)) {
-      // FIXME: Check the value.
-      // foo(NULL)
-      return true;
-    }
-
-    return isASafeCallArg(ArgOrigin.first);
+    return tryToFindPtrOrigin(Arg, /*StopAtFirstRefCountedObj=*/true,
+                              [](const clang::Expr *ArgOrigin, bool IsSafe) {
+                                if (IsSafe)
+                                  return true;
+                                if (isa<CXXNullPtrLiteralExpr>(ArgOrigin)) {
+                                  // foo(nullptr)
+                                  return true;
+                                }
+                                if (isa<IntegerLiteral>(ArgOrigin)) {
+                                  // FIXME: Check the value.
+                                  // foo(NULL)
+                                  return true;
+                                }
+                                if (isASafeCallArg(ArgOrigin))
+                                  return true;
+                                return false;
+                              });
   }
 
   bool shouldSkipCall(const CallExpr *CE) const {
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
index 98a73810b7af..0d9710a5e2d8 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
@@ -188,39 +188,50 @@ public:
       if (!InitExpr)
         return; // FIXME: later on we might warn on uninitialized vars too
 
-      const clang::Expr *const InitArgOrigin =
-          tryToFindPtrOrigin(InitExpr, /*StopAtFirstRefCountedObj=*/false)
-              .first;
-      if (!InitArgOrigin)
+      if (tryToFindPtrOrigin(
+              InitExpr, /*StopAtFirstRefCountedObj=*/false,
+              [&](const clang::Expr *InitArgOrigin, bool IsSafe) {
+                if (!InitArgOrigin)
+                  return true;
+
+                if (isa<CXXThisExpr>(InitArgOrigin))
+                  return true;
+
+                if (isa<CXXNullPtrLiteralExpr>(InitArgOrigin))
+                  return true;
+
+                if (isa<IntegerLiteral>(InitArgOrigin))
+                  return true;
+
+                if (auto *Ref = llvm::dyn_cast<DeclRefExpr>(InitArgOrigin)) {
+                  if (auto *MaybeGuardian =
+                          dyn_cast_or_null<VarDecl>(Ref->getFoundDecl())) {
+                    const auto *MaybeGuardianArgType =
+                        MaybeGuardian->getType().getTypePtr();
+                    if (MaybeGuardianArgType) {
+                      const CXXRecordDecl *const MaybeGuardianArgCXXRecord =
+                          MaybeGuardianArgType->getAsCXXRecordDecl();
+                      if (MaybeGuardianArgCXXRecord) {
+                        if (MaybeGuardian->isLocalVarDecl() &&
+                            (isRefCounted(MaybeGuardianArgCXXRecord) ||
+                             isRefcountedStringsHack(MaybeGuardian)) &&
+                            isGuardedScopeEmbeddedInGuardianScope(
+                                V, MaybeGuardian))
+                          return true;
+                      }
+                    }
+
+                    // Parameters are guaranteed to be safe for the duration of
+                    // the call by another checker.
+                    if (isa<ParmVarDecl>(MaybeGuardian))
+                      return true;
+                  }
+                }
+
+                return false;
+              }))
         return;
 
-      if (isa<CXXThisExpr>(InitArgOrigin))
-        return;
-
-      if (auto *Ref = llvm::dyn_cast<DeclRefExpr>(InitArgOrigin)) {
-        if (auto *MaybeGuardian =
-                dyn_cast_or_null<VarDecl>(Ref->getFoundDecl())) {
-          const auto *MaybeGuardianArgType =
-              MaybeGuardian->getType().getTypePtr();
-          if (MaybeGuardianArgType) {
-            const CXXRecordDecl *const MaybeGuardianArgCXXRecord =
-                MaybeGuardianArgType->getAsCXXRecordDecl();
-            if (MaybeGuardianArgCXXRecord) {
-              if (MaybeGuardian->isLocalVarDecl() &&
-                  (isRefCounted(MaybeGuardianArgCXXRecord) ||
-                   isRefcountedStringsHack(MaybeGuardian)) &&
-                  isGuardedScopeEmbeddedInGuardianScope(V, MaybeGuardian))
-                return;
-            }
-          }
-
-          // Parameters are guaranteed to be safe for the duration of the call
-          // by another checker.
-          if (isa<ParmVarDecl>(MaybeGuardian))
-            return;
-        }
-      }
-
       reportBug(V);
     }
   }
diff --git a/clang/test/Analysis/Checkers/WebKit/call-args.cpp b/clang/test/Analysis/Checkers/WebKit/call-args.cpp
index 45d900d4ba88..e1bee8a23a25 100644
--- a/clang/test/Analysis/Checkers/WebKit/call-args.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/call-args.cpp
@@ -344,3 +344,17 @@ namespace cxx_member_operator_call {
     // expected-warning@-1{{Call argument for parameter 'bad' is uncounted and unsafe}}
   }
 }
+
+namespace call_with_ptr_on_ref {
+  Ref<RefCountable> provideProtected();
+  void bar(RefCountable* bad);
+  bool baz();
+  void foo(bool v) {
+    bar(v ? nullptr : provideProtected().ptr());
+    bar(baz() ? provideProtected().ptr() : nullptr);
+    bar(v ? provide() : provideProtected().ptr());
+    // expected-warning@-1{{Call argument for parameter 'bad' is uncounted and unsafe}}
+    bar(v ? provideProtected().ptr() : provide());
+    // expected-warning@-1{{Call argument for parameter 'bad' is uncounted and unsafe}}
+  }
+}
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
index 8da1dc557a5a..632a82eb0d8d 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
@@ -198,3 +198,21 @@ void system_header() {
 }
 
 } // ignore_system_headers
+
+namespace conditional_op {
+RefCountable *provide_ref_ctnbl();
+bool bar();
+
+void foo() {
+  RefCountable *a = bar() ? nullptr : provide_ref_ctnbl();
+  // expected-warning@-1{{Local variable 'a' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+  RefPtr<RefCountable> b = provide_ref_ctnbl();
+  {
+    RefCountable* c = bar() ? nullptr : b.get();
+    c->method();
+    RefCountable* d = bar() ? b.get() : nullptr;
+    d->method();
+  }
+}
+
+} // namespace conditional_op
-- 
GitLab


From 2c5f470da6ab313f7c6a1aa53fb40dbcbde338f1 Mon Sep 17 00:00:00 2001
From: Chaitanya <Krishna.Sankisa@amd.com>
Date: Fri, 10 May 2024 10:49:48 +0530
Subject: [PATCH 0371/1206] [AMDGPU] Move LDS utilities from
 amdgpu-lower-module-lds pass to AMDGPUMemoryUtils (#88002)

This moves some of the utility methods from amdgpu-lower-module-lds pass to AMDGPUMemoryUtils.
---
 .../AMDGPU/AMDGPULowerModuleLDSPass.cpp       | 186 +--------------
 .../Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp | 215 +++++++++++++++++-
 .../Target/AMDGPU/Utils/AMDGPUMemoryUtils.h   |  31 ++-
 3 files changed, 245 insertions(+), 187 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
index c8bf9dd39e38..2c7163a77537 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
@@ -212,6 +212,7 @@
 #define DEBUG_TYPE "amdgpu-lower-module-lds"
 
 using namespace llvm;
+using namespace AMDGPU;
 
 namespace {
 
@@ -234,17 +235,6 @@ cl::opt<LoweringKind> LoweringKindLoc(
         clEnumValN(LoweringKind::hybrid, "hybrid",
                    "Lower via mixture of above strategies")));
 
-bool isKernelLDS(const Function *F) {
-  // Some weirdness here. AMDGPU::isKernelCC does not call into
-  // AMDGPU::isKernel with the calling conv, it instead calls into
-  // isModuleEntryFunction which returns true for more calling conventions
-  // than AMDGPU::isKernel does. There's a FIXME on AMDGPU::isKernel.
-  // There's also a test that checks that the LDS lowering does not hit on
-  // a graphics shader, denoted amdgpu_ps, so stay with the limited case.
-  // Putting LDS in the name of the function to draw attention to this.
-  return AMDGPU::isKernel(F->getCallingConv());
-}
-
 template <typename T> std::vector<T> sortByName(std::vector<T> &&V) {
   llvm::sort(V.begin(), V.end(), [](const auto *L, const auto *R) {
     return L->getName() < R->getName();
@@ -305,183 +295,9 @@ class AMDGPULowerModuleLDS {
         Decl, {}, {OperandBundleDefT<Value *>("ExplicitUse", UseInstance)});
   }
 
-  static bool eliminateConstantExprUsesOfLDSFromAllInstructions(Module &M) {
-    // Constants are uniqued within LLVM. A ConstantExpr referring to a LDS
-    // global may have uses from multiple different functions as a result.
-    // This pass specialises LDS variables with respect to the kernel that
-    // allocates them.
-
-    // This is semantically equivalent to (the unimplemented as slow):
-    // for (auto &F : M.functions())
-    //   for (auto &BB : F)
-    //     for (auto &I : BB)
-    //       for (Use &Op : I.operands())
-    //         if (constantExprUsesLDS(Op))
-    //           replaceConstantExprInFunction(I, Op);
-
-    SmallVector<Constant *> LDSGlobals;
-    for (auto &GV : M.globals())
-      if (AMDGPU::isLDSVariableToLower(GV))
-        LDSGlobals.push_back(&GV);
-
-    return convertUsersOfConstantsToInstructions(LDSGlobals);
-  }
-
 public:
   AMDGPULowerModuleLDS(const AMDGPUTargetMachine &TM_) : TM(TM_) {}
 
-  using FunctionVariableMap = DenseMap<Function *, DenseSet<GlobalVariable *>>;
-
-  using VariableFunctionMap = DenseMap<GlobalVariable *, DenseSet<Function *>>;
-
-  static void getUsesOfLDSByFunction(CallGraph const &CG, Module &M,
-                                     FunctionVariableMap &kernels,
-                                     FunctionVariableMap &functions) {
-
-    // Get uses from the current function, excluding uses by called functions
-    // Two output variables to avoid walking the globals list twice
-    for (auto &GV : M.globals()) {
-      if (!AMDGPU::isLDSVariableToLower(GV)) {
-        continue;
-      }
-
-      for (User *V : GV.users()) {
-        if (auto *I = dyn_cast<Instruction>(V)) {
-          Function *F = I->getFunction();
-          if (isKernelLDS(F)) {
-            kernels[F].insert(&GV);
-          } else {
-            functions[F].insert(&GV);
-          }
-        }
-      }
-    }
-  }
-
-  struct LDSUsesInfoTy {
-    FunctionVariableMap direct_access;
-    FunctionVariableMap indirect_access;
-  };
-
-  static LDSUsesInfoTy getTransitiveUsesOfLDS(CallGraph const &CG, Module &M) {
-
-    FunctionVariableMap direct_map_kernel;
-    FunctionVariableMap direct_map_function;
-    getUsesOfLDSByFunction(CG, M, direct_map_kernel, direct_map_function);
-
-    // Collect variables that are used by functions whose address has escaped
-    DenseSet<GlobalVariable *> VariablesReachableThroughFunctionPointer;
-    for (Function &F : M.functions()) {
-      if (!isKernelLDS(&F))
-        if (F.hasAddressTaken(nullptr,
-                              /* IgnoreCallbackUses */ false,
-                              /* IgnoreAssumeLikeCalls */ false,
-                              /* IgnoreLLVMUsed */ true,
-                              /* IgnoreArcAttachedCall */ false)) {
-          set_union(VariablesReachableThroughFunctionPointer,
-                    direct_map_function[&F]);
-        }
-    }
-
-    auto functionMakesUnknownCall = [&](const Function *F) -> bool {
-      assert(!F->isDeclaration());
-      for (const CallGraphNode::CallRecord &R : *CG[F]) {
-        if (!R.second->getFunction()) {
-          return true;
-        }
-      }
-      return false;
-    };
-
-    // Work out which variables are reachable through function calls
-    FunctionVariableMap transitive_map_function = direct_map_function;
-
-    // If the function makes any unknown call, assume the worst case that it can
-    // access all variables accessed by functions whose address escaped
-    for (Function &F : M.functions()) {
-      if (!F.isDeclaration() && functionMakesUnknownCall(&F)) {
-        if (!isKernelLDS(&F)) {
-          set_union(transitive_map_function[&F],
-                    VariablesReachableThroughFunctionPointer);
-        }
-      }
-    }
-
-    // Direct implementation of collecting all variables reachable from each
-    // function
-    for (Function &Func : M.functions()) {
-      if (Func.isDeclaration() || isKernelLDS(&Func))
-        continue;
-
-      DenseSet<Function *> seen; // catches cycles
-      SmallVector<Function *, 4> wip{&Func};
-
-      while (!wip.empty()) {
-        Function *F = wip.pop_back_val();
-
-        // Can accelerate this by referring to transitive map for functions that
-        // have already been computed, with more care than this
-        set_union(transitive_map_function[&Func], direct_map_function[F]);
-
-        for (const CallGraphNode::CallRecord &R : *CG[F]) {
-          Function *ith = R.second->getFunction();
-          if (ith) {
-            if (!seen.contains(ith)) {
-              seen.insert(ith);
-              wip.push_back(ith);
-            }
-          }
-        }
-      }
-    }
-
-    // direct_map_kernel lists which variables are used by the kernel
-    // find the variables which are used through a function call
-    FunctionVariableMap indirect_map_kernel;
-
-    for (Function &Func : M.functions()) {
-      if (Func.isDeclaration() || !isKernelLDS(&Func))
-        continue;
-
-      for (const CallGraphNode::CallRecord &R : *CG[&Func]) {
-        Function *ith = R.second->getFunction();
-        if (ith) {
-          set_union(indirect_map_kernel[&Func], transitive_map_function[ith]);
-        } else {
-          set_union(indirect_map_kernel[&Func],
-                    VariablesReachableThroughFunctionPointer);
-        }
-      }
-    }
-
-    // Verify that we fall into one of 2 cases:
-    //    - All variables are absolute: this is a re-run of the pass
-    //      so we don't have anything to do.
-    //    - No variables are absolute.
-    std::optional<bool> HasAbsoluteGVs;
-    for (auto &Map : {direct_map_kernel, indirect_map_kernel}) {
-      for (auto &[Fn, GVs] : Map) {
-        for (auto *GV : GVs) {
-          bool IsAbsolute = GV->isAbsoluteSymbolRef();
-          if (HasAbsoluteGVs.has_value()) {
-            if (*HasAbsoluteGVs != IsAbsolute) {
-              report_fatal_error(
-                  "Module cannot mix absolute and non-absolute LDS GVs");
-            }
-          } else
-            HasAbsoluteGVs = IsAbsolute;
-        }
-      }
-    }
-
-    // If we only had absolute GVs, we have nothing to do, return an empty
-    // result.
-    if (HasAbsoluteGVs && *HasAbsoluteGVs)
-      return {FunctionVariableMap(), FunctionVariableMap()};
-
-    return {std::move(direct_map_kernel), std::move(indirect_map_kernel)};
-  }
-
   struct LDSVariableReplacement {
     GlobalVariable *SGV = nullptr;
     DenseMap<GlobalVariable *, Constant *> LDSVarsToConstantGEP;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp
index 79c359a57554..239e0ee70572 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp
@@ -9,13 +9,16 @@
 #include "AMDGPUMemoryUtils.h"
 #include "AMDGPU.h"
 #include "AMDGPUBaseInfo.h"
+#include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/MemorySSA.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Operator.h"
 #include "llvm/IR/ReplaceConstant.h"
 
 #define DEBUG_TYPE "amdgpu-memory-utils"
@@ -26,7 +29,7 @@ namespace llvm {
 
 namespace AMDGPU {
 
-Align getAlign(DataLayout const &DL, const GlobalVariable *GV) {
+Align getAlign(const DataLayout &DL, const GlobalVariable *GV) {
   return DL.getValueOrABITypeAlignment(GV->getPointerAlignment(DL),
                                        GV->getValueType());
 }
@@ -61,6 +64,216 @@ bool isLDSVariableToLower(const GlobalVariable &GV) {
   return true;
 }
 
+bool eliminateConstantExprUsesOfLDSFromAllInstructions(Module &M) {
+  // Constants are uniqued within LLVM. A ConstantExpr referring to a LDS
+  // global may have uses from multiple different functions as a result.
+  // This pass specialises LDS variables with respect to the kernel that
+  // allocates them.
+
+  // This is semantically equivalent to (the unimplemented as slow):
+  // for (auto &F : M.functions())
+  //   for (auto &BB : F)
+  //     for (auto &I : BB)
+  //       for (Use &Op : I.operands())
+  //         if (constantExprUsesLDS(Op))
+  //           replaceConstantExprInFunction(I, Op);
+
+  SmallVector<Constant *> LDSGlobals;
+  for (auto &GV : M.globals())
+    if (AMDGPU::isLDSVariableToLower(GV))
+      LDSGlobals.push_back(&GV);
+  return convertUsersOfConstantsToInstructions(LDSGlobals);
+}
+
+void getUsesOfLDSByFunction(const CallGraph &CG, Module &M,
+                            FunctionVariableMap &kernels,
+                            FunctionVariableMap &Functions) {
+  // Get uses from the current function, excluding uses by called Functions
+  // Two output variables to avoid walking the globals list twice
+  for (auto &GV : M.globals()) {
+    if (!AMDGPU::isLDSVariableToLower(GV))
+      continue;
+    for (User *V : GV.users()) {
+      if (auto *I = dyn_cast<Instruction>(V)) {
+        Function *F = I->getFunction();
+        if (isKernelLDS(F))
+          kernels[F].insert(&GV);
+        else
+          Functions[F].insert(&GV);
+      }
+    }
+  }
+}
+
+bool isKernelLDS(const Function *F) {
+  // Some weirdness here. AMDGPU::isKernelCC does not call into
+  // AMDGPU::isKernel with the calling conv, it instead calls into
+  // isModuleEntryFunction which returns true for more calling conventions
+  // than AMDGPU::isKernel does. There's a FIXME on AMDGPU::isKernel.
+  // There's also a test that checks that the LDS lowering does not hit on
+  // a graphics shader, denoted amdgpu_ps, so stay with the limited case.
+  // Putting LDS in the name of the function to draw attention to this.
+  return AMDGPU::isKernel(F->getCallingConv());
+}
+
+LDSUsesInfoTy getTransitiveUsesOfLDS(const CallGraph &CG, Module &M) {
+
+  FunctionVariableMap DirectMapKernel;
+  FunctionVariableMap DirectMapFunction;
+  getUsesOfLDSByFunction(CG, M, DirectMapKernel, DirectMapFunction);
+
+  // Collect variables that are used by functions whose address has escaped
+  DenseSet<GlobalVariable *> VariablesReachableThroughFunctionPointer;
+  for (Function &F : M.functions()) {
+    if (!isKernelLDS(&F))
+      if (F.hasAddressTaken(nullptr,
+                            /* IgnoreCallbackUses */ false,
+                            /* IgnoreAssumeLikeCalls */ false,
+                            /* IgnoreLLVMUsed */ true,
+                            /* IgnoreArcAttachedCall */ false)) {
+        set_union(VariablesReachableThroughFunctionPointer,
+                  DirectMapFunction[&F]);
+      }
+  }
+
+  auto FunctionMakesUnknownCall = [&](const Function *F) -> bool {
+    assert(!F->isDeclaration());
+    for (const CallGraphNode::CallRecord &R : *CG[F]) {
+      if (!R.second->getFunction())
+        return true;
+    }
+    return false;
+  };
+
+  // Work out which variables are reachable through function calls
+  FunctionVariableMap TransitiveMapFunction = DirectMapFunction;
+
+  // If the function makes any unknown call, assume the worst case that it can
+  // access all variables accessed by functions whose address escaped
+  for (Function &F : M.functions()) {
+    if (!F.isDeclaration() && FunctionMakesUnknownCall(&F)) {
+      if (!isKernelLDS(&F)) {
+        set_union(TransitiveMapFunction[&F],
+                  VariablesReachableThroughFunctionPointer);
+      }
+    }
+  }
+
+  // Direct implementation of collecting all variables reachable from each
+  // function
+  for (Function &Func : M.functions()) {
+    if (Func.isDeclaration() || isKernelLDS(&Func))
+      continue;
+
+    DenseSet<Function *> seen; // catches cycles
+    SmallVector<Function *, 4> wip = {&Func};
+
+    while (!wip.empty()) {
+      Function *F = wip.pop_back_val();
+
+      // Can accelerate this by referring to transitive map for functions that
+      // have already been computed, with more care than this
+      set_union(TransitiveMapFunction[&Func], DirectMapFunction[F]);
+
+      for (const CallGraphNode::CallRecord &R : *CG[F]) {
+        Function *Ith = R.second->getFunction();
+        if (Ith) {
+          if (!seen.contains(Ith)) {
+            seen.insert(Ith);
+            wip.push_back(Ith);
+          }
+        }
+      }
+    }
+  }
+
+  // DirectMapKernel lists which variables are used by the kernel
+  // find the variables which are used through a function call
+  FunctionVariableMap IndirectMapKernel;
+
+  for (Function &Func : M.functions()) {
+    if (Func.isDeclaration() || !isKernelLDS(&Func))
+      continue;
+
+    for (const CallGraphNode::CallRecord &R : *CG[&Func]) {
+      Function *Ith = R.second->getFunction();
+      if (Ith) {
+        set_union(IndirectMapKernel[&Func], TransitiveMapFunction[Ith]);
+      } else {
+        set_union(IndirectMapKernel[&Func],
+                  VariablesReachableThroughFunctionPointer);
+      }
+    }
+  }
+
+  // Verify that we fall into one of 2 cases:
+  //    - All variables are absolute: this is a re-run of the pass
+  //      so we don't have anything to do.
+  //    - No variables are absolute.
+  std::optional<bool> HasAbsoluteGVs;
+  for (auto &Map : {DirectMapKernel, IndirectMapKernel}) {
+    for (auto &[Fn, GVs] : Map) {
+      for (auto *GV : GVs) {
+        bool IsAbsolute = GV->isAbsoluteSymbolRef();
+        if (HasAbsoluteGVs.has_value()) {
+          if (*HasAbsoluteGVs != IsAbsolute) {
+            report_fatal_error(
+                "Module cannot mix absolute and non-absolute LDS GVs");
+          }
+        } else
+          HasAbsoluteGVs = IsAbsolute;
+      }
+    }
+  }
+
+  // If we only had absolute GVs, we have nothing to do, return an empty
+  // result.
+  if (HasAbsoluteGVs && *HasAbsoluteGVs)
+    return {FunctionVariableMap(), FunctionVariableMap()};
+
+  return {std::move(DirectMapKernel), std::move(IndirectMapKernel)};
+}
+
+void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot,
+                               StringRef FnAttr) {
+  KernelRoot->removeFnAttr(FnAttr);
+
+  SmallVector<Function *> WorkList = {CG[KernelRoot]->getFunction()};
+  SmallPtrSet<Function *, 8> Visited;
+  bool SeenUnknownCall = false;
+
+  while (!WorkList.empty()) {
+    Function *F = WorkList.pop_back_val();
+
+    for (auto &CallRecord : *CG[F]) {
+      if (!CallRecord.second)
+        continue;
+
+      Function *Callee = CallRecord.second->getFunction();
+      if (!Callee) {
+        if (!SeenUnknownCall) {
+          SeenUnknownCall = true;
+
+          // If we see any indirect calls, assume nothing about potential
+          // targets.
+          // TODO: This could be refined to possible LDS global users.
+          for (auto &ExternalCallRecord : *CG.getExternalCallingNode()) {
+            Function *PotentialCallee =
+                ExternalCallRecord.second->getFunction();
+            assert(PotentialCallee);
+            if (!isKernelLDS(PotentialCallee))
+              PotentialCallee->removeFnAttr(FnAttr);
+          }
+        }
+      } else {
+        Callee->removeFnAttr(FnAttr);
+        if (Visited.insert(Callee).second)
+          WorkList.push_back(Callee);
+      }
+    }
+  }
+}
+
 bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA) {
   Instruction *DefInst = Def->getMemoryInst();
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h
index e42b27f8e09e..4d3ad328e131 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h
@@ -9,6 +9,9 @@
 #ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUMEMORYUTILS_H
 #define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUMEMORYUTILS_H
 
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+
 namespace llvm {
 
 struct Align;
@@ -19,14 +22,40 @@ class LoadInst;
 class MemoryDef;
 class MemorySSA;
 class Value;
+class Function;
+class CallGraph;
+class Module;
 
 namespace AMDGPU {
 
-Align getAlign(DataLayout const &DL, const GlobalVariable *GV);
+using FunctionVariableMap = DenseMap<Function *, DenseSet<GlobalVariable *>>;
+using VariableFunctionMap = DenseMap<GlobalVariable *, DenseSet<Function *>>;
+
+Align getAlign(const DataLayout &DL, const GlobalVariable *GV);
 
 bool isDynamicLDS(const GlobalVariable &GV);
 bool isLDSVariableToLower(const GlobalVariable &GV);
 
+struct LDSUsesInfoTy {
+  FunctionVariableMap direct_access;
+  FunctionVariableMap indirect_access;
+};
+
+bool eliminateConstantExprUsesOfLDSFromAllInstructions(Module &M);
+
+void getUsesOfLDSByFunction(const CallGraph &CG, Module &M,
+                            FunctionVariableMap &kernels,
+                            FunctionVariableMap &functions);
+
+bool isKernelLDS(const Function *F);
+
+LDSUsesInfoTy getTransitiveUsesOfLDS(const CallGraph &CG, Module &M);
+
+/// Strip FnAttr attribute from any functions where we may have
+/// introduced its use.
+void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot,
+                               StringRef FnAttr);
+
 /// Given a \p Def clobbering a load from \p Ptr according to the MSSA check
 /// if this is actually a memory update or an artificial clobber to facilitate
 /// ordering constraints.
-- 
GitLab


From 87f3407856e61a73798af4e41b28bc33b5bf4ce6 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Fri, 10 May 2024 13:25:37 +0800
Subject: [PATCH 0372/1206] [X86][Driver] Do not add `-evex512` for
 `-march=native` when the target doesn't support AVX512 (#91694)

---
 llvm/lib/TargetParser/Host.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 834f4536f93a..c5156c6cb802 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -1802,7 +1802,8 @@ bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   Features["rtm"]        = HasLeaf7 && ((EBX >> 11) & 1);
   // AVX512 is only supported if the OS supports the context save for it.
   Features["avx512f"]    = HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save;
-  Features["evex512"]    = Features["avx512f"];
+  if (Features["avx512f"])
+    Features["evex512"]  = true;
   Features["avx512dq"]   = HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save;
   Features["rdseed"]     = HasLeaf7 && ((EBX >> 18) & 1);
   Features["adx"]        = HasLeaf7 && ((EBX >> 19) & 1);
-- 
GitLab


From 06ad86361ab29d344a5e2e22903c2739743c77be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Danny=20M=C3=B6sch?= <danny.moesch@icloud.com>
Date: Fri, 10 May 2024 07:56:35 +0200
Subject: [PATCH 0373/1206] [NFC] Prefer `str.resize(len)` over `str.substr(0,
 len)` (#91067)

---
 llvm/include/llvm/Analysis/DOTGraphTraitsPass.h | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h b/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h
index da72fb511f82..7aea7a3b0f6d 100644
--- a/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h
+++ b/llvm/include/llvm/Analysis/DOTGraphTraitsPass.h
@@ -87,13 +87,12 @@ private:
 };
 
 static inline void shortenFileName(std::string &FN, unsigned char len = 250) {
-
-  FN = FN.substr(0, len);
-
+  if (FN.length() > len)
+    FN.resize(len);
   auto strLen = FN.length();
   while (strLen > 0) {
-    if (auto it = nameObj.find(FN); it != nameObj.end()) {
-      FN = FN.substr(0, --len);
+    if (nameObj.find(FN) != nameObj.end()) {
+      FN.resize(--len);
     } else {
       nameObj.insert(FN);
       break;
-- 
GitLab


From 135d92f903161e66ff82ab846acfbc5015ef3096 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 9 May 2024 23:12:08 -0700
Subject: [PATCH 0374/1206] [Driver] Use StringRef::operator== instead of
 StringRef::equals (NFC) (#91698)

I'm planning to remove StringRef::equals in favor of
StringRef::operator==.

- StringRef::operator==/!= outnumber StringRef::equals by a factor of
  13 under clang/ in terms of their usage.

- The elimination of StringRef::equals brings StringRef closer to
  std::string_view, which has operator== but not equals.

- S == "foo" is more readable than S.equals("foo"), especially for
  !Long.Expression.equals("str") vs Long.Expression != "str".
---
 clang/lib/Driver/Driver.cpp                |  6 +-
 clang/lib/Driver/ToolChains/AIX.cpp        |  4 +-
 clang/lib/Driver/ToolChains/AMDGPU.cpp     |  2 +-
 clang/lib/Driver/ToolChains/Clang.cpp      | 66 +++++++++++-----------
 clang/lib/Driver/ToolChains/CommonArgs.cpp |  2 +-
 clang/lib/Driver/ToolChains/Flang.cpp      | 13 ++---
 6 files changed, 45 insertions(+), 48 deletions(-)

diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 114320f5d314..7b36d8e5084c 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -564,9 +564,9 @@ static llvm::Triple computeTargetTriple(const Driver &D,
       StringRef ObjectMode = *ObjectModeValue;
       llvm::Triple::ArchType AT = llvm::Triple::UnknownArch;
 
-      if (ObjectMode.equals("64")) {
+      if (ObjectMode == "64") {
         AT = Target.get64BitArchVariant().getArch();
-      } else if (ObjectMode.equals("32")) {
+      } else if (ObjectMode == "32") {
         AT = Target.get32BitArchVariant().getArch();
       } else {
         D.Diag(diag::err_drv_invalid_object_mode) << ObjectMode;
@@ -6694,7 +6694,7 @@ llvm::StringRef clang::driver::getDriverMode(StringRef ProgName,
   return Opt.consume_front(OptName) ? Opt : "";
 }
 
-bool driver::IsClangCL(StringRef DriverMode) { return DriverMode.equals("cl"); }
+bool driver::IsClangCL(StringRef DriverMode) { return DriverMode == "cl"; }
 
 llvm::Error driver::expandResponseFiles(SmallVectorImpl<const char *> &Args,
                                         bool ClangCLMode,
diff --git a/clang/lib/Driver/ToolChains/AIX.cpp b/clang/lib/Driver/ToolChains/AIX.cpp
index aab98506adb9..85825e1ea65b 100644
--- a/clang/lib/Driver/ToolChains/AIX.cpp
+++ b/clang/lib/Driver/ToolChains/AIX.cpp
@@ -481,8 +481,8 @@ static void addTocDataOptions(const llvm::opt::ArgList &Args,
 
   // Currently only supported for small code model.
   if (TOCDataGloballyinEffect &&
-      (Args.getLastArgValue(options::OPT_mcmodel_EQ).equals("large") ||
-       Args.getLastArgValue(options::OPT_mcmodel_EQ).equals("medium"))) {
+      (Args.getLastArgValue(options::OPT_mcmodel_EQ) == "large" ||
+       Args.getLastArgValue(options::OPT_mcmodel_EQ) == "medium")) {
     D.Diag(clang::diag::warn_drv_unsupported_tocdata);
     return;
   }
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index 07965b487ea7..9ffea57b005d 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -732,7 +732,7 @@ AMDGPUToolChain::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch,
 
   checkTargetID(*DAL);
 
-  if (!Args.getLastArgValue(options::OPT_x).equals("cl"))
+  if (Args.getLastArgValue(options::OPT_x) != "cl")
     return DAL;
 
   // Phase 1 (.cl -> .bc)
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 775dc249999e..449eb9b2a965 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1526,7 +1526,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args,
 
   CmdArgs.push_back(
       Args.MakeArgString(Twine("-msign-return-address=") + Scope));
-  if (!Scope.equals("none"))
+  if (Scope != "none")
     CmdArgs.push_back(
         Args.MakeArgString(Twine("-msign-return-address-key=") + Key));
   if (BranchProtectionPAuthLR)
@@ -1719,10 +1719,9 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
   if (Arg *A = Args.getLastArg(options::OPT_msve_vector_bits_EQ)) {
     StringRef Val = A->getValue();
     const Driver &D = getToolChain().getDriver();
-    if (Val.equals("128") || Val.equals("256") || Val.equals("512") ||
-        Val.equals("1024") || Val.equals("2048") || Val.equals("128+") ||
-        Val.equals("256+") || Val.equals("512+") || Val.equals("1024+") ||
-        Val.equals("2048+")) {
+    if (Val == "128" || Val == "256" || Val == "512" || Val == "1024" ||
+        Val == "2048" || Val == "128+" || Val == "256+" || Val == "512+" ||
+        Val == "1024+" || Val == "2048+") {
       unsigned Bits = 0;
       if (!Val.consume_back("+")) {
         bool Invalid = Val.getAsInteger(10, Bits); (void)Invalid;
@@ -1736,7 +1735,7 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
       CmdArgs.push_back(
           Args.MakeArgString("-mvscale-min=" + llvm::Twine(Bits / 128)));
     // Silently drop requests for vector-length agnostic code as it's implied.
-    } else if (!Val.equals("scalable"))
+    } else if (Val != "scalable")
       // Handle the unsupported values passed to msve-vector-bits.
       D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << Val;
@@ -2098,7 +2097,7 @@ void Clang::AddRISCVTargetArgs(const ArgList &Args,
     // If the value is "zvl", use MinVLen from march. Otherwise, try to parse
     // as integer as long as we have a MinVLen.
     unsigned Bits = 0;
-    if (Val.equals("zvl") && MinVLen >= llvm::RISCV::RVVBitsPerBlock) {
+    if (Val == "zvl" && MinVLen >= llvm::RISCV::RVVBitsPerBlock) {
       Bits = MinVLen;
     } else if (!Val.getAsInteger(10, Bits)) {
       // Only accept power of 2 values beteen RVVBitsPerBlock and 65536 that
@@ -2115,7 +2114,7 @@ void Clang::AddRISCVTargetArgs(const ArgList &Args,
           Args.MakeArgString("-mvscale-max=" + llvm::Twine(VScaleMin)));
       CmdArgs.push_back(
           Args.MakeArgString("-mvscale-min=" + llvm::Twine(VScaleMin)));
-    } else if (!Val.equals("scalable")) {
+    } else if (Val != "scalable") {
       // Handle the unsupported values passed to mrvv-vector-bits.
       D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << Val;
@@ -2865,13 +2864,13 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
     case options::OPT_fcomplex_arithmetic_EQ: {
       LangOptions::ComplexRangeKind RangeVal;
       StringRef Val = A->getValue();
-      if (Val.equals("full"))
+      if (Val == "full")
         RangeVal = LangOptions::ComplexRangeKind::CX_Full;
-      else if (Val.equals("improved"))
+      else if (Val == "improved")
         RangeVal = LangOptions::ComplexRangeKind::CX_Improved;
-      else if (Val.equals("promoted"))
+      else if (Val == "promoted")
         RangeVal = LangOptions::ComplexRangeKind::CX_Promoted;
-      else if (Val.equals("basic"))
+      else if (Val == "basic")
         RangeVal = LangOptions::ComplexRangeKind::CX_Basic;
       else {
         D.Diag(diag::err_drv_unsupported_option_argument)
@@ -2910,24 +2909,24 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       FPContract = "on";
 
       StringRef Val = A->getValue();
-      if (OFastEnabled && !Val.equals("fast")) {
-          // Only -ffp-model=fast is compatible with OFast, ignore.
+      if (OFastEnabled && Val != "fast") {
+        // Only -ffp-model=fast is compatible with OFast, ignore.
         D.Diag(clang::diag::warn_drv_overriding_option)
             << Args.MakeArgString("-ffp-model=" + Val) << "-Ofast";
         break;
       }
       StrictFPModel = false;
-      if (!FPModel.empty() && !FPModel.equals(Val))
+      if (!FPModel.empty() && FPModel != Val)
         D.Diag(clang::diag::warn_drv_overriding_option)
             << Args.MakeArgString("-ffp-model=" + FPModel)
             << Args.MakeArgString("-ffp-model=" + Val);
-      if (Val.equals("fast")) {
+      if (Val == "fast") {
         FPModel = Val;
         applyFastMath();
-      } else if (Val.equals("precise")) {
+      } else if (Val == "precise") {
         FPModel = Val;
         FPContract = "on";
-      } else if (Val.equals("strict")) {
+      } else if (Val == "strict") {
         StrictFPModel = true;
         FPExceptionBehavior = "strict";
         FPModel = Val;
@@ -2957,7 +2956,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
     case options::OPT_fno_signed_zeros:     SignedZeros = false;      break;
     case options::OPT_ftrapping_math:
       if (!TrappingMathPresent && !FPExceptionBehavior.empty() &&
-          !FPExceptionBehavior.equals("strict"))
+          FPExceptionBehavior != "strict")
         // Warn that previous value of option is overridden.
         D.Diag(clang::diag::warn_drv_overriding_option)
             << Args.MakeArgString("-ffp-exception-behavior=" +
@@ -2969,7 +2968,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       break;
     case options::OPT_fno_trapping_math:
       if (!TrappingMathPresent && !FPExceptionBehavior.empty() &&
-          !FPExceptionBehavior.equals("ignore"))
+          FPExceptionBehavior != "ignore")
         // Warn that previous value of option is overridden.
         D.Diag(clang::diag::warn_drv_overriding_option)
             << Args.MakeArgString("-ffp-exception-behavior=" +
@@ -3008,8 +3007,8 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
     // Validate and pass through -ffp-contract option.
     case options::OPT_ffp_contract: {
       StringRef Val = A->getValue();
-      if (Val.equals("fast") || Val.equals("on") || Val.equals("off") ||
-          Val.equals("fast-honor-pragmas")) {
+      if (Val == "fast" || Val == "on" || Val == "off" ||
+          Val == "fast-honor-pragmas") {
         FPContract = Val;
         LastSeenFfpContractOption = Val;
       } else
@@ -3022,16 +3021,16 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
     case options::OPT_ffp_exception_behavior_EQ: {
       StringRef Val = A->getValue();
       if (!TrappingMathPresent && !FPExceptionBehavior.empty() &&
-          !FPExceptionBehavior.equals(Val))
+          FPExceptionBehavior != Val)
         // Warn that previous value of option is overridden.
         D.Diag(clang::diag::warn_drv_overriding_option)
             << Args.MakeArgString("-ffp-exception-behavior=" +
                                   FPExceptionBehavior)
             << Args.MakeArgString("-ffp-exception-behavior=" + Val);
       TrappingMath = TrappingMathPresent = false;
-      if (Val.equals("ignore") || Val.equals("maytrap"))
+      if (Val == "ignore" || Val == "maytrap")
         FPExceptionBehavior = Val;
-      else if (Val.equals("strict")) {
+      else if (Val == "strict") {
         FPExceptionBehavior = Val;
         TrappingMath = TrappingMathPresent = true;
       } else
@@ -3043,8 +3042,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
     // Validate and pass through -ffp-eval-method option.
     case options::OPT_ffp_eval_method_EQ: {
       StringRef Val = A->getValue();
-      if (Val.equals("double") || Val.equals("extended") ||
-          Val.equals("source"))
+      if (Val == "double" || Val == "extended" || Val == "source")
         FPEvalMethod = Val;
       else
         D.Diag(diag::err_drv_unsupported_option_argument)
@@ -3056,18 +3054,18 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       StringRef Val = A->getValue();
       const llvm::Triple::ArchType Arch = TC.getArch();
       if (Arch == llvm::Triple::x86 || Arch == llvm::Triple::x86_64) {
-        if (Val.equals("standard") || Val.equals("fast"))
+        if (Val == "standard" || Val == "fast")
           Float16ExcessPrecision = Val;
         // To make it GCC compatible, allow the value of "16" which
         // means disable excess precision, the same meaning than clang's
         // equivalent value "none".
-        else if (Val.equals("16"))
+        else if (Val == "16")
           Float16ExcessPrecision = "none";
         else
           D.Diag(diag::err_drv_unsupported_option_argument)
               << A->getSpelling() << Val;
       } else {
-        if (!(Val.equals("standard") || Val.equals("fast")))
+        if (!(Val == "standard" || Val == "fast"))
           D.Diag(diag::err_drv_unsupported_option_argument)
               << A->getSpelling() << Val;
       }
@@ -3149,7 +3147,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       // subsequent options conflict then emit warning diagnostic.
       if (HonorINFs && HonorNaNs && !AssociativeMath && !ReciprocalMath &&
           SignedZeros && TrappingMath && RoundingFPMath && !ApproxFunc &&
-          FPContract.equals("off"))
+          FPContract == "off")
         // OK: Current Arg doesn't conflict with -ffp-model=strict
         ;
       else {
@@ -3195,7 +3193,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
 
   if (TrappingMath) {
     // FP Exception Behavior is also set to strict
-    assert(FPExceptionBehavior.equals("strict"));
+    assert(FPExceptionBehavior == "strict");
   }
 
   // The default is IEEE.
@@ -3244,8 +3242,8 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
   if (!HonorINFs && !HonorNaNs && !MathErrno && AssociativeMath && ApproxFunc &&
       ReciprocalMath && !SignedZeros && !TrappingMath && !RoundingFPMath) {
     CmdArgs.push_back("-ffast-math");
-    if (FPModel.equals("fast")) {
-      if (FPContract.equals("fast"))
+    if (FPModel == "fast") {
+      if (FPContract == "fast")
         // All set, do nothing.
         ;
       else if (FPContract.empty())
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 6796b43a1550..71e993119436 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -346,7 +346,7 @@ void tools::addDirectoryList(const ArgList &Args, ArgStringList &CmdArgs,
     return; // Nothing to do.
 
   StringRef Name(ArgName);
-  if (Name.equals("-I") || Name.equals("-L") || Name.empty())
+  if (Name == "-I" || Name == "-L" || Name.empty())
     CombinedArg = true;
 
   StringRef Dirs(DirList);
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 436a9c418a5f..d275528b6905 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -170,10 +170,9 @@ void Flang::AddAArch64TargetArgs(const ArgList &Args,
   if (Arg *A = Args.getLastArg(options::OPT_msve_vector_bits_EQ)) {
     StringRef Val = A->getValue();
     const Driver &D = getToolChain().getDriver();
-    if (Val.equals("128") || Val.equals("256") || Val.equals("512") ||
-        Val.equals("1024") || Val.equals("2048") || Val.equals("128+") ||
-        Val.equals("256+") || Val.equals("512+") || Val.equals("1024+") ||
-        Val.equals("2048+")) {
+    if (Val == "128" || Val == "256" || Val == "512" || Val == "1024" ||
+        Val == "2048" || Val == "128+" || Val == "256+" || Val == "512+" ||
+        Val == "1024+" || Val == "2048+") {
       unsigned Bits = 0;
       if (!Val.consume_back("+")) {
         [[maybe_unused]] bool Invalid = Val.getAsInteger(10, Bits);
@@ -187,7 +186,7 @@ void Flang::AddAArch64TargetArgs(const ArgList &Args,
       CmdArgs.push_back(
           Args.MakeArgString("-mvscale-min=" + llvm::Twine(Bits / 128)));
       // Silently drop requests for vector-length agnostic code as it's implied.
-    } else if (!Val.equals("scalable"))
+    } else if (Val != "scalable")
       // Handle the unsupported values passed to msve-vector-bits.
       D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << Val;
@@ -214,7 +213,7 @@ void Flang::AddRISCVTargetArgs(const ArgList &Args,
     // If the value is "zvl", use MinVLen from march. Otherwise, try to parse
     // as integer as long as we have a MinVLen.
     unsigned Bits = 0;
-    if (Val.equals("zvl") && MinVLen >= llvm::RISCV::RVVBitsPerBlock) {
+    if (Val == "zvl" && MinVLen >= llvm::RISCV::RVVBitsPerBlock) {
       Bits = MinVLen;
     } else if (!Val.getAsInteger(10, Bits)) {
       // Only accept power of 2 values beteen RVVBitsPerBlock and 65536 that
@@ -231,7 +230,7 @@ void Flang::AddRISCVTargetArgs(const ArgList &Args,
           Args.MakeArgString("-mvscale-max=" + llvm::Twine(VScaleMin)));
       CmdArgs.push_back(
           Args.MakeArgString("-mvscale-min=" + llvm::Twine(VScaleMin)));
-    } else if (!Val.equals("scalable")) {
+    } else if (Val != "scalable") {
       // Handle the unsupported values passed to mrvv-vector-bits.
       D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << Val;
-- 
GitLab


From e7e13c6ffec58fe67a23d173387e96d5ebb8f84b Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Fri, 10 May 2024 16:09:34 +1000
Subject: [PATCH 0375/1206] [ORC] Fix another error fall-through in
 EPCGenericDylibManager::lookupAsync.

The early return added in this commit should have been added in cbf1535cc81.

No test-case: This would require a deliberately injected failure in a
remote-JIT test and we don't have the infrastructure for that at the moment.

rdar://126772381
---
 llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp b/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp
index 7c0d89012922..298bde46ab75 100644
--- a/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp
@@ -92,6 +92,7 @@ void EPCGenericDylibManager::lookupAsync(tpctypes::DylibHandle H,
         if (SerializationErr) {
           cantFail(Result.takeError());
           Complete(std::move(SerializationErr));
+          return;
         }
         Complete(std::move(Result));
       },
-- 
GitLab


From 0ebe48f068c0ca69f76ed68b621c9294acd75f76 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Fri, 10 May 2024 14:31:43 +0800
Subject: [PATCH 0376/1206] [RISCV] Move RISCVInsertVSETVLI after CSR/VXRM
 passes (#91701)

This further splits off #91440 to inch RISCVInsertVSETVLI closer to post
vector regalloc.

As noted in #91440, most of the diffs are from moving vsetvli insertion
after the vxrm/csr insertion passes, but these are getting conflated
with the changes from moving to LiveIntervals.

One idea was that we could try and remove some of these diffs by
manually moving back the vsetvlis past the vxrm/csr instructions. But
this meant having to touch up the LiveIntervals again which seemed to
lead to even more diffs.

This instead just moves RISCVInsertVSETVLI after RISCVInsertReadWriteCSR
and RISCVInsertWriteVXRM so we can isolate those changes.
---
 llvm/lib/Target/RISCV/RISCVTargetMachine.cpp  |   2 +-
 llvm/test/CodeGen/RISCV/O0-pipeline.ll        |   2 +-
 llvm/test/CodeGen/RISCV/O3-pipeline.ll        |   2 +-
 llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll        |  36 +-
 llvm/test/CodeGen/RISCV/rvv/commutable.ll     |  12 +-
 llvm/test/CodeGen/RISCV/rvv/ctlz-sdnode.ll    |  56 +--
 llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll        | 352 +++++++++---------
 llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll    |  32 +-
 .../CodeGen/RISCV/rvv/double-round-conv.ll    |  32 +-
 .../RISCV/rvv/fixed-vectors-ceil-vp.ll        |  38 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll   |   8 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-cttz.ll   |  16 +-
 .../RISCV/rvv/fixed-vectors-floor-vp.ll       |  38 +-
 .../RISCV/rvv/fixed-vectors-round-vp.ll       |  38 +-
 .../RISCV/rvv/fixed-vectors-roundeven-vp.ll   |  38 +-
 .../RISCV/rvv/fixed-vectors-roundtozero-vp.ll |  38 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-vaaddu.ll |  38 +-
 .../CodeGen/RISCV/rvv/float-round-conv.ll     |  48 +--
 llvm/test/CodeGen/RISCV/rvv/floor-vp.ll       |  36 +-
 llvm/test/CodeGen/RISCV/rvv/frm-insert.ll     |  84 ++---
 .../test/CodeGen/RISCV/rvv/half-round-conv.ll |  24 +-
 llvm/test/CodeGen/RISCV/rvv/masked-tama.ll    |   8 +-
 llvm/test/CodeGen/RISCV/rvv/masked-tamu.ll    |   8 +-
 llvm/test/CodeGen/RISCV/rvv/masked-tuma.ll    |   8 +-
 llvm/test/CodeGen/RISCV/rvv/masked-tumu.ll    |   8 +-
 llvm/test/CodeGen/RISCV/rvv/round-vp.ll       |  52 +--
 llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll   |  52 +--
 llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll |  52 +--
 .../rvv/rvv-peephole-vmerge-masked-vops.ll    |   4 +-
 .../RISCV/rvv/rvv-peephole-vmerge-vops.ll     |  14 +-
 .../CodeGen/RISCV/rvv/sf_vfnrclip_x_f_qf.ll   |  40 +-
 .../CodeGen/RISCV/rvv/sf_vfnrclip_xu_f_qf.ll  |  40 +-
 llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll    |  22 +-
 llvm/test/CodeGen/RISCV/rvv/vaadd.ll          | 176 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vaaddu-sdnode.ll  |  36 +-
 llvm/test/CodeGen/RISCV/rvv/vaaddu.ll         | 176 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vasub.ll          | 176 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vasubu.ll         | 176 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vfadd.ll          | 234 ++++++------
 llvm/test/CodeGen/RISCV/rvv/vfcvt-f-x.ll      | 120 +++---
 llvm/test/CodeGen/RISCV/rvv/vfcvt-f-xu.ll     | 120 +++---
 llvm/test/CodeGen/RISCV/rvv/vfcvt-x-f.ll      | 120 +++---
 llvm/test/CodeGen/RISCV/rvv/vfcvt-xu-f.ll     | 120 +++---
 llvm/test/CodeGen/RISCV/rvv/vfdiv.ll          | 234 ++++++------
 llvm/test/CodeGen/RISCV/rvv/vfmacc.ll         | 192 +++++-----
 llvm/test/CodeGen/RISCV/rvv/vfmadd.ll         | 192 +++++-----
 llvm/test/CodeGen/RISCV/rvv/vfmsac.ll         | 192 +++++-----
 llvm/test/CodeGen/RISCV/rvv/vfmsub.ll         | 192 +++++-----
 llvm/test/CodeGen/RISCV/rvv/vfmul.ll          | 234 ++++++------
 llvm/test/CodeGen/RISCV/rvv/vfncvt-f-f.ll     |  72 ++--
 llvm/test/CodeGen/RISCV/rvv/vfncvt-f-x.ll     |  72 ++--
 llvm/test/CodeGen/RISCV/rvv/vfncvt-f-xu.ll    |  72 ++--
 llvm/test/CodeGen/RISCV/rvv/vfncvt-x-f.ll     | 120 +++---
 llvm/test/CodeGen/RISCV/rvv/vfncvt-xu-f.ll    | 120 +++---
 llvm/test/CodeGen/RISCV/rvv/vfnmacc.ll        | 192 +++++-----
 llvm/test/CodeGen/RISCV/rvv/vfnmadd.ll        | 192 +++++-----
 llvm/test/CodeGen/RISCV/rvv/vfnmsac.ll        | 192 +++++-----
 llvm/test/CodeGen/RISCV/rvv/vfnmsub.ll        | 192 +++++-----
 llvm/test/CodeGen/RISCV/rvv/vfrdiv.ll         | 120 +++---
 llvm/test/CodeGen/RISCV/rvv/vfrec7.ll         | 120 +++---
 llvm/test/CodeGen/RISCV/rvv/vfredosum.ll      | 120 +++---
 llvm/test/CodeGen/RISCV/rvv/vfredusum.ll      | 120 +++---
 llvm/test/CodeGen/RISCV/rvv/vfsqrt.ll         | 120 +++---
 llvm/test/CodeGen/RISCV/rvv/vfwadd.ll         | 144 +++----
 llvm/test/CodeGen/RISCV/rvv/vfwadd.w.ll       | 244 ++++++------
 llvm/test/CodeGen/RISCV/rvv/vfwcvt-x-f.ll     |  72 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwcvt-xu-f.ll    |  72 ++--
 llvm/test/CodeGen/RISCV/rvv/vfwmacc.ll        | 144 +++----
 llvm/test/CodeGen/RISCV/rvv/vfwmsac.ll        | 144 +++----
 llvm/test/CodeGen/RISCV/rvv/vfwmul.ll         | 144 +++----
 llvm/test/CodeGen/RISCV/rvv/vfwnmacc.ll       | 144 +++----
 llvm/test/CodeGen/RISCV/rvv/vfwnmsac.ll       | 144 +++----
 llvm/test/CodeGen/RISCV/rvv/vfwredosum.ll     |  88 ++---
 llvm/test/CodeGen/RISCV/rvv/vfwredusum.ll     |  88 ++---
 llvm/test/CodeGen/RISCV/rvv/vfwsub.ll         | 144 +++----
 llvm/test/CodeGen/RISCV/rvv/vfwsub.w.ll       | 244 ++++++------
 llvm/test/CodeGen/RISCV/rvv/vnclip.ll         | 180 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vnclipu.ll        | 180 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vsmul-rv32.ll     | 160 ++++----
 llvm/test/CodeGen/RISCV/rvv/vsmul-rv64.ll     | 176 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vssra-rv32.ll     | 176 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vssra-rv64.ll     | 176 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vssrl-rv32.ll     | 176 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vssrl-rv64.ll     | 176 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vxrm-insert.ll    |  24 +-
 llvm/test/CodeGen/RISCV/rvv/vxrm.mir          |   6 +-
 86 files changed, 4519 insertions(+), 4519 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index b79568539046..7b2dcadc4191 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -541,9 +541,9 @@ void RISCVPassConfig::addPreRegAlloc() {
   addPass(createRISCVPreRAExpandPseudoPass());
   if (TM->getOptLevel() != CodeGenOptLevel::None)
     addPass(createRISCVMergeBaseOffsetOptPass());
-  addPass(createRISCVInsertVSETVLIPass());
   addPass(createRISCVInsertReadWriteCSRPass());
   addPass(createRISCVInsertWriteVXRMPass());
+  addPass(createRISCVInsertVSETVLIPass());
 }
 
 void RISCVPassConfig::addFastRegAlloc() {
diff --git a/llvm/test/CodeGen/RISCV/O0-pipeline.ll b/llvm/test/CodeGen/RISCV/O0-pipeline.ll
index 56bd4bd0c08f..c4a7f9562534 100644
--- a/llvm/test/CodeGen/RISCV/O0-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O0-pipeline.ll
@@ -40,9 +40,9 @@
 ; CHECK-NEXT:       Finalize ISel and expand pseudo-instructions
 ; CHECK-NEXT:       Local Stack Slot Allocation
 ; CHECK-NEXT:       RISC-V Pre-RA pseudo instruction expansion pass
-; CHECK-NEXT:       RISC-V Insert VSETVLI pass
 ; CHECK-NEXT:       RISC-V Insert Read/Write CSR Pass
 ; CHECK-NEXT:       RISC-V Insert Write VXRM Pass
+; CHECK-NEXT:       RISC-V Insert VSETVLI pass
 ; CHECK-NEXT:       Init Undef Pass
 ; CHECK-NEXT:       Eliminate PHI nodes for register allocation
 ; CHECK-NEXT:       Two-Address instruction pass
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index 04b055f9b216..4a71d3276d26 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -115,9 +115,9 @@
 ; RV64-NEXT:        RISC-V Optimize W Instructions
 ; CHECK-NEXT:       RISC-V Pre-RA pseudo instruction expansion pass
 ; CHECK-NEXT:       RISC-V Merge Base Offset
-; CHECK-NEXT:       RISC-V Insert VSETVLI pass
 ; CHECK-NEXT:       RISC-V Insert Read/Write CSR Pass
 ; CHECK-NEXT:       RISC-V Insert Write VXRM Pass
+; CHECK-NEXT:       RISC-V Insert VSETVLI pass
 ; CHECK-NEXT:       Detect Dead Lanes
 ; CHECK-NEXT:       Init Undef Pass
 ; CHECK-NEXT:       Process Implicit Definitions
diff --git a/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll b/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll
index 5b271606f08a..aa11e012af20 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ceil-vp.ll
@@ -15,8 +15,8 @@ define <vscale x 1 x half> @vp_ceil_vv_nxv1f16(<vscale x 1 x half> %va, <vscale
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -57,8 +57,8 @@ define <vscale x 2 x half> @vp_ceil_vv_nxv2f16(<vscale x 2 x half> %va, <vscale
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -99,8 +99,8 @@ define <vscale x 4 x half> @vp_ceil_vv_nxv4f16(<vscale x 4 x half> %va, <vscale
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -142,8 +142,8 @@ define <vscale x 8 x half> @vp_ceil_vv_nxv8f16(<vscale x 8 x half> %va, <vscale
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -186,8 +186,8 @@ define <vscale x 16 x half> @vp_ceil_vv_nxv16f16(<vscale x 16 x half> %va, <vsca
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -230,8 +230,8 @@ define <vscale x 32 x half> @vp_ceil_vv_nxv32f16(<vscale x 32 x half> %va, <vsca
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -273,8 +273,8 @@ define <vscale x 1 x float> @vp_ceil_vv_nxv1f32(<vscale x 1 x float> %va, <vscal
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -315,8 +315,8 @@ define <vscale x 2 x float> @vp_ceil_vv_nxv2f32(<vscale x 2 x float> %va, <vscal
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -358,8 +358,8 @@ define <vscale x 4 x float> @vp_ceil_vv_nxv4f32(<vscale x 4 x float> %va, <vscal
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -402,8 +402,8 @@ define <vscale x 8 x float> @vp_ceil_vv_nxv8f32(<vscale x 8 x float> %va, <vscal
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -446,8 +446,8 @@ define <vscale x 16 x float> @vp_ceil_vv_nxv16f32(<vscale x 16 x float> %va, <vs
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -489,8 +489,8 @@ define <vscale x 1 x double> @vp_ceil_vv_nxv1f64(<vscale x 1 x double> %va, <vsc
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -532,8 +532,8 @@ define <vscale x 2 x double> @vp_ceil_vv_nxv2f64(<vscale x 2 x double> %va, <vsc
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -576,8 +576,8 @@ define <vscale x 4 x double> @vp_ceil_vv_nxv4f64(<vscale x 4 x double> %va, <vsc
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -620,8 +620,8 @@ define <vscale x 7 x double> @vp_ceil_vv_nxv7f64(<vscale x 7 x double> %va, <vsc
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -664,8 +664,8 @@ define <vscale x 8 x double> @vp_ceil_vv_nxv8f64(<vscale x 8 x double> %va, <vsc
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -726,8 +726,8 @@ define <vscale x 16 x double> @vp_ceil_vv_nxv16f64(<vscale x 16 x double> %va, <
 ; CHECK-NEXT:    vfabs.v v8, v16, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v25, v8, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a2, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
 ; CHECK-NEXT:    fsrm a2
@@ -750,8 +750,8 @@ define <vscale x 16 x double> @vp_ceil_vv_nxv16f64(<vscale x 16 x double> %va, <
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v24, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/commutable.ll b/llvm/test/CodeGen/RISCV/rvv/commutable.ll
index d94b529bac01..5bca2eeb3fdd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/commutable.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/commutable.ll
@@ -720,8 +720,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vaadd.nxv1i64.nxv1i64(<vscale x 1 x i64>,
 define <vscale x 1 x i64> @commutable_vaadd_vv(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: commutable_vaadd_vv:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vadd.vv v8, v8, v8
@@ -737,8 +737,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vaadd.mask.nxv1i64.nxv1i64(<vscale x 1 x
 define <vscale x 1 x i64> @commutable_vaadd_vv_masked(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %mask, iXLen %2) {
 ; CHECK-LABEL: commutable_vaadd_vv_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vaadd.vv v10, v8, v9, v0.t
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
@@ -755,8 +755,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vaaddu.nxv1i64.nxv1i64(<vscale x 1 x i64>
 define <vscale x 1 x i64> @commutable_vaaddu_vv(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: commutable_vaaddu_vv:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vadd.vv v8, v8, v8
@@ -772,8 +772,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vaaddu.mask.nxv1i64.nxv1i64(<vscale x 1 x
 define <vscale x 1 x i64> @commutable_vaaddu_vv_masked(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %mask, iXLen %2) {
 ; CHECK-LABEL: commutable_vaaddu_vv_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v10, v8, v9, v0.t
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
@@ -790,8 +790,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vsmul.nxv1i64.nxv1i64(<vscale x 1 x i64>,
 define <vscale x 1 x i64> @commutable_vsmul_vv(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: commutable_vsmul_vv:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vadd.vv v8, v8, v8
@@ -807,8 +807,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vsmul.mask.nxv1i64.nxv1i64(<vscale x 1 x
 define <vscale x 1 x i64> @commutable_vsmul_vv_masked(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %mask, iXLen %2) {
 ; CHECK-LABEL: commutable_vsmul_vv_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vv v10, v8, v9, v0.t
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/ctlz-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/ctlz-sdnode.ll
index d756cfcf7077..41ec102c34ef 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ctlz-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ctlz-sdnode.ll
@@ -806,8 +806,8 @@ define <vscale x 1 x i32> @ctlz_nxv1i32(<vscale x 1 x i32> %va) {
 ;
 ; CHECK-F-LABEL: ctlz_nxv1i32:
 ; CHECK-F:       # %bb.0:
-; CHECK-F-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; CHECK-F-NEXT:    fsrmi a0, 1
+; CHECK-F-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; CHECK-F-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-F-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-F-NEXT:    li a1, 158
@@ -878,8 +878,8 @@ define <vscale x 2 x i32> @ctlz_nxv2i32(<vscale x 2 x i32> %va) {
 ;
 ; CHECK-F-LABEL: ctlz_nxv2i32:
 ; CHECK-F:       # %bb.0:
-; CHECK-F-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-F-NEXT:    fsrmi a0, 1
+; CHECK-F-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-F-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-F-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-F-NEXT:    li a1, 158
@@ -950,8 +950,8 @@ define <vscale x 4 x i32> @ctlz_nxv4i32(<vscale x 4 x i32> %va) {
 ;
 ; CHECK-F-LABEL: ctlz_nxv4i32:
 ; CHECK-F:       # %bb.0:
-; CHECK-F-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; CHECK-F-NEXT:    fsrmi a0, 1
+; CHECK-F-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; CHECK-F-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-F-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-F-NEXT:    li a1, 158
@@ -1022,8 +1022,8 @@ define <vscale x 8 x i32> @ctlz_nxv8i32(<vscale x 8 x i32> %va) {
 ;
 ; CHECK-F-LABEL: ctlz_nxv8i32:
 ; CHECK-F:       # %bb.0:
-; CHECK-F-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; CHECK-F-NEXT:    fsrmi a0, 1
+; CHECK-F-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; CHECK-F-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-F-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-F-NEXT:    li a1, 158
@@ -1094,8 +1094,8 @@ define <vscale x 16 x i32> @ctlz_nxv16i32(<vscale x 16 x i32> %va) {
 ;
 ; CHECK-F-LABEL: ctlz_nxv16i32:
 ; CHECK-F:       # %bb.0:
-; CHECK-F-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
 ; CHECK-F-NEXT:    fsrmi a0, 1
+; CHECK-F-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
 ; CHECK-F-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-F-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-F-NEXT:    li a1, 158
@@ -1107,8 +1107,8 @@ define <vscale x 16 x i32> @ctlz_nxv16i32(<vscale x 16 x i32> %va) {
 ;
 ; CHECK-D-LABEL: ctlz_nxv16i32:
 ; CHECK-D:       # %bb.0:
-; CHECK-D-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
 ; CHECK-D-NEXT:    fsrmi a0, 1
+; CHECK-D-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
 ; CHECK-D-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-D-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-D-NEXT:    li a1, 158
@@ -1234,8 +1234,8 @@ define <vscale x 1 x i64> @ctlz_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV32F-NEXT:    li a0, 190
 ; RV32F-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32F-NEXT:    vmv.v.x v9, a0
-; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v10, v8
 ; RV32F-NEXT:    vsrl.vi v8, v10, 23
 ; RV32F-NEXT:    vwsubu.wv v9, v9, v8
@@ -1262,8 +1262,8 @@ define <vscale x 1 x i64> @ctlz_nxv1i64(<vscale x 1 x i64> %va) {
 ;
 ; CHECK-D-LABEL: ctlz_nxv1i64:
 ; CHECK-D:       # %bb.0:
-; CHECK-D-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-D-NEXT:    fsrmi a0, 1
+; CHECK-D-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-D-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-D-NEXT:    li a1, 52
 ; CHECK-D-NEXT:    vsrl.vx v8, v8, a1
@@ -1390,8 +1390,8 @@ define <vscale x 2 x i64> @ctlz_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV32F-NEXT:    li a0, 190
 ; RV32F-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32F-NEXT:    vmv.v.x v10, a0
-; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v12, v8
 ; RV32F-NEXT:    vsrl.vi v8, v12, 23
 ; RV32F-NEXT:    vwsubu.wv v10, v10, v8
@@ -1418,8 +1418,8 @@ define <vscale x 2 x i64> @ctlz_nxv2i64(<vscale x 2 x i64> %va) {
 ;
 ; CHECK-D-LABEL: ctlz_nxv2i64:
 ; CHECK-D:       # %bb.0:
-; CHECK-D-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
 ; CHECK-D-NEXT:    fsrmi a0, 1
+; CHECK-D-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; CHECK-D-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-D-NEXT:    li a1, 52
 ; CHECK-D-NEXT:    vsrl.vx v8, v8, a1
@@ -1546,8 +1546,8 @@ define <vscale x 4 x i64> @ctlz_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV32F-NEXT:    li a0, 190
 ; RV32F-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32F-NEXT:    vmv.v.x v12, a0
-; RV32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v16, v8
 ; RV32F-NEXT:    vsrl.vi v8, v16, 23
 ; RV32F-NEXT:    vwsubu.wv v12, v12, v8
@@ -1574,8 +1574,8 @@ define <vscale x 4 x i64> @ctlz_nxv4i64(<vscale x 4 x i64> %va) {
 ;
 ; CHECK-D-LABEL: ctlz_nxv4i64:
 ; CHECK-D:       # %bb.0:
-; CHECK-D-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; CHECK-D-NEXT:    fsrmi a0, 1
+; CHECK-D-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-D-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-D-NEXT:    li a1, 52
 ; CHECK-D-NEXT:    vsrl.vx v8, v8, a1
@@ -1702,8 +1702,8 @@ define <vscale x 8 x i64> @ctlz_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV32F-NEXT:    li a0, 190
 ; RV32F-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32F-NEXT:    vmv.v.x v16, a0
-; RV32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v24, v8
 ; RV32F-NEXT:    vsrl.vi v8, v24, 23
 ; RV32F-NEXT:    vwsubu.wv v16, v16, v8
@@ -1730,8 +1730,8 @@ define <vscale x 8 x i64> @ctlz_nxv8i64(<vscale x 8 x i64> %va) {
 ;
 ; CHECK-D-LABEL: ctlz_nxv8i64:
 ; CHECK-D:       # %bb.0:
-; CHECK-D-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; CHECK-D-NEXT:    fsrmi a0, 1
+; CHECK-D-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; CHECK-D-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-D-NEXT:    li a1, 52
 ; CHECK-D-NEXT:    vsrl.vx v8, v8, a1
@@ -2497,8 +2497,8 @@ define <vscale x 1 x i32> @ctlz_zero_undef_nxv1i32(<vscale x 1 x i32> %va) {
 ;
 ; CHECK-F-LABEL: ctlz_zero_undef_nxv1i32:
 ; CHECK-F:       # %bb.0:
-; CHECK-F-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; CHECK-F-NEXT:    fsrmi a0, 1
+; CHECK-F-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; CHECK-F-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-F-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-F-NEXT:    li a1, 158
@@ -2564,8 +2564,8 @@ define <vscale x 2 x i32> @ctlz_zero_undef_nxv2i32(<vscale x 2 x i32> %va) {
 ;
 ; CHECK-F-LABEL: ctlz_zero_undef_nxv2i32:
 ; CHECK-F:       # %bb.0:
-; CHECK-F-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-F-NEXT:    fsrmi a0, 1
+; CHECK-F-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-F-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-F-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-F-NEXT:    li a1, 158
@@ -2631,8 +2631,8 @@ define <vscale x 4 x i32> @ctlz_zero_undef_nxv4i32(<vscale x 4 x i32> %va) {
 ;
 ; CHECK-F-LABEL: ctlz_zero_undef_nxv4i32:
 ; CHECK-F:       # %bb.0:
-; CHECK-F-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; CHECK-F-NEXT:    fsrmi a0, 1
+; CHECK-F-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; CHECK-F-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-F-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-F-NEXT:    li a1, 158
@@ -2698,8 +2698,8 @@ define <vscale x 8 x i32> @ctlz_zero_undef_nxv8i32(<vscale x 8 x i32> %va) {
 ;
 ; CHECK-F-LABEL: ctlz_zero_undef_nxv8i32:
 ; CHECK-F:       # %bb.0:
-; CHECK-F-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; CHECK-F-NEXT:    fsrmi a0, 1
+; CHECK-F-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; CHECK-F-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-F-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-F-NEXT:    li a1, 158
@@ -2765,8 +2765,8 @@ define <vscale x 16 x i32> @ctlz_zero_undef_nxv16i32(<vscale x 16 x i32> %va) {
 ;
 ; CHECK-F-LABEL: ctlz_zero_undef_nxv16i32:
 ; CHECK-F:       # %bb.0:
-; CHECK-F-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
 ; CHECK-F-NEXT:    fsrmi a0, 1
+; CHECK-F-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
 ; CHECK-F-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-F-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-F-NEXT:    li a1, 158
@@ -2776,8 +2776,8 @@ define <vscale x 16 x i32> @ctlz_zero_undef_nxv16i32(<vscale x 16 x i32> %va) {
 ;
 ; CHECK-D-LABEL: ctlz_zero_undef_nxv16i32:
 ; CHECK-D:       # %bb.0:
-; CHECK-D-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
 ; CHECK-D-NEXT:    fsrmi a0, 1
+; CHECK-D-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
 ; CHECK-D-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-D-NEXT:    vsrl.vi v8, v8, 23
 ; CHECK-D-NEXT:    li a1, 158
@@ -2900,8 +2900,8 @@ define <vscale x 1 x i64> @ctlz_zero_undef_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV32F-NEXT:    li a0, 190
 ; RV32F-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32F-NEXT:    vmv.v.x v9, a0
-; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v10, v8
 ; RV32F-NEXT:    vsrl.vi v8, v10, 23
 ; RV32F-NEXT:    vwsubu.wv v9, v9, v8
@@ -2923,8 +2923,8 @@ define <vscale x 1 x i64> @ctlz_zero_undef_nxv1i64(<vscale x 1 x i64> %va) {
 ;
 ; CHECK-D-LABEL: ctlz_zero_undef_nxv1i64:
 ; CHECK-D:       # %bb.0:
-; CHECK-D-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-D-NEXT:    fsrmi a0, 1
+; CHECK-D-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-D-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-D-NEXT:    li a1, 52
 ; CHECK-D-NEXT:    vsrl.vx v8, v8, a1
@@ -3048,8 +3048,8 @@ define <vscale x 2 x i64> @ctlz_zero_undef_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV32F-NEXT:    li a0, 190
 ; RV32F-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; RV32F-NEXT:    vmv.v.x v10, a0
-; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v12, v8
 ; RV32F-NEXT:    vsrl.vi v8, v12, 23
 ; RV32F-NEXT:    vwsubu.wv v10, v10, v8
@@ -3071,8 +3071,8 @@ define <vscale x 2 x i64> @ctlz_zero_undef_nxv2i64(<vscale x 2 x i64> %va) {
 ;
 ; CHECK-D-LABEL: ctlz_zero_undef_nxv2i64:
 ; CHECK-D:       # %bb.0:
-; CHECK-D-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
 ; CHECK-D-NEXT:    fsrmi a0, 1
+; CHECK-D-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; CHECK-D-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-D-NEXT:    li a1, 52
 ; CHECK-D-NEXT:    vsrl.vx v8, v8, a1
@@ -3196,8 +3196,8 @@ define <vscale x 4 x i64> @ctlz_zero_undef_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV32F-NEXT:    li a0, 190
 ; RV32F-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32F-NEXT:    vmv.v.x v12, a0
-; RV32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v16, v8
 ; RV32F-NEXT:    vsrl.vi v8, v16, 23
 ; RV32F-NEXT:    vwsubu.wv v12, v12, v8
@@ -3219,8 +3219,8 @@ define <vscale x 4 x i64> @ctlz_zero_undef_nxv4i64(<vscale x 4 x i64> %va) {
 ;
 ; CHECK-D-LABEL: ctlz_zero_undef_nxv4i64:
 ; CHECK-D:       # %bb.0:
-; CHECK-D-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; CHECK-D-NEXT:    fsrmi a0, 1
+; CHECK-D-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-D-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-D-NEXT:    li a1, 52
 ; CHECK-D-NEXT:    vsrl.vx v8, v8, a1
@@ -3345,8 +3345,8 @@ define <vscale x 8 x i64> @ctlz_zero_undef_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV32F-NEXT:    li a0, 190
 ; RV32F-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV32F-NEXT:    vmv.v.x v8, a0
-; RV32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v24, v16
 ; RV32F-NEXT:    vsrl.vi v16, v24, 23
 ; RV32F-NEXT:    vwsubu.wv v8, v8, v16
@@ -3367,8 +3367,8 @@ define <vscale x 8 x i64> @ctlz_zero_undef_nxv8i64(<vscale x 8 x i64> %va) {
 ;
 ; CHECK-D-LABEL: ctlz_zero_undef_nxv8i64:
 ; CHECK-D:       # %bb.0:
-; CHECK-D-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; CHECK-D-NEXT:    fsrmi a0, 1
+; CHECK-D-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; CHECK-D-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-D-NEXT:    li a1, 52
 ; CHECK-D-NEXT:    vsrl.vx v8, v8, a1
diff --git a/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
index 2a75e5ce7175..86086f5dc88f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
@@ -937,15 +937,15 @@ declare <vscale x 16 x i32> @llvm.vp.ctlz.nxv16i32(<vscale x 16 x i32>, i1 immar
 define <vscale x 16 x i32> @vp_ctlz_nxv16i32(<vscale x 16 x i32> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv16i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
 ; CHECK-NEXT:    vsrl.vi v8, v8, 23, v0.t
-; CHECK-NEXT:    li a1, 158
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 32
-; CHECK-NEXT:    vminu.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 158
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vminu.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv16i32:
@@ -960,15 +960,15 @@ define <vscale x 16 x i32> @vp_ctlz_nxv16i32(<vscale x 16 x i32> %va, <vscale x
 define <vscale x 16 x i32> @vp_ctlz_nxv16i32_unmasked(<vscale x 16 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv16i32_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-NEXT:    vsrl.vi v8, v8, 23
-; CHECK-NEXT:    li a1, 158
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 32
-; CHECK-NEXT:    vminu.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 158
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vminu.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv16i32_unmasked:
@@ -985,16 +985,16 @@ declare <vscale x 1 x i64> @llvm.vp.ctlz.nxv1i64(<vscale x 1 x i64>, i1 immarg,
 define <vscale x 1 x i64> @vp_ctlz_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv1i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    vminu.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vminu.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv1i64:
@@ -1009,16 +1009,16 @@ define <vscale x 1 x i64> @vp_ctlz_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x
 define <vscale x 1 x i64> @vp_ctlz_nxv1i64_unmasked(<vscale x 1 x i64> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv1i64_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    vminu.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vminu.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv1i64_unmasked:
@@ -1035,16 +1035,16 @@ declare <vscale x 2 x i64> @llvm.vp.ctlz.nxv2i64(<vscale x 2 x i64>, i1 immarg,
 define <vscale x 2 x i64> @vp_ctlz_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv2i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    vminu.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vminu.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv2i64:
@@ -1059,16 +1059,16 @@ define <vscale x 2 x i64> @vp_ctlz_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x
 define <vscale x 2 x i64> @vp_ctlz_nxv2i64_unmasked(<vscale x 2 x i64> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv2i64_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    vminu.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vminu.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv2i64_unmasked:
@@ -1085,16 +1085,16 @@ declare <vscale x 4 x i64> @llvm.vp.ctlz.nxv4i64(<vscale x 4 x i64>, i1 immarg,
 define <vscale x 4 x i64> @vp_ctlz_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv4i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    vminu.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vminu.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv4i64:
@@ -1109,16 +1109,16 @@ define <vscale x 4 x i64> @vp_ctlz_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4 x
 define <vscale x 4 x i64> @vp_ctlz_nxv4i64_unmasked(<vscale x 4 x i64> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv4i64_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    vminu.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vminu.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv4i64_unmasked:
@@ -1135,16 +1135,16 @@ declare <vscale x 7 x i64> @llvm.vp.ctlz.nxv7i64(<vscale x 7 x i64>, i1 immarg,
 define <vscale x 7 x i64> @vp_ctlz_nxv7i64(<vscale x 7 x i64> %va, <vscale x 7 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv7i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    vminu.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vminu.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv7i64:
@@ -1159,16 +1159,16 @@ define <vscale x 7 x i64> @vp_ctlz_nxv7i64(<vscale x 7 x i64> %va, <vscale x 7 x
 define <vscale x 7 x i64> @vp_ctlz_nxv7i64_unmasked(<vscale x 7 x i64> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv7i64_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    vminu.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vminu.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv7i64_unmasked:
@@ -1185,16 +1185,16 @@ declare <vscale x 8 x i64> @llvm.vp.ctlz.nxv8i64(<vscale x 8 x i64>, i1 immarg,
 define <vscale x 8 x i64> @vp_ctlz_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv8i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    vminu.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vminu.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv8i64:
@@ -1209,16 +1209,16 @@ define <vscale x 8 x i64> @vp_ctlz_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8 x
 define <vscale x 8 x i64> @vp_ctlz_nxv8i64_unmasked(<vscale x 8 x i64> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_nxv8i64_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 64
-; CHECK-NEXT:    vminu.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vminu.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv8i64_unmasked:
@@ -1244,10 +1244,10 @@ define <vscale x 16 x i64> @vp_ctlz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
 ; CHECK-NEXT:    and a2, a3, a2
+; CHECK-NEXT:    fsrmi a3, 1
 ; CHECK-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a2, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v16, v16, v0.t
-; CHECK-NEXT:    fsrm a2
+; CHECK-NEXT:    fsrm a3
 ; CHECK-NEXT:    li a2, 52
 ; CHECK-NEXT:    vsrl.vx v16, v16, a2, v0.t
 ; CHECK-NEXT:    li a3, 1086
@@ -1258,14 +1258,14 @@ define <vscale x 16 x i64> @vp_ctlz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:  .LBB46_2:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
 ; CHECK-NEXT:    vsrl.vx v8, v8, a2, v0.t
 ; CHECK-NEXT:    vrsub.vx v8, v8, a3, v0.t
 ; CHECK-NEXT:    vminu.vx v8, v8, a4, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv16i64:
@@ -1301,10 +1301,10 @@ define <vscale x 16 x i64> @vp_ctlz_nxv16i64_unmasked(<vscale x 16 x i64> %va, i
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
 ; CHECK-NEXT:    and a2, a3, a2
+; CHECK-NEXT:    fsrmi a3, 1
 ; CHECK-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a2, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v16, v16
-; CHECK-NEXT:    fsrm a2
+; CHECK-NEXT:    fsrm a3
 ; CHECK-NEXT:    li a2, 52
 ; CHECK-NEXT:    vsrl.vx v16, v16, a2
 ; CHECK-NEXT:    li a3, 1086
@@ -1315,13 +1315,13 @@ define <vscale x 16 x i64> @vp_ctlz_nxv16i64_unmasked(<vscale x 16 x i64> %va, i
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:  .LBB47_2:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-NEXT:    vsrl.vx v8, v8, a2
 ; CHECK-NEXT:    vrsub.vx v8, v8, a3
 ; CHECK-NEXT:    vminu.vx v8, v8, a4
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv16i64_unmasked:
@@ -2198,13 +2198,13 @@ define <vscale x 8 x i32> @vp_ctlz_zero_undef_nxv8i32_unmasked(<vscale x 8 x i32
 define <vscale x 16 x i32> @vp_ctlz_zero_undef_nxv16i32(<vscale x 16 x i32> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv16i32:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
 ; CHECK-NEXT:    vsrl.vi v8, v8, 23, v0.t
-; CHECK-NEXT:    li a1, 158
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 158
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv16i32:
@@ -2219,13 +2219,13 @@ define <vscale x 16 x i32> @vp_ctlz_zero_undef_nxv16i32(<vscale x 16 x i32> %va,
 define <vscale x 16 x i32> @vp_ctlz_zero_undef_nxv16i32_unmasked(<vscale x 16 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv16i32_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-NEXT:    vsrl.vi v8, v8, 23
-; CHECK-NEXT:    li a1, 158
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 158
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv16i32_unmasked:
@@ -2241,14 +2241,14 @@ define <vscale x 16 x i32> @vp_ctlz_zero_undef_nxv16i32_unmasked(<vscale x 16 x
 define <vscale x 1 x i64> @vp_ctlz_zero_undef_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv1i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv1i64:
@@ -2263,14 +2263,14 @@ define <vscale x 1 x i64> @vp_ctlz_zero_undef_nxv1i64(<vscale x 1 x i64> %va, <v
 define <vscale x 1 x i64> @vp_ctlz_zero_undef_nxv1i64_unmasked(<vscale x 1 x i64> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv1i64_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv1i64_unmasked:
@@ -2286,14 +2286,14 @@ define <vscale x 1 x i64> @vp_ctlz_zero_undef_nxv1i64_unmasked(<vscale x 1 x i64
 define <vscale x 2 x i64> @vp_ctlz_zero_undef_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv2i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv2i64:
@@ -2308,14 +2308,14 @@ define <vscale x 2 x i64> @vp_ctlz_zero_undef_nxv2i64(<vscale x 2 x i64> %va, <v
 define <vscale x 2 x i64> @vp_ctlz_zero_undef_nxv2i64_unmasked(<vscale x 2 x i64> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv2i64_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv2i64_unmasked:
@@ -2331,14 +2331,14 @@ define <vscale x 2 x i64> @vp_ctlz_zero_undef_nxv2i64_unmasked(<vscale x 2 x i64
 define <vscale x 4 x i64> @vp_ctlz_zero_undef_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv4i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv4i64:
@@ -2353,14 +2353,14 @@ define <vscale x 4 x i64> @vp_ctlz_zero_undef_nxv4i64(<vscale x 4 x i64> %va, <v
 define <vscale x 4 x i64> @vp_ctlz_zero_undef_nxv4i64_unmasked(<vscale x 4 x i64> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv4i64_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv4i64_unmasked:
@@ -2376,14 +2376,14 @@ define <vscale x 4 x i64> @vp_ctlz_zero_undef_nxv4i64_unmasked(<vscale x 4 x i64
 define <vscale x 7 x i64> @vp_ctlz_zero_undef_nxv7i64(<vscale x 7 x i64> %va, <vscale x 7 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv7i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv7i64:
@@ -2398,14 +2398,14 @@ define <vscale x 7 x i64> @vp_ctlz_zero_undef_nxv7i64(<vscale x 7 x i64> %va, <v
 define <vscale x 7 x i64> @vp_ctlz_zero_undef_nxv7i64_unmasked(<vscale x 7 x i64> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv7i64_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv7i64_unmasked:
@@ -2421,14 +2421,14 @@ define <vscale x 7 x i64> @vp_ctlz_zero_undef_nxv7i64_unmasked(<vscale x 7 x i64
 define <vscale x 8 x i64> @vp_ctlz_zero_undef_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv8i64:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv8i64:
@@ -2443,14 +2443,14 @@ define <vscale x 8 x i64> @vp_ctlz_zero_undef_nxv8i64(<vscale x 8 x i64> %va, <v
 define <vscale x 8 x i64> @vp_ctlz_zero_undef_nxv8i64_unmasked(<vscale x 8 x i64> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_ctlz_zero_undef_nxv8i64_unmasked:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    li a1, 52
-; CHECK-NEXT:    vsrl.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 1086
-; CHECK-NEXT:    vrsub.vx v8, v8, a1
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    li a0, 52
+; CHECK-NEXT:    vsrl.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 1086
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv8i64_unmasked:
@@ -2474,10 +2474,10 @@ define <vscale x 16 x i64> @vp_ctlz_zero_undef_nxv16i64(<vscale x 16 x i64> %va,
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
 ; CHECK-NEXT:    and a2, a3, a2
+; CHECK-NEXT:    fsrmi a3, 1
 ; CHECK-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a2, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v16, v16, v0.t
-; CHECK-NEXT:    fsrm a2
+; CHECK-NEXT:    fsrm a3
 ; CHECK-NEXT:    li a2, 52
 ; CHECK-NEXT:    vsrl.vx v16, v16, a2, v0.t
 ; CHECK-NEXT:    li a3, 1086
@@ -2486,13 +2486,13 @@ define <vscale x 16 x i64> @vp_ctlz_zero_undef_nxv16i64(<vscale x 16 x i64> %va,
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:  .LBB94_2:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8, v0.t
 ; CHECK-NEXT:    vsrl.vx v8, v8, a2, v0.t
 ; CHECK-NEXT:    vrsub.vx v8, v8, a3, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv16i64:
@@ -2528,10 +2528,10 @@ define <vscale x 16 x i64> @vp_ctlz_zero_undef_nxv16i64_unmasked(<vscale x 16 x
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
 ; CHECK-NEXT:    and a2, a3, a2
+; CHECK-NEXT:    fsrmi a3, 1
 ; CHECK-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a2, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v16, v16
-; CHECK-NEXT:    fsrm a2
+; CHECK-NEXT:    fsrm a3
 ; CHECK-NEXT:    li a2, 52
 ; CHECK-NEXT:    vsrl.vx v16, v16, a2
 ; CHECK-NEXT:    li a3, 1086
@@ -2540,12 +2540,12 @@ define <vscale x 16 x i64> @vp_ctlz_zero_undef_nxv16i64_unmasked(<vscale x 16 x
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:  .LBB95_2:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
 ; CHECK-NEXT:    vsrl.vx v8, v8, a2
 ; CHECK-NEXT:    vrsub.vx v8, v8, a3
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_zero_undef_nxv16i64_unmasked:
diff --git a/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll
index d13f4d2dca1f..479664c6f5f6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll
@@ -1219,8 +1219,8 @@ define <vscale x 1 x i64> @cttz_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV32F-NEXT:    vmseq.vx v0, v8, zero
 ; RV32F-NEXT:    vrsub.vi v9, v8, 0
 ; RV32F-NEXT:    vand.vv v8, v8, v9
-; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v9, v8
 ; RV32F-NEXT:    vsrl.vi v8, v9, 23
 ; RV32F-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
@@ -1237,8 +1237,8 @@ define <vscale x 1 x i64> @cttz_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64F-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; RV64F-NEXT:    vrsub.vi v9, v8, 0
 ; RV64F-NEXT:    vand.vv v9, v8, v9
-; RV64F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV64F-NEXT:    fsrmi a0, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v10, v9
 ; RV64F-NEXT:    vsrl.vi v9, v10, 23
 ; RV64F-NEXT:    li a1, 127
@@ -1381,8 +1381,8 @@ define <vscale x 2 x i64> @cttz_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV32F-NEXT:    vmseq.vx v0, v8, zero
 ; RV32F-NEXT:    vrsub.vi v10, v8, 0
 ; RV32F-NEXT:    vand.vv v8, v8, v10
-; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v10, v8
 ; RV32F-NEXT:    vsrl.vi v8, v10, 23
 ; RV32F-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
@@ -1399,8 +1399,8 @@ define <vscale x 2 x i64> @cttz_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64F-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
 ; RV64F-NEXT:    vrsub.vi v10, v8, 0
 ; RV64F-NEXT:    vand.vv v10, v8, v10
-; RV64F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64F-NEXT:    fsrmi a0, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v12, v10
 ; RV64F-NEXT:    vsrl.vi v10, v12, 23
 ; RV64F-NEXT:    li a1, 127
@@ -1543,8 +1543,8 @@ define <vscale x 4 x i64> @cttz_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV32F-NEXT:    vmseq.vx v0, v8, zero
 ; RV32F-NEXT:    vrsub.vi v12, v8, 0
 ; RV32F-NEXT:    vand.vv v8, v8, v12
-; RV32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v12, v8
 ; RV32F-NEXT:    vsrl.vi v8, v12, 23
 ; RV32F-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
@@ -1561,8 +1561,8 @@ define <vscale x 4 x i64> @cttz_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64F-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; RV64F-NEXT:    vrsub.vi v12, v8, 0
 ; RV64F-NEXT:    vand.vv v12, v8, v12
-; RV64F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV64F-NEXT:    fsrmi a0, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v16, v12
 ; RV64F-NEXT:    vsrl.vi v12, v16, 23
 ; RV64F-NEXT:    li a1, 127
@@ -1705,8 +1705,8 @@ define <vscale x 8 x i64> @cttz_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV32F-NEXT:    vmseq.vx v0, v8, zero
 ; RV32F-NEXT:    vrsub.vi v16, v8, 0
 ; RV32F-NEXT:    vand.vv v8, v8, v16
-; RV32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v16, v8
 ; RV32F-NEXT:    vsrl.vi v8, v16, 23
 ; RV32F-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
@@ -1723,8 +1723,8 @@ define <vscale x 8 x i64> @cttz_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64F-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; RV64F-NEXT:    vrsub.vi v16, v8, 0
 ; RV64F-NEXT:    vand.vv v16, v8, v16
-; RV64F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV64F-NEXT:    fsrmi a0, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v24, v16
 ; RV64F-NEXT:    vsrl.vi v16, v24, 23
 ; RV64F-NEXT:    li a1, 127
@@ -2892,8 +2892,8 @@ define <vscale x 1 x i64> @cttz_zero_undef_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV32F-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; RV32F-NEXT:    vrsub.vi v9, v8, 0
 ; RV32F-NEXT:    vand.vv v8, v8, v9
-; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v9, v8
 ; RV32F-NEXT:    vsrl.vi v8, v9, 23
 ; RV32F-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
@@ -2908,8 +2908,8 @@ define <vscale x 1 x i64> @cttz_zero_undef_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64F-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; RV64F-NEXT:    vrsub.vi v9, v8, 0
 ; RV64F-NEXT:    vand.vv v8, v8, v9
-; RV64F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV64F-NEXT:    fsrmi a0, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v9, v8
 ; RV64F-NEXT:    vsrl.vi v9, v9, 23
 ; RV64F-NEXT:    li a1, 127
@@ -3026,8 +3026,8 @@ define <vscale x 2 x i64> @cttz_zero_undef_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV32F-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
 ; RV32F-NEXT:    vrsub.vi v10, v8, 0
 ; RV32F-NEXT:    vand.vv v8, v8, v10
-; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v10, v8
 ; RV32F-NEXT:    vsrl.vi v8, v10, 23
 ; RV32F-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
@@ -3042,8 +3042,8 @@ define <vscale x 2 x i64> @cttz_zero_undef_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64F-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
 ; RV64F-NEXT:    vrsub.vi v10, v8, 0
 ; RV64F-NEXT:    vand.vv v8, v8, v10
-; RV64F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64F-NEXT:    fsrmi a0, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v10, v8
 ; RV64F-NEXT:    vsrl.vi v10, v10, 23
 ; RV64F-NEXT:    li a1, 127
@@ -3160,8 +3160,8 @@ define <vscale x 4 x i64> @cttz_zero_undef_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV32F-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; RV32F-NEXT:    vrsub.vi v12, v8, 0
 ; RV32F-NEXT:    vand.vv v8, v8, v12
-; RV32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v12, v8
 ; RV32F-NEXT:    vsrl.vi v8, v12, 23
 ; RV32F-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
@@ -3176,8 +3176,8 @@ define <vscale x 4 x i64> @cttz_zero_undef_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64F-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; RV64F-NEXT:    vrsub.vi v12, v8, 0
 ; RV64F-NEXT:    vand.vv v8, v8, v12
-; RV64F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV64F-NEXT:    fsrmi a0, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v12, v8
 ; RV64F-NEXT:    vsrl.vi v12, v12, 23
 ; RV64F-NEXT:    li a1, 127
@@ -3294,8 +3294,8 @@ define <vscale x 8 x i64> @cttz_zero_undef_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV32F-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; RV32F-NEXT:    vrsub.vi v16, v8, 0
 ; RV32F-NEXT:    vand.vv v8, v8, v16
-; RV32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32F-NEXT:    fsrmi a0, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v16, v8
 ; RV32F-NEXT:    vsrl.vi v8, v16, 23
 ; RV32F-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
@@ -3310,8 +3310,8 @@ define <vscale x 8 x i64> @cttz_zero_undef_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64F-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; RV64F-NEXT:    vrsub.vi v16, v8, 0
 ; RV64F-NEXT:    vand.vv v8, v8, v16
-; RV64F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV64F-NEXT:    fsrmi a0, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v16, v8
 ; RV64F-NEXT:    vsrl.vi v16, v16, 23
 ; RV64F-NEXT:    li a1, 127
diff --git a/llvm/test/CodeGen/RISCV/rvv/double-round-conv.ll b/llvm/test/CodeGen/RISCV/rvv/double-round-conv.ll
index ee9ad097b442..8c63c2d4be8c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/double-round-conv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/double-round-conv.ll
@@ -677,8 +677,8 @@ define <vscale x 1 x i16> @ceil_nxv1f64_to_ui16(<vscale x 1 x double> %x) {
 define <vscale x 1 x i32> @ceil_nxv1f64_to_si32(<vscale x 1 x double> %x) {
 ; RV32-LABEL: ceil_nxv1f64_to_si32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    vfncvt.x.f.w v9, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv1r.v v8, v9
@@ -686,8 +686,8 @@ define <vscale x 1 x i32> @ceil_nxv1f64_to_si32(<vscale x 1 x double> %x) {
 ;
 ; RV64-LABEL: ceil_nxv1f64_to_si32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    vfncvt.x.f.w v9, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv1r.v v8, v9
@@ -700,8 +700,8 @@ define <vscale x 1 x i32> @ceil_nxv1f64_to_si32(<vscale x 1 x double> %x) {
 define <vscale x 1 x i32> @ceil_nxv1f64_to_ui32(<vscale x 1 x double> %x) {
 ; RV32-LABEL: ceil_nxv1f64_to_ui32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    vfncvt.xu.f.w v9, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv1r.v v8, v9
@@ -709,8 +709,8 @@ define <vscale x 1 x i32> @ceil_nxv1f64_to_ui32(<vscale x 1 x double> %x) {
 ;
 ; RV64-LABEL: ceil_nxv1f64_to_ui32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    vfncvt.xu.f.w v9, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv1r.v v8, v9
@@ -723,16 +723,16 @@ define <vscale x 1 x i32> @ceil_nxv1f64_to_ui32(<vscale x 1 x double> %x) {
 define <vscale x 1 x i64> @ceil_nxv1f64_to_si64(<vscale x 1 x double> %x) {
 ; RV32-LABEL: ceil_nxv1f64_to_si64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ceil_nxv1f64_to_si64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    ret
@@ -744,16 +744,16 @@ define <vscale x 1 x i64> @ceil_nxv1f64_to_si64(<vscale x 1 x double> %x) {
 define <vscale x 1 x i64> @ceil_nxv1f64_to_ui64(<vscale x 1 x double> %x) {
 ; RV32-LABEL: ceil_nxv1f64_to_ui64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV32-NEXT:    vfcvt.xu.f.v v8, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ceil_nxv1f64_to_ui64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; RV64-NEXT:    vfcvt.xu.f.v v8, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    ret
@@ -951,8 +951,8 @@ define <vscale x 4 x i16> @ceil_nxv4f64_to_ui16(<vscale x 4 x double> %x) {
 define <vscale x 4 x i32> @ceil_nxv4f64_to_si32(<vscale x 4 x double> %x) {
 ; RV32-LABEL: ceil_nxv4f64_to_si32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV32-NEXT:    vfncvt.x.f.w v12, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv.v.v v8, v12
@@ -960,8 +960,8 @@ define <vscale x 4 x i32> @ceil_nxv4f64_to_si32(<vscale x 4 x double> %x) {
 ;
 ; RV64-LABEL: ceil_nxv4f64_to_si32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV64-NEXT:    vfncvt.x.f.w v12, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv.v.v v8, v12
@@ -974,8 +974,8 @@ define <vscale x 4 x i32> @ceil_nxv4f64_to_si32(<vscale x 4 x double> %x) {
 define <vscale x 4 x i32> @ceil_nxv4f64_to_ui32(<vscale x 4 x double> %x) {
 ; RV32-LABEL: ceil_nxv4f64_to_ui32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV32-NEXT:    vfncvt.xu.f.w v12, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv.v.v v8, v12
@@ -983,8 +983,8 @@ define <vscale x 4 x i32> @ceil_nxv4f64_to_ui32(<vscale x 4 x double> %x) {
 ;
 ; RV64-LABEL: ceil_nxv4f64_to_ui32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV64-NEXT:    vfncvt.xu.f.w v12, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv.v.v v8, v12
@@ -997,16 +997,16 @@ define <vscale x 4 x i32> @ceil_nxv4f64_to_ui32(<vscale x 4 x double> %x) {
 define <vscale x 4 x i64> @ceil_nxv4f64_to_si64(<vscale x 4 x double> %x) {
 ; RV32-LABEL: ceil_nxv4f64_to_si64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ceil_nxv4f64_to_si64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    ret
@@ -1018,16 +1018,16 @@ define <vscale x 4 x i64> @ceil_nxv4f64_to_si64(<vscale x 4 x double> %x) {
 define <vscale x 4 x i64> @ceil_nxv4f64_to_ui64(<vscale x 4 x double> %x) {
 ; RV32-LABEL: ceil_nxv4f64_to_ui64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV32-NEXT:    vfcvt.xu.f.v v8, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ceil_nxv4f64_to_ui64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; RV64-NEXT:    vfcvt.xu.f.v v8, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll
index 5d024f140fd5..3e2af7e8267b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ceil-vp.ll
@@ -19,8 +19,8 @@ define <2 x half> @vp_ceil_v2f16(<2 x half> %va, <2 x i1> %m, i32 zeroext %evl)
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 3
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -38,8 +38,8 @@ define <2 x half> @vp_ceil_v2f16(<2 x half> %va, <2 x i1> %m, i32 zeroext %evl)
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 3
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -101,8 +101,8 @@ define <4 x half> @vp_ceil_v4f16(<4 x half> %va, <4 x i1> %m, i32 zeroext %evl)
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 3
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -120,8 +120,8 @@ define <4 x half> @vp_ceil_v4f16(<4 x half> %va, <4 x i1> %m, i32 zeroext %evl)
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 3
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -183,8 +183,8 @@ define <8 x half> @vp_ceil_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %evl)
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 3
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -203,8 +203,8 @@ define <8 x half> @vp_ceil_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %evl)
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 3
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v9
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v12, v10, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -268,8 +268,8 @@ define <16 x half> @vp_ceil_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %e
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 3
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v10
 ; ZVFH-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -289,8 +289,8 @@ define <16 x half> @vp_ceil_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %e
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v10, v16, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 3
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v10
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v16, v12, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -353,8 +353,8 @@ define <2 x float> @vp_ceil_v2f32(<2 x float> %va, <2 x i1> %m, i32 zeroext %evl
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -395,8 +395,8 @@ define <4 x float> @vp_ceil_v4f32(<4 x float> %va, <4 x i1> %m, i32 zeroext %evl
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -438,8 +438,8 @@ define <8 x float> @vp_ceil_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext %evl
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -482,8 +482,8 @@ define <16 x float> @vp_ceil_v16f32(<16 x float> %va, <16 x i1> %m, i32 zeroext
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -525,8 +525,8 @@ define <2 x double> @vp_ceil_v2f64(<2 x double> %va, <2 x i1> %m, i32 zeroext %e
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -568,8 +568,8 @@ define <4 x double> @vp_ceil_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroext %e
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -612,8 +612,8 @@ define <8 x double> @vp_ceil_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroext %e
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -656,8 +656,8 @@ define <15 x double> @vp_ceil_v15f64(<15 x double> %va, <15 x i1> %m, i32 zeroex
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -700,8 +700,8 @@ define <16 x double> @vp_ceil_v16f64(<16 x double> %va, <16 x i1> %m, i32 zeroex
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -764,8 +764,8 @@ define <32 x double> @vp_ceil_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroex
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v25, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a1, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a1
@@ -788,8 +788,8 @@ define <32 x double> @vp_ceil_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroex
 ; CHECK-NEXT:    vfabs.v v8, v16, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v24, v8, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
 ; CHECK-NEXT:    fsrm a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll
index 277146cc1403..49e5a1c79c43 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz.ll
@@ -353,8 +353,8 @@ define void @ctlz_v2i64(ptr %x, ptr %y) nounwind {
 ; RV32F-NEXT:    vle64.v v8, (a0)
 ; RV32F-NEXT:    li a1, 190
 ; RV32F-NEXT:    vmv.v.x v9, a1
-; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    fsrmi a1, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v10, v8
 ; RV32F-NEXT:    fsrm a1
 ; RV32F-NEXT:    vsrl.vi v8, v10, 23
@@ -762,8 +762,8 @@ define void @ctlz_v4i64(ptr %x, ptr %y) nounwind {
 ; RV32F-NEXT:    vle64.v v8, (a0)
 ; RV32F-NEXT:    li a1, 190
 ; RV32F-NEXT:    vmv.v.x v10, a1
-; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    fsrmi a1, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v12, v8
 ; RV32F-NEXT:    fsrm a1
 ; RV32F-NEXT:    vsrl.vi v8, v12, 23
@@ -1152,8 +1152,8 @@ define void @ctlz_zero_undef_v2i64(ptr %x, ptr %y) nounwind {
 ; RV32F-NEXT:    vle64.v v8, (a0)
 ; RV32F-NEXT:    li a1, 190
 ; RV32F-NEXT:    vmv.v.x v9, a1
-; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    fsrmi a1, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v10, v8
 ; RV32F-NEXT:    fsrm a1
 ; RV32F-NEXT:    vsrl.vi v8, v10, 23
@@ -1537,8 +1537,8 @@ define void @ctlz_zero_undef_v4i64(ptr %x, ptr %y) nounwind {
 ; RV32F-NEXT:    vle64.v v8, (a0)
 ; RV32F-NEXT:    li a1, 190
 ; RV32F-NEXT:    vmv.v.x v10, a1
-; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    fsrmi a1, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v12, v8
 ; RV32F-NEXT:    fsrm a1
 ; RV32F-NEXT:    vsrl.vi v8, v12, 23
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
index 8c8da6d1e003..ea3a78ae0bec 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
@@ -336,8 +336,8 @@ define void @cttz_v2i64(ptr %x, ptr %y) nounwind {
 ; RV32F-NEXT:    vle64.v v8, (a0)
 ; RV32F-NEXT:    vrsub.vi v9, v8, 0
 ; RV32F-NEXT:    vand.vv v9, v8, v9
-; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    fsrmi a1, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v10, v9
 ; RV32F-NEXT:    fsrm a1
 ; RV32F-NEXT:    vsrl.vi v9, v10, 23
@@ -357,8 +357,8 @@ define void @cttz_v2i64(ptr %x, ptr %y) nounwind {
 ; RV64F-NEXT:    vle64.v v8, (a0)
 ; RV64F-NEXT:    vrsub.vi v9, v8, 0
 ; RV64F-NEXT:    vand.vv v9, v8, v9
-; RV64F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV64F-NEXT:    fsrmi a1, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v10, v9
 ; RV64F-NEXT:    fsrm a1
 ; RV64F-NEXT:    vsrl.vi v9, v10, 23
@@ -737,8 +737,8 @@ define void @cttz_v4i64(ptr %x, ptr %y) nounwind {
 ; RV32F-NEXT:    vle64.v v8, (a0)
 ; RV32F-NEXT:    vrsub.vi v10, v8, 0
 ; RV32F-NEXT:    vand.vv v10, v8, v10
-; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    fsrmi a1, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v12, v10
 ; RV32F-NEXT:    fsrm a1
 ; RV32F-NEXT:    vsrl.vi v10, v12, 23
@@ -758,8 +758,8 @@ define void @cttz_v4i64(ptr %x, ptr %y) nounwind {
 ; RV64F-NEXT:    vle64.v v8, (a0)
 ; RV64F-NEXT:    vrsub.vi v10, v8, 0
 ; RV64F-NEXT:    vand.vv v10, v8, v10
-; RV64F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64F-NEXT:    fsrmi a1, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v12, v10
 ; RV64F-NEXT:    fsrm a1
 ; RV64F-NEXT:    vsrl.vi v10, v12, 23
@@ -1115,8 +1115,8 @@ define void @cttz_zero_undef_v2i64(ptr %x, ptr %y) nounwind {
 ; RV32F-NEXT:    vle64.v v8, (a0)
 ; RV32F-NEXT:    vrsub.vi v9, v8, 0
 ; RV32F-NEXT:    vand.vv v8, v8, v9
-; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    fsrmi a1, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v9, v8
 ; RV32F-NEXT:    fsrm a1
 ; RV32F-NEXT:    vsrl.vi v8, v9, 23
@@ -1133,8 +1133,8 @@ define void @cttz_zero_undef_v2i64(ptr %x, ptr %y) nounwind {
 ; RV64F-NEXT:    vle64.v v8, (a0)
 ; RV64F-NEXT:    vrsub.vi v9, v8, 0
 ; RV64F-NEXT:    vand.vv v8, v8, v9
-; RV64F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV64F-NEXT:    fsrmi a1, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v9, v8
 ; RV64F-NEXT:    fsrm a1
 ; RV64F-NEXT:    vsrl.vi v8, v9, 23
@@ -1486,8 +1486,8 @@ define void @cttz_zero_undef_v4i64(ptr %x, ptr %y) nounwind {
 ; RV32F-NEXT:    vle64.v v8, (a0)
 ; RV32F-NEXT:    vrsub.vi v10, v8, 0
 ; RV32F-NEXT:    vand.vv v8, v8, v10
-; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    fsrmi a1, 1
+; RV32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32F-NEXT:    vfncvt.f.xu.w v10, v8
 ; RV32F-NEXT:    fsrm a1
 ; RV32F-NEXT:    vsrl.vi v8, v10, 23
@@ -1504,8 +1504,8 @@ define void @cttz_zero_undef_v4i64(ptr %x, ptr %y) nounwind {
 ; RV64F-NEXT:    vle64.v v8, (a0)
 ; RV64F-NEXT:    vrsub.vi v10, v8, 0
 ; RV64F-NEXT:    vand.vv v8, v8, v10
-; RV64F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64F-NEXT:    fsrmi a1, 1
+; RV64F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64F-NEXT:    vfncvt.f.xu.w v10, v8
 ; RV64F-NEXT:    fsrm a1
 ; RV64F-NEXT:    vsrl.vi v8, v10, 23
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll
index 6c2be509f7c2..287dd510674d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-floor-vp.ll
@@ -19,8 +19,8 @@ define <2 x half> @vp_floor_v2f16(<2 x half> %va, <2 x i1> %m, i32 zeroext %evl)
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 2
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -38,8 +38,8 @@ define <2 x half> @vp_floor_v2f16(<2 x half> %va, <2 x i1> %m, i32 zeroext %evl)
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 2
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -101,8 +101,8 @@ define <4 x half> @vp_floor_v4f16(<4 x half> %va, <4 x i1> %m, i32 zeroext %evl)
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 2
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -120,8 +120,8 @@ define <4 x half> @vp_floor_v4f16(<4 x half> %va, <4 x i1> %m, i32 zeroext %evl)
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 2
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -183,8 +183,8 @@ define <8 x half> @vp_floor_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %evl)
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 2
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -203,8 +203,8 @@ define <8 x half> @vp_floor_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %evl)
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 2
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v9
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v12, v10, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -268,8 +268,8 @@ define <16 x half> @vp_floor_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 2
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v10
 ; ZVFH-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -289,8 +289,8 @@ define <16 x half> @vp_floor_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v10, v16, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 2
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v10
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v16, v12, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -353,8 +353,8 @@ define <2 x float> @vp_floor_v2f32(<2 x float> %va, <2 x i1> %m, i32 zeroext %ev
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -395,8 +395,8 @@ define <4 x float> @vp_floor_v4f32(<4 x float> %va, <4 x i1> %m, i32 zeroext %ev
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -438,8 +438,8 @@ define <8 x float> @vp_floor_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext %ev
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -482,8 +482,8 @@ define <16 x float> @vp_floor_v16f32(<16 x float> %va, <16 x i1> %m, i32 zeroext
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -525,8 +525,8 @@ define <2 x double> @vp_floor_v2f64(<2 x double> %va, <2 x i1> %m, i32 zeroext %
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -568,8 +568,8 @@ define <4 x double> @vp_floor_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroext %
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -612,8 +612,8 @@ define <8 x double> @vp_floor_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroext %
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -656,8 +656,8 @@ define <15 x double> @vp_floor_v15f64(<15 x double> %va, <15 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -700,8 +700,8 @@ define <16 x double> @vp_floor_v16f64(<16 x double> %va, <16 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -764,8 +764,8 @@ define <32 x double> @vp_floor_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v25, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a1, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a1
@@ -788,8 +788,8 @@ define <32 x double> @vp_floor_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vfabs.v v8, v16, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v24, v8, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
 ; CHECK-NEXT:    fsrm a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll
index 6f045349423c..716cf7b0f46f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-round-vp.ll
@@ -19,8 +19,8 @@ define <2 x half> @vp_round_v2f16(<2 x half> %va, <2 x i1> %m, i32 zeroext %evl)
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 4
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -38,8 +38,8 @@ define <2 x half> @vp_round_v2f16(<2 x half> %va, <2 x i1> %m, i32 zeroext %evl)
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -101,8 +101,8 @@ define <4 x half> @vp_round_v4f16(<4 x half> %va, <4 x i1> %m, i32 zeroext %evl)
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 4
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -120,8 +120,8 @@ define <4 x half> @vp_round_v4f16(<4 x half> %va, <4 x i1> %m, i32 zeroext %evl)
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -183,8 +183,8 @@ define <8 x half> @vp_round_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %evl)
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 4
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -203,8 +203,8 @@ define <8 x half> @vp_round_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %evl)
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v9
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v12, v10, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -268,8 +268,8 @@ define <16 x half> @vp_round_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 4
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v10
 ; ZVFH-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -289,8 +289,8 @@ define <16 x half> @vp_round_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroext %
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v10, v16, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v10
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v16, v12, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -353,8 +353,8 @@ define <2 x float> @vp_round_v2f32(<2 x float> %va, <2 x i1> %m, i32 zeroext %ev
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -395,8 +395,8 @@ define <4 x float> @vp_round_v4f32(<4 x float> %va, <4 x i1> %m, i32 zeroext %ev
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -438,8 +438,8 @@ define <8 x float> @vp_round_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext %ev
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -482,8 +482,8 @@ define <16 x float> @vp_round_v16f32(<16 x float> %va, <16 x i1> %m, i32 zeroext
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -525,8 +525,8 @@ define <2 x double> @vp_round_v2f64(<2 x double> %va, <2 x i1> %m, i32 zeroext %
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -568,8 +568,8 @@ define <4 x double> @vp_round_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroext %
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -612,8 +612,8 @@ define <8 x double> @vp_round_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroext %
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -656,8 +656,8 @@ define <15 x double> @vp_round_v15f64(<15 x double> %va, <15 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -700,8 +700,8 @@ define <16 x double> @vp_round_v16f64(<16 x double> %va, <16 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -764,8 +764,8 @@ define <32 x double> @vp_round_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v25, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a1, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a1
@@ -788,8 +788,8 @@ define <32 x double> @vp_round_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vfabs.v v8, v16, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v24, v8, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
 ; CHECK-NEXT:    fsrm a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll
index 738d7e37c50b..603f9397dc90 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundeven-vp.ll
@@ -19,8 +19,8 @@ define <2 x half> @vp_roundeven_v2f16(<2 x half> %va, <2 x i1> %m, i32 zeroext %
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 0
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -38,8 +38,8 @@ define <2 x half> @vp_roundeven_v2f16(<2 x half> %va, <2 x i1> %m, i32 zeroext %
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -101,8 +101,8 @@ define <4 x half> @vp_roundeven_v4f16(<4 x half> %va, <4 x i1> %m, i32 zeroext %
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 0
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -120,8 +120,8 @@ define <4 x half> @vp_roundeven_v4f16(<4 x half> %va, <4 x i1> %m, i32 zeroext %
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -183,8 +183,8 @@ define <8 x half> @vp_roundeven_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 0
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -203,8 +203,8 @@ define <8 x half> @vp_roundeven_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v9
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v12, v10, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -268,8 +268,8 @@ define <16 x half> @vp_roundeven_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroe
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 0
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v10
 ; ZVFH-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -289,8 +289,8 @@ define <16 x half> @vp_roundeven_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroe
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v10, v16, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v10
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v16, v12, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -353,8 +353,8 @@ define <2 x float> @vp_roundeven_v2f32(<2 x float> %va, <2 x i1> %m, i32 zeroext
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -395,8 +395,8 @@ define <4 x float> @vp_roundeven_v4f32(<4 x float> %va, <4 x i1> %m, i32 zeroext
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -438,8 +438,8 @@ define <8 x float> @vp_roundeven_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -482,8 +482,8 @@ define <16 x float> @vp_roundeven_v16f32(<16 x float> %va, <16 x i1> %m, i32 zer
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -525,8 +525,8 @@ define <2 x double> @vp_roundeven_v2f64(<2 x double> %va, <2 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -568,8 +568,8 @@ define <4 x double> @vp_roundeven_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -612,8 +612,8 @@ define <8 x double> @vp_roundeven_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -656,8 +656,8 @@ define <15 x double> @vp_roundeven_v15f64(<15 x double> %va, <15 x i1> %m, i32 z
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -700,8 +700,8 @@ define <16 x double> @vp_roundeven_v16f64(<16 x double> %va, <16 x i1> %m, i32 z
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -764,8 +764,8 @@ define <32 x double> @vp_roundeven_v32f64(<32 x double> %va, <32 x i1> %m, i32 z
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v25, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a1, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a1
@@ -788,8 +788,8 @@ define <32 x double> @vp_roundeven_v32f64(<32 x double> %va, <32 x i1> %m, i32 z
 ; CHECK-NEXT:    vfabs.v v8, v16, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v24, v8, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
 ; CHECK-NEXT:    fsrm a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll
index 6f5b7875266b..a5adfc36887a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-roundtozero-vp.ll
@@ -19,8 +19,8 @@ define <2 x half> @vp_roundtozero_v2f16(<2 x half> %va, <2 x i1> %m, i32 zeroext
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 1
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -38,8 +38,8 @@ define <2 x half> @vp_roundtozero_v2f16(<2 x half> %va, <2 x i1> %m, i32 zeroext
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -101,8 +101,8 @@ define <4 x half> @vp_roundtozero_v4f16(<4 x half> %va, <4 x i1> %m, i32 zeroext
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 1
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -120,8 +120,8 @@ define <4 x half> @vp_roundtozero_v4f16(<4 x half> %va, <4 x i1> %m, i32 zeroext
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -183,8 +183,8 @@ define <8 x half> @vp_roundtozero_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 1
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -203,8 +203,8 @@ define <8 x half> @vp_roundtozero_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v9
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v12, v10, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -268,8 +268,8 @@ define <16 x half> @vp_roundtozero_v16f16(<16 x half> %va, <16 x i1> %m, i32 zer
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 1
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v10
 ; ZVFH-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -289,8 +289,8 @@ define <16 x half> @vp_roundtozero_v16f16(<16 x half> %va, <16 x i1> %m, i32 zer
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v10, v16, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v10
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v16, v12, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -353,8 +353,8 @@ define <2 x float> @vp_roundtozero_v2f32(<2 x float> %va, <2 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -395,8 +395,8 @@ define <4 x float> @vp_roundtozero_v4f32(<4 x float> %va, <4 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -438,8 +438,8 @@ define <8 x float> @vp_roundtozero_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -482,8 +482,8 @@ define <16 x float> @vp_roundtozero_v16f32(<16 x float> %va, <16 x i1> %m, i32 z
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -525,8 +525,8 @@ define <2 x double> @vp_roundtozero_v2f64(<2 x double> %va, <2 x i1> %m, i32 zer
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -568,8 +568,8 @@ define <4 x double> @vp_roundtozero_v4f64(<4 x double> %va, <4 x i1> %m, i32 zer
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -612,8 +612,8 @@ define <8 x double> @vp_roundtozero_v8f64(<8 x double> %va, <8 x i1> %m, i32 zer
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -656,8 +656,8 @@ define <15 x double> @vp_roundtozero_v15f64(<15 x double> %va, <15 x i1> %m, i32
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -700,8 +700,8 @@ define <16 x double> @vp_roundtozero_v16f64(<16 x double> %va, <16 x i1> %m, i32
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -764,8 +764,8 @@ define <32 x double> @vp_roundtozero_v32f64(<32 x double> %va, <32 x i1> %m, i32
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v25, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a1, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a1
@@ -788,8 +788,8 @@ define <32 x double> @vp_roundtozero_v32f64(<32 x double> %va, <32 x i1> %m, i32
 ; CHECK-NEXT:    vfabs.v v8, v16, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v24, v8, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
 ; CHECK-NEXT:    fsrm a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vaaddu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vaaddu.ll
index 70b547759938..600290a62515 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vaaddu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vaaddu.ll
@@ -5,8 +5,8 @@
 define <8 x i8> @vaaddu_vv_v8i8_floor(<8 x i8> %x, <8 x i8> %y) {
 ; CHECK-LABEL: vaaddu_vv_v8i8_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i8> %x to <8 x i16>
@@ -20,8 +20,8 @@ define <8 x i8> @vaaddu_vv_v8i8_floor(<8 x i8> %x, <8 x i8> %y) {
 define <8 x i8> @vaaddu_vx_v8i8_floor(<8 x i8> %x, i8 %y) {
 ; CHECK-LABEL: vaaddu_vx_v8i8_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i8> %x to <8 x i16>
@@ -53,8 +53,8 @@ define <8 x i8> @vaaddu_vv_v8i8_floor_sexti16(<8 x i8> %x, <8 x i8> %y) {
 define <8 x i8> @vaaddu_vv_v8i8_floor_zexti32(<8 x i8> %x, <8 x i8> %y) {
 ; CHECK-LABEL: vaaddu_vv_v8i8_floor_zexti32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i8> %x to <8 x i32>
@@ -83,8 +83,8 @@ define <8 x i8> @vaaddu_vv_v8i8_floor_lshr2(<8 x i8> %x, <8 x i8> %y) {
 define <8 x i16> @vaaddu_vv_v8i16_floor(<8 x i16> %x, <8 x i16> %y) {
 ; CHECK-LABEL: vaaddu_vv_v8i16_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i16> %x to <8 x i32>
@@ -98,8 +98,8 @@ define <8 x i16> @vaaddu_vv_v8i16_floor(<8 x i16> %x, <8 x i16> %y) {
 define <8 x i16> @vaaddu_vx_v8i16_floor(<8 x i16> %x, i16 %y) {
 ; CHECK-LABEL: vaaddu_vx_v8i16_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i16> %x to <8 x i32>
@@ -115,8 +115,8 @@ define <8 x i16> @vaaddu_vx_v8i16_floor(<8 x i16> %x, i16 %y) {
 define <8 x i32> @vaaddu_vv_v8i32_floor(<8 x i32> %x, <8 x i32> %y) {
 ; CHECK-LABEL: vaaddu_vv_v8i32_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i32> %x to <8 x i64>
@@ -130,8 +130,8 @@ define <8 x i32> @vaaddu_vv_v8i32_floor(<8 x i32> %x, <8 x i32> %y) {
 define <8 x i32> @vaaddu_vx_v8i32_floor(<8 x i32> %x, i32 %y) {
 ; CHECK-LABEL: vaaddu_vx_v8i32_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i32> %x to <8 x i64>
@@ -147,8 +147,8 @@ define <8 x i32> @vaaddu_vx_v8i32_floor(<8 x i32> %x, i32 %y) {
 define <8 x i64> @vaaddu_vv_v8i64_floor(<8 x i64> %x, <8 x i64> %y) {
 ; CHECK-LABEL: vaaddu_vv_v8i64_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i64> %x to <8 x i128>
@@ -197,8 +197,8 @@ define <8 x i64> @vaaddu_vx_v8i64_floor(<8 x i64> %x, i64 %y) {
 ;
 ; RV64-LABEL: vaaddu_vx_v8i64_floor:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 2
+; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-NEXT:    vaaddu.vx v8, v8, a0
 ; RV64-NEXT:    ret
   %xzv = zext <8 x i64> %x to <8 x i128>
@@ -214,8 +214,8 @@ define <8 x i64> @vaaddu_vx_v8i64_floor(<8 x i64> %x, i64 %y) {
 define <8 x i8> @vaaddu_vv_v8i8_ceil(<8 x i8> %x, <8 x i8> %y) {
 ; CHECK-LABEL: vaaddu_vv_v8i8_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i8> %x to <8 x i16>
@@ -230,8 +230,8 @@ define <8 x i8> @vaaddu_vv_v8i8_ceil(<8 x i8> %x, <8 x i8> %y) {
 define <8 x i8> @vaaddu_vx_v8i8_ceil(<8 x i8> %x, i8 %y) {
 ; CHECK-LABEL: vaaddu_vx_v8i8_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i8> %x to <8 x i16>
@@ -267,8 +267,8 @@ define <8 x i8> @vaaddu_vv_v8i8_ceil_sexti16(<8 x i8> %x, <8 x i8> %y) {
 define <8 x i8> @vaaddu_vv_v8i8_ceil_zexti32(<8 x i8> %x, <8 x i8> %y) {
 ; CHECK-LABEL: vaaddu_vv_v8i8_ceil_zexti32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i8> %x to <8 x i32>
@@ -305,8 +305,8 @@ define <8 x i8> @vaaddu_vv_v8i8_ceil_add2(<8 x i8> %x, <8 x i8> %y) {
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vwaddu.vv v10, v8, v9
 ; CHECK-NEXT:    li a0, 2
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v10, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnsrl.wi v8, v8, 0
@@ -323,8 +323,8 @@ define <8 x i8> @vaaddu_vv_v8i8_ceil_add2(<8 x i8> %x, <8 x i8> %y) {
 define <8 x i16> @vaaddu_vv_v8i16_ceil(<8 x i16> %x, <8 x i16> %y) {
 ; CHECK-LABEL: vaaddu_vv_v8i16_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i16> %x to <8 x i32>
@@ -339,8 +339,8 @@ define <8 x i16> @vaaddu_vv_v8i16_ceil(<8 x i16> %x, <8 x i16> %y) {
 define <8 x i16> @vaaddu_vx_v8i16_ceil(<8 x i16> %x, i16 %y) {
 ; CHECK-LABEL: vaaddu_vx_v8i16_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i16> %x to <8 x i32>
@@ -357,8 +357,8 @@ define <8 x i16> @vaaddu_vx_v8i16_ceil(<8 x i16> %x, i16 %y) {
 define <8 x i32> @vaaddu_vv_v8i32_ceil(<8 x i32> %x, <8 x i32> %y) {
 ; CHECK-LABEL: vaaddu_vv_v8i32_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i32> %x to <8 x i64>
@@ -373,8 +373,8 @@ define <8 x i32> @vaaddu_vv_v8i32_ceil(<8 x i32> %x, <8 x i32> %y) {
 define <8 x i32> @vaaddu_vx_v8i32_ceil(<8 x i32> %x, i32 %y) {
 ; CHECK-LABEL: vaaddu_vx_v8i32_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i32> %x to <8 x i64>
@@ -391,8 +391,8 @@ define <8 x i32> @vaaddu_vx_v8i32_ceil(<8 x i32> %x, i32 %y) {
 define <8 x i64> @vaaddu_vv_v8i64_ceil(<8 x i64> %x, <8 x i64> %y) {
 ; CHECK-LABEL: vaaddu_vv_v8i64_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %xzv = zext <8 x i64> %x to <8 x i128>
@@ -443,8 +443,8 @@ define <8 x i64> @vaaddu_vx_v8i64_ceil(<8 x i64> %x, i64 %y) {
 ;
 ; RV64-LABEL: vaaddu_vx_v8i64_ceil:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-NEXT:    vaaddu.vx v8, v8, a0
 ; RV64-NEXT:    ret
   %xzv = zext <8 x i64> %x to <8 x i128>
diff --git a/llvm/test/CodeGen/RISCV/rvv/float-round-conv.ll b/llvm/test/CodeGen/RISCV/rvv/float-round-conv.ll
index 9dcb6d211cb9..b7661bd826fe 100644
--- a/llvm/test/CodeGen/RISCV/rvv/float-round-conv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/float-round-conv.ll
@@ -487,8 +487,8 @@ define <vscale x 1 x i8> @ceil_nxv1f32_to_ui8(<vscale x 1 x float> %x) {
 define <vscale x 1 x i16> @ceil_nxv1f32_to_si16(<vscale x 1 x float> %x) {
 ; RV32-LABEL: ceil_nxv1f32_to_si16:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; RV32-NEXT:    vfncvt.x.f.w v9, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv1r.v v8, v9
@@ -496,8 +496,8 @@ define <vscale x 1 x i16> @ceil_nxv1f32_to_si16(<vscale x 1 x float> %x) {
 ;
 ; RV64-LABEL: ceil_nxv1f32_to_si16:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; RV64-NEXT:    vfncvt.x.f.w v9, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv1r.v v8, v9
@@ -510,8 +510,8 @@ define <vscale x 1 x i16> @ceil_nxv1f32_to_si16(<vscale x 1 x float> %x) {
 define <vscale x 1 x i16> @ceil_nxv1f32_to_ui16(<vscale x 1 x float> %x) {
 ; RV32-LABEL: ceil_nxv1f32_to_ui16:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; RV32-NEXT:    vfncvt.xu.f.w v9, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv1r.v v8, v9
@@ -519,8 +519,8 @@ define <vscale x 1 x i16> @ceil_nxv1f32_to_ui16(<vscale x 1 x float> %x) {
 ;
 ; RV64-LABEL: ceil_nxv1f32_to_ui16:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; RV64-NEXT:    vfncvt.xu.f.w v9, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv1r.v v8, v9
@@ -533,16 +533,16 @@ define <vscale x 1 x i16> @ceil_nxv1f32_to_ui16(<vscale x 1 x float> %x) {
 define <vscale x 1 x i32> @ceil_nxv1f32_to_si32(<vscale x 1 x float> %x) {
 ; RV32-LABEL: ceil_nxv1f32_to_si32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ceil_nxv1f32_to_si32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    ret
@@ -554,16 +554,16 @@ define <vscale x 1 x i32> @ceil_nxv1f32_to_si32(<vscale x 1 x float> %x) {
 define <vscale x 1 x i32> @ceil_nxv1f32_to_ui32(<vscale x 1 x float> %x) {
 ; RV32-LABEL: ceil_nxv1f32_to_ui32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    vfcvt.xu.f.v v8, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ceil_nxv1f32_to_ui32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    vfcvt.xu.f.v v8, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    ret
@@ -575,8 +575,8 @@ define <vscale x 1 x i32> @ceil_nxv1f32_to_ui32(<vscale x 1 x float> %x) {
 define <vscale x 1 x i64> @ceil_nxv1f32_to_si64(<vscale x 1 x float> %x) {
 ; RV32-LABEL: ceil_nxv1f32_to_si64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    vfwcvt.x.f.v v9, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv1r.v v8, v9
@@ -584,8 +584,8 @@ define <vscale x 1 x i64> @ceil_nxv1f32_to_si64(<vscale x 1 x float> %x) {
 ;
 ; RV64-LABEL: ceil_nxv1f32_to_si64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    vfwcvt.x.f.v v9, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv1r.v v8, v9
@@ -598,8 +598,8 @@ define <vscale x 1 x i64> @ceil_nxv1f32_to_si64(<vscale x 1 x float> %x) {
 define <vscale x 1 x i64> @ceil_nxv1f32_to_ui64(<vscale x 1 x float> %x) {
 ; RV32-LABEL: ceil_nxv1f32_to_ui64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV32-NEXT:    vfwcvt.xu.f.v v9, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv1r.v v8, v9
@@ -607,8 +607,8 @@ define <vscale x 1 x i64> @ceil_nxv1f32_to_ui64(<vscale x 1 x float> %x) {
 ;
 ; RV64-LABEL: ceil_nxv1f32_to_ui64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; RV64-NEXT:    vfwcvt.xu.f.v v9, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv1r.v v8, v9
@@ -713,8 +713,8 @@ define <vscale x 4 x i8> @ceil_nxv4f32_to_ui8(<vscale x 4 x float> %x) {
 define <vscale x 4 x i16> @ceil_nxv4f32_to_si16(<vscale x 4 x float> %x) {
 ; RV32-LABEL: ceil_nxv4f32_to_si16:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; RV32-NEXT:    vfncvt.x.f.w v10, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv.v.v v8, v10
@@ -722,8 +722,8 @@ define <vscale x 4 x i16> @ceil_nxv4f32_to_si16(<vscale x 4 x float> %x) {
 ;
 ; RV64-LABEL: ceil_nxv4f32_to_si16:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; RV64-NEXT:    vfncvt.x.f.w v10, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv.v.v v8, v10
@@ -736,8 +736,8 @@ define <vscale x 4 x i16> @ceil_nxv4f32_to_si16(<vscale x 4 x float> %x) {
 define <vscale x 4 x i16> @ceil_nxv4f32_to_ui16(<vscale x 4 x float> %x) {
 ; RV32-LABEL: ceil_nxv4f32_to_ui16:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; RV32-NEXT:    vfncvt.xu.f.w v10, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv.v.v v8, v10
@@ -745,8 +745,8 @@ define <vscale x 4 x i16> @ceil_nxv4f32_to_ui16(<vscale x 4 x float> %x) {
 ;
 ; RV64-LABEL: ceil_nxv4f32_to_ui16:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; RV64-NEXT:    vfncvt.xu.f.w v10, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv.v.v v8, v10
@@ -759,16 +759,16 @@ define <vscale x 4 x i16> @ceil_nxv4f32_to_ui16(<vscale x 4 x float> %x) {
 define <vscale x 4 x i32> @ceil_nxv4f32_to_si32(<vscale x 4 x float> %x) {
 ; RV32-LABEL: ceil_nxv4f32_to_si32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ceil_nxv4f32_to_si32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    ret
@@ -780,16 +780,16 @@ define <vscale x 4 x i32> @ceil_nxv4f32_to_si32(<vscale x 4 x float> %x) {
 define <vscale x 4 x i32> @ceil_nxv4f32_to_ui32(<vscale x 4 x float> %x) {
 ; RV32-LABEL: ceil_nxv4f32_to_ui32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV32-NEXT:    vfcvt.xu.f.v v8, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ceil_nxv4f32_to_ui32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV64-NEXT:    vfcvt.xu.f.v v8, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    ret
@@ -801,8 +801,8 @@ define <vscale x 4 x i32> @ceil_nxv4f32_to_ui32(<vscale x 4 x float> %x) {
 define <vscale x 4 x i64> @ceil_nxv4f32_to_si64(<vscale x 4 x float> %x) {
 ; RV32-LABEL: ceil_nxv4f32_to_si64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV32-NEXT:    vfwcvt.x.f.v v12, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv4r.v v8, v12
@@ -810,8 +810,8 @@ define <vscale x 4 x i64> @ceil_nxv4f32_to_si64(<vscale x 4 x float> %x) {
 ;
 ; RV64-LABEL: ceil_nxv4f32_to_si64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV64-NEXT:    vfwcvt.x.f.v v12, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv4r.v v8, v12
@@ -824,8 +824,8 @@ define <vscale x 4 x i64> @ceil_nxv4f32_to_si64(<vscale x 4 x float> %x) {
 define <vscale x 4 x i64> @ceil_nxv4f32_to_ui64(<vscale x 4 x float> %x) {
 ; RV32-LABEL: ceil_nxv4f32_to_ui64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV32-NEXT:    fsrmi a0, 3
+; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV32-NEXT:    vfwcvt.xu.f.v v12, v8
 ; RV32-NEXT:    fsrm a0
 ; RV32-NEXT:    vmv4r.v v8, v12
@@ -833,8 +833,8 @@ define <vscale x 4 x i64> @ceil_nxv4f32_to_ui64(<vscale x 4 x float> %x) {
 ;
 ; RV64-LABEL: ceil_nxv4f32_to_ui64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; RV64-NEXT:    fsrmi a0, 3
+; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV64-NEXT:    vfwcvt.xu.f.v v12, v8
 ; RV64-NEXT:    fsrm a0
 ; RV64-NEXT:    vmv4r.v v8, v12
diff --git a/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll b/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll
index 9c4706b2bda7..d464b491bbbe 100644
--- a/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/floor-vp.ll
@@ -15,8 +15,8 @@ define <vscale x 1 x half> @vp_floor_nxv1f16(<vscale x 1 x half> %va, <vscale x
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -57,8 +57,8 @@ define <vscale x 2 x half> @vp_floor_nxv2f16(<vscale x 2 x half> %va, <vscale x
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -99,8 +99,8 @@ define <vscale x 4 x half> @vp_floor_nxv4f16(<vscale x 4 x half> %va, <vscale x
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -142,8 +142,8 @@ define <vscale x 8 x half> @vp_floor_nxv8f16(<vscale x 8 x half> %va, <vscale x
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -186,8 +186,8 @@ define <vscale x 16 x half> @vp_floor_nxv16f16(<vscale x 16 x half> %va, <vscale
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -230,8 +230,8 @@ define <vscale x 32 x half> @vp_floor_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -273,8 +273,8 @@ define <vscale x 1 x float> @vp_floor_nxv1f32(<vscale x 1 x float> %va, <vscale
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -315,8 +315,8 @@ define <vscale x 2 x float> @vp_floor_nxv2f32(<vscale x 2 x float> %va, <vscale
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -358,8 +358,8 @@ define <vscale x 4 x float> @vp_floor_nxv4f32(<vscale x 4 x float> %va, <vscale
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -402,8 +402,8 @@ define <vscale x 8 x float> @vp_floor_nxv8f32(<vscale x 8 x float> %va, <vscale
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -446,8 +446,8 @@ define <vscale x 16 x float> @vp_floor_nxv16f32(<vscale x 16 x float> %va, <vsca
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -489,8 +489,8 @@ define <vscale x 1 x double> @vp_floor_nxv1f64(<vscale x 1 x double> %va, <vscal
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -532,8 +532,8 @@ define <vscale x 2 x double> @vp_floor_nxv2f64(<vscale x 2 x double> %va, <vscal
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -576,8 +576,8 @@ define <vscale x 4 x double> @vp_floor_nxv4f64(<vscale x 4 x double> %va, <vscal
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -620,8 +620,8 @@ define <vscale x 7 x double> @vp_floor_nxv7f64(<vscale x 7 x double> %va, <vscal
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -664,8 +664,8 @@ define <vscale x 8 x double> @vp_floor_nxv8f64(<vscale x 8 x double> %va, <vscal
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -726,8 +726,8 @@ define <vscale x 16 x double> @vp_floor_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    vfabs.v v8, v16, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v25, v8, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a2, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
 ; CHECK-NEXT:    fsrm a2
@@ -750,8 +750,8 @@ define <vscale x 16 x double> @vp_floor_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v24, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll b/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll
index cdbc6e8d8d55..0e102d98c79c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/frm-insert.ll
@@ -13,19 +13,19 @@ declare <vscale x 1 x float> @llvm.riscv.vfadd.nxv1f32.nxv1f32(
 define <vscale x 1 x float> @test(<vscale x 1 x float> %0, <vscale x 1 x float> %1, i64 %2) nounwind {
 ; CHECK-LABEL: test:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
 ; CHECK-NEXT:    vfadd.vv v8, v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; UNOPT-LABEL: test:
 ; UNOPT:       # %bb.0: # %entry
+; UNOPT-NEXT:    fsrmi a1, 0
 ; UNOPT-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; UNOPT-NEXT:    fsrmi a0, 0
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v9
-; UNOPT-NEXT:    fsrm a0
+; UNOPT-NEXT:    fsrm a1
 ; UNOPT-NEXT:    fsrmi a0, 0
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v8
 ; UNOPT-NEXT:    fsrm a0
@@ -48,20 +48,20 @@ entry:
 define <vscale x 1 x float> @test2(<vscale x 1 x float> %0, <vscale x 1 x float> %1, i64 %2) nounwind {
 ; CHECK-LABEL: test2:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
 ; CHECK-NEXT:    fsrmi 1
 ; CHECK-NEXT:    vfadd.vv v8, v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; UNOPT-LABEL: test2:
 ; UNOPT:       # %bb.0: # %entry
+; UNOPT-NEXT:    fsrmi a1, 0
 ; UNOPT-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; UNOPT-NEXT:    fsrmi a0, 0
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v9
-; UNOPT-NEXT:    fsrm a0
+; UNOPT-NEXT:    fsrm a1
 ; UNOPT-NEXT:    fsrmi a0, 1
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v8
 ; UNOPT-NEXT:    fsrm a0
@@ -132,12 +132,12 @@ define <vscale x 1 x float> @before_call1(<vscale x 1 x float> %0, <vscale x 1 x
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 1
 ; CHECK-NEXT:    sub sp, sp, a1
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    addi a1, sp, 32
-; CHECK-NEXT:    vs1r.v v8, (a1) # Unknown-size Folded Spill
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    addi a0, sp, 32
+; CHECK-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    call foo
 ; CHECK-NEXT:    addi a0, sp, 32
 ; CHECK-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
@@ -155,12 +155,12 @@ define <vscale x 1 x float> @before_call1(<vscale x 1 x float> %0, <vscale x 1 x
 ; UNOPT-NEXT:    csrr a1, vlenb
 ; UNOPT-NEXT:    slli a1, a1, 1
 ; UNOPT-NEXT:    sub sp, sp, a1
+; UNOPT-NEXT:    fsrmi a1, 0
 ; UNOPT-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; UNOPT-NEXT:    fsrmi a0, 0
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v9
-; UNOPT-NEXT:    addi a1, sp, 32
-; UNOPT-NEXT:    vs1r.v v8, (a1) # Unknown-size Folded Spill
-; UNOPT-NEXT:    fsrm a0
+; UNOPT-NEXT:    addi a0, sp, 32
+; UNOPT-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
+; UNOPT-NEXT:    fsrm a1
 ; UNOPT-NEXT:    call foo
 ; UNOPT-NEXT:    addi a0, sp, 32
 ; UNOPT-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
@@ -240,12 +240,12 @@ define <vscale x 1 x float> @after_call1(<vscale x 1 x float> %0, <vscale x 1 x
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 1
 ; CHECK-NEXT:    sub sp, sp, a1
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    addi a1, sp, 32
-; CHECK-NEXT:    vs1r.v v8, (a1) # Unknown-size Folded Spill
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    addi a0, sp, 32
+; CHECK-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    call foo
 ; CHECK-NEXT:    addi a0, sp, 32
 ; CHECK-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
@@ -263,12 +263,12 @@ define <vscale x 1 x float> @after_call1(<vscale x 1 x float> %0, <vscale x 1 x
 ; UNOPT-NEXT:    csrr a1, vlenb
 ; UNOPT-NEXT:    slli a1, a1, 1
 ; UNOPT-NEXT:    sub sp, sp, a1
+; UNOPT-NEXT:    fsrmi a1, 0
 ; UNOPT-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; UNOPT-NEXT:    fsrmi a0, 0
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v9
-; UNOPT-NEXT:    addi a1, sp, 32
-; UNOPT-NEXT:    vs1r.v v8, (a1) # Unknown-size Folded Spill
-; UNOPT-NEXT:    fsrm a0
+; UNOPT-NEXT:    addi a0, sp, 32
+; UNOPT-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
+; UNOPT-NEXT:    fsrm a1
 ; UNOPT-NEXT:    call foo
 ; UNOPT-NEXT:    addi a0, sp, 32
 ; UNOPT-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
@@ -360,20 +360,20 @@ entry:
 define <vscale x 1 x float> @before_asm1(<vscale x 1 x float> %0, <vscale x 1 x float> %1, i64 %2) nounwind {
 ; CHECK-LABEL: before_asm1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    ret
 ;
 ; UNOPT-LABEL: before_asm1:
 ; UNOPT:       # %bb.0: # %entry
+; UNOPT-NEXT:    fsrmi a1, 0
 ; UNOPT-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; UNOPT-NEXT:    fsrmi a0, 0
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v9
-; UNOPT-NEXT:    fsrm a0
+; UNOPT-NEXT:    fsrm a1
 ; UNOPT-NEXT:    #APP
 ; UNOPT-NEXT:    #NO_APP
 ; UNOPT-NEXT:    ret
@@ -416,20 +416,20 @@ entry:
 define <vscale x 1 x float> @after_asm1(<vscale x 1 x float> %0, <vscale x 1 x float> %1, i64 %2) nounwind {
 ; CHECK-LABEL: after_asm1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
 ; CHECK-NEXT:    ret
 ;
 ; UNOPT-LABEL: after_asm1:
 ; UNOPT:       # %bb.0: # %entry
+; UNOPT-NEXT:    fsrmi a1, 0
 ; UNOPT-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; UNOPT-NEXT:    fsrmi a0, 0
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v9
-; UNOPT-NEXT:    fsrm a0
+; UNOPT-NEXT:    fsrm a1
 ; UNOPT-NEXT:    #APP
 ; UNOPT-NEXT:    #NO_APP
 ; UNOPT-NEXT:    ret
@@ -476,10 +476,10 @@ declare i32 @llvm.get.rounding()
 define <vscale x 1 x float> @test5(<vscale x 1 x float> %0, <vscale x 1 x float> %1, i64 %2, ptr %p) nounwind {
 ; CHECK-LABEL: test5:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a2, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a2
 ; CHECK-NEXT:    frrm a0
 ; CHECK-NEXT:    slli a0, a0, 2
 ; CHECK-NEXT:    lui a2, 66
@@ -492,10 +492,10 @@ define <vscale x 1 x float> @test5(<vscale x 1 x float> %0, <vscale x 1 x float>
 ;
 ; UNOPT-LABEL: test5:
 ; UNOPT:       # %bb.0: # %entry
+; UNOPT-NEXT:    fsrmi a2, 0
 ; UNOPT-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; UNOPT-NEXT:    fsrmi a0, 0
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v9
-; UNOPT-NEXT:    fsrm a0
+; UNOPT-NEXT:    fsrm a2
 ; UNOPT-NEXT:    frrm a0
 ; UNOPT-NEXT:    slli a0, a0, 2
 ; UNOPT-NEXT:    lui a2, 66
@@ -559,10 +559,10 @@ define <vscale x 1 x float> @after_fsrm2(<vscale x 1 x float> %0, <vscale x 1 x
 ; UNOPT-LABEL: after_fsrm2:
 ; UNOPT:       # %bb.0: # %entry
 ; UNOPT-NEXT:    fsrmi 4
+; UNOPT-NEXT:    fsrmi a1, 4
 ; UNOPT-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; UNOPT-NEXT:    fsrmi a0, 4
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v9
-; UNOPT-NEXT:    fsrm a0
+; UNOPT-NEXT:    fsrm a1
 ; UNOPT-NEXT:    ret
 entry:
   call void @llvm.set.rounding(i32 4)
@@ -579,19 +579,19 @@ define <vscale x 1 x float> @after_fsrm3(<vscale x 1 x float> %0, <vscale x 1 x
 ; CHECK-LABEL: after_fsrm3:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fsrmi 4
+; CHECK-NEXT:    fsrmi a1, 5
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 5
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 ;
 ; UNOPT-LABEL: after_fsrm3:
 ; UNOPT:       # %bb.0: # %entry
 ; UNOPT-NEXT:    fsrmi 4
+; UNOPT-NEXT:    fsrmi a1, 5
 ; UNOPT-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; UNOPT-NEXT:    fsrmi a0, 5
 ; UNOPT-NEXT:    vfadd.vv v8, v8, v9
-; UNOPT-NEXT:    fsrm a0
+; UNOPT-NEXT:    fsrm a1
 ; UNOPT-NEXT:    ret
 entry:
   call void @llvm.set.rounding(i32 4)
diff --git a/llvm/test/CodeGen/RISCV/rvv/half-round-conv.ll b/llvm/test/CodeGen/RISCV/rvv/half-round-conv.ll
index 6de62214ccc4..5cd9996c7ba3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/half-round-conv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/half-round-conv.ll
@@ -249,8 +249,8 @@ declare <vscale x 1 x half> @llvm.ceil.nxv1f16(<vscale x 1 x half>)
 define <vscale x 1 x i8> @ceil_nxv1f16_to_si8(<vscale x 1 x half> %x) {
 ; CHECK-LABEL: ceil_nxv1f16_to_si8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vfncvt.x.f.w v9, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vmv1r.v v8, v9
@@ -263,8 +263,8 @@ define <vscale x 1 x i8> @ceil_nxv1f16_to_si8(<vscale x 1 x half> %x) {
 define <vscale x 1 x i8> @ceil_nxv1f16_to_ui8(<vscale x 1 x half> %x) {
 ; CHECK-LABEL: ceil_nxv1f16_to_ui8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vfncvt.xu.f.w v9, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vmv1r.v v8, v9
@@ -277,8 +277,8 @@ define <vscale x 1 x i8> @ceil_nxv1f16_to_ui8(<vscale x 1 x half> %x) {
 define <vscale x 1 x i16> @ceil_nxv1f16_to_si16(<vscale x 1 x half> %x) {
 ; CHECK-LABEL: ceil_nxv1f16_to_si16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -290,8 +290,8 @@ define <vscale x 1 x i16> @ceil_nxv1f16_to_si16(<vscale x 1 x half> %x) {
 define <vscale x 1 x i16> @ceil_nxv1f16_to_ui16(<vscale x 1 x half> %x) {
 ; CHECK-LABEL: ceil_nxv1f16_to_ui16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -303,8 +303,8 @@ define <vscale x 1 x i16> @ceil_nxv1f16_to_ui16(<vscale x 1 x half> %x) {
 define <vscale x 1 x i32> @ceil_nxv1f16_to_si32(<vscale x 1 x half> %x) {
 ; CHECK-LABEL: ceil_nxv1f16_to_si32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvt.x.f.v v9, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vmv1r.v v8, v9
@@ -317,8 +317,8 @@ define <vscale x 1 x i32> @ceil_nxv1f16_to_si32(<vscale x 1 x half> %x) {
 define <vscale x 1 x i32> @ceil_nxv1f16_to_ui32(<vscale x 1 x half> %x) {
 ; CHECK-LABEL: ceil_nxv1f16_to_ui32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvt.xu.f.v v9, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vmv1r.v v8, v9
@@ -451,8 +451,8 @@ declare <vscale x 4 x half> @llvm.ceil.nxv4f16(<vscale x 4 x half>)
 define <vscale x 4 x i8> @ceil_nxv4f16_to_si8(<vscale x 4 x half> %x) {
 ; CHECK-LABEL: ceil_nxv4f16_to_si8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vfncvt.x.f.w v9, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vmv1r.v v8, v9
@@ -465,8 +465,8 @@ define <vscale x 4 x i8> @ceil_nxv4f16_to_si8(<vscale x 4 x half> %x) {
 define <vscale x 4 x i8> @ceil_nxv4f16_to_ui8(<vscale x 4 x half> %x) {
 ; CHECK-LABEL: ceil_nxv4f16_to_ui8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vfncvt.xu.f.w v9, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vmv1r.v v8, v9
@@ -479,8 +479,8 @@ define <vscale x 4 x i8> @ceil_nxv4f16_to_ui8(<vscale x 4 x half> %x) {
 define <vscale x 4 x i16> @ceil_nxv4f16_to_si16(<vscale x 4 x half> %x) {
 ; CHECK-LABEL: ceil_nxv4f16_to_si16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -492,8 +492,8 @@ define <vscale x 4 x i16> @ceil_nxv4f16_to_si16(<vscale x 4 x half> %x) {
 define <vscale x 4 x i16> @ceil_nxv4f16_to_ui16(<vscale x 4 x half> %x) {
 ; CHECK-LABEL: ceil_nxv4f16_to_ui16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -505,8 +505,8 @@ define <vscale x 4 x i16> @ceil_nxv4f16_to_ui16(<vscale x 4 x half> %x) {
 define <vscale x 4 x i32> @ceil_nxv4f16_to_si32(<vscale x 4 x half> %x) {
 ; CHECK-LABEL: ceil_nxv4f16_to_si32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvt.x.f.v v10, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vmv2r.v v8, v10
@@ -519,8 +519,8 @@ define <vscale x 4 x i32> @ceil_nxv4f16_to_si32(<vscale x 4 x half> %x) {
 define <vscale x 4 x i32> @ceil_nxv4f16_to_ui32(<vscale x 4 x half> %x) {
 ; CHECK-LABEL: ceil_nxv4f16_to_ui32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfwcvt.xu.f.v v10, v8
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vmv2r.v v8, v10
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll b/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
index d81079da64bd..f87fa3ec6f16 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-tama.ll
@@ -516,8 +516,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaadd.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vaadd_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -543,8 +543,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -570,8 +570,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vssrl_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -597,8 +597,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vnclip_mask_wv_nxv1i8_nxv1i16_nxv1i8(<vscale x 1 x i16> %0, <vscale x 1 x i8> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv1i8_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vnclip.wv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-tamu.ll b/llvm/test/CodeGen/RISCV/rvv/masked-tamu.ll
index c8bff58b00e4..4098270d365a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-tamu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-tamu.ll
@@ -489,8 +489,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaadd.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vaadd_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -515,8 +515,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -541,8 +541,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vssrl_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vssrl.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -567,8 +567,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vnclip_mask_wv_nxv1i8_nxv1i16_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv1i8_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-tuma.ll b/llvm/test/CodeGen/RISCV/rvv/masked-tuma.ll
index 409a008ec7cf..4cd7e143be66 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-tuma.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-tuma.ll
@@ -489,8 +489,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaadd.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vaadd_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -515,8 +515,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -541,8 +541,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vssrl_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vssrl.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -567,8 +567,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vnclip_mask_wv_nxv1i8_nxv1i16_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv1i8_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vnclip.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-tumu.ll b/llvm/test/CodeGen/RISCV/rvv/masked-tumu.ll
index 90054bcc5f36..c8719e6a2e7c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-tumu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-tumu.ll
@@ -489,8 +489,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaadd.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vaadd_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -515,8 +515,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -541,8 +541,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vssrl_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
 ; CHECK-NEXT:    vssrl.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -567,8 +567,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vnclip_mask_wv_nxv1i8_nxv1i16_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv1i8_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, mu
 ; CHECK-NEXT:    vnclip.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/round-vp.ll b/llvm/test/CodeGen/RISCV/rvv/round-vp.ll
index eb4994914fad..edeac1acf3b0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/round-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/round-vp.ll
@@ -19,8 +19,8 @@ define <vscale x 1 x half> @vp_round_nxv1f16(<vscale x 1 x half> %va, <vscale x
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 4
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -38,8 +38,8 @@ define <vscale x 1 x half> @vp_round_nxv1f16(<vscale x 1 x half> %va, <vscale x
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -101,8 +101,8 @@ define <vscale x 2 x half> @vp_round_nxv2f16(<vscale x 2 x half> %va, <vscale x
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 4
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -120,8 +120,8 @@ define <vscale x 2 x half> @vp_round_nxv2f16(<vscale x 2 x half> %va, <vscale x
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -183,8 +183,8 @@ define <vscale x 4 x half> @vp_round_nxv4f16(<vscale x 4 x half> %va, <vscale x
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 4
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -203,8 +203,8 @@ define <vscale x 4 x half> @vp_round_nxv4f16(<vscale x 4 x half> %va, <vscale x
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v9
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v12, v10, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -268,8 +268,8 @@ define <vscale x 8 x half> @vp_round_nxv8f16(<vscale x 8 x half> %va, <vscale x
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 4
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v10
 ; ZVFH-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -289,8 +289,8 @@ define <vscale x 8 x half> @vp_round_nxv8f16(<vscale x 8 x half> %va, <vscale x
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v10, v16, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v10
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v16, v12, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -354,8 +354,8 @@ define <vscale x 16 x half> @vp_round_nxv16f16(<vscale x 16 x half> %va, <vscale
 ; ZVFH-NEXT:    vfabs.v v16, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 4
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v12
 ; ZVFH-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -375,8 +375,8 @@ define <vscale x 16 x half> @vp_round_nxv16f16(<vscale x 16 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v12, v24, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v12
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v24, v16, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -440,8 +440,8 @@ define <vscale x 32 x half> @vp_round_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFH-NEXT:    vfabs.v v24, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 4
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v16
 ; ZVFH-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -479,8 +479,8 @@ define <vscale x 32 x half> @vp_round_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a2
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v17, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a2, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v17
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    fsrm a2
@@ -501,8 +501,8 @@ define <vscale x 32 x half> @vp_round_nxv32f16(<vscale x 32 x half> %va, <vscale
 ; ZVFHMIN-NEXT:    vfabs.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v16, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v16
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -567,8 +567,8 @@ define <vscale x 32 x half> @vp_round_nxv32f16_unmasked(<vscale x 32 x half> %va
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a2
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v16, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a2, 4
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v16
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    fsrm a2
@@ -615,8 +615,8 @@ define <vscale x 1 x float> @vp_round_nxv1f32(<vscale x 1 x float> %va, <vscale
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -657,8 +657,8 @@ define <vscale x 2 x float> @vp_round_nxv2f32(<vscale x 2 x float> %va, <vscale
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -700,8 +700,8 @@ define <vscale x 4 x float> @vp_round_nxv4f32(<vscale x 4 x float> %va, <vscale
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -744,8 +744,8 @@ define <vscale x 8 x float> @vp_round_nxv8f32(<vscale x 8 x float> %va, <vscale
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -788,8 +788,8 @@ define <vscale x 16 x float> @vp_round_nxv16f32(<vscale x 16 x float> %va, <vsca
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -831,8 +831,8 @@ define <vscale x 1 x double> @vp_round_nxv1f64(<vscale x 1 x double> %va, <vscal
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -874,8 +874,8 @@ define <vscale x 2 x double> @vp_round_nxv2f64(<vscale x 2 x double> %va, <vscal
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -918,8 +918,8 @@ define <vscale x 4 x double> @vp_round_nxv4f64(<vscale x 4 x double> %va, <vscal
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -962,8 +962,8 @@ define <vscale x 7 x double> @vp_round_nxv7f64(<vscale x 7 x double> %va, <vscal
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -1006,8 +1006,8 @@ define <vscale x 8 x double> @vp_round_nxv8f64(<vscale x 8 x double> %va, <vscal
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -1068,8 +1068,8 @@ define <vscale x 16 x double> @vp_round_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    vfabs.v v8, v16, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v25, v8, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a2, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
 ; CHECK-NEXT:    fsrm a2
@@ -1092,8 +1092,8 @@ define <vscale x 16 x double> @vp_round_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v24, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll b/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll
index f366a2922d07..a77c58ba9ec5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/roundeven-vp.ll
@@ -19,8 +19,8 @@ define <vscale x 1 x half> @vp_roundeven_nxv1f16(<vscale x 1 x half> %va, <vscal
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 0
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -38,8 +38,8 @@ define <vscale x 1 x half> @vp_roundeven_nxv1f16(<vscale x 1 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -101,8 +101,8 @@ define <vscale x 2 x half> @vp_roundeven_nxv2f16(<vscale x 2 x half> %va, <vscal
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 0
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -120,8 +120,8 @@ define <vscale x 2 x half> @vp_roundeven_nxv2f16(<vscale x 2 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -183,8 +183,8 @@ define <vscale x 4 x half> @vp_roundeven_nxv4f16(<vscale x 4 x half> %va, <vscal
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 0
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -203,8 +203,8 @@ define <vscale x 4 x half> @vp_roundeven_nxv4f16(<vscale x 4 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v9
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v12, v10, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -268,8 +268,8 @@ define <vscale x 8 x half> @vp_roundeven_nxv8f16(<vscale x 8 x half> %va, <vscal
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 0
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v10
 ; ZVFH-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -289,8 +289,8 @@ define <vscale x 8 x half> @vp_roundeven_nxv8f16(<vscale x 8 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v10, v16, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v10
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v16, v12, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -354,8 +354,8 @@ define <vscale x 16 x half> @vp_roundeven_nxv16f16(<vscale x 16 x half> %va, <vs
 ; ZVFH-NEXT:    vfabs.v v16, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 0
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v12
 ; ZVFH-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -375,8 +375,8 @@ define <vscale x 16 x half> @vp_roundeven_nxv16f16(<vscale x 16 x half> %va, <vs
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v12, v24, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v12
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v24, v16, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -440,8 +440,8 @@ define <vscale x 32 x half> @vp_roundeven_nxv32f16(<vscale x 32 x half> %va, <vs
 ; ZVFH-NEXT:    vfabs.v v24, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 0
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v16
 ; ZVFH-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -479,8 +479,8 @@ define <vscale x 32 x half> @vp_roundeven_nxv32f16(<vscale x 32 x half> %va, <vs
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a2
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v17, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a2, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v17
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    fsrm a2
@@ -501,8 +501,8 @@ define <vscale x 32 x half> @vp_roundeven_nxv32f16(<vscale x 32 x half> %va, <vs
 ; ZVFHMIN-NEXT:    vfabs.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v16, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v16
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -567,8 +567,8 @@ define <vscale x 32 x half> @vp_roundeven_nxv32f16_unmasked(<vscale x 32 x half>
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a2
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v16, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a2, 0
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v16
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    fsrm a2
@@ -615,8 +615,8 @@ define <vscale x 1 x float> @vp_roundeven_nxv1f32(<vscale x 1 x float> %va, <vsc
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -657,8 +657,8 @@ define <vscale x 2 x float> @vp_roundeven_nxv2f32(<vscale x 2 x float> %va, <vsc
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -700,8 +700,8 @@ define <vscale x 4 x float> @vp_roundeven_nxv4f32(<vscale x 4 x float> %va, <vsc
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -744,8 +744,8 @@ define <vscale x 8 x float> @vp_roundeven_nxv8f32(<vscale x 8 x float> %va, <vsc
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -788,8 +788,8 @@ define <vscale x 16 x float> @vp_roundeven_nxv16f32(<vscale x 16 x float> %va, <
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -831,8 +831,8 @@ define <vscale x 1 x double> @vp_roundeven_nxv1f64(<vscale x 1 x double> %va, <v
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -874,8 +874,8 @@ define <vscale x 2 x double> @vp_roundeven_nxv2f64(<vscale x 2 x double> %va, <v
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -918,8 +918,8 @@ define <vscale x 4 x double> @vp_roundeven_nxv4f64(<vscale x 4 x double> %va, <v
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -962,8 +962,8 @@ define <vscale x 7 x double> @vp_roundeven_nxv7f64(<vscale x 7 x double> %va, <v
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -1006,8 +1006,8 @@ define <vscale x 8 x double> @vp_roundeven_nxv8f64(<vscale x 8 x double> %va, <v
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -1068,8 +1068,8 @@ define <vscale x 16 x double> @vp_roundeven_nxv16f64(<vscale x 16 x double> %va,
 ; CHECK-NEXT:    vfabs.v v8, v16, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v25, v8, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a2, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
 ; CHECK-NEXT:    fsrm a2
@@ -1092,8 +1092,8 @@ define <vscale x 16 x double> @vp_roundeven_nxv16f64(<vscale x 16 x double> %va,
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v24, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll b/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll
index 79c940bdf089..71a53c525551 100644
--- a/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/roundtozero-vp.ll
@@ -19,8 +19,8 @@ define <vscale x 1 x half> @vp_roundtozero_nxv1f16(<vscale x 1 x half> %va, <vsc
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 1
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -38,8 +38,8 @@ define <vscale x 1 x half> @vp_roundtozero_nxv1f16(<vscale x 1 x half> %va, <vsc
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -101,8 +101,8 @@ define <vscale x 2 x half> @vp_roundtozero_nxv2f16(<vscale x 2 x half> %va, <vsc
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 1
+; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -120,8 +120,8 @@ define <vscale x 2 x half> @vp_roundtozero_nxv2f16(<vscale x 2 x half> %va, <vsc
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v0, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
@@ -183,8 +183,8 @@ define <vscale x 4 x half> @vp_roundtozero_nxv4f16(<vscale x 4 x half> %va, <vsc
 ; ZVFH-NEXT:    vfabs.v v9, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 1
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -203,8 +203,8 @@ define <vscale x 4 x half> @vp_roundtozero_nxv4f16(<vscale x 4 x half> %va, <vsc
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v9, v12, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v9
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v12, v10, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -268,8 +268,8 @@ define <vscale x 8 x half> @vp_roundtozero_nxv8f16(<vscale x 8 x half> %va, <vsc
 ; ZVFH-NEXT:    vfabs.v v12, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 1
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v10
 ; ZVFH-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -289,8 +289,8 @@ define <vscale x 8 x half> @vp_roundtozero_nxv8f16(<vscale x 8 x half> %va, <vsc
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v10, v16, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v10
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v16, v12, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -354,8 +354,8 @@ define <vscale x 16 x half> @vp_roundtozero_nxv16f16(<vscale x 16 x half> %va, <
 ; ZVFH-NEXT:    vfabs.v v16, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 1
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v12
 ; ZVFH-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -375,8 +375,8 @@ define <vscale x 16 x half> @vp_roundtozero_nxv16f16(<vscale x 16 x half> %va, <
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v12, v24, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v12
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v24, v16, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -440,8 +440,8 @@ define <vscale x 32 x half> @vp_roundtozero_nxv32f16(<vscale x 32 x half> %va, <
 ; ZVFH-NEXT:    vfabs.v v24, v8, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
 ; ZVFH-NEXT:    fsrmi a0, 1
+; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v0, v16
 ; ZVFH-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; ZVFH-NEXT:    fsrm a0
@@ -479,8 +479,8 @@ define <vscale x 32 x half> @vp_roundtozero_nxv32f16(<vscale x 32 x half> %va, <
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a2
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v17, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a2, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v17
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    fsrm a2
@@ -501,8 +501,8 @@ define <vscale x 32 x half> @vp_roundtozero_nxv32f16(<vscale x 32 x half> %va, <
 ; ZVFHMIN-NEXT:    vfabs.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v16, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a0, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v16
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    fsrm a0
@@ -567,8 +567,8 @@ define <vscale x 32 x half> @vp_roundtozero_nxv32f16_unmasked(<vscale x 32 x hal
 ; ZVFHMIN-NEXT:    fmv.w.x fa5, a2
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vmflt.vf v16, v8, fa5, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    fsrmi a2, 1
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v16
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    fsrm a2
@@ -615,8 +615,8 @@ define <vscale x 1 x float> @vp_roundtozero_nxv1f32(<vscale x 1 x float> %va, <v
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -657,8 +657,8 @@ define <vscale x 2 x float> @vp_roundtozero_nxv2f32(<vscale x 2 x float> %va, <v
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -700,8 +700,8 @@ define <vscale x 4 x float> @vp_roundtozero_nxv4f32(<vscale x 4 x float> %va, <v
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -744,8 +744,8 @@ define <vscale x 8 x float> @vp_roundtozero_nxv8f32(<vscale x 8 x float> %va, <v
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -788,8 +788,8 @@ define <vscale x 16 x float> @vp_roundtozero_nxv16f32(<vscale x 16 x float> %va,
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -831,8 +831,8 @@ define <vscale x 1 x double> @vp_roundtozero_nxv1f64(<vscale x 1 x double> %va,
 ; CHECK-NEXT:    vfabs.v v9, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
@@ -874,8 +874,8 @@ define <vscale x 2 x double> @vp_roundtozero_nxv2f64(<vscale x 2 x double> %va,
 ; CHECK-NEXT:    vfabs.v v12, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vmflt.vf v10, v12, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -918,8 +918,8 @@ define <vscale x 4 x double> @vp_roundtozero_nxv4f64(<vscale x 4 x double> %va,
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vmflt.vf v12, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -962,8 +962,8 @@ define <vscale x 7 x double> @vp_roundtozero_nxv7f64(<vscale x 7 x double> %va,
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -1006,8 +1006,8 @@ define <vscale x 8 x double> @vp_roundtozero_nxv8f64(<vscale x 8 x double> %va,
 ; CHECK-NEXT:    vfabs.v v24, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v16, v24, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
@@ -1068,8 +1068,8 @@ define <vscale x 16 x double> @vp_roundtozero_nxv16f64(<vscale x 16 x double> %v
 ; CHECK-NEXT:    vfabs.v v8, v16, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v25, v8, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a2, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
 ; CHECK-NEXT:    fsrm a2
@@ -1092,8 +1092,8 @@ define <vscale x 16 x double> @vp_roundtozero_nxv16f64(<vscale x 16 x double> %v
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vmflt.vf v24, v16, fa5, v0.t
-; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    fsrmi a0, 1
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    fsrm a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-masked-vops.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-masked-vops.ll
index 8cefbac59ce6..033a1d7e297f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-masked-vops.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-masked-vops.ll
@@ -215,10 +215,10 @@ declare <vscale x 2 x i32> @llvm.vp.merge.nxv2i32(<vscale x 2 x i1>, <vscale x 2
 define <vscale x 2 x i32> @vmerge_vfcvt_rm(<vscale x 2 x i32> %passthru, <vscale x 2 x float> %a, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vmerge_vfcvt_rm:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 2
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 2
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %floor = call <vscale x 2 x float> @llvm.floor.nxv2f32(<vscale x 2 x float> %a)
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
index 64b3a6f2b4b3..1a3a1a6c1ee6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
@@ -919,8 +919,8 @@ entry:
 define <vscale x 1 x i16> @test_vaaddu(<vscale x 1 x i16> %var_11, i16 zeroext %var_9, <vscale x 1 x i1> %var_5, <vscale x 1 x i16> %var_0) {
 ; CHECK-LABEL: test_vaaddu:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 3, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetivli zero, 3, e16, mf4, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v9, v8, a0, v0.t
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
@@ -966,13 +966,13 @@ declare <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv2f32(
 define <vscale x 2 x float> @vfredusum(<vscale x 2 x float> %passthru, <vscale x 2 x float> %x, <vscale x 2 x float> %y, <vscale x 2 x i1> %m, i64 %vl) {
 ; CHECK-LABEL: vfredusum:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vmv1r.v v11, v8
 ; CHECK-NEXT:    vfredusum.vs v11, v9, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, tu, ma
 ; CHECK-NEXT:    vmerge.vvm v8, v8, v11, v0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv2f32(
     <vscale x 2 x float> %passthru,
@@ -1002,10 +1002,10 @@ define <vscale x 2 x i32> @vredsum_allones_mask(<vscale x 2 x i32> %passthru, <v
 define <vscale x 2 x float> @vfredusum_allones_mask(<vscale x 2 x float> %passthru, <vscale x 2 x float> %x, <vscale x 2 x float> %y, i64 %vl) {
 ; CHECK-LABEL: vfredusum_allones_mask:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv2f32(
     <vscale x 2 x float> %passthru,
@@ -1136,10 +1136,10 @@ define <vscale x 2 x i64> @vpmerge_vwsub.w_tied(<vscale x 2 x i64> %passthru, <v
 define <vscale x 2 x double> @vpmerge_vfwsub.w_tied(<vscale x 2 x double> %passthru, <vscale x 2 x double> %x, <vscale x 2 x float> %y, <vscale x 2 x i1> %mask, i32 zeroext %vl) {
 ; CHECK-LABEL: vpmerge_vfwsub.w_tied:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    fsrmi a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 1
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
   %vl.zext = zext i32 %vl to i64
   %a = call <vscale x 2 x double> @llvm.riscv.vfwsub.w.nxv2f64.nxv2f32(<vscale x 2 x double> %passthru, <vscale x 2 x double> %passthru, <vscale x 2 x float> %y, i64 1, i64 %vl.zext)
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_x_f_qf.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_x_f_qf.ll
index b44b57394321..3c19616576f5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_x_f_qf.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_x_f_qf.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.nxv1i8.nxv1f32.iXLen(
 define <vscale x 1 x i8> @intrinsic_sf_vfnrclip_x_f_qf_nxv1i8_nxv1f32(<vscale x 1 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_x_f_qf_nxv1i8_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.x.f.qf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -39,10 +39,10 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.mask.nxv1i8.nxv1f32.iXL
 define <vscale x 1 x i8> @intrinsic_sf_vfnrclip_x_f_qf_mask_nxv1i8_nxv1f32(<vscale x 1 x i8> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_x_f_qf_mask_nxv1i8_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.x.f.qf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.mask.nxv1i8.nxv1f32.iXLen(
@@ -64,10 +64,10 @@ declare <vscale x 2 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.nxv2i8.nxv2f32.iXLen(
 define <vscale x 2 x i8> @intrinsic_sf_vfnrclip_x_f_qf_nxv2i8_nxv2f32(<vscale x 2 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_x_f_qf_nxv2i8_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.x.f.qf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -90,10 +90,10 @@ declare <vscale x 2 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.mask.nxv2i8.nxv2f32.iXL
 define <vscale x 2 x i8> @intrinsic_sf_vfnrclip_x_f_qf_mask_nxv2i8_nxv2f32(<vscale x 2 x i8> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_x_f_qf_mask_nxv2i8_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.x.f.qf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.mask.nxv2i8.nxv2f32.iXLen(
@@ -115,10 +115,10 @@ declare <vscale x 4 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.nxv4i8.nxv4f32.iXLen(
 define <vscale x 4 x i8> @intrinsic_sf_vfnrclip_x_f_qf_nxv4i8_nxv4f32(<vscale x 4 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_x_f_qf_nxv4i8_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.x.f.qf v10, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -141,10 +141,10 @@ declare <vscale x 4 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.mask.nxv4i8.nxv4f32.iXL
 define <vscale x 4 x i8> @intrinsic_sf_vfnrclip_x_f_qf_mask_nxv4i8_nxv4f32(<vscale x 4 x i8> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_x_f_qf_mask_nxv4i8_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.x.f.qf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.mask.nxv4i8.nxv4f32.iXLen(
@@ -166,10 +166,10 @@ declare <vscale x 8 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.nxv8i8.nxv8f32.iXLen(
 define <vscale x 8 x i8> @intrinsic_sf_vfnrclip_x_f_qf_nxv8i8_nxv8f32(<vscale x 8 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_x_f_qf_nxv8i8_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.x.f.qf v12, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -192,10 +192,10 @@ declare <vscale x 8 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.mask.nxv8i8.nxv8f32.iXL
 define <vscale x 8 x i8> @intrinsic_sf_vfnrclip_x_f_qf_mask_nxv8i8_nxv8f32(<vscale x 8 x i8> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_x_f_qf_mask_nxv8i8_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.x.f.qf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.mask.nxv8i8.nxv8f32.iXLen(
@@ -217,10 +217,10 @@ declare <vscale x 16 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.nxv16i8.nxv16f32.iXLen
 define <vscale x 16 x i8> @intrinsic_sf_vfnrclip_x_f_qf_nxv16i8_nxv16f32(<vscale x 16 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_x_f_qf_nxv16i8_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.x.f.qf v16, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -243,10 +243,10 @@ declare <vscale x 16 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.mask.nxv16i8.nxv16f32.
 define <vscale x 16 x i8> @intrinsic_sf_vfnrclip_x_f_qf_mask_nxv16i8_nxv16f32(<vscale x 16 x i8> %0, <vscale x 16 x float> %1, float %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_x_f_qf_mask_nxv16i8_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.x.f.qf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.sf.vfnrclip.x.f.qf.mask.nxv16i8.nxv16f32.iXLen(
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_xu_f_qf.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_xu_f_qf.ll
index bc2f7ca7dc86..dbcee311c6e3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_xu_f_qf.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vfnrclip_xu_f_qf.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.nxv1i8.nxv1f32.iXLen(
 define <vscale x 1 x i8> @intrinsic_sf_vfnrclip_xu_f_qf_nxv1i8_nxv1f32(<vscale x 1 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_xu_f_qf_nxv1i8_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.xu.f.qf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -39,10 +39,10 @@ declare <vscale x 1 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.mask.nxv1i8.nxv1f32.iX
 define <vscale x 1 x i8> @intrinsic_sf_vfnrclip_xu_f_qf_mask_nxv1i8_nxv1f32(<vscale x 1 x i8> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_xu_f_qf_mask_nxv1i8_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.xu.f.qf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.mask.nxv1i8.nxv1f32.iXLen(
@@ -64,10 +64,10 @@ declare <vscale x 2 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.nxv2i8.nxv2f32.iXLen(
 define <vscale x 2 x i8> @intrinsic_sf_vfnrclip_xu_f_qf_nxv2i8_nxv2f32(<vscale x 2 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_xu_f_qf_nxv2i8_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.xu.f.qf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -90,10 +90,10 @@ declare <vscale x 2 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.mask.nxv2i8.nxv2f32.iX
 define <vscale x 2 x i8> @intrinsic_sf_vfnrclip_xu_f_qf_mask_nxv2i8_nxv2f32(<vscale x 2 x i8> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_xu_f_qf_mask_nxv2i8_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.xu.f.qf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.mask.nxv2i8.nxv2f32.iXLen(
@@ -115,10 +115,10 @@ declare <vscale x 4 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.nxv4i8.nxv4f32.iXLen(
 define <vscale x 4 x i8> @intrinsic_sf_vfnrclip_xu_f_qf_nxv4i8_nxv4f32(<vscale x 4 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_xu_f_qf_nxv4i8_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.xu.f.qf v10, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -141,10 +141,10 @@ declare <vscale x 4 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.mask.nxv4i8.nxv4f32.iX
 define <vscale x 4 x i8> @intrinsic_sf_vfnrclip_xu_f_qf_mask_nxv4i8_nxv4f32(<vscale x 4 x i8> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_xu_f_qf_mask_nxv4i8_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.xu.f.qf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.mask.nxv4i8.nxv4f32.iXLen(
@@ -166,10 +166,10 @@ declare <vscale x 8 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.nxv8i8.nxv8f32.iXLen(
 define <vscale x 8 x i8> @intrinsic_sf_vfnrclip_xu_f_qf_nxv8i8_nxv8f32(<vscale x 8 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_xu_f_qf_nxv8i8_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.xu.f.qf v12, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -192,10 +192,10 @@ declare <vscale x 8 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.mask.nxv8i8.nxv8f32.iX
 define <vscale x 8 x i8> @intrinsic_sf_vfnrclip_xu_f_qf_mask_nxv8i8_nxv8f32(<vscale x 8 x i8> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_xu_f_qf_mask_nxv8i8_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.xu.f.qf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.mask.nxv8i8.nxv8f32.iXLen(
@@ -217,10 +217,10 @@ declare <vscale x 16 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.nxv16i8.nxv16f32.iXLe
 define <vscale x 16 x i8> @intrinsic_sf_vfnrclip_xu_f_qf_nxv16i8_nxv16f32(<vscale x 16 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_xu_f_qf_nxv16i8_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.xu.f.qf v16, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -243,10 +243,10 @@ declare <vscale x 16 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.mask.nxv16i8.nxv16f32
 define <vscale x 16 x i8> @intrinsic_sf_vfnrclip_xu_f_qf_mask_nxv16i8_nxv16f32(<vscale x 16 x i8> %0, <vscale x 16 x float> %1, float %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_sf_vfnrclip_xu_f_qf_mask_nxv16i8_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    sf.vfnrclip.xu.f.qf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.sf.vfnrclip.xu.f.qf.mask.nxv16i8.nxv16f32.iXLen(
diff --git a/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll b/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
index 2926a23c8b27..25e3468dcb62 100644
--- a/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
@@ -110,8 +110,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaadd.rm.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vaadd_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -133,8 +133,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaaddu.rm.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vaaddu_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -199,8 +199,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vasub.rm.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vasub_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vasub.vv v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -222,8 +222,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vasubu.rm.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vasubu_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vasubu.vv v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -950,8 +950,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vnclip_wv_nxv1i8_nxv1i16_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv1i8_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vnclip.wv v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -974,8 +974,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vnclipu_wv_nxv1i8_nxv1i16_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv1i8_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vnclipu.wv v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -1271,8 +1271,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -1301,16 +1301,16 @@ define <vscale x 1 x i64> @intrinsic_vsmul_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
 ; RV32-NEXT:    vlse64.v v10, (a0), zero
-; RV32-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
 ; RV32-NEXT:    csrwi vxrm, 0
+; RV32-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
 ; RV32-NEXT:    vsmul.vv v8, v9, v10
 ; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: intrinsic_vsmul_vx_nxv1i64_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, tu, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, tu, ma
 ; RV64-NEXT:    vsmul.vx v8, v9, a0
 ; RV64-NEXT:    ret
 entry:
@@ -1376,8 +1376,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vssra_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vssra.vv v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -1400,8 +1400,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vssrl_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vssrl.vv v8, v9, v10
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vaadd.ll b/llvm/test/CodeGen/RISCV/rvv/vaadd.ll
index 82cd4bf162b9..096e60b6285f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vaadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vaadd.ll
@@ -13,8 +13,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaadd.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vaadd_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -37,8 +37,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaadd.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vaadd_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -61,8 +61,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vaadd.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vaadd_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -85,8 +85,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vaadd.mask.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vaadd_mask_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, <vscale x 2 x i8> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -109,8 +109,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vaadd.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vaadd_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -133,8 +133,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vaadd.mask.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vaadd_mask_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, <vscale x 4 x i8> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -157,8 +157,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vaadd.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vaadd_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -181,8 +181,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vaadd.mask.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vaadd_mask_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -205,8 +205,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vaadd.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vaadd_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -229,8 +229,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vaadd.mask.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vaadd_mask_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <vscale x 16 x i8> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -253,8 +253,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vaadd.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vaadd_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -277,8 +277,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vaadd.mask.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vaadd_mask_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, <vscale x 32 x i8> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -301,8 +301,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vaadd.nxv64i8.nxv64i8(
 define <vscale x 64 x i8> @intrinsic_vaadd_vv_nxv64i8_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -326,8 +326,8 @@ define <vscale x 64 x i8> @intrinsic_vaadd_mask_vv_nxv64i8_nxv64i8_nxv64i8(<vsca
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8r.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -350,8 +350,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vaadd.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vaadd_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -374,8 +374,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vaadd.mask.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vaadd_mask_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, <vscale x 1 x i16> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -398,8 +398,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vaadd.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vaadd_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -422,8 +422,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vaadd.mask.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vaadd_mask_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, <vscale x 2 x i16> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -446,8 +446,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vaadd.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vaadd_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -470,8 +470,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vaadd.mask.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vaadd_mask_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, <vscale x 4 x i16> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -494,8 +494,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vaadd.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vaadd_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -518,8 +518,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vaadd.mask.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vaadd_mask_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <vscale x 8 x i16> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -542,8 +542,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vaadd.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vaadd_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -566,8 +566,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vaadd.mask.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vaadd_mask_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, <vscale x 16 x i16> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -590,8 +590,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vaadd.nxv32i16.nxv32i16(
 define <vscale x 32 x i16> @intrinsic_vaadd_vv_nxv32i16_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -615,8 +615,8 @@ define <vscale x 32 x i16> @intrinsic_vaadd_mask_vv_nxv32i16_nxv32i16_nxv32i16(<
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -639,8 +639,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vaadd.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vaadd_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -663,8 +663,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vaadd.mask.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vaadd_mask_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, <vscale x 1 x i32> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -687,8 +687,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vaadd.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vaadd_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -711,8 +711,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vaadd.mask.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vaadd_mask_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, <vscale x 2 x i32> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -735,8 +735,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vaadd.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vaadd_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -759,8 +759,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vaadd.mask.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vaadd_mask_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <vscale x 4 x i32> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -783,8 +783,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vaadd.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vaadd_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -807,8 +807,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vaadd.mask.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vaadd_mask_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, <vscale x 8 x i32> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -831,8 +831,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vaadd.nxv16i32.nxv16i32(
 define <vscale x 16 x i32> @intrinsic_vaadd_vv_nxv16i32_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -856,8 +856,8 @@ define <vscale x 16 x i32> @intrinsic_vaadd_mask_vv_nxv16i32_nxv16i32_nxv16i32(<
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -880,8 +880,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vaadd.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vaadd_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -904,8 +904,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vaadd.mask.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vaadd_mask_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i64> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -928,8 +928,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vaadd.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vaadd_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -952,8 +952,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vaadd.mask.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vaadd_mask_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -976,8 +976,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vaadd.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vaadd_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -1000,8 +1000,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vaadd.mask.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vaadd_mask_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, <vscale x 4 x i64> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1024,8 +1024,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vaadd.nxv8i64.nxv8i64(
 define <vscale x 8 x i64> @intrinsic_vaadd_vv_nxv8i64_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -1049,8 +1049,8 @@ define <vscale x 8 x i64> @intrinsic_vaadd_mask_vv_nxv8i64_nxv8i64_nxv8i64(<vsca
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    vaadd.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1073,8 +1073,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaadd.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vaadd_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1097,8 +1097,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaadd.mask.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vaadd_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1121,8 +1121,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vaadd.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vaadd_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1145,8 +1145,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vaadd.mask.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vaadd_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1169,8 +1169,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vaadd.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vaadd_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1193,8 +1193,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vaadd.mask.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vaadd_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1217,8 +1217,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vaadd.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vaadd_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1241,8 +1241,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vaadd.mask.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vaadd_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1265,8 +1265,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vaadd.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vaadd_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1289,8 +1289,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vaadd.mask.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vaadd_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1313,8 +1313,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vaadd.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vaadd_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1337,8 +1337,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vaadd.mask.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vaadd_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1361,8 +1361,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vaadd.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vaadd_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1385,8 +1385,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vaadd.mask.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vaadd_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1409,8 +1409,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vaadd.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vaadd_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1433,8 +1433,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vaadd.mask.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vaadd_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1457,8 +1457,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vaadd.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vaadd_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1481,8 +1481,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vaadd.mask.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vaadd_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1505,8 +1505,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vaadd.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vaadd_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1529,8 +1529,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vaadd.mask.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vaadd_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1553,8 +1553,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vaadd.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vaadd_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1577,8 +1577,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vaadd.mask.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vaadd_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1601,8 +1601,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vaadd.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vaadd_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1625,8 +1625,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vaadd.mask.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vaadd_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1649,8 +1649,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vaadd.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vaadd_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1673,8 +1673,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vaadd.mask.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vaadd_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1697,8 +1697,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vaadd.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vaadd_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1721,8 +1721,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vaadd.mask.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vaadd_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1745,8 +1745,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vaadd.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vaadd_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1769,8 +1769,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vaadd.mask.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vaadd_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1793,8 +1793,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vaadd.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vaadd_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1817,8 +1817,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vaadd.mask.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vaadd_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1841,8 +1841,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vaadd.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vaadd_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1865,8 +1865,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vaadd.mask.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vaadd_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1889,8 +1889,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vaadd.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vaadd_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vaadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1913,8 +1913,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vaadd.mask.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vaadd_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vaadd.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1950,8 +1950,8 @@ define <vscale x 1 x i64> @intrinsic_vaadd_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ;
 ; RV64-LABEL: intrinsic_vaadd_vx_nxv1i64_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; RV64-NEXT:    vaadd.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -1987,8 +1987,8 @@ define <vscale x 1 x i64> @intrinsic_vaadd_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ;
 ; RV64-LABEL: intrinsic_vaadd_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vaadd.vx v8, v9, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2024,8 +2024,8 @@ define <vscale x 2 x i64> @intrinsic_vaadd_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ;
 ; RV64-LABEL: intrinsic_vaadd_vx_nxv2i64_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; RV64-NEXT:    vaadd.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2061,8 +2061,8 @@ define <vscale x 2 x i64> @intrinsic_vaadd_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ;
 ; RV64-LABEL: intrinsic_vaadd_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vaadd.vx v8, v10, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2098,8 +2098,8 @@ define <vscale x 4 x i64> @intrinsic_vaadd_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ;
 ; RV64-LABEL: intrinsic_vaadd_vx_nxv4i64_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    vaadd.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2135,8 +2135,8 @@ define <vscale x 4 x i64> @intrinsic_vaadd_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ;
 ; RV64-LABEL: intrinsic_vaadd_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vaadd.vx v8, v12, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2172,8 +2172,8 @@ define <vscale x 8 x i64> @intrinsic_vaadd_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ;
 ; RV64-LABEL: intrinsic_vaadd_vx_nxv8i64_nxv8i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    vaadd.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2209,8 +2209,8 @@ define <vscale x 8 x i64> @intrinsic_vaadd_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ;
 ; RV64-LABEL: intrinsic_vaadd_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; RV64-NEXT:    vaadd.vx v8, v16, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vaaddu-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vaaddu-sdnode.ll
index 5b14014a252f..dd2c14b037ee 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vaaddu-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vaaddu-sdnode.ll
@@ -5,8 +5,8 @@
 define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_floor(<vscale x 8 x i8> %x, <vscale x 8 x i8> %y) {
 ; CHECK-LABEL: vaaddu_vv_nxv8i8_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i8> %x to <vscale x 8 x i16>
@@ -20,8 +20,8 @@ define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_floor(<vscale x 8 x i8> %x, <vscale x
 define <vscale x 8 x i8> @vaaddu_vx_nxv8i8_floor(<vscale x 8 x i8> %x, i8 %y) {
 ; CHECK-LABEL: vaaddu_vx_nxv8i8_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i8> %x to <vscale x 8 x i16>
@@ -52,8 +52,8 @@ define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_floor_sexti16(<vscale x 8 x i8> %x, <
 define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_floor_zexti32(<vscale x 8 x i8> %x, <vscale x 8 x i8> %y) {
 ; CHECK-LABEL: vaaddu_vv_nxv8i8_floor_zexti32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i8> %x to <vscale x 8 x i32>
@@ -82,8 +82,8 @@ define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_floor_lshr2(<vscale x 8 x i8> %x, <vs
 define <vscale x 8 x i16> @vaaddu_vv_nxv8i16_floor(<vscale x 8 x i16> %x, <vscale x 8 x i16> %y) {
 ; CHECK-LABEL: vaaddu_vv_nxv8i16_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i16> %x to <vscale x 8 x i32>
@@ -97,8 +97,8 @@ define <vscale x 8 x i16> @vaaddu_vv_nxv8i16_floor(<vscale x 8 x i16> %x, <vscal
 define <vscale x 8 x i16> @vaaddu_vx_nxv8i16_floor(<vscale x 8 x i16> %x, i16 %y) {
 ; CHECK-LABEL: vaaddu_vx_nxv8i16_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i16> %x to <vscale x 8 x i32>
@@ -114,8 +114,8 @@ define <vscale x 8 x i16> @vaaddu_vx_nxv8i16_floor(<vscale x 8 x i16> %x, i16 %y
 define <vscale x 8 x i32> @vaaddu_vv_nxv8i32_floor(<vscale x 8 x i32> %x, <vscale x 8 x i32> %y) {
 ; CHECK-LABEL: vaaddu_vv_nxv8i32_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i32> %x to <vscale x 8 x i64>
@@ -129,8 +129,8 @@ define <vscale x 8 x i32> @vaaddu_vv_nxv8i32_floor(<vscale x 8 x i32> %x, <vscal
 define <vscale x 8 x i32> @vaaddu_vx_nxv8i32_floor(<vscale x 8 x i32> %x, i32 %y) {
 ; CHECK-LABEL: vaaddu_vx_nxv8i32_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i32> %x to <vscale x 8 x i64>
@@ -146,8 +146,8 @@ define <vscale x 8 x i32> @vaaddu_vx_nxv8i32_floor(<vscale x 8 x i32> %x, i32 %y
 define <vscale x 8 x i64> @vaaddu_vv_nxv8i64_floor(<vscale x 8 x i64> %x, <vscale x 8 x i64> %y) {
 ; CHECK-LABEL: vaaddu_vv_nxv8i64_floor:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i64> %x to <vscale x 8 x i128>
@@ -175,8 +175,8 @@ define <vscale x 8 x i64> @vaaddu_vx_nxv8i64_floor(<vscale x 8 x i64> %x, i64 %y
 ;
 ; RV64-LABEL: vaaddu_vx_nxv8i64_floor:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 2
+; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vaaddu.vx v8, v8, a0
 ; RV64-NEXT:    ret
   %xzv = zext <vscale x 8 x i64> %x to <vscale x 8 x i128>
@@ -192,8 +192,8 @@ define <vscale x 8 x i64> @vaaddu_vx_nxv8i64_floor(<vscale x 8 x i64> %x, i64 %y
 define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_ceil(<vscale x 8 x i8> %x, <vscale x 8 x i8> %y) {
 ; CHECK-LABEL: vaaddu_vv_nxv8i8_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i8> %x to <vscale x 8 x i16>
@@ -208,8 +208,8 @@ define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_ceil(<vscale x 8 x i8> %x, <vscale x
 define <vscale x 8 x i8> @vaaddu_vx_nxv8i8_ceil(<vscale x 8 x i8> %x, i8 %y) {
 ; CHECK-LABEL: vaaddu_vx_nxv8i8_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i8> %x to <vscale x 8 x i16>
@@ -245,8 +245,8 @@ define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_ceil_sexti16(<vscale x 8 x i8> %x, <v
 define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_ceil_zexti32(<vscale x 8 x i8> %x, <vscale x 8 x i8> %y) {
 ; CHECK-LABEL: vaaddu_vv_nxv8i8_ceil_zexti32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i8> %x to <vscale x 8 x i32>
@@ -299,8 +299,8 @@ define <vscale x 8 x i8> @vaaddu_vv_nxv8i8_ceil_add2(<vscale x 8 x i8> %x, <vsca
 define <vscale x 8 x i16> @vaaddu_vv_nxv8i16_ceil(<vscale x 8 x i16> %x, <vscale x 8 x i16> %y) {
 ; CHECK-LABEL: vaaddu_vv_nxv8i16_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i16> %x to <vscale x 8 x i32>
@@ -315,8 +315,8 @@ define <vscale x 8 x i16> @vaaddu_vv_nxv8i16_ceil(<vscale x 8 x i16> %x, <vscale
 define <vscale x 8 x i16> @vaaddu_vx_nxv8i16_ceil(<vscale x 8 x i16> %x, i16 %y) {
 ; CHECK-LABEL: vaaddu_vx_nxv8i16_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i16> %x to <vscale x 8 x i32>
@@ -333,8 +333,8 @@ define <vscale x 8 x i16> @vaaddu_vx_nxv8i16_ceil(<vscale x 8 x i16> %x, i16 %y)
 define <vscale x 8 x i32> @vaaddu_vv_nxv8i32_ceil(<vscale x 8 x i32> %x, <vscale x 8 x i32> %y) {
 ; CHECK-LABEL: vaaddu_vv_nxv8i32_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i32> %x to <vscale x 8 x i64>
@@ -349,8 +349,8 @@ define <vscale x 8 x i32> @vaaddu_vv_nxv8i32_ceil(<vscale x 8 x i32> %x, <vscale
 define <vscale x 8 x i32> @vaaddu_vx_nxv8i32_ceil(<vscale x 8 x i32> %x, i32 %y) {
 ; CHECK-LABEL: vaaddu_vx_nxv8i32_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i32> %x to <vscale x 8 x i64>
@@ -367,8 +367,8 @@ define <vscale x 8 x i32> @vaaddu_vx_nxv8i32_ceil(<vscale x 8 x i32> %x, i32 %y)
 define <vscale x 8 x i64> @vaaddu_vv_nxv8i64_ceil(<vscale x 8 x i64> %x, <vscale x 8 x i64> %y) {
 ; CHECK-LABEL: vaaddu_vv_nxv8i64_ceil:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v16
 ; CHECK-NEXT:    ret
   %xzv = zext <vscale x 8 x i64> %x to <vscale x 8 x i128>
@@ -397,8 +397,8 @@ define <vscale x 8 x i64> @vaaddu_vx_nxv8i64_ceil(<vscale x 8 x i64> %x, i64 %y)
 ;
 ; RV64-LABEL: vaaddu_vx_nxv8i64_ceil:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vaaddu.vx v8, v8, a0
 ; RV64-NEXT:    ret
   %xzv = zext <vscale x 8 x i64> %x to <vscale x 8 x i128>
diff --git a/llvm/test/CodeGen/RISCV/rvv/vaaddu.ll b/llvm/test/CodeGen/RISCV/rvv/vaaddu.ll
index eba87d7061d3..a15a1932360a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vaaddu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vaaddu.ll
@@ -13,8 +13,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaaddu.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vaaddu_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -37,8 +37,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaaddu.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vaaddu_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -61,8 +61,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vaaddu.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vaaddu_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -85,8 +85,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vaaddu.mask.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vaaddu_mask_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, <vscale x 2 x i8> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -109,8 +109,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vaaddu.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vaaddu_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -133,8 +133,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vaaddu.mask.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vaaddu_mask_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, <vscale x 4 x i8> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -157,8 +157,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vaaddu.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vaaddu_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -181,8 +181,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vaaddu.mask.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vaaddu_mask_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -205,8 +205,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vaaddu.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vaaddu_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -229,8 +229,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vaaddu.mask.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vaaddu_mask_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <vscale x 16 x i8> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -253,8 +253,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vaaddu.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vaaddu_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -277,8 +277,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vaaddu.mask.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vaaddu_mask_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, <vscale x 32 x i8> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -301,8 +301,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vaaddu.nxv64i8.nxv64i8(
 define <vscale x 64 x i8> @intrinsic_vaaddu_vv_nxv64i8_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -326,8 +326,8 @@ define <vscale x 64 x i8> @intrinsic_vaaddu_mask_vv_nxv64i8_nxv64i8_nxv64i8(<vsc
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8r.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -350,8 +350,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vaaddu.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vaaddu_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -374,8 +374,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vaaddu.mask.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vaaddu_mask_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, <vscale x 1 x i16> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -398,8 +398,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vaaddu.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vaaddu_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -422,8 +422,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vaaddu.mask.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vaaddu_mask_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, <vscale x 2 x i16> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -446,8 +446,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vaaddu.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vaaddu_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -470,8 +470,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vaaddu.mask.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vaaddu_mask_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, <vscale x 4 x i16> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -494,8 +494,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vaaddu.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vaaddu_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -518,8 +518,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vaaddu.mask.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vaaddu_mask_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <vscale x 8 x i16> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -542,8 +542,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vaaddu.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vaaddu_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -566,8 +566,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vaaddu.mask.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vaaddu_mask_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, <vscale x 16 x i16> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -590,8 +590,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vaaddu.nxv32i16.nxv32i16(
 define <vscale x 32 x i16> @intrinsic_vaaddu_vv_nxv32i16_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -615,8 +615,8 @@ define <vscale x 32 x i16> @intrinsic_vaaddu_mask_vv_nxv32i16_nxv32i16_nxv32i16(
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -639,8 +639,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vaaddu.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vaaddu_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -663,8 +663,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vaaddu.mask.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vaaddu_mask_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, <vscale x 1 x i32> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -687,8 +687,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vaaddu.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vaaddu_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -711,8 +711,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vaaddu.mask.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vaaddu_mask_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, <vscale x 2 x i32> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -735,8 +735,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vaaddu.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vaaddu_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -759,8 +759,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vaaddu.mask.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vaaddu_mask_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <vscale x 4 x i32> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -783,8 +783,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vaaddu.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vaaddu_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -807,8 +807,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vaaddu.mask.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vaaddu_mask_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, <vscale x 8 x i32> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -831,8 +831,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vaaddu.nxv16i32.nxv16i32(
 define <vscale x 16 x i32> @intrinsic_vaaddu_vv_nxv16i32_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -856,8 +856,8 @@ define <vscale x 16 x i32> @intrinsic_vaaddu_mask_vv_nxv16i32_nxv16i32_nxv16i32(
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -880,8 +880,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vaaddu.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vaaddu_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -904,8 +904,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vaaddu.mask.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vaaddu_mask_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i64> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -928,8 +928,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vaaddu.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vaaddu_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -952,8 +952,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vaaddu.mask.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vaaddu_mask_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -976,8 +976,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vaaddu.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vaaddu_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -1000,8 +1000,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vaaddu.mask.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vaaddu_mask_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, <vscale x 4 x i64> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1024,8 +1024,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vaaddu.nxv8i64.nxv8i64(
 define <vscale x 8 x i64> @intrinsic_vaaddu_vv_nxv8i64_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vaaddu.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -1049,8 +1049,8 @@ define <vscale x 8 x i64> @intrinsic_vaaddu_mask_vv_nxv8i64_nxv8i64_nxv8i64(<vsc
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    vaaddu.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1073,8 +1073,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaaddu.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vaaddu_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1097,8 +1097,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vaaddu.mask.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vaaddu_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1121,8 +1121,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vaaddu.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vaaddu_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1145,8 +1145,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vaaddu.mask.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vaaddu_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1169,8 +1169,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vaaddu.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vaaddu_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1193,8 +1193,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vaaddu.mask.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vaaddu_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1217,8 +1217,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vaaddu.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vaaddu_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1241,8 +1241,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vaaddu.mask.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vaaddu_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1265,8 +1265,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vaaddu.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vaaddu_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1289,8 +1289,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vaaddu.mask.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vaaddu_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1313,8 +1313,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vaaddu.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vaaddu_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1337,8 +1337,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vaaddu.mask.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vaaddu_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1361,8 +1361,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vaaddu.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vaaddu_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1385,8 +1385,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vaaddu.mask.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vaaddu_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1409,8 +1409,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vaaddu.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vaaddu_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1433,8 +1433,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vaaddu.mask.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vaaddu_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1457,8 +1457,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vaaddu.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vaaddu_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1481,8 +1481,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vaaddu.mask.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vaaddu_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1505,8 +1505,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vaaddu.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vaaddu_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1529,8 +1529,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vaaddu.mask.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vaaddu_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1553,8 +1553,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vaaddu.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vaaddu_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1577,8 +1577,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vaaddu.mask.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vaaddu_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1601,8 +1601,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vaaddu.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vaaddu_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1625,8 +1625,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vaaddu.mask.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vaaddu_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1649,8 +1649,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vaaddu.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vaaddu_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1673,8 +1673,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vaaddu.mask.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vaaddu_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1697,8 +1697,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vaaddu.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vaaddu_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1721,8 +1721,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vaaddu.mask.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vaaddu_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1745,8 +1745,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vaaddu.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vaaddu_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1769,8 +1769,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vaaddu.mask.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vaaddu_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1793,8 +1793,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vaaddu.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vaaddu_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1817,8 +1817,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vaaddu.mask.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vaaddu_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1841,8 +1841,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vaaddu.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vaaddu_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1865,8 +1865,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vaaddu.mask.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vaaddu_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1889,8 +1889,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vaaddu.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vaaddu_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vaaddu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1913,8 +1913,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vaaddu.mask.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vaaddu_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vaaddu_mask_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vaaddu.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1950,8 +1950,8 @@ define <vscale x 1 x i64> @intrinsic_vaaddu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ;
 ; RV64-LABEL: intrinsic_vaaddu_vx_nxv1i64_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; RV64-NEXT:    vaaddu.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -1987,8 +1987,8 @@ define <vscale x 1 x i64> @intrinsic_vaaddu_mask_vx_nxv1i64_nxv1i64_i64(<vscale
 ;
 ; RV64-LABEL: intrinsic_vaaddu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vaaddu.vx v8, v9, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2024,8 +2024,8 @@ define <vscale x 2 x i64> @intrinsic_vaaddu_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ;
 ; RV64-LABEL: intrinsic_vaaddu_vx_nxv2i64_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; RV64-NEXT:    vaaddu.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2061,8 +2061,8 @@ define <vscale x 2 x i64> @intrinsic_vaaddu_mask_vx_nxv2i64_nxv2i64_i64(<vscale
 ;
 ; RV64-LABEL: intrinsic_vaaddu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vaaddu.vx v8, v10, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2098,8 +2098,8 @@ define <vscale x 4 x i64> @intrinsic_vaaddu_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ;
 ; RV64-LABEL: intrinsic_vaaddu_vx_nxv4i64_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    vaaddu.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2135,8 +2135,8 @@ define <vscale x 4 x i64> @intrinsic_vaaddu_mask_vx_nxv4i64_nxv4i64_i64(<vscale
 ;
 ; RV64-LABEL: intrinsic_vaaddu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vaaddu.vx v8, v12, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2172,8 +2172,8 @@ define <vscale x 8 x i64> @intrinsic_vaaddu_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ;
 ; RV64-LABEL: intrinsic_vaaddu_vx_nxv8i64_nxv8i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    vaaddu.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2209,8 +2209,8 @@ define <vscale x 8 x i64> @intrinsic_vaaddu_mask_vx_nxv8i64_nxv8i64_i64(<vscale
 ;
 ; RV64-LABEL: intrinsic_vaaddu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; RV64-NEXT:    vaaddu.vx v8, v16, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vasub.ll b/llvm/test/CodeGen/RISCV/rvv/vasub.ll
index d69910efb0ee..c3cb66f7f230 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vasub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vasub.ll
@@ -13,8 +13,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vasub.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vasub_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -37,8 +37,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vasub.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vasub_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -61,8 +61,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vasub.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vasub_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -85,8 +85,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vasub.mask.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vasub_mask_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, <vscale x 2 x i8> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -109,8 +109,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vasub.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vasub_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -133,8 +133,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vasub.mask.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vasub_mask_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, <vscale x 4 x i8> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -157,8 +157,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vasub.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vasub_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -181,8 +181,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vasub.mask.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vasub_mask_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -205,8 +205,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vasub.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vasub_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -229,8 +229,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vasub.mask.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vasub_mask_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <vscale x 16 x i8> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -253,8 +253,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vasub.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vasub_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -277,8 +277,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vasub.mask.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vasub_mask_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, <vscale x 32 x i8> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -301,8 +301,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vasub.nxv64i8.nxv64i8(
 define <vscale x 64 x i8> @intrinsic_vasub_vv_nxv64i8_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -326,8 +326,8 @@ define <vscale x 64 x i8> @intrinsic_vasub_mask_vv_nxv64i8_nxv64i8_nxv64i8(<vsca
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8r.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -350,8 +350,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vasub.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vasub_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -374,8 +374,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vasub.mask.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vasub_mask_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, <vscale x 1 x i16> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -398,8 +398,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vasub.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vasub_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -422,8 +422,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vasub.mask.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vasub_mask_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, <vscale x 2 x i16> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -446,8 +446,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vasub.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vasub_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -470,8 +470,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vasub.mask.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vasub_mask_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, <vscale x 4 x i16> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -494,8 +494,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vasub.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vasub_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -518,8 +518,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vasub.mask.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vasub_mask_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <vscale x 8 x i16> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -542,8 +542,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vasub.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vasub_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -566,8 +566,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vasub.mask.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vasub_mask_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, <vscale x 16 x i16> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -590,8 +590,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vasub.nxv32i16.nxv32i16(
 define <vscale x 32 x i16> @intrinsic_vasub_vv_nxv32i16_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -615,8 +615,8 @@ define <vscale x 32 x i16> @intrinsic_vasub_mask_vv_nxv32i16_nxv32i16_nxv32i16(<
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -639,8 +639,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vasub.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vasub_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -663,8 +663,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vasub.mask.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vasub_mask_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, <vscale x 1 x i32> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -687,8 +687,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vasub.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vasub_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -711,8 +711,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vasub.mask.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vasub_mask_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, <vscale x 2 x i32> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -735,8 +735,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vasub.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vasub_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -759,8 +759,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vasub.mask.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vasub_mask_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <vscale x 4 x i32> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -783,8 +783,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vasub.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vasub_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -807,8 +807,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vasub.mask.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vasub_mask_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, <vscale x 8 x i32> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -831,8 +831,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vasub.nxv16i32.nxv16i32(
 define <vscale x 16 x i32> @intrinsic_vasub_vv_nxv16i32_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -856,8 +856,8 @@ define <vscale x 16 x i32> @intrinsic_vasub_mask_vv_nxv16i32_nxv16i32_nxv16i32(<
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -880,8 +880,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vasub.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vasub_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -904,8 +904,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vasub.mask.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vasub_mask_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i64> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -928,8 +928,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vasub.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vasub_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -952,8 +952,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vasub.mask.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vasub_mask_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -976,8 +976,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vasub.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vasub_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -1000,8 +1000,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vasub.mask.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vasub_mask_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, <vscale x 4 x i64> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1024,8 +1024,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vasub.nxv8i64.nxv8i64(
 define <vscale x 8 x i64> @intrinsic_vasub_vv_nxv8i64_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vasub.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -1049,8 +1049,8 @@ define <vscale x 8 x i64> @intrinsic_vasub_mask_vv_nxv8i64_nxv8i64_nxv8i64(<vsca
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    vasub.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1073,8 +1073,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vasub.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vasub_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1097,8 +1097,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vasub.mask.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vasub_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1121,8 +1121,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vasub.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vasub_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1145,8 +1145,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vasub.mask.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vasub_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1169,8 +1169,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vasub.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vasub_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1193,8 +1193,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vasub.mask.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vasub_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1217,8 +1217,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vasub.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vasub_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1241,8 +1241,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vasub.mask.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vasub_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1265,8 +1265,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vasub.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vasub_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1289,8 +1289,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vasub.mask.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vasub_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1313,8 +1313,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vasub.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vasub_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1337,8 +1337,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vasub.mask.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vasub_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1361,8 +1361,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vasub.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vasub_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1385,8 +1385,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vasub.mask.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vasub_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1409,8 +1409,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vasub.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vasub_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1433,8 +1433,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vasub.mask.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vasub_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1457,8 +1457,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vasub.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vasub_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1481,8 +1481,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vasub.mask.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vasub_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1505,8 +1505,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vasub.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vasub_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1529,8 +1529,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vasub.mask.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vasub_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1553,8 +1553,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vasub.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vasub_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1577,8 +1577,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vasub.mask.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vasub_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1601,8 +1601,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vasub.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vasub_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1625,8 +1625,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vasub.mask.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vasub_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1649,8 +1649,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vasub.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vasub_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1673,8 +1673,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vasub.mask.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vasub_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1697,8 +1697,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vasub.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vasub_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1721,8 +1721,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vasub.mask.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vasub_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1745,8 +1745,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vasub.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vasub_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1769,8 +1769,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vasub.mask.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vasub_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1793,8 +1793,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vasub.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vasub_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1817,8 +1817,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vasub.mask.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vasub_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1841,8 +1841,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vasub.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vasub_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1865,8 +1865,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vasub.mask.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vasub_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1889,8 +1889,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vasub.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vasub_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vasub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1913,8 +1913,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vasub.mask.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vasub_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vasub.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1950,8 +1950,8 @@ define <vscale x 1 x i64> @intrinsic_vasub_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ;
 ; RV64-LABEL: intrinsic_vasub_vx_nxv1i64_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; RV64-NEXT:    vasub.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -1987,8 +1987,8 @@ define <vscale x 1 x i64> @intrinsic_vasub_mask_vx_nxv1i64_nxv1i64_i64(<vscale x
 ;
 ; RV64-LABEL: intrinsic_vasub_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vasub.vx v8, v9, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2024,8 +2024,8 @@ define <vscale x 2 x i64> @intrinsic_vasub_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ;
 ; RV64-LABEL: intrinsic_vasub_vx_nxv2i64_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; RV64-NEXT:    vasub.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2061,8 +2061,8 @@ define <vscale x 2 x i64> @intrinsic_vasub_mask_vx_nxv2i64_nxv2i64_i64(<vscale x
 ;
 ; RV64-LABEL: intrinsic_vasub_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vasub.vx v8, v10, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2098,8 +2098,8 @@ define <vscale x 4 x i64> @intrinsic_vasub_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ;
 ; RV64-LABEL: intrinsic_vasub_vx_nxv4i64_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    vasub.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2135,8 +2135,8 @@ define <vscale x 4 x i64> @intrinsic_vasub_mask_vx_nxv4i64_nxv4i64_i64(<vscale x
 ;
 ; RV64-LABEL: intrinsic_vasub_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vasub.vx v8, v12, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2172,8 +2172,8 @@ define <vscale x 8 x i64> @intrinsic_vasub_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ;
 ; RV64-LABEL: intrinsic_vasub_vx_nxv8i64_nxv8i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    vasub.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2209,8 +2209,8 @@ define <vscale x 8 x i64> @intrinsic_vasub_mask_vx_nxv8i64_nxv8i64_i64(<vscale x
 ;
 ; RV64-LABEL: intrinsic_vasub_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; RV64-NEXT:    vasub.vx v8, v16, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vasubu.ll b/llvm/test/CodeGen/RISCV/rvv/vasubu.ll
index 4228e067199f..6d790a9ce027 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vasubu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vasubu.ll
@@ -13,8 +13,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vasubu.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vasubu_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -37,8 +37,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vasubu.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vasubu_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -61,8 +61,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vasubu.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vasubu_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -85,8 +85,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vasubu.mask.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vasubu_mask_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, <vscale x 2 x i8> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -109,8 +109,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vasubu.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vasubu_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -133,8 +133,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vasubu.mask.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vasubu_mask_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, <vscale x 4 x i8> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -157,8 +157,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vasubu.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vasubu_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -181,8 +181,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vasubu.mask.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vasubu_mask_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -205,8 +205,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vasubu.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vasubu_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -229,8 +229,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vasubu.mask.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vasubu_mask_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <vscale x 16 x i8> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -253,8 +253,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vasubu.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vasubu_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -277,8 +277,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vasubu.mask.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vasubu_mask_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, <vscale x 32 x i8> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -301,8 +301,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vasubu.nxv64i8.nxv64i8(
 define <vscale x 64 x i8> @intrinsic_vasubu_vv_nxv64i8_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -326,8 +326,8 @@ define <vscale x 64 x i8> @intrinsic_vasubu_mask_vv_nxv64i8_nxv64i8_nxv64i8(<vsc
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8r.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -350,8 +350,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vasubu.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vasubu_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -374,8 +374,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vasubu.mask.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vasubu_mask_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, <vscale x 1 x i16> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -398,8 +398,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vasubu.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vasubu_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -422,8 +422,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vasubu.mask.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vasubu_mask_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, <vscale x 2 x i16> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -446,8 +446,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vasubu.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vasubu_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -470,8 +470,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vasubu.mask.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vasubu_mask_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, <vscale x 4 x i16> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -494,8 +494,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vasubu.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vasubu_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -518,8 +518,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vasubu.mask.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vasubu_mask_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <vscale x 8 x i16> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -542,8 +542,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vasubu.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vasubu_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -566,8 +566,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vasubu.mask.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vasubu_mask_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, <vscale x 16 x i16> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -590,8 +590,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vasubu.nxv32i16.nxv32i16(
 define <vscale x 32 x i16> @intrinsic_vasubu_vv_nxv32i16_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -615,8 +615,8 @@ define <vscale x 32 x i16> @intrinsic_vasubu_mask_vv_nxv32i16_nxv32i16_nxv32i16(
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -639,8 +639,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vasubu.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vasubu_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -663,8 +663,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vasubu.mask.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vasubu_mask_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, <vscale x 1 x i32> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -687,8 +687,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vasubu.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vasubu_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -711,8 +711,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vasubu.mask.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vasubu_mask_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, <vscale x 2 x i32> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -735,8 +735,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vasubu.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vasubu_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -759,8 +759,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vasubu.mask.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vasubu_mask_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <vscale x 4 x i32> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -783,8 +783,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vasubu.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vasubu_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -807,8 +807,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vasubu.mask.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vasubu_mask_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, <vscale x 8 x i32> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -831,8 +831,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vasubu.nxv16i32.nxv16i32(
 define <vscale x 16 x i32> @intrinsic_vasubu_vv_nxv16i32_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -856,8 +856,8 @@ define <vscale x 16 x i32> @intrinsic_vasubu_mask_vv_nxv16i32_nxv16i32_nxv16i32(
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -880,8 +880,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vasubu.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vasubu_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -904,8 +904,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vasubu.mask.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vasubu_mask_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i64> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -928,8 +928,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vasubu.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vasubu_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -952,8 +952,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vasubu.mask.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vasubu_mask_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -976,8 +976,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vasubu.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vasubu_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -1000,8 +1000,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vasubu.mask.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vasubu_mask_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, <vscale x 4 x i64> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1024,8 +1024,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vasubu.nxv8i64.nxv8i64(
 define <vscale x 8 x i64> @intrinsic_vasubu_vv_nxv8i64_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vasubu.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -1049,8 +1049,8 @@ define <vscale x 8 x i64> @intrinsic_vasubu_mask_vv_nxv8i64_nxv8i64_nxv8i64(<vsc
 ; CHECK-LABEL: intrinsic_vasubu_mask_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    vasubu.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1073,8 +1073,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vasubu.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vasubu_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1097,8 +1097,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vasubu.mask.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vasubu_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1121,8 +1121,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vasubu.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vasubu_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1145,8 +1145,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vasubu.mask.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vasubu_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1169,8 +1169,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vasubu.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vasubu_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1193,8 +1193,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vasubu.mask.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vasubu_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1217,8 +1217,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vasubu.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vasubu_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1241,8 +1241,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vasubu.mask.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vasubu_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1265,8 +1265,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vasubu.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vasubu_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1289,8 +1289,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vasubu.mask.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vasubu_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1313,8 +1313,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vasubu.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vasubu_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1337,8 +1337,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vasubu.mask.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vasubu_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1361,8 +1361,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vasubu.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vasubu_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1385,8 +1385,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vasubu.mask.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vasubu_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1409,8 +1409,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vasubu.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vasubu_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1433,8 +1433,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vasubu.mask.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vasubu_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1457,8 +1457,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vasubu.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vasubu_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1481,8 +1481,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vasubu.mask.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vasubu_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1505,8 +1505,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vasubu.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vasubu_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1529,8 +1529,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vasubu.mask.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vasubu_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1553,8 +1553,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vasubu.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vasubu_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1577,8 +1577,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vasubu.mask.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vasubu_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1601,8 +1601,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vasubu.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vasubu_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1625,8 +1625,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vasubu.mask.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vasubu_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1649,8 +1649,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vasubu.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vasubu_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1673,8 +1673,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vasubu.mask.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vasubu_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1697,8 +1697,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vasubu.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vasubu_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1721,8 +1721,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vasubu.mask.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vasubu_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1745,8 +1745,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vasubu.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vasubu_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1769,8 +1769,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vasubu.mask.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vasubu_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1793,8 +1793,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vasubu.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vasubu_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1817,8 +1817,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vasubu.mask.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vasubu_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1841,8 +1841,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vasubu.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vasubu_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1865,8 +1865,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vasubu.mask.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vasubu_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1889,8 +1889,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vasubu.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vasubu_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vasubu.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1913,8 +1913,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vasubu.mask.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vasubu_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vasubu_mask_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 1
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vasubu.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1950,8 +1950,8 @@ define <vscale x 1 x i64> @intrinsic_vasubu_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x
 ;
 ; RV64-LABEL: intrinsic_vasubu_vx_nxv1i64_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; RV64-NEXT:    vasubu.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -1987,8 +1987,8 @@ define <vscale x 1 x i64> @intrinsic_vasubu_mask_vx_nxv1i64_nxv1i64_i64(<vscale
 ;
 ; RV64-LABEL: intrinsic_vasubu_mask_vx_nxv1i64_nxv1i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; RV64-NEXT:    vasubu.vx v8, v9, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2024,8 +2024,8 @@ define <vscale x 2 x i64> @intrinsic_vasubu_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x
 ;
 ; RV64-LABEL: intrinsic_vasubu_vx_nxv2i64_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; RV64-NEXT:    vasubu.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2061,8 +2061,8 @@ define <vscale x 2 x i64> @intrinsic_vasubu_mask_vx_nxv2i64_nxv2i64_i64(<vscale
 ;
 ; RV64-LABEL: intrinsic_vasubu_mask_vx_nxv2i64_nxv2i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; RV64-NEXT:    vasubu.vx v8, v10, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2098,8 +2098,8 @@ define <vscale x 4 x i64> @intrinsic_vasubu_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x
 ;
 ; RV64-LABEL: intrinsic_vasubu_vx_nxv4i64_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; RV64-NEXT:    vasubu.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2135,8 +2135,8 @@ define <vscale x 4 x i64> @intrinsic_vasubu_mask_vx_nxv4i64_nxv4i64_i64(<vscale
 ;
 ; RV64-LABEL: intrinsic_vasubu_mask_vx_nxv4i64_nxv4i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; RV64-NEXT:    vasubu.vx v8, v12, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
@@ -2172,8 +2172,8 @@ define <vscale x 8 x i64> @intrinsic_vasubu_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x
 ;
 ; RV64-LABEL: intrinsic_vasubu_vx_nxv8i64_nxv8i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    vasubu.vx v8, v8, a0
 ; RV64-NEXT:    ret
 entry:
@@ -2209,8 +2209,8 @@ define <vscale x 8 x i64> @intrinsic_vasubu_mask_vx_nxv8i64_nxv8i64_i64(<vscale
 ;
 ; RV64-LABEL: intrinsic_vasubu_mask_vx_nxv8i64_nxv8i64_i64:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; RV64-NEXT:    csrwi vxrm, 1
+; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; RV64-NEXT:    vasubu.vx v8, v16, a0, v0.t
 ; RV64-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfadd.ll b/llvm/test/CodeGen/RISCV/rvv/vfadd.ll
index 6816307d1096..ae7d7d5d1962 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfadd.ll
@@ -23,10 +23,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfadd.nxv1f16.nxv1f16(
 define <vscale x 1 x half> @intrinsic_vfadd_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfadd.nxv1f16.nxv1f16(
@@ -48,10 +48,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfadd.mask.nxv1f16.nxv1f16(
 define <vscale x 1 x half> @intrinsic_vfadd_mask_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfadd.mask.nxv1f16.nxv1f16(
@@ -73,10 +73,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfadd.nxv2f16.nxv2f16(
 define <vscale x 2 x half> @intrinsic_vfadd_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfadd.nxv2f16.nxv2f16(
@@ -98,10 +98,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfadd.mask.nxv2f16.nxv2f16(
 define <vscale x 2 x half> @intrinsic_vfadd_mask_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfadd.mask.nxv2f16.nxv2f16(
@@ -123,10 +123,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfadd.nxv4f16.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfadd_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfadd.nxv4f16.nxv4f16(
@@ -148,10 +148,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfadd.mask.nxv4f16.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfadd_mask_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfadd.mask.nxv4f16.nxv4f16(
@@ -173,10 +173,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfadd.nxv8f16.nxv8f16(
 define <vscale x 8 x half> @intrinsic_vfadd_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfadd.nxv8f16.nxv8f16(
@@ -198,10 +198,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfadd.mask.nxv8f16.nxv8f16(
 define <vscale x 8 x half> @intrinsic_vfadd_mask_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfadd.mask.nxv8f16.nxv8f16(
@@ -223,10 +223,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfadd.nxv16f16.nxv16f16(
 define <vscale x 16 x half> @intrinsic_vfadd_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfadd.nxv16f16.nxv16f16(
@@ -248,10 +248,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfadd.mask.nxv16f16.nxv16f16(
 define <vscale x 16 x half> @intrinsic_vfadd_mask_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfadd.mask.nxv16f16.nxv16f16(
@@ -273,10 +273,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfadd.nxv32f16.nxv32f16(
 define <vscale x 32 x half> @intrinsic_vfadd_vv_nxv32f16_nxv32f16_nxv32f16(<vscale x 32 x half> %0, <vscale x 32 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv32f16_nxv32f16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfadd.nxv32f16.nxv32f16(
@@ -299,8 +299,8 @@ define <vscale x 32 x half> @intrinsic_vfadd_mask_vv_nxv32f16_nxv32f16_nxv32f16(
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv32f16_nxv32f16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vfadd.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -324,10 +324,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfadd.nxv1f32.nxv1f32(
 define <vscale x 1 x float> @intrinsic_vfadd_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfadd.nxv1f32.nxv1f32(
@@ -349,10 +349,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfadd.mask.nxv1f32.nxv1f32(
 define <vscale x 1 x float> @intrinsic_vfadd_mask_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfadd.mask.nxv1f32.nxv1f32(
@@ -374,10 +374,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfadd.nxv2f32.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfadd_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfadd.nxv2f32.nxv2f32(
@@ -399,10 +399,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfadd.mask.nxv2f32.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfadd_mask_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfadd.mask.nxv2f32.nxv2f32(
@@ -424,10 +424,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(
 define <vscale x 4 x float> @intrinsic_vfadd_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(
@@ -449,10 +449,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfadd.mask.nxv4f32.nxv4f32(
 define <vscale x 4 x float> @intrinsic_vfadd_mask_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfadd.mask.nxv4f32.nxv4f32(
@@ -474,10 +474,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfadd.nxv8f32.nxv8f32(
 define <vscale x 8 x float> @intrinsic_vfadd_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfadd.nxv8f32.nxv8f32(
@@ -499,10 +499,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfadd.mask.nxv8f32.nxv8f32(
 define <vscale x 8 x float> @intrinsic_vfadd_mask_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfadd.mask.nxv8f32.nxv8f32(
@@ -524,10 +524,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfadd.nxv16f32.nxv16f32(
 define <vscale x 16 x float> @intrinsic_vfadd_vv_nxv16f32_nxv16f32_nxv16f32(<vscale x 16 x float> %0, <vscale x 16 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv16f32_nxv16f32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfadd.nxv16f32.nxv16f32(
@@ -550,8 +550,8 @@ define <vscale x 16 x float> @intrinsic_vfadd_mask_vv_nxv16f32_nxv16f32_nxv16f32
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv16f32_nxv16f32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vfadd.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -575,10 +575,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfadd.nxv1f64.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfadd_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfadd.nxv1f64.nxv1f64(
@@ -600,10 +600,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfadd.mask.nxv1f64.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfadd_mask_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfadd.mask.nxv1f64.nxv1f64(
@@ -625,10 +625,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfadd.nxv2f64.nxv2f64(
 define <vscale x 2 x double> @intrinsic_vfadd_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfadd.nxv2f64.nxv2f64(
@@ -650,10 +650,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfadd.mask.nxv2f64.nxv2f64(
 define <vscale x 2 x double> @intrinsic_vfadd_mask_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfadd.mask.nxv2f64.nxv2f64(
@@ -675,10 +675,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(
 define <vscale x 4 x double> @intrinsic_vfadd_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.nxv4f64(
@@ -700,10 +700,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfadd.mask.nxv4f64.nxv4f64(
 define <vscale x 4 x double> @intrinsic_vfadd_mask_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfadd.mask.nxv4f64.nxv4f64(
@@ -725,10 +725,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfadd.nxv8f64.nxv8f64(
 define <vscale x 8 x double> @intrinsic_vfadd_vv_nxv8f64_nxv8f64_nxv8f64(<vscale x 8 x double> %0, <vscale x 8 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv8f64_nxv8f64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfadd.nxv8f64.nxv8f64(
@@ -751,8 +751,8 @@ define <vscale x 8 x double> @intrinsic_vfadd_mask_vv_nxv8f64_nxv8f64_nxv8f64(<v
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv8f64_nxv8f64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    vfadd.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -776,10 +776,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfadd.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfadd_vf_nxv1f16_nxv1f16_f16(<vscale x 1 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv1f16_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfadd.nxv1f16.f16(
@@ -801,10 +801,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfadd.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfadd_mask_vf_nxv1f16_nxv1f16_f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv1f16_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfadd.mask.nxv1f16.f16(
@@ -826,10 +826,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfadd.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfadd_vf_nxv2f16_nxv2f16_f16(<vscale x 2 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv2f16_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfadd.nxv2f16.f16(
@@ -851,10 +851,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfadd.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfadd_mask_vf_nxv2f16_nxv2f16_f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv2f16_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfadd.mask.nxv2f16.f16(
@@ -876,10 +876,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfadd.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfadd_vf_nxv4f16_nxv4f16_f16(<vscale x 4 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv4f16_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfadd.nxv4f16.f16(
@@ -901,10 +901,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfadd.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfadd_mask_vf_nxv4f16_nxv4f16_f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv4f16_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfadd.mask.nxv4f16.f16(
@@ -926,10 +926,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfadd.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfadd_vf_nxv8f16_nxv8f16_f16(<vscale x 8 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv8f16_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfadd.nxv8f16.f16(
@@ -951,10 +951,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfadd.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfadd_mask_vf_nxv8f16_nxv8f16_f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv8f16_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfadd.mask.nxv8f16.f16(
@@ -976,10 +976,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfadd.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfadd_vf_nxv16f16_nxv16f16_f16(<vscale x 16 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv16f16_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfadd.nxv16f16.f16(
@@ -1001,10 +1001,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfadd.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfadd_mask_vf_nxv16f16_nxv16f16_f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv16f16_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfadd.mask.nxv16f16.f16(
@@ -1026,10 +1026,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfadd.nxv32f16.f16(
 define <vscale x 32 x half> @intrinsic_vfadd_vf_nxv32f16_nxv32f16_f16(<vscale x 32 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv32f16_nxv32f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfadd.nxv32f16.f16(
@@ -1051,10 +1051,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfadd.mask.nxv32f16.f16(
 define <vscale x 32 x half> @intrinsic_vfadd_mask_vf_nxv32f16_nxv32f16_f16(<vscale x 32 x half> %0, <vscale x 32 x half> %1, half %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv32f16_nxv32f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfadd.mask.nxv32f16.f16(
@@ -1076,10 +1076,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfadd.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfadd_vf_nxv1f32_nxv1f32_f32(<vscale x 1 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv1f32_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfadd.nxv1f32.f32(
@@ -1101,10 +1101,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfadd.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfadd_mask_vf_nxv1f32_nxv1f32_f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv1f32_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfadd.mask.nxv1f32.f32(
@@ -1126,10 +1126,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfadd.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfadd_vf_nxv2f32_nxv2f32_f32(<vscale x 2 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv2f32_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfadd.nxv2f32.f32(
@@ -1151,10 +1151,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfadd.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfadd_mask_vf_nxv2f32_nxv2f32_f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv2f32_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfadd.mask.nxv2f32.f32(
@@ -1176,10 +1176,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfadd_vf_nxv4f32_nxv4f32_f32(<vscale x 4 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv4f32_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.f32(
@@ -1201,10 +1201,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfadd.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfadd_mask_vf_nxv4f32_nxv4f32_f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv4f32_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfadd.mask.nxv4f32.f32(
@@ -1226,10 +1226,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfadd.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfadd_vf_nxv8f32_nxv8f32_f32(<vscale x 8 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv8f32_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfadd.nxv8f32.f32(
@@ -1251,10 +1251,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfadd.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfadd_mask_vf_nxv8f32_nxv8f32_f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv8f32_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfadd.mask.nxv8f32.f32(
@@ -1276,10 +1276,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfadd.nxv16f32.f32(
 define <vscale x 16 x float> @intrinsic_vfadd_vf_nxv16f32_nxv16f32_f32(<vscale x 16 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv16f32_nxv16f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfadd.nxv16f32.f32(
@@ -1301,10 +1301,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfadd.mask.nxv16f32.f32(
 define <vscale x 16 x float> @intrinsic_vfadd_mask_vf_nxv16f32_nxv16f32_f32(<vscale x 16 x float> %0, <vscale x 16 x float> %1, float %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv16f32_nxv16f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfadd.mask.nxv16f32.f32(
@@ -1326,10 +1326,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfadd.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfadd_vf_nxv1f64_nxv1f64_f64(<vscale x 1 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv1f64_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfadd.nxv1f64.f64(
@@ -1351,10 +1351,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfadd.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfadd_mask_vf_nxv1f64_nxv1f64_f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, double %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv1f64_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfadd.mask.nxv1f64.f64(
@@ -1376,10 +1376,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfadd.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfadd_vf_nxv2f64_nxv2f64_f64(<vscale x 2 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv2f64_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfadd.nxv2f64.f64(
@@ -1401,10 +1401,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfadd.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfadd_mask_vf_nxv2f64_nxv2f64_f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, double %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv2f64_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfadd.mask.nxv2f64.f64(
@@ -1426,10 +1426,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfadd_vf_nxv4f64_nxv4f64_f64(<vscale x 4 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv4f64_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfadd.nxv4f64.f64(
@@ -1451,10 +1451,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfadd.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfadd_mask_vf_nxv4f64_nxv4f64_f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, double %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv4f64_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfadd.mask.nxv4f64.f64(
@@ -1476,10 +1476,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfadd.nxv8f64.f64(
 define <vscale x 8 x double> @intrinsic_vfadd_vf_nxv8f64_nxv8f64_f64(<vscale x 8 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv8f64_nxv8f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfadd.nxv8f64.f64(
@@ -1501,10 +1501,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfadd.mask.nxv8f64.f64(
 define <vscale x 8 x double> @intrinsic_vfadd_mask_vf_nxv8f64_nxv8f64_f64(<vscale x 8 x double> %0, <vscale x 8 x double> %1, double %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv8f64_nxv8f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfadd.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfadd.mask.nxv8f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-x.ll b/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-x.ll
index 626848839b07..bc8440920cd8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-x.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-x.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfcvt.f.x.v.nxv1f16.nxv1i16(
 define <vscale x 1 x half> @intrinsic_vfcvt_f.x.v_nxv1f16_nxv1i16(<vscale x 1 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv1f16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfcvt.f.x.v.nxv1f16.nxv1i16(
@@ -35,10 +35,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv1f16.nxv1i16(
 define <vscale x 1 x half> @intrinsic_vfcvt_mask_f.x.v_nxv1f16_nxv1i16(<vscale x 1 x half> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv1f16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv1f16.nxv1i16(
@@ -58,10 +58,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfcvt.f.x.v.nxv2f16.nxv2i16(
 define <vscale x 2 x half> @intrinsic_vfcvt_f.x.v_nxv2f16_nxv2i16(<vscale x 2 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv2f16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfcvt.f.x.v.nxv2f16.nxv2i16(
@@ -81,10 +81,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv2f16.nxv2i16(
 define <vscale x 2 x half> @intrinsic_vfcvt_mask_f.x.v_nxv2f16_nxv2i16(<vscale x 2 x half> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv2f16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv2f16.nxv2i16(
@@ -104,10 +104,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfcvt.f.x.v.nxv4f16.nxv4i16(
 define <vscale x 4 x half> @intrinsic_vfcvt_f.x.v_nxv4f16_nxv4i16(<vscale x 4 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv4f16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfcvt.f.x.v.nxv4f16.nxv4i16(
@@ -127,10 +127,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv4f16.nxv4i16(
 define <vscale x 4 x half> @intrinsic_vfcvt_mask_f.x.v_nxv4f16_nxv4i16(<vscale x 4 x half> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv4f16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv4f16.nxv4i16(
@@ -150,10 +150,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfcvt.f.x.v.nxv8f16.nxv8i16(
 define <vscale x 8 x half> @intrinsic_vfcvt_f.x.v_nxv8f16_nxv8i16(<vscale x 8 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv8f16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfcvt.f.x.v.nxv8f16.nxv8i16(
@@ -173,10 +173,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv8f16.nxv8i16(
 define <vscale x 8 x half> @intrinsic_vfcvt_mask_f.x.v_nxv8f16_nxv8i16(<vscale x 8 x half> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv8f16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv8f16.nxv8i16(
@@ -196,10 +196,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfcvt.f.x.v.nxv16f16.nxv16i16(
 define <vscale x 16 x half> @intrinsic_vfcvt_f.x.v_nxv16f16_nxv16i16(<vscale x 16 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv16f16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfcvt.f.x.v.nxv16f16.nxv16i16(
@@ -219,10 +219,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv16f16.nxv16i16(
 define <vscale x 16 x half> @intrinsic_vfcvt_mask_f.x.v_nxv16f16_nxv16i16(<vscale x 16 x half> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv16f16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv16f16.nxv16i16(
@@ -242,10 +242,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfcvt.f.x.v.nxv32f16.nxv32i16(
 define <vscale x 32 x half> @intrinsic_vfcvt_f.x.v_nxv32f16_nxv32i16(<vscale x 32 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv32f16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfcvt.f.x.v.nxv32f16.nxv32i16(
@@ -265,10 +265,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv32f16.nxv32i16(
 define <vscale x 32 x half> @intrinsic_vfcvt_mask_f.x.v_nxv32f16_nxv32i16(<vscale x 32 x half> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv32f16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfcvt.f.x.v.mask.nxv32f16.nxv32i16(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfcvt.f.x.v.nxv1f32.nxv1i32(
 define <vscale x 1 x float> @intrinsic_vfcvt_f.x.v_nxv1f32_nxv1i32(<vscale x 1 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv1f32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfcvt.f.x.v.nxv1f32.nxv1i32(
@@ -311,10 +311,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfcvt.f.x.v.mask.nxv1f32.nxv1i32(
 define <vscale x 1 x float> @intrinsic_vfcvt_mask_f.x.v_nxv1f32_nxv1i32(<vscale x 1 x float> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv1f32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfcvt.f.x.v.mask.nxv1f32.nxv1i32(
@@ -334,10 +334,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfcvt.f.x.v.nxv2f32.nxv2i32(
 define <vscale x 2 x float> @intrinsic_vfcvt_f.x.v_nxv2f32_nxv2i32(<vscale x 2 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv2f32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfcvt.f.x.v.nxv2f32.nxv2i32(
@@ -357,10 +357,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfcvt.f.x.v.mask.nxv2f32.nxv2i32(
 define <vscale x 2 x float> @intrinsic_vfcvt_mask_f.x.v_nxv2f32_nxv2i32(<vscale x 2 x float> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv2f32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfcvt.f.x.v.mask.nxv2f32.nxv2i32(
@@ -380,10 +380,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfcvt.f.x.v.nxv4f32.nxv4i32(
 define <vscale x 4 x float> @intrinsic_vfcvt_f.x.v_nxv4f32_nxv4i32(<vscale x 4 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv4f32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfcvt.f.x.v.nxv4f32.nxv4i32(
@@ -403,10 +403,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfcvt.f.x.v.mask.nxv4f32.nxv4i32(
 define <vscale x 4 x float> @intrinsic_vfcvt_mask_f.x.v_nxv4f32_nxv4i32(<vscale x 4 x float> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv4f32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfcvt.f.x.v.mask.nxv4f32.nxv4i32(
@@ -426,10 +426,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfcvt.f.x.v.nxv8f32.nxv8i32(
 define <vscale x 8 x float> @intrinsic_vfcvt_f.x.v_nxv8f32_nxv8i32(<vscale x 8 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv8f32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfcvt.f.x.v.nxv8f32.nxv8i32(
@@ -449,10 +449,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfcvt.f.x.v.mask.nxv8f32.nxv8i32(
 define <vscale x 8 x float> @intrinsic_vfcvt_mask_f.x.v_nxv8f32_nxv8i32(<vscale x 8 x float> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv8f32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfcvt.f.x.v.mask.nxv8f32.nxv8i32(
@@ -472,10 +472,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfcvt.f.x.v.nxv16f32.nxv16i32(
 define <vscale x 16 x float> @intrinsic_vfcvt_f.x.v_nxv16f32_nxv16i32(<vscale x 16 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv16f32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfcvt.f.x.v.nxv16f32.nxv16i32(
@@ -495,10 +495,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfcvt.f.x.v.mask.nxv16f32.nxv16i32(
 define <vscale x 16 x float> @intrinsic_vfcvt_mask_f.x.v_nxv16f32_nxv16i32(<vscale x 16 x float> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv16f32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfcvt.f.x.v.mask.nxv16f32.nxv16i32(
@@ -518,10 +518,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfcvt.f.x.v.nxv1f64.nxv1i64(
 define <vscale x 1 x double> @intrinsic_vfcvt_f.x.v_nxv1f64_nxv1i64(<vscale x 1 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv1f64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfcvt.f.x.v.nxv1f64.nxv1i64(
@@ -541,10 +541,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfcvt.f.x.v.mask.nxv1f64.nxv1i64(
 define <vscale x 1 x double> @intrinsic_vfcvt_mask_f.x.v_nxv1f64_nxv1i64(<vscale x 1 x double> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv1f64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfcvt.f.x.v.mask.nxv1f64.nxv1i64(
@@ -564,10 +564,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfcvt.f.x.v.nxv2f64.nxv2i64(
 define <vscale x 2 x double> @intrinsic_vfcvt_f.x.v_nxv2f64_nxv2i64(<vscale x 2 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv2f64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfcvt.f.x.v.nxv2f64.nxv2i64(
@@ -587,10 +587,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfcvt.f.x.v.mask.nxv2f64.nxv2i64(
 define <vscale x 2 x double> @intrinsic_vfcvt_mask_f.x.v_nxv2f64_nxv2i64(<vscale x 2 x double> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv2f64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfcvt.f.x.v.mask.nxv2f64.nxv2i64(
@@ -610,10 +610,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfcvt.f.x.v.nxv4f64.nxv4i64(
 define <vscale x 4 x double> @intrinsic_vfcvt_f.x.v_nxv4f64_nxv4i64(<vscale x 4 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv4f64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfcvt.f.x.v.nxv4f64.nxv4i64(
@@ -633,10 +633,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfcvt.f.x.v.mask.nxv4f64.nxv4i64(
 define <vscale x 4 x double> @intrinsic_vfcvt_mask_f.x.v_nxv4f64_nxv4i64(<vscale x 4 x double> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv4f64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfcvt.f.x.v.mask.nxv4f64.nxv4i64(
@@ -656,10 +656,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfcvt.f.x.v.nxv8f64.nxv8i64(
 define <vscale x 8 x double> @intrinsic_vfcvt_f.x.v_nxv8f64_nxv8i64(<vscale x 8 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.x.v_nxv8f64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfcvt.f.x.v.nxv8f64.nxv8i64(
@@ -679,10 +679,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfcvt.f.x.v.mask.nxv8f64.nxv8i64(
 define <vscale x 8 x double> @intrinsic_vfcvt_mask_f.x.v_nxv8f64_nxv8i64(<vscale x 8 x double> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x.v_nxv8f64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.x.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfcvt.f.x.v.mask.nxv8f64.nxv8i64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-xu.ll b/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-xu.ll
index 9109df44ec7f..9cf47f993ee4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-xu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcvt-f-xu.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfcvt.f.xu.v.nxv1f16.nxv1i16(
 define <vscale x 1 x half> @intrinsic_vfcvt_f.xu.v_nxv1f16_nxv1i16(<vscale x 1 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv1f16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfcvt.f.xu.v.nxv1f16.nxv1i16(
@@ -35,10 +35,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv1f16.nxv1i16(
 define <vscale x 1 x half> @intrinsic_vfcvt_mask_f.xu.v_nxv1f16_nxv1i16(<vscale x 1 x half> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv1f16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv1f16.nxv1i16(
@@ -58,10 +58,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfcvt.f.xu.v.nxv2f16.nxv2i16(
 define <vscale x 2 x half> @intrinsic_vfcvt_f.xu.v_nxv2f16_nxv2i16(<vscale x 2 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv2f16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfcvt.f.xu.v.nxv2f16.nxv2i16(
@@ -81,10 +81,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv2f16.nxv2i16(
 define <vscale x 2 x half> @intrinsic_vfcvt_mask_f.xu.v_nxv2f16_nxv2i16(<vscale x 2 x half> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv2f16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv2f16.nxv2i16(
@@ -104,10 +104,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfcvt.f.xu.v.nxv4f16.nxv4i16(
 define <vscale x 4 x half> @intrinsic_vfcvt_f.xu.v_nxv4f16_nxv4i16(<vscale x 4 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv4f16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfcvt.f.xu.v.nxv4f16.nxv4i16(
@@ -127,10 +127,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv4f16.nxv4i16(
 define <vscale x 4 x half> @intrinsic_vfcvt_mask_f.xu.v_nxv4f16_nxv4i16(<vscale x 4 x half> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv4f16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv4f16.nxv4i16(
@@ -150,10 +150,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfcvt.f.xu.v.nxv8f16.nxv8i16(
 define <vscale x 8 x half> @intrinsic_vfcvt_f.xu.v_nxv8f16_nxv8i16(<vscale x 8 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv8f16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfcvt.f.xu.v.nxv8f16.nxv8i16(
@@ -173,10 +173,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv8f16.nxv8i16(
 define <vscale x 8 x half> @intrinsic_vfcvt_mask_f.xu.v_nxv8f16_nxv8i16(<vscale x 8 x half> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv8f16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv8f16.nxv8i16(
@@ -196,10 +196,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfcvt.f.xu.v.nxv16f16.nxv16i16(
 define <vscale x 16 x half> @intrinsic_vfcvt_f.xu.v_nxv16f16_nxv16i16(<vscale x 16 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv16f16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfcvt.f.xu.v.nxv16f16.nxv16i16(
@@ -219,10 +219,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv16f16.nxv16i16(
 define <vscale x 16 x half> @intrinsic_vfcvt_mask_f.xu.v_nxv16f16_nxv16i16(<vscale x 16 x half> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv16f16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv16f16.nxv16i16(
@@ -242,10 +242,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfcvt.f.xu.v.nxv32f16.nxv32i16(
 define <vscale x 32 x half> @intrinsic_vfcvt_f.xu.v_nxv32f16_nxv32i16(<vscale x 32 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv32f16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfcvt.f.xu.v.nxv32f16.nxv32i16(
@@ -265,10 +265,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv32f16.nxv32i16(
 define <vscale x 32 x half> @intrinsic_vfcvt_mask_f.xu.v_nxv32f16_nxv32i16(<vscale x 32 x half> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv32f16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfcvt.f.xu.v.mask.nxv32f16.nxv32i16(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfcvt.f.xu.v.nxv1f32.nxv1i32(
 define <vscale x 1 x float> @intrinsic_vfcvt_f.xu.v_nxv1f32_nxv1i32(<vscale x 1 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv1f32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfcvt.f.xu.v.nxv1f32.nxv1i32(
@@ -311,10 +311,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfcvt.f.xu.v.mask.nxv1f32.nxv1i32(
 define <vscale x 1 x float> @intrinsic_vfcvt_mask_f.xu.v_nxv1f32_nxv1i32(<vscale x 1 x float> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv1f32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfcvt.f.xu.v.mask.nxv1f32.nxv1i32(
@@ -334,10 +334,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfcvt.f.xu.v.nxv2f32.nxv2i32(
 define <vscale x 2 x float> @intrinsic_vfcvt_f.xu.v_nxv2f32_nxv2i32(<vscale x 2 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv2f32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfcvt.f.xu.v.nxv2f32.nxv2i32(
@@ -357,10 +357,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfcvt.f.xu.v.mask.nxv2f32.nxv2i32(
 define <vscale x 2 x float> @intrinsic_vfcvt_mask_f.xu.v_nxv2f32_nxv2i32(<vscale x 2 x float> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv2f32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfcvt.f.xu.v.mask.nxv2f32.nxv2i32(
@@ -380,10 +380,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfcvt.f.xu.v.nxv4f32.nxv4i32(
 define <vscale x 4 x float> @intrinsic_vfcvt_f.xu.v_nxv4f32_nxv4i32(<vscale x 4 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv4f32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfcvt.f.xu.v.nxv4f32.nxv4i32(
@@ -403,10 +403,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfcvt.f.xu.v.mask.nxv4f32.nxv4i32(
 define <vscale x 4 x float> @intrinsic_vfcvt_mask_f.xu.v_nxv4f32_nxv4i32(<vscale x 4 x float> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv4f32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfcvt.f.xu.v.mask.nxv4f32.nxv4i32(
@@ -426,10 +426,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfcvt.f.xu.v.nxv8f32.nxv8i32(
 define <vscale x 8 x float> @intrinsic_vfcvt_f.xu.v_nxv8f32_nxv8i32(<vscale x 8 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv8f32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfcvt.f.xu.v.nxv8f32.nxv8i32(
@@ -449,10 +449,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfcvt.f.xu.v.mask.nxv8f32.nxv8i32(
 define <vscale x 8 x float> @intrinsic_vfcvt_mask_f.xu.v_nxv8f32_nxv8i32(<vscale x 8 x float> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv8f32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfcvt.f.xu.v.mask.nxv8f32.nxv8i32(
@@ -472,10 +472,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfcvt.f.xu.v.nxv16f32.nxv16i32(
 define <vscale x 16 x float> @intrinsic_vfcvt_f.xu.v_nxv16f32_nxv16i32(<vscale x 16 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv16f32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfcvt.f.xu.v.nxv16f32.nxv16i32(
@@ -495,10 +495,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfcvt.f.xu.v.mask.nxv16f32.nxv16i32(
 define <vscale x 16 x float> @intrinsic_vfcvt_mask_f.xu.v_nxv16f32_nxv16i32(<vscale x 16 x float> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv16f32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfcvt.f.xu.v.mask.nxv16f32.nxv16i32(
@@ -518,10 +518,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfcvt.f.xu.v.nxv1f64.nxv1i64(
 define <vscale x 1 x double> @intrinsic_vfcvt_f.xu.v_nxv1f64_nxv1i64(<vscale x 1 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv1f64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfcvt.f.xu.v.nxv1f64.nxv1i64(
@@ -541,10 +541,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfcvt.f.xu.v.mask.nxv1f64.nxv1i64(
 define <vscale x 1 x double> @intrinsic_vfcvt_mask_f.xu.v_nxv1f64_nxv1i64(<vscale x 1 x double> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv1f64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfcvt.f.xu.v.mask.nxv1f64.nxv1i64(
@@ -564,10 +564,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfcvt.f.xu.v.nxv2f64.nxv2i64(
 define <vscale x 2 x double> @intrinsic_vfcvt_f.xu.v_nxv2f64_nxv2i64(<vscale x 2 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv2f64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfcvt.f.xu.v.nxv2f64.nxv2i64(
@@ -587,10 +587,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfcvt.f.xu.v.mask.nxv2f64.nxv2i64(
 define <vscale x 2 x double> @intrinsic_vfcvt_mask_f.xu.v_nxv2f64_nxv2i64(<vscale x 2 x double> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv2f64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfcvt.f.xu.v.mask.nxv2f64.nxv2i64(
@@ -610,10 +610,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfcvt.f.xu.v.nxv4f64.nxv4i64(
 define <vscale x 4 x double> @intrinsic_vfcvt_f.xu.v_nxv4f64_nxv4i64(<vscale x 4 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv4f64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfcvt.f.xu.v.nxv4f64.nxv4i64(
@@ -633,10 +633,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfcvt.f.xu.v.mask.nxv4f64.nxv4i64(
 define <vscale x 4 x double> @intrinsic_vfcvt_mask_f.xu.v_nxv4f64_nxv4i64(<vscale x 4 x double> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv4f64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfcvt.f.xu.v.mask.nxv4f64.nxv4i64(
@@ -656,10 +656,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfcvt.f.xu.v.nxv8f64.nxv8i64(
 define <vscale x 8 x double> @intrinsic_vfcvt_f.xu.v_nxv8f64_nxv8i64(<vscale x 8 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu.v_nxv8f64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfcvt.f.xu.v.nxv8f64.nxv8i64(
@@ -679,10 +679,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfcvt.f.xu.v.mask.nxv8f64.nxv8i64(
 define <vscale x 8 x double> @intrinsic_vfcvt_mask_f.xu.v_nxv8f64_nxv8i64(<vscale x 8 x double> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu.v_nxv8f64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.f.xu.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfcvt.f.xu.v.mask.nxv8f64.nxv8i64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcvt-x-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfcvt-x-f.ll
index 1147ec331b78..68a85530ea24 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcvt-x-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcvt-x-f.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x i16> @llvm.riscv.vfcvt.x.f.v.nxv1i16.nxv1f16(
 define <vscale x 1 x i16> @intrinsic_vfcvt_x.f.v_nxv1i16_nxv1f16(<vscale x 1 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv1i16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vfcvt.x.f.v.nxv1i16.nxv1f16(
@@ -35,10 +35,10 @@ declare <vscale x 1 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv1i16.nxv1f16(
 define <vscale x 1 x i16> @intrinsic_vfcvt_mask_x.f.v_nxv1i16_nxv1f16(<vscale x 1 x i16> %0, <vscale x 1 x half> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv1i16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv1i16.nxv1f16(
@@ -58,10 +58,10 @@ declare <vscale x 2 x i16> @llvm.riscv.vfcvt.x.f.v.nxv2i16.nxv2f16(
 define <vscale x 2 x i16> @intrinsic_vfcvt_x.f.v_nxv2i16_nxv2f16(<vscale x 2 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv2i16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vfcvt.x.f.v.nxv2i16.nxv2f16(
@@ -81,10 +81,10 @@ declare <vscale x 2 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv2i16.nxv2f16(
 define <vscale x 2 x i16> @intrinsic_vfcvt_mask_x.f.v_nxv2i16_nxv2f16(<vscale x 2 x i16> %0, <vscale x 2 x half> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv2i16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv2i16.nxv2f16(
@@ -104,10 +104,10 @@ declare <vscale x 4 x i16> @llvm.riscv.vfcvt.x.f.v.nxv4i16.nxv4f16(
 define <vscale x 4 x i16> @intrinsic_vfcvt_x.f.v_nxv4i16_nxv4f16(<vscale x 4 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv4i16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vfcvt.x.f.v.nxv4i16.nxv4f16(
@@ -127,10 +127,10 @@ declare <vscale x 4 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv4i16.nxv4f16(
 define <vscale x 4 x i16> @intrinsic_vfcvt_mask_x.f.v_nxv4i16_nxv4f16(<vscale x 4 x i16> %0, <vscale x 4 x half> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv4i16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv4i16.nxv4f16(
@@ -150,10 +150,10 @@ declare <vscale x 8 x i16> @llvm.riscv.vfcvt.x.f.v.nxv8i16.nxv8f16(
 define <vscale x 8 x i16> @intrinsic_vfcvt_x.f.v_nxv8i16_nxv8f16(<vscale x 8 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv8i16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vfcvt.x.f.v.nxv8i16.nxv8f16(
@@ -173,10 +173,10 @@ declare <vscale x 8 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv8i16.nxv8f16(
 define <vscale x 8 x i16> @intrinsic_vfcvt_mask_x.f.v_nxv8i16_nxv8f16(<vscale x 8 x i16> %0, <vscale x 8 x half> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv8i16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv8i16.nxv8f16(
@@ -196,10 +196,10 @@ declare <vscale x 16 x i16> @llvm.riscv.vfcvt.x.f.v.nxv16i16.nxv16f16(
 define <vscale x 16 x i16> @intrinsic_vfcvt_x.f.v_nxv16i16_nxv16f16(<vscale x 16 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv16i16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vfcvt.x.f.v.nxv16i16.nxv16f16(
@@ -219,10 +219,10 @@ declare <vscale x 16 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv16i16.nxv16f16(
 define <vscale x 16 x i16> @intrinsic_vfcvt_mask_x.f.v_nxv16i16_nxv16f16(<vscale x 16 x i16> %0, <vscale x 16 x half> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv16i16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv16i16.nxv16f16(
@@ -242,10 +242,10 @@ declare <vscale x 32 x i16> @llvm.riscv.vfcvt.x.f.v.nxv32i16.nxv32f16(
 define <vscale x 32 x i16> @intrinsic_vfcvt_x.f.v_nxv32i16_nxv32f16(<vscale x 32 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv32i16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x i16> @llvm.riscv.vfcvt.x.f.v.nxv32i16.nxv32f16(
@@ -265,10 +265,10 @@ declare <vscale x 32 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv32i16.nxv32f16(
 define <vscale x 32 x i16> @intrinsic_vfcvt_mask_x.f.v_nxv32i16_nxv32f16(<vscale x 32 x i16> %0, <vscale x 32 x half> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv32i16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x i16> @llvm.riscv.vfcvt.x.f.v.mask.nxv32i16.nxv32f16(
@@ -288,10 +288,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfcvt.x.f.v.nxv1i32.nxv1f32(
 define <vscale x 1 x i32> @intrinsic_vfcvt_x.f.v_nxv1i32_nxv1f32(<vscale x 1 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv1i32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vfcvt.x.f.v.nxv1i32.nxv1f32(
@@ -311,10 +311,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfcvt.x.f.v.mask.nxv1i32.nxv1f32(
 define <vscale x 1 x i32> @intrinsic_vfcvt_mask_x.f.v_nxv1i32_nxv1f32(<vscale x 1 x i32> %0, <vscale x 1 x float> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv1i32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vfcvt.x.f.v.mask.nxv1i32.nxv1f32(
@@ -334,10 +334,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfcvt.x.f.v.nxv2i32.nxv2f32(
 define <vscale x 2 x i32> @intrinsic_vfcvt_x.f.v_nxv2i32_nxv2f32(<vscale x 2 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv2i32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vfcvt.x.f.v.nxv2i32.nxv2f32(
@@ -357,10 +357,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfcvt.x.f.v.mask.nxv2i32.nxv2f32(
 define <vscale x 2 x i32> @intrinsic_vfcvt_mask_x.f.v_nxv2i32_nxv2f32(<vscale x 2 x i32> %0, <vscale x 2 x float> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv2i32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vfcvt.x.f.v.mask.nxv2i32.nxv2f32(
@@ -380,10 +380,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfcvt.x.f.v.nxv4i32.nxv4f32(
 define <vscale x 4 x i32> @intrinsic_vfcvt_x.f.v_nxv4i32_nxv4f32(<vscale x 4 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv4i32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vfcvt.x.f.v.nxv4i32.nxv4f32(
@@ -403,10 +403,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfcvt.x.f.v.mask.nxv4i32.nxv4f32(
 define <vscale x 4 x i32> @intrinsic_vfcvt_mask_x.f.v_nxv4i32_nxv4f32(<vscale x 4 x i32> %0, <vscale x 4 x float> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv4i32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vfcvt.x.f.v.mask.nxv4i32.nxv4f32(
@@ -426,10 +426,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfcvt.x.f.v.nxv8i32.nxv8f32(
 define <vscale x 8 x i32> @intrinsic_vfcvt_x.f.v_nxv8i32_nxv8f32(<vscale x 8 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv8i32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vfcvt.x.f.v.nxv8i32.nxv8f32(
@@ -449,10 +449,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfcvt.x.f.v.mask.nxv8i32.nxv8f32(
 define <vscale x 8 x i32> @intrinsic_vfcvt_mask_x.f.v_nxv8i32_nxv8f32(<vscale x 8 x i32> %0, <vscale x 8 x float> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv8i32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vfcvt.x.f.v.mask.nxv8i32.nxv8f32(
@@ -472,10 +472,10 @@ declare <vscale x 16 x i32> @llvm.riscv.vfcvt.x.f.v.nxv16i32.nxv16f32(
 define <vscale x 16 x i32> @intrinsic_vfcvt_x.f.v_nxv16i32_nxv16f32(<vscale x 16 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv16i32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vfcvt.x.f.v.nxv16i32.nxv16f32(
@@ -495,10 +495,10 @@ declare <vscale x 16 x i32> @llvm.riscv.vfcvt.x.f.v.mask.nxv16i32.nxv16f32(
 define <vscale x 16 x i32> @intrinsic_vfcvt_mask_x.f.v_nxv16i32_nxv16f32(<vscale x 16 x i32> %0, <vscale x 16 x float> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv16i32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vfcvt.x.f.v.mask.nxv16i32.nxv16f32(
@@ -518,10 +518,10 @@ declare <vscale x 1 x i64> @llvm.riscv.vfcvt.x.f.v.nxv1i64.nxv1f64(
 define <vscale x 1 x i64> @intrinsic_vfcvt_x.f.v_nxv1i64_nxv1f64(<vscale x 1 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv1i64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vfcvt.x.f.v.nxv1i64.nxv1f64(
@@ -541,10 +541,10 @@ declare <vscale x 1 x i64> @llvm.riscv.vfcvt.x.f.v.mask.nxv1i64.nxv1f64(
 define <vscale x 1 x i64> @intrinsic_vfcvt_mask_x.f.v_nxv1i64_nxv1f64(<vscale x 1 x i64> %0, <vscale x 1 x double> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv1i64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vfcvt.x.f.v.mask.nxv1i64.nxv1f64(
@@ -564,10 +564,10 @@ declare <vscale x 2 x i64> @llvm.riscv.vfcvt.x.f.v.nxv2i64.nxv2f64(
 define <vscale x 2 x i64> @intrinsic_vfcvt_x.f.v_nxv2i64_nxv2f64(<vscale x 2 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv2i64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vfcvt.x.f.v.nxv2i64.nxv2f64(
@@ -587,10 +587,10 @@ declare <vscale x 2 x i64> @llvm.riscv.vfcvt.x.f.v.mask.nxv2i64.nxv2f64(
 define <vscale x 2 x i64> @intrinsic_vfcvt_mask_x.f.v_nxv2i64_nxv2f64(<vscale x 2 x i64> %0, <vscale x 2 x double> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv2i64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vfcvt.x.f.v.mask.nxv2i64.nxv2f64(
@@ -610,10 +610,10 @@ declare <vscale x 4 x i64> @llvm.riscv.vfcvt.x.f.v.nxv4i64.nxv4f64(
 define <vscale x 4 x i64> @intrinsic_vfcvt_x.f.v_nxv4i64_nxv4f64(<vscale x 4 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv4i64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vfcvt.x.f.v.nxv4i64.nxv4f64(
@@ -633,10 +633,10 @@ declare <vscale x 4 x i64> @llvm.riscv.vfcvt.x.f.v.mask.nxv4i64.nxv4f64(
 define <vscale x 4 x i64> @intrinsic_vfcvt_mask_x.f.v_nxv4i64_nxv4f64(<vscale x 4 x i64> %0, <vscale x 4 x double> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv4i64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vfcvt.x.f.v.mask.nxv4i64.nxv4f64(
@@ -656,10 +656,10 @@ declare <vscale x 8 x i64> @llvm.riscv.vfcvt.x.f.v.nxv8i64.nxv8f64(
 define <vscale x 8 x i64> @intrinsic_vfcvt_x.f.v_nxv8i64_nxv8f64(<vscale x 8 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_x.f.v_nxv8i64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vfcvt.x.f.v.nxv8i64.nxv8f64(
@@ -679,10 +679,10 @@ declare <vscale x 8 x i64> @llvm.riscv.vfcvt.x.f.v.mask.nxv8i64.nxv8f64(
 define <vscale x 8 x i64> @intrinsic_vfcvt_mask_x.f.v_nxv8i64_nxv8f64(<vscale x 8 x i64> %0, <vscale x 8 x double> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f.v_nxv8i64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.x.f.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vfcvt.x.f.v.mask.nxv8i64.nxv8f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfcvt-xu-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfcvt-xu-f.ll
index cd227196b4f4..93716ba7f451 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfcvt-xu-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfcvt-xu-f.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv1i16.nxv1f16(
 define <vscale x 1 x i16> @intrinsic_vfcvt_xu.f.v_nxv1i16_nxv1f16(<vscale x 1 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv1i16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv1i16.nxv1f16(
@@ -35,10 +35,10 @@ declare <vscale x 1 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv1i16.nxv1f16(
 define <vscale x 1 x i16> @intrinsic_vfcvt_mask_xu.f.v_nxv1i16_nxv1f16(<vscale x 1 x i16> %0, <vscale x 1 x half> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv1i16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv1i16.nxv1f16(
@@ -58,10 +58,10 @@ declare <vscale x 2 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv2i16.nxv2f16(
 define <vscale x 2 x i16> @intrinsic_vfcvt_xu.f.v_nxv2i16_nxv2f16(<vscale x 2 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv2i16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv2i16.nxv2f16(
@@ -81,10 +81,10 @@ declare <vscale x 2 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv2i16.nxv2f16(
 define <vscale x 2 x i16> @intrinsic_vfcvt_mask_xu.f.v_nxv2i16_nxv2f16(<vscale x 2 x i16> %0, <vscale x 2 x half> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv2i16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv2i16.nxv2f16(
@@ -104,10 +104,10 @@ declare <vscale x 4 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv4i16.nxv4f16(
 define <vscale x 4 x i16> @intrinsic_vfcvt_xu.f.v_nxv4i16_nxv4f16(<vscale x 4 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv4i16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv4i16.nxv4f16(
@@ -127,10 +127,10 @@ declare <vscale x 4 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv4i16.nxv4f16(
 define <vscale x 4 x i16> @intrinsic_vfcvt_mask_xu.f.v_nxv4i16_nxv4f16(<vscale x 4 x i16> %0, <vscale x 4 x half> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv4i16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv4i16.nxv4f16(
@@ -150,10 +150,10 @@ declare <vscale x 8 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv8i16.nxv8f16(
 define <vscale x 8 x i16> @intrinsic_vfcvt_xu.f.v_nxv8i16_nxv8f16(<vscale x 8 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv8i16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv8i16.nxv8f16(
@@ -173,10 +173,10 @@ declare <vscale x 8 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv8i16.nxv8f16(
 define <vscale x 8 x i16> @intrinsic_vfcvt_mask_xu.f.v_nxv8i16_nxv8f16(<vscale x 8 x i16> %0, <vscale x 8 x half> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv8i16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv8i16.nxv8f16(
@@ -196,10 +196,10 @@ declare <vscale x 16 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv16i16.nxv16f16(
 define <vscale x 16 x i16> @intrinsic_vfcvt_xu.f.v_nxv16i16_nxv16f16(<vscale x 16 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv16i16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv16i16.nxv16f16(
@@ -219,10 +219,10 @@ declare <vscale x 16 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv16i16.nxv16f16(
 define <vscale x 16 x i16> @intrinsic_vfcvt_mask_xu.f.v_nxv16i16_nxv16f16(<vscale x 16 x i16> %0, <vscale x 16 x half> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv16i16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv16i16.nxv16f16(
@@ -242,10 +242,10 @@ declare <vscale x 32 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv32i16.nxv32f16(
 define <vscale x 32 x i16> @intrinsic_vfcvt_xu.f.v_nxv32i16_nxv32f16(<vscale x 32 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv32i16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x i16> @llvm.riscv.vfcvt.xu.f.v.nxv32i16.nxv32f16(
@@ -265,10 +265,10 @@ declare <vscale x 32 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv32i16.nxv32f16(
 define <vscale x 32 x i16> @intrinsic_vfcvt_mask_xu.f.v_nxv32i16_nxv32f16(<vscale x 32 x i16> %0, <vscale x 32 x half> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv32i16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x i16> @llvm.riscv.vfcvt.xu.f.v.mask.nxv32i16.nxv32f16(
@@ -288,10 +288,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfcvt.xu.f.v.nxv1i32.nxv1f32(
 define <vscale x 1 x i32> @intrinsic_vfcvt_xu.f.v_nxv1i32_nxv1f32(<vscale x 1 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv1i32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vfcvt.xu.f.v.nxv1i32.nxv1f32(
@@ -311,10 +311,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfcvt.xu.f.v.mask.nxv1i32.nxv1f32(
 define <vscale x 1 x i32> @intrinsic_vfcvt_mask_xu.f.v_nxv1i32_nxv1f32(<vscale x 1 x i32> %0, <vscale x 1 x float> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv1i32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vfcvt.xu.f.v.mask.nxv1i32.nxv1f32(
@@ -334,10 +334,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfcvt.xu.f.v.nxv2i32.nxv2f32(
 define <vscale x 2 x i32> @intrinsic_vfcvt_xu.f.v_nxv2i32_nxv2f32(<vscale x 2 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv2i32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vfcvt.xu.f.v.nxv2i32.nxv2f32(
@@ -357,10 +357,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfcvt.xu.f.v.mask.nxv2i32.nxv2f32(
 define <vscale x 2 x i32> @intrinsic_vfcvt_mask_xu.f.v_nxv2i32_nxv2f32(<vscale x 2 x i32> %0, <vscale x 2 x float> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv2i32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vfcvt.xu.f.v.mask.nxv2i32.nxv2f32(
@@ -380,10 +380,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfcvt.xu.f.v.nxv4i32.nxv4f32(
 define <vscale x 4 x i32> @intrinsic_vfcvt_xu.f.v_nxv4i32_nxv4f32(<vscale x 4 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv4i32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vfcvt.xu.f.v.nxv4i32.nxv4f32(
@@ -403,10 +403,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfcvt.xu.f.v.mask.nxv4i32.nxv4f32(
 define <vscale x 4 x i32> @intrinsic_vfcvt_mask_xu.f.v_nxv4i32_nxv4f32(<vscale x 4 x i32> %0, <vscale x 4 x float> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv4i32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vfcvt.xu.f.v.mask.nxv4i32.nxv4f32(
@@ -426,10 +426,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfcvt.xu.f.v.nxv8i32.nxv8f32(
 define <vscale x 8 x i32> @intrinsic_vfcvt_xu.f.v_nxv8i32_nxv8f32(<vscale x 8 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv8i32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vfcvt.xu.f.v.nxv8i32.nxv8f32(
@@ -449,10 +449,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfcvt.xu.f.v.mask.nxv8i32.nxv8f32(
 define <vscale x 8 x i32> @intrinsic_vfcvt_mask_xu.f.v_nxv8i32_nxv8f32(<vscale x 8 x i32> %0, <vscale x 8 x float> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv8i32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vfcvt.xu.f.v.mask.nxv8i32.nxv8f32(
@@ -472,10 +472,10 @@ declare <vscale x 16 x i32> @llvm.riscv.vfcvt.xu.f.v.nxv16i32.nxv16f32(
 define <vscale x 16 x i32> @intrinsic_vfcvt_xu.f.v_nxv16i32_nxv16f32(<vscale x 16 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv16i32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vfcvt.xu.f.v.nxv16i32.nxv16f32(
@@ -495,10 +495,10 @@ declare <vscale x 16 x i32> @llvm.riscv.vfcvt.xu.f.v.mask.nxv16i32.nxv16f32(
 define <vscale x 16 x i32> @intrinsic_vfcvt_mask_xu.f.v_nxv16i32_nxv16f32(<vscale x 16 x i32> %0, <vscale x 16 x float> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv16i32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vfcvt.xu.f.v.mask.nxv16i32.nxv16f32(
@@ -518,10 +518,10 @@ declare <vscale x 1 x i64> @llvm.riscv.vfcvt.xu.f.v.nxv1i64.nxv1f64(
 define <vscale x 1 x i64> @intrinsic_vfcvt_xu.f.v_nxv1i64_nxv1f64(<vscale x 1 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv1i64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vfcvt.xu.f.v.nxv1i64.nxv1f64(
@@ -541,10 +541,10 @@ declare <vscale x 1 x i64> @llvm.riscv.vfcvt.xu.f.v.mask.nxv1i64.nxv1f64(
 define <vscale x 1 x i64> @intrinsic_vfcvt_mask_xu.f.v_nxv1i64_nxv1f64(<vscale x 1 x i64> %0, <vscale x 1 x double> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv1i64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vfcvt.xu.f.v.mask.nxv1i64.nxv1f64(
@@ -564,10 +564,10 @@ declare <vscale x 2 x i64> @llvm.riscv.vfcvt.xu.f.v.nxv2i64.nxv2f64(
 define <vscale x 2 x i64> @intrinsic_vfcvt_xu.f.v_nxv2i64_nxv2f64(<vscale x 2 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv2i64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vfcvt.xu.f.v.nxv2i64.nxv2f64(
@@ -587,10 +587,10 @@ declare <vscale x 2 x i64> @llvm.riscv.vfcvt.xu.f.v.mask.nxv2i64.nxv2f64(
 define <vscale x 2 x i64> @intrinsic_vfcvt_mask_xu.f.v_nxv2i64_nxv2f64(<vscale x 2 x i64> %0, <vscale x 2 x double> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv2i64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vfcvt.xu.f.v.mask.nxv2i64.nxv2f64(
@@ -610,10 +610,10 @@ declare <vscale x 4 x i64> @llvm.riscv.vfcvt.xu.f.v.nxv4i64.nxv4f64(
 define <vscale x 4 x i64> @intrinsic_vfcvt_xu.f.v_nxv4i64_nxv4f64(<vscale x 4 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv4i64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vfcvt.xu.f.v.nxv4i64.nxv4f64(
@@ -633,10 +633,10 @@ declare <vscale x 4 x i64> @llvm.riscv.vfcvt.xu.f.v.mask.nxv4i64.nxv4f64(
 define <vscale x 4 x i64> @intrinsic_vfcvt_mask_xu.f.v_nxv4i64_nxv4f64(<vscale x 4 x i64> %0, <vscale x 4 x double> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv4i64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vfcvt.xu.f.v.mask.nxv4i64.nxv4f64(
@@ -656,10 +656,10 @@ declare <vscale x 8 x i64> @llvm.riscv.vfcvt.xu.f.v.nxv8i64.nxv8f64(
 define <vscale x 8 x i64> @intrinsic_vfcvt_xu.f.v_nxv8i64_nxv8f64(<vscale x 8 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f.v_nxv8i64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vfcvt.xu.f.v.nxv8i64.nxv8f64(
@@ -679,10 +679,10 @@ declare <vscale x 8 x i64> @llvm.riscv.vfcvt.xu.f.v.mask.nxv8i64.nxv8f64(
 define <vscale x 8 x i64> @intrinsic_vfcvt_mask_xu.f.v_nxv8i64_nxv8f64(<vscale x 8 x i64> %0, <vscale x 8 x double> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f.v_nxv8i64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfcvt.xu.f.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vfcvt.xu.f.v.mask.nxv8i64.nxv8f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfdiv.ll b/llvm/test/CodeGen/RISCV/rvv/vfdiv.ll
index 7e77fb7dc2ed..3f67c433bcbf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfdiv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfdiv.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfdiv.nxv1f16.nxv1f16(
 define <vscale x 1 x half> @intrinsic_vfdiv_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfdiv.nxv1f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfdiv.mask.nxv1f16.nxv1f16(
 define <vscale x 1 x half> @intrinsic_vfdiv_mask_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfdiv.mask.nxv1f16.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfdiv.nxv2f16.nxv2f16(
 define <vscale x 2 x half> @intrinsic_vfdiv_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfdiv.nxv2f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfdiv.mask.nxv2f16.nxv2f16(
 define <vscale x 2 x half> @intrinsic_vfdiv_mask_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfdiv.mask.nxv2f16.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfdiv.nxv4f16.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfdiv_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfdiv.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfdiv.mask.nxv4f16.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfdiv_mask_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfdiv.mask.nxv4f16.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfdiv.nxv8f16.nxv8f16(
 define <vscale x 8 x half> @intrinsic_vfdiv_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfdiv.nxv8f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfdiv.mask.nxv8f16.nxv8f16(
 define <vscale x 8 x half> @intrinsic_vfdiv_mask_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfdiv.mask.nxv8f16.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfdiv.nxv16f16.nxv16f16(
 define <vscale x 16 x half> @intrinsic_vfdiv_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfdiv.nxv16f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfdiv.mask.nxv16f16.nxv16f16(
 define <vscale x 16 x half> @intrinsic_vfdiv_mask_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfdiv.mask.nxv16f16.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfdiv.nxv32f16.nxv32f16(
 define <vscale x 32 x half> @intrinsic_vfdiv_vv_nxv32f16_nxv32f16_nxv32f16(<vscale x 32 x half> %0, <vscale x 32 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv32f16_nxv32f16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfdiv.nxv32f16.nxv32f16(
@@ -289,8 +289,8 @@ define <vscale x 32 x half> @intrinsic_vfdiv_mask_vv_nxv32f16_nxv32f16_nxv32f16(
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv32f16_nxv32f16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vfdiv.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -314,10 +314,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfdiv.nxv1f32.nxv1f32(
 define <vscale x 1 x float> @intrinsic_vfdiv_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfdiv.nxv1f32.nxv1f32(
@@ -339,10 +339,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfdiv.mask.nxv1f32.nxv1f32(
 define <vscale x 1 x float> @intrinsic_vfdiv_mask_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfdiv.mask.nxv1f32.nxv1f32(
@@ -364,10 +364,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfdiv.nxv2f32.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfdiv_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfdiv.nxv2f32.nxv2f32(
@@ -389,10 +389,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfdiv.mask.nxv2f32.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfdiv_mask_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfdiv.mask.nxv2f32.nxv2f32(
@@ -414,10 +414,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfdiv.nxv4f32.nxv4f32(
 define <vscale x 4 x float> @intrinsic_vfdiv_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfdiv.nxv4f32.nxv4f32(
@@ -439,10 +439,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfdiv.mask.nxv4f32.nxv4f32(
 define <vscale x 4 x float> @intrinsic_vfdiv_mask_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfdiv.mask.nxv4f32.nxv4f32(
@@ -464,10 +464,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfdiv.nxv8f32.nxv8f32(
 define <vscale x 8 x float> @intrinsic_vfdiv_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfdiv.nxv8f32.nxv8f32(
@@ -489,10 +489,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfdiv.mask.nxv8f32.nxv8f32(
 define <vscale x 8 x float> @intrinsic_vfdiv_mask_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfdiv.mask.nxv8f32.nxv8f32(
@@ -514,10 +514,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfdiv.nxv16f32.nxv16f32(
 define <vscale x 16 x float> @intrinsic_vfdiv_vv_nxv16f32_nxv16f32_nxv16f32(<vscale x 16 x float> %0, <vscale x 16 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv16f32_nxv16f32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfdiv.nxv16f32.nxv16f32(
@@ -540,8 +540,8 @@ define <vscale x 16 x float> @intrinsic_vfdiv_mask_vv_nxv16f32_nxv16f32_nxv16f32
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv16f32_nxv16f32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vfdiv.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -565,10 +565,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfdiv.nxv1f64.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfdiv_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfdiv.nxv1f64.nxv1f64(
@@ -590,10 +590,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfdiv.mask.nxv1f64.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfdiv_mask_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfdiv.mask.nxv1f64.nxv1f64(
@@ -615,10 +615,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfdiv.nxv2f64.nxv2f64(
 define <vscale x 2 x double> @intrinsic_vfdiv_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfdiv.nxv2f64.nxv2f64(
@@ -640,10 +640,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfdiv.mask.nxv2f64.nxv2f64(
 define <vscale x 2 x double> @intrinsic_vfdiv_mask_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfdiv.mask.nxv2f64.nxv2f64(
@@ -665,10 +665,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfdiv.nxv4f64.nxv4f64(
 define <vscale x 4 x double> @intrinsic_vfdiv_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfdiv.nxv4f64.nxv4f64(
@@ -690,10 +690,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfdiv.mask.nxv4f64.nxv4f64(
 define <vscale x 4 x double> @intrinsic_vfdiv_mask_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfdiv.mask.nxv4f64.nxv4f64(
@@ -715,10 +715,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfdiv.nxv8f64.nxv8f64(
 define <vscale x 8 x double> @intrinsic_vfdiv_vv_nxv8f64_nxv8f64_nxv8f64(<vscale x 8 x double> %0, <vscale x 8 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv8f64_nxv8f64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfdiv.nxv8f64.nxv8f64(
@@ -741,8 +741,8 @@ define <vscale x 8 x double> @intrinsic_vfdiv_mask_vv_nxv8f64_nxv8f64_nxv8f64(<v
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv8f64_nxv8f64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    vfdiv.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -766,10 +766,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfdiv.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfdiv_vf_nxv1f16_nxv1f16_f16(<vscale x 1 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv1f16_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfdiv.nxv1f16.f16(
@@ -791,10 +791,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfdiv.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfdiv_mask_vf_nxv1f16_nxv1f16_f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv1f16_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfdiv.mask.nxv1f16.f16(
@@ -816,10 +816,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfdiv.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfdiv_vf_nxv2f16_nxv2f16_f16(<vscale x 2 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv2f16_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfdiv.nxv2f16.f16(
@@ -841,10 +841,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfdiv.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfdiv_mask_vf_nxv2f16_nxv2f16_f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv2f16_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfdiv.mask.nxv2f16.f16(
@@ -866,10 +866,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfdiv.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfdiv_vf_nxv4f16_nxv4f16_f16(<vscale x 4 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv4f16_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfdiv.nxv4f16.f16(
@@ -891,10 +891,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfdiv.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfdiv_mask_vf_nxv4f16_nxv4f16_f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv4f16_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfdiv.mask.nxv4f16.f16(
@@ -916,10 +916,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfdiv.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfdiv_vf_nxv8f16_nxv8f16_f16(<vscale x 8 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv8f16_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfdiv.nxv8f16.f16(
@@ -941,10 +941,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfdiv.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfdiv_mask_vf_nxv8f16_nxv8f16_f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv8f16_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfdiv.mask.nxv8f16.f16(
@@ -966,10 +966,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfdiv.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfdiv_vf_nxv16f16_nxv16f16_f16(<vscale x 16 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv16f16_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfdiv.nxv16f16.f16(
@@ -991,10 +991,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfdiv.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfdiv_mask_vf_nxv16f16_nxv16f16_f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv16f16_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfdiv.mask.nxv16f16.f16(
@@ -1016,10 +1016,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfdiv.nxv32f16.f16(
 define <vscale x 32 x half> @intrinsic_vfdiv_vf_nxv32f16_nxv32f16_f16(<vscale x 32 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv32f16_nxv32f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfdiv.nxv32f16.f16(
@@ -1041,10 +1041,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfdiv.mask.nxv32f16.f16(
 define <vscale x 32 x half> @intrinsic_vfdiv_mask_vf_nxv32f16_nxv32f16_f16(<vscale x 32 x half> %0, <vscale x 32 x half> %1, half %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv32f16_nxv32f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfdiv.mask.nxv32f16.f16(
@@ -1066,10 +1066,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfdiv.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfdiv_vf_nxv1f32_nxv1f32_f32(<vscale x 1 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv1f32_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfdiv.nxv1f32.f32(
@@ -1091,10 +1091,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfdiv.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfdiv_mask_vf_nxv1f32_nxv1f32_f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv1f32_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfdiv.mask.nxv1f32.f32(
@@ -1116,10 +1116,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfdiv.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfdiv_vf_nxv2f32_nxv2f32_f32(<vscale x 2 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv2f32_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfdiv.nxv2f32.f32(
@@ -1141,10 +1141,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfdiv.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfdiv_mask_vf_nxv2f32_nxv2f32_f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv2f32_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfdiv.mask.nxv2f32.f32(
@@ -1166,10 +1166,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfdiv.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfdiv_vf_nxv4f32_nxv4f32_f32(<vscale x 4 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv4f32_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfdiv.nxv4f32.f32(
@@ -1191,10 +1191,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfdiv.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfdiv_mask_vf_nxv4f32_nxv4f32_f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv4f32_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfdiv.mask.nxv4f32.f32(
@@ -1216,10 +1216,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfdiv.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfdiv_vf_nxv8f32_nxv8f32_f32(<vscale x 8 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv8f32_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfdiv.nxv8f32.f32(
@@ -1241,10 +1241,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfdiv.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfdiv_mask_vf_nxv8f32_nxv8f32_f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv8f32_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfdiv.mask.nxv8f32.f32(
@@ -1266,10 +1266,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfdiv.nxv16f32.f32(
 define <vscale x 16 x float> @intrinsic_vfdiv_vf_nxv16f32_nxv16f32_f32(<vscale x 16 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv16f32_nxv16f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfdiv.nxv16f32.f32(
@@ -1291,10 +1291,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfdiv.mask.nxv16f32.f32(
 define <vscale x 16 x float> @intrinsic_vfdiv_mask_vf_nxv16f32_nxv16f32_f32(<vscale x 16 x float> %0, <vscale x 16 x float> %1, float %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv16f32_nxv16f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfdiv.mask.nxv16f32.f32(
@@ -1316,10 +1316,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfdiv.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfdiv_vf_nxv1f64_nxv1f64_f64(<vscale x 1 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv1f64_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfdiv.nxv1f64.f64(
@@ -1341,10 +1341,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfdiv.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfdiv_mask_vf_nxv1f64_nxv1f64_f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, double %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv1f64_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfdiv.mask.nxv1f64.f64(
@@ -1366,10 +1366,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfdiv.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfdiv_vf_nxv2f64_nxv2f64_f64(<vscale x 2 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv2f64_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfdiv.nxv2f64.f64(
@@ -1391,10 +1391,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfdiv.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfdiv_mask_vf_nxv2f64_nxv2f64_f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, double %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv2f64_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfdiv.mask.nxv2f64.f64(
@@ -1416,10 +1416,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfdiv.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfdiv_vf_nxv4f64_nxv4f64_f64(<vscale x 4 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv4f64_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfdiv.nxv4f64.f64(
@@ -1441,10 +1441,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfdiv.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfdiv_mask_vf_nxv4f64_nxv4f64_f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, double %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv4f64_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfdiv.mask.nxv4f64.f64(
@@ -1466,10 +1466,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfdiv.nxv8f64.f64(
 define <vscale x 8 x double> @intrinsic_vfdiv_vf_nxv8f64_nxv8f64_f64(<vscale x 8 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv8f64_nxv8f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfdiv.nxv8f64.f64(
@@ -1491,10 +1491,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfdiv.mask.nxv8f64.f64(
 define <vscale x 8 x double> @intrinsic_vfdiv_mask_vf_nxv8f64_nxv8f64_f64(<vscale x 8 x double> %0, <vscale x 8 x double> %1, double %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv8f64_nxv8f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfdiv.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfdiv.mask.nxv8f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmacc.ll b/llvm/test/CodeGen/RISCV/rvv/vfmacc.ll
index 73d0178a939c..5586b52b64ec 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmacc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmacc.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmacc.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfmacc_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmacc.nxv1f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmacc.mask.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfmacc_mask_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmacc.mask.nxv1f16.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmacc.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfmacc_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmacc.nxv2f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmacc.mask.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfmacc_mask_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmacc.mask.nxv2f16.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmacc.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfmacc_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmacc.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmacc.mask.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfmacc_mask_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmacc.mask.nxv4f16.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmacc.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfmacc_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmacc.nxv8f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmacc.mask.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfmacc_mask_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmacc.mask.nxv8f16.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmacc.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfmacc_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmacc.nxv16f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmacc.mask.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfmacc_mask_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmacc.mask.nxv16f16.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmacc.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfmacc_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmacc.nxv1f32.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmacc.mask.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfmacc_mask_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmacc.mask.nxv1f32.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmacc.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfmacc_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmacc.nxv2f32.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmacc.mask.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfmacc_mask_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmacc.mask.nxv2f32.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmacc.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfmacc_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmacc.nxv4f32.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmacc.mask.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfmacc_mask_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmacc.mask.nxv4f32.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmacc.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfmacc_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmacc.nxv8f32.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmacc.mask.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfmacc_mask_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmacc.mask.nxv8f32.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmacc.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfmacc_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmacc.nxv1f64.nxv1f64(
@@ -488,10 +488,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmacc.mask.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfmacc_mask_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmacc.mask.nxv1f64.nxv1f64(
@@ -513,10 +513,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmacc.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfmacc_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmacc.nxv2f64.nxv2f64(
@@ -538,10 +538,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmacc.mask.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfmacc_mask_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmacc.mask.nxv2f64.nxv2f64(
@@ -563,10 +563,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmacc.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfmacc_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmacc.nxv4f64.nxv4f64(
@@ -588,10 +588,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmacc.mask.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfmacc_mask_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmacc.mask.nxv4f64.nxv4f64(
@@ -613,10 +613,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmacc.nxv1f16.f16(
 define <vscale x 1 x half>  @intrinsic_vfmacc_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmacc.nxv1f16.f16(
@@ -638,10 +638,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmacc.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfmacc_mask_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmacc.mask.nxv1f16.f16(
@@ -663,10 +663,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmacc.nxv2f16.f16(
 define <vscale x 2 x half>  @intrinsic_vfmacc_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmacc.nxv2f16.f16(
@@ -688,10 +688,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmacc.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfmacc_mask_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmacc.mask.nxv2f16.f16(
@@ -713,10 +713,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmacc.nxv4f16.f16(
 define <vscale x 4 x half>  @intrinsic_vfmacc_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmacc.nxv4f16.f16(
@@ -738,10 +738,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmacc.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfmacc_mask_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmacc.mask.nxv4f16.f16(
@@ -763,10 +763,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmacc.nxv8f16.f16(
 define <vscale x 8 x half>  @intrinsic_vfmacc_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmacc.nxv8f16.f16(
@@ -788,10 +788,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmacc.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfmacc_mask_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmacc.mask.nxv8f16.f16(
@@ -813,10 +813,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmacc.nxv16f16.f16(
 define <vscale x 16 x half>  @intrinsic_vfmacc_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmacc.nxv16f16.f16(
@@ -838,10 +838,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmacc.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfmacc_mask_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmacc.mask.nxv16f16.f16(
@@ -863,10 +863,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmacc.nxv1f32.f32(
 define <vscale x 1 x float>  @intrinsic_vfmacc_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmacc.nxv1f32.f32(
@@ -888,10 +888,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmacc.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfmacc_mask_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmacc.mask.nxv1f32.f32(
@@ -913,10 +913,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmacc.nxv2f32.f32(
 define <vscale x 2 x float>  @intrinsic_vfmacc_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmacc.nxv2f32.f32(
@@ -938,10 +938,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmacc.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfmacc_mask_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmacc.mask.nxv2f32.f32(
@@ -963,10 +963,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmacc.nxv4f32.f32(
 define <vscale x 4 x float>  @intrinsic_vfmacc_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmacc.nxv4f32.f32(
@@ -988,10 +988,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmacc.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfmacc_mask_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmacc.mask.nxv4f32.f32(
@@ -1013,10 +1013,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmacc.nxv8f32.f32(
 define <vscale x 8 x float>  @intrinsic_vfmacc_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmacc.nxv8f32.f32(
@@ -1038,10 +1038,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmacc.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfmacc_mask_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmacc.mask.nxv8f32.f32(
@@ -1063,10 +1063,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmacc.nxv1f64.f64(
 define <vscale x 1 x double>  @intrinsic_vfmacc_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmacc.nxv1f64.f64(
@@ -1088,10 +1088,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmacc.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfmacc_mask_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmacc.mask.nxv1f64.f64(
@@ -1113,10 +1113,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmacc.nxv2f64.f64(
 define <vscale x 2 x double>  @intrinsic_vfmacc_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmacc.nxv2f64.f64(
@@ -1138,10 +1138,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmacc.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfmacc_mask_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmacc.mask.nxv2f64.f64(
@@ -1163,10 +1163,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmacc.nxv4f64.f64(
 define <vscale x 4 x double>  @intrinsic_vfmacc_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmacc.nxv4f64.f64(
@@ -1188,10 +1188,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmacc.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfmacc_mask_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmacc.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmacc.mask.nxv4f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmadd.ll b/llvm/test/CodeGen/RISCV/rvv/vfmadd.ll
index caad65c78e66..c44690d23f08 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmadd.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmadd.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfmadd_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmadd.nxv1f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmadd.mask.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfmadd_mask_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmadd.mask.nxv1f16.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmadd.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfmadd_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmadd.nxv2f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmadd.mask.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfmadd_mask_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmadd.mask.nxv2f16.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmadd.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfmadd_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmadd.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmadd.mask.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfmadd_mask_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmadd.mask.nxv4f16.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmadd.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfmadd_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmadd.nxv8f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmadd.mask.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfmadd_mask_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmadd.mask.nxv8f16.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmadd.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfmadd_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmadd.nxv16f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmadd.mask.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfmadd_mask_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmadd.mask.nxv16f16.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmadd.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfmadd_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmadd.nxv1f32.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmadd.mask.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfmadd_mask_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmadd.mask.nxv1f32.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmadd.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfmadd_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmadd.nxv2f32.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmadd.mask.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfmadd_mask_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmadd.mask.nxv2f32.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmadd.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfmadd_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmadd.nxv4f32.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmadd.mask.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfmadd_mask_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmadd.mask.nxv4f32.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmadd.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfmadd_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmadd.nxv8f32.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmadd.mask.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfmadd_mask_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmadd.mask.nxv8f32.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmadd.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfmadd_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmadd.nxv1f64.nxv1f64(
@@ -488,10 +488,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmadd.mask.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfmadd_mask_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmadd.mask.nxv1f64.nxv1f64(
@@ -513,10 +513,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmadd.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfmadd_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmadd.nxv2f64.nxv2f64(
@@ -538,10 +538,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmadd.mask.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfmadd_mask_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmadd.mask.nxv2f64.nxv2f64(
@@ -563,10 +563,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmadd.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfmadd_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmadd.nxv4f64.nxv4f64(
@@ -588,10 +588,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmadd.mask.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfmadd_mask_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmadd.mask.nxv4f64.nxv4f64(
@@ -613,10 +613,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmadd.nxv1f16.f16(
 define <vscale x 1 x half>  @intrinsic_vfmadd_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmadd.nxv1f16.f16(
@@ -638,10 +638,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmadd.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfmadd_mask_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmadd.mask.nxv1f16.f16(
@@ -663,10 +663,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmadd.nxv2f16.f16(
 define <vscale x 2 x half>  @intrinsic_vfmadd_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmadd.nxv2f16.f16(
@@ -688,10 +688,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmadd.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfmadd_mask_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmadd.mask.nxv2f16.f16(
@@ -713,10 +713,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmadd.nxv4f16.f16(
 define <vscale x 4 x half>  @intrinsic_vfmadd_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmadd.nxv4f16.f16(
@@ -738,10 +738,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmadd.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfmadd_mask_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmadd.mask.nxv4f16.f16(
@@ -763,10 +763,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmadd.nxv8f16.f16(
 define <vscale x 8 x half>  @intrinsic_vfmadd_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmadd.nxv8f16.f16(
@@ -788,10 +788,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmadd.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfmadd_mask_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmadd.mask.nxv8f16.f16(
@@ -813,10 +813,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmadd.nxv16f16.f16(
 define <vscale x 16 x half>  @intrinsic_vfmadd_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmadd.nxv16f16.f16(
@@ -838,10 +838,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmadd.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfmadd_mask_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmadd.mask.nxv16f16.f16(
@@ -863,10 +863,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmadd.nxv1f32.f32(
 define <vscale x 1 x float>  @intrinsic_vfmadd_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmadd.nxv1f32.f32(
@@ -888,10 +888,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmadd.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfmadd_mask_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmadd.mask.nxv1f32.f32(
@@ -913,10 +913,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmadd.nxv2f32.f32(
 define <vscale x 2 x float>  @intrinsic_vfmadd_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmadd.nxv2f32.f32(
@@ -938,10 +938,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmadd.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfmadd_mask_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmadd.mask.nxv2f32.f32(
@@ -963,10 +963,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmadd.nxv4f32.f32(
 define <vscale x 4 x float>  @intrinsic_vfmadd_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmadd.nxv4f32.f32(
@@ -988,10 +988,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmadd.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfmadd_mask_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmadd.mask.nxv4f32.f32(
@@ -1013,10 +1013,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmadd.nxv8f32.f32(
 define <vscale x 8 x float>  @intrinsic_vfmadd_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmadd.nxv8f32.f32(
@@ -1038,10 +1038,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmadd.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfmadd_mask_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmadd.mask.nxv8f32.f32(
@@ -1063,10 +1063,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmadd.nxv1f64.f64(
 define <vscale x 1 x double>  @intrinsic_vfmadd_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmadd.nxv1f64.f64(
@@ -1088,10 +1088,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmadd.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfmadd_mask_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmadd.mask.nxv1f64.f64(
@@ -1113,10 +1113,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmadd.nxv2f64.f64(
 define <vscale x 2 x double>  @intrinsic_vfmadd_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmadd.nxv2f64.f64(
@@ -1138,10 +1138,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmadd.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfmadd_mask_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmadd.mask.nxv2f64.f64(
@@ -1163,10 +1163,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmadd.nxv4f64.f64(
 define <vscale x 4 x double>  @intrinsic_vfmadd_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmadd.nxv4f64.f64(
@@ -1188,10 +1188,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmadd.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfmadd_mask_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmadd.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmadd.mask.nxv4f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmsac.ll b/llvm/test/CodeGen/RISCV/rvv/vfmsac.ll
index e668a70050e4..4eac7b63fd88 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmsac.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmsac.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmsac.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfmsac_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmsac.nxv1f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmsac.mask.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfmsac_mask_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmsac.mask.nxv1f16.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmsac.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfmsac_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmsac.nxv2f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmsac.mask.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfmsac_mask_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmsac.mask.nxv2f16.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmsac.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfmsac_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmsac.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmsac.mask.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfmsac_mask_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmsac.mask.nxv4f16.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmsac.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfmsac_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmsac.nxv8f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmsac.mask.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfmsac_mask_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmsac.mask.nxv8f16.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmsac.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfmsac_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmsac.nxv16f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmsac.mask.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfmsac_mask_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmsac.mask.nxv16f16.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmsac.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfmsac_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmsac.nxv1f32.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmsac.mask.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfmsac_mask_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmsac.mask.nxv1f32.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmsac.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfmsac_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmsac.nxv2f32.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmsac.mask.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfmsac_mask_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmsac.mask.nxv2f32.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmsac.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfmsac_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmsac.nxv4f32.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmsac.mask.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfmsac_mask_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmsac.mask.nxv4f32.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmsac.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfmsac_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmsac.nxv8f32.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmsac.mask.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfmsac_mask_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmsac.mask.nxv8f32.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmsac.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfmsac_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmsac.nxv1f64.nxv1f64(
@@ -488,10 +488,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmsac.mask.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfmsac_mask_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmsac.mask.nxv1f64.nxv1f64(
@@ -513,10 +513,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmsac.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfmsac_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmsac.nxv2f64.nxv2f64(
@@ -538,10 +538,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmsac.mask.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfmsac_mask_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmsac.mask.nxv2f64.nxv2f64(
@@ -563,10 +563,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmsac.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfmsac_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmsac.nxv4f64.nxv4f64(
@@ -588,10 +588,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmsac.mask.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfmsac_mask_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmsac.mask.nxv4f64.nxv4f64(
@@ -613,10 +613,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmsac.nxv1f16.f16(
 define <vscale x 1 x half>  @intrinsic_vfmsac_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmsac.nxv1f16.f16(
@@ -638,10 +638,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmsac.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfmsac_mask_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmsac.mask.nxv1f16.f16(
@@ -663,10 +663,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmsac.nxv2f16.f16(
 define <vscale x 2 x half>  @intrinsic_vfmsac_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmsac.nxv2f16.f16(
@@ -688,10 +688,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmsac.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfmsac_mask_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmsac.mask.nxv2f16.f16(
@@ -713,10 +713,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmsac.nxv4f16.f16(
 define <vscale x 4 x half>  @intrinsic_vfmsac_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmsac.nxv4f16.f16(
@@ -738,10 +738,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmsac.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfmsac_mask_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmsac.mask.nxv4f16.f16(
@@ -763,10 +763,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmsac.nxv8f16.f16(
 define <vscale x 8 x half>  @intrinsic_vfmsac_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmsac.nxv8f16.f16(
@@ -788,10 +788,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmsac.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfmsac_mask_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmsac.mask.nxv8f16.f16(
@@ -813,10 +813,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmsac.nxv16f16.f16(
 define <vscale x 16 x half>  @intrinsic_vfmsac_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmsac.nxv16f16.f16(
@@ -838,10 +838,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmsac.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfmsac_mask_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmsac.mask.nxv16f16.f16(
@@ -863,10 +863,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmsac.nxv1f32.f32(
 define <vscale x 1 x float>  @intrinsic_vfmsac_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmsac.nxv1f32.f32(
@@ -888,10 +888,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmsac.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfmsac_mask_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmsac.mask.nxv1f32.f32(
@@ -913,10 +913,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmsac.nxv2f32.f32(
 define <vscale x 2 x float>  @intrinsic_vfmsac_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmsac.nxv2f32.f32(
@@ -938,10 +938,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmsac.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfmsac_mask_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmsac.mask.nxv2f32.f32(
@@ -963,10 +963,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmsac.nxv4f32.f32(
 define <vscale x 4 x float>  @intrinsic_vfmsac_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmsac.nxv4f32.f32(
@@ -988,10 +988,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmsac.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfmsac_mask_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmsac.mask.nxv4f32.f32(
@@ -1013,10 +1013,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmsac.nxv8f32.f32(
 define <vscale x 8 x float>  @intrinsic_vfmsac_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmsac.nxv8f32.f32(
@@ -1038,10 +1038,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmsac.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfmsac_mask_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmsac.mask.nxv8f32.f32(
@@ -1063,10 +1063,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmsac.nxv1f64.f64(
 define <vscale x 1 x double>  @intrinsic_vfmsac_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmsac.nxv1f64.f64(
@@ -1088,10 +1088,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmsac.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfmsac_mask_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmsac.mask.nxv1f64.f64(
@@ -1113,10 +1113,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmsac.nxv2f64.f64(
 define <vscale x 2 x double>  @intrinsic_vfmsac_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmsac.nxv2f64.f64(
@@ -1138,10 +1138,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmsac.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfmsac_mask_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmsac.mask.nxv2f64.f64(
@@ -1163,10 +1163,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmsac.nxv4f64.f64(
 define <vscale x 4 x double>  @intrinsic_vfmsac_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmsac.nxv4f64.f64(
@@ -1188,10 +1188,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmsac.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfmsac_mask_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsac.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmsac.mask.nxv4f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmsub.ll b/llvm/test/CodeGen/RISCV/rvv/vfmsub.ll
index 4cda25e18911..626b40e132c7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmsub.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmsub.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfmsub_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmsub.nxv1f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmsub.mask.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfmsub_mask_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmsub.mask.nxv1f16.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmsub.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfmsub_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmsub.nxv2f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmsub.mask.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfmsub_mask_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmsub.mask.nxv2f16.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmsub.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfmsub_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmsub.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmsub.mask.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfmsub_mask_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmsub.mask.nxv4f16.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmsub.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfmsub_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmsub.nxv8f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmsub.mask.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfmsub_mask_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmsub.mask.nxv8f16.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmsub.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfmsub_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmsub.nxv16f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmsub.mask.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfmsub_mask_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmsub.mask.nxv16f16.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmsub.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfmsub_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmsub.nxv1f32.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmsub.mask.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfmsub_mask_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmsub.mask.nxv1f32.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmsub.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfmsub_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmsub.nxv2f32.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmsub.mask.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfmsub_mask_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmsub.mask.nxv2f32.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmsub.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfmsub_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmsub.nxv4f32.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmsub.mask.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfmsub_mask_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmsub.mask.nxv4f32.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmsub.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfmsub_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmsub.nxv8f32.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmsub.mask.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfmsub_mask_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmsub.mask.nxv8f32.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmsub.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfmsub_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmsub.nxv1f64.nxv1f64(
@@ -488,10 +488,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmsub.mask.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfmsub_mask_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmsub.mask.nxv1f64.nxv1f64(
@@ -513,10 +513,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmsub.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfmsub_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmsub.nxv2f64.nxv2f64(
@@ -538,10 +538,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmsub.mask.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfmsub_mask_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmsub.mask.nxv2f64.nxv2f64(
@@ -563,10 +563,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmsub.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfmsub_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmsub.nxv4f64.nxv4f64(
@@ -588,10 +588,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmsub.mask.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfmsub_mask_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmsub.mask.nxv4f64.nxv4f64(
@@ -613,10 +613,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmsub.nxv1f16.f16(
 define <vscale x 1 x half>  @intrinsic_vfmsub_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmsub.nxv1f16.f16(
@@ -638,10 +638,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmsub.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfmsub_mask_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmsub.mask.nxv1f16.f16(
@@ -663,10 +663,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmsub.nxv2f16.f16(
 define <vscale x 2 x half>  @intrinsic_vfmsub_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmsub.nxv2f16.f16(
@@ -688,10 +688,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmsub.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfmsub_mask_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmsub.mask.nxv2f16.f16(
@@ -713,10 +713,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmsub.nxv4f16.f16(
 define <vscale x 4 x half>  @intrinsic_vfmsub_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmsub.nxv4f16.f16(
@@ -738,10 +738,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmsub.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfmsub_mask_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmsub.mask.nxv4f16.f16(
@@ -763,10 +763,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmsub.nxv8f16.f16(
 define <vscale x 8 x half>  @intrinsic_vfmsub_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmsub.nxv8f16.f16(
@@ -788,10 +788,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmsub.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfmsub_mask_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmsub.mask.nxv8f16.f16(
@@ -813,10 +813,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmsub.nxv16f16.f16(
 define <vscale x 16 x half>  @intrinsic_vfmsub_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmsub.nxv16f16.f16(
@@ -838,10 +838,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmsub.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfmsub_mask_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmsub.mask.nxv16f16.f16(
@@ -863,10 +863,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmsub.nxv1f32.f32(
 define <vscale x 1 x float>  @intrinsic_vfmsub_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmsub.nxv1f32.f32(
@@ -888,10 +888,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmsub.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfmsub_mask_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmsub.mask.nxv1f32.f32(
@@ -913,10 +913,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmsub.nxv2f32.f32(
 define <vscale x 2 x float>  @intrinsic_vfmsub_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmsub.nxv2f32.f32(
@@ -938,10 +938,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmsub.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfmsub_mask_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmsub.mask.nxv2f32.f32(
@@ -963,10 +963,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmsub.nxv4f32.f32(
 define <vscale x 4 x float>  @intrinsic_vfmsub_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmsub.nxv4f32.f32(
@@ -988,10 +988,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmsub.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfmsub_mask_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmsub.mask.nxv4f32.f32(
@@ -1013,10 +1013,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmsub.nxv8f32.f32(
 define <vscale x 8 x float>  @intrinsic_vfmsub_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmsub.nxv8f32.f32(
@@ -1038,10 +1038,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmsub.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfmsub_mask_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmsub.mask.nxv8f32.f32(
@@ -1063,10 +1063,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmsub.nxv1f64.f64(
 define <vscale x 1 x double>  @intrinsic_vfmsub_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmsub.nxv1f64.f64(
@@ -1088,10 +1088,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmsub.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfmsub_mask_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmsub.mask.nxv1f64.f64(
@@ -1113,10 +1113,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmsub.nxv2f64.f64(
 define <vscale x 2 x double>  @intrinsic_vfmsub_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmsub.nxv2f64.f64(
@@ -1138,10 +1138,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmsub.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfmsub_mask_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmsub.mask.nxv2f64.f64(
@@ -1163,10 +1163,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmsub.nxv4f64.f64(
 define <vscale x 4 x double>  @intrinsic_vfmsub_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmsub.nxv4f64.f64(
@@ -1188,10 +1188,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmsub.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfmsub_mask_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmsub.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmsub.mask.nxv4f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmul.ll b/llvm/test/CodeGen/RISCV/rvv/vfmul.ll
index ee1d197e091f..b73d03fe36c7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmul.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmul.nxv1f16.nxv1f16(
 define <vscale x 1 x half> @intrinsic_vfmul_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmul.nxv1f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmul.mask.nxv1f16.nxv1f16(
 define <vscale x 1 x half> @intrinsic_vfmul_mask_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmul.mask.nxv1f16.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmul.nxv2f16.nxv2f16(
 define <vscale x 2 x half> @intrinsic_vfmul_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmul.nxv2f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmul.mask.nxv2f16.nxv2f16(
 define <vscale x 2 x half> @intrinsic_vfmul_mask_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmul.mask.nxv2f16.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmul.nxv4f16.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfmul_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmul.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmul.mask.nxv4f16.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfmul_mask_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmul.mask.nxv4f16.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmul.nxv8f16.nxv8f16(
 define <vscale x 8 x half> @intrinsic_vfmul_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmul.nxv8f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmul.mask.nxv8f16.nxv8f16(
 define <vscale x 8 x half> @intrinsic_vfmul_mask_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmul.mask.nxv8f16.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmul.nxv16f16.nxv16f16(
 define <vscale x 16 x half> @intrinsic_vfmul_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmul.nxv16f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmul.mask.nxv16f16.nxv16f16(
 define <vscale x 16 x half> @intrinsic_vfmul_mask_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmul.mask.nxv16f16.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfmul.nxv32f16.nxv32f16(
 define <vscale x 32 x half> @intrinsic_vfmul_vv_nxv32f16_nxv32f16_nxv32f16(<vscale x 32 x half> %0, <vscale x 32 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv32f16_nxv32f16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfmul.nxv32f16.nxv32f16(
@@ -289,8 +289,8 @@ define <vscale x 32 x half> @intrinsic_vfmul_mask_vv_nxv32f16_nxv32f16_nxv32f16(
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv32f16_nxv32f16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vfmul.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -314,10 +314,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmul.nxv1f32.nxv1f32(
 define <vscale x 1 x float> @intrinsic_vfmul_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmul.nxv1f32.nxv1f32(
@@ -339,10 +339,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmul.mask.nxv1f32.nxv1f32(
 define <vscale x 1 x float> @intrinsic_vfmul_mask_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmul.mask.nxv1f32.nxv1f32(
@@ -364,10 +364,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmul.nxv2f32.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfmul_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmul.nxv2f32.nxv2f32(
@@ -389,10 +389,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmul.mask.nxv2f32.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfmul_mask_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmul.mask.nxv2f32.nxv2f32(
@@ -414,10 +414,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmul.nxv4f32.nxv4f32(
 define <vscale x 4 x float> @intrinsic_vfmul_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmul.nxv4f32.nxv4f32(
@@ -439,10 +439,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmul.mask.nxv4f32.nxv4f32(
 define <vscale x 4 x float> @intrinsic_vfmul_mask_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmul.mask.nxv4f32.nxv4f32(
@@ -464,10 +464,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmul.nxv8f32.nxv8f32(
 define <vscale x 8 x float> @intrinsic_vfmul_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmul.nxv8f32.nxv8f32(
@@ -489,10 +489,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmul.mask.nxv8f32.nxv8f32(
 define <vscale x 8 x float> @intrinsic_vfmul_mask_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmul.mask.nxv8f32.nxv8f32(
@@ -514,10 +514,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfmul.nxv16f32.nxv16f32(
 define <vscale x 16 x float> @intrinsic_vfmul_vv_nxv16f32_nxv16f32_nxv16f32(<vscale x 16 x float> %0, <vscale x 16 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv16f32_nxv16f32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfmul.nxv16f32.nxv16f32(
@@ -540,8 +540,8 @@ define <vscale x 16 x float> @intrinsic_vfmul_mask_vv_nxv16f32_nxv16f32_nxv16f32
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv16f32_nxv16f32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vfmul.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -565,10 +565,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmul.nxv1f64.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfmul_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmul.nxv1f64.nxv1f64(
@@ -590,10 +590,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmul.mask.nxv1f64.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfmul_mask_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmul.mask.nxv1f64.nxv1f64(
@@ -615,10 +615,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmul.nxv2f64.nxv2f64(
 define <vscale x 2 x double> @intrinsic_vfmul_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmul.nxv2f64.nxv2f64(
@@ -640,10 +640,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmul.mask.nxv2f64.nxv2f64(
 define <vscale x 2 x double> @intrinsic_vfmul_mask_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmul.mask.nxv2f64.nxv2f64(
@@ -665,10 +665,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmul.nxv4f64.nxv4f64(
 define <vscale x 4 x double> @intrinsic_vfmul_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmul.nxv4f64.nxv4f64(
@@ -690,10 +690,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmul.mask.nxv4f64.nxv4f64(
 define <vscale x 4 x double> @intrinsic_vfmul_mask_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmul.mask.nxv4f64.nxv4f64(
@@ -715,10 +715,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfmul.nxv8f64.nxv8f64(
 define <vscale x 8 x double> @intrinsic_vfmul_vv_nxv8f64_nxv8f64_nxv8f64(<vscale x 8 x double> %0, <vscale x 8 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv8f64_nxv8f64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfmul.nxv8f64.nxv8f64(
@@ -741,8 +741,8 @@ define <vscale x 8 x double> @intrinsic_vfmul_mask_vv_nxv8f64_nxv8f64_nxv8f64(<v
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv8f64_nxv8f64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    vfmul.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -766,10 +766,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmul.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfmul_vf_nxv1f16_nxv1f16_f16(<vscale x 1 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv1f16_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmul.nxv1f16.f16(
@@ -791,10 +791,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfmul.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfmul_mask_vf_nxv1f16_nxv1f16_f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv1f16_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfmul.mask.nxv1f16.f16(
@@ -816,10 +816,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmul.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfmul_vf_nxv2f16_nxv2f16_f16(<vscale x 2 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv2f16_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmul.nxv2f16.f16(
@@ -841,10 +841,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfmul.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfmul_mask_vf_nxv2f16_nxv2f16_f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv2f16_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfmul.mask.nxv2f16.f16(
@@ -866,10 +866,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmul.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfmul_vf_nxv4f16_nxv4f16_f16(<vscale x 4 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv4f16_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmul.nxv4f16.f16(
@@ -891,10 +891,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfmul.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfmul_mask_vf_nxv4f16_nxv4f16_f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv4f16_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfmul.mask.nxv4f16.f16(
@@ -916,10 +916,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmul.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfmul_vf_nxv8f16_nxv8f16_f16(<vscale x 8 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv8f16_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmul.nxv8f16.f16(
@@ -941,10 +941,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfmul.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfmul_mask_vf_nxv8f16_nxv8f16_f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv8f16_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfmul.mask.nxv8f16.f16(
@@ -966,10 +966,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmul.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfmul_vf_nxv16f16_nxv16f16_f16(<vscale x 16 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv16f16_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmul.nxv16f16.f16(
@@ -991,10 +991,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfmul.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfmul_mask_vf_nxv16f16_nxv16f16_f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv16f16_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfmul.mask.nxv16f16.f16(
@@ -1016,10 +1016,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfmul.nxv32f16.f16(
 define <vscale x 32 x half> @intrinsic_vfmul_vf_nxv32f16_nxv32f16_f16(<vscale x 32 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv32f16_nxv32f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfmul.nxv32f16.f16(
@@ -1041,10 +1041,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfmul.mask.nxv32f16.f16(
 define <vscale x 32 x half> @intrinsic_vfmul_mask_vf_nxv32f16_nxv32f16_f16(<vscale x 32 x half> %0, <vscale x 32 x half> %1, half %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv32f16_nxv32f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfmul.mask.nxv32f16.f16(
@@ -1066,10 +1066,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmul.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfmul_vf_nxv1f32_nxv1f32_f32(<vscale x 1 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv1f32_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmul.nxv1f32.f32(
@@ -1091,10 +1091,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfmul.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfmul_mask_vf_nxv1f32_nxv1f32_f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv1f32_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfmul.mask.nxv1f32.f32(
@@ -1116,10 +1116,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmul.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfmul_vf_nxv2f32_nxv2f32_f32(<vscale x 2 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv2f32_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmul.nxv2f32.f32(
@@ -1141,10 +1141,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfmul.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfmul_mask_vf_nxv2f32_nxv2f32_f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv2f32_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfmul.mask.nxv2f32.f32(
@@ -1166,10 +1166,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmul.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfmul_vf_nxv4f32_nxv4f32_f32(<vscale x 4 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv4f32_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmul.nxv4f32.f32(
@@ -1191,10 +1191,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfmul.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfmul_mask_vf_nxv4f32_nxv4f32_f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv4f32_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfmul.mask.nxv4f32.f32(
@@ -1216,10 +1216,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmul.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfmul_vf_nxv8f32_nxv8f32_f32(<vscale x 8 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv8f32_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmul.nxv8f32.f32(
@@ -1241,10 +1241,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfmul.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfmul_mask_vf_nxv8f32_nxv8f32_f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv8f32_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfmul.mask.nxv8f32.f32(
@@ -1266,10 +1266,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfmul.nxv16f32.f32(
 define <vscale x 16 x float> @intrinsic_vfmul_vf_nxv16f32_nxv16f32_f32(<vscale x 16 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv16f32_nxv16f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfmul.nxv16f32.f32(
@@ -1291,10 +1291,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfmul.mask.nxv16f32.f32(
 define <vscale x 16 x float> @intrinsic_vfmul_mask_vf_nxv16f32_nxv16f32_f32(<vscale x 16 x float> %0, <vscale x 16 x float> %1, float %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv16f32_nxv16f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfmul.mask.nxv16f32.f32(
@@ -1316,10 +1316,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmul.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfmul_vf_nxv1f64_nxv1f64_f64(<vscale x 1 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv1f64_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmul.nxv1f64.f64(
@@ -1341,10 +1341,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfmul.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfmul_mask_vf_nxv1f64_nxv1f64_f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, double %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv1f64_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfmul.mask.nxv1f64.f64(
@@ -1366,10 +1366,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmul.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfmul_vf_nxv2f64_nxv2f64_f64(<vscale x 2 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv2f64_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmul.nxv2f64.f64(
@@ -1391,10 +1391,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfmul.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfmul_mask_vf_nxv2f64_nxv2f64_f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, double %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv2f64_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfmul.mask.nxv2f64.f64(
@@ -1416,10 +1416,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmul.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfmul_vf_nxv4f64_nxv4f64_f64(<vscale x 4 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv4f64_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmul.nxv4f64.f64(
@@ -1441,10 +1441,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfmul.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfmul_mask_vf_nxv4f64_nxv4f64_f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, double %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv4f64_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfmul.mask.nxv4f64.f64(
@@ -1466,10 +1466,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfmul.nxv8f64.f64(
 define <vscale x 8 x double> @intrinsic_vfmul_vf_nxv8f64_nxv8f64_f64(<vscale x 8 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv8f64_nxv8f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfmul.nxv8f64.f64(
@@ -1491,10 +1491,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfmul.mask.nxv8f64.f64(
 define <vscale x 8 x double> @intrinsic_vfmul_mask_vf_nxv8f64_nxv8f64_f64(<vscale x 8 x double> %0, <vscale x 8 x double> %1, double %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv8f64_nxv8f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfmul.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfmul.mask.nxv8f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-f.ll
index 2de7d78df881..183ffa8a668a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-f.ll
@@ -15,10 +15,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfncvt.f.f.w.nxv1f16.nxv1f32(
 define <vscale x 1 x half> @intrinsic_vfncvt_f.f.w_nxv1f16_nxv1f32(<vscale x 1 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.f.w_nxv1f16_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -39,10 +39,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfncvt.f.f.w.mask.nxv1f16.nxv1f32(
 define <vscale x 1 x half> @intrinsic_vfncvt_mask_f.f.w_nxv1f16_nxv1f32(<vscale x 1 x half> %0, <vscale x 1 x float> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f.w_nxv1f16_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfncvt.f.f.w.mask.nxv1f16.nxv1f32(
@@ -62,10 +62,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfncvt.f.f.w.nxv2f16.nxv2f32(
 define <vscale x 2 x half> @intrinsic_vfncvt_f.f.w_nxv2f16_nxv2f32(<vscale x 2 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.f.w_nxv2f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -86,10 +86,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfncvt.f.f.w.mask.nxv2f16.nxv2f32(
 define <vscale x 2 x half> @intrinsic_vfncvt_mask_f.f.w_nxv2f16_nxv2f32(<vscale x 2 x half> %0, <vscale x 2 x float> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f.w_nxv2f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfncvt.f.f.w.mask.nxv2f16.nxv2f32(
@@ -109,10 +109,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfncvt.f.f.w.nxv4f16.nxv4f32(
 define <vscale x 4 x half> @intrinsic_vfncvt_f.f.w_nxv4f16_nxv4f32(<vscale x 4 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.f.w_nxv4f16_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -133,10 +133,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfncvt.f.f.w.mask.nxv4f16.nxv4f32(
 define <vscale x 4 x half> @intrinsic_vfncvt_mask_f.f.w_nxv4f16_nxv4f32(<vscale x 4 x half> %0, <vscale x 4 x float> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f.w_nxv4f16_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfncvt.f.f.w.mask.nxv4f16.nxv4f32(
@@ -156,10 +156,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfncvt.f.f.w.nxv8f16.nxv8f32(
 define <vscale x 8 x half> @intrinsic_vfncvt_f.f.w_nxv8f16_nxv8f32(<vscale x 8 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.f.w_nxv8f16_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -180,10 +180,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfncvt.f.f.w.mask.nxv8f16.nxv8f32(
 define <vscale x 8 x half> @intrinsic_vfncvt_mask_f.f.w_nxv8f16_nxv8f32(<vscale x 8 x half> %0, <vscale x 8 x float> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f.w_nxv8f16_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfncvt.f.f.w.mask.nxv8f16.nxv8f32(
@@ -203,10 +203,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfncvt.f.f.w.nxv16f16.nxv16f32(
 define <vscale x 16 x half> @intrinsic_vfncvt_f.f.w_nxv16f16_nxv16f32(<vscale x 16 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.f.w_nxv16f16_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -227,10 +227,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfncvt.f.f.w.mask.nxv16f16.nxv16f32(
 define <vscale x 16 x half> @intrinsic_vfncvt_mask_f.f.w_nxv16f16_nxv16f32(<vscale x 16 x half> %0, <vscale x 16 x float> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f.w_nxv16f16_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfncvt.f.f.w.mask.nxv16f16.nxv16f32(
@@ -250,10 +250,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfncvt.f.f.w.nxv1f32.nxv1f64(
 define <vscale x 1 x float> @intrinsic_vfncvt_f.f.w_nxv1f32_nxv1f64(<vscale x 1 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.f.w_nxv1f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -274,10 +274,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfncvt.f.f.w.mask.nxv1f32.nxv1f64(
 define <vscale x 1 x float> @intrinsic_vfncvt_mask_f.f.w_nxv1f32_nxv1f64(<vscale x 1 x float> %0, <vscale x 1 x double> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f.w_nxv1f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfncvt.f.f.w.mask.nxv1f32.nxv1f64(
@@ -297,10 +297,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfncvt.f.f.w.nxv2f32.nxv2f64(
 define <vscale x 2 x float> @intrinsic_vfncvt_f.f.w_nxv2f32_nxv2f64(<vscale x 2 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.f.w_nxv2f32_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -321,10 +321,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfncvt.f.f.w.mask.nxv2f32.nxv2f64(
 define <vscale x 2 x float> @intrinsic_vfncvt_mask_f.f.w_nxv2f32_nxv2f64(<vscale x 2 x float> %0, <vscale x 2 x double> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f.w_nxv2f32_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfncvt.f.f.w.mask.nxv2f32.nxv2f64(
@@ -344,10 +344,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfncvt.f.f.w.nxv4f32.nxv4f64(
 define <vscale x 4 x float> @intrinsic_vfncvt_f.f.w_nxv4f32_nxv4f64(<vscale x 4 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.f.w_nxv4f32_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -368,10 +368,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfncvt.f.f.w.mask.nxv4f32.nxv4f64(
 define <vscale x 4 x float> @intrinsic_vfncvt_mask_f.f.w_nxv4f32_nxv4f64(<vscale x 4 x float> %0, <vscale x 4 x double> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f.w_nxv4f32_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfncvt.f.f.w.mask.nxv4f32.nxv4f64(
@@ -391,10 +391,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfncvt.f.f.w.nxv8f32.nxv8f64(
 define <vscale x 8 x float> @intrinsic_vfncvt_f.f.w_nxv8f32_nxv8f64(<vscale x 8 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.f.w_nxv8f32_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -415,10 +415,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfncvt.f.f.w.mask.nxv8f32.nxv8f64(
 define <vscale x 8 x float> @intrinsic_vfncvt_mask_f.f.w_nxv8f32_nxv8f64(<vscale x 8 x float> %0, <vscale x 8 x double> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f.w_nxv8f32_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.f.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfncvt.f.f.w.mask.nxv8f32.nxv8f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-x.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-x.ll
index 7f2714b2fbfc..aef119faf5f7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-x.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-x.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfncvt.f.x.w.nxv1f16.nxv1i32(
 define <vscale x 1 x half> @intrinsic_vfncvt_f.x.w_nxv1f16_nxv1i32(<vscale x 1 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.x.w_nxv1f16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -36,10 +36,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfncvt.f.x.w.mask.nxv1f16.nxv1i32(
 define <vscale x 1 x half> @intrinsic_vfncvt_mask_f.x.w_nxv1f16_nxv1i32(<vscale x 1 x half> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x.w_nxv1f16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfncvt.f.x.w.mask.nxv1f16.nxv1i32(
@@ -59,10 +59,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfncvt.f.x.w.nxv2f16.nxv2i32(
 define <vscale x 2 x half> @intrinsic_vfncvt_f.x.w_nxv2f16_nxv2i32(<vscale x 2 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.x.w_nxv2f16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -83,10 +83,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfncvt.f.x.w.mask.nxv2f16.nxv2i32(
 define <vscale x 2 x half> @intrinsic_vfncvt_mask_f.x.w_nxv2f16_nxv2i32(<vscale x 2 x half> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x.w_nxv2f16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfncvt.f.x.w.mask.nxv2f16.nxv2i32(
@@ -106,10 +106,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfncvt.f.x.w.nxv4f16.nxv4i32(
 define <vscale x 4 x half> @intrinsic_vfncvt_f.x.w_nxv4f16_nxv4i32(<vscale x 4 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.x.w_nxv4f16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -130,10 +130,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfncvt.f.x.w.mask.nxv4f16.nxv4i32(
 define <vscale x 4 x half> @intrinsic_vfncvt_mask_f.x.w_nxv4f16_nxv4i32(<vscale x 4 x half> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x.w_nxv4f16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfncvt.f.x.w.mask.nxv4f16.nxv4i32(
@@ -153,10 +153,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfncvt.f.x.w.nxv8f16.nxv8i32(
 define <vscale x 8 x half> @intrinsic_vfncvt_f.x.w_nxv8f16_nxv8i32(<vscale x 8 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.x.w_nxv8f16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -177,10 +177,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfncvt.f.x.w.mask.nxv8f16.nxv8i32(
 define <vscale x 8 x half> @intrinsic_vfncvt_mask_f.x.w_nxv8f16_nxv8i32(<vscale x 8 x half> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x.w_nxv8f16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfncvt.f.x.w.mask.nxv8f16.nxv8i32(
@@ -200,10 +200,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfncvt.f.x.w.nxv16f16.nxv16i32(
 define <vscale x 16 x half> @intrinsic_vfncvt_f.x.w_nxv16f16_nxv16i32(<vscale x 16 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.x.w_nxv16f16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -224,10 +224,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfncvt.f.x.w.mask.nxv16f16.nxv16i32(
 define <vscale x 16 x half> @intrinsic_vfncvt_mask_f.x.w_nxv16f16_nxv16i32(<vscale x 16 x half> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x.w_nxv16f16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfncvt.f.x.w.mask.nxv16f16.nxv16i32(
@@ -247,10 +247,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfncvt.f.x.w.nxv1f32.nxv1i64(
 define <vscale x 1 x float> @intrinsic_vfncvt_f.x.w_nxv1f32_nxv1i64(<vscale x 1 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.x.w_nxv1f32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -271,10 +271,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfncvt.f.x.w.mask.nxv1f32.nxv1i64(
 define <vscale x 1 x float> @intrinsic_vfncvt_mask_f.x.w_nxv1f32_nxv1i64(<vscale x 1 x float> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x.w_nxv1f32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfncvt.f.x.w.mask.nxv1f32.nxv1i64(
@@ -294,10 +294,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfncvt.f.x.w.nxv2f32.nxv2i64(
 define <vscale x 2 x float> @intrinsic_vfncvt_f.x.w_nxv2f32_nxv2i64(<vscale x 2 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.x.w_nxv2f32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -318,10 +318,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfncvt.f.x.w.mask.nxv2f32.nxv2i64(
 define <vscale x 2 x float> @intrinsic_vfncvt_mask_f.x.w_nxv2f32_nxv2i64(<vscale x 2 x float> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x.w_nxv2f32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfncvt.f.x.w.mask.nxv2f32.nxv2i64(
@@ -341,10 +341,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfncvt.f.x.w.nxv4f32.nxv4i64(
 define <vscale x 4 x float> @intrinsic_vfncvt_f.x.w_nxv4f32_nxv4i64(<vscale x 4 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.x.w_nxv4f32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -365,10 +365,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfncvt.f.x.w.mask.nxv4f32.nxv4i64(
 define <vscale x 4 x float> @intrinsic_vfncvt_mask_f.x.w_nxv4f32_nxv4i64(<vscale x 4 x float> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x.w_nxv4f32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfncvt.f.x.w.mask.nxv4f32.nxv4i64(
@@ -388,10 +388,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfncvt.f.x.w.nxv8f32.nxv8i64(
 define <vscale x 8 x float> @intrinsic_vfncvt_f.x.w_nxv8f32_nxv8i64(<vscale x 8 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.x.w_nxv8f32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -412,10 +412,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfncvt.f.x.w.mask.nxv8f32.nxv8i64(
 define <vscale x 8 x float> @intrinsic_vfncvt_mask_f.x.w_nxv8f32_nxv8i64(<vscale x 8 x float> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x.w_nxv8f32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.x.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfncvt.f.x.w.mask.nxv8f32.nxv8i64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-xu.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-xu.ll
index 1aeee4317cb3..bc287e4bdef1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-xu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-f-xu.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfncvt.f.xu.w.nxv1f16.nxv1i32(
 define <vscale x 1 x half> @intrinsic_vfncvt_f.xu.w_nxv1f16_nxv1i32(<vscale x 1 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu.w_nxv1f16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -36,10 +36,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfncvt.f.xu.w.mask.nxv1f16.nxv1i32(
 define <vscale x 1 x half> @intrinsic_vfncvt_mask_f.xu.w_nxv1f16_nxv1i32(<vscale x 1 x half> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu.w_nxv1f16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfncvt.f.xu.w.mask.nxv1f16.nxv1i32(
@@ -59,10 +59,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfncvt.f.xu.w.nxv2f16.nxv2i32(
 define <vscale x 2 x half> @intrinsic_vfncvt_f.xu.w_nxv2f16_nxv2i32(<vscale x 2 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu.w_nxv2f16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -83,10 +83,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfncvt.f.xu.w.mask.nxv2f16.nxv2i32(
 define <vscale x 2 x half> @intrinsic_vfncvt_mask_f.xu.w_nxv2f16_nxv2i32(<vscale x 2 x half> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu.w_nxv2f16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfncvt.f.xu.w.mask.nxv2f16.nxv2i32(
@@ -106,10 +106,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfncvt.f.xu.w.nxv4f16.nxv4i32(
 define <vscale x 4 x half> @intrinsic_vfncvt_f.xu.w_nxv4f16_nxv4i32(<vscale x 4 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu.w_nxv4f16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -130,10 +130,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfncvt.f.xu.w.mask.nxv4f16.nxv4i32(
 define <vscale x 4 x half> @intrinsic_vfncvt_mask_f.xu.w_nxv4f16_nxv4i32(<vscale x 4 x half> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu.w_nxv4f16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfncvt.f.xu.w.mask.nxv4f16.nxv4i32(
@@ -153,10 +153,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfncvt.f.xu.w.nxv8f16.nxv8i32(
 define <vscale x 8 x half> @intrinsic_vfncvt_f.xu.w_nxv8f16_nxv8i32(<vscale x 8 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu.w_nxv8f16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -177,10 +177,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfncvt.f.xu.w.mask.nxv8f16.nxv8i32(
 define <vscale x 8 x half> @intrinsic_vfncvt_mask_f.xu.w_nxv8f16_nxv8i32(<vscale x 8 x half> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu.w_nxv8f16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfncvt.f.xu.w.mask.nxv8f16.nxv8i32(
@@ -200,10 +200,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfncvt.f.xu.w.nxv16f16.nxv16i32(
 define <vscale x 16 x half> @intrinsic_vfncvt_f.xu.w_nxv16f16_nxv16i32(<vscale x 16 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu.w_nxv16f16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -224,10 +224,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfncvt.f.xu.w.mask.nxv16f16.nxv16i32(
 define <vscale x 16 x half> @intrinsic_vfncvt_mask_f.xu.w_nxv16f16_nxv16i32(<vscale x 16 x half> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu.w_nxv16f16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfncvt.f.xu.w.mask.nxv16f16.nxv16i32(
@@ -247,10 +247,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfncvt.f.xu.w.nxv1f32.nxv1i64(
 define <vscale x 1 x float> @intrinsic_vfncvt_f.xu.w_nxv1f32_nxv1i64(<vscale x 1 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu.w_nxv1f32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -271,10 +271,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfncvt.f.xu.w.mask.nxv1f32.nxv1i64(
 define <vscale x 1 x float> @intrinsic_vfncvt_mask_f.xu.w_nxv1f32_nxv1i64(<vscale x 1 x float> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu.w_nxv1f32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfncvt.f.xu.w.mask.nxv1f32.nxv1i64(
@@ -294,10 +294,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfncvt.f.xu.w.nxv2f32.nxv2i64(
 define <vscale x 2 x float> @intrinsic_vfncvt_f.xu.w_nxv2f32_nxv2i64(<vscale x 2 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu.w_nxv2f32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -318,10 +318,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfncvt.f.xu.w.mask.nxv2f32.nxv2i64(
 define <vscale x 2 x float> @intrinsic_vfncvt_mask_f.xu.w_nxv2f32_nxv2i64(<vscale x 2 x float> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu.w_nxv2f32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfncvt.f.xu.w.mask.nxv2f32.nxv2i64(
@@ -341,10 +341,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfncvt.f.xu.w.nxv4f32.nxv4i64(
 define <vscale x 4 x float> @intrinsic_vfncvt_f.xu.w_nxv4f32_nxv4i64(<vscale x 4 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu.w_nxv4f32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -365,10 +365,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfncvt.f.xu.w.mask.nxv4f32.nxv4i64(
 define <vscale x 4 x float> @intrinsic_vfncvt_mask_f.xu.w_nxv4f32_nxv4i64(<vscale x 4 x float> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu.w_nxv4f32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfncvt.f.xu.w.mask.nxv4f32.nxv4i64(
@@ -388,10 +388,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfncvt.f.xu.w.nxv8f32.nxv8i64(
 define <vscale x 8 x float> @intrinsic_vfncvt_f.xu.w_nxv8f32_nxv8i64(<vscale x 8 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu.w_nxv8f32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -412,10 +412,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfncvt.f.xu.w.mask.nxv8f32.nxv8i64(
 define <vscale x 8 x float> @intrinsic_vfncvt_mask_f.xu.w_nxv8f32_nxv8i64(<vscale x 8 x float> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu.w_nxv8f32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.f.xu.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfncvt.f.xu.w.mask.nxv8f32.nxv8i64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-x-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-x-f.ll
index 8309e3fb857f..e4b39c655a10 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-x-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-x-f.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x i8> @llvm.riscv.vfncvt.x.f.w.nxv1i8.nxv1f16(
 define <vscale x 1 x i8> @intrinsic_vfncvt_x.f.w_nxv1i8_nxv1f16(<vscale x 1 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv1i8_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -36,10 +36,10 @@ declare <vscale x 1 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv1i8.nxv1f16(
 define <vscale x 1 x i8> @intrinsic_vfncvt_mask_x.f.w_nxv1i8_nxv1f16(<vscale x 1 x i8> %0, <vscale x 1 x half> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv1i8_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv1i8.nxv1f16(
@@ -59,10 +59,10 @@ declare <vscale x 2 x i8> @llvm.riscv.vfncvt.x.f.w.nxv2i8.nxv2f16(
 define <vscale x 2 x i8> @intrinsic_vfncvt_x.f.w_nxv2i8_nxv2f16(<vscale x 2 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv2i8_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -83,10 +83,10 @@ declare <vscale x 2 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv2i8.nxv2f16(
 define <vscale x 2 x i8> @intrinsic_vfncvt_mask_x.f.w_nxv2i8_nxv2f16(<vscale x 2 x i8> %0, <vscale x 2 x half> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv2i8_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv2i8.nxv2f16(
@@ -106,10 +106,10 @@ declare <vscale x 4 x i8> @llvm.riscv.vfncvt.x.f.w.nxv4i8.nxv4f16(
 define <vscale x 4 x i8> @intrinsic_vfncvt_x.f.w_nxv4i8_nxv4f16(<vscale x 4 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv4i8_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -130,10 +130,10 @@ declare <vscale x 4 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv4i8.nxv4f16(
 define <vscale x 4 x i8> @intrinsic_vfncvt_mask_x.f.w_nxv4i8_nxv4f16(<vscale x 4 x i8> %0, <vscale x 4 x half> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv4i8_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv4i8.nxv4f16(
@@ -153,10 +153,10 @@ declare <vscale x 8 x i8> @llvm.riscv.vfncvt.x.f.w.nxv8i8.nxv8f16(
 define <vscale x 8 x i8> @intrinsic_vfncvt_x.f.w_nxv8i8_nxv8f16(<vscale x 8 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv8i8_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -177,10 +177,10 @@ declare <vscale x 8 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv8i8.nxv8f16(
 define <vscale x 8 x i8> @intrinsic_vfncvt_mask_x.f.w_nxv8i8_nxv8f16(<vscale x 8 x i8> %0, <vscale x 8 x half> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv8i8_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv8i8.nxv8f16(
@@ -200,10 +200,10 @@ declare <vscale x 16 x i8> @llvm.riscv.vfncvt.x.f.w.nxv16i8.nxv16f16(
 define <vscale x 16 x i8> @intrinsic_vfncvt_x.f.w_nxv16i8_nxv16f16(<vscale x 16 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv16i8_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -224,10 +224,10 @@ declare <vscale x 16 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv16i8.nxv16f16(
 define <vscale x 16 x i8> @intrinsic_vfncvt_mask_x.f.w_nxv16i8_nxv16f16(<vscale x 16 x i8> %0, <vscale x 16 x half> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv16i8_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv16i8.nxv16f16(
@@ -247,10 +247,10 @@ declare <vscale x 32 x i8> @llvm.riscv.vfncvt.x.f.w.nxv32i8.nxv32f16(
 define <vscale x 32 x i8> @intrinsic_vfncvt_x.f.w_nxv32i8_nxv32f16(<vscale x 32 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv32i8_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -271,10 +271,10 @@ declare <vscale x 32 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv32i8.nxv32f16(
 define <vscale x 32 x i8> @intrinsic_vfncvt_mask_x.f.w_nxv32i8_nxv32f16(<vscale x 32 x i8> %0, <vscale x 32 x half> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv32i8_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x i8> @llvm.riscv.vfncvt.x.f.w.mask.nxv32i8.nxv32f16(
@@ -294,10 +294,10 @@ declare <vscale x 1 x i16> @llvm.riscv.vfncvt.x.f.w.nxv1i16.nxv1f32(
 define <vscale x 1 x i16> @intrinsic_vfncvt_x.f.w_nxv1i16_nxv1f32(<vscale x 1 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv1i16_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -318,10 +318,10 @@ declare <vscale x 1 x i16> @llvm.riscv.vfncvt.x.f.w.mask.nxv1i16.nxv1f32(
 define <vscale x 1 x i16> @intrinsic_vfncvt_mask_x.f.w_nxv1i16_nxv1f32(<vscale x 1 x i16> %0, <vscale x 1 x float> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv1i16_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vfncvt.x.f.w.mask.nxv1i16.nxv1f32(
@@ -341,10 +341,10 @@ declare <vscale x 2 x i16> @llvm.riscv.vfncvt.x.f.w.nxv2i16.nxv2f32(
 define <vscale x 2 x i16> @intrinsic_vfncvt_x.f.w_nxv2i16_nxv2f32(<vscale x 2 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv2i16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -365,10 +365,10 @@ declare <vscale x 2 x i16> @llvm.riscv.vfncvt.x.f.w.mask.nxv2i16.nxv2f32(
 define <vscale x 2 x i16> @intrinsic_vfncvt_mask_x.f.w_nxv2i16_nxv2f32(<vscale x 2 x i16> %0, <vscale x 2 x float> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv2i16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vfncvt.x.f.w.mask.nxv2i16.nxv2f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x i16> @llvm.riscv.vfncvt.x.f.w.nxv4i16.nxv4f32(
 define <vscale x 4 x i16> @intrinsic_vfncvt_x.f.w_nxv4i16_nxv4f32(<vscale x 4 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv4i16_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -412,10 +412,10 @@ declare <vscale x 4 x i16> @llvm.riscv.vfncvt.x.f.w.mask.nxv4i16.nxv4f32(
 define <vscale x 4 x i16> @intrinsic_vfncvt_mask_x.f.w_nxv4i16_nxv4f32(<vscale x 4 x i16> %0, <vscale x 4 x float> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv4i16_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vfncvt.x.f.w.mask.nxv4i16.nxv4f32(
@@ -435,10 +435,10 @@ declare <vscale x 8 x i16> @llvm.riscv.vfncvt.x.f.w.nxv8i16.nxv8f32(
 define <vscale x 8 x i16> @intrinsic_vfncvt_x.f.w_nxv8i16_nxv8f32(<vscale x 8 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv8i16_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -459,10 +459,10 @@ declare <vscale x 8 x i16> @llvm.riscv.vfncvt.x.f.w.mask.nxv8i16.nxv8f32(
 define <vscale x 8 x i16> @intrinsic_vfncvt_mask_x.f.w_nxv8i16_nxv8f32(<vscale x 8 x i16> %0, <vscale x 8 x float> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv8i16_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vfncvt.x.f.w.mask.nxv8i16.nxv8f32(
@@ -482,10 +482,10 @@ declare <vscale x 16 x i16> @llvm.riscv.vfncvt.x.f.w.nxv16i16.nxv16f32(
 define <vscale x 16 x i16> @intrinsic_vfncvt_x.f.w_nxv16i16_nxv16f32(<vscale x 16 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv16i16_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -506,10 +506,10 @@ declare <vscale x 16 x i16> @llvm.riscv.vfncvt.x.f.w.mask.nxv16i16.nxv16f32(
 define <vscale x 16 x i16> @intrinsic_vfncvt_mask_x.f.w_nxv16i16_nxv16f32(<vscale x 16 x i16> %0, <vscale x 16 x float> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv16i16_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vfncvt.x.f.w.mask.nxv16i16.nxv16f32(
@@ -529,10 +529,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfncvt.x.f.w.nxv1i32.nxv1f64(
 define <vscale x 1 x i32> @intrinsic_vfncvt_x.f.w_nxv1i32_nxv1f64(<vscale x 1 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv1i32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -553,10 +553,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfncvt.x.f.w.mask.nxv1i32.nxv1f64(
 define <vscale x 1 x i32> @intrinsic_vfncvt_mask_x.f.w_nxv1i32_nxv1f64(<vscale x 1 x i32> %0, <vscale x 1 x double> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv1i32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vfncvt.x.f.w.mask.nxv1i32.nxv1f64(
@@ -576,10 +576,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfncvt.x.f.w.nxv2i32.nxv2f64(
 define <vscale x 2 x i32> @intrinsic_vfncvt_x.f.w_nxv2i32_nxv2f64(<vscale x 2 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv2i32_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -600,10 +600,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfncvt.x.f.w.mask.nxv2i32.nxv2f64(
 define <vscale x 2 x i32> @intrinsic_vfncvt_mask_x.f.w_nxv2i32_nxv2f64(<vscale x 2 x i32> %0, <vscale x 2 x double> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv2i32_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vfncvt.x.f.w.mask.nxv2i32.nxv2f64(
@@ -623,10 +623,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfncvt.x.f.w.nxv4i32.nxv4f64(
 define <vscale x 4 x i32> @intrinsic_vfncvt_x.f.w_nxv4i32_nxv4f64(<vscale x 4 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv4i32_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -647,10 +647,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfncvt.x.f.w.mask.nxv4i32.nxv4f64(
 define <vscale x 4 x i32> @intrinsic_vfncvt_mask_x.f.w_nxv4i32_nxv4f64(<vscale x 4 x i32> %0, <vscale x 4 x double> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv4i32_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vfncvt.x.f.w.mask.nxv4i32.nxv4f64(
@@ -670,10 +670,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfncvt.x.f.w.nxv8i32.nxv8f64(
 define <vscale x 8 x i32> @intrinsic_vfncvt_x.f.w_nxv8i32_nxv8f64(<vscale x 8 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_x.f.w_nxv8i32_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -694,10 +694,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfncvt.x.f.w.mask.nxv8i32.nxv8f64(
 define <vscale x 8 x i32> @intrinsic_vfncvt_mask_x.f.w_nxv8i32_nxv8f64(<vscale x 8 x i32> %0, <vscale x 8 x double> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f.w_nxv8i32_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.x.f.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vfncvt.x.f.w.mask.nxv8i32.nxv8f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfncvt-xu-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfncvt-xu-f.ll
index 3a3abacc8fc3..fd922438d05b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfncvt-xu-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfncvt-xu-f.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x i8> @llvm.riscv.vfncvt.xu.f.w.nxv1i8.nxv1f16(
 define <vscale x 1 x i8> @intrinsic_vfncvt_xu.f.w_nxv1i8_nxv1f16(<vscale x 1 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv1i8_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -36,10 +36,10 @@ declare <vscale x 1 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv1i8.nxv1f16(
 define <vscale x 1 x i8> @intrinsic_vfncvt_mask_xu.f.w_nxv1i8_nxv1f16(<vscale x 1 x i8> %0, <vscale x 1 x half> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv1i8_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv1i8.nxv1f16(
@@ -59,10 +59,10 @@ declare <vscale x 2 x i8> @llvm.riscv.vfncvt.xu.f.w.nxv2i8.nxv2f16(
 define <vscale x 2 x i8> @intrinsic_vfncvt_xu.f.w_nxv2i8_nxv2f16(<vscale x 2 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv2i8_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -83,10 +83,10 @@ declare <vscale x 2 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv2i8.nxv2f16(
 define <vscale x 2 x i8> @intrinsic_vfncvt_mask_xu.f.w_nxv2i8_nxv2f16(<vscale x 2 x i8> %0, <vscale x 2 x half> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv2i8_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv2i8.nxv2f16(
@@ -106,10 +106,10 @@ declare <vscale x 4 x i8> @llvm.riscv.vfncvt.xu.f.w.nxv4i8.nxv4f16(
 define <vscale x 4 x i8> @intrinsic_vfncvt_xu.f.w_nxv4i8_nxv4f16(<vscale x 4 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv4i8_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -130,10 +130,10 @@ declare <vscale x 4 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv4i8.nxv4f16(
 define <vscale x 4 x i8> @intrinsic_vfncvt_mask_xu.f.w_nxv4i8_nxv4f16(<vscale x 4 x i8> %0, <vscale x 4 x half> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv4i8_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv4i8.nxv4f16(
@@ -153,10 +153,10 @@ declare <vscale x 8 x i8> @llvm.riscv.vfncvt.xu.f.w.nxv8i8.nxv8f16(
 define <vscale x 8 x i8> @intrinsic_vfncvt_xu.f.w_nxv8i8_nxv8f16(<vscale x 8 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv8i8_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -177,10 +177,10 @@ declare <vscale x 8 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv8i8.nxv8f16(
 define <vscale x 8 x i8> @intrinsic_vfncvt_mask_xu.f.w_nxv8i8_nxv8f16(<vscale x 8 x i8> %0, <vscale x 8 x half> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv8i8_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv8i8.nxv8f16(
@@ -200,10 +200,10 @@ declare <vscale x 16 x i8> @llvm.riscv.vfncvt.xu.f.w.nxv16i8.nxv16f16(
 define <vscale x 16 x i8> @intrinsic_vfncvt_xu.f.w_nxv16i8_nxv16f16(<vscale x 16 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv16i8_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -224,10 +224,10 @@ declare <vscale x 16 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv16i8.nxv16f16(
 define <vscale x 16 x i8> @intrinsic_vfncvt_mask_xu.f.w_nxv16i8_nxv16f16(<vscale x 16 x i8> %0, <vscale x 16 x half> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv16i8_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv16i8.nxv16f16(
@@ -247,10 +247,10 @@ declare <vscale x 32 x i8> @llvm.riscv.vfncvt.xu.f.w.nxv32i8.nxv32f16(
 define <vscale x 32 x i8> @intrinsic_vfncvt_xu.f.w_nxv32i8_nxv32f16(<vscale x 32 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv32i8_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -271,10 +271,10 @@ declare <vscale x 32 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv32i8.nxv32f16(
 define <vscale x 32 x i8> @intrinsic_vfncvt_mask_xu.f.w_nxv32i8_nxv32f16(<vscale x 32 x i8> %0, <vscale x 32 x half> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv32i8_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x i8> @llvm.riscv.vfncvt.xu.f.w.mask.nxv32i8.nxv32f16(
@@ -294,10 +294,10 @@ declare <vscale x 1 x i16> @llvm.riscv.vfncvt.xu.f.w.nxv1i16.nxv1f32(
 define <vscale x 1 x i16> @intrinsic_vfncvt_xu.f.w_nxv1i16_nxv1f32(<vscale x 1 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv1i16_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -318,10 +318,10 @@ declare <vscale x 1 x i16> @llvm.riscv.vfncvt.xu.f.w.mask.nxv1i16.nxv1f32(
 define <vscale x 1 x i16> @intrinsic_vfncvt_mask_xu.f.w_nxv1i16_nxv1f32(<vscale x 1 x i16> %0, <vscale x 1 x float> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv1i16_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i16> @llvm.riscv.vfncvt.xu.f.w.mask.nxv1i16.nxv1f32(
@@ -341,10 +341,10 @@ declare <vscale x 2 x i16> @llvm.riscv.vfncvt.xu.f.w.nxv2i16.nxv2f32(
 define <vscale x 2 x i16> @intrinsic_vfncvt_xu.f.w_nxv2i16_nxv2f32(<vscale x 2 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv2i16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -365,10 +365,10 @@ declare <vscale x 2 x i16> @llvm.riscv.vfncvt.xu.f.w.mask.nxv2i16.nxv2f32(
 define <vscale x 2 x i16> @intrinsic_vfncvt_mask_xu.f.w_nxv2i16_nxv2f32(<vscale x 2 x i16> %0, <vscale x 2 x float> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv2i16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i16> @llvm.riscv.vfncvt.xu.f.w.mask.nxv2i16.nxv2f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x i16> @llvm.riscv.vfncvt.xu.f.w.nxv4i16.nxv4f32(
 define <vscale x 4 x i16> @intrinsic_vfncvt_xu.f.w_nxv4i16_nxv4f32(<vscale x 4 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv4i16_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -412,10 +412,10 @@ declare <vscale x 4 x i16> @llvm.riscv.vfncvt.xu.f.w.mask.nxv4i16.nxv4f32(
 define <vscale x 4 x i16> @intrinsic_vfncvt_mask_xu.f.w_nxv4i16_nxv4f32(<vscale x 4 x i16> %0, <vscale x 4 x float> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv4i16_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i16> @llvm.riscv.vfncvt.xu.f.w.mask.nxv4i16.nxv4f32(
@@ -435,10 +435,10 @@ declare <vscale x 8 x i16> @llvm.riscv.vfncvt.xu.f.w.nxv8i16.nxv8f32(
 define <vscale x 8 x i16> @intrinsic_vfncvt_xu.f.w_nxv8i16_nxv8f32(<vscale x 8 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv8i16_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -459,10 +459,10 @@ declare <vscale x 8 x i16> @llvm.riscv.vfncvt.xu.f.w.mask.nxv8i16.nxv8f32(
 define <vscale x 8 x i16> @intrinsic_vfncvt_mask_xu.f.w_nxv8i16_nxv8f32(<vscale x 8 x i16> %0, <vscale x 8 x float> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv8i16_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i16> @llvm.riscv.vfncvt.xu.f.w.mask.nxv8i16.nxv8f32(
@@ -482,10 +482,10 @@ declare <vscale x 16 x i16> @llvm.riscv.vfncvt.xu.f.w.nxv16i16.nxv16f32(
 define <vscale x 16 x i16> @intrinsic_vfncvt_xu.f.w_nxv16i16_nxv16f32(<vscale x 16 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv16i16_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -506,10 +506,10 @@ declare <vscale x 16 x i16> @llvm.riscv.vfncvt.xu.f.w.mask.nxv16i16.nxv16f32(
 define <vscale x 16 x i16> @intrinsic_vfncvt_mask_xu.f.w_nxv16i16_nxv16f32(<vscale x 16 x i16> %0, <vscale x 16 x float> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv16i16_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i16> @llvm.riscv.vfncvt.xu.f.w.mask.nxv16i16.nxv16f32(
@@ -529,10 +529,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfncvt.xu.f.w.nxv1i32.nxv1f64(
 define <vscale x 1 x i32> @intrinsic_vfncvt_xu.f.w_nxv1i32_nxv1f64(<vscale x 1 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv1i32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -553,10 +553,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfncvt.xu.f.w.mask.nxv1i32.nxv1f64(
 define <vscale x 1 x i32> @intrinsic_vfncvt_mask_xu.f.w_nxv1i32_nxv1f64(<vscale x 1 x i32> %0, <vscale x 1 x double> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv1i32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vfncvt.xu.f.w.mask.nxv1i32.nxv1f64(
@@ -576,10 +576,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfncvt.xu.f.w.nxv2i32.nxv2f64(
 define <vscale x 2 x i32> @intrinsic_vfncvt_xu.f.w_nxv2i32_nxv2f64(<vscale x 2 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv2i32_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -600,10 +600,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfncvt.xu.f.w.mask.nxv2i32.nxv2f64(
 define <vscale x 2 x i32> @intrinsic_vfncvt_mask_xu.f.w_nxv2i32_nxv2f64(<vscale x 2 x i32> %0, <vscale x 2 x double> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv2i32_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vfncvt.xu.f.w.mask.nxv2i32.nxv2f64(
@@ -623,10 +623,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfncvt.xu.f.w.nxv4i32.nxv4f64(
 define <vscale x 4 x i32> @intrinsic_vfncvt_xu.f.w_nxv4i32_nxv4f64(<vscale x 4 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv4i32_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -647,10 +647,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfncvt.xu.f.w.mask.nxv4i32.nxv4f64(
 define <vscale x 4 x i32> @intrinsic_vfncvt_mask_xu.f.w_nxv4i32_nxv4f64(<vscale x 4 x i32> %0, <vscale x 4 x double> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv4i32_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vfncvt.xu.f.w.mask.nxv4i32.nxv4f64(
@@ -670,10 +670,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfncvt.xu.f.w.nxv8i32.nxv8f64(
 define <vscale x 8 x i32> @intrinsic_vfncvt_xu.f.w_nxv8i32_nxv8f64(<vscale x 8 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f.w_nxv8i32_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -694,10 +694,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfncvt.xu.f.w.mask.nxv8i32.nxv8f64(
 define <vscale x 8 x i32> @intrinsic_vfncvt_mask_xu.f.w_nxv8i32_nxv8f64(<vscale x 8 x i32> %0, <vscale x 8 x double> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f.w_nxv8i32_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfncvt.xu.f.w v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vfncvt.xu.f.w.mask.nxv8i32.nxv8f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmacc.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmacc.ll
index bdfa211dfdcb..01f4715274b6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmacc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmacc.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmacc.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfnmacc_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmacc.nxv1f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmacc.mask.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfnmacc_mask_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmacc.mask.nxv1f16.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmacc.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfnmacc_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmacc.nxv2f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmacc.mask.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfnmacc_mask_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmacc.mask.nxv2f16.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmacc.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfnmacc_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmacc.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmacc.mask.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfnmacc_mask_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmacc.mask.nxv4f16.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmacc.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfnmacc_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmacc.nxv8f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmacc.mask.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfnmacc_mask_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmacc.mask.nxv8f16.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmacc.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfnmacc_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmacc.nxv16f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmacc.mask.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfnmacc_mask_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmacc.mask.nxv16f16.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmacc.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfnmacc_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmacc.nxv1f32.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmacc.mask.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfnmacc_mask_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmacc.mask.nxv1f32.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmacc.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfnmacc_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmacc.nxv2f32.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmacc.mask.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfnmacc_mask_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmacc.mask.nxv2f32.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmacc.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfnmacc_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmacc.nxv4f32.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmacc.mask.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfnmacc_mask_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmacc.mask.nxv4f32.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmacc.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfnmacc_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmacc.nxv8f32.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmacc.mask.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfnmacc_mask_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmacc.mask.nxv8f32.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmacc.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfnmacc_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmacc.nxv1f64.nxv1f64(
@@ -488,10 +488,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmacc.mask.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfnmacc_mask_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmacc.mask.nxv1f64.nxv1f64(
@@ -513,10 +513,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmacc.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfnmacc_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmacc.nxv2f64.nxv2f64(
@@ -538,10 +538,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmacc.mask.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfnmacc_mask_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmacc.mask.nxv2f64.nxv2f64(
@@ -563,10 +563,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmacc.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfnmacc_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmacc.nxv4f64.nxv4f64(
@@ -588,10 +588,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmacc.mask.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfnmacc_mask_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmacc.mask.nxv4f64.nxv4f64(
@@ -613,10 +613,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmacc.nxv1f16.f16(
 define <vscale x 1 x half>  @intrinsic_vfnmacc_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmacc.nxv1f16.f16(
@@ -638,10 +638,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmacc.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfnmacc_mask_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmacc.mask.nxv1f16.f16(
@@ -663,10 +663,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmacc.nxv2f16.f16(
 define <vscale x 2 x half>  @intrinsic_vfnmacc_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmacc.nxv2f16.f16(
@@ -688,10 +688,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmacc.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfnmacc_mask_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmacc.mask.nxv2f16.f16(
@@ -713,10 +713,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmacc.nxv4f16.f16(
 define <vscale x 4 x half>  @intrinsic_vfnmacc_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmacc.nxv4f16.f16(
@@ -738,10 +738,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmacc.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfnmacc_mask_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmacc.mask.nxv4f16.f16(
@@ -763,10 +763,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmacc.nxv8f16.f16(
 define <vscale x 8 x half>  @intrinsic_vfnmacc_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmacc.nxv8f16.f16(
@@ -788,10 +788,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmacc.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfnmacc_mask_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmacc.mask.nxv8f16.f16(
@@ -813,10 +813,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmacc.nxv16f16.f16(
 define <vscale x 16 x half>  @intrinsic_vfnmacc_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmacc.nxv16f16.f16(
@@ -838,10 +838,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmacc.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfnmacc_mask_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmacc.mask.nxv16f16.f16(
@@ -863,10 +863,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmacc.nxv1f32.f32(
 define <vscale x 1 x float>  @intrinsic_vfnmacc_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmacc.nxv1f32.f32(
@@ -888,10 +888,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmacc.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfnmacc_mask_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmacc.mask.nxv1f32.f32(
@@ -913,10 +913,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmacc.nxv2f32.f32(
 define <vscale x 2 x float>  @intrinsic_vfnmacc_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmacc.nxv2f32.f32(
@@ -938,10 +938,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmacc.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfnmacc_mask_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmacc.mask.nxv2f32.f32(
@@ -963,10 +963,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmacc.nxv4f32.f32(
 define <vscale x 4 x float>  @intrinsic_vfnmacc_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmacc.nxv4f32.f32(
@@ -988,10 +988,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmacc.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfnmacc_mask_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmacc.mask.nxv4f32.f32(
@@ -1013,10 +1013,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmacc.nxv8f32.f32(
 define <vscale x 8 x float>  @intrinsic_vfnmacc_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmacc.nxv8f32.f32(
@@ -1038,10 +1038,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmacc.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfnmacc_mask_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmacc.mask.nxv8f32.f32(
@@ -1063,10 +1063,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmacc.nxv1f64.f64(
 define <vscale x 1 x double>  @intrinsic_vfnmacc_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmacc.nxv1f64.f64(
@@ -1088,10 +1088,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmacc.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfnmacc_mask_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmacc.mask.nxv1f64.f64(
@@ -1113,10 +1113,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmacc.nxv2f64.f64(
 define <vscale x 2 x double>  @intrinsic_vfnmacc_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmacc.nxv2f64.f64(
@@ -1138,10 +1138,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmacc.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfnmacc_mask_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmacc.mask.nxv2f64.f64(
@@ -1163,10 +1163,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmacc.nxv4f64.f64(
 define <vscale x 4 x double>  @intrinsic_vfnmacc_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmacc.nxv4f64.f64(
@@ -1188,10 +1188,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmacc.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfnmacc_mask_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmacc.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmacc.mask.nxv4f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmadd.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmadd.ll
index 4eb2e7caba24..ae4cfef35e61 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmadd.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmadd.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfnmadd_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmadd.nxv1f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmadd.mask.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfnmadd_mask_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmadd.mask.nxv1f16.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmadd.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfnmadd_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmadd.nxv2f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmadd.mask.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfnmadd_mask_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmadd.mask.nxv2f16.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmadd.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfnmadd_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmadd.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmadd.mask.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfnmadd_mask_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmadd.mask.nxv4f16.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmadd.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfnmadd_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmadd.nxv8f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmadd.mask.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfnmadd_mask_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmadd.mask.nxv8f16.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmadd.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfnmadd_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmadd.nxv16f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmadd.mask.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfnmadd_mask_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmadd.mask.nxv16f16.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmadd.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfnmadd_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmadd.nxv1f32.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmadd.mask.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfnmadd_mask_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmadd.mask.nxv1f32.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmadd.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfnmadd_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmadd.nxv2f32.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmadd.mask.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfnmadd_mask_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmadd.mask.nxv2f32.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmadd.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfnmadd_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmadd.nxv4f32.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmadd.mask.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfnmadd_mask_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmadd.mask.nxv4f32.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmadd.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfnmadd_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmadd.nxv8f32.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmadd.mask.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfnmadd_mask_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmadd.mask.nxv8f32.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmadd.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfnmadd_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmadd.nxv1f64.nxv1f64(
@@ -488,10 +488,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmadd.mask.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfnmadd_mask_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmadd.mask.nxv1f64.nxv1f64(
@@ -513,10 +513,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmadd.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfnmadd_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmadd.nxv2f64.nxv2f64(
@@ -538,10 +538,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmadd.mask.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfnmadd_mask_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmadd.mask.nxv2f64.nxv2f64(
@@ -563,10 +563,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmadd.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfnmadd_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmadd.nxv4f64.nxv4f64(
@@ -588,10 +588,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmadd.mask.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfnmadd_mask_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmadd.mask.nxv4f64.nxv4f64(
@@ -613,10 +613,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmadd.nxv1f16.f16(
 define <vscale x 1 x half>  @intrinsic_vfnmadd_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmadd.nxv1f16.f16(
@@ -638,10 +638,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmadd.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfnmadd_mask_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmadd.mask.nxv1f16.f16(
@@ -663,10 +663,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmadd.nxv2f16.f16(
 define <vscale x 2 x half>  @intrinsic_vfnmadd_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmadd.nxv2f16.f16(
@@ -688,10 +688,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmadd.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfnmadd_mask_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmadd.mask.nxv2f16.f16(
@@ -713,10 +713,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmadd.nxv4f16.f16(
 define <vscale x 4 x half>  @intrinsic_vfnmadd_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmadd.nxv4f16.f16(
@@ -738,10 +738,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmadd.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfnmadd_mask_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmadd.mask.nxv4f16.f16(
@@ -763,10 +763,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmadd.nxv8f16.f16(
 define <vscale x 8 x half>  @intrinsic_vfnmadd_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmadd.nxv8f16.f16(
@@ -788,10 +788,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmadd.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfnmadd_mask_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmadd.mask.nxv8f16.f16(
@@ -813,10 +813,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmadd.nxv16f16.f16(
 define <vscale x 16 x half>  @intrinsic_vfnmadd_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmadd.nxv16f16.f16(
@@ -838,10 +838,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmadd.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfnmadd_mask_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmadd.mask.nxv16f16.f16(
@@ -863,10 +863,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmadd.nxv1f32.f32(
 define <vscale x 1 x float>  @intrinsic_vfnmadd_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmadd.nxv1f32.f32(
@@ -888,10 +888,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmadd.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfnmadd_mask_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmadd.mask.nxv1f32.f32(
@@ -913,10 +913,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmadd.nxv2f32.f32(
 define <vscale x 2 x float>  @intrinsic_vfnmadd_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmadd.nxv2f32.f32(
@@ -938,10 +938,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmadd.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfnmadd_mask_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmadd.mask.nxv2f32.f32(
@@ -963,10 +963,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmadd.nxv4f32.f32(
 define <vscale x 4 x float>  @intrinsic_vfnmadd_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmadd.nxv4f32.f32(
@@ -988,10 +988,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmadd.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfnmadd_mask_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmadd.mask.nxv4f32.f32(
@@ -1013,10 +1013,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmadd.nxv8f32.f32(
 define <vscale x 8 x float>  @intrinsic_vfnmadd_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmadd.nxv8f32.f32(
@@ -1038,10 +1038,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmadd.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfnmadd_mask_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmadd.mask.nxv8f32.f32(
@@ -1063,10 +1063,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmadd.nxv1f64.f64(
 define <vscale x 1 x double>  @intrinsic_vfnmadd_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmadd.nxv1f64.f64(
@@ -1088,10 +1088,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmadd.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfnmadd_mask_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmadd.mask.nxv1f64.f64(
@@ -1113,10 +1113,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmadd.nxv2f64.f64(
 define <vscale x 2 x double>  @intrinsic_vfnmadd_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmadd.nxv2f64.f64(
@@ -1138,10 +1138,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmadd.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfnmadd_mask_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmadd.mask.nxv2f64.f64(
@@ -1163,10 +1163,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmadd.nxv4f64.f64(
 define <vscale x 4 x double>  @intrinsic_vfnmadd_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmadd.nxv4f64.f64(
@@ -1188,10 +1188,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmadd.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfnmadd_mask_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmadd.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmadd.mask.nxv4f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmsac.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmsac.ll
index dc30540bc0af..071f546b4f60 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmsac.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmsac.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmsac.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfnmsac_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmsac.nxv1f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmsac.mask.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfnmsac_mask_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmsac.mask.nxv1f16.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmsac.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfnmsac_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmsac.nxv2f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmsac.mask.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfnmsac_mask_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmsac.mask.nxv2f16.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmsac.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfnmsac_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmsac.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmsac.mask.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfnmsac_mask_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmsac.mask.nxv4f16.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmsac.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfnmsac_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmsac.nxv8f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmsac.mask.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfnmsac_mask_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmsac.mask.nxv8f16.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmsac.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfnmsac_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmsac.nxv16f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmsac.mask.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfnmsac_mask_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmsac.mask.nxv16f16.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmsac.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfnmsac_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmsac.nxv1f32.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmsac.mask.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfnmsac_mask_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmsac.mask.nxv1f32.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmsac.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfnmsac_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmsac.nxv2f32.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmsac.mask.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfnmsac_mask_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmsac.mask.nxv2f32.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmsac.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfnmsac_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmsac.nxv4f32.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmsac.mask.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfnmsac_mask_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmsac.mask.nxv4f32.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmsac.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfnmsac_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmsac.nxv8f32.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmsac.mask.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfnmsac_mask_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmsac.mask.nxv8f32.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmsac.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfnmsac_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmsac.nxv1f64.nxv1f64(
@@ -488,10 +488,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmsac.mask.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfnmsac_mask_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmsac.mask.nxv1f64.nxv1f64(
@@ -513,10 +513,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmsac.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfnmsac_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmsac.nxv2f64.nxv2f64(
@@ -538,10 +538,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmsac.mask.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfnmsac_mask_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmsac.mask.nxv2f64.nxv2f64(
@@ -563,10 +563,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmsac.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfnmsac_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmsac.nxv4f64.nxv4f64(
@@ -588,10 +588,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmsac.mask.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfnmsac_mask_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmsac.mask.nxv4f64.nxv4f64(
@@ -613,10 +613,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmsac.nxv1f16.f16(
 define <vscale x 1 x half>  @intrinsic_vfnmsac_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmsac.nxv1f16.f16(
@@ -638,10 +638,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmsac.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfnmsac_mask_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmsac.mask.nxv1f16.f16(
@@ -663,10 +663,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmsac.nxv2f16.f16(
 define <vscale x 2 x half>  @intrinsic_vfnmsac_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmsac.nxv2f16.f16(
@@ -688,10 +688,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmsac.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfnmsac_mask_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmsac.mask.nxv2f16.f16(
@@ -713,10 +713,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmsac.nxv4f16.f16(
 define <vscale x 4 x half>  @intrinsic_vfnmsac_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmsac.nxv4f16.f16(
@@ -738,10 +738,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmsac.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfnmsac_mask_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmsac.mask.nxv4f16.f16(
@@ -763,10 +763,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmsac.nxv8f16.f16(
 define <vscale x 8 x half>  @intrinsic_vfnmsac_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmsac.nxv8f16.f16(
@@ -788,10 +788,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmsac.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfnmsac_mask_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmsac.mask.nxv8f16.f16(
@@ -813,10 +813,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmsac.nxv16f16.f16(
 define <vscale x 16 x half>  @intrinsic_vfnmsac_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmsac.nxv16f16.f16(
@@ -838,10 +838,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmsac.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfnmsac_mask_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmsac.mask.nxv16f16.f16(
@@ -863,10 +863,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmsac.nxv1f32.f32(
 define <vscale x 1 x float>  @intrinsic_vfnmsac_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmsac.nxv1f32.f32(
@@ -888,10 +888,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmsac.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfnmsac_mask_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmsac.mask.nxv1f32.f32(
@@ -913,10 +913,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmsac.nxv2f32.f32(
 define <vscale x 2 x float>  @intrinsic_vfnmsac_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmsac.nxv2f32.f32(
@@ -938,10 +938,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmsac.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfnmsac_mask_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmsac.mask.nxv2f32.f32(
@@ -963,10 +963,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmsac.nxv4f32.f32(
 define <vscale x 4 x float>  @intrinsic_vfnmsac_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmsac.nxv4f32.f32(
@@ -988,10 +988,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmsac.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfnmsac_mask_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmsac.mask.nxv4f32.f32(
@@ -1013,10 +1013,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmsac.nxv8f32.f32(
 define <vscale x 8 x float>  @intrinsic_vfnmsac_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmsac.nxv8f32.f32(
@@ -1038,10 +1038,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmsac.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfnmsac_mask_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmsac.mask.nxv8f32.f32(
@@ -1063,10 +1063,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmsac.nxv1f64.f64(
 define <vscale x 1 x double>  @intrinsic_vfnmsac_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmsac.nxv1f64.f64(
@@ -1088,10 +1088,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmsac.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfnmsac_mask_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmsac.mask.nxv1f64.f64(
@@ -1113,10 +1113,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmsac.nxv2f64.f64(
 define <vscale x 2 x double>  @intrinsic_vfnmsac_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmsac.nxv2f64.f64(
@@ -1138,10 +1138,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmsac.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfnmsac_mask_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmsac.mask.nxv2f64.f64(
@@ -1163,10 +1163,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmsac.nxv4f64.f64(
 define <vscale x 4 x double>  @intrinsic_vfnmsac_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmsac.nxv4f64.f64(
@@ -1188,10 +1188,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmsac.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfnmsac_mask_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsac.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmsac.mask.nxv4f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfnmsub.ll b/llvm/test/CodeGen/RISCV/rvv/vfnmsub.ll
index cadddb016c4f..4922cf40e503 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfnmsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfnmsub.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmsub.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfnmsub_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmsub.nxv1f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmsub.mask.nxv1f16.nxv1f16(
 define <vscale x 1 x half>  @intrinsic_vfnmsub_mask_vv_nxv1f16_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv1f16_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmsub.mask.nxv1f16.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmsub.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfnmsub_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmsub.nxv2f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmsub.mask.nxv2f16.nxv2f16(
 define <vscale x 2 x half>  @intrinsic_vfnmsub_mask_vv_nxv2f16_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv2f16_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmsub.mask.nxv2f16.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmsub.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfnmsub_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmsub.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmsub.mask.nxv4f16.nxv4f16(
 define <vscale x 4 x half>  @intrinsic_vfnmsub_mask_vv_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmsub.mask.nxv4f16.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmsub.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfnmsub_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmsub.nxv8f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmsub.mask.nxv8f16.nxv8f16(
 define <vscale x 8 x half>  @intrinsic_vfnmsub_mask_vv_nxv8f16_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv8f16_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmsub.mask.nxv8f16.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmsub.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfnmsub_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmsub.nxv16f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmsub.mask.nxv16f16.nxv16f16(
 define <vscale x 16 x half>  @intrinsic_vfnmsub_mask_vv_nxv16f16_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv16f16_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmsub.mask.nxv16f16.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmsub.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfnmsub_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmsub.nxv1f32.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmsub.mask.nxv1f32.nxv1f32(
 define <vscale x 1 x float>  @intrinsic_vfnmsub_mask_vv_nxv1f32_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv1f32_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmsub.mask.nxv1f32.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmsub.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfnmsub_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmsub.nxv2f32.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmsub.mask.nxv2f32.nxv2f32(
 define <vscale x 2 x float>  @intrinsic_vfnmsub_mask_vv_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmsub.mask.nxv2f32.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmsub.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfnmsub_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmsub.nxv4f32.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmsub.mask.nxv4f32.nxv4f32(
 define <vscale x 4 x float>  @intrinsic_vfnmsub_mask_vv_nxv4f32_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv4f32_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmsub.mask.nxv4f32.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmsub.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfnmsub_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmsub.nxv8f32.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmsub.mask.nxv8f32.nxv8f32(
 define <vscale x 8 x float>  @intrinsic_vfnmsub_mask_vv_nxv8f32_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv8f32_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmsub.mask.nxv8f32.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmsub.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfnmsub_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmsub.nxv1f64.nxv1f64(
@@ -488,10 +488,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmsub.mask.nxv1f64.nxv1f64(
 define <vscale x 1 x double>  @intrinsic_vfnmsub_mask_vv_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmsub.mask.nxv1f64.nxv1f64(
@@ -513,10 +513,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmsub.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfnmsub_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v10, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmsub.nxv2f64.nxv2f64(
@@ -538,10 +538,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmsub.mask.nxv2f64.nxv2f64(
 define <vscale x 2 x double>  @intrinsic_vfnmsub_mask_vv_nxv2f64_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv2f64_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmsub.mask.nxv2f64.nxv2f64(
@@ -563,10 +563,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmsub.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfnmsub_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v12, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmsub.nxv4f64.nxv4f64(
@@ -588,10 +588,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmsub.mask.nxv4f64.nxv4f64(
 define <vscale x 4 x double>  @intrinsic_vfnmsub_mask_vv_nxv4f64_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv4f64_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmsub.mask.nxv4f64.nxv4f64(
@@ -613,10 +613,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmsub.nxv1f16.f16(
 define <vscale x 1 x half>  @intrinsic_vfnmsub_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmsub.nxv1f16.f16(
@@ -638,10 +638,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfnmsub.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfnmsub_mask_vf_nxv1f16_f16_nxv1f16(<vscale x 1 x half> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv1f16_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfnmsub.mask.nxv1f16.f16(
@@ -663,10 +663,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmsub.nxv2f16.f16(
 define <vscale x 2 x half>  @intrinsic_vfnmsub_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmsub.nxv2f16.f16(
@@ -688,10 +688,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfnmsub.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfnmsub_mask_vf_nxv2f16_f16_nxv2f16(<vscale x 2 x half> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv2f16_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfnmsub.mask.nxv2f16.f16(
@@ -713,10 +713,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmsub.nxv4f16.f16(
 define <vscale x 4 x half>  @intrinsic_vfnmsub_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmsub.nxv4f16.f16(
@@ -738,10 +738,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfnmsub.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfnmsub_mask_vf_nxv4f16_f16_nxv4f16(<vscale x 4 x half> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv4f16_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfnmsub.mask.nxv4f16.f16(
@@ -763,10 +763,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmsub.nxv8f16.f16(
 define <vscale x 8 x half>  @intrinsic_vfnmsub_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmsub.nxv8f16.f16(
@@ -788,10 +788,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfnmsub.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfnmsub_mask_vf_nxv8f16_f16_nxv8f16(<vscale x 8 x half> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv8f16_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfnmsub.mask.nxv8f16.f16(
@@ -813,10 +813,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmsub.nxv16f16.f16(
 define <vscale x 16 x half>  @intrinsic_vfnmsub_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmsub.nxv16f16.f16(
@@ -838,10 +838,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfnmsub.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfnmsub_mask_vf_nxv16f16_f16_nxv16f16(<vscale x 16 x half> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv16f16_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfnmsub.mask.nxv16f16.f16(
@@ -863,10 +863,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmsub.nxv1f32.f32(
 define <vscale x 1 x float>  @intrinsic_vfnmsub_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmsub.nxv1f32.f32(
@@ -888,10 +888,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfnmsub.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfnmsub_mask_vf_nxv1f32_f32_nxv1f32(<vscale x 1 x float> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv1f32_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfnmsub.mask.nxv1f32.f32(
@@ -913,10 +913,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmsub.nxv2f32.f32(
 define <vscale x 2 x float>  @intrinsic_vfnmsub_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmsub.nxv2f32.f32(
@@ -938,10 +938,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfnmsub.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfnmsub_mask_vf_nxv2f32_f32_nxv2f32(<vscale x 2 x float> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv2f32_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfnmsub.mask.nxv2f32.f32(
@@ -963,10 +963,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmsub.nxv4f32.f32(
 define <vscale x 4 x float>  @intrinsic_vfnmsub_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmsub.nxv4f32.f32(
@@ -988,10 +988,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfnmsub.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfnmsub_mask_vf_nxv4f32_f32_nxv4f32(<vscale x 4 x float> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv4f32_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfnmsub.mask.nxv4f32.f32(
@@ -1013,10 +1013,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmsub.nxv8f32.f32(
 define <vscale x 8 x float>  @intrinsic_vfnmsub_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmsub.nxv8f32.f32(
@@ -1038,10 +1038,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfnmsub.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfnmsub_mask_vf_nxv8f32_f32_nxv8f32(<vscale x 8 x float> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv8f32_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfnmsub.mask.nxv8f32.f32(
@@ -1063,10 +1063,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmsub.nxv1f64.f64(
 define <vscale x 1 x double>  @intrinsic_vfnmsub_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmsub.nxv1f64.f64(
@@ -1088,10 +1088,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfnmsub.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfnmsub_mask_vf_nxv1f64_f64_nxv1f64(<vscale x 1 x double> %0, double %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv1f64_f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfnmsub.mask.nxv1f64.f64(
@@ -1113,10 +1113,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmsub.nxv2f64.f64(
 define <vscale x 2 x double>  @intrinsic_vfnmsub_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmsub.nxv2f64.f64(
@@ -1138,10 +1138,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfnmsub.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfnmsub_mask_vf_nxv2f64_f64_nxv2f64(<vscale x 2 x double> %0, double %1, <vscale x 2 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv2f64_f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfnmsub.mask.nxv2f64.f64(
@@ -1163,10 +1163,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmsub.nxv4f64.f64(
 define <vscale x 4 x double>  @intrinsic_vfnmsub_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmsub.nxv4f64.f64(
@@ -1188,10 +1188,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfnmsub.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfnmsub_mask_vf_nxv4f64_f64_nxv4f64(<vscale x 4 x double> %0, double %1, <vscale x 4 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv4f64_f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfnmsub.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfnmsub.mask.nxv4f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfrdiv.ll b/llvm/test/CodeGen/RISCV/rvv/vfrdiv.ll
index f17c226ada0d..f73e7dce9212 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfrdiv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfrdiv.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfrdiv.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfrdiv_vf_nxv1f16_nxv1f16_f16(<vscale x 1 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv1f16_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfrdiv.nxv1f16.f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfrdiv.mask.nxv1f16.f16(
 define <vscale x 1 x half> @intrinsic_vfrdiv_mask_vf_nxv1f16_nxv1f16_f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv1f16_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfrdiv.mask.nxv1f16.f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfrdiv.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfrdiv_vf_nxv2f16_nxv2f16_f16(<vscale x 2 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv2f16_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfrdiv.nxv2f16.f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfrdiv.mask.nxv2f16.f16(
 define <vscale x 2 x half> @intrinsic_vfrdiv_mask_vf_nxv2f16_nxv2f16_f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv2f16_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfrdiv.mask.nxv2f16.f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfrdiv.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfrdiv_vf_nxv4f16_nxv4f16_f16(<vscale x 4 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv4f16_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfrdiv.nxv4f16.f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfrdiv.mask.nxv4f16.f16(
 define <vscale x 4 x half> @intrinsic_vfrdiv_mask_vf_nxv4f16_nxv4f16_f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv4f16_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfrdiv.mask.nxv4f16.f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfrdiv.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfrdiv_vf_nxv8f16_nxv8f16_f16(<vscale x 8 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv8f16_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfrdiv.nxv8f16.f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfrdiv.mask.nxv8f16.f16(
 define <vscale x 8 x half> @intrinsic_vfrdiv_mask_vf_nxv8f16_nxv8f16_f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv8f16_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfrdiv.mask.nxv8f16.f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfrdiv.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfrdiv_vf_nxv16f16_nxv16f16_f16(<vscale x 16 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv16f16_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfrdiv.nxv16f16.f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfrdiv.mask.nxv16f16.f16(
 define <vscale x 16 x half> @intrinsic_vfrdiv_mask_vf_nxv16f16_nxv16f16_f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv16f16_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfrdiv.mask.nxv16f16.f16(
@@ -263,10 +263,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfrdiv.nxv32f16.f16(
 define <vscale x 32 x half> @intrinsic_vfrdiv_vf_nxv32f16_nxv32f16_f16(<vscale x 32 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv32f16_nxv32f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfrdiv.nxv32f16.f16(
@@ -288,10 +288,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfrdiv.mask.nxv32f16.f16(
 define <vscale x 32 x half> @intrinsic_vfrdiv_mask_vf_nxv32f16_nxv32f16_f16(<vscale x 32 x half> %0, <vscale x 32 x half> %1, half %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv32f16_nxv32f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfrdiv.mask.nxv32f16.f16(
@@ -313,10 +313,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfrdiv.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfrdiv_vf_nxv1f32_nxv1f32_f32(<vscale x 1 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv1f32_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfrdiv.nxv1f32.f32(
@@ -338,10 +338,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfrdiv.mask.nxv1f32.f32(
 define <vscale x 1 x float> @intrinsic_vfrdiv_mask_vf_nxv1f32_nxv1f32_f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv1f32_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfrdiv.mask.nxv1f32.f32(
@@ -363,10 +363,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfrdiv.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfrdiv_vf_nxv2f32_nxv2f32_f32(<vscale x 2 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv2f32_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfrdiv.nxv2f32.f32(
@@ -388,10 +388,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfrdiv.mask.nxv2f32.f32(
 define <vscale x 2 x float> @intrinsic_vfrdiv_mask_vf_nxv2f32_nxv2f32_f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv2f32_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfrdiv.mask.nxv2f32.f32(
@@ -413,10 +413,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfrdiv.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfrdiv_vf_nxv4f32_nxv4f32_f32(<vscale x 4 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv4f32_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfrdiv.nxv4f32.f32(
@@ -438,10 +438,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfrdiv.mask.nxv4f32.f32(
 define <vscale x 4 x float> @intrinsic_vfrdiv_mask_vf_nxv4f32_nxv4f32_f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv4f32_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfrdiv.mask.nxv4f32.f32(
@@ -463,10 +463,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfrdiv.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfrdiv_vf_nxv8f32_nxv8f32_f32(<vscale x 8 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv8f32_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfrdiv.nxv8f32.f32(
@@ -488,10 +488,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfrdiv.mask.nxv8f32.f32(
 define <vscale x 8 x float> @intrinsic_vfrdiv_mask_vf_nxv8f32_nxv8f32_f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv8f32_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfrdiv.mask.nxv8f32.f32(
@@ -513,10 +513,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfrdiv.nxv16f32.f32(
 define <vscale x 16 x float> @intrinsic_vfrdiv_vf_nxv16f32_nxv16f32_f32(<vscale x 16 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv16f32_nxv16f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfrdiv.nxv16f32.f32(
@@ -538,10 +538,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfrdiv.mask.nxv16f32.f32(
 define <vscale x 16 x float> @intrinsic_vfrdiv_mask_vf_nxv16f32_nxv16f32_f32(<vscale x 16 x float> %0, <vscale x 16 x float> %1, float %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv16f32_nxv16f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfrdiv.mask.nxv16f32.f32(
@@ -563,10 +563,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfrdiv.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfrdiv_vf_nxv1f64_nxv1f64_f64(<vscale x 1 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv1f64_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfrdiv.nxv1f64.f64(
@@ -588,10 +588,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfrdiv.mask.nxv1f64.f64(
 define <vscale x 1 x double> @intrinsic_vfrdiv_mask_vf_nxv1f64_nxv1f64_f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, double %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv1f64_nxv1f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfrdiv.mask.nxv1f64.f64(
@@ -613,10 +613,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfrdiv.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfrdiv_vf_nxv2f64_nxv2f64_f64(<vscale x 2 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv2f64_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfrdiv.nxv2f64.f64(
@@ -638,10 +638,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfrdiv.mask.nxv2f64.f64(
 define <vscale x 2 x double> @intrinsic_vfrdiv_mask_vf_nxv2f64_nxv2f64_f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, double %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv2f64_nxv2f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfrdiv.mask.nxv2f64.f64(
@@ -663,10 +663,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfrdiv.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfrdiv_vf_nxv4f64_nxv4f64_f64(<vscale x 4 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv4f64_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfrdiv.nxv4f64.f64(
@@ -688,10 +688,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfrdiv.mask.nxv4f64.f64(
 define <vscale x 4 x double> @intrinsic_vfrdiv_mask_vf_nxv4f64_nxv4f64_f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, double %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv4f64_nxv4f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfrdiv.mask.nxv4f64.f64(
@@ -713,10 +713,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfrdiv.nxv8f64.f64(
 define <vscale x 8 x double> @intrinsic_vfrdiv_vf_nxv8f64_nxv8f64_f64(<vscale x 8 x double> %0, double %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv8f64_nxv8f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfrdiv.nxv8f64.f64(
@@ -738,10 +738,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfrdiv.mask.nxv8f64.f64(
 define <vscale x 8 x double> @intrinsic_vfrdiv_mask_vf_nxv8f64_nxv8f64_f64(<vscale x 8 x double> %0, <vscale x 8 x double> %1, double %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv8f64_nxv8f64_f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrdiv.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfrdiv.mask.nxv8f64.f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfrec7.ll b/llvm/test/CodeGen/RISCV/rvv/vfrec7.ll
index 0204f0373d93..914b3b33fbe5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfrec7.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfrec7.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfrec7.nxv1f16(
 define <vscale x 1 x half> @intrinsic_vfrec7_v_nxv1f16_nxv1f16(<vscale x 1 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfrec7.nxv1f16(
@@ -35,10 +35,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfrec7.mask.nxv1f16(
 define <vscale x 1 x half> @intrinsic_vfrec7_mask_v_nxv1f16_nxv1f16(<vscale x 1 x i1> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfrec7.mask.nxv1f16(
@@ -58,10 +58,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfrec7.nxv2f16(
 define <vscale x 2 x half> @intrinsic_vfrec7_v_nxv2f16_nxv2f16(<vscale x 2 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfrec7.nxv2f16(
@@ -81,10 +81,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfrec7.mask.nxv2f16(
 define <vscale x 2 x half> @intrinsic_vfrec7_mask_v_nxv2f16_nxv2f16(<vscale x 2 x i1> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfrec7.mask.nxv2f16(
@@ -104,10 +104,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfrec7.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfrec7_v_nxv4f16_nxv4f16(<vscale x 4 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfrec7.nxv4f16(
@@ -127,10 +127,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfrec7.mask.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfrec7_mask_v_nxv4f16_nxv4f16(<vscale x 4 x i1> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfrec7.mask.nxv4f16(
@@ -150,10 +150,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfrec7.nxv8f16(
 define <vscale x 8 x half> @intrinsic_vfrec7_v_nxv8f16_nxv8f16(<vscale x 8 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfrec7.nxv8f16(
@@ -173,10 +173,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfrec7.mask.nxv8f16(
 define <vscale x 8 x half> @intrinsic_vfrec7_mask_v_nxv8f16_nxv8f16(<vscale x 8 x i1> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfrec7.mask.nxv8f16(
@@ -196,10 +196,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfrec7.nxv16f16(
 define <vscale x 16 x half> @intrinsic_vfrec7_v_nxv16f16_nxv16f16(<vscale x 16 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfrec7.nxv16f16(
@@ -219,10 +219,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfrec7.mask.nxv16f16(
 define <vscale x 16 x half> @intrinsic_vfrec7_mask_v_nxv16f16_nxv16f16(<vscale x 16 x i1> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfrec7.mask.nxv16f16(
@@ -242,10 +242,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfrec7.nxv32f16(
 define <vscale x 32 x half> @intrinsic_vfrec7_v_nxv32f16_nxv32f16(<vscale x 32 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv32f16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfrec7.nxv32f16(
@@ -265,10 +265,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfrec7.mask.nxv32f16(
 define <vscale x 32 x half> @intrinsic_vfrec7_mask_v_nxv32f16_nxv32f16(<vscale x 32 x i1> %0, <vscale x 32 x half> %1, <vscale x 32 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv32f16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfrec7.mask.nxv32f16(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfrec7.nxv1f32(
 define <vscale x 1 x float> @intrinsic_vfrec7_v_nxv1f32_nxv1f32(<vscale x 1 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfrec7.nxv1f32(
@@ -311,10 +311,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfrec7.mask.nxv1f32(
 define <vscale x 1 x float> @intrinsic_vfrec7_mask_v_nxv1f32_nxv1f32(<vscale x 1 x i1> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfrec7.mask.nxv1f32(
@@ -334,10 +334,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfrec7.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfrec7_v_nxv2f32_nxv2f32(<vscale x 2 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfrec7.nxv2f32(
@@ -357,10 +357,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfrec7.mask.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfrec7_mask_v_nxv2f32_nxv2f32(<vscale x 2 x i1> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfrec7.mask.nxv2f32(
@@ -380,10 +380,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfrec7.nxv4f32(
 define <vscale x 4 x float> @intrinsic_vfrec7_v_nxv4f32_nxv4f32(<vscale x 4 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfrec7.nxv4f32(
@@ -403,10 +403,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfrec7.mask.nxv4f32(
 define <vscale x 4 x float> @intrinsic_vfrec7_mask_v_nxv4f32_nxv4f32(<vscale x 4 x i1> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfrec7.mask.nxv4f32(
@@ -426,10 +426,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfrec7.nxv8f32(
 define <vscale x 8 x float> @intrinsic_vfrec7_v_nxv8f32_nxv8f32(<vscale x 8 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfrec7.nxv8f32(
@@ -449,10 +449,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfrec7.mask.nxv8f32(
 define <vscale x 8 x float> @intrinsic_vfrec7_mask_v_nxv8f32_nxv8f32(<vscale x 8 x i1> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfrec7.mask.nxv8f32(
@@ -472,10 +472,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfrec7.nxv16f32(
 define <vscale x 16 x float> @intrinsic_vfrec7_v_nxv16f32_nxv16f32(<vscale x 16 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv16f32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfrec7.nxv16f32(
@@ -495,10 +495,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfrec7.mask.nxv16f32(
 define <vscale x 16 x float> @intrinsic_vfrec7_mask_v_nxv16f32_nxv16f32(<vscale x 16 x i1> %0, <vscale x 16 x float> %1, <vscale x 16 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv16f32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfrec7.mask.nxv16f32(
@@ -518,10 +518,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfrec7.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfrec7_v_nxv1f64_nxv1f64(<vscale x 1 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfrec7.nxv1f64(
@@ -541,10 +541,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfrec7.mask.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfrec7_mask_v_nxv1f64_nxv1f64(<vscale x 1 x i1> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfrec7.mask.nxv1f64(
@@ -564,10 +564,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfrec7.nxv2f64(
 define <vscale x 2 x double> @intrinsic_vfrec7_v_nxv2f64_nxv2f64(<vscale x 2 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfrec7.nxv2f64(
@@ -587,10 +587,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfrec7.mask.nxv2f64(
 define <vscale x 2 x double> @intrinsic_vfrec7_mask_v_nxv2f64_nxv2f64(<vscale x 2 x i1> %0, <vscale x 2 x double> %1, <vscale x 2 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfrec7.mask.nxv2f64(
@@ -610,10 +610,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfrec7.nxv4f64(
 define <vscale x 4 x double> @intrinsic_vfrec7_v_nxv4f64_nxv4f64(<vscale x 4 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfrec7.nxv4f64(
@@ -633,10 +633,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfrec7.mask.nxv4f64(
 define <vscale x 4 x double> @intrinsic_vfrec7_mask_v_nxv4f64_nxv4f64(<vscale x 4 x i1> %0, <vscale x 4 x double> %1, <vscale x 4 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfrec7.mask.nxv4f64(
@@ -656,10 +656,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfrec7.nxv8f64(
 define <vscale x 8 x double> @intrinsic_vfrec7_v_nxv8f64_nxv8f64(<vscale x 8 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_v_nxv8f64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfrec7.nxv8f64(
@@ -679,10 +679,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfrec7.mask.nxv8f64(
 define <vscale x 8 x double> @intrinsic_vfrec7_mask_v_nxv8f64_nxv8f64(<vscale x 8 x i1> %0, <vscale x 8 x double> %1, <vscale x 8 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfrec7_mask_v_nxv8f64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfrec7.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfrec7.mask.nxv8f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfredosum.ll b/llvm/test/CodeGen/RISCV/rvv/vfredosum.ll
index 19dde75969e3..6de9c82002f5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfredosum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfredosum.ll
@@ -13,10 +13,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv1f16(
 define <vscale x 4 x half> @intrinsic_vfredosum_vs_nxv4f16_nxv1f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 1 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv4f16_nxv1f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv1f16.nxv1i1(
 define <vscale x 4 x half> @intrinsic_vfredosum_mask_vs_nxv4f16_nxv1f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 1 x half> %1, <vscale x 4 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv4f16_nxv1f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv1f16.nxv1i1(
@@ -63,10 +63,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv2f16(
 define <vscale x 4 x half> @intrinsic_vfredosum_vs_nxv4f16_nxv2f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 2 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv4f16_nxv2f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv2f16.nxv2i1(
 define <vscale x 4 x half> @intrinsic_vfredosum_mask_vs_nxv4f16_nxv2f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 2 x half> %1, <vscale x 4 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv4f16_nxv2f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv2f16.nxv2i1(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfredosum_vs_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv4f16.nxv4i1(
 define <vscale x 4 x half> @intrinsic_vfredosum_mask_vs_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv4f16.nxv4i1(
@@ -163,10 +163,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv8f16(
 define <vscale x 4 x half> @intrinsic_vfredosum_vs_nxv4f16_nxv8f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 8 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv4f16_nxv8f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v10, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv8f16.nxv8i1(
 define <vscale x 4 x half> @intrinsic_vfredosum_mask_vs_nxv4f16_nxv8f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 8 x half> %1, <vscale x 4 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv4f16_nxv8f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v10, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv8f16.nxv8i1(
@@ -213,10 +213,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv16f16(
 define <vscale x 4 x half> @intrinsic_vfredosum_vs_nxv4f16_nxv16f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 16 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv4f16_nxv16f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v12, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv16f16.nxv16i1(
 define <vscale x 4 x half> @intrinsic_vfredosum_mask_vs_nxv4f16_nxv16f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 16 x half> %1, <vscale x 4 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv4f16_nxv16f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v12, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv16f16.nxv16i1(
@@ -263,10 +263,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv32f16(
 define <vscale x 4 x half> @intrinsic_vfredosum_vs_nxv4f16_nxv32f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 32 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv4f16_nxv32f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v16, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.nxv4f16.nxv32f16(
@@ -288,10 +288,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv32f16.nxv32i1(
 define <vscale x 4 x half> @intrinsic_vfredosum_mask_vs_nxv4f16_nxv32f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 32 x half> %1, <vscale x 4 x half> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv4f16_nxv32f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v16, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredosum.mask.nxv4f16.nxv32f16.nxv32i1(
@@ -313,10 +313,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredosum.nxv2f32.nxv1f32(
 define <vscale x 2 x float> @intrinsic_vfredosum_vs_nxv2f32_nxv1f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 1 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv2f32_nxv1f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredosum.nxv2f32.nxv1f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredosum.mask.nxv2f32.nxv1f32.nxv1i1(
 define <vscale x 2 x float> @intrinsic_vfredosum_mask_vs_nxv2f32_nxv1f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 1 x float> %1, <vscale x 2 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv2f32_nxv1f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredosum.mask.nxv2f32.nxv1f32.nxv1i1(
@@ -363,10 +363,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredosum.nxv2f32.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfredosum_vs_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredosum.nxv2f32.nxv2f32(
@@ -388,10 +388,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredosum.mask.nxv2f32.nxv2f32.nxv2i1(
 define <vscale x 2 x float> @intrinsic_vfredosum_mask_vs_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredosum.mask.nxv2f32.nxv2f32.nxv2i1(
@@ -413,10 +413,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredosum.nxv2f32.nxv4f32(
 define <vscale x 2 x float> @intrinsic_vfredosum_vs_nxv2f32_nxv4f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 4 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv2f32_nxv4f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v10, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredosum.nxv2f32.nxv4f32(
@@ -438,10 +438,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredosum.mask.nxv2f32.nxv4f32.nxv4i1(
 define <vscale x 2 x float> @intrinsic_vfredosum_mask_vs_nxv2f32_nxv4f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 4 x float> %1, <vscale x 2 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv2f32_nxv4f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v10, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredosum.mask.nxv2f32.nxv4f32.nxv4i1(
@@ -463,10 +463,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredosum.nxv2f32.nxv8f32(
 define <vscale x 2 x float> @intrinsic_vfredosum_vs_nxv2f32_nxv8f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 8 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv2f32_nxv8f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v12, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredosum.nxv2f32.nxv8f32(
@@ -488,10 +488,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredosum.mask.nxv2f32.nxv8f32.nxv8i1(
 define <vscale x 2 x float> @intrinsic_vfredosum_mask_vs_nxv2f32_nxv8f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 8 x float> %1, <vscale x 2 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv2f32_nxv8f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v12, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredosum.mask.nxv2f32.nxv8f32.nxv8i1(
@@ -513,10 +513,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredosum.nxv2f32.nxv16f32(
 define <vscale x 2 x float> @intrinsic_vfredosum_vs_nxv2f32_nxv16f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 16 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv2f32_nxv16f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v16, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredosum.nxv2f32.nxv16f32(
@@ -538,10 +538,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredosum.mask.nxv2f32.nxv16f32.nxv16i1
 define <vscale x 2 x float> @intrinsic_vfredosum_mask_vs_nxv2f32_nxv16f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 16 x float> %1, <vscale x 2 x float> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv2f32_nxv16f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v16, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredosum.mask.nxv2f32.nxv16f32.nxv16i1(
@@ -563,10 +563,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredosum.nxv1f64.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfredosum_vs_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredosum.nxv1f64.nxv1f64(
@@ -588,10 +588,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredosum.mask.nxv1f64.nxv1f64.nxv1i1(
 define <vscale x 1 x double> @intrinsic_vfredosum_mask_vs_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredosum.mask.nxv1f64.nxv1f64.nxv1i1(
@@ -613,10 +613,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredosum.nxv1f64.nxv2f64(
 define <vscale x 1 x double> @intrinsic_vfredosum_vs_nxv1f64_nxv2f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 2 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv1f64_nxv2f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v10, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredosum.nxv1f64.nxv2f64(
@@ -638,10 +638,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredosum.mask.nxv1f64.nxv2f64.nxv2i1(
 define <vscale x 1 x double> @intrinsic_vfredosum_mask_vs_nxv1f64_nxv2f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 2 x double> %1, <vscale x 1 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv1f64_nxv2f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v10, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredosum.mask.nxv1f64.nxv2f64.nxv2i1(
@@ -663,10 +663,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredosum.nxv1f64.nxv4f64(
 define <vscale x 1 x double> @intrinsic_vfredosum_vs_nxv1f64_nxv4f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 4 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv1f64_nxv4f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v12, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredosum.nxv1f64.nxv4f64(
@@ -688,10 +688,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredosum.mask.nxv1f64.nxv4f64.nxv4i1(
 define <vscale x 1 x double> @intrinsic_vfredosum_mask_vs_nxv1f64_nxv4f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 4 x double> %1, <vscale x 1 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv1f64_nxv4f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v12, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredosum.mask.nxv1f64.nxv4f64.nxv4i1(
@@ -713,10 +713,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredosum.nxv1f64.nxv8f64(
 define <vscale x 1 x double> @intrinsic_vfredosum_vs_nxv1f64_nxv8f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 8 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv1f64_nxv8f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v16, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredosum.nxv1f64.nxv8f64(
@@ -738,10 +738,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredosum.mask.nxv1f64.nxv8f64.nxv8i1(
 define <vscale x 1 x double> @intrinsic_vfredosum_mask_vs_nxv1f64_nxv8f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 8 x double> %1, <vscale x 1 x double> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv1f64_nxv8f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredosum.vs v8, v16, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredosum.mask.nxv1f64.nxv8f64.nxv8i1(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfredusum.ll b/llvm/test/CodeGen/RISCV/rvv/vfredusum.ll
index bd2a5a901fb8..ffef9ef728a1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfredusum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfredusum.ll
@@ -13,10 +13,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv1f16(
 define <vscale x 4 x half> @intrinsic_vfredusum_vs_nxv4f16_nxv1f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 1 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv4f16_nxv1f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv1f16.nxv1i1(
 define <vscale x 4 x half> @intrinsic_vfredusum_mask_vs_nxv4f16_nxv1f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 1 x half> %1, <vscale x 4 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv4f16_nxv1f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv1f16.nxv1i1(
@@ -63,10 +63,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv2f16(
 define <vscale x 4 x half> @intrinsic_vfredusum_vs_nxv4f16_nxv2f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 2 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv4f16_nxv2f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv2f16.nxv2i1(
 define <vscale x 4 x half> @intrinsic_vfredusum_mask_vs_nxv4f16_nxv2f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 2 x half> %1, <vscale x 4 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv4f16_nxv2f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv2f16.nxv2i1(
@@ -113,10 +113,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfredusum_vs_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv4f16.nxv4i1(
 define <vscale x 4 x half> @intrinsic_vfredusum_mask_vs_nxv4f16_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv4f16_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv4f16.nxv4i1(
@@ -163,10 +163,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv8f16(
 define <vscale x 4 x half> @intrinsic_vfredusum_vs_nxv4f16_nxv8f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 8 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv4f16_nxv8f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v10, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv8f16.nxv8i1(
 define <vscale x 4 x half> @intrinsic_vfredusum_mask_vs_nxv4f16_nxv8f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 8 x half> %1, <vscale x 4 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv4f16_nxv8f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v10, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv8f16.nxv8i1(
@@ -213,10 +213,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv16f16(
 define <vscale x 4 x half> @intrinsic_vfredusum_vs_nxv4f16_nxv16f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 16 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv4f16_nxv16f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v12, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv16f16.nxv16i1(
 define <vscale x 4 x half> @intrinsic_vfredusum_mask_vs_nxv4f16_nxv16f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 16 x half> %1, <vscale x 4 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv4f16_nxv16f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v12, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv16f16.nxv16i1(
@@ -263,10 +263,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv32f16(
 define <vscale x 4 x half> @intrinsic_vfredusum_vs_nxv4f16_nxv32f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 32 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv4f16_nxv32f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v16, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.nxv4f16.nxv32f16(
@@ -288,10 +288,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv32f16.nxv32i1(
 define <vscale x 4 x half> @intrinsic_vfredusum_mask_vs_nxv4f16_nxv32f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 32 x half> %1, <vscale x 4 x half> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv4f16_nxv32f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v16, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfredusum.mask.nxv4f16.nxv32f16.nxv32i1(
@@ -313,10 +313,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv1f32(
 define <vscale x 2 x float> @intrinsic_vfredusum_vs_nxv2f32_nxv1f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 1 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv2f32_nxv1f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv1f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredusum.mask.nxv2f32.nxv1f32.nxv1i1(
 define <vscale x 2 x float> @intrinsic_vfredusum_mask_vs_nxv2f32_nxv1f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 1 x float> %1, <vscale x 2 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv2f32_nxv1f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.mask.nxv2f32.nxv1f32.nxv1i1(
@@ -363,10 +363,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfredusum_vs_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv2f32(
@@ -388,10 +388,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredusum.mask.nxv2f32.nxv2f32.nxv2i1(
 define <vscale x 2 x float> @intrinsic_vfredusum_mask_vs_nxv2f32_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv2f32_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.mask.nxv2f32.nxv2f32.nxv2i1(
@@ -413,10 +413,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv4f32(
 define <vscale x 2 x float> @intrinsic_vfredusum_vs_nxv2f32_nxv4f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 4 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv2f32_nxv4f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v10, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv4f32(
@@ -438,10 +438,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredusum.mask.nxv2f32.nxv4f32.nxv4i1(
 define <vscale x 2 x float> @intrinsic_vfredusum_mask_vs_nxv2f32_nxv4f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 4 x float> %1, <vscale x 2 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv2f32_nxv4f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v10, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.mask.nxv2f32.nxv4f32.nxv4i1(
@@ -463,10 +463,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv8f32(
 define <vscale x 2 x float> @intrinsic_vfredusum_vs_nxv2f32_nxv8f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 8 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv2f32_nxv8f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v12, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv8f32(
@@ -488,10 +488,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredusum.mask.nxv2f32.nxv8f32.nxv8i1(
 define <vscale x 2 x float> @intrinsic_vfredusum_mask_vs_nxv2f32_nxv8f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 8 x float> %1, <vscale x 2 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv2f32_nxv8f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v12, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.mask.nxv2f32.nxv8f32.nxv8i1(
@@ -513,10 +513,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv16f32(
 define <vscale x 2 x float> @intrinsic_vfredusum_vs_nxv2f32_nxv16f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 16 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv2f32_nxv16f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v16, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.nxv2f32.nxv16f32(
@@ -538,10 +538,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfredusum.mask.nxv2f32.nxv16f32.nxv16i1
 define <vscale x 2 x float> @intrinsic_vfredusum_mask_vs_nxv2f32_nxv16f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 16 x float> %1, <vscale x 2 x float> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv2f32_nxv16f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v16, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfredusum.mask.nxv2f32.nxv16f32.nxv16i1(
@@ -563,10 +563,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredusum.nxv1f64.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfredusum_vs_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredusum.nxv1f64.nxv1f64(
@@ -588,10 +588,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredusum.mask.nxv1f64.nxv1f64.nxv1i1(
 define <vscale x 1 x double> @intrinsic_vfredusum_mask_vs_nxv1f64_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv1f64_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredusum.mask.nxv1f64.nxv1f64.nxv1i1(
@@ -613,10 +613,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredusum.nxv1f64.nxv2f64(
 define <vscale x 1 x double> @intrinsic_vfredusum_vs_nxv1f64_nxv2f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 2 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv1f64_nxv2f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v10, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredusum.nxv1f64.nxv2f64(
@@ -638,10 +638,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredusum.mask.nxv1f64.nxv2f64.nxv2i1(
 define <vscale x 1 x double> @intrinsic_vfredusum_mask_vs_nxv1f64_nxv2f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 2 x double> %1, <vscale x 1 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv1f64_nxv2f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v10, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredusum.mask.nxv1f64.nxv2f64.nxv2i1(
@@ -663,10 +663,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredusum.nxv1f64.nxv4f64(
 define <vscale x 1 x double> @intrinsic_vfredusum_vs_nxv1f64_nxv4f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 4 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv1f64_nxv4f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v12, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredusum.nxv1f64.nxv4f64(
@@ -688,10 +688,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredusum.mask.nxv1f64.nxv4f64.nxv4i1(
 define <vscale x 1 x double> @intrinsic_vfredusum_mask_vs_nxv1f64_nxv4f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 4 x double> %1, <vscale x 1 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv1f64_nxv4f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v12, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredusum.mask.nxv1f64.nxv4f64.nxv4i1(
@@ -713,10 +713,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredusum.nxv1f64.nxv8f64(
 define <vscale x 1 x double> @intrinsic_vfredusum_vs_nxv1f64_nxv8f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 8 x double> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_vs_nxv1f64_nxv8f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v16, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredusum.nxv1f64.nxv8f64(
@@ -738,10 +738,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfredusum.mask.nxv1f64.nxv8f64.nxv8i1(
 define <vscale x 1 x double> @intrinsic_vfredusum_mask_vs_nxv1f64_nxv8f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 8 x double> %1, <vscale x 1 x double> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfredusum_mask_vs_nxv1f64_nxv8f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfredusum.vs v8, v16, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfredusum.mask.nxv1f64.nxv8f64.nxv8i1(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfsqrt.ll b/llvm/test/CodeGen/RISCV/rvv/vfsqrt.ll
index 0f61e6a7d406..3e3eea9f353c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfsqrt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfsqrt.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfsqrt.nxv1f16(
 define <vscale x 1 x half> @intrinsic_vfsqrt_v_nxv1f16_nxv1f16(<vscale x 1 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfsqrt.nxv1f16(
@@ -35,10 +35,10 @@ declare <vscale x 1 x half> @llvm.riscv.vfsqrt.mask.nxv1f16(
 define <vscale x 1 x half> @intrinsic_vfsqrt_mask_v_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x half> @llvm.riscv.vfsqrt.mask.nxv1f16(
@@ -58,10 +58,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfsqrt.nxv2f16(
 define <vscale x 2 x half> @intrinsic_vfsqrt_v_nxv2f16_nxv2f16(<vscale x 2 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfsqrt.nxv2f16(
@@ -81,10 +81,10 @@ declare <vscale x 2 x half> @llvm.riscv.vfsqrt.mask.nxv2f16(
 define <vscale x 2 x half> @intrinsic_vfsqrt_mask_v_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x half> @llvm.riscv.vfsqrt.mask.nxv2f16(
@@ -104,10 +104,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfsqrt.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfsqrt_v_nxv4f16_nxv4f16(<vscale x 4 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfsqrt.nxv4f16(
@@ -127,10 +127,10 @@ declare <vscale x 4 x half> @llvm.riscv.vfsqrt.mask.nxv4f16(
 define <vscale x 4 x half> @intrinsic_vfsqrt_mask_v_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x half> @llvm.riscv.vfsqrt.mask.nxv4f16(
@@ -150,10 +150,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfsqrt.nxv8f16(
 define <vscale x 8 x half> @intrinsic_vfsqrt_v_nxv8f16_nxv8f16(<vscale x 8 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfsqrt.nxv8f16(
@@ -173,10 +173,10 @@ declare <vscale x 8 x half> @llvm.riscv.vfsqrt.mask.nxv8f16(
 define <vscale x 8 x half> @intrinsic_vfsqrt_mask_v_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x half> @llvm.riscv.vfsqrt.mask.nxv8f16(
@@ -196,10 +196,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfsqrt.nxv16f16(
 define <vscale x 16 x half> @intrinsic_vfsqrt_v_nxv16f16_nxv16f16(<vscale x 16 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfsqrt.nxv16f16(
@@ -219,10 +219,10 @@ declare <vscale x 16 x half> @llvm.riscv.vfsqrt.mask.nxv16f16(
 define <vscale x 16 x half> @intrinsic_vfsqrt_mask_v_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x half> @llvm.riscv.vfsqrt.mask.nxv16f16(
@@ -242,10 +242,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfsqrt.nxv32f16(
 define <vscale x 32 x half> @intrinsic_vfsqrt_v_nxv32f16_nxv32f16(<vscale x 32 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv32f16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfsqrt.nxv32f16(
@@ -265,10 +265,10 @@ declare <vscale x 32 x half> @llvm.riscv.vfsqrt.mask.nxv32f16(
 define <vscale x 32 x half> @intrinsic_vfsqrt_mask_v_nxv32f16_nxv32f16(<vscale x 32 x half> %0, <vscale x 32 x half> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv32f16_nxv32f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 32 x half> @llvm.riscv.vfsqrt.mask.nxv32f16(
@@ -288,10 +288,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfsqrt.nxv1f32(
 define <vscale x 1 x float> @intrinsic_vfsqrt_v_nxv1f32_nxv1f32(<vscale x 1 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfsqrt.nxv1f32(
@@ -311,10 +311,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfsqrt.mask.nxv1f32(
 define <vscale x 1 x float> @intrinsic_vfsqrt_mask_v_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfsqrt.mask.nxv1f32(
@@ -334,10 +334,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfsqrt.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfsqrt_v_nxv2f32_nxv2f32(<vscale x 2 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfsqrt.nxv2f32(
@@ -357,10 +357,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfsqrt.mask.nxv2f32(
 define <vscale x 2 x float> @intrinsic_vfsqrt_mask_v_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfsqrt.mask.nxv2f32(
@@ -380,10 +380,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfsqrt.nxv4f32(
 define <vscale x 4 x float> @intrinsic_vfsqrt_v_nxv4f32_nxv4f32(<vscale x 4 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfsqrt.nxv4f32(
@@ -403,10 +403,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfsqrt.mask.nxv4f32(
 define <vscale x 4 x float> @intrinsic_vfsqrt_mask_v_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfsqrt.mask.nxv4f32(
@@ -426,10 +426,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfsqrt.nxv8f32(
 define <vscale x 8 x float> @intrinsic_vfsqrt_v_nxv8f32_nxv8f32(<vscale x 8 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfsqrt.nxv8f32(
@@ -449,10 +449,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfsqrt.mask.nxv8f32(
 define <vscale x 8 x float> @intrinsic_vfsqrt_mask_v_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfsqrt.mask.nxv8f32(
@@ -472,10 +472,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfsqrt.nxv16f32(
 define <vscale x 16 x float> @intrinsic_vfsqrt_v_nxv16f32_nxv16f32(<vscale x 16 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv16f32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfsqrt.nxv16f32(
@@ -495,10 +495,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfsqrt.mask.nxv16f32(
 define <vscale x 16 x float> @intrinsic_vfsqrt_mask_v_nxv16f32_nxv16f32(<vscale x 16 x float> %0, <vscale x 16 x float> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv16f32_nxv16f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfsqrt.mask.nxv16f32(
@@ -518,10 +518,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfsqrt.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfsqrt_v_nxv1f64_nxv1f64(<vscale x 1 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfsqrt.nxv1f64(
@@ -541,10 +541,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfsqrt.mask.nxv1f64(
 define <vscale x 1 x double> @intrinsic_vfsqrt_mask_v_nxv1f64_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv1f64_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfsqrt.mask.nxv1f64(
@@ -564,10 +564,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfsqrt.nxv2f64(
 define <vscale x 2 x double> @intrinsic_vfsqrt_v_nxv2f64_nxv2f64(<vscale x 2 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfsqrt.nxv2f64(
@@ -587,10 +587,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfsqrt.mask.nxv2f64(
 define <vscale x 2 x double> @intrinsic_vfsqrt_mask_v_nxv2f64_nxv2f64(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv2f64_nxv2f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfsqrt.mask.nxv2f64(
@@ -610,10 +610,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfsqrt.nxv4f64(
 define <vscale x 4 x double> @intrinsic_vfsqrt_v_nxv4f64_nxv4f64(<vscale x 4 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfsqrt.nxv4f64(
@@ -633,10 +633,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfsqrt.mask.nxv4f64(
 define <vscale x 4 x double> @intrinsic_vfsqrt_mask_v_nxv4f64_nxv4f64(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv4f64_nxv4f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfsqrt.mask.nxv4f64(
@@ -656,10 +656,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfsqrt.nxv8f64(
 define <vscale x 8 x double> @intrinsic_vfsqrt_v_nxv8f64_nxv8f64(<vscale x 8 x double> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv8f64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfsqrt.nxv8f64(
@@ -679,10 +679,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfsqrt.mask.nxv8f64(
 define <vscale x 8 x double> @intrinsic_vfsqrt_mask_v_nxv8f64_nxv8f64(<vscale x 8 x double> %0, <vscale x 8 x double> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv8f64_nxv8f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfsqrt.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfsqrt.mask.nxv8f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwadd.ll b/llvm/test/CodeGen/RISCV/rvv/vfwadd.ll
index cb7047be9753..b42a1fe46e67 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwadd.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwadd.nxv1f32.nxv1f16.nxv1f16(
 define <vscale x 1 x float> @intrinsic_vfwadd_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -39,10 +39,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwadd.mask.nxv1f32.nxv1f16.nxv1f16(
 define <vscale x 1 x float> @intrinsic_vfwadd_mask_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwadd.mask.nxv1f32.nxv1f16.nxv1f16(
@@ -64,10 +64,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwadd.nxv2f32.nxv2f16.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwadd_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -90,10 +90,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwadd.mask.nxv2f32.nxv2f16.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwadd_mask_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwadd.mask.nxv2f32.nxv2f16.nxv2f16(
@@ -115,10 +115,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwadd.nxv4f32.nxv4f16.nxv4f16(
 define <vscale x 4 x float> @intrinsic_vfwadd_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -141,10 +141,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwadd.mask.nxv4f32.nxv4f16.nxv4f16(
 define <vscale x 4 x float> @intrinsic_vfwadd_mask_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwadd.mask.nxv4f32.nxv4f16.nxv4f16(
@@ -166,10 +166,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwadd.nxv8f32.nxv8f16.nxv8f16(
 define <vscale x 8 x float> @intrinsic_vfwadd_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v12, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -192,10 +192,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwadd.mask.nxv8f32.nxv8f16.nxv8f16(
 define <vscale x 8 x float> @intrinsic_vfwadd_mask_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwadd.mask.nxv8f32.nxv8f16.nxv8f16(
@@ -217,10 +217,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwadd.nxv16f32.nxv16f16.nxv16f16(
 define <vscale x 16 x float> @intrinsic_vfwadd_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v16, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -243,10 +243,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwadd.mask.nxv16f32.nxv16f16.nxv16f16
 define <vscale x 16 x float> @intrinsic_vfwadd_mask_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwadd.mask.nxv16f32.nxv16f16.nxv16f16(
@@ -268,10 +268,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwadd.nxv1f64.nxv1f32.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwadd_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -294,10 +294,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwadd.mask.nxv1f64.nxv1f32.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwadd_mask_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwadd.mask.nxv1f64.nxv1f32.nxv1f32(
@@ -319,10 +319,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwadd.nxv2f64.nxv2f32.nxv2f32(
 define <vscale x 2 x double> @intrinsic_vfwadd_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -345,10 +345,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwadd.mask.nxv2f64.nxv2f32.nxv2f32(
 define <vscale x 2 x double> @intrinsic_vfwadd_mask_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwadd.mask.nxv2f64.nxv2f32.nxv2f32(
@@ -370,10 +370,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwadd.nxv4f64.nxv4f32.nxv4f32(
 define <vscale x 4 x double> @intrinsic_vfwadd_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v12, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -396,10 +396,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwadd.mask.nxv4f64.nxv4f32.nxv4f32(
 define <vscale x 4 x double> @intrinsic_vfwadd_mask_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwadd.mask.nxv4f64.nxv4f32.nxv4f32(
@@ -421,10 +421,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwadd.nxv8f64.nxv8f32.nxv8f32(
 define <vscale x 8 x double> @intrinsic_vfwadd_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v16, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -447,10 +447,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwadd.mask.nxv8f64.nxv8f32.nxv8f32(
 define <vscale x 8 x double> @intrinsic_vfwadd_mask_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwadd.mask.nxv8f64.nxv8f32.nxv8f32(
@@ -472,10 +472,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwadd.nxv1f32.nxv1f16.f16(
 define <vscale x 1 x float> @intrinsic_vfwadd_vf_nxv1f32_nxv1f16_f16(<vscale x 1 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv1f32_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -498,10 +498,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwadd.mask.nxv1f32.nxv1f16.f16(
 define <vscale x 1 x float> @intrinsic_vfwadd_mask_vf_nxv1f32_nxv1f16_f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv1f32_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwadd.mask.nxv1f32.nxv1f16.f16(
@@ -523,10 +523,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwadd.nxv2f32.nxv2f16.f16(
 define <vscale x 2 x float> @intrinsic_vfwadd_vf_nxv2f32_nxv2f16_f16(<vscale x 2 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv2f32_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -549,10 +549,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwadd.mask.nxv2f32.nxv2f16.f16(
 define <vscale x 2 x float> @intrinsic_vfwadd_mask_vf_nxv2f32_nxv2f16_f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv2f32_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwadd.mask.nxv2f32.nxv2f16.f16(
@@ -574,10 +574,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwadd.nxv4f32.nxv4f16.f16(
 define <vscale x 4 x float> @intrinsic_vfwadd_vf_nxv4f32_nxv4f16_f16(<vscale x 4 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv4f32_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v10, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -600,10 +600,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwadd.mask.nxv4f32.nxv4f16.f16(
 define <vscale x 4 x float> @intrinsic_vfwadd_mask_vf_nxv4f32_nxv4f16_f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv4f32_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwadd.mask.nxv4f32.nxv4f16.f16(
@@ -625,10 +625,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwadd.nxv8f32.nxv8f16.f16(
 define <vscale x 8 x float> @intrinsic_vfwadd_vf_nxv8f32_nxv8f16_f16(<vscale x 8 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv8f32_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v12, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -651,10 +651,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwadd.mask.nxv8f32.nxv8f16.f16(
 define <vscale x 8 x float> @intrinsic_vfwadd_mask_vf_nxv8f32_nxv8f16_f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv8f32_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwadd.mask.nxv8f32.nxv8f16.f16(
@@ -676,10 +676,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwadd.nxv16f32.nxv16f16.f16(
 define <vscale x 16 x float> @intrinsic_vfwadd_vf_nxv16f32_nxv16f16_f16(<vscale x 16 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv16f32_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v16, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -702,10 +702,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwadd.mask.nxv16f32.nxv16f16.f16(
 define <vscale x 16 x float> @intrinsic_vfwadd_mask_vf_nxv16f32_nxv16f16_f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv16f32_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwadd.mask.nxv16f32.nxv16f16.f16(
@@ -727,10 +727,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwadd.nxv1f64.nxv1f32.f32(
 define <vscale x 1 x double> @intrinsic_vfwadd_vf_nxv1f64_nxv1f32_f32(<vscale x 1 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv1f64_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -753,10 +753,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwadd.mask.nxv1f64.nxv1f32.f32(
 define <vscale x 1 x double> @intrinsic_vfwadd_mask_vf_nxv1f64_nxv1f32_f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv1f64_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwadd.mask.nxv1f64.nxv1f32.f32(
@@ -778,10 +778,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwadd.nxv2f64.nxv2f32.f32(
 define <vscale x 2 x double> @intrinsic_vfwadd_vf_nxv2f64_nxv2f32_f32(<vscale x 2 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv2f64_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v10, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -804,10 +804,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwadd.mask.nxv2f64.nxv2f32.f32(
 define <vscale x 2 x double> @intrinsic_vfwadd_mask_vf_nxv2f64_nxv2f32_f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv2f64_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwadd.mask.nxv2f64.nxv2f32.f32(
@@ -829,10 +829,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwadd.nxv4f64.nxv4f32.f32(
 define <vscale x 4 x double> @intrinsic_vfwadd_vf_nxv4f64_nxv4f32_f32(<vscale x 4 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv4f64_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v12, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -855,10 +855,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwadd.mask.nxv4f64.nxv4f32.f32(
 define <vscale x 4 x double> @intrinsic_vfwadd_mask_vf_nxv4f64_nxv4f32_f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv4f64_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwadd.mask.nxv4f64.nxv4f32.f32(
@@ -880,10 +880,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwadd.nxv8f64.nxv8f32.f32(
 define <vscale x 8 x double> @intrinsic_vfwadd_vf_nxv8f64_nxv8f32_f32(<vscale x 8 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv8f64_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v16, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -906,10 +906,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwadd.mask.nxv8f64.nxv8f32.f32(
 define <vscale x 8 x double> @intrinsic_vfwadd_mask_vf_nxv8f64_nxv8f32_f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv8f64_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwadd.mask.nxv8f64.nxv8f32.f32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwadd.w.ll b/llvm/test/CodeGen/RISCV/rvv/vfwadd.w.ll
index 2a318c53a5fb..76246eba9480 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwadd.w.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwadd.w.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwadd.w.nxv1f32.nxv1f16(
 define <vscale x 1 x float> @intrinsic_vfwadd.w_wv_nxv1f32_nxv1f32_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv1f32_nxv1f32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwadd.w.nxv1f32.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwadd.w.mask.nxv1f32.nxv1f16(
 define <vscale x 1 x float> @intrinsic_vfwadd.w_mask_wv_nxv1f32_nxv1f32_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv1f32_nxv1f32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwadd.w.mask.nxv1f32.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwadd.w.nxv2f32.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwadd.w_wv_nxv2f32_nxv2f32_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv2f32_nxv2f32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwadd.w.nxv2f32.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwadd.w.mask.nxv2f32.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwadd.w_mask_wv_nxv2f32_nxv2f32_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv2f32_nxv2f32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwadd.w.mask.nxv2f32.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwadd.w.nxv4f32.nxv4f16(
 define <vscale x 4 x float> @intrinsic_vfwadd.w_wv_nxv4f32_nxv4f32_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv4f32_nxv4f32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwadd.w.nxv4f32.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwadd.w.mask.nxv4f32.nxv4f16(
 define <vscale x 4 x float> @intrinsic_vfwadd.w_mask_wv_nxv4f32_nxv4f32_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv4f32_nxv4f32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwadd.w.mask.nxv4f32.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwadd.w.nxv8f32.nxv8f16(
 define <vscale x 8 x float> @intrinsic_vfwadd.w_wv_nxv8f32_nxv8f32_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv8f32_nxv8f32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwadd.w.nxv8f32.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwadd.w.mask.nxv8f32.nxv8f16(
 define <vscale x 8 x float> @intrinsic_vfwadd.w_mask_wv_nxv8f32_nxv8f32_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv8f32_nxv8f32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwadd.w.mask.nxv8f32.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwadd.w.nxv16f32.nxv16f16(
 define <vscale x 16 x float> @intrinsic_vfwadd.w_wv_nxv16f32_nxv16f32_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv16f32_nxv16f32_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwadd.w.nxv16f32.nxv16f16(
@@ -239,8 +239,8 @@ define <vscale x 16 x float> @intrinsic_vfwadd.w_mask_wv_nxv16f32_nxv16f32_nxv16
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv16f32_nxv16f32_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl4re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vfwadd.wv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -264,10 +264,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwadd.w.nxv1f64.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwadd.w_wv_nxv1f64_nxv1f64_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv1f64_nxv1f64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwadd.w.nxv1f64.nxv1f32(
@@ -289,10 +289,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwadd.w.mask.nxv1f64.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwadd.w_mask_wv_nxv1f64_nxv1f64_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv1f64_nxv1f64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwadd.w.mask.nxv1f64.nxv1f32(
@@ -314,10 +314,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwadd.w.nxv2f64.nxv2f32(
 define <vscale x 2 x double> @intrinsic_vfwadd.w_wv_nxv2f64_nxv2f64_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv2f64_nxv2f64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwadd.w.nxv2f64.nxv2f32(
@@ -339,10 +339,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwadd.w.mask.nxv2f64.nxv2f32(
 define <vscale x 2 x double> @intrinsic_vfwadd.w_mask_wv_nxv2f64_nxv2f64_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv2f64_nxv2f64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwadd.w.mask.nxv2f64.nxv2f32(
@@ -364,10 +364,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwadd.w.nxv4f64.nxv4f32(
 define <vscale x 4 x double> @intrinsic_vfwadd.w_wv_nxv4f64_nxv4f64_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv4f64_nxv4f64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwadd.w.nxv4f64.nxv4f32(
@@ -389,10 +389,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwadd.w.mask.nxv4f64.nxv4f32(
 define <vscale x 4 x double> @intrinsic_vfwadd.w_mask_wv_nxv4f64_nxv4f64_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv4f64_nxv4f64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwadd.w.mask.nxv4f64.nxv4f32(
@@ -414,10 +414,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwadd.w.nxv8f64.nxv8f32(
 define <vscale x 8 x double> @intrinsic_vfwadd.w_wv_nxv8f64_nxv8f64_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv8f64_nxv8f64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwadd.w.nxv8f64.nxv8f32(
@@ -440,8 +440,8 @@ define <vscale x 8 x double> @intrinsic_vfwadd.w_mask_wv_nxv8f64_nxv8f64_nxv8f32
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv8f64_nxv8f64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl4re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vfwadd.wv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -465,10 +465,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwadd.w.nxv1f32.f16(
 define <vscale x 1 x float> @intrinsic_vfwadd.w_wf_nxv1f32_nxv1f32_f16(<vscale x 1 x float> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv1f32_nxv1f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwadd.w.nxv1f32.f16(
@@ -490,10 +490,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwadd.w.mask.nxv1f32.f16(
 define <vscale x 1 x float> @intrinsic_vfwadd.w_mask_wf_nxv1f32_nxv1f32_f16(<vscale x 1 x float> %0, <vscale x 1 x float> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv1f32_nxv1f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwadd.w.mask.nxv1f32.f16(
@@ -515,10 +515,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwadd.w.nxv2f32.f16(
 define <vscale x 2 x float> @intrinsic_vfwadd.w_wf_nxv2f32_nxv2f32_f16(<vscale x 2 x float> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv2f32_nxv2f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwadd.w.nxv2f32.f16(
@@ -540,10 +540,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwadd.w.mask.nxv2f32.f16(
 define <vscale x 2 x float> @intrinsic_vfwadd.w_mask_wf_nxv2f32_nxv2f32_f16(<vscale x 2 x float> %0, <vscale x 2 x float> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv2f32_nxv2f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwadd.w.mask.nxv2f32.f16(
@@ -565,10 +565,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwadd.w.nxv4f32.f16(
 define <vscale x 4 x float> @intrinsic_vfwadd.w_wf_nxv4f32_nxv4f32_f16(<vscale x 4 x float> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv4f32_nxv4f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwadd.w.nxv4f32.f16(
@@ -590,10 +590,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwadd.w.mask.nxv4f32.f16(
 define <vscale x 4 x float> @intrinsic_vfwadd.w_mask_wf_nxv4f32_nxv4f32_f16(<vscale x 4 x float> %0, <vscale x 4 x float> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv4f32_nxv4f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwadd.w.mask.nxv4f32.f16(
@@ -615,10 +615,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwadd.w.nxv8f32.f16(
 define <vscale x 8 x float> @intrinsic_vfwadd.w_wf_nxv8f32_nxv8f32_f16(<vscale x 8 x float> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv8f32_nxv8f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwadd.w.nxv8f32.f16(
@@ -640,10 +640,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwadd.w.mask.nxv8f32.f16(
 define <vscale x 8 x float> @intrinsic_vfwadd.w_mask_wf_nxv8f32_nxv8f32_f16(<vscale x 8 x float> %0, <vscale x 8 x float> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv8f32_nxv8f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwadd.w.mask.nxv8f32.f16(
@@ -665,10 +665,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwadd.w.nxv16f32.f16(
 define <vscale x 16 x float> @intrinsic_vfwadd.w_wf_nxv16f32_nxv16f32_f16(<vscale x 16 x float> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv16f32_nxv16f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwadd.w.nxv16f32.f16(
@@ -690,10 +690,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwadd.w.mask.nxv16f32.f16(
 define <vscale x 16 x float> @intrinsic_vfwadd.w_mask_wf_nxv16f32_nxv16f32_f16(<vscale x 16 x float> %0, <vscale x 16 x float> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv16f32_nxv16f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwadd.w.mask.nxv16f32.f16(
@@ -715,10 +715,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwadd.w.nxv1f64.f32(
 define <vscale x 1 x double> @intrinsic_vfwadd.w_wf_nxv1f64_nxv1f64_f32(<vscale x 1 x double> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv1f64_nxv1f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwadd.w.nxv1f64.f32(
@@ -740,10 +740,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwadd.w.mask.nxv1f64.f32(
 define <vscale x 1 x double> @intrinsic_vfwadd.w_mask_wf_nxv1f64_nxv1f64_f32(<vscale x 1 x double> %0, <vscale x 1 x double> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv1f64_nxv1f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwadd.w.mask.nxv1f64.f32(
@@ -765,10 +765,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwadd.w.nxv2f64.f32(
 define <vscale x 2 x double> @intrinsic_vfwadd.w_wf_nxv2f64_nxv2f64_f32(<vscale x 2 x double> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv2f64_nxv2f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwadd.w.nxv2f64.f32(
@@ -790,10 +790,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwadd.w.mask.nxv2f64.f32(
 define <vscale x 2 x double> @intrinsic_vfwadd.w_mask_wf_nxv2f64_nxv2f64_f32(<vscale x 2 x double> %0, <vscale x 2 x double> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv2f64_nxv2f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwadd.w.mask.nxv2f64.f32(
@@ -815,10 +815,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwadd.w.nxv4f64.f32(
 define <vscale x 4 x double> @intrinsic_vfwadd.w_wf_nxv4f64_nxv4f64_f32(<vscale x 4 x double> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv4f64_nxv4f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwadd.w.nxv4f64.f32(
@@ -840,10 +840,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwadd.w.mask.nxv4f64.f32(
 define <vscale x 4 x double> @intrinsic_vfwadd.w_mask_wf_nxv4f64_nxv4f64_f32(<vscale x 4 x double> %0, <vscale x 4 x double> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv4f64_nxv4f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwadd.w.mask.nxv4f64.f32(
@@ -865,10 +865,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwadd.w.nxv8f64.f32(
 define <vscale x 8 x double> @intrinsic_vfwadd.w_wf_nxv8f64_nxv8f64_f32(<vscale x 8 x double> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv8f64_nxv8f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwadd.w.nxv8f64.f32(
@@ -890,10 +890,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwadd.w.mask.nxv8f64.f32(
 define <vscale x 8 x double> @intrinsic_vfwadd.w_mask_wf_nxv8f64_nxv8f64_f32(<vscale x 8 x double> %0, <vscale x 8 x double> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv8f64_nxv8f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwadd.w.mask.nxv8f64.f32(
@@ -909,10 +909,10 @@ entry:
 define <vscale x 1 x float> @intrinsic_vfwadd.w_mask_wv_tie_nxv1f32_nxv1f32_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_tie_nxv1f32_nxv1f32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwadd.w.mask.nxv1f32.nxv1f16(
@@ -928,10 +928,10 @@ entry:
 define <vscale x 2 x float> @intrinsic_vfwadd.w_mask_wv_tie_nxv2f32_nxv2f32_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_tie_nxv2f32_nxv2f32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwadd.w.mask.nxv2f32.nxv2f16(
@@ -947,10 +947,10 @@ entry:
 define <vscale x 4 x float> @intrinsic_vfwadd.w_mask_wv_tie_nxv4f32_nxv4f32_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_tie_nxv4f32_nxv4f32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwadd.w.mask.nxv4f32.nxv4f16(
@@ -966,10 +966,10 @@ entry:
 define <vscale x 8 x float> @intrinsic_vfwadd.w_mask_wv_tie_nxv8f32_nxv8f32_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_tie_nxv8f32_nxv8f32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwadd.w.mask.nxv8f32.nxv8f16(
@@ -985,10 +985,10 @@ entry:
 define <vscale x 16 x float> @intrinsic_vfwadd.w_mask_wv_tie_nxv16f32_nxv16f32_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_tie_nxv16f32_nxv16f32_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwadd.w.mask.nxv16f32.nxv16f16(
@@ -1004,10 +1004,10 @@ entry:
 define <vscale x 1 x double> @intrinsic_vfwadd.w_mask_wv_tie_nxv1f64_nxv1f64_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_tie_nxv1f64_nxv1f64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwadd.w.mask.nxv1f64.nxv1f32(
@@ -1023,10 +1023,10 @@ entry:
 define <vscale x 2 x double> @intrinsic_vfwadd.w_mask_wv_tie_nxv2f64_nxv2f64_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_tie_nxv2f64_nxv2f64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwadd.w.mask.nxv2f64.nxv2f32(
@@ -1042,10 +1042,10 @@ entry:
 define <vscale x 4 x double> @intrinsic_vfwadd.w_mask_wv_tie_nxv4f64_nxv4f64_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_tie_nxv4f64_nxv4f64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwadd.w.mask.nxv4f64.nxv4f32(
@@ -1061,10 +1061,10 @@ entry:
 define <vscale x 8 x double> @intrinsic_vfwadd.w_mask_wv_tie_nxv8f64_nxv8f64_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_tie_nxv8f64_nxv8f64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v8, v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwadd.w.mask.nxv8f64.nxv8f32(
@@ -1080,10 +1080,10 @@ entry:
 define <vscale x 1 x float> @intrinsic_vfwadd.w_mask_wf_tie_nxv1f32_nxv1f32_f16(<vscale x 1 x float> %0, half %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_tie_nxv1f32_nxv1f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwadd.w.mask.nxv1f32.f16(
@@ -1099,10 +1099,10 @@ entry:
 define <vscale x 2 x float> @intrinsic_vfwadd.w_mask_wf_tie_nxv2f32_nxv2f32_f16(<vscale x 2 x float> %0, half %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_tie_nxv2f32_nxv2f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwadd.w.mask.nxv2f32.f16(
@@ -1118,10 +1118,10 @@ entry:
 define <vscale x 4 x float> @intrinsic_vfwadd.w_mask_wf_tie_nxv4f32_nxv4f32_f16(<vscale x 4 x float> %0, half %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_tie_nxv4f32_nxv4f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwadd.w.mask.nxv4f32.f16(
@@ -1137,10 +1137,10 @@ entry:
 define <vscale x 8 x float> @intrinsic_vfwadd.w_mask_wf_tie_nxv8f32_nxv8f32_f16(<vscale x 8 x float> %0, half %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_tie_nxv8f32_nxv8f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwadd.w.mask.nxv8f32.f16(
@@ -1156,10 +1156,10 @@ entry:
 define <vscale x 16 x float> @intrinsic_vfwadd.w_mask_wf_tie_nxv16f32_nxv16f32_f16(<vscale x 16 x float> %0, half %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_tie_nxv16f32_nxv16f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwadd.w.mask.nxv16f32.f16(
@@ -1175,10 +1175,10 @@ entry:
 define <vscale x 1 x double> @intrinsic_vfwadd.w_mask_wf_tie_nxv1f64_nxv1f64_f32(<vscale x 1 x double> %0, float %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_tie_nxv1f64_nxv1f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwadd.w.mask.nxv1f64.f32(
@@ -1194,10 +1194,10 @@ entry:
 define <vscale x 2 x double> @intrinsic_vfwadd.w_mask_wf_tie_nxv2f64_nxv2f64_f32(<vscale x 2 x double> %0, float %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_tie_nxv2f64_nxv2f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwadd.w.mask.nxv2f64.f32(
@@ -1213,10 +1213,10 @@ entry:
 define <vscale x 4 x double> @intrinsic_vfwadd.w_mask_wf_tie_nxv4f64_nxv4f64_f32(<vscale x 4 x double> %0, float %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_tie_nxv4f64_nxv4f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwadd.w.mask.nxv4f64.f32(
@@ -1232,10 +1232,10 @@ entry:
 define <vscale x 8 x double> @intrinsic_vfwadd.w_mask_wf_tie_nxv8f64_nxv8f64_f32(<vscale x 8 x double> %0, float %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_tie_nxv8f64_nxv8f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwadd.w.mask.nxv8f64.f32(
@@ -1251,10 +1251,10 @@ entry:
 define <vscale x 1 x float> @intrinsic_vfwadd.w_wv_untie_nxv1f32_nxv1f32_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_untie_nxv1f32_nxv1f32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v10, v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -1270,10 +1270,10 @@ entry:
 define <vscale x 2 x float> @intrinsic_vfwadd.w_wv_untie_nxv2f32_nxv2f32_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_untie_nxv2f32_nxv2f32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v10, v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -1289,10 +1289,10 @@ entry:
 define <vscale x 4 x float> @intrinsic_vfwadd.w_wv_untie_nxv4f32_nxv4f32_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_untie_nxv4f32_nxv4f32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v12, v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -1308,10 +1308,10 @@ entry:
 define <vscale x 8 x float> @intrinsic_vfwadd.w_wv_untie_nxv8f32_nxv8f32_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_untie_nxv8f32_nxv8f32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v16, v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -1327,10 +1327,10 @@ entry:
 define <vscale x 1 x double> @intrinsic_vfwadd.w_wv_untie_nxv1f64_nxv1f64_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_untie_nxv1f64_nxv1f64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v10, v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -1346,10 +1346,10 @@ entry:
 define <vscale x 2 x double> @intrinsic_vfwadd.w_wv_untie_nxv2f64_nxv2f64_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_untie_nxv2f64_nxv2f64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v12, v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -1365,10 +1365,10 @@ entry:
 define <vscale x 4 x double> @intrinsic_vfwadd.w_wv_untie_nxv4f64_nxv4f64_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_untie_nxv4f64_nxv4f64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v16, v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -1384,10 +1384,10 @@ entry:
 define <vscale x 8 x double> @intrinsic_vfwadd.w_wv_untie_nxv8f64_nxv8f64_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_untie_nxv8f64_nxv8f64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwadd.wv v24, v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v24
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-x-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-x-f.ll
index ba7ba4e4c2bb..23b10250dfa4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-x-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-x-f.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfwcvt.x.f.v.nxv1i32.nxv1f16(
 define <vscale x 1 x i32> @intrinsic_vfwcvt_x.f.v_nxv1i32_nxv1f16(<vscale x 1 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f.v_nxv1i32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -36,10 +36,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfwcvt.x.f.v.mask.nxv1i32.nxv1f16(
 define <vscale x 1 x i32> @intrinsic_vfwcvt_mask_x.f.v_nxv1i32_nxv1f16(<vscale x 1 x i32> %0, <vscale x 1 x half> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f.v_nxv1i32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vfwcvt.x.f.v.mask.nxv1i32.nxv1f16(
@@ -59,10 +59,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfwcvt.x.f.v.nxv2i32.nxv2f16(
 define <vscale x 2 x i32> @intrinsic_vfwcvt_x.f.v_nxv2i32_nxv2f16(<vscale x 2 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f.v_nxv2i32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -83,10 +83,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfwcvt.x.f.v.mask.nxv2i32.nxv2f16(
 define <vscale x 2 x i32> @intrinsic_vfwcvt_mask_x.f.v_nxv2i32_nxv2f16(<vscale x 2 x i32> %0, <vscale x 2 x half> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f.v_nxv2i32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vfwcvt.x.f.v.mask.nxv2i32.nxv2f16(
@@ -106,10 +106,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfwcvt.x.f.v.nxv4i32.nxv4f16(
 define <vscale x 4 x i32> @intrinsic_vfwcvt_x.f.v_nxv4i32_nxv4f16(<vscale x 4 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f.v_nxv4i32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -130,10 +130,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfwcvt.x.f.v.mask.nxv4i32.nxv4f16(
 define <vscale x 4 x i32> @intrinsic_vfwcvt_mask_x.f.v_nxv4i32_nxv4f16(<vscale x 4 x i32> %0, <vscale x 4 x half> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f.v_nxv4i32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vfwcvt.x.f.v.mask.nxv4i32.nxv4f16(
@@ -153,10 +153,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfwcvt.x.f.v.nxv8i32.nxv8f16(
 define <vscale x 8 x i32> @intrinsic_vfwcvt_x.f.v_nxv8i32_nxv8f16(<vscale x 8 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f.v_nxv8i32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -177,10 +177,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfwcvt.x.f.v.mask.nxv8i32.nxv8f16(
 define <vscale x 8 x i32> @intrinsic_vfwcvt_mask_x.f.v_nxv8i32_nxv8f16(<vscale x 8 x i32> %0, <vscale x 8 x half> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f.v_nxv8i32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vfwcvt.x.f.v.mask.nxv8i32.nxv8f16(
@@ -200,10 +200,10 @@ declare <vscale x 16 x i32> @llvm.riscv.vfwcvt.x.f.v.nxv16i32.nxv16f16(
 define <vscale x 16 x i32> @intrinsic_vfwcvt_x.f.v_nxv16i32_nxv16f16(<vscale x 16 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f.v_nxv16i32_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -224,10 +224,10 @@ declare <vscale x 16 x i32> @llvm.riscv.vfwcvt.x.f.v.mask.nxv16i32.nxv16f16(
 define <vscale x 16 x i32> @intrinsic_vfwcvt_mask_x.f.v_nxv16i32_nxv16f16(<vscale x 16 x i32> %0, <vscale x 16 x half> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f.v_nxv16i32_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vfwcvt.x.f.v.mask.nxv16i32.nxv16f16(
@@ -247,10 +247,10 @@ declare <vscale x 1 x i64> @llvm.riscv.vfwcvt.x.f.v.nxv1i64.nxv1f32(
 define <vscale x 1 x i64> @intrinsic_vfwcvt_x.f.v_nxv1i64_nxv1f32(<vscale x 1 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f.v_nxv1i64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -271,10 +271,10 @@ declare <vscale x 1 x i64> @llvm.riscv.vfwcvt.x.f.v.mask.nxv1i64.nxv1f32(
 define <vscale x 1 x i64> @intrinsic_vfwcvt_mask_x.f.v_nxv1i64_nxv1f32(<vscale x 1 x i64> %0, <vscale x 1 x float> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f.v_nxv1i64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vfwcvt.x.f.v.mask.nxv1i64.nxv1f32(
@@ -294,10 +294,10 @@ declare <vscale x 2 x i64> @llvm.riscv.vfwcvt.x.f.v.nxv2i64.nxv2f32(
 define <vscale x 2 x i64> @intrinsic_vfwcvt_x.f.v_nxv2i64_nxv2f32(<vscale x 2 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f.v_nxv2i64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -318,10 +318,10 @@ declare <vscale x 2 x i64> @llvm.riscv.vfwcvt.x.f.v.mask.nxv2i64.nxv2f32(
 define <vscale x 2 x i64> @intrinsic_vfwcvt_mask_x.f.v_nxv2i64_nxv2f32(<vscale x 2 x i64> %0, <vscale x 2 x float> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f.v_nxv2i64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vfwcvt.x.f.v.mask.nxv2i64.nxv2f32(
@@ -341,10 +341,10 @@ declare <vscale x 4 x i64> @llvm.riscv.vfwcvt.x.f.v.nxv4i64.nxv4f32(
 define <vscale x 4 x i64> @intrinsic_vfwcvt_x.f.v_nxv4i64_nxv4f32(<vscale x 4 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f.v_nxv4i64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -365,10 +365,10 @@ declare <vscale x 4 x i64> @llvm.riscv.vfwcvt.x.f.v.mask.nxv4i64.nxv4f32(
 define <vscale x 4 x i64> @intrinsic_vfwcvt_mask_x.f.v_nxv4i64_nxv4f32(<vscale x 4 x i64> %0, <vscale x 4 x float> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f.v_nxv4i64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vfwcvt.x.f.v.mask.nxv4i64.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 8 x i64> @llvm.riscv.vfwcvt.x.f.v.nxv8i64.nxv8f32(
 define <vscale x 8 x i64> @intrinsic_vfwcvt_x.f.v_nxv8i64_nxv8f32(<vscale x 8 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f.v_nxv8i64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -412,10 +412,10 @@ declare <vscale x 8 x i64> @llvm.riscv.vfwcvt.x.f.v.mask.nxv8i64.nxv8f32(
 define <vscale x 8 x i64> @intrinsic_vfwcvt_mask_x.f.v_nxv8i64_nxv8f32(<vscale x 8 x i64> %0, <vscale x 8 x float> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f.v_nxv8i64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.x.f.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vfwcvt.x.f.v.mask.nxv8i64.nxv8f32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-xu-f.ll b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-xu-f.ll
index 82cea184920b..f6779ec9ba5a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwcvt-xu-f.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwcvt-xu-f.ll
@@ -12,10 +12,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfwcvt.xu.f.v.nxv1i32.nxv1f16(
 define <vscale x 1 x i32> @intrinsic_vfwcvt_xu.f.v_nxv1i32_nxv1f16(<vscale x 1 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f.v_nxv1i32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -36,10 +36,10 @@ declare <vscale x 1 x i32> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv1i32.nxv1f16(
 define <vscale x 1 x i32> @intrinsic_vfwcvt_mask_xu.f.v_nxv1i32_nxv1f16(<vscale x 1 x i32> %0, <vscale x 1 x half> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f.v_nxv1i32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i32> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv1i32.nxv1f16(
@@ -59,10 +59,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfwcvt.xu.f.v.nxv2i32.nxv2f16(
 define <vscale x 2 x i32> @intrinsic_vfwcvt_xu.f.v_nxv2i32_nxv2f16(<vscale x 2 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f.v_nxv2i32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -83,10 +83,10 @@ declare <vscale x 2 x i32> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv2i32.nxv2f16(
 define <vscale x 2 x i32> @intrinsic_vfwcvt_mask_xu.f.v_nxv2i32_nxv2f16(<vscale x 2 x i32> %0, <vscale x 2 x half> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f.v_nxv2i32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i32> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv2i32.nxv2f16(
@@ -106,10 +106,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfwcvt.xu.f.v.nxv4i32.nxv4f16(
 define <vscale x 4 x i32> @intrinsic_vfwcvt_xu.f.v_nxv4i32_nxv4f16(<vscale x 4 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f.v_nxv4i32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -130,10 +130,10 @@ declare <vscale x 4 x i32> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv4i32.nxv4f16(
 define <vscale x 4 x i32> @intrinsic_vfwcvt_mask_xu.f.v_nxv4i32_nxv4f16(<vscale x 4 x i32> %0, <vscale x 4 x half> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f.v_nxv4i32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i32> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv4i32.nxv4f16(
@@ -153,10 +153,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfwcvt.xu.f.v.nxv8i32.nxv8f16(
 define <vscale x 8 x i32> @intrinsic_vfwcvt_xu.f.v_nxv8i32_nxv8f16(<vscale x 8 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f.v_nxv8i32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -177,10 +177,10 @@ declare <vscale x 8 x i32> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv8i32.nxv8f16(
 define <vscale x 8 x i32> @intrinsic_vfwcvt_mask_xu.f.v_nxv8i32_nxv8f16(<vscale x 8 x i32> %0, <vscale x 8 x half> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f.v_nxv8i32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i32> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv8i32.nxv8f16(
@@ -200,10 +200,10 @@ declare <vscale x 16 x i32> @llvm.riscv.vfwcvt.xu.f.v.nxv16i32.nxv16f16(
 define <vscale x 16 x i32> @intrinsic_vfwcvt_xu.f.v_nxv16i32_nxv16f16(<vscale x 16 x half> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f.v_nxv16i32_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -224,10 +224,10 @@ declare <vscale x 16 x i32> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv16i32.nxv16f16(
 define <vscale x 16 x i32> @intrinsic_vfwcvt_mask_xu.f.v_nxv16i32_nxv16f16(<vscale x 16 x i32> %0, <vscale x 16 x half> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f.v_nxv16i32_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x i32> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv16i32.nxv16f16(
@@ -247,10 +247,10 @@ declare <vscale x 1 x i64> @llvm.riscv.vfwcvt.xu.f.v.nxv1i64.nxv1f32(
 define <vscale x 1 x i64> @intrinsic_vfwcvt_xu.f.v_nxv1i64_nxv1f32(<vscale x 1 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f.v_nxv1i64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -271,10 +271,10 @@ declare <vscale x 1 x i64> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv1i64.nxv1f32(
 define <vscale x 1 x i64> @intrinsic_vfwcvt_mask_xu.f.v_nxv1i64_nxv1f32(<vscale x 1 x i64> %0, <vscale x 1 x float> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f.v_nxv1i64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x i64> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv1i64.nxv1f32(
@@ -294,10 +294,10 @@ declare <vscale x 2 x i64> @llvm.riscv.vfwcvt.xu.f.v.nxv2i64.nxv2f32(
 define <vscale x 2 x i64> @intrinsic_vfwcvt_xu.f.v_nxv2i64_nxv2f32(<vscale x 2 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f.v_nxv2i64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -318,10 +318,10 @@ declare <vscale x 2 x i64> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv2i64.nxv2f32(
 define <vscale x 2 x i64> @intrinsic_vfwcvt_mask_xu.f.v_nxv2i64_nxv2f32(<vscale x 2 x i64> %0, <vscale x 2 x float> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f.v_nxv2i64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x i64> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv2i64.nxv2f32(
@@ -341,10 +341,10 @@ declare <vscale x 4 x i64> @llvm.riscv.vfwcvt.xu.f.v.nxv4i64.nxv4f32(
 define <vscale x 4 x i64> @intrinsic_vfwcvt_xu.f.v_nxv4i64_nxv4f32(<vscale x 4 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f.v_nxv4i64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -365,10 +365,10 @@ declare <vscale x 4 x i64> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv4i64.nxv4f32(
 define <vscale x 4 x i64> @intrinsic_vfwcvt_mask_xu.f.v_nxv4i64_nxv4f32(<vscale x 4 x i64> %0, <vscale x 4 x float> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f.v_nxv4i64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x i64> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv4i64.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 8 x i64> @llvm.riscv.vfwcvt.xu.f.v.nxv8i64.nxv8f32(
 define <vscale x 8 x i64> @intrinsic_vfwcvt_xu.f.v_nxv8i64_nxv8f32(<vscale x 8 x float> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f.v_nxv8i64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -412,10 +412,10 @@ declare <vscale x 8 x i64> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv8i64.nxv8f32(
 define <vscale x 8 x i64> @intrinsic_vfwcvt_mask_xu.f.v_nxv8i64_nxv8f32(<vscale x 8 x i64> %0, <vscale x 8 x float> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f.v_nxv8i64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwcvt.xu.f.v v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x i64> @llvm.riscv.vfwcvt.xu.f.v.mask.nxv8i64.nxv8f32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwmacc.ll b/llvm/test/CodeGen/RISCV/rvv/vfwmacc.ll
index b3ff91d92ce9..225ba1c14031 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwmacc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwmacc.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmacc.nxv1f32.nxv1f16(
 define <vscale x 1 x float>  @intrinsic_vfwmacc_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwmacc.nxv1f32.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmacc.mask.nxv1f32.nxv1f16(
 define <vscale x 1 x float>  @intrinsic_vfwmacc_mask_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwmacc.mask.nxv1f32.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmacc.nxv2f32.nxv2f16(
 define <vscale x 2 x float>  @intrinsic_vfwmacc_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwmacc.nxv2f32.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmacc.mask.nxv2f32.nxv2f16(
 define <vscale x 2 x float>  @intrinsic_vfwmacc_mask_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwmacc.mask.nxv2f32.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmacc.nxv4f32.nxv4f16(
 define <vscale x 4 x float>  @intrinsic_vfwmacc_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v10, v11
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwmacc.nxv4f32.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmacc.mask.nxv4f32.nxv4f16(
 define <vscale x 4 x float>  @intrinsic_vfwmacc_mask_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwmacc.mask.nxv4f32.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmacc.nxv8f32.nxv8f16(
 define <vscale x 8 x float>  @intrinsic_vfwmacc_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v12, v14
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwmacc.nxv8f32.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmacc.mask.nxv8f32.nxv8f16(
 define <vscale x 8 x float>  @intrinsic_vfwmacc_mask_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwmacc.mask.nxv8f32.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmacc.nxv16f32.nxv16f16(
 define <vscale x 16 x float>  @intrinsic_vfwmacc_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v16, v20
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwmacc.nxv16f32.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmacc.mask.nxv16f32.nxv16f16(
 define <vscale x 16 x float>  @intrinsic_vfwmacc_mask_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwmacc.mask.nxv16f32.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmacc.nxv1f64.nxv1f32(
 define <vscale x 1 x double>  @intrinsic_vfwmacc_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwmacc.nxv1f64.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmacc.mask.nxv1f64.nxv1f32(
 define <vscale x 1 x double>  @intrinsic_vfwmacc_mask_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwmacc.mask.nxv1f64.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmacc.nxv2f64.nxv2f32(
 define <vscale x 2 x double>  @intrinsic_vfwmacc_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v10, v11
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwmacc.nxv2f64.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmacc.mask.nxv2f64.nxv2f32(
 define <vscale x 2 x double>  @intrinsic_vfwmacc_mask_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwmacc.mask.nxv2f64.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmacc.nxv4f64.nxv4f32(
 define <vscale x 4 x double>  @intrinsic_vfwmacc_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v12, v14
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwmacc.nxv4f64.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmacc.mask.nxv4f64.nxv4f32(
 define <vscale x 4 x double>  @intrinsic_vfwmacc_mask_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwmacc.mask.nxv4f64.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmacc.nxv8f64.nxv8f32(
 define <vscale x 8 x double>  @intrinsic_vfwmacc_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v16, v20
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwmacc.nxv8f64.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmacc.mask.nxv8f64.nxv8f32(
 define <vscale x 8 x double>  @intrinsic_vfwmacc_mask_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwmacc.mask.nxv8f64.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmacc.nxv1f32.f16(
 define <vscale x 1 x float>  @intrinsic_vfwmacc_vf_nxv1f32_f16_nxv1f16(<vscale x 1 x float> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vf_nxv1f32_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwmacc.nxv1f32.f16(
@@ -488,10 +488,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmacc.mask.nxv1f32.f16(
 define <vscale x 1 x float> @intrinsic_vfwmacc_mask_vf_nxv1f32_f16_nxv1f16(<vscale x 1 x float> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vf_nxv1f32_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwmacc.mask.nxv1f32.f16(
@@ -513,10 +513,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmacc.nxv2f32.f16(
 define <vscale x 2 x float>  @intrinsic_vfwmacc_vf_nxv2f32_f16_nxv2f16(<vscale x 2 x float> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vf_nxv2f32_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwmacc.nxv2f32.f16(
@@ -538,10 +538,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmacc.mask.nxv2f32.f16(
 define <vscale x 2 x float> @intrinsic_vfwmacc_mask_vf_nxv2f32_f16_nxv2f16(<vscale x 2 x float> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vf_nxv2f32_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwmacc.mask.nxv2f32.f16(
@@ -563,10 +563,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmacc.nxv4f32.f16(
 define <vscale x 4 x float>  @intrinsic_vfwmacc_vf_nxv4f32_f16_nxv4f16(<vscale x 4 x float> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vf_nxv4f32_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwmacc.nxv4f32.f16(
@@ -588,10 +588,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmacc.mask.nxv4f32.f16(
 define <vscale x 4 x float> @intrinsic_vfwmacc_mask_vf_nxv4f32_f16_nxv4f16(<vscale x 4 x float> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vf_nxv4f32_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwmacc.mask.nxv4f32.f16(
@@ -613,10 +613,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmacc.nxv8f32.f16(
 define <vscale x 8 x float>  @intrinsic_vfwmacc_vf_nxv8f32_f16_nxv8f16(<vscale x 8 x float> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vf_nxv8f32_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwmacc.nxv8f32.f16(
@@ -638,10 +638,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmacc.mask.nxv8f32.f16(
 define <vscale x 8 x float> @intrinsic_vfwmacc_mask_vf_nxv8f32_f16_nxv8f16(<vscale x 8 x float> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vf_nxv8f32_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwmacc.mask.nxv8f32.f16(
@@ -663,10 +663,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmacc.nxv16f32.f16(
 define <vscale x 16 x float>  @intrinsic_vfwmacc_vf_nxv16f32_f16_nxv16f16(<vscale x 16 x float> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vf_nxv16f32_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwmacc.nxv16f32.f16(
@@ -688,10 +688,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmacc.mask.nxv16f32.f16(
 define <vscale x 16 x float> @intrinsic_vfwmacc_mask_vf_nxv16f32_f16_nxv16f16(<vscale x 16 x float> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vf_nxv16f32_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwmacc.mask.nxv16f32.f16(
@@ -713,10 +713,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmacc.nxv1f64.f32(
 define <vscale x 1 x double>  @intrinsic_vfwmacc_vf_nxv1f64_f32_nxv1f32(<vscale x 1 x double> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vf_nxv1f64_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwmacc.nxv1f64.f32(
@@ -738,10 +738,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmacc.mask.nxv1f64.f32(
 define <vscale x 1 x double> @intrinsic_vfwmacc_mask_vf_nxv1f64_f32_nxv1f32(<vscale x 1 x double> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vf_nxv1f64_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwmacc.mask.nxv1f64.f32(
@@ -763,10 +763,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmacc.nxv2f64.f32(
 define <vscale x 2 x double>  @intrinsic_vfwmacc_vf_nxv2f64_f32_nxv2f32(<vscale x 2 x double> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vf_nxv2f64_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwmacc.nxv2f64.f32(
@@ -788,10 +788,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmacc.mask.nxv2f64.f32(
 define <vscale x 2 x double> @intrinsic_vfwmacc_mask_vf_nxv2f64_f32_nxv2f32(<vscale x 2 x double> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vf_nxv2f64_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwmacc.mask.nxv2f64.f32(
@@ -813,10 +813,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmacc.nxv4f64.f32(
 define <vscale x 4 x double>  @intrinsic_vfwmacc_vf_nxv4f64_f32_nxv4f32(<vscale x 4 x double> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vf_nxv4f64_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwmacc.nxv4f64.f32(
@@ -838,10 +838,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmacc.mask.nxv4f64.f32(
 define <vscale x 4 x double> @intrinsic_vfwmacc_mask_vf_nxv4f64_f32_nxv4f32(<vscale x 4 x double> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vf_nxv4f64_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwmacc.mask.nxv4f64.f32(
@@ -863,10 +863,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmacc.nxv8f64.f32(
 define <vscale x 8 x double>  @intrinsic_vfwmacc_vf_nxv8f64_f32_nxv8f32(<vscale x 8 x double> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_vf_nxv8f64_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwmacc.nxv8f64.f32(
@@ -888,10 +888,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmacc.mask.nxv8f64.f32(
 define <vscale x 8 x double> @intrinsic_vfwmacc_mask_vf_nxv8f64_f32_nxv8f32(<vscale x 8 x double> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmacc_mask_vf_nxv8f64_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmacc.vf v8, fa0, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwmacc.mask.nxv8f64.f32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwmsac.ll b/llvm/test/CodeGen/RISCV/rvv/vfwmsac.ll
index 103eeb08f8c8..5e3f63b95b2f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwmsac.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwmsac.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmsac.nxv1f32.nxv1f16(
 define <vscale x 1 x float>  @intrinsic_vfwmsac_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwmsac.nxv1f32.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmsac.mask.nxv1f32.nxv1f16(
 define <vscale x 1 x float>  @intrinsic_vfwmsac_mask_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwmsac.mask.nxv1f32.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmsac.nxv2f32.nxv2f16(
 define <vscale x 2 x float>  @intrinsic_vfwmsac_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwmsac.nxv2f32.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmsac.mask.nxv2f32.nxv2f16(
 define <vscale x 2 x float>  @intrinsic_vfwmsac_mask_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwmsac.mask.nxv2f32.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmsac.nxv4f32.nxv4f16(
 define <vscale x 4 x float>  @intrinsic_vfwmsac_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v10, v11
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwmsac.nxv4f32.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmsac.mask.nxv4f32.nxv4f16(
 define <vscale x 4 x float>  @intrinsic_vfwmsac_mask_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwmsac.mask.nxv4f32.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmsac.nxv8f32.nxv8f16(
 define <vscale x 8 x float>  @intrinsic_vfwmsac_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v12, v14
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwmsac.nxv8f32.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmsac.mask.nxv8f32.nxv8f16(
 define <vscale x 8 x float>  @intrinsic_vfwmsac_mask_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwmsac.mask.nxv8f32.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmsac.nxv16f32.nxv16f16(
 define <vscale x 16 x float>  @intrinsic_vfwmsac_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v16, v20
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwmsac.nxv16f32.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmsac.mask.nxv16f32.nxv16f16(
 define <vscale x 16 x float>  @intrinsic_vfwmsac_mask_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwmsac.mask.nxv16f32.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmsac.nxv1f64.nxv1f32(
 define <vscale x 1 x double>  @intrinsic_vfwmsac_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwmsac.nxv1f64.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmsac.mask.nxv1f64.nxv1f32(
 define <vscale x 1 x double>  @intrinsic_vfwmsac_mask_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwmsac.mask.nxv1f64.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmsac.nxv2f64.nxv2f32(
 define <vscale x 2 x double>  @intrinsic_vfwmsac_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v10, v11
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwmsac.nxv2f64.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmsac.mask.nxv2f64.nxv2f32(
 define <vscale x 2 x double>  @intrinsic_vfwmsac_mask_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwmsac.mask.nxv2f64.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmsac.nxv4f64.nxv4f32(
 define <vscale x 4 x double>  @intrinsic_vfwmsac_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v12, v14
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwmsac.nxv4f64.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmsac.mask.nxv4f64.nxv4f32(
 define <vscale x 4 x double>  @intrinsic_vfwmsac_mask_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwmsac.mask.nxv4f64.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmsac.nxv8f64.nxv8f32(
 define <vscale x 8 x double>  @intrinsic_vfwmsac_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v16, v20
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwmsac.nxv8f64.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmsac.mask.nxv8f64.nxv8f32(
 define <vscale x 8 x double>  @intrinsic_vfwmsac_mask_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwmsac.mask.nxv8f64.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmsac.nxv1f32.f16(
 define <vscale x 1 x float>  @intrinsic_vfwmsac_vf_nxv1f32_f16_nxv1f16(<vscale x 1 x float> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vf_nxv1f32_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwmsac.nxv1f32.f16(
@@ -488,10 +488,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmsac.mask.nxv1f32.f16(
 define <vscale x 1 x float> @intrinsic_vfwmsac_mask_vf_nxv1f32_f16_nxv1f16(<vscale x 1 x float> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vf_nxv1f32_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwmsac.mask.nxv1f32.f16(
@@ -513,10 +513,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmsac.nxv2f32.f16(
 define <vscale x 2 x float>  @intrinsic_vfwmsac_vf_nxv2f32_f16_nxv2f16(<vscale x 2 x float> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vf_nxv2f32_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwmsac.nxv2f32.f16(
@@ -538,10 +538,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmsac.mask.nxv2f32.f16(
 define <vscale x 2 x float> @intrinsic_vfwmsac_mask_vf_nxv2f32_f16_nxv2f16(<vscale x 2 x float> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vf_nxv2f32_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwmsac.mask.nxv2f32.f16(
@@ -563,10 +563,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmsac.nxv4f32.f16(
 define <vscale x 4 x float>  @intrinsic_vfwmsac_vf_nxv4f32_f16_nxv4f16(<vscale x 4 x float> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vf_nxv4f32_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwmsac.nxv4f32.f16(
@@ -588,10 +588,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmsac.mask.nxv4f32.f16(
 define <vscale x 4 x float> @intrinsic_vfwmsac_mask_vf_nxv4f32_f16_nxv4f16(<vscale x 4 x float> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vf_nxv4f32_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwmsac.mask.nxv4f32.f16(
@@ -613,10 +613,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmsac.nxv8f32.f16(
 define <vscale x 8 x float>  @intrinsic_vfwmsac_vf_nxv8f32_f16_nxv8f16(<vscale x 8 x float> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vf_nxv8f32_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwmsac.nxv8f32.f16(
@@ -638,10 +638,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmsac.mask.nxv8f32.f16(
 define <vscale x 8 x float> @intrinsic_vfwmsac_mask_vf_nxv8f32_f16_nxv8f16(<vscale x 8 x float> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vf_nxv8f32_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwmsac.mask.nxv8f32.f16(
@@ -663,10 +663,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmsac.nxv16f32.f16(
 define <vscale x 16 x float>  @intrinsic_vfwmsac_vf_nxv16f32_f16_nxv16f16(<vscale x 16 x float> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vf_nxv16f32_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwmsac.nxv16f32.f16(
@@ -688,10 +688,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmsac.mask.nxv16f32.f16(
 define <vscale x 16 x float> @intrinsic_vfwmsac_mask_vf_nxv16f32_f16_nxv16f16(<vscale x 16 x float> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vf_nxv16f32_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwmsac.mask.nxv16f32.f16(
@@ -713,10 +713,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmsac.nxv1f64.f32(
 define <vscale x 1 x double>  @intrinsic_vfwmsac_vf_nxv1f64_f32_nxv1f32(<vscale x 1 x double> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vf_nxv1f64_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwmsac.nxv1f64.f32(
@@ -738,10 +738,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmsac.mask.nxv1f64.f32(
 define <vscale x 1 x double> @intrinsic_vfwmsac_mask_vf_nxv1f64_f32_nxv1f32(<vscale x 1 x double> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vf_nxv1f64_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwmsac.mask.nxv1f64.f32(
@@ -763,10 +763,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmsac.nxv2f64.f32(
 define <vscale x 2 x double>  @intrinsic_vfwmsac_vf_nxv2f64_f32_nxv2f32(<vscale x 2 x double> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vf_nxv2f64_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwmsac.nxv2f64.f32(
@@ -788,10 +788,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmsac.mask.nxv2f64.f32(
 define <vscale x 2 x double> @intrinsic_vfwmsac_mask_vf_nxv2f64_f32_nxv2f32(<vscale x 2 x double> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vf_nxv2f64_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwmsac.mask.nxv2f64.f32(
@@ -813,10 +813,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmsac.nxv4f64.f32(
 define <vscale x 4 x double>  @intrinsic_vfwmsac_vf_nxv4f64_f32_nxv4f32(<vscale x 4 x double> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vf_nxv4f64_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwmsac.nxv4f64.f32(
@@ -838,10 +838,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmsac.mask.nxv4f64.f32(
 define <vscale x 4 x double> @intrinsic_vfwmsac_mask_vf_nxv4f64_f32_nxv4f32(<vscale x 4 x double> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vf_nxv4f64_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwmsac.mask.nxv4f64.f32(
@@ -863,10 +863,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmsac.nxv8f64.f32(
 define <vscale x 8 x double>  @intrinsic_vfwmsac_vf_nxv8f64_f32_nxv8f32(<vscale x 8 x double> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_vf_nxv8f64_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwmsac.nxv8f64.f32(
@@ -888,10 +888,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmsac.mask.nxv8f64.f32(
 define <vscale x 8 x double> @intrinsic_vfwmsac_mask_vf_nxv8f64_f32_nxv8f32(<vscale x 8 x double> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmsac_mask_vf_nxv8f64_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmsac.vf v8, fa0, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwmsac.mask.nxv8f64.f32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwmul.ll b/llvm/test/CodeGen/RISCV/rvv/vfwmul.ll
index 2f9fc24de3aa..bc5759f469ad 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwmul.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmul.nxv1f32.nxv1f16.nxv1f16(
 define <vscale x 1 x float> @intrinsic_vfwmul_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -39,10 +39,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmul.mask.nxv1f32.nxv1f16.nxv1f16(
 define <vscale x 1 x float> @intrinsic_vfwmul_mask_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwmul.mask.nxv1f32.nxv1f16.nxv1f16(
@@ -64,10 +64,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmul.nxv2f32.nxv2f16.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwmul_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -90,10 +90,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmul.mask.nxv2f32.nxv2f16.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwmul_mask_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwmul.mask.nxv2f32.nxv2f16.nxv2f16(
@@ -115,10 +115,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmul.nxv4f32.nxv4f16.nxv4f16(
 define <vscale x 4 x float> @intrinsic_vfwmul_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -141,10 +141,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmul.mask.nxv4f32.nxv4f16.nxv4f16(
 define <vscale x 4 x float> @intrinsic_vfwmul_mask_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwmul.mask.nxv4f32.nxv4f16.nxv4f16(
@@ -166,10 +166,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmul.nxv8f32.nxv8f16.nxv8f16(
 define <vscale x 8 x float> @intrinsic_vfwmul_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v12, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -192,10 +192,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmul.mask.nxv8f32.nxv8f16.nxv8f16(
 define <vscale x 8 x float> @intrinsic_vfwmul_mask_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwmul.mask.nxv8f32.nxv8f16.nxv8f16(
@@ -217,10 +217,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmul.nxv16f32.nxv16f16.nxv16f16(
 define <vscale x 16 x float> @intrinsic_vfwmul_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v16, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -243,10 +243,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmul.mask.nxv16f32.nxv16f16.nxv16f16
 define <vscale x 16 x float> @intrinsic_vfwmul_mask_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwmul.mask.nxv16f32.nxv16f16.nxv16f16(
@@ -268,10 +268,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmul.nxv1f64.nxv1f32.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwmul_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -294,10 +294,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmul.mask.nxv1f64.nxv1f32.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwmul_mask_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwmul.mask.nxv1f64.nxv1f32.nxv1f32(
@@ -319,10 +319,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmul.nxv2f64.nxv2f32.nxv2f32(
 define <vscale x 2 x double> @intrinsic_vfwmul_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -345,10 +345,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmul.mask.nxv2f64.nxv2f32.nxv2f32(
 define <vscale x 2 x double> @intrinsic_vfwmul_mask_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwmul.mask.nxv2f64.nxv2f32.nxv2f32(
@@ -370,10 +370,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmul.nxv4f64.nxv4f32.nxv4f32(
 define <vscale x 4 x double> @intrinsic_vfwmul_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v12, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -396,10 +396,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmul.mask.nxv4f64.nxv4f32.nxv4f32(
 define <vscale x 4 x double> @intrinsic_vfwmul_mask_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwmul.mask.nxv4f64.nxv4f32.nxv4f32(
@@ -421,10 +421,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmul.nxv8f64.nxv8f32.nxv8f32(
 define <vscale x 8 x double> @intrinsic_vfwmul_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v16, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -447,10 +447,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmul.mask.nxv8f64.nxv8f32.nxv8f32(
 define <vscale x 8 x double> @intrinsic_vfwmul_mask_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwmul.mask.nxv8f64.nxv8f32.nxv8f32(
@@ -472,10 +472,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmul.nxv1f32.nxv1f16.f16(
 define <vscale x 1 x float> @intrinsic_vfwmul_vf_nxv1f32_nxv1f16_f16(<vscale x 1 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv1f32_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -498,10 +498,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwmul.mask.nxv1f32.nxv1f16.f16(
 define <vscale x 1 x float> @intrinsic_vfwmul_mask_vf_nxv1f32_nxv1f16_f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv1f32_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwmul.mask.nxv1f32.nxv1f16.f16(
@@ -523,10 +523,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmul.nxv2f32.nxv2f16.f16(
 define <vscale x 2 x float> @intrinsic_vfwmul_vf_nxv2f32_nxv2f16_f16(<vscale x 2 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv2f32_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -549,10 +549,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwmul.mask.nxv2f32.nxv2f16.f16(
 define <vscale x 2 x float> @intrinsic_vfwmul_mask_vf_nxv2f32_nxv2f16_f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv2f32_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwmul.mask.nxv2f32.nxv2f16.f16(
@@ -574,10 +574,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmul.nxv4f32.nxv4f16.f16(
 define <vscale x 4 x float> @intrinsic_vfwmul_vf_nxv4f32_nxv4f16_f16(<vscale x 4 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv4f32_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v10, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -600,10 +600,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwmul.mask.nxv4f32.nxv4f16.f16(
 define <vscale x 4 x float> @intrinsic_vfwmul_mask_vf_nxv4f32_nxv4f16_f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv4f32_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwmul.mask.nxv4f32.nxv4f16.f16(
@@ -625,10 +625,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmul.nxv8f32.nxv8f16.f16(
 define <vscale x 8 x float> @intrinsic_vfwmul_vf_nxv8f32_nxv8f16_f16(<vscale x 8 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv8f32_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v12, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -651,10 +651,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwmul.mask.nxv8f32.nxv8f16.f16(
 define <vscale x 8 x float> @intrinsic_vfwmul_mask_vf_nxv8f32_nxv8f16_f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv8f32_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwmul.mask.nxv8f32.nxv8f16.f16(
@@ -676,10 +676,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmul.nxv16f32.nxv16f16.f16(
 define <vscale x 16 x float> @intrinsic_vfwmul_vf_nxv16f32_nxv16f16_f16(<vscale x 16 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv16f32_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v16, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -702,10 +702,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwmul.mask.nxv16f32.nxv16f16.f16(
 define <vscale x 16 x float> @intrinsic_vfwmul_mask_vf_nxv16f32_nxv16f16_f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv16f32_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwmul.mask.nxv16f32.nxv16f16.f16(
@@ -727,10 +727,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmul.nxv1f64.nxv1f32.f32(
 define <vscale x 1 x double> @intrinsic_vfwmul_vf_nxv1f64_nxv1f32_f32(<vscale x 1 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv1f64_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -753,10 +753,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwmul.mask.nxv1f64.nxv1f32.f32(
 define <vscale x 1 x double> @intrinsic_vfwmul_mask_vf_nxv1f64_nxv1f32_f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv1f64_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwmul.mask.nxv1f64.nxv1f32.f32(
@@ -778,10 +778,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmul.nxv2f64.nxv2f32.f32(
 define <vscale x 2 x double> @intrinsic_vfwmul_vf_nxv2f64_nxv2f32_f32(<vscale x 2 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv2f64_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v10, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -804,10 +804,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwmul.mask.nxv2f64.nxv2f32.f32(
 define <vscale x 2 x double> @intrinsic_vfwmul_mask_vf_nxv2f64_nxv2f32_f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv2f64_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwmul.mask.nxv2f64.nxv2f32.f32(
@@ -829,10 +829,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmul.nxv4f64.nxv4f32.f32(
 define <vscale x 4 x double> @intrinsic_vfwmul_vf_nxv4f64_nxv4f32_f32(<vscale x 4 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv4f64_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v12, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -855,10 +855,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwmul.mask.nxv4f64.nxv4f32.f32(
 define <vscale x 4 x double> @intrinsic_vfwmul_mask_vf_nxv4f64_nxv4f32_f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv4f64_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwmul.mask.nxv4f64.nxv4f32.f32(
@@ -880,10 +880,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmul.nxv8f64.nxv8f32.f32(
 define <vscale x 8 x double> @intrinsic_vfwmul_vf_nxv8f64_nxv8f32_f32(<vscale x 8 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv8f64_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v16, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -906,10 +906,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwmul.mask.nxv8f64.nxv8f32.f32(
 define <vscale x 8 x double> @intrinsic_vfwmul_mask_vf_nxv8f64_nxv8f32_f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv8f64_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwmul.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwmul.mask.nxv8f64.nxv8f32.f32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwnmacc.ll b/llvm/test/CodeGen/RISCV/rvv/vfwnmacc.ll
index ca2d2a33159b..fc8e15273f08 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwnmacc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwnmacc.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwnmacc.nxv1f32.nxv1f16(
 define <vscale x 1 x float>  @intrinsic_vfwnmacc_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwnmacc.nxv1f32.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwnmacc.mask.nxv1f32.nxv1f16(
 define <vscale x 1 x float>  @intrinsic_vfwnmacc_mask_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwnmacc.mask.nxv1f32.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwnmacc.nxv2f32.nxv2f16(
 define <vscale x 2 x float>  @intrinsic_vfwnmacc_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwnmacc.nxv2f32.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwnmacc.mask.nxv2f32.nxv2f16(
 define <vscale x 2 x float>  @intrinsic_vfwnmacc_mask_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwnmacc.mask.nxv2f32.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwnmacc.nxv4f32.nxv4f16(
 define <vscale x 4 x float>  @intrinsic_vfwnmacc_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v10, v11
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwnmacc.nxv4f32.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwnmacc.mask.nxv4f32.nxv4f16(
 define <vscale x 4 x float>  @intrinsic_vfwnmacc_mask_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwnmacc.mask.nxv4f32.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwnmacc.nxv8f32.nxv8f16(
 define <vscale x 8 x float>  @intrinsic_vfwnmacc_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v12, v14
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwnmacc.nxv8f32.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwnmacc.mask.nxv8f32.nxv8f16(
 define <vscale x 8 x float>  @intrinsic_vfwnmacc_mask_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwnmacc.mask.nxv8f32.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwnmacc.nxv16f32.nxv16f16(
 define <vscale x 16 x float>  @intrinsic_vfwnmacc_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v16, v20
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwnmacc.nxv16f32.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwnmacc.mask.nxv16f32.nxv16f16(
 define <vscale x 16 x float>  @intrinsic_vfwnmacc_mask_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwnmacc.mask.nxv16f32.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwnmacc.nxv1f64.nxv1f32(
 define <vscale x 1 x double>  @intrinsic_vfwnmacc_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwnmacc.nxv1f64.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwnmacc.mask.nxv1f64.nxv1f32(
 define <vscale x 1 x double>  @intrinsic_vfwnmacc_mask_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwnmacc.mask.nxv1f64.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwnmacc.nxv2f64.nxv2f32(
 define <vscale x 2 x double>  @intrinsic_vfwnmacc_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v10, v11
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwnmacc.nxv2f64.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwnmacc.mask.nxv2f64.nxv2f32(
 define <vscale x 2 x double>  @intrinsic_vfwnmacc_mask_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwnmacc.mask.nxv2f64.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwnmacc.nxv4f64.nxv4f32(
 define <vscale x 4 x double>  @intrinsic_vfwnmacc_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v12, v14
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwnmacc.nxv4f64.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwnmacc.mask.nxv4f64.nxv4f32(
 define <vscale x 4 x double>  @intrinsic_vfwnmacc_mask_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwnmacc.mask.nxv4f64.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwnmacc.nxv8f64.nxv8f32(
 define <vscale x 8 x double>  @intrinsic_vfwnmacc_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v16, v20
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwnmacc.nxv8f64.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwnmacc.mask.nxv8f64.nxv8f32(
 define <vscale x 8 x double>  @intrinsic_vfwnmacc_mask_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwnmacc.mask.nxv8f64.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwnmacc.nxv1f32.f16(
 define <vscale x 1 x float>  @intrinsic_vfwnmacc_vf_nxv1f32_f16_nxv1f16(<vscale x 1 x float> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vf_nxv1f32_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwnmacc.nxv1f32.f16(
@@ -488,10 +488,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwnmacc.mask.nxv1f32.f16(
 define <vscale x 1 x float> @intrinsic_vfwnmacc_mask_vf_nxv1f32_f16_nxv1f16(<vscale x 1 x float> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vf_nxv1f32_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwnmacc.mask.nxv1f32.f16(
@@ -513,10 +513,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwnmacc.nxv2f32.f16(
 define <vscale x 2 x float>  @intrinsic_vfwnmacc_vf_nxv2f32_f16_nxv2f16(<vscale x 2 x float> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vf_nxv2f32_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwnmacc.nxv2f32.f16(
@@ -538,10 +538,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwnmacc.mask.nxv2f32.f16(
 define <vscale x 2 x float> @intrinsic_vfwnmacc_mask_vf_nxv2f32_f16_nxv2f16(<vscale x 2 x float> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vf_nxv2f32_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwnmacc.mask.nxv2f32.f16(
@@ -563,10 +563,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwnmacc.nxv4f32.f16(
 define <vscale x 4 x float>  @intrinsic_vfwnmacc_vf_nxv4f32_f16_nxv4f16(<vscale x 4 x float> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vf_nxv4f32_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwnmacc.nxv4f32.f16(
@@ -588,10 +588,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwnmacc.mask.nxv4f32.f16(
 define <vscale x 4 x float> @intrinsic_vfwnmacc_mask_vf_nxv4f32_f16_nxv4f16(<vscale x 4 x float> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vf_nxv4f32_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwnmacc.mask.nxv4f32.f16(
@@ -613,10 +613,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwnmacc.nxv8f32.f16(
 define <vscale x 8 x float>  @intrinsic_vfwnmacc_vf_nxv8f32_f16_nxv8f16(<vscale x 8 x float> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vf_nxv8f32_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwnmacc.nxv8f32.f16(
@@ -638,10 +638,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwnmacc.mask.nxv8f32.f16(
 define <vscale x 8 x float> @intrinsic_vfwnmacc_mask_vf_nxv8f32_f16_nxv8f16(<vscale x 8 x float> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vf_nxv8f32_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwnmacc.mask.nxv8f32.f16(
@@ -663,10 +663,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwnmacc.nxv16f32.f16(
 define <vscale x 16 x float>  @intrinsic_vfwnmacc_vf_nxv16f32_f16_nxv16f16(<vscale x 16 x float> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vf_nxv16f32_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwnmacc.nxv16f32.f16(
@@ -688,10 +688,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwnmacc.mask.nxv16f32.f16(
 define <vscale x 16 x float> @intrinsic_vfwnmacc_mask_vf_nxv16f32_f16_nxv16f16(<vscale x 16 x float> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vf_nxv16f32_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwnmacc.mask.nxv16f32.f16(
@@ -713,10 +713,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwnmacc.nxv1f64.f32(
 define <vscale x 1 x double>  @intrinsic_vfwnmacc_vf_nxv1f64_f32_nxv1f32(<vscale x 1 x double> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vf_nxv1f64_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwnmacc.nxv1f64.f32(
@@ -738,10 +738,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwnmacc.mask.nxv1f64.f32(
 define <vscale x 1 x double> @intrinsic_vfwnmacc_mask_vf_nxv1f64_f32_nxv1f32(<vscale x 1 x double> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vf_nxv1f64_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwnmacc.mask.nxv1f64.f32(
@@ -763,10 +763,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwnmacc.nxv2f64.f32(
 define <vscale x 2 x double>  @intrinsic_vfwnmacc_vf_nxv2f64_f32_nxv2f32(<vscale x 2 x double> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vf_nxv2f64_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwnmacc.nxv2f64.f32(
@@ -788,10 +788,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwnmacc.mask.nxv2f64.f32(
 define <vscale x 2 x double> @intrinsic_vfwnmacc_mask_vf_nxv2f64_f32_nxv2f32(<vscale x 2 x double> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vf_nxv2f64_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwnmacc.mask.nxv2f64.f32(
@@ -813,10 +813,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwnmacc.nxv4f64.f32(
 define <vscale x 4 x double>  @intrinsic_vfwnmacc_vf_nxv4f64_f32_nxv4f32(<vscale x 4 x double> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vf_nxv4f64_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwnmacc.nxv4f64.f32(
@@ -838,10 +838,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwnmacc.mask.nxv4f64.f32(
 define <vscale x 4 x double> @intrinsic_vfwnmacc_mask_vf_nxv4f64_f32_nxv4f32(<vscale x 4 x double> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vf_nxv4f64_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwnmacc.mask.nxv4f64.f32(
@@ -863,10 +863,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwnmacc.nxv8f64.f32(
 define <vscale x 8 x double>  @intrinsic_vfwnmacc_vf_nxv8f64_f32_nxv8f32(<vscale x 8 x double> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_vf_nxv8f64_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwnmacc.nxv8f64.f32(
@@ -888,10 +888,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwnmacc.mask.nxv8f64.f32(
 define <vscale x 8 x double> @intrinsic_vfwnmacc_mask_vf_nxv8f64_f32_nxv8f32(<vscale x 8 x double> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmacc_mask_vf_nxv8f64_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmacc.vf v8, fa0, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwnmacc.mask.nxv8f64.f32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwnmsac.ll b/llvm/test/CodeGen/RISCV/rvv/vfwnmsac.ll
index 648727dce246..b51faf9082c8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwnmsac.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwnmsac.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwnmsac.nxv1f32.nxv1f16(
 define <vscale x 1 x float>  @intrinsic_vfwnmsac_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwnmsac.nxv1f32.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwnmsac.mask.nxv1f32.nxv1f16(
 define <vscale x 1 x float>  @intrinsic_vfwnmsac_mask_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwnmsac.mask.nxv1f32.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwnmsac.nxv2f32.nxv2f16(
 define <vscale x 2 x float>  @intrinsic_vfwnmsac_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwnmsac.nxv2f32.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwnmsac.mask.nxv2f32.nxv2f16(
 define <vscale x 2 x float>  @intrinsic_vfwnmsac_mask_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwnmsac.mask.nxv2f32.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwnmsac.nxv4f32.nxv4f16(
 define <vscale x 4 x float>  @intrinsic_vfwnmsac_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v10, v11
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwnmsac.nxv4f32.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwnmsac.mask.nxv4f32.nxv4f16(
 define <vscale x 4 x float>  @intrinsic_vfwnmsac_mask_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwnmsac.mask.nxv4f32.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwnmsac.nxv8f32.nxv8f16(
 define <vscale x 8 x float>  @intrinsic_vfwnmsac_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v12, v14
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwnmsac.nxv8f32.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwnmsac.mask.nxv8f32.nxv8f16(
 define <vscale x 8 x float>  @intrinsic_vfwnmsac_mask_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwnmsac.mask.nxv8f32.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwnmsac.nxv16f32.nxv16f16(
 define <vscale x 16 x float>  @intrinsic_vfwnmsac_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v16, v20
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwnmsac.nxv16f32.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwnmsac.mask.nxv16f32.nxv16f16(
 define <vscale x 16 x float>  @intrinsic_vfwnmsac_mask_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwnmsac.mask.nxv16f32.nxv16f16(
@@ -263,10 +263,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwnmsac.nxv1f64.nxv1f32(
 define <vscale x 1 x double>  @intrinsic_vfwnmsac_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwnmsac.nxv1f64.nxv1f32(
@@ -288,10 +288,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwnmsac.mask.nxv1f64.nxv1f32(
 define <vscale x 1 x double>  @intrinsic_vfwnmsac_mask_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwnmsac.mask.nxv1f64.nxv1f32(
@@ -313,10 +313,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwnmsac.nxv2f64.nxv2f32(
 define <vscale x 2 x double>  @intrinsic_vfwnmsac_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v10, v11
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwnmsac.nxv2f64.nxv2f32(
@@ -338,10 +338,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwnmsac.mask.nxv2f64.nxv2f32(
 define <vscale x 2 x double>  @intrinsic_vfwnmsac_mask_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwnmsac.mask.nxv2f64.nxv2f32(
@@ -363,10 +363,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwnmsac.nxv4f64.nxv4f32(
 define <vscale x 4 x double>  @intrinsic_vfwnmsac_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v12, v14
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwnmsac.nxv4f64.nxv4f32(
@@ -388,10 +388,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwnmsac.mask.nxv4f64.nxv4f32(
 define <vscale x 4 x double>  @intrinsic_vfwnmsac_mask_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwnmsac.mask.nxv4f64.nxv4f32(
@@ -413,10 +413,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwnmsac.nxv8f64.nxv8f32(
 define <vscale x 8 x double>  @intrinsic_vfwnmsac_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v16, v20
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwnmsac.nxv8f64.nxv8f32(
@@ -438,10 +438,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwnmsac.mask.nxv8f64.nxv8f32(
 define <vscale x 8 x double>  @intrinsic_vfwnmsac_mask_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwnmsac.mask.nxv8f64.nxv8f32(
@@ -463,10 +463,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwnmsac.nxv1f32.f16(
 define <vscale x 1 x float>  @intrinsic_vfwnmsac_vf_nxv1f32_f16_nxv1f16(<vscale x 1 x float> %0, half %1, <vscale x 1 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vf_nxv1f32_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwnmsac.nxv1f32.f16(
@@ -488,10 +488,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwnmsac.mask.nxv1f32.f16(
 define <vscale x 1 x float> @intrinsic_vfwnmsac_mask_vf_nxv1f32_f16_nxv1f16(<vscale x 1 x float> %0, half %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vf_nxv1f32_f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwnmsac.mask.nxv1f32.f16(
@@ -513,10 +513,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwnmsac.nxv2f32.f16(
 define <vscale x 2 x float>  @intrinsic_vfwnmsac_vf_nxv2f32_f16_nxv2f16(<vscale x 2 x float> %0, half %1, <vscale x 2 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vf_nxv2f32_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwnmsac.nxv2f32.f16(
@@ -538,10 +538,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwnmsac.mask.nxv2f32.f16(
 define <vscale x 2 x float> @intrinsic_vfwnmsac_mask_vf_nxv2f32_f16_nxv2f16(<vscale x 2 x float> %0, half %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vf_nxv2f32_f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwnmsac.mask.nxv2f32.f16(
@@ -563,10 +563,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwnmsac.nxv4f32.f16(
 define <vscale x 4 x float>  @intrinsic_vfwnmsac_vf_nxv4f32_f16_nxv4f16(<vscale x 4 x float> %0, half %1, <vscale x 4 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vf_nxv4f32_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwnmsac.nxv4f32.f16(
@@ -588,10 +588,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwnmsac.mask.nxv4f32.f16(
 define <vscale x 4 x float> @intrinsic_vfwnmsac_mask_vf_nxv4f32_f16_nxv4f16(<vscale x 4 x float> %0, half %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vf_nxv4f32_f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwnmsac.mask.nxv4f32.f16(
@@ -613,10 +613,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwnmsac.nxv8f32.f16(
 define <vscale x 8 x float>  @intrinsic_vfwnmsac_vf_nxv8f32_f16_nxv8f16(<vscale x 8 x float> %0, half %1, <vscale x 8 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vf_nxv8f32_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwnmsac.nxv8f32.f16(
@@ -638,10 +638,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwnmsac.mask.nxv8f32.f16(
 define <vscale x 8 x float> @intrinsic_vfwnmsac_mask_vf_nxv8f32_f16_nxv8f16(<vscale x 8 x float> %0, half %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vf_nxv8f32_f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwnmsac.mask.nxv8f32.f16(
@@ -663,10 +663,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwnmsac.nxv16f32.f16(
 define <vscale x 16 x float>  @intrinsic_vfwnmsac_vf_nxv16f32_f16_nxv16f16(<vscale x 16 x float> %0, half %1, <vscale x 16 x half> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vf_nxv16f32_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwnmsac.nxv16f32.f16(
@@ -688,10 +688,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwnmsac.mask.nxv16f32.f16(
 define <vscale x 16 x float> @intrinsic_vfwnmsac_mask_vf_nxv16f32_f16_nxv16f16(<vscale x 16 x float> %0, half %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vf_nxv16f32_f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwnmsac.mask.nxv16f32.f16(
@@ -713,10 +713,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwnmsac.nxv1f64.f32(
 define <vscale x 1 x double>  @intrinsic_vfwnmsac_vf_nxv1f64_f32_nxv1f32(<vscale x 1 x double> %0, float %1, <vscale x 1 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vf_nxv1f64_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwnmsac.nxv1f64.f32(
@@ -738,10 +738,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwnmsac.mask.nxv1f64.f32(
 define <vscale x 1 x double> @intrinsic_vfwnmsac_mask_vf_nxv1f64_f32_nxv1f32(<vscale x 1 x double> %0, float %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vf_nxv1f64_f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwnmsac.mask.nxv1f64.f32(
@@ -763,10 +763,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwnmsac.nxv2f64.f32(
 define <vscale x 2 x double>  @intrinsic_vfwnmsac_vf_nxv2f64_f32_nxv2f32(<vscale x 2 x double> %0, float %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vf_nxv2f64_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwnmsac.nxv2f64.f32(
@@ -788,10 +788,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwnmsac.mask.nxv2f64.f32(
 define <vscale x 2 x double> @intrinsic_vfwnmsac_mask_vf_nxv2f64_f32_nxv2f32(<vscale x 2 x double> %0, float %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vf_nxv2f64_f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwnmsac.mask.nxv2f64.f32(
@@ -813,10 +813,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwnmsac.nxv4f64.f32(
 define <vscale x 4 x double>  @intrinsic_vfwnmsac_vf_nxv4f64_f32_nxv4f32(<vscale x 4 x double> %0, float %1, <vscale x 4 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vf_nxv4f64_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwnmsac.nxv4f64.f32(
@@ -838,10 +838,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwnmsac.mask.nxv4f64.f32(
 define <vscale x 4 x double> @intrinsic_vfwnmsac_mask_vf_nxv4f64_f32_nxv4f32(<vscale x 4 x double> %0, float %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vf_nxv4f64_f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwnmsac.mask.nxv4f64.f32(
@@ -863,10 +863,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwnmsac.nxv8f64.f32(
 define <vscale x 8 x double>  @intrinsic_vfwnmsac_vf_nxv8f64_f32_nxv8f32(<vscale x 8 x double> %0, float %1, <vscale x 8 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_vf_nxv8f64_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwnmsac.nxv8f64.f32(
@@ -888,10 +888,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwnmsac.mask.nxv8f64.f32(
 define <vscale x 8 x double> @intrinsic_vfwnmsac_mask_vf_nxv8f64_f32_nxv8f32(<vscale x 8 x double> %0, float %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwnmsac_mask_vf_nxv8f64_f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, tu, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwnmsac.vf v8, fa0, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwnmsac.mask.nxv8f64.f32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwredosum.ll b/llvm/test/CodeGen/RISCV/rvv/vfwredosum.ll
index 2184ab413c55..cb2bea0b50e1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwredosum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwredosum.ll
@@ -13,10 +13,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv1f16(
 define <vscale x 2 x float> @intrinsic_vfwredosum_vs_nxv2f32_nxv1f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 1 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv2f32_nxv1f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv1f16.nxv2f32
 define <vscale x 2 x float> @intrinsic_vfwredosum_mask_vs_nxv2f32_nxv1f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 1 x half> %1, <vscale x 2 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv2f32_nxv1f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv1f16.nxv2f32(
@@ -63,10 +63,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwredosum_vs_nxv2f32_nxv2f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv2f32_nxv2f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv2f16.nxv2f32
 define <vscale x 2 x float> @intrinsic_vfwredosum_mask_vs_nxv2f32_nxv2f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv2f32_nxv2f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv2f16.nxv2f32(
@@ -113,10 +113,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv4f16(
 define <vscale x 2 x float> @intrinsic_vfwredosum_vs_nxv2f32_nxv4f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 4 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv2f32_nxv4f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv4f16.nxv2f32
 define <vscale x 2 x float> @intrinsic_vfwredosum_mask_vs_nxv2f32_nxv4f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 4 x half> %1, <vscale x 2 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv2f32_nxv4f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv4f16.nxv2f32(
@@ -163,10 +163,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv8f16(
 define <vscale x 2 x float> @intrinsic_vfwredosum_vs_nxv2f32_nxv8f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 8 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv2f32_nxv8f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v10, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv8f16.nxv2f32
 define <vscale x 2 x float> @intrinsic_vfwredosum_mask_vs_nxv2f32_nxv8f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 8 x half> %1, <vscale x 2 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv2f32_nxv8f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v10, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv8f16.nxv2f32(
@@ -213,10 +213,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv16f16(
 define <vscale x 2 x float> @intrinsic_vfwredosum_vs_nxv2f32_nxv16f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 16 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv2f32_nxv16f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v12, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv16f16.nxv2f3
 define <vscale x 2 x float> @intrinsic_vfwredosum_mask_vs_nxv2f32_nxv16f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 16 x half> %1, <vscale x 2 x float> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv2f32_nxv16f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v12, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv16f16.nxv2f32(
@@ -263,10 +263,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv32f16(
 define <vscale x 2 x float> @intrinsic_vfwredosum_vs_nxv2f32_nxv32f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 32 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv2f32_nxv32f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v16, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.nxv2f32.nxv32f16(
@@ -288,10 +288,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv32f16(
 define <vscale x 2 x float> @intrinsic_vfwredosum_mask_vs_nxv2f32_nxv32f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 32 x half> %1, <vscale x 2 x float> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv2f32_nxv32f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v16, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredosum.mask.nxv2f32.nxv32f16(
@@ -313,10 +313,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredosum.nxv1f64.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwredosum_vs_nxv1f64_nxv1f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv1f64_nxv1f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredosum.nxv1f64.nxv1f32(
@@ -338,10 +338,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredosum.mask.nxv1f64.nxv1f32.nxv1f6
 define <vscale x 1 x double> @intrinsic_vfwredosum_mask_vs_nxv1f64_nxv1f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv1f64_nxv1f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredosum.mask.nxv1f64.nxv1f32.nxv1f64(
@@ -363,10 +363,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredosum.nxv1f64.nxv2f32(
 define <vscale x 1 x double> @intrinsic_vfwredosum_vs_nxv1f64_nxv2f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 2 x float> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv1f64_nxv2f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredosum.nxv1f64.nxv2f32(
@@ -388,10 +388,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredosum.mask.nxv1f64.nxv2f32.nxv1f6
 define <vscale x 1 x double> @intrinsic_vfwredosum_mask_vs_nxv1f64_nxv2f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 2 x float> %1, <vscale x 1 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv1f64_nxv2f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredosum.mask.nxv1f64.nxv2f32.nxv1f64(
@@ -413,10 +413,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredosum.nxv1f64.nxv4f32(
 define <vscale x 1 x double> @intrinsic_vfwredosum_vs_nxv1f64_nxv4f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 4 x float> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv1f64_nxv4f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v10, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredosum.nxv1f64.nxv4f32(
@@ -438,10 +438,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredosum.mask.nxv1f64.nxv4f32.nxv1f6
 define <vscale x 1 x double> @intrinsic_vfwredosum_mask_vs_nxv1f64_nxv4f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 4 x float> %1, <vscale x 1 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv1f64_nxv4f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v10, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredosum.mask.nxv1f64.nxv4f32.nxv1f64(
@@ -463,10 +463,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredosum.nxv1f64.nxv8f32(
 define <vscale x 1 x double> @intrinsic_vfwredosum_vs_nxv1f64_nxv8f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 8 x float> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv1f64_nxv8f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v12, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredosum.nxv1f64.nxv8f32(
@@ -488,10 +488,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredosum.mask.nxv1f64.nxv8f32.nxv1f6
 define <vscale x 1 x double> @intrinsic_vfwredosum_mask_vs_nxv1f64_nxv8f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 8 x float> %1, <vscale x 1 x double> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv1f64_nxv8f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v12, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredosum.mask.nxv1f64.nxv8f32.nxv1f64(
@@ -513,10 +513,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredosum.nxv1f64.nxv16f32(
 define <vscale x 1 x double> @intrinsic_vfwredosum_vs_nxv1f64_nxv16f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 16 x float> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv1f64_nxv16f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v16, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredosum.nxv1f64.nxv16f32(
@@ -538,10 +538,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredosum.mask.nxv1f64.nxv16f32.nxv1f
 define <vscale x 1 x double> @intrinsic_vfwredosum_mask_vs_nxv1f64_nxv16f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 16 x float> %1, <vscale x 1 x double> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv1f64_nxv16f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredosum.vs v8, v16, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredosum.mask.nxv1f64.nxv16f32.nxv1f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwredusum.ll b/llvm/test/CodeGen/RISCV/rvv/vfwredusum.ll
index d3d76e575978..66c2da047cfa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwredusum.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwredusum.ll
@@ -13,10 +13,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv1f16(
 define <vscale x 2 x float> @intrinsic_vfwredusum_vs_nxv2f32_nxv1f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 1 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_vs_nxv2f32_nxv1f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv1f16.nxv2f32
 define <vscale x 2 x float> @intrinsic_vfwredusum_mask_vs_nxv2f32_nxv1f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 1 x half> %1, <vscale x 2 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_mask_vs_nxv2f32_nxv1f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv1f16.nxv2f32(
@@ -63,10 +63,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwredusum_vs_nxv2f32_nxv2f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_vs_nxv2f32_nxv2f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv2f16.nxv2f32
 define <vscale x 2 x float> @intrinsic_vfwredusum_mask_vs_nxv2f32_nxv2f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_mask_vs_nxv2f32_nxv2f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv2f16.nxv2f32(
@@ -113,10 +113,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv4f16(
 define <vscale x 2 x float> @intrinsic_vfwredusum_vs_nxv2f32_nxv4f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 4 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_vs_nxv2f32_nxv4f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv4f16.nxv2f32
 define <vscale x 2 x float> @intrinsic_vfwredusum_mask_vs_nxv2f32_nxv4f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 4 x half> %1, <vscale x 2 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_mask_vs_nxv2f32_nxv4f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv4f16.nxv2f32(
@@ -163,10 +163,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv8f16(
 define <vscale x 2 x float> @intrinsic_vfwredusum_vs_nxv2f32_nxv8f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 8 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_vs_nxv2f32_nxv8f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v10, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv8f16.nxv2f32
 define <vscale x 2 x float> @intrinsic_vfwredusum_mask_vs_nxv2f32_nxv8f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 8 x half> %1, <vscale x 2 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_mask_vs_nxv2f32_nxv8f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v10, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv8f16.nxv2f32(
@@ -213,10 +213,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv16f16(
 define <vscale x 2 x float> @intrinsic_vfwredusum_vs_nxv2f32_nxv16f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 16 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_vs_nxv2f32_nxv16f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v12, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv16f16(
@@ -238,10 +238,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv16f16.nxv2f3
 define <vscale x 2 x float> @intrinsic_vfwredusum_mask_vs_nxv2f32_nxv16f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 16 x half> %1, <vscale x 2 x float> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_mask_vs_nxv2f32_nxv16f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v12, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv16f16.nxv2f32(
@@ -263,10 +263,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv32f16(
 define <vscale x 2 x float> @intrinsic_vfwredusum_vs_nxv2f32_nxv32f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 32 x half> %1, <vscale x 2 x float> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_vs_nxv2f32_nxv32f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v16, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.nxv2f32.nxv32f16(
@@ -288,10 +288,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv32f16(
 define <vscale x 2 x float> @intrinsic_vfwredusum_mask_vs_nxv2f32_nxv32f16_nxv2f32(<vscale x 2 x float> %0, <vscale x 32 x half> %1, <vscale x 2 x float> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_mask_vs_nxv2f32_nxv32f16_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v16, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwredusum.mask.nxv2f32.nxv32f16(
@@ -313,10 +313,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredusum.nxv1f64.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwredusum_vs_nxv1f64_nxv1f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_vs_nxv1f64_nxv1f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredusum.nxv1f64.nxv1f32(
@@ -338,10 +338,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredusum.mask.nxv1f64.nxv1f32.nxv1f6
 define <vscale x 1 x double> @intrinsic_vfwredusum_mask_vs_nxv1f64_nxv1f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_mask_vs_nxv1f64_nxv1f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredusum.mask.nxv1f64.nxv1f32.nxv1f64(
@@ -363,10 +363,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredusum.nxv1f64.nxv2f32(
 define <vscale x 1 x double> @intrinsic_vfwredusum_vs_nxv1f64_nxv2f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 2 x float> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_vs_nxv1f64_nxv2f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v9, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredusum.nxv1f64.nxv2f32(
@@ -388,10 +388,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredusum.mask.nxv1f64.nxv2f32.nxv1f6
 define <vscale x 1 x double> @intrinsic_vfwredusum_mask_vs_nxv1f64_nxv2f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 2 x float> %1, <vscale x 1 x double> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_mask_vs_nxv1f64_nxv2f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredusum.mask.nxv1f64.nxv2f32.nxv1f64(
@@ -413,10 +413,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredusum.nxv1f64.nxv4f32(
 define <vscale x 1 x double> @intrinsic_vfwredusum_vs_nxv1f64_nxv4f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 4 x float> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_vs_nxv1f64_nxv4f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v10, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredusum.nxv1f64.nxv4f32(
@@ -438,10 +438,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredusum.mask.nxv1f64.nxv4f32.nxv1f6
 define <vscale x 1 x double> @intrinsic_vfwredusum_mask_vs_nxv1f64_nxv4f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 4 x float> %1, <vscale x 1 x double> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_mask_vs_nxv1f64_nxv4f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v10, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredusum.mask.nxv1f64.nxv4f32.nxv1f64(
@@ -463,10 +463,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredusum.nxv1f64.nxv8f32(
 define <vscale x 1 x double> @intrinsic_vfwredusum_vs_nxv1f64_nxv8f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 8 x float> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_vs_nxv1f64_nxv8f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v12, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredusum.nxv1f64.nxv8f32(
@@ -488,10 +488,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredusum.mask.nxv1f64.nxv8f32.nxv1f6
 define <vscale x 1 x double> @intrinsic_vfwredusum_mask_vs_nxv1f64_nxv8f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 8 x float> %1, <vscale x 1 x double> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_mask_vs_nxv1f64_nxv8f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v12, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredusum.mask.nxv1f64.nxv8f32.nxv1f64(
@@ -513,10 +513,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredusum.nxv1f64.nxv16f32(
 define <vscale x 1 x double> @intrinsic_vfwredusum_vs_nxv1f64_nxv16f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 16 x float> %1, <vscale x 1 x double> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_vs_nxv1f64_nxv16f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v16, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredusum.nxv1f64.nxv16f32(
@@ -538,10 +538,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwredusum.mask.nxv1f64.nxv16f32.nxv1f
 define <vscale x 1 x double> @intrinsic_vfwredusum_mask_vs_nxv1f64_nxv16f32_nxv1f64(<vscale x 1 x double> %0, <vscale x 16 x float> %1, <vscale x 1 x double> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwredusum_mask_vs_nxv1f64_nxv16f32_nxv1f64:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwredusum.vs v8, v16, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwredusum.mask.nxv1f64.nxv16f32.nxv1f64(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwsub.ll b/llvm/test/CodeGen/RISCV/rvv/vfwsub.ll
index bb72f70f111b..0e3e5f8aabfd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwsub.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwsub.nxv1f32.nxv1f16.nxv1f16(
 define <vscale x 1 x float> @intrinsic_vfwsub_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -39,10 +39,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwsub.mask.nxv1f32.nxv1f16.nxv1f16(
 define <vscale x 1 x float> @intrinsic_vfwsub_mask_vv_nxv1f32_nxv1f16_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv1f32_nxv1f16_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwsub.mask.nxv1f32.nxv1f16.nxv1f16(
@@ -64,10 +64,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwsub.nxv2f32.nxv2f16.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwsub_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -90,10 +90,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwsub.mask.nxv2f32.nxv2f16.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwsub_mask_vv_nxv2f32_nxv2f16_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv2f32_nxv2f16_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwsub.mask.nxv2f32.nxv2f16.nxv2f16(
@@ -115,10 +115,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwsub.nxv4f32.nxv4f16.nxv4f16(
 define <vscale x 4 x float> @intrinsic_vfwsub_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -141,10 +141,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwsub.mask.nxv4f32.nxv4f16.nxv4f16(
 define <vscale x 4 x float> @intrinsic_vfwsub_mask_vv_nxv4f32_nxv4f16_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv4f32_nxv4f16_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwsub.mask.nxv4f32.nxv4f16.nxv4f16(
@@ -166,10 +166,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwsub.nxv8f32.nxv8f16.nxv8f16(
 define <vscale x 8 x float> @intrinsic_vfwsub_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v12, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -192,10 +192,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwsub.mask.nxv8f32.nxv8f16.nxv8f16(
 define <vscale x 8 x float> @intrinsic_vfwsub_mask_vv_nxv8f32_nxv8f16_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv8f32_nxv8f16_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwsub.mask.nxv8f32.nxv8f16.nxv8f16(
@@ -217,10 +217,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwsub.nxv16f32.nxv16f16.nxv16f16(
 define <vscale x 16 x float> @intrinsic_vfwsub_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x half> %0, <vscale x 16 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v16, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -243,10 +243,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwsub.mask.nxv16f32.nxv16f16.nxv16f16
 define <vscale x 16 x float> @intrinsic_vfwsub_mask_vv_nxv16f32_nxv16f16_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x half> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv16f32_nxv16f16_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwsub.mask.nxv16f32.nxv16f16.nxv16f16(
@@ -268,10 +268,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwsub.nxv1f64.nxv1f32.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwsub_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -294,10 +294,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwsub.mask.nxv1f64.nxv1f32.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwsub_mask_vv_nxv1f64_nxv1f32_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv1f64_nxv1f32_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwsub.mask.nxv1f64.nxv1f32.nxv1f32(
@@ -319,10 +319,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwsub.nxv2f64.nxv2f32.nxv2f32(
 define <vscale x 2 x double> @intrinsic_vfwsub_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v10, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -345,10 +345,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwsub.mask.nxv2f64.nxv2f32.nxv2f32(
 define <vscale x 2 x double> @intrinsic_vfwsub_mask_vv_nxv2f64_nxv2f32_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv2f64_nxv2f32_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v8, v10, v11, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwsub.mask.nxv2f64.nxv2f32.nxv2f32(
@@ -370,10 +370,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwsub.nxv4f64.nxv4f32.nxv4f32(
 define <vscale x 4 x double> @intrinsic_vfwsub_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v12, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -396,10 +396,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwsub.mask.nxv4f64.nxv4f32.nxv4f32(
 define <vscale x 4 x double> @intrinsic_vfwsub_mask_vv_nxv4f64_nxv4f32_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv4f64_nxv4f32_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v8, v12, v14, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwsub.mask.nxv4f64.nxv4f32.nxv4f32(
@@ -421,10 +421,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwsub.nxv8f64.nxv8f32.nxv8f32(
 define <vscale x 8 x double> @intrinsic_vfwsub_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v16, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -447,10 +447,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwsub.mask.nxv8f64.nxv8f32.nxv8f32(
 define <vscale x 8 x double> @intrinsic_vfwsub_mask_vv_nxv8f64_nxv8f32_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x float> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv8f64_nxv8f32_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vv v8, v16, v20, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwsub.mask.nxv8f64.nxv8f32.nxv8f32(
@@ -472,10 +472,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwsub.nxv1f32.nxv1f16.f16(
 define <vscale x 1 x float> @intrinsic_vfwsub_vf_nxv1f32_nxv1f16_f16(<vscale x 1 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv1f32_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -498,10 +498,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwsub.mask.nxv1f32.nxv1f16.f16(
 define <vscale x 1 x float> @intrinsic_vfwsub_mask_vf_nxv1f32_nxv1f16_f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv1f32_nxv1f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwsub.mask.nxv1f32.nxv1f16.f16(
@@ -523,10 +523,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwsub.nxv2f32.nxv2f16.f16(
 define <vscale x 2 x float> @intrinsic_vfwsub_vf_nxv2f32_nxv2f16_f16(<vscale x 2 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv2f32_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -549,10 +549,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwsub.mask.nxv2f32.nxv2f16.f16(
 define <vscale x 2 x float> @intrinsic_vfwsub_mask_vf_nxv2f32_nxv2f16_f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv2f32_nxv2f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwsub.mask.nxv2f32.nxv2f16.f16(
@@ -574,10 +574,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwsub.nxv4f32.nxv4f16.f16(
 define <vscale x 4 x float> @intrinsic_vfwsub_vf_nxv4f32_nxv4f16_f16(<vscale x 4 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv4f32_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v10, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -600,10 +600,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwsub.mask.nxv4f32.nxv4f16.f16(
 define <vscale x 4 x float> @intrinsic_vfwsub_mask_vf_nxv4f32_nxv4f16_f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv4f32_nxv4f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwsub.mask.nxv4f32.nxv4f16.f16(
@@ -625,10 +625,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwsub.nxv8f32.nxv8f16.f16(
 define <vscale x 8 x float> @intrinsic_vfwsub_vf_nxv8f32_nxv8f16_f16(<vscale x 8 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv8f32_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v12, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -651,10 +651,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwsub.mask.nxv8f32.nxv8f16.f16(
 define <vscale x 8 x float> @intrinsic_vfwsub_mask_vf_nxv8f32_nxv8f16_f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv8f32_nxv8f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwsub.mask.nxv8f32.nxv8f16.f16(
@@ -676,10 +676,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwsub.nxv16f32.nxv16f16.f16(
 define <vscale x 16 x float> @intrinsic_vfwsub_vf_nxv16f32_nxv16f16_f16(<vscale x 16 x half> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv16f32_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v16, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -702,10 +702,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwsub.mask.nxv16f32.nxv16f16.f16(
 define <vscale x 16 x float> @intrinsic_vfwsub_mask_vf_nxv16f32_nxv16f16_f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv16f32_nxv16f16_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwsub.mask.nxv16f32.nxv16f16.f16(
@@ -727,10 +727,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwsub.nxv1f64.nxv1f32.f32(
 define <vscale x 1 x double> @intrinsic_vfwsub_vf_nxv1f64_nxv1f32_f32(<vscale x 1 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv1f64_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v9, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -753,10 +753,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwsub.mask.nxv1f64.nxv1f32.f32(
 define <vscale x 1 x double> @intrinsic_vfwsub_mask_vf_nxv1f64_nxv1f32_f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv1f64_nxv1f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwsub.mask.nxv1f64.nxv1f32.f32(
@@ -778,10 +778,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwsub.nxv2f64.nxv2f32.f32(
 define <vscale x 2 x double> @intrinsic_vfwsub_vf_nxv2f64_nxv2f32_f32(<vscale x 2 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv2f64_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v10, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -804,10 +804,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwsub.mask.nxv2f64.nxv2f32.f32(
 define <vscale x 2 x double> @intrinsic_vfwsub_mask_vf_nxv2f64_nxv2f32_f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv2f64_nxv2f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwsub.mask.nxv2f64.nxv2f32.f32(
@@ -829,10 +829,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwsub.nxv4f64.nxv4f32.f32(
 define <vscale x 4 x double> @intrinsic_vfwsub_vf_nxv4f64_nxv4f32_f32(<vscale x 4 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv4f64_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v12, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -855,10 +855,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwsub.mask.nxv4f64.nxv4f32.f32(
 define <vscale x 4 x double> @intrinsic_vfwsub_mask_vf_nxv4f64_nxv4f32_f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv4f64_nxv4f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwsub.mask.nxv4f64.nxv4f32.f32(
@@ -880,10 +880,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwsub.nxv8f64.nxv8f32.f32(
 define <vscale x 8 x double> @intrinsic_vfwsub_vf_nxv8f64_nxv8f32_f32(<vscale x 8 x float> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv8f64_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v16, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -906,10 +906,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwsub.mask.nxv8f64.nxv8f32.f32(
 define <vscale x 8 x double> @intrinsic_vfwsub_mask_vf_nxv8f64_nxv8f32_f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv8f64_nxv8f32_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.vf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwsub.mask.nxv8f64.nxv8f32.f32(
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwsub.w.ll b/llvm/test/CodeGen/RISCV/rvv/vfwsub.w.ll
index 722fed5138f7..90f92226dcdd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwsub.w.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwsub.w.ll
@@ -13,10 +13,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwsub.w.nxv1f32.nxv1f16(
 define <vscale x 1 x float> @intrinsic_vfwsub.w_wv_nxv1f32_nxv1f32_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv1f32_nxv1f32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwsub.w.nxv1f32.nxv1f16(
@@ -38,10 +38,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwsub.w.mask.nxv1f32.nxv1f16(
 define <vscale x 1 x float> @intrinsic_vfwsub.w_mask_wv_nxv1f32_nxv1f32_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x float> %1, <vscale x 1 x half> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv1f32_nxv1f32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwsub.w.mask.nxv1f32.nxv1f16(
@@ -63,10 +63,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwsub.w.nxv2f32.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwsub.w_wv_nxv2f32_nxv2f32_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv2f32_nxv2f32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwsub.w.nxv2f32.nxv2f16(
@@ -88,10 +88,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwsub.w.mask.nxv2f32.nxv2f16(
 define <vscale x 2 x float> @intrinsic_vfwsub.w_mask_wv_nxv2f32_nxv2f32_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x float> %1, <vscale x 2 x half> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv2f32_nxv2f32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwsub.w.mask.nxv2f32.nxv2f16(
@@ -113,10 +113,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwsub.w.nxv4f32.nxv4f16(
 define <vscale x 4 x float> @intrinsic_vfwsub.w_wv_nxv4f32_nxv4f32_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv4f32_nxv4f32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwsub.w.nxv4f32.nxv4f16(
@@ -138,10 +138,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwsub.w.mask.nxv4f32.nxv4f16(
 define <vscale x 4 x float> @intrinsic_vfwsub.w_mask_wv_nxv4f32_nxv4f32_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x float> %1, <vscale x 4 x half> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv4f32_nxv4f32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwsub.w.mask.nxv4f32.nxv4f16(
@@ -163,10 +163,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwsub.w.nxv8f32.nxv8f16(
 define <vscale x 8 x float> @intrinsic_vfwsub.w_wv_nxv8f32_nxv8f32_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv8f32_nxv8f32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwsub.w.nxv8f32.nxv8f16(
@@ -188,10 +188,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwsub.w.mask.nxv8f32.nxv8f16(
 define <vscale x 8 x float> @intrinsic_vfwsub.w_mask_wv_nxv8f32_nxv8f32_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x float> %1, <vscale x 8 x half> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv8f32_nxv8f32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwsub.w.mask.nxv8f32.nxv8f16(
@@ -213,10 +213,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwsub.w.nxv16f32.nxv16f16(
 define <vscale x 16 x float> @intrinsic_vfwsub.w_wv_nxv16f32_nxv16f32_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv16f32_nxv16f32_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwsub.w.nxv16f32.nxv16f16(
@@ -239,8 +239,8 @@ define <vscale x 16 x float> @intrinsic_vfwsub.w_mask_wv_nxv16f32_nxv16f32_nxv16
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv16f32_nxv16f32_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl4re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vfwsub.wv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -264,10 +264,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwsub.w.nxv1f64.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwsub.w_wv_nxv1f64_nxv1f64_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv1f64_nxv1f64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v9
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwsub.w.nxv1f64.nxv1f32(
@@ -289,10 +289,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwsub.w.mask.nxv1f64.nxv1f32(
 define <vscale x 1 x double> @intrinsic_vfwsub.w_mask_wv_nxv1f64_nxv1f64_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x double> %1, <vscale x 1 x float> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv1f64_nxv1f64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v9, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwsub.w.mask.nxv1f64.nxv1f32(
@@ -314,10 +314,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwsub.w.nxv2f64.nxv2f32(
 define <vscale x 2 x double> @intrinsic_vfwsub.w_wv_nxv2f64_nxv2f64_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv2f64_nxv2f64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v10
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwsub.w.nxv2f64.nxv2f32(
@@ -339,10 +339,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwsub.w.mask.nxv2f64.nxv2f32(
 define <vscale x 2 x double> @intrinsic_vfwsub.w_mask_wv_nxv2f64_nxv2f64_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x double> %1, <vscale x 2 x float> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv2f64_nxv2f64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v10, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwsub.w.mask.nxv2f64.nxv2f32(
@@ -364,10 +364,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwsub.w.nxv4f64.nxv4f32(
 define <vscale x 4 x double> @intrinsic_vfwsub.w_wv_nxv4f64_nxv4f64_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv4f64_nxv4f64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v12
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwsub.w.nxv4f64.nxv4f32(
@@ -389,10 +389,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwsub.w.mask.nxv4f64.nxv4f32(
 define <vscale x 4 x double> @intrinsic_vfwsub.w_mask_wv_nxv4f64_nxv4f64_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x double> %1, <vscale x 4 x float> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv4f64_nxv4f64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v12, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwsub.w.mask.nxv4f64.nxv4f32(
@@ -414,10 +414,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwsub.w.nxv8f64.nxv8f32(
 define <vscale x 8 x double> @intrinsic_vfwsub.w_wv_nxv8f64_nxv8f64_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv8f64_nxv8f64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v16
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwsub.w.nxv8f64.nxv8f32(
@@ -440,8 +440,8 @@ define <vscale x 8 x double> @intrinsic_vfwsub.w_mask_wv_nxv8f64_nxv8f64_nxv8f32
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv8f64_nxv8f64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl4re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vfwsub.wv v8, v16, v24, v0.t
 ; CHECK-NEXT:    fsrm a0
 ; CHECK-NEXT:    ret
@@ -465,10 +465,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwsub.w.nxv1f32.f16(
 define <vscale x 1 x float> @intrinsic_vfwsub.w_wf_nxv1f32_nxv1f32_f16(<vscale x 1 x float> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv1f32_nxv1f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwsub.w.nxv1f32.f16(
@@ -490,10 +490,10 @@ declare <vscale x 1 x float> @llvm.riscv.vfwsub.w.mask.nxv1f32.f16(
 define <vscale x 1 x float> @intrinsic_vfwsub.w_mask_wf_nxv1f32_nxv1f32_f16(<vscale x 1 x float> %0, <vscale x 1 x float> %1, half %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv1f32_nxv1f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwsub.w.mask.nxv1f32.f16(
@@ -515,10 +515,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwsub.w.nxv2f32.f16(
 define <vscale x 2 x float> @intrinsic_vfwsub.w_wf_nxv2f32_nxv2f32_f16(<vscale x 2 x float> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv2f32_nxv2f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwsub.w.nxv2f32.f16(
@@ -540,10 +540,10 @@ declare <vscale x 2 x float> @llvm.riscv.vfwsub.w.mask.nxv2f32.f16(
 define <vscale x 2 x float> @intrinsic_vfwsub.w_mask_wf_nxv2f32_nxv2f32_f16(<vscale x 2 x float> %0, <vscale x 2 x float> %1, half %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv2f32_nxv2f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwsub.w.mask.nxv2f32.f16(
@@ -565,10 +565,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwsub.w.nxv4f32.f16(
 define <vscale x 4 x float> @intrinsic_vfwsub.w_wf_nxv4f32_nxv4f32_f16(<vscale x 4 x float> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv4f32_nxv4f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwsub.w.nxv4f32.f16(
@@ -590,10 +590,10 @@ declare <vscale x 4 x float> @llvm.riscv.vfwsub.w.mask.nxv4f32.f16(
 define <vscale x 4 x float> @intrinsic_vfwsub.w_mask_wf_nxv4f32_nxv4f32_f16(<vscale x 4 x float> %0, <vscale x 4 x float> %1, half %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv4f32_nxv4f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwsub.w.mask.nxv4f32.f16(
@@ -615,10 +615,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwsub.w.nxv8f32.f16(
 define <vscale x 8 x float> @intrinsic_vfwsub.w_wf_nxv8f32_nxv8f32_f16(<vscale x 8 x float> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv8f32_nxv8f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwsub.w.nxv8f32.f16(
@@ -640,10 +640,10 @@ declare <vscale x 8 x float> @llvm.riscv.vfwsub.w.mask.nxv8f32.f16(
 define <vscale x 8 x float> @intrinsic_vfwsub.w_mask_wf_nxv8f32_nxv8f32_f16(<vscale x 8 x float> %0, <vscale x 8 x float> %1, half %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv8f32_nxv8f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwsub.w.mask.nxv8f32.f16(
@@ -665,10 +665,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwsub.w.nxv16f32.f16(
 define <vscale x 16 x float> @intrinsic_vfwsub.w_wf_nxv16f32_nxv16f32_f16(<vscale x 16 x float> %0, half %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv16f32_nxv16f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwsub.w.nxv16f32.f16(
@@ -690,10 +690,10 @@ declare <vscale x 16 x float> @llvm.riscv.vfwsub.w.mask.nxv16f32.f16(
 define <vscale x 16 x float> @intrinsic_vfwsub.w_mask_wf_nxv16f32_nxv16f32_f16(<vscale x 16 x float> %0, <vscale x 16 x float> %1, half %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv16f32_nxv16f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwsub.w.mask.nxv16f32.f16(
@@ -715,10 +715,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwsub.w.nxv1f64.f32(
 define <vscale x 1 x double> @intrinsic_vfwsub.w_wf_nxv1f64_nxv1f64_f32(<vscale x 1 x double> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv1f64_nxv1f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwsub.w.nxv1f64.f32(
@@ -740,10 +740,10 @@ declare <vscale x 1 x double> @llvm.riscv.vfwsub.w.mask.nxv1f64.f32(
 define <vscale x 1 x double> @intrinsic_vfwsub.w_mask_wf_nxv1f64_nxv1f64_f32(<vscale x 1 x double> %0, <vscale x 1 x double> %1, float %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv1f64_nxv1f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v9, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwsub.w.mask.nxv1f64.f32(
@@ -765,10 +765,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwsub.w.nxv2f64.f32(
 define <vscale x 2 x double> @intrinsic_vfwsub.w_wf_nxv2f64_nxv2f64_f32(<vscale x 2 x double> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv2f64_nxv2f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwsub.w.nxv2f64.f32(
@@ -790,10 +790,10 @@ declare <vscale x 2 x double> @llvm.riscv.vfwsub.w.mask.nxv2f64.f32(
 define <vscale x 2 x double> @intrinsic_vfwsub.w_mask_wf_nxv2f64_nxv2f64_f32(<vscale x 2 x double> %0, <vscale x 2 x double> %1, float %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv2f64_nxv2f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v10, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwsub.w.mask.nxv2f64.f32(
@@ -815,10 +815,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwsub.w.nxv4f64.f32(
 define <vscale x 4 x double> @intrinsic_vfwsub.w_wf_nxv4f64_nxv4f64_f32(<vscale x 4 x double> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv4f64_nxv4f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwsub.w.nxv4f64.f32(
@@ -840,10 +840,10 @@ declare <vscale x 4 x double> @llvm.riscv.vfwsub.w.mask.nxv4f64.f32(
 define <vscale x 4 x double> @intrinsic_vfwsub.w_mask_wf_nxv4f64_nxv4f64_f32(<vscale x 4 x double> %0, <vscale x 4 x double> %1, float %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv4f64_nxv4f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v12, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwsub.w.mask.nxv4f64.f32(
@@ -865,10 +865,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwsub.w.nxv8f64.f32(
 define <vscale x 8 x double> @intrinsic_vfwsub.w_wf_nxv8f64_nxv8f64_f32(<vscale x 8 x double> %0, float %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv8f64_nxv8f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwsub.w.nxv8f64.f32(
@@ -890,10 +890,10 @@ declare <vscale x 8 x double> @llvm.riscv.vfwsub.w.mask.nxv8f64.f32(
 define <vscale x 8 x double> @intrinsic_vfwsub.w_mask_wf_nxv8f64_nxv8f64_f32(<vscale x 8 x double> %0, <vscale x 8 x double> %1, float %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv8f64_nxv8f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v16, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwsub.w.mask.nxv8f64.f32(
@@ -909,10 +909,10 @@ entry:
 define <vscale x 1 x float> @intrinsic_vfwsub.w_mask_wv_tie_nxv1f32_nxv1f32_nxv1f16(<vscale x 1 x float> %0, <vscale x 1 x half> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_tie_nxv1f32_nxv1f32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwsub.w.mask.nxv1f32.nxv1f16(
@@ -928,10 +928,10 @@ entry:
 define <vscale x 2 x float> @intrinsic_vfwsub.w_mask_wv_tie_nxv2f32_nxv2f32_nxv2f16(<vscale x 2 x float> %0, <vscale x 2 x half> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_tie_nxv2f32_nxv2f32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwsub.w.mask.nxv2f32.nxv2f16(
@@ -947,10 +947,10 @@ entry:
 define <vscale x 4 x float> @intrinsic_vfwsub.w_mask_wv_tie_nxv4f32_nxv4f32_nxv4f16(<vscale x 4 x float> %0, <vscale x 4 x half> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_tie_nxv4f32_nxv4f32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwsub.w.mask.nxv4f32.nxv4f16(
@@ -966,10 +966,10 @@ entry:
 define <vscale x 8 x float> @intrinsic_vfwsub.w_mask_wv_tie_nxv8f32_nxv8f32_nxv8f16(<vscale x 8 x float> %0, <vscale x 8 x half> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_tie_nxv8f32_nxv8f32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwsub.w.mask.nxv8f32.nxv8f16(
@@ -985,10 +985,10 @@ entry:
 define <vscale x 16 x float> @intrinsic_vfwsub.w_mask_wv_tie_nxv16f32_nxv16f32_nxv16f16(<vscale x 16 x float> %0, <vscale x 16 x half> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_tie_nxv16f32_nxv16f32_nxv16f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwsub.w.mask.nxv16f32.nxv16f16(
@@ -1004,10 +1004,10 @@ entry:
 define <vscale x 1 x double> @intrinsic_vfwsub.w_mask_wv_tie_nxv1f64_nxv1f64_nxv1f32(<vscale x 1 x double> %0, <vscale x 1 x float> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_tie_nxv1f64_nxv1f64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v9, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwsub.w.mask.nxv1f64.nxv1f32(
@@ -1023,10 +1023,10 @@ entry:
 define <vscale x 2 x double> @intrinsic_vfwsub.w_mask_wv_tie_nxv2f64_nxv2f64_nxv2f32(<vscale x 2 x double> %0, <vscale x 2 x float> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_tie_nxv2f64_nxv2f64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v10, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwsub.w.mask.nxv2f64.nxv2f32(
@@ -1042,10 +1042,10 @@ entry:
 define <vscale x 4 x double> @intrinsic_vfwsub.w_mask_wv_tie_nxv4f64_nxv4f64_nxv4f32(<vscale x 4 x double> %0, <vscale x 4 x float> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_tie_nxv4f64_nxv4f64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v12, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwsub.w.mask.nxv4f64.nxv4f32(
@@ -1061,10 +1061,10 @@ entry:
 define <vscale x 8 x double> @intrinsic_vfwsub.w_mask_wv_tie_nxv8f64_nxv8f64_nxv8f32(<vscale x 8 x double> %0, <vscale x 8 x float> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_tie_nxv8f64_nxv8f64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v8, v8, v16, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwsub.w.mask.nxv8f64.nxv8f32(
@@ -1080,10 +1080,10 @@ entry:
 define <vscale x 1 x float> @intrinsic_vfwsub.w_mask_wf_tie_nxv1f32_nxv1f32_f16(<vscale x 1 x float> %0, half %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_tie_nxv1f32_nxv1f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x float> @llvm.riscv.vfwsub.w.mask.nxv1f32.f16(
@@ -1099,10 +1099,10 @@ entry:
 define <vscale x 2 x float> @intrinsic_vfwsub.w_mask_wf_tie_nxv2f32_nxv2f32_f16(<vscale x 2 x float> %0, half %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_tie_nxv2f32_nxv2f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x float> @llvm.riscv.vfwsub.w.mask.nxv2f32.f16(
@@ -1118,10 +1118,10 @@ entry:
 define <vscale x 4 x float> @intrinsic_vfwsub.w_mask_wf_tie_nxv4f32_nxv4f32_f16(<vscale x 4 x float> %0, half %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_tie_nxv4f32_nxv4f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x float> @llvm.riscv.vfwsub.w.mask.nxv4f32.f16(
@@ -1137,10 +1137,10 @@ entry:
 define <vscale x 8 x float> @intrinsic_vfwsub.w_mask_wf_tie_nxv8f32_nxv8f32_f16(<vscale x 8 x float> %0, half %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_tie_nxv8f32_nxv8f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x float> @llvm.riscv.vfwsub.w.mask.nxv8f32.f16(
@@ -1156,10 +1156,10 @@ entry:
 define <vscale x 16 x float> @intrinsic_vfwsub.w_mask_wf_tie_nxv16f32_nxv16f32_f16(<vscale x 16 x float> %0, half %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_tie_nxv16f32_nxv16f32_f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 16 x float> @llvm.riscv.vfwsub.w.mask.nxv16f32.f16(
@@ -1175,10 +1175,10 @@ entry:
 define <vscale x 1 x double> @intrinsic_vfwsub.w_mask_wf_tie_nxv1f64_nxv1f64_f32(<vscale x 1 x double> %0, float %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_tie_nxv1f64_nxv1f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 1 x double> @llvm.riscv.vfwsub.w.mask.nxv1f64.f32(
@@ -1194,10 +1194,10 @@ entry:
 define <vscale x 2 x double> @intrinsic_vfwsub.w_mask_wf_tie_nxv2f64_nxv2f64_f32(<vscale x 2 x double> %0, float %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_tie_nxv2f64_nxv2f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 2 x double> @llvm.riscv.vfwsub.w.mask.nxv2f64.f32(
@@ -1213,10 +1213,10 @@ entry:
 define <vscale x 4 x double> @intrinsic_vfwsub.w_mask_wf_tie_nxv4f64_nxv4f64_f32(<vscale x 4 x double> %0, float %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_tie_nxv4f64_nxv4f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 4 x double> @llvm.riscv.vfwsub.w.mask.nxv4f64.f32(
@@ -1232,10 +1232,10 @@ entry:
 define <vscale x 8 x double> @intrinsic_vfwsub.w_mask_wf_tie_nxv8f64_nxv8f64_f32(<vscale x 8 x double> %0, float %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_tie_nxv8f64_nxv8f64_f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wf v8, v8, fa0, v0.t
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    ret
 entry:
   %a = call <vscale x 8 x double> @llvm.riscv.vfwsub.w.mask.nxv8f64.f32(
@@ -1251,10 +1251,10 @@ entry:
 define <vscale x 1 x float> @intrinsic_vfwsub.w_wv_untie_nxv1f32_nxv1f32_nxv1f16(<vscale x 1 x half> %0, <vscale x 1 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_untie_nxv1f32_nxv1f32_nxv1f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v10, v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -1270,10 +1270,10 @@ entry:
 define <vscale x 2 x float> @intrinsic_vfwsub.w_wv_untie_nxv2f32_nxv2f32_nxv2f16(<vscale x 2 x half> %0, <vscale x 2 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_untie_nxv2f32_nxv2f32_nxv2f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v10, v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -1289,10 +1289,10 @@ entry:
 define <vscale x 4 x float> @intrinsic_vfwsub.w_wv_untie_nxv4f32_nxv4f32_nxv4f16(<vscale x 4 x half> %0, <vscale x 4 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_untie_nxv4f32_nxv4f32_nxv4f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v12, v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -1308,10 +1308,10 @@ entry:
 define <vscale x 8 x float> @intrinsic_vfwsub.w_wv_untie_nxv8f32_nxv8f32_nxv8f16(<vscale x 8 x half> %0, <vscale x 8 x float> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_untie_nxv8f32_nxv8f32_nxv8f16:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v16, v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -1327,10 +1327,10 @@ entry:
 define <vscale x 1 x double> @intrinsic_vfwsub.w_wv_untie_nxv1f64_nxv1f64_nxv1f32(<vscale x 1 x float> %0, <vscale x 1 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_untie_nxv1f64_nxv1f64_nxv1f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v10, v9, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -1346,10 +1346,10 @@ entry:
 define <vscale x 2 x double> @intrinsic_vfwsub.w_wv_untie_nxv2f64_nxv2f64_nxv2f32(<vscale x 2 x float> %0, <vscale x 2 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_untie_nxv2f64_nxv2f64_nxv2f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v12, v10, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv2r.v v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -1365,10 +1365,10 @@ entry:
 define <vscale x 4 x double> @intrinsic_vfwsub.w_wv_untie_nxv4f64_nxv4f64_nxv4f32(<vscale x 4 x float> %0, <vscale x 4 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_untie_nxv4f64_nxv4f64_nxv4f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v16, v12, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv4r.v v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -1384,10 +1384,10 @@ entry:
 define <vscale x 8 x double> @intrinsic_vfwsub.w_wv_untie_nxv8f64_nxv8f64_nxv8f32(<vscale x 8 x float> %0, <vscale x 8 x double> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_untie_nxv8f64_nxv8f64_nxv8f32:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fsrmi a1, 0
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    fsrmi a0, 0
 ; CHECK-NEXT:    vfwsub.wv v24, v16, v8
-; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    fsrm a1
 ; CHECK-NEXT:    vmv8r.v v8, v24
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnclip.ll b/llvm/test/CodeGen/RISCV/rvv/vnclip.ll
index 54f4c17dd7ed..8902b1a28f8c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnclip.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnclip.ll
@@ -13,8 +13,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vnclip_wv_nxv1i8_nxv1i16_nxv1i8(<vscale x 1 x i16> %0, <vscale x 1 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv1i8_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vnclip.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -37,8 +37,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vnclip_mask_wv_nxv1i8_nxv1i16_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv1i8_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -61,8 +61,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vnclip_wv_nxv2i8_nxv2i16_nxv2i8(<vscale x 2 x i16> %0, <vscale x 2 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv2i8_nxv2i16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclip.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -85,8 +85,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vnclip_mask_wv_nxv2i8_nxv2i16_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i8> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv2i8_nxv2i16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -109,8 +109,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vnclip_wv_nxv4i8_nxv4i16_nxv4i8(<vscale x 4 x i16> %0, <vscale x 4 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv4i8_nxv4i16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclip.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -133,8 +133,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vnclip_mask_wv_nxv4i8_nxv4i16_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i8> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv4i8_nxv4i16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -157,8 +157,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vnclip_wv_nxv8i8_nxv8i16_nxv8i8(<vscale x 8 x i16> %0, <vscale x 8 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv8i8_nxv8i16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vnclip.wv v11, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v11
 ; CHECK-NEXT:    ret
@@ -182,8 +182,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vnclip_mask_wv_nxv8i8_nxv8i16_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i8> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv8i8_nxv8i16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v10, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -206,8 +206,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vnclip_wv_nxv16i8_nxv16i16_nxv16i8(<vscale x 16 x i16> %0, <vscale x 16 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv16i8_nxv16i16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vnclip.wv v14, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v14
 ; CHECK-NEXT:    ret
@@ -231,8 +231,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vnclip_mask_wv_nxv16i8_nxv16i16_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i8> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv16i8_nxv16i16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v12, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -255,8 +255,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vnclip_wv_nxv32i8_nxv32i16_nxv32i8(<vscale x 32 x i16> %0, <vscale x 32 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv32i8_nxv32i16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vnclip.wv v20, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v20
 ; CHECK-NEXT:    ret
@@ -280,8 +280,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vnclip_mask_wv_nxv32i8_nxv32i16_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i8> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv32i8_nxv32i16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v16, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -304,8 +304,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vnclip_wv_nxv1i16_nxv1i32_nxv1i16(<vscale x 1 x i32> %0, <vscale x 1 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv1i16_nxv1i32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vnclip.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -328,8 +328,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vnclip_mask_wv_nxv1i16_nxv1i32_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i16> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv1i16_nxv1i32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -352,8 +352,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vnclip_wv_nxv2i16_nxv2i32_nxv2i16(<vscale x 2 x i32> %0, <vscale x 2 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv2i16_nxv2i32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclip.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -376,8 +376,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vnclip_mask_wv_nxv2i16_nxv2i32_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i16> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv2i16_nxv2i32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -400,8 +400,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vnclip_wv_nxv4i16_nxv4i32_nxv4i16(<vscale x 4 x i32> %0, <vscale x 4 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv4i16_nxv4i32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclip.wv v11, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v11
 ; CHECK-NEXT:    ret
@@ -425,8 +425,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vnclip_mask_wv_nxv4i16_nxv4i32_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i16> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv4i16_nxv4i32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v10, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -449,8 +449,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vnclip_wv_nxv8i16_nxv8i32_nxv8i16(<vscale x 8 x i32> %0, <vscale x 8 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv8i16_nxv8i32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vnclip.wv v14, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v14
 ; CHECK-NEXT:    ret
@@ -474,8 +474,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vnclip_mask_wv_nxv8i16_nxv8i32_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i16> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv8i16_nxv8i32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v12, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -498,8 +498,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vnclip_wv_nxv16i16_nxv16i32_nxv16i16(<vscale x 16 x i32> %0, <vscale x 16 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv16i16_nxv16i32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vnclip.wv v20, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v20
 ; CHECK-NEXT:    ret
@@ -523,8 +523,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vnclip_mask_wv_nxv16i16_nxv16i32_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i16> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv16i16_nxv16i32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v16, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -547,8 +547,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vnclip.nxv1i32.nxv1i64.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vnclip_wv_nxv1i32_nxv1i64_nxv1i32(<vscale x 1 x i64> %0, <vscale x 1 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv1i32_nxv1i64_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vnclip.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -571,8 +571,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vnclip.mask.nxv1i32.nxv1i64.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vnclip_mask_wv_nxv1i32_nxv1i64_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i32> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv1i32_nxv1i64_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -595,8 +595,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vnclip.nxv2i32.nxv2i64.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vnclip_wv_nxv2i32_nxv2i64_nxv2i32(<vscale x 2 x i64> %0, <vscale x 2 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv2i32_nxv2i64_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclip.wv v11, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v11
 ; CHECK-NEXT:    ret
@@ -620,8 +620,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vnclip.mask.nxv2i32.nxv2i64.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vnclip_mask_wv_nxv2i32_nxv2i64_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i32> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv2i32_nxv2i64_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v10, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -644,8 +644,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vnclip.nxv4i32.nxv4i64.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vnclip_wv_nxv4i32_nxv4i64_nxv4i32(<vscale x 4 x i64> %0, <vscale x 4 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv4i32_nxv4i64_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclip.wv v14, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v14
 ; CHECK-NEXT:    ret
@@ -669,8 +669,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vnclip.mask.nxv4i32.nxv4i64.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vnclip_mask_wv_nxv4i32_nxv4i64_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i32> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv4i32_nxv4i64_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v12, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -693,8 +693,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vnclip.nxv8i32.nxv8i64.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vnclip_wv_nxv8i32_nxv8i64_nxv8i32(<vscale x 8 x i64> %0, <vscale x 8 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_wv_nxv8i32_nxv8i64_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vnclip.wv v20, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v20
 ; CHECK-NEXT:    ret
@@ -718,8 +718,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vnclip.mask.nxv8i32.nxv8i64.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vnclip_mask_wv_nxv8i32_nxv8i64_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i32> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_wv_nxv8i32_nxv8i64_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vnclip.wv v8, v16, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -741,8 +741,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclip.nxv1i8.nxv1i16(
 define <vscale x 1 x i8> @intrinsic_vnclip_vx_nxv1i8_nxv1i16(<vscale x 1 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vnclip.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -765,8 +765,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclip.mask.nxv1i8.nxv1i16(
 define <vscale x 1 x i8> @intrinsic_vnclip_mask_vx_nxv1i8_nxv1i16(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, iXLen %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -788,8 +788,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vnclip.nxv2i8.nxv2i16(
 define <vscale x 2 x i8> @intrinsic_vnclip_vx_nxv2i8_nxv2i16(<vscale x 2 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclip.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -812,8 +812,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vnclip.mask.nxv2i8.nxv2i16(
 define <vscale x 2 x i8> @intrinsic_vnclip_mask_vx_nxv2i8_nxv2i16(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, iXLen %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -835,8 +835,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vnclip.nxv4i8.nxv4i16(
 define <vscale x 4 x i8> @intrinsic_vnclip_vx_nxv4i8_nxv4i16(<vscale x 4 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclip.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -859,8 +859,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vnclip.mask.nxv4i8.nxv4i16(
 define <vscale x 4 x i8> @intrinsic_vnclip_mask_vx_nxv4i8_nxv4i16(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, iXLen %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -882,8 +882,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vnclip.nxv8i8.nxv8i16(
 define <vscale x 8 x i8> @intrinsic_vnclip_vx_nxv8i8_nxv8i16(<vscale x 8 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vnclip.wx v10, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -907,8 +907,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vnclip.mask.nxv8i8.nxv8i16(
 define <vscale x 8 x i8> @intrinsic_vnclip_mask_vx_nxv8i8_nxv8i16(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, iXLen %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -930,8 +930,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vnclip.nxv16i8.nxv16i16(
 define <vscale x 16 x i8> @intrinsic_vnclip_vx_nxv16i8_nxv16i16(<vscale x 16 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vnclip.wx v12, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -955,8 +955,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vnclip.mask.nxv16i8.nxv16i16(
 define <vscale x 16 x i8> @intrinsic_vnclip_mask_vx_nxv16i8_nxv16i16(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, iXLen %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -978,8 +978,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vnclip.nxv32i8.nxv32i16(
 define <vscale x 32 x i8> @intrinsic_vnclip_vx_nxv32i8_nxv32i16(<vscale x 32 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vnclip.wx v16, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1003,8 +1003,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vnclip.mask.nxv32i8.nxv32i16(
 define <vscale x 32 x i8> @intrinsic_vnclip_mask_vx_nxv32i8_nxv32i16(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, iXLen %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1026,8 +1026,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vnclip.nxv1i16.nxv1i32(
 define <vscale x 1 x i16> @intrinsic_vnclip_vx_nxv1i16_nxv1i32(<vscale x 1 x i32> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vnclip.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1050,8 +1050,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vnclip.mask.nxv1i16.nxv1i32(
 define <vscale x 1 x i16> @intrinsic_vnclip_mask_vx_nxv1i16_nxv1i32(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, iXLen %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1073,8 +1073,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vnclip.nxv2i16.nxv2i32(
 define <vscale x 2 x i16> @intrinsic_vnclip_vx_nxv2i16_nxv2i32(<vscale x 2 x i32> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclip.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1097,8 +1097,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vnclip.mask.nxv2i16.nxv2i32(
 define <vscale x 2 x i16> @intrinsic_vnclip_mask_vx_nxv2i16_nxv2i32(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, iXLen %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1120,8 +1120,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vnclip.nxv4i16.nxv4i32(
 define <vscale x 4 x i16> @intrinsic_vnclip_vx_nxv4i16_nxv4i32(<vscale x 4 x i32> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclip.wx v10, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -1145,8 +1145,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vnclip.mask.nxv4i16.nxv4i32(
 define <vscale x 4 x i16> @intrinsic_vnclip_mask_vx_nxv4i16_nxv4i32(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, iXLen %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1168,8 +1168,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vnclip.nxv8i16.nxv8i32(
 define <vscale x 8 x i16> @intrinsic_vnclip_vx_nxv8i16_nxv8i32(<vscale x 8 x i32> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vnclip.wx v12, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -1193,8 +1193,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vnclip.mask.nxv8i16.nxv8i32(
 define <vscale x 8 x i16> @intrinsic_vnclip_mask_vx_nxv8i16_nxv8i32(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, iXLen %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1216,8 +1216,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vnclip.nxv16i16.nxv16i32(
 define <vscale x 16 x i16> @intrinsic_vnclip_vx_nxv16i16_nxv16i32(<vscale x 16 x i32> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vnclip.wx v16, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1241,8 +1241,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vnclip.mask.nxv16i16.nxv16i32(
 define <vscale x 16 x i16> @intrinsic_vnclip_mask_vx_nxv16i16_nxv16i32(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, iXLen %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1264,8 +1264,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vnclip.nxv1i32.nxv1i64(
 define <vscale x 1 x i32> @intrinsic_vnclip_vx_nxv1i32_nxv1i64(<vscale x 1 x i64> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vnclip.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1288,8 +1288,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vnclip.mask.nxv1i32.nxv1i64(
 define <vscale x 1 x i32> @intrinsic_vnclip_mask_vx_nxv1i32_nxv1i64(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, iXLen %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1311,8 +1311,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vnclip.nxv2i32.nxv2i64(
 define <vscale x 2 x i32> @intrinsic_vnclip_vx_nxv2i32_nxv2i64(<vscale x 2 x i64> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclip.wx v10, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -1336,8 +1336,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vnclip.mask.nxv2i32.nxv2i64(
 define <vscale x 2 x i32> @intrinsic_vnclip_mask_vx_nxv2i32_nxv2i64(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, iXLen %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1359,8 +1359,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vnclip.nxv4i32.nxv4i64(
 define <vscale x 4 x i32> @intrinsic_vnclip_vx_nxv4i32_nxv4i64(<vscale x 4 x i64> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclip.wx v12, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -1384,8 +1384,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vnclip.mask.nxv4i32.nxv4i64(
 define <vscale x 4 x i32> @intrinsic_vnclip_mask_vx_nxv4i32_nxv4i64(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, iXLen %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1407,8 +1407,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vnclip.nxv8i32.nxv8i64(
 define <vscale x 8 x i32> @intrinsic_vnclip_vx_nxv8i32_nxv8i64(<vscale x 8 x i64> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vnclip.wx v16, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1432,8 +1432,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vnclip.mask.nxv8i32.nxv8i64(
 define <vscale x 8 x i32> @intrinsic_vnclip_mask_vx_nxv8i32_nxv8i64(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, iXLen %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vnclip.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1450,8 +1450,8 @@ entry:
 define <vscale x 1 x i8> @intrinsic_vnclip_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vnclip.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1467,8 +1467,8 @@ entry:
 define <vscale x 1 x i8> @intrinsic_vnclip_mask_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1485,8 +1485,8 @@ entry:
 define <vscale x 2 x i8> @intrinsic_vnclip_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclip.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1502,8 +1502,8 @@ entry:
 define <vscale x 2 x i8> @intrinsic_vnclip_mask_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1520,8 +1520,8 @@ entry:
 define <vscale x 4 x i8> @intrinsic_vnclip_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclip.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1537,8 +1537,8 @@ entry:
 define <vscale x 4 x i8> @intrinsic_vnclip_mask_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1555,8 +1555,8 @@ entry:
 define <vscale x 8 x i8> @intrinsic_vnclip_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vnclip.wi v10, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -1573,8 +1573,8 @@ entry:
 define <vscale x 8 x i8> @intrinsic_vnclip_mask_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1591,8 +1591,8 @@ entry:
 define <vscale x 16 x i8> @intrinsic_vnclip_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vnclip.wi v12, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -1609,8 +1609,8 @@ entry:
 define <vscale x 16 x i8> @intrinsic_vnclip_mask_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1627,8 +1627,8 @@ entry:
 define <vscale x 32 x i8> @intrinsic_vnclip_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vnclip.wi v16, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1645,8 +1645,8 @@ entry:
 define <vscale x 32 x i8> @intrinsic_vnclip_mask_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1663,8 +1663,8 @@ entry:
 define <vscale x 1 x i16> @intrinsic_vnclip_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vnclip.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1680,8 +1680,8 @@ entry:
 define <vscale x 1 x i16> @intrinsic_vnclip_mask_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1698,8 +1698,8 @@ entry:
 define <vscale x 2 x i16> @intrinsic_vnclip_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclip.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1715,8 +1715,8 @@ entry:
 define <vscale x 2 x i16> @intrinsic_vnclip_mask_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1733,8 +1733,8 @@ entry:
 define <vscale x 4 x i16> @intrinsic_vnclip_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclip.wi v10, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -1751,8 +1751,8 @@ entry:
 define <vscale x 4 x i16> @intrinsic_vnclip_mask_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1769,8 +1769,8 @@ entry:
 define <vscale x 8 x i16> @intrinsic_vnclip_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vnclip.wi v12, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -1787,8 +1787,8 @@ entry:
 define <vscale x 8 x i16> @intrinsic_vnclip_mask_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1805,8 +1805,8 @@ entry:
 define <vscale x 16 x i16> @intrinsic_vnclip_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vnclip.wi v16, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1823,8 +1823,8 @@ entry:
 define <vscale x 16 x i16> @intrinsic_vnclip_mask_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1841,8 +1841,8 @@ entry:
 define <vscale x 1 x i32> @intrinsic_vnclip_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vnclip.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1858,8 +1858,8 @@ entry:
 define <vscale x 1 x i32> @intrinsic_vnclip_mask_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1876,8 +1876,8 @@ entry:
 define <vscale x 2 x i32> @intrinsic_vnclip_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclip.wi v10, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -1894,8 +1894,8 @@ entry:
 define <vscale x 2 x i32> @intrinsic_vnclip_mask_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1912,8 +1912,8 @@ entry:
 define <vscale x 4 x i32> @intrinsic_vnclip_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclip.wi v12, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -1930,8 +1930,8 @@ entry:
 define <vscale x 4 x i32> @intrinsic_vnclip_mask_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1948,8 +1948,8 @@ entry:
 define <vscale x 8 x i32> @intrinsic_vnclip_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vnclip.wi v16, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1966,8 +1966,8 @@ entry:
 define <vscale x 8 x i32> @intrinsic_vnclip_mask_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclip_mask_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vnclip.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vnclipu.ll b/llvm/test/CodeGen/RISCV/rvv/vnclipu.ll
index 39980504f887..a1804e7d98a4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vnclipu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vnclipu.ll
@@ -13,8 +13,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vnclipu_wv_nxv1i8_nxv1i16_nxv1i8(<vscale x 1 x i16> %0, <vscale x 1 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv1i8_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -37,8 +37,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vnclipu_mask_wv_nxv1i8_nxv1i16_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv1i8_nxv1i16_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -61,8 +61,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vnclipu_wv_nxv2i8_nxv2i16_nxv2i8(<vscale x 2 x i16> %0, <vscale x 2 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv2i8_nxv2i16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -85,8 +85,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vnclipu_mask_wv_nxv2i8_nxv2i16_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i8> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv2i8_nxv2i16_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -109,8 +109,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vnclipu_wv_nxv4i8_nxv4i16_nxv4i8(<vscale x 4 x i16> %0, <vscale x 4 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv4i8_nxv4i16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -133,8 +133,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vnclipu_mask_wv_nxv4i8_nxv4i16_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i8> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv4i8_nxv4i16_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -157,8 +157,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vnclipu_wv_nxv8i8_nxv8i16_nxv8i8(<vscale x 8 x i16> %0, <vscale x 8 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv8i8_nxv8i16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v11, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v11
 ; CHECK-NEXT:    ret
@@ -182,8 +182,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vnclipu_mask_wv_nxv8i8_nxv8i16_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i8> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv8i8_nxv8i16_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v10, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -206,8 +206,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vnclipu_wv_nxv16i8_nxv16i16_nxv16i8(<vscale x 16 x i16> %0, <vscale x 16 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv16i8_nxv16i16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v14, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v14
 ; CHECK-NEXT:    ret
@@ -231,8 +231,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vnclipu_mask_wv_nxv16i8_nxv16i16_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i8> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv16i8_nxv16i16_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v12, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -255,8 +255,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vnclipu_wv_nxv32i8_nxv32i16_nxv32i8(<vscale x 32 x i16> %0, <vscale x 32 x i8> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv32i8_nxv32i16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v20, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v20
 ; CHECK-NEXT:    ret
@@ -280,8 +280,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vnclipu_mask_wv_nxv32i8_nxv32i16_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i8> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv32i8_nxv32i16_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v16, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -304,8 +304,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vnclipu_wv_nxv1i16_nxv1i32_nxv1i16(<vscale x 1 x i32> %0, <vscale x 1 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv1i16_nxv1i32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -328,8 +328,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vnclipu_mask_wv_nxv1i16_nxv1i32_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i16> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv1i16_nxv1i32_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -352,8 +352,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vnclipu_wv_nxv2i16_nxv2i32_nxv2i16(<vscale x 2 x i32> %0, <vscale x 2 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv2i16_nxv2i32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -376,8 +376,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vnclipu_mask_wv_nxv2i16_nxv2i32_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i16> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv2i16_nxv2i32_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -400,8 +400,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vnclipu_wv_nxv4i16_nxv4i32_nxv4i16(<vscale x 4 x i32> %0, <vscale x 4 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv4i16_nxv4i32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v11, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v11
 ; CHECK-NEXT:    ret
@@ -425,8 +425,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vnclipu_mask_wv_nxv4i16_nxv4i32_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i16> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv4i16_nxv4i32_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v10, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -449,8 +449,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vnclipu_wv_nxv8i16_nxv8i32_nxv8i16(<vscale x 8 x i32> %0, <vscale x 8 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv8i16_nxv8i32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v14, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v14
 ; CHECK-NEXT:    ret
@@ -474,8 +474,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vnclipu_mask_wv_nxv8i16_nxv8i32_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i16> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv8i16_nxv8i32_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v12, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -498,8 +498,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vnclipu_wv_nxv16i16_nxv16i32_nxv16i16(<vscale x 16 x i32> %0, <vscale x 16 x i16> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv16i16_nxv16i32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v20, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v20
 ; CHECK-NEXT:    ret
@@ -523,8 +523,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vnclipu_mask_wv_nxv16i16_nxv16i32_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i16> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv16i16_nxv16i32_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v16, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -547,8 +547,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vnclipu.nxv1i32.nxv1i64.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vnclipu_wv_nxv1i32_nxv1i64_nxv1i32(<vscale x 1 x i64> %0, <vscale x 1 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv1i32_nxv1i64_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -571,8 +571,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vnclipu.mask.nxv1i32.nxv1i64.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vnclipu_mask_wv_nxv1i32_nxv1i64_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i32> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv1i32_nxv1i64_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -595,8 +595,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vnclipu.nxv2i32.nxv2i64.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vnclipu_wv_nxv2i32_nxv2i64_nxv2i32(<vscale x 2 x i64> %0, <vscale x 2 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv2i32_nxv2i64_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v11, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v11
 ; CHECK-NEXT:    ret
@@ -620,8 +620,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vnclipu.mask.nxv2i32.nxv2i64.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vnclipu_mask_wv_nxv2i32_nxv2i64_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i32> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv2i32_nxv2i64_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v10, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -644,8 +644,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vnclipu.nxv4i32.nxv4i64.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vnclipu_wv_nxv4i32_nxv4i64_nxv4i32(<vscale x 4 x i64> %0, <vscale x 4 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv4i32_nxv4i64_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v14, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v14
 ; CHECK-NEXT:    ret
@@ -669,8 +669,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vnclipu.mask.nxv4i32.nxv4i64.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vnclipu_mask_wv_nxv4i32_nxv4i64_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i32> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv4i32_nxv4i64_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v12, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -693,8 +693,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vnclipu.nxv8i32.nxv8i64.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vnclipu_wv_nxv8i32_nxv8i64_nxv8i32(<vscale x 8 x i64> %0, <vscale x 8 x i32> %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_wv_nxv8i32_nxv8i64_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vnclipu.wv v20, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v20
 ; CHECK-NEXT:    ret
@@ -718,8 +718,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vnclipu.mask.nxv8i32.nxv8i64.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vnclipu_mask_wv_nxv8i32_nxv8i64_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i32> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_wv_nxv8i32_nxv8i64_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vnclipu.wv v8, v16, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -741,8 +741,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclipu.nxv1i8.nxv1i16(
 define <vscale x 1 x i8> @intrinsic_vnclipu_vx_nxv1i8_nxv1i16(<vscale x 1 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -765,8 +765,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vnclipu.mask.nxv1i8.nxv1i16(
 define <vscale x 1 x i8> @intrinsic_vnclipu_mask_vx_nxv1i8_nxv1i16(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, iXLen %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv1i8_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -788,8 +788,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vnclipu.nxv2i8.nxv2i16(
 define <vscale x 2 x i8> @intrinsic_vnclipu_vx_nxv2i8_nxv2i16(<vscale x 2 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -812,8 +812,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vnclipu.mask.nxv2i8.nxv2i16(
 define <vscale x 2 x i8> @intrinsic_vnclipu_mask_vx_nxv2i8_nxv2i16(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, iXLen %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv2i8_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -835,8 +835,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vnclipu.nxv4i8.nxv4i16(
 define <vscale x 4 x i8> @intrinsic_vnclipu_vx_nxv4i8_nxv4i16(<vscale x 4 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -859,8 +859,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vnclipu.mask.nxv4i8.nxv4i16(
 define <vscale x 4 x i8> @intrinsic_vnclipu_mask_vx_nxv4i8_nxv4i16(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, iXLen %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv4i8_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -882,8 +882,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vnclipu.nxv8i8.nxv8i16(
 define <vscale x 8 x i8> @intrinsic_vnclipu_vx_nxv8i8_nxv8i16(<vscale x 8 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v10, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -907,8 +907,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vnclipu.mask.nxv8i8.nxv8i16(
 define <vscale x 8 x i8> @intrinsic_vnclipu_mask_vx_nxv8i8_nxv8i16(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, iXLen %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv8i8_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -930,8 +930,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vnclipu.nxv16i8.nxv16i16(
 define <vscale x 16 x i8> @intrinsic_vnclipu_vx_nxv16i8_nxv16i16(<vscale x 16 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v12, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -955,8 +955,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vnclipu.mask.nxv16i8.nxv16i16(
 define <vscale x 16 x i8> @intrinsic_vnclipu_mask_vx_nxv16i8_nxv16i16(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, iXLen %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv16i8_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -978,8 +978,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vnclipu.nxv32i8.nxv32i16(
 define <vscale x 32 x i8> @intrinsic_vnclipu_vx_nxv32i8_nxv32i16(<vscale x 32 x i16> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v16, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1003,8 +1003,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vnclipu.mask.nxv32i8.nxv32i16(
 define <vscale x 32 x i8> @intrinsic_vnclipu_mask_vx_nxv32i8_nxv32i16(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, iXLen %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv32i8_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1026,8 +1026,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vnclipu.nxv1i16.nxv1i32(
 define <vscale x 1 x i16> @intrinsic_vnclipu_vx_nxv1i16_nxv1i32(<vscale x 1 x i32> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1050,8 +1050,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vnclipu.mask.nxv1i16.nxv1i32(
 define <vscale x 1 x i16> @intrinsic_vnclipu_mask_vx_nxv1i16_nxv1i32(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, iXLen %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv1i16_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1073,8 +1073,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vnclipu.nxv2i16.nxv2i32(
 define <vscale x 2 x i16> @intrinsic_vnclipu_vx_nxv2i16_nxv2i32(<vscale x 2 x i32> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1097,8 +1097,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vnclipu.mask.nxv2i16.nxv2i32(
 define <vscale x 2 x i16> @intrinsic_vnclipu_mask_vx_nxv2i16_nxv2i32(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, iXLen %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv2i16_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1120,8 +1120,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vnclipu.nxv4i16.nxv4i32(
 define <vscale x 4 x i16> @intrinsic_vnclipu_vx_nxv4i16_nxv4i32(<vscale x 4 x i32> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v10, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -1145,8 +1145,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vnclipu.mask.nxv4i16.nxv4i32(
 define <vscale x 4 x i16> @intrinsic_vnclipu_mask_vx_nxv4i16_nxv4i32(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, iXLen %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv4i16_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1168,8 +1168,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vnclipu.nxv8i16.nxv8i32(
 define <vscale x 8 x i16> @intrinsic_vnclipu_vx_nxv8i16_nxv8i32(<vscale x 8 x i32> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v12, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -1193,8 +1193,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vnclipu.mask.nxv8i16.nxv8i32(
 define <vscale x 8 x i16> @intrinsic_vnclipu_mask_vx_nxv8i16_nxv8i32(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, iXLen %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv8i16_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1216,8 +1216,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vnclipu.nxv16i16.nxv16i32(
 define <vscale x 16 x i16> @intrinsic_vnclipu_vx_nxv16i16_nxv16i32(<vscale x 16 x i32> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v16, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1241,8 +1241,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vnclipu.mask.nxv16i16.nxv16i32(
 define <vscale x 16 x i16> @intrinsic_vnclipu_mask_vx_nxv16i16_nxv16i32(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, iXLen %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv16i16_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1264,8 +1264,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vnclipu.nxv1i32.nxv1i64(
 define <vscale x 1 x i32> @intrinsic_vnclipu_vx_nxv1i32_nxv1i64(<vscale x 1 x i64> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1288,8 +1288,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vnclipu.mask.nxv1i32.nxv1i64(
 define <vscale x 1 x i32> @intrinsic_vnclipu_mask_vx_nxv1i32_nxv1i64(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, iXLen %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv1i32_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1311,8 +1311,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vnclipu.nxv2i32.nxv2i64(
 define <vscale x 2 x i32> @intrinsic_vnclipu_vx_nxv2i32_nxv2i64(<vscale x 2 x i64> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v10, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -1336,8 +1336,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vnclipu.mask.nxv2i32.nxv2i64(
 define <vscale x 2 x i32> @intrinsic_vnclipu_mask_vx_nxv2i32_nxv2i64(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, iXLen %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv2i32_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1359,8 +1359,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vnclipu.nxv4i32.nxv4i64(
 define <vscale x 4 x i32> @intrinsic_vnclipu_vx_nxv4i32_nxv4i64(<vscale x 4 x i64> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v12, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -1384,8 +1384,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vnclipu.mask.nxv4i32.nxv4i64(
 define <vscale x 4 x i32> @intrinsic_vnclipu_mask_vx_nxv4i32_nxv4i64(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, iXLen %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv4i32_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1407,8 +1407,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vnclipu.nxv8i32.nxv8i64(
 define <vscale x 8 x i32> @intrinsic_vnclipu_vx_nxv8i32_nxv8i64(<vscale x 8 x i64> %0, iXLen %1, iXLen %2) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vnclipu.wx v16, v8, a0
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1432,8 +1432,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vnclipu.mask.nxv8i32.nxv8i64(
 define <vscale x 8 x i32> @intrinsic_vnclipu_mask_vx_nxv8i32_nxv8i64(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, iXLen %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vx_nxv8i32_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vnclipu.wx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1450,8 +1450,8 @@ entry:
 define <vscale x 1 x i8> @intrinsic_vnclipu_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1467,8 +1467,8 @@ entry:
 define <vscale x 1 x i8> @intrinsic_vnclipu_mask_vi_nxv1i8_nxv1i16_i8(<vscale x 1 x i8> %0, <vscale x 1 x i16> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv1i8_nxv1i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1485,8 +1485,8 @@ entry:
 define <vscale x 2 x i8> @intrinsic_vnclipu_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1502,8 +1502,8 @@ entry:
 define <vscale x 2 x i8> @intrinsic_vnclipu_mask_vi_nxv2i8_nxv2i16_i8(<vscale x 2 x i8> %0, <vscale x 2 x i16> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv2i8_nxv2i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1520,8 +1520,8 @@ entry:
 define <vscale x 4 x i8> @intrinsic_vnclipu_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1537,8 +1537,8 @@ entry:
 define <vscale x 4 x i8> @intrinsic_vnclipu_mask_vi_nxv4i8_nxv4i16_i8(<vscale x 4 x i8> %0, <vscale x 4 x i16> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv4i8_nxv4i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1555,8 +1555,8 @@ entry:
 define <vscale x 8 x i8> @intrinsic_vnclipu_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -1573,8 +1573,8 @@ entry:
 define <vscale x 8 x i8> @intrinsic_vnclipu_mask_vi_nxv8i8_nxv8i16_i8(<vscale x 8 x i8> %0, <vscale x 8 x i16> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv8i8_nxv8i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1591,8 +1591,8 @@ entry:
 define <vscale x 16 x i8> @intrinsic_vnclipu_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -1609,8 +1609,8 @@ entry:
 define <vscale x 16 x i8> @intrinsic_vnclipu_mask_vi_nxv16i8_nxv16i16_i8(<vscale x 16 x i8> %0, <vscale x 16 x i16> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv16i8_nxv16i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1627,8 +1627,8 @@ entry:
 define <vscale x 32 x i8> @intrinsic_vnclipu_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i16> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v16, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1645,8 +1645,8 @@ entry:
 define <vscale x 32 x i8> @intrinsic_vnclipu_mask_vi_nxv32i8_nxv32i16_i8(<vscale x 32 x i8> %0, <vscale x 32 x i16> %1, <vscale x 32 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv32i8_nxv32i16_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1663,8 +1663,8 @@ entry:
 define <vscale x 1 x i16> @intrinsic_vnclipu_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1680,8 +1680,8 @@ entry:
 define <vscale x 1 x i16> @intrinsic_vnclipu_mask_vi_nxv1i16_nxv1i32_i16(<vscale x 1 x i16> %0, <vscale x 1 x i32> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv1i16_nxv1i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1698,8 +1698,8 @@ entry:
 define <vscale x 2 x i16> @intrinsic_vnclipu_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1715,8 +1715,8 @@ entry:
 define <vscale x 2 x i16> @intrinsic_vnclipu_mask_vi_nxv2i16_nxv2i32_i16(<vscale x 2 x i16> %0, <vscale x 2 x i32> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv2i16_nxv2i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1733,8 +1733,8 @@ entry:
 define <vscale x 4 x i16> @intrinsic_vnclipu_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -1751,8 +1751,8 @@ entry:
 define <vscale x 4 x i16> @intrinsic_vnclipu_mask_vi_nxv4i16_nxv4i32_i16(<vscale x 4 x i16> %0, <vscale x 4 x i32> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv4i16_nxv4i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1769,8 +1769,8 @@ entry:
 define <vscale x 8 x i16> @intrinsic_vnclipu_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -1787,8 +1787,8 @@ entry:
 define <vscale x 8 x i16> @intrinsic_vnclipu_mask_vi_nxv8i16_nxv8i32_i16(<vscale x 8 x i16> %0, <vscale x 8 x i32> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv8i16_nxv8i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1805,8 +1805,8 @@ entry:
 define <vscale x 16 x i16> @intrinsic_vnclipu_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i32> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v16, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1823,8 +1823,8 @@ entry:
 define <vscale x 16 x i16> @intrinsic_vnclipu_mask_vi_nxv16i16_nxv16i32_i16(<vscale x 16 x i16> %0, <vscale x 16 x i32> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv16i16_nxv16i32_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1841,8 +1841,8 @@ entry:
 define <vscale x 1 x i32> @intrinsic_vnclipu_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 9
 ; CHECK-NEXT:    ret
 entry:
@@ -1858,8 +1858,8 @@ entry:
 define <vscale x 1 x i32> @intrinsic_vnclipu_mask_vi_nxv1i32_nxv1i64_i32(<vscale x 1 x i32> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv1i32_nxv1i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v9, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1876,8 +1876,8 @@ entry:
 define <vscale x 2 x i32> @intrinsic_vnclipu_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -1894,8 +1894,8 @@ entry:
 define <vscale x 2 x i32> @intrinsic_vnclipu_mask_vi_nxv2i32_nxv2i64_i32(<vscale x 2 x i32> %0, <vscale x 2 x i64> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv2i32_nxv2i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1912,8 +1912,8 @@ entry:
 define <vscale x 4 x i32> @intrinsic_vnclipu_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -1930,8 +1930,8 @@ entry:
 define <vscale x 4 x i32> @intrinsic_vnclipu_mask_vi_nxv4i32_nxv4i64_i32(<vscale x 4 x i32> %0, <vscale x 4 x i64> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv4i32_nxv4i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1948,8 +1948,8 @@ entry:
 define <vscale x 8 x i32> @intrinsic_vnclipu_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i64> %0, iXLen %1) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v16, v8, 9
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
@@ -1966,8 +1966,8 @@ entry:
 define <vscale x 8 x i32> @intrinsic_vnclipu_mask_vi_nxv8i32_nxv8i64_i32(<vscale x 8 x i32> %0, <vscale x 8 x i64> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: intrinsic_vnclipu_mask_vi_nxv8i32_nxv8i64_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vnclipu.wi v8, v16, 9, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsmul-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vsmul-rv32.ll
index d1fcb0f47cb5..e7d8ae635f75 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsmul-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsmul-rv32.ll
@@ -15,8 +15,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -39,8 +39,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -63,8 +63,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vsmul.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vsmul_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -87,8 +87,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vsmul.mask.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vsmul_mask_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, <vscale x 2 x i8> %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -111,8 +111,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vsmul.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vsmul_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -135,8 +135,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vsmul.mask.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vsmul_mask_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, <vscale x 4 x i8> %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -159,8 +159,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vsmul.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vsmul_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -183,8 +183,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vsmul.mask.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vsmul_mask_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -207,8 +207,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vsmul.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vsmul_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -231,8 +231,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vsmul.mask.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vsmul_mask_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <vscale x 16 x i8> %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -255,8 +255,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vsmul.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vsmul_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -279,8 +279,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vsmul.mask.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vsmul_mask_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, <vscale x 32 x i8> %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -303,8 +303,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vsmul.nxv64i8.nxv64i8(
 define <vscale x 64 x i8> @intrinsic_vsmul_vv_nxv64i8_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -328,8 +328,8 @@ define <vscale x 64 x i8> @intrinsic_vsmul_mask_vv_nxv64i8_nxv64i8_nxv64i8(<vsca
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8r.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -352,8 +352,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vsmul.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vsmul_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -376,8 +376,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vsmul.mask.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vsmul_mask_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, <vscale x 1 x i16> %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -400,8 +400,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vsmul.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vsmul_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -424,8 +424,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vsmul.mask.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vsmul_mask_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, <vscale x 2 x i16> %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -448,8 +448,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vsmul.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vsmul_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -472,8 +472,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vsmul.mask.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vsmul_mask_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, <vscale x 4 x i16> %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -496,8 +496,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vsmul.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vsmul_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -520,8 +520,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vsmul.mask.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vsmul_mask_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <vscale x 8 x i16> %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -544,8 +544,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vsmul.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vsmul_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -568,8 +568,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vsmul.mask.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vsmul_mask_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, <vscale x 16 x i16> %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -592,8 +592,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vsmul.nxv32i16.nxv32i16(
 define <vscale x 32 x i16> @intrinsic_vsmul_vv_nxv32i16_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -617,8 +617,8 @@ define <vscale x 32 x i16> @intrinsic_vsmul_mask_vv_nxv32i16_nxv32i16_nxv32i16(<
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -641,8 +641,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vsmul.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vsmul_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -665,8 +665,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vsmul.mask.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vsmul_mask_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, <vscale x 1 x i32> %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -689,8 +689,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vsmul.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vsmul_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -713,8 +713,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vsmul.mask.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vsmul_mask_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, <vscale x 2 x i32> %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -737,8 +737,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vsmul.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vsmul_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -761,8 +761,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vsmul.mask.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vsmul_mask_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <vscale x 4 x i32> %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -785,8 +785,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vsmul.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vsmul_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -809,8 +809,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vsmul.mask.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vsmul_mask_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, <vscale x 8 x i32> %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -833,8 +833,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vsmul.nxv16i32.nxv16i32(
 define <vscale x 16 x i32> @intrinsic_vsmul_vv_nxv16i32_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -858,8 +858,8 @@ define <vscale x 16 x i32> @intrinsic_vsmul_mask_vv_nxv16i32_nxv16i32_nxv16i32(<
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -882,8 +882,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vsmul.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vsmul_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -906,8 +906,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vsmul.mask.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vsmul_mask_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i64> %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -930,8 +930,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vsmul.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vsmul_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -954,8 +954,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vsmul.mask.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vsmul_mask_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -978,8 +978,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vsmul.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vsmul_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -1002,8 +1002,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vsmul.mask.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vsmul_mask_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, <vscale x 4 x i64> %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1026,8 +1026,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vsmul.nxv8i64.nxv8i64(
 define <vscale x 8 x i64> @intrinsic_vsmul_vv_nxv8i64_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -1051,8 +1051,8 @@ define <vscale x 8 x i64> @intrinsic_vsmul_mask_vv_nxv8i64_nxv8i64_nxv8i64(<vsca
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1075,8 +1075,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1099,8 +1099,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.mask.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1123,8 +1123,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vsmul.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vsmul_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1147,8 +1147,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vsmul.mask.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vsmul_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1171,8 +1171,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vsmul.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vsmul_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1195,8 +1195,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vsmul.mask.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vsmul_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1219,8 +1219,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vsmul.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vsmul_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1243,8 +1243,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vsmul.mask.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vsmul_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1267,8 +1267,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vsmul.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vsmul_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1291,8 +1291,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vsmul.mask.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vsmul_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1315,8 +1315,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vsmul.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vsmul_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1339,8 +1339,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vsmul.mask.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vsmul_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1363,8 +1363,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vsmul.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vsmul_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1387,8 +1387,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vsmul.mask.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vsmul_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1411,8 +1411,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vsmul.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vsmul_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1435,8 +1435,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vsmul.mask.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vsmul_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1459,8 +1459,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vsmul.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vsmul_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1483,8 +1483,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vsmul.mask.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vsmul_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1507,8 +1507,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vsmul.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vsmul_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1531,8 +1531,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vsmul.mask.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vsmul_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1555,8 +1555,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vsmul.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vsmul_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1579,8 +1579,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vsmul.mask.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vsmul_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1603,8 +1603,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vsmul.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vsmul_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1627,8 +1627,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vsmul.mask.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vsmul_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1651,8 +1651,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vsmul.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vsmul_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1675,8 +1675,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vsmul.mask.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vsmul_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1698,8 +1698,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vsmul.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vsmul_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1722,8 +1722,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vsmul.mask.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vsmul_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1745,8 +1745,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vsmul.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vsmul_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1769,8 +1769,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vsmul.mask.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vsmul_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1792,8 +1792,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vsmul.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vsmul_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1816,8 +1816,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vsmul.mask.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vsmul_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1839,8 +1839,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vsmul.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vsmul_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1863,8 +1863,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vsmul.mask.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vsmul_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1886,8 +1886,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vsmul.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vsmul_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1910,8 +1910,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vsmul.mask.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vsmul_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i32 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsmul-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vsmul-rv64.ll
index 1fe1baf1cef2..66bc5c9103a4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsmul-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsmul-rv64.ll
@@ -15,8 +15,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -39,8 +39,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.mask.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i8_nxv1i8_nxv1i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -63,8 +63,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vsmul.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vsmul_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -87,8 +87,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vsmul.mask.nxv2i8.nxv2i8(
 define <vscale x 2 x i8> @intrinsic_vsmul_mask_vv_nxv2i8_nxv2i8_nxv2i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, <vscale x 2 x i8> %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv2i8_nxv2i8_nxv2i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -111,8 +111,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vsmul.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vsmul_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -135,8 +135,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vsmul.mask.nxv4i8.nxv4i8(
 define <vscale x 4 x i8> @intrinsic_vsmul_mask_vv_nxv4i8_nxv4i8_nxv4i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, <vscale x 4 x i8> %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv4i8_nxv4i8_nxv4i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -159,8 +159,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vsmul.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vsmul_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -183,8 +183,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vsmul.mask.nxv8i8.nxv8i8(
 define <vscale x 8 x i8> @intrinsic_vsmul_mask_vv_nxv8i8_nxv8i8_nxv8i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i8_nxv8i8_nxv8i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -207,8 +207,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vsmul.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vsmul_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -231,8 +231,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vsmul.mask.nxv16i8.nxv16i8(
 define <vscale x 16 x i8> @intrinsic_vsmul_mask_vv_nxv16i8_nxv16i8_nxv16i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, <vscale x 16 x i8> %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv16i8_nxv16i8_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -255,8 +255,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vsmul.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vsmul_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -279,8 +279,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vsmul.mask.nxv32i8.nxv32i8(
 define <vscale x 32 x i8> @intrinsic_vsmul_mask_vv_nxv32i8_nxv32i8_nxv32i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, <vscale x 32 x i8> %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv32i8_nxv32i8_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -303,8 +303,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vsmul.nxv64i8.nxv64i8(
 define <vscale x 64 x i8> @intrinsic_vsmul_vv_nxv64i8_nxv64i8_nxv64i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -328,8 +328,8 @@ define <vscale x 64 x i8> @intrinsic_vsmul_mask_vv_nxv64i8_nxv64i8_nxv64i8(<vsca
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv64i8_nxv64i8_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8r.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -352,8 +352,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vsmul.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vsmul_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -376,8 +376,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vsmul.mask.nxv1i16.nxv1i16(
 define <vscale x 1 x i16> @intrinsic_vsmul_mask_vv_nxv1i16_nxv1i16_nxv1i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, <vscale x 1 x i16> %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i16_nxv1i16_nxv1i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -400,8 +400,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vsmul.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vsmul_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -424,8 +424,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vsmul.mask.nxv2i16.nxv2i16(
 define <vscale x 2 x i16> @intrinsic_vsmul_mask_vv_nxv2i16_nxv2i16_nxv2i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, <vscale x 2 x i16> %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv2i16_nxv2i16_nxv2i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -448,8 +448,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vsmul.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vsmul_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -472,8 +472,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vsmul.mask.nxv4i16.nxv4i16(
 define <vscale x 4 x i16> @intrinsic_vsmul_mask_vv_nxv4i16_nxv4i16_nxv4i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, <vscale x 4 x i16> %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv4i16_nxv4i16_nxv4i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -496,8 +496,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vsmul.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vsmul_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -520,8 +520,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vsmul.mask.nxv8i16.nxv8i16(
 define <vscale x 8 x i16> @intrinsic_vsmul_mask_vv_nxv8i16_nxv8i16_nxv8i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, <vscale x 8 x i16> %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i16_nxv8i16_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -544,8 +544,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vsmul.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vsmul_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -568,8 +568,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vsmul.mask.nxv16i16.nxv16i16(
 define <vscale x 16 x i16> @intrinsic_vsmul_mask_vv_nxv16i16_nxv16i16_nxv16i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, <vscale x 16 x i16> %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv16i16_nxv16i16_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -592,8 +592,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vsmul.nxv32i16.nxv32i16(
 define <vscale x 32 x i16> @intrinsic_vsmul_vv_nxv32i16_nxv32i16_nxv32i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -617,8 +617,8 @@ define <vscale x 32 x i16> @intrinsic_vsmul_mask_vv_nxv32i16_nxv32i16_nxv32i16(<
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv32i16_nxv32i16_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re16.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -641,8 +641,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vsmul.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vsmul_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -665,8 +665,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vsmul.mask.nxv1i32.nxv1i32(
 define <vscale x 1 x i32> @intrinsic_vsmul_mask_vv_nxv1i32_nxv1i32_nxv1i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, <vscale x 1 x i32> %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i32_nxv1i32_nxv1i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -689,8 +689,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vsmul.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vsmul_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -713,8 +713,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vsmul.mask.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @intrinsic_vsmul_mask_vv_nxv2i32_nxv2i32_nxv2i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, <vscale x 2 x i32> %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv2i32_nxv2i32_nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -737,8 +737,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vsmul.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vsmul_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -761,8 +761,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vsmul.mask.nxv4i32.nxv4i32(
 define <vscale x 4 x i32> @intrinsic_vsmul_mask_vv_nxv4i32_nxv4i32_nxv4i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, <vscale x 4 x i32> %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv4i32_nxv4i32_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -785,8 +785,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vsmul.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vsmul_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -809,8 +809,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vsmul.mask.nxv8i32.nxv8i32(
 define <vscale x 8 x i32> @intrinsic_vsmul_mask_vv_nxv8i32_nxv8i32_nxv8i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, <vscale x 8 x i32> %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i32_nxv8i32_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -833,8 +833,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vsmul.nxv16i32.nxv16i32(
 define <vscale x 16 x i32> @intrinsic_vsmul_vv_nxv16i32_nxv16i32_nxv16i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -858,8 +858,8 @@ define <vscale x 16 x i32> @intrinsic_vsmul_mask_vv_nxv16i32_nxv16i32_nxv16i32(<
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv16i32_nxv16i32_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re32.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -882,8 +882,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vsmul.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vsmul_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -906,8 +906,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vsmul.mask.nxv1i64.nxv1i64(
 define <vscale x 1 x i64> @intrinsic_vsmul_mask_vv_nxv1i64_nxv1i64_nxv1i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i64> %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i64_nxv1i64_nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -930,8 +930,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vsmul.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vsmul_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -954,8 +954,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vsmul.mask.nxv2i64.nxv2i64(
 define <vscale x 2 x i64> @intrinsic_vsmul_mask_vv_nxv2i64_nxv2i64_nxv2i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, <vscale x 2 x i64> %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv2i64_nxv2i64_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v10, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -978,8 +978,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vsmul.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vsmul_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -1002,8 +1002,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vsmul.mask.nxv4i64.nxv4i64(
 define <vscale x 4 x i64> @intrinsic_vsmul_mask_vv_nxv4i64_nxv4i64_nxv4i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, <vscale x 4 x i64> %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv4i64_nxv4i64_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v12, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1026,8 +1026,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vsmul.nxv8i64.nxv8i64(
 define <vscale x 8 x i64> @intrinsic_vsmul_vv_nxv8i64_nxv8i64_nxv8i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -1051,8 +1051,8 @@ define <vscale x 8 x i64> @intrinsic_vsmul_mask_vv_nxv8i64_nxv8i64_nxv8i64(<vsca
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i64_nxv8i64_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vl8re64.v v24, (a0)
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vv v8, v16, v24, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1075,8 +1075,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, i8 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1099,8 +1099,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vsmul.mask.nxv1i8.i8(
 define <vscale x 1 x i8> @intrinsic_vsmul_mask_vx_nxv1i8_nxv1i8_i8(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i8 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv1i8_nxv1i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1123,8 +1123,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vsmul.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vsmul_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, i8 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1147,8 +1147,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vsmul.mask.nxv2i8.i8(
 define <vscale x 2 x i8> @intrinsic_vsmul_mask_vx_nxv2i8_nxv2i8_i8(<vscale x 2 x i8> %0, <vscale x 2 x i8> %1, i8 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv2i8_nxv2i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1171,8 +1171,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vsmul.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vsmul_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, i8 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1195,8 +1195,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vsmul.mask.nxv4i8.i8(
 define <vscale x 4 x i8> @intrinsic_vsmul_mask_vx_nxv4i8_nxv4i8_i8(<vscale x 4 x i8> %0, <vscale x 4 x i8> %1, i8 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv4i8_nxv4i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1219,8 +1219,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vsmul.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vsmul_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, i8 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1243,8 +1243,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vsmul.mask.nxv8i8.i8(
 define <vscale x 8 x i8> @intrinsic_vsmul_mask_vx_nxv8i8_nxv8i8_i8(<vscale x 8 x i8> %0, <vscale x 8 x i8> %1, i8 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv8i8_nxv8i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1267,8 +1267,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vsmul.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vsmul_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, i8 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1291,8 +1291,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vsmul.mask.nxv16i8.i8(
 define <vscale x 16 x i8> @intrinsic_vsmul_mask_vx_nxv16i8_nxv16i8_i8(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1, i8 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv16i8_nxv16i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1315,8 +1315,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vsmul.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vsmul_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, i8 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1339,8 +1339,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vsmul.mask.nxv32i8.i8(
 define <vscale x 32 x i8> @intrinsic_vsmul_mask_vx_nxv32i8_nxv32i8_i8(<vscale x 32 x i8> %0, <vscale x 32 x i8> %1, i8 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv32i8_nxv32i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1363,8 +1363,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vsmul.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vsmul_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, i8 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1387,8 +1387,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vsmul.mask.nxv64i8.i8(
 define <vscale x 64 x i8> @intrinsic_vsmul_mask_vx_nxv64i8_nxv64i8_i8(<vscale x 64 x i8> %0, <vscale x 64 x i8> %1, i8 %2, <vscale x 64 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv64i8_nxv64i8_i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1411,8 +1411,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vsmul.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vsmul_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, i16 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1435,8 +1435,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vsmul.mask.nxv1i16.i16(
 define <vscale x 1 x i16> @intrinsic_vsmul_mask_vx_nxv1i16_nxv1i16_i16(<vscale x 1 x i16> %0, <vscale x 1 x i16> %1, i16 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv1i16_nxv1i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1459,8 +1459,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vsmul.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vsmul_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, i16 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1483,8 +1483,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vsmul.mask.nxv2i16.i16(
 define <vscale x 2 x i16> @intrinsic_vsmul_mask_vx_nxv2i16_nxv2i16_i16(<vscale x 2 x i16> %0, <vscale x 2 x i16> %1, i16 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv2i16_nxv2i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1507,8 +1507,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vsmul.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vsmul_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, i16 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1531,8 +1531,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vsmul.mask.nxv4i16.i16(
 define <vscale x 4 x i16> @intrinsic_vsmul_mask_vx_nxv4i16_nxv4i16_i16(<vscale x 4 x i16> %0, <vscale x 4 x i16> %1, i16 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv4i16_nxv4i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1555,8 +1555,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vsmul.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vsmul_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, i16 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1579,8 +1579,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vsmul.mask.nxv8i16.i16(
 define <vscale x 8 x i16> @intrinsic_vsmul_mask_vx_nxv8i16_nxv8i16_i16(<vscale x 8 x i16> %0, <vscale x 8 x i16> %1, i16 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv8i16_nxv8i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1603,8 +1603,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vsmul.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vsmul_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, i16 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1627,8 +1627,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vsmul.mask.nxv16i16.i16(
 define <vscale x 16 x i16> @intrinsic_vsmul_mask_vx_nxv16i16_nxv16i16_i16(<vscale x 16 x i16> %0, <vscale x 16 x i16> %1, i16 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv16i16_nxv16i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1651,8 +1651,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vsmul.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vsmul_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, i16 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1675,8 +1675,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vsmul.mask.nxv32i16.i16(
 define <vscale x 32 x i16> @intrinsic_vsmul_mask_vx_nxv32i16_nxv32i16_i16(<vscale x 32 x i16> %0, <vscale x 32 x i16> %1, i16 %2, <vscale x 32 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv32i16_nxv32i16_i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1699,8 +1699,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vsmul.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vsmul_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, i32 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1723,8 +1723,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vsmul.mask.nxv1i32.i32(
 define <vscale x 1 x i32> @intrinsic_vsmul_mask_vx_nxv1i32_nxv1i32_i32(<vscale x 1 x i32> %0, <vscale x 1 x i32> %1, i32 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv1i32_nxv1i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1747,8 +1747,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vsmul.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vsmul_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, i32 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1771,8 +1771,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vsmul.mask.nxv2i32.i32(
 define <vscale x 2 x i32> @intrinsic_vsmul_mask_vx_nxv2i32_nxv2i32_i32(<vscale x 2 x i32> %0, <vscale x 2 x i32> %1, i32 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv2i32_nxv2i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1795,8 +1795,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vsmul.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vsmul_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, i32 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1819,8 +1819,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vsmul.mask.nxv4i32.i32(
 define <vscale x 4 x i32> @intrinsic_vsmul_mask_vx_nxv4i32_nxv4i32_i32(<vscale x 4 x i32> %0, <vscale x 4 x i32> %1, i32 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv4i32_nxv4i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1843,8 +1843,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vsmul.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vsmul_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, i32 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1867,8 +1867,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vsmul.mask.nxv8i32.i32(
 define <vscale x 8 x i32> @intrinsic_vsmul_mask_vx_nxv8i32_nxv8i32_i32(<vscale x 8 x i32> %0, <vscale x 8 x i32> %1, i32 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv8i32_nxv8i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1891,8 +1891,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vsmul.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vsmul_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, i32 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1915,8 +1915,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vsmul.mask.nxv16i32.i32(
 define <vscale x 16 x i32> @intrinsic_vsmul_mask_vx_nxv16i32_nxv16i32_i32(<vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i32 %2, <vscale x 16 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv16i32_nxv16i32_i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1938,8 +1938,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vsmul.nxv1i64.i64(
 define <vscale x 1 x i64> @intrinsic_vsmul_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, i64 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv1i64_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -1962,8 +1962,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vsmul.mask.nxv1i64.i64(
 define <vscale x 1 x i64> @intrinsic_vsmul_mask_vx_nxv1i64_nxv1i64_i64(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, i64 %2, <vscale x 1 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv1i64_nxv1i64_i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1985,8 +1985,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vsmul.nxv2i64.i64(
 define <vscale x 2 x i64> @intrinsic_vsmul_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv2i64_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2009,8 +2009,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vsmul.mask.nxv2i64.i64(
 define <vscale x 2 x i64> @intrinsic_vsmul_mask_vx_nxv2i64_nxv2i64_i64(<vscale x 2 x i64> %0, <vscale x 2 x i64> %1, i64 %2, <vscale x 2 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv2i64_nxv2i64_i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v10, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -2032,8 +2032,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vsmul.nxv4i64.i64(
 define <vscale x 4 x i64> @intrinsic_vsmul_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv4i64_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2056,8 +2056,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vsmul.mask.nxv4i64.i64(
 define <vscale x 4 x i64> @intrinsic_vsmul_mask_vx_nxv4i64_nxv4i64_i64(<vscale x 4 x i64> %0, <vscale x 4 x i64> %1, i64 %2, <vscale x 4 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv4i64_nxv4i64_i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v12, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -2079,8 +2079,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vsmul.nxv8i64.i64(
 define <vscale x 8 x i64> @intrinsic_vsmul_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv8i64_nxv8i64_i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vsmul.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -2103,8 +2103,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vsmul.mask.nxv8i64.i64(
 define <vscale x 8 x i64> @intrinsic_vsmul_mask_vx_nxv8i64_nxv8i64_i64(<vscale x 8 x i64> %0, <vscale x 8 x i64> %1, i64 %2, <vscale x 8 x i1> %3, i64 %4) nounwind {
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv8i64_nxv8i64_i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, mu
 ; CHECK-NEXT:    vsmul.vx v8, v16, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssra-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vssra-rv32.ll
index 8e28dd490a87..7fd1b05bb444 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssra-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssra-rv32.ll
@@ -5,8 +5,8 @@
 define <vscale x 1 x i8> @test_vssra_vv_i8mf8(<vscale x 1 x i8> %op1, <vscale x 1 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -19,8 +19,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8.nxv1i8.i32(<vscale x 1 x i8>,
 define <vscale x 1 x i8> @test_vssra_vx_i8mf8(<vscale x 1 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -33,8 +33,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8.i32.i32(<vscale x 1 x i8>, <v
 define <vscale x 2 x i8> @test_vssra_vv_i8mf4(<vscale x 2 x i8> %op1, <vscale x 2 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -47,8 +47,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8.nxv2i8.i32(<vscale x 2 x i8>,
 define <vscale x 2 x i8> @test_vssra_vx_i8mf4(<vscale x 2 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -61,8 +61,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8.i32.i32(<vscale x 2 x i8>, <v
 define <vscale x 4 x i8> @test_vssra_vv_i8mf2(<vscale x 4 x i8> %op1, <vscale x 4 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -75,8 +75,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8.nxv4i8.i32(<vscale x 4 x i8>,
 define <vscale x 4 x i8> @test_vssra_vx_i8mf2(<vscale x 4 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -89,8 +89,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8.i32.i32(<vscale x 4 x i8>, <v
 define <vscale x 8 x i8> @test_vssra_vv_i8m1(<vscale x 8 x i8> %op1, <vscale x 8 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -103,8 +103,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8.nxv8i8.i32(<vscale x 8 x i8>,
 define <vscale x 8 x i8> @test_vssra_vx_i8m1(<vscale x 8 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -117,8 +117,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8.i32.i32(<vscale x 8 x i8>, <v
 define <vscale x 16 x i8> @test_vssra_vv_i8m2(<vscale x 16 x i8> %op1, <vscale x 16 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -131,8 +131,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8.nxv16i8.i32(<vscale x 16 x
 define <vscale x 16 x i8> @test_vssra_vx_i8m2(<vscale x 16 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -145,8 +145,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8.i32.i32(<vscale x 16 x i8>,
 define <vscale x 32 x i8> @test_vssra_vv_i8m4(<vscale x 32 x i8> %op1, <vscale x 32 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -159,8 +159,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8.nxv32i8.i32(<vscale x 32 x
 define <vscale x 32 x i8> @test_vssra_vx_i8m4(<vscale x 32 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -173,8 +173,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8.i32.i32(<vscale x 32 x i8>,
 define <vscale x 64 x i8> @test_vssra_vv_i8m8(<vscale x 64 x i8> %op1, <vscale x 64 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -187,8 +187,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8.nxv64i8.i32(<vscale x 64 x
 define <vscale x 64 x i8> @test_vssra_vx_i8m8(<vscale x 64 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -201,8 +201,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8.i32.i32(<vscale x 64 x i8>,
 define <vscale x 1 x i16> @test_vssra_vv_i16mf4(<vscale x 1 x i16> %op1, <vscale x 1 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -215,8 +215,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16.nxv1i16.i32(<vscale x 1 x i
 define <vscale x 1 x i16> @test_vssra_vx_i16mf4(<vscale x 1 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -229,8 +229,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16.i32.i32(<vscale x 1 x i16>,
 define <vscale x 2 x i16> @test_vssra_vv_i16mf2(<vscale x 2 x i16> %op1, <vscale x 2 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -243,8 +243,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16.nxv2i16.i32(<vscale x 2 x i
 define <vscale x 2 x i16> @test_vssra_vx_i16mf2(<vscale x 2 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -257,8 +257,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16.i32.i32(<vscale x 2 x i16>,
 define <vscale x 4 x i16> @test_vssra_vv_i16m1(<vscale x 4 x i16> %op1, <vscale x 4 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -271,8 +271,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16.nxv4i16.i32(<vscale x 4 x i
 define <vscale x 4 x i16> @test_vssra_vx_i16m1(<vscale x 4 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -285,8 +285,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16.i32.i32(<vscale x 4 x i16>,
 define <vscale x 8 x i16> @test_vssra_vv_i16m2(<vscale x 8 x i16> %op1, <vscale x 8 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -299,8 +299,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16.nxv8i16.i32(<vscale x 8 x i
 define <vscale x 8 x i16> @test_vssra_vx_i16m2(<vscale x 8 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -313,8 +313,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16.i32.i32(<vscale x 8 x i16>,
 define <vscale x 16 x i16> @test_vssra_vv_i16m4(<vscale x 16 x i16> %op1, <vscale x 16 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -327,8 +327,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16.nxv16i16.i32(<vscale x 16
 define <vscale x 16 x i16> @test_vssra_vx_i16m4(<vscale x 16 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -341,8 +341,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16.i32.i32(<vscale x 16 x i1
 define <vscale x 32 x i16> @test_vssra_vv_i16m8(<vscale x 32 x i16> %op1, <vscale x 32 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -355,8 +355,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16.nxv32i16.i32(<vscale x 32
 define <vscale x 32 x i16> @test_vssra_vx_i16m8(<vscale x 32 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -369,8 +369,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16.i32.i32(<vscale x 32 x i1
 define <vscale x 1 x i32> @test_vssra_vv_i32mf2(<vscale x 1 x i32> %op1, <vscale x 1 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -383,8 +383,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32.nxv1i32.i32(<vscale x 1 x i
 define <vscale x 1 x i32> @test_vssra_vx_i32mf2(<vscale x 1 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -397,8 +397,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32.i32.i32(<vscale x 1 x i32>,
 define <vscale x 2 x i32> @test_vssra_vv_i32m1(<vscale x 2 x i32> %op1, <vscale x 2 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -411,8 +411,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32.nxv2i32.i32(<vscale x 2 x i
 define <vscale x 2 x i32> @test_vssra_vx_i32m1(<vscale x 2 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -425,8 +425,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32.i32.i32(<vscale x 2 x i32>,
 define <vscale x 4 x i32> @test_vssra_vv_i32m2(<vscale x 4 x i32> %op1, <vscale x 4 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -439,8 +439,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.nxv4i32.i32(<vscale x 4 x i
 define <vscale x 4 x i32> @test_vssra_vx_i32m2(<vscale x 4 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -453,8 +453,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.i32.i32(<vscale x 4 x i32>,
 define <vscale x 8 x i32> @test_vssra_vv_i32m4(<vscale x 8 x i32> %op1, <vscale x 8 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -467,8 +467,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32.nxv8i32.i32(<vscale x 8 x i
 define <vscale x 8 x i32> @test_vssra_vx_i32m4(<vscale x 8 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -481,8 +481,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32.i32.i32(<vscale x 8 x i32>,
 define <vscale x 16 x i32> @test_vssra_vv_i32m8(<vscale x 16 x i32> %op1, <vscale x 16 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -495,8 +495,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32.nxv16i32.i32(<vscale x 16
 define <vscale x 16 x i32> @test_vssra_vx_i32m8(<vscale x 16 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -509,8 +509,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32.i32.i32(<vscale x 16 x i3
 define <vscale x 1 x i64> @test_vssra_vv_i64m1(<vscale x 1 x i64> %op1, <vscale x 1 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -523,8 +523,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.nxv1i64.i32(<vscale x 1 x i
 define <vscale x 1 x i64> @test_vssra_vx_i64m1(<vscale x 1 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -537,8 +537,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.i32.i32(<vscale x 1 x i64>,
 define <vscale x 2 x i64> @test_vssra_vv_i64m2(<vscale x 2 x i64> %op1, <vscale x 2 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -551,8 +551,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.nxv2i64.i32(<vscale x 2 x i
 define <vscale x 2 x i64> @test_vssra_vx_i64m2(<vscale x 2 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -565,8 +565,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.i32.i32(<vscale x 2 x i64>,
 define <vscale x 4 x i64> @test_vssra_vv_i64m4(<vscale x 4 x i64> %op1, <vscale x 4 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -579,8 +579,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.nxv4i64.i32(<vscale x 4 x i
 define <vscale x 4 x i64> @test_vssra_vx_i64m4(<vscale x 4 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -593,8 +593,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.i32.i32(<vscale x 4 x i64>,
 define <vscale x 8 x i64> @test_vssra_vv_i64m8(<vscale x 8 x i64> %op1, <vscale x 8 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -607,8 +607,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.nxv8i64.i32(<vscale x 8 x i
 define <vscale x 8 x i64> @test_vssra_vx_i64m8(<vscale x 8 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -621,8 +621,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.i32.i32(<vscale x 8 x i64>,
 define <vscale x 1 x i8> @test_vssra_vv_i8mf8_m(<vscale x 1 x i1> %mask, <vscale x 1 x i8> %op1, <vscale x 1 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -635,8 +635,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8.nxv1i8.i32(<vscale x 1 x
 define <vscale x 1 x i8> @test_vssra_vx_i8mf8_m(<vscale x 1 x i1> %mask, <vscale x 1 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -649,8 +649,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8.i32.i32(<vscale x 1 x i8
 define <vscale x 2 x i8> @test_vssra_vv_i8mf4_m(<vscale x 2 x i1> %mask, <vscale x 2 x i8> %op1, <vscale x 2 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -663,8 +663,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8.nxv2i8.i32(<vscale x 2 x
 define <vscale x 2 x i8> @test_vssra_vx_i8mf4_m(<vscale x 2 x i1> %mask, <vscale x 2 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -677,8 +677,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8.i32.i32(<vscale x 2 x i8
 define <vscale x 4 x i8> @test_vssra_vv_i8mf2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i8> %op1, <vscale x 4 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -691,8 +691,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8.nxv4i8.i32(<vscale x 4 x
 define <vscale x 4 x i8> @test_vssra_vx_i8mf2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -705,8 +705,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8.i32.i32(<vscale x 4 x i8
 define <vscale x 8 x i8> @test_vssra_vv_i8m1_m(<vscale x 8 x i1> %mask, <vscale x 8 x i8> %op1, <vscale x 8 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -719,8 +719,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8.nxv8i8.i32(<vscale x 8 x
 define <vscale x 8 x i8> @test_vssra_vx_i8m1_m(<vscale x 8 x i1> %mask, <vscale x 8 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -733,8 +733,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8.i32.i32(<vscale x 8 x i8
 define <vscale x 16 x i8> @test_vssra_vv_i8m2_m(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %op1, <vscale x 16 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -747,8 +747,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8.nxv16i8.i32(<vscale x
 define <vscale x 16 x i8> @test_vssra_vx_i8m2_m(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -761,8 +761,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8.i32.i32(<vscale x 16 x
 define <vscale x 32 x i8> @test_vssra_vv_i8m4_m(<vscale x 32 x i1> %mask, <vscale x 32 x i8> %op1, <vscale x 32 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -775,8 +775,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8.nxv32i8.i32(<vscale x
 define <vscale x 32 x i8> @test_vssra_vx_i8m4_m(<vscale x 32 x i1> %mask, <vscale x 32 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -789,8 +789,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8.i32.i32(<vscale x 32 x
 define <vscale x 64 x i8> @test_vssra_vv_i8m8_m(<vscale x 64 x i1> %mask, <vscale x 64 x i8> %op1, <vscale x 64 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -803,8 +803,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8.nxv64i8.i32(<vscale x
 define <vscale x 64 x i8> @test_vssra_vx_i8m8_m(<vscale x 64 x i1> %mask, <vscale x 64 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -817,8 +817,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8.i32.i32(<vscale x 64 x
 define <vscale x 1 x i16> @test_vssra_vv_i16mf4_m(<vscale x 1 x i1> %mask, <vscale x 1 x i16> %op1, <vscale x 1 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -831,8 +831,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16.nxv1i16.i32(<vscale x
 define <vscale x 1 x i16> @test_vssra_vx_i16mf4_m(<vscale x 1 x i1> %mask, <vscale x 1 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -845,8 +845,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16.i32.i32(<vscale x 1 x
 define <vscale x 2 x i16> @test_vssra_vv_i16mf2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i16> %op1, <vscale x 2 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -859,8 +859,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16.nxv2i16.i32(<vscale x
 define <vscale x 2 x i16> @test_vssra_vx_i16mf2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -873,8 +873,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16.i32.i32(<vscale x 2 x
 define <vscale x 4 x i16> @test_vssra_vv_i16m1_m(<vscale x 4 x i1> %mask, <vscale x 4 x i16> %op1, <vscale x 4 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -887,8 +887,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16.nxv4i16.i32(<vscale x
 define <vscale x 4 x i16> @test_vssra_vx_i16m1_m(<vscale x 4 x i1> %mask, <vscale x 4 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -901,8 +901,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16.i32.i32(<vscale x 4 x
 define <vscale x 8 x i16> @test_vssra_vv_i16m2_m(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %op1, <vscale x 8 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -915,8 +915,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16.nxv8i16.i32(<vscale x
 define <vscale x 8 x i16> @test_vssra_vx_i16m2_m(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -929,8 +929,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16.i32.i32(<vscale x 8 x
 define <vscale x 16 x i16> @test_vssra_vv_i16m4_m(<vscale x 16 x i1> %mask, <vscale x 16 x i16> %op1, <vscale x 16 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -943,8 +943,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16.nxv16i16.i32(<vscale
 define <vscale x 16 x i16> @test_vssra_vx_i16m4_m(<vscale x 16 x i1> %mask, <vscale x 16 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -957,8 +957,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16.i32.i32(<vscale x 16
 define <vscale x 32 x i16> @test_vssra_vv_i16m8_m(<vscale x 32 x i1> %mask, <vscale x 32 x i16> %op1, <vscale x 32 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -971,8 +971,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16.nxv32i16.i32(<vscale
 define <vscale x 32 x i16> @test_vssra_vx_i16m8_m(<vscale x 32 x i1> %mask, <vscale x 32 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -985,8 +985,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16.i32.i32(<vscale x 32
 define <vscale x 1 x i32> @test_vssra_vv_i32mf2_m(<vscale x 1 x i1> %mask, <vscale x 1 x i32> %op1, <vscale x 1 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -999,8 +999,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32.nxv1i32.i32(<vscale x
 define <vscale x 1 x i32> @test_vssra_vx_i32mf2_m(<vscale x 1 x i1> %mask, <vscale x 1 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1013,8 +1013,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32.i32.i32(<vscale x 1 x
 define <vscale x 2 x i32> @test_vssra_vv_i32m1_m(<vscale x 2 x i1> %mask, <vscale x 2 x i32> %op1, <vscale x 2 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1027,8 +1027,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32.nxv2i32.i32(<vscale x
 define <vscale x 2 x i32> @test_vssra_vx_i32m1_m(<vscale x 2 x i1> %mask, <vscale x 2 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1041,8 +1041,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32.i32.i32(<vscale x 2 x
 define <vscale x 4 x i32> @test_vssra_vv_i32m2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i32> %op1, <vscale x 4 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1055,8 +1055,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32.nxv4i32.i32(<vscale x
 define <vscale x 4 x i32> @test_vssra_vx_i32m2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1069,8 +1069,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32.i32.i32(<vscale x 4 x
 define <vscale x 8 x i32> @test_vssra_vv_i32m4_m(<vscale x 8 x i1> %mask, <vscale x 8 x i32> %op1, <vscale x 8 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1083,8 +1083,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32.nxv8i32.i32(<vscale x
 define <vscale x 8 x i32> @test_vssra_vx_i32m4_m(<vscale x 8 x i1> %mask, <vscale x 8 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1097,8 +1097,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32.i32.i32(<vscale x 8 x
 define <vscale x 16 x i32> @test_vssra_vv_i32m8_m(<vscale x 16 x i1> %mask, <vscale x 16 x i32> %op1, <vscale x 16 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1111,8 +1111,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32.nxv16i32.i32(<vscale
 define <vscale x 16 x i32> @test_vssra_vx_i32m8_m(<vscale x 16 x i1> %mask, <vscale x 16 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1125,8 +1125,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32.i32.i32(<vscale x 16
 define <vscale x 1 x i64> @test_vssra_vv_i64m1_m(<vscale x 1 x i1> %mask, <vscale x 1 x i64> %op1, <vscale x 1 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1139,8 +1139,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.nxv1i64.i32(<vscale x
 define <vscale x 1 x i64> @test_vssra_vx_i64m1_m(<vscale x 1 x i1> %mask, <vscale x 1 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1153,8 +1153,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.i32.i32(<vscale x 1 x
 define <vscale x 2 x i64> @test_vssra_vv_i64m2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %op1, <vscale x 2 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1167,8 +1167,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.nxv2i64.i32(<vscale x
 define <vscale x 2 x i64> @test_vssra_vx_i64m2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1181,8 +1181,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.i32.i32(<vscale x 2 x
 define <vscale x 4 x i64> @test_vssra_vv_i64m4_m(<vscale x 4 x i1> %mask, <vscale x 4 x i64> %op1, <vscale x 4 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1195,8 +1195,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.nxv4i64.i32(<vscale x
 define <vscale x 4 x i64> @test_vssra_vx_i64m4_m(<vscale x 4 x i1> %mask, <vscale x 4 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1209,8 +1209,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.i32.i32(<vscale x 4 x
 define <vscale x 8 x i64> @test_vssra_vv_i64m8_m(<vscale x 8 x i1> %mask, <vscale x 8 x i64> %op1, <vscale x 8 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1223,8 +1223,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64.nxv8i64.i32(<vscale x
 define <vscale x 8 x i64> @test_vssra_vx_i64m8_m(<vscale x 8 x i1> %mask, <vscale x 8 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssra-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vssra-rv64.ll
index 96ca5e32cf36..b7a84e58e6e6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssra-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssra-rv64.ll
@@ -5,8 +5,8 @@
 define <vscale x 1 x i8> @test_vssra_vv_i8mf8(<vscale x 1 x i8> %op1, <vscale x 1 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -19,8 +19,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8.nxv1i8.i64(<vscale x 1 x i8>,
 define <vscale x 1 x i8> @test_vssra_vx_i8mf8(<vscale x 1 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -33,8 +33,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssra.nxv1i8.i64.i64(<vscale x 1 x i8>, <v
 define <vscale x 2 x i8> @test_vssra_vv_i8mf4(<vscale x 2 x i8> %op1, <vscale x 2 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -47,8 +47,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8.nxv2i8.i64(<vscale x 2 x i8>,
 define <vscale x 2 x i8> @test_vssra_vx_i8mf4(<vscale x 2 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -61,8 +61,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssra.nxv2i8.i64.i64(<vscale x 2 x i8>, <v
 define <vscale x 4 x i8> @test_vssra_vv_i8mf2(<vscale x 4 x i8> %op1, <vscale x 4 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -75,8 +75,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8.nxv4i8.i64(<vscale x 4 x i8>,
 define <vscale x 4 x i8> @test_vssra_vx_i8mf2(<vscale x 4 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -89,8 +89,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssra.nxv4i8.i64.i64(<vscale x 4 x i8>, <v
 define <vscale x 8 x i8> @test_vssra_vv_i8m1(<vscale x 8 x i8> %op1, <vscale x 8 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -103,8 +103,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8.nxv8i8.i64(<vscale x 8 x i8>,
 define <vscale x 8 x i8> @test_vssra_vx_i8m1(<vscale x 8 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -117,8 +117,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssra.nxv8i8.i64.i64(<vscale x 8 x i8>, <v
 define <vscale x 16 x i8> @test_vssra_vv_i8m2(<vscale x 16 x i8> %op1, <vscale x 16 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -131,8 +131,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8.nxv16i8.i64(<vscale x 16 x
 define <vscale x 16 x i8> @test_vssra_vx_i8m2(<vscale x 16 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -145,8 +145,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssra.nxv16i8.i64.i64(<vscale x 16 x i8>,
 define <vscale x 32 x i8> @test_vssra_vv_i8m4(<vscale x 32 x i8> %op1, <vscale x 32 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -159,8 +159,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8.nxv32i8.i64(<vscale x 32 x
 define <vscale x 32 x i8> @test_vssra_vx_i8m4(<vscale x 32 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -173,8 +173,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssra.nxv32i8.i64.i64(<vscale x 32 x i8>,
 define <vscale x 64 x i8> @test_vssra_vv_i8m8(<vscale x 64 x i8> %op1, <vscale x 64 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -187,8 +187,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8.nxv64i8.i64(<vscale x 64 x
 define <vscale x 64 x i8> @test_vssra_vx_i8m8(<vscale x 64 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -201,8 +201,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssra.nxv64i8.i64.i64(<vscale x 64 x i8>,
 define <vscale x 1 x i16> @test_vssra_vv_i16mf4(<vscale x 1 x i16> %op1, <vscale x 1 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -215,8 +215,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16.nxv1i16.i64(<vscale x 1 x i
 define <vscale x 1 x i16> @test_vssra_vx_i16mf4(<vscale x 1 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -229,8 +229,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssra.nxv1i16.i64.i64(<vscale x 1 x i16>,
 define <vscale x 2 x i16> @test_vssra_vv_i16mf2(<vscale x 2 x i16> %op1, <vscale x 2 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -243,8 +243,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16.nxv2i16.i64(<vscale x 2 x i
 define <vscale x 2 x i16> @test_vssra_vx_i16mf2(<vscale x 2 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -257,8 +257,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssra.nxv2i16.i64.i64(<vscale x 2 x i16>,
 define <vscale x 4 x i16> @test_vssra_vv_i16m1(<vscale x 4 x i16> %op1, <vscale x 4 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -271,8 +271,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16.nxv4i16.i64(<vscale x 4 x i
 define <vscale x 4 x i16> @test_vssra_vx_i16m1(<vscale x 4 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -285,8 +285,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssra.nxv4i16.i64.i64(<vscale x 4 x i16>,
 define <vscale x 8 x i16> @test_vssra_vv_i16m2(<vscale x 8 x i16> %op1, <vscale x 8 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -299,8 +299,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16.nxv8i16.i64(<vscale x 8 x i
 define <vscale x 8 x i16> @test_vssra_vx_i16m2(<vscale x 8 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -313,8 +313,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssra.nxv8i16.i64.i64(<vscale x 8 x i16>,
 define <vscale x 16 x i16> @test_vssra_vv_i16m4(<vscale x 16 x i16> %op1, <vscale x 16 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -327,8 +327,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16.nxv16i16.i64(<vscale x 16
 define <vscale x 16 x i16> @test_vssra_vx_i16m4(<vscale x 16 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -341,8 +341,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssra.nxv16i16.i64.i64(<vscale x 16 x i1
 define <vscale x 32 x i16> @test_vssra_vv_i16m8(<vscale x 32 x i16> %op1, <vscale x 32 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -355,8 +355,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16.nxv32i16.i64(<vscale x 32
 define <vscale x 32 x i16> @test_vssra_vx_i16m8(<vscale x 32 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -369,8 +369,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssra.nxv32i16.i64.i64(<vscale x 32 x i1
 define <vscale x 1 x i32> @test_vssra_vv_i32mf2(<vscale x 1 x i32> %op1, <vscale x 1 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -383,8 +383,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32.nxv1i32.i64(<vscale x 1 x i
 define <vscale x 1 x i32> @test_vssra_vx_i32mf2(<vscale x 1 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -397,8 +397,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssra.nxv1i32.i64.i64(<vscale x 1 x i32>,
 define <vscale x 2 x i32> @test_vssra_vv_i32m1(<vscale x 2 x i32> %op1, <vscale x 2 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -411,8 +411,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32.nxv2i32.i64(<vscale x 2 x i
 define <vscale x 2 x i32> @test_vssra_vx_i32m1(<vscale x 2 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -425,8 +425,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssra.nxv2i32.i64.i64(<vscale x 2 x i32>,
 define <vscale x 4 x i32> @test_vssra_vv_i32m2(<vscale x 4 x i32> %op1, <vscale x 4 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -439,8 +439,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.nxv4i32.i64(<vscale x 4 x i
 define <vscale x 4 x i32> @test_vssra_vx_i32m2(<vscale x 4 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -453,8 +453,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssra.nxv4i32.i64.i64(<vscale x 4 x i32>,
 define <vscale x 8 x i32> @test_vssra_vv_i32m4(<vscale x 8 x i32> %op1, <vscale x 8 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -467,8 +467,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32.nxv8i32.i64(<vscale x 8 x i
 define <vscale x 8 x i32> @test_vssra_vx_i32m4(<vscale x 8 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -481,8 +481,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssra.nxv8i32.i64.i64(<vscale x 8 x i32>,
 define <vscale x 16 x i32> @test_vssra_vv_i32m8(<vscale x 16 x i32> %op1, <vscale x 16 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -495,8 +495,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32.nxv16i32.i64(<vscale x 16
 define <vscale x 16 x i32> @test_vssra_vx_i32m8(<vscale x 16 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -509,8 +509,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssra.nxv16i32.i64.i64(<vscale x 16 x i3
 define <vscale x 1 x i64> @test_vssra_vv_i64m1(<vscale x 1 x i64> %op1, <vscale x 1 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -523,8 +523,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.nxv1i64.i64(<vscale x 1 x i
 define <vscale x 1 x i64> @test_vssra_vx_i64m1(<vscale x 1 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -537,8 +537,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssra.nxv1i64.i64.i64(<vscale x 1 x i64>,
 define <vscale x 2 x i64> @test_vssra_vv_i64m2(<vscale x 2 x i64> %op1, <vscale x 2 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -551,8 +551,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.nxv2i64.i64(<vscale x 2 x i
 define <vscale x 2 x i64> @test_vssra_vx_i64m2(<vscale x 2 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -565,8 +565,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssra.nxv2i64.i64.i64(<vscale x 2 x i64>,
 define <vscale x 4 x i64> @test_vssra_vv_i64m4(<vscale x 4 x i64> %op1, <vscale x 4 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -579,8 +579,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.nxv4i64.i64(<vscale x 4 x i
 define <vscale x 4 x i64> @test_vssra_vx_i64m4(<vscale x 4 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -593,8 +593,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssra.nxv4i64.i64.i64(<vscale x 4 x i64>,
 define <vscale x 8 x i64> @test_vssra_vv_i64m8(<vscale x 8 x i64> %op1, <vscale x 8 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -607,8 +607,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.nxv8i64.i64(<vscale x 8 x i
 define <vscale x 8 x i64> @test_vssra_vx_i64m8(<vscale x 8 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -621,8 +621,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssra.nxv8i64.i64.i64(<vscale x 8 x i64>,
 define <vscale x 1 x i8> @test_vssra_vv_i8mf8_m(<vscale x 1 x i1> %mask, <vscale x 1 x i8> %op1, <vscale x 1 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -635,8 +635,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8.nxv1i8.i64(<vscale x 1 x
 define <vscale x 1 x i8> @test_vssra_vx_i8mf8_m(<vscale x 1 x i1> %mask, <vscale x 1 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -649,8 +649,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssra.mask.nxv1i8.i64.i64(<vscale x 1 x i8
 define <vscale x 2 x i8> @test_vssra_vv_i8mf4_m(<vscale x 2 x i1> %mask, <vscale x 2 x i8> %op1, <vscale x 2 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -663,8 +663,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8.nxv2i8.i64(<vscale x 2 x
 define <vscale x 2 x i8> @test_vssra_vx_i8mf4_m(<vscale x 2 x i1> %mask, <vscale x 2 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -677,8 +677,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssra.mask.nxv2i8.i64.i64(<vscale x 2 x i8
 define <vscale x 4 x i8> @test_vssra_vv_i8mf2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i8> %op1, <vscale x 4 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -691,8 +691,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8.nxv4i8.i64(<vscale x 4 x
 define <vscale x 4 x i8> @test_vssra_vx_i8mf2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -705,8 +705,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssra.mask.nxv4i8.i64.i64(<vscale x 4 x i8
 define <vscale x 8 x i8> @test_vssra_vv_i8m1_m(<vscale x 8 x i1> %mask, <vscale x 8 x i8> %op1, <vscale x 8 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -719,8 +719,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x
 define <vscale x 8 x i8> @test_vssra_vx_i8m1_m(<vscale x 8 x i1> %mask, <vscale x 8 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -733,8 +733,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssra.mask.nxv8i8.i64.i64(<vscale x 8 x i8
 define <vscale x 16 x i8> @test_vssra_vv_i8m2_m(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %op1, <vscale x 16 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -747,8 +747,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8.nxv16i8.i64(<vscale x
 define <vscale x 16 x i8> @test_vssra_vx_i8m2_m(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -761,8 +761,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssra.mask.nxv16i8.i64.i64(<vscale x 16 x
 define <vscale x 32 x i8> @test_vssra_vv_i8m4_m(<vscale x 32 x i1> %mask, <vscale x 32 x i8> %op1, <vscale x 32 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -775,8 +775,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8.nxv32i8.i64(<vscale x
 define <vscale x 32 x i8> @test_vssra_vx_i8m4_m(<vscale x 32 x i1> %mask, <vscale x 32 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -789,8 +789,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssra.mask.nxv32i8.i64.i64(<vscale x 32 x
 define <vscale x 64 x i8> @test_vssra_vv_i8m8_m(<vscale x 64 x i1> %mask, <vscale x 64 x i8> %op1, <vscale x 64 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i8m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -803,8 +803,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8.nxv64i8.i64(<vscale x
 define <vscale x 64 x i8> @test_vssra_vx_i8m8_m(<vscale x 64 x i1> %mask, <vscale x 64 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i8m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -817,8 +817,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssra.mask.nxv64i8.i64.i64(<vscale x 64 x
 define <vscale x 1 x i16> @test_vssra_vv_i16mf4_m(<vscale x 1 x i1> %mask, <vscale x 1 x i16> %op1, <vscale x 1 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -831,8 +831,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16.nxv1i16.i64(<vscale x
 define <vscale x 1 x i16> @test_vssra_vx_i16mf4_m(<vscale x 1 x i1> %mask, <vscale x 1 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -845,8 +845,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssra.mask.nxv1i16.i64.i64(<vscale x 1 x
 define <vscale x 2 x i16> @test_vssra_vv_i16mf2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i16> %op1, <vscale x 2 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -859,8 +859,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16.nxv2i16.i64(<vscale x
 define <vscale x 2 x i16> @test_vssra_vx_i16mf2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -873,8 +873,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssra.mask.nxv2i16.i64.i64(<vscale x 2 x
 define <vscale x 4 x i16> @test_vssra_vv_i16m1_m(<vscale x 4 x i1> %mask, <vscale x 4 x i16> %op1, <vscale x 4 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -887,8 +887,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16.nxv4i16.i64(<vscale x
 define <vscale x 4 x i16> @test_vssra_vx_i16m1_m(<vscale x 4 x i1> %mask, <vscale x 4 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -901,8 +901,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssra.mask.nxv4i16.i64.i64(<vscale x 4 x
 define <vscale x 8 x i16> @test_vssra_vv_i16m2_m(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %op1, <vscale x 8 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -915,8 +915,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16.nxv8i16.i64(<vscale x
 define <vscale x 8 x i16> @test_vssra_vx_i16m2_m(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -929,8 +929,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssra.mask.nxv8i16.i64.i64(<vscale x 8 x
 define <vscale x 16 x i16> @test_vssra_vv_i16m4_m(<vscale x 16 x i1> %mask, <vscale x 16 x i16> %op1, <vscale x 16 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -943,8 +943,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16.nxv16i16.i64(<vscale
 define <vscale x 16 x i16> @test_vssra_vx_i16m4_m(<vscale x 16 x i1> %mask, <vscale x 16 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -957,8 +957,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssra.mask.nxv16i16.i64.i64(<vscale x 16
 define <vscale x 32 x i16> @test_vssra_vv_i16m8_m(<vscale x 32 x i1> %mask, <vscale x 32 x i16> %op1, <vscale x 32 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i16m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -971,8 +971,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16.nxv32i16.i64(<vscale
 define <vscale x 32 x i16> @test_vssra_vx_i16m8_m(<vscale x 32 x i1> %mask, <vscale x 32 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i16m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -985,8 +985,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssra.mask.nxv32i16.i64.i64(<vscale x 32
 define <vscale x 1 x i32> @test_vssra_vv_i32mf2_m(<vscale x 1 x i1> %mask, <vscale x 1 x i32> %op1, <vscale x 1 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -999,8 +999,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32.nxv1i32.i64(<vscale x
 define <vscale x 1 x i32> @test_vssra_vx_i32mf2_m(<vscale x 1 x i1> %mask, <vscale x 1 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1013,8 +1013,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssra.mask.nxv1i32.i64.i64(<vscale x 1 x
 define <vscale x 2 x i32> @test_vssra_vv_i32m1_m(<vscale x 2 x i1> %mask, <vscale x 2 x i32> %op1, <vscale x 2 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1027,8 +1027,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32.nxv2i32.i64(<vscale x
 define <vscale x 2 x i32> @test_vssra_vx_i32m1_m(<vscale x 2 x i1> %mask, <vscale x 2 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1041,8 +1041,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssra.mask.nxv2i32.i64.i64(<vscale x 2 x
 define <vscale x 4 x i32> @test_vssra_vv_i32m2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i32> %op1, <vscale x 4 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1055,8 +1055,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32.nxv4i32.i64(<vscale x
 define <vscale x 4 x i32> @test_vssra_vx_i32m2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1069,8 +1069,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssra.mask.nxv4i32.i64.i64(<vscale x 4 x
 define <vscale x 8 x i32> @test_vssra_vv_i32m4_m(<vscale x 8 x i1> %mask, <vscale x 8 x i32> %op1, <vscale x 8 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1083,8 +1083,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32.nxv8i32.i64(<vscale x
 define <vscale x 8 x i32> @test_vssra_vx_i32m4_m(<vscale x 8 x i1> %mask, <vscale x 8 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1097,8 +1097,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssra.mask.nxv8i32.i64.i64(<vscale x 8 x
 define <vscale x 16 x i32> @test_vssra_vv_i32m8_m(<vscale x 16 x i1> %mask, <vscale x 16 x i32> %op1, <vscale x 16 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i32m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1111,8 +1111,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32.nxv16i32.i64(<vscale
 define <vscale x 16 x i32> @test_vssra_vx_i32m8_m(<vscale x 16 x i1> %mask, <vscale x 16 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i32m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1125,8 +1125,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssra.mask.nxv16i32.i64.i64(<vscale x 16
 define <vscale x 1 x i64> @test_vssra_vv_i64m1_m(<vscale x 1 x i1> %mask, <vscale x 1 x i64> %op1, <vscale x 1 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1139,8 +1139,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.nxv1i64.i64(<vscale x
 define <vscale x 1 x i64> @test_vssra_vx_i64m1_m(<vscale x 1 x i1> %mask, <vscale x 1 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1153,8 +1153,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssra.mask.nxv1i64.i64.i64(<vscale x 1 x
 define <vscale x 2 x i64> @test_vssra_vv_i64m2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %op1, <vscale x 2 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1167,8 +1167,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.nxv2i64.i64(<vscale x
 define <vscale x 2 x i64> @test_vssra_vx_i64m2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1181,8 +1181,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssra.mask.nxv2i64.i64.i64(<vscale x 2 x
 define <vscale x 4 x i64> @test_vssra_vv_i64m4_m(<vscale x 4 x i1> %mask, <vscale x 4 x i64> %op1, <vscale x 4 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1195,8 +1195,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.nxv4i64.i64(<vscale x
 define <vscale x 4 x i64> @test_vssra_vx_i64m4_m(<vscale x 4 x i1> %mask, <vscale x 4 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1209,8 +1209,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssra.mask.nxv4i64.i64.i64(<vscale x 4 x
 define <vscale x 8 x i64> @test_vssra_vv_i64m8_m(<vscale x 8 x i1> %mask, <vscale x 8 x i64> %op1, <vscale x 8 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vv_i64m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vssra.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1223,8 +1223,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssra.mask.nxv8i64.nxv8i64.i64(<vscale x
 define <vscale x 8 x i64> @test_vssra_vx_i64m8_m(<vscale x 8 x i1> %mask, <vscale x 8 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssra_vx_i64m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vssra.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssrl-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vssrl-rv32.ll
index c1a064984dcc..0c2cdff65776 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssrl-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssrl-rv32.ll
@@ -5,8 +5,8 @@
 define <vscale x 1 x i8> @test_vssrl_vv_u8mf8(<vscale x 1 x i8> %op1, <vscale x 1 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -19,8 +19,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8.nxv1i8.i32(<vscale x 1 x i8>,
 define <vscale x 1 x i8> @test_vssrl_vx_u8mf8(<vscale x 1 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -33,8 +33,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8.i32.i32(<vscale x 1 x i8>, <v
 define <vscale x 2 x i8> @test_vssrl_vv_u8mf4(<vscale x 2 x i8> %op1, <vscale x 2 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -47,8 +47,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8.nxv2i8.i32(<vscale x 2 x i8>,
 define <vscale x 2 x i8> @test_vssrl_vx_u8mf4(<vscale x 2 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -61,8 +61,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8.i32.i32(<vscale x 2 x i8>, <v
 define <vscale x 4 x i8> @test_vssrl_vv_u8mf2(<vscale x 4 x i8> %op1, <vscale x 4 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -75,8 +75,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8.nxv4i8.i32(<vscale x 4 x i8>,
 define <vscale x 4 x i8> @test_vssrl_vx_u8mf2(<vscale x 4 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -89,8 +89,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8.i32.i32(<vscale x 4 x i8>, <v
 define <vscale x 8 x i8> @test_vssrl_vv_u8m1(<vscale x 8 x i8> %op1, <vscale x 8 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -103,8 +103,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8.nxv8i8.i32(<vscale x 8 x i8>,
 define <vscale x 8 x i8> @test_vssrl_vx_u8m1(<vscale x 8 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -117,8 +117,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8.i32.i32(<vscale x 8 x i8>, <v
 define <vscale x 16 x i8> @test_vssrl_vv_u8m2(<vscale x 16 x i8> %op1, <vscale x 16 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -131,8 +131,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8.nxv16i8.i32(<vscale x 16 x
 define <vscale x 16 x i8> @test_vssrl_vx_u8m2(<vscale x 16 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -145,8 +145,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8.i32.i32(<vscale x 16 x i8>,
 define <vscale x 32 x i8> @test_vssrl_vv_u8m4(<vscale x 32 x i8> %op1, <vscale x 32 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -159,8 +159,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8.nxv32i8.i32(<vscale x 32 x
 define <vscale x 32 x i8> @test_vssrl_vx_u8m4(<vscale x 32 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -173,8 +173,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8.i32.i32(<vscale x 32 x i8>,
 define <vscale x 64 x i8> @test_vssrl_vv_u8m8(<vscale x 64 x i8> %op1, <vscale x 64 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -187,8 +187,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8.nxv64i8.i32(<vscale x 64 x
 define <vscale x 64 x i8> @test_vssrl_vx_u8m8(<vscale x 64 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -201,8 +201,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8.i32.i32(<vscale x 64 x i8>,
 define <vscale x 1 x i16> @test_vssrl_vv_u16mf4(<vscale x 1 x i16> %op1, <vscale x 1 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -215,8 +215,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16.nxv1i16.i32(<vscale x 1 x i
 define <vscale x 1 x i16> @test_vssrl_vx_u16mf4(<vscale x 1 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -229,8 +229,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16.i32.i32(<vscale x 1 x i16>,
 define <vscale x 2 x i16> @test_vssrl_vv_u16mf2(<vscale x 2 x i16> %op1, <vscale x 2 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -243,8 +243,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16.nxv2i16.i32(<vscale x 2 x i
 define <vscale x 2 x i16> @test_vssrl_vx_u16mf2(<vscale x 2 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -257,8 +257,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16.i32.i32(<vscale x 2 x i16>,
 define <vscale x 4 x i16> @test_vssrl_vv_u16m1(<vscale x 4 x i16> %op1, <vscale x 4 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -271,8 +271,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16.nxv4i16.i32(<vscale x 4 x i
 define <vscale x 4 x i16> @test_vssrl_vx_u16m1(<vscale x 4 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -285,8 +285,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16.i32.i32(<vscale x 4 x i16>,
 define <vscale x 8 x i16> @test_vssrl_vv_u16m2(<vscale x 8 x i16> %op1, <vscale x 8 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -299,8 +299,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16.nxv8i16.i32(<vscale x 8 x i
 define <vscale x 8 x i16> @test_vssrl_vx_u16m2(<vscale x 8 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -313,8 +313,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16.i32.i32(<vscale x 8 x i16>,
 define <vscale x 16 x i16> @test_vssrl_vv_u16m4(<vscale x 16 x i16> %op1, <vscale x 16 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -327,8 +327,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16.nxv16i16.i32(<vscale x 16
 define <vscale x 16 x i16> @test_vssrl_vx_u16m4(<vscale x 16 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -341,8 +341,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16.i32.i32(<vscale x 16 x i1
 define <vscale x 32 x i16> @test_vssrl_vv_u16m8(<vscale x 32 x i16> %op1, <vscale x 32 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -355,8 +355,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16.nxv32i16.i32(<vscale x 32
 define <vscale x 32 x i16> @test_vssrl_vx_u16m8(<vscale x 32 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -369,8 +369,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16.i32.i32(<vscale x 32 x i1
 define <vscale x 1 x i32> @test_vssrl_vv_u32mf2(<vscale x 1 x i32> %op1, <vscale x 1 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -383,8 +383,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32.nxv1i32.i32(<vscale x 1 x i
 define <vscale x 1 x i32> @test_vssrl_vx_u32mf2(<vscale x 1 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -397,8 +397,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32.i32.i32(<vscale x 1 x i32>,
 define <vscale x 2 x i32> @test_vssrl_vv_u32m1(<vscale x 2 x i32> %op1, <vscale x 2 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -411,8 +411,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32.nxv2i32.i32(<vscale x 2 x i
 define <vscale x 2 x i32> @test_vssrl_vx_u32m1(<vscale x 2 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -425,8 +425,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32.i32.i32(<vscale x 2 x i32>,
 define <vscale x 4 x i32> @test_vssrl_vv_u32m2(<vscale x 4 x i32> %op1, <vscale x 4 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -439,8 +439,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.nxv4i32.i32(<vscale x 4 x i
 define <vscale x 4 x i32> @test_vssrl_vx_u32m2(<vscale x 4 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -453,8 +453,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.i32.i32(<vscale x 4 x i32>,
 define <vscale x 8 x i32> @test_vssrl_vv_u32m4(<vscale x 8 x i32> %op1, <vscale x 8 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -467,8 +467,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32.nxv8i32.i32(<vscale x 8 x i
 define <vscale x 8 x i32> @test_vssrl_vx_u32m4(<vscale x 8 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -481,8 +481,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32.i32.i32(<vscale x 8 x i32>,
 define <vscale x 16 x i32> @test_vssrl_vv_u32m8(<vscale x 16 x i32> %op1, <vscale x 16 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -495,8 +495,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32.nxv16i32.i32(<vscale x 16
 define <vscale x 16 x i32> @test_vssrl_vx_u32m8(<vscale x 16 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -509,8 +509,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32.i32.i32(<vscale x 16 x i3
 define <vscale x 1 x i64> @test_vssrl_vv_u64m1(<vscale x 1 x i64> %op1, <vscale x 1 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -523,8 +523,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.nxv1i64.i32(<vscale x 1 x i
 define <vscale x 1 x i64> @test_vssrl_vx_u64m1(<vscale x 1 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -537,8 +537,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.i32.i32(<vscale x 1 x i64>,
 define <vscale x 2 x i64> @test_vssrl_vv_u64m2(<vscale x 2 x i64> %op1, <vscale x 2 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -551,8 +551,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.nxv2i64.i32(<vscale x 2 x i
 define <vscale x 2 x i64> @test_vssrl_vx_u64m2(<vscale x 2 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -565,8 +565,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.i32.i32(<vscale x 2 x i64>,
 define <vscale x 4 x i64> @test_vssrl_vv_u64m4(<vscale x 4 x i64> %op1, <vscale x 4 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -579,8 +579,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.nxv4i64.i32(<vscale x 4 x i
 define <vscale x 4 x i64> @test_vssrl_vx_u64m4(<vscale x 4 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -593,8 +593,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.i32.i32(<vscale x 4 x i64>,
 define <vscale x 8 x i64> @test_vssrl_vv_u64m8(<vscale x 8 x i64> %op1, <vscale x 8 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -607,8 +607,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.nxv8i64.i32(<vscale x 8 x i
 define <vscale x 8 x i64> @test_vssrl_vx_u64m8(<vscale x 8 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -621,8 +621,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.i32.i32(<vscale x 8 x i64>,
 define <vscale x 1 x i8> @test_vssrl_vv_u8mf8_m(<vscale x 1 x i1> %mask, <vscale x 1 x i8> %op1, <vscale x 1 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -635,8 +635,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.nxv1i8.i32(<vscale x 1 x
 define <vscale x 1 x i8> @test_vssrl_vx_u8mf8_m(<vscale x 1 x i1> %mask, <vscale x 1 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -649,8 +649,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.i32.i32(<vscale x 1 x i8
 define <vscale x 2 x i8> @test_vssrl_vv_u8mf4_m(<vscale x 2 x i1> %mask, <vscale x 2 x i8> %op1, <vscale x 2 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -663,8 +663,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8.nxv2i8.i32(<vscale x 2 x
 define <vscale x 2 x i8> @test_vssrl_vx_u8mf4_m(<vscale x 2 x i1> %mask, <vscale x 2 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -677,8 +677,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8.i32.i32(<vscale x 2 x i8
 define <vscale x 4 x i8> @test_vssrl_vv_u8mf2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i8> %op1, <vscale x 4 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -691,8 +691,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8.nxv4i8.i32(<vscale x 4 x
 define <vscale x 4 x i8> @test_vssrl_vx_u8mf2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -705,8 +705,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8.i32.i32(<vscale x 4 x i8
 define <vscale x 8 x i8> @test_vssrl_vv_u8m1_m(<vscale x 8 x i1> %mask, <vscale x 8 x i8> %op1, <vscale x 8 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -719,8 +719,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8.nxv8i8.i32(<vscale x 8 x
 define <vscale x 8 x i8> @test_vssrl_vx_u8m1_m(<vscale x 8 x i1> %mask, <vscale x 8 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -733,8 +733,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8.i32.i32(<vscale x 8 x i8
 define <vscale x 16 x i8> @test_vssrl_vv_u8m2_m(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %op1, <vscale x 16 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -747,8 +747,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8.nxv16i8.i32(<vscale x
 define <vscale x 16 x i8> @test_vssrl_vx_u8m2_m(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -761,8 +761,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8.i32.i32(<vscale x 16 x
 define <vscale x 32 x i8> @test_vssrl_vv_u8m4_m(<vscale x 32 x i1> %mask, <vscale x 32 x i8> %op1, <vscale x 32 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -775,8 +775,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8.nxv32i8.i32(<vscale x
 define <vscale x 32 x i8> @test_vssrl_vx_u8m4_m(<vscale x 32 x i1> %mask, <vscale x 32 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -789,8 +789,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8.i32.i32(<vscale x 32 x
 define <vscale x 64 x i8> @test_vssrl_vv_u8m8_m(<vscale x 64 x i1> %mask, <vscale x 64 x i8> %op1, <vscale x 64 x i8> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -803,8 +803,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8.nxv64i8.i32(<vscale x
 define <vscale x 64 x i8> @test_vssrl_vx_u8m8_m(<vscale x 64 x i1> %mask, <vscale x 64 x i8> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -817,8 +817,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8.i32.i32(<vscale x 64 x
 define <vscale x 1 x i16> @test_vssrl_vv_u16mf4_m(<vscale x 1 x i1> %mask, <vscale x 1 x i16> %op1, <vscale x 1 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -831,8 +831,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16.nxv1i16.i32(<vscale x
 define <vscale x 1 x i16> @test_vssrl_vx_u16mf4_m(<vscale x 1 x i1> %mask, <vscale x 1 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -845,8 +845,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16.i32.i32(<vscale x 1 x
 define <vscale x 2 x i16> @test_vssrl_vv_u16mf2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i16> %op1, <vscale x 2 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -859,8 +859,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16.nxv2i16.i32(<vscale x
 define <vscale x 2 x i16> @test_vssrl_vx_u16mf2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -873,8 +873,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16.i32.i32(<vscale x 2 x
 define <vscale x 4 x i16> @test_vssrl_vv_u16m1_m(<vscale x 4 x i1> %mask, <vscale x 4 x i16> %op1, <vscale x 4 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -887,8 +887,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16.nxv4i16.i32(<vscale x
 define <vscale x 4 x i16> @test_vssrl_vx_u16m1_m(<vscale x 4 x i1> %mask, <vscale x 4 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -901,8 +901,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16.i32.i32(<vscale x 4 x
 define <vscale x 8 x i16> @test_vssrl_vv_u16m2_m(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %op1, <vscale x 8 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -915,8 +915,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16.nxv8i16.i32(<vscale x
 define <vscale x 8 x i16> @test_vssrl_vx_u16m2_m(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -929,8 +929,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16.i32.i32(<vscale x 8 x
 define <vscale x 16 x i16> @test_vssrl_vv_u16m4_m(<vscale x 16 x i1> %mask, <vscale x 16 x i16> %op1, <vscale x 16 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -943,8 +943,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16.nxv16i16.i32(<vscale
 define <vscale x 16 x i16> @test_vssrl_vx_u16m4_m(<vscale x 16 x i1> %mask, <vscale x 16 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -957,8 +957,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16.i32.i32(<vscale x 16
 define <vscale x 32 x i16> @test_vssrl_vv_u16m8_m(<vscale x 32 x i1> %mask, <vscale x 32 x i16> %op1, <vscale x 32 x i16> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -971,8 +971,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16.nxv32i16.i32(<vscale
 define <vscale x 32 x i16> @test_vssrl_vx_u16m8_m(<vscale x 32 x i1> %mask, <vscale x 32 x i16> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -985,8 +985,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16.i32.i32(<vscale x 32
 define <vscale x 1 x i32> @test_vssrl_vv_u32mf2_m(<vscale x 1 x i1> %mask, <vscale x 1 x i32> %op1, <vscale x 1 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -999,8 +999,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32.nxv1i32.i32(<vscale x
 define <vscale x 1 x i32> @test_vssrl_vx_u32mf2_m(<vscale x 1 x i1> %mask, <vscale x 1 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1013,8 +1013,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32.i32.i32(<vscale x 1 x
 define <vscale x 2 x i32> @test_vssrl_vv_u32m1_m(<vscale x 2 x i1> %mask, <vscale x 2 x i32> %op1, <vscale x 2 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1027,8 +1027,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32.nxv2i32.i32(<vscale x
 define <vscale x 2 x i32> @test_vssrl_vx_u32m1_m(<vscale x 2 x i1> %mask, <vscale x 2 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1041,8 +1041,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32.i32.i32(<vscale x 2 x
 define <vscale x 4 x i32> @test_vssrl_vv_u32m2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i32> %op1, <vscale x 4 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1055,8 +1055,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32.nxv4i32.i32(<vscale x
 define <vscale x 4 x i32> @test_vssrl_vx_u32m2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1069,8 +1069,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32.i32.i32(<vscale x 4 x
 define <vscale x 8 x i32> @test_vssrl_vv_u32m4_m(<vscale x 8 x i1> %mask, <vscale x 8 x i32> %op1, <vscale x 8 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1083,8 +1083,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32.nxv8i32.i32(<vscale x
 define <vscale x 8 x i32> @test_vssrl_vx_u32m4_m(<vscale x 8 x i1> %mask, <vscale x 8 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1097,8 +1097,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32.i32.i32(<vscale x 8 x
 define <vscale x 16 x i32> @test_vssrl_vv_u32m8_m(<vscale x 16 x i1> %mask, <vscale x 16 x i32> %op1, <vscale x 16 x i32> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1111,8 +1111,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32.nxv16i32.i32(<vscale
 define <vscale x 16 x i32> @test_vssrl_vx_u32m8_m(<vscale x 16 x i1> %mask, <vscale x 16 x i32> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1125,8 +1125,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32.i32.i32(<vscale x 16
 define <vscale x 1 x i64> @test_vssrl_vv_u64m1_m(<vscale x 1 x i1> %mask, <vscale x 1 x i64> %op1, <vscale x 1 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1139,8 +1139,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.nxv1i64.i32(<vscale x
 define <vscale x 1 x i64> @test_vssrl_vx_u64m1_m(<vscale x 1 x i1> %mask, <vscale x 1 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1153,8 +1153,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.i32.i32(<vscale x 1 x
 define <vscale x 2 x i64> @test_vssrl_vv_u64m2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %op1, <vscale x 2 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1167,8 +1167,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.nxv2i64.i32(<vscale x
 define <vscale x 2 x i64> @test_vssrl_vx_u64m2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1181,8 +1181,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.i32.i32(<vscale x 2 x
 define <vscale x 4 x i64> @test_vssrl_vv_u64m4_m(<vscale x 4 x i1> %mask, <vscale x 4 x i64> %op1, <vscale x 4 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1195,8 +1195,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.nxv4i64.i32(<vscale x
 define <vscale x 4 x i64> @test_vssrl_vx_u64m4_m(<vscale x 4 x i1> %mask, <vscale x 4 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1209,8 +1209,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.i32.i32(<vscale x 4 x
 define <vscale x 8 x i64> @test_vssrl_vv_u64m8_m(<vscale x 8 x i1> %mask, <vscale x 8 x i64> %op1, <vscale x 8 x i64> %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1223,8 +1223,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64.nxv8i64.i32(<vscale x
 define <vscale x 8 x i64> @test_vssrl_vx_u64m8_m(<vscale x 8 x i1> %mask, <vscale x 8 x i64> %op1, i32 %shift, i32 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vssrl-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vssrl-rv64.ll
index 0a465db64b7a..fe80854bb264 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vssrl-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vssrl-rv64.ll
@@ -5,8 +5,8 @@
 define <vscale x 1 x i8> @test_vssrl_vv_u8mf8(<vscale x 1 x i8> %op1, <vscale x 1 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -19,8 +19,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8.nxv1i8.i64(<vscale x 1 x i8>,
 define <vscale x 1 x i8> @test_vssrl_vx_u8mf8(<vscale x 1 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -33,8 +33,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.nxv1i8.i64.i64(<vscale x 1 x i8>, <v
 define <vscale x 2 x i8> @test_vssrl_vv_u8mf4(<vscale x 2 x i8> %op1, <vscale x 2 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -47,8 +47,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8.nxv2i8.i64(<vscale x 2 x i8>,
 define <vscale x 2 x i8> @test_vssrl_vx_u8mf4(<vscale x 2 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -61,8 +61,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssrl.nxv2i8.i64.i64(<vscale x 2 x i8>, <v
 define <vscale x 4 x i8> @test_vssrl_vv_u8mf2(<vscale x 4 x i8> %op1, <vscale x 4 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -75,8 +75,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8.nxv4i8.i64(<vscale x 4 x i8>,
 define <vscale x 4 x i8> @test_vssrl_vx_u8mf2(<vscale x 4 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -89,8 +89,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssrl.nxv4i8.i64.i64(<vscale x 4 x i8>, <v
 define <vscale x 8 x i8> @test_vssrl_vv_u8m1(<vscale x 8 x i8> %op1, <vscale x 8 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -103,8 +103,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8.nxv8i8.i64(<vscale x 8 x i8>,
 define <vscale x 8 x i8> @test_vssrl_vx_u8m1(<vscale x 8 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -117,8 +117,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssrl.nxv8i8.i64.i64(<vscale x 8 x i8>, <v
 define <vscale x 16 x i8> @test_vssrl_vv_u8m2(<vscale x 16 x i8> %op1, <vscale x 16 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -131,8 +131,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8.nxv16i8.i64(<vscale x 16 x
 define <vscale x 16 x i8> @test_vssrl_vx_u8m2(<vscale x 16 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -145,8 +145,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssrl.nxv16i8.i64.i64(<vscale x 16 x i8>,
 define <vscale x 32 x i8> @test_vssrl_vv_u8m4(<vscale x 32 x i8> %op1, <vscale x 32 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -159,8 +159,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8.nxv32i8.i64(<vscale x 32 x
 define <vscale x 32 x i8> @test_vssrl_vx_u8m4(<vscale x 32 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -173,8 +173,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssrl.nxv32i8.i64.i64(<vscale x 32 x i8>,
 define <vscale x 64 x i8> @test_vssrl_vv_u8m8(<vscale x 64 x i8> %op1, <vscale x 64 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -187,8 +187,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8.nxv64i8.i64(<vscale x 64 x
 define <vscale x 64 x i8> @test_vssrl_vx_u8m8(<vscale x 64 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -201,8 +201,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssrl.nxv64i8.i64.i64(<vscale x 64 x i8>,
 define <vscale x 1 x i16> @test_vssrl_vv_u16mf4(<vscale x 1 x i16> %op1, <vscale x 1 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -215,8 +215,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16.nxv1i16.i64(<vscale x 1 x i
 define <vscale x 1 x i16> @test_vssrl_vx_u16mf4(<vscale x 1 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16mf4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -229,8 +229,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssrl.nxv1i16.i64.i64(<vscale x 1 x i16>,
 define <vscale x 2 x i16> @test_vssrl_vv_u16mf2(<vscale x 2 x i16> %op1, <vscale x 2 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -243,8 +243,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16.nxv2i16.i64(<vscale x 2 x i
 define <vscale x 2 x i16> @test_vssrl_vx_u16mf2(<vscale x 2 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -257,8 +257,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssrl.nxv2i16.i64.i64(<vscale x 2 x i16>,
 define <vscale x 4 x i16> @test_vssrl_vv_u16m1(<vscale x 4 x i16> %op1, <vscale x 4 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -271,8 +271,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16.nxv4i16.i64(<vscale x 4 x i
 define <vscale x 4 x i16> @test_vssrl_vx_u16m1(<vscale x 4 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -285,8 +285,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssrl.nxv4i16.i64.i64(<vscale x 4 x i16>,
 define <vscale x 8 x i16> @test_vssrl_vv_u16m2(<vscale x 8 x i16> %op1, <vscale x 8 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -299,8 +299,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16.nxv8i16.i64(<vscale x 8 x i
 define <vscale x 8 x i16> @test_vssrl_vx_u16m2(<vscale x 8 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -313,8 +313,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssrl.nxv8i16.i64.i64(<vscale x 8 x i16>,
 define <vscale x 16 x i16> @test_vssrl_vv_u16m4(<vscale x 16 x i16> %op1, <vscale x 16 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -327,8 +327,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16.nxv16i16.i64(<vscale x 16
 define <vscale x 16 x i16> @test_vssrl_vx_u16m4(<vscale x 16 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -341,8 +341,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssrl.nxv16i16.i64.i64(<vscale x 16 x i1
 define <vscale x 32 x i16> @test_vssrl_vv_u16m8(<vscale x 32 x i16> %op1, <vscale x 32 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -355,8 +355,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16.nxv32i16.i64(<vscale x 32
 define <vscale x 32 x i16> @test_vssrl_vx_u16m8(<vscale x 32 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -369,8 +369,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssrl.nxv32i16.i64.i64(<vscale x 32 x i1
 define <vscale x 1 x i32> @test_vssrl_vv_u32mf2(<vscale x 1 x i32> %op1, <vscale x 1 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -383,8 +383,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32.nxv1i32.i64(<vscale x 1 x i
 define <vscale x 1 x i32> @test_vssrl_vx_u32mf2(<vscale x 1 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32mf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -397,8 +397,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssrl.nxv1i32.i64.i64(<vscale x 1 x i32>,
 define <vscale x 2 x i32> @test_vssrl_vv_u32m1(<vscale x 2 x i32> %op1, <vscale x 2 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -411,8 +411,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32.nxv2i32.i64(<vscale x 2 x i
 define <vscale x 2 x i32> @test_vssrl_vx_u32m1(<vscale x 2 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -425,8 +425,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssrl.nxv2i32.i64.i64(<vscale x 2 x i32>,
 define <vscale x 4 x i32> @test_vssrl_vv_u32m2(<vscale x 4 x i32> %op1, <vscale x 4 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -439,8 +439,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.nxv4i32.i64(<vscale x 4 x i
 define <vscale x 4 x i32> @test_vssrl_vx_u32m2(<vscale x 4 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -453,8 +453,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssrl.nxv4i32.i64.i64(<vscale x 4 x i32>,
 define <vscale x 8 x i32> @test_vssrl_vv_u32m4(<vscale x 8 x i32> %op1, <vscale x 8 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -467,8 +467,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32.nxv8i32.i64(<vscale x 8 x i
 define <vscale x 8 x i32> @test_vssrl_vx_u32m4(<vscale x 8 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -481,8 +481,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssrl.nxv8i32.i64.i64(<vscale x 8 x i32>,
 define <vscale x 16 x i32> @test_vssrl_vv_u32m8(<vscale x 16 x i32> %op1, <vscale x 16 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -495,8 +495,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32.nxv16i32.i64(<vscale x 16
 define <vscale x 16 x i32> @test_vssrl_vx_u32m8(<vscale x 16 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -509,8 +509,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssrl.nxv16i32.i64.i64(<vscale x 16 x i3
 define <vscale x 1 x i64> @test_vssrl_vv_u64m1(<vscale x 1 x i64> %op1, <vscale x 1 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
@@ -523,8 +523,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.nxv1i64.i64(<vscale x 1 x i
 define <vscale x 1 x i64> @test_vssrl_vx_u64m1(<vscale x 1 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -537,8 +537,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssrl.nxv1i64.i64.i64(<vscale x 1 x i64>,
 define <vscale x 2 x i64> @test_vssrl_vv_u64m2(<vscale x 2 x i64> %op1, <vscale x 2 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -551,8 +551,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.nxv2i64.i64(<vscale x 2 x i
 define <vscale x 2 x i64> @test_vssrl_vx_u64m2(<vscale x 2 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -565,8 +565,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssrl.nxv2i64.i64.i64(<vscale x 2 x i64>,
 define <vscale x 4 x i64> @test_vssrl_vv_u64m4(<vscale x 4 x i64> %op1, <vscale x 4 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12
 ; CHECK-NEXT:    ret
 entry:
@@ -579,8 +579,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.nxv4i64.i64(<vscale x 4 x i
 define <vscale x 4 x i64> @test_vssrl_vx_u64m4(<vscale x 4 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -593,8 +593,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssrl.nxv4i64.i64.i64(<vscale x 4 x i64>,
 define <vscale x 8 x i64> @test_vssrl_vv_u64m8(<vscale x 8 x i64> %op1, <vscale x 8 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -607,8 +607,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.nxv8i64.i64(<vscale x 8 x i
 define <vscale x 8 x i64> @test_vssrl_vx_u64m8(<vscale x 8 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -621,8 +621,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssrl.nxv8i64.i64.i64(<vscale x 8 x i64>,
 define <vscale x 1 x i8> @test_vssrl_vv_u8mf8_m(<vscale x 1 x i1> %mask, <vscale x 1 x i8> %op1, <vscale x 1 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -635,8 +635,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.nxv1i8.i64(<vscale x 1 x
 define <vscale x 1 x i8> @test_vssrl_vx_u8mf8_m(<vscale x 1 x i1> %mask, <vscale x 1 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -649,8 +649,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vssrl.mask.nxv1i8.i64.i64(<vscale x 1 x i8
 define <vscale x 2 x i8> @test_vssrl_vv_u8mf4_m(<vscale x 2 x i1> %mask, <vscale x 2 x i8> %op1, <vscale x 2 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -663,8 +663,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8.nxv2i8.i64(<vscale x 2 x
 define <vscale x 2 x i8> @test_vssrl_vx_u8mf4_m(<vscale x 2 x i1> %mask, <vscale x 2 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -677,8 +677,8 @@ declare <vscale x 2 x i8> @llvm.riscv.vssrl.mask.nxv2i8.i64.i64(<vscale x 2 x i8
 define <vscale x 4 x i8> @test_vssrl_vv_u8mf2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i8> %op1, <vscale x 4 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -691,8 +691,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8.nxv4i8.i64(<vscale x 4 x
 define <vscale x 4 x i8> @test_vssrl_vx_u8mf2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -705,8 +705,8 @@ declare <vscale x 4 x i8> @llvm.riscv.vssrl.mask.nxv4i8.i64.i64(<vscale x 4 x i8
 define <vscale x 8 x i8> @test_vssrl_vv_u8m1_m(<vscale x 8 x i1> %mask, <vscale x 8 x i8> %op1, <vscale x 8 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -719,8 +719,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x
 define <vscale x 8 x i8> @test_vssrl_vx_u8m1_m(<vscale x 8 x i1> %mask, <vscale x 8 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -733,8 +733,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vssrl.mask.nxv8i8.i64.i64(<vscale x 8 x i8
 define <vscale x 16 x i8> @test_vssrl_vv_u8m2_m(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %op1, <vscale x 16 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -747,8 +747,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8.nxv16i8.i64(<vscale x
 define <vscale x 16 x i8> @test_vssrl_vx_u8m2_m(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -761,8 +761,8 @@ declare <vscale x 16 x i8> @llvm.riscv.vssrl.mask.nxv16i8.i64.i64(<vscale x 16 x
 define <vscale x 32 x i8> @test_vssrl_vv_u8m4_m(<vscale x 32 x i1> %mask, <vscale x 32 x i8> %op1, <vscale x 32 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -775,8 +775,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8.nxv32i8.i64(<vscale x
 define <vscale x 32 x i8> @test_vssrl_vx_u8m4_m(<vscale x 32 x i1> %mask, <vscale x 32 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -789,8 +789,8 @@ declare <vscale x 32 x i8> @llvm.riscv.vssrl.mask.nxv32i8.i64.i64(<vscale x 32 x
 define <vscale x 64 x i8> @test_vssrl_vv_u8m8_m(<vscale x 64 x i1> %mask, <vscale x 64 x i8> %op1, <vscale x 64 x i8> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u8m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -803,8 +803,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8.nxv64i8.i64(<vscale x
 define <vscale x 64 x i8> @test_vssrl_vx_u8m8_m(<vscale x 64 x i1> %mask, <vscale x 64 x i8> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u8m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -817,8 +817,8 @@ declare <vscale x 64 x i8> @llvm.riscv.vssrl.mask.nxv64i8.i64.i64(<vscale x 64 x
 define <vscale x 1 x i16> @test_vssrl_vv_u16mf4_m(<vscale x 1 x i1> %mask, <vscale x 1 x i16> %op1, <vscale x 1 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -831,8 +831,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16.nxv1i16.i64(<vscale x
 define <vscale x 1 x i16> @test_vssrl_vx_u16mf4_m(<vscale x 1 x i1> %mask, <vscale x 1 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16mf4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -845,8 +845,8 @@ declare <vscale x 1 x i16> @llvm.riscv.vssrl.mask.nxv1i16.i64.i64(<vscale x 1 x
 define <vscale x 2 x i16> @test_vssrl_vv_u16mf2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i16> %op1, <vscale x 2 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -859,8 +859,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16.nxv2i16.i64(<vscale x
 define <vscale x 2 x i16> @test_vssrl_vx_u16mf2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -873,8 +873,8 @@ declare <vscale x 2 x i16> @llvm.riscv.vssrl.mask.nxv2i16.i64.i64(<vscale x 2 x
 define <vscale x 4 x i16> @test_vssrl_vv_u16m1_m(<vscale x 4 x i1> %mask, <vscale x 4 x i16> %op1, <vscale x 4 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -887,8 +887,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16.nxv4i16.i64(<vscale x
 define <vscale x 4 x i16> @test_vssrl_vx_u16m1_m(<vscale x 4 x i1> %mask, <vscale x 4 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -901,8 +901,8 @@ declare <vscale x 4 x i16> @llvm.riscv.vssrl.mask.nxv4i16.i64.i64(<vscale x 4 x
 define <vscale x 8 x i16> @test_vssrl_vv_u16m2_m(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %op1, <vscale x 8 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -915,8 +915,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16.nxv8i16.i64(<vscale x
 define <vscale x 8 x i16> @test_vssrl_vx_u16m2_m(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -929,8 +929,8 @@ declare <vscale x 8 x i16> @llvm.riscv.vssrl.mask.nxv8i16.i64.i64(<vscale x 8 x
 define <vscale x 16 x i16> @test_vssrl_vv_u16m4_m(<vscale x 16 x i1> %mask, <vscale x 16 x i16> %op1, <vscale x 16 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -943,8 +943,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16.nxv16i16.i64(<vscale
 define <vscale x 16 x i16> @test_vssrl_vx_u16m4_m(<vscale x 16 x i1> %mask, <vscale x 16 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -957,8 +957,8 @@ declare <vscale x 16 x i16> @llvm.riscv.vssrl.mask.nxv16i16.i64.i64(<vscale x 16
 define <vscale x 32 x i16> @test_vssrl_vv_u16m8_m(<vscale x 32 x i1> %mask, <vscale x 32 x i16> %op1, <vscale x 32 x i16> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u16m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -971,8 +971,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16.nxv32i16.i64(<vscale
 define <vscale x 32 x i16> @test_vssrl_vx_u16m8_m(<vscale x 32 x i1> %mask, <vscale x 32 x i16> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u16m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -985,8 +985,8 @@ declare <vscale x 32 x i16> @llvm.riscv.vssrl.mask.nxv32i16.i64.i64(<vscale x 32
 define <vscale x 1 x i32> @test_vssrl_vv_u32mf2_m(<vscale x 1 x i1> %mask, <vscale x 1 x i32> %op1, <vscale x 1 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -999,8 +999,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32.nxv1i32.i64(<vscale x
 define <vscale x 1 x i32> @test_vssrl_vx_u32mf2_m(<vscale x 1 x i1> %mask, <vscale x 1 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32mf2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1013,8 +1013,8 @@ declare <vscale x 1 x i32> @llvm.riscv.vssrl.mask.nxv1i32.i64.i64(<vscale x 1 x
 define <vscale x 2 x i32> @test_vssrl_vv_u32m1_m(<vscale x 2 x i1> %mask, <vscale x 2 x i32> %op1, <vscale x 2 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1027,8 +1027,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32.nxv2i32.i64(<vscale x
 define <vscale x 2 x i32> @test_vssrl_vx_u32m1_m(<vscale x 2 x i1> %mask, <vscale x 2 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1041,8 +1041,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vssrl.mask.nxv2i32.i64.i64(<vscale x 2 x
 define <vscale x 4 x i32> @test_vssrl_vv_u32m2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i32> %op1, <vscale x 4 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1055,8 +1055,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32.nxv4i32.i64(<vscale x
 define <vscale x 4 x i32> @test_vssrl_vx_u32m2_m(<vscale x 4 x i1> %mask, <vscale x 4 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1069,8 +1069,8 @@ declare <vscale x 4 x i32> @llvm.riscv.vssrl.mask.nxv4i32.i64.i64(<vscale x 4 x
 define <vscale x 8 x i32> @test_vssrl_vv_u32m4_m(<vscale x 8 x i1> %mask, <vscale x 8 x i32> %op1, <vscale x 8 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1083,8 +1083,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32.nxv8i32.i64(<vscale x
 define <vscale x 8 x i32> @test_vssrl_vx_u32m4_m(<vscale x 8 x i1> %mask, <vscale x 8 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1097,8 +1097,8 @@ declare <vscale x 8 x i32> @llvm.riscv.vssrl.mask.nxv8i32.i64.i64(<vscale x 8 x
 define <vscale x 16 x i32> @test_vssrl_vv_u32m8_m(<vscale x 16 x i1> %mask, <vscale x 16 x i32> %op1, <vscale x 16 x i32> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u32m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1111,8 +1111,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32.nxv16i32.i64(<vscale
 define <vscale x 16 x i32> @test_vssrl_vx_u32m8_m(<vscale x 16 x i1> %mask, <vscale x 16 x i32> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u32m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1125,8 +1125,8 @@ declare <vscale x 16 x i32> @llvm.riscv.vssrl.mask.nxv16i32.i64.i64(<vscale x 16
 define <vscale x 1 x i64> @test_vssrl_vv_u64m1_m(<vscale x 1 x i1> %mask, <vscale x 1 x i64> %op1, <vscale x 1 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1139,8 +1139,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.nxv1i64.i64(<vscale x
 define <vscale x 1 x i64> @test_vssrl_vx_u64m1_m(<vscale x 1 x i1> %mask, <vscale x 1 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m1_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1153,8 +1153,8 @@ declare <vscale x 1 x i64> @llvm.riscv.vssrl.mask.nxv1i64.i64.i64(<vscale x 1 x
 define <vscale x 2 x i64> @test_vssrl_vv_u64m2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %op1, <vscale x 2 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v10, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1167,8 +1167,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.nxv2i64.i64(<vscale x
 define <vscale x 2 x i64> @test_vssrl_vx_u64m2_m(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m2_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1181,8 +1181,8 @@ declare <vscale x 2 x i64> @llvm.riscv.vssrl.mask.nxv2i64.i64.i64(<vscale x 2 x
 define <vscale x 4 x i64> @test_vssrl_vv_u64m4_m(<vscale x 4 x i1> %mask, <vscale x 4 x i64> %op1, <vscale x 4 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v12, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1195,8 +1195,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.nxv4i64.i64(<vscale x
 define <vscale x 4 x i64> @test_vssrl_vx_u64m4_m(<vscale x 4 x i1> %mask, <vscale x 4 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m4_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m4, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1209,8 +1209,8 @@ declare <vscale x 4 x i64> @llvm.riscv.vssrl.mask.nxv4i64.i64.i64(<vscale x 4 x
 define <vscale x 8 x i64> @test_vssrl_vv_u64m8_m(<vscale x 8 x i1> %mask, <vscale x 8 x i64> %op1, <vscale x 8 x i64> %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vv_u64m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    ret
 entry:
@@ -1223,8 +1223,8 @@ declare <vscale x 8 x i64> @llvm.riscv.vssrl.mask.nxv8i64.nxv8i64.i64(<vscale x
 define <vscale x 8 x i64> @test_vssrl_vx_u64m8_m(<vscale x 8 x i1> %mask, <vscale x 8 x i64> %op1, i64 %shift, i64 %vl) {
 ; CHECK-LABEL: test_vssrl_vx_u64m8_m:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-NEXT:    vssrl.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vxrm-insert.ll b/llvm/test/CodeGen/RISCV/rvv/vxrm-insert.ll
index 10175218a440..c5f34eee3118 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vxrm-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vxrm-insert.ll
@@ -20,8 +20,8 @@ declare <vscale x 1 x i8> @llvm.riscv.vasub.nxv1i8.nxv1i8(
 define <vscale x 1 x i8> @test1(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: test1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    vaadd.vv v8, v8, v10
 ; CHECK-NEXT:    ret
@@ -44,8 +44,8 @@ entry:
 define <vscale x 1 x i8> @test2(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: test2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 2
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    csrwi vxrm, 0
 ; CHECK-NEXT:    vaadd.vv v8, v8, v10
@@ -80,12 +80,12 @@ define <vscale x 1 x i8> @test3(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vsc
 ; RV32-NEXT:    mv s0, a0
 ; RV32-NEXT:    addi a1, sp, 16
 ; RV32-NEXT:    vs1r.v v10, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; RV32-NEXT:    csrwi vxrm, 0
+; RV32-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; RV32-NEXT:    vaadd.vv v8, v8, v9
 ; RV32-NEXT:    call foo
-; RV32-NEXT:    vsetvli zero, s0, e8, mf8, ta, ma
 ; RV32-NEXT:    csrwi vxrm, 0
+; RV32-NEXT:    vsetvli zero, s0, e8, mf8, ta, ma
 ; RV32-NEXT:    addi a0, sp, 16
 ; RV32-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vaadd.vv v8, v8, v9
@@ -108,12 +108,12 @@ define <vscale x 1 x i8> @test3(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vsc
 ; RV64-NEXT:    mv s0, a0
 ; RV64-NEXT:    addi a1, sp, 16
 ; RV64-NEXT:    vs1r.v v10, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; RV64-NEXT:    vaadd.vv v8, v8, v9
 ; RV64-NEXT:    call foo
-; RV64-NEXT:    vsetvli zero, s0, e8, mf8, ta, ma
 ; RV64-NEXT:    csrwi vxrm, 0
+; RV64-NEXT:    vsetvli zero, s0, e8, mf8, ta, ma
 ; RV64-NEXT:    addi a0, sp, 16
 ; RV64-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
 ; RV64-NEXT:    vaadd.vv v8, v8, v9
@@ -144,13 +144,13 @@ entry:
 define <vscale x 1 x i8> @test4(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vscale x 1 x i8> %2, iXLen %3) nounwind {
 ; CHECK-LABEL: test4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v10
 ; CHECK-NEXT:    ret
 entry:
@@ -174,8 +174,8 @@ define <vscale x 1 x i8> @test5(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vsc
 ; CHECK-LABEL: test5:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andi a1, a1, 1
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    beqz a1, .LBB4_2
 ; CHECK-NEXT:  # %bb.1: # %condblock
@@ -249,8 +249,8 @@ define <vscale x 1 x i8> @test7(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, <vsc
 ; CHECK-LABEL: test7:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andi a1, a1, 1
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v8, v8, v9
 ; CHECK-NEXT:    beqz a1, .LBB6_2
 ; CHECK-NEXT:  # %bb.1: # %trueblock
@@ -480,8 +480,8 @@ define <vscale x 1 x i8> @test12(i1 %c1, <vscale x 1 x i8> %0, <vscale x 1 x i8>
 ; CHECK-LABEL: test12:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andi a0, a0, 1
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v9, v8, v9
 ; CHECK-NEXT:    beqz a0, .LBB11_2
 ; CHECK-NEXT:  # %bb.1: # %block1
@@ -513,8 +513,8 @@ define <vscale x 1 x i8> @test13(i1 %c1, i1 %c2, i1 %c3, <vscale x 1 x i8> %0, <
 ; CHECK-LABEL: test13:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    andi a0, a0, 1
-; CHECK-NEXT:    vsetvli zero, a3, e8, mf8, ta, ma
 ; CHECK-NEXT:    csrwi vxrm, 0
+; CHECK-NEXT:    vsetvli zero, a3, e8, mf8, ta, ma
 ; CHECK-NEXT:    vaadd.vv v10, v8, v9
 ; CHECK-NEXT:    beqz a0, .LBB12_2
 ; CHECK-NEXT:  # %bb.1: # %block1
diff --git a/llvm/test/CodeGen/RISCV/rvv/vxrm.mir b/llvm/test/CodeGen/RISCV/rvv/vxrm.mir
index eac3cfca209e..2bac1eeb9060 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vxrm.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vxrm.mir
@@ -11,14 +11,14 @@ body:     |
     ; MIR-LABEL: name: verify_vxrm
     ; MIR: liveins: $v8, $v9, $x10
     ; MIR-NEXT: {{  $}}
-    ; MIR-NEXT: dead $x0 = PseudoVSETVLI killed renamable $x10, 197 /* e8, mf8, ta, ma */, implicit-def $vl, implicit-def $vtype
     ; MIR-NEXT: WriteVXRMImm 0, implicit-def $vxrm
-    ; MIR-NEXT: renamable $v8 = PseudoVAADD_VV_MF8 undef $v8, killed renamable $v8, killed renamable $v9, 0, $noreg, 3 /* e8 */, 0  /* tu, mu */, implicit $vl, implicit $vtype, implicit $vxrm
+    ; MIR-NEXT: dead $x0 = PseudoVSETVLI killed renamable $x10, 197 /* e8, mf8, ta, ma */, implicit-def $vl, implicit-def $vtype
+    ; MIR-NEXT: renamable $v8 = PseudoVAADD_VV_MF8 undef $v8, killed renamable $v8, killed renamable $v9, 0, $noreg, 3 /* e8 */, 0  /* tu, mu */, implicit $vxrm, implicit $vl, implicit $vtype
     ; MIR-NEXT: PseudoRET implicit $v8
     ; ASM-LABEL: verify_vxrm:
     ; ASM:        # %bb.0:
-    ; ASM-NEXT:    vsetvli	zero, a0, e8, mf8, ta, ma
     ; ASM-NEXT:    csrwi	vxrm, 0
+    ; ASM-NEXT:    vsetvli	zero, a0, e8, mf8, ta, ma
     ; ASM-NEXT:    vaadd.vv	v8, v8, v9
     ; ASM-NEXT:    ret
     %0:vr = COPY $v8
-- 
GitLab


From 5bde2aa1080ba90021f8f5e0c48744ddfc0d6f15 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Fri, 10 May 2024 08:34:42 +0200
Subject: [PATCH 0377/1206] [lldb] Improve type name parsing (#91586)

Parsing of '::' scopes in TypeQuery was very naive and failed for names
with '::''s in template arguments. Interestingly, one of the functions
it was calling (Type::GetTypeScopeAndBasename) was already doing the
same thing, and getting it (mostly (*)) right. This refactors the
function so that it can return the scope results, fixing the parsing of
names like std::vector<int, std::allocator<int>>::iterator.

Two callers of GetTypeScopeAndBasename are deleted as the functions are
not used (I presume they stopped being used once we started pruning type
search results more eagerly).

(*) This implementation is still not correct when one takes c++
operators into account -- e.g., something like `X<&A::operator<>::T` is
a legitimate type name. We do have an implementation that is able to
handle names like these (CPlusPlusLanguage::MethodName), but using it is
not trivial, because it is hidden in a language plugin and specific to
method name parsing.

---------

Co-authored-by: Michael Buch <michaelbuch12@gmail.com>
---
 lldb/include/lldb/Symbol/Type.h               |  35 ++++-
 lldb/include/lldb/Symbol/TypeList.h           |   9 --
 lldb/include/lldb/Symbol/TypeMap.h            |   4 -
 lldb/source/Symbol/Type.cpp                   | 130 ++++++++----------
 lldb/source/Symbol/TypeList.cpp               | 109 ---------------
 lldb/source/Symbol/TypeMap.cpp                |  73 ----------
 .../python_api/sbmodule/FindTypes/Makefile    |   3 +
 .../FindTypes/TestSBModuleFindTypes.py        |  40 ++++++
 .../python_api/sbmodule/FindTypes/main.cpp    |  17 +++
 lldb/unittests/Symbol/TestType.cpp            |  62 ++++-----
 10 files changed, 180 insertions(+), 302 deletions(-)
 create mode 100644 lldb/test/API/python_api/sbmodule/FindTypes/Makefile
 create mode 100644 lldb/test/API/python_api/sbmodule/FindTypes/TestSBModuleFindTypes.py
 create mode 100644 lldb/test/API/python_api/sbmodule/FindTypes/main.cpp

diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index 1c4f7b5601b0..7aa0852676e4 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -21,6 +21,8 @@
 
 #include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLForwardCompat.h"
+#include "llvm/Support/raw_ostream.h"
 
 #include <optional>
 #include <set>
@@ -492,12 +494,37 @@ public:
 
   static int Compare(const Type &a, const Type &b);
 
+  // Represents a parsed type name coming out of GetTypeScopeAndBasename. The
+  // structure holds StringRefs pointing to portions of the original name, and
+  // so must not be used after the name is destroyed.
+  struct ParsedName {
+    lldb::TypeClass type_class = lldb::eTypeClassAny;
+
+    // Scopes of the type, starting with the outermost. Absolute type references
+    // have a "::" as the first scope.
+    llvm::SmallVector<llvm::StringRef> scope;
+
+    llvm::StringRef basename;
+
+    friend bool operator==(const ParsedName &lhs, const ParsedName &rhs) {
+      return lhs.type_class == rhs.type_class && lhs.scope == rhs.scope &&
+             lhs.basename == rhs.basename;
+    }
+
+    friend llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
+                                         const ParsedName &name) {
+      return os << llvm::formatv(
+                 "Type::ParsedName({0:x}, [{1}], {2})",
+                 llvm::to_underlying(name.type_class),
+                 llvm::make_range(name.scope.begin(), name.scope.end()),
+                 name.basename);
+    }
+  };
   // From a fully qualified typename, split the type into the type basename and
   // the remaining type scope (namespaces/classes).
-  static bool GetTypeScopeAndBasename(llvm::StringRef name,
-                                      llvm::StringRef &scope,
-                                      llvm::StringRef &basename,
-                                      lldb::TypeClass &type_class);
+  static std::optional<ParsedName>
+  GetTypeScopeAndBasename(llvm::StringRef name);
+
   void SetEncodingType(Type *encoding_type) { m_encoding_type = encoding_type; }
 
   uint32_t GetEncodingMask();
diff --git a/lldb/include/lldb/Symbol/TypeList.h b/lldb/include/lldb/Symbol/TypeList.h
index 403469c989f5..d58772ad5b62 100644
--- a/lldb/include/lldb/Symbol/TypeList.h
+++ b/lldb/include/lldb/Symbol/TypeList.h
@@ -49,15 +49,6 @@ public:
 
   void ForEach(std::function<bool(lldb::TypeSP &type_sp)> const &callback);
 
-  void RemoveMismatchedTypes(llvm::StringRef qualified_typename,
-                             bool exact_match);
-
-  void RemoveMismatchedTypes(llvm::StringRef type_scope,
-                             llvm::StringRef type_basename,
-                             lldb::TypeClass type_class, bool exact_match);
-
-  void RemoveMismatchedTypes(lldb::TypeClass type_class);
-
 private:
   typedef collection::iterator iterator;
   typedef collection::const_iterator const_iterator;
diff --git a/lldb/include/lldb/Symbol/TypeMap.h b/lldb/include/lldb/Symbol/TypeMap.h
index 433711875e55..89011efab5c3 100644
--- a/lldb/include/lldb/Symbol/TypeMap.h
+++ b/lldb/include/lldb/Symbol/TypeMap.h
@@ -55,10 +55,6 @@ public:
 
   bool Remove(const lldb::TypeSP &type_sp);
 
-  void RemoveMismatchedTypes(llvm::StringRef type_scope,
-                             llvm::StringRef type_basename,
-                             lldb::TypeClass type_class, bool exact_match);
-
 private:
   typedef collection::iterator iterator;
   typedef collection::const_iterator const_iterator;
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index b85c38097ebe..6bf69c2ded28 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -29,6 +29,7 @@
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/Target.h"
+#include "lldb/lldb-enumerations.h"
 
 #include "llvm/ADT/StringRef.h"
 
@@ -85,27 +86,23 @@ static CompilerContextKind ConvertTypeClass(lldb::TypeClass type_class) {
 
 TypeQuery::TypeQuery(llvm::StringRef name, TypeQueryOptions options)
     : m_options(options) {
-  llvm::StringRef scope, basename;
-  lldb::TypeClass type_class = lldb::eTypeClassAny;
-  if (Type::GetTypeScopeAndBasename(name, scope, basename, type_class)) {
-    if (scope.consume_front("::"))
-      m_options |= e_exact_match;
+  if (std::optional<Type::ParsedName> parsed_name =
+          Type::GetTypeScopeAndBasename(name)) {
+    llvm::ArrayRef scope = parsed_name->scope;
     if (!scope.empty()) {
-      std::pair<llvm::StringRef, llvm::StringRef> scope_pair =
-          scope.split("::");
-      while (!scope_pair.second.empty()) {
-        m_context.push_back({CompilerContextKind::AnyDeclContext,
-                             ConstString(scope_pair.first.str())});
-        scope_pair = scope_pair.second.split("::");
+      if (scope[0] == "::") {
+        m_options |= e_exact_match;
+        scope = scope.drop_front();
+      }
+      for (llvm::StringRef s : scope) {
+        m_context.push_back(
+            {CompilerContextKind::AnyDeclContext, ConstString(s)});
       }
-      m_context.push_back({CompilerContextKind::AnyDeclContext,
-                           ConstString(scope_pair.first.str())});
     }
-    m_context.push_back(
-        {ConvertTypeClass(type_class), ConstString(basename.str())});
+    m_context.push_back({ConvertTypeClass(parsed_name->type_class),
+                         ConstString(parsed_name->basename)});
   } else {
-    m_context.push_back(
-        {CompilerContextKind::AnyType, ConstString(name.str())});
+    m_context.push_back({CompilerContextKind::AnyType, ConstString(name)});
   }
 }
 
@@ -773,65 +770,56 @@ ConstString Type::GetQualifiedName() {
   return GetForwardCompilerType().GetTypeName();
 }
 
-bool Type::GetTypeScopeAndBasename(llvm::StringRef name,
-                                   llvm::StringRef &scope,
-                                   llvm::StringRef &basename,
-                                   TypeClass &type_class) {
-  type_class = eTypeClassAny;
+std::optional<Type::ParsedName>
+Type::GetTypeScopeAndBasename(llvm::StringRef name) {
+  ParsedName result;
 
   if (name.empty())
-    return false;
-
-  // Clear the scope in case we have just a type class and a basename.
-  scope = llvm::StringRef();
-  basename = name;
-  if (basename.consume_front("struct "))
-    type_class = eTypeClassStruct;
-  else if (basename.consume_front("class "))
-    type_class = eTypeClassClass;
-  else if (basename.consume_front("union "))
-    type_class = eTypeClassUnion;
-  else if (basename.consume_front("enum "))
-    type_class = eTypeClassEnumeration;
-  else if (basename.consume_front("typedef "))
-    type_class = eTypeClassTypedef;
-
-  size_t namespace_separator = basename.find("::");
-  if (namespace_separator == llvm::StringRef::npos) {
-    // If "name" started a type class we need to return true with no scope.
-    return type_class != eTypeClassAny;
-  }
-
-  size_t template_begin = basename.find('<');
-  while (namespace_separator != llvm::StringRef::npos) {
-    if (template_begin != llvm::StringRef::npos &&
-        namespace_separator > template_begin) {
-      size_t template_depth = 1;
-      llvm::StringRef template_arg =
-          basename.drop_front(template_begin + 1);
-      while (template_depth > 0 && !template_arg.empty()) {
-        if (template_arg.front() == '<')
-          template_depth++;
-        else if (template_arg.front() == '>')
-          template_depth--;
-        template_arg = template_arg.drop_front(1);
+    return std::nullopt;
+
+  if (name.consume_front("struct "))
+    result.type_class = eTypeClassStruct;
+  else if (name.consume_front("class "))
+    result.type_class = eTypeClassClass;
+  else if (name.consume_front("union "))
+    result.type_class = eTypeClassUnion;
+  else if (name.consume_front("enum "))
+    result.type_class = eTypeClassEnumeration;
+  else if (name.consume_front("typedef "))
+    result.type_class = eTypeClassTypedef;
+
+  if (name.consume_front("::"))
+    result.scope.push_back("::");
+
+  bool prev_is_colon = false;
+  size_t template_depth = 0;
+  size_t name_begin = 0;
+  for (const auto &pos : llvm::enumerate(name)) {
+    switch (pos.value()) {
+    case ':':
+      if (prev_is_colon && template_depth == 0) {
+        result.scope.push_back(name.slice(name_begin, pos.index() - 1));
+        name_begin = pos.index() + 1;
       }
-      if (template_depth != 0)
-        return false; // We have an invalid type name. Bail out.
-      if (template_arg.empty())
-        break; // The template ends at the end of the full name.
-      basename = template_arg;
-    } else {
-      basename = basename.drop_front(namespace_separator + 2);
+      break;
+    case '<':
+      ++template_depth;
+      break;
+    case '>':
+      if (template_depth == 0)
+        return std::nullopt; // Invalid name.
+      --template_depth;
+      break;
     }
-    template_begin = basename.find('<');
-    namespace_separator = basename.find("::");
-  }
-  if (basename.size() < name.size()) {
-    scope = name.take_front(name.size() - basename.size());
-    return true;
+    prev_is_colon = pos.value() == ':';
   }
-  return false;
+
+  if (name_begin < name.size() && template_depth == 0)
+    result.basename = name.substr(name_begin);
+  else
+    return std::nullopt;
+
+  return result;
 }
 
 ModuleSP Type::GetModule() {
diff --git a/lldb/source/Symbol/TypeList.cpp b/lldb/source/Symbol/TypeList.cpp
index 2e101e0a8f57..574887189315 100644
--- a/lldb/source/Symbol/TypeList.cpp
+++ b/lldb/source/Symbol/TypeList.cpp
@@ -96,112 +96,3 @@ void TypeList::Dump(Stream *s, bool show_context) {
     if (Type *t = pos->get())
       t->Dump(s, show_context);
 }
-
-void TypeList::RemoveMismatchedTypes(llvm::StringRef qualified_typename,
-                                     bool exact_match) {
-  llvm::StringRef type_scope;
-  llvm::StringRef type_basename;
-  TypeClass type_class = eTypeClassAny;
-  if (!Type::GetTypeScopeAndBasename(qualified_typename, type_scope,
-                                     type_basename, type_class)) {
-    type_basename = qualified_typename;
-    type_scope = "";
-  }
-  return RemoveMismatchedTypes(type_scope, type_basename, type_class,
-                               exact_match);
-}
-
-void TypeList::RemoveMismatchedTypes(llvm::StringRef type_scope,
-                                     llvm::StringRef type_basename,
-                                     TypeClass type_class, bool exact_match) {
-  // Our "collection" type currently is a std::map which doesn't have any good
-  // way to iterate and remove items from the map so we currently just make a
-  // new list and add all of the matching types to it, and then swap it into
-  // m_types at the end
-  collection matching_types;
-
-  iterator pos, end = m_types.end();
-
-  for (pos = m_types.begin(); pos != end; ++pos) {
-    Type *the_type = pos->get();
-    bool keep_match = false;
-    TypeClass match_type_class = eTypeClassAny;
-
-    if (type_class != eTypeClassAny) {
-      match_type_class = the_type->GetForwardCompilerType().GetTypeClass();
-      if ((match_type_class & type_class) == 0)
-        continue;
-    }
-
-    ConstString match_type_name_const_str(the_type->GetQualifiedName());
-    if (match_type_name_const_str) {
-      const char *match_type_name = match_type_name_const_str.GetCString();
-      llvm::StringRef match_type_scope;
-      llvm::StringRef match_type_basename;
-      if (Type::GetTypeScopeAndBasename(match_type_name, match_type_scope,
-                                        match_type_basename,
-                                        match_type_class)) {
-        if (match_type_basename == type_basename) {
-          const size_t type_scope_size = type_scope.size();
-          const size_t match_type_scope_size = match_type_scope.size();
-          if (exact_match || (type_scope_size == match_type_scope_size)) {
-            keep_match = match_type_scope == type_scope;
-          } else {
-            if (match_type_scope_size > type_scope_size) {
-              const size_t type_scope_pos = match_type_scope.rfind(type_scope);
-              if (type_scope_pos == match_type_scope_size - type_scope_size) {
-                if (type_scope_pos >= 2) {
-                  // Our match scope ends with the type scope we were looking
-                  // for, but we need to make sure what comes before the
-                  // matching type scope is a namespace boundary in case we are
-                  // trying to match: type_basename = "d" type_scope = "b::c::"
-                  // We want to match:
-                  //  match_type_scope "a::b::c::"
-                  // But not:
-                  //  match_type_scope "a::bb::c::"
-                  // So below we make sure what comes before "b::c::" in
-                  // match_type_scope is "::", or the namespace boundary
-                  if (match_type_scope[type_scope_pos - 1] == ':' &&
-                      match_type_scope[type_scope_pos - 2] == ':') {
-                    keep_match = true;
-                  }
-                }
-              }
-            }
-          }
-        }
-      } else {
-        // The type we are currently looking at doesn't exists in a namespace
-        // or class, so it only matches if there is no type scope...
-        keep_match = type_scope.empty() && type_basename == match_type_name;
-      }
-    }
-
-    if (keep_match) {
-      matching_types.push_back(*pos);
-    }
-  }
-  m_types.swap(matching_types);
-}
-
-void TypeList::RemoveMismatchedTypes(TypeClass type_class) {
-  if (type_class == eTypeClassAny)
-    return;
-
-  // Our "collection" type currently is a std::map which doesn't have any good
-  // way to iterate and remove items from the map so we currently just make a
-  // new list and add all of the matching types to it, and then swap it into
-  // m_types at the end
-  collection matching_types;
-
-  iterator pos, end = m_types.end();
-
-  for (pos = m_types.begin(); pos != end; ++pos) {
-    Type *the_type = pos->get();
-    TypeClass match_type_class =
-        the_type->GetForwardCompilerType().GetTypeClass();
-    if (match_type_class & type_class)
-      matching_types.push_back(*pos);
-  }
-  m_types.swap(matching_types);
-}
diff --git a/lldb/source/Symbol/TypeMap.cpp b/lldb/source/Symbol/TypeMap.cpp
index 8933de53749c..9d7c05f318a1 100644
--- a/lldb/source/Symbol/TypeMap.cpp
+++ b/lldb/source/Symbol/TypeMap.cpp
@@ -132,76 +132,3 @@ void TypeMap::Dump(Stream *s, bool show_context,
   for (const auto &pair : m_types)
     pair.second->Dump(s, show_context, level);
 }
-
-void TypeMap::RemoveMismatchedTypes(llvm::StringRef type_scope,
-                                    llvm::StringRef type_basename,
-                                    TypeClass type_class, bool exact_match) {
-  // Our "collection" type currently is a std::map which doesn't have any good
-  // way to iterate and remove items from the map so we currently just make a
-  // new list and add all of the matching types to it, and then swap it into
-  // m_types at the end
-  collection matching_types;
-
-  iterator pos, end = m_types.end();
-
-  for (pos = m_types.begin(); pos != end; ++pos) {
-    Type *the_type = pos->second.get();
-    bool keep_match = false;
-    TypeClass match_type_class = eTypeClassAny;
-
-    if (type_class != eTypeClassAny) {
-      match_type_class = the_type->GetForwardCompilerType().GetTypeClass();
-      if ((match_type_class & type_class) == 0)
-        continue;
-    }
-
-    ConstString match_type_name_const_str(the_type->GetQualifiedName());
-    if (match_type_name_const_str) {
-      const char *match_type_name = match_type_name_const_str.GetCString();
-      llvm::StringRef match_type_scope;
-      llvm::StringRef match_type_basename;
-      if (Type::GetTypeScopeAndBasename(match_type_name, match_type_scope,
-                                        match_type_basename,
-                                        match_type_class)) {
-        if (match_type_basename == type_basename) {
-          const size_t type_scope_size = type_scope.size();
-          const size_t match_type_scope_size = match_type_scope.size();
-          if (exact_match || (type_scope_size == match_type_scope_size)) {
-            keep_match = match_type_scope == type_scope;
-          } else {
-            if (match_type_scope_size > type_scope_size) {
-              const size_t type_scope_pos = match_type_scope.rfind(type_scope);
-              if (type_scope_pos == match_type_scope_size - type_scope_size) {
-                if (type_scope_pos >= 2) {
-                  // Our match scope ends with the type scope we were looking
-                  // for, but we need to make sure what comes before the
-                  // matching type scope is a namespace boundary in case we are
-                  // trying to match: type_basename = "d" type_scope = "b::c::"
-                  // We want to match:
-                  //  match_type_scope "a::b::c::"
-                  // But not:
-                  //  match_type_scope "a::bb::c::"
-                  // So below we make sure what comes before "b::c::" in
-                  // match_type_scope is "::", or the namespace boundary
-                  if (match_type_scope[type_scope_pos - 1] == ':' &&
-                      match_type_scope[type_scope_pos - 2] == ':') {
-                    keep_match = true;
-                  }
-                }
-              }
-            }
-          }
-        }
-      } else {
-        // The type we are currently looking at doesn't exists in a namespace
-        // or class, so it only matches if there is no type scope...
-        keep_match = type_scope.empty() && type_basename == match_type_name;
-      }
-    }
-
-    if (keep_match) {
-      matching_types.insert(*pos);
-    }
-  }
-  m_types.swap(matching_types);
-}
diff --git a/lldb/test/API/python_api/sbmodule/FindTypes/Makefile b/lldb/test/API/python_api/sbmodule/FindTypes/Makefile
new file mode 100644
index 000000000000..99998b20bcb0
--- /dev/null
+++ b/lldb/test/API/python_api/sbmodule/FindTypes/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/python_api/sbmodule/FindTypes/TestSBModuleFindTypes.py b/lldb/test/API/python_api/sbmodule/FindTypes/TestSBModuleFindTypes.py
new file mode 100644
index 000000000000..5c3d2b4187dd
--- /dev/null
+++ b/lldb/test/API/python_api/sbmodule/FindTypes/TestSBModuleFindTypes.py
@@ -0,0 +1,40 @@
+"""Test the SBModule::FindTypes."""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestSBModuleFindTypes(TestBase):
+    def test_lookup_in_template_scopes(self):
+        self.build()
+        spec = lldb.SBModuleSpec()
+        spec.SetFileSpec(lldb.SBFileSpec(self.getBuildArtifact()))
+        module = lldb.SBModule(spec)
+
+        self.assertEqual(
+            set([t.GetName() for t in module.FindTypes("LookMeUp")]),
+            set(
+                [
+                    "ns1::Foo<void>::LookMeUp",
+                    "ns2::Bar<void>::LookMeUp",
+                    "ns1::Foo<ns2::Bar<void> >::LookMeUp",
+                ]
+            ),
+        )
+
+        self.assertEqual(
+            set([t.GetName() for t in module.FindTypes("ns1::Foo<void>::LookMeUp")]),
+            set(["ns1::Foo<void>::LookMeUp"]),
+        )
+
+        self.assertEqual(
+            set(
+                [
+                    t.GetName()
+                    for t in module.FindTypes("ns1::Foo<ns2::Bar<void> >::LookMeUp")
+                ]
+            ),
+            set(["ns1::Foo<ns2::Bar<void> >::LookMeUp"]),
+        )
diff --git a/lldb/test/API/python_api/sbmodule/FindTypes/main.cpp b/lldb/test/API/python_api/sbmodule/FindTypes/main.cpp
new file mode 100644
index 000000000000..cb2646ce312a
--- /dev/null
+++ b/lldb/test/API/python_api/sbmodule/FindTypes/main.cpp
@@ -0,0 +1,17 @@
+namespace ns1 {
+template <typename T> struct Foo {
+  struct LookMeUp {};
+};
+} // namespace ns1
+
+namespace ns2 {
+template <typename T> struct Bar {
+  struct LookMeUp {};
+};
+} // namespace ns2
+
+ns1::Foo<void>::LookMeUp l1;
+ns2::Bar<void>::LookMeUp l2;
+ns1::Foo<ns2::Bar<void>>::LookMeUp l3;
+
+int main() {}
diff --git a/lldb/unittests/Symbol/TestType.cpp b/lldb/unittests/Symbol/TestType.cpp
index 73f5811434fd..da849d804e4d 100644
--- a/lldb/unittests/Symbol/TestType.cpp
+++ b/lldb/unittests/Symbol/TestType.cpp
@@ -10,43 +10,41 @@
 #include "gtest/gtest.h"
 
 #include "lldb/Symbol/Type.h"
+#include "lldb/lldb-enumerations.h"
 
 using namespace lldb;
 using namespace lldb_private;
 
-namespace {
-void TestGetTypeScopeAndBasenameHelper(const char *full_type,
-                                       bool expected_is_scoped,
-                                       const char *expected_scope,
-                                       const char *expected_name) {
-  llvm::StringRef scope, name;
-  lldb::TypeClass type_class;
-  bool is_scoped =
-      Type::GetTypeScopeAndBasename(full_type, scope, name, type_class);
-  EXPECT_EQ(is_scoped, expected_is_scoped);
-  if (expected_is_scoped) {
-    EXPECT_EQ(scope, expected_scope);
-    EXPECT_EQ(name, expected_name);
-  }
-}
-}
-
 TEST(Type, GetTypeScopeAndBasename) {
-  TestGetTypeScopeAndBasenameHelper("int", false, "", "");
-  TestGetTypeScopeAndBasenameHelper("std::string", true, "std::", "string");
-  TestGetTypeScopeAndBasenameHelper("std::set<int>", true, "std::", "set<int>");
-  TestGetTypeScopeAndBasenameHelper("std::set<int, std::less<int>>", true,
-                                    "std::", "set<int, std::less<int>>");
-  TestGetTypeScopeAndBasenameHelper("std::string::iterator", true,
-                                    "std::string::", "iterator");
-  TestGetTypeScopeAndBasenameHelper("std::set<int>::iterator", true,
-                                    "std::set<int>::", "iterator");
-  TestGetTypeScopeAndBasenameHelper(
-      "std::set<int, std::less<int>>::iterator", true,
-      "std::set<int, std::less<int>>::", "iterator");
-  TestGetTypeScopeAndBasenameHelper(
-      "std::set<int, std::less<int>>::iterator<bool>", true,
-      "std::set<int, std::less<int>>::", "iterator<bool>");
+  EXPECT_EQ(Type::GetTypeScopeAndBasename("int"),
+            (Type::ParsedName{eTypeClassAny, {}, "int"}));
+  EXPECT_EQ(Type::GetTypeScopeAndBasename("std::string"),
+            (Type::ParsedName{eTypeClassAny, {"std"}, "string"}));
+  EXPECT_EQ(Type::GetTypeScopeAndBasename("::std::string"),
+            (Type::ParsedName{eTypeClassAny, {"::", "std"}, "string"}));
+  EXPECT_EQ(Type::GetTypeScopeAndBasename("struct std::string"),
+            (Type::ParsedName{eTypeClassStruct, {"std"}, "string"}));
+  EXPECT_EQ(Type::GetTypeScopeAndBasename("std::set<int>"),
+            (Type::ParsedName{eTypeClassAny, {"std"}, "set<int>"}));
+  EXPECT_EQ(
+      Type::GetTypeScopeAndBasename("std::set<int, std::less<int>>"),
+      (Type::ParsedName{eTypeClassAny, {"std"}, "set<int, std::less<int>>"}));
+  EXPECT_EQ(Type::GetTypeScopeAndBasename("std::string::iterator"),
+            (Type::ParsedName{eTypeClassAny, {"std", "string"}, "iterator"}));
+  EXPECT_EQ(Type::GetTypeScopeAndBasename("std::set<int>::iterator"),
+            (Type::ParsedName{eTypeClassAny, {"std", "set<int>"}, "iterator"}));
+  EXPECT_EQ(
+      Type::GetTypeScopeAndBasename("std::set<int, std::less<int>>::iterator"),
+      (Type::ParsedName{
+          eTypeClassAny, {"std", "set<int, std::less<int>>"}, "iterator"}));
+  EXPECT_EQ(Type::GetTypeScopeAndBasename(
+                "std::set<int, std::less<int>>::iterator<bool>"),
+            (Type::ParsedName{eTypeClassAny,
+                              {"std", "set<int, std::less<int>>"},
+                              "iterator<bool>"}));
+
+  EXPECT_EQ(Type::GetTypeScopeAndBasename("std::"), std::nullopt);
+  EXPECT_EQ(Type::GetTypeScopeAndBasename("foo<::bar"), std::nullopt);
 }
 
 TEST(Type, CompilerContextPattern) {
-- 
GitLab


From 2dbe89d15046bedcc36a5de1242e20aa91a5e598 Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Fri, 10 May 2024 08:50:44 +0200
Subject: [PATCH 0378/1206] [Clang] Implement
 __reference_converts_from_temporary (#91199)

This completes the required language support for P2255R2.
---
 clang/docs/LanguageExtensions.rst        |   5 +-
 clang/docs/ReleaseNotes.rst              |   3 +
 clang/include/clang/Basic/TokenKinds.def |   1 +
 clang/lib/Lex/PPMacroExpansion.cpp       |   2 -
 clang/lib/Parse/ParseDeclCXX.cpp         |   5 +-
 clang/lib/Parse/ParseExpr.cpp            |   1 -
 clang/lib/Sema/SemaExprCXX.cpp           | 157 +++++++++++++----------
 clang/test/SemaCXX/type-traits.cpp       |  80 ++++++++++++
 clang/www/cxx_status.html                |  10 +-
 9 files changed, 182 insertions(+), 82 deletions(-)

diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 3627a780886a..a09c409f8f91 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -1662,8 +1662,11 @@ The following type trait primitives are supported by Clang. Those traits marked
   ``T`` from ``U`` is ill-formed.
   Deprecated, use ``__reference_constructs_from_temporary``.
 * ``__reference_constructs_from_temporary(T, U)`` (C++)
-  Returns true if a reference ``T`` can be constructed from a temporary of type
+  Returns true if a reference ``T`` can be direct-initialized from a temporary of type
   a non-cv-qualified ``U``.
+* ``__reference_converts_from_temporary(T, U)`` (C++)
+    Returns true if a reference ``T`` can be copy-initialized from a temporary of type
+    a non-cv-qualified ``U``.
 * ``__underlying_type`` (C++, GNU, Microsoft)
 
 In addition, the following expression traits are supported:
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 4547636318a7..eef627ff2e31 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -182,6 +182,9 @@ C++23 Feature Support
 
 - Implemented `P2448R2: Relaxing some constexpr restrictions <https://wg21.link/P2448R2>`_.
 
+- Added a ``__reference_converts_from_temporary`` builtin, completing the necessary compiler support for
+  `P2255R2: Type trait to determine if a reference binds to a temporary <https://wg21.link/P2255R2>`_.
+
 C++2c Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index a27fbed358a6..56c4b17f769d 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -537,6 +537,7 @@ TYPE_TRAIT_1(__is_referenceable, IsReferenceable, KEYCXX)
 TYPE_TRAIT_1(__can_pass_in_regs, CanPassInRegs, KEYCXX)
 TYPE_TRAIT_2(__reference_binds_to_temporary, ReferenceBindsToTemporary, KEYCXX)
 TYPE_TRAIT_2(__reference_constructs_from_temporary, ReferenceConstructsFromTemporary, KEYCXX)
+TYPE_TRAIT_2(__reference_converts_from_temporary, ReferenceConvertsFromTemporary, KEYCXX)
 
 // Embarcadero Expression Traits
 EXPRESSION_TRAIT(__is_lvalue_expr, IsLValueExpr, KEYCXX)
diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
index a5f22f01682d..a478e0badb0c 100644
--- a/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -1714,8 +1714,6 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
           return llvm::StringSwitch<bool>(II->getName())
               .Case("__array_rank", true)
               .Case("__array_extent", true)
-              .Case("__reference_binds_to_temporary", true)
-              .Case("__reference_constructs_from_temporary", true)
 #define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) .Case("__" #Trait, true)
 #include "clang/Basic/TransformTypeTraits.def"
               .Default(false);
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 8e0e86824829..96c9708c3711 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -1779,9 +1779,8 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
           tok::kw___is_union,
           tok::kw___is_unsigned,
           tok::kw___is_void,
-          tok::kw___is_volatile,
-          tok::kw___reference_binds_to_temporary,
-          tok::kw___reference_constructs_from_temporary))
+          tok::kw___is_volatile
+      ))
     // GNU libstdc++ 4.2 and libc++ use certain intrinsic names as the
     // name of struct templates, but some are keywords in GCC >= 4.3
     // and Clang. Therefore, when we see the token sequence "struct
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 5f5f9a79c8c4..0551b8314f9f 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -1166,7 +1166,6 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
           REVERTIBLE_TYPE_TRAIT(__is_void);
           REVERTIBLE_TYPE_TRAIT(__is_volatile);
           REVERTIBLE_TYPE_TRAIT(__reference_binds_to_temporary);
-          REVERTIBLE_TYPE_TRAIT(__reference_constructs_from_temporary);
 #define TRANSFORM_TYPE_TRAIT_DEF(_, Trait)                                     \
   REVERTIBLE_TYPE_TRAIT(RTT_JOIN(__, Trait));
 #include "clang/Basic/TransformTypeTraits.def"
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index c1cb03e4ec7a..ae844bc69914 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -5627,6 +5627,77 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, TypeTrait UTT,
 static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, const TypeSourceInfo *Lhs,
                                     const TypeSourceInfo *Rhs, SourceLocation KeyLoc);
 
+static ExprResult CheckConvertibilityForTypeTraits(Sema &Self,
+                                                   const TypeSourceInfo *Lhs,
+                                                   const TypeSourceInfo *Rhs,
+                                                   SourceLocation KeyLoc) {
+
+  QualType LhsT = Lhs->getType();
+  QualType RhsT = Rhs->getType();
+
+  // C++0x [meta.rel]p4:
+  //   Given the following function prototype:
+  //
+  //     template <class T>
+  //       typename add_rvalue_reference<T>::type create();
+  //
+  //   the predicate condition for a template specialization
+  //   is_convertible<From, To> shall be satisfied if and only if
+  //   the return expression in the following code would be
+  //   well-formed, including any implicit conversions to the return
+  //   type of the function:
+  //
+  //     To test() {
+  //       return create<From>();
+  //     }
+  //
+  //   Access checking is performed as if in a context unrelated to To and
+  //   From. Only the validity of the immediate context of the expression
+  //   of the return-statement (including conversions to the return type)
+  //   is considered.
+  //
+  // We model the initialization as a copy-initialization of a temporary
+  // of the appropriate type, which for this expression is identical to the
+  // return statement (since NRVO doesn't apply).
+
+  // Functions aren't allowed to return function or array types.
+  if (RhsT->isFunctionType() || RhsT->isArrayType())
+    return ExprError();
+
+  // A function definition requires a complete, non-abstract return type.
+  if (!Self.isCompleteType(Rhs->getTypeLoc().getBeginLoc(), RhsT) ||
+      Self.isAbstractType(Rhs->getTypeLoc().getBeginLoc(), RhsT))
+    return ExprError();
+
+  // Compute the result of add_rvalue_reference.
+  if (LhsT->isObjectType() || LhsT->isFunctionType())
+    LhsT = Self.Context.getRValueReferenceType(LhsT);
+
+  // Build a fake source and destination for initialization.
+  InitializedEntity To(InitializedEntity::InitializeTemporary(RhsT));
+  OpaqueValueExpr From(KeyLoc, LhsT.getNonLValueExprType(Self.Context),
+                       Expr::getValueKindForType(LhsT));
+  Expr *FromPtr = &From;
+  InitializationKind Kind =
+      InitializationKind::CreateCopy(KeyLoc, SourceLocation());
+
+  // Perform the initialization in an unevaluated context within a SFINAE
+  // trap at translation unit scope.
+  EnterExpressionEvaluationContext Unevaluated(
+      Self, Sema::ExpressionEvaluationContext::Unevaluated);
+  Sema::SFINAETrap SFINAE(Self, /*AccessCheckingSFINAE=*/true);
+  Sema::ContextRAII TUContext(Self, Self.Context.getTranslationUnitDecl());
+  InitializationSequence Init(Self, To, Kind, FromPtr);
+  if (Init.Failed())
+    return ExprError();
+
+  ExprResult Result = Init.Perform(Self, To, Kind, FromPtr);
+  if (Result.isInvalid() || SFINAE.hasErrorOccurred())
+    return ExprError();
+
+  return Result;
+}
+
 static bool EvaluateBooleanTypeTrait(Sema &S, TypeTrait Kind,
                                      SourceLocation KWLoc,
                                      ArrayRef<TypeSourceInfo *> Args,
@@ -5640,13 +5711,16 @@ static bool EvaluateBooleanTypeTrait(Sema &S, TypeTrait Kind,
 
   // Evaluate ReferenceBindsToTemporary and ReferenceConstructsFromTemporary
   // alongside the IsConstructible traits to avoid duplication.
-  if (Kind <= BTT_Last && Kind != BTT_ReferenceBindsToTemporary && Kind != BTT_ReferenceConstructsFromTemporary)
+  if (Kind <= BTT_Last && Kind != BTT_ReferenceBindsToTemporary &&
+      Kind != BTT_ReferenceConstructsFromTemporary &&
+      Kind != BTT_ReferenceConvertsFromTemporary)
     return EvaluateBinaryTypeTrait(S, Kind, Args[0],
                                    Args[1], RParenLoc);
 
   switch (Kind) {
   case clang::BTT_ReferenceBindsToTemporary:
   case clang::BTT_ReferenceConstructsFromTemporary:
+  case clang::BTT_ReferenceConvertsFromTemporary:
   case clang::TT_IsConstructible:
   case clang::TT_IsNothrowConstructible:
   case clang::TT_IsTriviallyConstructible: {
@@ -5710,8 +5784,10 @@ static bool EvaluateBooleanTypeTrait(Sema &S, TypeTrait Kind,
     Sema::ContextRAII TUContext(S, S.Context.getTranslationUnitDecl());
     InitializedEntity To(
         InitializedEntity::InitializeTemporary(S.Context, Args[0]));
-    InitializationKind InitKind(InitializationKind::CreateDirect(KWLoc, KWLoc,
-                                                                 RParenLoc));
+    InitializationKind InitKind(
+        Kind == clang::BTT_ReferenceConvertsFromTemporary
+            ? InitializationKind::CreateCopy(KWLoc, KWLoc)
+            : InitializationKind::CreateDirect(KWLoc, KWLoc, RParenLoc));
     InitializationSequence Init(S, To, InitKind, ArgExprs);
     if (Init.Failed())
       return false;
@@ -5723,7 +5799,9 @@ static bool EvaluateBooleanTypeTrait(Sema &S, TypeTrait Kind,
     if (Kind == clang::TT_IsConstructible)
       return true;
 
-    if (Kind == clang::BTT_ReferenceBindsToTemporary || Kind == clang::BTT_ReferenceConstructsFromTemporary) {
+    if (Kind == clang::BTT_ReferenceBindsToTemporary ||
+        Kind == clang::BTT_ReferenceConstructsFromTemporary ||
+        Kind == clang::BTT_ReferenceConvertsFromTemporary) {
       if (!T->isReferenceType())
         return false;
 
@@ -5737,9 +5815,12 @@ static bool EvaluateBooleanTypeTrait(Sema &S, TypeTrait Kind,
       if (U->isReferenceType())
         return false;
 
-      TypeSourceInfo *TPtr = S.Context.CreateTypeSourceInfo(S.Context.getPointerType(S.BuiltinRemoveReference(T, UnaryTransformType::RemoveCVRef, {})));
-      TypeSourceInfo *UPtr = S.Context.CreateTypeSourceInfo(S.Context.getPointerType(S.BuiltinRemoveReference(U, UnaryTransformType::RemoveCVRef, {})));
-      return EvaluateBinaryTypeTrait(S, TypeTrait::BTT_IsConvertibleTo, UPtr, TPtr, RParenLoc);
+      TypeSourceInfo *TPtr = S.Context.CreateTypeSourceInfo(
+          S.Context.getPointerType(T.getNonReferenceType()));
+      TypeSourceInfo *UPtr = S.Context.CreateTypeSourceInfo(
+          S.Context.getPointerType(U.getNonReferenceType()));
+      return !CheckConvertibilityForTypeTraits(S, UPtr, TPtr, RParenLoc)
+                  .isInvalid();
     }
 
     if (Kind == clang::TT_IsNothrowConstructible)
@@ -5945,68 +6026,12 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, const TypeSourceI
   case BTT_IsConvertible:
   case BTT_IsConvertibleTo:
   case BTT_IsNothrowConvertible: {
-    // C++0x [meta.rel]p4:
-    //   Given the following function prototype:
-    //
-    //     template <class T>
-    //       typename add_rvalue_reference<T>::type create();
-    //
-    //   the predicate condition for a template specialization
-    //   is_convertible<From, To> shall be satisfied if and only if
-    //   the return expression in the following code would be
-    //   well-formed, including any implicit conversions to the return
-    //   type of the function:
-    //
-    //     To test() {
-    //       return create<From>();
-    //     }
-    //
-    //   Access checking is performed as if in a context unrelated to To and
-    //   From. Only the validity of the immediate context of the expression
-    //   of the return-statement (including conversions to the return type)
-    //   is considered.
-    //
-    // We model the initialization as a copy-initialization of a temporary
-    // of the appropriate type, which for this expression is identical to the
-    // return statement (since NRVO doesn't apply).
-
-    // Functions aren't allowed to return function or array types.
-    if (RhsT->isFunctionType() || RhsT->isArrayType())
-      return false;
-
-    // A return statement in a void function must have void type.
     if (RhsT->isVoidType())
       return LhsT->isVoidType();
 
-    // A function definition requires a complete, non-abstract return type.
-    if (!Self.isCompleteType(Rhs->getTypeLoc().getBeginLoc(), RhsT) ||
-        Self.isAbstractType(Rhs->getTypeLoc().getBeginLoc(), RhsT))
-      return false;
-
-    // Compute the result of add_rvalue_reference.
-    if (LhsT->isObjectType() || LhsT->isFunctionType())
-      LhsT = Self.Context.getRValueReferenceType(LhsT);
-
-    // Build a fake source and destination for initialization.
-    InitializedEntity To(InitializedEntity::InitializeTemporary(RhsT));
-    OpaqueValueExpr From(KeyLoc, LhsT.getNonLValueExprType(Self.Context),
-                         Expr::getValueKindForType(LhsT));
-    Expr *FromPtr = &From;
-    InitializationKind Kind(InitializationKind::CreateCopy(KeyLoc,
-                                                           SourceLocation()));
-
-    // Perform the initialization in an unevaluated context within a SFINAE
-    // trap at translation unit scope.
-    EnterExpressionEvaluationContext Unevaluated(
-        Self, Sema::ExpressionEvaluationContext::Unevaluated);
-    Sema::SFINAETrap SFINAE(Self, /*AccessCheckingSFINAE=*/true);
-    Sema::ContextRAII TUContext(Self, Self.Context.getTranslationUnitDecl());
-    InitializationSequence Init(Self, To, Kind, FromPtr);
-    if (Init.Failed())
-      return false;
-
-    ExprResult Result = Init.Perform(Self, To, Kind, FromPtr);
-    if (Result.isInvalid() || SFINAE.hasErrorOccurred())
+    ExprResult Result =
+        CheckConvertibilityForTypeTraits(Self, Lhs, Rhs, KeyLoc);
+    if (Result.isInvalid())
       return false;
 
     if (BTT != BTT_IsNothrowConvertible)
diff --git a/clang/test/SemaCXX/type-traits.cpp b/clang/test/SemaCXX/type-traits.cpp
index 01991887b284..f2fd45762abf 100644
--- a/clang/test/SemaCXX/type-traits.cpp
+++ b/clang/test/SemaCXX/type-traits.cpp
@@ -2908,6 +2908,12 @@ struct ConvertsToRef {
   operator RefType() const { return static_cast<RefType>(obj); }
   mutable T obj = 42;
 };
+template <class T, class RefType = T &>
+class ConvertsToRefPrivate {
+  operator RefType() const { return static_cast<RefType>(obj); }
+  mutable T obj = 42;
+};
+
 
 void reference_binds_to_temporary_checks() {
   static_assert(!(__reference_binds_to_temporary(int &, int &)));
@@ -2937,6 +2943,8 @@ void reference_binds_to_temporary_checks() {
 
   static_assert((__is_constructible(int const &, LongRef)));
   static_assert((__reference_binds_to_temporary(int const &, LongRef)));
+  static_assert(!__reference_binds_to_temporary(int const &, ConvertsToRefPrivate<long, long &>));
+
 
   // Test that it doesn't accept non-reference types as input.
   static_assert(!(__reference_binds_to_temporary(int, long)));
@@ -2944,6 +2952,17 @@ void reference_binds_to_temporary_checks() {
   static_assert((__reference_binds_to_temporary(const int &, long)));
 }
 
+
+struct ExplicitConversionRvalueRef {
+    operator int();
+    explicit operator int&&();
+};
+
+struct ExplicitConversionRef {
+    operator int();
+    explicit operator int&();
+};
+
 void reference_constructs_from_temporary_checks() {
   static_assert(!__reference_constructs_from_temporary(int &, int &));
   static_assert(!__reference_constructs_from_temporary(int &, int &&));
@@ -2973,6 +2992,8 @@ void reference_constructs_from_temporary_checks() {
 
   static_assert(__is_constructible(int const &, LongRef));
   static_assert(__reference_constructs_from_temporary(int const &, LongRef));
+  static_assert(!__reference_constructs_from_temporary(int const &, ConvertsToRefPrivate<long, long &>));
+
 
   // Test that it doesn't accept non-reference types as input.
   static_assert(!__reference_constructs_from_temporary(int, long));
@@ -2987,6 +3008,65 @@ void reference_constructs_from_temporary_checks() {
   static_assert(!__reference_constructs_from_temporary(const int&, int&&));
   static_assert(__reference_constructs_from_temporary(int&&, long&&));
   static_assert(__reference_constructs_from_temporary(int&&, long));
+
+
+  static_assert(!__reference_constructs_from_temporary(int&, ExplicitConversionRef));
+  static_assert(!__reference_constructs_from_temporary(const int&, ExplicitConversionRef));
+  static_assert(!__reference_constructs_from_temporary(int&&, ExplicitConversionRvalueRef));
+
+
+}
+
+void reference_converts_from_temporary_checks() {
+  static_assert(!__reference_converts_from_temporary(int &, int &));
+  static_assert(!__reference_converts_from_temporary(int &, int &&));
+
+  static_assert(!__reference_converts_from_temporary(int const &, int &));
+  static_assert(!__reference_converts_from_temporary(int const &, int const &));
+  static_assert(!__reference_converts_from_temporary(int const &, int &&));
+
+  static_assert(!__reference_converts_from_temporary(int &, long &)); // doesn't construct
+
+  static_assert(__reference_converts_from_temporary(int const &, long &));
+  static_assert(__reference_converts_from_temporary(int const &, long &&));
+  static_assert(__reference_converts_from_temporary(int &&, long &));
+
+  using LRef = ConvertsToRef<int, int &>;
+  using RRef = ConvertsToRef<int, int &&>;
+  using CLRef = ConvertsToRef<int, const int &>;
+  using LongRef = ConvertsToRef<long, long &>;
+  static_assert(__is_constructible(int &, LRef));
+  static_assert(!__reference_converts_from_temporary(int &, LRef));
+
+  static_assert(__is_constructible(int &&, RRef));
+  static_assert(!__reference_converts_from_temporary(int &&, RRef));
+
+  static_assert(__is_constructible(int const &, CLRef));
+  static_assert(!__reference_converts_from_temporary(int &&, CLRef));
+
+  static_assert(__is_constructible(int const &, LongRef));
+  static_assert(__reference_converts_from_temporary(int const &, LongRef));
+  static_assert(!__reference_converts_from_temporary(int const &, ConvertsToRefPrivate<long, long &>));
+
+
+  // Test that it doesn't accept non-reference types as input.
+  static_assert(!__reference_converts_from_temporary(int, long));
+
+  static_assert(__reference_converts_from_temporary(const int &, long));
+
+  // Additional checks
+  static_assert(__reference_converts_from_temporary(POD const&, Derives));
+  static_assert(__reference_converts_from_temporary(int&&, int));
+  static_assert(__reference_converts_from_temporary(const int&, int));
+  static_assert(!__reference_converts_from_temporary(int&&, int&&));
+  static_assert(!__reference_converts_from_temporary(const int&, int&&));
+  static_assert(__reference_converts_from_temporary(int&&, long&&));
+  static_assert(__reference_converts_from_temporary(int&&, long));
+
+  static_assert(!__reference_converts_from_temporary(int&, ExplicitConversionRef));
+  static_assert(__reference_converts_from_temporary(const int&, ExplicitConversionRef));
+  static_assert(__reference_converts_from_temporary(int&&, ExplicitConversionRvalueRef));
+
 }
 
 void array_rank() {
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index 6e0599cc9fe0..1338f544ffcb 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -347,15 +347,7 @@ C++23, informally referred to as C++26.</p>
     <tr>
       <td>Type trait to determine if a reference binds to a temporary</td>
       <td><a href="https://wg21.link/P2255R2">P2255R2</a></td>
-      <td class="partial" align="center">
-        <details><summary>Partial</summary>
-          Clang provides <tt>__reference_constructs_from_temporary</tt> type
-          trait builtin, with which <tt>std::reference_constructs_from_temporary</tt>
-          is implemented. <tt>__reference_converts_from_temporary</tt> needs to be
-          provided, following the normal cross-vendor convention to implement
-          traits requiring compiler support directly.
-        </details></td>
-      </td>
+      <td class="unreleased" align="center">Clang 19</td>
     </tr>
     <!-- July 2022 papers -->
     <tr>
-- 
GitLab


From 7e52ad3b5b9509d0873965e8492ab01141342822 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Fri, 10 May 2024 15:49:53 +0900
Subject: [PATCH 0379/1206] Fix a warning for #91455 [-Wc++20-extensions]

---
 .../Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/unittests/Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp b/clang/unittests/Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp
index 0a2514a2d7c1..d194742dbea7 100644
--- a/clang/unittests/Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/WatchedLiteralsSolverTest.cpp
@@ -20,7 +20,7 @@ SolverTest<WatchedLiteralsSolver>::createSolverWithLowTimeout() {
 namespace {
 
 INSTANTIATE_TYPED_TEST_SUITE_P(WatchedLiteralsSolverTest, SolverTest,
-                               WatchedLiteralsSolver);
+                               WatchedLiteralsSolver, );
 
 } // namespace
 } // namespace clang::dataflow::test
-- 
GitLab


From 2a61eebc66c0903cf3834a520b1f975ac3cdf92b Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Thu, 9 May 2024 19:34:43 +0900
Subject: [PATCH 0380/1206] Cleanup asserts in BranchParameters and
 DecisionParameters

---
 clang/lib/CodeGen/CoverageMappingGen.cpp                 | 5 +----
 llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h | 5 +----
 llvm/include/llvm/ProfileData/Coverage/MCDCTypes.h       | 8 ++++++--
 llvm/lib/ProfileData/Coverage/CoverageMapping.cpp        | 2 --
 llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp  | 1 -
 5 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp
index 733686d4946b..ce2f39aeb082 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -191,10 +191,7 @@ public:
   bool isBranch() const { return FalseCount.has_value(); }
 
   bool isMCDCDecision() const {
-    const auto *DecisionParams =
-        std::get_if<mcdc::DecisionParameters>(&MCDCParams);
-    assert(!DecisionParams || DecisionParams->NumConditions > 0);
-    return DecisionParams;
+    return std::holds_alternative<mcdc::DecisionParameters>(MCDCParams);
   }
 
   const auto &getMCDCDecisionParams() const {
diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
index 7a8b6639f297..da0310404524 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -462,10 +462,7 @@ public:
 
   CounterMappingRegion getDecisionRegion() const { return Region; }
   unsigned getNumConditions() const {
-    unsigned NumConditions = Region.getDecisionParams().NumConditions;
-    assert(NumConditions != 0 &&
-           "In MC/DC, NumConditions should never be zero!");
-    return NumConditions;
+    return Region.getDecisionParams().NumConditions;
   }
   unsigned getNumTestVectors() const { return TV.size(); }
   bool isCondFolded(unsigned Condition) const { return Folded[Condition]; }
diff --git a/llvm/include/llvm/ProfileData/Coverage/MCDCTypes.h b/llvm/include/llvm/ProfileData/Coverage/MCDCTypes.h
index 8c78bed4dec5..191e4ead95ea 100644
--- a/llvm/include/llvm/ProfileData/Coverage/MCDCTypes.h
+++ b/llvm/include/llvm/ProfileData/Coverage/MCDCTypes.h
@@ -33,7 +33,9 @@ struct DecisionParameters {
 
   DecisionParameters() = delete;
   DecisionParameters(unsigned BitmapIdx, unsigned NumConditions)
-      : BitmapIdx(BitmapIdx), NumConditions(NumConditions) {}
+      : BitmapIdx(BitmapIdx), NumConditions(NumConditions) {
+    assert(NumConditions > 0);
+  }
 };
 
 struct BranchParameters {
@@ -44,7 +46,9 @@ struct BranchParameters {
 
   BranchParameters() = delete;
   BranchParameters(ConditionID ID, const ConditionIDs &Conds)
-      : ID(ID), Conds(Conds) {}
+      : ID(ID), Conds(Conds) {
+    assert(ID >= 0);
+  }
 };
 
 /// The type of MC/DC-specific parameters.
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
index 6c77ce017c03..8c81bbe8e9c4 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -338,7 +338,6 @@ public:
 #endif
     for (const auto *Branch : Branches) {
       const auto &BranchParams = Branch->getBranchParams();
-      assert(BranchParams.ID >= 0 && "CondID isn't set");
       assert(SeenIDs.insert(BranchParams.ID).second && "Duplicate CondID");
       NextIDs[BranchParams.ID] = BranchParams.Conds;
     }
@@ -694,7 +693,6 @@ private:
       assert(Branch.Kind == CounterMappingRegion::MCDCBranchRegion);
 
       auto ConditionID = Branch.getBranchParams().ID;
-      assert(ConditionID >= 0 && "ConditionID should be positive");
 
       if (ConditionIDs.contains(ConditionID) ||
           ConditionID >= DecisionParams.NumConditions)
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
index 5036bde5aca7..adfd22804356 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
@@ -256,7 +256,6 @@ void CoverageMappingWriter::write(raw_ostream &OS) {
         // They are written as internal values plus 1.
         const auto &BranchParams = I->getBranchParams();
         ParamsShouldBeNull = false;
-        assert(BranchParams.ID >= 0);
         unsigned ID1 = BranchParams.ID + 1;
         unsigned TID1 = BranchParams.Conds[true] + 1;
         unsigned FID1 = BranchParams.Conds[false] + 1;
-- 
GitLab


From 6ce4c4ca2bb846c2b8d64dc2b3d3496785c9edff Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Thu, 9 May 2024 23:56:56 -0700
Subject: [PATCH 0381/1206] [clang-format][NFC] Drop a redundant
 clang::format::

---
 clang/lib/Format/UnwrappedLineParser.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 71557b127fb7..310b75485e08 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -3385,7 +3385,7 @@ void UnwrappedLineParser::parseAccessSpecifier() {
 /// \brief Parses a requires, decides if it is a clause or an expression.
 /// \pre The current token has to be the requires keyword.
 /// \returns true if it parsed a clause.
-bool clang::format::UnwrappedLineParser::parseRequires() {
+bool UnwrappedLineParser::parseRequires() {
   assert(FormatTok->is(tok::kw_requires) && "'requires' expected");
   auto RequiresToken = FormatTok;
 
-- 
GitLab


From 6aac30fa43f094ac25269bda163dc89a88cb8da7 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles@arm.com>
Date: Fri, 10 May 2024 08:09:02 +0100
Subject: [PATCH 0382/1206] Update FEAT_PAuth_LR behaviour for AArch64 (#90614)

Currently, LLVM enables `-mbranch-protection=standard` as `bti+pac-ret`.
To align LLVM with the behaviour in GNU, this has been updated to
`bti+pac-ret+pc` when FEAT_PAuth_LR is enabled as an optional feature
via the `-mcpu=` options. If this is not enabled, then this will revert
to the existing behaviour.
---
 clang/lib/Basic/Targets/AArch64.cpp           |  2 +-
 clang/lib/Driver/ToolChains/Clang.cpp         | 20 ++++++++++++++++++-
 .../Preprocessor/aarch64-target-features.c    |  4 ++++
 llvm/docs/ReleaseNotes.rst                    |  5 +++++
 .../llvm/TargetParser/AArch64TargetParser.h   |  2 ++
 .../llvm/TargetParser/ARMTargetParserCommon.h |  2 +-
 llvm/lib/TargetParser/AArch64TargetParser.cpp |  5 +++++
 .../TargetParser/ARMTargetParserCommon.cpp    |  3 ++-
 8 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 4b1545339f69..5db1ce78c657 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -225,7 +225,7 @@ bool AArch64TargetInfo::validateBranchProtection(StringRef Spec, StringRef,
                                                  BranchProtectionInfo &BPI,
                                                  StringRef &Err) const {
   llvm::ARM::ParsedBranchProtection PBP;
-  if (!llvm::ARM::parseBranchProtection(Spec, PBP, Err))
+  if (!llvm::ARM::parseBranchProtection(Spec, PBP, Err, HasPAuthLR))
     return false;
 
   BPI.SignReturnAddr =
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 449eb9b2a965..f81c2024ae48 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -55,6 +55,7 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/YAMLParser.h"
+#include "llvm/TargetParser/AArch64TargetParser.h"
 #include "llvm/TargetParser/ARMTargetParserCommon.h"
 #include "llvm/TargetParser/Host.h"
 #include "llvm/TargetParser/LoongArchTargetParser.h"
@@ -1511,7 +1512,24 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args,
   } else {
     StringRef DiagMsg;
     llvm::ARM::ParsedBranchProtection PBP;
-    if (!llvm::ARM::parseBranchProtection(A->getValue(), PBP, DiagMsg))
+    bool EnablePAuthLR = false;
+
+    // To know if we need to enable PAuth-LR As part of the standard branch
+    // protection option, it needs to be determined if the feature has been
+    // activated in the `march` argument. This information is stored within the
+    // CmdArgs variable and can be found using a search.
+    if (isAArch64) {
+      auto isPAuthLR = [](const char *member) {
+        llvm::AArch64::ExtensionInfo pauthlr_extension =
+            llvm::AArch64::getExtensionByID(llvm::AArch64::AEK_PAUTHLR);
+        return (pauthlr_extension.Feature.compare(member) == 0);
+      };
+
+      if (std::any_of(CmdArgs.begin(), CmdArgs.end(), isPAuthLR))
+        EnablePAuthLR = true;
+    }
+    if (!llvm::ARM::parseBranchProtection(A->getValue(), PBP, DiagMsg,
+                                          EnablePAuthLR))
       D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << DiagMsg;
     if (!isAArch64 && PBP.Key == "b_key")
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index 4d10eeafa884..82304a15a04a 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -616,6 +616,9 @@
 // ================== Check Armv9.5-A Pointer Authentication Enhancements(PAuth_LR).
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-LR-OFF %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv9.5-a -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-LR-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv9.5-a -mbranch-protection=standard -x c -E -dM %s -o - | FileCheck -check-prefixes=CHECK-PAUTH-LR-OFF,CHECK-BRANCH-PROTECTION-NO-PC %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv9.5-a+pauth-lr -mbranch-protection=standard -x c -E -dM %s -o - | FileCheck -check-prefixes=CHECK-PAUTH-LR,CHECK-BRANCH-PROTECTION-PC %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv9.5-a+nopauth-lr -mbranch-protection=standard -x c -E -dM %s -o - | FileCheck -check-prefixes=CHECK-PAUTH-LR-OFF,CHECK-BRANCH-PROTECTION-NO-PC %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a+pauth -mbranch-protection=none -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-LR-OFF %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a+pauth-lr -mbranch-protection=none -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-LR %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a+pauth-lr -mbranch-protection=bti -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-LR %s
@@ -636,6 +639,7 @@
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a+pauth-lr -mbranch-protection=pac-ret+pc+b-key -x c -E -dM %s -o - | FileCheck -check-prefixes=CHECK-PAUTH-LR,CHECK-BRANCH-PROTECTION-PC-BKEY %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a+pauth-lr -mbranch-protection=pac-ret+pc+leaf -x c -E -dM %s -o - | FileCheck -check-prefixes=CHECK-PAUTH-LR,CHECK-BRANCH-PROTECTION-PC-LEAF %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a+pauth-lr -mbranch-protection=pac-ret+pc+leaf+b-key -x c -E -dM %s -o - | FileCheck -check-prefixes=CHECK-PAUTH-LR,CHECK-BRANCH-PROTECTION-PC-LEAF-BKEY %s
+// CHECK-BRANCH-PROTECTION-NO-PC:        #define __ARM_FEATURE_PAC_DEFAULT 1
 // CHECK-BRANCH-PROTECTION-PC:           #define __ARM_FEATURE_PAC_DEFAULT 9
 // CHECK-BRANCH-PROTECTION-PC-BKEY:      #define __ARM_FEATURE_PAC_DEFAULT 10
 // CHECK-BRANCH-PROTECTION-PC-LEAF:      #define __ARM_FEATURE_PAC_DEFAULT 13
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 26f1d33f6800..f2577e1684f5 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -76,6 +76,11 @@ Changes to the AArch64 Backend
 * Added support for Cortex-A78AE, Cortex-A520AE, Cortex-A720AE,
   Cortex-R82AE, Neoverse-N3, Neoverse-V3 and Neoverse-V3AE CPUs.
 
+* ``-mbranch-protection=standard`` now enables FEAT_PAuth_LR by
+  default when the feature is enabled. The new behaviour results 
+  in ``standard`` being equal to ``bti+pac-ret+pc`` when ``+pauth-lr``
+  is passed as part of ``-mcpu=`` options.
+
 Changes to the AMDGPU Backend
 -----------------------------
 
diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index 1124420daf8d..20c3f95173c2 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -676,6 +676,8 @@ inline constexpr Alias CpuAliases[] = {{"cobalt-100", "neoverse-n2"},
 
 inline constexpr Alias ExtAliases[] = {{"rdma", "rdm"}};
 
+const ExtensionInfo &getExtensionByID(ArchExtKind(ExtID));
+
 bool getExtensionFeatures(
     const AArch64::ExtensionBitset &Extensions,
     std::vector<StringRef> &Features);
diff --git a/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h b/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h
index 8ae553ca80dd..f6115718e9f5 100644
--- a/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h
+++ b/llvm/include/llvm/TargetParser/ARMTargetParserCommon.h
@@ -46,7 +46,7 @@ struct ParsedBranchProtection {
 };
 
 bool parseBranchProtection(StringRef Spec, ParsedBranchProtection &PBP,
-                           StringRef &Err);
+                           StringRef &Err, bool EnablePAuthLR = false);
 
 } // namespace ARM
 } // namespace llvm
diff --git a/llvm/lib/TargetParser/AArch64TargetParser.cpp b/llvm/lib/TargetParser/AArch64TargetParser.cpp
index 71099462d5ec..026214e7e2ea 100644
--- a/llvm/lib/TargetParser/AArch64TargetParser.cpp
+++ b/llvm/lib/TargetParser/AArch64TargetParser.cpp
@@ -280,3 +280,8 @@ bool AArch64::ExtensionSet::parseModifier(StringRef Modifier) {
   }
   return false;
 }
+
+const AArch64::ExtensionInfo &
+AArch64::getExtensionByID(AArch64::ArchExtKind ExtID) {
+  return lookupExtensionByID(ExtID);
+}
diff --git a/llvm/lib/TargetParser/ARMTargetParserCommon.cpp b/llvm/lib/TargetParser/ARMTargetParserCommon.cpp
index 45d04f9bcbfb..d6ce6581bb1a 100644
--- a/llvm/lib/TargetParser/ARMTargetParserCommon.cpp
+++ b/llvm/lib/TargetParser/ARMTargetParserCommon.cpp
@@ -139,7 +139,7 @@ ARM::EndianKind ARM::parseArchEndian(StringRef Arch) {
 // returned in `PBP`. Returns false in error, with `Err` containing
 // an erroneous part of the spec.
 bool ARM::parseBranchProtection(StringRef Spec, ParsedBranchProtection &PBP,
-                                StringRef &Err) {
+                                StringRef &Err, bool EnablePAuthLR) {
   PBP = {"none", "a_key", false, false, false};
   if (Spec == "none")
     return true; // defaults are ok
@@ -148,6 +148,7 @@ bool ARM::parseBranchProtection(StringRef Spec, ParsedBranchProtection &PBP,
     PBP.Scope = "non-leaf";
     PBP.BranchTargetEnforcement = true;
     PBP.GuardedControlStack = true;
+    PBP.BranchProtectionPAuthLR = EnablePAuthLR;
     return true;
   }
 
-- 
GitLab


From 5d24217c2c1c06358168cae65d3ff8632b28cd7d Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Fri, 10 May 2024 00:38:52 -0700
Subject: [PATCH 0383/1206] [Clang] Pass -fseparate-named-sections from the
 driver (#91567)

This is a follow up to #91028.
---
 clang/lib/Driver/ToolChains/Clang.cpp        | 2 ++
 clang/test/Driver/fseparate-named-sections.c | 4 ++++
 2 files changed, 6 insertions(+)
 create mode 100644 clang/test/Driver/fseparate-named-sections.c

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index f81c2024ae48..42feb1650574 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -6146,6 +6146,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.addOptOutFlag(CmdArgs, options::OPT_funique_section_names,
                      options::OPT_fno_unique_section_names);
+  Args.addOptInFlag(CmdArgs, options::OPT_fseparate_named_sections,
+                    options::OPT_fno_separate_named_sections);
   Args.addOptInFlag(CmdArgs, options::OPT_funique_internal_linkage_names,
                     options::OPT_fno_unique_internal_linkage_names);
   Args.addOptInFlag(CmdArgs, options::OPT_funique_basic_block_section_names,
diff --git a/clang/test/Driver/fseparate-named-sections.c b/clang/test/Driver/fseparate-named-sections.c
new file mode 100644
index 000000000000..6264b8fcf0d8
--- /dev/null
+++ b/clang/test/Driver/fseparate-named-sections.c
@@ -0,0 +1,4 @@
+// RUN: %clang -### -fseparate-named-sections %s -c 2>&1 | FileCheck -check-prefix=CHECK-OPT %s
+// RUN: %clang -### -fseparate-named-sections -fno-separate-named-sections %s -c 2>&1 | FileCheck -check-prefix=CHECK-NOOPT %s
+// CHECK-OPT: "-fseparate-named-sections"
+// CHECK-NOOPT-NOT: "-fseparate-named-sections"
-- 
GitLab


From a76518cadc5eaa6b6d07334e2b5bc08382aabe49 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Fri, 10 May 2024 09:20:48 +0100
Subject: [PATCH 0384/1206] [lldb][ELF] Return address class map changes from
 symbol table parsing methods (#91585)

Instead of updating the member of the ObjectFileELF instance. This means
that if one object file asks another to parse the symbol table, that
first object's can update its address class map with the same changes
that the other object did.

(I'm not returning a reference to the other object's m_address_class_map
member because there may be other things in there not related to the
symbol table being parsed)

This will fix the code added in
https://github.com/llvm/llvm-project/pull/90622 which broke the test
`Expr/TestStringLiteralExpr.test` on 32 bit Arm Linux.

This happened because we had the program file, then asked for a better
object file, which returned the same program file again. This creates a
second ObjectFileELF for the same file, so when we tell the second
instance to parse the symbol table it actually calls into the first
instance, leaving the address class map of the second instance empty.

Which caused us to put an Arm breakpoint instuction at a Thumb return
address and broke the ability to call mmap.
---
 .../Plugins/ObjectFile/ELF/ObjectFileELF.cpp  | 77 ++++++++++++-------
 .../Plugins/ObjectFile/ELF/ObjectFileELF.h    | 21 ++---
 2 files changed, 60 insertions(+), 38 deletions(-)

diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
index 1646ee9aa34a..d88f2d083019 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
@@ -2060,13 +2060,17 @@ static char FindArmAarch64MappingSymbol(const char *symbol_name) {
 #define IS_MICROMIPS(ST_OTHER) (((ST_OTHER)&STO_MIPS_ISA) == STO_MICROMIPS)
 
 // private
-unsigned ObjectFileELF::ParseSymbols(Symtab *symtab, user_id_t start_id,
-                                     SectionList *section_list,
-                                     const size_t num_symbols,
-                                     const DataExtractor &symtab_data,
-                                     const DataExtractor &strtab_data) {
+std::pair<unsigned, ObjectFileELF::FileAddressToAddressClassMap>
+ObjectFileELF::ParseSymbols(Symtab *symtab, user_id_t start_id,
+                            SectionList *section_list, const size_t num_symbols,
+                            const DataExtractor &symtab_data,
+                            const DataExtractor &strtab_data) {
   ELFSymbol symbol;
   lldb::offset_t offset = 0;
+  // The changes these symbols would make to the class map. We will also update
+  // m_address_class_map but need to tell the caller what changed because the
+  // caller may be another object file.
+  FileAddressToAddressClassMap address_class_map;
 
   static ConstString text_section_name(".text");
   static ConstString init_section_name(".init");
@@ -2213,18 +2217,18 @@ unsigned ObjectFileELF::ParseSymbols(Symtab *symtab, user_id_t start_id,
             switch (mapping_symbol) {
             case 'a':
               // $a[.<any>]* - marks an ARM instruction sequence
-              m_address_class_map[symbol.st_value] = AddressClass::eCode;
+              address_class_map[symbol.st_value] = AddressClass::eCode;
               break;
             case 'b':
             case 't':
               // $b[.<any>]* - marks a THUMB BL instruction sequence
               // $t[.<any>]* - marks a THUMB instruction sequence
-              m_address_class_map[symbol.st_value] =
+              address_class_map[symbol.st_value] =
                   AddressClass::eCodeAlternateISA;
               break;
             case 'd':
               // $d[.<any>]* - marks a data item sequence (e.g. lit pool)
-              m_address_class_map[symbol.st_value] = AddressClass::eData;
+              address_class_map[symbol.st_value] = AddressClass::eData;
               break;
             }
           }
@@ -2238,11 +2242,11 @@ unsigned ObjectFileELF::ParseSymbols(Symtab *symtab, user_id_t start_id,
             switch (mapping_symbol) {
             case 'x':
               // $x[.<any>]* - marks an A64 instruction sequence
-              m_address_class_map[symbol.st_value] = AddressClass::eCode;
+              address_class_map[symbol.st_value] = AddressClass::eCode;
               break;
             case 'd':
               // $d[.<any>]* - marks a data item sequence (e.g. lit pool)
-              m_address_class_map[symbol.st_value] = AddressClass::eData;
+              address_class_map[symbol.st_value] = AddressClass::eData;
               break;
             }
           }
@@ -2260,11 +2264,11 @@ unsigned ObjectFileELF::ParseSymbols(Symtab *symtab, user_id_t start_id,
             // conjunction with symbol.st_value to produce the final
             // symbol_value that we store in the symtab.
             symbol_value_offset = -1;
-            m_address_class_map[symbol.st_value ^ 1] =
+            address_class_map[symbol.st_value ^ 1] =
                 AddressClass::eCodeAlternateISA;
           } else {
             // This address is ARM
-            m_address_class_map[symbol.st_value] = AddressClass::eCode;
+            address_class_map[symbol.st_value] = AddressClass::eCode;
           }
         }
       }
@@ -2285,17 +2289,17 @@ unsigned ObjectFileELF::ParseSymbols(Symtab *symtab, user_id_t start_id,
       */
       if (arch.IsMIPS()) {
         if (IS_MICROMIPS(symbol.st_other))
-          m_address_class_map[symbol.st_value] = AddressClass::eCodeAlternateISA;
+          address_class_map[symbol.st_value] = AddressClass::eCodeAlternateISA;
         else if ((symbol.st_value & 1) && (symbol_type == eSymbolTypeCode)) {
           symbol.st_value = symbol.st_value & (~1ull);
-          m_address_class_map[symbol.st_value] = AddressClass::eCodeAlternateISA;
+          address_class_map[symbol.st_value] = AddressClass::eCodeAlternateISA;
         } else {
           if (symbol_type == eSymbolTypeCode)
-            m_address_class_map[symbol.st_value] = AddressClass::eCode;
+            address_class_map[symbol.st_value] = AddressClass::eCode;
           else if (symbol_type == eSymbolTypeData)
-            m_address_class_map[symbol.st_value] = AddressClass::eData;
+            address_class_map[symbol.st_value] = AddressClass::eData;
           else
-            m_address_class_map[symbol.st_value] = AddressClass::eUnknown;
+            address_class_map[symbol.st_value] = AddressClass::eUnknown;
         }
       }
     }
@@ -2392,24 +2396,33 @@ unsigned ObjectFileELF::ParseSymbols(Symtab *symtab, user_id_t start_id,
       dc_symbol.SetIsWeak(true);
     symtab->AddSymbol(dc_symbol);
   }
-  return i;
+
+  m_address_class_map.merge(address_class_map);
+  return {i, address_class_map};
 }
 
-unsigned ObjectFileELF::ParseSymbolTable(Symtab *symbol_table,
-                                         user_id_t start_id,
-                                         lldb_private::Section *symtab) {
+std::pair<unsigned, ObjectFileELF::FileAddressToAddressClassMap>
+ObjectFileELF::ParseSymbolTable(Symtab *symbol_table, user_id_t start_id,
+                                lldb_private::Section *symtab) {
   if (symtab->GetObjectFile() != this) {
     // If the symbol table section is owned by a different object file, have it
     // do the parsing.
     ObjectFileELF *obj_file_elf =
         static_cast<ObjectFileELF *>(symtab->GetObjectFile());
-    return obj_file_elf->ParseSymbolTable(symbol_table, start_id, symtab);
+    auto [num_symbols, address_class_map] =
+        obj_file_elf->ParseSymbolTable(symbol_table, start_id, symtab);
+
+    // The other object file returned the changes it made to its address
+    // class map, make the same changes to ours.
+    m_address_class_map.merge(address_class_map);
+
+    return {num_symbols, address_class_map};
   }
 
   // Get section list for this object file.
   SectionList *section_list = m_sections_up.get();
   if (!section_list)
-    return 0;
+    return {};
 
   user_id_t symtab_id = symtab->GetID();
   const ELFSectionHeaderInfo *symtab_hdr = GetSectionHeaderByIndex(symtab_id);
@@ -2435,7 +2448,7 @@ unsigned ObjectFileELF::ParseSymbolTable(Symtab *symbol_table,
     }
   }
 
-  return 0;
+  return {0, {}};
 }
 
 size_t ObjectFileELF::ParseDynamicSymbols() {
@@ -2972,8 +2985,12 @@ void ObjectFileELF::ParseSymtab(Symtab &lldb_symtab) {
   // while the reverse is not necessarily true.
   Section *symtab =
       section_list->FindSectionByType(eSectionTypeELFSymbolTable, true).get();
-  if (symtab)
-    symbol_id += ParseSymbolTable(&lldb_symtab, symbol_id, symtab);
+  if (symtab) {
+    auto [num_symbols, address_class_map] =
+        ParseSymbolTable(&lldb_symtab, symbol_id, symtab);
+    m_address_class_map.merge(address_class_map);
+    symbol_id += num_symbols;
+  }
 
   // The symtab section is non-allocable and can be stripped, while the
   // .dynsym section which should always be always be there. To support the
@@ -2986,8 +3003,12 @@ void ObjectFileELF::ParseSymtab(Symtab &lldb_symtab) {
     Section *dynsym =
         section_list->FindSectionByType(eSectionTypeELFDynamicSymbols, true)
             .get();
-    if (dynsym)
-      symbol_id += ParseSymbolTable(&lldb_symtab, symbol_id, dynsym);
+    if (dynsym) {
+      auto [num_symbols, address_class_map] =
+          ParseSymbolTable(&lldb_symtab, symbol_id, dynsym);
+      symbol_id += num_symbols;
+      m_address_class_map.merge(address_class_map);
+    }
   }
 
   // DT_JMPREL
diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
index bc8e34981a9d..716bbe01638f 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
@@ -285,18 +285,19 @@ private:
 
   /// Populates the symbol table with all non-dynamic linker symbols.  This
   /// method will parse the symbols only once.  Returns the number of symbols
-  /// parsed.
-  unsigned ParseSymbolTable(lldb_private::Symtab *symbol_table,
-                            lldb::user_id_t start_id,
-                            lldb_private::Section *symtab);
+  /// parsed and a map of address types (used by targets like Arm that have
+  /// an alternative ISA mode like Thumb).
+  std::pair<unsigned, FileAddressToAddressClassMap>
+  ParseSymbolTable(lldb_private::Symtab *symbol_table, lldb::user_id_t start_id,
+                   lldb_private::Section *symtab);
 
   /// Helper routine for ParseSymbolTable().
-  unsigned ParseSymbols(lldb_private::Symtab *symbol_table,
-                        lldb::user_id_t start_id,
-                        lldb_private::SectionList *section_list,
-                        const size_t num_symbols,
-                        const lldb_private::DataExtractor &symtab_data,
-                        const lldb_private::DataExtractor &strtab_data);
+  std::pair<unsigned, FileAddressToAddressClassMap>
+  ParseSymbols(lldb_private::Symtab *symbol_table, lldb::user_id_t start_id,
+               lldb_private::SectionList *section_list,
+               const size_t num_symbols,
+               const lldb_private::DataExtractor &symtab_data,
+               const lldb_private::DataExtractor &strtab_data);
 
   /// Scans the relocation entries and adds a set of artificial symbols to the
   /// given symbol table for each PLT slot.  Returns the number of symbols
-- 
GitLab


From 23b673e5b4b73b42864fcd7d63c1e974317ed4d6 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Fri, 10 May 2024 09:27:02 +0100
Subject: [PATCH 0385/1206] [DAG][AArch64] Handle vscale addressing modes in
 reassociationCanBreakAddressingModePattern (#89908)

reassociationCanBreakAddressingModePattern tries to prevent bad add
reassociations that would break adrressing mode patterns. This adds
support for vscale offset addressing modes, making sure we don't break
patterns that already exist. It does not optimize _to_ the correct
addressing modes yet, but prevents us from optimizating _away_ from
them.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 42 ++++++++++++++-
 llvm/test/CodeGen/AArch64/sve-reassocadd.ll   | 54 +++++++------------
 2 files changed, 58 insertions(+), 38 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 4589d201d620..fddc97d8901a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -1083,7 +1083,44 @@ bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc,
   // (load/store (add, (add, x, y), offset2)) ->
   // (load/store (add, (add, x, offset2), y)).
 
-  if (Opc != ISD::ADD || N0.getOpcode() != ISD::ADD)
+  if (N0.getOpcode() != ISD::ADD)
+    return false;
+
+  // Check for vscale addressing modes.
+  // (load/store (add/sub (add x, y), vscale))
+  // (load/store (add/sub (add x, y), (lsl vscale, C)))
+  // (load/store (add/sub (add x, y), (mul vscale, C)))
+  if ((N1.getOpcode() == ISD::VSCALE ||
+       ((N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::MUL) &&
+        N1.getOperand(0).getOpcode() == ISD::VSCALE &&
+        isa<ConstantSDNode>(N1.getOperand(1)))) &&
+      N1.getValueType().getFixedSizeInBits() <= 64) {
+    int64_t ScalableOffset =
+        N1.getOpcode() == ISD::VSCALE
+            ? N1.getConstantOperandVal(0)
+            : (N1.getOperand(0).getConstantOperandVal(0) *
+               (N1.getOpcode() == ISD::SHL ? (1 << N1.getConstantOperandVal(1))
+                                           : N1.getConstantOperandVal(1)));
+    if (Opc == ISD::SUB)
+      ScalableOffset = -ScalableOffset;
+    if (all_of(N->uses(), [&](SDNode *Node) {
+          if (auto *LoadStore = dyn_cast<MemSDNode>(Node);
+              LoadStore && LoadStore->getBasePtr().getNode() == N) {
+            TargetLoweringBase::AddrMode AM;
+            AM.HasBaseReg = true;
+            AM.ScalableOffset = ScalableOffset;
+            EVT VT = LoadStore->getMemoryVT();
+            unsigned AS = LoadStore->getAddressSpace();
+            Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
+            return TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy,
+                                             AS);
+          }
+          return false;
+        }))
+      return true;
+  }
+
+  if (Opc != ISD::ADD)
     return false;
 
   auto *C2 = dyn_cast<ConstantSDNode>(N1);
@@ -3971,7 +4008,8 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
 
   // Hoist one-use addition by non-opaque constant:
   //   (x + C) - y  ->  (x - y) + C
-  if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() &&
+  if (!reassociationCanBreakAddressingModePattern(ISD::SUB, DL, N, N0, N1) &&
+      N0.getOpcode() == ISD::ADD && N0.hasOneUse() &&
       isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
     SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
     return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1));
diff --git a/llvm/test/CodeGen/AArch64/sve-reassocadd.ll b/llvm/test/CodeGen/AArch64/sve-reassocadd.ll
index c7261200a567..f54098b29a27 100644
--- a/llvm/test/CodeGen/AArch64/sve-reassocadd.ll
+++ b/llvm/test/CodeGen/AArch64/sve-reassocadd.ll
@@ -22,11 +22,9 @@ entry:
 define <vscale x 16 x i8> @i8_4s_1v(ptr %b) {
 ; CHECK-LABEL: i8_4s_1v:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rdvl x8, #1
 ; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    mov w9, #4 // =0x4
-; CHECK-NEXT:    add x8, x0, x8
-; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x8, x9]
+; CHECK-NEXT:    add x8, x0, #4
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x8, #1, mul vl]
 ; CHECK-NEXT:    ret
 entry:
   %add.ptr = getelementptr inbounds i8, ptr %b, i64 4
@@ -58,11 +56,9 @@ entry:
 define <vscale x 8 x i16> @i16_8s_1v(ptr %b) {
 ; CHECK-LABEL: i16_8s_1v:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rdvl x8, #1
 ; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    mov x9, #4 // =0x4
-; CHECK-NEXT:    add x8, x0, x8
-; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x8, x9, lsl #1]
+; CHECK-NEXT:    add x8, x0, #8
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x8, #1, mul vl]
 ; CHECK-NEXT:    ret
 entry:
   %add.ptr = getelementptr inbounds i8, ptr %b, i64 8
@@ -94,11 +90,9 @@ entry:
 define <vscale x 8 x i16> @i16_8s_2v(ptr %b) {
 ; CHECK-LABEL: i16_8s_2v:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rdvl x8, #2
 ; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    mov x9, #4 // =0x4
-; CHECK-NEXT:    add x8, x0, x8
-; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x8, x9, lsl #1]
+; CHECK-NEXT:    add x8, x0, #8
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x8, #2, mul vl]
 ; CHECK-NEXT:    ret
 entry:
   %add.ptr = getelementptr inbounds i8, ptr %b, i64 8
@@ -130,11 +124,9 @@ entry:
 define <vscale x 4 x i32> @i32_16s_2v(ptr %b) {
 ; CHECK-LABEL: i32_16s_2v:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rdvl x8, #1
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    mov x9, #4 // =0x4
-; CHECK-NEXT:    add x8, x0, x8
-; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x8, x9, lsl #2]
+; CHECK-NEXT:    add x8, x0, #16
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x8, #1, mul vl]
 ; CHECK-NEXT:    ret
 entry:
   %add.ptr = getelementptr inbounds i8, ptr %b, i64 16
@@ -166,11 +158,9 @@ entry:
 define <vscale x 2 x i64> @i64_32s_2v(ptr %b) {
 ; CHECK-LABEL: i64_32s_2v:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rdvl x8, #1
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov x9, #4 // =0x4
-; CHECK-NEXT:    add x8, x0, x8
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8, x9, lsl #3]
+; CHECK-NEXT:    add x8, x0, #32
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8, #1, mul vl]
 ; CHECK-NEXT:    ret
 entry:
   %add.ptr = getelementptr inbounds i8, ptr %b, i64 32
@@ -203,11 +193,9 @@ entry:
 define <vscale x 16 x i8> @i8_4s_m2v(ptr %b) {
 ; CHECK-LABEL: i8_4s_m2v:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cnth x8, all, mul #4
 ; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    mov w9, #4 // =0x4
-; CHECK-NEXT:    sub x8, x0, x8
-; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x8, x9]
+; CHECK-NEXT:    add x8, x0, #4
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x8, #-2, mul vl]
 ; CHECK-NEXT:    ret
 entry:
   %add.ptr = getelementptr inbounds i8, ptr %b, i64 4
@@ -239,11 +227,9 @@ entry:
 define <vscale x 8 x i16> @i16_8s_m2v(ptr %b) {
 ; CHECK-LABEL: i16_8s_m2v:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cnth x8, all, mul #4
 ; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    mov x9, #4 // =0x4
-; CHECK-NEXT:    sub x8, x0, x8
-; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x8, x9, lsl #1]
+; CHECK-NEXT:    add x8, x0, #8
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x8, #-2, mul vl]
 ; CHECK-NEXT:    ret
 entry:
   %add.ptr = getelementptr inbounds i8, ptr %b, i64 8
@@ -275,11 +261,9 @@ entry:
 define <vscale x 4 x i32> @i32_16s_m2v(ptr %b) {
 ; CHECK-LABEL: i32_16s_m2v:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cnth x8, all, mul #4
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    mov x9, #4 // =0x4
-; CHECK-NEXT:    sub x8, x0, x8
-; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x8, x9, lsl #2]
+; CHECK-NEXT:    add x8, x0, #16
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x8, #-2, mul vl]
 ; CHECK-NEXT:    ret
 entry:
   %add.ptr = getelementptr inbounds i8, ptr %b, i64 16
@@ -311,11 +295,9 @@ entry:
 define <vscale x 2 x i64> @i64_32s_m2v(ptr %b) {
 ; CHECK-LABEL: i64_32s_m2v:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cnth x8, all, mul #4
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov x9, #4 // =0x4
-; CHECK-NEXT:    sub x8, x0, x8
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8, x9, lsl #3]
+; CHECK-NEXT:    add x8, x0, #32
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8, #-2, mul vl]
 ; CHECK-NEXT:    ret
 entry:
   %add.ptr = getelementptr inbounds i8, ptr %b, i64 32
-- 
GitLab


From e6d29be566d19a6558597ed1ede4783e85485749 Mon Sep 17 00:00:00 2001
From: Tomas Matheson <tomas.matheson@arm.com>
Date: Fri, 10 May 2024 09:56:11 +0100
Subject: [PATCH 0386/1206] [clang] fix FMV test for Win x Arm builds (#91490)

---
 clang/test/Driver/aarch64-fmv.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/test/Driver/aarch64-fmv.c b/clang/test/Driver/aarch64-fmv.c
index 873a88964e9b..e7d01d1d5906 100644
--- a/clang/test/Driver/aarch64-fmv.c
+++ b/clang/test/Driver/aarch64-fmv.c
@@ -11,8 +11,8 @@
 // RUN: %clang --target=aarch64-linux-android23 --rtlib=compiler-rt  -### -c %s 2>&1 | FileCheck -check-prefix=FMV-ENABLED %s
 
 // FMV is disabled without compiler-rt:
-// RUN: %clang --target=aarch64 -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
-// RUN: %clang --target=aarch64-linux-gnu -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
+// RUN: %clang --rtlib=libgcc --target=aarch64 -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
+// RUN: %clang --rtlib=libgcc --target=aarch64-linux-gnu -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
 
 // Disabled for older android versions:
 // RUN: %clang --rtlib=compiler-rt --target=aarch64-linux-android               -### -c %s 2>&1 | FileCheck -check-prefix=FMV-DISABLED %s
-- 
GitLab


From 641949654910a533076e35517d084e2961adbdfa Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer@amd.com>
Date: Fri, 10 May 2024 10:12:24 +0100
Subject: [PATCH 0387/1206] [flang][OMPIRBuilder] Keep debug location in sync
 with insert point. (#89953)

A customer reported an issue which I have reduced to the test in the PR.
If built with debug info enabled, the build fails with the following
error in the verifier.

!dbg attachment points at wrong subprogram for function

The problem happened because some of the functions in OMPIRBuilder.cpp
updated the insertion point with the passed in location but did not
change the current debug location. This caused a stale debug location to
be attached to the instruction.

I have solved it by replacing restoreIP with updateToLocation which
updates both the insertion point and debug location. The
updateToLocation is used in many places already, so this PR brings
functions that I have changed in line with rest of the file.

Slight issue is that I am not checking the return type of
updateToLocation as there is no good value I could return in that case.
But if we have a condition where updateToLocation will return false,
these functions will fail in any case.

I have added a test that checks that build does not fail. I was not sure
what is the correct location for the test should be. Happy to move it to
more appropriate location.
---
 flang/test/Integration/debug-loc-1.f90    | 30 +++++++++++++++++++++++
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 12 ++++-----
 2 files changed, 36 insertions(+), 6 deletions(-)
 create mode 100644 flang/test/Integration/debug-loc-1.f90

diff --git a/flang/test/Integration/debug-loc-1.f90 b/flang/test/Integration/debug-loc-1.f90
new file mode 100644
index 000000000000..5fe2c8e31dd9
--- /dev/null
+++ b/flang/test/Integration/debug-loc-1.f90
@@ -0,0 +1,30 @@
+!RUN: %flang_fc1 -emit-llvm -debug-info-kind=line-tables-only -fopenmp %s -o - | FileCheck %s
+
+! Test that this file builds without an error.
+
+module debugloc
+contains
+subroutine test1
+implicit none
+ integer :: i
+ real, save :: var
+
+! CHECK: DILocation(line: [[@LINE+1]], {{.*}})
+!$omp parallel do
+do i=1,100
+  var = var + 0.1
+end do
+!$omp end parallel do
+
+end subroutine test1
+
+subroutine test2
+
+real, save :: tp
+!$omp threadprivate (tp)
+! CHECK: DILocation(line: [[@LINE+1]], {{.*}})
+  tp = tp + 1
+
+end subroutine test2
+
+end module debugloc
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 9f0c07ef0da9..42ea20919a5e 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -4401,7 +4401,7 @@ CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
                                           Value *Size, Value *Allocator,
                                           std::string Name) {
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  Builder.restoreIP(Loc.IP);
+  updateToLocation(Loc);
 
   uint32_t SrcLocStrSize;
   Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
@@ -4418,7 +4418,7 @@ CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
                                          Value *Addr, Value *Allocator,
                                          std::string Name) {
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  Builder.restoreIP(Loc.IP);
+  updateToLocation(Loc);
 
   uint32_t SrcLocStrSize;
   Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
@@ -4434,7 +4434,7 @@ CallInst *OpenMPIRBuilder::createOMPInteropInit(
     omp::OMPInteropType InteropType, Value *Device, Value *NumDependences,
     Value *DependenceAddress, bool HaveNowaitClause) {
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  Builder.restoreIP(Loc.IP);
+  updateToLocation(Loc);
 
   uint32_t SrcLocStrSize;
   Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
@@ -4462,7 +4462,7 @@ CallInst *OpenMPIRBuilder::createOMPInteropDestroy(
     const LocationDescription &Loc, Value *InteropVar, Value *Device,
     Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause) {
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  Builder.restoreIP(Loc.IP);
+  updateToLocation(Loc);
 
   uint32_t SrcLocStrSize;
   Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
@@ -4491,7 +4491,7 @@ CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,
                                                Value *DependenceAddress,
                                                bool HaveNowaitClause) {
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  Builder.restoreIP(Loc.IP);
+  updateToLocation(Loc);
   uint32_t SrcLocStrSize;
   Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
   Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
@@ -4517,7 +4517,7 @@ CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
     const LocationDescription &Loc, llvm::Value *Pointer,
     llvm::ConstantInt *Size, const llvm::Twine &Name) {
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  Builder.restoreIP(Loc.IP);
+  updateToLocation(Loc);
 
   uint32_t SrcLocStrSize;
   Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
-- 
GitLab


From fc57f88f007497a4ead0ec8607ac66e1847b02d6 Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Fri, 10 May 2024 11:15:26 +0200
Subject: [PATCH 0388/1206] [Clang] Fix Undefined Behavior introduced by #91199
 (#91718)

We stack allocated an OpaqueExpr that would be used after it was
destroyed.

e.g https://lab.llvm.org/buildbot/#/builders/57/builds/34909
---
 clang/lib/Sema/SemaExprCXX.cpp | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index ae844bc69914..c181092113e1 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -5627,10 +5627,9 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, TypeTrait UTT,
 static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, const TypeSourceInfo *Lhs,
                                     const TypeSourceInfo *Rhs, SourceLocation KeyLoc);
 
-static ExprResult CheckConvertibilityForTypeTraits(Sema &Self,
-                                                   const TypeSourceInfo *Lhs,
-                                                   const TypeSourceInfo *Rhs,
-                                                   SourceLocation KeyLoc) {
+static ExprResult CheckConvertibilityForTypeTraits(
+    Sema &Self, const TypeSourceInfo *Lhs, const TypeSourceInfo *Rhs,
+    SourceLocation KeyLoc, llvm::BumpPtrAllocator &OpaqueExprAllocator) {
 
   QualType LhsT = Lhs->getType();
   QualType RhsT = Rhs->getType();
@@ -5675,9 +5674,9 @@ static ExprResult CheckConvertibilityForTypeTraits(Sema &Self,
 
   // Build a fake source and destination for initialization.
   InitializedEntity To(InitializedEntity::InitializeTemporary(RhsT));
-  OpaqueValueExpr From(KeyLoc, LhsT.getNonLValueExprType(Self.Context),
-                       Expr::getValueKindForType(LhsT));
-  Expr *FromPtr = &From;
+  Expr *From = new (OpaqueExprAllocator.Allocate<OpaqueValueExpr>())
+      OpaqueValueExpr(KeyLoc, LhsT.getNonLValueExprType(Self.Context),
+                      Expr::getValueKindForType(LhsT));
   InitializationKind Kind =
       InitializationKind::CreateCopy(KeyLoc, SourceLocation());
 
@@ -5687,11 +5686,11 @@ static ExprResult CheckConvertibilityForTypeTraits(Sema &Self,
       Self, Sema::ExpressionEvaluationContext::Unevaluated);
   Sema::SFINAETrap SFINAE(Self, /*AccessCheckingSFINAE=*/true);
   Sema::ContextRAII TUContext(Self, Self.Context.getTranslationUnitDecl());
-  InitializationSequence Init(Self, To, Kind, FromPtr);
+  InitializationSequence Init(Self, To, Kind, From);
   if (Init.Failed())
     return ExprError();
 
-  ExprResult Result = Init.Perform(Self, To, Kind, FromPtr);
+  ExprResult Result = Init.Perform(Self, To, Kind, From);
   if (Result.isInvalid() || SFINAE.hasErrorOccurred())
     return ExprError();
 
@@ -5819,7 +5818,8 @@ static bool EvaluateBooleanTypeTrait(Sema &S, TypeTrait Kind,
           S.Context.getPointerType(T.getNonReferenceType()));
       TypeSourceInfo *UPtr = S.Context.CreateTypeSourceInfo(
           S.Context.getPointerType(U.getNonReferenceType()));
-      return !CheckConvertibilityForTypeTraits(S, UPtr, TPtr, RParenLoc)
+      return !CheckConvertibilityForTypeTraits(S, UPtr, TPtr, RParenLoc,
+                                               OpaqueExprAllocator)
                   .isInvalid();
     }
 
@@ -6028,9 +6028,9 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, const TypeSourceI
   case BTT_IsNothrowConvertible: {
     if (RhsT->isVoidType())
       return LhsT->isVoidType();
-
-    ExprResult Result =
-        CheckConvertibilityForTypeTraits(Self, Lhs, Rhs, KeyLoc);
+    llvm::BumpPtrAllocator OpaqueExprAllocator;
+    ExprResult Result = CheckConvertibilityForTypeTraits(Self, Lhs, Rhs, KeyLoc,
+                                                         OpaqueExprAllocator);
     if (Result.isInvalid())
       return false;
 
-- 
GitLab


From 1aca8ed5a7eeed264fdc2694deca8a4a4dba3689 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Fri, 10 May 2024 09:25:03 +0000
Subject: [PATCH 0389/1206] [lldb][ELF] Add a comment to explain address class
 map type

It was pointed out that ordering is crucial here, so note that.

I also looked into using a vector instead, as described in
https://llvm.org/docs/ProgrammersManual.html#dss-sortedvectorset.

Which this is in theory perfect for, but we have at least 2 places
that update the map and both would need to sort/unique each time.
Plus this code is pretty bug prone.

If there is future refactoring it's one thing to consider.
---
 lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
index 716bbe01638f..844e981b1d89 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
@@ -187,6 +187,9 @@ private:
   typedef DynamicSymbolColl::iterator DynamicSymbolCollIter;
   typedef DynamicSymbolColl::const_iterator DynamicSymbolCollConstIter;
 
+  /// An ordered map of file address to address class. Used on architectures
+  /// like Arm where there is an alternative ISA mode like Thumb. The container
+  /// is ordered so that it can be binary searched.
   typedef std::map<lldb::addr_t, lldb_private::AddressClass>
       FileAddressToAddressClassMap;
 
-- 
GitLab


From b277bf56d7654877a1c4b59dc08bc96b4d75b649 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Fri, 10 May 2024 10:53:57 +0100
Subject: [PATCH 0390/1206] [LLVM][CodeGen][SVE] Clean up lowering of
 VECTOR_SPLICE operations. (#91330)

Remove DAG combine that is performing type legalisation and instead
add isel patterns for all legal types.
---
 .../SelectionDAG/SelectionDAGBuilder.cpp      |   3 +-
 .../Target/AArch64/AArch64ISelLowering.cpp    |  38 ++-----
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  23 ++--
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |  17 +--
 .../AArch64/named-vector-shuffles-sve.ll      | 104 ++++++++++++++++++
 5 files changed, 136 insertions(+), 49 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index eac4297b89b5..b76036a22992 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -12240,9 +12240,8 @@ void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
 
   // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
   if (VT.isScalableVector()) {
-    MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
     setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
-                             DAG.getConstant(Imm, DL, IdxVT)));
+                             DAG.getVectorIdxConstant(Imm, DL)));
     return;
   }
 
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 7344387ffe55..29b66a72d21d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1048,9 +1048,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
 
   setTargetDAGCombine({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND,
-                       ISD::VECTOR_SPLICE, ISD::SIGN_EXTEND_INREG,
-                       ISD::CONCAT_VECTORS, ISD::EXTRACT_SUBVECTOR,
-                       ISD::INSERT_SUBVECTOR, ISD::STORE, ISD::BUILD_VECTOR});
+                       ISD::SIGN_EXTEND_INREG, ISD::CONCAT_VECTORS,
+                       ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR,
+                       ISD::STORE, ISD::BUILD_VECTOR});
   setTargetDAGCombine(ISD::TRUNCATE);
   setTargetDAGCombine(ISD::LOAD);
 
@@ -1580,6 +1580,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::MLOAD, VT, Custom);
       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
+      setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
 
       if (!Subtarget->isLittleEndian())
         setOperationAction(ISD::BITCAST, VT, Expand);
@@ -10102,10 +10103,9 @@ SDValue AArch64TargetLowering::LowerVECTOR_SPLICE(SDValue Op,
                        Op.getOperand(1));
   }
 
-  // This will select to an EXT instruction, which has a maximum immediate
-  // value of 255, hence 2048-bits is the maximum value we can lower.
-  if (IdxVal >= 0 &&
-      IdxVal < int64_t(2048 / Ty.getVectorElementType().getSizeInBits()))
+  // We can select to an EXT instruction when indexing the first 256 bytes.
+  unsigned BlockSize = AArch64::SVEBitsPerBlock / Ty.getVectorMinNumElements();
+  if (IdxVal >= 0 && (IdxVal * BlockSize / 8) < 256)
     return Op;
 
   return SDValue();
@@ -24255,28 +24255,6 @@ performInsertVectorEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
   return performPostLD1Combine(N, DCI, true);
 }
 
-static SDValue performSVESpliceCombine(SDNode *N, SelectionDAG &DAG) {
-  EVT Ty = N->getValueType(0);
-  if (Ty.isInteger())
-    return SDValue();
-
-  EVT IntTy = Ty.changeVectorElementTypeToInteger();
-  EVT ExtIntTy = getPackedSVEVectorVT(IntTy.getVectorElementCount());
-  if (ExtIntTy.getVectorElementType().getScalarSizeInBits() <
-      IntTy.getVectorElementType().getScalarSizeInBits())
-    return SDValue();
-
-  SDLoc DL(N);
-  SDValue LHS = DAG.getAnyExtOrTrunc(DAG.getBitcast(IntTy, N->getOperand(0)),
-                                     DL, ExtIntTy);
-  SDValue RHS = DAG.getAnyExtOrTrunc(DAG.getBitcast(IntTy, N->getOperand(1)),
-                                     DL, ExtIntTy);
-  SDValue Idx = N->getOperand(2);
-  SDValue Splice = DAG.getNode(ISD::VECTOR_SPLICE, DL, ExtIntTy, LHS, RHS, Idx);
-  SDValue Trunc = DAG.getAnyExtOrTrunc(Splice, DL, IntTy);
-  return DAG.getBitcast(Ty, Trunc);
-}
-
 static SDValue performFPExtendCombine(SDNode *N, SelectionDAG &DAG,
                                       TargetLowering::DAGCombinerInfo &DCI,
                                       const AArch64Subtarget *Subtarget) {
@@ -24661,8 +24639,6 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::MGATHER:
   case ISD::MSCATTER:
     return performMaskedGatherScatterCombine(N, DCI, DAG);
-  case ISD::VECTOR_SPLICE:
-    return performSVESpliceCombine(N, DAG);
   case ISD::FP_EXTEND:
     return performFPExtendCombine(N, DAG, DCI, Subtarget);
   case AArch64ISD::BRCOND:
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 62e68de1359f..64e545aa26b4 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -1994,14 +1994,21 @@ let Predicates = [HasSVEorSME] in {
             (LASTB_VPZ_D (PTRUE_D 31), ZPR:$Z1), dsub))>;
 
   // Splice with lane bigger or equal to 0
-  def : Pat<(nxv16i8 (vector_splice (nxv16i8 ZPR:$Z1), (nxv16i8 ZPR:$Z2), (i64 (sve_ext_imm_0_255 i32:$index)))),
-            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
-  def : Pat<(nxv8i16 (vector_splice (nxv8i16 ZPR:$Z1), (nxv8i16 ZPR:$Z2), (i64 (sve_ext_imm_0_127 i32:$index)))),
-            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
-  def : Pat<(nxv4i32 (vector_splice (nxv4i32 ZPR:$Z1), (nxv4i32 ZPR:$Z2), (i64 (sve_ext_imm_0_63 i32:$index)))),
-            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
-  def : Pat<(nxv2i64 (vector_splice (nxv2i64 ZPR:$Z1), (nxv2i64 ZPR:$Z2), (i64 (sve_ext_imm_0_31 i32:$index)))),
-            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
+  foreach VT = [nxv16i8] in
+    def : Pat<(VT (vector_splice (VT ZPR:$Z1), (VT ZPR:$Z2), (i64 (sve_ext_imm_0_255 i32:$index)))),
+              (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
+
+  foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
+    def : Pat<(VT (vector_splice (VT ZPR:$Z1), (VT ZPR:$Z2), (i64 (sve_ext_imm_0_127 i32:$index)))),
+              (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
+
+  foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
+    def : Pat<(VT (vector_splice (VT ZPR:$Z1), (VT ZPR:$Z2), (i64 (sve_ext_imm_0_63 i32:$index)))),
+              (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
+
+  foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
+    def : Pat<(VT (vector_splice (VT ZPR:$Z1), (VT ZPR:$Z2), (i64 (sve_ext_imm_0_31 i32:$index)))),
+              (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
 
   defm CMPHS_PPzZZ : sve_int_cmp_0<0b000, "cmphs", SETUGE, SETULE>;
   defm CMPHI_PPzZZ : sve_int_cmp_0<0b001, "cmphi", SETUGT, SETULT>;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 69c3238c7d61..fc7d3cdda4ac 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -7060,16 +7060,17 @@ multiclass sve_int_perm_splice<string asm, SDPatternOperator op> {
   def _S : sve_int_perm_splice<0b10, asm, ZPR32>;
   def _D : sve_int_perm_splice<0b11, asm, ZPR64>;
 
-  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
-  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
-  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
-  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
+ foreach VT = [nxv16i8] in
+   def : SVE_3_Op_Pat<VT, op, nxv16i1, VT, VT, !cast<Instruction>(NAME # _B)>;
 
-  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1,  nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
-  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1,  nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
-  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1,  nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
+ foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
+   def : SVE_3_Op_Pat<VT, op, nxv8i1, VT, VT, !cast<Instruction>(NAME # _H)>;
 
-  def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;
+ foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
+   def : SVE_3_Op_Pat<VT, op, nxv4i1, VT, VT, !cast<Instruction>(NAME # _S)>;
+
+ foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
+   def : SVE_3_Op_Pat<VT, op, nxv2i1, VT, VT, !cast<Instruction>(NAME # _D)>;
 }
 
 class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm,
diff --git a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
index f5763cd61033..d1171bc31247 100644
--- a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
+++ b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
@@ -692,6 +692,104 @@ define <vscale x 2 x double> @splice_nxv2f64_neg3(<vscale x 2 x double> %a, <vsc
   ret <vscale x 2 x double> %res
 }
 
+define <vscale x 2 x bfloat> @splice_nxv2bf16_neg_idx(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv2bf16_neg_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    rev p0.d, p0.d
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b, i32 -1)
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 2 x bfloat> @splice_nxv2bf16_neg2_idx(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv2bf16_neg2_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    rev p0.d, p0.d
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b, i32 -2)
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 2 x bfloat> @splice_nxv2bf16_first_idx(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv2bf16_first_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #8
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b, i32 1)
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 2 x bfloat> @splice_nxv2bf16_last_idx(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) vscale_range(16,16) #0 {
+; CHECK-LABEL: splice_nxv2bf16_last_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #248
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b, i32 31)
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 4 x bfloat> @splice_nxv4bf16_neg_idx(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv4bf16_neg_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s, vl1
+; CHECK-NEXT:    rev p0.s, p0.s
+; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b, i32 -1)
+  ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 4 x bfloat> @splice_nxv4bf16_neg3_idx(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv4bf16_neg3_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s, vl3
+; CHECK-NEXT:    rev p0.s, p0.s
+; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b, i32 -3)
+  ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 4 x bfloat> @splice_nxv4bf16_first_idx(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv4bf16_first_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #4
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b, i32 1)
+  ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 4 x bfloat> @splice_nxv4bf16_last_idx(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) vscale_range(16,16) #0 {
+; CHECK-LABEL: splice_nxv4bf16_last_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #252
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b, i32 63)
+  ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 8 x bfloat> @splice_nxv8bf16_first_idx(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv8bf16_first_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #2
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, i32 1)
+  ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 8 x bfloat> @splice_nxv8bf16_last_idx(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) vscale_range(16,16) #0 {
+; CHECK-LABEL: splice_nxv8bf16_last_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #254
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, i32 127)
+  ret <vscale x 8 x bfloat> %res
+}
+
 ; Ensure predicate based splice is promoted to use ZPRs.
 define <vscale x 2 x i1> @splice_nxv2i1(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv2i1:
@@ -834,12 +932,14 @@ declare <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1>, <vscale
 declare <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1>, <vscale x 4 x i1>, i32)
 declare <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1>, <vscale x 8 x i1>, i32)
 declare <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, i32)
+
 declare <vscale x 2 x i8> @llvm.vector.splice.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>, i32)
 declare <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, i32)
 declare <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, i32)
 declare <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, i32)
 declare <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32>, <vscale x 8 x i32>, i32)
 declare <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, i32)
+
 declare <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half>, <vscale x 2 x half>, i32)
 declare <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>, i32)
 declare <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, i32)
@@ -848,4 +948,8 @@ declare <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float>, <
 declare <vscale x 16 x float> @llvm.vector.splice.nxv16f32(<vscale x 16 x float>, <vscale x 16 x float>, i32)
 declare <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, i32)
 
+declare <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>, i32)
+declare <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>, i32)
+declare <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)
+
 attributes #0 = { nounwind "target-features"="+sve" }
-- 
GitLab


From 64d4ade3bb7d0d8cc87a777f4964e68f2f25edf9 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Fri, 10 May 2024 11:14:26 +0100
Subject: [PATCH 0391/1206] [AArch64] Add intrinsics for 16-bit non-widening
 FMLA/FMLS (#88553)

According to the specification in
https://github.com/ARM-software/acle/pull/309
add the following intrinsics

void svmla[_single]_za16[_f16]_vg1x2(uint32_t slice, svfloat16x2_t zn,
svfloat16_t zm)
void svmla[_single]_za16[_f16]_vg1x4(uint32_t slice, svfloat16x4_t zn,
svfloat16_t zm)
void svmls[_single]_za16[_f16]_vg1x2(uint32_t slice, svfloat16x2_t zn,
svfloat16_t zm)
void svmls[_single]_za16[_f16]_vg1x4(uint32_t slice, svfloat16x4_t zn,
svfloat16_t zm)

void svmla_za16[_f16]_vg1x2(uint32_t slice, svfloat16x2_t zn,
svfloat16x2_t zm)
void svmla_za16[_f16]_vg1x4(uint32_t slice, svfloat16x4_t zn,
svfloat16x4_t zm)
void svmls_za16[_f16]_vg1x2(uint32_t slice, svfloat16x2_t zn,
svfloat16x2_t zm)
void svmls_za16[_f16]_vg1x4(uint32_t slice, svfloat16x4_t zn,
svfloat16x4_t zm)

void svmla_lane_za16[_f16]_vg1x2(uint32_t slice, svfloat16x2_t zn,
svfloat16_t zm, uint64_t imm_idx)
void svmla_lane_za16[_f16]_vg1x4(uint32_t slice, svfloat16x4_t zn,
svfloat16_t zm, uint64_t imm_idx)
void svmls_lane_za16[_f16]_vg1x2(uint32_t slice, svfloat16x2_t zn,
svfloat16_t zm, uint64_t imm_idx)
void svmls_lane_za16[_f16]_vg1x4(uint32_t slice, svfloat16x4_t zn,
svfloat16_t zm, uint64_t imm_idx)

as well as the corresponding `_bf16` variants.
---
 clang/include/clang/Basic/arm_sme.td          |  34 +
 .../acle_sme2_fmlas16.c                       | 592 ++++++++++++++++++
 .../acle_sme2_fmlas16.c                       |  90 +++
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |  59 +-
 llvm/lib/Target/AArch64/SMEInstrFormats.td    |  54 +-
 .../AArch64/sme2-intrinsics-fmlas16.ll        | 462 ++++++++++++++
 6 files changed, 1254 insertions(+), 37 deletions(-)
 create mode 100644 clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c
 create mode 100644 clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c
 create mode 100644 llvm/test/CodeGen/AArch64/sme2-intrinsics-fmlas16.ll

diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index 1ac6d5170ea2..77ea53fb83fa 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -458,6 +458,40 @@ let TargetGuard = "sme2,sme-f64f64" in {
   def SVMLS_LANE_VG1x4_F64 : Inst<"svmls_lane_za64[_{d}]_vg1x4", "vm4di", "d", MergeNone, "aarch64_sme_fmls_lane_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_1>]>;
 }
 
+let TargetGuard = "sme-f16f16" in {
+  def SVMLA_MULTI_VG1x2_F16 : Inst<"svmla_za16[_f16]_vg1x2", "vm22", "h", MergeNone, "aarch64_sme_fmla_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLA_MULTI_VG1x4_F16 : Inst<"svmla_za16[_f16]_vg1x4", "vm44", "h", MergeNone, "aarch64_sme_fmla_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_MULTI_VG1x2_F16 : Inst<"svmls_za16[_f16]_vg1x2", "vm22", "h", MergeNone, "aarch64_sme_fmls_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_MULTI_VG1x4_F16 : Inst<"svmls_za16[_f16]_vg1x4", "vm44", "h", MergeNone, "aarch64_sme_fmls_vg1x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLA_SINGLE_VG1x2_F16 : Inst<"svmla[_single]_za16[_f16]_vg1x2", "vm2d", "h", MergeNone, "aarch64_sme_fmla_single_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLA_SINGLE_VG1x4_F16 : Inst<"svmla[_single]_za16[_f16]_vg1x4", "vm4d", "h", MergeNone, "aarch64_sme_fmla_single_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_SINGLE_VG1x2_F16 : Inst<"svmls[_single]_za16[_f16]_vg1x2", "vm2d", "h", MergeNone, "aarch64_sme_fmls_single_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_SINGLE_VG1x4_F16 : Inst<"svmls[_single]_za16[_f16]_vg1x4", "vm4d", "h", MergeNone, "aarch64_sme_fmls_single_vg1x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLA_LANE_VG1x2_F16 : Inst<"svmla_lane_za16[_f16]_vg1x2", "vm2di", "h", MergeNone, "aarch64_sme_fmla_lane_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLA_LANE_VG1x4_F16 : Inst<"svmla_lane_za16[_f16]_vg1x4", "vm4di", "h", MergeNone, "aarch64_sme_fmla_lane_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLS_LANE_VG1x2_F16 : Inst<"svmls_lane_za16[_f16]_vg1x2", "vm2di", "h", MergeNone, "aarch64_sme_fmls_lane_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLS_LANE_VG1x4_F16 : Inst<"svmls_lane_za16[_f16]_vg1x4", "vm4di", "h", MergeNone, "aarch64_sme_fmls_lane_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+}
+
+let TargetGuard = "sme2,b16b16" in {
+  def SVMLA_MULTI_VG1x2_BF16 : Inst<"svmla_za16[_bf16]_vg1x2", "vm22", "b", MergeNone, "aarch64_sme_fmla_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLA_MULTI_VG1x4_BF16 : Inst<"svmla_za16[_bf16]_vg1x4", "vm44", "b", MergeNone, "aarch64_sme_fmla_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_MULTI_VG1x2_BF16 : Inst<"svmls_za16[_bf16]_vg1x2", "vm22", "b", MergeNone, "aarch64_sme_fmls_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_MULTI_VG1x4_BF16 : Inst<"svmls_za16[_bf16]_vg1x4", "vm44", "b", MergeNone, "aarch64_sme_fmls_vg1x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLA_SINGLE_VG1x2_BF16 : Inst<"svmla[_single]_za16[_bf16]_vg1x2", "vm2d", "b", MergeNone, "aarch64_sme_fmla_single_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLA_SINGLE_VG1x4_BF16 : Inst<"svmla[_single]_za16[_bf16]_vg1x4", "vm4d", "b", MergeNone, "aarch64_sme_fmla_single_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_SINGLE_VG1x2_BF16 : Inst<"svmls[_single]_za16[_bf16]_vg1x2", "vm2d", "b", MergeNone, "aarch64_sme_fmls_single_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_SINGLE_VG1x4_BF16 : Inst<"svmls[_single]_za16[_bf16]_vg1x4", "vm4d", "b", MergeNone, "aarch64_sme_fmls_single_vg1x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLA_LANE_VG1x2_BF16 : Inst<"svmla_lane_za16[_bf16]_vg1x2", "vm2di", "b", MergeNone, "aarch64_sme_fmla_lane_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLA_LANE_VG1x4_BF16 : Inst<"svmla_lane_za16[_bf16]_vg1x4", "vm4di", "b", MergeNone, "aarch64_sme_fmla_lane_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLS_LANE_VG1x2_BF16 : Inst<"svmls_lane_za16[_bf16]_vg1x2", "vm2di", "b", MergeNone, "aarch64_sme_fmls_lane_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLS_LANE_VG1x4_BF16 : Inst<"svmls_lane_za16[_bf16]_vg1x4", "vm4di", "b", MergeNone, "aarch64_sme_fmls_lane_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+}
+
 // FMLAL/FMLSL/UMLAL/SMLAL
 // SMLALL/UMLALL/USMLALL/SUMLALL
 let TargetGuard = "sme2" in {
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c
new file mode 100644
index 000000000000..ecc415545414
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c
@@ -0,0 +1,592 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
+// RUN: %clang_cc1                               -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1                        -x c++ -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s --check-prefix CHECK-CXX
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS        -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -x c++ -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s --check-prefix CHECK-CXX
+
+// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2p1 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -S -Werror -Wall %s -o /dev/null
+
+// REQUIRES: aarch64-registered-target
+#include <arm_sme.h>
+
+#ifdef SME_OVERLOADED_FORMS
+#define SME_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED, A5) A1##A3##A5
+#else
+#define SME_ACLE_FUNC(A1, A2, A3, A4, A5) A1##A2##A3##A4##A5
+#endif
+
+// CHECK-LABEL: define dso_local void @test_svmla_single_za16_f16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z32test_svmla_single_za16_f16_vg1x2j13svfloat16x2_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_single_za16_f16_vg1x2(uint32_t slice, svfloat16x2_t zn, svfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla,_single,_za16,_f16,_vg1x2)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_single_za16_f16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z32test_svmla_single_za16_f16_vg1x4j13svfloat16x4_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_single_za16_f16_vg1x4(uint32_t slice, svfloat16x4_t zn, svfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla,_single,_za16,_f16,_vg1x4)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_single_za16_f16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z32test_svmls_single_za16_f16_vg1x2j13svfloat16x2_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_single_za16_f16_vg1x2(uint32_t slice, svfloat16x2_t zn, svfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls,_single,_za16,_f16,_vg1x2)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_single_za16_f16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z32test_svmls_single_za16_f16_vg1x4j13svfloat16x4_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_single_za16_f16_vg1x4(uint32_t slice, svfloat16x4_t zn, svfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls,_single,_za16,_f16,_vg1x4)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_za16_f16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 16 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svmla_za16_f16_vg1x2j13svfloat16x2_tS_(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 16 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_za16_f16_vg1x2(uint32_t slice, svfloat16x2_t zn, svfloat16x2_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla,,_za16,_f16,_vg1x2)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_za16_f16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 32 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svmla_za16_f16_vg1x4j13svfloat16x4_tS_(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 32 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_za16_f16_vg1x4(uint32_t slice, svfloat16x4_t zn, svfloat16x4_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla,,_za16,_f16,_vg1x4)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_za16_f16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 16 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svmls_za16_f16_vg1x2j13svfloat16x2_tS_(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 16 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZM]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_za16_f16_vg1x2(uint32_t slice, svfloat16x2_t zn, svfloat16x2_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls,,_za16,_f16,_vg1x2)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_za16_f16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 32 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svmls_za16_f16_vg1x4j13svfloat16x4_tS_(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 32 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZM]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_za16_f16_vg1x4(uint32_t slice, svfloat16x4_t zn, svfloat16x4_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls,,_za16,_f16,_vg1x4)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_lane_za16_f16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM]], i32 7)
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z30test_svmla_lane_za16_f16_vg1x2j13svfloat16x2_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM]], i32 7)
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_lane_za16_f16_vg1x2(uint32_t slice, svfloat16x2_t zn, svfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla_lane,,_za16,_f16,_vg1x2)(slice, zn, zm, 7);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_lane_za16_f16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM]], i32 7)
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z30test_svmla_lane_za16_f16_vg1x4j13svfloat16x4_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM]], i32 7)
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_lane_za16_f16_vg1x4(uint32_t slice, svfloat16x4_t zn, svfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla_lane,,_za16,_f16,_vg1x4)(slice, zn, zm, 7);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_lane_za16_f16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM]], i32 7)
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z30test_svmls_lane_za16_f16_vg1x2j13svfloat16x2_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[ZM]], i32 7)
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_lane_za16_f16_vg1x2(uint32_t slice, svfloat16x2_t zn, svfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls_lane,,_za16,_f16,_vg1x2)(slice, zn, zm, 7);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_lane_za16_f16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM]], i32 7)
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z30test_svmls_lane_za16_f16_vg1x4j13svfloat16x4_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[ZM]], i32 7)
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_lane_za16_f16_vg1x4(uint32_t slice, svfloat16x4_t zn, svfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls_lane,,_za16,_f16,_vg1x4)(slice, zn, zm, 7);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_single_za16_bf16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z33test_svmla_single_za16_bf16_vg1x2j14svbfloat16x2_tu14__SVBfloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_single_za16_bf16_vg1x2(uint32_t slice, svbfloat16x2_t zn, svbfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla, _single, _za16, _bf16, _vg1x2)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_single_za16_bf16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z33test_svmla_single_za16_bf16_vg1x4j14svbfloat16x4_tu14__SVBfloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_single_za16_bf16_vg1x4(uint32_t slice, svbfloat16x4_t zn, svbfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla, _single, _za16, _bf16, _vg1x4)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_single_za16_bf16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z33test_svmls_single_za16_bf16_vg1x2j14svbfloat16x2_tu14__SVBfloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_single_za16_bf16_vg1x2(uint32_t slice, svbfloat16x2_t zn, svbfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls, _single, _za16, _bf16, _vg1x2)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_single_za16_bf16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z33test_svmls_single_za16_bf16_vg1x4j14svbfloat16x4_tu14__SVBfloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_single_za16_bf16_vg1x4(uint32_t slice, svbfloat16x4_t zn, svbfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls, _single, _za16, _bf16, _vg1x4)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_za16_bf16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 16 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svmla_za16_bf16_vg1x2j14svbfloat16x2_tS_(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 16 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_za16_bf16_vg1x2(uint32_t slice, svbfloat16x2_t zn, svbfloat16x2_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla, , _za16, _bf16, _vg1x2)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_za16_bf16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 32 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svmla_za16_bf16_vg1x4j14svbfloat16x4_tS_(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 32 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_za16_bf16_vg1x4(uint32_t slice, svbfloat16x4_t zn, svbfloat16x4_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla, , _za16, _bf16, _vg1x4)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_za16_bf16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 16 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svmls_za16_bf16_vg1x2j14svbfloat16x2_tS_(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 16 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_za16_bf16_vg1x2(uint32_t slice, svbfloat16x2_t zn, svbfloat16x2_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls, , _za16, _bf16, _vg1x2)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_za16_bf16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 32 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svmls_za16_bf16_vg1x4j14svbfloat16x4_tS_(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 32 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_za16_bf16_vg1x4(uint32_t slice, svbfloat16x4_t zn, svbfloat16x4_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls, , _za16, _bf16, _vg1x4)(slice, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_lane_za16_bf16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM]], i32 7)
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z31test_svmla_lane_za16_bf16_vg1x2j14svbfloat16x2_tu14__SVBfloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM]], i32 7)
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_lane_za16_bf16_vg1x2(uint32_t slice, svbfloat16x2_t zn, svbfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla_lane, , _za16, _bf16, _vg1x2)(slice, zn, zm, 7);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmla_lane_za16_bf16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM]], i32 7)
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z31test_svmla_lane_za16_bf16_vg1x4j14svbfloat16x4_tu14__SVBfloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM]], i32 7)
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmla_lane_za16_bf16_vg1x4(uint32_t slice, svbfloat16x4_t zn, svbfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmla_lane, , _za16, _bf16, _vg1x4)(slice, zn, zm, 7);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_lane_za16_bf16_vg1x2(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM]], i32 7)
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z31test_svmls_lane_za16_bf16_vg1x2j14svbfloat16x2_tu14__SVBfloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM]], i32 7)
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_lane_za16_bf16_vg1x2(uint32_t slice, svbfloat16x2_t zn, svbfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls_lane, , _za16, _bf16, _vg1x2)(slice, zn, zm, 7);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmls_lane_za16_bf16_vg1x4(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM]], i32 7)
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z31test_svmls_lane_za16_bf16_vg1x4j14svbfloat16x4_tu14__SVBfloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM]], i32 7)
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmls_lane_za16_bf16_vg1x4(uint32_t slice, svbfloat16x4_t zn, svbfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmls_lane, , _za16, _bf16, _vg1x4)(slice, zn, zm, 7);
+}
diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c
new file mode 100644
index 000000000000..b1582569971d
--- /dev/null
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c
@@ -0,0 +1,90 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -verify -emit-llvm %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+
+void test_features_f16f16(uint32_t slice,
+                          svfloat16_t zm,
+                          svfloat16x2_t zn2, svfloat16x2_t zm2,
+                          svfloat16x4_t zn4, svfloat16x4_t zm4,
+                          svbfloat16_t bzm,
+                          svbfloat16x2_t bzn2, svbfloat16x2_t bzm2,
+                          svbfloat16x4_t bzn4, svbfloat16x4_t bzm4)
+
+   __arm_streaming __arm_inout("za") {
+  // expected-error@+1 {{'svmla_single_za16_f16_vg1x2' needs target feature sme-f16f16}}
+  svmla_single_za16_f16_vg1x2(slice, zn2, zm);
+  // expected-error@+1 {{'svmla_single_za16_f16_vg1x4' needs target feature sme-f16f16}}
+  svmla_single_za16_f16_vg1x4(slice, zn4, zm);
+  // expected-error@+1 {{'svmls_single_za16_f16_vg1x2' needs target feature sme-f16f16}}
+  svmls_single_za16_f16_vg1x2(slice, zn2, zm);
+  // expected-error@+1 {{'svmls_single_za16_f16_vg1x4' needs target feature sme-f16f16}}
+  svmls_single_za16_f16_vg1x4(slice, zn4, zm);
+  // expected-error@+1 {{'svmla_za16_f16_vg1x2' needs target feature sme-f16f16}}
+  svmla_za16_f16_vg1x2(slice, zn2, zm2);
+  // expected-error@+1 {{'svmla_za16_f16_vg1x4' needs target feature sme-f16f16}}
+  svmla_za16_f16_vg1x4(slice, zn4, zm4);
+  // expected-error@+1 {{'svmls_za16_f16_vg1x2' needs target feature sme-f16f16}}
+  svmls_za16_f16_vg1x2(slice, zn2, zm2);
+  // expected-error@+1 {{'svmls_za16_f16_vg1x4' needs target feature sme-f16f16}}
+  svmls_za16_f16_vg1x4(slice, zn4, zm4);
+  // expected-error@+1 {{'svmla_lane_za16_f16_vg1x2' needs target feature sme-f16f16}}
+  svmla_lane_za16_f16_vg1x2(slice, zn2, zm, 7);
+  // expected-error@+1 {{'svmla_lane_za16_f16_vg1x4' needs target feature sme-f16f16}}
+  svmla_lane_za16_f16_vg1x4(slice, zn4, zm, 7);
+  // expected-error@+1 {{'svmls_lane_za16_f16_vg1x2' needs target feature sme-f16f16}}
+  svmls_lane_za16_f16_vg1x2(slice, zn2, zm, 7);
+  // expected-error@+1 {{'svmls_lane_za16_f16_vg1x4' needs target feature sme-f16f16}}
+  svmls_lane_za16_f16_vg1x4(slice, zn4, zm, 7);
+
+  // expected-error@+1 {{'svmla_single_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
+  svmla_single_za16_bf16_vg1x2(slice, bzn2, bzm);
+  // expected-error@+1 {{'svmla_single_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
+  svmla_single_za16_bf16_vg1x4(slice, bzn4, bzm);
+  // expected-error@+1 {{'svmls_single_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
+  svmls_single_za16_bf16_vg1x2(slice, bzn2, bzm);
+  // expected-error@+1 {{'svmls_single_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
+  svmls_single_za16_bf16_vg1x4(slice, bzn4, bzm);
+  // expected-error@+1 {{'svmla_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
+  svmla_za16_bf16_vg1x2(slice, bzn2, bzm2);
+  // expected-error@+1 {{'svmla_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
+  svmla_za16_bf16_vg1x4(slice, bzn4, bzm4);
+  // expected-error@+1 {{'svmls_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
+  svmls_za16_bf16_vg1x2(slice, bzn2, bzm2);
+  // expected-error@+1 {{'svmls_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
+  svmls_za16_bf16_vg1x4(slice, bzn4, bzm4);
+  // expected-error@+1 {{'svmla_lane_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
+  svmla_lane_za16_bf16_vg1x2(slice, bzn2, bzm, 7);
+  // expected-error@+1 {{'svmla_lane_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
+  svmla_lane_za16_bf16_vg1x4(slice, bzn4, bzm, 7);
+  // expected-error@+1 {{'svmls_lane_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
+  svmls_lane_za16_bf16_vg1x2(slice, bzn2, bzm, 7);
+  // expected-error@+1 {{'svmls_lane_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
+  svmls_lane_za16_bf16_vg1x4(slice, bzn4, bzm, 7);
+}
+
+
+void test_imm(uint32_t slice, svfloat16_t zm, svfloat16x2_t zn2,svfloat16x4_t zn4,
+              svbfloat16_t bzm, svbfloat16x2_t bzn2, svbfloat16x4_t bzn4)
+  __arm_streaming __arm_inout("za") {
+
+  // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svmla_lane_za16_f16_vg1x2(slice, zn2, zm, -1);
+  // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svmla_lane_za16_f16_vg1x4(slice, zn4, zm, -1);
+  // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svmls_lane_za16_f16_vg1x2(slice, zn2, zm, -1);
+  // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svmls_lane_za16_f16_vg1x4(slice, zn4, zm, -1);
+
+  // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svmla_lane_za16_bf16_vg1x2(slice, bzn2, bzm, -1);
+  // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svmla_lane_za16_bf16_vg1x4(slice, bzn4, bzm, -1);
+  // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svmls_lane_za16_bf16_vg1x2(slice, bzn2, bzm, -1);
+  // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svmls_lane_za16_bf16_vg1x4(slice, bzn4, bzm, -1);
+}
\ No newline at end of file
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index 574178c8d524..a2e8c530c1df 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -797,22 +797,20 @@ defm FADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fadd", 0b0100, MatrixOp16
 defm FADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fadd", 0b0100, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
 defm FSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fsub", 0b0101, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
 defm FSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fsub", 0b0101, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
-}
 
-let Predicates = [HasSMEF16F16] in {
-defm FMLA_VG2_M2ZZI_H : sme2p1_multi_vec_array_vg2_index_16b<"fmla", 0b00, 0b100, ZZ_h_mul_r, ZPR4b16>;
-defm FMLA_VG4_M4ZZI_H : sme2p1_multi_vec_array_vg4_index_16b<"fmla", 0b000, ZZZZ_h_mul_r, ZPR4b16>;
-defm FMLA_VG2_M2ZZ_H :  sme2_dot_mla_add_sub_array_vg24_single<"fmla", 0b0011100, MatrixOp16, ZZ_h, ZPR4b16>;
-defm FMLA_VG4_M4ZZ_H :  sme2_dot_mla_add_sub_array_vg24_single<"fmla", 0b0111100, MatrixOp16, ZZZZ_h, ZPR4b16>;
-defm FMLA_VG2_M2Z4Z_H : sme2_dot_mla_add_sub_array_vg2_multi<"fmla", 0b0100001, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
-defm FMLA_VG4_M4Z4Z_H : sme2_dot_mla_add_sub_array_vg4_multi<"fmla", 0b0100001, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
-
-defm FMLS_VG2_M2ZZI_H : sme2p1_multi_vec_array_vg2_index_16b<"fmls", 0b00, 0b101, ZZ_h_mul_r, ZPR4b16>;
-defm FMLS_VG4_M4ZZI_H : sme2p1_multi_vec_array_vg4_index_16b<"fmls", 0b001, ZZZZ_h_mul_r, ZPR4b16>;
-defm FMLS_VG2_M2ZZ_H :  sme2_dot_mla_add_sub_array_vg24_single<"fmls", 0b0011101, MatrixOp16, ZZ_h, ZPR4b16>;
-defm FMLS_VG4_M4ZZ_H :  sme2_dot_mla_add_sub_array_vg24_single<"fmls", 0b0111101, MatrixOp16, ZZZZ_h, ZPR4b16>;
-defm FMLS_VG2_M2Z2Z_H : sme2_dot_mla_add_sub_array_vg2_multi<"fmls", 0b0100011, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
-defm FMLS_VG4_M4Z2Z_H : sme2_dot_mla_add_sub_array_vg4_multi<"fmls", 0b0100011, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
+defm FMLA_VG2_M2ZZI_H : sme2p1_multi_vec_array_vg2_index_16b<"fmla", 0b00, 0b100, ZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fmla_lane_vg1x2>;
+defm FMLA_VG4_M4ZZI_H : sme2p1_multi_vec_array_vg4_index_16b<"fmla", 0b000, ZZZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fmla_lane_vg1x4>;
+defm FMLA_VG2_M2ZZ_H :  sme2_dot_mla_add_sub_array_vg2_single<"fmla", 0b0011100, MatrixOp16, ZZ_h, ZPR4b16, nxv8f16, int_aarch64_sme_fmla_single_vg1x2>;
+defm FMLA_VG4_M4ZZ_H :  sme2_dot_mla_add_sub_array_vg4_single<"fmla", 0b0111100, MatrixOp16, ZZZZ_h, ZPR4b16, nxv8f16, int_aarch64_sme_fmla_single_vg1x4>;
+defm FMLA_VG2_M2Z4Z_H : sme2_dot_mla_add_sub_array_vg2_multi<"fmla", 0b0100001, MatrixOp16, ZZ_h_mul_r, nxv8f16, int_aarch64_sme_fmla_vg1x2>;
+defm FMLA_VG4_M4Z4Z_H : sme2_dot_mla_add_sub_array_vg4_multi<"fmla", 0b0100001, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, int_aarch64_sme_fmla_vg1x4>;
+
+defm FMLS_VG2_M2ZZI_H : sme2p1_multi_vec_array_vg2_index_16b<"fmls", 0b00, 0b101, ZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fmls_lane_vg1x2>;
+defm FMLS_VG4_M4ZZI_H : sme2p1_multi_vec_array_vg4_index_16b<"fmls", 0b001, ZZZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fmls_lane_vg1x4>;
+defm FMLS_VG2_M2ZZ_H :  sme2_dot_mla_add_sub_array_vg2_single<"fmls", 0b0011101, MatrixOp16, ZZ_h, ZPR4b16, nxv8f16, int_aarch64_sme_fmls_single_vg1x2>;
+defm FMLS_VG4_M4ZZ_H :  sme2_dot_mla_add_sub_array_vg4_single<"fmls", 0b0111101, MatrixOp16, ZZZZ_h, ZPR4b16, nxv8f16, int_aarch64_sme_fmls_single_vg1x4>;
+defm FMLS_VG2_M2Z2Z_H : sme2_dot_mla_add_sub_array_vg2_multi<"fmls", 0b0100011, MatrixOp16, ZZ_h_mul_r, nxv8f16, int_aarch64_sme_fmls_vg1x2>;
+defm FMLS_VG4_M4Z2Z_H : sme2_dot_mla_add_sub_array_vg4_multi<"fmls", 0b0100011, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, int_aarch64_sme_fmls_vg1x4>;
 
 defm FCVT_2ZZ_H  : sme2p1_fp_cvt_vector_vg2_single<"fcvt", 0b0>;
 defm FCVTL_2ZZ_H : sme2p1_fp_cvt_vector_vg2_single<"fcvtl", 0b1>;
@@ -827,20 +825,19 @@ defm BFADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfadd", 0b1100, MatrixOp
 defm BFSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfsub", 0b1101, MatrixOp16, ZZ_h_mul_r,  nxv8bf16, null_frag>;
 defm BFSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfsub", 0b1101, MatrixOp16, ZZZZ_h_mul_r,  nxv8bf16, null_frag>;
 
-defm BFMLA_VG2_M2ZZI : sme2p1_multi_vec_array_vg2_index_16b<"bfmla", 0b00, 0b110, ZZ_h_mul_r, ZPR4b16>;
-defm BFMLA_VG4_M4ZZI : sme2p1_multi_vec_array_vg4_index_16b<"bfmla", 0b010, ZZZZ_h_mul_r, ZPR4b16>;
-defm BFMLA_VG2_M2ZZ : sme2_dot_mla_add_sub_array_vg24_single<"bfmla", 0b1011100, MatrixOp16, ZZ_h, ZPR4b16>;
-defm BFMLA_VG4_M4ZZ : sme2_dot_mla_add_sub_array_vg24_single<"bfmla", 0b1111100, MatrixOp16, ZZZZ_h, ZPR4b16>;
-defm BFMLA_VG2_M2Z2Z : sme2_dot_mla_add_sub_array_vg2_multi<"bfmla", 0b1100001, MatrixOp16, ZZ_h_mul_r, nxv8bf16, null_frag>;
-defm BFMLA_VG4_M4Z4Z : sme2_dot_mla_add_sub_array_vg4_multi<"bfmla", 0b1100001, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16, null_frag>;
-
-defm BFMLS_VG2_M2ZZI : sme2p1_multi_vec_array_vg2_index_16b<"bfmls", 0b00, 0b111, ZZ_h_mul_r, ZPR4b16>;
-defm BFMLS_VG4_M4ZZI : sme2p1_multi_vec_array_vg4_index_16b<"bfmls", 0b011, ZZZZ_h_mul_r, ZPR4b16>;
-defm BFMLS_VG2_M2ZZ : sme2_dot_mla_add_sub_array_vg24_single<"bfmls", 0b1011101, MatrixOp16, ZZ_h, ZPR4b16>;
-defm BFMLS_VG4_M4ZZ : sme2_dot_mla_add_sub_array_vg24_single<"bfmls", 0b1111101, MatrixOp16, ZZZZ_h, ZPR4b16>;
-defm BFMLS_VG2_M2Z2Z : sme2_dot_mla_add_sub_array_vg2_multi<"bfmls", 0b1100011, MatrixOp16, ZZ_h_mul_r, nxv8bf16, null_frag>;
-defm BFMLS_VG4_M4Z4Z : sme2_dot_mla_add_sub_array_vg4_multi<"bfmls", 0b1100011, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16, null_frag>;
+defm BFMLA_VG2_M2ZZI : sme2p1_multi_vec_array_vg2_index_16b<"bfmla", 0b00, 0b110, ZZ_h_mul_r, ZPR4b16, nxv8bf16, int_aarch64_sme_fmla_lane_vg1x2>;
+defm BFMLA_VG4_M4ZZI : sme2p1_multi_vec_array_vg4_index_16b<"bfmla", 0b010, ZZZZ_h_mul_r, ZPR4b16, nxv8bf16, int_aarch64_sme_fmla_lane_vg1x4>;
+defm BFMLA_VG2_M2ZZ  : sme2_dot_mla_add_sub_array_vg2_single<"bfmla", 0b1011100, MatrixOp16, ZZ_h, ZPR4b16, nxv8bf16, int_aarch64_sme_fmla_single_vg1x2>;
+defm BFMLA_VG4_M4ZZ  : sme2_dot_mla_add_sub_array_vg4_single<"bfmla", 0b1111100, MatrixOp16, ZZZZ_h, ZPR4b16, nxv8bf16, int_aarch64_sme_fmla_single_vg1x4>;
+defm BFMLA_VG2_M2Z2Z : sme2_dot_mla_add_sub_array_vg2_multi<"bfmla", 0b1100001, MatrixOp16, ZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fmla_vg1x2>;
+defm BFMLA_VG4_M4Z4Z : sme2_dot_mla_add_sub_array_vg4_multi<"bfmla", 0b1100001, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fmla_vg1x4>;
 
+defm BFMLS_VG2_M2ZZI : sme2p1_multi_vec_array_vg2_index_16b<"bfmls", 0b00, 0b111, ZZ_h_mul_r, ZPR4b16, nxv8bf16, int_aarch64_sme_fmls_lane_vg1x2>;
+defm BFMLS_VG4_M4ZZI : sme2p1_multi_vec_array_vg4_index_16b<"bfmls", 0b011, ZZZZ_h_mul_r, ZPR4b16, nxv8bf16, int_aarch64_sme_fmls_lane_vg1x4>;
+defm BFMLS_VG2_M2ZZ  : sme2_dot_mla_add_sub_array_vg2_single<"bfmls", 0b1011101, MatrixOp16, ZZ_h, ZPR4b16, nxv8bf16, int_aarch64_sme_fmls_single_vg1x2>;
+defm BFMLS_VG4_M4ZZ  : sme2_dot_mla_add_sub_array_vg4_single<"bfmls", 0b1111101, MatrixOp16, ZZZZ_h, ZPR4b16, nxv8bf16, int_aarch64_sme_fmls_single_vg1x4>;
+defm BFMLS_VG2_M2Z2Z : sme2_dot_mla_add_sub_array_vg2_multi<"bfmls", 0b1100011, MatrixOp16, ZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fmls_vg1x2>;
+defm BFMLS_VG4_M4Z4Z : sme2_dot_mla_add_sub_array_vg4_multi<"bfmls", 0b1100011, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fmls_vg1x4>;
 
 defm BFMAX_VG2_2ZZ  : sme2p1_bf_max_min_vector_vg2_single<"bfmax", 0b0010000>;
 defm BFMAX_VG4_4ZZ  : sme2p1_bf_max_min_vector_vg4_single<"bfmax", 0b0010000>;
@@ -909,9 +906,9 @@ def LUTI4_S_4ZZT2Z  : sme2_luti4_vector_vg4_strided<0b00, 0b00, "luti4">;
 } //[HasSME2p1, HasSME_LUTv2]
 
 let Predicates = [HasSMEF8F16] in {
-defm FVDOT_VG2_M2ZZI_BtoH : sme2p1_multi_vec_array_vg2_index_16b<"fvdot", 0b11, 0b110, ZZ_b_mul_r, ZPR4b8>;
-defm FDOT_VG2_M2ZZI_BtoH  : sme2p1_multi_vec_array_vg2_index_16b<"fdot",    0b11, 0b010, ZZ_b_mul_r, ZPR4b8>;
-defm FDOT_VG4_M4ZZI_BtoH  : sme2p1_multi_vec_array_vg4_index_16b<"fdot",    0b100, ZZZZ_b_mul_r, ZPR4b8>;
+defm FVDOT_VG2_M2ZZI_BtoH : sme2p1_multi_vec_array_vg2_index_f8f16<"fvdot", 0b11, 0b110, ZZ_b_mul_r, ZPR4b8>;
+defm FDOT_VG2_M2ZZI_BtoH  : sme2p1_multi_vec_array_vg2_index_f8f16<"fdot",  0b11, 0b010, ZZ_b_mul_r, ZPR4b8>;
+defm FDOT_VG4_M4ZZI_BtoH  : sme2p1_multi_vec_array_vg4_index_f8f16<"fdot",    0b100, ZZZZ_b_mul_r, ZPR4b8>;
 defm FDOT_VG2_M2ZZ_BtoH   :  sme2_dot_mla_add_sub_array_vg24_single<"fdot", 0b0010001, MatrixOp16, ZZ_b, ZPR4b8>;
 defm FDOT_VG4_M4ZZ_BtoH   :  sme2_dot_mla_add_sub_array_vg24_single<"fdot", 0b0110001, MatrixOp16, ZZZZ_b, ZPR4b8>;
 // TODO: Replace nxv16i8 by nxv16f8
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 3363aab4b093..724dd07225cd 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -2448,9 +2448,29 @@ multiclass sme2_multi_vec_array_vg2_index_32b<string mnemonic, bits<2> sz, bits<
 }
 
 // SME2.1 multi-vec ternary indexed two registers 16-bit
-// SME2 multi-vec indexed FP8 two-way dot product to FP16 two registers
 multiclass sme2p1_multi_vec_array_vg2_index_16b<string mnemonic, bits<2> sz, bits<3> op,
-                                                RegisterOperand multi_vector_ty, ZPRRegOp zpr_ty> {
+                                                RegisterOperand multi_vector_ty, ZPRRegOp vector_ty,
+                                                ValueType vt, SDPatternOperator intrinsic> {
+  def NAME : sme2_multi_vec_array_vg2_index<sz, {op{2},?,?,op{1-0},?}, MatrixOp16,
+                                            multi_vector_ty, vector_ty,
+                                            VectorIndexH, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+    bits<3> i;
+    let Inst{11-10} = i{2-1};
+    let Inst{3}     = i{0};
+  }
+
+  def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexH32b, SMEMatrixArray>;
+
+  def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexH32b_timm, tileslice16>;
+
+  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",
+        (!cast<Instruction>(NAME) MatrixOp16:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,
+        multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexH:$i), 0>;
+}
+
+// SME2 multi-vec indexed FP8 two-way dot product to FP16 two registers
+multiclass sme2p1_multi_vec_array_vg2_index_f8f16<string mnemonic, bits<2> sz, bits<3> op,
+                                                  RegisterOperand multi_vector_ty, ZPRRegOp zpr_ty> {
   def NAME : sme2_multi_vec_array_vg2_index<sz, {op{2},?,?,op{1-0},?}, MatrixOp16,
                                             multi_vector_ty, zpr_ty,
                                             VectorIndexH, mnemonic> {
@@ -2569,10 +2589,10 @@ multiclass sme2_multi_vec_array_vg4_index_32b<string mnemonic, bits<4> op,
         multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>;
 }
 
-// SME2.1 multi-vec ternary indexed four registers 16-bit
-multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<3> op,
-                                                RegisterOperand multi_vector_ty,
-                                                ZPRRegOp zpr_ty> {
+// SME2.1 multi-vec ternary indexed four registers 16-bit (FP8)
+multiclass sme2p1_multi_vec_array_vg4_index_f8f16<string mnemonic, bits<3> op,
+                                                  RegisterOperand multi_vector_ty,
+                                                  ZPRRegOp zpr_ty> {
   def NAME : sme2_multi_vec_array_vg4_index<0b0,{0b1,?,?,op,?}, MatrixOp16,
                                             multi_vector_ty, zpr_ty,
                                             VectorIndexH, mnemonic>{
@@ -2586,6 +2606,28 @@ multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<3> op,
         sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm, VectorIndexH:$i), 0>;
 }
 
+// SME2.1 multi-vec ternary indexed four registers 16-bit
+multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<3> op,
+                                                RegisterOperand multi_vector_ty,
+                                                ZPRRegOp vector_ty, ValueType vt,
+                                                SDPatternOperator intrinsic> {
+  def NAME : sme2_multi_vec_array_vg4_index<0b0,{0b1,?,?,op,?}, MatrixOp16,
+                                            multi_vector_ty, vector_ty,
+                                            VectorIndexH, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+    bits<3> i;
+    let Inst{11-10} = i{2-1};
+    let Inst{3}     = i{0};
+  }
+
+  def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexH32b_timm, SMEMatrixArray>;
+
+  def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexH32b_timm, tileslice16>;
+
+  def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",
+        (!cast<Instruction>(NAME) MatrixOp16:$ZAda,  MatrixIndexGPR32Op8_11:$Rv,
+        sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexH:$i), 0>;
+}
+
 // SME2 multi-vec ternary indexed four registers 64-bit
 class sme2_multi_vec_array_vg4_index_64b<bits<3> op,
                                          RegisterOperand multi_vector_ty,
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-fmlas16.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fmlas16.ll
new file mode 100644
index 000000000000..3e807b7e6338
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fmlas16.ll
@@ -0,0 +1,462 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "// kill:.*$" --version 4
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+
+target triple = "aarch64-linux"
+
+define void @test_fmla_f16_vg2_single(i32 %slice, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1, <vscale x 8 x half> %b) #0 {
+; CHECK-LABEL: test_fmla_f16_vg2_single:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmla za.h[w8, 0, vgx2], { z0.h, z1.h }, z2.h
+; CHECK:    fmla za.h[w8, 7, vgx2], { z0.h, z1.h }, z2.h
+; CHECK:    ret
+  call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv8f16(i32 %slice, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1, <vscale x 8 x half> %b)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv8f16(i32 %slice.7, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1, <vscale x 8 x half> %b)
+  ret void
+}
+
+define void @test_fmla_f16_vg4_single(i32 %slice, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+; CHECK-LABEL: test_fmla_f16_vg4_single:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmla za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h
+; CHECK:    fmla za.h[w8, 7, vgx4], { z0.h - z3.h }, z4.h
+; CHECK:    ret
+                                      <vscale x 8 x half> %a2, <vscale x 8 x half> %a3, <vscale x 8 x half> %b) #0 {
+  call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv8f16(i32 %slice, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                        <vscale x 8 x half> %a2, <vscale x 8 x half> %a3, <vscale x 8 x half> %b)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv8f16(i32 %slice.7, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                        <vscale x 8 x half> %a2, <vscale x 8 x half> %a3, <vscale x 8 x half> %b)
+  ret void
+}
+
+define void @test_fmls_f16_vg2_single(i32 %slice, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1, <vscale x 8 x half> %b) #0 {
+; CHECK-LABEL: test_fmls_f16_vg2_single:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmls za.h[w8, 0, vgx2], { z0.h, z1.h }, z2.h
+; CHECK:    fmls za.h[w8, 7, vgx2], { z0.h, z1.h }, z2.h
+; CHECK:    ret
+  call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv8f16(i32 %slice, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1, <vscale x 8 x half> %b)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv8f16(i32 %slice.7, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1, <vscale x 8 x half> %b)
+  ret void
+}
+
+define void @test_fmls_f16_vg4_single(i32 %slice, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+; CHECK-LABEL: test_fmls_f16_vg4_single:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmls za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h
+; CHECK:    fmls za.h[w8, 7, vgx4], { z0.h - z3.h }, z4.h
+; CHECK:    ret
+                                      <vscale x 8 x half> %a2, <vscale x 8 x half> %a3, <vscale x 8 x half> %b) #0 {
+  call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv8f16(i32 %slice, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                        <vscale x 8 x half> %a2, <vscale x 8 x half> %a3, <vscale x 8 x half> %b)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv8f16(i32 %slice.7, <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                        <vscale x 8 x half> %a2, <vscale x 8 x half> %a3, <vscale x 8 x half> %b)
+  ret void
+}
+
+define void @test_fmla_f16_vg2_multi(i32 %slice,
+; CHECK-LABEL: test_fmla_f16_vg2_multi:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmla za.h[w8, 0, vgx2], { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK:    fmla za.h[w8, 7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK:    ret
+                                     <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                     <vscale x 8 x half> %b0, <vscale x 8 x half> %b1) #0 {
+  call void @llvm.aarch64.sme.fmla.vg1x2.nxv8f16(i32 %slice,
+                                                <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                <vscale x 8 x half> %b0, <vscale x 8 x half> %b1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.vg1x2.nxv8f16(i32 %slice.7,
+                                                 <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                 <vscale x 8 x half> %b0, <vscale x 8 x half> %b1)
+  ret void
+}
+
+define void @test_fmla_f16_vg4_multi(i32 %slice,
+; CHECK-LABEL: test_fmla_f16_vg4_multi:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmla za.h[w8, 0, vgx4], { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK:    fmla za.h[w8, 7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK:    ret
+                                     <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                     <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                     <vscale x 8 x half> %b0, <vscale x 8 x half> %b1,
+                                     <vscale x 8 x half> %b2, <vscale x 8 x half> %b3) #0 {
+  call void @llvm.aarch64.sme.fmla.vg1x4.nxv8f16(i32 %slice,
+                                                 <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                 <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                                 <vscale x 8 x half> %b0, <vscale x 8 x half> %b1,
+                                                 <vscale x 8 x half> %b2, <vscale x 8 x half> %b3)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.vg1x4.nxv8f16(i32 %slice.7,
+                                                 <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                 <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                                 <vscale x 8 x half> %b0, <vscale x 8 x half> %b1,
+                                                 <vscale x 8 x half> %b2, <vscale x 8 x half> %b3)
+  ret void
+}
+
+define void @test_fmls_f16_vg2_multi(i32 %slice,
+; CHECK-LABEL: test_fmls_f16_vg2_multi:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmls za.h[w8, 0, vgx2], { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK:    fmls za.h[w8, 7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK:    ret
+                                     <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                     <vscale x 8 x half> %b0, <vscale x 8 x half> %b1) #0 {
+  call void @llvm.aarch64.sme.fmls.vg1x2.nxv8f16(i32 %slice,
+                                                <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                <vscale x 8 x half> %b0, <vscale x 8 x half> %b1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.vg1x2.nxv8f16(i32 %slice.7,
+                                                 <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                 <vscale x 8 x half> %b0, <vscale x 8 x half> %b1)
+  ret void
+}
+
+define void @test_fmls_f16_vg4_multi(i32 %slice,
+; CHECK-LABEL: test_fmls_f16_vg4_multi:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmls za.h[w8, 0, vgx4], { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK:    fmls za.h[w8, 7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK:    ret
+                                     <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                     <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                     <vscale x 8 x half> %b0, <vscale x 8 x half> %b1,
+                                     <vscale x 8 x half> %b2, <vscale x 8 x half> %b3) #0 {
+  call void @llvm.aarch64.sme.fmls.vg1x4.nxv8f16(i32 %slice,
+                                                 <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                 <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                                 <vscale x 8 x half> %b0, <vscale x 8 x half> %b1,
+                                                 <vscale x 8 x half> %b2, <vscale x 8 x half> %b3)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.vg1x4.nxv8f16(i32 %slice.7,
+                                                 <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                 <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                                 <vscale x 8 x half> %b0, <vscale x 8 x half> %b1,
+                                                 <vscale x 8 x half> %b2, <vscale x 8 x half> %b3)
+  ret void
+}
+
+define void @test_fmla_f16_vg2_index(i32 %slice,
+; CHECK-LABEL: test_fmla_f16_vg2_index:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmla za.h[w8, 0, vgx2], { z0.h, z1.h }, z2.h[7]
+; CHECK:    fmla za.h[w8, 7, vgx2], { z0.h, z1.h }, z2.h[7]
+; CHECK:    ret
+                                     <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                     <vscale x 8 x half> %b) #0 {
+  call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv8f16(i32 %slice,
+                                                      <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                      <vscale x 8 x half> %b, i32 7);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv8f16(i32 %slice.7,
+                                                     <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                     <vscale x 8 x half> %b, i32 7);
+  ret void
+}
+
+define void @test_fmla_f16_vg4_index(i32 %slice,
+; CHECK-LABEL: test_fmla_f16_vg4_index:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmla za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h[7]
+; CHECK:    fmla za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h[7]
+; CHECK:    ret
+                                     <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                     <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                     <vscale x 8 x half> %b) #0 {
+  call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv8f16(i32 %slice,
+                                                      <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                      <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                                      <vscale x 8 x half> %b, i32 7);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv8f16(i32 %slice,
+                                                      <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                      <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                                      <vscale x 8 x half> %b, i32 7);
+  ret void
+}
+
+define void @test_fmls_f16_vg2_index(i32 %slice,
+; CHECK-LABEL: test_fmls_f16_vg2_index:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmls za.h[w8, 0, vgx2], { z0.h, z1.h }, z2.h[7]
+; CHECK:    fmls za.h[w8, 7, vgx2], { z0.h, z1.h }, z2.h[7]
+; CHECK:    ret
+                                     <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                     <vscale x 8 x half> %b) #0 {
+  call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv8f16(i32 %slice,
+                                                      <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                      <vscale x 8 x half> %b, i32 7);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv8f16(i32 %slice.7,
+                                                     <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                     <vscale x 8 x half> %b, i32 7);
+  ret void
+}
+
+define void @test_fmls_f16_vg4_index(i32 %slice,
+; CHECK-LABEL: test_fmls_f16_vg4_index:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    fmls za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h[7]
+; CHECK:    fmls za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h[7]
+; CHECK:    ret
+                                     <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                     <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                     <vscale x 8 x half> %b) #0 {
+  call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv8f16(i32 %slice,
+                                                      <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                      <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                                      <vscale x 8 x half> %b, i32 7);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv8f16(i32 %slice,
+                                                      <vscale x 8 x half> %a0, <vscale x 8 x half> %a1,
+                                                      <vscale x 8 x half> %a2, <vscale x 8 x half> %a3,
+                                                      <vscale x 8 x half> %b, i32 7);
+  ret void
+}
+
+define void @test_fmla_bf16_vg2_single(i32 %slice, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fmla_bf16_vg2_single:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmla za.h[w8, 0, vgx2], { z0.h, z1.h }, z2.h
+; CHECK:    bfmla za.h[w8, 7, vgx2], { z0.h, z1.h }, z2.h
+; CHECK:    ret
+  call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1, <vscale x 8 x bfloat> %b)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.single.vg1x2.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1, <vscale x 8 x bfloat> %b)
+  ret void
+}
+
+define void @test_fmla_bf16_vg4_single(i32 %slice, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+; CHECK-LABEL: test_fmla_bf16_vg4_single:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmla za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h
+; CHECK:    bfmla za.h[w8, 7, vgx4], { z0.h - z3.h }, z4.h
+; CHECK:    ret
+                                       <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3, <vscale x 8 x bfloat> %b) #0 {
+  call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                         <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3, <vscale x 8 x bfloat> %b)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.single.vg1x4.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                         <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3, <vscale x 8 x bfloat> %b)
+  ret void
+}
+
+define void @test_fmls_bf16_vg2_single(i32 %slice, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: test_fmls_bf16_vg2_single:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmls za.h[w8, 0, vgx2], { z0.h, z1.h }, z2.h
+; CHECK:    bfmls za.h[w8, 7, vgx2], { z0.h, z1.h }, z2.h
+; CHECK:    ret
+  call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1, <vscale x 8 x bfloat> %b)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.single.vg1x2.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1, <vscale x 8 x bfloat> %b)
+  ret void
+}
+
+define void @test_fmls_bf16_vg4_single(i32 %slice, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+; CHECK-LABEL: test_fmls_bf16_vg4_single:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmls za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h
+; CHECK:    bfmls za.h[w8, 7, vgx4], { z0.h - z3.h }, z4.h
+; CHECK:    ret
+                                       <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3, <vscale x 8 x bfloat> %b) #0 {
+  call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                         <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3, <vscale x 8 x bfloat> %b)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.single.vg1x4.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                         <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3, <vscale x 8 x bfloat> %b)
+  ret void
+}
+
+define void @test_fmla_bf16_vg2_multi(i32 %slice,
+; CHECK-LABEL: test_fmla_bf16_vg2_multi:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmla za.h[w8, 0, vgx2], { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK:    bfmla za.h[w8, 7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK:    ret
+                                      <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                      <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1) #0 {
+  call void @llvm.aarch64.sme.fmla.vg1x2.nxv8bf16(i32 %slice,
+                                                 <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                 <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.vg1x2.nxv8bf16(i32 %slice.7,
+                                                  <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                  <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1)
+  ret void
+}
+
+define void @test_fmla_bf16_vg4_multi(i32 %slice,
+; CHECK-LABEL: test_fmla_bf16_vg4_multi:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmla za.h[w8, 0, vgx4], { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK:    bfmla za.h[w8, 7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK:    ret
+                                      <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                      <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                      <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1,
+                                      <vscale x 8 x bfloat> %b2, <vscale x 8 x bfloat> %b3) #0 {
+  call void @llvm.aarch64.sme.fmla.vg1x4.nxv8bf16(i32 %slice,
+                                                  <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                  <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                                  <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1,
+                                                  <vscale x 8 x bfloat> %b2, <vscale x 8 x bfloat> %b3)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.vg1x4.nxv8bf16(i32 %slice.7,
+                                                  <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                  <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                                  <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1,
+                                                  <vscale x 8 x bfloat> %b2, <vscale x 8 x bfloat> %b3)
+  ret void
+}
+
+define void @test_fmls_bf16_vg2_multi(i32 %slice,
+; CHECK-LABEL: test_fmls_bf16_vg2_multi:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmls za.h[w8, 0, vgx2], { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK:    bfmls za.h[w8, 7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK:    ret
+                                     <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                     <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1) #0 {
+  call void @llvm.aarch64.sme.fmls.vg1x2.nxv8bf16(i32 %slice,
+                                                <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.vg1x2.nxv8bf16(i32 %slice.7,
+                                                 <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                 <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1)
+  ret void
+}
+
+define void @test_fmls_bf16_vg4_multi(i32 %slice,
+; CHECK-LABEL: test_fmls_bf16_vg4_multi:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmls za.h[w8, 0, vgx4], { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK:    bfmls za.h[w8, 7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK:    ret
+                                     <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                     <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                     <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1,
+                                     <vscale x 8 x bfloat> %b2, <vscale x 8 x bfloat> %b3) #0 {
+  call void @llvm.aarch64.sme.fmls.vg1x4.nxv8bf16(i32 %slice,
+                                                 <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                 <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                                 <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1,
+                                                 <vscale x 8 x bfloat> %b2, <vscale x 8 x bfloat> %b3)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.vg1x4.nxv8bf16(i32 %slice.7,
+                                                 <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                 <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                                 <vscale x 8 x bfloat> %b0, <vscale x 8 x bfloat> %b1,
+                                                 <vscale x 8 x bfloat> %b2, <vscale x 8 x bfloat> %b3)
+  ret void
+}
+
+define void @test_fmla_bf16_vg2_index(i32 %slice,
+; CHECK-LABEL: test_fmla_bf16_vg2_index:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmla za.h[w8, 0, vgx2], { z0.h, z1.h }, z2.h[7]
+; CHECK:    bfmla za.h[w8, 7, vgx2], { z0.h, z1.h }, z2.h[7]
+; CHECK:    ret
+                                      <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                      <vscale x 8 x bfloat> %b) #0 {
+  call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv8bf16(i32 %slice,
+                                                       <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                       <vscale x 8 x bfloat> %b, i32 7);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.lane.vg1x2.nxv8bf16(i32 %slice.7,
+                                                       <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                       <vscale x 8 x bfloat> %b, i32 7);
+  ret void
+}
+
+define void @test_fmla_bf16_vg4_index(i32 %slice,
+; CHECK-LABEL: test_fmla_bf16_vg4_index:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmla za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h[7]
+; CHECK:    bfmla za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h[7]
+; CHECK:    ret
+                                      <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                      <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                      <vscale x 8 x bfloat> %b) #0 {
+  call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv8bf16(i32 %slice,
+                                                       <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                       <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                                       <vscale x 8 x bfloat> %b, i32 7);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmla.lane.vg1x4.nxv8bf16(i32 %slice,
+                                                       <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                       <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                                       <vscale x 8 x bfloat> %b, i32 7);
+  ret void
+}
+
+define void @test_fmls_bf16_vg2_index(i32 %slice,
+; CHECK-LABEL: test_fmls_bf16_vg2_index:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmls za.h[w8, 0, vgx2], { z0.h, z1.h }, z2.h[7]
+; CHECK:    bfmls za.h[w8, 7, vgx2], { z0.h, z1.h }, z2.h[7]
+; CHECK:    ret
+                                      <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                      <vscale x 8 x bfloat> %b) #0 {
+  call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv8bf16(i32 %slice,
+                                                       <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                       <vscale x 8 x bfloat> %b, i32 7);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.lane.vg1x2.nxv8bf16(i32 %slice.7,
+                                                      <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                      <vscale x 8 x bfloat> %b, i32 7);
+  ret void
+}
+
+define void @test_fmls_bf16_vg4_index(i32 %slice,
+; CHECK-LABEL: test_fmls_bf16_vg4_index:
+; CHECK:  // %bb.0:
+; CHECK:    mov w8, w0
+; CHECK:    bfmls za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h[7]
+; CHECK:    bfmls za.h[w8, 0, vgx4], { z0.h - z3.h }, z4.h[7]
+; CHECK:    ret
+                                      <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                      <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                      <vscale x 8 x bfloat> %b) #0 {
+  call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv8bf16(i32 %slice,
+                                                       <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                       <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                                       <vscale x 8 x bfloat> %b, i32 7);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.fmls.lane.vg1x4.nxv8bf16(i32 %slice,
+                                                       <vscale x 8 x bfloat> %a0, <vscale x 8 x bfloat> %a1,
+                                                       <vscale x 8 x bfloat> %a2, <vscale x 8 x bfloat> %a3,
+                                                       <vscale x 8 x bfloat> %b, i32 7);
+  ret void
+}
+
+attributes #0 = { nounwind "target-features"="+sme2p1,+sme-f16f16,+b16b16" }
-- 
GitLab


From 2e8d8155969f90b8f17634ce9a8e4541fb21dbab Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter@arm.com>
Date: Fri, 10 May 2024 11:22:11 +0100
Subject: [PATCH 0392/1206] [TTI] Support scalable offsets in
 getScalingFactorCost (#88113)

Part of the work to support vscale-relative immediates in LSR.
---
 llvm/include/llvm/Analysis/TargetTransformInfo.h       | 6 +++---
 llvm/include/llvm/Analysis/TargetTransformInfoImpl.h   | 8 +++++---
 llvm/include/llvm/CodeGen/BasicTTIImpl.h               | 5 +++--
 llvm/lib/Analysis/TargetTransformInfo.cpp              | 2 +-
 llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 5 +++--
 llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h   | 2 +-
 llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp         | 5 +++--
 llvm/lib/Target/ARM/ARMTargetTransformInfo.h           | 2 +-
 llvm/lib/Target/X86/X86TargetTransformInfo.cpp         | 5 +++--
 llvm/lib/Target/X86/X86TargetTransformInfo.h           | 2 +-
 llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp      | 6 ++++--
 11 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 1c76821fe5e4..f0eb83c143e2 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -834,7 +834,7 @@ public:
   /// If the AM is not supported, it returns a negative value.
   /// TODO: Handle pre/postinc as well.
   InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
-                                       int64_t BaseOffset, bool HasBaseReg,
+                                       StackOffset BaseOffset, bool HasBaseReg,
                                        int64_t Scale,
                                        unsigned AddrSpace = 0) const;
 
@@ -1891,7 +1891,7 @@ public:
   virtual bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) = 0;
   virtual bool prefersVectorizedAddressing() = 0;
   virtual InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
-                                               int64_t BaseOffset,
+                                               StackOffset BaseOffset,
                                                bool HasBaseReg, int64_t Scale,
                                                unsigned AddrSpace) = 0;
   virtual bool LSRWithInstrQueries() = 0;
@@ -2403,7 +2403,7 @@ public:
     return Impl.prefersVectorizedAddressing();
   }
   InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
-                                       int64_t BaseOffset, bool HasBaseReg,
+                                       StackOffset BaseOffset, bool HasBaseReg,
                                        int64_t Scale,
                                        unsigned AddrSpace) override {
     return Impl.getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, Scale,
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 4d5cd963e092..262ebdb3cbef 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -32,6 +32,7 @@ class Function;
 /// Base class for use as a mix-in that aids implementing
 /// a TargetTransformInfo-compatible class.
 class TargetTransformInfoImplBase {
+
 protected:
   typedef TargetTransformInfo TTI;
 
@@ -326,12 +327,13 @@ public:
   bool prefersVectorizedAddressing() const { return true; }
 
   InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
-                                       int64_t BaseOffset, bool HasBaseReg,
+                                       StackOffset BaseOffset, bool HasBaseReg,
                                        int64_t Scale,
                                        unsigned AddrSpace) const {
     // Guess that all legal addressing mode are free.
-    if (isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, Scale,
-                              AddrSpace))
+    if (isLegalAddressingMode(Ty, BaseGV, BaseOffset.getFixed(), HasBaseReg,
+                              Scale, AddrSpace, /*I=*/nullptr,
+                              BaseOffset.getScalable()))
       return 0;
     return -1;
   }
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index bcb60c656296..fa481886b268 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -404,13 +404,14 @@ public:
   }
 
   InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
-                                       int64_t BaseOffset, bool HasBaseReg,
+                                       StackOffset BaseOffset, bool HasBaseReg,
                                        int64_t Scale, unsigned AddrSpace) {
     TargetLoweringBase::AddrMode AM;
     AM.BaseGV = BaseGV;
-    AM.BaseOffs = BaseOffset;
+    AM.BaseOffs = BaseOffset.getFixed();
     AM.HasBaseReg = HasBaseReg;
     AM.Scale = Scale;
+    AM.ScalableOffset = BaseOffset.getScalable();
     if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace))
       return 0;
     return -1;
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 33c899fe8899..00443ace46f7 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -531,7 +531,7 @@ bool TargetTransformInfo::prefersVectorizedAddressing() const {
 }
 
 InstructionCost TargetTransformInfo::getScalingFactorCost(
-    Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg,
+    Type *Ty, GlobalValue *BaseGV, StackOffset BaseOffset, bool HasBaseReg,
     int64_t Scale, unsigned AddrSpace) const {
   InstructionCost Cost = TTIImpl->getScalingFactorCost(
       Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace);
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 4b410826f4bb..f49c73dc7951 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4183,7 +4183,7 @@ bool AArch64TTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) {
 
 InstructionCost
 AArch64TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
-                                     int64_t BaseOffset, bool HasBaseReg,
+                                     StackOffset BaseOffset, bool HasBaseReg,
                                      int64_t Scale, unsigned AddrSpace) const {
   // Scaling factors are not free at all.
   // Operands                     | Rt Latency
@@ -4194,9 +4194,10 @@ AArch64TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
   // Rt, [Xn, Wm, <extend> #imm]  |
   TargetLoweringBase::AddrMode AM;
   AM.BaseGV = BaseGV;
-  AM.BaseOffs = BaseOffset;
+  AM.BaseOffs = BaseOffset.getFixed();
   AM.HasBaseReg = HasBaseReg;
   AM.Scale = Scale;
+  AM.ScalableOffset = BaseOffset.getScalable();
   if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace))
     // Scale represents reg2 * scale, thus account for 1 if
     // it is not equal to 0 or 1.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 678c132e6a80..2f44aaa3e26a 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -407,7 +407,7 @@ public:
   /// If the AM is supported, the return value must be >= 0.
   /// If the AM is not supported, it returns a negative value.
   InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
-                                       int64_t BaseOffset, bool HasBaseReg,
+                                       StackOffset BaseOffset, bool HasBaseReg,
                                        int64_t Scale, unsigned AddrSpace) const;
   /// @}
 
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index ee87f7f0e555..7db2e8ee7e6f 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -2571,14 +2571,15 @@ bool ARMTTIImpl::preferPredicatedReductionSelect(
 }
 
 InstructionCost ARMTTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
-                                                 int64_t BaseOffset,
+                                                 StackOffset BaseOffset,
                                                  bool HasBaseReg, int64_t Scale,
                                                  unsigned AddrSpace) const {
   TargetLoweringBase::AddrMode AM;
   AM.BaseGV = BaseGV;
-  AM.BaseOffs = BaseOffset;
+  AM.BaseOffs = BaseOffset.getFixed();
   AM.HasBaseReg = HasBaseReg;
   AM.Scale = Scale;
+  AM.ScalableOffset = BaseOffset.getScalable();
   if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace)) {
     if (ST->hasFPAO())
       return AM.Scale < 0 ? 1 : 0; // positive offsets execute faster
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 58eab45b9641..8c4b92b85688 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -303,7 +303,7 @@ public:
   /// If the AM is supported, the return value must be >= 0.
   /// If the AM is not supported, the return value must be negative.
   InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
-                                       int64_t BaseOffset, bool HasBaseReg,
+                                       StackOffset BaseOffset, bool HasBaseReg,
                                        int64_t Scale, unsigned AddrSpace) const;
 
   bool maybeLoweredToCall(Instruction &I);
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 6b7cddc6d72e..d43480d0a012 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -6741,7 +6741,7 @@ InstructionCost X86TTIImpl::getInterleavedMemoryOpCost(
 }
 
 InstructionCost X86TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
-                                                 int64_t BaseOffset,
+                                                 StackOffset BaseOffset,
                                                  bool HasBaseReg, int64_t Scale,
                                                  unsigned AddrSpace) const {
   // Scaling factors are not free at all.
@@ -6764,9 +6764,10 @@ InstructionCost X86TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
   // vmovaps %ymm1, (%r8) can use port 2, 3, or 7.
   TargetLoweringBase::AddrMode AM;
   AM.BaseGV = BaseGV;
-  AM.BaseOffs = BaseOffset;
+  AM.BaseOffs = BaseOffset.getFixed();
   AM.HasBaseReg = HasBaseReg;
   AM.Scale = Scale;
+  AM.ScalableOffset = BaseOffset.getScalable();
   if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace))
     // Scale represents reg2 * scale, thus account for 1
     // as soon as we use a second register.
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index b50193074573..d720cc136b8a 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -253,7 +253,7 @@ public:
   /// If the AM is supported, the return value must be >= 0.
   /// If the AM is not supported, it returns a negative value.
   InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
-                                       int64_t BaseOffset, bool HasBaseReg,
+                                       StackOffset BaseOffset, bool HasBaseReg,
                                        int64_t Scale, unsigned AddrSpace) const;
 
   bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index ec42e2d6e193..eb1904ccaff3 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -1817,10 +1817,12 @@ static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI,
   case LSRUse::Address: {
     // Check the scaling factor cost with both the min and max offsets.
     InstructionCost ScaleCostMinOffset = TTI.getScalingFactorCost(
-        LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MinOffset, F.HasBaseReg,
+        LU.AccessTy.MemTy, F.BaseGV,
+        StackOffset::getFixed(F.BaseOffset + LU.MinOffset), F.HasBaseReg,
         F.Scale, LU.AccessTy.AddrSpace);
     InstructionCost ScaleCostMaxOffset = TTI.getScalingFactorCost(
-        LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MaxOffset, F.HasBaseReg,
+        LU.AccessTy.MemTy, F.BaseGV,
+        StackOffset::getFixed(F.BaseOffset + LU.MaxOffset), F.HasBaseReg,
         F.Scale, LU.AccessTy.AddrSpace);
 
     assert(ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() &&
-- 
GitLab


From 28767afd53353d9333b0adf6f0fafa1592092532 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 10 May 2024 11:47:13 +0100
Subject: [PATCH 0393/1206] [LAA] Support backward dependences with
 non-constant distance. (#91525)

Following up to 933f49248, also update the code reasoning about
backwards dependences to support non-constant distances.

Update the code to use the signed minimum distance instead of a constant
distance

This means e checked the lower bound of the dependence distance and the
distance may be larger at runtime (and safe for vectorization). Whether
to classify it as Unknown or Backwards depends on the vector width and
LAA was updated to take TTI to get the maximum vector register width.

If the minimum dependence distance is larger than the max vector width,
we consider it as backwards-vectorizable. Otherwise we classify them as
Unknown, so we re-try with runtime checks.

PR: https://github.com/llvm/llvm-project/pull/91525
---
 .../llvm/Analysis/LoopAccessAnalysis.h        |  23 +-
 llvm/lib/Analysis/LoopAccessAnalysis.cpp      |  93 ++++--
 llvm/lib/Transforms/Scalar/LoopFlatten.cpp    |   2 +-
 .../Transforms/Scalar/LoopVersioningLICM.cpp  |   2 +-
 .../multiple-strides-rt-memory-checks.ll      |   2 +-
 .../non-constant-distance-backward.ll         | 292 ++++++++++--------
 .../Transforms/Vectorize/VPlanSlpTest.cpp     |   2 +-
 7 files changed, 247 insertions(+), 169 deletions(-)

diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index cf998e66ee48..6ebd0fb8477a 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -181,8 +181,10 @@ public:
                const SmallVectorImpl<Instruction *> &Instrs) const;
   };
 
-  MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L)
-      : PSE(PSE), InnermostLoop(L) {}
+  MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L,
+                   unsigned MaxTargetVectorWidthInBits)
+      : PSE(PSE), InnermostLoop(L),
+        MaxTargetVectorWidthInBits(MaxTargetVectorWidthInBits) {}
 
   /// Register the location (instructions are given increasing numbers)
   /// of a write access.
@@ -314,6 +316,12 @@ private:
   /// RecordDependences is true.
   SmallVector<Dependence, 8> Dependences;
 
+  /// The maximum width of a target's vector registers multiplied by 2 to also
+  /// roughly account for additional interleaving. Is used to decide if a
+  /// backwards dependence with non-constant stride should be classified as
+  /// backwards-vectorizable or unknown (triggering a runtime check).
+  unsigned MaxTargetVectorWidthInBits = 0;
+
   /// Check whether there is a plausible dependence between the two
   /// accesses.
   ///
@@ -575,8 +583,9 @@ private:
 /// PSE must be emitted in order for the results of this analysis to be valid.
 class LoopAccessInfo {
 public:
-  LoopAccessInfo(Loop *L, ScalarEvolution *SE, const TargetLibraryInfo *TLI,
-                 AAResults *AA, DominatorTree *DT, LoopInfo *LI);
+  LoopAccessInfo(Loop *L, ScalarEvolution *SE, const TargetTransformInfo *TTI,
+                 const TargetLibraryInfo *TLI, AAResults *AA, DominatorTree *DT,
+                 LoopInfo *LI);
 
   /// Return true we can analyze the memory accesses in the loop and there are
   /// no memory dependence cycles. Note that for dependences between loads &
@@ -799,12 +808,14 @@ class LoopAccessInfoManager {
   AAResults &AA;
   DominatorTree &DT;
   LoopInfo &LI;
+  TargetTransformInfo *TTI;
   const TargetLibraryInfo *TLI = nullptr;
 
 public:
   LoopAccessInfoManager(ScalarEvolution &SE, AAResults &AA, DominatorTree &DT,
-                        LoopInfo &LI, const TargetLibraryInfo *TLI)
-      : SE(SE), AA(AA), DT(DT), LI(LI), TLI(TLI) {}
+                        LoopInfo &LI, TargetTransformInfo *TTI,
+                        const TargetLibraryInfo *TLI)
+      : SE(SE), AA(AA), DT(DT), LI(LI), TTI(TTI), TLI(TLI) {}
 
   const LoopAccessInfo &getInfo(Loop &L);
 
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 6fc7da168b42..d071e5332440 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -31,6 +31,7 @@
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/BasicBlock.h"
@@ -2122,17 +2123,24 @@ MemoryDepChecker::Dependence::DepType MemoryDepChecker::isDependent(
     return Dependence::Forward;
   }
 
-  if (!C) {
-    // TODO: FoundNonConstantDistanceDependence is used as a necessary condition
-    // to consider retrying with runtime checks. Historically, we did not set it
-    // when strides were different but there is no inherent reason to.
+  int64_t MinDistance = SE.getSignedRangeMin(Dist).getSExtValue();
+  // Below we only handle strictly positive distances.
+  if (MinDistance <= 0) {
     FoundNonConstantDistanceDependence |= CommonStride.has_value();
-    LLVM_DEBUG(dbgs() << "LAA: Dependence because of non-constant distance\n");
     return Dependence::Unknown;
   }
 
-  if (!SE.isKnownPositive(Dist))
-    return Dependence::Unknown;
+  if (!isa<SCEVConstant>(Dist)) {
+    // Previously this case would be treated as Unknown, possibly setting
+    // FoundNonConstantDistanceDependence to force re-trying with runtime
+    // checks. Until the TODO below is addressed, set it here to preserve
+    // original behavior w.r.t. re-trying with runtime checks.
+    // TODO: FoundNonConstantDistanceDependence is used as a necessary
+    // condition to consider retrying with runtime checks. Historically, we
+    // did not set it when strides were different but there is no inherent
+    // reason to.
+    FoundNonConstantDistanceDependence |= CommonStride.has_value();
+  }
 
   if (!HasSameSize) {
     LLVM_DEBUG(dbgs() << "LAA: ReadWrite-Write positive dependency with "
@@ -2140,14 +2148,9 @@ MemoryDepChecker::Dependence::DepType MemoryDepChecker::isDependent(
     return Dependence::Unknown;
   }
 
-  // The logic below currently only supports StrideA ==  StrideB, i.e. there's a
-  // common stride.
   if (!CommonStride)
     return Dependence::Unknown;
 
-  const APInt &Val = C->getAPInt();
-  int64_t Distance = Val.getSExtValue();
-
   // Bail out early if passed-in parameters make vectorization not feasible.
   unsigned ForcedFactor = (VectorizerParams::VectorizationFactor ?
                            VectorizerParams::VectorizationFactor : 1);
@@ -2172,8 +2175,8 @@ MemoryDepChecker::Dependence::DepType MemoryDepChecker::isDependent(
   //     | A[0] |      | A[2] |      | A[4] |      | A[6] |      |
   //                              | B[0] |      | B[2] |      | B[4] |
   //
-  // Distance needs for vectorizing iterations except the last iteration:
-  // 4 * 2 * (MinNumIter - 1). Distance needs for the last iteration: 4.
+  // MinDistance needs for vectorizing iterations except the last iteration:
+  // 4 * 2 * (MinNumIter - 1). MinDistance needs for the last iteration: 4.
   // So the minimum distance needed is: 4 * 2 * (MinNumIter - 1) + 4.
   //
   // If MinNumIter is 2, it is vectorizable as the minimum distance needed is
@@ -2182,11 +2185,22 @@ MemoryDepChecker::Dependence::DepType MemoryDepChecker::isDependent(
   // If MinNumIter is 4 (Say if a user forces the vectorization factor to be 4),
   // the minimum distance needed is 28, which is greater than distance. It is
   // not safe to do vectorization.
+
+  // We know that Dist is positive, but it may not be constant. Use the signed
+  // minimum for computations below, as this ensures we compute the closest
+  // possible dependence distance.
   uint64_t MinDistanceNeeded =
-      TypeByteSize * (*CommonStride) * (MinNumIter - 1) + TypeByteSize;
-  if (MinDistanceNeeded > static_cast<uint64_t>(Distance)) {
-    LLVM_DEBUG(dbgs() << "LAA: Failure because of positive distance "
-                      << Distance << '\n');
+      TypeByteSize * *CommonStride * (MinNumIter - 1) + TypeByteSize;
+  if (MinDistanceNeeded > static_cast<uint64_t>(MinDistance)) {
+    if (!isa<SCEVConstant>(Dist)) {
+      // For non-constant distances, we checked the lower bound of the
+      // dependence distance and the distance may be larger at runtime (and safe
+      // for vectorization). Classify it as Unknown, so we re-try with runtime
+      // checks.
+      return Dependence::Unknown;
+    }
+    LLVM_DEBUG(dbgs() << "LAA: Failure because of positive minimum distance "
+                      << MinDistance << '\n');
     return Dependence::Backward;
   }
 
@@ -2215,12 +2229,13 @@ MemoryDepChecker::Dependence::DepType MemoryDepChecker::isDependent(
   // is 8, which is less than 2 and forbidden vectorization, But actually
   // both A and B could be vectorized by 2 iterations.
   MinDepDistBytes =
-      std::min(static_cast<uint64_t>(Distance), MinDepDistBytes);
+      std::min(static_cast<uint64_t>(MinDistance), MinDepDistBytes);
 
   bool IsTrueDataDependence = (!AIsWrite && BIsWrite);
   uint64_t MinDepDistBytesOld = MinDepDistBytes;
   if (IsTrueDataDependence && EnableForwardingConflictDetection &&
-      couldPreventStoreLoadForward(Distance, TypeByteSize)) {
+      isa<SCEVConstant>(Dist) &&
+      couldPreventStoreLoadForward(MinDistance, TypeByteSize)) {
     // Sanity check that we didn't update MinDepDistBytes when calling
     // couldPreventStoreLoadForward
     assert(MinDepDistBytes == MinDepDistBytesOld &&
@@ -2232,10 +2247,18 @@ MemoryDepChecker::Dependence::DepType MemoryDepChecker::isDependent(
 
   // An update to MinDepDistBytes requires an update to MaxSafeVectorWidthInBits
   // since there is a backwards dependency.
-  uint64_t MaxVF = MinDepDistBytes / (TypeByteSize * (*CommonStride));
-  LLVM_DEBUG(dbgs() << "LAA: Positive distance " << Val.getSExtValue()
+  uint64_t MaxVF = MinDepDistBytes / (TypeByteSize * *CommonStride);
+  LLVM_DEBUG(dbgs() << "LAA: Positive min distance " << MinDistance
                     << " with max VF = " << MaxVF << '\n');
+
   uint64_t MaxVFInBits = MaxVF * TypeByteSize * 8;
+  if (!isa<SCEVConstant>(Dist) && MaxVFInBits < MaxTargetVectorWidthInBits) {
+    // For non-constant distances, we checked the lower bound of the dependence
+    // distance and the distance may be larger at runtime (and safe for
+    // vectorization). Classify it as Unknown, so we re-try with runtime checks.
+    return Dependence::Unknown;
+  }
+
   MaxSafeVectorWidthInBits = std::min(MaxSafeVectorWidthInBits, MaxVFInBits);
   return Dependence::BackwardVectorizable;
 }
@@ -3018,11 +3041,28 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
 }
 
 LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
+                               const TargetTransformInfo *TTI,
                                const TargetLibraryInfo *TLI, AAResults *AA,
                                DominatorTree *DT, LoopInfo *LI)
     : PSE(std::make_unique<PredicatedScalarEvolution>(*SE, *L)),
-      PtrRtChecking(nullptr),
-      DepChecker(std::make_unique<MemoryDepChecker>(*PSE, L)), TheLoop(L) {
+      PtrRtChecking(nullptr), TheLoop(L) {
+  unsigned MaxTargetVectorWidthInBits = std::numeric_limits<unsigned>::max();
+  if (TTI) {
+    TypeSize FixedWidth =
+        TTI->getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector);
+    if (FixedWidth.isNonZero()) {
+      // Scale the vector width by 2 as rough estimate to also consider
+      // interleaving.
+      MaxTargetVectorWidthInBits = FixedWidth.getFixedValue() * 2;
+    }
+
+    TypeSize ScalableWidth =
+        TTI->getRegisterBitWidth(TargetTransformInfo::RGK_ScalableVector);
+    if (ScalableWidth.isNonZero())
+      MaxTargetVectorWidthInBits = std::numeric_limits<unsigned>::max();
+  }
+  DepChecker =
+      std::make_unique<MemoryDepChecker>(*PSE, L, MaxTargetVectorWidthInBits);
   PtrRtChecking = std::make_unique<RuntimePointerChecking>(*DepChecker, SE);
   if (canAnalyzeLoop()) {
     analyzeLoop(AA, LI, TLI, DT);
@@ -3082,7 +3122,7 @@ const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L) {
 
   if (I.second)
     I.first->second =
-        std::make_unique<LoopAccessInfo>(&L, &SE, TLI, &AA, &DT, &LI);
+        std::make_unique<LoopAccessInfo>(&L, &SE, TTI, TLI, &AA, &DT, &LI);
 
   return *I.first->second;
 }
@@ -3111,8 +3151,9 @@ LoopAccessInfoManager LoopAccessAnalysis::run(Function &F,
   auto &AA = FAM.getResult<AAManager>(F);
   auto &DT = FAM.getResult<DominatorTreeAnalysis>(F);
   auto &LI = FAM.getResult<LoopAnalysis>(F);
+  auto &TTI = FAM.getResult<TargetIRAnalysis>(F);
   auto &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
-  return LoopAccessInfoManager(SE, AA, DT, LI, &TLI);
+  return LoopAccessInfoManager(SE, AA, DT, LI, &TTI, &TLI);
 }
 
 AnalysisKey LoopAccessAnalysis::Key;
diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
index 0e9cf328f149..a7f8a22ece27 100644
--- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
@@ -1005,7 +1005,7 @@ PreservedAnalyses LoopFlattenPass::run(LoopNest &LN, LoopAnalysisManager &LAM,
   // in simplified form, and also needs LCSSA. Running
   // this pass will simplify all loops that contain inner loops,
   // regardless of whether anything ends up being flattened.
-  LoopAccessInfoManager LAIM(AR.SE, AR.AA, AR.DT, AR.LI, nullptr);
+  LoopAccessInfoManager LAIM(AR.SE, AR.AA, AR.DT, AR.LI, &AR.TTI, nullptr);
   for (Loop *InnerLoop : LN.getLoops()) {
     auto *OuterLoop = InnerLoop->getParentLoop();
     if (!OuterLoop)
diff --git a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
index f39c24484840..663715948241 100644
--- a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
@@ -582,7 +582,7 @@ PreservedAnalyses LoopVersioningLICMPass::run(Loop &L, LoopAnalysisManager &AM,
   const Function *F = L.getHeader()->getParent();
   OptimizationRemarkEmitter ORE(F);
 
-  LoopAccessInfoManager LAIs(*SE, *AA, *DT, LAR.LI, nullptr);
+  LoopAccessInfoManager LAIs(*SE, *AA, *DT, LAR.LI, nullptr, nullptr);
   if (!LoopVersioningLICM(AA, SE, &ORE, LAIs, LAR.LI, &L).run(DT))
     return PreservedAnalyses::all();
   return getLoopPassPreservedAnalyses();
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/multiple-strides-rt-memory-checks.ll b/llvm/test/Analysis/LoopAccessAnalysis/multiple-strides-rt-memory-checks.ll
index 3b4f2a170d8c..d96a6ea7c555 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/multiple-strides-rt-memory-checks.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/multiple-strides-rt-memory-checks.ll
@@ -23,7 +23,7 @@
 
 ; CHECK: function 'Test':
 ; CHECK:   .inner:
-; CHECK-NEXT:     Memory dependences are safe with run-time checks
+; CHECK-NEXT:     Memory dependences are safe with a maximum safe vector width of 2048 bits with run-time checks
 ; CHECK-NEXT:     Dependences:
 ; CHECK-NEXT:     Run-time memory checks:
 ; CHECK:          Check 0:
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/non-constant-distance-backward.ll b/llvm/test/Analysis/LoopAccessAnalysis/non-constant-distance-backward.ll
index 5a95dcca1050..0058135a30d6 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/non-constant-distance-backward.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/non-constant-distance-backward.ll
@@ -1,45 +1,47 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck %s
-; RUN: opt -passes='print<access-info>' -disable-output -mtriple=arm64-apple-macosx %s 2>&1 | FileCheck %s
-; RUN: opt -passes='print<access-info>' -disable-output -mtriple=arm64-apple-macosx -mattr=+sve %s 2>&1 | FileCheck %s
+; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck --check-prefixes=COMMON,MAXLEN %s
+; RUN: opt -passes='print<access-info>' -disable-output -mtriple=arm64-apple-macosx %s 2>&1 | FileCheck --check-prefixes=COMMON,VW128 %s
+; RUN: opt -passes='print<access-info>' -disable-output -mtriple=arm64-apple-macosx -mattr=+sve %s 2>&1 | FileCheck --check-prefixes=COMMON,MAXLEN %s
 
 ; REQUIRES: aarch64-registered-target
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
+; If the dependence distance is not a constant, whether it gets identified as backwards or unknown depends on the minimum distance and the target's vector length.
+
 define void @backward_min_distance_8(ptr %A, i64 %N) {
-; CHECK-LABEL: 'backward_min_distance_8'
-; CHECK-NEXT:    loop:
-; CHECK-NEXT:      Memory dependences are safe with run-time checks
-; CHECK-NEXT:      Dependences:
-; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Check 0:
-; CHECK-NEXT:        Comparing group ([[GRP1:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
-; CHECK-NEXT:        Against group ([[GRP2:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
-; CHECK-NEXT:      Grouped accesses:
-; CHECK-NEXT:        Group [[GRP1]]:
-; CHECK-NEXT:          (Low: {(1 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(257 + %A),+,1}<nw><%outer.header>)
-; CHECK-NEXT:            Member: {{\{\{}}(1 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP2]]:
-; CHECK-NEXT:          (Low: %A High: (256 + %A))
-; CHECK-NEXT:            Member: {%A,+,1}<nuw><%loop>
-; CHECK-EMPTY:
-; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
-; CHECK-NEXT:      SCEV assumptions:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Expressions re-written:
-; CHECK-NEXT:    outer.header:
-; CHECK-NEXT:      Report: loop is not the innermost loop
-; CHECK-NEXT:      Dependences:
-; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Grouped accesses:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
-; CHECK-NEXT:      SCEV assumptions:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Expressions re-written:
+; COMMON-LABEL: 'backward_min_distance_8'
+; COMMON-NEXT:    loop:
+; COMMON-NEXT:      Memory dependences are safe with run-time checks
+; COMMON-NEXT:      Dependences:
+; COMMON-NEXT:      Run-time memory checks:
+; COMMON-NEXT:      Check 0:
+; COMMON-NEXT:        Comparing group ([[GRP1:0x[0-9a-f]+]]):
+; COMMON-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+; COMMON-NEXT:        Against group ([[GRP2:0x[0-9a-f]+]]):
+; COMMON-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+; COMMON-NEXT:      Grouped accesses:
+; COMMON-NEXT:        Group [[GRP1]]:
+; COMMON-NEXT:          (Low: {(1 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(257 + %A),+,1}<nw><%outer.header>)
+; COMMON-NEXT:            Member: {{\{\{}}(1 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
+; COMMON-NEXT:        Group [[GRP2]]:
+; COMMON-NEXT:          (Low: %A High: (256 + %A))
+; COMMON-NEXT:            Member: {%A,+,1}<nuw><%loop>
+; COMMON-EMPTY:
+; COMMON-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; COMMON-NEXT:      SCEV assumptions:
+; COMMON-EMPTY:
+; COMMON-NEXT:      Expressions re-written:
+; COMMON-NEXT:    outer.header:
+; COMMON-NEXT:      Report: loop is not the innermost loop
+; COMMON-NEXT:      Dependences:
+; COMMON-NEXT:      Run-time memory checks:
+; COMMON-NEXT:      Grouped accesses:
+; COMMON-EMPTY:
+; COMMON-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; COMMON-NEXT:      SCEV assumptions:
+; COMMON-EMPTY:
+; COMMON-NEXT:      Expressions re-written:
 ;
 entry:
   br label %outer.header
@@ -70,38 +72,38 @@ exit:
 }
 
 define void @backward_min_distance_120(ptr %A, i64 %N) {
-; CHECK-LABEL: 'backward_min_distance_120'
-; CHECK-NEXT:    loop:
-; CHECK-NEXT:      Memory dependences are safe with run-time checks
-; CHECK-NEXT:      Dependences:
-; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Check 0:
-; CHECK-NEXT:        Comparing group ([[GRP3:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
-; CHECK-NEXT:        Against group ([[GRP4:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
-; CHECK-NEXT:      Grouped accesses:
-; CHECK-NEXT:        Group [[GRP3]]:
-; CHECK-NEXT:          (Low: {(15 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(271 + %A),+,1}<nw><%outer.header>)
-; CHECK-NEXT:            Member: {{\{\{}}(15 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP4]]:
-; CHECK-NEXT:          (Low: %A High: (256 + %A))
-; CHECK-NEXT:            Member: {%A,+,1}<nuw><%loop>
-; CHECK-EMPTY:
-; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
-; CHECK-NEXT:      SCEV assumptions:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Expressions re-written:
-; CHECK-NEXT:    outer.header:
-; CHECK-NEXT:      Report: loop is not the innermost loop
-; CHECK-NEXT:      Dependences:
-; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Grouped accesses:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
-; CHECK-NEXT:      SCEV assumptions:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Expressions re-written:
+; COMMON-LABEL: 'backward_min_distance_120'
+; COMMON-NEXT:    loop:
+; COMMON-NEXT:      Memory dependences are safe with run-time checks
+; COMMON-NEXT:      Dependences:
+; COMMON-NEXT:      Run-time memory checks:
+; COMMON-NEXT:      Check 0:
+; COMMON-NEXT:        Comparing group ([[GRP3:0x[0-9a-f]+]]):
+; COMMON-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+; COMMON-NEXT:        Against group ([[GRP4:0x[0-9a-f]+]]):
+; COMMON-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+; COMMON-NEXT:      Grouped accesses:
+; COMMON-NEXT:        Group [[GRP3]]:
+; COMMON-NEXT:          (Low: {(15 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(271 + %A),+,1}<nw><%outer.header>)
+; COMMON-NEXT:            Member: {{\{\{}}(15 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
+; COMMON-NEXT:        Group [[GRP4]]:
+; COMMON-NEXT:          (Low: %A High: (256 + %A))
+; COMMON-NEXT:            Member: {%A,+,1}<nuw><%loop>
+; COMMON-EMPTY:
+; COMMON-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; COMMON-NEXT:      SCEV assumptions:
+; COMMON-EMPTY:
+; COMMON-NEXT:      Expressions re-written:
+; COMMON-NEXT:    outer.header:
+; COMMON-NEXT:      Report: loop is not the innermost loop
+; COMMON-NEXT:      Dependences:
+; COMMON-NEXT:      Run-time memory checks:
+; COMMON-NEXT:      Grouped accesses:
+; COMMON-EMPTY:
+; COMMON-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; COMMON-NEXT:      SCEV assumptions:
+; COMMON-EMPTY:
+; COMMON-NEXT:      Expressions re-written:
 ;
 entry:
   br label %outer.header
@@ -131,41 +133,39 @@ exit:
   ret void
 }
 
-
-declare void @llvm.assume(i1)
 define void @backward_min_distance_128(ptr %A, i64 %N) {
-; CHECK-LABEL: 'backward_min_distance_128'
-; CHECK-NEXT:    loop:
-; CHECK-NEXT:      Memory dependences are safe with run-time checks
-; CHECK-NEXT:      Dependences:
-; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Check 0:
-; CHECK-NEXT:        Comparing group ([[GRP5:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
-; CHECK-NEXT:        Against group ([[GRP6:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
-; CHECK-NEXT:      Grouped accesses:
-; CHECK-NEXT:        Group [[GRP5]]:
-; CHECK-NEXT:          (Low: {(16 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(272 + %A),+,1}<nw><%outer.header>)
-; CHECK-NEXT:            Member: {{\{\{}}(16 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP6]]:
-; CHECK-NEXT:          (Low: %A High: (256 + %A))
-; CHECK-NEXT:            Member: {%A,+,1}<nuw><%loop>
-; CHECK-EMPTY:
-; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
-; CHECK-NEXT:      SCEV assumptions:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Expressions re-written:
-; CHECK-NEXT:    outer.header:
-; CHECK-NEXT:      Report: loop is not the innermost loop
-; CHECK-NEXT:      Dependences:
-; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Grouped accesses:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
-; CHECK-NEXT:      SCEV assumptions:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Expressions re-written:
+; COMMON-LABEL: 'backward_min_distance_128'
+; COMMON-NEXT:    loop:
+; COMMON-NEXT:      Memory dependences are safe with run-time checks
+; COMMON-NEXT:      Dependences:
+; COMMON-NEXT:      Run-time memory checks:
+; COMMON-NEXT:      Check 0:
+; COMMON-NEXT:        Comparing group ([[GRP13:0x[0-9a-f]+]]):
+; COMMON-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+; COMMON-NEXT:        Against group ([[GRP14:0x[0-9a-f]+]]):
+; COMMON-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+; COMMON-NEXT:      Grouped accesses:
+; COMMON-NEXT:        Group [[GRP13]]:
+; COMMON-NEXT:          (Low: {(16 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(272 + %A),+,1}<nw><%outer.header>)
+; COMMON-NEXT:            Member: {{\{\{}}(16 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
+; COMMON-NEXT:        Group [[GRP14]]:
+; COMMON-NEXT:          (Low: %A High: (256 + %A))
+; COMMON-NEXT:            Member: {%A,+,1}<nuw><%loop>
+; COMMON-EMPTY:
+; COMMON-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; COMMON-NEXT:      SCEV assumptions:
+; COMMON-EMPTY:
+; COMMON-NEXT:      Expressions re-written:
+; COMMON-NEXT:    outer.header:
+; COMMON-NEXT:      Report: loop is not the innermost loop
+; COMMON-NEXT:      Dependences:
+; COMMON-NEXT:      Run-time memory checks:
+; COMMON-NEXT:      Grouped accesses:
+; COMMON-EMPTY:
+; COMMON-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; COMMON-NEXT:      SCEV assumptions:
+; COMMON-EMPTY:
+; COMMON-NEXT:      Expressions re-written:
 ;
 entry:
   br label %outer.header
@@ -196,38 +196,64 @@ exit:
 }
 
 define void @backward_min_distance_256(ptr %A, i64 %N) {
-; CHECK-LABEL: 'backward_min_distance_256'
-; CHECK-NEXT:    loop:
-; CHECK-NEXT:      Memory dependences are safe with run-time checks
-; CHECK-NEXT:      Dependences:
-; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Check 0:
-; CHECK-NEXT:        Comparing group ([[GRP7:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
-; CHECK-NEXT:        Against group ([[GRP8:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
-; CHECK-NEXT:      Grouped accesses:
-; CHECK-NEXT:        Group [[GRP7]]:
-; CHECK-NEXT:          (Low: {(32 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(288 + %A),+,1}<nw><%outer.header>)
-; CHECK-NEXT:            Member: {{\{\{}}(32 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP8]]:
-; CHECK-NEXT:          (Low: %A High: (256 + %A))
-; CHECK-NEXT:            Member: {%A,+,1}<nuw><%loop>
-; CHECK-EMPTY:
-; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
-; CHECK-NEXT:      SCEV assumptions:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Expressions re-written:
-; CHECK-NEXT:    outer.header:
-; CHECK-NEXT:      Report: loop is not the innermost loop
-; CHECK-NEXT:      Dependences:
-; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Grouped accesses:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
-; CHECK-NEXT:      SCEV assumptions:
-; CHECK-EMPTY:
-; CHECK-NEXT:      Expressions re-written:
+; MAXLEN-LABEL: 'backward_min_distance_256'
+; MAXLEN-NEXT:    loop:
+; MAXLEN-NEXT:      Memory dependences are safe with run-time checks
+; MAXLEN-NEXT:      Dependences:
+; MAXLEN-NEXT:      Run-time memory checks:
+; MAXLEN-NEXT:      Check 0:
+; MAXLEN-NEXT:        Comparing group ([[GRP17:0x[0-9a-f]+]]):
+; MAXLEN-NEXT:          %gep.off.iv = getelementptr inbounds i8, ptr %gep.off, i64 %iv
+; MAXLEN-NEXT:        Against group ([[GRP18:0x[0-9a-f]+]]):
+; MAXLEN-NEXT:          %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+; MAXLEN-NEXT:      Grouped accesses:
+; MAXLEN-NEXT:        Group [[GRP17]]:
+; MAXLEN-NEXT:          (Low: {(32 + %A)<nuw>,+,1}<nuw><%outer.header> High: {(288 + %A),+,1}<nw><%outer.header>)
+; MAXLEN-NEXT:            Member: {{\{\{}}(32 + %A)<nuw>,+,1}<nuw><%outer.header>,+,1}<nuw><%loop>
+; MAXLEN-NEXT:        Group [[GRP18]]:
+; MAXLEN-NEXT:          (Low: %A High: (256 + %A))
+; MAXLEN-NEXT:            Member: {%A,+,1}<nuw><%loop>
+; MAXLEN-EMPTY:
+; MAXLEN-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; MAXLEN-NEXT:      SCEV assumptions:
+; MAXLEN-EMPTY:
+; MAXLEN-NEXT:      Expressions re-written:
+; MAXLEN-NEXT:    outer.header:
+; MAXLEN-NEXT:      Report: loop is not the innermost loop
+; MAXLEN-NEXT:      Dependences:
+; MAXLEN-NEXT:      Run-time memory checks:
+; MAXLEN-NEXT:      Grouped accesses:
+; MAXLEN-EMPTY:
+; MAXLEN-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; MAXLEN-NEXT:      SCEV assumptions:
+; MAXLEN-EMPTY:
+; MAXLEN-NEXT:      Expressions re-written:
+;
+; VW128-LABEL: 'backward_min_distance_256'
+; VW128-NEXT:    loop:
+; VW128-NEXT:      Memory dependences are safe with a maximum safe vector width of 256 bits
+; VW128-NEXT:      Dependences:
+; VW128-NEXT:        BackwardVectorizable:
+; VW128-NEXT:            %l = load i8, ptr %gep, align 4 ->
+; VW128-NEXT:            store i8 %add, ptr %gep.off.iv, align 4
+; VW128-EMPTY:
+; VW128-NEXT:      Run-time memory checks:
+; VW128-NEXT:      Grouped accesses:
+; VW128-EMPTY:
+; VW128-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; VW128-NEXT:      SCEV assumptions:
+; VW128-EMPTY:
+; VW128-NEXT:      Expressions re-written:
+; VW128-NEXT:    outer.header:
+; VW128-NEXT:      Report: loop is not the innermost loop
+; VW128-NEXT:      Dependences:
+; VW128-NEXT:      Run-time memory checks:
+; VW128-NEXT:      Grouped accesses:
+; VW128-EMPTY:
+; VW128-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; VW128-NEXT:      SCEV assumptions:
+; VW128-EMPTY:
+; VW128-NEXT:      Expressions re-written:
 ;
 entry:
   br label %outer.header
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
index 396919763c93..910fc24455a6 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
@@ -44,7 +44,7 @@ protected:
     AARes.reset(new AAResults(TLI));
     AARes->addAAResult(*BasicAA);
     PSE.reset(new PredicatedScalarEvolution(*SE, *L));
-    LAI.reset(new LoopAccessInfo(L, &*SE, &TLI, &*AARes, &*DT, &*LI));
+    LAI.reset(new LoopAccessInfo(L, &*SE, nullptr, &TLI, &*AARes, &*DT, &*LI));
     IAI.reset(new InterleavedAccessInfo(*PSE, L, &*DT, &*LI, &*LAI));
     IAI->analyzeInterleaving(false);
     return {Plan, *IAI};
-- 
GitLab


From fa4e8995cb6b6281a94261bee34242a39fd9f462 Mon Sep 17 00:00:00 2001
From: Tomas Matheson <tomas.matheson@arm.com>
Date: Fri, 10 May 2024 11:47:05 +0100
Subject: [PATCH 0394/1206] [AArch64] Make wfxt a full Extension (#90987)

Before #90987 WFxT did not have an AEK_WFXT, so it was assumed to be an
FMV-only extension. However it also had a SubtargetFeature. This commit
combines the two. This fixes an issue where -mattr=+wfxt was ignored,
but has the side effect of allowing +wfxt as an option to -march.
---
 llvm/lib/Target/AArch64/AArch64Features.td | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index ddc324ea14ed..755a1bdc8e23 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -90,7 +90,6 @@ def : FMVOnlyExtension<"FEAT_SVE_BF16", "sve-bf16", "+sve,+bf16,+fullfp16,+fp-ar
 def : FMVOnlyExtension<"FEAT_SVE_EBF16", "sve-ebf16", "+sve,+bf16,+fullfp16,+fp-armv8,+neon", 330>;
 def : FMVOnlyExtension<"FEAT_SVE_I8MM", "sve-i8mm", "+sve,+i8mm,+fullfp16,+fp-armv8,+neon", 340>;
 def : FMVOnlyExtension<"FEAT_SVE_PMULL128", "sve2-pmull128", "+sve2,+sve,+sve2-aes,+fullfp16,+fp-armv8,+neon", 390>;
-def : FMVOnlyExtension<"FEAT_WFXT", "wfxt", "+wfxt", 550>;
 
 
 // Each SubtargetFeature which corresponds to an Arm Architecture feature should
@@ -596,9 +595,9 @@ def FeatureMatMulFP64 : Extension<"f64mm", "MatMulFP64",
 def FeatureXS : SubtargetFeature<"xs", "HasXS",
     "true", "Enable Armv8.7-A limited-TLB-maintenance instruction (FEAT_XS)">;
 
-// FIXME link with FMVExtension?
-def FeatureWFxT : SubtargetFeature<"wfxt", "HasWFxT",
-    "true", "Enable Armv8.7-A WFET and WFIT instruction (FEAT_WFxT)">;
+def FeatureWFxT : Extension<"wfxt", "WFxT",
+    "Enable Armv8.7-A WFET and WFIT instruction (FEAT_WFxT)", [],
+    "FEAT_WFXT", "", 550>;
 
 def FeatureHCX : SubtargetFeature<
     "hcx", "HasHCX", "true", "Enable Armv8.7-A HCRX_EL2 system register (FEAT_HCX)">;
-- 
GitLab


From 2371a6410dd83e82862e1c4dce1f7411efbe6b1c Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Fri, 10 May 2024 11:57:08 +0100
Subject: [PATCH 0395/1206] [AArch64] Add intrinsics for non-widening
 FMOPA/FMOPS (#88105)

According to the specification in
https://github.com/ARM-software/acle/pull/309 this adds the intrinsics

    void svmopa_za16[_f16]_m(uint64_t tile, svbool_t pn, svbool_t pm,
                             svfloat16_t zn, svfloat16_t zm)
        __arm_streaming __arm_inout("za");
    void svmops_za16[_f16]_m(uint64_t tile, svbool_t pn,
svbool_t pm, svfloat16_t zn, svfloat16_t zm)
        __arm_streaming __arm_inout("za");

as well as the corresponding `bf16` variants.
---
 clang/include/clang/Basic/arm_sme.td          | 24 +++++
 .../acle_sme2_mopa_nonwide.c                  | 97 +++++++++++++++++++
 .../acle_sme2_mopa_nonwide.c                  | 34 +++++++
 llvm/include/llvm/IR/IntrinsicsAArch64.td     |  3 +
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td | 10 +-
 llvm/lib/Target/AArch64/SMEInstrFormats.td    | 16 ++-
 .../CodeGen/AArch64/sme2-intrinsics-mopa.ll   | 42 ++++++++
 7 files changed, 219 insertions(+), 7 deletions(-)
 create mode 100644 clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c
 create mode 100644 clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c
 create mode 100644 llvm/test/CodeGen/AArch64/sme2-intrinsics-mopa.ll

diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index 77ea53fb83fa..7808ee559932 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -708,3 +708,27 @@ let TargetGuard = "sme2" in {
   def SVLUTI2_LANE_ZT_X2 : Inst<"svluti2_lane_zt_{d}_x2", "2.di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti2_lane_zt_x2", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_7>]>;
   def SVLUTI4_LANE_ZT_X2 : Inst<"svluti4_lane_zt_{d}_x2", "2.di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti4_lane_zt_x2", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_3>]>;
 }
+
+////////////////////////////////////////////////////////////////////////////////
+// SME2p1 - FMOPA, FMOPS (non-widening)
+let TargetGuard = "sme2,b16b16" in {
+  def SVMOPA_BF16_NW : SInst<"svmopa_za16[_bf16]_m", "viPPdd", "b",
+                             MergeNone, "aarch64_sme_mopa",
+                             [IsStreaming, IsInOutZA],
+                             [ImmCheck<0, ImmCheck0_1>]>;
+  def SVMOPS_BF16_NW : SInst<"svmops_za16[_bf16]_m", "viPPdd", "b",
+                             MergeNone, "aarch64_sme_mops",
+                             [IsStreaming, IsInOutZA],
+                             [ImmCheck<0, ImmCheck0_1>]>;
+}
+
+let TargetGuard = "sme-f16f16" in {
+  def SVMOPA_F16_NW : SInst<"svmopa_za16[_f16]_m", "viPPdd", "h",
+                            MergeNone, "aarch64_sme_mopa",
+                            [IsStreaming, IsInOutZA],
+                            [ImmCheck<0, ImmCheck0_1>]>;
+  def SVMOPS_F16_NW : SInst<"svmops_za16[_f16]_m", "viPPdd", "h",
+                            MergeNone, "aarch64_sme_mops",
+                            [IsStreaming, IsInOutZA],
+                            [ImmCheck<0, ImmCheck0_1>]>;
+}
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c
new file mode 100644
index 000000000000..626bb6d3cf6f
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c
@@ -0,0 +1,97 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
+// RUN: %clang_cc1                               -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2p1 -target-feature  +b16b16 -target-feature +sme-f16f16 -O2 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK
+// RUN: %clang_cc1                        -x c++ -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2p1 -target-feature  +b16b16 -target-feature +sme-f16f16 -O2 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK-CXX
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS        -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2p1 -target-feature  +b16b16 -target-feature +sme-f16f16 -O2 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -x c++ -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2p1 -target-feature  +b16b16 -target-feature +sme-f16f16 -O2 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK-CXX
+
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -x c++ -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2p1 -target-feature  +b16b16 -target-feature +sme-f16f16 -S -O2 -Werror -o /dev/null %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+#ifdef SME_OVERLOADED_FORMS
+#define SME_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
+#else
+#define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
+#endif
+
+// CHECK-LABEL: define dso_local void @test_svmopa_za16_bf16(
+// CHECK-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv8bf16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x bfloat> [[ZN]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svmopa_za16_bf16u10__SVBool_tS_u14__SVBfloat16_tS0_(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv8bf16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x bfloat> [[ZN]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmopa_za16_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmopa_za16, _bf16, _m)(0, pn, pm, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmops_za16_bf16(
+// CHECK-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mops.nxv8bf16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x bfloat> [[ZN]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svmops_za16_bf16u10__SVBool_tS_u14__SVBfloat16_tS0_(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mops.nxv8bf16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x bfloat> [[ZN]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmops_za16_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmops_za16, _bf16, _m)(0, pn, pm, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmopa_za16_f16(
+// CHECK-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv8f16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x half> [[ZN]], <vscale x 8 x half> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmopa_za16_f16u10__SVBool_tS_u13__SVFloat16_tS0_(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv8f16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x half> [[ZN]], <vscale x 8 x half> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmopa_za16_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmopa_za16, _f16, _m)(0, pn, pm, zn, zm);
+}
+
+// CHECK-LABEL: define dso_local void @test_svmops_za16_f16(
+// CHECK-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mops.nxv8f16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x half> [[ZN]], <vscale x 8 x half> [[ZM]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmops_za16_f16u10__SVBool_tS_u13__SVFloat16_tS0_(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mops.nxv8f16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x half> [[ZN]], <vscale x 8 x half> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svmops_za16_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) __arm_streaming __arm_inout("za") {
+  SME_ACLE_FUNC(svmops_za16, _f16, _m)(0, pn, pm, zn, zm);
+}
diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c
new file mode 100644
index 000000000000..201ad4b8ff7f
--- /dev/null
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -verify -emit-llvm %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+void test_features(svbool_t pn, svbool_t pm,
+                   svfloat16_t zn, svfloat16_t zm,
+                   svbfloat16_t znb, svbfloat16_t zmb)
+  __arm_streaming __arm_inout("za") {
+// expected-error@+1 {{'svmopa_za16_bf16_m' needs target feature sme2,b16b16}}
+  svmopa_za16_bf16_m(0, pn, pm, znb, zmb);
+// expected-error@+1 {{'svmops_za16_bf16_m' needs target feature sme2,b16b16}}
+  svmops_za16_bf16_m(0, pn, pm, znb, zmb);
+// expected-error@+1 {{'svmopa_za16_f16_m' needs target feature sme-f16f16}}
+  svmopa_za16_f16_m(0, pn, pm, zn, zm);
+// expected-error@+1 {{'svmops_za16_f16_m' needs target feature sme-f16f16}}
+  svmops_za16_f16_m(0, pn, pm, zn, zm);
+}
+
+void test_imm(svbool_t pn, svbool_t pm,
+              svfloat16_t zn, svfloat16_t zm,
+              svbfloat16_t znb, svbfloat16_t zmb)
+  __arm_streaming __arm_inout("za") {
+// expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svmopa_za16_bf16_m(-1, pn, pm, znb, zmb);
+// expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svmops_za16_bf16_m(-1, pn, pm, znb, zmb);
+// expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svmopa_za16_f16_m(-1, pn, pm, zn, zm);
+// expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svmops_za16_f16_m(-1, pn, pm, zn, zm);
+}
+
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index e31e00a9c76f..e0630a6649dd 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -3649,3 +3649,6 @@ def int_aarch64_sve_pmov_to_pred_lane_zero : SVE2_1VectorArg_Pred_Intrinsic;
 def int_aarch64_sve_pmov_to_vector_lane_merging : SVE2_Pred_1VectorArgIndexed_Intrinsic;
    
 def int_aarch64_sve_pmov_to_vector_lane_zeroing : SVE2_Pred_1VectorArg_Intrinsic;
+
+def int_aarch64_sme_mopa_nonwide : SME_OuterProduct_Intrinsic;
+def int_aarch64_sme_mops_nonwide : SME_OuterProduct_Intrinsic;
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index a2e8c530c1df..c5cbdce476ca 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -815,8 +815,8 @@ defm FMLS_VG4_M4Z2Z_H : sme2_dot_mla_add_sub_array_vg4_multi<"fmls", 0b0100011,
 defm FCVT_2ZZ_H  : sme2p1_fp_cvt_vector_vg2_single<"fcvt", 0b0>;
 defm FCVTL_2ZZ_H : sme2p1_fp_cvt_vector_vg2_single<"fcvtl", 0b1>;
 
-defm FMOPA_MPPZZ_H : sme2p1_fmop_tile_fp16<"fmopa", 0b0, 0b0, 0b11, ZPR16>;
-defm FMOPS_MPPZZ_H : sme2p1_fmop_tile_fp16<"fmops", 0b0, 0b1, 0b11, ZPR16>;
+defm FMOPA_MPPZZ_H : sme2p1_fmop_tile_fp16<"fmopa", 0b0, 0b0, nxv8f16, int_aarch64_sme_mopa>;
+defm FMOPS_MPPZZ_H : sme2p1_fmop_tile_fp16<"fmops", 0b0, 0b1, nxv8f16, int_aarch64_sme_mops>;
 }
 
 let Predicates = [HasSME2, HasB16B16] in {
@@ -862,8 +862,8 @@ defm BFMINNM_VG4_4Z2Z : sme2p1_bf_max_min_vector_vg4_multi<"bfminnm",  0b0010011
 defm BFCLAMP_VG2_2ZZZ: sme2p1_bfclamp_vector_vg2_multi<"bfclamp">;
 defm BFCLAMP_VG4_4ZZZ: sme2p1_bfclamp_vector_vg4_multi<"bfclamp">;
 
-defm BFMOPA_MPPZZ_H : sme2p1_fmop_tile_fp16<"bfmopa", 0b1, 0b0, 0b11, ZPR16>;
-defm BFMOPS_MPPZZ_H : sme2p1_fmop_tile_fp16<"bfmops", 0b1, 0b1, 0b11, ZPR16>;
+defm BFMOPA_MPPZZ_H : sme2p1_fmop_tile_fp16<"bfmopa", 0b1, 0b0, nxv8bf16, int_aarch64_sme_mopa>;
+defm BFMOPS_MPPZZ_H : sme2p1_fmop_tile_fp16<"bfmops", 0b1, 0b1, nxv8bf16, int_aarch64_sme_mops>;
 }
 
 let Predicates = [HasSME2, HasFP8] in {
@@ -925,7 +925,7 @@ defm FMLAL_VG4_M4ZZ_BtoH  :  sme2_fp_mla_long_array_vg4_single<"fmlal", 0b001, M
 defm FMLAL_VG2_M2Z2Z_BtoH : sme2_fp_mla_long_array_vg2_multi<"fmlal",   0b100, MatrixOp16, ZZ_b_mul_r, nxv16i8, null_frag>;
 defm FMLAL_VG4_M4Z4Z_BtoH : sme2_fp_mla_long_array_vg4_multi<"fmlal",   0b100, MatrixOp16, ZZZZ_b_mul_r, nxv16i8, null_frag>;
 
-defm FMOPA_MPPZZ_BtoH     : sme2p1_fmop_tile_fp16<"fmopa", 0b1, 0b0, 0b01, ZPR8>;
+defm FMOPA_MPPZZ_BtoH     : sme2p1_fmop_tile_f8f16<"fmopa", 0b1, 0b0, 0b01>;
 
 } //[HasSMEF8F16]
 
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 724dd07225cd..50ee37b0dfeb 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -286,14 +286,26 @@ multiclass sme_outer_product_fp64<bit S, string mnemonic, SDPatternOperator op>
   def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_7, nxv2i1, nxv2f64>;
 }
 
-multiclass sme2p1_fmop_tile_fp16<string mnemonic, bit bf, bit s, bits<2> op, ZPRRegOp zpr_ty>{
-  def NAME : sme_fp_outer_product_inst<s, {0,bf}, op, TileOp16, zpr_ty, mnemonic> {
+multiclass sme2p1_fmop_tile_f8f16<string mnemonic, bit bf, bit s, bits<2> op> {
+  def NAME : sme_fp_outer_product_inst<s, {0,bf}, op, TileOp16, ZPR8, mnemonic> {
     bits<1> ZAda;
     let Inst{2-1} = 0b00;
     let Inst{0}   = ZAda;
   }
 }
 
+multiclass sme2p1_fmop_tile_fp16<string mnemonic, bit bf, bit s, ValueType vt, SDPatternOperator intrinsic = null_frag> {
+  def NAME : sme_fp_outer_product_inst<s, {0,bf}, 0b11, TileOp16, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+    bits<1> ZAda;
+    let Inst{2-1} = 0b00;
+    let Inst{0}   = ZAda;
+  }
+
+  def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileH>, SMEPseudo2Instr<NAME, 0>;
+
+  def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_1, nxv8i1, vt>;
+}
+
 class sme_int_outer_product_inst<bits<3> opc, bit sz, bit sme2,
                                  MatrixTileOperand za_ty, ZPRRegOp zpr_ty,
                                  string mnemonic>
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-mopa.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-mopa.ll
new file mode 100644
index 000000000000..fa0fd4360702
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-mopa.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+
+target triple = "aarch64-linux"
+
+define void @mopa_bf16(<vscale x 8 x i1> %pn, <vscale x 8 x i1> %pm, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) #0 {
+; CHECK-LABEL: mopa_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bfmopa za0.h, p0/m, p1/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.mopa.nxv8bf16(i32 0, <vscale x 8 x i1> %pn, <vscale x 8 x i1> %pm, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm)
+  ret void
+}
+
+define void @mopa_f16(<vscale x 8 x i1> %pn, <vscale x 8 x i1> %pm, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm) #0 {
+; CHECK-LABEL: mopa_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmopa za1.h, p0/m, p1/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.mopa.nxv8f16(i32 1, <vscale x 8 x i1> %pn, <vscale x 8 x i1> %pm, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm)
+  ret void
+}
+
+define void @mops_bf16(<vscale x 8 x i1> %pn, <vscale x 8 x i1> %pm, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) #0 {
+; CHECK-LABEL: mops_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bfmops za0.h, p0/m, p1/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.mops.nxv8bf16(i32 0, <vscale x 8 x i1> %pn, <vscale x 8 x i1> %pm, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm)
+  ret void
+}
+
+define void @mops_f16(<vscale x 8 x i1> %pn, <vscale x 8 x i1> %pm, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm) #0 {
+; CHECK-LABEL: mops_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmops za1.h, p0/m, p1/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.mops.nxv8f16(i32 1, <vscale x 8 x i1> %pn, <vscale x 8 x i1> %pm, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm)
+  ret void
+}
+
+attributes #0 = {nounwind "target-features" = "+sme,+sme2p1,+bf16,+sme-f16f16,+b16b16" }
-- 
GitLab


From f5e49279c01436971001e107a0a3435510b9ae98 Mon Sep 17 00:00:00 2001
From: Petar Avramovic <Petar.Avramovic@amd.com>
Date: Fri, 10 May 2024 13:02:05 +0200
Subject: [PATCH 0396/1206] AMDGPU: fix isSafeToSink expecting exactly one
 predecessor (#89224)

isSafeToSink needs to check if machine cycle has divergent exit branch
but first it needs the MBB that contains cycle exit branch.
Early-tailduplication can delete exit block created by structurize-cfg
so there is still exactly one cycle exit block but the new cycle exit
block can have multiple predecessors.
Simplify search for MBBs that contain cycle exit branch by introducing
helper method getExitingBlocks in GenericCycle.

Fixes #89200
---
 llvm/include/llvm/ADT/GenericCycleImpl.h      | 15 +++
 llvm/include/llvm/ADT/GenericCycleInfo.h      |  4 +
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        | 12 +--
 .../AMDGPU/GlobalISel/is-safe-to-sink-bug.ll  | 93 +++++++++++++++++++
 4 files changed, 117 insertions(+), 7 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/is-safe-to-sink-bug.ll

diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index 74faff98b903..ab9c421a4469 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -66,6 +66,21 @@ void GenericCycle<ContextT>::getExitBlocks(
   }
 }
 
+template <typename ContextT>
+void GenericCycle<ContextT>::getExitingBlocks(
+    SmallVectorImpl<BlockT *> &TmpStorage) const {
+  TmpStorage.clear();
+
+  for (BlockT *Block : blocks()) {
+    for (BlockT *Succ : successors(Block)) {
+      if (!contains(Succ)) {
+        TmpStorage.push_back(Block);
+        break;
+      }
+    }
+  }
+}
+
 template <typename ContextT>
 auto GenericCycle<ContextT>::getCyclePreheader() const -> BlockT * {
   BlockT *Predecessor = getCyclePredecessor();
diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h
index 83c4c2759d46..b601fc9bae38 100644
--- a/llvm/include/llvm/ADT/GenericCycleInfo.h
+++ b/llvm/include/llvm/ADT/GenericCycleInfo.h
@@ -126,6 +126,10 @@ public:
   /// branched to.
   void getExitBlocks(SmallVectorImpl<BlockT *> &TmpStorage) const;
 
+  /// Return all blocks of this cycle that have successor outside of this cycle.
+  /// These blocks have cycle exit branch.
+  void getExitingBlocks(SmallVectorImpl<BlockT *> &TmpStorage) const;
+
   /// Return the preheader block for this cycle. Pre-header is well-defined for
   /// reducible cycle in docs/LoopTerminology.rst as: the only one entering
   /// block and its only edge is to the entry block. Return null for irreducible
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 6599d0abd135..08351c49b223 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -213,15 +213,13 @@ bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
       // Check if there is a FromCycle that contains SgprDef's basic block but
       // does not contain SuccToSinkTo and also has divergent exit condition.
       while (FromCycle && !FromCycle->contains(ToCycle)) {
-        // After structurize-cfg, there should be exactly one cycle exit.
-        SmallVector<MachineBasicBlock *, 1> ExitBlocks;
-        FromCycle->getExitBlocks(ExitBlocks);
-        assert(ExitBlocks.size() == 1);
-        assert(ExitBlocks[0]->getSinglePredecessor());
+        SmallVector<MachineBasicBlock *, 1> ExitingBlocks;
+        FromCycle->getExitingBlocks(ExitingBlocks);
 
         // FromCycle has divergent exit condition.
-        if (hasDivergentBranch(ExitBlocks[0]->getSinglePredecessor())) {
-          return false;
+        for (MachineBasicBlock *ExitingBlock : ExitingBlocks) {
+          if (hasDivergentBranch(ExitingBlock))
+            return false;
         }
 
         FromCycle = FromCycle->getParentCycle();
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/is-safe-to-sink-bug.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/is-safe-to-sink-bug.ll
new file mode 100644
index 000000000000..d3bc661f5940
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/is-safe-to-sink-bug.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -global-isel -verify-machineinstrs < %s | FileCheck %s
+
+; early-tailduplication deletes cycle exit block created by structurize-cfg
+; that had exactly one predecessor. Now, new cycle exit block has two
+; predecessors, we need to find predecessor that belongs to the cycle.
+
+define amdgpu_ps void @_amdgpu_ps_main(i1 %arg) {
+; CHECK-LABEL: _amdgpu_ps_main:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_mov_b32 s4, 0
+; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
+; CHECK-NEXT:    s_mov_b32 s5, s4
+; CHECK-NEXT:    s_mov_b32 s6, s4
+; CHECK-NEXT:    s_mov_b32 s7, s4
+; CHECK-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; CHECK-NEXT:    s_buffer_load_dword s1, s[4:7], 0x0
+; CHECK-NEXT:    s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; CHECK-NEXT:    s_mov_b32 s10, -1
+; CHECK-NEXT:    s_mov_b32 s11, 0x31c16000
+; CHECK-NEXT:    s_add_u32 s8, s8, s0
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s0, 0, v0
+; CHECK-NEXT:    s_addc_u32 s9, s9, 0
+; CHECK-NEXT:    s_mov_b32 s32, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_cmp_ge_i32 s1, 0
+; CHECK-NEXT:    s_cbranch_scc0 .LBB0_2
+; CHECK-NEXT:  .LBB0_1: ; %bb12
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, 1.0, 0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    s_mov_b64 s[0:1], s[8:9]
+; CHECK-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; CHECK-NEXT:    s_swappc_b64 s[30:31], 0
+; CHECK-NEXT:  .LBB0_2: ; %bb2.preheader
+; CHECK-NEXT:    s_mov_b32 s1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, s1
+; CHECK-NEXT:    s_branch .LBB0_4
+; CHECK-NEXT:    .p2align 6
+; CHECK-NEXT:  .LBB0_3: ; %bb6
+; CHECK-NEXT:    ; in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s3
+; CHECK-NEXT:    s_and_b32 s2, 1, s2
+; CHECK-NEXT:    v_or_b32_e32 v1, 1, v0
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s2, 0, s2
+; CHECK-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v0
+; CHECK-NEXT:    v_mov_b32_e32 v0, v1
+; CHECK-NEXT:    s_and_b32 s4, s2, s1
+; CHECK-NEXT:    s_andn2_b32 s1, s1, exec_lo
+; CHECK-NEXT:    s_and_b32 s2, exec_lo, s4
+; CHECK-NEXT:    s_or_b32 s1, s1, s2
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_1
+; CHECK-NEXT:  .LBB0_4: ; %bb2
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_mov_b32 s2, 0
+; CHECK-NEXT:    s_and_saveexec_b32 s3, s0
+; CHECK-NEXT:    s_cbranch_execz .LBB0_3
+; CHECK-NEXT:  ; %bb.5: ; %bb5
+; CHECK-NEXT:    ; in Loop: Header=BB0_4 Depth=1
+; CHECK-NEXT:    s_mov_b32 s2, 1
+; CHECK-NEXT:    s_branch .LBB0_3
+bb:
+  %i = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> zeroinitializer, i32 0, i32 0)
+  %i1 = icmp slt i32 %i, 0
+  br i1 %i1, label %bb2, label %bb12
+
+bb2:
+  %i3 = phi i1 [ %i9, %bb6 ], [ false, %bb ]
+  %i4 = phi i32 [ %i10, %bb6 ], [ 0, %bb ]
+  br i1 %arg, label %bb5, label %bb6
+
+bb5:
+  br label %bb6
+
+bb6:
+  %i7 = phi i32 [ 0, %bb2 ], [ 1, %bb5 ]
+  %i8 = icmp ne i32 %i7, 0
+  %i9 = select i1 %i8, i1 %i3, i1 false
+  %i10 = or i32 %i4, 1
+  %i11 = icmp slt i32 %i4, 0
+  br i1 %i11, label %bb2, label %bb12
+
+bb12:
+  %i13 = phi i1 [ false, %bb ], [ %i9, %bb6 ]
+  %i14 = select i1 %i13, float 0.000000e+00, float 1.000000e+00
+  %i15 = insertelement <4 x float> zeroinitializer, float %i14, i64 0
+  call amdgpu_gfx addrspace(4) void null(<4 x float> %i15, i32 0)
+  unreachable
+}
+
+declare i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32>, i32, i32 immarg)
-- 
GitLab


From 73681b8fee930274e6dc11d4471b44666d6d0dfd Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Fri, 10 May 2024 12:12:44 +0100
Subject: [PATCH 0397/1206] [NFC][LLVM] Simplify SVE isel DAG patterns.
 (#91510)

We have many instances of (Ty ZPR:$op) than can be written as Ty:$Op.
Whilst other operands can also be simplified this patch focuses on
removing redundant instances of PPR, PNR and ZPR only.
---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 624 +++++++++---------
 1 file changed, 311 insertions(+), 313 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 64e545aa26b4..d4405a230613 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -1269,33 +1269,33 @@ let Predicates = [HasSVE] in {
 
   multiclass sve_masked_gather_x2_scaled<ValueType Ty, SDPatternOperator Load, string Inst> {
     // base + vector of scaled offsets
-    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (nxv2i64 ZPR:$offs))),
+    def : Pat<(Ty (Load (SVEDup0Undef), nxv2i1:$gp, GPR64:$base, nxv2i64:$offs)),
               (!cast<Instruction>(Inst # _SCALED) PPR:$gp, GPR64:$base, ZPR:$offs)>;
     // base + vector of signed 32bit scaled offsets
-    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (sext_inreg (nxv2i64 ZPR:$offs), nxv2i32))),
+    def : Pat<(Ty (Load (SVEDup0Undef), nxv2i1:$gp, GPR64:$base, (sext_inreg nxv2i64:$offs, nxv2i32))),
               (!cast<Instruction>(Inst # _SXTW_SCALED) PPR:$gp, GPR64:$base, ZPR:$offs)>;
     // base + vector of unsigned 32bit scaled offsets
-    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (and (nxv2i64 ZPR:$offs), (nxv2i64 (splat_vector (i64 0xFFFFFFFF)))))),
+    def : Pat<(Ty (Load (SVEDup0Undef), nxv2i1:$gp, GPR64:$base, (and nxv2i64:$offs, (nxv2i64 (splat_vector (i64 0xFFFFFFFF)))))),
               (!cast<Instruction>(Inst # _UXTW_SCALED) PPR:$gp, GPR64:$base, ZPR:$offs)>;
   }
 
   multiclass sve_masked_gather_x2_unscaled<ValueType Ty, SDPatternOperator Load, string Inst, Operand ImmTy> {
     // vector of pointers + immediate offset (includes zero)
-    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), (i64 ImmTy:$imm), (nxv2i64 ZPR:$ptrs))),
+    def : Pat<(Ty (Load (SVEDup0Undef), nxv2i1:$gp, (i64 ImmTy:$imm), nxv2i64:$ptrs)),
               (!cast<Instruction>(Inst # _IMM) PPR:$gp, ZPR:$ptrs, ImmTy:$imm)>;
     // base + vector of offsets
-    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (nxv2i64 ZPR:$offs))),
+    def : Pat<(Ty (Load (SVEDup0Undef), nxv2i1:$gp, GPR64:$base, nxv2i64:$offs)),
               (!cast<Instruction>(Inst) PPR:$gp, GPR64:$base, ZPR:$offs)>;
     // base + vector of signed 32bit offsets
-    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (sext_inreg (nxv2i64 ZPR:$offs), nxv2i32))),
+    def : Pat<(Ty (Load (SVEDup0Undef), nxv2i1:$gp, GPR64:$base, (sext_inreg nxv2i64:$offs, nxv2i32))),
               (!cast<Instruction>(Inst # _SXTW) PPR:$gp, GPR64:$base, ZPR:$offs)>;
     // base + vector of unsigned 32bit offsets
-    def : Pat<(Ty (Load (SVEDup0Undef), (nxv2i1 PPR:$gp), GPR64:$base, (and (nxv2i64 ZPR:$offs), (nxv2i64 (splat_vector (i64 0xFFFFFFFF)))))),
+    def : Pat<(Ty (Load (SVEDup0Undef), nxv2i1:$gp, GPR64:$base, (and nxv2i64:$offs, (nxv2i64 (splat_vector (i64 0xFFFFFFFF)))))),
               (!cast<Instruction>(Inst # _UXTW) PPR:$gp, GPR64:$base, ZPR:$offs)>;
   }
 
   multiclass sve_masked_gather_x4<ValueType Ty, SDPatternOperator Load, Instruction Inst> {
-    def : Pat<(Ty (Load (SVEDup0Undef), (nxv4i1 PPR:$gp), GPR64:$base, (nxv4i32 ZPR:$offs))),
+    def : Pat<(Ty (Load (SVEDup0Undef), nxv4i1:$gp, GPR64:$base, nxv4i32:$offs)),
               (Inst PPR:$gp, GPR64:$base, ZPR:$offs)>;
   }
 
@@ -1503,33 +1503,33 @@ let Predicates = [HasSVE] in {
 
   multiclass sve_masked_scatter_x2_scaled<ValueType Ty, SDPatternOperator Store, string Inst> {
     // base + vector of scaled offsets
-    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (nxv2i64 ZPR:$offs)),
+    def : Pat<(Store Ty:$data, nxv2i1:$gp, GPR64:$base, nxv2i64:$offs),
               (!cast<Instruction>(Inst # _SCALED) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
     // base + vector of signed 32bit scaled offsets
-    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (sext_inreg (nxv2i64 ZPR:$offs), nxv2i32)),
+    def : Pat<(Store Ty:$data, nxv2i1:$gp, GPR64:$base, (sext_inreg nxv2i64:$offs, nxv2i32)),
               (!cast<Instruction>(Inst # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
     // base + vector of unsigned 32bit scaled offsets
-    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (and (nxv2i64 ZPR:$offs), (nxv2i64 (splat_vector (i64 0xFFFFFFFF))))),
+    def : Pat<(Store Ty:$data, nxv2i1:$gp, GPR64:$base, (and nxv2i64:$offs, (nxv2i64 (splat_vector (i64 0xFFFFFFFF))))),
               (!cast<Instruction>(Inst # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
   }
 
   multiclass sve_masked_scatter_x2_unscaled<ValueType Ty, SDPatternOperator Store, string Inst, Operand ImmTy> {
     // vector of pointers + immediate offset (includes zero)
-    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), (i64 ImmTy:$imm), (nxv2i64 ZPR:$ptrs)),
+    def : Pat<(Store Ty:$data, nxv2i1:$gp, (i64 ImmTy:$imm), nxv2i64:$ptrs),
               (!cast<Instruction>(Inst # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, ImmTy:$imm)>;
     // base + vector of offsets
-    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (nxv2i64 ZPR:$offs)),
+    def : Pat<(Store Ty:$data, nxv2i1:$gp, GPR64:$base, nxv2i64:$offs),
               (!cast<Instruction>(Inst) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
     // base + vector of signed 32bit offsets
-    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (sext_inreg (nxv2i64 ZPR:$offs), nxv2i32)),
+    def : Pat<(Store Ty:$data, nxv2i1:$gp, GPR64:$base, (sext_inreg nxv2i64:$offs, nxv2i32)),
               (!cast<Instruction>(Inst # _SXTW) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
     // base + vector of unsigned 32bit offsets
-    def : Pat<(Store (Ty ZPR:$data), (nxv2i1 PPR:$gp), GPR64:$base, (and (nxv2i64 ZPR:$offs), (nxv2i64 (splat_vector (i64 0xFFFFFFFF))))),
+    def : Pat<(Store Ty:$data, nxv2i1:$gp, GPR64:$base, (and nxv2i64:$offs, (nxv2i64 (splat_vector (i64 0xFFFFFFFF))))),
               (!cast<Instruction>(Inst # _UXTW) ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
   }
 
   multiclass sve_masked_scatter_x4<ValueType Ty, SDPatternOperator Store, Instruction Inst> {
-    def : Pat<(Store (Ty ZPR:$data), (nxv4i1 PPR:$gp), GPR64:$base, (nxv4i32 ZPR:$offs)),
+    def : Pat<(Store Ty:$data, nxv4i1:$gp, GPR64:$base, nxv4i32:$offs),
               (Inst ZPR:$data, PPR:$gp, GPR64:$base, ZPR:$offs)>;
   }
 
@@ -1791,159 +1791,159 @@ let Predicates = [HasSVEorSME] in {
   defm TRN2_PPP : sve_int_perm_bin_perm_pp<0b101, "trn2", AArch64trn2, int_aarch64_sve_trn2_b16, int_aarch64_sve_trn2_b32, int_aarch64_sve_trn2_b64>;
 
   // Extract lo/hi halves of legal predicate types.
-  def : Pat<(nxv1i1 (extract_subvector (nxv2i1 PPR:$Ps), (i64 0))),
+  def : Pat<(nxv1i1 (extract_subvector nxv2i1:$Ps, (i64 0))),
             (PUNPKLO_PP PPR:$Ps)>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv2i1 PPR:$Ps), (i64 1))),
+  def : Pat<(nxv1i1 (extract_subvector nxv2i1:$Ps, (i64 1))),
             (PUNPKHI_PP PPR:$Ps)>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv4i1 PPR:$Ps), (i64 0))),
+  def : Pat<(nxv2i1 (extract_subvector nxv4i1:$Ps, (i64 0))),
             (PUNPKLO_PP PPR:$Ps)>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv4i1 PPR:$Ps), (i64 2))),
+  def : Pat<(nxv2i1 (extract_subvector nxv4i1:$Ps, (i64 2))),
             (PUNPKHI_PP PPR:$Ps)>;
-  def : Pat<(nxv4i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 0))),
+  def : Pat<(nxv4i1 (extract_subvector nxv8i1:$Ps, (i64 0))),
             (PUNPKLO_PP PPR:$Ps)>;
-  def : Pat<(nxv4i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 4))),
+  def : Pat<(nxv4i1 (extract_subvector nxv8i1:$Ps, (i64 4))),
             (PUNPKHI_PP PPR:$Ps)>;
-  def : Pat<(nxv8i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 0))),
+  def : Pat<(nxv8i1 (extract_subvector nxv16i1:$Ps, (i64 0))),
             (PUNPKLO_PP PPR:$Ps)>;
-  def : Pat<(nxv8i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 8))),
+  def : Pat<(nxv8i1 (extract_subvector nxv16i1:$Ps, (i64 8))),
             (PUNPKHI_PP PPR:$Ps)>;
 
-  def : Pat<(nxv1i1 (extract_subvector (nxv4i1 PPR:$Ps), (i64 0))),
+  def : Pat<(nxv1i1 (extract_subvector nxv4i1:$Ps, (i64 0))),
             (PUNPKLO_PP (PUNPKLO_PP PPR:$Ps))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv4i1 PPR:$Ps), (i64 1))),
+  def : Pat<(nxv1i1 (extract_subvector nxv4i1:$Ps, (i64 1))),
             (PUNPKHI_PP (PUNPKLO_PP PPR:$Ps))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv4i1 PPR:$Ps), (i64 2))),
+  def : Pat<(nxv1i1 (extract_subvector nxv4i1:$Ps, (i64 2))),
             (PUNPKLO_PP (PUNPKHI_PP PPR:$Ps))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv4i1 PPR:$Ps), (i64 3))),
+  def : Pat<(nxv1i1 (extract_subvector nxv4i1:$Ps, (i64 3))),
             (PUNPKHI_PP (PUNPKHI_PP PPR:$Ps))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 0))),
+  def : Pat<(nxv2i1 (extract_subvector nxv8i1:$Ps, (i64 0))),
             (PUNPKLO_PP (PUNPKLO_PP PPR:$Ps))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 2))),
+  def : Pat<(nxv2i1 (extract_subvector nxv8i1:$Ps, (i64 2))),
             (PUNPKHI_PP (PUNPKLO_PP PPR:$Ps))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 4))),
+  def : Pat<(nxv2i1 (extract_subvector nxv8i1:$Ps, (i64 4))),
             (PUNPKLO_PP (PUNPKHI_PP PPR:$Ps))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 6))),
+  def : Pat<(nxv2i1 (extract_subvector nxv8i1:$Ps, (i64 6))),
             (PUNPKHI_PP (PUNPKHI_PP PPR:$Ps))>;
-  def : Pat<(nxv4i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 0))),
+  def : Pat<(nxv4i1 (extract_subvector nxv16i1:$Ps, (i64 0))),
             (PUNPKLO_PP (PUNPKLO_PP PPR:$Ps))>;
-  def : Pat<(nxv4i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 4))),
+  def : Pat<(nxv4i1 (extract_subvector nxv16i1:$Ps, (i64 4))),
             (PUNPKHI_PP (PUNPKLO_PP PPR:$Ps))>;
-  def : Pat<(nxv4i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 8))),
+  def : Pat<(nxv4i1 (extract_subvector nxv16i1:$Ps, (i64 8))),
             (PUNPKLO_PP (PUNPKHI_PP PPR:$Ps))>;
-  def : Pat<(nxv4i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 12))),
+  def : Pat<(nxv4i1 (extract_subvector nxv16i1:$Ps, (i64 12))),
             (PUNPKHI_PP (PUNPKHI_PP PPR:$Ps))>;
 
 
-  def : Pat<(nxv1i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 0))),
+  def : Pat<(nxv1i1 (extract_subvector nxv8i1:$Ps, (i64 0))),
             (PUNPKLO_PP (PUNPKLO_PP (PUNPKLO_PP PPR:$Ps)))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 1))),
+  def : Pat<(nxv1i1 (extract_subvector nxv8i1:$Ps, (i64 1))),
             (PUNPKHI_PP (PUNPKLO_PP (PUNPKLO_PP PPR:$Ps)))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 2))),
+  def : Pat<(nxv1i1 (extract_subvector nxv8i1:$Ps, (i64 2))),
             (PUNPKLO_PP (PUNPKHI_PP (PUNPKLO_PP PPR:$Ps)))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 3))),
+  def : Pat<(nxv1i1 (extract_subvector nxv8i1:$Ps, (i64 3))),
             (PUNPKHI_PP (PUNPKHI_PP (PUNPKLO_PP PPR:$Ps)))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 4))),
+  def : Pat<(nxv1i1 (extract_subvector nxv8i1:$Ps, (i64 4))),
             (PUNPKLO_PP (PUNPKLO_PP (PUNPKHI_PP PPR:$Ps)))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 5))),
+  def : Pat<(nxv1i1 (extract_subvector nxv8i1:$Ps, (i64 5))),
             (PUNPKHI_PP (PUNPKLO_PP (PUNPKHI_PP PPR:$Ps)))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 6))),
+  def : Pat<(nxv1i1 (extract_subvector nxv8i1:$Ps, (i64 6))),
             (PUNPKLO_PP (PUNPKHI_PP (PUNPKHI_PP PPR:$Ps)))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv8i1 PPR:$Ps), (i64 7))),
+  def : Pat<(nxv1i1 (extract_subvector nxv8i1:$Ps, (i64 7))),
             (PUNPKHI_PP (PUNPKHI_PP (PUNPKHI_PP PPR:$Ps)))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 0))),
+  def : Pat<(nxv2i1 (extract_subvector nxv16i1:$Ps, (i64 0))),
             (PUNPKLO_PP (PUNPKLO_PP (PUNPKLO_PP PPR:$Ps)))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 2))),
+  def : Pat<(nxv2i1 (extract_subvector nxv16i1:$Ps, (i64 2))),
             (PUNPKHI_PP (PUNPKLO_PP (PUNPKLO_PP PPR:$Ps)))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 4))),
+  def : Pat<(nxv2i1 (extract_subvector nxv16i1:$Ps, (i64 4))),
             (PUNPKLO_PP (PUNPKHI_PP (PUNPKLO_PP PPR:$Ps)))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 6))),
+  def : Pat<(nxv2i1 (extract_subvector nxv16i1:$Ps, (i64 6))),
             (PUNPKHI_PP (PUNPKHI_PP (PUNPKLO_PP PPR:$Ps)))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 8))),
+  def : Pat<(nxv2i1 (extract_subvector nxv16i1:$Ps, (i64 8))),
             (PUNPKLO_PP (PUNPKLO_PP (PUNPKHI_PP PPR:$Ps)))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 10))),
+  def : Pat<(nxv2i1 (extract_subvector nxv16i1:$Ps, (i64 10))),
             (PUNPKHI_PP (PUNPKLO_PP (PUNPKHI_PP PPR:$Ps)))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 12))),
+  def : Pat<(nxv2i1 (extract_subvector nxv16i1:$Ps, (i64 12))),
             (PUNPKLO_PP (PUNPKHI_PP (PUNPKHI_PP PPR:$Ps)))>;
-  def : Pat<(nxv2i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 14))),
+  def : Pat<(nxv2i1 (extract_subvector nxv16i1:$Ps, (i64 14))),
             (PUNPKHI_PP (PUNPKHI_PP (PUNPKHI_PP PPR:$Ps)))>;
 
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 0))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 0))),
             (PUNPKLO_PP (PUNPKLO_PP (PUNPKLO_PP  (PUNPKLO_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 1))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 1))),
             (PUNPKHI_PP (PUNPKLO_PP (PUNPKLO_PP  (PUNPKLO_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 2))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 2))),
             (PUNPKLO_PP (PUNPKHI_PP (PUNPKLO_PP  (PUNPKLO_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 3))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 3))),
             (PUNPKHI_PP (PUNPKHI_PP (PUNPKLO_PP  (PUNPKLO_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 4))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 4))),
             (PUNPKLO_PP (PUNPKLO_PP (PUNPKHI_PP  (PUNPKLO_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 5))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 5))),
             (PUNPKHI_PP (PUNPKLO_PP (PUNPKHI_PP  (PUNPKLO_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 6))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 6))),
             (PUNPKLO_PP (PUNPKHI_PP (PUNPKHI_PP  (PUNPKLO_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 7))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 7))),
             (PUNPKHI_PP (PUNPKHI_PP (PUNPKHI_PP  (PUNPKLO_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 8))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 8))),
             (PUNPKLO_PP (PUNPKLO_PP (PUNPKLO_PP  (PUNPKHI_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 9))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 9))),
             (PUNPKHI_PP (PUNPKLO_PP (PUNPKLO_PP  (PUNPKHI_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 10))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 10))),
             (PUNPKLO_PP (PUNPKHI_PP (PUNPKLO_PP  (PUNPKHI_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 11))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 11))),
             (PUNPKHI_PP (PUNPKHI_PP (PUNPKLO_PP  (PUNPKHI_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 12))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 12))),
             (PUNPKLO_PP (PUNPKLO_PP (PUNPKHI_PP  (PUNPKHI_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 13))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 13))),
             (PUNPKHI_PP (PUNPKLO_PP (PUNPKHI_PP  (PUNPKHI_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 14))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 14))),
             (PUNPKLO_PP (PUNPKHI_PP (PUNPKHI_PP  (PUNPKHI_PP PPR:$Ps))))>;
-  def : Pat<(nxv1i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 15))),
+  def : Pat<(nxv1i1 (extract_subvector nxv16i1:$Ps, (i64 15))),
             (PUNPKHI_PP (PUNPKHI_PP (PUNPKHI_PP  (PUNPKHI_PP PPR:$Ps))))>;
 
   // Extract subvectors from FP SVE vectors
-  def : Pat<(nxv2f16 (extract_subvector (nxv4f16 ZPR:$Zs), (i64 0))),
+  def : Pat<(nxv2f16 (extract_subvector nxv4f16:$Zs, (i64 0))),
             (UUNPKLO_ZZ_D ZPR:$Zs)>;
-  def : Pat<(nxv2f16 (extract_subvector (nxv4f16 ZPR:$Zs), (i64 2))),
+  def : Pat<(nxv2f16 (extract_subvector nxv4f16:$Zs, (i64 2))),
             (UUNPKHI_ZZ_D ZPR:$Zs)>;
-  def : Pat<(nxv4f16 (extract_subvector (nxv8f16 ZPR:$Zs), (i64 0))),
+  def : Pat<(nxv4f16 (extract_subvector nxv8f16:$Zs, (i64 0))),
             (UUNPKLO_ZZ_S ZPR:$Zs)>;
-  def : Pat<(nxv4f16 (extract_subvector (nxv8f16 ZPR:$Zs), (i64 4))),
+  def : Pat<(nxv4f16 (extract_subvector nxv8f16:$Zs, (i64 4))),
             (UUNPKHI_ZZ_S ZPR:$Zs)>;
-  def : Pat<(nxv2f32 (extract_subvector (nxv4f32 ZPR:$Zs), (i64 0))),
+  def : Pat<(nxv2f32 (extract_subvector nxv4f32:$Zs, (i64 0))),
             (UUNPKLO_ZZ_D ZPR:$Zs)>;
-  def : Pat<(nxv2f32 (extract_subvector (nxv4f32 ZPR:$Zs), (i64 2))),
+  def : Pat<(nxv2f32 (extract_subvector nxv4f32:$Zs, (i64 2))),
             (UUNPKHI_ZZ_D ZPR:$Zs)>;
 
-  def : Pat<(nxv2bf16 (extract_subvector (nxv4bf16 ZPR:$Zs), (i64 0))),
+  def : Pat<(nxv2bf16 (extract_subvector nxv4bf16:$Zs, (i64 0))),
             (UUNPKLO_ZZ_D ZPR:$Zs)>;
-  def : Pat<(nxv2bf16 (extract_subvector (nxv4bf16 ZPR:$Zs), (i64 2))),
+  def : Pat<(nxv2bf16 (extract_subvector nxv4bf16:$Zs, (i64 2))),
             (UUNPKHI_ZZ_D ZPR:$Zs)>;
-  def : Pat<(nxv4bf16 (extract_subvector (nxv8bf16 ZPR:$Zs), (i64 0))),
+  def : Pat<(nxv4bf16 (extract_subvector nxv8bf16:$Zs, (i64 0))),
             (UUNPKLO_ZZ_S ZPR:$Zs)>;
-  def : Pat<(nxv4bf16 (extract_subvector (nxv8bf16 ZPR:$Zs), (i64 4))),
+  def : Pat<(nxv4bf16 (extract_subvector nxv8bf16:$Zs, (i64 4))),
             (UUNPKHI_ZZ_S ZPR:$Zs)>;
 
-  def : Pat<(nxv2f16 (extract_subvector (nxv8f16 ZPR:$Zs), (i64 0))),
+  def : Pat<(nxv2f16 (extract_subvector nxv8f16:$Zs, (i64 0))),
             (UUNPKLO_ZZ_D (UUNPKLO_ZZ_S ZPR:$Zs))>;
-  def : Pat<(nxv2f16 (extract_subvector (nxv8f16 ZPR:$Zs), (i64 2))),
+  def : Pat<(nxv2f16 (extract_subvector nxv8f16:$Zs, (i64 2))),
             (UUNPKHI_ZZ_D (UUNPKLO_ZZ_S ZPR:$Zs))>;
-  def : Pat<(nxv2f16 (extract_subvector (nxv8f16 ZPR:$Zs), (i64 4))),
+  def : Pat<(nxv2f16 (extract_subvector nxv8f16:$Zs, (i64 4))),
             (UUNPKLO_ZZ_D (UUNPKHI_ZZ_S ZPR:$Zs))>;
-  def : Pat<(nxv2f16 (extract_subvector (nxv8f16 ZPR:$Zs), (i64 6))),
+  def : Pat<(nxv2f16 (extract_subvector nxv8f16:$Zs, (i64 6))),
             (UUNPKHI_ZZ_D (UUNPKHI_ZZ_S ZPR:$Zs))>;
 
-  def : Pat<(nxv2bf16 (extract_subvector (nxv8bf16 ZPR:$Zs), (i64 0))),
+  def : Pat<(nxv2bf16 (extract_subvector nxv8bf16:$Zs, (i64 0))),
             (UUNPKLO_ZZ_D (UUNPKLO_ZZ_S ZPR:$Zs))>;
-  def : Pat<(nxv2bf16 (extract_subvector (nxv8bf16 ZPR:$Zs), (i64 2))),
+  def : Pat<(nxv2bf16 (extract_subvector nxv8bf16:$Zs, (i64 2))),
             (UUNPKHI_ZZ_D (UUNPKLO_ZZ_S ZPR:$Zs))>;
-  def : Pat<(nxv2bf16 (extract_subvector (nxv8bf16 ZPR:$Zs), (i64 4))),
+  def : Pat<(nxv2bf16 (extract_subvector nxv8bf16:$Zs, (i64 4))),
             (UUNPKLO_ZZ_D (UUNPKHI_ZZ_S ZPR:$Zs))>;
-  def : Pat<(nxv2bf16 (extract_subvector (nxv8bf16 ZPR:$Zs), (i64 6))),
+  def : Pat<(nxv2bf16 (extract_subvector nxv8bf16:$Zs, (i64 6))),
             (UUNPKHI_ZZ_D (UUNPKHI_ZZ_S ZPR:$Zs))>;
 
   // extract/insert 64-bit fixed length vector from/into a scalable vector
   foreach VT = [v8i8, v4i16, v2i32, v1i64, v4f16, v2f32, v1f64, v4bf16] in {
-    def : Pat<(VT (vector_extract_subvec (SVEContainerVT<VT>.Value ZPR:$Zs), (i64 0))),
+    def : Pat<(VT (vector_extract_subvec SVEContainerVT<VT>.Value:$Zs, (i64 0))),
               (EXTRACT_SUBREG ZPR:$Zs, dsub)>;
     def : Pat<(SVEContainerVT<VT>.Value (vector_insert_subvec undef, (VT V64:$src), (i64 0))),
               (INSERT_SUBREG (IMPLICIT_DEF), $src, dsub)>;
@@ -1951,7 +1951,7 @@ let Predicates = [HasSVEorSME] in {
 
   // extract/insert 128-bit fixed length vector from/into a scalable vector
   foreach VT = [v16i8, v8i16, v4i32, v2i64, v8f16, v4f32, v2f64, v8bf16] in {
-    def : Pat<(VT (vector_extract_subvec (SVEContainerVT<VT>.Value ZPR:$Zs), (i64 0))),
+    def : Pat<(VT (vector_extract_subvec SVEContainerVT<VT>.Value:$Zs, (i64 0))),
               (EXTRACT_SUBREG ZPR:$Zs, zsub)>;
     def : Pat<(SVEContainerVT<VT>.Value (vector_insert_subvec undef, (VT V128:$src), (i64 0))),
               (INSERT_SUBREG (IMPLICIT_DEF), $src, zsub)>;
@@ -1980,34 +1980,34 @@ let Predicates = [HasSVEorSME] in {
             (UZP1_ZZZ_H $v1, $v2)>;
 
   // Splice with lane equal to -1
-  def : Pat<(nxv16i8 (vector_splice (nxv16i8 ZPR:$Z1), (nxv16i8 ZPR:$Z2), (i64 -1))),
+  def : Pat<(nxv16i8 (vector_splice nxv16i8:$Z1, nxv16i8:$Z2, (i64 -1))),
             (INSR_ZV_B ZPR:$Z2, (INSERT_SUBREG (IMPLICIT_DEF),
             (LASTB_VPZ_B (PTRUE_B 31), ZPR:$Z1), bsub))>;
-  def : Pat<(nxv8i16 (vector_splice (nxv8i16 ZPR:$Z1), (nxv8i16 ZPR:$Z2), (i64 -1))),
+  def : Pat<(nxv8i16 (vector_splice nxv8i16:$Z1, nxv8i16:$Z2, (i64 -1))),
             (INSR_ZV_H ZPR:$Z2, (INSERT_SUBREG (IMPLICIT_DEF),
             (LASTB_VPZ_H (PTRUE_H 31), ZPR:$Z1), hsub))>;
-  def : Pat<(nxv4i32 (vector_splice (nxv4i32 ZPR:$Z1), (nxv4i32 ZPR:$Z2), (i64 -1))),
+  def : Pat<(nxv4i32 (vector_splice nxv4i32:$Z1, nxv4i32:$Z2, (i64 -1))),
             (INSR_ZV_S ZPR:$Z2, (INSERT_SUBREG (IMPLICIT_DEF),
             (LASTB_VPZ_S (PTRUE_S 31), ZPR:$Z1), ssub))>;
-  def : Pat<(nxv2i64 (vector_splice (nxv2i64 ZPR:$Z1), (nxv2i64 ZPR:$Z2), (i64 -1))),
+  def : Pat<(nxv2i64 (vector_splice nxv2i64:$Z1, nxv2i64:$Z2, (i64 -1))),
             (INSR_ZV_D ZPR:$Z2, (INSERT_SUBREG (IMPLICIT_DEF),
             (LASTB_VPZ_D (PTRUE_D 31), ZPR:$Z1), dsub))>;
 
   // Splice with lane bigger or equal to 0
   foreach VT = [nxv16i8] in
-    def : Pat<(VT (vector_splice (VT ZPR:$Z1), (VT ZPR:$Z2), (i64 (sve_ext_imm_0_255 i32:$index)))),
+    def : Pat<(VT (vector_splice VT:$Z1, VT:$Z2, (i64 (sve_ext_imm_0_255 i32:$index)))),
               (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
 
   foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
-    def : Pat<(VT (vector_splice (VT ZPR:$Z1), (VT ZPR:$Z2), (i64 (sve_ext_imm_0_127 i32:$index)))),
+    def : Pat<(VT (vector_splice VT:$Z1, VT:$Z2, (i64 (sve_ext_imm_0_127 i32:$index)))),
               (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
 
   foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
-    def : Pat<(VT (vector_splice (VT ZPR:$Z1), (VT ZPR:$Z2), (i64 (sve_ext_imm_0_63 i32:$index)))),
+    def : Pat<(VT (vector_splice VT:$Z1, VT:$Z2, (i64 (sve_ext_imm_0_63 i32:$index)))),
               (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
 
   foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
-    def : Pat<(VT (vector_splice (VT ZPR:$Z1), (VT ZPR:$Z2), (i64 (sve_ext_imm_0_31 i32:$index)))),
+    def : Pat<(VT (vector_splice VT:$Z1, VT:$Z2, (i64 (sve_ext_imm_0_31 i32:$index)))),
               (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
 
   defm CMPHS_PPzZZ : sve_int_cmp_0<0b000, "cmphs", SETUGE, SETULE>;
@@ -2263,59 +2263,59 @@ let Predicates = [HasSVEorSME] in {
   defm FCVTZU_ZPmZ_DtoD : sve_fp_2op_p_zd< 0b1111111, "fcvtzu", ZPR64, ZPR64, null_frag,                     AArch64fcvtzu_mt, nxv2i64, nxv2i1, nxv2f64, ElementSizeD>;
 
   //These patterns exist to improve the code quality of conversions on unpacked types.
-  def : Pat<(nxv2f32 (AArch64fcvte_mt (nxv2i1 (SVEAllActive):$Pg), (nxv2f16 ZPR:$Zs), (nxv2f32 ZPR:$Zd))),
+  def : Pat<(nxv2f32 (AArch64fcvte_mt (nxv2i1 (SVEAllActive:$Pg)), nxv2f16:$Zs, nxv2f32:$Zd)),
             (FCVT_ZPmZ_HtoS_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
   // FP_ROUND has an additional 'precise' flag which indicates the type of rounding.
   // This is ignored by the pattern below where it is matched by (i64 timm0_1)
-  def : Pat<(nxv2f16 (AArch64fcvtr_mt (nxv2i1 (SVEAllActive):$Pg), (nxv2f32 ZPR:$Zs), (i64 timm0_1), (nxv2f16 ZPR:$Zd))),
+  def : Pat<(nxv2f16 (AArch64fcvtr_mt (nxv2i1 (SVEAllActive:$Pg)), nxv2f32:$Zs, (i64 timm0_1), nxv2f16:$Zd)),
             (FCVT_ZPmZ_StoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
   // Signed integer -> Floating-point 
   def : Pat<(nxv2f16 (AArch64scvtf_mt (nxv2i1 (SVEAllActive):$Pg),
-                      (sext_inreg (nxv2i64 ZPR:$Zs), nxv2i16), (nxv2f16 ZPR:$Zd))),
+                      (sext_inreg nxv2i64:$Zs, nxv2i16), nxv2f16:$Zd)),
             (SCVTF_ZPmZ_HtoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
   def : Pat<(nxv4f16 (AArch64scvtf_mt (nxv4i1 (SVEAllActive):$Pg),
-                      (sext_inreg (nxv4i32 ZPR:$Zs), nxv4i16), (nxv4f16 ZPR:$Zd))),
+                      (sext_inreg nxv4i32:$Zs, nxv4i16), nxv4f16:$Zd)),
             (SCVTF_ZPmZ_HtoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
   def : Pat<(nxv2f16 (AArch64scvtf_mt (nxv2i1 (SVEAllActive):$Pg),
-                      (sext_inreg (nxv2i64 ZPR:$Zs), nxv2i32), (nxv2f16 ZPR:$Zd))),
+                      (sext_inreg nxv2i64:$Zs, nxv2i32), nxv2f16:$Zd)),
             (SCVTF_ZPmZ_StoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
   def : Pat<(nxv2f32 (AArch64scvtf_mt (nxv2i1 (SVEAllActive):$Pg),
-                      (sext_inreg (nxv2i64 ZPR:$Zs), nxv2i32), (nxv2f32 ZPR:$Zd))),
+                      (sext_inreg nxv2i64:$Zs, nxv2i32), nxv2f32:$Zd)),
             (SCVTF_ZPmZ_StoS_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
   def : Pat<(nxv2f64 (AArch64scvtf_mt (nxv2i1 (SVEAllActive):$Pg),
-                      (sext_inreg (nxv2i64 ZPR:$Zs), nxv2i32), (nxv2f64 ZPR:$Zd))),
+                      (sext_inreg nxv2i64:$Zs, nxv2i32), nxv2f64:$Zd)),
             (SCVTF_ZPmZ_StoD_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
   // Unsigned integer -> Floating-point
-  def : Pat<(nxv2f16 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive):$Pg),
-                      (and (nxv2i64 ZPR:$Zs),
-                       (nxv2i64 (splat_vector (i64 0xFFFF)))), (nxv2f16 ZPR:$Zd))),
+  def : Pat<(nxv2f16 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive:$Pg)),
+                      (and nxv2i64:$Zs,
+                       (nxv2i64 (splat_vector (i64 0xFFFF)))), nxv2f16:$Zd)),
             (UCVTF_ZPmZ_HtoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
-  def : Pat<(nxv2f16 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive):$Pg),
-                      (and (nxv2i64 ZPR:$Zs),
-                       (nxv2i64 (splat_vector (i64 0xFFFFFFFF)))), (nxv2f16 ZPR:$Zd))),
+  def : Pat<(nxv2f16 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive:$Pg)),
+                      (and nxv2i64:$Zs,
+                       (nxv2i64 (splat_vector (i64 0xFFFFFFFF)))), nxv2f16:$Zd)),
             (UCVTF_ZPmZ_StoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
-  def : Pat<(nxv4f16 (AArch64ucvtf_mt (nxv4i1 (SVEAllActive):$Pg),
-                      (and (nxv4i32 ZPR:$Zs),
-                       (nxv4i32 (splat_vector (i32 0xFFFF)))), (nxv4f16 ZPR:$Zd))),
+  def : Pat<(nxv4f16 (AArch64ucvtf_mt (nxv4i1 (SVEAllActive:$Pg)),
+                      (and nxv4i32:$Zs,
+                       (nxv4i32 (splat_vector (i32 0xFFFF)))), nxv4f16:$Zd)),
             (UCVTF_ZPmZ_HtoH_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
-  def : Pat<(nxv2f32 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive):$Pg),
-                      (and (nxv2i64 ZPR:$Zs),
-                       (nxv2i64 (splat_vector (i64 0xFFFFFFFF)))), (nxv2f32 ZPR:$Zd))),
+  def : Pat<(nxv2f32 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive:$Pg)),
+                      (and nxv2i64:$Zs,
+                       (nxv2i64 (splat_vector (i64 0xFFFFFFFF)))), nxv2f32:$Zd)),
             (UCVTF_ZPmZ_StoS_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
-  def : Pat<(nxv2f64 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive):$Pg),
-                      (and (nxv2i64 ZPR:$Zs),
-                       (nxv2i64 (splat_vector (i64 0xFFFFFFFF)))), (nxv2f64 ZPR:$Zd))),
+  def : Pat<(nxv2f64 (AArch64ucvtf_mt (nxv2i1 (SVEAllActive:$Pg)),
+                      (and nxv2i64:$Zs,
+                       (nxv2i64 (splat_vector (i64 0xFFFFFFFF)))), nxv2f64:$Zd)),
             (UCVTF_ZPmZ_StoD_UNDEF ZPR:$Zd, PPR:$Pg, ZPR:$Zs)>;
 
   defm FRINTN_ZPmZ : sve_fp_2op_p_zd_HSD<0b00000, "frintn", AArch64frintn_mt>;
@@ -2510,12 +2510,12 @@ let Predicates = [HasSVEorSME] in {
   defm : ld1rq_pat<nxv4i32, AArch64ld1rq_z, LD1RQ_W, am_sve_regreg_lsl2>;
   defm : ld1rq_pat<nxv2i64, AArch64ld1rq_z, LD1RQ_D, am_sve_regreg_lsl3>;
 
-  def : Pat<(sext_inreg (nxv2i64 ZPR:$Zs), nxv2i32), (SXTW_ZPmZ_D_UNDEF (IMPLICIT_DEF), (PTRUE_D 31), ZPR:$Zs)>;
-  def : Pat<(sext_inreg (nxv2i64 ZPR:$Zs), nxv2i16), (SXTH_ZPmZ_D_UNDEF (IMPLICIT_DEF), (PTRUE_D 31), ZPR:$Zs)>;
-  def : Pat<(sext_inreg (nxv2i64 ZPR:$Zs), nxv2i8),  (SXTB_ZPmZ_D_UNDEF (IMPLICIT_DEF), (PTRUE_D 31), ZPR:$Zs)>;
-  def : Pat<(sext_inreg (nxv4i32 ZPR:$Zs), nxv4i16), (SXTH_ZPmZ_S_UNDEF (IMPLICIT_DEF), (PTRUE_S 31), ZPR:$Zs)>;
-  def : Pat<(sext_inreg (nxv4i32 ZPR:$Zs), nxv4i8),  (SXTB_ZPmZ_S_UNDEF (IMPLICIT_DEF), (PTRUE_S 31), ZPR:$Zs)>;
-  def : Pat<(sext_inreg (nxv8i16 ZPR:$Zs), nxv8i8),  (SXTB_ZPmZ_H_UNDEF (IMPLICIT_DEF), (PTRUE_H 31), ZPR:$Zs)>;
+  def : Pat<(sext_inreg nxv2i64:$Zs, nxv2i32), (SXTW_ZPmZ_D_UNDEF (IMPLICIT_DEF), (PTRUE_D 31), ZPR:$Zs)>;
+  def : Pat<(sext_inreg nxv2i64:$Zs, nxv2i16), (SXTH_ZPmZ_D_UNDEF (IMPLICIT_DEF), (PTRUE_D 31), ZPR:$Zs)>;
+  def : Pat<(sext_inreg nxv2i64:$Zs, nxv2i8),  (SXTB_ZPmZ_D_UNDEF (IMPLICIT_DEF), (PTRUE_D 31), ZPR:$Zs)>;
+  def : Pat<(sext_inreg nxv4i32:$Zs, nxv4i16), (SXTH_ZPmZ_S_UNDEF (IMPLICIT_DEF), (PTRUE_S 31), ZPR:$Zs)>;
+  def : Pat<(sext_inreg nxv4i32:$Zs, nxv4i8),  (SXTB_ZPmZ_S_UNDEF (IMPLICIT_DEF), (PTRUE_S 31), ZPR:$Zs)>;
+  def : Pat<(sext_inreg nxv8i16:$Zs, nxv8i8),  (SXTB_ZPmZ_H_UNDEF (IMPLICIT_DEF), (PTRUE_H 31), ZPR:$Zs)>;
 
   // General case that we ideally never want to match.
   def : Pat<(vscale GPR64:$scale), (MADDXrrr (UBFMXri (RDVLI_XI 1), 4, 63), $scale, XZR)>;
@@ -2621,109 +2621,109 @@ let Predicates = [HasSVEorSME] in {
   // constraint that none of the bits change when stored to memory as one
   // type, and reloaded as another type.
   let Predicates = [IsLE] in {
-    def : Pat<(nxv16i8 (bitconvert (nxv8i16 ZPR:$src))), (nxv16i8 ZPR:$src)>;
-    def : Pat<(nxv16i8 (bitconvert (nxv4i32 ZPR:$src))), (nxv16i8 ZPR:$src)>;
-    def : Pat<(nxv16i8 (bitconvert (nxv2i64 ZPR:$src))), (nxv16i8 ZPR:$src)>;
-    def : Pat<(nxv16i8 (bitconvert (nxv8f16 ZPR:$src))), (nxv16i8 ZPR:$src)>;
-    def : Pat<(nxv16i8 (bitconvert (nxv4f32 ZPR:$src))), (nxv16i8 ZPR:$src)>;
-    def : Pat<(nxv16i8 (bitconvert (nxv2f64 ZPR:$src))), (nxv16i8 ZPR:$src)>;
-
-    def : Pat<(nxv8i16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8i16 ZPR:$src)>;
-    def : Pat<(nxv8i16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8i16 ZPR:$src)>;
-    def : Pat<(nxv8i16 (bitconvert (nxv2i64 ZPR:$src))), (nxv8i16 ZPR:$src)>;
-    def : Pat<(nxv8i16 (bitconvert (nxv8f16 ZPR:$src))), (nxv8i16 ZPR:$src)>;
-    def : Pat<(nxv8i16 (bitconvert (nxv4f32 ZPR:$src))), (nxv8i16 ZPR:$src)>;
-    def : Pat<(nxv8i16 (bitconvert (nxv2f64 ZPR:$src))), (nxv8i16 ZPR:$src)>;
-
-    def : Pat<(nxv4i32 (bitconvert (nxv16i8 ZPR:$src))), (nxv4i32 ZPR:$src)>;
-    def : Pat<(nxv4i32 (bitconvert (nxv8i16 ZPR:$src))), (nxv4i32 ZPR:$src)>;
-    def : Pat<(nxv4i32 (bitconvert (nxv2i64 ZPR:$src))), (nxv4i32 ZPR:$src)>;
-    def : Pat<(nxv4i32 (bitconvert (nxv8f16 ZPR:$src))), (nxv4i32 ZPR:$src)>;
-    def : Pat<(nxv4i32 (bitconvert (nxv4f32 ZPR:$src))), (nxv4i32 ZPR:$src)>;
-    def : Pat<(nxv4i32 (bitconvert (nxv2f64 ZPR:$src))), (nxv4i32 ZPR:$src)>;
-
-    def : Pat<(nxv2i64 (bitconvert (nxv16i8 ZPR:$src))), (nxv2i64 ZPR:$src)>;
-    def : Pat<(nxv2i64 (bitconvert (nxv8i16 ZPR:$src))), (nxv2i64 ZPR:$src)>;
-    def : Pat<(nxv2i64 (bitconvert (nxv4i32 ZPR:$src))), (nxv2i64 ZPR:$src)>;
-    def : Pat<(nxv2i64 (bitconvert (nxv8f16 ZPR:$src))), (nxv2i64 ZPR:$src)>;
-    def : Pat<(nxv2i64 (bitconvert (nxv4f32 ZPR:$src))), (nxv2i64 ZPR:$src)>;
-    def : Pat<(nxv2i64 (bitconvert (nxv2f64 ZPR:$src))), (nxv2i64 ZPR:$src)>;
-
-    def : Pat<(nxv8f16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8f16 ZPR:$src)>;
-    def : Pat<(nxv8f16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8f16 ZPR:$src)>;
-    def : Pat<(nxv8f16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8f16 ZPR:$src)>;
-    def : Pat<(nxv8f16 (bitconvert (nxv2i64 ZPR:$src))), (nxv8f16 ZPR:$src)>;
-    def : Pat<(nxv8f16 (bitconvert (nxv4f32 ZPR:$src))), (nxv8f16 ZPR:$src)>;
-    def : Pat<(nxv8f16 (bitconvert (nxv2f64 ZPR:$src))), (nxv8f16 ZPR:$src)>;
-
-    def : Pat<(nxv4f32 (bitconvert (nxv16i8 ZPR:$src))), (nxv4f32 ZPR:$src)>;
-    def : Pat<(nxv4f32 (bitconvert (nxv8i16 ZPR:$src))), (nxv4f32 ZPR:$src)>;
-    def : Pat<(nxv4f32 (bitconvert (nxv4i32 ZPR:$src))), (nxv4f32 ZPR:$src)>;
-    def : Pat<(nxv4f32 (bitconvert (nxv2i64 ZPR:$src))), (nxv4f32 ZPR:$src)>;
-    def : Pat<(nxv4f32 (bitconvert (nxv8f16 ZPR:$src))), (nxv4f32 ZPR:$src)>;
-    def : Pat<(nxv4f32 (bitconvert (nxv2f64 ZPR:$src))), (nxv4f32 ZPR:$src)>;
-
-    def : Pat<(nxv2f64 (bitconvert (nxv16i8 ZPR:$src))), (nxv2f64 ZPR:$src)>;
-    def : Pat<(nxv2f64 (bitconvert (nxv8i16 ZPR:$src))), (nxv2f64 ZPR:$src)>;
-    def : Pat<(nxv2f64 (bitconvert (nxv4i32 ZPR:$src))), (nxv2f64 ZPR:$src)>;
-    def : Pat<(nxv2f64 (bitconvert (nxv2i64 ZPR:$src))), (nxv2f64 ZPR:$src)>;
-    def : Pat<(nxv2f64 (bitconvert (nxv8f16 ZPR:$src))), (nxv2f64 ZPR:$src)>;
-    def : Pat<(nxv2f64 (bitconvert (nxv4f32 ZPR:$src))), (nxv2f64 ZPR:$src)>;
-
-    def : Pat<(nxv8bf16 (bitconvert (nxv16i8 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
-    def : Pat<(nxv8bf16 (bitconvert (nxv8i16 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
-    def : Pat<(nxv8bf16 (bitconvert (nxv4i32 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
-    def : Pat<(nxv8bf16 (bitconvert (nxv2i64 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
-    def : Pat<(nxv8bf16 (bitconvert (nxv8f16 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
-    def : Pat<(nxv8bf16 (bitconvert (nxv4f32 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
-    def : Pat<(nxv8bf16 (bitconvert (nxv2f64 ZPR:$src))), (nxv8bf16 ZPR:$src)>;
-
-    def : Pat<(nxv16i8 (bitconvert (nxv8bf16 ZPR:$src))), (nxv16i8 ZPR:$src)>;
-    def : Pat<(nxv8i16 (bitconvert (nxv8bf16 ZPR:$src))), (nxv8i16 ZPR:$src)>;
-    def : Pat<(nxv4i32 (bitconvert (nxv8bf16 ZPR:$src))), (nxv4i32 ZPR:$src)>;
-    def : Pat<(nxv2i64 (bitconvert (nxv8bf16 ZPR:$src))), (nxv2i64 ZPR:$src)>;
-    def : Pat<(nxv8f16 (bitconvert (nxv8bf16 ZPR:$src))), (nxv8f16 ZPR:$src)>;
-    def : Pat<(nxv4f32 (bitconvert (nxv8bf16 ZPR:$src))), (nxv4f32 ZPR:$src)>;
-    def : Pat<(nxv2f64 (bitconvert (nxv8bf16 ZPR:$src))), (nxv2f64 ZPR:$src)>;
-
-    def : Pat<(nxv16i1 (bitconvert (aarch64svcount PNR:$src))), (nxv16i1 PPR:$src)>;
-    def : Pat<(aarch64svcount (bitconvert (nxv16i1 PPR:$src))), (aarch64svcount PNR:$src)>;
+    def : Pat<(nxv16i8 (bitconvert nxv8i16:$src)), (nxv16i8 ZPR:$src)>;
+    def : Pat<(nxv16i8 (bitconvert nxv4i32:$src)), (nxv16i8 ZPR:$src)>;
+    def : Pat<(nxv16i8 (bitconvert nxv2i64:$src)), (nxv16i8 ZPR:$src)>;
+    def : Pat<(nxv16i8 (bitconvert nxv8f16:$src)), (nxv16i8 ZPR:$src)>;
+    def : Pat<(nxv16i8 (bitconvert nxv4f32:$src)), (nxv16i8 ZPR:$src)>;
+    def : Pat<(nxv16i8 (bitconvert nxv2f64:$src)), (nxv16i8 ZPR:$src)>;
+
+    def : Pat<(nxv8i16 (bitconvert nxv16i8:$src)), (nxv8i16 ZPR:$src)>;
+    def : Pat<(nxv8i16 (bitconvert nxv4i32:$src)), (nxv8i16 ZPR:$src)>;
+    def : Pat<(nxv8i16 (bitconvert nxv2i64:$src)), (nxv8i16 ZPR:$src)>;
+    def : Pat<(nxv8i16 (bitconvert nxv8f16:$src)), (nxv8i16 ZPR:$src)>;
+    def : Pat<(nxv8i16 (bitconvert nxv4f32:$src)), (nxv8i16 ZPR:$src)>;
+    def : Pat<(nxv8i16 (bitconvert nxv2f64:$src)), (nxv8i16 ZPR:$src)>;
+
+    def : Pat<(nxv4i32 (bitconvert nxv16i8:$src)), (nxv4i32 ZPR:$src)>;
+    def : Pat<(nxv4i32 (bitconvert nxv8i16:$src)), (nxv4i32 ZPR:$src)>;
+    def : Pat<(nxv4i32 (bitconvert nxv2i64:$src)), (nxv4i32 ZPR:$src)>;
+    def : Pat<(nxv4i32 (bitconvert nxv8f16:$src)), (nxv4i32 ZPR:$src)>;
+    def : Pat<(nxv4i32 (bitconvert nxv4f32:$src)), (nxv4i32 ZPR:$src)>;
+    def : Pat<(nxv4i32 (bitconvert nxv2f64:$src)), (nxv4i32 ZPR:$src)>;
+
+    def : Pat<(nxv2i64 (bitconvert nxv16i8:$src)), (nxv2i64 ZPR:$src)>;
+    def : Pat<(nxv2i64 (bitconvert nxv8i16:$src)), (nxv2i64 ZPR:$src)>;
+    def : Pat<(nxv2i64 (bitconvert nxv4i32:$src)), (nxv2i64 ZPR:$src)>;
+    def : Pat<(nxv2i64 (bitconvert nxv8f16:$src)), (nxv2i64 ZPR:$src)>;
+    def : Pat<(nxv2i64 (bitconvert nxv4f32:$src)), (nxv2i64 ZPR:$src)>;
+    def : Pat<(nxv2i64 (bitconvert nxv2f64:$src)), (nxv2i64 ZPR:$src)>;
+
+    def : Pat<(nxv8f16 (bitconvert nxv16i8:$src)), (nxv8f16 ZPR:$src)>;
+    def : Pat<(nxv8f16 (bitconvert nxv8i16:$src)), (nxv8f16 ZPR:$src)>;
+    def : Pat<(nxv8f16 (bitconvert nxv4i32:$src)), (nxv8f16 ZPR:$src)>;
+    def : Pat<(nxv8f16 (bitconvert nxv2i64:$src)), (nxv8f16 ZPR:$src)>;
+    def : Pat<(nxv8f16 (bitconvert nxv4f32:$src)), (nxv8f16 ZPR:$src)>;
+    def : Pat<(nxv8f16 (bitconvert nxv2f64:$src)), (nxv8f16 ZPR:$src)>;
+
+    def : Pat<(nxv4f32 (bitconvert nxv16i8:$src)), (nxv4f32 ZPR:$src)>;
+    def : Pat<(nxv4f32 (bitconvert nxv8i16:$src)), (nxv4f32 ZPR:$src)>;
+    def : Pat<(nxv4f32 (bitconvert nxv4i32:$src)), (nxv4f32 ZPR:$src)>;
+    def : Pat<(nxv4f32 (bitconvert nxv2i64:$src)), (nxv4f32 ZPR:$src)>;
+    def : Pat<(nxv4f32 (bitconvert nxv8f16:$src)), (nxv4f32 ZPR:$src)>;
+    def : Pat<(nxv4f32 (bitconvert nxv2f64:$src)), (nxv4f32 ZPR:$src)>;
+
+    def : Pat<(nxv2f64 (bitconvert nxv16i8:$src)), (nxv2f64 ZPR:$src)>;
+    def : Pat<(nxv2f64 (bitconvert nxv8i16:$src)), (nxv2f64 ZPR:$src)>;
+    def : Pat<(nxv2f64 (bitconvert nxv4i32:$src)), (nxv2f64 ZPR:$src)>;
+    def : Pat<(nxv2f64 (bitconvert nxv2i64:$src)), (nxv2f64 ZPR:$src)>;
+    def : Pat<(nxv2f64 (bitconvert nxv8f16:$src)), (nxv2f64 ZPR:$src)>;
+    def : Pat<(nxv2f64 (bitconvert nxv4f32:$src)), (nxv2f64 ZPR:$src)>;
+
+    def : Pat<(nxv8bf16 (bitconvert nxv16i8:$src)), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert nxv8i16:$src)), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert nxv4i32:$src)), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert nxv2i64:$src)), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert nxv8f16:$src)), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert nxv4f32:$src)), (nxv8bf16 ZPR:$src)>;
+    def : Pat<(nxv8bf16 (bitconvert nxv2f64:$src)), (nxv8bf16 ZPR:$src)>;
+
+    def : Pat<(nxv16i8 (bitconvert nxv8bf16:$src)), (nxv16i8 ZPR:$src)>;
+    def : Pat<(nxv8i16 (bitconvert nxv8bf16:$src)), (nxv8i16 ZPR:$src)>;
+    def : Pat<(nxv4i32 (bitconvert nxv8bf16:$src)), (nxv4i32 ZPR:$src)>;
+    def : Pat<(nxv2i64 (bitconvert nxv8bf16:$src)), (nxv2i64 ZPR:$src)>;
+    def : Pat<(nxv8f16 (bitconvert nxv8bf16:$src)), (nxv8f16 ZPR:$src)>;
+    def : Pat<(nxv4f32 (bitconvert nxv8bf16:$src)), (nxv4f32 ZPR:$src)>;
+    def : Pat<(nxv2f64 (bitconvert nxv8bf16:$src)), (nxv2f64 ZPR:$src)>;
+
+    def : Pat<(nxv16i1 (bitconvert aarch64svcount:$src)), (nxv16i1 PPR:$src)>;
+    def : Pat<(aarch64svcount (bitconvert nxv16i1:$src)), (aarch64svcount PNR:$src)>;
   }
 
   // These allow casting from/to unpacked predicate types.
-  def : Pat<(nxv16i1 (reinterpret_cast (nxv16i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv16i1 (reinterpret_cast (nxv8i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv16i1 (reinterpret_cast (nxv4i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv16i1 (reinterpret_cast (nxv2i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv16i1 (reinterpret_cast (nxv1i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv8i1 (reinterpret_cast (nxv16i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv8i1 (reinterpret_cast  (nxv4i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv8i1 (reinterpret_cast  (nxv2i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv8i1 (reinterpret_cast  (nxv1i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv4i1 (reinterpret_cast (nxv16i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv4i1 (reinterpret_cast  (nxv8i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv4i1 (reinterpret_cast  (nxv2i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv4i1 (reinterpret_cast  (nxv1i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv2i1 (reinterpret_cast (nxv16i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv2i1 (reinterpret_cast  (nxv8i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv2i1 (reinterpret_cast  (nxv4i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv2i1 (reinterpret_cast  (nxv1i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv1i1 (reinterpret_cast (nxv16i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv1i1 (reinterpret_cast  (nxv8i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv1i1 (reinterpret_cast  (nxv4i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
-  def : Pat<(nxv1i1 (reinterpret_cast  (nxv2i1 PPR:$src))), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv16i1 (reinterpret_cast nxv16i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv16i1 (reinterpret_cast nxv8i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv16i1 (reinterpret_cast nxv4i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv16i1 (reinterpret_cast nxv2i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv16i1 (reinterpret_cast nxv1i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv8i1 (reinterpret_cast nxv16i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv8i1 (reinterpret_cast  nxv4i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv8i1 (reinterpret_cast  nxv2i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv8i1 (reinterpret_cast  nxv1i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv4i1 (reinterpret_cast nxv16i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv4i1 (reinterpret_cast  nxv8i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv4i1 (reinterpret_cast  nxv2i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv4i1 (reinterpret_cast  nxv1i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv2i1 (reinterpret_cast nxv16i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv2i1 (reinterpret_cast  nxv8i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv2i1 (reinterpret_cast  nxv4i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv2i1 (reinterpret_cast  nxv1i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv1i1 (reinterpret_cast nxv16i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv1i1 (reinterpret_cast  nxv8i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv1i1 (reinterpret_cast  nxv4i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
+  def : Pat<(nxv1i1 (reinterpret_cast  nxv2i1:$src)), (COPY_TO_REGCLASS PPR:$src, PPR)>;
 
   // These allow casting from/to unpacked floating-point types.
-  def : Pat<(nxv2f16 (reinterpret_cast (nxv8f16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
-  def : Pat<(nxv8f16 (reinterpret_cast (nxv2f16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
-  def : Pat<(nxv4f16 (reinterpret_cast (nxv8f16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
-  def : Pat<(nxv8f16 (reinterpret_cast (nxv4f16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
-  def : Pat<(nxv2f32 (reinterpret_cast (nxv4f32 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
-  def : Pat<(nxv4f32 (reinterpret_cast (nxv2f32 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
-  def : Pat<(nxv2bf16 (reinterpret_cast (nxv8bf16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
-  def : Pat<(nxv8bf16 (reinterpret_cast (nxv2bf16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
-  def : Pat<(nxv4bf16 (reinterpret_cast (nxv8bf16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
-  def : Pat<(nxv8bf16 (reinterpret_cast (nxv4bf16 ZPR:$src))), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
+  def : Pat<(nxv2f16 (reinterpret_cast nxv8f16:$src)), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
+  def : Pat<(nxv8f16 (reinterpret_cast nxv2f16:$src)), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
+  def : Pat<(nxv4f16 (reinterpret_cast nxv8f16:$src)), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
+  def : Pat<(nxv8f16 (reinterpret_cast nxv4f16:$src)), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
+  def : Pat<(nxv2f32 (reinterpret_cast nxv4f32:$src)), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
+  def : Pat<(nxv4f32 (reinterpret_cast nxv2f32:$src)), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
+  def : Pat<(nxv2bf16 (reinterpret_cast nxv8bf16:$src)), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
+  def : Pat<(nxv8bf16 (reinterpret_cast nxv2bf16:$src)), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
+  def : Pat<(nxv4bf16 (reinterpret_cast nxv8bf16:$src)), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
+  def : Pat<(nxv8bf16 (reinterpret_cast nxv4bf16:$src)), (COPY_TO_REGCLASS ZPR:$src, ZPR)>;
 
   def : Pat<(nxv16i1 (and PPR:$Ps1, PPR:$Ps2)),
             (AND_PPzPP (PTRUE_B 31), PPR:$Ps1, PPR:$Ps2)>;
@@ -2788,14 +2788,14 @@ let Predicates = [HasSVEorSME] in {
   multiclass pred_store<ValueType Ty, ValueType PredTy, SDPatternOperator Store,
                         Instruction RegRegInst, Instruction RegImmInst, ComplexPattern AddrCP> {
     let AddedComplexity = 1 in {
-      def _reg_reg : Pat<(Store (Ty ZPR:$vec), (AddrCP GPR64:$base, GPR64:$offset), (PredTy PPR:$gp)),
+      def _reg_reg : Pat<(Store Ty:$vec, (AddrCP GPR64:$base, GPR64:$offset), PredTy:$gp),
                          (RegRegInst ZPR:$vec, PPR:$gp, GPR64:$base, GPR64:$offset)>;
     }
     let AddedComplexity = 2 in {
-      def _reg_imm : Pat<(Store (Ty ZPR:$vec), (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset), (PredTy PPR:$gp)),
+      def _reg_imm : Pat<(Store Ty:$vec, (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset), PredTy:$gp),
                          (RegImmInst ZPR:$vec, PPR:$gp, GPR64:$base, simm4s1:$offset)>;
     }
-    def _default : Pat<(Store (Ty ZPR:$vec), GPR64:$base, (PredTy PPR:$gp)),
+    def _default : Pat<(Store Ty:$vec, GPR64:$base, PredTy:$gp),
                        (RegImmInst ZPR:$vec, PPR:$gp, GPR64:$base, (i64 0))>;
   }
 
@@ -2840,15 +2840,15 @@ let Predicates = [HasSVEorSME] in {
                           Instruction RegImmInst, Instruction PTrue,
                           ComplexPattern AddrCP> {
     let AddedComplexity = 1 in {
-      def _reg : Pat<(Store (Ty ZPR:$val), (AddrCP GPR64sp:$base, GPR64:$offset)),
+      def _reg : Pat<(Store Ty:$val, (AddrCP GPR64sp:$base, GPR64:$offset)),
                      (RegRegInst ZPR:$val, (PTrue 31), GPR64sp:$base, GPR64:$offset)>;
     }
     let AddedComplexity = 2 in {
-      def _imm : Pat<(Store (Ty ZPR:$val), (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
+      def _imm : Pat<(Store Ty:$val, (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset)),
                      (RegImmInst ZPR:$val, (PTrue 31), GPR64sp:$base, simm4s1:$offset)>;
     }
 
-    def : Pat<(Store (Ty ZPR:$val), GPR64:$base),
+    def : Pat<(Store Ty:$val, GPR64:$base),
               (RegImmInst ZPR:$val, (PTrue 31), GPR64:$base, (i64 0))>;
   }
 
@@ -2927,7 +2927,7 @@ let Predicates = [HasSVEorSME] in {
     let Predicates = [IsLE] in {
       def : Pat<(Ty (load (am_sve_regreg_lsl0 GPR64sp:$base, GPR64:$offset))),
                 (LD1B (PTRUE_B 31), GPR64sp:$base, GPR64:$offset)>;
-      def : Pat<(store (Ty ZPR:$val), (am_sve_regreg_lsl0 GPR64sp:$base, GPR64:$offset)),
+      def : Pat<(store Ty:$val, (am_sve_regreg_lsl0 GPR64sp:$base, GPR64:$offset)),
                 (ST1B ZPR:$val, (PTRUE_B 31), GPR64sp:$base, GPR64:$offset)>;
     }
   }
@@ -3095,18 +3095,18 @@ let Predicates = [HasSVEorSME] in {
                  SDPatternOperator Store, ValueType PredTy, ValueType MemVT, ComplexPattern AddrCP> {
     // reg + reg
     let AddedComplexity = 1 in {
-      def : Pat<(Store (Ty ZPR:$vec), (AddrCP GPR64:$base, GPR64:$offset), (PredTy PPR:$gp), MemVT),
+      def : Pat<(Store Ty:$vec, (AddrCP GPR64:$base, GPR64:$offset), PredTy:$gp, MemVT),
                 (RegRegInst ZPR:$vec, PPR:$gp, GPR64sp:$base, GPR64:$offset)>;
     }
 
     // scalar + immediate (mul vl)
     let AddedComplexity = 2 in {
-      def : Pat<(Store (Ty ZPR:$vec), (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset), (PredTy PPR:$gp), MemVT),
+      def : Pat<(Store Ty:$vec, (am_sve_indexed_s4 GPR64sp:$base, simm4s1:$offset), PredTy:$gp, MemVT),
                 (RegImmInst ZPR:$vec, PPR:$gp, GPR64sp:$base, simm4s1:$offset)>;
     }
 
     // base
-    def : Pat<(Store (Ty ZPR:$vec), GPR64:$base, (PredTy PPR:$gp), MemVT),
+    def : Pat<(Store Ty:$vec, GPR64:$base, (PredTy PPR:$gp), MemVT),
               (RegImmInst ZPR:$vec, PPR:$gp, GPR64:$base, (i64 0))>;
   }
 
@@ -3158,44 +3158,44 @@ let Predicates = [HasSVEorSME] in {
             (INSERT_SUBREG (nxv2f64 (IMPLICIT_DEF)), FPR64:$src, dsub)>;
 
   // Insert scalar into vector[0]
-  def : Pat<(nxv16i8 (vector_insert (nxv16i8 ZPR:$vec), (i32 GPR32:$src), 0)),
+  def : Pat<(nxv16i8 (vector_insert nxv16i8:$vec, (i32 GPR32:$src), 0)),
             (CPY_ZPmR_B ZPR:$vec, (PTRUE_B 1), GPR32:$src)>;
-  def : Pat<(nxv8i16 (vector_insert (nxv8i16 ZPR:$vec), (i32 GPR32:$src), 0)),
+  def : Pat<(nxv8i16 (vector_insert nxv8i16:$vec, (i32 GPR32:$src), 0)),
             (CPY_ZPmR_H ZPR:$vec, (PTRUE_H 1), GPR32:$src)>;
-  def : Pat<(nxv4i32 (vector_insert (nxv4i32 ZPR:$vec), (i32 GPR32:$src), 0)),
+  def : Pat<(nxv4i32 (vector_insert nxv4i32:$vec, (i32 GPR32:$src), 0)),
             (CPY_ZPmR_S ZPR:$vec, (PTRUE_S 1), GPR32:$src)>;
-  def : Pat<(nxv2i64 (vector_insert (nxv2i64 ZPR:$vec), (i64 GPR64:$src), 0)),
+  def : Pat<(nxv2i64 (vector_insert nxv2i64:$vec, (i64 GPR64:$src), 0)),
             (CPY_ZPmR_D ZPR:$vec, (PTRUE_D 1), GPR64:$src)>;
 
-  def : Pat<(nxv8f16 (vector_insert (nxv8f16 ZPR:$vec), (f16 FPR16:$src), 0)),
+  def : Pat<(nxv8f16 (vector_insert nxv8f16:$vec, (f16 FPR16:$src), 0)),
             (SEL_ZPZZ_H (PTRUE_H 1), (INSERT_SUBREG (IMPLICIT_DEF), FPR16:$src, hsub), ZPR:$vec)>;
-  def : Pat<(nxv8bf16 (vector_insert (nxv8bf16 ZPR:$vec), (bf16 FPR16:$src), 0)),
+  def : Pat<(nxv8bf16 (vector_insert nxv8bf16:$vec, (bf16 FPR16:$src), 0)),
             (SEL_ZPZZ_H (PTRUE_H 1), (INSERT_SUBREG (IMPLICIT_DEF), FPR16:$src, hsub), ZPR:$vec)>;
-  def : Pat<(nxv4f32 (vector_insert (nxv4f32 ZPR:$vec), (f32 FPR32:$src), 0)),
+  def : Pat<(nxv4f32 (vector_insert nxv4f32:$vec, (f32 FPR32:$src), 0)),
             (SEL_ZPZZ_S (PTRUE_S 1), (INSERT_SUBREG (IMPLICIT_DEF), FPR32:$src, ssub), ZPR:$vec)>;
-  def : Pat<(nxv2f64 (vector_insert (nxv2f64 ZPR:$vec), (f64 FPR64:$src), 0)),
+  def : Pat<(nxv2f64 (vector_insert nxv2f64:$vec, (f64 FPR64:$src), 0)),
             (SEL_ZPZZ_D (PTRUE_D 1), (INSERT_SUBREG (IMPLICIT_DEF), FPR64:$src, dsub), ZPR:$vec)>;
 
   // Insert scalar into vector with scalar index
-  def : Pat<(nxv16i8 (vector_insert (nxv16i8 ZPR:$vec), GPR32:$src, GPR64:$index)),
+  def : Pat<(nxv16i8 (vector_insert nxv16i8:$vec, GPR32:$src, GPR64:$index)),
             (CPY_ZPmR_B ZPR:$vec,
                         (CMPEQ_PPzZZ_B (PTRUE_B 31),
                                        (INDEX_II_B 0, 1),
                                        (DUP_ZR_B (i32 (EXTRACT_SUBREG GPR64:$index, sub_32)))),
                         GPR32:$src)>;
-  def : Pat<(nxv8i16 (vector_insert (nxv8i16 ZPR:$vec), GPR32:$src, GPR64:$index)),
+  def : Pat<(nxv8i16 (vector_insert nxv8i16:$vec, GPR32:$src, GPR64:$index)),
             (CPY_ZPmR_H ZPR:$vec,
                         (CMPEQ_PPzZZ_H (PTRUE_H 31),
                                        (INDEX_II_H 0, 1),
                                        (DUP_ZR_H (i32 (EXTRACT_SUBREG GPR64:$index, sub_32)))),
                         GPR32:$src)>;
-  def : Pat<(nxv4i32 (vector_insert (nxv4i32 ZPR:$vec), GPR32:$src, GPR64:$index)),
+  def : Pat<(nxv4i32 (vector_insert nxv4i32:$vec, GPR32:$src, GPR64:$index)),
             (CPY_ZPmR_S ZPR:$vec,
                         (CMPEQ_PPzZZ_S (PTRUE_S 31),
                                        (INDEX_II_S 0, 1),
                                        (DUP_ZR_S (i32 (EXTRACT_SUBREG GPR64:$index, sub_32)))),
                         GPR32:$src)>;
-  def : Pat<(nxv2i64 (vector_insert (nxv2i64 ZPR:$vec), GPR64:$src, GPR64:$index)),
+  def : Pat<(nxv2i64 (vector_insert nxv2i64:$vec, GPR64:$src, GPR64:$index)),
             (CPY_ZPmR_D ZPR:$vec,
                         (CMPEQ_PPzZZ_D (PTRUE_D 31),
                                        (INDEX_II_D 0, 1),
@@ -3203,55 +3203,55 @@ let Predicates = [HasSVEorSME] in {
                         GPR64:$src)>;
 
   // Insert FP scalar into vector with scalar index
-  def : Pat<(nxv2f16 (vector_insert (nxv2f16 ZPR:$vec), (f16 FPR16:$src), GPR64:$index)),
+  def : Pat<(nxv2f16 (vector_insert nxv2f16:$vec, (f16 FPR16:$src), GPR64:$index)),
             (CPY_ZPmV_H ZPR:$vec,
                         (CMPEQ_PPzZZ_D (PTRUE_D 31),
                                        (INDEX_II_D 0, 1),
                                        (DUP_ZR_D GPR64:$index)),
                         $src)>;
-  def : Pat<(nxv4f16 (vector_insert (nxv4f16 ZPR:$vec), (f16 FPR16:$src), GPR64:$index)),
+  def : Pat<(nxv4f16 (vector_insert nxv4f16:$vec, (f16 FPR16:$src), GPR64:$index)),
             (CPY_ZPmV_H ZPR:$vec,
                         (CMPEQ_PPzZZ_S (PTRUE_S 31),
                                        (INDEX_II_S 0, 1),
                                        (DUP_ZR_S (i32 (EXTRACT_SUBREG GPR64:$index, sub_32)))),
                         $src)>;
-  def : Pat<(nxv8f16 (vector_insert (nxv8f16 ZPR:$vec), (f16 FPR16:$src), GPR64:$index)),
+  def : Pat<(nxv8f16 (vector_insert nxv8f16:$vec, (f16 FPR16:$src), GPR64:$index)),
             (CPY_ZPmV_H ZPR:$vec,
                         (CMPEQ_PPzZZ_H (PTRUE_H 31),
                                        (INDEX_II_H 0, 1),
                                        (DUP_ZR_H (i32 (EXTRACT_SUBREG GPR64:$index, sub_32)))),
                         $src)>;
-  def : Pat<(nxv2bf16 (vector_insert (nxv2bf16 ZPR:$vec), (bf16 FPR16:$src), GPR64:$index)),
+  def : Pat<(nxv2bf16 (vector_insert nxv2bf16:$vec, (bf16 FPR16:$src), GPR64:$index)),
             (CPY_ZPmV_H ZPR:$vec,
                         (CMPEQ_PPzZZ_D (PTRUE_D 31),
                                        (INDEX_II_D 0, 1),
                                        (DUP_ZR_D GPR64:$index)),
                         $src)>;
-  def : Pat<(nxv4bf16 (vector_insert (nxv4bf16 ZPR:$vec), (bf16 FPR16:$src), GPR64:$index)),
+  def : Pat<(nxv4bf16 (vector_insert nxv4bf16:$vec, (bf16 FPR16:$src), GPR64:$index)),
             (CPY_ZPmV_H ZPR:$vec,
                         (CMPEQ_PPzZZ_S (PTRUE_S 31),
                                        (INDEX_II_S 0, 1),
                                        (DUP_ZR_S (i32 (EXTRACT_SUBREG GPR64:$index, sub_32)))),
                         $src)>;
-  def : Pat<(nxv8bf16 (vector_insert (nxv8bf16 ZPR:$vec), (bf16 FPR16:$src), GPR64:$index)),
+  def : Pat<(nxv8bf16 (vector_insert nxv8bf16:$vec, (bf16 FPR16:$src), GPR64:$index)),
             (CPY_ZPmV_H ZPR:$vec,
                         (CMPEQ_PPzZZ_H (PTRUE_H 31),
                                        (INDEX_II_H 0, 1),
                                        (DUP_ZR_H (i32 (EXTRACT_SUBREG GPR64:$index, sub_32)))),
                         $src)>;
-  def : Pat<(nxv2f32 (vector_insert (nxv2f32 ZPR:$vec), (f32 FPR32:$src), GPR64:$index)),
+  def : Pat<(nxv2f32 (vector_insert nxv2f32:$vec, (f32 FPR32:$src), GPR64:$index)),
             (CPY_ZPmV_S ZPR:$vec,
                         (CMPEQ_PPzZZ_D (PTRUE_D 31),
                                        (INDEX_II_D 0, 1),
                                        (DUP_ZR_D GPR64:$index)),
                         $src) >;
-  def : Pat<(nxv4f32 (vector_insert (nxv4f32 ZPR:$vec), (f32 FPR32:$src), GPR64:$index)),
+  def : Pat<(nxv4f32 (vector_insert nxv4f32:$vec, (f32 FPR32:$src), GPR64:$index)),
             (CPY_ZPmV_S ZPR:$vec,
                         (CMPEQ_PPzZZ_S (PTRUE_S 31),
                                        (INDEX_II_S 0, 1),
                                        (DUP_ZR_S (i32 (EXTRACT_SUBREG GPR64:$index, sub_32)))),
                         $src)>;
-  def : Pat<(nxv2f64 (vector_insert (nxv2f64 ZPR:$vec), (f64 FPR64:$src), GPR64:$index)),
+  def : Pat<(nxv2f64 (vector_insert nxv2f64:$vec, (f64 FPR64:$src), GPR64:$index)),
             (CPY_ZPmV_D ZPR:$vec,
                         (CMPEQ_PPzZZ_D (PTRUE_D 31),
                                        (INDEX_II_D 0, 1),
@@ -3259,139 +3259,139 @@ let Predicates = [HasSVEorSME] in {
                         $src)>;
 
   // Extract element from vector with scalar index
-  def : Pat<(i32 (vector_extract (nxv16i8 ZPR:$vec), GPR64:$index)),
+  def : Pat<(i32 (vector_extract nxv16i8:$vec, GPR64:$index)),
             (LASTB_RPZ_B (WHILELS_PXX_B XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(i32 (vector_extract (nxv8i16 ZPR:$vec), GPR64:$index)),
+  def : Pat<(i32 (vector_extract nxv8i16:$vec, GPR64:$index)),
             (LASTB_RPZ_H (WHILELS_PXX_H XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(i32 (vector_extract (nxv4i32 ZPR:$vec), GPR64:$index)),
+  def : Pat<(i32 (vector_extract nxv4i32:$vec, GPR64:$index)),
             (LASTB_RPZ_S (WHILELS_PXX_S XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(i64 (vector_extract (nxv2i64 ZPR:$vec), GPR64:$index)),
+  def : Pat<(i64 (vector_extract nxv2i64:$vec, GPR64:$index)),
             (LASTB_RPZ_D (WHILELS_PXX_D XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(f16 (vector_extract (nxv8f16 ZPR:$vec), GPR64:$index)),
+  def : Pat<(f16 (vector_extract nxv8f16:$vec, GPR64:$index)),
             (LASTB_VPZ_H (WHILELS_PXX_H XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(f16 (vector_extract (nxv4f16 ZPR:$vec), GPR64:$index)),
+  def : Pat<(f16 (vector_extract nxv4f16:$vec, GPR64:$index)),
             (LASTB_VPZ_H (WHILELS_PXX_S XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(f16 (vector_extract (nxv2f16 ZPR:$vec), GPR64:$index)),
+  def : Pat<(f16 (vector_extract nxv2f16:$vec, GPR64:$index)),
             (LASTB_VPZ_H (WHILELS_PXX_D XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(bf16 (vector_extract (nxv8bf16 ZPR:$vec), GPR64:$index)),
+  def : Pat<(bf16 (vector_extract nxv8bf16:$vec, GPR64:$index)),
             (LASTB_VPZ_H (WHILELS_PXX_H XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(bf16 (vector_extract (nxv4bf16 ZPR:$vec), GPR64:$index)),
+  def : Pat<(bf16 (vector_extract nxv4bf16:$vec, GPR64:$index)),
             (LASTB_VPZ_H (WHILELS_PXX_S XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(bf16 (vector_extract (nxv2bf16 ZPR:$vec), GPR64:$index)),
+  def : Pat<(bf16 (vector_extract nxv2bf16:$vec, GPR64:$index)),
             (LASTB_VPZ_H (WHILELS_PXX_D XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(f32 (vector_extract (nxv4f32 ZPR:$vec), GPR64:$index)),
+  def : Pat<(f32 (vector_extract nxv4f32:$vec, GPR64:$index)),
             (LASTB_VPZ_S (WHILELS_PXX_S XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(f32 (vector_extract (nxv2f32 ZPR:$vec), GPR64:$index)),
+  def : Pat<(f32 (vector_extract nxv2f32:$vec, GPR64:$index)),
             (LASTB_VPZ_S (WHILELS_PXX_D XZR, GPR64:$index), ZPR:$vec)>;
-  def : Pat<(f64 (vector_extract (nxv2f64 ZPR:$vec), GPR64:$index)),
+  def : Pat<(f64 (vector_extract nxv2f64:$vec, GPR64:$index)),
             (LASTB_VPZ_D (WHILELS_PXX_D XZR, GPR64:$index), ZPR:$vec)>;
 
   // Extract element from vector with immediate index
-  def : Pat<(i32 (vector_extract (nxv16i8 ZPR:$vec), sve_elm_idx_extdup_b:$index)),
+  def : Pat<(i32 (vector_extract nxv16i8:$vec, sve_elm_idx_extdup_b:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_B ZPR:$vec, sve_elm_idx_extdup_b:$index), ssub)>;
-  def : Pat<(i32 (vector_extract (nxv8i16 ZPR:$vec), sve_elm_idx_extdup_h:$index)),
+  def : Pat<(i32 (vector_extract nxv8i16:$vec, sve_elm_idx_extdup_h:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_H ZPR:$vec, sve_elm_idx_extdup_h:$index), ssub)>;
-  def : Pat<(i32 (vector_extract (nxv4i32 ZPR:$vec), sve_elm_idx_extdup_s:$index)),
+  def : Pat<(i32 (vector_extract nxv4i32:$vec, sve_elm_idx_extdup_s:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_S ZPR:$vec, sve_elm_idx_extdup_s:$index), ssub)>;
-  def : Pat<(i64 (vector_extract (nxv2i64 ZPR:$vec), sve_elm_idx_extdup_d:$index)),
+  def : Pat<(i64 (vector_extract nxv2i64:$vec, sve_elm_idx_extdup_d:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_D ZPR:$vec, sve_elm_idx_extdup_d:$index), dsub)>;
-  def : Pat<(f16 (vector_extract (nxv8f16 ZPR:$vec), sve_elm_idx_extdup_h:$index)),
+  def : Pat<(f16 (vector_extract nxv8f16:$vec, sve_elm_idx_extdup_h:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_H ZPR:$vec, sve_elm_idx_extdup_h:$index), hsub)>;
-  def : Pat<(f16 (vector_extract (nxv4f16 ZPR:$vec), sve_elm_idx_extdup_s:$index)),
+  def : Pat<(f16 (vector_extract nxv4f16:$vec, sve_elm_idx_extdup_s:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_S ZPR:$vec, sve_elm_idx_extdup_s:$index), hsub)>;
-  def : Pat<(f16 (vector_extract (nxv2f16 ZPR:$vec), sve_elm_idx_extdup_d:$index)),
+  def : Pat<(f16 (vector_extract nxv2f16:$vec, sve_elm_idx_extdup_d:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_D ZPR:$vec, sve_elm_idx_extdup_d:$index), hsub)>;
-  def : Pat<(bf16 (vector_extract (nxv8bf16 ZPR:$vec), sve_elm_idx_extdup_h:$index)),
+  def : Pat<(bf16 (vector_extract nxv8bf16:$vec, sve_elm_idx_extdup_h:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_H ZPR:$vec, sve_elm_idx_extdup_h:$index), hsub)>;
-  def : Pat<(bf16 (vector_extract (nxv4bf16 ZPR:$vec), sve_elm_idx_extdup_s:$index)),
+  def : Pat<(bf16 (vector_extract nxv4bf16:$vec, sve_elm_idx_extdup_s:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_S ZPR:$vec, sve_elm_idx_extdup_s:$index), hsub)>;
-  def : Pat<(bf16 (vector_extract (nxv2bf16 ZPR:$vec), sve_elm_idx_extdup_d:$index)),
+  def : Pat<(bf16 (vector_extract nxv2bf16:$vec, sve_elm_idx_extdup_d:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_D ZPR:$vec, sve_elm_idx_extdup_d:$index), hsub)>;
-  def : Pat<(f32 (vector_extract (nxv4f32 ZPR:$vec), sve_elm_idx_extdup_s:$index)),
+  def : Pat<(f32 (vector_extract nxv4f32:$vec, sve_elm_idx_extdup_s:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_S ZPR:$vec, sve_elm_idx_extdup_s:$index), ssub)>;
-  def : Pat<(f32 (vector_extract (nxv2f32 ZPR:$vec), sve_elm_idx_extdup_d:$index)),
+  def : Pat<(f32 (vector_extract nxv2f32:$vec, sve_elm_idx_extdup_d:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_D ZPR:$vec, sve_elm_idx_extdup_d:$index), ssub)>;
-  def : Pat<(f64 (vector_extract (nxv2f64 ZPR:$vec), sve_elm_idx_extdup_d:$index)),
+  def : Pat<(f64 (vector_extract nxv2f64:$vec, sve_elm_idx_extdup_d:$index)),
             (EXTRACT_SUBREG (DUP_ZZI_D ZPR:$vec, sve_elm_idx_extdup_d:$index), dsub)>;
 
   // Extract element from vector with immediate index that's within the bottom 128-bits.
   let Predicates = [IsNeonAvailable], AddedComplexity = 1 in {
-  def : Pat<(i32 (vector_extract (nxv16i8 ZPR:$vec), VectorIndexB:$index)),
+  def : Pat<(i32 (vector_extract nxv16i8:$vec, VectorIndexB:$index)),
             (UMOVvi8 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index)>;
-  def : Pat<(i32 (vector_extract (nxv8i16 ZPR:$vec), VectorIndexH:$index)),
+  def : Pat<(i32 (vector_extract nxv8i16:$vec, VectorIndexH:$index)),
             (UMOVvi16 (v8i16 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexH:$index)>;
-  def : Pat<(i32 (vector_extract (nxv4i32 ZPR:$vec), VectorIndexS:$index)),
+  def : Pat<(i32 (vector_extract nxv4i32:$vec, VectorIndexS:$index)),
             (UMOVvi32 (v4i32 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexS:$index)>;
-  def : Pat<(i64 (vector_extract (nxv2i64 ZPR:$vec), VectorIndexD:$index)),
+  def : Pat<(i64 (vector_extract nxv2i64:$vec, VectorIndexD:$index)),
             (UMOVvi64 (v2i64 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexD:$index)>;
   } // End IsNeonAvailable
 
   let Predicates = [IsNeonAvailable] in {
-  def : Pat<(sext_inreg (vector_extract (nxv16i8 ZPR:$vec), VectorIndexB:$index), i8),
+  def : Pat<(sext_inreg (vector_extract nxv16i8:$vec, VectorIndexB:$index), i8),
             (SMOVvi8to32 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index)>;
-  def : Pat<(sext_inreg (anyext (i32 (vector_extract (nxv16i8 ZPR:$vec), VectorIndexB:$index))), i8),
+  def : Pat<(sext_inreg (anyext (i32 (vector_extract nxv16i8:$vec, VectorIndexB:$index))), i8),
             (SMOVvi8to64 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index)>;
 
-  def : Pat<(sext_inreg (vector_extract (nxv8i16 ZPR:$vec), VectorIndexH:$index), i16),
+  def : Pat<(sext_inreg (vector_extract nxv8i16:$vec, VectorIndexH:$index), i16),
             (SMOVvi16to32 (v8i16 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexH:$index)>;
-  def : Pat<(sext_inreg (anyext (i32 (vector_extract (nxv8i16 ZPR:$vec), VectorIndexH:$index))), i16),
+  def : Pat<(sext_inreg (anyext (i32 (vector_extract nxv8i16:$vec, VectorIndexH:$index))), i16),
             (SMOVvi16to64 (v8i16 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexH:$index)>;
 
-  def : Pat<(sext (i32 (vector_extract (nxv4i32 ZPR:$vec), VectorIndexS:$index))),
+  def : Pat<(sext (i32 (vector_extract nxv4i32:$vec, VectorIndexS:$index))),
             (SMOVvi32to64 (v4i32 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexS:$index)>;
   } // End IsNeonAvailable
 
   // Extract first element from vector.
   let AddedComplexity = 2 in {
-  def : Pat<(i32 (vector_extract (nxv16i8 ZPR:$Zs), (i64 0))),
+  def : Pat<(i32 (vector_extract nxv16i8:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, ssub)>;
-  def : Pat<(i32 (vector_extract (nxv8i16 ZPR:$Zs), (i64 0))),
+  def : Pat<(i32 (vector_extract nxv8i16:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, ssub)>;
-  def : Pat<(i32 (vector_extract (nxv4i32 ZPR:$Zs), (i64 0))),
+  def : Pat<(i32 (vector_extract nxv4i32:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, ssub)>;
-  def : Pat<(i64 (vector_extract (nxv2i64 ZPR:$Zs), (i64 0))),
+  def : Pat<(i64 (vector_extract nxv2i64:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, dsub)>;
-  def : Pat<(f16 (vector_extract (nxv8f16 ZPR:$Zs), (i64 0))),
+  def : Pat<(f16 (vector_extract nxv8f16:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, hsub)>;
-  def : Pat<(f16 (vector_extract (nxv4f16 ZPR:$Zs), (i64 0))),
+  def : Pat<(f16 (vector_extract nxv4f16:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, hsub)>;
-  def : Pat<(f16 (vector_extract (nxv2f16 ZPR:$Zs), (i64 0))),
+  def : Pat<(f16 (vector_extract nxv2f16:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, hsub)>;
-  def : Pat<(bf16 (vector_extract (nxv8bf16 ZPR:$Zs), (i64 0))),
+  def : Pat<(bf16 (vector_extract nxv8bf16:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, hsub)>;
-  def : Pat<(bf16 (vector_extract (nxv4bf16 ZPR:$Zs), (i64 0))),
+  def : Pat<(bf16 (vector_extract nxv4bf16:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, hsub)>;
-  def : Pat<(bf16 (vector_extract (nxv2bf16 ZPR:$Zs), (i64 0))),
+  def : Pat<(bf16 (vector_extract nxv2bf16:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, hsub)>;
-  def : Pat<(f32 (vector_extract (nxv4f32 ZPR:$Zs), (i64 0))),
+  def : Pat<(f32 (vector_extract nxv4f32:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, ssub)>;
-  def : Pat<(f32 (vector_extract (nxv2f32 ZPR:$Zs), (i64 0))),
+  def : Pat<(f32 (vector_extract nxv2f32:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, ssub)>;
-  def : Pat<(f64 (vector_extract (nxv2f64 ZPR:$Zs), (i64 0))),
+  def : Pat<(f64 (vector_extract nxv2f64:$Zs, (i64 0))),
             (EXTRACT_SUBREG ZPR:$Zs, dsub)>;
   }
 
   multiclass sve_predicated_add<SDNode extend, int value> {
-    def : Pat<(nxv16i8 (add ZPR:$op, (extend (nxv16i1 PPR:$pred)))),
+    def : Pat<(nxv16i8 (add ZPR:$op, (extend nxv16i1:$pred))),
               (ADD_ZPmZ_B PPR:$pred, ZPR:$op, (DUP_ZI_B value, 0))>;
-    def : Pat<(nxv8i16 (add ZPR:$op, (extend (nxv8i1 PPR:$pred)))),
+    def : Pat<(nxv8i16 (add ZPR:$op, (extend nxv8i1:$pred))),
               (ADD_ZPmZ_H PPR:$pred, ZPR:$op, (DUP_ZI_H value, 0))>;
-    def : Pat<(nxv4i32 (add ZPR:$op, (extend (nxv4i1 PPR:$pred)))),
+    def : Pat<(nxv4i32 (add ZPR:$op, (extend nxv4i1:$pred))),
               (ADD_ZPmZ_S PPR:$pred, ZPR:$op, (DUP_ZI_S value, 0))>;
-    def : Pat<(nxv2i64 (add ZPR:$op, (extend (nxv2i1 PPR:$pred)))),
+    def : Pat<(nxv2i64 (add ZPR:$op, (extend nxv2i1:$pred))),
               (ADD_ZPmZ_D PPR:$pred, ZPR:$op, (DUP_ZI_D value, 0))>;
   }
 
   defm : sve_predicated_add<zext, 1>;
   defm : sve_predicated_add<sext, 255>;
 
-  def : Pat<(nxv16i8 (sub ZPR:$op, (sext (nxv16i1 PPR:$pred)))),
+  def : Pat<(nxv16i8 (sub ZPR:$op, (sext nxv16i1:$pred))),
             (SUB_ZPmZ_B PPR:$pred, ZPR:$op, (DUP_ZI_B 255, 0))>;
-  def : Pat<(nxv8i16 (sub ZPR:$op, (sext (nxv8i1 PPR:$pred)))),
+  def : Pat<(nxv8i16 (sub ZPR:$op, (sext nxv8i1:$pred))),
             (SUB_ZPmZ_H PPR:$pred, ZPR:$op, (DUP_ZI_H 255, 0))>;
-  def : Pat<(nxv4i32 (sub ZPR:$op, (sext (nxv4i1 PPR:$pred)))),
+  def : Pat<(nxv4i32 (sub ZPR:$op, (sext nxv4i1:$pred))),
             (SUB_ZPmZ_S PPR:$pred, ZPR:$op, (DUP_ZI_S 255, 0))>;
-  def : Pat<(nxv2i64 (sub ZPR:$op, (sext (nxv2i1 PPR:$pred)))),
+  def : Pat<(nxv2i64 (sub ZPR:$op, (sext nxv2i1:$pred))),
             (SUB_ZPmZ_D PPR:$pred, ZPR:$op, (DUP_ZI_D 255, 0))>;
 } // End HasSVEorSME
 
@@ -3995,8 +3995,7 @@ defm STNT1D_4Z_IMM : sve2p1_mem_cst_si_4z<"stnt1d", 0b11, 0b1, ZZZZ_d_mul_r>;
 
 multiclass store_pn_x2<ValueType Ty, SDPatternOperator Store,
                         Instruction RegImmInst> {
-  def : Pat<(Store (Ty ZPR:$vec0), (Ty ZPR:$vec1),
-                   (aarch64svcount PNR:$PNg), GPR64:$base),
+  def : Pat<(Store Ty:$vec0, Ty:$vec1, aarch64svcount:$PNg, GPR64:$base),
             (RegImmInst (REG_SEQUENCE ZPR2Mul2, Ty:$vec0, zsub0, Ty:$vec1, zsub1),
                          PNR:$PNg, GPR64:$base, (i64 0))>;
 }
@@ -4021,8 +4020,7 @@ defm : store_pn_x2<nxv2f64, int_aarch64_sve_stnt1_pn_x2, STNT1D_2Z_IMM>;
 
 multiclass store_pn_x4<ValueType Ty, SDPatternOperator Store,
                         Instruction RegImmInst> {
-  def : Pat<(Store (Ty ZPR:$vec0), (Ty ZPR:$vec1), (Ty ZPR:$vec2), (Ty ZPR:$vec3),
-                   (aarch64svcount PNR:$PNg), GPR64:$base),
+  def : Pat<(Store Ty:$vec0, Ty:$vec1, Ty:$vec2, Ty:$vec3, aarch64svcount:$PNg, GPR64:$base),
             (RegImmInst (REG_SEQUENCE ZPR4Mul4, Ty:$vec0, zsub0, Ty:$vec1, zsub1,
                                                 Ty:$vec2, zsub2, Ty:$vec3, zsub3),
                         PNR:$PNg, GPR64:$base, (i64 0))>;
-- 
GitLab


From 5ad7a210ff850eed1f255f81f0609efedabe3bb7 Mon Sep 17 00:00:00 2001
From: Tomas Matheson <tomas.matheson@arm.com>
Date: Fri, 10 May 2024 12:14:11 +0100
Subject: [PATCH 0398/1206] [AArch64] wfxt must depend on itself (#90987)

---
 llvm/lib/Target/AArch64/AArch64Features.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 755a1bdc8e23..3425ac413302 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -597,7 +597,7 @@ def FeatureXS : SubtargetFeature<"xs", "HasXS",
 
 def FeatureWFxT : Extension<"wfxt", "WFxT",
     "Enable Armv8.7-A WFET and WFIT instruction (FEAT_WFxT)", [],
-    "FEAT_WFXT", "", 550>;
+    "FEAT_WFXT", "+wfxt", 550>;
 
 def FeatureHCX : SubtargetFeature<
     "hcx", "HasHCX", "true", "Enable Armv8.7-A HCRX_EL2 system register (FEAT_HCX)">;
-- 
GitLab


From f52ca632787a5d4227689726e14dae6749e1e650 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 10 May 2024 12:26:45 +0100
Subject: [PATCH 0399/1206] [LAA] Drop x86_64 target triple to fix test on
 builds with X86.

Follow-up o fix test after 28767afd53353d9333b0adf6f0fafa1592092532.
---
 .../LoopAccessAnalysis/multiple-strides-rt-memory-checks.ll    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/test/Analysis/LoopAccessAnalysis/multiple-strides-rt-memory-checks.ll b/llvm/test/Analysis/LoopAccessAnalysis/multiple-strides-rt-memory-checks.ll
index d96a6ea7c555..cb50b2c75ccb 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/multiple-strides-rt-memory-checks.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/multiple-strides-rt-memory-checks.ll
@@ -23,14 +23,13 @@
 
 ; CHECK: function 'Test':
 ; CHECK:   .inner:
-; CHECK-NEXT:     Memory dependences are safe with a maximum safe vector width of 2048 bits with run-time checks
+; CHECK-NEXT:     Memory dependences are safe with run-time checks
 ; CHECK-NEXT:     Dependences:
 ; CHECK-NEXT:     Run-time memory checks:
 ; CHECK:          Check 0:
 ; CHECK:          Check 1:
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
 
 %struct.s = type { [32 x i32], [32 x i32], [32 x [32 x i32]] }
 
-- 
GitLab


From 200f3bd39562f4d605f13567398025d30fa27d61 Mon Sep 17 00:00:00 2001
From: Qizhi Hu <836744285@qq.com>
Date: Fri, 10 May 2024 20:14:08 +0800
Subject: [PATCH 0400/1206] [Clang][Sema] access checking of friend declaration
 should not be delayed (#91430)

attempt to fix https://github.com/llvm/llvm-project/issues/12361
Consider this example:
```cpp
class D {
    class E{
        class F{};
        friend  void foo(D::E::F& q);
        };
    friend  void foo(D::E::F& q);
    };

void foo(D::E::F& q) {}
```
The first friend declaration of foo is correct. After that, the second
friend declaration delayed access checking and set its previous
declaration to be the first one. When doing access checking of `F`(which
is private filed of `E`), we put its canonical declaration(the first
friend declaration) into `EffectiveContext.Functions`. Actually, we are
still checking the first one. This is incorrect due to the delayed
checking.
Creating a new scope to indicate we are parsing a friend declaration and
doing access checking in time.
---
 clang/docs/ReleaseNotes.rst      |  1 +
 clang/include/clang/Sema/Scope.h |  6 ++++++
 clang/lib/Parse/ParseDecl.cpp    |  7 +++++--
 clang/lib/Sema/Scope.cpp         |  1 +
 clang/lib/Sema/SemaAccess.cpp    | 30 +++++++++++++++++++++++++-----
 clang/test/SemaCXX/PR12361.cpp   | 30 ++++++++++++++++++++++++++++++
 6 files changed, 68 insertions(+), 7 deletions(-)
 create mode 100644 clang/test/SemaCXX/PR12361.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index eef627ff2e31..7c5dcc59c701 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -706,6 +706,7 @@ Bug Fixes to C++ Support
   within initializers for variables that are usable in constant expressions or are constant
   initialized, rather than evaluating them as a part of the larger manifestly constant evaluated
   expression.
+- Fix a bug in access control checking due to dealyed checking of friend declaration. Fixes (#GH12361).
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Sema/Scope.h b/clang/include/clang/Sema/Scope.h
index 1752a25111a7..084db7303421 100644
--- a/clang/include/clang/Sema/Scope.h
+++ b/clang/include/clang/Sema/Scope.h
@@ -159,6 +159,9 @@ public:
 
     /// This is a scope of type alias declaration.
     TypeAliasScope = 0x20000000,
+
+    /// This is a scope of friend declaration.
+    FriendScope = 0x40000000,
   };
 
 private:
@@ -586,6 +589,9 @@ public:
   /// Determine whether this scope is a type alias scope.
   bool isTypeAliasScope() const { return getFlags() & Scope::TypeAliasScope; }
 
+  /// Determine whether this scope is a friend scope.
+  bool isFriendScope() const { return getFlags() & Scope::FriendScope; }
+
   /// Returns if rhs has a higher scope depth than this.
   ///
   /// The caller is responsible for calling this only if one of the two scopes
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 2c11ae693c35..5b5fc02ad402 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -4331,9 +4331,12 @@ void Parser::ParseDeclarationSpecifiers(
 
     // friend
     case tok::kw_friend:
-      if (DSContext == DeclSpecContext::DSC_class)
+      if (DSContext == DeclSpecContext::DSC_class) {
         isInvalid = DS.SetFriendSpec(Loc, PrevSpec, DiagID);
-      else {
+        Scope *CurS = getCurScope();
+        if (!isInvalid && CurS)
+          CurS->setFlags(CurS->getFlags() | Scope::FriendScope);
+      } else {
         PrevSpec = ""; // not actually used by the diagnostic
         DiagID = diag::err_friend_invalid_in_context;
         isInvalid = true;
diff --git a/clang/lib/Sema/Scope.cpp b/clang/lib/Sema/Scope.cpp
index 11a41753a1bd..c08073e80ff3 100644
--- a/clang/lib/Sema/Scope.cpp
+++ b/clang/lib/Sema/Scope.cpp
@@ -229,6 +229,7 @@ void Scope::dumpImpl(raw_ostream &OS) const {
       {ClassInheritanceScope, "ClassInheritanceScope"},
       {CatchScope, "CatchScope"},
       {OpenACCComputeConstructScope, "OpenACCComputeConstructScope"},
+      {FriendScope, "FriendScope"},
   };
 
   for (auto Info : FlagInfo) {
diff --git a/clang/lib/Sema/SemaAccess.cpp b/clang/lib/Sema/SemaAccess.cpp
index 6a707eeb66d0..979a64b065f3 100644
--- a/clang/lib/Sema/SemaAccess.cpp
+++ b/clang/lib/Sema/SemaAccess.cpp
@@ -1473,12 +1473,32 @@ static Sema::AccessResult CheckAccess(Sema &S, SourceLocation Loc,
   // specifier, like this:
   //   A::private_type A::foo() { ... }
   //
-  // Or we might be parsing something that will turn out to be a friend:
-  //   void foo(A::private_type);
-  //   void B::foo(A::private_type);
+  // friend declaration should not be delayed because it may lead to incorrect
+  // redeclaration chain, such as:
+  //   class D {
+  //    class E{
+  //     class F{};
+  //     friend  void foo(D::E::F& q);
+  //    };
+  //    friend  void foo(D::E::F& q);
+  //   };
   if (S.DelayedDiagnostics.shouldDelayDiagnostics()) {
-    S.DelayedDiagnostics.add(DelayedDiagnostic::makeAccess(Loc, Entity));
-    return Sema::AR_delayed;
+    // [class.friend]p9:
+    // A member nominated by a friend declaration shall be accessible in the
+    // class containing the friend declaration. The meaning of the friend
+    // declaration is the same whether the friend declaration appears in the
+    // private, protected, or public ([class.mem]) portion of the class
+    // member-specification.
+    Scope *TS = S.getCurScope();
+    bool IsFriendDeclaration = false;
+    while (TS && !IsFriendDeclaration) {
+      IsFriendDeclaration = TS->isFriendScope();
+      TS = TS->getParent();
+    }
+    if (!IsFriendDeclaration) {
+      S.DelayedDiagnostics.add(DelayedDiagnostic::makeAccess(Loc, Entity));
+      return Sema::AR_delayed;
+    }
   }
 
   EffectiveContext EC(S.CurContext);
diff --git a/clang/test/SemaCXX/PR12361.cpp b/clang/test/SemaCXX/PR12361.cpp
new file mode 100644
index 000000000000..95ceb45b7ba0
--- /dev/null
+++ b/clang/test/SemaCXX/PR12361.cpp
@@ -0,0 +1,30 @@
+ // RUN: %clang_cc1 -fsyntax-only -verify -std=c++98 %s
+ // RUN: %clang_cc1 -fsyntax-only -verify -std=c++17 %s
+ 
+class D {
+    class E{
+        class F{}; // expected-note{{implicitly declared private here}}
+        friend  void foo(D::E::F& q);
+        };
+    friend  void foo(D::E::F& q); // expected-error{{'F' is a private member of 'D::E'}}
+    };
+
+void foo(D::E::F& q) {}
+
+class D1 {
+    class E1{
+        class F1{}; // expected-note{{implicitly declared private here}}
+        friend  D1::E1::F1 foo1();
+        };
+    friend  D1::E1::F1 foo1(); // expected-error{{'F1' is a private member of 'D1::E1'}}
+    };
+
+D1::E1::F1 foo1() { return D1::E1::F1(); }
+
+class D2 {
+    class E2{
+        class F2{};
+        friend  void foo2();
+        };
+    friend  void foo2(){ D2::E2::F2 c;}
+    };
-- 
GitLab


From 8c852ab57932a5cd954cb0d050c3d2ab486428df Mon Sep 17 00:00:00 2001
From: Younan Zhang <zyn7109@gmail.com>
Date: Fri, 10 May 2024 20:47:15 +0800
Subject: [PATCH 0401/1206] [Clang][Sema] Revise the transformation of CTAD
 parameters of nested class templates (#91628)

This fixes a regression introduced by bee78b88f.

When we form a deduction guide for a constructor, basically, we do the
following work:
- Collect template parameters from the constructor's surrounding class
template, if present.
- Collect template parameters from the constructor.
- Splice these template parameters together into a new template
parameter list.
- Turn all the references (e.g. the function parameter list) to the
invented parameter list by applying a `TreeTransform` to the function
type.

In the previous fix, we handled cases of nested class templates by
substituting the "outer" template parameters (i.e. those not declared at
the surrounding class template or the constructor) with the
instantiating template arguments. The approach per se makes sense, but
there was a flaw in the following case:

```cpp
template <typename U, typename... Us> struct X {
  template <typename V> struct Y {
    template <typename T> Y(T) {}
  };

  template <typename T> Y(T) -> Y<T>;
};

X<int>::Y y(42);
```

While we're transforming the parameters for `Y(T)`, we first attempt to
transform all references to `V` and `T`; then, we handle the references
to outer parameters `U` and `Us` using the template arguments from
`X<int>` by transforming the same `ParamDecl`. However, the first step
results in the reference `T` being `<template-param-0-1>` because the
invented `T` is the last of the parameter list of the deduction guide,
and what we're substituting with is a corresponding parameter pack
(which is `Us`, though empty). Hence we're messing up the substitution.

I think we can resolve it by reversing the substitution order, which
means handling outer template parameters first and then the inner
parameters.

There's no release note because this is a regression in 18, and I hope
we can catch up with the last release.

Fixes https://github.com/llvm/llvm-project/issues/88142
---
 clang/lib/Sema/SemaTemplate.cpp               | 25 ++++++++++++++-----
 .../nested-implicit-deduction-guides.cpp      | 14 +++++++++++
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 2fce2238f9c1..480c0103ae33 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -2492,9 +2492,6 @@ struct ConvertConstructorToDeductionGuideTransform {
       Args.addOuterRetainedLevel();
     }
 
-    if (NestedPattern)
-      Args.addOuterRetainedLevels(NestedPattern->getTemplateDepth());
-
     FunctionProtoTypeLoc FPTL = CD->getTypeSourceInfo()->getTypeLoc()
                                    .getAsAdjusted<FunctionProtoTypeLoc>();
     assert(FPTL && "no prototype for constructor declaration");
@@ -2584,11 +2581,27 @@ private:
 
     //    -- The types of the function parameters are those of the constructor.
     for (auto *OldParam : TL.getParams()) {
-      ParmVarDecl *NewParam =
-          transformFunctionTypeParam(OldParam, Args, MaterializedTypedefs);
-      if (NestedPattern && NewParam)
+      ParmVarDecl *NewParam = OldParam;
+      // Given
+      //   template <class T> struct C {
+      //     template <class U> struct D {
+      //       template <class V> D(U, V);
+      //     };
+      //   };
+      // First, transform all the references to template parameters that are
+      // defined outside of the surrounding class template. That is T in the
+      // above example.
+      if (NestedPattern) {
         NewParam = transformFunctionTypeParam(NewParam, OuterInstantiationArgs,
                                               MaterializedTypedefs);
+        if (!NewParam)
+          return QualType();
+      }
+      // Then, transform all the references to template parameters that are
+      // defined at the class template and the constructor. In this example,
+      // they're U and V, respectively.
+      NewParam =
+          transformFunctionTypeParam(NewParam, Args, MaterializedTypedefs);
       if (!NewParam)
         return QualType();
       ParamTypes.push_back(NewParam->getType());
diff --git a/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp b/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp
index 38b6706595a1..f289dc045286 100644
--- a/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp
+++ b/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp
@@ -84,3 +84,17 @@ nested_init_list<int>::concept_fail nil_invalid{1, ""};
 // expected-note@#INIT_LIST_INNER_INVALID {{candidate template ignored: substitution failure [with F = const char *]: constraints not satisfied for class template 'concept_fail' [with F = const char *]}}
 // expected-note@#INIT_LIST_INNER_INVALID {{candidate function template not viable: requires 1 argument, but 2 were provided}}
 // expected-note@#INIT_LIST_INNER_INVALID {{candidate function template not viable: requires 0 arguments, but 2 were provided}}
+
+namespace GH88142 {
+
+template <typename, typename...> struct X {
+  template <typename> struct Y {
+    template <typename T> Y(T) {}
+  };
+
+  template <typename T> Y(T) -> Y<T>;
+};
+
+X<int>::Y y(42);
+
+} // namespace PR88142
-- 
GitLab


From 452f4393c70e0ffecf8e394f82b1eaeaa8d224af Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Fri, 10 May 2024 13:47:22 +0100
Subject: [PATCH 0402/1206] [InstCombine] Precommit tests for #86111

The upcoming patch adds logic to prefer to use constants close to
power-of-two in these ashr exact + slt/ult patterns when it has a choice
on which constant can be used.
---
 .../Transforms/InstCombine/icmp-shr-lt-gt.ll  | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll b/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
index 1b8efe4351c6..4dd5b0925914 100644
--- a/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
@@ -3800,3 +3800,64 @@ define i1 @ashrslt_03_15_exact(i4 %x) {
   ret i1 %c
 }
 
+; TODO: The resulting compared constant can be safely replaced with one that
+; is closer to a power of two.
+define i1 @ashr_slt_exact_near_pow2_cmpval(i8 %x) {
+; CHECK-LABEL: @ashr_slt_exact_near_pow2_cmpval(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X:%.*]], 10
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i8 %x, 1
+  %c = icmp slt i8 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashr_ult_exact_near_pow2_cmpval(i8 %x) {
+; CHECK-LABEL: @ashr_ult_exact_near_pow2_cmpval(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[X:%.*]], 10
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i8 %x, 1
+  %c = icmp ult i8 %s, 5
+  ret i1 %c
+}
+
+define i1 @negtest_near_pow2_cmpval_ashr_slt_noexact(i8 %x) {
+; CHECK-LABEL: @negtest_near_pow2_cmpval_ashr_slt_noexact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X:%.*]], 10
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i8 %x, 1
+  %c = icmp slt i8 %s, 5
+  ret i1 %c
+}
+
+define i1 @negtest_near_pow2_cmpval_ashr_wrong_cmp_pred(i8 %x) {
+; CHECK-LABEL: @negtest_near_pow2_cmpval_ashr_wrong_cmp_pred(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[X:%.*]], 10
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i8 %x, 1
+  %c = icmp eq i8 %s, 5
+  ret i1 %c
+}
+
+define i1 @negtest_near_pow2_cmpval_isnt_close_to_pow2(i8 %x) {
+; CHECK-LABEL: @negtest_near_pow2_cmpval_isnt_close_to_pow2(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X:%.*]], 12
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i8 %x, 1
+  %c = icmp slt i8 %s, 6
+  ret i1 %c
+}
+
+define i1 @negtest_near_pow2_cmpval_would_overflow_into_signbit(i8 %x) {
+; CHECK-LABEL: @negtest_near_pow2_cmpval_would_overflow_into_signbit(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i8 %x, 2
+  %c = icmp ult i8 %s, 33
+  ret i1 %c
+}
-- 
GitLab


From 3be8e2c95d3dca5b2fdea889649a69dce8605e65 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Fri, 10 May 2024 13:50:03 +0100
Subject: [PATCH 0403/1206] [InstCombine] Prefer to keep power-of-2 constants
 when combining ashr exact and slt/ult of a constant (#86111)

We have flexibility in what constant to use when combining an `ashr
exact` with a slt or ult of a constant, and it's not possible to revisit
this decision later in the compilation pipeline after the `ashr exact`
is removed. Keeping a constant close to power-of-2 (pow2val + 1) should
be no worse than neutral, and in some cases may allow better codegen
later on for targets that can more cheaply generate power of 2 (which
may be selectable if converting back to setle/setge) or near power of 2
constants.

Alive2 proofs:
<https://alive2.llvm.org/ce/z/2BmPnq> and
<https://alive2.llvm.org/ce/z/DtuhnR>
---
 .../lib/Transforms/InstCombine/InstCombineCompares.cpp | 10 ++++++++++
 llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll     | 10 ++++------
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index e1a3194a1beb..9883d02c87a3 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -2479,6 +2479,16 @@ Instruction *InstCombinerImpl::foldICmpShrConstant(ICmpInst &Cmp,
   // those conditions rather than checking them. This is difficult because of
   // undef/poison (PR34838).
   if (IsAShr && Shr->hasOneUse()) {
+    if (IsExact && (Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) &&
+        (C - 1).isPowerOf2() && C.countLeadingZeros() > ShAmtVal) {
+      // When C - 1 is a power of two and the transform can be legally
+      // performed, prefer this form so the produced constant is close to a
+      // power of two.
+      // icmp slt/ult (ashr exact X, ShAmtC), C
+      // --> icmp slt/ult X, (C - 1) << ShAmtC) + 1
+      APInt ShiftedC = (C - 1).shl(ShAmtVal) + 1;
+      return new ICmpInst(Pred, X, ConstantInt::get(ShrTy, ShiftedC));
+    }
     if (IsExact || Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) {
       // When ShAmtC can be shifted losslessly:
       // icmp PRED (ashr exact X, ShAmtC), C --> icmp PRED X, (C << ShAmtC)
diff --git a/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll b/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
index 4dd5b0925914..5f09964fd93a 100644
--- a/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
@@ -3379,7 +3379,7 @@ define i1 @ashrslt_01_01_exact(i4 %x) {
 
 define i1 @ashrslt_01_02_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_02_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 4
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -3389,7 +3389,7 @@ define i1 @ashrslt_01_02_exact(i4 %x) {
 
 define i1 @ashrslt_01_03_exact(i4 %x) {
 ; CHECK-LABEL: @ashrslt_01_03_exact(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 6
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 [[X:%.*]], 5
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i4 %x, 1
@@ -3800,11 +3800,9 @@ define i1 @ashrslt_03_15_exact(i4 %x) {
   ret i1 %c
 }
 
-; TODO: The resulting compared constant can be safely replaced with one that
-; is closer to a power of two.
 define i1 @ashr_slt_exact_near_pow2_cmpval(i8 %x) {
 ; CHECK-LABEL: @ashr_slt_exact_near_pow2_cmpval(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X:%.*]], 10
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X:%.*]], 9
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i8 %x, 1
@@ -3814,7 +3812,7 @@ define i1 @ashr_slt_exact_near_pow2_cmpval(i8 %x) {
 
 define i1 @ashr_ult_exact_near_pow2_cmpval(i8 %x) {
 ; CHECK-LABEL: @ashr_ult_exact_near_pow2_cmpval(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[X:%.*]], 10
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[X:%.*]], 9
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %s = ashr exact i8 %x, 1
-- 
GitLab


From d48bf8aef2abeb915b1e04e1b78051869088df42 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk@dianqk.net>
Date: Fri, 10 May 2024 19:19:51 +0800
Subject: [PATCH 0404/1206] Reapply "[InlineCost] Correct the default branch
 cost for the switch statement (#85160)"

This reverts commit c6e4f6309184814dfc4bb855ddbdb5375cc971e0.
---
 llvm/lib/Analysis/InlineCost.cpp              |  25 ++--
 .../Inline/inline-cost-switch-default.ll      | 130 ++++++++++++++++++
 .../Inline/inline-switch-default-2.ll         |  21 +--
 .../Inline/inline-switch-default.ll           |  26 +---
 4 files changed, 153 insertions(+), 49 deletions(-)
 create mode 100644 llvm/test/Transforms/Inline/inline-cost-switch-default.ll

diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index a531064e304d..f5b17dca4973 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -701,21 +701,26 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
 
   void onFinalizeSwitch(unsigned JumpTableSize, unsigned NumCaseCluster,
                         bool DefaultDestUndefined) override {
-    if (!DefaultDestUndefined)
-      addCost(2 * InstrCost);
     // If suitable for a jump table, consider the cost for the table size and
     // branch to destination.
     // Maximum valid cost increased in this function.
     if (JumpTableSize) {
+      // Suppose a default branch includes one compare and one conditional
+      // branch if it's reachable.
+      if (!DefaultDestUndefined)
+        addCost(2 * InstrCost);
+      // Suppose a jump table requires one load and one jump instruction.
       int64_t JTCost =
-          static_cast<int64_t>(JumpTableSize) * InstrCost + 4 * InstrCost;
+          static_cast<int64_t>(JumpTableSize) * InstrCost + 2 * InstrCost;
       addCost(JTCost);
       return;
     }
 
     if (NumCaseCluster <= 3) {
       // Suppose a comparison includes one compare and one conditional branch.
-      addCost(NumCaseCluster * 2 * InstrCost);
+      // We can reduce a set of instructions if the default branch is
+      // undefined.
+      addCost((NumCaseCluster - DefaultDestUndefined) * 2 * InstrCost);
       return;
     }
 
@@ -1152,7 +1157,7 @@ private:
   // FIXME: These constants are taken from the heuristic-based cost visitor.
   // These should be removed entirely in a later revision to avoid reliance on
   // heuristics in the ML inliner.
-  static constexpr int JTCostMultiplier = 4;
+  static constexpr int JTCostMultiplier = 2;
   static constexpr int CaseClusterCostMultiplier = 2;
   static constexpr int SwitchDefaultDestCostMultiplier = 2;
   static constexpr int SwitchCostMultiplier = 2;
@@ -1235,11 +1240,10 @@ private:
 
   void onFinalizeSwitch(unsigned JumpTableSize, unsigned NumCaseCluster,
                         bool DefaultDestUndefined) override {
-    if (!DefaultDestUndefined)
-      increment(InlineCostFeatureIndex::switch_default_dest_penalty,
-                SwitchDefaultDestCostMultiplier * InstrCost);
-
     if (JumpTableSize) {
+      if (!DefaultDestUndefined)
+        increment(InlineCostFeatureIndex::switch_default_dest_penalty,
+                  SwitchDefaultDestCostMultiplier * InstrCost);
       int64_t JTCost = static_cast<int64_t>(JumpTableSize) * InstrCost +
                        JTCostMultiplier * InstrCost;
       increment(InlineCostFeatureIndex::jump_table_penalty, JTCost);
@@ -1248,7 +1252,8 @@ private:
 
     if (NumCaseCluster <= 3) {
       increment(InlineCostFeatureIndex::case_cluster_penalty,
-                NumCaseCluster * CaseClusterCostMultiplier * InstrCost);
+                (NumCaseCluster - DefaultDestUndefined) *
+                    CaseClusterCostMultiplier * InstrCost);
       return;
     }
 
diff --git a/llvm/test/Transforms/Inline/inline-cost-switch-default.ll b/llvm/test/Transforms/Inline/inline-cost-switch-default.ll
new file mode 100644
index 000000000000..e3768ac8233a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-cost-switch-default.ll
@@ -0,0 +1,130 @@
+; RUN: opt -S -passes=inline %s -debug-only=inline-cost -min-jump-table-entries=4 --disable-output 2>&1 | FileCheck %s -check-prefix=LOOKUPTABLE -match-full-lines
+; RUN: opt -S -passes=inline %s -debug-only=inline-cost -min-jump-table-entries=5 --disable-output 2>&1 | FileCheck %s -check-prefix=SWITCH -match-full-lines
+; REQUIRES: x86_64-linux, asserts
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i64 @main(i64 %a) {
+  %b = call i64 @small_switch_default(i64 %a)
+  %c = call i64 @small_switch_no_default(i64 %a)
+  %d = call i64 @lookup_table_default(i64 %a)
+  %e = call i64 @lookup_table_no_default(i64 %a)
+  ret i64 %b
+}
+
+; SWITCH-LABEL: Analyzing call of small_switch_default{{.*}}
+; SWITCH: Cost: 0
+define i64 @small_switch_default(i64 %a) {
+  switch i64 %a, label %default_branch [
+  i64 -1, label %branch_0
+  i64 8, label %branch_1
+  i64 52, label %branch_2
+  ]
+
+branch_0:
+  br label %exit
+
+branch_1:
+  br label %exit
+
+branch_2:
+  br label %exit
+
+default_branch:
+  br label %exit
+
+exit:
+  %b = phi i64 [ 5, %branch_0 ], [ 9, %branch_1 ], [ 2, %branch_2 ], [ 3, %default_branch ]
+  ret i64 %b
+}
+
+; SWITCH-LABEL: Analyzing call of small_switch_no_default{{.*}}
+; SWITCH: Cost: -10
+define i64 @small_switch_no_default(i64 %a) {
+  switch i64 %a, label %unreachabledefault [
+  i64 -1, label %branch_0
+  i64 8, label %branch_1
+  i64 52, label %branch_2
+  ]
+
+branch_0:
+  br label %exit
+
+branch_1:
+  br label %exit
+
+branch_2:
+  br label %exit
+
+unreachabledefault:
+  unreachable
+
+exit:
+  %b = phi i64 [ 5, %branch_0 ], [ 9, %branch_1 ], [ 2, %branch_2 ]
+  ret i64 %b
+}
+
+; LOOKUPTABLE-LABEL: Analyzing call of lookup_table_default{{.*}}
+; LOOKUPTABLE: Cost: 10
+; SWITCH-LABEL: Analyzing call of lookup_table_default{{.*}}
+; SWITCH: Cost: 20
+define i64 @lookup_table_default(i64 %a) {
+  switch i64 %a, label %default_branch [
+  i64 0, label %branch_0
+  i64 1, label %branch_1
+  i64 2, label %branch_2
+  i64 3, label %branch_3
+  ]
+
+branch_0:
+  br label %exit
+
+branch_1:
+  br label %exit
+
+branch_2:
+  br label %exit
+
+branch_3:
+  br label %exit
+
+default_branch:
+  br label %exit
+
+exit:
+  %b = phi i64 [ 5, %branch_0 ], [ 9, %branch_1 ], [ 2, %branch_2 ], [ 7, %branch_3 ], [ 3, %default_branch ]
+  ret i64 %b
+}
+
+; LOOKUPTABLE-LABEL: Analyzing call of lookup_table_no_default{{.*}}
+; LOOKUPTABLE: Cost: 0
+; SWITCH-LABEL: Analyzing call of lookup_table_no_default{{.*}}
+; SWITCH: Cost: 20
+define i64 @lookup_table_no_default(i64 %a) {
+  switch i64 %a, label %unreachabledefault [
+  i64 0, label %branch_0
+  i64 1, label %branch_1
+  i64 2, label %branch_2
+  i64 3, label %branch_3
+  ]
+
+branch_0:
+  br label %exit
+
+branch_1:
+  br label %exit
+
+branch_2:
+  br label %exit
+
+branch_3:
+  br label %exit
+
+unreachabledefault:
+  unreachable
+
+exit:
+  %b = phi i64 [ 5, %branch_0 ], [ 9, %branch_1 ], [ 2, %branch_2 ], [ 7, %branch_3 ]
+  ret i64 %b
+}
diff --git a/llvm/test/Transforms/Inline/inline-switch-default-2.ll b/llvm/test/Transforms/Inline/inline-switch-default-2.ll
index 82dae1c27648..169cb2cff9b8 100644
--- a/llvm/test/Transforms/Inline/inline-switch-default-2.ll
+++ b/llvm/test/Transforms/Inline/inline-switch-default-2.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt %s -S -passes=inline -inline-threshold=21 | FileCheck %s
+; RUN: opt %s -S -passes=inline -inline-threshold=11 | FileCheck %s
 
 ; Check for scenarios without TTI.
 
@@ -16,24 +16,7 @@ define i64 @foo1(i64 %a) {
 define i64 @foo2(i64 %a) {
 ; CHECK-LABEL: define i64 @foo2(
 ; CHECK-SAME: i64 [[A:%.*]]) {
-; CHECK-NEXT:    switch i64 [[A]], label [[UNREACHABLEDEFAULT_I:%.*]] [
-; CHECK-NEXT:      i64 0, label [[BRANCH_0_I:%.*]]
-; CHECK-NEXT:      i64 2, label [[BRANCH_2_I:%.*]]
-; CHECK-NEXT:      i64 4, label [[BRANCH_4_I:%.*]]
-; CHECK-NEXT:      i64 6, label [[BRANCH_6_I:%.*]]
-; CHECK-NEXT:    ]
-; CHECK:       branch_0.i:
-; CHECK-NEXT:    br label [[BAR2_EXIT:%.*]]
-; CHECK:       branch_2.i:
-; CHECK-NEXT:    br label [[BAR2_EXIT]]
-; CHECK:       branch_4.i:
-; CHECK-NEXT:    br label [[BAR2_EXIT]]
-; CHECK:       branch_6.i:
-; CHECK-NEXT:    br label [[BAR2_EXIT]]
-; CHECK:       unreachabledefault.i:
-; CHECK-NEXT:    unreachable
-; CHECK:       bar2.exit:
-; CHECK-NEXT:    [[B_I:%.*]] = phi i64 [ 5, [[BRANCH_0_I]] ], [ 9, [[BRANCH_2_I]] ], [ 2, [[BRANCH_4_I]] ], [ 7, [[BRANCH_6_I]] ]
+; CHECK-NEXT:    [[B_I:%.*]] = call i64 @bar2(i64 [[A]])
 ; CHECK-NEXT:    ret i64 [[B_I]]
 ;
   %b = call i64 @bar2(i64 %a)
diff --git a/llvm/test/Transforms/Inline/inline-switch-default.ll b/llvm/test/Transforms/Inline/inline-switch-default.ll
index 44f1304e82df..288d414fe0e0 100644
--- a/llvm/test/Transforms/Inline/inline-switch-default.ll
+++ b/llvm/test/Transforms/Inline/inline-switch-default.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt %s -S -passes=inline -inline-threshold=26 -min-jump-table-entries=4 | FileCheck %s -check-prefix=LOOKUPTABLE
-; RUN: opt %s -S -passes=inline -inline-threshold=21 -min-jump-table-entries=5 | FileCheck %s -check-prefix=SWITCH
+; RUN: opt %s -S -passes=inline -inline-threshold=16 -min-jump-table-entries=4 | FileCheck %s -check-prefix=LOOKUPTABLE
+; RUN: opt %s -S -passes=inline -inline-threshold=11 -min-jump-table-entries=5 | FileCheck %s -check-prefix=SWITCH
+; REQUIRES: x86_64-linux
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -22,6 +23,8 @@ define i64 @foo1(i64 %a) {
   ret i64 %b
 }
 
+; Since the default branch is undefined behavior,
+; we can inline `bar2`: https://github.com/llvm/llvm-project/issues/90929
 define i64 @foo2(i64 %a) {
 ; LOOKUPTABLE-LABEL: define i64 @foo2(
 ; LOOKUPTABLE-SAME: i64 [[A:%.*]]) {
@@ -47,24 +50,7 @@ define i64 @foo2(i64 %a) {
 ;
 ; SWITCH-LABEL: define i64 @foo2(
 ; SWITCH-SAME: i64 [[A:%.*]]) {
-; SWITCH-NEXT:    switch i64 [[A]], label [[UNREACHABLEDEFAULT_I:%.*]] [
-; SWITCH-NEXT:      i64 0, label [[BRANCH_0_I:%.*]]
-; SWITCH-NEXT:      i64 2, label [[BRANCH_2_I:%.*]]
-; SWITCH-NEXT:      i64 4, label [[BRANCH_4_I:%.*]]
-; SWITCH-NEXT:      i64 6, label [[BRANCH_6_I:%.*]]
-; SWITCH-NEXT:    ]
-; SWITCH:       branch_0.i:
-; SWITCH-NEXT:    br label [[BAR2_EXIT:%.*]]
-; SWITCH:       branch_2.i:
-; SWITCH-NEXT:    br label [[BAR2_EXIT]]
-; SWITCH:       branch_4.i:
-; SWITCH-NEXT:    br label [[BAR2_EXIT]]
-; SWITCH:       branch_6.i:
-; SWITCH-NEXT:    br label [[BAR2_EXIT]]
-; SWITCH:       unreachabledefault.i:
-; SWITCH-NEXT:    unreachable
-; SWITCH:       bar2.exit:
-; SWITCH-NEXT:    [[B_I:%.*]] = phi i64 [ 5, [[BRANCH_0_I]] ], [ 9, [[BRANCH_2_I]] ], [ 2, [[BRANCH_4_I]] ], [ 7, [[BRANCH_6_I]] ]
+; SWITCH-NEXT:    [[B_I:%.*]] = call i64 @bar2(i64 [[A]])
 ; SWITCH-NEXT:    ret i64 [[B_I]]
 ;
   %b = call i64 @bar2(i64 %a)
-- 
GitLab


From 561b6ab96e9d5b38a5d2672e6cc6823389b75a3f Mon Sep 17 00:00:00 2001
From: Xing Xue <xingxue@outlook.com>
Date: Fri, 10 May 2024 09:23:02 -0400
Subject: [PATCH 0405/1206] [OpenMP][AIX] Implement __kmp_get_load_balance()
 for AIX (#91520)

AIX has the `/proc` filesystem where `/proc/<pid>/lwp/<tid>/lwpsinfo` has
the thread state in binary, similar to Linux's
`/proc/<pid>/task/<tid>/stat` where the state is in ASCII. However, the
definition of state info `R` in `lwpsinfo` is `runnable`. In Linux,
state `R` means the thread is `running`. Therefore, `lwpsinfo` is not
ideal for our purpose of getting the current load of the system. This
patch uses `perfstat_cpu()` in AIX system library `libperfstat.a` to
obtain the number of threads current running on logical CPUs.
---
 openmp/runtime/CMakeLists.txt       |  9 ++--
 openmp/runtime/src/z_Linux_util.cpp | 83 +++++++++++++++++++++++++----
 2 files changed, 80 insertions(+), 12 deletions(-)

diff --git a/openmp/runtime/CMakeLists.txt b/openmp/runtime/CMakeLists.txt
index 57ed54bcdc7b..bcae02eba6a5 100644
--- a/openmp/runtime/CMakeLists.txt
+++ b/openmp/runtime/CMakeLists.txt
@@ -132,10 +132,13 @@ set(LIBOMP_ASMFLAGS "" CACHE STRING
   "Appended user specified assembler flags.")
 set(LIBOMP_LDFLAGS "" CACHE STRING
   "Appended user specified linker flags.")
-if("${LIBOMP_ARCH}" STREQUAL "ppc" AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
-  # PPC (32-bit) on AIX needs libatomic for __atomic_load_8, etc.
-  set(LIBOMP_LIBFLAGS "-latomic" CACHE STRING
+if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+  set(LIBOMP_LIBFLAGS "-lperfstat" CACHE STRING
     "Appended user specified linked libs flags. (e.g., -lm)")
+  if("${LIBOMP_ARCH}" STREQUAL "ppc")
+    # PPC (32-bit) on AIX needs libatomic for __atomic_load_8, etc.
+    set(LIBOMP_LIBFLAGS "${LIBOMP_LIBFLAGS} -latomic")
+  endif()
 else()
   set(LIBOMP_LIBFLAGS "" CACHE STRING
     "Appended user specified linked libs flags. (e.g., -lm)")
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index affb577a5393..7c90740ae5bd 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -31,6 +31,7 @@
 #include <sys/resource.h>
 #if KMP_OS_AIX
 #include <sys/ldr.h>
+#include <libperfstat.h>
 #else
 #include <sys/syscall.h>
 #endif
@@ -2427,6 +2428,79 @@ int __kmp_get_load_balance(int max) {
   return ret_avg;
 }
 
+#elif KMP_OS_AIX
+
+// The function returns number of running (not sleeping) threads, or -1 in case
+// of error.
+int __kmp_get_load_balance(int max) {
+
+  static int glb_running_threads = 0; // Saved count of the running threads for
+                                      // the thread balance algorithm.
+  static double glb_call_time = 0; // Thread balance algorithm call time.
+  int running_threads = 0; // Number of running threads in the system.
+
+  double call_time = 0.0;
+
+  __kmp_elapsed(&call_time);
+
+  if (glb_call_time &&
+      (call_time - glb_call_time < __kmp_load_balance_interval))
+    return glb_running_threads;
+
+  glb_call_time = call_time;
+
+  if (max <= 0) {
+    max = INT_MAX;
+  }
+
+  // Check how many perfstat_cpu_t structures are available.
+  int logical_cpus = perfstat_cpu(NULL, NULL, sizeof(perfstat_cpu_t), 0);
+  if (logical_cpus <= 0) {
+    glb_call_time = -1;
+    return -1;
+  }
+
+  perfstat_cpu_t *cpu_stat = (perfstat_cpu_t *)KMP_INTERNAL_MALLOC(
+      logical_cpus * sizeof(perfstat_cpu_t));
+  if (cpu_stat == NULL) {
+    glb_call_time = -1;
+    return -1;
+  }
+
+  // Set first CPU as the name of the first logical CPU for which the info is
+  // desired.
+  perfstat_id_t first_cpu_name;
+  strcpy(first_cpu_name.name, FIRST_CPU);
+
+  // Get the stat info of logical CPUs.
+  int rc = perfstat_cpu(&first_cpu_name, cpu_stat, sizeof(perfstat_cpu_t),
+                        logical_cpus);
+  KMP_DEBUG_ASSERT(rc == logical_cpus);
+  if (rc <= 0) {
+    KMP_INTERNAL_FREE(cpu_stat);
+    glb_call_time = -1;
+    return -1;
+  }
+  for (int i = 0; i < logical_cpus; ++i) {
+    running_threads += cpu_stat[i].runque;
+    if (running_threads >= max)
+      break;
+  }
+
+  // There _might_ be a timing hole where the thread executing this
+  // code gets skipped in the load balance, and running_threads is 0.
+  // Assert in the debug builds only!!!
+  KMP_DEBUG_ASSERT(running_threads > 0);
+  if (running_threads <= 0)
+    running_threads = 1;
+
+  KMP_INTERNAL_FREE(cpu_stat);
+
+  glb_running_threads = running_threads;
+
+  return running_threads;
+}
+
 #else // Linux* OS
 
 // The function returns number of running (not sleeping) threads, or -1 in case
@@ -2498,14 +2572,9 @@ int __kmp_get_load_balance(int max) {
 
   proc_entry = readdir(proc_dir);
   while (proc_entry != NULL) {
-#if KMP_OS_AIX
-    // Proc entry name starts with a digit. Assume it is a  process' directory.
-    if (isdigit(proc_entry->d_name[0])) {
-#else
     // Proc entry is a directory and name starts with a digit. Assume it is a
     // process' directory.
     if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) {
-#endif
 
 #ifdef KMP_DEBUG
       ++total_processes;
@@ -2549,11 +2618,7 @@ int __kmp_get_load_balance(int max) {
         task_entry = readdir(task_dir);
         while (task_entry != NULL) {
           // It is a directory and name starts with a digit.
-#if KMP_OS_AIX
-          if (isdigit(task_entry->d_name[0])) {
-#else
           if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) {
-#endif
 
             // Construct complete stat file path. Easiest way would be:
             //  __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str,
-- 
GitLab


From 195d8ac26d91ca798733c3a5f58d67992d43503d Mon Sep 17 00:00:00 2001
From: Xiang Li <python3kgae@outlook.com>
Date: Fri, 10 May 2024 06:29:23 -0700
Subject: [PATCH 0406/1206] [DirectX] Fix DXIL part header version encoding
 (#91506)

Move MinorVersion be the lower 8 bit.
Set DXIL version in DXContainerObjectWriter::writeObject.


Fixes #89952
---
 llvm/include/llvm/BinaryFormat/DXContainer.h |  2 +-
 llvm/include/llvm/TargetParser/Triple.h      |  2 +-
 llvm/lib/MC/MCDXContainerWriter.cpp          |  3 +
 llvm/lib/TargetParser/Triple.cpp             |  2 +
 llvm/test/CodeGen/DirectX/embed-dxil.ll      |  4 +-
 llvm/unittests/Object/DXContainerTest.cpp    | 59 +++++++++++++++++---
 6 files changed, 61 insertions(+), 11 deletions(-)

diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index e8d03f806715..e5fcda63910d 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -103,8 +103,8 @@ struct PartHeader {
 
 struct BitcodeHeader {
   uint8_t Magic[4];     // ACSII "DXIL".
-  uint8_t MajorVersion; // DXIL version.
   uint8_t MinorVersion; // DXIL version.
+  uint8_t MajorVersion; // DXIL version.
   uint16_t Unused;
   uint32_t Offset; // Offset to LLVM bitcode (from start of header).
   uint32_t Size;   // Size of LLVM bitcode (in bytes).
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 8f9d99816931..b3bb354b38ff 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -429,7 +429,7 @@ public:
   /// (SubArch).  This should only be called with Vulkan SPIR-V triples.
   VersionTuple getVulkanVersion() const;
 
-  /// Parse the DXIL version number from the DXIL version
+  /// Parse the DXIL version number from the OSVersion and DXIL version
   /// (SubArch).  This should only be called with DXIL triples.
   VersionTuple getDXILVersion() const;
 
diff --git a/llvm/lib/MC/MCDXContainerWriter.cpp b/llvm/lib/MC/MCDXContainerWriter.cpp
index 0580dc7e4282..015899278f37 100644
--- a/llvm/lib/MC/MCDXContainerWriter.cpp
+++ b/llvm/lib/MC/MCDXContainerWriter.cpp
@@ -127,6 +127,9 @@ uint64_t DXContainerObjectWriter::writeObject(MCAssembler &Asm,
       // The program header's size field is in 32-bit words.
       Header.Size = (SectionSize + sizeof(dxbc::ProgramHeader) + 3) / 4;
       memcpy(Header.Bitcode.Magic, "DXIL", 4);
+      VersionTuple DXILVersion = TT.getDXILVersion();
+      Header.Bitcode.MajorVersion = DXILVersion.getMajor();
+      Header.Bitcode.MinorVersion = DXILVersion.getMinor().value_or(0);
       Header.Bitcode.Offset = sizeof(dxbc::BitcodeHeader);
       Header.Bitcode.Size = SectionSize;
       if (sys::IsBigEndianHost)
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index f8269a51dc0b..4fc1ff5aaa05 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -1510,6 +1510,8 @@ VersionTuple Triple::getDXILVersion() const {
   if (getArch() != dxil || getOS() != ShaderModel)
     llvm_unreachable("invalid DXIL triple");
   StringRef Arch = getArchName();
+  if (getSubArch() == NoSubArch)
+    Arch = getDXILArchNameFromShaderModel(getOSName());
   Arch.consume_front("dxilv");
   VersionTuple DXILVersion = parseVersionFromName(Arch);
   // FIXME: validate DXIL version against Shader Model version.
diff --git a/llvm/test/CodeGen/DirectX/embed-dxil.ll b/llvm/test/CodeGen/DirectX/embed-dxil.ll
index 306e5c385b5a..9f4fb19d86fa 100644
--- a/llvm/test/CodeGen/DirectX/embed-dxil.ll
+++ b/llvm/test/CodeGen/DirectX/embed-dxil.ll
@@ -42,8 +42,8 @@ define i32 @add(i32 %a, i32 %b) {
 ; DXC-NEXT:       MinorVersion:    5
 ; DXC-NEXT:       ShaderKind:      6
 ; DXC-NEXT:       Size:            [[#div(SIZE,4)]]
-; DXC-NEXT:       DXILMajorVersion: [[#]]
-; DXC-NEXT:       DXILMinorVersion: [[#]]
+; DXC-NEXT:       DXILMajorVersion: 1
+; DXC-NEXT:       DXILMinorVersion: 5
 ; DXC-NEXT:       DXILSize:        [[#SIZE - 24]]
 ; DXC-NEXT:       DXIL:            [ 0x42, 0x43, 0xC0, 0xDE,
 ; DXC:      - Name:            SFI0
diff --git a/llvm/unittests/Object/DXContainerTest.cpp b/llvm/unittests/Object/DXContainerTest.cpp
index da640225617d..098da331ab56 100644
--- a/llvm/unittests/Object/DXContainerTest.cpp
+++ b/llvm/unittests/Object/DXContainerTest.cpp
@@ -126,6 +126,51 @@ TEST(DXCFile, ParseOverlappingParts) {
           "Part offset for part 1 begins before the previous part ends"));
 }
 
+// This test verify DXILMajorVersion and DXILMinorVersion are correctly parsed.
+// This test is based on the binary output constructed from this yaml.
+// --- !dxcontainer
+// Header:
+//   Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+//                      0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+//   Version:
+//     Major:           1
+//     Minor:           0
+//   PartCount:       1
+// Parts:
+//   - Name:            DXIL
+//     Size:            28
+//     Program:
+//       MajorVersion:    6
+//       MinorVersion:    5
+//       ShaderKind:      5
+//       Size:            8
+//       DXILMajorVersion: 1
+//       DXILMinorVersion: 5
+//       DXILSize:        4
+//       DXIL:            [ 0x42, 0x43, 0xC0, 0xDE, ]
+// ...
+TEST(DXCFile, ParseDXILPart) {
+  uint8_t Buffer[] = {
+      0x44, 0x58, 0x42, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+      0x48, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
+      0x44, 0x58, 0x49, 0x4c, 0x1c, 0x00, 0x00, 0x00, 0x65, 0x00, 0x05, 0x00,
+      0x08, 0x00, 0x00, 0x00, 0x44, 0x58, 0x49, 0x4c, 0x05, 0x01, 0x00, 0x00,
+      0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x42, 0x43, 0xc0, 0xde};
+  DXContainer C =
+      llvm::cantFail(DXContainer::create(getMemoryBuffer<116>(Buffer)));
+  EXPECT_EQ(C.getHeader().PartCount, 1u);
+  const std::optional<object::DXContainer::DXILData> &DXIL = C.getDXIL();
+  EXPECT_TRUE(DXIL.has_value());
+  dxbc::ProgramHeader Header = DXIL->first;
+  EXPECT_EQ(Header.MajorVersion, 6u);
+  EXPECT_EQ(Header.MinorVersion, 5u);
+  EXPECT_EQ(Header.ShaderKind, 5u);
+  EXPECT_EQ(Header.Size, 8u);
+  EXPECT_EQ(Header.Bitcode.MajorVersion, 1u);
+  EXPECT_EQ(Header.Bitcode.MinorVersion, 5u);
+}
+
 TEST(DXCFile, ParseEmptyParts) {
   uint8_t Buffer[] = {
       0x44, 0x58, 0x42, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -240,8 +285,8 @@ Parts:
       MinorVersion:    0
       ShaderKind:      14
       Size:            6
-      DXILMajorVersion: 0
-      DXILMinorVersion: 1
+      DXILMajorVersion: 1
+      DXILMinorVersion: 0
       DXILSize:        0
 ...
 )";
@@ -361,8 +406,8 @@ Parts:
 //       MinorVersion:    0
 //       ShaderKind:      14
 //       Size:            6
-//       DXILMajorVersion: 0
-//       DXILMinorVersion: 1
+//       DXILMajorVersion: 1
+//       DXILMinorVersion: 0
 //       DXILSize:        0
 //   - Name:            PSV0
 //     Size:            36
@@ -477,7 +522,7 @@ TEST(DXCFile, MaliciousFiles) {
 //
 // --- !dxcontainer
 // Header:
-//   Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+//   Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
 //                      0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
 //   Version:
 //     Major:           1
@@ -491,8 +536,8 @@ TEST(DXCFile, MaliciousFiles) {
 //       MinorVersion:    0
 //       ShaderKind:      14
 //       Size:            6
-//       DXILMajorVersion: 0
-//       DXILMinorVersion: 1
+//       DXILMajorVersion: 1
+//       DXILMinorVersion: 0
 //       DXILSize:        0
 //   - Name:            PSV0
 //     Size:            100
-- 
GitLab


From 4cf3f032283d8426c9b7829c7ccf0ab01939c7db Mon Sep 17 00:00:00 2001
From: Congcong Cai <congcongcai0907@163.com>
Date: Fri, 10 May 2024 21:51:07 +0800
Subject: [PATCH 0407/1206] [clang-tidy] use llvm::any_of refactor
 getAnalyzerCheckersAndPackages [NFC] (#91713)

---
 clang-tools-extra/clang-tidy/ClangTidy.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index b877ea06dc05..1cd7cdd10bc2 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -373,11 +373,11 @@ static CheckersList getAnalyzerCheckersAndPackages(ClangTidyContext &Context,
 
   const auto &RegisteredCheckers =
       AnalyzerOptions::getRegisteredCheckers(IncludeExperimental);
-  bool AnalyzerChecksEnabled = false;
-  for (StringRef CheckName : RegisteredCheckers) {
-    std::string ClangTidyCheckName((AnalyzerCheckNamePrefix + CheckName).str());
-    AnalyzerChecksEnabled |= Context.isCheckEnabled(ClangTidyCheckName);
-  }
+  const bool AnalyzerChecksEnabled =
+      llvm::any_of(RegisteredCheckers, [&](StringRef CheckName) -> bool {
+        return Context.isCheckEnabled(
+            (AnalyzerCheckNamePrefix + CheckName).str());
+      });
 
   if (!AnalyzerChecksEnabled)
     return List;
-- 
GitLab


From 8fc9e3d577c02d2b97c952fbafb75db0100462a9 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Fri, 10 May 2024 14:58:48 +0100
Subject: [PATCH 0408/1206] [DAG] Lower frem of power-2 using div/trunc/mul+sub
 (#91148)

If we are lowering a frem and the divisor is known to be an integer power-2, we
can use the formula 'frem = x - trunc(x / d) * d'. This avoids the more
expensive call to fmod. The results are identical as fmod so long as d is a
power-2 (so the mul does not round incorrectly), and the sign of the return is
either always positive or not important for zeroes (nsz).

Unfortunately Alive2 does not handle this well at the moment. I was using
exhaustive checking to test this:
(https://gist.github.com/davemgreen/6078015f30d3bacd1e9572f8db5d4b64).

I found this in cpythons implementation of float_pow. I currently added it as a
DAG combine for frem with power-2 fp constants.
---
 llvm/include/llvm/CodeGen/SelectionDAG.h      |   8 +
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  20 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  17 +
 llvm/test/CodeGen/AArch64/frem-power2.ll      | 373 ++++++++++--------
 llvm/test/CodeGen/ARM/frem-power2.ll          |  18 +-
 5 files changed, 273 insertions(+), 163 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 4b1b58d4af0b..c08e57ba3f67 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1996,6 +1996,10 @@ public:
   /// is set.
   bool isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth = 0) const;
 
+  /// Test if the given _fp_ value is known to be an integer power-of-2, either
+  /// positive or negative.
+  bool isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth = 0) const;
+
   /// Return the number of times the sign bit of the register is replicated into
   /// the other bits. We know that at least 1 bit is always equal to the sign
   /// bit (itself), but other cases can give us information. For example,
@@ -2111,6 +2115,10 @@ public:
   /// Test whether the given SDValue is known to contain non-zero value(s).
   bool isKnownNeverZero(SDValue Op, unsigned Depth = 0) const;
 
+  /// Test whether the given float value is known to be positive. +0.0, +inf and
+  /// +nan are considered positive, -0.0, -inf and -nan are not.
+  bool cannotBeOrderedNegativeFP(SDValue Op) const;
+
   /// Test whether two SDValues are known to compare equal. This
   /// is true if they are the same value, or if one is negative zero and the
   /// other positive zero.
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index fddc97d8901a..be919b7a8922 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -17365,17 +17365,35 @@ SDValue DAGCombiner::visitFREM(SDNode *N) {
   EVT VT = N->getValueType(0);
   SDNodeFlags Flags = N->getFlags();
   SelectionDAG::FlagInserter FlagsInserter(DAG, N);
+  SDLoc DL(N);
 
   if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
     return R;
 
   // fold (frem c1, c2) -> fmod(c1,c2)
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FREM, SDLoc(N), VT, {N0, N1}))
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FREM, DL, VT, {N0, N1}))
     return C;
 
   if (SDValue NewSel = foldBinOpIntoSelect(N))
     return NewSel;
 
+  // Lower frem N0, N1 => x - trunc(N0 / N1) * N1, providing N1 is an integer
+  // power of 2.
+  if (!TLI.isOperationLegal(ISD::FREM, VT) &&
+      TLI.isOperationLegalOrCustom(ISD::FMUL, VT) &&
+      TLI.isOperationLegalOrCustom(ISD::FDIV, VT) &&
+      TLI.isOperationLegalOrCustom(ISD::FTRUNC, VT) &&
+      DAG.isKnownToBeAPowerOfTwoFP(N1) &&
+      (Flags.hasNoSignedZeros() || DAG.cannotBeOrderedNegativeFP(N0))) {
+    SDValue Div = DAG.getNode(ISD::FDIV, DL, VT, N0, N1);
+    SDValue Rnd = DAG.getNode(ISD::FTRUNC, DL, VT, Div);
+    if (TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT))
+      return DAG.getNode(ISD::FMA, DL, VT, DAG.getNode(ISD::FNEG, DL, VT, Rnd),
+                         N1, N0);
+    SDValue Mul = DAG.getNode(ISD::FMUL, DL, VT, Rnd, N1);
+    return DAG.getNode(ISD::FSUB, DL, VT, N0, Mul);
+  }
+
   return SDValue();
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index eef5acd03234..9c1f3c1e3431 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -4373,6 +4373,16 @@ bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const {
   return false;
 }
 
+bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
+  if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(Val, true))
+    return C1->getValueAPF().getExactLog2Abs() >= 0;
+
+  if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
+    return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
+
+  return false;
+}
+
 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
   EVT VT = Op.getValueType();
 
@@ -5555,6 +5565,13 @@ bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
   return computeKnownBits(Op, Depth).isNonZero();
 }
 
+bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
+  if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(Op, true))
+    return !C1->isNegative();
+
+  return Op.getOpcode() == ISD::FABS;
+}
+
 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
   // Check the obvious case.
   if (A == B) return true;
diff --git a/llvm/test/CodeGen/AArch64/frem-power2.ll b/llvm/test/CodeGen/AArch64/frem-power2.ll
index 5d627fcd6b65..402e03c5e265 100644
--- a/llvm/test/CodeGen/AArch64/frem-power2.ll
+++ b/llvm/test/CodeGen/AArch64/frem-power2.ll
@@ -13,31 +13,57 @@ entry:
 }
 
 define float @frem2_nsz(float %x) {
-; CHECK-LABEL: frem2_nsz:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s1, #2.00000000
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem2_nsz:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, #2.00000000
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem2_nsz:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %fmod = frem nsz float %x, 2.0
   ret float %fmod
 }
 
 define float @frem2_fast(float %x) {
-; CHECK-LABEL: frem2_fast:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s1, #2.00000000
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem2_fast:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, #0.50000000
+; CHECK-SD-NEXT:    fmov s2, #-2.00000000
+; CHECK-SD-NEXT:    fmul s1, s0, s1
+; CHECK-SD-NEXT:    frintz s1, s1
+; CHECK-SD-NEXT:    fmadd s0, s1, s2, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem2_fast:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %fmod = frem fast float %x, 2.0
   ret float %fmod
 }
 
 define float @frem2_abs(float %x) {
-; CHECK-LABEL: frem2_abs:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fabs s0, s0
-; CHECK-NEXT:    fmov s1, #2.00000000
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem2_abs:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fabs s0, s0
+; CHECK-SD-NEXT:    fmov s1, #2.00000000
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem2_abs:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fabs s0, s0
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %a = tail call float @llvm.fabs.f32(float %x)
   %fmod = frem float %a, 2.0
@@ -47,14 +73,11 @@ entry:
 define half @hrem2_nsz(half %x) {
 ; CHECK-SD-LABEL: hrem2_nsz:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    fcvt s0, h0
-; CHECK-SD-NEXT:    fmov s1, #2.00000000
-; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    fcvt h0, s0
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    fmov h1, #2.00000000
+; CHECK-SD-NEXT:    fmov h2, #-2.00000000
+; CHECK-SD-NEXT:    fdiv h1, h0, h1
+; CHECK-SD-NEXT:    frintz h1, h1
+; CHECK-SD-NEXT:    fmadd h0, h1, h2, h0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: hrem2_nsz:
@@ -75,10 +98,18 @@ entry:
 }
 
 define double @drem2_nsz(double %x) {
-; CHECK-LABEL: drem2_nsz:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov d1, #2.00000000
-; CHECK-NEXT:    b fmod
+; CHECK-SD-LABEL: drem2_nsz:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov d1, #2.00000000
+; CHECK-SD-NEXT:    fdiv d2, d0, d1
+; CHECK-SD-NEXT:    frintz d2, d2
+; CHECK-SD-NEXT:    fmsub d0, d2, d1, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: drem2_nsz:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov d1, #2.00000000
+; CHECK-GI-NEXT:    b fmod
 entry:
   %fmod = frem nsz double %x, 2.0
   ret double %fmod
@@ -105,10 +136,16 @@ entry:
 }
 
 define float @frem1_nsz(float %x) {
-; CHECK-LABEL: frem1_nsz:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s1, #1.00000000
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem1_nsz:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    frintz s1, s0
+; CHECK-SD-NEXT:    fsub s0, s0, s1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem1_nsz:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s1, #1.00000000
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %fmod = frem nsz float %x, 1.0
   ret float %fmod
@@ -125,21 +162,38 @@ entry:
 }
 
 define float @fremm2_nsz(float %x) {
-; CHECK-LABEL: fremm2_nsz:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s1, #-2.00000000
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: fremm2_nsz:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, #-2.00000000
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fremm2_nsz:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s1, #-2.00000000
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %fmod = frem nsz float %x, -2.0
   ret float %fmod
 }
 
 define float @frem4_abs(float %x) {
-; CHECK-LABEL: frem4_abs:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fabs s0, s0
-; CHECK-NEXT:    fmov s1, #4.00000000
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem4_abs:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fabs s0, s0
+; CHECK-SD-NEXT:    fmov s1, #4.00000000
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem4_abs:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fabs s0, s0
+; CHECK-GI-NEXT:    fmov s1, #4.00000000
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %a = tail call float @llvm.fabs.f32(float %x)
   %fmod = frem float %a, 4.0
@@ -147,11 +201,20 @@ entry:
 }
 
 define float @frem16_abs(float %x) {
-; CHECK-LABEL: frem16_abs:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fabs s0, s0
-; CHECK-NEXT:    fmov s1, #16.00000000
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem16_abs:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fabs s0, s0
+; CHECK-SD-NEXT:    fmov s1, #16.00000000
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem16_abs:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fabs s0, s0
+; CHECK-GI-NEXT:    fmov s1, #16.00000000
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %a = tail call float @llvm.fabs.f32(float %x)
   %fmod = frem float %a, 16.0
@@ -159,12 +222,22 @@ entry:
 }
 
 define float @frem4294967296_abs(float %x) {
-; CHECK-LABEL: frem4294967296_abs:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fabs s0, s0
-; CHECK-NEXT:    mov w8, #1333788672 // =0x4f800000
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem4294967296_abs:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fabs s0, s0
+; CHECK-SD-NEXT:    mov w8, #1333788672 // =0x4f800000
+; CHECK-SD-NEXT:    fmov s1, w8
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem4294967296_abs:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fabs s0, s0
+; CHECK-GI-NEXT:    mov w8, #1333788672 // =0x4f800000
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %a = tail call float @llvm.fabs.f32(float %x)
   %fmod = frem float %a, 4294967296.0
@@ -172,12 +245,22 @@ entry:
 }
 
 define float @frem1152921504606846976_abs(float %x) {
-; CHECK-LABEL: frem1152921504606846976_abs:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fabs s0, s0
-; CHECK-NEXT:    mov w8, #1568669696 // =0x5d800000
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem1152921504606846976_abs:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fabs s0, s0
+; CHECK-SD-NEXT:    mov w8, #1568669696 // =0x5d800000
+; CHECK-SD-NEXT:    fmov s1, w8
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem1152921504606846976_abs:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fabs s0, s0
+; CHECK-GI-NEXT:    mov w8, #1568669696 // =0x5d800000
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %a = tail call float @llvm.fabs.f32(float %x)
   %fmod = frem float %a, 1152921504606846976.0
@@ -185,12 +268,22 @@ entry:
 }
 
 define float @frem4611686018427387904_abs(float %x) {
-; CHECK-LABEL: frem4611686018427387904_abs:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fabs s0, s0
-; CHECK-NEXT:    mov w8, #1585446912 // =0x5e800000
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem4611686018427387904_abs:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fabs s0, s0
+; CHECK-SD-NEXT:    mov w8, #1585446912 // =0x5e800000
+; CHECK-SD-NEXT:    fmov s1, w8
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem4611686018427387904_abs:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fabs s0, s0
+; CHECK-GI-NEXT:    mov w8, #1585446912 // =0x5e800000
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %a = tail call float @llvm.fabs.f32(float %x)
   %fmod = frem float %a, 4611686018427387904.0
@@ -198,11 +291,20 @@ entry:
 }
 
 define float @frem9223372036854775808_abs(float %x) {
-; CHECK-LABEL: frem9223372036854775808_abs:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fabs s0, s0
-; CHECK-NEXT:    movi v1.2s, #95, lsl #24
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem9223372036854775808_abs:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.2s, #95, lsl #24
+; CHECK-SD-NEXT:    fabs s0, s0
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem9223372036854775808_abs:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fabs s0, s0
+; CHECK-GI-NEXT:    movi v1.2s, #95, lsl #24
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %a = tail call float @llvm.fabs.f32(float %x)
   %fmod = frem float %a, 9223372036854775808.0
@@ -212,42 +314,10 @@ entry:
 define <4 x float> @frem2_nsz_vec(<4 x float> %x) {
 ; CHECK-SD-LABEL: frem2_nsz_vec:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    mov s0, v0.s[1]
-; CHECK-SD-NEXT:    fmov s1, #2.00000000
-; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    fmov s1, #2.00000000
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-SD-NEXT:    fmov s1, #2.00000000
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov s0, v0.s[2]
-; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    mov v1.s[2], v0.s[0]
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov s0, v0.s[3]
-; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    fmov s1, #2.00000000
-; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v1.s[3], v0.s[0]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    add sp, sp, #48
+; CHECK-SD-NEXT:    movi v1.4s, #64, lsl #24
+; CHECK-SD-NEXT:    fdiv v2.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    frintz v2.4s, v2.4s
+; CHECK-SD-NEXT:    fmls v0.4s, v1.4s, v2.4s
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem2_nsz_vec:
@@ -302,48 +372,12 @@ entry:
 define <4 x float> @frem1152921504606846976_absv(<4 x float> %x) {
 ; CHECK-SD-LABEL: frem1152921504606846976_absv:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str d8, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -8
-; CHECK-SD-NEXT:    .cfi_offset b8, -16
-; CHECK-SD-NEXT:    fabs v0.4s, v0.4s
 ; CHECK-SD-NEXT:    mov w8, #1568669696 // =0x5d800000
-; CHECK-SD-NEXT:    fmov s8, w8
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    mov s0, v0.s[1]
-; CHECK-SD-NEXT:    fmov s1, s8
-; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    fmov s1, s8
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-SD-NEXT:    fmov s1, s8
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov s0, v0.s[2]
-; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    mov v1.s[2], v0.s[0]
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov s0, v0.s[3]
-; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    fmov s1, s8
-; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ldr d8, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v1.s[3], v0.s[0]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    add sp, sp, #48
+; CHECK-SD-NEXT:    fabs v0.4s, v0.4s
+; CHECK-SD-NEXT:    dup v1.4s, w8
+; CHECK-SD-NEXT:    fdiv v2.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    frintz v2.4s, v2.4s
+; CHECK-SD-NEXT:    fmls v0.4s, v1.4s, v2.4s
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem1152921504606846976_absv:
@@ -401,12 +435,22 @@ entry:
 }
 
 define float @frem2_nsz_sitofp(float %x, i32 %sa) {
-; CHECK-LABEL: frem2_nsz_sitofp:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #1 // =0x1
-; CHECK-NEXT:    lsl w8, w8, w0
-; CHECK-NEXT:    scvtf s1, w8
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem2_nsz_sitofp:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #1 // =0x1
+; CHECK-SD-NEXT:    lsl w8, w8, w0
+; CHECK-SD-NEXT:    scvtf s1, w8
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem2_nsz_sitofp:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    lsl w8, w8, w0
+; CHECK-GI-NEXT:    scvtf s1, w8
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %s = shl i32 1, %sa
   %y = sitofp i32 %s to float
@@ -415,12 +459,22 @@ entry:
 }
 
 define float @frem2_nsz_uitofp(float %x, i32 %sa) {
-; CHECK-LABEL: frem2_nsz_uitofp:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #1 // =0x1
-; CHECK-NEXT:    lsl w8, w8, w0
-; CHECK-NEXT:    ucvtf s1, w8
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem2_nsz_uitofp:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #1 // =0x1
+; CHECK-SD-NEXT:    lsl w8, w8, w0
+; CHECK-SD-NEXT:    ucvtf s1, w8
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem2_nsz_uitofp:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    lsl w8, w8, w0
+; CHECK-GI-NEXT:    ucvtf s1, w8
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %s = shl i32 1, %sa
   %y = uitofp i32 %s to float
@@ -432,10 +486,13 @@ define float @frem2_const_sitofp(float %x, i32 %sa) {
 ; CHECK-SD-LABEL: frem2_const_sitofp:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    mov w8, #1 // =0x1
-; CHECK-SD-NEXT:    fmov s0, #12.50000000
+; CHECK-SD-NEXT:    fmov s1, #12.50000000
 ; CHECK-SD-NEXT:    lsl w8, w8, w0
-; CHECK-SD-NEXT:    scvtf s1, w8
-; CHECK-SD-NEXT:    b fmodf
+; CHECK-SD-NEXT:    scvtf s0, w8
+; CHECK-SD-NEXT:    fdiv s2, s1, s0
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s0, s1
+; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem2_const_sitofp:
 ; CHECK-GI:       // %bb.0: // %entry
diff --git a/llvm/test/CodeGen/ARM/frem-power2.ll b/llvm/test/CodeGen/ARM/frem-power2.ll
index 8052c8c35bcf..7f52943175ac 100644
--- a/llvm/test/CodeGen/ARM/frem-power2.ll
+++ b/llvm/test/CodeGen/ARM/frem-power2.ll
@@ -37,13 +37,23 @@ define float @frem4_nsz(float %x) {
 ;
 ; CHECK-FP-LABEL: frem4_nsz:
 ; CHECK-FP:       @ %bb.0: @ %entry
-; CHECK-FP-NEXT:    mov.w r1, #1082130432
-; CHECK-FP-NEXT:    b fmodf
+; CHECK-FP-NEXT:    vmov.f32 s0, #4.000000e+00
+; CHECK-FP-NEXT:    vmov s2, r0
+; CHECK-FP-NEXT:    vdiv.f32 s4, s2, s0
+; CHECK-FP-NEXT:    vrintz.f32 s4, s4
+; CHECK-FP-NEXT:    vfms.f32 s2, s4, s0
+; CHECK-FP-NEXT:    vmov r0, s2
+; CHECK-FP-NEXT:    bx lr
 ;
 ; CHECK-M33-LABEL: frem4_nsz:
 ; CHECK-M33:       @ %bb.0: @ %entry
-; CHECK-M33-NEXT:    mov.w r1, #1082130432
-; CHECK-M33-NEXT:    b fmodf
+; CHECK-M33-NEXT:    vmov.f32 s0, #4.000000e+00
+; CHECK-M33-NEXT:    vmov s2, r0
+; CHECK-M33-NEXT:    vdiv.f32 s4, s2, s0
+; CHECK-M33-NEXT:    vrintz.f32 s4, s4
+; CHECK-M33-NEXT:    vmls.f32 s2, s4, s0
+; CHECK-M33-NEXT:    vmov r0, s2
+; CHECK-M33-NEXT:    bx lr
 entry:
   %fmod = frem nsz float %x, 4.0
   ret float %fmod
-- 
GitLab


From 63177422a834f4b81d59b827b5f2a1c5a9083749 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Fri, 10 May 2024 07:02:10 -0700
Subject: [PATCH 0409/1206] [OpenACC][NFC] Fix isa behavior for OpenACC types

I discovered while working on a different patch that I'd not implemented
the 'classof' for any of the Clauses, which resulted in 'isa' always
returning 'true' for all of the types.  This patch goes through all the
existing clauses and adds 'classof' such that it will work correctly.

Additionally, in doing this, I found a bug where I was doing a cast to
the wrong type in the ASTWriter, so this fixes that problem as well.
---
 clang/include/clang/AST/OpenACCClause.h | 71 ++++++++++++++++++++++++-
 clang/include/clang/AST/StmtOpenACC.h   |  4 ++
 clang/lib/AST/OpenACCClause.cpp         | 27 ++++++++++
 clang/lib/Serialization/ASTWriter.cpp   |  2 +-
 4 files changed, 102 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index d8332816a499..3d0b1ab9d31e 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -36,7 +36,7 @@ public:
   SourceLocation getBeginLoc() const { return Location.getBegin(); }
   SourceLocation getEndLoc() const { return Location.getEnd(); }
 
-  static bool classof(const OpenACCClause *) { return true; }
+  static bool classof(const OpenACCClause *) { return false; }
 
   using child_iterator = StmtIterator;
   using const_child_iterator = ConstStmtIterator;
@@ -63,6 +63,8 @@ protected:
       : OpenACCClause(K, BeginLoc, EndLoc), LParenLoc(LParenLoc) {}
 
 public:
+  static bool classof(const OpenACCClause *C);
+
   SourceLocation getLParenLoc() const { return LParenLoc; }
 
   child_range children() {
@@ -92,6 +94,9 @@ protected:
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Default;
+  }
   OpenACCDefaultClauseKind getDefaultClauseKind() const {
     return DefaultClauseKind;
   }
@@ -116,6 +121,8 @@ protected:
         ConditionExpr(ConditionExpr) {}
 
 public:
+  static bool classof(const OpenACCClause *C);
+
   bool hasConditionExpr() const { return ConditionExpr; }
   const Expr *getConditionExpr() const { return ConditionExpr; }
   Expr *getConditionExpr() { return ConditionExpr; }
@@ -143,6 +150,9 @@ protected:
                   Expr *ConditionExpr, SourceLocation EndLoc);
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::If;
+  }
   static OpenACCIfClause *Create(const ASTContext &C, SourceLocation BeginLoc,
                                  SourceLocation LParenLoc, Expr *ConditionExpr,
                                  SourceLocation EndLoc);
@@ -154,6 +164,9 @@ class OpenACCSelfClause : public OpenACCClauseWithCondition {
                     Expr *ConditionExpr, SourceLocation EndLoc);
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Self;
+  }
   static OpenACCSelfClause *Create(const ASTContext &C, SourceLocation BeginLoc,
                                    SourceLocation LParenLoc,
                                    Expr *ConditionExpr, SourceLocation EndLoc);
@@ -180,6 +193,7 @@ protected:
   llvm::ArrayRef<Expr *> getExprs() const { return Exprs; }
 
 public:
+  static bool classof(const OpenACCClause *C);
   child_range children() {
     return child_range(reinterpret_cast<Stmt **>(Exprs.begin()),
                        reinterpret_cast<Stmt **>(Exprs.end()));
@@ -214,6 +228,9 @@ class OpenACCWaitClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Wait;
+  }
   static OpenACCWaitClause *Create(const ASTContext &C, SourceLocation BeginLoc,
                                    SourceLocation LParenLoc, Expr *DevNumExpr,
                                    SourceLocation QueuesLoc,
@@ -246,6 +263,9 @@ class OpenACCNumGangsClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::NumGangs;
+  }
   static OpenACCNumGangsClause *
   Create(const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
          ArrayRef<Expr *> IntExprs, SourceLocation EndLoc);
@@ -275,6 +295,7 @@ protected:
   }
 
 public:
+  static bool classof(const OpenACCClause *C);
   bool hasIntExpr() const { return !getExprs().empty(); }
   const Expr *getIntExpr() const {
     return hasIntExpr() ? getExprs()[0] : nullptr;
@@ -288,6 +309,9 @@ class OpenACCNumWorkersClause : public OpenACCClauseWithSingleIntExpr {
                           Expr *IntExpr, SourceLocation EndLoc);
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::NumWorkers;
+  }
   static OpenACCNumWorkersClause *Create(const ASTContext &C,
                                          SourceLocation BeginLoc,
                                          SourceLocation LParenLoc,
@@ -299,6 +323,9 @@ class OpenACCVectorLengthClause : public OpenACCClauseWithSingleIntExpr {
                             Expr *IntExpr, SourceLocation EndLoc);
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::VectorLength;
+  }
   static OpenACCVectorLengthClause *
   Create(const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
          Expr *IntExpr, SourceLocation EndLoc);
@@ -309,6 +336,9 @@ class OpenACCAsyncClause : public OpenACCClauseWithSingleIntExpr {
                      Expr *IntExpr, SourceLocation EndLoc);
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Async;
+  }
   static OpenACCAsyncClause *Create(const ASTContext &C,
                                     SourceLocation BeginLoc,
                                     SourceLocation LParenLoc, Expr *IntExpr,
@@ -326,6 +356,7 @@ protected:
       : OpenACCClauseWithExprs(K, BeginLoc, LParenLoc, EndLoc) {}
 
 public:
+  static bool classof(const OpenACCClause *C);
   ArrayRef<Expr *> getVarList() { return getExprs(); }
   ArrayRef<Expr *> getVarList() const { return getExprs(); }
 };
@@ -344,6 +375,9 @@ class OpenACCPrivateClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Private;
+  }
   static OpenACCPrivateClause *
   Create(const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
          ArrayRef<Expr *> VarList, SourceLocation EndLoc);
@@ -363,6 +397,9 @@ class OpenACCFirstPrivateClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::FirstPrivate;
+  }
   static OpenACCFirstPrivateClause *
   Create(const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
          ArrayRef<Expr *> VarList, SourceLocation EndLoc);
@@ -382,6 +419,9 @@ class OpenACCDevicePtrClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::DevicePtr;
+  }
   static OpenACCDevicePtrClause *
   Create(const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
          ArrayRef<Expr *> VarList, SourceLocation EndLoc);
@@ -401,6 +441,9 @@ class OpenACCAttachClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Attach;
+  }
   static OpenACCAttachClause *
   Create(const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
          ArrayRef<Expr *> VarList, SourceLocation EndLoc);
@@ -420,6 +463,9 @@ class OpenACCNoCreateClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::NoCreate;
+  }
   static OpenACCNoCreateClause *
   Create(const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
          ArrayRef<Expr *> VarList, SourceLocation EndLoc);
@@ -439,6 +485,9 @@ class OpenACCPresentClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Present;
+  }
   static OpenACCPresentClause *
   Create(const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
          ArrayRef<Expr *> VarList, SourceLocation EndLoc);
@@ -462,6 +511,11 @@ class OpenACCCopyClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Copy ||
+           C->getClauseKind() == OpenACCClauseKind::PCopy ||
+           C->getClauseKind() == OpenACCClauseKind::PresentOrCopy;
+  }
   static OpenACCCopyClause *
   Create(const ASTContext &C, OpenACCClauseKind Spelling,
          SourceLocation BeginLoc, SourceLocation LParenLoc,
@@ -488,6 +542,11 @@ class OpenACCCopyInClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::CopyIn ||
+           C->getClauseKind() == OpenACCClauseKind::PCopyIn ||
+           C->getClauseKind() == OpenACCClauseKind::PresentOrCopyIn;
+  }
   bool isReadOnly() const { return IsReadOnly; }
   static OpenACCCopyInClause *
   Create(const ASTContext &C, OpenACCClauseKind Spelling,
@@ -515,6 +574,11 @@ class OpenACCCopyOutClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::CopyOut ||
+           C->getClauseKind() == OpenACCClauseKind::PCopyOut ||
+           C->getClauseKind() == OpenACCClauseKind::PresentOrCopyOut;
+  }
   bool isZero() const { return IsZero; }
   static OpenACCCopyOutClause *
   Create(const ASTContext &C, OpenACCClauseKind Spelling,
@@ -542,6 +606,11 @@ class OpenACCCreateClause final
   }
 
 public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Create ||
+           C->getClauseKind() == OpenACCClauseKind::PCreate ||
+           C->getClauseKind() == OpenACCClauseKind::PresentOrCreate;
+  }
   bool isZero() const { return IsZero; }
   static OpenACCCreateClause *
   Create(const ASTContext &C, OpenACCClauseKind Spelling,
diff --git a/clang/include/clang/AST/StmtOpenACC.h b/clang/include/clang/AST/StmtOpenACC.h
index 66f8f844e0b2..b706864798ba 100644
--- a/clang/include/clang/AST/StmtOpenACC.h
+++ b/clang/include/clang/AST/StmtOpenACC.h
@@ -93,6 +93,10 @@ protected:
   }
 
 public:
+  static bool classof(const Stmt *T) {
+    return false;
+  }
+
   child_range children() {
     if (getAssociatedStmt())
       return child_range(&AssociatedStmt, &AssociatedStmt + 1);
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index be079556a87a..ee13437b97b4 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -17,6 +17,33 @@
 
 using namespace clang;
 
+bool OpenACCClauseWithParams::classof(const OpenACCClause *C) {
+  return OpenACCClauseWithCondition::classof(C) ||
+         OpenACCClauseWithExprs::classof(C);
+}
+bool OpenACCClauseWithExprs::classof(const OpenACCClause *C) {
+  return OpenACCWaitClause::classof(C) || OpenACCNumGangsClause::classof(C) ||
+         OpenACCClauseWithSingleIntExpr::classof(C) ||
+         OpenACCClauseWithVarList::classof(C);
+}
+bool OpenACCClauseWithVarList::classof(const OpenACCClause *C) {
+  return OpenACCPrivateClause::classof(C) ||
+         OpenACCFirstPrivateClause::classof(C) ||
+         OpenACCDevicePtrClause::classof(C) ||
+         OpenACCDevicePtrClause::classof(C) ||
+         OpenACCAttachClause::classof(C) || OpenACCNoCreateClause::classof(C) ||
+         OpenACCPresentClause::classof(C) || OpenACCCopyClause::classof(C) ||
+         OpenACCCopyInClause::classof(C) || OpenACCCopyOutClause::classof(C) ||
+         OpenACCCreateClause::classof(C);
+}
+bool OpenACCClauseWithCondition::classof(const OpenACCClause *C) {
+  return OpenACCIfClause::classof(C) || OpenACCSelfClause::classof(C);
+}
+bool OpenACCClauseWithSingleIntExpr::classof(const OpenACCClause *C) {
+  return OpenACCNumWorkersClause::classof(C) ||
+         OpenACCVectorLengthClause::classof(C) ||
+         OpenACCAsyncClause::classof(C);
+}
 OpenACCDefaultClause *OpenACCDefaultClause::Create(const ASTContext &C,
                                                    OpenACCDefaultClauseKind K,
                                                    SourceLocation BeginLoc,
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index e7b9050165bb..ab07cf3efa45 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -7816,7 +7816,7 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     return;
   }
   case OpenACCClauseKind::Self: {
-    const auto *SC = cast<OpenACCIfClause>(C);
+    const auto *SC = cast<OpenACCSelfClause>(C);
     writeSourceLocation(SC->getLParenLoc());
     writeBool(SC->hasConditionExpr());
     if (SC->hasConditionExpr())
-- 
GitLab


From 331f22af4b4c849a97c97e6803e0c8cab57fa10b Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Fri, 10 May 2024 15:04:38 +0100
Subject: [PATCH 0410/1206] [AMDGPU] Remove unnecessary predicates from
 aliases. NFC. (#91602)

So long as the target of the alias is predicated with HasImageInsts or
similar, the alias itself does not need this predicate.
---
 llvm/lib/Target/AMDGPU/EXPInstructions.td  | 1 -
 llvm/lib/Target/AMDGPU/MIMGInstructions.td | 1 -
 2 files changed, 2 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/EXPInstructions.td b/llvm/lib/Target/AMDGPU/EXPInstructions.td
index e02652e62a57..5e426b5acd50 100644
--- a/llvm/lib/Target/AMDGPU/EXPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/EXPInstructions.td
@@ -125,7 +125,6 @@ multiclass VEXPORT_Real_gfx12 {
   }
   def : AMDGPUMnemonicAlias<"exp", "export"> {
     let AssemblerPredicate = isGFX12Plus;
-    let OtherPredicates = [HasExportInsts];
   }
 }
 
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 9f6277f1257f..351263d07976 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -1079,7 +1079,6 @@ multiclass MIMG_Atomic_Addr_Helper_m <mimgopc op, string asm,
   if !and(op.HAS_GFX12, !not(!empty(renamed))) then
     def : AMDGPUMnemonicAlias<asm, renamed> {
       let AssemblerPredicate = isGFX12Plus;
-      let OtherPredicates = [HasImageInsts];
       bit IsAtomicRet; // Unused
       MIMGBaseOpcode BaseOpcode; // Unused
       int VDataDwords; // Unused
-- 
GitLab


From 52271a5c11f6abde1fa1221db304212b5eb8ec7c Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990@gmail.com>
Date: Fri, 10 May 2024 15:13:00 +0100
Subject: [PATCH 0411/1206] [libc++] Make `constexpr std::variant`. Implement
 P2231R1 (#83335)

Fixes #86686
---
 libcxx/docs/ReleaseNotes/19.rst               |   1 +
 libcxx/docs/Status/Cxx20.rst                  |   1 -
 libcxx/docs/Status/Cxx20Papers.csv            |   2 +-
 libcxx/include/variant                        | 212 ++++----
 .../variant.variant/variant.assign/T.pass.cpp | 202 ++++++-
 .../variant.assign/copy.pass.cpp              | 430 +++++++++------
 .../variant.assign/move.pass.cpp              | 316 ++++++-----
 .../variant.ctor/copy.pass.cpp                | 106 ++--
 .../variant.ctor/default.pass.cpp             |  26 +-
 .../variant.ctor/move.pass.cpp                | 113 ++--
 .../variant.dtor/dtor.pass.cpp                |  71 ++-
 .../variant.mod/emplace_index_args.pass.cpp   |  44 +-
 .../emplace_index_init_list_args.pass.cpp     |  35 +-
 .../variant.mod/emplace_type_args.pass.cpp    |  51 +-
 .../emplace_type_init_list_args.pass.cpp      |  44 +-
 .../variant.swap/swap.pass.cpp                | 504 +++++++++---------
 16 files changed, 1322 insertions(+), 836 deletions(-)

diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index 3fc007f45985..83fcd40bb80c 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -52,6 +52,7 @@ Implemented Papers
 - P3029R1 - Better ``mdspan``'s CTAD
 - P2387R3 - Pipe support for user-defined range adaptors
 - P2713R1 - Escaping improvements in ``std::format``
+- P2231R1 - Missing ``constexpr`` in ``std::optional`` and ``std::variant``
 
 Improvements and New Features
 -----------------------------
diff --git a/libcxx/docs/Status/Cxx20.rst b/libcxx/docs/Status/Cxx20.rst
index 23289dc6e596..b08b99394fbb 100644
--- a/libcxx/docs/Status/Cxx20.rst
+++ b/libcxx/docs/Status/Cxx20.rst
@@ -47,7 +47,6 @@ Paper Status
    .. [#note-P0619] P0619: Only sections D.8, D.9, D.10 and D.13 are implemented. Sections D.4, D.7, D.11, and D.12 remain undone.
    .. [#note-P0883.1] P0883: shared_ptr and floating-point changes weren't applied as they themselves aren't implemented yet.
    .. [#note-P0883.2] P0883: ``ATOMIC_FLAG_INIT`` was marked deprecated in version 14.0, but was undeprecated with the implementation of LWG3659 in version 15.0.
-   .. [#note-P2231] P2231: Optional is complete. The changes to variant haven't been implemented yet.
    .. [#note-P0660] P0660: The paper is implemented but the features are experimental and can be enabled via ``-fexperimental-library``.
    .. [#note-P0355] P0355: The implementation status is:
 
diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index d31720b7576d..955aa5f614af 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -192,7 +192,7 @@
 "`P2106R0 <https://wg21.link/P2106R0>`__","LWG","Alternative wording for GB315 and GB316","Prague","|Complete|","15.0","|ranges|"
 "`P2116R0 <https://wg21.link/P2116R0>`__","LWG","Remove tuple-like protocol support from fixed-extent span","Prague","|Complete|","11.0"
 "","","","","","",""
-"`P2231R1 <https://wg21.link/P2231R1>`__","LWG","Missing constexpr in std::optional and std::variant","June 2021","|Partial| [#note-P2231]_","13.0"
+"`P2231R1 <https://wg21.link/P2231R1>`__","LWG","Missing constexpr in std::optional and std::variant","June 2021","|Complete|","19.0"
 "`P2325R3 <https://wg21.link/P2325R3>`__","LWG","Views should not be required to be default constructible","June 2021","|Complete|","16.0","|ranges|"
 "`P2210R2 <https://wg21.link/P2210R2>`__","LWG","Superior String Splitting","June 2021","|Complete|","16.0","|ranges|"
 "`P2216R3 <https://wg21.link/P2216R3>`__","LWG","std::format improvements","June 2021","|Complete|","15.0"
diff --git a/libcxx/include/variant b/libcxx/include/variant
index 34150bd45284..631ffceab5f6 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -42,26 +42,28 @@ namespace std {
         in_place_index_t<I>, initializer_list<U>, Args&&...);
 
     // 20.7.2.2, destructor
-    ~variant();
+    constexpr ~variant();                                             // constexpr since c++20
 
     // 20.7.2.3, assignment
     constexpr variant& operator=(const variant&);
     constexpr variant& operator=(variant&&) noexcept(see below);
 
-    template <class T> variant& operator=(T&&) noexcept(see below);
+    template <class T>
+    constexpr variant& operator=(T&&) noexcept(see below);            // constexpr since c++20
 
     // 20.7.2.4, modifiers
     template <class T, class... Args>
-    T& emplace(Args&&...);
+    constexpr T& emplace(Args&&...);                                  // constexpr since c++20
 
     template <class T, class U, class... Args>
-    T& emplace(initializer_list<U>, Args&&...);
+    constexpr T& emplace(initializer_list<U>, Args&&...);             // constexpr since c++20
 
     template <size_t I, class... Args>
-    variant_alternative_t<I, variant>& emplace(Args&&...);
+    constexpr variant_alternative_t<I, variant>& emplace(Args&&...);  // constexpr since c++20
 
     template <size_t I, class U, class...  Args>
-    variant_alternative_t<I, variant>& emplace(initializer_list<U>, Args&&...);
+    constexpr variant_alternative_t<I, variant>&
+        emplace(initializer_list<U>, Args&&...);                      // constexpr since c++20
 
     // 20.7.2.5, value status
     constexpr bool valueless_by_exception() const noexcept;
@@ -221,6 +223,7 @@ namespace std {
 #include <__functional/operations.h>
 #include <__functional/unary_function.h>
 #include <__memory/addressof.h>
+#include <__memory/construct_at.h>
 #include <__tuple/find_index.h>
 #include <__tuple/sfinae_helpers.h>
 #include <__type_traits/add_const.h>
@@ -663,7 +666,8 @@ private:
 
 template <size_t _Index, class _Tp>
 struct _LIBCPP_TEMPLATE_VIS __alt {
-  using __value_type = _Tp;
+  using __value_type              = _Tp;
+  static constexpr size_t __index = _Index;
 
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI explicit constexpr __alt(in_place_t, _Args&&... __args)
@@ -678,7 +682,7 @@ union _LIBCPP_TEMPLATE_VIS __union;
 template <_Trait _DestructibleTrait, size_t _Index>
 union _LIBCPP_TEMPLATE_VIS __union<_DestructibleTrait, _Index> {};
 
-#  define _LIBCPP_VARIANT_UNION(destructible_trait, destructor)                                                        \
+#  define _LIBCPP_VARIANT_UNION(destructible_trait, destructor_definition)                                             \
     template <size_t _Index, class _Tp, class... _Types>                                                               \
     union _LIBCPP_TEMPLATE_VIS __union<destructible_trait, _Index, _Tp, _Types...> {                                   \
     public:                                                                                                            \
@@ -692,13 +696,11 @@ union _LIBCPP_TEMPLATE_VIS __union<_DestructibleTrait, _Index> {};
       _LIBCPP_HIDE_FROM_ABI explicit constexpr __union(in_place_index_t<_Ip>, _Args&&... __args)                       \
           : __tail(in_place_index<_Ip - 1>, std::forward<_Args>(__args)...) {}                                         \
                                                                                                                        \
-      __union(const __union&) = default;                                                                               \
-      __union(__union&&)      = default;                                                                               \
-                                                                                                                       \
-      destructor;                                                                                                      \
-                                                                                                                       \
-      __union& operator=(const __union&) = default;                                                                    \
-      __union& operator=(__union&&)      = default;                                                                    \
+      _LIBCPP_HIDE_FROM_ABI __union(const __union&)            = default;                                              \
+      _LIBCPP_HIDE_FROM_ABI __union(__union&&)                 = default;                                              \
+      _LIBCPP_HIDE_FROM_ABI __union& operator=(const __union&) = default;                                              \
+      _LIBCPP_HIDE_FROM_ABI __union& operator=(__union&&)      = default;                                              \
+      destructor_definition;                                                                                           \
                                                                                                                        \
     private:                                                                                                           \
       char __dummy;                                                                                                    \
@@ -708,10 +710,11 @@ union _LIBCPP_TEMPLATE_VIS __union<_DestructibleTrait, _Index> {};
       friend struct __access::__union;                                                                                 \
     }
 
-_LIBCPP_VARIANT_UNION(_Trait::_TriviallyAvailable, ~__union() = default);
+_LIBCPP_VARIANT_UNION(_Trait::_TriviallyAvailable,
+                      _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__union() = default);
 _LIBCPP_VARIANT_UNION(
-    _Trait::_Available, _LIBCPP_HIDE_FROM_ABI ~__union() {} _LIBCPP_EAT_SEMICOLON);
-_LIBCPP_VARIANT_UNION(_Trait::_Unavailable, ~__union() = delete);
+    _Trait::_Available, _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__union() {} _LIBCPP_EAT_SEMICOLON);
+_LIBCPP_VARIANT_UNION(_Trait::_Unavailable, _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__union() = delete);
 
 #  undef _LIBCPP_VARIANT_UNION
 
@@ -754,7 +757,7 @@ protected:
 template <class _Traits, _Trait = _Traits::__destructible_trait>
 class _LIBCPP_TEMPLATE_VIS __dtor;
 
-#  define _LIBCPP_VARIANT_DESTRUCTOR(destructible_trait, destructor, destroy)                                          \
+#  define _LIBCPP_VARIANT_DESTRUCTOR(destructible_trait, destructor_definition, destroy)                               \
     template <class... _Types>                                                                                         \
     class _LIBCPP_TEMPLATE_VIS __dtor<__traits<_Types...>, destructible_trait>                                         \
         : public __base<destructible_trait, _Types...> {                                                               \
@@ -764,28 +767,27 @@ class _LIBCPP_TEMPLATE_VIS __dtor;
     public:                                                                                                            \
       using __base_type::__base_type;                                                                                  \
       using __base_type::operator=;                                                                                    \
-                                                                                                                       \
-      __dtor(const __dtor&)            = default;                                                                      \
-      __dtor(__dtor&&)                 = default;                                                                      \
-      __dtor& operator=(const __dtor&) = default;                                                                      \
-      __dtor& operator=(__dtor&&)      = default;                                                                      \
-      destructor;                                                                                                      \
+      _LIBCPP_HIDE_FROM_ABI __dtor(const __dtor&)            = default;                                                \
+      _LIBCPP_HIDE_FROM_ABI __dtor(__dtor&&)                 = default;                                                \
+      _LIBCPP_HIDE_FROM_ABI __dtor& operator=(const __dtor&) = default;                                                \
+      _LIBCPP_HIDE_FROM_ABI __dtor& operator=(__dtor&&)      = default;                                                \
+      destructor_definition;                                                                                           \
                                                                                                                        \
     protected:                                                                                                         \
-      inline _LIBCPP_HIDE_FROM_ABI destroy;                                                                            \
+      destroy;                                                                                                         \
     }
 
 _LIBCPP_VARIANT_DESTRUCTOR(
     _Trait::_TriviallyAvailable,
-    ~__dtor() = default, //
-    _LIBCPP_HIDE_FROM_ABI void __destroy() noexcept {
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__dtor() = default,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy() noexcept {
       this->__index = __variant_npos<__index_t>;
     } _LIBCPP_EAT_SEMICOLON);
 
 _LIBCPP_VARIANT_DESTRUCTOR(
     _Trait::_Available,
-    _LIBCPP_HIDE_FROM_ABI ~__dtor() { __destroy(); } _LIBCPP_EAT_SEMICOLON,
-    _LIBCPP_HIDE_FROM_ABI void __destroy() noexcept {
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__dtor() { __destroy(); } _LIBCPP_EAT_SEMICOLON,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy() noexcept {
       if (!this->valueless_by_exception()) {
         __visitation::__base::__visit_alt(
             [](auto& __alt) noexcept {
@@ -797,7 +799,9 @@ _LIBCPP_VARIANT_DESTRUCTOR(
       this->__index = __variant_npos<__index_t>;
     } _LIBCPP_EAT_SEMICOLON);
 
-_LIBCPP_VARIANT_DESTRUCTOR(_Trait::_Unavailable, ~__dtor() = delete, void __destroy() noexcept = delete);
+_LIBCPP_VARIANT_DESTRUCTOR(_Trait::_Unavailable,
+                           _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__dtor()                 = delete,
+                           _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy() noexcept = delete);
 
 #  undef _LIBCPP_VARIANT_DESTRUCTOR
 
@@ -810,23 +814,18 @@ public:
   using __base_type::operator=;
 
 protected:
-  template <size_t _Ip, class _Tp, class... _Args>
-  _LIBCPP_HIDE_FROM_ABI static _Tp& __construct_alt(__alt<_Ip, _Tp>& __a, _Args&&... __args) {
-    ::new ((void*)std::addressof(__a)) __alt<_Ip, _Tp>(in_place, std::forward<_Args>(__args)...);
-    return __a.__value;
-  }
-
   template <class _Rhs>
-  _LIBCPP_HIDE_FROM_ABI static void __generic_construct(__ctor& __lhs, _Rhs&& __rhs) {
+  _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX20 void __generic_construct(__ctor& __lhs, _Rhs&& __rhs) {
     __lhs.__destroy();
     if (!__rhs.valueless_by_exception()) {
       auto __rhs_index = __rhs.index();
       __visitation::__base::__visit_alt_at(
           __rhs_index,
-          [](auto& __lhs_alt, auto&& __rhs_alt) {
-            __construct_alt(__lhs_alt, std::forward<decltype(__rhs_alt)>(__rhs_alt).__value);
+          [&__lhs](auto&& __rhs_alt) {
+            std::__construct_at(std::addressof(__lhs.__data),
+                                in_place_index<__decay_t<decltype(__rhs_alt)>::__index>,
+                                std::forward<decltype(__rhs_alt)>(__rhs_alt).__value);
           },
-          __lhs,
           std::forward<_Rhs>(__rhs));
       __lhs.__index = __rhs_index;
     }
@@ -836,7 +835,7 @@ protected:
 template <class _Traits, _Trait = _Traits::__move_constructible_trait>
 class _LIBCPP_TEMPLATE_VIS __move_constructor;
 
-#  define _LIBCPP_VARIANT_MOVE_CONSTRUCTOR(move_constructible_trait, move_constructor)                                 \
+#  define _LIBCPP_VARIANT_MOVE_CONSTRUCTOR(move_constructible_trait, move_constructor_definition)                      \
     template <class... _Types>                                                                                         \
     class _LIBCPP_TEMPLATE_VIS __move_constructor<__traits<_Types...>, move_constructible_trait>                       \
         : public __ctor<__traits<_Types...>> {                                                                         \
@@ -846,32 +845,35 @@ class _LIBCPP_TEMPLATE_VIS __move_constructor;
       using __base_type::__base_type;                                                                                  \
       using __base_type::operator=;                                                                                    \
                                                                                                                        \
-      __move_constructor(const __move_constructor&)            = default;                                              \
-      ~__move_constructor()                                    = default;                                              \
-      __move_constructor& operator=(const __move_constructor&) = default;                                              \
-      __move_constructor& operator=(__move_constructor&&)      = default;                                              \
-      move_constructor;                                                                                                \
+      _LIBCPP_HIDE_FROM_ABI __move_constructor(const __move_constructor&)            = default;                        \
+      _LIBCPP_HIDE_FROM_ABI ~__move_constructor()                                    = default;                        \
+      _LIBCPP_HIDE_FROM_ABI __move_constructor& operator=(const __move_constructor&) = default;                        \
+      _LIBCPP_HIDE_FROM_ABI __move_constructor& operator=(__move_constructor&&)      = default;                        \
+      move_constructor_definition;                                                                                     \
     }
 
-_LIBCPP_VARIANT_MOVE_CONSTRUCTOR(_Trait::_TriviallyAvailable,
-                                 __move_constructor(__move_constructor&& __that) = default);
+_LIBCPP_VARIANT_MOVE_CONSTRUCTOR(
+    _Trait::_TriviallyAvailable,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_constructor(__move_constructor&& __that) = default);
 
 _LIBCPP_VARIANT_MOVE_CONSTRUCTOR(
     _Trait::_Available,
-    _LIBCPP_HIDE_FROM_ABI __move_constructor(__move_constructor&& __that) noexcept(
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_constructor(__move_constructor&& __that) noexcept(
         __all<is_nothrow_move_constructible_v<_Types>...>::value)
     : __move_constructor(__valueless_t{}) {
       this->__generic_construct(*this, std::move(__that));
     } _LIBCPP_EAT_SEMICOLON);
 
-_LIBCPP_VARIANT_MOVE_CONSTRUCTOR(_Trait::_Unavailable, __move_constructor(__move_constructor&&) = delete);
+_LIBCPP_VARIANT_MOVE_CONSTRUCTOR(
+    _Trait::_Unavailable,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_constructor(__move_constructor&&) = delete);
 
 #  undef _LIBCPP_VARIANT_MOVE_CONSTRUCTOR
 
 template <class _Traits, _Trait = _Traits::__copy_constructible_trait>
 class _LIBCPP_TEMPLATE_VIS __copy_constructor;
 
-#  define _LIBCPP_VARIANT_COPY_CONSTRUCTOR(copy_constructible_trait, copy_constructor)                                 \
+#  define _LIBCPP_VARIANT_COPY_CONSTRUCTOR(copy_constructible_trait, copy_constructor_definition)                      \
     template <class... _Types>                                                                                         \
     class _LIBCPP_TEMPLATE_VIS __copy_constructor<__traits<_Types...>, copy_constructible_trait>                       \
         : public __move_constructor<__traits<_Types...>> {                                                             \
@@ -881,21 +883,25 @@ class _LIBCPP_TEMPLATE_VIS __copy_constructor;
       using __base_type::__base_type;                                                                                  \
       using __base_type::operator=;                                                                                    \
                                                                                                                        \
-      __copy_constructor(__copy_constructor&&)                 = default;                                              \
-      ~__copy_constructor()                                    = default;                                              \
-      __copy_constructor& operator=(const __copy_constructor&) = default;                                              \
-      __copy_constructor& operator=(__copy_constructor&&)      = default;                                              \
-      copy_constructor;                                                                                                \
-    } // namespace __variant_detail
+      _LIBCPP_HIDE_FROM_ABI __copy_constructor(__copy_constructor&&)                 = default;                        \
+      _LIBCPP_HIDE_FROM_ABI ~__copy_constructor()                                    = default;                        \
+      _LIBCPP_HIDE_FROM_ABI __copy_constructor& operator=(const __copy_constructor&) = default;                        \
+      _LIBCPP_HIDE_FROM_ABI __copy_constructor& operator=(__copy_constructor&&)      = default;                        \
+      copy_constructor_definition;                                                                                     \
+    }
 
-_LIBCPP_VARIANT_COPY_CONSTRUCTOR(_Trait::_TriviallyAvailable,
-                                 __copy_constructor(const __copy_constructor& __that) = default);
+_LIBCPP_VARIANT_COPY_CONSTRUCTOR(
+    _Trait::_TriviallyAvailable,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_constructor(const __copy_constructor& __that) = default);
 
 _LIBCPP_VARIANT_COPY_CONSTRUCTOR(
-    _Trait::_Available, _LIBCPP_HIDE_FROM_ABI __copy_constructor(const __copy_constructor& __that)
+    _Trait::_Available,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_constructor(const __copy_constructor& __that)
     : __copy_constructor(__valueless_t{}) { this->__generic_construct(*this, __that); } _LIBCPP_EAT_SEMICOLON);
 
-_LIBCPP_VARIANT_COPY_CONSTRUCTOR(_Trait::_Unavailable, __copy_constructor(const __copy_constructor&) = delete);
+_LIBCPP_VARIANT_COPY_CONSTRUCTOR(
+    _Trait::_Unavailable,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_constructor(const __copy_constructor&) = delete);
 
 #  undef _LIBCPP_VARIANT_COPY_CONSTRUCTOR
 
@@ -908,22 +914,24 @@ public:
   using __base_type::operator=;
 
   template <size_t _Ip, class... _Args>
-  _LIBCPP_HIDE_FROM_ABI auto& __emplace(_Args&&... __args) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 auto& __emplace(_Args&&... __args) {
     this->__destroy();
-    auto& __res   = this->__construct_alt(__access::__base::__get_alt<_Ip>(*this), std::forward<_Args>(__args)...);
+    std::__construct_at(std::addressof(this->__data), in_place_index<_Ip>, std::forward<_Args>(__args)...);
     this->__index = _Ip;
-    return __res;
+    return __access::__base::__get_alt<_Ip>(*this).__value;
   }
 
 protected:
   template <size_t _Ip, class _Tp, class _Arg>
-  _LIBCPP_HIDE_FROM_ABI void __assign_alt(__alt<_Ip, _Tp>& __a, _Arg&& __arg) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __assign_alt(__alt<_Ip, _Tp>& __a, _Arg&& __arg) {
     if (this->index() == _Ip) {
       __a.__value = std::forward<_Arg>(__arg);
     } else {
       struct {
-        _LIBCPP_HIDDEN void operator()(true_type) const { __this->__emplace<_Ip>(std::forward<_Arg>(__arg)); }
-        _LIBCPP_HIDDEN void operator()(false_type) const {
+        _LIBCPP_HIDDEN _LIBCPP_CONSTEXPR_SINCE_CXX20 void operator()(true_type) const {
+          __this->__emplace<_Ip>(std::forward<_Arg>(__arg));
+        }
+        _LIBCPP_HIDDEN _LIBCPP_CONSTEXPR_SINCE_CXX20 void operator()(false_type) const {
           __this->__emplace<_Ip>(_Tp(std::forward<_Arg>(__arg)));
         }
         __assignment* __this;
@@ -934,7 +942,7 @@ protected:
   }
 
   template <class _That>
-  _LIBCPP_HIDE_FROM_ABI void __generic_assign(_That&& __that) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __generic_assign(_That&& __that) {
     if (this->valueless_by_exception() && __that.valueless_by_exception()) {
       // do nothing.
     } else if (__that.valueless_by_exception()) {
@@ -954,7 +962,7 @@ protected:
 template <class _Traits, _Trait = _Traits::__move_assignable_trait>
 class _LIBCPP_TEMPLATE_VIS __move_assignment;
 
-#  define _LIBCPP_VARIANT_MOVE_ASSIGNMENT(move_assignable_trait, move_assignment)                                      \
+#  define _LIBCPP_VARIANT_MOVE_ASSIGNMENT(move_assignable_trait, move_assignment_definition)                           \
     template <class... _Types>                                                                                         \
     class _LIBCPP_TEMPLATE_VIS __move_assignment<__traits<_Types...>, move_assignable_trait>                           \
         : public __assignment<__traits<_Types...>> {                                                                   \
@@ -964,33 +972,36 @@ class _LIBCPP_TEMPLATE_VIS __move_assignment;
       using __base_type::__base_type;                                                                                  \
       using __base_type::operator=;                                                                                    \
                                                                                                                        \
-      __move_assignment(const __move_assignment&)            = default;                                                \
-      __move_assignment(__move_assignment&&)                 = default;                                                \
-      ~__move_assignment()                                   = default;                                                \
-      __move_assignment& operator=(const __move_assignment&) = default;                                                \
-      move_assignment;                                                                                                 \
+      _LIBCPP_HIDE_FROM_ABI __move_assignment(const __move_assignment&)            = default;                          \
+      _LIBCPP_HIDE_FROM_ABI __move_assignment(__move_assignment&&)                 = default;                          \
+      _LIBCPP_HIDE_FROM_ABI ~__move_assignment()                                   = default;                          \
+      _LIBCPP_HIDE_FROM_ABI __move_assignment& operator=(const __move_assignment&) = default;                          \
+      move_assignment_definition;                                                                                      \
     }
 
 _LIBCPP_VARIANT_MOVE_ASSIGNMENT(_Trait::_TriviallyAvailable,
-                                __move_assignment& operator=(__move_assignment&& __that) = default);
+                                _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_assignment& operator=(
+                                    __move_assignment&& __that) = default);
 
 _LIBCPP_VARIANT_MOVE_ASSIGNMENT(
     _Trait::_Available,
-    _LIBCPP_HIDE_FROM_ABI __move_assignment&
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_assignment&
     operator=(__move_assignment&& __that) noexcept(
         __all<(is_nothrow_move_constructible_v<_Types> && is_nothrow_move_assignable_v<_Types>)...>::value) {
       this->__generic_assign(std::move(__that));
       return *this;
     } _LIBCPP_EAT_SEMICOLON);
 
-_LIBCPP_VARIANT_MOVE_ASSIGNMENT(_Trait::_Unavailable, __move_assignment& operator=(__move_assignment&&) = delete);
+_LIBCPP_VARIANT_MOVE_ASSIGNMENT(
+    _Trait::_Unavailable,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __move_assignment& operator=(__move_assignment&&) = delete);
 
 #  undef _LIBCPP_VARIANT_MOVE_ASSIGNMENT
 
 template <class _Traits, _Trait = _Traits::__copy_assignable_trait>
 class _LIBCPP_TEMPLATE_VIS __copy_assignment;
 
-#  define _LIBCPP_VARIANT_COPY_ASSIGNMENT(copy_assignable_trait, copy_assignment)                                      \
+#  define _LIBCPP_VARIANT_COPY_ASSIGNMENT(copy_assignable_trait, copy_assignment_definition)                           \
     template <class... _Types>                                                                                         \
     class _LIBCPP_TEMPLATE_VIS __copy_assignment<__traits<_Types...>, copy_assignable_trait>                           \
         : public __move_assignment<__traits<_Types...>> {                                                              \
@@ -1000,23 +1011,28 @@ class _LIBCPP_TEMPLATE_VIS __copy_assignment;
       using __base_type::__base_type;                                                                                  \
       using __base_type::operator=;                                                                                    \
                                                                                                                        \
-      __copy_assignment(const __copy_assignment&)       = default;                                                     \
-      __copy_assignment(__copy_assignment&&)            = default;                                                     \
-      ~__copy_assignment()                              = default;                                                     \
-      __copy_assignment& operator=(__copy_assignment&&) = default;                                                     \
-      copy_assignment;                                                                                                 \
+      _LIBCPP_HIDE_FROM_ABI __copy_assignment(const __copy_assignment&)       = default;                               \
+      _LIBCPP_HIDE_FROM_ABI __copy_assignment(__copy_assignment&&)            = default;                               \
+      _LIBCPP_HIDE_FROM_ABI ~__copy_assignment()                              = default;                               \
+      _LIBCPP_HIDE_FROM_ABI __copy_assignment& operator=(__copy_assignment&&) = default;                               \
+      copy_assignment_definition;                                                                                      \
     }
 
 _LIBCPP_VARIANT_COPY_ASSIGNMENT(_Trait::_TriviallyAvailable,
-                                __copy_assignment& operator=(const __copy_assignment& __that) = default);
+                                _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_assignment& operator=(
+                                    const __copy_assignment& __that) = default);
 
 _LIBCPP_VARIANT_COPY_ASSIGNMENT(
-    _Trait::_Available, _LIBCPP_HIDE_FROM_ABI __copy_assignment& operator=(const __copy_assignment& __that) {
+    _Trait::_Available,
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_assignment&
+    operator=(const __copy_assignment& __that) {
       this->__generic_assign(__that);
       return *this;
     } _LIBCPP_EAT_SEMICOLON);
 
-_LIBCPP_VARIANT_COPY_ASSIGNMENT(_Trait::_Unavailable, __copy_assignment& operator=(const __copy_assignment&) = delete);
+_LIBCPP_VARIANT_COPY_ASSIGNMENT(_Trait::_Unavailable,
+                                _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __copy_assignment& operator=(
+                                    const __copy_assignment&) = delete);
 
 #  undef _LIBCPP_VARIANT_COPY_ASSIGNMENT
 
@@ -1032,11 +1048,11 @@ public:
   _LIBCPP_HIDE_FROM_ABI __impl& operator=(__impl&&)      = default;
 
   template <size_t _Ip, class _Arg>
-  _LIBCPP_HIDE_FROM_ABI void __assign(_Arg&& __arg) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __assign(_Arg&& __arg) {
     this->__assign_alt(__access::__base::__get_alt<_Ip>(*this), std::forward<_Arg>(__arg));
   }
 
-  inline _LIBCPP_HIDE_FROM_ABI void __swap(__impl& __that) {
+  inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __swap(__impl& __that) {
     if (this->valueless_by_exception() && __that.valueless_by_exception()) {
       // do nothing.
     } else if (this->index() == __that.index()) {
@@ -1081,7 +1097,7 @@ public:
   }
 
 private:
-  inline _LIBCPP_HIDE_FROM_ABI bool __move_nothrow() const {
+  constexpr inline _LIBCPP_HIDE_FROM_ABI bool __move_nothrow() const {
     constexpr bool __results[] = {is_nothrow_move_constructible_v<_Types>...};
     return this->valueless_by_exception() || __results[this->index()];
   }
@@ -1223,7 +1239,7 @@ public:
       _Args&&... __args) noexcept(is_nothrow_constructible_v<_Tp, initializer_list< _Up>&, _Args...>)
       : __impl_(in_place_index<_Ip>, __il, std::forward<_Args>(__args)...) {}
 
-  _LIBCPP_HIDE_FROM_ABI ~variant() = default;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~variant() = default;
 
   _LIBCPP_HIDE_FROM_ABI constexpr variant& operator=(const variant&) = default;
   _LIBCPP_HIDE_FROM_ABI constexpr variant& operator=(variant&&)      = default;
@@ -1233,7 +1249,7 @@ public:
              class _Tp  = __variant_detail::__best_match_t<_Arg, _Types...>,
              size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
              enable_if_t<is_assignable_v<_Tp&, _Arg> && is_constructible_v<_Tp, _Arg>, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI variant&
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 variant&
   operator=(_Arg&& __arg) noexcept(is_nothrow_assignable_v<_Tp&, _Arg> && is_nothrow_constructible_v<_Tp, _Arg>) {
     __impl_.template __assign<_Ip>(std::forward<_Arg>(__arg));
     return *this;
@@ -1244,7 +1260,7 @@ public:
              enable_if_t<(_Ip < sizeof...(_Types)), int>         = 0,
              class _Tp                                           = variant_alternative_t<_Ip, variant<_Types...>>,
              enable_if_t<is_constructible_v<_Tp, _Args...>, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI _Tp& emplace(_Args&&... __args) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(_Args&&... __args) {
     return __impl_.template __emplace<_Ip>(std::forward<_Args>(__args)...);
   }
 
@@ -1254,7 +1270,7 @@ public:
              enable_if_t<(_Ip < sizeof...(_Types)), int> = 0,
              class _Tp                                   = variant_alternative_t<_Ip, variant<_Types...>>,
              enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI _Tp& emplace(initializer_list<_Up> __il, _Args&&... __args) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(initializer_list<_Up> __il, _Args&&... __args) {
     return __impl_.template __emplace<_Ip>(__il, std::forward<_Args>(__args)...);
   }
 
@@ -1262,7 +1278,7 @@ public:
              class... _Args,
              size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
              enable_if_t<is_constructible_v<_Tp, _Args...>, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI _Tp& emplace(_Args&&... __args) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(_Args&&... __args) {
     return __impl_.template __emplace<_Ip>(std::forward<_Args>(__args)...);
   }
 
@@ -1271,7 +1287,7 @@ public:
              class... _Args,
              size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
              enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI _Tp& emplace(initializer_list<_Up> __il, _Args&&... __args) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(initializer_list<_Up> __il, _Args&&... __args) {
     return __impl_.template __emplace<_Ip>(__il, std::forward<_Args>(__args)...);
   }
 
@@ -1285,7 +1301,7 @@ public:
              enable_if_t< __all<(__dependent_type<is_move_constructible<_Types>, _Dummy>::value &&
                                  __dependent_type<is_swappable<_Types>, _Dummy>::value)...>::value,
                           int> = 0>
-  _LIBCPP_HIDE_FROM_ABI void swap(variant& __that) noexcept(
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(variant& __that) noexcept(
       __all<(is_nothrow_move_constructible_v<_Types> && is_nothrow_swappable_v<_Types>)...>::value) {
     __impl_.__swap(__that.__impl_);
   }
@@ -1568,7 +1584,7 @@ visit(_Visitor&& __visitor, _Vs&&... __vs) {
 #  endif
 
 template <class... _Types>
-_LIBCPP_HIDE_FROM_ABI auto
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 auto
 swap(variant<_Types...>& __lhs, variant<_Types...>& __rhs) noexcept(noexcept(__lhs.swap(__rhs)))
     -> decltype(__lhs.swap(__rhs)) {
   return __lhs.swap(__rhs);
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp
index 4b9eaba2d2ba..98faf84fa52d 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp
@@ -33,17 +33,17 @@ struct Dummy {
 
 struct ThrowsCtorT {
   ThrowsCtorT(int) noexcept(false) {}
-  ThrowsCtorT &operator=(int) noexcept { return *this; }
+  ThrowsCtorT& operator=(int) noexcept { return *this; }
 };
 
 struct ThrowsAssignT {
   ThrowsAssignT(int) noexcept {}
-  ThrowsAssignT &operator=(int) noexcept(false) { return *this; }
+  ThrowsAssignT& operator=(int) noexcept(false) { return *this; }
 };
 
 struct NoThrowT {
   NoThrowT(int) noexcept {}
-  NoThrowT &operator=(int) noexcept { return *this; }
+  NoThrowT& operator=(int) noexcept { return *this; }
 };
 
 } // namespace MetaHelpers
@@ -55,7 +55,7 @@ struct ThrowsCtorT {
   int value;
   ThrowsCtorT() : value(0) {}
   ThrowsCtorT(int) noexcept(false) { throw 42; }
-  ThrowsCtorT &operator=(int v) noexcept {
+  ThrowsCtorT& operator=(int v) noexcept {
     value = v;
     return *this;
   }
@@ -64,9 +64,12 @@ struct ThrowsCtorT {
 struct MoveCrashes {
   int value;
   MoveCrashes(int v = 0) noexcept : value{v} {}
-  MoveCrashes(MoveCrashes &&) noexcept { assert(false); }
-  MoveCrashes &operator=(MoveCrashes &&) noexcept { assert(false); return *this; }
-  MoveCrashes &operator=(int v) noexcept {
+  MoveCrashes(MoveCrashes&&) noexcept { assert(false); }
+  MoveCrashes& operator=(MoveCrashes&&) noexcept {
+    assert(false);
+    return *this;
+  }
+  MoveCrashes& operator=(int v) noexcept {
     value = v;
     return *this;
   }
@@ -76,8 +79,8 @@ struct ThrowsCtorTandMove {
   int value;
   ThrowsCtorTandMove() : value(0) {}
   ThrowsCtorTandMove(int) noexcept(false) { throw 42; }
-  ThrowsCtorTandMove(ThrowsCtorTandMove &&) noexcept(false) { assert(false); }
-  ThrowsCtorTandMove &operator=(int v) noexcept {
+  ThrowsCtorTandMove(ThrowsCtorTandMove&&) noexcept(false) { assert(false); }
+  ThrowsCtorTandMove& operator=(int v) noexcept {
     value = v;
     return *this;
   }
@@ -87,14 +90,14 @@ struct ThrowsAssignT {
   int value;
   ThrowsAssignT() : value(0) {}
   ThrowsAssignT(int v) noexcept : value(v) {}
-  ThrowsAssignT &operator=(int) noexcept(false) { throw 42; }
+  ThrowsAssignT& operator=(int) noexcept(false) { throw 42; }
 };
 
 struct NoThrowT {
   int value;
   NoThrowT() : value(0) {}
   NoThrowT(int v) noexcept : value(v) {}
-  NoThrowT &operator=(int v) noexcept {
+  NoThrowT& operator=(int v) noexcept {
     value = v;
     return *this;
   }
@@ -103,7 +106,7 @@ struct NoThrowT {
 #endif // !defined(TEST_HAS_NO_EXCEPTIONS)
 } // namespace RuntimeHelpers
 
-void test_T_assignment_noexcept() {
+constexpr void test_T_assignment_noexcept() {
   using namespace MetaHelpers;
   {
     using V = std::variant<Dummy, NoThrowT>;
@@ -119,17 +122,17 @@ void test_T_assignment_noexcept() {
   }
 }
 
-void test_T_assignment_sfinae() {
+constexpr void test_T_assignment_sfinae() {
   {
     using V = std::variant<long, long long>;
     static_assert(!std::is_assignable<V, int>::value, "ambiguous");
   }
   {
     using V = std::variant<std::string, std::string>;
-    static_assert(!std::is_assignable<V, const char *>::value, "ambiguous");
+    static_assert(!std::is_assignable<V, const char*>::value, "ambiguous");
   }
   {
-    using V = std::variant<std::string, void *>;
+    using V = std::variant<std::string, void*>;
     static_assert(!std::is_assignable<V, int>::value, "no matching operator=");
   }
   {
@@ -138,8 +141,7 @@ void test_T_assignment_sfinae() {
   }
   {
     using V = std::variant<std::unique_ptr<int>, bool>;
-    static_assert(!std::is_assignable<V, std::unique_ptr<char>>::value,
-                  "no explicit bool in operator=");
+    static_assert(!std::is_assignable<V, std::unique_ptr<char>>::value, "no explicit bool in operator=");
     struct X {
       operator void*();
     };
@@ -152,12 +154,11 @@ void test_T_assignment_sfinae() {
       operator X();
     };
     using V = std::variant<X>;
-    static_assert(std::is_assignable<V, Y>::value,
-                  "regression on user-defined conversions in operator=");
+    static_assert(std::is_assignable<V, Y>::value, "regression on user-defined conversions in operator=");
   }
 }
 
-void test_T_assignment_basic() {
+TEST_CONSTEXPR_CXX20 void test_T_assignment_basic() {
   {
     std::variant<int> v(43);
     v = 42;
@@ -184,19 +185,146 @@ void test_T_assignment_basic() {
   }
   {
     std::variant<std::string, bool> v = true;
-    v = "bar";
+    v                                 = "bar";
     assert(v.index() == 0);
     assert(std::get<0>(v) == "bar");
   }
+}
+
+void test_T_assignment_basic_no_constexpr() {
+  std::variant<bool, std::unique_ptr<int>> v;
+  v = nullptr;
+  assert(v.index() == 1);
+  assert(std::get<1>(v) == nullptr);
+}
+
+struct TraceStat {
+  int construct      = 0;
+  int copy_construct = 0;
+  int copy_assign    = 0;
+  int move_construct = 0;
+  int move_assign    = 0;
+  int T_copy_assign  = 0;
+  int T_move_assign  = 0;
+  int destroy        = 0;
+};
+
+template <bool CtorNoexcept, bool MoveCtorNoexcept>
+struct Trace {
+  struct T {};
+
+  constexpr Trace(TraceStat* s) noexcept(CtorNoexcept) : stat(s) { ++s->construct; }
+  constexpr Trace(T) noexcept(CtorNoexcept) : stat(nullptr) {}
+  constexpr Trace(const Trace& o) : stat(o.stat) { ++stat->copy_construct; }
+  constexpr Trace(Trace&& o) noexcept(MoveCtorNoexcept) : stat(o.stat) { ++stat->move_construct; }
+  constexpr Trace& operator=(const Trace&) {
+    ++stat->copy_assign;
+    return *this;
+  }
+  constexpr Trace& operator=(Trace&&) noexcept {
+    ++stat->move_assign;
+    return *this;
+  }
+
+  constexpr Trace& operator=(const T&) {
+    ++stat->T_copy_assign;
+    return *this;
+  }
+  constexpr Trace& operator=(T&&) noexcept {
+    ++stat->T_move_assign;
+    return *this;
+  }
+  TEST_CONSTEXPR_CXX20 ~Trace() { ++stat->destroy; }
+
+  TraceStat* stat;
+};
+
+TEST_CONSTEXPR_CXX20 void test_T_assignment_performs_construction() {
   {
-    std::variant<bool, std::unique_ptr<int>> v;
-    v = nullptr;
-    assert(v.index() == 1);
-    assert(std::get<1>(v) == nullptr);
+    using V = std::variant<int, Trace<false, false>>;
+    TraceStat stat;
+    V v{1};
+    v = &stat;
+    assert(stat.construct == 1);
+    assert(stat.copy_construct == 0);
+    assert(stat.move_construct == 0);
+    assert(stat.copy_assign == 0);
+    assert(stat.move_assign == 0);
+    assert(stat.destroy == 0);
+  }
+  {
+    using V = std::variant<int, Trace<false, true>>;
+    TraceStat stat;
+    V v{1};
+    v = &stat;
+    assert(stat.construct == 1);
+    assert(stat.copy_construct == 0);
+    assert(stat.move_construct == 1);
+    assert(stat.copy_assign == 0);
+    assert(stat.move_assign == 0);
+    assert(stat.destroy == 1);
+  }
+
+  {
+    using V = std::variant<int, Trace<true, false>>;
+    TraceStat stat;
+    V v{1};
+    v = &stat;
+    assert(stat.construct == 1);
+    assert(stat.copy_construct == 0);
+    assert(stat.move_construct == 0);
+    assert(stat.copy_assign == 0);
+    assert(stat.move_assign == 0);
+    assert(stat.destroy == 0);
+  }
+
+  {
+    using V = std::variant<int, Trace<true, true>>;
+    TraceStat stat;
+    V v{1};
+    v = &stat;
+    assert(stat.construct == 1);
+    assert(stat.copy_construct == 0);
+    assert(stat.move_construct == 0);
+    assert(stat.copy_assign == 0);
+    assert(stat.move_assign == 0);
+    assert(stat.destroy == 0);
   }
 }
 
-void test_T_assignment_performs_construction() {
+TEST_CONSTEXPR_CXX20 void test_T_assignment_performs_assignment() {
+  {
+    using V = std::variant<int, Trace<false, false>>;
+    TraceStat stat;
+    V v{&stat};
+    v = Trace<false, false>::T{};
+    assert(stat.construct == 1);
+    assert(stat.copy_construct == 0);
+    assert(stat.move_construct == 0);
+    assert(stat.copy_assign == 0);
+    assert(stat.move_assign == 0);
+    assert(stat.T_copy_assign == 0);
+    assert(stat.T_move_assign == 1);
+    assert(stat.destroy == 0);
+  }
+  {
+    using V = std::variant<int, Trace<false, false>>;
+    TraceStat stat;
+    V v{&stat};
+    Trace<false, false>::T t;
+    v = t;
+    assert(stat.construct == 1);
+    assert(stat.copy_construct == 0);
+    assert(stat.move_construct == 0);
+    assert(stat.copy_assign == 0);
+    assert(stat.move_assign == 0);
+    assert(stat.T_copy_assign == 1);
+    assert(stat.T_move_assign == 0);
+    assert(stat.destroy == 0);
+  }
+}
+
+void test_T_assignment_performs_construction_throw() {
   using namespace RuntimeHelpers;
 #ifndef TEST_HAS_NO_EXCEPTIONS
   {
@@ -220,7 +348,7 @@ void test_T_assignment_performs_construction() {
 #endif // TEST_HAS_NO_EXCEPTIONS
 }
 
-void test_T_assignment_performs_assignment() {
+void test_T_assignment_performs_assignment_throw() {
   using namespace RuntimeHelpers;
 #ifndef TEST_HAS_NO_EXCEPTIONS
   {
@@ -262,7 +390,7 @@ void test_T_assignment_performs_assignment() {
 #endif // TEST_HAS_NO_EXCEPTIONS
 }
 
-void test_T_assignment_vector_bool() {
+TEST_CONSTEXPR_CXX20 void test_T_assignment_vector_bool() {
   std::vector<bool> vec = {true};
   std::variant<bool, int> v;
   v = vec[0];
@@ -270,7 +398,13 @@ void test_T_assignment_vector_bool() {
   assert(std::get<0>(v) == true);
 }
 
-int main(int, char**) {
+void non_constexpr_test() {
+  test_T_assignment_basic_no_constexpr();
+  test_T_assignment_performs_construction_throw();
+  test_T_assignment_performs_assignment_throw();
+}
+
+TEST_CONSTEXPR_CXX20 bool test() {
   test_T_assignment_basic();
   test_T_assignment_performs_construction();
   test_T_assignment_performs_assignment();
@@ -278,5 +412,15 @@ int main(int, char**) {
   test_T_assignment_sfinae();
   test_T_assignment_vector_bool();
 
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  non_constexpr_test();
+
+#if TEST_STD_VER >= 20
+  static_assert(test());
+#endif
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/copy.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/copy.pass.cpp
index 096d365d2d75..a6d3f34114eb 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/copy.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/copy.pass.cpp
@@ -22,88 +22,108 @@
 #include "test_macros.h"
 
 struct NoCopy {
-  NoCopy(const NoCopy &) = delete;
-  NoCopy &operator=(const NoCopy &) = default;
+  NoCopy(const NoCopy&)            = delete;
+  NoCopy& operator=(const NoCopy&) = default;
 };
 
 struct CopyOnly {
-  CopyOnly(const CopyOnly &) = default;
-  CopyOnly(CopyOnly &&) = delete;
-  CopyOnly &operator=(const CopyOnly &) = default;
-  CopyOnly &operator=(CopyOnly &&) = delete;
+  CopyOnly(const CopyOnly&)            = default;
+  CopyOnly(CopyOnly&&)                 = delete;
+  CopyOnly& operator=(const CopyOnly&) = default;
+  CopyOnly& operator=(CopyOnly&&)      = delete;
 };
 
 struct MoveOnly {
-  MoveOnly(const MoveOnly &) = delete;
-  MoveOnly(MoveOnly &&) = default;
-  MoveOnly &operator=(const MoveOnly &) = default;
+  MoveOnly(const MoveOnly&)            = delete;
+  MoveOnly(MoveOnly&&)                 = default;
+  MoveOnly& operator=(const MoveOnly&) = default;
 };
 
 struct MoveOnlyNT {
-  MoveOnlyNT(const MoveOnlyNT &) = delete;
-  MoveOnlyNT(MoveOnlyNT &&) {}
-  MoveOnlyNT &operator=(const MoveOnlyNT &) = default;
+  MoveOnlyNT(const MoveOnlyNT&) = delete;
+  MoveOnlyNT(MoveOnlyNT&&) {}
+  MoveOnlyNT& operator=(const MoveOnlyNT&) = default;
 };
 
 struct CopyAssign {
-  static int alive;
-  static int copy_construct;
-  static int copy_assign;
-  static int move_construct;
-  static int move_assign;
-  static void reset() {
-    copy_construct = copy_assign = move_construct = move_assign = alive = 0;
-  }
-  CopyAssign(int v) : value(v) { ++alive; }
-  CopyAssign(const CopyAssign &o) : value(o.value) {
-    ++alive;
-    ++copy_construct;
-  }
-  CopyAssign(CopyAssign &&o) noexcept : value(o.value) {
+  constexpr CopyAssign(int v, int* alv, int* cpy_ctr, int* cpy_assi, int* move_ctr, int* move_assi)
+      : value(v),
+        alive(alv),
+        copy_construct(cpy_ctr),
+        copy_assign(cpy_assi),
+        move_construct(move_ctr),
+        move_assign(move_assi) {
+    ++*alive;
+  }
+  constexpr CopyAssign(const CopyAssign& o)
+      : value(o.value),
+        alive(o.alive),
+        copy_construct(o.copy_construct),
+        copy_assign(o.copy_assign),
+        move_construct(o.move_construct),
+        move_assign(o.move_assign) {
+    ++*alive;
+    ++*copy_construct;
+  }
+  constexpr CopyAssign(CopyAssign&& o) noexcept
+      : value(o.value),
+        alive(o.alive),
+        copy_construct(o.copy_construct),
+        copy_assign(o.copy_assign),
+        move_construct(o.move_construct),
+        move_assign(o.move_assign) {
     o.value = -1;
-    ++alive;
-    ++move_construct;
-  }
-  CopyAssign &operator=(const CopyAssign &o) {
-    value = o.value;
-    ++copy_assign;
+    ++*alive;
+    ++*move_construct;
+  }
+  constexpr CopyAssign& operator=(const CopyAssign& o) {
+    value          = o.value;
+    alive          = o.alive;
+    copy_construct = o.copy_construct;
+    copy_assign    = o.copy_assign;
+    move_construct = o.move_construct;
+    move_assign    = o.move_assign;
+    ++*copy_assign;
     return *this;
   }
-  CopyAssign &operator=(CopyAssign &&o) noexcept {
-    value = o.value;
-    o.value = -1;
-    ++move_assign;
+  constexpr CopyAssign& operator=(CopyAssign&& o) noexcept {
+    value          = o.value;
+    alive          = o.alive;
+    copy_construct = o.copy_construct;
+    copy_assign    = o.copy_assign;
+    move_construct = o.move_construct;
+    move_assign    = o.move_assign;
+    o.value        = -1;
+    ++*move_assign;
     return *this;
   }
-  ~CopyAssign() { --alive; }
+  TEST_CONSTEXPR_CXX20 ~CopyAssign() { --*alive; }
   int value;
+  int* alive;
+  int* copy_construct;
+  int* copy_assign;
+  int* move_construct;
+  int* move_assign;
 };
 
-int CopyAssign::alive = 0;
-int CopyAssign::copy_construct = 0;
-int CopyAssign::copy_assign = 0;
-int CopyAssign::move_construct = 0;
-int CopyAssign::move_assign = 0;
-
 struct CopyMaybeThrows {
-  CopyMaybeThrows(const CopyMaybeThrows &);
-  CopyMaybeThrows &operator=(const CopyMaybeThrows &);
+  CopyMaybeThrows(const CopyMaybeThrows&);
+  CopyMaybeThrows& operator=(const CopyMaybeThrows&);
 };
 struct CopyDoesThrow {
-  CopyDoesThrow(const CopyDoesThrow &) noexcept(false);
-  CopyDoesThrow &operator=(const CopyDoesThrow &) noexcept(false);
+  CopyDoesThrow(const CopyDoesThrow&) noexcept(false);
+  CopyDoesThrow& operator=(const CopyDoesThrow&) noexcept(false);
 };
 
-
 struct NTCopyAssign {
   constexpr NTCopyAssign(int v) : value(v) {}
-  NTCopyAssign(const NTCopyAssign &) = default;
-  NTCopyAssign(NTCopyAssign &&) = default;
-  NTCopyAssign &operator=(const NTCopyAssign &that) {
+  NTCopyAssign(const NTCopyAssign&) = default;
+  NTCopyAssign(NTCopyAssign&&)      = default;
+  NTCopyAssign& operator=(const NTCopyAssign& that) {
     value = that.value;
     return *this;
   };
-  NTCopyAssign &operator=(NTCopyAssign &&) = delete;
+  NTCopyAssign& operator=(NTCopyAssign&&) = delete;
   int value;
 };
 
@@ -112,10 +132,10 @@ static_assert(std::is_copy_assignable<NTCopyAssign>::value, "");
 
 struct TCopyAssign {
   constexpr TCopyAssign(int v) : value(v) {}
-  TCopyAssign(const TCopyAssign &) = default;
-  TCopyAssign(TCopyAssign &&) = default;
-  TCopyAssign &operator=(const TCopyAssign &) = default;
-  TCopyAssign &operator=(TCopyAssign &&) = delete;
+  TCopyAssign(const TCopyAssign&)            = default;
+  TCopyAssign(TCopyAssign&&)                 = default;
+  TCopyAssign& operator=(const TCopyAssign&) = default;
+  TCopyAssign& operator=(TCopyAssign&&)      = delete;
   int value;
 };
 
@@ -123,11 +143,11 @@ static_assert(std::is_trivially_copy_assignable<TCopyAssign>::value, "");
 
 struct TCopyAssignNTMoveAssign {
   constexpr TCopyAssignNTMoveAssign(int v) : value(v) {}
-  TCopyAssignNTMoveAssign(const TCopyAssignNTMoveAssign &) = default;
-  TCopyAssignNTMoveAssign(TCopyAssignNTMoveAssign &&) = default;
-  TCopyAssignNTMoveAssign &operator=(const TCopyAssignNTMoveAssign &) = default;
-  TCopyAssignNTMoveAssign &operator=(TCopyAssignNTMoveAssign &&that) {
-    value = that.value;
+  TCopyAssignNTMoveAssign(const TCopyAssignNTMoveAssign&)            = default;
+  TCopyAssignNTMoveAssign(TCopyAssignNTMoveAssign&&)                 = default;
+  TCopyAssignNTMoveAssign& operator=(const TCopyAssignNTMoveAssign&) = default;
+  TCopyAssignNTMoveAssign& operator=(TCopyAssignNTMoveAssign&& that) {
+    value      = that.value;
     that.value = -1;
     return *this;
   }
@@ -139,17 +159,20 @@ static_assert(std::is_trivially_copy_assignable_v<TCopyAssignNTMoveAssign>, "");
 #ifndef TEST_HAS_NO_EXCEPTIONS
 struct CopyThrows {
   CopyThrows() = default;
-  CopyThrows(const CopyThrows &) { throw 42; }
-  CopyThrows &operator=(const CopyThrows &) { throw 42; }
+  CopyThrows(const CopyThrows&) { throw 42; }
+  CopyThrows& operator=(const CopyThrows&) { throw 42; }
 };
 
 struct CopyCannotThrow {
   static int alive;
   CopyCannotThrow() { ++alive; }
-  CopyCannotThrow(const CopyCannotThrow &) noexcept { ++alive; }
-  CopyCannotThrow(CopyCannotThrow &&) noexcept { assert(false); }
-  CopyCannotThrow &operator=(const CopyCannotThrow &) noexcept = default;
-  CopyCannotThrow &operator=(CopyCannotThrow &&) noexcept { assert(false); return *this; }
+  CopyCannotThrow(const CopyCannotThrow&) noexcept { ++alive; }
+  CopyCannotThrow(CopyCannotThrow&&) noexcept { assert(false); }
+  CopyCannotThrow& operator=(const CopyCannotThrow&) noexcept = default;
+  CopyCannotThrow& operator=(CopyCannotThrow&&) noexcept {
+    assert(false);
+    return *this;
+  }
 };
 
 int CopyCannotThrow::alive = 0;
@@ -157,10 +180,10 @@ int CopyCannotThrow::alive = 0;
 struct MoveThrows {
   static int alive;
   MoveThrows() { ++alive; }
-  MoveThrows(const MoveThrows &) { ++alive; }
-  MoveThrows(MoveThrows &&) { throw 42; }
-  MoveThrows &operator=(const MoveThrows &) { return *this; }
-  MoveThrows &operator=(MoveThrows &&) { throw 42; }
+  MoveThrows(const MoveThrows&) { ++alive; }
+  MoveThrows(MoveThrows&&) { throw 42; }
+  MoveThrows& operator=(const MoveThrows&) { return *this; }
+  MoveThrows& operator=(MoveThrows&&) { throw 42; }
   ~MoveThrows() { --alive; }
 };
 
@@ -169,20 +192,21 @@ int MoveThrows::alive = 0;
 struct MakeEmptyT {
   static int alive;
   MakeEmptyT() { ++alive; }
-  MakeEmptyT(const MakeEmptyT &) {
+  MakeEmptyT(const MakeEmptyT&) {
     ++alive;
     // Don't throw from the copy constructor since variant's assignment
     // operator performs a copy before committing to the assignment.
   }
-  MakeEmptyT(MakeEmptyT &&) { throw 42; }
-  MakeEmptyT &operator=(const MakeEmptyT &) { throw 42; }
-  MakeEmptyT &operator=(MakeEmptyT &&) { throw 42; }
+  MakeEmptyT(MakeEmptyT&&) { throw 42; }
+  MakeEmptyT& operator=(const MakeEmptyT&) { throw 42; }
+  MakeEmptyT& operator=(MakeEmptyT&&) { throw 42; }
   ~MakeEmptyT() { --alive; }
 };
 
 int MakeEmptyT::alive = 0;
 
-template <class Variant> void makeEmpty(Variant &v) {
+template <class Variant>
+void makeEmpty(Variant& v) {
   Variant v2(std::in_place_type<MakeEmptyT>);
   try {
     v = std::move(v2);
@@ -193,7 +217,7 @@ template <class Variant> void makeEmpty(Variant &v) {
 }
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-void test_copy_assignment_not_noexcept() {
+constexpr void test_copy_assignment_not_noexcept() {
   {
     using V = std::variant<CopyMaybeThrows>;
     static_assert(!std::is_nothrow_copy_assignable<V>::value, "");
@@ -204,7 +228,7 @@ void test_copy_assignment_not_noexcept() {
   }
 }
 
-void test_copy_assignment_sfinae() {
+constexpr void test_copy_assignment_sfinae() {
   {
     using V = std::variant<int, long>;
     static_assert(std::is_copy_assignable<V>::value, "");
@@ -259,7 +283,7 @@ void test_copy_assignment_empty_empty() {
     makeEmpty(v1);
     V v2(std::in_place_index<0>);
     makeEmpty(v2);
-    V &vref = (v1 = v2);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.valueless_by_exception());
     assert(v1.index() == std::variant_npos);
@@ -275,7 +299,7 @@ void test_copy_assignment_non_empty_empty() {
     V v1(std::in_place_index<0>, 42);
     V v2(std::in_place_index<0>);
     makeEmpty(v2);
-    V &vref = (v1 = v2);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.valueless_by_exception());
     assert(v1.index() == std::variant_npos);
@@ -285,7 +309,7 @@ void test_copy_assignment_non_empty_empty() {
     V v1(std::in_place_index<2>, "hello");
     V v2(std::in_place_index<0>);
     makeEmpty(v2);
-    V &vref = (v1 = v2);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.valueless_by_exception());
     assert(v1.index() == std::variant_npos);
@@ -301,7 +325,7 @@ void test_copy_assignment_empty_non_empty() {
     V v1(std::in_place_index<0>);
     makeEmpty(v1);
     V v2(std::in_place_index<0>, 42);
-    V &vref = (v1 = v2);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.index() == 0);
     assert(std::get<0>(v1) == 42);
@@ -311,7 +335,7 @@ void test_copy_assignment_empty_non_empty() {
     V v1(std::in_place_index<0>);
     makeEmpty(v1);
     V v2(std::in_place_type<std::string>, "hello");
-    V &vref = (v1 = v2);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.index() == 2);
     assert(std::get<2>(v1) == "hello");
@@ -319,14 +343,18 @@ void test_copy_assignment_empty_non_empty() {
 #endif // TEST_HAS_NO_EXCEPTIONS
 }
 
-template <typename T> struct Result { std::size_t index; T value; };
+template <typename T>
+struct Result {
+  std::size_t index;
+  T value;
+};
 
-void test_copy_assignment_same_index() {
+TEST_CONSTEXPR_CXX20 void test_copy_assignment_same_index() {
   {
     using V = std::variant<int>;
     V v1(43);
     V v2(42);
-    V &vref = (v1 = v2);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.index() == 0);
     assert(std::get<0>(v1) == 42);
@@ -335,40 +363,28 @@ void test_copy_assignment_same_index() {
     using V = std::variant<int, long, unsigned>;
     V v1(43l);
     V v2(42l);
-    V &vref = (v1 = v2);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.index() == 1);
     assert(std::get<1>(v1) == 42);
   }
   {
-    using V = std::variant<int, CopyAssign, unsigned>;
-    V v1(std::in_place_type<CopyAssign>, 43);
-    V v2(std::in_place_type<CopyAssign>, 42);
-    CopyAssign::reset();
-    V &vref = (v1 = v2);
+    using V            = std::variant<int, CopyAssign, unsigned>;
+    int alive          = 0;
+    int copy_construct = 0;
+    int copy_assign    = 0;
+    int move_construct = 0;
+    int move_assign    = 0;
+    V v1(std::in_place_type<CopyAssign>, 43, &alive, &copy_construct, &copy_assign, &move_construct, &move_assign);
+    V v2(std::in_place_type<CopyAssign>, 42, &alive, &copy_construct, &copy_assign, &move_construct, &move_assign);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.index() == 1);
     assert(std::get<1>(v1).value == 42);
-    assert(CopyAssign::copy_construct == 0);
-    assert(CopyAssign::move_construct == 0);
-    assert(CopyAssign::copy_assign == 1);
+    assert(copy_construct == 0);
+    assert(move_construct == 0);
+    assert(copy_assign == 1);
   }
-#ifndef TEST_HAS_NO_EXCEPTIONS
-  using MET = MakeEmptyT;
-  {
-    using V = std::variant<int, MET, std::string>;
-    V v1(std::in_place_type<MET>);
-    MET &mref = std::get<1>(v1);
-    V v2(std::in_place_type<MET>);
-    try {
-      v1 = v2;
-      assert(false);
-    } catch (...) {
-    }
-    assert(v1.index() == 1);
-    assert(&std::get<1>(v1) == &mref);
-  }
-#endif // TEST_HAS_NO_EXCEPTIONS
 
   // Make sure we properly propagate triviality, which implies constexpr-ness (see P0602R4).
   {
@@ -429,34 +445,88 @@ void test_copy_assignment_same_index() {
   }
 }
 
-void test_copy_assignment_different_index() {
+TEST_CONSTEXPR_CXX20 void test_copy_assignment_different_index() {
   {
     using V = std::variant<int, long, unsigned>;
     V v1(43);
     V v2(42l);
-    V &vref = (v1 = v2);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.index() == 1);
     assert(std::get<1>(v1) == 42);
   }
   {
-    using V = std::variant<int, CopyAssign, unsigned>;
-    CopyAssign::reset();
+    using V            = std::variant<int, CopyAssign, unsigned>;
+    int alive          = 0;
+    int copy_construct = 0;
+    int copy_assign    = 0;
+    int move_construct = 0;
+    int move_assign    = 0;
     V v1(std::in_place_type<unsigned>, 43u);
-    V v2(std::in_place_type<CopyAssign>, 42);
-    assert(CopyAssign::copy_construct == 0);
-    assert(CopyAssign::move_construct == 0);
-    assert(CopyAssign::alive == 1);
-    V &vref = (v1 = v2);
+    V v2(std::in_place_type<CopyAssign>, 42, &alive, &copy_construct, &copy_assign, &move_construct, &move_assign);
+    assert(copy_construct == 0);
+    assert(move_construct == 0);
+    assert(alive == 1);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.index() == 1);
     assert(std::get<1>(v1).value == 42);
-    assert(CopyAssign::alive == 2);
-    assert(CopyAssign::copy_construct == 1);
-    assert(CopyAssign::move_construct == 1);
-    assert(CopyAssign::copy_assign == 0);
+    assert(alive == 2);
+    assert(copy_construct == 1);
+    assert(move_construct == 1);
+    assert(copy_assign == 0);
+  }
+
+  // Make sure we properly propagate triviality, which implies constexpr-ness (see P0602R4).
+  {
+    struct {
+      constexpr Result<long> operator()() const {
+        using V = std::variant<int, long, unsigned>;
+        V v(43);
+        V v2(42l);
+        v = v2;
+        return {v.index(), std::get<1>(v)};
+      }
+    } test;
+    constexpr auto result = test();
+    static_assert(result.index == 1, "");
+    static_assert(result.value == 42l, "");
   }
+  {
+    struct {
+      constexpr Result<int> operator()() const {
+        using V = std::variant<int, TCopyAssign, unsigned>;
+        V v(std::in_place_type<unsigned>, 43u);
+        V v2(std::in_place_type<TCopyAssign>, 42);
+        v = v2;
+        return {v.index(), std::get<1>(v).value};
+      }
+    } test;
+    constexpr auto result = test();
+    static_assert(result.index == 1, "");
+    static_assert(result.value == 42, "");
+  }
+}
+
+void test_assignment_throw() {
 #ifndef TEST_HAS_NO_EXCEPTIONS
+  using MET = MakeEmptyT;
+  // same index
+  {
+    using V = std::variant<int, MET, std::string>;
+    V v1(std::in_place_type<MET>);
+    MET& mref = std::get<1>(v1);
+    V v2(std::in_place_type<MET>);
+    try {
+      v1 = v2;
+      assert(false);
+    } catch (...) {
+    }
+    assert(v1.index() == 1);
+    assert(&std::get<1>(v1) == &mref);
+  }
+
+  // difference indices
   {
     using V = std::variant<int, CopyThrows, std::string>;
     V v1(std::in_place_type<std::string>, "hello");
@@ -496,7 +566,7 @@ void test_copy_assignment_different_index() {
     using V = std::variant<int, CopyThrows, std::string>;
     V v1(std::in_place_type<CopyThrows>);
     V v2(std::in_place_type<std::string>, "hello");
-    V &vref = (v1 = v2);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.index() == 2);
     assert(std::get<2>(v1) == "hello");
@@ -507,7 +577,7 @@ void test_copy_assignment_different_index() {
     using V = std::variant<int, MoveThrows, std::string>;
     V v1(std::in_place_type<MoveThrows>);
     V v2(std::in_place_type<std::string>, "hello");
-    V &vref = (v1 = v2);
+    V& vref = (v1 = v2);
     assert(&vref == &v1);
     assert(v1.index() == 2);
     assert(std::get<2>(v1) == "hello");
@@ -515,69 +585,83 @@ void test_copy_assignment_different_index() {
     assert(std::get<2>(v2) == "hello");
   }
 #endif // TEST_HAS_NO_EXCEPTIONS
-
-  // Make sure we properly propagate triviality, which implies constexpr-ness (see P0602R4).
-  {
-    struct {
-      constexpr Result<long> operator()() const {
-        using V = std::variant<int, long, unsigned>;
-        V v(43);
-        V v2(42l);
-        v = v2;
-        return {v.index(), std::get<1>(v)};
-      }
-    } test;
-    constexpr auto result = test();
-    static_assert(result.index == 1, "");
-    static_assert(result.value == 42l, "");
-  }
-  {
-    struct {
-      constexpr Result<int> operator()() const {
-        using V = std::variant<int, TCopyAssign, unsigned>;
-        V v(std::in_place_type<unsigned>, 43u);
-        V v2(std::in_place_type<TCopyAssign>, 42);
-        v = v2;
-        return {v.index(), std::get<1>(v).value};
-      }
-    } test;
-    constexpr auto result = test();
-    static_assert(result.index == 1, "");
-    static_assert(result.value == 42, "");
-  }
 }
 
-template <std::size_t NewIdx, class ValueType>
-constexpr bool test_constexpr_assign_imp(
-    std::variant<long, void*, int>&& v, ValueType&& new_value)
-{
-  const std::variant<long, void*, int> cp(
-      std::forward<ValueType>(new_value));
+template <std::size_t NewIdx, class T, class ValueType>
+constexpr void test_constexpr_assign_imp(T&& v, ValueType&& new_value) {
+  using Variant = std::decay_t<T>;
+  const Variant cp(std::forward<ValueType>(new_value));
   v = cp;
-  return v.index() == NewIdx &&
-        std::get<NewIdx>(v) == std::get<NewIdx>(cp);
+  assert(v.index() == NewIdx);
+  assert(std::get<NewIdx>(v) == std::get<NewIdx>(cp));
 }
 
-void test_constexpr_copy_assignment() {
+constexpr void test_constexpr_copy_assignment_trivial() {
   // Make sure we properly propagate triviality, which implies constexpr-ness (see P0602R4).
   using V = std::variant<long, void*, int>;
   static_assert(std::is_trivially_copyable<V>::value, "");
   static_assert(std::is_trivially_copy_assignable<V>::value, "");
-  static_assert(test_constexpr_assign_imp<0>(V(42l), 101l), "");
-  static_assert(test_constexpr_assign_imp<0>(V(nullptr), 101l), "");
-  static_assert(test_constexpr_assign_imp<1>(V(42l), nullptr), "");
-  static_assert(test_constexpr_assign_imp<2>(V(42l), 101), "");
+  test_constexpr_assign_imp<0>(V(42l), 101l);
+  test_constexpr_assign_imp<0>(V(nullptr), 101l);
+  test_constexpr_assign_imp<1>(V(42l), nullptr);
+  test_constexpr_assign_imp<2>(V(42l), 101);
 }
 
-int main(int, char**) {
+struct NonTrivialCopyAssign {
+  int i = 0;
+  constexpr NonTrivialCopyAssign(int ii) : i(ii) {}
+  constexpr NonTrivialCopyAssign(const NonTrivialCopyAssign& other) : i(other.i) {}
+  constexpr NonTrivialCopyAssign& operator=(const NonTrivialCopyAssign& o) {
+    i = o.i;
+    return *this;
+  }
+  TEST_CONSTEXPR_CXX20 ~NonTrivialCopyAssign() = default;
+  friend constexpr bool operator==(const NonTrivialCopyAssign& x, const NonTrivialCopyAssign& y) { return x.i == y.i; }
+};
+
+constexpr void test_constexpr_copy_assignment_non_trivial() {
+  // Make sure we properly propagate triviality, which implies constexpr-ness (see P0602R4).
+  using V = std::variant<long, void*, NonTrivialCopyAssign>;
+  static_assert(!std::is_trivially_copyable<V>::value, "");
+  static_assert(!std::is_trivially_copy_assignable<V>::value, "");
+  test_constexpr_assign_imp<0>(V(42l), 101l);
+  test_constexpr_assign_imp<0>(V(nullptr), 101l);
+  test_constexpr_assign_imp<1>(V(42l), nullptr);
+  test_constexpr_assign_imp<2>(V(42l), NonTrivialCopyAssign(5));
+  test_constexpr_assign_imp<2>(V(NonTrivialCopyAssign(3)), NonTrivialCopyAssign(5));
+}
+
+void non_constexpr_test() {
   test_copy_assignment_empty_empty();
   test_copy_assignment_non_empty_empty();
   test_copy_assignment_empty_non_empty();
-  test_copy_assignment_same_index();
-  test_copy_assignment_different_index();
+  test_assignment_throw();
+}
+
+constexpr bool cxx17_constexpr_test() {
   test_copy_assignment_sfinae();
   test_copy_assignment_not_noexcept();
-  test_constexpr_copy_assignment();
+  test_constexpr_copy_assignment_trivial();
 
+  return true;
+}
+
+TEST_CONSTEXPR_CXX20 bool cxx20_constexpr_test() {
+  test_copy_assignment_same_index();
+  test_copy_assignment_different_index();
+  test_constexpr_copy_assignment_non_trivial();
+
+  return true;
+}
+
+int main(int, char**) {
+  non_constexpr_test();
+  cxx17_constexpr_test();
+  cxx20_constexpr_test();
+
+  static_assert(cxx17_constexpr_test());
+#if TEST_STD_VER >= 20
+  static_assert(cxx20_constexpr_test());
+#endif
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/move.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/move.pass.cpp
index 84094347aed3..157ff68f3748 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/move.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/move.pass.cpp
@@ -24,71 +24,70 @@
 #include "variant_test_helpers.h"
 
 struct NoCopy {
-  NoCopy(const NoCopy &) = delete;
-  NoCopy &operator=(const NoCopy &) = default;
+  NoCopy(const NoCopy&)            = delete;
+  NoCopy& operator=(const NoCopy&) = default;
 };
 
 struct CopyOnly {
-  CopyOnly(const CopyOnly &) = default;
-  CopyOnly(CopyOnly &&) = delete;
-  CopyOnly &operator=(const CopyOnly &) = default;
-  CopyOnly &operator=(CopyOnly &&) = delete;
+  CopyOnly(const CopyOnly&)            = default;
+  CopyOnly(CopyOnly&&)                 = delete;
+  CopyOnly& operator=(const CopyOnly&) = default;
+  CopyOnly& operator=(CopyOnly&&)      = delete;
 };
 
 struct MoveOnly {
-  MoveOnly(const MoveOnly &) = delete;
-  MoveOnly(MoveOnly &&) = default;
-  MoveOnly &operator=(const MoveOnly &) = delete;
-  MoveOnly &operator=(MoveOnly &&) = default;
+  MoveOnly(const MoveOnly&)            = delete;
+  MoveOnly(MoveOnly&&)                 = default;
+  MoveOnly& operator=(const MoveOnly&) = delete;
+  MoveOnly& operator=(MoveOnly&&)      = default;
 };
 
 struct MoveOnlyNT {
-  MoveOnlyNT(const MoveOnlyNT &) = delete;
-  MoveOnlyNT(MoveOnlyNT &&) {}
-  MoveOnlyNT &operator=(const MoveOnlyNT &) = delete;
-  MoveOnlyNT &operator=(MoveOnlyNT &&) = default;
+  MoveOnlyNT(const MoveOnlyNT&) = delete;
+  MoveOnlyNT(MoveOnlyNT&&) {}
+  MoveOnlyNT& operator=(const MoveOnlyNT&) = delete;
+  MoveOnlyNT& operator=(MoveOnlyNT&&)      = default;
 };
 
 struct MoveOnlyOddNothrow {
-  MoveOnlyOddNothrow(MoveOnlyOddNothrow &&) noexcept(false) {}
-  MoveOnlyOddNothrow(const MoveOnlyOddNothrow &) = delete;
-  MoveOnlyOddNothrow &operator=(MoveOnlyOddNothrow &&) noexcept = default;
-  MoveOnlyOddNothrow &operator=(const MoveOnlyOddNothrow &) = delete;
+  MoveOnlyOddNothrow(MoveOnlyOddNothrow&&) noexcept(false) {}
+  MoveOnlyOddNothrow(const MoveOnlyOddNothrow&)                = delete;
+  MoveOnlyOddNothrow& operator=(MoveOnlyOddNothrow&&) noexcept = default;
+  MoveOnlyOddNothrow& operator=(const MoveOnlyOddNothrow&)     = delete;
 };
 
 struct MoveAssignOnly {
-  MoveAssignOnly(MoveAssignOnly &&) = delete;
-  MoveAssignOnly &operator=(MoveAssignOnly &&) = default;
+  MoveAssignOnly(MoveAssignOnly&&)            = delete;
+  MoveAssignOnly& operator=(MoveAssignOnly&&) = default;
 };
 
 struct MoveAssign {
-  static int move_construct;
-  static int move_assign;
-  static void reset() { move_construct = move_assign = 0; }
-  MoveAssign(int v) : value(v) {}
-  MoveAssign(MoveAssign &&o) : value(o.value) {
-    ++move_construct;
+  constexpr MoveAssign(int v, int* move_ctor, int* move_assi)
+      : value(v), move_construct(move_ctor), move_assign(move_assi) {}
+  constexpr MoveAssign(MoveAssign&& o) : value(o.value), move_construct(o.move_construct), move_assign(o.move_assign) {
+    ++*move_construct;
     o.value = -1;
   }
-  MoveAssign &operator=(MoveAssign &&o) {
-    value = o.value;
-    ++move_assign;
+  constexpr MoveAssign& operator=(MoveAssign&& o) {
+    value          = o.value;
+    move_construct = o.move_construct;
+    move_assign    = o.move_assign;
+    ++*move_assign;
     o.value = -1;
     return *this;
   }
   int value;
+  int* move_construct;
+  int* move_assign;
 };
 
-int MoveAssign::move_construct = 0;
-int MoveAssign::move_assign = 0;
-
 struct NTMoveAssign {
   constexpr NTMoveAssign(int v) : value(v) {}
-  NTMoveAssign(const NTMoveAssign &) = default;
-  NTMoveAssign(NTMoveAssign &&) = default;
-  NTMoveAssign &operator=(const NTMoveAssign &that) = default;
-  NTMoveAssign &operator=(NTMoveAssign &&that) {
-    value = that.value;
+  NTMoveAssign(const NTMoveAssign&)                 = default;
+  NTMoveAssign(NTMoveAssign&&)                      = default;
+  NTMoveAssign& operator=(const NTMoveAssign& that) = default;
+  NTMoveAssign& operator=(NTMoveAssign&& that) {
+    value      = that.value;
     that.value = -1;
     return *this;
   };
@@ -100,10 +99,10 @@ static_assert(std::is_move_assignable<NTMoveAssign>::value, "");
 
 struct TMoveAssign {
   constexpr TMoveAssign(int v) : value(v) {}
-  TMoveAssign(const TMoveAssign &) = delete;
-  TMoveAssign(TMoveAssign &&) = default;
-  TMoveAssign &operator=(const TMoveAssign &) = delete;
-  TMoveAssign &operator=(TMoveAssign &&) = default;
+  TMoveAssign(const TMoveAssign&)            = delete;
+  TMoveAssign(TMoveAssign&&)                 = default;
+  TMoveAssign& operator=(const TMoveAssign&) = delete;
+  TMoveAssign& operator=(TMoveAssign&&)      = default;
   int value;
 };
 
@@ -111,13 +110,13 @@ static_assert(std::is_trivially_move_assignable<TMoveAssign>::value, "");
 
 struct TMoveAssignNTCopyAssign {
   constexpr TMoveAssignNTCopyAssign(int v) : value(v) {}
-  TMoveAssignNTCopyAssign(const TMoveAssignNTCopyAssign &) = default;
-  TMoveAssignNTCopyAssign(TMoveAssignNTCopyAssign &&) = default;
-  TMoveAssignNTCopyAssign &operator=(const TMoveAssignNTCopyAssign &that) {
+  TMoveAssignNTCopyAssign(const TMoveAssignNTCopyAssign&) = default;
+  TMoveAssignNTCopyAssign(TMoveAssignNTCopyAssign&&)      = default;
+  TMoveAssignNTCopyAssign& operator=(const TMoveAssignNTCopyAssign& that) {
     value = that.value;
     return *this;
   }
-  TMoveAssignNTCopyAssign &operator=(TMoveAssignNTCopyAssign &&) = default;
+  TMoveAssignNTCopyAssign& operator=(TMoveAssignNTCopyAssign&&) = default;
   int value;
 };
 
@@ -127,16 +126,13 @@ struct TrivialCopyNontrivialMove {
   TrivialCopyNontrivialMove(TrivialCopyNontrivialMove const&) = default;
   TrivialCopyNontrivialMove(TrivialCopyNontrivialMove&&) noexcept {}
   TrivialCopyNontrivialMove& operator=(TrivialCopyNontrivialMove const&) = default;
-  TrivialCopyNontrivialMove& operator=(TrivialCopyNontrivialMove&&) noexcept {
-    return *this;
-  }
+  TrivialCopyNontrivialMove& operator=(TrivialCopyNontrivialMove&&) noexcept { return *this; }
 };
 
 static_assert(std::is_trivially_copy_assignable_v<TrivialCopyNontrivialMove>, "");
 static_assert(!std::is_trivially_move_assignable_v<TrivialCopyNontrivialMove>, "");
 
-
-void test_move_assignment_noexcept() {
+constexpr void test_move_assignment_noexcept() {
   {
     using V = std::variant<int>;
     static_assert(std::is_nothrow_move_assignable<V>::value, "");
@@ -163,7 +159,7 @@ void test_move_assignment_noexcept() {
   }
 }
 
-void test_move_assignment_sfinae() {
+constexpr void test_move_assignment_sfinae() {
   {
     using V = std::variant<int, long>;
     static_assert(std::is_move_assignable<V>::value, "");
@@ -228,7 +224,7 @@ void test_move_assignment_empty_empty() {
     makeEmpty(v1);
     V v2(std::in_place_index<0>);
     makeEmpty(v2);
-    V &vref = (v1 = std::move(v2));
+    V& vref = (v1 = std::move(v2));
     assert(&vref == &v1);
     assert(v1.valueless_by_exception());
     assert(v1.index() == std::variant_npos);
@@ -244,7 +240,7 @@ void test_move_assignment_non_empty_empty() {
     V v1(std::in_place_index<0>, 42);
     V v2(std::in_place_index<0>);
     makeEmpty(v2);
-    V &vref = (v1 = std::move(v2));
+    V& vref = (v1 = std::move(v2));
     assert(&vref == &v1);
     assert(v1.valueless_by_exception());
     assert(v1.index() == std::variant_npos);
@@ -254,7 +250,7 @@ void test_move_assignment_non_empty_empty() {
     V v1(std::in_place_index<2>, "hello");
     V v2(std::in_place_index<0>);
     makeEmpty(v2);
-    V &vref = (v1 = std::move(v2));
+    V& vref = (v1 = std::move(v2));
     assert(&vref == &v1);
     assert(v1.valueless_by_exception());
     assert(v1.index() == std::variant_npos);
@@ -270,7 +266,7 @@ void test_move_assignment_empty_non_empty() {
     V v1(std::in_place_index<0>);
     makeEmpty(v1);
     V v2(std::in_place_index<0>, 42);
-    V &vref = (v1 = std::move(v2));
+    V& vref = (v1 = std::move(v2));
     assert(&vref == &v1);
     assert(v1.index() == 0);
     assert(std::get<0>(v1) == 42);
@@ -280,7 +276,7 @@ void test_move_assignment_empty_non_empty() {
     V v1(std::in_place_index<0>);
     makeEmpty(v1);
     V v2(std::in_place_type<std::string>, "hello");
-    V &vref = (v1 = std::move(v2));
+    V& vref = (v1 = std::move(v2));
     assert(&vref == &v1);
     assert(v1.index() == 2);
     assert(std::get<2>(v1) == "hello");
@@ -288,14 +284,18 @@ void test_move_assignment_empty_non_empty() {
 #endif // TEST_HAS_NO_EXCEPTIONS
 }
 
-template <typename T> struct Result { std::size_t index; T value; };
+template <typename T>
+struct Result {
+  std::size_t index;
+  T value;
+};
 
-void test_move_assignment_same_index() {
+TEST_CONSTEXPR_CXX20 void test_move_assignment_same_index() {
   {
     using V = std::variant<int>;
     V v1(43);
     V v2(42);
-    V &vref = (v1 = std::move(v2));
+    V& vref = (v1 = std::move(v2));
     assert(&vref == &v1);
     assert(v1.index() == 0);
     assert(std::get<0>(v1) == 42);
@@ -304,39 +304,24 @@ void test_move_assignment_same_index() {
     using V = std::variant<int, long, unsigned>;
     V v1(43l);
     V v2(42l);
-    V &vref = (v1 = std::move(v2));
+    V& vref = (v1 = std::move(v2));
     assert(&vref == &v1);
     assert(v1.index() == 1);
     assert(std::get<1>(v1) == 42);
   }
   {
-    using V = std::variant<int, MoveAssign, unsigned>;
-    V v1(std::in_place_type<MoveAssign>, 43);
-    V v2(std::in_place_type<MoveAssign>, 42);
-    MoveAssign::reset();
-    V &vref = (v1 = std::move(v2));
+    using V            = std::variant<int, MoveAssign, unsigned>;
+    int move_construct = 0;
+    int move_assign    = 0;
+    V v1(std::in_place_type<MoveAssign>, 43, &move_construct, &move_assign);
+    V v2(std::in_place_type<MoveAssign>, 42, &move_construct, &move_assign);
+    V& vref = (v1 = std::move(v2));
     assert(&vref == &v1);
     assert(v1.index() == 1);
     assert(std::get<1>(v1).value == 42);
-    assert(MoveAssign::move_construct == 0);
-    assert(MoveAssign::move_assign == 1);
-  }
-#ifndef TEST_HAS_NO_EXCEPTIONS
-  using MET = MakeEmptyT;
-  {
-    using V = std::variant<int, MET, std::string>;
-    V v1(std::in_place_type<MET>);
-    MET &mref = std::get<1>(v1);
-    V v2(std::in_place_type<MET>);
-    try {
-      v1 = std::move(v2);
-      assert(false);
-    } catch (...) {
-    }
-    assert(v1.index() == 1);
-    assert(&std::get<1>(v1) == &mref);
+    assert(move_construct == 0);
+    assert(move_assign == 1);
   }
-#endif // TEST_HAS_NO_EXCEPTIONS
 
   // Make sure we properly propagate triviality, which implies constexpr-ness (see P0602R4).
   {
@@ -383,52 +368,29 @@ void test_move_assignment_same_index() {
   }
 }
 
-void test_move_assignment_different_index() {
+TEST_CONSTEXPR_CXX20 void test_move_assignment_different_index() {
   {
     using V = std::variant<int, long, unsigned>;
     V v1(43);
     V v2(42l);
-    V &vref = (v1 = std::move(v2));
+    V& vref = (v1 = std::move(v2));
     assert(&vref == &v1);
     assert(v1.index() == 1);
     assert(std::get<1>(v1) == 42);
   }
   {
-    using V = std::variant<int, MoveAssign, unsigned>;
+    using V            = std::variant<int, MoveAssign, unsigned>;
+    int move_construct = 0;
+    int move_assign    = 0;
     V v1(std::in_place_type<unsigned>, 43u);
-    V v2(std::in_place_type<MoveAssign>, 42);
-    MoveAssign::reset();
-    V &vref = (v1 = std::move(v2));
+    V v2(std::in_place_type<MoveAssign>, 42, &move_construct, &move_assign);
+    V& vref = (v1 = std::move(v2));
     assert(&vref == &v1);
     assert(v1.index() == 1);
     assert(std::get<1>(v1).value == 42);
-    assert(MoveAssign::move_construct == 1);
-    assert(MoveAssign::move_assign == 0);
+    assert(move_construct == 1);
+    assert(move_assign == 0);
   }
-#ifndef TEST_HAS_NO_EXCEPTIONS
-  using MET = MakeEmptyT;
-  {
-    using V = std::variant<int, MET, std::string>;
-    V v1(std::in_place_type<int>);
-    V v2(std::in_place_type<MET>);
-    try {
-      v1 = std::move(v2);
-      assert(false);
-    } catch (...) {
-    }
-    assert(v1.valueless_by_exception());
-    assert(v1.index() == std::variant_npos);
-  }
-  {
-    using V = std::variant<int, MET, std::string>;
-    V v1(std::in_place_type<MET>);
-    V v2(std::in_place_type<std::string>, "hello");
-    V &vref = (v1 = std::move(v2));
-    assert(&vref == &v1);
-    assert(v1.index() == 2);
-    assert(std::get<2>(v1) == "hello");
-  }
-#endif // TEST_HAS_NO_EXCEPTIONS
 
   // Make sure we properly propagate triviality, which implies constexpr-ness (see P0602R4).
   {
@@ -461,38 +423,126 @@ void test_move_assignment_different_index() {
   }
 }
 
-template <std::size_t NewIdx, class ValueType>
-constexpr bool test_constexpr_assign_imp(
-    std::variant<long, void*, int>&& v, ValueType&& new_value)
-{
-  std::variant<long, void*, int> v2(
-      std::forward<ValueType>(new_value));
+void test_assignment_throw() {
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  using MET = MakeEmptyT;
+  // same index
+  {
+    using V = std::variant<int, MET, std::string>;
+    V v1(std::in_place_type<MET>);
+    MET& mref = std::get<1>(v1);
+    V v2(std::in_place_type<MET>);
+    try {
+      v1 = std::move(v2);
+      assert(false);
+    } catch (...) {
+    }
+    assert(v1.index() == 1);
+    assert(&std::get<1>(v1) == &mref);
+  }
+
+  // different indices
+  {
+    using V = std::variant<int, MET, std::string>;
+    V v1(std::in_place_type<int>);
+    V v2(std::in_place_type<MET>);
+    try {
+      v1 = std::move(v2);
+      assert(false);
+    } catch (...) {
+    }
+    assert(v1.valueless_by_exception());
+    assert(v1.index() == std::variant_npos);
+  }
+  {
+    using V = std::variant<int, MET, std::string>;
+    V v1(std::in_place_type<MET>);
+    V v2(std::in_place_type<std::string>, "hello");
+    V& vref = (v1 = std::move(v2));
+    assert(&vref == &v1);
+    assert(v1.index() == 2);
+    assert(std::get<2>(v1) == "hello");
+  }
+#endif // TEST_HAS_NO_EXCEPTIONS
+}
+
+template <std::size_t NewIdx, class T, class ValueType>
+constexpr void test_constexpr_assign_imp(T&& v, ValueType&& new_value) {
+  using Variant = std::decay_t<T>;
+  Variant v2(std::forward<ValueType>(new_value));
   const auto cp = v2;
-  v = std::move(v2);
-  return v.index() == NewIdx &&
-        std::get<NewIdx>(v) == std::get<NewIdx>(cp);
+  v             = std::move(v2);
+  assert(v.index() == NewIdx);
+  assert(std::get<NewIdx>(v) == std::get<NewIdx>(cp));
 }
 
-void test_constexpr_move_assignment() {
+constexpr void test_constexpr_move_assignment_trivial() {
   // Make sure we properly propagate triviality, which implies constexpr-ness (see P0602R4).
   using V = std::variant<long, void*, int>;
   static_assert(std::is_trivially_copyable<V>::value, "");
   static_assert(std::is_trivially_move_assignable<V>::value, "");
-  static_assert(test_constexpr_assign_imp<0>(V(42l), 101l), "");
-  static_assert(test_constexpr_assign_imp<0>(V(nullptr), 101l), "");
-  static_assert(test_constexpr_assign_imp<1>(V(42l), nullptr), "");
-  static_assert(test_constexpr_assign_imp<2>(V(42l), 101), "");
+  test_constexpr_assign_imp<0>(V(42l), 101l);
+  test_constexpr_assign_imp<0>(V(nullptr), 101l);
+  test_constexpr_assign_imp<1>(V(42l), nullptr);
+  test_constexpr_assign_imp<2>(V(42l), 101);
 }
 
-int main(int, char**) {
+struct NonTrivialMoveAssign {
+  int i = 0;
+  constexpr NonTrivialMoveAssign(int ii) : i(ii) {}
+  constexpr NonTrivialMoveAssign(const NonTrivialMoveAssign& other) = default;
+  constexpr NonTrivialMoveAssign(NonTrivialMoveAssign&& other) : i(other.i) {}
+  constexpr NonTrivialMoveAssign& operator=(const NonTrivialMoveAssign&) = default;
+  constexpr NonTrivialMoveAssign& operator=(NonTrivialMoveAssign&& o) {
+    i = o.i;
+    return *this;
+  }
+  TEST_CONSTEXPR_CXX20 ~NonTrivialMoveAssign() = default;
+  friend constexpr bool operator==(const NonTrivialMoveAssign& x, const NonTrivialMoveAssign& y) { return x.i == y.i; }
+};
+
+TEST_CONSTEXPR_CXX20 void test_constexpr_move_assignment_non_trivial() {
+  using V = std::variant<long, void*, NonTrivialMoveAssign>;
+  static_assert(!std::is_trivially_copyable<V>::value);
+  static_assert(!std::is_trivially_move_assignable<V>::value);
+  test_constexpr_assign_imp<0>(V(42l), 101l);
+  test_constexpr_assign_imp<0>(V(nullptr), 101l);
+  test_constexpr_assign_imp<1>(V(42l), nullptr);
+  test_constexpr_assign_imp<2>(V(42l), NonTrivialMoveAssign(5));
+  test_constexpr_assign_imp<2>(V(NonTrivialMoveAssign(3)), NonTrivialMoveAssign(5));
+}
+
+void non_constexpr_test() {
   test_move_assignment_empty_empty();
   test_move_assignment_non_empty_empty();
   test_move_assignment_empty_non_empty();
-  test_move_assignment_same_index();
-  test_move_assignment_different_index();
+  test_assignment_throw();
+}
+
+constexpr bool cxx17_constexpr_test() {
   test_move_assignment_sfinae();
   test_move_assignment_noexcept();
-  test_constexpr_move_assignment();
+  test_constexpr_move_assignment_trivial();
+
+  return true;
+}
 
+TEST_CONSTEXPR_CXX20 bool cxx20_constexpr_test() {
+  test_move_assignment_same_index();
+  test_move_assignment_different_index();
+  test_constexpr_move_assignment_non_trivial();
+
+  return true;
+}
+
+int main(int, char**) {
+  non_constexpr_test();
+  cxx17_constexpr_test();
+  cxx20_constexpr_test();
+
+  static_assert(cxx17_constexpr_test());
+#if TEST_STD_VER >= 20
+  static_assert(cxx20_constexpr_test());
+#endif
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/copy.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/copy.pass.cpp
index d1e5768f58d2..820ff9e0d1a9 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/copy.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/copy.pass.cpp
@@ -22,30 +22,30 @@
 #include "test_workarounds.h"
 
 struct NonT {
-  NonT(int v) : value(v) {}
-  NonT(const NonT &o) : value(o.value) {}
+  constexpr NonT(int v) : value(v) {}
+  constexpr NonT(const NonT& o) : value(o.value) {}
   int value;
 };
 static_assert(!std::is_trivially_copy_constructible<NonT>::value, "");
 
 struct NoCopy {
-  NoCopy(const NoCopy &) = delete;
+  NoCopy(const NoCopy&) = delete;
 };
 
 struct MoveOnly {
-  MoveOnly(const MoveOnly &) = delete;
-  MoveOnly(MoveOnly &&) = default;
+  MoveOnly(const MoveOnly&) = delete;
+  MoveOnly(MoveOnly&&)      = default;
 };
 
 struct MoveOnlyNT {
-  MoveOnlyNT(const MoveOnlyNT &) = delete;
-  MoveOnlyNT(MoveOnlyNT &&) {}
+  MoveOnlyNT(const MoveOnlyNT&) = delete;
+  MoveOnlyNT(MoveOnlyNT&&) {}
 };
 
 struct NTCopy {
   constexpr NTCopy(int v) : value(v) {}
-  NTCopy(const NTCopy &that) : value(that.value) {}
-  NTCopy(NTCopy &&) = delete;
+  NTCopy(const NTCopy& that) : value(that.value) {}
+  NTCopy(NTCopy&&) = delete;
   int value;
 };
 
@@ -54,8 +54,8 @@ static_assert(std::is_copy_constructible<NTCopy>::value, "");
 
 struct TCopy {
   constexpr TCopy(int v) : value(v) {}
-  TCopy(TCopy const &) = default;
-  TCopy(TCopy &&) = delete;
+  TCopy(TCopy const&) = default;
+  TCopy(TCopy&&)      = delete;
   int value;
 };
 
@@ -74,20 +74,21 @@ static_assert(std::is_trivially_copy_constructible<TCopyNTMove>::value, "");
 struct MakeEmptyT {
   static int alive;
   MakeEmptyT() { ++alive; }
-  MakeEmptyT(const MakeEmptyT &) {
+  MakeEmptyT(const MakeEmptyT&) {
     ++alive;
     // Don't throw from the copy constructor since variant's assignment
     // operator performs a copy before committing to the assignment.
   }
-  MakeEmptyT(MakeEmptyT &&) { throw 42; }
-  MakeEmptyT &operator=(const MakeEmptyT &) { throw 42; }
-  MakeEmptyT &operator=(MakeEmptyT &&) { throw 42; }
+  MakeEmptyT(MakeEmptyT&&) { throw 42; }
+  MakeEmptyT& operator=(const MakeEmptyT&) { throw 42; }
+  MakeEmptyT& operator=(MakeEmptyT&&) { throw 42; }
   ~MakeEmptyT() { --alive; }
 };
 
 int MakeEmptyT::alive = 0;
 
-template <class Variant> void makeEmpty(Variant &v) {
+template <class Variant>
+void makeEmpty(Variant& v) {
   Variant v2(std::in_place_type<MakeEmptyT>);
   try {
     v = std::move(v2);
@@ -98,7 +99,7 @@ template <class Variant> void makeEmpty(Variant &v) {
 }
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-void test_copy_ctor_sfinae() {
+constexpr void test_copy_ctor_sfinae() {
   {
     using V = std::variant<int, long>;
     static_assert(std::is_copy_constructible<V>::value, "");
@@ -136,7 +137,7 @@ void test_copy_ctor_sfinae() {
   }
 }
 
-void test_copy_ctor_basic() {
+TEST_CONSTEXPR_CXX20 void test_copy_ctor_basic() {
   {
     std::variant<int> v(std::in_place_index<0>, 42);
     std::variant<int> v2 = v;
@@ -214,21 +215,21 @@ void test_copy_ctor_valueless_by_exception() {
   using V = std::variant<int, MakeEmptyT>;
   V v1;
   makeEmpty(v1);
-  const V &cv1 = v1;
+  const V& cv1 = v1;
   V v(cv1);
   assert(v.valueless_by_exception());
 #endif // TEST_HAS_NO_EXCEPTIONS
 }
 
-template <std::size_t Idx>
-constexpr bool test_constexpr_copy_ctor_imp(std::variant<long, void*, const int> const& v) {
+template <std::size_t Idx, class T>
+constexpr void test_constexpr_copy_ctor_imp(const T& v) {
   auto v2 = v;
-  return v2.index() == v.index() &&
-         v2.index() == Idx &&
-         std::get<Idx>(v2) == std::get<Idx>(v);
+  assert(v2.index() == v.index());
+  assert(v2.index() == Idx);
+  assert(std::get<Idx>(v2) == std::get<Idx>(v));
 }
 
-void test_constexpr_copy_ctor() {
+constexpr void test_constexpr_copy_ctor_trivial() {
   // Make sure we properly propagate triviality, which implies constexpr-ness (see P0602R4).
   using V = std::variant<long, void*, const int>;
 #ifdef TEST_WORKAROUND_MSVC_BROKEN_IS_TRIVIALLY_COPYABLE
@@ -237,18 +238,57 @@ void test_constexpr_copy_ctor() {
   static_assert(std::is_trivially_move_constructible<V>::value, "");
   static_assert(!std::is_copy_assignable<V>::value, "");
   static_assert(!std::is_move_assignable<V>::value, "");
-#else // TEST_WORKAROUND_MSVC_BROKEN_IS_TRIVIALLY_COPYABLE
+#else  // TEST_WORKAROUND_MSVC_BROKEN_IS_TRIVIALLY_COPYABLE
   static_assert(std::is_trivially_copyable<V>::value, "");
 #endif // TEST_WORKAROUND_MSVC_BROKEN_IS_TRIVIALLY_COPYABLE
-  static_assert(test_constexpr_copy_ctor_imp<0>(V(42l)), "");
-  static_assert(test_constexpr_copy_ctor_imp<1>(V(nullptr)), "");
-  static_assert(test_constexpr_copy_ctor_imp<2>(V(101)), "");
+  static_assert(std::is_trivially_copy_constructible<V>::value, "");
+  test_constexpr_copy_ctor_imp<0>(V(42l));
+  test_constexpr_copy_ctor_imp<1>(V(nullptr));
+  test_constexpr_copy_ctor_imp<2>(V(101));
 }
 
-int main(int, char**) {
-  test_copy_ctor_basic();
-  test_copy_ctor_valueless_by_exception();
+struct NonTrivialCopyCtor {
+  int i = 0;
+  constexpr NonTrivialCopyCtor(int ii) : i(ii) {}
+  constexpr NonTrivialCopyCtor(const NonTrivialCopyCtor& other) : i(other.i) {}
+  constexpr NonTrivialCopyCtor(NonTrivialCopyCtor&& other) = default;
+  TEST_CONSTEXPR_CXX20 ~NonTrivialCopyCtor()               = default;
+  friend constexpr bool operator==(const NonTrivialCopyCtor& x, const NonTrivialCopyCtor& y) { return x.i == y.i; }
+};
+
+TEST_CONSTEXPR_CXX20 void test_constexpr_copy_ctor_non_trivial() {
+  // Test !is_trivially_move_constructible
+  using V = std::variant<long, NonTrivialCopyCtor, void*>;
+  static_assert(!std::is_trivially_copy_constructible<V>::value, "");
+  test_constexpr_copy_ctor_imp<0>(V(42l));
+  test_constexpr_copy_ctor_imp<1>(V(NonTrivialCopyCtor(5)));
+  test_constexpr_copy_ctor_imp<2>(V(nullptr));
+}
+
+void non_constexpr_test() { test_copy_ctor_valueless_by_exception(); }
+
+constexpr bool cxx17_constexpr_test() {
   test_copy_ctor_sfinae();
-  test_constexpr_copy_ctor();
+  test_constexpr_copy_ctor_trivial();
+
+  return true;
+}
+
+TEST_CONSTEXPR_CXX20 bool cxx20_constexpr_test() {
+  test_copy_ctor_basic();
+  test_constexpr_copy_ctor_non_trivial();
+
+  return true;
+}
+
+int main(int, char**) {
+  non_constexpr_test();
+  cxx17_constexpr_test();
+  cxx20_constexpr_test();
+
+  static_assert(cxx17_constexpr_test());
+#if TEST_STD_VER >= 20
+  static_assert(cxx20_constexpr_test());
+#endif
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/default.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/default.pass.cpp
index 40db038a0033..9abf4d758d84 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/default.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/default.pass.cpp
@@ -17,6 +17,7 @@
 #include <cassert>
 #include <type_traits>
 #include <variant>
+#include <string>
 
 #include "test_macros.h"
 #include "variant_test_helpers.h"
@@ -35,7 +36,7 @@ struct DefaultCtorThrows {
 };
 #endif
 
-void test_default_ctor_sfinae() {
+constexpr void test_default_ctor_sfinae() {
   {
     using V = std::variant<std::monostate, int>;
     static_assert(std::is_default_constructible<V>::value, "");
@@ -46,7 +47,7 @@ void test_default_ctor_sfinae() {
   }
 }
 
-void test_default_ctor_noexcept() {
+constexpr void test_default_ctor_noexcept() {
   {
     using V = std::variant<int>;
     static_assert(std::is_nothrow_default_constructible<V>::value, "");
@@ -63,7 +64,7 @@ void test_default_ctor_throws() {
   try {
     V v;
     assert(false);
-  } catch (const int &ex) {
+  } catch (const int& ex) {
     assert(ex == 42);
   } catch (...) {
     assert(false);
@@ -71,7 +72,7 @@ void test_default_ctor_throws() {
 #endif
 }
 
-void test_default_ctor_basic() {
+constexpr void test_default_ctor_basic() {
   {
     std::variant<int> v;
     assert(v.index() == 0);
@@ -107,11 +108,24 @@ void test_default_ctor_basic() {
   }
 }
 
-int main(int, char**) {
+constexpr void issue_86686() {
+#if TEST_STD_VER >= 20
+  static_assert(std::variant<std::string>{}.index() == 0);
+#endif
+}
+
+constexpr bool test() {
   test_default_ctor_basic();
   test_default_ctor_sfinae();
   test_default_ctor_noexcept();
-  test_default_ctor_throws();
+  issue_86686();
 
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  test_default_ctor_throws();
+  static_assert(test());
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/move.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/move.pass.cpp
index e2518fe29caf..4e8453c23cf5 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/move.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/move.pass.cpp
@@ -23,31 +23,31 @@
 #include "test_workarounds.h"
 
 struct ThrowsMove {
-  ThrowsMove(ThrowsMove &&) noexcept(false) {}
+  ThrowsMove(ThrowsMove&&) noexcept(false) {}
 };
 
 struct NoCopy {
-  NoCopy(const NoCopy &) = delete;
+  NoCopy(const NoCopy&) = delete;
 };
 
 struct MoveOnly {
   int value;
-  MoveOnly(int v) : value(v) {}
-  MoveOnly(const MoveOnly &) = delete;
-  MoveOnly(MoveOnly &&) = default;
+  constexpr MoveOnly(int v) : value(v) {}
+  MoveOnly(const MoveOnly&) = delete;
+  MoveOnly(MoveOnly&&)      = default;
 };
 
 struct MoveOnlyNT {
   int value;
-  MoveOnlyNT(int v) : value(v) {}
-  MoveOnlyNT(const MoveOnlyNT &) = delete;
-  MoveOnlyNT(MoveOnlyNT &&other) : value(other.value) { other.value = -1; }
+  constexpr MoveOnlyNT(int v) : value(v) {}
+  MoveOnlyNT(const MoveOnlyNT&) = delete;
+  constexpr MoveOnlyNT(MoveOnlyNT&& other) : value(other.value) { other.value = -1; }
 };
 
 struct NTMove {
   constexpr NTMove(int v) : value(v) {}
-  NTMove(const NTMove &) = delete;
-  NTMove(NTMove &&that) : value(that.value) { that.value = -1; }
+  NTMove(const NTMove&) = delete;
+  NTMove(NTMove&& that) : value(that.value) { that.value = -1; }
   int value;
 };
 
@@ -56,8 +56,8 @@ static_assert(std::is_move_constructible<NTMove>::value, "");
 
 struct TMove {
   constexpr TMove(int v) : value(v) {}
-  TMove(const TMove &) = delete;
-  TMove(TMove &&) = default;
+  TMove(const TMove&) = delete;
+  TMove(TMove&&)      = default;
   int value;
 };
 
@@ -76,20 +76,21 @@ static_assert(std::is_trivially_move_constructible<TMoveNTCopy>::value, "");
 struct MakeEmptyT {
   static int alive;
   MakeEmptyT() { ++alive; }
-  MakeEmptyT(const MakeEmptyT &) {
+  MakeEmptyT(const MakeEmptyT&) {
     ++alive;
     // Don't throw from the copy constructor since variant's assignment
     // operator performs a copy before committing to the assignment.
   }
-  MakeEmptyT(MakeEmptyT &&) { throw 42; }
-  MakeEmptyT &operator=(const MakeEmptyT &) { throw 42; }
-  MakeEmptyT &operator=(MakeEmptyT &&) { throw 42; }
+  MakeEmptyT(MakeEmptyT&&) { throw 42; }
+  MakeEmptyT& operator=(const MakeEmptyT&) { throw 42; }
+  MakeEmptyT& operator=(MakeEmptyT&&) { throw 42; }
   ~MakeEmptyT() { --alive; }
 };
 
 int MakeEmptyT::alive = 0;
 
-template <class Variant> void makeEmpty(Variant &v) {
+template <class Variant>
+void makeEmpty(Variant& v) {
   Variant v2(std::in_place_type<MakeEmptyT>);
   try {
     v = std::move(v2);
@@ -100,7 +101,7 @@ template <class Variant> void makeEmpty(Variant &v) {
 }
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-void test_move_noexcept() {
+constexpr void test_move_noexcept() {
   {
     using V = std::variant<int, long>;
     static_assert(std::is_nothrow_move_constructible<V>::value, "");
@@ -119,7 +120,7 @@ void test_move_noexcept() {
   }
 }
 
-void test_move_ctor_sfinae() {
+constexpr void test_move_ctor_sfinae() {
   {
     using V = std::variant<int, long>;
     static_assert(std::is_move_constructible<V>::value, "");
@@ -158,9 +159,12 @@ void test_move_ctor_sfinae() {
 }
 
 template <typename T>
-struct Result { std::size_t index; T value; };
+struct Result {
+  std::size_t index;
+  T value;
+};
 
-void test_move_ctor_basic() {
+TEST_CONSTEXPR_CXX20 void test_move_ctor_basic() {
   {
     std::variant<int> v(std::in_place_index<0>, 42);
     std::variant<int> v2 = std::move(v);
@@ -289,16 +293,16 @@ void test_move_ctor_valueless_by_exception() {
 #endif // TEST_HAS_NO_EXCEPTIONS
 }
 
-template <std::size_t Idx>
-constexpr bool test_constexpr_ctor_imp(std::variant<long, void*, const int> const& v) {
+template <std::size_t Idx, class T>
+constexpr void test_constexpr_ctor_imp(const T& v) {
   auto copy = v;
-  auto v2 = std::move(copy);
-  return v2.index() == v.index() &&
-         v2.index() == Idx &&
-        std::get<Idx>(v2) == std::get<Idx>(v);
+  auto v2   = std::move(copy);
+  assert(v2.index() == v.index());
+  assert(v2.index() == Idx);
+  assert(std::get<Idx>(v2) == std::get<Idx>(v));
 }
 
-void test_constexpr_move_ctor() {
+constexpr void test_constexpr_move_ctor_trivial() {
   // Make sure we properly propagate triviality, which implies constexpr-ness (see P0602R4).
   using V = std::variant<long, void*, const int>;
 #ifdef TEST_WORKAROUND_MSVC_BROKEN_IS_TRIVIALLY_COPYABLE
@@ -307,21 +311,58 @@ void test_constexpr_move_ctor() {
   static_assert(std::is_trivially_move_constructible<V>::value, "");
   static_assert(!std::is_copy_assignable<V>::value, "");
   static_assert(!std::is_move_assignable<V>::value, "");
-#else // TEST_WORKAROUND_MSVC_BROKEN_IS_TRIVIALLY_COPYABLE
+#else  // TEST_WORKAROUND_MSVC_BROKEN_IS_TRIVIALLY_COPYABLE
   static_assert(std::is_trivially_copyable<V>::value, "");
 #endif // TEST_WORKAROUND_MSVC_BROKEN_IS_TRIVIALLY_COPYABLE
   static_assert(std::is_trivially_move_constructible<V>::value, "");
-  static_assert(test_constexpr_ctor_imp<0>(V(42l)), "");
-  static_assert(test_constexpr_ctor_imp<1>(V(nullptr)), "");
-  static_assert(test_constexpr_ctor_imp<2>(V(101)), "");
+  test_constexpr_ctor_imp<0>(V(42l));
+  test_constexpr_ctor_imp<1>(V(nullptr));
+  test_constexpr_ctor_imp<2>(V(101));
 }
 
-int main(int, char**) {
-  test_move_ctor_basic();
-  test_move_ctor_valueless_by_exception();
+struct NonTrivialMoveCtor {
+  int i = 0;
+  constexpr NonTrivialMoveCtor(int ii) : i(ii) {}
+  constexpr NonTrivialMoveCtor(const NonTrivialMoveCtor& other) = default;
+  constexpr NonTrivialMoveCtor(NonTrivialMoveCtor&& other) : i(other.i) {}
+  TEST_CONSTEXPR_CXX20 ~NonTrivialMoveCtor() = default;
+  friend constexpr bool operator==(const NonTrivialMoveCtor& x, const NonTrivialMoveCtor& y) { return x.i == y.i; }
+};
+
+TEST_CONSTEXPR_CXX20 void test_constexpr_move_ctor_non_trivial() {
+  using V = std::variant<long, NonTrivialMoveCtor, void*>;
+  static_assert(!std::is_trivially_move_constructible<V>::value, "");
+  test_constexpr_ctor_imp<0>(V(42l));
+  test_constexpr_ctor_imp<1>(V(NonTrivialMoveCtor(5)));
+  test_constexpr_ctor_imp<2>(V(nullptr));
+}
+
+void non_constexpr_test() { test_move_ctor_valueless_by_exception(); }
+
+constexpr bool cxx17_constexpr_test() {
   test_move_noexcept();
   test_move_ctor_sfinae();
-  test_constexpr_move_ctor();
+  test_constexpr_move_ctor_trivial();
+
+  return true;
+}
+
+TEST_CONSTEXPR_CXX20 bool cxx20_constexpr_test() {
+  test_move_ctor_basic();
+  test_constexpr_move_ctor_non_trivial();
+
+  return true;
+}
+
+int main(int, char**) {
+  non_constexpr_test();
+  cxx17_constexpr_test();
+  cxx20_constexpr_test();
+
+  static_assert(cxx17_constexpr_test());
+#if TEST_STD_VER >= 20
+  static_assert(cxx20_constexpr_test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.dtor/dtor.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.dtor/dtor.pass.cpp
index 2e026038c97a..53c5283b2edc 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.dtor/dtor.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.dtor/dtor.pass.cpp
@@ -21,55 +21,76 @@
 #include "test_macros.h"
 
 struct NonTDtor {
-  static int count;
-  NonTDtor() = default;
-  ~NonTDtor() { ++count; }
+  int* count;
+  constexpr NonTDtor(int* a, int*) : count(a) {}
+  TEST_CONSTEXPR_CXX20 ~NonTDtor() { ++*count; }
 };
-int NonTDtor::count = 0;
 static_assert(!std::is_trivially_destructible<NonTDtor>::value, "");
 
 struct NonTDtor1 {
-  static int count;
-  NonTDtor1() = default;
-  ~NonTDtor1() { ++count; }
+  int* count;
+  constexpr NonTDtor1(int*, int* b) : count(b) {}
+  TEST_CONSTEXPR_CXX20 ~NonTDtor1() { ++*count; }
 };
-int NonTDtor1::count = 0;
 static_assert(!std::is_trivially_destructible<NonTDtor1>::value, "");
 
 struct TDtor {
-  TDtor(const TDtor &) {} // non-trivial copy
-  ~TDtor() = default;
+  constexpr TDtor() = default;
+  constexpr TDtor(const TDtor&) {} // non-trivial copy
+  TEST_CONSTEXPR_CXX20 ~TDtor() = default;
 };
 static_assert(!std::is_trivially_copy_constructible<TDtor>::value, "");
 static_assert(std::is_trivially_destructible<TDtor>::value, "");
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX20 bool test() {
   {
     using V = std::variant<int, long, TDtor>;
     static_assert(std::is_trivially_destructible<V>::value, "");
+    [[maybe_unused]] V v(std::in_place_index<2>);
   }
   {
     using V = std::variant<NonTDtor, int, NonTDtor1>;
     static_assert(!std::is_trivially_destructible<V>::value, "");
     {
-      V v(std::in_place_index<0>);
-      assert(NonTDtor::count == 0);
-      assert(NonTDtor1::count == 0);
+      int count0 = 0;
+      int count1 = 0;
+      {
+        V v(std::in_place_index<0>, &count0, &count1);
+        assert(count0 == 0);
+        assert(count1 == 0);
+      }
+      assert(count0 == 1);
+      assert(count1 == 0);
+    }
+    {
+      int count0 = 0;
+      int count1 = 0;
+      { V v(std::in_place_index<1>); }
+      assert(count0 == 0);
+      assert(count1 == 0);
     }
-    assert(NonTDtor::count == 1);
-    assert(NonTDtor1::count == 0);
-    NonTDtor::count = 0;
-    { V v(std::in_place_index<1>); }
-    assert(NonTDtor::count == 0);
-    assert(NonTDtor1::count == 0);
     {
-      V v(std::in_place_index<2>);
-      assert(NonTDtor::count == 0);
-      assert(NonTDtor1::count == 0);
+      int count0 = 0;
+      int count1 = 0;
+      {
+        V v(std::in_place_index<2>, &count0, &count1);
+        assert(count0 == 0);
+        assert(count1 == 0);
+      }
+      assert(count0 == 0);
+      assert(count1 == 1);
     }
-    assert(NonTDtor::count == 0);
-    assert(NonTDtor1::count == 1);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  test();
+
+#if TEST_STD_VER >= 20
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_args.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_args.pass.cpp
index 2fe9033dd816..f98d968f0eae 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_args.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_args.pass.cpp
@@ -26,8 +26,8 @@
 #include "variant_test_helpers.h"
 
 template <class Var, std::size_t I, class... Args>
-constexpr auto test_emplace_exists_imp(int) -> decltype(
-    std::declval<Var>().template emplace<I>(std::declval<Args>()...), true) {
+constexpr auto test_emplace_exists_imp(int)
+    -> decltype(std::declval<Var>().template emplace<I>(std::declval<Args>()...), true) {
   return true;
 }
 
@@ -36,28 +36,32 @@ constexpr auto test_emplace_exists_imp(long) -> bool {
   return false;
 }
 
-template <class Var, std::size_t I, class... Args> constexpr bool emplace_exists() {
+template <class Var, std::size_t I, class... Args>
+constexpr bool emplace_exists() {
   return test_emplace_exists_imp<Var, I, Args...>(0);
 }
 
-void test_emplace_sfinae() {
+constexpr void test_emplace_sfinae() {
   {
-    using V = std::variant<int, void *, const void *, TestTypes::NoCtors>;
+    using V = std::variant<int, void*, const void*, TestTypes::NoCtors>;
     static_assert(emplace_exists<V, 0>(), "");
     static_assert(emplace_exists<V, 0, int>(), "");
-    static_assert(!emplace_exists<V, 0, decltype(nullptr)>(),
-                  "cannot construct");
+    static_assert(!emplace_exists<V, 0, decltype(nullptr)>(), "cannot construct");
     static_assert(emplace_exists<V, 1, decltype(nullptr)>(), "");
-    static_assert(emplace_exists<V, 1, int *>(), "");
-    static_assert(!emplace_exists<V, 1, const int *>(), "");
+    static_assert(emplace_exists<V, 1, int*>(), "");
+    static_assert(!emplace_exists<V, 1, const int*>(), "");
     static_assert(!emplace_exists<V, 1, int>(), "cannot construct");
-    static_assert(emplace_exists<V, 2, const int *>(), "");
-    static_assert(emplace_exists<V, 2, int *>(), "");
+    static_assert(emplace_exists<V, 2, const int*>(), "");
+    static_assert(emplace_exists<V, 2, int*>(), "");
     static_assert(!emplace_exists<V, 3>(), "cannot construct");
   }
 }
 
-void test_basic() {
+struct NoCtor {
+  NoCtor() = delete;
+};
+
+TEST_CONSTEXPR_CXX20 void test_basic() {
   {
     using V = std::variant<int>;
     V v(42);
@@ -70,9 +74,9 @@ void test_basic() {
     assert(std::get<0>(v) == 42);
     assert(&ref2 == &std::get<0>(v));
   }
+
   {
-    using V =
-        std::variant<int, long, const void *, TestTypes::NoCtors, std::string>;
+    using V     = std::variant<int, long, const void*, NoCtor, std::string>;
     const int x = 100;
     V v(std::in_place_index<0>, -1);
     // default emplace a value
@@ -92,9 +96,19 @@ void test_basic() {
   }
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX20 bool test() {
   test_basic();
   test_emplace_sfinae();
 
+  return true;
+}
+
+int main(int, char**) {
+  test();
+
+#if TEST_STD_VER >= 20
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_init_list_args.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_init_list_args.pass.cpp
index 9068aacc4359..4c635570bd56 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_init_list_args.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_init_list_args.pass.cpp
@@ -32,13 +32,12 @@ struct InitList {
 struct InitListArg {
   std::size_t size;
   int value;
-  constexpr InitListArg(std::initializer_list<int> il, int v)
-      : size(il.size()), value(v) {}
+  constexpr InitListArg(std::initializer_list<int> il, int v) : size(il.size()), value(v) {}
 };
 
 template <class Var, std::size_t I, class... Args>
-constexpr auto test_emplace_exists_imp(int) -> decltype(
-    std::declval<Var>().template emplace<I>(std::declval<Args>()...), true) {
+constexpr auto test_emplace_exists_imp(int)
+    -> decltype(std::declval<Var>().template emplace<I>(std::declval<Args>()...), true) {
   return true;
 }
 
@@ -47,13 +46,13 @@ constexpr auto test_emplace_exists_imp(long) -> bool {
   return false;
 }
 
-template <class Var, std::size_t I, class... Args> constexpr bool emplace_exists() {
+template <class Var, std::size_t I, class... Args>
+constexpr bool emplace_exists() {
   return test_emplace_exists_imp<Var, I, Args...>(0);
 }
 
-void test_emplace_sfinae() {
-  using V =
-      std::variant<int, TestTypes::NoCtors, InitList, InitListArg, long, long>;
+constexpr void test_emplace_sfinae() {
+  using V  = std::variant<int, TestTypes::NoCtors, InitList, InitListArg, long, long>;
   using IL = std::initializer_list<int>;
   static_assert(!emplace_exists<V, 1, IL>(), "no such constructor");
   static_assert(emplace_exists<V, 2, IL>(), "");
@@ -65,8 +64,12 @@ void test_emplace_sfinae() {
   static_assert(!emplace_exists<V, 3, IL, int, int>(), "too many args");
 }
 
-void test_basic() {
-  using V = std::variant<int, InitList, InitListArg, TestTypes::NoCtors>;
+struct NoCtor {
+  NoCtor() = delete;
+};
+
+TEST_CONSTEXPR_CXX20 void test_basic() {
+  using V = std::variant<int, InitList, InitListArg, NoCtor>;
   V v;
   auto& ref1 = v.emplace<1>({1, 2, 3});
   static_assert(std::is_same_v<InitList&, decltype(ref1)>, "");
@@ -83,9 +86,19 @@ void test_basic() {
   assert(&ref3 == &std::get<1>(v));
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX20 bool test() {
   test_basic();
   test_emplace_sfinae();
 
+  return true;
+}
+
+int main(int, char**) {
+  test();
+
+#if TEST_STD_VER >= 20
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_args.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_args.pass.cpp
index 4e9f67775d10..c2ed54d8a625 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_args.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_args.pass.cpp
@@ -25,8 +25,8 @@
 #include "variant_test_helpers.h"
 
 template <class Var, class T, class... Args>
-constexpr auto test_emplace_exists_imp(int) -> decltype(
-    std::declval<Var>().template emplace<T>(std::declval<Args>()...), true) {
+constexpr auto test_emplace_exists_imp(int)
+    -> decltype(std::declval<Var>().template emplace<T>(std::declval<Args>()...), true) {
   return true;
 }
 
@@ -35,28 +35,32 @@ constexpr auto test_emplace_exists_imp(long) -> bool {
   return false;
 }
 
-template <class... Args> constexpr bool emplace_exists() {
+template <class... Args>
+constexpr bool emplace_exists() {
   return test_emplace_exists_imp<Args...>(0);
 }
 
-void test_emplace_sfinae() {
+constexpr void test_emplace_sfinae() {
   {
-    using V = std::variant<int, void *, const void *, TestTypes::NoCtors>;
+    using V = std::variant<int, void*, const void*, TestTypes::NoCtors>;
     static_assert(emplace_exists<V, int>(), "");
     static_assert(emplace_exists<V, int, int>(), "");
-    static_assert(!emplace_exists<V, int, decltype(nullptr)>(),
-                  "cannot construct");
-    static_assert(emplace_exists<V, void *, decltype(nullptr)>(), "");
-    static_assert(!emplace_exists<V, void *, int>(), "cannot construct");
-    static_assert(emplace_exists<V, void *, int *>(), "");
-    static_assert(!emplace_exists<V, void *, const int *>(), "");
-    static_assert(emplace_exists<V, const void *, const int *>(), "");
-    static_assert(emplace_exists<V, const void *, int *>(), "");
+    static_assert(!emplace_exists<V, int, decltype(nullptr)>(), "cannot construct");
+    static_assert(emplace_exists<V, void*, decltype(nullptr)>(), "");
+    static_assert(!emplace_exists<V, void*, int>(), "cannot construct");
+    static_assert(emplace_exists<V, void*, int*>(), "");
+    static_assert(!emplace_exists<V, void*, const int*>(), "");
+    static_assert(emplace_exists<V, const void*, const int*>(), "");
+    static_assert(emplace_exists<V, const void*, int*>(), "");
     static_assert(!emplace_exists<V, TestTypes::NoCtors>(), "cannot construct");
   }
 }
 
-void test_basic() {
+struct NoCtor {
+  NoCtor() = delete;
+};
+
+TEST_CONSTEXPR_CXX20 void test_basic() {
   {
     using V = std::variant<int>;
     V v(42);
@@ -70,8 +74,7 @@ void test_basic() {
     assert(&ref2 == &std::get<0>(v));
   }
   {
-    using V =
-        std::variant<int, long, const void *, TestTypes::NoCtors, std::string>;
+    using V     = std::variant<int, long, const void*, NoCtor, std::string>;
     const int x = 100;
     V v(std::in_place_type<int>, -1);
     // default emplace a value
@@ -79,8 +82,8 @@ void test_basic() {
     static_assert(std::is_same_v<long&, decltype(ref1)>, "");
     assert(std::get<1>(v) == 0);
     assert(&ref1 == &std::get<1>(v));
-    auto& ref2 = v.emplace<const void *>(&x);
-    static_assert(std::is_same_v<const void *&, decltype(ref2)>, "");
+    auto& ref2 = v.emplace<const void*>(&x);
+    static_assert(std::is_same_v<const void*&, decltype(ref2)>, "");
     assert(std::get<2>(v) == &x);
     assert(&ref2 == &std::get<2>(v));
     // emplace with multiple args
@@ -91,9 +94,19 @@ void test_basic() {
   }
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX20 bool test() {
   test_basic();
   test_emplace_sfinae();
 
+  return true;
+}
+
+int main(int, char**) {
+  test();
+
+#if TEST_STD_VER >= 20
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_init_list_args.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_init_list_args.pass.cpp
index 74d834b9b345..644f2418b925 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_init_list_args.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_init_list_args.pass.cpp
@@ -32,13 +32,12 @@ struct InitList {
 struct InitListArg {
   std::size_t size;
   int value;
-  constexpr InitListArg(std::initializer_list<int> il, int v)
-      : size(il.size()), value(v) {}
+  constexpr InitListArg(std::initializer_list<int> il, int v) : size(il.size()), value(v) {}
 };
 
 template <class Var, class T, class... Args>
-constexpr auto test_emplace_exists_imp(int) -> decltype(
-    std::declval<Var>().template emplace<T>(std::declval<Args>()...), true) {
+constexpr auto test_emplace_exists_imp(int)
+    -> decltype(std::declval<Var>().template emplace<T>(std::declval<Args>()...), true) {
   return true;
 }
 
@@ -47,13 +46,13 @@ constexpr auto test_emplace_exists_imp(long) -> bool {
   return false;
 }
 
-template <class... Args> constexpr bool emplace_exists() {
+template <class... Args>
+constexpr bool emplace_exists() {
   return test_emplace_exists_imp<Args...>(0);
 }
 
-void test_emplace_sfinae() {
-  using V =
-      std::variant<int, TestTypes::NoCtors, InitList, InitListArg, long, long>;
+constexpr void test_emplace_sfinae() {
+  using V  = std::variant<int, TestTypes::NoCtors, InitList, InitListArg, long, long>;
   using IL = std::initializer_list<int>;
   static_assert(emplace_exists<V, InitList, IL>(), "");
   static_assert(!emplace_exists<V, InitList, int>(), "args don't match");
@@ -61,31 +60,44 @@ void test_emplace_sfinae() {
   static_assert(emplace_exists<V, InitListArg, IL, int>(), "");
   static_assert(!emplace_exists<V, InitListArg, int>(), "args don't match");
   static_assert(!emplace_exists<V, InitListArg, IL>(), "too few args");
-  static_assert(!emplace_exists<V, InitListArg, IL, int, int>(),
-                "too many args");
+  static_assert(!emplace_exists<V, InitListArg, IL, int, int>(), "too many args");
 }
 
-void test_basic() {
-  using V = std::variant<int, InitList, InitListArg, TestTypes::NoCtors>;
+struct NoCtor {
+  NoCtor() = delete;
+};
+
+TEST_CONSTEXPR_CXX20 void test_basic() {
+  using V = std::variant<int, InitList, InitListArg, NoCtor>;
   V v;
   auto& ref1 = v.emplace<InitList>({1, 2, 3});
-  static_assert(std::is_same_v<InitList&,decltype(ref1)>, "");
+  static_assert(std::is_same_v<InitList&, decltype(ref1)>, "");
   assert(std::get<InitList>(v).size == 3);
   assert(&ref1 == &std::get<InitList>(v));
   auto& ref2 = v.emplace<InitListArg>({1, 2, 3, 4}, 42);
-  static_assert(std::is_same_v<InitListArg&,decltype(ref2)>, "");
+  static_assert(std::is_same_v<InitListArg&, decltype(ref2)>, "");
   assert(std::get<InitListArg>(v).size == 4);
   assert(std::get<InitListArg>(v).value == 42);
   assert(&ref2 == &std::get<InitListArg>(v));
   auto& ref3 = v.emplace<InitList>({1});
-  static_assert(std::is_same_v<InitList&,decltype(ref3)>, "");
+  static_assert(std::is_same_v<InitList&, decltype(ref3)>, "");
   assert(std::get<InitList>(v).size == 1);
   assert(&ref3 == &std::get<InitList>(v));
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX20 bool test() {
   test_basic();
   test_emplace_sfinae();
 
+  return true;
+}
+
+int main(int, char**) {
+  test();
+
+#if TEST_STD_VER >= 20
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.swap/swap.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.swap/swap.pass.cpp
index 1802bc4670bb..db05691c5581 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.swap/swap.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.swap/swap.pass.cpp
@@ -25,37 +25,39 @@
 #include "variant_test_helpers.h"
 
 struct NotSwappable {};
-void swap(NotSwappable &, NotSwappable &) = delete;
+void swap(NotSwappable&, NotSwappable&) = delete;
 
 struct NotCopyable {
-  NotCopyable() = default;
-  NotCopyable(const NotCopyable &) = delete;
-  NotCopyable &operator=(const NotCopyable &) = delete;
+  NotCopyable()                              = default;
+  NotCopyable(const NotCopyable&)            = delete;
+  NotCopyable& operator=(const NotCopyable&) = delete;
 };
 
 struct NotCopyableWithSwap {
-  NotCopyableWithSwap() = default;
-  NotCopyableWithSwap(const NotCopyableWithSwap &) = delete;
-  NotCopyableWithSwap &operator=(const NotCopyableWithSwap &) = delete;
+  NotCopyableWithSwap()                                      = default;
+  NotCopyableWithSwap(const NotCopyableWithSwap&)            = delete;
+  NotCopyableWithSwap& operator=(const NotCopyableWithSwap&) = delete;
 };
-void swap(NotCopyableWithSwap &, NotCopyableWithSwap) {}
+constexpr void swap(NotCopyableWithSwap&, NotCopyableWithSwap) {}
 
 struct NotMoveAssignable {
-  NotMoveAssignable() = default;
-  NotMoveAssignable(NotMoveAssignable &&) = default;
-  NotMoveAssignable &operator=(NotMoveAssignable &&) = delete;
+  NotMoveAssignable()                               = default;
+  NotMoveAssignable(NotMoveAssignable&&)            = default;
+  NotMoveAssignable& operator=(NotMoveAssignable&&) = delete;
 };
 
 struct NotMoveAssignableWithSwap {
-  NotMoveAssignableWithSwap() = default;
-  NotMoveAssignableWithSwap(NotMoveAssignableWithSwap &&) = default;
-  NotMoveAssignableWithSwap &operator=(NotMoveAssignableWithSwap &&) = delete;
+  NotMoveAssignableWithSwap()                                       = default;
+  NotMoveAssignableWithSwap(NotMoveAssignableWithSwap&&)            = default;
+  NotMoveAssignableWithSwap& operator=(NotMoveAssignableWithSwap&&) = delete;
 };
-void swap(NotMoveAssignableWithSwap &, NotMoveAssignableWithSwap &) noexcept {}
+constexpr void swap(NotMoveAssignableWithSwap&, NotMoveAssignableWithSwap&) noexcept {}
 
-template <bool Throws> void do_throw() {}
+template <bool Throws>
+constexpr void do_throw() {}
 
-template <> void do_throw<true>() {
+template <>
+void do_throw<true>() {
 #ifndef TEST_HAS_NO_EXCEPTIONS
   throw 42;
 #else
@@ -63,60 +65,49 @@ template <> void do_throw<true>() {
 #endif
 }
 
-template <bool NT_Copy, bool NT_Move, bool NT_CopyAssign, bool NT_MoveAssign,
-          bool NT_Swap, bool EnableSwap = true>
+template <bool NT_Copy, bool NT_Move, bool NT_CopyAssign, bool NT_MoveAssign, bool NT_Swap, bool EnableSwap = true>
 struct NothrowTypeImp {
-  static int move_called;
-  static int move_assign_called;
-  static int swap_called;
-  static void reset() { move_called = move_assign_called = swap_called = 0; }
-  NothrowTypeImp() = default;
-  explicit NothrowTypeImp(int v) : value(v) {}
-  NothrowTypeImp(const NothrowTypeImp &o) noexcept(NT_Copy) : value(o.value) {
-    assert(false);
-  } // never called by test
-  NothrowTypeImp(NothrowTypeImp &&o) noexcept(NT_Move) : value(o.value) {
-    ++move_called;
+  int value;
+  int* move_called;
+  int* move_assign_called;
+  int* swap_called;
+
+  constexpr NothrowTypeImp(int v, int* mv_ctr, int* mv_assign, int* swap)
+      : value(v), move_called(mv_ctr), move_assign_called(mv_assign), swap_called(swap) {}
+
+  NothrowTypeImp(const NothrowTypeImp& o) noexcept(NT_Copy) : value(o.value) { assert(false); } // never called by test
+
+  constexpr NothrowTypeImp(NothrowTypeImp&& o) noexcept(NT_Move)
+      : value(o.value),
+        move_called(o.move_called),
+        move_assign_called(o.move_assign_called),
+        swap_called(o.swap_called) {
+    ++*move_called;
     do_throw<!NT_Move>();
     o.value = -1;
   }
-  NothrowTypeImp &operator=(const NothrowTypeImp &) noexcept(NT_CopyAssign) {
+
+  NothrowTypeImp& operator=(const NothrowTypeImp&) noexcept(NT_CopyAssign) {
     assert(false);
     return *this;
   } // never called by the tests
-  NothrowTypeImp &operator=(NothrowTypeImp &&o) noexcept(NT_MoveAssign) {
-    ++move_assign_called;
+
+  constexpr NothrowTypeImp& operator=(NothrowTypeImp&& o) noexcept(NT_MoveAssign) {
+    ++*move_assign_called;
     do_throw<!NT_MoveAssign>();
-    value = o.value;
+    value   = o.value;
     o.value = -1;
     return *this;
   }
-  int value;
 };
-template <bool NT_Copy, bool NT_Move, bool NT_CopyAssign, bool NT_MoveAssign,
-          bool NT_Swap, bool EnableSwap>
-int NothrowTypeImp<NT_Copy, NT_Move, NT_CopyAssign, NT_MoveAssign, NT_Swap,
-                   EnableSwap>::move_called = 0;
-template <bool NT_Copy, bool NT_Move, bool NT_CopyAssign, bool NT_MoveAssign,
-          bool NT_Swap, bool EnableSwap>
-int NothrowTypeImp<NT_Copy, NT_Move, NT_CopyAssign, NT_MoveAssign, NT_Swap,
-                   EnableSwap>::move_assign_called = 0;
-template <bool NT_Copy, bool NT_Move, bool NT_CopyAssign, bool NT_MoveAssign,
-          bool NT_Swap, bool EnableSwap>
-int NothrowTypeImp<NT_Copy, NT_Move, NT_CopyAssign, NT_MoveAssign, NT_Swap,
-                   EnableSwap>::swap_called = 0;
-
-template <bool NT_Copy, bool NT_Move, bool NT_CopyAssign, bool NT_MoveAssign,
-          bool NT_Swap>
-void swap(NothrowTypeImp<NT_Copy, NT_Move, NT_CopyAssign, NT_MoveAssign,
-                         NT_Swap, true> &lhs,
-          NothrowTypeImp<NT_Copy, NT_Move, NT_CopyAssign, NT_MoveAssign,
-                         NT_Swap, true> &rhs) noexcept(NT_Swap) {
-  lhs.swap_called++;
+
+template <bool NT_Copy, bool NT_Move, bool NT_CopyAssign, bool NT_MoveAssign, bool NT_Swap>
+constexpr void
+swap(NothrowTypeImp<NT_Copy, NT_Move, NT_CopyAssign, NT_MoveAssign, NT_Swap, true>& lhs,
+     NothrowTypeImp<NT_Copy, NT_Move, NT_CopyAssign, NT_MoveAssign, NT_Swap, true>& rhs) noexcept(NT_Swap) {
+  ++*lhs.swap_called;
   do_throw<!NT_Swap>();
-  int tmp = lhs.value;
-  lhs.value = rhs.value;
-  rhs.value = tmp;
+  std::swap(lhs.value, rhs.value);
 }
 
 // throwing copy, nothrow move ctor/assign, no swap provided
@@ -124,53 +115,42 @@ using NothrowMoveable = NothrowTypeImp<false, true, false, true, false, false>;
 // throwing copy and move assign, nothrow move ctor, no swap provided
 using NothrowMoveCtor = NothrowTypeImp<false, true, false, false, false, false>;
 // nothrow move ctor, throwing move assignment, swap provided
-using NothrowMoveCtorWithThrowingSwap =
-    NothrowTypeImp<false, true, false, false, false, true>;
+using NothrowMoveCtorWithThrowingSwap = NothrowTypeImp<false, true, false, false, false, true>;
 // throwing move ctor, nothrow move assignment, no swap provided
-using ThrowingMoveCtor =
-    NothrowTypeImp<false, false, false, true, false, false>;
+using ThrowingMoveCtor = NothrowTypeImp<false, false, false, true, false, false>;
 // throwing special members, nothrowing swap
-using ThrowingTypeWithNothrowSwap =
-    NothrowTypeImp<false, false, false, false, true, true>;
-using NothrowTypeWithThrowingSwap =
-    NothrowTypeImp<true, true, true, true, false, true>;
+using ThrowingTypeWithNothrowSwap = NothrowTypeImp<false, false, false, false, true, true>;
+using NothrowTypeWithThrowingSwap = NothrowTypeImp<true, true, true, true, false, true>;
 // throwing move assign with nothrow move and nothrow swap
-using ThrowingMoveAssignNothrowMoveCtorWithSwap =
-    NothrowTypeImp<false, true, false, false, true, true>;
+using ThrowingMoveAssignNothrowMoveCtorWithSwap = NothrowTypeImp<false, true, false, false, true, true>;
 // throwing move assign with nothrow move but no swap.
-using ThrowingMoveAssignNothrowMoveCtor =
-    NothrowTypeImp<false, true, false, false, false, false>;
+using ThrowingMoveAssignNothrowMoveCtor = NothrowTypeImp<false, true, false, false, false, false>;
 
 struct NonThrowingNonNoexceptType {
-  static int move_called;
-  static void reset() { move_called = 0; }
-  NonThrowingNonNoexceptType() = default;
-  NonThrowingNonNoexceptType(int v) : value(v) {}
-  NonThrowingNonNoexceptType(NonThrowingNonNoexceptType &&o) noexcept(false)
-      : value(o.value) {
-    ++move_called;
+  int value;
+  int* move_called;
+  constexpr NonThrowingNonNoexceptType(int v, int* mv_called) : value(v), move_called(mv_called) {}
+  constexpr NonThrowingNonNoexceptType(NonThrowingNonNoexceptType&& o) noexcept(false)
+      : value(o.value), move_called(o.move_called) {
+    ++*move_called;
     o.value = -1;
   }
-  NonThrowingNonNoexceptType &
-  operator=(NonThrowingNonNoexceptType &&) noexcept(false) {
+  NonThrowingNonNoexceptType& operator=(NonThrowingNonNoexceptType&&) noexcept(false) {
     assert(false); // never called by the tests.
     return *this;
   }
-  int value;
 };
-int NonThrowingNonNoexceptType::move_called = 0;
 
 struct ThrowsOnSecondMove {
   int value;
   int move_count;
   ThrowsOnSecondMove(int v) : value(v), move_count(0) {}
-  ThrowsOnSecondMove(ThrowsOnSecondMove &&o) noexcept(false)
-      : value(o.value), move_count(o.move_count + 1) {
+  ThrowsOnSecondMove(ThrowsOnSecondMove&& o) noexcept(false) : value(o.value), move_count(o.move_count + 1) {
     if (move_count == 2)
       do_throw<true>();
     o.value = -1;
   }
-  ThrowsOnSecondMove &operator=(ThrowsOnSecondMove &&) {
+  ThrowsOnSecondMove& operator=(ThrowsOnSecondMove&&) {
     assert(false); // not called by test
     return *this;
   }
@@ -224,265 +204,293 @@ void test_swap_valueless_by_exception() {
 #endif
 }
 
-void test_swap_same_alternative() {
+TEST_CONSTEXPR_CXX20 void test_swap_same_alternative() {
   {
-    using T = ThrowingTypeWithNothrowSwap;
-    using V = std::variant<T, int>;
-    T::reset();
-    V v1(std::in_place_index<0>, 42);
-    V v2(std::in_place_index<0>, 100);
+    using V                = std::variant<ThrowingTypeWithNothrowSwap, int>;
+    int move_called        = 0;
+    int move_assign_called = 0;
+    int swap_called        = 0;
+    V v1(std::in_place_index<0>, 42, &move_called, &move_assign_called, &swap_called);
+    V v2(std::in_place_index<0>, 100, &move_called, &move_assign_called, &swap_called);
     v1.swap(v2);
-    assert(T::swap_called == 1);
+    assert(swap_called == 1);
     assert(std::get<0>(v1).value == 100);
     assert(std::get<0>(v2).value == 42);
     swap(v1, v2);
-    assert(T::swap_called == 2);
+    assert(swap_called == 2);
     assert(std::get<0>(v1).value == 42);
     assert(std::get<0>(v2).value == 100);
+
+    assert(move_called == 0);
+    assert(move_assign_called == 0);
   }
   {
-    using T = NothrowMoveable;
-    using V = std::variant<T, int>;
-    T::reset();
-    V v1(std::in_place_index<0>, 42);
-    V v2(std::in_place_index<0>, 100);
+    using V                = std::variant<NothrowMoveable, int>;
+    int move_called        = 0;
+    int move_assign_called = 0;
+    int swap_called        = 0;
+    V v1(std::in_place_index<0>, 42, &move_called, &move_assign_called, &swap_called);
+    V v2(std::in_place_index<0>, 100, &move_called, &move_assign_called, &swap_called);
     v1.swap(v2);
-    assert(T::swap_called == 0);
-    assert(T::move_called == 1);
-    assert(T::move_assign_called == 2);
+    assert(swap_called == 0);
+    assert(move_called == 1);
+    assert(move_assign_called == 2);
     assert(std::get<0>(v1).value == 100);
     assert(std::get<0>(v2).value == 42);
-    T::reset();
+
+    move_called        = 0;
+    move_assign_called = 0;
+    swap_called        = 0;
+
     swap(v1, v2);
-    assert(T::swap_called == 0);
-    assert(T::move_called == 1);
-    assert(T::move_assign_called == 2);
+    assert(swap_called == 0);
+    assert(move_called == 1);
+    assert(move_assign_called == 2);
     assert(std::get<0>(v1).value == 42);
     assert(std::get<0>(v2).value == 100);
   }
+}
+
+void test_swap_same_alternative_throws(){
 #ifndef TEST_HAS_NO_EXCEPTIONS
-  {
-    using T = NothrowTypeWithThrowingSwap;
-    using V = std::variant<T, int>;
-    T::reset();
-    V v1(std::in_place_index<0>, 42);
-    V v2(std::in_place_index<0>, 100);
-    try {
-      v1.swap(v2);
-      assert(false);
-    } catch (int) {
-    }
-    assert(T::swap_called == 1);
-    assert(T::move_called == 0);
-    assert(T::move_assign_called == 0);
-    assert(std::get<0>(v1).value == 42);
-    assert(std::get<0>(v2).value == 100);
-  }
-  {
-    using T = ThrowingMoveCtor;
-    using V = std::variant<T, int>;
-    T::reset();
-    V v1(std::in_place_index<0>, 42);
-    V v2(std::in_place_index<0>, 100);
-    try {
-      v1.swap(v2);
-      assert(false);
-    } catch (int) {
-    }
-    assert(T::move_called == 1); // call threw
-    assert(T::move_assign_called == 0);
-    assert(std::get<0>(v1).value ==
-           42); // throw happened before v1 was moved from
-    assert(std::get<0>(v2).value == 100);
+    {using V = std::variant<NothrowTypeWithThrowingSwap, int>;
+int move_called        = 0;
+int move_assign_called = 0;
+int swap_called        = 0;
+V v1(std::in_place_index<0>, 42, &move_called, &move_assign_called, &swap_called);
+V v2(std::in_place_index<0>, 100, &move_called, &move_assign_called, &swap_called);
+try {
+  v1.swap(v2);
+  assert(false);
+} catch (int) {
+}
+assert(swap_called == 1);
+assert(move_called == 0);
+assert(move_assign_called == 0);
+assert(std::get<0>(v1).value == 42);
+assert(std::get<0>(v2).value == 100);
+}
+
+{
+  using V                = std::variant<ThrowingMoveCtor, int>;
+  int move_called        = 0;
+  int move_assign_called = 0;
+  int swap_called        = 0;
+  V v1(std::in_place_index<0>, 42, &move_called, &move_assign_called, &swap_called);
+  V v2(std::in_place_index<0>, 100, &move_called, &move_assign_called, &swap_called);
+  try {
+    v1.swap(v2);
+    assert(false);
+  } catch (int) {
   }
-  {
-    using T = ThrowingMoveAssignNothrowMoveCtor;
-    using V = std::variant<T, int>;
-    T::reset();
-    V v1(std::in_place_index<0>, 42);
-    V v2(std::in_place_index<0>, 100);
-    try {
-      v1.swap(v2);
-      assert(false);
-    } catch (int) {
-    }
-    assert(T::move_called == 1);
-    assert(T::move_assign_called == 1);  // call threw and didn't complete
-    assert(std::get<0>(v1).value == -1); // v1 was moved from
-    assert(std::get<0>(v2).value == 100);
+  assert(move_called == 1); // call threw
+  assert(move_assign_called == 0);
+  assert(swap_called == 0);
+  assert(std::get<0>(v1).value == 42); // throw happened before v1 was moved from
+  assert(std::get<0>(v2).value == 100);
+}
+{
+  using V                = std::variant<ThrowingMoveAssignNothrowMoveCtor, int>;
+  int move_called        = 0;
+  int move_assign_called = 0;
+  int swap_called        = 0;
+  V v1(std::in_place_index<0>, 42, &move_called, &move_assign_called, &swap_called);
+  V v2(std::in_place_index<0>, 100, &move_called, &move_assign_called, &swap_called);
+  try {
+    v1.swap(v2);
+    assert(false);
+  } catch (int) {
   }
+  assert(move_called == 1);
+  assert(move_assign_called == 1); // call threw and didn't complete
+  assert(swap_called == 0);
+  assert(std::get<0>(v1).value == -1); // v1 was moved from
+  assert(std::get<0>(v2).value == 100);
+}
 #endif
 }
 
-void test_swap_different_alternatives() {
+TEST_CONSTEXPR_CXX20 void test_swap_different_alternatives() {
   {
-    using T = NothrowMoveCtorWithThrowingSwap;
-    using V = std::variant<T, int>;
-    T::reset();
-    V v1(std::in_place_index<0>, 42);
+    using V                = std::variant<NothrowMoveCtorWithThrowingSwap, int>;
+    int move_called        = 0;
+    int move_assign_called = 0;
+    int swap_called        = 0;
+    V v1(std::in_place_index<0>, 42, &move_called, &move_assign_called, &swap_called);
     V v2(std::in_place_index<1>, 100);
     v1.swap(v2);
-    assert(T::swap_called == 0);
+    assert(swap_called == 0);
     // The libc++ implementation double copies the argument, and not
     // the variant swap is called on.
-    LIBCPP_ASSERT(T::move_called == 1);
-    assert(T::move_called <= 2);
-    assert(T::move_assign_called == 0);
+    LIBCPP_ASSERT(move_called == 1);
+    assert(move_called <= 2);
+    assert(move_assign_called == 0);
     assert(std::get<1>(v1) == 100);
     assert(std::get<0>(v2).value == 42);
-    T::reset();
+
+    move_called        = 0;
+    move_assign_called = 0;
+    swap_called        = 0;
+
     swap(v1, v2);
-    assert(T::swap_called == 0);
-    LIBCPP_ASSERT(T::move_called == 2);
-    assert(T::move_called <= 2);
-    assert(T::move_assign_called == 0);
+    assert(swap_called == 0);
+    LIBCPP_ASSERT(move_called == 2);
+    assert(move_called <= 2);
+    assert(move_assign_called == 0);
     assert(std::get<0>(v1).value == 42);
     assert(std::get<1>(v2) == 100);
   }
+}
+
+void test_swap_different_alternatives_throws() {
 #ifndef TEST_HAS_NO_EXCEPTIONS
   {
-    using T1 = ThrowingTypeWithNothrowSwap;
-    using T2 = NonThrowingNonNoexceptType;
-    using V = std::variant<T1, T2>;
-    T1::reset();
-    T2::reset();
-    V v1(std::in_place_index<0>, 42);
-    V v2(std::in_place_index<1>, 100);
+    using V                 = std::variant<ThrowingTypeWithNothrowSwap, NonThrowingNonNoexceptType>;
+    int move_called1        = 0;
+    int move_assign_called1 = 0;
+    int swap_called1        = 0;
+    int move_called2        = 0;
+    V v1(std::in_place_index<0>, 42, &move_called1, &move_assign_called1, &swap_called1);
+    V v2(std::in_place_index<1>, 100, &move_called2);
     try {
       v1.swap(v2);
       assert(false);
     } catch (int) {
     }
-    assert(T1::swap_called == 0);
-    assert(T1::move_called == 1); // throws
-    assert(T1::move_assign_called == 0);
+    assert(swap_called1 == 0);
+    assert(move_called1 == 1); // throws
+    assert(move_assign_called1 == 0);
     // FIXME: libc++ shouldn't move from T2 here.
-    LIBCPP_ASSERT(T2::move_called == 1);
-    assert(T2::move_called <= 1);
+    LIBCPP_ASSERT(move_called2 == 1);
+    assert(move_called2 <= 1);
     assert(std::get<0>(v1).value == 42);
-    if (T2::move_called != 0)
+    if (move_called2 != 0)
       assert(v2.valueless_by_exception());
     else
       assert(std::get<1>(v2).value == 100);
   }
   {
-    using T1 = NonThrowingNonNoexceptType;
-    using T2 = ThrowingTypeWithNothrowSwap;
-    using V = std::variant<T1, T2>;
-    T1::reset();
-    T2::reset();
-    V v1(std::in_place_index<0>, 42);
-    V v2(std::in_place_index<1>, 100);
+    using V                 = std::variant<NonThrowingNonNoexceptType, ThrowingTypeWithNothrowSwap>;
+    int move_called1        = 0;
+    int move_called2        = 0;
+    int move_assign_called2 = 0;
+    int swap_called2        = 0;
+    V v1(std::in_place_index<0>, 42, &move_called1);
+    V v2(std::in_place_index<1>, 100, &move_called2, &move_assign_called2, &swap_called2);
     try {
       v1.swap(v2);
       assert(false);
     } catch (int) {
     }
-    LIBCPP_ASSERT(T1::move_called == 0);
-    assert(T1::move_called <= 1);
-    assert(T2::swap_called == 0);
-    assert(T2::move_called == 1); // throws
-    assert(T2::move_assign_called == 0);
-    if (T1::move_called != 0)
+    LIBCPP_ASSERT(move_called1 == 0);
+    assert(move_called1 <= 1);
+    assert(swap_called2 == 0);
+    assert(move_called2 == 1); // throws
+    assert(move_assign_called2 == 0);
+    if (move_called1 != 0)
       assert(v1.valueless_by_exception());
     else
       assert(std::get<0>(v1).value == 42);
     assert(std::get<1>(v2).value == 100);
   }
 // FIXME: The tests below are just very libc++ specific
-#ifdef _LIBCPP_VERSION
+#  ifdef _LIBCPP_VERSION
   {
-    using T1 = ThrowsOnSecondMove;
-    using T2 = NonThrowingNonNoexceptType;
-    using V = std::variant<T1, T2>;
-    T2::reset();
+    using V         = std::variant<ThrowsOnSecondMove, NonThrowingNonNoexceptType>;
+    int move_called = 0;
     V v1(std::in_place_index<0>, 42);
-    V v2(std::in_place_index<1>, 100);
+    V v2(std::in_place_index<1>, 100, &move_called);
     v1.swap(v2);
-    assert(T2::move_called == 2);
+    assert(move_called == 2);
     assert(std::get<1>(v1).value == 100);
     assert(std::get<0>(v2).value == 42);
     assert(std::get<0>(v2).move_count == 1);
   }
   {
-    using T1 = NonThrowingNonNoexceptType;
-    using T2 = ThrowsOnSecondMove;
-    using V = std::variant<T1, T2>;
-    T1::reset();
-    V v1(std::in_place_index<0>, 42);
+    using V         = std::variant<NonThrowingNonNoexceptType, ThrowsOnSecondMove>;
+    int move_called = 0;
+    V v1(std::in_place_index<0>, 42, &move_called);
     V v2(std::in_place_index<1>, 100);
     try {
       v1.swap(v2);
       assert(false);
     } catch (int) {
     }
-    assert(T1::move_called == 1);
+    assert(move_called == 1);
     assert(v1.valueless_by_exception());
     assert(std::get<0>(v2).value == 42);
   }
-#endif
-// testing libc++ extension. If either variant stores a nothrow move
-// constructible type v1.swap(v2) provides the strong exception safety
-// guarantee.
-#ifdef _LIBCPP_VERSION
+#  endif
+  // testing libc++ extension. If either variant stores a nothrow move
+  // constructible type v1.swap(v2) provides the strong exception safety
+  // guarantee.
+#  ifdef _LIBCPP_VERSION
   {
-
-    using T1 = ThrowingTypeWithNothrowSwap;
-    using T2 = NothrowMoveable;
-    using V = std::variant<T1, T2>;
-    T1::reset();
-    T2::reset();
-    V v1(std::in_place_index<0>, 42);
-    V v2(std::in_place_index<1>, 100);
+    using V                 = std::variant<ThrowingTypeWithNothrowSwap, NothrowMoveable>;
+    int move_called1        = 0;
+    int move_assign_called1 = 0;
+    int swap_called1        = 0;
+    int move_called2        = 0;
+    int move_assign_called2 = 0;
+    int swap_called2        = 0;
+    V v1(std::in_place_index<0>, 42, &move_called1, &move_assign_called1, &swap_called1);
+    V v2(std::in_place_index<1>, 100, &move_called2, &move_assign_called2, &swap_called2);
     try {
       v1.swap(v2);
       assert(false);
     } catch (int) {
     }
-    assert(T1::swap_called == 0);
-    assert(T1::move_called == 1);
-    assert(T1::move_assign_called == 0);
-    assert(T2::swap_called == 0);
-    assert(T2::move_called == 2);
-    assert(T2::move_assign_called == 0);
+    assert(swap_called1 == 0);
+    assert(move_called1 == 1);
+    assert(move_assign_called1 == 0);
+    assert(swap_called2 == 0);
+    assert(move_called2 == 2);
+    assert(move_assign_called2 == 0);
     assert(std::get<0>(v1).value == 42);
     assert(std::get<1>(v2).value == 100);
     // swap again, but call v2's swap.
-    T1::reset();
-    T2::reset();
+
+    move_called1        = 0;
+    move_assign_called1 = 0;
+    swap_called1        = 0;
+    move_called2        = 0;
+    move_assign_called2 = 0;
+    swap_called2        = 0;
+
     try {
       v2.swap(v1);
       assert(false);
     } catch (int) {
     }
-    assert(T1::swap_called == 0);
-    assert(T1::move_called == 1);
-    assert(T1::move_assign_called == 0);
-    assert(T2::swap_called == 0);
-    assert(T2::move_called == 2);
-    assert(T2::move_assign_called == 0);
+    assert(swap_called1 == 0);
+    assert(move_called1 == 1);
+    assert(move_assign_called1 == 0);
+    assert(swap_called2 == 0);
+    assert(move_called2 == 2);
+    assert(move_assign_called2 == 0);
     assert(std::get<0>(v1).value == 42);
     assert(std::get<1>(v2).value == 100);
   }
-#endif // _LIBCPP_VERSION
+#  endif // _LIBCPP_VERSION
 #endif
 }
 
 template <class Var>
-constexpr auto has_swap_member_imp(int)
-    -> decltype(std::declval<Var &>().swap(std::declval<Var &>()), true) {
+constexpr auto has_swap_member_imp(int) -> decltype(std::declval<Var&>().swap(std::declval<Var&>()), true) {
   return true;
 }
 
-template <class Var> constexpr auto has_swap_member_imp(long) -> bool {
+template <class Var>
+constexpr auto has_swap_member_imp(long) -> bool {
   return false;
 }
 
-template <class Var> constexpr bool has_swap_member() {
+template <class Var>
+constexpr bool has_swap_member() {
   return has_swap_member_imp<Var>(0);
 }
 
-void test_swap_sfinae() {
+constexpr void test_swap_sfinae() {
   {
     // This variant type does not provide either a member or non-member swap
     // but is still swappable via the generic swap algorithm, since the
@@ -508,7 +516,7 @@ void test_swap_sfinae() {
   }
 }
 
-void test_swap_noexcept() {
+_LIBCPP_CONSTEXPR_SINCE_CXX20 void test_swap_noexcept() {
   {
     using V = std::variant<int, NothrowMoveable>;
     static_assert(std::is_swappable_v<V> && has_swap_member<V>(), "");
@@ -581,12 +589,28 @@ void test_swap_noexcept() {
 template class std::variant<int, NotSwappable>;
 #endif
 
-int main(int, char**) {
+void non_constexpr_test() {
   test_swap_valueless_by_exception();
+  test_swap_same_alternative_throws();
+  test_swap_different_alternatives_throws();
+}
+
+TEST_CONSTEXPR_CXX20 bool test() {
   test_swap_same_alternative();
   test_swap_different_alternatives();
   test_swap_sfinae();
   test_swap_noexcept();
 
+  return true;
+}
+
+int main(int, char**) {
+  non_constexpr_test();
+  test();
+
+#if TEST_STD_VER >= 20
+  static_assert(test());
+#endif
+
   return 0;
 }
-- 
GitLab


From 9232591b04d7a4586e88bdbd1c3e513775c73560 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 10 May 2024 08:42:52 -0600
Subject: [PATCH 0412/1206] [libc++][NFC] Use TestEachPointerType in
 TestEachAtomicType (#91480)

That way, if we ever expand TestEachPointerType we will pick up those
changes in TestEachAtomicType.
---
 libcxx/test/support/atomic_helpers.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/libcxx/test/support/atomic_helpers.h b/libcxx/test/support/atomic_helpers.h
index 9a32b1ffe85e..0266a0961067 100644
--- a/libcxx/test/support/atomic_helpers.h
+++ b/libcxx/test/support/atomic_helpers.h
@@ -116,6 +116,7 @@ template <template <class TestArg> class TestFunctor>
 struct TestEachAtomicType {
   void operator()() const {
     TestEachIntegralType<TestFunctor>()();
+    TestEachPointerType<TestFunctor>()();
     TestFunctor<UserAtomicType>()();
     /*
             Note: These aren't going to be lock-free,
@@ -128,8 +129,6 @@ struct TestEachAtomicType {
         TestFunctor<PaddedUserAtomicType>()();
         TestFunctor<WeirdUserAtomicType>()();
 */
-    TestFunctor<int*>()();
-    TestFunctor<const int*>()();
     TestFunctor<float>()();
     TestFunctor<double>()();
   }
-- 
GitLab


From c5c8ae43545e917e624edd93695d1b265bae1e5c Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <razvan.lupusoru@gmail.com>
Date: Fri, 10 May 2024 07:45:35 -0700
Subject: [PATCH 0413/1206] [flang] Use i16 instead of i2 for cmdstat (#91647)

The code for preparing cmdstat was generating an i2 constant with value
0, casting it, and then storing it into i16 storage. Just generate i16
constant directly.
---
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp   | 6 ++----
 flang/test/Lower/Intrinsics/system-optional.f90 | 3 +--
 flang/test/Lower/Intrinsics/system.f90          | 6 ++----
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 9d72e76e2369..dcbbc39b84ea 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -6306,11 +6306,9 @@ void IntrinsicLibrary::genSystem(llvm::ArrayRef<fir::ExtendedValue> args) {
   // Create a dummmy cmdstat to prevent EXECUTE_COMMAND_LINE terminate itself
   // when cmdstat is assigned with a non-zero value but not present
   mlir::Value tempValue =
-      builder.createIntegerConstant(loc, builder.getI2Type(), 0);
+      builder.createIntegerConstant(loc, builder.getI16Type(), 0);
   mlir::Value temp = builder.createTemporary(loc, builder.getI16Type());
-  mlir::Value castVal =
-      builder.createConvert(loc, builder.getI16Type(), tempValue);
-  builder.create<fir::StoreOp>(loc, castVal, temp);
+  builder.create<fir::StoreOp>(loc, tempValue, temp);
   mlir::Value cmdstatBox = builder.createBox(loc, temp);
 
   mlir::Value cmdmsgBox =
diff --git a/flang/test/Lower/Intrinsics/system-optional.f90 b/flang/test/Lower/Intrinsics/system-optional.f90
index 8a2db132d672..8001e76fb93b 100644
--- a/flang/test/Lower/Intrinsics/system-optional.f90
+++ b/flang/test/Lower/Intrinsics/system-optional.f90
@@ -19,8 +19,7 @@ call system(command, exitstat)
 ! CHECK-NEXT:    %[[absentIntBox:.*]] = fir.absent !fir.box<i32>
 ! CHECK-NEXT:    %[[exitstatRealBox:.*]] = arith.select %[[exitstatIsPresent]], %[[exitstatBox]], %[[absentIntBox]] : !fir.box<i32>
 ! CHECK-NEXT:    %[[true:.*]] = arith.constant true
-! CHECK-NEXT:    %[[c0_i2:.*]] = arith.constant 0 : i2
-! CHECK-NEXT:    %[[c0_i16:.*]] = fir.convert %[[c0_i2]] : (i2) -> i16
+! CHECK-NEXT:    %[[c0_i16:.*]] = arith.constant 0 : i16
 ! CHECK-NEXT:    fir.store %[[c0_i16]] to %[[cmdstatVal]] : !fir.ref<i16>
 ! CHECK-NEXT:    %[[cmdstatBox:.*]] = fir.embox %[[cmdstatVal]] : (!fir.ref<i16>) -> !fir.box<i16>
 ! CHECK-NEXT:    %[[absentBox:.*]] = fir.absent !fir.box<none>
diff --git a/flang/test/Lower/Intrinsics/system.f90 b/flang/test/Lower/Intrinsics/system.f90
index 4ce3c553891d..71655938113f 100644
--- a/flang/test/Lower/Intrinsics/system.f90
+++ b/flang/test/Lower/Intrinsics/system.f90
@@ -15,8 +15,7 @@ call system(command, exitstat)
 ! CHECK-NEXT:   %[[commandBox:.*]] = fir.embox %[[commandDeclare]]#1 typeparams %[[commandUnbox]]#1 : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
 ! CHECK-NEXT:   %[[exitstatBox:.*]] = fir.embox %[[exitstatDeclare]]#1 : (!fir.ref<i32>) -> !fir.box<i32>
 ! CHECK-NEXT:   %[[true:.*]] = arith.constant true
-! CHECK-NEXT:   %[[c0_i2:.*]] = arith.constant 0 : i2
-! CHECK-NEXT:   %[[c0_i16:.*]] = fir.convert %[[c0_i2]] : (i2) -> i16
+! CHECK-NEXT:   %[[c0_i16:.*]] = arith.constant 0 : i16
 ! CHECK-NEXT:   fir.store %[[c0_i16]] to %[[cmdstatVal]] : !fir.ref<i16>
 ! CHECK-NEXT:   %[[cmdstatBox:.*]] = fir.embox %[[cmdstatVal]] : (!fir.ref<i16>) -> !fir.box<i16>
 ! CHECK-NEXT:   %[[absentBox:.*]] = fir.absent !fir.box<none>
@@ -41,8 +40,7 @@ call system(command)
 ! CHECK-NEXT:   %[[commandBox:.*]] = fir.embox %[[commandDeclare]]#1 typeparams %[[commandUnbox]]#1 : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
 ! CHECK-NEXT:   %[[true:.*]] = arith.constant true
 ! CHECK-NEXT:   %[[absentBox:.*]] = fir.absent !fir.box<none>
-! CHECK-NEXT:   %[[c0_i2:.*]] = arith.constant 0 : i2
-! CHECK-NEXT:   %[[c0_i16:.*]] = fir.convert %[[c0_i2]] : (i2) -> i16
+! CHECK-NEXT:   %[[c0_i16:.*]] = arith.constant 0 : i16
 ! CHECK-NEXT:   fir.store %[[c0_i16]] to %[[cmdstatVal]] : !fir.ref<i16>
 ! CHECK-NEXT:   %[[cmdstatBox:.*]] = fir.embox %[[cmdstatVal]] : (!fir.ref<i16>) -> !fir.box<i16>
 ! CHECK-NEXT:   %[[absentBox2:.*]] = fir.absent !fir.box<none>
-- 
GitLab


From 92dfe20acd4b101c58d7eb72cb7c0a9d4ae36385 Mon Sep 17 00:00:00 2001
From: lntue <35648136+lntue@users.noreply.github.com>
Date: Fri, 10 May 2024 11:07:41 -0400
Subject: [PATCH 0414/1206] [libc][math] Fix exact cases for powf. (#91488)

It was reported from the CORE-MATH project that the `powf`
implementation did not round correctly when `x^y` is either exact for
exactly half-way.

This PR deals with the potential exact cases when `y` is an integer `2 <
y < 25`. In such cases, the results of `x^y` is exactly representable in
double precision.
---
 libc/src/math/generic/powf.cpp   | 23 +++++++++++++++++++++--
 libc/test/src/math/powf_test.cpp | 19 +++++++++++++------
 2 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/libc/src/math/generic/powf.cpp b/libc/src/math/generic/powf.cpp
index 0450ffd711ff..59efc3f424c7 100644
--- a/libc/src/math/generic/powf.cpp
+++ b/libc/src/math/generic/powf.cpp
@@ -528,7 +528,7 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) {
   // So if |y| > 151 * 2^24, and x is finite:
   //   |y * log2(x)| = 0 or > 151.
   // Hence x^y will either overflow or underflow if x is not zero.
-  if (LIBC_UNLIKELY((y_abs & 0x007f'ffff) == 0) || (y_abs > 0x4f170000)) {
+  if (LIBC_UNLIKELY((y_abs & 0x0007'ffff) == 0) || (y_abs > 0x4f170000)) {
     // Exceptional exponents.
     switch (y_abs) {
     case 0x0000'0000: { // y = +-0.0f
@@ -572,6 +572,26 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) {
         // case 0xbf00'0000:  // pow(x, -1/2) = rsqrt(x)
         //   return rsqrt(x);
       }
+      if (is_integer(y) && (y_u > 0x4000'0000) && (y_u <= 0x41c0'0000)) {
+        // Check for exact cases when 2 < y < 25 and y is an integer.
+        int msb =
+            (x_abs == 0) ? (FloatBits::TOTAL_LEN - 2) : cpp::countl_zero(x_abs);
+        msb = (msb > FloatBits::EXP_LEN) ? msb : FloatBits::EXP_LEN;
+        int lsb = (x_abs == 0) ? 0 : cpp::countr_zero(x_abs);
+        lsb = (lsb > FloatBits::FRACTION_LEN) ? FloatBits::FRACTION_LEN : lsb;
+        int extra_bits = FloatBits::TOTAL_LEN - 2 - lsb - msb;
+        int iter = static_cast<int>(y);
+
+        if (extra_bits * iter <= FloatBits::FRACTION_LEN + 2) {
+          // The result is either exact or exactly half-way.
+          // But it is exactly representable in double precision.
+          double x_d = static_cast<double>(x);
+          double result = x_d;
+          for (int i = 1; i < iter; ++i)
+            result *= x_d;
+          return static_cast<float>(result);
+        }
+      }
       if (y_abs > 0x4f17'0000) {
         if (y_abs > 0x7f80'0000) {
           // y is NaN
@@ -834,7 +854,6 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) {
   return static_cast<float>(
              powf_double_double(idx_x, dx, y6, lo6_hi, exp2_hi_mid_dd)) +
          0.0f;
-  // return static_cast<float>(r);
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/test/src/math/powf_test.cpp b/libc/test/src/math/powf_test.cpp
index 69135593cd32..797913e5b7ee 100644
--- a/libc/test/src/math/powf_test.cpp
+++ b/libc/test/src/math/powf_test.cpp
@@ -22,14 +22,21 @@ using LIBC_NAMESPACE::testing::tlog;
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 TEST_F(LlvmLibcPowfTest, TrickyInputs) {
-  constexpr int N = 11;
+  constexpr int N = 13;
   constexpr mpfr::BinaryInput<float> INPUTS[N] = {
-      {0x1.290bbp-124f, 0x1.1e6d92p-25f}, {0x1.2e9fb6p+5f, -0x1.1b82b6p-18f},
-      {0x1.6877f6p+60f, -0x1.75f1c6p-4f}, {0x1.0936acp-63f, -0x1.55200ep-15f},
-      {0x1.d6d72ap+43f, -0x1.749ccap-5f}, {0x1.4afb2ap-40f, 0x1.063198p+0f},
-      {0x1.0124dep+0f, -0x1.fdb016p+9f},  {0x1.1058p+0f, 0x1.ap+64f},
-      {0x1.1058p+0f, -0x1.ap+64f},        {0x1.1058p+0f, 0x1.ap+64f},
+      {0x1.290bbp-124f, 0x1.1e6d92p-25f},
+      {0x1.2e9fb6p+5f, -0x1.1b82b6p-18f},
+      {0x1.6877f6p+60f, -0x1.75f1c6p-4f},
+      {0x1.0936acp-63f, -0x1.55200ep-15f},
+      {0x1.d6d72ap+43f, -0x1.749ccap-5f},
+      {0x1.4afb2ap-40f, 0x1.063198p+0f},
+      {0x1.0124dep+0f, -0x1.fdb016p+9f},
+      {0x1.1058p+0f, 0x1.ap+64f},
+      {0x1.1058p+0f, -0x1.ap+64f},
+      {0x1.1058p+0f, 0x1.ap+64f},
       {0x1.fa32d4p-1f, 0x1.67a62ep+12f},
+      {-0x1.8p-49, 0x1.8p+1},
+      {0x1.8p-48, 0x1.8p+1},
   };
 
   for (int i = 0; i < N; ++i) {
-- 
GitLab


From 023cdfcc1a5bdef7f12bb6da9328f93b477c38b8 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Fri, 10 May 2024 17:30:05 +0200
Subject: [PATCH 0415/1206] Update stale comments of the demangler, NFC
 (#91740)

To reflect what the actual code does.
---
 libcxxabi/src/demangle/ItaniumDemangle.h     | 3 ++-
 llvm/include/llvm/Demangle/ItaniumDemangle.h | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/libcxxabi/src/demangle/ItaniumDemangle.h b/libcxxabi/src/demangle/ItaniumDemangle.h
index 4a0444d407ea..36bf45463636 100644
--- a/libcxxabi/src/demangle/ItaniumDemangle.h
+++ b/libcxxabi/src/demangle/ItaniumDemangle.h
@@ -5715,6 +5715,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParam() {
 }
 
 // <template-param-decl> ::= Ty                          # type parameter
+//                       ::= Tk <concept name> [<template-args>] # constrained type parameter
 //                       ::= Tn <type>                   # non-type parameter
 //                       ::= Tt <template-param-decl>* E # template parameter
 //                       ::= Tp <template-param-decl>    # parameter pack
@@ -5846,7 +5847,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateArg() {
   }
 }
 
-// <template-args> ::= I <template-arg>* E
+// <template-args> ::= I <template-arg>* [Q <requires-clause expr>] E
 //     extension, the abi says <template-arg>+
 template <typename Derived, typename Alloc>
 Node *
diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h
index d33af157543f..e7c008be32f9 100644
--- a/llvm/include/llvm/Demangle/ItaniumDemangle.h
+++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h
@@ -5714,6 +5714,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateParam() {
 }
 
 // <template-param-decl> ::= Ty                          # type parameter
+//                       ::= Tk <concept name> [<template-args>] # constrained type parameter
 //                       ::= Tn <type>                   # non-type parameter
 //                       ::= Tt <template-param-decl>* E # template parameter
 //                       ::= Tp <template-param-decl>    # parameter pack
@@ -5845,7 +5846,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseTemplateArg() {
   }
 }
 
-// <template-args> ::= I <template-arg>* E
+// <template-args> ::= I <template-arg>* [Q <requires-clause expr>] E
 //     extension, the abi says <template-arg>+
 template <typename Derived, typename Alloc>
 Node *
-- 
GitLab


From b8f4f39d3d43d348e31fc5709b72e1f51392e52d Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Fri, 10 May 2024 11:55:02 -0400
Subject: [PATCH 0416/1206] [libc] avoid cmpxchg on the fastpath of callonce
 (#91748)

Avoid `cmpxchg` operation if the function has already been called.
The destination operand of `cmpxchg` may receive a write cycle without
regard to the result of the comparison
---
 libc/src/__support/threads/linux/callonce.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/libc/src/__support/threads/linux/callonce.cpp b/libc/src/__support/threads/linux/callonce.cpp
index 1c29db5f5c1a..b48a514a4487 100644
--- a/libc/src/__support/threads/linux/callonce.cpp
+++ b/libc/src/__support/threads/linux/callonce.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/threads/callonce.h"
+#include "src/__support/macros/optimization.h"
 #include "src/__support/threads/linux/futex_utils.h"
 
 namespace LIBC_NAMESPACE {
@@ -21,6 +22,12 @@ int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
 
   FutexWordType not_called = NOT_CALLED;
 
+  // Avoid cmpxchg operation if the function has already been called.
+  // The destination operand of cmpxchg may receive a write cycle without
+  // regard to the result of the comparison
+  if (LIBC_LIKELY(futex_word->load(cpp::MemoryOrder::RELAXED) == FINISH))
+    return 0;
+
   // The call_once call can return only after the called function |func|
   // returns. So, we use futexes to synchronize calls with the same flag value.
   if (futex_word->compare_exchange_strong(not_called, START)) {
-- 
GitLab


From d8f8ac8f5f9c787df94ba1ed99b04795afa40ba7 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 10 May 2024 08:59:56 -0700
Subject: [PATCH 0417/1206] [RISCV] Don't pre-split before the loop in
 parseNormalizedArchString. (#91684)

We can extract each extension as we process them without much
complexity.

I changed the error message for cases where there are double underscores
or a trailing underscore. I think this is an improvement over the
previous error.
---
 llvm/lib/TargetParser/RISCVISAInfo.cpp           | 15 ++++++++++++---
 llvm/unittests/TargetParser/RISCVISAInfoTest.cpp |  9 +++++++--
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 975cb5897b76..8ae0c355479c 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -451,9 +451,18 @@ RISCVISAInfo::parseNormalizedArchString(StringRef Arch) {
   // Each extension is of the form ${name}${major_version}p${minor_version}
   // and separated by _. Split by _ and then extract the name and version
   // information for each extension.
-  SmallVector<StringRef, 8> Split;
-  Arch.split(Split, '_');
-  for (StringRef Ext : Split) {
+  while (!Arch.empty()) {
+    if (Arch[0] == '_') {
+      if (Arch.size() == 1 || Arch[1] == '_')
+        return createStringError(errc::invalid_argument,
+                                 "extension name missing after separator '_'");
+      Arch = Arch.drop_front();
+    }
+
+    size_t Idx = Arch.find('_');
+    StringRef Ext = Arch.slice(0, Idx);
+    Arch = Arch.slice(Idx, StringRef::npos);
+
     StringRef Prefix, MinorVersionStr;
     std::tie(Prefix, MinorVersionStr) = Ext.rsplit('p');
     if (MinorVersionStr.empty())
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index f9e386a85fea..7f2d1eb8c017 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -38,12 +38,17 @@ TEST(ParseNormalizedArchString, RejectsInvalidBaseISA) {
 }
 
 TEST(ParseNormalizedArchString, RejectsMalformedInputs) {
-  for (StringRef Input :
-       {"rv64i2p0_", "rv32i2p0__a2p0", "rv64e2p", "rv32i", "rv64ip1"}) {
+  for (StringRef Input : {"rv64e2p", "rv32i", "rv64ip1"}) {
     EXPECT_EQ(
         toString(RISCVISAInfo::parseNormalizedArchString(Input).takeError()),
         "extension lacks version in expected format");
   }
+
+  for (StringRef Input : {"rv64i2p0_", "rv32i2p0__a2p0"}) {
+    EXPECT_EQ(
+        toString(RISCVISAInfo::parseNormalizedArchString(Input).takeError()),
+        "extension name missing after separator '_'");
+  }
 }
 
 TEST(ParseNormalizedArchString, RejectsOnlyVersion) {
-- 
GitLab


From d358b2de458e0611a3f474a9c3e3deff926c07c4 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 10 May 2024 18:50:42 +0300
Subject: [PATCH 0418/1206] [clang][NFC] Rename C++ CWG DR test files to use
 `cwg` prefix

This is a follow-up for ed128c7df9b4e60bfd814dc9fd22de1dde4a2c1c
---
 clang/test/CXX/drs/{dr0xx.cpp => cwg0xx.cpp}   | 0
 clang/test/CXX/drs/{dr10xx.cpp => cwg10xx.cpp} | 0
 clang/test/CXX/drs/{dr11xx.cpp => cwg11xx.cpp} | 0
 clang/test/CXX/drs/{dr12xx.cpp => cwg12xx.cpp} | 0
 clang/test/CXX/drs/{dr13xx.cpp => cwg13xx.cpp} | 0
 clang/test/CXX/drs/{dr14xx.cpp => cwg14xx.cpp} | 0
 clang/test/CXX/drs/{dr15xx.cpp => cwg15xx.cpp} | 0
 clang/test/CXX/drs/{dr16xx.cpp => cwg16xx.cpp} | 0
 clang/test/CXX/drs/{dr17xx.cpp => cwg17xx.cpp} | 0
 clang/test/CXX/drs/{dr18xx.cpp => cwg18xx.cpp} | 0
 clang/test/CXX/drs/{dr19xx.cpp => cwg19xx.cpp} | 0
 clang/test/CXX/drs/{dr1xx.cpp => cwg1xx.cpp}   | 0
 clang/test/CXX/drs/{dr20xx.cpp => cwg20xx.cpp} | 0
 clang/test/CXX/drs/{dr21xx.cpp => cwg21xx.cpp} | 0
 clang/test/CXX/drs/{dr22xx.cpp => cwg22xx.cpp} | 0
 clang/test/CXX/drs/{dr23xx.cpp => cwg23xx.cpp} | 0
 clang/test/CXX/drs/{dr24xx.cpp => cwg24xx.cpp} | 0
 clang/test/CXX/drs/{dr25xx.cpp => cwg25xx.cpp} | 0
 clang/test/CXX/drs/{dr26xx.cpp => cwg26xx.cpp} | 0
 clang/test/CXX/drs/{dr27xx.cpp => cwg27xx.cpp} | 0
 clang/test/CXX/drs/{dr28xx.cpp => cwg28xx.cpp} | 0
 clang/test/CXX/drs/{dr2xx.cpp => cwg2xx.cpp}   | 0
 clang/test/CXX/drs/{dr3xx.cpp => cwg3xx.cpp}   | 0
 clang/test/CXX/drs/{dr4xx.cpp => cwg4xx.cpp}   | 0
 clang/test/CXX/drs/{dr5xx.cpp => cwg5xx.cpp}   | 0
 clang/test/CXX/drs/{dr6xx.cpp => cwg6xx.cpp}   | 0
 clang/test/CXX/drs/{dr7xx.cpp => cwg7xx.cpp}   | 0
 clang/test/CXX/drs/{dr8xx.cpp => cwg8xx.cpp}   | 0
 clang/test/CXX/drs/{dr9xx.cpp => cwg9xx.cpp}   | 0
 29 files changed, 0 insertions(+), 0 deletions(-)
 rename clang/test/CXX/drs/{dr0xx.cpp => cwg0xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr10xx.cpp => cwg10xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr11xx.cpp => cwg11xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr12xx.cpp => cwg12xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr13xx.cpp => cwg13xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr14xx.cpp => cwg14xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr15xx.cpp => cwg15xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr16xx.cpp => cwg16xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr17xx.cpp => cwg17xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr18xx.cpp => cwg18xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr19xx.cpp => cwg19xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr1xx.cpp => cwg1xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr20xx.cpp => cwg20xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr21xx.cpp => cwg21xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr22xx.cpp => cwg22xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr23xx.cpp => cwg23xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr24xx.cpp => cwg24xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr25xx.cpp => cwg25xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr26xx.cpp => cwg26xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr27xx.cpp => cwg27xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr28xx.cpp => cwg28xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr2xx.cpp => cwg2xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr3xx.cpp => cwg3xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr4xx.cpp => cwg4xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr5xx.cpp => cwg5xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr6xx.cpp => cwg6xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr7xx.cpp => cwg7xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr8xx.cpp => cwg8xx.cpp} (100%)
 rename clang/test/CXX/drs/{dr9xx.cpp => cwg9xx.cpp} (100%)

diff --git a/clang/test/CXX/drs/dr0xx.cpp b/clang/test/CXX/drs/cwg0xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr0xx.cpp
rename to clang/test/CXX/drs/cwg0xx.cpp
diff --git a/clang/test/CXX/drs/dr10xx.cpp b/clang/test/CXX/drs/cwg10xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr10xx.cpp
rename to clang/test/CXX/drs/cwg10xx.cpp
diff --git a/clang/test/CXX/drs/dr11xx.cpp b/clang/test/CXX/drs/cwg11xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr11xx.cpp
rename to clang/test/CXX/drs/cwg11xx.cpp
diff --git a/clang/test/CXX/drs/dr12xx.cpp b/clang/test/CXX/drs/cwg12xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr12xx.cpp
rename to clang/test/CXX/drs/cwg12xx.cpp
diff --git a/clang/test/CXX/drs/dr13xx.cpp b/clang/test/CXX/drs/cwg13xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr13xx.cpp
rename to clang/test/CXX/drs/cwg13xx.cpp
diff --git a/clang/test/CXX/drs/dr14xx.cpp b/clang/test/CXX/drs/cwg14xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr14xx.cpp
rename to clang/test/CXX/drs/cwg14xx.cpp
diff --git a/clang/test/CXX/drs/dr15xx.cpp b/clang/test/CXX/drs/cwg15xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr15xx.cpp
rename to clang/test/CXX/drs/cwg15xx.cpp
diff --git a/clang/test/CXX/drs/dr16xx.cpp b/clang/test/CXX/drs/cwg16xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr16xx.cpp
rename to clang/test/CXX/drs/cwg16xx.cpp
diff --git a/clang/test/CXX/drs/dr17xx.cpp b/clang/test/CXX/drs/cwg17xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr17xx.cpp
rename to clang/test/CXX/drs/cwg17xx.cpp
diff --git a/clang/test/CXX/drs/dr18xx.cpp b/clang/test/CXX/drs/cwg18xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr18xx.cpp
rename to clang/test/CXX/drs/cwg18xx.cpp
diff --git a/clang/test/CXX/drs/dr19xx.cpp b/clang/test/CXX/drs/cwg19xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr19xx.cpp
rename to clang/test/CXX/drs/cwg19xx.cpp
diff --git a/clang/test/CXX/drs/dr1xx.cpp b/clang/test/CXX/drs/cwg1xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr1xx.cpp
rename to clang/test/CXX/drs/cwg1xx.cpp
diff --git a/clang/test/CXX/drs/dr20xx.cpp b/clang/test/CXX/drs/cwg20xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr20xx.cpp
rename to clang/test/CXX/drs/cwg20xx.cpp
diff --git a/clang/test/CXX/drs/dr21xx.cpp b/clang/test/CXX/drs/cwg21xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr21xx.cpp
rename to clang/test/CXX/drs/cwg21xx.cpp
diff --git a/clang/test/CXX/drs/dr22xx.cpp b/clang/test/CXX/drs/cwg22xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr22xx.cpp
rename to clang/test/CXX/drs/cwg22xx.cpp
diff --git a/clang/test/CXX/drs/dr23xx.cpp b/clang/test/CXX/drs/cwg23xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr23xx.cpp
rename to clang/test/CXX/drs/cwg23xx.cpp
diff --git a/clang/test/CXX/drs/dr24xx.cpp b/clang/test/CXX/drs/cwg24xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr24xx.cpp
rename to clang/test/CXX/drs/cwg24xx.cpp
diff --git a/clang/test/CXX/drs/dr25xx.cpp b/clang/test/CXX/drs/cwg25xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr25xx.cpp
rename to clang/test/CXX/drs/cwg25xx.cpp
diff --git a/clang/test/CXX/drs/dr26xx.cpp b/clang/test/CXX/drs/cwg26xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr26xx.cpp
rename to clang/test/CXX/drs/cwg26xx.cpp
diff --git a/clang/test/CXX/drs/dr27xx.cpp b/clang/test/CXX/drs/cwg27xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr27xx.cpp
rename to clang/test/CXX/drs/cwg27xx.cpp
diff --git a/clang/test/CXX/drs/dr28xx.cpp b/clang/test/CXX/drs/cwg28xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr28xx.cpp
rename to clang/test/CXX/drs/cwg28xx.cpp
diff --git a/clang/test/CXX/drs/dr2xx.cpp b/clang/test/CXX/drs/cwg2xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr2xx.cpp
rename to clang/test/CXX/drs/cwg2xx.cpp
diff --git a/clang/test/CXX/drs/dr3xx.cpp b/clang/test/CXX/drs/cwg3xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr3xx.cpp
rename to clang/test/CXX/drs/cwg3xx.cpp
diff --git a/clang/test/CXX/drs/dr4xx.cpp b/clang/test/CXX/drs/cwg4xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr4xx.cpp
rename to clang/test/CXX/drs/cwg4xx.cpp
diff --git a/clang/test/CXX/drs/dr5xx.cpp b/clang/test/CXX/drs/cwg5xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr5xx.cpp
rename to clang/test/CXX/drs/cwg5xx.cpp
diff --git a/clang/test/CXX/drs/dr6xx.cpp b/clang/test/CXX/drs/cwg6xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr6xx.cpp
rename to clang/test/CXX/drs/cwg6xx.cpp
diff --git a/clang/test/CXX/drs/dr7xx.cpp b/clang/test/CXX/drs/cwg7xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr7xx.cpp
rename to clang/test/CXX/drs/cwg7xx.cpp
diff --git a/clang/test/CXX/drs/dr8xx.cpp b/clang/test/CXX/drs/cwg8xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr8xx.cpp
rename to clang/test/CXX/drs/cwg8xx.cpp
diff --git a/clang/test/CXX/drs/dr9xx.cpp b/clang/test/CXX/drs/cwg9xx.cpp
similarity index 100%
rename from clang/test/CXX/drs/dr9xx.cpp
rename to clang/test/CXX/drs/cwg9xx.cpp
-- 
GitLab


From ef71c79d85fb3f94f19a92a84ab44719d112c1bd Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 10 May 2024 19:02:50 +0300
Subject: [PATCH 0419/1206] [clang][NFC] Update C++ DR status page

---
 clang/www/cxx_dr_status.html | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 875521bd505d..992dc61d65e4 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -16823,7 +16823,7 @@ objects</td>
   </tr>
   <tr class="open" id="2836">
     <td><a href="https://cplusplus.github.io/CWG/issues/2836.html">2836</a></td>
-    <td>review</td>
+    <td>tentatively ready</td>
     <td>Conversion rank of <TT>long double</TT> and extended floating-point types</td>
     <td align="center">Not resolved</td>
   </tr>
@@ -16997,7 +16997,7 @@ objects</td>
   </tr>
   <tr class="open" id="2865">
     <td><a href="https://cplusplus.github.io/CWG/issues/2865.html">2865</a></td>
-    <td>open</td>
+    <td>tentatively ready</td>
     <td>Regression on result of conditional operator</td>
     <td align="center">Not resolved</td>
   </tr>
@@ -17009,7 +17009,7 @@ objects</td>
   </tr>
   <tr class="open" id="2867">
     <td><a href="https://cplusplus.github.io/CWG/issues/2867.html">2867</a></td>
-    <td>open</td>
+    <td>review</td>
     <td>Order of initialization for structured bindings</td>
     <td align="center">Not resolved</td>
   </tr>
@@ -17126,6 +17126,18 @@ objects</td>
     <td>open</td>
     <td>Temporaries and trivial potentially-throwing special member functions</td>
     <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2887">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2887.html">2887</a></td>
+    <td>open</td>
+    <td>Missing compatibility entries for xvalues</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2888">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2888.html">2888</a></td>
+    <td>open</td>
+    <td>Missing cases for reference and array types for argument-dependent lookup</td>
+    <td align="center">Not resolved</td>
   </tr></table>
 
 </div>
-- 
GitLab


From 099c1527f6088305fc253e9598685faa85544a8e Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Fri, 10 May 2024 18:13:47 +0200
Subject: [PATCH 0420/1206] [AST] Print the separator "," for template
 arguments in ConceptReference::print (#91750)

---
 clang/lib/AST/ASTConcept.cpp                  |  6 +-
 clang/unittests/AST/CMakeLists.txt            |  1 +
 clang/unittests/AST/ConceptPrinterTest.cpp    | 57 +++++++++++++++++++
 .../gn/secondary/clang/unittests/AST/BUILD.gn |  1 +
 4 files changed, 64 insertions(+), 1 deletion(-)
 create mode 100644 clang/unittests/AST/ConceptPrinterTest.cpp

diff --git a/clang/lib/AST/ASTConcept.cpp b/clang/lib/AST/ASTConcept.cpp
index b3ec99448b3e..0387fc9f6aec 100644
--- a/clang/lib/AST/ASTConcept.cpp
+++ b/clang/lib/AST/ASTConcept.cpp
@@ -15,6 +15,7 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringExtras.h"
 
 using namespace clang;
 
@@ -106,9 +107,12 @@ void ConceptReference::print(llvm::raw_ostream &OS,
   ConceptName.printName(OS, Policy);
   if (hasExplicitTemplateArgs()) {
     OS << "<";
+    llvm::ListSeparator Sep(", ");
     // FIXME: Find corresponding parameter for argument
-    for (auto &ArgLoc : ArgsAsWritten->arguments())
+    for (auto &ArgLoc : ArgsAsWritten->arguments()) {
+      OS << Sep;
       ArgLoc.getArgument().print(Policy, OS, /*IncludeType*/ false);
+    }
     OS << ">";
   }
 }
diff --git a/clang/unittests/AST/CMakeLists.txt b/clang/unittests/AST/CMakeLists.txt
index 54765e36db00..29d2b39cff8b 100644
--- a/clang/unittests/AST/CMakeLists.txt
+++ b/clang/unittests/AST/CMakeLists.txt
@@ -24,6 +24,7 @@ add_clang_unittest(ASTTests
   CommentLexer.cpp
   CommentParser.cpp
   CommentTextTest.cpp
+  ConceptPrinterTest.cpp
   DataCollectionTest.cpp
   DeclPrinterTest.cpp
   DeclTest.cpp
diff --git a/clang/unittests/AST/ConceptPrinterTest.cpp b/clang/unittests/AST/ConceptPrinterTest.cpp
new file mode 100644
index 000000000000..bd1f6bbfa565
--- /dev/null
+++ b/clang/unittests/AST/ConceptPrinterTest.cpp
@@ -0,0 +1,57 @@
+//===- unittests/AST/ConceptPrinterTest.cpp --- Concept printer tests -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ASTPrint.h"
+#include "clang/AST/ASTConcept.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/ExprConcepts.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/SmallString.h"
+#include "gtest/gtest.h"
+
+using namespace clang;
+using namespace ast_matchers;
+using namespace tooling;
+
+namespace {
+
+static void PrintConceptReference(raw_ostream &Out, const ASTContext *Context,
+                                  const ConceptSpecializationExpr *T,
+                                  PrintingPolicyAdjuster PolicyAdjuster) {
+  assert(T && T->getConceptReference() &&
+         "Expected non-null concept reference");
+
+  PrintingPolicy Policy = Context->getPrintingPolicy();
+  if (PolicyAdjuster)
+    PolicyAdjuster(Policy);
+  T->getConceptReference()->print(Out, Policy);
+}
+
+::testing::AssertionResult
+PrintedConceptMatches(StringRef Code, const std::vector<std::string> &Args,
+                      const StatementMatcher &NodeMatch,
+                      StringRef ExpectedPrinted) {
+  return PrintedNodeMatches<ConceptSpecializationExpr>(
+      Code, Args, NodeMatch, ExpectedPrinted, "", PrintConceptReference);
+}
+const internal::VariadicDynCastAllOfMatcher<Stmt, ConceptSpecializationExpr>
+    conceptSpecializationExpr;
+} // unnamed namespace
+
+TEST(ConceptPrinter, ConceptReference) {
+  std::string Code = R"cpp(
+    template <typename, typename> concept D = true;
+    template<typename T, typename U>
+    requires D<T, U>
+    void g(T);
+  )cpp";
+  auto Matcher = conceptSpecializationExpr().bind("id");
+
+  ASSERT_TRUE(PrintedConceptMatches(Code, {"-std=c++20"}, Matcher, "D<T, U>"));
+}
diff --git a/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn
index 1fb3d621275e..f66e86ad938b 100644
--- a/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn
@@ -32,6 +32,7 @@ unittest("ASTTests") {
     "CommentLexer.cpp",
     "CommentParser.cpp",
     "CommentTextTest.cpp",
+    "ConceptPrinterTest.cpp",
     "DataCollectionTest.cpp",
     "DeclPrinterTest.cpp",
     "DeclTest.cpp",
-- 
GitLab


From 9a7262c2601874e5aa64c5db19746770212d4b44 Mon Sep 17 00:00:00 2001
From: Zequan Wu <zequanwu@google.com>
Date: Fri, 10 May 2024 12:26:52 -0400
Subject: [PATCH 0421/1206] [lldb][DWARF] Delay struct/class/union definition
 DIE searching when parsing declaration DIEs. (#90663)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is the implementation for
https://discourse.llvm.org/t/rfc-delay-definition-die-searching-when-parse-a-declaration-die-for-record-type/78526.

#### Motivation
Currently, lldb eagerly searches for definition DIE when parsing a
declaration DIE for struct/class/union definition DIE. It will search
for all definition DIEs with the same unqualified name (just
`DW_AT_name` ) and then find out those DIEs with same fully qualified
name. Then lldb will try to resolve those DIEs to create the Types from
definition DIEs. It works fine most time. However, when built with
`-gsimple-template-names`, the search graph expands very quickly,
because for the specialized-template classes, they don’t have template
parameter names encoded inside `DW_AT_name`. They have
`DW_TAG_template_type_parameter` to reference the types used as template
parameters. In order to identify if a definition DIE matches a
declaration DIE, lldb needs to resolve all template parameter types
first and those template parameter types might be template classes as
well, and so on… So, the search graph explodes, causing a lot
unnecessary searching/type-resolving to just get the fully qualified
names for a specialized-template class. This causes lldb stack overflow
for us internally on template-heavy libraries.

#### Implementation
Instead of searching for definition DIEs when parsing declaration DIEs,
we always construct the record type from the DIE regardless if it's
definition or declaration. The process of searching for definition DIE
is refactored to `DWARFASTParserClang::FindDefinitionTypeForDIE` which
is invoked when 1) completing the type on
`SymbolFileDWARF::CompleteType`. 2) the record type needs to start its
definition as a containing type so that nested classes can be added into
it in `PrepareContextToReceiveMembers`.

The key difference is `SymbolFileDWARF::ResolveType` return a `Type*`
that might be created from declaration DIE, which means it hasn't starts
its definition yet. We also need to change according in places where we
want the type to start definition, like `PrepareContextToReceiveMembers`
(I'm not aware of any other places, but this should be a simple call to
`SymbolFileDWARF::FindDefinitionDIE`)

#### Result
It fixes the stack overflow of lldb for the internal binary built with
simple template name. When constructing the fully qualified name built
with `-gsimple-template-names`, it gets the name of the type parameter
by resolving the referenced DIE, which might be a declaration (we won't
try to search for the definition DIE to just get the name).
I got rough measurement about the time using the same commands (set
breakpoint, run, expr this, exit). For the binary built without
`-gsimple-template-names`, this change has no impact on time, still
taking 41 seconds to complete. When built with
`-gsimple-template-names`, it also takes about 41 seconds to complete
wit this change.
---
 .../Plugins/SymbolFile/DWARF/DWARFASTParser.h |   2 +
 .../SymbolFile/DWARF/DWARFASTParserClang.cpp  | 397 ++++++++++--------
 .../SymbolFile/DWARF/DWARFASTParserClang.h    | 197 ++++-----
 .../SymbolFile/DWARF/SymbolFileDWARF.cpp      |  44 +-
 .../SymbolFile/DWARF/SymbolFileDWARF.h        |   4 +
 .../SymbolFile/DWARF/UniqueDWARFASTType.cpp   | 107 ++---
 .../SymbolFile/DWARF/UniqueDWARFASTType.h     |  36 +-
 .../delayed-definition-die-searching.test     |  34 ++
 8 files changed, 438 insertions(+), 383 deletions(-)
 create mode 100644 lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
index 66db396279e0..e144cf0f9bd9 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
@@ -60,6 +60,8 @@ public:
 
   virtual ConstString GetDIEClassTemplateParams(const DWARFDIE &die) = 0;
 
+  virtual lldb_private::Type *FindDefinitionTypeForDIE(const DWARFDIE &die) = 0;
+
   static std::optional<SymbolFile::ArrayInfo>
   ParseChildArrayInfo(const DWARFDIE &parent_die,
                       const ExecutionContext *exe_ctx = nullptr);
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index f8101aba5c62..e0b1b430b266 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -154,6 +154,26 @@ static bool TagIsRecordType(dw_tag_t tag) {
   }
 }
 
+static bool IsForwardDeclaration(const DWARFDIE &die,
+                                 const ParsedDWARFTypeAttributes &attrs,
+                                 LanguageType cu_language) {
+  if (attrs.is_forward_declaration)
+    return true;
+
+  // Work around an issue with clang at the moment where forward
+  // declarations for objective C classes are emitted as:
+  //  DW_TAG_structure_type [2]
+  //  DW_AT_name( "ForwardObjcClass" )
+  //  DW_AT_byte_size( 0x00 )
+  //  DW_AT_decl_file( "..." )
+  //  DW_AT_decl_line( 1 )
+  //
+  // Note that there is no DW_AT_declaration and there are no children,
+  // and the byte size is zero.
+  return attrs.byte_size && *attrs.byte_size == 0 && attrs.name &&
+         !die.HasChildren() && cu_language == eLanguageTypeObjC;
+}
+
 TypeSP DWARFASTParserClang::ParseTypeFromClangModule(const SymbolContext &sc,
                                                      const DWARFDIE &die,
                                                      Log *log) {
@@ -249,11 +269,9 @@ static void ForcefullyCompleteType(CompilerType type) {
 /// This function serves a similar purpose as RequireCompleteType above, but it
 /// avoids completing the type if it is not immediately necessary. It only
 /// ensures we _can_ complete the type later.
-static void PrepareContextToReceiveMembers(TypeSystemClang &ast,
-                                           ClangASTImporter &ast_importer,
-                                           clang::DeclContext *decl_ctx,
-                                           DWARFDIE die,
-                                           const char *type_name_cstr) {
+void DWARFASTParserClang::PrepareContextToReceiveMembers(
+    clang::DeclContext *decl_ctx, const DWARFDIE &decl_ctx_die,
+    const DWARFDIE &die, const char *type_name_cstr) {
   auto *tag_decl_ctx = clang::dyn_cast<clang::TagDecl>(decl_ctx);
   if (!tag_decl_ctx)
     return; // Non-tag context are always ready.
@@ -268,7 +286,8 @@ static void PrepareContextToReceiveMembers(TypeSystemClang &ast,
   // gmodules case), we can complete the type by doing a full import.
 
   // If this type was not imported from an external AST, there's nothing to do.
-  CompilerType type = ast.GetTypeForDecl(tag_decl_ctx);
+  CompilerType type = m_ast.GetTypeForDecl(tag_decl_ctx);
+  ClangASTImporter &ast_importer = GetClangASTImporter();
   if (type && ast_importer.CanImport(type)) {
     auto qual_type = ClangUtil::GetQualType(type);
     if (ast_importer.RequireCompleteType(qual_type))
@@ -279,6 +298,13 @@ static void PrepareContextToReceiveMembers(TypeSystemClang &ast,
         type_name_cstr ? type_name_cstr : "", die.GetOffset());
   }
 
+  // By searching for the definition DIE of the decl_ctx type, we will either:
+  // 1. Found the the definition DIE and start its definition with
+  // TypeSystemClang::StartTagDeclarationDefinition.
+  // 2. Unable to find it, then need to forcefully complete it.
+  FindDefinitionTypeForDIE(decl_ctx_die);
+  if (tag_decl_ctx->isCompleteDefinition() || tag_decl_ctx->isBeingDefined())
+    return;
   // We don't have a type definition and/or the import failed. We must
   // forcefully complete the type to avoid crashes.
   ForcefullyCompleteType(type);
@@ -620,10 +646,11 @@ DWARFASTParserClang::ParseTypeModifier(const SymbolContext &sc,
   if (tag == DW_TAG_typedef) {
     // DeclContext will be populated when the clang type is materialized in
     // Type::ResolveCompilerType.
-    PrepareContextToReceiveMembers(
-        m_ast, GetClangASTImporter(),
-        GetClangDeclContextContainingDIE(die, nullptr), die,
-        attrs.name.GetCString());
+    DWARFDIE decl_ctx_die;
+    clang::DeclContext *decl_ctx =
+        GetClangDeclContextContainingDIE(die, &decl_ctx_die);
+    PrepareContextToReceiveMembers(decl_ctx, decl_ctx_die, die,
+                                   attrs.name.GetCString());
 
     if (attrs.type.IsValid()) {
       // Try to parse a typedef from the (DWARF embedded in the) Clang
@@ -1103,32 +1130,6 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
         // struct and see if this is actually a C++ method
         Type *class_type = dwarf->ResolveType(decl_ctx_die);
         if (class_type) {
-          if (class_type->GetID() != decl_ctx_die.GetID() ||
-              IsClangModuleFwdDecl(decl_ctx_die)) {
-
-            // We uniqued the parent class of this function to another
-            // class so we now need to associate all dies under
-            // "decl_ctx_die" to DIEs in the DIE for "class_type"...
-            DWARFDIE class_type_die = dwarf->GetDIE(class_type->GetID());
-
-            if (class_type_die) {
-              std::vector<DWARFDIE> failures;
-
-              CopyUniqueClassMethodTypes(decl_ctx_die, class_type_die,
-                                         class_type, failures);
-
-              // FIXME do something with these failures that's
-              // smarter than just dropping them on the ground.
-              // Unfortunately classes don't like having stuff added
-              // to them after their definitions are complete...
-
-              Type *type_ptr = dwarf->GetDIEToType()[die.GetDIE()];
-              if (type_ptr && type_ptr != DIE_IS_BEING_PARSED) {
-                return type_ptr->shared_from_this();
-              }
-            }
-          }
-
           if (attrs.specification.IsValid()) {
             // We have a specification which we are going to base our
             // function prototype off of, so we need this type to be
@@ -1263,6 +1264,39 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
               }
             }
           }
+          // By here, we should have already completed the c++ class_type
+          // because if either specification or abstract_origin is present, we
+          // call GetClangDeclContextForDIE to resolve the DW_TAG_subprogram
+          // refered by this one until we reached the DW_TAG_subprogram without
+          // specification or abstract_origin (the else branch above). Then the
+          // above GetFullCompilerType() will complete the class_type if it's
+          // not completed yet. After that, we will have the mapping from DIEs
+          // in class_type_die to DeclContexts in m_die_to_decl_ctx.
+          if (class_type->GetID() != decl_ctx_die.GetID() ||
+              IsClangModuleFwdDecl(decl_ctx_die)) {
+
+            // We uniqued the parent class of this function to another
+            // class so we now need to associate all dies under
+            // "decl_ctx_die" to DIEs in the DIE for "class_type"...
+            DWARFDIE class_type_die = dwarf->GetDIE(class_type->GetID());
+
+            if (class_type_die) {
+              std::vector<DWARFDIE> failures;
+
+              CopyUniqueClassMethodTypes(decl_ctx_die, class_type_die,
+                                         class_type, failures);
+
+              // FIXME do something with these failures that's
+              // smarter than just dropping them on the ground.
+              // Unfortunately classes don't like having stuff added
+              // to them after their definitions are complete...
+
+              Type *type_ptr = dwarf->GetDIEToType()[die.GetDIE()];
+              if (type_ptr && type_ptr != DIE_IS_BEING_PARSED) {
+                return type_ptr->shared_from_this();
+              }
+            }
+          }
         }
       }
     }
@@ -1635,6 +1669,93 @@ DWARFASTParserClang::GetCPlusPlusQualifiedName(const DWARFDIE &die) {
   return qualified_name;
 }
 
+lldb_private::Type *
+DWARFASTParserClang::FindDefinitionTypeForDIE(const DWARFDIE &die) {
+  SymbolFileDWARF *dwarf = die.GetDWARF();
+  ParsedDWARFTypeAttributes attrs(die);
+  bool is_forward_declaration = IsForwardDeclaration(
+      die, attrs, SymbolFileDWARF::GetLanguage(*die.GetCU()));
+  if (!is_forward_declaration)
+    return dwarf->GetDIEToType()[die.GetDIE()];
+
+  const dw_tag_t tag = die.Tag();
+  TypeSP type_sp;
+  Log *log = GetLog(DWARFLog::TypeCompletion | DWARFLog::Lookups);
+  if (log) {
+    dwarf->GetObjectFile()->GetModule()->LogMessage(
+        log,
+        "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a "
+        "forward declaration DIE, trying to find definition DIE",
+        static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
+        attrs.name.GetCString());
+  }
+  // We haven't parse definition die for this type, starting to search for it.
+  // After we found the definition die, the GetDeclarationDIEToDefinitionDIE()
+  // map will have the new mapping from this declaration die to definition die.
+  if (attrs.class_language == eLanguageTypeObjC ||
+      attrs.class_language == eLanguageTypeObjC_plus_plus) {
+    if (!attrs.is_complete_objc_class &&
+        die.Supports_DW_AT_APPLE_objc_complete_type()) {
+      // We have a valid eSymbolTypeObjCClass class symbol whose name
+      // matches the current objective C class that we are trying to find
+      // and this DIE isn't the complete definition (we checked
+      // is_complete_objc_class above and know it is false), so the real
+      // definition is in here somewhere
+      type_sp =
+          dwarf->FindCompleteObjCDefinitionTypeForDIE(die, attrs.name, true);
+
+      if (!type_sp) {
+        SymbolFileDWARFDebugMap *debug_map_symfile =
+            dwarf->GetDebugMapSymfile();
+        if (debug_map_symfile) {
+          // We weren't able to find a full declaration in this DWARF,
+          // see if we have a declaration anywhere else...
+          type_sp = debug_map_symfile->FindCompleteObjCDefinitionTypeForDIE(
+              die, attrs.name, true);
+        }
+      }
+
+      if (type_sp && log) {
+        dwarf->GetObjectFile()->GetModule()->LogMessage(
+            log,
+            "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is an "
+            "incomplete objc type, complete type is {5:x8}",
+            static_cast<void *>(this), die.GetOffset(),
+            DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(),
+            type_sp->GetID());
+      }
+    }
+  }
+
+  type_sp = dwarf->FindDefinitionTypeForDWARFDeclContext(die);
+  if (!type_sp) {
+    SymbolFileDWARFDebugMap *debug_map_symfile = dwarf->GetDebugMapSymfile();
+    if (debug_map_symfile) {
+      // We weren't able to find a full declaration in this DWARF, see
+      // if we have a declaration anywhere else...
+      type_sp = debug_map_symfile->FindDefinitionTypeForDWARFDeclContext(die);
+    }
+    if (type_sp && log) {
+      dwarf->GetObjectFile()->GetModule()->LogMessage(
+          log,
+          "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a "
+          "forward declaration, complete type is {4:x8}",
+          static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
+          attrs.name.GetCString(), type_sp->GetID());
+    }
+  }
+
+  if (!type_sp && log) {
+    dwarf->GetObjectFile()->GetModule()->LogMessage(
+        log,
+        "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a "
+        "forward declaration, unable to find definition DIE for it",
+        static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
+        attrs.name.GetCString());
+  }
+  return type_sp.get();
+}
+
 TypeSP
 DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
                                            const DWARFDIE &die,
@@ -1646,14 +1767,10 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
   LanguageType cu_language = SymbolFileDWARF::GetLanguage(*die.GetCU());
   Log *log = GetLog(DWARFLog::TypeCompletion | DWARFLog::Lookups);
 
-  // UniqueDWARFASTType is large, so don't create a local variables on the
-  // stack, put it on the heap. This function is often called recursively and
-  // clang isn't good at sharing the stack space for variables in different
-  // blocks.
-  auto unique_ast_entry_up = std::make_unique<UniqueDWARFASTType>();
-
   ConstString unique_typename(attrs.name);
   Declaration unique_decl(attrs.decl);
+  uint64_t byte_size = attrs.byte_size.value_or(0);
+  attrs.is_forward_declaration = IsForwardDeclaration(die, attrs, cu_language);
 
   if (attrs.name) {
     if (Language::LanguageIsCPlusPlus(cu_language)) {
@@ -1666,14 +1783,42 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
       unique_decl.Clear();
     }
 
-    if (dwarf->GetUniqueDWARFASTTypeMap().Find(
-            unique_typename, die, unique_decl, attrs.byte_size.value_or(-1),
-            *unique_ast_entry_up)) {
-      type_sp = unique_ast_entry_up->m_type_sp;
+    if (UniqueDWARFASTType *unique_ast_entry_type =
+            dwarf->GetUniqueDWARFASTTypeMap().Find(
+                unique_typename, die, unique_decl, byte_size,
+                attrs.is_forward_declaration)) {
+      type_sp = unique_ast_entry_type->m_type_sp;
       if (type_sp) {
         dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
         LinkDeclContextToDIE(
-            GetCachedClangDeclContextForDIE(unique_ast_entry_up->m_die), die);
+            GetCachedClangDeclContextForDIE(unique_ast_entry_type->m_die), die);
+        if (!attrs.is_forward_declaration) {
+          // If the DIE being parsed in this function is a definition and the
+          // entry in the map is a declaration, then we need to update the entry
+          // to point to the definition DIE.
+          if (unique_ast_entry_type->m_is_forward_declaration) {
+            unique_ast_entry_type->m_die = die;
+            unique_ast_entry_type->m_byte_size = byte_size;
+            unique_ast_entry_type->m_declaration = unique_decl;
+            unique_ast_entry_type->m_is_forward_declaration = false;
+            // Need to update Type ID to refer to the definition DIE. because
+            // it's used in ParseSubroutine to determine if we need to copy cxx
+            // method types from a declaration DIE to this definition DIE.
+            type_sp->SetID(die.GetID());
+            clang_type = type_sp->GetForwardCompilerType();
+            if (attrs.class_language != eLanguageTypeObjC &&
+                attrs.class_language != eLanguageTypeObjC_plus_plus)
+              TypeSystemClang::StartTagDeclarationDefinition(clang_type);
+
+            CompilerType compiler_type_no_qualifiers =
+                ClangUtil::RemoveFastQualifiers(clang_type);
+            auto result = dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace(
+                compiler_type_no_qualifiers.GetOpaqueQualType(),
+                *die.GetDIERef());
+            if (!result.second)
+              result.first->second = *die.GetDIERef();
+          }
+        }
         return type_sp;
       }
     }
@@ -1695,125 +1840,21 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
     default_accessibility = eAccessPrivate;
   }
 
-  if (attrs.byte_size && *attrs.byte_size == 0 && attrs.name &&
-      !die.HasChildren() && cu_language == eLanguageTypeObjC) {
-    // Work around an issue with clang at the moment where forward
-    // declarations for objective C classes are emitted as:
-    //  DW_TAG_structure_type [2]
-    //  DW_AT_name( "ForwardObjcClass" )
-    //  DW_AT_byte_size( 0x00 )
-    //  DW_AT_decl_file( "..." )
-    //  DW_AT_decl_line( 1 )
-    //
-    // Note that there is no DW_AT_declaration and there are no children,
-    // and the byte size is zero.
-    attrs.is_forward_declaration = true;
-  }
-
-  if (attrs.class_language == eLanguageTypeObjC ||
-      attrs.class_language == eLanguageTypeObjC_plus_plus) {
-    if (!attrs.is_complete_objc_class &&
-        die.Supports_DW_AT_APPLE_objc_complete_type()) {
-      // We have a valid eSymbolTypeObjCClass class symbol whose name
-      // matches the current objective C class that we are trying to find
-      // and this DIE isn't the complete definition (we checked
-      // is_complete_objc_class above and know it is false), so the real
-      // definition is in here somewhere
-      type_sp =
-          dwarf->FindCompleteObjCDefinitionTypeForDIE(die, attrs.name, true);
-
-      if (!type_sp) {
-        SymbolFileDWARFDebugMap *debug_map_symfile =
-            dwarf->GetDebugMapSymfile();
-        if (debug_map_symfile) {
-          // We weren't able to find a full declaration in this DWARF,
-          // see if we have a declaration anywhere else...
-          type_sp = debug_map_symfile->FindCompleteObjCDefinitionTypeForDIE(
-              die, attrs.name, true);
-        }
-      }
-
-      if (type_sp) {
-        if (log) {
-          dwarf->GetObjectFile()->GetModule()->LogMessage(
-              log,
-              "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is an "
-              "incomplete objc type, complete type is {5:x8}",
-              static_cast<void *>(this), die.GetOffset(),
-              DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(),
-              type_sp->GetID());
-        }
-
-        // We found a real definition for this type elsewhere so lets use
-        // it and cache the fact that we found a complete type for this
-        // die
-        dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
-        return type_sp;
-      }
-    }
-  }
-
   if (attrs.is_forward_declaration) {
-    // We have a forward declaration to a type and we need to try and
-    // find a full declaration. We look in the current type index just in
-    // case we have a forward declaration followed by an actual
-    // declarations in the DWARF. If this fails, we need to look
-    // elsewhere...
-    if (log) {
-      dwarf->GetObjectFile()->GetModule()->LogMessage(
-          log,
-          "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is a "
-          "forward declaration, trying to find complete type",
-          static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
-          tag, attrs.name.GetCString());
-    }
-
     // See if the type comes from a Clang module and if so, track down
     // that type.
     type_sp = ParseTypeFromClangModule(sc, die, log);
     if (type_sp)
       return type_sp;
-
-    // type_sp = FindDefinitionTypeForDIE (dwarf_cu, die,
-    // type_name_const_str);
-    type_sp = dwarf->FindDefinitionTypeForDWARFDeclContext(die);
-
-    if (!type_sp) {
-      SymbolFileDWARFDebugMap *debug_map_symfile = dwarf->GetDebugMapSymfile();
-      if (debug_map_symfile) {
-        // We weren't able to find a full declaration in this DWARF, see
-        // if we have a declaration anywhere else...
-        type_sp = debug_map_symfile->FindDefinitionTypeForDWARFDeclContext(die);
-      }
-    }
-
-    if (type_sp) {
-      if (log) {
-        dwarf->GetObjectFile()->GetModule()->LogMessage(
-            log,
-            "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is a "
-            "forward declaration, complete type is {5:x8}",
-            static_cast<void *>(this), die.GetOffset(),
-            DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(),
-            type_sp->GetID());
-      }
-
-      // We found a real definition for this type elsewhere so lets use
-      // it and cache the fact that we found a complete type for this die
-      dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
-      clang::DeclContext *defn_decl_ctx =
-          GetCachedClangDeclContextForDIE(dwarf->GetDIE(type_sp->GetID()));
-      if (defn_decl_ctx)
-        LinkDeclContextToDIE(defn_decl_ctx, die);
-      return type_sp;
-    }
   }
+
   assert(tag_decl_kind != -1);
   UNUSED_IF_ASSERT_DISABLED(tag_decl_kind);
-  bool clang_type_was_created = false;
-  clang::DeclContext *decl_ctx = GetClangDeclContextContainingDIE(die, nullptr);
+  DWARFDIE decl_ctx_die;
+  clang::DeclContext *decl_ctx =
+      GetClangDeclContextContainingDIE(die, &decl_ctx_die);
 
-  PrepareContextToReceiveMembers(m_ast, GetClangASTImporter(), decl_ctx, die,
+  PrepareContextToReceiveMembers(decl_ctx, decl_ctx_die, die,
                                  attrs.name.GetCString());
 
   if (attrs.accessibility == eAccessNone && decl_ctx) {
@@ -1852,20 +1893,17 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
             tag_decl_kind, template_param_infos);
     clang_type =
         m_ast.CreateClassTemplateSpecializationType(class_specialization_decl);
-    clang_type_was_created = true;
 
     m_ast.SetMetadata(class_template_decl, metadata);
     m_ast.SetMetadata(class_specialization_decl, metadata);
   }
 
-  if (!clang_type_was_created) {
-    clang_type_was_created = true;
+  if (!clang_type) {
     clang_type = m_ast.CreateRecordType(
         decl_ctx, GetOwningClangModule(die), attrs.accessibility,
         attrs.name.GetCString(), tag_decl_kind, attrs.class_language, &metadata,
         attrs.exports_symbols);
   }
-
   // Store a forward declaration to this class type in case any
   // parameters in any class methods need it for the clang types for
   // function prototypes.
@@ -1876,13 +1914,19 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
       Type::ResolveState::Forward,
       TypePayloadClang(OptionalClangModuleID(), attrs.is_complete_objc_class));
 
+  // UniqueDWARFASTType is large, so don't create a local variables on the
+  // stack, put it on the heap. This function is often called recursively and
+  // clang isn't good at sharing the stack space for variables in different
+  // blocks.
+  auto unique_ast_entry_up = std::make_unique<UniqueDWARFASTType>();
   // Add our type to the unique type map so we don't end up creating many
   // copies of the same type over and over in the ASTContext for our
   // module
   unique_ast_entry_up->m_type_sp = type_sp;
   unique_ast_entry_up->m_die = die;
   unique_ast_entry_up->m_declaration = unique_decl;
-  unique_ast_entry_up->m_byte_size = attrs.byte_size.value_or(0);
+  unique_ast_entry_up->m_byte_size = byte_size;
+  unique_ast_entry_up->m_is_forward_declaration = attrs.is_forward_declaration;
   dwarf->GetUniqueDWARFASTTypeMap().Insert(unique_typename,
                                            *unique_ast_entry_up);
 
@@ -1923,7 +1967,7 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
           GetClangASTImporter().SetRecordLayout(record_decl, layout);
         }
       }
-    } else if (clang_type_was_created) {
+    } else {
       // Start the definition if the class is not objective C since the
       // underlying decls respond to isCompleteDefinition(). Objective
       // C decls don't respond to isCompleteDefinition() so we can't
@@ -1935,26 +1979,21 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
       if (attrs.class_language != eLanguageTypeObjC &&
           attrs.class_language != eLanguageTypeObjC_plus_plus)
         TypeSystemClang::StartTagDeclarationDefinition(clang_type);
-
-      // Leave this as a forward declaration until we need to know the
-      // details of the type. lldb_private::Type will automatically call
-      // the SymbolFile virtual function
-      // "SymbolFileDWARF::CompleteType(Type *)" When the definition
-      // needs to be defined.
-      assert(!dwarf->GetForwardDeclCompilerTypeToDIE().count(
-                 ClangUtil::RemoveFastQualifiers(clang_type)
-                     .GetOpaqueQualType()) &&
-             "Type already in the forward declaration map!");
-      // Can't assume m_ast.GetSymbolFile() is actually a
-      // SymbolFileDWARF, it can be a SymbolFileDWARFDebugMap for Apple
-      // binaries.
-      dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace(
-          ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType(),
-          *die.GetDIERef());
-      m_ast.SetHasExternalStorage(clang_type.GetOpaqueQualType(), true);
     }
   }
 
+  // If this is a declaration DIE, leave this as a forward declaration until we
+  // need to know the details of the type. lldb_private::Type will automatically
+  // call the SymbolFile virtual function "SymbolFileDWARF::CompleteType(Type
+  // *)" When the definition needs to be defined.
+  assert(!dwarf->GetForwardDeclCompilerTypeToDIE().count(
+             ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType()) &&
+         "Type already in the forward declaration map!");
+  dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace(
+      ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType(),
+      *die.GetDIERef());
+  m_ast.SetHasExternalStorage(clang_type.GetOpaqueQualType(), true);
+
   // If we made a clang type, set the trivial abi if applicable: We only
   // do this for pass by value - which implies the Trivial ABI. There
   // isn't a way to assert that something that would normally be pass by
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
index 8d4af203bb28..853b8ccc3036 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
@@ -42,40 +42,40 @@ struct ParsedDWARFTypeAttributes;
 
 class DWARFASTParserClang : public lldb_private::plugin::dwarf::DWARFASTParser {
 public:
+  typedef lldb_private::plugin::dwarf::DWARFDIE DWARFDIE;
+
   DWARFASTParserClang(lldb_private::TypeSystemClang &ast);
 
   ~DWARFASTParserClang() override;
 
   // DWARFASTParser interface.
-  lldb::TypeSP
-  ParseTypeFromDWARF(const lldb_private::SymbolContext &sc,
-                     const lldb_private::plugin::dwarf::DWARFDIE &die,
-                     bool *type_is_new_ptr) override;
+  lldb::TypeSP ParseTypeFromDWARF(const lldb_private::SymbolContext &sc,
+                                  const DWARFDIE &die,
+                                  bool *type_is_new_ptr) override;
 
-  lldb_private::ConstString ConstructDemangledNameFromDWARF(
-      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
+  lldb_private::ConstString
+  ConstructDemangledNameFromDWARF(const DWARFDIE &die) override;
 
   lldb_private::Function *
   ParseFunctionFromDWARF(lldb_private::CompileUnit &comp_unit,
-                         const lldb_private::plugin::dwarf::DWARFDIE &die,
+                         const DWARFDIE &die,
                          const lldb_private::AddressRange &func_range) override;
 
   bool
-  CompleteTypeFromDWARF(const lldb_private::plugin::dwarf::DWARFDIE &die,
-                        lldb_private::Type *type,
+  CompleteTypeFromDWARF(const DWARFDIE &die, lldb_private::Type *type,
                         lldb_private::CompilerType &compiler_type) override;
 
-  lldb_private::CompilerDecl GetDeclForUIDFromDWARF(
-      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
+  lldb_private::CompilerDecl
+  GetDeclForUIDFromDWARF(const DWARFDIE &die) override;
 
   void EnsureAllDIEsInDeclContextHaveBeenParsed(
       lldb_private::CompilerDeclContext decl_context) override;
 
-  lldb_private::CompilerDeclContext GetDeclContextForUIDFromDWARF(
-      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
+  lldb_private::CompilerDeclContext
+  GetDeclContextForUIDFromDWARF(const DWARFDIE &die) override;
 
-  lldb_private::CompilerDeclContext GetDeclContextContainingUIDFromDWARF(
-      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
+  lldb_private::CompilerDeclContext
+  GetDeclContextContainingUIDFromDWARF(const DWARFDIE &die) override;
 
   lldb_private::ClangASTImporter &GetClangASTImporter();
 
@@ -105,8 +105,13 @@ public:
   /// \return A string, including surrounding '<>', of the template parameters.
   /// If the DIE's name already has '<>', returns an empty ConstString because
   /// it's assumed that the caller is using the DIE name anyway.
-  lldb_private::ConstString GetDIEClassTemplateParams(
-      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
+  lldb_private::ConstString
+  GetDIEClassTemplateParams(const DWARFDIE &die) override;
+
+  // Searching for definition DIE for the given DIE and return the type
+  // associated with the definition DIE, or nullptr if definition DIE is not
+  // found.
+  lldb_private::Type *FindDefinitionTypeForDIE(const DWARFDIE &die) override;
 
 protected:
   /// Protected typedefs and members.
@@ -118,8 +123,7 @@ protected:
       const lldb_private::plugin::dwarf::DWARFDebugInfoEntry *,
       clang::DeclContext *>
       DIEToDeclContextMap;
-  typedef std::multimap<const clang::DeclContext *,
-                        const lldb_private::plugin::dwarf::DWARFDIE>
+  typedef std::multimap<const clang::DeclContext *, const DWARFDIE>
       DeclContextToDIEMap;
   typedef llvm::DenseMap<
       const lldb_private::plugin::dwarf::DWARFDebugInfoEntry *,
@@ -137,14 +141,11 @@ protected:
   std::unique_ptr<lldb_private::ClangASTImporter> m_clang_ast_importer_up;
   /// @}
 
-  clang::DeclContext *
-  GetDeclContextForBlock(const lldb_private::plugin::dwarf::DWARFDIE &die);
+  clang::DeclContext *GetDeclContextForBlock(const DWARFDIE &die);
 
-  clang::BlockDecl *
-  ResolveBlockDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
+  clang::BlockDecl *ResolveBlockDIE(const DWARFDIE &die);
 
-  clang::NamespaceDecl *
-  ResolveNamespaceDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
+  clang::NamespaceDecl *ResolveNamespaceDIE(const DWARFDIE &die);
 
   /// Returns the namespace decl that a DW_TAG_imported_declaration imports.
   ///
@@ -155,96 +156,86 @@ protected:
   ///          'die' imports. If the imported entity is not a namespace
   ///          or another import declaration, returns nullptr. If an error
   ///          occurs, returns nullptr.
-  clang::NamespaceDecl *ResolveImportedDeclarationDIE(
-      const lldb_private::plugin::dwarf::DWARFDIE &die);
+  clang::NamespaceDecl *ResolveImportedDeclarationDIE(const DWARFDIE &die);
 
-  bool ParseTemplateDIE(const lldb_private::plugin::dwarf::DWARFDIE &die,
+  bool ParseTemplateDIE(const DWARFDIE &die,
                         lldb_private::TypeSystemClang::TemplateParameterInfos
                             &template_param_infos);
 
   bool ParseTemplateParameterInfos(
-      const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+      const DWARFDIE &parent_die,
       lldb_private::TypeSystemClang::TemplateParameterInfos
           &template_param_infos);
 
-  std::string
-  GetCPlusPlusQualifiedName(const lldb_private::plugin::dwarf::DWARFDIE &die);
+  std::string GetCPlusPlusQualifiedName(const DWARFDIE &die);
 
   bool ParseChildMembers(
-      const lldb_private::plugin::dwarf::DWARFDIE &die,
-      lldb_private::CompilerType &class_compiler_type,
+      const DWARFDIE &die, lldb_private::CompilerType &class_compiler_type,
       std::vector<std::unique_ptr<clang::CXXBaseSpecifier>> &base_classes,
-      std::vector<lldb_private::plugin::dwarf::DWARFDIE> &member_function_dies,
-      std::vector<lldb_private::plugin::dwarf::DWARFDIE> &contained_type_dies,
+      std::vector<DWARFDIE> &member_function_dies,
+      std::vector<DWARFDIE> &contained_type_dies,
       DelayedPropertyList &delayed_properties,
       const lldb::AccessType default_accessibility,
       lldb_private::ClangASTImporter::LayoutInfo &layout_info);
 
   size_t
   ParseChildParameters(clang::DeclContext *containing_decl_ctx,
-                       const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
-                       bool skip_artificial, bool &is_static, bool &is_variadic,
+                       const DWARFDIE &parent_die, bool skip_artificial,
+                       bool &is_static, bool &is_variadic,
                        bool &has_template_params,
                        std::vector<lldb_private::CompilerType> &function_args,
                        std::vector<clang::ParmVarDecl *> &function_param_decls,
                        unsigned &type_quals);
 
-  size_t ParseChildEnumerators(
-      lldb_private::CompilerType &compiler_type, bool is_signed,
-      uint32_t enumerator_byte_size,
-      const lldb_private::plugin::dwarf::DWARFDIE &parent_die);
+  size_t ParseChildEnumerators(lldb_private::CompilerType &compiler_type,
+                               bool is_signed, uint32_t enumerator_byte_size,
+                               const DWARFDIE &parent_die);
 
   /// Parse a structure, class, or union type DIE.
-  lldb::TypeSP
-  ParseStructureLikeDIE(const lldb_private::SymbolContext &sc,
-                        const lldb_private::plugin::dwarf::DWARFDIE &die,
-                        ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP ParseStructureLikeDIE(const lldb_private::SymbolContext &sc,
+                                     const DWARFDIE &die,
+                                     ParsedDWARFTypeAttributes &attrs);
 
-  clang::Decl *
-  GetClangDeclForDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
+  clang::Decl *GetClangDeclForDIE(const DWARFDIE &die);
 
-  clang::DeclContext *
-  GetClangDeclContextForDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
+  clang::DeclContext *GetClangDeclContextForDIE(const DWARFDIE &die);
 
-  clang::DeclContext *GetClangDeclContextContainingDIE(
-      const lldb_private::plugin::dwarf::DWARFDIE &die,
-      lldb_private::plugin::dwarf::DWARFDIE *decl_ctx_die);
-  lldb_private::OptionalClangModuleID
-  GetOwningClangModule(const lldb_private::plugin::dwarf::DWARFDIE &die);
+  clang::DeclContext *GetClangDeclContextContainingDIE(const DWARFDIE &die,
+                                                       DWARFDIE *decl_ctx_die);
+  lldb_private::OptionalClangModuleID GetOwningClangModule(const DWARFDIE &die);
 
-  bool CopyUniqueClassMethodTypes(
-      const lldb_private::plugin::dwarf::DWARFDIE &src_class_die,
-      const lldb_private::plugin::dwarf::DWARFDIE &dst_class_die,
-      lldb_private::Type *class_type,
-      std::vector<lldb_private::plugin::dwarf::DWARFDIE> &failures);
+  bool CopyUniqueClassMethodTypes(const DWARFDIE &src_class_die,
+                                  const DWARFDIE &dst_class_die,
+                                  lldb_private::Type *class_type,
+                                  std::vector<DWARFDIE> &failures);
 
-  clang::DeclContext *GetCachedClangDeclContextForDIE(
-      const lldb_private::plugin::dwarf::DWARFDIE &die);
+  clang::DeclContext *GetCachedClangDeclContextForDIE(const DWARFDIE &die);
 
-  void LinkDeclContextToDIE(clang::DeclContext *decl_ctx,
-                            const lldb_private::plugin::dwarf::DWARFDIE &die);
+  void LinkDeclContextToDIE(clang::DeclContext *decl_ctx, const DWARFDIE &die);
 
-  void LinkDeclToDIE(clang::Decl *decl,
-                     const lldb_private::plugin::dwarf::DWARFDIE &die);
+  void LinkDeclToDIE(clang::Decl *decl, const DWARFDIE &die);
 
   /// If \p type_sp is valid, calculate and set its symbol context scope, and
   /// update the type list for its backing symbol file.
   ///
   /// Returns \p type_sp.
-  lldb::TypeSP UpdateSymbolContextScopeForType(
-      const lldb_private::SymbolContext &sc,
-      const lldb_private::plugin::dwarf::DWARFDIE &die, lldb::TypeSP type_sp);
+  lldb::TypeSP
+  UpdateSymbolContextScopeForType(const lldb_private::SymbolContext &sc,
+                                  const DWARFDIE &die, lldb::TypeSP type_sp);
 
   /// Follow Clang Module Skeleton CU references to find a type definition.
-  lldb::TypeSP
-  ParseTypeFromClangModule(const lldb_private::SymbolContext &sc,
-                           const lldb_private::plugin::dwarf::DWARFDIE &die,
-                           lldb_private::Log *log);
+  lldb::TypeSP ParseTypeFromClangModule(const lldb_private::SymbolContext &sc,
+                                        const DWARFDIE &die,
+                                        lldb_private::Log *log);
 
   // Return true if this type is a declaration to a type in an external
   // module.
-  lldb::ModuleSP
-  GetModuleForType(const lldb_private::plugin::dwarf::DWARFDIE &die);
+  lldb::ModuleSP GetModuleForType(const DWARFDIE &die);
+
+  void PrepareContextToReceiveMembers(clang::DeclContext *decl_ctx,
+                                      const DWARFDIE &decl_ctx_die,
+                                      const DWARFDIE &die,
+                                      const char *type_name_cstr);
 
   static bool classof(const DWARFASTParser *Parser) {
     return Parser->GetKind() == Kind::DWARFASTParserClang;
@@ -274,10 +265,8 @@ private:
 
   /// Parsed form of all attributes that are relevant for parsing type members.
   struct MemberAttributes {
-    explicit MemberAttributes(
-        const lldb_private::plugin::dwarf::DWARFDIE &die,
-        const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
-        lldb::ModuleSP module_sp);
+    explicit MemberAttributes(const DWARFDIE &die, const DWARFDIE &parent_die,
+                              lldb::ModuleSP module_sp);
     const char *name = nullptr;
     /// Indicates how many bits into the word (according to the host endianness)
     /// the low-order bit of the field starts. Can be negative.
@@ -324,15 +313,12 @@ private:
   /// created property.
   /// \param delayed_properties The list of delayed properties that the result
   /// will be appended to.
-  void
-  ParseObjCProperty(const lldb_private::plugin::dwarf::DWARFDIE &die,
-                    const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
-                    const lldb_private::CompilerType &class_clang_type,
-                    DelayedPropertyList &delayed_properties);
+  void ParseObjCProperty(const DWARFDIE &die, const DWARFDIE &parent_die,
+                         const lldb_private::CompilerType &class_clang_type,
+                         DelayedPropertyList &delayed_properties);
 
   void
-  ParseSingleMember(const lldb_private::plugin::dwarf::DWARFDIE &die,
-                    const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+  ParseSingleMember(const DWARFDIE &die, const DWARFDIE &parent_die,
                     const lldb_private::CompilerType &class_clang_type,
                     lldb::AccessType default_accessibility,
                     lldb_private::ClangASTImporter::LayoutInfo &layout_info,
@@ -350,31 +336,25 @@ private:
   /// \param[in] class_clang_type The parent RecordType of the static
   ///                             member this function will create.
   void CreateStaticMemberVariable(
-      const lldb_private::plugin::dwarf::DWARFDIE &die,
-      const MemberAttributes &attrs,
+      const DWARFDIE &die, const MemberAttributes &attrs,
       const lldb_private::CompilerType &class_clang_type);
 
-  bool CompleteRecordType(const lldb_private::plugin::dwarf::DWARFDIE &die,
-                          lldb_private::Type *type,
+  bool CompleteRecordType(const DWARFDIE &die, lldb_private::Type *type,
                           lldb_private::CompilerType &clang_type);
-  bool CompleteEnumType(const lldb_private::plugin::dwarf::DWARFDIE &die,
-                        lldb_private::Type *type,
+  bool CompleteEnumType(const DWARFDIE &die, lldb_private::Type *type,
                         lldb_private::CompilerType &clang_type);
 
-  lldb::TypeSP
-  ParseTypeModifier(const lldb_private::SymbolContext &sc,
-                    const lldb_private::plugin::dwarf::DWARFDIE &die,
-                    ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP ParseTypeModifier(const lldb_private::SymbolContext &sc,
+                                 const DWARFDIE &die,
+                                 ParsedDWARFTypeAttributes &attrs);
   lldb::TypeSP ParseEnum(const lldb_private::SymbolContext &sc,
-                         const lldb_private::plugin::dwarf::DWARFDIE &die,
-                         ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP ParseSubroutine(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                         const DWARFDIE &die, ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP ParseSubroutine(const DWARFDIE &die,
                                const ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP ParseArrayType(const lldb_private::plugin::dwarf::DWARFDIE &die,
+  lldb::TypeSP ParseArrayType(const DWARFDIE &die,
                               const ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP
-  ParsePointerToMemberType(const lldb_private::plugin::dwarf::DWARFDIE &die,
-                           const ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP ParsePointerToMemberType(const DWARFDIE &die,
+                                        const ParsedDWARFTypeAttributes &attrs);
 
   /// Parses a DW_TAG_inheritance DIE into a base/super class.
   ///
@@ -391,8 +371,7 @@ private:
   /// \param layout_info The layout information that will be updated for C++
   /// base classes with the base offset.
   void ParseInheritance(
-      const lldb_private::plugin::dwarf::DWARFDIE &die,
-      const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+      const DWARFDIE &die, const DWARFDIE &parent_die,
       const lldb_private::CompilerType class_clang_type,
       const lldb::AccessType default_accessibility,
       const lldb::ModuleSP &module_sp,
@@ -409,8 +388,7 @@ private:
   /// \param layout_info The layout information that will be updated for
   //   base classes with the base offset
   void
-  ParseRustVariantPart(lldb_private::plugin::dwarf::DWARFDIE &die,
-                       const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+  ParseRustVariantPart(DWARFDIE &die, const DWARFDIE &parent_die,
                        lldb_private::CompilerType &class_clang_type,
                        const lldb::AccessType default_accesibility,
                        lldb_private::ClangASTImporter::LayoutInfo &layout_info);
@@ -420,8 +398,9 @@ private:
 /// Some attributes are relevant for all kinds of types (declaration), while
 /// others are only meaningful to a specific type (is_virtual)
 struct ParsedDWARFTypeAttributes {
-  explicit ParsedDWARFTypeAttributes(
-      const lldb_private::plugin::dwarf::DWARFDIE &die);
+  typedef lldb_private::plugin::dwarf::DWARFDIE DWARFDIE;
+
+  explicit ParsedDWARFTypeAttributes(const DWARFDIE &die);
 
   lldb::AccessType accessibility = lldb::eAccessNone;
   bool is_artificial = false;
@@ -438,7 +417,7 @@ struct ParsedDWARFTypeAttributes {
   const char *mangled_name = nullptr;
   lldb_private::ConstString name;
   lldb_private::Declaration decl;
-  lldb_private::plugin::dwarf::DWARFDIE object_pointer;
+  DWARFDIE object_pointer;
   lldb_private::plugin::dwarf::DWARFFormValue abstract_origin;
   lldb_private::plugin::dwarf::DWARFFormValue containing_type;
   lldb_private::plugin::dwarf::DWARFFormValue signature;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index f6f152726bf7..5a07fd30fbf7 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -1632,27 +1632,33 @@ bool SymbolFileDWARF::CompleteType(CompilerType &compiler_type) {
     return true;
   }
 
-  DWARFDIE dwarf_die = GetDIE(die_it->getSecond());
-  if (dwarf_die) {
-    // Once we start resolving this type, remove it from the forward
-    // declaration map in case anyone child members or other types require this
-    // type to get resolved. The type will get resolved when all of the calls
-    // to SymbolFileDWARF::ResolveClangOpaqueTypeDefinition are done.
-    GetForwardDeclCompilerTypeToDIE().erase(die_it);
-
-    Type *type = GetDIEToType().lookup(dwarf_die.GetDIE());
+  // Once we start resolving this type, remove it from the forward
+  // declaration map in case anyone's child members or other types require this
+  // type to get resolved.
+  DWARFDIE dwarf_die = GetDIE(die_it->second);
+  GetForwardDeclCompilerTypeToDIE().erase(die_it);
+  Type *type = nullptr;
+  if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU()))
+    type = dwarf_ast->FindDefinitionTypeForDIE(dwarf_die);
+  if (!type)
+    return false;
 
-    Log *log = GetLog(DWARFLog::DebugInfo | DWARFLog::TypeCompletion);
-    if (log)
-      GetObjectFile()->GetModule()->LogMessageVerboseBacktrace(
-          log, "{0:x8}: {1} ({2}) '{3}' resolving forward declaration...",
-          dwarf_die.GetID(), DW_TAG_value_to_name(dwarf_die.Tag()),
-          dwarf_die.Tag(), type->GetName().AsCString());
-    assert(compiler_type);
-    if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU()))
-      return dwarf_ast->CompleteTypeFromDWARF(dwarf_die, type, compiler_type);
+  die_it = GetForwardDeclCompilerTypeToDIE().find(
+      compiler_type_no_qualifiers.GetOpaqueQualType());
+  if (die_it != GetForwardDeclCompilerTypeToDIE().end()) {
+    dwarf_die = GetDIE(die_it->getSecond());
+    GetForwardDeclCompilerTypeToDIE().erase(die_it);
   }
-  return false;
+
+  if (Log *log = GetLog(DWARFLog::DebugInfo | DWARFLog::TypeCompletion))
+    GetObjectFile()->GetModule()->LogMessageVerboseBacktrace(
+        log, "{0:x8}: {1} ({2}) '{3}' resolving forward declaration...",
+        dwarf_die.GetID(), DW_TAG_value_to_name(dwarf_die.Tag()),
+        dwarf_die.Tag(), type->GetName().AsCString());
+  assert(compiler_type);
+  if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU()))
+    return dwarf_ast->CompleteTypeFromDWARF(dwarf_die, type, compiler_type);
+  return true;
 }
 
 Type *SymbolFileDWARF::ResolveType(const DWARFDIE &die,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index 7282c08c6857..94aa810680c5 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -533,8 +533,12 @@ protected:
   NameToOffsetMap m_function_scope_qualified_name_map;
   std::unique_ptr<DWARFDebugRanges> m_ranges;
   UniqueDWARFASTTypeMap m_unique_ast_type_map;
+  // A map from DIE to lldb_private::Type. For record type, the key might be
+  // either declaration DIE or definition DIE.
   DIEToTypePtr m_die_to_type;
   DIEToVariableSP m_die_to_variable_sp;
+  // A map from CompilerType to the struct/class/union/enum DIE (might be a
+  // declaration or a definition) that is used to construct it.
   CompilerTypeToDIE m_forward_decl_compiler_type_to_die;
   llvm::DenseMap<dw_offset_t, std::unique_ptr<SupportFileList>>
       m_type_unit_support_files;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
index 223518f0ae82..4762356034ca 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
@@ -13,66 +13,67 @@
 using namespace lldb_private::dwarf;
 using namespace lldb_private::plugin::dwarf;
 
-bool UniqueDWARFASTTypeList::Find(const DWARFDIE &die,
-                                  const lldb_private::Declaration &decl,
-                                  const int32_t byte_size,
-                                  UniqueDWARFASTType &entry) const {
-  for (const UniqueDWARFASTType &udt : m_collection) {
+UniqueDWARFASTType *UniqueDWARFASTTypeList::Find(
+    const DWARFDIE &die, const lldb_private::Declaration &decl,
+    const int32_t byte_size, bool is_forward_declaration) {
+  for (UniqueDWARFASTType &udt : m_collection) {
     // Make sure the tags match
     if (udt.m_die.Tag() == die.Tag()) {
-      // Validate byte sizes of both types only if both are valid.
-      if (udt.m_byte_size < 0 || byte_size < 0 ||
-          udt.m_byte_size == byte_size) {
-        // Make sure the file and line match
-        if (udt.m_declaration == decl) {
-          // The type has the same name, and was defined on the same file and
-          // line. Now verify all of the parent DIEs match.
-          DWARFDIE parent_arg_die = die.GetParent();
-          DWARFDIE parent_pos_die = udt.m_die.GetParent();
-          bool match = true;
-          bool done = false;
-          while (!done && match && parent_arg_die && parent_pos_die) {
-            const dw_tag_t parent_arg_tag = parent_arg_die.Tag();
-            const dw_tag_t parent_pos_tag = parent_pos_die.Tag();
-            if (parent_arg_tag == parent_pos_tag) {
-              switch (parent_arg_tag) {
-              case DW_TAG_class_type:
-              case DW_TAG_structure_type:
-              case DW_TAG_union_type:
-              case DW_TAG_namespace: {
-                const char *parent_arg_die_name = parent_arg_die.GetName();
-                if (parent_arg_die_name ==
-                    nullptr) // Anonymous (i.e. no-name) struct
-                {
-                  match = false;
-                } else {
-                  const char *parent_pos_die_name = parent_pos_die.GetName();
-                  if (parent_pos_die_name == nullptr ||
-                      ((parent_arg_die_name != parent_pos_die_name) &&
-                       strcmp(parent_arg_die_name, parent_pos_die_name)))
-                    match = false;
-                }
-              } break;
-
-              case DW_TAG_compile_unit:
-              case DW_TAG_partial_unit:
-                done = true;
-                break;
-              default:
-                break;
-              }
+      // If they are not both definition DIEs or both declaration DIEs, then
+      // don't check for byte size and declaration location, because declaration
+      // DIEs usually don't have those info.
+      bool matching_size_declaration =
+          udt.m_is_forward_declaration != is_forward_declaration
+              ? true
+              : (udt.m_byte_size < 0 || byte_size < 0 ||
+                 udt.m_byte_size == byte_size) &&
+                    udt.m_declaration == decl;
+      if (!matching_size_declaration)
+        continue;
+      // The type has the same name, and was defined on the same file and
+      // line. Now verify all of the parent DIEs match.
+      DWARFDIE parent_arg_die = die.GetParent();
+      DWARFDIE parent_pos_die = udt.m_die.GetParent();
+      bool match = true;
+      bool done = false;
+      while (!done && match && parent_arg_die && parent_pos_die) {
+        const dw_tag_t parent_arg_tag = parent_arg_die.Tag();
+        const dw_tag_t parent_pos_tag = parent_pos_die.Tag();
+        if (parent_arg_tag == parent_pos_tag) {
+          switch (parent_arg_tag) {
+          case DW_TAG_class_type:
+          case DW_TAG_structure_type:
+          case DW_TAG_union_type:
+          case DW_TAG_namespace: {
+            const char *parent_arg_die_name = parent_arg_die.GetName();
+            if (parent_arg_die_name == nullptr) {
+              // Anonymous (i.e. no-name) struct
+              match = false;
+            } else {
+              const char *parent_pos_die_name = parent_pos_die.GetName();
+              if (parent_pos_die_name == nullptr ||
+                  ((parent_arg_die_name != parent_pos_die_name) &&
+                   strcmp(parent_arg_die_name, parent_pos_die_name)))
+                match = false;
             }
-            parent_arg_die = parent_arg_die.GetParent();
-            parent_pos_die = parent_pos_die.GetParent();
-          }
+          } break;
 
-          if (match) {
-            entry = udt;
-            return true;
+          case DW_TAG_compile_unit:
+          case DW_TAG_partial_unit:
+            done = true;
+            break;
+          default:
+            break;
           }
         }
+        parent_arg_die = parent_arg_die.GetParent();
+        parent_pos_die = parent_pos_die.GetParent();
+      }
+
+      if (match) {
+        return &udt;
       }
     }
   }
-  return false;
+  return nullptr;
 }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
index bf3cbae55e5c..29e5c02dcbe1 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
@@ -23,31 +23,19 @@ public:
   // Constructors and Destructors
   UniqueDWARFASTType() : m_type_sp(), m_die(), m_declaration() {}
 
-  UniqueDWARFASTType(lldb::TypeSP &type_sp, const DWARFDIE &die,
-                     const Declaration &decl, int32_t byte_size)
-      : m_type_sp(type_sp), m_die(die), m_declaration(decl),
-        m_byte_size(byte_size) {}
-
   UniqueDWARFASTType(const UniqueDWARFASTType &rhs)
       : m_type_sp(rhs.m_type_sp), m_die(rhs.m_die),
-        m_declaration(rhs.m_declaration), m_byte_size(rhs.m_byte_size) {}
+        m_declaration(rhs.m_declaration), m_byte_size(rhs.m_byte_size),
+        m_is_forward_declaration(rhs.m_is_forward_declaration) {}
 
   ~UniqueDWARFASTType() = default;
 
-  UniqueDWARFASTType &operator=(const UniqueDWARFASTType &rhs) {
-    if (this != &rhs) {
-      m_type_sp = rhs.m_type_sp;
-      m_die = rhs.m_die;
-      m_declaration = rhs.m_declaration;
-      m_byte_size = rhs.m_byte_size;
-    }
-    return *this;
-  }
-
   lldb::TypeSP m_type_sp;
   DWARFDIE m_die;
   Declaration m_declaration;
   int32_t m_byte_size = -1;
+  // True if the m_die is a forward declaration DIE.
+  bool m_is_forward_declaration = true;
 };
 
 class UniqueDWARFASTTypeList {
@@ -62,8 +50,9 @@ public:
     m_collection.push_back(entry);
   }
 
-  bool Find(const DWARFDIE &die, const Declaration &decl,
-            const int32_t byte_size, UniqueDWARFASTType &entry) const;
+  UniqueDWARFASTType *Find(const DWARFDIE &die, const Declaration &decl,
+                           const int32_t byte_size,
+                           bool is_forward_declaration);
 
 protected:
   typedef std::vector<UniqueDWARFASTType> collection;
@@ -80,14 +69,15 @@ public:
     m_collection[name.GetCString()].Append(entry);
   }
 
-  bool Find(ConstString name, const DWARFDIE &die, const Declaration &decl,
-            const int32_t byte_size, UniqueDWARFASTType &entry) const {
+  UniqueDWARFASTType *Find(ConstString name, const DWARFDIE &die,
+                           const Declaration &decl, const int32_t byte_size,
+                           bool is_forward_declaration) {
     const char *unique_name_cstr = name.GetCString();
-    collection::const_iterator pos = m_collection.find(unique_name_cstr);
+    collection::iterator pos = m_collection.find(unique_name_cstr);
     if (pos != m_collection.end()) {
-      return pos->second.Find(die, decl, byte_size, entry);
+      return pos->second.Find(die, decl, byte_size, is_forward_declaration);
     }
-    return false;
+    return nullptr;
   }
 
 protected:
diff --git a/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test b/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
new file mode 100644
index 000000000000..a2b36dadedd2
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
@@ -0,0 +1,34 @@
+# Test definition DIE searching is delayed until complete type is required.
+
+# RUN: split-file %s %t
+# RUN: %clangxx_host %t/main.cpp %t/t1_def.cpp -g -o %t.out
+# RUN: %lldb -b %t.out -s %t/lldb.cmd | FileCheck %s
+
+# CHECK: (lldb) p v1
+# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't2<t1>'
+# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't1'
+# CHECK: DW_TAG_structure_type (DW_TAG_structure_type) 't2<t1>' resolving forward declaration...
+# CHECK: (t2<t1>)  {}
+# CHECK: (lldb) p v2
+# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't1'
+# CHECK: DW_TAG_structure_type (DW_TAG_structure_type) 't1' resolving forward declaration...
+
+#--- lldb.cmd
+log enable dwarf comp
+p v1
+p v2
+
+#--- main.cpp
+template<typename T>
+struct t2 {
+};
+struct t1;
+t2<t1> v1; // this CU doesn't have definition DIE for t1, but only declaration DIE for it.
+int main() {
+}
+
+#--- t1_def.cpp
+struct t1 { // this CU contains definition DIE for t1.
+  int x;
+};
+t1 v2;
\ No newline at end of file
-- 
GitLab


From 062f6fe324e98b0994e49bc14eb45b20aa0807c4 Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Fri, 10 May 2024 09:35:41 -0700
Subject: [PATCH 0422/1206] [InstallAPI] Support mutually exclusive parse
 options (#90686)

Projects like libc use mutually exclusive macros to compile files
multiple times and then merge the result into the final library. For
installapi to accept these, we'd need to parse the same declarations in
different ways. This patch adds the basic pipelining for installapi to
create the correct TBD file.

* -Xproject allows: -fmodules, -fobjc-arc, fvisibility=hidden, prefix
headers
* -Xlabel allows: -D and -U settings
* Error on 'private' and 'public' labels -X<label>
* Xplatform allows: -iframework <path> This is to support the case where
zippered frameworks want to pass in iOSSupport search path.
---
 .../clang/Basic/DiagnosticInstallAPIKinds.td  |   1 +
 clang/include/clang/InstallAPI/MachO.h        |   2 +
 .../Modules/module.modulemap                  |   3 +
 .../Inputs/LibFoo/usr/include/foo.h           |  15 +
 .../Inputs/LibFoo/usr/include/macro_defs.h    |  15 +
 .../Inputs/LibFoo/usr/include/public.h        |   9 +
 .../Zippered.framework/Headers/Zippered.h     |  20 +
 .../PrivateHeaders/Zippered_Private.h         |   9 +
 .../InstallAPI/Inputs/Zippered/Zippered.tbd   |  47 +++
 .../InstallAPI/Inputs/Zippered/Zippered.yaml  | 383 ++++++++++++++++++
 clang/test/InstallAPI/exclusive-passes-2.test |  58 +++
 .../InstallAPI/exclusive-passes-platform.test | 286 +++++++++++++
 .../InstallAPI/exclusive-passes-zippered.test |  56 +++
 clang/test/InstallAPI/exclusive-passes.test   |  54 +++
 .../InstallAPI/invalid-exclusive-passes.test  |  37 ++
 .../InstallAPI/project-header-only-args.test  |  81 ++++
 .../clang-installapi/ClangInstallAPI.cpp      |  22 +-
 .../tools/clang-installapi/InstallAPIOpts.td  |  30 ++
 clang/tools/clang-installapi/Options.cpp      | 187 +++++++--
 clang/tools/clang-installapi/Options.h        |  13 +-
 llvm/include/llvm/TextAPI/Utils.h             |   9 +
 llvm/lib/TextAPI/Utils.cpp                    |  10 +
 22 files changed, 1316 insertions(+), 31 deletions(-)
 create mode 100644 clang/test/InstallAPI/Inputs/Foundation/Foundation.framework/Modules/module.modulemap
 create mode 100644 clang/test/InstallAPI/Inputs/LibFoo/usr/include/foo.h
 create mode 100644 clang/test/InstallAPI/Inputs/LibFoo/usr/include/macro_defs.h
 create mode 100644 clang/test/InstallAPI/Inputs/LibFoo/usr/include/public.h
 create mode 100644 clang/test/InstallAPI/Inputs/Zippered/Zippered.framework/Headers/Zippered.h
 create mode 100644 clang/test/InstallAPI/Inputs/Zippered/Zippered.framework/PrivateHeaders/Zippered_Private.h
 create mode 100644 clang/test/InstallAPI/Inputs/Zippered/Zippered.tbd
 create mode 100644 clang/test/InstallAPI/Inputs/Zippered/Zippered.yaml
 create mode 100644 clang/test/InstallAPI/exclusive-passes-2.test
 create mode 100644 clang/test/InstallAPI/exclusive-passes-platform.test
 create mode 100644 clang/test/InstallAPI/exclusive-passes-zippered.test
 create mode 100644 clang/test/InstallAPI/exclusive-passes.test
 create mode 100644 clang/test/InstallAPI/invalid-exclusive-passes.test
 create mode 100644 clang/test/InstallAPI/project-header-only-args.test

diff --git a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
index 6896e0f5aa59..674742431dcb 100644
--- a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
+++ b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
@@ -25,6 +25,7 @@ def err_unsupported_vendor : Error<"vendor '%0' is not supported: '%1'">;
 def err_unsupported_environment : Error<"environment '%0' is not supported: '%1'">;
 def err_unsupported_os : Error<"os '%0' is not supported: '%1'">;
 def err_cannot_read_input_list : Error<"could not read %select{alias list|filelist}0 '%1': %2">;
+def err_invalid_label: Error<"label '%0' is reserved: use a different label name for -X<label>">;
 } // end of command line category.
 
 let CategoryName = "Verification" in {
diff --git a/clang/include/clang/InstallAPI/MachO.h b/clang/include/clang/InstallAPI/MachO.h
index 9da91a62e233..1ea544412f4c 100644
--- a/clang/include/clang/InstallAPI/MachO.h
+++ b/clang/include/clang/InstallAPI/MachO.h
@@ -45,6 +45,8 @@ using SimpleSymbol = llvm::MachO::SimpleSymbol;
 using FileType = llvm::MachO::FileType;
 using PackedVersion = llvm::MachO::PackedVersion;
 using PathSeq = llvm::MachO::PathSeq;
+using PlatformType = llvm::MachO::PlatformType;
+using PathToPlatformSeq = llvm::MachO::PathToPlatformSeq;
 using Target = llvm::MachO::Target;
 using TargetList = llvm::MachO::TargetList;
 
diff --git a/clang/test/InstallAPI/Inputs/Foundation/Foundation.framework/Modules/module.modulemap b/clang/test/InstallAPI/Inputs/Foundation/Foundation.framework/Modules/module.modulemap
new file mode 100644
index 000000000000..2bb688da1fa4
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Foundation/Foundation.framework/Modules/module.modulemap
@@ -0,0 +1,3 @@
+framework module Foundation [system] {
+    umbrella header "Foundation.h"
+}
diff --git a/clang/test/InstallAPI/Inputs/LibFoo/usr/include/foo.h b/clang/test/InstallAPI/Inputs/LibFoo/usr/include/foo.h
new file mode 100644
index 000000000000..e131da67ab7e
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/LibFoo/usr/include/foo.h
@@ -0,0 +1,15 @@
+#ifndef FOO_H
+#define FOO_H 
+#include <macro_defs.h> 
+
+#if defined(Foo) 
+  #define FOO "FooLib$" 
+#else 
+  #define FOO 
+#endif 
+
+#define __STRING(x)     #x
+#define PLATFORM_ALIAS(sym)	__asm("_" FOO __STRING(sym) DARWIN LINUX)
+extern int foo() PLATFORM_ALIAS(foo);
+
+#endif 
diff --git a/clang/test/InstallAPI/Inputs/LibFoo/usr/include/macro_defs.h b/clang/test/InstallAPI/Inputs/LibFoo/usr/include/macro_defs.h
new file mode 100644
index 000000000000..25566909da06
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/LibFoo/usr/include/macro_defs.h
@@ -0,0 +1,15 @@
+#ifndef MACRO_DEFS_H
+#define MACRO_DEFS_H 
+
+#if defined(NONDarwin) 
+  #define LINUX "$linux"
+  #define DARWIN 
+#elif defined(Darwin) 
+  #define LINUX 
+  #define DARWIN "$darwin" 
+#else 
+  #define LINUX 
+  #define DARWIN 
+#endif 
+
+#endif // MACRO_DEFS_H
diff --git a/clang/test/InstallAPI/Inputs/LibFoo/usr/include/public.h b/clang/test/InstallAPI/Inputs/LibFoo/usr/include/public.h
new file mode 100644
index 000000000000..fcc6d91adb77
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/LibFoo/usr/include/public.h
@@ -0,0 +1,9 @@
+#ifndef PUBLIC_H
+#define PUBLIC_H 
+#include <macro_defs.h>
+
+#define __STRING(x)     #x
+#define PLATFORM_ALIAS(sym)	__asm("_" __STRING(sym) DARWIN LINUX)
+extern int foo() PLATFORM_ALIAS(foo);
+
+#endif 
diff --git a/clang/test/InstallAPI/Inputs/Zippered/Zippered.framework/Headers/Zippered.h b/clang/test/InstallAPI/Inputs/Zippered/Zippered.framework/Headers/Zippered.h
new file mode 100644
index 000000000000..ec1b03318be1
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Zippered/Zippered.framework/Headers/Zippered.h
@@ -0,0 +1,20 @@
+#if !__is_target_environment(macabi)
+typedef int MyType;
+#else
+typedef float MyType;
+#endif
+
+extern MyType invalidAPI();
+
+#define OS_AVAILABLE(_target, _availability)                                   \
+  __attribute__((availability(_target, _availability)))
+extern int macOSAPI() OS_AVAILABLE(macos, introduced=10.14) OS_AVAILABLE(ios, unavailable);
+extern int iOSAPI() OS_AVAILABLE(ios, introduced=12.0) OS_AVAILABLE(macos, unavailable);
+extern int commonAPI() OS_AVAILABLE(macos, introduced=10.14) OS_AVAILABLE(ios, introduced=12.0);
+
+extern int obsoletedMacOSAPI() OS_AVAILABLE(macos, obsoleted=10.14) OS_AVAILABLE(ios, unavailable);
+
+#if !__is_target_environment(macabi)
+extern int macOSAPI2() OS_AVAILABLE(macos, introduced = 10.14)
+    OS_AVAILABLE(ios, unavailable);
+#endif
diff --git a/clang/test/InstallAPI/Inputs/Zippered/Zippered.framework/PrivateHeaders/Zippered_Private.h b/clang/test/InstallAPI/Inputs/Zippered/Zippered.framework/PrivateHeaders/Zippered_Private.h
new file mode 100644
index 000000000000..2182a17275cb
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Zippered/Zippered.framework/PrivateHeaders/Zippered_Private.h
@@ -0,0 +1,9 @@
+#if __is_target_environment(macabi)
+extern int a;
+@class UIImage;
+UIImage *image;
+#else
+extern long a;
+@class NSImage;
+NSImage *image;
+#endif
diff --git a/clang/test/InstallAPI/Inputs/Zippered/Zippered.tbd b/clang/test/InstallAPI/Inputs/Zippered/Zippered.tbd
new file mode 100644
index 000000000000..6ceb589ff0cd
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Zippered/Zippered.tbd
@@ -0,0 +1,47 @@
+{
+  "main_library": {
+    "exported_symbols": [
+      {
+        "data": {
+          "global": [
+            "_image", "_a"
+          ]
+        },
+        "text": {
+          "global": [
+            "_invalidAPI", "_commonAPI"
+          ]
+        }
+      },
+      {
+        "targets": [ "x86_64-maccatalyst" ],
+        "text": {
+          "global": [ "_iOSAPI"]
+        }
+      },
+      {
+        "targets": [ "x86_64-macos" ],
+        "text": {
+          "global": [ "_macOSAPI", "_macOSAPI2" ]
+        }
+      }
+    ],
+    "flags": [
+      {
+        "attributes": ["not_app_extension_safe"]
+      }
+    ],
+    "install_names": [
+      {"name": "/System/Library/Frameworks/Zippered.framework/Versions/A/Zippered"}
+    ],
+    "target_info": [
+      {
+        "min_deployment": "13", "target": "x86_64-macos"
+      },
+      {
+        "min_deployment": "16", "target": "x86_64-maccatalyst"
+      }
+    ]
+  },
+  "tapi_tbd_version": 5
+}
diff --git a/clang/test/InstallAPI/Inputs/Zippered/Zippered.yaml b/clang/test/InstallAPI/Inputs/Zippered/Zippered.yaml
new file mode 100644
index 000000000000..284cd46416fd
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Zippered/Zippered.yaml
@@ -0,0 +1,383 @@
+# Generated from: 
+# xcrun -sdk macosx clang --target=x86_64-apple-macos13 --target-variant=x86_64-apple-ios16-macabi
+# -dynamiclib 
+#
+#import "Zippered.h"
+#import "Zippered_Private.h"
+# MyType invalidAPI() { return 0; }
+# int macOSAPI() { return 0; }
+# int macOSAPI2() { return 0; }
+# int iOSAPI() { return 0; }
+# int commonAPI() { return 0; }
+# int obsoletedMacOSAPI() { return 0; }
+# 
+# #if __is_target_environment(macabi)
+# int a = 0;
+# UIImage *image = 0;
+# #else
+# long a = 0;
+# NSImage *image = 0;
+# #endif
+
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x1000007
+  cpusubtype:      0x3
+  filetype:        0x6
+  ncmds:           15
+  sizeofcmds:      1584
+  flags:           0x100085
+  reserved:        0x0
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         312
+    segname:         __TEXT
+    vmaddr:          0
+    vmsize:          12288
+    fileoff:         0
+    filesize:        12288
+    maxprot:         5
+    initprot:        5
+    nsects:          3
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x1090
+        size:            88
+        offset:          0x1090
+        align:           4
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         554889E531C05DC30F1F840000000000554889E531C05DC30F1F840000000000554889E531C05DC30F1F840000000000554889E531C05DC30F1F840000000000554889E531C05DC30F1F840000000000554889E531C05DC3
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x10E8
+        size:            4152
+        offset:          0x10E8
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         010000001C000000010000002000000000000000200000000200000000000001901000003800000038000000E81000000000000038000000030000000C0001001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+      - sectname:        __eh_frame
+        segname:         __TEXT
+        addr:            0x2120
+        size:            24
+        offset:          0x2120
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x6000000B
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         1400000000000000017A520001781001100C070890010000
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         152
+    segname:         __DATA_CONST
+    vmaddr:          12288
+    vmsize:          4096
+    fileoff:         12288
+    filesize:        4096
+    maxprot:         3
+    initprot:        3
+    nsects:          1
+    flags:           16
+    Sections:
+      - sectname:        __objc_imageinfo
+        segname:         __DATA_CONST
+        addr:            0x3000
+        size:            8
+        offset:          0x3000
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         '0000000040000000'
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         152
+    segname:         __DATA
+    vmaddr:          16384
+    vmsize:          4096
+    fileoff:         16384
+    filesize:        0
+    maxprot:         3
+    initprot:        3
+    nsects:          1
+    flags:           0
+    Sections:
+      - sectname:        __common
+        segname:         __DATA
+        addr:            0x4000
+        size:            16
+        offset:          0x0
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x1
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          20480
+    vmsize:          384
+    fileoff:         16384
+    filesize:        384
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      0
+    rebase_size:     0
+    bind_off:        0
+    bind_size:       0
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   0
+    lazy_bind_size:  0
+    export_off:      16384
+    export_size:     128
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          16520
+    nsyms:           9
+    stroff:          16664
+    strsize:         104
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      8
+    iundefsym:       8
+    nundefsym:       1
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_ID_DYLIB
+    cmdsize:         96
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 65536
+      compatibility_version: 65536
+    Content:         '/System/Library/Frameworks/Zippered.framework/Versions/A/Zippered'
+    ZeroPadBytes:    7
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            4C4C44B0-5555-3144-A126-166C8AB77CD1
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        1
+    minos:           851968
+    sdk:             983040
+    ntools:          1
+    Tools:
+      - tool:            4
+        version:         1245184
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        6
+    minos:           1048576
+    sdk:             1048576
+    ntools:          1
+    Tools:
+      - tool:            4
+        version:         1245184
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 14942208
+      compatibility_version: 65536
+    Content:         '/usr/lib/libobjc.A.dylib'
+    ZeroPadBytes:    8
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 88539136
+      compatibility_version: 65536
+    Content:         '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    6
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         16512
+    datasize:        8
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         16520
+    datasize:        0
+LinkEditData:
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0
+    Address:         0x0
+    Other:           0x0
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      5
+        Name:            _
+        Flags:           0x0
+        Address:         0x0
+        Other:           0x0
+        ImportName:      ''
+        Children:
+          - TerminalSize:    4
+            NodeOffset:      53
+            Name:            a
+            Flags:           0x0
+            Address:         0x4000
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    0
+            NodeOffset:      59
+            Name:            i
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    3
+                NodeOffset:      85
+                Name:            OSAPI
+                Flags:           0x0
+                Address:         0x10C0
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    4
+                NodeOffset:      90
+                Name:            mage
+                Flags:           0x0
+                Address:         0x4008
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    3
+                NodeOffset:      96
+                Name:            nvalidAPI
+                Flags:           0x0
+                Address:         0x1090
+                Other:           0x0
+                ImportName:      ''
+          - TerminalSize:    3
+            NodeOffset:      101
+            Name:            obsoletedMacOSAPI
+            Flags:           0x0
+            Address:         0x10E0
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    3
+            NodeOffset:      106
+            Name:            macOSAPI
+            Flags:           0x0
+            Address:         0x10A0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    3
+                NodeOffset:      114
+                Name:            '2'
+                Flags:           0x0
+                Address:         0x10B0
+                Other:           0x0
+                ImportName:      ''
+          - TerminalSize:    3
+            NodeOffset:      119
+            Name:            commonAPI
+            Flags:           0x0
+            Address:         0x10D0
+            Other:           0x0
+            ImportName:      ''
+  NameList:
+    - n_strx:          2
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4240
+    - n_strx:          14
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4256
+    - n_strx:          24
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4272
+    - n_strx:          35
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4288
+    - n_strx:          43
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4304
+    - n_strx:          54
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4320
+    - n_strx:          73
+      n_type:          0xF
+      n_sect:          5
+      n_desc:          0
+      n_value:         16384
+    - n_strx:          76
+      n_type:          0xF
+      n_sect:          5
+      n_desc:          0
+      n_value:         16392
+    - n_strx:          83
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          512
+      n_value:         0
+  StringTable:
+    - ' '
+    - _invalidAPI
+    - _macOSAPI
+    - _macOSAPI2
+    - _iOSAPI
+    - _commonAPI
+    - _obsoletedMacOSAPI
+    - _a
+    - _image
+    - dyld_stub_binder
+    - ''
+    - ''
+    - ''
+    - ''
+  FunctionStarts:  [ 0x1090, 0x10A0, 0x10B0, 0x10C0, 0x10D0, 0x10E0 ]
+...
diff --git a/clang/test/InstallAPI/exclusive-passes-2.test b/clang/test/InstallAPI/exclusive-passes-2.test
new file mode 100644
index 000000000000..3e7a6d777d5a
--- /dev/null
+++ b/clang/test/InstallAPI/exclusive-passes-2.test
@@ -0,0 +1,58 @@
+; RUN: rm -rf %t
+; RUN: split-file %s %t
+
+// All passes should include Foo macro definition.
+; RUN: clang-installapi -target arm64-apple-macos12 \
+; RUN: -install_name @rpath/libfoo.dylib -current_version 1 \
+; RUN: -compatibility_version 1 \
+; RUN: -I%S/Inputs/LibFoo/usr/include -dynamiclib \
+; RUN: -extra-public-header %S/Inputs/LibFoo/usr/include/foo.h \
+; RUN: -o %t/output.tbd \
+; RUN: -DFoo -XApple -DDarwin=1 -XElf -DNONDarwin=1 2>&1 | FileCheck -allow-empty %s 
+; RUN: llvm-readtapi --compare %t/output.tbd %t/expected.tbd 2>&1 | FileCheck -allow-empty %s
+
+; CHECK-NOT: error
+; CHECK-NOT: warning
+
+
+//--- expected.tbd
+{
+  "main_library": {
+    "exported_symbols": [
+      {
+        "text": {
+          "global": [
+            "_FooLib$foo$darwin",
+            "_FooLib$foo$linux",
+            "_FooLib$foo"
+          ]
+        }
+      }
+    ],
+    "flags": [
+      {
+        "attributes": [
+          "not_app_extension_safe"
+        ]
+      }
+    ],
+    "install_names": [
+      {
+        "name": "@rpath/libfoo.dylib"
+      }
+    ],
+    "target_info": [
+      {
+        "min_deployment": "12",
+        "target": "arm64-macos"
+      }
+    ]
+  },
+  "tapi_tbd_version": 5
+}
+
+//--- options.json
+{
+  "Apple" : ["-DDarwin=1"],
+  "Elf" : ["-DNONDarwin"]
+}
diff --git a/clang/test/InstallAPI/exclusive-passes-platform.test b/clang/test/InstallAPI/exclusive-passes-platform.test
new file mode 100644
index 000000000000..c5a79cf9a30d
--- /dev/null
+++ b/clang/test/InstallAPI/exclusive-passes-platform.test
@@ -0,0 +1,286 @@
+; RUN: rm -rf %t
+; RUN: split-file %s %t
+; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json
+; RUN: yaml2obj %t/Xplatform.yaml -o=%t/Xplatform
+
+// Check that in zippered mode, a successful pass runs in both macos & maccatalyst.
+; RUN: clang-installapi %t/inputs.json \
+; RUN: --target=x86_64-apple-macos10.15 -darwin-target-variant x86_64-apple-ios13.1-macabi \
+; RUN: -Xplatform_ios-macabi -iframework %t/System/iOSSupport/System/Library/Frameworks \
+; RUN: -install_name /System/Library/Frameworks/Xplatform.framework/Versions/A/Xplatform \
+; RUN: -fdefine-target-os-macros --verify-against=%t/Xplatform --verify-mode=Pedantic \
+; RUN: -o Xplatform.tbd  -F%t/Frameworks \
+: RUN: -current_version 1 -compatibility_version 1 2>&1 | FileCheck --allow-empty %s\
+; RUN: --implicit-check-not warning: --implicit-check-not error:
+
+// A missing header error should be invoked in macos pass because it wasn't given the needed search path.
+; RUN: mv %t/Xplatform-macosx.h %t/Frameworks/Xplatform.framework/Headers/Xplatform.h
+
+; RUN: not clang-installapi %t/inputs.json \
+; RUN: --target=x86_64-apple-macos10.15 -darwin-target-variant x86_64-apple-ios13.1-macabi \
+; RUN: -Xplatform_ios-macabi -iframework %t/System/iOSSupport/System/Library/Frameworks \
+; RUN: -install_name /System/Library/Frameworks/Xplatform.framework/Versions/A/Xplatform \
+; RUN: -fdefine-target-os-macros --verify-against=%t/Xplatform --verify-mode=Pedantic \
+; RUN: -o Xplatform.tbd  -F%t/Frameworks \
+: RUN: -current_version 1 -compatibility_version 1 2>&1 | FileCheck -check-prefix=MACOSFAIL %s 
+
+; MACOSFAIL: fatal error: 'IOSMac/IOSMac.h' file not found
+
+;--- Frameworks/Xplatform.framework/Headers/Xplatform.h
+#if TARGET_OS_MACCATALYST
+#include <IOSMac/IOSMac.h>
+#endif
+
+inline int foo() {
+  int x = 1;
+#if TARGET_OS_MACCATALYST
+  x += iOSAPI();
+#endif
+  return x;
+}
+
+extern int bar();
+
+;--- Xplatform-macosx.h
+#include <IOSMac/IOSMac.h>
+inline int foo() {
+  int x = 1;
+  return x;
+}
+
+extern int bar();
+
+;--- System/iOSSupport/System/Library/Frameworks/IOSMac.framework/Headers/IOSMac.h
+extern int iOSAPI();
+
+;--- Xplatform.yaml
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x1000007
+  cpusubtype:      0x3
+  filetype:        0x6
+  ncmds:           16
+  sizeofcmds:      968
+  flags:           0x100085
+  reserved:        0x0
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         232
+    segname:         __TEXT
+    vmaddr:          0
+    vmsize:          32768
+    fileoff:         0
+    filesize:        32768
+    maxprot:         5
+    initprot:        5
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x4FAD
+        size:            11
+        offset:          0x4FAD
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         554889E5B8010000005DC3
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x4FB8
+        size:            72
+        offset:          0x4FB8
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         010000001C000000000000001C000000000000001C00000002000000AD4F00003400000034000000B94F00000000000034000000030000000C000100100001000000000000000001
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         152
+    segname:         __DATA
+    vmaddr:          32768
+    vmsize:          16384
+    fileoff:         32768
+    filesize:        16384
+    maxprot:         3
+    initprot:        3
+    nsects:          1
+    flags:           0
+    Sections:
+      - sectname:        __objc_imageinfo
+        segname:         __DATA
+        addr:            0x8000
+        size:            8
+        offset:          0x8000
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         '0000000040000000'
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          49152
+    vmsize:          16384
+    fileoff:         49152
+    filesize:        88
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_ID_DYLIB
+    cmdsize:         96
+    dylib:
+      name:            24
+      timestamp:       1
+      current_version: 65536
+      compatibility_version: 65536
+    Content:   '/System/Library/Frameworks/Xplatform.framework/Versions/A/Xplatform'
+    ZeroPadBytes:    5
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      0
+    rebase_size:     0
+    bind_off:        0
+    bind_size:       0
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   0
+    lazy_bind_size:  0
+    export_off:      49152
+    export_size:     16
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          49184
+    nsyms:           2
+    stroff:          49216
+    strsize:         24
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      1
+    iundefsym:       1
+    nundefsym:       1
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            4AA4F126-BD02-359C-B3EF-E53AD399B590
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        1
+    minos:           659200
+    sdk:             721152
+    ntools:          1
+    Tools:
+      - tool:            3
+        version:         46008832
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        6
+    minos:           0x00d0100 
+    sdk:             851968
+    ntools:          1
+    Tools:
+      - tool:            3
+        version:         46008832
+  - cmd:             LC_SOURCE_VERSION
+    cmdsize:         16
+    version:         0
+  - cmd:             LC_SEGMENT_SPLIT_INFO
+    cmdsize:         16
+    dataoff:         49168
+    datasize:        8
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       2
+      current_version: 14942208
+      compatibility_version: 65536
+    Content:   '/usr/lib/libobjc.A.dylib'
+    ZeroPadBytes:    8
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       2
+      current_version: 84687873
+      compatibility_version: 65536
+    Content:   '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    6
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         49176
+    datasize:        8
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         49184
+    datasize:        0
+LinkEditData:
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0
+    Address:         0x0
+    Other:           0x0
+    ImportName:      ''
+    Children:
+      - TerminalSize:    4
+        NodeOffset:      8
+        Name:            _bar
+        Flags:           0x0
+        Address:         0x4FAD
+        Other:           0x0
+        ImportName:      ''
+  NameList:
+    - n_strx:          2
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         20397
+    - n_strx:          7
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          512
+      n_value:         0
+  StringTable:
+    - ' '
+    - _bar
+    - dyld_stub_binder
+...
+
+;--- inputs.json.in
+{
+  "headers": [ 
+  {
+    "path" : "DSTROOT/Frameworks/Xplatform.framework/Headers/Xplatform.h",
+    "type" : "public"
+  }
+  ],
+  "version": "3"
+}
diff --git a/clang/test/InstallAPI/exclusive-passes-zippered.test b/clang/test/InstallAPI/exclusive-passes-zippered.test
new file mode 100644
index 000000000000..73d9361d553e
--- /dev/null
+++ b/clang/test/InstallAPI/exclusive-passes-zippered.test
@@ -0,0 +1,56 @@
+; RUN: rm -rf %t
+; RUN: split-file %s %t
+; RUN: mkdir -p %t/Frameworks/
+; RUN: cp -r %S/Inputs/Zippered/Zippered.framework %t/Frameworks/
+; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json
+; RUN: yaml2obj %S/Inputs/Zippered/Zippered.yaml -o %t/Frameworks/Zippered.framework/Zippered
+
+; RUN: clang-installapi \
+; RUN: --target=x86_64-apple-macos13 -darwin-target-variant x86_64-apple-ios16-macabi \
+; RUN: -install_name /System/Library/Frameworks/Zippered.framework/Versions/A/Zippered \
+; RUN: -current_version 1 -compatibility_version 1 %t/inputs.json \
+; RUN: --verify-against=%t/Frameworks/Zippered.framework/Zippered \
+; RUN: -isysroot %S/Inputs/MacOSX13.0.sdk -F%t/Frameworks \
+; RUN: --verify-mode=Pedantic -o %t/Zippered.tbd \
+; RUN: --extra-private-header=%t/Extra.h 2>&1 | FileCheck -allow-empty %s \
+; RUN: --implicit-check-not warning: --implicit-check-not error:
+; RUN: llvm-readtapi -compare %t/Zippered.tbd %S/Inputs/Zippered/Zippered.tbd
+
+// Flag extra symbols exposed by macro definition.
+; RUN: not clang-installapi \
+; RUN: --target=x86_64-apple-macos13 -darwin-target-variant x86_64-apple-ios16-macabi \
+; RUN: -install_name /System/Library/Frameworks/Zippered.framework/Versions/A/Zippered \
+; RUN: -current_version 1 -compatibility_version 1 %t/inputs.json \
+; RUN: -isysroot %S/Inputs/MacOSX13.0.sdk -F%t/Frameworks \
+; RUN: --verify-mode=Pedantic -o %t/Zippered.tbd -v \
+; RUN: --verify-against=%t/Frameworks/Zippered.framework/Zippered \
+; RUN: --extra-private-header=%t/Extra.h -XExtra -DExtra 2>&1 | FileCheck %s --check-prefix=MACRO_DEF
+
+; MACRO_DEF-COUNT-2: "-D" "Extra"
+; MACRO_DEF: violations found for x86_64-apple-macos13
+; MACRO_DEF: Extra.h:5:12: error: declaration has external linkage, but dynamic library doesn't have symbol 'foo$bar'
+; MACRO_DEF-COUNT-2: "-D" "Extra"
+; MACRO_DEF-NOT: violations found for x86_64-apple-ios16-macabi
+
+;--- Extra.h
+#define __STRING(x)     #x
+
+#if defined(Extra)
+  #define MACRO_DEF "$bar"
+extern int foo() __asm("_" __STRING(foo) MACRO_DEF) __attribute__((availability(ios, unavailable)));
+#endif
+
+;--- inputs.json.in
+{
+  "headers": [ 
+  {
+    "path" : "DSTROOT/Frameworks/Zippered.framework/PrivateHeaders/Zippered_Private.h",
+    "type" : "private"
+  }, 
+  {
+    "path" : "DSTROOT/Frameworks/Zippered.framework/Headers/Zippered.h",
+    "type" : "public"
+  }
+  ],
+  "version": "3"
+}
diff --git a/clang/test/InstallAPI/exclusive-passes.test b/clang/test/InstallAPI/exclusive-passes.test
new file mode 100644
index 000000000000..29b0fc3d7a2a
--- /dev/null
+++ b/clang/test/InstallAPI/exclusive-passes.test
@@ -0,0 +1,54 @@
+; RUN: rm -rf %t
+; RUN: split-file %s %t
+
+; RUN: clang-installapi \
+; RUN: -target arm64-apple-macos12 -install_name @rpath/libfoo.dylib \
+; RUN: -current_version 1 -compatibility_version 1 \
+; RUN: -XApple -DDarwin=1 -XElf -DNONDarwin=1 \
+; RUN: -I%S/Inputs/LibFoo/usr/include -dynamiclib \
+; RUN: -extra-public-header %S/Inputs/LibFoo/usr/include/public.h \
+; RUN: -o %t/output.tbd -v 2>&1 | FileCheck %s --check-prefix=INSTALLAPI
+; RUN: llvm-readtapi --compare %t/output.tbd %t/expected.tbd 2>&1 | FileCheck -allow-empty %s
+
+; CHECK-NOT: error
+; CHECK-NOT: warning
+
+; INSTALLAPI: Public Headers:
+; INSTALLAPI: Apple Public Headers:
+; INSTALLAPI: Elf Public Headers:
+
+;--- expected.tbd
+{
+  "main_library": {
+    "exported_symbols": [
+      {
+        "text": {
+          "global": [
+            "_foo$darwin",
+            "_foo$linux",
+            "_foo"
+          ]
+        }
+      }
+    ],
+    "flags": [
+      {
+        "attributes": [
+          "not_app_extension_safe"
+        ]
+      }
+    ],
+    "install_names": [
+      {
+        "name": "@rpath/libfoo.dylib"
+      }
+    ],
+    "target_info": [
+      {
+        "min_deployment": "12",
+        "target": "arm64-macos"
+      }
+    ]
+  },
+  "tapi_tbd_version": 5
+}
diff --git a/clang/test/InstallAPI/invalid-exclusive-passes.test b/clang/test/InstallAPI/invalid-exclusive-passes.test
new file mode 100644
index 000000000000..c23c918f0bfb
--- /dev/null
+++ b/clang/test/InstallAPI/invalid-exclusive-passes.test
@@ -0,0 +1,37 @@
+; RUN: rm -rf %t 
+; RUN: split-file %s %t
+
+// Validate arguments not allowed with -X
+; RUN: not clang-installapi \
+; RUN: -target arm64-apple-macos12 \
+; RUN: -install_name @rpath/libfoo.dylib \
+; RUN: -current_version 1 -compatibility_version 1 \
+; RUN: -XApple -I/fake/path -I%t %t/inputs.json \
+; RUN: -dynamiclib -o %t/output.tbd  2>&1 | FileCheck %s --check-prefix=INVALID_OPT
+; INVALID_OPT: error: invalid argument '-XApple' not allowed with '-I/fake/path'
+
+// Validate reserved labels.
+; RUN: not clang-installapi \
+; RUN: -target arm64-apple-macos12 \
+; RUN: -install_name @rpath/libfoo.dylib \
+; RUN: -current_version 1 -compatibility_version 1 \
+; RUN: -XApple -DDarwin=1 -XElf -DNONDarwin=1 \
+; RUN: -I%t -dynamiclib -o %t/output.tbd %t/inputs.json \
+; RUN: -XPrivate -DInvalid=1 2>&1 | FileCheck %s --check-prefix=INVALID_LABELS
+; INVALID_LABELS: error: label 'Private' is reserved: use a different label name for -X<label>
+
+// Validate arguments not allowed with -Xproject
+; RUN: not clang-installapi \
+; RUN: -target arm64-apple-macos12 \
+; RUN: -install_name @rpath/libfoo.dylib \
+; RUN: -current_version 1 -compatibility_version 1 \
+; RUN: -Xproject -fprofile-instr-generate \
+; RUN: %t/inputs.json -I%t -dynamiclib \
+; RUN: -o %t/output.tbd 2>&1 | FileCheck %s --check-prefix=INVALID_PROJECT_OPT
+; INVALID_PROJECT_OPT: error: invalid argument '-Xproject' not allowed with '-fprofile-instr-generate'
+
+;--- inputs.json
+{
+  "headers": [ ],
+  "version": "3"
+}
diff --git a/clang/test/InstallAPI/project-header-only-args.test b/clang/test/InstallAPI/project-header-only-args.test
new file mode 100644
index 000000000000..7147c83b0f5d
--- /dev/null
+++ b/clang/test/InstallAPI/project-header-only-args.test
@@ -0,0 +1,81 @@
+; RUN: rm -rf %t
+; RUN: split-file %s %t
+; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json
+
+; RUN: clang-installapi \
+; RUN: -target arm64-apple-macos12 -install_name @rpath/libfoo.dylib \
+; RUN: -current_version 1 -compatibility_version 1 \
+; RUN: -Xproject -fmodules -I%t/usr/include \
+; RUN: -F %S/Inputs/Foundation/ \
+; RUN: -exclude-public-header %t/usr/include/public.h \
+; RUN: -extra-project-header %t/project.h -I%t -dynamiclib \
+; RUN: %t/inputs.json \
+; RUN: -o %t/output.tbd 2>&1 | FileCheck %s --allow-empty
+; RUN: llvm-readtapi --compare %t/output.tbd %t/expected.tbd 2>&1 | FileCheck %s --allow-empty
+
+; RUN: not clang-installapi \
+; RUN: -target arm64-apple-macos12 -install_name @rpath/libfoo.dylib \
+; RUN: -current_version 1 -compatibility_version 1 \
+; RUN: -Xproject -fmodules -I%t/usr/include \
+; RUN: -extra-project-header %t/project.h \
+; RUN: -F %S/Inputs/Foundation/ \
+; RUN: %t/inputs.json \
+; RUN: -I%t -dynamiclib -o %t/output.tbd 2>&1 | FileCheck %s --check-prefix=PUBLIC
+
+; CHECK-NOT: error
+; CHECK-NOT: warning
+
+; PUBLIC: public.h:1:1: error: use of '@import' when modules are disabled
+; PUBLIC-NEXT: @import Foundation;
+
+//--- usr/include/public.h
+@import Foundation;
+extern int foo();
+
+//--- project.h
+@import Foundation;
+extern int bar();
+
+//--- expected.tbd
+{
+  "main_library": {
+    "exported_symbols": [
+      {
+        "text": {
+          "global": [
+            "_bar"
+          ]
+        }
+      }
+    ],
+    "flags": [
+      {
+        "attributes": [
+          "not_app_extension_safe"
+        ]
+      }
+    ],
+    "install_names": [
+      {
+        "name": "@rpath/libfoo.dylib"
+      }
+    ],
+    "target_info": [
+      {
+        "min_deployment": "12",
+        "target": "arm64-macos"
+      }
+    ]
+  },
+  "tapi_tbd_version": 5
+}
+
+;--- inputs.json.in
+{
+  "headers": [ {
+    "path" : "DSTROOT/usr/include/public.h",
+    "type" : "public"
+  }
+  ],
+  "version": "3"
+}
diff --git a/clang/tools/clang-installapi/ClangInstallAPI.cpp b/clang/tools/clang-installapi/ClangInstallAPI.cpp
index add28ab4fcda..308e5285e325 100644
--- a/clang/tools/clang-installapi/ClangInstallAPI.cpp
+++ b/clang/tools/clang-installapi/ClangInstallAPI.cpp
@@ -39,7 +39,7 @@ using namespace clang::driver::options;
 using namespace llvm::opt;
 using namespace llvm::MachO;
 
-static bool runFrontend(StringRef ProgName, bool Verbose,
+static bool runFrontend(StringRef ProgName, Twine Label, bool Verbose,
                         InstallAPIContext &Ctx,
                         llvm::vfs::InMemoryFileSystem *FS,
                         const ArrayRef<std::string> InitialArgs) {
@@ -50,7 +50,7 @@ static bool runFrontend(StringRef ProgName, bool Verbose,
     return true;
 
   if (Verbose)
-    llvm::errs() << getName(Ctx.Type) << " Headers:\n"
+    llvm::errs() << Label << " Headers:\n"
                  << ProcessedInput->getBuffer() << "\n\n";
 
   std::string InputFile = ProcessedInput->getBufferIdentifier().str();
@@ -65,7 +65,6 @@ static bool runFrontend(StringRef ProgName, bool Verbose,
   // Create & run invocation.
   clang::tooling::ToolInvocation Invocation(
       std::move(Args), std::make_unique<InstallAPIAction>(Ctx), Ctx.FM);
-
   return Invocation.run();
 }
 
@@ -127,10 +126,23 @@ static bool run(ArrayRef<const char *> Args, const char *ProgName) {
     Ctx.Slice = std::make_shared<FrontendRecordsSlice>(Trip);
     for (const HeaderType Type :
          {HeaderType::Public, HeaderType::Private, HeaderType::Project}) {
+      std::vector<std::string> ArgStrings = Opts.getClangFrontendArgs();
+      Opts.addConditionalCC1Args(ArgStrings, Trip, Type);
       Ctx.Type = Type;
-      if (!runFrontend(ProgName, Opts.DriverOpts.Verbose, Ctx,
-                       InMemoryFileSystem.get(), Opts.getClangFrontendArgs()))
+      StringRef HeaderLabel = getName(Ctx.Type);
+      if (!runFrontend(ProgName, HeaderLabel, Opts.DriverOpts.Verbose, Ctx,
+                       InMemoryFileSystem.get(), ArgStrings))
         return EXIT_FAILURE;
+
+      // Run extra passes for unique compiler arguments.
+      for (const auto &[Label, ExtraArgs] : Opts.FEOpts.UniqueArgs) {
+        std::vector<std::string> FinalArguments = ArgStrings;
+        llvm::append_range(FinalArguments, ExtraArgs);
+        if (!runFrontend(ProgName, Label + " " + HeaderLabel,
+                         Opts.DriverOpts.Verbose, Ctx, InMemoryFileSystem.get(),
+                         FinalArguments))
+          return EXIT_FAILURE;
+      }
     }
     FrontendRecords.emplace_back(std::move(Ctx.Slice));
   }
diff --git a/clang/tools/clang-installapi/InstallAPIOpts.td b/clang/tools/clang-installapi/InstallAPIOpts.td
index 8b1998c280dd..a95a7a80a9d2 100644
--- a/clang/tools/clang-installapi/InstallAPIOpts.td
+++ b/clang/tools/clang-installapi/InstallAPIOpts.td
@@ -90,6 +90,15 @@ def project_umbrella_header : Separate<["-"], "project-umbrella-header">,
 def project_umbrella_header_EQ : Joined<["--"], "project-umbrella-header=">,
   Alias<project_umbrella_header>;
 
+//
+/// X<label> overrides.
+//
+def Xplatform__ : Joined<["-"], "Xplatform_">;
+def Xproject : Joined<["-"], "Xproject">;
+def X__ : Joined<["-"], "X">,
+  HelpText<"Pass <arg> to run unique clang invocation identified as <label>">, 
+  MetaVarName<"<label> <arg>">;
+
 //
 /// Overidden clang options for different behavior.
 //
@@ -108,4 +117,25 @@ def reexport_library : Separate<["-"], "reexport_library">, MetaVarName<"<path>"
 def reexport_framework : Separate<["-"], "reexport_framework">,
   HelpText<"Re-export the specified framework">;
 
+// Xproject supported options.
+def fobjc_arc : Flag<["-"], "fobjc-arc">,
+  HelpText<"Synthesize retain and release calls for Objective-C pointers">;
+def include_ : JoinedOrSeparate<["-", "--"], "include">,
+  MetaVarName<"<file>">, HelpText<"Include file before parsing, can only be used with -Xproject">;
+def fvisibility_EQ : Joined<["-"], "fvisibility=">,
+  HelpText<"Set the default symbol visibility for all global declarations">;
+def fmodules : Flag <["-"], "fmodules">,
+  HelpText<"Enable the 'modules' language feature">;
+def fmodules_cache_path : Joined<["-"], "fmodules-cache-path=">,
+  MetaVarName<"<directory>">,
+  HelpText<"Specify the module cache path">;
+
+// Xplatform supported options.
+def iframework : JoinedOrSeparate<["-"], "iframework">,
+  HelpText<"Add directory to SYSTEM framework search path">;
+
+// X<label> prefixes supported options.
+def D : JoinedOrSeparate<["-"], "D">, HelpText<"Define macro">;
+def U : JoinedOrSeparate<["-"], "U">, HelpText<"Undefine macro">;
+
 
diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index 21f04a291b2f..5396ad23620b 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -189,12 +189,120 @@ bool Options::processDriverOptions(InputArgList &Args) {
 
 bool Options::processInstallAPIXOptions(InputArgList &Args) {
   for (arg_iterator It = Args.begin(), End = Args.end(); It != End; ++It) {
-    if ((*It)->getOption().matches(OPT_Xarch__)) {
+    Arg *A = *It;
+    if (A->getOption().matches(OPT_Xarch__)) {
       if (!processXarchOption(Args, It))
         return false;
+      continue;
+    } else if (A->getOption().matches(OPT_Xplatform__)) {
+      if (!processXplatformOption(Args, It))
+        return false;
+      continue;
+    } else if (A->getOption().matches(OPT_Xproject)) {
+      if (!processXprojectOption(Args, It))
+        return false;
+      continue;
+    } else if (!A->getOption().matches(OPT_X__))
+      continue;
+
+    // Handle any user defined labels.
+    const StringRef Label = A->getValue(0);
+
+    // Ban "public" and "private" labels.
+    if ((Label.lower() == "public") || (Label.lower() == "private")) {
+      Diags->Report(diag::err_invalid_label) << Label;
+      return false;
     }
+
+    auto NextIt = std::next(It);
+    if (NextIt == End) {
+      Diags->Report(clang::diag::err_drv_missing_argument)
+          << A->getAsString(Args) << 1;
+      return false;
+    }
+    Arg *NextA = *NextIt;
+    switch ((ID)NextA->getOption().getID()) {
+    case OPT_D:
+    case OPT_U:
+      break;
+    default:
+      Diags->Report(clang::diag::err_drv_argument_not_allowed_with)
+          << A->getAsString(Args) << NextA->getAsString(Args);
+      return false;
+    }
+    const StringRef ASpelling = NextA->getSpelling();
+    const auto &AValues = NextA->getValues();
+    if (AValues.empty())
+      FEOpts.UniqueArgs[Label].emplace_back(ASpelling.str());
+    else
+      for (const StringRef Val : AValues)
+        FEOpts.UniqueArgs[Label].emplace_back((ASpelling + Val).str());
+
+    A->claim();
+    NextA->claim();
   }
-  // TODO: Add support for the all of the X* options installapi supports.
+
+  return true;
+}
+
+bool Options::processXplatformOption(InputArgList &Args, arg_iterator Curr) {
+  Arg *A = *Curr;
+
+  PlatformType Platform = getPlatformFromName(A->getValue(0));
+  if (Platform == PLATFORM_UNKNOWN) {
+    Diags->Report(diag::err_unsupported_os)
+        << getPlatformName(Platform) << A->getAsString(Args);
+    return false;
+  }
+  auto NextIt = std::next(Curr);
+  if (NextIt == Args.end()) {
+    Diags->Report(diag::err_drv_missing_argument) << A->getAsString(Args) << 1;
+    return false;
+  }
+
+  Arg *NextA = *NextIt;
+  switch ((ID)NextA->getOption().getID()) {
+  case OPT_iframework:
+    FEOpts.SystemFwkPaths.emplace_back(NextA->getValue(), Platform);
+    break;
+  default:
+    Diags->Report(diag::err_drv_invalid_argument_to_option)
+        << A->getAsString(Args) << NextA->getAsString(Args);
+    return false;
+  }
+
+  A->claim();
+  NextA->claim();
+
+  return true;
+}
+
+bool Options::processXprojectOption(InputArgList &Args, arg_iterator Curr) {
+  Arg *A = *Curr;
+  auto NextIt = std::next(Curr);
+  if (NextIt == Args.end()) {
+    Diags->Report(diag::err_drv_missing_argument) << A->getAsString(Args) << 1;
+    return false;
+  }
+
+  Arg *NextA = *NextIt;
+  switch ((ID)NextA->getOption().getID()) {
+  case OPT_fobjc_arc:
+  case OPT_fmodules:
+  case OPT_fmodules_cache_path:
+  case OPT_include_:
+  case OPT_fvisibility_EQ:
+    break;
+  default:
+    Diags->Report(diag::err_drv_argument_not_allowed_with)
+        << A->getAsString(Args) << NextA->getAsString(Args);
+    return false;
+  }
+
+  ProjectLevelArgs.push_back(NextA->getSpelling().str());
+  llvm::copy(NextA->getValues(), std::back_inserter(ProjectLevelArgs));
+  A->claim();
+  NextA->claim();
 
   return true;
 }
@@ -333,10 +441,10 @@ bool Options::processFrontendOptions(InputArgList &Args) {
     }
   }
 
-  // Capture system frameworks.
-  // TODO: Support passing framework paths per platform.
+  // Capture system frameworks for all platforms.
   for (const Arg *A : Args.filtered(drv::OPT_iframework))
-    FEOpts.SystemFwkPaths.emplace_back(A->getValue());
+    FEOpts.SystemFwkPaths.emplace_back(A->getValue(),
+                                       std::optional<PlatformType>{});
 
   // Capture framework paths.
   PathSeq FrameworkPaths;
@@ -359,7 +467,8 @@ bool Options::processFrontendOptions(InputArgList &Args) {
   for (const StringRef FwkPath : DefaultFrameworkPaths) {
     SmallString<PATH_MAX> Path(FEOpts.ISysroot);
     sys::path::append(Path, FwkPath);
-    FEOpts.SystemFwkPaths.emplace_back(Path.str());
+    FEOpts.SystemFwkPaths.emplace_back(Path.str(),
+                                       std::optional<PlatformType>{});
   }
 
   return true;
@@ -510,7 +619,11 @@ Options::processAndFilterOutInstallAPIOptions(ArrayRef<const char *> Args) {
   for (const Arg *A : ParsedArgs) {
     if (A->isClaimed())
       continue;
-    llvm::copy(A->getValues(), std::back_inserter(ClangDriverArgs));
+    // Forward along unclaimed but overlapping arguments to the clang driver.
+    if (A->getOption().getID() > (unsigned)OPT_UNKNOWN) {
+      ClangDriverArgs.push_back(A->getSpelling().data());
+    } else
+      llvm::copy(A->getValues(), std::back_inserter(ClangDriverArgs));
   }
   return ClangDriverArgs;
 }
@@ -622,12 +735,30 @@ std::pair<LibAttrs, ReexportedInterfaces> Options::getReexportedLibraries() {
     return true;
   };
 
+  PlatformSet Platforms;
+  llvm::for_each(DriverOpts.Targets,
+                 [&](const auto &T) { Platforms.insert(T.first.Platform); });
   // Populate search paths by looking at user paths before system ones.
   PathSeq FwkSearchPaths(FEOpts.FwkPaths.begin(), FEOpts.FwkPaths.end());
-  // FIXME: System framework paths need to reset if installapi is invoked with
-  // different platforms.
-  FwkSearchPaths.insert(FwkSearchPaths.end(), FEOpts.SystemFwkPaths.begin(),
-                        FEOpts.SystemFwkPaths.end());
+  for (const PlatformType P : Platforms) {
+    PathSeq PlatformSearchPaths = getPathsForPlatform(FEOpts.SystemFwkPaths, P);
+    FwkSearchPaths.insert(FwkSearchPaths.end(), PlatformSearchPaths.begin(),
+                          PlatformSearchPaths.end());
+    for (const StringMapEntry<ArchitectureSet> &Lib :
+         LinkerOpts.ReexportedFrameworks) {
+      std::string Name = (Lib.getKey() + ".framework/" + Lib.getKey()).str();
+      std::string Path = findLibrary(Name, *FM, FwkSearchPaths, {}, {});
+      if (Path.empty()) {
+        Diags->Report(diag::err_cannot_find_reexport) << false << Lib.getKey();
+        return {};
+      }
+      if (DriverOpts.TraceLibraryLocation)
+        errs() << Path << "\n";
+
+      AccumulateReexports(Path, Lib.getValue());
+    }
+    FwkSearchPaths.resize(FwkSearchPaths.size() - PlatformSearchPaths.size());
+  }
 
   for (const StringMapEntry<ArchitectureSet> &Lib :
        LinkerOpts.ReexportedLibraries) {
@@ -647,20 +778,6 @@ std::pair<LibAttrs, ReexportedInterfaces> Options::getReexportedLibraries() {
        LinkerOpts.ReexportedLibraryPaths)
     AccumulateReexports(Lib.getKey(), Lib.getValue());
 
-  for (const StringMapEntry<ArchitectureSet> &Lib :
-       LinkerOpts.ReexportedFrameworks) {
-    std::string Name = (Lib.getKey() + ".framework/" + Lib.getKey()).str();
-    std::string Path = findLibrary(Name, *FM, FwkSearchPaths, {}, {});
-    if (Path.empty()) {
-      Diags->Report(diag::err_cannot_find_reexport) << false << Lib.getKey();
-      return {};
-    }
-    if (DriverOpts.TraceLibraryLocation)
-      errs() << Path << "\n";
-
-    AccumulateReexports(Path, Lib.getValue());
-  }
-
   return {std::move(Reexports), std::move(ReexportIFs)};
 }
 
@@ -876,5 +993,25 @@ InstallAPIContext Options::createContext() {
   return Ctx;
 }
 
+void Options::addConditionalCC1Args(std::vector<std::string> &ArgStrings,
+                                    const llvm::Triple &Targ,
+                                    const HeaderType Type) {
+  // Unique to architecture (Xarch) options hold no arguments to pass along for
+  // frontend.
+
+  // Add specific to platform arguments.
+  PathSeq PlatformSearchPaths =
+      getPathsForPlatform(FEOpts.SystemFwkPaths, mapToPlatformType(Targ));
+  llvm::for_each(PlatformSearchPaths, [&ArgStrings](const StringRef Path) {
+    ArgStrings.push_back("-iframework");
+    ArgStrings.push_back(Path.str());
+  });
+
+  // Add specific to header type arguments.
+  if (Type == HeaderType::Project)
+    for (const StringRef A : ProjectLevelArgs)
+      ArgStrings.emplace_back(A);
+}
+
 } // namespace installapi
 } // namespace clang
diff --git a/clang/tools/clang-installapi/Options.h b/clang/tools/clang-installapi/Options.h
index e9ac75889ad3..fd1e10065d10 100644
--- a/clang/tools/clang-installapi/Options.h
+++ b/clang/tools/clang-installapi/Options.h
@@ -133,6 +133,9 @@ struct LinkerOptions {
 };
 
 struct FrontendOptions {
+  /// \brief Unique clang options to pass per key in map.
+  llvm::StringMap<std::vector<std::string>> UniqueArgs;
+
   /// \brief The language mode to parse headers in.
   Language LangMode = Language::ObjC;
 
@@ -143,7 +146,7 @@ struct FrontendOptions {
   PathSeq FwkPaths;
 
   /// \brief Additional SYSTEM framework search paths.
-  PathSeq SystemFwkPaths;
+  PathToPlatformSeq SystemFwkPaths;
 };
 
 using arg_iterator = llvm::opt::arg_iterator<llvm::opt::Arg **>;
@@ -156,6 +159,8 @@ private:
   processAndFilterOutInstallAPIOptions(ArrayRef<const char *> Args);
   bool processInstallAPIXOptions(llvm::opt::InputArgList &Args);
   bool processXarchOption(llvm::opt::InputArgList &Args, arg_iterator Curr);
+  bool processXplatformOption(llvm::opt::InputArgList &Args, arg_iterator Curr);
+  bool processXprojectOption(llvm::opt::InputArgList &Args, arg_iterator Curr);
 
 public:
   /// The various options grouped together.
@@ -176,6 +181,11 @@ public:
   /// ones.
   std::vector<std::string> &getClangFrontendArgs() { return FrontendArgs; }
 
+  /// \brief Add relevant, but conditionalized by active target and header type,
+  /// arguments for constructing a CC1 invocation.
+  void addConditionalCC1Args(std::vector<std::string> &ArgStrings,
+                             const llvm::Triple &Targ, const HeaderType Type);
+
 private:
   bool addFilePaths(llvm::opt::InputArgList &Args, PathSeq &Headers,
                     llvm::opt::OptSpecifier ID);
@@ -186,6 +196,7 @@ private:
   FileManager *FM;
   std::vector<std::string> FrontendArgs;
   llvm::DenseMap<const llvm::opt::Arg *, Architecture> ArgToArchMap;
+  std::vector<std::string> ProjectLevelArgs;
 };
 
 enum ID {
diff --git a/llvm/include/llvm/TextAPI/Utils.h b/llvm/include/llvm/TextAPI/Utils.h
index 87550851f091..00dfd63e14f9 100644
--- a/llvm/include/llvm/TextAPI/Utils.h
+++ b/llvm/include/llvm/TextAPI/Utils.h
@@ -32,6 +32,8 @@
 namespace llvm::MachO {
 
 using PathSeq = std::vector<std::string>;
+using PathToPlatform = std::pair<std::string, std::optional<PlatformType>>;
+using PathToPlatformSeq = std::vector<PathToPlatform>;
 
 // Defines simple struct for storing symbolic links.
 struct SymLink {
@@ -87,5 +89,12 @@ using AliasMap = std::map<AliasEntry, AliasEntry>;
 /// \return Lookup table of alias to their base symbol.
 Expected<AliasMap> parseAliasList(std::unique_ptr<llvm::MemoryBuffer> &Buffer);
 
+/// Pickup active paths for a given platform.
+///
+/// \param Paths File or search paths to pick up.
+/// \param Platform Platform to collect paths for.
+PathSeq getPathsForPlatform(const PathToPlatformSeq &Paths,
+                            PlatformType Platform);
+
 } // namespace llvm::MachO
 #endif // LLVM_TEXTAPI_UTILS_H
diff --git a/llvm/lib/TextAPI/Utils.cpp b/llvm/lib/TextAPI/Utils.cpp
index 3b5e11e29de4..08f14f65177e 100644
--- a/llvm/lib/TextAPI/Utils.cpp
+++ b/llvm/lib/TextAPI/Utils.cpp
@@ -232,3 +232,13 @@ llvm::MachO::parseAliasList(std::unique_ptr<llvm::MemoryBuffer> &Buffer) {
 
   return Aliases;
 }
+
+PathSeq llvm::MachO::getPathsForPlatform(const PathToPlatformSeq &Paths,
+                                         PlatformType Platform) {
+  PathSeq Result;
+  for (const auto &[Path, CurrP] : Paths) {
+    if (!CurrP.has_value() || CurrP.value() == Platform)
+      Result.push_back(Path);
+  }
+  return Result;
+}
-- 
GitLab


From 871f4839f988a1ef59ea0371e0f25c8651a899f2 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Fri, 10 May 2024 18:56:21 +0200
Subject: [PATCH 0423/1206] [lldb-dap] Fix a race during shutdown (#91591)

lldb-dap was setting a flag which was meant to shut it down as soon as
it sent a terminated event. The problem with this flag is two-fold:
- as far as I can tell (definitely not an expert here), there's no
justification for this in the protocol spec. The only way I found to
shut the server down was to send it a disconnect request.
- the flag did not actually work most of the time, because it's only
checked between requests so nothing will happen if the server starts
listening for a new request before a different thread manages to send
the terminated event. And since the next request is usually the
disconnect request, everything will operate normally.

The combination of these two things meant that the issue was largely
unnoticable, except for rare flaky test failures, which happened when
the handler thread was too slow, and checked the flag after it has
already been said. This caused the test suite to complain as it did not
get a response to the disconnect request. This situation could be
s(t)imulated by adding a sleep to the and of the main loop, which
delayed the flag check, and caused the DAP tests to fail reliably.

This patch changes the shutdown condition to only trigger when the
disconnect request has been received. Since the flag can now only be set
from the handler thread, it no longer needs to be atomic.
---
 lldb/tools/lldb-dap/DAP.cpp      |  5 ++---
 lldb/tools/lldb-dap/DAP.h        |  2 +-
 lldb/tools/lldb-dap/lldb-dap.cpp | 25 +++++++------------------
 3 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp
index b254ddfef0d5..55ff1493c101 100644
--- a/lldb/tools/lldb-dap/DAP.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -39,8 +39,7 @@ DAP::DAP()
            {"objc_throw", "Objective-C Throw", lldb::eLanguageTypeObjC},
            {"swift_catch", "Swift Catch", lldb::eLanguageTypeSwift},
            {"swift_throw", "Swift Throw", lldb::eLanguageTypeSwift}}),
-      focus_tid(LLDB_INVALID_THREAD_ID), sent_terminated_event(false),
-      stop_at_entry(false), is_attach(false),
+      focus_tid(LLDB_INVALID_THREAD_ID), stop_at_entry(false), is_attach(false),
       enable_auto_variable_summaries(false),
       enable_synthetic_child_debugging(false),
       restarting_process_id(LLDB_INVALID_PROCESS_ID),
@@ -623,7 +622,7 @@ bool DAP::HandleObject(const llvm::json::Object &object) {
 }
 
 llvm::Error DAP::Loop() {
-  while (!sent_terminated_event) {
+  while (!disconnecting) {
     llvm::json::Object object;
     lldb_dap::PacketStatus status = GetNextObject(object);
 
diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index 5c70a056fea4..bbd9d46ba3a0 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -168,7 +168,7 @@ struct DAP {
   // arguments if we get a RestartRequest.
   std::optional<llvm::json::Object> last_launch_or_attach_request;
   lldb::tid_t focus_tid;
-  std::atomic<bool> sent_terminated_event;
+  bool disconnecting = false;
   bool stop_at_entry;
   bool is_attach;
   bool enable_auto_variable_summaries;
diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp
index f35abd665e84..96da458be21d 100644
--- a/lldb/tools/lldb-dap/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/lldb-dap.cpp
@@ -226,26 +226,14 @@ void SendContinuedEvent() {
 // Send a "terminated" event to indicate the process is done being
 // debugged.
 void SendTerminatedEvent() {
-  // If an inferior exits prior to the processing of a disconnect request, then
-  // the threads executing EventThreadFunction and request_discontinue
-  // respectively may call SendTerminatedEvent simultaneously. Without any
-  // synchronization, the thread executing EventThreadFunction may set
-  // g_dap.sent_terminated_event before the thread executing
-  // request_discontinue has had a chance to test it, in which case the latter
-  // would move ahead to issue a response to the disconnect request. Said
-  // response may get dispatched ahead of the terminated event compelling the
-  // client to terminate the debug session without consuming any console output
-  // that might've been generated by the execution of terminateCommands. So,
-  // synchronize simultaneous calls to SendTerminatedEvent.
+  // Prevent races if the process exits while we're being asked to disconnect.
   static std::mutex mutex;
   std::lock_guard<std::mutex> locker(mutex);
-  if (!g_dap.sent_terminated_event) {
-    g_dap.sent_terminated_event = true;
-    g_dap.RunTerminateCommands();
-    // Send a "terminated" event
-    llvm::json::Object event(CreateTerminatedEventObject());
-    g_dap.SendJSON(llvm::json::Value(std::move(event)));
-  }
+
+  g_dap.RunTerminateCommands();
+  // Send a "terminated" event
+  llvm::json::Object event(CreateTerminatedEventObject());
+  g_dap.SendJSON(llvm::json::Value(std::move(event)));
 }
 
 // Send a thread stopped event for all threads as long as the process
@@ -1003,6 +991,7 @@ void request_disconnect(const llvm::json::Object &request) {
     g_dap.broadcaster.BroadcastEventByType(eBroadcastBitStopProgressThread);
     g_dap.progress_event_thread.join();
   }
+  g_dap.disconnecting = true;
 }
 
 void request_exceptionInfo(const llvm::json::Object &request) {
-- 
GitLab


From 3bde7983986d8ce637f6bb506860859249787751 Mon Sep 17 00:00:00 2001
From: Alex Langford <alangford@apple.com>
Date: Fri, 10 May 2024 10:00:36 -0700
Subject: [PATCH 0424/1206] [lldb] Put SBSourceLanguageName in lldb namespace
 (#91685)

---
 lldb/scripts/generate-sbapi-dwarf-enum.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lldb/scripts/generate-sbapi-dwarf-enum.py b/lldb/scripts/generate-sbapi-dwarf-enum.py
index f7a13e5efffe..7fd603798631 100755
--- a/lldb/scripts/generate-sbapi-dwarf-enum.py
+++ b/lldb/scripts/generate-sbapi-dwarf-enum.py
@@ -15,6 +15,8 @@ HEADER = """\
 
 #ifndef LLDB_API_SBLANGUAGE_H
 #define LLDB_API_SBLANGUAGE_H
+
+namespace lldb {
 /// Used by \\ref SBExpressionOptions.
 /// These enumerations use the same language enumerations as the DWARF
 /// specification for ease of use and consistency.
@@ -24,6 +26,8 @@ enum SBSourceLanguageName : uint16_t {
 FOOTER = """\
 };
 
+} // namespace lldb
+
 #endif
 """
 
-- 
GitLab


From 265953cc26b40c4f9a3300baa18c2b7a45074b74 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Fri, 10 May 2024 10:42:40 -0700
Subject: [PATCH 0425/1206] [ctx_profile] Arena should zero-initialize its
 allocation area.

---
 compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp        | 4 ++++
 compiler-rt/lib/ctx_profile/CtxInstrProfiling.h          | 6 +++++-
 .../lib/ctx_profile/tests/CtxInstrProfilingTest.cpp      | 9 +++++++++
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
index c5d167bf996a..cff39eeafba6 100644
--- a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
+++ b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
@@ -132,6 +132,10 @@ __thread ContextNode **volatile __llvm_ctx_profile_callsite[2] = {0, 0};
 __thread ContextRoot *volatile __llvm_ctx_profile_current_context_root =
     nullptr;
 
+Arena::Arena(uint32_t Size) : Size(Size) {
+  __sanitizer::internal_memset(start(), 0, Size);
+}
+
 // FIXME(mtrofin): use malloc / mmap instead of sanitizer common APIs to reduce
 // the dependency on the latter.
 Arena *Arena::allocateNewArena(size_t Size, Arena *Prev) {
diff --git a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
index 69ce796b71e3..f55068e98dd4 100644
--- a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
+++ b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
@@ -15,6 +15,9 @@
 
 using namespace llvm::ctx_profile;
 
+// Forward-declare for the one unittest checking Arena construction zeroes out
+// its allocatable space.
+class ArenaTest_ZeroInit_Test;
 namespace __ctx_profile {
 
 static constexpr size_t ExpectedAlignment = 8;
@@ -51,7 +54,8 @@ public:
   const char *pos() const { return start() + Pos; }
 
 private:
-  explicit Arena(uint32_t Size) : Size(Size) {}
+  friend class ::ArenaTest_ZeroInit_Test;
+  explicit Arena(uint32_t Size);
   ~Arena() = delete;
 
   char *start() { return reinterpret_cast<char *>(&this[1]); }
diff --git a/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp b/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
index 1e96aea19ce4..b8d4342c4112 100644
--- a/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
+++ b/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
@@ -12,6 +12,15 @@ public:
   ContextRoot Root;
 };
 
+TEST(ArenaTest, ZeroInit) {
+  char Buffer[1024];
+  memset(Buffer, 1, 1024);
+  Arena *A = new (Buffer) Arena(10);
+  for (auto I = 0U; I < A->size(); ++I)
+    EXPECT_EQ(A->pos()[I], 0);
+  EXPECT_EQ(A->size(), 10);
+}
+
 TEST(ArenaTest, Basic) {
   Arena *A = Arena::allocateNewArena(1024);
   EXPECT_EQ(A->size(), 1024U);
-- 
GitLab


From 7c555cb2349b7f5f3b8336c40f38965aca354472 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 10 May 2024 10:47:04 -0700
Subject: [PATCH 0426/1206] [flang][cuda] Reapply restriction on kernel
 subprogram but not device (#91764)

The restriction was completely removed in #89677. This was a bit too
much. Reapply the restriction on elemental, pure and recursive but only
for kernel subprogram (`grid_global` and `global` attributes).
---
 flang/lib/Semantics/check-declarations.cpp |  7 +++++++
 flang/test/Semantics/cuf02.cuf             | 18 +++++++++++++++---
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index 26efa288b5ae..ce7870b8d54e 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -1470,6 +1470,13 @@ void CheckHelper::CheckSubprogram(
     messages_.Say(symbol.name(),
         "A function may not have ATTRIBUTES(GLOBAL) or ATTRIBUTES(GRID_GLOBAL)"_err_en_US);
   }
+  if (cudaAttrs &&
+      (*cudaAttrs == common::CUDASubprogramAttrs::Global ||
+          *cudaAttrs == common::CUDASubprogramAttrs::Grid_Global) &&
+      symbol.attrs().HasAny({Attr::RECURSIVE, Attr::PURE, Attr::ELEMENTAL})) {
+    messages_.Say(symbol.name(),
+        "A kernel subprogram may not be RECURSIVE, PURE, or ELEMENTAL"_err_en_US);
+  }
   if (cudaAttrs && *cudaAttrs != common::CUDASubprogramAttrs::Host) {
     // CUDA device subprogram checks
     if (ClassifyProcedure(symbol) == ProcedureDefinitionClass::Internal) {
diff --git a/flang/test/Semantics/cuf02.cuf b/flang/test/Semantics/cuf02.cuf
index a4a229565a3e..58cb3cf49011 100644
--- a/flang/test/Semantics/cuf02.cuf
+++ b/flang/test/Semantics/cuf02.cuf
@@ -29,11 +29,23 @@ module m
   !ERROR: A function may not have ATTRIBUTES(GLOBAL) or ATTRIBUTES(GRID_GLOBAL)
   attributes(global) real function f1
   end
-  recursive attributes(global) subroutine s7 ! ok
+  !ERROR: A kernel subprogram may not be RECURSIVE, PURE, or ELEMENTAL
+  recursive attributes(global) subroutine s7
   end
-  pure attributes(global) subroutine s8 ! ok
+  !ERROR: A kernel subprogram may not be RECURSIVE, PURE, or ELEMENTAL
+  pure attributes(global) subroutine s8
   end
-  elemental attributes(global) subroutine s9 ! ok
+  !ERROR: A kernel subprogram may not be RECURSIVE, PURE, or ELEMENTAL
+  elemental attributes(global) subroutine s9
+  end
+  !ERROR: A kernel subprogram may not be RECURSIVE, PURE, or ELEMENTAL
+  recursive attributes(grid_global) subroutine s10
+  end
+  !ERROR: A kernel subprogram may not be RECURSIVE, PURE, or ELEMENTAL
+  pure attributes(grid_global) subroutine s11
+  end
+  !ERROR: A kernel subprogram may not be RECURSIVE, PURE, or ELEMENTAL
+  elemental attributes(grid_global) subroutine s12
   end
 end
 
-- 
GitLab


From 842f7b0fe7f70f5f66d925e57bab0123eedcc5d2 Mon Sep 17 00:00:00 2001
From: Tomas Matheson <tomas.matheson@arm.com>
Date: Fri, 10 May 2024 18:46:04 +0100
Subject: [PATCH 0427/1206] [AArch64] make +dit non-FMV-only (#90987)

Required to fix an llvm-test-suite failure:

    error: expected writable system register or pstate: "msr DIT, x0"
---
 llvm/lib/Target/AArch64/AArch64Features.td | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 3425ac413302..920ca7f4fbfc 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -71,7 +71,6 @@ class FMVOnlyExtension<string FMVBit, string Name, string Deps, int Priority>
 }
 
 def : FMVOnlyExtension<"FEAT_DGH", "dgh", "", 260>;
-def : FMVOnlyExtension<"FEAT_DIT", "dit", "+dit", 180>;
 def : FMVOnlyExtension<"FEAT_DPB", "dpb", "+ccpp", 190>;
 def : FMVOnlyExtension<"FEAT_DPB2", "dpb2", "+ccpp,+ccdp", 200>;
 def : FMVOnlyExtension<"FEAT_EBF16", "ebf16", "+bf16", 290>;
@@ -455,9 +454,10 @@ def FeatureMPAM : SubtargetFeature<
     "mpam", "HasMPAM", "true",
     "Enable v8.4-A Memory system Partitioning and Monitoring extension (FEAT_MPAM)">;
 
-def FeatureDIT : SubtargetFeature<
-    "dit", "HasDIT", "true",
-    "Enable v8.4-A Data Independent Timing instructions (FEAT_DIT)">;
+def FeatureDIT : Extension<
+    "dit", "DIT",
+    "Enable v8.4-A Data Independent Timing instructions (FEAT_DIT)", [],
+    "FEAT_DIT", "+dit", 180>;
 
 def FeatureTRACEV8_4 : SubtargetFeature<
     "tracev8.4", "HasTRACEV8_4", "true",
-- 
GitLab


From 514d80b4feea3c788c1b0821959e96f63c8b8f3d Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 10 May 2024 10:49:52 -0700
Subject: [PATCH 0428/1206] [Driver,test] Add -Werror to
 -ffp-contract/-ffp-model tests

Test that these combinations do not lead to a warning.

-funsafe-math-optimizations -ffp-contract=off
and
-ffast-math -ffp-contract=off

may issue a warning after #91271.
---
 clang/test/Driver/fp-contract.c | 151 ++++++++++++++++----------------
 clang/test/Driver/fp-model.c    |   6 +-
 2 files changed, 77 insertions(+), 80 deletions(-)

diff --git a/clang/test/Driver/fp-contract.c b/clang/test/Driver/fp-contract.c
index 660f67fad3cc..e2691dc211cc 100644
--- a/clang/test/Driver/fp-contract.c
+++ b/clang/test/Driver/fp-contract.c
@@ -10,112 +10,112 @@
 // RUN: %clang -### -fno-fast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -ffast-math -ffp-contract=on -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=on -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 // CHECK-FPC-ON:       "-ffp-contract=on"
 
-// RUN: %clang -### -ffast-math -ffp-contract=off -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=off -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-OFF %s
 // CHECK-FPC-OFF:      "-ffp-contract=off"
 
-// RUN: %clang -### -ffast-math -ffp-contract=fast -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=fast -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffast-math -ffp-contract=fast-honor-pragmas -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=fast-honor-pragmas -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST-HONOR %s
 // CHECK-FPC-FAST-HONOR:     "-ffp-contract=fast-honor-pragmas"
 
-// RUN: %clang -### -ffp-contract=fast -ffast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=fast -ffast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
-// RUN: %clang -### -ffp-contract=on -ffast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=on -ffast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
-// RUN: %clang -### -ffp-contract=off -ffast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=off -ffast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=fast -fno-fast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=fast -fno-fast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
-// RUN: %clang -### -ffp-contract=on -fno-fast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=on -fno-fast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
-// RUN: %clang -### -ffp-contract=off -fno-fast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=off -fno-fast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
 
-// RUN: %clang -### -ffast-math -ffp-contract=fast -ffp-contract=on -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=fast -ffp-contract=on -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -ffast-math -ffp-contract=on -ffp-contract=off -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=on -ffp-contract=off -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -ffast-math -ffp-contract=on -ffp-contract=fast -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=on -ffp-contract=fast -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffast-math -ffp-contract=off -ffp-contract=on -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=off -ffp-contract=on -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -ffast-math -ffp-contract=off -ffp-contract=fast \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=off -ffp-contract=fast \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffast-math -ffp-contract=on -fno-fast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=on -fno-fast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -ffast-math -ffp-contract=off -fno-fast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=off -fno-fast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -ffast-math -ffp-contract=fast -fno-fast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -ffp-contract=fast -fno-fast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffast-math -fno-fast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffast-math -fno-fast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -fno-fast-math -ffast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -fno-fast-math -ffast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -fno-fast-math -ffp-contract=on -c %s 2>&1 \
+// RUN: %clang -### -Werror -fno-fast-math -ffp-contract=on -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -fno-fast-math -ffp-contract=off -c %s 2>&1 \
+// RUN: %clang -### -Werror -fno-fast-math -ffp-contract=off -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -fno-fast-math -ffp-contract=fast -c %s 2>&1 \
+// RUN: %clang -### -Werror -fno-fast-math -ffp-contract=fast -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=fast -fno-fast-math -ffp-contract=on \
+// RUN: %clang -### -Werror -ffp-contract=fast -fno-fast-math -ffp-contract=on \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -ffp-contract=fast -fno-fast-math -ffp-contract=off \
+// RUN: %clang -### -Werror -ffp-contract=fast -fno-fast-math -ffp-contract=off \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -ffp-contract=off -fno-fast-math -ffp-contract=fast \
+// RUN: %clang -### -Werror -ffp-contract=off -fno-fast-math -ffp-contract=fast \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=off -fno-fast-math -ffp-contract=on \
+// RUN: %clang -### -Werror -ffp-contract=off -fno-fast-math -ffp-contract=on \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -ffp-contract=on -ffast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=on -ffast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=off -ffast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=off -ffast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=fast -ffast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=fast -ffast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=on -ffast-math -fno-fast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=on -ffast-math -fno-fast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -ffp-contract=off -ffast-math -fno-fast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=off -ffast-math -fno-fast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -ffp-contract=fast -ffast-math -fno-fast-math -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=fast -ffast-math -fno-fast-math -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -fno-fast-math -ffast-math -ffp-contract=fast \
+// RUN: %clang -### -Werror -fno-fast-math -ffast-math -ffp-contract=fast \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -fno-fast-math -ffast-math -ffp-contract=on \
+// RUN: %clang -### -Werror -fno-fast-math -ffast-math -ffp-contract=on \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -fno-fast-math -ffast-math -ffp-contract=off \
+// RUN: %clang -### -Werror -fno-fast-math -ffast-math -ffp-contract=off \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
 // funsafe-math-optimizations, fno-unsafe-math-optimizations
@@ -125,116 +125,113 @@
 // RUN: %clang -### -fno-unsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -funsafe-math-optimizations -ffp-contract=on -c %s 2>&1 \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -ffp-contract=on -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -funsafe-math-optimizations -ffp-contract=off -c %s 2>&1 \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -ffp-contract=off -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -funsafe-math-optimizations -ffp-contract=fast -c %s 2>&1 \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -ffp-contract=fast -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=fast -funsafe-math-optimizations -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=fast -funsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
-// RUN: %clang -### -ffp-contract=on -funsafe-math-optimizations -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=on -funsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
-// RUN: %clang -### -ffp-contract=off -funsafe-math-optimizations -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=off -funsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=fast -fno-unsafe-math-optimizations -c \
+// RUN: %clang -### -Werror -ffp-contract=fast -fno-unsafe-math-optimizations -c \
 // RUN: %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-FAST %s
-// RUN: %clang -### -ffp-contract=on -fno-unsafe-math-optimizations -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=on -fno-unsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
-// RUN: %clang -### -ffp-contract=off -fno-unsafe-math-optimizations -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=off -fno-unsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -funsafe-math-optimizations -ffp-contract=fast \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -ffp-contract=fast \
 // RUN: -ffp-contract=on -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -funsafe-math-optimizations -ffp-contract=on \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -ffp-contract=on \
 // RUN: -ffp-contract=off -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -funsafe-math-optimizations -ffp-contract=on \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -ffp-contract=on \
 // RUN: -ffp-contract=fast -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -funsafe-math-optimizations -ffp-contract=off \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -ffp-contract=off \
 // RUN: -ffp-contract=on -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -funsafe-math-optimizations -ffp-contract=off \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -ffp-contract=off \
 // RUN: -ffp-contract=fast \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -funsafe-math-optimizations -ffp-contract=on \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -ffp-contract=on \
 // RUN: -fno-unsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -funsafe-math-optimizations -ffp-contract=off \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -ffp-contract=off \
 // RUN: -fno-unsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -funsafe-math-optimizations -ffp-contract=fast \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -ffp-contract=fast \
 // RUN: -fno-unsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -funsafe-math-optimizations -fno-unsafe-math-optimizations \
+// RUN: %clang -### -Werror -funsafe-math-optimizations -fno-unsafe-math-optimizations \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -fno-unsafe-math-optimizations -funsafe-math-optimizations \
+// RUN: %clang -### -Werror -fno-unsafe-math-optimizations -funsafe-math-optimizations \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -fno-unsafe-math-optimizations -ffp-contract=on -c %s 2>&1 \
+// RUN: %clang -### -Werror -fno-unsafe-math-optimizations -ffp-contract=on -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -fno-unsafe-math-optimizations -ffp-contract=off -c %s 2>&1 \
+// RUN: %clang -### -Werror -fno-unsafe-math-optimizations -ffp-contract=off -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -fno-unsafe-math-optimizations -ffp-contract=fast -c %s 2>&1 \
+// RUN: %clang -### -Werror -fno-unsafe-math-optimizations -ffp-contract=fast -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=fast -fno-unsafe-math-optimizations \
+// RUN: %clang -### -Werror -ffp-contract=fast -fno-unsafe-math-optimizations \
 // RUN: -ffp-contract=on \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -ffp-contract=fast -fno-unsafe-math-optimizations \
+// RUN: %clang -### -Werror -ffp-contract=fast -fno-unsafe-math-optimizations \
 // RUN: -ffp-contract=off \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -ffp-contract=off -fno-unsafe-math-optimizations \
+// RUN: %clang -### -Werror -ffp-contract=off -fno-unsafe-math-optimizations \
 // RUN: -ffp-contract=fast \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=off -fno-unsafe-math-optimizations \
+// RUN: %clang -### -Werror -ffp-contract=off -fno-unsafe-math-optimizations \
 // RUN: -ffp-contract=on -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -ffp-contract=on -funsafe-math-optimizations -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=on -funsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=off -funsafe-math-optimizations -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=off -funsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=fast -funsafe-math-optimizations -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=fast -funsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -ffp-contract=on -funsafe-math-optimizations \
-// RUN: -fno-unsafe-math-optimizations -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=on -funsafe-math-optimizations -fno-unsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -ffp-contract=off -funsafe-math-optimizations \
-// RUN: -fno-unsafe-math-optimizations -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=off -funsafe-math-optimizations -fno-unsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
-// RUN: %clang -### -ffp-contract=fast -funsafe-math-optimizations \
-// RUN: -fno-unsafe-math-optimizations -c %s 2>&1 \
+// RUN: %clang -### -Werror -ffp-contract=fast -funsafe-math-optimizations -fno-unsafe-math-optimizations -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -fno-unsafe-math-optimizations -funsafe-math-optimizations \
-// RUN: -ffp-contract=fast \
+// RUN: %clang -### -Werror -fno-unsafe-math-optimizations -funsafe-math-optimizations \
+// RUN:   -ffp-contract=fast \
 // RUN: -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-FAST %s
 
-// RUN: %clang -### -fno-unsafe-math-optimizations -funsafe-math-optimizations \
-// RUN: -ffp-contract=on -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-ON %s
+// RUN: %clang -### -Werror -fno-unsafe-math-optimizations -funsafe-math-optimizations \
+// RUN:   -ffp-contract=on -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-ON %s
 
-// RUN: %clang -### -fno-unsafe-math-optimizations -funsafe-math-optimizations \
-// RUN: -ffp-contract=off -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-OFF %s
+// RUN: %clang -### -Werror -fno-unsafe-math-optimizations -funsafe-math-optimizations \
+// RUN:   -ffp-contract=off -c %s 2>&1 | FileCheck --check-prefix=CHECK-FPC-OFF %s
 
diff --git a/clang/test/Driver/fp-model.c b/clang/test/Driver/fp-model.c
index 9d1245239911..644523394d6b 100644
--- a/clang/test/Driver/fp-model.c
+++ b/clang/test/Driver/fp-model.c
@@ -150,7 +150,7 @@
 // CHECK-FEB-IGNORE: "-fno-rounding-math"
 // CHECK-FEB-IGNORE: "-ffp-exception-behavior=ignore"
 
-// RUN: %clang -### -nostdinc -ffast-math -ffp-model=fast -c %s 2>&1 \
+// RUN: %clang -### -nostdinc -Werror -ffast-math -ffp-model=fast -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FASTMATH-FPM-FAST %s
 // CHECK-FASTMATH-FPM-FAST: "-cc1"
 // CHECK-FASTMATH-FPM-FAST: "-menable-no-infs"
@@ -165,7 +165,7 @@
 // CHECK-FASTMATH-FPM-FAST: "-ffast-math"
 // CHECK-FASTMATH-FPM-FAST: "-ffinite-math-only"
 
-// RUN: %clang -### -nostdinc -ffast-math -ffp-model=precise -c %s 2>&1 \
+// RUN: %clang -### -nostdinc -Werror -ffast-math -ffp-model=precise -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FASTMATH-FPM-PRECISE %s
 // CHECK-FASTMATH-FPM-PRECISE:     "-cc1"
 // CHECK-FASTMATH-FPM-PRECISE-NOT: "-menable-no-infs"
@@ -180,7 +180,7 @@
 // CHECK-FASTMATH-FPM-PRECISE-NOT: "-ffast-math"
 // CHECK-FASTMATH-FPM-PRECISE-NOT: "-ffinite-math-only"
 
-// RUN: %clang -### -nostdinc -ffast-math -ffp-model=strict -c %s 2>&1 \
+// RUN: %clang -### -nostdinc -Werror -ffast-math -ffp-model=strict -c %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-FASTMATH-FPM-STRICT %s
 // CHECK-FASTMATH-FPM-STRICT:     "-cc1"
 // CHECK-FASTMATH-FPM-STRICT-NOT: "-menable-no-infs"
-- 
GitLab


From e513f2c69b13322d0289cbb74c91a84996382baa Mon Sep 17 00:00:00 2001
From: Spenser Bauman <sbauman@mathworks.com>
Date: Fri, 10 May 2024 13:56:06 -0400
Subject: [PATCH 0429/1206] [mlir][tosa] Cleanups for post-merge review
 comments in tosa-infer-shapes (#87660)

This change addresses some of the additional review feedback on
https://github.com/llvm/llvm-project/pull/87234.

A summary of the changes:

1. Cleaned up the language to use 'roll back' rather than revert to
reduce the chance of confusion. Improved some function names as well.
2. Eliminated string comparisons on dialect names.
3. Prevented the introduction of redundant tensor.cast operations for
the same value.

---------

Co-authored-by: Spenser Bauman <sabauma@fastmail>
---
 .../Tosa/Transforms/TosaInferShapes.cpp       | 41 ++++++++++---------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
index 8614559e2a6f..b1d572054184 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Utils/ShapeUtils.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
@@ -39,13 +40,10 @@ namespace {
 // type-inference related interface.
 // When a non-replaceable use is encountered, the value is wrapped in a
 // cast back to the original type after inference.
-bool isReplaceableUser(Operation *user) {
-  // Handle unregistered dialects.
+bool canBeRefined(Operation *user) {
   if (!user->getDialect())
     return false;
-
-  return user->getDialect()->getNamespace() ==
-             TosaDialect::getDialectNamespace() ||
+  return user->getDialect()->getTypeID() == TypeID::get<TosaDialect>() ||
          isa<InferTypeOpInterface, InferShapedTypeOpInterface>(user);
 }
 
@@ -53,16 +51,16 @@ bool isReplaceableUser(Operation *user) {
 // updated. For the tosa.while_loop operation, types are speculatively updated
 // within the body region to determine the output type of the while_loop. This
 // process is performed until a fixed point is reached, then the types are
-// reverted.
+// rolled back.
 //
-// This class encapsulates the state information needed to perform the reversion
+// This class encapsulates the state information needed to perform the roll back
 // process or to commit to the final changes.
 class TypeModificationState {
 public:
   TypeModificationState() = default;
 
   ~TypeModificationState() {
-    // Ensure the recorded modifications are either committed or reverted.
+    // Ensure the recorded modifications are either committed or rolled back.
     assert(oldTypes.empty() && "unhandled type modifications");
   }
 
@@ -74,10 +72,9 @@ public:
     }
   }
 
-  // Revert changes made to the types in the IR by setting all the affected
+  // Roll back changes made to the types in the IR by setting all the affected
   // values to their old types.
-  void revert() {
-    // Otherwise revert the changes.
+  void rollBack() {
     for (auto [value, type] : oldTypes)
       value.setType(type);
 
@@ -91,15 +88,18 @@ public:
     // For each use whose type changed, cast the value with the new type back to
     // the old type.
     for (auto [value, oldType] : oldTypes) {
+      tensor::CastOp castedValue;
       for (auto &use : value.getUses()) {
-        if (isReplaceableUser(use.getOwner()))
+        if (canBeRefined(use.getOwner()))
           continue;
 
-        OpBuilder builder(value.getContext());
-        builder.setInsertionPoint(use.getOwner());
+        // Cache the cast to avoid generating duplicates
+        if (!castedValue) {
+          ImplicitLocOpBuilder builder{value.getLoc(), use.getOwner()};
+          castedValue = builder.create<tensor::CastOp>(oldType, value);
+        }
 
-        Location loc = value.getLoc();
-        use.set(builder.create<tensor::CastOp>(loc, oldType, value));
+        use.set(castedValue);
       }
     }
 
@@ -211,8 +211,8 @@ void propagateShapesToTosaWhile(Operation &op, TypeModificationState &state) {
       argTypes[i] = newType;
     }
 
-    // Revert all changes made during the speculative part of the algorithm.
-    localState.revert();
+    // Roll back all changes made during the speculative part of the algorithm.
+    localState.rollBack();
   }
 
   // We now set the block arguments according to the most recent shape
@@ -228,10 +228,11 @@ void propagateShapesToTosaWhile(Operation &op, TypeModificationState &state) {
 }
 
 void propagateShapesInRegion(Region &region, TypeModificationState &state) {
+  Dialect *tosaDialect = region.getContext()->getLoadedDialect<TosaDialect>();
+
   for (auto &block : region) {
     for (Operation &op : block) {
-      if (!op.getDialect() ||
-          op.getDialect()->getNamespace() != TosaDialect::getDialectNamespace())
+      if (op.getDialect() != tosaDialect)
         continue;
 
       propagateShapesToTosaIf(op, state);
-- 
GitLab


From af0b293b5068e71e34704fa4c997fc3295e673fe Mon Sep 17 00:00:00 2001
From: agozillon <Andrew.Gozillon@amd.com>
Date: Fri, 10 May 2024 20:05:53 +0200
Subject: [PATCH 0430/1206] [Flang][OpenMP] Derived type member map fortran
 offload runtime tests (#82850)

This is a large series of runtime tests that help to add coverage for
the specific cases intended to be supported by the PR stack
that extends derived type map support in Flang+OpenMP. Primarily this will add functionality coverage, there's cases where
things may work, but not optimally (or at least similarly to the status quo in Clang), additional IR tests are added in the
relevant segments of the related PRs to test for breakages like that.
---
 .../target-map-derived-type-full-1.f90        |  45 ++++++++
 .../target-map-derived-type-full-2.f90        |  60 +++++++++++
 ...arget-map-derived-type-full-implicit-1.f90 |  46 ++++++++
 ...arget-map-derived-type-full-implicit-2.f90 |  61 +++++++++++
 ...double-large-nested-dtype-multi-member.f90 | 101 ++++++++++++++++++
 ...t-map-double-nested-dtype-array-bounds.f90 |  47 ++++++++
 ...ouble-nested-dtype-double-array-bounds.f90 |  47 ++++++++
 ...-map-double-nested-dtype-single-member.f90 |  47 ++++++++
 ...pe-arr-bounds-member-enter-exit-update.f90 |  49 +++++++++
 ...map-dtype-arr-bounds-member-enter-exit.f90 |  49 +++++++++
 ...dtype-explicit-individual-array-member.f90 |  33 ++++++
 ...-multi-explicit-array-3D-member-bounds.f90 |  45 ++++++++
 ...ype-multi-explicit-array-member-bounds.f90 |  38 +++++++
 ...-map-dtype-multi-explicit-array-member.f90 |  39 +++++++
 ...target-map-dtype-multi-explicit-member.f90 |  33 ++++++
 .../fortran/target-map-enter-exit-array-2.f90 |   2 -
 .../target-map-enter-exit-array-bounds.f90    |   2 -
 .../fortran/target-map-enter-exit-scalar.f90  |   1 -
 ...target-map-individual-dtype-member-map.f90 |  33 ++++++
 ...et-map-large-nested-dtype-multi-member.f90 |  83 ++++++++++++++
 ...target-map-nested-dtype-complex-member.f90 |  55 ++++++++++
 ...target-map-nested-dtype-derived-member.f90 |  51 +++++++++
 .../target-map-nested-dtype-multi-member.f90  |  47 ++++++++
 .../target-map-nested-dtype-single-member.f90 |  40 +++++++
 .../target-map-two-dtype-explicit-member.f90  |  35 ++++++
 ...type-individual-member-array-1D-bounds.f90 |  39 +++++++
 ...type-mixed-implicit-explicit-capture-1.f90 |  35 ++++++
 ...type-mixed-implicit-explicit-capture-2.f90 |  41 +++++++
 ...two-dtype-multi-member-array-1D-bounds.f90 |  51 +++++++++
 ...-map-two-nested-dtype-member-array-map.f90 |  56 ++++++++++
 30 files changed, 1306 insertions(+), 5 deletions(-)
 create mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-1.f90
 create mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-2.f90
 create mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
 create mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
 create mode 100644 offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
 create mode 100644 offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90

diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-1.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
new file mode 100644
index 000000000000..cb03708554fe
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
@@ -0,0 +1,45 @@
+! Offloading test checking interaction of an
+! explicit derived type mapping when mapped 
+! to target and assinging one derived type
+! to another
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar
+    integer(4) :: ix = 0
+    real(4) :: rx = 0.0
+    complex(4) :: zx = (0,0)
+    end type scalar  
+  
+    type(scalar) :: in
+    type(scalar) :: out
+    in%ix = 10
+    in%rx = 2.0
+    in%zx = (2, 10)
+  
+  !$omp target map(from:out) map(to:in)
+      out = in 
+  !$omp end target
+  
+    print*, in%ix
+    print*, in%rx
+    write (*,*) in%zx
+  
+    print*, out%ix
+    print*, out%rx
+    write (*,*)  out%zx
+end program main
+
+!CHECK: 10
+!CHECK: 2.
+!CHECK: (2.,10.)
+!CHECK: 10
+!CHECK: 2.
+!CHECK: (2.,10.)
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-2.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-2.f90
new file mode 100644
index 000000000000..0095b0fdf86a
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-2.f90
@@ -0,0 +1,60 @@
+! Offloading test checking interaction of an
+! explicit derived type mapping when mapped to 
+! target and assigning to individual members
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar
+    integer(4) :: ix = 0
+    real(4) :: rx = 0.0
+    complex(4) :: zx = (0,0)
+    integer(4) :: array(5)
+    end type scalar 
+  
+    type(scalar) :: out
+    type(scalar) :: in
+  
+    in%ix = 10
+    in%rx = 2.0
+    in%zx = (2, 10)
+  
+    do i = 1, 5
+      in%array(i) = i
+    end do 
+  
+  !$omp target map(from:out) map(to:in)
+    out%ix = in%ix
+    out%rx = in%rx
+    out%zx = in%zx
+  
+    do i = 1, 5
+      out%array(i) = in%array(i)
+    end do 
+  !$omp end target
+  
+    print*, in%ix
+    print*, in%rx
+    print*, in%array
+    write (*,*) in%zx
+
+    print*, out%ix
+    print*, out%rx
+    print*, out%array
+    write (*,*)  out%zx
+end program main
+
+!CHECK: 10
+!CHECK: 2.
+!CHECK: 1 2 3 4 5
+!CHECK: (2.,10.)
+!CHECK: 10
+!CHECK: 2.
+!CHECK: 1 2 3 4 5
+!CHECK: (2.,10.)
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
new file mode 100644
index 000000000000..f57e2c70d155
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
@@ -0,0 +1,46 @@
+! Offloading test checking interaction of an
+! implicit derived type mapping when mapped 
+! to target and assinging one derived type
+! to another
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar
+    integer(4) :: ix = 0
+    real(4) :: rx = 0.0
+    complex(4) :: zx = (0,0)
+    end type scalar  
+  
+    type(scalar) :: in
+    type(scalar) :: out
+    in%ix = 10
+    in%rx = 2.0
+    in%zx = (2, 10)
+  
+  !$omp target map(from:out)
+      out = in 
+  !$omp end target
+  
+    print*, in%ix
+    print*, in%rx
+    write (*,*) in%zx
+
+    print*, out%ix
+    print*, out%rx
+    write (*,*)  out%zx
+  end program main
+
+!CHECK: 10
+!CHECK: 2.
+!CHECK: (2.,10.)
+!CHECK: 10
+!CHECK: 2.
+!CHECK: (2.,10.)
+  
\ No newline at end of file
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
new file mode 100644
index 000000000000..92d3454d462a
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
@@ -0,0 +1,61 @@
+! Offloading test checking interaction of an
+! explicit derived type mapping when mapped 
+! to target and assinging one derived type
+! to another
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar
+    integer(4) :: ix = 0
+    real(4) :: rx = 0.0
+    complex(4) :: zx = (0,0)
+    integer(4) :: array(5)
+    end type scalar 
+  
+    type(scalar) :: out
+    type(scalar) :: in
+  
+    in%ix = 10
+    in%rx = 2.0
+    in%zx = (2, 10)
+  
+    do i = 1, 5
+      in%array(i) = i
+    end do 
+  
+  !$omp target
+    out%ix = in%ix
+    out%rx = in%rx
+    out%zx = in%zx
+  
+    do i = 1, 5
+      out%array(i) = in%array(i)
+    end do 
+  !$omp end target
+  
+    print*, in%ix
+    print*, in%rx
+    print*, in%array
+    write (*,*) in%zx
+
+    print*, out%ix
+    print*, out%rx
+    print*, out%array
+    write (*,*)  out%zx
+end program main
+
+!CHECK: 10
+!CHECK: 2.
+!CHECK: 1 2 3 4 5
+!CHECK: (2.,10.)
+!CHECK: 10
+!CHECK: 2.
+!CHECK: 1 2 3 4 5
+!CHECK: (2.,10.)
diff --git a/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
new file mode 100644
index 000000000000..31774be19146
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
@@ -0,0 +1,101 @@
+! Offloading test checking interaction of an
+! explicit member map from two large nested
+! derived types
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer1
+    real(4) :: i4
+    real(4) :: j4
+    real(4) :: k4
+    end type bottom_layer1
+
+    type :: bottom_layer2
+      integer(4) :: i3
+      integer(4) :: j3
+      integer(4) :: k3
+    end type bottom_layer2 
+
+    type :: middle_layer
+     real(4) :: array_i2(10)
+     real(4) :: i2
+     real(4) :: array_j2(10)
+     type(bottom_layer1) :: nest 
+     type(bottom_layer2) :: nest2 
+    end type middle_layer
+
+    type :: top_layer
+    real(4) :: i
+    integer(4) :: array_i(10)
+    real(4) :: j
+    integer, allocatable :: array_j(:)
+    integer(4) :: k
+    type(middle_layer) :: nested
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+    type(top_layer) :: top_dtype2
+
+    top_dtype2%nested%nest%i4 = 10
+    top_dtype2%nested%nest%j4 = 12
+    top_dtype2%nested%nest%k4 = 54
+    
+    top_dtype2%nested%nest2%i3 = 20
+    top_dtype2%nested%nest2%j3 = 40
+    top_dtype2%nested%nest2%k3 = 60
+    
+    top_dtype2%nested%i2 = 200
+
+      do i = 1, 10
+        top_dtype2%array_i(i) = i
+      end do
+
+!$omp target map(from: top_dtype%nested%nest%j4, top_dtype%nested%nest%i4, top_dtype%nested%nest%k4) &
+!$omp map(from: top_dtype%array_i, top_dtype%nested%nest2%i3, top_dtype%nested%i2) &
+!$omp map(from: top_dtype%nested%nest2%k3, top_dtype%nested%nest2%j3) &
+!$omp map(to: top_dtype2%nested%nest%j4, top_dtype2%nested%nest%i4, top_dtype2%nested%nest%k4) &
+!$omp map(to: top_dtype2%array_i, top_dtype2%nested%nest2%i3, top_dtype2%nested%i2) &
+!$omp map(to: top_dtype2%nested%nest2%k3, top_dtype2%nested%nest2%j3)
+    top_dtype%nested%nest%i4 = top_dtype2%nested%nest%i4
+    top_dtype%nested%nest%j4 = top_dtype2%nested%nest%j4 
+    top_dtype%nested%nest%k4 = top_dtype2%nested%nest%k4
+    
+    top_dtype%nested%nest2%i3 = top_dtype2%nested%nest2%i3
+    top_dtype%nested%nest2%j3 = top_dtype2%nested%nest2%j3
+    top_dtype%nested%nest2%k3 = top_dtype2%nested%nest2%k3
+    
+    top_dtype%nested%i2 = top_dtype2%nested%i2
+
+    do i = 1, 10
+      top_dtype%array_i(i) = top_dtype2%array_i(i)
+    end do
+!$omp end target
+  
+  print *, top_dtype%nested%nest%i4
+  print *, top_dtype%nested%nest%j4
+  print *, top_dtype%nested%nest%k4
+
+  print *, top_dtype%nested%nest2%i3
+  print *, top_dtype%nested%nest2%j3
+  print *, top_dtype%nested%nest2%k3
+  
+  print *, top_dtype%nested%i2
+
+  print *, top_dtype%array_i 
+end program main
+
+!CHECK: 10.
+!CHECK: 12.
+!CHECK: 54.
+!CHECK: 20
+!CHECK: 40
+!CHECK: 60
+!CHECK: 200.
+!CHECK: 1 2 3 4 5 6 7 8 9 10
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
new file mode 100644
index 000000000000..cecfb9e84a59
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
@@ -0,0 +1,47 @@
+! Offloading test checking interaction of two
+! explicit arrau member maps with bounds from 
+! two nested derived types 
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+    type(top_layer) :: top_dtype2
+
+!$omp target map(tofrom: top_dtype%nested%array_i2(4:8), top_dtype2%nested%array_j2(4:8))
+    do i = 4, 8 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    do i = 4, 8 
+      top_dtype2%nested%array_j2(i) = i * 2
+    end do 
+!$omp end target
+  
+  print *, top_dtype%nested%array_i2
+  print *, top_dtype2%nested%array_j2
+end program main
+
+!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0.
+!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
new file mode 100644
index 000000000000..a8762a0829cc
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
@@ -0,0 +1,47 @@
+! Offloading test checking interaction of two
+! explicit array member maps with array bounds 
+! from two nested derived types 
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+    type(top_layer) :: top_dtype2
+
+!$omp target map(tofrom: top_dtype%nested%array_i2(4:8), top_dtype2%nested%array_j2(4:8))
+    do i = 4, 8 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    do i = 4, 8 
+      top_dtype2%nested%array_j2(i) = i * 2
+    end do 
+!$omp end target
+  
+  print *, top_dtype%nested%array_i2
+  print *, top_dtype2%nested%array_j2
+end program main
+
+!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0
+!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
new file mode 100644
index 000000000000..9ecb394dbe46
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
@@ -0,0 +1,47 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of two
+! derived types for a single array member each
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+    type(top_layer) :: top_dtype2
+
+!$omp target map(tofrom: top_dtype%nested%array_i2, top_dtype2%nested%array_j2)
+    do i = 1, 10 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    do i = 1, 10 
+      top_dtype2%nested%array_j2(i) = i * 2
+    end do 
+!$omp end target
+
+  print *, top_dtype%nested%array_i2
+  print *, top_dtype2%nested%array_j2
+end program main
+
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90 b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
new file mode 100644
index 000000000000..3b3ec96b9bab
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
@@ -0,0 +1,49 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of 
+! an array with bounds when mapped to 
+! target using a combination of update,
+! enter and exit directives.
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+        integer(4) :: array(10)
+    end type scalar_array
+
+    type(scalar_array) :: scalar_arr
+
+    do I = 1, 10
+        scalar_arr%array(I) = I + I
+    end do
+
+  !$omp target enter data map(to: scalar_arr%array(3:6))
+
+    ! overwrite our target data with an update.  
+    do I = 1, 10
+        scalar_arr%array(I) = 10
+    end do
+
+  !$omp target update to(scalar_arr%array(3:6))
+
+  ! The compiler/runtime is less friendly about read/write out of 
+  ! bounds when using enter and exit, we have to specifically loop
+  ! over the correct range
+   !$omp target
+    do i=3,6
+        scalar_arr%array(i) = scalar_arr%array(i) + i
+    end do
+  !$omp end target 
+
+  !$omp target exit data map(from: scalar_arr%array(3:6))
+  
+  print*, scalar_arr%array
+end program
+
+!CHECK: 10 10 13 14 15 16 10 10 10 10
diff --git a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90 b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
new file mode 100644
index 000000000000..5f7e9f946826
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
@@ -0,0 +1,49 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of 
+! an array with bounds when mapped to 
+! target using a combination of enter and 
+! exit directives.
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+        integer(4) :: array(10)
+    end type scalar_array
+
+    type(scalar_array) :: scalar_arr
+
+    do I = 1, 10
+        scalar_arr%array(I) = I + I
+    end do
+
+    !$omp target enter data map(to: scalar_arr%array(3:6))
+    
+    ! Shouldn't overwrite data already locked in
+    ! on target via enter, which will then be 
+    ! overwritten by our exit
+    do I = 1, 10
+        scalar_arr%array(I) = 10
+    end do
+
+  ! The compiler/runtime is less friendly about read/write out of 
+  ! bounds when using enter and exit, we have to specifically loop
+  ! over the correct range
+   !$omp target
+    do i=3,6
+        scalar_arr%array(i) = scalar_arr%array(i) + i
+    end do
+  !$omp end target 
+
+  !$omp target exit data map(from: scalar_arr%array(3:6))
+  
+  print*, scalar_arr%array
+end program
+
+!CHECK: 10 10 9 12 15 18 10 10 10 10
diff --git a/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90 b/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
new file mode 100644
index 000000000000..907b16ffedf5
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
@@ -0,0 +1,33 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of 
+! an array when mapped to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+end type scalar_array
+  
+   type(scalar_array) :: scalar_arr
+    
+  !$omp target map(tofrom:scalar_arr%array_y)
+    do i = 1, 10
+      scalar_arr%array_y(i) = i
+    end do
+  !$omp end target
+
+  print *, scalar_arr%array_y
+end program main
+
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
new file mode 100644
index 000000000000..110fb648980c
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
@@ -0,0 +1,45 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of 
+! two arrays with explicit bounds when 
+! mapped to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+    real(4) :: break_0
+    integer(4) :: array_x(3,3,3)
+    real(4) :: break_1
+    integer(4) :: array_y(3,3,3)
+    real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr
+    
+    do i = 1, 3
+      do j = 1, 3
+        do k = 1, 3
+            scalar_arr%array_x(i, j, k) = 42
+            scalar_arr%array_y(i, j, k) = 0 ! Will get overwritten by garbage values in target
+        end do
+       end do
+    end do
+
+  !$omp target map(tofrom:scalar_arr%array_x(1:3, 1:3, 2:2), scalar_arr%array_y(1:3, 1:3, 1:3))
+    do j = 1, 3
+      do k = 1, 3
+        scalar_arr%array_y(k, j, 2) = scalar_arr%array_x(k, j, 2)
+      end do
+    end do
+  !$omp end target
+
+  print *, scalar_arr%array_y
+end program main
+
+!CHECK: 0 0 0 0 0 0 0 0 0 42 42 42 42 42 42 42 42 42 0 0 0 0 0 0 0 0
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
new file mode 100644
index 000000000000..b7f6e2ddfb3b
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
@@ -0,0 +1,38 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of 
+! two arrays with explicit bounds when 
+! mapped to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr
+    
+  do i = 1, 10
+    scalar_arr%array_x(i) = i
+  end do
+
+  !$omp target map(tofrom:scalar_arr%array_x(3:6), scalar_arr%array_y(3:6))
+    do i = 1, 10
+      scalar_arr%array_y(i) = scalar_arr%array_x(i)
+    end do
+  !$omp end target
+
+  print*, scalar_arr%array_y
+end program main
+
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
new file mode 100644
index 000000000000..c44a58dbebc8
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
@@ -0,0 +1,39 @@
+! Offloading test checking interaction of an
+! derived type mapping of two explicit array
+! members to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr
+    
+  do i = 1, 10
+    scalar_arr%array_x(i) = i
+  end do
+
+  !$omp target map(tofrom:scalar_arr%array_x, scalar_arr%array_y)
+    do i = 1, 10
+      scalar_arr%array_y(i) = scalar_arr%array_x(i)
+    end do
+  !$omp end target
+
+  print*, scalar_arr%array_x
+  print*, scalar_arr%array_y
+end program main
+
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
new file mode 100644
index 000000000000..a4205bc0179e
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
@@ -0,0 +1,33 @@
+! Offloading test checking interaction of an
+! derived type mapping of two explicit 
+! members to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar   
+      integer(4) :: ix = 0  
+      real(4) :: rx = 0.0
+      complex(4) :: zx = (0,0)
+      real(4) :: ry = 1.0
+    end type scalar  
+    
+      type(scalar) :: scalar_struct
+      
+    !$omp target map(from:scalar_struct%rx, scalar_struct%ry)
+      scalar_struct%rx = 21.0
+      scalar_struct%ry = 27.0
+    !$omp end target
+  
+    print*, scalar_struct%rx
+    print*, scalar_struct%ry
+end program main
+  
+!CHECK: 21.
+!CHECK: 27.
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90 b/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
index 489c2532a762..8d35a281caf9 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
@@ -17,7 +17,6 @@ program main
     end do
 
     !$omp target enter data map(to: array)
-
     ! Shouldn't overwrite data already locked in
     ! on target via enter, this will then be 
     ! overwritten by our exit
@@ -32,7 +31,6 @@ program main
   !$omp end target 
 
   !$omp target exit data map(from: array)
-
   print*, array
 end program
 
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 b/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
index 3c8c3507ed72..d842cd15c65a 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
@@ -19,7 +19,6 @@ program main
     end do
 
     !$omp target enter data map(to: array(3:6))
-
     ! Shouldn't overwrite data already locked in
     ! on target via enter, which will then be 
     ! overwritten by our exit
@@ -37,7 +36,6 @@ program main
   !$omp end target 
 
   !$omp target exit data map(from: array(3:6))
-
   print *, array
 end program
 
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90 b/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
index 29a0b5ee3e62..70ed3d747829 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
@@ -14,7 +14,6 @@ program main
     scalar = 10
 
     !$omp target enter data map(to: scalar)
-
     !ignored, as we've already attached
     scalar = 20
 
diff --git a/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90 b/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
new file mode 100644
index 000000000000..4cdf41db70d4
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
@@ -0,0 +1,33 @@
+! Offloading test checking interaction of an
+! single explicit member map from a single
+! derived type.
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    real :: test
+    type :: scalar
+        integer(4) :: ix = 0
+        real(4) :: rx = 0.0
+        complex(4) :: zx = (0,0)
+        real(4) :: ry = 1.0
+    end type scalar  
+  
+    type(scalar) :: scalar_struct
+    scalar_struct%rx = 2.0
+    test = 21.0
+
+  !$omp target map(from:scalar_struct%rx)
+    scalar_struct%rx = test
+  !$omp end target
+
+  print *, scalar_struct%rx
+end program main
+
+!CHECK: 21.
diff --git a/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
new file mode 100644
index 000000000000..2412381e62e7
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
@@ -0,0 +1,83 @@
+! Offloading test checking interaction of an
+! explicit member map a large nested derived 
+! type
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer1
+    real(4) :: i4
+    real(4) :: j4
+    real(4) :: k4
+    end type bottom_layer1
+
+    type :: bottom_layer2
+      integer(4) :: i3
+      integer(4) :: j3
+      integer(4) :: k3
+    end type bottom_layer2 
+
+    type :: middle_layer
+     real(4) :: array_i2(10)
+     real(4) :: i2
+     real(4) :: array_j2(10)
+     type(bottom_layer1) :: nest 
+     type(bottom_layer2) :: nest2 
+    end type middle_layer
+
+    type :: top_layer
+    real(4) :: i
+    integer(4) :: array_i(10)
+    real(4) :: j
+    integer, allocatable :: array_j(:)
+    integer(4) :: k
+    type(middle_layer) :: nested
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+
+    top_dtype%nested%nest%j4 = 12
+!$omp target map(tofrom: top_dtype%nested%nest%j4, top_dtype%nested%nest%i4, top_dtype%nested%nest%k4) &
+!$omp map(tofrom: top_dtype%array_i, top_dtype%nested%nest2%i3, top_dtype%nested%i2, top_dtype%nested%nest2%k3, top_dtype%nested%nest2%j3)
+    top_dtype%nested%nest%i4 = 10
+    top_dtype%nested%nest%j4 = 12 + top_dtype%nested%nest%j4
+    top_dtype%nested%nest%k4 = 54
+    
+    top_dtype%nested%nest2%i3 = 20
+    top_dtype%nested%nest2%j3 = 40
+    top_dtype%nested%nest2%k3 = 60
+    
+    top_dtype%nested%i2 = 200
+
+      do i = 1, 10
+        top_dtype%array_i(i) = i
+      end do
+!$omp end target
+  
+  print *, top_dtype%nested%nest%i4
+  print *, top_dtype%nested%nest%j4
+  print *, top_dtype%nested%nest%k4
+
+  print *, top_dtype%nested%nest2%i3
+  print *, top_dtype%nested%nest2%j3
+  print *, top_dtype%nested%nest2%k3
+  
+  print *, top_dtype%nested%i2
+
+  print *, top_dtype%array_i 
+end program main
+
+!CHECK: 10.
+!CHECK: 24.
+!CHECK: 54.
+!CHECK: 20
+!CHECK: 40
+!CHECK: 60
+!CHECK: 200.
+!CHECK: 1 2 3 4 5 6 7 8 9 10
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
new file mode 100644
index 000000000000..b9e8221357c8
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
@@ -0,0 +1,55 @@
+! Offloading test checking interaction of an
+! nested derived type member map of a complex
+! number member
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      complex  :: j2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+      complex :: l
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+
+!$omp target map(tofrom: top_dtype%nested%i2, top_dtype%k, top_dtype%nested%j2, top_dtype%nested%array_i2, top_dtype%l)
+    do i = 1, 10 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    top_dtype%l = (10,20)
+    top_dtype%nested%j2 = (510,210)
+    
+    top_dtype%nested%i2 = 30.30
+    top_dtype%k = 74
+!$omp end target
+  
+  print *, top_dtype%nested%i2
+  print *, top_dtype%k
+  print *, top_dtype%nested%array_i2
+  print *, top_dtype%l
+  print *, top_dtype%nested%j2
+end program main
+
+!CHECK: 30.299999237060547
+!CHECK: 74
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
+!CHECK: (10.,20.)
+!CHECK: (510.,210.)
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
new file mode 100644
index 000000000000..4712ae6210d6
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
@@ -0,0 +1,51 @@
+! Offloading test checking interaction of an
+! nested derived type member map with the 
+! inclusion of an entire nested derived 
+! type being mapped
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+      type(bottom_layer) :: nested2
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+
+!$omp target map(tofrom: top_dtype%k, top_dtype%nested2%array_i2, top_dtype%nested)
+    do i = 1, 10 
+      top_dtype%nested2%array_i2(i) = i * 2
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    top_dtype%nested%i2 = 30.30
+    top_dtype%k = 74
+!$omp end target
+  
+  print *, top_dtype%nested%i2
+  print *, top_dtype%k
+  print *, top_dtype%nested%array_i2
+  print *, top_dtype%nested2%array_i2
+end program main
+
+!CHECK: 30.299999237060547
+!CHECK: 74
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
new file mode 100644
index 000000000000..6f4d5ad5c15b
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
@@ -0,0 +1,47 @@
+! Offloading test checking interaction of an
+! explicit member map from a small nested
+! derived type
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+
+!$omp target map(tofrom: top_dtype%nested%i2, top_dtype%k, top_dtype%nested%array_i2)
+    do i = 1, 10 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    top_dtype%nested%i2 = 30.30
+    top_dtype%k = 74
+!$omp end target
+  
+  print *, top_dtype%nested%i2
+  print *, top_dtype%k
+  print *, top_dtype%nested%array_i2
+end program main
+
+!CHECK: 30.299999237060547
+!CHECK: 74
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
new file mode 100644
index 000000000000..046fc13eb93c
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
@@ -0,0 +1,40 @@
+! Offloading test checking interaction of an
+! single explicit member map from a nested
+! derived type.
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+
+!$omp target map(tofrom: top_dtype%nested%array_i2)
+    do i = 1, 10 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+!$omp end target
+  
+  print *, top_dtype%nested%array_i2
+end program main
+
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90 b/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
new file mode 100644
index 000000000000..b080b437e381
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
@@ -0,0 +1,35 @@
+! Offloading test checking interaction of two
+! derived type's with one explicit member
+! each being mapped with bounds to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+        real(4) :: break_0
+        real(4) :: array_x(10)
+        real(4) :: break_1
+        real(4) :: array_y(10)
+        real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+  !$omp target map(tofrom:scalar_arr1%break_1, scalar_arr2%break_3)
+    scalar_arr2%break_3 = 10
+    scalar_arr1%break_1 = 15
+  !$omp end target
+
+  print*, scalar_arr1%break_1
+  print*, scalar_arr2%break_3
+end program main
+
+!CHECK: 15.
+!CHECK: 10.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90 b/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
new file mode 100644
index 000000000000..7cb7846ed0b2
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
@@ -0,0 +1,39 @@
+! Offloading test checking interaction of two
+! derived type's with a single explicit array
+! member each being mapped with bounds to 
+! target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+
+  !$omp target map(tofrom:scalar_arr1%array_x(3:6), scalar_arr2%array_x(3:6))
+    do i = 3, 6
+      scalar_arr2%array_x(i) = i
+      scalar_arr1%array_x(i) = i
+    end do
+  !$omp end target
+
+  print*, scalar_arr1%array_x  
+  print*, scalar_arr2%array_x
+end program main
+
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90 b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
new file mode 100644
index 000000000000..fbe6b305464a
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
@@ -0,0 +1,35 @@
+! Offloading test checking interaction of two
+! derived type's with a mix of explicit and
+! implicit member mapping to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+        real(4) :: break_0
+        real(4) :: array_x(10)
+        real(4) :: break_1
+        real(4) :: array_y(10)
+        real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+  !$omp target map(tofrom:scalar_arr1%break_1)
+    scalar_arr2%break_3 = 10
+    scalar_arr1%break_1 = 15
+  !$omp end target
+
+  print*, scalar_arr1%break_1
+  print*, scalar_arr2%break_3
+end program main
+
+!CHECK: 15.
+!CHECK: 10.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90 b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
new file mode 100644
index 000000000000..503329d90628
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
@@ -0,0 +1,41 @@
+! Offloading test checking interaction of two
+! derived type's with a mix of explicit and
+! implicit member mapping of arrays to target
+! one with bounds.
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+        real(4) :: break_0
+        real(4) :: array_x(10)
+        real(4) :: break_1
+        real(4) :: array_y(10)
+        real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+  do i = 1, 10
+    scalar_arr1%array_x(i) = i
+  end do 
+
+  !$omp target map(tofrom:scalar_arr2%array_x(3:6))
+    do i = 3, 6
+      scalar_arr2%array_x(i) = scalar_arr1%array_x(i)
+    end do
+  !$omp end target
+
+  print*, scalar_arr1%array_x
+  print*, scalar_arr2%array_x
+end program main
+
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90 b/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
new file mode 100644
index 000000000000..ed350c54dcb2
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
@@ -0,0 +1,51 @@
+! Offloading test checking interaction of two
+! derived type's with two explicit array
+! members each being mapped with bounds to 
+! target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+  do i = 1, 10
+    scalar_arr1%array_x(i) = i
+    scalar_arr2%array_x(i) = i
+  end do
+
+  !$omp target map(tofrom:scalar_arr1%array_x(3:6), scalar_arr1%array_y(3:6), scalar_arr2%array_x(3:6), scalar_arr2%array_y(3:6))
+    do i = 1, 10
+      scalar_arr2%array_y(i) = scalar_arr1%array_x(i)
+    end do
+    
+    do i = 1, 10
+      scalar_arr1%array_y(i) = scalar_arr2%array_x(i)
+    end do
+  !$omp end target
+
+  print*, scalar_arr1%array_x
+  print*, scalar_arr2%array_y
+
+  print*, scalar_arr2%array_x
+  print*, scalar_arr1%array_y
+end program main
+
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90 b/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90
new file mode 100644
index 000000000000..42d9197f6e1e
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90
@@ -0,0 +1,56 @@
+! Offloading test checking interaction of an
+! explicit member map utilising array bounds
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: array
+     real(4) :: array_z(10)
+     real(4) :: break_4
+     real(4) :: array_ix(10)
+    end type array
+
+    type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+    type(array) :: nested
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+  do i = 1, 10
+    scalar_arr1%nested%array_z(i) = i
+    scalar_arr2%nested%array_z(i) = i
+  end do
+
+  !$omp target map(tofrom:scalar_arr1%nested%array_z(3:6), scalar_arr1%nested%array_ix(3:6), scalar_arr2%nested%array_z(3:6), scalar_arr2%nested%array_ix(3:6))
+    do i = 3, 6
+      scalar_arr2%nested%array_ix(i) = scalar_arr1%nested%array_z(i)
+    end do
+    
+    do i = 3, 6
+      scalar_arr1%nested%array_ix(i) = scalar_arr2%nested%array_z(i)
+    end do
+  !$omp end target
+
+  print*, scalar_arr1%nested%array_ix
+  print*, scalar_arr2%nested%array_z
+
+  print*, scalar_arr2%nested%array_ix
+  print*, scalar_arr1%nested%array_z
+end program main
+
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
-- 
GitLab


From 05f88b1769db90c10ca66810720b2092f9d73e34 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Fri, 10 May 2024 20:06:21 +0200
Subject: [PATCH 0431/1206] [libc++][NFC] Replace __apply_cv with __copy_cv or
 __copy_cvref (#90867)

`__apply_cv_t` and `__copy_cvref_t` are very closely related. They are
in fact identical except that `__copy_cvref_t` handles rvalue references
properly. Some uses don't actually require handling of references, so
they are replaced with `__copy_cv_t`.
---
 libcxx/include/CMakeLists.txt                |  1 -
 libcxx/include/__tuple/make_tuple_types.h    |  6 ++--
 libcxx/include/__type_traits/apply_cv.h      | 38 --------------------
 libcxx/include/__type_traits/make_signed.h   |  3 +-
 libcxx/include/__type_traits/make_unsigned.h |  4 +--
 libcxx/include/cwchar                        |  4 +--
 libcxx/include/module.modulemap              |  5 ---
 libcxx/include/tuple                         |  7 ++--
 libcxx/include/type_traits                   |  1 -
 9 files changed, 11 insertions(+), 58 deletions(-)
 delete mode 100644 libcxx/include/__type_traits/apply_cv.h

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index fd7eb125e007..96d7b4037e10 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -733,7 +733,6 @@ set(files
   __type_traits/aligned_storage.h
   __type_traits/aligned_union.h
   __type_traits/alignment_of.h
-  __type_traits/apply_cv.h
   __type_traits/can_extract_key.h
   __type_traits/common_reference.h
   __type_traits/common_type.h
diff --git a/libcxx/include/__tuple/make_tuple_types.h b/libcxx/include/__tuple/make_tuple_types.h
index 43161b17cfa3..9e0fefae2f2f 100644
--- a/libcxx/include/__tuple/make_tuple_types.h
+++ b/libcxx/include/__tuple/make_tuple_types.h
@@ -16,7 +16,7 @@
 #include <__tuple/tuple_indices.h>
 #include <__tuple/tuple_size.h>
 #include <__tuple/tuple_types.h>
-#include <__type_traits/apply_cv.h>
+#include <__type_traits/copy_cvref.h>
 #include <__type_traits/remove_cv.h>
 #include <__type_traits/remove_reference.h>
 #include <cstddef>
@@ -41,7 +41,7 @@ template <template <class...> class _Tuple, class... _Types, size_t... _Idx>
 struct __make_tuple_types_flat<_Tuple<_Types...>, __tuple_indices<_Idx...>> {
   // Specialization for pair, tuple, and __tuple_types
   template <class _Tp>
-  using __apply_quals _LIBCPP_NODEBUG = __tuple_types<__apply_cv_t<_Tp, __type_pack_element<_Idx, _Types...>>...>;
+  using __apply_quals _LIBCPP_NODEBUG = __tuple_types<__copy_cvref_t<_Tp, __type_pack_element<_Idx, _Types...>>...>;
 };
 
 template <class _Vt, size_t _Np, size_t... _Idx>
@@ -49,7 +49,7 @@ struct __make_tuple_types_flat<array<_Vt, _Np>, __tuple_indices<_Idx...>> {
   template <size_t>
   using __value_type = _Vt;
   template <class _Tp>
-  using __apply_quals = __tuple_types<__apply_cv_t<_Tp, __value_type<_Idx>>...>;
+  using __apply_quals = __tuple_types<__copy_cvref_t<_Tp, __value_type<_Idx>>...>;
 };
 
 template <class _Tp,
diff --git a/libcxx/include/__type_traits/apply_cv.h b/libcxx/include/__type_traits/apply_cv.h
deleted file mode 100644
index 723af95b8d92..000000000000
--- a/libcxx/include/__type_traits/apply_cv.h
+++ /dev/null
@@ -1,38 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _LIBCPP___TYPE_TRAITS_APPLY_CV_H
-#define _LIBCPP___TYPE_TRAITS_APPLY_CV_H
-
-#include <__config>
-#include <__type_traits/copy_cv.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-template <class _Tp>
-struct __apply_cv_impl {
-  template <class _Up>
-  using __apply _LIBCPP_NODEBUG = __copy_cv_t<_Tp, _Up>;
-};
-
-template <class _Tp>
-struct __apply_cv_impl<_Tp&> {
-  template <class _Up>
-  using __apply _LIBCPP_NODEBUG = __copy_cv_t<_Tp, _Up>&;
-};
-
-template <class _Tp, class _Up>
-using __apply_cv_t _LIBCPP_NODEBUG = typename __apply_cv_impl<_Tp>::template __apply<_Up>;
-
-_LIBCPP_END_NAMESPACE_STD
-
-#endif // _LIBCPP___TYPE_TRAITS_APPLY_CV_H
diff --git a/libcxx/include/__type_traits/make_signed.h b/libcxx/include/__type_traits/make_signed.h
index 1a8a35f3859d..c1fc009d9ba2 100644
--- a/libcxx/include/__type_traits/make_signed.h
+++ b/libcxx/include/__type_traits/make_signed.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_MAKE_SIGNED_H
 
 #include <__config>
-#include <__type_traits/apply_cv.h>
 #include <__type_traits/is_enum.h>
 #include <__type_traits/is_integral.h>
 #include <__type_traits/nat.h>
@@ -70,7 +69,7 @@ template <> struct __make_signed<__uint128_t,        true> {typedef __int128_t t
 // clang-format on
 
 template <class _Tp>
-using __make_signed_t = __apply_cv_t<_Tp, typename __make_signed<__remove_cv_t<_Tp> >::type>;
+using __make_signed_t = __copy_cv_t<_Tp, typename __make_signed<__remove_cv_t<_Tp> >::type>;
 
 #endif // __has_builtin(__make_signed)
 
diff --git a/libcxx/include/__type_traits/make_unsigned.h b/libcxx/include/__type_traits/make_unsigned.h
index 98967371e773..282cd2d91131 100644
--- a/libcxx/include/__type_traits/make_unsigned.h
+++ b/libcxx/include/__type_traits/make_unsigned.h
@@ -10,8 +10,8 @@
 #define _LIBCPP___TYPE_TRAITS_MAKE_UNSIGNED_H
 
 #include <__config>
-#include <__type_traits/apply_cv.h>
 #include <__type_traits/conditional.h>
+#include <__type_traits/copy_cv.h>
 #include <__type_traits/is_enum.h>
 #include <__type_traits/is_integral.h>
 #include <__type_traits/is_unsigned.h>
@@ -72,7 +72,7 @@ template <> struct __make_unsigned<__uint128_t,        true> {typedef __uint128_
 // clang-format on
 
 template <class _Tp>
-using __make_unsigned_t = __apply_cv_t<_Tp, typename __make_unsigned<__remove_cv_t<_Tp> >::type>;
+using __make_unsigned_t = __copy_cv_t<_Tp, typename __make_unsigned<__remove_cv_t<_Tp> >::type>;
 
 #endif // __has_builtin(__make_unsigned)
 
diff --git a/libcxx/include/cwchar b/libcxx/include/cwchar
index 4cc6f56c389b..08cfac58c846 100644
--- a/libcxx/include/cwchar
+++ b/libcxx/include/cwchar
@@ -103,7 +103,7 @@ size_t wcsrtombs(char* restrict dst, const wchar_t** restrict src, size_t len,
 */
 
 #include <__config>
-#include <__type_traits/apply_cv.h>
+#include <__type_traits/copy_cv.h>
 #include <__type_traits/is_constant_evaluated.h>
 #include <__type_traits/is_equality_comparable.h>
 #include <__type_traits/is_same.h>
@@ -236,7 +236,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __constexpr_wmemchr(_Tp
     wchar_t __value_buffer = 0;
     __builtin_memcpy(&__value_buffer, &__value, sizeof(wchar_t));
     return reinterpret_cast<_Tp*>(
-        __builtin_wmemchr(reinterpret_cast<__apply_cv_t<_Tp, wchar_t>*>(__str), __value_buffer, __count));
+        __builtin_wmemchr(reinterpret_cast<__copy_cv_t<_Tp, wchar_t>*>(__str), __value_buffer, __count));
   }
 #  if _LIBCPP_STD_VER >= 17
   else if constexpr (is_same_v<remove_cv_t<_Tp>, wchar_t>)
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 2974d12500c4..0ee0cdce61bc 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1860,11 +1860,6 @@ module std_private_type_traits_add_volatile                              [system
 module std_private_type_traits_aligned_storage                           [system] { header "__type_traits/aligned_storage.h" }
 module std_private_type_traits_aligned_union                             [system] { header "__type_traits/aligned_union.h" }
 module std_private_type_traits_alignment_of                              [system] { header "__type_traits/alignment_of.h" }
-module std_private_type_traits_apply_cv                                  [system] {
-  header "__type_traits/apply_cv.h"
-  export std_private_type_traits_is_const
-  export std_private_type_traits_is_volatile
-}
 module std_private_type_traits_can_extract_key                           [system] { header "__type_traits/can_extract_key.h" }
 module std_private_type_traits_common_reference                          [system] {
   header "__type_traits/common_reference.h"
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index ace5f2328a45..e7b43af7d13c 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -222,7 +222,6 @@ template <class... Types>
 #include <__tuple/tuple_like_ext.h>
 #include <__tuple/tuple_size.h>
 #include <__tuple/tuple_types.h>
-#include <__type_traits/apply_cv.h>
 #include <__type_traits/common_reference.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/conditional.h>
@@ -1286,14 +1285,14 @@ struct __tuple_cat_return_ref_imp;
 template <class... _Types, size_t... _I0, class _Tuple0>
 struct __tuple_cat_return_ref_imp<tuple<_Types...>, __tuple_indices<_I0...>, _Tuple0> {
   typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tuple0> _T0;
-  typedef tuple<_Types..., __apply_cv_t<_Tuple0, typename tuple_element<_I0, _T0>::type>&&...> type;
+  typedef tuple<_Types..., __copy_cvref_t<_Tuple0, typename tuple_element<_I0, _T0>::type>&&...> type;
 };
 
 template <class... _Types, size_t... _I0, class _Tuple0, class _Tuple1, class... _Tuples>
 struct __tuple_cat_return_ref_imp<tuple<_Types...>, __tuple_indices<_I0...>, _Tuple0, _Tuple1, _Tuples...>
     : public __tuple_cat_return_ref_imp<
           tuple<_Types...,
-                __apply_cv_t<_Tuple0, typename tuple_element<_I0, __libcpp_remove_reference_t<_Tuple0>>::type>&&...>,
+                __copy_cvref_t<_Tuple0, typename tuple_element<_I0, __libcpp_remove_reference_t<_Tuple0>>::type>&&...>,
           typename __make_tuple_indices<tuple_size<__libcpp_remove_reference_t<_Tuple1> >::value>::type,
           _Tuple1,
           _Tuples...> {};
@@ -1327,7 +1326,7 @@ struct __tuple_cat<tuple<_Types...>, __tuple_indices<_I0...>, __tuple_indices<_J
     (void)__t; // avoid unused parameter warning on GCC when _I0 is empty
     typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tuple0> _T0;
     typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tuple1> _T1;
-    return __tuple_cat<tuple<_Types..., __apply_cv_t<_Tuple0, typename tuple_element<_J0, _T0>::type>&&...>,
+    return __tuple_cat<tuple<_Types..., __copy_cvref_t<_Tuple0, typename tuple_element<_J0, _T0>::type>&&...>,
                        typename __make_tuple_indices<sizeof...(_Types) + tuple_size<_T0>::value>::type,
                        typename __make_tuple_indices<tuple_size<_T1>::value>::type>()(
         std::forward_as_tuple(
diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits
index 10f9b881c0e4..aee9fcf4137f 100644
--- a/libcxx/include/type_traits
+++ b/libcxx/include/type_traits
@@ -428,7 +428,6 @@ namespace std
 #include <__type_traits/aligned_storage.h>
 #include <__type_traits/aligned_union.h>
 #include <__type_traits/alignment_of.h>
-#include <__type_traits/apply_cv.h>
 #include <__type_traits/can_extract_key.h>
 #include <__type_traits/common_reference.h>
 #include <__type_traits/common_type.h>
-- 
GitLab


From cfdc2bc72e1747f18516e59bdfa4d7447ce4e0ce Mon Sep 17 00:00:00 2001
From: agozillon <Andrew.Gozillon@amd.com>
Date: Fri, 10 May 2024 20:10:18 +0200
Subject: [PATCH 0432/1206] Revert "[Flang][OpenMP] Derived type member map
 fortran offload runtime tests" (#91784)

Reverts llvm/llvm-project#82850, applied accidentally without the rest of the PR series.
---
 .../target-map-derived-type-full-1.f90        |  45 --------
 .../target-map-derived-type-full-2.f90        |  60 -----------
 ...arget-map-derived-type-full-implicit-1.f90 |  46 --------
 ...arget-map-derived-type-full-implicit-2.f90 |  61 -----------
 ...double-large-nested-dtype-multi-member.f90 | 101 ------------------
 ...t-map-double-nested-dtype-array-bounds.f90 |  47 --------
 ...ouble-nested-dtype-double-array-bounds.f90 |  47 --------
 ...-map-double-nested-dtype-single-member.f90 |  47 --------
 ...pe-arr-bounds-member-enter-exit-update.f90 |  49 ---------
 ...map-dtype-arr-bounds-member-enter-exit.f90 |  49 ---------
 ...dtype-explicit-individual-array-member.f90 |  33 ------
 ...-multi-explicit-array-3D-member-bounds.f90 |  45 --------
 ...ype-multi-explicit-array-member-bounds.f90 |  38 -------
 ...-map-dtype-multi-explicit-array-member.f90 |  39 -------
 ...target-map-dtype-multi-explicit-member.f90 |  33 ------
 .../fortran/target-map-enter-exit-array-2.f90 |   2 +
 .../target-map-enter-exit-array-bounds.f90    |   2 +
 .../fortran/target-map-enter-exit-scalar.f90  |   1 +
 ...target-map-individual-dtype-member-map.f90 |  33 ------
 ...et-map-large-nested-dtype-multi-member.f90 |  83 --------------
 ...target-map-nested-dtype-complex-member.f90 |  55 ----------
 ...target-map-nested-dtype-derived-member.f90 |  51 ---------
 .../target-map-nested-dtype-multi-member.f90  |  47 --------
 .../target-map-nested-dtype-single-member.f90 |  40 -------
 .../target-map-two-dtype-explicit-member.f90  |  35 ------
 ...type-individual-member-array-1D-bounds.f90 |  39 -------
 ...type-mixed-implicit-explicit-capture-1.f90 |  35 ------
 ...type-mixed-implicit-explicit-capture-2.f90 |  41 -------
 ...two-dtype-multi-member-array-1D-bounds.f90 |  51 ---------
 ...-map-two-nested-dtype-member-array-map.f90 |  56 ----------
 30 files changed, 5 insertions(+), 1306 deletions(-)
 delete mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-1.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-2.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
 delete mode 100644 offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90

diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-1.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
deleted file mode 100644
index cb03708554fe..000000000000
--- a/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
+++ /dev/null
@@ -1,45 +0,0 @@
-! Offloading test checking interaction of an
-! explicit derived type mapping when mapped 
-! to target and assinging one derived type
-! to another
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar
-    integer(4) :: ix = 0
-    real(4) :: rx = 0.0
-    complex(4) :: zx = (0,0)
-    end type scalar  
-  
-    type(scalar) :: in
-    type(scalar) :: out
-    in%ix = 10
-    in%rx = 2.0
-    in%zx = (2, 10)
-  
-  !$omp target map(from:out) map(to:in)
-      out = in 
-  !$omp end target
-  
-    print*, in%ix
-    print*, in%rx
-    write (*,*) in%zx
-  
-    print*, out%ix
-    print*, out%rx
-    write (*,*)  out%zx
-end program main
-
-!CHECK: 10
-!CHECK: 2.
-!CHECK: (2.,10.)
-!CHECK: 10
-!CHECK: 2.
-!CHECK: (2.,10.)
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-2.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-2.f90
deleted file mode 100644
index 0095b0fdf86a..000000000000
--- a/offload/test/offloading/fortran/target-map-derived-type-full-2.f90
+++ /dev/null
@@ -1,60 +0,0 @@
-! Offloading test checking interaction of an
-! explicit derived type mapping when mapped to 
-! target and assigning to individual members
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar
-    integer(4) :: ix = 0
-    real(4) :: rx = 0.0
-    complex(4) :: zx = (0,0)
-    integer(4) :: array(5)
-    end type scalar 
-  
-    type(scalar) :: out
-    type(scalar) :: in
-  
-    in%ix = 10
-    in%rx = 2.0
-    in%zx = (2, 10)
-  
-    do i = 1, 5
-      in%array(i) = i
-    end do 
-  
-  !$omp target map(from:out) map(to:in)
-    out%ix = in%ix
-    out%rx = in%rx
-    out%zx = in%zx
-  
-    do i = 1, 5
-      out%array(i) = in%array(i)
-    end do 
-  !$omp end target
-  
-    print*, in%ix
-    print*, in%rx
-    print*, in%array
-    write (*,*) in%zx
-
-    print*, out%ix
-    print*, out%rx
-    print*, out%array
-    write (*,*)  out%zx
-end program main
-
-!CHECK: 10
-!CHECK: 2.
-!CHECK: 1 2 3 4 5
-!CHECK: (2.,10.)
-!CHECK: 10
-!CHECK: 2.
-!CHECK: 1 2 3 4 5
-!CHECK: (2.,10.)
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
deleted file mode 100644
index f57e2c70d155..000000000000
--- a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
+++ /dev/null
@@ -1,46 +0,0 @@
-! Offloading test checking interaction of an
-! implicit derived type mapping when mapped 
-! to target and assinging one derived type
-! to another
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar
-    integer(4) :: ix = 0
-    real(4) :: rx = 0.0
-    complex(4) :: zx = (0,0)
-    end type scalar  
-  
-    type(scalar) :: in
-    type(scalar) :: out
-    in%ix = 10
-    in%rx = 2.0
-    in%zx = (2, 10)
-  
-  !$omp target map(from:out)
-      out = in 
-  !$omp end target
-  
-    print*, in%ix
-    print*, in%rx
-    write (*,*) in%zx
-
-    print*, out%ix
-    print*, out%rx
-    write (*,*)  out%zx
-  end program main
-
-!CHECK: 10
-!CHECK: 2.
-!CHECK: (2.,10.)
-!CHECK: 10
-!CHECK: 2.
-!CHECK: (2.,10.)
-  
\ No newline at end of file
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
deleted file mode 100644
index 92d3454d462a..000000000000
--- a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
+++ /dev/null
@@ -1,61 +0,0 @@
-! Offloading test checking interaction of an
-! explicit derived type mapping when mapped 
-! to target and assinging one derived type
-! to another
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar
-    integer(4) :: ix = 0
-    real(4) :: rx = 0.0
-    complex(4) :: zx = (0,0)
-    integer(4) :: array(5)
-    end type scalar 
-  
-    type(scalar) :: out
-    type(scalar) :: in
-  
-    in%ix = 10
-    in%rx = 2.0
-    in%zx = (2, 10)
-  
-    do i = 1, 5
-      in%array(i) = i
-    end do 
-  
-  !$omp target
-    out%ix = in%ix
-    out%rx = in%rx
-    out%zx = in%zx
-  
-    do i = 1, 5
-      out%array(i) = in%array(i)
-    end do 
-  !$omp end target
-  
-    print*, in%ix
-    print*, in%rx
-    print*, in%array
-    write (*,*) in%zx
-
-    print*, out%ix
-    print*, out%rx
-    print*, out%array
-    write (*,*)  out%zx
-end program main
-
-!CHECK: 10
-!CHECK: 2.
-!CHECK: 1 2 3 4 5
-!CHECK: (2.,10.)
-!CHECK: 10
-!CHECK: 2.
-!CHECK: 1 2 3 4 5
-!CHECK: (2.,10.)
diff --git a/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
deleted file mode 100644
index 31774be19146..000000000000
--- a/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
+++ /dev/null
@@ -1,101 +0,0 @@
-! Offloading test checking interaction of an
-! explicit member map from two large nested
-! derived types
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: bottom_layer1
-    real(4) :: i4
-    real(4) :: j4
-    real(4) :: k4
-    end type bottom_layer1
-
-    type :: bottom_layer2
-      integer(4) :: i3
-      integer(4) :: j3
-      integer(4) :: k3
-    end type bottom_layer2 
-
-    type :: middle_layer
-     real(4) :: array_i2(10)
-     real(4) :: i2
-     real(4) :: array_j2(10)
-     type(bottom_layer1) :: nest 
-     type(bottom_layer2) :: nest2 
-    end type middle_layer
-
-    type :: top_layer
-    real(4) :: i
-    integer(4) :: array_i(10)
-    real(4) :: j
-    integer, allocatable :: array_j(:)
-    integer(4) :: k
-    type(middle_layer) :: nested
-    end type top_layer
-    
-    type(top_layer) :: top_dtype
-    type(top_layer) :: top_dtype2
-
-    top_dtype2%nested%nest%i4 = 10
-    top_dtype2%nested%nest%j4 = 12
-    top_dtype2%nested%nest%k4 = 54
-    
-    top_dtype2%nested%nest2%i3 = 20
-    top_dtype2%nested%nest2%j3 = 40
-    top_dtype2%nested%nest2%k3 = 60
-    
-    top_dtype2%nested%i2 = 200
-
-      do i = 1, 10
-        top_dtype2%array_i(i) = i
-      end do
-
-!$omp target map(from: top_dtype%nested%nest%j4, top_dtype%nested%nest%i4, top_dtype%nested%nest%k4) &
-!$omp map(from: top_dtype%array_i, top_dtype%nested%nest2%i3, top_dtype%nested%i2) &
-!$omp map(from: top_dtype%nested%nest2%k3, top_dtype%nested%nest2%j3) &
-!$omp map(to: top_dtype2%nested%nest%j4, top_dtype2%nested%nest%i4, top_dtype2%nested%nest%k4) &
-!$omp map(to: top_dtype2%array_i, top_dtype2%nested%nest2%i3, top_dtype2%nested%i2) &
-!$omp map(to: top_dtype2%nested%nest2%k3, top_dtype2%nested%nest2%j3)
-    top_dtype%nested%nest%i4 = top_dtype2%nested%nest%i4
-    top_dtype%nested%nest%j4 = top_dtype2%nested%nest%j4 
-    top_dtype%nested%nest%k4 = top_dtype2%nested%nest%k4
-    
-    top_dtype%nested%nest2%i3 = top_dtype2%nested%nest2%i3
-    top_dtype%nested%nest2%j3 = top_dtype2%nested%nest2%j3
-    top_dtype%nested%nest2%k3 = top_dtype2%nested%nest2%k3
-    
-    top_dtype%nested%i2 = top_dtype2%nested%i2
-
-    do i = 1, 10
-      top_dtype%array_i(i) = top_dtype2%array_i(i)
-    end do
-!$omp end target
-  
-  print *, top_dtype%nested%nest%i4
-  print *, top_dtype%nested%nest%j4
-  print *, top_dtype%nested%nest%k4
-
-  print *, top_dtype%nested%nest2%i3
-  print *, top_dtype%nested%nest2%j3
-  print *, top_dtype%nested%nest2%k3
-  
-  print *, top_dtype%nested%i2
-
-  print *, top_dtype%array_i 
-end program main
-
-!CHECK: 10.
-!CHECK: 12.
-!CHECK: 54.
-!CHECK: 20
-!CHECK: 40
-!CHECK: 60
-!CHECK: 200.
-!CHECK: 1 2 3 4 5 6 7 8 9 10
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
deleted file mode 100644
index cecfb9e84a59..000000000000
--- a/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
+++ /dev/null
@@ -1,47 +0,0 @@
-! Offloading test checking interaction of two
-! explicit arrau member maps with bounds from 
-! two nested derived types 
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: bottom_layer
-      real(8) :: i2
-      real(4) :: array_i2(10)
-      real(4) :: array_j2(10)
-    end type bottom_layer
-
-    type :: top_layer
-      real(4) :: i
-      integer(4) :: array_i(10)
-      real(4) :: j
-      type(bottom_layer) :: nested
-      integer, allocatable :: array_j(:)
-      integer(4) :: k
-    end type top_layer
-    
-    type(top_layer) :: top_dtype
-    type(top_layer) :: top_dtype2
-
-!$omp target map(tofrom: top_dtype%nested%array_i2(4:8), top_dtype2%nested%array_j2(4:8))
-    do i = 4, 8 
-      top_dtype%nested%array_i2(i) = i * 2
-    end do 
-
-    do i = 4, 8 
-      top_dtype2%nested%array_j2(i) = i * 2
-    end do 
-!$omp end target
-  
-  print *, top_dtype%nested%array_i2
-  print *, top_dtype2%nested%array_j2
-end program main
-
-!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0.
-!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
deleted file mode 100644
index a8762a0829cc..000000000000
--- a/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
+++ /dev/null
@@ -1,47 +0,0 @@
-! Offloading test checking interaction of two
-! explicit array member maps with array bounds 
-! from two nested derived types 
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: bottom_layer
-      real(8) :: i2
-      real(4) :: array_i2(10)
-      real(4) :: array_j2(10)
-    end type bottom_layer
-
-    type :: top_layer
-      real(4) :: i
-      integer(4) :: array_i(10)
-      real(4) :: j
-      type(bottom_layer) :: nested
-      integer, allocatable :: array_j(:)
-      integer(4) :: k
-    end type top_layer
-    
-    type(top_layer) :: top_dtype
-    type(top_layer) :: top_dtype2
-
-!$omp target map(tofrom: top_dtype%nested%array_i2(4:8), top_dtype2%nested%array_j2(4:8))
-    do i = 4, 8 
-      top_dtype%nested%array_i2(i) = i * 2
-    end do 
-
-    do i = 4, 8 
-      top_dtype2%nested%array_j2(i) = i * 2
-    end do 
-!$omp end target
-  
-  print *, top_dtype%nested%array_i2
-  print *, top_dtype2%nested%array_j2
-end program main
-
-!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0
-!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
deleted file mode 100644
index 9ecb394dbe46..000000000000
--- a/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
+++ /dev/null
@@ -1,47 +0,0 @@
-! Offloading test checking interaction of an
-! explicit derived type member mapping of two
-! derived types for a single array member each
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: bottom_layer
-      real(8) :: i2
-      real(4) :: array_i2(10)
-      real(4) :: array_j2(10)
-    end type bottom_layer
-
-    type :: top_layer
-      real(4) :: i
-      integer(4) :: array_i(10)
-      real(4) :: j
-      type(bottom_layer) :: nested
-      integer, allocatable :: array_j(:)
-      integer(4) :: k
-    end type top_layer
-    
-    type(top_layer) :: top_dtype
-    type(top_layer) :: top_dtype2
-
-!$omp target map(tofrom: top_dtype%nested%array_i2, top_dtype2%nested%array_j2)
-    do i = 1, 10 
-      top_dtype%nested%array_i2(i) = i * 2
-    end do 
-
-    do i = 1, 10 
-      top_dtype2%nested%array_j2(i) = i * 2
-    end do 
-!$omp end target
-
-  print *, top_dtype%nested%array_i2
-  print *, top_dtype2%nested%array_j2
-end program main
-
-!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
-!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90 b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
deleted file mode 100644
index 3b3ec96b9bab..000000000000
--- a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
+++ /dev/null
@@ -1,49 +0,0 @@
-! Offloading test checking interaction of an
-! explicit derived type member mapping of 
-! an array with bounds when mapped to 
-! target using a combination of update,
-! enter and exit directives.
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar_array
-        integer(4) :: array(10)
-    end type scalar_array
-
-    type(scalar_array) :: scalar_arr
-
-    do I = 1, 10
-        scalar_arr%array(I) = I + I
-    end do
-
-  !$omp target enter data map(to: scalar_arr%array(3:6))
-
-    ! overwrite our target data with an update.  
-    do I = 1, 10
-        scalar_arr%array(I) = 10
-    end do
-
-  !$omp target update to(scalar_arr%array(3:6))
-
-  ! The compiler/runtime is less friendly about read/write out of 
-  ! bounds when using enter and exit, we have to specifically loop
-  ! over the correct range
-   !$omp target
-    do i=3,6
-        scalar_arr%array(i) = scalar_arr%array(i) + i
-    end do
-  !$omp end target 
-
-  !$omp target exit data map(from: scalar_arr%array(3:6))
-  
-  print*, scalar_arr%array
-end program
-
-!CHECK: 10 10 13 14 15 16 10 10 10 10
diff --git a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90 b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
deleted file mode 100644
index 5f7e9f946826..000000000000
--- a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
+++ /dev/null
@@ -1,49 +0,0 @@
-! Offloading test checking interaction of an
-! explicit derived type member mapping of 
-! an array with bounds when mapped to 
-! target using a combination of enter and 
-! exit directives.
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar_array
-        integer(4) :: array(10)
-    end type scalar_array
-
-    type(scalar_array) :: scalar_arr
-
-    do I = 1, 10
-        scalar_arr%array(I) = I + I
-    end do
-
-    !$omp target enter data map(to: scalar_arr%array(3:6))
-    
-    ! Shouldn't overwrite data already locked in
-    ! on target via enter, which will then be 
-    ! overwritten by our exit
-    do I = 1, 10
-        scalar_arr%array(I) = 10
-    end do
-
-  ! The compiler/runtime is less friendly about read/write out of 
-  ! bounds when using enter and exit, we have to specifically loop
-  ! over the correct range
-   !$omp target
-    do i=3,6
-        scalar_arr%array(i) = scalar_arr%array(i) + i
-    end do
-  !$omp end target 
-
-  !$omp target exit data map(from: scalar_arr%array(3:6))
-  
-  print*, scalar_arr%array
-end program
-
-!CHECK: 10 10 9 12 15 18 10 10 10 10
diff --git a/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90 b/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
deleted file mode 100644
index 907b16ffedf5..000000000000
--- a/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
+++ /dev/null
@@ -1,33 +0,0 @@
-! Offloading test checking interaction of an
-! explicit derived type member mapping of 
-! an array when mapped to target
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-type :: scalar_array
-    real(4) :: break_0
-    real(4) :: array_x(10)
-    real(4) :: break_1
-    real(4) :: array_y(10)
-    real(4) :: break_3
-end type scalar_array
-  
-   type(scalar_array) :: scalar_arr
-    
-  !$omp target map(tofrom:scalar_arr%array_y)
-    do i = 1, 10
-      scalar_arr%array_y(i) = i
-    end do
-  !$omp end target
-
-  print *, scalar_arr%array_y
-end program main
-
-!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
deleted file mode 100644
index 110fb648980c..000000000000
--- a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
+++ /dev/null
@@ -1,45 +0,0 @@
-! Offloading test checking interaction of an
-! explicit derived type member mapping of 
-! two arrays with explicit bounds when 
-! mapped to target
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar_array
-    real(4) :: break_0
-    integer(4) :: array_x(3,3,3)
-    real(4) :: break_1
-    integer(4) :: array_y(3,3,3)
-    real(4) :: break_3
-    end type scalar_array
-  
-    type(scalar_array) :: scalar_arr
-    
-    do i = 1, 3
-      do j = 1, 3
-        do k = 1, 3
-            scalar_arr%array_x(i, j, k) = 42
-            scalar_arr%array_y(i, j, k) = 0 ! Will get overwritten by garbage values in target
-        end do
-       end do
-    end do
-
-  !$omp target map(tofrom:scalar_arr%array_x(1:3, 1:3, 2:2), scalar_arr%array_y(1:3, 1:3, 1:3))
-    do j = 1, 3
-      do k = 1, 3
-        scalar_arr%array_y(k, j, 2) = scalar_arr%array_x(k, j, 2)
-      end do
-    end do
-  !$omp end target
-
-  print *, scalar_arr%array_y
-end program main
-
-!CHECK: 0 0 0 0 0 0 0 0 0 42 42 42 42 42 42 42 42 42 0 0 0 0 0 0 0 0
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
deleted file mode 100644
index b7f6e2ddfb3b..000000000000
--- a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
+++ /dev/null
@@ -1,38 +0,0 @@
-! Offloading test checking interaction of an
-! explicit derived type member mapping of 
-! two arrays with explicit bounds when 
-! mapped to target
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar_array
-    real(4) :: break_0
-    real(4) :: array_x(10)
-    real(4) :: break_1
-    real(4) :: array_y(10)
-    real(4) :: break_3
-    end type scalar_array
-  
-    type(scalar_array) :: scalar_arr
-    
-  do i = 1, 10
-    scalar_arr%array_x(i) = i
-  end do
-
-  !$omp target map(tofrom:scalar_arr%array_x(3:6), scalar_arr%array_y(3:6))
-    do i = 1, 10
-      scalar_arr%array_y(i) = scalar_arr%array_x(i)
-    end do
-  !$omp end target
-
-  print*, scalar_arr%array_y
-end program main
-
-!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
deleted file mode 100644
index c44a58dbebc8..000000000000
--- a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
+++ /dev/null
@@ -1,39 +0,0 @@
-! Offloading test checking interaction of an
-! derived type mapping of two explicit array
-! members to target
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar_array
-    real(4) :: break_0
-    real(4) :: array_x(10)
-    real(4) :: break_1
-    real(4) :: array_y(10)
-    real(4) :: break_3
-    end type scalar_array
-  
-    type(scalar_array) :: scalar_arr
-    
-  do i = 1, 10
-    scalar_arr%array_x(i) = i
-  end do
-
-  !$omp target map(tofrom:scalar_arr%array_x, scalar_arr%array_y)
-    do i = 1, 10
-      scalar_arr%array_y(i) = scalar_arr%array_x(i)
-    end do
-  !$omp end target
-
-  print*, scalar_arr%array_x
-  print*, scalar_arr%array_y
-end program main
-
-!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
-!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
deleted file mode 100644
index a4205bc0179e..000000000000
--- a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
+++ /dev/null
@@ -1,33 +0,0 @@
-! Offloading test checking interaction of an
-! derived type mapping of two explicit 
-! members to target
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar   
-      integer(4) :: ix = 0  
-      real(4) :: rx = 0.0
-      complex(4) :: zx = (0,0)
-      real(4) :: ry = 1.0
-    end type scalar  
-    
-      type(scalar) :: scalar_struct
-      
-    !$omp target map(from:scalar_struct%rx, scalar_struct%ry)
-      scalar_struct%rx = 21.0
-      scalar_struct%ry = 27.0
-    !$omp end target
-  
-    print*, scalar_struct%rx
-    print*, scalar_struct%ry
-end program main
-  
-!CHECK: 21.
-!CHECK: 27.
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90 b/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
index 8d35a281caf9..489c2532a762 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
@@ -17,6 +17,7 @@ program main
     end do
 
     !$omp target enter data map(to: array)
+
     ! Shouldn't overwrite data already locked in
     ! on target via enter, this will then be 
     ! overwritten by our exit
@@ -31,6 +32,7 @@ program main
   !$omp end target 
 
   !$omp target exit data map(from: array)
+
   print*, array
 end program
 
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 b/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
index d842cd15c65a..3c8c3507ed72 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
@@ -19,6 +19,7 @@ program main
     end do
 
     !$omp target enter data map(to: array(3:6))
+
     ! Shouldn't overwrite data already locked in
     ! on target via enter, which will then be 
     ! overwritten by our exit
@@ -36,6 +37,7 @@ program main
   !$omp end target 
 
   !$omp target exit data map(from: array(3:6))
+
   print *, array
 end program
 
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90 b/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
index 70ed3d747829..29a0b5ee3e62 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
@@ -14,6 +14,7 @@ program main
     scalar = 10
 
     !$omp target enter data map(to: scalar)
+
     !ignored, as we've already attached
     scalar = 20
 
diff --git a/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90 b/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
deleted file mode 100644
index 4cdf41db70d4..000000000000
--- a/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
+++ /dev/null
@@ -1,33 +0,0 @@
-! Offloading test checking interaction of an
-! single explicit member map from a single
-! derived type.
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    real :: test
-    type :: scalar
-        integer(4) :: ix = 0
-        real(4) :: rx = 0.0
-        complex(4) :: zx = (0,0)
-        real(4) :: ry = 1.0
-    end type scalar  
-  
-    type(scalar) :: scalar_struct
-    scalar_struct%rx = 2.0
-    test = 21.0
-
-  !$omp target map(from:scalar_struct%rx)
-    scalar_struct%rx = test
-  !$omp end target
-
-  print *, scalar_struct%rx
-end program main
-
-!CHECK: 21.
diff --git a/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
deleted file mode 100644
index 2412381e62e7..000000000000
--- a/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
+++ /dev/null
@@ -1,83 +0,0 @@
-! Offloading test checking interaction of an
-! explicit member map a large nested derived 
-! type
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: bottom_layer1
-    real(4) :: i4
-    real(4) :: j4
-    real(4) :: k4
-    end type bottom_layer1
-
-    type :: bottom_layer2
-      integer(4) :: i3
-      integer(4) :: j3
-      integer(4) :: k3
-    end type bottom_layer2 
-
-    type :: middle_layer
-     real(4) :: array_i2(10)
-     real(4) :: i2
-     real(4) :: array_j2(10)
-     type(bottom_layer1) :: nest 
-     type(bottom_layer2) :: nest2 
-    end type middle_layer
-
-    type :: top_layer
-    real(4) :: i
-    integer(4) :: array_i(10)
-    real(4) :: j
-    integer, allocatable :: array_j(:)
-    integer(4) :: k
-    type(middle_layer) :: nested
-    end type top_layer
-    
-    type(top_layer) :: top_dtype
-
-    top_dtype%nested%nest%j4 = 12
-!$omp target map(tofrom: top_dtype%nested%nest%j4, top_dtype%nested%nest%i4, top_dtype%nested%nest%k4) &
-!$omp map(tofrom: top_dtype%array_i, top_dtype%nested%nest2%i3, top_dtype%nested%i2, top_dtype%nested%nest2%k3, top_dtype%nested%nest2%j3)
-    top_dtype%nested%nest%i4 = 10
-    top_dtype%nested%nest%j4 = 12 + top_dtype%nested%nest%j4
-    top_dtype%nested%nest%k4 = 54
-    
-    top_dtype%nested%nest2%i3 = 20
-    top_dtype%nested%nest2%j3 = 40
-    top_dtype%nested%nest2%k3 = 60
-    
-    top_dtype%nested%i2 = 200
-
-      do i = 1, 10
-        top_dtype%array_i(i) = i
-      end do
-!$omp end target
-  
-  print *, top_dtype%nested%nest%i4
-  print *, top_dtype%nested%nest%j4
-  print *, top_dtype%nested%nest%k4
-
-  print *, top_dtype%nested%nest2%i3
-  print *, top_dtype%nested%nest2%j3
-  print *, top_dtype%nested%nest2%k3
-  
-  print *, top_dtype%nested%i2
-
-  print *, top_dtype%array_i 
-end program main
-
-!CHECK: 10.
-!CHECK: 24.
-!CHECK: 54.
-!CHECK: 20
-!CHECK: 40
-!CHECK: 60
-!CHECK: 200.
-!CHECK: 1 2 3 4 5 6 7 8 9 10
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
deleted file mode 100644
index b9e8221357c8..000000000000
--- a/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
+++ /dev/null
@@ -1,55 +0,0 @@
-! Offloading test checking interaction of an
-! nested derived type member map of a complex
-! number member
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: bottom_layer
-      real(8) :: i2
-      complex  :: j2
-      real(4) :: array_i2(10)
-      real(4) :: array_j2(10)
-    end type bottom_layer
-
-    type :: top_layer
-      real(4) :: i
-      integer(4) :: array_i(10)
-      real(4) :: j
-      type(bottom_layer) :: nested
-      integer, allocatable :: array_j(:)
-      integer(4) :: k
-      complex :: l
-    end type top_layer
-    
-    type(top_layer) :: top_dtype
-
-!$omp target map(tofrom: top_dtype%nested%i2, top_dtype%k, top_dtype%nested%j2, top_dtype%nested%array_i2, top_dtype%l)
-    do i = 1, 10 
-      top_dtype%nested%array_i2(i) = i * 2
-    end do 
-
-    top_dtype%l = (10,20)
-    top_dtype%nested%j2 = (510,210)
-    
-    top_dtype%nested%i2 = 30.30
-    top_dtype%k = 74
-!$omp end target
-  
-  print *, top_dtype%nested%i2
-  print *, top_dtype%k
-  print *, top_dtype%nested%array_i2
-  print *, top_dtype%l
-  print *, top_dtype%nested%j2
-end program main
-
-!CHECK: 30.299999237060547
-!CHECK: 74
-!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
-!CHECK: (10.,20.)
-!CHECK: (510.,210.)
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
deleted file mode 100644
index 4712ae6210d6..000000000000
--- a/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
+++ /dev/null
@@ -1,51 +0,0 @@
-! Offloading test checking interaction of an
-! nested derived type member map with the 
-! inclusion of an entire nested derived 
-! type being mapped
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: bottom_layer
-      real(8) :: i2
-      real(4) :: array_i2(10)
-      real(4) :: array_j2(10)
-    end type bottom_layer
-
-    type :: top_layer
-      real(4) :: i
-      integer(4) :: array_i(10)
-      real(4) :: j
-      type(bottom_layer) :: nested
-      integer, allocatable :: array_j(:)
-      integer(4) :: k
-      type(bottom_layer) :: nested2
-    end type top_layer
-    
-    type(top_layer) :: top_dtype
-
-!$omp target map(tofrom: top_dtype%k, top_dtype%nested2%array_i2, top_dtype%nested)
-    do i = 1, 10 
-      top_dtype%nested2%array_i2(i) = i * 2
-      top_dtype%nested%array_i2(i) = i * 2
-    end do 
-
-    top_dtype%nested%i2 = 30.30
-    top_dtype%k = 74
-!$omp end target
-  
-  print *, top_dtype%nested%i2
-  print *, top_dtype%k
-  print *, top_dtype%nested%array_i2
-  print *, top_dtype%nested2%array_i2
-end program main
-
-!CHECK: 30.299999237060547
-!CHECK: 74
-!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
-!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
deleted file mode 100644
index 6f4d5ad5c15b..000000000000
--- a/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
+++ /dev/null
@@ -1,47 +0,0 @@
-! Offloading test checking interaction of an
-! explicit member map from a small nested
-! derived type
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: bottom_layer
-      real(8) :: i2
-      real(4) :: array_i2(10)
-      real(4) :: array_j2(10)
-    end type bottom_layer
-
-    type :: top_layer
-      real(4) :: i
-      integer(4) :: array_i(10)
-      real(4) :: j
-      type(bottom_layer) :: nested
-      integer, allocatable :: array_j(:)
-      integer(4) :: k
-    end type top_layer
-    
-    type(top_layer) :: top_dtype
-
-!$omp target map(tofrom: top_dtype%nested%i2, top_dtype%k, top_dtype%nested%array_i2)
-    do i = 1, 10 
-      top_dtype%nested%array_i2(i) = i * 2
-    end do 
-
-    top_dtype%nested%i2 = 30.30
-    top_dtype%k = 74
-!$omp end target
-  
-  print *, top_dtype%nested%i2
-  print *, top_dtype%k
-  print *, top_dtype%nested%array_i2
-end program main
-
-!CHECK: 30.299999237060547
-!CHECK: 74
-!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
deleted file mode 100644
index 046fc13eb93c..000000000000
--- a/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
+++ /dev/null
@@ -1,40 +0,0 @@
-! Offloading test checking interaction of an
-! single explicit member map from a nested
-! derived type.
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: bottom_layer
-      real(8) :: i2
-      real(4) :: array_i2(10)
-      real(4) :: array_j2(10)
-    end type bottom_layer
-
-    type :: top_layer
-      real(4) :: i
-      integer(4) :: array_i(10)
-      real(4) :: j
-      type(bottom_layer) :: nested
-      integer, allocatable :: array_j(:)
-      integer(4) :: k
-    end type top_layer
-    
-    type(top_layer) :: top_dtype
-
-!$omp target map(tofrom: top_dtype%nested%array_i2)
-    do i = 1, 10 
-      top_dtype%nested%array_i2(i) = i * 2
-    end do 
-!$omp end target
-  
-  print *, top_dtype%nested%array_i2
-end program main
-
-!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90 b/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
deleted file mode 100644
index b080b437e381..000000000000
--- a/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
+++ /dev/null
@@ -1,35 +0,0 @@
-! Offloading test checking interaction of two
-! derived type's with one explicit member
-! each being mapped with bounds to target
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar_array
-        real(4) :: break_0
-        real(4) :: array_x(10)
-        real(4) :: break_1
-        real(4) :: array_y(10)
-        real(4) :: break_3
-    end type scalar_array
-  
-    type(scalar_array) :: scalar_arr1
-    type(scalar_array) :: scalar_arr2
-    
-  !$omp target map(tofrom:scalar_arr1%break_1, scalar_arr2%break_3)
-    scalar_arr2%break_3 = 10
-    scalar_arr1%break_1 = 15
-  !$omp end target
-
-  print*, scalar_arr1%break_1
-  print*, scalar_arr2%break_3
-end program main
-
-!CHECK: 15.
-!CHECK: 10.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90 b/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
deleted file mode 100644
index 7cb7846ed0b2..000000000000
--- a/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
+++ /dev/null
@@ -1,39 +0,0 @@
-! Offloading test checking interaction of two
-! derived type's with a single explicit array
-! member each being mapped with bounds to 
-! target
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar_array
-    real(4) :: break_0
-    real(4) :: array_x(10)
-    real(4) :: break_1
-    real(4) :: array_y(10)
-    real(4) :: break_3
-    end type scalar_array
-  
-    type(scalar_array) :: scalar_arr1
-    type(scalar_array) :: scalar_arr2
-    
-
-  !$omp target map(tofrom:scalar_arr1%array_x(3:6), scalar_arr2%array_x(3:6))
-    do i = 3, 6
-      scalar_arr2%array_x(i) = i
-      scalar_arr1%array_x(i) = i
-    end do
-  !$omp end target
-
-  print*, scalar_arr1%array_x  
-  print*, scalar_arr2%array_x
-end program main
-
-!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
-!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90 b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
deleted file mode 100644
index fbe6b305464a..000000000000
--- a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
+++ /dev/null
@@ -1,35 +0,0 @@
-! Offloading test checking interaction of two
-! derived type's with a mix of explicit and
-! implicit member mapping to target
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar_array
-        real(4) :: break_0
-        real(4) :: array_x(10)
-        real(4) :: break_1
-        real(4) :: array_y(10)
-        real(4) :: break_3
-    end type scalar_array
-  
-    type(scalar_array) :: scalar_arr1
-    type(scalar_array) :: scalar_arr2
-    
-  !$omp target map(tofrom:scalar_arr1%break_1)
-    scalar_arr2%break_3 = 10
-    scalar_arr1%break_1 = 15
-  !$omp end target
-
-  print*, scalar_arr1%break_1
-  print*, scalar_arr2%break_3
-end program main
-
-!CHECK: 15.
-!CHECK: 10.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90 b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
deleted file mode 100644
index 503329d90628..000000000000
--- a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
+++ /dev/null
@@ -1,41 +0,0 @@
-! Offloading test checking interaction of two
-! derived type's with a mix of explicit and
-! implicit member mapping of arrays to target
-! one with bounds.
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar_array
-        real(4) :: break_0
-        real(4) :: array_x(10)
-        real(4) :: break_1
-        real(4) :: array_y(10)
-        real(4) :: break_3
-    end type scalar_array
-  
-    type(scalar_array) :: scalar_arr1
-    type(scalar_array) :: scalar_arr2
-    
-  do i = 1, 10
-    scalar_arr1%array_x(i) = i
-  end do 
-
-  !$omp target map(tofrom:scalar_arr2%array_x(3:6))
-    do i = 3, 6
-      scalar_arr2%array_x(i) = scalar_arr1%array_x(i)
-    end do
-  !$omp end target
-
-  print*, scalar_arr1%array_x
-  print*, scalar_arr2%array_x
-end program main
-
-!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
-!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90 b/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
deleted file mode 100644
index ed350c54dcb2..000000000000
--- a/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
+++ /dev/null
@@ -1,51 +0,0 @@
-! Offloading test checking interaction of two
-! derived type's with two explicit array
-! members each being mapped with bounds to 
-! target
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: scalar_array
-    real(4) :: break_0
-    real(4) :: array_x(10)
-    real(4) :: break_1
-    real(4) :: array_y(10)
-    real(4) :: break_3
-    end type scalar_array
-  
-    type(scalar_array) :: scalar_arr1
-    type(scalar_array) :: scalar_arr2
-    
-  do i = 1, 10
-    scalar_arr1%array_x(i) = i
-    scalar_arr2%array_x(i) = i
-  end do
-
-  !$omp target map(tofrom:scalar_arr1%array_x(3:6), scalar_arr1%array_y(3:6), scalar_arr2%array_x(3:6), scalar_arr2%array_y(3:6))
-    do i = 1, 10
-      scalar_arr2%array_y(i) = scalar_arr1%array_x(i)
-    end do
-    
-    do i = 1, 10
-      scalar_arr1%array_y(i) = scalar_arr2%array_x(i)
-    end do
-  !$omp end target
-
-  print*, scalar_arr1%array_x
-  print*, scalar_arr2%array_y
-
-  print*, scalar_arr2%array_x
-  print*, scalar_arr1%array_y
-end program main
-
-!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
-!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
-!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
-!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90 b/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90
deleted file mode 100644
index 42d9197f6e1e..000000000000
--- a/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90
+++ /dev/null
@@ -1,56 +0,0 @@
-! Offloading test checking interaction of an
-! explicit member map utilising array bounds
-! REQUIRES: flang, amdgcn-amd-amdhsa
-! UNSUPPORTED: nvptx64-nvidia-cuda
-! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
-! UNSUPPORTED: aarch64-unknown-linux-gnu
-! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
-! UNSUPPORTED: x86_64-pc-linux-gnu
-! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
-
-! RUN: %libomptarget-compile-fortran-run-and-check-generic
-program main
-    type :: array
-     real(4) :: array_z(10)
-     real(4) :: break_4
-     real(4) :: array_ix(10)
-    end type array
-
-    type :: scalar_array
-    real(4) :: break_0
-    real(4) :: array_x(10)
-    real(4) :: break_1
-    real(4) :: array_y(10)
-    real(4) :: break_3
-    type(array) :: nested
-    end type scalar_array
-  
-    type(scalar_array) :: scalar_arr1
-    type(scalar_array) :: scalar_arr2
-    
-  do i = 1, 10
-    scalar_arr1%nested%array_z(i) = i
-    scalar_arr2%nested%array_z(i) = i
-  end do
-
-  !$omp target map(tofrom:scalar_arr1%nested%array_z(3:6), scalar_arr1%nested%array_ix(3:6), scalar_arr2%nested%array_z(3:6), scalar_arr2%nested%array_ix(3:6))
-    do i = 3, 6
-      scalar_arr2%nested%array_ix(i) = scalar_arr1%nested%array_z(i)
-    end do
-    
-    do i = 3, 6
-      scalar_arr1%nested%array_ix(i) = scalar_arr2%nested%array_z(i)
-    end do
-  !$omp end target
-
-  print*, scalar_arr1%nested%array_ix
-  print*, scalar_arr2%nested%array_z
-
-  print*, scalar_arr2%nested%array_ix
-  print*, scalar_arr1%nested%array_z
-end program main
-
-!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
-!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
-!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
-!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
-- 
GitLab


From 77a59c32105379b289ee3f7b6abbdf483bcb65c1 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Fri, 10 May 2024 11:27:44 -0700
Subject: [PATCH 0433/1206] [ctx_profile] Fix signed-ness in
 CtxInstrProfilingTest.cpp

Follow-up from `265953c`
---
 compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp b/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
index b8d4342c4112..d9f08b1e7efe 100644
--- a/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
+++ b/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
@@ -17,8 +17,8 @@ TEST(ArenaTest, ZeroInit) {
   memset(Buffer, 1, 1024);
   Arena *A = new (Buffer) Arena(10);
   for (auto I = 0U; I < A->size(); ++I)
-    EXPECT_EQ(A->pos()[I], 0);
-  EXPECT_EQ(A->size(), 10);
+    EXPECT_EQ(A->pos()[I], static_cast<char>(0));
+  EXPECT_EQ(A->size(), 10U);
 }
 
 TEST(ArenaTest, Basic) {
-- 
GitLab


From 13af97a70e7202507dcca89d2f732e5126d2bbcd Mon Sep 17 00:00:00 2001
From: Peiming Liu <peiming@google.com>
Date: Fri, 10 May 2024 11:36:01 -0700
Subject: [PATCH 0434/1206] [mlir][sparse] allow multiple COO segments in
 sparse encodings. (#91786)

**NOTE**: we still have implementation holes when handling multiple COO
segments in the encoding. But the format should be considered to be
legal.
---
 .../SparseTensor/IR/SparseTensorDialect.cpp       | 15 ++++++++++-----
 .../Dialect/SparseTensor/roundtrip_encoding.mlir  | 11 +++++++++++
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index 4cc6ee971d4a..4adb1c19096a 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -788,24 +788,29 @@ LogicalResult SparseTensorEncodingAttr::verify(
     return emitError() << "unexpected position bitwidth: " << posWidth;
   if (!acceptBitWidth(crdWidth))
     return emitError() << "unexpected coordinate bitwidth: " << crdWidth;
-  if (auto it = std::find_if(lvlTypes.begin(), lvlTypes.end(), isSingletonLT);
-      it != std::end(lvlTypes)) {
+
+  // Verify every COO segment.
+  auto *it = std::find_if(lvlTypes.begin(), lvlTypes.end(), isSingletonLT);
+  while (it != lvlTypes.end()) {
     if (it == lvlTypes.begin() ||
-        (!isCompressedLT(*(it - 1)) && !isLooseCompressedLT(*(it - 1))))
+        !(it - 1)->isa<LevelFormat::Compressed, LevelFormat::LooseCompressed>())
       return emitError() << "expected compressed or loose_compressed level "
                             "before singleton level";
-    if (!std::all_of(it, lvlTypes.end(),
+
+    auto *curCOOEnd = std::find_if_not(it, lvlTypes.end(), isSingletonLT);
+    if (!std::all_of(it, curCOOEnd,
                      [](LevelType i) { return isSingletonLT(i); }))
       return emitError() << "expected all singleton lvlTypes "
                             "following a singleton level";
     // We can potentially support mixed SoA/AoS singleton levels.
-    if (!std::all_of(it, lvlTypes.end(), [it](LevelType i) {
+    if (!std::all_of(it, curCOOEnd, [it](LevelType i) {
           return it->isa<LevelPropNonDefault::SoA>() ==
                  i.isa<LevelPropNonDefault::SoA>();
         })) {
       return emitError() << "expected all singleton lvlTypes stored in the "
                             "same memory layout (SoA vs AoS).";
     }
+    it = std::find_if(curCOOEnd, lvlTypes.end(), isSingletonLT);
   }
 
   auto lastBatch = std::find_if(lvlTypes.rbegin(), lvlTypes.rend(), isBatchLT);
diff --git a/mlir/test/Dialect/SparseTensor/roundtrip_encoding.mlir b/mlir/test/Dialect/SparseTensor/roundtrip_encoding.mlir
index 7fb1c76c1a1f..44710cad246c 100644
--- a/mlir/test/Dialect/SparseTensor/roundtrip_encoding.mlir
+++ b/mlir/test/Dialect/SparseTensor/roundtrip_encoding.mlir
@@ -156,6 +156,17 @@ func.func private @sparse_coo(tensor<?x?xf32, #COO>)
 
 // -----
 
+#COO_DENSE = #sparse_tensor.encoding<{
+  map = (d0, d1, d2) -> (d0 : compressed(nonunique), d1 : singleton, d2: dense)
+}>
+
+// CHECK-DAG: #[[$COO:.*]] = #sparse_tensor.encoding<{ map = (d0, d1, d2) -> (d0 : compressed(nonunique), d1 : singleton, d2 : dense) }>
+// CHECK-LABEL: func private @sparse_coo_trailing_dense(
+// CHECK-SAME: tensor<?x?x1xf32, #[[$COO]]>)
+func.func private @sparse_coo_trailing_dense(tensor<?x?x1xf32, #COO_DENSE>)
+
+// -----
+
 #BCOO = #sparse_tensor.encoding<{
   map = (d0, d1, d2) -> (d0 : dense, d1 : loose_compressed(nonunique), d2 : singleton)
 }>
-- 
GitLab


From d655054395e2dba2b949e170d7764cc8c9c8a597 Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Fri, 10 May 2024 12:42:17 -0600
Subject: [PATCH 0435/1206] Revert "[DirectX] Fix DXIL part header version
 encoding" (#91791)

Test failures on big endian bots after this change.

Reverts llvm/llvm-project#91506
---
 llvm/include/llvm/BinaryFormat/DXContainer.h |  2 +-
 llvm/include/llvm/TargetParser/Triple.h      |  2 +-
 llvm/lib/MC/MCDXContainerWriter.cpp          |  3 -
 llvm/lib/TargetParser/Triple.cpp             |  2 -
 llvm/test/CodeGen/DirectX/embed-dxil.ll      |  4 +-
 llvm/unittests/Object/DXContainerTest.cpp    | 59 +++-----------------
 6 files changed, 11 insertions(+), 61 deletions(-)

diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index e5fcda63910d..e8d03f806715 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -103,8 +103,8 @@ struct PartHeader {
 
 struct BitcodeHeader {
   uint8_t Magic[4];     // ACSII "DXIL".
-  uint8_t MinorVersion; // DXIL version.
   uint8_t MajorVersion; // DXIL version.
+  uint8_t MinorVersion; // DXIL version.
   uint16_t Unused;
   uint32_t Offset; // Offset to LLVM bitcode (from start of header).
   uint32_t Size;   // Size of LLVM bitcode (in bytes).
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index b3bb354b38ff..8f9d99816931 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -429,7 +429,7 @@ public:
   /// (SubArch).  This should only be called with Vulkan SPIR-V triples.
   VersionTuple getVulkanVersion() const;
 
-  /// Parse the DXIL version number from the OSVersion and DXIL version
+  /// Parse the DXIL version number from the DXIL version
   /// (SubArch).  This should only be called with DXIL triples.
   VersionTuple getDXILVersion() const;
 
diff --git a/llvm/lib/MC/MCDXContainerWriter.cpp b/llvm/lib/MC/MCDXContainerWriter.cpp
index 015899278f37..0580dc7e4282 100644
--- a/llvm/lib/MC/MCDXContainerWriter.cpp
+++ b/llvm/lib/MC/MCDXContainerWriter.cpp
@@ -127,9 +127,6 @@ uint64_t DXContainerObjectWriter::writeObject(MCAssembler &Asm,
       // The program header's size field is in 32-bit words.
       Header.Size = (SectionSize + sizeof(dxbc::ProgramHeader) + 3) / 4;
       memcpy(Header.Bitcode.Magic, "DXIL", 4);
-      VersionTuple DXILVersion = TT.getDXILVersion();
-      Header.Bitcode.MajorVersion = DXILVersion.getMajor();
-      Header.Bitcode.MinorVersion = DXILVersion.getMinor().value_or(0);
       Header.Bitcode.Offset = sizeof(dxbc::BitcodeHeader);
       Header.Bitcode.Size = SectionSize;
       if (sys::IsBigEndianHost)
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index 4fc1ff5aaa05..f8269a51dc0b 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -1510,8 +1510,6 @@ VersionTuple Triple::getDXILVersion() const {
   if (getArch() != dxil || getOS() != ShaderModel)
     llvm_unreachable("invalid DXIL triple");
   StringRef Arch = getArchName();
-  if (getSubArch() == NoSubArch)
-    Arch = getDXILArchNameFromShaderModel(getOSName());
   Arch.consume_front("dxilv");
   VersionTuple DXILVersion = parseVersionFromName(Arch);
   // FIXME: validate DXIL version against Shader Model version.
diff --git a/llvm/test/CodeGen/DirectX/embed-dxil.ll b/llvm/test/CodeGen/DirectX/embed-dxil.ll
index 9f4fb19d86fa..306e5c385b5a 100644
--- a/llvm/test/CodeGen/DirectX/embed-dxil.ll
+++ b/llvm/test/CodeGen/DirectX/embed-dxil.ll
@@ -42,8 +42,8 @@ define i32 @add(i32 %a, i32 %b) {
 ; DXC-NEXT:       MinorVersion:    5
 ; DXC-NEXT:       ShaderKind:      6
 ; DXC-NEXT:       Size:            [[#div(SIZE,4)]]
-; DXC-NEXT:       DXILMajorVersion: 1
-; DXC-NEXT:       DXILMinorVersion: 5
+; DXC-NEXT:       DXILMajorVersion: [[#]]
+; DXC-NEXT:       DXILMinorVersion: [[#]]
 ; DXC-NEXT:       DXILSize:        [[#SIZE - 24]]
 ; DXC-NEXT:       DXIL:            [ 0x42, 0x43, 0xC0, 0xDE,
 ; DXC:      - Name:            SFI0
diff --git a/llvm/unittests/Object/DXContainerTest.cpp b/llvm/unittests/Object/DXContainerTest.cpp
index 098da331ab56..da640225617d 100644
--- a/llvm/unittests/Object/DXContainerTest.cpp
+++ b/llvm/unittests/Object/DXContainerTest.cpp
@@ -126,51 +126,6 @@ TEST(DXCFile, ParseOverlappingParts) {
           "Part offset for part 1 begins before the previous part ends"));
 }
 
-// This test verify DXILMajorVersion and DXILMinorVersion are correctly parsed.
-// This test is based on the binary output constructed from this yaml.
-// --- !dxcontainer
-// Header:
-//   Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-//                      0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
-//   Version:
-//     Major:           1
-//     Minor:           0
-//   PartCount:       1
-// Parts:
-//   - Name:            DXIL
-//     Size:            28
-//     Program:
-//       MajorVersion:    6
-//       MinorVersion:    5
-//       ShaderKind:      5
-//       Size:            8
-//       DXILMajorVersion: 1
-//       DXILMinorVersion: 5
-//       DXILSize:        4
-//       DXIL:            [ 0x42, 0x43, 0xC0, 0xDE, ]
-// ...
-TEST(DXCFile, ParseDXILPart) {
-  uint8_t Buffer[] = {
-      0x44, 0x58, 0x42, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
-      0x48, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
-      0x44, 0x58, 0x49, 0x4c, 0x1c, 0x00, 0x00, 0x00, 0x65, 0x00, 0x05, 0x00,
-      0x08, 0x00, 0x00, 0x00, 0x44, 0x58, 0x49, 0x4c, 0x05, 0x01, 0x00, 0x00,
-      0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x42, 0x43, 0xc0, 0xde};
-  DXContainer C =
-      llvm::cantFail(DXContainer::create(getMemoryBuffer<116>(Buffer)));
-  EXPECT_EQ(C.getHeader().PartCount, 1u);
-  const std::optional<object::DXContainer::DXILData> &DXIL = C.getDXIL();
-  EXPECT_TRUE(DXIL.has_value());
-  dxbc::ProgramHeader Header = DXIL->first;
-  EXPECT_EQ(Header.MajorVersion, 6u);
-  EXPECT_EQ(Header.MinorVersion, 5u);
-  EXPECT_EQ(Header.ShaderKind, 5u);
-  EXPECT_EQ(Header.Size, 8u);
-  EXPECT_EQ(Header.Bitcode.MajorVersion, 1u);
-  EXPECT_EQ(Header.Bitcode.MinorVersion, 5u);
-}
-
 TEST(DXCFile, ParseEmptyParts) {
   uint8_t Buffer[] = {
       0x44, 0x58, 0x42, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -285,8 +240,8 @@ Parts:
       MinorVersion:    0
       ShaderKind:      14
       Size:            6
-      DXILMajorVersion: 1
-      DXILMinorVersion: 0
+      DXILMajorVersion: 0
+      DXILMinorVersion: 1
       DXILSize:        0
 ...
 )";
@@ -406,8 +361,8 @@ Parts:
 //       MinorVersion:    0
 //       ShaderKind:      14
 //       Size:            6
-//       DXILMajorVersion: 1
-//       DXILMinorVersion: 0
+//       DXILMajorVersion: 0
+//       DXILMinorVersion: 1
 //       DXILSize:        0
 //   - Name:            PSV0
 //     Size:            36
@@ -522,7 +477,7 @@ TEST(DXCFile, MaliciousFiles) {
 //
 // --- !dxcontainer
 // Header:
-//   Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+//   Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
 //                      0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
 //   Version:
 //     Major:           1
@@ -536,8 +491,8 @@ TEST(DXCFile, MaliciousFiles) {
 //       MinorVersion:    0
 //       ShaderKind:      14
 //       Size:            6
-//       DXILMajorVersion: 1
-//       DXILMinorVersion: 0
+//       DXILMajorVersion: 0
+//       DXILMinorVersion: 1
 //       DXILSize:        0
 //   - Name:            PSV0
 //     Size:            100
-- 
GitLab


From 88da8756a6f54a8ae441750eaa890c2450b94490 Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Fri, 10 May 2024 20:01:42 +0100
Subject: [PATCH 0436/1206] [AArch64] Combine getActiveLaneMask with
 vector_extract (#81139)

... into a `whilelo` instruction with a pair of predicate registers.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  62 +++++++
 .../AArch64/get-active-lane-mask-extract.ll   | 156 ++++++++++++++++++
 2 files changed, 218 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 29b66a72d21d..4f2f170922ec 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -20535,6 +20535,66 @@ static SDValue convertMergedOpToPredOp(SDNode *N, unsigned Opc,
   return SDValue();
 }
 
+static SDValue tryCombineWhileLo(SDNode *N,
+                                 TargetLowering::DAGCombinerInfo &DCI,
+                                 const AArch64Subtarget *Subtarget) {
+  if (DCI.isBeforeLegalize())
+    return SDValue();
+
+  if (!Subtarget->hasSVE2p1())
+    return SDValue();
+
+  if (!N->hasNUsesOfValue(2, 0))
+    return SDValue();
+
+  const uint64_t HalfSize = N->getValueType(0).getVectorMinNumElements() / 2;
+  if (HalfSize < 2)
+    return SDValue();
+
+  auto It = N->use_begin();
+  SDNode *Lo = *It++;
+  SDNode *Hi = *It;
+
+  if (Lo->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
+      Hi->getOpcode() != ISD::EXTRACT_SUBVECTOR)
+    return SDValue();
+
+  uint64_t OffLo = Lo->getConstantOperandVal(1);
+  uint64_t OffHi = Hi->getConstantOperandVal(1);
+
+  if (OffLo > OffHi) {
+    std::swap(Lo, Hi);
+    std::swap(OffLo, OffHi);
+  }
+
+  if (OffLo != 0 || OffHi != HalfSize)
+    return SDValue();
+
+  EVT HalfVec = Lo->getValueType(0);
+  if (HalfVec != Hi->getValueType(0) ||
+      HalfVec.getVectorElementCount() != ElementCount::getScalable(HalfSize))
+    return SDValue();
+
+  SelectionDAG &DAG = DCI.DAG;
+  SDLoc DL(N);
+  SDValue ID =
+      DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo_x2, DL, MVT::i64);
+  SDValue Idx = N->getOperand(1);
+  SDValue TC = N->getOperand(2);
+  if (Idx.getValueType() != MVT::i64) {
+    Idx = DAG.getZExtOrTrunc(Idx, DL, MVT::i64);
+    TC = DAG.getZExtOrTrunc(TC, DL, MVT::i64);
+  }
+  auto R =
+      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL,
+                  {Lo->getValueType(0), Hi->getValueType(0)}, {ID, Idx, TC});
+
+  DCI.CombineTo(Lo, R.getValue(0));
+  DCI.CombineTo(Hi, R.getValue(1));
+
+  return SDValue(N, 0);
+}
+
 static SDValue performIntrinsicCombine(SDNode *N,
                                        TargetLowering::DAGCombinerInfo &DCI,
                                        const AArch64Subtarget *Subtarget) {
@@ -20832,6 +20892,8 @@ static SDValue performIntrinsicCombine(SDNode *N,
   case Intrinsic::aarch64_sve_ptest_last:
     return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
                     AArch64CC::LAST_ACTIVE);
+  case Intrinsic::aarch64_sve_whilelo:
+    return tryCombineWhileLo(N, DCI, Subtarget);
   }
   return SDValue();
 }
diff --git a/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll b/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
new file mode 100644
index 000000000000..2d84a69f3144
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mattr=+sve    < %s | FileCheck %s -check-prefix CHECK-SVE
+; RUN: llc -mattr=+sve2p1 < %s | FileCheck %s -check-prefix CHECK-SVE2p1
+target triple = "aarch64-linux"
+
+; Test combining of getActiveLaneMask with a pair of extract_vector operations.
+
+define void @test_2x8bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0 {
+; CHECK-SVE-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    whilelo p1.b, w0, w1
+; CHECK-SVE-NEXT:    punpklo p0.h, p1.b
+; CHECK-SVE-NEXT:    punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT:    b use
+;
+; CHECK-SVE2p1-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
+; CHECK-SVE2p1:       // %bb.0:
+; CHECK-SVE2p1-NEXT:    mov w8, w1
+; CHECK-SVE2p1-NEXT:    mov w9, w0
+; CHECK-SVE2p1-NEXT:    whilelo { p0.h, p1.h }, x9, x8
+; CHECK-SVE2p1-NEXT:    b use
+    %r = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 %i, i32 %n)
+    %v0 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 0)
+    %v1 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 8)
+    tail call void @use(<vscale x 8 x i1> %v0, <vscale x 8 x i1> %v1)
+    ret void
+}
+
+define void @test_2x8bit_mask_with_64bit_index_and_trip_count(i64 %i, i64 %n) #0 {
+; CHECK-SVE-LABEL: test_2x8bit_mask_with_64bit_index_and_trip_count:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    whilelo p1.b, x0, x1
+; CHECK-SVE-NEXT:    punpklo p0.h, p1.b
+; CHECK-SVE-NEXT:    punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT:    b use
+;
+; CHECK-SVE2p1-LABEL: test_2x8bit_mask_with_64bit_index_and_trip_count:
+; CHECK-SVE2p1:       // %bb.0:
+; CHECK-SVE2p1-NEXT:    whilelo { p0.h, p1.h }, x0, x1
+; CHECK-SVE2p1-NEXT:    b use
+    %r = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 %i, i64 %n)
+    %v0 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 0)
+    %v1 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 8)
+    tail call void @use(<vscale x 8 x i1> %v0, <vscale x 8 x i1> %v1)
+    ret void
+}
+
+define void @test_edge_case_2x1bit_mask(i64 %i, i64 %n) #0 {
+; CHECK-SVE-LABEL: test_edge_case_2x1bit_mask:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    whilelo p1.d, x0, x1
+; CHECK-SVE-NEXT:    punpklo p0.h, p1.b
+; CHECK-SVE-NEXT:    punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT:    b use
+;
+; CHECK-SVE2p1-LABEL: test_edge_case_2x1bit_mask:
+; CHECK-SVE2p1:       // %bb.0:
+; CHECK-SVE2p1-NEXT:    whilelo p1.d, x0, x1
+; CHECK-SVE2p1-NEXT:    punpklo p0.h, p1.b
+; CHECK-SVE2p1-NEXT:    punpkhi p1.h, p1.b
+; CHECK-SVE2p1-NEXT:    b use
+    %r = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 %i, i64 %n)
+    %v0 = call <vscale x 1 x i1> @llvm.vector.extract.nxv1i1.nxv2i1.i64(<vscale x 2 x i1> %r, i64 0)
+    %v1 = call <vscale x 1 x i1> @llvm.vector.extract.nxv1i1.nxv2i1.i64(<vscale x 2 x i1> %r, i64 1)
+    tail call void @use(<vscale x 1 x i1> %v0, <vscale x 1 x i1> %v1)
+    ret void
+}
+
+define void @test_boring_case_2x2bit_mask(i64 %i, i64 %n) #0 {
+; CHECK-SVE-LABEL: test_boring_case_2x2bit_mask:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    whilelo p1.s, x0, x1
+; CHECK-SVE-NEXT:    punpklo p0.h, p1.b
+; CHECK-SVE-NEXT:    punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT:    b use
+;
+; CHECK-SVE2p1-LABEL: test_boring_case_2x2bit_mask:
+; CHECK-SVE2p1:       // %bb.0:
+; CHECK-SVE2p1-NEXT:    whilelo { p0.d, p1.d }, x0, x1
+; CHECK-SVE2p1-NEXT:    b use
+    %r = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 %i, i64 %n)
+    %v0 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1.i64(<vscale x 4 x i1> %r, i64 0)
+    %v1 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1.i64(<vscale x 4 x i1> %r, i64 2)
+    tail call void @use(<vscale x 2 x i1> %v0, <vscale x 2 x i1> %v1)
+    ret void
+}
+
+; Negative test for when not extracting exactly two halves of the source vector
+define void @test_partial_extract(i64 %i, i64 %n) #0 {
+; CHECK-SVE-LABEL: test_partial_extract:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    whilelo p0.h, x0, x1
+; CHECK-SVE-NEXT:    punpklo p1.h, p0.b
+; CHECK-SVE-NEXT:    punpkhi p2.h, p0.b
+; CHECK-SVE-NEXT:    punpklo p0.h, p1.b
+; CHECK-SVE-NEXT:    punpklo p1.h, p2.b
+; CHECK-SVE-NEXT:    b use
+;
+; CHECK-SVE2p1-LABEL: test_partial_extract:
+; CHECK-SVE2p1:       // %bb.0:
+; CHECK-SVE2p1-NEXT:    whilelo p0.h, x0, x1
+; CHECK-SVE2p1-NEXT:    punpklo p1.h, p0.b
+; CHECK-SVE2p1-NEXT:    punpkhi p2.h, p0.b
+; CHECK-SVE2p1-NEXT:    punpklo p0.h, p1.b
+; CHECK-SVE2p1-NEXT:    punpklo p1.h, p2.b
+; CHECK-SVE2p1-NEXT:    b use
+    %r = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %i, i64 %n)
+    %v0 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 0)
+    %v1 = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 4)
+    tail call void @use(<vscale x 2 x i1> %v0, <vscale x 2 x i1> %v1)
+    ret void
+}
+
+;; Negative test for when extracting a fixed-length vector.
+define void @test_fixed_extract(i64 %i, i64 %n) #0 {
+; CHECK-SVE-LABEL: test_fixed_extract:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    whilelo p0.h, x0, x1
+; CHECK-SVE-NEXT:    cset w8, mi
+; CHECK-SVE-NEXT:    mov z0.h, p0/z, #1 // =0x1
+; CHECK-SVE-NEXT:    umov w9, v0.h[4]
+; CHECK-SVE-NEXT:    umov w10, v0.h[1]
+; CHECK-SVE-NEXT:    umov w11, v0.h[5]
+; CHECK-SVE-NEXT:    fmov s0, w8
+; CHECK-SVE-NEXT:    fmov s1, w9
+; CHECK-SVE-NEXT:    mov v0.s[1], w10
+; CHECK-SVE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SVE-NEXT:    mov v1.s[1], w11
+; CHECK-SVE-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SVE-NEXT:    b use
+;
+; CHECK-SVE2p1-LABEL: test_fixed_extract:
+; CHECK-SVE2p1:       // %bb.0:
+; CHECK-SVE2p1-NEXT:    whilelo p0.h, x0, x1
+; CHECK-SVE2p1-NEXT:    cset w8, mi
+; CHECK-SVE2p1-NEXT:    mov z0.h, p0/z, #1 // =0x1
+; CHECK-SVE2p1-NEXT:    umov w9, v0.h[4]
+; CHECK-SVE2p1-NEXT:    umov w10, v0.h[1]
+; CHECK-SVE2p1-NEXT:    umov w11, v0.h[5]
+; CHECK-SVE2p1-NEXT:    fmov s0, w8
+; CHECK-SVE2p1-NEXT:    fmov s1, w9
+; CHECK-SVE2p1-NEXT:    mov v0.s[1], w10
+; CHECK-SVE2p1-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SVE2p1-NEXT:    mov v1.s[1], w11
+; CHECK-SVE2p1-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SVE2p1-NEXT:    b use
+    %r = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %i, i64 %n)
+    %v0 = call <2 x i1> @llvm.vector.extract.v2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 0)
+    %v1 = call <2 x i1> @llvm.vector.extract.v2i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 4)
+    tail call void @use(<2 x i1> %v0, <2 x i1> %v1)
+    ret void
+}
+
+declare void @use(...)
+
+attributes #0 = { nounwind }
-- 
GitLab


From 00e3620a0e7ceb8ab1c0cae303e40ad2528fdebe Mon Sep 17 00:00:00 2001
From: Joyce <joycebrum@google.com>
Date: Fri, 10 May 2024 16:05:54 -0300
Subject: [PATCH 0437/1206] Generate a new requirements.txt to fix CVEs
 (#90109)

Hi! Here is a patch for #81859 that fix the vulnerabilities found in
gitpython, cryptography, urllib3 and requests.

I have just regenerated the requirements.txt files running pip-compile
again. Fortunately, this was enough to set all the dependencies on safe
versions.

I have also checked if new vulnerabilities were introduced by running
scorecard on my fork, but none has been introduced.

Thanks!

Signed-off-by: Joyce Brum <joycebrum@google.com>
---
 llvm/utils/git/requirements.txt            | 30 ++++++++++------------
 llvm/utils/git/requirements_formatting.txt | 24 ++++++++---------
 2 files changed, 26 insertions(+), 28 deletions(-)

diff --git a/llvm/utils/git/requirements.txt b/llvm/utils/git/requirements.txt
index bed449e6bf9f..0ff62a8ea5be 100644
--- a/llvm/utils/git/requirements.txt
+++ b/llvm/utils/git/requirements.txt
@@ -4,41 +4,39 @@
 #
 #    pip-compile --output-file=requirements.txt requirements.txt.in
 #
-certifi==2023.7.22
+certifi==2024.2.2
     # via
     #   -r requirements.txt.in
     #   requests
-cffi==1.15.1
+cffi==1.16.0
     # via
     #   cryptography
     #   pynacl
-charset-normalizer==2.1.1
+charset-normalizer==3.3.2
     # via requests
-cryptography==41.0.3
+cryptography==42.0.5
     # via pyjwt
-deprecated==1.2.13
+deprecated==1.2.14
     # via pygithub
-gitdb==4.0.9
+gitdb==4.0.11
     # via gitpython
-gitpython==3.1.32
+gitpython==3.1.43
     # via -r requirements.txt.in
-idna==3.4
+idna==3.7
     # via requests
-pycparser==2.21
+pycparser==2.22
     # via cffi
 pygithub==1.59.1
     # via -r requirements.txt.in
-pyjwt[crypto]==2.5.0
+pyjwt[crypto]==2.8.0
     # via pygithub
 pynacl==1.5.0
     # via pygithub
-requests==2.28.1
+requests==2.31.0
     # via pygithub
-smmap==5.0.0
+smmap==5.0.1
     # via gitdb
-types-cryptography==3.3.23.2
-    # via pyjwt
-urllib3==1.26.12
+urllib3==2.2.1
     # via requests
-wrapt==1.14.1
+wrapt==1.16.0
     # via deprecated
diff --git a/llvm/utils/git/requirements_formatting.txt b/llvm/utils/git/requirements_formatting.txt
index ff744f0d4225..4c9dbd8755ab 100644
--- a/llvm/utils/git/requirements_formatting.txt
+++ b/llvm/utils/git/requirements_formatting.txt
@@ -4,37 +4,37 @@
 #
 #    pip-compile --output-file=llvm/utils/git/requirements_formatting.txt llvm/utils/git/requirements_formatting.txt.in
 #
-black==23.9.1
+black==23.12.1
     # via
     #   -r llvm/utils/git/requirements_formatting.txt.in
     #   darker
-certifi==2023.7.22
+certifi==2024.2.2
     # via requests
-cffi==1.15.1
+cffi==1.16.0
     # via
     #   cryptography
     #   pynacl
-charset-normalizer==3.2.0
+charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via black
-cryptography==41.0.3
+cryptography==42.0.5
     # via pyjwt
 darker==1.7.2
     # via -r llvm/utils/git/requirements_formatting.txt.in
 deprecated==1.2.14
     # via pygithub
-idna==3.4
+idna==3.7
     # via requests
 mypy-extensions==1.0.0
     # via black
-packaging==23.1
+packaging==24.0
     # via black
-pathspec==0.11.2
+pathspec==0.12.1
     # via black
-platformdirs==3.10.0
+platformdirs==4.2.1
     # via black
-pycparser==2.21
+pycparser==2.22
     # via cffi
 pygithub==1.59.1
     # via -r llvm/utils/git/requirements_formatting.txt.in
@@ -46,7 +46,7 @@ requests==2.31.0
     # via pygithub
 toml==0.10.2
     # via darker
-urllib3==2.0.4
+urllib3==2.2.1
     # via requests
-wrapt==1.15.0
+wrapt==1.16.0
     # via deprecated
-- 
GitLab


From d009bd7c69ee338b9f550d65edd5ddaa4e032e00 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 10 May 2024 19:08:54 +0000
Subject: [PATCH 0438/1206] [gn build] Port 05f88b1769db

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 9645bff18ae7..9dc06399877f 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -802,7 +802,6 @@ if (current_toolchain == default_toolchain) {
       "__type_traits/aligned_storage.h",
       "__type_traits/aligned_union.h",
       "__type_traits/alignment_of.h",
-      "__type_traits/apply_cv.h",
       "__type_traits/can_extract_key.h",
       "__type_traits/common_reference.h",
       "__type_traits/common_type.h",
-- 
GitLab


From 7fd6cb293975b05406e24aba1cc8e83275cb6bf9 Mon Sep 17 00:00:00 2001
From: Andrew Gozillon <Andrew.Gozillon@amd.com>
Date: Mon, 12 Feb 2024 08:28:20 -0600
Subject: [PATCH 0439/1206] [Flang][OpenMP] Derived type member map fortran
 offload runtime tests

This is a large series of runtime tests that help to add coverage for the specific cases intended to be supported by the PR stack
that extends derived type map support in Flang+OpenMP. Primarily this will add functionality coverage, there's cases where
things may work, but not optimally (or at least similarly to the status quo in Clang), addiitonal IR tests are added in the
relevant segments of the related PRs to test for breakages like that.

Pull Request: https://github.com/llvm/llvm-project/pull/82850
---
 .../target-map-derived-type-full-1.f90        |  45 ++++++++
 .../target-map-derived-type-full-2.f90        |  60 +++++++++++
 ...arget-map-derived-type-full-implicit-1.f90 |  46 ++++++++
 ...arget-map-derived-type-full-implicit-2.f90 |  61 +++++++++++
 ...double-large-nested-dtype-multi-member.f90 | 101 ++++++++++++++++++
 ...t-map-double-nested-dtype-array-bounds.f90 |  47 ++++++++
 ...ouble-nested-dtype-double-array-bounds.f90 |  47 ++++++++
 ...-map-double-nested-dtype-single-member.f90 |  47 ++++++++
 ...pe-arr-bounds-member-enter-exit-update.f90 |  49 +++++++++
 ...map-dtype-arr-bounds-member-enter-exit.f90 |  49 +++++++++
 ...dtype-explicit-individual-array-member.f90 |  33 ++++++
 ...-multi-explicit-array-3D-member-bounds.f90 |  45 ++++++++
 ...ype-multi-explicit-array-member-bounds.f90 |  38 +++++++
 ...-map-dtype-multi-explicit-array-member.f90 |  39 +++++++
 ...target-map-dtype-multi-explicit-member.f90 |  33 ++++++
 .../fortran/target-map-enter-exit-array-2.f90 |   2 -
 .../target-map-enter-exit-array-bounds.f90    |   2 -
 .../fortran/target-map-enter-exit-scalar.f90  |   1 -
 ...target-map-individual-dtype-member-map.f90 |  33 ++++++
 ...et-map-large-nested-dtype-multi-member.f90 |  83 ++++++++++++++
 ...target-map-nested-dtype-complex-member.f90 |  55 ++++++++++
 ...target-map-nested-dtype-derived-member.f90 |  51 +++++++++
 .../target-map-nested-dtype-multi-member.f90  |  47 ++++++++
 .../target-map-nested-dtype-single-member.f90 |  40 +++++++
 .../target-map-two-dtype-explicit-member.f90  |  35 ++++++
 ...type-individual-member-array-1D-bounds.f90 |  39 +++++++
 ...type-mixed-implicit-explicit-capture-1.f90 |  35 ++++++
 ...type-mixed-implicit-explicit-capture-2.f90 |  41 +++++++
 ...two-dtype-multi-member-array-1D-bounds.f90 |  51 +++++++++
 ...-map-two-nested-dtype-member-array-map.f90 |  56 ++++++++++
 30 files changed, 1306 insertions(+), 5 deletions(-)
 create mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-1.f90
 create mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-2.f90
 create mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
 create mode 100644 offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
 create mode 100644 offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
 create mode 100644 offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
 create mode 100644 offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90

diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-1.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
new file mode 100644
index 000000000000..cb03708554fe
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
@@ -0,0 +1,45 @@
+! Offloading test checking interaction of an
+! explicit derived type mapping when mapped 
+! to target and assinging one derived type
+! to another
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar
+    integer(4) :: ix = 0
+    real(4) :: rx = 0.0
+    complex(4) :: zx = (0,0)
+    end type scalar  
+  
+    type(scalar) :: in
+    type(scalar) :: out
+    in%ix = 10
+    in%rx = 2.0
+    in%zx = (2, 10)
+  
+  !$omp target map(from:out) map(to:in)
+      out = in 
+  !$omp end target
+  
+    print*, in%ix
+    print*, in%rx
+    write (*,*) in%zx
+  
+    print*, out%ix
+    print*, out%rx
+    write (*,*)  out%zx
+end program main
+
+!CHECK: 10
+!CHECK: 2.
+!CHECK: (2.,10.)
+!CHECK: 10
+!CHECK: 2.
+!CHECK: (2.,10.)
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-2.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-2.f90
new file mode 100644
index 000000000000..0095b0fdf86a
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-2.f90
@@ -0,0 +1,60 @@
+! Offloading test checking interaction of an
+! explicit derived type mapping when mapped to 
+! target and assigning to individual members
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar
+    integer(4) :: ix = 0
+    real(4) :: rx = 0.0
+    complex(4) :: zx = (0,0)
+    integer(4) :: array(5)
+    end type scalar 
+  
+    type(scalar) :: out
+    type(scalar) :: in
+  
+    in%ix = 10
+    in%rx = 2.0
+    in%zx = (2, 10)
+  
+    do i = 1, 5
+      in%array(i) = i
+    end do 
+  
+  !$omp target map(from:out) map(to:in)
+    out%ix = in%ix
+    out%rx = in%rx
+    out%zx = in%zx
+  
+    do i = 1, 5
+      out%array(i) = in%array(i)
+    end do 
+  !$omp end target
+  
+    print*, in%ix
+    print*, in%rx
+    print*, in%array
+    write (*,*) in%zx
+
+    print*, out%ix
+    print*, out%rx
+    print*, out%array
+    write (*,*)  out%zx
+end program main
+
+!CHECK: 10
+!CHECK: 2.
+!CHECK: 1 2 3 4 5
+!CHECK: (2.,10.)
+!CHECK: 10
+!CHECK: 2.
+!CHECK: 1 2 3 4 5
+!CHECK: (2.,10.)
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
new file mode 100644
index 000000000000..f57e2c70d155
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
@@ -0,0 +1,46 @@
+! Offloading test checking interaction of an
+! implicit derived type mapping when mapped 
+! to target and assinging one derived type
+! to another
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar
+    integer(4) :: ix = 0
+    real(4) :: rx = 0.0
+    complex(4) :: zx = (0,0)
+    end type scalar  
+  
+    type(scalar) :: in
+    type(scalar) :: out
+    in%ix = 10
+    in%rx = 2.0
+    in%zx = (2, 10)
+  
+  !$omp target map(from:out)
+      out = in 
+  !$omp end target
+  
+    print*, in%ix
+    print*, in%rx
+    write (*,*) in%zx
+
+    print*, out%ix
+    print*, out%rx
+    write (*,*)  out%zx
+  end program main
+
+!CHECK: 10
+!CHECK: 2.
+!CHECK: (2.,10.)
+!CHECK: 10
+!CHECK: 2.
+!CHECK: (2.,10.)
+  
\ No newline at end of file
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
new file mode 100644
index 000000000000..92d3454d462a
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
@@ -0,0 +1,61 @@
+! Offloading test checking interaction of an
+! explicit derived type mapping when mapped 
+! to target and assinging one derived type
+! to another
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar
+    integer(4) :: ix = 0
+    real(4) :: rx = 0.0
+    complex(4) :: zx = (0,0)
+    integer(4) :: array(5)
+    end type scalar 
+  
+    type(scalar) :: out
+    type(scalar) :: in
+  
+    in%ix = 10
+    in%rx = 2.0
+    in%zx = (2, 10)
+  
+    do i = 1, 5
+      in%array(i) = i
+    end do 
+  
+  !$omp target
+    out%ix = in%ix
+    out%rx = in%rx
+    out%zx = in%zx
+  
+    do i = 1, 5
+      out%array(i) = in%array(i)
+    end do 
+  !$omp end target
+  
+    print*, in%ix
+    print*, in%rx
+    print*, in%array
+    write (*,*) in%zx
+
+    print*, out%ix
+    print*, out%rx
+    print*, out%array
+    write (*,*)  out%zx
+end program main
+
+!CHECK: 10
+!CHECK: 2.
+!CHECK: 1 2 3 4 5
+!CHECK: (2.,10.)
+!CHECK: 10
+!CHECK: 2.
+!CHECK: 1 2 3 4 5
+!CHECK: (2.,10.)
diff --git a/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
new file mode 100644
index 000000000000..31774be19146
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
@@ -0,0 +1,101 @@
+! Offloading test checking interaction of an
+! explicit member map from two large nested
+! derived types
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer1
+    real(4) :: i4
+    real(4) :: j4
+    real(4) :: k4
+    end type bottom_layer1
+
+    type :: bottom_layer2
+      integer(4) :: i3
+      integer(4) :: j3
+      integer(4) :: k3
+    end type bottom_layer2 
+
+    type :: middle_layer
+     real(4) :: array_i2(10)
+     real(4) :: i2
+     real(4) :: array_j2(10)
+     type(bottom_layer1) :: nest 
+     type(bottom_layer2) :: nest2 
+    end type middle_layer
+
+    type :: top_layer
+    real(4) :: i
+    integer(4) :: array_i(10)
+    real(4) :: j
+    integer, allocatable :: array_j(:)
+    integer(4) :: k
+    type(middle_layer) :: nested
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+    type(top_layer) :: top_dtype2
+
+    top_dtype2%nested%nest%i4 = 10
+    top_dtype2%nested%nest%j4 = 12
+    top_dtype2%nested%nest%k4 = 54
+    
+    top_dtype2%nested%nest2%i3 = 20
+    top_dtype2%nested%nest2%j3 = 40
+    top_dtype2%nested%nest2%k3 = 60
+    
+    top_dtype2%nested%i2 = 200
+
+      do i = 1, 10
+        top_dtype2%array_i(i) = i
+      end do
+
+!$omp target map(from: top_dtype%nested%nest%j4, top_dtype%nested%nest%i4, top_dtype%nested%nest%k4) &
+!$omp map(from: top_dtype%array_i, top_dtype%nested%nest2%i3, top_dtype%nested%i2) &
+!$omp map(from: top_dtype%nested%nest2%k3, top_dtype%nested%nest2%j3) &
+!$omp map(to: top_dtype2%nested%nest%j4, top_dtype2%nested%nest%i4, top_dtype2%nested%nest%k4) &
+!$omp map(to: top_dtype2%array_i, top_dtype2%nested%nest2%i3, top_dtype2%nested%i2) &
+!$omp map(to: top_dtype2%nested%nest2%k3, top_dtype2%nested%nest2%j3)
+    top_dtype%nested%nest%i4 = top_dtype2%nested%nest%i4
+    top_dtype%nested%nest%j4 = top_dtype2%nested%nest%j4 
+    top_dtype%nested%nest%k4 = top_dtype2%nested%nest%k4
+    
+    top_dtype%nested%nest2%i3 = top_dtype2%nested%nest2%i3
+    top_dtype%nested%nest2%j3 = top_dtype2%nested%nest2%j3
+    top_dtype%nested%nest2%k3 = top_dtype2%nested%nest2%k3
+    
+    top_dtype%nested%i2 = top_dtype2%nested%i2
+
+    do i = 1, 10
+      top_dtype%array_i(i) = top_dtype2%array_i(i)
+    end do
+!$omp end target
+  
+  print *, top_dtype%nested%nest%i4
+  print *, top_dtype%nested%nest%j4
+  print *, top_dtype%nested%nest%k4
+
+  print *, top_dtype%nested%nest2%i3
+  print *, top_dtype%nested%nest2%j3
+  print *, top_dtype%nested%nest2%k3
+  
+  print *, top_dtype%nested%i2
+
+  print *, top_dtype%array_i 
+end program main
+
+!CHECK: 10.
+!CHECK: 12.
+!CHECK: 54.
+!CHECK: 20
+!CHECK: 40
+!CHECK: 60
+!CHECK: 200.
+!CHECK: 1 2 3 4 5 6 7 8 9 10
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
new file mode 100644
index 000000000000..cecfb9e84a59
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
@@ -0,0 +1,47 @@
+! Offloading test checking interaction of two
+! explicit arrau member maps with bounds from 
+! two nested derived types 
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+    type(top_layer) :: top_dtype2
+
+!$omp target map(tofrom: top_dtype%nested%array_i2(4:8), top_dtype2%nested%array_j2(4:8))
+    do i = 4, 8 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    do i = 4, 8 
+      top_dtype2%nested%array_j2(i) = i * 2
+    end do 
+!$omp end target
+  
+  print *, top_dtype%nested%array_i2
+  print *, top_dtype2%nested%array_j2
+end program main
+
+!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0.
+!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
new file mode 100644
index 000000000000..a8762a0829cc
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
@@ -0,0 +1,47 @@
+! Offloading test checking interaction of two
+! explicit array member maps with array bounds 
+! from two nested derived types 
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+    type(top_layer) :: top_dtype2
+
+!$omp target map(tofrom: top_dtype%nested%array_i2(4:8), top_dtype2%nested%array_j2(4:8))
+    do i = 4, 8 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    do i = 4, 8 
+      top_dtype2%nested%array_j2(i) = i * 2
+    end do 
+!$omp end target
+  
+  print *, top_dtype%nested%array_i2
+  print *, top_dtype2%nested%array_j2
+end program main
+
+!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0
+!CHECK: 0. 0. 0. 8. 10. 12. 14. 16. 0. 0
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
new file mode 100644
index 000000000000..9ecb394dbe46
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
@@ -0,0 +1,47 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of two
+! derived types for a single array member each
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+    type(top_layer) :: top_dtype2
+
+!$omp target map(tofrom: top_dtype%nested%array_i2, top_dtype2%nested%array_j2)
+    do i = 1, 10 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    do i = 1, 10 
+      top_dtype2%nested%array_j2(i) = i * 2
+    end do 
+!$omp end target
+
+  print *, top_dtype%nested%array_i2
+  print *, top_dtype2%nested%array_j2
+end program main
+
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90 b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
new file mode 100644
index 000000000000..3b3ec96b9bab
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
@@ -0,0 +1,49 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of 
+! an array with bounds when mapped to 
+! target using a combination of update,
+! enter and exit directives.
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+        integer(4) :: array(10)
+    end type scalar_array
+
+    type(scalar_array) :: scalar_arr
+
+    do I = 1, 10
+        scalar_arr%array(I) = I + I
+    end do
+
+  !$omp target enter data map(to: scalar_arr%array(3:6))
+
+    ! overwrite our target data with an update.  
+    do I = 1, 10
+        scalar_arr%array(I) = 10
+    end do
+
+  !$omp target update to(scalar_arr%array(3:6))
+
+  ! The compiler/runtime is less friendly about read/write out of 
+  ! bounds when using enter and exit, we have to specifically loop
+  ! over the correct range
+   !$omp target
+    do i=3,6
+        scalar_arr%array(i) = scalar_arr%array(i) + i
+    end do
+  !$omp end target 
+
+  !$omp target exit data map(from: scalar_arr%array(3:6))
+  
+  print*, scalar_arr%array
+end program
+
+!CHECK: 10 10 13 14 15 16 10 10 10 10
diff --git a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90 b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
new file mode 100644
index 000000000000..5f7e9f946826
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
@@ -0,0 +1,49 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of 
+! an array with bounds when mapped to 
+! target using a combination of enter and 
+! exit directives.
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+        integer(4) :: array(10)
+    end type scalar_array
+
+    type(scalar_array) :: scalar_arr
+
+    do I = 1, 10
+        scalar_arr%array(I) = I + I
+    end do
+
+    !$omp target enter data map(to: scalar_arr%array(3:6))
+    
+    ! Shouldn't overwrite data already locked in
+    ! on target via enter, which will then be 
+    ! overwritten by our exit
+    do I = 1, 10
+        scalar_arr%array(I) = 10
+    end do
+
+  ! The compiler/runtime is less friendly about read/write out of 
+  ! bounds when using enter and exit, we have to specifically loop
+  ! over the correct range
+   !$omp target
+    do i=3,6
+        scalar_arr%array(i) = scalar_arr%array(i) + i
+    end do
+  !$omp end target 
+
+  !$omp target exit data map(from: scalar_arr%array(3:6))
+  
+  print*, scalar_arr%array
+end program
+
+!CHECK: 10 10 9 12 15 18 10 10 10 10
diff --git a/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90 b/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
new file mode 100644
index 000000000000..907b16ffedf5
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
@@ -0,0 +1,33 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of 
+! an array when mapped to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+end type scalar_array
+  
+   type(scalar_array) :: scalar_arr
+    
+  !$omp target map(tofrom:scalar_arr%array_y)
+    do i = 1, 10
+      scalar_arr%array_y(i) = i
+    end do
+  !$omp end target
+
+  print *, scalar_arr%array_y
+end program main
+
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
new file mode 100644
index 000000000000..110fb648980c
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
@@ -0,0 +1,45 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of 
+! two arrays with explicit bounds when 
+! mapped to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+    real(4) :: break_0
+    integer(4) :: array_x(3,3,3)
+    real(4) :: break_1
+    integer(4) :: array_y(3,3,3)
+    real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr
+    
+    do i = 1, 3
+      do j = 1, 3
+        do k = 1, 3
+            scalar_arr%array_x(i, j, k) = 42
+            scalar_arr%array_y(i, j, k) = 0 ! Will get overwritten by garbage values in target
+        end do
+       end do
+    end do
+
+  !$omp target map(tofrom:scalar_arr%array_x(1:3, 1:3, 2:2), scalar_arr%array_y(1:3, 1:3, 1:3))
+    do j = 1, 3
+      do k = 1, 3
+        scalar_arr%array_y(k, j, 2) = scalar_arr%array_x(k, j, 2)
+      end do
+    end do
+  !$omp end target
+
+  print *, scalar_arr%array_y
+end program main
+
+!CHECK: 0 0 0 0 0 0 0 0 0 42 42 42 42 42 42 42 42 42 0 0 0 0 0 0 0 0
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
new file mode 100644
index 000000000000..b7f6e2ddfb3b
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
@@ -0,0 +1,38 @@
+! Offloading test checking interaction of an
+! explicit derived type member mapping of 
+! two arrays with explicit bounds when 
+! mapped to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr
+    
+  do i = 1, 10
+    scalar_arr%array_x(i) = i
+  end do
+
+  !$omp target map(tofrom:scalar_arr%array_x(3:6), scalar_arr%array_y(3:6))
+    do i = 1, 10
+      scalar_arr%array_y(i) = scalar_arr%array_x(i)
+    end do
+  !$omp end target
+
+  print*, scalar_arr%array_y
+end program main
+
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
new file mode 100644
index 000000000000..c44a58dbebc8
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
@@ -0,0 +1,39 @@
+! Offloading test checking interaction of an
+! derived type mapping of two explicit array
+! members to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr
+    
+  do i = 1, 10
+    scalar_arr%array_x(i) = i
+  end do
+
+  !$omp target map(tofrom:scalar_arr%array_x, scalar_arr%array_y)
+    do i = 1, 10
+      scalar_arr%array_y(i) = scalar_arr%array_x(i)
+    end do
+  !$omp end target
+
+  print*, scalar_arr%array_x
+  print*, scalar_arr%array_y
+end program main
+
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
new file mode 100644
index 000000000000..a4205bc0179e
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
@@ -0,0 +1,33 @@
+! Offloading test checking interaction of an
+! derived type mapping of two explicit 
+! members to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar   
+      integer(4) :: ix = 0  
+      real(4) :: rx = 0.0
+      complex(4) :: zx = (0,0)
+      real(4) :: ry = 1.0
+    end type scalar  
+    
+      type(scalar) :: scalar_struct
+      
+    !$omp target map(from:scalar_struct%rx, scalar_struct%ry)
+      scalar_struct%rx = 21.0
+      scalar_struct%ry = 27.0
+    !$omp end target
+  
+    print*, scalar_struct%rx
+    print*, scalar_struct%ry
+end program main
+  
+!CHECK: 21.
+!CHECK: 27.
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90 b/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
index 489c2532a762..8d35a281caf9 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
@@ -17,7 +17,6 @@ program main
     end do
 
     !$omp target enter data map(to: array)
-
     ! Shouldn't overwrite data already locked in
     ! on target via enter, this will then be 
     ! overwritten by our exit
@@ -32,7 +31,6 @@ program main
   !$omp end target 
 
   !$omp target exit data map(from: array)
-
   print*, array
 end program
 
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 b/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
index 3c8c3507ed72..d842cd15c65a 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
@@ -19,7 +19,6 @@ program main
     end do
 
     !$omp target enter data map(to: array(3:6))
-
     ! Shouldn't overwrite data already locked in
     ! on target via enter, which will then be 
     ! overwritten by our exit
@@ -37,7 +36,6 @@ program main
   !$omp end target 
 
   !$omp target exit data map(from: array(3:6))
-
   print *, array
 end program
 
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90 b/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
index 29a0b5ee3e62..70ed3d747829 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
@@ -14,7 +14,6 @@ program main
     scalar = 10
 
     !$omp target enter data map(to: scalar)
-
     !ignored, as we've already attached
     scalar = 20
 
diff --git a/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90 b/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
new file mode 100644
index 000000000000..4cdf41db70d4
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
@@ -0,0 +1,33 @@
+! Offloading test checking interaction of an
+! single explicit member map from a single
+! derived type.
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    real :: test
+    type :: scalar
+        integer(4) :: ix = 0
+        real(4) :: rx = 0.0
+        complex(4) :: zx = (0,0)
+        real(4) :: ry = 1.0
+    end type scalar  
+  
+    type(scalar) :: scalar_struct
+    scalar_struct%rx = 2.0
+    test = 21.0
+
+  !$omp target map(from:scalar_struct%rx)
+    scalar_struct%rx = test
+  !$omp end target
+
+  print *, scalar_struct%rx
+end program main
+
+!CHECK: 21.
diff --git a/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
new file mode 100644
index 000000000000..2412381e62e7
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
@@ -0,0 +1,83 @@
+! Offloading test checking interaction of an
+! explicit member map a large nested derived 
+! type
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer1
+    real(4) :: i4
+    real(4) :: j4
+    real(4) :: k4
+    end type bottom_layer1
+
+    type :: bottom_layer2
+      integer(4) :: i3
+      integer(4) :: j3
+      integer(4) :: k3
+    end type bottom_layer2 
+
+    type :: middle_layer
+     real(4) :: array_i2(10)
+     real(4) :: i2
+     real(4) :: array_j2(10)
+     type(bottom_layer1) :: nest 
+     type(bottom_layer2) :: nest2 
+    end type middle_layer
+
+    type :: top_layer
+    real(4) :: i
+    integer(4) :: array_i(10)
+    real(4) :: j
+    integer, allocatable :: array_j(:)
+    integer(4) :: k
+    type(middle_layer) :: nested
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+
+    top_dtype%nested%nest%j4 = 12
+!$omp target map(tofrom: top_dtype%nested%nest%j4, top_dtype%nested%nest%i4, top_dtype%nested%nest%k4) &
+!$omp map(tofrom: top_dtype%array_i, top_dtype%nested%nest2%i3, top_dtype%nested%i2, top_dtype%nested%nest2%k3, top_dtype%nested%nest2%j3)
+    top_dtype%nested%nest%i4 = 10
+    top_dtype%nested%nest%j4 = 12 + top_dtype%nested%nest%j4
+    top_dtype%nested%nest%k4 = 54
+    
+    top_dtype%nested%nest2%i3 = 20
+    top_dtype%nested%nest2%j3 = 40
+    top_dtype%nested%nest2%k3 = 60
+    
+    top_dtype%nested%i2 = 200
+
+      do i = 1, 10
+        top_dtype%array_i(i) = i
+      end do
+!$omp end target
+  
+  print *, top_dtype%nested%nest%i4
+  print *, top_dtype%nested%nest%j4
+  print *, top_dtype%nested%nest%k4
+
+  print *, top_dtype%nested%nest2%i3
+  print *, top_dtype%nested%nest2%j3
+  print *, top_dtype%nested%nest2%k3
+  
+  print *, top_dtype%nested%i2
+
+  print *, top_dtype%array_i 
+end program main
+
+!CHECK: 10.
+!CHECK: 24.
+!CHECK: 54.
+!CHECK: 20
+!CHECK: 40
+!CHECK: 60
+!CHECK: 200.
+!CHECK: 1 2 3 4 5 6 7 8 9 10
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
new file mode 100644
index 000000000000..b9e8221357c8
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
@@ -0,0 +1,55 @@
+! Offloading test checking interaction of an
+! nested derived type member map of a complex
+! number member
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      complex  :: j2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+      complex :: l
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+
+!$omp target map(tofrom: top_dtype%nested%i2, top_dtype%k, top_dtype%nested%j2, top_dtype%nested%array_i2, top_dtype%l)
+    do i = 1, 10 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    top_dtype%l = (10,20)
+    top_dtype%nested%j2 = (510,210)
+    
+    top_dtype%nested%i2 = 30.30
+    top_dtype%k = 74
+!$omp end target
+  
+  print *, top_dtype%nested%i2
+  print *, top_dtype%k
+  print *, top_dtype%nested%array_i2
+  print *, top_dtype%l
+  print *, top_dtype%nested%j2
+end program main
+
+!CHECK: 30.299999237060547
+!CHECK: 74
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
+!CHECK: (10.,20.)
+!CHECK: (510.,210.)
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
new file mode 100644
index 000000000000..4712ae6210d6
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
@@ -0,0 +1,51 @@
+! Offloading test checking interaction of an
+! nested derived type member map with the 
+! inclusion of an entire nested derived 
+! type being mapped
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+      type(bottom_layer) :: nested2
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+
+!$omp target map(tofrom: top_dtype%k, top_dtype%nested2%array_i2, top_dtype%nested)
+    do i = 1, 10 
+      top_dtype%nested2%array_i2(i) = i * 2
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    top_dtype%nested%i2 = 30.30
+    top_dtype%k = 74
+!$omp end target
+  
+  print *, top_dtype%nested%i2
+  print *, top_dtype%k
+  print *, top_dtype%nested%array_i2
+  print *, top_dtype%nested2%array_i2
+end program main
+
+!CHECK: 30.299999237060547
+!CHECK: 74
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
new file mode 100644
index 000000000000..6f4d5ad5c15b
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
@@ -0,0 +1,47 @@
+! Offloading test checking interaction of an
+! explicit member map from a small nested
+! derived type
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+
+!$omp target map(tofrom: top_dtype%nested%i2, top_dtype%k, top_dtype%nested%array_i2)
+    do i = 1, 10 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+
+    top_dtype%nested%i2 = 30.30
+    top_dtype%k = 74
+!$omp end target
+  
+  print *, top_dtype%nested%i2
+  print *, top_dtype%k
+  print *, top_dtype%nested%array_i2
+end program main
+
+!CHECK: 30.299999237060547
+!CHECK: 74
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
new file mode 100644
index 000000000000..046fc13eb93c
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
@@ -0,0 +1,40 @@
+! Offloading test checking interaction of an
+! single explicit member map from a nested
+! derived type.
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: bottom_layer
+      real(8) :: i2
+      real(4) :: array_i2(10)
+      real(4) :: array_j2(10)
+    end type bottom_layer
+
+    type :: top_layer
+      real(4) :: i
+      integer(4) :: array_i(10)
+      real(4) :: j
+      type(bottom_layer) :: nested
+      integer, allocatable :: array_j(:)
+      integer(4) :: k
+    end type top_layer
+    
+    type(top_layer) :: top_dtype
+
+!$omp target map(tofrom: top_dtype%nested%array_i2)
+    do i = 1, 10 
+      top_dtype%nested%array_i2(i) = i * 2
+    end do 
+!$omp end target
+  
+  print *, top_dtype%nested%array_i2
+end program main
+
+!CHECK: 2. 4. 6. 8. 10. 12. 14. 16. 18. 20.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90 b/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
new file mode 100644
index 000000000000..b080b437e381
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
@@ -0,0 +1,35 @@
+! Offloading test checking interaction of two
+! derived type's with one explicit member
+! each being mapped with bounds to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+        real(4) :: break_0
+        real(4) :: array_x(10)
+        real(4) :: break_1
+        real(4) :: array_y(10)
+        real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+  !$omp target map(tofrom:scalar_arr1%break_1, scalar_arr2%break_3)
+    scalar_arr2%break_3 = 10
+    scalar_arr1%break_1 = 15
+  !$omp end target
+
+  print*, scalar_arr1%break_1
+  print*, scalar_arr2%break_3
+end program main
+
+!CHECK: 15.
+!CHECK: 10.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90 b/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
new file mode 100644
index 000000000000..7cb7846ed0b2
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
@@ -0,0 +1,39 @@
+! Offloading test checking interaction of two
+! derived type's with a single explicit array
+! member each being mapped with bounds to 
+! target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+
+  !$omp target map(tofrom:scalar_arr1%array_x(3:6), scalar_arr2%array_x(3:6))
+    do i = 3, 6
+      scalar_arr2%array_x(i) = i
+      scalar_arr1%array_x(i) = i
+    end do
+  !$omp end target
+
+  print*, scalar_arr1%array_x  
+  print*, scalar_arr2%array_x
+end program main
+
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90 b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
new file mode 100644
index 000000000000..fbe6b305464a
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
@@ -0,0 +1,35 @@
+! Offloading test checking interaction of two
+! derived type's with a mix of explicit and
+! implicit member mapping to target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+        real(4) :: break_0
+        real(4) :: array_x(10)
+        real(4) :: break_1
+        real(4) :: array_y(10)
+        real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+  !$omp target map(tofrom:scalar_arr1%break_1)
+    scalar_arr2%break_3 = 10
+    scalar_arr1%break_1 = 15
+  !$omp end target
+
+  print*, scalar_arr1%break_1
+  print*, scalar_arr2%break_3
+end program main
+
+!CHECK: 15.
+!CHECK: 10.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90 b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
new file mode 100644
index 000000000000..503329d90628
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
@@ -0,0 +1,41 @@
+! Offloading test checking interaction of two
+! derived type's with a mix of explicit and
+! implicit member mapping of arrays to target
+! one with bounds.
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+        real(4) :: break_0
+        real(4) :: array_x(10)
+        real(4) :: break_1
+        real(4) :: array_y(10)
+        real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+  do i = 1, 10
+    scalar_arr1%array_x(i) = i
+  end do 
+
+  !$omp target map(tofrom:scalar_arr2%array_x(3:6))
+    do i = 3, 6
+      scalar_arr2%array_x(i) = scalar_arr1%array_x(i)
+    end do
+  !$omp end target
+
+  print*, scalar_arr1%array_x
+  print*, scalar_arr2%array_x
+end program main
+
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90 b/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
new file mode 100644
index 000000000000..ed350c54dcb2
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
@@ -0,0 +1,51 @@
+! Offloading test checking interaction of two
+! derived type's with two explicit array
+! members each being mapped with bounds to 
+! target
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+  do i = 1, 10
+    scalar_arr1%array_x(i) = i
+    scalar_arr2%array_x(i) = i
+  end do
+
+  !$omp target map(tofrom:scalar_arr1%array_x(3:6), scalar_arr1%array_y(3:6), scalar_arr2%array_x(3:6), scalar_arr2%array_y(3:6))
+    do i = 1, 10
+      scalar_arr2%array_y(i) = scalar_arr1%array_x(i)
+    end do
+    
+    do i = 1, 10
+      scalar_arr1%array_y(i) = scalar_arr2%array_x(i)
+    end do
+  !$omp end target
+
+  print*, scalar_arr1%array_x
+  print*, scalar_arr2%array_y
+
+  print*, scalar_arr2%array_x
+  print*, scalar_arr1%array_y
+end program main
+
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
diff --git a/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90 b/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90
new file mode 100644
index 000000000000..42d9197f6e1e
--- /dev/null
+++ b/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90
@@ -0,0 +1,56 @@
+! Offloading test checking interaction of an
+! explicit member map utilising array bounds
+! REQUIRES: flang, amdgcn-amd-amdhsa
+! UNSUPPORTED: nvptx64-nvidia-cuda
+! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
+! UNSUPPORTED: aarch64-unknown-linux-gnu
+! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+! UNSUPPORTED: x86_64-pc-linux-gnu
+! UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program main
+    type :: array
+     real(4) :: array_z(10)
+     real(4) :: break_4
+     real(4) :: array_ix(10)
+    end type array
+
+    type :: scalar_array
+    real(4) :: break_0
+    real(4) :: array_x(10)
+    real(4) :: break_1
+    real(4) :: array_y(10)
+    real(4) :: break_3
+    type(array) :: nested
+    end type scalar_array
+  
+    type(scalar_array) :: scalar_arr1
+    type(scalar_array) :: scalar_arr2
+    
+  do i = 1, 10
+    scalar_arr1%nested%array_z(i) = i
+    scalar_arr2%nested%array_z(i) = i
+  end do
+
+  !$omp target map(tofrom:scalar_arr1%nested%array_z(3:6), scalar_arr1%nested%array_ix(3:6), scalar_arr2%nested%array_z(3:6), scalar_arr2%nested%array_ix(3:6))
+    do i = 3, 6
+      scalar_arr2%nested%array_ix(i) = scalar_arr1%nested%array_z(i)
+    end do
+    
+    do i = 3, 6
+      scalar_arr1%nested%array_ix(i) = scalar_arr2%nested%array_z(i)
+    end do
+  !$omp end target
+
+  print*, scalar_arr1%nested%array_ix
+  print*, scalar_arr2%nested%array_z
+
+  print*, scalar_arr2%nested%array_ix
+  print*, scalar_arr1%nested%array_z
+end program main
+
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
+!CHECK: 0. 0. 3. 4. 5. 6. 0. 0. 0. 0.
+!CHECK: 1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
-- 
GitLab


From 50df0ff515697090983b3cb951c48377bcfb6ddf Mon Sep 17 00:00:00 2001
From: Andrew Gozillon <Andrew.Gozillon@amd.com>
Date: Mon, 12 Feb 2024 10:20:30 -0600
Subject: [PATCH 0440/1206] [OpenMP][MLIR] Add new arguments to map_info to
 help support record type maps

This PR adds two new fields to omp.map_info, one BoolAttr and one I64ArrayAttr.

The BoolAttr is named partial_map, and is a flag that indicates if the record type captured by
the map_info operation is a partial map, or if it is mapped in its entirety, this currently helps
the later lowering determine the type of map entries that need to be generated.

The I64ArrayAttr named members_index is intended to track the placement of each member
map_info operations (and by extension mapped member variable) placement in the parent
record type. This may need to be extended to an N-D array for nested member mapping.

Pull Request: https://github.com/llvm/llvm-project/pull/82851
---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 14 +++-
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 73 +++++++++++++++++++
 mlir/test/Dialect/OpenMP/ops.mlir             | 37 +++++++++-
 3 files changed, 117 insertions(+), 7 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index a641588eaa8d..f248be1639fe 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1423,10 +1423,12 @@ def MapInfoOp : OpenMP_Op<"map.info", [AttrSizedOperandSegments]> {
                        TypeAttr:$var_type,
                        Optional<OpenMP_PointerLikeType>:$var_ptr_ptr,
                        Variadic<OpenMP_PointerLikeType>:$members,
+                       OptionalAttr<AnyIntElementsAttr>:$members_index,
                        Variadic<MapBoundsType>:$bounds, /* rank-0 to rank-{n-1} */
                        OptionalAttr<UI64Attr>:$map_type,
                        OptionalAttr<VariableCaptureKindAttr>:$map_capture_type,
-                       OptionalAttr<StrAttr>:$name);
+                       OptionalAttr<StrAttr>:$name,
+                       DefaultValuedAttr<BoolAttr, "false">:$partial_map);
   let results = (outs OpenMP_PointerLikeType:$omp_ptr);
 
   let description = [{
@@ -1462,10 +1464,14 @@ def MapInfoOp : OpenMP_Op<"map.info", [AttrSizedOperandSegments]> {
     - `var_type`: The type of the variable to copy.
     - `var_ptr_ptr`: Used when the variable copied is a member of a class, structure
       or derived type and refers to the originating struct.
-    - `members`:  Used to indicate mapped child members for the current MapInfoOp,
+    - `members`: Used to indicate mapped child members for the current MapInfoOp,
        represented as other MapInfoOp's, utilised in cases where a parent structure
        type and members of the structure type are being mapped at the same time.
        For example: map(to: parent, parent->member, parent->member2[:10])
+    - `members_index`: Used to indicate the ordering of members within the containing
+       parent (generally a record type such as a structure, class or derived type),
+       e.g. struct {int x, float y, double z}, x would be 0, y would be 1, and z
+       would be 2. This aids the mapping.
     - `bounds`: Used when copying slices of array's, pointers or pointer members of
        objects (e.g. derived types or classes), indicates the bounds to be copied
        of the variable. When it's an array slice it is in rank order where rank 0
@@ -1476,6 +1482,8 @@ def MapInfoOp : OpenMP_Op<"map.info", [AttrSizedOperandSegments]> {
     - 'map_capture_type': Capture type for the variable e.g. this, byref, byvalue, byvla
        this can affect how the variable is lowered.
     - `name`: Holds the name of variable as specified in user clause (including bounds).
+    - `partial_map`: The record type being mapped will not be mapped in its entirety,
+       it may be used however, in a mapping to bind it's mapped components together.
   }];
 
   let assemblyFormat = [{
@@ -1484,7 +1492,7 @@ def MapInfoOp : OpenMP_Op<"map.info", [AttrSizedOperandSegments]> {
         `var_ptr_ptr` `(` $var_ptr_ptr `:` type($var_ptr_ptr) `)`
       | `map_clauses` `(` custom<MapClause>($map_type) `)`
       | `capture` `(` custom<CaptureType>($map_capture_type) `)`
-      | `members` `(` $members `:` type($members) `)`
+      | `members` `(` $members `:` custom<MembersIndex>($members_index) `:` type($members) `)`
       | `bounds` `(` $bounds `)`
     ) `->` type($omp_ptr) attr-dict
   }];
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index e016a326ecc7..61073af2aa4d 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -994,6 +994,79 @@ static void printMapClause(OpAsmPrinter &p, Operation *op,
   }
 }
 
+static ParseResult parseMembersIndex(OpAsmParser &parser,
+                                     DenseIntElementsAttr &membersIdx) {
+  SmallVector<APInt> values;
+  int64_t value;
+  int64_t shape[2] = {0, 0};
+  unsigned shapeTmp = 0;
+  auto parseIndices = [&]() -> ParseResult {
+    if (parser.parseInteger(value))
+      return failure();
+    shapeTmp++;
+    values.push_back(APInt(32, value));
+    return success();
+  };
+
+  do {
+    if (failed(parser.parseLSquare()))
+      return failure();
+
+    if (parser.parseCommaSeparatedList(parseIndices))
+      return failure();
+
+    if (failed(parser.parseRSquare()))
+      return failure();
+
+    // Only set once, if any indices are not the same size
+    // we error out in the next check as that's unsupported
+    if (shape[1] == 0)
+      shape[1] = shapeTmp;
+
+    // Verify that the recently parsed list is equal to the
+    // first one we parsed, they must be equal lengths to
+    // keep the rectangular shape DenseIntElementsAttr
+    // requires
+    if (shapeTmp != shape[1])
+      return failure();
+
+    shapeTmp = 0;
+    shape[0]++;
+  } while (succeeded(parser.parseOptionalComma()));
+
+  if (!values.empty()) {
+    ShapedType valueType =
+        VectorType::get(shape, IntegerType::get(parser.getContext(), 32));
+    membersIdx = DenseIntElementsAttr::get(valueType, values);
+  }
+
+  return success();
+}
+
+static void printMembersIndex(OpAsmPrinter &p, MapInfoOp op,
+                              DenseIntElementsAttr membersIdx) {
+  llvm::ArrayRef<int64_t> shape = membersIdx.getShapedType().getShape();
+  assert(shape.size() <= 2);
+  
+  if (!membersIdx)
+    return;
+
+  for (int i = 0; i < shape[0]; ++i) {
+    p << "[";
+    int rowOffset = i * shape[1];
+    for (int j = 0; j < shape[1]; ++j) {
+      p << membersIdx.getValues<
+          int32_t>()[rowOffset + j];
+      if ((j + 1) < shape[1])
+        p << ",";
+    }
+    p << "]";
+
+    if ((i + 1) < shape[0])
+      p << ", ";
+  }
+}
+
 static ParseResult
 parseMapEntries(OpAsmParser &parser,
                 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &mapOperands,
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 828c9d2c3b84..420cb226d593 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -2306,8 +2306,6 @@ func.func @omp_requires_multiple() -> ()
   return
 }
 
-// -----
-
 // CHECK-LABEL: @opaque_pointers_atomic_rwu
 // CHECK-SAME: (%[[v:.*]]: !llvm.ptr, %[[x:.*]]: !llvm.ptr)
 func.func @opaque_pointers_atomic_rwu(%v: !llvm.ptr, %x: !llvm.ptr) {
@@ -2417,8 +2415,8 @@ func.func @omp_target_update_data (%if_cond : i1, %device : si32, %map1: memref<
 func.func @omp_targets_is_allocatable(%arg0: !llvm.ptr, %arg1: !llvm.ptr) -> () {
   // CHECK: %[[MAP0:.*]] = omp.map.info var_ptr(%[[ARG0]] : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}  
   %mapv1 = omp.map.info var_ptr(%arg0 : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  // CHECK: %[[MAP1:.*]] = omp.map.info var_ptr(%[[ARG1]] : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>) map_clauses(tofrom) capture(ByRef) members(%[[MAP0]] : !llvm.ptr) -> !llvm.ptr {name = ""}
-  %mapv2 = omp.map.info var_ptr(%arg1 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>)   map_clauses(tofrom) capture(ByRef) members(%mapv1 : !llvm.ptr) -> !llvm.ptr {name = ""}  
+  // CHECK: %[[MAP1:.*]] = omp.map.info var_ptr(%[[ARG1]] : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>) map_clauses(tofrom) capture(ByRef) members(%[[MAP0]] : [0] : !llvm.ptr) -> !llvm.ptr {name = ""}
+  %mapv2 = omp.map.info var_ptr(%arg1 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>)   map_clauses(tofrom) capture(ByRef) members(%mapv1 : [0] : !llvm.ptr) -> !llvm.ptr {name = ""}  
   // CHECK: omp.target map_entries(%[[MAP0]] -> {{.*}}, %[[MAP1]] -> {{.*}} : !llvm.ptr, !llvm.ptr)
   omp.target map_entries(%mapv1 -> %arg2, %mapv2 -> %arg3 : !llvm.ptr, !llvm.ptr) {
     ^bb0(%arg2: !llvm.ptr, %arg3 : !llvm.ptr):
@@ -2473,6 +2471,37 @@ func.func @omp_target_enter_update_exit_data_depend(%a: memref<?xi32>, %b: memre
   }
   // CHECK: omp.target_exit_data map_entries([[MAP2]] : memref<?xi32>) depend(taskdependin -> [[ARG2]] : memref<?xi32>)
   omp.target_exit_data map_entries(%map_c : memref<?xi32>) depend(taskdependin -> %c : memref<?xi32>)
+
+  return
+}
+
+// CHECK-LABEL: omp_map_with_members
+// CHECK-SAME: (%[[ARG0:.*]]: !llvm.ptr, %[[ARG1:.*]]: !llvm.ptr, %[[ARG2:.*]]: !llvm.ptr, %[[ARG3:.*]]: !llvm.ptr, %[[ARG4:.*]]: !llvm.ptr, %[[ARG5:.*]]: !llvm.ptr)
+func.func @omp_map_with_members(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr, %arg3: !llvm.ptr, %arg4: !llvm.ptr, %arg5: !llvm.ptr) -> () {
+  // CHECK: %[[MAP0:.*]] = omp.map.info var_ptr(%[[ARG0]] : !llvm.ptr, i32) map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}  
+  %mapv1 = omp.map.info var_ptr(%arg0 : !llvm.ptr, i32) map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
+
+  // CHECK: %[[MAP1:.*]] = omp.map.info var_ptr(%[[ARG1]] : !llvm.ptr, f32) map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}  
+  %mapv2 = omp.map.info var_ptr(%arg1 : !llvm.ptr, f32) map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
+  
+  // CHECK: %[[MAP2:.*]] = omp.map.info var_ptr(%[[ARG2]] : !llvm.ptr, !llvm.struct<(i32, f32)>) map_clauses(to) capture(ByRef) members(%[[MAP0]], %[[MAP1]] : [0], [1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {name = "", partial_map = true}
+  %mapv3 = omp.map.info var_ptr(%arg2 : !llvm.ptr, !llvm.struct<(i32, f32)>)   map_clauses(to) capture(ByRef) members(%mapv1, %mapv2 : [0], [1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {name = "", partial_map = true}  
+  
+  // CHECK: omp.target_enter_data map_entries(%[[MAP0]], %[[MAP1]], %[[MAP2]] : !llvm.ptr, !llvm.ptr, !llvm.ptr)
+  omp.target_enter_data map_entries(%mapv1, %mapv2, %mapv3 : !llvm.ptr, !llvm.ptr, !llvm.ptr){}
+
+  // CHECK: %[[MAP3:.*]] = omp.map.info var_ptr(%[[ARG3]] : !llvm.ptr, i32) map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}  
+  %mapv4 = omp.map.info var_ptr(%arg3 : !llvm.ptr, i32) map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+
+  // CHECK: %[[MAP4:.*]] = omp.map.info var_ptr(%[[ARG4]] : !llvm.ptr, f32) map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}  
+  %mapv5 = omp.map.info var_ptr(%arg4 : !llvm.ptr, f32) map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+  
+  // CHECK: %[[MAP5:.*]] = omp.map.info var_ptr(%[[ARG5]] : !llvm.ptr, !llvm.struct<(i32, struct<(i32, f32)>)>) map_clauses(from) capture(ByRef) members(%[[MAP3]], %[[MAP4]] : [1,0], [1,1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {name = "", partial_map = true}
+  %mapv6 = omp.map.info var_ptr(%arg5 : !llvm.ptr, !llvm.struct<(i32, struct<(i32, f32)>)>) map_clauses(from) capture(ByRef) members(%mapv4, %mapv5 : [1,0], [1,1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {name = "", partial_map = true}  
+ 
+  // CHECK: omp.target_exit_data map_entries(%[[MAP3]], %[[MAP4]], %[[MAP5]] : !llvm.ptr, !llvm.ptr, !llvm.ptr)
+  omp.target_exit_data map_entries(%mapv4, %mapv5, %mapv6 : !llvm.ptr, !llvm.ptr, !llvm.ptr){}
+
   return
 }
 
-- 
GitLab


From 462435f089b7ca67ed1eed60f369e16b71e76b74 Mon Sep 17 00:00:00 2001
From: Andrew Gozillon <Andrew.Gozillon@amd.com>
Date: Mon, 12 Feb 2024 10:47:14 -0600
Subject: [PATCH 0441/1206] [OpenMP][MLIR] Extend record member map support for
 omp dialect to LLVM-IR

This patch seeks to refactor slightly and extend the current record type map
support that was put in place for Fortran's descriptor types to handle explicit
member mapping for record types at a single level of depth (the case of explicit
mapping of nested record types is currently unsupported).

This patch seeks to support this by extending the OpenMPToLLVMIRTranslation phase
to more generally support record types, building on the prior groundwork in the
Fortran allocatables/pointers patch. It now supports different kinds of record type
mapping, in this case full record type mapping and then explicit member mapping
in which there is a special case for certain types when mapped individually to not
require any parent map link in the kernel argument structure. To facilitate this
required:
   *  The movement of the setting of the map flag type "ptr_and_obj" to respective
         frontends, now supporting it as a possible flag that can be read and printed
         in mlir form. Some minor changes to declare target map type setting was
         neccesary for this.
   * The addition of a member index array operand, which tracks the position
       of the member in the parent, required for caclulating the appropriate size
       to offload to the target, alongside the parents offload pointer (always the
       first member currently being mapped).
   * A partial mapping attribute operand, to indicate if the entire record type is
       being mapped or just member components, aiding the ability to lower
       record types in the different manners that are possible.
   * Refactoring bounds calculation for record types and general arrays to one
       location (as well as load/store generation prior to assigning to the kernel
       argument structure), as a side affect enter/exit/update/data mapping
       should now be more correct and fully support bounds mapping, previously
       this would have only worked for target.

Pull Request: https://github.com/llvm/llvm-project/pull/82852
---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 257 +++++++++++++-----
 ...target-fortran-allocatable-types-host.mlir |  15 +-
 mlir/test/Target/LLVMIR/omptarget-llvm.mlir   |  28 +-
 ...arget-nested-record-type-mapping-host.mlir |  69 +++++
 .../omptarget-record-type-mapping-host.mlir   |  62 +++++
 5 files changed, 343 insertions(+), 88 deletions(-)
 create mode 100644 mlir/test/Target/LLVMIR/omptarget-nested-record-type-mapping-host.mlir
 create mode 100644 mlir/test/Target/LLVMIR/omptarget-record-type-mapping-host.mlir

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index bfd7d65912bd..0c3412666732 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -33,7 +33,9 @@
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 
 #include <any>
+#include <cstdint>
 #include <iterator>
+#include <numeric>
 #include <optional>
 #include <utility>
 
@@ -2037,7 +2039,7 @@ llvm::Value *getSizeInBytes(DataLayout &dl, const mlir::Type &type,
         if (auto boundOp = mlir::dyn_cast_if_present<mlir::omp::MapBoundsOp>(
                 bounds.getDefiningOp())) {
           // The below calculation for the size to be mapped calculated from the
-          // map_info's bounds is: (elemCount * [UB - LB] + 1), later we
+          // map.info's bounds is: (elemCount * [UB - LB] + 1), later we
           // multiply by the underlying element types byte size to get the full
           // size to be offloaded based on the bounds
           elementCount = builder.CreateMul(
@@ -2089,9 +2091,9 @@ void collectMapDataFromMapOperands(MapInfoData &mapData,
 
       mapData.BaseType.push_back(
           moduleTranslation.convertType(mapOp.getVarType()));
-      mapData.Sizes.push_back(getSizeInBytes(
-          dl, mapOp.getVarType(), mapOp, mapData.BasePointers.back(),
-          mapData.BaseType.back(), builder, moduleTranslation));
+      mapData.Sizes.push_back(
+          getSizeInBytes(dl, mapOp.getVarType(), mapOp, mapData.Pointers.back(),
+                         mapData.BaseType.back(), builder, moduleTranslation));
       mapData.MapClause.push_back(mapOp.getOperation());
       mapData.Types.push_back(
           llvm::omp::OpenMPOffloadMappingFlags(mapOp.getMapType().value()));
@@ -2122,6 +2124,67 @@ void collectMapDataFromMapOperands(MapInfoData &mapData,
   }
 }
 
+static int getMapDataMemberIdx(MapInfoData &mapData,
+                               mlir::omp::MapInfoOp memberOp) {
+  auto *res = llvm::find(mapData.MapClause, memberOp);
+  assert(res != mapData.MapClause.end() &&
+         "MapInfoOp for member not found in MapData, cannot return index");
+  return std::distance(mapData.MapClause.begin(), res);
+}
+
+static mlir::omp::MapInfoOp
+getFirstOrLastMappedMemberPtr(mlir::omp::MapInfoOp mapInfo, bool first) {
+  mlir::DenseIntElementsAttr indexAttr = mapInfo.getMembersIndexAttr();
+
+  // Only 1 member has been mapped, we can return it.
+  if (indexAttr.size() == 1)
+    if (auto mapOp = mlir::dyn_cast<mlir::omp::MapInfoOp>(
+            mapInfo.getMembers()[0].getDefiningOp()))
+      return mapOp;
+
+  llvm::ArrayRef<int64_t> shape = indexAttr.getShapedType().getShape();
+  llvm::SmallVector<size_t> indices(shape[0]);
+  std::iota(indices.begin(), indices.end(), 0);
+
+  llvm::sort(
+      indices.begin(), indices.end(), [&](const size_t a, const size_t b) {
+        auto indexValues = indexAttr.getValues<int32_t>();
+        for (int i = 0;
+             i < shape[1];
+             ++i) {
+          int aIndex = indexValues[a * shape[1] + i];
+          int bIndex = indexValues[b * shape[1] + i];
+
+          if (aIndex != -1 && bIndex == -1)
+            return false;
+
+          if (aIndex == -1 && bIndex != -1)
+            return true;
+
+          if (aIndex == -1)
+            return first;
+
+          if (bIndex == -1)
+            return !first;
+
+          // A is earlier in the record type layout than B
+          if (aIndex < bIndex)
+            return first;
+
+          if (bIndex < aIndex)
+            return !first;
+        }
+
+        // iterated the entire list and couldn't make a decision, all elements
+        // were likely the same, return true for now similar to reaching the end
+        // of both and finding invalid indices.
+        return true;
+      });
+
+    return llvm::cast<mlir::omp::MapInfoOp>(
+          mapInfo.getMembers()[indices.front()].getDefiningOp());
+}
+
 /// This function calculates the array/pointer offset for map data provided
 /// with bounds operations, e.g. when provided something like the following:
 ///
@@ -2227,6 +2290,9 @@ calculateBoundsOffset(LLVM::ModuleTranslation &moduleTranslation,
 // which is utilised in subsequent member mappings (by modifying there map type
 // with it) to indicate that a member is part of this parent and should be
 // treated by the runtime as such. Important to achieve the correct mapping.
+//
+// This function borrows a lot from Clang's emitCombinedEntry function
+// inside of CGOpenMPRuntime.cpp
 static llvm::omp::OpenMPOffloadMappingFlags mapParentWithMembers(
     LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder,
     llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl,
@@ -2242,7 +2308,6 @@ static llvm::omp::OpenMPOffloadMappingFlags mapParentWithMembers(
   combinedInfo.Names.emplace_back(LLVM::createMappingInformation(
       mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));
   combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIndex]);
-  combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIndex]);
 
   // Calculate size of the parent object being mapped based on the
   // addresses at runtime, highAddr - lowAddr = size. This of course
@@ -2251,42 +2316,68 @@ static llvm::omp::OpenMPOffloadMappingFlags mapParentWithMembers(
   // Fortran pointers and allocatables, the mapping of the pointed to
   // data by the descriptor (which itself, is a structure containing
   // runtime information on the dynamically allocated data).
-  llvm::Value *lowAddr = builder.CreatePointerCast(
-      mapData.Pointers[mapDataIndex], builder.getPtrTy());
-  llvm::Value *highAddr = builder.CreatePointerCast(
-      builder.CreateConstGEP1_32(mapData.BaseType[mapDataIndex],
-                                 mapData.Pointers[mapDataIndex], 1),
-      builder.getPtrTy());
+  auto parentClause =
+      llvm::cast<mlir::omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);
+
+  llvm::Value *lowAddr, *highAddr;
+  if (!parentClause.getPartialMap()) {
+    lowAddr = builder.CreatePointerCast(mapData.Pointers[mapDataIndex],
+                                        builder.getPtrTy());
+    highAddr = builder.CreatePointerCast(
+        builder.CreateConstGEP1_32(mapData.BaseType[mapDataIndex],
+                                   mapData.Pointers[mapDataIndex], 1),
+        builder.getPtrTy());
+    combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIndex]);
+  } else {
+    auto mapOp =
+        mlir::dyn_cast<mlir::omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);
+    int firstMemberIdx = getMapDataMemberIdx(
+        mapData, getFirstOrLastMappedMemberPtr(mapOp, true));
+    lowAddr = builder.CreatePointerCast(mapData.Pointers[firstMemberIdx],
+                                        builder.getPtrTy());
+    int lastMemberIdx = getMapDataMemberIdx(
+        mapData, getFirstOrLastMappedMemberPtr(mapOp, false));
+    highAddr = builder.CreatePointerCast(
+        builder.CreateGEP(mapData.BaseType[lastMemberIdx],
+                          mapData.Pointers[lastMemberIdx], builder.getInt64(1)),
+        builder.getPtrTy());
+    combinedInfo.Pointers.emplace_back(mapData.Pointers[firstMemberIdx]);
+  }
+
   llvm::Value *size = builder.CreateIntCast(
       builder.CreatePtrDiff(builder.getInt8Ty(), highAddr, lowAddr),
       builder.getInt64Ty(),
       /*isSigned=*/false);
   combinedInfo.Sizes.push_back(size);
 
-  // This creates the initial MEMBER_OF mapping that consists of
-  // the parent/top level container (same as above effectively, except
-  // with a fixed initial compile time size and seperate maptype which
-  // indicates the true mape type (tofrom etc.) and that it is a part
-  // of a larger mapping and indicating the link between it and it's
-  // members that are also explicitly mapped).
+  // TODO: This will need to be expanded to include the whole host of logic for
+  // the map flags that Clang currently supports (e.g. it should take the map
+  // flag of the parent map flag, remove the OMP_MAP_TARGET_PARAM and do some
+  // further case specific flag modifications). For the moment, it handles what
+  // we support as expected.
   llvm::omp::OpenMPOffloadMappingFlags mapFlag =
       llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
-  if (isTargetParams)
-    mapFlag &= ~llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
 
   llvm::omp::OpenMPOffloadMappingFlags memberOfFlag =
       ompBuilder.getMemberOfFlag(combinedInfo.BasePointers.size() - 1);
   ompBuilder.setCorrectMemberOfFlag(mapFlag, memberOfFlag);
 
-  combinedInfo.Types.emplace_back(mapFlag);
-  combinedInfo.DevicePointers.emplace_back(
-      llvm::OpenMPIRBuilder::DeviceInfoTy::None);
-  combinedInfo.Names.emplace_back(LLVM::createMappingInformation(
-      mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));
-  combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIndex]);
-  combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIndex]);
-  combinedInfo.Sizes.emplace_back(mapData.Sizes[mapDataIndex]);
-
+  // This creates the initial MEMBER_OF mapping that consists of
+  // the parent/top level container (same as above effectively, except
+  // with a fixed initial compile time size and seperate maptype which
+  // indicates the true mape type (tofrom etc.). This parent mapping is
+  // only relevant if the structure in its totality is being mapped,
+  // otherwise the above suffices.
+  if (!parentClause.getPartialMap()) {
+    combinedInfo.Types.emplace_back(mapFlag);
+    combinedInfo.DevicePointers.emplace_back(
+        llvm::OpenMPIRBuilder::DeviceInfoTy::None);
+    combinedInfo.Names.emplace_back(LLVM::createMappingInformation(
+        mapData.MapClause[mapDataIndex]->getLoc(), ompBuilder));
+    combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIndex]);
+    combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIndex]);
+    combinedInfo.Sizes.emplace_back(mapData.Sizes[mapDataIndex]);
+  }
   return memberOfFlag;
 }
 
@@ -2319,21 +2410,17 @@ static void processMapMembersWithParent(
     uint64_t mapDataIndex, llvm::omp::OpenMPOffloadMappingFlags memberOfFlag) {
 
   auto parentClause =
-      mlir::dyn_cast<mlir::omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);
+      llvm::cast<mlir::omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);
 
   for (auto mappedMembers : parentClause.getMembers()) {
     auto memberClause =
-        mlir::dyn_cast<mlir::omp::MapInfoOp>(mappedMembers.getDefiningOp());
-    int memberDataIdx = -1;
-    for (size_t i = 0; i < mapData.MapClause.size(); ++i) {
-      if (mapData.MapClause[i] == memberClause)
-        memberDataIdx = i;
-    }
+        llvm::cast<mlir::omp::MapInfoOp>(mappedMembers.getDefiningOp());
+    int memberDataIdx = getMapDataMemberIdx(mapData, memberClause);
 
     assert(memberDataIdx >= 0 && "could not find mapped member of structure");
 
     // Same MemberOfFlag to indicate its link with parent and other members
-    // of, and we flag that it's part of a pointer and object coupling.
+    // of.
     auto mapFlag =
         llvm::omp::OpenMPOffloadMappingFlags(memberClause.getMapType().value());
     mapFlag &= ~llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
@@ -2347,18 +2434,81 @@ static void processMapMembersWithParent(
         llvm::OpenMPIRBuilder::DeviceInfoTy::None);
     combinedInfo.Names.emplace_back(
         LLVM::createMappingInformation(memberClause.getLoc(), ompBuilder));
-
-    combinedInfo.BasePointers.emplace_back(mapData.BasePointers[memberDataIdx]);
+    combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIndex]);
     combinedInfo.Pointers.emplace_back(mapData.Pointers[memberDataIdx]);
     combinedInfo.Sizes.emplace_back(mapData.Sizes[memberDataIdx]);
   }
 }
 
+static void
+processIndividualMap(MapInfoData &mapData, size_t mapDataIdx,
+                     llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo,
+                     bool isTargetParams, int mapDataParentIdx = -1) {
+  // Declare Target Mappings are excluded from being marked as
+  // OMP_MAP_TARGET_PARAM as they are not passed as parameters, they're
+  // marked with OMP_MAP_PTR_AND_OBJ instead.
+  auto mapFlag = mapData.Types[mapDataIdx];
+  auto mapInfoOp =
+      llvm::cast<mlir::omp::MapInfoOp>(mapData.MapClause[mapDataIdx]);
+
+  bool isPtrTy = checkIfPointerMap(mapInfoOp);
+  if (isPtrTy)
+    mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
+
+  if (isTargetParams && !mapData.IsDeclareTarget[mapDataIdx])
+    mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
+
+  if (mapInfoOp.getMapCaptureType().value() ==
+          mlir::omp::VariableCaptureKind::ByCopy &&
+      !isPtrTy)
+    mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL;
+
+  // if we're provided a mapDataParentIdx, then the data being mapped is
+  // part of a larger object (in a parent <-> member mapping) and in this
+  // case our BasePointer should be the parent.
+  if (mapDataParentIdx >= 0)
+    combinedInfo.BasePointers.emplace_back(
+        mapData.BasePointers[mapDataParentIdx]);
+  else
+    combinedInfo.BasePointers.emplace_back(mapData.BasePointers[mapDataIdx]);
+
+  combinedInfo.Pointers.emplace_back(mapData.Pointers[mapDataIdx]);
+  combinedInfo.DevicePointers.emplace_back(mapData.DevicePointers[mapDataIdx]);
+  combinedInfo.Names.emplace_back(mapData.Names[mapDataIdx]);
+  combinedInfo.Types.emplace_back(mapFlag);
+  combinedInfo.Sizes.emplace_back(mapData.Sizes[mapDataIdx]);
+}
+
 static void processMapWithMembersOf(
     LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder,
     llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl,
     llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo, MapInfoData &mapData,
     uint64_t mapDataIndex, bool isTargetParams) {
+  auto parentClause =
+      llvm::cast<mlir::omp::MapInfoOp>(mapData.MapClause[mapDataIndex]);
+
+  // If we have a partial map (no parent referenced in the map clauses of the
+  // directive, only members) and only a single member, we do not need to bind
+  // the map of the member to the parent, we can pass the member seperately.
+  if (parentClause.getMembers().size() == 1 && parentClause.getPartialMap()) {
+    auto memberClause = llvm::cast<mlir::omp::MapInfoOp>(
+        parentClause.getMembers()[0].getDefiningOp());
+    int memberDataIdx = getMapDataMemberIdx(mapData, memberClause);
+    // Note: Clang treats arrays with explicit bounds that fall into this
+    // category as a parent with map case, however, it seems this isn't a
+    // requirement, and processing them as an individual map is fine. So,
+    // we will handle them as individual maps for the moment, as it's
+    // difficult for us to check this as we always require bounds to be
+    // specified currently and it's also marginally more optimal (single
+    // map rather than two). The difference may come from the fact that
+    // Clang maps array without bounds as pointers (which we do not
+    // currently do), whereas we treat them as arrays in all cases
+    // currently.
+    processIndividualMap(mapData, memberDataIdx, combinedInfo, isTargetParams,
+                         mapDataIndex);
+    return;
+  }
+
   llvm::omp::OpenMPOffloadMappingFlags memberOfParentFlag =
       mapParentWithMembers(moduleTranslation, builder, ompBuilder, dl,
                            combinedInfo, mapData, mapDataIndex, isTargetParams);
@@ -2477,12 +2627,8 @@ static void genMapInfos(llvm::IRBuilderBase &builder,
   // utilise the size from any component of MapInfoData, if we can't
   // something is missing from the initial MapInfoData construction.
   for (size_t i = 0; i < mapData.MapClause.size(); ++i) {
-    // NOTE/TODO: We currently do not handle member mapping seperately from it's
-    // parent or explicit mapping of a parent and member in the same operation,
-    // this will need to change in the near future, for now we primarily handle
-    // descriptor mapping from fortran, generalised as mapping record types
-    // with implicit member maps. This lowering needs further generalisation to
-    // fully support fortran derived types, and C/C++ structures and classes.
+    // NOTE/TODO: We currently do not support arbitrary depth record
+    // type mapping.
     if (mapData.IsAMember[i])
       continue;
 
@@ -2493,28 +2639,7 @@ static void genMapInfos(llvm::IRBuilderBase &builder,
       continue;
     }
 
-    auto mapFlag = mapData.Types[i];
-    bool isPtrTy = checkIfPointerMap(mapInfoOp);
-    if (isPtrTy)
-      mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
-
-    // Declare Target Mappings are excluded from being marked as
-    // OMP_MAP_TARGET_PARAM as they are not passed as parameters.
-    if (isTargetParams && !mapData.IsDeclareTarget[i])
-      mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
-
-    if (auto mapInfoOp = dyn_cast<mlir::omp::MapInfoOp>(mapData.MapClause[i]))
-      if (mapInfoOp.getMapCaptureType().value() ==
-              mlir::omp::VariableCaptureKind::ByCopy &&
-          !isPtrTy)
-        mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL;
-
-    combinedInfo.BasePointers.emplace_back(mapData.BasePointers[i]);
-    combinedInfo.Pointers.emplace_back(mapData.Pointers[i]);
-    combinedInfo.DevicePointers.emplace_back(mapData.DevicePointers[i]);
-    combinedInfo.Names.emplace_back(mapData.Names[i]);
-    combinedInfo.Types.emplace_back(mapFlag);
-    combinedInfo.Sizes.emplace_back(mapData.Sizes[i]);
+    processIndividualMap(mapData, i, combinedInfo, isTargetParams);
   }
 
   auto findMapInfo = [&combinedInfo](llvm::Value *val, unsigned &index) {
diff --git a/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir b/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir
index 7cb22dbb10b1..429bb379ee1b 100644
--- a/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir
@@ -2,10 +2,9 @@
 
 // This test checks the offload sizes, map types and base pointers and pointers
 // provided to the OpenMP kernel argument structure are correct when lowering 
-// to LLVM-IR from MLIR when the fortran allocatables flag is switched on and 
-// a fortran allocatable descriptor type is provided alongside the omp.map.info,
-// the test utilises mapping of array sections, full arrays and individual 
-// allocated scalars.
+// to LLVM-IR from MLIR when a fortran allocatable descriptor type is provided 
+// alongside the omp.map.info, the test utilises mapping of array sections, 
+// full arrays and individual allocated scalars.
 
 module attributes {omp.is_target_device = false} {
   llvm.func @_QQmain() {
@@ -27,7 +26,7 @@ module attributes {omp.is_target_device = false} {
     %15 = omp.map.bounds lower_bound(%7 : i64) upper_bound(%14 : i64) extent(%11 : i64) stride(%13 : i64) start_idx(%9 : i64) {stride_in_bytes = true}
     %16 = llvm.getelementptr %3[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>
     %17 = omp.map.info var_ptr(%3 : !llvm.ptr, f32) var_ptr_ptr(%16 : !llvm.ptr) map_clauses(tofrom) capture(ByRef) bounds(%15) -> !llvm.ptr {name = "full_arr"}
-    %18 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>) map_clauses(tofrom) capture(ByRef) members(%17 : !llvm.ptr) -> !llvm.ptr {name = "full_arr"}
+    %18 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>) map_clauses(tofrom) capture(ByRef) members(%17 : [0] : !llvm.ptr) -> !llvm.ptr {name = "full_arr"}
     %19 = llvm.getelementptr %6[0, 7, %7, 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>
     %20 = llvm.load %19 : !llvm.ptr -> i64
     %21 = llvm.getelementptr %6[0, 7, %7, 1] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>
@@ -39,10 +38,10 @@ module attributes {omp.is_target_device = false} {
     %27 = omp.map.bounds lower_bound(%25 : i64) upper_bound(%26 : i64) extent(%22 : i64) stride(%24 : i64) start_idx(%20 : i64) {stride_in_bytes = true}
     %28 = llvm.getelementptr %6[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>
     %29 = omp.map.info var_ptr(%6 : !llvm.ptr, i32) var_ptr_ptr(%28 : !llvm.ptr) map_clauses(tofrom) capture(ByRef) bounds(%27) -> !llvm.ptr {name = "sect_arr(2:5)"}
-    %30 = omp.map.info var_ptr(%6 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>) map_clauses(tofrom) capture(ByRef) members(%29 : !llvm.ptr) -> !llvm.ptr {name = "sect_arr(2:5)"}
+    %30 = omp.map.info var_ptr(%6 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>) map_clauses(tofrom) capture(ByRef) members(%29 : [0] : !llvm.ptr) -> !llvm.ptr {name = "sect_arr(2:5)"}
     %31 = llvm.getelementptr %5[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>
     %32 = omp.map.info var_ptr(%5 : !llvm.ptr, f32) var_ptr_ptr(%31 : !llvm.ptr) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = "scalar"}
-    %33 = omp.map.info var_ptr(%5 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>) map_clauses(tofrom) capture(ByRef) members(%32 : !llvm.ptr) -> !llvm.ptr {name = "scalar"}
+    %33 = omp.map.info var_ptr(%5 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>) map_clauses(tofrom) capture(ByRef) members(%32 : [0] : !llvm.ptr) -> !llvm.ptr {name = "scalar"}
     omp.target map_entries(%17 -> %arg0, %18 -> %arg1, %29 -> %arg2, %30 -> %arg3, %32 -> %arg4, %33 -> %arg5 : !llvm.ptr, !llvm.ptr, !llvm.ptr, !llvm.ptr, !llvm.ptr, !llvm.ptr) {
     ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr, %arg3: !llvm.ptr, %arg4: !llvm.ptr, %arg5: !llvm.ptr):
       omp.terminator
@@ -142,6 +141,6 @@ module attributes {omp.is_target_device = false} {
 // CHECK: %[[OFFLOADPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_ptrs, i32 0, i32 7
 // CHECK: store ptr %[[SCALAR_ALLOCA]], ptr %[[OFFLOADPTRS]], align 8
 // CHECK: %[[OFFLOADBASEPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_baseptrs, i32 0, i32 8
-// CHECK: store ptr %[[SCALAR_BASE]], ptr %[[OFFLOADBASEPTRS]], align 8
+// CHECK: store ptr %[[SCALAR_ALLOCA]], ptr %[[OFFLOADBASEPTRS]], align 8
 // CHECK: %[[OFFLOADPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_ptrs, i32 0, i32 8
 // CHECK: store ptr %[[SCALAR_PTR_LOAD]], ptr %[[OFFLOADPTRS]], align 8
diff --git a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
index 2f629675442d..18189ea63981 100644
--- a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
@@ -69,18 +69,18 @@ llvm.func @_QPopenmp_target_data_region(%0 : !llvm.ptr) {
 // CHECK:         %[[ARR_OFFSET:.*]] = getelementptr inbounds [1024 x i32], ptr %[[ARR_DATA:.*]], i64 0, i64 0
 // CHECK:         %[[VAL_5:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
 // CHECK:         store ptr %[[ARR_DATA]], ptr %[[VAL_5]], align 8
-// CHECK:         %[[VAL_7:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         store ptr %[[ARR_OFFSET]], ptr %[[VAL_7]], align 8
-// CHECK:         %[[VAL_8:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_2]], i64 0, i64 0
-// CHECK:         store ptr null, ptr %[[VAL_8]], align 8
-// CHECK:         %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
-// CHECK:         %[[VAL_10:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_9]], ptr %[[VAL_10]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
-// CHECK:         %[[VAL_11:.*]] = getelementptr [1024 x i32], ptr %[[ARR_DATA]], i32 0, i64 0
-// CHECK:         store i32 99, ptr %[[VAL_11]], align 4
-// CHECK:         %[[VAL_12:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
-// CHECK:         %[[VAL_13:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_end_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_12]], ptr %[[VAL_13]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
+// CHECK:         %[[VAL_6:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         store ptr %[[ARR_OFFSET]], ptr %[[VAL_6]], align 8
+// CHECK:         %[[VAL_7:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_2]], i64 0, i64 0
+// CHECK:         store ptr null, ptr %[[VAL_7]], align 8
+// CHECK:         %[[VAL_8:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:         %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_8]], ptr %[[VAL_9]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
+// CHECK:         %[[VAL_10:.*]] = getelementptr [1024 x i32], ptr %[[ARR_DATA]], i32 0, i64 0
+// CHECK:         store i32 99, ptr %[[VAL_10]], align 4
+// CHECK:         %[[VAL_11:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:         %[[VAL_12:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         call void @__tgt_target_data_end_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_11]], ptr %[[VAL_12]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
 // CHECK:         ret void
 
 // -----
@@ -157,13 +157,13 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) {
 // CHECK:         %[[ARR_OFFSET1:.*]] = getelementptr inbounds [1024 x i32], ptr %[[VAL_16:.*]], i64 0, i64 0
 // CHECK:         %[[ARR_OFFSET2:.*]] = getelementptr inbounds [512 x i32], ptr %[[VAL_20:.*]], i64 0, i64 0
 // CHECK:         %[[VAL_15:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 0
-// CHECK:         store ptr %[[VAL_16:.*]], ptr %[[VAL_15]], align 8
+// CHECK:         store ptr %[[VAL_16]], ptr %[[VAL_15]], align 8
 // CHECK:         %[[VAL_17:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 0
 // CHECK:         store ptr %[[ARR_OFFSET1]], ptr %[[VAL_17]], align 8
 // CHECK:         %[[VAL_18:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i64 0, i64 0
 // CHECK:         store ptr null, ptr %[[VAL_18]], align 8
 // CHECK:         %[[VAL_19:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 1
-// CHECK:         store ptr %[[VAL_20:.*]], ptr %[[VAL_19]], align 8
+// CHECK:         store ptr %[[VAL_20]], ptr %[[VAL_19]], align 8
 // CHECK:         %[[VAL_21:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 1
 // CHECK:         store ptr %[[ARR_OFFSET2]], ptr %[[VAL_21]], align 8
 // CHECK:         %[[VAL_22:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i64 0, i64 1
diff --git a/mlir/test/Target/LLVMIR/omptarget-nested-record-type-mapping-host.mlir b/mlir/test/Target/LLVMIR/omptarget-nested-record-type-mapping-host.mlir
new file mode 100644
index 000000000000..e4d82d4a58c8
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-nested-record-type-mapping-host.mlir
@@ -0,0 +1,69 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// This test checks the offload sizes, map types and base pointers and pointers
+// provided to the OpenMP kernel argument structure are correct when lowering
+// to LLVM-IR from MLIR when performing explicit member mapping of a record type
+// that includes another nested record type (C++/C class/structure, Fortran
+// derived type) where members of both the nested and outer record type have
+// members mapped.
+
+module attributes {omp.is_target_device = false} {
+llvm.func @_QQmain() {
+    %0 = llvm.mlir.constant(10 : index) : i64
+    %1 = llvm.mlir.constant(4 : index) : i64
+    %2 = llvm.mlir.constant(1 : index) : i64
+    %3 = llvm.mlir.constant(1 : i64) : i64
+    %4 = llvm.alloca %3 x !llvm.struct<(f32, array<10 x i32>, struct<(f32, i32)>, i32)> : (i64) -> !llvm.ptr
+    %5 = llvm.getelementptr %4[0, 3] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(f32, array<10 x i32>, struct<(f32, i32)>, i32)>
+    %6 = omp.map.info var_ptr(%5 : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr
+    %7 = llvm.getelementptr %4[0, 2, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(f32, array<10 x i32>, struct<(f32, i32)>, i32)>
+    %8 = omp.map.info var_ptr(%7 : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr
+    %9 = llvm.getelementptr %4[0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(f32, array<10 x i32>, struct<(f32, i32)>, i32)>
+    %10 = omp.map.bounds lower_bound(%2 : i64) upper_bound(%1 : i64) extent(%0 : i64) stride(%2 : i64) start_idx(%2 : i64)
+    %11 = omp.map.info var_ptr(%9 : !llvm.ptr, !llvm.array<10 x i32>) map_clauses(tofrom) capture(ByRef) bounds(%10) -> !llvm.ptr
+    %12 = omp.map.info var_ptr(%4 : !llvm.ptr, !llvm.struct<(f32, array<10 x i32>, struct<(f32, i32)>, i32)>) map_clauses(tofrom) capture(ByRef) members(%6, %8, %11 : [3, -1], [2, 1], [1, -1] : !llvm.ptr, !llvm.ptr, !llvm.ptr) -> !llvm.ptr {partial_map = true}
+    omp.target map_entries(%6 -> %arg0, %8 -> %arg1, %11 -> %arg2, %12 -> %arg3 : !llvm.ptr, !llvm.ptr, !llvm.ptr, !llvm.ptr) {
+    ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr, %arg3: !llvm.ptr):
+      omp.terminator
+    }
+    llvm.return
+  }
+}
+
+// CHECK: @.offload_sizes = private unnamed_addr constant [4 x i64] [i64 0, i64 4, i64 4, i64 16]
+// CHECK: @.offload_maptypes = private unnamed_addr constant [4 x i64] [i64 32, i64 281474976710659, i64 281474976710659, i64 281474976710659]
+
+// CHECK: define void @_QQmain()
+// CHECK: %[[ALLOCA:.*]] = alloca { float, [10 x i32], { float, i32 }, i32 }, i64 1, align 8
+// CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr { float, [10 x i32], { float, i32 }, i32 }, ptr %[[ALLOCA]], i32 0, i32 3 
+// CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr { float, [10 x i32], { float, i32 }, i32 }, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+// CHECK: %[[MEMBER_ACCESS_3:.*]] = getelementptr { float, [10 x i32], { float, i32 }, i32 }, ptr %[[ALLOCA]], i32 0, i32 1
+
+// CHECK: %[[LAST_MEMBER:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS_3]], i64 0, i64 1
+// CHECK: %[[FIRST_MEMBER:.*]] = getelementptr i32, ptr %[[MEMBER_ACCESS_1]], i64 1
+// CHECK: %[[FIRST_MEMBER_OFF:.*]] = ptrtoint ptr %[[FIRST_MEMBER]] to i64
+// CHECK: %[[SECOND_MEMBER_OFF:.*]] = ptrtoint ptr %[[LAST_MEMBER]] to i64
+// CHECK: %[[MEMBER_DIFF:.*]] = sub i64 %[[FIRST_MEMBER_OFF]], %[[SECOND_MEMBER_OFF]]
+// CHECK: %[[OFFLOAD_SIZE:.*]] = sdiv exact i64 %[[MEMBER_DIFF]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+
+// CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+// CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
+// CHECK: %[[PTR_ARR:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+// CHECK: store ptr %[[LAST_MEMBER]], ptr %[[PTR_ARR]], align 8
+// CHECK: %[[SIZE_ARR:.*]] = getelementptr inbounds [4 x i64], ptr %.offload_sizes, i32 0, i32 0
+// CHECK: store i64 %[[OFFLOAD_SIZE]], ptr %[[SIZE_ARR]], align 8
+
+// CHECK: %[[BASE_PTR_ARR_2:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_baseptrs, i32 0, i32 1
+// CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR_2]], align 8
+// CHECK: %[[PTR_ARR_2:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_ptrs, i32 0, i32 1
+// CHECK: store ptr %[[MEMBER_ACCESS_1]], ptr %[[PTR_ARR_2]], align 8
+
+// CHECK: %[[BASE_PTR_ARR_3:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_baseptrs, i32 0, i32 2
+// CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR_3]], align 8
+// CHECK: %[[PTR_ARR_3:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_ptrs, i32 0, i32 2
+// CHECK: store ptr %[[MEMBER_ACCESS_2]], ptr %[[PTR_ARR_3]], align 8
+
+// CHECK: %[[BASE_PTR_ARR_4:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_baseptrs, i32 0, i32 3
+// CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR_4]], align 8
+// CHECK: %[[PTR_ARR_4:.*]] = getelementptr inbounds [4 x ptr], ptr %.offload_ptrs, i32 0, i32 3
+// CHECK: store ptr %[[LAST_MEMBER]], ptr %[[PTR_ARR_4]], align 8
diff --git a/mlir/test/Target/LLVMIR/omptarget-record-type-mapping-host.mlir b/mlir/test/Target/LLVMIR/omptarget-record-type-mapping-host.mlir
new file mode 100644
index 000000000000..c7a87e44d653
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-record-type-mapping-host.mlir
@@ -0,0 +1,62 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// This test checks the offload sizes, map types and base pointers and pointers
+// provided to the OpenMP kernel argument structure are correct when lowering
+// to LLVM-IR from MLIR when performing explicit member mapping of a record type
+// (C++/C class/structure, Fortran derived type) where only members of the record
+// type are mapped.
+
+module attributes {omp.is_target_device = false} {
+llvm.func @_QQmain() {
+    %0 = llvm.mlir.constant(10 : index) : i64
+    %1 = llvm.mlir.constant(4 : index) : i64
+    %2 = llvm.mlir.constant(1 : index) : i64
+    %3 = llvm.mlir.constant(1 : i64) : i64
+    %4 = llvm.alloca %3 x !llvm.struct<(f32, array<10 x i32>, i32)> : (i64) -> !llvm.ptr
+    %5 = llvm.mlir.constant(2 : i32) : i32
+    %6 = llvm.getelementptr %4[0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(f32, array<10 x i32>, i32)>
+    %7 = omp.map.info var_ptr(%6 : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr
+    %8 = llvm.mlir.constant(1 : i32) : i32
+    %9 = llvm.getelementptr %4[0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(f32, array<10 x i32>, i32)>
+    %10 = omp.map.bounds lower_bound(%2 : i64) upper_bound(%1 : i64) extent(%0 : i64) stride(%2 : i64) start_idx(%2 : i64)
+    %11 = omp.map.info var_ptr(%9 : !llvm.ptr, !llvm.array<10 x i32>) map_clauses(tofrom) capture(ByRef) bounds(%10) -> !llvm.ptr
+    %12 = omp.map.info var_ptr(%4 : !llvm.ptr, !llvm.struct<(f32, array<10 x i32>, i32)>) map_clauses(tofrom) capture(ByRef) members(%7, %11 : [2], [1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {partial_map = true}
+    omp.target map_entries(%7 -> %arg0, %11 -> %arg1, %12 -> %arg2 : !llvm.ptr, !llvm.ptr, !llvm.ptr) {
+    ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr):
+      omp.terminator
+    }
+    llvm.return
+  }
+}
+
+// CHECK: @.offload_sizes = private unnamed_addr constant [3 x i64] [i64 0, i64 4, i64 16]
+// CHECK: @.offload_maptypes = private unnamed_addr constant [3 x i64] [i64 32, i64 281474976710659, i64 281474976710659]
+
+// CHECK: define void @_QQmain()
+// CHECK: %[[ALLOCA:.*]] = alloca { float, [10 x i32], i32 }, i64 1, align 8
+// CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr { float, [10 x i32], i32 }, ptr %[[ALLOCA]], i32 0, i32 2
+// CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr { float, [10 x i32], i32 }, ptr %[[ALLOCA]], i32 0, i32 1
+
+// CHECK: %[[LAST_MEMBER:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS_2]], i64 0, i64 1
+// CHECK: %[[FIRST_MEMBER:.*]] = getelementptr i32, ptr %[[MEMBER_ACCESS_1]], i64 1
+// CHECK: %[[FIRST_MEMBER_OFF:.*]] = ptrtoint ptr %[[FIRST_MEMBER]] to i64
+// CHECK: %[[SECOND_MEMBER_OFF:.*]] = ptrtoint ptr %[[LAST_MEMBER]] to i64
+// CHECK: %[[MEMBER_DIFF:.*]] = sub i64 %[[FIRST_MEMBER_OFF]], %[[SECOND_MEMBER_OFF]]
+// CHECK: %[[OFFLOAD_SIZE:.*]] = sdiv exact i64 %[[MEMBER_DIFF]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+
+// CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+// CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
+// CHECK: %[[PTR_ARR:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+// CHECK: store ptr %[[LAST_MEMBER]], ptr %[[PTR_ARR]], align 8
+// CHECK: %[[SIZE_ARR:.*]] = getelementptr inbounds [3 x i64], ptr %.offload_sizes, i32 0, i32 0
+// CHECK: store i64 %[[OFFLOAD_SIZE]], ptr %[[SIZE_ARR]], align 8
+
+// CHECK: %[[BASE_PTR_ARR_2:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_baseptrs, i32 0, i32 1
+// CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR_2]], align 8
+// CHECK: %[[PTR_ARR_2:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_ptrs, i32 0, i32 1
+// CHECK: store ptr %[[MEMBER_ACCESS_1]], ptr %[[PTR_ARR_2]], align 8
+
+// CHECK: %[[BASE_PTR_ARR_3:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_baseptrs, i32 0, i32 2
+// CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR_3]], align 8
+// CHECK: %[[PTR_ARR_3:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_ptrs, i32 0, i32 2
+// CHECK: store ptr %[[LAST_MEMBER]], ptr %[[PTR_ARR_3]], align 8
-- 
GitLab


From 435e850ba97ab567a14b6c84d2b27cadb771cb27 Mon Sep 17 00:00:00 2001
From: Andrew Gozillon <Andrew.Gozillon@amd.com>
Date: Mon, 12 Feb 2024 10:53:28 -0600
Subject: [PATCH 0442/1206] [Flang][OpenMP][MLIR] Initial derived type member
 map support

This patch is one in a series of four patches that seeks to refactor
slightly and extend the current record type map support that was
put in place for Fortran's descriptor types to handle explicit
member mapping for record types at a single level of depth.

For example, the below case where two members of a Fortran
derived type are mapped explicitly:

''''
  type :: scalar_and_array
    real(4) :: real
    integer(4) :: array(10)
    integer(4) :: int
  end type scalar_and_array
  type(scalar_and_array) :: scalar_arr

  !$omp target map(tofrom: scalar_arr%int, scalar_arr%real)
''''

Current cases of derived type mapping left for future work are:
  > explicit member mapping of nested members (e.g. two layers of
     record types where we explicitly map a member from the internal
     record type)
  > Fortran's automagical mapping of all elements and nested elements
     of a derived type
  > explicit member mapping of a derived type and then constituient members
     (redundant in Fortran due to former case but still legal as far as I am aware)
  > explicit member mapping of a record type (may be handled reasonably, just
     not fully tested in this iteration)
  > explicit member mapping for Fortran allocatable types (a variation of nested
     record types)

This patch seeks to support this by extending the Flang-new OpenMP lowering to
support generation of this newly required information, creating the neccessary
parent <-to-> member map_info links, calculating the member indices and
setting if it's a partial map.

The OMPDescriptorMapInfoGen pass has also been generalized into a map
finalization phase, now named OMPMapInfoFinalization. This pass was extended
to support the insertion of member maps into the BlockArg and MapOperands of
relevant map carrying operations. Similar to the method in which descriptor types
are expanded and constituient members inserted.

Pull Request: https://github.com/llvm/llvm-project/pull/82853
---
 flang/docs/OpenMP-descriptor-management.md    |   4 +-
 .../flang/Optimizer/Transforms/Passes.h       |   2 +-
 .../flang/Optimizer/Transforms/Passes.td      |   6 +-
 flang/include/flang/Tools/CLOptions.inc       |   2 +-
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  67 +++--
 flang/lib/Lower/OpenMP/ClauseProcessor.h      |  28 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  11 +-
 flang/lib/Lower/OpenMP/Utils.cpp              | 219 ++++++++++++++-
 flang/lib/Lower/OpenMP/Utils.h                |  34 ++-
 flang/lib/Optimizer/Transforms/CMakeLists.txt |   2 +-
 .../Transforms/OMPDescriptorMapInfoGen.cpp    | 168 -----------
 .../Transforms/OMPMapInfoFinalization.cpp     | 261 ++++++++++++++++++
 .../Fir/convert-to-llvm-openmp-and-fir.fir    |  61 +++-
 .../OpenMP/map-types-and-sizes.f90            | 238 +++++++++++++++-
 .../Lower/OpenMP/allocatable-array-bounds.f90 |   6 +-
 flang/test/Lower/OpenMP/allocatable-map.f90   |   8 +-
 flang/test/Lower/OpenMP/array-bounds.f90      |   4 +-
 flang/test/Lower/OpenMP/derived-type-map.f90  | 238 ++++++++++++++++
 flang/test/Lower/OpenMP/map-component-ref.f90 |   2 +-
 flang/test/Lower/OpenMP/target.f90            |   2 +-
 .../omp-descriptor-map-info-gen.fir           |  44 ---
 .../Transforms/omp-map-info-finalization.fir  |  99 +++++++
 22 files changed, 1223 insertions(+), 283 deletions(-)
 delete mode 100644 flang/lib/Optimizer/Transforms/OMPDescriptorMapInfoGen.cpp
 create mode 100644 flang/lib/Optimizer/Transforms/OMPMapInfoFinalization.cpp
 create mode 100644 flang/test/Lower/OpenMP/derived-type-map.f90
 delete mode 100644 flang/test/Transforms/omp-descriptor-map-info-gen.fir
 create mode 100644 flang/test/Transforms/omp-map-info-finalization.fir

diff --git a/flang/docs/OpenMP-descriptor-management.md b/flang/docs/OpenMP-descriptor-management.md
index 368ff3e911fc..d0eb01b00f9b 100644
--- a/flang/docs/OpenMP-descriptor-management.md
+++ b/flang/docs/OpenMP-descriptor-management.md
@@ -44,7 +44,7 @@ Currently, Flang will lower these descriptor types in the OpenMP lowering (lower
 to all other map types, generating an omp.MapInfoOp containing relevant information required for lowering
 the OpenMP dialect to LLVM-IR during the final stages of the MLIR lowering. However, after 
 the lowering to FIR/HLFIR has been performed an OpenMP dialect specific pass for Fortran, 
-`OMPDescriptorMapInfoGenPass` (Optimizer/OMPDescriptorMapInfoGen.cpp) will expand the 
+`OMPMapInfoFinalizationPass` (Optimizer/OMPMapInfoFinalization.cpp) will expand the 
 `omp.MapInfoOp`'s containing descriptors (which currently will be a `BoxType` or `BoxAddrOp`) into multiple 
 mappings, with one extra per pointer member in the descriptor that is supported on top of the original
 descriptor map operation. These pointers members are linked to the parent descriptor by adding them to 
@@ -53,7 +53,7 @@ owning operation's (`omp.TargetOp`, `omp.TargetDataOp` etc.) map operand list an
 operation is `IsolatedFromAbove`, it also inserts them as `BlockArgs` to canonicalize the mappings and
 simplify lowering.
 
-An example transformation by the `OMPDescriptorMapInfoGenPass`:
+An example transformation by the `OMPMapInfoFinalizationPass`:
 
 ```
 
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index 470ed8a125ac..ae1d72a3526b 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -68,7 +68,7 @@ std::unique_ptr<mlir::Pass> createAlgebraicSimplificationPass();
 std::unique_ptr<mlir::Pass>
 createAlgebraicSimplificationPass(const mlir::GreedyRewriteConfig &config);
 
-std::unique_ptr<mlir::Pass> createOMPDescriptorMapInfoGenPass();
+std::unique_ptr<mlir::Pass> createOMPMapInfoFinalizationPass();
 std::unique_ptr<mlir::Pass> createOMPFunctionFilteringPass();
 std::unique_ptr<mlir::OperationPass<mlir::ModuleOp>>
 createOMPMarkDeclareTargetPass();
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 1eaaa32a508a..e22c1b5f338b 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -321,15 +321,15 @@ def LoopVersioning : Pass<"loop-versioning", "mlir::func::FuncOp"> {
   let dependentDialects = [ "fir::FIROpsDialect" ];
 }
 
-def OMPDescriptorMapInfoGenPass
-    : Pass<"omp-descriptor-map-info-gen", "mlir::func::FuncOp"> {
+def OMPMapInfoFinalizationPass
+    : Pass<"omp-map-info-finalization", "mlir::func::FuncOp"> {
   let summary = "expands OpenMP MapInfo operations containing descriptors";
   let description = [{
     Expands MapInfo operations containing descriptor types into multiple 
     MapInfo's for each pointer element in the descriptor that requires 
     explicit individual mapping by the OpenMP runtime.
   }];
-  let constructor = "::fir::createOMPDescriptorMapInfoGenPass()";
+  let constructor = "::fir::createOMPMapInfoFinalizationPass()";
   let dependentDialects = ["mlir::omp::OpenMPDialect"];
 }
 
diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc
index 79a2a4f63cfc..cc3431d5b71d 100644
--- a/flang/include/flang/Tools/CLOptions.inc
+++ b/flang/include/flang/Tools/CLOptions.inc
@@ -335,7 +335,7 @@ inline void createHLFIRToFIRPassPipeline(
 /// rather than the host device.
 inline void createOpenMPFIRPassPipeline(
     mlir::PassManager &pm, bool isTargetDevice) {
-  pm.addPass(fir::createOMPDescriptorMapInfoGenPass());
+  pm.addPass(fir::createOMPMapInfoFinalizationPass());
   pm.addPass(fir::createOMPMarkDeclareTargetPass());
   if (isTargetDevice)
     pm.addPass(fir::createOMPFunctionFilteringPass());
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index b7198c951c8f..0ea87314d571 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -814,30 +814,6 @@ bool ClauseProcessor::processLink(
       });
 }
 
-mlir::omp::MapInfoOp
-createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
-                mlir::Value baseAddr, mlir::Value varPtrPtr, std::string name,
-                llvm::ArrayRef<mlir::Value> bounds,
-                llvm::ArrayRef<mlir::Value> members, uint64_t mapType,
-                mlir::omp::VariableCaptureKind mapCaptureType, mlir::Type retTy,
-                bool isVal) {
-  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(baseAddr.getType())) {
-    baseAddr = builder.create<fir::BoxAddrOp>(loc, baseAddr);
-    retTy = baseAddr.getType();
-  }
-
-  mlir::TypeAttr varType = mlir::TypeAttr::get(
-      llvm::cast<mlir::omp::PointerLikeType>(retTy).getElementType());
-
-  mlir::omp::MapInfoOp op = builder.create<mlir::omp::MapInfoOp>(
-      loc, retTy, baseAddr, varType, varPtrPtr, members, bounds,
-      builder.getIntegerAttr(builder.getIntegerType(64, false), mapType),
-      builder.getAttr<mlir::omp::VariableCaptureKindAttr>(mapCaptureType),
-      builder.getStringAttr(name));
-
-  return op;
-}
-
 bool ClauseProcessor::processMap(
     mlir::Location currentLocation, Fortran::lower::StatementContext &stmtCtx,
     mlir::omp::MapClauseOps &result,
@@ -845,7 +821,17 @@ bool ClauseProcessor::processMap(
     llvm::SmallVectorImpl<mlir::Location> *mapSymLocs,
     llvm::SmallVectorImpl<mlir::Type> *mapSymTypes) const {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  return findRepeatableClause<omp::clause::Map>(
+  // We always require tracking of symbols, even if the caller does not,
+  // so we create an optionally used local set of symbols when the mapSyms
+  // argument is not present.
+  llvm::SmallVector<const Fortran::semantics::Symbol *> localMapSyms;
+  llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *ptrMapSyms =
+      mapSyms ? mapSyms : &localMapSyms;
+  std::map<const Fortran::semantics::Symbol *,
+           llvm::SmallVector<OmpMapMemberIndicesData>>
+      parentMemberIndices;
+
+  bool clauseFound = findRepeatableClause<omp::clause::Map>(
       [&](const omp::clause::Map &clause,
           const Fortran::parser::CharBlock &source) {
         using Map = omp::clause::Map;
@@ -910,24 +896,33 @@ bool ClauseProcessor::processMap(
           // Explicit map captures are captured ByRef by default,
           // optimisation passes may alter this to ByCopy or other capture
           // types to optimise
-          mlir::Value mapOp = createMapInfoOp(
-              firOpBuilder, clauseLocation, symAddr, mlir::Value{},
-              asFortran.str(), bounds, {},
+          mlir::omp::MapInfoOp mapOp = createMapInfoOp(
+              firOpBuilder, clauseLocation, symAddr,
+              /*varPtrPtr=*/mlir::Value{}, asFortran.str(), bounds,
+              /*members=*/{}, /*membersIndex=*/mlir::DenseIntElementsAttr{},
               static_cast<
                   std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
                   mapTypeBits),
               mlir::omp::VariableCaptureKind::ByRef, symAddr.getType());
 
-          result.mapVars.push_back(mapOp);
-
-          if (mapSyms)
-            mapSyms->push_back(object.id());
-          if (mapSymLocs)
-            mapSymLocs->push_back(symAddr.getLoc());
-          if (mapSymTypes)
-            mapSymTypes->push_back(symAddr.getType());
+          if (object.id()->owner().IsDerivedType()) {
+            addChildIndexAndMapToParent(object, parentMemberIndices, mapOp,
+                                        semaCtx);
+          } else {
+            result.mapVars.push_back(mapOp);
+            ptrMapSyms->push_back(object.id());
+            if (mapSymTypes)
+              mapSymTypes->push_back(symAddr.getType());
+            if (mapSymLocs)
+              mapSymLocs->push_back(symAddr.getLoc());
+          }
         }
       });
+
+  insertChildMapInfoIntoParent(converter, parentMemberIndices, result.mapVars,
+                               *ptrMapSyms, mapSymTypes, mapSymLocs);
+
+  return clauseFound;
 }
 
 bool ClauseProcessor::processReduction(
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 78c148ab0216..54ffcd0c964b 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -185,7 +185,12 @@ template <typename T>
 bool ClauseProcessor::processMotionClauses(
     Fortran::lower::StatementContext &stmtCtx,
     mlir::omp::MapClauseOps &result) {
-  return findRepeatableClause<T>(
+  std::map<const Fortran::semantics::Symbol *,
+           llvm::SmallVector<OmpMapMemberIndicesData>>
+      parentMemberIndices;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> mapSymbols;
+
+  bool clauseFound = findRepeatableClause<T>(
       [&](const T &clause, const Fortran::parser::CharBlock &source) {
         mlir::Location clauseLocation = converter.genLocation(source);
         fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
@@ -203,6 +208,7 @@ bool ClauseProcessor::processMotionClauses(
         for (const omp::Object &object : objects) {
           llvm::SmallVector<mlir::Value> bounds;
           std::stringstream asFortran;
+
           Fortran::lower::AddrAndBoundsInfo info =
               Fortran::lower::gatherDataOperandAddrAndBounds<
                   mlir::omp::MapBoundsOp, mlir::omp::MapBoundsType>(
@@ -218,17 +224,29 @@ bool ClauseProcessor::processMotionClauses(
           // Explicit map captures are captured ByRef by default,
           // optimisation passes may alter this to ByCopy or other capture
           // types to optimise
-          mlir::Value mapOp = createMapInfoOp(
-              firOpBuilder, clauseLocation, symAddr, mlir::Value{},
-              asFortran.str(), bounds, {},
+          mlir::omp::MapInfoOp mapOp = createMapInfoOp(
+              firOpBuilder, clauseLocation, symAddr,
+              /*varPtrPtr=*/mlir::Value{}, asFortran.str(), bounds,
+              /*members=*/{}, /*membersIndex=*/mlir::DenseIntElementsAttr{},
               static_cast<
                   std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
                   mapTypeBits),
               mlir::omp::VariableCaptureKind::ByRef, symAddr.getType());
 
-          result.mapVars.push_back(mapOp);
+          if (object.id()->owner().IsDerivedType()) {
+            addChildIndexAndMapToParent(object, parentMemberIndices, mapOp,
+                                        semaCtx);
+          } else {
+            result.mapVars.push_back(mapOp);
+            mapSymbols.push_back(object.id());
+          }
         }
       });
+
+  insertChildMapInfoIntoParent(converter, parentMemberIndices, result.mapVars,
+                               mapSymbols,
+                               /*mapSymTypes=*/nullptr, /*mapSymLocs=*/nullptr);
+  return clauseFound;
 }
 
 template <typename... Ts>
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 44011ad78f2e..f23902d6a823 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -939,8 +939,10 @@ genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
         std::stringstream name;
         firOpBuilder.setInsertionPoint(targetOp);
         mlir::Value mapOp = createMapInfoOp(
-            firOpBuilder, copyVal.getLoc(), copyVal, mlir::Value{}, name.str(),
-            bounds, llvm::SmallVector<mlir::Value>{},
+            firOpBuilder, copyVal.getLoc(), copyVal,
+            /*varPtrPtr=*/mlir::Value{}, name.str(), bounds,
+            /*members=*/llvm::SmallVector<mlir::Value>{},
+            /*membersIndex=*/mlir::DenseIntElementsAttr{},
             static_cast<
                 std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
                 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT),
@@ -1637,8 +1639,9 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
         }
 
         mlir::Value mapOp = createMapInfoOp(
-            firOpBuilder, baseOp.getLoc(), baseOp, mlir::Value{}, name.str(),
-            bounds, {},
+            firOpBuilder, baseOp.getLoc(), baseOp, /*varPtrPtr=*/mlir::Value{},
+            name.str(), bounds, /*members=*/{},
+            /*membersIndex=*/mlir::DenseIntElementsAttr{},
             static_cast<
                 std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
                 mapFlag),
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index c38e0c18cac8..b183705c5677 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -22,6 +22,9 @@
 #include <flang/Semantics/tools.h>
 #include <llvm/Support/CommandLine.h>
 
+#include <algorithm>
+#include <numeric>
+
 llvm::cl::opt<bool> treatIndexAsSection(
     "openmp-treat-index-as-section",
     llvm::cl::desc("In the OpenMP data clauses treat `a(N)` as `a(N:N)`."),
@@ -116,6 +119,216 @@ void gatherFuncAndVarSyms(
     symbolAndClause.emplace_back(clause, *object.id());
 }
 
+mlir::omp::MapInfoOp
+createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
+                mlir::Value baseAddr, mlir::Value varPtrPtr, std::string name,
+                llvm::ArrayRef<mlir::Value> bounds,
+                llvm::ArrayRef<mlir::Value> members,
+                mlir::DenseIntElementsAttr membersIndex, uint64_t mapType,
+                mlir::omp::VariableCaptureKind mapCaptureType, mlir::Type retTy,
+                bool partialMap) {
+  if (auto boxTy = baseAddr.getType().dyn_cast<fir::BaseBoxType>()) {
+    baseAddr = builder.create<fir::BoxAddrOp>(loc, baseAddr);
+    retTy = baseAddr.getType();
+  }
+
+  mlir::TypeAttr varType = mlir::TypeAttr::get(
+      llvm::cast<mlir::omp::PointerLikeType>(retTy).getElementType());
+
+  mlir::omp::MapInfoOp op = builder.create<mlir::omp::MapInfoOp>(
+      loc, retTy, baseAddr, varType, varPtrPtr, members, membersIndex, bounds,
+      builder.getIntegerAttr(builder.getIntegerType(64, false), mapType),
+      builder.getAttr<mlir::omp::VariableCaptureKindAttr>(mapCaptureType),
+      builder.getStringAttr(name), builder.getBoolAttr(partialMap));
+
+  return op;
+}
+
+static int
+getComponentPlacementInParent(const Fortran::semantics::Symbol *componentSym) {
+  const auto *derived =
+      componentSym->owner()
+          .derivedTypeSpec()
+          ->typeSymbol()
+          .detailsIf<Fortran::semantics::DerivedTypeDetails>();
+  assert(derived &&
+         "expected derived type details when processing component symbol");
+  for (auto [placement, name] : llvm::enumerate(derived->componentNames()))
+    if (name == componentSym->name())
+      return placement;
+  return -1;
+}
+
+static std::optional<Object>
+getComponentObject(std::optional<Object> object,
+                   Fortran::semantics::SemanticsContext &semaCtx) {
+  if (!object)
+    return std::nullopt;
+
+  auto ref = evaluate::ExtractDataRef(*object.value().ref());
+  if (!ref)
+    return std::nullopt;
+
+  if (std::holds_alternative<evaluate::Component>(ref->u))
+    return object;
+
+  auto baseObj = getBaseObject(object.value(), semaCtx);
+  if (!baseObj)
+    return std::nullopt;
+
+  return getComponentObject(baseObj.value(), semaCtx);
+}
+
+static void
+generateMemberPlacementIndices(const Object &object,
+                               llvm::SmallVectorImpl<int> &indices,
+                               Fortran::semantics::SemanticsContext &semaCtx) {
+  auto compObj = getComponentObject(object, semaCtx);
+  while (compObj) {
+    indices.push_back(getComponentPlacementInParent(compObj->id()));
+    compObj =
+        getComponentObject(getBaseObject(compObj.value(), semaCtx), semaCtx);
+  }
+
+  indices = llvm::SmallVector<int>{llvm::reverse(indices)};
+}
+
+void addChildIndexAndMapToParent(
+    const omp::Object &object,
+    std::map<const Fortran::semantics::Symbol *,
+             llvm::SmallVector<OmpMapMemberIndicesData>> &parentMemberIndices,
+    mlir::omp::MapInfoOp &mapOp,
+    Fortran::semantics::SemanticsContext &semaCtx) {
+  std::optional<Fortran::evaluate::DataRef> dataRef =
+      ExtractDataRef(object.designator);
+  assert(dataRef.has_value() &&
+         "DataRef could not be extracted during mapping of derived type "
+         "cannot proceed");
+  const Fortran::semantics::Symbol *parentSym = &dataRef->GetFirstSymbol();
+  assert(parentSym && "Could not find parent symbol during lower of "
+                      "a component member in OpenMP map clause");
+  llvm::SmallVector<int> indices;
+  generateMemberPlacementIndices(object, indices, semaCtx);
+  parentMemberIndices[parentSym].push_back({indices, mapOp});
+}
+
+static void calculateShapeAndFillIndices(
+    llvm::SmallVectorImpl<int64_t> &shape,
+    llvm::SmallVectorImpl<OmpMapMemberIndicesData> &memberPlacementData) {
+  shape.push_back(memberPlacementData.size());
+  size_t largestIndicesSize =
+      std::max_element(memberPlacementData.begin(), memberPlacementData.end(),
+                       [](auto a, auto b) {
+                         return a.memberPlacementIndices.size() <
+                                b.memberPlacementIndices.size();
+                       })
+          ->memberPlacementIndices.size();
+  shape.push_back(largestIndicesSize);
+
+  // DenseElementsAttr expects a rectangular shape for the data, so all
+  // index lists have to be of the same length, this emplaces -1 as filler.
+  for (auto &v : memberPlacementData) {
+    if (v.memberPlacementIndices.size() < largestIndicesSize) {
+      auto *prevEnd = v.memberPlacementIndices.end();
+      v.memberPlacementIndices.resize(largestIndicesSize);
+      std::fill(prevEnd, v.memberPlacementIndices.end(), -1);
+    }
+  }
+}
+
+static mlir::DenseIntElementsAttr createDenseElementsAttrFromIndices(
+    llvm::SmallVectorImpl<OmpMapMemberIndicesData> &memberPlacementData,
+    fir::FirOpBuilder &builder) {
+  llvm::SmallVector<int64_t> shape;
+  calculateShapeAndFillIndices(shape, memberPlacementData);
+
+  llvm::SmallVector<int> indicesFlattened = std::accumulate(
+      memberPlacementData.begin(), memberPlacementData.end(),
+      llvm::SmallVector<int>(),
+      [](llvm::SmallVector<int> &x, OmpMapMemberIndicesData y) {
+        x.insert(x.end(), y.memberPlacementIndices.begin(),
+                 y.memberPlacementIndices.end());
+        return x;
+      });
+
+  return mlir::DenseIntElementsAttr::get(
+      mlir::VectorType::get(shape,
+                            mlir::IntegerType::get(builder.getContext(), 32)),
+      indicesFlattened);
+}
+
+void insertChildMapInfoIntoParent(
+    Fortran::lower::AbstractConverter &converter,
+    std::map<const Fortran::semantics::Symbol *,
+             llvm::SmallVector<OmpMapMemberIndicesData>> &parentMemberIndices,
+    llvm::SmallVectorImpl<mlir::Value> &mapOperands,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &mapSyms,
+    llvm::SmallVectorImpl<mlir::Type> *mapSymTypes,
+    llvm::SmallVectorImpl<mlir::Location> *mapSymLocs) {
+  for (auto indices : parentMemberIndices) {
+    bool parentExists = false;
+    size_t parentIdx;
+    for (parentIdx = 0; parentIdx < mapSyms.size(); ++parentIdx) {
+      if (mapSyms[parentIdx] == indices.first) {
+        parentExists = true;
+        break;
+      }
+    }
+
+    if (parentExists) {
+      auto mapOp = llvm::cast<mlir::omp::MapInfoOp>(
+          mapOperands[parentIdx].getDefiningOp());
+
+      // NOTE: To maintain appropriate SSA ordering, we move the parent map
+      // which will now have references to its children after the last
+      // of its members to be generated. This is necessary when a user
+      // has defined a series of parent and children maps where the parent
+      // precedes the children. An alternative, may be to do
+      // delayed generation of map info operations from the clauses and
+      // organize them first before generation.
+      mapOp->moveAfter(indices.second.back().memberMap);
+
+      for (auto memberIndicesData : indices.second)
+        mapOp.getMembersMutable().append(
+            memberIndicesData.memberMap.getResult());
+
+      mapOp.setMembersIndexAttr(createDenseElementsAttrFromIndices(
+          indices.second, converter.getFirOpBuilder()));
+    } else {
+      // NOTE: We take the map type of the first child, this may not
+      // be the correct thing to do, however, we shall see. For the moment
+      // it allows this to work with enter and exit without causing MLIR
+      // verification issues. The more appropriate thing may be to take
+      // the "main" map type clause from the directive being used.
+      uint64_t mapType = indices.second[0].memberMap.getMapType().value_or(0);
+
+      // create parent to emplace and bind members
+      mlir::Value origSymbol = converter.getSymbolAddress(*indices.first);
+
+      llvm::SmallVector<mlir::Value> members;
+      for (OmpMapMemberIndicesData memberIndicesData : indices.second)
+        members.push_back((mlir::Value)memberIndicesData.memberMap);
+
+      mlir::Value mapOp = createMapInfoOp(
+          converter.getFirOpBuilder(), origSymbol.getLoc(), origSymbol,
+          /*varPtrPtr=*/mlir::Value(), indices.first->name().ToString(),
+          /*bounds=*/{}, members,
+          createDenseElementsAttrFromIndices(indices.second,
+                                             converter.getFirOpBuilder()),
+          mapType, mlir::omp::VariableCaptureKind::ByRef, origSymbol.getType(),
+          /*partialMap=*/true);
+
+      mapOperands.push_back(mapOp);
+      mapSyms.push_back(indices.first);
+
+      if (mapSymTypes)
+        mapSymTypes->push_back(mapOp.getType());
+      if (mapSymLocs)
+        mapSymLocs->push_back(mapOp.getLoc());
+    }
+  }
+}
+
 Fortran::semantics::Symbol *
 getOmpObjectSymbol(const Fortran::parser::OmpObject &ompObject) {
   Fortran::semantics::Symbol *sym = nullptr;
@@ -125,7 +338,11 @@ getOmpObjectSymbol(const Fortran::parser::OmpObject &ompObject) {
             if (auto *arrayEle =
                     Fortran::parser::Unwrap<Fortran::parser::ArrayElement>(
                         designator)) {
-              sym = GetFirstName(arrayEle->base).symbol;
+              // Use getLastName to retrieve the arrays symbol, this will
+              // provide the farthest right symbol (the last) in a designator,
+              // i.e. providing something like the following:
+              // "dtype1%dtype2%array[2:10]", will result in "array"
+              sym = GetLastName(arrayEle->base).symbol;
             } else if (auto *structComp = Fortran::parser::Unwrap<
                            Fortran::parser::StructureComponent>(designator)) {
               sym = structComp->component.symbol;
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 5e0ebba23bf3..8fbb18fa8656 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -21,7 +21,6 @@ extern llvm::cl::opt<bool> enableDelayedPrivatization;
 namespace fir {
 class FirOpBuilder;
 } // namespace fir
-
 namespace Fortran {
 
 namespace semantics {
@@ -46,13 +45,42 @@ using DeclareTargetCapturePair =
     std::pair<mlir::omp::DeclareTargetCaptureClause,
               const Fortran::semantics::Symbol &>;
 
+// A small helper structure for keeping track of a component members MapInfoOp
+// and index data when lowering OpenMP map clauses. Keeps track of the
+// placement of the component in the derived type hierarchy it rests within,
+// alongside the generated mlir::omp::MapInfoOp for the mapped component.
+struct OmpMapMemberIndicesData {
+  // The indices representing the component members placement in its derived
+  // type parents hierarchy.
+  llvm::SmallVector<int> memberPlacementIndices;
+
+  // Placement of the member in the member vector.
+  mlir::omp::MapInfoOp memberMap;
+};
+
 mlir::omp::MapInfoOp
 createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
                 mlir::Value baseAddr, mlir::Value varPtrPtr, std::string name,
                 mlir::ArrayRef<mlir::Value> bounds,
-                mlir::ArrayRef<mlir::Value> members, uint64_t mapType,
+                mlir::ArrayRef<mlir::Value> members,
+                mlir::DenseIntElementsAttr membersIndex, uint64_t mapType,
                 mlir::omp::VariableCaptureKind mapCaptureType, mlir::Type retTy,
-                bool isVal = false);
+                bool partialMap = false);
+
+void addChildIndexAndMapToParent(
+    const omp::Object &object,
+    std::map<const Fortran::semantics::Symbol *,
+             llvm::SmallVector<OmpMapMemberIndicesData>> &parentMemberIndices,
+    mlir::omp::MapInfoOp &mapOp, Fortran::semantics::SemanticsContext &semaCtx);
+
+void insertChildMapInfoIntoParent(
+    Fortran::lower::AbstractConverter &converter,
+    std::map<const Fortran::semantics::Symbol *,
+             llvm::SmallVector<OmpMapMemberIndicesData>> &parentMemberIndices,
+    llvm::SmallVectorImpl<mlir::Value> &mapOperands,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &mapSyms,
+    llvm::SmallVectorImpl<mlir::Type> *mapSymTypes,
+    llvm::SmallVectorImpl<mlir::Location> *mapSymLocs);
 
 mlir::Type getLoopVarType(Fortran::lower::AbstractConverter &converter,
                           std::size_t loopVarTypeSize);
diff --git a/flang/lib/Optimizer/Transforms/CMakeLists.txt b/flang/lib/Optimizer/Transforms/CMakeLists.txt
index 5a542f237f8f..308b5ed06623 100644
--- a/flang/lib/Optimizer/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/Transforms/CMakeLists.txt
@@ -17,8 +17,8 @@ add_flang_library(FIRTransforms
   AddDebugInfo.cpp
   PolymorphicOpConversion.cpp
   LoopVersioning.cpp
-  OMPDescriptorMapInfoGen.cpp
   OMPFunctionFiltering.cpp
+  OMPMapInfoFinalization.cpp
   OMPMarkDeclareTarget.cpp
   VScaleAttr.cpp
   FunctionAttr.cpp
diff --git a/flang/lib/Optimizer/Transforms/OMPDescriptorMapInfoGen.cpp b/flang/lib/Optimizer/Transforms/OMPDescriptorMapInfoGen.cpp
deleted file mode 100644
index 6ffcf0746c76..000000000000
--- a/flang/lib/Optimizer/Transforms/OMPDescriptorMapInfoGen.cpp
+++ /dev/null
@@ -1,168 +0,0 @@
-//===- OMPDescriptorMapInfoGen.cpp
-//---------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-/// \file
-/// An OpenMP dialect related pass for FIR/HLFIR which expands MapInfoOp's
-/// containing descriptor related types (fir::BoxType's) into multiple
-/// MapInfoOp's containing the parent descriptor and pointer member components
-/// for individual mapping, treating the descriptor type as a record type for
-/// later lowering in the OpenMP dialect.
-//===----------------------------------------------------------------------===//
-
-#include "flang/Optimizer/Builder/FIRBuilder.h"
-#include "flang/Optimizer/Dialect/FIRType.h"
-#include "flang/Optimizer/Dialect/Support/KindMapping.h"
-#include "flang/Optimizer/Transforms/Passes.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
-#include "mlir/IR/BuiltinDialect.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/Operation.h"
-#include "mlir/IR/SymbolTable.h"
-#include "mlir/Pass/Pass.h"
-#include "mlir/Support/LLVM.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include <iterator>
-
-namespace fir {
-#define GEN_PASS_DEF_OMPDESCRIPTORMAPINFOGENPASS
-#include "flang/Optimizer/Transforms/Passes.h.inc"
-} // namespace fir
-
-namespace {
-class OMPDescriptorMapInfoGenPass
-    : public fir::impl::OMPDescriptorMapInfoGenPassBase<
-          OMPDescriptorMapInfoGenPass> {
-
-  void genDescriptorMemberMaps(mlir::omp::MapInfoOp op,
-                               fir::FirOpBuilder &builder,
-                               mlir::Operation *target) {
-    mlir::Location loc = builder.getUnknownLoc();
-    mlir::Value descriptor = op.getVarPtr();
-
-    // If we enter this function, but the mapped type itself is not the
-    // descriptor, then it's likely the address of the descriptor so we
-    // must retrieve the descriptor SSA.
-    if (!fir::isTypeWithDescriptor(op.getVarType())) {
-      if (auto addrOp = mlir::dyn_cast_if_present<fir::BoxAddrOp>(
-              op.getVarPtr().getDefiningOp())) {
-        descriptor = addrOp.getVal();
-      }
-    }
-
-    // The fir::BoxOffsetOp only works with !fir.ref<!fir.box<...>> types, as
-    // allowing it to access non-reference box operations can cause some
-    // problematic SSA IR. However, in the case of assumed shape's the type
-    // is not a !fir.ref, in these cases to retrieve the appropriate
-    // !fir.ref<!fir.box<...>> to access the data we need to map we must
-    // perform an alloca and then store to it and retrieve the data from the new
-    // alloca.
-    if (mlir::isa<fir::BaseBoxType>(descriptor.getType())) {
-      mlir::OpBuilder::InsertPoint insPt = builder.saveInsertionPoint();
-      builder.setInsertionPointToStart(builder.getAllocaBlock());
-      auto alloca = builder.create<fir::AllocaOp>(loc, descriptor.getType());
-      builder.restoreInsertionPoint(insPt);
-      builder.create<fir::StoreOp>(loc, descriptor, alloca);
-      descriptor = alloca;
-    }
-
-    mlir::Value baseAddrAddr = builder.create<fir::BoxOffsetOp>(
-        loc, descriptor, fir::BoxFieldAttr::base_addr);
-
-    // Member of the descriptor pointing at the allocated data
-    mlir::Value baseAddr = builder.create<mlir::omp::MapInfoOp>(
-        loc, baseAddrAddr.getType(), descriptor,
-        llvm::cast<mlir::omp::PointerLikeType>(
-            fir::unwrapRefType(baseAddrAddr.getType()))
-            .getElementType(),
-        baseAddrAddr, mlir::SmallVector<mlir::Value>{}, op.getBounds(),
-        builder.getIntegerAttr(builder.getIntegerType(64, false),
-                               op.getMapType().value()),
-        builder.getAttr<mlir::omp::VariableCaptureKindAttr>(
-            mlir::omp::VariableCaptureKind::ByRef),
-        builder.getStringAttr("") /*name*/);
-
-    // TODO: map the addendum segment of the descriptor, similarly to the
-    // above base address/data pointer member.
-
-    if (auto mapClauseOwner =
-            llvm::dyn_cast<mlir::omp::MapClauseOwningOpInterface>(target)) {
-      llvm::SmallVector<mlir::Value> newMapOps;
-      mlir::OperandRange mapOperandsArr = mapClauseOwner.getMapOperands();
-
-      for (size_t i = 0; i < mapOperandsArr.size(); ++i) {
-        if (mapOperandsArr[i] == op) {
-          // Push new implicit maps generated for the descriptor.
-          newMapOps.push_back(baseAddr);
-
-          // for TargetOp's which have IsolatedFromAbove we must align the
-          // new additional map operand with an appropriate BlockArgument,
-          // as the printing and later processing currently requires a 1:1
-          // mapping of BlockArgs to MapInfoOp's at the same placement in
-          // each array (BlockArgs and MapOperands).
-          if (auto targetOp = llvm::dyn_cast<mlir::omp::TargetOp>(target))
-            targetOp.getRegion().insertArgument(i, baseAddr.getType(), loc);
-        }
-        newMapOps.push_back(mapOperandsArr[i]);
-      }
-      mapClauseOwner.getMapOperandsMutable().assign(newMapOps);
-    }
-
-    mlir::Value newDescParentMapOp = builder.create<mlir::omp::MapInfoOp>(
-        op->getLoc(), op.getResult().getType(), descriptor,
-        fir::unwrapRefType(descriptor.getType()), mlir::Value{},
-        mlir::SmallVector<mlir::Value>{baseAddr},
-        mlir::SmallVector<mlir::Value>{},
-        builder.getIntegerAttr(builder.getIntegerType(64, false),
-                               op.getMapType().value()),
-        op.getMapCaptureTypeAttr(), op.getNameAttr());
-    op.replaceAllUsesWith(newDescParentMapOp);
-    op->erase();
-  }
-
-  // This pass executes on mlir::ModuleOp's finding omp::MapInfoOp's containing
-  // descriptor based types (allocatables, pointers, assumed shape etc.) and
-  // expanding them into multiple omp::MapInfoOp's for each pointer member
-  // contained within the descriptor.
-  void runOnOperation() override {
-    mlir::func::FuncOp func = getOperation();
-    mlir::ModuleOp module = func->getParentOfType<mlir::ModuleOp>();
-    fir::KindMapping kindMap = fir::getKindMapping(module);
-    fir::FirOpBuilder builder{module, std::move(kindMap)};
-
-    func->walk([&](mlir::omp::MapInfoOp op) {
-      if (fir::isTypeWithDescriptor(op.getVarType()) ||
-          mlir::isa_and_present<fir::BoxAddrOp>(
-              op.getVarPtr().getDefiningOp())) {
-        builder.setInsertionPoint(op);
-        // TODO: Currently only supports a single user for the MapInfoOp, this
-        // is fine for the moment as the Fortran Frontend will generate a
-        // new MapInfoOp per Target operation for the moment. However, when/if
-        // we optimise/cleanup the IR, it likely isn't too difficult to
-        // extend this function, it would require some modification to create a
-        // single new MapInfoOp per new MapInfoOp generated and share it across
-        // all users appropriately, making sure to only add a single member link
-        // per new generation for the original originating descriptor MapInfoOp.
-        assert(llvm::hasSingleElement(op->getUsers()) &&
-               "OMPDescriptorMapInfoGen currently only supports single users "
-               "of a MapInfoOp");
-        genDescriptorMemberMaps(op, builder, *op->getUsers().begin());
-      }
-    });
-  }
-};
-
-} // namespace
-
-namespace fir {
-std::unique_ptr<mlir::Pass> createOMPDescriptorMapInfoGenPass() {
-  return std::make_unique<OMPDescriptorMapInfoGenPass>();
-}
-} // namespace fir
diff --git a/flang/lib/Optimizer/Transforms/OMPMapInfoFinalization.cpp b/flang/lib/Optimizer/Transforms/OMPMapInfoFinalization.cpp
new file mode 100644
index 000000000000..5a5d4b9e0da4
--- /dev/null
+++ b/flang/lib/Optimizer/Transforms/OMPMapInfoFinalization.cpp
@@ -0,0 +1,261 @@
+//===- OMPMapInfoFinalization.cpp
+//---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+/// \file
+/// An OpenMP dialect related pass for FIR/HLFIR which performs some
+/// pre-processing of MapInfoOp's after the module has been lowered to
+/// finalize them.
+///
+/// For example, it expands MapInfoOp's containing descriptor related
+/// types (fir::BoxType's) into multiple MapInfoOp's containing the parent
+/// descriptor and pointer member components for individual mapping,
+/// treating the descriptor type as a record type for later lowering in the
+/// OpenMP dialect.
+///
+/// The pass also adds MapInfoOp's that are members of a parent object but are
+/// not directly used in the body of a target region to its BlockArgument list
+/// to maintain consistency across all MapInfoOp's tied to a region directly or
+/// indirectly via a parent object.
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "flang/Optimizer/Dialect/Support/KindMapping.h"
+#include "flang/Optimizer/Transforms/Passes.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/IR/BuiltinDialect.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/IR/SymbolTable.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Support/LLVM.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
+#include <iterator>
+
+namespace fir {
+#define GEN_PASS_DEF_OMPMAPINFOFINALIZATIONPASS
+#include "flang/Optimizer/Transforms/Passes.h.inc"
+} // namespace fir
+
+namespace {
+class OMPMapInfoFinalizationPass
+    : public fir::impl::OMPMapInfoFinalizationPassBase<
+          OMPMapInfoFinalizationPass> {
+
+  void genDescriptorMemberMaps(mlir::omp::MapInfoOp op,
+                               fir::FirOpBuilder &builder,
+                               mlir::Operation *target) {
+    mlir::Location loc = op.getLoc();
+    mlir::Value descriptor = op.getVarPtr();
+
+    // If we enter this function, but the mapped type itself is not the
+    // descriptor, then it's likely the address of the descriptor so we
+    // must retrieve the descriptor SSA.
+    if (!fir::isTypeWithDescriptor(op.getVarType())) {
+      if (auto addrOp = mlir::dyn_cast_if_present<fir::BoxAddrOp>(
+              op.getVarPtr().getDefiningOp())) {
+        descriptor = addrOp.getVal();
+      }
+    }
+
+    // The fir::BoxOffsetOp only works with !fir.ref<!fir.box<...>> types, as
+    // allowing it to access non-reference box operations can cause some
+    // problematic SSA IR. However, in the case of assumed shape's the type
+    // is not a !fir.ref, in these cases to retrieve the appropriate
+    // !fir.ref<!fir.box<...>> to access the data we need to map we must
+    // perform an alloca and then store to it and retrieve the data from the new
+    // alloca.
+    if (mlir::isa<fir::BaseBoxType>(descriptor.getType())) {
+      mlir::OpBuilder::InsertPoint insPt = builder.saveInsertionPoint();
+      builder.setInsertionPointToStart(builder.getAllocaBlock());
+      auto alloca = builder.create<fir::AllocaOp>(loc, descriptor.getType());
+      builder.restoreInsertionPoint(insPt);
+      builder.create<fir::StoreOp>(loc, descriptor, alloca);
+      descriptor = alloca;
+    }
+
+    mlir::Value baseAddrAddr = builder.create<fir::BoxOffsetOp>(
+        loc, descriptor, fir::BoxFieldAttr::base_addr);
+
+    // Member of the descriptor pointing at the allocated data
+    mlir::Value baseAddr = builder.create<mlir::omp::MapInfoOp>(
+        loc, baseAddrAddr.getType(), descriptor,
+        mlir::TypeAttr::get(llvm::cast<mlir::omp::PointerLikeType>(
+                                fir::unwrapRefType(baseAddrAddr.getType()))
+                                .getElementType()),
+        baseAddrAddr, /*members=*/mlir::SmallVector<mlir::Value>{},
+        /*member_index=*/mlir::DenseIntElementsAttr{}, op.getBounds(),
+        builder.getIntegerAttr(builder.getIntegerType(64, false),
+                               op.getMapType().value()),
+        builder.getAttr<mlir::omp::VariableCaptureKindAttr>(
+            mlir::omp::VariableCaptureKind::ByRef),
+        /*name=*/builder.getStringAttr(""),
+        /*partial_map=*/builder.getBoolAttr(false));
+
+    // TODO: map the addendum segment of the descriptor, similarly to the
+    // above base address/data pointer member.
+
+    if (auto mapClauseOwner =
+            llvm::dyn_cast<mlir::omp::MapClauseOwningOpInterface>(target)) {
+      llvm::SmallVector<mlir::Value> newMapOps;
+      mlir::OperandRange mapOperandsArr = mapClauseOwner.getMapOperands();
+
+      for (size_t i = 0; i < mapOperandsArr.size(); ++i) {
+        if (mapOperandsArr[i] == op) {
+          // Push new implicit maps generated for the descriptor.
+          newMapOps.push_back(baseAddr);
+
+          // for TargetOp's which have IsolatedFromAbove we must align the
+          // new additional map operand with an appropriate BlockArgument,
+          // as the printing and later processing currently requires a 1:1
+          // mapping of BlockArgs to MapInfoOp's at the same placement in
+          // each array (BlockArgs and MapOperands).
+          if (auto targetOp = llvm::dyn_cast<mlir::omp::TargetOp>(target))
+            targetOp.getRegion().insertArgument(i, baseAddr.getType(), loc);
+        }
+        newMapOps.push_back(mapOperandsArr[i]);
+      }
+      mapClauseOwner.getMapOperandsMutable().assign(newMapOps);
+    }
+
+    mlir::Value newDescParentMapOp = builder.create<mlir::omp::MapInfoOp>(
+        op->getLoc(), op.getResult().getType(), descriptor,
+        mlir::TypeAttr::get(fir::unwrapRefType(descriptor.getType())),
+        /*varPtrPtr=*/mlir::Value{},
+        /*members=*/mlir::SmallVector<mlir::Value>{baseAddr},
+        /*members_index=*/
+        mlir::DenseIntElementsAttr::get(
+            mlir::VectorType::get(
+                llvm::ArrayRef<int64_t>({1, 1}),
+                mlir::IntegerType::get(builder.getContext(), 32)),
+            llvm::ArrayRef<int32_t>({0})),
+        /*bounds=*/mlir::SmallVector<mlir::Value>{},
+        builder.getIntegerAttr(builder.getIntegerType(64, false),
+                               op.getMapType().value()),
+        op.getMapCaptureTypeAttr(), op.getNameAttr(), op.getPartialMapAttr());
+    op.replaceAllUsesWith(newDescParentMapOp);
+    op->erase();
+  }
+
+  // We add all mapped record members not directly used in the target region
+  // to the block arguments in front of their parent and we place them into
+  // the map operands list for consistency.
+  //
+  // These indirect uses (via accesses to their parent) will still be
+  // mapped individually in most cases, and a parent mapping doesn't
+  // guarantee the parent will be mapped in its totality, partial
+  // mapping is common.
+  //
+  // For example:
+  //    map(tofrom: x%y)
+  //
+  // Will generate a mapping for "x" (the parent) and "y" (the member).
+  // The parent "x" will not be mapped, but the member "y" will.
+  // However, we must have the parent as a BlockArg and MapOperand
+  // in these cases, to maintain the correct uses within the region and
+  // to help tracking that the member is part of a larger object.
+  //
+  // In the case of:
+  //    map(tofrom: x%y, x%z)
+  //
+  // The parent member becomes more critical, as we perform a partial
+  // structure mapping where we link the mapping of the members y
+  // and z together via the parent x. We do this at a kernel argument
+  // level in LLVM IR and not just MLIR, which is important to maintain
+  // similarity to Clang and for the runtime to do the correct thing.
+  // However, we still do not map the structure in its totality but
+  // rather we generate an un-sized "binding" map entry for it.
+  //
+  // In the case of:
+  //    map(tofrom: x, x%y, x%z)
+  //
+  // We do actually map the entirety of "x", so the explicit mapping of
+  // x%y, x%z becomes unnecessary. It is redundant to write this from a
+  // Fortran OpenMP perspective (although it is legal), as even if the
+  // members were allocatables or pointers, we are mandated by the
+  // specification to map these (and any recursive components) in their
+  // entirety, which is different to the C++ equivalent, which requires
+  // explicit mapping of these segments.
+  void addImplicitMembersToTarget(mlir::omp::MapInfoOp op,
+                                  fir::FirOpBuilder &builder,
+                                  mlir::Operation *target) {
+    auto mapClauseOwner =
+        llvm::dyn_cast<mlir::omp::MapClauseOwningOpInterface>(target);
+    if (!mapClauseOwner)
+      return;
+
+    llvm::SmallVector<mlir::Value> newMapOps;
+    mlir::OperandRange mapOperandsArr = mapClauseOwner.getMapOperands();
+    auto targetOp = llvm::dyn_cast<mlir::omp::TargetOp>(target);
+
+    for (size_t i = 0; i < mapOperandsArr.size(); ++i) {
+      if (mapOperandsArr[i] == op) {
+        for (auto [j, mapMember] : llvm::enumerate(op.getMembers())) {
+          newMapOps.push_back(mapMember);
+          // for TargetOp's which have IsolatedFromAbove we must align the
+          // new additional map operand with an appropriate BlockArgument,
+          // as the printing and later processing currently requires a 1:1
+          // mapping of BlockArgs to MapInfoOp's at the same placement in
+          // each array (BlockArgs and MapOperands).
+          if (targetOp) {
+            targetOp.getRegion().insertArgument(i + j, mapMember.getType(),
+                                                targetOp->getLoc());
+          }
+        }
+      }
+      newMapOps.push_back(mapOperandsArr[i]);
+    }
+    mapClauseOwner.getMapOperandsMutable().assign(newMapOps);
+  }
+
+  // This pass executes on mlir::ModuleOp's finding omp::MapInfoOp's containing
+  // descriptor based types (allocatables, pointers, assumed shape etc.) and
+  // expanding them into multiple omp::MapInfoOp's for each pointer member
+  // contained within the descriptor.
+  void runOnOperation() override {
+    mlir::func::FuncOp func = getOperation();
+    mlir::ModuleOp module = func->getParentOfType<mlir::ModuleOp>();
+    fir::KindMapping kindMap = fir::getKindMapping(module);
+    fir::FirOpBuilder builder{module, std::move(kindMap)};
+
+    func->walk([&](mlir::omp::MapInfoOp op) {
+      // TODO: Currently only supports a single user for the MapInfoOp, this
+      // is fine for the moment as the Fortran Frontend will generate a
+      // new MapInfoOp per Target operation for the moment. However, when/if
+      // we optimise/cleanup the IR, it likely isn't too difficult to
+      // extend this function, it would require some modification to create a
+      // single new MapInfoOp per new MapInfoOp generated and share it across
+      // all users appropriately, making sure to only add a single member link
+      // per new generation for the original originating descriptor MapInfoOp.
+      assert(llvm::hasSingleElement(op->getUsers()) &&
+             "OMPMapInfoFinalization currently only supports single users "
+             "of a MapInfoOp");
+
+      if (!op.getMembers().empty()) {
+        addImplicitMembersToTarget(op, builder, *op->getUsers().begin());
+      } else if (fir::isTypeWithDescriptor(op.getVarType()) ||
+                 mlir::isa_and_present<fir::BoxAddrOp>(
+                     op.getVarPtr().getDefiningOp())) {
+        builder.setInsertionPoint(op);
+        genDescriptorMemberMaps(op, builder, *op->getUsers().begin());
+      }
+    });
+  }
+};
+
+} // namespace
+
+namespace fir {
+std::unique_ptr<mlir::Pass> createOMPMapInfoFinalizationPass() {
+  return std::make_unique<OMPMapInfoFinalizationPass>();
+}
+} // namespace fir
diff --git a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
index 8cf4f566964f..72cd0a763e71 100644
--- a/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
+++ b/flang/test/Fir/convert-to-llvm-openmp-and-fir.fir
@@ -937,11 +937,68 @@ func.func @omp_map_info_descriptor_type_conversion(%arg0 : !fir.ref<!fir.box<!fi
   %0 = fir.box_offset %arg0 base_addr : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
   // CHECK: %[[MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[GEP]] : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
   %1 = omp.map.info var_ptr(%0 : !fir.llvm_ptr<!fir.ref<i32>>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.llvm_ptr<!fir.ref<i32>> {name = ""}
-  // CHECK: %[[DESC_MAP:.*]] = omp.map.info var_ptr(%[[ARG_0]] : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>) map_clauses(always, delete) capture(ByRef) members(%[[MEMBER_MAP]] : !llvm.ptr) -> !llvm.ptr {name = ""}
-  %2 = omp.map.info var_ptr(%arg0 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(always, delete) capture(ByRef) members(%1 : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.ref<!fir.box<!fir.heap<i32>>> {name = ""}
+  // CHECK: %[[DESC_MAP:.*]] = omp.map.info var_ptr(%[[ARG_0]] : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>) map_clauses(always, delete) capture(ByRef) members(%[[MEMBER_MAP]] : [0] : !llvm.ptr) -> !llvm.ptr {name = ""}
+  %2 = omp.map.info var_ptr(%arg0 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(always, delete) capture(ByRef) members(%1 : [0] : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.ref<!fir.box<!fir.heap<i32>>> {name = ""}
   // CHECK: omp.target_exit_data map_entries(%[[DESC_MAP]] : !llvm.ptr) 
   omp.target_exit_data   map_entries(%2 : !fir.ref<!fir.box<!fir.heap<i32>>>)
   return 
 }
 
 // -----
+
+// CHECK-LABEL:  llvm.func @omp_map_info_derived_type_explicit_member_conversion
+// CHECK-SAME:   %[[ARG_0:.*]]: !llvm.ptr)
+
+func.func @omp_map_info_derived_type_explicit_member_conversion(%arg0 : !fir.ref<!fir.type<_QFderived_type{real:f32,array:!fir.array<10xi32>,int:i32}>>) {
+  // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ARG_0]][0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFderived_type", (f32, array<10 x i32>, i32)>
+  %0 = fir.field_index int, !fir.type<_QFderived_type{real:f32,array:!fir.array<10xi32>,int:i32}>
+  %1 = fir.coordinate_of %arg0, %0 : (!fir.ref<!fir.type<_QFderived_type{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.field) -> !fir.ref<i32>
+  // CHECK: %[[MAP_MEMBER_1:.*]] = omp.map.info var_ptr(%[[GEP]] : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = "dtype%int"}
+  %2 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "dtype%int"}
+  // CHECK: %[[GEP_2:.*]] = llvm.getelementptr %[[ARG_0]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFderived_type", (f32, array<10 x i32>, i32)>
+  %3 = fir.field_index real, !fir.type<_QFderived_type{real:f32,array:!fir.array<10xi32>,int:i32}>
+  %4 = fir.coordinate_of %arg0, %3 : (!fir.ref<!fir.type<_QFderived_type{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.field) -> !fir.ref<f32>
+  // CHECK: %[[MAP_MEMBER_2:.*]] = omp.map.info var_ptr(%[[GEP_2]] : !llvm.ptr, f32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = "dtype%real"}
+  %5 = omp.map.info var_ptr(%4 : !fir.ref<f32>, f32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<f32> {name = "dtype%real"}    
+  // CHECK: %[[MAP_PARENT:.*]] = omp.map.info var_ptr(%[[ARG_0]] : !llvm.ptr, !llvm.struct<"_QFderived_type", (f32, array<10 x i32>, i32)>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_MEMBER_1]], %[[MAP_MEMBER_2]] : [2], [0] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {name = "dtype", partial_map = true} 
+  %6 = omp.map.info var_ptr(%arg0 : !fir.ref<!fir.type<_QFderived_type{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFderived_type{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(tofrom) capture(ByRef) members(%2, %5 : [2], [0] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFderived_type{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "dtype", partial_map = true}
+  // CHECK: omp.target map_entries(%[[MAP_MEMBER_1]] -> %[[ARG_1:.*]], %[[MAP_MEMBER_2]] -> %[[ARG_2:.*]], %[[MAP_PARENT]] -> %[[ARG_3:.*]] : !llvm.ptr, !llvm.ptr, !llvm.ptr) {
+  // CHECK: ^bb0(%[[ARG_1]]: !llvm.ptr, %[[ARG_2]]: !llvm.ptr, %[[ARG_3]]: !llvm.ptr):
+  omp.target map_entries(%2 -> %arg1, %5 -> %arg2, %6 -> %arg3 : !fir.ref<i32>, !fir.ref<f32>, !fir.ref<!fir.type<_QFderived_type{real:f32,array:!fir.array<10xi32>,int:i32}>>) {
+  ^bb0(%arg1: !fir.ref<f32>, %arg2: !fir.ref<i32>, %arg3: !fir.ref<!fir.type<_QFderived_type{real:f32,array:!fir.array<10xi32>,int:i32}>>):
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL:  llvm.func @omp_map_info_nested_derived_type_explicit_member_conversion
+// CHECK-SAME:   %[[ARG_0:.*]]: !llvm.ptr)
+
+func.func @omp_map_info_nested_derived_type_explicit_member_conversion(%arg0 : !fir.ref<!fir.type<_QFTtop_layer{array_i:!fir.array<10xi32>,nested:!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>,k:i32}>>) {
+    // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ARG_0]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFTtop_layer", (array<10 x i32>, struct<"_QFTbottom_layer", (array<10 x f32>, f64)>, i32)>
+    %0 = fir.field_index nested, !fir.type<_QFTtop_layer{array_i:!fir.array<10xi32>,nested:!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>,k:i32}>
+    %1 = fir.coordinate_of %arg0, %0 : (!fir.ref<!fir.type<_QFTtop_layer{array_i:!fir.array<10xi32>,nested:!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>,k:i32}>>, !fir.field) -> !fir.ref<!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>>
+    // CHECK: %[[GEP_2:.*]] = llvm.getelementptr %[[GEP]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFTbottom_layer", (array<10 x f32>, f64)>
+    %2 = fir.field_index i2, !fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>
+    %3 = fir.coordinate_of %1, %2 : (!fir.ref<!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>>, !fir.field) -> !fir.ref<f64>
+    // CHECK: %[[MAP_MEMBER_1:.*]] = omp.map.info var_ptr(%[[GEP_2]] : !llvm.ptr, f64) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr
+    %4 = omp.map.info var_ptr(%3 : !fir.ref<f64>, f64) map_clauses(tofrom) capture(ByRef) -> !fir.ref<f64>
+    // CHECK: %[[GEP_3:.*]] = llvm.getelementptr %[[ARG_0]][0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFTtop_layer", (array<10 x i32>, struct<"_QFTbottom_layer", (array<10 x f32>, f64)>, i32)>
+    %5 = fir.field_index k, !fir.type<_QFTtop_layer{array_i:!fir.array<10xi32>,nested:!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>,k:i32}>
+    %6 = fir.coordinate_of %arg0, %5 : (!fir.ref<!fir.type<_QFTtop_layer{array_i:!fir.array<10xi32>,nested:!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>,k:i32}>>, !fir.field) -> !fir.ref<i32>
+    // CHECK: %[[MAP_MEMBER_2:.*]] = omp.map.info var_ptr(%[[GEP_3]] : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr
+    %7 = omp.map.info var_ptr(%6 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32>
+    // CHECK: %[[PARENT_MAP:.*]] = omp.map.info var_ptr(%[[ARG_0]] : !llvm.ptr, !llvm.struct<"_QFTtop_layer", (array<10 x i32>, struct<"_QFTbottom_layer", (array<10 x f32>, f64)>, i32)>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_MEMBER_1]], %[[MAP_MEMBER_2]] : [1,1], [2,-1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {partial_map = true}
+    %9 = omp.map.info var_ptr(%arg0 : !fir.ref<!fir.type<_QFTtop_layer{array_i:!fir.array<10xi32>,nested:!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>,k:i32}>>, !fir.type<_QFTtop_layer{array_i:!fir.array<10xi32>,nested:!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>,k:i32}>) map_clauses(tofrom) capture(ByRef) members(%4, %7 : [1,1], [2,-1] : !fir.ref<f64>, !fir.ref<i32>) -> !fir.ref<!fir.type<_QFTtop_layer{array_i:!fir.array<10xi32>,nested:!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>,k:i32}>> {partial_map = true}
+    // CHECK: omp.target map_entries(%[[MAP_MEMBER_1]] -> %{{.*}}, %[[MAP_MEMBER_2]] -> %{{.*}}, %[[PARENT_MAP]] -> %{{.*}} : !llvm.ptr, !llvm.ptr, !llvm.ptr) {
+    // CHECK: ^bb0(%{{.*}}: !llvm.ptr, %{{.*}}: !llvm.ptr, %{{.*}}: !llvm.ptr):
+    omp.target map_entries(%4 -> %arg1, %7 -> %arg2, %9 -> %arg3 : !fir.ref<f64>, !fir.ref<i32>, !fir.ref<!fir.type<_QFTtop_layer{array_i:!fir.array<10xi32>,nested:!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>,k:i32}>>) {
+     ^bb0(%arg1: !fir.ref<i32>, %arg2: !fir.ref<f64>, %arg3: !fir.ref<!fir.type<_QFTtop_layer{array_i:!fir.array<10xi32>,nested:!fir.type<_QFTbottom_layer{array_i2:!fir.array<10xf32>,i2:f64}>,k:i32}>>):
+      omp.terminator
+    }
+  return
+}
+
+// -----
diff --git a/flang/test/Integration/OpenMP/map-types-and-sizes.f90 b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
index 283ac227f343..f3a20690f05a 100644
--- a/flang/test/Integration/OpenMP/map-types-and-sizes.f90
+++ b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
@@ -69,6 +69,147 @@ subroutine mapType_allocatable_explicit
   !$omp end target
   deallocate(a)
 end subroutine mapType_allocatable_explicit
+ 
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [1 x i64] [i64 48]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [1 x i64] [i64 547]
+subroutine mapType_derived_implicit
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    integer(4) :: int
+  end type scalar_and_array
+  type(scalar_and_array) :: scalar_arr 
+  
+  !$omp target
+     scalar_arr%int = 1
+  !$omp end target
+end subroutine mapType_derived_implicit
+
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [1 x i64] [i64 48]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [1 x i64] [i64 35]
+subroutine mapType_derived_explicit
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    integer(4) :: int
+  end type scalar_and_array
+  type(scalar_and_array) :: scalar_arr 
+  
+  !$omp target map(tofrom: scalar_arr)
+     scalar_arr%int = 1
+  !$omp end target
+end subroutine mapType_derived_explicit
+
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [1 x i64] [i64 40]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [1 x i64] [i64 35]
+subroutine mapType_derived_explicit_single_member
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    integer(4) :: int
+  end type scalar_and_array
+  type(scalar_and_array) :: scalar_arr 
+  
+  !$omp target map(tofrom: scalar_arr%array)
+     scalar_arr%array(1) = 1
+  !$omp end target
+end subroutine mapType_derived_explicit_single_member
+
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [3 x i64] [i64 0, i64 4, i64 4]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [3 x i64] [i64 32, i64 281474976710659, i64 281474976710659]
+subroutine mapType_derived_explicit_multiple_members
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    integer(4) :: int
+  end type scalar_and_array
+  type(scalar_and_array) :: scalar_arr 
+  
+  !$omp target map(tofrom: scalar_arr%int, scalar_arr%real)
+     scalar_arr%int = 1
+  !$omp end target
+end subroutine mapType_derived_explicit_multiple_members
+
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [1 x i64] [i64 16]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [1 x i64] [i64 35]
+subroutine mapType_derived_explicit_member_with_bounds
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    integer(4) :: int
+  end type scalar_and_array
+  type(scalar_and_array) :: scalar_arr 
+  
+  !$omp target map(tofrom: scalar_arr%array(2:5))
+     scalar_arr%array(3) = 3
+  !$omp end target
+end subroutine mapType_derived_explicit_member_with_bounds
+
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [1 x i64] [i64 4]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [1 x i64] [i64 35]
+subroutine mapType_derived_explicit_nested_single_member
+  type :: nested
+    integer(4) :: int
+    real(4) :: real
+    integer(4) :: array(10)
+  end type nested
+
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    type(nested) :: nest
+    integer(4) :: int
+  end type scalar_and_array
+  type(scalar_and_array) :: scalar_arr 
+  
+  !$omp target map(tofrom: scalar_arr%nest%real)
+    scalar_arr%nest%real = 1
+  !$omp end target
+end subroutine mapType_derived_explicit_nested_single_member
+
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [3 x i64] [i64 0, i64 4, i64 4]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [3 x i64] [i64 32, i64 281474976710659, i64 281474976710659]
+subroutine mapType_derived_explicit_multiple_nested_members
+  type :: nested
+    integer(4) :: int
+    real(4) :: real
+    integer(4) :: array(10)
+  end type nested
+
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    type(nested) :: nest
+    integer(4) :: int
+  end type scalar_and_array
+  type(scalar_and_array) :: scalar_arr 
+  
+!$omp target map(tofrom: scalar_arr%nest%int, scalar_arr%nest%real)
+  scalar_arr%nest%int = 1
+!$omp end target
+end subroutine mapType_derived_explicit_multiple_nested_members
+
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [1 x i64] [i64 16]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [1 x i64] [i64 35]
+subroutine mapType_derived_explicit_nested_member_with_bounds
+  type :: nested
+    integer(4) :: int
+    real(4) :: real
+    integer(4) :: array(10)
+  end type nested
+
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    type(nested) :: nest
+    integer(4) :: int
+  end type scalar_and_array
+  type(scalar_and_array) :: scalar_arr 
+  
+!$omp target map(tofrom: scalar_arr%nest%array(2:5))
+    scalar_arr%nest%array(3) = 3
+!$omp end target
+end subroutine mapType_derived_explicit_nested_member_with_bounds
 
 !CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [2 x i64] [i64 8, i64 4]
 !CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [2 x i64] [i64 544, i64 800]
@@ -100,7 +241,6 @@ end subroutine mapType_char
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [3 x i64], ptr %.offload_sizes, i32 0, i32 0
 !CHECK: store i64 %[[DIV]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 
-
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_allocatable_explicit_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8 }, i64 1, align 8
 !CHECK: %[[ALLOCA_GEP:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8 }, ptr %[[ALLOCA]], i32 1
@@ -110,3 +250,99 @@ end subroutine mapType_char
 !CHECK: %[[DIV:.*]] = sdiv exact i64 %[[SIZE_DIFF]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [3 x i64], ptr %.offload_sizes, i32 0, i32 0
 !CHECK: store i64 %[[DIV]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
+
+!CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_implicit_{{.*}}
+!CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_implicitTscalar_and_array, i64 1, align 8
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+!CHECK: store ptr %[[ALLOCA]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+
+!CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_{{.*}}
+!CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicitTscalar_and_array, i64 1, align 8
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+!CHECK: store ptr %[[ALLOCA]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+
+!CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_single_member_{{.*}}
+!CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_single_memberTscalar_and_array, i64 1, align 8
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 1
+!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS]], i64 0, i64 0
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+!CHECK: store ptr %[[ARR_OFF]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+
+!CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_multiple_members_{{.*}}
+!CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_multiple_membersTscalar_and_array, i64 1, align 8
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 0
+!CHECK: %[[ARR_END_OFF:.*]] = getelementptr i32, ptr %[[MEMBER_ACCESS_1]], i64 1
+!CHECK: %[[ARR_END:.*]] = ptrtoint ptr %[[ARR_END_OFF]] to i64
+!CHECK: %[[FIRST_MEMBER:.*]] = ptrtoint ptr %[[MEMBER_ACCESS_2]] to i64
+!CHECK: %[[SIZE_DIFF:.*]] = sub i64 %[[ARR_END]], %[[FIRST_MEMBER]]
+!CHECK: %[[SIZE:.*]] = sdiv exact i64 %[[SIZE_DIFF]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+!CHECK: store ptr %[[MEMBER_ACCESS_2]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+!CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [3 x i64], ptr %.offload_sizes, i32 0, i32 0
+!CHECK: store i64 %[[SIZE]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
+!CHECK: %[[BASE_PTR_ARR_2:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_baseptrs, i32 0, i32 1
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR_2]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR_2:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_ptrs, i32 0, i32 1
+!CHECK: store ptr %[[MEMBER_ACCESS_1]], ptr %[[OFFLOAD_PTR_ARR_2]], align 8
+!CHECK: %[[BASE_PTR_ARR_3:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_baseptrs, i32 0, i32 2
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR_3]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR_3:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_ptrs, i32 0, i32 2
+!CHECK: store ptr %[[MEMBER_ACCESS_2]], ptr %[[OFFLOAD_PTR_ARR_3]], align 8
+
+!CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_member_with_bounds_{{.*}}
+!CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array, i64 1, align 8
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 1
+!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS]], i64 0, i64 1
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+!CHECK: store ptr %[[ARR_OFF]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+
+!CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_single_member_{{.*}}
+!CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, i64 1, align 8
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+!CHECK: store ptr %[[MEMBER_ACCESS]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+
+!CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_multiple_nested_members_{{.*}}
+!CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, i64 1, align 8
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 0
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: %[[ARR_END_OFF:.*]] = getelementptr float, ptr %[[MEMBER_ACCESS_2]], i64 1
+!CHECK: %[[ARR_END:.*]] = ptrtoint ptr %[[ARR_END_OFF]] to i64
+!CHECK: %[[FIRST_MEMBER:.*]] = ptrtoint ptr %[[MEMBER_ACCESS_1]] to i64
+!CHECK: %[[SIZE_DIFF:.*]] = sub i64 %[[ARR_END]], %[[FIRST_MEMBER]]
+!CHECK: %[[SIZE:.*]] = sdiv exact i64 %[[SIZE_DIFF]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64)
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+!CHECK: store ptr %[[MEMBER_ACCESS_1]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+!CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [3 x i64], ptr %.offload_sizes, i32 0, i32 0
+!CHECK: store i64 %[[SIZE]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
+!CHECK: %[[BASE_PTR_ARR_2:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_baseptrs, i32 0, i32 1
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR_2]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR_2:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_ptrs, i32 0, i32 1
+!CHECK: store ptr %[[MEMBER_ACCESS_1]], ptr %[[OFFLOAD_PTR_ARR_2]], align 8
+!CHECK: %[[BASE_PTR_ARR_3:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_baseptrs, i32 0, i32 2
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR_3]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR_3:.*]] = getelementptr inbounds [3 x ptr], ptr %.offload_ptrs, i32 0, i32 2
+!CHECK: store ptr %[[MEMBER_ACCESS_2]], ptr %[[OFFLOAD_PTR_ARR_3]], align 8
+
+!CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_member_with_bounds_{{.*}}
+!CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, i64 1, align 8
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 2
+!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS]], i64 0, i64 1
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+!CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
+!CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+!CHECK: store ptr %[[ARR_OFF]], ptr %[[OFFLOAD_PTR_ARR]], align 8
diff --git a/flang/test/Lower/OpenMP/allocatable-array-bounds.f90 b/flang/test/Lower/OpenMP/allocatable-array-bounds.f90
index 14a50692bc60..87ca400e82e2 100644
--- a/flang/test/Lower/OpenMP/allocatable-array-bounds.f90
+++ b/flang/test/Lower/OpenMP/allocatable-array-bounds.f90
@@ -24,7 +24,7 @@
 !HOST: %[[BOUNDS_1:.*]] = omp.map.bounds lower_bound(%[[LB_1]] : index) upper_bound(%[[UB_1]] : index) extent(%[[BOX_3]]#1 : index) stride(%[[BOX_2]]#2 : index) start_idx(%[[BOX_1]]#0 : index) {stride_in_bytes = true}
 !HOST: %[[VAR_PTR_PTR:.*]] = fir.box_offset %[[DECLARE_1]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
 !HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[DECLARE_1]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.array<?xi32>) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) bounds(%[[BOUNDS_1]]) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>> {name = ""}    
-!HOST: %[[MAP_INFO_1:.*]] = omp.map.info var_ptr(%[[DECLARE_1]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {name = "sp_read(2:5)"}
+!HOST: %[[MAP_INFO_1:.*]] = omp.map.info var_ptr(%[[DECLARE_1]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {name = "sp_read(2:5)"}
 
 !HOST: %[[LOAD_3:.*]] = fir.load %[[DECLARE_2]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 !HOST: %[[LOAD_4:.*]] = fir.load %[[DECLARE_2]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
@@ -42,7 +42,7 @@
 !HOST: %[[BOUNDS_2:.*]] = omp.map.bounds lower_bound(%[[LB_2]] : index) upper_bound(%[[UB_2]] : index) extent(%[[BOX_5]]#1 : index) stride(%[[BOX_4]]#2 : index) start_idx(%[[BOX_3]]#0 : index) {stride_in_bytes = true}
 !HOST: %[[VAR_PTR_PTR:.*]] = fir.box_offset %[[DECLARE_2]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
 !HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[DECLARE_2]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.array<?xi32>) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) bounds(%[[BOUNDS_2]]) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>> {name = ""}    
-!HOST: %[[MAP_INFO_2:.*]] = omp.map.info var_ptr(%[[DECLARE_2]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {name = "sp_write(2:5)"}
+!HOST: %[[MAP_INFO_2:.*]] = omp.map.info var_ptr(%[[DECLARE_2]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {name = "sp_write(2:5)"}
 
 subroutine read_write_section()
     integer, allocatable :: sp_read(:)
@@ -81,7 +81,7 @@ module assumed_allocatable_array_routines
 !HOST: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%[[LB]] : index) upper_bound(%[[UB]] : index) extent(%[[BOX_3]]#1 : index) stride(%[[BOX_2]]#2 : index) start_idx(%[[BOX_1]]#0 : index) {stride_in_bytes = true}
 !HOST: %[[VAR_PTR_PTR:.*]] = fir.box_offset %[[DECLARE]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
 !HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.array<?xi32>) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) bounds(%[[BOUNDS]]) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>> {name = ""}    
-!HOST: %[[MAP_INFO:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {name = "arr_read_write(2:5)"}    
+!HOST: %[[MAP_INFO:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.box<!fir.heap<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {name = "arr_read_write(2:5)"}    
 subroutine assumed_shape_array(arr_read_write)
     integer, allocatable, intent(inout) :: arr_read_write(:)
 
diff --git a/flang/test/Lower/OpenMP/allocatable-map.f90 b/flang/test/Lower/OpenMP/allocatable-map.f90
index 396d45373b84..a9f576a6f099 100644
--- a/flang/test/Lower/OpenMP/allocatable-map.f90
+++ b/flang/test/Lower/OpenMP/allocatable-map.f90
@@ -2,12 +2,12 @@
 
 !HLFIRDIALECT: %[[POINTER:.*]]:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFpointer_routineEpoint"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
 !HLFIRDIALECT: %[[BOX_OFF:.*]] = fir.box_offset %[[POINTER]]#1 base_addr : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-!HLFIRDIALECT: %[[POINTER_MAP_MEMBER:.*]] = omp.map.info var_ptr(%[[POINTER]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, i32) var_ptr_ptr(%[[BOX_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>)  map_clauses(implicit, tofrom) capture(ByRef) -> !fir.llvm_ptr<!fir.ref<i32>> {name = ""}
-!HLFIRDIALECT: %[[POINTER_MAP:.*]] = omp.map.info var_ptr(%[[POINTER]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(implicit, tofrom) capture(ByRef) members(%[[POINTER_MAP_MEMBER]] : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.ref<!fir.box<!fir.ptr<i32>>> {name = "point"}
-!HLFIRDIALECT: omp.target map_entries({{.*}}, %[[POINTER_MAP_MEMBER]] -> {{.*}}, %[[POINTER_MAP]] -> {{.*}} : {{.*}}, !fir.llvm_ptr<!fir.ref<i32>>, !fir.ref<!fir.box<!fir.ptr<i32>>>) {
+!HLFIRDIALECT: %[[POINTER_MAP_MEMBER:.*]] = omp.map.info var_ptr(%[[POINTER]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, i32) var_ptr_ptr(%[[BOX_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>)  map_clauses(tofrom) capture(ByRef) -> !fir.llvm_ptr<!fir.ref<i32>> {name = ""}
+!HLFIRDIALECT: %[[POINTER_MAP:.*]] = omp.map.info var_ptr(%[[POINTER]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) members(%[[POINTER_MAP_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.ref<!fir.box<!fir.ptr<i32>>> {name = "point"}
+!HLFIRDIALECT: omp.target map_entries(%[[POINTER_MAP_MEMBER]] -> {{.*}}, %[[POINTER_MAP]] -> {{.*}} : !fir.llvm_ptr<!fir.ref<i32>>, !fir.ref<!fir.box<!fir.ptr<i32>>>) {
 subroutine pointer_routine()
     integer, pointer :: point 
-!$omp target map(tofrom:pointer)
+!$omp target map(tofrom:point)
     point = 1
 !$omp end target
 end subroutine pointer_routine
diff --git a/flang/test/Lower/OpenMP/array-bounds.f90 b/flang/test/Lower/OpenMP/array-bounds.f90
index 8c9d8944bf33..f235d5041ab2 100644
--- a/flang/test/Lower/OpenMP/array-bounds.f90
+++ b/flang/test/Lower/OpenMP/array-bounds.f90
@@ -52,7 +52,7 @@ module assumed_array_routines
 !HOST: %[[BOUNDS:.*]] = omp.map.bounds   lower_bound(%[[C3]] : index) upper_bound(%[[C4]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) start_idx(%[[C0]] : index) {stride_in_bytes = true}
 !HOST: %[[VAR_PTR_PTR:.*]] = fir.box_offset %0 base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
 !HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.array<?xi32>) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) bounds(%[[BOUNDS]]) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>> {name = ""}
-!HOST: %[[MAP:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.array<?xi32>> {name = "arr_read_write(2:5)"}
+!HOST: %[[MAP:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.array<?xi32>> {name = "arr_read_write(2:5)"}
 !HOST: omp.target   map_entries(%[[MAP_INFO_MEMBER]] -> %{{.*}}, %[[MAP]] -> %{{.*}}, {{.*}} -> {{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, !fir.ref<!fir.array<?xi32>>, !fir.ref<i32>) {
     subroutine assumed_shape_array(arr_read_write)
             integer, intent(inout) :: arr_read_write(:)
@@ -77,7 +77,7 @@ module assumed_array_routines
 !HOST: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%c1{{.*}} : index) upper_bound(%c4{{.*}} : index) extent(%[[EXT]] : index) stride(%[[DIMS0]]#2 : index) start_idx(%c1{{.*}} : index) {stride_in_bytes = true}
 !HOST: %[[VAR_PTR_PTR:.*]] = fir.box_offset %[[INTERMEDIATE_ALLOCA]] base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
 !HOST: %[[MAP_INFO_MEMBER:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.array<?xi32>) var_ptr_ptr(%[[VAR_PTR_PTR]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) map_clauses(tofrom) capture(ByRef) bounds(%[[BOUNDS]]) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>> {name = ""}
-!HOST: %[[MAP:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.array<?xi32>> {name = "arr_read_write(2:5)"}
+!HOST: %[[MAP:.*]] = omp.map.info var_ptr(%[[INTERMEDIATE_ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_INFO_MEMBER]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.array<?xi32>> {name = "arr_read_write(2:5)"}
 !HOST: omp.target map_entries(%[[MAP_INFO_MEMBER]] -> %{{.*}}, %[[MAP]] -> %{{.*}}, {{.*}} -> {{.*}} : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, !fir.ref<!fir.array<?xi32>>, !fir.ref<i32>) {
     subroutine assumed_size_array(arr_read_write)
         integer, intent(inout) :: arr_read_write(*)
diff --git a/flang/test/Lower/OpenMP/derived-type-map.f90 b/flang/test/Lower/OpenMP/derived-type-map.f90
new file mode 100644
index 000000000000..6121b450f062
--- /dev/null
+++ b/flang/test/Lower/OpenMP/derived-type-map.f90
@@ -0,0 +1,238 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+
+!CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_implicitEscalar_arr"}
+!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_implicitEscalar_arr"} : (!fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) -> (!fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>)
+!CHECK: %[[MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(implicit, tofrom) capture(ByRef) -> !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr"}
+!CHECK:     omp.target map_entries(%[[MAP]] -> %[[ARG0:.*]] : !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) {
+!CHECK:         ^bb0(%[[ARG0]]: !fir.ref<!fir.type<_QFmaptype_derived_implicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>):
+subroutine mapType_derived_implicit
+    type :: scalar_and_array
+      real(4) :: real
+      integer(4) :: array(10)
+      integer(4) :: int
+    end type scalar_and_array
+    type(scalar_and_array) :: scalar_arr 
+    
+    !$omp target
+       scalar_arr%int = 1
+    !$omp end target
+end subroutine mapType_derived_implicit
+
+!CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_explicitEscalar_arr"}
+!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_explicitEscalar_arr"} : (!fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) -> (!fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>)
+!CHECK: %[[MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#0 : !fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(tofrom) capture(ByRef) -> !fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr"}
+!CHECK:  omp.target map_entries(%[[MAP]] -> %[[ARG0:.*]] : !fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) {
+!CHECK:    ^bb0(%[[ARG0]]: !fir.ref<!fir.type<_QFmaptype_derived_explicitTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>):
+subroutine mapType_derived_explicit
+    type :: scalar_and_array
+      real(4) :: real
+      integer(4) :: array(10)
+      integer(4) :: int
+    end type scalar_and_array
+    type(scalar_and_array) :: scalar_arr 
+    
+    !$omp target map(tofrom: scalar_arr)
+       scalar_arr%int = 1
+    !$omp end target
+end subroutine mapType_derived_explicit
+
+!CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.type<_QFmaptype_derived_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_explicit_single_memberEscalar_arr"}
+!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_explicit_single_memberEscalar_arr"} : (!fir.ref<!fir.type<_QFmaptype_derived_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) -> (!fir.ref<!fir.type<_QFmaptype_derived_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.ref<!fir.type<_QFmaptype_derived_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>)
+!CHECK: %[[MEMBER:.*]] = hlfir.designate %[[DECLARE]]#0{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFmaptype_derived_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xi32>>
+!CHECK: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%{{.*}} : index) upper_bound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) start_idx(%{{.*}} : index)
+!CHECK: %[[MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[MEMBER]] : !fir.ref<!fir.array<10xi32>>, !fir.array<10xi32>) map_clauses(tofrom) capture(ByRef) bounds(%[[BOUNDS]]) -> !fir.ref<!fir.array<10xi32>> {name = "scalar_arr%array"}
+!CHECK: %[[PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<_QFmaptype_derived_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(tofrom) capture(ByRef) members(%[[MEMBER_MAP]] : [1] : !fir.ref<!fir.array<10xi32>>) -> !fir.ref<!fir.type<_QFmaptype_derived_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr", partial_map = true}
+!CHECK: omp.target map_entries(%[[MEMBER_MAP]] -> %[[ARG0:.*]], %[[PARENT_MAP]] -> %[[ARG1:.*]] : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.type<_QFmaptype_derived_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) {
+!CHECK:  ^bb0(%[[ARG0]]: !fir.ref<!fir.array<10xi32>>, %[[ARG1]]: !fir.ref<!fir.type<_QFmaptype_derived_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>):
+subroutine mapType_derived_explicit_single_member
+    type :: scalar_and_array
+      real(4) :: real
+      integer(4) :: array(10)
+      integer(4) :: int
+    end type scalar_and_array
+    type(scalar_and_array) :: scalar_arr 
+    
+    !$omp target map(tofrom: scalar_arr%array)
+       scalar_arr%array(1) = 1
+    !$omp end target
+end subroutine mapType_derived_explicit_single_member
+
+!CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.type<_QFmaptype_derived_explicit_multiple_membersTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_explicit_multiple_membersEscalar_arr"}
+!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_explicit_multiple_membersEscalar_arr"} : (!fir.ref<!fir.type<_QFmaptype_derived_explicit_multiple_membersTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) -> (!fir.ref<!fir.type<_QFmaptype_derived_explicit_multiple_membersTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.ref<!fir.type<_QFmaptype_derived_explicit_multiple_membersTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>)
+!CHECK: %[[MEMBER1:.*]] = hlfir.designate %[[DECLARE]]#0{"int"}   : (!fir.ref<!fir.type<_QFmaptype_derived_explicit_multiple_membersTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) -> !fir.ref<i32>
+!CHECK: %[[MEMBER_MAP_1:.*]] = omp.map.info var_ptr(%[[MEMBER1]] : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "scalar_arr%int"}
+!CHECK: %[[MEMBER2:.*]] = hlfir.designate %[[DECLARE]]#0{"real"}   : (!fir.ref<!fir.type<_QFmaptype_derived_explicit_multiple_membersTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) -> !fir.ref<f32>
+!CHECK: %[[MEMBER_MAP_2:.*]] = omp.map.info var_ptr(%[[MEMBER2]] : !fir.ref<f32>, f32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<f32> {name = "scalar_arr%real"}
+!CHECK: %[[PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<_QFmaptype_derived_explicit_multiple_membersTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_explicit_multiple_membersTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(tofrom) capture(ByRef) members(%[[MEMBER_MAP_1]], %[[MEMBER_MAP_2]] : [2], [0] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_explicit_multiple_membersTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr", partial_map = true}
+!CHECK: omp.target map_entries(%[[MEMBER_MAP_1]] -> %[[ARG0:.*]], %[[MEMBER_MAP_2]] -> %[[ARG1:.*]], %[[PARENT_MAP]] -> %[[ARG2:.*]] : !fir.ref<i32>, !fir.ref<f32>, !fir.ref<!fir.type<_QFmaptype_derived_explicit_multiple_membersTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) {
+!CHECK:  ^bb0(%[[ARG0]]: !fir.ref<i32>, %[[ARG1]]: !fir.ref<f32>, %[[ARG2]]: !fir.ref<!fir.type<_QFmaptype_derived_explicit_multiple_membersTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>):
+subroutine mapType_derived_explicit_multiple_members
+    type :: scalar_and_array
+      real(4) :: real
+      integer(4) :: array(10)
+      integer(4) :: int
+    end type scalar_and_array
+    type(scalar_and_array) :: scalar_arr 
+    
+    !$omp target map(tofrom: scalar_arr%int, scalar_arr%real)
+       scalar_arr%int = 1
+    !$omp end target
+end subroutine mapType_derived_explicit_multiple_members
+  
+!CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_explicit_member_with_boundsEscalar_arr"}
+!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_explicit_member_with_boundsEscalar_arr"} : (!fir.ref<!fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) -> (!fir.ref<!fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.ref<!fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>)
+!CHECK: %[[MEMBER:.*]] = hlfir.designate %[[DECLARE]]#0{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xi32>>
+!CHECK: %{{.*}} = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 1 : index
+!CHECK: %[[UB:.*]] = arith.constant 4 : index
+!CHECK: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%[[LB]] : index) upper_bound(%[[UB]] : index) extent(%{{.*}} : index) stride(%{{.*}} : index) start_idx(%{{.*}} : index)
+!CHECK: %[[MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[MEMBER]] : !fir.ref<!fir.array<10xi32>>, !fir.array<10xi32>) map_clauses(tofrom) capture(ByRef) bounds(%20) -> !fir.ref<!fir.array<10xi32>> {name = "scalar_arr%array(2:5)"}
+!CHECK: %[[PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : !fir.ref<!fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>, !fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>) map_clauses(tofrom) capture(ByRef) members(%[[MEMBER_MAP]] : [1] : !fir.ref<!fir.array<10xi32>>) -> !fir.ref<!fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>> {name = "scalar_arr", partial_map = true}
+!CHECK: omp.target map_entries(%[[MEMBER_MAP]] -> %[[ARG0:.*]], %[[PARENT_MAP]] -> %[[ARG1:.*]] : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>) {
+!CHECK: ^bb0(%[[ARG0]]: !fir.ref<!fir.array<10xi32>>, %[[ARG1]]: !fir.ref<!fir.type<_QFmaptype_derived_explicit_member_with_boundsTscalar_and_array{real:f32,array:!fir.array<10xi32>,int:i32}>>):
+subroutine mapType_derived_explicit_member_with_bounds
+    type :: scalar_and_array
+      real(4) :: real
+      integer(4) :: array(10)
+      integer(4) :: int
+    end type scalar_and_array
+    type(scalar_and_array) :: scalar_arr 
+    
+    !$omp target map(tofrom: scalar_arr%array(2:5))
+       scalar_arr%array(3) = 3
+    !$omp end target
+end subroutine mapType_derived_explicit_member_with_bounds
+
+!CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.type<_QFmaptype_derived_nested_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,nest:!fir.type<_QFmaptype_derived_nested_explicit_single_memberTnested{int:i32,real:f32,array:!fir.array<10xi32>}>,int:i32}> {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_nested_explicit_single_memberEscalar_arr"}
+!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_nested_explicit_single_memberEscalar_arr"} : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,nest:!fir.type<_QFmaptype_derived_nested_explicit_single_memberTnested{int:i32,real:f32,array:!fir.array<10xi32>}>,int:i32}>>) -> {{.*}}
+!CHECK: %[[NEST:.*]] = hlfir.designate %[[DECLARE]]#0{"nest"}   : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_single_memberTscalar_and_array{real:f32,array:!fir.array<10xi32>,nest:!fir.type<_QFmaptype_derived_nested_explicit_single_memberTnested{int:i32,real:f32,array:!fir.array<10xi32>}>,int:i32}>>) -> {{.*}}
+!CHECK: %[[NEST_MEMBER:.*]] = hlfir.designate %[[NEST]]{"array"}   shape %{{.*}} : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_single_memberTnested{int:i32,real:f32,array:!fir.array<10xi32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xi32>>
+!CHECK: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%{{.*}} : index) upper_bound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) start_idx(%{{.*}} : index)
+!CHECK: %[[MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[NEST_MEMBER]] : !fir.ref<!fir.array<10xi32>>, !fir.array<10xi32>) map_clauses(tofrom) capture(ByRef) bounds(%[[BOUNDS]]) -> !fir.ref<!fir.array<10xi32>> {name = "scalar_arr%nest%array"}
+!CHECK: %[[PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : {{.*}}) map_clauses(tofrom) capture(ByRef) members(%35 : [2,2] : !fir.ref<!fir.array<10xi32>>) -> {{.*}} {name = "scalar_arr", partial_map = true}
+!CHECK: omp.target map_entries(%[[MEMBER_MAP]] -> %[[ARG0:.*]], %[[PARENT_MAP]] -> %[[ARG1:.*]] : {{.*}}, {{.*}}) {
+!CHECK:  ^bb0(%[[ARG0]]: {{.*}}, %[[ARG1]]: {{.*}}):
+subroutine mapType_derived_nested_explicit_single_member
+  type :: nested
+    integer(4) :: int
+    real(4) :: real
+    integer(4) :: array(10)
+  end type nested
+
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    type(nested) :: nest
+    integer(4) :: int
+  end type scalar_and_array
+  
+  type(scalar_and_array) :: scalar_arr 
+
+  !$omp target map(tofrom: scalar_arr%nest%array)
+    scalar_arr%nest%array(1) = 1
+  !$omp end target
+end subroutine mapType_derived_nested_explicit_single_member
+
+!CHECK: %[[ALLOCA:.*]] = fir.alloca {{.*}} {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_nested_explicit_multiple_membersEscalar_arr"}
+!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_nested_explicit_multiple_membersEscalar_arr"} : ({{.*}}) -> {{.*}}
+!CHECK: %[[NEST:.*]] = hlfir.designate %[[DECLARE]]#0{"nest"}   : ({{.*}}) -> {{.*}}
+!CHECK: %[[NEST_MEMBER1:.*]] = hlfir.designate %[[NEST]]{"int"}   : ({{.*}}) -> !fir.ref<i32>
+!CHECK: %[[MEMBER_MAP_1:.*]] = omp.map.info var_ptr(%[[NEST_MEMBER1]] : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "scalar_arr%nest%int"}
+!CHECK: %[[NEST:.*]] = hlfir.designate %[[DECLARE]]#0{"nest"}   : ({{.*}}) -> {{.*}}
+!CHECK: %[[NEST_MEMBER2:.*]] = hlfir.designate %[[NEST]]{"real"}   : ({{.*}}) -> !fir.ref<f32>
+!CHECK: %[[MEMBER_MAP_2:.*]] = omp.map.info var_ptr(%[[NEST_MEMBER2]] : !fir.ref<f32>, f32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<f32> {name = "scalar_arr%nest%real"}
+!CHECK: %[[PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : {{.*}}, {{.*}}) map_clauses(tofrom) capture(ByRef) members(%[[MEMBER_MAP_1]], %[[MEMBER_MAP_2]] : [2,0], [2,1] : !fir.ref<i32>, !fir.ref<f32>) -> {{.*}} {name = "scalar_arr", partial_map = true}
+!CHECK: omp.target map_entries(%[[MEMBER_MAP_1]] -> %[[ARG0:.*]], %[[MEMBER_MAP_2]] -> %[[ARG1:.*]], %[[PARENT_MAP]] -> %[[ARG2:.*]] : !fir.ref<i32>, !fir.ref<f32>, {{.*}}) {
+!CHECK: ^bb0(%[[ARG0]]: !fir.ref<i32>, %[[ARG1]]: !fir.ref<f32>, %[[ARG2]]: {{.*}}):
+subroutine mapType_derived_nested_explicit_multiple_members
+  type :: nested
+    integer(4) :: int
+    real(4) :: real
+    integer(4) :: array(10)
+  end type nested
+
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    type(nested) :: nest
+    integer(4) :: int
+  end type scalar_and_array
+
+  type(scalar_and_array) :: scalar_arr 
+
+  !$omp target map(tofrom: scalar_arr%nest%int, scalar_arr%nest%real)
+    scalar_arr%nest%int = 1
+  !$omp end target
+end subroutine mapType_derived_nested_explicit_multiple_members
+
+!CHECK: %[[ALLOCA:.*]] = fir.alloca {{.*}} {bindc_name = "scalar_arr", uniq_name = "_QFmaptype_derived_nested_explicit_member_with_boundsEscalar_arr"}
+!CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFmaptype_derived_nested_explicit_member_with_boundsEscalar_arr"} : {{.*}} -> {{.*}}
+!CHECK: %[[NEST:.*]] = hlfir.designate %[[DECLARE]]#0{"nest"}   : {{.*}} -> {{.*}}
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[NEST_MEMBER:.*]] = hlfir.designate %[[NEST]]{"array"}   {{.*}} : {{.*}} -> !fir.ref<!fir.array<10xi32>>
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[C1_2:.*]] = arith.constant 1 : index
+!CHECK: %[[C4:.*]] = arith.constant 4 : index
+!CHECK: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%[[C1_2]] : index) upper_bound(%[[C4]] : index) extent(%[[C10]] : index) stride(%[[C1]] : index) start_idx(%[[C1]] : index)
+!CHECK: %[[MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[NEST_MEMBER]] : !fir.ref<!fir.array<10xi32>>, !fir.array<10xi32>) map_clauses(tofrom) capture(ByRef) bounds(%[[BOUNDS]]) -> !fir.ref<!fir.array<10xi32>> {name = "scalar_arr%nest%array(2:5)"}
+!CHECK: %[[PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE]]#1 : {{.*}}, {{.*}}) map_clauses(tofrom) capture(ByRef) members(%[[MEMBER_MAP]] : [2,2] : !fir.ref<!fir.array<10xi32>>) -> {{.*}} {name = "scalar_arr", partial_map = true}
+!CHECK: omp.target map_entries(%[[MEMBER_MAP]] -> %[[ARG0:.*]], %[[PARENT_MAP]] -> %[[ARG1:.*]] : !fir.ref<!fir.array<10xi32>>, {{.*}}) {
+!CHECK: ^bb0(%[[ARG0]]: !fir.ref<!fir.array<10xi32>>, %[[ARG1]]: {{.*}}):
+subroutine mapType_derived_nested_explicit_member_with_bounds
+  type :: nested
+    integer(4) :: int
+    real(4) :: real
+    integer(4) :: array(10)
+  end type nested
+
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    type(nested) :: nest
+    integer(4) :: int
+  end type scalar_and_array
+  
+  type(scalar_and_array) :: scalar_arr 
+  
+  !$omp target map(tofrom: scalar_arr%nest%array(2:5))
+    scalar_arr%nest%array(3) = 3
+  !$omp end target
+end subroutine mapType_derived_nested_explicit_member_with_bounds
+
+!CHECK: %[[ALLOCA_1:.*]] = fir.alloca {{.*}} {bindc_name = "scalar_arr1", uniq_name = "_QFmaptype_multilpe_derived_nested_explicit_memberEscalar_arr1"}
+!CHECK: %[[DECLARE_1:.*]]:2 = hlfir.declare %[[ALLOCA_1]] {uniq_name = "_QFmaptype_multilpe_derived_nested_explicit_memberEscalar_arr1"} : {{.*}} -> {{.*}}
+!CHECK: %[[ALLOCA_2:.*]] = fir.alloca {{.*}} {bindc_name = "scalar_arr2", uniq_name = "_QFmaptype_multilpe_derived_nested_explicit_memberEscalar_arr2"}
+!CHECK: %[[DECLARE_2:.*]]:2 = hlfir.declare %[[ALLOCA_2]] {uniq_name = "_QFmaptype_multilpe_derived_nested_explicit_memberEscalar_arr2"} : {{.*}} -> {{.*}}
+!CHECK: %[[PARENT_1:.*]] = hlfir.designate %[[DECLARE_1]]#0{"nest"}   : {{.*}} -> {{.*}}
+!CHECK: %[[MEMBER_1:.*]] = hlfir.designate %[[PARENT_1]]{"int"}   : {{.*}} -> !fir.ref<i32>
+!CHECK: %[[MAP_MEMBER_1:.*]] = omp.map.info var_ptr(%[[MEMBER_1]] : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "scalar_arr1%nest%int"}
+!CHECK: %[[PARENT_2:.*]] = hlfir.designate %[[DECLARE_2]]#0{"nest"}   : {{.*}} -> {{.*}}
+!CHECK: %[[MEMBER_2:.*]] = hlfir.designate %[[PARENT_2]]{"int"}   : {{.*}} -> !fir.ref<i32>
+!CHECK: %[[MAP_MEMBER_2:.*]] = omp.map.info var_ptr(%[[MEMBER_2]] : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "scalar_arr2%nest%int"}
+!CHECK: %[[MAP_PARENT_1:.*]] = omp.map.info var_ptr(%[[DECLARE_1]]#1 : {{.*}}) map_clauses(tofrom) capture(ByRef) members(%[[MAP_MEMBER_1]] : [2,0] : !fir.ref<i32>) -> {{.*}} {name = "scalar_arr1", partial_map = true}
+!CHECK: %[[MAP_PARENT_2:.*]] = omp.map.info var_ptr(%[[DECLARE_2]]#1 : {{.*}}) map_clauses(tofrom) capture(ByRef) members(%[[MAP_MEMBER_2]] : [2,0] : !fir.ref<i32>) -> {{.*}} {name = "scalar_arr2", partial_map = true}
+!CHECK: omp.target map_entries(%[[MAP_MEMBER_1]] -> %[[ARG0:.*]], %[[MAP_PARENT_1]] -> %[[ARG1:.*]], %[[MAP_MEMBER_2]] -> %[[ARG2:.*]], %[[MAP_PARENT_2:.*]] -> %[[ARG3:.*]] : !fir.ref<i32>, {{.*}}, !fir.ref<i32>, {{.*}}) {
+!CHECK: ^bb0(%[[ARG0]]: !fir.ref<i32>, %[[ARG1]]: {{.*}}, %[[ARG2]]: !fir.ref<i32>, %[[ARG3]]: {{.*}}):
+subroutine mapType_multilpe_derived_nested_explicit_member
+  type :: nested
+    integer(4) :: int
+    real(4) :: real
+    integer(4) :: array(10)
+  end type nested
+
+  type :: scalar_and_array
+    real(4) :: real
+    integer(4) :: array(10)
+    type(nested) :: nest
+    integer(4) :: int
+  end type scalar_and_array
+  
+  type(scalar_and_array) :: scalar_arr1
+  type(scalar_and_array) :: scalar_arr2
+
+!$omp target map(tofrom:scalar_arr1%nest%int, scalar_arr2%nest%int)
+  scalar_arr1%nest%int = 3
+  scalar_arr2%nest%int = 2
+!$omp end target
+end subroutine mapType_multilpe_derived_nested_explicit_member
diff --git a/flang/test/Lower/OpenMP/map-component-ref.f90 b/flang/test/Lower/OpenMP/map-component-ref.f90
index 435bc8e6bd36..2c582667f38d 100644
--- a/flang/test/Lower/OpenMP/map-component-ref.f90
+++ b/flang/test/Lower/OpenMP/map-component-ref.f90
@@ -5,7 +5,7 @@
 ! CHECK: %[[V1:[0-9]+]]:2 = hlfir.declare %[[V0]] {uniq_name = "_QFfooEa"} : (!fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>) -> (!fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>, !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>)
 ! CHECK: %[[V2:[0-9]+]] = hlfir.designate %[[V1]]#0{"a1"}   : (!fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>) -> !fir.ref<i32>
 ! CHECK: %[[V3:[0-9]+]] = omp.map.info var_ptr(%[[V2]] : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "a%a1"}
-! CHECK: %[[V4:[0-9]+]] = omp.map.info var_ptr(%[[V1]]#1 : !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>, !fir.type<_QFfooTt0{a0:i32,a1:i32}>) map_clauses(implicit, tofrom) capture(ByRef) -> !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>> {name = "a"}
+! CHECK: %[[V4:[0-9]+]] = omp.map.info var_ptr(%[[V1]]#1 : !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>, !fir.type<_QFfooTt0{a0:i32,a1:i32}>) map_clauses(tofrom) capture(ByRef) members(%[[V3]] : [1] : !fir.ref<i32>) -> !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>> {name = "a", partial_map = true}
 ! CHECK: omp.target map_entries(%[[V3]] -> %arg0, %[[V4]] -> %arg1 : !fir.ref<i32>, !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>) {
 ! CHECK: ^bb0(%arg0: !fir.ref<i32>, %arg1: !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>):
 ! CHECK:   %[[V5:[0-9]+]]:2 = hlfir.declare %arg1 {uniq_name = "_QFfooEa"} : (!fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>) -> (!fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>, !fir.ref<!fir.type<_QFfooTt0{a0:i32,a1:i32}>>)
diff --git a/flang/test/Lower/OpenMP/target.f90 b/flang/test/Lower/OpenMP/target.f90
index e3b3799e5e7a..9bb855e44694 100644
--- a/flang/test/Lower/OpenMP/target.f90
+++ b/flang/test/Lower/OpenMP/target.f90
@@ -532,7 +532,7 @@ end subroutine omp_target_device_ptr
    !CHECK: %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "a", uniq_name = "_QFomp_target_device_addrEa"}
    !CHECK: %[[VAL_0_DECL:.*]]:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFomp_target_device_addrEa"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
    !CHECK: %[[MAP_MEMBERS:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, i32) var_ptr_ptr({{.*}} : !fir.llvm_ptr<!fir.ref<i32>>) map_clauses(tofrom) capture(ByRef) -> !fir.llvm_ptr<!fir.ref<i32>> {name = ""}
-   !CHECK: %[[MAP:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_MEMBERS]] : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.ref<!fir.box<!fir.ptr<i32>>> {name = "a"}
+   !CHECK: %[[MAP:.*]] = omp.map.info var_ptr({{.*}} : !fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.box<!fir.ptr<i32>>) map_clauses(tofrom) capture(ByRef) members(%[[MAP_MEMBERS]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.ref<!fir.box<!fir.ptr<i32>>> {name = "a"}
    !CHECK: omp.target_data map_entries(%[[MAP_MEMBERS]], %[[MAP]] : {{.*}}) use_device_addr(%[[VAL_0_DECL]]#1 : !fir.ref<!fir.box<!fir.ptr<i32>>>) {
    !$omp target data map(tofrom: a) use_device_addr(a)
    !CHECK: ^bb0(%[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>>):
diff --git a/flang/test/Transforms/omp-descriptor-map-info-gen.fir b/flang/test/Transforms/omp-descriptor-map-info-gen.fir
deleted file mode 100644
index 05d05b0ecb59..000000000000
--- a/flang/test/Transforms/omp-descriptor-map-info-gen.fir
+++ /dev/null
@@ -1,44 +0,0 @@
-// RUN: fir-opt --omp-descriptor-map-info-gen %s | FileCheck %s
-
-module attributes {omp.is_target_device = false} {
-  func.func @test_descriptor_expansion_pass(%arg0: !fir.box<!fir.array<?xi32>>) {
-    %0 = fir.alloca !fir.box<!fir.heap<i32>>
-    %1 = fir.zero_bits !fir.heap<i32>
-    %2:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "test"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-    %3 = fir.embox %1 : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
-    fir.store %3 to %0 : !fir.ref<!fir.box<!fir.heap<i32>>>
-    %4:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "test2"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
-    %5 = fir.allocmem i32 {fir.must_be_heap = true}
-    %6 = fir.embox %5 : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
-    fir.store %6 to %4#1 : !fir.ref<!fir.box<!fir.heap<i32>>>
-    %c0 = arith.constant 1 : index  
-    %c1 = arith.constant 0 : index
-    %c2 = arith.constant 10 : index
-    %dims:3 = fir.box_dims %2#1, %c1 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
-    %bounds = omp.map.bounds lower_bound(%c1 : index) upper_bound(%c2 : index) extent(%dims#1 : index) stride(%dims#2 : index) start_idx(%c0 : index) {stride_in_bytes = true}
-    %7 = fir.box_addr %2#1 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
-    %8 = omp.map.info var_ptr(%4#1 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(tofrom) capture(ByRef) -> !fir.ref<!fir.box<!fir.heap<i32>>>
-    %9 = omp.map.info var_ptr(%7 : !fir.ref<!fir.array<?xi32>>, !fir.array<?xi32>) map_clauses(from) capture(ByRef) bounds(%bounds) -> !fir.ref<!fir.array<?xi32>>
-    omp.target map_entries(%8 -> %arg1, %9 -> %arg2 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.array<?xi32>>) {
-    ^bb0(%arg1: !fir.ref<!fir.box<!fir.heap<i32>>>, %arg2: !fir.ref<!fir.array<?xi32>>):
-      omp.terminator
-    }
-    return 
-  }
-}
- 
-// CHECK: func.func @test_descriptor_expansion_pass(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>) {
-// CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
-// CHECK: %[[ALLOCA2:.*]] = fir.alloca !fir.box<!fir.heap<i32>>
-// CHECK: %[[DECLARE1:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "test"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-// CHECK: %[[DECLARE2:.*]]:2 = hlfir.declare %[[ALLOCA2]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "test2"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
-// CHECK: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%{{.*}} : index) upper_bound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) start_idx(%{{.*}} : index) {stride_in_bytes = true}
-// CHECK: %[[BASE_ADDR_OFF:.*]] = fir.box_offset %[[DECLARE2]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
-// CHECK: %[[DESC_MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE2]]#1 : !fir.ref<!fir.box<!fir.heap<i32>>>, i32) var_ptr_ptr(%[[BASE_ADDR_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>) map_clauses(tofrom) capture(ByRef) -> !fir.llvm_ptr<!fir.ref<i32>> {name = ""}
-// CHECK: %[[DESC_PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE2]]#1 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(tofrom) capture(ByRef) members(%[[DESC_MEMBER_MAP]] : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.ref<!fir.box<!fir.heap<i32>>>
-// CHECK: fir.store %[[DECLARE1]]#1 to %[[ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
-// CHECK: %[[BASE_ADDR_OFF_2:.*]] = fir.box_offset %[[ALLOCA]] base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
-// CHECK: %[[DESC_MEMBER_MAP_2:.*]] = omp.map.info var_ptr(%[[ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.array<?xi32>) var_ptr_ptr(%[[BASE_ADDR_OFF_2]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) map_clauses(from) capture(ByRef) bounds(%[[BOUNDS]]) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>> {name = ""}
-// CHECK: %[[DESC_PARENT_MAP_2:.*]] = omp.map.info var_ptr(%[[ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(from) capture(ByRef) members(%15 : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.array<?xi32>>
-// CHECK: omp.target map_entries(%[[DESC_MEMBER_MAP]] -> %[[ARG1:.*]], %[[DESC_PARENT_MAP]] -> %[[ARG2:.*]], %[[DESC_MEMBER_MAP_2]] -> %[[ARG3:.*]], %[[DESC_PARENT_MAP_2]] -> %[[ARG4:.*]] : {{.*}}) {
-// CHECK: ^bb0(%[[ARG1]]: !fir.llvm_ptr<!fir.ref<i32>>, %[[ARG2]]: !fir.ref<!fir.box<!fir.heap<i32>>>, %[[ARG3]]: !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, %[[ARG4]]: !fir.ref<!fir.array<?xi32>>):
diff --git a/flang/test/Transforms/omp-map-info-finalization.fir b/flang/test/Transforms/omp-map-info-finalization.fir
new file mode 100644
index 000000000000..9d776b674151
--- /dev/null
+++ b/flang/test/Transforms/omp-map-info-finalization.fir
@@ -0,0 +1,99 @@
+// RUN: fir-opt --split-input-file --omp-map-info-finalization %s | FileCheck %s
+
+module attributes {omp.is_target_device = false} {
+  func.func @test_descriptor_expansion_pass(%arg0: !fir.box<!fir.array<?xi32>>) {
+    %0 = fir.alloca !fir.box<!fir.heap<i32>>
+    %1 = fir.zero_bits !fir.heap<i32>
+    %2:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "test"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+    %3 = fir.embox %1 : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+    fir.store %3 to %0 : !fir.ref<!fir.box<!fir.heap<i32>>>
+    %4:2 = hlfir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "test2"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+    %5 = fir.allocmem i32 {fir.must_be_heap = true}
+    %6 = fir.embox %5 : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+    fir.store %6 to %4#1 : !fir.ref<!fir.box<!fir.heap<i32>>>
+    %c0 = arith.constant 1 : index  
+    %c1 = arith.constant 0 : index
+    %c2 = arith.constant 10 : index
+    %dims:3 = fir.box_dims %2#1, %c1 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+    %bounds = omp.map.bounds lower_bound(%c1 : index) upper_bound(%c2 : index) extent(%dims#1 : index) stride(%dims#2 : index) start_idx(%c0 : index) {stride_in_bytes = true}
+    %7 = fir.box_addr %2#1 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
+    %8 = omp.map.info var_ptr(%4#1 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(tofrom) capture(ByRef) -> !fir.ref<!fir.box<!fir.heap<i32>>>
+    %9 = omp.map.info var_ptr(%7 : !fir.ref<!fir.array<?xi32>>, !fir.array<?xi32>) map_clauses(from) capture(ByRef) bounds(%bounds) -> !fir.ref<!fir.array<?xi32>>
+    omp.target map_entries(%8 -> %arg1, %9 -> %arg2 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.array<?xi32>>) {
+    ^bb0(%arg1: !fir.ref<!fir.box<!fir.heap<i32>>>, %arg2: !fir.ref<!fir.array<?xi32>>):
+      omp.terminator
+    }
+    return 
+  }
+}
+ 
+// CHECK: func.func @test_descriptor_expansion_pass(%[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>) {
+// CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
+// CHECK: %[[ALLOCA2:.*]] = fir.alloca !fir.box<!fir.heap<i32>>
+// CHECK: %[[DECLARE1:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "test"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK: %[[DECLARE2:.*]]:2 = hlfir.declare %[[ALLOCA2]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "test2"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+// CHECK: %[[BOUNDS:.*]] = omp.map.bounds lower_bound(%{{.*}} : index) upper_bound(%{{.*}} : index) extent(%{{.*}} : index) stride(%{{.*}} : index) start_idx(%{{.*}} : index) {stride_in_bytes = true}
+// CHECK: %[[BASE_ADDR_OFF:.*]] = fir.box_offset %[[DECLARE2]]#1 base_addr : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> !fir.llvm_ptr<!fir.ref<i32>>
+// CHECK: %[[DESC_MEMBER_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE2]]#1 : !fir.ref<!fir.box<!fir.heap<i32>>>, i32) var_ptr_ptr(%[[BASE_ADDR_OFF]] : !fir.llvm_ptr<!fir.ref<i32>>) map_clauses(tofrom) capture(ByRef) -> !fir.llvm_ptr<!fir.ref<i32>> {name = ""}
+// CHECK: %[[DESC_PARENT_MAP:.*]] = omp.map.info var_ptr(%[[DECLARE2]]#1 : !fir.ref<!fir.box<!fir.heap<i32>>>, !fir.box<!fir.heap<i32>>) map_clauses(tofrom) capture(ByRef) members(%[[DESC_MEMBER_MAP]] : [0] : !fir.llvm_ptr<!fir.ref<i32>>) -> !fir.ref<!fir.box<!fir.heap<i32>>>
+// CHECK: fir.store %[[DECLARE1]]#1 to %[[ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
+// CHECK: %[[BASE_ADDR_OFF_2:.*]] = fir.box_offset %[[ALLOCA]] base_addr : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>
+// CHECK: %[[DESC_MEMBER_MAP_2:.*]] = omp.map.info var_ptr(%[[ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.array<?xi32>) var_ptr_ptr(%[[BASE_ADDR_OFF_2]] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) map_clauses(from) capture(ByRef) bounds(%[[BOUNDS]]) -> !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>> {name = ""}
+// CHECK: %[[DESC_PARENT_MAP_2:.*]] = omp.map.info var_ptr(%[[ALLOCA]] : !fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.box<!fir.array<?xi32>>) map_clauses(from) capture(ByRef) members(%[[DESC_MEMBER_MAP_2]] : [0] : !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>) -> !fir.ref<!fir.array<?xi32>>
+// CHECK: omp.target map_entries(%[[DESC_MEMBER_MAP]] -> %[[ARG1:.*]], %[[DESC_PARENT_MAP]] -> %[[ARG2:.*]], %[[DESC_MEMBER_MAP_2]] -> %[[ARG3:.*]], %[[DESC_PARENT_MAP_2]] -> %[[ARG4:.*]] : {{.*}}) {
+// CHECK: ^bb0(%[[ARG1]]: !fir.llvm_ptr<!fir.ref<i32>>, %[[ARG2]]: !fir.ref<!fir.box<!fir.heap<i32>>>, %[[ARG3]]: !fir.llvm_ptr<!fir.ref<!fir.array<?xi32>>>, %[[ARG4]]: !fir.ref<!fir.array<?xi32>>):
+
+// -----
+
+module attributes {omp.is_target_device = false} {
+  func.func @test_derived_type_map_operand_and_block_addition(%arg0: !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) {
+    %0 = hlfir.designate %arg0{"rx"}   : (!fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) -> !fir.ref<f32>
+    %1 = omp.map.info var_ptr(%0 : !fir.ref<f32>, f32) map_clauses(from) capture(ByRef) -> !fir.ref<f32> {name = "scalar_struct%rx"}
+    %2 = hlfir.designate %arg0{"ry"}   : (!fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) -> !fir.ref<f32>
+    %3 = omp.map.info var_ptr(%2 : !fir.ref<f32>, f32) map_clauses(from) capture(ByRef) -> !fir.ref<f32> {name = "scalar_struct%ry"}
+    %4 = omp.map.info var_ptr(%arg0 : !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>, !fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>) map_clauses(from) capture(ByRef) members(%1, %3 : [1], [4] : !fir.ref<f32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>> {name = "scalar_struct", partial_map = true}
+    omp.target map_entries(%4 -> %arg1 : !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) {
+    ^bb0(%arg1: !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>):
+      omp.terminator
+    }
+    return
+  }
+}
+
+// CHECK: func.func @test_derived_type_map_operand_and_block_addition(%{{.*}}: !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) { 
+// CHECK:   %[[MAP_MEMBER_1:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<f32>, f32) map_clauses(from) capture(ByRef) -> !fir.ref<f32> {name = "scalar_struct%rx"}
+// CHECK:   %[[MAP_MEMBER_2:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<f32>, f32) map_clauses(from) capture(ByRef) -> !fir.ref<f32> {name = "scalar_struct%ry"}
+// CHECK:   %[[MAP_PARENT:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>, !fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>) map_clauses(from) capture(ByRef) members(%[[MAP_MEMBER_1]], %[[MAP_MEMBER_2]] : [1], [4] : !fir.ref<f32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>> {name = "scalar_struct", partial_map = true}
+// CHECK:   omp.target map_entries(%[[MAP_MEMBER_1]] -> %[[ARG1:.*]], %[[MAP_MEMBER_2]] -> %[[ARG2:.*]], %[[MAP_PARENT]] -> %[[ARG3:.*]] : !fir.ref<f32>, !fir.ref<f32>, !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>) {
+// CHECK:     ^bb0(%[[ARG1]]: !fir.ref<f32>, %[[ARG2]]: !fir.ref<f32>, %[[ARG3]]: !fir.ref<!fir.type<_QFTdtype{ix:i32,rx:f32,zx:!fir.complex<4>,nested:!fir.box<!fir.heap<!fir.type<_QFTdtype>>>,ry:f32}>>):
+
+// -----
+
+module attributes {omp.is_target_device = false} {
+func.func @test_nested_derived_type_map_operand_and_block_addition(%arg0: !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {        
+    %0 = fir.declare %arg0 {uniq_name = "_QFmaptype_derived_nested_explicit_multiple_membersEsa"} : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>
+    %1 = fir.field_index n, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>
+    %2 = fir.coordinate_of %0, %1 : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.field) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>
+    %3 = fir.field_index i, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>
+    %4 = fir.coordinate_of %2, %3 : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>, !fir.field) -> !fir.ref<i32>
+    %5 = omp.map.info var_ptr(%4 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "sa%n%i"}
+    %6 = fir.field_index n, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>
+    %7 = fir.coordinate_of %0, %6 : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.field) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>
+    %8 = fir.field_index r, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>
+    %9 = fir.coordinate_of %7, %8 : (!fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>>, !fir.field) -> !fir.ref<f32>
+    %10 = omp.map.info var_ptr(%9 : !fir.ref<f32>, f32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<f32> {name = "sa%n%r"}
+    %11 = omp.map.info var_ptr(%0 : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>, !fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>) map_clauses(tofrom) capture(ByRef) members(%5, %10 : [1,0], [1,1] : !fir.ref<i32>, !fir.ref<f32>) -> !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>> {name = "sa", partial_map = true}
+    omp.target map_entries(%11 -> %arg1 : !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) {
+    ^bb0(%arg1: !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>):
+      omp.terminator
+    }
+    return
+  }
+}
+
+// CHECK: func.func @test_nested_derived_type_map_operand_and_block_addition(%{{.*}}: !fir.ref<!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTscalar_and_array{r:f32,n:!fir.type<_QFmaptype_derived_nested_explicit_multiple_membersTnested{i:i32,r:f32}>}>>) { 
+// CHECK:   %[[MAP_MEMBER_1:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "sa%n%i"}
+// CHECK:   %[[MAP_MEMBER_2:.*]] = omp.map.info var_ptr(%{{.*}} : !fir.ref<f32>, f32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<f32> {name = "sa%n%r"}
+// CHECK:   %[[MAP_PARENT:.*]] = omp.map.info var_ptr(%{{.*}} : {{.*}}, {{.*}}) map_clauses(tofrom) capture(ByRef) members(%[[MAP_MEMBER_1]], %[[MAP_MEMBER_2]] : [1,0], [1,1] : !fir.ref<i32>, !fir.ref<f32>) -> {{.*}} {name = "sa", partial_map = true}
+// CHECK:   omp.target map_entries(%[[MAP_MEMBER_1]] -> %[[ARG1:.*]], %[[MAP_MEMBER_2]] -> %[[ARG2:.*]], %[[MAP_PARENT]] -> %[[ARG3:.*]] : !fir.ref<i32>, !fir.ref<f32>, {{.*}}) {
+// CHECK:     ^bb0(%[[ARG1]]: !fir.ref<i32>, %[[ARG2]]: !fir.ref<f32>, %[[ARG3]]: {{.*}}):
-- 
GitLab


From 7237bef5da57e8d67db25df7c22379e81e5903a7 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 10 May 2024 12:08:59 -0700
Subject: [PATCH 0443/1206] [RISCV] Use map::count instead of hasExtension in
 computeDefaultABI. NFC

hasExtension checks if the extension name is a known extension name.
That should always be true for the extensions listed here so we can
skip that check.
---
 llvm/lib/TargetParser/RISCVISAInfo.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 8ae0c355479c..931c2aa41ac7 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -998,19 +998,19 @@ RISCVISAInfo::postProcessAndChecking(std::unique_ptr<RISCVISAInfo> &&ISAInfo) {
 
 StringRef RISCVISAInfo::computeDefaultABI() const {
   if (XLen == 32) {
-    if (hasExtension("e"))
+    if (Exts.count("e"))
       return "ilp32e";
-    if (hasExtension("d"))
+    if (Exts.count("d"))
       return "ilp32d";
-    if (hasExtension("f"))
+    if (Exts.count("f"))
       return "ilp32f";
     return "ilp32";
   } else if (XLen == 64) {
-    if (hasExtension("e"))
+    if (Exts.count("e"))
       return "lp64e";
-    if (hasExtension("d"))
+    if (Exts.count("d"))
       return "lp64d";
-    if (hasExtension("f"))
+    if (Exts.count("f"))
       return "lp64f";
     return "lp64";
   }
-- 
GitLab


From d8e73752a5f4f79ef4293d8f446c03062010233d Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Fri, 10 May 2024 15:32:04 -0400
Subject: [PATCH 0444/1206] Reland "[libc][NFC] adjust time related
 implementations"" (#91687)

Reverts llvm/llvm-project#91657 and Relands #91485
---
 libc/hdr/CMakeLists.txt                       |  9 ++++
 libc/hdr/time_macros.h                        | 22 +++++++++
 libc/hdr/types/CMakeLists.txt                 | 45 +++++++++++++++++++
 libc/hdr/types/clock_t.h                      | 22 +++++++++
 libc/hdr/types/clockid_t.h                    | 22 +++++++++
 libc/hdr/types/struct_timeval.h               | 21 +++++++++
 libc/hdr/types/suseconds_t.h                  | 22 +++++++++
 libc/hdr/types/time_t.h                       | 22 +++++++++
 libc/src/__support/CMakeLists.txt             |  2 +
 libc/src/__support/time/CMakeLists.txt        | 12 +++++
 libc/src/__support/time/linux/CMakeLists.txt  | 14 ++++++
 .../time/linux/clock_gettime.cpp}             | 23 +++-------
 libc/src/__support/time/linux/clock_gettime.h | 23 ++++++++++
 libc/src/__support/time/units.h               | 38 ++++++++++++++++
 libc/src/time/clock.h                         |  2 +-
 libc/src/time/clock_gettime.h                 |  5 ++-
 libc/src/time/gettimeofday.h                  |  2 +-
 libc/src/time/gpu/CMakeLists.txt              |  3 ++
 libc/src/time/gpu/clock.cpp                   |  3 +-
 libc/src/time/gpu/time_utils.h                |  3 +-
 libc/src/time/linux/CMakeLists.txt            | 30 +++++++------
 libc/src/time/linux/clock.cpp                 | 20 ++++-----
 libc/src/time/linux/clock_gettime.cpp         |  9 +---
 libc/src/time/linux/gettimeofday.cpp          | 14 +++---
 libc/src/time/linux/time.cpp                  | 12 ++---
 libc/src/time/nanosleep.h                     |  4 +-
 libc/src/time/time_func.h                     |  2 +-
 27 files changed, 331 insertions(+), 75 deletions(-)
 create mode 100644 libc/hdr/time_macros.h
 create mode 100644 libc/hdr/types/clock_t.h
 create mode 100644 libc/hdr/types/clockid_t.h
 create mode 100644 libc/hdr/types/struct_timeval.h
 create mode 100644 libc/hdr/types/suseconds_t.h
 create mode 100644 libc/hdr/types/time_t.h
 create mode 100644 libc/src/__support/time/CMakeLists.txt
 create mode 100644 libc/src/__support/time/linux/CMakeLists.txt
 rename libc/src/{time/linux/clockGetTimeImpl.h => __support/time/linux/clock_gettime.cpp} (64%)
 create mode 100644 libc/src/__support/time/linux/clock_gettime.h
 create mode 100644 libc/src/__support/time/units.h

diff --git a/libc/hdr/CMakeLists.txt b/libc/hdr/CMakeLists.txt
index 179b05e6ee96..754934251430 100644
--- a/libc/hdr/CMakeLists.txt
+++ b/libc/hdr/CMakeLists.txt
@@ -68,4 +68,13 @@ add_proxy_header_library(
     libc.include.llvm-libc-macros.sys_epoll_macros
 )
 
+add_proxy_header_library(
+  time_macros
+  HDRS
+    time_macros.h
+  FULL_BUILD_DEPENDS
+    libc.include.time
+    libc.include.llvm-libc-macros.time_macros
+)
+
 add_subdirectory(types)
diff --git a/libc/hdr/time_macros.h b/libc/hdr/time_macros.h
new file mode 100644
index 000000000000..dc36fe66f7a8
--- /dev/null
+++ b/libc/hdr/time_macros.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from time.h ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TIME_MACROS_H
+#define LLVM_LIBC_HDR_TIME_MACROS_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-macros/time-macros.h"
+
+#else // Overlay mode
+
+#include <time.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TIME_MACROS_H
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 46a66ec59020..3a1bb2f3c340 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -63,3 +63,48 @@ add_proxy_header_library(
     libc.include.llvm-libc-types.fexcept_t
     libc.include.fenv
 )
+
+add_proxy_header_library(
+  time_t
+  HDRS
+    time_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.time_t
+    libc.include.time
+)
+
+add_proxy_header_library(
+  clockid_t
+  HDRS
+    clockid_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.clockid_t
+    libc.include.sys_types
+)
+
+add_proxy_header_library(
+  clock_t
+  HDRS
+    clock_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.clock_t
+    libc.include.time
+)
+
+add_proxy_header_library(
+  suseconds_t
+  HDRS
+    suseconds_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.suseconds_t
+    libc.include.sys_time
+)
+
+add_proxy_header_library(
+  struct_timeval
+  HDRS
+    struct_timeval.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.struct_timeval
+    libc.include.sys_time
+)
diff --git a/libc/hdr/types/clock_t.h b/libc/hdr/types/clock_t.h
new file mode 100644
index 000000000000..b0b658e96c3d
--- /dev/null
+++ b/libc/hdr/types/clock_t.h
@@ -0,0 +1,22 @@
+//===-- Proxy for clock_t -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_CLOCK_T_H
+#define LLVM_LIBC_HDR_TYPES_CLOCK_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/clock_t.h"
+
+#else // Overlay mode
+
+#include <sys/types.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_CLOCK_T_H
diff --git a/libc/hdr/types/clockid_t.h b/libc/hdr/types/clockid_t.h
new file mode 100644
index 000000000000..333342072a2f
--- /dev/null
+++ b/libc/hdr/types/clockid_t.h
@@ -0,0 +1,22 @@
+//===-- Proxy for clockid_t -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_CLOCKID_T_H
+#define LLVM_LIBC_HDR_TYPES_CLOCKID_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/clockid_t.h"
+
+#else // Overlay mode
+
+#include <sys/types.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_CLOCKID_T_H
diff --git a/libc/hdr/types/struct_timeval.h b/libc/hdr/types/struct_timeval.h
new file mode 100644
index 000000000000..8fc321a52d71
--- /dev/null
+++ b/libc/hdr/types/struct_timeval.h
@@ -0,0 +1,21 @@
+//===-- Proxy for struct timeval  ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_HDR_TYPES_STRUCT_TIMEVAL_H
+#define LLVM_LIBC_HDR_TYPES_STRUCT_TIMEVAL_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/struct_timeval.h"
+
+#else
+
+#include <sys/time.h>
+
+#endif // LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_STRUCT_TIMEVAL_H
diff --git a/libc/hdr/types/suseconds_t.h b/libc/hdr/types/suseconds_t.h
new file mode 100644
index 000000000000..72e54a965f75
--- /dev/null
+++ b/libc/hdr/types/suseconds_t.h
@@ -0,0 +1,22 @@
+//===-- Proxy for suseconds_t ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TIMES_SUSECONDS_T_H
+#define LLVM_LIBC_HDR_TIMES_SUSECONDS_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/suseconds_t.h"
+
+#else // Overlay mode
+
+#include <sys/types.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // #ifndef LLVM_LIBC_HDR_TIMES_SUSECONDS_T_H
diff --git a/libc/hdr/types/time_t.h b/libc/hdr/types/time_t.h
new file mode 100644
index 000000000000..fc9a1506a2cd
--- /dev/null
+++ b/libc/hdr/types/time_t.h
@@ -0,0 +1,22 @@
+//===-- Proxy for time_t --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_TIME_T_H
+#define LLVM_LIBC_HDR_TYPES_TIME_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/time_t.h"
+
+#else // Overlay mode
+
+#include <time.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_TIME_T_H
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index dcae55e050bf..32d693ec6a26 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -281,3 +281,5 @@ add_subdirectory(File)
 add_subdirectory(HashTable)
 
 add_subdirectory(fixed_point)
+
+add_subdirectory(time)
diff --git a/libc/src/__support/time/CMakeLists.txt b/libc/src/__support/time/CMakeLists.txt
new file mode 100644
index 000000000000..89ddffb09938
--- /dev/null
+++ b/libc/src/__support/time/CMakeLists.txt
@@ -0,0 +1,12 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${LIBC_TARGET_OS})
+endif()
+
+add_header_library(
+  units
+  HDRS
+    units.h
+  DEPENDS
+    libc.src.__support.common
+    libc.hdr.types.time_t
+)
diff --git a/libc/src/__support/time/linux/CMakeLists.txt b/libc/src/__support/time/linux/CMakeLists.txt
new file mode 100644
index 000000000000..f04d550555e1
--- /dev/null
+++ b/libc/src/__support/time/linux/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_object_library(
+  clock_gettime
+  HDRS
+    clock_gettime.h
+  SRCS
+    clock_gettime.cpp
+  DEPENDS
+    libc.include.sys_syscall
+    libc.hdr.types.struct_timespec
+    libc.hdr.types.clockid_t
+    libc.src.__support.common
+    libc.src.__support.error_or
+    libc.src.__support.OSUtil.osutil
+)
diff --git a/libc/src/time/linux/clockGetTimeImpl.h b/libc/src/__support/time/linux/clock_gettime.cpp
similarity index 64%
rename from libc/src/time/linux/clockGetTimeImpl.h
rename to libc/src/__support/time/linux/clock_gettime.cpp
index 8c8c9fcf845c..7f266b282a39 100644
--- a/libc/src/time/linux/clockGetTimeImpl.h
+++ b/libc/src/__support/time/linux/clock_gettime.cpp
@@ -1,4 +1,4 @@
-//===- Linux implementation of the POSIX clock_gettime function -*- C++ -*-===//
+//===--- clock_gettime linux implementation ---------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,23 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_TIME_LINUX_CLOCKGETTIMEIMPL_H
-#define LLVM_LIBC_SRC_TIME_LINUX_CLOCKGETTIMEIMPL_H
-
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
-#include "src/__support/common.h"
-#include "src/__support/error_or.h"
-#include "src/errno/libc_errno.h"
-
-#include <stdint.h>      // For int64_t.
-#include <sys/syscall.h> // For syscall numbers.
-#include <time.h>
-
+#include "src/__support/time/linux/clock_gettime.h"
+#include "src/__support/OSUtil/syscall.h"
+#include <sys/syscall.h>
 namespace LIBC_NAMESPACE {
 namespace internal {
-
-LIBC_INLINE ErrorOr<int> clock_gettimeimpl(clockid_t clockid,
-                                           struct timespec *ts) {
+ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts) {
 #if SYS_clock_gettime
   int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_clock_gettime,
                                               static_cast<long>(clockid),
@@ -44,5 +33,3 @@ LIBC_INLINE ErrorOr<int> clock_gettimeimpl(clockid_t clockid,
 
 } // namespace internal
 } // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC_TIME_LINUX_CLOCKGETTIMEIMPL_H
diff --git a/libc/src/__support/time/linux/clock_gettime.h b/libc/src/__support/time/linux/clock_gettime.h
new file mode 100644
index 000000000000..b1572726f630
--- /dev/null
+++ b/libc/src/__support/time/linux/clock_gettime.h
@@ -0,0 +1,23 @@
+//===--- clock_gettime linux implementation ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
+#define LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
+#include "hdr/types/clockid_t.h"
+#include "hdr/types/struct_timespec.h"
+#include "src/__support/common.h"
+
+#include "src/__support/error_or.h"
+
+namespace LIBC_NAMESPACE {
+namespace internal {
+ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts);
+}
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
diff --git a/libc/src/__support/time/units.h b/libc/src/__support/time/units.h
new file mode 100644
index 000000000000..f6bd19f9b139
--- /dev/null
+++ b/libc/src/__support/time/units.h
@@ -0,0 +1,38 @@
+//===--- Time units conversion ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_UNITS_H
+#define LLVM_LIBC_SRC___SUPPORT_TIME_UNITS_H
+
+#include "hdr/types/time_t.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+namespace time_units {
+LIBC_INLINE constexpr time_t operator""_s_ns(unsigned long long s) {
+  return s * 1'000'000'000;
+}
+LIBC_INLINE constexpr time_t operator""_s_us(unsigned long long s) {
+  return s * 1'000'000;
+}
+LIBC_INLINE constexpr time_t operator""_s_ms(unsigned long long s) {
+  return s * 1'000;
+}
+LIBC_INLINE constexpr time_t operator""_ms_ns(unsigned long long ms) {
+  return ms * 1'000'000;
+}
+LIBC_INLINE constexpr time_t operator""_ms_us(unsigned long long ms) {
+  return ms * 1'000;
+}
+LIBC_INLINE constexpr time_t operator""_us_ns(unsigned long long us) {
+  return us * 1'000;
+}
+} // namespace time_units
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_TIME_UNITS_H
diff --git a/libc/src/time/clock.h b/libc/src/time/clock.h
index d4af7656644a..f5d14d036e13 100644
--- a/libc/src/time/clock.h
+++ b/libc/src/time/clock.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_TIME_CLOCK_H
 #define LLVM_LIBC_SRC_TIME_CLOCK_H
 
-#include <time.h>
+#include "hdr/types/clock_t.h"
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/time/clock_gettime.h b/libc/src/time/clock_gettime.h
index 72e2e1949feb..48e81a355429 100644
--- a/libc/src/time/clock_gettime.h
+++ b/libc/src/time/clock_gettime.h
@@ -9,11 +9,12 @@
 #ifndef LLVM_LIBC_SRC_TIME_CLOCK_GETTIME_H
 #define LLVM_LIBC_SRC_TIME_CLOCK_GETTIME_H
 
-#include <time.h>
+#include "hdr/types/clockid_t.h"
+#include "hdr/types/struct_timespec.h"
 
 namespace LIBC_NAMESPACE {
 
-int clock_gettime(clockid_t clockid, struct timespec *tp);
+int clock_gettime(clockid_t clockid, timespec *tp);
 
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/src/time/gettimeofday.h b/libc/src/time/gettimeofday.h
index 880b94cee731..62ee31edcad6 100644
--- a/libc/src/time/gettimeofday.h
+++ b/libc/src/time/gettimeofday.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_TIME_GETTIMEOFDAY_H
 #define LLVM_LIBC_SRC_TIME_GETTIMEOFDAY_H
 
-#include <time.h>
+#include "hdr/types/struct_timeval.h"
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/time/gpu/CMakeLists.txt b/libc/src/time/gpu/CMakeLists.txt
index bb79d92399b3..088271d88191 100644
--- a/libc/src/time/gpu/CMakeLists.txt
+++ b/libc/src/time/gpu/CMakeLists.txt
@@ -4,6 +4,9 @@ add_object_library(
     time_utils.cpp
   HDRS
     time_utils.h
+  DEPENDS
+    libc.hdr.types.clock_t
+    libc.hdr.time_macros
 )
 
 add_entrypoint_object(
diff --git a/libc/src/time/gpu/clock.cpp b/libc/src/time/gpu/clock.cpp
index 86cc97e2a3bf..8ddfc27975bb 100644
--- a/libc/src/time/gpu/clock.cpp
+++ b/libc/src/time/gpu/clock.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "time_utils.h"
-
 #include "src/time/clock.h"
+#include "src/time/gpu/time_utils.h"
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/time/gpu/time_utils.h b/libc/src/time/gpu/time_utils.h
index 8a9a5f0f65b8..3f1fd11c1791 100644
--- a/libc/src/time/gpu/time_utils.h
+++ b/libc/src/time/gpu/time_utils.h
@@ -9,8 +9,9 @@
 #ifndef LLVM_LIBC_SRC_TIME_GPU_TIME_UTILS_H
 #define LLVM_LIBC_SRC_TIME_GPU_TIME_UTILS_H
 
+#include "hdr/time_macros.h"
+#include "hdr/types/clock_t.h"
 #include "src/__support/GPU/utils.h"
-
 namespace LIBC_NAMESPACE {
 
 #if defined(LIBC_TARGET_ARCH_IS_AMDGPU)
diff --git a/libc/src/time/linux/CMakeLists.txt b/libc/src/time/linux/CMakeLists.txt
index df79bf598626..c15fb44ad5d1 100644
--- a/libc/src/time/linux/CMakeLists.txt
+++ b/libc/src/time/linux/CMakeLists.txt
@@ -5,9 +5,9 @@ add_entrypoint_object(
   HDRS
     ../time_func.h
   DEPENDS
-    libc.include.time
-    libc.include.sys_syscall
-    libc.src.__support.OSUtil.osutil
+    libc.hdr.time_macros
+    libc.hdr.types.time_t
+    libc.src.__support.time.linux.clock_gettime
     libc.src.errno.errno
 )
 
@@ -18,10 +18,11 @@ add_entrypoint_object(
   HDRS
     ../clock.h
   DEPENDS
-    libc.include.time
-    libc.include.sys_syscall
+    libc.hdr.time_macros
+    libc.hdr.types.clock_t
+    libc.src.__support.time.units
+    libc.src.__support.time.linux.clock_gettime
     libc.src.__support.CPP.limits
-    libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
 
@@ -32,10 +33,10 @@ add_entrypoint_object(
   HDRS
     ../nanosleep.h
   DEPENDS
-    libc.include.time
+    libc.hdr.types.struct_timespec
     libc.include.sys_syscall
-    libc.src.__support.CPP.limits
     libc.src.__support.OSUtil.osutil
+    libc.src.__support.CPP.limits
     libc.src.errno.errno
 )
 
@@ -46,9 +47,9 @@ add_entrypoint_object(
   HDRS
     ../clock_gettime.h
   DEPENDS
-    libc.include.time
-    libc.include.sys_syscall
-    libc.src.__support.OSUtil.osutil
+    libc.hdr.types.clockid_t
+    libc.hdr.types.struct_timespec
+    libc.src.__support.time.linux.clock_gettime
     libc.src.errno.errno
 )
 
@@ -59,8 +60,9 @@ add_entrypoint_object(
   HDRS
     ../gettimeofday.h
   DEPENDS
-    libc.include.time
-    libc.include.sys_syscall
-    libc.src.__support.OSUtil.osutil
+    libc.hdr.time_macros
+    libc.hdr.types.suseconds_t
+    libc.src.__support.time.linux.clock_gettime
+    libc.src.__support.time.units
     libc.src.errno.errno
 )
diff --git a/libc/src/time/linux/clock.cpp b/libc/src/time/linux/clock.cpp
index 1e95f0526bc9..2c1eee8e5d60 100644
--- a/libc/src/time/linux/clock.cpp
+++ b/libc/src/time/linux/clock.cpp
@@ -7,21 +7,19 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/time/clock.h"
-
+#include "hdr/time_macros.h"
 #include "src/__support/CPP/limits.h"
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/time/linux/clock_gettime.h"
+#include "src/__support/time/units.h"
 #include "src/errno/libc_errno.h"
-#include "src/time/linux/clockGetTimeImpl.h"
-
-#include <sys/syscall.h> // For syscall numbers.
-#include <time.h>
 
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(clock_t, clock, ()) {
+  using namespace time_units;
   struct timespec ts;
-  auto result = internal::clock_gettimeimpl(CLOCK_PROCESS_CPUTIME_ID, &ts);
+  auto result = internal::clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
   if (!result.has_value()) {
     libc_errno = result.error();
     return -1;
@@ -34,15 +32,15 @@ LLVM_LIBC_FUNCTION(clock_t, clock, ()) {
       cpp::numeric_limits<clock_t>::max() / CLOCKS_PER_SEC;
   if (ts.tv_sec > CLOCK_SECS_MAX)
     return clock_t(-1);
-  if (ts.tv_nsec / 1000000000 > CLOCK_SECS_MAX - ts.tv_sec)
+  if (ts.tv_nsec / 1_s_ns > CLOCK_SECS_MAX - ts.tv_sec)
     return clock_t(-1);
 
   // For the integer computation converting tv_nsec to clocks to work
   // correctly, we want CLOCKS_PER_SEC to be less than 1000000000.
-  static_assert(1000000000 > CLOCKS_PER_SEC,
-                "Expected CLOCKS_PER_SEC to be less than 1000000000.");
+  static_assert(1_s_ns > CLOCKS_PER_SEC,
+                "Expected CLOCKS_PER_SEC to be less than 1'000'000'000.");
   return clock_t(ts.tv_sec * CLOCKS_PER_SEC +
-                 ts.tv_nsec / (1000000000 / CLOCKS_PER_SEC));
+                 ts.tv_nsec / (1_s_ns / CLOCKS_PER_SEC));
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/time/linux/clock_gettime.cpp b/libc/src/time/linux/clock_gettime.cpp
index 47e974a866c8..d7b8cfd245bc 100644
--- a/libc/src/time/linux/clock_gettime.cpp
+++ b/libc/src/time/linux/clock_gettime.cpp
@@ -7,21 +7,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/time/clock_gettime.h"
-
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
+#include "src/__support/time/linux/clock_gettime.h"
 #include "src/errno/libc_errno.h"
-#include "src/time/linux/clockGetTimeImpl.h"
-
-#include <sys/syscall.h> // For syscall numbers.
-#include <time.h>
 
 namespace LIBC_NAMESPACE {
 
 // TODO(michaelrj): Move this into time/linux with the other syscalls.
 LLVM_LIBC_FUNCTION(int, clock_gettime,
                    (clockid_t clockid, struct timespec *ts)) {
-  auto result = internal::clock_gettimeimpl(clockid, ts);
+  auto result = internal::clock_gettime(clockid, ts);
 
   // A negative return value indicates an error with the magnitude of the
   // value being the error code.
diff --git a/libc/src/time/linux/gettimeofday.cpp b/libc/src/time/linux/gettimeofday.cpp
index 07ab4d579176..f868f5ff4d4b 100644
--- a/libc/src/time/linux/gettimeofday.cpp
+++ b/libc/src/time/linux/gettimeofday.cpp
@@ -7,24 +7,24 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/time/gettimeofday.h"
-
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "hdr/time_macros.h"
+#include "hdr/types/suseconds_t.h"
 #include "src/__support/common.h"
+#include "src/__support/time/linux/clock_gettime.h"
+#include "src/__support/time/units.h"
 #include "src/errno/libc_errno.h"
-#include "src/time/linux/clockGetTimeImpl.h"
-
-#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE {
 
 // TODO(michaelrj): Move this into time/linux with the other syscalls.
 LLVM_LIBC_FUNCTION(int, gettimeofday,
                    (struct timeval * tv, [[maybe_unused]] void *unused)) {
+  using namespace time_units;
   if (tv == nullptr)
     return 0;
 
   struct timespec ts;
-  auto result = internal::clock_gettimeimpl(CLOCK_REALTIME, &ts);
+  auto result = internal::clock_gettime(CLOCK_REALTIME, &ts);
 
   // A negative return value indicates an error with the magnitude of the
   // value being the error code.
@@ -34,7 +34,7 @@ LLVM_LIBC_FUNCTION(int, gettimeofday,
   }
 
   tv->tv_sec = ts.tv_sec;
-  tv->tv_usec = static_cast<suseconds_t>(ts.tv_nsec / 1000);
+  tv->tv_usec = static_cast<suseconds_t>(ts.tv_nsec / 1_us_ns);
   return 0;
 }
 
diff --git a/libc/src/time/linux/time.cpp b/libc/src/time/linux/time.cpp
index e286fae095b2..32f531efb6d1 100644
--- a/libc/src/time/linux/time.cpp
+++ b/libc/src/time/linux/time.cpp
@@ -6,22 +6,18 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/time/time_func.h"
-
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "hdr/time_macros.h"
 #include "src/__support/common.h"
+#include "src/__support/time/linux/clock_gettime.h"
 #include "src/errno/libc_errno.h"
-#include "src/time/linux/clockGetTimeImpl.h"
-
-#include <sys/syscall.h> // For syscall numbers.
-#include <time.h>
+#include "src/time/time_func.h"
 
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(time_t, time, (time_t * tp)) {
   // TODO: Use the Linux VDSO to fetch the time and avoid the syscall.
   struct timespec ts;
-  auto result = internal::clock_gettimeimpl(CLOCK_REALTIME, &ts);
+  auto result = internal::clock_gettime(CLOCK_REALTIME, &ts);
   if (!result.has_value()) {
     libc_errno = result.error();
     return -1;
diff --git a/libc/src/time/nanosleep.h b/libc/src/time/nanosleep.h
index 757394232c07..2309666b2304 100644
--- a/libc/src/time/nanosleep.h
+++ b/libc/src/time/nanosleep.h
@@ -9,11 +9,11 @@
 #ifndef LLVM_LIBC_SRC_TIME_NANOSLEEP_H
 #define LLVM_LIBC_SRC_TIME_NANOSLEEP_H
 
-#include <time.h>
+#include "hdr/types/struct_timespec.h"
 
 namespace LIBC_NAMESPACE {
 
-int nanosleep(const struct timespec *req, struct timespec *rem);
+int nanosleep(const timespec *req, timespec *rem);
 
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/src/time/time_func.h b/libc/src/time/time_func.h
index beb02020b575..2a5239220942 100644
--- a/libc/src/time/time_func.h
+++ b/libc/src/time/time_func.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_TIME_TIME_FUNC_H
 #define LLVM_LIBC_SRC_TIME_TIME_FUNC_H
 
-#include <time.h>
+#include "hdr/types/time_t.h"
 
 // Note this header file is named time_func.h to avoid conflicts with the
 // public header file time.h.
-- 
GitLab


From d2d08ea93b5a5958f46ed03e65ce1808e9fbbd0e Mon Sep 17 00:00:00 2001
From: agozillon <Andrew.Gozillon@amd.com>
Date: Fri, 10 May 2024 14:42:45 -0500
Subject: [PATCH 0445/1206] [Offload] Fix new fortran OpenMP offload test
 failures via addition of requires

---
 .../fortran/target-map-nested-dtype-complex-member.f90           | 1 +
 .../fortran/target-map-nested-dtype-derived-member.f90           | 1 +
 2 files changed, 2 insertions(+)

diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
index b9e8221357c8..9d29639ca101 100644
--- a/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
@@ -1,6 +1,7 @@
 ! Offloading test checking interaction of an
 ! nested derived type member map of a complex
 ! number member
+! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
 ! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
 ! UNSUPPORTED: aarch64-unknown-linux-gnu
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
index 4712ae6210d6..1be8cb32c8e6 100644
--- a/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
@@ -2,6 +2,7 @@
 ! nested derived type member map with the 
 ! inclusion of an entire nested derived 
 ! type being mapped
+! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
 ! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
 ! UNSUPPORTED: aarch64-unknown-linux-gnu
-- 
GitLab


From 91feb130d5cd3cafce94bbaf7ad67d1542623a75 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <r@artagnon.com>
Date: Fri, 10 May 2024 20:51:20 +0100
Subject: [PATCH 0446/1206] ISel/AArch64: custom lower vector ISD::[L]LRINT
 (#89035)

Since 98c90a1 (ISel: introduce vector ISD::LRINT, ISD::LLRINT; custom
RISCV lowering), ISD::LRINT and ISD::LLRINT now have vector variants,
that are custom lowered on RISCV, and scalarized on all other targets.
Since 2302e4c (Reland "VectorUtils: mark xrint as trivially
vectorizable"), lrint and llrint are trivially vectorizable, so all the
vectorizers in-tree will produce vector variants when possible. Add a
custom lowering for AArch64 to custom-lower the vector variants natively
using a combination of frintx, fcvte, and fcvtzs.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |   43 +-
 llvm/lib/Target/AArch64/AArch64ISelLowering.h |    1 +
 .../AArch64/sve-fixed-vector-llrint.ll        |  866 +++++++
 .../CodeGen/AArch64/sve-fixed-vector-lrint.ll | 1616 +++++++++++++
 llvm/test/CodeGen/AArch64/sve-llrint.ll       | 1763 +++++++++++++++
 llvm/test/CodeGen/AArch64/sve-lrint.ll        | 1764 +++++++++++++++
 llvm/test/CodeGen/AArch64/vector-llrint.ll    |  440 ++--
 llvm/test/CodeGen/AArch64/vector-lrint.ll     | 2006 ++++++++++++-----
 8 files changed, 7740 insertions(+), 759 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-llrint.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-lrint.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 4f2f170922ec..a6c020c6b823 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1304,6 +1304,15 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
           setOperationAction(Op, Ty, Legal);
     }
 
+    // LRINT and LLRINT.
+    for (auto Op : {ISD::LRINT, ISD::LLRINT}) {
+      for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64})
+        setOperationAction(Op, Ty, Custom);
+      if (Subtarget->hasFullFP16())
+        for (MVT Ty : {MVT::v4f16, MVT::v8f16})
+          setOperationAction(Op, Ty, Custom);
+    }
+
     setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
 
     setOperationAction(ISD::BITCAST, MVT::i2, Custom);
@@ -1525,6 +1534,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::FFLOOR, VT, Custom);
       setOperationAction(ISD::FNEARBYINT, VT, Custom);
       setOperationAction(ISD::FRINT, VT, Custom);
+      setOperationAction(ISD::LRINT, VT, Custom);
+      setOperationAction(ISD::LLRINT, VT, Custom);
       setOperationAction(ISD::FROUND, VT, Custom);
       setOperationAction(ISD::FROUNDEVEN, VT, Custom);
       setOperationAction(ISD::FTRUNC, VT, Custom);
@@ -1666,7 +1677,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
         setOperationAction(ISD::MULHU, VT, Custom);
       }
 
-
       // Use SVE for vectors with more than 2 elements.
       for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32})
         setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
@@ -1940,6 +1950,8 @@ void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
   setOperationAction(ISD::FP_TO_SINT, VT, Default);
   setOperationAction(ISD::FP_TO_UINT, VT, Default);
   setOperationAction(ISD::FRINT, VT, Default);
+  setOperationAction(ISD::LRINT, VT, Default);
+  setOperationAction(ISD::LLRINT, VT, Default);
   setOperationAction(ISD::FROUND, VT, Default);
   setOperationAction(ISD::FROUNDEVEN, VT, Default);
   setOperationAction(ISD::FSQRT, VT, Default);
@@ -4363,6 +4375,26 @@ SDValue AArch64TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
   return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Sat);
 }
 
+SDValue AArch64TargetLowering::LowerVectorXRINT(SDValue Op,
+                                                SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+  SDValue Src = Op.getOperand(0);
+  SDLoc DL(Op);
+
+  assert(VT.isVector() && "Expected vector type");
+
+  EVT CastVT =
+      VT.changeVectorElementType(Src.getValueType().getVectorElementType());
+
+  // Round the floating-point value into a floating-point register with the
+  // current rounding mode.
+  SDValue FOp = DAG.getNode(ISD::FRINT, DL, CastVT, Src);
+
+  // Truncate the rounded floating point to an integer.
+  return DAG.getNode(ISD::FP_TO_SINT_SAT, DL, VT, FOp,
+                     DAG.getValueType(VT.getVectorElementType()));
+}
+
 SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
                                                     SelectionDAG &DAG) const {
   // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
@@ -6686,10 +6718,13 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
     return LowerVECTOR_DEINTERLEAVE(Op, DAG);
   case ISD::VECTOR_INTERLEAVE:
     return LowerVECTOR_INTERLEAVE(Op, DAG);
-  case ISD::LROUND:
-  case ISD::LLROUND:
   case ISD::LRINT:
-  case ISD::LLRINT: {
+  case ISD::LLRINT:
+    if (Op.getValueType().isVector())
+      return LowerVectorXRINT(Op, DAG);
+    [[fallthrough]];
+  case ISD::LROUND:
+  case ISD::LLROUND: {
     assert((Op.getOperand(0).getValueType() == MVT::f16 ||
             Op.getOperand(0).getValueType() == MVT::bf16) &&
            "Expected custom lowering of rounding operations only for f16");
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index fbdc4de5617f..b3e282a04060 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -1165,6 +1165,7 @@ private:
   SDValue LowerVectorFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerVectorXRINT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll b/llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll
new file mode 100644
index 000000000000..9137eae269d9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll
@@ -0,0 +1,866 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve -aarch64-sve-vector-bits-min=256 | FileCheck %s
+
+define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
+; CHECK-LABEL: llrint_v1i64_v1f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx h0, h0
+; CHECK-NEXT:    fcvtzs x8, h0
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f16(<1 x half> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f16(<1 x half>)
+
+define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
+; CHECK-LABEL: llrint_v1i64_v2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov h1, v0.h[1]
+; CHECK-NEXT:    frintx h0, h0
+; CHECK-NEXT:    frintx h1, h1
+; CHECK-NEXT:    fcvtzs x8, h0
+; CHECK-NEXT:    fcvtzs x9, h1
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    ret
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half>)
+
+define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
+; CHECK-LABEL: llrint_v4i64_v4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx v0.4h, v0.4h
+; CHECK-NEXT:    mov h1, v0.h[2]
+; CHECK-NEXT:    mov h2, v0.h[3]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    fcvtzs x9, h0
+; CHECK-NEXT:    fcvtzs x8, h1
+; CHECK-NEXT:    fcvtzs x10, h2
+; CHECK-NEXT:    fcvtzs x11, h3
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    mov v0.d[1], x11
+; CHECK-NEXT:    mov v1.d[1], x10
+; CHECK-NEXT:    ret
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half>)
+
+define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
+; CHECK-LABEL: llrint_v8i64_v8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    frintx v0.4h, v0.4h
+; CHECK-NEXT:    frintx v1.4h, v1.4h
+; CHECK-NEXT:    mov h4, v0.h[2]
+; CHECK-NEXT:    mov h2, v0.h[1]
+; CHECK-NEXT:    mov h7, v0.h[3]
+; CHECK-NEXT:    fcvtzs x8, h0
+; CHECK-NEXT:    mov h3, v1.h[2]
+; CHECK-NEXT:    mov h5, v1.h[3]
+; CHECK-NEXT:    mov h6, v1.h[1]
+; CHECK-NEXT:    fcvtzs x11, h1
+; CHECK-NEXT:    fcvtzs x12, h4
+; CHECK-NEXT:    fcvtzs x9, h2
+; CHECK-NEXT:    fcvtzs x15, h7
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fcvtzs x10, h3
+; CHECK-NEXT:    fcvtzs x13, h5
+; CHECK-NEXT:    fcvtzs x14, h6
+; CHECK-NEXT:    fmov d1, x12
+; CHECK-NEXT:    fmov d2, x11
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    fmov d3, x10
+; CHECK-NEXT:    mov v1.d[1], x15
+; CHECK-NEXT:    mov v2.d[1], x14
+; CHECK-NEXT:    mov v3.d[1], x13
+; CHECK-NEXT:    ret
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half>)
+
+define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
+; CHECK-LABEL: llrint_v16i64_v16f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    frintx v1.4h, v1.4h
+; CHECK-NEXT:    frintx v3.4h, v0.4h
+; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    frintx v2.4h, v2.4h
+; CHECK-NEXT:    mov h4, v1.h[2]
+; CHECK-NEXT:    mov h5, v3.h[2]
+; CHECK-NEXT:    frintx v0.4h, v0.4h
+; CHECK-NEXT:    mov h6, v3.h[1]
+; CHECK-NEXT:    fcvtzs x9, h3
+; CHECK-NEXT:    mov h16, v1.h[1]
+; CHECK-NEXT:    fcvtzs x12, h1
+; CHECK-NEXT:    mov h3, v3.h[3]
+; CHECK-NEXT:    mov h17, v1.h[3]
+; CHECK-NEXT:    mov h7, v2.h[3]
+; CHECK-NEXT:    fcvtzs x8, h4
+; CHECK-NEXT:    fcvtzs x10, h5
+; CHECK-NEXT:    mov h4, v2.h[2]
+; CHECK-NEXT:    mov h5, v0.h[2]
+; CHECK-NEXT:    fcvtzs x11, h6
+; CHECK-NEXT:    mov h6, v0.h[3]
+; CHECK-NEXT:    fcvtzs x15, h2
+; CHECK-NEXT:    mov h2, v2.h[1]
+; CHECK-NEXT:    fcvtzs x14, h0
+; CHECK-NEXT:    fcvtzs x17, h3
+; CHECK-NEXT:    fcvtzs x0, h17
+; CHECK-NEXT:    fcvtzs x13, h7
+; CHECK-NEXT:    mov h7, v0.h[1]
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    fcvtzs x16, h4
+; CHECK-NEXT:    fcvtzs x9, h5
+; CHECK-NEXT:    fmov d4, x12
+; CHECK-NEXT:    fcvtzs x12, h16
+; CHECK-NEXT:    fmov d1, x10
+; CHECK-NEXT:    fcvtzs x10, h6
+; CHECK-NEXT:    fmov d5, x8
+; CHECK-NEXT:    fcvtzs x8, h2
+; CHECK-NEXT:    fmov d2, x14
+; CHECK-NEXT:    fcvtzs x18, h7
+; CHECK-NEXT:    fmov d6, x15
+; CHECK-NEXT:    mov v0.d[1], x11
+; CHECK-NEXT:    fmov d3, x9
+; CHECK-NEXT:    fmov d7, x16
+; CHECK-NEXT:    mov v1.d[1], x17
+; CHECK-NEXT:    mov v4.d[1], x12
+; CHECK-NEXT:    mov v5.d[1], x0
+; CHECK-NEXT:    mov v6.d[1], x8
+; CHECK-NEXT:    mov v2.d[1], x18
+; CHECK-NEXT:    mov v3.d[1], x10
+; CHECK-NEXT:    mov v7.d[1], x13
+; CHECK-NEXT:    ret
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half>)
+
+define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
+; CHECK-LABEL: llrint_v32i64_v32f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    sub x9, sp, #272
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
+; CHECK-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    frintx v5.4h, v0.4h
+; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    ext v17.16b, v2.16b, v2.16b, #8
+; CHECK-NEXT:    frintx v1.4h, v1.4h
+; CHECK-NEXT:    frintx v2.4h, v2.4h
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    mov h6, v5.h[3]
+; CHECK-NEXT:    frintx v0.4h, v0.4h
+; CHECK-NEXT:    mov h7, v5.h[2]
+; CHECK-NEXT:    mov h16, v5.h[1]
+; CHECK-NEXT:    frintx v4.4h, v4.4h
+; CHECK-NEXT:    fcvtzs x12, h5
+; CHECK-NEXT:    ext v5.16b, v3.16b, v3.16b, #8
+; CHECK-NEXT:    frintx v17.4h, v17.4h
+; CHECK-NEXT:    frintx v3.4h, v3.4h
+; CHECK-NEXT:    fcvtzs x9, h6
+; CHECK-NEXT:    mov h6, v0.h[3]
+; CHECK-NEXT:    fcvtzs x10, h7
+; CHECK-NEXT:    mov h7, v0.h[2]
+; CHECK-NEXT:    fcvtzs x11, h16
+; CHECK-NEXT:    mov h16, v0.h[1]
+; CHECK-NEXT:    fcvtzs x13, h6
+; CHECK-NEXT:    mov h6, v4.h[3]
+; CHECK-NEXT:    stp x10, x9, [sp, #48]
+; CHECK-NEXT:    fcvtzs x9, h7
+; CHECK-NEXT:    mov h7, v4.h[2]
+; CHECK-NEXT:    fcvtzs x10, h16
+; CHECK-NEXT:    mov h16, v4.h[1]
+; CHECK-NEXT:    stp x12, x11, [sp, #32]
+; CHECK-NEXT:    fcvtzs x11, h0
+; CHECK-NEXT:    frintx v0.4h, v5.4h
+; CHECK-NEXT:    mov h5, v17.h[3]
+; CHECK-NEXT:    fcvtzs x12, h6
+; CHECK-NEXT:    mov h6, v17.h[2]
+; CHECK-NEXT:    stp x9, x13, [sp, #16]
+; CHECK-NEXT:    fcvtzs x13, h7
+; CHECK-NEXT:    mov h7, v17.h[1]
+; CHECK-NEXT:    fcvtzs x9, h16
+; CHECK-NEXT:    stp x11, x10, [sp]
+; CHECK-NEXT:    fcvtzs x10, h4
+; CHECK-NEXT:    fcvtzs x11, h5
+; CHECK-NEXT:    mov h4, v0.h[3]
+; CHECK-NEXT:    mov h5, v0.h[2]
+; CHECK-NEXT:    stp x13, x12, [sp, #80]
+; CHECK-NEXT:    fcvtzs x12, h6
+; CHECK-NEXT:    fcvtzs x13, h7
+; CHECK-NEXT:    mov h6, v0.h[1]
+; CHECK-NEXT:    stp x10, x9, [sp, #64]
+; CHECK-NEXT:    fcvtzs x9, h17
+; CHECK-NEXT:    mov h7, v1.h[3]
+; CHECK-NEXT:    fcvtzs x10, h4
+; CHECK-NEXT:    mov h4, v1.h[2]
+; CHECK-NEXT:    stp x12, x11, [sp, #144]
+; CHECK-NEXT:    fcvtzs x11, h5
+; CHECK-NEXT:    mov h5, v1.h[1]
+; CHECK-NEXT:    fcvtzs x12, h6
+; CHECK-NEXT:    stp x9, x13, [sp, #128]
+; CHECK-NEXT:    fcvtzs x9, h0
+; CHECK-NEXT:    fcvtzs x13, h7
+; CHECK-NEXT:    mov h0, v2.h[3]
+; CHECK-NEXT:    stp x11, x10, [sp, #208]
+; CHECK-NEXT:    fcvtzs x10, h4
+; CHECK-NEXT:    mov h4, v2.h[2]
+; CHECK-NEXT:    fcvtzs x11, h5
+; CHECK-NEXT:    mov h5, v2.h[1]
+; CHECK-NEXT:    stp x9, x12, [sp, #192]
+; CHECK-NEXT:    fcvtzs x9, h1
+; CHECK-NEXT:    fcvtzs x12, h0
+; CHECK-NEXT:    mov h0, v3.h[3]
+; CHECK-NEXT:    mov h1, v3.h[2]
+; CHECK-NEXT:    stp x10, x13, [sp, #112]
+; CHECK-NEXT:    fcvtzs x10, h4
+; CHECK-NEXT:    mov h4, v3.h[1]
+; CHECK-NEXT:    fcvtzs x13, h5
+; CHECK-NEXT:    stp x9, x11, [sp, #96]
+; CHECK-NEXT:    fcvtzs x9, h2
+; CHECK-NEXT:    fcvtzs x11, h0
+; CHECK-NEXT:    stp x10, x12, [sp, #176]
+; CHECK-NEXT:    fcvtzs x10, h1
+; CHECK-NEXT:    fcvtzs x12, h4
+; CHECK-NEXT:    stp x9, x13, [sp, #160]
+; CHECK-NEXT:    fcvtzs x9, h3
+; CHECK-NEXT:    stp x10, x11, [sp, #240]
+; CHECK-NEXT:    add x10, sp, #64
+; CHECK-NEXT:    stp x9, x12, [sp, #224]
+; CHECK-NEXT:    add x9, sp, #32
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x9]
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x10]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x9]
+; CHECK-NEXT:    add x9, sp, #224
+; CHECK-NEXT:    add x10, sp, #128
+; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x9]
+; CHECK-NEXT:    add x9, sp, #160
+; CHECK-NEXT:    ld1d { z4.d }, p0/z, [x10]
+; CHECK-NEXT:    add x10, sp, #96
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x9]
+; CHECK-NEXT:    add x9, sp, #192
+; CHECK-NEXT:    ld1d { z6.d }, p0/z, [x10]
+; CHECK-NEXT:    mov x10, #24 // =0x18
+; CHECK-NEXT:    ld1d { z7.d }, p0/z, [x9]
+; CHECK-NEXT:    mov x9, #16 // =0x10
+; CHECK-NEXT:    st1d { z3.d }, p0, [x8, x10, lsl #3]
+; CHECK-NEXT:    st1d { z5.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #8 // =0x8
+; CHECK-NEXT:    st1d { z6.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #28 // =0x1c
+; CHECK-NEXT:    st1d { z7.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #20 // =0x14
+; CHECK-NEXT:    st1d { z4.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #12 // =0xc
+; CHECK-NEXT:    st1d { z2.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #4 // =0x4
+; CHECK-NEXT:    st1d { z1.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <32 x i64> @llvm.llrint.v32i64.v32f16(<32 x half> %x)
+  ret <32 x i64> %a
+}
+declare <32 x i64> @llvm.llrint.v32i64.v32f16(<32 x half>)
+
+define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
+; CHECK-LABEL: llrint_v1i64_v1f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
+
+define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
+; CHECK-LABEL: llrint_v2i64_v2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx v0.2s, v0.2s
+; CHECK-NEXT:    mov s1, v0.s[1]
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fcvtzs x9, s1
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    ret
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
+
+define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
+; CHECK-LABEL: llrint_v4i64_v4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx v0.4s, v0.4s
+; CHECK-NEXT:    mov s1, v0.s[2]
+; CHECK-NEXT:    mov s2, v0.s[3]
+; CHECK-NEXT:    mov s3, v0.s[1]
+; CHECK-NEXT:    fcvtzs x9, s0
+; CHECK-NEXT:    fcvtzs x8, s1
+; CHECK-NEXT:    fcvtzs x10, s2
+; CHECK-NEXT:    fcvtzs x11, s3
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    mov v0.d[1], x11
+; CHECK-NEXT:    mov v1.d[1], x10
+; CHECK-NEXT:    ret
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
+
+define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
+; CHECK-LABEL: llrint_v8i64_v8f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx v0.4s, v0.4s
+; CHECK-NEXT:    frintx v1.4s, v1.4s
+; CHECK-NEXT:    mov s3, v1.s[2]
+; CHECK-NEXT:    mov s4, v0.s[2]
+; CHECK-NEXT:    mov s2, v0.s[1]
+; CHECK-NEXT:    mov s5, v1.s[3]
+; CHECK-NEXT:    mov s6, v1.s[1]
+; CHECK-NEXT:    mov s7, v0.s[3]
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fcvtzs x10, s1
+; CHECK-NEXT:    fcvtzs x11, s3
+; CHECK-NEXT:    fcvtzs x12, s4
+; CHECK-NEXT:    fcvtzs x9, s2
+; CHECK-NEXT:    fcvtzs x13, s5
+; CHECK-NEXT:    fcvtzs x14, s6
+; CHECK-NEXT:    fcvtzs x15, s7
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d2, x10
+; CHECK-NEXT:    fmov d1, x12
+; CHECK-NEXT:    fmov d3, x11
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    mov v2.d[1], x14
+; CHECK-NEXT:    mov v1.d[1], x15
+; CHECK-NEXT:    mov v3.d[1], x13
+; CHECK-NEXT:    ret
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
+
+define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
+; CHECK-LABEL: llrint_v16i64_v16f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx v3.4s, v3.4s
+; CHECK-NEXT:    frintx v2.4s, v2.4s
+; CHECK-NEXT:    frintx v1.4s, v1.4s
+; CHECK-NEXT:    frintx v0.4s, v0.4s
+; CHECK-NEXT:    mov s4, v3.s[2]
+; CHECK-NEXT:    mov s5, v2.s[2]
+; CHECK-NEXT:    mov s6, v1.s[2]
+; CHECK-NEXT:    mov s7, v0.s[2]
+; CHECK-NEXT:    fcvtzs x10, s1
+; CHECK-NEXT:    fcvtzs x11, s0
+; CHECK-NEXT:    mov s16, v0.s[1]
+; CHECK-NEXT:    mov s17, v1.s[1]
+; CHECK-NEXT:    mov s18, v3.s[1]
+; CHECK-NEXT:    fcvtzs x14, s3
+; CHECK-NEXT:    fcvtzs x16, s2
+; CHECK-NEXT:    fcvtzs x8, s4
+; CHECK-NEXT:    mov s4, v2.s[1]
+; CHECK-NEXT:    fcvtzs x9, s5
+; CHECK-NEXT:    mov s5, v1.s[3]
+; CHECK-NEXT:    fcvtzs x12, s6
+; CHECK-NEXT:    mov s6, v0.s[3]
+; CHECK-NEXT:    fcvtzs x13, s7
+; CHECK-NEXT:    mov s7, v3.s[3]
+; CHECK-NEXT:    fmov d0, x11
+; CHECK-NEXT:    fcvtzs x17, s16
+; CHECK-NEXT:    fcvtzs x18, s18
+; CHECK-NEXT:    fcvtzs x15, s4
+; CHECK-NEXT:    mov s4, v2.s[3]
+; CHECK-NEXT:    fmov d2, x10
+; CHECK-NEXT:    fcvtzs x11, s5
+; CHECK-NEXT:    fcvtzs x10, s6
+; CHECK-NEXT:    fmov d3, x12
+; CHECK-NEXT:    fmov d1, x13
+; CHECK-NEXT:    fcvtzs x12, s17
+; CHECK-NEXT:    fcvtzs x13, s7
+; CHECK-NEXT:    fmov d5, x9
+; CHECK-NEXT:    fmov d6, x14
+; CHECK-NEXT:    fmov d7, x8
+; CHECK-NEXT:    fcvtzs x0, s4
+; CHECK-NEXT:    fmov d4, x16
+; CHECK-NEXT:    mov v0.d[1], x17
+; CHECK-NEXT:    mov v1.d[1], x10
+; CHECK-NEXT:    mov v3.d[1], x11
+; CHECK-NEXT:    mov v2.d[1], x12
+; CHECK-NEXT:    mov v6.d[1], x18
+; CHECK-NEXT:    mov v7.d[1], x13
+; CHECK-NEXT:    mov v4.d[1], x15
+; CHECK-NEXT:    mov v5.d[1], x0
+; CHECK-NEXT:    ret
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
+
+define <32 x i64> @llrint_v32i64_v32f32(<32 x float> %x) {
+; CHECK-LABEL: llrint_v32i64_v32f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    sub x9, sp, #272
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
+; CHECK-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    frintx v0.4s, v0.4s
+; CHECK-NEXT:    frintx v1.4s, v1.4s
+; CHECK-NEXT:    frintx v2.4s, v2.4s
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    mov s16, v0.s[3]
+; CHECK-NEXT:    mov s17, v0.s[2]
+; CHECK-NEXT:    mov s18, v0.s[1]
+; CHECK-NEXT:    fcvtzs x12, s0
+; CHECK-NEXT:    frintx v0.4s, v3.4s
+; CHECK-NEXT:    mov s3, v2.s[3]
+; CHECK-NEXT:    fcvtzs x9, s16
+; CHECK-NEXT:    mov s16, v1.s[3]
+; CHECK-NEXT:    fcvtzs x10, s17
+; CHECK-NEXT:    mov s17, v1.s[2]
+; CHECK-NEXT:    fcvtzs x11, s18
+; CHECK-NEXT:    mov s18, v1.s[1]
+; CHECK-NEXT:    fcvtzs x13, s16
+; CHECK-NEXT:    stp x10, x9, [sp, #16]
+; CHECK-NEXT:    mov s16, v2.s[2]
+; CHECK-NEXT:    fcvtzs x9, s17
+; CHECK-NEXT:    fcvtzs x10, s18
+; CHECK-NEXT:    mov s17, v2.s[1]
+; CHECK-NEXT:    stp x12, x11, [sp]
+; CHECK-NEXT:    fcvtzs x11, s1
+; CHECK-NEXT:    frintx v1.4s, v4.4s
+; CHECK-NEXT:    fcvtzs x12, s3
+; CHECK-NEXT:    mov s3, v0.s[3]
+; CHECK-NEXT:    mov s4, v0.s[2]
+; CHECK-NEXT:    stp x9, x13, [sp, #48]
+; CHECK-NEXT:    fcvtzs x13, s16
+; CHECK-NEXT:    fcvtzs x9, s17
+; CHECK-NEXT:    mov s16, v0.s[1]
+; CHECK-NEXT:    stp x11, x10, [sp, #32]
+; CHECK-NEXT:    fcvtzs x10, s2
+; CHECK-NEXT:    frintx v2.4s, v5.4s
+; CHECK-NEXT:    fcvtzs x11, s3
+; CHECK-NEXT:    mov s3, v1.s[3]
+; CHECK-NEXT:    mov s5, v1.s[1]
+; CHECK-NEXT:    stp x13, x12, [sp, #80]
+; CHECK-NEXT:    fcvtzs x12, s4
+; CHECK-NEXT:    mov s4, v1.s[2]
+; CHECK-NEXT:    fcvtzs x13, s16
+; CHECK-NEXT:    stp x10, x9, [sp, #64]
+; CHECK-NEXT:    fcvtzs x9, s0
+; CHECK-NEXT:    mov s0, v2.s[3]
+; CHECK-NEXT:    fcvtzs x10, s3
+; CHECK-NEXT:    frintx v3.4s, v6.4s
+; CHECK-NEXT:    stp x12, x11, [sp, #112]
+; CHECK-NEXT:    fcvtzs x11, s4
+; CHECK-NEXT:    mov s4, v2.s[2]
+; CHECK-NEXT:    fcvtzs x12, s5
+; CHECK-NEXT:    mov s5, v2.s[1]
+; CHECK-NEXT:    stp x9, x13, [sp, #96]
+; CHECK-NEXT:    fcvtzs x9, s1
+; CHECK-NEXT:    fcvtzs x13, s0
+; CHECK-NEXT:    mov s0, v3.s[3]
+; CHECK-NEXT:    frintx v1.4s, v7.4s
+; CHECK-NEXT:    stp x11, x10, [sp, #144]
+; CHECK-NEXT:    fcvtzs x10, s4
+; CHECK-NEXT:    mov s4, v3.s[2]
+; CHECK-NEXT:    fcvtzs x11, s5
+; CHECK-NEXT:    mov s5, v3.s[1]
+; CHECK-NEXT:    stp x9, x12, [sp, #128]
+; CHECK-NEXT:    fcvtzs x9, s2
+; CHECK-NEXT:    fcvtzs x12, s0
+; CHECK-NEXT:    mov s0, v1.s[3]
+; CHECK-NEXT:    mov s2, v1.s[2]
+; CHECK-NEXT:    stp x10, x13, [sp, #176]
+; CHECK-NEXT:    fcvtzs x10, s4
+; CHECK-NEXT:    mov s4, v1.s[1]
+; CHECK-NEXT:    fcvtzs x13, s5
+; CHECK-NEXT:    stp x9, x11, [sp, #160]
+; CHECK-NEXT:    fcvtzs x9, s3
+; CHECK-NEXT:    fcvtzs x11, s0
+; CHECK-NEXT:    stp x10, x12, [sp, #208]
+; CHECK-NEXT:    fcvtzs x10, s2
+; CHECK-NEXT:    fcvtzs x12, s4
+; CHECK-NEXT:    stp x9, x13, [sp, #192]
+; CHECK-NEXT:    fcvtzs x9, s1
+; CHECK-NEXT:    stp x10, x11, [sp, #240]
+; CHECK-NEXT:    add x10, sp, #64
+; CHECK-NEXT:    stp x9, x12, [sp, #224]
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x9]
+; CHECK-NEXT:    add x9, sp, #32
+; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x10]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x9]
+; CHECK-NEXT:    add x9, sp, #224
+; CHECK-NEXT:    add x10, sp, #96
+; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x9]
+; CHECK-NEXT:    add x9, sp, #192
+; CHECK-NEXT:    ld1d { z4.d }, p0/z, [x10]
+; CHECK-NEXT:    add x10, sp, #160
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x9]
+; CHECK-NEXT:    add x9, sp, #128
+; CHECK-NEXT:    ld1d { z6.d }, p0/z, [x10]
+; CHECK-NEXT:    mov x10, #28 // =0x1c
+; CHECK-NEXT:    ld1d { z7.d }, p0/z, [x9]
+; CHECK-NEXT:    mov x9, #24 // =0x18
+; CHECK-NEXT:    st1d { z3.d }, p0, [x8, x10, lsl #3]
+; CHECK-NEXT:    st1d { z5.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #20 // =0x14
+; CHECK-NEXT:    st1d { z6.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #16 // =0x10
+; CHECK-NEXT:    st1d { z7.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #12 // =0xc
+; CHECK-NEXT:    st1d { z4.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #8 // =0x8
+; CHECK-NEXT:    st1d { z2.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #4 // =0x4
+; CHECK-NEXT:    st1d { z1.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <32 x i64> @llvm.llrint.v32i64.v32f32(<32 x float> %x)
+  ret <32 x i64> %a
+}
+declare <32 x i64> @llvm.llrint.v32i64.v32f32(<32 x float>)
+
+define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
+; CHECK-LABEL: llrint_v1i64_v1f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx d0, d0
+; CHECK-NEXT:    fcvtzs x8, d0
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
+
+define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
+; CHECK-LABEL: llrint_v2i64_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    ret
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
+
+define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
+; CHECK-LABEL: llrint_v4i64_v4f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    mov z1.d, z0.d[2]
+; CHECK-NEXT:    mov z2.d, z0.d[3]
+; CHECK-NEXT:    mov z3.d, z0.d[1]
+; CHECK-NEXT:    fcvtzs x9, d0
+; CHECK-NEXT:    fcvtzs x8, d1
+; CHECK-NEXT:    fcvtzs x10, d2
+; CHECK-NEXT:    fcvtzs x11, d3
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    mov v0.d[1], x11
+; CHECK-NEXT:    mov v1.d[1], x10
+; CHECK-NEXT:    ret
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
+
+define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
+; CHECK-LABEL: llrint_v8i64_v8f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    splice z2.d, p0, z2.d, z3.d
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z1, z2
+; CHECK-NEXT:    frintx z1.d, p0/m, z2.d
+; CHECK-NEXT:    mov z4.d, z1.d[2]
+; CHECK-NEXT:    mov z5.d, z0.d[2]
+; CHECK-NEXT:    mov z2.d, z0.d[1]
+; CHECK-NEXT:    mov z3.d, z1.d[3]
+; CHECK-NEXT:    mov z6.d, z0.d[3]
+; CHECK-NEXT:    fcvtzs x8, d0
+; CHECK-NEXT:    mov z0.d, z1.d[1]
+; CHECK-NEXT:    fcvtzs x10, d1
+; CHECK-NEXT:    fcvtzs x11, d4
+; CHECK-NEXT:    fcvtzs x12, d5
+; CHECK-NEXT:    fcvtzs x9, d2
+; CHECK-NEXT:    fcvtzs x13, d3
+; CHECK-NEXT:    fcvtzs x14, d6
+; CHECK-NEXT:    fcvtzs x15, d0
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d2, x10
+; CHECK-NEXT:    fmov d1, x12
+; CHECK-NEXT:    fmov d3, x11
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    mov v2.d[1], x15
+; CHECK-NEXT:    mov v1.d[1], x14
+; CHECK-NEXT:    mov v3.d[1], x13
+; CHECK-NEXT:    ret
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double>)
+
+define <16 x i64> @llrint_v16f64(<16 x double> %x) {
+; CHECK-LABEL: llrint_v16f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d, vl2
+; CHECK-NEXT:    // kill: def $q6 killed $q6 def $z6
+; CHECK-NEXT:    // kill: def $q4 killed $q4 def $z4
+; CHECK-NEXT:    // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT:    // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    splice z6.d, p1, z6.d, z7.d
+; CHECK-NEXT:    splice z4.d, p1, z4.d, z5.d
+; CHECK-NEXT:    splice z2.d, p1, z2.d, z3.d
+; CHECK-NEXT:    splice z0.d, p1, z0.d, z1.d
+; CHECK-NEXT:    movprfx z3, z6
+; CHECK-NEXT:    frintx z3.d, p0/m, z6.d
+; CHECK-NEXT:    movprfx z1, z4
+; CHECK-NEXT:    frintx z1.d, p0/m, z4.d
+; CHECK-NEXT:    frintx z2.d, p0/m, z2.d
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    mov z4.d, z3.d[2]
+; CHECK-NEXT:    mov z5.d, z1.d[2]
+; CHECK-NEXT:    mov z6.d, z2.d[3]
+; CHECK-NEXT:    fcvtzs x11, d0
+; CHECK-NEXT:    fcvtzs x12, d1
+; CHECK-NEXT:    fcvtzs x13, d2
+; CHECK-NEXT:    fcvtzs x14, d3
+; CHECK-NEXT:    mov z7.d, z3.d[3]
+; CHECK-NEXT:    mov z16.d, z1.d[3]
+; CHECK-NEXT:    fcvtzs x9, d4
+; CHECK-NEXT:    fcvtzs x10, d5
+; CHECK-NEXT:    mov z4.d, z2.d[2]
+; CHECK-NEXT:    mov z5.d, z0.d[2]
+; CHECK-NEXT:    fcvtzs x8, d6
+; CHECK-NEXT:    mov z2.d, z2.d[1]
+; CHECK-NEXT:    mov z6.d, z0.d[3]
+; CHECK-NEXT:    mov z1.d, z1.d[1]
+; CHECK-NEXT:    mov z3.d, z3.d[1]
+; CHECK-NEXT:    fcvtzs x15, d4
+; CHECK-NEXT:    mov z4.d, z0.d[1]
+; CHECK-NEXT:    fmov d0, x11
+; CHECK-NEXT:    fcvtzs x16, d5
+; CHECK-NEXT:    fcvtzs x11, d2
+; CHECK-NEXT:    fmov d2, x13
+; CHECK-NEXT:    fcvtzs x17, d7
+; CHECK-NEXT:    fcvtzs x18, d16
+; CHECK-NEXT:    fcvtzs x0, d3
+; CHECK-NEXT:    fcvtzs x13, d4
+; CHECK-NEXT:    fmov d4, x12
+; CHECK-NEXT:    fcvtzs x12, d6
+; CHECK-NEXT:    fmov d6, x14
+; CHECK-NEXT:    fcvtzs x14, d1
+; CHECK-NEXT:    fmov d3, x15
+; CHECK-NEXT:    fmov d1, x16
+; CHECK-NEXT:    fmov d5, x10
+; CHECK-NEXT:    fmov d7, x9
+; CHECK-NEXT:    mov v2.d[1], x11
+; CHECK-NEXT:    mov v0.d[1], x13
+; CHECK-NEXT:    mov v3.d[1], x8
+; CHECK-NEXT:    mov v6.d[1], x0
+; CHECK-NEXT:    mov v4.d[1], x14
+; CHECK-NEXT:    mov v1.d[1], x12
+; CHECK-NEXT:    mov v5.d[1], x18
+; CHECK-NEXT:    mov v7.d[1], x17
+; CHECK-NEXT:    ret
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double>)
+
+define <32 x i64> @llrint_v32f64(<32 x double> %x) {
+; CHECK-LABEL: llrint_v32f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    sub x9, sp, #272
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
+; CHECK-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    ptrue p1.d, vl2
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT:    // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT:    // kill: def $q6 killed $q6 def $z6
+; CHECK-NEXT:    // kill: def $q4 killed $q4 def $z4
+; CHECK-NEXT:    // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    splice z0.d, p1, z0.d, z1.d
+; CHECK-NEXT:    splice z2.d, p1, z2.d, z3.d
+; CHECK-NEXT:    splice z4.d, p1, z4.d, z5.d
+; CHECK-NEXT:    splice z6.d, p1, z6.d, z7.d
+; CHECK-NEXT:    ldp q5, q19, [x29, #16]
+; CHECK-NEXT:    movprfx z3, z0
+; CHECK-NEXT:    frintx z3.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z16, z2
+; CHECK-NEXT:    frintx z16.d, p0/m, z2.d
+; CHECK-NEXT:    frintx z4.d, p0/m, z4.d
+; CHECK-NEXT:    splice z5.d, p1, z5.d, z19.d
+; CHECK-NEXT:    frintx z6.d, p0/m, z6.d
+; CHECK-NEXT:    ldp q2, q17, [x29, #48]
+; CHECK-NEXT:    ldp q0, q1, [x29, #112]
+; CHECK-NEXT:    mov z18.d, z3.d[3]
+; CHECK-NEXT:    mov z7.d, z3.d[2]
+; CHECK-NEXT:    fcvtzs x9, d3
+; CHECK-NEXT:    mov z3.d, z3.d[1]
+; CHECK-NEXT:    mov z20.d, z16.d[3]
+; CHECK-NEXT:    fcvtzs x12, d16
+; CHECK-NEXT:    splice z2.d, p1, z2.d, z17.d
+; CHECK-NEXT:    frintx z5.d, p0/m, z5.d
+; CHECK-NEXT:    splice z0.d, p1, z0.d, z1.d
+; CHECK-NEXT:    fcvtzs x10, d18
+; CHECK-NEXT:    fcvtzs x11, d7
+; CHECK-NEXT:    mov z18.d, z16.d[2]
+; CHECK-NEXT:    mov z7.d, z16.d[1]
+; CHECK-NEXT:    fcvtzs x13, d3
+; CHECK-NEXT:    fcvtzs x14, d20
+; CHECK-NEXT:    str x9, [sp, #128]
+; CHECK-NEXT:    mov z16.d, z4.d[3]
+; CHECK-NEXT:    fcvtzs x9, d18
+; CHECK-NEXT:    mov z18.d, z4.d[2]
+; CHECK-NEXT:    frintx z2.d, p0/m, z2.d
+; CHECK-NEXT:    stp x11, x10, [sp, #144]
+; CHECK-NEXT:    fcvtzs x10, d7
+; CHECK-NEXT:    mov z7.d, z4.d[1]
+; CHECK-NEXT:    str x13, [sp, #136]
+; CHECK-NEXT:    fcvtzs x11, d16
+; CHECK-NEXT:    mov z16.d, z6.d[3]
+; CHECK-NEXT:    fcvtzs x13, d18
+; CHECK-NEXT:    ldp q3, q19, [x29, #80]
+; CHECK-NEXT:    stp x9, x14, [sp, #176]
+; CHECK-NEXT:    fcvtzs x9, d4
+; CHECK-NEXT:    mov z4.d, z6.d[2]
+; CHECK-NEXT:    stp x12, x10, [sp, #160]
+; CHECK-NEXT:    fcvtzs x10, d7
+; CHECK-NEXT:    mov z7.d, z6.d[1]
+; CHECK-NEXT:    fcvtzs x12, d6
+; CHECK-NEXT:    splice z3.d, p1, z3.d, z19.d
+; CHECK-NEXT:    mov z6.d, z5.d[2]
+; CHECK-NEXT:    stp x13, x11, [sp, #208]
+; CHECK-NEXT:    fcvtzs x11, d16
+; CHECK-NEXT:    fcvtzs x13, d4
+; CHECK-NEXT:    mov z4.d, z5.d[3]
+; CHECK-NEXT:    mov z1.d, z5.d[1]
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    stp x9, x10, [sp, #192]
+; CHECK-NEXT:    fcvtzs x9, d7
+; CHECK-NEXT:    frintx z3.d, p0/m, z3.d
+; CHECK-NEXT:    fcvtzs x10, d4
+; CHECK-NEXT:    stp x13, x11, [sp, #240]
+; CHECK-NEXT:    fcvtzs x11, d6
+; CHECK-NEXT:    mov z4.d, z2.d[3]
+; CHECK-NEXT:    fcvtzs x13, d2
+; CHECK-NEXT:    stp x12, x9, [sp, #224]
+; CHECK-NEXT:    fcvtzs x9, d5
+; CHECK-NEXT:    fcvtzs x12, d1
+; CHECK-NEXT:    mov z5.d, z2.d[2]
+; CHECK-NEXT:    mov z1.d, z2.d[1]
+; CHECK-NEXT:    mov z2.d, z3.d[2]
+; CHECK-NEXT:    stp x11, x10, [sp, #16]
+; CHECK-NEXT:    fcvtzs x10, d4
+; CHECK-NEXT:    mov z4.d, z3.d[3]
+; CHECK-NEXT:    fcvtzs x11, d5
+; CHECK-NEXT:    stp x9, x12, [sp]
+; CHECK-NEXT:    fcvtzs x9, d1
+; CHECK-NEXT:    mov z1.d, z3.d[1]
+; CHECK-NEXT:    fcvtzs x12, d4
+; CHECK-NEXT:    stp x11, x10, [sp, #48]
+; CHECK-NEXT:    fcvtzs x10, d2
+; CHECK-NEXT:    fcvtzs x11, d3
+; CHECK-NEXT:    stp x13, x9, [sp, #32]
+; CHECK-NEXT:    fcvtzs x9, d1
+; CHECK-NEXT:    mov z2.d, z0.d[3]
+; CHECK-NEXT:    mov z3.d, z0.d[2]
+; CHECK-NEXT:    mov z1.d, z0.d[1]
+; CHECK-NEXT:    fcvtzs x13, d2
+; CHECK-NEXT:    stp x10, x12, [sp, #80]
+; CHECK-NEXT:    fcvtzs x12, d0
+; CHECK-NEXT:    fcvtzs x10, d3
+; CHECK-NEXT:    stp x11, x9, [sp, #64]
+; CHECK-NEXT:    fcvtzs x9, d1
+; CHECK-NEXT:    stp x10, x13, [sp, #112]
+; CHECK-NEXT:    add x10, sp, #192
+; CHECK-NEXT:    stp x12, x9, [sp, #96]
+; CHECK-NEXT:    add x9, sp, #128
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x9]
+; CHECK-NEXT:    add x9, sp, #160
+; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x10]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x9]
+; CHECK-NEXT:    add x9, sp, #96
+; CHECK-NEXT:    add x10, sp, #224
+; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x9]
+; CHECK-NEXT:    add x9, sp, #64
+; CHECK-NEXT:    ld1d { z4.d }, p0/z, [x10]
+; CHECK-NEXT:    add x10, sp, #32
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x9]
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    ld1d { z6.d }, p0/z, [x10]
+; CHECK-NEXT:    mov x10, #28 // =0x1c
+; CHECK-NEXT:    ld1d { z7.d }, p0/z, [x9]
+; CHECK-NEXT:    mov x9, #24 // =0x18
+; CHECK-NEXT:    st1d { z3.d }, p0, [x8, x10, lsl #3]
+; CHECK-NEXT:    st1d { z5.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #20 // =0x14
+; CHECK-NEXT:    st1d { z6.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #16 // =0x10
+; CHECK-NEXT:    st1d { z7.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #12 // =0xc
+; CHECK-NEXT:    st1d { z4.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #8 // =0x8
+; CHECK-NEXT:    st1d { z2.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    mov x9, #4 // =0x4
+; CHECK-NEXT:    st1d { z1.d }, p0, [x8, x9, lsl #3]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <32 x i64> @llvm.llrint.v32i64.v16f64(<32 x double> %x)
+  ret <32 x i64> %a
+}
+declare <32 x i64> @llvm.llrint.v32i64.v32f64(<32 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll b/llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll
new file mode 100644
index 000000000000..9bdbe9b8ac62
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll
@@ -0,0 +1,1616 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=aarch64 -mattr=+sve \
+; RUN:   -aarch64-sve-vector-bits-min=256 | FileCheck --check-prefixes=CHECK-i32 %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=aarch64 -mattr=+sve \
+; RUN:   -aarch64-sve-vector-bits-min=256 | FileCheck --check-prefixes=CHECK-i64 %s
+
+define <1 x iXLen> @lrint_v1f16(<1 x half> %x) {
+; CHECK-i32-LABEL: lrint_v1f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx h0, h0
+; CHECK-i32-NEXT:    fcvtzs w8, h0
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v1f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx h0, h0
+; CHECK-i64-NEXT:    fcvtzs x8, h0
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    ret
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f16(<1 x half> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f16(<1 x half>)
+
+define <2 x iXLen> @lrint_v2f16(<2 x half> %x) {
+; CHECK-i32-LABEL: lrint_v2f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-i32-NEXT:    mov h1, v0.h[1]
+; CHECK-i32-NEXT:    frintx h0, h0
+; CHECK-i32-NEXT:    frintx h1, h1
+; CHECK-i32-NEXT:    fcvtzs w8, h0
+; CHECK-i32-NEXT:    fcvtzs w9, h1
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    mov v0.s[1], w9
+; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v2f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-i64-NEXT:    mov h1, v0.h[1]
+; CHECK-i64-NEXT:    frintx h0, h0
+; CHECK-i64-NEXT:    frintx h1, h1
+; CHECK-i64-NEXT:    fcvtzs x8, h0
+; CHECK-i64-NEXT:    fcvtzs x9, h1
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    mov v0.d[1], x9
+; CHECK-i64-NEXT:    ret
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f16(<2 x half> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f16(<2 x half>)
+
+define <4 x iXLen> @lrint_v4f16(<4 x half> %x) {
+; CHECK-i32-LABEL: lrint_v4f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.4h, v0.4h
+; CHECK-i32-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-i32-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v4f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v0.4h, v0.4h
+; CHECK-i64-NEXT:    mov h1, v0.h[2]
+; CHECK-i64-NEXT:    mov h2, v0.h[3]
+; CHECK-i64-NEXT:    mov h3, v0.h[1]
+; CHECK-i64-NEXT:    fcvtzs x9, h0
+; CHECK-i64-NEXT:    fcvtzs x8, h1
+; CHECK-i64-NEXT:    fcvtzs x10, h2
+; CHECK-i64-NEXT:    fcvtzs x11, h3
+; CHECK-i64-NEXT:    fmov d0, x9
+; CHECK-i64-NEXT:    fmov d1, x8
+; CHECK-i64-NEXT:    mov v0.d[1], x11
+; CHECK-i64-NEXT:    mov v1.d[1], x10
+; CHECK-i64-NEXT:    ret
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f16(<4 x half> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f16(<4 x half>)
+
+define <8 x iXLen> @lrint_v8f16(<8 x half> %x) {
+; CHECK-i32-LABEL: lrint_v8f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v2.8h, v0.8h
+; CHECK-i32-NEXT:    mov h0, v2.h[4]
+; CHECK-i32-NEXT:    mov h1, v2.h[5]
+; CHECK-i32-NEXT:    mov h3, v2.h[1]
+; CHECK-i32-NEXT:    fcvtzs w9, h2
+; CHECK-i32-NEXT:    mov h4, v2.h[6]
+; CHECK-i32-NEXT:    fcvtzs w8, h0
+; CHECK-i32-NEXT:    mov h0, v2.h[2]
+; CHECK-i32-NEXT:    fcvtzs w10, h1
+; CHECK-i32-NEXT:    fcvtzs w11, h3
+; CHECK-i32-NEXT:    mov h3, v2.h[7]
+; CHECK-i32-NEXT:    fcvtzs w12, h4
+; CHECK-i32-NEXT:    mov h2, v2.h[3]
+; CHECK-i32-NEXT:    fcvtzs w13, h0
+; CHECK-i32-NEXT:    fmov s0, w9
+; CHECK-i32-NEXT:    fmov s1, w8
+; CHECK-i32-NEXT:    fcvtzs w8, h3
+; CHECK-i32-NEXT:    fcvtzs w9, h2
+; CHECK-i32-NEXT:    mov v0.s[1], w11
+; CHECK-i32-NEXT:    mov v1.s[1], w10
+; CHECK-i32-NEXT:    mov v0.s[2], w13
+; CHECK-i32-NEXT:    mov v1.s[2], w12
+; CHECK-i32-NEXT:    mov v0.s[3], w9
+; CHECK-i32-NEXT:    mov v1.s[3], w8
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v8f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-i64-NEXT:    frintx v0.4h, v0.4h
+; CHECK-i64-NEXT:    frintx v1.4h, v1.4h
+; CHECK-i64-NEXT:    mov h4, v0.h[2]
+; CHECK-i64-NEXT:    mov h2, v0.h[1]
+; CHECK-i64-NEXT:    mov h7, v0.h[3]
+; CHECK-i64-NEXT:    fcvtzs x8, h0
+; CHECK-i64-NEXT:    mov h3, v1.h[2]
+; CHECK-i64-NEXT:    mov h5, v1.h[3]
+; CHECK-i64-NEXT:    mov h6, v1.h[1]
+; CHECK-i64-NEXT:    fcvtzs x11, h1
+; CHECK-i64-NEXT:    fcvtzs x12, h4
+; CHECK-i64-NEXT:    fcvtzs x9, h2
+; CHECK-i64-NEXT:    fcvtzs x15, h7
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    fcvtzs x10, h3
+; CHECK-i64-NEXT:    fcvtzs x13, h5
+; CHECK-i64-NEXT:    fcvtzs x14, h6
+; CHECK-i64-NEXT:    fmov d1, x12
+; CHECK-i64-NEXT:    fmov d2, x11
+; CHECK-i64-NEXT:    mov v0.d[1], x9
+; CHECK-i64-NEXT:    fmov d3, x10
+; CHECK-i64-NEXT:    mov v1.d[1], x15
+; CHECK-i64-NEXT:    mov v2.d[1], x14
+; CHECK-i64-NEXT:    mov v3.d[1], x13
+; CHECK-i64-NEXT:    ret
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f16(<8 x half> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f16(<8 x half>)
+
+define <16 x iXLen> @lrint_v16f16(<16 x half> %x) {
+; CHECK-i32-LABEL: lrint_v16f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v1.8h, v1.8h
+; CHECK-i32-NEXT:    frintx v0.8h, v0.8h
+; CHECK-i32-NEXT:    mov h3, v1.h[4]
+; CHECK-i32-NEXT:    mov h2, v1.h[5]
+; CHECK-i32-NEXT:    mov h5, v0.h[4]
+; CHECK-i32-NEXT:    mov h4, v1.h[1]
+; CHECK-i32-NEXT:    mov h6, v0.h[1]
+; CHECK-i32-NEXT:    fcvtzs w11, h0
+; CHECK-i32-NEXT:    fcvtzs w14, h1
+; CHECK-i32-NEXT:    mov h7, v1.h[6]
+; CHECK-i32-NEXT:    mov h16, v1.h[3]
+; CHECK-i32-NEXT:    mov h17, v0.h[7]
+; CHECK-i32-NEXT:    mov h18, v0.h[3]
+; CHECK-i32-NEXT:    fcvtzs w9, h3
+; CHECK-i32-NEXT:    mov h3, v0.h[5]
+; CHECK-i32-NEXT:    fcvtzs w8, h2
+; CHECK-i32-NEXT:    mov h2, v1.h[2]
+; CHECK-i32-NEXT:    fcvtzs w12, h5
+; CHECK-i32-NEXT:    fcvtzs w10, h4
+; CHECK-i32-NEXT:    mov h4, v0.h[6]
+; CHECK-i32-NEXT:    mov h5, v0.h[2]
+; CHECK-i32-NEXT:    fcvtzs w13, h6
+; CHECK-i32-NEXT:    mov h6, v1.h[7]
+; CHECK-i32-NEXT:    fmov s0, w11
+; CHECK-i32-NEXT:    fcvtzs w16, h7
+; CHECK-i32-NEXT:    fcvtzs w15, h3
+; CHECK-i32-NEXT:    fmov s3, w9
+; CHECK-i32-NEXT:    fcvtzs w9, h16
+; CHECK-i32-NEXT:    fcvtzs w17, h2
+; CHECK-i32-NEXT:    fmov s1, w12
+; CHECK-i32-NEXT:    fmov s2, w14
+; CHECK-i32-NEXT:    fcvtzs w11, h4
+; CHECK-i32-NEXT:    fcvtzs w18, h5
+; CHECK-i32-NEXT:    mov v0.s[1], w13
+; CHECK-i32-NEXT:    mov v3.s[1], w8
+; CHECK-i32-NEXT:    fcvtzs w8, h6
+; CHECK-i32-NEXT:    fcvtzs w12, h18
+; CHECK-i32-NEXT:    mov v1.s[1], w15
+; CHECK-i32-NEXT:    mov v2.s[1], w10
+; CHECK-i32-NEXT:    fcvtzs w10, h17
+; CHECK-i32-NEXT:    mov v0.s[2], w18
+; CHECK-i32-NEXT:    mov v3.s[2], w16
+; CHECK-i32-NEXT:    mov v1.s[2], w11
+; CHECK-i32-NEXT:    mov v2.s[2], w17
+; CHECK-i32-NEXT:    mov v0.s[3], w12
+; CHECK-i32-NEXT:    mov v3.s[3], w8
+; CHECK-i32-NEXT:    mov v1.s[3], w10
+; CHECK-i32-NEXT:    mov v2.s[3], w9
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v16f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ext v2.16b, v1.16b, v1.16b, #8
+; CHECK-i64-NEXT:    frintx v1.4h, v1.4h
+; CHECK-i64-NEXT:    frintx v3.4h, v0.4h
+; CHECK-i64-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-i64-NEXT:    frintx v2.4h, v2.4h
+; CHECK-i64-NEXT:    mov h4, v1.h[2]
+; CHECK-i64-NEXT:    mov h5, v3.h[2]
+; CHECK-i64-NEXT:    frintx v0.4h, v0.4h
+; CHECK-i64-NEXT:    mov h6, v3.h[1]
+; CHECK-i64-NEXT:    fcvtzs x9, h3
+; CHECK-i64-NEXT:    mov h16, v1.h[1]
+; CHECK-i64-NEXT:    fcvtzs x12, h1
+; CHECK-i64-NEXT:    mov h3, v3.h[3]
+; CHECK-i64-NEXT:    mov h17, v1.h[3]
+; CHECK-i64-NEXT:    mov h7, v2.h[3]
+; CHECK-i64-NEXT:    fcvtzs x8, h4
+; CHECK-i64-NEXT:    fcvtzs x10, h5
+; CHECK-i64-NEXT:    mov h4, v2.h[2]
+; CHECK-i64-NEXT:    mov h5, v0.h[2]
+; CHECK-i64-NEXT:    fcvtzs x11, h6
+; CHECK-i64-NEXT:    mov h6, v0.h[3]
+; CHECK-i64-NEXT:    fcvtzs x15, h2
+; CHECK-i64-NEXT:    mov h2, v2.h[1]
+; CHECK-i64-NEXT:    fcvtzs x14, h0
+; CHECK-i64-NEXT:    fcvtzs x17, h3
+; CHECK-i64-NEXT:    fcvtzs x0, h17
+; CHECK-i64-NEXT:    fcvtzs x13, h7
+; CHECK-i64-NEXT:    mov h7, v0.h[1]
+; CHECK-i64-NEXT:    fmov d0, x9
+; CHECK-i64-NEXT:    fcvtzs x16, h4
+; CHECK-i64-NEXT:    fcvtzs x9, h5
+; CHECK-i64-NEXT:    fmov d4, x12
+; CHECK-i64-NEXT:    fcvtzs x12, h16
+; CHECK-i64-NEXT:    fmov d1, x10
+; CHECK-i64-NEXT:    fcvtzs x10, h6
+; CHECK-i64-NEXT:    fmov d5, x8
+; CHECK-i64-NEXT:    fcvtzs x8, h2
+; CHECK-i64-NEXT:    fmov d2, x14
+; CHECK-i64-NEXT:    fcvtzs x18, h7
+; CHECK-i64-NEXT:    fmov d6, x15
+; CHECK-i64-NEXT:    mov v0.d[1], x11
+; CHECK-i64-NEXT:    fmov d3, x9
+; CHECK-i64-NEXT:    fmov d7, x16
+; CHECK-i64-NEXT:    mov v1.d[1], x17
+; CHECK-i64-NEXT:    mov v4.d[1], x12
+; CHECK-i64-NEXT:    mov v5.d[1], x0
+; CHECK-i64-NEXT:    mov v6.d[1], x8
+; CHECK-i64-NEXT:    mov v2.d[1], x18
+; CHECK-i64-NEXT:    mov v3.d[1], x10
+; CHECK-i64-NEXT:    mov v7.d[1], x13
+; CHECK-i64-NEXT:    ret
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f16(<16 x half> %x)
+  ret <16 x iXLen> %a
+}
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f16(<16 x half>)
+
+define <32 x iXLen> @lrint_v32f16(<32 x half> %x) {
+; CHECK-i32-LABEL: lrint_v32f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    stp x26, x25, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp x24, x23, [sp, #16] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-i32-NEXT:    .cfi_offset w19, -8
+; CHECK-i32-NEXT:    .cfi_offset w20, -16
+; CHECK-i32-NEXT:    .cfi_offset w21, -24
+; CHECK-i32-NEXT:    .cfi_offset w22, -32
+; CHECK-i32-NEXT:    .cfi_offset w23, -40
+; CHECK-i32-NEXT:    .cfi_offset w24, -48
+; CHECK-i32-NEXT:    .cfi_offset w25, -56
+; CHECK-i32-NEXT:    .cfi_offset w26, -64
+; CHECK-i32-NEXT:    frintx v3.8h, v3.8h
+; CHECK-i32-NEXT:    frintx v2.8h, v2.8h
+; CHECK-i32-NEXT:    frintx v1.8h, v1.8h
+; CHECK-i32-NEXT:    frintx v0.8h, v0.8h
+; CHECK-i32-NEXT:    mov h4, v3.h[7]
+; CHECK-i32-NEXT:    mov h5, v3.h[6]
+; CHECK-i32-NEXT:    mov h6, v3.h[5]
+; CHECK-i32-NEXT:    mov h7, v3.h[4]
+; CHECK-i32-NEXT:    mov h16, v3.h[3]
+; CHECK-i32-NEXT:    mov h17, v3.h[2]
+; CHECK-i32-NEXT:    mov h18, v3.h[1]
+; CHECK-i32-NEXT:    mov h19, v2.h[7]
+; CHECK-i32-NEXT:    fcvtzs w1, h3
+; CHECK-i32-NEXT:    mov h3, v1.h[6]
+; CHECK-i32-NEXT:    fcvtzs w7, h2
+; CHECK-i32-NEXT:    fcvtzs w22, h0
+; CHECK-i32-NEXT:    fcvtzs w8, h4
+; CHECK-i32-NEXT:    mov h4, v2.h[6]
+; CHECK-i32-NEXT:    fcvtzs w10, h5
+; CHECK-i32-NEXT:    mov h5, v2.h[5]
+; CHECK-i32-NEXT:    fcvtzs w12, h6
+; CHECK-i32-NEXT:    mov h6, v2.h[4]
+; CHECK-i32-NEXT:    fcvtzs w13, h7
+; CHECK-i32-NEXT:    mov h7, v2.h[3]
+; CHECK-i32-NEXT:    fcvtzs w9, h16
+; CHECK-i32-NEXT:    fcvtzs w11, h17
+; CHECK-i32-NEXT:    mov h16, v2.h[2]
+; CHECK-i32-NEXT:    mov h17, v2.h[1]
+; CHECK-i32-NEXT:    fcvtzs w17, h4
+; CHECK-i32-NEXT:    mov h4, v1.h[5]
+; CHECK-i32-NEXT:    mov h2, v0.h[5]
+; CHECK-i32-NEXT:    fcvtzs w0, h5
+; CHECK-i32-NEXT:    fcvtzs w3, h6
+; CHECK-i32-NEXT:    mov h5, v1.h[4]
+; CHECK-i32-NEXT:    mov h6, v0.h[4]
+; CHECK-i32-NEXT:    fcvtzs w16, h7
+; CHECK-i32-NEXT:    mov h7, v0.h[1]
+; CHECK-i32-NEXT:    fcvtzs w15, h18
+; CHECK-i32-NEXT:    fcvtzs w2, h3
+; CHECK-i32-NEXT:    mov h3, v1.h[2]
+; CHECK-i32-NEXT:    fcvtzs w19, h4
+; CHECK-i32-NEXT:    mov h4, v1.h[1]
+; CHECK-i32-NEXT:    mov h18, v0.h[6]
+; CHECK-i32-NEXT:    fcvtzs w20, h5
+; CHECK-i32-NEXT:    fcvtzs w23, h2
+; CHECK-i32-NEXT:    mov h2, v0.h[2]
+; CHECK-i32-NEXT:    fcvtzs w21, h6
+; CHECK-i32-NEXT:    fcvtzs w25, h1
+; CHECK-i32-NEXT:    fcvtzs w4, h17
+; CHECK-i32-NEXT:    fcvtzs w24, h7
+; CHECK-i32-NEXT:    fcvtzs w14, h19
+; CHECK-i32-NEXT:    fcvtzs w18, h16
+; CHECK-i32-NEXT:    fcvtzs w26, h4
+; CHECK-i32-NEXT:    mov h16, v1.h[7]
+; CHECK-i32-NEXT:    mov h17, v1.h[3]
+; CHECK-i32-NEXT:    fcvtzs w5, h3
+; CHECK-i32-NEXT:    mov h19, v0.h[7]
+; CHECK-i32-NEXT:    fcvtzs w6, h18
+; CHECK-i32-NEXT:    mov h18, v0.h[3]
+; CHECK-i32-NEXT:    fmov s0, w22
+; CHECK-i32-NEXT:    fmov s1, w21
+; CHECK-i32-NEXT:    fcvtzs w21, h2
+; CHECK-i32-NEXT:    fmov s2, w25
+; CHECK-i32-NEXT:    fmov s3, w20
+; CHECK-i32-NEXT:    fmov s4, w7
+; CHECK-i32-NEXT:    fmov s5, w3
+; CHECK-i32-NEXT:    fmov s6, w1
+; CHECK-i32-NEXT:    fmov s7, w13
+; CHECK-i32-NEXT:    mov v0.s[1], w24
+; CHECK-i32-NEXT:    mov v1.s[1], w23
+; CHECK-i32-NEXT:    ldp x24, x23, [sp, #16] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v2.s[1], w26
+; CHECK-i32-NEXT:    mov v3.s[1], w19
+; CHECK-i32-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v4.s[1], w4
+; CHECK-i32-NEXT:    mov v5.s[1], w0
+; CHECK-i32-NEXT:    mov v6.s[1], w15
+; CHECK-i32-NEXT:    mov v7.s[1], w12
+; CHECK-i32-NEXT:    fcvtzs w12, h16
+; CHECK-i32-NEXT:    fcvtzs w13, h17
+; CHECK-i32-NEXT:    fcvtzs w15, h19
+; CHECK-i32-NEXT:    fcvtzs w0, h18
+; CHECK-i32-NEXT:    mov v0.s[2], w21
+; CHECK-i32-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v1.s[2], w6
+; CHECK-i32-NEXT:    mov v2.s[2], w5
+; CHECK-i32-NEXT:    mov v3.s[2], w2
+; CHECK-i32-NEXT:    mov v4.s[2], w18
+; CHECK-i32-NEXT:    mov v5.s[2], w17
+; CHECK-i32-NEXT:    mov v6.s[2], w11
+; CHECK-i32-NEXT:    mov v7.s[2], w10
+; CHECK-i32-NEXT:    mov v0.s[3], w0
+; CHECK-i32-NEXT:    mov v1.s[3], w15
+; CHECK-i32-NEXT:    mov v2.s[3], w13
+; CHECK-i32-NEXT:    mov v3.s[3], w12
+; CHECK-i32-NEXT:    mov v4.s[3], w16
+; CHECK-i32-NEXT:    mov v5.s[3], w14
+; CHECK-i32-NEXT:    mov v6.s[3], w9
+; CHECK-i32-NEXT:    mov v7.s[3], w8
+; CHECK-i32-NEXT:    ldp x26, x25, [sp], #64 // 16-byte Folded Reload
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v32f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-i64-NEXT:    sub x9, sp, #272
+; CHECK-i64-NEXT:    mov x29, sp
+; CHECK-i64-NEXT:    and sp, x9, #0xffffffffffffffe0
+; CHECK-i64-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-i64-NEXT:    .cfi_offset w30, -8
+; CHECK-i64-NEXT:    .cfi_offset w29, -16
+; CHECK-i64-NEXT:    frintx v5.4h, v0.4h
+; CHECK-i64-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-i64-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-i64-NEXT:    ext v17.16b, v2.16b, v2.16b, #8
+; CHECK-i64-NEXT:    frintx v1.4h, v1.4h
+; CHECK-i64-NEXT:    frintx v2.4h, v2.4h
+; CHECK-i64-NEXT:    ptrue p0.d, vl4
+; CHECK-i64-NEXT:    mov h6, v5.h[3]
+; CHECK-i64-NEXT:    frintx v0.4h, v0.4h
+; CHECK-i64-NEXT:    mov h7, v5.h[2]
+; CHECK-i64-NEXT:    mov h16, v5.h[1]
+; CHECK-i64-NEXT:    frintx v4.4h, v4.4h
+; CHECK-i64-NEXT:    fcvtzs x12, h5
+; CHECK-i64-NEXT:    ext v5.16b, v3.16b, v3.16b, #8
+; CHECK-i64-NEXT:    frintx v17.4h, v17.4h
+; CHECK-i64-NEXT:    frintx v3.4h, v3.4h
+; CHECK-i64-NEXT:    fcvtzs x9, h6
+; CHECK-i64-NEXT:    mov h6, v0.h[3]
+; CHECK-i64-NEXT:    fcvtzs x10, h7
+; CHECK-i64-NEXT:    mov h7, v0.h[2]
+; CHECK-i64-NEXT:    fcvtzs x11, h16
+; CHECK-i64-NEXT:    mov h16, v0.h[1]
+; CHECK-i64-NEXT:    fcvtzs x13, h6
+; CHECK-i64-NEXT:    mov h6, v4.h[3]
+; CHECK-i64-NEXT:    stp x10, x9, [sp, #48]
+; CHECK-i64-NEXT:    fcvtzs x9, h7
+; CHECK-i64-NEXT:    mov h7, v4.h[2]
+; CHECK-i64-NEXT:    fcvtzs x10, h16
+; CHECK-i64-NEXT:    mov h16, v4.h[1]
+; CHECK-i64-NEXT:    stp x12, x11, [sp, #32]
+; CHECK-i64-NEXT:    fcvtzs x11, h0
+; CHECK-i64-NEXT:    frintx v0.4h, v5.4h
+; CHECK-i64-NEXT:    mov h5, v17.h[3]
+; CHECK-i64-NEXT:    fcvtzs x12, h6
+; CHECK-i64-NEXT:    mov h6, v17.h[2]
+; CHECK-i64-NEXT:    stp x9, x13, [sp, #16]
+; CHECK-i64-NEXT:    fcvtzs x13, h7
+; CHECK-i64-NEXT:    mov h7, v17.h[1]
+; CHECK-i64-NEXT:    fcvtzs x9, h16
+; CHECK-i64-NEXT:    stp x11, x10, [sp]
+; CHECK-i64-NEXT:    fcvtzs x10, h4
+; CHECK-i64-NEXT:    fcvtzs x11, h5
+; CHECK-i64-NEXT:    mov h4, v0.h[3]
+; CHECK-i64-NEXT:    mov h5, v0.h[2]
+; CHECK-i64-NEXT:    stp x13, x12, [sp, #80]
+; CHECK-i64-NEXT:    fcvtzs x12, h6
+; CHECK-i64-NEXT:    fcvtzs x13, h7
+; CHECK-i64-NEXT:    mov h6, v0.h[1]
+; CHECK-i64-NEXT:    stp x10, x9, [sp, #64]
+; CHECK-i64-NEXT:    fcvtzs x9, h17
+; CHECK-i64-NEXT:    mov h7, v1.h[3]
+; CHECK-i64-NEXT:    fcvtzs x10, h4
+; CHECK-i64-NEXT:    mov h4, v1.h[2]
+; CHECK-i64-NEXT:    stp x12, x11, [sp, #144]
+; CHECK-i64-NEXT:    fcvtzs x11, h5
+; CHECK-i64-NEXT:    mov h5, v1.h[1]
+; CHECK-i64-NEXT:    fcvtzs x12, h6
+; CHECK-i64-NEXT:    stp x9, x13, [sp, #128]
+; CHECK-i64-NEXT:    fcvtzs x9, h0
+; CHECK-i64-NEXT:    fcvtzs x13, h7
+; CHECK-i64-NEXT:    mov h0, v2.h[3]
+; CHECK-i64-NEXT:    stp x11, x10, [sp, #208]
+; CHECK-i64-NEXT:    fcvtzs x10, h4
+; CHECK-i64-NEXT:    mov h4, v2.h[2]
+; CHECK-i64-NEXT:    fcvtzs x11, h5
+; CHECK-i64-NEXT:    mov h5, v2.h[1]
+; CHECK-i64-NEXT:    stp x9, x12, [sp, #192]
+; CHECK-i64-NEXT:    fcvtzs x9, h1
+; CHECK-i64-NEXT:    fcvtzs x12, h0
+; CHECK-i64-NEXT:    mov h0, v3.h[3]
+; CHECK-i64-NEXT:    mov h1, v3.h[2]
+; CHECK-i64-NEXT:    stp x10, x13, [sp, #112]
+; CHECK-i64-NEXT:    fcvtzs x10, h4
+; CHECK-i64-NEXT:    mov h4, v3.h[1]
+; CHECK-i64-NEXT:    fcvtzs x13, h5
+; CHECK-i64-NEXT:    stp x9, x11, [sp, #96]
+; CHECK-i64-NEXT:    fcvtzs x9, h2
+; CHECK-i64-NEXT:    fcvtzs x11, h0
+; CHECK-i64-NEXT:    stp x10, x12, [sp, #176]
+; CHECK-i64-NEXT:    fcvtzs x10, h1
+; CHECK-i64-NEXT:    fcvtzs x12, h4
+; CHECK-i64-NEXT:    stp x9, x13, [sp, #160]
+; CHECK-i64-NEXT:    fcvtzs x9, h3
+; CHECK-i64-NEXT:    stp x10, x11, [sp, #240]
+; CHECK-i64-NEXT:    add x10, sp, #64
+; CHECK-i64-NEXT:    stp x9, x12, [sp, #224]
+; CHECK-i64-NEXT:    add x9, sp, #32
+; CHECK-i64-NEXT:    ld1d { z0.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    mov x9, sp
+; CHECK-i64-NEXT:    ld1d { z2.d }, p0/z, [x10]
+; CHECK-i64-NEXT:    ld1d { z1.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    add x9, sp, #224
+; CHECK-i64-NEXT:    add x10, sp, #128
+; CHECK-i64-NEXT:    ld1d { z3.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    add x9, sp, #160
+; CHECK-i64-NEXT:    ld1d { z4.d }, p0/z, [x10]
+; CHECK-i64-NEXT:    add x10, sp, #96
+; CHECK-i64-NEXT:    ld1d { z5.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    add x9, sp, #192
+; CHECK-i64-NEXT:    ld1d { z6.d }, p0/z, [x10]
+; CHECK-i64-NEXT:    mov x10, #24 // =0x18
+; CHECK-i64-NEXT:    ld1d { z7.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    mov x9, #16 // =0x10
+; CHECK-i64-NEXT:    st1d { z3.d }, p0, [x8, x10, lsl #3]
+; CHECK-i64-NEXT:    st1d { z5.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #8 // =0x8
+; CHECK-i64-NEXT:    st1d { z6.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #28 // =0x1c
+; CHECK-i64-NEXT:    st1d { z7.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #20 // =0x14
+; CHECK-i64-NEXT:    st1d { z4.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #12 // =0xc
+; CHECK-i64-NEXT:    st1d { z2.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #4 // =0x4
+; CHECK-i64-NEXT:    st1d { z1.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-i64-NEXT:    mov sp, x29
+; CHECK-i64-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ret
+  %a = call <32 x iXLen> @llvm.lrint.v32iXLen.v32f16(<32 x half> %x)
+  ret <32 x iXLen> %a
+}
+declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f16(<32 x half>)
+
+define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
+; CHECK-i32-LABEL: lrint_v1f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.2s, v0.2s
+; CHECK-i32-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v1f32:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-i64-NEXT:    frintx s0, s0
+; CHECK-i64-NEXT:    fcvtzs x8, s0
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    ret
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float>)
+
+define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
+; CHECK-i32-LABEL: lrint_v2f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.2s, v0.2s
+; CHECK-i32-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v2f32:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v0.2s, v0.2s
+; CHECK-i64-NEXT:    mov s1, v0.s[1]
+; CHECK-i64-NEXT:    fcvtzs x8, s0
+; CHECK-i64-NEXT:    fcvtzs x9, s1
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    mov v0.d[1], x9
+; CHECK-i64-NEXT:    ret
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float>)
+
+define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
+; CHECK-i32-LABEL: lrint_v4f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.4s, v0.4s
+; CHECK-i32-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v4f32:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v0.4s, v0.4s
+; CHECK-i64-NEXT:    mov s1, v0.s[2]
+; CHECK-i64-NEXT:    mov s2, v0.s[3]
+; CHECK-i64-NEXT:    mov s3, v0.s[1]
+; CHECK-i64-NEXT:    fcvtzs x9, s0
+; CHECK-i64-NEXT:    fcvtzs x8, s1
+; CHECK-i64-NEXT:    fcvtzs x10, s2
+; CHECK-i64-NEXT:    fcvtzs x11, s3
+; CHECK-i64-NEXT:    fmov d0, x9
+; CHECK-i64-NEXT:    fmov d1, x8
+; CHECK-i64-NEXT:    mov v0.d[1], x11
+; CHECK-i64-NEXT:    mov v1.d[1], x10
+; CHECK-i64-NEXT:    ret
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float>)
+
+define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
+; CHECK-i32-LABEL: lrint_v8f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    ptrue p0.d, vl2
+; CHECK-i32-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i32-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-i32-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i32-NEXT:    ptrue p0.s, vl8
+; CHECK-i32-NEXT:    movprfx z2, z0
+; CHECK-i32-NEXT:    frintx z2.s, p0/m, z0.s
+; CHECK-i32-NEXT:    mov z0.s, z2.s[4]
+; CHECK-i32-NEXT:    mov z1.s, z2.s[5]
+; CHECK-i32-NEXT:    mov z3.s, z2.s[1]
+; CHECK-i32-NEXT:    fcvtzs w9, s2
+; CHECK-i32-NEXT:    fcvtzs w8, s0
+; CHECK-i32-NEXT:    mov z0.s, z2.s[6]
+; CHECK-i32-NEXT:    fcvtzs w10, s1
+; CHECK-i32-NEXT:    mov z1.s, z2.s[2]
+; CHECK-i32-NEXT:    fcvtzs w11, s3
+; CHECK-i32-NEXT:    mov z3.s, z2.s[7]
+; CHECK-i32-NEXT:    mov z2.s, z2.s[3]
+; CHECK-i32-NEXT:    fcvtzs w12, s0
+; CHECK-i32-NEXT:    fmov s0, w9
+; CHECK-i32-NEXT:    fcvtzs w13, s1
+; CHECK-i32-NEXT:    fmov s1, w8
+; CHECK-i32-NEXT:    fcvtzs w8, s3
+; CHECK-i32-NEXT:    fcvtzs w9, s2
+; CHECK-i32-NEXT:    mov v0.s[1], w11
+; CHECK-i32-NEXT:    mov v1.s[1], w10
+; CHECK-i32-NEXT:    mov v0.s[2], w13
+; CHECK-i32-NEXT:    mov v1.s[2], w12
+; CHECK-i32-NEXT:    mov v0.s[3], w9
+; CHECK-i32-NEXT:    mov v1.s[3], w8
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v8f32:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v0.4s, v0.4s
+; CHECK-i64-NEXT:    frintx v1.4s, v1.4s
+; CHECK-i64-NEXT:    mov s3, v1.s[2]
+; CHECK-i64-NEXT:    mov s4, v0.s[2]
+; CHECK-i64-NEXT:    mov s2, v0.s[1]
+; CHECK-i64-NEXT:    mov s5, v1.s[3]
+; CHECK-i64-NEXT:    mov s6, v1.s[1]
+; CHECK-i64-NEXT:    mov s7, v0.s[3]
+; CHECK-i64-NEXT:    fcvtzs x8, s0
+; CHECK-i64-NEXT:    fcvtzs x10, s1
+; CHECK-i64-NEXT:    fcvtzs x11, s3
+; CHECK-i64-NEXT:    fcvtzs x12, s4
+; CHECK-i64-NEXT:    fcvtzs x9, s2
+; CHECK-i64-NEXT:    fcvtzs x13, s5
+; CHECK-i64-NEXT:    fcvtzs x14, s6
+; CHECK-i64-NEXT:    fcvtzs x15, s7
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    fmov d2, x10
+; CHECK-i64-NEXT:    fmov d1, x12
+; CHECK-i64-NEXT:    fmov d3, x11
+; CHECK-i64-NEXT:    mov v0.d[1], x9
+; CHECK-i64-NEXT:    mov v2.d[1], x14
+; CHECK-i64-NEXT:    mov v1.d[1], x15
+; CHECK-i64-NEXT:    mov v3.d[1], x13
+; CHECK-i64-NEXT:    ret
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float>)
+
+define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
+; CHECK-i32-LABEL: lrint_v16f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    ptrue p0.d, vl2
+; CHECK-i32-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-i32-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-i32-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i32-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-i32-NEXT:    splice z2.d, p0, z2.d, z3.d
+; CHECK-i32-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i32-NEXT:    ptrue p0.s, vl8
+; CHECK-i32-NEXT:    movprfx z1, z2
+; CHECK-i32-NEXT:    frintx z1.s, p0/m, z2.s
+; CHECK-i32-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-i32-NEXT:    mov z2.s, z1.s[5]
+; CHECK-i32-NEXT:    mov z3.s, z1.s[4]
+; CHECK-i32-NEXT:    mov z5.s, z0.s[5]
+; CHECK-i32-NEXT:    mov z7.s, z0.s[1]
+; CHECK-i32-NEXT:    fcvtzs w11, s0
+; CHECK-i32-NEXT:    fcvtzs w13, s1
+; CHECK-i32-NEXT:    mov z4.s, z1.s[7]
+; CHECK-i32-NEXT:    mov z6.s, z1.s[6]
+; CHECK-i32-NEXT:    mov z16.s, z0.s[7]
+; CHECK-i32-NEXT:    fcvtzs w8, s2
+; CHECK-i32-NEXT:    mov z2.s, z0.s[4]
+; CHECK-i32-NEXT:    fcvtzs w9, s3
+; CHECK-i32-NEXT:    mov z3.s, z1.s[1]
+; CHECK-i32-NEXT:    fcvtzs w10, s5
+; CHECK-i32-NEXT:    fcvtzs w12, s7
+; CHECK-i32-NEXT:    mov z5.s, z0.s[6]
+; CHECK-i32-NEXT:    mov z7.s, z1.s[2]
+; CHECK-i32-NEXT:    mov z17.s, z1.s[3]
+; CHECK-i32-NEXT:    fcvtzs w14, s2
+; CHECK-i32-NEXT:    mov z2.s, z0.s[2]
+; CHECK-i32-NEXT:    mov z18.s, z0.s[3]
+; CHECK-i32-NEXT:    fcvtzs w15, s3
+; CHECK-i32-NEXT:    fmov s0, w11
+; CHECK-i32-NEXT:    fmov s3, w9
+; CHECK-i32-NEXT:    fcvtzs w16, s6
+; CHECK-i32-NEXT:    fcvtzs w17, s5
+; CHECK-i32-NEXT:    fcvtzs w11, s7
+; CHECK-i32-NEXT:    fcvtzs w18, s2
+; CHECK-i32-NEXT:    fmov s2, w13
+; CHECK-i32-NEXT:    fcvtzs w9, s16
+; CHECK-i32-NEXT:    fmov s1, w14
+; CHECK-i32-NEXT:    mov v0.s[1], w12
+; CHECK-i32-NEXT:    mov v3.s[1], w8
+; CHECK-i32-NEXT:    fcvtzs w8, s4
+; CHECK-i32-NEXT:    fcvtzs w12, s18
+; CHECK-i32-NEXT:    mov v2.s[1], w15
+; CHECK-i32-NEXT:    mov v1.s[1], w10
+; CHECK-i32-NEXT:    fcvtzs w10, s17
+; CHECK-i32-NEXT:    mov v0.s[2], w18
+; CHECK-i32-NEXT:    mov v3.s[2], w16
+; CHECK-i32-NEXT:    mov v2.s[2], w11
+; CHECK-i32-NEXT:    mov v1.s[2], w17
+; CHECK-i32-NEXT:    mov v0.s[3], w12
+; CHECK-i32-NEXT:    mov v3.s[3], w8
+; CHECK-i32-NEXT:    mov v2.s[3], w10
+; CHECK-i32-NEXT:    mov v1.s[3], w9
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v16f32:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v3.4s, v3.4s
+; CHECK-i64-NEXT:    frintx v2.4s, v2.4s
+; CHECK-i64-NEXT:    frintx v1.4s, v1.4s
+; CHECK-i64-NEXT:    frintx v0.4s, v0.4s
+; CHECK-i64-NEXT:    mov s4, v3.s[2]
+; CHECK-i64-NEXT:    mov s5, v2.s[2]
+; CHECK-i64-NEXT:    mov s6, v1.s[2]
+; CHECK-i64-NEXT:    mov s7, v0.s[2]
+; CHECK-i64-NEXT:    fcvtzs x10, s1
+; CHECK-i64-NEXT:    fcvtzs x11, s0
+; CHECK-i64-NEXT:    mov s16, v0.s[1]
+; CHECK-i64-NEXT:    mov s17, v1.s[1]
+; CHECK-i64-NEXT:    mov s18, v3.s[1]
+; CHECK-i64-NEXT:    fcvtzs x14, s3
+; CHECK-i64-NEXT:    fcvtzs x16, s2
+; CHECK-i64-NEXT:    fcvtzs x8, s4
+; CHECK-i64-NEXT:    mov s4, v2.s[1]
+; CHECK-i64-NEXT:    fcvtzs x9, s5
+; CHECK-i64-NEXT:    mov s5, v1.s[3]
+; CHECK-i64-NEXT:    fcvtzs x12, s6
+; CHECK-i64-NEXT:    mov s6, v0.s[3]
+; CHECK-i64-NEXT:    fcvtzs x13, s7
+; CHECK-i64-NEXT:    mov s7, v3.s[3]
+; CHECK-i64-NEXT:    fmov d0, x11
+; CHECK-i64-NEXT:    fcvtzs x17, s16
+; CHECK-i64-NEXT:    fcvtzs x18, s18
+; CHECK-i64-NEXT:    fcvtzs x15, s4
+; CHECK-i64-NEXT:    mov s4, v2.s[3]
+; CHECK-i64-NEXT:    fmov d2, x10
+; CHECK-i64-NEXT:    fcvtzs x11, s5
+; CHECK-i64-NEXT:    fcvtzs x10, s6
+; CHECK-i64-NEXT:    fmov d3, x12
+; CHECK-i64-NEXT:    fmov d1, x13
+; CHECK-i64-NEXT:    fcvtzs x12, s17
+; CHECK-i64-NEXT:    fcvtzs x13, s7
+; CHECK-i64-NEXT:    fmov d5, x9
+; CHECK-i64-NEXT:    fmov d6, x14
+; CHECK-i64-NEXT:    fmov d7, x8
+; CHECK-i64-NEXT:    fcvtzs x0, s4
+; CHECK-i64-NEXT:    fmov d4, x16
+; CHECK-i64-NEXT:    mov v0.d[1], x17
+; CHECK-i64-NEXT:    mov v1.d[1], x10
+; CHECK-i64-NEXT:    mov v3.d[1], x11
+; CHECK-i64-NEXT:    mov v2.d[1], x12
+; CHECK-i64-NEXT:    mov v6.d[1], x18
+; CHECK-i64-NEXT:    mov v7.d[1], x13
+; CHECK-i64-NEXT:    mov v4.d[1], x15
+; CHECK-i64-NEXT:    mov v5.d[1], x0
+; CHECK-i64-NEXT:    ret
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float> %x)
+  ret <16 x iXLen> %a
+}
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float>)
+
+define <32 x iXLen> @lrint_v32f32(<32 x float> %x) {
+; CHECK-i32-LABEL: lrint_v32f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    str x27, [sp, #-80]! // 8-byte Folded Spill
+; CHECK-i32-NEXT:    stp x26, x25, [sp, #16] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp x24, x23, [sp, #32] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-i32-NEXT:    .cfi_offset w19, -8
+; CHECK-i32-NEXT:    .cfi_offset w20, -16
+; CHECK-i32-NEXT:    .cfi_offset w21, -24
+; CHECK-i32-NEXT:    .cfi_offset w22, -32
+; CHECK-i32-NEXT:    .cfi_offset w23, -40
+; CHECK-i32-NEXT:    .cfi_offset w24, -48
+; CHECK-i32-NEXT:    .cfi_offset w25, -56
+; CHECK-i32-NEXT:    .cfi_offset w26, -64
+; CHECK-i32-NEXT:    .cfi_offset w27, -80
+; CHECK-i32-NEXT:    ptrue p1.d, vl2
+; CHECK-i32-NEXT:    // kill: def $q6 killed $q6 def $z6
+; CHECK-i32-NEXT:    // kill: def $q7 killed $q7 def $z7
+; CHECK-i32-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-i32-NEXT:    // kill: def $q4 killed $q4 def $z4
+; CHECK-i32-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-i32-NEXT:    // kill: def $q5 killed $q5 def $z5
+; CHECK-i32-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-i32-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i32-NEXT:    ptrue p0.s, vl8
+; CHECK-i32-NEXT:    splice z6.d, p1, z6.d, z7.d
+; CHECK-i32-NEXT:    splice z2.d, p1, z2.d, z3.d
+; CHECK-i32-NEXT:    splice z4.d, p1, z4.d, z5.d
+; CHECK-i32-NEXT:    splice z0.d, p1, z0.d, z1.d
+; CHECK-i32-NEXT:    movprfx z3, z6
+; CHECK-i32-NEXT:    frintx z3.s, p0/m, z6.s
+; CHECK-i32-NEXT:    frintx z2.s, p0/m, z2.s
+; CHECK-i32-NEXT:    movprfx z1, z4
+; CHECK-i32-NEXT:    frintx z1.s, p0/m, z4.s
+; CHECK-i32-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-i32-NEXT:    mov z4.s, z3.s[7]
+; CHECK-i32-NEXT:    mov z5.s, z3.s[6]
+; CHECK-i32-NEXT:    mov z6.s, z3.s[5]
+; CHECK-i32-NEXT:    mov z16.s, z1.s[7]
+; CHECK-i32-NEXT:    mov z7.s, z3.s[4]
+; CHECK-i32-NEXT:    mov z17.s, z1.s[6]
+; CHECK-i32-NEXT:    mov z18.s, z1.s[5]
+; CHECK-i32-NEXT:    mov z19.s, z1.s[4]
+; CHECK-i32-NEXT:    fcvtzs w7, s3
+; CHECK-i32-NEXT:    fcvtzs w8, s4
+; CHECK-i32-NEXT:    mov z4.s, z2.s[7]
+; CHECK-i32-NEXT:    fcvtzs w10, s5
+; CHECK-i32-NEXT:    mov z5.s, z2.s[6]
+; CHECK-i32-NEXT:    fcvtzs w13, s6
+; CHECK-i32-NEXT:    fcvtzs w9, s16
+; CHECK-i32-NEXT:    mov z6.s, z2.s[4]
+; CHECK-i32-NEXT:    mov z16.s, z0.s[6]
+; CHECK-i32-NEXT:    fcvtzs w14, s7
+; CHECK-i32-NEXT:    fcvtzs w11, s4
+; CHECK-i32-NEXT:    mov z4.s, z2.s[5]
+; CHECK-i32-NEXT:    mov z7.s, z0.s[7]
+; CHECK-i32-NEXT:    fcvtzs w16, s5
+; CHECK-i32-NEXT:    mov z5.s, z0.s[4]
+; CHECK-i32-NEXT:    fcvtzs w12, s17
+; CHECK-i32-NEXT:    fcvtzs w15, s18
+; CHECK-i32-NEXT:    fcvtzs w17, s19
+; CHECK-i32-NEXT:    mov z17.s, z0.s[5]
+; CHECK-i32-NEXT:    fcvtzs w3, s4
+; CHECK-i32-NEXT:    mov z4.s, z3.s[1]
+; CHECK-i32-NEXT:    mov z18.s, z3.s[2]
+; CHECK-i32-NEXT:    fcvtzs w4, s6
+; CHECK-i32-NEXT:    fcvtzs w0, s16
+; CHECK-i32-NEXT:    fcvtzs w6, s5
+; CHECK-i32-NEXT:    mov z16.s, z3.s[3]
+; CHECK-i32-NEXT:    mov z3.s, z0.s[1]
+; CHECK-i32-NEXT:    mov z5.s, z1.s[1]
+; CHECK-i32-NEXT:    mov z6.s, z2.s[1]
+; CHECK-i32-NEXT:    fcvtzs w21, s1
+; CHECK-i32-NEXT:    fcvtzs w22, s0
+; CHECK-i32-NEXT:    fcvtzs w23, s2
+; CHECK-i32-NEXT:    fcvtzs w18, s7
+; CHECK-i32-NEXT:    fcvtzs w2, s4
+; CHECK-i32-NEXT:    mov z4.s, z1.s[2]
+; CHECK-i32-NEXT:    mov z7.s, z2.s[2]
+; CHECK-i32-NEXT:    fcvtzs w5, s17
+; CHECK-i32-NEXT:    fcvtzs w24, s3
+; CHECK-i32-NEXT:    fcvtzs w25, s5
+; CHECK-i32-NEXT:    fcvtzs w26, s6
+; CHECK-i32-NEXT:    fcvtzs w1, s18
+; CHECK-i32-NEXT:    mov z18.s, z0.s[2]
+; CHECK-i32-NEXT:    mov z17.s, z1.s[3]
+; CHECK-i32-NEXT:    fcvtzs w19, s4
+; CHECK-i32-NEXT:    mov z19.s, z2.s[3]
+; CHECK-i32-NEXT:    fcvtzs w20, s7
+; CHECK-i32-NEXT:    mov z20.s, z0.s[3]
+; CHECK-i32-NEXT:    fmov s0, w22
+; CHECK-i32-NEXT:    fmov s2, w23
+; CHECK-i32-NEXT:    fmov s4, w21
+; CHECK-i32-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    fmov s1, w6
+; CHECK-i32-NEXT:    fmov s6, w7
+; CHECK-i32-NEXT:    fmov s3, w4
+; CHECK-i32-NEXT:    fmov s5, w17
+; CHECK-i32-NEXT:    fmov s7, w14
+; CHECK-i32-NEXT:    fcvtzs w27, s18
+; CHECK-i32-NEXT:    mov v0.s[1], w24
+; CHECK-i32-NEXT:    ldp x24, x23, [sp, #32] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v2.s[1], w26
+; CHECK-i32-NEXT:    mov v4.s[1], w25
+; CHECK-i32-NEXT:    mov v1.s[1], w5
+; CHECK-i32-NEXT:    ldp x26, x25, [sp, #16] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v3.s[1], w3
+; CHECK-i32-NEXT:    mov v6.s[1], w2
+; CHECK-i32-NEXT:    mov v5.s[1], w15
+; CHECK-i32-NEXT:    mov v7.s[1], w13
+; CHECK-i32-NEXT:    fcvtzs w13, s16
+; CHECK-i32-NEXT:    fcvtzs w14, s17
+; CHECK-i32-NEXT:    fcvtzs w15, s19
+; CHECK-i32-NEXT:    fcvtzs w17, s20
+; CHECK-i32-NEXT:    mov v0.s[2], w27
+; CHECK-i32-NEXT:    mov v1.s[2], w0
+; CHECK-i32-NEXT:    mov v2.s[2], w20
+; CHECK-i32-NEXT:    mov v4.s[2], w19
+; CHECK-i32-NEXT:    mov v3.s[2], w16
+; CHECK-i32-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v6.s[2], w1
+; CHECK-i32-NEXT:    mov v5.s[2], w12
+; CHECK-i32-NEXT:    mov v7.s[2], w10
+; CHECK-i32-NEXT:    mov v0.s[3], w17
+; CHECK-i32-NEXT:    mov v1.s[3], w18
+; CHECK-i32-NEXT:    mov v2.s[3], w15
+; CHECK-i32-NEXT:    mov v4.s[3], w14
+; CHECK-i32-NEXT:    mov v3.s[3], w11
+; CHECK-i32-NEXT:    mov v6.s[3], w13
+; CHECK-i32-NEXT:    mov v5.s[3], w9
+; CHECK-i32-NEXT:    mov v7.s[3], w8
+; CHECK-i32-NEXT:    ldr x27, [sp], #80 // 8-byte Folded Reload
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v32f32:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-i64-NEXT:    sub x9, sp, #272
+; CHECK-i64-NEXT:    mov x29, sp
+; CHECK-i64-NEXT:    and sp, x9, #0xffffffffffffffe0
+; CHECK-i64-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-i64-NEXT:    .cfi_offset w30, -8
+; CHECK-i64-NEXT:    .cfi_offset w29, -16
+; CHECK-i64-NEXT:    frintx v0.4s, v0.4s
+; CHECK-i64-NEXT:    frintx v1.4s, v1.4s
+; CHECK-i64-NEXT:    frintx v2.4s, v2.4s
+; CHECK-i64-NEXT:    ptrue p0.d, vl4
+; CHECK-i64-NEXT:    mov s16, v0.s[3]
+; CHECK-i64-NEXT:    mov s17, v0.s[2]
+; CHECK-i64-NEXT:    mov s18, v0.s[1]
+; CHECK-i64-NEXT:    fcvtzs x12, s0
+; CHECK-i64-NEXT:    frintx v0.4s, v3.4s
+; CHECK-i64-NEXT:    mov s3, v2.s[3]
+; CHECK-i64-NEXT:    fcvtzs x9, s16
+; CHECK-i64-NEXT:    mov s16, v1.s[3]
+; CHECK-i64-NEXT:    fcvtzs x10, s17
+; CHECK-i64-NEXT:    mov s17, v1.s[2]
+; CHECK-i64-NEXT:    fcvtzs x11, s18
+; CHECK-i64-NEXT:    mov s18, v1.s[1]
+; CHECK-i64-NEXT:    fcvtzs x13, s16
+; CHECK-i64-NEXT:    stp x10, x9, [sp, #16]
+; CHECK-i64-NEXT:    mov s16, v2.s[2]
+; CHECK-i64-NEXT:    fcvtzs x9, s17
+; CHECK-i64-NEXT:    fcvtzs x10, s18
+; CHECK-i64-NEXT:    mov s17, v2.s[1]
+; CHECK-i64-NEXT:    stp x12, x11, [sp]
+; CHECK-i64-NEXT:    fcvtzs x11, s1
+; CHECK-i64-NEXT:    frintx v1.4s, v4.4s
+; CHECK-i64-NEXT:    fcvtzs x12, s3
+; CHECK-i64-NEXT:    mov s3, v0.s[3]
+; CHECK-i64-NEXT:    mov s4, v0.s[2]
+; CHECK-i64-NEXT:    stp x9, x13, [sp, #48]
+; CHECK-i64-NEXT:    fcvtzs x13, s16
+; CHECK-i64-NEXT:    fcvtzs x9, s17
+; CHECK-i64-NEXT:    mov s16, v0.s[1]
+; CHECK-i64-NEXT:    stp x11, x10, [sp, #32]
+; CHECK-i64-NEXT:    fcvtzs x10, s2
+; CHECK-i64-NEXT:    frintx v2.4s, v5.4s
+; CHECK-i64-NEXT:    fcvtzs x11, s3
+; CHECK-i64-NEXT:    mov s3, v1.s[3]
+; CHECK-i64-NEXT:    mov s5, v1.s[1]
+; CHECK-i64-NEXT:    stp x13, x12, [sp, #80]
+; CHECK-i64-NEXT:    fcvtzs x12, s4
+; CHECK-i64-NEXT:    mov s4, v1.s[2]
+; CHECK-i64-NEXT:    fcvtzs x13, s16
+; CHECK-i64-NEXT:    stp x10, x9, [sp, #64]
+; CHECK-i64-NEXT:    fcvtzs x9, s0
+; CHECK-i64-NEXT:    mov s0, v2.s[3]
+; CHECK-i64-NEXT:    fcvtzs x10, s3
+; CHECK-i64-NEXT:    frintx v3.4s, v6.4s
+; CHECK-i64-NEXT:    stp x12, x11, [sp, #112]
+; CHECK-i64-NEXT:    fcvtzs x11, s4
+; CHECK-i64-NEXT:    mov s4, v2.s[2]
+; CHECK-i64-NEXT:    fcvtzs x12, s5
+; CHECK-i64-NEXT:    mov s5, v2.s[1]
+; CHECK-i64-NEXT:    stp x9, x13, [sp, #96]
+; CHECK-i64-NEXT:    fcvtzs x9, s1
+; CHECK-i64-NEXT:    fcvtzs x13, s0
+; CHECK-i64-NEXT:    mov s0, v3.s[3]
+; CHECK-i64-NEXT:    frintx v1.4s, v7.4s
+; CHECK-i64-NEXT:    stp x11, x10, [sp, #144]
+; CHECK-i64-NEXT:    fcvtzs x10, s4
+; CHECK-i64-NEXT:    mov s4, v3.s[2]
+; CHECK-i64-NEXT:    fcvtzs x11, s5
+; CHECK-i64-NEXT:    mov s5, v3.s[1]
+; CHECK-i64-NEXT:    stp x9, x12, [sp, #128]
+; CHECK-i64-NEXT:    fcvtzs x9, s2
+; CHECK-i64-NEXT:    fcvtzs x12, s0
+; CHECK-i64-NEXT:    mov s0, v1.s[3]
+; CHECK-i64-NEXT:    mov s2, v1.s[2]
+; CHECK-i64-NEXT:    stp x10, x13, [sp, #176]
+; CHECK-i64-NEXT:    fcvtzs x10, s4
+; CHECK-i64-NEXT:    mov s4, v1.s[1]
+; CHECK-i64-NEXT:    fcvtzs x13, s5
+; CHECK-i64-NEXT:    stp x9, x11, [sp, #160]
+; CHECK-i64-NEXT:    fcvtzs x9, s3
+; CHECK-i64-NEXT:    fcvtzs x11, s0
+; CHECK-i64-NEXT:    stp x10, x12, [sp, #208]
+; CHECK-i64-NEXT:    fcvtzs x10, s2
+; CHECK-i64-NEXT:    fcvtzs x12, s4
+; CHECK-i64-NEXT:    stp x9, x13, [sp, #192]
+; CHECK-i64-NEXT:    fcvtzs x9, s1
+; CHECK-i64-NEXT:    stp x10, x11, [sp, #240]
+; CHECK-i64-NEXT:    add x10, sp, #64
+; CHECK-i64-NEXT:    stp x9, x12, [sp, #224]
+; CHECK-i64-NEXT:    mov x9, sp
+; CHECK-i64-NEXT:    ld1d { z0.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    add x9, sp, #32
+; CHECK-i64-NEXT:    ld1d { z2.d }, p0/z, [x10]
+; CHECK-i64-NEXT:    ld1d { z1.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    add x9, sp, #224
+; CHECK-i64-NEXT:    add x10, sp, #96
+; CHECK-i64-NEXT:    ld1d { z3.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    add x9, sp, #192
+; CHECK-i64-NEXT:    ld1d { z4.d }, p0/z, [x10]
+; CHECK-i64-NEXT:    add x10, sp, #160
+; CHECK-i64-NEXT:    ld1d { z5.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    add x9, sp, #128
+; CHECK-i64-NEXT:    ld1d { z6.d }, p0/z, [x10]
+; CHECK-i64-NEXT:    mov x10, #28 // =0x1c
+; CHECK-i64-NEXT:    ld1d { z7.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    mov x9, #24 // =0x18
+; CHECK-i64-NEXT:    st1d { z3.d }, p0, [x8, x10, lsl #3]
+; CHECK-i64-NEXT:    st1d { z5.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #20 // =0x14
+; CHECK-i64-NEXT:    st1d { z6.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #16 // =0x10
+; CHECK-i64-NEXT:    st1d { z7.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #12 // =0xc
+; CHECK-i64-NEXT:    st1d { z4.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #8 // =0x8
+; CHECK-i64-NEXT:    st1d { z2.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #4 // =0x4
+; CHECK-i64-NEXT:    st1d { z1.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-i64-NEXT:    mov sp, x29
+; CHECK-i64-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ret
+  %a = call <32 x iXLen> @llvm.lrint.v32iXLen.v32f32(<32 x float> %x)
+  ret <32 x iXLen> %a
+}
+declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f32(<32 x float>)
+
+define <1 x iXLen> @lrint_v1f64(<1 x double> %x) {
+; CHECK-i32-LABEL: lrint_v1f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx d0, d0
+; CHECK-i32-NEXT:    fcvtzs w8, d0
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v1f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx d0, d0
+; CHECK-i64-NEXT:    fcvtzs x8, d0
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    ret
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double>)
+
+define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
+; CHECK-i32-LABEL: lrint_v2f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.2d, v0.2d
+; CHECK-i32-NEXT:    mov d1, v0.d[1]
+; CHECK-i32-NEXT:    fcvtzs w8, d0
+; CHECK-i32-NEXT:    fcvtzs w9, d1
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    mov v0.s[1], w9
+; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v2f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v0.2d, v0.2d
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-i64-NEXT:    ret
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double>)
+
+define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
+; CHECK-i32-LABEL: lrint_v4f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    ptrue p0.d, vl2
+; CHECK-i32-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i32-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-i32-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i32-NEXT:    ptrue p0.d, vl4
+; CHECK-i32-NEXT:    movprfx z1, z0
+; CHECK-i32-NEXT:    frintx z1.d, p0/m, z0.d
+; CHECK-i32-NEXT:    mov z0.d, z1.d[1]
+; CHECK-i32-NEXT:    fcvtzs w8, d1
+; CHECK-i32-NEXT:    mov z2.d, z1.d[2]
+; CHECK-i32-NEXT:    mov z1.d, z1.d[3]
+; CHECK-i32-NEXT:    fcvtzs w9, d0
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    fcvtzs w8, d2
+; CHECK-i32-NEXT:    mov v0.s[1], w9
+; CHECK-i32-NEXT:    mov v0.s[2], w8
+; CHECK-i32-NEXT:    fcvtzs w8, d1
+; CHECK-i32-NEXT:    mov v0.s[3], w8
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v4f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ptrue p0.d, vl2
+; CHECK-i64-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i64-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-i64-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i64-NEXT:    ptrue p0.d, vl4
+; CHECK-i64-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-i64-NEXT:    mov z1.d, z0.d[2]
+; CHECK-i64-NEXT:    mov z2.d, z0.d[3]
+; CHECK-i64-NEXT:    mov z3.d, z0.d[1]
+; CHECK-i64-NEXT:    fcvtzs x9, d0
+; CHECK-i64-NEXT:    fcvtzs x8, d1
+; CHECK-i64-NEXT:    fcvtzs x10, d2
+; CHECK-i64-NEXT:    fcvtzs x11, d3
+; CHECK-i64-NEXT:    fmov d0, x9
+; CHECK-i64-NEXT:    fmov d1, x8
+; CHECK-i64-NEXT:    mov v0.d[1], x11
+; CHECK-i64-NEXT:    mov v1.d[1], x10
+; CHECK-i64-NEXT:    ret
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double>)
+
+define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
+; CHECK-i32-LABEL: lrint_v8f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    ptrue p0.d, vl2
+; CHECK-i32-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-i32-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i32-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-i32-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-i32-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i32-NEXT:    splice z2.d, p0, z2.d, z3.d
+; CHECK-i32-NEXT:    ptrue p0.d, vl4
+; CHECK-i32-NEXT:    movprfx z3, z0
+; CHECK-i32-NEXT:    frintx z3.d, p0/m, z0.d
+; CHECK-i32-NEXT:    frintx z2.d, p0/m, z2.d
+; CHECK-i32-NEXT:    mov z0.d, z3.d[1]
+; CHECK-i32-NEXT:    mov z1.d, z2.d[1]
+; CHECK-i32-NEXT:    fcvtzs w8, d3
+; CHECK-i32-NEXT:    fcvtzs w9, d2
+; CHECK-i32-NEXT:    mov z4.d, z3.d[2]
+; CHECK-i32-NEXT:    mov z5.d, z2.d[2]
+; CHECK-i32-NEXT:    mov z3.d, z3.d[3]
+; CHECK-i32-NEXT:    mov z2.d, z2.d[3]
+; CHECK-i32-NEXT:    fcvtzs w10, d0
+; CHECK-i32-NEXT:    fcvtzs w11, d1
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    fcvtzs w8, d4
+; CHECK-i32-NEXT:    fmov s1, w9
+; CHECK-i32-NEXT:    fcvtzs w9, d5
+; CHECK-i32-NEXT:    mov v0.s[1], w10
+; CHECK-i32-NEXT:    mov v1.s[1], w11
+; CHECK-i32-NEXT:    mov v0.s[2], w8
+; CHECK-i32-NEXT:    fcvtzs w8, d3
+; CHECK-i32-NEXT:    mov v1.s[2], w9
+; CHECK-i32-NEXT:    fcvtzs w9, d2
+; CHECK-i32-NEXT:    mov v0.s[3], w8
+; CHECK-i32-NEXT:    mov v1.s[3], w9
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v8f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ptrue p0.d, vl2
+; CHECK-i64-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-i64-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i64-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-i64-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-i64-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i64-NEXT:    splice z2.d, p0, z2.d, z3.d
+; CHECK-i64-NEXT:    ptrue p0.d, vl4
+; CHECK-i64-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-i64-NEXT:    movprfx z1, z2
+; CHECK-i64-NEXT:    frintx z1.d, p0/m, z2.d
+; CHECK-i64-NEXT:    mov z4.d, z1.d[2]
+; CHECK-i64-NEXT:    mov z5.d, z0.d[2]
+; CHECK-i64-NEXT:    mov z2.d, z0.d[1]
+; CHECK-i64-NEXT:    mov z3.d, z1.d[3]
+; CHECK-i64-NEXT:    mov z6.d, z0.d[3]
+; CHECK-i64-NEXT:    fcvtzs x8, d0
+; CHECK-i64-NEXT:    mov z0.d, z1.d[1]
+; CHECK-i64-NEXT:    fcvtzs x10, d1
+; CHECK-i64-NEXT:    fcvtzs x11, d4
+; CHECK-i64-NEXT:    fcvtzs x12, d5
+; CHECK-i64-NEXT:    fcvtzs x9, d2
+; CHECK-i64-NEXT:    fcvtzs x13, d3
+; CHECK-i64-NEXT:    fcvtzs x14, d6
+; CHECK-i64-NEXT:    fcvtzs x15, d0
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    fmov d2, x10
+; CHECK-i64-NEXT:    fmov d1, x12
+; CHECK-i64-NEXT:    fmov d3, x11
+; CHECK-i64-NEXT:    mov v0.d[1], x9
+; CHECK-i64-NEXT:    mov v2.d[1], x15
+; CHECK-i64-NEXT:    mov v1.d[1], x14
+; CHECK-i64-NEXT:    mov v3.d[1], x13
+; CHECK-i64-NEXT:    ret
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f64(<8 x double> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f64(<8 x double>)
+
+define <16 x iXLen> @lrint_v16f64(<16 x double> %x) {
+; CHECK-i32-LABEL: lrint_v16f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    ptrue p1.d, vl2
+; CHECK-i32-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i32-NEXT:    // kill: def $q6 killed $q6 def $z6
+; CHECK-i32-NEXT:    // kill: def $q4 killed $q4 def $z4
+; CHECK-i32-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-i32-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-i32-NEXT:    // kill: def $q7 killed $q7 def $z7
+; CHECK-i32-NEXT:    // kill: def $q5 killed $q5 def $z5
+; CHECK-i32-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-i32-NEXT:    ptrue p0.d, vl4
+; CHECK-i32-NEXT:    splice z0.d, p1, z0.d, z1.d
+; CHECK-i32-NEXT:    splice z2.d, p1, z2.d, z3.d
+; CHECK-i32-NEXT:    splice z4.d, p1, z4.d, z5.d
+; CHECK-i32-NEXT:    splice z6.d, p1, z6.d, z7.d
+; CHECK-i32-NEXT:    movprfx z5, z0
+; CHECK-i32-NEXT:    frintx z5.d, p0/m, z0.d
+; CHECK-i32-NEXT:    movprfx z7, z2
+; CHECK-i32-NEXT:    frintx z7.d, p0/m, z2.d
+; CHECK-i32-NEXT:    frintx z4.d, p0/m, z4.d
+; CHECK-i32-NEXT:    frintx z6.d, p0/m, z6.d
+; CHECK-i32-NEXT:    fcvtzs w8, d5
+; CHECK-i32-NEXT:    mov z0.d, z5.d[1]
+; CHECK-i32-NEXT:    mov z1.d, z7.d[1]
+; CHECK-i32-NEXT:    fcvtzs w9, d7
+; CHECK-i32-NEXT:    mov z3.d, z4.d[1]
+; CHECK-i32-NEXT:    fcvtzs w10, d4
+; CHECK-i32-NEXT:    mov z16.d, z6.d[1]
+; CHECK-i32-NEXT:    fcvtzs w12, d6
+; CHECK-i32-NEXT:    mov z2.d, z5.d[2]
+; CHECK-i32-NEXT:    fcvtzs w11, d0
+; CHECK-i32-NEXT:    fcvtzs w13, d1
+; CHECK-i32-NEXT:    mov z17.d, z7.d[2]
+; CHECK-i32-NEXT:    fcvtzs w14, d3
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    mov z18.d, z4.d[2]
+; CHECK-i32-NEXT:    fcvtzs w8, d16
+; CHECK-i32-NEXT:    mov z19.d, z6.d[2]
+; CHECK-i32-NEXT:    fcvtzs w15, d2
+; CHECK-i32-NEXT:    fmov s1, w9
+; CHECK-i32-NEXT:    fmov s2, w10
+; CHECK-i32-NEXT:    fmov s3, w12
+; CHECK-i32-NEXT:    fcvtzs w9, d17
+; CHECK-i32-NEXT:    fcvtzs w10, d18
+; CHECK-i32-NEXT:    mov v0.s[1], w11
+; CHECK-i32-NEXT:    fcvtzs w11, d19
+; CHECK-i32-NEXT:    mov z5.d, z5.d[3]
+; CHECK-i32-NEXT:    mov z7.d, z7.d[3]
+; CHECK-i32-NEXT:    mov v1.s[1], w13
+; CHECK-i32-NEXT:    mov v2.s[1], w14
+; CHECK-i32-NEXT:    mov v3.s[1], w8
+; CHECK-i32-NEXT:    mov z4.d, z4.d[3]
+; CHECK-i32-NEXT:    mov z6.d, z6.d[3]
+; CHECK-i32-NEXT:    mov v0.s[2], w15
+; CHECK-i32-NEXT:    fcvtzs w8, d5
+; CHECK-i32-NEXT:    mov v1.s[2], w9
+; CHECK-i32-NEXT:    fcvtzs w9, d7
+; CHECK-i32-NEXT:    mov v2.s[2], w10
+; CHECK-i32-NEXT:    fcvtzs w10, d4
+; CHECK-i32-NEXT:    mov v3.s[2], w11
+; CHECK-i32-NEXT:    fcvtzs w11, d6
+; CHECK-i32-NEXT:    mov v0.s[3], w8
+; CHECK-i32-NEXT:    mov v1.s[3], w9
+; CHECK-i32-NEXT:    mov v2.s[3], w10
+; CHECK-i32-NEXT:    mov v3.s[3], w11
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v16f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ptrue p1.d, vl2
+; CHECK-i64-NEXT:    // kill: def $q6 killed $q6 def $z6
+; CHECK-i64-NEXT:    // kill: def $q4 killed $q4 def $z4
+; CHECK-i64-NEXT:    // kill: def $q7 killed $q7 def $z7
+; CHECK-i64-NEXT:    // kill: def $q5 killed $q5 def $z5
+; CHECK-i64-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-i64-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i64-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-i64-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-i64-NEXT:    ptrue p0.d, vl4
+; CHECK-i64-NEXT:    splice z6.d, p1, z6.d, z7.d
+; CHECK-i64-NEXT:    splice z4.d, p1, z4.d, z5.d
+; CHECK-i64-NEXT:    splice z2.d, p1, z2.d, z3.d
+; CHECK-i64-NEXT:    splice z0.d, p1, z0.d, z1.d
+; CHECK-i64-NEXT:    movprfx z3, z6
+; CHECK-i64-NEXT:    frintx z3.d, p0/m, z6.d
+; CHECK-i64-NEXT:    movprfx z1, z4
+; CHECK-i64-NEXT:    frintx z1.d, p0/m, z4.d
+; CHECK-i64-NEXT:    frintx z2.d, p0/m, z2.d
+; CHECK-i64-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-i64-NEXT:    mov z4.d, z3.d[2]
+; CHECK-i64-NEXT:    mov z5.d, z1.d[2]
+; CHECK-i64-NEXT:    mov z6.d, z2.d[3]
+; CHECK-i64-NEXT:    fcvtzs x11, d0
+; CHECK-i64-NEXT:    fcvtzs x12, d1
+; CHECK-i64-NEXT:    fcvtzs x13, d2
+; CHECK-i64-NEXT:    fcvtzs x14, d3
+; CHECK-i64-NEXT:    mov z7.d, z3.d[3]
+; CHECK-i64-NEXT:    mov z16.d, z1.d[3]
+; CHECK-i64-NEXT:    fcvtzs x9, d4
+; CHECK-i64-NEXT:    fcvtzs x10, d5
+; CHECK-i64-NEXT:    mov z4.d, z2.d[2]
+; CHECK-i64-NEXT:    mov z5.d, z0.d[2]
+; CHECK-i64-NEXT:    fcvtzs x8, d6
+; CHECK-i64-NEXT:    mov z2.d, z2.d[1]
+; CHECK-i64-NEXT:    mov z6.d, z0.d[3]
+; CHECK-i64-NEXT:    mov z1.d, z1.d[1]
+; CHECK-i64-NEXT:    mov z3.d, z3.d[1]
+; CHECK-i64-NEXT:    fcvtzs x15, d4
+; CHECK-i64-NEXT:    mov z4.d, z0.d[1]
+; CHECK-i64-NEXT:    fmov d0, x11
+; CHECK-i64-NEXT:    fcvtzs x16, d5
+; CHECK-i64-NEXT:    fcvtzs x11, d2
+; CHECK-i64-NEXT:    fmov d2, x13
+; CHECK-i64-NEXT:    fcvtzs x17, d7
+; CHECK-i64-NEXT:    fcvtzs x18, d16
+; CHECK-i64-NEXT:    fcvtzs x0, d3
+; CHECK-i64-NEXT:    fcvtzs x13, d4
+; CHECK-i64-NEXT:    fmov d4, x12
+; CHECK-i64-NEXT:    fcvtzs x12, d6
+; CHECK-i64-NEXT:    fmov d6, x14
+; CHECK-i64-NEXT:    fcvtzs x14, d1
+; CHECK-i64-NEXT:    fmov d3, x15
+; CHECK-i64-NEXT:    fmov d1, x16
+; CHECK-i64-NEXT:    fmov d5, x10
+; CHECK-i64-NEXT:    fmov d7, x9
+; CHECK-i64-NEXT:    mov v2.d[1], x11
+; CHECK-i64-NEXT:    mov v0.d[1], x13
+; CHECK-i64-NEXT:    mov v3.d[1], x8
+; CHECK-i64-NEXT:    mov v6.d[1], x0
+; CHECK-i64-NEXT:    mov v4.d[1], x14
+; CHECK-i64-NEXT:    mov v1.d[1], x12
+; CHECK-i64-NEXT:    mov v5.d[1], x18
+; CHECK-i64-NEXT:    mov v7.d[1], x17
+; CHECK-i64-NEXT:    ret
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f64(<16 x double> %x)
+  ret <16 x iXLen> %a
+}
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f64(<16 x double>)
+
+define <32 x iXLen> @lrint_v32f64(<32 x double> %x) {
+; CHECK-i32-LABEL: lrint_v32f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    ptrue p1.d, vl2
+; CHECK-i32-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i32-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-i32-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-i32-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-i32-NEXT:    // kill: def $q4 killed $q4 def $z4
+; CHECK-i32-NEXT:    // kill: def $q5 killed $q5 def $z5
+; CHECK-i32-NEXT:    // kill: def $q7 killed $q7 def $z7
+; CHECK-i32-NEXT:    // kill: def $q6 killed $q6 def $z6
+; CHECK-i32-NEXT:    ptrue p0.d, vl4
+; CHECK-i32-NEXT:    splice z0.d, p1, z0.d, z1.d
+; CHECK-i32-NEXT:    splice z2.d, p1, z2.d, z3.d
+; CHECK-i32-NEXT:    splice z4.d, p1, z4.d, z5.d
+; CHECK-i32-NEXT:    ldp q1, q3, [sp]
+; CHECK-i32-NEXT:    splice z6.d, p1, z6.d, z7.d
+; CHECK-i32-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-i32-NEXT:    splice z1.d, p1, z1.d, z3.d
+; CHECK-i32-NEXT:    movprfx z18, z2
+; CHECK-i32-NEXT:    frintx z18.d, p0/m, z2.d
+; CHECK-i32-NEXT:    ldp q5, q3, [sp, #96]
+; CHECK-i32-NEXT:    ldp q2, q7, [sp, #64]
+; CHECK-i32-NEXT:    splice z5.d, p1, z5.d, z3.d
+; CHECK-i32-NEXT:    movprfx z3, z4
+; CHECK-i32-NEXT:    frintx z3.d, p0/m, z4.d
+; CHECK-i32-NEXT:    mov z4.d, z0.d[1]
+; CHECK-i32-NEXT:    fcvtzs w8, d0
+; CHECK-i32-NEXT:    splice z2.d, p1, z2.d, z7.d
+; CHECK-i32-NEXT:    mov z19.d, z18.d[1]
+; CHECK-i32-NEXT:    ldp q7, q16, [sp, #32]
+; CHECK-i32-NEXT:    movprfx z17, z1
+; CHECK-i32-NEXT:    frintx z17.d, p0/m, z1.d
+; CHECK-i32-NEXT:    fcvtzs w10, d4
+; CHECK-i32-NEXT:    mov z1.d, z0.d[2]
+; CHECK-i32-NEXT:    fcvtzs w9, d18
+; CHECK-i32-NEXT:    mov z4.d, z0.d[3]
+; CHECK-i32-NEXT:    fcvtzs w11, d19
+; CHECK-i32-NEXT:    mov z20.d, z18.d[3]
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    splice z7.d, p1, z7.d, z16.d
+; CHECK-i32-NEXT:    movprfx z16, z6
+; CHECK-i32-NEXT:    frintx z16.d, p0/m, z6.d
+; CHECK-i32-NEXT:    mov z6.d, z18.d[2]
+; CHECK-i32-NEXT:    mov z18.d, z3.d[1]
+; CHECK-i32-NEXT:    fcvtzs w12, d3
+; CHECK-i32-NEXT:    fcvtzs w13, d1
+; CHECK-i32-NEXT:    fmov s1, w9
+; CHECK-i32-NEXT:    movprfx z19, z2
+; CHECK-i32-NEXT:    frintx z19.d, p0/m, z2.d
+; CHECK-i32-NEXT:    mov v0.s[1], w10
+; CHECK-i32-NEXT:    mov z21.d, z3.d[2]
+; CHECK-i32-NEXT:    fcvtzs w8, d4
+; CHECK-i32-NEXT:    fcvtzs w14, d6
+; CHECK-i32-NEXT:    mov z6.d, z16.d[1]
+; CHECK-i32-NEXT:    fcvtzs w15, d18
+; CHECK-i32-NEXT:    movprfx z18, z7
+; CHECK-i32-NEXT:    frintx z18.d, p0/m, z7.d
+; CHECK-i32-NEXT:    mov v1.s[1], w11
+; CHECK-i32-NEXT:    fmov s2, w12
+; CHECK-i32-NEXT:    mov z7.d, z17.d[1]
+; CHECK-i32-NEXT:    mov z4.d, z16.d[2]
+; CHECK-i32-NEXT:    fcvtzs w16, d16
+; CHECK-i32-NEXT:    mov v0.s[2], w13
+; CHECK-i32-NEXT:    fcvtzs w13, d17
+; CHECK-i32-NEXT:    fcvtzs w12, d6
+; CHECK-i32-NEXT:    mov z6.d, z19.d[1]
+; CHECK-i32-NEXT:    fcvtzs w11, d21
+; CHECK-i32-NEXT:    movprfx z21, z5
+; CHECK-i32-NEXT:    frintx z21.d, p0/m, z5.d
+; CHECK-i32-NEXT:    mov z3.d, z3.d[3]
+; CHECK-i32-NEXT:    mov v2.s[1], w15
+; CHECK-i32-NEXT:    mov z5.d, z18.d[1]
+; CHECK-i32-NEXT:    fcvtzs w15, d7
+; CHECK-i32-NEXT:    fcvtzs w0, d19
+; CHECK-i32-NEXT:    mov v1.s[2], w14
+; CHECK-i32-NEXT:    fcvtzs w14, d4
+; CHECK-i32-NEXT:    mov z7.d, z18.d[2]
+; CHECK-i32-NEXT:    fmov s4, w13
+; CHECK-i32-NEXT:    fcvtzs w13, d6
+; CHECK-i32-NEXT:    mov z6.d, z19.d[2]
+; CHECK-i32-NEXT:    fcvtzs w10, d3
+; CHECK-i32-NEXT:    fmov s3, w16
+; CHECK-i32-NEXT:    fcvtzs w17, d18
+; CHECK-i32-NEXT:    fcvtzs w18, d5
+; CHECK-i32-NEXT:    mov z5.d, z21.d[1]
+; CHECK-i32-NEXT:    fcvtzs w2, d21
+; CHECK-i32-NEXT:    fcvtzs w1, d7
+; CHECK-i32-NEXT:    mov z7.d, z21.d[2]
+; CHECK-i32-NEXT:    mov v4.s[1], w15
+; CHECK-i32-NEXT:    fcvtzs w15, d6
+; CHECK-i32-NEXT:    fmov s6, w0
+; CHECK-i32-NEXT:    mov v3.s[1], w12
+; CHECK-i32-NEXT:    fcvtzs w9, d20
+; CHECK-i32-NEXT:    fcvtzs w12, d5
+; CHECK-i32-NEXT:    mov z20.d, z17.d[2]
+; CHECK-i32-NEXT:    fmov s5, w17
+; CHECK-i32-NEXT:    mov z16.d, z16.d[3]
+; CHECK-i32-NEXT:    mov z17.d, z17.d[3]
+; CHECK-i32-NEXT:    mov z18.d, z18.d[3]
+; CHECK-i32-NEXT:    mov v6.s[1], w13
+; CHECK-i32-NEXT:    fcvtzs w13, d7
+; CHECK-i32-NEXT:    fmov s7, w2
+; CHECK-i32-NEXT:    fcvtzs w16, d20
+; CHECK-i32-NEXT:    mov v5.s[1], w18
+; CHECK-i32-NEXT:    mov z19.d, z19.d[3]
+; CHECK-i32-NEXT:    mov z20.d, z21.d[3]
+; CHECK-i32-NEXT:    mov v2.s[2], w11
+; CHECK-i32-NEXT:    mov v3.s[2], w14
+; CHECK-i32-NEXT:    mov v7.s[1], w12
+; CHECK-i32-NEXT:    fcvtzs w11, d16
+; CHECK-i32-NEXT:    fcvtzs w12, d17
+; CHECK-i32-NEXT:    fcvtzs w14, d18
+; CHECK-i32-NEXT:    mov v6.s[2], w15
+; CHECK-i32-NEXT:    fcvtzs w15, d19
+; CHECK-i32-NEXT:    mov v4.s[2], w16
+; CHECK-i32-NEXT:    mov v5.s[2], w1
+; CHECK-i32-NEXT:    mov v0.s[3], w8
+; CHECK-i32-NEXT:    mov v1.s[3], w9
+; CHECK-i32-NEXT:    mov v2.s[3], w10
+; CHECK-i32-NEXT:    mov v7.s[2], w13
+; CHECK-i32-NEXT:    fcvtzs w13, d20
+; CHECK-i32-NEXT:    mov v3.s[3], w11
+; CHECK-i32-NEXT:    mov v6.s[3], w15
+; CHECK-i32-NEXT:    mov v4.s[3], w12
+; CHECK-i32-NEXT:    mov v5.s[3], w14
+; CHECK-i32-NEXT:    mov v7.s[3], w13
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v32f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-i64-NEXT:    sub x9, sp, #272
+; CHECK-i64-NEXT:    mov x29, sp
+; CHECK-i64-NEXT:    and sp, x9, #0xffffffffffffffe0
+; CHECK-i64-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-i64-NEXT:    .cfi_offset w30, -8
+; CHECK-i64-NEXT:    .cfi_offset w29, -16
+; CHECK-i64-NEXT:    ptrue p1.d, vl2
+; CHECK-i64-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i64-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-i64-NEXT:    // kill: def $q3 killed $q3 def $z3
+; CHECK-i64-NEXT:    // kill: def $q2 killed $q2 def $z2
+; CHECK-i64-NEXT:    // kill: def $q7 killed $q7 def $z7
+; CHECK-i64-NEXT:    // kill: def $q6 killed $q6 def $z6
+; CHECK-i64-NEXT:    // kill: def $q4 killed $q4 def $z4
+; CHECK-i64-NEXT:    // kill: def $q5 killed $q5 def $z5
+; CHECK-i64-NEXT:    ptrue p0.d, vl4
+; CHECK-i64-NEXT:    splice z0.d, p1, z0.d, z1.d
+; CHECK-i64-NEXT:    splice z2.d, p1, z2.d, z3.d
+; CHECK-i64-NEXT:    splice z4.d, p1, z4.d, z5.d
+; CHECK-i64-NEXT:    splice z6.d, p1, z6.d, z7.d
+; CHECK-i64-NEXT:    ldp q5, q19, [x29, #16]
+; CHECK-i64-NEXT:    movprfx z3, z0
+; CHECK-i64-NEXT:    frintx z3.d, p0/m, z0.d
+; CHECK-i64-NEXT:    movprfx z16, z2
+; CHECK-i64-NEXT:    frintx z16.d, p0/m, z2.d
+; CHECK-i64-NEXT:    frintx z4.d, p0/m, z4.d
+; CHECK-i64-NEXT:    splice z5.d, p1, z5.d, z19.d
+; CHECK-i64-NEXT:    frintx z6.d, p0/m, z6.d
+; CHECK-i64-NEXT:    ldp q2, q17, [x29, #48]
+; CHECK-i64-NEXT:    ldp q0, q1, [x29, #112]
+; CHECK-i64-NEXT:    mov z18.d, z3.d[3]
+; CHECK-i64-NEXT:    mov z7.d, z3.d[2]
+; CHECK-i64-NEXT:    fcvtzs x9, d3
+; CHECK-i64-NEXT:    mov z3.d, z3.d[1]
+; CHECK-i64-NEXT:    mov z20.d, z16.d[3]
+; CHECK-i64-NEXT:    fcvtzs x12, d16
+; CHECK-i64-NEXT:    splice z2.d, p1, z2.d, z17.d
+; CHECK-i64-NEXT:    frintx z5.d, p0/m, z5.d
+; CHECK-i64-NEXT:    splice z0.d, p1, z0.d, z1.d
+; CHECK-i64-NEXT:    fcvtzs x10, d18
+; CHECK-i64-NEXT:    fcvtzs x11, d7
+; CHECK-i64-NEXT:    mov z18.d, z16.d[2]
+; CHECK-i64-NEXT:    mov z7.d, z16.d[1]
+; CHECK-i64-NEXT:    fcvtzs x13, d3
+; CHECK-i64-NEXT:    fcvtzs x14, d20
+; CHECK-i64-NEXT:    str x9, [sp, #128]
+; CHECK-i64-NEXT:    mov z16.d, z4.d[3]
+; CHECK-i64-NEXT:    fcvtzs x9, d18
+; CHECK-i64-NEXT:    mov z18.d, z4.d[2]
+; CHECK-i64-NEXT:    frintx z2.d, p0/m, z2.d
+; CHECK-i64-NEXT:    stp x11, x10, [sp, #144]
+; CHECK-i64-NEXT:    fcvtzs x10, d7
+; CHECK-i64-NEXT:    mov z7.d, z4.d[1]
+; CHECK-i64-NEXT:    str x13, [sp, #136]
+; CHECK-i64-NEXT:    fcvtzs x11, d16
+; CHECK-i64-NEXT:    mov z16.d, z6.d[3]
+; CHECK-i64-NEXT:    fcvtzs x13, d18
+; CHECK-i64-NEXT:    ldp q3, q19, [x29, #80]
+; CHECK-i64-NEXT:    stp x9, x14, [sp, #176]
+; CHECK-i64-NEXT:    fcvtzs x9, d4
+; CHECK-i64-NEXT:    mov z4.d, z6.d[2]
+; CHECK-i64-NEXT:    stp x12, x10, [sp, #160]
+; CHECK-i64-NEXT:    fcvtzs x10, d7
+; CHECK-i64-NEXT:    mov z7.d, z6.d[1]
+; CHECK-i64-NEXT:    fcvtzs x12, d6
+; CHECK-i64-NEXT:    splice z3.d, p1, z3.d, z19.d
+; CHECK-i64-NEXT:    mov z6.d, z5.d[2]
+; CHECK-i64-NEXT:    stp x13, x11, [sp, #208]
+; CHECK-i64-NEXT:    fcvtzs x11, d16
+; CHECK-i64-NEXT:    fcvtzs x13, d4
+; CHECK-i64-NEXT:    mov z4.d, z5.d[3]
+; CHECK-i64-NEXT:    mov z1.d, z5.d[1]
+; CHECK-i64-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-i64-NEXT:    stp x9, x10, [sp, #192]
+; CHECK-i64-NEXT:    fcvtzs x9, d7
+; CHECK-i64-NEXT:    frintx z3.d, p0/m, z3.d
+; CHECK-i64-NEXT:    fcvtzs x10, d4
+; CHECK-i64-NEXT:    stp x13, x11, [sp, #240]
+; CHECK-i64-NEXT:    fcvtzs x11, d6
+; CHECK-i64-NEXT:    mov z4.d, z2.d[3]
+; CHECK-i64-NEXT:    fcvtzs x13, d2
+; CHECK-i64-NEXT:    stp x12, x9, [sp, #224]
+; CHECK-i64-NEXT:    fcvtzs x9, d5
+; CHECK-i64-NEXT:    fcvtzs x12, d1
+; CHECK-i64-NEXT:    mov z5.d, z2.d[2]
+; CHECK-i64-NEXT:    mov z1.d, z2.d[1]
+; CHECK-i64-NEXT:    mov z2.d, z3.d[2]
+; CHECK-i64-NEXT:    stp x11, x10, [sp, #16]
+; CHECK-i64-NEXT:    fcvtzs x10, d4
+; CHECK-i64-NEXT:    mov z4.d, z3.d[3]
+; CHECK-i64-NEXT:    fcvtzs x11, d5
+; CHECK-i64-NEXT:    stp x9, x12, [sp]
+; CHECK-i64-NEXT:    fcvtzs x9, d1
+; CHECK-i64-NEXT:    mov z1.d, z3.d[1]
+; CHECK-i64-NEXT:    fcvtzs x12, d4
+; CHECK-i64-NEXT:    stp x11, x10, [sp, #48]
+; CHECK-i64-NEXT:    fcvtzs x10, d2
+; CHECK-i64-NEXT:    fcvtzs x11, d3
+; CHECK-i64-NEXT:    stp x13, x9, [sp, #32]
+; CHECK-i64-NEXT:    fcvtzs x9, d1
+; CHECK-i64-NEXT:    mov z2.d, z0.d[3]
+; CHECK-i64-NEXT:    mov z3.d, z0.d[2]
+; CHECK-i64-NEXT:    mov z1.d, z0.d[1]
+; CHECK-i64-NEXT:    fcvtzs x13, d2
+; CHECK-i64-NEXT:    stp x10, x12, [sp, #80]
+; CHECK-i64-NEXT:    fcvtzs x12, d0
+; CHECK-i64-NEXT:    fcvtzs x10, d3
+; CHECK-i64-NEXT:    stp x11, x9, [sp, #64]
+; CHECK-i64-NEXT:    fcvtzs x9, d1
+; CHECK-i64-NEXT:    stp x10, x13, [sp, #112]
+; CHECK-i64-NEXT:    add x10, sp, #192
+; CHECK-i64-NEXT:    stp x12, x9, [sp, #96]
+; CHECK-i64-NEXT:    add x9, sp, #128
+; CHECK-i64-NEXT:    ld1d { z0.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    add x9, sp, #160
+; CHECK-i64-NEXT:    ld1d { z2.d }, p0/z, [x10]
+; CHECK-i64-NEXT:    ld1d { z1.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    add x9, sp, #96
+; CHECK-i64-NEXT:    add x10, sp, #224
+; CHECK-i64-NEXT:    ld1d { z3.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    add x9, sp, #64
+; CHECK-i64-NEXT:    ld1d { z4.d }, p0/z, [x10]
+; CHECK-i64-NEXT:    add x10, sp, #32
+; CHECK-i64-NEXT:    ld1d { z5.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    mov x9, sp
+; CHECK-i64-NEXT:    ld1d { z6.d }, p0/z, [x10]
+; CHECK-i64-NEXT:    mov x10, #28 // =0x1c
+; CHECK-i64-NEXT:    ld1d { z7.d }, p0/z, [x9]
+; CHECK-i64-NEXT:    mov x9, #24 // =0x18
+; CHECK-i64-NEXT:    st1d { z3.d }, p0, [x8, x10, lsl #3]
+; CHECK-i64-NEXT:    st1d { z5.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #20 // =0x14
+; CHECK-i64-NEXT:    st1d { z6.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #16 // =0x10
+; CHECK-i64-NEXT:    st1d { z7.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #12 // =0xc
+; CHECK-i64-NEXT:    st1d { z4.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #8 // =0x8
+; CHECK-i64-NEXT:    st1d { z2.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    mov x9, #4 // =0x4
+; CHECK-i64-NEXT:    st1d { z1.d }, p0, [x8, x9, lsl #3]
+; CHECK-i64-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-i64-NEXT:    mov sp, x29
+; CHECK-i64-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ret
+  %a = call <32 x iXLen> @llvm.lrint.v32iXLen.v16f64(<32 x double> %x)
+  ret <32 x iXLen> %a
+}
+declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f64(<32 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-llrint.ll b/llvm/test/CodeGen/AArch64/sve-llrint.ll
new file mode 100644
index 000000000000..a881af161201
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-llrint.ll
@@ -0,0 +1,1763 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve | FileCheck %s
+
+define <vscale x 1 x i64> @llrint_v1i64_v1f16(<vscale x 1 x half> %x) {
+; CHECK-LABEL: llrint_v1i64_v1f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
+; CHECK-NEXT:    mov z3.h, w8
+; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.h
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z0.h, z3.h
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f16(<vscale x 1 x half> %x)
+  ret <vscale x 1 x i64> %a
+}
+declare <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f16(<vscale x 1 x half>)
+
+define <vscale x 2 x i64> @llrint_v1i64_v2f16(<vscale x 2 x half> %x) {
+; CHECK-LABEL: llrint_v1i64_v2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
+; CHECK-NEXT:    mov z3.h, w8
+; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.h
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z0.h, z3.h
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f16(<vscale x 2 x half> %x)
+  ret <vscale x 2 x i64> %a
+}
+declare <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f16(<vscale x 2 x half>)
+
+define <vscale x 4 x i64> @llrint_v4i64_v4f16(<vscale x 4 x half> %x) {
+; CHECK-LABEL: llrint_v4i64_v4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    uunpklo z1.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z2.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    mov z3.h, w8
+; CHECK-NEXT:    mov z6.d, #0x7fffffffffffffff
+; CHECK-NEXT:    frintx z1.h, p0/m, z1.h
+; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
+; CHECK-NEXT:    fcmge p1.h, p0/z, z1.h, z2.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z0.h, z2.h
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z4, z1
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z1.h
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z0.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z1.h, z3.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z0.h, z3.h
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    sel z3.d, p1, z2.d, z4.d
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z1.h, z1.h
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
+; CHECK-NEXT:    sel z2.d, p2, z2.d, z5.d
+; CHECK-NEXT:    sel z0.d, p3, z6.d, z3.d
+; CHECK-NEXT:    sel z1.d, p4, z6.d, z2.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f16(<vscale x 4 x half> %x)
+  ret <vscale x 4 x i64> %a
+}
+declare <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f16(<vscale x 4 x half>)
+
+define <vscale x 8 x i64> @llrint_v8i64_v8f16(<vscale x 8 x half> %x) {
+; CHECK-LABEL: llrint_v8i64_v8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    uunpklo z1.s, z0.h
+; CHECK-NEXT:    uunpkhi z0.s, z0.h
+; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z4.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    mov z6.h, w8
+; CHECK-NEXT:    mov z26.d, #0x7fffffffffffffff
+; CHECK-NEXT:    uunpklo z2.d, z1.s
+; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    uunpklo z3.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    frintx z2.h, p0/m, z2.h
+; CHECK-NEXT:    frintx z1.h, p0/m, z1.h
+; CHECK-NEXT:    frintx z3.h, p0/m, z3.h
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    frintx z5.h, p0/m, z0.h
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p1.h, p0/z, z2.h, z4.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z1.h, z4.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z3.h, z4.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z5.h, z4.h
+; CHECK-NEXT:    movprfx z4, z2
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z2.h
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z1.h
+; CHECK-NEXT:    movprfx z24, z3
+; CHECK-NEXT:    fcvtzs z24.d, p0/m, z3.h
+; CHECK-NEXT:    movprfx z25, z5
+; CHECK-NEXT:    fcvtzs z25.d, p0/m, z5.h
+; CHECK-NEXT:    fcmgt p7.h, p0/z, z3.h, z6.h
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z2.h, z6.h
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z1.h, z6.h
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    mov z4.d, p1/m, z0.d
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z5.h, z6.h
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    sel z6.d, p2, z0.d, z7.d
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z2.h, z2.h
+; CHECK-NEXT:    sel z7.d, p3, z0.d, z24.d
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z1.h, z1.h
+; CHECK-NEXT:    sel z24.d, p4, z0.d, z25.d
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z3.h, z3.h
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z5.h, z5.h
+; CHECK-NEXT:    sel z0.d, p5, z26.d, z4.d
+; CHECK-NEXT:    sel z1.d, p6, z26.d, z6.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z2.d, p7, z26.d, z7.d
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z3.d, p1, z26.d, z24.d
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f16(<vscale x 8 x half> %x)
+  ret <vscale x 8 x i64> %a
+}
+declare <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f16(<vscale x 8 x half>)
+
+define <vscale x 16 x i64> @llrint_v16i64_v16f16(<vscale x 16 x half> %x) {
+; CHECK-LABEL: llrint_v16i64_v16f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    uunpklo z2.s, z0.h
+; CHECK-NEXT:    uunpkhi z0.s, z0.h
+; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    uunpklo z4.s, z1.h
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpkhi z1.s, z1.h
+; CHECK-NEXT:    mov z5.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    mov z25.d, #0x8000000000000000
+; CHECK-NEXT:    mov z27.h, w8
+; CHECK-NEXT:    mov z7.d, #0x7fffffffffffffff
+; CHECK-NEXT:    uunpklo z3.d, z2.s
+; CHECK-NEXT:    uunpkhi z2.d, z2.s
+; CHECK-NEXT:    uunpklo z6.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    uunpklo z24.d, z4.s
+; CHECK-NEXT:    uunpkhi z4.d, z4.s
+; CHECK-NEXT:    uunpklo z26.d, z1.s
+; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    frintx z2.h, p0/m, z2.h
+; CHECK-NEXT:    frintx z3.h, p0/m, z3.h
+; CHECK-NEXT:    frintx z6.h, p0/m, z6.h
+; CHECK-NEXT:    movprfx z28, z0
+; CHECK-NEXT:    frintx z28.h, p0/m, z0.h
+; CHECK-NEXT:    movprfx z29, z4
+; CHECK-NEXT:    frintx z29.h, p0/m, z4.h
+; CHECK-NEXT:    frintx z24.h, p0/m, z24.h
+; CHECK-NEXT:    movprfx z30, z1
+; CHECK-NEXT:    frintx z30.h, p0/m, z1.h
+; CHECK-NEXT:    frintx z26.h, p0/m, z26.h
+; CHECK-NEXT:    fcmge p5.h, p0/z, z2.h, z5.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z3.h, z5.h
+; CHECK-NEXT:    movprfx z1, z2
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z2.h
+; CHECK-NEXT:    movprfx z0, z3
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z3.h
+; CHECK-NEXT:    fcmge p6.h, p0/z, z6.h, z5.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z3.h, z27.h
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z3.h, z3.h
+; CHECK-NEXT:    fcmge p7.h, p0/z, z28.h, z5.h
+; CHECK-NEXT:    movprfx z3, z6
+; CHECK-NEXT:    fcvtzs z3.d, p0/m, z6.h
+; CHECK-NEXT:    fcmge p8.h, p0/z, z24.h, z5.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z2.h, z27.h
+; CHECK-NEXT:    fcmge p9.h, p0/z, z26.h, z5.h
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    movprfx z4, z24
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z24.h
+; CHECK-NEXT:    fcmge p10.h, p0/z, z30.h, z5.h
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    movprfx z31, z26
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z26.h
+; CHECK-NEXT:    movprfx z8, z30
+; CHECK-NEXT:    fcvtzs z8.d, p0/m, z30.h
+; CHECK-NEXT:    mov z1.d, p5/m, z25.d
+; CHECK-NEXT:    fcmge p5.h, p0/z, z29.h, z5.h
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    mov z0.d, p2/m, z25.d
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z2.h, z2.h
+; CHECK-NEXT:    movprfx z2, z28
+; CHECK-NEXT:    fcvtzs z2.d, p0/m, z28.h
+; CHECK-NEXT:    movprfx z5, z29
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z29.h
+; CHECK-NEXT:    not p7.b, p0/z, p7.b
+; CHECK-NEXT:    mov z3.d, p6/m, z25.d
+; CHECK-NEXT:    not p6.b, p0/z, p8.b
+; CHECK-NEXT:    fcmgt p8.h, p0/z, z6.h, z27.h
+; CHECK-NEXT:    mov z1.d, p4/m, z7.d
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    mov z0.d, p3/m, z7.d
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z29.h, z27.h
+; CHECK-NEXT:    sel z9.d, p7, z25.d, z2.d
+; CHECK-NEXT:    not p7.b, p0/z, p9.b
+; CHECK-NEXT:    mov z4.d, p6/m, z25.d
+; CHECK-NEXT:    not p6.b, p0/z, p10.b
+; CHECK-NEXT:    fcmgt p10.h, p0/z, z28.h, z27.h
+; CHECK-NEXT:    mov z5.d, p5/m, z25.d
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z24.h, z27.h
+; CHECK-NEXT:    fcmuo p9.h, p0/z, z6.h, z6.h
+; CHECK-NEXT:    sel z6.d, p7, z25.d, z31.d
+; CHECK-NEXT:    sel z25.d, p6, z25.d, z8.d
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z26.h, z27.h
+; CHECK-NEXT:    fcmgt p7.h, p0/z, z30.h, z27.h
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z28.h, z28.h
+; CHECK-NEXT:    sel z2.d, p8, z7.d, z3.d
+; CHECK-NEXT:    sel z3.d, p10, z7.d, z9.d
+; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    fcmuo p8.h, p0/z, z29.h, z29.h
+; CHECK-NEXT:    mov z4.d, p5/m, z7.d
+; CHECK-NEXT:    fcmuo p5.h, p0/z, z24.h, z24.h
+; CHECK-NEXT:    fcmuo p10.h, p0/z, z26.h, z26.h
+; CHECK-NEXT:    mov z5.d, p3/m, z7.d
+; CHECK-NEXT:    mov z6.d, p6/m, z7.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z30.h, z30.h
+; CHECK-NEXT:    sel z7.d, p7, z7.d, z25.d
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z6.d, p10/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f16(<vscale x 16 x half> %x)
+  ret <vscale x 16 x i64> %a
+}
+declare <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f16(<vscale x 16 x half>)
+
+define <vscale x 32 x i64> @llrint_v32i64_v32f16(<vscale x 32 x half> %x) {
+; CHECK-LABEL: llrint_v32i64_v32f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-17
+; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x88, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 136 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
+; CHECK-NEXT:    uunpkhi z5.s, z0.h
+; CHECK-NEXT:    uunpklo z4.s, z0.h
+; CHECK-NEXT:    mov w9, #64511 // =0xfbff
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpklo z6.s, z1.h
+; CHECK-NEXT:    mov z30.h, w9
+; CHECK-NEXT:    uunpkhi z10.s, z1.h
+; CHECK-NEXT:    mov w9, #31743 // =0x7bff
+; CHECK-NEXT:    mov z29.d, #0x8000000000000000
+; CHECK-NEXT:    uunpklo z8.s, z2.h
+; CHECK-NEXT:    uunpkhi z13.s, z3.h
+; CHECK-NEXT:    uunpklo z18.s, z3.h
+; CHECK-NEXT:    uunpklo z7.d, z5.s
+; CHECK-NEXT:    uunpklo z0.d, z4.s
+; CHECK-NEXT:    uunpkhi z4.d, z4.s
+; CHECK-NEXT:    uunpkhi z24.d, z5.s
+; CHECK-NEXT:    uunpklo z25.d, z6.s
+; CHECK-NEXT:    uunpkhi z26.d, z6.s
+; CHECK-NEXT:    uunpklo z27.d, z10.s
+; CHECK-NEXT:    uunpkhi z10.d, z10.s
+; CHECK-NEXT:    uunpklo z12.d, z8.s
+; CHECK-NEXT:    uunpkhi z16.d, z8.s
+; CHECK-NEXT:    movprfx z5, z7
+; CHECK-NEXT:    frintx z5.h, p0/m, z7.h
+; CHECK-NEXT:    movprfx z1, z4
+; CHECK-NEXT:    frintx z1.h, p0/m, z4.h
+; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
+; CHECK-NEXT:    movprfx z6, z24
+; CHECK-NEXT:    frintx z6.h, p0/m, z24.h
+; CHECK-NEXT:    movprfx z24, z25
+; CHECK-NEXT:    frintx z24.h, p0/m, z25.h
+; CHECK-NEXT:    movprfx z25, z26
+; CHECK-NEXT:    frintx z25.h, p0/m, z26.h
+; CHECK-NEXT:    movprfx z28, z27
+; CHECK-NEXT:    frintx z28.h, p0/m, z27.h
+; CHECK-NEXT:    movprfx z8, z10
+; CHECK-NEXT:    frintx z8.h, p0/m, z10.h
+; CHECK-NEXT:    mov z7.h, w9
+; CHECK-NEXT:    mov z4.d, #0x7fffffffffffffff
+; CHECK-NEXT:    rdvl x9, #15
+; CHECK-NEXT:    fcmge p3.h, p0/z, z5.h, z30.h
+; CHECK-NEXT:    movprfx z11, z5
+; CHECK-NEXT:    fcvtzs z11.d, p0/m, z5.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z1.h, z30.h
+; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, z30.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z6.h, z30.h
+; CHECK-NEXT:    movprfx z9, z6
+; CHECK-NEXT:    fcvtzs z9.d, p0/m, z6.h
+; CHECK-NEXT:    movprfx z15, z25
+; CHECK-NEXT:    fcvtzs z15.d, p0/m, z25.h
+; CHECK-NEXT:    movprfx z14, z24
+; CHECK-NEXT:    fcvtzs z14.d, p0/m, z24.h
+; CHECK-NEXT:    movprfx z26, z0
+; CHECK-NEXT:    fcvtzs z26.d, p0/m, z0.h
+; CHECK-NEXT:    movprfx z19, z28
+; CHECK-NEXT:    fcvtzs z19.d, p0/m, z28.h
+; CHECK-NEXT:    movprfx z31, z1
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z1.h
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    not p6.b, p0/z, p2.b
+; CHECK-NEXT:    fcmge p2.h, p0/z, z25.h, z30.h
+; CHECK-NEXT:    sel z27.d, p3, z29.d, z11.d
+; CHECK-NEXT:    uunpkhi z11.s, z2.h
+; CHECK-NEXT:    not p5.b, p0/z, p1.b
+; CHECK-NEXT:    fcmge p1.h, p0/z, z24.h, z30.h
+; CHECK-NEXT:    not p3.b, p0/z, p4.b
+; CHECK-NEXT:    fcmge p4.h, p0/z, z28.h, z30.h
+; CHECK-NEXT:    mov z26.d, p5/m, z29.d
+; CHECK-NEXT:    mov z31.d, p6/m, z29.d
+; CHECK-NEXT:    sel z2.d, p3, z29.d, z9.d
+; CHECK-NEXT:    movprfx z9, z12
+; CHECK-NEXT:    frintx z9.h, p0/m, z12.h
+; CHECK-NEXT:    uunpkhi z12.d, z13.s
+; CHECK-NEXT:    uunpklo z17.d, z11.s
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    sel z3.d, p2, z29.d, z15.d
+; CHECK-NEXT:    uunpklo z15.d, z13.s
+; CHECK-NEXT:    fcmge p2.h, p0/z, z8.h, z30.h
+; CHECK-NEXT:    sel z10.d, p1, z29.d, z14.d
+; CHECK-NEXT:    movprfx z14, z16
+; CHECK-NEXT:    frintx z14.h, p0/m, z16.h
+; CHECK-NEXT:    uunpkhi z16.d, z18.s
+; CHECK-NEXT:    movprfx z13, z17
+; CHECK-NEXT:    frintx z13.h, p0/m, z17.h
+; CHECK-NEXT:    movprfx z20, z12
+; CHECK-NEXT:    frintx z20.h, p0/m, z12.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z9.h, z30.h
+; CHECK-NEXT:    uunpkhi z17.d, z11.s
+; CHECK-NEXT:    uunpklo z18.d, z18.s
+; CHECK-NEXT:    movprfx z12, z8
+; CHECK-NEXT:    fcvtzs z12.d, p0/m, z8.h
+; CHECK-NEXT:    movprfx z21, z15
+; CHECK-NEXT:    frintx z21.h, p0/m, z15.h
+; CHECK-NEXT:    not p1.b, p0/z, p4.b
+; CHECK-NEXT:    movprfx z15, z9
+; CHECK-NEXT:    fcvtzs z15.d, p0/m, z9.h
+; CHECK-NEXT:    frintx z16.h, p0/m, z16.h
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    movprfx z22, z14
+; CHECK-NEXT:    fcvtzs z22.d, p0/m, z14.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z13.h, z30.h
+; CHECK-NEXT:    fcmge p5.h, p0/z, z20.h, z30.h
+; CHECK-NEXT:    sel z11.d, p1, z29.d, z19.d
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    frintx z17.h, p0/m, z17.h
+; CHECK-NEXT:    frintx z18.h, p0/m, z18.h
+; CHECK-NEXT:    movprfx z19, z20
+; CHECK-NEXT:    fcvtzs z19.d, p0/m, z20.h
+; CHECK-NEXT:    mov z12.d, p2/m, z29.d
+; CHECK-NEXT:    fcmge p2.h, p0/z, z21.h, z30.h
+; CHECK-NEXT:    fcmge p1.h, p0/z, z14.h, z30.h
+; CHECK-NEXT:    mov z15.d, p3/m, z29.d
+; CHECK-NEXT:    movprfx z23, z21
+; CHECK-NEXT:    fcvtzs z23.d, p0/m, z21.h
+; CHECK-NEXT:    not p3.b, p0/z, p4.b
+; CHECK-NEXT:    fcmge p4.h, p0/z, z16.h, z30.h
+; CHECK-NEXT:    fcmgt p8.h, p0/z, z21.h, z7.h
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    fcmge p6.h, p0/z, z17.h, z30.h
+; CHECK-NEXT:    fcmge p7.h, p0/z, z18.h, z30.h
+; CHECK-NEXT:    movprfx z30, z16
+; CHECK-NEXT:    fcvtzs z30.d, p0/m, z16.h
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    fcmuo p9.h, p0/z, z21.h, z21.h
+; CHECK-NEXT:    mov z19.d, p5/m, z29.d
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z20.h, z7.h
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z23.d, p2/m, z29.d
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z20.h, z20.h
+; CHECK-NEXT:    movprfx z20, z18
+; CHECK-NEXT:    fcvtzs z20.d, p0/m, z18.h
+; CHECK-NEXT:    movprfx z21, z13
+; CHECK-NEXT:    fcvtzs z21.d, p0/m, z13.h
+; CHECK-NEXT:    mov z22.d, p1/m, z29.d
+; CHECK-NEXT:    not p1.b, p0/z, p7.b
+; CHECK-NEXT:    mov z30.d, p4/m, z29.d
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z18.h, z7.h
+; CHECK-NEXT:    mov z19.d, p5/m, z4.d
+; CHECK-NEXT:    fcmuo p7.h, p0/z, z18.h, z18.h
+; CHECK-NEXT:    movprfx z18, z17
+; CHECK-NEXT:    fcvtzs z18.d, p0/m, z17.h
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z16.h, z7.h
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    mov z23.d, p8/m, z4.d
+; CHECK-NEXT:    mov z20.d, p1/m, z29.d
+; CHECK-NEXT:    mov z21.d, p3/m, z29.d
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z16.h, z16.h
+; CHECK-NEXT:    mov z19.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z17.h, z7.h
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    sel z29.d, p6, z29.d, z18.d
+; CHECK-NEXT:    mov z23.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z14.h, z7.h
+; CHECK-NEXT:    mov z30.d, p5/m, z4.d
+; CHECK-NEXT:    sel z16.d, p4, z4.d, z20.d
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z17.h, z17.h
+; CHECK-NEXT:    st1b { z19.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #14
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z1.h, z7.h
+; CHECK-NEXT:    st1b { z23.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #13
+; CHECK-NEXT:    mov z29.d, p2/m, z4.d
+; CHECK-NEXT:    mov z30.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z13.h, z7.h
+; CHECK-NEXT:    mov z16.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z9.h, z7.h
+; CHECK-NEXT:    fcmuo p7.h, p0/z, z14.h, z14.h
+; CHECK-NEXT:    mov z29.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z13.h, z13.h
+; CHECK-NEXT:    st1b { z30.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #12
+; CHECK-NEXT:    sel z30.d, p5, z4.d, z31.d
+; CHECK-NEXT:    st1b { z16.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #11
+; CHECK-NEXT:    sel z31.d, p3, z4.d, z21.d
+; CHECK-NEXT:    st1b { z29.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #10
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z24.h, z7.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z28.h, z7.h
+; CHECK-NEXT:    sel z13.d, p2, z4.d, z15.d
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z9.h, z9.h
+; CHECK-NEXT:    sel z29.d, p6, z4.d, z22.d
+; CHECK-NEXT:    mov z31.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z8.h, z7.h
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z5.h, z7.h
+; CHECK-NEXT:    sel z9.d, p5, z4.d, z10.d
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z6.h, z7.h
+; CHECK-NEXT:    st1b { z31.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #9
+; CHECK-NEXT:    mov z29.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    sel z10.d, p3, z4.d, z11.d
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z25.h, z7.h
+; CHECK-NEXT:    mov z13.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p7.h, p0/z, z8.h, z8.h
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z28.h, z28.h
+; CHECK-NEXT:    sel z28.d, p4, z4.d, z12.d
+; CHECK-NEXT:    st1b { z29.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #8
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z25.h, z25.h
+; CHECK-NEXT:    st1b { z13.b }, p1, [x8, x9]
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z24.h, z24.h
+; CHECK-NEXT:    mov z2.d, p5/m, z4.d
+; CHECK-NEXT:    mov z3.d, p3/m, z4.d
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z0.h, z7.h
+; CHECK-NEXT:    mov z28.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p7.h, p0/z, z6.h, z6.h
+; CHECK-NEXT:    mov z10.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z5.h, z5.h
+; CHECK-NEXT:    sel z5.d, p6, z4.d, z27.d
+; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z1.h, z1.h
+; CHECK-NEXT:    mov z9.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z28.d }, p0, [x8, #7, mul vl]
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z0.h, z0.h
+; CHECK-NEXT:    sel z0.d, p3, z4.d, z26.d
+; CHECK-NEXT:    st1d { z10.d }, p0, [x8, #6, mul vl]
+; CHECK-NEXT:    mov z2.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z3.d }, p0, [x8, #5, mul vl]
+; CHECK-NEXT:    mov z5.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z9.d }, p0, [x8, #4, mul vl]
+; CHECK-NEXT:    mov z30.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z2.d }, p0, [x8, #3, mul vl]
+; CHECK-NEXT:    st1d { z5.d }, p0, [x8, #2, mul vl]
+; CHECK-NEXT:    st1d { z30.d }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #17
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv32f16(<vscale x 32 x half> %x)
+  ret <vscale x 32 x i64> %a
+}
+declare <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv32f16(<vscale x 32 x half>)
+
+define <vscale x 1 x i64> @llrint_v1i64_v1f32(<vscale x 1 x float> %x) {
+; CHECK-LABEL: llrint_v1i64_v1f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-NEXT:    mov z3.s, w8
+; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.s
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z0.s, z3.s
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f32(<vscale x 1 x float> %x)
+  ret <vscale x 1 x i64> %a
+}
+declare <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f32(<vscale x 1 x float>)
+
+define <vscale x 2 x i64> @llrint_v2i64_v2f32(<vscale x 2 x float> %x) {
+; CHECK-LABEL: llrint_v2i64_v2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-NEXT:    mov z3.s, w8
+; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.s
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z0.s, z3.s
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f32(<vscale x 2 x float> %x)
+  ret <vscale x 2 x i64> %a
+}
+declare <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f32(<vscale x 2 x float>)
+
+define <vscale x 4 x i64> @llrint_v4i64_v4f32(<vscale x 4 x float> %x) {
+; CHECK-LABEL: llrint_v4i64_v4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    uunpklo z1.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z2.s, w8
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    mov z3.s, w8
+; CHECK-NEXT:    mov z6.d, #0x7fffffffffffffff
+; CHECK-NEXT:    frintx z1.s, p0/m, z1.s
+; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z1.s, z2.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z2.s
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z4, z1
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z1.s
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z0.s
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z1.s, z3.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z0.s, z3.s
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    sel z3.d, p1, z2.d, z4.d
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z1.s, z1.s
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    sel z2.d, p2, z2.d, z5.d
+; CHECK-NEXT:    sel z0.d, p3, z6.d, z3.d
+; CHECK-NEXT:    sel z1.d, p4, z6.d, z2.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f32(<vscale x 4 x float> %x)
+  ret <vscale x 4 x i64> %a
+}
+declare <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f32(<vscale x 4 x float>)
+
+define <vscale x 8 x i64> @llrint_v8i64_v8f32(<vscale x 8 x float> %x) {
+; CHECK-LABEL: llrint_v8i64_v8f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    uunpklo z2.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    uunpklo z3.d, z1.s
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    mov z4.s, w8
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    mov z5.d, #0x8000000000000000
+; CHECK-NEXT:    mov z6.s, w8
+; CHECK-NEXT:    mov z26.d, #0x7fffffffffffffff
+; CHECK-NEXT:    frintx z2.s, p0/m, z2.s
+; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-NEXT:    frintx z3.s, p0/m, z3.s
+; CHECK-NEXT:    frintx z1.s, p0/m, z1.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z2.s, z4.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z4.s
+; CHECK-NEXT:    movprfx z7, z0
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z0.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z3.s, z4.s
+; CHECK-NEXT:    fcmge p4.s, p0/z, z1.s, z4.s
+; CHECK-NEXT:    movprfx z4, z2
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z2.s
+; CHECK-NEXT:    movprfx z24, z3
+; CHECK-NEXT:    fcvtzs z24.d, p0/m, z3.s
+; CHECK-NEXT:    movprfx z25, z1
+; CHECK-NEXT:    fcvtzs z25.d, p0/m, z1.s
+; CHECK-NEXT:    fcmgt p7.s, p0/z, z3.s, z6.s
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z2.s, z6.s
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z0.s, z6.s
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    mov z4.d, p1/m, z5.d
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z6.s
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    sel z6.d, p2, z5.d, z7.d
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z2.s, z2.s
+; CHECK-NEXT:    sel z7.d, p3, z5.d, z24.d
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    sel z5.d, p4, z5.d, z25.d
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z3.s, z3.s
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z1.s, z1.s
+; CHECK-NEXT:    sel z0.d, p5, z26.d, z4.d
+; CHECK-NEXT:    sel z1.d, p6, z26.d, z6.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z2.d, p7, z26.d, z7.d
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z3.d, p1, z26.d, z5.d
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f32(<vscale x 8 x float> %x)
+  ret <vscale x 8 x i64> %a
+}
+declare <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f32(<vscale x 8 x float>)
+
+define <vscale x 16 x i64> @llrint_v16i64_v16f32(<vscale x 16 x float> %x) {
+; CHECK-LABEL: llrint_v16i64_v16f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-2
+; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    uunpklo z4.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpklo z7.d, z1.s
+; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    uunpklo z24.d, z2.s
+; CHECK-NEXT:    uunpkhi z2.d, z2.s
+; CHECK-NEXT:    uunpklo z25.d, z3.s
+; CHECK-NEXT:    uunpkhi z3.d, z3.s
+; CHECK-NEXT:    mov z26.d, #0x7fffffffffffffff
+; CHECK-NEXT:    movprfx z5, z4
+; CHECK-NEXT:    frintx z5.s, p0/m, z4.s
+; CHECK-NEXT:    movprfx z6, z0
+; CHECK-NEXT:    frintx z6.s, p0/m, z0.s
+; CHECK-NEXT:    mov z4.s, w8
+; CHECK-NEXT:    frintx z7.s, p0/m, z7.s
+; CHECK-NEXT:    movprfx z28, z1
+; CHECK-NEXT:    frintx z28.s, p0/m, z1.s
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    frintx z24.s, p0/m, z24.s
+; CHECK-NEXT:    movprfx z29, z2
+; CHECK-NEXT:    frintx z29.s, p0/m, z2.s
+; CHECK-NEXT:    frintx z25.s, p0/m, z25.s
+; CHECK-NEXT:    movprfx z30, z3
+; CHECK-NEXT:    frintx z30.s, p0/m, z3.s
+; CHECK-NEXT:    mov z27.s, w8
+; CHECK-NEXT:    fcmge p1.s, p0/z, z5.s, z4.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z6.s, z4.s
+; CHECK-NEXT:    movprfx z1, z5
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z5.s
+; CHECK-NEXT:    movprfx z2, z6
+; CHECK-NEXT:    fcvtzs z2.d, p0/m, z6.s
+; CHECK-NEXT:    fcmge p5.s, p0/z, z7.s, z4.s
+; CHECK-NEXT:    fcmge p6.s, p0/z, z28.s, z4.s
+; CHECK-NEXT:    movprfx z3, z7
+; CHECK-NEXT:    fcvtzs z3.d, p0/m, z7.s
+; CHECK-NEXT:    fcmge p8.s, p0/z, z29.s, z4.s
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z5.s, z27.s
+; CHECK-NEXT:    fcmgt p7.s, p0/z, z6.s, z27.s
+; CHECK-NEXT:    fcmge p9.s, p0/z, z25.s, z4.s
+; CHECK-NEXT:    movprfx z31, z25
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z25.s
+; CHECK-NEXT:    not p4.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z5.s, z5.s
+; CHECK-NEXT:    movprfx z5, z28
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z28.s
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    fcmge p10.s, p0/z, z30.s, z4.s
+; CHECK-NEXT:    movprfx z8, z30
+; CHECK-NEXT:    fcvtzs z8.d, p0/m, z30.s
+; CHECK-NEXT:    mov z1.d, p4/m, z0.d
+; CHECK-NEXT:    fcmge p4.s, p0/z, z24.s, z4.s
+; CHECK-NEXT:    movprfx z4, z29
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z29.s
+; CHECK-NEXT:    mov z2.d, p2/m, z0.d
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z6.s, z6.s
+; CHECK-NEXT:    movprfx z6, z24
+; CHECK-NEXT:    fcvtzs z6.d, p0/m, z24.s
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z3.d, p5/m, z0.d
+; CHECK-NEXT:    not p5.b, p0/z, p8.b
+; CHECK-NEXT:    mov z5.d, p6/m, z0.d
+; CHECK-NEXT:    fcmgt p8.s, p0/z, z7.s, z27.s
+; CHECK-NEXT:    not p6.b, p0/z, p9.b
+; CHECK-NEXT:    mov z6.d, p4/m, z0.d
+; CHECK-NEXT:    fcmuo p9.s, p0/z, z7.s, z7.s
+; CHECK-NEXT:    not p4.b, p0/z, p10.b
+; CHECK-NEXT:    fcmgt p10.s, p0/z, z28.s, z27.s
+; CHECK-NEXT:    sel z7.d, p5, z0.d, z4.d
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z24.s, z27.s
+; CHECK-NEXT:    mov z31.d, p6/m, z0.d
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z30.s, z27.s
+; CHECK-NEXT:    mov z8.d, p4/m, z0.d
+; CHECK-NEXT:    sel z0.d, p3, z26.d, z1.d
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z29.s, z27.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z25.s, z27.s
+; CHECK-NEXT:    sel z1.d, p7, z26.d, z2.d
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z28.s, z28.s
+; CHECK-NEXT:    sel z2.d, p8, z26.d, z3.d
+; CHECK-NEXT:    sel z3.d, p10, z26.d, z5.d
+; CHECK-NEXT:    fcmuo p8.s, p0/z, z29.s, z29.s
+; CHECK-NEXT:    sel z4.d, p5, z26.d, z6.d
+; CHECK-NEXT:    fcmuo p5.s, p0/z, z24.s, z24.s
+; CHECK-NEXT:    fcmuo p10.s, p0/z, z25.s, z25.s
+; CHECK-NEXT:    sel z5.d, p3, z26.d, z7.d
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z30.s, z30.s
+; CHECK-NEXT:    sel z7.d, p6, z26.d, z8.d
+; CHECK-NEXT:    ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z6.d, p4, z26.d, z31.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    mov z3.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z6.d, p10/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float> %x)
+  ret <vscale x 16 x i64> %a
+}
+declare <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float>)
+
+define <vscale x 32 x i64> @llrint_v32i64_v32f32(<vscale x 32 x float> %x) {
+; CHECK-LABEL: llrint_v32i64_v32f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-17
+; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x88, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 136 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
+; CHECK-NEXT:    uunpklo z24.d, z0.s
+; CHECK-NEXT:    uunpkhi z25.d, z0.s
+; CHECK-NEXT:    mov w9, #-553648128 // =0xdf000000
+; CHECK-NEXT:    uunpklo z26.d, z1.s
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpkhi z27.d, z1.s
+; CHECK-NEXT:    mov z31.s, w9
+; CHECK-NEXT:    mov w9, #1593835519 // =0x5effffff
+; CHECK-NEXT:    uunpklo z28.d, z2.s
+; CHECK-NEXT:    mov z8.d, #0x8000000000000000
+; CHECK-NEXT:    uunpklo z30.d, z3.s
+; CHECK-NEXT:    uunpklo z13.d, z4.s
+; CHECK-NEXT:    movprfx z0, z24
+; CHECK-NEXT:    frintx z0.s, p0/m, z24.s
+; CHECK-NEXT:    movprfx z1, z25
+; CHECK-NEXT:    frintx z1.s, p0/m, z25.s
+; CHECK-NEXT:    uunpkhi z15.d, z4.s
+; CHECK-NEXT:    movprfx z24, z26
+; CHECK-NEXT:    frintx z24.s, p0/m, z26.s
+; CHECK-NEXT:    uunpkhi z26.d, z2.s
+; CHECK-NEXT:    movprfx z25, z27
+; CHECK-NEXT:    frintx z25.s, p0/m, z27.s
+; CHECK-NEXT:    movprfx z27, z28
+; CHECK-NEXT:    frintx z27.s, p0/m, z28.s
+; CHECK-NEXT:    uunpklo z16.d, z5.s
+; CHECK-NEXT:    uunpkhi z17.d, z7.s
+; CHECK-NEXT:    frintx z30.s, p0/m, z30.s
+; CHECK-NEXT:    uunpklo z18.d, z7.s
+; CHECK-NEXT:    uunpklo z21.d, z6.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z31.s
+; CHECK-NEXT:    movprfx z9, z0
+; CHECK-NEXT:    fcvtzs z9.d, p0/m, z0.s
+; CHECK-NEXT:    movprfx z10, z1
+; CHECK-NEXT:    fcvtzs z10.d, p0/m, z1.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z1.s, z31.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z24.s, z31.s
+; CHECK-NEXT:    movprfx z11, z24
+; CHECK-NEXT:    fcvtzs z11.d, p0/m, z24.s
+; CHECK-NEXT:    movprfx z29, z26
+; CHECK-NEXT:    frintx z29.s, p0/m, z26.s
+; CHECK-NEXT:    fcmge p4.s, p0/z, z25.s, z31.s
+; CHECK-NEXT:    fcmge p5.s, p0/z, z27.s, z31.s
+; CHECK-NEXT:    movprfx z12, z27
+; CHECK-NEXT:    fcvtzs z12.d, p0/m, z27.s
+; CHECK-NEXT:    movprfx z19, z30
+; CHECK-NEXT:    fcvtzs z19.d, p0/m, z30.s
+; CHECK-NEXT:    movprfx z7, z16
+; CHECK-NEXT:    frintx z7.s, p0/m, z16.s
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    frintx z17.s, p0/m, z17.s
+; CHECK-NEXT:    uunpkhi z16.d, z5.s
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    frintx z18.s, p0/m, z18.s
+; CHECK-NEXT:    mov z28.s, w9
+; CHECK-NEXT:    not p6.b, p0/z, p3.b
+; CHECK-NEXT:    sel z26.d, p1, z8.d, z9.d
+; CHECK-NEXT:    movprfx z14, z29
+; CHECK-NEXT:    fcvtzs z14.d, p0/m, z29.s
+; CHECK-NEXT:    sel z9.d, p2, z8.d, z10.d
+; CHECK-NEXT:    uunpkhi z10.d, z3.s
+; CHECK-NEXT:    rdvl x9, #15
+; CHECK-NEXT:    sel z3.d, p6, z8.d, z11.d
+; CHECK-NEXT:    movprfx z11, z25
+; CHECK-NEXT:    fcvtzs z11.d, p0/m, z25.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z29.s, z31.s
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    fcmge p1.s, p0/z, z30.s, z31.s
+; CHECK-NEXT:    movprfx z23, z18
+; CHECK-NEXT:    fcvtzs z23.d, p0/m, z18.s
+; CHECK-NEXT:    not p2.b, p0/z, p5.b
+; CHECK-NEXT:    fcmge p5.s, p0/z, z17.s, z31.s
+; CHECK-NEXT:    frintx z16.s, p0/m, z16.s
+; CHECK-NEXT:    frintx z10.s, p0/m, z10.s
+; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcmgt p8.s, p0/z, z18.s, z28.s
+; CHECK-NEXT:    sel z4.d, p4, z8.d, z11.d
+; CHECK-NEXT:    movprfx z11, z13
+; CHECK-NEXT:    frintx z11.s, p0/m, z13.s
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    sel z13.d, p2, z8.d, z12.d
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmge p4.s, p0/z, z7.s, z31.s
+; CHECK-NEXT:    sel z12.d, p3, z8.d, z14.d
+; CHECK-NEXT:    movprfx z14, z15
+; CHECK-NEXT:    frintx z14.s, p0/m, z15.s
+; CHECK-NEXT:    uunpkhi z15.d, z6.s
+; CHECK-NEXT:    movprfx z20, z10
+; CHECK-NEXT:    fcvtzs z20.d, p0/m, z10.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z10.s, z31.s
+; CHECK-NEXT:    sel z5.d, p1, z8.d, z19.d
+; CHECK-NEXT:    movprfx z19, z11
+; CHECK-NEXT:    fcvtzs z19.d, p0/m, z11.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z11.s, z31.s
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    fcmge p6.s, p0/z, z16.s, z31.s
+; CHECK-NEXT:    fcmuo p9.s, p0/z, z18.s, z18.s
+; CHECK-NEXT:    movprfx z22, z15
+; CHECK-NEXT:    frintx z22.s, p0/m, z15.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z14.s, z31.s
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    sel z6.d, p2, z8.d, z20.d
+; CHECK-NEXT:    movprfx z20, z21
+; CHECK-NEXT:    frintx z20.s, p0/m, z21.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z18.s, z31.s
+; CHECK-NEXT:    sel z15.d, p3, z8.d, z19.d
+; CHECK-NEXT:    movprfx z19, z17
+; CHECK-NEXT:    fcvtzs z19.d, p0/m, z17.s
+; CHECK-NEXT:    not p3.b, p0/z, p4.b
+; CHECK-NEXT:    fcmge p4.s, p0/z, z22.s, z31.s
+; CHECK-NEXT:    movprfx z21, z14
+; CHECK-NEXT:    fcvtzs z21.d, p0/m, z14.s
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    movprfx z18, z7
+; CHECK-NEXT:    fcvtzs z18.d, p0/m, z7.s
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    fcmge p7.s, p0/z, z20.s, z31.s
+; CHECK-NEXT:    movprfx z31, z22
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z22.s
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    mov z19.d, p5/m, z8.d
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z17.s, z28.s
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z23.d, p2/m, z8.d
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z17.s, z17.s
+; CHECK-NEXT:    movprfx z17, z20
+; CHECK-NEXT:    fcvtzs z17.d, p0/m, z20.s
+; CHECK-NEXT:    mov z21.d, p1/m, z8.d
+; CHECK-NEXT:    mov z18.d, p3/m, z8.d
+; CHECK-NEXT:    not p1.b, p0/z, p7.b
+; CHECK-NEXT:    mov z31.d, p4/m, z8.d
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z20.s, z28.s
+; CHECK-NEXT:    mov z19.d, p5/m, z2.d
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z20.s, z20.s
+; CHECK-NEXT:    movprfx z20, z16
+; CHECK-NEXT:    fcvtzs z20.d, p0/m, z16.s
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z22.s, z28.s
+; CHECK-NEXT:    mov z23.d, p8/m, z2.d
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z22.s, z22.s
+; CHECK-NEXT:    mov z17.d, p1/m, z8.d
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    mov z19.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z16.s, z28.s
+; CHECK-NEXT:    sel z8.d, p6, z8.d, z20.d
+; CHECK-NEXT:    mov z23.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z14.s, z28.s
+; CHECK-NEXT:    mov z31.d, p5/m, z2.d
+; CHECK-NEXT:    mov z17.d, p4/m, z2.d
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z16.s, z16.s
+; CHECK-NEXT:    st1b { z19.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #14
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z1.s, z28.s
+; CHECK-NEXT:    st1b { z23.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #13
+; CHECK-NEXT:    mov z8.d, p2/m, z2.d
+; CHECK-NEXT:    mov z31.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z7.s, z28.s
+; CHECK-NEXT:    mov z17.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z11.s, z28.s
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z14.s, z14.s
+; CHECK-NEXT:    mov z8.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z7.s, z7.s
+; CHECK-NEXT:    sel z7.d, p5, z2.d, z9.d
+; CHECK-NEXT:    st1b { z31.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #12
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z27.s, z28.s
+; CHECK-NEXT:    st1b { z17.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #11
+; CHECK-NEXT:    sel z31.d, p3, z2.d, z18.d
+; CHECK-NEXT:    st1b { z8.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #10
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z30.s, z28.s
+; CHECK-NEXT:    sel z9.d, p2, z2.d, z15.d
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z11.s, z11.s
+; CHECK-NEXT:    sel z8.d, p6, z2.d, z21.d
+; CHECK-NEXT:    mov z31.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z10.s, z28.s
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z24.s, z28.s
+; CHECK-NEXT:    sel z11.d, p5, z2.d, z13.d
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z25.s, z28.s
+; CHECK-NEXT:    mov z8.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    mov z5.d, p3/m, z2.d
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z29.s, z28.s
+; CHECK-NEXT:    st1b { z31.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #9
+; CHECK-NEXT:    mov z9.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z10.s, z10.s
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z30.s, z30.s
+; CHECK-NEXT:    mov z6.d, p4/m, z2.d
+; CHECK-NEXT:    st1b { z8.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #8
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z29.s, z29.s
+; CHECK-NEXT:    st1b { z9.b }, p1, [x8, x9]
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z27.s, z27.s
+; CHECK-NEXT:    sel z27.d, p3, z2.d, z12.d
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z0.s, z28.s
+; CHECK-NEXT:    mov z4.d, p5/m, z2.d
+; CHECK-NEXT:    mov z3.d, p6/m, z2.d
+; CHECK-NEXT:    mov z6.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z25.s, z25.s
+; CHECK-NEXT:    mov z5.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z24.s, z24.s
+; CHECK-NEXT:    mov z27.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z1.s, z1.s
+; CHECK-NEXT:    mov z11.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    st1d { z6.d }, p0, [x8, #7, mul vl]
+; CHECK-NEXT:    sel z0.d, p3, z2.d, z26.d
+; CHECK-NEXT:    st1d { z5.d }, p0, [x8, #6, mul vl]
+; CHECK-NEXT:    mov z4.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z27.d }, p0, [x8, #5, mul vl]
+; CHECK-NEXT:    mov z3.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z7.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z11.d }, p0, [x8, #4, mul vl]
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z4.d }, p0, [x8, #3, mul vl]
+; CHECK-NEXT:    st1d { z3.d }, p0, [x8, #2, mul vl]
+; CHECK-NEXT:    st1d { z7.d }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #17
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv32f32(<vscale x 32 x float> %x)
+  ret <vscale x 32 x i64> %a
+}
+declare <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv32f32(<vscale x 32 x float>)
+
+define <vscale x 1 x i64> @llrint_v1i64_v1f64(<vscale x 1 x double> %x) {
+; CHECK-LABEL: llrint_v1i64_v1f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    mov z3.d, x8
+; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z0.d, z3.d
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f64(<vscale x 1 x double> %x)
+  ret <vscale x 1 x i64> %a
+}
+declare <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f64(<vscale x 1 x double>)
+
+define <vscale x 2 x i64> @llrint_v2i64_v2f64(<vscale x 2 x double> %x) {
+; CHECK-LABEL: llrint_v2i64_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    mov z3.d, x8
+; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z0.d, z3.d
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f64(<vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %a
+}
+declare <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f64(<vscale x 2 x double>)
+
+define <vscale x 4 x i64> @llrint_v4i64_v4f64(<vscale x 4 x double> %x) {
+; CHECK-LABEL: llrint_v4i64_v4f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z6.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z2.d, x8
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
+; CHECK-NEXT:    mov z3.d, x8
+; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z2.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z2.d
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z5, z1
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z1.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z0.d, z3.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z1.d, z3.d
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    sel z3.d, p1, z2.d, z4.d
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    sel z2.d, p2, z2.d, z5.d
+; CHECK-NEXT:    sel z0.d, p3, z6.d, z3.d
+; CHECK-NEXT:    sel z1.d, p4, z6.d, z2.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f64(<vscale x 4 x double> %x)
+  ret <vscale x 4 x i64> %a
+}
+declare <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f64(<vscale x 4 x double>)
+
+define <vscale x 8 x i64> @llrint_v8i64_v8f64(<vscale x 8 x double> %x) {
+; CHECK-LABEL: llrint_v8i64_v8f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z5.d, #0x8000000000000000
+; CHECK-NEXT:    mov z4.d, x8
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    mov z26.d, #0x7fffffffffffffff
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
+; CHECK-NEXT:    frintx z2.d, p0/m, z2.d
+; CHECK-NEXT:    frintx z3.d, p0/m, z3.d
+; CHECK-NEXT:    mov z6.d, x8
+; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z4.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z4.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z2.d, z4.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z3.d, z4.d
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z1.d
+; CHECK-NEXT:    movprfx z24, z2
+; CHECK-NEXT:    fcvtzs z24.d, p0/m, z2.d
+; CHECK-NEXT:    movprfx z25, z3
+; CHECK-NEXT:    fcvtzs z25.d, p0/m, z3.d
+; CHECK-NEXT:    fcmgt p7.d, p0/z, z2.d, z6.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z0.d, z6.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z1.d, z6.d
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    mov z4.d, p1/m, z5.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z3.d, z6.d
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    sel z6.d, p2, z5.d, z7.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    sel z7.d, p3, z5.d, z24.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    sel z5.d, p4, z5.d, z25.d
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z3.d, z3.d
+; CHECK-NEXT:    sel z0.d, p5, z26.d, z4.d
+; CHECK-NEXT:    sel z1.d, p6, z26.d, z6.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z2.d, p7, z26.d, z7.d
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z3.d, p1, z26.d, z5.d
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f64(<vscale x 8 x double> %x)
+  ret <vscale x 8 x i64> %a
+}
+declare <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f64(<vscale x 8 x double>)
+
+define <vscale x 16 x i64> @llrint_v16f64(<vscale x 16 x double> %x) {
+; CHECK-LABEL: llrint_v16f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-2
+; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z24.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z25.d, x8
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    movprfx z26, z0
+; CHECK-NEXT:    frintx z26.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z27, z1
+; CHECK-NEXT:    frintx z27.d, p0/m, z1.d
+; CHECK-NEXT:    frintx z2.d, p0/m, z2.d
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    frintx z3.d, p0/m, z3.d
+; CHECK-NEXT:    movprfx z28, z4
+; CHECK-NEXT:    frintx z28.d, p0/m, z4.d
+; CHECK-NEXT:    frintx z5.d, p0/m, z5.d
+; CHECK-NEXT:    frintx z6.d, p0/m, z6.d
+; CHECK-NEXT:    frintx z7.d, p0/m, z7.d
+; CHECK-NEXT:    fcmge p1.d, p0/z, z26.d, z25.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z27.d, z25.d
+; CHECK-NEXT:    movprfx z4, z26
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z26.d
+; CHECK-NEXT:    fcmge p5.d, p0/z, z2.d, z25.d
+; CHECK-NEXT:    movprfx z29, z27
+; CHECK-NEXT:    fcvtzs z29.d, p0/m, z27.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z26.d, z1.d
+; CHECK-NEXT:    fcmge p6.d, p0/z, z3.d, z25.d
+; CHECK-NEXT:    fcmge p8.d, p0/z, z5.d, z25.d
+; CHECK-NEXT:    fcmgt p7.d, p0/z, z27.d, z1.d
+; CHECK-NEXT:    fcmge p9.d, p0/z, z6.d, z25.d
+; CHECK-NEXT:    movprfx z30, z28
+; CHECK-NEXT:    fcvtzs z30.d, p0/m, z28.d
+; CHECK-NEXT:    fcmge p10.d, p0/z, z7.d, z25.d
+; CHECK-NEXT:    not p4.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z26.d, z26.d
+; CHECK-NEXT:    movprfx z26, z2
+; CHECK-NEXT:    fcvtzs z26.d, p0/m, z2.d
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    movprfx z31, z6
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z6.d
+; CHECK-NEXT:    movprfx z8, z7
+; CHECK-NEXT:    fcvtzs z8.d, p0/m, z7.d
+; CHECK-NEXT:    mov z4.d, p4/m, z0.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z28.d, z25.d
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    mov z29.d, p2/m, z0.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z27.d, z27.d
+; CHECK-NEXT:    movprfx z27, z3
+; CHECK-NEXT:    fcvtzs z27.d, p0/m, z3.d
+; CHECK-NEXT:    sel z25.d, p5, z0.d, z26.d
+; CHECK-NEXT:    movprfx z26, z5
+; CHECK-NEXT:    fcvtzs z26.d, p0/m, z5.d
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    not p5.b, p0/z, p8.b
+; CHECK-NEXT:    fcmgt p8.d, p0/z, z2.d, z1.d
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z27.d, p6/m, z0.d
+; CHECK-NEXT:    not p6.b, p0/z, p9.b
+; CHECK-NEXT:    fcmuo p9.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    mov z30.d, p4/m, z0.d
+; CHECK-NEXT:    not p4.b, p0/z, p10.b
+; CHECK-NEXT:    fcmgt p10.d, p0/z, z3.d, z1.d
+; CHECK-NEXT:    mov z26.d, p5/m, z0.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z28.d, z1.d
+; CHECK-NEXT:    mov z31.d, p6/m, z0.d
+; CHECK-NEXT:    mov z8.d, p4/m, z0.d
+; CHECK-NEXT:    sel z0.d, p3, z24.d, z4.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z5.d, z1.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z6.d, z1.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z7.d, z1.d
+; CHECK-NEXT:    sel z1.d, p7, z24.d, z29.d
+; CHECK-NEXT:    fcmuo p7.d, p0/z, z3.d, z3.d
+; CHECK-NEXT:    sel z2.d, p8, z24.d, z25.d
+; CHECK-NEXT:    sel z3.d, p10, z24.d, z27.d
+; CHECK-NEXT:    sel z4.d, p5, z24.d, z30.d
+; CHECK-NEXT:    fcmuo p5.d, p0/z, z28.d, z28.d
+; CHECK-NEXT:    fcmuo p8.d, p0/z, z5.d, z5.d
+; CHECK-NEXT:    fcmuo p10.d, p0/z, z6.d, z6.d
+; CHECK-NEXT:    sel z5.d, p3, z24.d, z26.d
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z7.d, z7.d
+; CHECK-NEXT:    sel z6.d, p4, z24.d, z31.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z7.d, p6, z24.d, z8.d
+; CHECK-NEXT:    ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    mov z6.d, p10/m, #0 // =0x0
+; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f64(<vscale x 16 x double> %x)
+  ret <vscale x 16 x i64> %a
+}
+declare <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f64(<vscale x 16 x double>)
+
+define <vscale x 32 x i64> @llrint_v32f64(<vscale x 32 x double> %x) {
+; CHECK-LABEL: llrint_v32f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-12
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xe0, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 96 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    rdvl x9, #8
+; CHECK-NEXT:    rdvl x10, #9
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    rdvl x11, #10
+; CHECK-NEXT:    mov x12, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    ld1b { z0.b }, p1/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z1.b }, p1/z, [x0, x10]
+; CHECK-NEXT:    mov z2.d, x12
+; CHECK-NEXT:    rdvl x14, #13
+; CHECK-NEXT:    rdvl x13, #12
+; CHECK-NEXT:    rdvl x12, #11
+; CHECK-NEXT:    ld1b { z6.b }, p1/z, [x0, x14]
+; CHECK-NEXT:    ld1b { z7.b }, p1/z, [x0, x13]
+; CHECK-NEXT:    mov z3.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z24, z0
+; CHECK-NEXT:    frintx z24.d, p0/m, z0.d
+; CHECK-NEXT:    ld1b { z0.b }, p1/z, [x0, x11]
+; CHECK-NEXT:    movprfx z5, z1
+; CHECK-NEXT:    frintx z5.d, p0/m, z1.d
+; CHECK-NEXT:    ld1b { z1.b }, p1/z, [x0, x12]
+; CHECK-NEXT:    mov x15, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    rdvl x16, #15
+; CHECK-NEXT:    movprfx z30, z6
+; CHECK-NEXT:    frintx z30.d, p0/m, z6.d
+; CHECK-NEXT:    movprfx z28, z7
+; CHECK-NEXT:    frintx z28.d, p0/m, z7.d
+; CHECK-NEXT:    ld1b { z8.b }, p1/z, [x0, x16]
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    frintx z4.d, p0/m, z0.d
+; CHECK-NEXT:    mov z0.d, #0x7fffffffffffffff
+; CHECK-NEXT:    ld1d { z18.d }, p0/z, [x0]
+; CHECK-NEXT:    fcmge p3.d, p0/z, z5.d, z2.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z24.d, z2.d
+; CHECK-NEXT:    movprfx z6, z5
+; CHECK-NEXT:    fcvtzs z6.d, p0/m, z5.d
+; CHECK-NEXT:    movprfx z27, z1
+; CHECK-NEXT:    frintx z27.d, p0/m, z1.d
+; CHECK-NEXT:    movprfx z25, z24
+; CHECK-NEXT:    fcvtzs z25.d, p0/m, z24.d
+; CHECK-NEXT:    mov z1.d, x15
+; CHECK-NEXT:    rdvl x15, #14
+; CHECK-NEXT:    movprfx z9, z28
+; CHECK-NEXT:    fcvtzs z9.d, p0/m, z28.d
+; CHECK-NEXT:    movprfx z13, z8
+; CHECK-NEXT:    frintx z13.d, p0/m, z8.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z4.d, z2.d
+; CHECK-NEXT:    movprfx z7, z4
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z4.d
+; CHECK-NEXT:    ld1d { z15.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z24.d, z1.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z5.d, z1.d
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    fcmge p7.d, p0/z, z27.d, z2.d
+; CHECK-NEXT:    movprfx z26, z27
+; CHECK-NEXT:    fcvtzs z26.d, p0/m, z27.d
+; CHECK-NEXT:    sel z29.d, p3, z3.d, z6.d
+; CHECK-NEXT:    ld1b { z6.b }, p1/z, [x0, x15]
+; CHECK-NEXT:    fcmge p3.d, p0/z, z28.d, z2.d
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z25.d, p2/m, z3.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z4.d, z1.d
+; CHECK-NEXT:    movprfx z16, z13
+; CHECK-NEXT:    fcvtzs z16.d, p0/m, z13.d
+; CHECK-NEXT:    ld1d { z17.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1d { z14.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT:    sel z31.d, p4, z3.d, z7.d
+; CHECK-NEXT:    movprfx z11, z6
+; CHECK-NEXT:    frintx z11.d, p0/m, z6.d
+; CHECK-NEXT:    not p7.b, p0/z, p7.b
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    sel z6.d, p5, z0.d, z25.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z27.d, z1.d
+; CHECK-NEXT:    sel z7.d, p6, z0.d, z29.d
+; CHECK-NEXT:    mov z26.d, p7/m, z3.d
+; CHECK-NEXT:    fcmge p5.d, p0/z, z13.d, z2.d
+; CHECK-NEXT:    sel z25.d, p2, z0.d, z31.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z30.d, z2.d
+; CHECK-NEXT:    sel z29.d, p3, z3.d, z9.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z11.d, z2.d
+; CHECK-NEXT:    movprfx z31, z30
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z30.d
+; CHECK-NEXT:    movprfx z9, z11
+; CHECK-NEXT:    fcvtzs z9.d, p0/m, z11.d
+; CHECK-NEXT:    mov z26.d, p4/m, z0.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z28.d, z1.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z30.d, z1.d
+; CHECK-NEXT:    not p7.b, p0/z, p5.b
+; CHECK-NEXT:    fcmuo p5.d, p0/z, z27.d, z27.d
+; CHECK-NEXT:    fcmgt p8.d, p0/z, z13.d, z1.d
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    movprfx z27, z18
+; CHECK-NEXT:    frintx z27.d, p0/m, z18.d
+; CHECK-NEXT:    ld1d { z8.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    mov z16.d, p7/m, z3.d
+; CHECK-NEXT:    fcmuo p7.d, p0/z, z13.d, z13.d
+; CHECK-NEXT:    mov z31.d, p2/m, z3.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z11.d, z1.d
+; CHECK-NEXT:    mov z29.d, p4/m, z0.d
+; CHECK-NEXT:    mov z9.d, p3/m, z3.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z28.d, z28.d
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z30.d, z30.d
+; CHECK-NEXT:    movprfx z28, z17
+; CHECK-NEXT:    frintx z28.d, p0/m, z17.d
+; CHECK-NEXT:    movprfx z30, z15
+; CHECK-NEXT:    frintx z30.d, p0/m, z15.d
+; CHECK-NEXT:    ld1d { z13.d }, p0/z, [x0, #4, mul vl]
+; CHECK-NEXT:    mov z31.d, p6/m, z0.d
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z11.d, z11.d
+; CHECK-NEXT:    sel z11.d, p8, z0.d, z16.d
+; CHECK-NEXT:    mov z9.d, p2/m, z0.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z24.d, z24.d
+; CHECK-NEXT:    movprfx z24, z14
+; CHECK-NEXT:    frintx z24.d, p0/m, z14.d
+; CHECK-NEXT:    fcmge p8.d, p0/z, z27.d, z2.d
+; CHECK-NEXT:    ld1d { z10.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT:    ld1d { z12.d }, p0/z, [x0, #5, mul vl]
+; CHECK-NEXT:    mov z26.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z29.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p5.d, p0/z, z28.d, z2.d
+; CHECK-NEXT:    movprfx z14, z27
+; CHECK-NEXT:    fcvtzs z14.d, p0/m, z27.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z30.d, z2.d
+; CHECK-NEXT:    frintx z13.d, p0/m, z13.d
+; CHECK-NEXT:    mov z31.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p4.d, p0/z, z24.d, z2.d
+; CHECK-NEXT:    mov z9.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    movprfx z15, z28
+; CHECK-NEXT:    fcvtzs z15.d, p0/m, z28.d
+; CHECK-NEXT:    not p6.b, p0/z, p8.b
+; CHECK-NEXT:    movprfx z16, z30
+; CHECK-NEXT:    fcvtzs z16.d, p0/m, z30.d
+; CHECK-NEXT:    frintx z12.d, p0/m, z12.d
+; CHECK-NEXT:    frintx z10.d, p0/m, z10.d
+; CHECK-NEXT:    movprfx z17, z24
+; CHECK-NEXT:    fcvtzs z17.d, p0/m, z24.d
+; CHECK-NEXT:    movprfx z18, z8
+; CHECK-NEXT:    frintx z18.d, p0/m, z8.d
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    sel z8.d, p6, z3.d, z14.d
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    fcmge p6.d, p0/z, z13.d, z2.d
+; CHECK-NEXT:    mov z11.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    sel z14.d, p5, z3.d, z15.d
+; CHECK-NEXT:    fcmuo p7.d, p0/z, z5.d, z5.d
+; CHECK-NEXT:    sel z15.d, p3, z3.d, z16.d
+; CHECK-NEXT:    movprfx z16, z13
+; CHECK-NEXT:    fcvtzs z16.d, p0/m, z13.d
+; CHECK-NEXT:    fcmge p5.d, p0/z, z12.d, z2.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z10.d, z2.d
+; CHECK-NEXT:    sel z5.d, p4, z3.d, z17.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z18.d, z2.d
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    movprfx z2, z12
+; CHECK-NEXT:    fcvtzs z2.d, p0/m, z12.d
+; CHECK-NEXT:    movprfx z17, z10
+; CHECK-NEXT:    fcvtzs z17.d, p0/m, z10.d
+; CHECK-NEXT:    st1b { z11.b }, p1, [x8, x16]
+; CHECK-NEXT:    movprfx z11, z18
+; CHECK-NEXT:    fcvtzs z11.d, p0/m, z18.d
+; CHECK-NEXT:    mov z6.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    st1b { z9.b }, p1, [x8, x15]
+; CHECK-NEXT:    sel z9.d, p6, z3.d, z16.d
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z4.d, z4.d
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z18.d, z1.d
+; CHECK-NEXT:    mov z7.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    st1b { z31.b }, p1, [x8, x14]
+; CHECK-NEXT:    fcmgt p7.d, p0/z, z24.d, z1.d
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z2.d, p5/m, z3.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z28.d, z1.d
+; CHECK-NEXT:    sel z4.d, p3, z3.d, z17.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z13.d, z1.d
+; CHECK-NEXT:    mov z25.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    sel z3.d, p4, z3.d, z11.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z10.d, z1.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z12.d, z1.d
+; CHECK-NEXT:    st1b { z29.b }, p1, [x8, x13]
+; CHECK-NEXT:    st1b { z26.b }, p1, [x8, x12]
+; CHECK-NEXT:    sel z26.d, p5, z0.d, z14.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z30.d, z1.d
+; CHECK-NEXT:    sel z29.d, p3, z0.d, z9.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z18.d, z18.d
+; CHECK-NEXT:    mov z3.d, p2/m, z0.d
+; CHECK-NEXT:    st1b { z25.b }, p1, [x8, x11]
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z10.d, z10.d
+; CHECK-NEXT:    mov z4.d, p4/m, z0.d
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z12.d, z12.d
+; CHECK-NEXT:    st1b { z7.b }, p1, [x8, x10]
+; CHECK-NEXT:    mov z2.d, p6/m, z0.d
+; CHECK-NEXT:    st1b { z6.b }, p1, [x8, x9]
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z13.d, z13.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z27.d, z1.d
+; CHECK-NEXT:    mov z3.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z24.d, z24.d
+; CHECK-NEXT:    sel z1.d, p7, z0.d, z5.d
+; CHECK-NEXT:    mov z4.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z30.d, z30.d
+; CHECK-NEXT:    sel z5.d, p5, z0.d, z15.d
+; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z28.d, z28.d
+; CHECK-NEXT:    mov z29.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z27.d, z27.d
+; CHECK-NEXT:    sel z0.d, p6, z0.d, z8.d
+; CHECK-NEXT:    mov z1.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z3.d }, p0, [x8, #7, mul vl]
+; CHECK-NEXT:    mov z5.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z4.d }, p0, [x8, #6, mul vl]
+; CHECK-NEXT:    mov z26.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z2.d }, p0, [x8, #5, mul vl]
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z29.d }, p0, [x8, #4, mul vl]
+; CHECK-NEXT:    st1d { z1.d }, p0, [x8, #3, mul vl]
+; CHECK-NEXT:    st1d { z5.d }, p0, [x8, #2, mul vl]
+; CHECK-NEXT:    st1d { z26.d }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-NEXT:    ldr z18, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #12
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv16f64(<vscale x 32 x double> %x)
+  ret <vscale x 32 x i64> %a
+}
+declare <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv32f64(<vscale x 32 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-lrint.ll b/llvm/test/CodeGen/AArch64/sve-lrint.ll
new file mode 100644
index 000000000000..2a1432d881e5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-lrint.ll
@@ -0,0 +1,1764 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=aarch64 -mattr=+sve |\
+; RUN:   FileCheck --check-prefixes=CHECK %s
+
+define <vscale x 1 x iXLen> @lrint_v1f16(<vscale x 1 x half> %x) {
+; CHECK-LABEL: lrint_v1f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
+; CHECK-NEXT:    mov z3.h, w8
+; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.h
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z0.h, z3.h
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f16(<vscale x 1 x half> %x)
+  ret <vscale x 1 x iXLen> %a
+}
+declare <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f16(<vscale x 1 x half>)
+
+define <vscale x 2 x iXLen> @lrint_v2f16(<vscale x 2 x half> %x) {
+; CHECK-LABEL: lrint_v2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
+; CHECK-NEXT:    mov z3.h, w8
+; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.h
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z0.h, z3.h
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f16(<vscale x 2 x half> %x)
+  ret <vscale x 2 x iXLen> %a
+}
+declare <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f16(<vscale x 2 x half>)
+
+define <vscale x 4 x iXLen> @lrint_v4f16(<vscale x 4 x half> %x) {
+; CHECK-LABEL: lrint_v4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    uunpklo z1.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z2.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    mov z3.h, w8
+; CHECK-NEXT:    mov z6.d, #0x7fffffffffffffff
+; CHECK-NEXT:    frintx z1.h, p0/m, z1.h
+; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
+; CHECK-NEXT:    fcmge p1.h, p0/z, z1.h, z2.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z0.h, z2.h
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z4, z1
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z1.h
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z0.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z1.h, z3.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z0.h, z3.h
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    sel z3.d, p1, z2.d, z4.d
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z1.h, z1.h
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
+; CHECK-NEXT:    sel z2.d, p2, z2.d, z5.d
+; CHECK-NEXT:    sel z0.d, p3, z6.d, z3.d
+; CHECK-NEXT:    sel z1.d, p4, z6.d, z2.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x iXLen> @llvm.lrint.nxv4iXLen.nxv4f16(<vscale x 4 x half> %x)
+  ret <vscale x 4 x iXLen> %a
+}
+declare <vscale x 4 x iXLen> @llvm.lrint.nxv4iXLen.nxv4f16(<vscale x 4 x half>)
+
+define <vscale x 8 x iXLen> @lrint_v8f16(<vscale x 8 x half> %x) {
+; CHECK-LABEL: lrint_v8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    uunpklo z1.s, z0.h
+; CHECK-NEXT:    uunpkhi z0.s, z0.h
+; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z4.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    mov z6.h, w8
+; CHECK-NEXT:    mov z26.d, #0x7fffffffffffffff
+; CHECK-NEXT:    uunpklo z2.d, z1.s
+; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    uunpklo z3.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    frintx z2.h, p0/m, z2.h
+; CHECK-NEXT:    frintx z1.h, p0/m, z1.h
+; CHECK-NEXT:    frintx z3.h, p0/m, z3.h
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    frintx z5.h, p0/m, z0.h
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p1.h, p0/z, z2.h, z4.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z1.h, z4.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z3.h, z4.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z5.h, z4.h
+; CHECK-NEXT:    movprfx z4, z2
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z2.h
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z1.h
+; CHECK-NEXT:    movprfx z24, z3
+; CHECK-NEXT:    fcvtzs z24.d, p0/m, z3.h
+; CHECK-NEXT:    movprfx z25, z5
+; CHECK-NEXT:    fcvtzs z25.d, p0/m, z5.h
+; CHECK-NEXT:    fcmgt p7.h, p0/z, z3.h, z6.h
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z2.h, z6.h
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z1.h, z6.h
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    mov z4.d, p1/m, z0.d
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z5.h, z6.h
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    sel z6.d, p2, z0.d, z7.d
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z2.h, z2.h
+; CHECK-NEXT:    sel z7.d, p3, z0.d, z24.d
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z1.h, z1.h
+; CHECK-NEXT:    sel z24.d, p4, z0.d, z25.d
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z3.h, z3.h
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z5.h, z5.h
+; CHECK-NEXT:    sel z0.d, p5, z26.d, z4.d
+; CHECK-NEXT:    sel z1.d, p6, z26.d, z6.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z2.d, p7, z26.d, z7.d
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z3.d, p1, z26.d, z24.d
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x iXLen> @llvm.lrint.nxv8iXLen.nxv8f16(<vscale x 8 x half> %x)
+  ret <vscale x 8 x iXLen> %a
+}
+declare <vscale x 8 x iXLen> @llvm.lrint.nxv8iXLen.nxv8f16(<vscale x 8 x half>)
+
+define <vscale x 16 x iXLen> @lrint_v16f16(<vscale x 16 x half> %x) {
+; CHECK-LABEL: lrint_v16f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-3
+; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    uunpklo z2.s, z0.h
+; CHECK-NEXT:    uunpkhi z0.s, z0.h
+; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    uunpklo z4.s, z1.h
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpkhi z1.s, z1.h
+; CHECK-NEXT:    mov z5.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    mov z25.d, #0x8000000000000000
+; CHECK-NEXT:    mov z27.h, w8
+; CHECK-NEXT:    mov z7.d, #0x7fffffffffffffff
+; CHECK-NEXT:    uunpklo z3.d, z2.s
+; CHECK-NEXT:    uunpkhi z2.d, z2.s
+; CHECK-NEXT:    uunpklo z6.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    uunpklo z24.d, z4.s
+; CHECK-NEXT:    uunpkhi z4.d, z4.s
+; CHECK-NEXT:    uunpklo z26.d, z1.s
+; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    frintx z2.h, p0/m, z2.h
+; CHECK-NEXT:    frintx z3.h, p0/m, z3.h
+; CHECK-NEXT:    frintx z6.h, p0/m, z6.h
+; CHECK-NEXT:    movprfx z28, z0
+; CHECK-NEXT:    frintx z28.h, p0/m, z0.h
+; CHECK-NEXT:    movprfx z29, z4
+; CHECK-NEXT:    frintx z29.h, p0/m, z4.h
+; CHECK-NEXT:    frintx z24.h, p0/m, z24.h
+; CHECK-NEXT:    movprfx z30, z1
+; CHECK-NEXT:    frintx z30.h, p0/m, z1.h
+; CHECK-NEXT:    frintx z26.h, p0/m, z26.h
+; CHECK-NEXT:    fcmge p5.h, p0/z, z2.h, z5.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z3.h, z5.h
+; CHECK-NEXT:    movprfx z1, z2
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z2.h
+; CHECK-NEXT:    movprfx z0, z3
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z3.h
+; CHECK-NEXT:    fcmge p6.h, p0/z, z6.h, z5.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z3.h, z27.h
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z3.h, z3.h
+; CHECK-NEXT:    fcmge p7.h, p0/z, z28.h, z5.h
+; CHECK-NEXT:    movprfx z3, z6
+; CHECK-NEXT:    fcvtzs z3.d, p0/m, z6.h
+; CHECK-NEXT:    fcmge p8.h, p0/z, z24.h, z5.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z2.h, z27.h
+; CHECK-NEXT:    fcmge p9.h, p0/z, z26.h, z5.h
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    movprfx z4, z24
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z24.h
+; CHECK-NEXT:    fcmge p10.h, p0/z, z30.h, z5.h
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    movprfx z31, z26
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z26.h
+; CHECK-NEXT:    movprfx z8, z30
+; CHECK-NEXT:    fcvtzs z8.d, p0/m, z30.h
+; CHECK-NEXT:    mov z1.d, p5/m, z25.d
+; CHECK-NEXT:    fcmge p5.h, p0/z, z29.h, z5.h
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    mov z0.d, p2/m, z25.d
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z2.h, z2.h
+; CHECK-NEXT:    movprfx z2, z28
+; CHECK-NEXT:    fcvtzs z2.d, p0/m, z28.h
+; CHECK-NEXT:    movprfx z5, z29
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z29.h
+; CHECK-NEXT:    not p7.b, p0/z, p7.b
+; CHECK-NEXT:    mov z3.d, p6/m, z25.d
+; CHECK-NEXT:    not p6.b, p0/z, p8.b
+; CHECK-NEXT:    fcmgt p8.h, p0/z, z6.h, z27.h
+; CHECK-NEXT:    mov z1.d, p4/m, z7.d
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    mov z0.d, p3/m, z7.d
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z29.h, z27.h
+; CHECK-NEXT:    sel z9.d, p7, z25.d, z2.d
+; CHECK-NEXT:    not p7.b, p0/z, p9.b
+; CHECK-NEXT:    mov z4.d, p6/m, z25.d
+; CHECK-NEXT:    not p6.b, p0/z, p10.b
+; CHECK-NEXT:    fcmgt p10.h, p0/z, z28.h, z27.h
+; CHECK-NEXT:    mov z5.d, p5/m, z25.d
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z24.h, z27.h
+; CHECK-NEXT:    fcmuo p9.h, p0/z, z6.h, z6.h
+; CHECK-NEXT:    sel z6.d, p7, z25.d, z31.d
+; CHECK-NEXT:    sel z25.d, p6, z25.d, z8.d
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z26.h, z27.h
+; CHECK-NEXT:    fcmgt p7.h, p0/z, z30.h, z27.h
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z28.h, z28.h
+; CHECK-NEXT:    sel z2.d, p8, z7.d, z3.d
+; CHECK-NEXT:    sel z3.d, p10, z7.d, z9.d
+; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    fcmuo p8.h, p0/z, z29.h, z29.h
+; CHECK-NEXT:    mov z4.d, p5/m, z7.d
+; CHECK-NEXT:    fcmuo p5.h, p0/z, z24.h, z24.h
+; CHECK-NEXT:    fcmuo p10.h, p0/z, z26.h, z26.h
+; CHECK-NEXT:    mov z5.d, p3/m, z7.d
+; CHECK-NEXT:    mov z6.d, p6/m, z7.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z30.h, z30.h
+; CHECK-NEXT:    sel z7.d, p7, z7.d, z25.d
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z6.d, p10/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #3
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x iXLen> @llvm.lrint.nxv16iXLen.nxv16f16(<vscale x 16 x half> %x)
+  ret <vscale x 16 x iXLen> %a
+}
+declare <vscale x 16 x iXLen> @llvm.lrint.nxv16iXLen.nxv16f16(<vscale x 16 x half>)
+
+define <vscale x 32 x iXLen> @lrint_v32f16(<vscale x 32 x half> %x) {
+; CHECK-LABEL: lrint_v32f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-17
+; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x88, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 136 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
+; CHECK-NEXT:    uunpkhi z5.s, z0.h
+; CHECK-NEXT:    uunpklo z4.s, z0.h
+; CHECK-NEXT:    mov w9, #64511 // =0xfbff
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpklo z6.s, z1.h
+; CHECK-NEXT:    mov z30.h, w9
+; CHECK-NEXT:    uunpkhi z10.s, z1.h
+; CHECK-NEXT:    mov w9, #31743 // =0x7bff
+; CHECK-NEXT:    mov z29.d, #0x8000000000000000
+; CHECK-NEXT:    uunpklo z8.s, z2.h
+; CHECK-NEXT:    uunpkhi z13.s, z3.h
+; CHECK-NEXT:    uunpklo z18.s, z3.h
+; CHECK-NEXT:    uunpklo z7.d, z5.s
+; CHECK-NEXT:    uunpklo z0.d, z4.s
+; CHECK-NEXT:    uunpkhi z4.d, z4.s
+; CHECK-NEXT:    uunpkhi z24.d, z5.s
+; CHECK-NEXT:    uunpklo z25.d, z6.s
+; CHECK-NEXT:    uunpkhi z26.d, z6.s
+; CHECK-NEXT:    uunpklo z27.d, z10.s
+; CHECK-NEXT:    uunpkhi z10.d, z10.s
+; CHECK-NEXT:    uunpklo z12.d, z8.s
+; CHECK-NEXT:    uunpkhi z16.d, z8.s
+; CHECK-NEXT:    movprfx z5, z7
+; CHECK-NEXT:    frintx z5.h, p0/m, z7.h
+; CHECK-NEXT:    movprfx z1, z4
+; CHECK-NEXT:    frintx z1.h, p0/m, z4.h
+; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
+; CHECK-NEXT:    movprfx z6, z24
+; CHECK-NEXT:    frintx z6.h, p0/m, z24.h
+; CHECK-NEXT:    movprfx z24, z25
+; CHECK-NEXT:    frintx z24.h, p0/m, z25.h
+; CHECK-NEXT:    movprfx z25, z26
+; CHECK-NEXT:    frintx z25.h, p0/m, z26.h
+; CHECK-NEXT:    movprfx z28, z27
+; CHECK-NEXT:    frintx z28.h, p0/m, z27.h
+; CHECK-NEXT:    movprfx z8, z10
+; CHECK-NEXT:    frintx z8.h, p0/m, z10.h
+; CHECK-NEXT:    mov z7.h, w9
+; CHECK-NEXT:    mov z4.d, #0x7fffffffffffffff
+; CHECK-NEXT:    rdvl x9, #15
+; CHECK-NEXT:    fcmge p3.h, p0/z, z5.h, z30.h
+; CHECK-NEXT:    movprfx z11, z5
+; CHECK-NEXT:    fcvtzs z11.d, p0/m, z5.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z1.h, z30.h
+; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, z30.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z6.h, z30.h
+; CHECK-NEXT:    movprfx z9, z6
+; CHECK-NEXT:    fcvtzs z9.d, p0/m, z6.h
+; CHECK-NEXT:    movprfx z15, z25
+; CHECK-NEXT:    fcvtzs z15.d, p0/m, z25.h
+; CHECK-NEXT:    movprfx z14, z24
+; CHECK-NEXT:    fcvtzs z14.d, p0/m, z24.h
+; CHECK-NEXT:    movprfx z26, z0
+; CHECK-NEXT:    fcvtzs z26.d, p0/m, z0.h
+; CHECK-NEXT:    movprfx z19, z28
+; CHECK-NEXT:    fcvtzs z19.d, p0/m, z28.h
+; CHECK-NEXT:    movprfx z31, z1
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z1.h
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    not p6.b, p0/z, p2.b
+; CHECK-NEXT:    fcmge p2.h, p0/z, z25.h, z30.h
+; CHECK-NEXT:    sel z27.d, p3, z29.d, z11.d
+; CHECK-NEXT:    uunpkhi z11.s, z2.h
+; CHECK-NEXT:    not p5.b, p0/z, p1.b
+; CHECK-NEXT:    fcmge p1.h, p0/z, z24.h, z30.h
+; CHECK-NEXT:    not p3.b, p0/z, p4.b
+; CHECK-NEXT:    fcmge p4.h, p0/z, z28.h, z30.h
+; CHECK-NEXT:    mov z26.d, p5/m, z29.d
+; CHECK-NEXT:    mov z31.d, p6/m, z29.d
+; CHECK-NEXT:    sel z2.d, p3, z29.d, z9.d
+; CHECK-NEXT:    movprfx z9, z12
+; CHECK-NEXT:    frintx z9.h, p0/m, z12.h
+; CHECK-NEXT:    uunpkhi z12.d, z13.s
+; CHECK-NEXT:    uunpklo z17.d, z11.s
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    sel z3.d, p2, z29.d, z15.d
+; CHECK-NEXT:    uunpklo z15.d, z13.s
+; CHECK-NEXT:    fcmge p2.h, p0/z, z8.h, z30.h
+; CHECK-NEXT:    sel z10.d, p1, z29.d, z14.d
+; CHECK-NEXT:    movprfx z14, z16
+; CHECK-NEXT:    frintx z14.h, p0/m, z16.h
+; CHECK-NEXT:    uunpkhi z16.d, z18.s
+; CHECK-NEXT:    movprfx z13, z17
+; CHECK-NEXT:    frintx z13.h, p0/m, z17.h
+; CHECK-NEXT:    movprfx z20, z12
+; CHECK-NEXT:    frintx z20.h, p0/m, z12.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z9.h, z30.h
+; CHECK-NEXT:    uunpkhi z17.d, z11.s
+; CHECK-NEXT:    uunpklo z18.d, z18.s
+; CHECK-NEXT:    movprfx z12, z8
+; CHECK-NEXT:    fcvtzs z12.d, p0/m, z8.h
+; CHECK-NEXT:    movprfx z21, z15
+; CHECK-NEXT:    frintx z21.h, p0/m, z15.h
+; CHECK-NEXT:    not p1.b, p0/z, p4.b
+; CHECK-NEXT:    movprfx z15, z9
+; CHECK-NEXT:    fcvtzs z15.d, p0/m, z9.h
+; CHECK-NEXT:    frintx z16.h, p0/m, z16.h
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    movprfx z22, z14
+; CHECK-NEXT:    fcvtzs z22.d, p0/m, z14.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z13.h, z30.h
+; CHECK-NEXT:    fcmge p5.h, p0/z, z20.h, z30.h
+; CHECK-NEXT:    sel z11.d, p1, z29.d, z19.d
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    frintx z17.h, p0/m, z17.h
+; CHECK-NEXT:    frintx z18.h, p0/m, z18.h
+; CHECK-NEXT:    movprfx z19, z20
+; CHECK-NEXT:    fcvtzs z19.d, p0/m, z20.h
+; CHECK-NEXT:    mov z12.d, p2/m, z29.d
+; CHECK-NEXT:    fcmge p2.h, p0/z, z21.h, z30.h
+; CHECK-NEXT:    fcmge p1.h, p0/z, z14.h, z30.h
+; CHECK-NEXT:    mov z15.d, p3/m, z29.d
+; CHECK-NEXT:    movprfx z23, z21
+; CHECK-NEXT:    fcvtzs z23.d, p0/m, z21.h
+; CHECK-NEXT:    not p3.b, p0/z, p4.b
+; CHECK-NEXT:    fcmge p4.h, p0/z, z16.h, z30.h
+; CHECK-NEXT:    fcmgt p8.h, p0/z, z21.h, z7.h
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    fcmge p6.h, p0/z, z17.h, z30.h
+; CHECK-NEXT:    fcmge p7.h, p0/z, z18.h, z30.h
+; CHECK-NEXT:    movprfx z30, z16
+; CHECK-NEXT:    fcvtzs z30.d, p0/m, z16.h
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    fcmuo p9.h, p0/z, z21.h, z21.h
+; CHECK-NEXT:    mov z19.d, p5/m, z29.d
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z20.h, z7.h
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z23.d, p2/m, z29.d
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z20.h, z20.h
+; CHECK-NEXT:    movprfx z20, z18
+; CHECK-NEXT:    fcvtzs z20.d, p0/m, z18.h
+; CHECK-NEXT:    movprfx z21, z13
+; CHECK-NEXT:    fcvtzs z21.d, p0/m, z13.h
+; CHECK-NEXT:    mov z22.d, p1/m, z29.d
+; CHECK-NEXT:    not p1.b, p0/z, p7.b
+; CHECK-NEXT:    mov z30.d, p4/m, z29.d
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z18.h, z7.h
+; CHECK-NEXT:    mov z19.d, p5/m, z4.d
+; CHECK-NEXT:    fcmuo p7.h, p0/z, z18.h, z18.h
+; CHECK-NEXT:    movprfx z18, z17
+; CHECK-NEXT:    fcvtzs z18.d, p0/m, z17.h
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z16.h, z7.h
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    mov z23.d, p8/m, z4.d
+; CHECK-NEXT:    mov z20.d, p1/m, z29.d
+; CHECK-NEXT:    mov z21.d, p3/m, z29.d
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z16.h, z16.h
+; CHECK-NEXT:    mov z19.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z17.h, z7.h
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    sel z29.d, p6, z29.d, z18.d
+; CHECK-NEXT:    mov z23.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z14.h, z7.h
+; CHECK-NEXT:    mov z30.d, p5/m, z4.d
+; CHECK-NEXT:    sel z16.d, p4, z4.d, z20.d
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z17.h, z17.h
+; CHECK-NEXT:    st1b { z19.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #14
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z1.h, z7.h
+; CHECK-NEXT:    st1b { z23.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #13
+; CHECK-NEXT:    mov z29.d, p2/m, z4.d
+; CHECK-NEXT:    mov z30.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z13.h, z7.h
+; CHECK-NEXT:    mov z16.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z9.h, z7.h
+; CHECK-NEXT:    fcmuo p7.h, p0/z, z14.h, z14.h
+; CHECK-NEXT:    mov z29.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z13.h, z13.h
+; CHECK-NEXT:    st1b { z30.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #12
+; CHECK-NEXT:    sel z30.d, p5, z4.d, z31.d
+; CHECK-NEXT:    st1b { z16.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #11
+; CHECK-NEXT:    sel z31.d, p3, z4.d, z21.d
+; CHECK-NEXT:    st1b { z29.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #10
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z24.h, z7.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z28.h, z7.h
+; CHECK-NEXT:    sel z13.d, p2, z4.d, z15.d
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z9.h, z9.h
+; CHECK-NEXT:    sel z29.d, p6, z4.d, z22.d
+; CHECK-NEXT:    mov z31.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z8.h, z7.h
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z5.h, z7.h
+; CHECK-NEXT:    sel z9.d, p5, z4.d, z10.d
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z6.h, z7.h
+; CHECK-NEXT:    st1b { z31.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #9
+; CHECK-NEXT:    mov z29.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    sel z10.d, p3, z4.d, z11.d
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z25.h, z7.h
+; CHECK-NEXT:    mov z13.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p7.h, p0/z, z8.h, z8.h
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z28.h, z28.h
+; CHECK-NEXT:    sel z28.d, p4, z4.d, z12.d
+; CHECK-NEXT:    st1b { z29.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #8
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z25.h, z25.h
+; CHECK-NEXT:    st1b { z13.b }, p1, [x8, x9]
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z24.h, z24.h
+; CHECK-NEXT:    mov z2.d, p5/m, z4.d
+; CHECK-NEXT:    mov z3.d, p3/m, z4.d
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z0.h, z7.h
+; CHECK-NEXT:    mov z28.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p7.h, p0/z, z6.h, z6.h
+; CHECK-NEXT:    mov z10.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z5.h, z5.h
+; CHECK-NEXT:    sel z5.d, p6, z4.d, z27.d
+; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z1.h, z1.h
+; CHECK-NEXT:    mov z9.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z28.d }, p0, [x8, #7, mul vl]
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z0.h, z0.h
+; CHECK-NEXT:    sel z0.d, p3, z4.d, z26.d
+; CHECK-NEXT:    st1d { z10.d }, p0, [x8, #6, mul vl]
+; CHECK-NEXT:    mov z2.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z3.d }, p0, [x8, #5, mul vl]
+; CHECK-NEXT:    mov z5.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z9.d }, p0, [x8, #4, mul vl]
+; CHECK-NEXT:    mov z30.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z2.d }, p0, [x8, #3, mul vl]
+; CHECK-NEXT:    st1d { z5.d }, p0, [x8, #2, mul vl]
+; CHECK-NEXT:    st1d { z30.d }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #17
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 32 x iXLen> @llvm.lrint.nxv32iXLen.nxv32f16(<vscale x 32 x half> %x)
+  ret <vscale x 32 x iXLen> %a
+}
+declare <vscale x 32 x iXLen> @llvm.lrint.nxv32iXLen.nxv32f16(<vscale x 32 x half>)
+
+define <vscale x 1 x iXLen> @lrint_v1f32(<vscale x 1 x float> %x) {
+; CHECK-LABEL: lrint_v1f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-NEXT:    mov z3.s, w8
+; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.s
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z0.s, z3.s
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f32(<vscale x 1 x float> %x)
+  ret <vscale x 1 x iXLen> %a
+}
+declare <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f32(<vscale x 1 x float>)
+
+define <vscale x 2 x iXLen> @lrint_v2f32(<vscale x 2 x float> %x) {
+; CHECK-LABEL: lrint_v2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-NEXT:    mov z3.s, w8
+; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.s
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z0.s, z3.s
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f32(<vscale x 2 x float> %x)
+  ret <vscale x 2 x iXLen> %a
+}
+declare <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f32(<vscale x 2 x float>)
+
+define <vscale x 4 x iXLen> @lrint_v4f32(<vscale x 4 x float> %x) {
+; CHECK-LABEL: lrint_v4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    uunpklo z1.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z2.s, w8
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    mov z3.s, w8
+; CHECK-NEXT:    mov z6.d, #0x7fffffffffffffff
+; CHECK-NEXT:    frintx z1.s, p0/m, z1.s
+; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z1.s, z2.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z2.s
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z4, z1
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z1.s
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z0.s
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z1.s, z3.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z0.s, z3.s
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    sel z3.d, p1, z2.d, z4.d
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z1.s, z1.s
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    sel z2.d, p2, z2.d, z5.d
+; CHECK-NEXT:    sel z0.d, p3, z6.d, z3.d
+; CHECK-NEXT:    sel z1.d, p4, z6.d, z2.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x iXLen> @llvm.lrint.nxv4iXLen.nxv4f32(<vscale x 4 x float> %x)
+  ret <vscale x 4 x iXLen> %a
+}
+declare <vscale x 4 x iXLen> @llvm.lrint.nxv4iXLen.nxv4f32(<vscale x 4 x float>)
+
+define <vscale x 8 x iXLen> @lrint_v8f32(<vscale x 8 x float> %x) {
+; CHECK-LABEL: lrint_v8f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    uunpklo z2.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    uunpklo z3.d, z1.s
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    mov z4.s, w8
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    mov z5.d, #0x8000000000000000
+; CHECK-NEXT:    mov z6.s, w8
+; CHECK-NEXT:    mov z26.d, #0x7fffffffffffffff
+; CHECK-NEXT:    frintx z2.s, p0/m, z2.s
+; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
+; CHECK-NEXT:    frintx z3.s, p0/m, z3.s
+; CHECK-NEXT:    frintx z1.s, p0/m, z1.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z2.s, z4.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z4.s
+; CHECK-NEXT:    movprfx z7, z0
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z0.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z3.s, z4.s
+; CHECK-NEXT:    fcmge p4.s, p0/z, z1.s, z4.s
+; CHECK-NEXT:    movprfx z4, z2
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z2.s
+; CHECK-NEXT:    movprfx z24, z3
+; CHECK-NEXT:    fcvtzs z24.d, p0/m, z3.s
+; CHECK-NEXT:    movprfx z25, z1
+; CHECK-NEXT:    fcvtzs z25.d, p0/m, z1.s
+; CHECK-NEXT:    fcmgt p7.s, p0/z, z3.s, z6.s
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z2.s, z6.s
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z0.s, z6.s
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    mov z4.d, p1/m, z5.d
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z6.s
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    sel z6.d, p2, z5.d, z7.d
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z2.s, z2.s
+; CHECK-NEXT:    sel z7.d, p3, z5.d, z24.d
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    sel z5.d, p4, z5.d, z25.d
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z3.s, z3.s
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z1.s, z1.s
+; CHECK-NEXT:    sel z0.d, p5, z26.d, z4.d
+; CHECK-NEXT:    sel z1.d, p6, z26.d, z6.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z2.d, p7, z26.d, z7.d
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z3.d, p1, z26.d, z5.d
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x iXLen> @llvm.lrint.nxv8iXLen.nxv8f32(<vscale x 8 x float> %x)
+  ret <vscale x 8 x iXLen> %a
+}
+declare <vscale x 8 x iXLen> @llvm.lrint.nxv8iXLen.nxv8f32(<vscale x 8 x float>)
+
+define <vscale x 16 x iXLen> @lrint_v16f32(<vscale x 16 x float> %x) {
+; CHECK-LABEL: lrint_v16f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-2
+; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    uunpklo z4.d, z0.s
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpklo z7.d, z1.s
+; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    uunpklo z24.d, z2.s
+; CHECK-NEXT:    uunpkhi z2.d, z2.s
+; CHECK-NEXT:    uunpklo z25.d, z3.s
+; CHECK-NEXT:    uunpkhi z3.d, z3.s
+; CHECK-NEXT:    mov z26.d, #0x7fffffffffffffff
+; CHECK-NEXT:    movprfx z5, z4
+; CHECK-NEXT:    frintx z5.s, p0/m, z4.s
+; CHECK-NEXT:    movprfx z6, z0
+; CHECK-NEXT:    frintx z6.s, p0/m, z0.s
+; CHECK-NEXT:    mov z4.s, w8
+; CHECK-NEXT:    frintx z7.s, p0/m, z7.s
+; CHECK-NEXT:    movprfx z28, z1
+; CHECK-NEXT:    frintx z28.s, p0/m, z1.s
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    frintx z24.s, p0/m, z24.s
+; CHECK-NEXT:    movprfx z29, z2
+; CHECK-NEXT:    frintx z29.s, p0/m, z2.s
+; CHECK-NEXT:    frintx z25.s, p0/m, z25.s
+; CHECK-NEXT:    movprfx z30, z3
+; CHECK-NEXT:    frintx z30.s, p0/m, z3.s
+; CHECK-NEXT:    mov z27.s, w8
+; CHECK-NEXT:    fcmge p1.s, p0/z, z5.s, z4.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z6.s, z4.s
+; CHECK-NEXT:    movprfx z1, z5
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z5.s
+; CHECK-NEXT:    movprfx z2, z6
+; CHECK-NEXT:    fcvtzs z2.d, p0/m, z6.s
+; CHECK-NEXT:    fcmge p5.s, p0/z, z7.s, z4.s
+; CHECK-NEXT:    fcmge p6.s, p0/z, z28.s, z4.s
+; CHECK-NEXT:    movprfx z3, z7
+; CHECK-NEXT:    fcvtzs z3.d, p0/m, z7.s
+; CHECK-NEXT:    fcmge p8.s, p0/z, z29.s, z4.s
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z5.s, z27.s
+; CHECK-NEXT:    fcmgt p7.s, p0/z, z6.s, z27.s
+; CHECK-NEXT:    fcmge p9.s, p0/z, z25.s, z4.s
+; CHECK-NEXT:    movprfx z31, z25
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z25.s
+; CHECK-NEXT:    not p4.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z5.s, z5.s
+; CHECK-NEXT:    movprfx z5, z28
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z28.s
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    fcmge p10.s, p0/z, z30.s, z4.s
+; CHECK-NEXT:    movprfx z8, z30
+; CHECK-NEXT:    fcvtzs z8.d, p0/m, z30.s
+; CHECK-NEXT:    mov z1.d, p4/m, z0.d
+; CHECK-NEXT:    fcmge p4.s, p0/z, z24.s, z4.s
+; CHECK-NEXT:    movprfx z4, z29
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z29.s
+; CHECK-NEXT:    mov z2.d, p2/m, z0.d
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z6.s, z6.s
+; CHECK-NEXT:    movprfx z6, z24
+; CHECK-NEXT:    fcvtzs z6.d, p0/m, z24.s
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z3.d, p5/m, z0.d
+; CHECK-NEXT:    not p5.b, p0/z, p8.b
+; CHECK-NEXT:    mov z5.d, p6/m, z0.d
+; CHECK-NEXT:    fcmgt p8.s, p0/z, z7.s, z27.s
+; CHECK-NEXT:    not p6.b, p0/z, p9.b
+; CHECK-NEXT:    mov z6.d, p4/m, z0.d
+; CHECK-NEXT:    fcmuo p9.s, p0/z, z7.s, z7.s
+; CHECK-NEXT:    not p4.b, p0/z, p10.b
+; CHECK-NEXT:    fcmgt p10.s, p0/z, z28.s, z27.s
+; CHECK-NEXT:    sel z7.d, p5, z0.d, z4.d
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z24.s, z27.s
+; CHECK-NEXT:    mov z31.d, p6/m, z0.d
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z30.s, z27.s
+; CHECK-NEXT:    mov z8.d, p4/m, z0.d
+; CHECK-NEXT:    sel z0.d, p3, z26.d, z1.d
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z29.s, z27.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z25.s, z27.s
+; CHECK-NEXT:    sel z1.d, p7, z26.d, z2.d
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z28.s, z28.s
+; CHECK-NEXT:    sel z2.d, p8, z26.d, z3.d
+; CHECK-NEXT:    sel z3.d, p10, z26.d, z5.d
+; CHECK-NEXT:    fcmuo p8.s, p0/z, z29.s, z29.s
+; CHECK-NEXT:    sel z4.d, p5, z26.d, z6.d
+; CHECK-NEXT:    fcmuo p5.s, p0/z, z24.s, z24.s
+; CHECK-NEXT:    fcmuo p10.s, p0/z, z25.s, z25.s
+; CHECK-NEXT:    sel z5.d, p3, z26.d, z7.d
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z30.s, z30.s
+; CHECK-NEXT:    sel z7.d, p6, z26.d, z8.d
+; CHECK-NEXT:    ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z6.d, p4, z26.d, z31.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    mov z3.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z6.d, p10/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x iXLen> @llvm.lrint.nxv16iXLen.nxv16f32(<vscale x 16 x float> %x)
+  ret <vscale x 16 x iXLen> %a
+}
+declare <vscale x 16 x iXLen> @llvm.lrint.nxv16iXLen.nxv16f32(<vscale x 16 x float>)
+
+define <vscale x 32 x iXLen> @lrint_v32f32(<vscale x 32 x float> %x) {
+; CHECK-LABEL: lrint_v32f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-17
+; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x88, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 136 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
+; CHECK-NEXT:    uunpklo z24.d, z0.s
+; CHECK-NEXT:    uunpkhi z25.d, z0.s
+; CHECK-NEXT:    mov w9, #-553648128 // =0xdf000000
+; CHECK-NEXT:    uunpklo z26.d, z1.s
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpkhi z27.d, z1.s
+; CHECK-NEXT:    mov z31.s, w9
+; CHECK-NEXT:    mov w9, #1593835519 // =0x5effffff
+; CHECK-NEXT:    uunpklo z28.d, z2.s
+; CHECK-NEXT:    mov z8.d, #0x8000000000000000
+; CHECK-NEXT:    uunpklo z30.d, z3.s
+; CHECK-NEXT:    uunpklo z13.d, z4.s
+; CHECK-NEXT:    movprfx z0, z24
+; CHECK-NEXT:    frintx z0.s, p0/m, z24.s
+; CHECK-NEXT:    movprfx z1, z25
+; CHECK-NEXT:    frintx z1.s, p0/m, z25.s
+; CHECK-NEXT:    uunpkhi z15.d, z4.s
+; CHECK-NEXT:    movprfx z24, z26
+; CHECK-NEXT:    frintx z24.s, p0/m, z26.s
+; CHECK-NEXT:    uunpkhi z26.d, z2.s
+; CHECK-NEXT:    movprfx z25, z27
+; CHECK-NEXT:    frintx z25.s, p0/m, z27.s
+; CHECK-NEXT:    movprfx z27, z28
+; CHECK-NEXT:    frintx z27.s, p0/m, z28.s
+; CHECK-NEXT:    uunpklo z16.d, z5.s
+; CHECK-NEXT:    uunpkhi z17.d, z7.s
+; CHECK-NEXT:    frintx z30.s, p0/m, z30.s
+; CHECK-NEXT:    uunpklo z18.d, z7.s
+; CHECK-NEXT:    uunpklo z21.d, z6.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z31.s
+; CHECK-NEXT:    movprfx z9, z0
+; CHECK-NEXT:    fcvtzs z9.d, p0/m, z0.s
+; CHECK-NEXT:    movprfx z10, z1
+; CHECK-NEXT:    fcvtzs z10.d, p0/m, z1.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z1.s, z31.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z24.s, z31.s
+; CHECK-NEXT:    movprfx z11, z24
+; CHECK-NEXT:    fcvtzs z11.d, p0/m, z24.s
+; CHECK-NEXT:    movprfx z29, z26
+; CHECK-NEXT:    frintx z29.s, p0/m, z26.s
+; CHECK-NEXT:    fcmge p4.s, p0/z, z25.s, z31.s
+; CHECK-NEXT:    fcmge p5.s, p0/z, z27.s, z31.s
+; CHECK-NEXT:    movprfx z12, z27
+; CHECK-NEXT:    fcvtzs z12.d, p0/m, z27.s
+; CHECK-NEXT:    movprfx z19, z30
+; CHECK-NEXT:    fcvtzs z19.d, p0/m, z30.s
+; CHECK-NEXT:    movprfx z7, z16
+; CHECK-NEXT:    frintx z7.s, p0/m, z16.s
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    frintx z17.s, p0/m, z17.s
+; CHECK-NEXT:    uunpkhi z16.d, z5.s
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    frintx z18.s, p0/m, z18.s
+; CHECK-NEXT:    mov z28.s, w9
+; CHECK-NEXT:    not p6.b, p0/z, p3.b
+; CHECK-NEXT:    sel z26.d, p1, z8.d, z9.d
+; CHECK-NEXT:    movprfx z14, z29
+; CHECK-NEXT:    fcvtzs z14.d, p0/m, z29.s
+; CHECK-NEXT:    sel z9.d, p2, z8.d, z10.d
+; CHECK-NEXT:    uunpkhi z10.d, z3.s
+; CHECK-NEXT:    rdvl x9, #15
+; CHECK-NEXT:    sel z3.d, p6, z8.d, z11.d
+; CHECK-NEXT:    movprfx z11, z25
+; CHECK-NEXT:    fcvtzs z11.d, p0/m, z25.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z29.s, z31.s
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    fcmge p1.s, p0/z, z30.s, z31.s
+; CHECK-NEXT:    movprfx z23, z18
+; CHECK-NEXT:    fcvtzs z23.d, p0/m, z18.s
+; CHECK-NEXT:    not p2.b, p0/z, p5.b
+; CHECK-NEXT:    fcmge p5.s, p0/z, z17.s, z31.s
+; CHECK-NEXT:    frintx z16.s, p0/m, z16.s
+; CHECK-NEXT:    frintx z10.s, p0/m, z10.s
+; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcmgt p8.s, p0/z, z18.s, z28.s
+; CHECK-NEXT:    sel z4.d, p4, z8.d, z11.d
+; CHECK-NEXT:    movprfx z11, z13
+; CHECK-NEXT:    frintx z11.s, p0/m, z13.s
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    sel z13.d, p2, z8.d, z12.d
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmge p4.s, p0/z, z7.s, z31.s
+; CHECK-NEXT:    sel z12.d, p3, z8.d, z14.d
+; CHECK-NEXT:    movprfx z14, z15
+; CHECK-NEXT:    frintx z14.s, p0/m, z15.s
+; CHECK-NEXT:    uunpkhi z15.d, z6.s
+; CHECK-NEXT:    movprfx z20, z10
+; CHECK-NEXT:    fcvtzs z20.d, p0/m, z10.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z10.s, z31.s
+; CHECK-NEXT:    sel z5.d, p1, z8.d, z19.d
+; CHECK-NEXT:    movprfx z19, z11
+; CHECK-NEXT:    fcvtzs z19.d, p0/m, z11.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z11.s, z31.s
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    fcmge p6.s, p0/z, z16.s, z31.s
+; CHECK-NEXT:    fcmuo p9.s, p0/z, z18.s, z18.s
+; CHECK-NEXT:    movprfx z22, z15
+; CHECK-NEXT:    frintx z22.s, p0/m, z15.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z14.s, z31.s
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    sel z6.d, p2, z8.d, z20.d
+; CHECK-NEXT:    movprfx z20, z21
+; CHECK-NEXT:    frintx z20.s, p0/m, z21.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z18.s, z31.s
+; CHECK-NEXT:    sel z15.d, p3, z8.d, z19.d
+; CHECK-NEXT:    movprfx z19, z17
+; CHECK-NEXT:    fcvtzs z19.d, p0/m, z17.s
+; CHECK-NEXT:    not p3.b, p0/z, p4.b
+; CHECK-NEXT:    fcmge p4.s, p0/z, z22.s, z31.s
+; CHECK-NEXT:    movprfx z21, z14
+; CHECK-NEXT:    fcvtzs z21.d, p0/m, z14.s
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    movprfx z18, z7
+; CHECK-NEXT:    fcvtzs z18.d, p0/m, z7.s
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    fcmge p7.s, p0/z, z20.s, z31.s
+; CHECK-NEXT:    movprfx z31, z22
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z22.s
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    mov z19.d, p5/m, z8.d
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z17.s, z28.s
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z23.d, p2/m, z8.d
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z17.s, z17.s
+; CHECK-NEXT:    movprfx z17, z20
+; CHECK-NEXT:    fcvtzs z17.d, p0/m, z20.s
+; CHECK-NEXT:    mov z21.d, p1/m, z8.d
+; CHECK-NEXT:    mov z18.d, p3/m, z8.d
+; CHECK-NEXT:    not p1.b, p0/z, p7.b
+; CHECK-NEXT:    mov z31.d, p4/m, z8.d
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z20.s, z28.s
+; CHECK-NEXT:    mov z19.d, p5/m, z2.d
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z20.s, z20.s
+; CHECK-NEXT:    movprfx z20, z16
+; CHECK-NEXT:    fcvtzs z20.d, p0/m, z16.s
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z22.s, z28.s
+; CHECK-NEXT:    mov z23.d, p8/m, z2.d
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z22.s, z22.s
+; CHECK-NEXT:    mov z17.d, p1/m, z8.d
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    mov z19.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z16.s, z28.s
+; CHECK-NEXT:    sel z8.d, p6, z8.d, z20.d
+; CHECK-NEXT:    mov z23.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z14.s, z28.s
+; CHECK-NEXT:    mov z31.d, p5/m, z2.d
+; CHECK-NEXT:    mov z17.d, p4/m, z2.d
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z16.s, z16.s
+; CHECK-NEXT:    st1b { z19.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #14
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z1.s, z28.s
+; CHECK-NEXT:    st1b { z23.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #13
+; CHECK-NEXT:    mov z8.d, p2/m, z2.d
+; CHECK-NEXT:    mov z31.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z7.s, z28.s
+; CHECK-NEXT:    mov z17.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z11.s, z28.s
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z14.s, z14.s
+; CHECK-NEXT:    mov z8.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z7.s, z7.s
+; CHECK-NEXT:    sel z7.d, p5, z2.d, z9.d
+; CHECK-NEXT:    st1b { z31.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #12
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z27.s, z28.s
+; CHECK-NEXT:    st1b { z17.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #11
+; CHECK-NEXT:    sel z31.d, p3, z2.d, z18.d
+; CHECK-NEXT:    st1b { z8.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #10
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z30.s, z28.s
+; CHECK-NEXT:    sel z9.d, p2, z2.d, z15.d
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z11.s, z11.s
+; CHECK-NEXT:    sel z8.d, p6, z2.d, z21.d
+; CHECK-NEXT:    mov z31.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z10.s, z28.s
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z24.s, z28.s
+; CHECK-NEXT:    sel z11.d, p5, z2.d, z13.d
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z25.s, z28.s
+; CHECK-NEXT:    mov z8.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    mov z5.d, p3/m, z2.d
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z29.s, z28.s
+; CHECK-NEXT:    st1b { z31.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #9
+; CHECK-NEXT:    mov z9.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z10.s, z10.s
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z30.s, z30.s
+; CHECK-NEXT:    mov z6.d, p4/m, z2.d
+; CHECK-NEXT:    st1b { z8.b }, p1, [x8, x9]
+; CHECK-NEXT:    rdvl x9, #8
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z29.s, z29.s
+; CHECK-NEXT:    st1b { z9.b }, p1, [x8, x9]
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z27.s, z27.s
+; CHECK-NEXT:    sel z27.d, p3, z2.d, z12.d
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z0.s, z28.s
+; CHECK-NEXT:    mov z4.d, p5/m, z2.d
+; CHECK-NEXT:    mov z3.d, p6/m, z2.d
+; CHECK-NEXT:    mov z6.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z25.s, z25.s
+; CHECK-NEXT:    mov z5.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z24.s, z24.s
+; CHECK-NEXT:    mov z27.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z1.s, z1.s
+; CHECK-NEXT:    mov z11.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    st1d { z6.d }, p0, [x8, #7, mul vl]
+; CHECK-NEXT:    sel z0.d, p3, z2.d, z26.d
+; CHECK-NEXT:    st1d { z5.d }, p0, [x8, #6, mul vl]
+; CHECK-NEXT:    mov z4.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z27.d }, p0, [x8, #5, mul vl]
+; CHECK-NEXT:    mov z3.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z7.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z11.d }, p0, [x8, #4, mul vl]
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z4.d }, p0, [x8, #3, mul vl]
+; CHECK-NEXT:    st1d { z3.d }, p0, [x8, #2, mul vl]
+; CHECK-NEXT:    st1d { z7.d }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #17
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 32 x iXLen> @llvm.lrint.nxv32iXLen.nxv32f32(<vscale x 32 x float> %x)
+  ret <vscale x 32 x iXLen> %a
+}
+declare <vscale x 32 x iXLen> @llvm.lrint.nxv32iXLen.nxv32f32(<vscale x 32 x float>)
+
+define <vscale x 1 x iXLen> @lrint_v1f64(<vscale x 1 x double> %x) {
+; CHECK-LABEL: lrint_v1f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    mov z3.d, x8
+; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z0.d, z3.d
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f64(<vscale x 1 x double> %x)
+  ret <vscale x 1 x iXLen> %a
+}
+declare <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f64(<vscale x 1 x double>)
+
+define <vscale x 2 x iXLen> @lrint_v2f64(<vscale x 2 x double> %x) {
+; CHECK-LABEL: lrint_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    mov z3.d, x8
+; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z0.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z0.d, z3.d
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    mov z1.d, p1/m, z2.d
+; CHECK-NEXT:    sel z0.d, p2, z3.d, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f64(<vscale x 2 x double> %x)
+  ret <vscale x 2 x iXLen> %a
+}
+declare <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f64(<vscale x 2 x double>)
+
+define <vscale x 4 x iXLen> @lrint_v4f64(<vscale x 4 x double> %x) {
+; CHECK-LABEL: lrint_v4f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z6.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z2.d, x8
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
+; CHECK-NEXT:    mov z3.d, x8
+; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z2.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z2.d
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z5, z1
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z1.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z0.d, z3.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z1.d, z3.d
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    sel z3.d, p1, z2.d, z4.d
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    sel z2.d, p2, z2.d, z5.d
+; CHECK-NEXT:    sel z0.d, p3, z6.d, z3.d
+; CHECK-NEXT:    sel z1.d, p4, z6.d, z2.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x iXLen> @llvm.lrint.nxv4iXLen.nxv4f64(<vscale x 4 x double> %x)
+  ret <vscale x 4 x iXLen> %a
+}
+declare <vscale x 4 x iXLen> @llvm.lrint.nxv4iXLen.nxv4f64(<vscale x 4 x double>)
+
+define <vscale x 8 x iXLen> @lrint_v8f64(<vscale x 8 x double> %x) {
+; CHECK-LABEL: lrint_v8f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z5.d, #0x8000000000000000
+; CHECK-NEXT:    mov z4.d, x8
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    mov z26.d, #0x7fffffffffffffff
+; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
+; CHECK-NEXT:    frintx z2.d, p0/m, z2.d
+; CHECK-NEXT:    frintx z3.d, p0/m, z3.d
+; CHECK-NEXT:    mov z6.d, x8
+; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z4.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z4.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z2.d, z4.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z3.d, z4.d
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z1.d
+; CHECK-NEXT:    movprfx z24, z2
+; CHECK-NEXT:    fcvtzs z24.d, p0/m, z2.d
+; CHECK-NEXT:    movprfx z25, z3
+; CHECK-NEXT:    fcvtzs z25.d, p0/m, z3.d
+; CHECK-NEXT:    fcmgt p7.d, p0/z, z2.d, z6.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z0.d, z6.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z1.d, z6.d
+; CHECK-NEXT:    not p1.b, p0/z, p1.b
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    mov z4.d, p1/m, z5.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z3.d, z6.d
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    sel z6.d, p2, z5.d, z7.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    sel z7.d, p3, z5.d, z24.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    sel z5.d, p4, z5.d, z25.d
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z3.d, z3.d
+; CHECK-NEXT:    sel z0.d, p5, z26.d, z4.d
+; CHECK-NEXT:    sel z1.d, p6, z26.d, z6.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z2.d, p7, z26.d, z7.d
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z3.d, p1, z26.d, z5.d
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x iXLen> @llvm.lrint.nxv8iXLen.nxv8f64(<vscale x 8 x double> %x)
+  ret <vscale x 8 x iXLen> %a
+}
+declare <vscale x 8 x iXLen> @llvm.lrint.nxv8iXLen.nxv8f64(<vscale x 8 x double>)
+
+define <vscale x 16 x iXLen> @lrint_v16f64(<vscale x 16 x double> %x) {
+; CHECK-LABEL: lrint_v16f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-2
+; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z24.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z25.d, x8
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    movprfx z26, z0
+; CHECK-NEXT:    frintx z26.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z27, z1
+; CHECK-NEXT:    frintx z27.d, p0/m, z1.d
+; CHECK-NEXT:    frintx z2.d, p0/m, z2.d
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    frintx z3.d, p0/m, z3.d
+; CHECK-NEXT:    movprfx z28, z4
+; CHECK-NEXT:    frintx z28.d, p0/m, z4.d
+; CHECK-NEXT:    frintx z5.d, p0/m, z5.d
+; CHECK-NEXT:    frintx z6.d, p0/m, z6.d
+; CHECK-NEXT:    frintx z7.d, p0/m, z7.d
+; CHECK-NEXT:    fcmge p1.d, p0/z, z26.d, z25.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z27.d, z25.d
+; CHECK-NEXT:    movprfx z4, z26
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z26.d
+; CHECK-NEXT:    fcmge p5.d, p0/z, z2.d, z25.d
+; CHECK-NEXT:    movprfx z29, z27
+; CHECK-NEXT:    fcvtzs z29.d, p0/m, z27.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z26.d, z1.d
+; CHECK-NEXT:    fcmge p6.d, p0/z, z3.d, z25.d
+; CHECK-NEXT:    fcmge p8.d, p0/z, z5.d, z25.d
+; CHECK-NEXT:    fcmgt p7.d, p0/z, z27.d, z1.d
+; CHECK-NEXT:    fcmge p9.d, p0/z, z6.d, z25.d
+; CHECK-NEXT:    movprfx z30, z28
+; CHECK-NEXT:    fcvtzs z30.d, p0/m, z28.d
+; CHECK-NEXT:    fcmge p10.d, p0/z, z7.d, z25.d
+; CHECK-NEXT:    not p4.b, p0/z, p1.b
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z26.d, z26.d
+; CHECK-NEXT:    movprfx z26, z2
+; CHECK-NEXT:    fcvtzs z26.d, p0/m, z2.d
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    movprfx z31, z6
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z6.d
+; CHECK-NEXT:    movprfx z8, z7
+; CHECK-NEXT:    fcvtzs z8.d, p0/m, z7.d
+; CHECK-NEXT:    mov z4.d, p4/m, z0.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z28.d, z25.d
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    mov z29.d, p2/m, z0.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z27.d, z27.d
+; CHECK-NEXT:    movprfx z27, z3
+; CHECK-NEXT:    fcvtzs z27.d, p0/m, z3.d
+; CHECK-NEXT:    sel z25.d, p5, z0.d, z26.d
+; CHECK-NEXT:    movprfx z26, z5
+; CHECK-NEXT:    fcvtzs z26.d, p0/m, z5.d
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    not p5.b, p0/z, p8.b
+; CHECK-NEXT:    fcmgt p8.d, p0/z, z2.d, z1.d
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z27.d, p6/m, z0.d
+; CHECK-NEXT:    not p6.b, p0/z, p9.b
+; CHECK-NEXT:    fcmuo p9.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    mov z30.d, p4/m, z0.d
+; CHECK-NEXT:    not p4.b, p0/z, p10.b
+; CHECK-NEXT:    fcmgt p10.d, p0/z, z3.d, z1.d
+; CHECK-NEXT:    mov z26.d, p5/m, z0.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z28.d, z1.d
+; CHECK-NEXT:    mov z31.d, p6/m, z0.d
+; CHECK-NEXT:    mov z8.d, p4/m, z0.d
+; CHECK-NEXT:    sel z0.d, p3, z24.d, z4.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z5.d, z1.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z6.d, z1.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z7.d, z1.d
+; CHECK-NEXT:    sel z1.d, p7, z24.d, z29.d
+; CHECK-NEXT:    fcmuo p7.d, p0/z, z3.d, z3.d
+; CHECK-NEXT:    sel z2.d, p8, z24.d, z25.d
+; CHECK-NEXT:    sel z3.d, p10, z24.d, z27.d
+; CHECK-NEXT:    sel z4.d, p5, z24.d, z30.d
+; CHECK-NEXT:    fcmuo p5.d, p0/z, z28.d, z28.d
+; CHECK-NEXT:    fcmuo p8.d, p0/z, z5.d, z5.d
+; CHECK-NEXT:    fcmuo p10.d, p0/z, z6.d, z6.d
+; CHECK-NEXT:    sel z5.d, p3, z24.d, z26.d
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z7.d, z7.d
+; CHECK-NEXT:    sel z6.d, p4, z24.d, z31.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z7.d, p6, z24.d, z8.d
+; CHECK-NEXT:    ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    mov z6.d, p10/m, #0 // =0x0
+; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x iXLen> @llvm.lrint.nxv16iXLen.nxv16f64(<vscale x 16 x double> %x)
+  ret <vscale x 16 x iXLen> %a
+}
+declare <vscale x 16 x iXLen> @llvm.lrint.nxv16iXLen.nxv16f64(<vscale x 16 x double>)
+
+define <vscale x 32 x iXLen> @lrint_v32f64(<vscale x 32 x double> %x) {
+; CHECK-LABEL: lrint_v32f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-12
+; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xe0, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 96 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    rdvl x9, #8
+; CHECK-NEXT:    rdvl x10, #9
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    rdvl x11, #10
+; CHECK-NEXT:    mov x12, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    ld1b { z0.b }, p1/z, [x0, x9]
+; CHECK-NEXT:    ld1b { z1.b }, p1/z, [x0, x10]
+; CHECK-NEXT:    mov z2.d, x12
+; CHECK-NEXT:    rdvl x14, #13
+; CHECK-NEXT:    rdvl x13, #12
+; CHECK-NEXT:    rdvl x12, #11
+; CHECK-NEXT:    ld1b { z6.b }, p1/z, [x0, x14]
+; CHECK-NEXT:    ld1b { z7.b }, p1/z, [x0, x13]
+; CHECK-NEXT:    mov z3.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z24, z0
+; CHECK-NEXT:    frintx z24.d, p0/m, z0.d
+; CHECK-NEXT:    ld1b { z0.b }, p1/z, [x0, x11]
+; CHECK-NEXT:    movprfx z5, z1
+; CHECK-NEXT:    frintx z5.d, p0/m, z1.d
+; CHECK-NEXT:    ld1b { z1.b }, p1/z, [x0, x12]
+; CHECK-NEXT:    mov x15, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    rdvl x16, #15
+; CHECK-NEXT:    movprfx z30, z6
+; CHECK-NEXT:    frintx z30.d, p0/m, z6.d
+; CHECK-NEXT:    movprfx z28, z7
+; CHECK-NEXT:    frintx z28.d, p0/m, z7.d
+; CHECK-NEXT:    ld1b { z8.b }, p1/z, [x0, x16]
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    frintx z4.d, p0/m, z0.d
+; CHECK-NEXT:    mov z0.d, #0x7fffffffffffffff
+; CHECK-NEXT:    ld1d { z18.d }, p0/z, [x0]
+; CHECK-NEXT:    fcmge p3.d, p0/z, z5.d, z2.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z24.d, z2.d
+; CHECK-NEXT:    movprfx z6, z5
+; CHECK-NEXT:    fcvtzs z6.d, p0/m, z5.d
+; CHECK-NEXT:    movprfx z27, z1
+; CHECK-NEXT:    frintx z27.d, p0/m, z1.d
+; CHECK-NEXT:    movprfx z25, z24
+; CHECK-NEXT:    fcvtzs z25.d, p0/m, z24.d
+; CHECK-NEXT:    mov z1.d, x15
+; CHECK-NEXT:    rdvl x15, #14
+; CHECK-NEXT:    movprfx z9, z28
+; CHECK-NEXT:    fcvtzs z9.d, p0/m, z28.d
+; CHECK-NEXT:    movprfx z13, z8
+; CHECK-NEXT:    frintx z13.d, p0/m, z8.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z4.d, z2.d
+; CHECK-NEXT:    movprfx z7, z4
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z4.d
+; CHECK-NEXT:    ld1d { z15.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z24.d, z1.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z5.d, z1.d
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    fcmge p7.d, p0/z, z27.d, z2.d
+; CHECK-NEXT:    movprfx z26, z27
+; CHECK-NEXT:    fcvtzs z26.d, p0/m, z27.d
+; CHECK-NEXT:    sel z29.d, p3, z3.d, z6.d
+; CHECK-NEXT:    ld1b { z6.b }, p1/z, [x0, x15]
+; CHECK-NEXT:    fcmge p3.d, p0/z, z28.d, z2.d
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z25.d, p2/m, z3.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z4.d, z1.d
+; CHECK-NEXT:    movprfx z16, z13
+; CHECK-NEXT:    fcvtzs z16.d, p0/m, z13.d
+; CHECK-NEXT:    ld1d { z17.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1d { z14.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT:    sel z31.d, p4, z3.d, z7.d
+; CHECK-NEXT:    movprfx z11, z6
+; CHECK-NEXT:    frintx z11.d, p0/m, z6.d
+; CHECK-NEXT:    not p7.b, p0/z, p7.b
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    sel z6.d, p5, z0.d, z25.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z27.d, z1.d
+; CHECK-NEXT:    sel z7.d, p6, z0.d, z29.d
+; CHECK-NEXT:    mov z26.d, p7/m, z3.d
+; CHECK-NEXT:    fcmge p5.d, p0/z, z13.d, z2.d
+; CHECK-NEXT:    sel z25.d, p2, z0.d, z31.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z30.d, z2.d
+; CHECK-NEXT:    sel z29.d, p3, z3.d, z9.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z11.d, z2.d
+; CHECK-NEXT:    movprfx z31, z30
+; CHECK-NEXT:    fcvtzs z31.d, p0/m, z30.d
+; CHECK-NEXT:    movprfx z9, z11
+; CHECK-NEXT:    fcvtzs z9.d, p0/m, z11.d
+; CHECK-NEXT:    mov z26.d, p4/m, z0.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z28.d, z1.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z30.d, z1.d
+; CHECK-NEXT:    not p7.b, p0/z, p5.b
+; CHECK-NEXT:    fcmuo p5.d, p0/z, z27.d, z27.d
+; CHECK-NEXT:    fcmgt p8.d, p0/z, z13.d, z1.d
+; CHECK-NEXT:    not p2.b, p0/z, p2.b
+; CHECK-NEXT:    movprfx z27, z18
+; CHECK-NEXT:    frintx z27.d, p0/m, z18.d
+; CHECK-NEXT:    ld1d { z8.d }, p0/z, [x0, #7, mul vl]
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    mov z16.d, p7/m, z3.d
+; CHECK-NEXT:    fcmuo p7.d, p0/z, z13.d, z13.d
+; CHECK-NEXT:    mov z31.d, p2/m, z3.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z11.d, z1.d
+; CHECK-NEXT:    mov z29.d, p4/m, z0.d
+; CHECK-NEXT:    mov z9.d, p3/m, z3.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z28.d, z28.d
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z30.d, z30.d
+; CHECK-NEXT:    movprfx z28, z17
+; CHECK-NEXT:    frintx z28.d, p0/m, z17.d
+; CHECK-NEXT:    movprfx z30, z15
+; CHECK-NEXT:    frintx z30.d, p0/m, z15.d
+; CHECK-NEXT:    ld1d { z13.d }, p0/z, [x0, #4, mul vl]
+; CHECK-NEXT:    mov z31.d, p6/m, z0.d
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z11.d, z11.d
+; CHECK-NEXT:    sel z11.d, p8, z0.d, z16.d
+; CHECK-NEXT:    mov z9.d, p2/m, z0.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z24.d, z24.d
+; CHECK-NEXT:    movprfx z24, z14
+; CHECK-NEXT:    frintx z24.d, p0/m, z14.d
+; CHECK-NEXT:    fcmge p8.d, p0/z, z27.d, z2.d
+; CHECK-NEXT:    ld1d { z10.d }, p0/z, [x0, #6, mul vl]
+; CHECK-NEXT:    ld1d { z12.d }, p0/z, [x0, #5, mul vl]
+; CHECK-NEXT:    mov z26.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z29.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p5.d, p0/z, z28.d, z2.d
+; CHECK-NEXT:    movprfx z14, z27
+; CHECK-NEXT:    fcvtzs z14.d, p0/m, z27.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z30.d, z2.d
+; CHECK-NEXT:    frintx z13.d, p0/m, z13.d
+; CHECK-NEXT:    mov z31.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p4.d, p0/z, z24.d, z2.d
+; CHECK-NEXT:    mov z9.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    movprfx z15, z28
+; CHECK-NEXT:    fcvtzs z15.d, p0/m, z28.d
+; CHECK-NEXT:    not p6.b, p0/z, p8.b
+; CHECK-NEXT:    movprfx z16, z30
+; CHECK-NEXT:    fcvtzs z16.d, p0/m, z30.d
+; CHECK-NEXT:    frintx z12.d, p0/m, z12.d
+; CHECK-NEXT:    frintx z10.d, p0/m, z10.d
+; CHECK-NEXT:    movprfx z17, z24
+; CHECK-NEXT:    fcvtzs z17.d, p0/m, z24.d
+; CHECK-NEXT:    movprfx z18, z8
+; CHECK-NEXT:    frintx z18.d, p0/m, z8.d
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    sel z8.d, p6, z3.d, z14.d
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    fcmge p6.d, p0/z, z13.d, z2.d
+; CHECK-NEXT:    mov z11.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    sel z14.d, p5, z3.d, z15.d
+; CHECK-NEXT:    fcmuo p7.d, p0/z, z5.d, z5.d
+; CHECK-NEXT:    sel z15.d, p3, z3.d, z16.d
+; CHECK-NEXT:    movprfx z16, z13
+; CHECK-NEXT:    fcvtzs z16.d, p0/m, z13.d
+; CHECK-NEXT:    fcmge p5.d, p0/z, z12.d, z2.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z10.d, z2.d
+; CHECK-NEXT:    sel z5.d, p4, z3.d, z17.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z18.d, z2.d
+; CHECK-NEXT:    not p6.b, p0/z, p6.b
+; CHECK-NEXT:    movprfx z2, z12
+; CHECK-NEXT:    fcvtzs z2.d, p0/m, z12.d
+; CHECK-NEXT:    movprfx z17, z10
+; CHECK-NEXT:    fcvtzs z17.d, p0/m, z10.d
+; CHECK-NEXT:    st1b { z11.b }, p1, [x8, x16]
+; CHECK-NEXT:    movprfx z11, z18
+; CHECK-NEXT:    fcvtzs z11.d, p0/m, z18.d
+; CHECK-NEXT:    mov z6.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    st1b { z9.b }, p1, [x8, x15]
+; CHECK-NEXT:    sel z9.d, p6, z3.d, z16.d
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z4.d, z4.d
+; CHECK-NEXT:    not p5.b, p0/z, p5.b
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z18.d, z1.d
+; CHECK-NEXT:    mov z7.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    not p3.b, p0/z, p3.b
+; CHECK-NEXT:    st1b { z31.b }, p1, [x8, x14]
+; CHECK-NEXT:    fcmgt p7.d, p0/z, z24.d, z1.d
+; CHECK-NEXT:    not p4.b, p0/z, p4.b
+; CHECK-NEXT:    mov z2.d, p5/m, z3.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z28.d, z1.d
+; CHECK-NEXT:    sel z4.d, p3, z3.d, z17.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z13.d, z1.d
+; CHECK-NEXT:    mov z25.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    sel z3.d, p4, z3.d, z11.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z10.d, z1.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z12.d, z1.d
+; CHECK-NEXT:    st1b { z29.b }, p1, [x8, x13]
+; CHECK-NEXT:    st1b { z26.b }, p1, [x8, x12]
+; CHECK-NEXT:    sel z26.d, p5, z0.d, z14.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z30.d, z1.d
+; CHECK-NEXT:    sel z29.d, p3, z0.d, z9.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z18.d, z18.d
+; CHECK-NEXT:    mov z3.d, p2/m, z0.d
+; CHECK-NEXT:    st1b { z25.b }, p1, [x8, x11]
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z10.d, z10.d
+; CHECK-NEXT:    mov z4.d, p4/m, z0.d
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z12.d, z12.d
+; CHECK-NEXT:    st1b { z7.b }, p1, [x8, x10]
+; CHECK-NEXT:    mov z2.d, p6/m, z0.d
+; CHECK-NEXT:    st1b { z6.b }, p1, [x8, x9]
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z13.d, z13.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z27.d, z1.d
+; CHECK-NEXT:    mov z3.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z24.d, z24.d
+; CHECK-NEXT:    sel z1.d, p7, z0.d, z5.d
+; CHECK-NEXT:    mov z4.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z30.d, z30.d
+; CHECK-NEXT:    sel z5.d, p5, z0.d, z15.d
+; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z28.d, z28.d
+; CHECK-NEXT:    mov z29.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z27.d, z27.d
+; CHECK-NEXT:    sel z0.d, p6, z0.d, z8.d
+; CHECK-NEXT:    mov z1.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z3.d }, p0, [x8, #7, mul vl]
+; CHECK-NEXT:    mov z5.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z4.d }, p0, [x8, #6, mul vl]
+; CHECK-NEXT:    mov z26.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z2.d }, p0, [x8, #5, mul vl]
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    st1d { z29.d }, p0, [x8, #4, mul vl]
+; CHECK-NEXT:    st1d { z1.d }, p0, [x8, #3, mul vl]
+; CHECK-NEXT:    st1d { z5.d }, p0, [x8, #2, mul vl]
+; CHECK-NEXT:    st1d { z26.d }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x8]
+; CHECK-NEXT:    ldr z18, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #12
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <vscale x 32 x iXLen> @llvm.lrint.nxv32iXLen.nxv16f64(<vscale x 32 x double> %x)
+  ret <vscale x 32 x iXLen> %a
+}
+declare <vscale x 32 x iXLen> @llvm.lrint.nxv32iXLen.nxv32f64(<vscale x 32 x double>)
diff --git a/llvm/test/CodeGen/AArch64/vector-llrint.ll b/llvm/test/CodeGen/AArch64/vector-llrint.ll
index beb2b6a13460..b7e743b5085f 100644
--- a/llvm/test/CodeGen/AArch64/vector-llrint.ll
+++ b/llvm/test/CodeGen/AArch64/vector-llrint.ll
@@ -386,10 +386,8 @@ declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
 define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
 ; CHECK-LABEL: llrint_v2i64_v2f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    frintx v0.2s, v0.2s
 ; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    frintx s1, s1
 ; CHECK-NEXT:    fcvtzs x8, s0
 ; CHECK-NEXT:    fcvtzs x9, s1
 ; CHECK-NEXT:    fmov d0, x8
@@ -404,20 +402,18 @@ define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: llrint_v4i64_v4f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    mov s3, v0.s[1]
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    mov s2, v1.s[1]
-; CHECK-NEXT:    frintx s1, s1
-; CHECK-NEXT:    frintx s3, s3
-; CHECK-NEXT:    fcvtzs x9, s0
-; CHECK-NEXT:    frintx s2, s2
-; CHECK-NEXT:    fcvtzs x8, s1
-; CHECK-NEXT:    fcvtzs x11, s3
-; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    frintx v0.2s, v0.2s
+; CHECK-NEXT:    frintx v1.2s, v1.2s
+; CHECK-NEXT:    mov s2, v0.s[1]
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    mov s3, v1.s[1]
+; CHECK-NEXT:    fcvtzs x9, s1
 ; CHECK-NEXT:    fcvtzs x10, s2
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    mov v0.d[1], x11
-; CHECK-NEXT:    mov v1.d[1], x10
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fcvtzs x11, s3
+; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    mov v0.d[1], x10
+; CHECK-NEXT:    mov v1.d[1], x11
 ; CHECK-NEXT:    ret
   %a = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> %x)
   ret <4 x i64> %a
@@ -429,34 +425,30 @@ define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    frintx v0.2s, v0.2s
+; CHECK-NEXT:    frintx v1.2s, v1.2s
+; CHECK-NEXT:    frintx v2.2s, v2.2s
+; CHECK-NEXT:    frintx v3.2s, v3.2s
 ; CHECK-NEXT:    mov s4, v0.s[1]
-; CHECK-NEXT:    mov s7, v1.s[1]
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    frintx s1, s1
-; CHECK-NEXT:    mov s5, v2.s[1]
-; CHECK-NEXT:    mov s6, v3.s[1]
-; CHECK-NEXT:    frintx s2, s2
-; CHECK-NEXT:    frintx s3, s3
-; CHECK-NEXT:    frintx s4, s4
-; CHECK-NEXT:    frintx s7, s7
-; CHECK-NEXT:    fcvtzs x9, s0
-; CHECK-NEXT:    fcvtzs x12, s1
-; CHECK-NEXT:    frintx s5, s5
-; CHECK-NEXT:    frintx s6, s6
-; CHECK-NEXT:    fcvtzs x8, s2
-; CHECK-NEXT:    fcvtzs x10, s3
-; CHECK-NEXT:    fcvtzs x11, s4
-; CHECK-NEXT:    fcvtzs x15, s7
-; CHECK-NEXT:    fmov d0, x9
-; CHECK-NEXT:    fmov d2, x12
+; CHECK-NEXT:    mov s5, v1.s[1]
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fcvtzs x10, s1
+; CHECK-NEXT:    mov s6, v2.s[1]
+; CHECK-NEXT:    mov s7, v3.s[1]
+; CHECK-NEXT:    fcvtzs x11, s2
+; CHECK-NEXT:    fcvtzs x12, s3
+; CHECK-NEXT:    fcvtzs x9, s4
 ; CHECK-NEXT:    fcvtzs x13, s5
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d2, x10
 ; CHECK-NEXT:    fcvtzs x14, s6
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    fmov d3, x10
-; CHECK-NEXT:    mov v0.d[1], x11
-; CHECK-NEXT:    mov v2.d[1], x15
-; CHECK-NEXT:    mov v1.d[1], x13
-; CHECK-NEXT:    mov v3.d[1], x14
+; CHECK-NEXT:    fcvtzs x15, s7
+; CHECK-NEXT:    fmov d1, x11
+; CHECK-NEXT:    fmov d3, x12
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    mov v2.d[1], x13
+; CHECK-NEXT:    mov v1.d[1], x14
+; CHECK-NEXT:    mov v3.d[1], x15
 ; CHECK-NEXT:    ret
   %a = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> %x)
   ret <8 x i64> %a
@@ -466,72 +458,185 @@ declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
 define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 ; CHECK-LABEL: llrint_v16i64_v16f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v4.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v5.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    frintx v4.2s, v0.2s
+; CHECK-NEXT:    frintx v5.2s, v1.2s
+; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
 ; CHECK-NEXT:    ext v6.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT:    frintx s7, s0
-; CHECK-NEXT:    ext v16.16b, v3.16b, v3.16b, #8
-; CHECK-NEXT:    mov s0, v0.s[1]
-; CHECK-NEXT:    frintx s17, s4
-; CHECK-NEXT:    mov s4, v4.s[1]
-; CHECK-NEXT:    mov s18, v5.s[1]
-; CHECK-NEXT:    frintx s5, s5
-; CHECK-NEXT:    frintx s19, s6
-; CHECK-NEXT:    fcvtzs x8, s7
-; CHECK-NEXT:    frintx s7, s16
-; CHECK-NEXT:    mov s6, v6.s[1]
-; CHECK-NEXT:    mov s16, v16.s[1]
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    frintx s4, s4
-; CHECK-NEXT:    fcvtzs x9, s17
-; CHECK-NEXT:    frintx s17, s1
-; CHECK-NEXT:    mov s1, v1.s[1]
-; CHECK-NEXT:    frintx s18, s18
+; CHECK-NEXT:    ext v7.16b, v3.16b, v3.16b, #8
+; CHECK-NEXT:    frintx v2.2s, v2.2s
+; CHECK-NEXT:    frintx v3.2s, v3.2s
+; CHECK-NEXT:    mov s16, v4.s[1]
+; CHECK-NEXT:    mov s17, v5.s[1]
+; CHECK-NEXT:    fcvtzs x8, s4
+; CHECK-NEXT:    frintx v0.2s, v0.2s
+; CHECK-NEXT:    frintx v1.2s, v1.2s
+; CHECK-NEXT:    fcvtzs x9, s5
+; CHECK-NEXT:    frintx v4.2s, v6.2s
+; CHECK-NEXT:    frintx v5.2s, v7.2s
+; CHECK-NEXT:    fcvtzs x10, s2
+; CHECK-NEXT:    mov s6, v2.s[1]
+; CHECK-NEXT:    fcvtzs x13, s3
+; CHECK-NEXT:    mov s3, v3.s[1]
+; CHECK-NEXT:    fcvtzs x11, s16
+; CHECK-NEXT:    fcvtzs x12, s17
+; CHECK-NEXT:    mov s7, v0.s[1]
+; CHECK-NEXT:    mov s16, v1.s[1]
+; CHECK-NEXT:    fcvtzs x15, s1
+; CHECK-NEXT:    mov s1, v4.s[1]
+; CHECK-NEXT:    mov s17, v5.s[1]
+; CHECK-NEXT:    fcvtzs x14, s0
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fcvtzs x8, s4
+; CHECK-NEXT:    fmov d4, x10
 ; CHECK-NEXT:    fcvtzs x10, s5
-; CHECK-NEXT:    mov s5, v2.s[1]
-; CHECK-NEXT:    fcvtzs x11, s19
-; CHECK-NEXT:    mov s19, v3.s[1]
-; CHECK-NEXT:    frintx s2, s2
-; CHECK-NEXT:    fcvtzs x12, s7
-; CHECK-NEXT:    frintx s6, s6
-; CHECK-NEXT:    fcvtzs x13, s4
-; CHECK-NEXT:    frintx s4, s3
-; CHECK-NEXT:    frintx s16, s16
-; CHECK-NEXT:    fcvtzs x14, s18
-; CHECK-NEXT:    frintx s18, s1
-; CHECK-NEXT:    fcvtzs x15, s17
-; CHECK-NEXT:    frintx s20, s5
-; CHECK-NEXT:    frintx s17, s19
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    fcvtzs x9, s2
-; CHECK-NEXT:    fmov d5, x11
-; CHECK-NEXT:    fmov d3, x10
-; CHECK-NEXT:    fcvtzs x11, s4
-; CHECK-NEXT:    fcvtzs x10, s0
-; CHECK-NEXT:    fmov d7, x12
-; CHECK-NEXT:    fcvtzs x12, s18
-; CHECK-NEXT:    fcvtzs x17, s6
-; CHECK-NEXT:    fcvtzs x18, s16
-; CHECK-NEXT:    fcvtzs x16, s20
+; CHECK-NEXT:    fmov d2, x9
+; CHECK-NEXT:    fcvtzs x9, s6
+; CHECK-NEXT:    fmov d6, x13
+; CHECK-NEXT:    fcvtzs x13, s7
+; CHECK-NEXT:    fcvtzs x16, s16
+; CHECK-NEXT:    fcvtzs x17, s3
+; CHECK-NEXT:    fcvtzs x18, s1
 ; CHECK-NEXT:    fcvtzs x0, s17
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d2, x15
-; CHECK-NEXT:    fmov d4, x9
-; CHECK-NEXT:    mov v1.d[1], x13
-; CHECK-NEXT:    fmov d6, x11
-; CHECK-NEXT:    mov v3.d[1], x14
-; CHECK-NEXT:    mov v0.d[1], x10
-; CHECK-NEXT:    mov v5.d[1], x17
-; CHECK-NEXT:    mov v7.d[1], x18
+; CHECK-NEXT:    fmov d1, x14
+; CHECK-NEXT:    fmov d3, x15
+; CHECK-NEXT:    fmov d5, x8
+; CHECK-NEXT:    fmov d7, x10
+; CHECK-NEXT:    mov v0.d[1], x11
 ; CHECK-NEXT:    mov v2.d[1], x12
-; CHECK-NEXT:    mov v4.d[1], x16
-; CHECK-NEXT:    mov v6.d[1], x0
+; CHECK-NEXT:    mov v4.d[1], x9
+; CHECK-NEXT:    mov v1.d[1], x13
+; CHECK-NEXT:    mov v3.d[1], x16
+; CHECK-NEXT:    mov v6.d[1], x17
+; CHECK-NEXT:    mov v5.d[1], x18
+; CHECK-NEXT:    mov v7.d[1], x0
 ; CHECK-NEXT:    ret
   %a = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> %x)
   ret <16 x i64> %a
 }
 declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
 
+define <32 x i64> @llrint_v32i64_v32f32(<32 x float> %x) {
+; CHECK-LABEL: llrint_v32i64_v32f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v17.16b, v3.16b, v3.16b, #8
+; CHECK-NEXT:    ext v18.16b, v4.16b, v4.16b, #8
+; CHECK-NEXT:    ext v19.16b, v5.16b, v5.16b, #8
+; CHECK-NEXT:    ext v21.16b, v7.16b, v7.16b, #8
+; CHECK-NEXT:    ext v16.16b, v2.16b, v2.16b, #8
+; CHECK-NEXT:    ext v20.16b, v6.16b, v6.16b, #8
+; CHECK-NEXT:    frintx v7.2s, v7.2s
+; CHECK-NEXT:    frintx v24.2s, v6.2s
+; CHECK-NEXT:    frintx v23.2s, v5.2s
+; CHECK-NEXT:    frintx v4.2s, v4.2s
+; CHECK-NEXT:    frintx v3.2s, v3.2s
+; CHECK-NEXT:    frintx v17.2s, v17.2s
+; CHECK-NEXT:    frintx v18.2s, v18.2s
+; CHECK-NEXT:    frintx v22.2s, v19.2s
+; CHECK-NEXT:    frintx v21.2s, v21.2s
+; CHECK-NEXT:    frintx v16.2s, v16.2s
+; CHECK-NEXT:    frintx v20.2s, v20.2s
+; CHECK-NEXT:    mov s25, v7.s[1]
+; CHECK-NEXT:    fcvtzs x15, s7
+; CHECK-NEXT:    frintx v19.2s, v1.2s
+; CHECK-NEXT:    fcvtzs x16, s24
+; CHECK-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    fcvtzs x10, s17
+; CHECK-NEXT:    fcvtzs x11, s18
+; CHECK-NEXT:    mov s26, v22.s[1]
+; CHECK-NEXT:    fcvtzs x12, s22
+; CHECK-NEXT:    mov s22, v21.s[1]
+; CHECK-NEXT:    fcvtzs x14, s21
+; CHECK-NEXT:    mov s21, v24.s[1]
+; CHECK-NEXT:    fcvtzs x9, s16
+; CHECK-NEXT:    fcvtzs x13, s20
+; CHECK-NEXT:    mov s20, v20.s[1]
+; CHECK-NEXT:    fmov d24, x15
+; CHECK-NEXT:    mov s18, v18.s[1]
+; CHECK-NEXT:    fmov d6, x10
+; CHECK-NEXT:    fmov d7, x11
+; CHECK-NEXT:    fcvtzs x10, s25
+; CHECK-NEXT:    fcvtzs x11, s22
+; CHECK-NEXT:    fmov d25, x12
+; CHECK-NEXT:    frintx v22.2s, v2.2s
+; CHECK-NEXT:    fcvtzs x15, s21
+; CHECK-NEXT:    fmov d21, x14
+; CHECK-NEXT:    fmov d5, x9
+; CHECK-NEXT:    fcvtzs x9, s26
+; CHECK-NEXT:    fmov d26, x13
+; CHECK-NEXT:    fcvtzs x12, s20
+; CHECK-NEXT:    fcvtzs x13, s19
+; CHECK-NEXT:    mov s20, v23.s[1]
+; CHECK-NEXT:    mov v24.d[1], x10
+; CHECK-NEXT:    mov v21.d[1], x11
+; CHECK-NEXT:    fcvtzs x11, s23
+; CHECK-NEXT:    fcvtzs x10, s22
+; CHECK-NEXT:    mov s17, v17.s[1]
+; CHECK-NEXT:    frintx v1.2s, v1.2s
+; CHECK-NEXT:    mov s22, v22.s[1]
+; CHECK-NEXT:    mov v26.d[1], x12
+; CHECK-NEXT:    fcvtzs x12, s18
+; CHECK-NEXT:    mov v25.d[1], x9
+; CHECK-NEXT:    fmov d2, x13
+; CHECK-NEXT:    fcvtzs x13, s20
+; CHECK-NEXT:    fmov d20, x16
+; CHECK-NEXT:    stp q24, q21, [x8, #224]
+; CHECK-NEXT:    ext v21.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    fmov d18, x11
+; CHECK-NEXT:    fcvtzs x11, s4
+; CHECK-NEXT:    mov s4, v4.s[1]
+; CHECK-NEXT:    fmov d23, x10
+; CHECK-NEXT:    mov v20.d[1], x15
+; CHECK-NEXT:    fcvtzs x10, s3
+; CHECK-NEXT:    mov s3, v3.s[1]
+; CHECK-NEXT:    mov v18.d[1], x13
+; CHECK-NEXT:    frintx v0.2s, v0.2s
+; CHECK-NEXT:    mov s16, v16.s[1]
+; CHECK-NEXT:    frintx v21.2s, v21.2s
+; CHECK-NEXT:    fcvtzs x13, s17
+; CHECK-NEXT:    fcvtzs x14, s22
+; CHECK-NEXT:    fcvtzs x9, s4
+; CHECK-NEXT:    fmov d4, x11
+; CHECK-NEXT:    mov v7.d[1], x12
+; CHECK-NEXT:    stp q20, q26, [x8, #192]
+; CHECK-NEXT:    fmov d20, x10
+; CHECK-NEXT:    fcvtzs x10, s3
+; CHECK-NEXT:    stp q18, q25, [x8, #160]
+; CHECK-NEXT:    mov s18, v19.s[1]
+; CHECK-NEXT:    mov s3, v1.s[1]
+; CHECK-NEXT:    mov s17, v0.s[1]
+; CHECK-NEXT:    mov s19, v21.s[1]
+; CHECK-NEXT:    fcvtzs x11, s21
+; CHECK-NEXT:    mov v4.d[1], x9
+; CHECK-NEXT:    fcvtzs x9, s16
+; CHECK-NEXT:    fcvtzs x12, s1
+; CHECK-NEXT:    mov v6.d[1], x13
+; CHECK-NEXT:    fcvtzs x13, s0
+; CHECK-NEXT:    mov v20.d[1], x10
+; CHECK-NEXT:    fcvtzs x15, s18
+; CHECK-NEXT:    fcvtzs x10, s3
+; CHECK-NEXT:    mov v23.d[1], x14
+; CHECK-NEXT:    fcvtzs x14, s17
+; CHECK-NEXT:    fmov d3, x11
+; CHECK-NEXT:    stp q4, q7, [x8, #128]
+; CHECK-NEXT:    mov v5.d[1], x9
+; CHECK-NEXT:    fcvtzs x9, s19
+; CHECK-NEXT:    stp q20, q6, [x8, #96]
+; CHECK-NEXT:    fmov d0, x12
+; CHECK-NEXT:    fmov d1, x13
+; CHECK-NEXT:    mov v2.d[1], x15
+; CHECK-NEXT:    stp q23, q5, [x8, #64]
+; CHECK-NEXT:    mov v0.d[1], x10
+; CHECK-NEXT:    mov v1.d[1], x14
+; CHECK-NEXT:    mov v3.d[1], x9
+; CHECK-NEXT:    stp q2, q0, [x8, #32]
+; CHECK-NEXT:    stp q1, q3, [x8]
+; CHECK-NEXT:    ret
+  %a = call <32 x i64> @llvm.llrint.v32i64.v32f32(<32 x float> %x)
+  ret <32 x i64> %a
+}
+declare <32 x i64> @llvm.llrint.v32i64.v32f32(<32 x float>)
+
 define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
 ; CHECK-LABEL: llrint_v1i64_v1f64:
 ; CHECK:       // %bb.0:
@@ -547,13 +652,8 @@ declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
 define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
 ; CHECK-LABEL: llrint_v2i64_v2f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d1, v0.d[1]
-; CHECK-NEXT:    frintx d0, d0
-; CHECK-NEXT:    frintx d1, d1
-; CHECK-NEXT:    fcvtzs x8, d0
-; CHECK-NEXT:    fcvtzs x9, d1
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    frintx v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
 ; CHECK-NEXT:    ret
   %a = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> %x)
   ret <2 x i64> %a
@@ -563,20 +663,10 @@ declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
 define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
 ; CHECK-LABEL: llrint_v4i64_v4f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d2, v0.d[1]
-; CHECK-NEXT:    mov d3, v1.d[1]
-; CHECK-NEXT:    frintx d0, d0
-; CHECK-NEXT:    frintx d1, d1
-; CHECK-NEXT:    frintx d2, d2
-; CHECK-NEXT:    frintx d3, d3
-; CHECK-NEXT:    fcvtzs x8, d0
-; CHECK-NEXT:    fcvtzs x9, d1
-; CHECK-NEXT:    fcvtzs x10, d2
-; CHECK-NEXT:    fcvtzs x11, d3
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    mov v0.d[1], x10
-; CHECK-NEXT:    mov v1.d[1], x11
+; CHECK-NEXT:    frintx v0.2d, v0.2d
+; CHECK-NEXT:    frintx v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
 ; CHECK-NEXT:    ret
   %a = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> %x)
   ret <4 x i64> %a
@@ -586,36 +676,94 @@ declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
 define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
 ; CHECK-LABEL: llrint_v8i64_v8f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d4, v0.d[1]
-; CHECK-NEXT:    mov d5, v1.d[1]
-; CHECK-NEXT:    mov d6, v2.d[1]
-; CHECK-NEXT:    mov d7, v3.d[1]
-; CHECK-NEXT:    frintx d0, d0
-; CHECK-NEXT:    frintx d1, d1
-; CHECK-NEXT:    frintx d2, d2
-; CHECK-NEXT:    frintx d3, d3
-; CHECK-NEXT:    frintx d4, d4
-; CHECK-NEXT:    frintx d5, d5
-; CHECK-NEXT:    frintx d6, d6
-; CHECK-NEXT:    frintx d7, d7
-; CHECK-NEXT:    fcvtzs x8, d0
-; CHECK-NEXT:    fcvtzs x9, d1
-; CHECK-NEXT:    fcvtzs x10, d2
-; CHECK-NEXT:    fcvtzs x11, d3
-; CHECK-NEXT:    fcvtzs x12, d4
-; CHECK-NEXT:    fcvtzs x13, d5
-; CHECK-NEXT:    fcvtzs x14, d6
-; CHECK-NEXT:    fcvtzs x15, d7
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    fmov d2, x10
-; CHECK-NEXT:    fmov d3, x11
-; CHECK-NEXT:    mov v0.d[1], x12
-; CHECK-NEXT:    mov v1.d[1], x13
-; CHECK-NEXT:    mov v2.d[1], x14
-; CHECK-NEXT:    mov v3.d[1], x15
+; CHECK-NEXT:    frintx v0.2d, v0.2d
+; CHECK-NEXT:    frintx v1.2d, v1.2d
+; CHECK-NEXT:    frintx v2.2d, v2.2d
+; CHECK-NEXT:    frintx v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
 ; CHECK-NEXT:    ret
   %a = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> %x)
   ret <8 x i64> %a
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double>)
+
+define <16 x i64> @llrint_v16f64(<16 x double> %x) {
+; CHECK-LABEL: llrint_v16f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx v0.2d, v0.2d
+; CHECK-NEXT:    frintx v1.2d, v1.2d
+; CHECK-NEXT:    frintx v2.2d, v2.2d
+; CHECK-NEXT:    frintx v3.2d, v3.2d
+; CHECK-NEXT:    frintx v4.2d, v4.2d
+; CHECK-NEXT:    frintx v5.2d, v5.2d
+; CHECK-NEXT:    frintx v6.2d, v6.2d
+; CHECK-NEXT:    frintx v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    ret
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double>)
+
+define <32 x i64> @llrint_v32f64(<32 x double> %x) {
+; CHECK-LABEL: llrint_v32f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q17, q16, [sp, #96]
+; CHECK-NEXT:    frintx v7.2d, v7.2d
+; CHECK-NEXT:    ldp q19, q18, [sp, #64]
+; CHECK-NEXT:    frintx v6.2d, v6.2d
+; CHECK-NEXT:    ldp q21, q20, [sp, #32]
+; CHECK-NEXT:    frintx v5.2d, v5.2d
+; CHECK-NEXT:    frintx v16.2d, v16.2d
+; CHECK-NEXT:    frintx v17.2d, v17.2d
+; CHECK-NEXT:    frintx v4.2d, v4.2d
+; CHECK-NEXT:    frintx v18.2d, v18.2d
+; CHECK-NEXT:    frintx v19.2d, v19.2d
+; CHECK-NEXT:    frintx v3.2d, v3.2d
+; CHECK-NEXT:    ldp q23, q22, [sp]
+; CHECK-NEXT:    frintx v20.2d, v20.2d
+; CHECK-NEXT:    frintx v21.2d, v21.2d
+; CHECK-NEXT:    frintx v2.2d, v2.2d
+; CHECK-NEXT:    frintx v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NEXT:    frintx v0.2d, v0.2d
+; CHECK-NEXT:    frintx v22.2d, v22.2d
+; CHECK-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-NEXT:    frintx v23.2d, v23.2d
+; CHECK-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    stp q17, q16, [x8, #224]
+; CHECK-NEXT:    fcvtzs v16.2d, v21.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v17.2d, v22.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    stp q19, q18, [x8, #192]
+; CHECK-NEXT:    fcvtzs v18.2d, v23.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    stp q4, q5, [x8, #64]
+; CHECK-NEXT:    stp q6, q7, [x8, #96]
+; CHECK-NEXT:    stp q2, q3, [x8, #32]
+; CHECK-NEXT:    stp q0, q1, [x8]
+; CHECK-NEXT:    stp q18, q17, [x8, #128]
+; CHECK-NEXT:    stp q16, q20, [x8, #160]
+; CHECK-NEXT:    ret
+  %a = call <32 x i64> @llvm.llrint.v32i64.v16f64(<32 x double> %x)
+  ret <32 x i64> %a
+}
+declare <32 x i64> @llvm.llrint.v32i64.v32f64(<32 x double>)
diff --git a/llvm/test/CodeGen/AArch64/vector-lrint.ll b/llvm/test/CodeGen/AArch64/vector-lrint.ll
index db85b2342821..44f29f1420fe 100644
--- a/llvm/test/CodeGen/AArch64/vector-lrint.ll
+++ b/llvm/test/CodeGen/AArch64/vector-lrint.ll
@@ -1,642 +1,1430 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=aarch64 -mattr=+neon |\
+; RUN:   FileCheck %s --check-prefixes=CHECK-i32
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=aarch64 -mattr=+neon |\
+; RUN:   FileCheck %s --check-prefixes=CHECK-i64,CHECK-i64-SD
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=aarch64 -mattr=+neon \
+; RUN:   -global-isel -global-isel-abort=2 2>&1 |\
+; RUN:   FileCheck %s --check-prefixes=CHECK-i32,CHECK-i32-GI
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=aarch64 -mattr=+neon \
+; RUN:   -global-isel -global-isel-abort=2 2>&1 |\
+; RUN:   FileCheck %s --check-prefixes=CHECK-i64,CHECK-i64-GI
 
-; CHECK-GI:       warning: Instruction selection used fallback path for lrint_v2f16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v4f16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v8f16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v16i64_v16f16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v32i64_v32f16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v2f32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v4f32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v8f32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v16i64_v16f32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v2f64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v4f64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v8f64
+; CHECK-i32-GI:       warning: Instruction selection used fallback path for lrint_v1f16
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v2f16
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v4f16
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v8f16
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v16f16
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v32f16
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v1f32
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v2f32
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v4f32
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v8f32
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v16f32
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v32f32
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v1f64
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v2f64
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v4f64
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v8f64
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v16f64
+; CHECK-i32-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v32f64
 
-define <1 x i64> @lrint_v1f16(<1 x half> %x) {
-; CHECK-LABEL: lrint_v1f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fcvt s0, h0
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    ret
-  %a = call <1 x i64> @llvm.lrint.v1i64.v1f16(<1 x half> %x)
-  ret <1 x i64> %a
+; CHECK-i64-GI:       warning: Instruction selection used fallback path for lrint_v2f16
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v4f16
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v8f16
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v16f16
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v32f16
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v2f32
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v4f32
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v8f32
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v16f32
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v32f32
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v2f64
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v4f64
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v8f64
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v16f64
+; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v32f64
+
+define <1 x iXLen> @lrint_v1f16(<1 x half> %x) {
+; CHECK-i32-LABEL: lrint_v1f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    fcvt s0, h0
+; CHECK-i32-NEXT:    frintx s0, s0
+; CHECK-i32-NEXT:    fcvtzs w8, s0
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v1f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    fcvt s0, h0
+; CHECK-i64-NEXT:    frintx s0, s0
+; CHECK-i64-NEXT:    fcvtzs x8, s0
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    ret
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f16(<1 x half> %x)
+  ret <1 x iXLen> %a
 }
-declare <1 x i64> @llvm.lrint.v1i64.v1f16(<1 x half>)
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f16(<1 x half>)
 
-define <2 x i64> @lrint_v2f16(<2 x half> %x) {
-; CHECK-LABEL: lrint_v2f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov h1, v0.h[1]
-; CHECK-NEXT:    fcvt s0, h0
-; CHECK-NEXT:    fcvt s1, h1
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    frintx s1, s1
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fcvtzs x9, s1
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
-; CHECK-NEXT:    ret
-  %a = call <2 x i64> @llvm.lrint.v2i64.v2f16(<2 x half> %x)
-  ret <2 x i64> %a
+define <2 x iXLen> @lrint_v2f16(<2 x half> %x) {
+; CHECK-i32-LABEL: lrint_v2f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-i32-NEXT:    mov h1, v0.h[1]
+; CHECK-i32-NEXT:    fcvt s0, h0
+; CHECK-i32-NEXT:    fcvt s1, h1
+; CHECK-i32-NEXT:    frintx s0, s0
+; CHECK-i32-NEXT:    frintx s1, s1
+; CHECK-i32-NEXT:    fcvtzs w8, s0
+; CHECK-i32-NEXT:    fcvtzs w9, s1
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    mov v0.s[1], w9
+; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v2f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-i64-NEXT:    mov h1, v0.h[1]
+; CHECK-i64-NEXT:    fcvt s0, h0
+; CHECK-i64-NEXT:    fcvt s1, h1
+; CHECK-i64-NEXT:    frintx s0, s0
+; CHECK-i64-NEXT:    frintx s1, s1
+; CHECK-i64-NEXT:    fcvtzs x8, s0
+; CHECK-i64-NEXT:    fcvtzs x9, s1
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    mov v0.d[1], x9
+; CHECK-i64-NEXT:    ret
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f16(<2 x half> %x)
+  ret <2 x iXLen> %a
 }
-declare <2 x i64> @llvm.lrint.v2i64.v2f16(<2 x half>)
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f16(<2 x half>)
 
-define <4 x i64> @lrint_v4f16(<4 x half> %x) {
-; CHECK-LABEL: lrint_v4f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov h1, v0.h[2]
-; CHECK-NEXT:    mov h2, v0.h[1]
-; CHECK-NEXT:    mov h3, v0.h[3]
-; CHECK-NEXT:    fcvt s0, h0
-; CHECK-NEXT:    fcvt s1, h1
-; CHECK-NEXT:    fcvt s2, h2
-; CHECK-NEXT:    fcvt s3, h3
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    frintx s1, s1
-; CHECK-NEXT:    frintx s2, s2
-; CHECK-NEXT:    frintx s3, s3
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fcvtzs x9, s1
-; CHECK-NEXT:    fcvtzs x10, s2
-; CHECK-NEXT:    fcvtzs x11, s3
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    mov v0.d[1], x10
-; CHECK-NEXT:    mov v1.d[1], x11
-; CHECK-NEXT:    ret
-  %a = call <4 x i64> @llvm.lrint.v4i64.v4f16(<4 x half> %x)
-  ret <4 x i64> %a
+define <4 x iXLen> @lrint_v4f16(<4 x half> %x) {
+; CHECK-i32-LABEL: lrint_v4f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-i32-NEXT:    mov h1, v0.h[1]
+; CHECK-i32-NEXT:    fcvt s2, h0
+; CHECK-i32-NEXT:    mov h3, v0.h[2]
+; CHECK-i32-NEXT:    mov h0, v0.h[3]
+; CHECK-i32-NEXT:    fcvt s1, h1
+; CHECK-i32-NEXT:    frintx s2, s2
+; CHECK-i32-NEXT:    fcvt s3, h3
+; CHECK-i32-NEXT:    frintx s1, s1
+; CHECK-i32-NEXT:    fcvtzs w8, s2
+; CHECK-i32-NEXT:    fcvt s2, h0
+; CHECK-i32-NEXT:    fcvtzs w9, s1
+; CHECK-i32-NEXT:    frintx s1, s3
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    mov v0.s[1], w9
+; CHECK-i32-NEXT:    fcvtzs w8, s1
+; CHECK-i32-NEXT:    frintx s1, s2
+; CHECK-i32-NEXT:    mov v0.s[2], w8
+; CHECK-i32-NEXT:    fcvtzs w8, s1
+; CHECK-i32-NEXT:    mov v0.s[3], w8
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v4f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-i64-NEXT:    mov h1, v0.h[2]
+; CHECK-i64-NEXT:    mov h2, v0.h[1]
+; CHECK-i64-NEXT:    mov h3, v0.h[3]
+; CHECK-i64-NEXT:    fcvt s0, h0
+; CHECK-i64-NEXT:    fcvt s1, h1
+; CHECK-i64-NEXT:    fcvt s2, h2
+; CHECK-i64-NEXT:    fcvt s3, h3
+; CHECK-i64-NEXT:    frintx s0, s0
+; CHECK-i64-NEXT:    frintx s1, s1
+; CHECK-i64-NEXT:    frintx s2, s2
+; CHECK-i64-NEXT:    frintx s3, s3
+; CHECK-i64-NEXT:    fcvtzs x8, s0
+; CHECK-i64-NEXT:    fcvtzs x9, s1
+; CHECK-i64-NEXT:    fcvtzs x10, s2
+; CHECK-i64-NEXT:    fcvtzs x11, s3
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    fmov d1, x9
+; CHECK-i64-NEXT:    mov v0.d[1], x10
+; CHECK-i64-NEXT:    mov v1.d[1], x11
+; CHECK-i64-NEXT:    ret
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f16(<4 x half> %x)
+  ret <4 x iXLen> %a
 }
-declare <4 x i64> @llvm.lrint.v4i64.v4f16(<4 x half>)
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f16(<4 x half>)
 
-define <8 x i64> @lrint_v8f16(<8 x half> %x) {
-; CHECK-LABEL: lrint_v8f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    mov h4, v0.h[2]
-; CHECK-NEXT:    mov h3, v0.h[1]
-; CHECK-NEXT:    mov h7, v0.h[3]
-; CHECK-NEXT:    fcvt s0, h0
-; CHECK-NEXT:    mov h2, v1.h[2]
-; CHECK-NEXT:    mov h5, v1.h[1]
-; CHECK-NEXT:    mov h6, v1.h[3]
-; CHECK-NEXT:    fcvt s1, h1
-; CHECK-NEXT:    fcvt s4, h4
-; CHECK-NEXT:    fcvt s3, h3
-; CHECK-NEXT:    fcvt s7, h7
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    fcvt s2, h2
-; CHECK-NEXT:    fcvt s5, h5
-; CHECK-NEXT:    fcvt s6, h6
-; CHECK-NEXT:    frintx s1, s1
-; CHECK-NEXT:    frintx s4, s4
-; CHECK-NEXT:    frintx s3, s3
-; CHECK-NEXT:    frintx s7, s7
-; CHECK-NEXT:    fcvtzs x9, s0
-; CHECK-NEXT:    frintx s2, s2
-; CHECK-NEXT:    frintx s5, s5
-; CHECK-NEXT:    frintx s6, s6
-; CHECK-NEXT:    fcvtzs x8, s1
-; CHECK-NEXT:    fcvtzs x12, s4
-; CHECK-NEXT:    fcvtzs x11, s3
-; CHECK-NEXT:    fcvtzs x15, s7
-; CHECK-NEXT:    fmov d0, x9
-; CHECK-NEXT:    fcvtzs x10, s2
-; CHECK-NEXT:    fcvtzs x13, s5
-; CHECK-NEXT:    fcvtzs x14, s6
-; CHECK-NEXT:    fmov d2, x8
-; CHECK-NEXT:    fmov d1, x12
-; CHECK-NEXT:    mov v0.d[1], x11
-; CHECK-NEXT:    fmov d3, x10
-; CHECK-NEXT:    mov v2.d[1], x13
-; CHECK-NEXT:    mov v1.d[1], x15
-; CHECK-NEXT:    mov v3.d[1], x14
-; CHECK-NEXT:    ret
-  %a = call <8 x i64> @llvm.lrint.v8i64.v8f16(<8 x half> %x)
-  ret <8 x i64> %a
+define <8 x iXLen> @lrint_v8f16(<8 x half> %x) {
+; CHECK-i32-LABEL: lrint_v8f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-i32-NEXT:    mov h3, v0.h[1]
+; CHECK-i32-NEXT:    fcvt s6, h0
+; CHECK-i32-NEXT:    mov h4, v0.h[2]
+; CHECK-i32-NEXT:    mov h0, v0.h[3]
+; CHECK-i32-NEXT:    mov h2, v1.h[1]
+; CHECK-i32-NEXT:    fcvt s5, h1
+; CHECK-i32-NEXT:    mov h7, v1.h[2]
+; CHECK-i32-NEXT:    fcvt s3, h3
+; CHECK-i32-NEXT:    frintx s6, s6
+; CHECK-i32-NEXT:    fcvt s4, h4
+; CHECK-i32-NEXT:    mov h1, v1.h[3]
+; CHECK-i32-NEXT:    fcvt s2, h2
+; CHECK-i32-NEXT:    frintx s5, s5
+; CHECK-i32-NEXT:    fcvt s7, h7
+; CHECK-i32-NEXT:    frintx s3, s3
+; CHECK-i32-NEXT:    fcvtzs w9, s6
+; CHECK-i32-NEXT:    frintx s4, s4
+; CHECK-i32-NEXT:    frintx s2, s2
+; CHECK-i32-NEXT:    fcvtzs w8, s5
+; CHECK-i32-NEXT:    fcvt s5, h1
+; CHECK-i32-NEXT:    fcvtzs w11, s3
+; CHECK-i32-NEXT:    fcvt s3, h0
+; CHECK-i32-NEXT:    fmov s0, w9
+; CHECK-i32-NEXT:    fcvtzs w12, s4
+; CHECK-i32-NEXT:    fcvtzs w10, s2
+; CHECK-i32-NEXT:    frintx s2, s7
+; CHECK-i32-NEXT:    fmov s1, w8
+; CHECK-i32-NEXT:    mov v0.s[1], w11
+; CHECK-i32-NEXT:    fcvtzs w8, s2
+; CHECK-i32-NEXT:    mov v1.s[1], w10
+; CHECK-i32-NEXT:    frintx s2, s3
+; CHECK-i32-NEXT:    frintx s3, s5
+; CHECK-i32-NEXT:    mov v0.s[2], w12
+; CHECK-i32-NEXT:    mov v1.s[2], w8
+; CHECK-i32-NEXT:    fcvtzs w9, s2
+; CHECK-i32-NEXT:    fcvtzs w8, s3
+; CHECK-i32-NEXT:    mov v0.s[3], w9
+; CHECK-i32-NEXT:    mov v1.s[3], w8
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v8f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-i64-NEXT:    mov h4, v0.h[2]
+; CHECK-i64-NEXT:    mov h3, v0.h[1]
+; CHECK-i64-NEXT:    mov h7, v0.h[3]
+; CHECK-i64-NEXT:    fcvt s0, h0
+; CHECK-i64-NEXT:    mov h2, v1.h[2]
+; CHECK-i64-NEXT:    mov h5, v1.h[1]
+; CHECK-i64-NEXT:    mov h6, v1.h[3]
+; CHECK-i64-NEXT:    fcvt s1, h1
+; CHECK-i64-NEXT:    fcvt s4, h4
+; CHECK-i64-NEXT:    fcvt s3, h3
+; CHECK-i64-NEXT:    fcvt s7, h7
+; CHECK-i64-NEXT:    frintx s0, s0
+; CHECK-i64-NEXT:    fcvt s2, h2
+; CHECK-i64-NEXT:    fcvt s5, h5
+; CHECK-i64-NEXT:    fcvt s6, h6
+; CHECK-i64-NEXT:    frintx s1, s1
+; CHECK-i64-NEXT:    frintx s4, s4
+; CHECK-i64-NEXT:    frintx s3, s3
+; CHECK-i64-NEXT:    frintx s7, s7
+; CHECK-i64-NEXT:    fcvtzs x9, s0
+; CHECK-i64-NEXT:    frintx s2, s2
+; CHECK-i64-NEXT:    frintx s5, s5
+; CHECK-i64-NEXT:    frintx s6, s6
+; CHECK-i64-NEXT:    fcvtzs x8, s1
+; CHECK-i64-NEXT:    fcvtzs x12, s4
+; CHECK-i64-NEXT:    fcvtzs x11, s3
+; CHECK-i64-NEXT:    fcvtzs x15, s7
+; CHECK-i64-NEXT:    fmov d0, x9
+; CHECK-i64-NEXT:    fcvtzs x10, s2
+; CHECK-i64-NEXT:    fcvtzs x13, s5
+; CHECK-i64-NEXT:    fcvtzs x14, s6
+; CHECK-i64-NEXT:    fmov d2, x8
+; CHECK-i64-NEXT:    fmov d1, x12
+; CHECK-i64-NEXT:    mov v0.d[1], x11
+; CHECK-i64-NEXT:    fmov d3, x10
+; CHECK-i64-NEXT:    mov v2.d[1], x13
+; CHECK-i64-NEXT:    mov v1.d[1], x15
+; CHECK-i64-NEXT:    mov v3.d[1], x14
+; CHECK-i64-NEXT:    ret
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f16(<8 x half> %x)
+  ret <8 x iXLen> %a
 }
-declare <8 x i64> @llvm.lrint.v8i64.v8f16(<8 x half>)
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f16(<8 x half>)
 
-define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
-; CHECK-LABEL: lrint_v16i64_v16f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    mov h17, v0.h[1]
-; CHECK-NEXT:    mov h19, v0.h[2]
-; CHECK-NEXT:    fcvt s18, h0
-; CHECK-NEXT:    mov h0, v0.h[3]
-; CHECK-NEXT:    mov h4, v2.h[1]
-; CHECK-NEXT:    mov h5, v2.h[2]
-; CHECK-NEXT:    fcvt s7, h3
-; CHECK-NEXT:    fcvt s6, h2
-; CHECK-NEXT:    mov h16, v3.h[2]
-; CHECK-NEXT:    mov h2, v2.h[3]
-; CHECK-NEXT:    fcvt s17, h17
-; CHECK-NEXT:    fcvt s19, h19
-; CHECK-NEXT:    frintx s18, s18
-; CHECK-NEXT:    fcvt s0, h0
-; CHECK-NEXT:    fcvt s4, h4
-; CHECK-NEXT:    fcvt s5, h5
-; CHECK-NEXT:    frintx s7, s7
-; CHECK-NEXT:    frintx s6, s6
-; CHECK-NEXT:    fcvt s16, h16
-; CHECK-NEXT:    fcvt s2, h2
-; CHECK-NEXT:    frintx s17, s17
-; CHECK-NEXT:    frintx s19, s19
-; CHECK-NEXT:    fcvtzs x13, s18
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    frintx s4, s4
-; CHECK-NEXT:    frintx s5, s5
-; CHECK-NEXT:    fcvtzs x9, s7
-; CHECK-NEXT:    mov h7, v1.h[2]
-; CHECK-NEXT:    fcvtzs x8, s6
-; CHECK-NEXT:    mov h6, v1.h[1]
-; CHECK-NEXT:    frintx s16, s16
-; CHECK-NEXT:    fcvtzs x14, s17
-; CHECK-NEXT:    fcvtzs x15, s19
-; CHECK-NEXT:    fcvtzs x10, s4
-; CHECK-NEXT:    mov h4, v3.h[1]
-; CHECK-NEXT:    fcvtzs x11, s5
-; CHECK-NEXT:    mov h5, v1.h[3]
-; CHECK-NEXT:    mov h3, v3.h[3]
-; CHECK-NEXT:    fcvt s1, h1
-; CHECK-NEXT:    fcvt s7, h7
-; CHECK-NEXT:    fcvt s6, h6
-; CHECK-NEXT:    fcvtzs x12, s16
-; CHECK-NEXT:    frintx s16, s2
-; CHECK-NEXT:    fmov d2, x8
-; CHECK-NEXT:    fcvt s4, h4
-; CHECK-NEXT:    fcvt s3, h3
-; CHECK-NEXT:    fcvt s5, h5
-; CHECK-NEXT:    frintx s1, s1
-; CHECK-NEXT:    frintx s7, s7
-; CHECK-NEXT:    frintx s17, s6
-; CHECK-NEXT:    fmov d6, x9
-; CHECK-NEXT:    mov v2.d[1], x10
-; CHECK-NEXT:    frintx s4, s4
-; CHECK-NEXT:    frintx s18, s3
-; CHECK-NEXT:    frintx s5, s5
-; CHECK-NEXT:    fcvtzs x8, s1
-; CHECK-NEXT:    fcvtzs x9, s7
-; CHECK-NEXT:    fmov d3, x11
-; CHECK-NEXT:    fcvtzs x11, s0
-; CHECK-NEXT:    fmov d7, x12
-; CHECK-NEXT:    fcvtzs x12, s16
-; CHECK-NEXT:    fcvtzs x16, s17
-; CHECK-NEXT:    fcvtzs x17, s4
-; CHECK-NEXT:    fmov d0, x13
-; CHECK-NEXT:    fmov d1, x15
-; CHECK-NEXT:    fcvtzs x18, s18
-; CHECK-NEXT:    fcvtzs x0, s5
-; CHECK-NEXT:    fmov d4, x8
-; CHECK-NEXT:    fmov d5, x9
-; CHECK-NEXT:    mov v0.d[1], x14
-; CHECK-NEXT:    mov v1.d[1], x11
-; CHECK-NEXT:    mov v3.d[1], x12
-; CHECK-NEXT:    mov v4.d[1], x16
-; CHECK-NEXT:    mov v6.d[1], x17
-; CHECK-NEXT:    mov v7.d[1], x18
-; CHECK-NEXT:    mov v5.d[1], x0
-; CHECK-NEXT:    ret
-  %a = call <16 x i64> @llvm.lrint.v16i64.v16f16(<16 x half> %x)
-  ret <16 x i64> %a
+define <16 x iXLen> @lrint_v16f16(<16 x half> %x) {
+; CHECK-i32-LABEL: lrint_v16f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-i32-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-i32-NEXT:    mov h18, v0.h[1]
+; CHECK-i32-NEXT:    mov h19, v1.h[1]
+; CHECK-i32-NEXT:    fcvt s20, h0
+; CHECK-i32-NEXT:    mov h21, v0.h[2]
+; CHECK-i32-NEXT:    mov h0, v0.h[3]
+; CHECK-i32-NEXT:    mov h4, v2.h[1]
+; CHECK-i32-NEXT:    mov h5, v2.h[2]
+; CHECK-i32-NEXT:    fcvt s6, h2
+; CHECK-i32-NEXT:    fcvt s7, h3
+; CHECK-i32-NEXT:    mov h16, v3.h[1]
+; CHECK-i32-NEXT:    mov h17, v3.h[2]
+; CHECK-i32-NEXT:    fcvt s18, h18
+; CHECK-i32-NEXT:    fcvt s19, h19
+; CHECK-i32-NEXT:    mov h2, v2.h[3]
+; CHECK-i32-NEXT:    fcvt s4, h4
+; CHECK-i32-NEXT:    fcvt s5, h5
+; CHECK-i32-NEXT:    frintx s6, s6
+; CHECK-i32-NEXT:    frintx s7, s7
+; CHECK-i32-NEXT:    fcvt s16, h16
+; CHECK-i32-NEXT:    fcvt s17, h17
+; CHECK-i32-NEXT:    frintx s18, s18
+; CHECK-i32-NEXT:    fcvt s2, h2
+; CHECK-i32-NEXT:    frintx s4, s4
+; CHECK-i32-NEXT:    frintx s5, s5
+; CHECK-i32-NEXT:    fcvtzs w8, s6
+; CHECK-i32-NEXT:    fcvt s6, h1
+; CHECK-i32-NEXT:    fcvtzs w9, s7
+; CHECK-i32-NEXT:    mov h7, v1.h[2]
+; CHECK-i32-NEXT:    frintx s16, s16
+; CHECK-i32-NEXT:    fcvtzs w15, s18
+; CHECK-i32-NEXT:    fcvtzs w10, s4
+; CHECK-i32-NEXT:    frintx s4, s17
+; CHECK-i32-NEXT:    fcvtzs w11, s5
+; CHECK-i32-NEXT:    frintx s5, s20
+; CHECK-i32-NEXT:    fcvt s17, h21
+; CHECK-i32-NEXT:    frintx s6, s6
+; CHECK-i32-NEXT:    fcvtzs w12, s16
+; CHECK-i32-NEXT:    frintx s16, s19
+; CHECK-i32-NEXT:    fcvt s7, h7
+; CHECK-i32-NEXT:    mov h19, v1.h[3]
+; CHECK-i32-NEXT:    fmov s1, w8
+; CHECK-i32-NEXT:    fcvtzs w13, s4
+; CHECK-i32-NEXT:    mov h4, v3.h[3]
+; CHECK-i32-NEXT:    fmov s3, w9
+; CHECK-i32-NEXT:    fcvtzs w14, s5
+; CHECK-i32-NEXT:    frintx s5, s17
+; CHECK-i32-NEXT:    fcvtzs w16, s6
+; CHECK-i32-NEXT:    fcvt s17, h0
+; CHECK-i32-NEXT:    fcvtzs w8, s16
+; CHECK-i32-NEXT:    frintx s6, s7
+; CHECK-i32-NEXT:    fcvt s7, h19
+; CHECK-i32-NEXT:    mov v1.s[1], w10
+; CHECK-i32-NEXT:    mov v3.s[1], w12
+; CHECK-i32-NEXT:    fcvt s4, h4
+; CHECK-i32-NEXT:    fcvtzs w9, s5
+; CHECK-i32-NEXT:    fmov s0, w14
+; CHECK-i32-NEXT:    frintx s5, s2
+; CHECK-i32-NEXT:    fmov s2, w16
+; CHECK-i32-NEXT:    frintx s16, s17
+; CHECK-i32-NEXT:    fcvtzs w10, s6
+; CHECK-i32-NEXT:    frintx s6, s7
+; CHECK-i32-NEXT:    mov v1.s[2], w11
+; CHECK-i32-NEXT:    mov v3.s[2], w13
+; CHECK-i32-NEXT:    mov v0.s[1], w15
+; CHECK-i32-NEXT:    frintx s4, s4
+; CHECK-i32-NEXT:    mov v2.s[1], w8
+; CHECK-i32-NEXT:    fcvtzs w8, s5
+; CHECK-i32-NEXT:    fcvtzs w12, s16
+; CHECK-i32-NEXT:    mov v0.s[2], w9
+; CHECK-i32-NEXT:    fcvtzs w9, s4
+; CHECK-i32-NEXT:    mov v2.s[2], w10
+; CHECK-i32-NEXT:    fcvtzs w10, s6
+; CHECK-i32-NEXT:    mov v1.s[3], w8
+; CHECK-i32-NEXT:    mov v0.s[3], w12
+; CHECK-i32-NEXT:    mov v3.s[3], w9
+; CHECK-i32-NEXT:    mov v2.s[3], w10
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v16f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-i64-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-i64-NEXT:    mov h17, v0.h[1]
+; CHECK-i64-NEXT:    mov h19, v0.h[2]
+; CHECK-i64-NEXT:    fcvt s18, h0
+; CHECK-i64-NEXT:    mov h0, v0.h[3]
+; CHECK-i64-NEXT:    mov h4, v2.h[1]
+; CHECK-i64-NEXT:    mov h5, v2.h[2]
+; CHECK-i64-NEXT:    fcvt s7, h3
+; CHECK-i64-NEXT:    fcvt s6, h2
+; CHECK-i64-NEXT:    mov h16, v3.h[2]
+; CHECK-i64-NEXT:    mov h2, v2.h[3]
+; CHECK-i64-NEXT:    fcvt s17, h17
+; CHECK-i64-NEXT:    fcvt s19, h19
+; CHECK-i64-NEXT:    frintx s18, s18
+; CHECK-i64-NEXT:    fcvt s0, h0
+; CHECK-i64-NEXT:    fcvt s4, h4
+; CHECK-i64-NEXT:    fcvt s5, h5
+; CHECK-i64-NEXT:    frintx s7, s7
+; CHECK-i64-NEXT:    frintx s6, s6
+; CHECK-i64-NEXT:    fcvt s16, h16
+; CHECK-i64-NEXT:    fcvt s2, h2
+; CHECK-i64-NEXT:    frintx s17, s17
+; CHECK-i64-NEXT:    frintx s19, s19
+; CHECK-i64-NEXT:    fcvtzs x13, s18
+; CHECK-i64-NEXT:    frintx s0, s0
+; CHECK-i64-NEXT:    frintx s4, s4
+; CHECK-i64-NEXT:    frintx s5, s5
+; CHECK-i64-NEXT:    fcvtzs x9, s7
+; CHECK-i64-NEXT:    mov h7, v1.h[2]
+; CHECK-i64-NEXT:    fcvtzs x8, s6
+; CHECK-i64-NEXT:    mov h6, v1.h[1]
+; CHECK-i64-NEXT:    frintx s16, s16
+; CHECK-i64-NEXT:    fcvtzs x14, s17
+; CHECK-i64-NEXT:    fcvtzs x15, s19
+; CHECK-i64-NEXT:    fcvtzs x10, s4
+; CHECK-i64-NEXT:    mov h4, v3.h[1]
+; CHECK-i64-NEXT:    fcvtzs x11, s5
+; CHECK-i64-NEXT:    mov h5, v1.h[3]
+; CHECK-i64-NEXT:    mov h3, v3.h[3]
+; CHECK-i64-NEXT:    fcvt s1, h1
+; CHECK-i64-NEXT:    fcvt s7, h7
+; CHECK-i64-NEXT:    fcvt s6, h6
+; CHECK-i64-NEXT:    fcvtzs x12, s16
+; CHECK-i64-NEXT:    frintx s16, s2
+; CHECK-i64-NEXT:    fmov d2, x8
+; CHECK-i64-NEXT:    fcvt s4, h4
+; CHECK-i64-NEXT:    fcvt s3, h3
+; CHECK-i64-NEXT:    fcvt s5, h5
+; CHECK-i64-NEXT:    frintx s1, s1
+; CHECK-i64-NEXT:    frintx s7, s7
+; CHECK-i64-NEXT:    frintx s17, s6
+; CHECK-i64-NEXT:    fmov d6, x9
+; CHECK-i64-NEXT:    mov v2.d[1], x10
+; CHECK-i64-NEXT:    frintx s4, s4
+; CHECK-i64-NEXT:    frintx s18, s3
+; CHECK-i64-NEXT:    frintx s5, s5
+; CHECK-i64-NEXT:    fcvtzs x8, s1
+; CHECK-i64-NEXT:    fcvtzs x9, s7
+; CHECK-i64-NEXT:    fmov d3, x11
+; CHECK-i64-NEXT:    fcvtzs x11, s0
+; CHECK-i64-NEXT:    fmov d7, x12
+; CHECK-i64-NEXT:    fcvtzs x12, s16
+; CHECK-i64-NEXT:    fcvtzs x16, s17
+; CHECK-i64-NEXT:    fcvtzs x17, s4
+; CHECK-i64-NEXT:    fmov d0, x13
+; CHECK-i64-NEXT:    fmov d1, x15
+; CHECK-i64-NEXT:    fcvtzs x18, s18
+; CHECK-i64-NEXT:    fcvtzs x0, s5
+; CHECK-i64-NEXT:    fmov d4, x8
+; CHECK-i64-NEXT:    fmov d5, x9
+; CHECK-i64-NEXT:    mov v0.d[1], x14
+; CHECK-i64-NEXT:    mov v1.d[1], x11
+; CHECK-i64-NEXT:    mov v3.d[1], x12
+; CHECK-i64-NEXT:    mov v4.d[1], x16
+; CHECK-i64-NEXT:    mov v6.d[1], x17
+; CHECK-i64-NEXT:    mov v7.d[1], x18
+; CHECK-i64-NEXT:    mov v5.d[1], x0
+; CHECK-i64-NEXT:    ret
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f16(<16 x half> %x)
+  ret <16 x iXLen> %a
 }
-declare <16 x i64> @llvm.lrint.v16i64.v16f16(<16 x half>)
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f16(<16 x half>)
 
-define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
-; CHECK-LABEL: lrint_v32i64_v32f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
-; CHECK-NEXT:    ext v7.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    mov h19, v0.h[1]
-; CHECK-NEXT:    fcvt s21, h0
-; CHECK-NEXT:    mov h23, v1.h[2]
-; CHECK-NEXT:    fcvt s22, h1
-; CHECK-NEXT:    fcvt s26, h2
-; CHECK-NEXT:    mov h27, v2.h[1]
-; CHECK-NEXT:    mov h28, v2.h[2]
-; CHECK-NEXT:    mov h16, v4.h[2]
-; CHECK-NEXT:    fcvt s17, h5
-; CHECK-NEXT:    mov h18, v5.h[2]
-; CHECK-NEXT:    mov h20, v6.h[2]
-; CHECK-NEXT:    fcvt s24, h7
-; CHECK-NEXT:    fcvt s25, h6
-; CHECK-NEXT:    fcvt s19, h19
-; CHECK-NEXT:    frintx s22, s22
-; CHECK-NEXT:    fcvt s16, h16
-; CHECK-NEXT:    frintx s17, s17
-; CHECK-NEXT:    fcvt s18, h18
-; CHECK-NEXT:    fcvt s20, h20
-; CHECK-NEXT:    frintx s16, s16
-; CHECK-NEXT:    fcvtzs x12, s17
-; CHECK-NEXT:    frintx s17, s18
-; CHECK-NEXT:    frintx s18, s21
-; CHECK-NEXT:    fcvt s21, h23
-; CHECK-NEXT:    frintx s23, s24
-; CHECK-NEXT:    frintx s24, s25
-; CHECK-NEXT:    frintx s25, s19
-; CHECK-NEXT:    mov h19, v7.h[1]
-; CHECK-NEXT:    fcvtzs x13, s16
-; CHECK-NEXT:    frintx s16, s20
-; CHECK-NEXT:    frintx s20, s26
-; CHECK-NEXT:    fcvtzs x9, s23
-; CHECK-NEXT:    mov h23, v3.h[2]
-; CHECK-NEXT:    fcvt s26, h27
-; CHECK-NEXT:    fcvtzs x15, s24
-; CHECK-NEXT:    fcvtzs x10, s25
-; CHECK-NEXT:    fcvt s24, h28
-; CHECK-NEXT:    mov h25, v3.h[3]
-; CHECK-NEXT:    fcvtzs x14, s17
-; CHECK-NEXT:    frintx s21, s21
-; CHECK-NEXT:    fmov d17, x12
-; CHECK-NEXT:    fcvtzs x12, s16
-; CHECK-NEXT:    fmov d16, x13
-; CHECK-NEXT:    fcvtzs x13, s22
-; CHECK-NEXT:    fcvt s22, h3
-; CHECK-NEXT:    mov h3, v3.h[1]
-; CHECK-NEXT:    mov h27, v0.h[2]
-; CHECK-NEXT:    mov h28, v2.h[3]
-; CHECK-NEXT:    fcvt s23, h23
-; CHECK-NEXT:    frintx s26, s26
-; CHECK-NEXT:    fcvtzs x16, s20
-; CHECK-NEXT:    frintx s20, s24
-; CHECK-NEXT:    fcvt s24, h25
-; CHECK-NEXT:    fcvtzs x11, s18
-; CHECK-NEXT:    fmov d18, x14
-; CHECK-NEXT:    fcvtzs x14, s21
-; CHECK-NEXT:    frintx s22, s22
-; CHECK-NEXT:    fcvt s3, h3
-; CHECK-NEXT:    fcvt s25, h27
-; CHECK-NEXT:    fcvt s27, h28
-; CHECK-NEXT:    frintx s23, s23
-; CHECK-NEXT:    mov h21, v1.h[3]
-; CHECK-NEXT:    fmov d2, x15
-; CHECK-NEXT:    fcvtzs x15, s26
-; CHECK-NEXT:    fmov d26, x13
-; CHECK-NEXT:    mov h1, v1.h[1]
-; CHECK-NEXT:    fcvtzs x13, s20
-; CHECK-NEXT:    frintx s20, s24
-; CHECK-NEXT:    fmov d24, x14
-; CHECK-NEXT:    fcvtzs x14, s22
-; CHECK-NEXT:    frintx s3, s3
-; CHECK-NEXT:    fmov d22, x16
-; CHECK-NEXT:    frintx s27, s27
-; CHECK-NEXT:    fcvtzs x16, s23
-; CHECK-NEXT:    fcvt s21, h21
-; CHECK-NEXT:    frintx s25, s25
-; CHECK-NEXT:    fcvt s1, h1
-; CHECK-NEXT:    mov h0, v0.h[3]
-; CHECK-NEXT:    mov h23, v7.h[2]
-; CHECK-NEXT:    mov v22.d[1], x15
-; CHECK-NEXT:    fcvtzs x15, s20
-; CHECK-NEXT:    fmov d20, x13
-; CHECK-NEXT:    fcvtzs x13, s3
-; CHECK-NEXT:    fmov d3, x14
-; CHECK-NEXT:    fcvtzs x14, s27
-; CHECK-NEXT:    fmov d27, x16
-; CHECK-NEXT:    frintx s21, s21
-; CHECK-NEXT:    mov h7, v7.h[3]
-; CHECK-NEXT:    frintx s1, s1
-; CHECK-NEXT:    fcvt s0, h0
-; CHECK-NEXT:    fcvt s23, h23
-; CHECK-NEXT:    fcvt s19, h19
-; CHECK-NEXT:    mov v27.d[1], x15
-; CHECK-NEXT:    fcvtzs x15, s25
-; CHECK-NEXT:    mov h25, v6.h[3]
-; CHECK-NEXT:    mov h6, v6.h[1]
-; CHECK-NEXT:    mov v3.d[1], x13
-; CHECK-NEXT:    fcvtzs x13, s21
-; CHECK-NEXT:    mov h21, v5.h[1]
-; CHECK-NEXT:    mov h5, v5.h[3]
-; CHECK-NEXT:    mov v20.d[1], x14
-; CHECK-NEXT:    fcvtzs x14, s1
-; CHECK-NEXT:    mov h1, v4.h[1]
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    fcvt s25, h25
-; CHECK-NEXT:    fcvt s7, h7
-; CHECK-NEXT:    stp q3, q27, [x8, #192]
-; CHECK-NEXT:    fcvt s6, h6
-; CHECK-NEXT:    mov h3, v4.h[3]
-; CHECK-NEXT:    stp q22, q20, [x8, #128]
-; CHECK-NEXT:    fcvt s21, h21
-; CHECK-NEXT:    fcvt s5, h5
-; CHECK-NEXT:    mov v24.d[1], x13
-; CHECK-NEXT:    mov v26.d[1], x14
-; CHECK-NEXT:    fcvt s4, h4
-; CHECK-NEXT:    frintx s22, s25
-; CHECK-NEXT:    fmov d20, x12
-; CHECK-NEXT:    fcvt s1, h1
-; CHECK-NEXT:    frintx s6, s6
-; CHECK-NEXT:    fcvt s3, h3
-; CHECK-NEXT:    fcvtzs x12, s0
-; CHECK-NEXT:    frintx s5, s5
-; CHECK-NEXT:    frintx s21, s21
-; CHECK-NEXT:    fmov d0, x11
-; CHECK-NEXT:    stp q26, q24, [x8, #64]
-; CHECK-NEXT:    fmov d24, x15
-; CHECK-NEXT:    frintx s4, s4
-; CHECK-NEXT:    fcvtzs x11, s22
-; CHECK-NEXT:    frintx s22, s23
-; CHECK-NEXT:    frintx s1, s1
-; CHECK-NEXT:    fcvtzs x13, s6
-; CHECK-NEXT:    frintx s3, s3
-; CHECK-NEXT:    frintx s6, s7
-; CHECK-NEXT:    fcvtzs x14, s5
-; CHECK-NEXT:    mov v24.d[1], x12
-; CHECK-NEXT:    frintx s5, s19
-; CHECK-NEXT:    fcvtzs x12, s21
-; CHECK-NEXT:    mov v0.d[1], x10
-; CHECK-NEXT:    fcvtzs x10, s4
-; CHECK-NEXT:    mov v20.d[1], x11
-; CHECK-NEXT:    fcvtzs x11, s22
-; CHECK-NEXT:    mov v2.d[1], x13
-; CHECK-NEXT:    fcvtzs x15, s3
-; CHECK-NEXT:    fcvtzs x13, s1
-; CHECK-NEXT:    mov v18.d[1], x14
-; CHECK-NEXT:    fcvtzs x14, s6
-; CHECK-NEXT:    stp q0, q24, [x8]
-; CHECK-NEXT:    mov v17.d[1], x12
-; CHECK-NEXT:    fcvtzs x12, s5
-; CHECK-NEXT:    fmov d0, x10
-; CHECK-NEXT:    fmov d1, x11
-; CHECK-NEXT:    stp q2, q20, [x8, #224]
-; CHECK-NEXT:    fmov d2, x9
-; CHECK-NEXT:    mov v16.d[1], x15
-; CHECK-NEXT:    stp q17, q18, [x8, #160]
-; CHECK-NEXT:    mov v0.d[1], x13
-; CHECK-NEXT:    mov v1.d[1], x14
-; CHECK-NEXT:    mov v2.d[1], x12
-; CHECK-NEXT:    stp q0, q16, [x8, #96]
-; CHECK-NEXT:    stp q2, q1, [x8, #32]
-; CHECK-NEXT:    ret
-  %a = call <32 x i64> @llvm.lrint.v32i64.v32f16(<32 x half> %x)
-  ret <32 x i64> %a
+define <32 x iXLen> @lrint_v32f16(<32 x half> %x) {
+; CHECK-i32-LABEL: lrint_v32f16:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    ext v5.16b, v0.16b, v0.16b, #8
+; CHECK-i32-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-i32-NEXT:    ext v17.16b, v2.16b, v2.16b, #8
+; CHECK-i32-NEXT:    mov h6, v5.h[1]
+; CHECK-i32-NEXT:    fcvt s7, h5
+; CHECK-i32-NEXT:    mov h16, v5.h[2]
+; CHECK-i32-NEXT:    mov h5, v5.h[3]
+; CHECK-i32-NEXT:    mov h18, v4.h[1]
+; CHECK-i32-NEXT:    mov h20, v4.h[3]
+; CHECK-i32-NEXT:    mov h19, v4.h[2]
+; CHECK-i32-NEXT:    fcvt s21, h4
+; CHECK-i32-NEXT:    mov h23, v17.h[1]
+; CHECK-i32-NEXT:    ext v4.16b, v3.16b, v3.16b, #8
+; CHECK-i32-NEXT:    fcvt s22, h17
+; CHECK-i32-NEXT:    fcvt s6, h6
+; CHECK-i32-NEXT:    frintx s7, s7
+; CHECK-i32-NEXT:    fcvt s16, h16
+; CHECK-i32-NEXT:    fcvt s5, h5
+; CHECK-i32-NEXT:    fcvt s18, h18
+; CHECK-i32-NEXT:    fcvt s20, h20
+; CHECK-i32-NEXT:    fcvt s19, h19
+; CHECK-i32-NEXT:    frintx s22, s22
+; CHECK-i32-NEXT:    frintx s6, s6
+; CHECK-i32-NEXT:    fcvtzs w12, s7
+; CHECK-i32-NEXT:    frintx s7, s16
+; CHECK-i32-NEXT:    frintx s5, s5
+; CHECK-i32-NEXT:    frintx s16, s21
+; CHECK-i32-NEXT:    fcvt s21, h23
+; CHECK-i32-NEXT:    frintx s18, s18
+; CHECK-i32-NEXT:    frintx s20, s20
+; CHECK-i32-NEXT:    frintx s19, s19
+; CHECK-i32-NEXT:    fcvtzs w15, s22
+; CHECK-i32-NEXT:    mov h22, v1.h[2]
+; CHECK-i32-NEXT:    fcvtzs w17, s6
+; CHECK-i32-NEXT:    mov h6, v17.h[2]
+; CHECK-i32-NEXT:    mov h17, v17.h[3]
+; CHECK-i32-NEXT:    fcvtzs w9, s7
+; CHECK-i32-NEXT:    mov h7, v4.h[2]
+; CHECK-i32-NEXT:    fcvtzs w8, s5
+; CHECK-i32-NEXT:    mov h5, v4.h[1]
+; CHECK-i32-NEXT:    fcvtzs w13, s16
+; CHECK-i32-NEXT:    frintx s16, s21
+; CHECK-i32-NEXT:    fcvtzs w14, s18
+; CHECK-i32-NEXT:    fcvtzs w10, s20
+; CHECK-i32-NEXT:    fcvt s18, h4
+; CHECK-i32-NEXT:    fcvt s6, h6
+; CHECK-i32-NEXT:    fcvt s17, h17
+; CHECK-i32-NEXT:    mov h20, v0.h[2]
+; CHECK-i32-NEXT:    fcvt s7, h7
+; CHECK-i32-NEXT:    fcvtzs w11, s19
+; CHECK-i32-NEXT:    mov h19, v0.h[1]
+; CHECK-i32-NEXT:    fcvt s5, h5
+; CHECK-i32-NEXT:    fcvtzs w0, s16
+; CHECK-i32-NEXT:    mov h21, v1.h[1]
+; CHECK-i32-NEXT:    frintx s18, s18
+; CHECK-i32-NEXT:    mov h4, v4.h[3]
+; CHECK-i32-NEXT:    frintx s6, s6
+; CHECK-i32-NEXT:    frintx s16, s17
+; CHECK-i32-NEXT:    mov h17, v0.h[3]
+; CHECK-i32-NEXT:    fcvt s0, h0
+; CHECK-i32-NEXT:    fcvt s19, h19
+; CHECK-i32-NEXT:    frintx s5, s5
+; CHECK-i32-NEXT:    fcvtzs w2, s18
+; CHECK-i32-NEXT:    fcvt s18, h21
+; CHECK-i32-NEXT:    fcvt s21, h2
+; CHECK-i32-NEXT:    fcvtzs w18, s6
+; CHECK-i32-NEXT:    frintx s6, s7
+; CHECK-i32-NEXT:    fcvt s7, h20
+; CHECK-i32-NEXT:    fcvtzs w16, s16
+; CHECK-i32-NEXT:    fcvt s16, h17
+; CHECK-i32-NEXT:    fcvt s17, h1
+; CHECK-i32-NEXT:    frintx s0, s0
+; CHECK-i32-NEXT:    fcvtzs w3, s5
+; CHECK-i32-NEXT:    frintx s5, s19
+; CHECK-i32-NEXT:    fcvt s19, h22
+; CHECK-i32-NEXT:    mov h1, v1.h[3]
+; CHECK-i32-NEXT:    fcvtzs w1, s6
+; CHECK-i32-NEXT:    frintx s6, s7
+; CHECK-i32-NEXT:    mov h7, v2.h[1]
+; CHECK-i32-NEXT:    frintx s17, s17
+; CHECK-i32-NEXT:    frintx s20, s16
+; CHECK-i32-NEXT:    fmov s16, w12
+; CHECK-i32-NEXT:    fcvtzs w4, s0
+; CHECK-i32-NEXT:    frintx s0, s18
+; CHECK-i32-NEXT:    fcvtzs w5, s5
+; CHECK-i32-NEXT:    frintx s5, s19
+; CHECK-i32-NEXT:    frintx s18, s21
+; CHECK-i32-NEXT:    fcvt s19, h3
+; CHECK-i32-NEXT:    fcvtzs w12, s6
+; CHECK-i32-NEXT:    fcvt s6, h7
+; CHECK-i32-NEXT:    mov h7, v3.h[1]
+; CHECK-i32-NEXT:    fcvtzs w6, s17
+; CHECK-i32-NEXT:    fmov s17, w13
+; CHECK-i32-NEXT:    mov v16.s[1], w17
+; CHECK-i32-NEXT:    fcvtzs w17, s20
+; CHECK-i32-NEXT:    fcvtzs w7, s0
+; CHECK-i32-NEXT:    mov h0, v2.h[2]
+; CHECK-i32-NEXT:    mov h20, v3.h[2]
+; CHECK-i32-NEXT:    fcvtzs w13, s5
+; CHECK-i32-NEXT:    fmov s5, w15
+; CHECK-i32-NEXT:    frintx s6, s6
+; CHECK-i32-NEXT:    fcvt s7, h7
+; CHECK-i32-NEXT:    mov v17.s[1], w14
+; CHECK-i32-NEXT:    fcvtzs w14, s18
+; CHECK-i32-NEXT:    frintx s18, s19
+; CHECK-i32-NEXT:    mov h2, v2.h[3]
+; CHECK-i32-NEXT:    fcvt s0, h0
+; CHECK-i32-NEXT:    mov h3, v3.h[3]
+; CHECK-i32-NEXT:    mov v5.s[1], w0
+; CHECK-i32-NEXT:    fcvt s19, h20
+; CHECK-i32-NEXT:    fcvt s1, h1
+; CHECK-i32-NEXT:    mov v16.s[2], w9
+; CHECK-i32-NEXT:    fcvtzs w15, s6
+; CHECK-i32-NEXT:    frintx s6, s7
+; CHECK-i32-NEXT:    fmov s7, w2
+; CHECK-i32-NEXT:    fcvtzs w0, s18
+; CHECK-i32-NEXT:    fcvt s20, h2
+; CHECK-i32-NEXT:    fcvt s18, h4
+; CHECK-i32-NEXT:    frintx s21, s0
+; CHECK-i32-NEXT:    fcvt s3, h3
+; CHECK-i32-NEXT:    fmov s0, w4
+; CHECK-i32-NEXT:    frintx s19, s19
+; CHECK-i32-NEXT:    fmov s2, w6
+; CHECK-i32-NEXT:    fmov s4, w14
+; CHECK-i32-NEXT:    fcvtzs w2, s6
+; CHECK-i32-NEXT:    mov v7.s[1], w3
+; CHECK-i32-NEXT:    frintx s1, s1
+; CHECK-i32-NEXT:    fmov s6, w0
+; CHECK-i32-NEXT:    mov v0.s[1], w5
+; CHECK-i32-NEXT:    frintx s20, s20
+; CHECK-i32-NEXT:    mov v2.s[1], w7
+; CHECK-i32-NEXT:    fcvtzs w3, s21
+; CHECK-i32-NEXT:    mov v4.s[1], w15
+; CHECK-i32-NEXT:    fcvtzs w14, s19
+; CHECK-i32-NEXT:    frintx s18, s18
+; CHECK-i32-NEXT:    frintx s3, s3
+; CHECK-i32-NEXT:    mov v6.s[1], w2
+; CHECK-i32-NEXT:    mov v17.s[2], w11
+; CHECK-i32-NEXT:    fcvtzs w15, s1
+; CHECK-i32-NEXT:    fcvtzs w0, s20
+; CHECK-i32-NEXT:    mov v5.s[2], w18
+; CHECK-i32-NEXT:    mov v0.s[2], w12
+; CHECK-i32-NEXT:    mov v7.s[2], w1
+; CHECK-i32-NEXT:    mov v2.s[2], w13
+; CHECK-i32-NEXT:    mov v4.s[2], w3
+; CHECK-i32-NEXT:    fcvtzs w9, s18
+; CHECK-i32-NEXT:    fcvtzs w11, s3
+; CHECK-i32-NEXT:    mov v16.s[3], w8
+; CHECK-i32-NEXT:    mov v6.s[2], w14
+; CHECK-i32-NEXT:    mov v17.s[3], w10
+; CHECK-i32-NEXT:    mov v0.s[3], w17
+; CHECK-i32-NEXT:    mov v5.s[3], w16
+; CHECK-i32-NEXT:    mov v2.s[3], w15
+; CHECK-i32-NEXT:    mov v4.s[3], w0
+; CHECK-i32-NEXT:    mov v7.s[3], w9
+; CHECK-i32-NEXT:    mov v1.16b, v16.16b
+; CHECK-i32-NEXT:    mov v6.s[3], w11
+; CHECK-i32-NEXT:    mov v3.16b, v17.16b
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v32f16:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-i64-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-i64-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-i64-NEXT:    ext v7.16b, v0.16b, v0.16b, #8
+; CHECK-i64-NEXT:    mov h19, v0.h[1]
+; CHECK-i64-NEXT:    fcvt s21, h0
+; CHECK-i64-NEXT:    mov h23, v1.h[2]
+; CHECK-i64-NEXT:    fcvt s22, h1
+; CHECK-i64-NEXT:    fcvt s26, h2
+; CHECK-i64-NEXT:    mov h27, v2.h[1]
+; CHECK-i64-NEXT:    mov h28, v2.h[2]
+; CHECK-i64-NEXT:    mov h16, v4.h[2]
+; CHECK-i64-NEXT:    fcvt s17, h5
+; CHECK-i64-NEXT:    mov h18, v5.h[2]
+; CHECK-i64-NEXT:    mov h20, v6.h[2]
+; CHECK-i64-NEXT:    fcvt s24, h7
+; CHECK-i64-NEXT:    fcvt s25, h6
+; CHECK-i64-NEXT:    fcvt s19, h19
+; CHECK-i64-NEXT:    frintx s22, s22
+; CHECK-i64-NEXT:    fcvt s16, h16
+; CHECK-i64-NEXT:    frintx s17, s17
+; CHECK-i64-NEXT:    fcvt s18, h18
+; CHECK-i64-NEXT:    fcvt s20, h20
+; CHECK-i64-NEXT:    frintx s16, s16
+; CHECK-i64-NEXT:    fcvtzs x12, s17
+; CHECK-i64-NEXT:    frintx s17, s18
+; CHECK-i64-NEXT:    frintx s18, s21
+; CHECK-i64-NEXT:    fcvt s21, h23
+; CHECK-i64-NEXT:    frintx s23, s24
+; CHECK-i64-NEXT:    frintx s24, s25
+; CHECK-i64-NEXT:    frintx s25, s19
+; CHECK-i64-NEXT:    mov h19, v7.h[1]
+; CHECK-i64-NEXT:    fcvtzs x13, s16
+; CHECK-i64-NEXT:    frintx s16, s20
+; CHECK-i64-NEXT:    frintx s20, s26
+; CHECK-i64-NEXT:    fcvtzs x9, s23
+; CHECK-i64-NEXT:    mov h23, v3.h[2]
+; CHECK-i64-NEXT:    fcvt s26, h27
+; CHECK-i64-NEXT:    fcvtzs x15, s24
+; CHECK-i64-NEXT:    fcvtzs x10, s25
+; CHECK-i64-NEXT:    fcvt s24, h28
+; CHECK-i64-NEXT:    mov h25, v3.h[3]
+; CHECK-i64-NEXT:    fcvtzs x14, s17
+; CHECK-i64-NEXT:    frintx s21, s21
+; CHECK-i64-NEXT:    fmov d17, x12
+; CHECK-i64-NEXT:    fcvtzs x12, s16
+; CHECK-i64-NEXT:    fmov d16, x13
+; CHECK-i64-NEXT:    fcvtzs x13, s22
+; CHECK-i64-NEXT:    fcvt s22, h3
+; CHECK-i64-NEXT:    mov h3, v3.h[1]
+; CHECK-i64-NEXT:    mov h27, v0.h[2]
+; CHECK-i64-NEXT:    mov h28, v2.h[3]
+; CHECK-i64-NEXT:    fcvt s23, h23
+; CHECK-i64-NEXT:    frintx s26, s26
+; CHECK-i64-NEXT:    fcvtzs x16, s20
+; CHECK-i64-NEXT:    frintx s20, s24
+; CHECK-i64-NEXT:    fcvt s24, h25
+; CHECK-i64-NEXT:    fcvtzs x11, s18
+; CHECK-i64-NEXT:    fmov d18, x14
+; CHECK-i64-NEXT:    fcvtzs x14, s21
+; CHECK-i64-NEXT:    frintx s22, s22
+; CHECK-i64-NEXT:    fcvt s3, h3
+; CHECK-i64-NEXT:    fcvt s25, h27
+; CHECK-i64-NEXT:    fcvt s27, h28
+; CHECK-i64-NEXT:    frintx s23, s23
+; CHECK-i64-NEXT:    mov h21, v1.h[3]
+; CHECK-i64-NEXT:    fmov d2, x15
+; CHECK-i64-NEXT:    fcvtzs x15, s26
+; CHECK-i64-NEXT:    fmov d26, x13
+; CHECK-i64-NEXT:    mov h1, v1.h[1]
+; CHECK-i64-NEXT:    fcvtzs x13, s20
+; CHECK-i64-NEXT:    frintx s20, s24
+; CHECK-i64-NEXT:    fmov d24, x14
+; CHECK-i64-NEXT:    fcvtzs x14, s22
+; CHECK-i64-NEXT:    frintx s3, s3
+; CHECK-i64-NEXT:    fmov d22, x16
+; CHECK-i64-NEXT:    frintx s27, s27
+; CHECK-i64-NEXT:    fcvtzs x16, s23
+; CHECK-i64-NEXT:    fcvt s21, h21
+; CHECK-i64-NEXT:    frintx s25, s25
+; CHECK-i64-NEXT:    fcvt s1, h1
+; CHECK-i64-NEXT:    mov h0, v0.h[3]
+; CHECK-i64-NEXT:    mov h23, v7.h[2]
+; CHECK-i64-NEXT:    mov v22.d[1], x15
+; CHECK-i64-NEXT:    fcvtzs x15, s20
+; CHECK-i64-NEXT:    fmov d20, x13
+; CHECK-i64-NEXT:    fcvtzs x13, s3
+; CHECK-i64-NEXT:    fmov d3, x14
+; CHECK-i64-NEXT:    fcvtzs x14, s27
+; CHECK-i64-NEXT:    fmov d27, x16
+; CHECK-i64-NEXT:    frintx s21, s21
+; CHECK-i64-NEXT:    mov h7, v7.h[3]
+; CHECK-i64-NEXT:    frintx s1, s1
+; CHECK-i64-NEXT:    fcvt s0, h0
+; CHECK-i64-NEXT:    fcvt s23, h23
+; CHECK-i64-NEXT:    fcvt s19, h19
+; CHECK-i64-NEXT:    mov v27.d[1], x15
+; CHECK-i64-NEXT:    fcvtzs x15, s25
+; CHECK-i64-NEXT:    mov h25, v6.h[3]
+; CHECK-i64-NEXT:    mov h6, v6.h[1]
+; CHECK-i64-NEXT:    mov v3.d[1], x13
+; CHECK-i64-NEXT:    fcvtzs x13, s21
+; CHECK-i64-NEXT:    mov h21, v5.h[1]
+; CHECK-i64-NEXT:    mov h5, v5.h[3]
+; CHECK-i64-NEXT:    mov v20.d[1], x14
+; CHECK-i64-NEXT:    fcvtzs x14, s1
+; CHECK-i64-NEXT:    mov h1, v4.h[1]
+; CHECK-i64-NEXT:    frintx s0, s0
+; CHECK-i64-NEXT:    fcvt s25, h25
+; CHECK-i64-NEXT:    fcvt s7, h7
+; CHECK-i64-NEXT:    stp q3, q27, [x8, #192]
+; CHECK-i64-NEXT:    fcvt s6, h6
+; CHECK-i64-NEXT:    mov h3, v4.h[3]
+; CHECK-i64-NEXT:    stp q22, q20, [x8, #128]
+; CHECK-i64-NEXT:    fcvt s21, h21
+; CHECK-i64-NEXT:    fcvt s5, h5
+; CHECK-i64-NEXT:    mov v24.d[1], x13
+; CHECK-i64-NEXT:    mov v26.d[1], x14
+; CHECK-i64-NEXT:    fcvt s4, h4
+; CHECK-i64-NEXT:    frintx s22, s25
+; CHECK-i64-NEXT:    fmov d20, x12
+; CHECK-i64-NEXT:    fcvt s1, h1
+; CHECK-i64-NEXT:    frintx s6, s6
+; CHECK-i64-NEXT:    fcvt s3, h3
+; CHECK-i64-NEXT:    fcvtzs x12, s0
+; CHECK-i64-NEXT:    frintx s5, s5
+; CHECK-i64-NEXT:    frintx s21, s21
+; CHECK-i64-NEXT:    fmov d0, x11
+; CHECK-i64-NEXT:    stp q26, q24, [x8, #64]
+; CHECK-i64-NEXT:    fmov d24, x15
+; CHECK-i64-NEXT:    frintx s4, s4
+; CHECK-i64-NEXT:    fcvtzs x11, s22
+; CHECK-i64-NEXT:    frintx s22, s23
+; CHECK-i64-NEXT:    frintx s1, s1
+; CHECK-i64-NEXT:    fcvtzs x13, s6
+; CHECK-i64-NEXT:    frintx s3, s3
+; CHECK-i64-NEXT:    frintx s6, s7
+; CHECK-i64-NEXT:    fcvtzs x14, s5
+; CHECK-i64-NEXT:    mov v24.d[1], x12
+; CHECK-i64-NEXT:    frintx s5, s19
+; CHECK-i64-NEXT:    fcvtzs x12, s21
+; CHECK-i64-NEXT:    mov v0.d[1], x10
+; CHECK-i64-NEXT:    fcvtzs x10, s4
+; CHECK-i64-NEXT:    mov v20.d[1], x11
+; CHECK-i64-NEXT:    fcvtzs x11, s22
+; CHECK-i64-NEXT:    mov v2.d[1], x13
+; CHECK-i64-NEXT:    fcvtzs x15, s3
+; CHECK-i64-NEXT:    fcvtzs x13, s1
+; CHECK-i64-NEXT:    mov v18.d[1], x14
+; CHECK-i64-NEXT:    fcvtzs x14, s6
+; CHECK-i64-NEXT:    stp q0, q24, [x8]
+; CHECK-i64-NEXT:    mov v17.d[1], x12
+; CHECK-i64-NEXT:    fcvtzs x12, s5
+; CHECK-i64-NEXT:    fmov d0, x10
+; CHECK-i64-NEXT:    fmov d1, x11
+; CHECK-i64-NEXT:    stp q2, q20, [x8, #224]
+; CHECK-i64-NEXT:    fmov d2, x9
+; CHECK-i64-NEXT:    mov v16.d[1], x15
+; CHECK-i64-NEXT:    stp q17, q18, [x8, #160]
+; CHECK-i64-NEXT:    mov v0.d[1], x13
+; CHECK-i64-NEXT:    mov v1.d[1], x14
+; CHECK-i64-NEXT:    mov v2.d[1], x12
+; CHECK-i64-NEXT:    stp q0, q16, [x8, #96]
+; CHECK-i64-NEXT:    stp q2, q1, [x8, #32]
+; CHECK-i64-NEXT:    ret
+  %a = call <32 x iXLen> @llvm.lrint.v32iXLen.v32f16(<32 x half> %x)
+  ret <32 x iXLen> %a
 }
-declare <32 x i64> @llvm.lrint.v32i64.v32f16(<32 x half>)
+declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f16(<32 x half>)
 
-define <1 x i64> @lrint_v1f32(<1 x float> %x) {
-; CHECK-SD-LABEL: lrint_v1f32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    frintx s0, s0
-; CHECK-SD-NEXT:    fcvtzs x8, s0
-; CHECK-SD-NEXT:    fmov d0, x8
-; CHECK-SD-NEXT:    ret
+define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
+; CHECK-i32-LABEL: lrint_v1f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.2s, v0.2s
+; CHECK-i32-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-i32-NEXT:    ret
 ;
-; CHECK-GI-LABEL: lrint_v1f32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    frintx s0, s0
-; CHECK-GI-NEXT:    fcvtzs x8, s0
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    ret
-  %a = call <1 x i64> @llvm.lrint.v1i64.v1f32(<1 x float> %x)
-  ret <1 x i64> %a
+; CHECK-i64-SD-LABEL: lrint_v1f32:
+; CHECK-i64-SD:       // %bb.0:
+; CHECK-i64-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-i64-SD-NEXT:    frintx s0, s0
+; CHECK-i64-SD-NEXT:    fcvtzs x8, s0
+; CHECK-i64-SD-NEXT:    fmov d0, x8
+; CHECK-i64-SD-NEXT:    ret
+;
+; CHECK-i64-GI-LABEL: lrint_v1f32:
+; CHECK-i64-GI:       // %bb.0:
+; CHECK-i64-GI-NEXT:    frintx s0, s0
+; CHECK-i64-GI-NEXT:    fcvtzs x8, s0
+; CHECK-i64-GI-NEXT:    fmov d0, x8
+; CHECK-i64-GI-NEXT:    ret
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float> %x)
+  ret <1 x iXLen> %a
 }
-declare <1 x i64> @llvm.lrint.v1i64.v1f32(<1 x float>)
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float>)
 
-define <2 x i64> @lrint_v2f32(<2 x float> %x) {
-; CHECK-LABEL: lrint_v2f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    frintx s1, s1
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fcvtzs x9, s1
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
-; CHECK-NEXT:    ret
-  %a = call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> %x)
-  ret <2 x i64> %a
+define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
+; CHECK-i32-LABEL: lrint_v2f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.2s, v0.2s
+; CHECK-i32-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v2f32:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v0.2s, v0.2s
+; CHECK-i64-NEXT:    mov s1, v0.s[1]
+; CHECK-i64-NEXT:    fcvtzs x8, s0
+; CHECK-i64-NEXT:    fcvtzs x9, s1
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    mov v0.d[1], x9
+; CHECK-i64-NEXT:    ret
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float> %x)
+  ret <2 x iXLen> %a
 }
-declare <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float>)
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float>)
 
-define <4 x i64> @lrint_v4f32(<4 x float> %x) {
-; CHECK-LABEL: lrint_v4f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    mov s3, v0.s[1]
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    mov s2, v1.s[1]
-; CHECK-NEXT:    frintx s1, s1
-; CHECK-NEXT:    frintx s3, s3
-; CHECK-NEXT:    fcvtzs x9, s0
-; CHECK-NEXT:    frintx s2, s2
-; CHECK-NEXT:    fcvtzs x8, s1
-; CHECK-NEXT:    fcvtzs x11, s3
-; CHECK-NEXT:    fmov d0, x9
-; CHECK-NEXT:    fcvtzs x10, s2
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    mov v0.d[1], x11
-; CHECK-NEXT:    mov v1.d[1], x10
-; CHECK-NEXT:    ret
-  %a = call <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float> %x)
-  ret <4 x i64> %a
+define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
+; CHECK-i32-LABEL: lrint_v4f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.4s, v0.4s
+; CHECK-i32-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v4f32:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-i64-NEXT:    frintx v0.2s, v0.2s
+; CHECK-i64-NEXT:    frintx v1.2s, v1.2s
+; CHECK-i64-NEXT:    mov s2, v0.s[1]
+; CHECK-i64-NEXT:    fcvtzs x8, s0
+; CHECK-i64-NEXT:    mov s3, v1.s[1]
+; CHECK-i64-NEXT:    fcvtzs x9, s1
+; CHECK-i64-NEXT:    fcvtzs x10, s2
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    fcvtzs x11, s3
+; CHECK-i64-NEXT:    fmov d1, x9
+; CHECK-i64-NEXT:    mov v0.d[1], x10
+; CHECK-i64-NEXT:    mov v1.d[1], x11
+; CHECK-i64-NEXT:    ret
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float> %x)
+  ret <4 x iXLen> %a
 }
-declare <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float>)
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float>)
 
-define <8 x i64> @lrint_v8f32(<8 x float> %x) {
-; CHECK-LABEL: lrint_v8f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    mov s4, v0.s[1]
-; CHECK-NEXT:    mov s7, v1.s[1]
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    frintx s1, s1
-; CHECK-NEXT:    mov s5, v2.s[1]
-; CHECK-NEXT:    mov s6, v3.s[1]
-; CHECK-NEXT:    frintx s2, s2
-; CHECK-NEXT:    frintx s3, s3
-; CHECK-NEXT:    frintx s4, s4
-; CHECK-NEXT:    frintx s7, s7
-; CHECK-NEXT:    fcvtzs x9, s0
-; CHECK-NEXT:    fcvtzs x12, s1
-; CHECK-NEXT:    frintx s5, s5
-; CHECK-NEXT:    frintx s6, s6
-; CHECK-NEXT:    fcvtzs x8, s2
-; CHECK-NEXT:    fcvtzs x10, s3
-; CHECK-NEXT:    fcvtzs x11, s4
-; CHECK-NEXT:    fcvtzs x15, s7
-; CHECK-NEXT:    fmov d0, x9
-; CHECK-NEXT:    fmov d2, x12
-; CHECK-NEXT:    fcvtzs x13, s5
-; CHECK-NEXT:    fcvtzs x14, s6
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    fmov d3, x10
-; CHECK-NEXT:    mov v0.d[1], x11
-; CHECK-NEXT:    mov v2.d[1], x15
-; CHECK-NEXT:    mov v1.d[1], x13
-; CHECK-NEXT:    mov v3.d[1], x14
-; CHECK-NEXT:    ret
-  %a = call <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float> %x)
-  ret <8 x i64> %a
+define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
+; CHECK-i32-LABEL: lrint_v8f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.4s, v0.4s
+; CHECK-i32-NEXT:    frintx v1.4s, v1.4s
+; CHECK-i32-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-i32-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v8f32:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-i64-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-i64-NEXT:    frintx v0.2s, v0.2s
+; CHECK-i64-NEXT:    frintx v1.2s, v1.2s
+; CHECK-i64-NEXT:    frintx v2.2s, v2.2s
+; CHECK-i64-NEXT:    frintx v3.2s, v3.2s
+; CHECK-i64-NEXT:    mov s4, v0.s[1]
+; CHECK-i64-NEXT:    mov s5, v1.s[1]
+; CHECK-i64-NEXT:    fcvtzs x8, s0
+; CHECK-i64-NEXT:    fcvtzs x10, s1
+; CHECK-i64-NEXT:    mov s6, v2.s[1]
+; CHECK-i64-NEXT:    mov s7, v3.s[1]
+; CHECK-i64-NEXT:    fcvtzs x11, s2
+; CHECK-i64-NEXT:    fcvtzs x12, s3
+; CHECK-i64-NEXT:    fcvtzs x9, s4
+; CHECK-i64-NEXT:    fcvtzs x13, s5
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    fmov d2, x10
+; CHECK-i64-NEXT:    fcvtzs x14, s6
+; CHECK-i64-NEXT:    fcvtzs x15, s7
+; CHECK-i64-NEXT:    fmov d1, x11
+; CHECK-i64-NEXT:    fmov d3, x12
+; CHECK-i64-NEXT:    mov v0.d[1], x9
+; CHECK-i64-NEXT:    mov v2.d[1], x13
+; CHECK-i64-NEXT:    mov v1.d[1], x14
+; CHECK-i64-NEXT:    mov v3.d[1], x15
+; CHECK-i64-NEXT:    ret
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float> %x)
+  ret <8 x iXLen> %a
 }
-declare <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float>)
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float>)
 
-define <16 x i64> @lrint_v16i64_v16f32(<16 x float> %x) {
-; CHECK-LABEL: lrint_v16i64_v16f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v4.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v5.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    ext v6.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT:    frintx s7, s0
-; CHECK-NEXT:    ext v16.16b, v3.16b, v3.16b, #8
-; CHECK-NEXT:    mov s0, v0.s[1]
-; CHECK-NEXT:    frintx s17, s4
-; CHECK-NEXT:    mov s4, v4.s[1]
-; CHECK-NEXT:    mov s18, v5.s[1]
-; CHECK-NEXT:    frintx s5, s5
-; CHECK-NEXT:    frintx s19, s6
-; CHECK-NEXT:    fcvtzs x8, s7
-; CHECK-NEXT:    frintx s7, s16
-; CHECK-NEXT:    mov s6, v6.s[1]
-; CHECK-NEXT:    mov s16, v16.s[1]
-; CHECK-NEXT:    frintx s0, s0
-; CHECK-NEXT:    frintx s4, s4
-; CHECK-NEXT:    fcvtzs x9, s17
-; CHECK-NEXT:    frintx s17, s1
-; CHECK-NEXT:    mov s1, v1.s[1]
-; CHECK-NEXT:    frintx s18, s18
-; CHECK-NEXT:    fcvtzs x10, s5
-; CHECK-NEXT:    mov s5, v2.s[1]
-; CHECK-NEXT:    fcvtzs x11, s19
-; CHECK-NEXT:    mov s19, v3.s[1]
-; CHECK-NEXT:    frintx s2, s2
-; CHECK-NEXT:    fcvtzs x12, s7
-; CHECK-NEXT:    frintx s6, s6
-; CHECK-NEXT:    fcvtzs x13, s4
-; CHECK-NEXT:    frintx s4, s3
-; CHECK-NEXT:    frintx s16, s16
-; CHECK-NEXT:    fcvtzs x14, s18
-; CHECK-NEXT:    frintx s18, s1
-; CHECK-NEXT:    fcvtzs x15, s17
-; CHECK-NEXT:    frintx s20, s5
-; CHECK-NEXT:    frintx s17, s19
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    fcvtzs x9, s2
-; CHECK-NEXT:    fmov d5, x11
-; CHECK-NEXT:    fmov d3, x10
-; CHECK-NEXT:    fcvtzs x11, s4
-; CHECK-NEXT:    fcvtzs x10, s0
-; CHECK-NEXT:    fmov d7, x12
-; CHECK-NEXT:    fcvtzs x12, s18
-; CHECK-NEXT:    fcvtzs x17, s6
-; CHECK-NEXT:    fcvtzs x18, s16
-; CHECK-NEXT:    fcvtzs x16, s20
-; CHECK-NEXT:    fcvtzs x0, s17
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d2, x15
-; CHECK-NEXT:    fmov d4, x9
-; CHECK-NEXT:    mov v1.d[1], x13
-; CHECK-NEXT:    fmov d6, x11
-; CHECK-NEXT:    mov v3.d[1], x14
-; CHECK-NEXT:    mov v0.d[1], x10
-; CHECK-NEXT:    mov v5.d[1], x17
-; CHECK-NEXT:    mov v7.d[1], x18
-; CHECK-NEXT:    mov v2.d[1], x12
-; CHECK-NEXT:    mov v4.d[1], x16
-; CHECK-NEXT:    mov v6.d[1], x0
-; CHECK-NEXT:    ret
-  %a = call <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float> %x)
-  ret <16 x i64> %a
+define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
+; CHECK-i32-LABEL: lrint_v16f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.4s, v0.4s
+; CHECK-i32-NEXT:    frintx v1.4s, v1.4s
+; CHECK-i32-NEXT:    frintx v2.4s, v2.4s
+; CHECK-i32-NEXT:    frintx v3.4s, v3.4s
+; CHECK-i32-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-i32-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-i32-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-i32-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v16f32:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v4.2s, v0.2s
+; CHECK-i64-NEXT:    frintx v5.2s, v1.2s
+; CHECK-i64-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-i64-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-i64-NEXT:    ext v6.16b, v2.16b, v2.16b, #8
+; CHECK-i64-NEXT:    ext v7.16b, v3.16b, v3.16b, #8
+; CHECK-i64-NEXT:    frintx v2.2s, v2.2s
+; CHECK-i64-NEXT:    frintx v3.2s, v3.2s
+; CHECK-i64-NEXT:    mov s16, v4.s[1]
+; CHECK-i64-NEXT:    mov s17, v5.s[1]
+; CHECK-i64-NEXT:    fcvtzs x8, s4
+; CHECK-i64-NEXT:    frintx v0.2s, v0.2s
+; CHECK-i64-NEXT:    frintx v1.2s, v1.2s
+; CHECK-i64-NEXT:    fcvtzs x9, s5
+; CHECK-i64-NEXT:    frintx v4.2s, v6.2s
+; CHECK-i64-NEXT:    frintx v5.2s, v7.2s
+; CHECK-i64-NEXT:    fcvtzs x10, s2
+; CHECK-i64-NEXT:    mov s6, v2.s[1]
+; CHECK-i64-NEXT:    fcvtzs x13, s3
+; CHECK-i64-NEXT:    mov s3, v3.s[1]
+; CHECK-i64-NEXT:    fcvtzs x11, s16
+; CHECK-i64-NEXT:    fcvtzs x12, s17
+; CHECK-i64-NEXT:    mov s7, v0.s[1]
+; CHECK-i64-NEXT:    mov s16, v1.s[1]
+; CHECK-i64-NEXT:    fcvtzs x15, s1
+; CHECK-i64-NEXT:    mov s1, v4.s[1]
+; CHECK-i64-NEXT:    mov s17, v5.s[1]
+; CHECK-i64-NEXT:    fcvtzs x14, s0
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    fcvtzs x8, s4
+; CHECK-i64-NEXT:    fmov d4, x10
+; CHECK-i64-NEXT:    fcvtzs x10, s5
+; CHECK-i64-NEXT:    fmov d2, x9
+; CHECK-i64-NEXT:    fcvtzs x9, s6
+; CHECK-i64-NEXT:    fmov d6, x13
+; CHECK-i64-NEXT:    fcvtzs x13, s7
+; CHECK-i64-NEXT:    fcvtzs x16, s16
+; CHECK-i64-NEXT:    fcvtzs x17, s3
+; CHECK-i64-NEXT:    fcvtzs x18, s1
+; CHECK-i64-NEXT:    fcvtzs x0, s17
+; CHECK-i64-NEXT:    fmov d1, x14
+; CHECK-i64-NEXT:    fmov d3, x15
+; CHECK-i64-NEXT:    fmov d5, x8
+; CHECK-i64-NEXT:    fmov d7, x10
+; CHECK-i64-NEXT:    mov v0.d[1], x11
+; CHECK-i64-NEXT:    mov v2.d[1], x12
+; CHECK-i64-NEXT:    mov v4.d[1], x9
+; CHECK-i64-NEXT:    mov v1.d[1], x13
+; CHECK-i64-NEXT:    mov v3.d[1], x16
+; CHECK-i64-NEXT:    mov v6.d[1], x17
+; CHECK-i64-NEXT:    mov v5.d[1], x18
+; CHECK-i64-NEXT:    mov v7.d[1], x0
+; CHECK-i64-NEXT:    ret
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float> %x)
+  ret <16 x iXLen> %a
 }
-declare <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float>)
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float>)
 
-define <1 x i64> @lrint_v1f64(<1 x double> %x) {
-; CHECK-LABEL: lrint_v1f64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    frintx d0, d0
-; CHECK-NEXT:    fcvtzs x8, d0
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    ret
-  %a = call <1 x i64> @llvm.lrint.v1i64.v1f64(<1 x double> %x)
-  ret <1 x i64> %a
+define <32 x iXLen> @lrint_v32f32(<32 x float> %x) {
+; CHECK-i32-LABEL: lrint_v32f32:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.4s, v0.4s
+; CHECK-i32-NEXT:    frintx v1.4s, v1.4s
+; CHECK-i32-NEXT:    frintx v2.4s, v2.4s
+; CHECK-i32-NEXT:    frintx v3.4s, v3.4s
+; CHECK-i32-NEXT:    frintx v4.4s, v4.4s
+; CHECK-i32-NEXT:    frintx v5.4s, v5.4s
+; CHECK-i32-NEXT:    frintx v6.4s, v6.4s
+; CHECK-i32-NEXT:    frintx v7.4s, v7.4s
+; CHECK-i32-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-i32-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-i32-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-i32-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-i32-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-i32-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-i32-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-i32-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v32f32:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ext v17.16b, v3.16b, v3.16b, #8
+; CHECK-i64-NEXT:    ext v18.16b, v4.16b, v4.16b, #8
+; CHECK-i64-NEXT:    ext v19.16b, v5.16b, v5.16b, #8
+; CHECK-i64-NEXT:    ext v21.16b, v7.16b, v7.16b, #8
+; CHECK-i64-NEXT:    ext v16.16b, v2.16b, v2.16b, #8
+; CHECK-i64-NEXT:    ext v20.16b, v6.16b, v6.16b, #8
+; CHECK-i64-NEXT:    frintx v7.2s, v7.2s
+; CHECK-i64-NEXT:    frintx v24.2s, v6.2s
+; CHECK-i64-NEXT:    frintx v23.2s, v5.2s
+; CHECK-i64-NEXT:    frintx v4.2s, v4.2s
+; CHECK-i64-NEXT:    frintx v3.2s, v3.2s
+; CHECK-i64-NEXT:    frintx v17.2s, v17.2s
+; CHECK-i64-NEXT:    frintx v18.2s, v18.2s
+; CHECK-i64-NEXT:    frintx v22.2s, v19.2s
+; CHECK-i64-NEXT:    frintx v21.2s, v21.2s
+; CHECK-i64-NEXT:    frintx v16.2s, v16.2s
+; CHECK-i64-NEXT:    frintx v20.2s, v20.2s
+; CHECK-i64-NEXT:    mov s25, v7.s[1]
+; CHECK-i64-NEXT:    fcvtzs x15, s7
+; CHECK-i64-NEXT:    frintx v19.2s, v1.2s
+; CHECK-i64-NEXT:    fcvtzs x16, s24
+; CHECK-i64-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-i64-NEXT:    fcvtzs x10, s17
+; CHECK-i64-NEXT:    fcvtzs x11, s18
+; CHECK-i64-NEXT:    mov s26, v22.s[1]
+; CHECK-i64-NEXT:    fcvtzs x12, s22
+; CHECK-i64-NEXT:    mov s22, v21.s[1]
+; CHECK-i64-NEXT:    fcvtzs x14, s21
+; CHECK-i64-NEXT:    mov s21, v24.s[1]
+; CHECK-i64-NEXT:    fcvtzs x9, s16
+; CHECK-i64-NEXT:    fcvtzs x13, s20
+; CHECK-i64-NEXT:    mov s20, v20.s[1]
+; CHECK-i64-NEXT:    fmov d24, x15
+; CHECK-i64-NEXT:    mov s18, v18.s[1]
+; CHECK-i64-NEXT:    fmov d6, x10
+; CHECK-i64-NEXT:    fmov d7, x11
+; CHECK-i64-NEXT:    fcvtzs x10, s25
+; CHECK-i64-NEXT:    fcvtzs x11, s22
+; CHECK-i64-NEXT:    fmov d25, x12
+; CHECK-i64-NEXT:    frintx v22.2s, v2.2s
+; CHECK-i64-NEXT:    fcvtzs x15, s21
+; CHECK-i64-NEXT:    fmov d21, x14
+; CHECK-i64-NEXT:    fmov d5, x9
+; CHECK-i64-NEXT:    fcvtzs x9, s26
+; CHECK-i64-NEXT:    fmov d26, x13
+; CHECK-i64-NEXT:    fcvtzs x12, s20
+; CHECK-i64-NEXT:    fcvtzs x13, s19
+; CHECK-i64-NEXT:    mov s20, v23.s[1]
+; CHECK-i64-NEXT:    mov v24.d[1], x10
+; CHECK-i64-NEXT:    mov v21.d[1], x11
+; CHECK-i64-NEXT:    fcvtzs x11, s23
+; CHECK-i64-NEXT:    fcvtzs x10, s22
+; CHECK-i64-NEXT:    mov s17, v17.s[1]
+; CHECK-i64-NEXT:    frintx v1.2s, v1.2s
+; CHECK-i64-NEXT:    mov s22, v22.s[1]
+; CHECK-i64-NEXT:    mov v26.d[1], x12
+; CHECK-i64-NEXT:    fcvtzs x12, s18
+; CHECK-i64-NEXT:    mov v25.d[1], x9
+; CHECK-i64-NEXT:    fmov d2, x13
+; CHECK-i64-NEXT:    fcvtzs x13, s20
+; CHECK-i64-NEXT:    fmov d20, x16
+; CHECK-i64-NEXT:    stp q24, q21, [x8, #224]
+; CHECK-i64-NEXT:    ext v21.16b, v0.16b, v0.16b, #8
+; CHECK-i64-NEXT:    fmov d18, x11
+; CHECK-i64-NEXT:    fcvtzs x11, s4
+; CHECK-i64-NEXT:    mov s4, v4.s[1]
+; CHECK-i64-NEXT:    fmov d23, x10
+; CHECK-i64-NEXT:    mov v20.d[1], x15
+; CHECK-i64-NEXT:    fcvtzs x10, s3
+; CHECK-i64-NEXT:    mov s3, v3.s[1]
+; CHECK-i64-NEXT:    mov v18.d[1], x13
+; CHECK-i64-NEXT:    frintx v0.2s, v0.2s
+; CHECK-i64-NEXT:    mov s16, v16.s[1]
+; CHECK-i64-NEXT:    frintx v21.2s, v21.2s
+; CHECK-i64-NEXT:    fcvtzs x13, s17
+; CHECK-i64-NEXT:    fcvtzs x14, s22
+; CHECK-i64-NEXT:    fcvtzs x9, s4
+; CHECK-i64-NEXT:    fmov d4, x11
+; CHECK-i64-NEXT:    mov v7.d[1], x12
+; CHECK-i64-NEXT:    stp q20, q26, [x8, #192]
+; CHECK-i64-NEXT:    fmov d20, x10
+; CHECK-i64-NEXT:    fcvtzs x10, s3
+; CHECK-i64-NEXT:    stp q18, q25, [x8, #160]
+; CHECK-i64-NEXT:    mov s18, v19.s[1]
+; CHECK-i64-NEXT:    mov s3, v1.s[1]
+; CHECK-i64-NEXT:    mov s17, v0.s[1]
+; CHECK-i64-NEXT:    mov s19, v21.s[1]
+; CHECK-i64-NEXT:    fcvtzs x11, s21
+; CHECK-i64-NEXT:    mov v4.d[1], x9
+; CHECK-i64-NEXT:    fcvtzs x9, s16
+; CHECK-i64-NEXT:    fcvtzs x12, s1
+; CHECK-i64-NEXT:    mov v6.d[1], x13
+; CHECK-i64-NEXT:    fcvtzs x13, s0
+; CHECK-i64-NEXT:    mov v20.d[1], x10
+; CHECK-i64-NEXT:    fcvtzs x15, s18
+; CHECK-i64-NEXT:    fcvtzs x10, s3
+; CHECK-i64-NEXT:    mov v23.d[1], x14
+; CHECK-i64-NEXT:    fcvtzs x14, s17
+; CHECK-i64-NEXT:    fmov d3, x11
+; CHECK-i64-NEXT:    stp q4, q7, [x8, #128]
+; CHECK-i64-NEXT:    mov v5.d[1], x9
+; CHECK-i64-NEXT:    fcvtzs x9, s19
+; CHECK-i64-NEXT:    stp q20, q6, [x8, #96]
+; CHECK-i64-NEXT:    fmov d0, x12
+; CHECK-i64-NEXT:    fmov d1, x13
+; CHECK-i64-NEXT:    mov v2.d[1], x15
+; CHECK-i64-NEXT:    stp q23, q5, [x8, #64]
+; CHECK-i64-NEXT:    mov v0.d[1], x10
+; CHECK-i64-NEXT:    mov v1.d[1], x14
+; CHECK-i64-NEXT:    mov v3.d[1], x9
+; CHECK-i64-NEXT:    stp q2, q0, [x8, #32]
+; CHECK-i64-NEXT:    stp q1, q3, [x8]
+; CHECK-i64-NEXT:    ret
+  %a = call <32 x iXLen> @llvm.lrint.v32iXLen.v32f32(<32 x float> %x)
+  ret <32 x iXLen> %a
 }
-declare <1 x i64> @llvm.lrint.v1i64.v1f64(<1 x double>)
+declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f32(<32 x float>)
 
-define <2 x i64> @lrint_v2f64(<2 x double> %x) {
-; CHECK-LABEL: lrint_v2f64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d1, v0.d[1]
-; CHECK-NEXT:    frintx d0, d0
-; CHECK-NEXT:    frintx d1, d1
-; CHECK-NEXT:    fcvtzs x8, d0
-; CHECK-NEXT:    fcvtzs x9, d1
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
-; CHECK-NEXT:    ret
-  %a = call <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double> %x)
-  ret <2 x i64> %a
+define <1 x iXLen> @lrint_v1f64(<1 x double> %x) {
+; CHECK-i32-LABEL: lrint_v1f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx d0, d0
+; CHECK-i32-NEXT:    fcvtzs w8, d0
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v1f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx d0, d0
+; CHECK-i64-NEXT:    fcvtzs x8, d0
+; CHECK-i64-NEXT:    fmov d0, x8
+; CHECK-i64-NEXT:    ret
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double>)
+
+define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
+; CHECK-i32-LABEL: lrint_v2f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.2d, v0.2d
+; CHECK-i32-NEXT:    mov d1, v0.d[1]
+; CHECK-i32-NEXT:    fcvtzs w8, d0
+; CHECK-i32-NEXT:    fcvtzs w9, d1
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    mov v0.s[1], w9
+; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v2f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v0.2d, v0.2d
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-i64-NEXT:    ret
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double>)
+
+define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
+; CHECK-i32-LABEL: lrint_v4f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.2d, v0.2d
+; CHECK-i32-NEXT:    frintx v1.2d, v1.2d
+; CHECK-i32-NEXT:    mov d2, v0.d[1]
+; CHECK-i32-NEXT:    fcvtzs w8, d0
+; CHECK-i32-NEXT:    fcvtzs w9, d2
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    fcvtzs w8, d1
+; CHECK-i32-NEXT:    mov d1, v1.d[1]
+; CHECK-i32-NEXT:    mov v0.s[1], w9
+; CHECK-i32-NEXT:    mov v0.s[2], w8
+; CHECK-i32-NEXT:    fcvtzs w8, d1
+; CHECK-i32-NEXT:    mov v0.s[3], w8
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v4f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v0.2d, v0.2d
+; CHECK-i64-NEXT:    frintx v1.2d, v1.2d
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-i64-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-i64-NEXT:    ret
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double>)
+
+define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
+; CHECK-i32-LABEL: lrint_v8f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v2.2d, v2.2d
+; CHECK-i32-NEXT:    frintx v0.2d, v0.2d
+; CHECK-i32-NEXT:    frintx v3.2d, v3.2d
+; CHECK-i32-NEXT:    mov d4, v0.d[1]
+; CHECK-i32-NEXT:    mov d5, v2.d[1]
+; CHECK-i32-NEXT:    fcvtzs w8, d0
+; CHECK-i32-NEXT:    fcvtzs w9, d2
+; CHECK-i32-NEXT:    frintx v2.2d, v1.2d
+; CHECK-i32-NEXT:    fcvtzs w10, d4
+; CHECK-i32-NEXT:    fcvtzs w11, d5
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    fmov s1, w9
+; CHECK-i32-NEXT:    fcvtzs w8, d2
+; CHECK-i32-NEXT:    mov d2, v2.d[1]
+; CHECK-i32-NEXT:    fcvtzs w9, d3
+; CHECK-i32-NEXT:    mov d3, v3.d[1]
+; CHECK-i32-NEXT:    mov v0.s[1], w10
+; CHECK-i32-NEXT:    mov v1.s[1], w11
+; CHECK-i32-NEXT:    mov v0.s[2], w8
+; CHECK-i32-NEXT:    fcvtzs w8, d2
+; CHECK-i32-NEXT:    mov v1.s[2], w9
+; CHECK-i32-NEXT:    fcvtzs w9, d3
+; CHECK-i32-NEXT:    mov v0.s[3], w8
+; CHECK-i32-NEXT:    mov v1.s[3], w9
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v8f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v0.2d, v0.2d
+; CHECK-i64-NEXT:    frintx v1.2d, v1.2d
+; CHECK-i64-NEXT:    frintx v2.2d, v2.2d
+; CHECK-i64-NEXT:    frintx v3.2d, v3.2d
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-i64-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-i64-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-i64-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-i64-NEXT:    ret
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f64(<8 x double> %x)
+  ret <8 x iXLen> %a
 }
-declare <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double>)
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f64(<8 x double>)
 
-define <4 x i64> @lrint_v4f64(<4 x double> %x) {
-; CHECK-LABEL: lrint_v4f64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d2, v0.d[1]
-; CHECK-NEXT:    mov d3, v1.d[1]
-; CHECK-NEXT:    frintx d0, d0
-; CHECK-NEXT:    frintx d1, d1
-; CHECK-NEXT:    frintx d2, d2
-; CHECK-NEXT:    frintx d3, d3
-; CHECK-NEXT:    fcvtzs x8, d0
-; CHECK-NEXT:    fcvtzs x9, d1
-; CHECK-NEXT:    fcvtzs x10, d2
-; CHECK-NEXT:    fcvtzs x11, d3
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    mov v0.d[1], x10
-; CHECK-NEXT:    mov v1.d[1], x11
-; CHECK-NEXT:    ret
-  %a = call <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double> %x)
-  ret <4 x i64> %a
+define <16 x iXLen> @lrint_v16f64(<16 x double> %x) {
+; CHECK-i32-LABEL: lrint_v16f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v0.2d, v0.2d
+; CHECK-i32-NEXT:    frintx v2.2d, v2.2d
+; CHECK-i32-NEXT:    frintx v4.2d, v4.2d
+; CHECK-i32-NEXT:    frintx v6.2d, v6.2d
+; CHECK-i32-NEXT:    frintx v17.2d, v1.2d
+; CHECK-i32-NEXT:    frintx v5.2d, v5.2d
+; CHECK-i32-NEXT:    fcvtzs w8, d0
+; CHECK-i32-NEXT:    mov d16, v0.d[1]
+; CHECK-i32-NEXT:    fcvtzs w9, d2
+; CHECK-i32-NEXT:    mov d2, v2.d[1]
+; CHECK-i32-NEXT:    fcvtzs w10, d4
+; CHECK-i32-NEXT:    mov d4, v4.d[1]
+; CHECK-i32-NEXT:    fcvtzs w11, d6
+; CHECK-i32-NEXT:    mov d6, v6.d[1]
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    fcvtzs w8, d16
+; CHECK-i32-NEXT:    frintx v16.2d, v3.2d
+; CHECK-i32-NEXT:    fmov s1, w9
+; CHECK-i32-NEXT:    fcvtzs w9, d2
+; CHECK-i32-NEXT:    fmov s2, w10
+; CHECK-i32-NEXT:    fcvtzs w10, d4
+; CHECK-i32-NEXT:    frintx v4.2d, v7.2d
+; CHECK-i32-NEXT:    fmov s3, w11
+; CHECK-i32-NEXT:    fcvtzs w11, d6
+; CHECK-i32-NEXT:    mov d6, v17.d[1]
+; CHECK-i32-NEXT:    mov v0.s[1], w8
+; CHECK-i32-NEXT:    fcvtzs w8, d17
+; CHECK-i32-NEXT:    mov d7, v16.d[1]
+; CHECK-i32-NEXT:    mov v1.s[1], w9
+; CHECK-i32-NEXT:    fcvtzs w9, d16
+; CHECK-i32-NEXT:    mov v2.s[1], w10
+; CHECK-i32-NEXT:    fcvtzs w10, d5
+; CHECK-i32-NEXT:    mov d5, v5.d[1]
+; CHECK-i32-NEXT:    mov v3.s[1], w11
+; CHECK-i32-NEXT:    fcvtzs w11, d4
+; CHECK-i32-NEXT:    mov d4, v4.d[1]
+; CHECK-i32-NEXT:    mov v0.s[2], w8
+; CHECK-i32-NEXT:    fcvtzs w8, d6
+; CHECK-i32-NEXT:    mov v1.s[2], w9
+; CHECK-i32-NEXT:    fcvtzs w9, d7
+; CHECK-i32-NEXT:    mov v2.s[2], w10
+; CHECK-i32-NEXT:    fcvtzs w10, d5
+; CHECK-i32-NEXT:    mov v3.s[2], w11
+; CHECK-i32-NEXT:    fcvtzs w11, d4
+; CHECK-i32-NEXT:    mov v0.s[3], w8
+; CHECK-i32-NEXT:    mov v1.s[3], w9
+; CHECK-i32-NEXT:    mov v2.s[3], w10
+; CHECK-i32-NEXT:    mov v3.s[3], w11
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v16f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    frintx v0.2d, v0.2d
+; CHECK-i64-NEXT:    frintx v1.2d, v1.2d
+; CHECK-i64-NEXT:    frintx v2.2d, v2.2d
+; CHECK-i64-NEXT:    frintx v3.2d, v3.2d
+; CHECK-i64-NEXT:    frintx v4.2d, v4.2d
+; CHECK-i64-NEXT:    frintx v5.2d, v5.2d
+; CHECK-i64-NEXT:    frintx v6.2d, v6.2d
+; CHECK-i64-NEXT:    frintx v7.2d, v7.2d
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-i64-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-i64-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-i64-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-i64-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-i64-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-i64-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-i64-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-i64-NEXT:    ret
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f64(<16 x double> %x)
+  ret <16 x iXLen> %a
 }
-declare <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double>)
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f64(<16 x double>)
 
-define <8 x i64> @lrint_v8f64(<8 x double> %x) {
-; CHECK-LABEL: lrint_v8f64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d4, v0.d[1]
-; CHECK-NEXT:    mov d5, v1.d[1]
-; CHECK-NEXT:    mov d6, v2.d[1]
-; CHECK-NEXT:    mov d7, v3.d[1]
-; CHECK-NEXT:    frintx d0, d0
-; CHECK-NEXT:    frintx d1, d1
-; CHECK-NEXT:    frintx d2, d2
-; CHECK-NEXT:    frintx d3, d3
-; CHECK-NEXT:    frintx d4, d4
-; CHECK-NEXT:    frintx d5, d5
-; CHECK-NEXT:    frintx d6, d6
-; CHECK-NEXT:    frintx d7, d7
-; CHECK-NEXT:    fcvtzs x8, d0
-; CHECK-NEXT:    fcvtzs x9, d1
-; CHECK-NEXT:    fcvtzs x10, d2
-; CHECK-NEXT:    fcvtzs x11, d3
-; CHECK-NEXT:    fcvtzs x12, d4
-; CHECK-NEXT:    fcvtzs x13, d5
-; CHECK-NEXT:    fcvtzs x14, d6
-; CHECK-NEXT:    fcvtzs x15, d7
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    fmov d2, x10
-; CHECK-NEXT:    fmov d3, x11
-; CHECK-NEXT:    mov v0.d[1], x12
-; CHECK-NEXT:    mov v1.d[1], x13
-; CHECK-NEXT:    mov v2.d[1], x14
-; CHECK-NEXT:    mov v3.d[1], x15
-; CHECK-NEXT:    ret
-  %a = call <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double> %x)
-  ret <8 x i64> %a
+define <32 x iXLen> @lrint_v32f64(<32 x double> %x) {
+; CHECK-i32-LABEL: lrint_v32f64:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    frintx v17.2d, v0.2d
+; CHECK-i32-NEXT:    frintx v19.2d, v2.2d
+; CHECK-i32-NEXT:    frintx v0.2d, v1.2d
+; CHECK-i32-NEXT:    frintx v1.2d, v4.2d
+; CHECK-i32-NEXT:    frintx v2.2d, v3.2d
+; CHECK-i32-NEXT:    frintx v3.2d, v5.2d
+; CHECK-i32-NEXT:    ldp q16, q5, [sp]
+; CHECK-i32-NEXT:    frintx v18.2d, v6.2d
+; CHECK-i32-NEXT:    frintx v4.2d, v7.2d
+; CHECK-i32-NEXT:    ldp q22, q6, [sp, #64]
+; CHECK-i32-NEXT:    mov d20, v17.d[1]
+; CHECK-i32-NEXT:    mov d21, v19.d[1]
+; CHECK-i32-NEXT:    fcvtzs w8, d17
+; CHECK-i32-NEXT:    fcvtzs w9, d19
+; CHECK-i32-NEXT:    ldp q17, q7, [sp, #32]
+; CHECK-i32-NEXT:    fcvtzs w12, d0
+; CHECK-i32-NEXT:    mov d19, v1.d[1]
+; CHECK-i32-NEXT:    fcvtzs w13, d1
+; CHECK-i32-NEXT:    frintx v16.2d, v16.2d
+; CHECK-i32-NEXT:    mov d23, v18.d[1]
+; CHECK-i32-NEXT:    fcvtzs w15, d18
+; CHECK-i32-NEXT:    fcvtzs w10, d20
+; CHECK-i32-NEXT:    fcvtzs w11, d21
+; CHECK-i32-NEXT:    mov d21, v0.d[1]
+; CHECK-i32-NEXT:    fmov s0, w8
+; CHECK-i32-NEXT:    fmov s1, w9
+; CHECK-i32-NEXT:    frintx v17.2d, v17.2d
+; CHECK-i32-NEXT:    frintx v20.2d, v22.2d
+; CHECK-i32-NEXT:    mov d22, v2.d[1]
+; CHECK-i32-NEXT:    fcvtzs w14, d19
+; CHECK-i32-NEXT:    mov d18, v16.d[1]
+; CHECK-i32-NEXT:    frintx v7.2d, v7.2d
+; CHECK-i32-NEXT:    mov v0.s[1], w10
+; CHECK-i32-NEXT:    fcvtzs w10, d2
+; CHECK-i32-NEXT:    mov v1.s[1], w11
+; CHECK-i32-NEXT:    fcvtzs w8, d21
+; CHECK-i32-NEXT:    ldp q21, q19, [sp, #96]
+; CHECK-i32-NEXT:    fmov s2, w13
+; CHECK-i32-NEXT:    fcvtzs w11, d23
+; CHECK-i32-NEXT:    mov d23, v3.d[1]
+; CHECK-i32-NEXT:    fcvtzs w9, d22
+; CHECK-i32-NEXT:    mov d22, v17.d[1]
+; CHECK-i32-NEXT:    fcvtzs w13, d18
+; CHECK-i32-NEXT:    mov v0.s[2], w12
+; CHECK-i32-NEXT:    fcvtzs w12, d16
+; CHECK-i32-NEXT:    mov v1.s[2], w10
+; CHECK-i32-NEXT:    fcvtzs w10, d3
+; CHECK-i32-NEXT:    fmov s3, w15
+; CHECK-i32-NEXT:    frintx v21.2d, v21.2d
+; CHECK-i32-NEXT:    mov v2.s[1], w14
+; CHECK-i32-NEXT:    mov d16, v20.d[1]
+; CHECK-i32-NEXT:    fcvtzs w14, d17
+; CHECK-i32-NEXT:    mov d17, v4.d[1]
+; CHECK-i32-NEXT:    fcvtzs w15, d22
+; CHECK-i32-NEXT:    frintx v22.2d, v5.2d
+; CHECK-i32-NEXT:    mov v3.s[1], w11
+; CHECK-i32-NEXT:    fcvtzs w11, d4
+; CHECK-i32-NEXT:    fmov s4, w12
+; CHECK-i32-NEXT:    fcvtzs w12, d20
+; CHECK-i32-NEXT:    mov d18, v21.d[1]
+; CHECK-i32-NEXT:    mov d20, v7.d[1]
+; CHECK-i32-NEXT:    fmov s5, w14
+; CHECK-i32-NEXT:    fcvtzs w14, d21
+; CHECK-i32-NEXT:    mov v2.s[2], w10
+; CHECK-i32-NEXT:    mov v4.s[1], w13
+; CHECK-i32-NEXT:    fcvtzs w13, d16
+; CHECK-i32-NEXT:    frintx v16.2d, v6.2d
+; CHECK-i32-NEXT:    fcvtzs w10, d23
+; CHECK-i32-NEXT:    mov v3.s[2], w11
+; CHECK-i32-NEXT:    fcvtzs w11, d17
+; CHECK-i32-NEXT:    fmov s6, w12
+; CHECK-i32-NEXT:    mov v5.s[1], w15
+; CHECK-i32-NEXT:    fcvtzs w15, d18
+; CHECK-i32-NEXT:    frintx v18.2d, v19.2d
+; CHECK-i32-NEXT:    fcvtzs w12, d22
+; CHECK-i32-NEXT:    mov d19, v22.d[1]
+; CHECK-i32-NEXT:    mov v0.s[3], w8
+; CHECK-i32-NEXT:    mov v1.s[3], w9
+; CHECK-i32-NEXT:    mov v6.s[1], w13
+; CHECK-i32-NEXT:    fcvtzs w13, d7
+; CHECK-i32-NEXT:    fmov s7, w14
+; CHECK-i32-NEXT:    fcvtzs w14, d16
+; CHECK-i32-NEXT:    mov d16, v16.d[1]
+; CHECK-i32-NEXT:    mov v2.s[3], w10
+; CHECK-i32-NEXT:    mov v4.s[2], w12
+; CHECK-i32-NEXT:    fcvtzs w12, d19
+; CHECK-i32-NEXT:    mov v3.s[3], w11
+; CHECK-i32-NEXT:    mov v7.s[1], w15
+; CHECK-i32-NEXT:    fcvtzs w15, d18
+; CHECK-i32-NEXT:    mov d18, v18.d[1]
+; CHECK-i32-NEXT:    mov v5.s[2], w13
+; CHECK-i32-NEXT:    fcvtzs w13, d20
+; CHECK-i32-NEXT:    mov v6.s[2], w14
+; CHECK-i32-NEXT:    fcvtzs w14, d16
+; CHECK-i32-NEXT:    mov v4.s[3], w12
+; CHECK-i32-NEXT:    mov v7.s[2], w15
+; CHECK-i32-NEXT:    fcvtzs w15, d18
+; CHECK-i32-NEXT:    mov v5.s[3], w13
+; CHECK-i32-NEXT:    mov v6.s[3], w14
+; CHECK-i32-NEXT:    mov v7.s[3], w15
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v32f64:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    ldp q17, q16, [sp, #96]
+; CHECK-i64-NEXT:    frintx v7.2d, v7.2d
+; CHECK-i64-NEXT:    ldp q19, q18, [sp, #64]
+; CHECK-i64-NEXT:    frintx v6.2d, v6.2d
+; CHECK-i64-NEXT:    ldp q21, q20, [sp, #32]
+; CHECK-i64-NEXT:    frintx v5.2d, v5.2d
+; CHECK-i64-NEXT:    frintx v16.2d, v16.2d
+; CHECK-i64-NEXT:    frintx v17.2d, v17.2d
+; CHECK-i64-NEXT:    frintx v4.2d, v4.2d
+; CHECK-i64-NEXT:    frintx v18.2d, v18.2d
+; CHECK-i64-NEXT:    frintx v19.2d, v19.2d
+; CHECK-i64-NEXT:    frintx v3.2d, v3.2d
+; CHECK-i64-NEXT:    ldp q23, q22, [sp]
+; CHECK-i64-NEXT:    frintx v20.2d, v20.2d
+; CHECK-i64-NEXT:    frintx v21.2d, v21.2d
+; CHECK-i64-NEXT:    frintx v2.2d, v2.2d
+; CHECK-i64-NEXT:    frintx v1.2d, v1.2d
+; CHECK-i64-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-i64-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-i64-NEXT:    frintx v0.2d, v0.2d
+; CHECK-i64-NEXT:    frintx v22.2d, v22.2d
+; CHECK-i64-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-i64-NEXT:    frintx v23.2d, v23.2d
+; CHECK-i64-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-i64-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-i64-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-i64-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-i64-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-i64-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-i64-NEXT:    stp q17, q16, [x8, #224]
+; CHECK-i64-NEXT:    fcvtzs v16.2d, v21.2d
+; CHECK-i64-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-i64-NEXT:    fcvtzs v17.2d, v22.2d
+; CHECK-i64-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-i64-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-i64-NEXT:    stp q19, q18, [x8, #192]
+; CHECK-i64-NEXT:    fcvtzs v18.2d, v23.2d
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-i64-NEXT:    stp q4, q5, [x8, #64]
+; CHECK-i64-NEXT:    stp q6, q7, [x8, #96]
+; CHECK-i64-NEXT:    stp q2, q3, [x8, #32]
+; CHECK-i64-NEXT:    stp q0, q1, [x8]
+; CHECK-i64-NEXT:    stp q18, q17, [x8, #128]
+; CHECK-i64-NEXT:    stp q16, q20, [x8, #160]
+; CHECK-i64-NEXT:    ret
+  %a = call <32 x iXLen> @llvm.lrint.v32iXLen.v16f64(<32 x double> %x)
+  ret <32 x iXLen> %a
 }
-declare <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double>)
+declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f64(<32 x double>)
-- 
GitLab


From e3ca558ffb1441cb16da7aba021e12c6f110000f Mon Sep 17 00:00:00 2001
From: agozillon <Andrew.Gozillon@amd.com>
Date: Fri, 10 May 2024 14:55:43 -0500
Subject: [PATCH 0447/1206] [Flang] Remove deprecated cast style that snuck in
 during landing of 435e850ba97ab567a14b6c84d2b27cadb771cb27

---
 flang/lib/Lower/OpenMP/Utils.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index b183705c5677..eed63b226133 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -127,7 +127,7 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
                 mlir::DenseIntElementsAttr membersIndex, uint64_t mapType,
                 mlir::omp::VariableCaptureKind mapCaptureType, mlir::Type retTy,
                 bool partialMap) {
-  if (auto boxTy = baseAddr.getType().dyn_cast<fir::BaseBoxType>()) {
+  if (auto boxTy = llvm::dyn_cast<fir::BaseBoxType>(baseAddr.getType())) {
     baseAddr = builder.create<fir::BoxAddrOp>(loc, baseAddr);
     retTy = baseAddr.getType();
   }
-- 
GitLab


From a427aa9346295fe7dd3be5955214d28c8be2ad4a Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Fri, 10 May 2024 15:04:39 -0500
Subject: [PATCH 0448/1206] [flang][Lower] Treat directives with nested
 evaluations as constructs (#91614)

When generating block terminators in `genFIR(Evaluation)`, treat
`Directives` with nested evaluations the same way as `Constructs` to
determine the successor block.

This fixes https://github.com/llvm/llvm-project/issues/91526
---
 flang/lib/Lower/Bridge.cpp               | 10 +++++++---
 flang/test/Lower/branching-directive.f90 | 25 ++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Lower/branching-directive.f90

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4902886712e9..79d6bbf65cbf 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -4548,9 +4548,13 @@ private:
     // constructs, this can be done for either the end construct statement,
     // or for the construct itself, which will skip this code if the
     // end statement was visited first and generated a branch.
-    Fortran::lower::pft::Evaluation *successor =
-        eval.isConstruct() ? eval.getLastNestedEvaluation().lexicalSuccessor
-                           : eval.lexicalSuccessor;
+    Fortran::lower::pft::Evaluation *successor = [&]() {
+      if (eval.isConstruct() ||
+          (eval.isDirective() && eval.hasNestedEvaluations()))
+        return eval.getLastNestedEvaluation().lexicalSuccessor;
+      return eval.lexicalSuccessor;
+    }();
+
     if (successor && blockIsUnterminated()) {
       if (successor->isIntermediateConstructStmt() &&
           successor->parentConstruct->lowerAsUnstructured())
diff --git a/flang/test/Lower/branching-directive.f90 b/flang/test/Lower/branching-directive.f90
new file mode 100644
index 000000000000..a0a147f1053a
--- /dev/null
+++ b/flang/test/Lower/branching-directive.f90
@@ -0,0 +1,25 @@
+!RUN: flang-new -fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+
+!https://github.com/llvm/llvm-project/issues/91526
+
+!CHECK:   cf.cond_br %{{[0-9]+}}, ^bb[[THEN:[0-9]+]], ^bb[[ELSE:[0-9]+]]
+!CHECK: ^bb[[THEN]]:
+!CHECK:   cf.br ^bb[[EXIT:[0-9]+]]
+!CHECK: ^bb[[ELSE]]:
+!CHECK:   fir.call @_FortranAStopStatement
+!CHECK:   fir.unreachable
+!CHECK: ^bb[[EXIT]]:
+
+subroutine simple(y)
+  implicit none
+  logical, intent(in) :: y
+  integer :: i
+  if (y) then
+!$omp parallel
+    i = 1
+!$omp end parallel
+  else
+    stop 1
+  end if
+end subroutine simple
+
-- 
GitLab


From 6b9bca8faaa759c56fdd6e8697f9be38d201bd71 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Fri, 10 May 2024 13:20:51 -0700
Subject: [PATCH 0449/1206] [BOLT] Preserve Offset annotation in SCTC (#91693)

Offset annotation is used in writing BAT tables.

Test Plan: updated sctc-bug4.test
---
 bolt/lib/Passes/BinaryPasses.cpp |  5 +++++
 bolt/test/X86/sctc-bug4.test     | 10 +++++++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp
index c0ba73108f57..df6dbcddeed5 100644
--- a/bolt/lib/Passes/BinaryPasses.cpp
+++ b/bolt/lib/Passes/BinaryPasses.cpp
@@ -910,6 +910,11 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryFunction &BF) {
       auto &CTCAnnotation =
           MIB->getOrCreateAnnotationAs<uint64_t>(*CondBranch, "CTCTakenCount");
       CTCAnnotation = CTCTakenFreq;
+      // Preserve Offset annotation, used in BAT.
+      // Instr is a direct tail call instruction that was created when CTCs are
+      // first expanded, and has the original CTC offset set.
+      if (std::optional<uint32_t> Offset = MIB->getOffset(*Instr))
+        MIB->setOffset(*CondBranch, *Offset);
 
       // Remove the unused successor which may be eliminated later
       // if there are no other users.
diff --git a/bolt/test/X86/sctc-bug4.test b/bolt/test/X86/sctc-bug4.test
index 00f5ee429b63..92aca5110059 100644
--- a/bolt/test/X86/sctc-bug4.test
+++ b/bolt/test/X86/sctc-bug4.test
@@ -1,20 +1,23 @@
-# Check that fallthrough blocks are handled properly.
+# Check that fallthrough blocks are handled properly and Offset annotation is
+# set for conditional tail calls.
 
 RUN: %clang %cflags %S/Inputs/sctc_bug4.s -o %t
-RUN: llvm-bolt %t -o %t.null \
+RUN: llvm-bolt %t -o %t.null --enable-bat \
 RUN:   -funcs=test_func -print-sctc -sequential-disassembly 2>&1 | FileCheck %s
 
 CHECK:      .Ltmp2 (3 instructions, align : 1)
 CHECK-NEXT:   CFI State : 0
+CHECK-NEXT:   Input offset: 0x24
 CHECK-NEXT:   Predecessors: .LFT1
 CHECK-NEXT:     00000024: 	cmpq	$0x20, %rsi
-CHECK-NEXT:     00000028: 	ja	dummy # TAILCALL {{.*}}# CTCTakenCount: 0
+CHECK-NEXT:     00000028: 	ja	dummy # TAILCALL # Offset: 53 # CTCTakenCount: 0
 CHECK-NEXT:     0000002a: 	jmp .Ltmp4
 CHECK-NEXT:   Successors: .Ltmp4
 CHECK-NEXT:   CFI State: 0
 
 CHECK:      .Ltmp1 (2 instructions, align : 1)
 CHECK-NEXT:   CFI State : 0
+CHECK-NEXT:   Input offset: 0x2c
 CHECK-NEXT:   Predecessors: .LFT0
 CHECK-NEXT:     0000002c: 	xorq	%r11, %rax
 CHECK-NEXT:     0000002f: 	retq
@@ -22,4 +25,5 @@ CHECK-NEXT:   CFI State: 0
 
 CHECK:      .Ltmp4 (4 instructions, align : 1)
 CHECK-NEXT:  CFI State : 0
+CHECK-NEXT:  Input offset: 0x3a
 CHECK-NEXT:  Predecessors: .Ltmp2
-- 
GitLab


From e9f53e4095d8a8600b5c5d445c73e2d5a6f45abb Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 10 May 2024 13:22:07 -0700
Subject: [PATCH 0450/1206] [test] Move RISCV tests to
 clang/test/CodeGen/RISCV/

The directory was created by 2f1fe9a3a60d6f18998c5f3b7e643d4cbaa4e65d
(2020).

Pull Request: https://github.com/llvm/llvm-project/pull/91783
---
 .../attr-rvv-vector-bits-bitcast.c}                               | 0
 .../attr-rvv-vector-bits-call.c}                                  | 0
 .../attr-rvv-vector-bits-cast.c}                                  | 0
 .../attr-rvv-vector-bits-codegen.c}                               | 0
 .../attr-rvv-vector-bits-globals.c}                               | 0
 .../attr-rvv-vector-bits-types.c}                                 | 0
 clang/test/CodeGen/{ => RISCV}/riscv32-ilp32d-abi.cpp             | 0
 .../{riscv-rvv-vls-arith-ops.c => RISCV/rvv-vls-arith-ops.c}      | 0
 .../{riscv-rvv-vls-bitwise-ops.c => RISCV/rvv-vls-bitwise-ops.c}  | 0
 .../{riscv-rvv-vls-compare-ops.c => RISCV/rvv-vls-compare-ops.c}  | 0
 .../{riscv-rvv-vls-shift-ops.c => RISCV/rvv-vls-shift-ops.c}      | 0
 .../rvv-vls-subscript-ops.c}                                      | 0
 .../vector-bits-vscale-range.c}                                   | 0
 13 files changed, 0 insertions(+), 0 deletions(-)
 rename clang/test/CodeGen/{attr-riscv-rvv-vector-bits-bitcast.c => RISCV/attr-rvv-vector-bits-bitcast.c} (100%)
 rename clang/test/CodeGen/{attr-riscv-rvv-vector-bits-call.c => RISCV/attr-rvv-vector-bits-call.c} (100%)
 rename clang/test/CodeGen/{attr-riscv-rvv-vector-bits-cast.c => RISCV/attr-rvv-vector-bits-cast.c} (100%)
 rename clang/test/CodeGen/{attr-riscv-rvv-vector-bits-codegen.c => RISCV/attr-rvv-vector-bits-codegen.c} (100%)
 rename clang/test/CodeGen/{attr-riscv-rvv-vector-bits-globals.c => RISCV/attr-rvv-vector-bits-globals.c} (100%)
 rename clang/test/CodeGen/{attr-riscv-rvv-vector-bits-types.c => RISCV/attr-rvv-vector-bits-types.c} (100%)
 rename clang/test/CodeGen/{ => RISCV}/riscv32-ilp32d-abi.cpp (100%)
 rename clang/test/CodeGen/{riscv-rvv-vls-arith-ops.c => RISCV/rvv-vls-arith-ops.c} (100%)
 rename clang/test/CodeGen/{riscv-rvv-vls-bitwise-ops.c => RISCV/rvv-vls-bitwise-ops.c} (100%)
 rename clang/test/CodeGen/{riscv-rvv-vls-compare-ops.c => RISCV/rvv-vls-compare-ops.c} (100%)
 rename clang/test/CodeGen/{riscv-rvv-vls-shift-ops.c => RISCV/rvv-vls-shift-ops.c} (100%)
 rename clang/test/CodeGen/{riscv-rvv-vls-subscript-ops.c => RISCV/rvv-vls-subscript-ops.c} (100%)
 rename clang/test/CodeGen/{riscv-vector-bits-vscale-range.c => RISCV/vector-bits-vscale-range.c} (100%)

diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-bitcast.c b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-bitcast.c
similarity index 100%
rename from clang/test/CodeGen/attr-riscv-rvv-vector-bits-bitcast.c
rename to clang/test/CodeGen/RISCV/attr-rvv-vector-bits-bitcast.c
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-call.c b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-call.c
similarity index 100%
rename from clang/test/CodeGen/attr-riscv-rvv-vector-bits-call.c
rename to clang/test/CodeGen/RISCV/attr-rvv-vector-bits-call.c
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-cast.c b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-cast.c
similarity index 100%
rename from clang/test/CodeGen/attr-riscv-rvv-vector-bits-cast.c
rename to clang/test/CodeGen/RISCV/attr-rvv-vector-bits-cast.c
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-codegen.c b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-codegen.c
similarity index 100%
rename from clang/test/CodeGen/attr-riscv-rvv-vector-bits-codegen.c
rename to clang/test/CodeGen/RISCV/attr-rvv-vector-bits-codegen.c
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-globals.c b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-globals.c
similarity index 100%
rename from clang/test/CodeGen/attr-riscv-rvv-vector-bits-globals.c
rename to clang/test/CodeGen/RISCV/attr-rvv-vector-bits-globals.c
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-types.c b/clang/test/CodeGen/RISCV/attr-rvv-vector-bits-types.c
similarity index 100%
rename from clang/test/CodeGen/attr-riscv-rvv-vector-bits-types.c
rename to clang/test/CodeGen/RISCV/attr-rvv-vector-bits-types.c
diff --git a/clang/test/CodeGen/riscv32-ilp32d-abi.cpp b/clang/test/CodeGen/RISCV/riscv32-ilp32d-abi.cpp
similarity index 100%
rename from clang/test/CodeGen/riscv32-ilp32d-abi.cpp
rename to clang/test/CodeGen/RISCV/riscv32-ilp32d-abi.cpp
diff --git a/clang/test/CodeGen/riscv-rvv-vls-arith-ops.c b/clang/test/CodeGen/RISCV/rvv-vls-arith-ops.c
similarity index 100%
rename from clang/test/CodeGen/riscv-rvv-vls-arith-ops.c
rename to clang/test/CodeGen/RISCV/rvv-vls-arith-ops.c
diff --git a/clang/test/CodeGen/riscv-rvv-vls-bitwise-ops.c b/clang/test/CodeGen/RISCV/rvv-vls-bitwise-ops.c
similarity index 100%
rename from clang/test/CodeGen/riscv-rvv-vls-bitwise-ops.c
rename to clang/test/CodeGen/RISCV/rvv-vls-bitwise-ops.c
diff --git a/clang/test/CodeGen/riscv-rvv-vls-compare-ops.c b/clang/test/CodeGen/RISCV/rvv-vls-compare-ops.c
similarity index 100%
rename from clang/test/CodeGen/riscv-rvv-vls-compare-ops.c
rename to clang/test/CodeGen/RISCV/rvv-vls-compare-ops.c
diff --git a/clang/test/CodeGen/riscv-rvv-vls-shift-ops.c b/clang/test/CodeGen/RISCV/rvv-vls-shift-ops.c
similarity index 100%
rename from clang/test/CodeGen/riscv-rvv-vls-shift-ops.c
rename to clang/test/CodeGen/RISCV/rvv-vls-shift-ops.c
diff --git a/clang/test/CodeGen/riscv-rvv-vls-subscript-ops.c b/clang/test/CodeGen/RISCV/rvv-vls-subscript-ops.c
similarity index 100%
rename from clang/test/CodeGen/riscv-rvv-vls-subscript-ops.c
rename to clang/test/CodeGen/RISCV/rvv-vls-subscript-ops.c
diff --git a/clang/test/CodeGen/riscv-vector-bits-vscale-range.c b/clang/test/CodeGen/RISCV/vector-bits-vscale-range.c
similarity index 100%
rename from clang/test/CodeGen/riscv-vector-bits-vscale-range.c
rename to clang/test/CodeGen/RISCV/vector-bits-vscale-range.c
-- 
GitLab


From a7eff59f78f08f8ef0487dfe2a136fb311af4fd0 Mon Sep 17 00:00:00 2001
From: Zequan Wu <zequanwu@google.com>
Date: Fri, 10 May 2024 16:39:20 -0400
Subject: [PATCH 0451/1206] [lldb][DWARF] Do not complete type from declaration
 die. (#91799)

Fix the problem:
https://github.com/llvm/llvm-project/pull/90663#issuecomment-2105164917
by enhancing a double-check for #90663
---
 lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index e0b1b430b266..2a46be921612 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -2345,6 +2345,11 @@ bool DWARFASTParserClang::CompleteTypeFromDWARF(const DWARFDIE &die,
 
   if (!die)
     return false;
+  ParsedDWARFTypeAttributes attrs(die);
+  bool is_forward_declaration = IsForwardDeclaration(
+      die, attrs, SymbolFileDWARF::GetLanguage(*die.GetCU()));
+  if (is_forward_declaration)
+    return false;
 
   const dw_tag_t tag = die.Tag();
 
-- 
GitLab


From f32f6d199ac36e0c45433a648691f5165893deec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 10 May 2024 13:41:53 -0700
Subject: [PATCH 0452/1206] [flang][cuda] Implicitly load cudadevice module in
 device/global subprogram (#91668)

Some functions and subroutines are available in device context
(device/global). These functions have interfaces declared in the
`cudadevice` module.

This patch adds interfaces as `__cuda_device_builtins_<fctname>` in a
builtin module and they are USE'd rename in the `cudadevice` module. The
module is implicitly used in device/global subprograms.

The builtin module only contains procedures from section 3.6.4 for now.
---
 flang/include/flang/Semantics/semantics.h     |  5 +-
 flang/lib/Semantics/check-cuda.cpp            |  5 ++
 flang/lib/Semantics/resolve-names.cpp         | 13 ++++
 flang/lib/Semantics/semantics.cpp             |  8 ++
 flang/module/__cuda_device_builtins.f90       | 74 +++++++++++++++++++
 flang/module/cudadevice.f90                   | 21 ++++++
 .../Semantics/cuf-device-procedures01.cuf     | 35 +++++++++
 .../Semantics/cuf-device-procedures02.cuf     | 17 +++++
 flang/tools/f18/CMakeLists.txt                |  4 +
 9 files changed, 181 insertions(+), 1 deletion(-)
 create mode 100644 flang/module/__cuda_device_builtins.f90
 create mode 100644 flang/module/cudadevice.f90
 create mode 100644 flang/test/Semantics/cuf-device-procedures01.cuf
 create mode 100644 flang/test/Semantics/cuf-device-procedures02.cuf

diff --git a/flang/include/flang/Semantics/semantics.h b/flang/include/flang/Semantics/semantics.h
index e6ba71d53e92..167e61381639 100644
--- a/flang/include/flang/Semantics/semantics.h
+++ b/flang/include/flang/Semantics/semantics.h
@@ -215,8 +215,10 @@ public:
   void UseFortranBuiltinsModule();
   const Scope *GetBuiltinsScope() const { return builtinsScope_; }
 
-  void UsePPCBuiltinTypesModule();
   const Scope &GetCUDABuiltinsScope();
+  const Scope &GetCUDADeviceScope();
+
+  void UsePPCBuiltinTypesModule();
   void UsePPCBuiltinsModule();
   Scope *GetPPCBuiltinTypesScope() { return ppcBuiltinTypesScope_; }
   const Scope *GetPPCBuiltinsScope() const { return ppcBuiltinsScope_; }
@@ -292,6 +294,7 @@ private:
   const Scope *builtinsScope_{nullptr}; // module __Fortran_builtins
   Scope *ppcBuiltinTypesScope_{nullptr}; // module __Fortran_PPC_types
   std::optional<const Scope *> cudaBuiltinsScope_; // module __CUDA_builtins
+  std::optional<const Scope *> cudaDeviceScope_; // module cudadevice
   const Scope *ppcBuiltinsScope_{nullptr}; // module __ppc_intrinsics
   std::list<parser::Program> modFileParseTrees_;
   std::unique_ptr<CommonBlockMap> commonBlockMap_;
diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp
index 96ab90239263..2830d5f0be6e 100644
--- a/flang/lib/Semantics/check-cuda.cpp
+++ b/flang/lib/Semantics/check-cuda.cpp
@@ -82,6 +82,11 @@ struct DeviceExprChecker
           }
         }
       }
+      if (sym->owner().IsModule() &&
+          sym->owner().parent().IsIntrinsicModules() &&
+          DEREF(sym->owner().symbol()).name() == "__cuda_device_builtins") {
+        return {};
+      }
     } else if (x.GetSpecificIntrinsic()) {
       // TODO(CUDA): Check for unsupported intrinsics here
       return {};
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index e2875081b732..7cb0c6af937c 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -3797,6 +3797,19 @@ bool SubprogramVisitor::Pre(const parser::PrefixSpec::Attributes &attrs) {
         subp->set_cudaSubprogramAttrs(attr);
       }
     }
+    if (auto attrs{subp->cudaSubprogramAttrs()}) {
+      if (*attrs == common::CUDASubprogramAttrs::Global ||
+          *attrs == common::CUDASubprogramAttrs::Device) {
+        // Implicitly USE the cudadevice module by copying its symbols in the
+        // current scope.
+        const Scope &scope{context().GetCUDADeviceScope()};
+        for (auto sym : scope.GetSymbols()) {
+          if (!currScope().FindSymbol(sym->name())) {
+            currScope().CopySymbol(sym);
+          }
+        }
+      }
+    }
   }
   return false;
 }
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index 6ccd915c4dcb..d51cc62d804e 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -543,6 +543,14 @@ const Scope &SemanticsContext::GetCUDABuiltinsScope() {
   return **cudaBuiltinsScope_;
 }
 
+const Scope &SemanticsContext::GetCUDADeviceScope() {
+  if (!cudaDeviceScope_) {
+    cudaDeviceScope_ = GetBuiltinModule("cudadevice");
+    CHECK(cudaDeviceScope_.value() != nullptr);
+  }
+  return **cudaDeviceScope_;
+}
+
 void SemanticsContext::UsePPCBuiltinsModule() {
   if (ppcBuiltinsScope_ == nullptr) {
     ppcBuiltinsScope_ = GetBuiltinModule("__ppc_intrinsics");
diff --git a/flang/module/__cuda_device_builtins.f90 b/flang/module/__cuda_device_builtins.f90
new file mode 100644
index 000000000000..738dc97242f2
--- /dev/null
+++ b/flang/module/__cuda_device_builtins.f90
@@ -0,0 +1,74 @@
+!===-- module/__cuda_device_builtins.f90 -----------------------------------===!
+!
+! Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+! See https://llvm.org/LICENSE.txt for license information.
+! SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+!
+!===------------------------------------------------------------------------===!
+
+! CUDA Fortran procedures available in device subprogram
+
+module __CUDA_device_builtins  
+
+  implicit none
+
+  ! Set PRIVATE by default to explicitly only export what is meant
+  ! to be exported by this MODULE.
+  private
+
+  ! Synchronization Functions
+
+  interface
+    subroutine __cuda_device_builtins_syncthreads()
+    end subroutine
+  end interface
+  public :: __cuda_device_builtins_syncthreads
+
+  interface
+    integer function __cuda_device_builtins_syncthreads_and(value)
+      integer :: value
+    end function
+  end interface
+  public :: __cuda_device_builtins_syncthreads_and
+
+  interface
+    integer function __cuda_device_builtins_syncthreads_count(value)
+      integer :: value
+    end function
+  end interface
+  public :: __cuda_device_builtins_syncthreads_count
+
+  interface
+    integer function __cuda_device_builtins_syncthreads_or(int_value)
+    end function
+  end interface
+  public :: __cuda_device_builtins_syncthreads_or
+
+  interface
+    subroutine __cuda_device_builtins_syncwarp(mask)
+      integer :: mask
+    end subroutine
+  end interface
+  public :: __cuda_device_builtins_syncwarp
+
+  ! Memory Fences
+
+  interface
+    subroutine __cuda_device_builtins_threadfence()
+    end subroutine
+  end interface
+  public :: __cuda_device_builtins_threadfence
+
+  interface
+    subroutine __cuda_device_builtins_threadfence_block()
+    end subroutine
+  end interface
+  public :: __cuda_device_builtins_threadfence_block
+
+  interface
+    subroutine __cuda_device_builtins_threadfence_system()
+    end subroutine
+  end interface
+  public :: __cuda_device_builtins_threadfence_system
+
+end module
diff --git a/flang/module/cudadevice.f90 b/flang/module/cudadevice.f90
new file mode 100644
index 000000000000..b635d77ea452
--- /dev/null
+++ b/flang/module/cudadevice.f90
@@ -0,0 +1,21 @@
+!===-- module/cudedevice.f90 -----------------------------------------------===!
+!
+! Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+! See https://llvm.org/LICENSE.txt for license information.
+! SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+!
+!===------------------------------------------------------------------------===!
+
+! CUDA Fortran procedures available in device subprogram
+
+module cudadevice
+  use __cuda_device_builtins, only: &
+    syncthreads => __cuda_device_builtins_syncthreads, &
+    syncthreads_and => __cuda_device_builtins_syncthreads_and, &
+    syncthreads_count => __cuda_device_builtins_syncthreads_count, &
+    syncthreads_or => __cuda_device_builtins_syncthreads_or, &
+    syncwarp => __cuda_device_builtins_syncwarp, &
+    threadfence => __cuda_device_builtins_threadfence, &
+    threadfence_block => __cuda_device_builtins_threadfence_block, &
+    threadfence_system => __cuda_device_builtins_threadfence_system
+end module
diff --git a/flang/test/Semantics/cuf-device-procedures01.cuf b/flang/test/Semantics/cuf-device-procedures01.cuf
new file mode 100644
index 000000000000..e79423e3587a
--- /dev/null
+++ b/flang/test/Semantics/cuf-device-procedures01.cuf
@@ -0,0 +1,35 @@
+! RUN: %flang_fc1 -fdebug-dump-symbols %s | FileCheck %s
+
+! Test CUDA Fortran intrinsic can pass semantic
+
+attributes(global) subroutine devsub()
+  implicit none
+  integer :: ret
+
+  ! 3.6.4. Synchronization Functions
+  call syncthreads()
+  call syncwarp(1)
+  call threadfence()
+  call threadfence_block()
+  call threadfence_system()
+  ret = syncthreads_and(1)
+  ret = syncthreads_count(1)
+  ret = syncthreads_or(1)
+end
+
+! CHECK-LABEL: Subprogram scope: devsub
+! CHECK: syncthreads, EXTERNAL, PUBLIC (Subroutine): Use from __cuda_device_builtins_syncthreads in __cuda_device_builtins
+! CHECK: syncthreads_and, EXTERNAL, PUBLIC (Function): Use from __cuda_device_builtins_syncthreads_and in __cuda_device_builtins
+! CHECK: syncthreads_count, EXTERNAL, PUBLIC (Function): Use from __cuda_device_builtins_syncthreads_count in __cuda_device_builtins
+! CHECK: syncthreads_or, EXTERNAL, PUBLIC (Function): Use from __cuda_device_builtins_syncthreads_or in __cuda_device_builtins
+! CHECK: syncwarp, EXTERNAL, PUBLIC (Subroutine): Use from __cuda_device_builtins_syncwarp in __cuda_device_builtins
+! CHECK: threadfence, EXTERNAL, PUBLIC (Subroutine): Use from __cuda_device_builtins_threadfence in __cuda_device_builtins
+! CHECK: threadfence_block, EXTERNAL, PUBLIC (Subroutine): Use from __cuda_device_builtins_threadfence_block in __cuda_device_builtins
+! CHECK: threadfence_system, EXTERNAL, PUBLIC (Subroutine): Use from __cuda_device_builtins_threadfence_system in __cuda_device_builtins
+
+subroutine host()
+  call syncthreads()
+end subroutine
+
+! CHECK-LABEL: Subprogram scope: host
+! CHECK: syncthreads, EXTERNAL: HostAssoc{{$}}
diff --git a/flang/test/Semantics/cuf-device-procedures02.cuf b/flang/test/Semantics/cuf-device-procedures02.cuf
new file mode 100644
index 000000000000..ea6a094ed5c3
--- /dev/null
+++ b/flang/test/Semantics/cuf-device-procedures02.cuf
@@ -0,0 +1,17 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+
+module dev
+  integer, device :: syncthreads
+
+contains
+
+  attributes(device) subroutine sub1()
+    syncthreads = 1 ! syncthreads not overwritten by cudadevice
+  end subroutine
+
+  attributes(global) subroutine sub2()
+!ERROR: 'threadfence' is use-associated from module '__cuda_device_builtins' and cannot be re-declared
+    integer :: threadfence
+  end subroutine
+end module
+
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
index 64815a1f5da6..0222654c8e5d 100644
--- a/flang/tools/f18/CMakeLists.txt
+++ b/flang/tools/f18/CMakeLists.txt
@@ -12,6 +12,8 @@ set(MODULES
   "__ppc_intrinsics"
   "mma"
   "__cuda_builtins"
+  "__cuda_device_builtins"
+  "cudadevice"
   "ieee_arithmetic"
   "ieee_exceptions"
   "ieee_features"
@@ -31,6 +33,8 @@ if (NOT CMAKE_CROSSCOMPILING)
     elseif(${filename} STREQUAL "__ppc_intrinsics" OR
            ${filename} STREQUAL "mma")
       set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
+    elseif(${filename} STREQUAL "cudadevice")
+      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device_builtins.mod)
     else()
       set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
       if(NOT ${filename} STREQUAL "__fortran_type_info")
-- 
GitLab


From 666c686d5917be3f6260036ae536d9b44855ac34 Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Fri, 10 May 2024 14:20:09 -0700
Subject: [PATCH 0453/1206] [bazel] Codesign debugserver on macOS (#91789)

This tool doesn't work unless it's signed with the entitlements used
here. We should probably consider using the
macos_command_line_application rule from rules_apple which manages this
more flexibly for us, but for now this works. This uses apple_genrule as
opposed to genrule since the former encodes the Xcode environment info
into the action so it is correctly invalidated if that changes.
---
 utils/bazel/WORKSPACE                         | 13 +++++++
 .../llvm-project-overlay/lldb/BUILD.bazel     | 38 ++++++++++++-------
 2 files changed, 38 insertions(+), 13 deletions(-)

diff --git a/utils/bazel/WORKSPACE b/utils/bazel/WORKSPACE
index f4ae2c7ce456..298b64fd5629 100644
--- a/utils/bazel/WORKSPACE
+++ b/utils/bazel/WORKSPACE
@@ -56,6 +56,19 @@ maybe(
     name = "vulkan_sdk",
 )
 
+http_archive(
+    name = "build_bazel_apple_support",
+    sha256 = "c4bb2b7367c484382300aee75be598b92f847896fb31bbd22f3a2346adf66a80",
+    url = "https://github.com/bazelbuild/apple_support/releases/download/1.15.1/apple_support.1.15.1.tar.gz",
+)
+
+load(
+    "@build_bazel_apple_support//lib:repositories.bzl",
+    "apple_support_dependencies",
+)
+
+apple_support_dependencies()
+
 # llvm libc math tests reply on `mpfr`.
 # The availability of `mpfr` is controlled by a flag and can be either `disable`, `system` or `external`.
 # Continuous integration uses `system` to speed up the build process (see .bazelrc).
diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index b3a413c401cd..c6fc4e08aa72 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -5,6 +5,7 @@
 load("@bazel_skylib//lib:selects.bzl", "selects")
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
+load("@build_bazel_apple_support//rules:apple_genrule.bzl", "apple_genrule")
 load("//:vars.bzl", "LLVM_VERSION_MAJOR", "LLVM_VERSION_MINOR", "LLVM_VERSION_PATCH", "LLVM_VERSION_SUFFIX", "PACKAGE_VERSION")
 load("//lldb/source/Plugins:plugin_config.bzl", "DEFAULT_PLUGINS", "DEFAULT_SCRIPT_PLUGINS", "OBJCPP_COPTS")
 load("//mlir:tblgen.bzl", "gentbl_cc_library", "td_library")
@@ -800,24 +801,20 @@ gentbl_cc_library(
     deps = ["//llvm:OptParserTdFiles"],
 )
 
+alias(
+    name = "gdb-server",
+    actual = select({
+        "@platforms//os:macos": ":debugserver",
+        "//conditions:default": ":lldb-server",
+    }),
+)
+
 cc_binary(
     name = "lldb",
     srcs = glob([
         "tools/driver/*.cpp",
         "tools/driver/*.h",
     ]),
-    data = [
-        ":lldb-argdumper",
-    ] + select({
-        "@platforms//os:macos": [
-            ":debugserver",
-            ":lldb-server",
-        ],
-        "@platforms//os:linux": [
-            ":lldb-server",
-        ],
-        "//conditions:default": [],
-    }),
     deps = [
         ":APIHeaders",
         ":Host",
@@ -896,7 +893,7 @@ expand_template(
 )
 
 cc_binary(
-    name = "debugserver",
+    name = "debugserver_unsigned",
     srcs = [
         "tools/debugserver/source/debugserver.cpp",
         ":debugserver_version_gen",
@@ -914,6 +911,21 @@ cc_binary(
     ],
 )
 
+apple_genrule(
+    name = "debugserver_signed",
+    srcs = [":debugserver_unsigned"],
+    outs = ["debugserver"],
+    cmd = "cp $(SRCS) $(OUTS) && xcrun codesign -f -s - --entitlements $(location tools/debugserver/resources/debugserver-macosx-entitlements.plist) $(OUTS)",
+    tags = ["nobuildkite"],
+    target_compatible_with = select({
+        "@platforms//os:macos": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    tools = [
+        "tools/debugserver/resources/debugserver-macosx-entitlements.plist",
+    ],
+)
+
 cc_binary(
     name = "lldb-argdumper",
     srcs = glob(["tools/argdumper/*.cpp"]),
-- 
GitLab


From aa0776de464984e78ae1cc329bf541e9dd43631f Mon Sep 17 00:00:00 2001
From: Reid Kleckner <rnk@google.com>
Date: Fri, 10 May 2024 21:20:15 +0000
Subject: [PATCH 0454/1206] Revert "[Pipelines] Do not run CoroSplit and
 CoroCleanup in LTO pre-link pipeline (#90310)" and related patches

This change is incorrect when thinlto and asan are enabled, and this can
be observed by adding `-fsanitize=address` to the provided
coro-elide-thinlto.cpp test. It results in the error "Coroutines cannot
handle non static allocas yet", and ASan introduces a dynamic alloca.

In other words, we must preserve the invariant that CoroSplit runs
before ASan. If we move CoroSplit to the post post-link compile stage,
ASan has to be moved to the post-link compile stage first.  It would
also be correct to make CoroSplit handle dynamic allocas so the pass
ordering doesn't matter, but sanitizer instrumentation really ought to
be last, after coroutine splitting.

This reverts commit bafc5f42c0132171287d7cba7f5c14459be1f7b7.
This reverts commit b1b1bfa7bea0ce489b5ea9134e17a43c695df5ec.
This reverts commit 0232b77e145577ab78e3ed1fdbb7eacc5a7381ab.
This reverts commit fb2d3056618e3d03ba9a695627c7b002458e59f0.
This reverts commit 1cb33713910501c6352d0eb2a15b7a15e6e18695.
This reverts commit cd68d7b3c0ebf6da5e235cfabd5e6381737eb7fe.
---
 .../CodeGenCoroutines/coro-elide-thinlto.cpp  | 78 -------------------
 llvm/lib/Passes/PassBuilderPipelines.cpp      | 11 +--
 .../Other/new-pm-thinlto-prelink-defaults.ll  |  2 +
 .../new-pm-thinlto-prelink-pgo-defaults.ll    |  2 +
 ...w-pm-thinlto-prelink-samplepgo-defaults.ll |  2 +
 5 files changed, 10 insertions(+), 85 deletions(-)
 delete mode 100644 clang/test/CodeGenCoroutines/coro-elide-thinlto.cpp

diff --git a/clang/test/CodeGenCoroutines/coro-elide-thinlto.cpp b/clang/test/CodeGenCoroutines/coro-elide-thinlto.cpp
deleted file mode 100644
index 5b2d01465784..000000000000
--- a/clang/test/CodeGenCoroutines/coro-elide-thinlto.cpp
+++ /dev/null
@@ -1,78 +0,0 @@
-// REQUIRES: x86_64-linux
-// This tests that the coroutine elide optimization could happen succesfully with ThinLTO.
-// This test is adapted from coro-elide.cpp and splits functions into two files.
-//
-// RUN: split-file %s %t
-// RUN: %clang --target=x86_64-linux -std=c++20 -O2 -flto=thin -I %S -c %t/coro-elide-callee.cpp -o %t/coro-elide-callee.o
-// RUN: %clang --target=x86_64-linux -std=c++20 -O2 -flto=thin -I %S -c %t/coro-elide-caller.cpp -o %t/coro-elide-caller.o
-// RUN: llvm-lto -thinlto %t/coro-elide-callee.o %t/coro-elide-caller.o -o %t/summary
-// RUN: %clang_cc1 -triple x86_64-unknown-linux -O2 -x ir %t/coro-elide-caller.o -fthinlto-index=%t/summary.thinlto.bc -emit-llvm -o - | FileCheck %s
-
-//--- coro-elide-task.h
-#pragma once
-#include "Inputs/coroutine.h"
-
-struct Task {
-  struct promise_type {
-    struct FinalAwaiter {
-      bool await_ready() const noexcept { return false; }
-      template <typename PromiseType>
-      std::coroutine_handle<> await_suspend(std::coroutine_handle<PromiseType> h) noexcept {
-        if (!h)
-          return std::noop_coroutine();
-        return h.promise().continuation;
-      }
-      void await_resume() noexcept {}
-    };
-    Task get_return_object() noexcept {
-      return std::coroutine_handle<promise_type>::from_promise(*this);
-    }
-    std::suspend_always initial_suspend() noexcept { return {}; }
-    FinalAwaiter final_suspend() noexcept { return {}; }
-    void unhandled_exception() noexcept {}
-    void return_value(int x) noexcept {
-      _value = x;
-    }
-    std::coroutine_handle<> continuation;
-    int _value;
-  };
-
-  Task(std::coroutine_handle<promise_type> handle) : handle(handle) {}
-  ~Task() {
-    if (handle)
-      handle.destroy();
-  }
-
-  struct Awaiter {
-    bool await_ready() const noexcept { return false; }
-    void await_suspend(std::coroutine_handle<void> continuation) noexcept {}
-    int await_resume() noexcept {
-      return 43;
-    }
-  };
-
-  auto operator co_await() {
-    return Awaiter{};
-  }
-
-private:
-  std::coroutine_handle<promise_type> handle;
-};
-
-//--- coro-elide-callee.cpp
-#include "coro-elide-task.h"
-Task task0() {
-  co_return 43;
-}
-
-//--- coro-elide-caller.cpp
-#include "coro-elide-task.h"
-
-Task task0();
-
-Task task1() {
-  co_return co_await task0();
-}
-
-// CHECK-LABEL: define{{.*}} void @_Z5task1v.resume
-// CHECK-NOT: {{.*}}_Znwm
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 1d7f0510450c..72f273972f2f 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -968,8 +968,7 @@ PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
   MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
       RequireAnalysisPass<ShouldNotRunFunctionPassesAnalysis, Function>()));
 
-  if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink)
-    MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
+  MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
 
   // Make sure we don't affect potential future NoRerun CGSCC adaptors.
   MIWP.addLateModulePass(createModuleToFunctionPassAdaptor(
@@ -1011,9 +1010,8 @@ PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level,
       buildFunctionSimplificationPipeline(Level, Phase),
       PTO.EagerlyInvalidateAnalyses));
 
-  if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink)
-    MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
-        CoroSplitPass(Level != OptimizationLevel::O0)));
+  MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+      CoroSplitPass(Level != OptimizationLevel::O0)));
 
   return MPM;
 }
@@ -1190,8 +1188,7 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   // and argument promotion.
   MPM.addPass(DeadArgumentEliminationPass());
 
-  if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink)
-    MPM.addPass(CoroCleanupPass());
+  MPM.addPass(CoroCleanupPass());
 
   // Optimize globals now that functions are fully simplified.
   MPM.addPass(GlobalOptPass());
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll
index e2fd74306f80..6486639e07b4 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll
@@ -183,10 +183,12 @@
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Running analysis: ShouldNotRunFunctionPassesAnalysis
+; CHECK-O-NEXT: Running pass: CoroSplitPass
 ; CHECK-O-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Invalidating analysis: ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Invalidating analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
+; CHECK-O-NEXT: Running pass: CoroCleanupPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-O-NEXT: Running pass: GlobalDCEPass
 ; CHECK-EXT: Running pass: {{.*}}::Bye
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
index 13a63bbe4d9c..09f9f0f48bad 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
@@ -182,10 +182,12 @@
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Running analysis: ShouldNotRunFunctionPassesAnalysis
+; CHECK-O-NEXT: Running pass: CoroSplitPass
 ; CHECK-O-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Invalidating analysis: ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Invalidating analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
+; CHECK-O-NEXT: Running pass: CoroCleanupPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis on bar
 ; CHECK-O-NEXT: Running pass: GlobalDCEPass
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
index 3130da86fa99..47bdbfd2d357 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
@@ -147,10 +147,12 @@
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Running analysis: ShouldNotRunFunctionPassesAnalysis
+; CHECK-O-NEXT: Running pass: CoroSplitPass
 ; CHECK-O-NEXT: Running pass: InvalidateAnalysisPass<{{.*}}ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Invalidating analysis: ShouldNotRunFunctionPassesAnalysis
 ; CHECK-O-NEXT: Invalidating analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
+; CHECK-O-NEXT: Running pass: CoroCleanupPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
 ; CHECK-O-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O-NEXT: Running pass: AnnotationRemarksPass on foo
-- 
GitLab


From 2d5634a4b39d8e5497b6a67caa54049b3cfade8e Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Fri, 10 May 2024 18:32:34 -0300
Subject: [PATCH 0455/1206] [clang] Revert default behavior change of P0522R0
 implementation (#91811)

This partially reverts b86e0992bfa6c58be077d82d824016f590ac5d90.

Just the default is changed back, on the Driver side.
No Frontend changes.
The positive spelling of the flag is undeprecated.

No documentation changes or changelog entries because we plan to revert
this revert as soon as https://github.com/llvm/llvm-project/issues/62529
is fixed.
---
 clang/lib/Driver/ToolChains/Clang.cpp              | 14 +++++++-------
 .../Driver/frelaxed-template-template-args.cpp     |  6 ++++--
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 42feb1650574..f0cc018b6668 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -7249,15 +7249,15 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.addOptOutFlag(CmdArgs, options::OPT_fassume_unique_vtables,
                      options::OPT_fno_assume_unique_vtables);
 
-  // -frelaxed-template-template-args is deprecated.
-  if (Arg *A =
-          Args.getLastArg(options::OPT_frelaxed_template_template_args,
-                          options::OPT_fno_relaxed_template_template_args)) {
+  // -fno-relaxed-template-template-args is deprecated.
+  if (Arg *A = Args.getLastArg(options::OPT_frelaxed_template_template_args,
+                               options::OPT_fno_relaxed_template_template_args);
+      A &&
+      A->getOption().matches(options::OPT_fno_relaxed_template_template_args))
     D.Diag(diag::warn_drv_deprecated_arg)
         << A->getAsString(Args) << /*hasReplacement=*/false;
-    if (A->getOption().matches(options::OPT_fno_relaxed_template_template_args))
-      CmdArgs.push_back("-fno-relaxed-template-template-args");
-  }
+  else
+    CmdArgs.push_back("-fno-relaxed-template-template-args");
 
   // -fsized-deallocation is off by default, as it is an ABI-breaking change for
   // most platforms.
diff --git a/clang/test/Driver/frelaxed-template-template-args.cpp b/clang/test/Driver/frelaxed-template-template-args.cpp
index dd6265ba8375..136c360276a1 100644
--- a/clang/test/Driver/frelaxed-template-template-args.cpp
+++ b/clang/test/Driver/frelaxed-template-template-args.cpp
@@ -1,5 +1,7 @@
-// RUN: %clang -fsyntax-only -frelaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-ON %s
+// RUN: %clang -fsyntax-only -### %s 2>&1 | FileCheck --check-prefix=CHECK-DEF %s
+// RUN: %clang -fsyntax-only -frelaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-ON --allow-empty %s
 // RUN: %clang -fsyntax-only -fno-relaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-OFF %s
 
-// CHECK-ON:  warning: argument '-frelaxed-template-template-args' is deprecated [-Wdeprecated]
+// CHECK-DEF: "-cc1"{{.*}} "-fno-relaxed-template-template-args"
+// CHECK-ON-NOT: warning: argument '-frelaxed-template-template-args' is deprecated [-Wdeprecated]
 // CHECK-OFF: warning: argument '-fno-relaxed-template-template-args' is deprecated [-Wdeprecated]
-- 
GitLab


From 5d18d575d8d304e4336483d8be7394d0f4694cb1 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Fri, 10 May 2024 14:33:47 -0700
Subject: [PATCH 0456/1206] [AMDGPU] Make fneg/fabs/copysign legal for bf16
 (#91676)

These are just bit operations, exactly the same as with f16.
---
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp |   4 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |   7 +-
 llvm/lib/Target/AMDGPU/SIInstructions.td      |  16 +-
 llvm/test/CodeGen/AMDGPU/bf16.ll              | 378 +++++++-----------
 4 files changed, 169 insertions(+), 236 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 1e9132bcfaf9..d35a022ad680 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -946,14 +946,14 @@ bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
 
   // Packed operations do not have a fabs modifier.
   return VT == MVT::f32 || VT == MVT::f64 ||
-         (Subtarget->has16BitInsts() && VT == MVT::f16);
+         (Subtarget->has16BitInsts() && (VT == MVT::f16 || VT == MVT::bf16));
 }
 
 bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
   assert(VT.isFloatingPoint());
   // Report this based on the end legalized type.
   VT = VT.getScalarType();
-  return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16;
+  return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
 }
 
 bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 33bdd6195a04..0a3a56e9b3a0 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -225,10 +225,9 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::SELECT, MVT::bf16, Promote);
     AddPromotedToType(ISD::SELECT, MVT::bf16, MVT::i16);
 
-    // TODO: Could make these legal
-    setOperationAction(ISD::FABS, MVT::bf16, Expand);
-    setOperationAction(ISD::FNEG, MVT::bf16, Expand);
-    setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Expand);
+    setOperationAction(ISD::FABS, MVT::bf16, Legal);
+    setOperationAction(ISD::FNEG, MVT::bf16, Legal);
+    setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Legal);
 
     // We only need to custom lower because we can't specify an action for bf16
     // sources.
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index cca8d96f29c0..7592f8da1b4f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1908,20 +1908,22 @@ def : GCNPat <
   (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80000000)))
 >;
 
+foreach fp16vt = [f16, bf16] in {
 def : GCNPat <
-  (UniformUnaryFrag<fneg> (f16 SReg_32:$src)),
+  (UniformUnaryFrag<fneg> (fp16vt SReg_32:$src)),
   (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00008000)))
 >;
 
 def : GCNPat <
-  (UniformUnaryFrag<fabs> (f16 SReg_32:$src)),
+  (UniformUnaryFrag<fabs> (fp16vt SReg_32:$src)),
   (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00007fff)))
 >;
 
 def : GCNPat <
-  (UniformUnaryFrag<fneg> (fabs (f16 SReg_32:$src))),
+  (UniformUnaryFrag<fneg> (fabs (fp16vt SReg_32:$src))),
   (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00008000))) // Set sign bit
 >;
+} // End foreach fp16vt = ...
 
 def : GCNPat <
   (UniformUnaryFrag<fneg> (v2f16 SReg_32:$src)),
@@ -2030,20 +2032,22 @@ def : GCNPat <
   (V_XOR_B32_e64 (S_MOV_B32 (i32 0x80000000)), VGPR_32:$src)
 >;
 
+foreach fp16vt = [f16, bf16] in {
 def : GCNPat <
-  (fabs (f16 VGPR_32:$src)),
+  (fabs (fp16vt VGPR_32:$src)),
   (V_AND_B32_e64 (S_MOV_B32 (i32 0x00007fff)), VGPR_32:$src)
 >;
 
 def : GCNPat <
-  (fneg (f16 VGPR_32:$src)),
+  (fneg (fp16vt VGPR_32:$src)),
   (V_XOR_B32_e64 (S_MOV_B32 (i32 0x00008000)), VGPR_32:$src)
 >;
 
 def : GCNPat <
-  (fneg (fabs (f16 VGPR_32:$src))),
+  (fneg (fabs (fp16vt VGPR_32:$src))),
   (V_OR_B32_e64 (S_MOV_B32 (i32 0x00008000)), VGPR_32:$src) // Set sign bit
 >;
+} // End foreach fp16vt = ...
 
 def : GCNPat <
   (fneg (v2f16 VGPR_32:$src)),
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 4c9c34de7194..a86a3f6f279d 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -17008,33 +17008,27 @@ define amdgpu_ps i32 @s_fabs_bf16(bfloat inreg %a) {
 ;
 ; GFX8-LABEL: s_fabs_bf16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_mov_b32_e32 v0, 0x7fff
-; GFX8-NEXT:    v_and_b32_e32 v0, s0, v0
-; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    s_and_b32 s0, s0, 0x7fff
+; GFX8-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_fabs_bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x7fff
-; GFX9-NEXT:    v_and_b32_e32 v0, s0, v0
-; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    s_and_b32 s0, s0, 0x7fff
+; GFX9-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_fabs_bf16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_and_b32_e64 v0, 0x7fff, s0
-; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    s_and_b32 s0, s0, 0x7fff
+; GFX10-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_fabs_bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_and_b32_e64 v0, 0x7fff, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    s_and_b32 s0, s0, 0x7fff
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX11-NEXT:    ; return to shader part epilog
   %op = call bfloat @llvm.fabs.bf16(bfloat %a)
   %cast = bitcast bfloat %op to i16
@@ -17059,25 +17053,25 @@ define bfloat @v_fneg_bf16(bfloat %a) {
 ; GFX8-LABEL: v_fneg_bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_xor_b32_e32 v0, 0xffff8000, v0
+; GFX8-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_fneg_bf16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_xor_b32_e32 v0, 0xffff8000, v0
+; GFX9-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fneg_bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_xor_b32_e32 v0, 0xffff8000, v0
+; GFX10-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_fneg_bf16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_xor_b32_e32 v0, 0xffff8000, v0
+; GFX11-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %op = fneg bfloat %a
   ret bfloat %op
@@ -17089,49 +17083,41 @@ declare i32 @llvm.amdgcn.readfirstlane(i32)
 define amdgpu_ps i32 @s_fneg_bf16(bfloat inreg %a) {
 ; GCN-LABEL: s_fneg_bf16:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    v_mul_f32_e64 v0, 1.0, s0
+; GCN-NEXT:    v_mul_f32_e64 v0, -1.0, s0
 ; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GCN-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
 ; GCN-NEXT:    v_readfirstlane_b32 s0, v0
 ; GCN-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: s_fneg_bf16:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    v_mul_f32_e64 v0, 1.0, s0
+; GFX7-NEXT:    v_mul_f32_e64 v0, -1.0, s0
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
 ; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-LABEL: s_fneg_bf16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_mov_b32_e32 v0, 0xffff8000
-; GFX8-NEXT:    v_xor_b32_e32 v0, s0, v0
-; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    s_xor_b32 s0, s0, 0x8000
+; GFX8-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_fneg_bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff8000
-; GFX9-NEXT:    v_xor_b32_e32 v0, s0, v0
-; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    s_xor_b32 s0, s0, 0x8000
+; GFX9-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_fneg_bf16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_xor_b32_e64 v0, 0xffff8000, s0
-; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    s_xor_b32 s0, s0, 0x8000
+; GFX10-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_fneg_bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_xor_b32_e64 v0, 0xffff8000, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    s_xor_b32 s0, s0, 0x8000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX11-NEXT:    ; return to shader part epilog
   %op = fneg bfloat %a
   %cast = bitcast bfloat %op to i16
@@ -17166,25 +17152,25 @@ define bfloat @v_fneg_fabs_bf16(bfloat %a) {
 ; GFX8-LABEL: v_fneg_fabs_bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_or_b32_e32 v0, 0xffff8000, v0
+; GFX8-NEXT:    v_or_b32_e32 v0, 0x8000, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_fneg_fabs_bf16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_or_b32_e32 v0, 0xffff8000, v0
+; GFX9-NEXT:    v_or_b32_e32 v0, 0x8000, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fneg_fabs_bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_or_b32_e32 v0, 0xffff8000, v0
+; GFX10-NEXT:    v_or_b32_e32 v0, 0x8000, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_fneg_fabs_bf16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_or_b32_e32 v0, 0xffff8000, v0
+; GFX11-NEXT:    v_or_b32_e32 v0, 0x8000, v0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %fabs = call bfloat @llvm.fabs.bf16(bfloat %a)
   %op = fneg bfloat %fabs
@@ -17196,48 +17182,48 @@ define amdgpu_ps i32 @s_fneg_fabs_bf16(bfloat inreg %a) {
 ; GCN-LABEL: s_fneg_fabs_bf16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    v_mul_f32_e64 v0, 1.0, s0
-; GCN-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GCN-NEXT:    v_or_b32_e32 v0, 0x8000, v0
 ; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    s_and_b32 s0, s0, 0xffff0000
+; GCN-NEXT:    s_bitset0_b32 s0, 31
+; GCN-NEXT:    s_and_b32 s0, s0, 0xffff0000
+; GCN-NEXT:    s_xor_b32 s0, s0, 0x80000000
+; GCN-NEXT:    s_lshr_b32 s0, s0, 16
 ; GCN-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: s_fneg_fabs_bf16:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    v_mul_f32_e64 v0, 1.0, s0
-; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT:    v_or_b32_e32 v0, 0x8000, v0
 ; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    s_and_b32 s0, s0, 0xffff0000
+; GFX7-NEXT:    s_bitset0_b32 s0, 31
+; GFX7-NEXT:    s_and_b32 s0, s0, 0xffff0000
+; GFX7-NEXT:    s_xor_b32 s0, s0, 0x80000000
+; GFX7-NEXT:    s_lshr_b32 s0, s0, 16
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-LABEL: s_fneg_fabs_bf16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_mov_b32_e32 v0, 0xffff8000
-; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
-; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    s_bitset1_b32 s0, 15
+; GFX8-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_fneg_fabs_bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff8000
-; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
-; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    s_bitset1_b32 s0, 15
+; GFX9-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_fneg_fabs_bf16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_or_b32_e64 v0, 0xffff8000, s0
-; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    s_bitset1_b32 s0, 15
+; GFX10-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_fneg_fabs_bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_or_b32_e64 v0, 0xffff8000, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    s_bitset1_b32 s0, 15
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_b32 s0, 0xffff, s0
 ; GFX11-NEXT:    ; return to shader part epilog
   %fabs = call bfloat @llvm.fabs.bf16(bfloat %a)
   %op = fneg bfloat %fabs
@@ -27280,34 +27266,27 @@ define bfloat @v_copysign_bf16_bf16(bfloat %mag, bfloat %sign) {
 ; GFX8-LABEL: v_copysign_bf16_bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff8000, v1
-; GFX8-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_copysign_bf16_bf16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff8000, v1
-; GFX9-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_copysign_bf16_bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff8000, v1
-; GFX10-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_copysign_bf16_bf16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v1, 0xffff8000, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
   ret bfloat %op
@@ -27339,36 +27318,29 @@ define bfloat @v_copysign_bf16_s_bf16(bfloat %mag, bfloat inreg %sign) {
 ; GFX8-LABEL: v_copysign_bf16_s_bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v1, 0xffff8000
-; GFX8-NEXT:    v_and_b32_e32 v1, s4, v1
-; GFX8-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_bfi_b32 v0, s5, v0, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_copysign_bf16_s_bf16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff8000
-; GFX9-NEXT:    v_and_b32_e32 v1, s4, v1
-; GFX9-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_bfi_b32 v0, s5, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_copysign_bf16_s_bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_and_b32_e64 v1, 0xffff8000, s4
-; GFX10-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, v0, s4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_copysign_bf16_s_bf16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_and_b32_e64 v1, 0xffff8000, s0
-; GFX11-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, v0, s0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
   ret bfloat %op
@@ -27400,36 +27372,29 @@ define bfloat @v_copysign_s_bf16_bf16(bfloat inreg %mag, bfloat %sign) {
 ; GFX8-LABEL: v_copysign_s_bf16_bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v1, 0x7fff
-; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff8000, v0
-; GFX8-NEXT:    v_and_b32_e32 v1, s4, v1
-; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v1, s4
+; GFX8-NEXT:    v_bfi_b32 v0, s5, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_copysign_s_bf16_bf16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x7fff
-; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff8000, v0
-; GFX9-NEXT:    v_and_b32_e32 v1, s4, v1
-; GFX9-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_bfi_b32 v0, s5, v1, v0
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_copysign_s_bf16_bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff8000, v0
-; GFX10-NEXT:    v_and_b32_e64 v1, 0x7fff, s4
-; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, s4, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_copysign_s_bf16_bf16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff8000, v0
-; GFX11-NEXT:    v_and_b32_e64 v1, 0x7fff, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
   ret bfloat %op
@@ -27461,35 +27426,32 @@ define bfloat @v_copysign_bf16_f32(bfloat %mag, float %sign.f32) {
 ; GFX8-LABEL: v_copysign_bf16_f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX8-NEXT:    v_and_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_copysign_bf16_f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX9-NEXT:    v_and_b32_e32 v1, 0x80000000, v1
-; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_copysign_bf16_f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX10-NEXT:    v_and_b32_e32 v1, 0x80000000, v1
-; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_copysign_bf16_f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v1, 0x80000000, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sign = fptrunc float %sign.f32 to bfloat
   %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
@@ -27522,35 +27484,32 @@ define bfloat @v_copysign_bf16_f64(bfloat %mag, double %sign.f64) {
 ; GFX8-LABEL: v_copysign_bf16_f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX8-NEXT:    v_and_b32_e32 v1, 0x80000000, v2
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_copysign_bf16_f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX9-NEXT:    v_and_b32_e32 v1, 0x80000000, v2
-; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_copysign_bf16_f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX10-NEXT:    v_and_b32_e32 v1, 0x80000000, v2
-; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_copysign_bf16_f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v1, 0x80000000, v2
-; GFX11-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sign = fptrunc double %sign.f64 to bfloat
   %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
@@ -27583,34 +27542,27 @@ define bfloat @v_copysign_bf16_f16(bfloat %mag, half %sign.f16) {
 ; GFX8-LABEL: v_copysign_bf16_f16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff8000, v1
-; GFX8-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_copysign_bf16_f16:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff8000, v1
-; GFX9-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_bfi_b32 v0, s4, v0, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_copysign_bf16_f16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff8000, v1
-; GFX10-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_copysign_bf16_f16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v1, 0xffff8000, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sign = bitcast half %sign.f16 to bfloat
   %op = call bfloat @llvm.copysign.bf16(bfloat %mag, bfloat %sign)
@@ -27640,41 +27592,37 @@ define amdgpu_ps i32 @s_copysign_bf16_bf16(bfloat inreg %mag, bfloat inreg %sign
 ;
 ; GFX8-LABEL: s_copysign_bf16_bf16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_mov_b32_e32 v0, 0xffff8000
-; GFX8-NEXT:    v_mov_b32_e32 v1, 0x7fff
-; GFX8-NEXT:    v_and_b32_e32 v0, s1, v0
-; GFX8-NEXT:    v_and_b32_e32 v1, s0, v1
-; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_bfi_b32 v0, s2, v0, v1
 ; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_copysign_bf16_bf16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff8000
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x7fff
-; GFX9-NEXT:    v_and_b32_e32 v0, s1, v0
-; GFX9-NEXT:    v_and_b32_e32 v1, s0, v1
-; GFX9-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_bfi_b32 v0, s2, v0, v1
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_copysign_bf16_bf16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_and_b32_e64 v0, 0xffff8000, s1
-; GFX10-NEXT:    v_and_b32_e64 v1, 0x7fff, s0
-; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_copysign_bf16_bf16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_and_b32_e64 v0, 0xffff8000, s1
-; GFX11-NEXT:    v_and_b32_e64 v1, 0x7fff, s0
+; GFX11-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
@@ -27709,46 +27657,39 @@ define amdgpu_ps i32 @s_copysign_bf16_f32(bfloat inreg %mag, float inreg %sign.f
 ;
 ; GFX8-LABEL: s_copysign_bf16_f32:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_mov_b32_e32 v0, 0x7fff
-; GFX8-NEXT:    v_and_b32_e32 v0, s0, v0
-; GFX8-NEXT:    s_and_b32 s0, s1, 0x80000000
-; GFX8-NEXT:    s_lshr_b32 s0, s0, 16
-; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 16, s1
+; GFX8-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_bfi_b32 v0, s1, v1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_copysign_bf16_f32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x7fff
-; GFX9-NEXT:    v_and_b32_e32 v0, s0, v0
-; GFX9-NEXT:    s_and_b32 s0, s1, 0x80000000
-; GFX9-NEXT:    s_lshr_b32 s0, s0, 16
-; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e64 v0, 16, s1
+; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_bfi_b32 v0, s1, v1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_copysign_bf16_f32:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_and_b32_e64 v0, 0x7fff, s0
-; GFX10-NEXT:    s_and_b32 s0, s1, 0x80000000
-; GFX10-NEXT:    s_lshr_b32 s0, s0, 16
-; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    v_lshrrev_b32_e64 v0, 16, s1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_copysign_bf16_f32:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_and_b32_e64 v0, 0x7fff, s0
-; GFX11-NEXT:    s_and_b32 s0, s1, 0x80000000
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_lshr_b32 s0, s0, 16
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e64 v0, 16, s1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX11-NEXT:    ; return to shader part epilog
   %sign = fptrunc float %sign.f32 to bfloat
@@ -27782,46 +27723,39 @@ define amdgpu_ps i32 @s_copysign_bf16_f64(bfloat inreg %mag, double inreg %sign.
 ;
 ; GFX8-LABEL: s_copysign_bf16_f64:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_mov_b32_e32 v0, 0x7fff
-; GFX8-NEXT:    v_and_b32_e32 v0, s0, v0
-; GFX8-NEXT:    s_and_b32 s0, s2, 0x80000000
-; GFX8-NEXT:    s_lshr_b32 s0, s0, 16
-; GFX8-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 16, s2
+; GFX8-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_bfi_b32 v0, s1, v1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_copysign_bf16_f64:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x7fff
-; GFX9-NEXT:    v_and_b32_e32 v0, s0, v0
-; GFX9-NEXT:    s_and_b32 s0, s2, 0x80000000
-; GFX9-NEXT:    s_lshr_b32 s0, s0, 16
-; GFX9-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT:    v_lshrrev_b32_e64 v0, 16, s2
+; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_bfi_b32 v0, s1, v1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_copysign_bf16_f64:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_and_b32_e64 v0, 0x7fff, s0
-; GFX10-NEXT:    s_and_b32 s0, s2, 0x80000000
-; GFX10-NEXT:    s_lshr_b32 s0, s0, 16
-; GFX10-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX10-NEXT:    v_lshrrev_b32_e64 v0, 16, s2
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_copysign_bf16_f64:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_and_b32_e64 v0, 0x7fff, s0
-; GFX11-NEXT:    s_and_b32 s0, s2, 0x80000000
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_lshr_b32 s0, s0, 16
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    v_or_b32_e32 v0, s0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e64 v0, 16, s2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX11-NEXT:    ; return to shader part epilog
   %sign = fptrunc double %sign.f64 to bfloat
@@ -27855,41 +27789,37 @@ define amdgpu_ps i32 @s_copysign_bf16_f16(bfloat inreg %mag, half inreg %sign.f1
 ;
 ; GFX8-LABEL: s_copysign_bf16_f16:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    v_mov_b32_e32 v0, 0xffff8000
-; GFX8-NEXT:    v_mov_b32_e32 v1, 0x7fff
-; GFX8-NEXT:    v_and_b32_e32 v0, s1, v0
-; GFX8-NEXT:    v_and_b32_e32 v1, s0, v1
-; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_bfi_b32 v0, s2, v0, v1
 ; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX9-LABEL: s_copysign_bf16_f16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffff8000
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x7fff
-; GFX9-NEXT:    v_and_b32_e32 v0, s1, v0
-; GFX9-NEXT:    v_and_b32_e32 v1, s0, v1
-; GFX9-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_bfi_b32 v0, s2, v0, v1
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_copysign_bf16_f16:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_and_b32_e64 v0, 0xffff8000, s1
-; GFX10-NEXT:    v_and_b32_e64 v1, 0x7fff, s0
-; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: s_copysign_bf16_f16:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_and_b32_e64 v0, 0xffff8000, s1
-; GFX11-NEXT:    v_and_b32_e64 v1, 0x7fff, s0
+; GFX11-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fff, s0, v0
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
@@ -33706,7 +33636,7 @@ define bfloat @v_select_fneg_lhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX8-NEXT:    v_xor_b32_e32 v1, 0xffff8000, v1
+; GFX8-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -33715,7 +33645,7 @@ define bfloat @v_select_fneg_lhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_xor_b32_e32 v1, 0xffff8000, v1
+; GFX9-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -33724,7 +33654,7 @@ define bfloat @v_select_fneg_lhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX10-NEXT:    v_xor_b32_e32 v1, 0xffff8000, v1
+; GFX10-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -33733,7 +33663,7 @@ define bfloat @v_select_fneg_lhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX11-NEXT:    v_xor_b32_e32 v1, 0xffff8000, v1
+; GFX11-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc_lo
@@ -33770,7 +33700,7 @@ define bfloat @v_select_fneg_rhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0xffff8000, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0x8000, v2
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
@@ -33779,7 +33709,7 @@ define bfloat @v_select_fneg_rhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_xor_b32_e32 v2, 0xffff8000, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, 0x8000, v2
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -33788,7 +33718,7 @@ define bfloat @v_select_fneg_rhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX10-NEXT:    v_xor_b32_e32 v2, 0xffff8000, v2
+; GFX10-NEXT:    v_xor_b32_e32 v2, 0x8000, v2
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -33797,7 +33727,7 @@ define bfloat @v_select_fneg_rhs_bf16(i1 %cond, bfloat %a, bfloat %b) {
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX11-NEXT:    v_xor_b32_e32 v2, 0xffff8000, v2
+; GFX11-NEXT:    v_xor_b32_e32 v2, 0x8000, v2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc_lo
-- 
GitLab


From fb3f4b013c3acab0ea3cb14c4d29f4e6d9caa33c Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 10 May 2024 16:38:13 -0500
Subject: [PATCH 0457/1206] [libc] Add memory fence utility to the GPU
 utilities (#91756)

Summary:
GPUs like to execute instructions in the background until something
excplitely consumes them. We are working on adding some
microbenchmarking code, which requires flushing the pending memory
operations beforehand. This patch simply adds these utility functions
that will be used in the near future.
---
 libc/src/__support/GPU/amdgpu/utils.h | 5 +++++
 libc/src/__support/GPU/nvptx/utils.h  | 4 ++++
 2 files changed, 9 insertions(+)

diff --git a/libc/src/__support/GPU/amdgpu/utils.h b/libc/src/__support/GPU/amdgpu/utils.h
index 9b520a6bcf38..5f8ad74f6aea 100644
--- a/libc/src/__support/GPU/amdgpu/utils.h
+++ b/libc/src/__support/GPU/amdgpu/utils.h
@@ -140,6 +140,11 @@ LIBC_INLINE uint32_t get_lane_size() {
   __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "workgroup");
 }
 
+/// Waits for all pending memory operations to complete in program order.
+[[clang::convergent]] LIBC_INLINE void memory_fence() {
+  __builtin_amdgcn_fence(__ATOMIC_ACQ_REL, "");
+}
+
 /// Wait for all threads in the wavefront to converge, this is a noop on AMDGPU.
 [[clang::convergent]] LIBC_INLINE void sync_lane(uint64_t) {
   __builtin_amdgcn_wave_barrier();
diff --git a/libc/src/__support/GPU/nvptx/utils.h b/libc/src/__support/GPU/nvptx/utils.h
index 3f19afb83648..88b8ee2e31d3 100644
--- a/libc/src/__support/GPU/nvptx/utils.h
+++ b/libc/src/__support/GPU/nvptx/utils.h
@@ -118,9 +118,13 @@ LIBC_INLINE uint32_t get_lane_size() { return 32; }
   uint32_t mask = static_cast<uint32_t>(lane_mask);
   return __nvvm_vote_ballot_sync(mask, x);
 }
+
 /// Waits for all the threads in the block to converge and issues a fence.
 [[clang::convergent]] LIBC_INLINE void sync_threads() { __syncthreads(); }
 
+/// Waits for all pending memory operations to complete in program order.
+[[clang::convergent]] LIBC_INLINE void memory_fence() { __nvvm_membar_sys(); }
+
 /// Waits for all threads in the warp to reconverge for independent scheduling.
 [[clang::convergent]] LIBC_INLINE void sync_lane(uint64_t mask) {
   __nvvm_bar_warp_sync(static_cast<uint32_t>(mask));
-- 
GitLab


From 7e6879b245f7feaf00367874cbef7db973455553 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 9 May 2024 17:21:51 +0100
Subject: [PATCH 0458/1206] [X86] scalarizeExtractedBinop - reuse existing
 SDLoc. NFC.

---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index be919b7a8922..9fbafba9fea7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -22226,7 +22226,7 @@ SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
 /// Transform a vector binary operation into a scalar binary operation by moving
 /// the math/logic after an extract element of a vector.
 static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG,
-                                       bool LegalOperations) {
+                                       const SDLoc &DL, bool LegalOperations) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue Vec = ExtElt->getOperand(0);
   SDValue Index = ExtElt->getOperand(1);
@@ -22251,7 +22251,6 @@ static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG,
       ISD::isConstantSplatVector(Op1.getNode(), SplatVal)) {
     // extractelt (binop X, C), IndexC --> binop (extractelt X, IndexC), C'
     // extractelt (binop C, X), IndexC --> binop C', (extractelt X, IndexC)
-    SDLoc DL(ExtElt);
     EVT VT = ExtElt->getValueType(0);
     SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index);
     SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index);
@@ -22490,7 +22489,7 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
     }
   }
 
-  if (SDValue BO = scalarizeExtractedBinop(N, DAG, LegalOperations))
+  if (SDValue BO = scalarizeExtractedBinop(N, DAG, DL, LegalOperations))
     return BO;
 
   if (VecVT.isScalableVector())
-- 
GitLab


From 7f3e3785d0be00c727e3d2938b6348748499de99 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 10 May 2024 22:40:03 +0100
Subject: [PATCH 0459/1206] Fix MSVC "result of 32-bit shift implicitly
 converted to 64 bits" warning. NFC.

---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 9fbafba9fea7..a044b6dc4838 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -1095,12 +1095,12 @@ bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc,
         N1.getOperand(0).getOpcode() == ISD::VSCALE &&
         isa<ConstantSDNode>(N1.getOperand(1)))) &&
       N1.getValueType().getFixedSizeInBits() <= 64) {
-    int64_t ScalableOffset =
-        N1.getOpcode() == ISD::VSCALE
-            ? N1.getConstantOperandVal(0)
-            : (N1.getOperand(0).getConstantOperandVal(0) *
-               (N1.getOpcode() == ISD::SHL ? (1 << N1.getConstantOperandVal(1))
-                                           : N1.getConstantOperandVal(1)));
+    int64_t ScalableOffset = N1.getOpcode() == ISD::VSCALE
+                                 ? N1.getConstantOperandVal(0)
+                                 : (N1.getOperand(0).getConstantOperandVal(0) *
+                                    (N1.getOpcode() == ISD::SHL
+                                         ? (1LL << N1.getConstantOperandVal(1))
+                                         : N1.getConstantOperandVal(1)));
     if (Opc == ISD::SUB)
       ScalableOffset = -ScalableOffset;
     if (all_of(N->uses(), [&](SDNode *Node) {
-- 
GitLab


From cb037d071c43baab7e8ef3b818a21858905560c0 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Fri, 10 May 2024 14:57:55 -0700
Subject: [PATCH 0460/1206] [AMDGPU] Combine build_vector patterns for i16 and
 f16. NFCI. (#91806)

---
 llvm/lib/Target/AMDGPU/SIInstructions.td | 51 ++++++++----------------
 1 file changed, 17 insertions(+), 34 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 7592f8da1b4f..f9a660e334a0 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3166,49 +3166,33 @@ def : GCNPat <
   (v2f16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), VGPR_32:$src1))
 >;
 
-def : GCNPat <
-  (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 undef))),
-  (COPY_TO_REGCLASS SReg_32:$src0, SReg_32)
->;
-
-def : GCNPat <
-  (v2i16 (DivergentBinFrag<build_vector> (i16 VGPR_32:$src0), (i16 undef))),
-  (COPY_TO_REGCLASS VGPR_32:$src0, VGPR_32)
->;
+foreach vecTy = [v2i16, v2f16] in {
 
-def : GCNPat <
-  (v2f16 (build_vector f16:$src0, (f16 undef))),
-  (COPY $src0)
->;
+defvar Ty = vecTy.ElementType;
 
 def : GCNPat <
-  (v2i16 (UniformBinFrag<build_vector> (i16 undef), (i16 SReg_32:$src1))),
-  (S_LSHL_B32 SReg_32:$src1, (i32 16))
+  (vecTy (UniformBinFrag<build_vector> (Ty SReg_32:$src0), (Ty undef))),
+  (COPY_TO_REGCLASS SReg_32:$src0, SReg_32)
 >;
 
 def : GCNPat <
-  (v2i16 (DivergentBinFrag<build_vector> (i16 undef), (i16 VGPR_32:$src1))),
-  (v2i16 (V_LSHLREV_B32_e64 (i32 16), VGPR_32:$src1))
+  (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$src0), (Ty undef))),
+  (COPY_TO_REGCLASS VGPR_32:$src0, VGPR_32)
 >;
 
-
 def : GCNPat <
-  (v2f16 (UniformBinFrag<build_vector> (f16 undef), (f16 SReg_32:$src1))),
+  (vecTy (UniformBinFrag<build_vector> (Ty undef), (Ty SReg_32:$src1))),
   (S_LSHL_B32 SReg_32:$src1, (i32 16))
 >;
 
 def : GCNPat <
-  (v2f16 (DivergentBinFrag<build_vector> (f16 undef), (f16 VGPR_32:$src1))),
-  (v2f16 (V_LSHLREV_B32_e64 (i32 16), VGPR_32:$src1))
+  (vecTy (DivergentBinFrag<build_vector> (Ty undef), (Ty VGPR_32:$src1))),
+  (vecTy (V_LSHLREV_B32_e64 (i32 16), VGPR_32:$src1))
 >;
+} // End foreach Ty = ...
 }
 
 let SubtargetPredicate = HasVOP3PInsts in {
-def : GCNPat <
-  (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 SReg_32:$src1))),
-  (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1)
->;
-
 def : GCNPat <
   (v2i16 (DivergentBinFrag<build_vector> (i16 VGPR_32:$src0), (i16 VGPR_32:$src1))),
   (v2i16 (V_LSHL_OR_B32_e64 $src1, (i32 16), (i32 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), $src0))))
@@ -3227,18 +3211,17 @@ def : GCNPat <
   (S_PACK_HH_B32_B16 SReg_32:$src0, SReg_32:$src1)
 >;
 
-def : GCNPat <
-  (v2f16 (UniformBinFrag<build_vector> (f16 SReg_32:$src0), (f16 SReg_32:$src1))),
-  (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1)
->;
 
+foreach vecTy = [v2i16, v2f16] in {
 
-
-foreach Ty = [i16, f16] in {
-
-defvar vecTy = !if(!eq(Ty, i16), v2i16, v2f16);
+defvar Ty = vecTy.ElementType;
 defvar immzeroTy = !if(!eq(Ty, i16), immzero, fpimmzero);
 
+def : GCNPat <
+  (vecTy (UniformBinFrag<build_vector> (Ty SReg_32:$src0), (Ty SReg_32:$src1))),
+  (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1)
+>;
+
 // Take the lower 16 bits from each VGPR_32 and concat them
 def : GCNPat <
   (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$a), (Ty VGPR_32:$b))),
-- 
GitLab


From 3dcd604eb1d6612fda667793dbb52c5dfaa5fc4f Mon Sep 17 00:00:00 2001
From: Janek van Oirschot <5994977+JanekvO@users.noreply.github.com>
Date: Fri, 10 May 2024 23:31:17 +0100
Subject: [PATCH 0461/1206] [AMDGPU] Initialize MachineFunctionInfo for
 unittest (#91820)

Initializes the MachineFunctionInfo for unittest introduced in #88257

Additionally, also rephrases the test condition considering the abort/exit does not occur within emitKernel but is generally triggered through the Ctx.hadError() call.
---
 llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp b/llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp
index f2161f71e6e9..912974fa23bd 100644
--- a/llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp
+++ b/llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp
@@ -64,6 +64,7 @@ protected:
                                         TM->getTargetFeatureString(), *TM);
 
     MF = std::make_unique<MachineFunction>(*F, *TM, *ST, 1, *MMI);
+    MF->initTargetMachineFunctionInfo(*ST);
     PI.reset(*MF.get());
   }
 };
@@ -76,6 +77,11 @@ TEST_F(SIProgramInfoMCExprsTest, TestDeathHSAKernelEmit) {
   auto &Func = MF->getFunction();
   Func.setCallingConv(CallingConv::AMDGPU_KERNEL);
   AMDGPU::HSAMD::MetadataStreamerMsgPackV4 MD;
-  EXPECT_DEATH(MD.emitKernel(*MF, PI),
-               "could not resolve expression when required.");
+
+  testing::internal::CaptureStderr();
+  MD.emitKernel(*MF, PI);
+  std::string err = testing::internal::GetCapturedStderr();
+  EXPECT_EQ(
+      err, "<unknown>:0: error: could not resolve expression when required.\n");
+  EXPECT_TRUE(Ctx.hadError());
 }
-- 
GitLab


From 30aa49cbc2ece32bebcb06cc9e64f1daeb1dbbcb Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 10 May 2024 15:14:39 -0700
Subject: [PATCH 0462/1206] [RISCV] Remove the pre-split from
 RISCVISAInfo::parseArchString. NFCI

We can extract each extension as we process them without much
complexity.
---
 llvm/lib/TargetParser/RISCVISAInfo.cpp | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 931c2aa41ac7..e22dd6032cb0 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -660,10 +660,6 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
     break;
   }
 
-  if (Arch.back() == '_')
-    return createStringError(errc::invalid_argument,
-                             "extension name missing after separator '_'");
-
   // Skip baseline.
   StringRef Exts = Arch.drop_front(1);
 
@@ -703,18 +699,18 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
   // Consume the base ISA version number and any '_' between rvxxx and the
   // first extension
   Exts = Exts.drop_front(ConsumeLength);
-  Exts.consume_front("_");
 
-  SmallVector<StringRef, 8> SplitExts;
-  // Only split if the string is not empty. Otherwise the split will push an
-  // empty string into the vector.
-  if (!Exts.empty())
-    Exts.split(SplitExts, '_');
+  while (!Exts.empty()) {
+    if (Exts.front() == '_') {
+      if (Exts.size() == 1 || Exts[1] == '_')
+        return createStringError(errc::invalid_argument,
+                                 "extension name missing after separator '_'");
+      Exts = Exts.drop_front();
+    }
 
-  for (auto Ext : SplitExts) {
-    if (Ext.empty())
-      return createStringError(errc::invalid_argument,
-                               "extension name missing after separator '_'");
+    size_t Idx = Exts.find('_');
+    StringRef Ext = Exts.slice(0, Idx);
+    Exts = Exts.slice(Idx, StringRef::npos);
 
     do {
       if (RISCVISAUtils::AllStdExts.contains(Ext.front())) {
-- 
GitLab


From d9ce33a0eea701719651dafbf88deae0ece96976 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Fri, 10 May 2024 15:36:55 -0700
Subject: [PATCH 0463/1206] [libfuzzer] Prevent MSan false positive when
 printing log with -jobs (#91679)

libfuzzer's -jobs option will, depending on the number of CPUs, spin up
a
WorkerThread and end up printing the log file using CopyFileToErr.
This leads to an MSan false positive. This patch disables the MSan
interceptor checks,
similarly to other instances in https://reviews.llvm.org/D48891

Side-note: this false positive issue first appeared when printf()
was replaced by puts() (90b4d1bcb20180c591385131b12fa90d2e4860b1).
The interceptor check was always present; however, MSan does not
check_printf by default.
---
 compiler-rt/lib/fuzzer/FuzzerDriver.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
index 8674d788932f..3771abf5f532 100644
--- a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
@@ -229,6 +229,7 @@ static void PulseThread() {
 
 static void WorkerThread(const Command &BaseCmd, std::atomic<unsigned> *Counter,
                          unsigned NumJobs, std::atomic<bool> *HasErrors) {
+  ScopedDisableMsanInterceptorChecks S;
   while (true) {
     unsigned C = (*Counter)++;
     if (C >= NumJobs) break;
-- 
GitLab


From 4198aebc96cb0236fc63e29a92d886e6a2e3fedb Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Sat, 11 May 2024 00:42:56 +0200
Subject: [PATCH 0464/1206] [Clang] Fix tests broken by #91811 (#91822)

A few tests were checking for all driver flags.
We should revert this commit when we revert #91811
---
 clang/test/Driver/rewrite-legacy-objc.m | 6 +++---
 clang/test/Driver/rewrite-objc.m        | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/clang/test/Driver/rewrite-legacy-objc.m b/clang/test/Driver/rewrite-legacy-objc.m
index 413a7a7a61f0..d45fb8c405c5 100644
--- a/clang/test/Driver/rewrite-legacy-objc.m
+++ b/clang/test/Driver/rewrite-legacy-objc.m
@@ -3,11 +3,11 @@
 // TEST0: "-cc1"
 // TEST0: "-rewrite-objc"
 // FIXME: CHECK-NOT is broken somehow, it doesn't work here. Check adjacency instead.
-// TEST0: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fmax-type-align=16"
+// TEST0: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fno-relaxed-template-template-args" "-fmax-type-align=16"
 // TEST0: rewrite-legacy-objc.m"
 // RUN: %clang --target=i386-apple-macosx10.9.0 -rewrite-legacy-objc %s -o - -### 2>&1 | \
 // RUN:   FileCheck -check-prefix=TEST1 %s
 // RUN: %clang --target=i386-apple-macosx10.6.0 -rewrite-legacy-objc %s -o - -### 2>&1 | \
 // RUN:   FileCheck -check-prefix=TEST2 %s
-// TEST1: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fmax-type-align=16"
-// TEST2: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fmax-type-align=16"
+// TEST1: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fno-relaxed-template-template-args" "-fmax-type-align=16"
+// TEST2: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fno-relaxed-template-template-args" "-fmax-type-align=16"
diff --git a/clang/test/Driver/rewrite-objc.m b/clang/test/Driver/rewrite-objc.m
index de3577a770df..d19d38d8ab83 100644
--- a/clang/test/Driver/rewrite-objc.m
+++ b/clang/test/Driver/rewrite-objc.m
@@ -2,4 +2,4 @@
 // RUN:   FileCheck -check-prefix=TEST0 %s
 // TEST0: "-cc1" {{.*}} "-rewrite-objc"
 // FIXME: CHECK-NOT is broken somehow, it doesn't work here. Check adjacency instead.
-// TEST0: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fmax-type-align=16"
+// TEST0: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fno-relaxed-template-template-args" "-fmax-type-align=16"
-- 
GitLab


From f8063ffe73a3a1d704b9738169bb76ebb0f8a5e0 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu@sifive.com>
Date: Fri, 10 May 2024 16:01:47 -0700
Subject: [PATCH 0465/1206] [VP][RISCV] Add vp.reduce.fmaximum/fminimum and its
 RISC-V codegen (#91782)

`vp.reduce.fmaximum/fminimum` are the VP version of
`vector.reduce.fmaximum/fminimum`.
---
 llvm/docs/LangRef.rst                         | 140 ++++++++++++++++++
 llvm/include/llvm/IR/Intrinsics.td            |  10 ++
 llvm/include/llvm/IR/VPIntrinsics.def         |   8 +
 llvm/lib/CodeGen/ExpandVectorPredication.cpp  |  24 ++-
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |   6 +
 .../SelectionDAG/LegalizeVectorTypes.cpp      |   4 +
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   2 +
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  40 ++++-
 llvm/test/CodeGen/Generic/expand-vp.ll        |  83 ++++++++---
 llvm/test/CodeGen/RISCV/rvv/undef-vp-ops.ll   |  32 ++++
 .../CodeGen/RISCV/rvv/vreductions-fp-vp.ll    | 122 +++++++++++++++
 llvm/test/Verifier/vp-intrinsics.ll           |   8 +-
 llvm/unittests/IR/VPIntrinsicTest.cpp         |   3 +-
 13 files changed, 453 insertions(+), 29 deletions(-)

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 6f5a4644ffc2..d0515876f9e4 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -22182,6 +22182,146 @@ Examples:
       %also.r = call float @llvm.minnum.f32(float %reduction, float %start)
 
 
+.. _int_vp_reduce_fmaximum:
+
+'``llvm.vp.reduce.fmaximum.*``' Intrinsics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+This is an overloaded intrinsic.
+
+::
+
+      declare float @llvm.vp.reduce.fmaximum.v4f32(float <start_value>, <4 x float> <val>, <4 x i1> <mask>, float <vector_length>)
+      declare double @llvm.vp.reduce.fmaximum.nxv8f64(double <start_value>, <vscale x 8 x double> <val>, <vscale x 8 x i1> <mask>, i32 <vector_length>)
+
+Overview:
+"""""""""
+
+Predicated floating-point ``MAX`` reduction of a vector and a scalar starting
+value, returning the result as a scalar.
+
+
+Arguments:
+""""""""""
+
+The first operand is the start value of the reduction, which must be a scalar
+floating-point type equal to the result type. The second operand is the vector
+on which the reduction is performed and must be a vector of floating-point
+values whose element type is the result/start type. The third operand is the
+vector mask and is a vector of boolean values with the same number of elements
+as the vector operand. The fourth operand is the explicit vector length of the
+operation.
+
+Semantics:
+""""""""""
+
+The '``llvm.vp.reduce.fmaximum``' intrinsic performs the floating-point ``MAX``
+reduction (:ref:`llvm.vector.reduce.fmaximum <int_vector_reduce_fmaximum>`) of
+the vector operand ``val`` on each enabled lane, taking the maximum of that and
+the scalar ``start_value``. Disabled lanes are treated as containing the
+neutral value (i.e. having no effect on the reduction operation). If the vector
+length is zero, the result is the start value.
+
+The neutral value is dependent on the :ref:`fast-math flags <fastmath>`. If no
+flags are set or only the ``nnan`` is set, the neutral value is ``-Infinity``.
+If ``ninf`` is set, then the neutral value is the smallest floating-point value
+for the result type.
+
+This instruction has the same comparison semantics as the
+:ref:`llvm.vector.reduce.fmaximum <int_vector_reduce_fmaximum>` intrinsic (and
+thus the '``llvm.maximum.*``' intrinsic). That is, the result will always be a
+number unless any of the elements in the vector or the starting value is
+``NaN``. Namely, this intrinsic propagates ``NaN``. Also, -0.0 is considered
+less than +0.0.
+
+To ignore the start value, the neutral value can be used.
+
+Examples:
+"""""""""
+
+.. code-block:: llvm
+
+      %r = call float @llvm.vp.reduce.fmaximum.v4f32(float %float, <4 x float> %a, <4 x i1> %mask, i32 %evl)
+      ; %r is equivalent to %also.r, where lanes greater than or equal to %evl
+      ; are treated as though %mask were false for those lanes.
+
+      %masked.a = select <4 x i1> %mask, <4 x float> %a, <4 x float> <float -infinity, float -infinity, float -infinity, float -infinity>
+      %reduction = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> %masked.a)
+      %also.r = call float @llvm.maximum.f32(float %reduction, float %start)
+
+
+.. _int_vp_reduce_fminimum:
+
+'``llvm.vp.reduce.fminimum.*``' Intrinsics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+This is an overloaded intrinsic.
+
+::
+
+      declare float @llvm.vp.reduce.fminimum.v4f32(float <start_value>, <4 x float> <val>, <4 x i1> <mask>, float <vector_length>)
+      declare double @llvm.vp.reduce.fminimum.nxv8f64(double <start_value>, <vscale x 8 x double> <val>, <vscale x 8 x i1> <mask>, i32 <vector_length>)
+
+Overview:
+"""""""""
+
+Predicated floating-point ``MIN`` reduction of a vector and a scalar starting
+value, returning the result as a scalar.
+
+
+Arguments:
+""""""""""
+
+The first operand is the start value of the reduction, which must be a scalar
+floating-point type equal to the result type. The second operand is the vector
+on which the reduction is performed and must be a vector of floating-point
+values whose element type is the result/start type. The third operand is the
+vector mask and is a vector of boolean values with the same number of elements
+as the vector operand. The fourth operand is the explicit vector length of the
+operation.
+
+Semantics:
+""""""""""
+
+The '``llvm.vp.reduce.fminimum``' intrinsic performs the floating-point ``MIN``
+reduction (:ref:`llvm.vector.reduce.fminimum <int_vector_reduce_fminimum>`) of
+the vector operand ``val`` on each enabled lane, taking the minimum of that and
+the scalar ``start_value``. Disabled lanes are treated as containing the neutral
+value (i.e. having no effect on the reduction operation). If the vector length
+is zero, the result is the start value.
+
+The neutral value is dependent on the :ref:`fast-math flags <fastmath>`. If no
+flags are set or only the ``nnan`` is set, the neutral value is ``+Infinity``.
+If ``ninf`` is set, then the neutral value is the largest floating-point value
+for the result type.
+
+This instruction has the same comparison semantics as the
+:ref:`llvm.vector.reduce.fminimum <int_vector_reduce_fminimum>` intrinsic (and
+thus the '``llvm.minimum.*``' intrinsic). That is, the result will always be a
+number unless any of the elements in the vector or the starting value is
+``NaN``. Namely, this intrinsic propagates ``NaN``. Also, -0.0 is considered
+less than +0.0.
+
+To ignore the start value, the neutral value can be used.
+
+Examples:
+"""""""""
+
+.. code-block:: llvm
+
+      %r = call float @llvm.vp.reduce.fminimum.v4f32(float %start, <4 x float> %a, <4 x i1> %mask, i32 %evl)
+      ; %r is equivalent to %also.r, where lanes greater than or equal to %evl
+      ; are treated as though %mask were false for those lanes.
+
+      %masked.a = select <4 x i1> %mask, <4 x float> %a, <4 x float> <float infinity, float infinity, float infinity, float infinity>
+      %reduction = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> %masked.a)
+      %also.r = call float @llvm.minimum.f32(float %reduction, float %start)
+
+
 .. _int_get_active_lane_mask:
 
 '``llvm.get.active.lane.mask.*``' Intrinsics
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 29143123193b..42192d472ba6 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2243,6 +2243,16 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
                                llvm_anyvector_ty,
                                LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                llvm_i32_ty]>;
+  def int_vp_reduce_fmaximum : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
+                                 [ LLVMVectorElementType<0>,
+                                   llvm_anyvector_ty,
+                                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                                   llvm_i32_ty]>;
+  def int_vp_reduce_fminimum : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
+                                 [ LLVMVectorElementType<0>,
+                                   llvm_anyvector_ty,
+                                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                                   llvm_i32_ty]>;
 }
 
 let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn, ImmArg<ArgIndex<1>>] in {
diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def
index f1cc8bcae467..20f5bb2b531d 100644
--- a/llvm/include/llvm/IR/VPIntrinsics.def
+++ b/llvm/include/llvm/IR/VPIntrinsics.def
@@ -701,6 +701,14 @@ HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmax, VP_REDUCE_FMAX,
 HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN,
                              vector_reduce_fmin)
 
+// llvm.vp.reduce.fmaximum(start,x,mask,vlen)
+HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmaximum, VP_REDUCE_FMAXIMUM,
+                             vector_reduce_fmaximum)
+
+// llvm.vp.reduce.fminimum(start,x,mask,vlen)
+HELPER_REGISTER_REDUCTION_VP(vp_reduce_fminimum, VP_REDUCE_FMINIMUM,
+                             vector_reduce_fminimum)
+
 #undef HELPER_REGISTER_REDUCTION_VP
 
 // Specialized helper macro for VP reductions as above but with two forms:
diff --git a/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
index 8e623c85b737..dc35f33a3a05 100644
--- a/llvm/lib/CodeGen/ExpandVectorPredication.cpp
+++ b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
@@ -367,7 +367,8 @@ static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
                                          Type *EltTy) {
   bool Negative = false;
   unsigned EltBits = EltTy->getScalarSizeInBits();
-  switch (VPI.getIntrinsicID()) {
+  Intrinsic::ID VID = VPI.getIntrinsicID();
+  switch (VID) {
   default:
     llvm_unreachable("Expecting a VP reduction intrinsic");
   case Intrinsic::vp_reduce_add:
@@ -387,12 +388,17 @@ static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
     return ConstantInt::get(EltTy->getContext(),
                             APInt::getSignedMinValue(EltBits));
   case Intrinsic::vp_reduce_fmax:
+  case Intrinsic::vp_reduce_fmaximum:
     Negative = true;
     [[fallthrough]];
-  case Intrinsic::vp_reduce_fmin: {
+  case Intrinsic::vp_reduce_fmin:
+  case Intrinsic::vp_reduce_fminimum: {
+    bool PropagatesNaN = VID == Intrinsic::vp_reduce_fminimum ||
+                         VID == Intrinsic::vp_reduce_fmaximum;
     FastMathFlags Flags = VPI.getFastMathFlags();
     const fltSemantics &Semantics = EltTy->getFltSemantics();
-    return !Flags.noNaNs() ? ConstantFP::getQNaN(EltTy, Negative)
+    return (!Flags.noNaNs() && !PropagatesNaN)
+               ? ConstantFP::getQNaN(EltTy, Negative)
            : !Flags.noInfs()
                ? ConstantFP::getInfinity(EltTy, Negative)
                : ConstantFP::get(EltTy,
@@ -480,6 +486,18 @@ CachingVPExpander::expandPredicationInReduction(IRBuilder<> &Builder,
     Reduction =
         Builder.CreateBinaryIntrinsic(Intrinsic::minnum, Reduction, Start);
     break;
+  case Intrinsic::vp_reduce_fmaximum:
+    Reduction = Builder.CreateFPMaximumReduce(RedOp);
+    transferDecorations(*Reduction, VPI);
+    Reduction =
+        Builder.CreateBinaryIntrinsic(Intrinsic::maximum, Reduction, Start);
+    break;
+  case Intrinsic::vp_reduce_fminimum:
+    Reduction = Builder.CreateFPMinimumReduce(RedOp);
+    transferDecorations(*Reduction, VPI);
+    Reduction =
+        Builder.CreateBinaryIntrinsic(Intrinsic::minimum, Reduction, Start);
+    break;
   case Intrinsic::vp_reduce_fadd:
     Reduction = Builder.CreateFAddReduce(Start, RedOp);
     break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index b3ae419b20fe..fd97a1283b65 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1222,6 +1222,8 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
   case ISD::VP_REDUCE_UMIN:
   case ISD::VP_REDUCE_FMAX:
   case ISD::VP_REDUCE_FMIN:
+  case ISD::VP_REDUCE_FMAXIMUM:
+  case ISD::VP_REDUCE_FMINIMUM:
   case ISD::VP_REDUCE_SEQ_FADD:
   case ISD::VP_REDUCE_SEQ_FMUL:
     Action = TLI.getOperationAction(
@@ -5015,6 +5017,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
       Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
       Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
       Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
+      Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
+      Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
       Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
     OVT = Node->getOperand(1).getSimpleValueType();
   if (Node->getOpcode() == ISD::BR_CC ||
@@ -5687,6 +5691,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
   case ISD::VP_REDUCE_FMUL:
   case ISD::VP_REDUCE_FMAX:
   case ISD::VP_REDUCE_FMIN:
+  case ISD::VP_REDUCE_FMAXIMUM:
+  case ISD::VP_REDUCE_FMINIMUM:
   case ISD::VP_REDUCE_SEQ_FADD:
     Results.push_back(PromoteReduction(Node));
     break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 43db9b8e6be9..cd858003cf03 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -3148,6 +3148,8 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
   case ISD::VP_REDUCE_UMIN:
   case ISD::VP_REDUCE_FMAX:
   case ISD::VP_REDUCE_FMIN:
+  case ISD::VP_REDUCE_FMAXIMUM:
+  case ISD::VP_REDUCE_FMINIMUM:
     Res = SplitVecOp_VP_REDUCE(N, OpNo);
     break;
   case ISD::VP_CTTZ_ELTS:
@@ -6251,6 +6253,8 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
   case ISD::VP_REDUCE_UMIN:
   case ISD::VP_REDUCE_FMAX:
   case ISD::VP_REDUCE_FMIN:
+  case ISD::VP_REDUCE_FMAXIMUM:
+  case ISD::VP_REDUCE_FMINIMUM:
     Res = WidenVecOp_VP_REDUCE(N);
     break;
   case ISD::VP_CTTZ_ELTS:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 9c1f3c1e3431..0a258350c68a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -470,8 +470,10 @@ ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
   case ISD::VP_REDUCE_FMIN:
     return ISD::FMINNUM;
   case ISD::VECREDUCE_FMAXIMUM:
+  case ISD::VP_REDUCE_FMAXIMUM:
     return ISD::FMAXIMUM;
   case ISD::VECREDUCE_FMINIMUM:
+  case ISD::VP_REDUCE_FMINIMUM:
     return ISD::FMINIMUM;
   }
 }
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e0937989a6a4..60985edd9420 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -713,7 +713,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         ISD::VP_FRINT,       ISD::VP_FNEARBYINT,  ISD::VP_IS_FPCLASS,
         ISD::VP_FMINIMUM,    ISD::VP_FMAXIMUM,    ISD::VP_LRINT,
         ISD::VP_LLRINT,      ISD::EXPERIMENTAL_VP_REVERSE,
-        ISD::EXPERIMENTAL_VP_SPLICE};
+        ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
+        ISD::VP_REDUCE_FMAXIMUM};
 
     static const unsigned IntegerVecReduceOps[] = {
         ISD::VECREDUCE_ADD,  ISD::VECREDUCE_AND,  ISD::VECREDUCE_OR,
@@ -958,7 +959,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         ISD::VP_FFLOOR,      ISD::VP_FROUND,       ISD::VP_FROUNDEVEN,
         ISD::VP_FCOPYSIGN,   ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
         ISD::VP_FNEARBYINT,  ISD::VP_SETCC,        ISD::VP_FMINIMUM,
-        ISD::VP_FMAXIMUM};
+        ISD::VP_FMAXIMUM,    ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
 
     // Sets common operation actions on RVV floating-point vector types.
     const auto SetCommonVFPActions = [&](MVT VT) {
@@ -6661,6 +6662,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::VP_REDUCE_SEQ_FADD:
   case ISD::VP_REDUCE_FMIN:
   case ISD::VP_REDUCE_FMAX:
+  case ISD::VP_REDUCE_FMINIMUM:
+  case ISD::VP_REDUCE_FMAXIMUM:
     if (Op.getOperand(1).getValueType() == MVT::nxv32f16 &&
         (Subtarget.hasVInstructionsF16Minimal() &&
          !Subtarget.hasVInstructionsF16()))
@@ -9526,8 +9529,10 @@ static unsigned getRVVReductionOp(unsigned ISDOpcode) {
   case ISD::VP_REDUCE_SEQ_FADD:
     return RISCVISD::VECREDUCE_SEQ_FADD_VL;
   case ISD::VP_REDUCE_FMAX:
+  case ISD::VP_REDUCE_FMAXIMUM:
     return RISCVISD::VECREDUCE_FMAX_VL;
   case ISD::VP_REDUCE_FMIN:
+  case ISD::VP_REDUCE_FMINIMUM:
     return RISCVISD::VECREDUCE_FMIN_VL;
   }
 
@@ -9786,8 +9791,11 @@ SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
 SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
                                            SelectionDAG &DAG) const {
   SDLoc DL(Op);
+  unsigned Opc = Op.getOpcode();
+  SDValue Start = Op.getOperand(0);
   SDValue Vec = Op.getOperand(1);
   EVT VecEVT = Vec.getValueType();
+  MVT XLenVT = Subtarget.getXLenVT();
 
   // TODO: The type may need to be widened rather than split. Or widened before
   // it can be split.
@@ -9795,7 +9803,7 @@ SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
     return SDValue();
 
   MVT VecVT = VecEVT.getSimpleVT();
-  unsigned RVVOpcode = getRVVReductionOp(Op.getOpcode());
+  unsigned RVVOpcode = getRVVReductionOp(Opc);
 
   if (VecVT.isFixedLengthVector()) {
     auto ContainerVT = getContainerForFixedLengthVector(VecVT);
@@ -9804,8 +9812,30 @@ SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
 
   SDValue VL = Op.getOperand(3);
   SDValue Mask = Op.getOperand(2);
-  return lowerReductionSeq(RVVOpcode, Op.getSimpleValueType(), Op.getOperand(0),
-                           Vec, Mask, VL, DL, DAG, Subtarget);
+  SDValue Res =
+      lowerReductionSeq(RVVOpcode, Op.getSimpleValueType(), Op.getOperand(0),
+                        Vec, Mask, VL, DL, DAG, Subtarget);
+  if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
+      Op->getFlags().hasNoNaNs())
+    return Res;
+
+  // Propagate NaNs.
+  MVT PredVT = getMaskTypeFor(Vec.getSimpleValueType());
+  // Check if any of the elements in Vec is NaN.
+  SDValue IsNaN = DAG.getNode(
+      RISCVISD::SETCC_VL, DL, PredVT,
+      {Vec, Vec, DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(PredVT), Mask, VL});
+  SDValue VCPop = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, IsNaN, Mask, VL);
+  // Check if the start value is NaN.
+  SDValue StartIsNaN = DAG.getSetCC(DL, XLenVT, Start, Start, ISD::SETUO);
+  VCPop = DAG.getNode(ISD::OR, DL, XLenVT, VCPop, StartIsNaN);
+  SDValue NoNaNs = DAG.getSetCC(DL, XLenVT, VCPop,
+                                DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
+  MVT ResVT = Res.getSimpleValueType();
+  return DAG.getSelect(
+      DL, ResVT, NoNaNs, Res,
+      DAG.getConstantFP(APFloat::getNaN(DAG.EVTToAPFloatSemantics(ResVT)), DL,
+                        ResVT));
 }
 
 SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
diff --git a/llvm/test/CodeGen/Generic/expand-vp.ll b/llvm/test/CodeGen/Generic/expand-vp.ll
index 40d183273b86..4fee9a533b94 100644
--- a/llvm/test/CodeGen/Generic/expand-vp.ll
+++ b/llvm/test/CodeGen/Generic/expand-vp.ll
@@ -41,6 +41,8 @@ declare i32 @llvm.vp.reduce.umin.v4i32(i32, <4 x i32>, <4 x i1>, i32)
 declare i32 @llvm.vp.reduce.umax.v4i32(i32, <4 x i32>, <4 x i1>, i32)
 declare float @llvm.vp.reduce.fmin.v4f32(float, <4 x float>, <4 x i1>, i32)
 declare float @llvm.vp.reduce.fmax.v4f32(float, <4 x float>, <4 x i1>, i32)
+declare float @llvm.vp.reduce.fminimum.v4f32(float, <4 x float>, <4 x i1>, i32)
+declare float @llvm.vp.reduce.fmaximum.v4f32(float, <4 x float>, <4 x i1>, i32)
 declare float @llvm.vp.reduce.fadd.v4f32(float, <4 x float>, <4 x i1>, i32)
 declare float @llvm.vp.reduce.fmul.v4f32(float, <4 x float>, <4 x i1>, i32)
 ; Comparisons
@@ -133,10 +135,16 @@ define void @test_vp_reduce_fp_v4(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n
   %r3 = call float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
   %r4 = call nnan float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
   %r5 = call nnan ninf float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-  %r6 = call float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-  %r7 = call reassoc float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-  %r8 = call float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-  %r9 = call reassoc float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+  %r6 = call float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+  %r7 = call nnan float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+  %r8 = call nnan ninf float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+  %r9 = call float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+  %r10 = call nnan float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+  %r11 = call nnan ninf float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+  %r12 = call float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+  %r13 = call reassoc float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+  %r14 = call float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+  %r15 = call reassoc float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
   ret void
 }
 
@@ -254,6 +262,27 @@ define void @test_vp_cmp_v8(<8 x i32> %i0, <8 x i32> %i1, <8 x float> %f0, <8 x
 ; ALL-CONVERT:  [[FMAX_NNAN_NINF:%.+]] = select <4 x i1> %{{.+}}, <4 x float> %vf, <4 x float> <float 0xC7EFFFFFE0000000, float 0xC7EFFFFFE0000000, float 0xC7EFFFFFE0000000, float 0xC7EFFFFFE0000000>
 ; ALL-CONVERT-NEXT:  [[RED:%.+]] = call nnan ninf float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[FMAX_NNAN_NINF]])
 ; ALL-CONVERT-NEXT:  %{{.+}} = call nnan ninf float @llvm.maxnum.f32(float [[RED]], float %f)
+
+; ALL-CONVERT:       [[FMINIMUM:%.+]] = select <4 x i1> %{{.+}}, <4 x float> %vf, <4 x float> <float 0x7FF0000000000000, float 0x7FF0000000000000, float 0x7FF0000000000000, float 0x7FF0000000000000>
+; ALL-CONVERT-NEXT:  [[RED:%.+]] = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> [[FMINIMUM]])
+; ALL-CONVERT-NEXT:  %{{.+}} = call float @llvm.minimum.f32(float [[RED]], float %f)
+; ALL-CONVERT:       [[FMINIMUM_NNAN:%.+]] = select <4 x i1> %{{.+}}, <4 x float> %vf, <4 x float> <float 0x7FF0000000000000, float 0x7FF0000000000000, float 0x7FF0000000000000, float 0x7FF0000000000000>
+; ALL-CONVERT-NEXT:  [[RED:%.+]] = call nnan float @llvm.vector.reduce.fminimum.v4f32(<4 x float> [[FMINIMUM_NNAN]])
+; ALL-CONVERT-NEXT:  %{{.+}} = call nnan float @llvm.minimum.f32(float [[RED]], float %f)
+; ALL-CONVERT:       [[FMINIMUM_NNAN_NINF:%.+]] = select <4 x i1> %{{.+}}, <4 x float> %vf, <4 x float> <float 0x47EFFFFFE0000000, float 0x47EFFFFFE0000000, float 0x47EFFFFFE0000000, float 0x47EFFFFFE0000000>
+; ALL-CONVERT-NEXT:  [[RED:%.+]] = call nnan ninf float @llvm.vector.reduce.fminimum.v4f32(<4 x float> [[FMINIMUM_NNAN_NINF]])
+; ALL-CONVERT-NEXT:  %{{.+}} = call nnan ninf float @llvm.minimum.f32(float [[RED]], float %f)
+
+; ALL-CONVERT:  [[FMAXIMUM:%.+]] = select <4 x i1> %{{.+}}, <4 x float> %vf, <4 x float> <float 0xFFF0000000000000, float 0xFFF0000000000000, float 0xFFF0000000000000, float 0xFFF0000000000000>
+; ALL-CONVERT-NEXT:  [[RED:%.+]] = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[FMAXIMUM]])
+; ALL-CONVERT-NEXT:  %{{.+}} = call float @llvm.maximum.f32(float [[RED]], float %f)
+; ALL-CONVERT:  [[FMAXIMUM_NNAN:%.+]] = select <4 x i1> %{{.+}}, <4 x float> %vf, <4 x float> <float 0xFFF0000000000000, float 0xFFF0000000000000, float 0xFFF0000000000000, float 0xFFF0000000000000>
+; ALL-CONVERT-NEXT:  [[RED:%.+]] = call nnan float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[FMAXIMUM_NNAN]])
+; ALL-CONVERT-NEXT:  %{{.+}} = call nnan float @llvm.maximum.f32(float [[RED]], float %f)
+; ALL-CONVERT:  [[FMAXIMUM_NNAN_NINF:%.+]] = select <4 x i1> %{{.+}}, <4 x float> %vf, <4 x float> <float 0xC7EFFFFFE0000000, float 0xC7EFFFFFE0000000, float 0xC7EFFFFFE0000000, float 0xC7EFFFFFE0000000>
+; ALL-CONVERT-NEXT:  [[RED:%.+]] = call nnan ninf float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> [[FMAXIMUM_NNAN_NINF]])
+; ALL-CONVERT-NEXT:  %{{.+}} = call nnan ninf float @llvm.maximum.f32(float [[RED]], float %f)
+
 ; ALL-CONVERT:  [[FADD:%.+]] = select <4 x i1> %{{.+}}, <4 x float> %vf, <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>
 ; ALL-CONVERT-NEXT:  %{{.+}} = call float @llvm.vector.reduce.fadd.v4f32(float %f, <4 x float> [[FADD]])
 ; ALL-CONVERT:  [[FADD:%.+]] = select <4 x i1> %{{.+}}, <4 x float> %vf, <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>
@@ -328,16 +357,22 @@ define void @test_vp_cmp_v8(<8 x i32> %i0, <8 x i32> %i1, <8 x float> %f0, <8 x
 ; LEGAL_LEGAL-NEXT:  ret void
 
 ; LEGAL_LEGAL: define void @test_vp_reduce_fp_v4(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n) {
-; LEGAL_LEGAL-NEXT:  %r0 = call float @llvm.vp.reduce.fmin.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-; LEGAL_LEGAL-NEXT:  %r1 = call nnan float @llvm.vp.reduce.fmin.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-; LEGAL_LEGAL-NEXT:  %r2 = call nnan ninf float @llvm.vp.reduce.fmin.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-; LEGAL_LEGAL-NEXT:  %r3 = call float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-; LEGAL_LEGAL-NEXT:  %r4 = call nnan float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-; LEGAL_LEGAL-NEXT:  %r5 = call nnan ninf float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-; LEGAL_LEGAL-NEXT:  %r6 = call float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-; LEGAL_LEGAL-NEXT:  %r7 = call reassoc float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-; LEGAL_LEGAL-NEXT:  %r8 = call float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
-; LEGAL_LEGAL-NEXT:  %r9 = call reassoc float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r0 = call float @llvm.vp.reduce.fmin.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r1 = call nnan float @llvm.vp.reduce.fmin.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r2 = call nnan ninf float @llvm.vp.reduce.fmin.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r3 = call float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r4 = call nnan float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r5 = call nnan ninf float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r6 = call float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r7 = call nnan float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r8 = call nnan ninf float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r9 = call float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r10 = call nnan float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r11 = call nnan ninf float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r12 = call float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r13 = call reassoc float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r14 = call float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; LEGAL_LEGAL-NEXT: %r15 = call reassoc float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
 ; LEGAL_LEGAL-NEXT:  ret void
 
 ; LEGAL_LEGAL: define void @test_vp_cmp_v8(<8 x i32> %i0, <8 x i32> %i1, <8 x float> %f0, <8 x float> %f1, <8 x i1> %m, i32 %n) {
@@ -425,10 +460,16 @@ define void @test_vp_cmp_v8(<8 x i32> %i0, <8 x i32> %i1, <8 x float> %f0, <8 x
 ; DISCARD_LEGAL-NOT:  %r3 = call float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
 ; DISCARD_LEGAL-NOT:  %r4 = call nnan float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
 ; DISCARD_LEGAL-NOT:  %r5 = call nnan ninf float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
-; DISCARD_LEGAL-NOT:  %r6 = call float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
-; DISCARD_LEGAL-NOT:  %r7 = call reassoc float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
-; DISCARD_LEGAL-NOT:  %r8 = call float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
-; DISCARD_LEGAL-NOT:  %r9 = call reassoc float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
+; DISCARD_LEGAL-NOT:  %r6 = call float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; DISCARD_LEGAL-NOT:  %r7 = call nnan float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; DISCARD_LEGAL-NOT:  %r8 = call nnan ninf float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; DISCARD_LEGAL-NOT:  %r9 = call float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; DISCARD_LEGAL-NOT:  %r10 = call nnan float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; DISCARD_LEGAL-NOT:  %r11 = call nnan ninf float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; DISCARD_LEGAL-NOT:  %r12 = call float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; DISCARD_LEGAL-NOT:  %r13 = call reassoc float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; DISCARD_LEGAL-NOT:  %r14 = call float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
+; DISCARD_LEGAL-NOT:  %r15 = call reassoc float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 %n)
 ; DISCARD_LEGAL:      ret void
 
 ; DISCARD_LEGAL: define void @test_vp_cmp_v8(<8 x i32> %i0, <8 x i32> %i1, <8 x float> %f0, <8 x float> %f1, <8 x i1> %m, i32 %n) {
@@ -501,6 +542,12 @@ define void @test_vp_cmp_v8(<8 x i32> %i0, <8 x i32> %i1, <8 x float> %f0, <8 x
 ; CONVERT_LEGAL-NOT:   %{{.+}} = call float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
 ; CONVERT_LEGAL-NOT:   %{{.+}} = call nnan float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
 ; CONVERT_LEGAL-NOT:   %{{.+}} = call nnan ninf float @llvm.vp.reduce.fmax.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
+; CONVERT_LEGAL-NOT:   %{{.+}} = call float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> [[NEWM]], i32 4)
+; CONVERT_LEGAL-NOT:   %{{.+}} = call nnan float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
+; CONVERT_LEGAL-NOT:   %{{.+}} = call nnan ninf float @llvm.vp.reduce.fminimum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
+; CONVERT_LEGAL-NOT:   %{{.+}} = call float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
+; CONVERT_LEGAL-NOT:   %{{.+}} = call nnan float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
+; CONVERT_LEGAL-NOT:   %{{.+}} = call nnan ninf float @llvm.vp.reduce.fmaximum.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
 ; CONVERT_LEGAL-NOT:   %{{.+}} = call float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
 ; CONVERT_LEGAL-NOT:   %{{.+}} = call reassoc float @llvm.vp.reduce.fadd.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
 ; CONVERT_LEGAL-NOT:   %{{.+}} = call float @llvm.vp.reduce.fmul.v4f32(float %f, <4 x float> %vf, <4 x i1> %m, i32 4)
diff --git a/llvm/test/CodeGen/RISCV/rvv/undef-vp-ops.ll b/llvm/test/CodeGen/RISCV/rvv/undef-vp-ops.ll
index e9259b3a1d92..1bfc0f432eb5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/undef-vp-ops.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/undef-vp-ops.ll
@@ -631,3 +631,35 @@ define float @vreduce_fmax_v4f32_false_mask(float %start, <4 x float> %val, i32
   %s = call float @llvm.vp.reduce.fmax.v4f32(float %start, <4 x float> %val, <4 x i1> zeroinitializer, i32 %evl)
   ret float %s
 }
+
+define float @vreduce_fminimum_v4f32_zero_evl(float %start, <4 x float> %val, <4 x i1> %m) {
+; CHECK-LABEL: vreduce_fminimum_v4f32_zero_evl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %s = call float @llvm.vp.reduce.fminimum.v4f32(float %start, <4 x float> %val, <4 x i1> %m, i32 0)
+  ret float %s
+}
+
+define float @vreduce_fminimum_v4f32_false_mask(float %start, <4 x float> %val, i32 %evl) {
+; CHECK-LABEL: vreduce_fminimum_v4f32_false_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %s = call float @llvm.vp.reduce.fminimum.v4f32(float %start, <4 x float> %val, <4 x i1> zeroinitializer, i32 %evl)
+  ret float %s
+}
+
+define float @vreduce_fmaximum_v4f32_zero_evl(float %start, <4 x float> %val, <4 x i1> %m) {
+; CHECK-LABEL: vreduce_fmaximum_v4f32_zero_evl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %s = call float @llvm.vp.reduce.fmaximum.v4f32(float %start, <4 x float> %val, <4 x i1> %m, i32 0)
+  ret float %s
+}
+
+define float @vreduce_fmaximum_v4f32_false_mask(float %start, <4 x float> %val, i32 %evl) {
+; CHECK-LABEL: vreduce_fmaximum_v4f32_false_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %s = call float @llvm.vp.reduce.fmaximum.v4f32(float %start, <4 x float> %val, <4 x i1> zeroinitializer, i32 %evl)
+  ret float %s
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
index 4f7cb84c0864..46560fc501c6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vreductions-fp-vp.ll
@@ -579,3 +579,125 @@ define double @vpreduce_ord_fadd_nxv4f64(double %s, <vscale x 4 x double> %v, <v
   %r = call double @llvm.vp.reduce.fadd.nxv4f64(double %s, <vscale x 4 x double> %v, <vscale x 4 x i1> %m, i32 %evl)
   ret double %r
 }
+
+define float @vreduce_fminimum_nxv4f32(float %start, <vscale x 4 x float> %val, <vscale x 4 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vreduce_fminimum_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v10, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vfredmin.vs v10, v8, v10, v0.t
+; CHECK-NEXT:    vmfne.vv v11, v8, v8, v0.t
+; CHECK-NEXT:    vcpop.m a0, v11, v0.t
+; CHECK-NEXT:    feq.s a1, fa0, fa0
+; CHECK-NEXT:    xori a1, a1, 1
+; CHECK-NEXT:    or a0, a0, a1
+; CHECK-NEXT:    beqz a0, .LBB22_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lui a0, 523264
+; CHECK-NEXT:    fmv.w.x fa0, a0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB22_2:
+; CHECK-NEXT:    vfmv.f.s fa0, v10
+; CHECK-NEXT:    ret
+  %s = call float @llvm.vp.reduce.fminimum.nxv4f32(float %start, <vscale x 4 x float> %val, <vscale x 4 x i1> %m, i32 %evl)
+  ret float %s
+}
+
+define float @vreduce_fmaximum_nxv4f32(float %start, <vscale x 4 x float> %val, <vscale x 4 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vreduce_fmaximum_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v10, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vfredmax.vs v10, v8, v10, v0.t
+; CHECK-NEXT:    vmfne.vv v11, v8, v8, v0.t
+; CHECK-NEXT:    vcpop.m a0, v11, v0.t
+; CHECK-NEXT:    feq.s a1, fa0, fa0
+; CHECK-NEXT:    xori a1, a1, 1
+; CHECK-NEXT:    or a0, a0, a1
+; CHECK-NEXT:    beqz a0, .LBB23_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lui a0, 523264
+; CHECK-NEXT:    fmv.w.x fa0, a0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB23_2:
+; CHECK-NEXT:    vfmv.f.s fa0, v10
+; CHECK-NEXT:    ret
+  %s = call float @llvm.vp.reduce.fmaximum.nxv4f32(float %start, <vscale x 4 x float> %val, <vscale x 4 x i1> %m, i32 %evl)
+  ret float %s
+}
+
+define float @vreduce_fminimum_nnan_nxv4f32(float %start, <vscale x 4 x float> %val, <vscale x 4 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vreduce_fminimum_nnan_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v10, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vfredmin.vs v10, v8, v10, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v10
+; CHECK-NEXT:    ret
+  %s = call nnan float @llvm.vp.reduce.fminimum.nxv4f32(float %start, <vscale x 4 x float> %val, <vscale x 4 x i1> %m, i32 %evl)
+  ret float %s
+}
+
+define float @vreduce_fmaximum_nnan_nxv4f32(float %start, <vscale x 4 x float> %val, <vscale x 4 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vreduce_fmaximum_nnan_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v10, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vfredmax.vs v10, v8, v10, v0.t
+; CHECK-NEXT:    vfmv.f.s fa0, v10
+; CHECK-NEXT:    ret
+  %s = call nnan float @llvm.vp.reduce.fmaximum.nxv4f32(float %start, <vscale x 4 x float> %val, <vscale x 4 x i1> %m, i32 %evl)
+  ret float %s
+}
+
+define float @vreduce_fminimum_v4f32(float %start, <4 x float> %val, <4 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vreduce_fminimum_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vfredmin.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vmfne.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vcpop.m a0, v8, v0.t
+; CHECK-NEXT:    feq.s a1, fa0, fa0
+; CHECK-NEXT:    xori a1, a1, 1
+; CHECK-NEXT:    or a0, a0, a1
+; CHECK-NEXT:    beqz a0, .LBB26_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lui a0, 523264
+; CHECK-NEXT:    fmv.w.x fa0, a0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB26_2:
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
+  %s = call float @llvm.vp.reduce.fminimum.v4f32(float %start, <4 x float> %val, <4 x i1> %m, i32 %evl)
+  ret float %s
+}
+
+define float @vreduce_fmaximum_v4f32(float %start, <4 x float> %val, <4 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: vreduce_fmaximum_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v9, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vfredmax.vs v9, v8, v9, v0.t
+; CHECK-NEXT:    vmfne.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vcpop.m a0, v8, v0.t
+; CHECK-NEXT:    feq.s a1, fa0, fa0
+; CHECK-NEXT:    xori a1, a1, 1
+; CHECK-NEXT:    or a0, a0, a1
+; CHECK-NEXT:    beqz a0, .LBB27_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lui a0, 523264
+; CHECK-NEXT:    fmv.w.x fa0, a0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB27_2:
+; CHECK-NEXT:    vfmv.f.s fa0, v9
+; CHECK-NEXT:    ret
+  %s = call float @llvm.vp.reduce.fmaximum.v4f32(float %start, <4 x float> %val, <4 x i1> %m, i32 %evl)
+  ret float %s
+}
diff --git a/llvm/test/Verifier/vp-intrinsics.ll b/llvm/test/Verifier/vp-intrinsics.ll
index 765d67356c2b..9ed8279f94d7 100644
--- a/llvm/test/Verifier/vp-intrinsics.ll
+++ b/llvm/test/Verifier/vp-intrinsics.ll
@@ -43,8 +43,10 @@ define void @test_vp_reduction(i32 %x, <8 x i32> %vi, <8 x float> %vf, float %f,
   %r8 = call i32 @llvm.vp.reduce.umin.v8i32(i32 %x, <8 x i32> %vi, <8 x i1> %m, i32 %n)
   %r9 = call float @llvm.vp.reduce.fmin.v8f32(float %f, <8 x float> %vf, <8 x i1> %m, i32 %n)
   %rA = call float @llvm.vp.reduce.fmax.v8f32(float %f, <8 x float> %vf, <8 x i1> %m, i32 %n)
-  %rB = call float @llvm.vp.reduce.fadd.v8f32(float %f, <8 x float> %vf, <8 x i1> %m, i32 %n)
-  %rC = call float @llvm.vp.reduce.fmul.v8f32(float %f, <8 x float> %vf, <8 x i1> %m, i32 %n)
+  %rB = call float @llvm.vp.reduce.fminimum.v8f32(float %f, <8 x float> %vf, <8 x i1> %m, i32 %n)
+  %rC = call float @llvm.vp.reduce.fmaximum.v8f32(float %f, <8 x float> %vf, <8 x i1> %m, i32 %n)
+  %rD = call float @llvm.vp.reduce.fadd.v8f32(float %f, <8 x float> %vf, <8 x i1> %m, i32 %n)
+  %rE = call float @llvm.vp.reduce.fmul.v8f32(float %f, <8 x float> %vf, <8 x i1> %m, i32 %n)
   ret void
 }
 
@@ -113,6 +115,8 @@ declare i32 @llvm.vp.reduce.umax.v8i32(i32, <8 x i32>, <8 x i1>, i32)
 declare i32 @llvm.vp.reduce.umin.v8i32(i32, <8 x i32>, <8 x i1>, i32)
 declare float @llvm.vp.reduce.fmin.v8f32(float, <8 x float>, <8 x i1>, i32)
 declare float @llvm.vp.reduce.fmax.v8f32(float, <8 x float>, <8 x i1>, i32)
+declare float @llvm.vp.reduce.fminimum.v8f32(float, <8 x float>, <8 x i1>, i32)
+declare float @llvm.vp.reduce.fmaximum.v8f32(float, <8 x float>, <8 x i1>, i32)
 declare float @llvm.vp.reduce.fadd.v8f32(float, <8 x float>, <8 x i1>, i32)
 declare float @llvm.vp.reduce.fmul.v8f32(float, <8 x float>, <8 x i1>, i32)
 ; casts
diff --git a/llvm/unittests/IR/VPIntrinsicTest.cpp b/llvm/unittests/IR/VPIntrinsicTest.cpp
index 626ab2e9a9c5..d6508abd5197 100644
--- a/llvm/unittests/IR/VPIntrinsicTest.cpp
+++ b/llvm/unittests/IR/VPIntrinsicTest.cpp
@@ -27,7 +27,8 @@ namespace {
 static const char *ReductionIntOpcodes[] = {
     "add", "mul", "and", "or", "xor", "smin", "smax", "umin", "umax"};
 
-static const char *ReductionFPOpcodes[] = {"fadd", "fmul", "fmin", "fmax"};
+static const char *ReductionFPOpcodes[] = {"fadd", "fmul",     "fmin",
+                                           "fmax", "fminimum", "fmaximum"};
 
 class VPIntrinsicTest : public testing::Test {
 protected:
-- 
GitLab


From 504cf554639360525c3f746e7296a242350b2af9 Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Fri, 10 May 2024 17:17:38 -0700
Subject: [PATCH 0466/1206] [InstallAPI] Pass explicit module cache to avoid
 permissions issues.

Fixes: https://lab.llvm.org/buildbot/#/builders/192/builds/9313
---
 clang/test/InstallAPI/project-header-only-args.test | 3 +++
 clang/tools/clang-installapi/Options.cpp            | 7 +++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/clang/test/InstallAPI/project-header-only-args.test b/clang/test/InstallAPI/project-header-only-args.test
index 7147c83b0f5d..76fecce5b4a2 100644
--- a/clang/test/InstallAPI/project-header-only-args.test
+++ b/clang/test/InstallAPI/project-header-only-args.test
@@ -1,11 +1,13 @@
 ; RUN: rm -rf %t
 ; RUN: split-file %s %t
 ; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json
+; RUN: mkdir -p %t/modules.cache
 
 ; RUN: clang-installapi \
 ; RUN: -target arm64-apple-macos12 -install_name @rpath/libfoo.dylib \
 ; RUN: -current_version 1 -compatibility_version 1 \
 ; RUN: -Xproject -fmodules -I%t/usr/include \
+; RUN: -Xproject -fmodules-cache-path=%t/modules.cache \
 ; RUN: -F %S/Inputs/Foundation/ \
 ; RUN: -exclude-public-header %t/usr/include/public.h \
 ; RUN: -extra-project-header %t/project.h -I%t -dynamiclib \
@@ -17,6 +19,7 @@
 ; RUN: -target arm64-apple-macos12 -install_name @rpath/libfoo.dylib \
 ; RUN: -current_version 1 -compatibility_version 1 \
 ; RUN: -Xproject -fmodules -I%t/usr/include \
+; RUN: -Xproject -fmodules-cache-path=%t/modules.cache \
 ; RUN: -extra-project-header %t/project.h \
 ; RUN: -F %S/Inputs/Foundation/ \
 ; RUN: %t/inputs.json \
diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index 5396ad23620b..53340da704fc 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -299,8 +299,11 @@ bool Options::processXprojectOption(InputArgList &Args, arg_iterator Curr) {
     return false;
   }
 
-  ProjectLevelArgs.push_back(NextA->getSpelling().str());
-  llvm::copy(NextA->getValues(), std::back_inserter(ProjectLevelArgs));
+  std::string ArgString = NextA->getSpelling().str();
+  for (const StringRef Val : NextA->getValues())
+    ArgString += Val.str();
+
+  ProjectLevelArgs.push_back(ArgString);
   A->claim();
   NextA->claim();
 
-- 
GitLab


From 38b275598391dd6bd2d034e6b815c3febde0b1fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 10 May 2024 17:51:30 -0700
Subject: [PATCH 0467/1206] Revert "[flang][cuda] Implicitly load cudadevice
 module in device/global subprogram" (#91827)

Reverts llvm/llvm-project#91668
---
 flang/include/flang/Semantics/semantics.h     |  5 +-
 flang/lib/Semantics/check-cuda.cpp            |  5 --
 flang/lib/Semantics/resolve-names.cpp         | 13 ----
 flang/lib/Semantics/semantics.cpp             |  8 --
 flang/module/__cuda_device_builtins.f90       | 74 -------------------
 flang/module/cudadevice.f90                   | 21 ------
 .../Semantics/cuf-device-procedures01.cuf     | 35 ---------
 .../Semantics/cuf-device-procedures02.cuf     | 17 -----
 flang/tools/f18/CMakeLists.txt                |  4 -
 9 files changed, 1 insertion(+), 181 deletions(-)
 delete mode 100644 flang/module/__cuda_device_builtins.f90
 delete mode 100644 flang/module/cudadevice.f90
 delete mode 100644 flang/test/Semantics/cuf-device-procedures01.cuf
 delete mode 100644 flang/test/Semantics/cuf-device-procedures02.cuf

diff --git a/flang/include/flang/Semantics/semantics.h b/flang/include/flang/Semantics/semantics.h
index 167e61381639..e6ba71d53e92 100644
--- a/flang/include/flang/Semantics/semantics.h
+++ b/flang/include/flang/Semantics/semantics.h
@@ -215,10 +215,8 @@ public:
   void UseFortranBuiltinsModule();
   const Scope *GetBuiltinsScope() const { return builtinsScope_; }
 
-  const Scope &GetCUDABuiltinsScope();
-  const Scope &GetCUDADeviceScope();
-
   void UsePPCBuiltinTypesModule();
+  const Scope &GetCUDABuiltinsScope();
   void UsePPCBuiltinsModule();
   Scope *GetPPCBuiltinTypesScope() { return ppcBuiltinTypesScope_; }
   const Scope *GetPPCBuiltinsScope() const { return ppcBuiltinsScope_; }
@@ -294,7 +292,6 @@ private:
   const Scope *builtinsScope_{nullptr}; // module __Fortran_builtins
   Scope *ppcBuiltinTypesScope_{nullptr}; // module __Fortran_PPC_types
   std::optional<const Scope *> cudaBuiltinsScope_; // module __CUDA_builtins
-  std::optional<const Scope *> cudaDeviceScope_; // module cudadevice
   const Scope *ppcBuiltinsScope_{nullptr}; // module __ppc_intrinsics
   std::list<parser::Program> modFileParseTrees_;
   std::unique_ptr<CommonBlockMap> commonBlockMap_;
diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp
index 2830d5f0be6e..96ab90239263 100644
--- a/flang/lib/Semantics/check-cuda.cpp
+++ b/flang/lib/Semantics/check-cuda.cpp
@@ -82,11 +82,6 @@ struct DeviceExprChecker
           }
         }
       }
-      if (sym->owner().IsModule() &&
-          sym->owner().parent().IsIntrinsicModules() &&
-          DEREF(sym->owner().symbol()).name() == "__cuda_device_builtins") {
-        return {};
-      }
     } else if (x.GetSpecificIntrinsic()) {
       // TODO(CUDA): Check for unsupported intrinsics here
       return {};
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 7cb0c6af937c..e2875081b732 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -3797,19 +3797,6 @@ bool SubprogramVisitor::Pre(const parser::PrefixSpec::Attributes &attrs) {
         subp->set_cudaSubprogramAttrs(attr);
       }
     }
-    if (auto attrs{subp->cudaSubprogramAttrs()}) {
-      if (*attrs == common::CUDASubprogramAttrs::Global ||
-          *attrs == common::CUDASubprogramAttrs::Device) {
-        // Implicitly USE the cudadevice module by copying its symbols in the
-        // current scope.
-        const Scope &scope{context().GetCUDADeviceScope()};
-        for (auto sym : scope.GetSymbols()) {
-          if (!currScope().FindSymbol(sym->name())) {
-            currScope().CopySymbol(sym);
-          }
-        }
-      }
-    }
   }
   return false;
 }
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index d51cc62d804e..6ccd915c4dcb 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -543,14 +543,6 @@ const Scope &SemanticsContext::GetCUDABuiltinsScope() {
   return **cudaBuiltinsScope_;
 }
 
-const Scope &SemanticsContext::GetCUDADeviceScope() {
-  if (!cudaDeviceScope_) {
-    cudaDeviceScope_ = GetBuiltinModule("cudadevice");
-    CHECK(cudaDeviceScope_.value() != nullptr);
-  }
-  return **cudaDeviceScope_;
-}
-
 void SemanticsContext::UsePPCBuiltinsModule() {
   if (ppcBuiltinsScope_ == nullptr) {
     ppcBuiltinsScope_ = GetBuiltinModule("__ppc_intrinsics");
diff --git a/flang/module/__cuda_device_builtins.f90 b/flang/module/__cuda_device_builtins.f90
deleted file mode 100644
index 738dc97242f2..000000000000
--- a/flang/module/__cuda_device_builtins.f90
+++ /dev/null
@@ -1,74 +0,0 @@
-!===-- module/__cuda_device_builtins.f90 -----------------------------------===!
-!
-! Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-! See https://llvm.org/LICENSE.txt for license information.
-! SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-!
-!===------------------------------------------------------------------------===!
-
-! CUDA Fortran procedures available in device subprogram
-
-module __CUDA_device_builtins  
-
-  implicit none
-
-  ! Set PRIVATE by default to explicitly only export what is meant
-  ! to be exported by this MODULE.
-  private
-
-  ! Synchronization Functions
-
-  interface
-    subroutine __cuda_device_builtins_syncthreads()
-    end subroutine
-  end interface
-  public :: __cuda_device_builtins_syncthreads
-
-  interface
-    integer function __cuda_device_builtins_syncthreads_and(value)
-      integer :: value
-    end function
-  end interface
-  public :: __cuda_device_builtins_syncthreads_and
-
-  interface
-    integer function __cuda_device_builtins_syncthreads_count(value)
-      integer :: value
-    end function
-  end interface
-  public :: __cuda_device_builtins_syncthreads_count
-
-  interface
-    integer function __cuda_device_builtins_syncthreads_or(int_value)
-    end function
-  end interface
-  public :: __cuda_device_builtins_syncthreads_or
-
-  interface
-    subroutine __cuda_device_builtins_syncwarp(mask)
-      integer :: mask
-    end subroutine
-  end interface
-  public :: __cuda_device_builtins_syncwarp
-
-  ! Memory Fences
-
-  interface
-    subroutine __cuda_device_builtins_threadfence()
-    end subroutine
-  end interface
-  public :: __cuda_device_builtins_threadfence
-
-  interface
-    subroutine __cuda_device_builtins_threadfence_block()
-    end subroutine
-  end interface
-  public :: __cuda_device_builtins_threadfence_block
-
-  interface
-    subroutine __cuda_device_builtins_threadfence_system()
-    end subroutine
-  end interface
-  public :: __cuda_device_builtins_threadfence_system
-
-end module
diff --git a/flang/module/cudadevice.f90 b/flang/module/cudadevice.f90
deleted file mode 100644
index b635d77ea452..000000000000
--- a/flang/module/cudadevice.f90
+++ /dev/null
@@ -1,21 +0,0 @@
-!===-- module/cudedevice.f90 -----------------------------------------------===!
-!
-! Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-! See https://llvm.org/LICENSE.txt for license information.
-! SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-!
-!===------------------------------------------------------------------------===!
-
-! CUDA Fortran procedures available in device subprogram
-
-module cudadevice
-  use __cuda_device_builtins, only: &
-    syncthreads => __cuda_device_builtins_syncthreads, &
-    syncthreads_and => __cuda_device_builtins_syncthreads_and, &
-    syncthreads_count => __cuda_device_builtins_syncthreads_count, &
-    syncthreads_or => __cuda_device_builtins_syncthreads_or, &
-    syncwarp => __cuda_device_builtins_syncwarp, &
-    threadfence => __cuda_device_builtins_threadfence, &
-    threadfence_block => __cuda_device_builtins_threadfence_block, &
-    threadfence_system => __cuda_device_builtins_threadfence_system
-end module
diff --git a/flang/test/Semantics/cuf-device-procedures01.cuf b/flang/test/Semantics/cuf-device-procedures01.cuf
deleted file mode 100644
index e79423e3587a..000000000000
--- a/flang/test/Semantics/cuf-device-procedures01.cuf
+++ /dev/null
@@ -1,35 +0,0 @@
-! RUN: %flang_fc1 -fdebug-dump-symbols %s | FileCheck %s
-
-! Test CUDA Fortran intrinsic can pass semantic
-
-attributes(global) subroutine devsub()
-  implicit none
-  integer :: ret
-
-  ! 3.6.4. Synchronization Functions
-  call syncthreads()
-  call syncwarp(1)
-  call threadfence()
-  call threadfence_block()
-  call threadfence_system()
-  ret = syncthreads_and(1)
-  ret = syncthreads_count(1)
-  ret = syncthreads_or(1)
-end
-
-! CHECK-LABEL: Subprogram scope: devsub
-! CHECK: syncthreads, EXTERNAL, PUBLIC (Subroutine): Use from __cuda_device_builtins_syncthreads in __cuda_device_builtins
-! CHECK: syncthreads_and, EXTERNAL, PUBLIC (Function): Use from __cuda_device_builtins_syncthreads_and in __cuda_device_builtins
-! CHECK: syncthreads_count, EXTERNAL, PUBLIC (Function): Use from __cuda_device_builtins_syncthreads_count in __cuda_device_builtins
-! CHECK: syncthreads_or, EXTERNAL, PUBLIC (Function): Use from __cuda_device_builtins_syncthreads_or in __cuda_device_builtins
-! CHECK: syncwarp, EXTERNAL, PUBLIC (Subroutine): Use from __cuda_device_builtins_syncwarp in __cuda_device_builtins
-! CHECK: threadfence, EXTERNAL, PUBLIC (Subroutine): Use from __cuda_device_builtins_threadfence in __cuda_device_builtins
-! CHECK: threadfence_block, EXTERNAL, PUBLIC (Subroutine): Use from __cuda_device_builtins_threadfence_block in __cuda_device_builtins
-! CHECK: threadfence_system, EXTERNAL, PUBLIC (Subroutine): Use from __cuda_device_builtins_threadfence_system in __cuda_device_builtins
-
-subroutine host()
-  call syncthreads()
-end subroutine
-
-! CHECK-LABEL: Subprogram scope: host
-! CHECK: syncthreads, EXTERNAL: HostAssoc{{$}}
diff --git a/flang/test/Semantics/cuf-device-procedures02.cuf b/flang/test/Semantics/cuf-device-procedures02.cuf
deleted file mode 100644
index ea6a094ed5c3..000000000000
--- a/flang/test/Semantics/cuf-device-procedures02.cuf
+++ /dev/null
@@ -1,17 +0,0 @@
-! RUN: %python %S/test_errors.py %s %flang_fc1
-
-module dev
-  integer, device :: syncthreads
-
-contains
-
-  attributes(device) subroutine sub1()
-    syncthreads = 1 ! syncthreads not overwritten by cudadevice
-  end subroutine
-
-  attributes(global) subroutine sub2()
-!ERROR: 'threadfence' is use-associated from module '__cuda_device_builtins' and cannot be re-declared
-    integer :: threadfence
-  end subroutine
-end module
-
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
index 0222654c8e5d..64815a1f5da6 100644
--- a/flang/tools/f18/CMakeLists.txt
+++ b/flang/tools/f18/CMakeLists.txt
@@ -12,8 +12,6 @@ set(MODULES
   "__ppc_intrinsics"
   "mma"
   "__cuda_builtins"
-  "__cuda_device_builtins"
-  "cudadevice"
   "ieee_arithmetic"
   "ieee_exceptions"
   "ieee_features"
@@ -33,8 +31,6 @@ if (NOT CMAKE_CROSSCOMPILING)
     elseif(${filename} STREQUAL "__ppc_intrinsics" OR
            ${filename} STREQUAL "mma")
       set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "cudadevice")
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device_builtins.mod)
     else()
       set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
       if(NOT ${filename} STREQUAL "__fortran_type_info")
-- 
GitLab


From 11cb3c300d8e8f05dddbc1a4f8ab1c86e864544a Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Tue, 7 May 2024 11:58:14 -0500
Subject: [PATCH 0468/1206] [IR][PatternMatch] Make `m_Checked{Int,Fp}` accept
 `Constant *` output instead of `APInt *`

The `APInt *` version is pretty useless as any case one needs an
`APInt *` out, they could just replace whatever they have the
`m_Checked...` lambda with direct checks on the `APInt`.

Leaving other helpers such as `m_Negative`, `m_Power2`,
etc... unchanged as the `APInt` out version is used mostly for
convenience and rarely change functionality when converted output a
`Constant *`.

Closes #91377
---
 llvm/include/llvm/IR/PatternMatch.h |  32 +++++----
 llvm/unittests/IR/PatternMatch.cpp  | 104 ++++++++++++++--------------
 2 files changed, 72 insertions(+), 64 deletions(-)

diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 5d8f5c134bb5..171ddab977de 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -354,7 +354,8 @@ template <int64_t Val> inline constantint_match<Val> m_ConstantInt() {
 /// is true.
 template <typename Predicate, typename ConstantVal, bool AllowPoison>
 struct cstval_pred_ty : public Predicate {
-  template <typename ITy> bool match(ITy *V) {
+  const Constant **Res = nullptr;
+  template <typename ITy> bool match_impl(ITy *V) {
     if (const auto *CV = dyn_cast<ConstantVal>(V))
       return this->isValue(CV->getValue());
     if (const auto *VTy = dyn_cast<VectorType>(V->getType())) {
@@ -387,6 +388,15 @@ struct cstval_pred_ty : public Predicate {
     }
     return false;
   }
+
+  template <typename ITy> bool match(ITy *V) {
+    if (this->match_impl(V)) {
+      if (Res)
+        *Res = cast<Constant>(V);
+      return true;
+    }
+    return false;
+  }
 };
 
 /// specialization of cstval_pred_ty for ConstantInt
@@ -469,28 +479,24 @@ template <typename APTy> struct custom_checkfn {
 /// For vectors, poison elements are assumed to match.
 inline cst_pred_ty<custom_checkfn<APInt>>
 m_CheckedInt(function_ref<bool(const APInt &)> CheckFn) {
-  return cst_pred_ty<custom_checkfn<APInt>>{CheckFn};
+  return cst_pred_ty<custom_checkfn<APInt>>{{CheckFn}};
 }
 
-inline api_pred_ty<custom_checkfn<APInt>>
-m_CheckedInt(const APInt *&V, function_ref<bool(const APInt &)> CheckFn) {
-  api_pred_ty<custom_checkfn<APInt>> P(V);
-  P.CheckFn = CheckFn;
-  return P;
+inline cst_pred_ty<custom_checkfn<APInt>>
+m_CheckedInt(const Constant *&V, function_ref<bool(const APInt &)> CheckFn) {
+  return cst_pred_ty<custom_checkfn<APInt>>{{CheckFn}, &V};
 }
 
 /// Match a float or vector where CheckFn(ele) for each element is true.
 /// For vectors, poison elements are assumed to match.
 inline cstfp_pred_ty<custom_checkfn<APFloat>>
 m_CheckedFp(function_ref<bool(const APFloat &)> CheckFn) {
-  return cstfp_pred_ty<custom_checkfn<APFloat>>{CheckFn};
+  return cstfp_pred_ty<custom_checkfn<APFloat>>{{CheckFn}};
 }
 
-inline apf_pred_ty<custom_checkfn<APFloat>>
-m_CheckedFp(const APFloat *&V, function_ref<bool(const APFloat &)> CheckFn) {
-  apf_pred_ty<custom_checkfn<APFloat>> P(V);
-  P.CheckFn = CheckFn;
-  return P;
+inline cstfp_pred_ty<custom_checkfn<APFloat>>
+m_CheckedFp(const Constant *&V, function_ref<bool(const APFloat &)> CheckFn) {
+  return cstfp_pred_ty<custom_checkfn<APFloat>>{{CheckFn}, &V};
 }
 
 struct is_any_apint {
diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp
index d5a4a6a05687..9f91b4f3f993 100644
--- a/llvm/unittests/IR/PatternMatch.cpp
+++ b/llvm/unittests/IR/PatternMatch.cpp
@@ -613,8 +613,7 @@ TEST_F(PatternMatchTest, BitCast) {
 
 TEST_F(PatternMatchTest, CheckedInt) {
   Type *I8Ty = IRB.getInt8Ty();
-  const APInt *Res = nullptr;
-
+  const Constant * CRes = nullptr;
   auto CheckUgt1 = [](const APInt &C) { return C.ugt(1); };
   auto CheckTrue = [](const APInt &) { return true; };
   auto CheckFalse = [](const APInt &) { return false; };
@@ -625,38 +624,33 @@ TEST_F(PatternMatchTest, CheckedInt) {
     APInt APVal(8, Val);
     Constant *C = ConstantInt::get(I8Ty, Val);
 
-    Res = nullptr;
+    CRes = nullptr;
     EXPECT_TRUE(m_CheckedInt(CheckTrue).match(C));
-    EXPECT_TRUE(m_CheckedInt(Res, CheckTrue).match(C));
-    EXPECT_EQ(*Res, APVal);
+    EXPECT_TRUE(m_CheckedInt(CRes, CheckTrue).match(C));
+    EXPECT_EQ(CRes, C);
 
-    Res = nullptr;
+    CRes = nullptr;
     EXPECT_FALSE(m_CheckedInt(CheckFalse).match(C));
-    EXPECT_FALSE(m_CheckedInt(Res, CheckFalse).match(C));
+    EXPECT_FALSE(m_CheckedInt(CRes, CheckFalse).match(C));
+    EXPECT_EQ(CRes, nullptr);
 
-    Res = nullptr;
+    CRes = nullptr;
     EXPECT_EQ(CheckUgt1(APVal), m_CheckedInt(CheckUgt1).match(C));
-    EXPECT_EQ(CheckUgt1(APVal), m_CheckedInt(Res, CheckUgt1).match(C));
-    if (CheckUgt1(APVal)) {
-      EXPECT_NE(Res, nullptr);
-      EXPECT_EQ(*Res, APVal);
-    }
+    EXPECT_EQ(CheckUgt1(APVal), m_CheckedInt(CRes, CheckUgt1).match(C));
+    if (CheckUgt1(APVal))
+      EXPECT_EQ(CRes, C);
 
-    Res = nullptr;
+    CRes = nullptr;
     EXPECT_EQ(CheckNonZero(APVal), m_CheckedInt(CheckNonZero).match(C));
-    EXPECT_EQ(CheckNonZero(APVal), m_CheckedInt(Res, CheckNonZero).match(C));
-    if (CheckNonZero(APVal)) {
-      EXPECT_NE(Res, nullptr);
-      EXPECT_EQ(*Res, APVal);
-    }
+    EXPECT_EQ(CheckNonZero(APVal), m_CheckedInt(CRes, CheckNonZero).match(C));
+    if (CheckNonZero(APVal))
+      EXPECT_EQ(CRes, C);
 
-    Res = nullptr;
+    CRes = nullptr;
     EXPECT_EQ(CheckPow2(APVal), m_CheckedInt(CheckPow2).match(C));
-    EXPECT_EQ(CheckPow2(APVal), m_CheckedInt(Res, CheckPow2).match(C));
-    if (CheckPow2(APVal)) {
-      EXPECT_NE(Res, nullptr);
-      EXPECT_EQ(*Res, APVal);
-    }
+    EXPECT_EQ(CheckPow2(APVal), m_CheckedInt(CRes, CheckPow2).match(C));
+    if (CheckPow2(APVal))
+      EXPECT_EQ(CRes, C);
 
   };
 
@@ -666,20 +660,20 @@ TEST_F(PatternMatchTest, CheckedInt) {
   DoScalarCheck(3);
 
   EXPECT_FALSE(m_CheckedInt(CheckTrue).match(UndefValue::get(I8Ty)));
-  EXPECT_FALSE(m_CheckedInt(Res, CheckTrue).match(UndefValue::get(I8Ty)));
-  EXPECT_EQ(Res, nullptr);
+  EXPECT_FALSE(m_CheckedInt(CRes, CheckTrue).match(UndefValue::get(I8Ty)));
+  EXPECT_EQ(CRes, nullptr);
 
   EXPECT_FALSE(m_CheckedInt(CheckFalse).match(UndefValue::get(I8Ty)));
-  EXPECT_FALSE(m_CheckedInt(Res, CheckFalse).match(UndefValue::get(I8Ty)));
-  EXPECT_EQ(Res, nullptr);
+  EXPECT_FALSE(m_CheckedInt(CRes, CheckFalse).match(UndefValue::get(I8Ty)));
+  EXPECT_EQ(CRes, nullptr);
 
   EXPECT_FALSE(m_CheckedInt(CheckTrue).match(PoisonValue::get(I8Ty)));
-  EXPECT_FALSE(m_CheckedInt(Res, CheckTrue).match(PoisonValue::get(I8Ty)));
-  EXPECT_EQ(Res, nullptr);
+  EXPECT_FALSE(m_CheckedInt(CRes, CheckTrue).match(PoisonValue::get(I8Ty)));
+  EXPECT_EQ(CRes, nullptr);
 
   EXPECT_FALSE(m_CheckedInt(CheckFalse).match(PoisonValue::get(I8Ty)));
-  EXPECT_FALSE(m_CheckedInt(Res, CheckFalse).match(PoisonValue::get(I8Ty)));
-  EXPECT_EQ(Res, nullptr);
+  EXPECT_FALSE(m_CheckedInt(CRes, CheckFalse).match(PoisonValue::get(I8Ty)));
+  EXPECT_EQ(CRes, nullptr);
 
   auto DoVecCheckImpl = [&](ArrayRef<std::optional<int8_t>> Vals,
                             function_ref<bool(const APInt &)> CheckFn,
@@ -711,13 +705,13 @@ TEST_F(PatternMatchTest, CheckedInt) {
     EXPECT_EQ(!(HasUndef && !UndefAsPoison) && Okay.value_or(false),
               m_CheckedInt(CheckFn).match(C));
 
-    Res = nullptr;
-    bool Expec =
-        !(HasUndef && !UndefAsPoison) && AllSame && Okay.value_or(false);
-    EXPECT_EQ(Expec, m_CheckedInt(Res, CheckFn).match(C));
+    CRes = nullptr;
+    bool Expec = !(HasUndef && !UndefAsPoison) && Okay.value_or(false);
+    EXPECT_EQ(Expec, m_CheckedInt(CRes, CheckFn).match(C));
     if (Expec) {
-      EXPECT_NE(Res, nullptr);
-      EXPECT_EQ(*Res, *First);
+      EXPECT_NE(CRes, nullptr);
+      if (AllSame)
+        EXPECT_EQ(CRes, C);
     }
   };
   auto DoVecCheck = [&](ArrayRef<std::optional<int8_t>> Vals) {
@@ -1559,24 +1553,25 @@ TEST_F(PatternMatchTest, VectorUndefFloat) {
   EXPECT_FALSE(match(VectorNaNPoison, m_CheckedFp(CheckNonNaN)));
 
   const APFloat *C;
+  const Constant *CC;
   // Regardless of whether poison is allowed,
   // a fully undef/poison constant does not match.
   EXPECT_FALSE(match(ScalarUndef, m_APFloat(C)));
   EXPECT_FALSE(match(ScalarUndef, m_APFloatForbidPoison(C)));
   EXPECT_FALSE(match(ScalarUndef, m_APFloatAllowPoison(C)));
-  EXPECT_FALSE(match(ScalarUndef, m_CheckedFp(C, CheckTrue)));
+  EXPECT_FALSE(match(ScalarUndef, m_CheckedFp(CC, CheckTrue)));
   EXPECT_FALSE(match(VectorUndef, m_APFloat(C)));
   EXPECT_FALSE(match(VectorUndef, m_APFloatForbidPoison(C)));
   EXPECT_FALSE(match(VectorUndef, m_APFloatAllowPoison(C)));
-  EXPECT_FALSE(match(VectorUndef, m_CheckedFp(C, CheckTrue)));
+  EXPECT_FALSE(match(VectorUndef, m_CheckedFp(CC, CheckTrue)));
   EXPECT_FALSE(match(ScalarPoison, m_APFloat(C)));
   EXPECT_FALSE(match(ScalarPoison, m_APFloatForbidPoison(C)));
   EXPECT_FALSE(match(ScalarPoison, m_APFloatAllowPoison(C)));
-  EXPECT_FALSE(match(ScalarPoison, m_CheckedFp(C, CheckTrue)));
+  EXPECT_FALSE(match(ScalarPoison, m_CheckedFp(CC, CheckTrue)));
   EXPECT_FALSE(match(VectorPoison, m_APFloat(C)));
   EXPECT_FALSE(match(VectorPoison, m_APFloatForbidPoison(C)));
   EXPECT_FALSE(match(VectorPoison, m_APFloatAllowPoison(C)));
-  EXPECT_FALSE(match(VectorPoison, m_CheckedFp(C, CheckTrue)));
+  EXPECT_FALSE(match(VectorPoison, m_CheckedFp(CC, CheckTrue)));
 
   // We can always match simple constants and simple splats.
   C = nullptr;
@@ -1597,12 +1592,13 @@ TEST_F(PatternMatchTest, VectorUndefFloat) {
   C = nullptr;
   EXPECT_TRUE(match(VectorZero, m_APFloatAllowPoison(C)));
   EXPECT_TRUE(C->isZero());
-  C = nullptr;
-  EXPECT_TRUE(match(VectorZero, m_CheckedFp(C, CheckTrue)));
-  EXPECT_TRUE(C->isZero());
-  C = nullptr;
-  EXPECT_TRUE(match(VectorZero, m_CheckedFp(C, CheckNonNaN)));
-  EXPECT_TRUE(C->isZero());
+
+  CC = nullptr;
+  EXPECT_TRUE(match(VectorZero, m_CheckedFp(CC, CheckTrue)));
+  EXPECT_TRUE(CC->isNullValue());
+  CC = nullptr;
+  EXPECT_TRUE(match(VectorZero, m_CheckedFp(CC, CheckNonNaN)));
+  EXPECT_TRUE(CC->isNullValue());
 
   // Splats with undef are never allowed.
   // Whether splats with poison can be matched depends on the matcher.
@@ -1627,11 +1623,17 @@ TEST_F(PatternMatchTest, VectorUndefFloat) {
   C = nullptr;
   EXPECT_TRUE(match(VectorZeroPoison, m_Finite(C)));
   EXPECT_TRUE(C->isZero());
+  CC = nullptr;
   C = nullptr;
-  EXPECT_TRUE(match(VectorZeroPoison, m_CheckedFp(C, CheckTrue)));
+  EXPECT_TRUE(match(VectorZeroPoison, m_CheckedFp(CC, CheckTrue)));
+  EXPECT_NE(CC, nullptr);
+  EXPECT_TRUE(match(CC, m_APFloatAllowPoison(C)));
   EXPECT_TRUE(C->isZero());
+  CC = nullptr;
   C = nullptr;
-  EXPECT_TRUE(match(VectorZeroPoison, m_CheckedFp(C, CheckNonNaN)));
+  EXPECT_TRUE(match(VectorZeroPoison, m_CheckedFp(CC, CheckNonNaN)));
+  EXPECT_NE(CC, nullptr);
+  EXPECT_TRUE(match(CC, m_APFloatAllowPoison(C)));
   EXPECT_TRUE(C->isZero());
 }
 
-- 
GitLab


From df88d610ab10c1e3f22d7f5daf3794158972c584 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 10 May 2024 19:15:17 -0700
Subject: [PATCH 0469/1206] [Driver] Remove unused getInstalledDir

The uses have been removed by commit 2b5cd8be3af43e5aa5b76b6aeb1edd3141b803ca
---
 clang/include/clang/Driver/Driver.h | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 2ffc52bcb7ad..cc1538372d5f 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -424,11 +424,6 @@ public:
     return ClangExecutable.c_str();
   }
 
-  /// Get the path to where the clang executable was installed.
-  const char *getInstalledDir() const {
-    return Dir.c_str();
-  }
-
   bool isSaveTempsEnabled() const { return SaveTemps != SaveTempsNone; }
   bool isSaveTempsObj() const { return SaveTemps == SaveTempsObj; }
 
-- 
GitLab


From e62ce1f8842cca36eb14126d79dcca0a85bf6d36 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Fri, 10 May 2024 19:27:02 -0700
Subject: [PATCH 0470/1206] [clang-format] Fix
 FormatToken::isSimpleTypeSpecifier() (#91712)

Remove FormatToken::isSimpleTypeSpecifier() and call
Token::isSimpleTypeSpecifier(LangOpts) instead.
---
 clang/lib/Format/Format.cpp                   |  3 +-
 clang/lib/Format/FormatToken.cpp              | 46 +--------
 clang/lib/Format/FormatToken.h                |  8 +-
 clang/lib/Format/FormatTokenLexer.cpp         |  1 -
 clang/lib/Format/QualifierAlignmentFixer.cpp  | 24 +++--
 clang/lib/Format/QualifierAlignmentFixer.h    |  5 +-
 clang/lib/Format/TokenAnalyzer.cpp            |  4 +-
 clang/lib/Format/TokenAnalyzer.h              |  1 +
 clang/lib/Format/TokenAnnotator.cpp           | 43 +++++----
 clang/lib/Format/TokenAnnotator.h             |  6 +-
 clang/lib/Format/UnwrappedLineParser.cpp      | 19 ++--
 clang/lib/Format/UnwrappedLineParser.h        |  1 +
 clang/unittests/Format/QualifierFixerTest.cpp | 94 +++++++++++--------
 13 files changed, 121 insertions(+), 134 deletions(-)

diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index c4eac1c99a66..c5c79dd0f883 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -3858,8 +3858,7 @@ LangOptions getFormattingLangOpts(const FormatStyle &Style) {
   LangOpts.Digraphs = LexingStd >= FormatStyle::LS_Cpp11;
 
   LangOpts.LineComment = 1;
-  bool AlternativeOperators = Style.isCpp();
-  LangOpts.CXXOperatorNames = AlternativeOperators ? 1 : 0;
+  LangOpts.CXXOperatorNames = Style.isCpp();
   LangOpts.Bool = 1;
   LangOpts.ObjC = 1;
   LangOpts.MicrosoftExt = 1;    // To get kw___try, kw___finally.
diff --git a/clang/lib/Format/FormatToken.cpp b/clang/lib/Format/FormatToken.cpp
index 4fb70ffac706..85bec71ffbbc 100644
--- a/clang/lib/Format/FormatToken.cpp
+++ b/clang/lib/Format/FormatToken.cpp
@@ -34,43 +34,6 @@ const char *getTokenTypeName(TokenType Type) {
   return nullptr;
 }
 
-// FIXME: This is copy&pasted from Sema. Put it in a common place and remove
-// duplication.
-bool FormatToken::isSimpleTypeSpecifier() const {
-  switch (Tok.getKind()) {
-  case tok::kw_short:
-  case tok::kw_long:
-  case tok::kw___int64:
-  case tok::kw___int128:
-  case tok::kw_signed:
-  case tok::kw_unsigned:
-  case tok::kw_void:
-  case tok::kw_char:
-  case tok::kw_int:
-  case tok::kw_half:
-  case tok::kw_float:
-  case tok::kw_double:
-  case tok::kw___bf16:
-  case tok::kw__Float16:
-  case tok::kw___float128:
-  case tok::kw___ibm128:
-  case tok::kw_wchar_t:
-  case tok::kw_bool:
-#define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) case tok::kw___##Trait:
-#include "clang/Basic/TransformTypeTraits.def"
-  case tok::annot_typename:
-  case tok::kw_char8_t:
-  case tok::kw_char16_t:
-  case tok::kw_char32_t:
-  case tok::kw_typeof:
-  case tok::kw_decltype:
-  case tok::kw__Atomic:
-    return true;
-  default:
-    return false;
-  }
-}
-
 // Sorted common C++ non-keyword types.
 static SmallVector<StringRef> CppNonKeywordTypes = {
     "clock_t",  "int16_t",   "int32_t", "int64_t",   "int8_t",
@@ -78,15 +41,16 @@ static SmallVector<StringRef> CppNonKeywordTypes = {
     "uint32_t", "uint64_t",  "uint8_t", "uintptr_t",
 };
 
-bool FormatToken::isTypeName(bool IsCpp) const {
-  return is(TT_TypeName) || isSimpleTypeSpecifier() ||
+bool FormatToken::isTypeName(const LangOptions &LangOpts) const {
+  const bool IsCpp = LangOpts.CXXOperatorNames;
+  return is(TT_TypeName) || Tok.isSimpleTypeSpecifier(LangOpts) ||
          (IsCpp && is(tok::identifier) &&
           std::binary_search(CppNonKeywordTypes.begin(),
                              CppNonKeywordTypes.end(), TokenText));
 }
 
-bool FormatToken::isTypeOrIdentifier(bool IsCpp) const {
-  return isTypeName(IsCpp) || isOneOf(tok::kw_auto, tok::identifier);
+bool FormatToken::isTypeOrIdentifier(const LangOptions &LangOpts) const {
+  return isTypeName(LangOpts) || isOneOf(tok::kw_auto, tok::identifier);
 }
 
 bool FormatToken::isBlockIndentedInitRBrace(const FormatStyle &Style) const {
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 95f16fde5005..8792f4c75074 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -684,12 +684,8 @@ public:
            isAttribute();
   }
 
-  /// Determine whether the token is a simple-type-specifier.
-  [[nodiscard]] bool isSimpleTypeSpecifier() const;
-
-  [[nodiscard]] bool isTypeName(bool IsCpp) const;
-
-  [[nodiscard]] bool isTypeOrIdentifier(bool IsCpp) const;
+  [[nodiscard]] bool isTypeName(const LangOptions &LangOpts) const;
+  [[nodiscard]] bool isTypeOrIdentifier(const LangOptions &LangOpts) const;
 
   bool isObjCAccessSpecifier() const {
     return is(tok::at) && Next &&
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index f430d3764bab..e21b5a882b77 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -1442,7 +1442,6 @@ void FormatTokenLexer::readRawToken(FormatToken &Tok) {
 
 void FormatTokenLexer::resetLexer(unsigned Offset) {
   StringRef Buffer = SourceMgr.getBufferData(ID);
-  LangOpts = getFormattingLangOpts(Style);
   Lex.reset(new Lexer(SourceMgr.getLocForStartOfFile(ID), LangOpts,
                       Buffer.begin(), Buffer.begin() + Offset, Buffer.end()));
   Lex->SetKeepWhitespaceMode(true);
diff --git a/clang/lib/Format/QualifierAlignmentFixer.cpp b/clang/lib/Format/QualifierAlignmentFixer.cpp
index c26353045672..a904f0b773c6 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.cpp
+++ b/clang/lib/Format/QualifierAlignmentFixer.cpp
@@ -268,13 +268,11 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeRight(
   if (isPossibleMacro(TypeToken))
     return Tok;
 
-  const bool IsCpp = Style.isCpp();
-
   // The case `const long long int volatile` -> `long long int const volatile`
   // The case `long const long int volatile` -> `long long int const volatile`
   // The case `long long volatile int const` -> `long long int const volatile`
   // The case `const long long volatile int` -> `long long int const volatile`
-  if (TypeToken->isTypeName(IsCpp)) {
+  if (TypeToken->isTypeName(LangOpts)) {
     // The case `const decltype(foo)` -> `const decltype(foo)`
     // The case `const typeof(foo)` -> `const typeof(foo)`
     // The case `const _Atomic(foo)` -> `const _Atomic(foo)`
@@ -283,7 +281,7 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeRight(
 
     const FormatToken *LastSimpleTypeSpecifier = TypeToken;
     while (isQualifierOrType(LastSimpleTypeSpecifier->getNextNonComment(),
-                             IsCpp)) {
+                             LangOpts)) {
       LastSimpleTypeSpecifier = LastSimpleTypeSpecifier->getNextNonComment();
     }
 
@@ -295,7 +293,7 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeRight(
   // The case  `unsigned short const` -> `unsigned short const`
   // The case:
   // `unsigned short volatile const` -> `unsigned short const volatile`
-  if (PreviousCheck && PreviousCheck->isTypeName(IsCpp)) {
+  if (PreviousCheck && PreviousCheck->isTypeName(LangOpts)) {
     if (LastQual != Tok)
       rotateTokens(SourceMgr, Fixes, Tok, LastQual, /*Left=*/false);
     return Tok;
@@ -412,11 +410,11 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeLeft(
   // The case `volatile long long const int` -> `const volatile long long int`
   // The case `const long long volatile int` -> `const volatile long long int`
   // The case `long volatile long int const` -> `const volatile long long int`
-  if (const bool IsCpp = Style.isCpp(); TypeToken->isTypeName(IsCpp)) {
+  if (TypeToken->isTypeName(LangOpts)) {
     const FormatToken *LastSimpleTypeSpecifier = TypeToken;
     while (isConfiguredQualifierOrType(
         LastSimpleTypeSpecifier->getPreviousNonComment(),
-        ConfiguredQualifierTokens, IsCpp)) {
+        ConfiguredQualifierTokens, LangOpts)) {
       LastSimpleTypeSpecifier =
           LastSimpleTypeSpecifier->getPreviousNonComment();
     }
@@ -614,16 +612,16 @@ void prepareLeftRightOrderingForQualifierAlignmentFixer(
   }
 }
 
-bool LeftRightQualifierAlignmentFixer::isQualifierOrType(const FormatToken *Tok,
-                                                         bool IsCpp) {
-  return Tok &&
-         (Tok->isTypeName(IsCpp) || Tok->is(tok::kw_auto) || isQualifier(Tok));
+bool LeftRightQualifierAlignmentFixer::isQualifierOrType(
+    const FormatToken *Tok, const LangOptions &LangOpts) {
+  return Tok && (Tok->isTypeName(LangOpts) || Tok->is(tok::kw_auto) ||
+                 isQualifier(Tok));
 }
 
 bool LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
     const FormatToken *Tok, const std::vector<tok::TokenKind> &Qualifiers,
-    bool IsCpp) {
-  return Tok && (Tok->isTypeName(IsCpp) || Tok->is(tok::kw_auto) ||
+    const LangOptions &LangOpts) {
+  return Tok && (Tok->isTypeName(LangOpts) || Tok->is(tok::kw_auto) ||
                  isConfiguredQualifier(Tok, Qualifiers));
 }
 
diff --git a/clang/lib/Format/QualifierAlignmentFixer.h b/clang/lib/Format/QualifierAlignmentFixer.h
index e1cc27e62b13..710fa2dc0030 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.h
+++ b/clang/lib/Format/QualifierAlignmentFixer.h
@@ -71,11 +71,12 @@ public:
                                  tok::TokenKind QualifierType);
 
   // Is the Token a simple or qualifier type
-  static bool isQualifierOrType(const FormatToken *Tok, bool IsCpp = true);
+  static bool isQualifierOrType(const FormatToken *Tok,
+                                const LangOptions &LangOpts);
   static bool
   isConfiguredQualifierOrType(const FormatToken *Tok,
                               const std::vector<tok::TokenKind> &Qualifiers,
-                              bool IsCpp = true);
+                              const LangOptions &LangOpts);
 
   // Is the Token likely a Macro
   static bool isPossibleMacro(const FormatToken *Tok);
diff --git a/clang/lib/Format/TokenAnalyzer.cpp b/clang/lib/Format/TokenAnalyzer.cpp
index bd648c430f9b..804a2b0f5e8c 100644
--- a/clang/lib/Format/TokenAnalyzer.cpp
+++ b/clang/lib/Format/TokenAnalyzer.cpp
@@ -84,7 +84,7 @@ Environment::Environment(StringRef Code, StringRef FileName,
       NextStartColumn(NextStartColumn), LastStartColumn(LastStartColumn) {}
 
 TokenAnalyzer::TokenAnalyzer(const Environment &Env, const FormatStyle &Style)
-    : Style(Style), Env(Env),
+    : Style(Style), LangOpts(getFormattingLangOpts(Style)), Env(Env),
       AffectedRangeMgr(Env.getSourceManager(), Env.getCharRanges()),
       UnwrappedLines(1),
       Encoding(encoding::detectEncoding(
@@ -101,7 +101,7 @@ std::pair<tooling::Replacements, unsigned>
 TokenAnalyzer::process(bool SkipAnnotation) {
   tooling::Replacements Result;
   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
-  IdentifierTable IdentTable(getFormattingLangOpts(Style));
+  IdentifierTable IdentTable(LangOpts);
   FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(),
                        Env.getFirstStartColumn(), Style, Encoding, Allocator,
                        IdentTable);
diff --git a/clang/lib/Format/TokenAnalyzer.h b/clang/lib/Format/TokenAnalyzer.h
index b7494c395c8a..ef559099d325 100644
--- a/clang/lib/Format/TokenAnalyzer.h
+++ b/clang/lib/Format/TokenAnalyzer.h
@@ -92,6 +92,7 @@ protected:
   void finishRun() override;
 
   FormatStyle Style;
+  LangOptions LangOpts;
   // Stores Style, FileID and SourceManager etc.
   const Environment &Env;
   // AffectedRangeMgr stores ranges to be fixed.
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index e935d3e2709c..478cae23d3c8 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -126,7 +126,9 @@ public:
                    const AdditionalKeywords &Keywords,
                    SmallVector<ScopeType> &Scopes)
       : Style(Style), Line(Line), CurrentToken(Line.First), AutoFound(false),
-        IsCpp(Style.isCpp()), Keywords(Keywords), Scopes(Scopes) {
+        IsCpp(Style.isCpp()), LangOpts(getFormattingLangOpts(Style)),
+        Keywords(Keywords), Scopes(Scopes) {
+    assert(IsCpp == LangOpts.CXXOperatorNames);
     Contexts.push_back(Context(tok::unknown, 1, /*IsExpression=*/false));
     resetTokenMetadata();
   }
@@ -562,7 +564,7 @@ private:
           (CurrentToken->is(tok::l_paren) && CurrentToken->Next &&
            CurrentToken->Next->isOneOf(tok::star, tok::amp, tok::caret));
       if ((CurrentToken->Previous->isOneOf(tok::kw_const, tok::kw_auto) ||
-           CurrentToken->Previous->isTypeName(IsCpp)) &&
+           CurrentToken->Previous->isTypeName(LangOpts)) &&
           !(CurrentToken->is(tok::l_brace) ||
             (CurrentToken->is(tok::l_paren) && !ProbablyFunctionTypeLParen))) {
         Contexts.back().IsExpression = false;
@@ -2624,7 +2626,7 @@ private:
       return true;
 
     // MyClass a;
-    if (PreviousNotConst->isTypeName(IsCpp))
+    if (PreviousNotConst->isTypeName(LangOpts))
       return true;
 
     // type[] a in Java
@@ -2728,7 +2730,7 @@ private:
     }
 
     if (Tok.Next->is(tok::question) ||
-        (Tok.Next->is(tok::ampamp) && !Tok.Previous->isTypeName(IsCpp))) {
+        (Tok.Next->is(tok::ampamp) && !Tok.Previous->isTypeName(LangOpts))) {
       return false;
     }
 
@@ -2757,9 +2759,10 @@ private:
     }
 
     // Heuristically try to determine whether the parentheses contain a type.
-    auto IsQualifiedPointerOrReference = [](FormatToken *T, bool IsCpp) {
+    auto IsQualifiedPointerOrReference = [](FormatToken *T,
+                                            const LangOptions &LangOpts) {
       // This is used to handle cases such as x = (foo *const)&y;
-      assert(!T->isTypeName(IsCpp) && "Should have already been checked");
+      assert(!T->isTypeName(LangOpts) && "Should have already been checked");
       // Strip trailing qualifiers such as const or volatile when checking
       // whether the parens could be a cast to a pointer/reference type.
       while (T) {
@@ -2791,8 +2794,8 @@ private:
     bool ParensAreType =
         !Tok.Previous ||
         Tok.Previous->isOneOf(TT_TemplateCloser, TT_TypeDeclarationParen) ||
-        Tok.Previous->isTypeName(IsCpp) ||
-        IsQualifiedPointerOrReference(Tok.Previous, IsCpp);
+        Tok.Previous->isTypeName(LangOpts) ||
+        IsQualifiedPointerOrReference(Tok.Previous, LangOpts);
     bool ParensCouldEndDecl =
         Tok.Next->isOneOf(tok::equal, tok::semi, tok::l_brace, tok::greater);
     if (ParensAreType && !ParensCouldEndDecl)
@@ -3065,6 +3068,7 @@ private:
   FormatToken *CurrentToken;
   bool AutoFound;
   bool IsCpp;
+  LangOptions LangOpts;
   const AdditionalKeywords &Keywords;
 
   SmallVector<ScopeType> &Scopes;
@@ -3639,7 +3643,8 @@ void TokenAnnotator::annotate(AnnotatedLine &Line) {
 
 // This function heuristically determines whether 'Current' starts the name of a
 // function declaration.
-static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
+static bool isFunctionDeclarationName(const LangOptions &LangOpts,
+                                      const FormatToken &Current,
                                       const AnnotatedLine &Line,
                                       FormatToken *&ClosingParen) {
   assert(Current.Previous);
@@ -3658,7 +3663,7 @@ static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
   }
 
   auto skipOperatorName =
-      [IsCpp](const FormatToken *Next) -> const FormatToken * {
+      [&LangOpts](const FormatToken *Next) -> const FormatToken * {
     for (; Next; Next = Next->Next) {
       if (Next->is(TT_OverloadedOperatorLParen))
         return Next;
@@ -3677,7 +3682,7 @@ static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
         Next = Next->Next;
         continue;
       }
-      if ((Next->isTypeName(IsCpp) || Next->is(tok::identifier)) &&
+      if ((Next->isTypeName(LangOpts) || Next->is(tok::identifier)) &&
           Next->Next && Next->Next->isPointerOrReference()) {
         // For operator void*(), operator char*(), operator Foo*().
         Next = Next->Next;
@@ -3693,8 +3698,10 @@ static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
     return nullptr;
   };
 
+  const auto *Next = Current.Next;
+  const bool IsCpp = LangOpts.CXXOperatorNames;
+
   // Find parentheses of parameter list.
-  const FormatToken *Next = Current.Next;
   if (Current.is(tok::kw_operator)) {
     if (Previous.Tok.getIdentifierInfo() &&
         !Previous.isOneOf(tok::kw_return, tok::kw_co_return)) {
@@ -3774,7 +3781,7 @@ static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
       Tok = Tok->MatchingParen;
       continue;
     }
-    if (Tok->is(tok::kw_const) || Tok->isTypeName(IsCpp) ||
+    if (Tok->is(tok::kw_const) || Tok->isTypeName(LangOpts) ||
         Tok->isOneOf(TT_PointerOrReference, TT_StartOfName, tok::ellipsis)) {
       return true;
     }
@@ -3837,7 +3844,7 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) const {
       AfterLastAttribute = Tok;
     if (const bool IsCtorOrDtor = Tok->is(TT_CtorDtorDeclName);
         IsCtorOrDtor ||
-        isFunctionDeclarationName(IsCpp, *Tok, Line, ClosingParen)) {
+        isFunctionDeclarationName(LangOpts, *Tok, Line, ClosingParen)) {
       if (!IsCtorOrDtor)
         Tok->setFinalizedType(TT_FunctionDeclarationName);
       LineIsFunctionDeclaration = true;
@@ -4447,7 +4454,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     if (Left.Tok.isLiteral())
       return true;
     // for (auto a = 0, b = 0; const auto & c : {1, 2, 3})
-    if (Left.isTypeOrIdentifier(IsCpp) && Right.Next && Right.Next->Next &&
+    if (Left.isTypeOrIdentifier(LangOpts) && Right.Next && Right.Next->Next &&
         Right.Next->Next->is(TT_RangeBasedForLoopColon)) {
       return getTokenPointerOrReferenceAlignment(Right) !=
              FormatStyle::PAS_Left;
@@ -4490,7 +4497,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     if (Right.is(tok::l_brace) && Right.is(BK_Block))
       return true;
     // for (auto a = 0, b = 0; const auto& c : {1, 2, 3})
-    if (BeforeLeft && BeforeLeft->isTypeOrIdentifier(IsCpp) && Right.Next &&
+    if (BeforeLeft && BeforeLeft->isTypeOrIdentifier(LangOpts) && Right.Next &&
         Right.Next->is(TT_RangeBasedForLoopColon)) {
       return getTokenPointerOrReferenceAlignment(Left) !=
              FormatStyle::PAS_Right;
@@ -4534,7 +4541,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   if (Right.isPointerOrReference()) {
     const FormatToken *Previous = &Left;
     while (Previous && Previous->isNot(tok::kw_operator)) {
-      if (Previous->is(tok::identifier) || Previous->isTypeName(IsCpp)) {
+      if (Previous->is(tok::identifier) || Previous->isTypeName(LangOpts)) {
         Previous = Previous->getPreviousNonComment();
         continue;
       }
@@ -4723,7 +4730,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   if (!Style.isVerilog() &&
       (Left.isOneOf(tok::identifier, tok::greater, tok::r_square,
                     tok::r_paren) ||
-       Left.isTypeName(IsCpp)) &&
+       Left.isTypeName(LangOpts)) &&
       Right.is(tok::l_brace) && Right.getNextNonComment() &&
       Right.isNot(BK_Block)) {
     return false;
diff --git a/clang/lib/Format/TokenAnnotator.h b/clang/lib/Format/TokenAnnotator.h
index 25a24dccb1b8..d19d3d061e40 100644
--- a/clang/lib/Format/TokenAnnotator.h
+++ b/clang/lib/Format/TokenAnnotator.h
@@ -211,7 +211,10 @@ private:
 class TokenAnnotator {
 public:
   TokenAnnotator(const FormatStyle &Style, const AdditionalKeywords &Keywords)
-      : Style(Style), IsCpp(Style.isCpp()), Keywords(Keywords) {}
+      : Style(Style), IsCpp(Style.isCpp()),
+        LangOpts(getFormattingLangOpts(Style)), Keywords(Keywords) {
+    assert(IsCpp == LangOpts.CXXOperatorNames);
+  }
 
   /// Adapts the indent levels of comment lines to the indent of the
   /// subsequent line.
@@ -260,6 +263,7 @@ private:
   const FormatStyle &Style;
 
   bool IsCpp;
+  LangOptions LangOpts;
 
   const AdditionalKeywords &Keywords;
 
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 310b75485e08..4f1c2c5114e9 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -160,13 +160,16 @@ UnwrappedLineParser::UnwrappedLineParser(
     IdentifierTable &IdentTable)
     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
       CurrentLines(&Lines), Style(Style), IsCpp(Style.isCpp()),
-      Keywords(Keywords), CommentPragmasRegex(Style.CommentPragmas),
-      Tokens(nullptr), Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1),
+      LangOpts(getFormattingLangOpts(Style)), Keywords(Keywords),
+      CommentPragmasRegex(Style.CommentPragmas), Tokens(nullptr),
+      Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1),
       IncludeGuard(Style.IndentPPDirectives == FormatStyle::PPDIS_None
                        ? IG_Rejected
                        : IG_Inited),
       IncludeGuardToken(nullptr), FirstStartColumn(FirstStartColumn),
-      Macros(Style.Macros, SourceMgr, Style, Allocator, IdentTable) {}
+      Macros(Style.Macros, SourceMgr, Style, Allocator, IdentTable) {
+  assert(IsCpp == LangOpts.CXXOperatorNames);
+}
 
 void UnwrappedLineParser::reset() {
   PPBranchLevel = -1;
@@ -1870,7 +1873,7 @@ void UnwrappedLineParser::parseStructuralElement(
     case tok::caret:
       nextToken();
       // Block return type.
-      if (FormatTok->Tok.isAnyIdentifier() || FormatTok->isTypeName(IsCpp)) {
+      if (FormatTok->Tok.isAnyIdentifier() || FormatTok->isTypeName(LangOpts)) {
         nextToken();
         // Return types: pointers are ok too.
         while (FormatTok->is(tok::star))
@@ -2231,7 +2234,7 @@ bool UnwrappedLineParser::tryToParseLambda() {
   bool InTemplateParameterList = false;
 
   while (FormatTok->isNot(tok::l_brace)) {
-    if (FormatTok->isTypeName(IsCpp)) {
+    if (FormatTok->isTypeName(LangOpts)) {
       nextToken();
       continue;
     }
@@ -3448,7 +3451,7 @@ bool UnwrappedLineParser::parseRequires() {
     break;
   }
   default:
-    if (PreviousNonComment->isTypeOrIdentifier(IsCpp)) {
+    if (PreviousNonComment->isTypeOrIdentifier(LangOpts)) {
       // This is a requires clause.
       parseRequiresClause(RequiresToken);
       return true;
@@ -3511,7 +3514,7 @@ bool UnwrappedLineParser::parseRequires() {
       --OpenAngles;
       break;
     default:
-      if (NextToken->isTypeName(IsCpp)) {
+      if (NextToken->isTypeName(LangOpts)) {
         FormatTok = Tokens->setPosition(StoredPosition);
         parseRequiresExpression(RequiresToken);
         return false;
@@ -4027,7 +4030,7 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
       if (FormatTok->is(tok::l_square)) {
         FormatToken *Previous = FormatTok->Previous;
         if (!Previous || (Previous->isNot(tok::r_paren) &&
-                          !Previous->isTypeOrIdentifier(IsCpp))) {
+                          !Previous->isTypeOrIdentifier(LangOpts))) {
           // Don't try parsing a lambda if we had a closing parenthesis before,
           // it was probably a pointer to an array: int (*)[].
           if (!tryToParseLambda())
diff --git a/clang/lib/Format/UnwrappedLineParser.h b/clang/lib/Format/UnwrappedLineParser.h
index 2a0fe19d0957..d7963a4211bb 100644
--- a/clang/lib/Format/UnwrappedLineParser.h
+++ b/clang/lib/Format/UnwrappedLineParser.h
@@ -316,6 +316,7 @@ private:
 
   const FormatStyle &Style;
   bool IsCpp;
+  LangOptions LangOpts;
   const AdditionalKeywords &Keywords;
 
   llvm::Regex CommentPragmasRegex;
diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index 792d8f3c3a98..ada47549fd0b 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -27,11 +27,15 @@ namespace {
   EXPECT_EQ(VALUE, Style.FIELD) << "Unexpected value after parsing!"
 
 class QualifierFixerTest : public FormatTestBase {
+public:
+  QualifierFixerTest() : LangOpts(getFormattingLangOpts()) {}
+
 protected:
   TokenList annotate(llvm::StringRef Code,
                      const FormatStyle &Style = getLLVMStyle()) {
     return TestLexer(Allocator, Buffers, Style).annotate(Code);
   }
+  LangOptions LangOpts;
   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
 };
@@ -1060,65 +1064,75 @@ TEST_F(QualifierFixerTest, IsQualifierType) {
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
 
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[0], ConfiguredTokens));
+      Tokens[0], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[1], ConfiguredTokens));
+      Tokens[1], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[2], ConfiguredTokens));
+      Tokens[2], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[3], ConfiguredTokens));
+      Tokens[3], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[4], ConfiguredTokens));
+      Tokens[4], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[5], ConfiguredTokens));
+      Tokens[5], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[6], ConfiguredTokens));
+      Tokens[6], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[7], ConfiguredTokens));
+      Tokens[7], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[8], ConfiguredTokens));
+      Tokens[8], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[9], ConfiguredTokens));
-
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[0]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[1]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[2]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[3]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[4]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[5]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[6]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[7]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[8]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[9]));
+      Tokens[9], ConfiguredTokens, LangOpts));
+
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[0], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[1], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[2], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[3], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[4], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[5], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[6], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[7], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[8], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[9], LangOpts));
 
   auto NotTokens = annotate("for while do Foo Bar ");
   ASSERT_EQ(NotTokens.size(), 6u) << Tokens;
 
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[0], ConfiguredTokens));
+      NotTokens[0], ConfiguredTokens, LangOpts));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[1], ConfiguredTokens));
+      NotTokens[1], ConfiguredTokens, LangOpts));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[2], ConfiguredTokens));
+      NotTokens[2], ConfiguredTokens, LangOpts));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[3], ConfiguredTokens));
+      NotTokens[3], ConfiguredTokens, LangOpts));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[4], ConfiguredTokens));
+      NotTokens[4], ConfiguredTokens, LangOpts));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[5], ConfiguredTokens));
-
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[0]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[1]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[2]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[3]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[4]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[5]));
+      NotTokens[5], ConfiguredTokens, LangOpts));
+
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[0],
+                                                                   LangOpts));
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[1],
+                                                                   LangOpts));
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[2],
+                                                                   LangOpts));
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[3],
+                                                                   LangOpts));
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[4],
+                                                                   LangOpts));
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[5],
+                                                                   LangOpts));
 }
 
 TEST_F(QualifierFixerTest, IsMacro) {
-- 
GitLab


From 65c9b8460b1aa309d012a5fe031524dbb813d64e Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 10 May 2024 19:45:42 -0700
Subject: [PATCH 0471/1206] [Driver] Remove elf::script indirection. NFC

There are 100+ references.
Use a wrapper similar to a623a4c8b4d5cc933ebc1856ce30aac935f62b18
---
 lld/ELF/Driver.cpp       | 2 +-
 lld/ELF/LinkerScript.cpp | 2 +-
 lld/ELF/LinkerScript.h   | 7 ++++++-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index b29e1e1a67f1..dd33f4bd772f 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -151,7 +151,7 @@ bool link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
                                  "--error-limit=0 to see all errors)";
 
   config = ConfigWrapper();
-  script = std::make_unique<LinkerScript>();
+  script = ScriptWrapper();
 
   symAux.emplace_back();
 
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index f9d8dcc4f71d..c0a5014817b9 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -44,7 +44,7 @@ using namespace llvm::support::endian;
 using namespace lld;
 using namespace lld::elf;
 
-std::unique_ptr<LinkerScript> elf::script;
+ScriptWrapper elf::script;
 
 static bool isSectionPrefix(StringRef prefix, StringRef name) {
   return name.consume_front(prefix) && (name.empty() || name[0] == '.');
diff --git a/lld/ELF/LinkerScript.h b/lld/ELF/LinkerScript.h
index fa7c6eb9c0d8..b09cd12c46f9 100644
--- a/lld/ELF/LinkerScript.h
+++ b/lld/ELF/LinkerScript.h
@@ -402,7 +402,12 @@ public:
   llvm::MapVector<StringRef, SmallVector<StringRef, 0>> provideMap;
 };
 
-LLVM_LIBRARY_VISIBILITY extern std::unique_ptr<LinkerScript> script;
+struct ScriptWrapper {
+  LinkerScript s;
+  LinkerScript *operator->() { return &s; }
+};
+
+LLVM_LIBRARY_VISIBILITY extern ScriptWrapper script;
 
 } // end namespace lld::elf
 
-- 
GitLab


From 3676b0945c800e3105f648d178b331953246716a Mon Sep 17 00:00:00 2001
From: Congcong Cai <congcongcai0907@163.com>
Date: Sat, 11 May 2024 12:15:24 +0800
Subject: [PATCH 0472/1206] [clang-tidy] `readability-simplify-boolean-expr`
 avoid to warn expression expand from macro when ``IgnoreMacro`` option is
 enabled. (#91757)

Fixes: #91487
---
 .../readability/SimplifyBooleanExprCheck.cpp  | 58 +++++++++++--------
 .../readability/SimplifyBooleanExprCheck.h    |  2 +
 clang-tools-extra/docs/ReleaseNotes.rst       |  4 ++
 .../simplify-boolean-expr-macros.cpp          | 18 ++++--
 4 files changed, 53 insertions(+), 29 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.cpp b/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.cpp
index edb67614bd55..fd4730d9c8b9 100644
--- a/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SimplifyBooleanExprCheck.h"
+#include "clang/AST/Expr.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/Support/SaveAndRestore.h"
@@ -280,9 +281,8 @@ public:
     if (!S) {
       return true;
     }
-    if (Check->IgnoreMacros && S->getBeginLoc().isMacroID()) {
+    if (Check->canBeBypassed(S))
       return false;
-    }
     if (!shouldIgnore(S))
       StmtStack.push_back(S);
     return true;
@@ -513,17 +513,23 @@ public:
     return true;
   }
 
-  static bool isUnaryLNot(const Expr *E) {
-    return isa<UnaryOperator>(E) &&
+  bool isExpectedUnaryLNot(const Expr *E) {
+    return !Check->canBeBypassed(E) && isa<UnaryOperator>(E) &&
            cast<UnaryOperator>(E)->getOpcode() == UO_LNot;
   }
 
+  bool isExpectedBinaryOp(const Expr *E) {
+    const auto *BinaryOp = dyn_cast<BinaryOperator>(E);
+    return !Check->canBeBypassed(E) && BinaryOp && BinaryOp->isLogicalOp() &&
+           BinaryOp->getType()->isBooleanType();
+  }
+
   template <typename Functor>
   static bool checkEitherSide(const BinaryOperator *BO, Functor Func) {
     return Func(BO->getLHS()) || Func(BO->getRHS());
   }
 
-  static bool nestedDemorgan(const Expr *E, unsigned NestingLevel) {
+  bool nestedDemorgan(const Expr *E, unsigned NestingLevel) {
     const auto *BO = dyn_cast<BinaryOperator>(E->IgnoreUnlessSpelledInSource());
     if (!BO)
       return false;
@@ -539,15 +545,13 @@ public:
       return true;
     case BO_LAnd:
     case BO_LOr:
-      if (checkEitherSide(BO, isUnaryLNot))
-        return true;
-      if (NestingLevel) {
-        if (checkEitherSide(BO, [NestingLevel](const Expr *E) {
-              return nestedDemorgan(E, NestingLevel - 1);
-            }))
-          return true;
-      }
-      return false;
+      return checkEitherSide(
+                 BO,
+                 [this](const Expr *E) { return isExpectedUnaryLNot(E); }) ||
+             (NestingLevel &&
+              checkEitherSide(BO, [this, NestingLevel](const Expr *E) {
+                return nestedDemorgan(E, NestingLevel - 1);
+              }));
     default:
       return false;
     }
@@ -556,19 +560,19 @@ public:
   bool TraverseUnaryOperator(UnaryOperator *Op) {
     if (!Check->SimplifyDeMorgan || Op->getOpcode() != UO_LNot)
       return Base::TraverseUnaryOperator(Op);
-    Expr *SubImp = Op->getSubExpr()->IgnoreImplicit();
-    auto *Parens = dyn_cast<ParenExpr>(SubImp);
-    auto *BinaryOp =
-        Parens
-            ? dyn_cast<BinaryOperator>(Parens->getSubExpr()->IgnoreImplicit())
-            : dyn_cast<BinaryOperator>(SubImp);
-    if (!BinaryOp || !BinaryOp->isLogicalOp() ||
-        !BinaryOp->getType()->isBooleanType())
+    const Expr *SubImp = Op->getSubExpr()->IgnoreImplicit();
+    const auto *Parens = dyn_cast<ParenExpr>(SubImp);
+    const Expr *SubExpr =
+        Parens ? Parens->getSubExpr()->IgnoreImplicit() : SubImp;
+    if (!isExpectedBinaryOp(SubExpr))
       return Base::TraverseUnaryOperator(Op);
+    const auto *BinaryOp = cast<BinaryOperator>(SubExpr);
     if (Check->SimplifyDeMorganRelaxed ||
-        checkEitherSide(BinaryOp, isUnaryLNot) ||
-        checkEitherSide(BinaryOp,
-                        [](const Expr *E) { return nestedDemorgan(E, 1); })) {
+        checkEitherSide(
+            BinaryOp,
+            [this](const Expr *E) { return isExpectedUnaryLNot(E); }) ||
+        checkEitherSide(
+            BinaryOp, [this](const Expr *E) { return nestedDemorgan(E, 1); })) {
       if (Check->reportDeMorgan(Context, Op, BinaryOp, !IsProcessing, parent(),
                                 Parens) &&
           !Check->areDiagsSelfContained()) {
@@ -694,6 +698,10 @@ void SimplifyBooleanExprCheck::check(const MatchFinder::MatchResult &Result) {
   Visitor(this, *Result.Context).traverse();
 }
 
+bool SimplifyBooleanExprCheck::canBeBypassed(const Stmt *S) const {
+  return IgnoreMacros && S->getBeginLoc().isMacroID();
+}
+
 void SimplifyBooleanExprCheck::issueDiag(const ASTContext &Context,
                                          SourceLocation Loc,
                                          StringRef Description,
diff --git a/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.h b/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.h
index ccc6f3d879fc..63c3caa01e01 100644
--- a/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.h
@@ -64,6 +64,8 @@ private:
                  StringRef Description, SourceRange ReplacementRange,
                  StringRef Replacement);
 
+  bool canBeBypassed(const Stmt *S) const;
+
   const bool IgnoreMacros;
   const bool ChainedConditionalReturn;
   const bool ChainedConditionalAssignment;
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 3f0d25ec8c75..6a2b8d3b6ded 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -357,6 +357,10 @@ Changes in existing checks
   support calls to overloaded operators as base expression and provide fixes to
   expressions with side-effects.
 
+- Improved :doc:`readability-simplify-boolean-expr
+  <clang-tidy/checks/readability/simplify-boolean-expr>` check to avoid to emit
+  warning for macro when IgnoreMacro option is enabled.
+
 - Improved :doc:`readability-static-definition-in-anonymous-namespace
   <clang-tidy/checks/readability/static-definition-in-anonymous-namespace>`
   check by resolving fix-it overlaps in template code by disregarding implicit
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/simplify-boolean-expr-macros.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/simplify-boolean-expr-macros.cpp
index 7d0cfe7e27dc..d1df79e23a1e 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/simplify-boolean-expr-macros.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/simplify-boolean-expr-macros.cpp
@@ -6,6 +6,7 @@
 // RUN:     --
 
 #define NEGATE(expr) !(expr)
+#define NOT_AND_NOT(a, b) (!a && !b)
 
 bool without_macro(bool a, bool b) {
     return !(!a && b);
@@ -13,8 +14,17 @@ bool without_macro(bool a, bool b) {
     // CHECK-FIXES: return a || !b;
 }
 
-bool macro(bool a, bool b) {
-    return NEGATE(!a && b);
-    // CHECK-MESSAGES-MACROS: :[[@LINE-1]]:12: warning: boolean expression can be simplified by DeMorgan's theorem
-    // CHECK-FIXES: return NEGATE(!a && b);
+void macro(bool a, bool b) {
+    NEGATE(!a && b);
+    // CHECK-MESSAGES-MACROS: :[[@LINE-1]]:5: warning: boolean expression can be simplified by DeMorgan's theorem
+    // CHECK-FIXES: NEGATE(!a && b);
+    !NOT_AND_NOT(a, b);
+    // CHECK-MESSAGES-MACROS: :[[@LINE-1]]:5: warning: boolean expression can be simplified by DeMorgan's theorem
+    // CHECK-FIXES: !NOT_AND_NOT(a, b);
+    !(NEGATE(a) && b);
+    // CHECK-MESSAGES-MACROS: :[[@LINE-1]]:5: warning: boolean expression can be simplified by DeMorgan's theorem
+    // CHECK-FIXES: !(NEGATE(a) && b);
+    !(a && NEGATE(b));
+    // CHECK-MESSAGES-MACROS: :[[@LINE-1]]:5: warning: boolean expression can be simplified by DeMorgan's theorem
+    // CHECK-FIXES: !(a && NEGATE(b));
 }
-- 
GitLab


From bbcdd4f4b2f77b461e9cfd259b4c73d985c2dd9d Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Fri, 10 May 2024 22:18:50 -0700
Subject: [PATCH 0473/1206] [BOLT] Use disambiguated local names in BAT YAML

Align BAT YAML to fdata profile.

Test Plan: updated register-fragments-bolt-symbols.s

Reviewers: dcci, rafaelauler, ayermolo, maksfb

Reviewed By: dcci

Pull Request: https://github.com/llvm/llvm-project/pull/91773
---
 bolt/lib/Profile/DataAggregator.cpp             | 2 +-
 bolt/test/X86/register-fragments-bolt-symbols.s | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index d02e4499014e..f832f958def0 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -2339,7 +2339,7 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
         continue;
       BinaryFunction *BF = BC.getBinaryFunctionAtAddress(FuncAddress);
       assert(BF);
-      YamlBF.Name = FuncName.str();
+      YamlBF.Name = getLocationName(*BF);
       YamlBF.Id = BF->getFunctionNumber();
       YamlBF.Hash = BAT->getBFHash(FuncAddress);
       YamlBF.ExecCount = BF->getKnownExecutionCount();
diff --git a/bolt/test/X86/register-fragments-bolt-symbols.s b/bolt/test/X86/register-fragments-bolt-symbols.s
index fa9b70e0b2d8..6478adf19372 100644
--- a/bolt/test/X86/register-fragments-bolt-symbols.s
+++ b/bolt/test/X86/register-fragments-bolt-symbols.s
@@ -15,6 +15,8 @@
 # PREAGG: B X:0 #chain.cold.0# 1 0
 # RUN: perf2bolt %t.bolt -p %t.preagg --pa -o %t.bat.fdata -w %t.bat.yaml -v=1 \
 # RUN:   | FileCheck %s --check-prefix=CHECK-REGISTER
+# RUN: FileCheck --input-file %t.bat.fdata --check-prefix=CHECK-FDATA %s
+# RUN: FileCheck --input-file %t.bat.yaml --check-prefix=CHECK-YAML %s
 
 # CHECK-SYMS: l df *ABS*          [[#]] chain.s
 # CHECK-SYMS: l  F .bolt.org.text [[#]] chain
@@ -24,6 +26,9 @@
 
 # CHECK-REGISTER: BOLT-INFO: marking chain.cold.0/1(*2) as a fragment of chain/2(*2)
 
+# CHECK-FDATA: 0 [unknown] 0 1 chain/chain.s/2 10 0 1
+# CHECK-YAML: - name: 'chain/chain.s/2'
+
 .file "chain.s"
         .text
         .type   chain, @function
-- 
GitLab


From 4f127667ca57875f0984f81d5790479b902022e0 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Fri, 10 May 2024 22:23:45 -0700
Subject: [PATCH 0474/1206] [BOLT] Set entry counts in BAT YAML profile
 (#91775)

Align with DataReader::readProfile that sets entry block counts from
FuncBranchData->EntryData.

Test Plan: updated bolt-address-translation-yaml.test
---
 bolt/lib/Profile/DataAggregator.cpp              | 11 +++++++++++
 bolt/test/X86/bolt-address-translation-yaml.test |  4 ++++
 2 files changed, 15 insertions(+)

diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index f832f958def0..2f6380e186e1 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -2403,6 +2403,17 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
           return A.Offset < B.Offset;
         });
       }
+      // Set entry counts, similar to DataReader::readProfile.
+      for (const llvm::bolt::BranchInfo &BI : Branches.EntryData) {
+        if (!BlockMap.isInputBlock(BI.To.Offset)) {
+          if (opts::Verbosity >= 1)
+            errs() << "BOLT-WARNING: Unexpected EntryData in " << FuncName
+                   << " at 0x" << Twine::utohexstr(BI.To.Offset) << '\n';
+          continue;
+        }
+        const unsigned BlockIndex = BlockMap.getBBIndex(BI.To.Offset);
+        YamlBF.Blocks[BlockIndex].ExecCount += BI.Branches;
+      }
       // Drop blocks without a hash, won't be useful for stale matching.
       llvm::erase_if(YamlBF.Blocks,
                      [](const yaml::bolt::BinaryBasicBlockProfile &YamlBB) {
diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test
index b3d8a8839450..38a50f6cacaf 100644
--- a/bolt/test/X86/bolt-address-translation-yaml.test
+++ b/bolt/test/X86/bolt-address-translation-yaml.test
@@ -48,6 +48,10 @@ YAML-BAT-CHECK-NEXT:   hash:    0x6AF7E61EA3966722
 YAML-BAT-CHECK-NEXT:   exec:    25
 YAML-BAT-CHECK-NEXT:   nblocks: 15
 YAML-BAT-CHECK-NEXT:   blocks:
+YAML-BAT-CHECK-NEXT:   - bid:   0
+YAML-BAT-CHECK-NEXT:     insns: [[#]]
+YAML-BAT-CHECK-NEXT:     hash:  0x700F19D24600000
+YAML-BAT-CHECK-NEXT:     exec:  25
 YAML-BAT-CHECK:        - bid:   3
 YAML-BAT-CHECK-NEXT:     insns: [[#]]
 YAML-BAT-CHECK-NEXT:     hash:  0xDDA1DC5F69F900AC
-- 
GitLab


From 5cd280433e8e063bf3c2390eaf152e726db5b0fb Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Fri, 10 May 2024 23:07:23 -0700
Subject: [PATCH 0475/1206] [clang-format] Move QualifierFixerTest::LangOpts to
 IsQualifierType test

This should fix buildbot failures in:
https://lab.llvm.org/buildbot/#/builders/5/builds/43303
https://lab.llvm.org/buildbot/#/builders/168/builds/20347
---
 clang/unittests/Format/QualifierFixerTest.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index ada47549fd0b..5463bfbb65ca 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -27,15 +27,11 @@ namespace {
   EXPECT_EQ(VALUE, Style.FIELD) << "Unexpected value after parsing!"
 
 class QualifierFixerTest : public FormatTestBase {
-public:
-  QualifierFixerTest() : LangOpts(getFormattingLangOpts()) {}
-
 protected:
   TokenList annotate(llvm::StringRef Code,
                      const FormatStyle &Style = getLLVMStyle()) {
     return TestLexer(Allocator, Buffers, Style).annotate(Code);
   }
-  LangOptions LangOpts;
   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
 };
@@ -1059,6 +1055,8 @@ TEST_F(QualifierFixerTest, IsQualifierType) {
   ConfiguredTokens.push_back(tok::kw_constexpr);
   ConfiguredTokens.push_back(tok::kw_friend);
 
+  LangOptions LangOpts{getFormattingLangOpts()};
+
   auto Tokens = annotate(
       "const static inline auto restrict int double long constexpr friend");
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
-- 
GitLab


From 1351aabf542b25e3ff1d36f4f2abe6727d0f19aa Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Sat, 11 May 2024 10:20:27 +0400
Subject: [PATCH 0476/1206] [clang] Add test for CWG1820 "Qualified typedef
 names" (#91765)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

[P1787R6](https://wg21.link/p1787r6):
> [CWG1820](https://cplusplus.github.io/CWG/issues/1820.html) is
resolved by requiring that a qualified declarator-id declare an entity.

P1787R6 changes wording of [dcl.pre]/9. Quote from the current draft
([[dcl.pre]/5](https://eel.is/c++draft/dcl.pre#5)):
> If a
[declarator-id](https://eel.is/c++draft/dcl.decl.general#nt:declarator-id)
is a name, the
[init-declarator](https://eel.is/c++draft/dcl.decl.general#nt:init-declarator)
and (hence) the declaration introduce that
name[.](https://eel.is/c++draft/dcl.pre#5.sentence-1)
> [Note [3](https://eel.is/c++draft/dcl.pre#note-3): Otherwise, the
[declarator-id](https://eel.is/c++draft/dcl.decl.general#nt:declarator-id)
is a
[qualified-id](https://eel.is/c++draft/expr.prim.id.qual#nt:qualified-id)
or names a destructor or its
[unqualified-id](https://eel.is/c++draft/expr.prim.id.unqual#nt:unqualified-id)
is a [template-id](https://eel.is/c++draft/temp.names#nt:template-id)
and no name is
introduced[.](https://eel.is/c++draft/dcl.pre#5.sentence-2)
— end note]
---
 clang/test/CXX/drs/cwg18xx.cpp | 47 ++++++++++++++++++++++++++++++++++
 clang/www/cxx_dr_status.html   |  2 +-
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/clang/test/CXX/drs/cwg18xx.cpp b/clang/test/CXX/drs/cwg18xx.cpp
index 3a2248a1af55..ea322ff4c8be 100644
--- a/clang/test/CXX/drs/cwg18xx.cpp
+++ b/clang/test/CXX/drs/cwg18xx.cpp
@@ -222,6 +222,53 @@ namespace cwg1815 { // cwg1815: no
 #endif
 }
 
+namespace cwg1820 { // cwg1820: 3.5
+typedef int A;
+typedef int cwg1820::A;
+// expected-warning@-1 {{extra qualification on member 'A'}}
+// expected-error@-2 {{typedef declarator cannot be qualified}}
+
+namespace B {
+typedef int cwg1820::A;
+// expected-error@-1 {{cannot define or redeclare 'A' here because namespace 'B' does not enclose namespace 'cwg1820'}}
+// expected-error@-2 {{typedef declarator cannot be qualified}}
+}
+
+class C1 {
+  typedef int cwg1820::A;
+  // expected-error@-1 {{non-friend class member 'A' cannot have a qualified name}}
+  // expected-error@-2 {{typedef declarator cannot be qualified}}
+};
+
+template <typename>
+class C2 {
+  typedef int cwg1820::A;
+  // expected-error@-1 {{non-friend class member 'A' cannot have a qualified name}}
+  // expected-error@-2 {{typedef declarator cannot be qualified}}
+};
+
+void d1() {
+  typedef int cwg1820::A;
+  // expected-error@-1 {{definition or redeclaration of 'A' not allowed inside a function}}
+  // expected-error@-2 {{typedef declarator cannot be qualified}}
+}
+
+template<typename>
+void d2() {
+  typedef int cwg1820::A;
+  // expected-error@-1 {{definition or redeclaration of 'A' not allowed inside a function}}
+  // expected-error@-2 {{typedef declarator cannot be qualified}}
+}
+
+#if __cplusplus >= 201103L
+auto e = [] {
+  typedef int cwg1820::A;
+  // expected-error@-1 {{definition or redeclaration of 'A' not allowed inside a function}}
+  // expected-error@-2 {{typedef declarator cannot be qualified}}
+};
+#endif
+} // namespace cwg1820
+
 namespace cwg1821 { // cwg1821: 2.9
 struct A {
   template <typename> struct B {
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 992dc61d65e4..12ab59224e82 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -10728,7 +10728,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1820.html">1820</a></td>
     <td>CD6</td>
     <td>Qualified typedef names</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 3.5</td>
   </tr>
   <tr id="1821">
     <td><a href="https://cplusplus.github.io/CWG/issues/1821.html">1821</a></td>
-- 
GitLab


From 52c5a81001663d4ca8fca5e7b1634a6a5b0bd021 Mon Sep 17 00:00:00 2001
From: Jonas Hahnfeld <jonas.hahnfeld@cern.ch>
Date: Sat, 11 May 2024 09:18:05 +0200
Subject: [PATCH 0477/1206] [AMDGPU] Fix unittest linking error with
 `LLVM_LINK_LLVM_DYLIB` (#91727)

Follow-up to https://github.com/llvm/llvm-project/pull/88257
---
 llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h |  4 +++-
 llvm/lib/Target/AMDGPU/SIProgramInfo.h             |  3 ++-
 llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp  | 12 ++++--------
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
index 26229af638f2..0e3bc63919f0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
@@ -18,6 +18,7 @@
 #include "llvm/BinaryFormat/MsgPackDocument.h"
 #include "llvm/Support/AMDGPUMetadata.h"
 #include "llvm/Support/Alignment.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -61,7 +62,8 @@ protected:
                                msgpack::MapDocNode Kern) = 0;
 };
 
-class MetadataStreamerMsgPackV4 : public MetadataStreamer {
+class LLVM_EXTERNAL_VISIBILITY MetadataStreamerMsgPackV4
+    : public MetadataStreamer {
 protected:
   std::unique_ptr<msgpack::Document> HSAMetadataDoc =
       std::make_unique<msgpack::Document>();
diff --git a/llvm/lib/Target/AMDGPU/SIProgramInfo.h b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
index c0a353033c3c..e66e5a194c8b 100644
--- a/llvm/lib/Target/AMDGPU/SIProgramInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
@@ -17,6 +17,7 @@
 #define LLVM_LIB_TARGET_AMDGPU_SIPROGRAMINFO_H
 
 #include "llvm/IR/CallingConv.h"
+#include "llvm/Support/Compiler.h"
 #include <cstdint>
 
 namespace llvm {
@@ -27,7 +28,7 @@ class MCExpr;
 class MachineFunction;
 
 /// Track resource usage for kernels / entry functions.
-struct SIProgramInfo {
+struct LLVM_EXTERNAL_VISIBILITY SIProgramInfo {
     // Fields set in PGM_RSRC1 pm4 packet.
     const MCExpr *VGPRBlocks = nullptr;
     const MCExpr *SGPRBlocks = nullptr;
diff --git a/llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp b/llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp
index 912974fa23bd..57828a728931 100644
--- a/llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp
+++ b/llvm/unittests/MC/AMDGPU/SIProgramInfoMCExprs.cpp
@@ -7,9 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPUHSAMetadataStreamer.h"
-#include "AMDGPUTargetMachine.h"
-#include "GCNSubtarget.h"
 #include "SIProgramInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
@@ -25,9 +24,8 @@ using namespace llvm;
 
 class SIProgramInfoMCExprsTest : public testing::Test {
 protected:
-  std::unique_ptr<GCNTargetMachine> TM;
+  std::unique_ptr<LLVMTargetMachine> TM;
   std::unique_ptr<LLVMContext> Ctx;
-  std::unique_ptr<GCNSubtarget> ST;
   std::unique_ptr<MachineModuleInfo> MMI;
   std::unique_ptr<MachineFunction> MF;
   std::unique_ptr<Module> M;
@@ -49,7 +47,7 @@ protected:
     const Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error);
     TargetOptions Options;
 
-    TM.reset(static_cast<GCNTargetMachine *>(TheTarget->createTargetMachine(
+    TM.reset(static_cast<LLVMTargetMachine *>(TheTarget->createTargetMachine(
         Triple, CPU, FS, Options, std::nullopt, std::nullopt)));
 
     Ctx = std::make_unique<LLVMContext>();
@@ -59,9 +57,7 @@ protected:
     auto *F = Function::Create(FType, GlobalValue::ExternalLinkage, "Test", *M);
     MMI = std::make_unique<MachineModuleInfo>(TM.get());
 
-    ST = std::make_unique<GCNSubtarget>(TM->getTargetTriple(),
-                                        TM->getTargetCPU(),
-                                        TM->getTargetFeatureString(), *TM);
+    auto *ST = TM->getSubtargetImpl(*F);
 
     MF = std::make_unique<MachineFunction>(*F, *TM, *ST, 1, *MMI);
     MF->initTargetMachineFunctionInfo(*ST);
-- 
GitLab


From 575177f610274e909a17450c29d2e997f7a7f13d Mon Sep 17 00:00:00 2001
From: Shengchen Kan <shengchen.kan@intel.com>
Date: Sat, 11 May 2024 15:51:05 +0800
Subject: [PATCH 0478/1206] [X86] Add sub-feature nf (no flags update) for APX

This is a follow-up patch for #74199
---
 clang/include/clang/Driver/Options.td              | 4 ++--
 clang/lib/Basic/Targets/X86.cpp                    | 6 ++++++
 clang/lib/Basic/Targets/X86.h                      | 1 +
 clang/lib/Driver/ToolChains/Arch/X86.cpp           | 3 ++-
 clang/test/Driver/x86-target-features.c            | 2 ++
 clang/test/Preprocessor/x86_target_features.c      | 2 ++
 llvm/include/llvm/TargetParser/X86TargetParser.def | 1 +
 llvm/lib/Target/X86/X86.td                         | 2 ++
 llvm/lib/TargetParser/X86TargetParser.cpp          | 1 +
 9 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 73a2518480e9..ed3f1b8b2981 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -6273,9 +6273,9 @@ def mno_gather : Flag<["-"], "mno-gather">, Group<m_Group>,
 def mno_scatter : Flag<["-"], "mno-scatter">, Group<m_Group>,
                   HelpText<"Disable generation of scatter instructions in auto-vectorization(x86 only)">;
 def mapx_features_EQ : CommaJoined<["-"], "mapx-features=">, Group<m_x86_Features_Group>,
-    HelpText<"Enable features of APX">, Values<"egpr,push2pop2,ppx,ndd,ccmp,cf">;
+    HelpText<"Enable features of APX">, Values<"egpr,push2pop2,ppx,ndd,ccmp,nf,cf">;
 def mno_apx_features_EQ : CommaJoined<["-"], "mno-apx-features=">, Group<m_x86_Features_Group>,
-    HelpText<"Disable features of APX">, Values<"egpr,push2pop2,ppx,ndd,ccmp,cf">;
+    HelpText<"Disable features of APX">, Values<"egpr,push2pop2,ppx,ndd,ccmp,nf,cf">;
 // Features egpr, push2pop2, ppx and ndd are validated with llvm-test-suite && cpu2017 on Intel SDE.
 // For stability, we turn on these features only for -mapxf. After a feature pass the validation,
 // we will add it to -mapxf.
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 67e2126cf766..b823eaf6ce33 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -458,6 +458,8 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasNDD = true;
     } else if (Feature == "+ccmp") {
       HasCCMP = true;
+    } else if (Feature == "+nf") {
+      HasNF = true;
     } else if (Feature == "+cf") {
       HasCF = true;
     }
@@ -969,6 +971,8 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__NDD__");
   if (HasCCMP)
     Builder.defineMacro("__CCMP__");
+  if (HasNF)
+    Builder.defineMacro("__NF__");
   if (HasCF)
     Builder.defineMacro("__CF__");
   // Condition here is aligned with the feature set of mapxf in Options.td
@@ -1174,6 +1178,7 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const {
       .Case("ppx", true)
       .Case("ndd", true)
       .Case("ccmp", true)
+      .Case("nf", true)
       .Case("cf", true)
       .Default(false);
 }
@@ -1296,6 +1301,7 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
       .Case("ppx", HasPPX)
       .Case("ndd", HasNDD)
       .Case("ccmp", HasCCMP)
+      .Case("nf", HasNF)
       .Case("cf", HasCF)
       .Default(false);
 }
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index c14e4d5f433d..fc00407d461e 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -173,6 +173,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
   bool HasPPX = false;
   bool HasNDD = false;
   bool HasCCMP = false;
+  bool HasNF = false;
   bool HasCF = false;
 
 protected:
diff --git a/clang/lib/Driver/ToolChains/Arch/X86.cpp b/clang/lib/Driver/ToolChains/Arch/X86.cpp
index 53e26a9f8e22..8295d001ec6f 100644
--- a/clang/lib/Driver/ToolChains/Arch/X86.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/X86.cpp
@@ -273,7 +273,8 @@ void x86::getX86TargetFeatures(const Driver &D, const llvm::Triple &Triple,
 
       for (StringRef Value : A->getValues()) {
         if (Value == "egpr" || Value == "push2pop2" || Value == "ppx" ||
-            Value == "ndd" || Value == "ccmp" || Value == "cf") {
+            Value == "ndd" || Value == "ccmp" || Value == "nf" ||
+            Value == "cf") {
           Features.push_back(
               Args.MakeArgString((IsNegative ? "-" : "+") + Value));
           continue;
diff --git a/clang/test/Driver/x86-target-features.c b/clang/test/Driver/x86-target-features.c
index 998d5f37da69..25f8f66bc321 100644
--- a/clang/test/Driver/x86-target-features.c
+++ b/clang/test/Driver/x86-target-features.c
@@ -436,12 +436,14 @@
 // RUN: %clang -target x86_64-unknown-linux-gnu -mapx-features=ppx %s -### -o %t.o 2>&1 | FileCheck -check-prefix=PPX %s
 // RUN: %clang -target x86_64-unknown-linux-gnu -mapx-features=ndd %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NDD %s
 // RUN: %clang -target x86_64-unknown-linux-gnu -mapx-features=ccmp %s -### -o %t.o 2>&1 | FileCheck -check-prefix=CCMP %s
+// RUN: %clang -target x86_64-unknown-linux-gnu -mapx-features=nf %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NF %s
 // RUN: %clang -target x86_64-unknown-linux-gnu -mapx-features=cf %s -### -o %t.o 2>&1 | FileCheck -check-prefix=CF %s
 // EGPR: "-target-feature" "+egpr"
 // PUSH2POP2: "-target-feature" "+push2pop2"
 // PPX: "-target-feature" "+ppx"
 // NDD: "-target-feature" "+ndd"
 // CCMP: "-target-feature" "+ccmp"
+// NF: "-target-feature" "+nf"
 // CF: "-target-feature" "+cf"
 
 // RUN: %clang -target x86_64-unknown-linux-gnu -mapx-features=egpr,ndd %s -### -o %t.o 2>&1 | FileCheck -check-prefix=EGPR-NDD %s
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index 5602c59158fe..57104c9e7a50 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -802,6 +802,7 @@
 // RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapx-features=ppx -x c -E -dM -o - %s | FileCheck --check-prefix=PPX %s
 // RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapx-features=ndd -x c -E -dM -o - %s | FileCheck --check-prefix=NDD %s
 // RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapx-features=ccmp -x c -E -dM -o - %s | FileCheck --check-prefix=CCMP %s
+// RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapx-features=nf -x c -E -dM -o - %s | FileCheck --check-prefix=NF %s
 // RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapx-features=cf -x c -E -dM -o - %s | FileCheck --check-prefix=CF %s
 // RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapxf -x c -E -dM -o - %s | FileCheck --check-prefixes=EGPR,PUSH2POP2,PPX,NDD,APXF %s
 // APXF: #define __APX_F__ 1
@@ -809,5 +810,6 @@
 // CF: #define __CF__ 1
 // EGPR: #define __EGPR__ 1
 // NDD: #define __NDD__ 1
+// NF: #define __NF__ 1
 // PPX: #define __PPX__ 1
 // PUSH2POP2: #define __PUSH2POP2__ 1
diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def
index a9ed56fcd470..5670767ff7ed 100644
--- a/llvm/include/llvm/TargetParser/X86TargetParser.def
+++ b/llvm/include/llvm/TargetParser/X86TargetParser.def
@@ -253,6 +253,7 @@ X86_FEATURE_COMPAT(USERMSR,         "usermsr",                0)
 X86_FEATURE_COMPAT(AVX10_1,         "avx10.1-256",            0)
 X86_FEATURE_COMPAT(AVX10_1_512,     "avx10.1-512",            0)
 X86_FEATURE       (EVEX512,         "evex512")
+X86_FEATURE       (NF,              "nf")
 X86_FEATURE       (CF,              "cf")
 // These features aren't really CPU features, but the frontend can set them.
 X86_FEATURE       (RETPOLINE_EXTERNAL_THUNK,    "retpoline-external-thunk")
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 9f5b58d78fcc..54642ecde18c 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -351,6 +351,8 @@ def FeatureNDD : SubtargetFeature<"ndd", "HasNDD", "true",
                                   "Support non-destructive destination">;
 def FeatureCCMP : SubtargetFeature<"ccmp", "HasCCMP", "true",
                                    "Support conditional cmp & test instructions">;
+def FeatureNF : SubtargetFeature<"nf", "HasNF", "true",
+                                 "Support status flags update suppression">;
 def FeatureCF : SubtargetFeature<"cf", "HasCF", "true",
                                  "Support conditional faulting">;
 
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 21f46f576490..efe392b94545 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -632,6 +632,7 @@ constexpr FeatureBitset ImpliedFeaturesPush2Pop2 = {};
 constexpr FeatureBitset ImpliedFeaturesPPX = {};
 constexpr FeatureBitset ImpliedFeaturesNDD = {};
 constexpr FeatureBitset ImpliedFeaturesCCMP = {};
+constexpr FeatureBitset ImpliedFeaturesNF = {};
 constexpr FeatureBitset ImpliedFeaturesCF = {};
 
 constexpr FeatureInfo FeatureInfos[X86::CPU_FEATURE_MAX] = {
-- 
GitLab


From 31774b6a8a88b435ce79f9ba048ef8bb00e2817e Mon Sep 17 00:00:00 2001
From: Ryosuke Niwa <rniwa@webkit.org>
Date: Sat, 11 May 2024 01:19:14 -0700
Subject: [PATCH 0479/1206] [analyzer] Treat bitwise_cast, std::addressof, and
 new as trivial in WebKit checkers. (#91830)

---
 .../Checkers/WebKit/PtrTypesSemantics.cpp     |  9 ++++-
 .../Checkers/WebKit/uncounted-obj-arg.cpp     | 38 +++++++++++++++++++
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index 3abfa4cbb295..ad493587affa 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -350,8 +350,11 @@ public:
     const auto &Name = safeGetName(Callee);
 
     if (Name == "WTFCrashWithInfo" || Name == "WTFBreakpointTrap" ||
-        Name == "WTFReportAssertionFailure" ||
-        Name == "compilerFenceForCrash" || Name.find("__builtin") == 0)
+        Name == "WTFReportAssertionFailure" || Name == "isMainThread" ||
+        Name == "isMainThreadOrGCThread" || Name == "isMainRunLoop" ||
+        Name == "isWebThread" || Name == "isUIThread" ||
+        Name == "compilerFenceForCrash" || Name == "bitwise_cast" ||
+        Name == "addressof" || Name.find("__builtin") == 0)
       return true;
 
     return TrivialFunctionAnalysis::isTrivialImpl(Callee, Cache);
@@ -428,6 +431,8 @@ public:
     return TrivialFunctionAnalysis::isTrivialImpl(CE->getConstructor(), Cache);
   }
 
+  bool VisitCXXNewExpr(const CXXNewExpr *NE) { return VisitChildren(NE); }
+
   bool VisitImplicitCastExpr(const ImplicitCastExpr *ICE) {
     return Visit(ICE->getSubExpr());
   }
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
index 6ca7677511d7..073f3252160e 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
@@ -59,6 +59,18 @@ void WTFCrashWithInfo(int line, const char* file, const char* function, int coun
     WTFCrashWithInfoImpl(line, file, function, counter, wtfCrashArg(reason));
 }
 
+template<typename ToType, typename FromType>
+ToType bitwise_cast(FromType from);
+
+template<typename T>
+T* addressof(T& arg);
+
+bool isMainThread();
+bool isMainThreadOrGCThread();
+bool isMainRunLoop();
+bool isWebThread();
+bool isUIThread();
+
 enum class Flags : unsigned short {
   Flag1 = 1 << 0,
   Flag2 = 1 << 1,
@@ -234,6 +246,16 @@ public:
   void trivial38() { v++; if (__builtin_expect(!!(number), 1)) (*number)++; }
   int trivial39() { return -v; }
   int trivial40() { return v << 2; }
+  unsigned trivial41() { v = ++s_v; return v; }
+  unsigned trivial42() { return bitwise_cast<unsigned long>(nullptr); }
+  Number* trivial43() { return addressof(*number); }
+  Number* trivial44() { return new Number(1); }
+  ComplexNumber* trivial45() { return new ComplexNumber(); }
+  void trivial46() { ASSERT(isMainThread()); }
+  void trivial47() { ASSERT(isMainThreadOrGCThread()); }
+  void trivial48() { ASSERT(isMainRunLoop()); }
+  void trivial49() { ASSERT(isWebThread()); }
+  void trivial50() { ASSERT(isUIThread()); }
 
   static RefCounted& singleton() {
     static RefCounted s_RefCounted;
@@ -312,13 +334,17 @@ public:
   void nonTrivial16() { complex++; }
   ComplexNumber nonTrivial17() { return complex << 2; }
   ComplexNumber nonTrivial18() { return +complex; }
+  ComplexNumber* nonTrivial19() { return new ComplexNumber(complex); }
 
+  static unsigned s_v;
   unsigned v { 0 };
   Number* number { nullptr };
   ComplexNumber complex;
   Enum enumValue { Enum::Value1 };
 };
 
+unsigned RefCounted::s_v = 0;
+
 RefCounted* refCountedObj();
 
 void test()
@@ -377,6 +403,16 @@ public:
     getFieldTrivial().trivial38(); // no-warning
     getFieldTrivial().trivial39(); // no-warning
     getFieldTrivial().trivial40(); // no-warning
+    getFieldTrivial().trivial41(); // no-warning
+    getFieldTrivial().trivial42(); // no-warning
+    getFieldTrivial().trivial43(); // no-warning
+    getFieldTrivial().trivial44(); // no-warning
+    getFieldTrivial().trivial45(); // no-warning
+    getFieldTrivial().trivial46(); // no-warning
+    getFieldTrivial().trivial47(); // no-warning
+    getFieldTrivial().trivial48(); // no-warning
+    getFieldTrivial().trivial49(); // no-warning
+    getFieldTrivial().trivial50(); // no-warning
 
     RefCounted::singleton().trivial18(); // no-warning
     RefCounted::singleton().someFunction(); // no-warning
@@ -419,6 +455,8 @@ public:
     // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
     getFieldTrivial().nonTrivial18();
     // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    getFieldTrivial().nonTrivial19();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
   }
 };
 
-- 
GitLab


From 9d66dcaf172cdaa78a6709df989bcaaf90d2ccee Mon Sep 17 00:00:00 2001
From: Spenser Bauman <sbauman@mathworks.com>
Date: Sat, 11 May 2024 07:28:52 -0400
Subject: [PATCH 0480/1206] [mlir][tosa] Work around GCC bug in tosa-to-tensor
 (#91521)

GCC 12 and 13 generate incorrect code for a pattern in the
tosa-to-tensor pass responsible for lowering tosa.reshape. This results
in the tosa.reshape lowering producing IR which fails to verify. I've
narrowed down the set of cmake flags needed to reproduce the issue to
this:

    cmake -G Ninja ../llvm \
      -DLLVM_ENABLE_PROJECTS="mlir" \
      -DLLVM_TARGETS_TO_BUILD=host \
      -DLLVM_ENABLE_PROJECTS=mlir \
      -DCMAKE_BUILD_TYPE="Release" \
      -DCMAKE_CXX_FLAGS_RELEASE="-O2" \
      -DCMAKE_CXX_FLAGS="-O2" \
      -DCMAKE_CXX_COMPILER=g++ \
      -DCMAKE_C_COMPILER=gcc

This is the failing test case:

func.func @fails_in_gcc_12(%arg0: tensor<?xf32>) -> tensor<1x1x1x?xf32>
{
%0 = tosa.reshape %arg0 {new_shape = array<i64: 1, 1, 1, -1>} :
(tensor<?xf32>) -> tensor<1x1x1x?xf32>
      return %0 : tensor<1x1x1x?xf32>
    }

This should lower to a tensor.expand_shape operation like so:

    func.func @foo(%arg0: tensor<?xf32>) -> tensor<1x1x1x?xf32> {
      %c0 = arith.constant 0 : index
      %dim = tensor.dim %arg0, %c0 : tensor<?xf32>
      %c1 = arith.constant 1 : index
%expanded = tensor.expand_shape %arg0 [[0, 1, 2, 3]] output_shape [1, 1,
1, %dim] : tensor<?xf32> into tensor<1x1x1x?xf32>
      return %expanded : tensor<1x1x1x?xf32>
    }

Under GCC 12/13 with the above cmake configuration, the
tensor.expand_shape looks like this

%2 = "tensor.expand_shape"(%arg0) <{reassociation = [[0, 1, 2, 3]],
static_output_shape = array<i64>}> : (tensor<?xf32>) ->
tensor<?x1x1x?xf32>

The key difference is the computed output type of `tensor<?x1x1x?xf32>`
rather than the expected `tensor<1x1x1x?xf32>`. This expand_shape fails
to verify with this error message:

error: 'tensor.expand_shape' op expected number of static shape dims to
be equal to the output rank (4) but found 0 inputs instead

The problematic code is calculating the intermediate shape of the
generated tensor.expand_shape operation in the
expand_shape/collapse_shape sequence that implements tosa.reshape.

    // Compute result shape
    bool resultIsStatic = true;
    auto resultShape = llvm::map_to_vector(newShape, [&](int64_t size) {
      // Omitted

// If we do not know the total size of the tensor, keep this dimension
      // dynamic in the result shape.
      if (!inputIsStatic) {
        resultIsStatic = false;
        return ShapedType::kDynamic;
      }
    });

    if (resultIsStatic) {
      // do something
      return;
    }

    // do something else
    return;

The failure point seems to be the update of the resultIsStatic variable
in the lambda body. The assignment of false is not propagated to the use
in the if-statement, resulting in the branch being taken when it should
not.

I've found several modification to the code that gets around the bug.
The version I settled on is one which makes the logic a little more
obvious.
---
 mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp | 11 +++++------
 .../Conversion/TosaToTensor/tosa-to-tensor.mlir   | 15 +++++++++++++++
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index cd6da3558246..89f956a5e701 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -55,9 +55,8 @@ TensorType inferReshapeExpandedType(TensorType inputType,
   // Check if the input is static, and if so, get its total size
   bool inputIsStatic = inputType.hasStaticShape();
   int64_t totalSize = inputIsStatic ? inputType.getNumElements() : -1;
- 
+
   // Compute result shape
-  bool resultIsStatic = true;
   auto resultShape = llvm::map_to_vector(newShape, [&](int64_t size) -> int64_t {
     // If this is not a placeholder, do not change it
     if (size >= 0)
@@ -65,10 +64,8 @@ TensorType inferReshapeExpandedType(TensorType inputType,
 
     // If we do not know the total size of the tensor, keep this dimension
     // dynamic in the result shape.
-    if (!inputIsStatic) {
-      resultIsStatic = false;
+    if (!inputIsStatic)
       return ShapedType::kDynamic;
-    }
 
     // Calculate the product of all elements in 'newShape' except for the -1
     // placeholder, which we discard by negating the result.
@@ -84,12 +81,14 @@ TensorType inferReshapeExpandedType(TensorType inputType,
     return totalSize / totalSizeNoPlaceholder;
   });
 
+  bool resultIsStatic = !ShapedType::isDynamicShape(resultShape);
+
   // A syntactic restriction in 'tensor.expand_shape' forbids a dynamically
   // shaped input from being reshaped into a statically shaped result. We may
   // simply turn the first result dimension dynamic to address this.
   if (!inputIsStatic && resultIsStatic)
     resultShape[0] = ShapedType::kDynamic;
-  
+
   // The 'tensor.expand_shape' op also forbids a statically shaped input from
   // being reshaped into a dynamically shaped result, but the placeholder
   // inference algorithm above guarantees that this will never be the case.
diff --git a/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir b/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir
index b8c3d56f21f1..72e7e4cc8408 100644
--- a/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir
+++ b/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir
@@ -394,6 +394,21 @@ func.func @test_reshape_6d_down_s2s_auto(%arg0: tensor<1x2x3x5x7x11xf32>) -> ten
 
 // -----
 
+// This test would previously fail on GCC with certain compiler flags.
+// The GCC issue would cause invalid IR after tosa-to-tensor, so this test
+// locks down that the code goes through tosa-to-tensor and verifies.
+//
+// See https://github.com/llvm/llvm-project/pull/91521 for a full description.
+
+// CHECK-LABEL: reshape_bug_fix
+// CHECK: tensor.expand_shape
+func.func @reshape_bug_fix(%arg0: tensor<?xf32>) -> tensor<1x1x1x?xf32> {
+  %0 = tosa.reshape %arg0 {new_shape = array<i64: 1, 1, 1, -1>} : (tensor<?xf32>) -> tensor<1x1x1x?xf32>
+  return %0 : tensor<1x1x1x?xf32>
+}
+
+// -----
+
 // CHECK-LABEL: test_reshape_6d_down_s2s_explicit
 // CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<1x2x3x5x7x11xf32>
 // CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2], [3], [4, 5]] : tensor<1x2x3x5x7x11xf32> into tensor<6x5x77xf32>
-- 
GitLab


From 079fdef7d26d2004b5bfd467fddadf1906bc814b Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 10 May 2024 22:43:29 +0100
Subject: [PATCH 0481/1206] [TTI] getCommonMaskedMemoryOpCost - use the target
 getMemoryOpCost/getCFInstrCost implementations.

We were using the default implementations instead of the CRTP versions.
---
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      |  8 ++---
 .../Analysis/CostModel/AArch64/masked_ldst.ll | 36 +++++++++----------
 .../CostModel/AArch64/mem-op-cost-model.ll    | 32 ++++++++---------
 .../Analysis/CostModel/AArch64/sve-gather.ll  |  6 ++--
 .../CostModel/AArch64/sve-intrinsics.ll       |  8 ++---
 .../Analysis/CostModel/RISCV/masked_ldst.ll   |  8 ++---
 .../LoopVectorize/AArch64/masked-op-cost.ll   |  4 +--
 7 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index fa481886b268..2091432d4fe2 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -233,8 +233,8 @@ private:
 
     // The cost of the scalar loads/stores.
     InstructionCost MemoryOpCost =
-        VF * getMemoryOpCost(Opcode, VT->getElementType(), Alignment,
-                             AddressSpace, CostKind);
+        VF * thisT()->getMemoryOpCost(Opcode, VT->getElementType(), Alignment,
+                                      AddressSpace, CostKind);
 
     // Next, compute the cost of packing the result in a vector.
     InstructionCost PackingCost =
@@ -253,8 +253,8 @@ private:
           getScalarizationOverhead(
               FixedVectorType::get(Type::getInt1Ty(DataTy->getContext()), VF),
               /*Insert=*/false, /*Extract=*/true, CostKind) +
-          VF * (getCFInstrCost(Instruction::Br, CostKind) +
-                getCFInstrCost(Instruction::PHI, CostKind));
+          VF * (thisT()->getCFInstrCost(Instruction::Br, CostKind) +
+                thisT()->getCFInstrCost(Instruction::PHI, CostKind));
     }
 
     return AddrExtractCost + MemoryOpCost + PackingCost + ConditionalCost;
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
index f5ca6a22b60a..af41ed92319c 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_ldst.ll
@@ -5,24 +5,24 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @fixed() {
 ; CHECK-LABEL: 'fixed'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0(ptr undef, i32 8, <2 x i1> undef, <2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr undef, i32 8, <4 x i1> undef, <4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %v8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr undef, i32 8, <8 x i1> undef, <8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %v16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr undef, i32 8, <16 x i1> undef, <16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0(ptr undef, i32 8, <2 x i1> undef, <2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr undef, i32 8, <4 x i1> undef, <4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %v8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr undef, i32 8, <8 x i1> undef, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr undef, i32 8, <2 x i1> undef, <2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr undef, i32 8, <4 x i1> undef, <4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v2i64 = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr undef, i32 8, <2 x i1> undef, <2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2f16 = call <2 x half> @llvm.masked.load.v2f16.p0(ptr undef, i32 8, <2 x i1> undef, <2 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %v4f16 = call <4 x half> @llvm.masked.load.v4f16.p0(ptr undef, i32 8, <4 x i1> undef, <4 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %v8f16 = call <8 x half> @llvm.masked.load.v8f16.p0(ptr undef, i32 8, <8 x i1> undef, <8 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2f32 = call <2 x float> @llvm.masked.load.v2f32.p0(ptr undef, i32 8, <2 x i1> undef, <2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %v4f32 = call <4 x float> @llvm.masked.load.v4f32.p0(ptr undef, i32 8, <4 x i1> undef, <4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v2f64 = call <2 x double> @llvm.masked.load.v2f64.p0(ptr undef, i32 8, <2 x i1> undef, <2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v4i64 = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr undef, i32 8, <4 x i1> undef, <4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 216 for instruction: %v32f16 = call <32 x half> @llvm.masked.load.v32f16.p0(ptr undef, i32 8, <32 x i1> undef, <32 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0(ptr undef, i32 8, <2 x i1> undef, <2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr undef, i32 8, <4 x i1> undef, <4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr undef, i32 8, <8 x i1> undef, <8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr undef, i32 8, <16 x i1> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0(ptr undef, i32 8, <2 x i1> undef, <2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr undef, i32 8, <4 x i1> undef, <4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr undef, i32 8, <8 x i1> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr undef, i32 8, <2 x i1> undef, <2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr undef, i32 8, <4 x i1> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2i64 = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr undef, i32 8, <2 x i1> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2f16 = call <2 x half> @llvm.masked.load.v2f16.p0(ptr undef, i32 8, <2 x i1> undef, <2 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v4f16 = call <4 x half> @llvm.masked.load.v4f16.p0(ptr undef, i32 8, <4 x i1> undef, <4 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %v8f16 = call <8 x half> @llvm.masked.load.v8f16.p0(ptr undef, i32 8, <8 x i1> undef, <8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2f32 = call <2 x float> @llvm.masked.load.v2f32.p0(ptr undef, i32 8, <2 x i1> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v4f32 = call <4 x float> @llvm.masked.load.v4f32.p0(ptr undef, i32 8, <4 x i1> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2f64 = call <2 x double> @llvm.masked.load.v2f64.p0(ptr undef, i32 8, <2 x i1> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4i64 = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr undef, i32 8, <4 x i1> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 152 for instruction: %v32f16 = call <32 x half> @llvm.masked.load.v32f16.p0(ptr undef, i32 8, <32 x i1> undef, <32 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 entry:
diff --git a/llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll b/llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll
index 521a0900c844..6f9f64a26851 100644
--- a/llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll
@@ -212,11 +212,11 @@ define <4 x i8> @gather_load_4xi8_constant_mask(<4 x ptr> %ptrs) {
 define <4 x i8> @gather_load_4xi8_variable_mask(<4 x ptr> %ptrs, <4 x i1> %cond) {
 ; CHECK:         gather_load_4xi8_variable_mask
 ; CHECK-NEON-LABEL: 'gather_load_4xi8_variable_mask'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef)
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv
 ;
 ; CHECK-SVE-128-LABEL: 'gather_load_4xi8_variable_mask'
-; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef)
+; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef)
 ; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv
 ;
 ; CHECK-SVE-256-LABEL: 'gather_load_4xi8_variable_mask'
@@ -257,11 +257,11 @@ define void @scatter_store_4xi8_constant_mask(<4 x i8> %val, <4 x ptr> %ptrs) {
 define void @scatter_store_4xi8_variable_mask(<4 x i8> %val, <4 x ptr> %ptrs, <4 x i1> %cond) {
 ; CHECK:         scatter_store_4xi8_variable_mask
 ; CHECK-NEON-LABEL: 'scatter_store_4xi8_variable_mask'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-128-LABEL: 'scatter_store_4xi8_variable_mask'
-; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
+; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
 ; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-256-LABEL: 'scatter_store_4xi8_variable_mask'
@@ -302,11 +302,11 @@ define <4 x i32> @gather_load_4xi32_constant_mask(<4 x ptr> %ptrs) {
 define <4 x i32> @gather_load_4xi32_variable_mask(<4 x ptr> %ptrs, <4 x i1> %cond) {
 ; CHECK:         gather_load_4xi32_variable_mask
 ; CHECK-NEON-LABEL: 'gather_load_4xi32_variable_mask'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef)
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv
 ;
 ; CHECK-SVE-128-LABEL: 'gather_load_4xi32_variable_mask'
-; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef)
+; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef)
 ; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv
 ;
 ; CHECK-SVE-256-LABEL: 'gather_load_4xi32_variable_mask'
@@ -347,11 +347,11 @@ define void @scatter_store_4xi32_constant_mask(<4 x i32> %val, <4 x ptr> %ptrs)
 define void @scatter_store_4xi32_variable_mask(<4 x i32> %val, <4 x ptr> %ptrs, <4 x i1> %cond) {
 ; CHECK:         scatter_store_4xi32_variable_mask
 ; CHECK-NEON-LABEL: 'scatter_store_4xi32_variable_mask'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-128-LABEL: 'scatter_store_4xi32_variable_mask'
-; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
+; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
 ; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-256-LABEL: 'scatter_store_4xi32_variable_mask'
@@ -370,11 +370,11 @@ declare <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr>, i32, <256 x
 define void @sve_gather_vls(<256 x i1> %v256i1mask) {
 ; CHECK-LABEL: 'sve_scatter_vls'
 ; CHECK-NEON-LABEL: 'sve_gather_vls'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2304 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1792 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-128-LABEL: 'sve_gather_vls'
-; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 2304 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
+; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 1792 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
 ; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-256-LABEL: 'sve_gather_vls'
@@ -394,11 +394,11 @@ declare <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr>, i32, <256
 define void @sve_gather_vls_float(<256 x i1> %v256i1mask) {
 ; CHECK-LABEL: 'sve_gather_vls_float'
 ; CHECK-NEON-LABEL: 'sve_gather_vls_float'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2176 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1664 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-128-LABEL: 'sve_gather_vls_float'
-; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 2176 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
+; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 1664 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
 ; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-256-LABEL: 'sve_gather_vls_float'
@@ -418,11 +418,11 @@ declare void @llvm.masked.scatter.v256i8.v256p0(<256 x i8>, <256 x ptr>, i32, <2
 define void @sve_scatter_vls(<256 x i1> %v256i1mask){
 ; CHECK-LABEL: 'sve_scatter_vls'
 ; CHECK-NEON-LABEL: 'sve_scatter_vls'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2304 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1792 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask)
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-128-LABEL: 'sve_scatter_vls'
-; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 2304 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask)
+; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 1792 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask)
 ; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-256-LABEL: 'sve_scatter_vls'
@@ -442,11 +442,11 @@ declare void @llvm.masked.scatter.v512f16.v512p0(<512 x half>, <512 x ptr>, i32,
 define void @sve_scatter_vls_float(<512 x i1> %v512i1mask){
 ; CHECK-LABEL: 'sve_scatter_vls_float'
 ; CHECK-NEON-LABEL: 'sve_scatter_vls_float'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4480 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3456 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask)
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-128-LABEL: 'sve_scatter_vls_float'
-; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 4480 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask)
+; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 3456 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask)
 ; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SVE-256-LABEL: 'sve_scatter_vls_float'
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-gather.ll b/llvm/test/Analysis/CostModel/AArch64/sve-gather.ll
index a9c18e20c1f5..c05339d89d35 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-gather.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-gather.ll
@@ -107,15 +107,15 @@ define void @masked_gathers_no_vscale_range() #2 {
 
 define <2 x i128> @masked_gather_v1i128(<2 x ptr> %ld, <2 x i1> %masks, <2 x i128> %passthru) #3 {
 ; CHECK-LABEL: 'masked_gather_v1i128'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i128> %res
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_gather_v1i128'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i128> %res
 ;
 ; CHECK-VSCALE-1-LABEL: 'masked_gather_v1i128'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i128> %res
 ;
   %res = call <2 x i128> @llvm.masked.gather.v2i128.v2p0(<2 x ptr> %ld, i32 0, <2 x i1> %masks, <2 x i128> %passthru)
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
index 15c278b060c9..1ff280d75b4e 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
@@ -829,7 +829,7 @@ define <vscale x 8 x i32> @masked_gather_nxv8i32(<vscale x 8 x ptr> %ld, <vscale
 
 define <4 x i32> @masked_gather_v4i32(<4 x ptr> %ld, <4 x i1> %masks, <4 x i32> %passthru) {
 ; CHECK-LABEL: 'masked_gather_v4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_gather_v4i32'
@@ -842,7 +842,7 @@ define <4 x i32> @masked_gather_v4i32(<4 x ptr> %ld, <4 x i1> %masks, <4 x i32>
 
 define <1 x i128> @masked_gather_v1i128(<1 x ptr> %ld, <1 x i1> %masks, <1 x i128> %passthru) {
 ; CHECK-LABEL: 'masked_gather_v1i128'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <1 x i128> %res
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_gather_v1i128'
@@ -883,7 +883,7 @@ define void @masked_scatter_nxv8i32(<vscale x 8 x i32> %data, <vscale x 8 x ptr>
 
 define void @masked_scatter_v4i32(<4 x i32> %data, <4 x ptr> %ptrs, <4 x i1> %masks) {
 ; CHECK-LABEL: 'masked_scatter_v4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_scatter_v4i32'
@@ -897,7 +897,7 @@ define void @masked_scatter_v4i32(<4 x i32> %data, <4 x ptr> %ptrs, <4 x i1> %ma
 
 define void @masked_scatter_v1i128(<1 x i128> %data, <1 x ptr> %ptrs, <1 x i1> %masks) {
 ; CHECK-LABEL: 'masked_scatter_v1i128'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_scatter_v1i128'
diff --git a/llvm/test/Analysis/CostModel/RISCV/masked_ldst.ll b/llvm/test/Analysis/CostModel/RISCV/masked_ldst.ll
index 31bbc8b02a19..e6f53d4429c7 100644
--- a/llvm/test/Analysis/CostModel/RISCV/masked_ldst.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/masked_ldst.ll
@@ -13,14 +13,14 @@ define void @fixed() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr undef, i32 8, <2 x i1> undef, <2 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr undef, i32 8, <4 x i1> undef, <4 x i32> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr undef, i32 8, <2 x i1> undef, <2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v2f16 = call <2 x half> @llvm.masked.load.v2f16.p0(ptr undef, i32 8, <2 x i1> undef, <2 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v4f16 = call <4 x half> @llvm.masked.load.v4f16.p0(ptr undef, i32 8, <4 x i1> undef, <4 x half> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %v8f16 = call <8 x half> @llvm.masked.load.v8f16.p0(ptr undef, i32 8, <8 x i1> undef, <8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v2f16 = call <2 x half> @llvm.masked.load.v2f16.p0(ptr undef, i32 8, <2 x i1> undef, <2 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %v4f16 = call <4 x half> @llvm.masked.load.v4f16.p0(ptr undef, i32 8, <4 x i1> undef, <4 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %v8f16 = call <8 x half> @llvm.masked.load.v8f16.p0(ptr undef, i32 8, <8 x i1> undef, <8 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = call <2 x float> @llvm.masked.load.v2f32.p0(ptr undef, i32 8, <2 x i1> undef, <2 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = call <4 x float> @llvm.masked.load.v4f32.p0(ptr undef, i32 8, <4 x i1> undef, <4 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = call <2 x double> @llvm.masked.load.v2f64.p0(ptr undef, i32 8, <2 x i1> undef, <2 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr undef, i32 8, <4 x i1> undef, <4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 287 for instruction: %v32f16 = call <32 x half> @llvm.masked.load.v32f16.p0(ptr undef, i32 8, <32 x i1> undef, <32 x half> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 223 for instruction: %v32f16 = call <32 x half> @llvm.masked.load.v32f16.p0(ptr undef, i32 8, <32 x i1> undef, <32 x half> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-op-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-op-cost.ll
index 9969f881063c..6b4cfa091c45 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-op-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-op-cost.ll
@@ -5,8 +5,8 @@
 target triple = "aarch64-unknown-linux-gnu"
 
 ; CHECK-COST: Checking a loop in 'fixed_width'
-; CHECK-COST: Found an estimated cost of 14 for VF 2 For instruction:   store i32 2, ptr %arrayidx1, align 4
-; CHECK-COST: Found an estimated cost of 28 for VF 4 For instruction:   store i32 2, ptr %arrayidx1, align 4
+; CHECK-COST: Found an estimated cost of 10 for VF 2 For instruction:   store i32 2, ptr %arrayidx1, align 4
+; CHECK-COST: Found an estimated cost of 20 for VF 4 For instruction:   store i32 2, ptr %arrayidx1, align 4
 ; CHECK-COST: Selecting VF: 1.
 
 ; We should decide this loop is not worth vectorising using fixed width vectors
-- 
GitLab


From 23fe1fc6b78b7643a801ce3eb53d14b47b1dd0ff Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sat, 11 May 2024 13:37:48 +0100
Subject: [PATCH 0482/1206] [TTI][X86] getGSScalarCost - don't bother with
 adding cost of ICMP for each i1 mask element

These can nearly always be folded into the existing cost of the branch, and brings the throughput costs of the scalarised gather/scatter code much closer to the llvm-mca/uica estimates
---
 .../lib/Target/X86/X86TargetTransformInfo.cpp |   5 +-
 .../CostModel/X86/intrinsic-cost-kinds.ll     |   4 +-
 .../X86/masked-gather-i32-with-i8-index.ll    |   4 +-
 .../X86/masked-gather-i64-with-i8-index.ll    |   4 +-
 .../X86/masked-intrinsic-cost-inseltpoison.ll | 650 +++++++++---------
 .../CostModel/X86/masked-intrinsic-cost.ll    | 650 +++++++++---------
 6 files changed, 657 insertions(+), 660 deletions(-)

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index d43480d0a012..ebcc11533add 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5864,11 +5864,8 @@ InstructionCost X86TTIImpl::getGSScalarCost(unsigned Opcode,
         FixedVectorType::get(Type::getInt1Ty(SrcVTy->getContext()), VF);
     MaskUnpackCost = getScalarizationOverhead(
         MaskTy, DemandedElts, /*Insert=*/false, /*Extract=*/true, CostKind);
-    InstructionCost ScalarCompareCost = getCmpSelInstrCost(
-        Instruction::ICmp, Type::getInt1Ty(SrcVTy->getContext()), nullptr,
-        CmpInst::BAD_ICMP_PREDICATE, CostKind);
     InstructionCost BranchCost = getCFInstrCost(Instruction::Br, CostKind);
-    MaskUnpackCost += VF * (BranchCost + ScalarCompareCost);
+    MaskUnpackCost += VF * BranchCost;
   }
 
   InstructionCost AddressUnpackCost = getScalarizationOverhead(
diff --git a/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll b/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
index a55576089c7f..06429a510711 100644
--- a/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
+++ b/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
@@ -310,7 +310,7 @@ define void @fshl(i32 %a, i32 %b, i32 %c, <16 x i32> %va, <16 x i32> %vb, <16 x
 
 define void @maskedgather(<16 x ptr> %va, <16 x i1> %vb, <16 x float> %vc) {
 ; THRU-LABEL: 'maskedgather'
-; THRU-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; LATE-LABEL: 'maskedgather'
@@ -331,7 +331,7 @@ define void @maskedgather(<16 x ptr> %va, <16 x i1> %vb, <16 x float> %vc) {
 
 define void @maskedscatter(<16 x float> %va, <16 x ptr> %vb, <16 x i1> %vc) {
 ; THRU-LABEL: 'maskedscatter'
-; THRU-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> %va, <16 x ptr> %vb, i32 1, <16 x i1> %vc)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> %va, <16 x ptr> %vb, i32 1, <16 x i1> %vc)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; LATE-LABEL: 'maskedscatter'
diff --git a/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll
index 80b350254e94..b62d3fb25091 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll
@@ -49,8 +49,8 @@ define void @test() {
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, ptr %inB, align 4
-; AVX512:  LV: Found an estimated cost of 10 for VF 2 For instruction: %valB.loaded = load i32, ptr %inB, align 4
-; AVX512:  LV: Found an estimated cost of 21 for VF 4 For instruction: %valB.loaded = load i32, ptr %inB, align 4
+; AVX512:  LV: Found an estimated cost of 8 for VF 2 For instruction: %valB.loaded = load i32, ptr %inB, align 4
+; AVX512:  LV: Found an estimated cost of 17 for VF 4 For instruction: %valB.loaded = load i32, ptr %inB, align 4
 ; AVX512:  LV: Found an estimated cost of 10 for VF 8 For instruction: %valB.loaded = load i32, ptr %inB, align 4
 ; AVX512:  LV: Found an estimated cost of 18 for VF 16 For instruction: %valB.loaded = load i32, ptr %inB, align 4
 ; AVX512:  LV: Found an estimated cost of 36 for VF 32 For instruction: %valB.loaded = load i32, ptr %inB, align 4
diff --git a/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll
index 9146ca498237..1d3e45765e51 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll
@@ -49,8 +49,8 @@ define void @test() {
 ;
 ; AVX512-LABEL: 'test'
 ; AVX512:  LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, ptr %inB, align 8
-; AVX512:  LV: Found an estimated cost of 10 for VF 2 For instruction: %valB.loaded = load i64, ptr %inB, align 8
-; AVX512:  LV: Found an estimated cost of 22 for VF 4 For instruction: %valB.loaded = load i64, ptr %inB, align 8
+; AVX512:  LV: Found an estimated cost of 8 for VF 2 For instruction: %valB.loaded = load i64, ptr %inB, align 8
+; AVX512:  LV: Found an estimated cost of 18 for VF 4 For instruction: %valB.loaded = load i64, ptr %inB, align 8
 ; AVX512:  LV: Found an estimated cost of 10 for VF 8 For instruction: %valB.loaded = load i64, ptr %inB, align 8
 ; AVX512:  LV: Found an estimated cost of 20 for VF 16 For instruction: %valB.loaded = load i64, ptr %inB, align 8
 ; AVX512:  LV: Found an estimated cost of 40 for VF 32 For instruction: %valB.loaded = load i64, ptr %inB, align 8
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll
index b0f9f6d32a56..25d12da306ab 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll
@@ -732,118 +732,118 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
 
 define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_gather'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 162 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_gather'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_gather'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 278 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 214 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_gather'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_gather'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
@@ -851,73 +851,73 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; KNL-LABEL: 'masked_gather'
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKX-LABEL: 'masked_gather'
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
@@ -955,192 +955,192 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 
 define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_scatter'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 162 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_scatter'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_scatter'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 278 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 214 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_scatter'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_scatter'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; KNL-LABEL: 'masked_scatter'
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKX-LABEL: 'masked_scatter'
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
@@ -1788,19 +1788,19 @@ define <2 x i32> @test8(<2 x i32> %trigger, ptr %addr, <2 x i32> %dst) {
 
 define <2 x double> @test_gather_2f64(<2 x ptr> %ptrs, <2 x i1> %mask, <2 x double> %src0)  {
 ; SSE2-LABEL: 'test_gather_2f64'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 ; SSE42-LABEL: 'test_gather_2f64'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 ; AVX1-LABEL: 'test_gather_2f64'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 ; AVX2-LABEL: 'test_gather_2f64'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 ; SKL-LABEL: 'test_gather_2f64'
@@ -1808,7 +1808,7 @@ define <2 x double> @test_gather_2f64(<2 x ptr> %ptrs, <2 x i1> %mask, <2 x doub
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 ; AVX512-LABEL: 'test_gather_2f64'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
   %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
@@ -1817,19 +1817,19 @@ define <2 x double> @test_gather_2f64(<2 x ptr> %ptrs, <2 x i1> %mask, <2 x doub
 
 define <4 x i32> @test_gather_4i32(<4 x ptr> %ptrs, <4 x i1> %mask, <4 x i32> %src0)  {
 ; SSE2-LABEL: 'test_gather_4i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; SSE42-LABEL: 'test_gather_4i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; AVX1-LABEL: 'test_gather_4i32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; AVX2-LABEL: 'test_gather_4i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; SKL-LABEL: 'test_gather_4i32'
@@ -1837,7 +1837,7 @@ define <4 x i32> @test_gather_4i32(<4 x ptr> %ptrs, <4 x i1> %mask, <4 x i32> %s
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; KNL-LABEL: 'test_gather_4i32'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; SKX-LABEL: 'test_gather_4i32'
@@ -1929,25 +1929,25 @@ define <16 x float> @test_gather_16f32_var_mask(ptr %base, <16 x i32> %ind, <16
 ; SSE2-LABEL: 'test_gather_16f32_var_mask'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; SSE42-LABEL: 'test_gather_16f32_var_mask'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; AVX1-LABEL: 'test_gather_16f32_var_mask'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; AVX2-LABEL: 'test_gather_16f32_var_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; SKL-LABEL: 'test_gather_16f32_var_mask'
@@ -1973,25 +1973,25 @@ define <16 x float> @test_gather_16f32_ra_var_mask(<16 x ptr> %ptrs, <16 x i32>
 ; SSE2-LABEL: 'test_gather_16f32_ra_var_mask'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; SSE42-LABEL: 'test_gather_16f32_ra_var_mask'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; AVX1-LABEL: 'test_gather_16f32_ra_var_mask'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; AVX2-LABEL: 'test_gather_16f32_ra_var_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; SKL-LABEL: 'test_gather_16f32_ra_var_mask'
@@ -2078,7 +2078,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> poison, <16 x i32> zeroinitializer
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_scatter_16i32'
@@ -2086,7 +2086,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> poison, <16 x i32> zeroinitializer
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_scatter_16i32'
@@ -2094,7 +2094,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> poison, <16 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_scatter_16i32'
@@ -2102,7 +2102,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> poison, <16 x i32> zeroinitializer
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SKL-LABEL: 'test_scatter_16i32'
@@ -2110,7 +2110,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> poison, <16 x i32> zeroinitializer
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; SKL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_scatter_16i32'
@@ -2132,15 +2132,15 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 
 define void @test_scatter_8i32(<8 x i32>%a1, <8 x ptr> %ptr, <8 x i1>%mask) {
 ; SSE2-LABEL: 'test_scatter_8i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_scatter_8i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_scatter_8i32'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_scatter_8i32'
@@ -2153,19 +2153,19 @@ define void @test_scatter_8i32(<8 x i32>%a1, <8 x ptr> %ptr, <8 x i1>%mask) {
 
 define void @test_scatter_4i32(<4 x i32>%a1, <4 x ptr> %ptr, <4 x i1>%mask) {
 ; SSE2-LABEL: 'test_scatter_4i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_scatter_4i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_scatter_4i32'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; KNL-LABEL: 'test_scatter_4i32'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SKX-LABEL: 'test_scatter_4i32'
@@ -2180,25 +2180,25 @@ define <4 x float> @test_gather_4f32(ptr %ptr, <4 x i32> %ind, <4 x i1>%mask) {
 ; SSE2-LABEL: 'test_gather_4f32'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
 ;
 ; SSE42-LABEL: 'test_gather_4f32'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
 ;
 ; AVX1-LABEL: 'test_gather_4f32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
 ;
 ; AVX2-LABEL: 'test_gather_4f32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
 ;
 ; SKL-LABEL: 'test_gather_4f32'
@@ -2210,7 +2210,7 @@ define <4 x float> @test_gather_4f32(ptr %ptr, <4 x i32> %ind, <4 x i1>%mask) {
 ; KNL-LABEL: 'test_gather_4f32'
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
 ;
 ; SKX-LABEL: 'test_gather_4f32'
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
index 46123e9f6057..332d90ac4191 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
@@ -732,118 +732,118 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
 
 define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_gather'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 162 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_gather'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_gather'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 278 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 214 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_gather'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_gather'
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
@@ -851,73 +851,73 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; KNL-LABEL: 'masked_gather'
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKX-LABEL: 'masked_gather'
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x double> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x double> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x double> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x float> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i64> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i64> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i64> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> undef, i32 1, <1 x i1> %m1, <1 x i64> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i32> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i32> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> undef, i32 1, <2 x i1> %m2, <2 x i32> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: %V32I16 = call <32 x i16> @llvm.masked.gather.v32i16.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i16> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I16 = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i16> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i16> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I16 = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> undef, i32 1, <4 x i1> %m4, <4 x i16> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: %V64I8 = call <64 x i8> @llvm.masked.gather.v64i8.v64p0(<64 x ptr> undef, i32 1, <64 x i1> %m64, <64 x i8> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> undef, i32 1, <32 x i1> %m32, <32 x i8> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
@@ -955,192 +955,192 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
 
 define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8, <16 x i1> %m16, <32 x i1> %m32, <64 x i1> %m64) {
 ; SSE2-LABEL: 'masked_scatter'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 162 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SSE42-LABEL: 'masked_scatter'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX1-LABEL: 'masked_scatter'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 278 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 214 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; AVX2-LABEL: 'masked_scatter'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKL-LABEL: 'masked_scatter'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 139 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 107 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; KNL-LABEL: 'masked_scatter'
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
 ; SKX-LABEL: 'masked_scatter'
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1f64.v1p0(<1 x double> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i64.v8p0(<8 x i64> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.scatter.v1i64.v1p0(<1 x i64> undef, <1 x ptr> undef, i32 1, <1 x i1> %m1)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 175 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 87 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 347 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 173 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> undef, <2 x ptr> undef, i32 1, <2 x i1> %m2)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 143 for instruction: call void @llvm.masked.scatter.v32i16.v32p0(<32 x i16> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i16.v16p0(<16 x i16> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i16.v8p0(<8 x i16> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> undef, <4 x ptr> undef, i32 1, <4 x i1> %m4)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 283 for instruction: call void @llvm.masked.scatter.v64i8.v64p0(<64 x i8> undef, <64 x ptr> undef, i32 1, <64 x i1> %m64)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 141 for instruction: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> undef, i32 1, <32 x i1> %m32)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
+; SKX-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
 ; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 0
 ;
   call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
@@ -1788,19 +1788,19 @@ define <2 x i32> @test8(<2 x i32> %trigger, ptr %addr, <2 x i32> %dst) {
 
 define <2 x double> @test_gather_2f64(<2 x ptr> %ptrs, <2 x i1> %mask, <2 x double> %src0)  {
 ; SSE2-LABEL: 'test_gather_2f64'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 ; SSE42-LABEL: 'test_gather_2f64'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 ; AVX1-LABEL: 'test_gather_2f64'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 ; AVX2-LABEL: 'test_gather_2f64'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 ; SKL-LABEL: 'test_gather_2f64'
@@ -1808,7 +1808,7 @@ define <2 x double> @test_gather_2f64(<2 x ptr> %ptrs, <2 x i1> %mask, <2 x doub
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 ; AVX512-LABEL: 'test_gather_2f64'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
   %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
@@ -1817,19 +1817,19 @@ define <2 x double> @test_gather_2f64(<2 x ptr> %ptrs, <2 x i1> %mask, <2 x doub
 
 define <4 x i32> @test_gather_4i32(<4 x ptr> %ptrs, <4 x i1> %mask, <4 x i32> %src0)  {
 ; SSE2-LABEL: 'test_gather_4i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; SSE42-LABEL: 'test_gather_4i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; AVX1-LABEL: 'test_gather_4i32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; AVX2-LABEL: 'test_gather_4i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; SKL-LABEL: 'test_gather_4i32'
@@ -1837,7 +1837,7 @@ define <4 x i32> @test_gather_4i32(<4 x ptr> %ptrs, <4 x i1> %mask, <4 x i32> %s
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; KNL-LABEL: 'test_gather_4i32'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
 ; SKX-LABEL: 'test_gather_4i32'
@@ -1929,25 +1929,25 @@ define <16 x float> @test_gather_16f32_var_mask(ptr %base, <16 x i32> %ind, <16
 ; SSE2-LABEL: 'test_gather_16f32_var_mask'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; SSE42-LABEL: 'test_gather_16f32_var_mask'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; AVX1-LABEL: 'test_gather_16f32_var_mask'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; AVX2-LABEL: 'test_gather_16f32_var_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; SKL-LABEL: 'test_gather_16f32_var_mask'
@@ -1973,25 +1973,25 @@ define <16 x float> @test_gather_16f32_ra_var_mask(<16 x ptr> %ptrs, <16 x i32>
 ; SSE2-LABEL: 'test_gather_16f32_ra_var_mask'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; SSE42-LABEL: 'test_gather_16f32_ra_var_mask'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; AVX1-LABEL: 'test_gather_16f32_ra_var_mask'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; AVX2-LABEL: 'test_gather_16f32_ra_var_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
 ; SKL-LABEL: 'test_gather_16f32_ra_var_mask'
@@ -2078,7 +2078,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 77 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_scatter_16i32'
@@ -2086,7 +2086,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 65 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_scatter_16i32'
@@ -2094,7 +2094,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_scatter_16i32'
@@ -2102,7 +2102,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SKL-LABEL: 'test_scatter_16i32'
@@ -2110,7 +2110,7 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> undef, <16 x i32> zeroinitializer
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x ptr> %broadcast.splat, <16 x i32> %ind
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; SKL-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
+; SKL-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> %val, <16 x ptr> %gep.random, i32 4, <16 x i1> %imask)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_scatter_16i32'
@@ -2132,15 +2132,15 @@ define void @test_scatter_16i32(ptr %base, <16 x i32> %ind, i16 %mask, <16 x i32
 
 define void @test_scatter_8i32(<8 x i32>%a1, <8 x ptr> %ptr, <8 x i1>%mask) {
 ; SSE2-LABEL: 'test_scatter_8i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_scatter_8i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_scatter_8i32'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %a1, <8 x ptr> %ptr, i32 4, <8 x i1> %mask)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_scatter_8i32'
@@ -2153,19 +2153,19 @@ define void @test_scatter_8i32(<8 x i32>%a1, <8 x ptr> %ptr, <8 x i1>%mask) {
 
 define void @test_scatter_4i32(<4 x i32>%a1, <4 x ptr> %ptr, <4 x i1>%mask) {
 ; SSE2-LABEL: 'test_scatter_4i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_scatter_4i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_scatter_4i32'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; KNL-LABEL: 'test_scatter_4i32'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %a1, <4 x ptr> %ptr, i32 4, <4 x i1> %mask)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SKX-LABEL: 'test_scatter_4i32'
@@ -2180,25 +2180,25 @@ define <4 x float> @test_gather_4f32(ptr %ptr, <4 x i32> %ind, <4 x i1>%mask) {
 ; SSE2-LABEL: 'test_gather_4f32'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
 ;
 ; SSE42-LABEL: 'test_gather_4f32'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
 ;
 ; AVX1-LABEL: 'test_gather_4f32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
 ;
 ; AVX2-LABEL: 'test_gather_4f32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
 ;
 ; SKL-LABEL: 'test_gather_4f32'
@@ -2210,7 +2210,7 @@ define <4 x float> @test_gather_4f32(ptr %ptr, <4 x i32> %ind, <4 x i1>%mask) {
 ; KNL-LABEL: 'test_gather_4f32'
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, ptr %ptr, <4 x i64> %sext_ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
 ;
 ; SKX-LABEL: 'test_gather_4f32'
-- 
GitLab


From 3773191fc4e088fb06592e0404296f6051b1d4f7 Mon Sep 17 00:00:00 2001
From: Shan Huang <52285902006@stu.ecnu.edu.cn>
Date: Sat, 11 May 2024 23:10:00 +0800
Subject: [PATCH 0483/1206] [DebugInfo][JumpThreading] Fix missing debug
 location updates (#91581)

---
 llvm/lib/Transforms/Scalar/JumpThreading.cpp  |  6 +-
 .../JumpThreading/guard-split-debuginfo.ll    | 30 +++++++-
 .../preserving-debugloc-bitcast.ll            | 46 +++++++++++
 .../preserving-debugloc-fold-select.ll        | 76 +++++++++++++++++++
 4 files changed, 153 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/Transforms/JumpThreading/preserving-debugloc-bitcast.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/preserving-debugloc-fold-select.ll

diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 802467b5b183..74a8f1958dfe 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -1279,9 +1279,11 @@ bool JumpThreadingPass::simplifyPartiallyRedundantLoad(LoadInst *LoadI) {
     // only happen in dead loops.
     if (AvailableVal == LoadI)
       AvailableVal = PoisonValue::get(LoadI->getType());
-    if (AvailableVal->getType() != LoadI->getType())
+    if (AvailableVal->getType() != LoadI->getType()) {
       AvailableVal = CastInst::CreateBitOrPointerCast(
           AvailableVal, LoadI->getType(), "", LoadI->getIterator());
+      cast<Instruction>(AvailableVal)->setDebugLoc(LoadI->getDebugLoc());
+    }
     LoadI->replaceAllUsesWith(AvailableVal);
     LoadI->eraseFromParent();
     return true;
@@ -2987,6 +2989,7 @@ bool JumpThreadingPass::tryToUnfoldSelectInCurrBB(BasicBlock *BB) {
     PHINode *NewPN = PHINode::Create(SI->getType(), 2, "", SI->getIterator());
     NewPN->addIncoming(SI->getTrueValue(), Term->getParent());
     NewPN->addIncoming(SI->getFalseValue(), BB);
+    NewPN->setDebugLoc(SI->getDebugLoc());
     SI->replaceAllUsesWith(NewPN);
     SI->eraseFromParent();
     // NewBB and SplitBB are newly created blocks which require insertion.
@@ -3124,6 +3127,7 @@ bool JumpThreadingPass::threadGuard(BasicBlock *BB, IntrinsicInst *Guard,
       PHINode *NewPN = PHINode::Create(Inst->getType(), 2);
       NewPN->addIncoming(UnguardedMapping[Inst], UnguardedBlock);
       NewPN->addIncoming(GuardedMapping[Inst], GuardedBlock);
+      NewPN->setDebugLoc(Inst->getDebugLoc());
       NewPN->insertBefore(InsertionPoint);
       Inst->replaceAllUsesWith(NewPN);
     }
diff --git a/llvm/test/Transforms/JumpThreading/guard-split-debuginfo.ll b/llvm/test/Transforms/JumpThreading/guard-split-debuginfo.ll
index 05ff74939449..38fbe4de51ad 100644
--- a/llvm/test/Transforms/JumpThreading/guard-split-debuginfo.ll
+++ b/llvm/test/Transforms/JumpThreading/guard-split-debuginfo.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
 ; RUN: opt -S -passes=jump-threading %s -o - -S | FileCheck %s
 ; RUN: opt -S -passes=jump-threading %s -o - -S --try-experimental-debuginfo-iterators | FileCheck %s
 
@@ -7,6 +7,9 @@
 ; parent blocks. And that ino jump-threading, the old dbg.value gets
 ; deleted.
 
+; Test that JumpThreading's threadGuard() propagates the debug location
+; to the `phi` from the instruction it replaces (`%retval`)
+
 declare void @llvm.experimental.guard(i1, ...)
 
 declare i32 @f1()
@@ -20,20 +23,20 @@ define i32 @branch_implies_guard(i32 %a) !dbg !7 {
 ; CHECK-NEXT:    br i1 [[COND]], label [[T1_SPLIT:%.*]], label [[F1_SPLIT:%.*]], !dbg [[DBG12:![0-9]+]]
 ; CHECK:       T1.split:
 ; CHECK-NEXT:    [[V1:%.*]] = call i32 @f1(), !dbg [[DBG12]]
-; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 0, metadata [[META13:![0-9]+]], metadata !DIExpression()), !dbg [[DBG14:![0-9]+]]
+; CHECK-NEXT:    tail call void @llvm.dbg.value(metadata i32 0, metadata [[META13:![0-9]+]], metadata !DIExpression()), !dbg [[DBG14:![0-9]+]]
 ; CHECK-NEXT:    [[RETVAL3:%.*]] = add i32 [[V1]], 10, !dbg [[DBG12]]
 ; CHECK-NEXT:    [[CONDGUARD4:%.*]] = icmp slt i32 [[A]], 20, !dbg [[DBG12]]
 ; CHECK-NEXT:    br label [[MERGE:%.*]]
 ; CHECK:       F1.split:
 ; CHECK-NEXT:    [[V2:%.*]] = call i32 @f2(), !dbg [[DBG12]]
-; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 0, metadata [[META13]], metadata !DIExpression()), !dbg [[DBG14]]
+; CHECK-NEXT:    tail call void @llvm.dbg.value(metadata i32 0, metadata [[META13]], metadata !DIExpression()), !dbg [[DBG14]]
 ; CHECK-NEXT:    [[RETVAL1:%.*]] = add i32 [[V2]], 10, !dbg [[DBG12]]
 ; CHECK-NEXT:    [[CONDGUARD2:%.*]] = icmp slt i32 [[A]], 20, !dbg [[DBG12]]
 ; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CONDGUARD2]]) [ "deopt"() ]
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       Merge:
 ; CHECK-NEXT:    [[RETPHI:%.*]] = phi i32 [ [[V1]], [[T1_SPLIT]] ], [ [[V2]], [[F1_SPLIT]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[RETVAL3]], [[T1_SPLIT]] ], [ [[RETVAL1]], [[F1_SPLIT]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[RETVAL3]], [[T1_SPLIT]] ], [ [[RETVAL1]], [[F1_SPLIT]] ], !dbg [[DBG12]]
 ; CHECK-NEXT:    ret i32 [[TMP1]], !dbg [[DBG12]]
 ;
   %cond = icmp slt i32 %a, 10
@@ -78,3 +81,22 @@ Merge:
 !19 = distinct !DILexicalBlock(scope: !7, file: !1, line: 8, column: 7)
 !26 = !DILocation(line: 13, column: 3, scope: !7)
 
+;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+;.
+; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C99, file: [[META1:![0-9]+]], isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: [[META2:![0-9]+]], splitDebugInlining: false, nameTableKind: None)
+; CHECK: [[META1]] = !DIFile(filename: "test.c", directory: {{.*}})
+; CHECK: [[META2]] = !{}
+; CHECK: [[META3:![0-9]+]] = !{i32 7, !"Dwarf Version", i32 4}
+; CHECK: [[META4:![0-9]+]] = !{i32 2, !"Debug Info Version", i32 3}
+; CHECK: [[META5:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; CHECK: [[META6:![0-9]+]] = !{!""}
+; CHECK: [[META7:![0-9]+]] = distinct !DISubprogram(name: "foo", scope: [[META1]], file: [[META1]], line: 3, type: [[META8:![0-9]+]], scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: [[META0]], retainedNodes: [[META2]])
+; CHECK: [[META8]] = !DISubroutineType(types: [[META9:![0-9]+]])
+; CHECK: [[META9]] = !{[[META10:![0-9]+]], [[META11:![0-9]+]], [[META11]]}
+; CHECK: [[META10]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+; CHECK: [[META11]] = !DIBasicType(name: "long int", size: 64, encoding: DW_ATE_signed)
+; CHECK: [[DBG12]] = !DILocation(line: 13, column: 3, scope: [[META7]])
+; CHECK: [[META13]] = !DILocalVariable(name: "bar", arg: 1, scope: [[META7]], file: [[META1]], line: 3, type: [[META11]])
+; CHECK: [[DBG14]] = !DILocation(line: 0, scope: [[META7]])
+;.
diff --git a/llvm/test/Transforms/JumpThreading/preserving-debugloc-bitcast.ll b/llvm/test/Transforms/JumpThreading/preserving-debugloc-bitcast.ll
new file mode 100644
index 000000000000..319350e06f47
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/preserving-debugloc-bitcast.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -S -passes=jump-threading | FileCheck %s
+
+; Test that JumpThreading's `simplifyPartiallyRedundantLoad` propagates
+; the debug location to the `bitcast` from the LoadInst it replaces (`%b`).
+
+declare void @f1(...)
+
+define void @test8(ptr %0, ptr %1, ptr %2) !dbg !5 {
+; CHECK: @test8
+; CHECK:    [[TMP4:%.*]] = bitcast float [[A:%.*]] to i32, !dbg [[DBG9:![0-9]+]]
+; CHECK: [[DBG9]] = !DILocation(line: 2,
+;
+  %a = load float, ptr %0, align 4, !dbg !8
+  %b = load i32, ptr %0, align 4, !dbg !9
+  store float %a, ptr %1, align 4, !dbg !10
+  %c = icmp eq i32 %b, 8, !dbg !11
+  br i1 %c, label %ret1, label %ret2, !dbg !12
+
+ret1:                                             ; preds = %3
+  ret void, !dbg !13
+
+ret2:                                             ; preds = %3
+  %xxx = tail call i32 (...) @f1() #0, !dbg !14
+  ret void, !dbg !15
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!2, !3}
+!llvm.module.flags = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "test.ll", directory: "/")
+!2 = !{i32 8}
+!3 = !{i32 0}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "test8", linkageName: "test8", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !DILocation(line: 1, column: 1, scope: !5)
+!9 = !DILocation(line: 2, column: 1, scope: !5)
+!10 = !DILocation(line: 3, column: 1, scope: !5)
+!11 = !DILocation(line: 4, column: 1, scope: !5)
+!12 = !DILocation(line: 5, column: 1, scope: !5)
+!13 = !DILocation(line: 6, column: 1, scope: !5)
+!14 = !DILocation(line: 7, column: 1, scope: !5)
+!15 = !DILocation(line: 8, column: 1, scope: !5)
diff --git a/llvm/test/Transforms/JumpThreading/preserving-debugloc-fold-select.ll b/llvm/test/Transforms/JumpThreading/preserving-debugloc-fold-select.ll
new file mode 100644
index 000000000000..8fdec7210980
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/preserving-debugloc-fold-select.ll
@@ -0,0 +1,76 @@
+; RUN: opt < %s -S -passes=jump-threading | FileCheck %s
+
+; Test the debug location update of the newly created PHINode
+; which replaces the select instruction in .exit block.
+
+define i32 @unfold3(i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z, i32 %j) !dbg !5 {
+; CHECK:       .exit.thread4:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 {{.*}}, !dbg [[DBG29:![0-9]+]]
+; CHECK-NEXT:    ret i32 [[TMP0]], !dbg [[DBG30:![0-9]+]]
+;
+; CHECK: [[DBG29]] = !DILocation(line: 13,
+;
+entry:
+  %add3 = add nsw i32 %j, 2, !dbg !19
+  %cmp.i = icmp slt i32 %u, %v, !dbg !20
+  br i1 %cmp.i, label %.exit, label %cond.false.i, !dbg !21
+
+cond.false.i:                                     ; preds = %entry
+  %cmp4.i = icmp sgt i32 %u, %v, !dbg !22
+  br i1 %cmp4.i, label %.exit, label %cond.false.6.i, !dbg !23
+
+cond.false.6.i:                                   ; preds = %cond.false.i
+  %cmp8.i = icmp slt i32 %w, %x, !dbg !24
+  br i1 %cmp8.i, label %.exit, label %cond.false.10.i, !dbg !25
+
+cond.false.10.i:                                  ; preds = %cond.false.6.i
+  %cmp13.i = icmp sgt i32 %w, %x, !dbg !26
+  br i1 %cmp13.i, label %.exit, label %cond.false.15.i, !dbg !27
+
+cond.false.15.i:                                  ; preds = %cond.false.10.i
+  %phitmp = icmp sge i32 %y, %z, !dbg !28
+  br label %.exit, !dbg !29
+
+.exit:                                            ; preds = %cond.false.15.i, %cond.false.10.i, %cond.false.6.i, %cond.false.i, %entry
+  %cond23.i = phi i1 [ false, %entry ], [ true, %cond.false.i ], [ false, %cond.false.6.i ], [ %phitmp, %cond.false.15.i ], [ true, %cond.false.10.i ], !dbg !30
+  %j.add3 = select i1 %cond23.i, i32 %j, i32 %add3, !dbg !31
+  ret i32 %j.add3, !dbg !32
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!2, !3}
+!llvm.module.flags = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "preserving-debugloc-trytofoldselect.ll", directory: "/")
+!2 = !{i32 14}
+!3 = !{i32 8}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "unfold3", linkageName: "unfold3", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !{!9, !11, !13, !14, !15, !16, !17, !18}
+!9 = !DILocalVariable(name: "1", scope: !5, file: !1, line: 1, type: !10)
+!10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+!11 = !DILocalVariable(name: "2", scope: !5, file: !1, line: 2, type: !12)
+!12 = !DIBasicType(name: "ty8", size: 8, encoding: DW_ATE_unsigned)
+!13 = !DILocalVariable(name: "3", scope: !5, file: !1, line: 4, type: !12)
+!14 = !DILocalVariable(name: "4", scope: !5, file: !1, line: 6, type: !12)
+!15 = !DILocalVariable(name: "5", scope: !5, file: !1, line: 8, type: !12)
+!16 = !DILocalVariable(name: "6", scope: !5, file: !1, line: 10, type: !12)
+!17 = !DILocalVariable(name: "7", scope: !5, file: !1, line: 12, type: !12)
+!18 = !DILocalVariable(name: "8", scope: !5, file: !1, line: 13, type: !10)
+!19 = !DILocation(line: 1, column: 1, scope: !5)
+!20 = !DILocation(line: 2, column: 1, scope: !5)
+!21 = !DILocation(line: 3, column: 1, scope: !5)
+!22 = !DILocation(line: 4, column: 1, scope: !5)
+!23 = !DILocation(line: 5, column: 1, scope: !5)
+!24 = !DILocation(line: 6, column: 1, scope: !5)
+!25 = !DILocation(line: 7, column: 1, scope: !5)
+!26 = !DILocation(line: 8, column: 1, scope: !5)
+!27 = !DILocation(line: 9, column: 1, scope: !5)
+!28 = !DILocation(line: 10, column: 1, scope: !5)
+!29 = !DILocation(line: 11, column: 1, scope: !5)
+!30 = !DILocation(line: 12, column: 1, scope: !5)
+!31 = !DILocation(line: 13, column: 1, scope: !5)
+!32 = !DILocation(line: 14, column: 1, scope: !5)
-- 
GitLab


From cdd782183dc6a074baa1dbc92b37f1f4dbb4725f Mon Sep 17 00:00:00 2001
From: Shan Huang <52285902006@stu.ecnu.edu.cn>
Date: Sat, 11 May 2024 23:26:04 +0800
Subject: [PATCH 0484/1206] [DebugInfo][LICM] Fix missing debug location
 updates (#91729)

---
 llvm/lib/Transforms/Scalar/LICM.cpp           |  2 +
 .../LICM/debugloc-preserve-fmul-drop-fdiv.ll  | 66 +++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 llvm/test/Transforms/LICM/debugloc-preserve-fmul-drop-fdiv.ll

diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index e50413de46b1..6aa4188d1cc4 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -933,12 +933,14 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
         ReciprocalDivisor->setFastMathFlags(I.getFastMathFlags());
         SafetyInfo->insertInstructionTo(ReciprocalDivisor, I.getParent());
         ReciprocalDivisor->insertBefore(&I);
+        ReciprocalDivisor->setDebugLoc(I.getDebugLoc());
 
         auto Product =
             BinaryOperator::CreateFMul(I.getOperand(0), ReciprocalDivisor);
         Product->setFastMathFlags(I.getFastMathFlags());
         SafetyInfo->insertInstructionTo(Product, I.getParent());
         Product->insertAfter(&I);
+        Product->setDebugLoc(I.getDebugLoc());
         I.replaceAllUsesWith(Product);
         eraseInstruction(I, *SafetyInfo, MSSAU);
 
diff --git a/llvm/test/Transforms/LICM/debugloc-preserve-fmul-drop-fdiv.ll b/llvm/test/Transforms/LICM/debugloc-preserve-fmul-drop-fdiv.ll
new file mode 100644
index 000000000000..e6e6f077fd6d
--- /dev/null
+++ b/llvm/test/Transforms/LICM/debugloc-preserve-fmul-drop-fdiv.ll
@@ -0,0 +1,66 @@
+; RUN: opt -passes=licm -verify-memoryssa -S < %s | FileCheck %s
+
+; JumpThreading's hoistRegion() replaces the `fdiv` (%v6), of which the second
+; operand (%v) is a loop invariant, with a loop invariant `fdiv` and a `fmul`.
+; This test checks that the debug location propagates to the new `fmul` from
+; the old `fdiv` it replaces in block `loop` and the debug location drop of new
+; `fdiv`, which is hoisted to block `entry` after being created.
+
+define zeroext i1 @invariant_denom(double %v) !dbg !5 {
+; CHECK:       entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fdiv fast double 1.000000e+00, [[V:%.*]]{{$}}
+; CHECK:       loop:
+; CHECK:         [[TMP1:%.*]] = fmul fast double {{.*}}, !dbg [[DBG12:![0-9]+]]
+; CHECK:       [[DBG12]] = !DILocation(line: 5,
+;
+entry:
+  br label %loop, !dbg !8
+
+loop:                                             ; preds = %loop, %entry
+  %v3 = phi i32 [ 0, %entry ], [ %v11, %loop ], !dbg !9
+  %v4 = phi i32 [ 0, %entry ], [ %v12, %loop ], !dbg !10
+  %v5 = uitofp i32 %v4 to double, !dbg !11
+  %v6 = fdiv fast double %v5, %v, !dbg !12
+  %v7 = fptoui double %v6 to i64, !dbg !13
+  %v8 = and i64 %v7, 1, !dbg !14
+  %v9 = xor i64 %v8, 1, !dbg !15
+  %v10 = trunc i64 %v9 to i32, !dbg !16
+  %v11 = add i32 %v10, %v3, !dbg !17
+  %v12 = add nuw i32 %v4, 1, !dbg !18
+  %v13 = icmp eq i32 %v12, -1, !dbg !19
+  br i1 %v13, label %end, label %loop, !dbg !20
+
+end:                                              ; preds = %loop
+  %v15 = phi i32 [ %v11, %loop ], !dbg !21
+  %v16 = icmp ne i32 %v15, 0, !dbg !22
+  ret i1 %v16, !dbg !23
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!2, !3}
+!llvm.module.flags = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "test.ll", directory: "/")
+!2 = !{i32 16}
+!3 = !{i32 0}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "invariant_denom", linkageName: "invariant_denom", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !DILocation(line: 1, column: 1, scope: !5)
+!9 = !DILocation(line: 2, column: 1, scope: !5)
+!10 = !DILocation(line: 3, column: 1, scope: !5)
+!11 = !DILocation(line: 4, column: 1, scope: !5)
+!12 = !DILocation(line: 5, column: 1, scope: !5)
+!13 = !DILocation(line: 6, column: 1, scope: !5)
+!14 = !DILocation(line: 7, column: 1, scope: !5)
+!15 = !DILocation(line: 8, column: 1, scope: !5)
+!16 = !DILocation(line: 9, column: 1, scope: !5)
+!17 = !DILocation(line: 10, column: 1, scope: !5)
+!18 = !DILocation(line: 11, column: 1, scope: !5)
+!19 = !DILocation(line: 12, column: 1, scope: !5)
+!20 = !DILocation(line: 13, column: 1, scope: !5)
+!21 = !DILocation(line: 14, column: 1, scope: !5)
+!22 = !DILocation(line: 15, column: 1, scope: !5)
+!23 = !DILocation(line: 16, column: 1, scope: !5)
-- 
GitLab


From 1337622a492f4e77604b09ac8ff97042e46d8d42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9o=20Degioanni?=
 <theo.degioanni.llvm.deluge062@simplelogin.fr>
Date: Sat, 11 May 2024 19:45:34 +0200
Subject: [PATCH 0485/1206] [MLIR] Add IRDL dialect loading to C API (#91852)

Being able to add custom dialects is one of the big missing pieces of
the C API. This change should make it achievable via IRDL. Hopefully
this should open custom dialect definition to non-C++ users of MLIR.
---
 mlir/include/mlir-c/Dialect/IRDL.h   | 29 ++++++++++++++
 mlir/lib/CAPI/Dialect/CMakeLists.txt |  9 +++++
 mlir/lib/CAPI/Dialect/IRDL.cpp       | 18 +++++++++
 mlir/test/CAPI/CMakeLists.txt        |  7 ++++
 mlir/test/CAPI/irdl.c                | 58 ++++++++++++++++++++++++++++
 mlir/test/CMakeLists.txt             |  1 +
 mlir/test/lit.cfg.py                 |  1 +
 7 files changed, 123 insertions(+)
 create mode 100644 mlir/include/mlir-c/Dialect/IRDL.h
 create mode 100644 mlir/lib/CAPI/Dialect/IRDL.cpp
 create mode 100644 mlir/test/CAPI/irdl.c

diff --git a/mlir/include/mlir-c/Dialect/IRDL.h b/mlir/include/mlir-c/Dialect/IRDL.h
new file mode 100644
index 000000000000..c4d6ffd989af
--- /dev/null
+++ b/mlir/include/mlir-c/Dialect/IRDL.h
@@ -0,0 +1,29 @@
+//===-- mlir-c/Dialect/IRDL.h - C API for IRDL --------------------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM
+// Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_C_DIALECT_IRDL_H
+#define MLIR_C_DIALECT_IRDL_H
+
+#include "mlir-c/IR.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(IRDL, irdl);
+
+/// Loads all IRDL dialects in the provided module, registering the dialects in
+/// the module's associated context.
+MLIR_CAPI_EXPORTED MlirLogicalResult mlirLoadIRDLDialects(MlirModule module);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // MLIR_C_DIALECT_IRDL_H
diff --git a/mlir/lib/CAPI/Dialect/CMakeLists.txt b/mlir/lib/CAPI/Dialect/CMakeLists.txt
index 58b8739043f9..4e141b60ff8c 100644
--- a/mlir/lib/CAPI/Dialect/CMakeLists.txt
+++ b/mlir/lib/CAPI/Dialect/CMakeLists.txt
@@ -72,6 +72,15 @@ add_mlir_upstream_c_api_library(MLIRCAPIGPU
   MLIRPass
 )
 
+add_mlir_upstream_c_api_library(MLIRCAPIIRDL
+  IRDL.cpp
+
+  PARTIAL_SOURCES_INTENDED
+  LINK_LIBS PUBLIC
+  MLIRCAPIIR
+  MLIRIRDL
+)
+
 add_mlir_upstream_c_api_library(MLIRCAPILLVM
   LLVM.cpp
 
diff --git a/mlir/lib/CAPI/Dialect/IRDL.cpp b/mlir/lib/CAPI/Dialect/IRDL.cpp
new file mode 100644
index 000000000000..cb9dc8ceb679
--- /dev/null
+++ b/mlir/lib/CAPI/Dialect/IRDL.cpp
@@ -0,0 +1,18 @@
+//===- IRDL.cpp - C Interface for IRDL dialect ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir-c/Dialect/IRDL.h"
+#include "mlir/CAPI/Registration.h"
+#include "mlir/Dialect/IRDL/IR/IRDL.h"
+#include "mlir/Dialect/IRDL/IRDLLoading.h"
+
+MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(IRDL, irdl, mlir::irdl::IRDLDialect)
+
+MlirLogicalResult mlirLoadIRDLDialects(MlirModule module) {
+  return wrap(mlir::irdl::loadDialects(unwrap(module)));
+}
diff --git a/mlir/test/CAPI/CMakeLists.txt b/mlir/test/CAPI/CMakeLists.txt
index b9cd63ef7c67..57b342a5e26b 100644
--- a/mlir/test/CAPI/CMakeLists.txt
+++ b/mlir/test/CAPI/CMakeLists.txt
@@ -38,6 +38,13 @@ _add_capi_test_executable(mlir-capi-ir-test
     MLIRCAPIRegisterEverything
 )
 
+_add_capi_test_executable(mlir-capi-irdl-test
+  irdl.c
+  LINK_LIBS PRIVATE
+    MLIRCAPIIR
+    MLIRCAPIIRDL
+)
+
 _add_capi_test_executable(mlir-capi-llvm-test
   llvm.c
   LINK_LIBS PRIVATE
diff --git a/mlir/test/CAPI/irdl.c b/mlir/test/CAPI/irdl.c
new file mode 100644
index 000000000000..b35345b664b1
--- /dev/null
+++ b/mlir/test/CAPI/irdl.c
@@ -0,0 +1,58 @@
+//===- irdl.c - Test for the C bindings for IRDL registration -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM
+// Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+/* RUN: mlir-capi-irdl-test 2>&1 | FileCheck %s
+ */
+
+#include "mlir-c/Dialect/IRDL.h"
+#include "mlir-c/IR.h"
+
+const char irdlDialect[] = "\
+  irdl.dialect @foo {\
+    irdl.operation @op {\
+      %i32 = irdl.is i32\
+      irdl.results(%i32)\
+    }\
+  }\
+  irdl.dialect @bar {\
+    irdl.operation @op {\
+      %i32 = irdl.is i32\
+      irdl.operands(%i32)\
+    }\
+  }";
+
+// CHECK:      module {
+// CHECK-NEXT:   %[[RES:.*]] = "foo.op"() : () -> i32
+// CHECK-NEXT:   "bar.op"(%[[RES]]) :  (i32) -> ()
+// CHECK-NEXT: }
+const char newDialectUsage[] = "\
+  module {\
+    %res = \"foo.op\"() : () -> i32\
+    \"bar.op\"(%res) : (i32) -> ()\
+  }";
+
+int main(void) {
+  MlirContext ctx = mlirContextCreate();
+  mlirDialectHandleLoadDialect(mlirGetDialectHandle__irdl__(), ctx);
+
+  MlirModule dialectDecl =
+      mlirModuleCreateParse(ctx, mlirStringRefCreateFromCString(irdlDialect));
+
+  mlirLoadIRDLDialects(dialectDecl);
+  mlirModuleDestroy(dialectDecl);
+
+  MlirModule usingModule = mlirModuleCreateParse(
+      ctx, mlirStringRefCreateFromCString(newDialectUsage));
+
+  mlirOperationDump(mlirModuleGetOperation(usingModule));
+
+  mlirModuleDestroy(usingModule);
+  mlirContextDestroy(ctx);
+  return 0;
+}
diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt
index 5319a9cb33e0..8806a1dd9223 100644
--- a/mlir/test/CMakeLists.txt
+++ b/mlir/test/CMakeLists.txt
@@ -101,6 +101,7 @@ configure_lit_site_cfg(
 set(MLIR_TEST_DEPENDS
   FileCheck count not split-file
   mlir-capi-ir-test
+  mlir-capi-irdl-test
   mlir-capi-llvm-test
   mlir-capi-pass-test
   mlir-capi-quant-test
diff --git a/mlir/test/lit.cfg.py b/mlir/test/lit.cfg.py
index 4740e7d137e8..ea6d9ae71b77 100644
--- a/mlir/test/lit.cfg.py
+++ b/mlir/test/lit.cfg.py
@@ -100,6 +100,7 @@ tools = [
     "mlir-lsp-server",
     "mlir-capi-execution-engine-test",
     "mlir-capi-ir-test",
+    "mlir-capi-irdl-test",
     "mlir-capi-llvm-test",
     "mlir-capi-pass-test",
     "mlir-capi-pdl-test",
-- 
GitLab


From 379b77773cf653352f30f8c7cca393f4df9389be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Fri, 10 May 2024 15:41:21 +0200
Subject: [PATCH 0486/1206] [clang][Interp] Use pointee metadata size in
 isRoot()

Previously, isRoot() would return true for pointers with a base
of sizeof(InlineDescriptor), even if the actual metadata size of the
pointee was 0.
---
 clang/lib/AST/Interp/Pointer.h            | 32 ++++++++++-------------
 clang/unittests/AST/Interp/Descriptor.cpp |  2 +-
 2 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/clang/lib/AST/Interp/Pointer.h b/clang/lib/AST/Interp/Pointer.h
index c4d701bc71b7..79fab05670e9 100644
--- a/clang/lib/AST/Interp/Pointer.h
+++ b/clang/lib/AST/Interp/Pointer.h
@@ -226,8 +226,7 @@ public:
       return *this;
 
     // If at base, point to an array of base types.
-    if (asBlockPointer().Base == 0 ||
-        asBlockPointer().Base == sizeof(InlineDescriptor))
+    if (isRoot())
       return Pointer(asBlockPointer().Pointee, RootPtrMark, 0);
 
     // Step into the containing array, if inside one.
@@ -306,10 +305,8 @@ public:
   const Descriptor *getFieldDesc() const {
     if (isIntegralPointer())
       return asIntPointer().Desc;
-    if (isBlockPointer() &&
-        (asBlockPointer().Base == 0 ||
-         asBlockPointer().Base == sizeof(InlineDescriptor) ||
-         asBlockPointer().Base == RootPtrMark))
+
+    if (isRoot())
       return getDeclDesc();
     return getInlineDesc()->Desc;
   }
@@ -390,8 +387,7 @@ public:
     // If this points inside a dummy block, return true.
     // FIXME: This might change in the future. If it does, we need
     // to set the proper Ctor/Dtor functions for dummy Descriptors.
-    if (asBlockPointer().Base != 0 &&
-        asBlockPointer().Base != sizeof(InlineDescriptor) && isDummy())
+    if (!isRoot() && isDummy())
       return true;
     return getFieldDesc()->isUnknownSizeArray();
   }
@@ -403,9 +399,11 @@ public:
   }
   /// Pointer points directly to a block.
   bool isRoot() const {
-    return (asBlockPointer().Base == 0 ||
-            asBlockPointer().Base == RootPtrMark) &&
-           Offset == 0;
+    if (isZero() || isIntegralPointer())
+      return true;
+    return (asBlockPointer().Base ==
+                asBlockPointer().Pointee->getDescriptor()->getMetadataSize() ||
+            asBlockPointer().Base == 0);
   }
   /// If this pointer has an InlineDescriptor we can use to initialize.
   bool canBeInitialized() const {
@@ -487,9 +485,7 @@ public:
   bool isActive() const {
     if (!isBlockPointer())
       return true;
-    return asBlockPointer().Base == 0 ||
-           asBlockPointer().Base == sizeof(InlineDescriptor) ||
-           getInlineDesc()->IsActive;
+    return isRoot() || getInlineDesc()->IsActive;
   }
   /// Checks if a structure is a base class.
   bool isBaseClass() const { return isField() && getInlineDesc()->IsBase; }
@@ -508,10 +504,7 @@ public:
   bool isConst() const {
     if (isIntegralPointer())
       return true;
-    return (asBlockPointer().Base == 0 ||
-            asBlockPointer().Base == sizeof(InlineDescriptor))
-               ? getDeclDesc()->IsConst
-               : getInlineDesc()->IsConst;
+    return isRoot() ? getDeclDesc()->IsConst : getInlineDesc()->IsConst;
   }
 
   /// Returns the declaration ID.
@@ -567,6 +560,9 @@ public:
 
     if (!asBlockPointer().Pointee)
       return false;
+    if (isDummy())
+      return false;
+
     return isElementPastEnd() || getSize() == getOffset();
   }
 
diff --git a/clang/unittests/AST/Interp/Descriptor.cpp b/clang/unittests/AST/Interp/Descriptor.cpp
index 4ea0fbc285a9..053d579ea391 100644
--- a/clang/unittests/AST/Interp/Descriptor.cpp
+++ b/clang/unittests/AST/Interp/Descriptor.cpp
@@ -115,7 +115,7 @@ TEST(Descriptor, Primitives) {
 
   // Check pointer stuff.
   // Global variables have an inline descriptor.
-  ASSERT_FALSE(GlobalPtr.isRoot());
+  ASSERT_TRUE(GlobalPtr.isRoot());
   ASSERT_TRUE(GlobalPtr.isLive());
   ASSERT_FALSE(GlobalPtr.isZero());
   ASSERT_FALSE(GlobalPtr.isField());
-- 
GitLab


From deffae5da123b32098914d8346bf4358a6792bdc Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 11 May 2024 11:38:52 -0700
Subject: [PATCH 0487/1206] [clang] Use StringRef::operator== instead of
 StringRef::equals (NFC) (#91844)

I'm planning to remove StringRef::equals in favor of
StringRef::operator==.

- StringRef::operator==/!= outnumber StringRef::equals by a factor of
  24 under clang/ in terms of their usage.

- The elimination of StringRef::equals brings StringRef closer to
  std::string_view, which has operator== but not equals.

- S == "foo" is more readable than S.equals("foo"), especially for
  !Long.Expression.equals("str") vs Long.Expression != "str".
---
 clang/include/clang/Basic/IdentifierTable.h              | 2 +-
 clang/include/clang/Basic/SourceManager.h                | 6 +++---
 clang/lib/ARCMigrate/ObjCMT.cpp                          | 2 +-
 clang/lib/AST/PrintfFormatString.cpp                     | 8 ++++----
 clang/lib/ASTMatchers/Dynamic/Marshallers.cpp            | 4 ++--
 clang/lib/Basic/Builtins.cpp                             | 2 +-
 clang/lib/Basic/Diagnostic.cpp                           | 3 +--
 clang/lib/Basic/LangOptions.cpp                          | 2 +-
 clang/lib/Basic/Targets/ARM.cpp                          | 5 ++---
 clang/lib/Basic/Targets/PPC.h                            | 2 +-
 clang/lib/Basic/Targets/SystemZ.h                        | 2 +-
 clang/lib/Basic/Targets/X86.h                            | 6 +++---
 clang/lib/CodeGen/CodeGenFunction.cpp                    | 7 +++----
 clang/lib/Format/ContinuationIndenter.cpp                | 2 +-
 clang/lib/Format/Format.cpp                              | 2 +-
 clang/lib/Frontend/CompilerInvocation.cpp                | 2 +-
 clang/lib/Frontend/ModuleDependencyCollector.cpp         | 2 +-
 clang/lib/Lex/PPDirectives.cpp                           | 4 ++--
 clang/lib/Parse/ParseDecl.cpp                            | 2 +-
 clang/lib/Parse/ParseDeclCXX.cpp                         | 4 ++--
 clang/lib/Parse/ParseHLSL.cpp                            | 2 +-
 clang/lib/Parse/ParseOpenMP.cpp                          | 4 ++--
 clang/lib/Sema/SemaCodeComplete.cpp                      | 9 ++++-----
 clang/lib/Sema/SemaDecl.cpp                              | 8 ++++----
 clang/lib/Sema/SemaDeclCXX.cpp                           | 4 ++--
 clang/lib/Sema/SemaStmtAsm.cpp                           | 2 +-
 clang/lib/Sema/SemaTemplate.cpp                          | 4 ++--
 .../lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp  | 7 +++----
 .../StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp   | 2 +-
 .../lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp  | 2 +-
 clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp      | 4 ++--
 .../StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp | 4 ++--
 .../StaticAnalyzer/Checkers/ObjCContainersChecker.cpp    | 6 +++---
 clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp      | 2 +-
 clang/lib/StaticAnalyzer/Core/CheckerContext.cpp         | 2 +-
 clang/lib/Tooling/Tooling.cpp                            | 2 +-
 clang/tools/diagtool/ShowEnabledWarnings.cpp             | 4 ++--
 clang/tools/diagtool/TreeView.cpp                        | 2 +-
 clang/unittests/CodeGen/IRMatchers.h                     | 2 +-
 39 files changed, 68 insertions(+), 73 deletions(-)

diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index a893e6f4d3d3..ae9ebd9f5915 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -738,7 +738,7 @@ public:
     II->Entry = &Entry;
 
     // If this is the 'import' contextual keyword, mark it as such.
-    if (Name.equals("import"))
+    if (Name == "import")
       II->setModulesImport(true);
 
     return *II;
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index d2ece14da0b1..5258bab584f4 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -1504,7 +1504,7 @@ public:
     if (Presumed.isInvalid())
       return false;
     StringRef Filename(Presumed.getFilename());
-    return Filename.equals("<built-in>");
+    return Filename == "<built-in>";
   }
 
   /// Returns whether \p Loc is located in a <command line> file.
@@ -1513,7 +1513,7 @@ public:
     if (Presumed.isInvalid())
       return false;
     StringRef Filename(Presumed.getFilename());
-    return Filename.equals("<command line>");
+    return Filename == "<command line>";
   }
 
   /// Returns whether \p Loc is located in a <scratch space> file.
@@ -1522,7 +1522,7 @@ public:
     if (Presumed.isInvalid())
       return false;
     StringRef Filename(Presumed.getFilename());
-    return Filename.equals("<scratch space>");
+    return Filename == "<scratch space>";
   }
 
   /// Returns if a SourceLocation is in a system header.
diff --git a/clang/lib/ARCMigrate/ObjCMT.cpp b/clang/lib/ARCMigrate/ObjCMT.cpp
index b9dcfb8951b3..aaf41dc4039c 100644
--- a/clang/lib/ARCMigrate/ObjCMT.cpp
+++ b/clang/lib/ARCMigrate/ObjCMT.cpp
@@ -484,7 +484,7 @@ static void rewriteToObjCProperty(const ObjCMethodDecl *Getter,
 
   // Short circuit 'delegate' properties that contain the name "delegate" or
   // "dataSource", or have exact name "target" to have 'assign' attribute.
-  if (PropertyName.equals("target") || PropertyName.contains("delegate") ||
+  if (PropertyName == "target" || PropertyName.contains("delegate") ||
       PropertyName.contains("dataSource")) {
     QualType QT = Getter->getReturnType();
     if (!QT->isRealType())
diff --git a/clang/lib/AST/PrintfFormatString.cpp b/clang/lib/AST/PrintfFormatString.cpp
index fec8ce13e8c4..dd3b38fabb55 100644
--- a/clang/lib/AST/PrintfFormatString.cpp
+++ b/clang/lib/AST/PrintfFormatString.cpp
@@ -146,13 +146,13 @@ static PrintfSpecifierResult ParsePrintfSpecifier(FormatStringHandler &H,
           if (Warn && (Size == 0 || Size > 8))
             H.handleInvalidMaskType(MaskType);
           FS.setMaskType(MaskType);
-        } else if (MatchedStr.equals("sensitive"))
+        } else if (MatchedStr == "sensitive")
           PrivacyFlags = clang::analyze_os_log::OSLogBufferItem::IsSensitive;
         else if (PrivacyFlags !=
-                 clang::analyze_os_log::OSLogBufferItem::IsSensitive &&
-                 MatchedStr.equals("private"))
+                     clang::analyze_os_log::OSLogBufferItem::IsSensitive &&
+                 MatchedStr == "private")
           PrivacyFlags = clang::analyze_os_log::OSLogBufferItem::IsPrivate;
-        else if (PrivacyFlags == 0 && MatchedStr.equals("public"))
+        else if (PrivacyFlags == 0 && MatchedStr == "public")
           PrivacyFlags = clang::analyze_os_log::OSLogBufferItem::IsPublic;
       } else {
         size_t CommaOrBracePos =
diff --git a/clang/lib/ASTMatchers/Dynamic/Marshallers.cpp b/clang/lib/ASTMatchers/Dynamic/Marshallers.cpp
index cf9ae7c974a6..37c91abb5c83 100644
--- a/clang/lib/ASTMatchers/Dynamic/Marshallers.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Marshallers.cpp
@@ -21,7 +21,7 @@ getBestGuess(llvm::StringRef Search, llvm::ArrayRef<llvm::StringRef> Allowed,
   llvm::StringRef Res;
   for (const llvm::StringRef &Item : Allowed) {
     if (Item.equals_insensitive(Search)) {
-      assert(!Item.equals(Search) && "This should be handled earlier on.");
+      assert(Item != Search && "This should be handled earlier on.");
       MaxEditDistance = 1;
       Res = Item;
       continue;
@@ -41,7 +41,7 @@ getBestGuess(llvm::StringRef Search, llvm::ArrayRef<llvm::StringRef> Allowed,
       if (!NoPrefix.consume_front(DropPrefix))
         continue;
       if (NoPrefix.equals_insensitive(Search)) {
-        if (NoPrefix.equals(Search))
+        if (NoPrefix == Search)
           return Item.str();
         MaxEditDistance = 1;
         Res = Item;
diff --git a/clang/lib/Basic/Builtins.cpp b/clang/lib/Basic/Builtins.cpp
index 3467847ac167..b116abbe034f 100644
--- a/clang/lib/Basic/Builtins.cpp
+++ b/clang/lib/Basic/Builtins.cpp
@@ -64,7 +64,7 @@ bool Builtin::Context::isBuiltinFunc(llvm::StringRef FuncName) {
   bool InStdNamespace = FuncName.consume_front("std-");
   for (unsigned i = Builtin::NotBuiltin + 1; i != Builtin::FirstTSBuiltin;
        ++i) {
-    if (FuncName.equals(BuiltinInfo[i].Name) &&
+    if (FuncName == BuiltinInfo[i].Name &&
         (bool)strchr(BuiltinInfo[i].Attributes, 'z') == InStdNamespace)
       return strchr(BuiltinInfo[i].Attributes, 'f') != nullptr;
   }
diff --git a/clang/lib/Basic/Diagnostic.cpp b/clang/lib/Basic/Diagnostic.cpp
index 0208ccc31bd7..10136b4cd943 100644
--- a/clang/lib/Basic/Diagnostic.cpp
+++ b/clang/lib/Basic/Diagnostic.cpp
@@ -851,8 +851,7 @@ FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
   // When the diagnostic string is only "%0", the entire string is being given
   // by an outside source.  Remove unprintable characters from this string
   // and skip all the other string processing.
-  if (DiagEnd - DiagStr == 2 &&
-      StringRef(DiagStr, DiagEnd - DiagStr).equals("%0") &&
+  if (DiagEnd - DiagStr == 2 && StringRef(DiagStr, DiagEnd - DiagStr) == "%0" &&
       getArgKind(0) == DiagnosticsEngine::ak_std_string) {
     const std::string &S = getArgStdStr(0);
     EscapeStringForDiagnostic(S, OutStr);
diff --git a/clang/lib/Basic/LangOptions.cpp b/clang/lib/Basic/LangOptions.cpp
index a0adfbf61840..2b906463931d 100644
--- a/clang/lib/Basic/LangOptions.cpp
+++ b/clang/lib/Basic/LangOptions.cpp
@@ -48,7 +48,7 @@ void LangOptions::resetNonModularOptions() {
 
 bool LangOptions::isNoBuiltinFunc(StringRef FuncName) const {
   for (unsigned i = 0, e = NoBuiltinFuncs.size(); i != e; ++i)
-    if (FuncName.equals(NoBuiltinFuncs[i]))
+    if (FuncName == NoBuiltinFuncs[i])
       return true;
   return false;
 }
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index 877799c66ec4..7423626d7c3c 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -173,8 +173,7 @@ bool ARMTargetInfo::supportsThumb() const {
 }
 
 bool ARMTargetInfo::supportsThumb2() const {
-  return CPUAttr.equals("6T2") ||
-         (ArchVersion >= 7 && !CPUAttr.equals("8M_BASE"));
+  return CPUAttr == "6T2" || (ArchVersion >= 7 && CPUAttr != "8M_BASE");
 }
 
 StringRef ARMTargetInfo::getCPUAttr() const {
@@ -1162,7 +1161,7 @@ bool ARMTargetInfo::validateAsmConstraint(
     return true;
   case 'j': // An immediate integer between 0 and 65535 (valid for MOVW)
     // only available in ARMv6T2 and above
-    if (CPUAttr.equals("6T2") || ArchVersion >= 7) {
+    if (CPUAttr == "6T2" || ArchVersion >= 7) {
       Info.setRequiresImmediate(0, 65535);
       return true;
     }
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index 496b6131d09b..fc23c30c6852 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -360,7 +360,7 @@ public:
   bool hasBitIntType() const override { return true; }
 
   bool isSPRegName(StringRef RegName) const override {
-    return RegName.equals("r1") || RegName.equals("x1");
+    return RegName == "r1" || RegName == "x1";
   }
 
   // We support __builtin_cpu_supports/__builtin_cpu_is on targets that
diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h
index 73d3aa01a043..3bc6f2c1d308 100644
--- a/clang/lib/Basic/Targets/SystemZ.h
+++ b/clang/lib/Basic/Targets/SystemZ.h
@@ -84,7 +84,7 @@ public:
   ArrayRef<TargetInfo::AddlRegName> getGCCAddlRegNames() const override;
 
   bool isSPRegName(StringRef RegName) const override {
-    return RegName.equals("r15");
+    return RegName == "r15";
   }
 
   bool validateAsmConstraint(const char *&Name,
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index fc00407d461e..6a0a6cb84203 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -219,7 +219,7 @@ public:
   ArrayRef<TargetInfo::AddlRegName> getGCCAddlRegNames() const override;
 
   bool isSPRegName(StringRef RegName) const override {
-    return RegName.equals("esp") || RegName.equals("rsp");
+    return RegName == "esp" || RegName == "rsp";
   }
 
   bool supportsCpuSupports() const override { return true; }
@@ -247,7 +247,7 @@ public:
                                       bool &HasSizeMismatch) const override {
     // esp and ebp are the only 32-bit registers the x86 backend can currently
     // handle.
-    if (RegName.equals("esp") || RegName.equals("ebp")) {
+    if (RegName == "esp" || RegName == "ebp") {
       // Check that the register size is 32-bit.
       HasSizeMismatch = RegSize != 32;
       return true;
@@ -803,7 +803,7 @@ public:
                                       bool &HasSizeMismatch) const override {
     // rsp and rbp are the only 64-bit registers the x86 backend can currently
     // handle.
-    if (RegName.equals("rsp") || RegName.equals("rbp")) {
+    if (RegName == "rsp" || RegName == "rbp") {
       // Check that the register size is 64-bit.
       HasSizeMismatch = RegSize != 64;
       return true;
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 477814140a9e..9f16fcb43855 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -2765,10 +2765,9 @@ llvm::Value *CodeGenFunction::FormAArch64ResolverCondition(
     // only for features that are not enabled in the target. The exception is
     // for features whose extension instructions are executed as NOP on targets
     // without extension support.
-    if (!getContext().getTargetInfo().hasFeature(Feature) ||
-        Feature.equals("bti") || Feature.equals("memtag") ||
-        Feature.equals("memtag2") || Feature.equals("memtag3") ||
-        Feature.equals("dgh"))
+    if (!getContext().getTargetInfo().hasFeature(Feature) || Feature == "bti" ||
+        Feature == "memtag" || Feature == "memtag2" || Feature == "memtag3" ||
+        Feature == "dgh")
       CondFeatures.push_back(Feature);
   }
   if (!CondFeatures.empty()) {
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index ad0e2c3c620c..6b9fbfe0ebf5 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -1422,7 +1422,7 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) {
   // the next line.
   if (State.Line->InPragmaDirective) {
     FormatToken *PragmaType = State.Line->First->Next->Next;
-    if (PragmaType && PragmaType->TokenText.equals("omp"))
+    if (PragmaType && PragmaType->TokenText == "omp")
       return CurrentState.Indent + Style.ContinuationIndentWidth;
   }
 
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index c5c79dd0f883..52005a6c881f 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -3770,7 +3770,7 @@ reformat(const FormatStyle &Style, StringRef Code,
     tooling::Replacements NonNoOpFixes;
     for (const tooling::Replacement &Fix : Fixes) {
       StringRef OriginalCode = Code.substr(Fix.getOffset(), Fix.getLength());
-      if (!OriginalCode.equals(Fix.getReplacementText())) {
+      if (OriginalCode != Fix.getReplacementText()) {
         auto Err = NonNoOpFixes.add(Fix);
         if (Err) {
           llvm::errs() << "Error adding replacements : "
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index dbb5f5662ebf..14ee02c4cd58 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1969,7 +1969,7 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
       Diags.Report(diag::err_drv_invalid_value)
           << A->getAsString(Args) << A->getValue();
     else if (Val == llvm::FunctionReturnThunksKind::Extern &&
-             Args.getLastArgValue(OPT_mcmodel_EQ).equals("large"))
+             Args.getLastArgValue(OPT_mcmodel_EQ) == "large")
       Diags.Report(diag::err_drv_argument_not_allowed_with)
           << A->getAsString(Args)
           << Args.getLastArg(OPT_mcmodel_EQ)->getAsString(Args);
diff --git a/clang/lib/Frontend/ModuleDependencyCollector.cpp b/clang/lib/Frontend/ModuleDependencyCollector.cpp
index b88cb60ebdd2..e2883f1e027e 100644
--- a/clang/lib/Frontend/ModuleDependencyCollector.cpp
+++ b/clang/lib/Frontend/ModuleDependencyCollector.cpp
@@ -105,7 +105,7 @@ static bool isCaseSensitivePath(StringRef Path) {
   // already expects when sensitivity isn't setup.
   for (auto &C : Path)
     UpperDest.push_back(toUppercase(C));
-  if (!llvm::sys::fs::real_path(UpperDest, RealDest) && Path.equals(RealDest))
+  if (!llvm::sys::fs::real_path(UpperDest, RealDest) && Path == RealDest)
     return false;
   return true;
 }
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 82eb47bcd5bf..8e7386449dce 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -183,7 +183,7 @@ static MacroDiag shouldWarnOnMacroDef(Preprocessor &PP, IdentifierInfo *II) {
     return isFeatureTestMacro(Text) ? MD_NoWarn : MD_ReservedMacro;
   if (II->isKeyword(Lang))
     return MD_KeywordDef;
-  if (Lang.CPlusPlus11 && (Text.equals("override") || Text.equals("final")))
+  if (Lang.CPlusPlus11 && (Text == "override" || Text == "final"))
     return MD_KeywordDef;
   return MD_NoWarn;
 }
@@ -2807,7 +2807,7 @@ static bool isConfigurationPattern(Token &MacroName, MacroInfo *MI,
         if (TrimmedValue.ends_with("__"))
           TrimmedValue = TrimmedValue.drop_back(2);
       }
-      return TrimmedValue.equals(MacroText);
+      return TrimmedValue == MacroText;
     } else {
       return false;
     }
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 5b5fc02ad402..7fbaee5690bd 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -3889,7 +3889,7 @@ void Parser::ParseDeclarationSpecifiers(
       // parse errors if this really is a __declspec attribute. Attempt to
       // recognize that scenario and recover gracefully.
       if (!getLangOpts().DeclSpecKeyword && Tok.is(tok::identifier) &&
-          Tok.getIdentifierInfo()->getName().equals("__declspec")) {
+          Tok.getIdentifierInfo()->getName() == "__declspec") {
         Diag(Loc, diag::err_ms_attributes_not_enabled);
 
         // The next token should be an open paren. If it is, eat the entire
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 96c9708c3711..65ddebca49bc 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -4548,9 +4548,9 @@ static bool IsBuiltInOrStandardCXX11Attribute(IdentifierInfo *AttrName,
   case ParsedAttr::AT_Unlikely:
     return true;
   case ParsedAttr::AT_WarnUnusedResult:
-    return !ScopeName && AttrName->getName().equals("nodiscard");
+    return !ScopeName && AttrName->getName() == "nodiscard";
   case ParsedAttr::AT_Unused:
-    return !ScopeName && AttrName->getName().equals("maybe_unused");
+    return !ScopeName && AttrName->getName() == "maybe_unused";
   default:
     return false;
   }
diff --git a/clang/lib/Parse/ParseHLSL.cpp b/clang/lib/Parse/ParseHLSL.cpp
index e9c8d6dca7bf..4b72afe9986e 100644
--- a/clang/lib/Parse/ParseHLSL.cpp
+++ b/clang/lib/Parse/ParseHLSL.cpp
@@ -174,7 +174,7 @@ void Parser::ParseHLSLAnnotations(ParsedAttributes &Attrs,
       ArgExprs.push_back(ParseIdentifierLoc());
 
       // Add numeric_constant for fix-it.
-      if (SpaceStr.equals("space") && Tok.is(tok::numeric_constant))
+      if (SpaceStr == "space" && Tok.is(tok::numeric_constant))
         fixSeparateAttrArgAndNumber(SpaceStr, SpaceLoc, Tok, ArgExprs, *this,
                                     Actions.Context, PP);
     }
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 18ba1185ee8d..53eabe0c662e 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -740,7 +740,7 @@ static bool parseDeclareSimdClauses(
       BS = Out;
       BSRange = SourceRange(Tok.getLocation(), Tok.getEndLoc());
       P.ConsumeToken();
-    } else if (ClauseName.equals("simdlen")) {
+    } else if (ClauseName == "simdlen") {
       if (SimdLen.isUsable()) {
         P.Diag(Tok, diag::err_omp_more_one_clause)
             << getOpenMPDirectiveName(OMPD_declare_simd) << ClauseName << 0;
@@ -1106,7 +1106,7 @@ static ExprResult parseContextScore(Parser &P) {
   llvm::SmallString<16> Buffer;
   StringRef SelectorName =
       P.getPreprocessor().getSpelling(P.getCurToken(), Buffer);
-  if (!SelectorName.equals("score"))
+  if (SelectorName != "score")
     return ScoreExpr;
   (void)P.ConsumeToken();
   SourceLocation RLoc;
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 3f0ab10646fe..87aa0cacc249 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -4049,18 +4049,17 @@ unsigned clang::getMacroUsagePriority(StringRef MacroName,
   unsigned Priority = CCP_Macro;
 
   // Treat the "nil", "Nil" and "NULL" macros as null pointer constants.
-  if (MacroName.equals("nil") || MacroName.equals("NULL") ||
-      MacroName.equals("Nil")) {
+  if (MacroName == "nil" || MacroName == "NULL" || MacroName == "Nil") {
     Priority = CCP_Constant;
     if (PreferredTypeIsPointer)
       Priority = Priority / CCF_SimilarTypeMatch;
   }
   // Treat "YES", "NO", "true", and "false" as constants.
-  else if (MacroName.equals("YES") || MacroName.equals("NO") ||
-           MacroName.equals("true") || MacroName.equals("false"))
+  else if (MacroName == "YES" || MacroName == "NO" || MacroName == "true" ||
+           MacroName == "false")
     Priority = CCP_Constant;
   // Treat "bool" as a type.
-  else if (MacroName.equals("bool"))
+  else if (MacroName == "bool")
     Priority = CCP_Type + (LangOpts.ObjC ? CCD_bool_in_ObjC : 0);
 
   return Priority;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 590f37837eb2..fb913034bd83 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -735,8 +735,8 @@ void Sema::DiagnoseUnknownTypeName(IdentifierInfo *&II,
                          << II, CanRecover);
       } else if (DeclContext *DC = computeDeclContext(*SS, false)) {
         std::string CorrectedStr(Corrected.getAsString(getLangOpts()));
-        bool DroppedSpecifier = Corrected.WillReplaceSpecifier() &&
-                                II->getName().equals(CorrectedStr);
+        bool DroppedSpecifier =
+            Corrected.WillReplaceSpecifier() && II->getName() == CorrectedStr;
         diagnoseTypo(Corrected,
                      PDiag(IsTemplateName
                                ? diag::err_no_member_template_suggest
@@ -1007,7 +1007,7 @@ Corrected:
         } else {// FIXME: is this even reachable? Test it.
           std::string CorrectedStr(Corrected.getAsString(getLangOpts()));
           bool DroppedSpecifier = Corrected.WillReplaceSpecifier() &&
-                                  Name->getName().equals(CorrectedStr);
+                                  Name->getName() == CorrectedStr;
           diagnoseTypo(Corrected, PDiag(QualifiedDiag)
                                     << Name << computeDeclContext(SS, false)
                                     << DroppedSpecifier << SS.getRange());
@@ -16076,7 +16076,7 @@ static void diagnoseImplicitlyRetainedSelf(Sema &S) {
 
 static bool methodHasName(const FunctionDecl *FD, StringRef Name) {
   return isa<CXXMethodDecl>(FD) && FD->param_empty() &&
-         FD->getDeclName().isIdentifier() && FD->getName().equals(Name);
+         FD->getDeclName().isIdentifier() && FD->getName() == Name;
 }
 
 bool Sema::CanBeGetReturnObject(const FunctionDecl *FD) {
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index d77b9507066b..53238d355ea0 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -12234,8 +12234,8 @@ static bool TryNamespaceTypoCorrection(Sema &S, LookupResult &R, Scope *Sc,
       DiagnoseInvisibleNamespace(Corrected, S);
     } else if (DeclContext *DC = S.computeDeclContext(SS, false)) {
       std::string CorrectedStr(Corrected.getAsString(S.getLangOpts()));
-      bool DroppedSpecifier = Corrected.WillReplaceSpecifier() &&
-                              Ident->getName().equals(CorrectedStr);
+      bool DroppedSpecifier =
+          Corrected.WillReplaceSpecifier() && Ident->getName() == CorrectedStr;
       S.diagnoseTypo(Corrected,
                      S.PDiag(diag::err_using_directive_member_suggest)
                        << Ident << DC << DroppedSpecifier << SS.getRange(),
diff --git a/clang/lib/Sema/SemaStmtAsm.cpp b/clang/lib/Sema/SemaStmtAsm.cpp
index 83351b703c15..32d42f3c3f3b 100644
--- a/clang/lib/Sema/SemaStmtAsm.cpp
+++ b/clang/lib/Sema/SemaStmtAsm.cpp
@@ -829,7 +829,7 @@ bool Sema::LookupInlineAsmField(StringRef Base, StringRef Member,
   NamedDecl *FoundDecl = nullptr;
 
   // MS InlineAsm uses 'this' as a base
-  if (getLangOpts().CPlusPlus && Base.equals("this")) {
+  if (getLangOpts().CPlusPlus && Base == "this") {
     if (const Type *PT = getCurrentThisType().getTypePtrOrNull())
       FoundDecl = PT->getPointeeType()->getAsTagDecl();
   } else {
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 480c0103ae33..82badb2efb5c 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -4268,8 +4268,8 @@ checkBuiltinTemplateIdType(Sema &SemaRef, BuiltinTemplateDecl *BTD,
 /// Determine whether this alias template is "enable_if_t".
 /// libc++ >=14 uses "__enable_if_t" in C++11 mode.
 static bool isEnableIfAliasTemplate(TypeAliasTemplateDecl *AliasTemplate) {
-  return AliasTemplate->getName().equals("enable_if_t") ||
-         AliasTemplate->getName().equals("__enable_if_t");
+  return AliasTemplate->getName() == "enable_if_t" ||
+         AliasTemplate->getName() == "__enable_if_t";
 }
 
 /// Collect all of the separable terms in the given condition, which
diff --git a/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
index d17f5ddf0705..a0190c30bfd2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
@@ -1065,15 +1065,14 @@ void GenericTaintChecker::taintUnsafeSocketProtocol(const CallEvent &Call,
   const IdentifierInfo *ID = Call.getCalleeIdentifier();
   if (!ID)
     return;
-  if (!ID->getName().equals("socket"))
+  if (ID->getName() != "socket")
     return;
 
   SourceLocation DomLoc = Call.getArgExpr(0)->getExprLoc();
   StringRef DomName = C.getMacroNameOrSpelling(DomLoc);
   // Allow internal communication protocols.
-  bool SafeProtocol = DomName.equals("AF_SYSTEM") ||
-                      DomName.equals("AF_LOCAL") || DomName.equals("AF_UNIX") ||
-                      DomName.equals("AF_RESERVED_36");
+  bool SafeProtocol = DomName == "AF_SYSTEM" || DomName == "AF_LOCAL" ||
+                      DomName == "AF_UNIX" || DomName == "AF_RESERVED_36";
   if (SafeProtocol)
     return;
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
index fa51aa80216b..1cb3848cfed2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
@@ -41,7 +41,7 @@ static bool InNamespace(const Decl *D, StringRef NS) {
   if (!ND)
     return false;
   const IdentifierInfo *II = ND->getIdentifier();
-  if (!II || !II->getName().equals(NS))
+  if (!II || II->getName() != NS)
     return false;
   return isa<TranslationUnitDecl>(ND->getDeclContext());
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
index 882eb0236a18..f524c4c067c8 100644
--- a/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
@@ -1159,7 +1159,7 @@ void EmptyLocalizationContextChecker::MethodCrawler::VisitObjCMessageExpr(
   }
 
   if (isAnyIdentifier(Result.getKind())) {
-    if (Result.getRawIdentifier().equals("nil")) {
+    if (Result.getRawIdentifier() == "nil") {
       reportEmptyContextError(ME);
       return;
     }
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index ab89fb14046b..34af7fb131f5 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -1727,7 +1727,7 @@ static std::optional<bool> getFreeWhenDoneArg(const ObjCMethodCall &Call) {
 
   // FIXME: We should not rely on fully-constrained symbols being folded.
   for (unsigned i = 1; i < S.getNumArgs(); ++i)
-    if (S.getNameForSlot(i).equals("freeWhenDone"))
+    if (S.getNameForSlot(i) == "freeWhenDone")
       return !Call.getArgSVal(i).isZeroConstant();
 
   return std::nullopt;
@@ -3255,7 +3255,7 @@ bool MallocChecker::mayFreeAnyEscapedMemoryOrIsModeledExplicitly(
     if (FirstSlot.starts_with("addPointer") ||
         FirstSlot.starts_with("insertPointer") ||
         FirstSlot.starts_with("replacePointer") ||
-        FirstSlot.equals("valueWithPointer")) {
+        FirstSlot == "valueWithPointer") {
       return true;
     }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp
index 2b008d1c775a..6978d81faf1c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp
@@ -101,14 +101,14 @@ void WalkAST::VisitCallExpr(CallExpr *CE) {
   const Expr *Arg = nullptr;
   unsigned ArgNum;
 
-  if (Name.equals("CFArrayCreate") || Name.equals("CFSetCreate")) {
+  if (Name == "CFArrayCreate" || Name == "CFSetCreate") {
     if (CE->getNumArgs() != 4)
       return;
     ArgNum = 1;
     Arg = CE->getArg(ArgNum)->IgnoreParenCasts();
     if (hasPointerToPointerSizedType(Arg))
         return;
-  } else if (Name.equals("CFDictionaryCreate")) {
+  } else if (Name == "CFDictionaryCreate") {
     if (CE->getNumArgs() != 6)
       return;
     // Check first argument.
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp
index 28e88245ca95..4937af3b91c2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp
@@ -82,7 +82,7 @@ void ObjCContainersChecker::checkPostStmt(const CallExpr *CE,
     return;
 
   // Add array size information to the state.
-  if (Name.equals("CFArrayCreate")) {
+  if (Name == "CFArrayCreate") {
     if (CE->getNumArgs() < 3)
       return;
     // Note, we can visit the Create method in the post-visit because
@@ -92,7 +92,7 @@ void ObjCContainersChecker::checkPostStmt(const CallExpr *CE,
     return;
   }
 
-  if (Name.equals("CFArrayGetCount")) {
+  if (Name == "CFArrayGetCount") {
     addSizeInfo(CE->getArg(0), CE, C);
     return;
   }
@@ -105,7 +105,7 @@ void ObjCContainersChecker::checkPreStmt(const CallExpr *CE,
     return;
 
   // Check the array access.
-  if (Name.equals("CFArrayGetValueAtIndex")) {
+  if (Name == "CFArrayGetValueAtIndex") {
     ProgramStateRef State = C.getState();
     // Retrieve the size.
     // Find out if we saw this array symbol before and have information about
diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
index a7b6f6c1fb55..d4e020f7a72a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
@@ -1143,7 +1143,7 @@ void StreamChecker::evalFscanf(const FnDescription *Desc, const CallEvent &Call,
       return;
 
     if (auto const *Callee = Call.getCalleeIdentifier();
-        !Callee || !Callee->getName().equals("vfscanf")) {
+        !Callee || Callee->getName() != "vfscanf") {
       SmallVector<unsigned int> EscArgs;
       for (auto EscArg : llvm::seq(2u, Call.getNumArgs()))
         EscArgs.push_back(EscArg);
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
index 113abcd4c2ab..96464b30c078 100644
--- a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
@@ -104,7 +104,7 @@ bool CheckerContext::isCLibraryFunction(const FunctionDecl *FD,
     return true;
 
   StringRef FName = II->getName();
-  if (FName.equals(Name))
+  if (FName == Name)
     return true;
 
   if (FName.starts_with("__inline") && FName.contains(Name))
diff --git a/clang/lib/Tooling/Tooling.cpp b/clang/lib/Tooling/Tooling.cpp
index c5c3cdb47e92..ffacf9cf1f78 100644
--- a/clang/lib/Tooling/Tooling.cpp
+++ b/clang/lib/Tooling/Tooling.cpp
@@ -293,7 +293,7 @@ void addTargetAndModeForProgramName(std::vector<std::string> &CommandLine,
        ++Token) {
     StringRef TokenRef(*Token);
     ShouldAddTarget = ShouldAddTarget && !TokenRef.starts_with(TargetOPT) &&
-                      !TokenRef.equals(TargetOPTLegacy);
+                      TokenRef != TargetOPTLegacy;
     ShouldAddMode = ShouldAddMode && !TokenRef.starts_with(DriverModeOPT);
   }
   if (ShouldAddMode) {
diff --git a/clang/tools/diagtool/ShowEnabledWarnings.cpp b/clang/tools/diagtool/ShowEnabledWarnings.cpp
index 285efe6ae05b..66a295db054c 100644
--- a/clang/tools/diagtool/ShowEnabledWarnings.cpp
+++ b/clang/tools/diagtool/ShowEnabledWarnings.cpp
@@ -90,11 +90,11 @@ int ShowEnabledWarnings::run(unsigned int argc, char **argv, raw_ostream &Out) {
   bool ShouldShowLevels = true;
   if (argc > 0) {
     StringRef FirstArg(*argv);
-    if (FirstArg.equals("--no-levels")) {
+    if (FirstArg == "--no-levels") {
       ShouldShowLevels = false;
       --argc;
       ++argv;
-    } else if (FirstArg.equals("--levels")) {
+    } else if (FirstArg == "--levels") {
       ShouldShowLevels = true;
       --argc;
       ++argv;
diff --git a/clang/tools/diagtool/TreeView.cpp b/clang/tools/diagtool/TreeView.cpp
index 00d1097b5fbf..8d1ce14b0f52 100644
--- a/clang/tools/diagtool/TreeView.cpp
+++ b/clang/tools/diagtool/TreeView.cpp
@@ -144,7 +144,7 @@ int TreeView::run(unsigned int argc, char **argv, llvm::raw_ostream &out) {
   bool Internal = false;
   if (argc > 0) {
     StringRef FirstArg(*argv);
-    if (FirstArg.equals("--internal")) {
+    if (FirstArg == "--internal") {
       Internal = true;
       --argc;
       ++argv;
diff --git a/clang/unittests/CodeGen/IRMatchers.h b/clang/unittests/CodeGen/IRMatchers.h
index 47e420498036..3572a317f07a 100644
--- a/clang/unittests/CodeGen/IRMatchers.h
+++ b/clang/unittests/CodeGen/IRMatchers.h
@@ -317,7 +317,7 @@ public:
   NameMetaMatcher(StringRef N) : Name(N) {}
   bool matchEntity(const Metadata &M, MatcherContext &C) override {
     if (auto *MDS = dyn_cast<MDString>(&M))
-      return MDS->getString().equals(Name);
+      return MDS->getString() == Name;
     return false;
   }
 };
-- 
GitLab


From baffaf000fd4667f33b3756d0d3b645b1d926b44 Mon Sep 17 00:00:00 2001
From: Dave Lee <davelee.com@gmail.com>
Date: Sat, 11 May 2024 11:47:40 -0700
Subject: [PATCH 0488/1206] [lldb] Add required skipIfLLVMTargetMissing for X86
 (NFC)

---
 lldb/test/API/macosx/rosetta/TestRosetta.py         | 1 +
 lldb/test/API/macosx/universal64/TestUniversal64.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/lldb/test/API/macosx/rosetta/TestRosetta.py b/lldb/test/API/macosx/rosetta/TestRosetta.py
index ce40de475ef1..a812f558a8fc 100644
--- a/lldb/test/API/macosx/rosetta/TestRosetta.py
+++ b/lldb/test/API/macosx/rosetta/TestRosetta.py
@@ -40,6 +40,7 @@ class TestRosetta(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
     @skipUnlessAppleSilicon
+    @skipIfLLVMTargetMissing("X86")
     @skipIfDarwinEmbedded
     def test_rosetta(self):
         """There can be many tests in a test case - describe this test here."""
diff --git a/lldb/test/API/macosx/universal64/TestUniversal64.py b/lldb/test/API/macosx/universal64/TestUniversal64.py
index 98661443086e..893ff14d8113 100644
--- a/lldb/test/API/macosx/universal64/TestUniversal64.py
+++ b/lldb/test/API/macosx/universal64/TestUniversal64.py
@@ -17,6 +17,7 @@ class Universal64TestCase(TestBase):
         # actually launch them here.
 
     # The Makefile manually invokes clang.
+    @skipIfLLVMTargetMissing("X86")
     @skipIfAsan
     @skipUnlessDarwin
     @skipIfDarwinEmbedded
@@ -26,6 +27,7 @@ class Universal64TestCase(TestBase):
         self.do_test()
 
     # The Makefile manually invokes clang.
+    @skipIfLLVMTargetMissing("X86")
     @skipIfAsan
     @skipUnlessDarwin
     @skipIfDarwinEmbedded
-- 
GitLab


From 2b386888b78327db4906b12401e1a89438471548 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Sat, 11 May 2024 21:56:31 +0200
Subject: [PATCH 0489/1206] Reland: [clang] Use getDefaultArgRange instead of
 getDefaultArg to retrieve source location in
 `ConvertConstructorToDeductionGuideTransform`.

The commit fec471649fffaa3ec44e17801e5c9605825e58bb was reverted by accident in 7415524b45392651969374c067041daa82dc89e7.

Reland it with a testcase.
---
 clang/lib/Sema/SemaTemplate.cpp  |  2 +-
 clang/test/SemaTemplate/ctad.cpp | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 82badb2efb5c..0e7bd8dd8957 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -2679,7 +2679,7 @@ private:
       // placeholder to indicate there is a default argument.
       QualType ParamTy = NewDI->getType();
       NewDefArg = new (SemaRef.Context)
-          OpaqueValueExpr(OldParam->getDefaultArg()->getBeginLoc(),
+          OpaqueValueExpr(OldParam->getDefaultArgRange().getBegin(),
                           ParamTy.getNonLValueExprType(SemaRef.Context),
                           ParamTy->isLValueReferenceType()   ? VK_LValue
                           : ParamTy->isRValueReferenceType() ? VK_XValue
diff --git a/clang/test/SemaTemplate/ctad.cpp b/clang/test/SemaTemplate/ctad.cpp
index ec144d4f44ba..e981ea8d5ecf 100644
--- a/clang/test/SemaTemplate/ctad.cpp
+++ b/clang/test/SemaTemplate/ctad.cpp
@@ -54,3 +54,18 @@ template<class T, class B> struct Y { Y(T); };
 template<class T, class B=void> struct Y ;
 Y y(1);
 }
+
+namespace NoCrashOnGettingDefaultArgLoc {
+template <typename>
+class A {
+  A(int = 1); // expected-note {{candidate template ignored: couldn't infer template argumen}}
+};
+class C : A<int> {
+  using A::A;
+};
+template <typename>
+class D : C { // expected-note {{candidate function template not viable: requires 1 argument}}
+  using C::C;
+};
+D abc; // expected-error {{no viable constructor or deduction guide}}
+}
-- 
GitLab


From 05cc2d5fe10ca3e7786863d9c600e1a3f8db93eb Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Sat, 11 May 2024 23:32:48 +0200
Subject: [PATCH 0490/1206] [libc++] Vectorize std::mismatch with trivially
 equality comparable types (#87716)

---
 libcxx/include/CMakeLists.txt                 |   1 +
 libcxx/include/__algorithm/mismatch.h         |  52 +++++--
 libcxx/include/__algorithm/simd_utils.h       |  34 ++++-
 libcxx/include/__iterator/aliasing_iterator.h | 127 ++++++++++++++++++
 .../__type_traits/is_equality_comparable.h    |   2 +
 libcxx/include/module.modulemap               |   2 +
 .../iterators/aliasing_iterator.pass.cpp      |  45 +++++++
 .../mismatch/mismatch.pass.cpp                |  47 +++++--
 8 files changed, 285 insertions(+), 25 deletions(-)
 create mode 100644 libcxx/include/__iterator/aliasing_iterator.h
 create mode 100644 libcxx/test/libcxx/iterators/aliasing_iterator.pass.cpp

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 96d7b4037e10..01e9c247560c 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -446,6 +446,7 @@ set(files
   __ios/fpos.h
   __iterator/access.h
   __iterator/advance.h
+  __iterator/aliasing_iterator.h
   __iterator/back_insert_iterator.h
   __iterator/bounded_iter.h
   __iterator/common_iterator.h
diff --git a/libcxx/include/__algorithm/mismatch.h b/libcxx/include/__algorithm/mismatch.h
index c2b3f8938f71..632bec02406a 100644
--- a/libcxx/include/__algorithm/mismatch.h
+++ b/libcxx/include/__algorithm/mismatch.h
@@ -16,6 +16,7 @@
 #include <__algorithm/unwrap_iter.h>
 #include <__config>
 #include <__functional/identity.h>
+#include <__iterator/aliasing_iterator.h>
 #include <__type_traits/desugars_to.h>
 #include <__type_traits/invoke.h>
 #include <__type_traits/is_constant_evaluated.h>
@@ -55,18 +56,13 @@ __mismatch(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Pro
 
 #if _LIBCPP_VECTORIZE_ALGORITHMS
 
-template <class _Tp,
-          class _Pred,
-          class _Proj1,
-          class _Proj2,
-          __enable_if_t<is_integral<_Tp>::value && __desugars_to_v<__equal_tag, _Pred, _Tp, _Tp> &&
-                            __is_identity<_Proj1>::value && __is_identity<_Proj2>::value,
-                        int> = 0>
-_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*>
-__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+template <class _Iter>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter, _Iter>
+__mismatch_vectorized(_Iter __first1, _Iter __last1, _Iter __first2) {
+  using __value_type              = __iter_value_type<_Iter>;
   constexpr size_t __unroll_count = 4;
-  constexpr size_t __vec_size     = __native_vector_size<_Tp>;
-  using __vec                     = __simd_vector<_Tp, __vec_size>;
+  constexpr size_t __vec_size     = __native_vector_size<__value_type>;
+  using __vec                     = __simd_vector<__value_type, __vec_size>;
 
   if (!__libcpp_is_constant_evaluated()) {
     auto __orig_first1 = __first1;
@@ -116,9 +112,41 @@ __mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __
     } // else loop over the elements individually
   }
 
-  return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2);
+  __equal_to __pred;
+  __identity __proj;
+  return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj, __proj);
+}
+
+template <class _Tp,
+          class _Pred,
+          class _Proj1,
+          class _Proj2,
+          __enable_if_t<is_integral<_Tp>::value && __desugars_to_v<__equal_tag, _Pred, _Tp, _Tp> &&
+                            __is_identity<_Proj1>::value && __is_identity<_Proj2>::value,
+                        int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*>
+__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred&, _Proj1&, _Proj2&) {
+  return std::__mismatch_vectorized(__first1, __last1, __first2);
 }
 
+template <class _Tp,
+          class _Pred,
+          class _Proj1,
+          class _Proj2,
+          __enable_if_t<!is_integral<_Tp>::value && __desugars_to_v<__equal_tag, _Pred, _Tp, _Tp> &&
+                            __is_identity<_Proj1>::value && __is_identity<_Proj2>::value &&
+                            __can_map_to_integer_v<_Tp> && __libcpp_is_trivially_equality_comparable<_Tp, _Tp>::value,
+                        int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*>
+__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  if (__libcpp_is_constant_evaluated()) {
+    return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2);
+  } else {
+    using _Iter = __aliasing_iterator<_Tp*, __get_as_integer_type_t<_Tp>>;
+    auto __ret  = std::__mismatch_vectorized(_Iter(__first1), _Iter(__last1), _Iter(__first2));
+    return {__ret.first.__base(), __ret.second.__base()};
+  }
+}
 #endif // _LIBCPP_VECTORIZE_ALGORITHMS
 
 template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
diff --git a/libcxx/include/__algorithm/simd_utils.h b/libcxx/include/__algorithm/simd_utils.h
index 8d540ae2cce8..71d65e8f4afb 100644
--- a/libcxx/include/__algorithm/simd_utils.h
+++ b/libcxx/include/__algorithm/simd_utils.h
@@ -43,6 +43,34 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+template <class _Tp>
+inline constexpr bool __can_map_to_integer_v =
+    sizeof(_Tp) == alignof(_Tp) && (sizeof(_Tp) == 1 || sizeof(_Tp) == 2 || sizeof(_Tp) == 4 || sizeof(_Tp) == 8);
+
+template <size_t _TypeSize>
+struct __get_as_integer_type_impl;
+
+template <>
+struct __get_as_integer_type_impl<1> {
+  using type = uint8_t;
+};
+
+template <>
+struct __get_as_integer_type_impl<2> {
+  using type = uint16_t;
+};
+template <>
+struct __get_as_integer_type_impl<4> {
+  using type = uint32_t;
+};
+template <>
+struct __get_as_integer_type_impl<8> {
+  using type = uint64_t;
+};
+
+template <class _Tp>
+using __get_as_integer_type_t = typename __get_as_integer_type_impl<sizeof(_Tp)>::type;
+
 // This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance
 // in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms
 // as far as benchmarks are concerned.
@@ -80,10 +108,10 @@ template <class _VecT>
 using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{}));
 
 // This isn't inlined without always_inline when loading chars.
-template <class _VecT, class _Tp>
-_LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(const _Tp* __ptr) noexcept {
+template <class _VecT, class _Iter>
+_LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(_Iter __iter) noexcept {
   return [=]<size_t... _Indices>(index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept {
-    return _VecT{__ptr[_Indices]...};
+    return _VecT{__iter[_Indices]...};
   }(make_index_sequence<__simd_vector_size_v<_VecT>>{});
 }
 
diff --git a/libcxx/include/__iterator/aliasing_iterator.h b/libcxx/include/__iterator/aliasing_iterator.h
new file mode 100644
index 000000000000..94ba577078b5
--- /dev/null
+++ b/libcxx/include/__iterator/aliasing_iterator.h
@@ -0,0 +1,127 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_ALIASING_ITERATOR_H
+#define _LIBCPP___ITERATOR_ALIASING_ITERATOR_H
+
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/is_trivial.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// This iterator wrapper is used to type-pun an iterator to return a different type. This is done without UB by not
+// actually punning the type, but instead inspecting the object representation of the base type and copying that into
+// an instance of the alias type. For that reason the alias type has to be trivial. The alias is returned as a prvalue
+// when derferencing the iterator, since it is temporary storage. This wrapper is used to vectorize some algorithms.
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _BaseIter, class _Alias>
+struct __aliasing_iterator_wrapper {
+  class __iterator {
+    _BaseIter __base_ = nullptr;
+
+    using __iter_traits     = iterator_traits<_BaseIter>;
+    using __base_value_type = typename __iter_traits::value_type;
+
+    static_assert(__has_random_access_iterator_category<_BaseIter>::value,
+                  "The base iterator has to be a random access iterator!");
+
+  public:
+    using iterator_category = random_access_iterator_tag;
+    using value_type        = _Alias;
+    using difference_type   = ptrdiff_t;
+    using reference         = value_type&;
+    using pointer           = value_type*;
+
+    static_assert(is_trivial<value_type>::value);
+    static_assert(sizeof(__base_value_type) == sizeof(value_type));
+
+    _LIBCPP_HIDE_FROM_ABI __iterator() = default;
+    _LIBCPP_HIDE_FROM_ABI __iterator(_BaseIter __base) _NOEXCEPT : __base_(__base) {}
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator++() _NOEXCEPT {
+      ++__base_;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator operator++(int) _NOEXCEPT {
+      __iterator __tmp(*this);
+      ++__base_;
+      return __tmp;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator--() _NOEXCEPT {
+      --__base_;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator operator--(int) _NOEXCEPT {
+      __iterator __tmp(*this);
+      --__base_;
+      return __tmp;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend __iterator operator+(__iterator __iter, difference_type __n) _NOEXCEPT {
+      return __iterator(__iter.__base_ + __n);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend __iterator operator+(difference_type __n, __iterator __iter) _NOEXCEPT {
+      return __iterator(__n + __iter.__base_);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator+=(difference_type __n) _NOEXCEPT {
+      __base_ += __n;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend __iterator operator-(__iterator __iter, difference_type __n) _NOEXCEPT {
+      return __iterator(__iter.__base_ - __n);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend difference_type operator-(__iterator __lhs, __iterator __rhs) _NOEXCEPT {
+      return __lhs.__base_ - __rhs.__base_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI __iterator& operator-=(difference_type __n) _NOEXCEPT {
+      __base_ -= __n;
+      return *this;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _BaseIter __base() const _NOEXCEPT { return __base_; }
+
+    _LIBCPP_HIDE_FROM_ABI _Alias operator*() const _NOEXCEPT {
+      _Alias __val;
+      __builtin_memcpy(&__val, std::__to_address(__base_), sizeof(value_type));
+      return __val;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI value_type operator[](difference_type __n) const _NOEXCEPT { return *(*this + __n); }
+
+    _LIBCPP_HIDE_FROM_ABI friend bool operator==(const __iterator& __lhs, const __iterator& __rhs) _NOEXCEPT {
+      return __lhs.__base_ == __rhs.__base_;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI friend bool operator!=(const __iterator& __lhs, const __iterator& __rhs) _NOEXCEPT {
+      return __lhs.__base_ != __rhs.__base_;
+    }
+  };
+};
+
+// This is required to avoid ADL instantiations on _BaseT
+template <class _BaseT, class _Alias>
+using __aliasing_iterator = typename __aliasing_iterator_wrapper<_BaseT, _Alias>::__iterator;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_ALIASING_ITERATOR_H
diff --git a/libcxx/include/__type_traits/is_equality_comparable.h b/libcxx/include/__type_traits/is_equality_comparable.h
index d4142218b641..4397f743e5ee 100644
--- a/libcxx/include/__type_traits/is_equality_comparable.h
+++ b/libcxx/include/__type_traits/is_equality_comparable.h
@@ -44,6 +44,8 @@ struct __is_equality_comparable<_Tp, _Up, __void_t<decltype(std::declval<_Tp>()
 // pointers that don't have the same type (ignoring cv-qualifiers): pointers to virtual bases are equality comparable,
 //   but don't have the same bit-pattern. An exception to this is comparing to a void-pointer. There the bit-pattern is
 //   always compared.
+// objects with padding bytes: since objects with padding bytes may compare equal, even though their object
+//   representation may not be equivalent.
 
 template <class _Tp, class _Up, class = void>
 struct __libcpp_is_trivially_equality_comparable_impl : false_type {};
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 0ee0cdce61bc..70dac2f19846 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -700,6 +700,7 @@ module std_private_algorithm_minmax_element                              [system
 module std_private_algorithm_mismatch                                    [system] {
   header "__algorithm/mismatch.h"
   export std_private_algorithm_simd_utils
+  export std_private_iterator_aliasing_iterator
 }
 module std_private_algorithm_move                                        [system] { header "__algorithm/move.h" }
 module std_private_algorithm_move_backward                               [system] { header "__algorithm/move_backward.h" }
@@ -1390,6 +1391,7 @@ module std_private_iosfwd_streambuf_fwd [system] { header "__fwd/streambuf.h" }
 
 module std_private_iterator_access                  [system] { header "__iterator/access.h" }
 module std_private_iterator_advance                 [system] { header "__iterator/advance.h" }
+module std_private_iterator_aliasing_iterator       [system] { header "__iterator/aliasing_iterator.h" }
 module std_private_iterator_back_insert_iterator    [system] { header "__iterator/back_insert_iterator.h" }
 module std_private_iterator_bounded_iter            [system] { header "__iterator/bounded_iter.h" }
 module std_private_iterator_common_iterator         [system] { header "__iterator/common_iterator.h" }
diff --git a/libcxx/test/libcxx/iterators/aliasing_iterator.pass.cpp b/libcxx/test/libcxx/iterators/aliasing_iterator.pass.cpp
new file mode 100644
index 000000000000..60587d5bfe5d
--- /dev/null
+++ b/libcxx/test/libcxx/iterators/aliasing_iterator.pass.cpp
@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS(clang): -Wprivate-header
+
+#include <__iterator/aliasing_iterator.h>
+#include <cassert>
+
+struct NonTrivial {
+  int i_;
+
+  NonTrivial(int i) : i_(i) {}
+  NonTrivial(const NonTrivial& other) : i_(other.i_) {}
+
+  NonTrivial& operator=(const NonTrivial& other) {
+    i_ = other.i_;
+    return *this;
+  }
+
+  ~NonTrivial() {}
+};
+
+int main(int, char**) {
+  {
+    NonTrivial arr[] = {1, 2, 3, 4};
+    std::__aliasing_iterator<NonTrivial*, int> iter(arr);
+
+    assert(*iter == 1);
+    assert(iter[0] == 1);
+    assert(iter[1] == 2);
+    ++iter;
+    assert(*iter == 2);
+    assert(iter[-1] == 1);
+    assert(iter.__base() == arr + 1);
+    assert(iter == iter);
+    assert(iter != (iter + 1));
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp
index eb5f7cacdde3..dd37555ffcce 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp
@@ -66,14 +66,27 @@ TEST_CONSTEXPR_CXX20 void check(Container1 lhs, Container2 rhs, size_t offset) {
 #endif
 }
 
-struct NonTrivial {
+// Compares modulo 4 to make sure we only forward to the vectorized version if we are trivially equality comparable
+struct NonTrivialMod4Comp {
   int i_;
 
-  TEST_CONSTEXPR_CXX20 NonTrivial(int i) : i_(i) {}
-  TEST_CONSTEXPR_CXX20 NonTrivial(NonTrivial&& other) : i_(other.i_) { other.i_ = 0; }
+  TEST_CONSTEXPR_CXX20 NonTrivialMod4Comp(int i) : i_(i) {}
+  TEST_CONSTEXPR_CXX20 NonTrivialMod4Comp(NonTrivialMod4Comp&& other) : i_(other.i_) { other.i_ = 0; }
 
-  TEST_CONSTEXPR_CXX20 friend bool operator==(const NonTrivial& lhs, const NonTrivial& rhs) { return lhs.i_ == rhs.i_; }
+  TEST_CONSTEXPR_CXX20 friend bool operator==(const NonTrivialMod4Comp& lhs, const NonTrivialMod4Comp& rhs) {
+    return lhs.i_ % 4 == rhs.i_ % 4;
+  }
+};
+
+#if TEST_STD_VER >= 20
+struct TriviallyEqualityComparable {
+  int i_;
+
+  TEST_CONSTEXPR_CXX20 TriviallyEqualityComparable(int i) : i_(i) {}
+
+  TEST_CONSTEXPR_CXX20 friend bool operator==(TriviallyEqualityComparable, TriviallyEqualityComparable) = default;
 };
+#endif // TEST_STD_VER >= 20
 
 struct ModTwoComp {
   TEST_CONSTEXPR_CXX20 bool operator()(int lhs, int rhs) { return lhs % 2 == rhs % 2; }
@@ -136,16 +149,30 @@ TEST_CONSTEXPR_CXX20 bool test() {
   types::for_each(types::cpp17_input_iterator_list<int*>(), Test());
 
   { // use a non-integer type to also test the general case - all elements match
-    std::array<NonTrivial, 8> lhs = {1, 2, 3, 4, 5, 6, 7, 8};
-    std::array<NonTrivial, 8> rhs = {1, 2, 3, 4, 5, 6, 7, 8};
-    check<NonTrivial*>(std::move(lhs), std::move(rhs), 8);
+    std::array<NonTrivialMod4Comp, 8> lhs = {1, 2, 3, 4, 5, 6, 7, 8};
+    std::array<NonTrivialMod4Comp, 8> rhs = {1, 2, 3, 4, 1, 6, 7, 8};
+    check<NonTrivialMod4Comp*>(std::move(lhs), std::move(rhs), 8);
   }
 
   { // use a non-integer type to also test the general case - not all elements match
-    std::array<NonTrivial, 8> lhs = {1, 2, 3, 4, 7, 6, 7, 8};
-    std::array<NonTrivial, 8> rhs = {1, 2, 3, 4, 5, 6, 7, 8};
-    check<NonTrivial*>(std::move(lhs), std::move(rhs), 4);
+    std::array<NonTrivialMod4Comp, 8> lhs = {1, 2, 3, 4, 7, 6, 7, 8};
+    std::array<NonTrivialMod4Comp, 8> rhs = {1, 2, 3, 4, 5, 6, 7, 8};
+    check<NonTrivialMod4Comp*>(std::move(lhs), std::move(rhs), 4);
+  }
+
+#if TEST_STD_VER >= 20
+  { // trivially equality comparable class type to test forwarding to the vectorized version - all elements match
+    std::array<TriviallyEqualityComparable, 8> lhs = {1, 2, 3, 4, 5, 6, 7, 8};
+    std::array<TriviallyEqualityComparable, 8> rhs = {1, 2, 3, 4, 5, 6, 7, 8};
+    check<TriviallyEqualityComparable*>(std::move(lhs), std::move(rhs), 8);
+  }
+
+  { // trivially equality comparable class type to test forwarding to the vectorized version - not all elements match
+    std::array<TriviallyEqualityComparable, 8> lhs = {1, 2, 3, 4, 7, 6, 7, 8};
+    std::array<TriviallyEqualityComparable, 8> rhs = {1, 2, 3, 4, 5, 6, 7, 8};
+    check<TriviallyEqualityComparable*>(std::move(lhs), std::move(rhs), 4);
   }
+#endif // TEST_STD_VER >= 20
 
   return true;
 }
-- 
GitLab


From 91c917b19f076b510635a6168305f6f0117e7fad Mon Sep 17 00:00:00 2001
From: Ivan Murashko <ivan.murashko@gmail.com>
Date: Sat, 11 May 2024 23:14:56 +0100
Subject: [PATCH 0491/1206] [Modules] Enable MS Windows test for
 implicit-module-no-timestamp.cpp (#91738)

There is a follow-up commit for #90319. The Windows test was disabled in
that commit, but it should pass on this operating system. Therefore, it
would be beneficial to have it enabled for MS Windows.
---
 clang/test/Modules/implicit-module-no-timestamp.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/clang/test/Modules/implicit-module-no-timestamp.cpp b/clang/test/Modules/implicit-module-no-timestamp.cpp
index 1b681a610bab..50767b4a1146 100644
--- a/clang/test/Modules/implicit-module-no-timestamp.cpp
+++ b/clang/test/Modules/implicit-module-no-timestamp.cpp
@@ -1,4 +1,3 @@
-// UNSUPPORTED: system-windows
 // RUN: rm -rf %t
 // RUN: split-file %s %t
 // RUN: cd %t
-- 
GitLab


From 0869204cff22831d0bb19a82c99bf85e4deb4ae3 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Sat, 11 May 2024 15:01:14 -0700
Subject: [PATCH 0492/1206] [clang-format] Fix buildbot failures

This effectively reverts 5cd280433e8e and changes to QualifierFixerTest.cpp
from e62ce1f8842c.

Failed buidbots:
https://lab.llvm.org/buildbot/#/builders/236/builds/11223
https://lab.llvm.org/buildbot/#/builders/239/builds/6968
---
 clang/lib/Format/QualifierAlignmentFixer.cpp  | 18 ++--
 clang/lib/Format/QualifierAlignmentFixer.h    |  4 +-
 clang/unittests/Format/QualifierFixerTest.cpp | 92 ++++++++-----------
 3 files changed, 52 insertions(+), 62 deletions(-)

diff --git a/clang/lib/Format/QualifierAlignmentFixer.cpp b/clang/lib/Format/QualifierAlignmentFixer.cpp
index a904f0b773c6..077ce5e597a2 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.cpp
+++ b/clang/lib/Format/QualifierAlignmentFixer.cpp
@@ -281,7 +281,7 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeRight(
 
     const FormatToken *LastSimpleTypeSpecifier = TypeToken;
     while (isQualifierOrType(LastSimpleTypeSpecifier->getNextNonComment(),
-                             LangOpts)) {
+                             &LangOpts)) {
       LastSimpleTypeSpecifier = LastSimpleTypeSpecifier->getNextNonComment();
     }
 
@@ -414,7 +414,7 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeLeft(
     const FormatToken *LastSimpleTypeSpecifier = TypeToken;
     while (isConfiguredQualifierOrType(
         LastSimpleTypeSpecifier->getPreviousNonComment(),
-        ConfiguredQualifierTokens, LangOpts)) {
+        ConfiguredQualifierTokens, &LangOpts)) {
       LastSimpleTypeSpecifier =
           LastSimpleTypeSpecifier->getPreviousNonComment();
     }
@@ -613,16 +613,18 @@ void prepareLeftRightOrderingForQualifierAlignmentFixer(
 }
 
 bool LeftRightQualifierAlignmentFixer::isQualifierOrType(
-    const FormatToken *Tok, const LangOptions &LangOpts) {
-  return Tok && (Tok->isTypeName(LangOpts) || Tok->is(tok::kw_auto) ||
-                 isQualifier(Tok));
+    const FormatToken *Tok, const LangOptions *LangOpts) {
+  return Tok &&
+         (Tok->isTypeName(LangOpts ? *LangOpts : getFormattingLangOpts()) ||
+          Tok->is(tok::kw_auto) || isQualifier(Tok));
 }
 
 bool LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
     const FormatToken *Tok, const std::vector<tok::TokenKind> &Qualifiers,
-    const LangOptions &LangOpts) {
-  return Tok && (Tok->isTypeName(LangOpts) || Tok->is(tok::kw_auto) ||
-                 isConfiguredQualifier(Tok, Qualifiers));
+    const LangOptions *LangOpts) {
+  return Tok &&
+         (Tok->isTypeName(LangOpts ? *LangOpts : getFormattingLangOpts()) ||
+          Tok->is(tok::kw_auto) || isConfiguredQualifier(Tok, Qualifiers));
 }
 
 // If a token is an identifier and it's upper case, it could
diff --git a/clang/lib/Format/QualifierAlignmentFixer.h b/clang/lib/Format/QualifierAlignmentFixer.h
index 710fa2dc0030..97fcb42e4b2e 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.h
+++ b/clang/lib/Format/QualifierAlignmentFixer.h
@@ -72,11 +72,11 @@ public:
 
   // Is the Token a simple or qualifier type
   static bool isQualifierOrType(const FormatToken *Tok,
-                                const LangOptions &LangOpts);
+                                const LangOptions *LangOpts = nullptr);
   static bool
   isConfiguredQualifierOrType(const FormatToken *Tok,
                               const std::vector<tok::TokenKind> &Qualifiers,
-                              const LangOptions &LangOpts);
+                              const LangOptions *LangOpts = nullptr);
 
   // Is the Token likely a Macro
   static bool isPossibleMacro(const FormatToken *Tok);
diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index 5463bfbb65ca..792d8f3c3a98 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -1055,82 +1055,70 @@ TEST_F(QualifierFixerTest, IsQualifierType) {
   ConfiguredTokens.push_back(tok::kw_constexpr);
   ConfiguredTokens.push_back(tok::kw_friend);
 
-  LangOptions LangOpts{getFormattingLangOpts()};
-
   auto Tokens = annotate(
       "const static inline auto restrict int double long constexpr friend");
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
 
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[0], ConfiguredTokens, LangOpts));
+      Tokens[0], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[1], ConfiguredTokens, LangOpts));
+      Tokens[1], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[2], ConfiguredTokens, LangOpts));
+      Tokens[2], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[3], ConfiguredTokens, LangOpts));
+      Tokens[3], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[4], ConfiguredTokens, LangOpts));
+      Tokens[4], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[5], ConfiguredTokens, LangOpts));
+      Tokens[5], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[6], ConfiguredTokens, LangOpts));
+      Tokens[6], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[7], ConfiguredTokens, LangOpts));
+      Tokens[7], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[8], ConfiguredTokens, LangOpts));
+      Tokens[8], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[9], ConfiguredTokens, LangOpts));
-
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[0], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[1], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[2], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[3], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[4], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[5], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[6], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[7], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[8], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[9], LangOpts));
+      Tokens[9], ConfiguredTokens));
+
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[0]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[1]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[2]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[3]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[4]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[5]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[6]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[7]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[8]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[9]));
 
   auto NotTokens = annotate("for while do Foo Bar ");
   ASSERT_EQ(NotTokens.size(), 6u) << Tokens;
 
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[0], ConfiguredTokens, LangOpts));
+      NotTokens[0], ConfiguredTokens));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[1], ConfiguredTokens, LangOpts));
+      NotTokens[1], ConfiguredTokens));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[2], ConfiguredTokens, LangOpts));
+      NotTokens[2], ConfiguredTokens));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[3], ConfiguredTokens, LangOpts));
+      NotTokens[3], ConfiguredTokens));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[4], ConfiguredTokens, LangOpts));
+      NotTokens[4], ConfiguredTokens));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[5], ConfiguredTokens, LangOpts));
-
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[0],
-                                                                   LangOpts));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[1],
-                                                                   LangOpts));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[2],
-                                                                   LangOpts));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[3],
-                                                                   LangOpts));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[4],
-                                                                   LangOpts));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[5],
-                                                                   LangOpts));
+      NotTokens[5], ConfiguredTokens));
+
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[0]));
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[1]));
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[2]));
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[3]));
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[4]));
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[5]));
 }
 
 TEST_F(QualifierFixerTest, IsMacro) {
-- 
GitLab


From f42117c8517cc928c6373bad35ebf75d94fe865b Mon Sep 17 00:00:00 2001
From: Xiang Li <python3kgae@outlook.com>
Date: Sat, 11 May 2024 18:16:22 -0700
Subject: [PATCH 0493/1206] [DirectX] Replace bitfield version in
 ProgramHeader. (#91797)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Avoid using bitfield in dxbc::ProgramHeader.
It could potentially be read incorrectly on any host depending on the
compiler.

From [C++17's
[class.bit]](https://timsong-cpp.github.io/cppwp/n4659/class.bit#1)
> Bit-fields are packed into some addressable allocation unit. [ Note:
Bit-fields straddle allocation units on some machines and not on others.
Bit-fields are assigned right-to-left on some machines, left-to-right on
others.  — end note ]

For #91793
---
 llvm/include/llvm/BinaryFormat/DXContainer.h |  8 +++-
 llvm/lib/MC/MCDXContainerWriter.cpp          |  8 ++--
 llvm/lib/ObjectYAML/DXContainerEmitter.cpp   |  4 +-
 llvm/tools/obj2yaml/dxcontainer2yaml.cpp     |  4 +-
 llvm/unittests/Object/DXContainerTest.cpp    | 43 ++++++++++++++++++++
 5 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index e8d03f806715..847d8103e681 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -119,8 +119,7 @@ struct BitcodeHeader {
 };
 
 struct ProgramHeader {
-  uint8_t MinorVersion : 4;
-  uint8_t MajorVersion : 4;
+  uint8_t Version;
   uint8_t Unused;
   uint16_t ShaderKind;
   uint32_t Size; // Size in uint32_t words including this header.
@@ -131,6 +130,11 @@ struct ProgramHeader {
     sys::swapByteOrder(Size);
     Bitcode.swapBytes();
   }
+  uint8_t getMajorVersion() { return Version >> 4; }
+  uint8_t getMinorVersion() { return Version & 0xF; }
+  static uint8_t getVersion(uint8_t Major, uint8_t Minor) {
+    return (Major << 4) | Minor;
+  }
 };
 
 static_assert(sizeof(ProgramHeader) == 24, "ProgramHeader Size incorrect!");
diff --git a/llvm/lib/MC/MCDXContainerWriter.cpp b/llvm/lib/MC/MCDXContainerWriter.cpp
index 0580dc7e4282..ff64c6e538ac 100644
--- a/llvm/lib/MC/MCDXContainerWriter.cpp
+++ b/llvm/lib/MC/MCDXContainerWriter.cpp
@@ -117,9 +117,11 @@ uint64_t DXContainerObjectWriter::writeObject(MCAssembler &Asm,
 
       const Triple &TT = Asm.getContext().getTargetTriple();
       VersionTuple Version = TT.getOSVersion();
-      Header.MajorVersion = static_cast<uint8_t>(Version.getMajor());
-      if (Version.getMinor())
-        Header.MinorVersion = static_cast<uint8_t>(*Version.getMinor());
+      uint8_t MajorVersion = static_cast<uint8_t>(Version.getMajor());
+      uint8_t MinorVersion =
+          static_cast<uint8_t>(Version.getMinor().value_or(0));
+      Header.Version =
+          dxbc::ProgramHeader::getVersion(MajorVersion, MinorVersion);
       if (TT.hasEnvironment())
         Header.ShaderKind =
             static_cast<uint16_t>(TT.getEnvironment() - Triple::Pixel);
diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
index f3a518df3175..175f1a12f931 100644
--- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
@@ -131,8 +131,8 @@ void DXContainerWriter::writeParts(raw_ostream &OS) {
       if (!P.Program)
         continue;
       dxbc::ProgramHeader Header;
-      Header.MajorVersion = P.Program->MajorVersion;
-      Header.MinorVersion = P.Program->MinorVersion;
+      Header.Version = dxbc::ProgramHeader::getVersion(P.Program->MajorVersion,
+                                                       P.Program->MinorVersion);
       Header.Unused = 0;
       Header.ShaderKind = P.Program->ShaderKind;
       memcpy(Header.Bitcode.Magic, "DXIL", 4);
diff --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
index ec4f5c74498f..06966b188358 100644
--- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
+++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
@@ -58,8 +58,8 @@ dumpDXContainer(MemoryBufferRef Source) {
       assert(DXIL && "Since we are iterating and found a DXIL part, "
                      "this should never not have a value");
       NewPart.Program = DXContainerYAML::DXILProgram{
-          DXIL->first.MajorVersion,
-          DXIL->first.MinorVersion,
+          DXIL->first.getMajorVersion(),
+          DXIL->first.getMinorVersion(),
           DXIL->first.ShaderKind,
           DXIL->first.Size,
           DXIL->first.Bitcode.MajorVersion,
diff --git a/llvm/unittests/Object/DXContainerTest.cpp b/llvm/unittests/Object/DXContainerTest.cpp
index da640225617d..115de47a3cef 100644
--- a/llvm/unittests/Object/DXContainerTest.cpp
+++ b/llvm/unittests/Object/DXContainerTest.cpp
@@ -174,6 +174,49 @@ TEST(DXCFile, ParseEmptyParts) {
   }
 }
 
+// This test verify DXIL part are correctly parsed.
+// This test is based on the binary output constructed from this yaml.
+// --- !dxcontainer
+// Header:
+//   Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+//                      0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+//   Version:
+//     Major:           1
+//     Minor:           0
+//   PartCount:       1
+// Parts:
+//   - Name:            DXIL
+//     Size:            28
+//     Program:
+//       MajorVersion:    6
+//       MinorVersion:    5
+//       ShaderKind:      5
+//       Size:            8
+//       DXILMajorVersion: 1
+//       DXILMinorVersion: 5
+//       DXILSize:        4
+//       DXIL:            [ 0x42, 0x43, 0xC0, 0xDE, ]
+// ...
+TEST(DXCFile, ParseDXILPart) {
+  uint8_t Buffer[] = {
+      0x44, 0x58, 0x42, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+      0x48, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
+      0x44, 0x58, 0x49, 0x4c, 0x1c, 0x00, 0x00, 0x00, 0x65, 0x00, 0x05, 0x00,
+      0x08, 0x00, 0x00, 0x00, 0x44, 0x58, 0x49, 0x4c, 0x05, 0x01, 0x00, 0x00,
+      0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x42, 0x43, 0xc0, 0xde};
+  DXContainer C =
+      llvm::cantFail(DXContainer::create(getMemoryBuffer<116>(Buffer)));
+  EXPECT_EQ(C.getHeader().PartCount, 1u);
+  const std::optional<object::DXContainer::DXILData> &DXIL = C.getDXIL();
+  EXPECT_TRUE(DXIL.has_value());
+  dxbc::ProgramHeader Header = DXIL->first;
+  EXPECT_EQ(Header.getMajorVersion(), 6u);
+  EXPECT_EQ(Header.getMinorVersion(), 5u);
+  EXPECT_EQ(Header.ShaderKind, 5u);
+  EXPECT_EQ(Header.Size, 8u);
+}
+
 static Expected<DXContainer>
 generateDXContainer(StringRef Yaml, SmallVectorImpl<char> &BinaryData) {
   DXContainerYAML::Object Obj;
-- 
GitLab


From e586556e375fc5c4f7e76b5c299cb981f2016108 Mon Sep 17 00:00:00 2001
From: Shengchen Kan <shengchen.kan@intel.com>
Date: Sun, 12 May 2024 12:46:54 +0800
Subject: [PATCH 0494/1206] [X86][CodeGen] Add NDD entries for transform
 TEST+AND -> TEST

---
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 24 +++++++++----
 llvm/test/CodeGen/X86/cmp.ll            | 47 ++++++++-----------------
 2 files changed, 33 insertions(+), 38 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 14c62893766a..ea3b84d0ca9e 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -1560,7 +1560,9 @@ void X86DAGToDAGISel::PostprocessISelDAG() {
       SDValue And = N->getOperand(0);
       unsigned N0Opc = And.getMachineOpcode();
       if ((N0Opc == X86::AND8rr || N0Opc == X86::AND16rr ||
-           N0Opc == X86::AND32rr || N0Opc == X86::AND64rr) &&
+           N0Opc == X86::AND32rr || N0Opc == X86::AND64rr ||
+           N0Opc == X86::AND8rr_ND || N0Opc == X86::AND16rr_ND ||
+           N0Opc == X86::AND32rr_ND || N0Opc == X86::AND64rr_ND) &&
           !And->hasAnyUseOfValue(1)) {
         MachineSDNode *Test = CurDAG->getMachineNode(Opc, SDLoc(N),
                                                      MVT::i32,
@@ -1571,15 +1573,25 @@ void X86DAGToDAGISel::PostprocessISelDAG() {
         continue;
       }
       if ((N0Opc == X86::AND8rm || N0Opc == X86::AND16rm ||
-           N0Opc == X86::AND32rm || N0Opc == X86::AND64rm) &&
+           N0Opc == X86::AND32rm || N0Opc == X86::AND64rm ||
+           N0Opc == X86::AND8rm_ND || N0Opc == X86::AND16rm_ND ||
+           N0Opc == X86::AND32rm_ND || N0Opc == X86::AND64rm_ND) &&
           !And->hasAnyUseOfValue(1)) {
         unsigned NewOpc;
+#define CASE_ND(OP)                                                            \
+  case X86::OP:                                                                \
+  case X86::OP##_ND:
+#define FROM_TO(A, B)                                                          \
+  CASE_ND(A) NewOpc = X86::B;                                                  \
+  break;
         switch (N0Opc) {
-        case X86::AND8rm:  NewOpc = X86::TEST8mr; break;
-        case X86::AND16rm: NewOpc = X86::TEST16mr; break;
-        case X86::AND32rm: NewOpc = X86::TEST32mr; break;
-        case X86::AND64rm: NewOpc = X86::TEST64mr; break;
+          FROM_TO(AND8rm, TEST8mr);
+          FROM_TO(AND16rm, TEST16mr);
+          FROM_TO(AND32rm, TEST32mr);
+          FROM_TO(AND64rm, TEST64mr);
         }
+#undef FROM_TO
+#undef CASE_ND
 
         // Need to swap the memory and register operand.
         SDValue Ops[] = { And.getOperand(1),
diff --git a/llvm/test/CodeGen/X86/cmp.ll b/llvm/test/CodeGen/X86/cmp.ll
index 402da547613c..5a63d36a6be4 100644
--- a/llvm/test/CodeGen/X86/cmp.ll
+++ b/llvm/test/CodeGen/X86/cmp.ll
@@ -787,25 +787,15 @@ define i1 @shifted_mask64_testl(i64 %a) {
 }
 
 define i1 @shifted_mask64_extra_use_const(i64 %a) {
-; NO-NDD-LABEL: shifted_mask64_extra_use_const:
-; NO-NDD:       # %bb.0:
-; NO-NDD-NEXT:    movabsq $287104476244869120, %rcx # encoding: [0x48,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0xfc,0x03]
-; NO-NDD-NEXT:    # imm = 0x3FC000000000000
-; NO-NDD-NEXT:    testq %rcx, %rdi # encoding: [0x48,0x85,0xcf]
-; NO-NDD-NEXT:    setne %al # encoding: [0x0f,0x95,0xc0]
-; NO-NDD-NEXT:    movq %rcx, d64(%rip) # encoding: [0x48,0x89,0x0d,A,A,A,A]
-; NO-NDD-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
-; NO-NDD-NEXT:    retq # encoding: [0xc3]
-;
-; NDD-LABEL: shifted_mask64_extra_use_const:
-; NDD:       # %bb.0:
-; NDD-NEXT:    movabsq $287104476244869120, %rcx # encoding: [0x48,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0xfc,0x03]
-; NDD-NEXT:    # imm = 0x3FC000000000000
-; NDD-NEXT:    andq %rcx, %rdi, %rax # encoding: [0x62,0xf4,0xfc,0x18,0x21,0xcf]
-; NDD-NEXT:    setne %al # encoding: [0x0f,0x95,0xc0]
-; NDD-NEXT:    movq %rcx, d64(%rip) # encoding: [0x48,0x89,0x0d,A,A,A,A]
-; NDD-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
-; NDD-NEXT:    retq # encoding: [0xc3]
+; CHECK-LABEL: shifted_mask64_extra_use_const:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movabsq $287104476244869120, %rcx # encoding: [0x48,0xb9,0x00,0x00,0x00,0x00,0x00,0x00,0xfc,0x03]
+; CHECK-NEXT:    # imm = 0x3FC000000000000
+; CHECK-NEXT:    testq %rcx, %rdi # encoding: [0x48,0x85,0xcf]
+; CHECK-NEXT:    setne %al # encoding: [0x0f,0x95,0xc0]
+; CHECK-NEXT:    movq %rcx, d64(%rip) # encoding: [0x48,0x89,0x0d,A,A,A,A]
+; CHECK-NEXT:    # fixup A - offset: 3, value: d64-4, kind: reloc_riprel_4byte
+; CHECK-NEXT:    retq # encoding: [0xc3]
   %v0 = and i64 %a, 287104476244869120  ; 0xff << 50
   %v1 = icmp ne i64 %v0, 0
   store i64 287104476244869120, ptr @d64
@@ -954,19 +944,12 @@ declare i32 @f()
 ; The store makes sure the chain result of the load is used which used to
 ; prevent the post isel peephole from catching this.
 define i1 @fold_test_and_with_chain(ptr %x, ptr %y, i32 %z) {
-; NO-NDD-LABEL: fold_test_and_with_chain:
-; NO-NDD:       # %bb.0:
-; NO-NDD-NEXT:    testl %edx, (%rdi) # encoding: [0x85,0x17]
-; NO-NDD-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
-; NO-NDD-NEXT:    movl %edx, (%rsi) # encoding: [0x89,0x16]
-; NO-NDD-NEXT:    retq # encoding: [0xc3]
-;
-; NDD-LABEL: fold_test_and_with_chain:
-; NDD:       # %bb.0:
-; NDD-NEXT:    andl (%rdi), %edx, %eax # encoding: [0x62,0xf4,0x7c,0x18,0x23,0x17]
-; NDD-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
-; NDD-NEXT:    movl %edx, (%rsi) # encoding: [0x89,0x16]
-; NDD-NEXT:    retq # encoding: [0xc3]
+; CHECK-LABEL: fold_test_and_with_chain:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    testl %edx, (%rdi) # encoding: [0x85,0x17]
+; CHECK-NEXT:    sete %al # encoding: [0x0f,0x94,0xc0]
+; CHECK-NEXT:    movl %edx, (%rsi) # encoding: [0x89,0x16]
+; CHECK-NEXT:    retq # encoding: [0xc3]
   %a = load i32, ptr %x
   %b = and i32 %z, %a
   %c = icmp eq i32 %b, 0
-- 
GitLab


From 63224d717108d927e998da8a67050a6cc5dd74a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Sat, 11 May 2024 06:13:15 +0200
Subject: [PATCH 0495/1206] Reapply "Reapply "[clang][Interp] Create full type
 info for dummy pointers""

This reverts commit fb1c2dbc0a6ba641e0b39e0692ec6a4ee4444c4b.
---
 clang/lib/AST/Interp/Descriptor.cpp         |  8 -----
 clang/lib/AST/Interp/Descriptor.h           |  6 ++--
 clang/lib/AST/Interp/Interp.h               | 36 ++++++++-------------
 clang/lib/AST/Interp/Program.cpp            | 21 ++++++------
 clang/test/AST/Interp/builtin-align-cxx.cpp | 15 +--------
 clang/test/AST/Interp/c.c                   |  3 ++
 6 files changed, 33 insertions(+), 56 deletions(-)

diff --git a/clang/lib/AST/Interp/Descriptor.cpp b/clang/lib/AST/Interp/Descriptor.cpp
index 954c58c8cb37..d0466902247b 100644
--- a/clang/lib/AST/Interp/Descriptor.cpp
+++ b/clang/lib/AST/Interp/Descriptor.cpp
@@ -347,14 +347,6 @@ Descriptor::Descriptor(const DeclTy &D)
   assert(Source && "Missing source");
 }
 
-/// Dummy array.
-Descriptor::Descriptor(const DeclTy &D, UnknownSize)
-    : Source(D), ElemSize(1), Size(UnknownSizeMark), MDSize(0),
-      AllocSize(MDSize), ElemRecord(nullptr), IsConst(true), IsMutable(false),
-      IsTemporary(false), IsArray(true), IsDummy(true) {
-  assert(Source && "Missing source");
-}
-
 QualType Descriptor::getType() const {
   if (auto *E = asExpr())
     return E->getType();
diff --git a/clang/lib/AST/Interp/Descriptor.h b/clang/lib/AST/Interp/Descriptor.h
index cd20495c259c..fcb14e76e7eb 100644
--- a/clang/lib/AST/Interp/Descriptor.h
+++ b/clang/lib/AST/Interp/Descriptor.h
@@ -128,7 +128,7 @@ public:
   /// Flag indicating if the block is an array.
   const bool IsArray = false;
   /// Flag indicating if this is a dummy descriptor.
-  const bool IsDummy = false;
+  bool IsDummy = false;
 
   /// Storage management methods.
   const BlockCtorFn CtorFn = nullptr;
@@ -162,8 +162,8 @@ public:
   /// Allocates a dummy descriptor.
   Descriptor(const DeclTy &D);
 
-  /// Allocates a dummy array descriptor.
-  Descriptor(const DeclTy &D, UnknownSize);
+  /// Make this descriptor a dummy descriptor.
+  void makeDummy() { IsDummy = true; }
 
   QualType getType() const;
   QualType getElemQualType() const;
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index 66d30cc3fbaa..a0bf87430012 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -823,9 +823,9 @@ inline bool CmpHelperEQ<Pointer>(InterpState &S, CodePtr OpPC, CompareFn Fn) {
     // element in the same array are NOT equal. They have the same Base value,
     // but a different Offset. This is a pretty rare case, so we fix this here
     // by comparing pointers to the first elements.
-    if (!LHS.isZero() && !LHS.isDummy() && LHS.isArrayRoot())
+    if (!LHS.isZero() && LHS.isArrayRoot())
       VL = LHS.atIndex(0).getByteOffset();
-    if (!RHS.isZero() && !RHS.isDummy() && RHS.isArrayRoot())
+    if (!RHS.isZero() && RHS.isArrayRoot())
       VR = RHS.atIndex(0).getByteOffset();
 
     S.Stk.push<BoolT>(BoolT::from(Fn(Compare(VL, VR))));
@@ -1241,14 +1241,16 @@ inline bool GetPtrField(InterpState &S, CodePtr OpPC, uint32_t Off) {
       !CheckNull(S, OpPC, Ptr, CSK_Field))
     return false;
 
-  if (CheckDummy(S, OpPC, Ptr)) {
-    if (!CheckExtern(S, OpPC, Ptr))
-      return false;
-    if (!CheckRange(S, OpPC, Ptr, CSK_Field))
-      return false;
-    if (!CheckSubobject(S, OpPC, Ptr, CSK_Field))
-      return false;
-  }
+  if (!CheckExtern(S, OpPC, Ptr))
+    return false;
+  if (!CheckRange(S, OpPC, Ptr, CSK_Field))
+    return false;
+  if (!CheckSubobject(S, OpPC, Ptr, CSK_Field))
+    return false;
+
+  if (Ptr.isBlockPointer() && Off > Ptr.block()->getSize())
+    return false;
+
   S.Stk.push<Pointer>(Ptr.atField(Off));
   return true;
 }
@@ -2034,11 +2036,6 @@ inline bool ArrayElemPtr(InterpState &S, CodePtr OpPC) {
   if (!Ptr.isZero()) {
     if (!CheckArray(S, OpPC, Ptr))
       return false;
-
-    if (Ptr.isDummy()) {
-      S.Stk.push<Pointer>(Ptr);
-      return true;
-    }
   }
 
   if (!OffsetHelper<T, ArithOp::Add>(S, OpPC, Offset, Ptr))
@@ -2055,11 +2052,6 @@ inline bool ArrayElemPtrPop(InterpState &S, CodePtr OpPC) {
   if (!Ptr.isZero()) {
     if (!CheckArray(S, OpPC, Ptr))
       return false;
-
-    if (Ptr.isDummy()) {
-      S.Stk.push<Pointer>(Ptr);
-      return true;
-    }
   }
 
   if (!OffsetHelper<T, ArithOp::Add>(S, OpPC, Offset, Ptr))
@@ -2113,12 +2105,12 @@ inline bool CopyArray(InterpState &S, CodePtr OpPC, uint32_t SrcIndex, uint32_t
 inline bool ArrayDecay(InterpState &S, CodePtr OpPC) {
   const Pointer &Ptr = S.Stk.pop<Pointer>();
 
-  if (Ptr.isZero() || Ptr.isDummy()) {
+  if (Ptr.isZero()) {
     S.Stk.push<Pointer>(Ptr);
     return true;
   }
 
-  if (!Ptr.isUnknownSizeArray()) {
+  if (!Ptr.isUnknownSizeArray() || Ptr.isDummy()) {
     S.Stk.push<Pointer>(Ptr.atIndex(0));
     return true;
   }
diff --git a/clang/lib/AST/Interp/Program.cpp b/clang/lib/AST/Interp/Program.cpp
index 02075c20cf55..87c767f85e79 100644
--- a/clang/lib/AST/Interp/Program.cpp
+++ b/clang/lib/AST/Interp/Program.cpp
@@ -144,17 +144,20 @@ std::optional<unsigned> Program::getOrCreateDummy(const ValueDecl *VD) {
   if (auto It = DummyVariables.find(VD); It != DummyVariables.end())
     return It->second;
 
-  // Create dummy descriptor.
-  // We create desriptors of 'array of unknown size' if the type is an array
-  // type _and_ the size isn't known (it's not a ConstantArrayType). If the size
-  // is known however, we create a regular dummy pointer.
   Descriptor *Desc;
-  if (const auto *AT = VD->getType()->getAsArrayTypeUnsafe();
-      AT && !isa<ConstantArrayType>(AT))
-    Desc = allocateDescriptor(VD, Descriptor::UnknownSize{});
+  if (std::optional<PrimType> T = Ctx.classify(VD->getType()))
+    Desc = createDescriptor(VD, *T, std::nullopt, true, false);
   else
+    Desc = createDescriptor(VD, VD->getType().getTypePtr(), std::nullopt, true,
+                            false);
+  if (!Desc)
     Desc = allocateDescriptor(VD);
 
+  assert(Desc);
+  Desc->makeDummy();
+
+  assert(Desc->isDummy());
+
   // Allocate a block for storage.
   unsigned I = Globals.size();
 
@@ -310,8 +313,7 @@ Record *Program::getOrCreateRecord(const RecordDecl *RD) {
   for (const FieldDecl *FD : RD->fields()) {
     // Note that we DO create fields and descriptors
     // for unnamed bitfields here, even though we later ignore
-    // them everywhere. That's because so the FieldDecl's
-    // getFieldIndex() matches.
+    // them everywhere. That's so the FieldDecl's getFieldIndex() matches.
 
     // Reserve space for the field's descriptor and the offset.
     BaseSize += align(sizeof(InlineDescriptor));
@@ -344,6 +346,7 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
                                       Descriptor::MetadataSize MDSize,
                                       bool IsConst, bool IsTemporary,
                                       bool IsMutable, const Expr *Init) {
+
   // Classes and structures.
   if (const auto *RT = Ty->getAs<RecordType>()) {
     if (const auto *Record = getOrCreateRecord(RT->getDecl()))
diff --git a/clang/test/AST/Interp/builtin-align-cxx.cpp b/clang/test/AST/Interp/builtin-align-cxx.cpp
index 62d73dba929b..c4103953df02 100644
--- a/clang/test/AST/Interp/builtin-align-cxx.cpp
+++ b/clang/test/AST/Interp/builtin-align-cxx.cpp
@@ -2,19 +2,6 @@
 // RUN: %clang_cc1 -triple=x86_64-unknown-unknown -std=c++11 %s -fsyntax-only -verify=expected,both -fexperimental-new-constant-interpreter
 // RUN: %clang_cc1 -triple=x86_64-unknown-unknown -std=c++11 %s -fsyntax-only -verify=ref,both
 
-
-/// This is just a copy of the one from test/SemaCXX/ with some of the
-/// diagnostic output adapted.
-/// Also, align32array has an initializer now, which means it's not just
-/// a dummy pointer for us and we do actually have type information for it.
-/// In the future, we need to retain type information for dummy pointers as
-/// well, so here is a test that will break once we do that:
-namespace {
-  _Alignas(32) char heh[4];
-  static_assert(!__builtin_is_aligned(&heh[1], 4), ""); // expected-error {{failed}}
-}
-
-
 // Check that we don't crash when using dependent types in __builtin_align:
 template <typename a, a b>
 void *c(void *d) { // both-note{{candidate template ignored}}
@@ -177,7 +164,7 @@ static_assert(wrap_align_up(static_cast<bool>(1), const_value(1 << 21)), ""); //
 // both-note@-1{{in instantiation of function template specialization 'wrap_align_up<bool>' requested here}}
 
 // Check constant evaluation for pointers:
-_Alignas(32) char align32array[128] = {};
+_Alignas(32) char align32array[128];
 static_assert(&align32array[0] == &align32array[0], "");
 // __builtin_align_up/down can be constant evaluated as a no-op for values
 // that are known to have greater alignment:
diff --git a/clang/test/AST/Interp/c.c b/clang/test/AST/Interp/c.c
index 207da5fe8126..2c675f4418ef 100644
--- a/clang/test/AST/Interp/c.c
+++ b/clang/test/AST/Interp/c.c
@@ -270,3 +270,6 @@ int test3(void) {
   a[0] = test3; // all-error {{incompatible pointer to integer conversion assigning to 'int' from 'int (void)'}}
   return 0;
 }
+/// This tests that we have full type info, even for values we cannot read.
+int dummyarray[5];
+_Static_assert(&dummyarray[0] < &dummyarray[1], ""); // pedantic-warning {{GNU extension}}
-- 
GitLab


From c8864bceeb20582b4e7a739d8ba3e11052f0e49f Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks@fb.com>
Date: Sat, 11 May 2024 23:59:57 -0700
Subject: [PATCH 0496/1206] [BOLT] Fix race condition in a test (#91866)

Fix race condition in internal NFC test.
---
 bolt/test/runtime/bolt-reserved.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bolt/test/runtime/bolt-reserved.cpp b/bolt/test/runtime/bolt-reserved.cpp
index 5e93b4f7c3d4..c88b1e284d07 100644
--- a/bolt/test/runtime/bolt-reserved.cpp
+++ b/bolt/test/runtime/bolt-reserved.cpp
@@ -16,8 +16,8 @@
  * not enough for allocating new sections.
  */
 
-// RUN: %clang %s -o %t.exe -Wl,--no-eh-frame-hdr -Wl,-q -DTINY
-// RUN: not llvm-bolt %t.exe -o %t.bolt.exe 2>&1 | \
+// RUN: %clang %s -o %t.tiny.exe -Wl,--no-eh-frame-hdr -Wl,-q -DTINY
+// RUN: not llvm-bolt %t.tiny.exe -o %t.tiny.bolt.exe 2>&1 | \
 // RUN:   FileCheck %s --check-prefix=CHECK-TINY
 
 // CHECK-TINY: BOLT-ERROR: reserved space (1 byte) is smaller than required
-- 
GitLab


From 1d6bf0ca29322b08e8b50681d440e7182441b025 Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Sun, 12 May 2024 09:45:00 -0400
Subject: [PATCH 0497/1206] [clang][NFC] Remove class layout scissor (#89055)

PR #87090 amended `accumulateBitfields` to do the correct clipping. The scissor is no longer necessary and  `checkBitfieldClipping` can compute its location directly when needed.
---
 clang/lib/CodeGen/CGRecordLayoutBuilder.cpp | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
index 868b1ab98e04..5169be204c14 100644
--- a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -75,7 +75,7 @@ struct CGRecordLowering {
   // sentinel member type that ensures correct rounding.
   struct MemberInfo {
     CharUnits Offset;
-    enum InfoKind { VFPtr, VBPtr, Field, Base, VBase, Scissor } Kind;
+    enum InfoKind { VFPtr, VBPtr, Field, Base, VBase } Kind;
     llvm::Type *Data;
     union {
       const FieldDecl *FD;
@@ -197,7 +197,7 @@ struct CGRecordLowering {
                      const CXXRecordDecl *Query) const;
   void calculateZeroInit();
   CharUnits calculateTailClippingOffset(bool isNonVirtualBaseType) const;
-  void checkBitfieldClipping() const;
+  void checkBitfieldClipping(bool isNonVirtualBaseType) const;
   /// Determines if we need a packed llvm struct.
   void determinePacked(bool NVBaseType);
   /// Inserts padding everywhere it's needed.
@@ -299,8 +299,8 @@ void CGRecordLowering::lower(bool NVBaseType) {
       accumulateVBases();
   }
   llvm::stable_sort(Members);
+  checkBitfieldClipping(NVBaseType);
   Members.push_back(StorageInfo(Size, getIntNType(8)));
-  checkBitfieldClipping();
   determinePacked(NVBaseType);
   insertPadding();
   Members.pop_back();
@@ -894,8 +894,6 @@ CGRecordLowering::calculateTailClippingOffset(bool isNonVirtualBaseType) const {
 }
 
 void CGRecordLowering::accumulateVBases() {
-  Members.push_back(MemberInfo(calculateTailClippingOffset(false),
-                               MemberInfo::Scissor, nullptr, RD));
   for (const auto &Base : RD->vbases()) {
     const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl();
     if (BaseDecl->isEmpty())
@@ -950,18 +948,19 @@ void CGRecordLowering::calculateZeroInit() {
 }
 
 // Verify accumulateBitfields computed the correct storage representations.
-void CGRecordLowering::checkBitfieldClipping() const {
+void CGRecordLowering::checkBitfieldClipping(bool IsNonVirtualBaseType) const {
 #ifndef NDEBUG
+  auto ScissorOffset = calculateTailClippingOffset(IsNonVirtualBaseType);
   auto Tail = CharUnits::Zero();
   for (const auto &M : Members) {
-    // Only members with data and the scissor can cut into tail padding.
-    if (!M.Data && M.Kind != MemberInfo::Scissor)
+    // Only members with data could possibly overlap.
+    if (!M.Data)
       continue;
 
     assert(M.Offset >= Tail && "Bitfield access unit is not clipped");
-    Tail = M.Offset;
-    if (M.Data)
-      Tail += getSize(M.Data);
+    Tail = M.Offset + getSize(M.Data);
+    assert((Tail <= ScissorOffset || M.Offset >= ScissorOffset) &&
+           "Bitfield straddles scissor offset");
   }
 #endif
 }
-- 
GitLab


From a477004d822d02290f9d388aa22a42490fd8180d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 12 May 2024 15:21:08 +0100
Subject: [PATCH 0498/1206] [CostModel][X86] Remove getGSScalarCost and use
 getCommonMaskedMemoryOpCost directly

The generic getCommonMaskedMemoryOpCost now gives the same cost estimates for scalarized gather/scatter.
---
 .../lib/Target/X86/X86TargetTransformInfo.cpp | 61 +------------------
 llvm/lib/Target/X86/X86TargetTransformInfo.h  |  3 -
 2 files changed, 1 insertion(+), 63 deletions(-)

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index ebcc11533add..95cd35ac694b 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5841,59 +5841,12 @@ InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode,
                                            CostKind);
 }
 
-/// Return the cost of full scalarization of gather / scatter operation.
-///
-/// Opcode - Load or Store instruction.
-/// SrcVTy - The type of the data vector that should be gathered or scattered.
-/// VariableMask - The mask is non-constant at compile time.
-/// Alignment - Alignment for one element.
-/// AddressSpace - pointer[s] address space.
-/// TODO: Remove this and use getCommonMaskedMemoryOpCost directly.
-InstructionCost X86TTIImpl::getGSScalarCost(unsigned Opcode,
-                                            TTI::TargetCostKind CostKind,
-                                            Type *SrcVTy, bool VariableMask,
-                                            Align Alignment,
-                                            unsigned AddressSpace) {
-  Type *ScalarTy = SrcVTy->getScalarType();
-  unsigned VF = cast<FixedVectorType>(SrcVTy)->getNumElements();
-  APInt DemandedElts = APInt::getAllOnes(VF);
-
-  InstructionCost MaskUnpackCost = 0;
-  if (VariableMask) {
-    auto *MaskTy =
-        FixedVectorType::get(Type::getInt1Ty(SrcVTy->getContext()), VF);
-    MaskUnpackCost = getScalarizationOverhead(
-        MaskTy, DemandedElts, /*Insert=*/false, /*Extract=*/true, CostKind);
-    InstructionCost BranchCost = getCFInstrCost(Instruction::Br, CostKind);
-    MaskUnpackCost += VF * BranchCost;
-  }
-
-  InstructionCost AddressUnpackCost = getScalarizationOverhead(
-      FixedVectorType::get(PointerType::getUnqual(ScalarTy->getContext()), VF),
-      DemandedElts, /*Insert=*/false, /*Extract=*/true, CostKind);
-
-  // The cost of the scalar loads/stores.
-  InstructionCost MemoryOpCost =
-      VF * getMemoryOpCost(Opcode, ScalarTy, MaybeAlign(Alignment),
-                           AddressSpace, CostKind);
-
-  // The cost of forming the vector from loaded scalars/
-  // scalarizing the vector to perform scalar stores.
-  InstructionCost InsertExtractCost = getScalarizationOverhead(
-      cast<FixedVectorType>(SrcVTy), DemandedElts,
-      /*Insert=*/Opcode == Instruction::Load,
-      /*Extract=*/Opcode == Instruction::Store, CostKind);
-
-  return AddressUnpackCost + MemoryOpCost + MaskUnpackCost + InsertExtractCost;
-}
-
 /// Calculate the cost of Gather / Scatter operation
 InstructionCost X86TTIImpl::getGatherScatterOpCost(
     unsigned Opcode, Type *SrcVTy, const Value *Ptr, bool VariableMask,
     Align Alignment, TTI::TargetCostKind CostKind,
     const Instruction *I = nullptr) {
-  if (CostKind != TTI::TCK_RecipThroughput &&
-      ((Opcode == Instruction::Load &&
+  if (((Opcode == Instruction::Load &&
         (!isLegalMaskedGather(SrcVTy, Align(Alignment)) ||
          forceScalarizeMaskedGather(cast<VectorType>(SrcVTy),
                                     Align(Alignment)))) ||
@@ -5911,18 +5864,6 @@ InstructionCost X86TTIImpl::getGatherScatterOpCost(
         cast<VectorType>(Ptr->getType())->getElementType());
   assert(PtrTy && "Unexpected type for Ptr argument");
   unsigned AddressSpace = PtrTy->getAddressSpace();
-
-  if ((Opcode == Instruction::Load &&
-       (!isLegalMaskedGather(SrcVTy, Align(Alignment)) ||
-        forceScalarizeMaskedGather(cast<VectorType>(SrcVTy),
-                                   Align(Alignment)))) ||
-      (Opcode == Instruction::Store &&
-       (!isLegalMaskedScatter(SrcVTy, Align(Alignment)) ||
-        forceScalarizeMaskedScatter(cast<VectorType>(SrcVTy),
-                                    Align(Alignment)))))
-    return getGSScalarCost(Opcode, CostKind, SrcVTy, VariableMask, Alignment,
-                           AddressSpace);
-
   return getGSVectorCost(Opcode, CostKind, SrcVTy, Ptr, Alignment,
                          AddressSpace);
 }
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index d720cc136b8a..e14dc9fc0905 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -295,9 +295,6 @@ public:
 
 private:
   bool supportsGather() const;
-  InstructionCost getGSScalarCost(unsigned Opcode, TTI::TargetCostKind CostKind,
-                                  Type *DataTy, bool VariableMask,
-                                  Align Alignment, unsigned AddressSpace);
   InstructionCost getGSVectorCost(unsigned Opcode, TTI::TargetCostKind CostKind,
                                   Type *DataTy, const Value *Ptr,
                                   Align Alignment, unsigned AddressSpace);
-- 
GitLab


From 502e77df1fc4aa859db6709e14e93af6207e4dc4 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 12 May 2024 15:24:04 +0100
Subject: [PATCH 0499/1206] [CostModel][X86] getGSVectorCost - remove FIXME

getGSVectorCost has supported other TargetCostKind since a55127281b2ed5f24f848b9e5c70870ad170bc3f
---
 llvm/lib/Target/X86/X86TargetTransformInfo.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 95cd35ac694b..0a0bc7b32e87 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5769,7 +5769,6 @@ int X86TTIImpl::getScatterOverhead() const {
 }
 
 // Return an average cost of Gather / Scatter instruction, maybe improved later.
-// FIXME: Add TargetCostKind support.
 InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode,
                                             TTI::TargetCostKind CostKind,
                                             Type *SrcVTy, const Value *Ptr,
-- 
GitLab


From 78b3a00418ce6da0426a261a64a77608d0264fe5 Mon Sep 17 00:00:00 2001
From: Felix Schneider <fx.schn@gmail.com>
Date: Sun, 12 May 2024 18:11:42 +0200
Subject: [PATCH 0500/1206] [mlir] `int-range-optmizations`: Fix referencing of
 deleted ops (#91807)

The pass runs a `DataFlowSolver` and collects state information on the
input IR. Then, the rewrite driver and folding is applied. During
pattern application and folding it can happen that an Op from the input
IR is deleted and a new Op is created at the same address. When the
newly created Ops is looked up in the `DataFlowSolver` state memory, the
state of the original Op is returned.

This patch adds a method to `DataFlowSolver` which removes all state
related to a `ProgramPoint`. It also adds a listener to the Pass which
clears the state information of deleted Ops from the `DataFlowSolver`.

Fix https://github.com/llvm/llvm-project/issues/81228
---
 .../include/mlir/Analysis/DataFlowFramework.h | 11 ++++++++
 .../Transforms/IntRangeOptimizations.cpp      | 25 ++++++++++++++++++-
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Analysis/DataFlowFramework.h b/mlir/include/mlir/Analysis/DataFlowFramework.h
index c76cfac07fc7..2580ec28b519 100644
--- a/mlir/include/mlir/Analysis/DataFlowFramework.h
+++ b/mlir/include/mlir/Analysis/DataFlowFramework.h
@@ -242,6 +242,17 @@ public:
     return static_cast<const StateT *>(it->second.get());
   }
 
+  /// Erase any analysis state associated with the given program point.
+  template <typename PointT>
+  void eraseState(PointT point) {
+    ProgramPoint pp(point);
+
+    for (auto it = analysisStates.begin(); it != analysisStates.end(); ++it) {
+      if (it->first.first == pp)
+        analysisStates.erase(it);
+    }
+  }
+
   /// Get a uniqued program point instance. If one is not present, it is
   /// created with the provided arguments.
   template <typename PointT, typename... Args>
diff --git a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
index 92cad7cd1ef2..2473169962b9 100644
--- a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
@@ -102,6 +102,24 @@ static FailureOr<bool> handleUge(ConstantIntRanges lhs, ConstantIntRanges rhs) {
 }
 
 namespace {
+/// This class listens on IR transformations performed during a pass relying on
+/// information from a `DataflowSolver`. It erases state associated with the
+/// erased operation and its results from the `DataFlowSolver` so that Patterns
+/// do not accidentally query old state information for newly created Ops.
+class DataFlowListener : public RewriterBase::Listener {
+public:
+  DataFlowListener(DataFlowSolver &s) : s(s) {}
+
+protected:
+  void notifyOperationErased(Operation *op) override {
+    s.eraseState(op);
+    for (Value res : op->getResults())
+      s.eraseState(res);
+  }
+
+  DataFlowSolver &s;
+};
+
 struct ConvertCmpOp : public OpRewritePattern<arith::CmpIOp> {
 
   ConvertCmpOp(MLIRContext *context, DataFlowSolver &s)
@@ -167,10 +185,15 @@ struct IntRangeOptimizationsPass
     if (failed(solver.initializeAndRun(op)))
       return signalPassFailure();
 
+    DataFlowListener listener(solver);
+
     RewritePatternSet patterns(ctx);
     populateIntRangeOptimizationsPatterns(patterns, solver);
 
-    if (failed(applyPatternsAndFoldGreedily(op, std::move(patterns))))
+    GreedyRewriteConfig config;
+    config.listener = &listener;
+
+    if (failed(applyPatternsAndFoldGreedily(op, std::move(patterns), config)))
       signalPassFailure();
   }
 };
-- 
GitLab


From 17daa204feadf9c28fc13b7daa69c3cbe865b238 Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777@gmail.com>
Date: Mon, 13 May 2024 01:16:36 +0800
Subject: [PATCH 0501/1206] [Clang][CWG1815] Support lifetime extension of
 temporary created by aggregate initialization using a default member
 initializer (#87933)

This PR complete [DR1815](https://wg21.link/CWG1815) under the guidance
of `FIXME` comments. And reuse `CXXDefaultInitExpr` rewrite machinery to
clone the initializer expression on each use that would lifetime extend
its temporaries.

---------

Signed-off-by: yronglin <yronglin777@gmail.com>
---
 .../clang/Basic/DiagnosticSemaKinds.td        |  6 ----
 clang/lib/Sema/SemaExpr.cpp                   | 31 ++++++++++++-----
 clang/lib/Sema/SemaInit.cpp                   | 19 +----------
 clang/test/AST/ast-dump-default-init-json.cpp |  6 ++--
 clang/test/AST/ast-dump-default-init.cpp      |  2 +-
 .../Analysis/lifetime-extended-regions.cpp    |  9 ++---
 clang/test/CXX/drs/cwg16xx.cpp                |  2 --
 clang/test/CXX/drs/cwg18xx.cpp                | 19 ++++++++---
 clang/test/CXX/special/class.temporary/p6.cpp | 34 +++++++++++++++++++
 clang/test/SemaCXX/constexpr-default-arg.cpp  |  4 +--
 clang/test/SemaCXX/eval-crashes.cpp           |  6 ++--
 clang/www/cxx_dr_status.html                  |  2 +-
 12 files changed, 86 insertions(+), 54 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index d6863f90edb6..9e82130c9360 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -10035,12 +10035,6 @@ def warn_new_dangling_initializer_list : Warning<
   "the allocated initializer list}0 "
   "will be destroyed at the end of the full-expression">,
   InGroup<DanglingInitializerList>;
-def warn_unsupported_lifetime_extension : Warning<
-  "lifetime extension of "
-  "%select{temporary|backing array of initializer list}0 created "
-  "by aggregate initialization using a default member initializer "
-  "is not yet supported; lifetime of %select{temporary|backing array}0 "
-  "will end at the end of the full-expression">, InGroup<Dangling>;
 
 // For non-floating point, expressions of the form x == x or x != x
 // should result in a warning, since these always evaluate to a constant.
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index c688cb21f236..bb4b116fd73c 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -5777,10 +5777,9 @@ ExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc,
         Res = Immediate.TransformInitializer(Param->getInit(),
                                              /*NotCopy=*/false);
       });
-      if (Res.isInvalid())
-        return ExprError();
-      Res = ConvertParamDefaultArgument(Param, Res.get(),
-                                        Res.get()->getBeginLoc());
+      if (Res.isUsable())
+        Res = ConvertParamDefaultArgument(Param, Res.get(),
+                                          Res.get()->getBeginLoc());
       if (Res.isInvalid())
         return ExprError();
       Init = Res.get();
@@ -5816,7 +5815,7 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) {
   Expr *Init = nullptr;
 
   bool NestedDefaultChecking = isCheckingDefaultArgumentOrInitializer();
-
+  bool InLifetimeExtendingContext = isInLifetimeExtendingContext();
   EnterExpressionEvaluationContext EvalContext(
       *this, ExpressionEvaluationContext::PotentiallyEvaluated, Field);
 
@@ -5851,19 +5850,35 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) {
   ImmediateCallVisitor V(getASTContext());
   if (!NestedDefaultChecking)
     V.TraverseDecl(Field);
-  if (V.HasImmediateCalls) {
+
+  // CWG1815
+  // Support lifetime extension of temporary created by aggregate
+  // initialization using a default member initializer. We should always rebuild
+  // the initializer if it contains any temporaries (if the initializer
+  // expression is an ExprWithCleanups). Then make sure the normal lifetime
+  // extension code recurses into the default initializer and does lifetime
+  // extension when warranted.
+  bool ContainsAnyTemporaries =
+      isa_and_present<ExprWithCleanups>(Field->getInClassInitializer());
+  if (V.HasImmediateCalls || InLifetimeExtendingContext ||
+      ContainsAnyTemporaries) {
     ExprEvalContexts.back().DelayedDefaultInitializationContext = {Loc, Field,
                                                                    CurContext};
     ExprEvalContexts.back().IsCurrentlyCheckingDefaultArgumentOrInitializer =
         NestedDefaultChecking;
-
+    // Pass down lifetime extending flag, and collect temporaries in
+    // CreateMaterializeTemporaryExpr when we rewrite the call argument.
+    keepInLifetimeExtendingContext();
     EnsureImmediateInvocationInDefaultArgs Immediate(*this);
     ExprResult Res;
+
+    // Rebuild CXXDefaultInitExpr might cause diagnostics.
+    SFINAETrap Trap(*this);
     runWithSufficientStackSpace(Loc, [&] {
       Res = Immediate.TransformInitializer(Field->getInClassInitializer(),
                                            /*CXXDirectInit=*/false);
     });
-    if (!Res.isInvalid())
+    if (Res.isUsable())
       Res = ConvertMemberDefaultInitExpression(Field, Res.get(), Loc);
     if (Res.isInvalid()) {
       Field->setInvalidDecl();
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index c8049ae581f8..fe4a698a612e 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -8065,11 +8065,6 @@ static void visitLocalsRetainedByInitializer(IndirectLocalPath &Path,
 enum PathLifetimeKind {
   /// Lifetime-extend along this path.
   Extend,
-  /// We should lifetime-extend, but we don't because (due to technical
-  /// limitations) we can't. This happens for default member initializers,
-  /// which we don't clone for every use, so we don't have a unique
-  /// MaterializeTemporaryExpr to update.
-  ShouldExtend,
   /// Do not lifetime extend along this path.
   NoExtend
 };
@@ -8081,7 +8076,7 @@ shouldLifetimeExtendThroughPath(const IndirectLocalPath &Path) {
   PathLifetimeKind Kind = PathLifetimeKind::Extend;
   for (auto Elem : Path) {
     if (Elem.Kind == IndirectLocalPathEntry::DefaultInit)
-      Kind = PathLifetimeKind::ShouldExtend;
+      Kind = PathLifetimeKind::Extend;
     else if (Elem.Kind != IndirectLocalPathEntry::LambdaCaptureInit)
       return PathLifetimeKind::NoExtend;
   }
@@ -8201,18 +8196,6 @@ void Sema::checkInitializerLifetime(const InitializedEntity &Entity,
                               ExtendingEntity->allocateManglingNumber());
         // Also visit the temporaries lifetime-extended by this initializer.
         return true;
-
-      case PathLifetimeKind::ShouldExtend:
-        // We're supposed to lifetime-extend the temporary along this path (per
-        // the resolution of DR1815), but we don't support that yet.
-        //
-        // FIXME: Properly handle this situation. Perhaps the easiest approach
-        // would be to clone the initializer expression on each use that would
-        // lifetime extend its temporaries.
-        Diag(DiagLoc, diag::warn_unsupported_lifetime_extension)
-            << RK << DiagRange;
-        break;
-
       case PathLifetimeKind::NoExtend:
         // If the path goes through the initialization of a variable or field,
         // it can't possibly reach a temporary created in this full-expression.
diff --git a/clang/test/AST/ast-dump-default-init-json.cpp b/clang/test/AST/ast-dump-default-init-json.cpp
index 1058b4e3ea4d..f4949a9c9eed 100644
--- a/clang/test/AST/ast-dump-default-init-json.cpp
+++ b/clang/test/AST/ast-dump-default-init-json.cpp
@@ -789,10 +789,10 @@ void test() {
 // CHECK-NEXT:                  "valueCategory": "lvalue",
 // CHECK-NEXT:                  "extendingDecl": {
 // CHECK-NEXT:                   "id": "0x{{.*}}",
-// CHECK-NEXT:                   "kind": "FieldDecl",
-// CHECK-NEXT:                   "name": "a",
+// CHECK-NEXT:                   "kind": "VarDecl",
+// CHECK-NEXT:                   "name": "b",
 // CHECK-NEXT:                   "type": {
-// CHECK-NEXT:                    "qualType": "const A &"
+// CHECK-NEXT:                    "qualType": "B"
 // CHECK-NEXT:                   }
 // CHECK-NEXT:                  },
 // CHECK-NEXT:                  "storageDuration": "automatic",
diff --git a/clang/test/AST/ast-dump-default-init.cpp b/clang/test/AST/ast-dump-default-init.cpp
index 15b29f04bf21..26864fbf1542 100644
--- a/clang/test/AST/ast-dump-default-init.cpp
+++ b/clang/test/AST/ast-dump-default-init.cpp
@@ -13,7 +13,7 @@ void test() {
 }
 // CHECK: -CXXDefaultInitExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue has rewritten init
 // CHECK-NEXT:  `-ExprWithCleanups 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue
-// CHECK-NEXT:    `-MaterializeTemporaryExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue extended by Field 0x{{[^ ]*}} 'a' 'const A &'
+// CHECK-NEXT:    `-MaterializeTemporaryExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue extended by Var 0x{{[^ ]*}} 'b' 'B'
 // CHECK-NEXT:      `-ImplicitCastExpr 0x{{[^ ]*}} <{{.*}}> 'const A' <NoOp>
 // CHECK-NEXT:        `-CXXFunctionalCastExpr 0x{{[^ ]*}} <{{.*}}> 'A' functional cast to A <NoOp>
 // CHECK-NEXT:          `-InitListExpr 0x{{[^ ]*}} <{{.*}}> 'A'
diff --git a/clang/test/Analysis/lifetime-extended-regions.cpp b/clang/test/Analysis/lifetime-extended-regions.cpp
index 4e98bd4b0403..4458ad294af7 100644
--- a/clang/test/Analysis/lifetime-extended-regions.cpp
+++ b/clang/test/Analysis/lifetime-extended-regions.cpp
@@ -120,10 +120,11 @@ void aggregateWithReferences() {
   clang_analyzer_dump(viaReference);    // expected-warning-re {{&lifetime_extended_object{RefAggregate, viaReference, S{{[0-9]+}}} }}
   clang_analyzer_dump(viaReference.rx); // expected-warning-re {{&lifetime_extended_object{int, viaReference, S{{[0-9]+}}} }}
   clang_analyzer_dump(viaReference.ry); // expected-warning-re {{&lifetime_extended_object{Composite, viaReference, S{{[0-9]+}}} }}
-
-  // clang does not currently implement extending lifetime of object bound to reference members of aggregates,
-  // that are created from default member initializer (see `warn_unsupported_lifetime_extension` from `-Wdangling`)
-  RefAggregate defaultInitExtended{i}; // clang-bug does not extend `Composite`
+  
+  // FIXME: clang currently support extending lifetime of object bound to reference members of aggregates,
+  // that are created from default member initializer. But CFG and ExprEngine need to be updated to address this change.
+  // The following expect warning: {{&lifetime_extended_object{Composite, defaultInitExtended, S{{[0-9]+}}} }}
+  RefAggregate defaultInitExtended{i};
   clang_analyzer_dump(defaultInitExtended.ry); // expected-warning {{Unknown }}
 }
 
diff --git a/clang/test/CXX/drs/cwg16xx.cpp b/clang/test/CXX/drs/cwg16xx.cpp
index cf6b45ceabf2..82ef871939d2 100644
--- a/clang/test/CXX/drs/cwg16xx.cpp
+++ b/clang/test/CXX/drs/cwg16xx.cpp
@@ -483,8 +483,6 @@ namespace cwg1696 { // cwg1696: 7
     const A &a = A(); // #cwg1696-D1-a
   };
   D1 d1 = {}; // #cwg1696-d1
-  // since-cxx14-warning@-1 {{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported; lifetime of temporary will end at the end of the full-expression}}
-  //   since-cxx14-note@#cwg1696-D1-a {{initializing field 'a' with default member initializer}}
 
   struct D2 {
     const A &a = A(); // #cwg1696-D2-a
diff --git a/clang/test/CXX/drs/cwg18xx.cpp b/clang/test/CXX/drs/cwg18xx.cpp
index ea322ff4c8be..9eb749153e57 100644
--- a/clang/test/CXX/drs/cwg18xx.cpp
+++ b/clang/test/CXX/drs/cwg18xx.cpp
@@ -206,19 +206,28 @@ namespace cwg1814 { // cwg1814: yes
 #endif
 }
 
-namespace cwg1815 { // cwg1815: no
+namespace cwg1815 { // cwg1815: yes
 #if __cplusplus >= 201402L
-  // FIXME: needs codegen test
-  struct A { int &&r = 0; }; // #cwg1815-A 
+  struct A { int &&r = 0; };
   A a = {};
-  // since-cxx14-warning@-1 {{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported; lifetime of temporary will end at the end of the full-expression}} FIXME
-  //   since-cxx14-note@#cwg1815-A {{initializing field 'r' with default member initializer}}
 
   struct B { int &&r = 0; }; // #cwg1815-B
   // since-cxx14-error@-1 {{reference member 'r' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
   //   since-cxx14-note@#cwg1815-B {{initializing field 'r' with default member initializer}}
   //   since-cxx14-note@#cwg1815-b {{in implicit default constructor for 'cwg1815::B' first required here}}
   B b; // #cwg1815-b
+
+#if __cplusplus >= 201703L
+  struct C { const int &r = 0; };
+  constexpr C c = {}; // OK, since cwg1815
+  static_assert(c.r == 0);
+
+  constexpr int f() {
+    A a = {}; // OK, since cwg1815
+    return a.r;
+  }
+  static_assert(f() == 0);
+#endif
 #endif
 }
 
diff --git a/clang/test/CXX/special/class.temporary/p6.cpp b/clang/test/CXX/special/class.temporary/p6.cpp
index 5554363cc69a..a6d2adfd1fd2 100644
--- a/clang/test/CXX/special/class.temporary/p6.cpp
+++ b/clang/test/CXX/special/class.temporary/p6.cpp
@@ -269,6 +269,40 @@ void init_capture_init_list() {
   // CHECK: }
 }
 
+void check_dr1815() { // dr1815: yes
+#if __cplusplus >= 201402L
+
+  struct A {
+    int &&r = 0;
+    ~A() {}
+  };
+
+  struct B {
+    A &&a = A{};
+    ~B() {}
+  };
+  B a = {};
+  
+  // CHECK: call {{.*}}block_scope_begin_function
+  extern void block_scope_begin_function();
+  extern void block_scope_end_function();
+  block_scope_begin_function();
+  {
+    // CHECK: call void @_ZZ12check_dr1815vEN1BD1Ev
+    // CHECK: call void @_ZZ12check_dr1815vEN1AD1Ev
+    B b = {};
+  }
+  // CHECK: call {{.*}}block_scope_end_function
+  block_scope_end_function();
+
+  // CHECK: call {{.*}}some_other_function
+  extern void some_other_function();
+  some_other_function();
+  // CHECK: call void @_ZZ12check_dr1815vEN1BD1Ev
+  // CHECK: call void @_ZZ12check_dr1815vEN1AD1Ev
+#endif
+}
+
 namespace P2718R0 {
 namespace basic {
 template <typename E> using T2 = std::list<E>;
diff --git a/clang/test/SemaCXX/constexpr-default-arg.cpp b/clang/test/SemaCXX/constexpr-default-arg.cpp
index ec9b2927880b..901123bfb359 100644
--- a/clang/test/SemaCXX/constexpr-default-arg.cpp
+++ b/clang/test/SemaCXX/constexpr-default-arg.cpp
@@ -32,8 +32,8 @@ void test_default_arg2() {
 }
 
 // Check that multiple CXXDefaultInitExprs don't cause an assertion failure.
-struct A { int &&r = 0; }; // expected-note 2{{default member initializer}}
+struct A { int &&r = 0; };
 struct B { A x, y; };
-B b = {}; // expected-warning 2{{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported}}
+B b = {}; // expected-no-diagnostics
 
 }
diff --git a/clang/test/SemaCXX/eval-crashes.cpp b/clang/test/SemaCXX/eval-crashes.cpp
index 017df977b26b..a06f60f71e9c 100644
--- a/clang/test/SemaCXX/eval-crashes.cpp
+++ b/clang/test/SemaCXX/eval-crashes.cpp
@@ -25,11 +25,9 @@ namespace pr33140_0b {
 }
 
 namespace pr33140_2 {
-  // FIXME: The declaration of 'b' below should lifetime-extend two int
-  // temporaries.
-  struct A { int &&r = 0; }; // expected-note 2{{initializing field 'r' with default member initializer}}
+  struct A { int &&r = 0; };
   struct B { A x, y; };
-  B b = {}; // expected-warning 2{{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported}}
+  B b = {};
 }
 
 namespace pr33140_3 {
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 12ab59224e82..92fdcf5556ed 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -10698,7 +10698,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1815.html">1815</a></td>
     <td>CD4</td>
     <td>Lifetime extension in aggregate initialization</td>
-    <td class="none" align="center">No</td>
+    <td class="unreleased" align="center">Clang 19</td>
   </tr>
   <tr id="1816">
     <td><a href="https://cplusplus.github.io/CWG/issues/1816.html">1816</a></td>
-- 
GitLab


From b7ed097f29d712b1cc839e15ab68d2c8a2ce07cc Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Sun, 12 May 2024 20:31:11 +0100
Subject: [PATCH 0502/1206] [VectorCombine] Add intrinsics handling to
 shuffleToIdentity (#91000)

This is probably the most involved addition, as it tries to make use of
isTriviallyVectorizable with isVectorIntrinsicWithScalarOpAtArg to handle a
number of different intrinsics that are all lane-wise. Additional tests have
been added for some of the different intrinsics from
isVectorIntrinsicWithScalarOpAtArg / isVectorIntrinsicWithOverloadTypeAtArg.
---
 .../Transforms/Vectorize/VectorCombine.cpp    | 33 +++++++++-
 .../AArch64/shuffletoidentity.ll              | 62 ++++---------------
 2 files changed, 42 insertions(+), 53 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 8573a8adf53b..9d43fb4ab607 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -1729,7 +1729,9 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
       return false;
 
     // Look for an identity value.
-    if (Item[0].second == 0 && Item[0].first->getType() == Ty &&
+    if (Item[0].second == 0 &&
+        cast<FixedVectorType>(Item[0].first->getType())->getNumElements() ==
+            Ty->getNumElements() &&
         all_of(drop_begin(enumerate(Item)), [&](const auto &E) {
           return !E.value().first || (E.value().first == Item[0].first &&
                                       E.value().second == (int)E.index());
@@ -1773,6 +1775,20 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
       Worklist.push_back(GenerateInstLaneVectorFromOperand(Item, 1));
     } else if (isa<UnaryOperator>(Item[0].first)) {
       Worklist.push_back(GenerateInstLaneVectorFromOperand(Item, 0));
+    } else if (auto *II = dyn_cast<IntrinsicInst>(Item[0].first);
+               II && isTriviallyVectorizable(II->getIntrinsicID())) {
+      for (unsigned Op = 0, E = II->getNumOperands() - 1; Op < E; Op++) {
+        if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(), Op)) {
+          if (!all_of(drop_begin(Item), [&](InstLane &IL) {
+                return !IL.first ||
+                       (cast<Instruction>(IL.first)->getOperand(Op) ==
+                        cast<Instruction>(Item[0].first)->getOperand(Op));
+              }))
+            return false;
+          continue;
+        }
+        Worklist.push_back(GenerateInstLaneVectorFromOperand(Item, Op));
+      }
     } else {
       return false;
     }
@@ -1799,13 +1815,24 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
     }
 
     auto *I = cast<Instruction>(Item[0].first);
-    SmallVector<Value *> Ops(I->getNumOperands());
-    for (unsigned Idx = 0, E = I->getNumOperands(); Idx < E; Idx++)
+    auto *II = dyn_cast<IntrinsicInst>(I);
+    unsigned NumOps = I->getNumOperands() - (II ? 1 : 0);
+    SmallVector<Value *> Ops(NumOps);
+    for (unsigned Idx = 0; Idx < NumOps; Idx++) {
+      if (II && isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(), Idx)) {
+        Ops[Idx] = II->getOperand(Idx);
+        continue;
+      }
       Ops[Idx] = Generate(GenerateInstLaneVectorFromOperand(Item, Idx));
+    }
     Builder.SetInsertPoint(I);
+    Type *DstTy = FixedVectorType::get(I->getType()->getScalarType(),
+                                       Ty->getNumElements());
     if (auto BI = dyn_cast<BinaryOperator>(I))
       return Builder.CreateBinOp((Instruction::BinaryOps)BI->getOpcode(),
                                  Ops[0], Ops[1]);
+    if (II)
+      return Builder.CreateIntrinsic(DstTy, II->getIntrinsicID(), Ops);
     assert(isa<UnaryInstruction>(I) &&
            "Unexpected instruction type in Generate");
     return Builder.CreateUnOp((Instruction::UnaryOps)I->getOpcode(), Ops[0]);
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
index 0fb634fce135..b58f92d70936 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
@@ -102,11 +102,7 @@ define <8 x half> @fneg(<8 x half> %a, <8 x half> %b) {
 
 define <8 x i8> @abs(<8 x i8> %a) {
 ; CHECK-LABEL: @abs(
-; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
-; CHECK-NEXT:    [[ABT:%.*]] = call <4 x i8> @llvm.abs.v4i8(<4 x i8> [[AT]], i1 false)
-; CHECK-NEXT:    [[ABB:%.*]] = call <4 x i8> @llvm.abs.v4i8(<4 x i8> [[AB]], i1 false)
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i8> @llvm.abs.v8i8(<8 x i8> [[A:%.*]], i1 false)
 ; CHECK-NEXT:    ret <8 x i8> [[R]]
 ;
   %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -119,11 +115,7 @@ define <8 x i8> @abs(<8 x i8> %a) {
 
 define <8 x half> @powi(<8 x half> %a) {
 ; CHECK-LABEL: @powi(
-; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x half> [[A:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x half> [[A]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
-; CHECK-NEXT:    [[ABT:%.*]] = call <4 x half> @llvm.powi.v4f16.i32(<4 x half> [[AT]], i32 10)
-; CHECK-NEXT:    [[ABB:%.*]] = call <4 x half> @llvm.powi.v4f16.i32(<4 x half> [[AB]], i32 10)
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x half> [[ABT]], <4 x half> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = call <8 x half> @llvm.powi.v8f16.i32(<8 x half> [[A:%.*]], i32 10)
 ; CHECK-NEXT:    ret <8 x half> [[R]]
 ;
   %ab = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -171,11 +163,7 @@ declare <4 x half> @othercall(<4 x half>)
 
 define <8 x i32> @lrint(<8 x half> %a) {
 ; CHECK-LABEL: @lrint(
-; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x half> [[A:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x half> [[A]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
-; CHECK-NEXT:    [[ABT:%.*]] = call <4 x i32> @llvm.lrint.v4i32.v4f16(<4 x half> [[AT]])
-; CHECK-NEXT:    [[ABB:%.*]] = call <4 x i32> @llvm.lrint.v4i32.v4f16(<4 x half> [[AB]])
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[ABT]], <4 x i32> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i32> @llvm.lrint.v8i32.v8f16(<8 x half> [[A:%.*]])
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
 ;
   %ab = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -427,15 +415,7 @@ define <8 x i8> @icmpsel(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
 
 define <8 x half> @fma(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; CHECK-LABEL: @fma(
-; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x half> [[A:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x half> [[A]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
-; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x half> [[B:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x half> [[B]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
-; CHECK-NEXT:    [[CB:%.*]] = shufflevector <8 x half> [[C:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[CT:%.*]] = shufflevector <8 x half> [[C]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
-; CHECK-NEXT:    [[ABB:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[AB]], <4 x half> [[BB]], <4 x half> [[CB]])
-; CHECK-NEXT:    [[ABT:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[AT]], <4 x half> [[BT]], <4 x half> [[CT]])
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x half> [[ABT]], <4 x half> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], <8 x half> [[C:%.*]])
 ; CHECK-NEXT:    ret <8 x half> [[R]]
 ;
   %ab = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -487,19 +467,10 @@ define void @exttrunc(<8 x i32> %a, <8 x i32> %b, ptr %p) {
 
 define <8 x i8> @intrinsics_minmax(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: @intrinsics_minmax(
-; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
-; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
-; CHECK-NEXT:    [[ABT:%.*]] = call <4 x i8> @llvm.smin.v4i8(<4 x i8> [[AT]], <4 x i8> [[BT]])
-; CHECK-NEXT:    [[ABB:%.*]] = call <4 x i8> @llvm.smin.v4i8(<4 x i8> [[AB]], <4 x i8> [[BB]])
-; CHECK-NEXT:    [[ABT1:%.*]] = call <4 x i8> @llvm.smax.v4i8(<4 x i8> [[ABT]], <4 x i8> [[BT]])
-; CHECK-NEXT:    [[ABB1:%.*]] = call <4 x i8> @llvm.smax.v4i8(<4 x i8> [[ABB]], <4 x i8> [[BB]])
-; CHECK-NEXT:    [[ABT2:%.*]] = call <4 x i8> @llvm.umin.v4i8(<4 x i8> [[ABT1]], <4 x i8> [[BT]])
-; CHECK-NEXT:    [[ABB2:%.*]] = call <4 x i8> @llvm.umin.v4i8(<4 x i8> [[ABB1]], <4 x i8> [[BB]])
-; CHECK-NEXT:    [[ABT3:%.*]] = call <4 x i8> @llvm.umax.v4i8(<4 x i8> [[ABT2]], <4 x i8> [[BT]])
-; CHECK-NEXT:    [[ABB3:%.*]] = call <4 x i8> @llvm.umax.v4i8(<4 x i8> [[ABB2]], <4 x i8> [[BB]])
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT3]], <4 x i8> [[ABB3]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i8> @llvm.smin.v8i8(<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i8> @llvm.smax.v8i8(<8 x i8> [[TMP1]], <8 x i8> [[B]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.umin.v8i8(<8 x i8> [[TMP2]], <8 x i8> [[B]])
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i8> @llvm.umax.v8i8(<8 x i8> [[TMP3]], <8 x i8> [[B]])
 ; CHECK-NEXT:    ret <8 x i8> [[R]]
 ;
   %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -520,19 +491,10 @@ define <8 x i8> @intrinsics_minmax(<8 x i8> %a, <8 x i8> %b) {
 
 define <8 x i8> @intrinsics_addsat(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: @intrinsics_addsat(
-; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
-; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
-; CHECK-NEXT:    [[ABT:%.*]] = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> [[AT]], <4 x i8> [[BT]])
-; CHECK-NEXT:    [[ABB:%.*]] = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> [[AB]], <4 x i8> [[BB]])
-; CHECK-NEXT:    [[ABT1:%.*]] = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> [[ABT]], <4 x i8> [[BT]])
-; CHECK-NEXT:    [[ABB1:%.*]] = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> [[ABB]], <4 x i8> [[BB]])
-; CHECK-NEXT:    [[ABT2:%.*]] = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> [[ABT1]], <4 x i8> [[BT]])
-; CHECK-NEXT:    [[ABB2:%.*]] = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> [[ABB1]], <4 x i8> [[BB]])
-; CHECK-NEXT:    [[ABT3:%.*]] = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> [[ABT2]], <4 x i8> [[BT]])
-; CHECK-NEXT:    [[ABB3:%.*]] = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> [[ABB2]], <4 x i8> [[BB]])
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT3]], <4 x i8> [[ABB3]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> [[TMP1]], <8 x i8> [[B]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> [[TMP2]], <8 x i8> [[B]])
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> [[TMP3]], <8 x i8> [[B]])
 ; CHECK-NEXT:    ret <8 x i8> [[R]]
 ;
   %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-- 
GitLab


From b5af667b01458e9083256f2614df175916c73e5a Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Sun, 12 May 2024 17:11:09 -0700
Subject: [PATCH 0503/1206] [BOLT] Map branch source address to the containing
 basic block in BAT YAML

Fix an issue where the profile for all branches that have a BRANCHENTRY
is dropped. If the branch has an entry in BAT, it will be translated to
its input offset. We used to only permit the basic block offset as a
branch source. Perform a lookup of containing basic block instead.

Test Plan: Updated bolt-address-translation-yaml.test

Reviewers: maksfb, dcci, rafaelauler, ayermolo

Reviewed By: maksfb

Pull Request: https://github.com/llvm/llvm-project/pull/91273
---
 bolt/lib/Profile/DataAggregator.cpp           | 21 ++++++++++++-------
 .../blarge_new_bat_branchentry.preagg.txt     |  1 +
 .../X86/bolt-address-translation-yaml.test    | 11 ++++++++++
 3 files changed, 26 insertions(+), 7 deletions(-)
 create mode 100644 bolt/test/X86/Inputs/blarge_new_bat_branchentry.preagg.txt

diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 2f6380e186e1..9a71e227f233 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -23,6 +23,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
@@ -2378,10 +2379,19 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
         return CSI;
       };
 
+      // Lookup containing basic block offset and index
+      auto getBlock = [&BlockMap](uint32_t Offset) {
+        auto BlockIt = BlockMap.upper_bound(Offset);
+        if (LLVM_UNLIKELY(BlockIt == BlockMap.begin())) {
+          errs() << "BOLT-ERROR: invalid BAT section\n";
+          exit(1);
+        }
+        --BlockIt;
+        return std::pair(BlockIt->first, BlockIt->second.getBBIndex());
+      };
+
       for (const auto &[FromOffset, SuccKV] : Branches.IntraIndex) {
-        if (!BlockMap.isInputBlock(FromOffset))
-          continue;
-        const unsigned Index = BlockMap.getBBIndex(FromOffset);
+        const auto &[_, Index] = getBlock(FromOffset);
         yaml::bolt::BinaryBasicBlockProfile &YamlBB = YamlBF.Blocks[Index];
         for (const auto &[SuccOffset, SuccDataIdx] : SuccKV)
           if (BlockMap.isInputBlock(SuccOffset))
@@ -2389,10 +2399,7 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
                 getSuccessorInfo(SuccOffset, SuccDataIdx));
       }
       for (const auto &[FromOffset, CallTo] : Branches.InterIndex) {
-        auto BlockIt = BlockMap.upper_bound(FromOffset);
-        --BlockIt;
-        const unsigned BlockOffset = BlockIt->first;
-        const unsigned BlockIndex = BlockIt->second.getBBIndex();
+        const auto &[BlockOffset, BlockIndex] = getBlock(FromOffset);
         yaml::bolt::BinaryBasicBlockProfile &YamlBB = YamlBF.Blocks[BlockIndex];
         const uint32_t Offset = FromOffset - BlockOffset;
         for (const auto &[CallToLoc, CallToIdx] : CallTo)
diff --git a/bolt/test/X86/Inputs/blarge_new_bat_branchentry.preagg.txt b/bolt/test/X86/Inputs/blarge_new_bat_branchentry.preagg.txt
new file mode 100644
index 000000000000..546da92f94db
--- /dev/null
+++ b/bolt/test/X86/Inputs/blarge_new_bat_branchentry.preagg.txt
@@ -0,0 +1 @@
+B 80010c 800194 1 0
diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test
index 38a50f6cacaf..c15d6ce15ed0 100644
--- a/bolt/test/X86/bolt-address-translation-yaml.test
+++ b/bolt/test/X86/bolt-address-translation-yaml.test
@@ -5,6 +5,17 @@ RUN: llvm-bolt %t.exe -o %t.out --pa -p %p/Inputs/blarge_new.preagg.txt \
 RUN:   --reorder-blocks=ext-tsp --split-functions --split-strategy=cdsplit \
 RUN:   --reorder-functions=cdsort --enable-bat --dyno-stats --skip-funcs=main \
 RUN:   2>&1 | FileCheck --check-prefix WRITE-BAT-CHECK %s
+# Check that branch with entry in BAT is accounted for.
+RUN: perf2bolt %t.out --pa -p %p/Inputs/blarge_new_bat_branchentry.preagg.txt \
+RUN:   -w %t.yaml -o %t.fdata
+RUN: llvm-bolt %t.exe -data %t.fdata -w %t.yaml-fdata -o %t.null
+RUN: FileCheck --input-file %t.yaml --check-prefix BRANCHENTRY-YAML-CHECK %s
+RUN: FileCheck --input-file %t.yaml-fdata --check-prefix BRANCHENTRY-YAML-CHECK %s
+BRANCHENTRY-YAML-CHECK:    - name: SolveCubic
+BRANCHENTRY-YAML-CHECK:      bid: 0
+BRANCHENTRY-YAML-CHECK:      hash: 0x700F19D24600000
+BRANCHENTRY-YAML-CHECK-NEXT: succ: [ { bid: 7, cnt: 1 }
+# Large profile test
 RUN: perf2bolt %t.out --pa -p %p/Inputs/blarge_new_bat.preagg.txt -w %t.yaml -o %t.fdata \
 RUN:   2>&1 | FileCheck --check-prefix READ-BAT-CHECK %s
 RUN: FileCheck --input-file %t.yaml --check-prefix YAML-BAT-CHECK %s
-- 
GitLab


From 5bde8017a1109128d011510dcf4ba79140a224fe Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Mon, 13 May 2024 08:31:49 +0800
Subject: [PATCH 0504/1206] [X86][vectorcall] Pass built types byval when
 xmm0~6 exhausted (#91846)

This is how MSVC handles it. https://godbolt.org/z/fG386bjnf
---
 clang/lib/CodeGen/Targets/X86.cpp |  2 ++
 clang/test/CodeGen/vectorcall.c   | 16 ++++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 717a27fc9c57..29d98aad8fcb 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -792,6 +792,8 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State,
         return ABIArgInfo::getDirect();
       return ABIArgInfo::getExpand();
     }
+    if (IsVectorCall && Ty->isBuiltinType())
+      return ABIArgInfo::getDirect();
     return getIndirectResult(Ty, /*ByVal=*/false, State);
   }
 
diff --git a/clang/test/CodeGen/vectorcall.c b/clang/test/CodeGen/vectorcall.c
index cb53ecc70351..71dc3b0b9585 100644
--- a/clang/test/CodeGen/vectorcall.c
+++ b/clang/test/CodeGen/vectorcall.c
@@ -140,4 +140,20 @@ void __vectorcall vectorcall_indirect_vec(
 // X86-SAME: ptr inreg noundef %0,
 // X86-SAME: i32 inreg noundef %edx,
 // X86-SAME: ptr noundef %1)
+
+void __vectorcall vectorcall_indirect_fp(
+    double xmm0, double xmm1, double xmm2, double xmm3, double xmm4,
+    v4f32 xmm5, v4f32 ecx, int edx, double mem) {
+}
+
+// X86: define dso_local x86_vectorcallcc void @"\01vectorcall_indirect_fp@@{{[0-9]+}}"
+// X86-SAME: (double inreg noundef %xmm0,
+// X86-SAME: double inreg noundef %xmm1,
+// X86-SAME: double inreg noundef %xmm2,
+// X86-SAME: double inreg noundef %xmm3,
+// X86-SAME: double inreg noundef %xmm4,
+// X86-SAME: <4 x float> inreg noundef %xmm5,
+// X86-SAME: ptr inreg noundef %0,
+// X86-SAME: i32 inreg noundef %edx,
+// X86-SAME: double noundef %mem)
 #endif
-- 
GitLab


From 626025ac7796b70cde9fc0fd4f688c3441949d04 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Sun, 12 May 2024 17:23:19 -0700
Subject: [PATCH 0505/1206] Revert "[clang-format] Fix buildbot failures"

This reverts commit 0869204cff22831d0bb19a82c99bf85e4deb4ae3, which caused a
buildbot failure:
https://lab.llvm.org/buildbot/#/builders/5/builds/43322
---
 clang/lib/Format/QualifierAlignmentFixer.cpp  | 18 ++--
 clang/lib/Format/QualifierAlignmentFixer.h    |  4 +-
 clang/unittests/Format/QualifierFixerTest.cpp | 92 +++++++++++--------
 3 files changed, 62 insertions(+), 52 deletions(-)

diff --git a/clang/lib/Format/QualifierAlignmentFixer.cpp b/clang/lib/Format/QualifierAlignmentFixer.cpp
index 077ce5e597a2..a904f0b773c6 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.cpp
+++ b/clang/lib/Format/QualifierAlignmentFixer.cpp
@@ -281,7 +281,7 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeRight(
 
     const FormatToken *LastSimpleTypeSpecifier = TypeToken;
     while (isQualifierOrType(LastSimpleTypeSpecifier->getNextNonComment(),
-                             &LangOpts)) {
+                             LangOpts)) {
       LastSimpleTypeSpecifier = LastSimpleTypeSpecifier->getNextNonComment();
     }
 
@@ -414,7 +414,7 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeLeft(
     const FormatToken *LastSimpleTypeSpecifier = TypeToken;
     while (isConfiguredQualifierOrType(
         LastSimpleTypeSpecifier->getPreviousNonComment(),
-        ConfiguredQualifierTokens, &LangOpts)) {
+        ConfiguredQualifierTokens, LangOpts)) {
       LastSimpleTypeSpecifier =
           LastSimpleTypeSpecifier->getPreviousNonComment();
     }
@@ -613,18 +613,16 @@ void prepareLeftRightOrderingForQualifierAlignmentFixer(
 }
 
 bool LeftRightQualifierAlignmentFixer::isQualifierOrType(
-    const FormatToken *Tok, const LangOptions *LangOpts) {
-  return Tok &&
-         (Tok->isTypeName(LangOpts ? *LangOpts : getFormattingLangOpts()) ||
-          Tok->is(tok::kw_auto) || isQualifier(Tok));
+    const FormatToken *Tok, const LangOptions &LangOpts) {
+  return Tok && (Tok->isTypeName(LangOpts) || Tok->is(tok::kw_auto) ||
+                 isQualifier(Tok));
 }
 
 bool LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
     const FormatToken *Tok, const std::vector<tok::TokenKind> &Qualifiers,
-    const LangOptions *LangOpts) {
-  return Tok &&
-         (Tok->isTypeName(LangOpts ? *LangOpts : getFormattingLangOpts()) ||
-          Tok->is(tok::kw_auto) || isConfiguredQualifier(Tok, Qualifiers));
+    const LangOptions &LangOpts) {
+  return Tok && (Tok->isTypeName(LangOpts) || Tok->is(tok::kw_auto) ||
+                 isConfiguredQualifier(Tok, Qualifiers));
 }
 
 // If a token is an identifier and it's upper case, it could
diff --git a/clang/lib/Format/QualifierAlignmentFixer.h b/clang/lib/Format/QualifierAlignmentFixer.h
index 97fcb42e4b2e..710fa2dc0030 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.h
+++ b/clang/lib/Format/QualifierAlignmentFixer.h
@@ -72,11 +72,11 @@ public:
 
   // Is the Token a simple or qualifier type
   static bool isQualifierOrType(const FormatToken *Tok,
-                                const LangOptions *LangOpts = nullptr);
+                                const LangOptions &LangOpts);
   static bool
   isConfiguredQualifierOrType(const FormatToken *Tok,
                               const std::vector<tok::TokenKind> &Qualifiers,
-                              const LangOptions *LangOpts = nullptr);
+                              const LangOptions &LangOpts);
 
   // Is the Token likely a Macro
   static bool isPossibleMacro(const FormatToken *Tok);
diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index 792d8f3c3a98..5463bfbb65ca 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -1055,70 +1055,82 @@ TEST_F(QualifierFixerTest, IsQualifierType) {
   ConfiguredTokens.push_back(tok::kw_constexpr);
   ConfiguredTokens.push_back(tok::kw_friend);
 
+  LangOptions LangOpts{getFormattingLangOpts()};
+
   auto Tokens = annotate(
       "const static inline auto restrict int double long constexpr friend");
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
 
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[0], ConfiguredTokens));
+      Tokens[0], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[1], ConfiguredTokens));
+      Tokens[1], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[2], ConfiguredTokens));
+      Tokens[2], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[3], ConfiguredTokens));
+      Tokens[3], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[4], ConfiguredTokens));
+      Tokens[4], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[5], ConfiguredTokens));
+      Tokens[5], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[6], ConfiguredTokens));
+      Tokens[6], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[7], ConfiguredTokens));
+      Tokens[7], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[8], ConfiguredTokens));
+      Tokens[8], ConfiguredTokens, LangOpts));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[9], ConfiguredTokens));
-
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[0]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[1]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[2]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[3]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[4]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[5]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[6]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[7]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[8]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[9]));
+      Tokens[9], ConfiguredTokens, LangOpts));
+
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[0], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[1], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[2], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[3], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[4], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[5], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[6], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[7], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[8], LangOpts));
+  EXPECT_TRUE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[9], LangOpts));
 
   auto NotTokens = annotate("for while do Foo Bar ");
   ASSERT_EQ(NotTokens.size(), 6u) << Tokens;
 
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[0], ConfiguredTokens));
+      NotTokens[0], ConfiguredTokens, LangOpts));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[1], ConfiguredTokens));
+      NotTokens[1], ConfiguredTokens, LangOpts));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[2], ConfiguredTokens));
+      NotTokens[2], ConfiguredTokens, LangOpts));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[3], ConfiguredTokens));
+      NotTokens[3], ConfiguredTokens, LangOpts));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[4], ConfiguredTokens));
+      NotTokens[4], ConfiguredTokens, LangOpts));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[5], ConfiguredTokens));
-
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[0]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[1]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[2]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[3]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[4]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[5]));
+      NotTokens[5], ConfiguredTokens, LangOpts));
+
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[0],
+                                                                   LangOpts));
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[1],
+                                                                   LangOpts));
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[2],
+                                                                   LangOpts));
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[3],
+                                                                   LangOpts));
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[4],
+                                                                   LangOpts));
+  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[5],
+                                                                   LangOpts));
 }
 
 TEST_F(QualifierFixerTest, IsMacro) {
-- 
GitLab


From ed16e7aac44f2024b45d8c6c9dc2817d77d0ea97 Mon Sep 17 00:00:00 2001
From: Shih-Po Hung <shihpo.hung@sifive.com>
Date: Mon, 13 May 2024 09:47:57 +0800
Subject: [PATCH 0506/1206] [RISCV][TTI] Support fdiv/udiv/sdiv/srem/urem in
 getArithmeticInstrCost (#89170)

This patch made following changes:
1. Support ISD FDIV/UDIV/SDIV/UREM/SREM
2. Classify instructions which cost the same
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  47 +-
 .../test/Analysis/CostModel/RISCV/arith-fp.ll |  48 +-
 .../Analysis/CostModel/RISCV/arith-int.ll     | 828 ++++++++++++++----
 3 files changed, 710 insertions(+), 213 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 5f84175da703..984638c1e50d 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1619,29 +1619,58 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
   if (Op2Info.isConstant())
     ConstantMatCost += getConstantMatCost(1, Op2Info);
 
+  unsigned Op;
   switch (TLI->InstructionOpcodeToISD(Opcode)) {
   case ISD::ADD:
   case ISD::SUB:
-  case ISD::AND:
-  case ISD::OR:
-  case ISD::XOR:
+    Op = RISCV::VADD_VV;
+    break;
   case ISD::SHL:
   case ISD::SRL:
   case ISD::SRA:
+    Op = RISCV::VSLL_VV;
+    break;
+  case ISD::AND:
+  case ISD::OR:
+  case ISD::XOR:
+    Op = (Ty->getScalarSizeInBits() == 1) ? RISCV::VMAND_MM : RISCV::VAND_VV;
+    break;
   case ISD::MUL:
   case ISD::MULHS:
   case ISD::MULHU:
+    Op = RISCV::VMUL_VV;
+    break;
+  case ISD::SDIV:
+  case ISD::UDIV:
+    Op = RISCV::VDIV_VV;
+    break;
+  case ISD::SREM:
+  case ISD::UREM:
+    Op = RISCV::VREM_VV;
+    break;
   case ISD::FADD:
   case ISD::FSUB:
+    // TODO: Address FP16 with VFHMIN
+    Op = RISCV::VFADD_VV;
   case ISD::FMUL:
-  case ISD::FNEG: {
-    return ConstantMatCost + TLI->getLMULCost(LT.second) * LT.first * 1;
-  }
+    // TODO: Address FP16 with VFHMIN
+    Op = RISCV::VFMUL_VV;
+    break;
+  case ISD::FDIV:
+    Op = RISCV::VFDIV_VV;
+    break;
+  case ISD::FNEG:
+    Op = RISCV::VFSGNJN_VV;
+    break;
   default:
-    return ConstantMatCost +
-           BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CxtI);
+    // Assuming all other instructions have the same cost until a need arises to
+    // differentiate them.
+    return ConstantMatCost + BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind,
+                                                           Op1Info, Op2Info,
+                                                           Args, CxtI);
   }
+  return ConstantMatCost +
+         LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);
 }
 
 // TODO: Deduplicate from TargetTransformInfoImplCRTPBase.
diff --git a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
index 1dde88f366a3..d1e8bb015491 100644
--- a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
@@ -248,36 +248,36 @@ define i32 @fdiv() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fdiv half undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fdiv float undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fdiv double undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1F16 = fdiv <1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F16 = fdiv <2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F16 = fdiv <4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F16 = fdiv <8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = fdiv <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F16 = fdiv <1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fdiv <2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fdiv <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fdiv <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fdiv <16 x half> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32F16 = fdiv <32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F16 = fdiv <vscale x 1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F16 = fdiv <vscale x 2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F16 = fdiv <vscale x 4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F16 = fdiv <vscale x 1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F16 = fdiv <vscale x 2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F16 = fdiv <vscale x 4 x half> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F16 = fdiv <vscale x 8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV16F16 = fdiv <vscale x 16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV32F16 = fdiv <vscale x 32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1F32 = fdiv <1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = fdiv <2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F32 = fdiv <4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fdiv <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16F16 = fdiv <vscale x 16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32F16 = fdiv <vscale x 32 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F32 = fdiv <1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fdiv <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fdiv <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fdiv <8 x float> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fdiv <16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F32 = fdiv <vscale x 1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F32 = fdiv <vscale x 2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F32 = fdiv <vscale x 1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F32 = fdiv <vscale x 2 x float> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F32 = fdiv <vscale x 4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F32 = fdiv <vscale x 8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV16F32 = fdiv <vscale x 16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1F64 = fdiv <1 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = fdiv <2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fdiv <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F32 = fdiv <vscale x 8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F32 = fdiv <vscale x 16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = fdiv <1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fdiv <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fdiv <4 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fdiv <8 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F64 = fdiv <vscale x 1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F64 = fdiv <vscale x 1 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F64 = fdiv <vscale x 2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F64 = fdiv <vscale x 4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F64 = fdiv <vscale x 8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F64 = fdiv <vscale x 4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F64 = fdiv <vscale x 8 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %F16 = fdiv half undef, undef
diff --git a/llvm/test/Analysis/CostModel/RISCV/arith-int.ll b/llvm/test/Analysis/CostModel/RISCV/arith-int.ll
index b4afbb513166..c976f483fdfe 100644
--- a/llvm/test/Analysis/CostModel/RISCV/arith-int.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/arith-int.ll
@@ -705,72 +705,72 @@ define i32 @udiv() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = udiv i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = udiv i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = udiv i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = udiv <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = udiv <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = udiv <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = udiv <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = udiv <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = udiv <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = udiv <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = udiv <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = udiv <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = udiv <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = udiv <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = udiv <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = udiv <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = udiv <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = udiv <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = udiv <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = udiv <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = udiv <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = udiv <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = udiv <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = udiv <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = udiv <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = udiv <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = udiv <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = udiv <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = udiv <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = udiv <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = udiv <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = udiv <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = udiv <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = udiv <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = udiv <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = udiv <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'udiv'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = udiv i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = udiv i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = udiv i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = udiv <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = udiv <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = udiv <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = udiv <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = udiv <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = udiv <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = udiv <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = udiv <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = udiv <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = udiv <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = udiv <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = udiv <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = udiv <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = udiv <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = udiv <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = udiv <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = udiv <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = udiv <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = udiv <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = udiv <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = udiv <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = udiv <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = udiv <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = udiv <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = udiv <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = udiv <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = udiv <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = udiv <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = udiv <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = udiv <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = udiv <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = udiv <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = udiv <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = udiv <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = udiv <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = udiv <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = udiv i16 undef, undef
@@ -821,72 +821,72 @@ define i32 @urem() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = urem i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = urem i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = urem i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = urem <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = urem <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = urem <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = urem <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = urem <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = urem <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = urem <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = urem <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = urem <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = urem <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = urem <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = urem <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = urem <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = urem <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = urem <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = urem <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = urem <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = urem <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = urem <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = urem <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = urem <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = urem <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = urem <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = urem <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = urem <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = urem <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = urem <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = urem <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = urem <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = urem <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = urem <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = urem <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = urem <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'urem'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = urem i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = urem i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = urem i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = urem <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = urem <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = urem <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = urem <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = urem <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = urem <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = urem <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = urem <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = urem <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = urem <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = urem <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = urem <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = urem <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = urem <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = urem <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = urem <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = urem <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = urem <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = urem <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = urem <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = urem <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = urem <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = urem <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = urem <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = urem <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = urem <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = urem <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = urem <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = urem <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = urem <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = urem <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = urem <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = urem <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = urem <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = urem <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = urem <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = urem i16 undef, undef
@@ -937,72 +937,72 @@ define i32 @sdiv() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sdiv i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sdiv i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sdiv i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = sdiv <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = sdiv <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = sdiv <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = sdiv <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = sdiv <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = sdiv <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = sdiv <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = sdiv <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = sdiv <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = sdiv <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = sdiv <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = sdiv <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = sdiv <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = sdiv <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = sdiv <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'sdiv'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sdiv i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sdiv i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sdiv i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = sdiv <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = sdiv <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = sdiv <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = sdiv <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = sdiv <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = sdiv <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = sdiv <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = sdiv <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = sdiv <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = sdiv <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = sdiv <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = sdiv <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = sdiv <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = sdiv <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = sdiv <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = sdiv <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = sdiv <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = sdiv <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = sdiv i16 undef, undef
@@ -1053,72 +1053,72 @@ define i32 @srem() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = srem i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = srem i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = srem i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = srem <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = srem <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = srem <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = srem <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = srem <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = srem <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = srem <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = srem <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = srem <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = srem <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = srem <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = srem <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = srem <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = srem <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = srem <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = srem <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = srem <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = srem <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = srem <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = srem <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = srem <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = srem <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = srem <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = srem <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = srem <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = srem <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = srem <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = srem <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = srem <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = srem <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = srem <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = srem <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = srem <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'srem'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = srem i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = srem i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = srem i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = srem <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = srem <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = srem <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = srem <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = srem <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = srem <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = srem <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = srem <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = srem <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = srem <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = srem <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = srem <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = srem <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = srem <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = srem <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = srem <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = srem <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = srem <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = srem <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = srem <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = srem <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = srem <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = srem <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = srem <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = srem <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = srem <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = srem <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = srem <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = srem <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = srem <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = srem <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = srem <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = srem <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = srem <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = srem <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = srem <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = srem i16 undef, undef
@@ -1232,3 +1232,471 @@ define void @add_of_constant() {
 
   ret void
 }
+
+define i32 @and() {
+; CHECK-LABEL: 'and'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = and i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = and i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = and i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = and i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = and <1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = and <2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = and <4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = and <8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = and <16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = and <32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = and <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = and <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = and <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = and <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = and <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = and <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = and <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = and <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = and <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = and <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = and <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = and <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = and <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = and <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = and <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = and <vscale x 1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = and <vscale x 2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = and <vscale x 4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = and <vscale x 8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = and <vscale x 16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = and <vscale x 32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = and <vscale x 1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = and <vscale x 2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = and <vscale x 4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %35 = and <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = and <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %37 = and <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = and <vscale x 1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %39 = and <vscale x 2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %40 = and <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %41 = and <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %42 = and <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %43 = and <vscale x 1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %44 = and <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %45 = and <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %46 = and <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SIFIVE-X280-LABEL: 'and'
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = and i1 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = and i16 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = and i32 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = and i64 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = and <1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = and <2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = and <4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = and <8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = and <16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = and <32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = and <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = and <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = and <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = and <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = and <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %16 = and <32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = and <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = and <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = and <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = and <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = and <16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = and <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = and <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = and <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = and <8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = and <vscale x 1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = and <vscale x 2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = and <vscale x 4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = and <vscale x 8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = and <vscale x 16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = and <vscale x 32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = and <vscale x 1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = and <vscale x 2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %34 = and <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = and <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %36 = and <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %37 = and <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = and <vscale x 1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %39 = and <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %40 = and <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %41 = and <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %42 = and <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %43 = and <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %44 = and <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %45 = and <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %46 = and <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  and i1 undef, undef
+  and i16 undef, undef
+  and i32 undef, undef
+  and i64 undef, undef
+
+  and <1 x i1> undef, undef
+  and <2 x i1> undef, undef
+  and <4 x i1> undef, undef
+  and <8 x i1> undef, undef
+  and <16 x i1> undef, undef
+  and <32 x i1> undef, undef
+
+  and <1 x i16> undef, undef
+  and <2 x i16> undef, undef
+  and <4 x i16> undef, undef
+  and <8 x i16> undef, undef
+  and <16 x i16> undef, undef
+  and <32 x i16> undef, undef
+
+  and <1 x i32> undef, undef
+  and <2 x i32> undef, undef
+  and <4 x i32> undef, undef
+  and <8 x i32> undef, undef
+  and <16 x i32> undef, undef
+
+  and <1 x i64> undef, undef
+  and <2 x i64> undef, undef
+  and <4 x i64> undef, undef
+  and <8 x i64> undef, undef
+
+  and <vscale x 1 x i1> undef, undef
+  and <vscale x 2 x i1> undef, undef
+  and <vscale x 4 x i1> undef, undef
+  and <vscale x 8 x i1> undef, undef
+  and <vscale x 16 x i1> undef, undef
+  and <vscale x 32 x i1> undef, undef
+
+  and <vscale x 1 x i16> undef, undef
+  and <vscale x 2 x i16> undef, undef
+  and <vscale x 4 x i16> undef, undef
+  and <vscale x 8 x i16> undef, undef
+  and <vscale x 16 x i16> undef, undef
+  and <vscale x 32 x i16> undef, undef
+
+  and <vscale x 1 x i32> undef, undef
+  and <vscale x 2 x i32> undef, undef
+  and <vscale x 4 x i32> undef, undef
+  and <vscale x 8 x i32> undef, undef
+  and <vscale x 16 x i32> undef, undef
+
+  and <vscale x 1 x i64> undef, undef
+  and <vscale x 2 x i64> undef, undef
+  and <vscale x 4 x i64> undef, undef
+  and <vscale x 8 x i64> undef, undef
+  ret i32 undef
+}
+
+define i32 @or() {
+; CHECK-LABEL: 'or'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = or i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = or i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = or i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = or i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = or <1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = or <2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = or <4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = or <8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = or <16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = or <32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = or <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = or <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = or <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = or <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = or <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = or <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = or <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = or <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = or <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = or <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = or <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = or <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = or <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = or <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = or <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = or <vscale x 1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = or <vscale x 2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = or <vscale x 4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = or <vscale x 8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = or <vscale x 16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = or <vscale x 32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = or <vscale x 1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = or <vscale x 2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = or <vscale x 4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %35 = or <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = or <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %37 = or <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = or <vscale x 1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %39 = or <vscale x 2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %40 = or <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %41 = or <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %42 = or <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %43 = or <vscale x 1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %44 = or <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %45 = or <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %46 = or <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SIFIVE-X280-LABEL: 'or'
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = or i1 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = or i16 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = or i32 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = or i64 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = or <1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = or <2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = or <4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = or <8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = or <16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = or <32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = or <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = or <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = or <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = or <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = or <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %16 = or <32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = or <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = or <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = or <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = or <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = or <16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = or <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = or <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = or <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = or <8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = or <vscale x 1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = or <vscale x 2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = or <vscale x 4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = or <vscale x 8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = or <vscale x 16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = or <vscale x 32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = or <vscale x 1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = or <vscale x 2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %34 = or <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = or <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %36 = or <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %37 = or <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = or <vscale x 1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %39 = or <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %40 = or <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %41 = or <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %42 = or <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %43 = or <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %44 = or <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %45 = or <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %46 = or <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  or i1 undef, undef
+  or i16 undef, undef
+  or i32 undef, undef
+  or i64 undef, undef
+
+  or <1 x i1> undef, undef
+  or <2 x i1> undef, undef
+  or <4 x i1> undef, undef
+  or <8 x i1> undef, undef
+  or <16 x i1> undef, undef
+  or <32 x i1> undef, undef
+
+  or <1 x i16> undef, undef
+  or <2 x i16> undef, undef
+  or <4 x i16> undef, undef
+  or <8 x i16> undef, undef
+  or <16 x i16> undef, undef
+  or <32 x i16> undef, undef
+
+  or <1 x i32> undef, undef
+  or <2 x i32> undef, undef
+  or <4 x i32> undef, undef
+  or <8 x i32> undef, undef
+  or <16 x i32> undef, undef
+
+  or <1 x i64> undef, undef
+  or <2 x i64> undef, undef
+  or <4 x i64> undef, undef
+  or <8 x i64> undef, undef
+
+  or <vscale x 1 x i1> undef, undef
+  or <vscale x 2 x i1> undef, undef
+  or <vscale x 4 x i1> undef, undef
+  or <vscale x 8 x i1> undef, undef
+  or <vscale x 16 x i1> undef, undef
+  or <vscale x 32 x i1> undef, undef
+
+  or <vscale x 1 x i16> undef, undef
+  or <vscale x 2 x i16> undef, undef
+  or <vscale x 4 x i16> undef, undef
+  or <vscale x 8 x i16> undef, undef
+  or <vscale x 16 x i16> undef, undef
+  or <vscale x 32 x i16> undef, undef
+
+  or <vscale x 1 x i32> undef, undef
+  or <vscale x 2 x i32> undef, undef
+  or <vscale x 4 x i32> undef, undef
+  or <vscale x 8 x i32> undef, undef
+  or <vscale x 16 x i32> undef, undef
+
+  or <vscale x 1 x i64> undef, undef
+  or <vscale x 2 x i64> undef, undef
+  or <vscale x 4 x i64> undef, undef
+  or <vscale x 8 x i64> undef, undef
+  ret i32 undef
+}
+
+define i32 @xor() {
+; CHECK-LABEL: 'xor'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = xor i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = xor i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = xor i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = xor i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = xor <1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = xor <2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = xor <4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = xor <8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = xor <16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = xor <32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = xor <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = xor <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = xor <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = xor <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = xor <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = xor <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = xor <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = xor <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = xor <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = xor <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = xor <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = xor <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = xor <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = xor <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = xor <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = xor <vscale x 1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = xor <vscale x 2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = xor <vscale x 4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = xor <vscale x 8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = xor <vscale x 16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = xor <vscale x 32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = xor <vscale x 1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = xor <vscale x 2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = xor <vscale x 4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %35 = xor <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = xor <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %37 = xor <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = xor <vscale x 1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %39 = xor <vscale x 2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %40 = xor <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %41 = xor <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %42 = xor <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %43 = xor <vscale x 1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %44 = xor <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %45 = xor <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %46 = xor <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SIFIVE-X280-LABEL: 'xor'
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = xor i1 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = xor i16 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = xor i32 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = xor i64 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = xor <1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = xor <2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = xor <4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = xor <8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = xor <16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = xor <32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = xor <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = xor <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = xor <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = xor <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = xor <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %16 = xor <32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = xor <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = xor <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = xor <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = xor <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = xor <16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = xor <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = xor <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = xor <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = xor <8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = xor <vscale x 1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = xor <vscale x 2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = xor <vscale x 4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = xor <vscale x 8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = xor <vscale x 16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = xor <vscale x 32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = xor <vscale x 1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = xor <vscale x 2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %34 = xor <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = xor <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %36 = xor <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %37 = xor <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = xor <vscale x 1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %39 = xor <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %40 = xor <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %41 = xor <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %42 = xor <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %43 = xor <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %44 = xor <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %45 = xor <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %46 = xor <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  xor i1 undef, undef
+  xor i16 undef, undef
+  xor i32 undef, undef
+  xor i64 undef, undef
+
+  xor <1 x i1> undef, undef
+  xor <2 x i1> undef, undef
+  xor <4 x i1> undef, undef
+  xor <8 x i1> undef, undef
+  xor <16 x i1> undef, undef
+  xor <32 x i1> undef, undef
+
+  xor <1 x i16> undef, undef
+  xor <2 x i16> undef, undef
+  xor <4 x i16> undef, undef
+  xor <8 x i16> undef, undef
+  xor <16 x i16> undef, undef
+  xor <32 x i16> undef, undef
+
+  xor <1 x i32> undef, undef
+  xor <2 x i32> undef, undef
+  xor <4 x i32> undef, undef
+  xor <8 x i32> undef, undef
+  xor <16 x i32> undef, undef
+
+  xor <1 x i64> undef, undef
+  xor <2 x i64> undef, undef
+  xor <4 x i64> undef, undef
+  xor <8 x i64> undef, undef
+
+  xor <vscale x 1 x i1> undef, undef
+  xor <vscale x 2 x i1> undef, undef
+  xor <vscale x 4 x i1> undef, undef
+  xor <vscale x 8 x i1> undef, undef
+  xor <vscale x 16 x i1> undef, undef
+  xor <vscale x 32 x i1> undef, undef
+
+  xor <vscale x 1 x i16> undef, undef
+  xor <vscale x 2 x i16> undef, undef
+  xor <vscale x 4 x i16> undef, undef
+  xor <vscale x 8 x i16> undef, undef
+  xor <vscale x 16 x i16> undef, undef
+  xor <vscale x 32 x i16> undef, undef
+
+  xor <vscale x 1 x i32> undef, undef
+  xor <vscale x 2 x i32> undef, undef
+  xor <vscale x 4 x i32> undef, undef
+  xor <vscale x 8 x i32> undef, undef
+  xor <vscale x 16 x i32> undef, undef
+
+  xor <vscale x 1 x i64> undef, undef
+  xor <vscale x 2 x i64> undef, undef
+  xor <vscale x 4 x i64> undef, undef
+  xor <vscale x 8 x i64> undef, undef
+  ret i32 undef
+}
-- 
GitLab


From de641e289269061f8bdb138bb5c50e27ef4b354f Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Sun, 12 May 2024 18:53:05 -0700
Subject: [PATCH 0507/1206] [clang-format] Fix buildbot failures

Fix the following buildbot failures by making LangOpts in the unit test
static:
https://lab.llvm.org/buildbot/#/builders/236/builds/11223
https://lab.llvm.org/buildbot/#/builders/239/builds/6968
---
 clang/unittests/Format/QualifierFixerTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index 5463bfbb65ca..fdc392e5d948 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -1055,7 +1055,7 @@ TEST_F(QualifierFixerTest, IsQualifierType) {
   ConfiguredTokens.push_back(tok::kw_constexpr);
   ConfiguredTokens.push_back(tok::kw_friend);
 
-  LangOptions LangOpts{getFormattingLangOpts()};
+  static const LangOptions LangOpts{getFormattingLangOpts()};
 
   auto Tokens = annotate(
       "const static inline auto restrict int double long constexpr friend");
-- 
GitLab


From abe3c5ac19e455d8a6df3120fa5e7a6e5f9005a6 Mon Sep 17 00:00:00 2001
From: AdityaK <hiraditya@msn.com>
Date: Sun, 12 May 2024 19:41:54 -0700
Subject: [PATCH 0508/1206] [GVNSink] Fix non-determinisms by using a
 deterministic ordering (#90995)

GVNSink used to order instructions based on their pointer values and was
prone to non-determinism because of that.
This patch ensures all the values stored are using a deterministic
order. I have also added a verfier(`ModelledPHI::verifyModelledPHI`) to
assert when ordering isn't preserved.

Additionally, I have added a test case (mirror graph image of an
existing test) that would have failed before this patch.

Fixes: #77852
---
 llvm/lib/Transforms/Scalar/GVNSink.cpp        | 78 +++++++++++++++----
 .../test/Transforms/GVNSink/int_sideeffect.ll | 26 +++++++
 2 files changed, 91 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp
index d4907326eb0a..ddf01dc612bb 100644
--- a/llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -226,12 +226,22 @@ class ModelledPHI {
 public:
   ModelledPHI() = default;
 
-  ModelledPHI(const PHINode *PN) {
-    // BasicBlock comes first so we sort by basic block pointer order, then by value pointer order.
-    SmallVector<std::pair<BasicBlock *, Value *>, 4> Ops;
+  ModelledPHI(const PHINode *PN,
+              const DenseMap<const BasicBlock *, unsigned> &BlockOrder) {
+    // BasicBlock comes first so we sort by basic block pointer order,
+    // then by value pointer order. No need to call `verifyModelledPHI`
+    // As the Values and Blocks are populated in a deterministic order.
+    using OpsType = std::pair<BasicBlock *, Value *>;
+    SmallVector<OpsType, 4> Ops;
     for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I)
       Ops.push_back({PN->getIncomingBlock(I), PN->getIncomingValue(I)});
-    llvm::sort(Ops);
+
+    auto ComesBefore = [BlockOrder](OpsType O1, OpsType O2) {
+      return BlockOrder.lookup(O1.first) < BlockOrder.lookup(O2.first);
+    };
+    // Sort in a deterministic order.
+    llvm::sort(Ops, ComesBefore);
+
     for (auto &P : Ops) {
       Blocks.push_back(P.first);
       Values.push_back(P.second);
@@ -247,16 +257,38 @@ public:
     return M;
   }
 
+  void
+  verifyModelledPHI(const DenseMap<const BasicBlock *, unsigned> &BlockOrder) {
+    assert(Values.size() > 1 && Blocks.size() > 1 &&
+           "Modelling PHI with less than 2 values");
+    auto ComesBefore = [BlockOrder](const BasicBlock *BB1,
+                                    const BasicBlock *BB2) {
+      return BlockOrder.lookup(BB1) < BlockOrder.lookup(BB2);
+    };
+    assert(llvm::is_sorted(Blocks, ComesBefore));
+    int C = 0;
+    llvm::for_each(Values, [&C, this](const Value *V) {
+      if (!isa<UndefValue>(V)) {
+        const Instruction *I = cast<Instruction>(V);
+        assert(I->getParent() == this->Blocks[C]);
+      }
+      C++;
+    });
+  }
   /// Create a PHI from an array of incoming values and incoming blocks.
-  template <typename VArray, typename BArray>
-  ModelledPHI(const VArray &V, const BArray &B) {
+  ModelledPHI(SmallVectorImpl<Instruction *> &V,
+              SmallSetVector<BasicBlock *, 4> &B,
+              const DenseMap<const BasicBlock *, unsigned> &BlockOrder) {
+    // The order of Values and Blocks are already ordered by the caller.
     llvm::copy(V, std::back_inserter(Values));
     llvm::copy(B, std::back_inserter(Blocks));
+    verifyModelledPHI(BlockOrder);
   }
 
   /// Create a PHI from [I[OpNum] for I in Insts].
-  template <typename BArray>
-  ModelledPHI(ArrayRef<Instruction *> Insts, unsigned OpNum, const BArray &B) {
+  /// TODO: Figure out a way to verifyModelledPHI in this constructor.
+  ModelledPHI(ArrayRef<Instruction *> Insts, unsigned OpNum,
+              SmallSetVector<BasicBlock *, 4> &B) {
     llvm::copy(B, std::back_inserter(Blocks));
     for (auto *I : Insts)
       Values.push_back(I->getOperand(OpNum));
@@ -297,7 +329,8 @@ public:
 
   // Hash functor
   unsigned hash() const {
-      return (unsigned)hash_combine_range(Values.begin(), Values.end());
+    // Is deterministic because Values are saved in a specific order.
+    return (unsigned)hash_combine_range(Values.begin(), Values.end());
   }
 
   bool operator==(const ModelledPHI &Other) const {
@@ -566,7 +599,7 @@ public:
 
 class GVNSink {
 public:
-  GVNSink() = default;
+  GVNSink() {}
 
   bool run(Function &F) {
     LLVM_DEBUG(dbgs() << "GVNSink: running on function @" << F.getName()
@@ -575,6 +608,16 @@ public:
     unsigned NumSunk = 0;
     ReversePostOrderTraversal<Function*> RPOT(&F);
     VN.setReachableBBs(BasicBlocksSet(RPOT.begin(), RPOT.end()));
+    // Populate reverse post-order to order basic blocks in deterministic
+    // order. Any arbitrary ordering will work in this case as long as they are
+    // deterministic. The node ordering of newly created basic blocks
+    // are irrelevant because RPOT(for computing sinkable candidates) is also
+    // obtained ahead of time and only their order are relevant for this pass.
+    unsigned NodeOrdering = 0;
+    RPOTOrder[*RPOT.begin()] = ++NodeOrdering;
+    for (auto *BB : RPOT)
+      if (!pred_empty(BB))
+        RPOTOrder[BB] = ++NodeOrdering;
     for (auto *N : RPOT)
       NumSunk += sinkBB(N);
 
@@ -583,6 +626,7 @@ public:
 
 private:
   ValueTable VN;
+  DenseMap<const BasicBlock *, unsigned> RPOTOrder;
 
   bool shouldAvoidSinkingInstruction(Instruction *I) {
     // These instructions may change or break semantics if moved.
@@ -603,7 +647,7 @@ private:
   void analyzeInitialPHIs(BasicBlock *BB, ModelledPHISet &PHIs,
                           SmallPtrSetImpl<Value *> &PHIContents) {
     for (PHINode &PN : BB->phis()) {
-      auto MPHI = ModelledPHI(&PN);
+      auto MPHI = ModelledPHI(&PN, RPOTOrder);
       PHIs.insert(MPHI);
       for (auto *V : MPHI.getValues())
         PHIContents.insert(V);
@@ -691,7 +735,7 @@ GVNSink::analyzeInstructionForSinking(LockstepReverseIterator &LRI,
   }
 
   // The sunk instruction's results.
-  ModelledPHI NewPHI(NewInsts, ActivePreds);
+  ModelledPHI NewPHI(NewInsts, ActivePreds, RPOTOrder);
 
   // Does sinking this instruction render previous PHIs redundant?
   if (NeededPHIs.erase(NewPHI))
@@ -766,6 +810,9 @@ unsigned GVNSink::sinkBB(BasicBlock *BBEnd) {
              BBEnd->printAsOperand(dbgs()); dbgs() << "\n");
   SmallVector<BasicBlock *, 4> Preds;
   for (auto *B : predecessors(BBEnd)) {
+    // Bailout on basic blocks without predecessor(PR42346).
+    if (!RPOTOrder.count(B))
+      return 0;
     auto *T = B->getTerminator();
     if (isa<BranchInst>(T) || isa<SwitchInst>(T))
       Preds.push_back(B);
@@ -774,7 +821,11 @@ unsigned GVNSink::sinkBB(BasicBlock *BBEnd) {
   }
   if (Preds.size() < 2)
     return 0;
-  llvm::sort(Preds);
+  auto ComesBefore = [this](const BasicBlock *BB1, const BasicBlock *BB2) {
+    return RPOTOrder.lookup(BB1) < RPOTOrder.lookup(BB2);
+  };
+  // Sort in a deterministic order.
+  llvm::sort(Preds, ComesBefore);
 
   unsigned NumOrigPreds = Preds.size();
   // We can only sink instructions through unconditional branches.
@@ -889,5 +940,6 @@ PreservedAnalyses GVNSinkPass::run(Function &F, FunctionAnalysisManager &AM) {
   GVNSink G;
   if (!G.run(F))
     return PreservedAnalyses::all();
+
   return PreservedAnalyses::none();
 }
diff --git a/llvm/test/Transforms/GVNSink/int_sideeffect.ll b/llvm/test/Transforms/GVNSink/int_sideeffect.ll
index 3cc54e84f17c..9a3bc062dd94 100644
--- a/llvm/test/Transforms/GVNSink/int_sideeffect.ll
+++ b/llvm/test/Transforms/GVNSink/int_sideeffect.ll
@@ -28,3 +28,29 @@ if.end:
   ret float %phi
 }
 
+; CHECK-LABEL: scalarsSinkingReverse
+; CHECK-NOT: fmul
+; CHECK: = phi
+; CHECK: = fmul
+define float @scalarsSinkingReverse(float %d, float %m, float %a, i1 %cmp) {
+; This test is just a reverse(graph mirror) of the test
+; above to ensure GVNSink doesn't depend on the order of branches.
+entry:
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %add = fadd float %m, %a
+  %mul1 = fmul float %add, %d
+  br label %if.end
+
+if.else:
+  call void @llvm.sideeffect()
+  %sub = fsub float %m, %a
+  %mul0 = fmul float %sub, %d
+  br label %if.end
+
+if.end:
+  %phi = phi float [ %mul1, %if.then ], [ %mul0, %if.else ]
+  ret float %phi
+}
+
-- 
GitLab


From d67c3a4b1f465da681b829fda54619ff0b5bd1b5 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung@sifive.com>
Date: Sun, 12 May 2024 19:57:40 -0700
Subject: [PATCH 0509/1206] Revert "[RISCV][TTI] Support
 fdiv/udiv/sdiv/srem/urem in getArithmeticInstrCost (#89170)"

This reverts commit ed16e7aac44f2024b45d8c6c9dc2817d77d0ea97.
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  47 +-
 .../test/Analysis/CostModel/RISCV/arith-fp.ll |  48 +-
 .../Analysis/CostModel/RISCV/arith-int.ll     | 828 ++++--------------
 3 files changed, 213 insertions(+), 710 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 984638c1e50d..5f84175da703 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1619,58 +1619,29 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
   if (Op2Info.isConstant())
     ConstantMatCost += getConstantMatCost(1, Op2Info);
 
-  unsigned Op;
   switch (TLI->InstructionOpcodeToISD(Opcode)) {
   case ISD::ADD:
   case ISD::SUB:
-    Op = RISCV::VADD_VV;
-    break;
-  case ISD::SHL:
-  case ISD::SRL:
-  case ISD::SRA:
-    Op = RISCV::VSLL_VV;
-    break;
   case ISD::AND:
   case ISD::OR:
   case ISD::XOR:
-    Op = (Ty->getScalarSizeInBits() == 1) ? RISCV::VMAND_MM : RISCV::VAND_VV;
-    break;
+  case ISD::SHL:
+  case ISD::SRL:
+  case ISD::SRA:
   case ISD::MUL:
   case ISD::MULHS:
   case ISD::MULHU:
-    Op = RISCV::VMUL_VV;
-    break;
-  case ISD::SDIV:
-  case ISD::UDIV:
-    Op = RISCV::VDIV_VV;
-    break;
-  case ISD::SREM:
-  case ISD::UREM:
-    Op = RISCV::VREM_VV;
-    break;
   case ISD::FADD:
   case ISD::FSUB:
-    // TODO: Address FP16 with VFHMIN
-    Op = RISCV::VFADD_VV;
   case ISD::FMUL:
-    // TODO: Address FP16 with VFHMIN
-    Op = RISCV::VFMUL_VV;
-    break;
-  case ISD::FDIV:
-    Op = RISCV::VFDIV_VV;
-    break;
-  case ISD::FNEG:
-    Op = RISCV::VFSGNJN_VV;
-    break;
+  case ISD::FNEG: {
+    return ConstantMatCost + TLI->getLMULCost(LT.second) * LT.first * 1;
+  }
   default:
-    // Assuming all other instructions have the same cost until a need arises to
-    // differentiate them.
-    return ConstantMatCost + BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind,
-                                                           Op1Info, Op2Info,
-                                                           Args, CxtI);
+    return ConstantMatCost +
+           BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
+                                         Args, CxtI);
   }
-  return ConstantMatCost +
-         LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);
 }
 
 // TODO: Deduplicate from TargetTransformInfoImplCRTPBase.
diff --git a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
index d1e8bb015491..1dde88f366a3 100644
--- a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
@@ -248,36 +248,36 @@ define i32 @fdiv() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fdiv half undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fdiv float undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fdiv double undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F16 = fdiv <1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fdiv <2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fdiv <4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fdiv <8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fdiv <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1F16 = fdiv <1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F16 = fdiv <2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F16 = fdiv <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F16 = fdiv <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = fdiv <16 x half> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32F16 = fdiv <32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F16 = fdiv <vscale x 1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F16 = fdiv <vscale x 2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F16 = fdiv <vscale x 4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F16 = fdiv <vscale x 1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F16 = fdiv <vscale x 2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F16 = fdiv <vscale x 4 x half> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F16 = fdiv <vscale x 8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16F16 = fdiv <vscale x 16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32F16 = fdiv <vscale x 32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F32 = fdiv <1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fdiv <2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fdiv <4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fdiv <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV16F16 = fdiv <vscale x 16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV32F16 = fdiv <vscale x 32 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1F32 = fdiv <1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = fdiv <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F32 = fdiv <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fdiv <8 x float> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fdiv <16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F32 = fdiv <vscale x 1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F32 = fdiv <vscale x 2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F32 = fdiv <vscale x 1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F32 = fdiv <vscale x 2 x float> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F32 = fdiv <vscale x 4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F32 = fdiv <vscale x 8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F32 = fdiv <vscale x 16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = fdiv <1 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fdiv <2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fdiv <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F32 = fdiv <vscale x 8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV16F32 = fdiv <vscale x 16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1F64 = fdiv <1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = fdiv <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fdiv <4 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fdiv <8 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F64 = fdiv <vscale x 1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F64 = fdiv <vscale x 1 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F64 = fdiv <vscale x 2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F64 = fdiv <vscale x 4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F64 = fdiv <vscale x 8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F64 = fdiv <vscale x 4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F64 = fdiv <vscale x 8 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %F16 = fdiv half undef, undef
diff --git a/llvm/test/Analysis/CostModel/RISCV/arith-int.ll b/llvm/test/Analysis/CostModel/RISCV/arith-int.ll
index c976f483fdfe..b4afbb513166 100644
--- a/llvm/test/Analysis/CostModel/RISCV/arith-int.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/arith-int.ll
@@ -705,72 +705,72 @@ define i32 @udiv() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = udiv i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = udiv i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = udiv i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = udiv <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = udiv <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = udiv <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = udiv <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = udiv <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = udiv <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = udiv <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = udiv <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = udiv <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = udiv <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = udiv <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = udiv <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = udiv <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = udiv <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = udiv <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = udiv <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = udiv <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = udiv <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = udiv <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = udiv <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = udiv <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = udiv <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = udiv <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = udiv <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = udiv <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = udiv <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = udiv <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = udiv <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = udiv <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = udiv <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = udiv <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = udiv <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = udiv <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'udiv'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = udiv i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = udiv i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = udiv i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = udiv <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = udiv <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = udiv <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = udiv <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = udiv <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = udiv <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = udiv <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = udiv <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = udiv <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = udiv <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = udiv <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = udiv <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = udiv <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = udiv <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = udiv <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = udiv <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = udiv <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = udiv <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = udiv <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = udiv <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = udiv <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = udiv <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = udiv <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = udiv <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = udiv <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = udiv <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = udiv <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = udiv <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = udiv <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = udiv <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = udiv <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = udiv <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = udiv <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = udiv <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = udiv <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = udiv <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = udiv i16 undef, undef
@@ -821,72 +821,72 @@ define i32 @urem() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = urem i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = urem i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = urem i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = urem <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = urem <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = urem <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = urem <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = urem <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = urem <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = urem <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = urem <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = urem <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = urem <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = urem <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = urem <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = urem <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = urem <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = urem <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = urem <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = urem <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = urem <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = urem <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = urem <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = urem <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = urem <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = urem <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = urem <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = urem <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = urem <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = urem <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = urem <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = urem <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = urem <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = urem <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = urem <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = urem <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'urem'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = urem i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = urem i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = urem i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = urem <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = urem <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = urem <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = urem <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = urem <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = urem <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = urem <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = urem <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = urem <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = urem <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = urem <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = urem <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = urem <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = urem <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = urem <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = urem <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = urem <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = urem <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = urem <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = urem <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = urem <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = urem <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = urem <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = urem <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = urem <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = urem <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = urem <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = urem <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = urem <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = urem <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = urem <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = urem <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = urem <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = urem <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = urem <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = urem <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = urem i16 undef, undef
@@ -937,72 +937,72 @@ define i32 @sdiv() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sdiv i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sdiv i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sdiv i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = sdiv <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = sdiv <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = sdiv <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = sdiv <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = sdiv <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = sdiv <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = sdiv <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = sdiv <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = sdiv <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = sdiv <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = sdiv <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = sdiv <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = sdiv <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = sdiv <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = sdiv <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'sdiv'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sdiv i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sdiv i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sdiv i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = sdiv <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = sdiv <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = sdiv <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = sdiv <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = sdiv <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = sdiv <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = sdiv <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = sdiv <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = sdiv <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = sdiv <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = sdiv <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = sdiv <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = sdiv <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = sdiv <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = sdiv <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = sdiv <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = sdiv <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = sdiv <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = sdiv i16 undef, undef
@@ -1053,72 +1053,72 @@ define i32 @srem() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = srem i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = srem i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = srem i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = srem <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = srem <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = srem <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = srem <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = srem <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = srem <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = srem <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = srem <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = srem <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = srem <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = srem <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = srem <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = srem <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = srem <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = srem <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = srem <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = srem <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = srem <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = srem <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = srem <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = srem <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = srem <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = srem <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = srem <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = srem <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = srem <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = srem <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = srem <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = srem <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = srem <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = srem <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = srem <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = srem <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'srem'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = srem i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = srem i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = srem i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = srem <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = srem <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = srem <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = srem <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = srem <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = srem <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = srem <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = srem <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = srem <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = srem <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = srem <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = srem <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = srem <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = srem <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = srem <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = srem <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = srem <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = srem <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = srem <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = srem <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = srem <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = srem <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = srem <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = srem <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = srem <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = srem <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = srem <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = srem <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = srem <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = srem <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = srem <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = srem <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = srem <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = srem <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = srem <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = srem <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = srem i16 undef, undef
@@ -1232,471 +1232,3 @@ define void @add_of_constant() {
 
   ret void
 }
-
-define i32 @and() {
-; CHECK-LABEL: 'and'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = and i1 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = and i16 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = and i32 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = and i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = and <1 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = and <2 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = and <4 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = and <8 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = and <16 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = and <32 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = and <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = and <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = and <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = and <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = and <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = and <32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = and <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = and <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = and <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = and <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = and <16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = and <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = and <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = and <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = and <8 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = and <vscale x 1 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = and <vscale x 2 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = and <vscale x 4 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = and <vscale x 8 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = and <vscale x 16 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = and <vscale x 32 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = and <vscale x 1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = and <vscale x 2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = and <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %35 = and <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = and <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %37 = and <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = and <vscale x 1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %39 = and <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %40 = and <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %41 = and <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %42 = and <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %43 = and <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %44 = and <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %45 = and <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %46 = and <vscale x 8 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIFIVE-X280-LABEL: 'and'
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = and i1 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = and i16 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = and i32 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = and i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = and <1 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = and <2 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = and <4 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = and <8 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = and <16 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = and <32 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = and <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = and <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = and <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = and <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = and <16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %16 = and <32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = and <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = and <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = and <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = and <8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = and <16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = and <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = and <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = and <4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = and <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = and <vscale x 1 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = and <vscale x 2 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = and <vscale x 4 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = and <vscale x 8 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = and <vscale x 16 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = and <vscale x 32 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = and <vscale x 1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = and <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %34 = and <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = and <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %36 = and <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %37 = and <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = and <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %39 = and <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %40 = and <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %41 = and <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %42 = and <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %43 = and <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %44 = and <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %45 = and <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %46 = and <vscale x 8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-  and i1 undef, undef
-  and i16 undef, undef
-  and i32 undef, undef
-  and i64 undef, undef
-
-  and <1 x i1> undef, undef
-  and <2 x i1> undef, undef
-  and <4 x i1> undef, undef
-  and <8 x i1> undef, undef
-  and <16 x i1> undef, undef
-  and <32 x i1> undef, undef
-
-  and <1 x i16> undef, undef
-  and <2 x i16> undef, undef
-  and <4 x i16> undef, undef
-  and <8 x i16> undef, undef
-  and <16 x i16> undef, undef
-  and <32 x i16> undef, undef
-
-  and <1 x i32> undef, undef
-  and <2 x i32> undef, undef
-  and <4 x i32> undef, undef
-  and <8 x i32> undef, undef
-  and <16 x i32> undef, undef
-
-  and <1 x i64> undef, undef
-  and <2 x i64> undef, undef
-  and <4 x i64> undef, undef
-  and <8 x i64> undef, undef
-
-  and <vscale x 1 x i1> undef, undef
-  and <vscale x 2 x i1> undef, undef
-  and <vscale x 4 x i1> undef, undef
-  and <vscale x 8 x i1> undef, undef
-  and <vscale x 16 x i1> undef, undef
-  and <vscale x 32 x i1> undef, undef
-
-  and <vscale x 1 x i16> undef, undef
-  and <vscale x 2 x i16> undef, undef
-  and <vscale x 4 x i16> undef, undef
-  and <vscale x 8 x i16> undef, undef
-  and <vscale x 16 x i16> undef, undef
-  and <vscale x 32 x i16> undef, undef
-
-  and <vscale x 1 x i32> undef, undef
-  and <vscale x 2 x i32> undef, undef
-  and <vscale x 4 x i32> undef, undef
-  and <vscale x 8 x i32> undef, undef
-  and <vscale x 16 x i32> undef, undef
-
-  and <vscale x 1 x i64> undef, undef
-  and <vscale x 2 x i64> undef, undef
-  and <vscale x 4 x i64> undef, undef
-  and <vscale x 8 x i64> undef, undef
-  ret i32 undef
-}
-
-define i32 @or() {
-; CHECK-LABEL: 'or'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = or i1 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = or i16 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = or i32 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = or i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = or <1 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = or <2 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = or <4 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = or <8 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = or <16 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = or <32 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = or <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = or <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = or <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = or <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = or <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = or <32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = or <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = or <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = or <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = or <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = or <16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = or <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = or <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = or <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = or <8 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = or <vscale x 1 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = or <vscale x 2 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = or <vscale x 4 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = or <vscale x 8 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = or <vscale x 16 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = or <vscale x 32 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = or <vscale x 1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = or <vscale x 2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = or <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %35 = or <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = or <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %37 = or <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = or <vscale x 1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %39 = or <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %40 = or <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %41 = or <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %42 = or <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %43 = or <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %44 = or <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %45 = or <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %46 = or <vscale x 8 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIFIVE-X280-LABEL: 'or'
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = or i1 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = or i16 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = or i32 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = or i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = or <1 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = or <2 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = or <4 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = or <8 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = or <16 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = or <32 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = or <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = or <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = or <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = or <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = or <16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %16 = or <32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = or <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = or <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = or <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = or <8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = or <16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = or <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = or <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = or <4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = or <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = or <vscale x 1 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = or <vscale x 2 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = or <vscale x 4 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = or <vscale x 8 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = or <vscale x 16 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = or <vscale x 32 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = or <vscale x 1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = or <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %34 = or <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = or <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %36 = or <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %37 = or <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = or <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %39 = or <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %40 = or <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %41 = or <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %42 = or <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %43 = or <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %44 = or <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %45 = or <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %46 = or <vscale x 8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-  or i1 undef, undef
-  or i16 undef, undef
-  or i32 undef, undef
-  or i64 undef, undef
-
-  or <1 x i1> undef, undef
-  or <2 x i1> undef, undef
-  or <4 x i1> undef, undef
-  or <8 x i1> undef, undef
-  or <16 x i1> undef, undef
-  or <32 x i1> undef, undef
-
-  or <1 x i16> undef, undef
-  or <2 x i16> undef, undef
-  or <4 x i16> undef, undef
-  or <8 x i16> undef, undef
-  or <16 x i16> undef, undef
-  or <32 x i16> undef, undef
-
-  or <1 x i32> undef, undef
-  or <2 x i32> undef, undef
-  or <4 x i32> undef, undef
-  or <8 x i32> undef, undef
-  or <16 x i32> undef, undef
-
-  or <1 x i64> undef, undef
-  or <2 x i64> undef, undef
-  or <4 x i64> undef, undef
-  or <8 x i64> undef, undef
-
-  or <vscale x 1 x i1> undef, undef
-  or <vscale x 2 x i1> undef, undef
-  or <vscale x 4 x i1> undef, undef
-  or <vscale x 8 x i1> undef, undef
-  or <vscale x 16 x i1> undef, undef
-  or <vscale x 32 x i1> undef, undef
-
-  or <vscale x 1 x i16> undef, undef
-  or <vscale x 2 x i16> undef, undef
-  or <vscale x 4 x i16> undef, undef
-  or <vscale x 8 x i16> undef, undef
-  or <vscale x 16 x i16> undef, undef
-  or <vscale x 32 x i16> undef, undef
-
-  or <vscale x 1 x i32> undef, undef
-  or <vscale x 2 x i32> undef, undef
-  or <vscale x 4 x i32> undef, undef
-  or <vscale x 8 x i32> undef, undef
-  or <vscale x 16 x i32> undef, undef
-
-  or <vscale x 1 x i64> undef, undef
-  or <vscale x 2 x i64> undef, undef
-  or <vscale x 4 x i64> undef, undef
-  or <vscale x 8 x i64> undef, undef
-  ret i32 undef
-}
-
-define i32 @xor() {
-; CHECK-LABEL: 'xor'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = xor i1 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = xor i16 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = xor i32 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = xor i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = xor <1 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = xor <2 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = xor <4 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = xor <8 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = xor <16 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = xor <32 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = xor <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = xor <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = xor <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = xor <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = xor <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = xor <32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = xor <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = xor <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = xor <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = xor <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = xor <16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = xor <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = xor <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = xor <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = xor <8 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = xor <vscale x 1 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = xor <vscale x 2 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = xor <vscale x 4 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = xor <vscale x 8 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = xor <vscale x 16 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = xor <vscale x 32 x i1> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = xor <vscale x 1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = xor <vscale x 2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = xor <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %35 = xor <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = xor <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %37 = xor <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = xor <vscale x 1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %39 = xor <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %40 = xor <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %41 = xor <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %42 = xor <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %43 = xor <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %44 = xor <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %45 = xor <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %46 = xor <vscale x 8 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIFIVE-X280-LABEL: 'xor'
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = xor i1 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = xor i16 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = xor i32 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = xor i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = xor <1 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = xor <2 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = xor <4 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = xor <8 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = xor <16 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = xor <32 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = xor <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = xor <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = xor <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = xor <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = xor <16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %16 = xor <32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = xor <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = xor <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = xor <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = xor <8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = xor <16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = xor <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = xor <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = xor <4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = xor <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = xor <vscale x 1 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = xor <vscale x 2 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = xor <vscale x 4 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = xor <vscale x 8 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = xor <vscale x 16 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = xor <vscale x 32 x i1> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = xor <vscale x 1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = xor <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %34 = xor <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = xor <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %36 = xor <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %37 = xor <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = xor <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %39 = xor <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %40 = xor <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %41 = xor <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %42 = xor <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %43 = xor <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %44 = xor <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %45 = xor <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %46 = xor <vscale x 8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-  xor i1 undef, undef
-  xor i16 undef, undef
-  xor i32 undef, undef
-  xor i64 undef, undef
-
-  xor <1 x i1> undef, undef
-  xor <2 x i1> undef, undef
-  xor <4 x i1> undef, undef
-  xor <8 x i1> undef, undef
-  xor <16 x i1> undef, undef
-  xor <32 x i1> undef, undef
-
-  xor <1 x i16> undef, undef
-  xor <2 x i16> undef, undef
-  xor <4 x i16> undef, undef
-  xor <8 x i16> undef, undef
-  xor <16 x i16> undef, undef
-  xor <32 x i16> undef, undef
-
-  xor <1 x i32> undef, undef
-  xor <2 x i32> undef, undef
-  xor <4 x i32> undef, undef
-  xor <8 x i32> undef, undef
-  xor <16 x i32> undef, undef
-
-  xor <1 x i64> undef, undef
-  xor <2 x i64> undef, undef
-  xor <4 x i64> undef, undef
-  xor <8 x i64> undef, undef
-
-  xor <vscale x 1 x i1> undef, undef
-  xor <vscale x 2 x i1> undef, undef
-  xor <vscale x 4 x i1> undef, undef
-  xor <vscale x 8 x i1> undef, undef
-  xor <vscale x 16 x i1> undef, undef
-  xor <vscale x 32 x i1> undef, undef
-
-  xor <vscale x 1 x i16> undef, undef
-  xor <vscale x 2 x i16> undef, undef
-  xor <vscale x 4 x i16> undef, undef
-  xor <vscale x 8 x i16> undef, undef
-  xor <vscale x 16 x i16> undef, undef
-  xor <vscale x 32 x i16> undef, undef
-
-  xor <vscale x 1 x i32> undef, undef
-  xor <vscale x 2 x i32> undef, undef
-  xor <vscale x 4 x i32> undef, undef
-  xor <vscale x 8 x i32> undef, undef
-  xor <vscale x 16 x i32> undef, undef
-
-  xor <vscale x 1 x i64> undef, undef
-  xor <vscale x 2 x i64> undef, undef
-  xor <vscale x 4 x i64> undef, undef
-  xor <vscale x 8 x i64> undef, undef
-  ret i32 undef
-}
-- 
GitLab


From 22213d58832e8fce344a7222ed705e4c7f4eb6b1 Mon Sep 17 00:00:00 2001
From: Shih-Po Hung <shihpo.hung@sifive.com>
Date: Mon, 13 May 2024 09:47:57 +0800
Subject: [PATCH 0510/1206] Recommit [RISCV][TTI] Support
 fdiv/udiv/sdiv/srem/urem in getArithmeticInstrCost (#89170)

Insert a break to fix the implicit-fallthrough caught by sanitizer.

Original commit message:

This patch made following changes:
1. Support ISD FDIV/UDIV/SDIV/UREM/SREM
2. Classify instructions which cost the same
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  48 +-
 .../test/Analysis/CostModel/RISCV/arith-fp.ll |  48 +-
 .../Analysis/CostModel/RISCV/arith-int.ll     | 828 ++++++++++++++----
 3 files changed, 711 insertions(+), 213 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 5f84175da703..d94dff5f2b1f 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1619,29 +1619,59 @@ InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
   if (Op2Info.isConstant())
     ConstantMatCost += getConstantMatCost(1, Op2Info);
 
+  unsigned Op;
   switch (TLI->InstructionOpcodeToISD(Opcode)) {
   case ISD::ADD:
   case ISD::SUB:
-  case ISD::AND:
-  case ISD::OR:
-  case ISD::XOR:
+    Op = RISCV::VADD_VV;
+    break;
   case ISD::SHL:
   case ISD::SRL:
   case ISD::SRA:
+    Op = RISCV::VSLL_VV;
+    break;
+  case ISD::AND:
+  case ISD::OR:
+  case ISD::XOR:
+    Op = (Ty->getScalarSizeInBits() == 1) ? RISCV::VMAND_MM : RISCV::VAND_VV;
+    break;
   case ISD::MUL:
   case ISD::MULHS:
   case ISD::MULHU:
+    Op = RISCV::VMUL_VV;
+    break;
+  case ISD::SDIV:
+  case ISD::UDIV:
+    Op = RISCV::VDIV_VV;
+    break;
+  case ISD::SREM:
+  case ISD::UREM:
+    Op = RISCV::VREM_VV;
+    break;
   case ISD::FADD:
   case ISD::FSUB:
+    // TODO: Address FP16 with VFHMIN
+    Op = RISCV::VFADD_VV;
+    break;
   case ISD::FMUL:
-  case ISD::FNEG: {
-    return ConstantMatCost + TLI->getLMULCost(LT.second) * LT.first * 1;
-  }
+    // TODO: Address FP16 with VFHMIN
+    Op = RISCV::VFMUL_VV;
+    break;
+  case ISD::FDIV:
+    Op = RISCV::VFDIV_VV;
+    break;
+  case ISD::FNEG:
+    Op = RISCV::VFSGNJN_VV;
+    break;
   default:
-    return ConstantMatCost +
-           BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
-                                         Args, CxtI);
+    // Assuming all other instructions have the same cost until a need arises to
+    // differentiate them.
+    return ConstantMatCost + BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind,
+                                                           Op1Info, Op2Info,
+                                                           Args, CxtI);
   }
+  return ConstantMatCost +
+         LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);
 }
 
 // TODO: Deduplicate from TargetTransformInfoImplCRTPBase.
diff --git a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
index 1dde88f366a3..d1e8bb015491 100644
--- a/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/arith-fp.ll
@@ -248,36 +248,36 @@ define i32 @fdiv() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fdiv half undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fdiv float undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fdiv double undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1F16 = fdiv <1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F16 = fdiv <2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F16 = fdiv <4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F16 = fdiv <8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = fdiv <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F16 = fdiv <1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fdiv <2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fdiv <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fdiv <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fdiv <16 x half> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32F16 = fdiv <32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F16 = fdiv <vscale x 1 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F16 = fdiv <vscale x 2 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F16 = fdiv <vscale x 4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F16 = fdiv <vscale x 1 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F16 = fdiv <vscale x 2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4F16 = fdiv <vscale x 4 x half> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F16 = fdiv <vscale x 8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV16F16 = fdiv <vscale x 16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV32F16 = fdiv <vscale x 32 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1F32 = fdiv <1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = fdiv <2 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F32 = fdiv <4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fdiv <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16F16 = fdiv <vscale x 16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32F16 = fdiv <vscale x 32 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F32 = fdiv <1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fdiv <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fdiv <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fdiv <8 x float> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fdiv <16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F32 = fdiv <vscale x 1 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F32 = fdiv <vscale x 2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F32 = fdiv <vscale x 1 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2F32 = fdiv <vscale x 2 x float> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F32 = fdiv <vscale x 4 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F32 = fdiv <vscale x 8 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV16F32 = fdiv <vscale x 16 x float> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1F64 = fdiv <1 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = fdiv <2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fdiv <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8F32 = fdiv <vscale x 8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16F32 = fdiv <vscale x 16 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1F64 = fdiv <1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fdiv <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fdiv <4 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fdiv <8 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1F64 = fdiv <vscale x 1 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1F64 = fdiv <vscale x 1 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2F64 = fdiv <vscale x 2 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4F64 = fdiv <vscale x 4 x double> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8F64 = fdiv <vscale x 8 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4F64 = fdiv <vscale x 4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8F64 = fdiv <vscale x 8 x double> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %F16 = fdiv half undef, undef
diff --git a/llvm/test/Analysis/CostModel/RISCV/arith-int.ll b/llvm/test/Analysis/CostModel/RISCV/arith-int.ll
index b4afbb513166..c976f483fdfe 100644
--- a/llvm/test/Analysis/CostModel/RISCV/arith-int.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/arith-int.ll
@@ -705,72 +705,72 @@ define i32 @udiv() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = udiv i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = udiv i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = udiv i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = udiv <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = udiv <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = udiv <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = udiv <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = udiv <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = udiv <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = udiv <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = udiv <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = udiv <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = udiv <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = udiv <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = udiv <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = udiv <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = udiv <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = udiv <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = udiv <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = udiv <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = udiv <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = udiv <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = udiv <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = udiv <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = udiv <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = udiv <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = udiv <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = udiv <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = udiv <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = udiv <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = udiv <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = udiv <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = udiv <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = udiv <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = udiv <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = udiv <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'udiv'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = udiv i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = udiv i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = udiv i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = udiv <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = udiv <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = udiv <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = udiv <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = udiv <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = udiv <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = udiv <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = udiv <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = udiv <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = udiv <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = udiv <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = udiv <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = udiv <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = udiv <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = udiv <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = udiv <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = udiv <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = udiv <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = udiv <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = udiv <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = udiv <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = udiv <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = udiv <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = udiv <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = udiv <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = udiv <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = udiv <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = udiv <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = udiv <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = udiv <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = udiv <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = udiv <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = udiv <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = udiv <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = udiv <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = udiv <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = udiv <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = udiv <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = udiv <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = udiv <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = udiv <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = udiv <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = udiv <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = udiv <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = udiv <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = udiv i16 undef, undef
@@ -821,72 +821,72 @@ define i32 @urem() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = urem i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = urem i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = urem i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = urem <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = urem <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = urem <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = urem <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = urem <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = urem <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = urem <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = urem <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = urem <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = urem <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = urem <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = urem <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = urem <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = urem <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = urem <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = urem <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = urem <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = urem <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = urem <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = urem <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = urem <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = urem <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = urem <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = urem <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = urem <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = urem <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = urem <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = urem <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = urem <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = urem <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = urem <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = urem <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = urem <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'urem'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = urem i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = urem i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = urem i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = urem <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = urem <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = urem <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = urem <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = urem <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = urem <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = urem <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = urem <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = urem <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = urem <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = urem <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = urem <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = urem <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = urem <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = urem <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = urem <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = urem <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = urem <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = urem <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = urem <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = urem <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = urem <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = urem <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = urem <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = urem <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = urem <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = urem <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = urem <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = urem <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = urem <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = urem <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = urem <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = urem <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = urem <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = urem <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = urem <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = urem <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = urem <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = urem <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = urem <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = urem <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = urem <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = urem <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = urem <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = urem <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = urem i16 undef, undef
@@ -937,72 +937,72 @@ define i32 @sdiv() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sdiv i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sdiv i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sdiv i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = sdiv <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = sdiv <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = sdiv <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = sdiv <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = sdiv <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = sdiv <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = sdiv <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = sdiv <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = sdiv <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = sdiv <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = sdiv <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = sdiv <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = sdiv <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = sdiv <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = sdiv <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'sdiv'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sdiv i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sdiv i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = sdiv i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = sdiv <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = sdiv <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = sdiv <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = sdiv <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = sdiv <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = sdiv <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = sdiv <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = sdiv <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = sdiv <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = sdiv <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = sdiv <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = sdiv <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = sdiv <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = sdiv <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = sdiv <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = sdiv <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = sdiv <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = sdiv <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = sdiv <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = sdiv <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = sdiv <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = sdiv <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = sdiv <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = sdiv <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = sdiv <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = sdiv <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = sdiv <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = sdiv <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = sdiv <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = sdiv <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = sdiv <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = sdiv <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = sdiv <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = sdiv <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = sdiv <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = sdiv <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = sdiv i16 undef, undef
@@ -1053,72 +1053,72 @@ define i32 @srem() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = srem i16 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = srem i32 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = srem i64 undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = srem <1 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = srem <2 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = srem <4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = srem <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = srem <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = srem <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = srem <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = srem <8 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = srem <16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = srem <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = srem <32 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = srem <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = srem <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = srem <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = srem <1 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = srem <2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = srem <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = srem <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = srem <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = srem <4 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = srem <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = srem <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = srem <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = srem <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = srem <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = srem <1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = srem <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = srem <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = srem <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = srem <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = srem <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = srem <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = srem <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SIFIVE-X280-LABEL: 'srem'
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = srem i16 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = srem i32 undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = srem i64 undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I16 = srem <1 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I16 = srem <2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I16 = srem <4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = srem <8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = srem <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I16 = srem <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = srem <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = srem <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = srem <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = srem <16 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = srem <32 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I16 = srem <vscale x 1 x i16> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I16 = srem <vscale x 2 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I16 = srem <vscale x 4 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I32 = srem <1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I32 = srem <2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = srem <4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = srem <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV4I16 = srem <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV8I16 = srem <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV16I16 = srem <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV32I16 = srem <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I32 = srem <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = srem <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = srem <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = srem <8 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = srem <16 x i32> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I32 = srem <vscale x 1 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I32 = srem <vscale x 2 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1I64 = srem <1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = srem <2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = srem <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV2I32 = srem <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV4I32 = srem <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV8I32 = srem <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV16I32 = srem <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1I64 = srem <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = srem <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = srem <4 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = srem <8 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV1I64 = srem <vscale x 1 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
-; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NXV1I64 = srem <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NXV2I64 = srem <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NXV4I64 = srem <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NXV8I64 = srem <vscale x 8 x i64> undef, undef
 ; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %I16 = srem i16 undef, undef
@@ -1232,3 +1232,471 @@ define void @add_of_constant() {
 
   ret void
 }
+
+define i32 @and() {
+; CHECK-LABEL: 'and'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = and i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = and i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = and i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = and i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = and <1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = and <2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = and <4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = and <8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = and <16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = and <32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = and <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = and <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = and <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = and <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = and <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = and <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = and <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = and <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = and <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = and <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = and <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = and <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = and <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = and <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = and <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = and <vscale x 1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = and <vscale x 2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = and <vscale x 4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = and <vscale x 8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = and <vscale x 16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = and <vscale x 32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = and <vscale x 1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = and <vscale x 2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = and <vscale x 4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %35 = and <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = and <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %37 = and <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = and <vscale x 1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %39 = and <vscale x 2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %40 = and <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %41 = and <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %42 = and <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %43 = and <vscale x 1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %44 = and <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %45 = and <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %46 = and <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SIFIVE-X280-LABEL: 'and'
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = and i1 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = and i16 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = and i32 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = and i64 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = and <1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = and <2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = and <4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = and <8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = and <16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = and <32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = and <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = and <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = and <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = and <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = and <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %16 = and <32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = and <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = and <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = and <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = and <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = and <16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = and <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = and <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = and <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = and <8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = and <vscale x 1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = and <vscale x 2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = and <vscale x 4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = and <vscale x 8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = and <vscale x 16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = and <vscale x 32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = and <vscale x 1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = and <vscale x 2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %34 = and <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = and <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %36 = and <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %37 = and <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = and <vscale x 1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %39 = and <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %40 = and <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %41 = and <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %42 = and <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %43 = and <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %44 = and <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %45 = and <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %46 = and <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  and i1 undef, undef
+  and i16 undef, undef
+  and i32 undef, undef
+  and i64 undef, undef
+
+  and <1 x i1> undef, undef
+  and <2 x i1> undef, undef
+  and <4 x i1> undef, undef
+  and <8 x i1> undef, undef
+  and <16 x i1> undef, undef
+  and <32 x i1> undef, undef
+
+  and <1 x i16> undef, undef
+  and <2 x i16> undef, undef
+  and <4 x i16> undef, undef
+  and <8 x i16> undef, undef
+  and <16 x i16> undef, undef
+  and <32 x i16> undef, undef
+
+  and <1 x i32> undef, undef
+  and <2 x i32> undef, undef
+  and <4 x i32> undef, undef
+  and <8 x i32> undef, undef
+  and <16 x i32> undef, undef
+
+  and <1 x i64> undef, undef
+  and <2 x i64> undef, undef
+  and <4 x i64> undef, undef
+  and <8 x i64> undef, undef
+
+  and <vscale x 1 x i1> undef, undef
+  and <vscale x 2 x i1> undef, undef
+  and <vscale x 4 x i1> undef, undef
+  and <vscale x 8 x i1> undef, undef
+  and <vscale x 16 x i1> undef, undef
+  and <vscale x 32 x i1> undef, undef
+
+  and <vscale x 1 x i16> undef, undef
+  and <vscale x 2 x i16> undef, undef
+  and <vscale x 4 x i16> undef, undef
+  and <vscale x 8 x i16> undef, undef
+  and <vscale x 16 x i16> undef, undef
+  and <vscale x 32 x i16> undef, undef
+
+  and <vscale x 1 x i32> undef, undef
+  and <vscale x 2 x i32> undef, undef
+  and <vscale x 4 x i32> undef, undef
+  and <vscale x 8 x i32> undef, undef
+  and <vscale x 16 x i32> undef, undef
+
+  and <vscale x 1 x i64> undef, undef
+  and <vscale x 2 x i64> undef, undef
+  and <vscale x 4 x i64> undef, undef
+  and <vscale x 8 x i64> undef, undef
+  ret i32 undef
+}
+
+define i32 @or() {
+; CHECK-LABEL: 'or'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = or i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = or i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = or i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = or i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = or <1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = or <2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = or <4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = or <8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = or <16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = or <32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = or <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = or <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = or <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = or <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = or <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = or <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = or <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = or <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = or <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = or <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = or <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = or <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = or <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = or <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = or <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = or <vscale x 1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = or <vscale x 2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = or <vscale x 4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = or <vscale x 8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = or <vscale x 16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = or <vscale x 32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = or <vscale x 1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = or <vscale x 2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = or <vscale x 4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %35 = or <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = or <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %37 = or <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = or <vscale x 1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %39 = or <vscale x 2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %40 = or <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %41 = or <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %42 = or <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %43 = or <vscale x 1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %44 = or <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %45 = or <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %46 = or <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SIFIVE-X280-LABEL: 'or'
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = or i1 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = or i16 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = or i32 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = or i64 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = or <1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = or <2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = or <4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = or <8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = or <16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = or <32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = or <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = or <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = or <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = or <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = or <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %16 = or <32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = or <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = or <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = or <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = or <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = or <16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = or <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = or <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = or <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = or <8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = or <vscale x 1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = or <vscale x 2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = or <vscale x 4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = or <vscale x 8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = or <vscale x 16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = or <vscale x 32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = or <vscale x 1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = or <vscale x 2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %34 = or <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = or <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %36 = or <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %37 = or <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = or <vscale x 1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %39 = or <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %40 = or <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %41 = or <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %42 = or <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %43 = or <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %44 = or <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %45 = or <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %46 = or <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  or i1 undef, undef
+  or i16 undef, undef
+  or i32 undef, undef
+  or i64 undef, undef
+
+  or <1 x i1> undef, undef
+  or <2 x i1> undef, undef
+  or <4 x i1> undef, undef
+  or <8 x i1> undef, undef
+  or <16 x i1> undef, undef
+  or <32 x i1> undef, undef
+
+  or <1 x i16> undef, undef
+  or <2 x i16> undef, undef
+  or <4 x i16> undef, undef
+  or <8 x i16> undef, undef
+  or <16 x i16> undef, undef
+  or <32 x i16> undef, undef
+
+  or <1 x i32> undef, undef
+  or <2 x i32> undef, undef
+  or <4 x i32> undef, undef
+  or <8 x i32> undef, undef
+  or <16 x i32> undef, undef
+
+  or <1 x i64> undef, undef
+  or <2 x i64> undef, undef
+  or <4 x i64> undef, undef
+  or <8 x i64> undef, undef
+
+  or <vscale x 1 x i1> undef, undef
+  or <vscale x 2 x i1> undef, undef
+  or <vscale x 4 x i1> undef, undef
+  or <vscale x 8 x i1> undef, undef
+  or <vscale x 16 x i1> undef, undef
+  or <vscale x 32 x i1> undef, undef
+
+  or <vscale x 1 x i16> undef, undef
+  or <vscale x 2 x i16> undef, undef
+  or <vscale x 4 x i16> undef, undef
+  or <vscale x 8 x i16> undef, undef
+  or <vscale x 16 x i16> undef, undef
+  or <vscale x 32 x i16> undef, undef
+
+  or <vscale x 1 x i32> undef, undef
+  or <vscale x 2 x i32> undef, undef
+  or <vscale x 4 x i32> undef, undef
+  or <vscale x 8 x i32> undef, undef
+  or <vscale x 16 x i32> undef, undef
+
+  or <vscale x 1 x i64> undef, undef
+  or <vscale x 2 x i64> undef, undef
+  or <vscale x 4 x i64> undef, undef
+  or <vscale x 8 x i64> undef, undef
+  ret i32 undef
+}
+
+define i32 @xor() {
+; CHECK-LABEL: 'xor'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = xor i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = xor i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = xor i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = xor i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = xor <1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = xor <2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = xor <4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = xor <8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = xor <16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = xor <32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = xor <1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = xor <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = xor <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = xor <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = xor <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %16 = xor <32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = xor <1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = xor <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = xor <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = xor <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = xor <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = xor <1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = xor <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = xor <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = xor <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = xor <vscale x 1 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = xor <vscale x 2 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = xor <vscale x 4 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = xor <vscale x 8 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = xor <vscale x 16 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = xor <vscale x 32 x i1> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = xor <vscale x 1 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = xor <vscale x 2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %34 = xor <vscale x 4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %35 = xor <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = xor <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %37 = xor <vscale x 32 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = xor <vscale x 1 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %39 = xor <vscale x 2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %40 = xor <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %41 = xor <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %42 = xor <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %43 = xor <vscale x 1 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %44 = xor <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %45 = xor <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %46 = xor <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SIFIVE-X280-LABEL: 'xor'
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = xor i1 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = xor i16 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = xor i32 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = xor i64 undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = xor <1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = xor <2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = xor <4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = xor <8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = xor <16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = xor <32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = xor <1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = xor <2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = xor <4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = xor <8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = xor <16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %16 = xor <32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = xor <1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = xor <2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = xor <4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = xor <8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = xor <16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = xor <1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = xor <2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = xor <4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = xor <8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = xor <vscale x 1 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = xor <vscale x 2 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = xor <vscale x 4 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %29 = xor <vscale x 8 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = xor <vscale x 16 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = xor <vscale x 32 x i1> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %32 = xor <vscale x 1 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %33 = xor <vscale x 2 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %34 = xor <vscale x 4 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = xor <vscale x 8 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %36 = xor <vscale x 16 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %37 = xor <vscale x 32 x i16> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %38 = xor <vscale x 1 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %39 = xor <vscale x 2 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %40 = xor <vscale x 4 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %41 = xor <vscale x 8 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %42 = xor <vscale x 16 x i32> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %43 = xor <vscale x 1 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %44 = xor <vscale x 2 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %45 = xor <vscale x 4 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %46 = xor <vscale x 8 x i64> undef, undef
+; SIFIVE-X280-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  xor i1 undef, undef
+  xor i16 undef, undef
+  xor i32 undef, undef
+  xor i64 undef, undef
+
+  xor <1 x i1> undef, undef
+  xor <2 x i1> undef, undef
+  xor <4 x i1> undef, undef
+  xor <8 x i1> undef, undef
+  xor <16 x i1> undef, undef
+  xor <32 x i1> undef, undef
+
+  xor <1 x i16> undef, undef
+  xor <2 x i16> undef, undef
+  xor <4 x i16> undef, undef
+  xor <8 x i16> undef, undef
+  xor <16 x i16> undef, undef
+  xor <32 x i16> undef, undef
+
+  xor <1 x i32> undef, undef
+  xor <2 x i32> undef, undef
+  xor <4 x i32> undef, undef
+  xor <8 x i32> undef, undef
+  xor <16 x i32> undef, undef
+
+  xor <1 x i64> undef, undef
+  xor <2 x i64> undef, undef
+  xor <4 x i64> undef, undef
+  xor <8 x i64> undef, undef
+
+  xor <vscale x 1 x i1> undef, undef
+  xor <vscale x 2 x i1> undef, undef
+  xor <vscale x 4 x i1> undef, undef
+  xor <vscale x 8 x i1> undef, undef
+  xor <vscale x 16 x i1> undef, undef
+  xor <vscale x 32 x i1> undef, undef
+
+  xor <vscale x 1 x i16> undef, undef
+  xor <vscale x 2 x i16> undef, undef
+  xor <vscale x 4 x i16> undef, undef
+  xor <vscale x 8 x i16> undef, undef
+  xor <vscale x 16 x i16> undef, undef
+  xor <vscale x 32 x i16> undef, undef
+
+  xor <vscale x 1 x i32> undef, undef
+  xor <vscale x 2 x i32> undef, undef
+  xor <vscale x 4 x i32> undef, undef
+  xor <vscale x 8 x i32> undef, undef
+  xor <vscale x 16 x i32> undef, undef
+
+  xor <vscale x 1 x i64> undef, undef
+  xor <vscale x 2 x i64> undef, undef
+  xor <vscale x 4 x i64> undef, undef
+  xor <vscale x 8 x i64> undef, undef
+  ret i32 undef
+}
-- 
GitLab


From b1f04d57f5818914d7db506985e2932f217844bd Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 12 May 2024 21:15:36 -0700
Subject: [PATCH 0511/1206] [ELF,test] Fix typo in check prefixes

---
 lld/test/ELF/aarch64-thunk-reuse2.s          |  2 +-
 lld/test/ELF/linkerscript/orphan-phdrs2.test |  2 +-
 lld/test/ELF/mips-got-page-script.s          | 52 ++++++++++----------
 lld/test/ELF/ttext-tdata-tbss.s              | 12 ++---
 4 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/lld/test/ELF/aarch64-thunk-reuse2.s b/lld/test/ELF/aarch64-thunk-reuse2.s
index e9dd385605ad..c2cfee6f876c 100644
--- a/lld/test/ELF/aarch64-thunk-reuse2.s
+++ b/lld/test/ELF/aarch64-thunk-reuse2.s
@@ -14,7 +14,7 @@
 # CHECK:       <__AArch64ADRPThunk_>:
 # CHECK-NEXT:   8010708:       adrp    x16, 0x10000
 # CHECK-NEXT:                  add     x16, x16, #1792
-# CHECk-NEXT:                  br      x16
+# CHECK-NEXT:                  br      x16
 # CHECK-LABEL: <high>:
 # CHECK-NEXT:   8010714:       bl      0x8010708 <__AArch64ADRPThunk_>
 # CHECK-NEXT:                  b       0x8010708 <__AArch64ADRPThunk_>
diff --git a/lld/test/ELF/linkerscript/orphan-phdrs2.test b/lld/test/ELF/linkerscript/orphan-phdrs2.test
index c302e0e70b2b..d75c76da87e8 100644
--- a/lld/test/ELF/linkerscript/orphan-phdrs2.test
+++ b/lld/test/ELF/linkerscript/orphan-phdrs2.test
@@ -12,7 +12,7 @@
 # CHECK-NEXT:   Type {{.*}} Flg Align
 # CHECK-NEXT:   LOAD {{.*}} R E 0x
 # CHECK-NEXT:   LOAD {{.*}} RW  0x
-# CHECK-MEXT:   LOAD {{.*}} R   0x
+# CHECK-NEXT:   LOAD {{.*}} R   0x
 
 # CHECK:      Segment Sections...
 # CHECK-NEXT:   00 .text {{$}}
diff --git a/lld/test/ELF/mips-got-page-script.s b/lld/test/ELF/mips-got-page-script.s
index 4055fb6dabef..3caf5cc03afc 100644
--- a/lld/test/ELF/mips-got-page-script.s
+++ b/lld/test/ELF/mips-got-page-script.s
@@ -19,32 +19,32 @@
 # CHECK-NEXT: Value: 0x40000
 
 # CHECK:      Local entries [
-# CHECK-BEXT:    Entry {
-# CHECK-BEXT:      Address:
-# CHECK-BEXT:      Access:
-# CHECK-BEXT:      Initial: 0x10000
-# CHECK-BEXT:    }
-# CHECK-BEXT:    Entry {
-# CHECK-BEXT:      Address:
-# CHECK-BEXT:      Access:
-# CHECK-BEXT:      Initial: 0x20000
-# CHECK-BEXT:    }
-# CHECK-BEXT:    Entry {
-# CHECK-BEXT:      Address:
-# CHECK-BEXT:      Access:
-# CHECK-BEXT:      Initial: 0x30000
-# CHECK-BEXT:    }
-# CHECK-BEXT:    Entry {
-# CHECK-BEXT:      Address:
-# CHECK-BEXT:      Access:
-# CHECK-BEXT:      Initial: 0x40000
-# CHECK-BEXT:    }
-# CHECK-BEXT:    Entry {
-# CHECK-BEXT:      Address:
-# CHECK-BEXT:      Access:
-# CHECK-BEXT:      Initial: 0x50000
-# CHECK-BEXT:    }
-# CHECK-BEXT:  ]
+# CHECK-NEXT:    Entry {
+# CHECK-NEXT:      Address:
+# CHECK-NEXT:      Access:
+# CHECK-NEXT:      Initial: 0x10000
+# CHECK-NEXT:    }
+# CHECK-NEXT:    Entry {
+# CHECK-NEXT:      Address:
+# CHECK-NEXT:      Access:
+# CHECK-NEXT:      Initial: 0x20000
+# CHECK-NEXT:    }
+# CHECK-NEXT:    Entry {
+# CHECK-NEXT:      Address:
+# CHECK-NEXT:      Access:
+# CHECK-NEXT:      Initial: 0x30000
+# CHECK-NEXT:    }
+# CHECK-NEXT:    Entry {
+# CHECK-NEXT:      Address:
+# CHECK-NEXT:      Access:
+# CHECK-NEXT:      Initial: 0x40000
+# CHECK-NEXT:    }
+# CHECK-NEXT:    Entry {
+# CHECK-NEXT:      Address:
+# CHECK-NEXT:      Access:
+# CHECK-NEXT:      Initial: 0x50000
+# CHECK-NEXT:    }
+# CHECK-NEXT:  ]
 
   .option pic2
   .text
diff --git a/lld/test/ELF/ttext-tdata-tbss.s b/lld/test/ELF/ttext-tdata-tbss.s
index fb9c4d513174..c8254d696929 100644
--- a/lld/test/ELF/ttext-tdata-tbss.s
+++ b/lld/test/ELF/ttext-tdata-tbss.s
@@ -42,13 +42,13 @@
 # USER2-NEXT: LOAD 0x001000 0x0000000000001000
 
 ## With .text well above 200000 we don't need to change the image base
-# RUN: ld.lld -Ttext 0x201000 %t.o -o %t4
+# RUN: ld.lld -Ttext 0x201000 -z separate-loadable-segments %t.o -o %t4
 # RUN: llvm-readelf -S -l %t4 | FileCheck %s --check-prefix=USER3
-# USER3:     .text   PROGBITS 0000000000201000 001000 000001
-# USER3-NEX: .rodata PROGBITS 0000000000202000 002000 000008
-# USER3-NEX: .aw     PROGBITS 0000000000203000 003000 000008
-# USER3-NEX: .data   PROGBITS 0000000000203008 003008 000008
-# USER3-NEX: .bss    NOBITS   0000000000203010 003010 000008
+# USER3:      .text   PROGBITS 0000000000201000 001000 000001
+# USER3-NEXT: .rodata PROGBITS 0000000000202000 002000 000008
+# USER3-NEXT: .aw     PROGBITS 0000000000203000 003000 000008
+# USER3-NEXT: .data   PROGBITS 0000000000203008 003008 000008
+# USER3-NEXT: .bss    NOBITS   0000000000203010 003010 000008
 # USER3:      Type
 # USER3-NEXT: PHDR 0x000040 0x0000000000200040
 # USER3-NEXT: LOAD 0x000000 0x0000000000200000
-- 
GitLab


From cb4fca929bd10a5c160a0f9bd58f2d7499669a89 Mon Sep 17 00:00:00 2001
From: Vikram Hegde <115221833+vikramRH@users.noreply.github.com>
Date: Mon, 13 May 2024 10:20:37 +0530
Subject: [PATCH 0512/1206] [AMDGPU] Extend llvm.amdgcn.update.dpp intrinsic to
 support f64 (#91190)

Follow up patch to https://github.com/llvm/llvm-project/pull/89217,
before we make changes to atomic optimizer.
---
 llvm/lib/Target/AMDGPU/SIInstructions.td      |   4 +-
 llvm/lib/Target/AMDGPU/SIRegisterInfo.td      |   1 +
 llvm/lib/Target/AMDGPU/VOP1Instructions.td    |   9 +-
 .../GlobalISel/llvm.amdgcn.update.dpp.ll      | 337 +++++++++++++++++-
 .../CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll  | 123 ++++++-
 5 files changed, 457 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index f9a660e334a0..f9e811f54d05 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3325,13 +3325,15 @@ def : GCNPat <
                         (as_i1timm $bound_ctrl))
 >;
 
+foreach vt = Reg64Types.types in {
 def : GCNPat <
-  (i64 (int_amdgcn_update_dpp i64:$old, i64:$src, timm:$dpp_ctrl, timm:$row_mask,
+  (vt (int_amdgcn_update_dpp vt:$old, vt:$src, timm:$dpp_ctrl, timm:$row_mask,
                               timm:$bank_mask, timm:$bound_ctrl)),
   (V_MOV_B64_DPP_PSEUDO VReg_64_Align2:$old, VReg_64_Align2:$src, (as_i32timm $dpp_ctrl),
                         (as_i32timm $row_mask), (as_i32timm $bank_mask),
                         (as_i1timm $bound_ctrl))
 >;
+}
 
 //===----------------------------------------------------------------------===//
 // Fract Patterns
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
index 01ed565bb756..caac7126068e 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -586,6 +586,7 @@ class RegisterTypes<list<ValueType> reg_types> {
 
 def Reg16Types : RegisterTypes<[i16, f16, bf16]>;
 def Reg32Types : RegisterTypes<[i32, f32, v2i16, v2f16, v2bf16, p2, p3, p5, p6]>;
+def Reg64Types : RegisterTypes<[i64, f64, v2i32, v2f32, p0]>;
 
 let HasVGPR = 1 in {
 // VOP3 and VINTERP can access 256 lo and 256 hi registers.
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 012dca22eb4f..4a56fad0cd60 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -1341,7 +1341,8 @@ def : GCNPat <
                        (as_i1timm $bound_ctrl))
 >;
 
-class UpdateDPPPat<ValueType vt> : GCNPat <
+foreach vt = Reg32Types.types in {
+def : GCNPat <
   (vt (int_amdgcn_update_dpp vt:$old, vt:$src, timm:$dpp_ctrl,
                               timm:$row_mask, timm:$bank_mask,
                               timm:$bound_ctrl)),
@@ -1349,11 +1350,7 @@ class UpdateDPPPat<ValueType vt> : GCNPat <
                  (as_i32timm $row_mask), (as_i32timm $bank_mask),
                  (as_i1timm $bound_ctrl))
 >;
-
-def : UpdateDPPPat<i32>;
-def : UpdateDPPPat<f32>;
-def : UpdateDPPPat<v2i16>;
-def : UpdateDPPPat<v2f16>;
+}
 
 } // End OtherPredicates = [isGFX8Plus]
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll
index f7adfe47b64f..727184a36c00 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll
@@ -43,8 +43,8 @@ define amdgpu_kernel void @dpp_test(ptr addrspace(1) %out, i32 %in1, i32 %in2) {
   store i32 %tmp0, ptr addrspace(1) %out
   ret void
 }
-define amdgpu_kernel void @update_dpp64_test(ptr addrspace(1) %arg, i64 %in1, i64 %in2) {
-; GFX8-LABEL: update_dpp64_test:
+define amdgpu_kernel void @update_dppi64_test(ptr addrspace(1) %arg, i64 %in1, i64 %in2) {
+; GFX8-LABEL: update_dppi64_test:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
@@ -62,7 +62,7 @@ define amdgpu_kernel void @update_dpp64_test(ptr addrspace(1) %arg, i64 %in1, i6
 ; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
 ; GFX8-NEXT:    s_endpgm
 ;
-; GFX10-LABEL: update_dpp64_test:
+; GFX10-LABEL: update_dppi64_test:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
@@ -76,7 +76,7 @@ define amdgpu_kernel void @update_dpp64_test(ptr addrspace(1) %arg, i64 %in1, i6
 ; GFX10-NEXT:    global_store_dwordx2 v4, v[2:3], s[0:1]
 ; GFX10-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: update_dpp64_test:
+; GFX11-LABEL: update_dppi64_test:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
@@ -98,6 +98,335 @@ define amdgpu_kernel void @update_dpp64_test(ptr addrspace(1) %arg, i64 %in1, i6
   ret void
 }
 
+define amdgpu_kernel void @update_dppf64_test(ptr addrspace(1) %arg, double %in1, double %in2) {
+; GFX8-LABEL: update_dppf64_test:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s3
+; GFX8-NEXT:    v_mov_b32_e32 v4, s2
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX8-NEXT:    v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: update_dppf64_test:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_dwordx2 v[0:1], v4, s[0:1]
+; GFX10-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX10-NEXT:    v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX10-NEXT:    global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: update_dppf64_test:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT:    global_load_b64 v[0:1], v4, s[0:1]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX11-NEXT:    v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX11-NEXT:    global_store_b64 v4, v[2:3], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds double, ptr addrspace(1) %arg, i32 %id
+  %load = load double, ptr addrspace(1) %gep
+  %tmp0 = call double @llvm.amdgcn.update.dpp.f64(double %in1, double %load, i32 1, i32 1, i32 1, i1 false) #1
+  store double %tmp0, ptr addrspace(1) %gep
+  ret void
+}
+
+define amdgpu_kernel void @update_dppv2i32_test(ptr addrspace(1) %arg, <2 x i32> %in1, <2 x i32> %in2) {
+; GFX8-LABEL: update_dppv2i32_test:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s3
+; GFX8-NEXT:    v_mov_b32_e32 v4, s2
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX8-NEXT:    v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: update_dppv2i32_test:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_dwordx2 v[0:1], v4, s[0:1]
+; GFX10-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX10-NEXT:    v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX10-NEXT:    global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: update_dppv2i32_test:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT:    global_load_b64 v[0:1], v4, s[0:1]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX11-NEXT:    v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX11-NEXT:    global_store_b64 v4, v[2:3], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds <2 x i32>, ptr addrspace(1) %arg, i32 %id
+  %load = load <2 x i32>, ptr addrspace(1) %gep
+  %tmp0 = call <2 x i32> @llvm.amdgcn.update.dpp.v2i32(<2 x i32> %in1, <2 x i32> %load, i32 1, i32 1, i32 1, i1 false) #1
+  store <2 x i32> %tmp0, ptr addrspace(1) %gep
+  ret void
+}
+
+define amdgpu_kernel void @update_dppv2f32_test(ptr addrspace(1) %arg, <2 x float> %in1, <2 x float> %in2) {
+; GFX8-LABEL: update_dppv2f32_test:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s3
+; GFX8-NEXT:    v_mov_b32_e32 v4, s2
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX8-NEXT:    v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: update_dppv2f32_test:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_dwordx2 v[0:1], v4, s[0:1]
+; GFX10-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX10-NEXT:    v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX10-NEXT:    global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: update_dppv2f32_test:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT:    global_load_b64 v[0:1], v4, s[0:1]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX11-NEXT:    v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX11-NEXT:    global_store_b64 v4, v[2:3], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %arg, i32 %id
+  %load = load <2 x float>, ptr addrspace(1) %gep
+  %tmp0 = call <2 x float> @llvm.amdgcn.update.dpp.v2f32(<2 x float> %in1, <2 x float> %load, i32 1, i32 1, i32 1, i1 false) #1
+  store <2 x float> %tmp0, ptr addrspace(1) %gep
+  ret void
+}
+
+define amdgpu_kernel void @update_dpp_p0_test(ptr addrspace(1) %arg, ptr %in1, ptr %in2) {
+; GFX8-LABEL: update_dpp_p0_test:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s3
+; GFX8-NEXT:    v_mov_b32_e32 v4, s2
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX8-NEXT:    v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: update_dpp_p0_test:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_dwordx2 v[0:1], v4, s[0:1]
+; GFX10-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX10-NEXT:    v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX10-NEXT:    global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: update_dpp_p0_test:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT:    global_load_b64 v[0:1], v4, s[0:1]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX11-NEXT:    v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX11-NEXT:    global_store_b64 v4, v[2:3], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds ptr, ptr addrspace(1) %arg, i32 %id
+  %load = load ptr, ptr addrspace(1) %gep
+  %tmp0 = call ptr @llvm.amdgcn.update.dpp.v2f32(ptr %in1, ptr %load, i32 1, i32 1, i32 1, i1 false) #1
+  store ptr %tmp0, ptr addrspace(1) %gep
+  ret void
+}
+
+define amdgpu_kernel void @update_dpp_p3_test(ptr addrspace(3) %arg, ptr addrspace(3) %in1, ptr %in2) {
+; GFX8-LABEL: update_dpp_p3_test:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    ds_read_b32 v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX8-NEXT:    ds_write_b32 v0, v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: update_dpp_p3_test:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
+; GFX10-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-NEXT:    ds_read_b32 v1, v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX10-NEXT:    ds_write_b32 v0, v2
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: update_dpp_p3_test:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, s0, v0
+; GFX11-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-NEXT:    ds_load_b32 v1, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX11-NEXT:    ds_store_b32 v0, v2
+; GFX11-NEXT:    s_endpgm
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) %arg, i32 %id
+  %load = load ptr addrspace(3), ptr addrspace(3) %gep
+  %tmp0 = call ptr addrspace(3) @llvm.amdgcn.update.dpp.p3(ptr addrspace(3) %in1, ptr addrspace(3) %load, i32 1, i32 1, i32 1, i1 false) #1
+  store ptr addrspace(3) %tmp0, ptr addrspace(3) %gep
+  ret void
+}
+
+define amdgpu_kernel void @update_dpp_p5_test(ptr addrspace(5) %arg, ptr addrspace(5) %in1, ptr %in2) {
+; GFX8-LABEL: update_dpp_p5_test:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
+; GFX8-NEXT:    s_mov_b32 s90, -1
+; GFX8-NEXT:    s_mov_b32 s91, 0xe80000
+; GFX8-NEXT:    s_add_u32 s88, s88, s3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX8-NEXT:    s_addc_u32 s89, s89, 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    s_nop 0
+; GFX8-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX8-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: update_dpp_p5_test:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT:    s_mov_b32 s4, SCRATCH_RSRC_DWORD0
+; GFX10-NEXT:    s_mov_b32 s5, SCRATCH_RSRC_DWORD1
+; GFX10-NEXT:    s_mov_b32 s6, -1
+; GFX10-NEXT:    s_mov_b32 s7, 0x31c16000
+; GFX10-NEXT:    s_add_u32 s4, s4, s3
+; GFX10-NEXT:    s_addc_u32 s5, s5, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
+; GFX10-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-NEXT:    buffer_load_dword v1, v0, s[4:7], 0 offen
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX10-NEXT:    buffer_store_dword v2, v0, s[4:7], 0 offen
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: update_dpp_p5_test:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_add_nc_u32_e32 v0, s0, v0
+; GFX11-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-NEXT:    scratch_load_b32 v1, v0, off
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
+; GFX11-NEXT:    scratch_store_b32 v0, v2, off
+; GFX11-NEXT:    s_endpgm
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds ptr addrspace(5), ptr addrspace(5) %arg, i32 %id
+  %load = load ptr addrspace(5), ptr addrspace(5) %gep
+  %tmp0 = call ptr addrspace(5) @llvm.amdgcn.update.dpp.p5(ptr addrspace(5) %in1, ptr addrspace(5) %load, i32 1, i32 1, i32 1, i1 false) #1
+  store ptr addrspace(5) %tmp0, ptr addrspace(5) %gep
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32 immarg, i32 immarg, i32 immarg, i1 immarg) #1
 declare i64 @llvm.amdgcn.update.dpp.i64(i64, i64, i32 immarg, i32 immarg, i32 immarg, i1 immarg) #1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll
index e43daf46e1e0..b678378e5554 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll
@@ -55,11 +55,11 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}update_dpp64_test:
+; GCN-LABEL: {{^}}update_dppi64_test:
 ; GCN:     load_{{dwordx2|b64}} v[[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]]
 ; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
 ; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
-define amdgpu_kernel void @update_dpp64_test(ptr addrspace(1) %arg, i64 %in1, i64 %in2) {
+define amdgpu_kernel void @update_dppi64_test(ptr addrspace(1) %arg, i64 %in1, i64 %in2) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id
   %load = load i64, ptr addrspace(1) %gep
@@ -68,7 +68,83 @@ define amdgpu_kernel void @update_dpp64_test(ptr addrspace(1) %arg, i64 %in1, i6
   ret void
 }
 
-; GCN-LABEL: {{^}}update_dpp64_imm_old_test:
+; GCN-LABEL: {{^}}update_dppf64_test:
+; GCN:     load_{{dwordx2|b64}} v[[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]]
+; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+define amdgpu_kernel void @update_dppf64_test(ptr addrspace(1) %arg, double %in1, double %in2) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds double, ptr addrspace(1) %arg, i32 %id
+  %load = load double, ptr addrspace(1) %gep
+  %tmp0 = call double @llvm.amdgcn.update.dpp.f64(double %in1, double %load, i32 1, i32 1, i32 1, i1 false) #0
+  store double %tmp0, ptr addrspace(1) %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}update_dppv2i32_test:
+; GCN:     load_{{dwordx2|b64}} v[[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]]
+; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+define amdgpu_kernel void @update_dppv2i32_test(ptr addrspace(1) %arg, <2 x i32> %in1, <2 x i32> %in2) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds <2 x i32>, ptr addrspace(1) %arg, i32 %id
+  %load = load <2 x i32>, ptr addrspace(1) %gep
+  %tmp0 = call <2 x i32> @llvm.amdgcn.update.dpp.v2i32(<2 x i32> %in1, <2 x i32> %load, i32 1, i32 1, i32 1, i1 false) #0
+  store <2 x i32> %tmp0, ptr addrspace(1) %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}update_dppv2f32_test:
+; GCN:     load_{{dwordx2|b64}} v[[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]]
+; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+define amdgpu_kernel void @update_dppv2f32_test(ptr addrspace(1) %arg, <2 x float> %in1, <2 x float> %in2) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %arg, i32 %id
+  %load = load <2 x float>, ptr addrspace(1) %gep
+  %tmp0 = call <2 x float> @llvm.amdgcn.update.dpp.v2f32(<2 x float> %in1, <2 x float> %load, i32 1, i32 1, i32 1, i1 false) #0
+  store <2 x float> %tmp0, ptr addrspace(1) %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}update_dpp_p0_test:
+; GCN:     load_{{dwordx2|b64}} v[[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]]
+; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+define amdgpu_kernel void @update_dpp_p0_test(ptr addrspace(1) %arg, ptr %in1, ptr %in2) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds ptr, ptr addrspace(1) %arg, i32 %id
+  %load = load ptr, ptr addrspace(1) %gep
+  %tmp0 = call ptr @llvm.amdgcn.update.dpp.p0(ptr %in1, ptr %load, i32 1, i32 1, i32 1, i1 false) #0
+  store ptr %tmp0, ptr addrspace(1) %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}update_dpp_p3_test:
+; GCN: {{load|read}}_{{dword|b32}} v[[SRC:[0-9]+]]
+; GCN: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+define amdgpu_kernel void @update_dpp_p3_test(ptr addrspace(3) %arg, ptr addrspace(3) %in1, ptr %in2) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) %arg, i32 %id
+  %load = load ptr addrspace(3), ptr addrspace(3) %gep
+  %tmp0 = call ptr addrspace(3) @llvm.amdgcn.update.dpp.p3(ptr addrspace(3) %in1, ptr addrspace(3) %load, i32 1, i32 1, i32 1, i1 false) #0
+  store ptr addrspace(3) %tmp0, ptr addrspace(3) %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}update_dpp_p5_test:
+; GCN: {{load|read}}_{{dword|b32}} v[[SRC:[0-9]+]]
+; GCN: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+define amdgpu_kernel void @update_dpp_p5_test(ptr addrspace(5) %arg, ptr addrspace(5) %in1, ptr %in2) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds ptr addrspace(5), ptr addrspace(5) %arg, i32 %id
+  %load = load ptr addrspace(5), ptr addrspace(5) %gep
+  %tmp0 = call ptr addrspace(5) @llvm.amdgcn.update.dpp.p5(ptr addrspace(5) %in1, ptr addrspace(5) %load, i32 1, i32 1, i32 1, i1 false) #0
+  store ptr addrspace(5) %tmp0, ptr addrspace(5) %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}update_dppi64_imm_old_test:
 ; GCN-OPT-DAG: v_mov_b32_e32 v[[OLD_LO:[0-9]+]], 0x3afaedd9
 ; GFX8-OPT-DAG,GFX10-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], 0x7047
 ; GFX11-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], 0x7047
@@ -79,7 +155,7 @@ define amdgpu_kernel void @update_dpp64_test(ptr addrspace(1) %arg, i64 %in1, i6
 ; GFX8-OPT-DAG,GFX10-DAG,GFX11-DAG: v_mov_b32_dpp v[[OLD_HI]], v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
 ; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
 ; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
-define amdgpu_kernel void @update_dpp64_imm_old_test(ptr addrspace(1) %arg, i64 %in2) {
+define amdgpu_kernel void @update_dppi64_imm_old_test(ptr addrspace(1) %arg, i64 %in2) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id
   %load = load i64, ptr addrspace(1) %gep
@@ -88,7 +164,27 @@ define amdgpu_kernel void @update_dpp64_imm_old_test(ptr addrspace(1) %arg, i64
   ret void
 }
 
-; GCN-LABEL: {{^}}update_dpp64_imm_src_test:
+; GCN-LABEL: {{^}}update_dppf64_imm_old_test:
+; GCN-OPT-DAG: v_mov_b32_e32 v[[OLD_LO:[0-9]+]], 0x6b8564a
+; GFX8-OPT-DAG,GFX10-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], 0x405edce1
+; GFX11-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], 0x405edce1
+; GFX8-NOOPT-DAG: s_mov_b32 s[[SOLD_LO:[0-9]+]], 0x6b8564a
+; GFX8-NOOPT-DAG: s_mov_b32 s[[SOLD_HI:[0-9]+]], 0x405edce1
+; GCN-DAG: load_{{dwordx2|b64}} v[[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]]
+; GCN-OPT-DAG: v_mov_b32_dpp v[[OLD_LO]], v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GFX8-OPT-DAG,GFX10-DAG,GFX11-DAG: v_mov_b32_dpp v[[OLD_HI]], v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+define amdgpu_kernel void @update_dppf64_imm_old_test(ptr addrspace(1) %arg, double %in2) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id
+  %load = load double, ptr addrspace(1) %gep
+  %tmp0 = call double @llvm.amdgcn.update.dpp.f64(double 123.4512345123450, double %load, i32 1, i32 1, i32 1, i1 false) #0
+  store double %tmp0, ptr addrspace(1) %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}update_dppi64_imm_src_test:
 ; GCN-OPT-DAG: v_mov_b32_e32 v[[OLD_LO:[0-9]+]], 0x3afaedd9
 ; GCN-OPT-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], 0x7047
 ; GFX8-NOOPT-DAG: s_mov_b32 s[[SOLD_LO:[0-9]+]], 0x3afaedd9
@@ -97,12 +193,27 @@ define amdgpu_kernel void @update_dpp64_imm_old_test(ptr addrspace(1) %arg, i64
 ; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[OLD_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
 ; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
 ; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
-define amdgpu_kernel void @update_dpp64_imm_src_test(ptr addrspace(1) %out, i64 %in1) {
+define amdgpu_kernel void @update_dppi64_imm_src_test(ptr addrspace(1) %out, i64 %in1) {
   %tmp0 = call i64 @llvm.amdgcn.update.dpp.i64(i64 %in1, i64 123451234512345, i32 1, i32 1, i32 1, i1 false) #0
   store i64 %tmp0, ptr addrspace(1) %out
   ret void
 }
 
+; GCN-LABEL: {{^}}update_dppf64_imm_src_test:
+; GCN-OPT-DAG: v_mov_b32_e32 v[[OLD_LO:[0-9]+]], 0x6b8564a
+; GCN-OPT-DAG: v_mov_b32_e32 v[[OLD_HI:[0-9]+]], 0x405edce1
+; GFX8-NOOPT-DAG: s_mov_b32 s[[SOLD_LO:[0-9]+]], 0x6b8564a
+; GFX8-NOOPT-DAG: s_mov_b32 s[[SOLD_HI:[0-9]+]], 0x405edce1
+; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[OLD_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[OLD_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+; GCN-NOOPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}}
+define amdgpu_kernel void @update_dppf64_imm_src_test(ptr addrspace(1) %out, double %in1) {
+  %tmp0 = call double @llvm.amdgcn.update.dpp.f64(double %in1, double 123.451234512345, i32 1, i32 1, i32 1, i1 false) #0
+  store double %tmp0, ptr addrspace(1) %out
+  ret void
+}
+
 ; GCN-LABEL: {{^}}dpp_test_f32:
 ; GCN:  v_mov_b32_e32 [[DST:v[0-9]+]], s{{[0-9]+}}
 ; GCN:  v_mov_b32_e32 [[SRC:v[0-9]+]], s{{[0-9]+}}
-- 
GitLab


From 2163ae761808ca0e5478357384f6ddbacce279eb Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 13 May 2024 13:51:22 +0900
Subject: [PATCH 0513/1206] [LVStringPool] Remove incorrect std::move (NFCI)

"Value" is still used afterwards in the return value. In this case,
this doesn't actually make a difference because a move for a
primitive type is the same as a copy, so there is no actual
misbehavior. Still drop the std::move to make the code less
confusing.
---
 llvm/include/llvm/DebugInfo/LogicalView/Core/LVStringPool.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVStringPool.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVStringPool.h
index 4c596b5b1dde..8ce751a56c59 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVStringPool.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVStringPool.h
@@ -60,7 +60,7 @@ public:
     if (isValidIndex(Index))
       return Index;
     size_t Value = Entries.size();
-    ValueType *Entry = ValueType::create(Key, Allocator, std::move(Value));
+    ValueType *Entry = ValueType::create(Key, Allocator, Value);
     StringTable.insert(Entry);
     Entries.push_back(Entry);
     return Value;
-- 
GitLab


From eeafc9daa15d2d022bcdd456d4b8bafd23f5f121 Mon Sep 17 00:00:00 2001
From: Christian Ulmann <christianulmann@gmail.com>
Date: Mon, 13 May 2024 07:37:41 +0200
Subject: [PATCH 0514/1206] [MLIR][Mem2Reg] Fix multi slot handling & move
 retry handling (#91464)

This commit fixes Mem2Regs mutli-slot allocator handling and extends the
test dialect to test this.

Additionally, this modifies Mem2Reg's API to always attempt a full
promotion on all the passed in "allocators". This ensures that the pass
does not require unnecessary walks over the regions and improves caching
benefits.
---
 .../mlir/Interfaces/MemorySlotInterfaces.td   |  6 +-
 mlir/include/mlir/Transforms/Mem2Reg.h        |  6 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp |  8 +-
 .../Dialect/MemRef/IR/MemRefMemorySlot.cpp    |  8 +-
 mlir/lib/Transforms/Mem2Reg.cpp               | 85 ++++++++++++-------
 mlir/test/Transforms/mem2reg.mlir             | 28 ++++++
 mlir/test/lib/Dialect/Test/TestOpDefs.cpp     | 59 +++++++++++++
 mlir/test/lib/Dialect/Test/TestOps.h          |  1 +
 mlir/test/lib/Dialect/Test/TestOps.td         | 11 +++
 9 files changed, 173 insertions(+), 39 deletions(-)
 create mode 100644 mlir/test/Transforms/mem2reg.mlir

diff --git a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td
index adf182ac7069..e2409cbec5fd 100644
--- a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td
+++ b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td
@@ -68,8 +68,12 @@ def PromotableAllocationOpInterface
         Hook triggered once the promotion of a slot is complete. This can
         also clean up the created default value if necessary.
         This will only be called for slots declared by this operation.
+
+        Must return a new promotable allocation op if this operation produced
+        multiple promotable slots, nullopt otherwise.
       }],
-      "void", "handlePromotionComplete",
+      "::std::optional<::mlir::PromotableAllocationOpInterface>",
+        "handlePromotionComplete",
       (ins
         "const ::mlir::MemorySlot &":$slot, 
         "::mlir::Value":$defaultValue,
diff --git a/mlir/include/mlir/Transforms/Mem2Reg.h b/mlir/include/mlir/Transforms/Mem2Reg.h
index fee7fb312750..6986cad9ae12 100644
--- a/mlir/include/mlir/Transforms/Mem2Reg.h
+++ b/mlir/include/mlir/Transforms/Mem2Reg.h
@@ -9,7 +9,6 @@
 #ifndef MLIR_TRANSFORMS_MEM2REG_H
 #define MLIR_TRANSFORMS_MEM2REG_H
 
-#include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/MemorySlotInterfaces.h"
 #include "llvm/ADT/Statistic.h"
 
@@ -23,8 +22,9 @@ struct Mem2RegStatistics {
   llvm::Statistic *newBlockArgumentAmount = nullptr;
 };
 
-/// Attempts to promote the memory slots of the provided allocators. Succeeds if
-/// at least one memory slot was promoted.
+/// Attempts to promote the memory slots of the provided allocators. Iteratively
+/// retries the promotion of all slots as promoting one slot might enable
+/// subsequent promotions. Succeeds if at least one memory slot was promoted.
 LogicalResult
 tryToPromoteMemorySlots(ArrayRef<PromotableAllocationOpInterface> allocators,
                         OpBuilder &builder, const DataLayout &dataLayout,
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index 70102e1c8192..4fdf847a559c 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -50,12 +50,14 @@ void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
                                        declareOp.getLocationExpr());
 }
 
-void LLVM::AllocaOp::handlePromotionComplete(const MemorySlot &slot,
-                                             Value defaultValue,
-                                             OpBuilder &builder) {
+std::optional<PromotableAllocationOpInterface>
+LLVM::AllocaOp::handlePromotionComplete(const MemorySlot &slot,
+                                        Value defaultValue,
+                                        OpBuilder &builder) {
   if (defaultValue && defaultValue.use_empty())
     defaultValue.getDefiningOp()->erase();
   this->erase();
+  return std::nullopt;
 }
 
 SmallVector<DestructurableMemorySlot> LLVM::AllocaOp::getDestructurableSlots() {
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
index dca07e84ea73..e30598e6878f 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
@@ -96,12 +96,14 @@ Value memref::AllocaOp::getDefaultValue(const MemorySlot &slot,
       });
 }
 
-void memref::AllocaOp::handlePromotionComplete(const MemorySlot &slot,
-                                               Value defaultValue,
-                                               OpBuilder &builder) {
+std::optional<PromotableAllocationOpInterface>
+memref::AllocaOp::handlePromotionComplete(const MemorySlot &slot,
+                                          Value defaultValue,
+                                          OpBuilder &builder) {
   if (defaultValue.use_empty())
     defaultValue.getDefiningOp()->erase();
   this->erase();
+  return std::nullopt;
 }
 
 void memref::AllocaOp::handleBlockArgument(const MemorySlot &slot,
diff --git a/mlir/lib/Transforms/Mem2Reg.cpp b/mlir/lib/Transforms/Mem2Reg.cpp
index 8adbbcd01cb4..e096747741c0 100644
--- a/mlir/lib/Transforms/Mem2Reg.cpp
+++ b/mlir/lib/Transforms/Mem2Reg.cpp
@@ -173,7 +173,9 @@ public:
   /// Actually promotes the slot by mutating IR. Promoting a slot DOES
   /// invalidate the MemorySlotPromotionInfo of other slots. Preparation of
   /// promotion info should NOT be performed in batches.
-  void promoteSlot();
+  /// Returns a promotable allocation op if a new allocator was created, nullopt
+  /// otherwise.
+  std::optional<PromotableAllocationOpInterface> promoteSlot();
 
 private:
   /// Computes the reaching definition for all the operations that require
@@ -595,7 +597,8 @@ void MemorySlotPromoter::removeBlockingUses() {
          "after promotion, the slot pointer should not be used anymore");
 }
 
-void MemorySlotPromoter::promoteSlot() {
+std::optional<PromotableAllocationOpInterface>
+MemorySlotPromoter::promoteSlot() {
   computeReachingDefInRegion(slot.ptr.getParentRegion(),
                              getOrCreateDefaultValue());
 
@@ -622,7 +625,7 @@ void MemorySlotPromoter::promoteSlot() {
   if (statistics.promotedAmount)
     (*statistics.promotedAmount)++;
 
-  allocator.handlePromotionComplete(slot, defaultValue, builder);
+  return allocator.handlePromotionComplete(slot, defaultValue, builder);
 }
 
 LogicalResult mlir::tryToPromoteMemorySlots(
@@ -636,20 +639,50 @@ LogicalResult mlir::tryToPromoteMemorySlots(
   // lazily and cached to avoid expensive recomputation.
   BlockIndexCache blockIndexCache;
 
-  for (PromotableAllocationOpInterface allocator : allocators) {
-    for (MemorySlot slot : allocator.getPromotableSlots()) {
-      if (slot.ptr.use_empty())
-        continue;
-
-      MemorySlotPromotionAnalyzer analyzer(slot, dominance, dataLayout);
-      std::optional<MemorySlotPromotionInfo> info = analyzer.computeInfo();
-      if (info) {
-        MemorySlotPromoter(slot, allocator, builder, dominance, dataLayout,
-                           std::move(*info), statistics, blockIndexCache)
-            .promoteSlot();
-        promotedAny = true;
+  SmallVector<PromotableAllocationOpInterface> workList(allocators.begin(),
+                                                        allocators.end());
+
+  SmallVector<PromotableAllocationOpInterface> newWorkList;
+  newWorkList.reserve(workList.size());
+  while (true) {
+    bool changesInThisRound = false;
+    for (PromotableAllocationOpInterface allocator : workList) {
+      bool changedAllocator = false;
+      for (MemorySlot slot : allocator.getPromotableSlots()) {
+        if (slot.ptr.use_empty())
+          continue;
+
+        MemorySlotPromotionAnalyzer analyzer(slot, dominance, dataLayout);
+        std::optional<MemorySlotPromotionInfo> info = analyzer.computeInfo();
+        if (info) {
+          std::optional<PromotableAllocationOpInterface> newAllocator =
+              MemorySlotPromoter(slot, allocator, builder, dominance,
+                                 dataLayout, std::move(*info), statistics,
+                                 blockIndexCache)
+                  .promoteSlot();
+          changedAllocator = true;
+          // Add newly created allocators to the worklist for further
+          // processing.
+          if (newAllocator)
+            newWorkList.push_back(*newAllocator);
+
+          // A break is required, since promoting a slot may invalidate the
+          // remaining slots of an allocator.
+          break;
+        }
       }
+      if (!changedAllocator)
+        newWorkList.push_back(allocator);
+      changesInThisRound |= changedAllocator;
     }
+    if (!changesInThisRound)
+      break;
+    promotedAny = true;
+
+    // Swap the vector's backing memory and clear the entries in newWorkList
+    // afterwards. This ensures that additional heap allocations can be avoided.
+    workList.swap(newWorkList);
+    newWorkList.clear();
   }
 
   return success(promotedAny);
@@ -677,22 +710,16 @@ struct Mem2Reg : impl::Mem2RegBase<Mem2Reg> {
 
       OpBuilder builder(&region.front(), region.front().begin());
 
-      // Promoting a slot can allow for further promotion of other slots,
-      // promotion is tried until no promotion succeeds.
-      while (true) {
-        SmallVector<PromotableAllocationOpInterface> allocators;
-        // Build a list of allocators to attempt to promote the slots of.
-        region.walk([&](PromotableAllocationOpInterface allocator) {
-          allocators.emplace_back(allocator);
-        });
-
-        // Attempt promoting until no promotion succeeds.
-        if (failed(tryToPromoteMemorySlots(allocators, builder, dataLayout,
-                                           dominance, statistics)))
-          break;
+      SmallVector<PromotableAllocationOpInterface> allocators;
+      // Build a list of allocators to attempt to promote the slots of.
+      region.walk([&](PromotableAllocationOpInterface allocator) {
+        allocators.emplace_back(allocator);
+      });
 
+      // Attempt promoting as many of the slots as possible.
+      if (succeeded(tryToPromoteMemorySlots(allocators, builder, dataLayout,
+                                            dominance, statistics)))
         changed = true;
-      }
     }
     if (!changed)
       markAllAnalysesPreserved();
diff --git a/mlir/test/Transforms/mem2reg.mlir b/mlir/test/Transforms/mem2reg.mlir
new file mode 100644
index 000000000000..daeaa2da0763
--- /dev/null
+++ b/mlir/test/Transforms/mem2reg.mlir
@@ -0,0 +1,28 @@
+// RUN: mlir-opt %s --pass-pipeline='builtin.module(func.func(mem2reg))' --split-input-file | FileCheck %s
+
+// Verifies that allocators with mutliple slots are handled properly.
+
+// CHECK-LABEL: func.func @multi_slot_alloca
+func.func @multi_slot_alloca() -> (i32, i32) {
+  // CHECK-NOT: test.multi_slot_alloca
+  %1, %2 = test.multi_slot_alloca : () -> (memref<i32>, memref<i32>)
+  %3 = memref.load %1[] : memref<i32>
+  %4 = memref.load %2[] : memref<i32>
+  return %3, %4 : i32, i32
+}
+
+// -----
+
+// Verifies that a multi slot allocator can be partially promoted.
+
+func.func private @consumer(memref<i32>)
+
+// CHECK-LABEL: func.func @multi_slot_alloca_only_second
+func.func @multi_slot_alloca_only_second() -> (i32, i32) {
+  // CHECK: %{{[[:alnum:]]+}} = test.multi_slot_alloca
+  %1, %2 = test.multi_slot_alloca : () -> (memref<i32>, memref<i32>)
+  func.call @consumer(%1) : (memref<i32>) -> ()
+  %3 = memref.load %1[] : memref<i32>
+  %4 = memref.load %2[] : memref<i32>
+  return %3, %4 : i32, i32
+}
diff --git a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
index 08df2e5e1228..d22d48b139a0 100644
--- a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
+++ b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/Verifier.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
+#include "mlir/Interfaces/MemorySlotInterfaces.h"
 
 using namespace mlir;
 using namespace test;
@@ -1172,3 +1173,61 @@ void TestOpWithVersionedProperties::writeToMlirBytecode(
   writer.writeVarInt(prop.value1);
   writer.writeVarInt(prop.value2);
 }
+
+//===----------------------------------------------------------------------===//
+// TestMultiSlotAlloca
+//===----------------------------------------------------------------------===//
+
+llvm::SmallVector<MemorySlot> TestMultiSlotAlloca::getPromotableSlots() {
+  SmallVector<MemorySlot> slots;
+  for (Value result : getResults()) {
+    slots.push_back(MemorySlot{
+        result, cast<MemRefType>(result.getType()).getElementType()});
+  }
+  return slots;
+}
+
+Value TestMultiSlotAlloca::getDefaultValue(const MemorySlot &slot,
+                                           OpBuilder &builder) {
+  return builder.create<TestOpConstant>(getLoc(), slot.elemType,
+                                        builder.getI32IntegerAttr(42));
+}
+
+void TestMultiSlotAlloca::handleBlockArgument(const MemorySlot &slot,
+                                              BlockArgument argument,
+                                              OpBuilder &builder) {
+  // Not relevant for testing.
+}
+
+std::optional<PromotableAllocationOpInterface>
+TestMultiSlotAlloca::handlePromotionComplete(const MemorySlot &slot,
+                                             Value defaultValue,
+                                             OpBuilder &builder) {
+  if (defaultValue && defaultValue.use_empty())
+    defaultValue.getDefiningOp()->erase();
+
+  if (getNumResults() == 1) {
+    erase();
+    return std::nullopt;
+  }
+
+  SmallVector<Type> newTypes;
+  SmallVector<Value> remainingValues;
+
+  for (Value oldResult : getResults()) {
+    if (oldResult == slot.ptr)
+      continue;
+    remainingValues.push_back(oldResult);
+    newTypes.push_back(oldResult.getType());
+  }
+
+  OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPoint(*this);
+  auto replacement = builder.create<TestMultiSlotAlloca>(getLoc(), newTypes);
+  for (auto [oldResult, newResult] :
+       llvm::zip_equal(remainingValues, replacement.getResults()))
+    oldResult.replaceAllUsesWith(newResult);
+
+  erase();
+  return replacement;
+}
diff --git a/mlir/test/lib/Dialect/Test/TestOps.h b/mlir/test/lib/Dialect/Test/TestOps.h
index f9925855bb9d..837ccca56592 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.h
+++ b/mlir/test/lib/Dialect/Test/TestOps.h
@@ -36,6 +36,7 @@
 #include "mlir/Interfaces/InferIntRangeInterface.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
+#include "mlir/Interfaces/MemorySlotInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 5352d574ac39..e16ea2407314 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -28,6 +28,7 @@ include "mlir/Interfaces/DestinationStyleOpInterface.td"
 include "mlir/Interfaces/InferIntRangeInterface.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/LoopLikeInterface.td"
+include "mlir/Interfaces/MemorySlotInterfaces.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 
 
@@ -3167,4 +3168,14 @@ def TestOpOptionallyImplementingInterface
   let arguments = (ins BoolAttr:$implementsInterface);
 }
 
+//===----------------------------------------------------------------------===//
+// Test Mem2Reg
+//===----------------------------------------------------------------------===//
+
+def TestMultiSlotAlloca : TEST_Op<"multi_slot_alloca",
+    [DeclareOpInterfaceMethods<PromotableAllocationOpInterface>]> {
+  let results = (outs Variadic<MemRefOf<[I32]>>:$results);
+  let assemblyFormat = "attr-dict `:` functional-type(operands, results)";
+}
+
 #endif // TEST_OPS
-- 
GitLab


From 662267daea7e76ee3cee90c63ab2bc2964b77b76 Mon Sep 17 00:00:00 2001
From: Chen Zheng <czhengsz@cn.ibm.com>
Date: Mon, 13 May 2024 01:32:25 -0400
Subject: [PATCH 0515/1206] [PPC] add testcase, nfc

---
 .../CodeGen/PowerPC/aix-tocdata-fastisel.ll   | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-tocdata-fastisel.ll

diff --git a/llvm/test/CodeGen/PowerPC/aix-tocdata-fastisel.ll b/llvm/test/CodeGen/PowerPC/aix-tocdata-fastisel.ll
new file mode 100644
index 000000000000..5a7fcd1d0ddd
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-tocdata-fastisel.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -mtriple=powerpc64-ibm-aix-xcoff -fast-isel -verify-machineinstrs \
+; RUN:   -code-model=small | FileCheck %s --check-prefix=SMALL
+
+;; FIXME: when toc data for 64 big large code model is supported,
+;; add a run line for large code model too.
+
+@a = global i32 0, align 4 #0
+
+define signext i32 @foo() #1 {
+; SMALL-LABEL: foo:
+; SMALL:       # %bb.0: # %entry
+; SMALL-NEXT:    la 3, a[TD](2)
+; SMALL-NEXT:    lwz 3, 0(3)
+; SMALL-NEXT:    extsw 3, 3
+; SMALL-NEXT:    blr
+entry:
+  %0 = load i32, ptr @a, align 4
+  ret i32 %0
+}
+
+attributes #0 = { "toc-data" }
+attributes #1 = { noinline optnone }
-- 
GitLab


From e6785fd75284f53b9e23db6f249598e09f3fc39f Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 12 May 2024 23:02:37 -0700
Subject: [PATCH 0516/1206] [Scalar] Fix a warning

This patch fixes:

  llvm/lib/Transforms/Scalar/GVNSink.cpp:270:33: error: lambda capture
  'this' is not used [-Werror,-Wunused-lambda-capture]

While I am at it, this patch replaces llvm::for_each with a
range-based for loop.
---
 llvm/lib/Transforms/Scalar/GVNSink.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp
index ddf01dc612bb..95a4c644a91a 100644
--- a/llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -267,13 +267,13 @@ public:
     };
     assert(llvm::is_sorted(Blocks, ComesBefore));
     int C = 0;
-    llvm::for_each(Values, [&C, this](const Value *V) {
+    for (const Value *V : Values) {
       if (!isa<UndefValue>(V)) {
-        const Instruction *I = cast<Instruction>(V);
-        assert(I->getParent() == this->Blocks[C]);
+        assert(cast<Instruction>(V)->getParent() == Blocks[C]);
+        (void)C;
       }
       C++;
-    });
+    }
   }
   /// Create a PHI from an array of incoming values and incoming blocks.
   ModelledPHI(SmallVectorImpl<Instruction *> &V,
-- 
GitLab


From e74a34b6932965dfdc182b69f779e5bee551585a Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Mon, 13 May 2024 13:56:25 +0800
Subject: [PATCH 0517/1206] [NFC] [Serialization] Merge IdentID with
 IdentifierID

In ASTBitCodes.h, there are two type alias for the ID type of
Identifiers with the same underlying type. It is confusing. This patch
tries to merge the `IdentID` to `IdentifierID` to erase such confusion.
---
 .../include/clang/Frontend/MultiplexConsumer.h |  2 +-
 .../include/clang/Serialization/ASTBitCodes.h  |  9 +++------
 .../Serialization/ASTDeserializationListener.h |  2 +-
 clang/include/clang/Serialization/ASTReader.h  |  2 +-
 clang/include/clang/Serialization/ASTWriter.h  | 10 +++++-----
 clang/include/clang/Serialization/ModuleFile.h |  2 +-
 clang/lib/Frontend/FrontendAction.cpp          |  2 +-
 clang/lib/Frontend/MultiplexConsumer.cpp       |  2 +-
 clang/lib/Serialization/ASTReader.cpp          |  4 ++--
 clang/lib/Serialization/ASTReaderInternals.h   |  2 +-
 clang/lib/Serialization/ASTWriter.cpp          | 18 +++++++++---------
 11 files changed, 26 insertions(+), 29 deletions(-)

diff --git a/clang/include/clang/Frontend/MultiplexConsumer.h b/clang/include/clang/Frontend/MultiplexConsumer.h
index f29c8e92fded..4ed0d86d3cdf 100644
--- a/clang/include/clang/Frontend/MultiplexConsumer.h
+++ b/clang/include/clang/Frontend/MultiplexConsumer.h
@@ -32,7 +32,7 @@ public:
   MultiplexASTDeserializationListener(
       const std::vector<ASTDeserializationListener *> &L);
   void ReaderInitialized(ASTReader *Reader) override;
-  void IdentifierRead(serialization::IdentID ID, IdentifierInfo *II) override;
+  void IdentifierRead(serialization::IdentifierID ID, IdentifierInfo *II) override;
   void MacroRead(serialization::MacroID ID, MacroInfo *MI) override;
   void TypeRead(serialization::TypeIdx Idx, QualType T) override;
   void DeclRead(GlobalDeclID ID, const Decl *D) override;
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index ae9521e42709..d3538e43d3d7 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -61,6 +61,9 @@ const unsigned VERSION_MINOR = 1;
 /// and start at 1. 0 is reserved for NULL.
 using IdentifierID = uint32_t;
 
+/// The number of predefined identifier IDs.
+const unsigned int NUM_PREDEF_IDENT_IDS = 1;
+
 /// An ID number that refers to a declaration in an AST file. See the comments
 /// in DeclIDBase for details.
 using DeclID = DeclIDBase::DeclID;
@@ -123,12 +126,6 @@ struct UnsafeQualTypeDenseMapInfo {
   }
 };
 
-/// An ID number that refers to an identifier in an AST file.
-using IdentID = uint32_t;
-
-/// The number of predefined identifier IDs.
-const unsigned int NUM_PREDEF_IDENT_IDS = 1;
-
 /// An ID number that refers to a macro in an AST file.
 using MacroID = uint32_t;
 
diff --git a/clang/include/clang/Serialization/ASTDeserializationListener.h b/clang/include/clang/Serialization/ASTDeserializationListener.h
index 3ab7f1a91843..1d81a9ae3fe2 100644
--- a/clang/include/clang/Serialization/ASTDeserializationListener.h
+++ b/clang/include/clang/Serialization/ASTDeserializationListener.h
@@ -35,7 +35,7 @@ public:
   virtual void ReaderInitialized(ASTReader *Reader) { }
 
   /// An identifier was deserialized from the AST file.
-  virtual void IdentifierRead(serialization::IdentID ID,
+  virtual void IdentifierRead(serialization::IdentifierID ID,
                               IdentifierInfo *II) { }
   /// A macro was read from the AST file.
   virtual void MacroRead(serialization::MacroID ID, MacroInfo *MI) { }
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index e24fa121528f..1bb5fa27a241 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -667,7 +667,7 @@ private:
   std::vector<IdentifierInfo *> IdentifiersLoaded;
 
   using GlobalIdentifierMapType =
-      ContinuousRangeMap<serialization::IdentID, ModuleFile *, 4>;
+      ContinuousRangeMap<serialization::IdentifierID, ModuleFile *, 4>;
 
   /// Mapping from global identifier IDs to the module in which the
   /// identifier resides along with the offset that should be added to the
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index 482e9dd168cc..7bb0e81545bd 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -277,10 +277,10 @@ private:
   std::vector<serialization::UnalignedUInt64> TypeOffsets;
 
   /// The first ID number we can use for our own identifiers.
-  serialization::IdentID FirstIdentID = serialization::NUM_PREDEF_IDENT_IDS;
+  serialization::IdentifierID FirstIdentID = serialization::NUM_PREDEF_IDENT_IDS;
 
   /// The identifier ID that will be assigned to the next new identifier.
-  serialization::IdentID NextIdentID = FirstIdentID;
+  serialization::IdentifierID NextIdentID = FirstIdentID;
 
   /// Map that provides the ID numbers of each identifier in
   /// the output stream.
@@ -288,7 +288,7 @@ private:
   /// The ID numbers for identifiers are consecutive (in order of
   /// discovery), starting at 1. An ID of zero refers to a NULL
   /// IdentifierInfo.
-  llvm::MapVector<const IdentifierInfo *, serialization::IdentID> IdentifierIDs;
+  llvm::MapVector<const IdentifierInfo *, serialization::IdentifierID> IdentifierIDs;
 
   /// The first ID number we can use for our own macros.
   serialization::MacroID FirstMacroID = serialization::NUM_PREDEF_MACRO_IDS;
@@ -698,7 +698,7 @@ public:
   serialization::SelectorID getSelectorRef(Selector Sel);
 
   /// Get the unique number used to refer to the given identifier.
-  serialization::IdentID getIdentifierRef(const IdentifierInfo *II);
+  serialization::IdentifierID getIdentifierRef(const IdentifierInfo *II);
 
   /// Get the unique number used to refer to the given macro.
   serialization::MacroID getMacroRef(MacroInfo *MI, const IdentifierInfo *Name);
@@ -855,7 +855,7 @@ public:
 private:
   // ASTDeserializationListener implementation
   void ReaderInitialized(ASTReader *Reader) override;
-  void IdentifierRead(serialization::IdentID ID, IdentifierInfo *II) override;
+  void IdentifierRead(serialization::IdentifierID ID, IdentifierInfo *II) override;
   void MacroRead(serialization::MacroID ID, MacroInfo *MI) override;
   void TypeRead(serialization::TypeIdx Idx, QualType T) override;
   void SelectorRead(serialization::SelectorID ID, Selector Sel) override;
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index 8e0fce1fd48c..7d8cbe3d40f5 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -308,7 +308,7 @@ public:
   const uint32_t *IdentifierOffsets = nullptr;
 
   /// Base identifier ID for identifiers local to this module.
-  serialization::IdentID BaseIdentifierID = 0;
+  serialization::IdentifierID BaseIdentifierID = 0;
 
   /// Remapping table for identifier IDs in this module.
   ContinuousRangeMap<uint32_t, int, 2> IdentifierRemap;
diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp
index 9ae7664b4b49..a9c45e525c69 100644
--- a/clang/lib/Frontend/FrontendAction.cpp
+++ b/clang/lib/Frontend/FrontendAction.cpp
@@ -71,7 +71,7 @@ public:
     if (Previous)
       Previous->ReaderInitialized(Reader);
   }
-  void IdentifierRead(serialization::IdentID ID,
+  void IdentifierRead(serialization::IdentifierID ID,
                       IdentifierInfo *II) override {
     if (Previous)
       Previous->IdentifierRead(ID, II);
diff --git a/clang/lib/Frontend/MultiplexConsumer.cpp b/clang/lib/Frontend/MultiplexConsumer.cpp
index c74bfd86195f..8fdc7f55a500 100644
--- a/clang/lib/Frontend/MultiplexConsumer.cpp
+++ b/clang/lib/Frontend/MultiplexConsumer.cpp
@@ -35,7 +35,7 @@ void MultiplexASTDeserializationListener::ReaderInitialized(
 }
 
 void MultiplexASTDeserializationListener::IdentifierRead(
-    serialization::IdentID ID, IdentifierInfo *II) {
+    serialization::IdentifierID ID, IdentifierInfo *II) {
   for (size_t i = 0, e = Listeners.size(); i != e; ++i)
     Listeners[i]->IdentifierRead(ID, II);
 }
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 78e4df440641..7627996d2c32 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1005,7 +1005,7 @@ static bool readBit(unsigned &Bits) {
   return Value;
 }
 
-IdentID ASTIdentifierLookupTrait::ReadIdentifierID(const unsigned char *d) {
+IdentifierID ASTIdentifierLookupTrait::ReadIdentifierID(const unsigned char *d) {
   using namespace llvm::support;
 
   unsigned RawID = endian::readNext<uint32_t, llvm::endianness::little>(d);
@@ -1041,7 +1041,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   markIdentifierFromAST(Reader, *II);
   Reader.markIdentifierUpToDate(II);
 
-  IdentID ID = Reader.getGlobalIdentifierID(F, RawID);
+  IdentifierID ID = Reader.getGlobalIdentifierID(F, RawID);
   if (!IsInteresting) {
     // For uninteresting identifiers, there's nothing else to do. Just notify
     // the reader that we've finished loading this identifier.
diff --git a/clang/lib/Serialization/ASTReaderInternals.h b/clang/lib/Serialization/ASTReaderInternals.h
index 49268ad5251d..536b19f91691 100644
--- a/clang/lib/Serialization/ASTReaderInternals.h
+++ b/clang/lib/Serialization/ASTReaderInternals.h
@@ -175,7 +175,7 @@ public:
                      const unsigned char* d,
                      unsigned DataLen);
 
-  IdentID ReadIdentifierID(const unsigned char *d);
+  IdentifierID ReadIdentifierID(const unsigned char *d);
 
   ASTReader &getReader() const { return Reader; }
 };
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index ab07cf3efa45..6154ead589d3 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -3746,7 +3746,7 @@ public:
   using key_type = const IdentifierInfo *;
   using key_type_ref = key_type;
 
-  using data_type = IdentID;
+  using data_type = IdentifierID;
   using data_type_ref = data_type;
 
   using hash_value_type = unsigned;
@@ -3775,7 +3775,7 @@ public:
   }
 
   std::pair<unsigned, unsigned>
-  EmitKeyDataLength(raw_ostream &Out, const IdentifierInfo *II, IdentID ID) {
+  EmitKeyDataLength(raw_ostream &Out, const IdentifierInfo *II, IdentifierID ID) {
     // Record the location of the identifier data. This is used when generating
     // the mapping from persistent IDs to strings.
     Writer.SetIdentifierOffset(II, Out.tell());
@@ -3807,7 +3807,7 @@ public:
     Out.write(II->getNameStart(), KeyLen);
   }
 
-  void EmitData(raw_ostream &Out, const IdentifierInfo *II, IdentID ID,
+  void EmitData(raw_ostream &Out, const IdentifierInfo *II, IdentifierID ID,
                 unsigned) {
     using namespace llvm::support;
 
@@ -3891,7 +3891,7 @@ void ASTWriter::WriteIdentifierTable(Preprocessor &PP,
     IdentifierOffsets.resize(NextIdentID - FirstIdentID);
     for (auto IdentIDPair : IdentifierIDs) {
       const IdentifierInfo *II = IdentIDPair.first;
-      IdentID ID = IdentIDPair.second;
+      IdentifierID ID = IdentIDPair.second;
       assert(II && "NULL identifier in identifier table");
 
       // Write out identifiers if either the ID is local or the identifier has
@@ -4779,7 +4779,7 @@ void ASTWriter::AddVersionTuple(const VersionTuple &Version,
 /// Note that the identifier II occurs at the given offset
 /// within the identifier table.
 void ASTWriter::SetIdentifierOffset(const IdentifierInfo *II, uint32_t Offset) {
-  IdentID ID = IdentifierIDs[II];
+  IdentifierID ID = IdentifierIDs[II];
   // Only store offsets new to this AST file. Other identifier names are looked
   // up earlier in the chain and thus don't need an offset.
   if (ID >= FirstIdentID)
@@ -5945,11 +5945,11 @@ void ASTWriter::AddIdentifierRef(const IdentifierInfo *II, RecordDataImpl &Recor
   Record.push_back(getIdentifierRef(II));
 }
 
-IdentID ASTWriter::getIdentifierRef(const IdentifierInfo *II) {
+IdentifierID ASTWriter::getIdentifierRef(const IdentifierInfo *II) {
   if (!II)
     return 0;
 
-  IdentID &ID = IdentifierIDs[II];
+  IdentifierID &ID = IdentifierIDs[II];
   if (ID == 0)
     ID = NextIdentID++;
   return ID;
@@ -6610,9 +6610,9 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) {
   NextSubmoduleID = FirstSubmoduleID;
 }
 
-void ASTWriter::IdentifierRead(IdentID ID, IdentifierInfo *II) {
+void ASTWriter::IdentifierRead(IdentifierID ID, IdentifierInfo *II) {
   // Always keep the highest ID. See \p TypeRead() for more information.
-  IdentID &StoredID = IdentifierIDs[II];
+  IdentifierID &StoredID = IdentifierIDs[II];
   if (ID > StoredID)
     StoredID = ID;
 }
-- 
GitLab


From f841ca0c355ae53c96f615996d0aff4648da8618 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sun, 12 May 2024 23:08:40 -0700
Subject: [PATCH 0518/1206] Use StringRef::operator== instead of
 StringRef::equals (NFC) (#91864)

I'm planning to remove StringRef::equals in favor of
StringRef::operator==.

- StringRef::operator==/!= outnumber StringRef::equals by a factor of
  276 under llvm-project/ in terms of their usage.

- The elimination of StringRef::equals brings StringRef closer to
  std::string_view, which has operator== but not equals.

- S == "foo" is more readable than S.equals("foo"), especially for
  !Long.Expression.equals("str") vs Long.Expression != "str".
---
 bolt/lib/Profile/DataAggregator.cpp                    |  2 +-
 bolt/lib/Profile/DataReader.cpp                        |  3 +--
 bolt/lib/Rewrite/DWARFRewriter.cpp                     | 10 +++++-----
 bolt/lib/Rewrite/SDTRewriter.cpp                       |  2 +-
 clang-tools-extra/clang-tidy/ClangTidyCheck.cpp        |  2 +-
 .../bugprone/ForwardingReferenceOverloadCheck.cpp      |  4 ++--
 .../clang-tidy/modernize/LoopConvertCheck.cpp          |  2 +-
 .../clang-tidy/readability/IdentifierNamingCheck.cpp   |  2 +-
 .../readability/SuspiciousCallArgumentCheck.cpp        |  4 ++--
 clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp   |  3 +--
 .../clang-tidy/utils/RenamerClangTidyCheck.cpp         |  2 +-
 flang/lib/Frontend/CompilerInvocation.cpp              |  4 ++--
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp          |  2 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp                |  8 ++++----
 lld/COFF/DebugTypes.cpp                                |  2 +-
 lld/ELF/InputSection.cpp                               |  2 +-
 lld/ELF/Writer.cpp                                     |  2 +-
 lld/MachO/Driver.cpp                                   |  2 +-
 lld/wasm/InputChunks.cpp                               |  4 ++--
 19 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 9a71e227f233..302bcf1f2d87 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -2000,7 +2000,7 @@ std::error_code DataAggregator::parseMMapEvents() {
     std::pair<StringRef, MMapInfo> FileMMapInfo = FileMMapInfoRes.get();
     if (FileMMapInfo.second.PID == -1)
       continue;
-    if (FileMMapInfo.first.equals("(deleted)"))
+    if (FileMMapInfo.first == "(deleted)")
       continue;
 
     // Consider only the first mapping of the file for any given PID
diff --git a/bolt/lib/Profile/DataReader.cpp b/bolt/lib/Profile/DataReader.cpp
index 67f357fe4d3f..b2511ba10399 100644
--- a/bolt/lib/Profile/DataReader.cpp
+++ b/bolt/lib/Profile/DataReader.cpp
@@ -1205,8 +1205,7 @@ std::error_code DataReader::parse() {
 
     // Add entry data for branches to another function or branches
     // to entry points (including recursive calls)
-    if (BI.To.IsSymbol &&
-        (!BI.From.Name.equals(BI.To.Name) || BI.To.Offset == 0)) {
+    if (BI.To.IsSymbol && (BI.From.Name != BI.To.Name || BI.To.Offset == 0)) {
       I = GetOrCreateFuncEntry(BI.To.Name);
       I->second.EntryData.emplace_back(std::move(BI));
     }
diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp
index 26e4889faada..9d4297f913f3 100644
--- a/bolt/lib/Rewrite/DWARFRewriter.cpp
+++ b/bolt/lib/Rewrite/DWARFRewriter.cpp
@@ -1550,7 +1550,7 @@ CUOffsetMap DWARFRewriter::finalizeTypeSections(DIEBuilder &DIEBlder,
   for (const SectionRef &Section : Obj->sections()) {
     StringRef Contents = cantFail(Section.getContents());
     StringRef Name = cantFail(Section.getName());
-    if (Name.equals(".debug_types"))
+    if (Name == ".debug_types")
       BC.registerOrUpdateNoteSection(".debug_types", copyByteArray(Contents),
                                      Contents.size());
   }
@@ -1633,10 +1633,10 @@ void DWARFRewriter::finalizeDebugSections(
   for (const SectionRef &Secs : Obj->sections()) {
     StringRef Contents = cantFail(Secs.getContents());
     StringRef Name = cantFail(Secs.getName());
-    if (Name.equals(".debug_abbrev")) {
+    if (Name == ".debug_abbrev") {
       BC.registerOrUpdateNoteSection(".debug_abbrev", copyByteArray(Contents),
                                      Contents.size());
-    } else if (Name.equals(".debug_info")) {
+    } else if (Name == ".debug_info") {
       BC.registerOrUpdateNoteSection(".debug_info", copyByteArray(Contents),
                                      Contents.size());
     }
@@ -1771,7 +1771,7 @@ std::optional<StringRef> updateDebugData(
   };
   switch (SectionIter->second.second) {
   default: {
-    if (!SectionName.equals("debug_str.dwo"))
+    if (SectionName != "debug_str.dwo")
       errs() << "BOLT-WARNING: unsupported debug section: " << SectionName
              << "\n";
     return SectionContents;
@@ -1959,7 +1959,7 @@ void DWARFRewriter::updateDWP(DWARFUnit &CU,
       continue;
     }
 
-    if (SectionName.equals("debug_str.dwo")) {
+    if (SectionName == "debug_str.dwo") {
       CurStrSection = OutData;
     } else {
       // Since handleDebugDataPatching returned true, we already know this is
diff --git a/bolt/lib/Rewrite/SDTRewriter.cpp b/bolt/lib/Rewrite/SDTRewriter.cpp
index cc663b28990f..a3928c554ad6 100644
--- a/bolt/lib/Rewrite/SDTRewriter.cpp
+++ b/bolt/lib/Rewrite/SDTRewriter.cpp
@@ -87,7 +87,7 @@ void SDTRewriter::readSection() {
 
     StringRef Name = DE.getCStr(&Offset);
 
-    if (!Name.equals("stapsdt"))
+    if (Name != "stapsdt")
       errs() << "BOLT-WARNING: SDT note name \"" << Name
              << "\" is not expected\n";
 
diff --git a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
index 710b361e16c0..6028bb225813 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
@@ -171,7 +171,7 @@ std::optional<int64_t> ClangTidyCheck::OptionsView::getEnumInt(
     if (IgnoreCase) {
       if (Value.equals_insensitive(NameAndEnum.second))
         return NameAndEnum.first;
-    } else if (Value.equals(NameAndEnum.second)) {
+    } else if (Value == NameAndEnum.second) {
       return NameAndEnum.first;
     } else if (Value.equals_insensitive(NameAndEnum.second)) {
       Closest = NameAndEnum.second;
diff --git a/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp
index e7be8134781e..36687a8e761e 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp
@@ -25,8 +25,8 @@ AST_MATCHER(QualType, isEnableIf) {
     const NamedDecl *TypeDecl =
         Spec->getTemplateName().getAsTemplateDecl()->getTemplatedDecl();
     return TypeDecl->isInStdNamespace() &&
-           (TypeDecl->getName().equals("enable_if") ||
-            TypeDecl->getName().equals("enable_if_t"));
+           (TypeDecl->getName() == "enable_if" ||
+            TypeDecl->getName() == "enable_if_t");
   };
   const Type *BaseType = Node.getTypePtr();
   // Case: pointer or reference to enable_if.
diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
index 3229e302eb43..a1786ba5acfd 100644
--- a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
@@ -421,7 +421,7 @@ getContainerFromBeginEndCall(const Expr *Init, bool IsBegin, bool *IsArrow,
     return {};
   if (IsReverse && !Call->Name.consume_back("r"))
     return {};
-  if (!Call->Name.empty() && !Call->Name.equals("c"))
+  if (!Call->Name.empty() && Call->Name != "c")
     return {};
   return std::make_pair(Call->Container, Call->CallKind);
 }
diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index 27a12bfc5806..c3208392df15 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -1358,7 +1358,7 @@ IdentifierNamingCheck::getFailureInfo(
   std::replace(KindName.begin(), KindName.end(), '_', ' ');
 
   std::string Fixup = fixupWithStyle(Type, Name, Style, HNOption, ND);
-  if (StringRef(Fixup).equals(Name)) {
+  if (StringRef(Fixup) == Name) {
     if (!IgnoreFailedSplit) {
       LLVM_DEBUG(Location.print(llvm::dbgs(), SM);
                  llvm::dbgs()
diff --git a/clang-tools-extra/clang-tidy/readability/SuspiciousCallArgumentCheck.cpp b/clang-tools-extra/clang-tidy/readability/SuspiciousCallArgumentCheck.cpp
index 3eb80019ae75..18420d0c8488 100644
--- a/clang-tools-extra/clang-tidy/readability/SuspiciousCallArgumentCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/SuspiciousCallArgumentCheck.cpp
@@ -138,11 +138,11 @@ static bool applyAbbreviationHeuristic(
     const llvm::StringMap<std::string> &AbbreviationDictionary, StringRef Arg,
     StringRef Param) {
   if (AbbreviationDictionary.contains(Arg) &&
-      Param.equals(AbbreviationDictionary.lookup(Arg)))
+      Param == AbbreviationDictionary.lookup(Arg))
     return true;
 
   if (AbbreviationDictionary.contains(Param) &&
-      Arg.equals(AbbreviationDictionary.lookup(Param)))
+      Arg == AbbreviationDictionary.lookup(Param))
     return true;
 
   return false;
diff --git a/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp b/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
index a44720c47eca..0fa54b3847eb 100644
--- a/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
+++ b/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
@@ -88,8 +88,7 @@ determineIncludeKind(StringRef CanonicalFile, StringRef IncludeFile,
     if (FileCopy.consume_front(Parts.first) &&
         FileCopy.consume_back(Parts.second)) {
       // Determine the kind of this inclusion.
-      if (FileCopy.equals("/internal/") ||
-          FileCopy.equals("/proto/")) {
+      if (FileCopy == "/internal/" || FileCopy == "/proto/") {
         return IncludeSorter::IK_MainTUInclude;
       }
     }
diff --git a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
index f5ed61736540..e811f5519de2 100644
--- a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
@@ -86,7 +86,7 @@ static const NamedDecl *findDecl(const RecordDecl &RecDecl,
                                  StringRef DeclName) {
   for (const Decl *D : RecDecl.decls()) {
     if (const auto *ND = dyn_cast<NamedDecl>(D)) {
-      if (ND->getDeclName().isIdentifier() && ND->getName().equals(DeclName))
+      if (ND->getDeclName().isIdentifier() && ND->getName() == DeclName)
         return ND;
     }
   }
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 4318286e7415..db7fd3cccc7a 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -883,7 +883,7 @@ static bool parseDialectArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
   // -x cuda
   auto language = args.getLastArgValue(clang::driver::options::OPT_x);
-  if (language.equals("cuda")) {
+  if (language == "cuda") {
     res.getFrontendOpts().features.Enable(
         Fortran::common::LanguageFeature::CUDA);
   }
@@ -986,7 +986,7 @@ static bool parseDialectArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
   if (args.hasArg(clang::driver::options::OPT_std_EQ)) {
     auto standard = args.getLastArgValue(clang::driver::options::OPT_std_EQ);
     // We only allow f2018 as the given standard
-    if (standard.equals("f2018")) {
+    if (standard == "f2018") {
       res.setEnableConformanceChecks();
     } else {
       const unsigned diagID =
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index dcbbc39b84ea..58064d23eb08 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -1607,7 +1607,7 @@ static bool isIntrinsicModuleProcedure(llvm::StringRef name) {
 static bool isCoarrayIntrinsic(llvm::StringRef name) {
   return name.starts_with("atomic_") || name.starts_with("co_") ||
          name.contains("image") || name.ends_with("cobound") ||
-         name.equals("team_number");
+         name == "team_number";
 }
 
 /// Return the generic name of an intrinsic module procedure specific name.
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index b4705aa47992..21154902d23f 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -3497,7 +3497,7 @@ public:
     rewriter.startOpModification(op);
     auto callee = op.getCallee();
     if (callee)
-      if (callee->equals("hypotf"))
+      if (*callee == "hypotf")
         op.setCalleeAttr(mlir::SymbolRefAttr::get(op.getContext(), "_hypotf"));
 
     rewriter.finalizeOpModification(op);
@@ -3514,7 +3514,7 @@ public:
   matchAndRewrite(mlir::LLVM::LLVMFuncOp op,
                   mlir::PatternRewriter &rewriter) const override {
     rewriter.startOpModification(op);
-    if (op.getSymName().equals("hypotf"))
+    if (op.getSymName() == "hypotf")
       op.setSymNameAttr(rewriter.getStringAttr("_hypotf"));
     rewriter.finalizeOpModification(op);
     return mlir::success();
@@ -3629,11 +3629,11 @@ public:
             auto callee = op.getCallee();
             if (!callee)
               return true;
-            return !callee->equals("hypotf");
+            return *callee != "hypotf";
           });
       target.addDynamicallyLegalOp<mlir::LLVM::LLVMFuncOp>(
           [](mlir::LLVM::LLVMFuncOp op) {
-            return !op.getSymName().equals("hypotf");
+            return op.getSymName() != "hypotf";
           });
     }
 
diff --git a/lld/COFF/DebugTypes.cpp b/lld/COFF/DebugTypes.cpp
index a4c808e4c9a0..7689ad163a65 100644
--- a/lld/COFF/DebugTypes.cpp
+++ b/lld/COFF/DebugTypes.cpp
@@ -465,7 +465,7 @@ static bool equalsPath(StringRef path1, StringRef path2) {
 #if defined(_WIN32)
   return path1.equals_insensitive(path2);
 #else
-  return path1.equals(path2);
+  return path1 == path2;
 #endif
 }
 
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index fa48552b8f7a..fa81611e7c9e 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -1128,7 +1128,7 @@ void InputSectionBase::adjustSplitStackFunctionPrologues(uint8_t *buf,
   for (Relocation &rel : relocs()) {
     // Ignore calls into the split-stack api.
     if (rel.sym->getName().starts_with("__morestack")) {
-      if (rel.sym->getName().equals("__morestack"))
+      if (rel.sym->getName() == "__morestack")
         morestackCalls.push_back(&rel);
       continue;
     }
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 240c16a4d8f6..e400ed2ae945 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -577,7 +577,7 @@ static bool isRelroSection(const OutputSection *sec) {
   // for accessing .got as well, .got and .toc need to be close enough in the
   // virtual address space. Usually, .toc comes just after .got. Since we place
   // .got into RELRO, .toc needs to be placed into RELRO too.
-  if (sec->name.equals(".toc"))
+  if (sec->name == ".toc")
     return true;
 
   // .got.plt contains pointers to external function symbols. They are
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 65de531db04b..d4d8d53d69ee 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -1507,7 +1507,7 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
       StringRef sep = sys::path::get_separator();
       // real_path removes trailing slashes as part of the normalization, but
       // these are meaningful for our text based stripping
-      if (config->osoPrefix.equals(".") || config->osoPrefix.ends_with(sep))
+      if (config->osoPrefix == "." || config->osoPrefix.ends_with(sep))
         expanded += sep;
       config->osoPrefix = saver().save(expanded.str());
     }
diff --git a/lld/wasm/InputChunks.cpp b/lld/wasm/InputChunks.cpp
index 2074dd59c1dd..975225974aff 100644
--- a/lld/wasm/InputChunks.cpp
+++ b/lld/wasm/InputChunks.cpp
@@ -519,8 +519,8 @@ uint64_t InputSection::getTombstoneForSection(StringRef name) {
   // If they occur in DWARF debug symbols, we want to change the pc of the
   // function to -1 to avoid overlapping with a valid range. However for the
   // debug_ranges and debug_loc sections that would conflict with the existing
-  // meaning of -1 so we use -2.  
-  if (name.equals(".debug_ranges") || name.equals(".debug_loc"))
+  // meaning of -1 so we use -2.
+  if (name == ".debug_ranges" || name == ".debug_loc")
     return UINT64_C(-2);
   if (name.starts_with(".debug_"))
     return UINT64_C(-1);
-- 
GitLab


From 1fadb2b0c881ced247931f442fdee6c4ed96dccb Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Sun, 12 May 2024 23:06:06 -0700
Subject: [PATCH 0519/1206] Revert "[clang-format] Fix
 FormatToken::isSimpleTypeSpecifier() (#91712)"

This reverts commits e62ce1f8842c, 5cd280433e8e, and de641e289269 due to
buildbot failures.
---
 clang/lib/Format/Format.cpp                   |  3 +-
 clang/lib/Format/FormatToken.cpp              | 46 +++++++++-
 clang/lib/Format/FormatToken.h                |  8 +-
 clang/lib/Format/FormatTokenLexer.cpp         |  1 +
 clang/lib/Format/QualifierAlignmentFixer.cpp  | 24 ++---
 clang/lib/Format/QualifierAlignmentFixer.h    |  5 +-
 clang/lib/Format/TokenAnalyzer.cpp            |  4 +-
 clang/lib/Format/TokenAnalyzer.h              |  1 -
 clang/lib/Format/TokenAnnotator.cpp           | 43 ++++-----
 clang/lib/Format/TokenAnnotator.h             |  6 +-
 clang/lib/Format/UnwrappedLineParser.cpp      | 19 ++--
 clang/lib/Format/UnwrappedLineParser.h        |  1 -
 clang/unittests/Format/QualifierFixerTest.cpp | 92 ++++++++-----------
 13 files changed, 134 insertions(+), 119 deletions(-)

diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 52005a6c881f..8f027ffa20cc 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -3858,7 +3858,8 @@ LangOptions getFormattingLangOpts(const FormatStyle &Style) {
   LangOpts.Digraphs = LexingStd >= FormatStyle::LS_Cpp11;
 
   LangOpts.LineComment = 1;
-  LangOpts.CXXOperatorNames = Style.isCpp();
+  bool AlternativeOperators = Style.isCpp();
+  LangOpts.CXXOperatorNames = AlternativeOperators ? 1 : 0;
   LangOpts.Bool = 1;
   LangOpts.ObjC = 1;
   LangOpts.MicrosoftExt = 1;    // To get kw___try, kw___finally.
diff --git a/clang/lib/Format/FormatToken.cpp b/clang/lib/Format/FormatToken.cpp
index 85bec71ffbbc..4fb70ffac706 100644
--- a/clang/lib/Format/FormatToken.cpp
+++ b/clang/lib/Format/FormatToken.cpp
@@ -34,6 +34,43 @@ const char *getTokenTypeName(TokenType Type) {
   return nullptr;
 }
 
+// FIXME: This is copy&pasted from Sema. Put it in a common place and remove
+// duplication.
+bool FormatToken::isSimpleTypeSpecifier() const {
+  switch (Tok.getKind()) {
+  case tok::kw_short:
+  case tok::kw_long:
+  case tok::kw___int64:
+  case tok::kw___int128:
+  case tok::kw_signed:
+  case tok::kw_unsigned:
+  case tok::kw_void:
+  case tok::kw_char:
+  case tok::kw_int:
+  case tok::kw_half:
+  case tok::kw_float:
+  case tok::kw_double:
+  case tok::kw___bf16:
+  case tok::kw__Float16:
+  case tok::kw___float128:
+  case tok::kw___ibm128:
+  case tok::kw_wchar_t:
+  case tok::kw_bool:
+#define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) case tok::kw___##Trait:
+#include "clang/Basic/TransformTypeTraits.def"
+  case tok::annot_typename:
+  case tok::kw_char8_t:
+  case tok::kw_char16_t:
+  case tok::kw_char32_t:
+  case tok::kw_typeof:
+  case tok::kw_decltype:
+  case tok::kw__Atomic:
+    return true;
+  default:
+    return false;
+  }
+}
+
 // Sorted common C++ non-keyword types.
 static SmallVector<StringRef> CppNonKeywordTypes = {
     "clock_t",  "int16_t",   "int32_t", "int64_t",   "int8_t",
@@ -41,16 +78,15 @@ static SmallVector<StringRef> CppNonKeywordTypes = {
     "uint32_t", "uint64_t",  "uint8_t", "uintptr_t",
 };
 
-bool FormatToken::isTypeName(const LangOptions &LangOpts) const {
-  const bool IsCpp = LangOpts.CXXOperatorNames;
-  return is(TT_TypeName) || Tok.isSimpleTypeSpecifier(LangOpts) ||
+bool FormatToken::isTypeName(bool IsCpp) const {
+  return is(TT_TypeName) || isSimpleTypeSpecifier() ||
          (IsCpp && is(tok::identifier) &&
           std::binary_search(CppNonKeywordTypes.begin(),
                              CppNonKeywordTypes.end(), TokenText));
 }
 
-bool FormatToken::isTypeOrIdentifier(const LangOptions &LangOpts) const {
-  return isTypeName(LangOpts) || isOneOf(tok::kw_auto, tok::identifier);
+bool FormatToken::isTypeOrIdentifier(bool IsCpp) const {
+  return isTypeName(IsCpp) || isOneOf(tok::kw_auto, tok::identifier);
 }
 
 bool FormatToken::isBlockIndentedInitRBrace(const FormatStyle &Style) const {
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 8792f4c75074..95f16fde5005 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -684,8 +684,12 @@ public:
            isAttribute();
   }
 
-  [[nodiscard]] bool isTypeName(const LangOptions &LangOpts) const;
-  [[nodiscard]] bool isTypeOrIdentifier(const LangOptions &LangOpts) const;
+  /// Determine whether the token is a simple-type-specifier.
+  [[nodiscard]] bool isSimpleTypeSpecifier() const;
+
+  [[nodiscard]] bool isTypeName(bool IsCpp) const;
+
+  [[nodiscard]] bool isTypeOrIdentifier(bool IsCpp) const;
 
   bool isObjCAccessSpecifier() const {
     return is(tok::at) && Next &&
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index e21b5a882b77..f430d3764bab 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -1442,6 +1442,7 @@ void FormatTokenLexer::readRawToken(FormatToken &Tok) {
 
 void FormatTokenLexer::resetLexer(unsigned Offset) {
   StringRef Buffer = SourceMgr.getBufferData(ID);
+  LangOpts = getFormattingLangOpts(Style);
   Lex.reset(new Lexer(SourceMgr.getLocForStartOfFile(ID), LangOpts,
                       Buffer.begin(), Buffer.begin() + Offset, Buffer.end()));
   Lex->SetKeepWhitespaceMode(true);
diff --git a/clang/lib/Format/QualifierAlignmentFixer.cpp b/clang/lib/Format/QualifierAlignmentFixer.cpp
index a904f0b773c6..c26353045672 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.cpp
+++ b/clang/lib/Format/QualifierAlignmentFixer.cpp
@@ -268,11 +268,13 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeRight(
   if (isPossibleMacro(TypeToken))
     return Tok;
 
+  const bool IsCpp = Style.isCpp();
+
   // The case `const long long int volatile` -> `long long int const volatile`
   // The case `long const long int volatile` -> `long long int const volatile`
   // The case `long long volatile int const` -> `long long int const volatile`
   // The case `const long long volatile int` -> `long long int const volatile`
-  if (TypeToken->isTypeName(LangOpts)) {
+  if (TypeToken->isTypeName(IsCpp)) {
     // The case `const decltype(foo)` -> `const decltype(foo)`
     // The case `const typeof(foo)` -> `const typeof(foo)`
     // The case `const _Atomic(foo)` -> `const _Atomic(foo)`
@@ -281,7 +283,7 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeRight(
 
     const FormatToken *LastSimpleTypeSpecifier = TypeToken;
     while (isQualifierOrType(LastSimpleTypeSpecifier->getNextNonComment(),
-                             LangOpts)) {
+                             IsCpp)) {
       LastSimpleTypeSpecifier = LastSimpleTypeSpecifier->getNextNonComment();
     }
 
@@ -293,7 +295,7 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeRight(
   // The case  `unsigned short const` -> `unsigned short const`
   // The case:
   // `unsigned short volatile const` -> `unsigned short const volatile`
-  if (PreviousCheck && PreviousCheck->isTypeName(LangOpts)) {
+  if (PreviousCheck && PreviousCheck->isTypeName(IsCpp)) {
     if (LastQual != Tok)
       rotateTokens(SourceMgr, Fixes, Tok, LastQual, /*Left=*/false);
     return Tok;
@@ -410,11 +412,11 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeLeft(
   // The case `volatile long long const int` -> `const volatile long long int`
   // The case `const long long volatile int` -> `const volatile long long int`
   // The case `long volatile long int const` -> `const volatile long long int`
-  if (TypeToken->isTypeName(LangOpts)) {
+  if (const bool IsCpp = Style.isCpp(); TypeToken->isTypeName(IsCpp)) {
     const FormatToken *LastSimpleTypeSpecifier = TypeToken;
     while (isConfiguredQualifierOrType(
         LastSimpleTypeSpecifier->getPreviousNonComment(),
-        ConfiguredQualifierTokens, LangOpts)) {
+        ConfiguredQualifierTokens, IsCpp)) {
       LastSimpleTypeSpecifier =
           LastSimpleTypeSpecifier->getPreviousNonComment();
     }
@@ -612,16 +614,16 @@ void prepareLeftRightOrderingForQualifierAlignmentFixer(
   }
 }
 
-bool LeftRightQualifierAlignmentFixer::isQualifierOrType(
-    const FormatToken *Tok, const LangOptions &LangOpts) {
-  return Tok && (Tok->isTypeName(LangOpts) || Tok->is(tok::kw_auto) ||
-                 isQualifier(Tok));
+bool LeftRightQualifierAlignmentFixer::isQualifierOrType(const FormatToken *Tok,
+                                                         bool IsCpp) {
+  return Tok &&
+         (Tok->isTypeName(IsCpp) || Tok->is(tok::kw_auto) || isQualifier(Tok));
 }
 
 bool LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
     const FormatToken *Tok, const std::vector<tok::TokenKind> &Qualifiers,
-    const LangOptions &LangOpts) {
-  return Tok && (Tok->isTypeName(LangOpts) || Tok->is(tok::kw_auto) ||
+    bool IsCpp) {
+  return Tok && (Tok->isTypeName(IsCpp) || Tok->is(tok::kw_auto) ||
                  isConfiguredQualifier(Tok, Qualifiers));
 }
 
diff --git a/clang/lib/Format/QualifierAlignmentFixer.h b/clang/lib/Format/QualifierAlignmentFixer.h
index 710fa2dc0030..e1cc27e62b13 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.h
+++ b/clang/lib/Format/QualifierAlignmentFixer.h
@@ -71,12 +71,11 @@ public:
                                  tok::TokenKind QualifierType);
 
   // Is the Token a simple or qualifier type
-  static bool isQualifierOrType(const FormatToken *Tok,
-                                const LangOptions &LangOpts);
+  static bool isQualifierOrType(const FormatToken *Tok, bool IsCpp = true);
   static bool
   isConfiguredQualifierOrType(const FormatToken *Tok,
                               const std::vector<tok::TokenKind> &Qualifiers,
-                              const LangOptions &LangOpts);
+                              bool IsCpp = true);
 
   // Is the Token likely a Macro
   static bool isPossibleMacro(const FormatToken *Tok);
diff --git a/clang/lib/Format/TokenAnalyzer.cpp b/clang/lib/Format/TokenAnalyzer.cpp
index 804a2b0f5e8c..bd648c430f9b 100644
--- a/clang/lib/Format/TokenAnalyzer.cpp
+++ b/clang/lib/Format/TokenAnalyzer.cpp
@@ -84,7 +84,7 @@ Environment::Environment(StringRef Code, StringRef FileName,
       NextStartColumn(NextStartColumn), LastStartColumn(LastStartColumn) {}
 
 TokenAnalyzer::TokenAnalyzer(const Environment &Env, const FormatStyle &Style)
-    : Style(Style), LangOpts(getFormattingLangOpts(Style)), Env(Env),
+    : Style(Style), Env(Env),
       AffectedRangeMgr(Env.getSourceManager(), Env.getCharRanges()),
       UnwrappedLines(1),
       Encoding(encoding::detectEncoding(
@@ -101,7 +101,7 @@ std::pair<tooling::Replacements, unsigned>
 TokenAnalyzer::process(bool SkipAnnotation) {
   tooling::Replacements Result;
   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
-  IdentifierTable IdentTable(LangOpts);
+  IdentifierTable IdentTable(getFormattingLangOpts(Style));
   FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(),
                        Env.getFirstStartColumn(), Style, Encoding, Allocator,
                        IdentTable);
diff --git a/clang/lib/Format/TokenAnalyzer.h b/clang/lib/Format/TokenAnalyzer.h
index ef559099d325..b7494c395c8a 100644
--- a/clang/lib/Format/TokenAnalyzer.h
+++ b/clang/lib/Format/TokenAnalyzer.h
@@ -92,7 +92,6 @@ protected:
   void finishRun() override;
 
   FormatStyle Style;
-  LangOptions LangOpts;
   // Stores Style, FileID and SourceManager etc.
   const Environment &Env;
   // AffectedRangeMgr stores ranges to be fixed.
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 478cae23d3c8..e935d3e2709c 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -126,9 +126,7 @@ public:
                    const AdditionalKeywords &Keywords,
                    SmallVector<ScopeType> &Scopes)
       : Style(Style), Line(Line), CurrentToken(Line.First), AutoFound(false),
-        IsCpp(Style.isCpp()), LangOpts(getFormattingLangOpts(Style)),
-        Keywords(Keywords), Scopes(Scopes) {
-    assert(IsCpp == LangOpts.CXXOperatorNames);
+        IsCpp(Style.isCpp()), Keywords(Keywords), Scopes(Scopes) {
     Contexts.push_back(Context(tok::unknown, 1, /*IsExpression=*/false));
     resetTokenMetadata();
   }
@@ -564,7 +562,7 @@ private:
           (CurrentToken->is(tok::l_paren) && CurrentToken->Next &&
            CurrentToken->Next->isOneOf(tok::star, tok::amp, tok::caret));
       if ((CurrentToken->Previous->isOneOf(tok::kw_const, tok::kw_auto) ||
-           CurrentToken->Previous->isTypeName(LangOpts)) &&
+           CurrentToken->Previous->isTypeName(IsCpp)) &&
           !(CurrentToken->is(tok::l_brace) ||
             (CurrentToken->is(tok::l_paren) && !ProbablyFunctionTypeLParen))) {
         Contexts.back().IsExpression = false;
@@ -2626,7 +2624,7 @@ private:
       return true;
 
     // MyClass a;
-    if (PreviousNotConst->isTypeName(LangOpts))
+    if (PreviousNotConst->isTypeName(IsCpp))
       return true;
 
     // type[] a in Java
@@ -2730,7 +2728,7 @@ private:
     }
 
     if (Tok.Next->is(tok::question) ||
-        (Tok.Next->is(tok::ampamp) && !Tok.Previous->isTypeName(LangOpts))) {
+        (Tok.Next->is(tok::ampamp) && !Tok.Previous->isTypeName(IsCpp))) {
       return false;
     }
 
@@ -2759,10 +2757,9 @@ private:
     }
 
     // Heuristically try to determine whether the parentheses contain a type.
-    auto IsQualifiedPointerOrReference = [](FormatToken *T,
-                                            const LangOptions &LangOpts) {
+    auto IsQualifiedPointerOrReference = [](FormatToken *T, bool IsCpp) {
       // This is used to handle cases such as x = (foo *const)&y;
-      assert(!T->isTypeName(LangOpts) && "Should have already been checked");
+      assert(!T->isTypeName(IsCpp) && "Should have already been checked");
       // Strip trailing qualifiers such as const or volatile when checking
       // whether the parens could be a cast to a pointer/reference type.
       while (T) {
@@ -2794,8 +2791,8 @@ private:
     bool ParensAreType =
         !Tok.Previous ||
         Tok.Previous->isOneOf(TT_TemplateCloser, TT_TypeDeclarationParen) ||
-        Tok.Previous->isTypeName(LangOpts) ||
-        IsQualifiedPointerOrReference(Tok.Previous, LangOpts);
+        Tok.Previous->isTypeName(IsCpp) ||
+        IsQualifiedPointerOrReference(Tok.Previous, IsCpp);
     bool ParensCouldEndDecl =
         Tok.Next->isOneOf(tok::equal, tok::semi, tok::l_brace, tok::greater);
     if (ParensAreType && !ParensCouldEndDecl)
@@ -3068,7 +3065,6 @@ private:
   FormatToken *CurrentToken;
   bool AutoFound;
   bool IsCpp;
-  LangOptions LangOpts;
   const AdditionalKeywords &Keywords;
 
   SmallVector<ScopeType> &Scopes;
@@ -3643,8 +3639,7 @@ void TokenAnnotator::annotate(AnnotatedLine &Line) {
 
 // This function heuristically determines whether 'Current' starts the name of a
 // function declaration.
-static bool isFunctionDeclarationName(const LangOptions &LangOpts,
-                                      const FormatToken &Current,
+static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
                                       const AnnotatedLine &Line,
                                       FormatToken *&ClosingParen) {
   assert(Current.Previous);
@@ -3663,7 +3658,7 @@ static bool isFunctionDeclarationName(const LangOptions &LangOpts,
   }
 
   auto skipOperatorName =
-      [&LangOpts](const FormatToken *Next) -> const FormatToken * {
+      [IsCpp](const FormatToken *Next) -> const FormatToken * {
     for (; Next; Next = Next->Next) {
       if (Next->is(TT_OverloadedOperatorLParen))
         return Next;
@@ -3682,7 +3677,7 @@ static bool isFunctionDeclarationName(const LangOptions &LangOpts,
         Next = Next->Next;
         continue;
       }
-      if ((Next->isTypeName(LangOpts) || Next->is(tok::identifier)) &&
+      if ((Next->isTypeName(IsCpp) || Next->is(tok::identifier)) &&
           Next->Next && Next->Next->isPointerOrReference()) {
         // For operator void*(), operator char*(), operator Foo*().
         Next = Next->Next;
@@ -3698,10 +3693,8 @@ static bool isFunctionDeclarationName(const LangOptions &LangOpts,
     return nullptr;
   };
 
-  const auto *Next = Current.Next;
-  const bool IsCpp = LangOpts.CXXOperatorNames;
-
   // Find parentheses of parameter list.
+  const FormatToken *Next = Current.Next;
   if (Current.is(tok::kw_operator)) {
     if (Previous.Tok.getIdentifierInfo() &&
         !Previous.isOneOf(tok::kw_return, tok::kw_co_return)) {
@@ -3781,7 +3774,7 @@ static bool isFunctionDeclarationName(const LangOptions &LangOpts,
       Tok = Tok->MatchingParen;
       continue;
     }
-    if (Tok->is(tok::kw_const) || Tok->isTypeName(LangOpts) ||
+    if (Tok->is(tok::kw_const) || Tok->isTypeName(IsCpp) ||
         Tok->isOneOf(TT_PointerOrReference, TT_StartOfName, tok::ellipsis)) {
       return true;
     }
@@ -3844,7 +3837,7 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) const {
       AfterLastAttribute = Tok;
     if (const bool IsCtorOrDtor = Tok->is(TT_CtorDtorDeclName);
         IsCtorOrDtor ||
-        isFunctionDeclarationName(LangOpts, *Tok, Line, ClosingParen)) {
+        isFunctionDeclarationName(IsCpp, *Tok, Line, ClosingParen)) {
       if (!IsCtorOrDtor)
         Tok->setFinalizedType(TT_FunctionDeclarationName);
       LineIsFunctionDeclaration = true;
@@ -4454,7 +4447,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     if (Left.Tok.isLiteral())
       return true;
     // for (auto a = 0, b = 0; const auto & c : {1, 2, 3})
-    if (Left.isTypeOrIdentifier(LangOpts) && Right.Next && Right.Next->Next &&
+    if (Left.isTypeOrIdentifier(IsCpp) && Right.Next && Right.Next->Next &&
         Right.Next->Next->is(TT_RangeBasedForLoopColon)) {
       return getTokenPointerOrReferenceAlignment(Right) !=
              FormatStyle::PAS_Left;
@@ -4497,7 +4490,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     if (Right.is(tok::l_brace) && Right.is(BK_Block))
       return true;
     // for (auto a = 0, b = 0; const auto& c : {1, 2, 3})
-    if (BeforeLeft && BeforeLeft->isTypeOrIdentifier(LangOpts) && Right.Next &&
+    if (BeforeLeft && BeforeLeft->isTypeOrIdentifier(IsCpp) && Right.Next &&
         Right.Next->is(TT_RangeBasedForLoopColon)) {
       return getTokenPointerOrReferenceAlignment(Left) !=
              FormatStyle::PAS_Right;
@@ -4541,7 +4534,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   if (Right.isPointerOrReference()) {
     const FormatToken *Previous = &Left;
     while (Previous && Previous->isNot(tok::kw_operator)) {
-      if (Previous->is(tok::identifier) || Previous->isTypeName(LangOpts)) {
+      if (Previous->is(tok::identifier) || Previous->isTypeName(IsCpp)) {
         Previous = Previous->getPreviousNonComment();
         continue;
       }
@@ -4730,7 +4723,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   if (!Style.isVerilog() &&
       (Left.isOneOf(tok::identifier, tok::greater, tok::r_square,
                     tok::r_paren) ||
-       Left.isTypeName(LangOpts)) &&
+       Left.isTypeName(IsCpp)) &&
       Right.is(tok::l_brace) && Right.getNextNonComment() &&
       Right.isNot(BK_Block)) {
     return false;
diff --git a/clang/lib/Format/TokenAnnotator.h b/clang/lib/Format/TokenAnnotator.h
index d19d3d061e40..25a24dccb1b8 100644
--- a/clang/lib/Format/TokenAnnotator.h
+++ b/clang/lib/Format/TokenAnnotator.h
@@ -211,10 +211,7 @@ private:
 class TokenAnnotator {
 public:
   TokenAnnotator(const FormatStyle &Style, const AdditionalKeywords &Keywords)
-      : Style(Style), IsCpp(Style.isCpp()),
-        LangOpts(getFormattingLangOpts(Style)), Keywords(Keywords) {
-    assert(IsCpp == LangOpts.CXXOperatorNames);
-  }
+      : Style(Style), IsCpp(Style.isCpp()), Keywords(Keywords) {}
 
   /// Adapts the indent levels of comment lines to the indent of the
   /// subsequent line.
@@ -263,7 +260,6 @@ private:
   const FormatStyle &Style;
 
   bool IsCpp;
-  LangOptions LangOpts;
 
   const AdditionalKeywords &Keywords;
 
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 4f1c2c5114e9..310b75485e08 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -160,16 +160,13 @@ UnwrappedLineParser::UnwrappedLineParser(
     IdentifierTable &IdentTable)
     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
       CurrentLines(&Lines), Style(Style), IsCpp(Style.isCpp()),
-      LangOpts(getFormattingLangOpts(Style)), Keywords(Keywords),
-      CommentPragmasRegex(Style.CommentPragmas), Tokens(nullptr),
-      Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1),
+      Keywords(Keywords), CommentPragmasRegex(Style.CommentPragmas),
+      Tokens(nullptr), Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1),
       IncludeGuard(Style.IndentPPDirectives == FormatStyle::PPDIS_None
                        ? IG_Rejected
                        : IG_Inited),
       IncludeGuardToken(nullptr), FirstStartColumn(FirstStartColumn),
-      Macros(Style.Macros, SourceMgr, Style, Allocator, IdentTable) {
-  assert(IsCpp == LangOpts.CXXOperatorNames);
-}
+      Macros(Style.Macros, SourceMgr, Style, Allocator, IdentTable) {}
 
 void UnwrappedLineParser::reset() {
   PPBranchLevel = -1;
@@ -1873,7 +1870,7 @@ void UnwrappedLineParser::parseStructuralElement(
     case tok::caret:
       nextToken();
       // Block return type.
-      if (FormatTok->Tok.isAnyIdentifier() || FormatTok->isTypeName(LangOpts)) {
+      if (FormatTok->Tok.isAnyIdentifier() || FormatTok->isTypeName(IsCpp)) {
         nextToken();
         // Return types: pointers are ok too.
         while (FormatTok->is(tok::star))
@@ -2234,7 +2231,7 @@ bool UnwrappedLineParser::tryToParseLambda() {
   bool InTemplateParameterList = false;
 
   while (FormatTok->isNot(tok::l_brace)) {
-    if (FormatTok->isTypeName(LangOpts)) {
+    if (FormatTok->isTypeName(IsCpp)) {
       nextToken();
       continue;
     }
@@ -3451,7 +3448,7 @@ bool UnwrappedLineParser::parseRequires() {
     break;
   }
   default:
-    if (PreviousNonComment->isTypeOrIdentifier(LangOpts)) {
+    if (PreviousNonComment->isTypeOrIdentifier(IsCpp)) {
       // This is a requires clause.
       parseRequiresClause(RequiresToken);
       return true;
@@ -3514,7 +3511,7 @@ bool UnwrappedLineParser::parseRequires() {
       --OpenAngles;
       break;
     default:
-      if (NextToken->isTypeName(LangOpts)) {
+      if (NextToken->isTypeName(IsCpp)) {
         FormatTok = Tokens->setPosition(StoredPosition);
         parseRequiresExpression(RequiresToken);
         return false;
@@ -4030,7 +4027,7 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
       if (FormatTok->is(tok::l_square)) {
         FormatToken *Previous = FormatTok->Previous;
         if (!Previous || (Previous->isNot(tok::r_paren) &&
-                          !Previous->isTypeOrIdentifier(LangOpts))) {
+                          !Previous->isTypeOrIdentifier(IsCpp))) {
           // Don't try parsing a lambda if we had a closing parenthesis before,
           // it was probably a pointer to an array: int (*)[].
           if (!tryToParseLambda())
diff --git a/clang/lib/Format/UnwrappedLineParser.h b/clang/lib/Format/UnwrappedLineParser.h
index d7963a4211bb..2a0fe19d0957 100644
--- a/clang/lib/Format/UnwrappedLineParser.h
+++ b/clang/lib/Format/UnwrappedLineParser.h
@@ -316,7 +316,6 @@ private:
 
   const FormatStyle &Style;
   bool IsCpp;
-  LangOptions LangOpts;
   const AdditionalKeywords &Keywords;
 
   llvm::Regex CommentPragmasRegex;
diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index fdc392e5d948..792d8f3c3a98 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -1055,82 +1055,70 @@ TEST_F(QualifierFixerTest, IsQualifierType) {
   ConfiguredTokens.push_back(tok::kw_constexpr);
   ConfiguredTokens.push_back(tok::kw_friend);
 
-  static const LangOptions LangOpts{getFormattingLangOpts()};
-
   auto Tokens = annotate(
       "const static inline auto restrict int double long constexpr friend");
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
 
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[0], ConfiguredTokens, LangOpts));
+      Tokens[0], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[1], ConfiguredTokens, LangOpts));
+      Tokens[1], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[2], ConfiguredTokens, LangOpts));
+      Tokens[2], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[3], ConfiguredTokens, LangOpts));
+      Tokens[3], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[4], ConfiguredTokens, LangOpts));
+      Tokens[4], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[5], ConfiguredTokens, LangOpts));
+      Tokens[5], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[6], ConfiguredTokens, LangOpts));
+      Tokens[6], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[7], ConfiguredTokens, LangOpts));
+      Tokens[7], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[8], ConfiguredTokens, LangOpts));
+      Tokens[8], ConfiguredTokens));
   EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[9], ConfiguredTokens, LangOpts));
-
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[0], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[1], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[2], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[3], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[4], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[5], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[6], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[7], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[8], LangOpts));
-  EXPECT_TRUE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[9], LangOpts));
+      Tokens[9], ConfiguredTokens));
+
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[0]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[1]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[2]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[3]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[4]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[5]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[6]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[7]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[8]));
+  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[9]));
 
   auto NotTokens = annotate("for while do Foo Bar ");
   ASSERT_EQ(NotTokens.size(), 6u) << Tokens;
 
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[0], ConfiguredTokens, LangOpts));
+      NotTokens[0], ConfiguredTokens));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[1], ConfiguredTokens, LangOpts));
+      NotTokens[1], ConfiguredTokens));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[2], ConfiguredTokens, LangOpts));
+      NotTokens[2], ConfiguredTokens));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[3], ConfiguredTokens, LangOpts));
+      NotTokens[3], ConfiguredTokens));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[4], ConfiguredTokens, LangOpts));
+      NotTokens[4], ConfiguredTokens));
   EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[5], ConfiguredTokens, LangOpts));
-
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[0],
-                                                                   LangOpts));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[1],
-                                                                   LangOpts));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[2],
-                                                                   LangOpts));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[3],
-                                                                   LangOpts));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[4],
-                                                                   LangOpts));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[5],
-                                                                   LangOpts));
+      NotTokens[5], ConfiguredTokens));
+
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[0]));
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[1]));
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[2]));
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[3]));
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[4]));
+  EXPECT_FALSE(
+      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[5]));
 }
 
 TEST_F(QualifierFixerTest, IsMacro) {
-- 
GitLab


From b5f4210e9f51f938ae517f219f04f9ab431a2684 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Mon, 13 May 2024 14:27:59 +0800
Subject: [PATCH 0520/1206] [InstCombine] Drop nuw flag when CtlzOp is a sub
 nuw (#91776)

See the following case:
```
define i32 @src1(i32 %x) {
  %dec = sub nuw i32 -2, %x
  %ctlz = tail call i32 @llvm.ctlz.i32(i32 %dec, i1 false)
  %sub = sub nsw i32 32, %ctlz
  %shl = shl i32 1, %sub
  %ugt = icmp ult i32 %x, -2
  %sel = select i1 %ugt, i32 %shl, i32 1
  ret i32 %sel
}

define i32 @tgt1(i32 %x) {
  %dec = sub nuw i32 -2, %x
  %ctlz = tail call i32 @llvm.ctlz.i32(i32 %dec, i1 false)
  %sub = sub nsw i32 32, %ctlz
  %and = and i32 %sub, 31
  %shl = shl nuw i32 1, %and
  ret i32 %shl
}
```
`nuw` in `%dec` should be dropped after the select instruction is
eliminated.

Alive2: https://alive2.llvm.org/ce/z/7S9529

Fixes https://github.com/llvm/llvm-project/issues/91691.
---
 .../InstCombine/InstCombineSelect.cpp         | 14 ++++++--
 llvm/test/Transforms/InstCombine/bit_ceil.ll  | 36 +++++++++++++++++++
 2 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index ee090e012508..a3ddb402bf66 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -3343,7 +3343,8 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) {
 // pattern.
 static bool isSafeToRemoveBitCeilSelect(ICmpInst::Predicate Pred, Value *Cond0,
                                         const APInt *Cond1, Value *CtlzOp,
-                                        unsigned BitWidth) {
+                                        unsigned BitWidth,
+                                        bool &ShouldDropNUW) {
   // The challenge in recognizing std::bit_ceil(X) is that the operand is used
   // for the CTLZ proper and select condition, each possibly with some
   // operation like add and sub.
@@ -3366,6 +3367,8 @@ static bool isSafeToRemoveBitCeilSelect(ICmpInst::Predicate Pred, Value *Cond0,
   ConstantRange CR = ConstantRange::makeExactICmpRegion(
       CmpInst::getInversePredicate(Pred), *Cond1);
 
+  ShouldDropNUW = false;
+
   // Match the operation that's used to compute CtlzOp from CommonAncestor.  If
   // CtlzOp == CommonAncestor, return true as no operation is needed.  If a
   // match is found, execute the operation on CR, update CR, and return true.
@@ -3379,6 +3382,7 @@ static bool isSafeToRemoveBitCeilSelect(ICmpInst::Predicate Pred, Value *Cond0,
       return true;
     }
     if (match(CtlzOp, m_Sub(m_APInt(C), m_Specific(CommonAncestor)))) {
+      ShouldDropNUW = true;
       CR = ConstantRange(*C).sub(CR);
       return true;
     }
@@ -3448,14 +3452,20 @@ static Instruction *foldBitCeil(SelectInst &SI, IRBuilderBase &Builder) {
     Pred = CmpInst::getInversePredicate(Pred);
   }
 
+  bool ShouldDropNUW;
+
   if (!match(FalseVal, m_One()) ||
       !match(TrueVal,
              m_OneUse(m_Shl(m_One(), m_OneUse(m_Sub(m_SpecificInt(BitWidth),
                                                     m_Value(Ctlz)))))) ||
       !match(Ctlz, m_Intrinsic<Intrinsic::ctlz>(m_Value(CtlzOp), m_Zero())) ||
-      !isSafeToRemoveBitCeilSelect(Pred, Cond0, Cond1, CtlzOp, BitWidth))
+      !isSafeToRemoveBitCeilSelect(Pred, Cond0, Cond1, CtlzOp, BitWidth,
+                                   ShouldDropNUW))
     return nullptr;
 
+  if (ShouldDropNUW)
+    cast<Instruction>(CtlzOp)->setHasNoUnsignedWrap(false);
+
   // Build 1 << (-CTLZ & (BitWidth-1)).  The negation likely corresponds to a
   // single hardware instruction as opposed to BitWidth - CTLZ, where BitWidth
   // is an integer constant.  Masking with BitWidth-1 comes free on some
diff --git a/llvm/test/Transforms/InstCombine/bit_ceil.ll b/llvm/test/Transforms/InstCombine/bit_ceil.ll
index 16631afa4878..79665be01576 100644
--- a/llvm/test/Transforms/InstCombine/bit_ceil.ll
+++ b/llvm/test/Transforms/InstCombine/bit_ceil.ll
@@ -284,6 +284,42 @@ define <4 x i32> @bit_ceil_v4i32(<4 x i32> %x) {
   ret <4 x i32> %sel
 }
 
+define i32 @pr91691(i32 %0) {
+; CHECK-LABEL: @pr91691(
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 -2, [[TMP0:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[TMP2]], i1 false)
+; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw i32 0, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 31
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i32 1, [[TMP5]]
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+  %2 = sub nuw i32 -2, %0
+  %3 = tail call i32 @llvm.ctlz.i32(i32 %2, i1 false)
+  %4 = sub i32 32, %3
+  %5 = shl i32 1, %4
+  %6 = icmp ult i32 %0, -2
+  %7 = select i1 %6, i32 %5, i32 1
+  ret i32 %7
+}
+
+define i32 @pr91691_keep_nsw(i32 %0) {
+; CHECK-LABEL: @pr91691_keep_nsw(
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i32 -2, [[TMP0:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[TMP2]], i1 false)
+; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw i32 0, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 31
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i32 1, [[TMP5]]
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+  %2 = sub nsw i32 -2, %0
+  %3 = tail call i32 @llvm.ctlz.i32(i32 %2, i1 false)
+  %4 = sub i32 32, %3
+  %5 = shl i32 1, %4
+  %6 = icmp ult i32 %0, -2
+  %7 = select i1 %6, i32 %5, i32 1
+  ret i32 %7
+}
+
 declare i32 @llvm.ctlz.i32(i32, i1 immarg)
 declare i64 @llvm.ctlz.i64(i64, i1 immarg)
 declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1)
-- 
GitLab


From fd4efecac21d92428d2f804f43e85bdfa460bdd5 Mon Sep 17 00:00:00 2001
From: Kai Sasaki <lewuathe@gmail.com>
Date: Mon, 13 May 2024 15:31:41 +0900
Subject: [PATCH 0521/1206] [mlir] Fix warning due to non ISO standard
 __FUNCTION__ usage (#91851)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR fixes the warning message due to the non ISO standard usage of
`__FUNCTION__`

```
/home/lewuathe/llvm-project/mlir/test/CAPI/transform_interpreter.c: In function ‘testApplyNamedSequence’:
/home/lewuathe/llvm-project/mlir/test/CAPI/transform_interpreter.c:21:27: warning: ISO C does not support ‘__FUNCTION__’ predefined identifier [-Wpedantic]
   21 |   fprintf(stderr, "%s\n", __FUNCTION__);
      |
```

As `__FUNCTION__` is another name of `__func__` and it conforms to the
specification. We should be able to use `__func__` here.

Ref:
https://stackoverflow.com/questions/52962812/how-to-silence-gcc-pedantic-wpedantic-warning-regarding-function


Compiler

```
Ubuntu clang version 18.1.3 (1)
Target: x86_64-pc-linux-gnu
```
---
 mlir/test/CAPI/transform_interpreter.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/test/CAPI/transform_interpreter.c b/mlir/test/CAPI/transform_interpreter.c
index f1ab185e0e21..a849b2f24526 100644
--- a/mlir/test/CAPI/transform_interpreter.c
+++ b/mlir/test/CAPI/transform_interpreter.c
@@ -18,7 +18,7 @@
 #include <stdlib.h>
 
 int testApplyNamedSequence(MlirContext ctx) {
-  fprintf(stderr, "%s\n", __FUNCTION__);
+  fprintf(stderr, "%s\n", __func__);
 
   const char module[] =
       "module attributes {transform.with_named_sequence} {"
-- 
GitLab


From 5082feabd155a59d7ae0249713def100516b839d Mon Sep 17 00:00:00 2001
From: Alcaro <floating@muncher.se>
Date: Mon, 13 May 2024 08:33:39 +0200
Subject: [PATCH 0522/1206] [Documentation][Blocks ABI] Fix typoed integer
 (#65688)

Fix an integer value in the prose to match the rest of the content.
---
 clang/docs/Block-ABI-Apple.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/docs/Block-ABI-Apple.rst b/clang/docs/Block-ABI-Apple.rst
index 68f7a3819ca2..f46f2f991ad7 100644
--- a/clang/docs/Block-ABI-Apple.rst
+++ b/clang/docs/Block-ABI-Apple.rst
@@ -80,7 +80,7 @@ The following flags bits are in use thusly for a possible ABI.2010.3.16:
 In 10.6.ABI the (1<<29) was usually set and was always ignored by the runtime -
 it had been a transitional marker that did not get deleted after the
 transition. This bit is now paired with (1<<30), and represented as the pair
-(3<<30), for the following combinations of valid bit settings, and their
+(3<<29), for the following combinations of valid bit settings, and their
 meanings:
 
 .. code-block:: c
-- 
GitLab


From 5ca368501ae81ca364f66ee6053aa4f8104fdbdd Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Mon, 13 May 2024 06:44:37 +0000
Subject: [PATCH 0523/1206] [mlir][Bazel] Update BUILD file for
 1337622a492f4e77604b09ac8ff97042e46d8d42

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 6a7bc5c9fea0..6304b7b548d8 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -397,6 +397,7 @@ mlir_c_api_cc_library(
     name = "CAPIIR",
     srcs = [
         "lib/CAPI/Dialect/Func.cpp",
+        "lib/CAPI/Dialect/IRDL.cpp",
         "lib/CAPI/IR/AffineExpr.cpp",
         "lib/CAPI/IR/AffineMap.cpp",
         "lib/CAPI/IR/BuiltinAttributes.cpp",
@@ -415,6 +416,7 @@ mlir_c_api_cc_library(
         "include/mlir-c/BuiltinTypes.h",
         "include/mlir-c/Diagnostics.h",
         "include/mlir-c/Dialect/Func.h",
+        "include/mlir-c/Dialect/IRDL.h",
         "include/mlir-c/ExecutionEngine.h",
         "include/mlir-c/IR.h",
         "include/mlir-c/IntegerSet.h",
@@ -446,6 +448,7 @@ mlir_c_api_cc_library(
         ":AsmParser",
         ":ConversionPassIncGen",
         ":FuncDialect",
+        ":IRDLDialect",
         ":InferTypeOpInterface",
         ":Parser",
     ],
-- 
GitLab


From baa5beecc04aef9da258e31e4a9de1a27051ee5a Mon Sep 17 00:00:00 2001
From: tyb0807 <sontuan.vu119@gmail.com>
Date: Mon, 13 May 2024 09:08:04 +0200
Subject: [PATCH 0524/1206] [NFC] Make NVGPU casing consistent (#91903)

---
 mlir/lib/Bindings/Python/DialectNVGPU.cpp                 | 8 ++++----
 mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp           | 2 +-
 mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp | 6 +++---
 mlir/python/CMakeLists.txt                                | 2 +-
 mlir/python/mlir/dialects/nvgpu.py                        | 2 +-
 mlir/test/lib/Dialect/NVGPU/TestNVGPUTransforms.cpp       | 4 ++--
 mlir/tools/mlir-opt/mlir-opt.cpp                          | 4 ++--
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/mlir/lib/Bindings/Python/DialectNVGPU.cpp b/mlir/lib/Bindings/Python/DialectNVGPU.cpp
index 341e4d55bcf2..754e0a75b0ab 100644
--- a/mlir/lib/Bindings/Python/DialectNVGPU.cpp
+++ b/mlir/lib/Bindings/Python/DialectNVGPU.cpp
@@ -1,4 +1,4 @@
-//===--- DialectNvgpu.cpp - Pybind module for Nvgpu dialect API support ---===//
+//===--- DialectNVGPU.cpp - Pybind module for NVGPU dialect API support ---===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -17,7 +17,7 @@ using namespace mlir;
 using namespace mlir::python;
 using namespace mlir::python::adaptors;
 
-static void populateDialectNvgpuSubmodule(const pybind11::module &m) {
+static void populateDialectNVGPUSubmodule(const pybind11::module &m) {
   auto nvgpuTensorMapDescriptorType = mlir_type_subclass(
       m, "TensorMapDescriptorType", mlirTypeIsANVGPUTensorMapDescriptorType);
 
@@ -34,8 +34,8 @@ static void populateDialectNvgpuSubmodule(const pybind11::module &m) {
       py::arg("ctx") = py::none());
 }
 
-PYBIND11_MODULE(_mlirDialectsNvgpu, m) {
+PYBIND11_MODULE(_mlirDialectsNVGPU, m) {
   m.doc() = "MLIR NVGPU dialect.";
 
-  populateDialectNvgpuSubmodule(m);
+  populateDialectNVGPUSubmodule(m);
 }
diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
index 3f92372d7cea..782cc92f83fe 100644
--- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
+++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
@@ -441,7 +441,7 @@ struct PrepareContractToGPUMMA
   }
 };
 
-// Fold transpose op into the transfer read op. Nvgpu mma.sync op only supports
+// Fold transpose op into the transfer read op. NVGPU mma.sync op only supports
 // row-, column-, and row-major layout for matrixA, matrixB, and matrixC,
 // respectively. We can fold the transpose operation when loading the data from
 // Shared Memory to registers.
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
index 29a5bc9a7ae5..db085b386483 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
@@ -362,7 +362,7 @@ public:
 
 /// Folds nvgpu.device_async_copy subviews into the copy itself. This pattern
 /// is folds subview on src and dst memref of the copy.
-class NvgpuAsyncCopyOpSubViewOpFolder final
+class NVGPUAsyncCopyOpSubViewOpFolder final
     : public OpRewritePattern<nvgpu::DeviceAsyncCopyOp> {
 public:
   using OpRewritePattern<nvgpu::DeviceAsyncCopyOp>::OpRewritePattern;
@@ -694,7 +694,7 @@ LogicalResult StoreOpOfCollapseShapeOpFolder<OpTy>::matchAndRewrite(
   return success();
 }
 
-LogicalResult NvgpuAsyncCopyOpSubViewOpFolder::matchAndRewrite(
+LogicalResult NVGPUAsyncCopyOpSubViewOpFolder::matchAndRewrite(
     nvgpu::DeviceAsyncCopyOp copyOp, PatternRewriter &rewriter) const {
 
   LLVM_DEBUG(DBGS() << "copyOp       : " << copyOp << "\n");
@@ -769,7 +769,7 @@ void memref::populateFoldMemRefAliasOpPatterns(RewritePatternSet &patterns) {
                LoadOpOfCollapseShapeOpFolder<memref::LoadOp>,
                StoreOpOfCollapseShapeOpFolder<affine::AffineStoreOp>,
                StoreOpOfCollapseShapeOpFolder<memref::StoreOp>,
-               SubViewOfSubViewFolder, NvgpuAsyncCopyOpSubViewOpFolder>(
+               SubViewOfSubViewFolder, NVGPUAsyncCopyOpSubViewOpFolder>(
       patterns.getContext());
 }
 
diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt
index a6c78880c8e7..d8f2d1989fde 100644
--- a/mlir/python/CMakeLists.txt
+++ b/mlir/python/CMakeLists.txt
@@ -538,7 +538,7 @@ declare_mlir_python_extension(MLIRPythonExtension.Dialects.Quant.Pybind
 )
 
 declare_mlir_python_extension(MLIRPythonExtension.Dialects.NVGPU.Pybind
-  MODULE_NAME _mlirDialectsNvgpu
+  MODULE_NAME _mlirDialectsNVGPU
   ADD_TO_PARENT MLIRPythonSources.Dialects.nvgpu
   ROOT_DIR "${PYTHON_SOURCE_DIR}"
   SOURCES
diff --git a/mlir/python/mlir/dialects/nvgpu.py b/mlir/python/mlir/dialects/nvgpu.py
index e19bf610ea33..d6a54f2772f4 100644
--- a/mlir/python/mlir/dialects/nvgpu.py
+++ b/mlir/python/mlir/dialects/nvgpu.py
@@ -4,4 +4,4 @@
 
 from ._nvgpu_ops_gen import *
 from ._nvgpu_enum_gen import *
-from .._mlir_libs._mlirDialectsNvgpu import *
+from .._mlir_libs._mlirDialectsNVGPU import *
diff --git a/mlir/test/lib/Dialect/NVGPU/TestNVGPUTransforms.cpp b/mlir/test/lib/Dialect/NVGPU/TestNVGPUTransforms.cpp
index 74a15ba273d8..8ca29257b812 100644
--- a/mlir/test/lib/Dialect/NVGPU/TestNVGPUTransforms.cpp
+++ b/mlir/test/lib/Dialect/NVGPU/TestNVGPUTransforms.cpp
@@ -68,9 +68,9 @@ struct TestMmaSyncF32ToTF32Patterns
 
 namespace mlir {
 namespace test {
-void registerTestNvgpuLowerings() {
+void registerTestNVGPULowerings() {
   PassRegistration<TestMmaSyncF32ToTF32Patterns>();
 }
 
 } // namespace test
-} // namespace mlir
\ No newline at end of file
+} // namespace mlir
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 7e39cb62965e..1dfc5d178b61 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -143,7 +143,7 @@ void registerTestTransformDialectEraseSchedulePass();
 void registerTestWrittenToPass();
 void registerTestVectorLowerings();
 void registerTestVectorReductionToSPIRVDotProd();
-void registerTestNvgpuLowerings();
+void registerTestNVGPULowerings();
 #if MLIR_ENABLE_PDL_IN_PATTERNMATCH
 void registerTestDialectConversionPasses();
 void registerTestPDLByteCodePass();
@@ -270,7 +270,7 @@ void registerTestPasses() {
   mlir::test::registerTestTransformDialectEraseSchedulePass();
   mlir::test::registerTestVectorLowerings();
   mlir::test::registerTestVectorReductionToSPIRVDotProd();
-  mlir::test::registerTestNvgpuLowerings();
+  mlir::test::registerTestNVGPULowerings();
   mlir::test::registerTestWrittenToPass();
 #if MLIR_ENABLE_PDL_IN_PATTERNMATCH
   mlir::test::registerTestDialectConversionPasses();
-- 
GitLab


From 0bacffbbfc081b4147ac935512e2c5da9e3c06f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Mon, 13 May 2024 08:13:51 +0100
Subject: [PATCH 0525/1206] [mlir][vector] Update tests/patterns for
 vector.transpose (#91359)

Pretty much all logic that we have today for lowering vector.transpose
assumes fixed length vectors (it's done via vector.shuffle that don't
support scalable vectors). This patch updates related tests and patterns
to capture and document this limitation more explicitly.

Note that `vector.transpose` is a valid operation in the context of
scalable vectors, but we are yet to implement the missing lowerings.

Summary of changes:
* `@transpose_nx8x2xf32` is renamed as `@transpose_scalabl`e
  and moved near other tests using `lowering_strategy = "shuffle_1d"
  (to avoid duplicating TD sequences)
* tests specific to X86  (`avx2_lowering_strategy = true`) are moved to
  a dedicated file (to separate generic tests from target-specific
  tests)
* `@transpose10_nx4xnx1xf32` duplicated `@transpose10_4xnx1xf32` and was
  deleted (the latter is renamed as `@transpose10_4x1xf32_scalable` to
  match its fixed-width counterpart: `@transpose10_4x1xf32`)
---
 .../Transforms/LowerVectorTranspose.cpp       |   1 +
 .../CPU/X86/vector-transpose-lowering.mlir    | 493 +++++++++++++++
 .../Vector/vector-transpose-lowering.mlir     | 561 ++----------------
 3 files changed, 533 insertions(+), 522 deletions(-)
 create mode 100644 mlir/test/Dialect/Vector/CPU/X86/vector-transpose-lowering.mlir

diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
index 792550dcfaf2..7011c478fefb 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
@@ -352,6 +352,7 @@ public:
       return success();
     }
 
+    // TODO: Add support for scalable vectors
     if (inputType.isScalable())
       return failure();
 
diff --git a/mlir/test/Dialect/Vector/CPU/X86/vector-transpose-lowering.mlir b/mlir/test/Dialect/Vector/CPU/X86/vector-transpose-lowering.mlir
new file mode 100644
index 000000000000..ae2b5393ca44
--- /dev/null
+++ b/mlir/test/Dialect/Vector/CPU/X86/vector-transpose-lowering.mlir
@@ -0,0 +1,493 @@
+// RUN: mlir-opt %s --transform-interpreter --split-input-file | FileCheck %s
+
+// NOTE: This file tests lowerings that are implemented in the X86Vector
+// dialect. Since X86 does not support scalable vectors, all examples in this
+// file use fixed-width vectors.
+
+// CHECK-LABEL: func @transpose4x8
+func.func @transpose4x8xf32(%arg0: vector<4x8xf32>) -> vector<8x4xf32> {
+  //      CHECK: vector.extract {{.*}}[0]
+  // CHECK-NEXT: vector.extract {{.*}}[1]
+  // CHECK-NEXT: vector.extract {{.*}}[2]
+  // CHECK-NEXT: vector.extract {{.*}}[3]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.shape_cast {{.*}} vector<4x8xf32> to vector<32xf32>
+  // CHECK-NEXT: vector.shape_cast {{.*}} vector<32xf32> to vector<8x4xf32>
+  %0 = vector.transpose %arg0, [1, 0] : vector<4x8xf32> to vector<8x4xf32>
+  return %0 : vector<8x4xf32>
+}
+
+// CHECK-LABEL: func @transpose021_1x4x8
+func.func @transpose021_1x4x8xf32(%arg0: vector<1x4x8xf32>) -> vector<1x8x4xf32> {
+  //      CHECK: vector.extract {{.*}}[0, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 1]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 2]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 3]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.shape_cast {{.*}} vector<4x8xf32> to vector<32xf32>
+  // CHECK-NEXT: vector.shape_cast {{.*}} vector<32xf32> to vector<1x8x4xf32>
+  %0 = vector.transpose %arg0, [0, 2, 1] : vector<1x4x8xf32> to vector<1x8x4xf32>
+  return %0 : vector<1x8x4xf32>
+}
+
+// CHECK-LABEL: func @transpose8x8
+func.func @transpose8x8xf32(%arg0: vector<8x8xf32>) -> vector<8x8xf32> {
+  //      CHECK: vector.extract {{.*}}[0]
+  // CHECK-NEXT: vector.extract {{.*}}[1]
+  // CHECK-NEXT: vector.extract {{.*}}[2]
+  // CHECK-NEXT: vector.extract {{.*}}[3]
+  // CHECK-NEXT: vector.extract {{.*}}[4]
+  // CHECK-NEXT: vector.extract {{.*}}[5]
+  // CHECK-NEXT: vector.extract {{.*}}[6]
+  // CHECK-NEXT: vector.extract {{.*}}[7]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.insert {{.*}}[4]
+  // CHECK-NEXT: vector.insert {{.*}}[5]
+  // CHECK-NEXT: vector.insert {{.*}}[6]
+  // CHECK-NEXT: vector.insert {{.*}}[7]
+  %0 = vector.transpose %arg0, [1, 0] : vector<8x8xf32> to vector<8x8xf32>
+  return %0 : vector<8x8xf32>
+}
+
+// CHECK-LABEL: func @transpose021_1x8x8
+func.func @transpose021_1x8x8xf32(%arg0: vector<1x8x8xf32>) -> vector<1x8x8xf32> {
+  //      CHECK: vector.extract {{.*}}[0, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 1]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 2]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 3]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 4]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 5]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 6]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 7]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.insert {{.*}}[4]
+  // CHECK-NEXT: vector.insert {{.*}}[5]
+  // CHECK-NEXT: vector.insert {{.*}}[6]
+  // CHECK-NEXT: vector.insert {{.*}}[7]
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<1x8x8xf32>
+  %0 = vector.transpose %arg0, [0, 2, 1] : vector<1x8x8xf32> to vector<1x8x8xf32>
+  return %0 : vector<1x8x8xf32>
+}
+
+// CHECK-LABEL: func @transpose120_8x1x8
+func.func @transpose120_8x1x8xf32(%arg0: vector<8x1x8xf32>) -> vector<1x8x8xf32> {
+  //      CHECK: vector.extract {{.*}}[0, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[1, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[2, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[3, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[4, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[5, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[6, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[7, 0]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.insert {{.*}}[4]
+  // CHECK-NEXT: vector.insert {{.*}}[5]
+  // CHECK-NEXT: vector.insert {{.*}}[6]
+  // CHECK-NEXT: vector.insert {{.*}}[7]
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<1x8x8xf32>
+  %0 = vector.transpose %arg0, [1, 2, 0] : vector<8x1x8xf32> to vector<1x8x8xf32>
+  return %0 : vector<1x8x8xf32>
+}
+
+// CHECK-LABEL: func @transpose120_8x8x1
+func.func @transpose120_8x8x1xf32(%arg0: vector<8x8x1xf32>) -> vector<8x1x8xf32> {
+  //      CHECK: vector.shape_cast %{{.*}} : vector<8x8x1xf32> to vector<8x8xf32>
+  // CHECK-NEXT: vector.extract {{.*}}[0]
+  // CHECK-NEXT: vector.extract {{.*}}[1]
+  // CHECK-NEXT: vector.extract {{.*}}[2]
+  // CHECK-NEXT: vector.extract {{.*}}[3]
+  // CHECK-NEXT: vector.extract {{.*}}[4]
+  // CHECK-NEXT: vector.extract {{.*}}[5]
+  // CHECK-NEXT: vector.extract {{.*}}[6]
+  // CHECK-NEXT: vector.extract {{.*}}[7]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.insert {{.*}}[4]
+  // CHECK-NEXT: vector.insert {{.*}}[5]
+  // CHECK-NEXT: vector.insert {{.*}}[6]
+  // CHECK-NEXT: vector.insert {{.*}}[7]
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x1x8xf32>
+  %0 = vector.transpose %arg0, [1, 2, 0] : vector<8x8x1xf32> to vector<8x1x8xf32>
+  return %0 : vector<8x1x8xf32>
+}
+
+// CHECK-LABEL: func @transpose102_8x8x1
+func.func @transpose102_8x8x1xf32(%arg0: vector<8x8x1xf32>) -> vector<8x8x1xf32> {
+  //      CHECK: vector.shape_cast %{{.*}} : vector<8x8x1xf32> to vector<8x8xf32>
+  // CHECK-NEXT: vector.extract {{.*}}[0]
+  // CHECK-NEXT: vector.extract {{.*}}[1]
+  // CHECK-NEXT: vector.extract {{.*}}[2]
+  // CHECK-NEXT: vector.extract {{.*}}[3]
+  // CHECK-NEXT: vector.extract {{.*}}[4]
+  // CHECK-NEXT: vector.extract {{.*}}[5]
+  // CHECK-NEXT: vector.extract {{.*}}[6]
+  // CHECK-NEXT: vector.extract {{.*}}[7]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.insert {{.*}}[4]
+  // CHECK-NEXT: vector.insert {{.*}}[5]
+  // CHECK-NEXT: vector.insert {{.*}}[6]
+  // CHECK-NEXT: vector.insert {{.*}}[7]
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x8x1xf32>
+  %0 = vector.transpose %arg0, [1, 0, 2] : vector<8x8x1xf32> to vector<8x8x1xf32>
+  return %0 : vector<8x8x1xf32>
+}
+
+// CHECK-LABEL: func @transpose201_8x1x8
+func.func @transpose201_8x1x8xf32(%arg0: vector<8x1x8xf32>) -> vector<8x8x1xf32> {
+  //      CHECK: vector.extract {{.*}}[0, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[1, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[2, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[3, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[4, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[5, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[6, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[7, 0]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.insert {{.*}}[4]
+  // CHECK-NEXT: vector.insert {{.*}}[5]
+  // CHECK-NEXT: vector.insert {{.*}}[6]
+  // CHECK-NEXT: vector.insert {{.*}}[7]
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x8x1xf32>
+  %0 = vector.transpose %arg0, [2, 0, 1] : vector<8x1x8xf32> to vector<8x8x1xf32>
+  return %0 : vector<8x8x1xf32>
+}
+
+// CHECK-LABEL: func @transpose201_1x8x8
+func.func @transpose201_1x8x8xf32(%arg0: vector<1x8x8xf32>) -> vector<8x1x8xf32> {
+  //      CHECK: vector.extract {{.*}}[0, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 1]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 2]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 3]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 4]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 5]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 6]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 7]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.insert {{.*}}[4]
+  // CHECK-NEXT: vector.insert {{.*}}[5]
+  // CHECK-NEXT: vector.insert {{.*}}[6]
+  // CHECK-NEXT: vector.insert {{.*}}[7]
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x1x8xf32>
+  %0 = vector.transpose %arg0, [2, 0, 1] : vector<1x8x8xf32> to vector<8x1x8xf32>
+  return %0 : vector<8x1x8xf32>
+}
+
+// CHECK-LABEL: func @transpose210_8x1x8
+func.func @transpose210_8x1x8xf32(%arg0: vector<8x1x8xf32>) -> vector<8x1x8xf32> {
+  //      CHECK: vector.extract {{.*}}[0, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[1, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[2, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[3, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[4, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[5, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[6, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[7, 0]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.insert {{.*}}[4]
+  // CHECK-NEXT: vector.insert {{.*}}[5]
+  // CHECK-NEXT: vector.insert {{.*}}[6]
+  // CHECK-NEXT: vector.insert {{.*}}[7]
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x1x8xf32>
+  %0 = vector.transpose %arg0, [2, 1, 0] : vector<8x1x8xf32> to vector<8x1x8xf32>
+  return %0 : vector<8x1x8xf32>
+}
+
+// CHECK-LABEL: func @transpose210_8x8x1
+func.func @transpose210_8x8x1xf32(%arg0: vector<8x8x1xf32>) -> vector<1x8x8xf32> {
+  //      CHECK: vector.shape_cast %{{.*}} : vector<8x8x1xf32> to vector<8x8xf32>
+  // CHECK-NEXT: vector.extract {{.*}}[0]
+  // CHECK-NEXT: vector.extract {{.*}}[1]
+  // CHECK-NEXT: vector.extract {{.*}}[2]
+  // CHECK-NEXT: vector.extract {{.*}}[3]
+  // CHECK-NEXT: vector.extract {{.*}}[4]
+  // CHECK-NEXT: vector.extract {{.*}}[5]
+  // CHECK-NEXT: vector.extract {{.*}}[6]
+  // CHECK-NEXT: vector.extract {{.*}}[7]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.insert {{.*}}[4]
+  // CHECK-NEXT: vector.insert {{.*}}[5]
+  // CHECK-NEXT: vector.insert {{.*}}[6]
+  // CHECK-NEXT: vector.insert {{.*}}[7]
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<1x8x8xf32>
+  %0 = vector.transpose %arg0, [2, 1, 0] : vector<8x8x1xf32> to vector<1x8x8xf32>
+  return %0 : vector<1x8x8xf32>
+}
+
+// CHECK-LABEL: func @transpose210_1x8x8
+func.func @transpose210_1x8x8xf32(%arg0: vector<1x8x8xf32>) -> vector<8x8x1xf32> {
+  //      CHECK: vector.extract {{.*}}[0, 0]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 1]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 2]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 3]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 4]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 5]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 6]
+  // CHECK-NEXT: vector.extract {{.*}}[0, 7]
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
+  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
+  // CHECK-NEXT: vector.insert {{.*}}[0]
+  // CHECK-NEXT: vector.insert {{.*}}[1]
+  // CHECK-NEXT: vector.insert {{.*}}[2]
+  // CHECK-NEXT: vector.insert {{.*}}[3]
+  // CHECK-NEXT: vector.insert {{.*}}[4]
+  // CHECK-NEXT: vector.insert {{.*}}[5]
+  // CHECK-NEXT: vector.insert {{.*}}[6]
+  // CHECK-NEXT: vector.insert {{.*}}[7]
+  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x8x1xf32>
+  %0 = vector.transpose %arg0, [2, 1, 0] : vector<1x8x8xf32> to vector<8x8x1xf32>
+  return %0 : vector<8x8x1xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) {
+    %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func">
+    transform.apply_patterns to %func_op {
+      transform.apply_patterns.vector.lower_transpose avx2_lowering_strategy = true
+    } : !transform.op<"func.func">
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir b/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir
index 628a8ce50959..219a72df52a1 100644
--- a/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir
+++ b/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir
@@ -110,6 +110,17 @@ func.func @transpose(%arg0: vector<2x4xf32>) -> vector<4x2xf32> {
   return %0 : vector<4x2xf32>
 }
 
+/// Scalable vectors are not supported
+
+// CHECK-LABEL: func @transpose_scalable
+// CHECK-NOT: vector.shuffle
+// CHECK-NOT: vector.shape_cast
+// CHECK: vector.transpose
+func.func @transpose_scalable(%arg0: vector<2x[4]xf32>) -> vector<[4]x2xf32> {
+  %0 = vector.transpose %arg0, [1, 0] : vector<2x[4]xf32> to vector<[4]x2xf32>
+  return %0 : vector<[4]x2xf32>
+}
+
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) {
@@ -132,502 +143,22 @@ func.func @transpose(%arg0: vector<2x4xf32>) -> vector<4x2xf32> {
   return %0 : vector<4x2xf32>
 }
 
+/// Scalable vectors are not supported
 
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) {
-    %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func">
-    transform.apply_patterns to %func_op {
-      transform.apply_patterns.vector.lower_transpose lowering_strategy = "flat_transpose"
-    } : !transform.op<"func.func">
-    transform.yield
-  }
-}
-
-// -----
-
-// CHECK-LABEL: func @transpose4x8
-func.func @transpose4x8xf32(%arg0: vector<4x8xf32>) -> vector<8x4xf32> {
-  //      CHECK: vector.extract {{.*}}[0]
-  // CHECK-NEXT: vector.extract {{.*}}[1]
-  // CHECK-NEXT: vector.extract {{.*}}[2]
-  // CHECK-NEXT: vector.extract {{.*}}[3]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.shape_cast {{.*}} vector<4x8xf32> to vector<32xf32>
-  // CHECK-NEXT: vector.shape_cast {{.*}} vector<32xf32> to vector<8x4xf32>
-  %0 = vector.transpose %arg0, [1, 0] : vector<4x8xf32> to vector<8x4xf32>
-  return %0 : vector<8x4xf32>
-}
-
-// CHECK-LABEL: func @transpose021_1x4x8
-func.func @transpose021_1x4x8xf32(%arg0: vector<1x4x8xf32>) -> vector<1x8x4xf32> {
-  //      CHECK: vector.extract {{.*}}[0, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 1]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 2]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 3]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.shape_cast {{.*}} vector<4x8xf32> to vector<32xf32>
-  // CHECK-NEXT: vector.shape_cast {{.*}} vector<32xf32> to vector<1x8x4xf32>
-  %0 = vector.transpose %arg0, [0, 2, 1] : vector<1x4x8xf32> to vector<1x8x4xf32>
-  return %0 : vector<1x8x4xf32>
-}
-
-// CHECK-LABEL: func @transpose8x8
-func.func @transpose8x8xf32(%arg0: vector<8x8xf32>) -> vector<8x8xf32> {
-  //      CHECK: vector.extract {{.*}}[0]
-  // CHECK-NEXT: vector.extract {{.*}}[1]
-  // CHECK-NEXT: vector.extract {{.*}}[2]
-  // CHECK-NEXT: vector.extract {{.*}}[3]
-  // CHECK-NEXT: vector.extract {{.*}}[4]
-  // CHECK-NEXT: vector.extract {{.*}}[5]
-  // CHECK-NEXT: vector.extract {{.*}}[6]
-  // CHECK-NEXT: vector.extract {{.*}}[7]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.insert {{.*}}[4]
-  // CHECK-NEXT: vector.insert {{.*}}[5]
-  // CHECK-NEXT: vector.insert {{.*}}[6]
-  // CHECK-NEXT: vector.insert {{.*}}[7]
-  %0 = vector.transpose %arg0, [1, 0] : vector<8x8xf32> to vector<8x8xf32>
-  return %0 : vector<8x8xf32>
-}
-
-// CHECK-LABEL: func @transpose021_1x8x8
-func.func @transpose021_1x8x8xf32(%arg0: vector<1x8x8xf32>) -> vector<1x8x8xf32> {
-  //      CHECK: vector.extract {{.*}}[0, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 1]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 2]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 3]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 4]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 5]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 6]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 7]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.insert {{.*}}[4]
-  // CHECK-NEXT: vector.insert {{.*}}[5]
-  // CHECK-NEXT: vector.insert {{.*}}[6]
-  // CHECK-NEXT: vector.insert {{.*}}[7]
-  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<1x8x8xf32>
-  %0 = vector.transpose %arg0, [0, 2, 1] : vector<1x8x8xf32> to vector<1x8x8xf32>
-  return %0 : vector<1x8x8xf32>
-}
-
-// CHECK-LABEL: func @transpose120_8x1x8
-func.func @transpose120_8x1x8xf32(%arg0: vector<8x1x8xf32>) -> vector<1x8x8xf32> {
-  //      CHECK: vector.extract {{.*}}[0, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[1, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[2, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[3, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[4, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[5, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[6, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[7, 0]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.insert {{.*}}[4]
-  // CHECK-NEXT: vector.insert {{.*}}[5]
-  // CHECK-NEXT: vector.insert {{.*}}[6]
-  // CHECK-NEXT: vector.insert {{.*}}[7]
-  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<1x8x8xf32>
-  %0 = vector.transpose %arg0, [1, 2, 0] : vector<8x1x8xf32> to vector<1x8x8xf32>
-  return %0 : vector<1x8x8xf32>
-}
-
-// CHECK-LABEL: func @transpose120_8x8x1
-func.func @transpose120_8x8x1xf32(%arg0: vector<8x8x1xf32>) -> vector<8x1x8xf32> {
-  //      CHECK: vector.shape_cast %{{.*}} : vector<8x8x1xf32> to vector<8x8xf32>
-  // CHECK-NEXT: vector.extract {{.*}}[0]
-  // CHECK-NEXT: vector.extract {{.*}}[1]
-  // CHECK-NEXT: vector.extract {{.*}}[2]
-  // CHECK-NEXT: vector.extract {{.*}}[3]
-  // CHECK-NEXT: vector.extract {{.*}}[4]
-  // CHECK-NEXT: vector.extract {{.*}}[5]
-  // CHECK-NEXT: vector.extract {{.*}}[6]
-  // CHECK-NEXT: vector.extract {{.*}}[7]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.insert {{.*}}[4]
-  // CHECK-NEXT: vector.insert {{.*}}[5]
-  // CHECK-NEXT: vector.insert {{.*}}[6]
-  // CHECK-NEXT: vector.insert {{.*}}[7]
-  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x1x8xf32>
-  %0 = vector.transpose %arg0, [1, 2, 0] : vector<8x8x1xf32> to vector<8x1x8xf32>
-  return %0 : vector<8x1x8xf32>
-}
-
-// CHECK-LABEL: func @transpose102_8x8x1
-func.func @transpose102_8x8x1xf32(%arg0: vector<8x8x1xf32>) -> vector<8x8x1xf32> {
-  //      CHECK: vector.shape_cast %{{.*}} : vector<8x8x1xf32> to vector<8x8xf32>
-  // CHECK-NEXT: vector.extract {{.*}}[0]
-  // CHECK-NEXT: vector.extract {{.*}}[1]
-  // CHECK-NEXT: vector.extract {{.*}}[2]
-  // CHECK-NEXT: vector.extract {{.*}}[3]
-  // CHECK-NEXT: vector.extract {{.*}}[4]
-  // CHECK-NEXT: vector.extract {{.*}}[5]
-  // CHECK-NEXT: vector.extract {{.*}}[6]
-  // CHECK-NEXT: vector.extract {{.*}}[7]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.insert {{.*}}[4]
-  // CHECK-NEXT: vector.insert {{.*}}[5]
-  // CHECK-NEXT: vector.insert {{.*}}[6]
-  // CHECK-NEXT: vector.insert {{.*}}[7]
-  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x8x1xf32>
-  %0 = vector.transpose %arg0, [1, 0, 2] : vector<8x8x1xf32> to vector<8x8x1xf32>
-  return %0 : vector<8x8x1xf32>
-}
-
-// CHECK-LABEL: func @transpose201_8x1x8
-func.func @transpose201_8x1x8xf32(%arg0: vector<8x1x8xf32>) -> vector<8x8x1xf32> {
-  //      CHECK: vector.extract {{.*}}[0, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[1, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[2, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[3, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[4, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[5, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[6, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[7, 0]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.insert {{.*}}[4]
-  // CHECK-NEXT: vector.insert {{.*}}[5]
-  // CHECK-NEXT: vector.insert {{.*}}[6]
-  // CHECK-NEXT: vector.insert {{.*}}[7]
-  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x8x1xf32>
-  %0 = vector.transpose %arg0, [2, 0, 1] : vector<8x1x8xf32> to vector<8x8x1xf32>
-  return %0 : vector<8x8x1xf32>
-}
-
-// CHECK-LABEL: func @transpose201_1x8x8
-func.func @transpose201_1x8x8xf32(%arg0: vector<1x8x8xf32>) -> vector<8x1x8xf32> {
-  //      CHECK: vector.extract {{.*}}[0, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 1]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 2]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 3]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 4]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 5]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 6]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 7]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.insert {{.*}}[4]
-  // CHECK-NEXT: vector.insert {{.*}}[5]
-  // CHECK-NEXT: vector.insert {{.*}}[6]
-  // CHECK-NEXT: vector.insert {{.*}}[7]
-  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x1x8xf32>
-  %0 = vector.transpose %arg0, [2, 0, 1] : vector<1x8x8xf32> to vector<8x1x8xf32>
-  return %0 : vector<8x1x8xf32>
-}
-
-// CHECK-LABEL: func @transpose210_8x1x8
-func.func @transpose210_8x1x8xf32(%arg0: vector<8x1x8xf32>) -> vector<8x1x8xf32> {
-  //      CHECK: vector.extract {{.*}}[0, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[1, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[2, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[3, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[4, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[5, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[6, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[7, 0]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.insert {{.*}}[4]
-  // CHECK-NEXT: vector.insert {{.*}}[5]
-  // CHECK-NEXT: vector.insert {{.*}}[6]
-  // CHECK-NEXT: vector.insert {{.*}}[7]
-  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x1x8xf32>
-  %0 = vector.transpose %arg0, [2, 1, 0] : vector<8x1x8xf32> to vector<8x1x8xf32>
-  return %0 : vector<8x1x8xf32>
-}
-
-// CHECK-LABEL: func @transpose210_8x8x1
-func.func @transpose210_8x8x1xf32(%arg0: vector<8x8x1xf32>) -> vector<1x8x8xf32> {
-  //      CHECK: vector.shape_cast %{{.*}} : vector<8x8x1xf32> to vector<8x8xf32>
-  // CHECK-NEXT: vector.extract {{.*}}[0]
-  // CHECK-NEXT: vector.extract {{.*}}[1]
-  // CHECK-NEXT: vector.extract {{.*}}[2]
-  // CHECK-NEXT: vector.extract {{.*}}[3]
-  // CHECK-NEXT: vector.extract {{.*}}[4]
-  // CHECK-NEXT: vector.extract {{.*}}[5]
-  // CHECK-NEXT: vector.extract {{.*}}[6]
-  // CHECK-NEXT: vector.extract {{.*}}[7]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.insert {{.*}}[4]
-  // CHECK-NEXT: vector.insert {{.*}}[5]
-  // CHECK-NEXT: vector.insert {{.*}}[6]
-  // CHECK-NEXT: vector.insert {{.*}}[7]
-  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<1x8x8xf32>
-  %0 = vector.transpose %arg0, [2, 1, 0] : vector<8x8x1xf32> to vector<1x8x8xf32>
-  return %0 : vector<1x8x8xf32>
-}
-
-// CHECK-LABEL: func @transpose210_1x8x8
-func.func @transpose210_1x8x8xf32(%arg0: vector<1x8x8xf32>) -> vector<8x8x1xf32> {
-  //      CHECK: vector.extract {{.*}}[0, 0]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 1]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 2]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 3]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 4]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 5]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 6]
-  // CHECK-NEXT: vector.extract {{.*}}[0, 7]
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32>
-  // CHECK-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32>
-  // CHECK-NEXT: vector.insert {{.*}}[0]
-  // CHECK-NEXT: vector.insert {{.*}}[1]
-  // CHECK-NEXT: vector.insert {{.*}}[2]
-  // CHECK-NEXT: vector.insert {{.*}}[3]
-  // CHECK-NEXT: vector.insert {{.*}}[4]
-  // CHECK-NEXT: vector.insert {{.*}}[5]
-  // CHECK-NEXT: vector.insert {{.*}}[6]
-  // CHECK-NEXT: vector.insert {{.*}}[7]
-  // CHECK-NEXT: vector.shape_cast %{{.*}} : vector<8x8xf32> to vector<8x8x1xf32>
-  %0 = vector.transpose %arg0, [2, 1, 0] : vector<1x8x8xf32> to vector<8x8x1xf32>
-  return %0 : vector<8x8x1xf32>
+// CHECK-LABEL: func @transpose_scalable(
+func.func @transpose_scalable(%arg0: vector<2x[4]xf32>) -> vector<[4]x2xf32> {
+  // CHECK-NOT:       vector.shape_cast
+  // CHECK-NOT:       vector.flat_transpose
+  // CHECK:           vector.transpose
+  %0 = vector.transpose %arg0, [1, 0] : vector<2x[4]xf32> to vector<[4]x2xf32>
+  return %0 : vector<[4]x2xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) {
     %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func">
     transform.apply_patterns to %func_op {
-      transform.apply_patterns.vector.lower_transpose avx2_lowering_strategy = true
+      transform.apply_patterns.vector.lower_transpose lowering_strategy = "flat_transpose"
     } : !transform.op<"func.func">
     transform.yield
   }
@@ -635,6 +166,7 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+// CHECK-LABEL: @transpose_shuffle16x16xf32(
 func.func @transpose_shuffle16x16xf32(%arg0: vector<16x16xf32>) -> vector<16x16xf32> {
   // CHECK: vector.shuffle {{.*}} [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf32>, vector<16xf32>
   // CHECK: vector.shuffle {{.*}} [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf32>, vector<16xf32>
@@ -704,6 +236,14 @@ func.func @transpose_shuffle16x16xf32(%arg0: vector<16x16xf32>) -> vector<16x16x
   return %0 : vector<16x16xf32>
 }
 
+// CHECK-LABEL: @transpose_shuffle16x16xf32_scalable(
+func.func @transpose_shuffle16x16xf32_scalable(%arg0: vector<16x[16]xf32>) -> vector<[16]x16xf32> {
+  // CHECK-NOT: vector.shuffle
+  // CHECK: vector.transpose
+  %0 = vector.transpose %arg0, [1, 0] : vector<16x[16]xf32> to vector<[16]x16xf32>
+  return %0 : vector<[16]x16xf32>
+}
+
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) {
     %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func">
@@ -786,6 +326,14 @@ func.func @transpose021_shuffle16x16xf32(%arg0: vector<1x16x16xf32>) -> vector<1
   return %0 : vector<1x16x16xf32>
 }
 
+// CHECK-LABEL: func @transpose021_shuffle16x16xf32_scalable
+func.func @transpose021_shuffle16x16xf32_scalable(%arg0: vector<1x16x[16]xf32>) -> vector<1x[16]x16xf32> {
+  // CHECK-NOT: vector.shuffle
+  // CHECK: vector.transpose
+  %0 = vector.transpose %arg0, [0, 2, 1] : vector<1x16x[16]xf32> to vector<1x[16]x16xf32>
+  return %0 : vector<1x[16]x16xf32>
+}
+
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) {
     %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func">
@@ -830,23 +378,14 @@ func.func @transpose10_1xnx4xf32(%arg0: vector<1x[4]xf32>) -> vector<[4]x1xf32>
 
 /// Scalable unit dim should not be lowered to shape_cast.
 
-// CHECK-LABEL: func @transpose10_4xnx1xf32
-func.func @transpose10_4xnx1xf32(%arg0: vector<4x[1]xf32>) -> vector<[1]x4xf32> {
+// CHECK-LABEL: func @transpose10_4x1xf32_scalable
+func.func @transpose10_4x1xf32_scalable(%arg0: vector<4x[1]xf32>) -> vector<[1]x4xf32> {
   // CHECK-NOT: vector.shape_cast
   // CHECK: vector.transpose %{{.*}} : vector<4x[1]xf32> to vector<[1]x4xf32>
   %0 = vector.transpose %arg0, [1, 0] : vector<4x[1]xf32> to vector<[1]x4xf32>
   return %0 : vector<[1]x4xf32>
 }
 
-// CHECK-LABEL: func @transpose10_nx4xnx1xf32
-func.func @transpose10_nx4xnx1xf32(%arg0: vector<4x[1]xf32>) -> vector<[1]x4xf32> {
-  // CHECK-NOT: vector.shape_cast
-  // CHECK: vector.transpose %{{.*}} : vector<4x[1]xf32> to vector<[1]x4xf32>
-  %0 = vector.transpose %arg0, [1, 0] : vector<4x[1]xf32> to vector<[1]x4xf32>
-
-  return %0 : vector<[1]x4xf32>
-}
-
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) {
     %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func">
@@ -856,25 +395,3 @@ module attributes {transform.with_named_sequence} {
     transform.yield
   }
 }
-
-// -----
-
-// Scalable transposes should not be lowered to vector.shuffle.
-
-// CHECK-LABEL: func @transpose_nx8x2xf32
-func.func @transpose_nx8x2xf32(%arg0: vector<[8]x2xf32>) -> vector<2x[8]xf32> {
-  // CHECK-NOT: vector.shuffle
-  // CHECK: vector.transpose %{{.*}} : vector<[8]x2xf32> to vector<2x[8]xf32>
-  %0 = vector.transpose %arg0, [1, 0] : vector<[8]x2xf32> to vector<2x[8]xf32>
-  return %0 : vector<2x[8]xf32>
-}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) {
-    %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func">
-    transform.apply_patterns to %func_op {
-      transform.apply_patterns.vector.lower_transpose lowering_strategy = "shuffle_1d"
-    } : !transform.op<"func.func">
-    transform.yield
-  }
-}
-- 
GitLab


From d182877ba3f5ad93e061f68a9ecce38cb8cec418 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Mon, 13 May 2024 09:33:35 +0200
Subject: [PATCH 0526/1206] [clang] CTAD alias: fix the transformation for the
 require-clause expr (#90961)

In the clang AST, constraint nodes are deliberately not instantiated
unless they are actively being evaluated. Consequently, occurrences of
template parameters in the require-clause expression have a subtle
"depth" difference compared to normal occurrences in places, such as
function parameters. When transforming the require-clause, we must take
this distinction into account.

The existing implementation overlooks this consideration. This patch is
to rewrite the implementation of the require-clause transformation to
address this issue.

Fixes #90177
---
 clang/lib/Sema/SemaTemplate.cpp              | 148 +++++++++++++++++--
 clang/test/AST/ast-dump-ctad-alias.cpp       |  40 +++++
 clang/test/SemaCXX/cxx20-ctad-type-alias.cpp |  68 +++++++++
 3 files changed, 242 insertions(+), 14 deletions(-)
 create mode 100644 clang/test/AST/ast-dump-ctad-alias.cpp

diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 0e7bd8dd8957..bae00c629270 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -2758,31 +2758,149 @@ bool hasDeclaredDeductionGuides(DeclarationName Name, DeclContext *DC) {
   return false;
 }
 
+unsigned getTemplateParameterDepth(NamedDecl *TemplateParam) {
+  if (auto *TTP = dyn_cast<TemplateTypeParmDecl>(TemplateParam))
+    return TTP->getDepth();
+  if (auto *TTP = dyn_cast<TemplateTemplateParmDecl>(TemplateParam))
+    return TTP->getDepth();
+  if (auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(TemplateParam))
+    return NTTP->getDepth();
+  llvm_unreachable("Unhandled template parameter types");
+}
+
 NamedDecl *transformTemplateParameter(Sema &SemaRef, DeclContext *DC,
                                       NamedDecl *TemplateParam,
                                       MultiLevelTemplateArgumentList &Args,
-                                      unsigned NewIndex) {
+                                      unsigned NewIndex, unsigned NewDepth) {
   if (auto *TTP = dyn_cast<TemplateTypeParmDecl>(TemplateParam))
-    return transformTemplateTypeParam(SemaRef, DC, TTP, Args, TTP->getDepth(),
+    return transformTemplateTypeParam(SemaRef, DC, TTP, Args, NewDepth,
                                       NewIndex);
   if (auto *TTP = dyn_cast<TemplateTemplateParmDecl>(TemplateParam))
-    return transformTemplateParam(SemaRef, DC, TTP, Args, NewIndex,
-                                  TTP->getDepth());
+    return transformTemplateParam(SemaRef, DC, TTP, Args, NewIndex, NewDepth);
   if (auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(TemplateParam))
-    return transformTemplateParam(SemaRef, DC, NTTP, Args, NewIndex,
-                                  NTTP->getDepth());
+    return transformTemplateParam(SemaRef, DC, NTTP, Args, NewIndex, NewDepth);
   llvm_unreachable("Unhandled template parameter types");
 }
 
-Expr *transformRequireClause(Sema &SemaRef, FunctionTemplateDecl *FTD,
-                             llvm::ArrayRef<TemplateArgument> TransformedArgs) {
-  Expr *RC = FTD->getTemplateParameters()->getRequiresClause();
+// Transform the require-clause of F if any.
+// The return result is expected to be the require-clause for the synthesized
+// alias deduction guide.
+Expr *transformRequireClause(Sema &SemaRef, FunctionTemplateDecl *F,
+                             TypeAliasTemplateDecl *AliasTemplate,
+                             ArrayRef<DeducedTemplateArgument> DeduceResults) {
+  Expr *RC = F->getTemplateParameters()->getRequiresClause();
   if (!RC)
     return nullptr;
+
+  auto &Context = SemaRef.Context;
+  LocalInstantiationScope Scope(SemaRef);
+
+  // In the clang AST, constraint nodes are deliberately not instantiated unless
+  // they are actively being evaluated. Consequently, occurrences of template
+  // parameters in the require-clause expression have a subtle "depth"
+  // difference compared to normal occurrences in places, such as function
+  // parameters. When transforming the require-clause, we must take this
+  // distinction into account:
+  //
+  //   1) In the transformed require-clause, occurrences of template parameters
+  //   must use the "uninstantiated" depth;
+  //   2) When substituting on the require-clause expr of the underlying
+  //   deduction guide, we must use the entire set of template argument lists;
+  //
+  // It's important to note that we're performing this transformation on an
+  // *instantiated* AliasTemplate.
+
+  // For 1), if the alias template is nested within a class template, we
+  // calcualte the 'uninstantiated' depth by adding the substitution level back.
+  unsigned AdjustDepth = 0;
+  if (auto *PrimaryTemplate =
+          AliasTemplate->getInstantiatedFromMemberTemplate())
+    AdjustDepth = PrimaryTemplate->getTemplateDepth();
+
+  // We rebuild all template parameters with the uninstantiated depth, and
+  // build template arguments refer to them.
+  SmallVector<TemplateArgument> AdjustedAliasTemplateArgs;
+
+  for (auto *TP : *AliasTemplate->getTemplateParameters()) {
+    // Rebuild any internal references to earlier parameters and reindex
+    // as we go.
+    MultiLevelTemplateArgumentList Args;
+    Args.setKind(TemplateSubstitutionKind::Rewrite);
+    Args.addOuterTemplateArguments(AdjustedAliasTemplateArgs);
+    NamedDecl *NewParam = transformTemplateParameter(
+        SemaRef, AliasTemplate->getDeclContext(), TP, Args,
+        /*NewIndex=*/AdjustedAliasTemplateArgs.size(),
+        getTemplateParameterDepth(TP) + AdjustDepth);
+
+    auto NewTemplateArgument = Context.getCanonicalTemplateArgument(
+        Context.getInjectedTemplateArg(NewParam));
+    AdjustedAliasTemplateArgs.push_back(NewTemplateArgument);
+  }
+  // Template arguments used to transform the template arguments in
+  // DeducedResults.
+  SmallVector<TemplateArgument> TemplateArgsForBuildingRC(
+      F->getTemplateParameters()->size());
+  // Transform the transformed template args
   MultiLevelTemplateArgumentList Args;
   Args.setKind(TemplateSubstitutionKind::Rewrite);
-  Args.addOuterTemplateArguments(TransformedArgs);
-  ExprResult E = SemaRef.SubstExpr(RC, Args);
+  Args.addOuterTemplateArguments(AdjustedAliasTemplateArgs);
+
+  for (unsigned Index = 0; Index < DeduceResults.size(); ++Index) {
+    const auto &D = DeduceResults[Index];
+    if (D.isNull())
+      continue;
+    TemplateArgumentLoc Input =
+        SemaRef.getTrivialTemplateArgumentLoc(D, QualType(), SourceLocation{});
+    TemplateArgumentLoc Output;
+    if (!SemaRef.SubstTemplateArgument(Input, Args, Output)) {
+      assert(TemplateArgsForBuildingRC[Index].isNull() &&
+             "InstantiatedArgs must be null before setting");
+      TemplateArgsForBuildingRC[Index] = Output.getArgument();
+    }
+  }
+
+  // A list of template arguments for transforming the require-clause of F.
+  // It must contain the entire set of template argument lists.
+  MultiLevelTemplateArgumentList ArgsForBuildingRC;
+  ArgsForBuildingRC.setKind(clang::TemplateSubstitutionKind::Rewrite);
+  ArgsForBuildingRC.addOuterTemplateArguments(TemplateArgsForBuildingRC);
+  // For 2), if the underlying F is instantiated from a member template, we need
+  // the entire template argument list, as the constraint AST in the
+  // require-clause of F remains completely uninstantiated.
+  //
+  // For example:
+  //   template <typename T> // depth 0
+  //   struct Outer {
+  //      template <typename U>
+  //      struct Foo { Foo(U); };
+  //
+  //      template <typename U> // depth 1
+  //      requires C<U>
+  //      Foo(U) -> Foo<int>;
+  //   };
+  //   template <typename U>
+  //   using AFoo = Outer<int>::Foo<U>;
+  //
+  // In this scenario, the deduction guide for `Foo` inside `Outer<int>`:
+  //   - The occurrence of U in the require-expression is [depth:1, index:0]
+  //   - The occurrence of U in the function parameter is [depth:0, index:0]
+  //   - The template parameter of U is [depth:0, index:0]
+  //
+  // We add the outer template arguments which is [int] to the multi-level arg
+  // list to ensure that the occurrence U in `C<U>` will be replaced with int
+  // during the substitution.
+  if (F->getInstantiatedFromMemberTemplate()) {
+    auto OuterLevelArgs = SemaRef.getTemplateInstantiationArgs(
+        F, F->getLexicalDeclContext(),
+        /*Final=*/false, /*Innermost=*/std::nullopt,
+        /*RelativeToPrimary=*/true,
+        /*Pattern=*/nullptr,
+        /*ForConstraintInstantiation=*/true);
+    for (auto It : OuterLevelArgs)
+      ArgsForBuildingRC.addOuterTemplateArguments(It.Args);
+  }
+
+  ExprResult E = SemaRef.SubstExpr(RC, ArgsForBuildingRC);
   if (E.isInvalid())
     return nullptr;
   return E.getAs<Expr>();
@@ -2920,7 +3038,8 @@ BuildDeductionGuideForTypeAlias(Sema &SemaRef,
     Args.addOuterTemplateArguments(TransformedDeducedAliasArgs);
     NamedDecl *NewParam = transformTemplateParameter(
         SemaRef, AliasTemplate->getDeclContext(), TP, Args,
-        /*NewIndex=*/FPrimeTemplateParams.size());
+        /*NewIndex=*/FPrimeTemplateParams.size(),
+        getTemplateParameterDepth(TP));
     FPrimeTemplateParams.push_back(NewParam);
 
     auto NewTemplateArgument = Context.getCanonicalTemplateArgument(
@@ -2937,7 +3056,8 @@ BuildDeductionGuideForTypeAlias(Sema &SemaRef,
     // TemplateArgsForBuildingFPrime.
     Args.addOuterTemplateArguments(TemplateArgsForBuildingFPrime);
     NamedDecl *NewParam = transformTemplateParameter(
-        SemaRef, F->getDeclContext(), TP, Args, FPrimeTemplateParams.size());
+        SemaRef, F->getDeclContext(), TP, Args, FPrimeTemplateParams.size(),
+        getTemplateParameterDepth(TP));
     FPrimeTemplateParams.push_back(NewParam);
 
     assert(TemplateArgsForBuildingFPrime[FTemplateParamIdx].isNull() &&
@@ -2993,7 +3113,7 @@ BuildDeductionGuideForTypeAlias(Sema &SemaRef,
     auto *GG = cast<CXXDeductionGuideDecl>(FPrime);
 
     Expr *RequiresClause =
-        transformRequireClause(SemaRef, F, TemplateArgsForBuildingFPrime);
+        transformRequireClause(SemaRef, F, AliasTemplate, DeduceResults);
 
     // FIXME: implement the is_deducible constraint per C++
     // [over.match.class.deduct]p3.3:
diff --git a/clang/test/AST/ast-dump-ctad-alias.cpp b/clang/test/AST/ast-dump-ctad-alias.cpp
new file mode 100644
index 000000000000..423c3454ccb7
--- /dev/null
+++ b/clang/test/AST/ast-dump-ctad-alias.cpp
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -std=c++2a -ast-dump %s | FileCheck -strict-whitespace %s
+
+template <typename, typename>
+constexpr bool Concept = true;
+template<typename T> // depth 0
+struct Out {
+  template<typename U> // depth 1
+  struct Inner {
+    U t;
+  };
+
+  template<typename V> // depth1
+  requires Concept<T, V>
+  Inner(V) -> Inner<V>;
+};
+
+template <typename X>
+struct Out2 {
+  template<typename Y> // depth1
+  using AInner = Out<int>::Inner<Y>;
+};
+Out2<double>::AInner t(1.0);
+
+// Verify that the require-clause of alias deduction guide is transformed correctly:
+//   - Occurrence T should be replaced with `int`;
+//   - Occurrence V should be replaced with the Y with depth 1
+//
+// CHECK:      |   `-FunctionTemplateDecl {{.*}} <deduction guide for AInner>
+// CHECK-NEXT: |     |-TemplateTypeParmDecl {{.*}} typename depth 0 index 0 Y
+// CHECK-NEXT: |     |-UnresolvedLookupExpr {{.*}} '<dependent type>' lvalue (no ADL) = 'Concept' 
+// CHECK-NEXT: |     | |-TemplateArgument type 'int'
+// CHECK-NEXT: |     | | `-BuiltinType {{.*}} 'int'
+// CHECK-NEXT: |     | `-TemplateArgument type 'type-parameter-1-0'
+// CHECK-NEXT: |     |   `-TemplateTypeParmType {{.*}} 'type-parameter-1-0' dependent depth 1 index 0
+// CHECK-NEXT: |     |-CXXDeductionGuideDecl {{.*}} <deduction guide for AInner> 'auto (type-parameter-0-0) -> Inner<type-parameter-0-0>'
+// CHECK-NEXT: |     | `-ParmVarDecl {{.*}} 'type-parameter-0-0'
+// CHECK-NEXT: |     `-CXXDeductionGuideDecl {{.*}} used <deduction guide for AInner> 'auto (double) -> Inner<double>' implicit_instantiation
+// CHECK-NEXT: |       |-TemplateArgument type 'double'
+// CHECK-NEXT: |       | `-BuiltinType {{.*}} 'double'
+// CHECK-NEXT: |       `-ParmVarDecl {{.*}} 'double'
diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
index e8b4383f53c5..4c5595e409f2 100644
--- a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
+++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
@@ -321,3 +321,71 @@ AG ag(1.0);
 // choosen.
 static_assert(__is_same(decltype(ag.t1), int));
 } // namespace test23
+
+// GH90177
+// verify that the transformed require-clause of the alias deduction gudie has
+// the right depth info.
+namespace test24 {
+class Forward;
+class Key {};
+
+template <typename D>
+constexpr bool C = sizeof(D);
+
+// Case1: the alias template and the underlying deduction guide are in the same
+// scope.
+template <typename T>
+struct Case1 {
+  template <typename U>
+  struct Foo {
+    Foo(U);
+  };
+
+  template <typename V>
+  requires (C<V>)
+  Foo(V) -> Foo<V>;
+
+  template <typename Y>
+  using Alias = Foo<Y>;
+};
+// The require-clause should be evaluated on the type Key.
+Case1<Forward>::Alias t2 = Key();
+
+
+// Case2: the alias template and underlying deduction guide are in different
+// scope.
+template <typename T>
+struct Foo {
+  Foo(T);
+};
+template <typename U>
+requires (C<U>)
+Foo(U) -> Foo<U>;
+
+template <typename T>
+struct Case2 {
+  template <typename Y>
+  using Alias = Foo<Y>;
+};
+// The require-caluse should be evaluated on the type Key.
+Case2<Forward>::Alias t1 = Key();
+
+// Case3: crashes on the constexpr evaluator due to the mixed-up depth in
+// require-expr.
+template <class T1>
+struct A1 {
+  template<class T2>
+  struct A2 {
+    template <class T3>
+    struct Foo {
+      Foo(T3);
+    };
+    template <class T3>
+    requires C<T3>
+    Foo(T3) -> Foo<T3>;
+  };
+};
+template <typename U>
+using AFoo = A1<int>::A2<int>::Foo<U>;
+AFoo case3(1);
+} // namespace test24
-- 
GitLab


From 109ede496ecf6de5dabace08d73ec7604b343a6b Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 13 May 2024 08:46:04 +0100
Subject: [PATCH 0527/1206] [AArch64] Extend v2i64 fptosi.sat to v2f64 (#91714)

This helps it produce a single instruction for the saturate, as opposed
to having to scalarize.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |   9 +
 llvm/test/CodeGen/AArch64/fcvt_combine.ll     |   7 +-
 llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll |  16 +-
 .../test/CodeGen/AArch64/fptosi-sat-vector.ll |  23 +-
 .../test/CodeGen/AArch64/fptoui-sat-vector.ll |  23 +-
 .../AArch64/sve-fixed-vector-llrint.ll        |   7 +-
 .../CodeGen/AArch64/sve-fixed-vector-lrint.ll |   7 +-
 llvm/test/CodeGen/AArch64/vector-llrint.ll    | 283 ++++++------------
 llvm/test/CodeGen/AArch64/vector-lrint.ll     | 283 ++++++------------
 9 files changed, 221 insertions(+), 437 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a6c020c6b823..1e0071fffe66 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -4286,6 +4286,15 @@ AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
     return SDValue();
 
   SDLoc DL(Op);
+  // Expand to f64 if we are saturating to i64, to help produce keep the lanes
+  // the same width and produce a fcvtzu.
+  if (SatWidth == 64 && SrcElementWidth < 64) {
+    MVT F64VT = MVT::getVectorVT(MVT::f64, SrcVT.getVectorNumElements());
+    SrcVal = DAG.getNode(ISD::FP_EXTEND, DL, F64VT, SrcVal);
+    SrcVT = F64VT;
+    SrcElementVT = MVT::f64;
+    SrcElementWidth = 64;
+  }
   // Cases that we can emit directly.
   if (SrcElementWidth == DstElementWidth && SrcElementWidth == SatWidth)
     return DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal,
diff --git a/llvm/test/CodeGen/AArch64/fcvt_combine.ll b/llvm/test/CodeGen/AArch64/fcvt_combine.ll
index 29170aab9656..62669a6d99ea 100644
--- a/llvm/test/CodeGen/AArch64/fcvt_combine.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt_combine.ll
@@ -345,11 +345,8 @@ define <2 x i64> @test6_sat(<2 x float> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov v1.2s, #16.00000000
 ; CHECK-NEXT:    fmul v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fcvtzs x9, s1
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
 ; CHECK-NEXT:    ret
   %mul.i = fmul <2 x float> %f, <float 16.000000e+00, float 16.000000e+00>
   %vcvt.i = call <2 x i64> @llvm.fptosi.sat.v2i64.v2f32(<2 x float> %mul.i)
diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
index 2ea581359af6..4e8bfcd9d751 100644
--- a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
@@ -436,12 +436,8 @@ entry:
 define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-LABEL: stest_f32i64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fcvtzs x9, s1
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
 ; CHECK-NEXT:    ret
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
@@ -1056,12 +1052,8 @@ entry:
 define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-LABEL: stest_f32i64_mm:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fcvtzs x9, s1
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
 ; CHECK-NEXT:    ret
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
index c45885a38f15..d620a8851ee4 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
@@ -793,12 +793,8 @@ define <2 x i50> @test_signed_v2f32_v2i50(<2 x float> %f) {
 define <2 x i64> @test_signed_v2f32_v2i64(<2 x float> %f) {
 ; CHECK-LABEL: test_signed_v2f32_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fcvtzs x9, s1
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
 ; CHECK-NEXT:    ret
     %x = call <2 x i64> @llvm.fptosi.sat.v2f32.v2i64(<2 x float> %f)
     ret <2 x i64> %x
@@ -1060,17 +1056,10 @@ define <4 x i50> @test_signed_v4f32_v4i50(<4 x float> %f) {
 define <4 x i64> @test_signed_v4f32_v4i64(<4 x float> %f) {
 ; CHECK-LABEL: test_signed_v4f32_v4i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    mov s3, v0.s[1]
-; CHECK-NEXT:    fcvtzs x9, s0
-; CHECK-NEXT:    mov s2, v1.s[1]
-; CHECK-NEXT:    fcvtzs x8, s1
-; CHECK-NEXT:    fcvtzs x11, s3
-; CHECK-NEXT:    fmov d0, x9
-; CHECK-NEXT:    fcvtzs x10, s2
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    mov v0.d[1], x11
-; CHECK-NEXT:    mov v1.d[1], x10
+; CHECK-NEXT:    fcvtl2 v1.2d, v0.4s
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
 ; CHECK-NEXT:    ret
     %x = call <4 x i64> @llvm.fptosi.sat.v4f32.v4i64(<4 x float> %f)
     ret <4 x i64> %x
diff --git a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
index c94db3484994..16e04070b654 100644
--- a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
@@ -707,12 +707,8 @@ define <2 x i50> @test_unsigned_v2f32_v2i50(<2 x float> %f) {
 define <2 x i64> @test_unsigned_v2f32_v2i64(<2 x float> %f) {
 ; CHECK-LABEL: test_unsigned_v2f32_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fcvtzu x8, s0
-; CHECK-NEXT:    fcvtzu x9, s1
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
 ; CHECK-NEXT:    ret
     %x = call <2 x i64> @llvm.fptoui.sat.v2f32.v2i64(<2 x float> %f)
     ret <2 x i64> %x
@@ -927,17 +923,10 @@ define <4 x i50> @test_unsigned_v4f32_v4i50(<4 x float> %f) {
 define <4 x i64> @test_unsigned_v4f32_v4i64(<4 x float> %f) {
 ; CHECK-LABEL: test_unsigned_v4f32_v4i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    mov s3, v0.s[1]
-; CHECK-NEXT:    fcvtzu x9, s0
-; CHECK-NEXT:    mov s2, v1.s[1]
-; CHECK-NEXT:    fcvtzu x8, s1
-; CHECK-NEXT:    fcvtzu x11, s3
-; CHECK-NEXT:    fmov d0, x9
-; CHECK-NEXT:    fcvtzu x10, s2
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    mov v0.d[1], x11
-; CHECK-NEXT:    mov v1.d[1], x10
+; CHECK-NEXT:    fcvtl2 v1.2d, v0.4s
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
 ; CHECK-NEXT:    ret
     %x = call <4 x i64> @llvm.fptoui.sat.v4f32.v4i64(<4 x float> %f)
     ret <4 x i64> %x
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll b/llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll
index 9137eae269d9..c77861509e4a 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll
@@ -295,11 +295,8 @@ define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
 ; CHECK-LABEL: llrint_v2i64_v2f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.2s, v0.2s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fcvtzs x9, s1
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
 ; CHECK-NEXT:    ret
   %a = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> %x)
   ret <2 x i64> %a
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll b/llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll
index 9bdbe9b8ac62..6a97e7ad64bf 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll
@@ -534,11 +534,8 @@ define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
 ; CHECK-i64-LABEL: lrint_v2f32:
 ; CHECK-i64:       // %bb.0:
 ; CHECK-i64-NEXT:    frintx v0.2s, v0.2s
-; CHECK-i64-NEXT:    mov s1, v0.s[1]
-; CHECK-i64-NEXT:    fcvtzs x8, s0
-; CHECK-i64-NEXT:    fcvtzs x9, s1
-; CHECK-i64-NEXT:    fmov d0, x8
-; CHECK-i64-NEXT:    mov v0.d[1], x9
+; CHECK-i64-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
 ; CHECK-i64-NEXT:    ret
   %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float> %x)
   ret <2 x iXLen> %a
diff --git a/llvm/test/CodeGen/AArch64/vector-llrint.ll b/llvm/test/CodeGen/AArch64/vector-llrint.ll
index b7e743b5085f..5503de2b4c5d 100644
--- a/llvm/test/CodeGen/AArch64/vector-llrint.ll
+++ b/llvm/test/CodeGen/AArch64/vector-llrint.ll
@@ -387,11 +387,8 @@ define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
 ; CHECK-LABEL: llrint_v2i64_v2f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.2s, v0.2s
-; CHECK-NEXT:    mov s1, v0.s[1]
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fcvtzs x9, s1
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
 ; CHECK-NEXT:    ret
   %a = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> %x)
   ret <2 x i64> %a
@@ -404,16 +401,10 @@ define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    frintx v0.2s, v0.2s
 ; CHECK-NEXT:    frintx v1.2s, v1.2s
-; CHECK-NEXT:    mov s2, v0.s[1]
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    mov s3, v1.s[1]
-; CHECK-NEXT:    fcvtzs x9, s1
-; CHECK-NEXT:    fcvtzs x10, s2
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fcvtzs x11, s3
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    mov v0.d[1], x10
-; CHECK-NEXT:    mov v1.d[1], x11
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
 ; CHECK-NEXT:    ret
   %a = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> %x)
   ret <4 x i64> %a
@@ -429,26 +420,14 @@ define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 ; CHECK-NEXT:    frintx v1.2s, v1.2s
 ; CHECK-NEXT:    frintx v2.2s, v2.2s
 ; CHECK-NEXT:    frintx v3.2s, v3.2s
-; CHECK-NEXT:    mov s4, v0.s[1]
-; CHECK-NEXT:    mov s5, v1.s[1]
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fcvtzs x10, s1
-; CHECK-NEXT:    mov s6, v2.s[1]
-; CHECK-NEXT:    mov s7, v3.s[1]
-; CHECK-NEXT:    fcvtzs x11, s2
-; CHECK-NEXT:    fcvtzs x12, s3
-; CHECK-NEXT:    fcvtzs x9, s4
-; CHECK-NEXT:    fcvtzs x13, s5
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d2, x10
-; CHECK-NEXT:    fcvtzs x14, s6
-; CHECK-NEXT:    fcvtzs x15, s7
-; CHECK-NEXT:    fmov d1, x11
-; CHECK-NEXT:    fmov d3, x12
-; CHECK-NEXT:    mov v0.d[1], x9
-; CHECK-NEXT:    mov v2.d[1], x13
-; CHECK-NEXT:    mov v1.d[1], x14
-; CHECK-NEXT:    mov v3.d[1], x15
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-NEXT:    fcvtl v4.2d, v2.2s
+; CHECK-NEXT:    fcvtl v3.2d, v3.2s
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
 ; CHECK-NEXT:    ret
   %a = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> %x)
   ret <8 x i64> %a
@@ -458,58 +437,34 @@ declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
 define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 ; CHECK-LABEL: llrint_v16i64_v16f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    frintx v4.2s, v0.2s
-; CHECK-NEXT:    frintx v5.2s, v1.2s
-; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    ext v5.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    ext v6.16b, v2.16b, v2.16b, #8
 ; CHECK-NEXT:    ext v7.16b, v3.16b, v3.16b, #8
-; CHECK-NEXT:    frintx v2.2s, v2.2s
-; CHECK-NEXT:    frintx v3.2s, v3.2s
-; CHECK-NEXT:    mov s16, v4.s[1]
-; CHECK-NEXT:    mov s17, v5.s[1]
-; CHECK-NEXT:    fcvtzs x8, s4
 ; CHECK-NEXT:    frintx v0.2s, v0.2s
 ; CHECK-NEXT:    frintx v1.2s, v1.2s
-; CHECK-NEXT:    fcvtzs x9, s5
-; CHECK-NEXT:    frintx v4.2s, v6.2s
-; CHECK-NEXT:    frintx v5.2s, v7.2s
-; CHECK-NEXT:    fcvtzs x10, s2
-; CHECK-NEXT:    mov s6, v2.s[1]
-; CHECK-NEXT:    fcvtzs x13, s3
-; CHECK-NEXT:    mov s3, v3.s[1]
-; CHECK-NEXT:    fcvtzs x11, s16
-; CHECK-NEXT:    fcvtzs x12, s17
-; CHECK-NEXT:    mov s7, v0.s[1]
-; CHECK-NEXT:    mov s16, v1.s[1]
-; CHECK-NEXT:    fcvtzs x15, s1
-; CHECK-NEXT:    mov s1, v4.s[1]
-; CHECK-NEXT:    mov s17, v5.s[1]
-; CHECK-NEXT:    fcvtzs x14, s0
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fcvtzs x8, s4
-; CHECK-NEXT:    fmov d4, x10
-; CHECK-NEXT:    fcvtzs x10, s5
-; CHECK-NEXT:    fmov d2, x9
-; CHECK-NEXT:    fcvtzs x9, s6
-; CHECK-NEXT:    fmov d6, x13
-; CHECK-NEXT:    fcvtzs x13, s7
-; CHECK-NEXT:    fcvtzs x16, s16
-; CHECK-NEXT:    fcvtzs x17, s3
-; CHECK-NEXT:    fcvtzs x18, s1
-; CHECK-NEXT:    fcvtzs x0, s17
-; CHECK-NEXT:    fmov d1, x14
-; CHECK-NEXT:    fmov d3, x15
-; CHECK-NEXT:    fmov d5, x8
-; CHECK-NEXT:    fmov d7, x10
-; CHECK-NEXT:    mov v0.d[1], x11
-; CHECK-NEXT:    mov v2.d[1], x12
-; CHECK-NEXT:    mov v4.d[1], x9
-; CHECK-NEXT:    mov v1.d[1], x13
-; CHECK-NEXT:    mov v3.d[1], x16
-; CHECK-NEXT:    mov v6.d[1], x17
-; CHECK-NEXT:    mov v5.d[1], x18
-; CHECK-NEXT:    mov v7.d[1], x0
+; CHECK-NEXT:    frintx v2.2s, v2.2s
+; CHECK-NEXT:    frintx v3.2s, v3.2s
+; CHECK-NEXT:    frintx v5.2s, v5.2s
+; CHECK-NEXT:    frintx v4.2s, v4.2s
+; CHECK-NEXT:    frintx v6.2s, v6.2s
+; CHECK-NEXT:    frintx v7.2s, v7.2s
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-NEXT:    fcvtl v16.2d, v2.2s
+; CHECK-NEXT:    fcvtl v18.2d, v3.2s
+; CHECK-NEXT:    fcvtl v5.2d, v5.2s
+; CHECK-NEXT:    fcvtl v17.2d, v4.2s
+; CHECK-NEXT:    fcvtl v19.2d, v6.2s
+; CHECK-NEXT:    fcvtl v7.2d, v7.2s
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v16.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v18.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v17.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v19.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
 ; CHECK-NEXT:    ret
   %a = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> %x)
   ret <16 x i64> %a
@@ -519,118 +474,70 @@ declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
 define <32 x i64> @llrint_v32i64_v32f32(<32 x float> %x) {
 ; CHECK-LABEL: llrint_v32i64_v32f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v17.16b, v3.16b, v3.16b, #8
-; CHECK-NEXT:    ext v18.16b, v4.16b, v4.16b, #8
-; CHECK-NEXT:    ext v19.16b, v5.16b, v5.16b, #8
-; CHECK-NEXT:    ext v21.16b, v7.16b, v7.16b, #8
-; CHECK-NEXT:    ext v16.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT:    ext v20.16b, v6.16b, v6.16b, #8
+; CHECK-NEXT:    ext v16.16b, v7.16b, v7.16b, #8
+; CHECK-NEXT:    ext v17.16b, v6.16b, v6.16b, #8
 ; CHECK-NEXT:    frintx v7.2s, v7.2s
-; CHECK-NEXT:    frintx v24.2s, v6.2s
-; CHECK-NEXT:    frintx v23.2s, v5.2s
+; CHECK-NEXT:    frintx v6.2s, v6.2s
+; CHECK-NEXT:    ext v18.16b, v5.16b, v5.16b, #8
+; CHECK-NEXT:    ext v21.16b, v4.16b, v4.16b, #8
+; CHECK-NEXT:    ext v22.16b, v2.16b, v2.16b, #8
+; CHECK-NEXT:    frintx v5.2s, v5.2s
+; CHECK-NEXT:    ext v23.16b, v3.16b, v3.16b, #8
 ; CHECK-NEXT:    frintx v4.2s, v4.2s
-; CHECK-NEXT:    frintx v3.2s, v3.2s
+; CHECK-NEXT:    ext v19.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ext v20.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    frintx v16.2s, v16.2s
 ; CHECK-NEXT:    frintx v17.2s, v17.2s
+; CHECK-NEXT:    fcvtl v7.2d, v7.2s
+; CHECK-NEXT:    fcvtl v6.2d, v6.2s
 ; CHECK-NEXT:    frintx v18.2s, v18.2s
-; CHECK-NEXT:    frintx v22.2s, v19.2s
 ; CHECK-NEXT:    frintx v21.2s, v21.2s
-; CHECK-NEXT:    frintx v16.2s, v16.2s
-; CHECK-NEXT:    frintx v20.2s, v20.2s
-; CHECK-NEXT:    mov s25, v7.s[1]
-; CHECK-NEXT:    fcvtzs x15, s7
-; CHECK-NEXT:    frintx v19.2s, v1.2s
-; CHECK-NEXT:    fcvtzs x16, s24
-; CHECK-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    fcvtzs x10, s17
-; CHECK-NEXT:    fcvtzs x11, s18
-; CHECK-NEXT:    mov s26, v22.s[1]
-; CHECK-NEXT:    fcvtzs x12, s22
-; CHECK-NEXT:    mov s22, v21.s[1]
-; CHECK-NEXT:    fcvtzs x14, s21
-; CHECK-NEXT:    mov s21, v24.s[1]
-; CHECK-NEXT:    fcvtzs x9, s16
-; CHECK-NEXT:    fcvtzs x13, s20
-; CHECK-NEXT:    mov s20, v20.s[1]
-; CHECK-NEXT:    fmov d24, x15
-; CHECK-NEXT:    mov s18, v18.s[1]
-; CHECK-NEXT:    fmov d6, x10
-; CHECK-NEXT:    fmov d7, x11
-; CHECK-NEXT:    fcvtzs x10, s25
-; CHECK-NEXT:    fcvtzs x11, s22
-; CHECK-NEXT:    fmov d25, x12
-; CHECK-NEXT:    frintx v22.2s, v2.2s
-; CHECK-NEXT:    fcvtzs x15, s21
-; CHECK-NEXT:    fmov d21, x14
-; CHECK-NEXT:    fmov d5, x9
-; CHECK-NEXT:    fcvtzs x9, s26
-; CHECK-NEXT:    fmov d26, x13
-; CHECK-NEXT:    fcvtzs x12, s20
-; CHECK-NEXT:    fcvtzs x13, s19
-; CHECK-NEXT:    mov s20, v23.s[1]
-; CHECK-NEXT:    mov v24.d[1], x10
-; CHECK-NEXT:    mov v21.d[1], x11
-; CHECK-NEXT:    fcvtzs x11, s23
-; CHECK-NEXT:    fcvtzs x10, s22
-; CHECK-NEXT:    mov s17, v17.s[1]
+; CHECK-NEXT:    frintx v2.2s, v2.2s
+; CHECK-NEXT:    frintx v3.2s, v3.2s
+; CHECK-NEXT:    fcvtl v5.2d, v5.2s
+; CHECK-NEXT:    frintx v23.2s, v23.2s
+; CHECK-NEXT:    fcvtl v4.2d, v4.2s
 ; CHECK-NEXT:    frintx v1.2s, v1.2s
-; CHECK-NEXT:    mov s22, v22.s[1]
-; CHECK-NEXT:    mov v26.d[1], x12
-; CHECK-NEXT:    fcvtzs x12, s18
-; CHECK-NEXT:    mov v25.d[1], x9
-; CHECK-NEXT:    fmov d2, x13
-; CHECK-NEXT:    fcvtzs x13, s20
-; CHECK-NEXT:    fmov d20, x16
-; CHECK-NEXT:    stp q24, q21, [x8, #224]
-; CHECK-NEXT:    ext v21.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    fmov d18, x11
-; CHECK-NEXT:    fcvtzs x11, s4
-; CHECK-NEXT:    mov s4, v4.s[1]
-; CHECK-NEXT:    fmov d23, x10
-; CHECK-NEXT:    mov v20.d[1], x15
-; CHECK-NEXT:    fcvtzs x10, s3
-; CHECK-NEXT:    mov s3, v3.s[1]
-; CHECK-NEXT:    mov v18.d[1], x13
+; CHECK-NEXT:    fcvtl v16.2d, v16.2s
+; CHECK-NEXT:    fcvtl v17.2d, v17.2s
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtl v18.2d, v18.2s
+; CHECK-NEXT:    fcvtl v21.2d, v21.2s
+; CHECK-NEXT:    frintx v20.2s, v20.2s
+; CHECK-NEXT:    fcvtl v3.2d, v3.2s
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
 ; CHECK-NEXT:    frintx v0.2s, v0.2s
-; CHECK-NEXT:    mov s16, v16.s[1]
-; CHECK-NEXT:    frintx v21.2s, v21.2s
-; CHECK-NEXT:    fcvtzs x13, s17
-; CHECK-NEXT:    fcvtzs x14, s22
-; CHECK-NEXT:    fcvtzs x9, s4
-; CHECK-NEXT:    fmov d4, x11
-; CHECK-NEXT:    mov v7.d[1], x12
-; CHECK-NEXT:    stp q20, q26, [x8, #192]
-; CHECK-NEXT:    fmov d20, x10
-; CHECK-NEXT:    fcvtzs x10, s3
-; CHECK-NEXT:    stp q18, q25, [x8, #160]
-; CHECK-NEXT:    mov s18, v19.s[1]
-; CHECK-NEXT:    mov s3, v1.s[1]
-; CHECK-NEXT:    mov s17, v0.s[1]
-; CHECK-NEXT:    mov s19, v21.s[1]
-; CHECK-NEXT:    fcvtzs x11, s21
-; CHECK-NEXT:    mov v4.d[1], x9
-; CHECK-NEXT:    fcvtzs x9, s16
-; CHECK-NEXT:    fcvtzs x12, s1
-; CHECK-NEXT:    mov v6.d[1], x13
-; CHECK-NEXT:    fcvtzs x13, s0
-; CHECK-NEXT:    mov v20.d[1], x10
-; CHECK-NEXT:    fcvtzs x15, s18
-; CHECK-NEXT:    fcvtzs x10, s3
-; CHECK-NEXT:    mov v23.d[1], x14
-; CHECK-NEXT:    fcvtzs x14, s17
-; CHECK-NEXT:    fmov d3, x11
-; CHECK-NEXT:    stp q4, q7, [x8, #128]
-; CHECK-NEXT:    mov v5.d[1], x9
-; CHECK-NEXT:    fcvtzs x9, s19
-; CHECK-NEXT:    stp q20, q6, [x8, #96]
-; CHECK-NEXT:    fmov d0, x12
-; CHECK-NEXT:    fmov d1, x13
-; CHECK-NEXT:    mov v2.d[1], x15
-; CHECK-NEXT:    stp q23, q5, [x8, #64]
-; CHECK-NEXT:    mov v0.d[1], x10
-; CHECK-NEXT:    mov v1.d[1], x14
-; CHECK-NEXT:    mov v3.d[1], x9
-; CHECK-NEXT:    stp q2, q0, [x8, #32]
-; CHECK-NEXT:    stp q1, q3, [x8]
+; CHECK-NEXT:    fcvtl v2.2d, v2.2s
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    stp q6, q17, [x8, #192]
+; CHECK-NEXT:    fcvtl v6.2d, v23.2s
+; CHECK-NEXT:    frintx v17.2s, v19.2s
+; CHECK-NEXT:    stp q7, q16, [x8, #224]
+; CHECK-NEXT:    frintx v7.2s, v22.2s
+; CHECK-NEXT:    fcvtzs v16.2d, v18.2d
+; CHECK-NEXT:    fcvtzs v18.2d, v21.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    stp q5, q16, [x8, #160]
+; CHECK-NEXT:    fcvtl v7.2d, v7.2s
+; CHECK-NEXT:    fcvtl v5.2d, v20.2s
+; CHECK-NEXT:    stp q4, q18, [x8, #128]
+; CHECK-NEXT:    fcvtl v4.2d, v17.2s
+; CHECK-NEXT:    stp q3, q6, [x8, #96]
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v5.2d
+; CHECK-NEXT:    stp q1, q3, [x8, #32]
+; CHECK-NEXT:    stp q2, q7, [x8, #64]
+; CHECK-NEXT:    fcvtzs v2.2d, v4.2d
+; CHECK-NEXT:    stp q0, q2, [x8]
 ; CHECK-NEXT:    ret
   %a = call <32 x i64> @llvm.llrint.v32i64.v32f32(<32 x float> %x)
   ret <32 x i64> %a
diff --git a/llvm/test/CodeGen/AArch64/vector-lrint.ll b/llvm/test/CodeGen/AArch64/vector-lrint.ll
index 44f29f1420fe..602643264e7b 100644
--- a/llvm/test/CodeGen/AArch64/vector-lrint.ll
+++ b/llvm/test/CodeGen/AArch64/vector-lrint.ll
@@ -784,11 +784,8 @@ define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
 ; CHECK-i64-LABEL: lrint_v2f32:
 ; CHECK-i64:       // %bb.0:
 ; CHECK-i64-NEXT:    frintx v0.2s, v0.2s
-; CHECK-i64-NEXT:    mov s1, v0.s[1]
-; CHECK-i64-NEXT:    fcvtzs x8, s0
-; CHECK-i64-NEXT:    fcvtzs x9, s1
-; CHECK-i64-NEXT:    fmov d0, x8
-; CHECK-i64-NEXT:    mov v0.d[1], x9
+; CHECK-i64-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
 ; CHECK-i64-NEXT:    ret
   %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float> %x)
   ret <2 x iXLen> %a
@@ -807,16 +804,10 @@ define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
 ; CHECK-i64-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-i64-NEXT:    frintx v0.2s, v0.2s
 ; CHECK-i64-NEXT:    frintx v1.2s, v1.2s
-; CHECK-i64-NEXT:    mov s2, v0.s[1]
-; CHECK-i64-NEXT:    fcvtzs x8, s0
-; CHECK-i64-NEXT:    mov s3, v1.s[1]
-; CHECK-i64-NEXT:    fcvtzs x9, s1
-; CHECK-i64-NEXT:    fcvtzs x10, s2
-; CHECK-i64-NEXT:    fmov d0, x8
-; CHECK-i64-NEXT:    fcvtzs x11, s3
-; CHECK-i64-NEXT:    fmov d1, x9
-; CHECK-i64-NEXT:    mov v0.d[1], x10
-; CHECK-i64-NEXT:    mov v1.d[1], x11
+; CHECK-i64-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-i64-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-i64-NEXT:    fcvtzs v1.2d, v1.2d
 ; CHECK-i64-NEXT:    ret
   %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float> %x)
   ret <4 x iXLen> %a
@@ -840,26 +831,14 @@ define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
 ; CHECK-i64-NEXT:    frintx v1.2s, v1.2s
 ; CHECK-i64-NEXT:    frintx v2.2s, v2.2s
 ; CHECK-i64-NEXT:    frintx v3.2s, v3.2s
-; CHECK-i64-NEXT:    mov s4, v0.s[1]
-; CHECK-i64-NEXT:    mov s5, v1.s[1]
-; CHECK-i64-NEXT:    fcvtzs x8, s0
-; CHECK-i64-NEXT:    fcvtzs x10, s1
-; CHECK-i64-NEXT:    mov s6, v2.s[1]
-; CHECK-i64-NEXT:    mov s7, v3.s[1]
-; CHECK-i64-NEXT:    fcvtzs x11, s2
-; CHECK-i64-NEXT:    fcvtzs x12, s3
-; CHECK-i64-NEXT:    fcvtzs x9, s4
-; CHECK-i64-NEXT:    fcvtzs x13, s5
-; CHECK-i64-NEXT:    fmov d0, x8
-; CHECK-i64-NEXT:    fmov d2, x10
-; CHECK-i64-NEXT:    fcvtzs x14, s6
-; CHECK-i64-NEXT:    fcvtzs x15, s7
-; CHECK-i64-NEXT:    fmov d1, x11
-; CHECK-i64-NEXT:    fmov d3, x12
-; CHECK-i64-NEXT:    mov v0.d[1], x9
-; CHECK-i64-NEXT:    mov v2.d[1], x13
-; CHECK-i64-NEXT:    mov v1.d[1], x14
-; CHECK-i64-NEXT:    mov v3.d[1], x15
+; CHECK-i64-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-i64-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-i64-NEXT:    fcvtl v4.2d, v2.2s
+; CHECK-i64-NEXT:    fcvtl v3.2d, v3.2s
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-i64-NEXT:    fcvtzs v2.2d, v1.2d
+; CHECK-i64-NEXT:    fcvtzs v1.2d, v4.2d
+; CHECK-i64-NEXT:    fcvtzs v3.2d, v3.2d
 ; CHECK-i64-NEXT:    ret
   %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float> %x)
   ret <8 x iXLen> %a
@@ -881,58 +860,34 @@ define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
 ;
 ; CHECK-i64-LABEL: lrint_v16f32:
 ; CHECK-i64:       // %bb.0:
-; CHECK-i64-NEXT:    frintx v4.2s, v0.2s
-; CHECK-i64-NEXT:    frintx v5.2s, v1.2s
-; CHECK-i64-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-i64-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-i64-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-i64-NEXT:    ext v5.16b, v0.16b, v0.16b, #8
 ; CHECK-i64-NEXT:    ext v6.16b, v2.16b, v2.16b, #8
 ; CHECK-i64-NEXT:    ext v7.16b, v3.16b, v3.16b, #8
-; CHECK-i64-NEXT:    frintx v2.2s, v2.2s
-; CHECK-i64-NEXT:    frintx v3.2s, v3.2s
-; CHECK-i64-NEXT:    mov s16, v4.s[1]
-; CHECK-i64-NEXT:    mov s17, v5.s[1]
-; CHECK-i64-NEXT:    fcvtzs x8, s4
 ; CHECK-i64-NEXT:    frintx v0.2s, v0.2s
 ; CHECK-i64-NEXT:    frintx v1.2s, v1.2s
-; CHECK-i64-NEXT:    fcvtzs x9, s5
-; CHECK-i64-NEXT:    frintx v4.2s, v6.2s
-; CHECK-i64-NEXT:    frintx v5.2s, v7.2s
-; CHECK-i64-NEXT:    fcvtzs x10, s2
-; CHECK-i64-NEXT:    mov s6, v2.s[1]
-; CHECK-i64-NEXT:    fcvtzs x13, s3
-; CHECK-i64-NEXT:    mov s3, v3.s[1]
-; CHECK-i64-NEXT:    fcvtzs x11, s16
-; CHECK-i64-NEXT:    fcvtzs x12, s17
-; CHECK-i64-NEXT:    mov s7, v0.s[1]
-; CHECK-i64-NEXT:    mov s16, v1.s[1]
-; CHECK-i64-NEXT:    fcvtzs x15, s1
-; CHECK-i64-NEXT:    mov s1, v4.s[1]
-; CHECK-i64-NEXT:    mov s17, v5.s[1]
-; CHECK-i64-NEXT:    fcvtzs x14, s0
-; CHECK-i64-NEXT:    fmov d0, x8
-; CHECK-i64-NEXT:    fcvtzs x8, s4
-; CHECK-i64-NEXT:    fmov d4, x10
-; CHECK-i64-NEXT:    fcvtzs x10, s5
-; CHECK-i64-NEXT:    fmov d2, x9
-; CHECK-i64-NEXT:    fcvtzs x9, s6
-; CHECK-i64-NEXT:    fmov d6, x13
-; CHECK-i64-NEXT:    fcvtzs x13, s7
-; CHECK-i64-NEXT:    fcvtzs x16, s16
-; CHECK-i64-NEXT:    fcvtzs x17, s3
-; CHECK-i64-NEXT:    fcvtzs x18, s1
-; CHECK-i64-NEXT:    fcvtzs x0, s17
-; CHECK-i64-NEXT:    fmov d1, x14
-; CHECK-i64-NEXT:    fmov d3, x15
-; CHECK-i64-NEXT:    fmov d5, x8
-; CHECK-i64-NEXT:    fmov d7, x10
-; CHECK-i64-NEXT:    mov v0.d[1], x11
-; CHECK-i64-NEXT:    mov v2.d[1], x12
-; CHECK-i64-NEXT:    mov v4.d[1], x9
-; CHECK-i64-NEXT:    mov v1.d[1], x13
-; CHECK-i64-NEXT:    mov v3.d[1], x16
-; CHECK-i64-NEXT:    mov v6.d[1], x17
-; CHECK-i64-NEXT:    mov v5.d[1], x18
-; CHECK-i64-NEXT:    mov v7.d[1], x0
+; CHECK-i64-NEXT:    frintx v2.2s, v2.2s
+; CHECK-i64-NEXT:    frintx v3.2s, v3.2s
+; CHECK-i64-NEXT:    frintx v5.2s, v5.2s
+; CHECK-i64-NEXT:    frintx v4.2s, v4.2s
+; CHECK-i64-NEXT:    frintx v6.2s, v6.2s
+; CHECK-i64-NEXT:    frintx v7.2s, v7.2s
+; CHECK-i64-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-i64-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-i64-NEXT:    fcvtl v16.2d, v2.2s
+; CHECK-i64-NEXT:    fcvtl v18.2d, v3.2s
+; CHECK-i64-NEXT:    fcvtl v5.2d, v5.2s
+; CHECK-i64-NEXT:    fcvtl v17.2d, v4.2s
+; CHECK-i64-NEXT:    fcvtl v19.2d, v6.2s
+; CHECK-i64-NEXT:    fcvtl v7.2d, v7.2s
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-i64-NEXT:    fcvtzs v2.2d, v1.2d
+; CHECK-i64-NEXT:    fcvtzs v4.2d, v16.2d
+; CHECK-i64-NEXT:    fcvtzs v6.2d, v18.2d
+; CHECK-i64-NEXT:    fcvtzs v1.2d, v5.2d
+; CHECK-i64-NEXT:    fcvtzs v3.2d, v17.2d
+; CHECK-i64-NEXT:    fcvtzs v5.2d, v19.2d
+; CHECK-i64-NEXT:    fcvtzs v7.2d, v7.2d
 ; CHECK-i64-NEXT:    ret
   %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float> %x)
   ret <16 x iXLen> %a
@@ -962,118 +917,70 @@ define <32 x iXLen> @lrint_v32f32(<32 x float> %x) {
 ;
 ; CHECK-i64-LABEL: lrint_v32f32:
 ; CHECK-i64:       // %bb.0:
-; CHECK-i64-NEXT:    ext v17.16b, v3.16b, v3.16b, #8
-; CHECK-i64-NEXT:    ext v18.16b, v4.16b, v4.16b, #8
-; CHECK-i64-NEXT:    ext v19.16b, v5.16b, v5.16b, #8
-; CHECK-i64-NEXT:    ext v21.16b, v7.16b, v7.16b, #8
-; CHECK-i64-NEXT:    ext v16.16b, v2.16b, v2.16b, #8
-; CHECK-i64-NEXT:    ext v20.16b, v6.16b, v6.16b, #8
+; CHECK-i64-NEXT:    ext v16.16b, v7.16b, v7.16b, #8
+; CHECK-i64-NEXT:    ext v17.16b, v6.16b, v6.16b, #8
 ; CHECK-i64-NEXT:    frintx v7.2s, v7.2s
-; CHECK-i64-NEXT:    frintx v24.2s, v6.2s
-; CHECK-i64-NEXT:    frintx v23.2s, v5.2s
+; CHECK-i64-NEXT:    frintx v6.2s, v6.2s
+; CHECK-i64-NEXT:    ext v18.16b, v5.16b, v5.16b, #8
+; CHECK-i64-NEXT:    ext v21.16b, v4.16b, v4.16b, #8
+; CHECK-i64-NEXT:    ext v22.16b, v2.16b, v2.16b, #8
+; CHECK-i64-NEXT:    frintx v5.2s, v5.2s
+; CHECK-i64-NEXT:    ext v23.16b, v3.16b, v3.16b, #8
 ; CHECK-i64-NEXT:    frintx v4.2s, v4.2s
-; CHECK-i64-NEXT:    frintx v3.2s, v3.2s
+; CHECK-i64-NEXT:    ext v19.16b, v0.16b, v0.16b, #8
+; CHECK-i64-NEXT:    ext v20.16b, v1.16b, v1.16b, #8
+; CHECK-i64-NEXT:    frintx v16.2s, v16.2s
 ; CHECK-i64-NEXT:    frintx v17.2s, v17.2s
+; CHECK-i64-NEXT:    fcvtl v7.2d, v7.2s
+; CHECK-i64-NEXT:    fcvtl v6.2d, v6.2s
 ; CHECK-i64-NEXT:    frintx v18.2s, v18.2s
-; CHECK-i64-NEXT:    frintx v22.2s, v19.2s
 ; CHECK-i64-NEXT:    frintx v21.2s, v21.2s
-; CHECK-i64-NEXT:    frintx v16.2s, v16.2s
-; CHECK-i64-NEXT:    frintx v20.2s, v20.2s
-; CHECK-i64-NEXT:    mov s25, v7.s[1]
-; CHECK-i64-NEXT:    fcvtzs x15, s7
-; CHECK-i64-NEXT:    frintx v19.2s, v1.2s
-; CHECK-i64-NEXT:    fcvtzs x16, s24
-; CHECK-i64-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-i64-NEXT:    fcvtzs x10, s17
-; CHECK-i64-NEXT:    fcvtzs x11, s18
-; CHECK-i64-NEXT:    mov s26, v22.s[1]
-; CHECK-i64-NEXT:    fcvtzs x12, s22
-; CHECK-i64-NEXT:    mov s22, v21.s[1]
-; CHECK-i64-NEXT:    fcvtzs x14, s21
-; CHECK-i64-NEXT:    mov s21, v24.s[1]
-; CHECK-i64-NEXT:    fcvtzs x9, s16
-; CHECK-i64-NEXT:    fcvtzs x13, s20
-; CHECK-i64-NEXT:    mov s20, v20.s[1]
-; CHECK-i64-NEXT:    fmov d24, x15
-; CHECK-i64-NEXT:    mov s18, v18.s[1]
-; CHECK-i64-NEXT:    fmov d6, x10
-; CHECK-i64-NEXT:    fmov d7, x11
-; CHECK-i64-NEXT:    fcvtzs x10, s25
-; CHECK-i64-NEXT:    fcvtzs x11, s22
-; CHECK-i64-NEXT:    fmov d25, x12
-; CHECK-i64-NEXT:    frintx v22.2s, v2.2s
-; CHECK-i64-NEXT:    fcvtzs x15, s21
-; CHECK-i64-NEXT:    fmov d21, x14
-; CHECK-i64-NEXT:    fmov d5, x9
-; CHECK-i64-NEXT:    fcvtzs x9, s26
-; CHECK-i64-NEXT:    fmov d26, x13
-; CHECK-i64-NEXT:    fcvtzs x12, s20
-; CHECK-i64-NEXT:    fcvtzs x13, s19
-; CHECK-i64-NEXT:    mov s20, v23.s[1]
-; CHECK-i64-NEXT:    mov v24.d[1], x10
-; CHECK-i64-NEXT:    mov v21.d[1], x11
-; CHECK-i64-NEXT:    fcvtzs x11, s23
-; CHECK-i64-NEXT:    fcvtzs x10, s22
-; CHECK-i64-NEXT:    mov s17, v17.s[1]
+; CHECK-i64-NEXT:    frintx v2.2s, v2.2s
+; CHECK-i64-NEXT:    frintx v3.2s, v3.2s
+; CHECK-i64-NEXT:    fcvtl v5.2d, v5.2s
+; CHECK-i64-NEXT:    frintx v23.2s, v23.2s
+; CHECK-i64-NEXT:    fcvtl v4.2d, v4.2s
 ; CHECK-i64-NEXT:    frintx v1.2s, v1.2s
-; CHECK-i64-NEXT:    mov s22, v22.s[1]
-; CHECK-i64-NEXT:    mov v26.d[1], x12
-; CHECK-i64-NEXT:    fcvtzs x12, s18
-; CHECK-i64-NEXT:    mov v25.d[1], x9
-; CHECK-i64-NEXT:    fmov d2, x13
-; CHECK-i64-NEXT:    fcvtzs x13, s20
-; CHECK-i64-NEXT:    fmov d20, x16
-; CHECK-i64-NEXT:    stp q24, q21, [x8, #224]
-; CHECK-i64-NEXT:    ext v21.16b, v0.16b, v0.16b, #8
-; CHECK-i64-NEXT:    fmov d18, x11
-; CHECK-i64-NEXT:    fcvtzs x11, s4
-; CHECK-i64-NEXT:    mov s4, v4.s[1]
-; CHECK-i64-NEXT:    fmov d23, x10
-; CHECK-i64-NEXT:    mov v20.d[1], x15
-; CHECK-i64-NEXT:    fcvtzs x10, s3
-; CHECK-i64-NEXT:    mov s3, v3.s[1]
-; CHECK-i64-NEXT:    mov v18.d[1], x13
+; CHECK-i64-NEXT:    fcvtl v16.2d, v16.2s
+; CHECK-i64-NEXT:    fcvtl v17.2d, v17.2s
+; CHECK-i64-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-i64-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-i64-NEXT:    fcvtl v18.2d, v18.2s
+; CHECK-i64-NEXT:    fcvtl v21.2d, v21.2s
+; CHECK-i64-NEXT:    frintx v20.2s, v20.2s
+; CHECK-i64-NEXT:    fcvtl v3.2d, v3.2s
+; CHECK-i64-NEXT:    fcvtzs v5.2d, v5.2d
 ; CHECK-i64-NEXT:    frintx v0.2s, v0.2s
-; CHECK-i64-NEXT:    mov s16, v16.s[1]
-; CHECK-i64-NEXT:    frintx v21.2s, v21.2s
-; CHECK-i64-NEXT:    fcvtzs x13, s17
-; CHECK-i64-NEXT:    fcvtzs x14, s22
-; CHECK-i64-NEXT:    fcvtzs x9, s4
-; CHECK-i64-NEXT:    fmov d4, x11
-; CHECK-i64-NEXT:    mov v7.d[1], x12
-; CHECK-i64-NEXT:    stp q20, q26, [x8, #192]
-; CHECK-i64-NEXT:    fmov d20, x10
-; CHECK-i64-NEXT:    fcvtzs x10, s3
-; CHECK-i64-NEXT:    stp q18, q25, [x8, #160]
-; CHECK-i64-NEXT:    mov s18, v19.s[1]
-; CHECK-i64-NEXT:    mov s3, v1.s[1]
-; CHECK-i64-NEXT:    mov s17, v0.s[1]
-; CHECK-i64-NEXT:    mov s19, v21.s[1]
-; CHECK-i64-NEXT:    fcvtzs x11, s21
-; CHECK-i64-NEXT:    mov v4.d[1], x9
-; CHECK-i64-NEXT:    fcvtzs x9, s16
-; CHECK-i64-NEXT:    fcvtzs x12, s1
-; CHECK-i64-NEXT:    mov v6.d[1], x13
-; CHECK-i64-NEXT:    fcvtzs x13, s0
-; CHECK-i64-NEXT:    mov v20.d[1], x10
-; CHECK-i64-NEXT:    fcvtzs x15, s18
-; CHECK-i64-NEXT:    fcvtzs x10, s3
-; CHECK-i64-NEXT:    mov v23.d[1], x14
-; CHECK-i64-NEXT:    fcvtzs x14, s17
-; CHECK-i64-NEXT:    fmov d3, x11
-; CHECK-i64-NEXT:    stp q4, q7, [x8, #128]
-; CHECK-i64-NEXT:    mov v5.d[1], x9
-; CHECK-i64-NEXT:    fcvtzs x9, s19
-; CHECK-i64-NEXT:    stp q20, q6, [x8, #96]
-; CHECK-i64-NEXT:    fmov d0, x12
-; CHECK-i64-NEXT:    fmov d1, x13
-; CHECK-i64-NEXT:    mov v2.d[1], x15
-; CHECK-i64-NEXT:    stp q23, q5, [x8, #64]
-; CHECK-i64-NEXT:    mov v0.d[1], x10
-; CHECK-i64-NEXT:    mov v1.d[1], x14
-; CHECK-i64-NEXT:    mov v3.d[1], x9
-; CHECK-i64-NEXT:    stp q2, q0, [x8, #32]
-; CHECK-i64-NEXT:    stp q1, q3, [x8]
+; CHECK-i64-NEXT:    fcvtl v2.2d, v2.2s
+; CHECK-i64-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-i64-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-i64-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-i64-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-i64-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-i64-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-i64-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-i64-NEXT:    stp q6, q17, [x8, #192]
+; CHECK-i64-NEXT:    fcvtl v6.2d, v23.2s
+; CHECK-i64-NEXT:    frintx v17.2s, v19.2s
+; CHECK-i64-NEXT:    stp q7, q16, [x8, #224]
+; CHECK-i64-NEXT:    frintx v7.2s, v22.2s
+; CHECK-i64-NEXT:    fcvtzs v16.2d, v18.2d
+; CHECK-i64-NEXT:    fcvtzs v18.2d, v21.2d
+; CHECK-i64-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-i64-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-i64-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-i64-NEXT:    stp q5, q16, [x8, #160]
+; CHECK-i64-NEXT:    fcvtl v7.2d, v7.2s
+; CHECK-i64-NEXT:    fcvtl v5.2d, v20.2s
+; CHECK-i64-NEXT:    stp q4, q18, [x8, #128]
+; CHECK-i64-NEXT:    fcvtl v4.2d, v17.2s
+; CHECK-i64-NEXT:    stp q3, q6, [x8, #96]
+; CHECK-i64-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-i64-NEXT:    fcvtzs v3.2d, v5.2d
+; CHECK-i64-NEXT:    stp q1, q3, [x8, #32]
+; CHECK-i64-NEXT:    stp q2, q7, [x8, #64]
+; CHECK-i64-NEXT:    fcvtzs v2.2d, v4.2d
+; CHECK-i64-NEXT:    stp q0, q2, [x8]
 ; CHECK-i64-NEXT:    ret
   %a = call <32 x iXLen> @llvm.lrint.v32iXLen.v32f32(<32 x float> %x)
   ret <32 x iXLen> %a
-- 
GitLab


From ae2a18d6cbc3328410bdb7e629fdc59766b73e4b Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Mon, 13 May 2024 07:47:33 +0000
Subject: [PATCH 0528/1206] [mlir][Bazel] Adjust BUILD.bazel for
 eeafc9daa15d2d022bcdd456d4b8bafd23f5f121

---
 utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 0ebfcbe284bd..65b31dc97e2d 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -121,6 +121,7 @@ td_library(
         "//mlir:InferIntRangeInterfaceTdFiles",
         "//mlir:InferTypeOpInterfaceTdFiles",
         "//mlir:LinalgStructuredOpsTdFiles",
+        "//mlir:MemorySlotInterfacesTdFiles",
         "//mlir:OpBaseTdFiles",
         "//mlir:SideEffectInterfacesTdFiles",
     ],
@@ -418,6 +419,7 @@ cc_library(
         "//mlir:LLVMIRToLLVMTranslation",
         "//mlir:LinalgDialect",
         "//mlir:LoopLikeInterface",
+        "//mlir:MemorySlotInterfaces",
         "//mlir:Pass",
         "//mlir:Reducer",
         "//mlir:SideEffectInterfaces",
-- 
GitLab


From 279a659e9772e48d95ad7d81f6deb00ee31e35e1 Mon Sep 17 00:00:00 2001
From: Corentin Ferry <corentin.ferry@amd.com>
Date: Mon, 13 May 2024 10:15:39 +0200
Subject: [PATCH 0529/1206] [mlir][math] lower rsqrt to sqrt + fdiv (#91344)

This commit creates an expansion pattern to lower math.rsqrt(x) into
fdiv(1, sqrt(x)).
---
 .../mlir/Dialect/Math/Transforms/Passes.h     |  1 +
 .../Math/Transforms/ExpandPatterns.cpp        | 21 ++++++
 mlir/test/Dialect/Math/expand-math.mlir       | 70 +++++++++++++++++++
 mlir/test/lib/Dialect/Math/TestExpandMath.cpp |  1 +
 .../test-expand-math-approx.mlir              | 41 +++++++++++
 5 files changed, 134 insertions(+)

diff --git a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
index 24e6d9a8d98e..ba6977251564 100644
--- a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
@@ -42,6 +42,7 @@ void populateExpandPowFPattern(RewritePatternSet &patterns);
 void populateExpandFPowIPattern(RewritePatternSet &patterns);
 void populateExpandRoundFPattern(RewritePatternSet &patterns);
 void populateExpandRoundEvenPattern(RewritePatternSet &patterns);
+void populateExpandRsqrtPattern(RewritePatternSet &patterns);
 void populateMathAlgebraicSimplificationPatterns(RewritePatternSet &patterns);
 
 struct MathPolynomialApproximationOptions {
diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
index 5ccf3b6d72a2..80569d95137c 100644
--- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp
@@ -615,6 +615,23 @@ static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
   return success();
 }
 
+// Convert `math.rsqrt` into `arith.divf` + `math.sqrt`
+static LogicalResult convertRsqrtOp(math::RsqrtOp op,
+                                    PatternRewriter &rewriter) {
+
+  auto operand = op.getOperand();
+  auto operandTy = operand.getType();
+  auto eTy = getElementTypeOrSelf(operandTy);
+  if (!isa<FloatType>(eTy))
+    return failure();
+
+  Location loc = op->getLoc();
+  auto constOneFloat = createFloatConst(loc, operandTy, 1.0, rewriter);
+  auto sqrtOp = rewriter.create<math::SqrtOp>(loc, operand);
+  rewriter.replaceOpWithNewOp<arith::DivFOp>(op, constOneFloat, sqrtOp);
+  return success();
+}
+
 void mlir::populateExpandCtlzPattern(RewritePatternSet &patterns) {
   patterns.add(convertCtlzOp);
 }
@@ -678,3 +695,7 @@ void mlir::populateExpandFloorFPattern(RewritePatternSet &patterns) {
 void mlir::populateExpandRoundEvenPattern(RewritePatternSet &patterns) {
   patterns.add(convertRoundEvenOp);
 }
+
+void mlir::populateExpandRsqrtPattern(RewritePatternSet &patterns) {
+  patterns.add(convertRsqrtOp);
+}
diff --git a/mlir/test/Dialect/Math/expand-math.mlir b/mlir/test/Dialect/Math/expand-math.mlir
index 3d94b55126d0..016a7bbdeb56 100644
--- a/mlir/test/Dialect/Math/expand-math.mlir
+++ b/mlir/test/Dialect/Math/expand-math.mlir
@@ -658,3 +658,73 @@ func.func @math_fpowi_to_powf_scalar(%0 : f32, %1: i64) -> f32 {
 // CHECK:        %[[AND:.*]] = arith.andi %[[CMPF1]], %[[CMPF]] : i1
 // CHECK:        %[[SEL:.*]] = arith.select %[[AND]], %[[MUL1]], %[[EXP]] : f32
 // CHECK:       return %[[SEL]] : f32
+
+// -----
+
+// CHECK-LABEL:   func.func @rsqrt
+// CHECK-SAME:     (%[[ARG:.*]]: f16)
+// CHECK-SAME:    -> f16
+// CHECK-DAG:     %[[CST:.*]] = arith.constant 1.000000e+00 : f16
+// CHECK-DAG:     %[[SQRT:.*]] = math.sqrt %[[ARG]] : f16
+// CHECK-DAG:     %[[DIV:.*]] = arith.divf %[[CST]], %[[SQRT]] : f16
+// CHECK:         return %[[DIV]] : f16
+func.func @rsqrt16(%float: f16) -> (f16)  {
+  %float_result = math.rsqrt %float : f16
+  return %float_result : f16
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @rsqrt
+// CHECK-SAME:     (%[[ARG:.*]]: f32)
+// CHECK-SAME:    -> f32
+// CHECK-DAG:     %[[CST:.*]] = arith.constant 1.000000e+00 : f32
+// CHECK-DAG:     %[[SQRT:.*]] = math.sqrt %[[ARG]] : f32
+// CHECK-DAG:     %[[DIV:.*]] = arith.divf %[[CST]], %[[SQRT]] : f32
+// CHECK:         return %[[DIV]] : f32
+func.func @rsqrt32(%float: f32) -> (f32)  {
+  %float_result = math.rsqrt %float : f32
+  return %float_result : f32
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @rsqrt
+// CHECK-SAME:     (%[[ARG:.*]]: f64)
+// CHECK-SAME:    -> f64
+// CHECK-DAG:     %[[CST:.*]] = arith.constant 1.000000e+00 : f64
+// CHECK-DAG:     %[[SQRT:.*]] = math.sqrt %[[ARG]] : f64
+// CHECK-DAG:     %[[DIV:.*]] = arith.divf %[[CST]], %[[SQRT]] : f64
+// CHECK:         return %[[DIV]] : f64
+func.func @rsqrt64(%float: f64) -> (f64)  {
+  %float_result = math.rsqrt %float : f64
+  return %float_result : f64
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @rsqrt_vec
+// CHECK-SAME:     (%[[ARG:.*]]: vector<5xf32>)
+// CHECK-SAME:    -> vector<5xf32>
+// CHECK-DAG:     %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<5xf32>
+// CHECK-DAG:     %[[SQRT:.*]] = math.sqrt %[[ARG]] : vector<5xf32>
+// CHECK-DAG:     %[[DIV:.*]] = arith.divf %[[CST]], %[[SQRT]] : vector<5xf32>
+// CHECK:         return %[[DIV]] : vector<5xf32>
+func.func @rsqrt_vec(%float: vector<5xf32>) -> (vector<5xf32>)  {
+  %float_result = math.rsqrt %float : vector<5xf32>
+  return %float_result : vector<5xf32>
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @rsqrt_tns
+// CHECK-SAME:     (%[[ARG:.*]]: tensor<5x8xf32>)
+// CHECK-SAME:    -> tensor<5x8xf32>
+// CHECK-DAG:     %[[CST:.*]] = arith.constant dense<1.000000e+00> : tensor<5x8xf32>
+// CHECK-DAG:     %[[SQRT:.*]] = math.sqrt %[[ARG]] : tensor<5x8xf32>
+// CHECK-DAG:     %[[DIV:.*]] = arith.divf %[[CST]], %[[SQRT]] : tensor<5x8xf32>
+// CHECK:         return %[[DIV]] : tensor<5x8xf32>
+func.func @rsqrt_tns(%float: tensor<5x8xf32>) -> (tensor<5x8xf32>)  {
+  %float_result = math.rsqrt %float : tensor<5x8xf32>
+  return %float_result : tensor<5x8xf32>
+}
diff --git a/mlir/test/lib/Dialect/Math/TestExpandMath.cpp b/mlir/test/lib/Dialect/Math/TestExpandMath.cpp
index da48ccb6e5e0..69af2a08b97b 100644
--- a/mlir/test/lib/Dialect/Math/TestExpandMath.cpp
+++ b/mlir/test/lib/Dialect/Math/TestExpandMath.cpp
@@ -52,6 +52,7 @@ void TestExpandMathPass::runOnOperation() {
   populateExpandFPowIPattern(patterns);
   populateExpandRoundFPattern(patterns);
   populateExpandRoundEvenPattern(patterns);
+  populateExpandRsqrtPattern(patterns);
   (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
 }
 
diff --git a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
index 2b72acde6a3b..9b929b3c864d 100644
--- a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
+++ b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
@@ -833,6 +833,46 @@ func.func @atanh() {
   return
 }
 
+// -------------------------------------------------------------------------- //
+// Rsqrt.
+// -------------------------------------------------------------------------- //
+
+func.func @rsqrt_f32(%a : f32) {
+  %r = math.rsqrt %a : f32
+  vector.print %r : f32
+  return
+}
+
+func.func @rsqrt_3xf32(%a : vector<3xf32>) {
+  %r = math.rsqrt %a : vector<3xf32>
+  vector.print %r : vector<3xf32>
+  return
+}
+
+func.func @rsqrt() {
+  // CHECK: 1
+  %zero = arith.constant 1.0 : f32
+  call @rsqrt_f32(%zero) : (f32) -> ()
+
+  // CHECK: 0.707107
+  %cst1 = arith.constant 2.0 : f32
+  call @rsqrt_f32(%cst1) : (f32) -> ()
+
+  // CHECK: inf
+  %cst2 = arith.constant 0.0 : f32
+  call @rsqrt_f32(%cst2) : (f32) -> ()
+
+  // CHECK: -nan
+  %cst3 = arith.constant -1.0 : f32
+  call @rsqrt_f32(%cst3) : (f32) -> ()
+
+  // CHECK: 0.5, 1.41421, 0.57735
+  %vec_x = arith.constant dense<[4.0, 0.5, 3.0]> : vector<3xf32>
+  call @rsqrt_3xf32(%vec_x) : (vector<3xf32>) -> ()
+
+  return
+}
+
 func.func @main() {
   call @exp2f() : () -> ()
   call @roundf() : () -> ()
@@ -844,5 +884,6 @@ func.func @main() {
   call @asinh() : () -> ()
   call @acosh() : () -> ()
   call @atanh() : () -> ()
+  call @rsqrt() : () -> ()
   return
 }
-- 
GitLab


From d4f5cf267936a082196b0c22fe45c730b24b9fe0 Mon Sep 17 00:00:00 2001
From: Azmat Yusuf <azmatyusuf2022@gmail.com>
Date: Mon, 13 May 2024 14:01:10 +0530
Subject: [PATCH 0530/1206] [Clang] Added check for unexpanded pack in
 attribute [[assume]] (#91893)

Added a check for unexpanded parameter pack in attribute [[assume]].
Tested it with expected-error statements from clang fronted.

This fixes #91232.
---
 clang/lib/Sema/SemaStmtAttr.cpp     | 5 +++++
 clang/test/SemaCXX/cxx23-assume.cpp | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp
index 1c84830b6ddd..36f8ecadcfab 100644
--- a/clang/lib/Sema/SemaStmtAttr.cpp
+++ b/clang/lib/Sema/SemaStmtAttr.cpp
@@ -670,6 +670,11 @@ ExprResult Sema::ActOnCXXAssumeAttr(Stmt *St, const ParsedAttr &A,
   }
 
   auto *Assumption = A.getArgAsExpr(0);
+
+  if (DiagnoseUnexpandedParameterPack(Assumption)) {
+    return ExprError();
+  }
+
   if (Assumption->getDependence() == ExprDependence::None) {
     ExprResult Res = BuildCXXAssumeExpr(Assumption, A.getAttrName(), Range);
     if (Res.isInvalid())
diff --git a/clang/test/SemaCXX/cxx23-assume.cpp b/clang/test/SemaCXX/cxx23-assume.cpp
index 8676970de14f..e67d72ae0a99 100644
--- a/clang/test/SemaCXX/cxx23-assume.cpp
+++ b/clang/test/SemaCXX/cxx23-assume.cpp
@@ -138,3 +138,8 @@ constexpr int foo() {
 }
 
 static_assert(foo() == 0);
+
+template <bool ...val>
+void f() {
+    [[assume(val)]]; // expected-error {{expression contains unexpanded parameter pack}}
+}
-- 
GitLab


From 220756f1f92b335cbafdff67c570d096a6925d87 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 13 May 2024 09:49:09 +0100
Subject: [PATCH 0531/1206] [AArch64][Inline] Regenerate
 Inline/AArch64/binop.ll test check lines. NFC

Should hopefully help with #91854
---
 llvm/test/Transforms/Inline/AArch64/binop.ll | 277 ++++++++++++++++---
 1 file changed, 233 insertions(+), 44 deletions(-)

diff --git a/llvm/test/Transforms/Inline/AArch64/binop.ll b/llvm/test/Transforms/Inline/AArch64/binop.ll
index eb882282820b..3dd66689a257 100644
--- a/llvm/test/Transforms/Inline/AArch64/binop.ll
+++ b/llvm/test/Transforms/Inline/AArch64/binop.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; RUN: opt -passes=inline -mtriple=aarch64--linux-gnu -S -o - < %s -inline-threshold=0 | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
@@ -7,20 +8,35 @@ declare void @pad()
 @glbl = external global i32
 
 define i32 @outer_add1(i32 %a) {
-; CHECK-LABEL: @outer_add1(
-; CHECK-NOT: call i32 @add
+; CHECK-LABEL: define i32 @outer_add1(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[A]]
+;
   %C = call i32 @add(i32 %a, i32 0)
   ret i32 %C
 }
 
 define i32 @outer_add2(i32 %a) {
-; CHECK-LABEL: @outer_add2(
-; CHECK-NOT: call i32 @add
+; CHECK-LABEL: define i32 @outer_add2(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[A]]
+;
   %C = call i32 @add(i32 0, i32 %a)
   ret i32 %C
 }
 
 define i32 @add(i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @add(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A]], [[B]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
   %add = add i32 %a, %b
   call void @pad()
   store i32 0, ptr @glbl
@@ -30,13 +46,24 @@ define i32 @add(i32 %a, i32 %b) {
 
 
 define i32 @outer_sub1(i32 %a) {
-; CHECK-LABEL: @outer_sub1(
-; CHECK-NOT: call i32 @sub1
+; CHECK-LABEL: define i32 @outer_sub1(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[A]]
+;
   %C = call i32 @sub1(i32 %a, i32 0)
   ret i32 %C
 }
 
 define i32 @sub1(i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @sub1(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[A]], [[B]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
   %sub = sub i32 %a, %b
   call void @pad()
   store i32 0, ptr @glbl
@@ -45,13 +72,22 @@ define i32 @sub1(i32 %a, i32 %b) {
 
 
 define i32 @outer_sub2(i32 %a) {
-; CHECK-LABEL: @outer_sub2(
-; CHECK-NOT: call i32 @sub2
+; CHECK-LABEL: define i32 @outer_sub2(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i32 0
+;
   %C = call i32 @sub2(i32 %a)
   ret i32 %C
 }
 
 define i32 @sub2(i32 %a) {
+; CHECK-LABEL: define i32 @sub2(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[A]], [[A]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
   %sub = sub i32 %a, %a
   call void @pad()
   ret i32 %sub
@@ -60,20 +96,35 @@ define i32 @sub2(i32 %a) {
 
 
 define i32 @outer_mul1(i32 %a) {
-; CHECK-LABEL: @outer_mul1(
-; CHECK-NOT: call i32 @mul
+; CHECK-LABEL: define i32 @outer_mul1(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 0
+;
   %C = call i32 @mul(i32 %a, i32 0)
   ret i32 %C
 }
 
 define i32 @outer_mul2(i32 %a) {
-; CHECK-LABEL: @outer_mul2(
-; CHECK-NOT: call i32 @mul
+; CHECK-LABEL: define i32 @outer_mul2(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[A]]
+;
   %C = call i32 @mul(i32 %a, i32 1)
   ret i32 %C
 }
 
 define i32 @mul(i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @mul(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[A]], [[B]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
   %mul = mul i32 %a, %b
   call void @pad()
   store i32 0, ptr @glbl
@@ -83,20 +134,35 @@ define i32 @mul(i32 %a, i32 %b) {
 
 
 define i32 @outer_div1(i32 %a) {
-; CHECK-LABEL: @outer_div1(
-; CHECK-NOT: call i32 @div1
+; CHECK-LABEL: define i32 @outer_div1(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 0
+;
   %C = call i32 @div1(i32 0, i32 %a)
   ret i32 %C
 }
 
 define i32 @outer_div2(i32 %a) {
-; CHECK-LABEL: @outer_div2(
-; CHECK-NOT: call i32 @div1
+; CHECK-LABEL: define i32 @outer_div2(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[A]]
+;
   %C = call i32 @div1(i32 %a, i32 1)
   ret i32 %C
 }
 
 define i32 @div1(i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @div1(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[A]], [[B]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
   %div = sdiv i32 %a, %b
   call void @pad()
   store i32 0, ptr @glbl
@@ -105,13 +171,22 @@ define i32 @div1(i32 %a, i32 %b) {
 
 
 define i32 @outer_div3(i32 %a) {
-; CHECK-LABEL: @outer_div3(
-; CHECK-NOT: call i32 @div
+; CHECK-LABEL: define i32 @outer_div3(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i32 1
+;
   %C = call i32 @div2(i32 %a)
   ret i32 %C
 }
 
 define i32 @div2(i32 %a) {
+; CHECK-LABEL: define i32 @div2(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[A]], [[A]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
   %div = sdiv i32 %a, %a
   call void @pad()
   ret i32 %div
@@ -120,20 +195,35 @@ define i32 @div2(i32 %a) {
 
 
 define i32 @outer_rem1(i32 %a) {
-; CHECK-LABEL: @outer_rem1(
-; CHECK-NOT: call i32 @rem
+; CHECK-LABEL: define i32 @outer_rem1(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 0
+;
   %C = call i32 @rem1(i32 0, i32 %a)
   ret i32 %C
 }
 
 define i32 @outer_rem2(i32 %a) {
-; CHECK-LABEL: @outer_rem2(
-; CHECK-NOT: call i32 @rem
+; CHECK-LABEL: define i32 @outer_rem2(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 0
+;
   %C = call i32 @rem1(i32 %a, i32 1)
   ret i32 %C
 }
 
 define i32 @rem1(i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @rem1(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[A]], [[B]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[REM]]
+;
   %rem = urem i32 %a, %b
   call void @pad()
   store i32 0, ptr @glbl
@@ -142,13 +232,22 @@ define i32 @rem1(i32 %a, i32 %b) {
 
 
 define i32 @outer_rem3(i32 %a) {
-; CHECK-LABEL: @outer_rem3(
-; CHECK-NOT: call i32 @rem
+; CHECK-LABEL: define i32 @outer_rem3(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i32 0
+;
   %C = call i32 @rem2(i32 %a)
   ret i32 %C
 }
 
 define i32 @rem2(i32 %a) {
+; CHECK-LABEL: define i32 @rem2(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[A]], [[A]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i32 [[REM]]
+;
   %rem = urem i32 %a, %a
   call void @pad()
   ret i32 %rem
@@ -157,13 +256,24 @@ define i32 @rem2(i32 %a) {
 
 
 define i32 @outer_shl1(i32 %a) {
-; CHECK-LABEL: @outer_shl1(
-; CHECK-NOT: call i32 @shl
+; CHECK-LABEL: define i32 @outer_shl1(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[A]]
+;
   %C = call i32 @shl(i32 %a, i32 0)
   ret i32 %C
 }
 
 define i32 @shl(i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @shl(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[A]], [[B]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[SHL]]
+;
   %shl = shl i32 %a, %b
   call void @pad()
   store i32 0, ptr @glbl
@@ -173,13 +283,24 @@ define i32 @shl(i32 %a, i32 %b) {
 
 
 define i32 @outer_shr1(i32 %a) {
-; CHECK-LABEL: @outer_shr1(
-; CHECK-NOT: call i32 @shr
+; CHECK-LABEL: define i32 @outer_shr1(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[A]]
+;
   %C = call i32 @shr(i32 %a, i32 0)
   ret i32 %C
 }
 
 define i32 @shr(i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @shr(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[A]], [[B]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
   %shr = ashr i32 %a, %b
   call void @pad()
   store i32 0, ptr @glbl
@@ -189,20 +310,35 @@ define i32 @shr(i32 %a, i32 %b) {
 
 
 define i1 @outer_and1(i1 %a) {
-; check-label: @outer_and1(
-; check-not: call i1 @and1
+; CHECK-LABEL: define i1 @outer_and1(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i1 false
+;
   %c = call i1 @and1(i1 %a, i1 false)
   ret i1 %c
 }
 
 define i1 @outer_and2(i1 %a) {
-; check-label: @outer_and2(
-; check-not: call i1 @and1
+; CHECK-LABEL: define i1 @outer_and2(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i1 [[A]]
+;
   %c = call i1 @and1(i1 %a, i1 true)
   ret i1 %c
 }
 
 define i1 @and1(i1 %a, i1 %b) {
+; CHECK-LABEL: define i1 @and1(
+; CHECK-SAME: i1 [[A:%.*]], i1 [[B:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i1 [[AND]]
+;
   %and = and i1 %a, %b
   call void @pad()
   store i32 0, ptr @glbl
@@ -211,13 +347,22 @@ define i1 @and1(i1 %a, i1 %b) {
 
 
 define i1 @outer_and3(i1 %a) {
-; check-label: @outer_and3(
-; check-not: call i1 @and2
+; CHECK-LABEL: define i1 @outer_and3(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i1 [[A]]
+;
   %c = call i1 @and2(i1 %a)
   ret i1 %c
 }
 
 define i1 @and2(i1 %a) {
+; CHECK-LABEL: define i1 @and2(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[A]], [[A]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i1 [[AND]]
+;
   %and = and i1 %a, %a
   call void @pad()
   ret i1 %and
@@ -226,20 +371,35 @@ define i1 @and2(i1 %a) {
 
 
 define i1 @outer_or1(i1 %a) {
-; check-label: @outer_or1(
-; check-not: call i1 @or1
+; CHECK-LABEL: define i1 @outer_or1(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i1 [[A]]
+;
   %c = call i1 @or1(i1 %a, i1 false)
   ret i1 %c
 }
 
 define i1 @outer_or2(i1 %a) {
-; check-label: @outer_or2(
-; check-not: call i1 @or1
+; CHECK-LABEL: define i1 @outer_or2(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i1 true
+;
   %c = call i1 @or1(i1 %a, i1 true)
   ret i1 %c
 }
 
 define i1 @or1(i1 %a, i1 %b) {
+; CHECK-LABEL: define i1 @or1(
+; CHECK-SAME: i1 [[A:%.*]], i1 [[B:%.*]]) {
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i1 [[OR]]
+;
   %or = or i1 %a, %b
   call void @pad()
   store i32 0, ptr @glbl
@@ -248,13 +408,22 @@ define i1 @or1(i1 %a, i1 %b) {
 
 
 define i1 @outer_or3(i1 %a) {
-; check-label: @outer_or3(
-; check-not: call i1 @or2
+; CHECK-LABEL: define i1 @outer_or3(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i1 [[A]]
+;
   %c = call i1 @or2(i1 %a)
   ret i1 %c
 }
 
 define i1 @or2(i1 %a) {
+; CHECK-LABEL: define i1 @or2(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[A]], [[A]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i1 [[OR]]
+;
   %or = or i1 %a, %a
   call void @pad()
   ret i1 %or
@@ -263,13 +432,24 @@ define i1 @or2(i1 %a) {
 
 
 define i1 @outer_xor1(i1 %a) {
-; check-label: @outer_xor1(
-; check-not: call i1 @xor
+; CHECK-LABEL: define i1 @outer_xor1(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i1 [[A]]
+;
   %c = call i1 @xor1(i1 %a, i1 false)
   ret i1 %c
 }
 
 define i1 @xor1(i1 %a, i1 %b) {
+; CHECK-LABEL: define i1 @xor1(
+; CHECK-SAME: i1 [[A:%.*]], i1 [[B:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[A]], [[B]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    store i32 0, ptr @glbl, align 4
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
   %xor = xor i1 %a, %b
   call void @pad()
   store i32 0, ptr @glbl
@@ -278,13 +458,22 @@ define i1 @xor1(i1 %a, i1 %b) {
 
 
 define i1 @outer_xor3(i1 %a) {
-; check-label: @outer_xor3(
-; check-not: call i1 @xor
+; CHECK-LABEL: define i1 @outer_xor3(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i1 false
+;
   %c = call i1 @xor2(i1 %a)
   ret i1 %c
 }
 
 define i1 @xor2(i1 %a) {
+; CHECK-LABEL: define i1 @xor2(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[A]], [[A]]
+; CHECK-NEXT:    call void @pad()
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
   %xor = xor i1 %a, %a
   call void @pad()
   ret i1 %xor
-- 
GitLab


From 0fb7546c587198df11714cfc433d4c5552af0888 Mon Sep 17 00:00:00 2001
From: Tuan Chuong Goh <chuong.goh@arm.com>
Date: Mon, 13 May 2024 08:21:36 +0000
Subject: [PATCH 0532/1206] [AArch64][NFC] Pre-commit Test for Select G_ICMP
 instruction through TableGen (#89932)

---
 llvm/test/CodeGen/AArch64/icmp.ll | 1056 +++++++++++++++++++++++++++++
 1 file changed, 1056 insertions(+)

diff --git a/llvm/test/CodeGen/AArch64/icmp.ll b/llvm/test/CodeGen/AArch64/icmp.ll
index 8e10847e7aae..2292bc6d2038 100644
--- a/llvm/test/CodeGen/AArch64/icmp.ll
+++ b/llvm/test/CodeGen/AArch64/icmp.ll
@@ -52,6 +52,1062 @@ entry:
   ret i8 %s
 }
 
+define <2 x i1> @test_v2i64_eq(<2 x i64> %v1, <2 x i64> %v2) {
+; CHECK-LABEL: test_v2i64_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+  %cmp = icmp eq <2 x i64> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i64_eq(<4 x i64> %v1, <4 x i64> %v2) {
+; CHECK-SD-LABEL: test_v4i64_eq:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmeq v1.2d, v1.2d, v3.2d
+; CHECK-SD-NEXT:    cmeq v0.2d, v0.2d, v2.2d
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v4i64_eq:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmeq v0.2d, v0.2d, v2.2d
+; CHECK-GI-NEXT:    cmeq v1.2d, v1.2d, v3.2d
+; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %cmp = icmp eq <4 x i64> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i32_eq(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: test_v4i32_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %cmp = icmp eq <4 x i32> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i32_eq(<2 x i32> %v1, <2 x i32> %v2) {
+; CHECK-LABEL: test_v2i32_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp eq <2 x i32> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i16_eq(<2 x i16> %v1, <2 x i16> %v2) {
+; CHECK-SD-LABEL: test_v2i16_eq:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-SD-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v2i16_eq:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
+  %cmp = icmp eq <2 x i16> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i16_eq(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: test_v8i16_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %cmp = icmp eq <8 x i16> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i16_eq(<4 x i16> %v1, <4 x i16> %v2) {
+; CHECK-LABEL: test_v4i16_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+  %cmp = icmp eq <4 x i16> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <16 x i1> @test_v16i8_eq(<16 x i8> %v1, <16 x i8> %v2) {
+; CHECK-LABEL: test_v16i8_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %cmp = icmp eq <16 x i8> %v1, %v2
+  ret <16 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i8_eq(<8 x i8> %v1, <8 x i8> %v2) {
+; CHECK-LABEL: test_v8i8_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp eq <8 x i8> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i64_ne(<2 x i64> %v1, <2 x i64> %v2) {
+; CHECK-LABEL: test_v2i64_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+  %cmp = icmp ne <2 x i64> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i64_ne(<4 x i64> %v1, <4 x i64> %v2) {
+; CHECK-SD-LABEL: test_v4i64_ne:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmeq v1.2d, v1.2d, v3.2d
+; CHECK-SD-NEXT:    cmeq v0.2d, v0.2d, v2.2d
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    mvn v0.16b, v0.16b
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v4i64_ne:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmeq v0.2d, v0.2d, v2.2d
+; CHECK-GI-NEXT:    cmeq v1.2d, v1.2d, v3.2d
+; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
+; CHECK-GI-NEXT:    mvn v1.16b, v1.16b
+; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %cmp = icmp ne <4 x i64> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i32_ne(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: test_v4i32_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %cmp = icmp ne <4 x i32> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i32_ne(<2 x i32> %v1, <2 x i32> %v2) {
+; CHECK-LABEL: test_v2i32_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp ne <2 x i32> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i16_ne(<2 x i16> %v1, <2 x i16> %v2) {
+; CHECK-SD-LABEL: test_v2i16_ne:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-SD-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    mvn v0.8b, v0.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v2i16_ne:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    mvn v0.8b, v0.8b
+; CHECK-GI-NEXT:    ret
+  %cmp = icmp ne <2 x i16> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i16_ne(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: test_v8i16_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %cmp = icmp ne <8 x i16> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i16_ne(<4 x i16> %v1, <4 x i16> %v2) {
+; CHECK-LABEL: test_v4i16_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp ne <4 x i16> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <16 x i1> @test_v16i8_ne(<16 x i8> %v1, <16 x i8> %v2) {
+; CHECK-LABEL: test_v16i8_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    mvn v0.16b, v0.16b
+; CHECK-NEXT:    ret
+  %cmp = icmp ne <16 x i8> %v1, %v2
+  ret <16 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i8_ne(<8 x i8> %v1, <8 x i8> %v2) {
+; CHECK-LABEL: test_v8i8_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp ne <8 x i8> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i64_ugt(<2 x i64> %v1, <2 x i64> %v2) {
+; CHECK-LABEL: test_v2i64_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+  %cmp = icmp ugt <2 x i64> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i64_ugt(<4 x i64> %v1, <4 x i64> %v2) {
+; CHECK-SD-LABEL: test_v4i64_ugt:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmhi v1.2d, v1.2d, v3.2d
+; CHECK-SD-NEXT:    cmhi v0.2d, v0.2d, v2.2d
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v4i64_ugt:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmhi v0.2d, v0.2d, v2.2d
+; CHECK-GI-NEXT:    cmhi v1.2d, v1.2d, v3.2d
+; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %cmp = icmp ugt <4 x i64> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i32_ugt(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: test_v4i32_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %cmp = icmp ugt <4 x i32> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i32_ugt(<2 x i32> %v1, <2 x i32> %v2) {
+; CHECK-LABEL: test_v2i32_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp ugt <2 x i32> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i16_ugt(<2 x i16> %v1, <2 x i16> %v2) {
+; CHECK-SD-LABEL: test_v2i16_ugt:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-SD-NEXT:    cmhi v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v2i16_ugt:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    cmhi v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
+  %cmp = icmp ugt <2 x i16> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i16_ugt(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: test_v8i16_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %cmp = icmp ugt <8 x i16> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i16_ugt(<4 x i16> %v1, <4 x i16> %v2) {
+; CHECK-LABEL: test_v4i16_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+  %cmp = icmp ugt <4 x i16> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <16 x i1> @test_v16i8_ugt(<16 x i8> %v1, <16 x i8> %v2) {
+; CHECK-LABEL: test_v16i8_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %cmp = icmp ugt <16 x i8> %v1, %v2
+  ret <16 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i8_ugt(<8 x i8> %v1, <8 x i8> %v2) {
+; CHECK-LABEL: test_v8i8_ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp ugt <8 x i8> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i64_uge(<2 x i64> %v1, <2 x i64> %v2) {
+; CHECK-LABEL: test_v2i64_uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+  %cmp = icmp uge <2 x i64> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i64_uge(<4 x i64> %v1, <4 x i64> %v2) {
+; CHECK-SD-LABEL: test_v4i64_uge:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmhs v1.2d, v1.2d, v3.2d
+; CHECK-SD-NEXT:    cmhs v0.2d, v0.2d, v2.2d
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v4i64_uge:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmhs v0.2d, v0.2d, v2.2d
+; CHECK-GI-NEXT:    cmhs v1.2d, v1.2d, v3.2d
+; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %cmp = icmp uge <4 x i64> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i32_uge(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: test_v4i32_uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %cmp = icmp uge <4 x i32> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i32_uge(<2 x i32> %v1, <2 x i32> %v2) {
+; CHECK-LABEL: test_v2i32_uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp uge <2 x i32> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i16_uge(<2 x i16> %v1, <2 x i16> %v2) {
+; CHECK-SD-LABEL: test_v2i16_uge:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-SD-NEXT:    cmhs v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v2i16_uge:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    cmhs v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
+  %cmp = icmp uge <2 x i16> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i16_uge(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: test_v8i16_uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %cmp = icmp uge <8 x i16> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i16_uge(<4 x i16> %v1, <4 x i16> %v2) {
+; CHECK-LABEL: test_v4i16_uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+  %cmp = icmp uge <4 x i16> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <16 x i1> @test_v16i8_uge(<16 x i8> %v1, <16 x i8> %v2) {
+; CHECK-LABEL: test_v16i8_uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %cmp = icmp uge <16 x i8> %v1, %v2
+  ret <16 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i8_uge(<8 x i8> %v1, <8 x i8> %v2) {
+; CHECK-LABEL: test_v8i8_uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp uge <8 x i8> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i64_ult(<2 x i64> %v1, <2 x i64> %v2) {
+; CHECK-LABEL: test_v2i64_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.2d, v1.2d, v0.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+  %cmp = icmp ult <2 x i64> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i64_ult(<4 x i64> %v1, <4 x i64> %v2) {
+; CHECK-SD-LABEL: test_v4i64_ult:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmhi v1.2d, v3.2d, v1.2d
+; CHECK-SD-NEXT:    cmhi v0.2d, v2.2d, v0.2d
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v4i64_ult:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmhi v0.2d, v2.2d, v0.2d
+; CHECK-GI-NEXT:    cmhi v1.2d, v3.2d, v1.2d
+; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %cmp = icmp ult <4 x i64> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i32_ult(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: test_v4i32_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %cmp = icmp ult <4 x i32> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i32_ult(<2 x i32> %v1, <2 x i32> %v2) {
+; CHECK-LABEL: test_v2i32_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp ult <2 x i32> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i16_ult(<2 x i16> %v1, <2 x i16> %v2) {
+; CHECK-LABEL: test_v2i16_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT:    cmhi v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp ult <2 x i16> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i16_ult(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: test_v8i16_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %cmp = icmp ult <8 x i16> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i16_ult(<4 x i16> %v1, <4 x i16> %v2) {
+; CHECK-LABEL: test_v4i16_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+  %cmp = icmp ult <4 x i16> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <16 x i1> @test_v16i8_ult(<16 x i8> %v1, <16 x i8> %v2) {
+; CHECK-LABEL: test_v16i8_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    ret
+  %cmp = icmp ult <16 x i8> %v1, %v2
+  ret <16 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i8_ult(<8 x i8> %v1, <8 x i8> %v2) {
+; CHECK-LABEL: test_v8i8_ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhi v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp ult <8 x i8> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i64_ule(<2 x i64> %v1, <2 x i64> %v2) {
+; CHECK-LABEL: test_v2i64_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.2d, v1.2d, v0.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+  %cmp = icmp ule <2 x i64> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i64_ule(<4 x i64> %v1, <4 x i64> %v2) {
+; CHECK-SD-LABEL: test_v4i64_ule:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmhs v1.2d, v3.2d, v1.2d
+; CHECK-SD-NEXT:    cmhs v0.2d, v2.2d, v0.2d
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v4i64_ule:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmhs v0.2d, v2.2d, v0.2d
+; CHECK-GI-NEXT:    cmhs v1.2d, v3.2d, v1.2d
+; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %cmp = icmp ule <4 x i64> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i32_ule(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: test_v4i32_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %cmp = icmp ule <4 x i32> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i32_ule(<2 x i32> %v1, <2 x i32> %v2) {
+; CHECK-LABEL: test_v2i32_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp ule <2 x i32> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i16_ule(<2 x i16> %v1, <2 x i16> %v2) {
+; CHECK-LABEL: test_v2i16_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT:    cmhs v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp ule <2 x i16> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i16_ule(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: test_v8i16_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %cmp = icmp ule <8 x i16> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i16_ule(<4 x i16> %v1, <4 x i16> %v2) {
+; CHECK-LABEL: test_v4i16_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+  %cmp = icmp ule <4 x i16> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <16 x i1> @test_v16i8_ule(<16 x i8> %v1, <16 x i8> %v2) {
+; CHECK-LABEL: test_v16i8_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    ret
+  %cmp = icmp ule <16 x i8> %v1, %v2
+  ret <16 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i8_ule(<8 x i8> %v1, <8 x i8> %v2) {
+; CHECK-LABEL: test_v8i8_ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmhs v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp ule <8 x i8> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i64_sgt(<2 x i64> %v1, <2 x i64> %v2) {
+; CHECK-LABEL: test_v2i64_sgt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+  %cmp = icmp sgt <2 x i64> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i64_sgt(<4 x i64> %v1, <4 x i64> %v2) {
+; CHECK-SD-LABEL: test_v4i64_sgt:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmgt v1.2d, v1.2d, v3.2d
+; CHECK-SD-NEXT:    cmgt v0.2d, v0.2d, v2.2d
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v4i64_sgt:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmgt v0.2d, v0.2d, v2.2d
+; CHECK-GI-NEXT:    cmgt v1.2d, v1.2d, v3.2d
+; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %cmp = icmp sgt <4 x i64> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i32_sgt(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: test_v4i32_sgt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %cmp = icmp sgt <4 x i32> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i32_sgt(<2 x i32> %v1, <2 x i32> %v2) {
+; CHECK-LABEL: test_v2i32_sgt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp sgt <2 x i32> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i16_sgt(<2 x i16> %v1, <2 x i16> %v2) {
+; CHECK-SD-LABEL: test_v2i16_sgt:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    shl v1.2s, v1.2s, #16
+; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #16
+; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-SD-NEXT:    cmgt v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v2i16_sgt:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    shl v1.2s, v1.2s, #16
+; CHECK-GI-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    sshr v1.2s, v1.2s, #16
+; CHECK-GI-NEXT:    cmgt v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
+  %cmp = icmp sgt <2 x i16> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i16_sgt(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: test_v8i16_sgt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %cmp = icmp sgt <8 x i16> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i16_sgt(<4 x i16> %v1, <4 x i16> %v2) {
+; CHECK-LABEL: test_v4i16_sgt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+  %cmp = icmp sgt <4 x i16> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <16 x i1> @test_v16i8_sgt(<16 x i8> %v1, <16 x i8> %v2) {
+; CHECK-LABEL: test_v16i8_sgt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %cmp = icmp sgt <16 x i8> %v1, %v2
+  ret <16 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i8_sgt(<8 x i8> %v1, <8 x i8> %v2) {
+; CHECK-LABEL: test_v8i8_sgt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp sgt <8 x i8> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i64_sge(<2 x i64> %v1, <2 x i64> %v2) {
+; CHECK-LABEL: test_v2i64_sge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+  %cmp = icmp sge <2 x i64> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i64_sge(<4 x i64> %v1, <4 x i64> %v2) {
+; CHECK-SD-LABEL: test_v4i64_sge:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmge v1.2d, v1.2d, v3.2d
+; CHECK-SD-NEXT:    cmge v0.2d, v0.2d, v2.2d
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v4i64_sge:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmge v0.2d, v0.2d, v2.2d
+; CHECK-GI-NEXT:    cmge v1.2d, v1.2d, v3.2d
+; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %cmp = icmp sge <4 x i64> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i32_sge(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: test_v4i32_sge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %cmp = icmp sge <4 x i32> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i32_sge(<2 x i32> %v1, <2 x i32> %v2) {
+; CHECK-LABEL: test_v2i32_sge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp sge <2 x i32> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i16_sge(<2 x i16> %v1, <2 x i16> %v2) {
+; CHECK-SD-LABEL: test_v2i16_sge:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    shl v1.2s, v1.2s, #16
+; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #16
+; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-SD-NEXT:    cmge v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v2i16_sge:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    shl v1.2s, v1.2s, #16
+; CHECK-GI-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    sshr v1.2s, v1.2s, #16
+; CHECK-GI-NEXT:    cmge v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
+  %cmp = icmp sge <2 x i16> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i16_sge(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: test_v8i16_sge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %cmp = icmp sge <8 x i16> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i16_sge(<4 x i16> %v1, <4 x i16> %v2) {
+; CHECK-LABEL: test_v4i16_sge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+  %cmp = icmp sge <4 x i16> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <16 x i1> @test_v16i8_sge(<16 x i8> %v1, <16 x i8> %v2) {
+; CHECK-LABEL: test_v16i8_sge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %cmp = icmp sge <16 x i8> %v1, %v2
+  ret <16 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i8_sge(<8 x i8> %v1, <8 x i8> %v2) {
+; CHECK-LABEL: test_v8i8_sge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp sge <8 x i8> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i64_slt(<2 x i64> %v1, <2 x i64> %v2) {
+; CHECK-LABEL: test_v2i64_slt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.2d, v1.2d, v0.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <2 x i64> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i64_slt(<4 x i64> %v1, <4 x i64> %v2) {
+; CHECK-SD-LABEL: test_v4i64_slt:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmgt v1.2d, v3.2d, v1.2d
+; CHECK-SD-NEXT:    cmgt v0.2d, v2.2d, v0.2d
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v4i64_slt:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmgt v0.2d, v2.2d, v0.2d
+; CHECK-GI-NEXT:    cmgt v1.2d, v3.2d, v1.2d
+; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %cmp = icmp slt <4 x i64> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i32_slt(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: test_v4i32_slt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <4 x i32> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i32_slt(<2 x i32> %v1, <2 x i32> %v2) {
+; CHECK-LABEL: test_v2i32_slt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <2 x i32> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i16_slt(<2 x i16> %v1, <2 x i16> %v2) {
+; CHECK-LABEL: test_v2i16_slt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-NEXT:    shl v1.2s, v1.2s, #16
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-NEXT:    sshr v1.2s, v1.2s, #16
+; CHECK-NEXT:    cmgt v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <2 x i16> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i16_slt(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: test_v8i16_slt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <8 x i16> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i16_slt(<4 x i16> %v1, <4 x i16> %v2) {
+; CHECK-LABEL: test_v4i16_slt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <4 x i16> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <16 x i1> @test_v16i8_slt(<16 x i8> %v1, <16 x i8> %v2) {
+; CHECK-LABEL: test_v16i8_slt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <16 x i8> %v1, %v2
+  ret <16 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i8_slt(<8 x i8> %v1, <8 x i8> %v2) {
+; CHECK-LABEL: test_v8i8_slt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp slt <8 x i8> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i64_sle(<2 x i64> %v1, <2 x i64> %v2) {
+; CHECK-LABEL: test_v2i64_sle:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2d, v1.2d, v0.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+  %cmp = icmp sle <2 x i64> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i64_sle(<4 x i64> %v1, <4 x i64> %v2) {
+; CHECK-SD-LABEL: test_v4i64_sle:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmge v1.2d, v3.2d, v1.2d
+; CHECK-SD-NEXT:    cmge v0.2d, v2.2d, v0.2d
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v4i64_sle:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmge v0.2d, v2.2d, v0.2d
+; CHECK-GI-NEXT:    cmge v1.2d, v3.2d, v1.2d
+; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %cmp = icmp sle <4 x i64> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i32_sle(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: test_v4i32_sle:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %cmp = icmp sle <4 x i32> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i32_sle(<2 x i32> %v1, <2 x i32> %v2) {
+; CHECK-LABEL: test_v2i32_sle:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp sle <2 x i32> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test_v2i16_sle(<2 x i16> %v1, <2 x i16> %v2) {
+; CHECK-LABEL: test_v2i16_sle:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-NEXT:    shl v1.2s, v1.2s, #16
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-NEXT:    sshr v1.2s, v1.2s, #16
+; CHECK-NEXT:    cmge v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+  %cmp = icmp sle <2 x i16> %v1, %v2
+  ret <2 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i16_sle(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: test_v8i16_sle:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+  %cmp = icmp sle <8 x i16> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
+define <4 x i1> @test_v4i16_sle(<4 x i16> %v1, <4 x i16> %v2) {
+; CHECK-LABEL: test_v4i16_sle:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+  %cmp = icmp sle <4 x i16> %v1, %v2
+  ret <4 x i1> %cmp
+}
+
+define <16 x i1> @test_v16i8_sle(<16 x i8> %v1, <16 x i8> %v2) {
+; CHECK-LABEL: test_v16i8_sle:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    ret
+  %cmp = icmp sle <16 x i8> %v1, %v2
+  ret <16 x i1> %cmp
+}
+
+define <8 x i1> @test_v8i8_sle(<8 x i8> %v1, <8 x i8> %v2) {
+; CHECK-LABEL: test_v8i8_sle:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    ret
+  %cmp = icmp sle <8 x i8> %v1, %v2
+  ret <8 x i1> %cmp
+}
+
 define <2 x i64> @v2i64_i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %d, <2 x i64> %e) {
 ; CHECK-LABEL: v2i64_i64:
 ; CHECK:       // %bb.0: // %entry
-- 
GitLab


From c6c7afd21edd0d16ebda916ea4939949e4e0fa8e Mon Sep 17 00:00:00 2001
From: Matthias Gehre <matthias.gehre@amd.com>
Date: Mon, 13 May 2024 11:02:09 +0200
Subject: [PATCH 0533/1206] [mlir][math] fix rsqrt test to not check sign of
 NaN

Hotfix for "[mlir][math] lower rsqrt to sqrt + fdiv (#91344)"
---
 mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
index 9b929b3c864d..80d559cc6f73 100644
--- a/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
+++ b/mlir/test/mlir-cpu-runner/test-expand-math-approx.mlir
@@ -862,7 +862,7 @@ func.func @rsqrt() {
   %cst2 = arith.constant 0.0 : f32
   call @rsqrt_f32(%cst2) : (f32) -> ()
 
-  // CHECK: -nan
+  // CHECK: nan
   %cst3 = arith.constant -1.0 : f32
   call @rsqrt_f32(%cst3) : (f32) -> ()
 
-- 
GitLab


From 61d4ca872215d3dfff0b3c92151dcbdc546a0aab Mon Sep 17 00:00:00 2001
From: Daniel Grumberg <dgrumberg@apple.com>
Date: Mon, 13 May 2024 10:37:09 +0100
Subject: [PATCH 0534/1206] [clang][ExtractAPI] Distinguish between record kind
 for display and for RTTI (#91466)

rdar://127732562
---
 clang/include/clang/ExtractAPI/API.h          |  7 ++++--
 .../clang/ExtractAPI/ExtractAPIVisitor.h      | 25 +++++++++++--------
 .../Serialization/SymbolGraphSerializer.cpp   |  6 ++---
 3 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/clang/include/clang/ExtractAPI/API.h b/clang/include/clang/ExtractAPI/API.h
index d323e1668a72..bf291074fd06 100644
--- a/clang/include/clang/ExtractAPI/API.h
+++ b/clang/include/clang/ExtractAPI/API.h
@@ -266,6 +266,8 @@ struct APIRecord {
 
   AccessControl Access;
 
+  RecordKind KindForDisplay;
+
 private:
   const RecordKind Kind;
   friend class RecordContext;
@@ -277,6 +279,7 @@ public:
   APIRecord *getNextInContext() const { return NextInContext; }
 
   RecordKind getKind() const { return Kind; }
+  RecordKind getKindForDisplay() const { return KindForDisplay; }
 
   static APIRecord *castFromRecordContext(const RecordContext *Ctx);
   static RecordContext *castToRecordContext(const APIRecord *Record);
@@ -293,10 +296,10 @@ public:
         Availability(std::move(Availability)), Linkage(Linkage),
         Comment(Comment), Declaration(Declaration), SubHeading(SubHeading),
         IsFromSystemHeader(IsFromSystemHeader), Access(std::move(Access)),
-        Kind(Kind) {}
+        KindForDisplay(Kind), Kind(Kind) {}
 
   APIRecord(RecordKind Kind, StringRef USR, StringRef Name)
-      : USR(USR), Name(Name), Kind(Kind) {}
+      : USR(USR), Name(Name), KindForDisplay(Kind), Kind(Kind) {}
 
   // Pure virtual destructor to make APIRecord abstract
   virtual ~APIRecord() = 0;
diff --git a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
index 97cc457ea2a9..8ccebe457ed5 100644
--- a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
+++ b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
@@ -194,6 +194,15 @@ protected:
     return Bases;
   }
 
+  APIRecord::RecordKind getKindForDisplay(const CXXRecordDecl *Decl) {
+    if (Decl->isUnion())
+      return APIRecord::RK_Union;
+    if (Decl->isStruct())
+      return APIRecord::RK_Struct;
+
+    return APIRecord::RK_CXXClass;
+  }
+
   StringRef getOwningModuleName(const Decl &D) {
     if (auto *OwningModule = D.getImportedOwningModule())
       return OwningModule->Name;
@@ -599,13 +608,6 @@ bool ExtractAPIVisitorBase<Derived>::VisitCXXRecordDecl(
   DeclarationFragments SubHeading =
       DeclarationFragmentsBuilder::getSubHeading(Decl);
 
-  APIRecord::RecordKind Kind;
-  if (Decl->isUnion())
-    Kind = APIRecord::RecordKind::RK_Union;
-  else if (Decl->isStruct())
-    Kind = APIRecord::RecordKind::RK_Struct;
-  else
-    Kind = APIRecord::RecordKind::RK_CXXClass;
   auto Access = DeclarationFragmentsBuilder::getAccessControl(Decl);
 
   CXXClassRecord *Record;
@@ -619,13 +621,15 @@ bool ExtractAPIVisitorBase<Derived>::VisitCXXRecordDecl(
         AvailabilityInfo::createFromDecl(Decl), Comment, Declaration,
         SubHeading, Template(Decl->getDescribedClassTemplate()), Access,
         isInSystemHeader(Decl));
-  } else
+  } else {
     Record = API.createRecord<CXXClassRecord>(
         USR, Name, createHierarchyInformationForDecl(*Decl), Loc,
         AvailabilityInfo::createFromDecl(Decl), Comment, Declaration,
-        SubHeading, Kind, Access, isInSystemHeader(Decl),
-        isEmbeddedInVarDeclarator(*Decl));
+        SubHeading, APIRecord::RecordKind::RK_CXXClass, Access,
+        isInSystemHeader(Decl), isEmbeddedInVarDeclarator(*Decl));
+  }
 
+  Record->KindForDisplay = getKindForDisplay(Decl);
   Record->Bases = getBases(Decl);
 
   return true;
@@ -849,6 +853,7 @@ bool ExtractAPIVisitorBase<Derived>::
       Template(Decl), DeclarationFragmentsBuilder::getAccessControl(Decl),
       isInSystemHeader(Decl));
 
+  CTPSR->KindForDisplay = getKindForDisplay(Decl);
   CTPSR->Bases = getBases(Decl);
 
   return true;
diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index 34278b5d40c4..c16d4623f115 100644
--- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -514,7 +514,7 @@ Object serializeSymbolKind(APIRecord::RecordKind RK, Language Lang) {
 /// which is prefixed by the source language name, useful for tooling to parse
 /// the kind, and a \c displayName for rendering human-readable names.
 Object serializeSymbolKind(const APIRecord &Record, Language Lang) {
-  return serializeSymbolKind(Record.getKind(), Lang);
+  return serializeSymbolKind(Record.KindForDisplay, Lang);
 }
 
 /// Serialize the function signature field, as specified by the
@@ -591,8 +591,8 @@ Array generateParentContexts(const SmallVectorImpl<SymbolReference> &Parents,
     Elem["usr"] = Parent.USR;
     Elem["name"] = Parent.Name;
     if (Parent.Record)
-      Elem["kind"] =
-          serializeSymbolKind(Parent.Record->getKind(), Lang)["identifier"];
+      Elem["kind"] = serializeSymbolKind(Parent.Record->KindForDisplay,
+                                         Lang)["identifier"];
     else
       Elem["kind"] =
           serializeSymbolKind(APIRecord::RK_Unknown, Lang)["identifier"];
-- 
GitLab


From 119aecb955df91173d69c455bba0abd74271c215 Mon Sep 17 00:00:00 2001
From: Victor Campos <victor.campos@arm.com>
Date: Mon, 13 May 2024 11:14:35 +0100
Subject: [PATCH 0535/1206] [DebugInfo] Emit negative DW_AT_bit_offset in
 explicit signed form (#87994)

Before this patch, the value of DW_AT_bit_offset, used for bitfields
before DWARF version 4, was always emitted as an unsigned integer using
the form DW_FORM_data<n>. If the value was originally a signed integer,
for instance in the case of negative offsets, it was up to debug
information consumers to re-cast it to a signed integer.

This is problematic since the burden of deciding if the value should be
read as signed or unsigned was put onto the debug info consumers: the
DWARF specification doesn't define DW_AT_bit_offset's underlying type.
If a debugger decided to interpret this attribute in the form data<n> as
unsigned, then negative offsets would be completely broken.

The DWARF specification version 3 mentions in the Data Representation
section, page 127:

> If one of the DW_FORM_data<n> forms is used to represent a signed or
unsigned integer, it can be hard for a consumer to discover the context
necessary to determine which interpretation is intended. Producers are
therefore strongly encouraged to use DW_FORM_sdata or DW_FORM_udata for
signed and unsigned integers respectively, rather than DW_FORM_data<n>.

Therefore, the proposal is to use DW_FORM_sdata, which is explicitly
signed. This is an indication to consumers that the offset must be
parsed unambiguously as a signed integer.

Finally, gcc already uses DW_FORM_sdata for negative offsets, fixing the
potential ambiguity altogether.

This patch mimics gcc's behaviour by emitting negative values of
DW_AT_bit_offset using the DW_FORM_sdata form. This eliminates any
potential misinterpretation.

One could argue that all values should use DW_FORM_sdata, but for the
sake of parity with gcc, it is safe to restrict the change to negative
values.
---
 llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp     | 11 +++++++++--
 llvm/test/DebugInfo/ARM/bitfield.ll           |  2 +-
 llvm/test/DebugInfo/NVPTX/packed_bitfields.ll |  2 +-
 llvm/test/DebugInfo/X86/packed_bitfields.ll   |  2 +-
 4 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index 56c288ee95b4..6533e8281631 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -30,6 +30,7 @@
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include <cassert>
 #include <cstdint>
+#include <limits>
 #include <string>
 #include <utility>
 
@@ -1649,7 +1650,8 @@ DIE &DwarfUnit::constructMemberDIE(DIE &Buffer, const DIDerivedType *DT) {
         addUInt(MemberDie, dwarf::DW_AT_byte_size, std::nullopt, FieldSize / 8);
       addUInt(MemberDie, dwarf::DW_AT_bit_size, std::nullopt, Size);
 
-      uint64_t Offset = DT->getOffsetInBits();
+      assert(DT->getOffsetInBits() <= std::numeric_limits<int64_t>::max());
+      int64_t Offset = DT->getOffsetInBits();
       // We can't use DT->getAlignInBits() here: AlignInBits for member type
       // is non-zero if and only if alignment was forced (e.g. _Alignas()),
       // which can't be done with bitfields. Thus we use FieldSize here.
@@ -1669,7 +1671,12 @@ DIE &DwarfUnit::constructMemberDIE(DIE &Buffer, const DIDerivedType *DT) {
         if (Asm->getDataLayout().isLittleEndian())
           Offset = FieldSize - (Offset + Size);
 
-        addUInt(MemberDie, dwarf::DW_AT_bit_offset, std::nullopt, Offset);
+        if (Offset < 0)
+          addSInt(MemberDie, dwarf::DW_AT_bit_offset, dwarf::DW_FORM_sdata,
+                  Offset);
+        else
+          addUInt(MemberDie, dwarf::DW_AT_bit_offset, std::nullopt,
+                  (uint64_t)Offset);
         OffsetInBytes = FieldOffset >> 3;
       } else {
         addUInt(MemberDie, dwarf::DW_AT_data_bit_offset, std::nullopt, Offset);
diff --git a/llvm/test/DebugInfo/ARM/bitfield.ll b/llvm/test/DebugInfo/ARM/bitfield.ll
index 5bd06b785b15..672c61db6f49 100644
--- a/llvm/test/DebugInfo/ARM/bitfield.ll
+++ b/llvm/test/DebugInfo/ARM/bitfield.ll
@@ -12,7 +12,7 @@
 ; CHECK:          DW_AT_name {{.*}} "reserved"
 ; CHECK:          DW_AT_byte_size  {{.*}} (0x04)
 ; CHECK:          DW_AT_bit_size   {{.*}} (0x1c)
-; CHECK:          DW_AT_bit_offset {{.*}} (0xfffffffffffffff8)
+; CHECK:          DW_AT_bit_offset {{.*}} (-8)
 ; CHECK:          DW_AT_data_member_location {{.*}} (DW_OP_plus_uconst 0x0)
 
 %struct.anon = type { i8, [5 x i8] }
diff --git a/llvm/test/DebugInfo/NVPTX/packed_bitfields.ll b/llvm/test/DebugInfo/NVPTX/packed_bitfields.ll
index e2097d7f49b4..62ffa0a4001f 100644
--- a/llvm/test/DebugInfo/NVPTX/packed_bitfields.ll
+++ b/llvm/test/DebugInfo/NVPTX/packed_bitfields.ll
@@ -16,7 +16,7 @@
 ; CHECK-NEXT: .b8 1    // DW_AT_byte_size
 ; CHECK-NEXT: .b8 6    // DW_AT_bit_size
 ; Negative offset must be encoded as an unsigned integer.
-; CHECK-NEXT: .b64 0xffffffffffffffff // DW_AT_bit_offset
+; CHECK-NEXT: .b8 127  // DW_AT_bit_offset
 ; CHECK-NEXT: .b8 2    // DW_AT_data_member_location
 
 %struct.anon = type { i16 }
diff --git a/llvm/test/DebugInfo/X86/packed_bitfields.ll b/llvm/test/DebugInfo/X86/packed_bitfields.ll
index 0e541f09d227..614fa59c3678 100644
--- a/llvm/test/DebugInfo/X86/packed_bitfields.ll
+++ b/llvm/test/DebugInfo/X86/packed_bitfields.ll
@@ -15,7 +15,7 @@
 ; CHECK-NOT: DW_TAG_member
 ; CHECK:      DW_AT_byte_size  {{.*}} (0x01)
 ; CHECK-NEXT: DW_AT_bit_size   {{.*}} (0x06)
-; CHECK-NEXT: DW_AT_bit_offset {{.*}} (0xffffffffffffffff)
+; CHECK-NEXT: DW_AT_bit_offset {{.*}} (-1)
 ; CHECK-NEXT: DW_AT_data_member_location {{.*}} ({{.*}}0x0{{0*}})
 
 ; ModuleID = 'repro.c'
-- 
GitLab


From 3438d8ac1ba58d098ff8d25a814b2c8c22d5844b Mon Sep 17 00:00:00 2001
From: Xing Xue <xingxue@outlook.com>
Date: Mon, 13 May 2024 06:29:50 -0400
Subject: [PATCH 0536/1206] [libcxx] [test] Fix the locale ctype widen tests on
 AIX (#91744)

The C locale on AIX uses `ISO-8859-1`, where `0xFB` is a valid
character. Widening char(-5) succeeds and produces L'\u00fb' the same as
on macOS, FreeBSD, and Windows. This patch removes `XFAIL:
LIBCXX-AIX-FIXME` and uses the macOS, FreeBSD, and WIN32 code path for
AIX.
---
 .../category.ctype/locale.ctype.byname/widen_1.pass.cpp        | 3 +--
 .../category.ctype/locale.ctype.byname/widen_many.pass.cpp     | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_1.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_1.pass.cpp
index bafdfcea0460..959a4be9e1de 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_1.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_1.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: locale.en_US.UTF-8
-// XFAIL: LIBCXX-AIX-FIXME
 // XFAIL: no-wide-characters
 
 // <locale>
@@ -57,7 +56,7 @@ int main(int, char**)
             assert(f.widen('.') == L'.');
             assert(f.widen('a') == L'a');
             assert(f.widen('1') == L'1');
-#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_WIN32)
+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_WIN32) || defined(_AIX)
             assert(f.widen(char(-5)) == L'\u00fb');
 #else
             assert(f.widen(char(-5)) == wchar_t(-1));
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_many.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_many.pass.cpp
index 552eab1f2ab4..078b4a6fefb7 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_many.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/widen_many.pass.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: locale.en_US.UTF-8
-// XFAIL: LIBCXX-AIX-FIXME
 // XFAIL: no-wide-characters
 
 // <locale>
@@ -63,7 +62,7 @@ int main(int, char**)
             assert(v[3] == L'.');
             assert(v[4] == L'a');
             assert(v[5] == L'1');
-#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_WIN32)
+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_WIN32) || defined(_AIX)
             assert(v[6] == L'\xfb');
 #else
             assert(v[6] == wchar_t(-1));
-- 
GitLab


From 7eeccc1430eeaa434724522945245ba21c97ac57 Mon Sep 17 00:00:00 2001
From: Xing Xue <xingxue@outlook.com>
Date: Mon, 13 May 2024 06:30:25 -0400
Subject: [PATCH 0537/1206] [libcxx][test] Fix numpunct grouping tests on AIX
 (#91781)

The `grouping` string for locale `en_US.UTF-8` and `fr_FR.UTF-8` on AIX
is `3`. This is different from Linux's `3;3` but is the same as Windows.
This patch removes `XFAIL: LIBCXX-AIX-FIXME` and changes to use the
`WIN32` code path.
---
 .../locale.numpunct.byname/grouping.pass.cpp          | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/grouping.pass.cpp b/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/grouping.pass.cpp
index 2f2bc2fd8832..86c447d400aa 100644
--- a/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/grouping.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/facet.numpunct/locale.numpunct.byname/grouping.pass.cpp
@@ -9,7 +9,6 @@
 // NetBSD does not support LC_NUMERIC at the moment
 // XFAIL: netbsd
 
-// XFAIL: LIBCXX-AIX-FIXME
 // XFAIL: LIBCXX-FREEBSD-FIXME
 
 // REQUIRES: locale.en_US.UTF-8
@@ -49,7 +48,7 @@ int main(int, char**)
         {
             typedef char C;
             const std::numpunct<C>& np = std::use_facet<std::numpunct<C> >(l);
-#ifdef _WIN32
+#if defined(_WIN32) || defined(_AIX)
             assert(np.grouping() == "\3");
 #else
             assert(np.grouping() == "\3\3");
@@ -59,17 +58,17 @@ int main(int, char**)
         {
             typedef wchar_t C;
             const std::numpunct<C>& np = std::use_facet<std::numpunct<C> >(l);
-#ifdef _WIN32
+#  if defined(_WIN32) || defined(_AIX)
             assert(np.grouping() == "\3");
-#else
+#  else
             assert(np.grouping() == "\3\3");
-#endif
+#  endif
         }
 #endif
     }
     {
         std::locale l(LOCALE_fr_FR_UTF_8);
-#if defined(TEST_HAS_GLIBC) || defined(_WIN32)
+#if defined(TEST_HAS_GLIBC) || defined(_WIN32) || defined(_AIX)
         const char* const group = "\3";
 #else
         const char* const group = "\x7f";
-- 
GitLab


From fbb37e960616efcf7cd5c1ebbe95f75c65d565dc Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter@arm.com>
Date: Mon, 13 May 2024 11:35:28 +0100
Subject: [PATCH 0538/1206] [AArch64] Add an all-in-one histogram intrinsic

Based on discussion from
https://discourse.llvm.org/t/rfc-vectorization-support-for-histogram-count-operations/74788

Current interface is:

llvm.experimental.histogram(<vecty> ptrs, <intty> inc_amount, <vecty> mask)

The integer type used by 'inc_amount' needs to match the type of the buckets in memory.

The intrinsic covers the following operations:
  * Gather load
  * histogram on the elements of 'ptrs'
  * multiply the histogram results by 'inc_amount'
  * add the result of the multiply to the values loaded by the gather
  * scatter store the results of the add

Supports lowering to histcnt instructions for AArch64 targets, and scalarization for all others at present.
---
 llvm/docs/LangRef.rst                         |  54 +++++++++
 .../llvm/Analysis/TargetTransformInfo.h       |   7 ++
 .../llvm/Analysis/TargetTransformInfoImpl.h   |   4 +
 llvm/include/llvm/CodeGen/ISDOpcodes.h        |   5 +
 llvm/include/llvm/CodeGen/SelectionDAG.h      |   3 +
 llvm/include/llvm/CodeGen/SelectionDAGNodes.h |  33 +++++
 llvm/include/llvm/IR/Intrinsics.td            |   7 ++
 llvm/lib/Analysis/TargetTransformInfo.cpp     |   5 +
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  38 ++++++
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  63 ++++++++++
 .../SelectionDAG/SelectionDAGBuilder.h        |   1 +
 .../SelectionDAG/SelectionDAGDumper.cpp       |   3 +
 .../Target/AArch64/AArch64ISelLowering.cpp    |  63 ++++++++++
 llvm/lib/Target/AArch64/AArch64ISelLowering.h |   1 +
 .../Scalar/ScalarizeMaskedMemIntrin.cpp       |  69 +++++++++++
 .../AArch64/neon-scalarize-histogram.ll       | 114 ++++++++++++++++++
 llvm/test/CodeGen/AArch64/sve2-histcnt.ll     |  53 ++++++++
 17 files changed, 523 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2-histcnt.ll

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index d0515876f9e4..06809f8bf445 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -19143,6 +19143,60 @@ will be on any later loop iteration.
 This intrinsic will only return 0 if the input count is also 0. A non-zero input
 count will produce a non-zero result.
 
+'``llvm.experimental.vector.histogram.*``' Intrinsics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+These intrinsics are overloaded.
+
+These intrinsics represent histogram-like operations; that is, updating values
+in memory that may not be contiguous, and where multiple elements within a
+single vector may be updating the same value in memory.
+
+The update operation must be specified as part of the intrinsic name. For a
+simple histogram like the following the ``add`` operation would be used.
+
+.. code-block:: c
+
+    void simple_histogram(int *restrict buckets, unsigned *indices, int N, int inc) {
+      for (int i = 0; i < N; ++i)
+        buckets[indices[i]] += inc;
+    }
+
+More update operation types may be added in the future.
+
+::
+
+    declare <8 x i32> @llvm.experimental.vector.histogram.add.v8p0.i32(<8 x ptr> %ptrs, i32 %inc, <8 x i1> %mask)
+    declare <vscale x 2 x i64> @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %ptrs, i64 %inc, <vscale x 2 x i1> %mask)
+
+Arguments:
+""""""""""
+
+The first argument is a vector of pointers to the memory locations to be
+updated. The second argument is a scalar used to update the value from
+memory; it must match the type of value to be updated. The final argument
+is a mask value to exclude locations from being modified.
+
+Semantics:
+""""""""""
+
+The '``llvm.experimental.vector.histogram.*``' intrinsics are used to perform
+updates on potentially overlapping values in memory. The intrinsics represent
+the follow sequence of operations:
+
+1. Gather load from the ``ptrs`` operand, with element type matching that of
+   the ``inc`` operand.
+2. Update of the values loaded from memory. In the case of the ``add``
+   update operation, this means:
+
+   1. Perform a cross-vector histogram operation on the ``ptrs`` operand.
+   2. Multiply the result by the ``inc`` operand.
+   3. Add the result to the values loaded from memory
+3. Scatter the result of the update operation to the memory locations from
+   the ``ptrs`` operand.
+
+The ``mask`` operand will apply to at least the gather and scatter operations.
+
 Matrix Intrinsics
 -----------------
 
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index f0eb83c143e2..0c3a6b3742c7 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -797,6 +797,9 @@ public:
   /// Return true if the target supports strided load.
   bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const;
 
+  // Return true if the target supports masked vector histograms.
+  bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) const;
+
   /// Return true if this is an alternating opcode pattern that can be lowered
   /// to a single instruction on the target. In X86 this is for the addsub
   /// instruction which corrsponds to a Shuffle + Fadd + FSub pattern in IR.
@@ -1883,6 +1886,7 @@ public:
   virtual bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) = 0;
   virtual bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) = 0;
   virtual bool isLegalStridedLoadStore(Type *DataType, Align Alignment) = 0;
+  virtual bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) = 0;
   virtual bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
                                unsigned Opcode1,
                                const SmallBitVector &OpcodeMask) const = 0;
@@ -2386,6 +2390,9 @@ public:
   bool isLegalStridedLoadStore(Type *DataType, Align Alignment) override {
     return Impl.isLegalStridedLoadStore(DataType, Alignment);
   }
+  bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) override {
+    return Impl.isLegalMaskedVectorHistogram(AddrType, DataType);
+  }
   bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
                        const SmallBitVector &OpcodeMask) const override {
     return Impl.isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 262ebdb3cbef..9a57331d281d 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -316,6 +316,10 @@ public:
     return false;
   }
 
+  bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) const {
+    return false;
+  }
+
   bool enableOrderedReductions() const { return false; }
 
   bool hasDivRemOp(Type *DataType, bool IsSigned) const { return false; }
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 6429947958ee..d8af97957e48 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1402,6 +1402,11 @@ enum NodeType {
   // which is later translated to an implicit use in the MIR.
   CONVERGENCECTRL_GLUE,
 
+  // Experimental vector histogram intrinsic
+  // Operands: Input Chain, Inc, Mask, Base, Index, Scale, ID
+  // Output: Output Chain
+  EXPERIMENTAL_VECTOR_HISTOGRAM,
+
   /// BUILTIN_OP_END - This must be the last enum value in this list.
   /// The target-specific pre-isel opcode values start here.
   BUILTIN_OP_END
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index c08e57ba3f67..979ef8033eb5 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1526,6 +1526,9 @@ public:
                            ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
                            ISD::MemIndexType IndexType,
                            bool IsTruncating = false);
+  SDValue getMaskedHistogram(SDVTList VTs, EVT MemVT, const SDLoc &dl,
+                             ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
+                             ISD::MemIndexType IndexType);
 
   SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT,
                       MachineMemOperand *MMO);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index e7c710414545..ac94c6099d08 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -542,6 +542,7 @@ BEGIN_TWO_BYTE_PACK()
     friend class MaskedLoadStoreSDNode;
     friend class MaskedGatherScatterSDNode;
     friend class VPGatherScatterSDNode;
+    friend class MaskedHistogramSDNode;
 
     uint16_t : NumMemSDNodeBits;
 
@@ -552,6 +553,7 @@ BEGIN_TWO_BYTE_PACK()
     //   MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
     //   VPGatherScatterSDNode => enum ISD::MemIndexType
     //   MaskedGatherScatterSDNode => enum ISD::MemIndexType
+    //   MaskedHistogramSDNode => enum ISD::MemIndexType
     uint16_t AddressingMode : 3;
   };
   enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
@@ -564,6 +566,7 @@ BEGIN_TWO_BYTE_PACK()
     friend class MaskedLoadSDNode;
     friend class MaskedGatherSDNode;
     friend class VPGatherSDNode;
+    friend class MaskedHistogramSDNode;
 
     uint16_t : NumLSBaseSDNodeBits;
 
@@ -1420,6 +1423,7 @@ public:
       return getOperand(2);
     case ISD::MGATHER:
     case ISD::MSCATTER:
+    case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
       return getOperand(3);
     default:
       return getOperand(1);
@@ -1468,6 +1472,7 @@ public:
     case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
     case ISD::GET_FPENV_MEM:
     case ISD::SET_FPENV_MEM:
+    case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
       return true;
     default:
       return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
@@ -2953,6 +2958,34 @@ public:
   }
 };
 
+class MaskedHistogramSDNode : public MemSDNode {
+public:
+  friend class SelectionDAG;
+
+  MaskedHistogramSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
+                        EVT MemVT, MachineMemOperand *MMO,
+                        ISD::MemIndexType IndexType)
+      : MemSDNode(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, Order, DL, VTs, MemVT,
+                  MMO) {
+    LSBaseSDNodeBits.AddressingMode = IndexType;
+  }
+
+  ISD::MemIndexType getIndexType() const {
+    return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
+  }
+
+  const SDValue &getBasePtr() const { return getOperand(3); }
+  const SDValue &getIndex() const { return getOperand(4); }
+  const SDValue &getMask() const { return getOperand(2); }
+  const SDValue &getScale() const { return getOperand(5); }
+  const SDValue &getInc() const { return getOperand(1); }
+  const SDValue &getIntID() const { return getOperand(6); }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::EXPERIMENTAL_VECTOR_HISTOGRAM;
+  }
+};
+
 class FPStateAccessSDNode : public MemSDNode {
 public:
   friend class SelectionDAG;
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 42192d472ba6..f1c7d950f927 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1856,6 +1856,13 @@ def int_experimental_vp_strided_load  : DefaultAttrsIntrinsic<[llvm_anyvector_ty
                                llvm_i32_ty],
                              [ NoCapture<ArgIndex<0>>, IntrNoSync, IntrReadMem, IntrWillReturn, IntrArgMemOnly ]>;
 
+// Experimental histogram
+def int_experimental_vector_histogram_add : DefaultAttrsIntrinsic<[],
+                             [ llvm_anyvector_ty, // Vector of pointers
+                               llvm_anyint_ty,    // Increment
+                               LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], // Mask
+                             [ IntrArgMemOnly ]>;
+
 // Operators
 let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
   // Integer arithmetic
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 00443ace46f7..f6a458f7ded4 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -513,6 +513,11 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
   return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
 }
 
+bool TargetTransformInfo::isLegalMaskedVectorHistogram(Type *AddrType,
+                                                       Type *DataType) const {
+  return TTIImpl->isLegalMaskedVectorHistogram(AddrType, DataType);
+}
+
 bool TargetTransformInfo::enableOrderedReductions() const {
   return TTIImpl->enableOrderedReductions();
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 0a258350c68a..247f52370e4c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -9633,6 +9633,44 @@ SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
   return V;
 }
 
+SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
+                                         const SDLoc &dl, ArrayRef<SDValue> Ops,
+                                         MachineMemOperand *MMO,
+                                         ISD::MemIndexType IndexType) {
+  assert(Ops.size() == 7 && "Incompatible number of operands");
+
+  FoldingSetNodeID ID;
+  AddNodeIDNode(ID, ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTs, Ops);
+  ID.AddInteger(MemVT.getRawBits());
+  ID.AddInteger(getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
+      dl.getIROrder(), VTs, MemVT, MMO, IndexType));
+  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
+  ID.AddInteger(MMO->getFlags());
+  void *IP = nullptr;
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
+    cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
+    return SDValue(E, 0);
+  }
+
+  auto *N = newSDNode<MaskedHistogramSDNode>(dl.getIROrder(), dl.getDebugLoc(),
+                                             VTs, MemVT, MMO, IndexType);
+  createOperands(N, Ops);
+
+  assert(N->getMask().getValueType().getVectorElementCount() ==
+             N->getIndex().getValueType().getVectorElementCount() &&
+         "Vector width mismatch between mask and data");
+  assert(isa<ConstantSDNode>(N->getScale()) &&
+         N->getScale()->getAsAPIntVal().isPowerOf2() &&
+         "Scale should be a constant power of 2");
+  assert(N->getInc().getValueType().isInteger() && "Non integer update value");
+
+  CSEMap.InsertNode(N, IP);
+  InsertNode(N);
+  SDValue V(N, 0);
+  NewSDValueDbgMsg(V, "Creating new node: ", this);
+  return V;
+}
+
 SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
                                   EVT MemVT, MachineMemOperand *MMO) {
   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index b76036a22992..ca352da5d36e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6281,6 +6281,64 @@ void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
   }
 }
 
+void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
+                                               unsigned IntrinsicID) {
+  // For now, we're only lowering an 'add' histogram.
+  // We can add others later, e.g. saturating adds, min/max.
+  assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
+         "Tried to lower unsupported histogram type");
+  SDLoc sdl = getCurSDLoc();
+  Value *Ptr = I.getOperand(0);
+  SDValue Inc = getValue(I.getOperand(1));
+  SDValue Mask = getValue(I.getOperand(2));
+
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  DataLayout TargetDL = DAG.getDataLayout();
+  EVT VT = Inc.getValueType();
+  Align Alignment = DAG.getEVTAlign(VT);
+
+  const MDNode *Ranges = getRangeMetadata(I);
+
+  SDValue Root = DAG.getRoot();
+  SDValue Base;
+  SDValue Index;
+  ISD::MemIndexType IndexType;
+  SDValue Scale;
+  bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
+                                    I.getParent(), VT.getScalarStoreSize());
+
+  unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
+
+  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+      MachinePointerInfo(AS),
+      MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
+      MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
+
+  if (!UniformBase) {
+    Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
+    Index = getValue(Ptr);
+    IndexType = ISD::SIGNED_SCALED;
+    Scale =
+        DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
+  }
+
+  EVT IdxVT = Index.getValueType();
+  EVT EltTy = IdxVT.getVectorElementType();
+  if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
+    EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
+    Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
+  }
+
+  SDValue ID = DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32);
+
+  SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
+  SDValue Histogram = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), VT, sdl,
+                                             Ops, MMO, IndexType);
+
+  setValue(&I, Histogram);
+  DAG.setRoot(Histogram);
+}
+
 /// Lower the call to the specified intrinsic function.
 void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
                                              unsigned Intrinsic) {
@@ -7948,6 +8006,11 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
   case Intrinsic::experimental_convergence_entry:
   case Intrinsic::experimental_convergence_loop:
     visitConvergenceControl(I, Intrinsic);
+    return;
+  case Intrinsic::experimental_vector_histogram_add: {
+    visitVectorHistogram(I, Intrinsic);
+    return;
+  }
   }
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 211e1653de56..ae361f8c500a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -624,6 +624,7 @@ private:
   void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
   void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI);
   void visitConvergenceControl(const CallInst &I, unsigned Intrinsic);
+  void visitVectorHistogram(const CallInst &I, unsigned IntrinsicID);
   void visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT,
                    const SmallVectorImpl<SDValue> &OpValues);
   void visitVPStore(const VPIntrinsic &VPIntrin,
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 4ad4a938ca97..59742e90c679 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -529,6 +529,9 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::PATCHPOINT:
     return "patchpoint";
 
+  case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
+    return "histogram";
+
     // Vector Predication
 #define BEGIN_REGISTER_VP_SDNODE(SDID, LEGALARG, NAME, ...)                    \
   case ISD::SDID:                                                              \
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 1e0071fffe66..0f1db3cb17aa 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1618,6 +1618,11 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
         setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
     }
 
+    // Histcnt is SVE2 only
+    if (Subtarget->hasSVE2() && Subtarget->isSVEAvailable())
+      setOperationAction(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, MVT::Other,
+                         Custom);
+
     // NOTE: Currently this has to happen after computeRegisterProperties rather
     // than the preferred option of combining it with the addRegisterClass call.
     if (Subtarget->useSVEForFixedLengthVectors()) {
@@ -6775,6 +6780,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
     return LowerFunnelShift(Op, DAG);
   case ISD::FLDEXP:
     return LowerFLDEXP(Op, DAG);
+  case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
+    return LowerVECTOR_HISTOGRAM(Op, DAG);
   }
 }
 
@@ -27355,6 +27362,62 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
   return DAG.getMergeValues({Lo, Hi}, DL);
 }
 
+SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
+                                                     SelectionDAG &DAG) const {
+  // FIXME: Maybe share some code with LowerMGather/Scatter?
+  MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Op);
+  SDLoc DL(HG);
+  SDValue Chain = HG->getChain();
+  SDValue Inc = HG->getInc();
+  SDValue Mask = HG->getMask();
+  SDValue Ptr = HG->getBasePtr();
+  SDValue Index = HG->getIndex();
+  SDValue Scale = HG->getScale();
+  SDValue IntID = HG->getIntID();
+
+  // The Intrinsic ID determines the type of update operation.
+  ConstantSDNode *CID = cast<ConstantSDNode>(IntID.getNode());
+  // Right now, we only support 'add' as an update.
+  assert(CID->getZExtValue() == Intrinsic::experimental_vector_histogram_add &&
+         "Unexpected histogram update operation");
+
+  EVT IncVT = Inc.getValueType();
+  EVT IndexVT = Index.getValueType();
+  EVT MemVT = EVT::getVectorVT(*DAG.getContext(), IncVT,
+                               IndexVT.getVectorElementCount());
+  SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
+  SDValue PassThru = DAG.getSplatVector(MemVT, DL, Zero);
+  SDValue IncSplat = DAG.getSplatVector(MemVT, DL, Inc);
+  SDValue Ops[] = {Chain, PassThru, Mask, Ptr, Index, Scale};
+
+  // Set the MMO to load only, rather than load|store.
+  MachineMemOperand *GMMO = HG->getMemOperand();
+  GMMO->setFlags(MachineMemOperand::MOLoad);
+  ISD::MemIndexType IndexType = HG->getIndexType();
+  SDValue Gather =
+      DAG.getMaskedGather(DAG.getVTList(MemVT, MVT::Other), MemVT, DL, Ops,
+                          GMMO, IndexType, ISD::NON_EXTLOAD);
+
+  SDValue GChain = Gather.getValue(1);
+
+  // Perform the histcnt, multiply by inc, add to bucket data.
+  SDValue ID = DAG.getTargetConstant(Intrinsic::aarch64_sve_histcnt, DL, IncVT);
+  SDValue HistCnt =
+      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, ID, Mask, Index, Index);
+  SDValue Mul = DAG.getNode(ISD::MUL, DL, MemVT, HistCnt, IncSplat);
+  SDValue Add = DAG.getNode(ISD::ADD, DL, MemVT, Gather, Mul);
+
+  // Create a new MMO for the scatter.
+  MachineMemOperand *SMMO = DAG.getMachineFunction().getMachineMemOperand(
+      GMMO->getPointerInfo(), MachineMemOperand::MOStore, GMMO->getSize(),
+      GMMO->getAlign(), GMMO->getAAInfo());
+
+  SDValue ScatterOps[] = {GChain, Add, Mask, Ptr, Index, Scale};
+  SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MemVT, DL,
+                                         ScatterOps, SMMO, IndexType, false);
+  return Scatter;
+}
+
 SDValue
 AArch64TargetLowering::LowerFixedLengthFPToIntToSVE(SDValue Op,
                                                     SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index b3e282a04060..a44a3d35d2f9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -1149,6 +1149,7 @@ private:
   SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVECTOR_DEINTERLEAVE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVECTOR_INTERLEAVE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerVECTOR_HISTOGRAM(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerDIV(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index a4111fad5d9f..de80fa2c0502 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -862,6 +862,69 @@ static void scalarizeMaskedCompressStore(const DataLayout &DL, CallInst *CI,
   ModifiedDT = true;
 }
 
+static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
+                                           DomTreeUpdater *DTU,
+                                           bool &ModifiedDT) {
+  // If we extend histogram to return a result someday (like the updated vector)
+  // then we'll need to support it here.
+  assert(CI->getType()->isVoidTy() && "Histogram with non-void return.");
+  Value *Ptrs = CI->getArgOperand(0);
+  Value *Inc = CI->getArgOperand(1);
+  Value *Mask = CI->getArgOperand(2);
+
+  auto *AddrType = cast<FixedVectorType>(Ptrs->getType());
+  Type *EltTy = Inc->getType();
+
+  IRBuilder<> Builder(CI->getContext());
+  Instruction *InsertPt = CI;
+  Builder.SetInsertPoint(InsertPt);
+
+  Builder.SetCurrentDebugLocation(CI->getDebugLoc());
+
+  // FIXME: Do we need to add an alignment parameter to the intrinsic?
+  unsigned VectorWidth = AddrType->getNumElements();
+
+  // Shorten the way if the mask is a vector of constants.
+  if (isConstantIntVector(Mask)) {
+    for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
+      if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue())
+        continue;
+      Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
+      LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
+      Value *Add = Builder.CreateAdd(Load, Inc);
+      Builder.CreateStore(Add, Ptr);
+    }
+    CI->eraseFromParent();
+    return;
+  }
+
+  for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
+    Value *Predicate =
+        Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));
+
+    Instruction *ThenTerm =
+        SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
+                                  /*BranchWeights=*/nullptr, DTU);
+
+    BasicBlock *CondBlock = ThenTerm->getParent();
+    CondBlock->setName("cond.histogram.update");
+
+    Builder.SetInsertPoint(CondBlock->getTerminator());
+    Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
+    LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
+    Value *Add = Builder.CreateAdd(Load, Inc);
+    Builder.CreateStore(Add, Ptr);
+
+    // Create "else" block, fill it in the next iteration
+    BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
+    NewIfBlock->setName("else");
+    Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());
+  }
+
+  CI->eraseFromParent();
+  ModifiedDT = true;
+}
+
 static bool runImpl(Function &F, const TargetTransformInfo &TTI,
                     DominatorTree *DT) {
   std::optional<DomTreeUpdater> DTU;
@@ -938,6 +1001,12 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
     switch (II->getIntrinsicID()) {
     default:
       break;
+    case Intrinsic::experimental_vector_histogram_add:
+      if (TTI.isLegalMaskedVectorHistogram(CI->getArgOperand(0)->getType(),
+                                           CI->getArgOperand(1)->getType()))
+        return false;
+      scalarizeMaskedVectorHistogram(DL, CI, DTU, ModifiedDT);
+      break;
     case Intrinsic::masked_load:
       // Scalarize unsupported vector masked load
       if (TTI.isLegalMaskedLoad(
diff --git a/llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll b/llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll
new file mode 100644
index 000000000000..45f1429a810a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=aarch64 < %s -o - | FileCheck %s
+
+;; This test exercises the default lowering of the histogram to scalarized code.
+
+define void @histogram_i64(<2 x ptr> %buckets, i64 %inc, <2 x i1> %mask) {
+; CHECK-LABEL: histogram_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    fmov w8, s1
+; CHECK-NEXT:    tbnz w8, #0, .LBB0_3
+; CHECK-NEXT:  // %bb.1: // %else
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    tbnz w8, #0, .LBB0_4
+; CHECK-NEXT:  .LBB0_2: // %else2
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_3: // %cond.histogram.update
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    ldr x9, [x8]
+; CHECK-NEXT:    add x9, x9, x0
+; CHECK-NEXT:    str x9, [x8]
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    tbz w8, #0, .LBB0_2
+; CHECK-NEXT:  .LBB0_4: // %cond.histogram.update1
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    ldr x9, [x8]
+; CHECK-NEXT:    add x9, x9, x0
+; CHECK-NEXT:    str x9, [x8]
+; CHECK-NEXT:    ret
+  call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<2 x ptr> %buckets, i64 %inc, <2 x i1> %mask)
+  ret void
+}
+
+define void @histogram_i32_literal(ptr %base, <4 x i32> %indices, <4 x i1> %mask) {
+; CHECK-LABEL: histogram_i32_literal:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v2.2d, x0
+; CHECK-NEXT:    sshll v3.2d, v0.2s, #2
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    umov w8, v1.h[0]
+; CHECK-NEXT:    add v3.2d, v2.2d, v3.2d
+; CHECK-NEXT:    tbz w8, #0, .LBB1_2
+; CHECK-NEXT:  // %bb.1: // %cond.histogram.update
+; CHECK-NEXT:    fmov x8, d3
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    add w9, w9, #1
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:  .LBB1_2: // %else
+; CHECK-NEXT:    umov w8, v1.h[1]
+; CHECK-NEXT:    sshll2 v0.2d, v0.4s, #2
+; CHECK-NEXT:    tbz w8, #0, .LBB1_4
+; CHECK-NEXT:  // %bb.3: // %cond.histogram.update1
+; CHECK-NEXT:    mov x8, v3.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    add w9, w9, #1
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:  .LBB1_4: // %else2
+; CHECK-NEXT:    umov w8, v1.h[2]
+; CHECK-NEXT:    add v0.2d, v2.2d, v0.2d
+; CHECK-NEXT:    tbnz w8, #0, .LBB1_7
+; CHECK-NEXT:  // %bb.5: // %else4
+; CHECK-NEXT:    umov w8, v1.h[3]
+; CHECK-NEXT:    tbnz w8, #0, .LBB1_8
+; CHECK-NEXT:  .LBB1_6: // %else6
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_7: // %cond.histogram.update3
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    add w9, w9, #1
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    umov w8, v1.h[3]
+; CHECK-NEXT:    tbz w8, #0, .LBB1_6
+; CHECK-NEXT:  .LBB1_8: // %cond.histogram.update5
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    add w9, w9, #1
+; CHECK-NEXT:    str w9, [x8]
+; CHECK-NEXT:    ret
+
+  %buckets = getelementptr i32, ptr %base, <4 x i32> %indices
+  call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+  ret void
+}
+
+define void @histogram_i32_literal_alltruemask(ptr %base, <4 x i32> %indices) {
+; CHECK-LABEL: histogram_i32_literal_alltruemask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dup v1.2d, x0
+; CHECK-NEXT:    sshll v2.2d, v0.2s, #2
+; CHECK-NEXT:    sshll2 v0.2d, v0.4s, #2
+; CHECK-NEXT:    add v2.2d, v1.2d, v2.2d
+; CHECK-NEXT:    add v0.2d, v1.2d, v0.2d
+; CHECK-NEXT:    fmov x8, d2
+; CHECK-NEXT:    mov x9, v2.d[1]
+; CHECK-NEXT:    ldr w10, [x8]
+; CHECK-NEXT:    add w10, w10, #1
+; CHECK-NEXT:    str w10, [x8]
+; CHECK-NEXT:    ldr w8, [x9]
+; CHECK-NEXT:    add w8, w8, #1
+; CHECK-NEXT:    str w8, [x9]
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    mov x9, v0.d[1]
+; CHECK-NEXT:    ldr w10, [x8]
+; CHECK-NEXT:    add w10, w10, #1
+; CHECK-NEXT:    str w10, [x8]
+; CHECK-NEXT:    ldr w8, [x9]
+; CHECK-NEXT:    add w8, w8, #1
+; CHECK-NEXT:    str w8, [x9]
+; CHECK-NEXT:    ret
+
+  %buckets = getelementptr i32, ptr %base, <4 x i32> %indices
+  call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/sve2-histcnt.ll b/llvm/test/CodeGen/AArch64/sve2-histcnt.ll
new file mode 100644
index 000000000000..557a42116cdb
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-histcnt.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=aarch64 < %s -o - | FileCheck %s
+
+define void @histogram_i64(<vscale x 2 x ptr> %buckets, i64 %inc, <vscale x 2 x i1> %mask) #0 {
+; CHECK-LABEL: histogram_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    histcnt z1.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    mov z3.d, x0
+; CHECK-NEXT:    ld1d { z2.d }, p0/z, [z0.d]
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    mad z1.d, p1/m, z3.d, z2.d
+; CHECK-NEXT:    st1d { z1.d }, p0, [z0.d]
+; CHECK-NEXT:    ret
+  call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 %inc, <vscale x 2 x i1> %mask)
+  ret void
+}
+
+;; FIXME: We maybe need some dagcombines here? We're multiplying the output of the histcnt
+;;        by 1, so we should be able to remove that and directly add the histcnt to the
+;;        current bucket data.
+define void @histogram_i32_literal(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: histogram_i32_literal:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    histcnt z1.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    mov z3.s, #1 // =0x1
+; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x0, z0.s, sxtw #2]
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    mad z1.s, p1/m, z3.s, z2.s
+; CHECK-NEXT:    st1w { z1.s }, p0, [x0, z0.s, sxtw #2]
+; CHECK-NEXT:    ret
+
+  %buckets = getelementptr i32, ptr %base, <vscale x 4 x i32> %indices
+  call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
+  ret void
+}
+
+define void @histogram_i32_literal_noscale(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: histogram_i32_literal_noscale:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    histcnt z1.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    mov z3.s, #1 // =0x1
+; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    mad z1.s, p1/m, z3.s, z2.s
+; CHECK-NEXT:    st1w { z1.s }, p0, [x0, z0.s, sxtw]
+; CHECK-NEXT:    ret
+
+  %buckets = getelementptr i8, ptr %base, <vscale x 4 x i32> %indices
+  call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve2" vscale_range(1, 16) }
-- 
GitLab


From fcc1baaa99fda4f5633e82f47e8de33c99bbcdd2 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Mon, 13 May 2024 18:48:27 +0800
Subject: [PATCH 0539/1206] [AArch64] Fix -Wunused-variable in
 AArch64ISelLowering.cpp (NFC)

llvm-project/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp:27379:19:
error: unused variable 'CID' [-Werror,-Wunused-variable]
  ConstantSDNode *CID = cast<ConstantSDNode>(IntID.getNode());
                  ^
1 error generated.
---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 0f1db3cb17aa..2aa328e0a127 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27376,7 +27376,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
   SDValue IntID = HG->getIntID();
 
   // The Intrinsic ID determines the type of update operation.
-  ConstantSDNode *CID = cast<ConstantSDNode>(IntID.getNode());
+  [[maybe_unused]] ConstantSDNode *CID = cast<ConstantSDNode>(IntID.getNode());
   // Right now, we only support 'add' as an update.
   assert(CID->getZExtValue() == Intrinsic::experimental_vector_histogram_add &&
          "Unexpected histogram update operation");
-- 
GitLab


From 0a6103eaeb7f22c009f9add87c84780b6f7f293a Mon Sep 17 00:00:00 2001
From: Muhammad Omair Javaid <omair.javaid@linaro.org>
Date: Mon, 13 May 2024 16:01:17 +0500
Subject: [PATCH 0540/1206] Revert "[lldb] Attempt to fix
 signal-in-leaf-function-aarch64 on darwin"

This reverts commit b903badd73a2467fdd4e363231f2bf9b0704b546.

TestInterruptBacktrace was broken on AArch64/Windows as a result of this change.
see lldb-aarch64-windows buildbot here:
https://lab.llvm.org/buildbot/#/builders/219/builds/11261
---
 .../test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c | 2 +-
 lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test    | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c b/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
index fe020affcad0..9a751330623f 100644
--- a/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
+++ b/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
@@ -7,7 +7,7 @@ int __attribute__((naked)) signal_generating_add(int a, int b) {
       "ret");
 }
 
-void sigill_handler(int signo) { _exit(0); }
+void sigill_handler(int) { _exit(0); }
 
 int main() {
   signal(SIGILL, sigill_handler);
diff --git a/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test b/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test
index 09f17c174bbf..0580d0cf734a 100644
--- a/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test
+++ b/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test
@@ -4,9 +4,6 @@
 # RUN: %clang_host %S/Inputs/signal-in-leaf-function-aarch64.c -o %t
 # RUN: %lldb -s %s -o exit %t | FileCheck %s
 
-# Convert EXC_BAD_INSTRUCTION to SIGILL on darwin
-settings set platform.plugin.darwin.ignored-exceptions EXC_BAD_INSTRUCTION
-
 breakpoint set -n sigill_handler
 # CHECK: Breakpoint 1: where = {{.*}}`sigill_handler
 
-- 
GitLab


From 4b44502ac81259630b422e791a82e0252e6478c3 Mon Sep 17 00:00:00 2001
From: Muhammad Omair Javaid <omair.javaid@linaro.org>
Date: Mon, 13 May 2024 16:03:33 +0500
Subject: [PATCH 0541/1206] Revert "[lldb/aarch64] Fix unwinding when signal
 interrupts a leaf function (#91321)"

This reverts commit fd1bd53ba5a06f344698a55578f6a5d79c457e30.

TestInterruptBacktrace was broken on AArch64/Windows as a result of this change.
See lldb-aarch64-windows buildbot here:
https://lab.llvm.org/buildbot/#/builders/219/builds/11261
---
 .../ARM64/EmulateInstructionARM64.cpp         |  2 --
 .../UnwindAssemblyInstEmulation.cpp           |  4 +++-
 lldb/source/Target/RegisterContextUnwind.cpp  |  6 ++---
 .../Inputs/signal-in-leaf-function-aarch64.c  | 15 ------------
 .../signal-in-leaf-function-aarch64.test      | 24 -------------------
 .../ARM64/TestArm64InstEmulation.cpp          | 24 ++++---------------
 6 files changed, 10 insertions(+), 65 deletions(-)
 delete mode 100644 lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
 delete mode 100644 lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test

diff --git a/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp b/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
index 62ecac3e0831..6ca4fb052457 100644
--- a/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
+++ b/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
@@ -444,8 +444,6 @@ bool EmulateInstructionARM64::CreateFunctionEntryUnwind(
 
   // Our previous Call Frame Address is the stack pointer
   row->GetCFAValue().SetIsRegisterPlusOffset(gpr_sp_arm64, 0);
-  row->SetRegisterLocationToSame(gpr_lr_arm64, /*must_replace=*/false);
-  row->SetRegisterLocationToSame(gpr_fp_arm64, /*must_replace=*/false);
 
   unwind_plan.AppendRow(row);
   unwind_plan.SetSourceName("EmulateInstructionARM64");
diff --git a/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp b/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
index 49edd40544e3..c4a171ec7d01 100644
--- a/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
+++ b/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
@@ -424,6 +424,8 @@ size_t UnwindAssemblyInstEmulation::WriteMemory(
     log->PutString(strm.GetString());
   }
 
+  const bool cant_replace = false;
+
   switch (context.type) {
   default:
   case EmulateInstruction::eContextInvalid:
@@ -465,7 +467,7 @@ size_t UnwindAssemblyInstEmulation::WriteMemory(
         m_pushed_regs[reg_num] = addr;
         const int32_t offset = addr - m_initial_sp;
         m_curr_row->SetRegisterLocationToAtCFAPlusOffset(reg_num, offset,
-                                                         /*can_replace=*/true);
+                                                         cant_replace);
         m_curr_row_modified = true;
       }
     }
diff --git a/lldb/source/Target/RegisterContextUnwind.cpp b/lldb/source/Target/RegisterContextUnwind.cpp
index e2d712cb72ea..13e101413a47 100644
--- a/lldb/source/Target/RegisterContextUnwind.cpp
+++ b/lldb/source/Target/RegisterContextUnwind.cpp
@@ -1555,12 +1555,12 @@ RegisterContextUnwind::SavedLocationForRegister(
   }
 
   if (unwindplan_regloc.IsSame()) {
-    if (!m_all_registers_available &&
+    if (!IsFrameZero() &&
         (regnum.GetAsKind(eRegisterKindGeneric) == LLDB_REGNUM_GENERIC_PC ||
          regnum.GetAsKind(eRegisterKindGeneric) == LLDB_REGNUM_GENERIC_RA)) {
       UnwindLogMsg("register %s (%d) is marked as 'IsSame' - it is a pc or "
-                   "return address reg on a frame which does not have all "
-                   "registers available -- treat as if we have no information",
+                   "return address reg on a non-zero frame -- treat as if we "
+                   "have no information",
                    regnum.GetName(), regnum.GetAsKind(eRegisterKindLLDB));
       return UnwindLLDB::RegisterSearchResult::eRegisterNotFound;
     } else {
diff --git a/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c b/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
deleted file mode 100644
index 9a751330623f..000000000000
--- a/lldb/test/Shell/Unwind/Inputs/signal-in-leaf-function-aarch64.c
+++ /dev/null
@@ -1,15 +0,0 @@
-#include <signal.h>
-#include <unistd.h>
-
-int __attribute__((naked)) signal_generating_add(int a, int b) {
-  asm("add w0, w1, w0\n\t"
-      "udf #0xdead\n\t"
-      "ret");
-}
-
-void sigill_handler(int) { _exit(0); }
-
-int main() {
-  signal(SIGILL, sigill_handler);
-  return signal_generating_add(42, 47);
-}
diff --git a/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test b/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test
deleted file mode 100644
index 0580d0cf734a..000000000000
--- a/lldb/test/Shell/Unwind/signal-in-leaf-function-aarch64.test
+++ /dev/null
@@ -1,24 +0,0 @@
-# REQUIRES: target-aarch64 && native
-# UNSUPPORTED: system-windows
-
-# RUN: %clang_host %S/Inputs/signal-in-leaf-function-aarch64.c -o %t
-# RUN: %lldb -s %s -o exit %t | FileCheck %s
-
-breakpoint set -n sigill_handler
-# CHECK: Breakpoint 1: where = {{.*}}`sigill_handler
-
-run
-# CHECK: thread #1, {{.*}} stop reason = signal SIGILL
-
-thread backtrace
-# CHECK: frame #0: [[ADD:0x[0-9a-fA-F]*]] {{.*}}`signal_generating_add
-# CHECK: frame #1: [[MAIN:0x[0-9a-fA-F]*]] {{.*}}`main
-
-continue
-# CHECK: thread #1, {{.*}} stop reason = breakpoint 1
-
-thread backtrace
-# CHECK: frame #0: {{.*}}`sigill_handler
-# Unknown number of signal trampoline frames
-# CHECK: frame #{{[0-9]+}}: [[ADD]] {{.*}}`signal_generating_add
-# CHECK: frame #{{[0-9]+}}: [[MAIN]] {{.*}}`main
diff --git a/lldb/unittests/UnwindAssembly/ARM64/TestArm64InstEmulation.cpp b/lldb/unittests/UnwindAssembly/ARM64/TestArm64InstEmulation.cpp
index 9303d6f5f3c6..80abeb8fae9e 100644
--- a/lldb/unittests/UnwindAssembly/ARM64/TestArm64InstEmulation.cpp
+++ b/lldb/unittests/UnwindAssembly/ARM64/TestArm64InstEmulation.cpp
@@ -77,7 +77,7 @@ TEST_F(TestArm64InstEmulation, TestSimpleDarwinFunction) {
 
   // UnwindPlan we expect:
 
-  // row[0]:    0: CFA=sp +0 => fp= <same> lr= <same>
+  // row[0]:    0: CFA=sp +0 =>
   // row[1]:    4: CFA=sp+16 => fp=[CFA-16] lr=[CFA-8]
   // row[2]:    8: CFA=fp+16 => fp=[CFA-16] lr=[CFA-8]
   // row[2]:   16: CFA=sp+16 => fp=[CFA-16] lr=[CFA-8]
@@ -88,19 +88,13 @@ TEST_F(TestArm64InstEmulation, TestSimpleDarwinFunction) {
   EXPECT_TRUE(engine->GetNonCallSiteUnwindPlanFromAssembly(
       sample_range, data, sizeof(data), unwind_plan));
 
-  // CFA=sp +0 => fp= <same> lr= <same>
+  // CFA=sp +0
   row_sp = unwind_plan.GetRowForFunctionOffset(0);
   EXPECT_EQ(0ull, row_sp->GetOffset());
   EXPECT_TRUE(row_sp->GetCFAValue().GetRegisterNumber() == gpr_sp_arm64);
   EXPECT_TRUE(row_sp->GetCFAValue().IsRegisterPlusOffset() == true);
   EXPECT_EQ(0, row_sp->GetCFAValue().GetOffset());
 
-  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_fp_arm64, regloc));
-  EXPECT_TRUE(regloc.IsSame());
-
-  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_lr_arm64, regloc));
-  EXPECT_TRUE(regloc.IsSame());
-
   // CFA=sp+16 => fp=[CFA-16] lr=[CFA-8]
   row_sp = unwind_plan.GetRowForFunctionOffset(4);
   EXPECT_EQ(4ull, row_sp->GetOffset());
@@ -152,12 +146,6 @@ TEST_F(TestArm64InstEmulation, TestSimpleDarwinFunction) {
   EXPECT_TRUE(row_sp->GetCFAValue().GetRegisterNumber() == gpr_sp_arm64);
   EXPECT_TRUE(row_sp->GetCFAValue().IsRegisterPlusOffset() == true);
   EXPECT_EQ(0, row_sp->GetCFAValue().GetOffset());
-
-  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_fp_arm64, regloc));
-  EXPECT_TRUE(regloc.IsSame());
-
-  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_lr_arm64, regloc));
-  EXPECT_TRUE(regloc.IsSame());
 }
 
 TEST_F(TestArm64InstEmulation, TestMediumDarwinFunction) {
@@ -393,12 +381,8 @@ TEST_F(TestArm64InstEmulation, TestFramelessThreeEpilogueFunction) {
   EXPECT_FALSE(row_sp->GetRegisterInfo(gpr_x26_arm64, regloc));
   EXPECT_FALSE(row_sp->GetRegisterInfo(gpr_x27_arm64, regloc));
   EXPECT_FALSE(row_sp->GetRegisterInfo(gpr_x28_arm64, regloc));
-
-  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_fp_arm64, regloc));
-  EXPECT_TRUE(regloc.IsSame());
-
-  EXPECT_TRUE(row_sp->GetRegisterInfo(gpr_lr_arm64, regloc));
-  EXPECT_TRUE(regloc.IsSame());
+  EXPECT_FALSE(row_sp->GetRegisterInfo(gpr_fp_arm64, regloc));
+  EXPECT_FALSE(row_sp->GetRegisterInfo(gpr_lr_arm64, regloc));
 
   row_sp = unwind_plan.GetRowForFunctionOffset(36);
   EXPECT_TRUE(row_sp->GetCFAValue().GetRegisterNumber() == gpr_sp_arm64);
-- 
GitLab


From 99934daa9b795278b8cc168fad430e09473b4992 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Mon, 13 May 2024 19:12:05 +0800
Subject: [PATCH 0542/1206] [ADT] Introduce `APInt::clearHighBits` (#91938)

This patch addresses
https://github.com/llvm/llvm-project/pull/90034#discussion_r1579235844.
---
 llvm/include/llvm/ADT/APInt.h    |  7 ++++
 llvm/unittests/ADT/APIntTest.cpp | 65 ++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)

diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index 8d3c029b2e7e..2fd8b7ea636c 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -1398,6 +1398,13 @@ public:
     *this &= Keep;
   }
 
+  /// Set top hiBits bits to 0.
+  void clearHighBits(unsigned hiBits) {
+    assert(hiBits <= BitWidth && "More bits than bitwidth");
+    APInt Keep = getLowBitsSet(BitWidth, BitWidth - hiBits);
+    *this &= Keep;
+  }
+
   /// Set the sign bit to 0.
   void clearSignBit() { clearBit(BitWidth - 1); }
 
diff --git a/llvm/unittests/ADT/APIntTest.cpp b/llvm/unittests/ADT/APIntTest.cpp
index 46aaa47ee645..eb4b847185f5 100644
--- a/llvm/unittests/ADT/APIntTest.cpp
+++ b/llvm/unittests/ADT/APIntTest.cpp
@@ -2534,6 +2534,71 @@ TEST(APIntTest, clearLowBits) {
   EXPECT_EQ(16u, i32hi16.popcount());
 }
 
+TEST(APIntTest, clearHighBits) {
+  APInt i64hi32 = APInt::getAllOnes(64);
+  i64hi32.clearHighBits(32);
+  EXPECT_EQ(32u, i64hi32.countr_one());
+  EXPECT_EQ(0u, i64hi32.countr_zero());
+  EXPECT_EQ(32u, i64hi32.getActiveBits());
+  EXPECT_EQ(32u, i64hi32.countl_zero());
+  EXPECT_EQ(0u, i64hi32.countl_one());
+  EXPECT_EQ(32u, i64hi32.popcount());
+
+  APInt i128hi64 = APInt::getAllOnes(128);
+  i128hi64.clearHighBits(64);
+  EXPECT_EQ(64u, i128hi64.countr_one());
+  EXPECT_EQ(0u, i128hi64.countr_zero());
+  EXPECT_EQ(64u, i128hi64.getActiveBits());
+  EXPECT_EQ(64u, i128hi64.countl_zero());
+  EXPECT_EQ(0u, i128hi64.countl_one());
+  EXPECT_EQ(64u, i128hi64.popcount());
+
+  APInt i128hi24 = APInt::getAllOnes(128);
+  i128hi24.clearHighBits(104);
+  EXPECT_EQ(24u, i128hi24.countr_one());
+  EXPECT_EQ(0u, i128hi24.countr_zero());
+  EXPECT_EQ(24u, i128hi24.getActiveBits());
+  EXPECT_EQ(104u, i128hi24.countl_zero());
+  EXPECT_EQ(0u, i128hi24.countl_one());
+  EXPECT_EQ(24u, i128hi24.popcount());
+
+  APInt i128hi104 = APInt::getAllOnes(128);
+  i128hi104.clearHighBits(24);
+  EXPECT_EQ(104u, i128hi104.countr_one());
+  EXPECT_EQ(0u, i128hi104.countr_zero());
+  EXPECT_EQ(104u, i128hi104.getActiveBits());
+  EXPECT_EQ(24u, i128hi104.countl_zero());
+  EXPECT_EQ(0u, i128hi104.countl_one());
+  EXPECT_EQ(104u, i128hi104.popcount());
+
+  APInt i128hi0 = APInt::getAllOnes(128);
+  i128hi0.clearHighBits(128);
+  EXPECT_EQ(0u, i128hi0.countr_one());
+  EXPECT_EQ(128u, i128hi0.countr_zero());
+  EXPECT_EQ(0u, i128hi0.getActiveBits());
+  EXPECT_EQ(128u, i128hi0.countl_zero());
+  EXPECT_EQ(0u, i128hi0.countl_one());
+  EXPECT_EQ(0u, i128hi0.popcount());
+
+  APInt i80hi1 = APInt::getAllOnes(80);
+  i80hi1.clearHighBits(79);
+  EXPECT_EQ(1u, i80hi1.countr_one());
+  EXPECT_EQ(0u, i80hi1.countr_zero());
+  EXPECT_EQ(1u, i80hi1.getActiveBits());
+  EXPECT_EQ(79u, i80hi1.countl_zero());
+  EXPECT_EQ(0u, i80hi1.countl_one());
+  EXPECT_EQ(1u, i80hi1.popcount());
+
+  APInt i32hi16 = APInt::getAllOnes(32);
+  i32hi16.clearHighBits(16);
+  EXPECT_EQ(16u, i32hi16.countr_one());
+  EXPECT_EQ(0u, i32hi16.countr_zero());
+  EXPECT_EQ(16u, i32hi16.getActiveBits());
+  EXPECT_EQ(16u, i32hi16.countl_zero());
+  EXPECT_EQ(0u, i32hi16.countl_one());
+  EXPECT_EQ(16u, i32hi16.popcount());
+}
+
 TEST(APIntTest, abds) {
   using APIntOps::abds;
 
-- 
GitLab


From efe91cf78bccda90637c817e3e592b5f34e891d0 Mon Sep 17 00:00:00 2001
From: Rajveer Singh Bharadwaj <rajveer.developer@icloud.com>
Date: Mon, 13 May 2024 17:13:35 +0530
Subject: [PATCH 0543/1206] [clang][analyzer] Check for label location bindings
 in `DereferenceChecker` (#91119)

Resolves #89264

Values should not be stored in addresses of labels, this throws a fatal
error when this happens.

---------

Co-authored-by: Balazs Benics <benicsbalazs@gmail.com>
---
 .../Checkers/DereferenceChecker.cpp               | 15 ++++++++++++++-
 .../StaticAnalyzer/Core/BugReporterVisitors.cpp   |  3 +++
 clang/test/Analysis/gh-issue-89185.c              | 15 +++++++--------
 3 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
index 1cebfbbee77d..0355eede75ea 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
@@ -31,11 +31,13 @@ class DereferenceChecker
     : public Checker< check::Location,
                       check::Bind,
                       EventDispatcher<ImplicitNullDerefEvent> > {
-  enum DerefKind { NullPointer, UndefinedPointerValue };
+  enum DerefKind { NullPointer, UndefinedPointerValue, AddressOfLabel };
 
   BugType BT_Null{this, "Dereference of null pointer", categories::LogicError};
   BugType BT_Undef{this, "Dereference of undefined pointer value",
                    categories::LogicError};
+  BugType BT_Label{this, "Dereference of the address of a label",
+                   categories::LogicError};
 
   void reportBug(DerefKind K, ProgramStateRef State, const Stmt *S,
                  CheckerContext &C) const;
@@ -167,6 +169,11 @@ void DereferenceChecker::reportBug(DerefKind K, ProgramStateRef State,
     DerefStr1 = " results in an undefined pointer dereference";
     DerefStr2 = " results in a dereference of an undefined pointer value";
     break;
+  case DerefKind::AddressOfLabel:
+    BT = &BT_Label;
+    DerefStr1 = " results in an undefined pointer dereference";
+    DerefStr2 = " results in a dereference of an address of a label";
+    break;
   };
 
   // Generate an error node.
@@ -287,6 +294,12 @@ void DereferenceChecker::checkBind(SVal L, SVal V, const Stmt *S,
   if (V.isUndef())
     return;
 
+  // One should never write to label addresses.
+  if (auto Label = L.getAs<loc::GotoLabel>()) {
+    reportBug(DerefKind::AddressOfLabel, C.getState(), S, C);
+    return;
+  }
+
   const MemRegion *MR = L.getAsRegion();
   const TypedValueRegion *TVR = dyn_cast_or_null<TypedValueRegion>(MR);
   if (!TVR)
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index 984755fa7e50..487a3bd16b67 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -113,6 +113,9 @@ const Expr *bugreporter::getDerefExpr(const Stmt *S) {
       // Pointer arithmetic: '*(x + 2)' -> 'x') etc.
       if (const Expr *Inner = peelOffPointerArithmetic(B)) {
         E = Inner;
+      } else if (B->isAssignmentOp()) {
+        // Follow LHS of assignments: '*p = 404' -> 'p'.
+        E = B->getLHS();
       } else {
         // Probably more arithmetic can be pattern-matched here,
         // but for now give up.
diff --git a/clang/test/Analysis/gh-issue-89185.c b/clang/test/Analysis/gh-issue-89185.c
index 8a907f198a5f..49526d2daa86 100644
--- a/clang/test/Analysis/gh-issue-89185.c
+++ b/clang/test/Analysis/gh-issue-89185.c
@@ -1,14 +1,13 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,debug.ExprInspection -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,debug.ExprInspection -analyzer-output text -verify %s 
 
-void clang_analyzer_dump(char);
-void clang_analyzer_dump_ptr(char*);
+void clang_analyzer_warnIfReached(void);
 
 // https://github.com/llvm/llvm-project/issues/89185
 void binding_to_label_loc() {
-  char *b = &&MyLabel;
+  char *b = &&MyLabel; // expected-note {{'b' initialized here}}
 MyLabel:
-  *b = 0; // no-crash
-  clang_analyzer_dump_ptr(b); // expected-warning {{&&MyLabel}}
-  clang_analyzer_dump(*b); // expected-warning {{Unknown}}
-  // FIXME: We should never reach here, as storing to a label is invalid.
+  *b = 0;
+  // expected-warning@-1 {{Dereference of the address of a label}}
+  // expected-note@-2    {{Dereference of the address of a label}}
+  clang_analyzer_warnIfReached(); // no-warning: Unreachable due to fatal error.
 }
-- 
GitLab


From 345f57df16af7e4fac3a321035e504b5d49206f4 Mon Sep 17 00:00:00 2001
From: Ivan Butygin <ivan.butygin@gmail.com>
Date: Mon, 13 May 2024 14:47:37 +0300
Subject: [PATCH 0544/1206] [mlir][arith] Overflow flags propagation in arith
 canonicalizations. (#91646)

---
 .../Dialect/Arith/IR/ArithCanonicalization.td |  35 +++---
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |   8 ++
 mlir/test/Dialect/Arith/canonicalize.mlir     | 119 ++++++++++++++++++
 3 files changed, 145 insertions(+), 17 deletions(-)

diff --git a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
index 02d05780a7ac..6d7ac2be951d 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
+++ b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
@@ -24,10 +24,10 @@ def SubIntAttrs : NativeCodeCall<"subIntegerAttrs($_builder, $0, $1, $2)">;
 // Multiply two integer attributes and create a new one with the result.
 def MulIntAttrs : NativeCodeCall<"mulIntegerAttrs($_builder, $0, $1, $2)">;
 
-// TODO: Canonicalizations currently doesn't take into account integer overflow
-// flags and always reset them to default (wraparound) which is safe but can
-// inhibit later optimizations. Individual patterns must be reviewed for
-// better handling of overflow flags.
+// Merge overflow flags from 2 ops, selecting the most conservative combination.
+def MergeOverflow : NativeCodeCall<"mergeOverflowFlags($0, $1)">;
+
+// Default overflow flag (all wraparounds allowed).
 defvar DefOverflow = ConstantEnumCase<Arith_IntegerOverflowAttr, "none">;
 
 class cast<string type> : NativeCodeCall<"::mlir::cast<" # type # ">($0)">;
@@ -45,7 +45,7 @@ def AddIAddConstant :
           (Arith_AddIOp $x, (ConstantLikeMatcher APIntAttr:$c0), $ovf1),
           (ConstantLikeMatcher APIntAttr:$c1), $ovf2),
         (Arith_AddIOp $x, (Arith_ConstantOp (AddIntAttrs $res, $c0, $c1)),
-            DefOverflow)>;
+            (MergeOverflow $ovf1, $ovf2))>;
 
 // addi(subi(x, c0), c1) -> addi(x, c1 - c0)
 def AddISubConstantRHS :
@@ -53,7 +53,7 @@ def AddISubConstantRHS :
           (Arith_SubIOp $x, (ConstantLikeMatcher APIntAttr:$c0), $ovf1),
           (ConstantLikeMatcher APIntAttr:$c1), $ovf2),
         (Arith_AddIOp $x, (Arith_ConstantOp (SubIntAttrs $res, $c1, $c0)),
-            DefOverflow)>;
+            (MergeOverflow $ovf1, $ovf2))>;
 
 // addi(subi(c0, x), c1) -> subi(c0 + c1, x)
 def AddISubConstantLHS :
@@ -61,7 +61,7 @@ def AddISubConstantLHS :
           (Arith_SubIOp (ConstantLikeMatcher APIntAttr:$c0), $x, $ovf1),
           (ConstantLikeMatcher APIntAttr:$c1), $ovf2),
         (Arith_SubIOp (Arith_ConstantOp (AddIntAttrs $res, $c0, $c1)), $x,
-            DefOverflow)>;
+            (MergeOverflow $ovf1, $ovf2))>;
 
 def IsScalarOrSplatNegativeOne :
     Constraint<And<[
@@ -73,7 +73,7 @@ def AddIMulNegativeOneRhs :
     Pat<(Arith_AddIOp
            $x,
            (Arith_MulIOp $y, (ConstantLikeMatcher AnyAttr:$c0), $ovf1), $ovf2),
-        (Arith_SubIOp $x, $y, DefOverflow),
+        (Arith_SubIOp $x, $y, DefOverflow), // TODO: overflow flags
         [(IsScalarOrSplatNegativeOne $c0)]>;
 
 // addi(muli(x, -1), y) -> subi(y, x)
@@ -81,7 +81,7 @@ def AddIMulNegativeOneLhs :
     Pat<(Arith_AddIOp
            (Arith_MulIOp $x, (ConstantLikeMatcher AnyAttr:$c0), $ovf1),
            $y, $ovf2),
-        (Arith_SubIOp $y, $x, DefOverflow),
+        (Arith_SubIOp $y, $x, DefOverflow), // TODO: overflow flags
         [(IsScalarOrSplatNegativeOne $c0)]>;
 
 // muli(muli(x, c0), c1) -> muli(x, c0 * c1)
@@ -90,7 +90,7 @@ def MulIMulIConstant :
           (Arith_MulIOp $x, (ConstantLikeMatcher APIntAttr:$c0), $ovf1),
           (ConstantLikeMatcher APIntAttr:$c1), $ovf2),
         (Arith_MulIOp $x, (Arith_ConstantOp (MulIntAttrs $res, $c0, $c1)),
-            DefOverflow)>;
+            (MergeOverflow $ovf1, $ovf2))>;
 
 //===----------------------------------------------------------------------===//
 // AddUIExtendedOp
@@ -113,7 +113,7 @@ def SubIRHSAddConstant :
           (Arith_AddIOp $x, (ConstantLikeMatcher APIntAttr:$c0), $ovf1),
           (ConstantLikeMatcher APIntAttr:$c1), $ovf2),
         (Arith_AddIOp $x, (Arith_ConstantOp (SubIntAttrs $res, $c0, $c1)),
-            DefOverflow)>;
+            DefOverflow)>; // TODO: overflow flags
 
 // subi(c1, addi(x, c0)) -> subi(c1 - c0, x)
 def SubILHSAddConstant :
@@ -121,7 +121,7 @@ def SubILHSAddConstant :
           (ConstantLikeMatcher APIntAttr:$c1),
           (Arith_AddIOp $x, (ConstantLikeMatcher APIntAttr:$c0), $ovf1), $ovf2),
         (Arith_SubIOp (Arith_ConstantOp (SubIntAttrs $res, $c1, $c0)), $x,
-            DefOverflow)>;
+            (MergeOverflow $ovf1, $ovf2))>;
 
 // subi(subi(x, c0), c1) -> subi(x, c0 + c1)
 def SubIRHSSubConstantRHS :
@@ -129,7 +129,7 @@ def SubIRHSSubConstantRHS :
           (Arith_SubIOp $x, (ConstantLikeMatcher APIntAttr:$c0), $ovf1),
           (ConstantLikeMatcher APIntAttr:$c1), $ovf2),
         (Arith_SubIOp $x, (Arith_ConstantOp (AddIntAttrs $res, $c0, $c1)),
-            DefOverflow)>;
+            (MergeOverflow $ovf1, $ovf2))>;
 
 // subi(subi(c0, x), c1) -> subi(c0 - c1, x)
 def SubIRHSSubConstantLHS :
@@ -137,7 +137,7 @@ def SubIRHSSubConstantLHS :
           (Arith_SubIOp (ConstantLikeMatcher APIntAttr:$c0), $x, $ovf1),
           (ConstantLikeMatcher APIntAttr:$c1), $ovf2),
         (Arith_SubIOp (Arith_ConstantOp (SubIntAttrs $res, $c0, $c1)), $x,
-            DefOverflow)>;
+            (MergeOverflow $ovf1, $ovf2))>;
 
 // subi(c1, subi(x, c0)) -> subi(c0 + c1, x)
 def SubILHSSubConstantRHS :
@@ -145,7 +145,7 @@ def SubILHSSubConstantRHS :
           (ConstantLikeMatcher APIntAttr:$c1),
           (Arith_SubIOp $x, (ConstantLikeMatcher APIntAttr:$c0), $ovf1), $ovf2),
         (Arith_SubIOp (Arith_ConstantOp (AddIntAttrs $res, $c0, $c1)), $x,
-            DefOverflow)>;
+            (MergeOverflow $ovf1, $ovf2))>;
 
 // subi(c1, subi(c0, x)) -> addi(x, c1 - c0)
 def SubILHSSubConstantLHS :
@@ -153,12 +153,13 @@ def SubILHSSubConstantLHS :
           (ConstantLikeMatcher APIntAttr:$c1),
           (Arith_SubIOp (ConstantLikeMatcher APIntAttr:$c0), $x, $ovf1), $ovf2),
         (Arith_AddIOp $x, (Arith_ConstantOp (SubIntAttrs $res, $c1, $c0)),
-            DefOverflow)>;
+            (MergeOverflow $ovf1, $ovf2))>;
 
 // subi(subi(a, b), a) -> subi(0, b)
 def SubISubILHSRHSLHS :
     Pat<(Arith_SubIOp:$res (Arith_SubIOp $x, $y, $ovf1), $x, $ovf2),
-        (Arith_SubIOp (Arith_ConstantOp (GetZeroAttr $y)), $y, DefOverflow)>;
+        (Arith_SubIOp (Arith_ConstantOp (GetZeroAttr $y)), $y,
+            (MergeOverflow $ovf1, $ovf2))>;
 
 //===----------------------------------------------------------------------===//
 // MulSIExtendedOp
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index a1568d0ebba3..a0b50251c6b6 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -64,6 +64,14 @@ static IntegerAttr mulIntegerAttrs(PatternRewriter &builder, Value res,
   return applyToIntegerAttrs(builder, res, lhs, rhs, std::multiplies<APInt>());
 }
 
+// Merge overflow flags from 2 ops, selecting the most conservative combination.
+static IntegerOverflowFlagsAttr
+mergeOverflowFlags(IntegerOverflowFlagsAttr val1,
+                   IntegerOverflowFlagsAttr val2) {
+  return IntegerOverflowFlagsAttr::get(val1.getContext(),
+                                       val1.getValue() & val2.getValue());
+}
+
 /// Invert an integer comparison predicate.
 arith::CmpIPredicate arith::invertPredicate(arith::CmpIPredicate pred) {
   switch (pred) {
diff --git a/mlir/test/Dialect/Arith/canonicalize.mlir b/mlir/test/Dialect/Arith/canonicalize.mlir
index f7ce2123a93c..e4f95bb0545a 100644
--- a/mlir/test/Dialect/Arith/canonicalize.mlir
+++ b/mlir/test/Dialect/Arith/canonicalize.mlir
@@ -833,6 +833,30 @@ func.func @tripleAddAdd(%arg0: index) -> index {
   return %add2 : index
 }
 
+// CHECK-LABEL: @tripleAddAddOvf1
+//       CHECK:   %[[cres:.+]] = arith.constant 59 : index
+//       CHECK:   %[[add:.+]] = arith.addi %arg0, %[[cres]] overflow<nsw, nuw> : index
+//       CHECK:   return %[[add]]
+func.func @tripleAddAddOvf1(%arg0: index) -> index {
+  %c17 = arith.constant 17 : index
+  %c42 = arith.constant 42 : index
+  %add1 = arith.addi %c17, %arg0 overflow<nsw, nuw> : index
+  %add2 = arith.addi %c42, %add1 overflow<nsw, nuw> : index
+  return %add2 : index
+}
+
+// CHECK-LABEL: @tripleAddAddOvf2
+//       CHECK:   %[[cres:.+]] = arith.constant 59 : index
+//       CHECK:   %[[add:.+]] = arith.addi %arg0, %[[cres]] : index
+//       CHECK:   return %[[add]]
+func.func @tripleAddAddOvf2(%arg0: index) -> index {
+  %c17 = arith.constant 17 : index
+  %c42 = arith.constant 42 : index
+  %add1 = arith.addi %c17, %arg0 overflow<nsw> : index
+  %add2 = arith.addi %c42, %add1 overflow<nuw> : index
+  return %add2 : index
+}
+
 // CHECK-LABEL: @tripleAddSub0
 //       CHECK:   %[[cres:.+]] = arith.constant 59 : index
 //       CHECK:   %[[add:.+]] = arith.subi %[[cres]], %arg0 : index
@@ -845,6 +869,18 @@ func.func @tripleAddSub0(%arg0: index) -> index {
   return %add2 : index
 }
 
+// CHECK-LABEL: @tripleAddSub0Ovf
+//       CHECK:   %[[cres:.+]] = arith.constant 59 : index
+//       CHECK:   %[[add:.+]] = arith.subi %[[cres]], %arg0 overflow<nsw, nuw> : index
+//       CHECK:   return %[[add]]
+func.func @tripleAddSub0Ovf(%arg0: index) -> index {
+  %c17 = arith.constant 17 : index
+  %c42 = arith.constant 42 : index
+  %add1 = arith.subi %c17, %arg0 overflow<nsw, nuw> : index
+  %add2 = arith.addi %c42, %add1 overflow<nsw, nuw> : index
+  return %add2 : index
+}
+
 // CHECK-LABEL: @tripleAddSub1
 //       CHECK:   %[[cres:.+]] = arith.constant 25 : index
 //       CHECK:   %[[add:.+]] = arith.addi %arg0, %[[cres]] : index
@@ -857,6 +893,18 @@ func.func @tripleAddSub1(%arg0: index) -> index {
   return %add2 : index
 }
 
+// CHECK-LABEL: @tripleAddSub1Ovf
+//       CHECK:   %[[cres:.+]] = arith.constant 25 : index
+//       CHECK:   %[[add:.+]] = arith.addi %arg0, %[[cres]] overflow<nsw, nuw> : index
+//       CHECK:   return %[[add]]
+func.func @tripleAddSub1Ovf(%arg0: index) -> index {
+  %c17 = arith.constant 17 : index
+  %c42 = arith.constant 42 : index
+  %add1 = arith.subi %arg0, %c17 overflow<nsw, nuw> : index
+  %add2 = arith.addi %c42, %add1 overflow<nsw, nuw> : index
+  return %add2 : index
+}
+
 // CHECK-LABEL: @tripleSubAdd0
 //       CHECK:   %[[cres:.+]] = arith.constant 25 : index
 //       CHECK:   %[[add:.+]] = arith.subi %[[cres]], %arg0 : index
@@ -869,6 +917,18 @@ func.func @tripleSubAdd0(%arg0: index) -> index {
   return %add2 : index
 }
 
+// CHECK-LABEL: @tripleSubAdd0Ovf
+//       CHECK:   %[[cres:.+]] = arith.constant 25 : index
+//       CHECK:   %[[add:.+]] = arith.subi %[[cres]], %arg0 overflow<nsw, nuw> : index
+//       CHECK:   return %[[add]]
+func.func @tripleSubAdd0Ovf(%arg0: index) -> index {
+  %c17 = arith.constant 17 : index
+  %c42 = arith.constant 42 : index
+  %add1 = arith.addi %c17, %arg0 overflow<nsw, nuw> : index
+  %add2 = arith.subi %c42, %add1 overflow<nsw, nuw> : index
+  return %add2 : index
+}
+
 // CHECK-LABEL: @tripleSubAdd1
 //       CHECK:   %[[cres:.+]] = arith.constant -25 : index
 //       CHECK:   %[[add:.+]] = arith.addi %arg0, %[[cres]] : index
@@ -891,6 +951,16 @@ func.func @subSub0(%arg0: index, %arg1: index) -> index {
   return %sub2 : index
 }
 
+// CHECK-LABEL: @subSub0Ovf
+//       CHECK:   %[[c0:.+]] = arith.constant 0 : index
+//       CHECK:   %[[add:.+]] = arith.subi %[[c0]], %arg1 overflow<nsw, nuw> : index
+//       CHECK:   return %[[add]]
+func.func @subSub0Ovf(%arg0: index, %arg1: index) -> index {
+  %sub1 = arith.subi %arg0, %arg1 overflow<nsw, nuw> : index
+  %sub2 = arith.subi %sub1, %arg0 overflow<nsw, nuw> : index
+  return %sub2 : index
+}
+
 // CHECK-LABEL: @tripleSubSub0
 //       CHECK:   %[[cres:.+]] = arith.constant 25 : index
 //       CHECK:   %[[add:.+]] = arith.addi %arg0, %[[cres]] : index
@@ -903,6 +973,19 @@ func.func @tripleSubSub0(%arg0: index) -> index {
   return %add2 : index
 }
 
+// CHECK-LABEL: @tripleSubSub0Ovf
+//       CHECK:   %[[cres:.+]] = arith.constant 25 : index
+//       CHECK:   %[[add:.+]] = arith.addi %arg0, %[[cres]] overflow<nsw, nuw> : index
+//       CHECK:   return %[[add]]
+func.func @tripleSubSub0Ovf(%arg0: index) -> index {
+  %c17 = arith.constant 17 : index
+  %c42 = arith.constant 42 : index
+  %add1 = arith.subi %c17, %arg0 overflow<nsw, nuw> : index
+  %add2 = arith.subi %c42, %add1 overflow<nsw, nuw> : index
+  return %add2 : index
+}
+
+
 // CHECK-LABEL: @tripleSubSub1
 //       CHECK:   %[[cres:.+]] = arith.constant -25 : index
 //       CHECK:   %[[add:.+]] = arith.subi %[[cres]], %arg0 : index
@@ -915,6 +998,18 @@ func.func @tripleSubSub1(%arg0: index) -> index {
   return %add2 : index
 }
 
+// CHECK-LABEL: @tripleSubSub1Ovf
+//       CHECK:   %[[cres:.+]] = arith.constant -25 : index
+//       CHECK:   %[[add:.+]] = arith.subi %[[cres]], %arg0 overflow<nsw, nuw> : index
+//       CHECK:   return %[[add]]
+func.func @tripleSubSub1Ovf(%arg0: index) -> index {
+  %c17 = arith.constant 17 : index
+  %c42 = arith.constant 42 : index
+  %add1 = arith.subi %c17, %arg0 overflow<nsw, nuw> : index
+  %add2 = arith.subi %add1, %c42 overflow<nsw, nuw> : index
+  return %add2 : index
+}
+
 // CHECK-LABEL: @tripleSubSub2
 //       CHECK:   %[[cres:.+]] = arith.constant 59 : index
 //       CHECK:   %[[add:.+]] = arith.subi %[[cres]], %arg0 : index
@@ -927,6 +1022,18 @@ func.func @tripleSubSub2(%arg0: index) -> index {
   return %add2 : index
 }
 
+// CHECK-LABEL: @tripleSubSub2Ovf
+//       CHECK:   %[[cres:.+]] = arith.constant 59 : index
+//       CHECK:   %[[add:.+]] = arith.subi %[[cres]], %arg0 overflow<nsw, nuw> : index
+//       CHECK:   return %[[add]]
+func.func @tripleSubSub2Ovf(%arg0: index) -> index {
+  %c17 = arith.constant 17 : index
+  %c42 = arith.constant 42 : index
+  %add1 = arith.subi %arg0, %c17 overflow<nsw, nuw> : index
+  %add2 = arith.subi %c42, %add1 overflow<nsw, nuw> : index
+  return %add2 : index
+}
+
 // CHECK-LABEL: @tripleSubSub3
 //       CHECK:   %[[cres:.+]] = arith.constant 59 : index
 //       CHECK:   %[[add:.+]] = arith.subi %arg0, %[[cres]] : index
@@ -939,6 +1046,18 @@ func.func @tripleSubSub3(%arg0: index) -> index {
   return %add2 : index
 }
 
+// CHECK-LABEL: @tripleSubSub3Ovf
+//       CHECK:   %[[cres:.+]] = arith.constant 59 : index
+//       CHECK:   %[[add:.+]] = arith.subi %arg0, %[[cres]] overflow<nsw, nuw> : index
+//       CHECK:   return %[[add]]
+func.func @tripleSubSub3Ovf(%arg0: index) -> index {
+  %c17 = arith.constant 17 : index
+  %c42 = arith.constant 42 : index
+  %add1 = arith.subi %arg0, %c17 overflow<nsw, nuw> : index
+  %add2 = arith.subi %add1, %c42 overflow<nsw, nuw> : index
+  return %add2 : index
+}
+
 // CHECK-LABEL: @subAdd1
 //  CHECK-NEXT:   return %arg0
 func.func @subAdd1(%arg0: index, %arg1 : index) -> index {
-- 
GitLab


From c5b0da9d83971d94d3b26105b1e42d3a3826ef1e Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Mon, 13 May 2024 13:49:09 +0200
Subject: [PATCH 0545/1206] InstCombine: Preserve inbounds in PointerReplacer
 (#91735)

This avoids spurious test changes in a future commit.
---
 .../InstCombine/InstCombineLoadStoreAlloca.cpp       |  1 +
 .../InstCombine/AMDGPU/memcpy-from-constant.ll       | 12 ++++++------
 llvm/test/Transforms/InstCombine/memcpy-addrspace.ll |  4 ++--
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index c70872c12917..344f3ec74522 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -388,6 +388,7 @@ void PointerReplacer::replace(Instruction *I) {
         GetElementPtrInst::Create(GEP->getSourceElementType(), V, Indices);
     IC.InsertNewInstWith(NewI, GEP->getIterator());
     NewI->takeName(GEP);
+    NewI->setIsInBounds(GEP->isInBounds());
     WorkMap[GEP] = NewI;
   } else if (auto *BC = dyn_cast<BitCastInst>(I)) {
     auto *V = getReplacement(BC->getOperand(0));
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll b/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll
index 2d0e3d2edd90..c14d61b51ad7 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll
@@ -10,7 +10,7 @@ target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:3
 define i8 @memcpy_constant_arg_ptr_to_alloca(ptr addrspace(4) noalias readonly align 4 dereferenceable(32) %arg, i32 %idx) {
 ; CHECK-LABEL: @memcpy_constant_arg_ptr_to_alloca(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[IDX:%.*]] to i64
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [32 x i8], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i8], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(4) [[GEP]], align 1
 ; CHECK-NEXT:    ret i8 [[LOAD]]
 ;
@@ -24,7 +24,7 @@ define i8 @memcpy_constant_arg_ptr_to_alloca(ptr addrspace(4) noalias readonly a
 define i8 @memcpy_constant_arg_ptr_to_alloca_load_metadata(ptr addrspace(4) noalias readonly align 4 dereferenceable(32) %arg, i32 %idx) {
 ; CHECK-LABEL: @memcpy_constant_arg_ptr_to_alloca_load_metadata(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[IDX:%.*]] to i64
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [32 x i8], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i8], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(4) [[GEP]], align 1, !noalias [[META0:![0-9]+]]
 ; CHECK-NEXT:    ret i8 [[LOAD]]
 ;
@@ -38,7 +38,7 @@ define i8 @memcpy_constant_arg_ptr_to_alloca_load_metadata(ptr addrspace(4) noal
 define i64 @memcpy_constant_arg_ptr_to_alloca_load_alignment(ptr addrspace(4) noalias readonly align 4 dereferenceable(256) %arg, i32 %idx) {
 ; CHECK-LABEL: @memcpy_constant_arg_ptr_to_alloca_load_alignment(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[IDX:%.*]] to i64
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [32 x i64], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i64], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i64, ptr addrspace(4) [[GEP]], align 16
 ; CHECK-NEXT:    ret i64 [[LOAD]]
 ;
@@ -68,7 +68,7 @@ define i64 @memcpy_constant_arg_ptr_to_alloca_load_atomic(ptr addrspace(4) noali
 define i8 @memmove_constant_arg_ptr_to_alloca(ptr addrspace(4) noalias readonly align 4 dereferenceable(32) %arg, i32 %idx) {
 ; CHECK-LABEL: @memmove_constant_arg_ptr_to_alloca(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[IDX:%.*]] to i64
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [32 x i8], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i8], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(4) [[GEP]], align 1
 ; CHECK-NEXT:    ret i8 [[LOAD]]
 ;
@@ -83,7 +83,7 @@ define i8 @memmove_constant_arg_ptr_to_alloca(ptr addrspace(4) noalias readonly
 define amdgpu_kernel void @memcpy_constant_byref_arg_ptr_to_alloca(ptr addrspace(4) noalias readonly align 4 byref([32 x i8]) %arg, ptr addrspace(1) %out, i32 %idx) {
 ; CHECK-LABEL: @memcpy_constant_byref_arg_ptr_to_alloca(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[IDX:%.*]] to i64
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [32 x i8], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i8], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(4) [[GEP]], align 1
 ; CHECK-NEXT:    store i8 [[LOAD]], ptr addrspace(1) [[OUT:%.*]], align 1
 ; CHECK-NEXT:    ret void
@@ -138,7 +138,7 @@ define amdgpu_kernel void @memcpy_constant_intrinsic_ptr_to_alloca(ptr addrspace
 define i8 @memcpy_constant_arg_ptr_to_alloca_addrspacecast_to_flat(ptr addrspace(4) noalias readonly align 4 dereferenceable(32) %arg, i32 %idx) {
 ; CHECK-LABEL: @memcpy_constant_arg_ptr_to_alloca_addrspacecast_to_flat(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[IDX:%.*]] to i64
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [32 x i8], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i8], ptr addrspace(4) [[ARG:%.*]], i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(4) [[GEP]], align 1
 ; CHECK-NEXT:    ret i8 [[LOAD]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/memcpy-addrspace.ll b/llvm/test/Transforms/InstCombine/memcpy-addrspace.ll
index d1543696bfc0..2ec3994f30fc 100644
--- a/llvm/test/Transforms/InstCombine/memcpy-addrspace.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy-addrspace.ll
@@ -6,7 +6,7 @@
 define void @test_load(ptr addrspace(1) %out, i64 %x) {
 ; CHECK-LABEL: @test_load(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr [8 x i32], ptr addrspace(2) @test.data, i64 0, i64 [[X:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [8 x i32], ptr addrspace(2) @test.data, i64 0, i64 [[X:%.*]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(2) [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT:%.*]], i64 [[X]]
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr addrspace(1) [[ARRAYIDX1]], align 4
@@ -25,7 +25,7 @@ entry:
 define void @test_load_bitcast_chain(ptr addrspace(1) %out, i64 %x) {
 ; CHECK-LABEL: @test_load_bitcast_chain(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i32, ptr addrspace(2) @test.data, i64 [[X:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr addrspace(2) @test.data, i64 [[X:%.*]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(2) [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT:%.*]], i64 [[X]]
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr addrspace(1) [[ARRAYIDX1]], align 4
-- 
GitLab


From 91d7ca904c601d181c431bffbf2773165de2fabd Mon Sep 17 00:00:00 2001
From: Orlando Cazalet-Hyams <orlando.hyams@sony.com>
Date: Mon, 13 May 2024 12:49:42 +0100
Subject: [PATCH 0546/1206] [DebugInfo] Remap extracted DIAssignIDs in
 hotcoldsplit (#91940)

Fix #91814

When instructions are extracted into a new function the `DIAssignID` metadata
uses and attachments need to be remapped so that the stores and assignment
markers don't link to stores and assignment markers in the original function.

This matches existing inlining behaviour for DIAssignIDs.
---
 llvm/include/llvm/IR/DebugInfo.h              |  4 ++
 llvm/lib/IR/DebugInfo.cpp                     | 24 +++++++++
 llvm/lib/Transforms/Utils/CodeExtractor.cpp   |  6 ++-
 llvm/lib/Transforms/Utils/InlineFunction.cpp  | 21 +-------
 .../assignment-tracking/X86/hotcoldsplit.ll   | 50 +++++++++++++++++++
 5 files changed, 84 insertions(+), 21 deletions(-)
 create mode 100644 llvm/test/DebugInfo/assignment-tracking/X86/hotcoldsplit.ll

diff --git a/llvm/include/llvm/IR/DebugInfo.h b/llvm/include/llvm/IR/DebugInfo.h
index 53cede5409e2..5b80218d6c5c 100644
--- a/llvm/include/llvm/IR/DebugInfo.h
+++ b/llvm/include/llvm/IR/DebugInfo.h
@@ -268,6 +268,10 @@ bool calculateFragmentIntersect(
     uint64_t SliceSizeInBits, const DbgVariableRecord *DVRAssign,
     std::optional<DIExpression::FragmentInfo> &Result);
 
+/// Replace DIAssignID uses and attachments with IDs from \p Map.
+/// If an ID is unmapped a new ID is generated and added to \p Map.
+void remapAssignID(DenseMap<DIAssignID *, DIAssignID *> &Map, Instruction &I);
+
 /// Helper struct for trackAssignments, below. We don't use the similar
 /// DebugVariable class because trackAssignments doesn't (yet?) understand
 /// partial variables (fragment info) as input and want to make that clear and
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 7976904b1fe9..4c3f37ceaaa4 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -2130,6 +2130,30 @@ bool at::calculateFragmentIntersect(
                                         SliceSizeInBits, DVRAssign, Result);
 }
 
+/// Update inlined instructions' DIAssignID metadata. We need to do this
+/// otherwise a function inlined more than once into the same function
+/// will cause DIAssignID to be shared by many instructions.
+void at::remapAssignID(DenseMap<DIAssignID *, DIAssignID *> &Map,
+                       Instruction &I) {
+  auto GetNewID = [&Map](Metadata *Old) {
+    DIAssignID *OldID = cast<DIAssignID>(Old);
+    if (DIAssignID *NewID = Map.lookup(OldID))
+      return NewID;
+    DIAssignID *NewID = DIAssignID::getDistinct(OldID->getContext());
+    Map[OldID] = NewID;
+    return NewID;
+  };
+  // If we find a DIAssignID attachment or use, replace it with a new version.
+  for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
+    if (DVR.isDbgAssign())
+      DVR.setAssignId(GetNewID(DVR.getAssignID()));
+  }
+  if (auto *ID = I.getMetadata(LLVMContext::MD_DIAssignID))
+    I.setMetadata(LLVMContext::MD_DIAssignID, GetNewID(ID));
+  else if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(&I))
+    DAI->setAssignId(GetNewID(DAI->getAssignID()));
+}
+
 /// Collect constant properies (base, size, offset) of \p StoreDest.
 /// Return std::nullopt if any properties are not constants or the
 /// offset from the base pointer is negative.
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 6988292ac715..f2672b8e9118 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -1678,8 +1678,9 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
     DVR->getMarker()->MarkedInstr->dropOneDbgRecord(DVR);
   DIB.finalizeSubprogram(NewSP);
 
-  // Fix up the scope information attached to the line locations in the new
-  // function.
+  // Fix up the scope information attached to the line locations and the
+  // debug assignment metadata in the new function.
+  DenseMap<DIAssignID *, DIAssignID *> AssignmentIDMap;
   for (Instruction &I : instructions(NewFunc)) {
     if (const DebugLoc &DL = I.getDebugLoc())
       I.setDebugLoc(
@@ -1695,6 +1696,7 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
       return MD;
     };
     updateLoopMetadataDebugLocations(I, updateLoopInfoLoc);
+    at::remapAssignID(AssignmentIDMap, I);
   }
   if (!TheCall.getDebugLoc())
     TheCall.setDebugLoc(DILocation::get(Ctx, 0, 0, OldSP));
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 48bb76eb85e3..82daaedaa0e8 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1888,29 +1888,12 @@ static void trackInlinedStores(Function::iterator Start, Function::iterator End,
 /// otherwise a function inlined more than once into the same function
 /// will cause DIAssignID to be shared by many instructions.
 static void fixupAssignments(Function::iterator Start, Function::iterator End) {
-  // Map {Old, New} metadata. Not used directly - use GetNewID.
   DenseMap<DIAssignID *, DIAssignID *> Map;
-  auto GetNewID = [&Map](Metadata *Old) {
-    DIAssignID *OldID = cast<DIAssignID>(Old);
-    if (DIAssignID *NewID = Map.lookup(OldID))
-      return NewID;
-    DIAssignID *NewID = DIAssignID::getDistinct(OldID->getContext());
-    Map[OldID] = NewID;
-    return NewID;
-  };
   // Loop over all the inlined instructions. If we find a DIAssignID
   // attachment or use, replace it with a new version.
   for (auto BBI = Start; BBI != End; ++BBI) {
-    for (Instruction &I : *BBI) {
-      for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
-        if (DVR.isDbgAssign())
-          DVR.setAssignId(GetNewID(DVR.getAssignID()));
-      }
-      if (auto *ID = I.getMetadata(LLVMContext::MD_DIAssignID))
-        I.setMetadata(LLVMContext::MD_DIAssignID, GetNewID(ID));
-      else if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(&I))
-        DAI->setAssignId(GetNewID(DAI->getAssignID()));
-    }
+    for (Instruction &I : *BBI)
+      at::remapAssignID(Map, I);
   }
 }
 #undef DEBUG_TYPE
diff --git a/llvm/test/DebugInfo/assignment-tracking/X86/hotcoldsplit.ll b/llvm/test/DebugInfo/assignment-tracking/X86/hotcoldsplit.ll
new file mode 100644
index 000000000000..f3faba7122af
--- /dev/null
+++ b/llvm/test/DebugInfo/assignment-tracking/X86/hotcoldsplit.ll
@@ -0,0 +1,50 @@
+; RUN: opt %s -passes=hotcoldsplit -S | FileCheck %s
+
+;; Check the extracted DIAssignID gets remapped.
+
+; CHECK-LABEL: define void @_foo()
+; CHECK: common.ret:
+; CHECK-NEXT: dbg.assign(metadata i64 0, metadata ![[#]], metadata !DIExpression(DW_OP_LLVM_fragment, 0, 64), metadata ![[ID1:[0-9]+]], {{.*}}, metadata !DIExpression())
+
+; CHECK-LABEL: define internal void @_foo.cold.1()
+; CHECK: store i64 0, ptr null, align 8, !DIAssignID ![[ID2:[0-9]+]]
+
+; CHECK-DAG: ![[ID1]] = distinct !DIAssignID()
+; CHECK-DAG: ![[ID2]] = distinct !DIAssignID()
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @_foo() !dbg !4 {
+entry:
+  br i1 false, label %if.then7, label %common.ret
+
+common.ret:                                       ; preds = %entry
+  call void @llvm.dbg.assign(metadata i64 0, metadata !7, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 64), metadata !12, metadata ptr null, metadata !DIExpression()), !dbg !13
+  ret void
+
+if.then7:                                         ; preds = %entry
+  %call21 = load i1, ptr null, align 4294967296
+  store i64 0, ptr null, align 8, !DIAssignID !12
+  unreachable
+}
+
+declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "file.cpp", directory: "foo")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "foo", linkageName: "_foo", scope: !5, file: !1, line: 425, type: !6, scopeLine: 425, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!5 = !DINamespace(name: "llvm", scope: null)
+!6 = distinct !DISubroutineType(types: !2)
+!7 = !DILocalVariable(name: "Path", scope: !4, file: !1, line: 436, type: !8)
+!8 = !DIDerivedType(tag: DW_TAG_typedef, name: "string", scope: !9, file: !1, line: 79, baseType: !10)
+!9 = !DINamespace(name: "std", scope: null)
+!10 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "basic_string<char, std::char_traits<char>, std::allocator<char> >", scope: !11, file: !1, line: 85, size: 256, flags: DIFlagTypePassByReference | DIFlagNonTrivial, elements: !2, templateParams: !2, identifier: "_ZTSNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE")
+!11 = !DINamespace(name: "__cxx11", scope: !9, exportSymbols: true)
+!12 = distinct !DIAssignID()
+!13 = !DILocation(line: 0, scope: !4)
-- 
GitLab


From 32939a16bcb084c2572f201ee42613413784cf7e Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 13 May 2024 06:51:12 -0500
Subject: [PATCH 0547/1206] [Offload][NFC] Remove unused compiler definition
 from CMake

---
 offload/src/CMakeLists.txt | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/offload/src/CMakeLists.txt b/offload/src/CMakeLists.txt
index 8fe6d19d83eb..b474f29ea0be 100644
--- a/offload/src/CMakeLists.txt
+++ b/offload/src/CMakeLists.txt
@@ -72,11 +72,6 @@ endforeach()
 target_compile_options(omptarget PUBLIC ${offload_compile_flags})
 target_link_options(omptarget PUBLIC ${offload_link_flags})
 
-list(TRANSFORM LIBOMPTARGET_PLUGINS_TO_LOAD PREPEND "\"libomptarget.rtl.")
-list(TRANSFORM LIBOMPTARGET_PLUGINS_TO_LOAD APPEND "\"")
-list(JOIN LIBOMPTARGET_PLUGINS_TO_LOAD "," ENABLED_OFFLOAD_PLUGINS)
-target_compile_definitions(omptarget PRIVATE ENABLED_OFFLOAD_PLUGINS=${ENABLED_OFFLOAD_PLUGINS})
-
 # libomptarget.so needs to be aware of where the plugins live as they
 # are now separated in the build directory.
 set_target_properties(omptarget PROPERTIES
-- 
GitLab


From bc17361c2baa0351f7f19b716fbe76bc9f99e903 Mon Sep 17 00:00:00 2001
From: Muhammad Omair Javaid <omair.javaid@linaro.org>
Date: Mon, 13 May 2024 16:50:19 +0500
Subject: [PATCH 0548/1206] [lldb][DWARF] Fix
 delayed-definition-die-searching.test for Windows

This is follow up fix on top of 9a7262c2601874e5aa64c5db19746770212d4b44

This fixes delayed-definition-die-searching.test to use -gdwarf. This is
required to explicitly select DWARF instead of PDB on windows.

Fixe LLDB build lldb-aarch64-windows:
https://lab.llvm.org/buildbot/#/builders/219/builds/11303
---
 .../SymbolFile/DWARF/delayed-definition-die-searching.test    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test b/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
index a2b36dadedd2..836fcd7b587b 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
+++ b/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
@@ -1,7 +1,7 @@
 # Test definition DIE searching is delayed until complete type is required.
 
 # RUN: split-file %s %t
-# RUN: %clangxx_host %t/main.cpp %t/t1_def.cpp -g -o %t.out
+# RUN: %clangxx_host %t/main.cpp %t/t1_def.cpp -gdwarf -o %t.out
 # RUN: %lldb -b %t.out -s %t/lldb.cmd | FileCheck %s
 
 # CHECK: (lldb) p v1
@@ -31,4 +31,4 @@ int main() {
 struct t1 { // this CU contains definition DIE for t1.
   int x;
 };
-t1 v2;
\ No newline at end of file
+t1 v2;
-- 
GitLab


From 710d95d1ecb4b6d69507cb910274ef3077ddc9c9 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Mon, 13 May 2024 15:55:20 +0400
Subject: [PATCH 0549/1206] [lldb] Fixed the test TestSettings when run with a
 remote target (#91915)

The setting `platform.module-cache-directory` is a local path on the
host. It cannot be set to a working directory from the remote target.
This test failed in case of Windows host and Linux target because of the
incompatible path. Use the local build dir instead.
---
 lldb/test/API/commands/settings/TestSettings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/test/API/commands/settings/TestSettings.py b/lldb/test/API/commands/settings/TestSettings.py
index 104a9f09788c..385acceb7a8b 100644
--- a/lldb/test/API/commands/settings/TestSettings.py
+++ b/lldb/test/API/commands/settings/TestSettings.py
@@ -953,7 +953,7 @@ class SettingsCommandTestCase(TestBase):
 
         # Test OptionValueFileSpec
         self.verify_setting_value_json(
-            "platform.module-cache-directory", self.get_process_working_directory()
+            "platform.module-cache-directory", self.getBuildDir()
         )
 
         # Test OptionValueArray
-- 
GitLab


From 7ce3dd49eb80816e3af52022ba2521b28a068c7b Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Mon, 13 May 2024 15:58:39 +0400
Subject: [PATCH 0550/1206] [lldb] Fixed the test TestQuoting (#91886)

os.path.join() uses the path separator of the host OS by default.
outfile_arg will be incorrect in case of Windows host and Linux target.
Use lldbutil.append_to_process_working_directory() instead.
---
 lldb/test/API/commands/settings/quoting/TestQuoting.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lldb/test/API/commands/settings/quoting/TestQuoting.py b/lldb/test/API/commands/settings/quoting/TestQuoting.py
index 393f4be3c824..60eeeead4e0a 100644
--- a/lldb/test/API/commands/settings/quoting/TestQuoting.py
+++ b/lldb/test/API/commands/settings/quoting/TestQuoting.py
@@ -51,9 +51,7 @@ class SettingsCommandTestCase(TestBase):
         outfile = self.getBuildArtifact(filename)
 
         if lldb.remote_platform:
-            outfile_arg = os.path.join(
-                lldb.remote_platform.GetWorkingDirectory(), filename
-            )
+            outfile_arg = lldbutil.append_to_process_working_directory(self, filename)
         else:
             outfile_arg = outfile
 
-- 
GitLab


From 4ff45eee36f003cda18a3654df4086e69dc245b5 Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Mon, 13 May 2024 13:59:24 +0200
Subject: [PATCH 0551/1206] [GlobalISel][KnownBits] Simplify G_CONSTANT
 handling (#91946)

We called getIConstantVRegVal which again queried MRI to get the VReg
def. We already have the def, so just get the CImm directly. It can't
fail.
---
 llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index c8199a42d15c..32d607cfd71a 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -256,10 +256,7 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
     break;
   }
   case TargetOpcode::G_CONSTANT: {
-    auto CstVal = getIConstantVRegVal(R, MRI);
-    if (!CstVal)
-      break;
-    Known = KnownBits::makeConstant(*CstVal);
+    Known = KnownBits::makeConstant(MI.getOperand(1).getCImm()->getValue());
     break;
   }
   case TargetOpcode::G_FRAME_INDEX: {
-- 
GitLab


From 8823abea6fb029bf24abd501e43844e3f5b9c444 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 10 May 2024 14:45:32 +0200
Subject: [PATCH 0552/1206] InstCombine: Simplify vector initialization

---
 llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index 344f3ec74522..537890d9025f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -382,8 +382,7 @@ void PointerReplacer::replace(Instruction *I) {
   } else if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
     auto *V = getReplacement(GEP->getPointerOperand());
     assert(V && "Operand not replaced");
-    SmallVector<Value *, 8> Indices;
-    Indices.append(GEP->idx_begin(), GEP->idx_end());
+    SmallVector<Value *, 8> Indices(GEP->indices());
     auto *NewI =
         GetElementPtrInst::Create(GEP->getSourceElementType(), V, Indices);
     IC.InsertNewInstWith(NewI, GEP->getIterator());
-- 
GitLab


From 3f0e1d4cf09b0c90abfb1d06a26cc4c85c1f9568 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 13 May 2024 13:04:22 +0100
Subject: [PATCH 0553/1206] [SCEV] Swap order of arguments to
 MatchBinaryAddToConst (NFCI). (#91945)

The argument order to MatchBinaryAddToConst doesn't match the comment
and also is counter-intuitive (passing RHS before LHS, C2 before C1).

This patch adjusts the order to be inline with the calls above, which
should be equivalent, but more natural:
https://alive2.llvm.org/ce/z/ZWGp-Z

PR: https://github.com/llvm/llvm-project/pull/91945
---
 llvm/lib/Analysis/ScalarEvolution.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 7dc5aa084f3c..254d79183a1e 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -11274,7 +11274,7 @@ bool ScalarEvolution::isKnownPredicateViaNoOverflow(ICmpInst::Predicate Pred,
     [[fallthrough]];
   case ICmpInst::ICMP_ULE:
     // (X + C1)<nuw> u<= (X + C2)<nuw> for C1 u<= C2.
-    if (MatchBinaryAddToConst(RHS, LHS, C2, C1, SCEV::FlagNUW) && C1.ule(C2))
+    if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ule(C2))
       return true;
 
     break;
@@ -11284,7 +11284,7 @@ bool ScalarEvolution::isKnownPredicateViaNoOverflow(ICmpInst::Predicate Pred,
     [[fallthrough]];
   case ICmpInst::ICMP_ULT:
     // (X + C1)<nuw> u< (X + C2)<nuw> if C1 u< C2.
-    if (MatchBinaryAddToConst(RHS, LHS, C2, C1, SCEV::FlagNUW) && C1.ult(C2))
+    if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ult(C2))
       return true;
     break;
   }
-- 
GitLab


From 19a62fbe00930d7eaa9f948c8dd26d58f5422c00 Mon Sep 17 00:00:00 2001
From: Oleg Shyshkov <shyshkov@google.com>
Date: Mon, 13 May 2024 14:08:25 +0200
Subject: [PATCH 0554/1206] [OpenMP][MLIR] Fix llvm::sort comparator. (#91947)

llvm::sort requires the comparator to return `false` for equal elements,
otherwise it triggers `Your comparator is not a valid strict-weak
ordering` assert.
---
 .../LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp   | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 0c3412666732..282e640d3aaa 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -2175,10 +2175,10 @@ getFirstOrLastMappedMemberPtr(mlir::omp::MapInfoOp mapInfo, bool first) {
             return !first;
         }
 
-        // iterated the entire list and couldn't make a decision, all elements
-        // were likely the same, return true for now similar to reaching the end
-        // of both and finding invalid indices.
-        return true;
+        // Iterated the entire list and couldn't make a decision, all elements
+        // were likely the same. Return false, since the sort comparator should
+        // return false for equal elements.
+        return false;
       });
 
     return llvm::cast<mlir::omp::MapInfoOp>(
-- 
GitLab


From 5e9401c2c21efcd55aae42d0b3d68034d344b08d Mon Sep 17 00:00:00 2001
From: Chris Warner <73851242+cwarner-8702@users.noreply.github.com>
Date: Mon, 13 May 2024 05:11:39 -0700
Subject: [PATCH 0555/1206] [clang-query] Load queries and matchers from file
 during REPL cycle (#90603)

The clang-query tool has the ability to execute or pre-load queries from
a file when the tool is launched, but doesn't have the ability to do the
same from the interactive REPL prompt. Because the prompt also doesn't
seem to allow multi-line matchers, this can make prototyping and
iterating on more complicated matchers difficult.

Supporting a dynamic load at REPL time allows the cost of reading the
compilation database and building the AST to be imposed just once, and
allows faster prototyping.
---
 clang-tools-extra/clang-query/Query.cpp       | 22 +++++++++++++++++++
 clang-tools-extra/clang-query/Query.h         | 18 ++++++++++++++-
 clang-tools-extra/clang-query/QueryParser.cpp | 10 +++++++--
 .../clang-query/tool/ClangQuery.cpp           | 18 ++-------------
 clang-tools-extra/docs/ReleaseNotes.rst       |  4 +++-
 .../test/clang-query/Inputs/empty.script      |  1 +
 .../test/clang-query/Inputs/file.script       |  1 +
 .../clang-query/Inputs/runtime_file.script    |  5 +++++
 clang-tools-extra/test/clang-query/errors.c   |  2 ++
 .../test/clang-query/file-empty.c             |  2 ++
 .../test/clang-query/file-query.c             | 14 ++++++++++++
 .../unittests/clang-query/QueryParserTest.cpp |  4 +++-
 12 files changed, 80 insertions(+), 21 deletions(-)
 create mode 100644 clang-tools-extra/test/clang-query/Inputs/empty.script
 create mode 100644 clang-tools-extra/test/clang-query/Inputs/file.script
 create mode 100644 clang-tools-extra/test/clang-query/Inputs/runtime_file.script
 create mode 100644 clang-tools-extra/test/clang-query/file-empty.c
 create mode 100644 clang-tools-extra/test/clang-query/file-query.c

diff --git a/clang-tools-extra/clang-query/Query.cpp b/clang-tools-extra/clang-query/Query.cpp
index c436d6fa9498..9d5807a52fa8 100644
--- a/clang-tools-extra/clang-query/Query.cpp
+++ b/clang-tools-extra/clang-query/Query.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Query.h"
+#include "QueryParser.h"
 #include "QuerySession.h"
 #include "clang/AST/ASTDumper.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
@@ -281,5 +282,26 @@ const QueryKind SetQueryKind<bool>::value;
 const QueryKind SetQueryKind<OutputKind>::value;
 #endif
 
+bool FileQuery::run(llvm::raw_ostream &OS, QuerySession &QS) const {
+  auto Buffer = llvm::MemoryBuffer::getFile(StringRef{File}.trim());
+  if (!Buffer) {
+    if (Prefix.has_value())
+      llvm::errs() << *Prefix << ": ";
+    llvm::errs() << "cannot open " << File << ": "
+                 << Buffer.getError().message() << "\n";
+    return false;
+  }
+
+  StringRef FileContentRef(Buffer.get()->getBuffer());
+
+  while (!FileContentRef.empty()) {
+    QueryRef Q = QueryParser::parse(FileContentRef, QS);
+    if (!Q->run(llvm::outs(), QS))
+      return false;
+    FileContentRef = Q->RemainingContent;
+  }
+  return true;
+}
+
 } // namespace query
 } // namespace clang
diff --git a/clang-tools-extra/clang-query/Query.h b/clang-tools-extra/clang-query/Query.h
index 7aefa6bb5ee0..7242479633c2 100644
--- a/clang-tools-extra/clang-query/Query.h
+++ b/clang-tools-extra/clang-query/Query.h
@@ -30,7 +30,8 @@ enum QueryKind {
   QK_SetTraversalKind,
   QK_EnableOutputKind,
   QK_DisableOutputKind,
-  QK_Quit
+  QK_Quit,
+  QK_File
 };
 
 class QuerySession;
@@ -188,6 +189,21 @@ struct DisableOutputQuery : SetNonExclusiveOutputQuery {
   }
 };
 
+struct FileQuery : Query {
+  FileQuery(StringRef File, StringRef Prefix = StringRef())
+      : Query(QK_File), File(File),
+        Prefix(!Prefix.empty() ? std::optional<std::string>(Prefix)
+                               : std::nullopt) {}
+
+  bool run(llvm::raw_ostream &OS, QuerySession &QS) const override;
+
+  static bool classof(const Query *Q) { return Q->Kind == QK_File; }
+
+private:
+  std::string File;
+  std::optional<std::string> Prefix;
+};
+
 } // namespace query
 } // namespace clang
 
diff --git a/clang-tools-extra/clang-query/QueryParser.cpp b/clang-tools-extra/clang-query/QueryParser.cpp
index 162acc1a598d..85a442bdd7de 100644
--- a/clang-tools-extra/clang-query/QueryParser.cpp
+++ b/clang-tools-extra/clang-query/QueryParser.cpp
@@ -183,7 +183,8 @@ enum ParsedQueryKind {
   PQK_Unlet,
   PQK_Quit,
   PQK_Enable,
-  PQK_Disable
+  PQK_Disable,
+  PQK_File
 };
 
 enum ParsedQueryVariable {
@@ -222,12 +223,14 @@ QueryRef QueryParser::doParse() {
                               .Case("let", PQK_Let)
                               .Case("m", PQK_Match, /*IsCompletion=*/false)
                               .Case("match", PQK_Match)
-                              .Case("q", PQK_Quit,  /*IsCompletion=*/false)
+                              .Case("q", PQK_Quit, /*IsCompletion=*/false)
                               .Case("quit", PQK_Quit)
                               .Case("set", PQK_Set)
                               .Case("enable", PQK_Enable)
                               .Case("disable", PQK_Disable)
                               .Case("unlet", PQK_Unlet)
+                              .Case("f", PQK_File, /*IsCompletion=*/false)
+                              .Case("file", PQK_File)
                               .Default(PQK_Invalid);
 
   switch (QKind) {
@@ -351,6 +354,9 @@ QueryRef QueryParser::doParse() {
     return endQuery(new LetQuery(Name, VariantValue()));
   }
 
+  case PQK_File:
+    return new FileQuery(Line);
+
   case PQK_Invalid:
     return new InvalidQuery("unknown command: " + CommandStr);
   }
diff --git a/clang-tools-extra/clang-query/tool/ClangQuery.cpp b/clang-tools-extra/clang-query/tool/ClangQuery.cpp
index da7ac2701448..a2de7a2dced8 100644
--- a/clang-tools-extra/clang-query/tool/ClangQuery.cpp
+++ b/clang-tools-extra/clang-query/tool/ClangQuery.cpp
@@ -74,22 +74,8 @@ static cl::opt<std::string> PreloadFile(
 
 bool runCommandsInFile(const char *ExeName, std::string const &FileName,
                        QuerySession &QS) {
-  auto Buffer = llvm::MemoryBuffer::getFile(FileName);
-  if (!Buffer) {
-    llvm::errs() << ExeName << ": cannot open " << FileName << ": "
-                 << Buffer.getError().message() << "\n";
-    return true;
-  }
-
-  StringRef FileContentRef(Buffer.get()->getBuffer());
-
-  while (!FileContentRef.empty()) {
-    QueryRef Q = QueryParser::parse(FileContentRef, QS);
-    if (!Q->run(llvm::outs(), QS))
-      return true;
-    FileContentRef = Q->RemainingContent;
-  }
-  return false;
+  FileQuery Query(FileName, ExeName);
+  return !Query.run(llvm::errs(), QS);
 }
 
 int main(int argc, const char **argv) {
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 6a2b8d3b6ded..fc976ce3a33d 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -90,7 +90,9 @@ Improvements to clang-doc
 Improvements to clang-query
 ---------------------------
 
-The improvements are...
+- Added the `file` command to dynamically load a list of commands and matchers
+  from an external file, allowing the cost of reading the compilation database
+  and building the AST to be imposed just once for faster prototyping.
 
 Improvements to clang-rename
 ----------------------------
diff --git a/clang-tools-extra/test/clang-query/Inputs/empty.script b/clang-tools-extra/test/clang-query/Inputs/empty.script
new file mode 100644
index 000000000000..3c30abd1ae5d
--- /dev/null
+++ b/clang-tools-extra/test/clang-query/Inputs/empty.script
@@ -0,0 +1 @@
+# This file intentionally has no queries
diff --git a/clang-tools-extra/test/clang-query/Inputs/file.script b/clang-tools-extra/test/clang-query/Inputs/file.script
new file mode 100644
index 000000000000..b58e7bbc24bf
--- /dev/null
+++ b/clang-tools-extra/test/clang-query/Inputs/file.script
@@ -0,0 +1 @@
+f DIRECTORY/runtime_file.script
diff --git a/clang-tools-extra/test/clang-query/Inputs/runtime_file.script b/clang-tools-extra/test/clang-query/Inputs/runtime_file.script
new file mode 100644
index 000000000000..714d7f03b1bf
--- /dev/null
+++ b/clang-tools-extra/test/clang-query/Inputs/runtime_file.script
@@ -0,0 +1,5 @@
+set bind-root false
+
+l func functionDecl(hasName("bar"))
+m func.bind("f")
+m varDecl().bind("v")
\ No newline at end of file
diff --git a/clang-tools-extra/test/clang-query/errors.c b/clang-tools-extra/test/clang-query/errors.c
index bbb742125744..3b9059ab0257 100644
--- a/clang-tools-extra/test/clang-query/errors.c
+++ b/clang-tools-extra/test/clang-query/errors.c
@@ -1,10 +1,12 @@
 // RUN: not clang-query -c foo -c bar %s -- | FileCheck %s
 // RUN: not clang-query -f %S/Inputs/foo.script %s -- | FileCheck %s
 // RUN: not clang-query -f %S/Inputs/nonexistent.script %s -- 2>&1 | FileCheck --check-prefix=CHECK-NONEXISTENT %s
+// RUN: not clang-query -c 'file %S/Inputs/nonexistent.script' %s -- 2>&1 | FileCheck --check-prefix=CHECK-NONEXISTENT-FILEQUERY %s
 // RUN: not clang-query -c foo -f foo %s -- 2>&1 | FileCheck --check-prefix=CHECK-BOTH %s
 
 // CHECK: unknown command: foo
 // CHECK-NOT: unknown command: bar
 
 // CHECK-NONEXISTENT: cannot open {{.*}}nonexistent.script
+// CHECK-NONEXISTENT-FILEQUERY: cannot open {{.*}}nonexistent.script
 // CHECK-BOTH: cannot specify both -c and -f
diff --git a/clang-tools-extra/test/clang-query/file-empty.c b/clang-tools-extra/test/clang-query/file-empty.c
new file mode 100644
index 000000000000..15137c57e915
--- /dev/null
+++ b/clang-tools-extra/test/clang-query/file-empty.c
@@ -0,0 +1,2 @@
+// RUN: clang-query -c 'file %S/Inputs/empty.script' %s --
+// COM: no output expected; nothing to CHECK
diff --git a/clang-tools-extra/test/clang-query/file-query.c b/clang-tools-extra/test/clang-query/file-query.c
new file mode 100644
index 000000000000..10a44e7aaccf
--- /dev/null
+++ b/clang-tools-extra/test/clang-query/file-query.c
@@ -0,0 +1,14 @@
+// RUN: rm -rf %/t
+// RUN: mkdir %/t
+// RUN: cp %/S/Inputs/file.script %/t/file.script
+// RUN: cp %/S/Inputs/runtime_file.script %/t/runtime_file.script
+// Need to embed the correct temp path in the actual JSON-RPC requests.
+// RUN: sed -e "s|DIRECTORY|%/t|" %/t/file.script > %/t/file.script.temp
+
+// RUN: clang-query -c 'file %/t/file.script.temp' %s -- | FileCheck %s
+
+// CHECK: file-query.c:11:1: note: "f" binds here
+void bar(void) {}
+
+// CHECK: file-query.c:14:1: note: "v" binds here
+int baz{1};
diff --git a/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp b/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp
index 06b0d7b36590..b561e2bb9833 100644
--- a/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp
+++ b/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp
@@ -197,7 +197,7 @@ TEST_F(QueryParserTest, Comment) {
 TEST_F(QueryParserTest, Complete) {
   std::vector<llvm::LineEditor::Completion> Comps =
       QueryParser::complete("", 0, QS);
-  ASSERT_EQ(8u, Comps.size());
+  ASSERT_EQ(9u, Comps.size());
   EXPECT_EQ("help ", Comps[0].TypedText);
   EXPECT_EQ("help", Comps[0].DisplayText);
   EXPECT_EQ("let ", Comps[1].TypedText);
@@ -214,6 +214,8 @@ TEST_F(QueryParserTest, Complete) {
   EXPECT_EQ("disable", Comps[6].DisplayText);
   EXPECT_EQ("unlet ", Comps[7].TypedText);
   EXPECT_EQ("unlet", Comps[7].DisplayText);
+  EXPECT_EQ("file ", Comps[8].TypedText);
+  EXPECT_EQ("file", Comps[8].DisplayText);
 
   Comps = QueryParser::complete("set o", 5, QS);
   ASSERT_EQ(1u, Comps.size());
-- 
GitLab


From e76b257483e6c6743de0fa6eca4d0cc60e08385d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project@meinersbur.de>
Date: Mon, 13 May 2024 14:31:39 +0200
Subject: [PATCH 0556/1206] [Clang][OpenMP][Tile] Ensure AST node uniqueness.
 (#91325)

One of the constraints of an AST is that every node object must appear
at most once, hence we define lamdas that create a new AST node at every
use.
---
 clang/lib/Sema/SemaOpenMP.cpp | 75 +++++++++++++++++++++++------------
 1 file changed, 50 insertions(+), 25 deletions(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index cf5447f223d4..fff4c7350f0f 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15109,6 +15109,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
                                                 SourceLocation StartLoc,
                                                 SourceLocation EndLoc) {
   ASTContext &Context = getASTContext();
+  Scope *CurScope = SemaRef.getCurScope();
+
   auto SizesClauses =
       OMPExecutableDirective::getClausesOfKind<OMPSizesClause>(Clauses);
   if (SizesClauses.empty()) {
@@ -15137,6 +15139,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
                                     NumLoops, AStmt, nullptr, nullptr);
 
   SmallVector<Decl *, 4> PreInits;
+  CaptureVars CopyTransformer(SemaRef);
 
   // Create iteration variables for the generated loops.
   SmallVector<VarDecl *, 4> FloorIndVars;
@@ -15200,19 +15203,30 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     Expr *NumIterations = LoopHelper.NumIterations;
     auto *OrigCntVar = cast<DeclRefExpr>(LoopHelper.Counters[0]);
     QualType CntTy = OrigCntVar->getType();
-    Expr *DimTileSize = SizesClause->getSizesRefs()[I];
-    Scope *CurScope = SemaRef.getCurScope();
 
-    // Commonly used variables.
-    DeclRefExpr *TileIV = buildDeclRefExpr(SemaRef, TileIndVars[I], CntTy,
-                                           OrigCntVar->getExprLoc());
-    DeclRefExpr *FloorIV = buildDeclRefExpr(SemaRef, FloorIndVars[I], CntTy,
-                                            OrigCntVar->getExprLoc());
+    // Commonly used variables. One of the constraints of an AST is that every
+    // node object must appear at most once, hence we define lamdas that create
+    // a new AST node at every use.
+    auto MakeDimTileSize = [&SemaRef = this->SemaRef, &CopyTransformer, I,
+                            SizesClause]() -> Expr * {
+      Expr *DimTileSize = SizesClause->getSizesRefs()[I];
+      return AssertSuccess(CopyTransformer.TransformExpr(DimTileSize));
+    };
+    auto MakeTileIVRef = [&SemaRef = this->SemaRef, &TileIndVars, I, CntTy,
+                          OrigCntVar]() {
+      return buildDeclRefExpr(SemaRef, TileIndVars[I], CntTy,
+                              OrigCntVar->getExprLoc());
+    };
+    auto MakeFloorIVRef = [&SemaRef = this->SemaRef, &FloorIndVars, I, CntTy,
+                           OrigCntVar]() {
+      return buildDeclRefExpr(SemaRef, FloorIndVars[I], CntTy,
+                              OrigCntVar->getExprLoc());
+    };
 
     // For init-statement: auto .tile.iv = .floor.iv
-    SemaRef.AddInitializerToDecl(TileIndVars[I],
-                                 SemaRef.DefaultLvalueConversion(FloorIV).get(),
-                                 /*DirectInit=*/false);
+    SemaRef.AddInitializerToDecl(
+        TileIndVars[I], SemaRef.DefaultLvalueConversion(MakeFloorIVRef()).get(),
+        /*DirectInit=*/false);
     Decl *CounterDecl = TileIndVars[I];
     StmtResult InitStmt = new (Context)
         DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
@@ -15220,10 +15234,11 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     if (!InitStmt.isUsable())
       return StmtError();
 
-    // For cond-expression: .tile.iv < min(.floor.iv + DimTileSize,
-    // NumIterations)
-    ExprResult EndOfTile = SemaRef.BuildBinOp(
-        CurScope, LoopHelper.Cond->getExprLoc(), BO_Add, FloorIV, DimTileSize);
+    // For cond-expression:
+    //   .tile.iv < min(.floor.iv + DimTileSize, NumIterations)
+    ExprResult EndOfTile =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
+                           MakeFloorIVRef(), MakeDimTileSize());
     if (!EndOfTile.isUsable())
       return StmtError();
     ExprResult IsPartialTile =
@@ -15238,25 +15253,28 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
       return StmtError();
     ExprResult CondExpr =
         SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                           TileIV, MinTileAndIterSpace.get());
+                           MakeTileIVRef(), MinTileAndIterSpace.get());
     if (!CondExpr.isUsable())
       return StmtError();
 
     // For incr-statement: ++.tile.iv
     ExprResult IncrStmt = SemaRef.BuildUnaryOp(
-        CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, TileIV);
+        CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, MakeTileIVRef());
     if (!IncrStmt.isUsable())
       return StmtError();
 
     // Statements to set the original iteration variable's value from the
     // logical iteration number.
     // Generated for loop is:
+    // \code
     // Original_for_init;
-    // for (auto .tile.iv = .floor.iv; .tile.iv < min(.floor.iv + DimTileSize,
-    // NumIterations); ++.tile.iv) {
+    // for (auto .tile.iv = .floor.iv;
+    //      .tile.iv < min(.floor.iv + DimTileSize, NumIterations);
+    //      ++.tile.iv) {
     //   Original_Body;
     //   Original_counter_update;
     // }
+    // \endcode
     // FIXME: If the innermost body is an loop itself, inserting these
     // statements stops it being recognized  as a perfectly nested loop (e.g.
     // for applying tiling again). If this is the case, sink the expressions
@@ -15278,12 +15296,18 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     Expr *NumIterations = LoopHelper.NumIterations;
     DeclRefExpr *OrigCntVar = cast<DeclRefExpr>(LoopHelper.Counters[0]);
     QualType CntTy = OrigCntVar->getType();
-    Expr *DimTileSize = SizesClause->getSizesRefs()[I];
-    Scope *CurScope = SemaRef.getCurScope();
 
     // Commonly used variables.
-    DeclRefExpr *FloorIV = buildDeclRefExpr(SemaRef, FloorIndVars[I], CntTy,
-                                            OrigCntVar->getExprLoc());
+    auto MakeDimTileSize = [&SemaRef = this->SemaRef, &CopyTransformer, I,
+                            SizesClause]() -> Expr * {
+      Expr *DimTileSize = SizesClause->getSizesRefs()[I];
+      return AssertSuccess(CopyTransformer.TransformExpr(DimTileSize));
+    };
+    auto MakeFloorIVRef = [&SemaRef = this->SemaRef, &FloorIndVars, I, CntTy,
+                           OrigCntVar]() {
+      return buildDeclRefExpr(SemaRef, FloorIndVars[I], CntTy,
+                              OrigCntVar->getExprLoc());
+    };
 
     // For init-statement: auto .floor.iv = 0
     SemaRef.AddInitializerToDecl(
@@ -15298,15 +15322,16 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
       return StmtError();
 
     // For cond-expression: .floor.iv < NumIterations
-    ExprResult CondExpr = SemaRef.BuildBinOp(
-        CurScope, LoopHelper.Cond->getExprLoc(), BO_LT, FloorIV, NumIterations);
+    ExprResult CondExpr =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                           MakeFloorIVRef(), NumIterations);
     if (!CondExpr.isUsable())
       return StmtError();
 
     // For incr-statement: .floor.iv += DimTileSize
     ExprResult IncrStmt =
         SemaRef.BuildBinOp(CurScope, LoopHelper.Inc->getExprLoc(), BO_AddAssign,
-                           FloorIV, DimTileSize);
+                           MakeFloorIVRef(), MakeDimTileSize());
     if (!IncrStmt.isUsable())
       return StmtError();
 
-- 
GitLab


From 061db17a3075e34c55ebd463d06e16771e398e04 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 13 May 2024 13:40:27 +0100
Subject: [PATCH 0557/1206] Fix MSVC "signed/unsigned mismatch" warning. NFC.

---
 llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index 6533e8281631..1e33c2729e5d 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -1650,7 +1650,8 @@ DIE &DwarfUnit::constructMemberDIE(DIE &Buffer, const DIDerivedType *DT) {
         addUInt(MemberDie, dwarf::DW_AT_byte_size, std::nullopt, FieldSize / 8);
       addUInt(MemberDie, dwarf::DW_AT_bit_size, std::nullopt, Size);
 
-      assert(DT->getOffsetInBits() <= std::numeric_limits<int64_t>::max());
+      assert(DT->getOffsetInBits() <=
+             (uint64_t)std::numeric_limits<int64_t>::max());
       int64_t Offset = DT->getOffsetInBits();
       // We can't use DT->getAlignInBits() here: AlignInBits for member type
       // is non-zero if and only if alignment was forced (e.g. _Alignas()),
-- 
GitLab


From 1a4b113a41266b94fe217e5fe90d91db15d2356b Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 13 May 2024 13:44:32 +0100
Subject: [PATCH 0558/1206] [CostModel][X86] getCastInstrCost - update cost
 tables to support CostKinds

Add TypeConversionCostKindTblEntry to hold the costs kinds and update the cast tables to take the existing default codesize/latency/sizelatency values (I'll update these values in future commits).

I've moved AdjustCost to the end of the function to ensure we don't accidentally use it, apart from when we fallback to default cost calculations.
---
 .../lib/Target/X86/X86TargetTransformInfo.cpp | 1554 +++++++++--------
 1 file changed, 787 insertions(+), 767 deletions(-)

diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 0a0bc7b32e87..ac66144aeaae 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -99,6 +99,7 @@ struct CostKindCosts {
   }
 };
 using CostKindTblEntry = CostTblEntryT<CostKindCosts>;
+using TypeConversionCostKindTblEntry = TypeConversionCostTblEntryT<CostKindCosts>;
 
 TargetTransformInfo::PopcntSupportKind
 X86TTIImpl::getPopcntSupport(unsigned TyWidth) {
@@ -2138,811 +2139,803 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
 
-  // TODO: Allow non-throughput costs that aren't binary.
-  auto AdjustCost = [&CostKind](InstructionCost Cost,
-                                InstructionCost N = 1) -> InstructionCost {
-    if (CostKind != TTI::TCK_RecipThroughput)
-      return Cost == 0 ? 0 : N;
-    return Cost * N;
-  };
-
   // The cost tables include both specific, custom (non-legal) src/dst type
   // conversions and generic, legalized types. We test for customs first, before
   // falling back to legalization.
   // FIXME: Need a better design of the cost table to handle non-simple types of
   // potential massive combinations (elem_num x src_type x dst_type).
-  static const TypeConversionCostTblEntry AVX512BWConversionTbl[] {
-    { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v32i8, 1 },
-    { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v32i8, 1 },
+  static const TypeConversionCostKindTblEntry AVX512BWConversionTbl[]{
+    { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v32i8,  { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v32i8,  { 1, 1, 1, 1 } },
 
     // Mask sign extend has an instruction.
-    { ISD::SIGN_EXTEND, MVT::v2i8,   MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v2i16,  MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i8,   MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i16,  MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i8,   MVT::v8i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v8i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v8i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v16i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v32i8,  MVT::v32i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v32i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v64i8,  MVT::v64i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v64i1,  1 },
+    { ISD::SIGN_EXTEND, MVT::v2i8,   MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v2i16,  MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i8,   MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i16,  MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i8,   MVT::v8i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v8i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v8i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v16i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v32i8,  MVT::v32i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v32i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v64i8,  MVT::v64i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v64i1,  { 1, 1, 1, 1 } },
 
     // Mask zero extend is a sext + shift.
-    { ISD::ZERO_EXTEND, MVT::v2i8,   MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v2i16,  MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v4i8,   MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v4i16,  MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v8i8,   MVT::v8i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v8i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v8i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v16i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v32i8,  MVT::v32i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v32i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v64i8,  MVT::v64i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v64i1,  2 },
-
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i8,   2 },
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v16i8,  2 },
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i16,  2 },
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v8i16,  2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i8,   2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v16i8,  2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i16,  2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v8i16,  2 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i8,   2 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v16i8,  2 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i16,  2 },
-    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i8,  2 },
-    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i16, 2 },
-    { ISD::TRUNCATE,    MVT::v32i1,  MVT::v32i8,  2 },
-    { ISD::TRUNCATE,    MVT::v32i1,  MVT::v32i16, 2 },
-    { ISD::TRUNCATE,    MVT::v64i1,  MVT::v64i8,  2 },
-    { ISD::TRUNCATE,    MVT::v64i1,  MVT::v32i16, 2 },
-
-    { ISD::TRUNCATE,    MVT::v32i8,  MVT::v32i16, 2 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i16, 2 }, // widen to zmm
-    { ISD::TRUNCATE,    MVT::v2i8,   MVT::v2i16,  2 }, // vpmovwb
-    { ISD::TRUNCATE,    MVT::v4i8,   MVT::v4i16,  2 }, // vpmovwb
-    { ISD::TRUNCATE,    MVT::v8i8,   MVT::v8i16,  2 }, // vpmovwb
+    { ISD::ZERO_EXTEND, MVT::v2i8,   MVT::v2i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v2i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v2i16,  MVT::v2i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v2i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i8,   MVT::v4i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v4i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i16,  MVT::v4i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v4i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i8,   MVT::v8i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v8i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v8i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v16i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v32i8,  MVT::v32i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v32i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v64i8,  MVT::v64i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v64i1,  { 2, 1, 1, 1 } },
+
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i8,   { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i16,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i8,   { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i16,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i8,   { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i16, { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v32i1,  MVT::v32i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v32i1,  MVT::v32i16, { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v64i1,  MVT::v64i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v64i1,  MVT::v32i16, { 2, 1, 1, 1 } },
+
+    { ISD::TRUNCATE,    MVT::v32i8,  MVT::v32i16, { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i16, { 2, 1, 1, 1 } }, // widen to zmm
+    { ISD::TRUNCATE,    MVT::v2i8,   MVT::v2i16,  { 2, 1, 1, 1 } }, // vpmovwb
+    { ISD::TRUNCATE,    MVT::v4i8,   MVT::v4i16,  { 2, 1, 1, 1 } }, // vpmovwb
+    { ISD::TRUNCATE,    MVT::v8i8,   MVT::v8i16,  { 2, 1, 1, 1 } }, // vpmovwb
   };
 
-  static const TypeConversionCostTblEntry AVX512DQConversionTbl[] = {
+  static const TypeConversionCostKindTblEntry AVX512DQConversionTbl[] = {
     // Mask sign extend has an instruction.
-    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v16i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i1,  1 },
+    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v16i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i1,  { 1, 1, 1, 1 } },
 
     // Mask zero extend is a sext + shift.
-    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v16i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i1,  2 },
-
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i64,  2 },
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v4i32,  2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i32,  2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i64,  2 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i32,  2 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i64,  2 },
-    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i32, 2 },
-    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v8i64,  2 },
-
-    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i64,  1 },
-    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i64,  1 },
-
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i64,  1 },
-    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i64,  1 },
-
-    { ISD::FP_TO_SINT,  MVT::v8i64,  MVT::v8f32,  1 },
-    { ISD::FP_TO_SINT,  MVT::v8i64,  MVT::v8f64,  1 },
-
-    { ISD::FP_TO_UINT,  MVT::v8i64,  MVT::v8f32,  1 },
-    { ISD::FP_TO_UINT,  MVT::v8i64,  MVT::v8f64,  1 },
+    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v2i1,   { 2, 1, 1, 1, } },
+    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v2i1,   { 2, 1, 1, 1, } },
+    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v4i1,   { 2, 1, 1, 1, } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   { 2, 1, 1, 1, } },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   { 2, 1, 1, 1, } },
+    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v16i1,  { 2, 1, 1, 1, } },
+    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i1,   { 2, 1, 1, 1, } },
+    { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i1,  { 2, 1, 1, 1, } },
+
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i64,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v4i32,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i32,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i64,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i32,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i64,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i32, { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v8i64,  { 2, 1, 1, 1 } },
+
+    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i64,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i64,  { 1, 1, 1, 1 } },
+
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i64,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i64,  { 1, 1, 1, 1 } },
+
+    { ISD::FP_TO_SINT,  MVT::v8i64,  MVT::v8f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i64,  MVT::v8f64,  { 1, 1, 1, 1 } },
+
+    { ISD::FP_TO_UINT,  MVT::v8i64,  MVT::v8f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i64,  MVT::v8f64,  { 1, 1, 1, 1 } },
   };
 
   // TODO: For AVX512DQ + AVX512VL, we also have cheap casts for 128-bit and
   // 256-bit wide vectors.
 
-  static const TypeConversionCostTblEntry AVX512FConversionTbl[] = {
-    { ISD::FP_EXTEND, MVT::v8f64,   MVT::v8f32,  1 },
-    { ISD::FP_EXTEND, MVT::v8f64,   MVT::v16f32, 3 },
-    { ISD::FP_EXTEND, MVT::v16f64,  MVT::v16f32, 4 }, // 2*vcvtps2pd+vextractf64x4
-    { ISD::FP_ROUND,  MVT::v8f32,   MVT::v8f64,  1 },
-
-    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i8,   3 }, // sext+vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i8,   3 }, // sext+vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i8,   3 }, // sext+vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v16i8,  3 }, // sext+vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i16,  3 }, // sext+vpsllq+vptestmq
-    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i16,  3 }, // sext+vpsllq+vptestmq
-    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i16,  3 }, // sext+vpsllq+vptestmq
-    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v16i16, 3 }, // sext+vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i32,  2 }, // zmm vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i32,  2 }, // zmm vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i32,  2 }, // zmm vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v16i32, 2 }, // vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i64,  2 }, // zmm vpsllq+vptestmq
-    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i64,  2 }, // zmm vpsllq+vptestmq
-    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i64,  2 }, // vpsllq+vptestmq
-    { ISD::TRUNCATE,  MVT::v2i8,    MVT::v2i32,  2 }, // vpmovdb
-    { ISD::TRUNCATE,  MVT::v4i8,    MVT::v4i32,  2 }, // vpmovdb
-    { ISD::TRUNCATE,  MVT::v16i8,   MVT::v16i32, 2 }, // vpmovdb
-    { ISD::TRUNCATE,  MVT::v32i8,   MVT::v16i32, 2 }, // vpmovdb
-    { ISD::TRUNCATE,  MVT::v64i8,   MVT::v16i32, 2 }, // vpmovdb
-    { ISD::TRUNCATE,  MVT::v16i16,  MVT::v16i32, 2 }, // vpmovdw
-    { ISD::TRUNCATE,  MVT::v32i16,  MVT::v16i32, 2 }, // vpmovdw
-    { ISD::TRUNCATE,  MVT::v2i8,    MVT::v2i64,  2 }, // vpmovqb
-    { ISD::TRUNCATE,  MVT::v2i16,   MVT::v2i64,  1 }, // vpshufb
-    { ISD::TRUNCATE,  MVT::v8i8,    MVT::v8i64,  2 }, // vpmovqb
-    { ISD::TRUNCATE,  MVT::v16i8,   MVT::v8i64,  2 }, // vpmovqb
-    { ISD::TRUNCATE,  MVT::v32i8,   MVT::v8i64,  2 }, // vpmovqb
-    { ISD::TRUNCATE,  MVT::v64i8,   MVT::v8i64,  2 }, // vpmovqb
-    { ISD::TRUNCATE,  MVT::v8i16,   MVT::v8i64,  2 }, // vpmovqw
-    { ISD::TRUNCATE,  MVT::v16i16,  MVT::v8i64,  2 }, // vpmovqw
-    { ISD::TRUNCATE,  MVT::v32i16,  MVT::v8i64,  2 }, // vpmovqw
-    { ISD::TRUNCATE,  MVT::v8i32,   MVT::v8i64,  1 }, // vpmovqd
-    { ISD::TRUNCATE,  MVT::v4i32,   MVT::v4i64,  1 }, // zmm vpmovqd
-    { ISD::TRUNCATE,  MVT::v16i8,   MVT::v16i64, 5 },// 2*vpmovqd+concat+vpmovdb
-
-    { ISD::TRUNCATE,  MVT::v16i8,  MVT::v16i16,  3 }, // extend to v16i32
-    { ISD::TRUNCATE,  MVT::v32i8,  MVT::v32i16,  8 },
-    { ISD::TRUNCATE,  MVT::v64i8,  MVT::v32i16,  8 },
+  static const TypeConversionCostKindTblEntry AVX512FConversionTbl[] = {
+    { ISD::FP_EXTEND, MVT::v8f64,   MVT::v8f32,   { 1, 1, 1, 1 } },
+    { ISD::FP_EXTEND, MVT::v8f64,   MVT::v16f32,  { 3, 1, 1, 1 } },
+    { ISD::FP_EXTEND, MVT::v16f64,  MVT::v16f32,  { 4, 1, 1, 1 } }, // 2*vcvtps2pd+vextractf64x4
+    { ISD::FP_ROUND,  MVT::v8f32,   MVT::v8f64,   { 1, 1, 1, 1 } },
+
+    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i8,    { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i8,    { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i8,    { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v16i8,   { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i16,   { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
+    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i16,   { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
+    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i16,   { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
+    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v16i16,  { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i32,   { 2, 1, 1, 1 } }, // zmm vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i32,   { 2, 1, 1, 1 } }, // zmm vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i32,   { 2, 1, 1, 1 } }, // zmm vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v16i32,  { 2, 1, 1, 1 } }, // vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i64,   { 2, 1, 1, 1 } }, // zmm vpsllq+vptestmq
+    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i64,   { 2, 1, 1, 1 } }, // zmm vpsllq+vptestmq
+    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i64,   { 2, 1, 1, 1 } }, // vpsllq+vptestmq
+    { ISD::TRUNCATE,  MVT::v2i8,    MVT::v2i32,   { 2, 1, 1, 1 } }, // vpmovdb
+    { ISD::TRUNCATE,  MVT::v4i8,    MVT::v4i32,   { 2, 1, 1, 1 } }, // vpmovdb
+    { ISD::TRUNCATE,  MVT::v16i8,   MVT::v16i32,  { 2, 1, 1, 1 } }, // vpmovdb
+    { ISD::TRUNCATE,  MVT::v32i8,   MVT::v16i32,  { 2, 1, 1, 1 } }, // vpmovdb
+    { ISD::TRUNCATE,  MVT::v64i8,   MVT::v16i32,  { 2, 1, 1, 1 } }, // vpmovdb
+    { ISD::TRUNCATE,  MVT::v16i16,  MVT::v16i32,  { 2, 1, 1, 1 } }, // vpmovdw
+    { ISD::TRUNCATE,  MVT::v32i16,  MVT::v16i32,  { 2, 1, 1, 1 } }, // vpmovdw
+    { ISD::TRUNCATE,  MVT::v2i8,    MVT::v2i64,   { 2, 1, 1, 1 } }, // vpmovqb
+    { ISD::TRUNCATE,  MVT::v2i16,   MVT::v2i64,   { 1, 1, 1, 1 } }, // vpshufb
+    { ISD::TRUNCATE,  MVT::v8i8,    MVT::v8i64,   { 2, 1, 1, 1 } }, // vpmovqb
+    { ISD::TRUNCATE,  MVT::v16i8,   MVT::v8i64,   { 2, 1, 1, 1 } }, // vpmovqb
+    { ISD::TRUNCATE,  MVT::v32i8,   MVT::v8i64,   { 2, 1, 1, 1 } }, // vpmovqb
+    { ISD::TRUNCATE,  MVT::v64i8,   MVT::v8i64,   { 2, 1, 1, 1 } }, // vpmovqb
+    { ISD::TRUNCATE,  MVT::v8i16,   MVT::v8i64,   { 2, 1, 1, 1 } }, // vpmovqw
+    { ISD::TRUNCATE,  MVT::v16i16,  MVT::v8i64,   { 2, 1, 1, 1 } }, // vpmovqw
+    { ISD::TRUNCATE,  MVT::v32i16,  MVT::v8i64,   { 2, 1, 1, 1 } }, // vpmovqw
+    { ISD::TRUNCATE,  MVT::v8i32,   MVT::v8i64,   { 1, 1, 1, 1 } }, // vpmovqd
+    { ISD::TRUNCATE,  MVT::v4i32,   MVT::v4i64,   { 1, 1, 1, 1 } }, // zmm vpmovqd
+    { ISD::TRUNCATE,  MVT::v16i8,   MVT::v16i64,  { 5, 1, 1, 1 } },// 2*vpmovqd+concat+vpmovdb
+
+    { ISD::TRUNCATE,  MVT::v16i8,  MVT::v16i16,   { 3, 1, 1, 1 } }, // extend to v16i32
+    { ISD::TRUNCATE,  MVT::v32i8,  MVT::v32i16,   { 8, 1, 1, 1 } },
+    { ISD::TRUNCATE,  MVT::v64i8,  MVT::v32i16,   { 8, 1, 1, 1 } },
 
     // Sign extend is zmm vpternlogd+vptruncdb.
     // Zero extend is zmm broadcast load+vptruncdw.
-    { ISD::SIGN_EXTEND, MVT::v2i8,   MVT::v2i1,   3 },
-    { ISD::ZERO_EXTEND, MVT::v2i8,   MVT::v2i1,   4 },
-    { ISD::SIGN_EXTEND, MVT::v4i8,   MVT::v4i1,   3 },
-    { ISD::ZERO_EXTEND, MVT::v4i8,   MVT::v4i1,   4 },
-    { ISD::SIGN_EXTEND, MVT::v8i8,   MVT::v8i1,   3 },
-    { ISD::ZERO_EXTEND, MVT::v8i8,   MVT::v8i1,   4 },
-    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v16i1,  3 },
-    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v16i1,  4 },
+    { ISD::SIGN_EXTEND, MVT::v2i8,   MVT::v2i1,   { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v2i8,   MVT::v2i1,   { 4, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i8,   MVT::v4i1,   { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i8,   MVT::v4i1,   { 4, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i8,   MVT::v8i1,   { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i8,   MVT::v8i1,   { 4, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v16i1,  { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v16i1,  { 4, 1, 1, 1 } },
 
     // Sign extend is zmm vpternlogd+vptruncdw.
     // Zero extend is zmm vpternlogd+vptruncdw+vpsrlw.
-    { ISD::SIGN_EXTEND, MVT::v2i16,  MVT::v2i1,   3 },
-    { ISD::ZERO_EXTEND, MVT::v2i16,  MVT::v2i1,   4 },
-    { ISD::SIGN_EXTEND, MVT::v4i16,  MVT::v4i1,   3 },
-    { ISD::ZERO_EXTEND, MVT::v4i16,  MVT::v4i1,   4 },
-    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v8i1,   3 },
-    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v8i1,   4 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1,  3 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1,  4 },
-
-    { ISD::SIGN_EXTEND, MVT::v2i32,  MVT::v2i1,   1 }, // zmm vpternlogd
-    { ISD::ZERO_EXTEND, MVT::v2i32,  MVT::v2i1,   2 }, // zmm vpternlogd+psrld
-    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v4i1,   1 }, // zmm vpternlogd
-    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v4i1,   2 }, // zmm vpternlogd+psrld
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   1 }, // zmm vpternlogd
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   2 }, // zmm vpternlogd+psrld
-    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v2i1,   1 }, // zmm vpternlogq
-    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v2i1,   2 }, // zmm vpternlogq+psrlq
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   1 }, // zmm vpternlogq
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   2 }, // zmm vpternlogq+psrlq
-
-    { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i1,  1 }, // vpternlogd
-    { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i1,  2 }, // vpternlogd+psrld
-    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i1,   1 }, // vpternlogq
-    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i1,   2 }, // vpternlogq+psrlq
-
-    { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8,  1 },
-    { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8,  1 },
-    { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i16, 1 },
-    { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i16, 1 },
-    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i8,   1 },
-    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i8,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i16,  1 },
-    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i16,  1 },
-    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i32,  1 },
-    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i32,  1 },
-
-    { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v32i8,  3 }, // FIXME: May not be right
-    { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v32i8,  3 }, // FIXME: May not be right
-
-    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i1,   4 },
-    { ISD::SINT_TO_FP,  MVT::v16f32, MVT::v16i1,  3 },
-    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v16i8,  2 },
-    { ISD::SINT_TO_FP,  MVT::v16f32, MVT::v16i8,  1 },
-    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i16,  2 },
-    { ISD::SINT_TO_FP,  MVT::v16f32, MVT::v16i16, 1 },
-    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  1 },
-    { ISD::SINT_TO_FP,  MVT::v16f32, MVT::v16i32, 1 },
-
-    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i1,   4 },
-    { ISD::UINT_TO_FP,  MVT::v16f32, MVT::v16i1,  3 },
-    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v16i8,  2 },
-    { ISD::UINT_TO_FP,  MVT::v16f32, MVT::v16i8,  1 },
-    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i16,  2 },
-    { ISD::UINT_TO_FP,  MVT::v16f32, MVT::v16i16, 1 },
-    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  1 },
-    { ISD::UINT_TO_FP,  MVT::v16f32, MVT::v16i32, 1 },
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i64, 26 },
-    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i64,  5 },
-
-    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v16f32, 2 },
-    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v16f64, 7 },
-    { ISD::FP_TO_SINT,  MVT::v32i8,  MVT::v32f64,15 },
-    { ISD::FP_TO_SINT,  MVT::v64i8,  MVT::v64f32,11 },
-    { ISD::FP_TO_SINT,  MVT::v64i8,  MVT::v64f64,31 },
-    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v8f64,  3 },
-    { ISD::FP_TO_SINT,  MVT::v16i16, MVT::v16f64, 7 },
-    { ISD::FP_TO_SINT,  MVT::v32i16, MVT::v32f32, 5 },
-    { ISD::FP_TO_SINT,  MVT::v32i16, MVT::v32f64,15 },
-    { ISD::FP_TO_SINT,  MVT::v8i32,  MVT::v8f64,  1 },
-    { ISD::FP_TO_SINT,  MVT::v16i32, MVT::v16f64, 3 },
-
-    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v8f64,  1 },
-    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v8f64,  3 },
-    { ISD::FP_TO_UINT,  MVT::v8i8,   MVT::v8f64,  3 },
-    { ISD::FP_TO_UINT,  MVT::v16i32, MVT::v16f32, 1 },
-    { ISD::FP_TO_UINT,  MVT::v16i16, MVT::v16f32, 3 },
-    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v16f32, 3 },
+    { ISD::SIGN_EXTEND, MVT::v2i16,  MVT::v2i1,   { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v2i16,  MVT::v2i1,   { 4, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i16,  MVT::v4i1,   { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i16,  MVT::v4i1,   { 4, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v8i1,   { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v8i1,   { 4, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1,  { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1,  { 4, 1, 1, 1 } },
+
+    { ISD::SIGN_EXTEND, MVT::v2i32,  MVT::v2i1,   { 1, 1, 1, 1 } }, // zmm vpternlogd
+    { ISD::ZERO_EXTEND, MVT::v2i32,  MVT::v2i1,   { 2, 1, 1, 1 } }, // zmm vpternlogd+psrld
+    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v4i1,   { 1, 1, 1, 1 } }, // zmm vpternlogd
+    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v4i1,   { 2, 1, 1, 1 } }, // zmm vpternlogd+psrld
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   { 1, 1, 1, 1 } }, // zmm vpternlogd
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   { 2, 1, 1, 1 } }, // zmm vpternlogd+psrld
+    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v2i1,   { 1, 1, 1, 1 } }, // zmm vpternlogq
+    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v2i1,   { 2, 1, 1, 1 } }, // zmm vpternlogq+psrlq
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   { 1, 1, 1, 1 } }, // zmm vpternlogq
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   { 2, 1, 1, 1 } }, // zmm vpternlogq+psrlq
+
+    { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i1,  { 1, 1, 1, 1 } }, // vpternlogd
+    { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i1,  { 2, 1, 1, 1 } }, // vpternlogd+psrld
+    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i1,   { 1, 1, 1, 1 } }, // vpternlogq
+    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i1,   { 2, 1, 1, 1 } }, // vpternlogq+psrlq
+
+    { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i16, { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i16, { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i8,   { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i8,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i32,  { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i32,  { 1, 1, 1, 1 } },
+
+    { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v32i8,  { 3, 1, 1, 1 } }, // FIXME: May not be right
+    { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v32i8,  { 3, 1, 1, 1 } }, // FIXME: May not be right
+
+    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i1,   { 4, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v16f32, MVT::v16i1,  { 3, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v16f32, MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v16f32, MVT::v16i16, { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v16f32, MVT::v16i32, { 1, 1, 1, 1 } },
+
+    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i1,   { 4, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v16f32, MVT::v16i1,  { 3, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v16f32, MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v16f32, MVT::v16i16, { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v16f32, MVT::v16i32, { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i64,  {26, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i64,  { 5, 1, 1, 1 } },
+
+    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v16f32, { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v16f64, { 7, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v32i8,  MVT::v32f64, {15, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v64i8,  MVT::v64f32, {11, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v64i8,  MVT::v64f64, {31, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v8f64,  { 3, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v16i16, MVT::v16f64, { 7, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v32i16, MVT::v32f32, { 5, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v32i16, MVT::v32f64, {15, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i32,  MVT::v8f64,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v16i32, MVT::v16f64, { 3, 1, 1, 1 } },
+
+    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v8f64,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v8f64,  { 3, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i8,   MVT::v8f64,  { 3, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v16i32, MVT::v16f32, { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v16i16, MVT::v16f32, { 3, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v16f32, { 3, 1, 1, 1 } },
   };
 
-  static const TypeConversionCostTblEntry AVX512BWVLConversionTbl[] {
+  static const TypeConversionCostKindTblEntry AVX512BWVLConversionTbl[] {
     // Mask sign extend has an instruction.
-    { ISD::SIGN_EXTEND, MVT::v2i8,   MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v2i16,  MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i16,  MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i8,   MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i8,   MVT::v8i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v8i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v8i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v16i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v32i8,  MVT::v32i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v32i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v32i8,  MVT::v64i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v64i1,  1 },
+    { ISD::SIGN_EXTEND, MVT::v2i8,   MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v2i16,  MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i16,  MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i8,   MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i8,   MVT::v8i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v8i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v8i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v16i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v32i8,  MVT::v32i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v32i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v32i8,  MVT::v64i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v64i1,  { 1, 1, 1, 1 } },
 
     // Mask zero extend is a sext + shift.
-    { ISD::ZERO_EXTEND, MVT::v2i8,   MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v2i16,  MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v4i8,   MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v4i16,  MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v8i8,   MVT::v8i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v8i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v8i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v16i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v32i8,  MVT::v32i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v32i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v32i8,  MVT::v64i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v64i1,  2 },
-
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i8,   2 },
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v16i8,  2 },
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i16,  2 },
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v8i16,  2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i8,   2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v16i8,  2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i16,  2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v8i16,  2 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i8,   2 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v16i8,  2 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i16,  2 },
-    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i8,  2 },
-    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i16, 2 },
-    { ISD::TRUNCATE,    MVT::v32i1,  MVT::v32i8,  2 },
-    { ISD::TRUNCATE,    MVT::v32i1,  MVT::v16i16, 2 },
-    { ISD::TRUNCATE,    MVT::v64i1,  MVT::v32i8,  2 },
-    { ISD::TRUNCATE,    MVT::v64i1,  MVT::v16i16, 2 },
-
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i16, 2 },
+    { ISD::ZERO_EXTEND, MVT::v2i8,   MVT::v2i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v2i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v2i16,  MVT::v2i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v2i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i8,   MVT::v4i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v4i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i16,  MVT::v4i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v4i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i8,   MVT::v8i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v8i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v8i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v16i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v32i8,  MVT::v32i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v32i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v32i8,  MVT::v64i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v64i1,  { 2, 1, 1, 1 } },
+
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i8,   { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i16,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i8,   { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i16,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i8,   { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i16, { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v32i1,  MVT::v32i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v32i1,  MVT::v16i16, { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v64i1,  MVT::v32i8,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v64i1,  MVT::v16i16, { 2, 1, 1, 1 } },
+
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i16, { 2, 1, 1, 1 } },
   };
 
-  static const TypeConversionCostTblEntry AVX512DQVLConversionTbl[] = {
+  static const TypeConversionCostKindTblEntry AVX512DQVLConversionTbl[] = {
     // Mask sign extend has an instruction.
-    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v2i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v16i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v8i1,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v16i1,  1 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   1 },
+    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v2i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v16i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v8i1,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v16i1,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   { 1, 1, 1, 1 } },
 
     // Mask zero extend is a sext + shift.
-    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v2i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v16i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v8i1,   2 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v16i1,  2 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   2 },
-
-    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v4i64,  2 },
-    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v8i32,  2 },
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i64,  2 },
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v4i32,  2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i32,  2 },
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i64,  2 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v4i64,  2 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i32,  2 },
-
-    { ISD::SINT_TO_FP,  MVT::v2f32,  MVT::v2i64,  1 },
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  1 },
-    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v4i64,  1 },
-    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v4i64,  1 },
-
-    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i64,  1 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  1 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i64,  1 },
-    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i64,  1 },
-
-    { ISD::FP_TO_SINT,  MVT::v2i64,  MVT::v4f32,  1 },
-    { ISD::FP_TO_SINT,  MVT::v4i64,  MVT::v4f32,  1 },
-    { ISD::FP_TO_SINT,  MVT::v2i64,  MVT::v2f64,  1 },
-    { ISD::FP_TO_SINT,  MVT::v4i64,  MVT::v4f64,  1 },
-
-    { ISD::FP_TO_UINT,  MVT::v2i64,  MVT::v4f32,  1 },
-    { ISD::FP_TO_UINT,  MVT::v4i64,  MVT::v4f32,  1 },
-    { ISD::FP_TO_UINT,  MVT::v2i64,  MVT::v2f64,  1 },
-    { ISD::FP_TO_UINT,  MVT::v4i64,  MVT::v4f64,  1 },
+    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v2i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v2i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v4i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v16i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v8i1,   { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v16i1,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   { 2, 1, 1, 1 } },
+
+    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v4i64,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v8i32,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i64,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v4i32,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i32,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i64,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v4i64,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i32,  { 2, 1, 1, 1 } },
+
+    { ISD::SINT_TO_FP,  MVT::v2f32,  MVT::v2i64,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v4i64,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v4i64,  { 1, 1, 1, 1 } },
+
+    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i64,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i64,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i64,  { 1, 1, 1, 1 } },
+
+    { ISD::FP_TO_SINT,  MVT::v2i64,  MVT::v4f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v4i64,  MVT::v4f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v2i64,  MVT::v2f64,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v4i64,  MVT::v4f64,  { 1, 1, 1, 1 } },
+
+    { ISD::FP_TO_UINT,  MVT::v2i64,  MVT::v4f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i64,  MVT::v4f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v2i64,  MVT::v2f64,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i64,  MVT::v4f64,  { 1, 1, 1, 1 } },
   };
 
-  static const TypeConversionCostTblEntry AVX512VLConversionTbl[] = {
-    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i8,   3 }, // sext+vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i8,   3 }, // sext+vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i8,   3 }, // sext+vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v16i8,  8 }, // split+2*v8i8
-    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i16,  3 }, // sext+vpsllq+vptestmq
-    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i16,  3 }, // sext+vpsllq+vptestmq
-    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i16,  3 }, // sext+vpsllq+vptestmq
-    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v16i16, 8 }, // split+2*v8i16
-    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i32,  2 }, // vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i32,  2 }, // vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i32,  2 }, // vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v8i32,  2 }, // vpslld+vptestmd
-    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i64,  2 }, // vpsllq+vptestmq
-    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i64,  2 }, // vpsllq+vptestmq
-    { ISD::TRUNCATE,  MVT::v4i32,   MVT::v4i64,  1 }, // vpmovqd
-    { ISD::TRUNCATE,  MVT::v4i8,    MVT::v4i64,  2 }, // vpmovqb
-    { ISD::TRUNCATE,  MVT::v4i16,   MVT::v4i64,  2 }, // vpmovqw
-    { ISD::TRUNCATE,  MVT::v8i8,    MVT::v8i32,  2 }, // vpmovwb
+  static const TypeConversionCostKindTblEntry AVX512VLConversionTbl[] = {
+    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i8,    { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i8,    { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i8,    { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v16i8,   { 8, 1, 1, 1 } }, // split+2*v8i8
+    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i16,   { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
+    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i16,   { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
+    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i16,   { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
+    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v16i16,  { 8, 1, 1, 1 } }, // split+2*v8i16
+    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i32,   { 2, 1, 1, 1 } }, // vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i32,   { 2, 1, 1, 1 } }, // vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v8i1,    MVT::v8i32,   { 2, 1, 1, 1 } }, // vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v16i1,   MVT::v8i32,   { 2, 1, 1, 1 } }, // vpslld+vptestmd
+    { ISD::TRUNCATE,  MVT::v2i1,    MVT::v2i64,   { 2, 1, 1, 1 } }, // vpsllq+vptestmq
+    { ISD::TRUNCATE,  MVT::v4i1,    MVT::v4i64,   { 2, 1, 1, 1 } }, // vpsllq+vptestmq
+    { ISD::TRUNCATE,  MVT::v4i32,   MVT::v4i64,   { 1, 1, 1, 1 } }, // vpmovqd
+    { ISD::TRUNCATE,  MVT::v4i8,    MVT::v4i64,   { 2, 1, 1, 1 } }, // vpmovqb
+    { ISD::TRUNCATE,  MVT::v4i16,   MVT::v4i64,   { 2, 1, 1, 1 } }, // vpmovqw
+    { ISD::TRUNCATE,  MVT::v8i8,    MVT::v8i32,   { 2, 1, 1, 1 } }, // vpmovwb
 
     // sign extend is vpcmpeq+maskedmove+vpmovdw+vpacksswb
     // zero extend is vpcmpeq+maskedmove+vpmovdw+vpsrlw+vpackuswb
-    { ISD::SIGN_EXTEND, MVT::v2i8,   MVT::v2i1,   5 },
-    { ISD::ZERO_EXTEND, MVT::v2i8,   MVT::v2i1,   6 },
-    { ISD::SIGN_EXTEND, MVT::v4i8,   MVT::v4i1,   5 },
-    { ISD::ZERO_EXTEND, MVT::v4i8,   MVT::v4i1,   6 },
-    { ISD::SIGN_EXTEND, MVT::v8i8,   MVT::v8i1,   5 },
-    { ISD::ZERO_EXTEND, MVT::v8i8,   MVT::v8i1,   6 },
-    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v16i1, 10 },
-    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v16i1, 12 },
+    { ISD::SIGN_EXTEND, MVT::v2i8,   MVT::v2i1,   { 5, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v2i8,   MVT::v2i1,   { 6, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i8,   MVT::v4i1,   { 5, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i8,   MVT::v4i1,   { 6, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i8,   MVT::v8i1,   { 5, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i8,   MVT::v8i1,   { 6, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i8,  MVT::v16i1,  {10, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i8,  MVT::v16i1,  {12, 1, 1, 1 } },
 
     // sign extend is vpcmpeq+maskedmove+vpmovdw
     // zero extend is vpcmpeq+maskedmove+vpmovdw+vpsrlw
-    { ISD::SIGN_EXTEND, MVT::v2i16,  MVT::v2i1,   4 },
-    { ISD::ZERO_EXTEND, MVT::v2i16,  MVT::v2i1,   5 },
-    { ISD::SIGN_EXTEND, MVT::v4i16,  MVT::v4i1,   4 },
-    { ISD::ZERO_EXTEND, MVT::v4i16,  MVT::v4i1,   5 },
-    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v8i1,   4 },
-    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v8i1,   5 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1, 10 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1, 12 },
-
-    { ISD::SIGN_EXTEND, MVT::v2i32,  MVT::v2i1,   1 }, // vpternlogd
-    { ISD::ZERO_EXTEND, MVT::v2i32,  MVT::v2i1,   2 }, // vpternlogd+psrld
-    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v4i1,   1 }, // vpternlogd
-    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v4i1,   2 }, // vpternlogd+psrld
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   1 }, // vpternlogd
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   2 }, // vpternlogd+psrld
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v16i1,  1 }, // vpternlogd
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v16i1,  2 }, // vpternlogd+psrld
-
-    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v2i1,   1 }, // vpternlogq
-    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v2i1,   2 }, // vpternlogq+psrlq
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   1 }, // vpternlogq
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   2 }, // vpternlogq+psrlq
-
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v16i8,  1 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v16i8,  1 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v16i8,  1 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v16i8,  1 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8,  1 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8,  1 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v8i16,  1 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v8i16,  1 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i16,  1 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i16,  1 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i32,  1 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i32,  1 },
-
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  1 },
-    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  1 },
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  1 },
-    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  1 },
-
-    { ISD::UINT_TO_FP,  MVT::f32,    MVT::i64,    1 },
-    { ISD::UINT_TO_FP,  MVT::f64,    MVT::i64,    1 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  1 },
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  1 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  1 },
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  1 },
-    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i32,  1 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  1 },
-    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  1 },
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i32,  1 },
-    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i64,  5 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  5 },
-    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i64,  5 },
-
-    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v8f32,  2 },
-    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v16f32, 2 },
-    { ISD::FP_TO_SINT,  MVT::v32i8,  MVT::v32f32, 5 },
-
-    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f32,    1 },
-    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f64,    1 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f32,  1 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v2f64,  1 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f64,  1 },
-    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v8f32,  1 },
-    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v8f64,  1 },
+    { ISD::SIGN_EXTEND, MVT::v2i16,  MVT::v2i1,   { 4, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v2i16,  MVT::v2i1,   { 5, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i16,  MVT::v4i1,   { 4, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i16,  MVT::v4i1,   { 5, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v8i1,   { 4, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v8i1,   { 5, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1,  {10, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1,  {12, 1, 1, 1 } },
+
+    { ISD::SIGN_EXTEND, MVT::v2i32,  MVT::v2i1,   { 1, 1, 1, 1 } }, // vpternlogd
+    { ISD::ZERO_EXTEND, MVT::v2i32,  MVT::v2i1,   { 2, 1, 1, 1 } }, // vpternlogd+psrld
+    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v4i1,   { 1, 1, 1, 1 } }, // vpternlogd
+    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v4i1,   { 2, 1, 1, 1 } }, // vpternlogd+psrld
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   { 1, 1, 1, 1 } }, // vpternlogd
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   { 2, 1, 1, 1 } }, // vpternlogd+psrld
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v16i1,  { 1, 1, 1, 1 } }, // vpternlogd
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v16i1,  { 2, 1, 1, 1 } }, // vpternlogd+psrld
+
+    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v2i1,   { 1, 1, 1, 1 } }, // vpternlogq
+    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v2i1,   { 2, 1, 1, 1 } }, // vpternlogq+psrlq
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   { 1, 1, 1, 1 } }, // vpternlogq
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   { 2, 1, 1, 1 } }, // vpternlogq+psrlq
+
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i32,  { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i32,  { 1, 1, 1, 1 } },
+
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  { 1, 1, 1, 1 } },
+
+    { ISD::UINT_TO_FP,  MVT::f32,    MVT::i64,    { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::f64,    MVT::i64,    { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i32,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i32,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i64,  { 5, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  { 5, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i64,  { 5, 1, 1, 1 } },
+
+    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v8f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v16f32, { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v32i8,  MVT::v32f32, { 5, 1, 1, 1 } },
+
+    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f32,    { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f64,    { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v2f64,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f64,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v8f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v8f64,  { 1, 1, 1, 1 } },
   };
 
-  static const TypeConversionCostTblEntry AVX2ConversionTbl[] = {
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   3 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   3 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   3 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   3 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1,  1 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1,  1 },
-
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v16i8,  2 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v16i8,  2 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v16i8,  2 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v16i8,  2 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8,  2 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8,  2 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v8i16,  2 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v8i16,  2 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i16,  2 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i16,  2 },
-    { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i16, 3 },
-    { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i16, 3 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i32,  2 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i32,  2 },
-
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i32,  2 },
-
-    { ISD::TRUNCATE,    MVT::v16i16, MVT::v16i32, 4 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i32, 4 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v8i16,  1 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v4i32,  1 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v2i64,  1 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v8i32,  4 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v4i64,  4 },
-    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v4i32,  1 },
-    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v2i64,  1 },
-    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v4i64,  5 },
-    { ISD::TRUNCATE,    MVT::v4i32,  MVT::v4i64,  1 },
-    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v8i32,  2 },
-
-    { ISD::FP_EXTEND,   MVT::v8f64,  MVT::v8f32,  3 },
-    { ISD::FP_ROUND,    MVT::v8f32,  MVT::v8f64,  3 },
-
-    { ISD::FP_TO_SINT,  MVT::v16i16, MVT::v8f32,  1 },
-    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v4f64,  1 },
-    { ISD::FP_TO_SINT,  MVT::v8i32,  MVT::v8f32,  1 },
-    { ISD::FP_TO_SINT,  MVT::v8i32,  MVT::v8f64,  3 },
-
-    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f32,    3 },
-    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f64,    3 },
-    { ISD::FP_TO_UINT,  MVT::v16i16, MVT::v8f32,  1 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f32,  3 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v2f64,  4 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f64,  4 },
-    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v8f32,  3 },
-    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v4f64,  4 },
-
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  2 },
-    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  2 },
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  2 },
-    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  2 },
-    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  1 },
-    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i32,  1 },
-    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  3 },
-
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  2 },
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  2 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  2 },
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  2 },
-    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i32,  2 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i32,  1 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  2 },
-    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  2 },
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i32,  2 },
-    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  4 },
+  static const TypeConversionCostKindTblEntry AVX2ConversionTbl[] = {
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   { 3, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   { 3, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1,  { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1,  { 1, 1, 1, 1 } },
+
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i16, { 3, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i16, { 3, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i32,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i32,  { 2, 1, 1, 1 } },
+
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i32,  { 2, 1, 1, 1 } },
+
+    { ISD::TRUNCATE,    MVT::v16i16, MVT::v16i32, { 4, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i32, { 4, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v4i32,  { 1, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v2i64,  { 1, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v8i32,  { 4, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v4i64,  { 4, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v4i32,  { 1, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v2i64,  { 1, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v4i64,  { 5, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v4i32,  MVT::v4i64,  { 1, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v8i32,  { 2, 1, 1, 1 } },
+
+    { ISD::FP_EXTEND,   MVT::v8f64,  MVT::v8f32,  { 3, 1, 1, 1 } },
+    { ISD::FP_ROUND,    MVT::v8f32,  MVT::v8f64,  { 3, 1, 1, 1 } },
+
+    { ISD::FP_TO_SINT,  MVT::v16i16, MVT::v8f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v4f64,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i32,  MVT::v8f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i32,  MVT::v8f64,  { 3, 1, 1, 1 } },
+
+    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f32,    { 3, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f64,    { 3, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v16i16, MVT::v8f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f32,  { 3, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v2f64,  { 4, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f64,  { 4, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v8f32,  { 3, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v4f64,  { 4, 1, 1, 1 } },
+
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i32,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  { 3, 1, 1, 1 } },
+
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i32,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i32,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i32,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  { 4, 1, 1, 1 } },
   };
 
-  static const TypeConversionCostTblEntry AVXConversionTbl[] = {
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   4 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   4 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   4 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   4 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1,  4 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1,  4 },
-
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v16i8,  3 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v16i8,  3 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v16i8,  3 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v16i8,  3 },
-    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8,  3 },
-    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8,  3 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v8i16,  3 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v8i16,  3 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i16,  3 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i16,  3 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i32,  3 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i32,  3 },
-
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i64,  4 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i32,  5 },
-    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i16, 4 },
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i64,  9 },
-    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i64, 11 },
-
-    { ISD::TRUNCATE,    MVT::v16i16, MVT::v16i32, 6 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i32, 6 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i16, 2 }, // and+extract+packuswb
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v8i32,  5 },
-    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v8i32,  5 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v4i64,  5 },
-    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v4i64,  3 }, // and+extract+2*packusdw
-    { ISD::TRUNCATE,    MVT::v4i32,  MVT::v4i64,  2 },
-
-    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v4i1,   3 },
-    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v4i1,   3 },
-    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i1,   8 },
-    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  4 },
-    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v16i8,  2 },
-    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  4 },
-    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v8i16,  2 },
-    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  2 },
-    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i32,  2 },
-    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  4 },
-    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v2i64,  5 },
-    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v4i64,  8 },
-
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i1,   7 },
-    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i1,   7 },
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i1,   6 },
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  4 },
-    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v16i8,  2 },
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  4 },
-    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v8i16,  2 },
-    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i32,  4 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i32,  4 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  5 },
-    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  6 },
-    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i32,  8 },
-    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i32, 10 },
-    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i64, 10 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i64, 18 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  5 },
-    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i64, 10 },
-
-    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v8f32,  2 },
-    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v4f64,  2 },
-    { ISD::FP_TO_SINT,  MVT::v32i8,  MVT::v8f32,  2 },
-    { ISD::FP_TO_SINT,  MVT::v32i8,  MVT::v4f64,  2 },
-    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v8f32,  2 },
-    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v4f64,  2 },
-    { ISD::FP_TO_SINT,  MVT::v16i16, MVT::v8f32,  2 },
-    { ISD::FP_TO_SINT,  MVT::v16i16, MVT::v4f64,  2 },
-    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v4f64,  2 },
-    { ISD::FP_TO_SINT,  MVT::v8i32,  MVT::v8f32,  2 },
-    { ISD::FP_TO_SINT,  MVT::v8i32,  MVT::v8f64,  5 },
-
-    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v8f32,  2 },
-    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v4f64,  2 },
-    { ISD::FP_TO_UINT,  MVT::v32i8,  MVT::v8f32,  2 },
-    { ISD::FP_TO_UINT,  MVT::v32i8,  MVT::v4f64,  2 },
-    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v8f32,  2 },
-    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v4f64,  2 },
-    { ISD::FP_TO_UINT,  MVT::v16i16, MVT::v8f32,  2 },
-    { ISD::FP_TO_UINT,  MVT::v16i16, MVT::v4f64,  2 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f32,  3 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v2f64,  4 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f64,  6 },
-    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v8f32,  7 },
-    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v4f64,  7 },
-
-    { ISD::FP_EXTEND,   MVT::v4f64,  MVT::v4f32,  1 },
-    { ISD::FP_ROUND,    MVT::v4f32,  MVT::v4f64,  1 },
+  static const TypeConversionCostKindTblEntry AVXConversionTbl[] = {
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   { 4, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   { 4, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   { 4, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   { 4, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1,  { 4, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1,  { 4, 1, 1, 1 } },
+
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v16i8,  { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v16i8,  { 3, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v16i8,  { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v16i8,  { 3, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8,  { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8,  { 3, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v8i16,  { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v8i16,  { 3, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i16,  { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i16,  { 3, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i32,  { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i32,  { 3, 1, 1, 1 } },
+
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i64,  { 4, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i32,  { 5, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i16, { 4, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i64,  { 9, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i1,  MVT::v16i64, {11, 1, 1, 1 } },
+
+    { ISD::TRUNCATE,    MVT::v16i16, MVT::v16i32, { 6, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i32, { 6, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i16, { 2, 1, 1, 1 } }, // and+extract+packuswb
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v8i32,  { 5, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v8i32,  { 5, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v4i64,  { 5, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v4i64,  { 3, 1, 1, 1 } }, // and+extract+2*packusdw
+    { ISD::TRUNCATE,    MVT::v4i32,  MVT::v4i64,  { 2, 1, 1, 1 } },
+
+    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v4i1,   { 3, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v4i1,   { 3, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i1,   { 8, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  { 4, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  { 4, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  { 2, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f32,  MVT::v8i32,  { 2, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  { 4, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v2i64,  { 5, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v4i64,  { 8, 1, 1, 1 } },
+
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i1,   { 7, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i1,   { 7, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i1,   { 6, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v16i8,  { 4, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i16,  { 4, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i32,  { 4, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i32,  { 4, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  { 5, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  { 6, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i32,  { 8, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v8f64,  MVT::v8i32,  {10, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i64,  {10, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i64,  {18, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  { 5, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f64,  MVT::v4i64,  {10, 1, 1, 1 } },
+
+    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v8f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v4f64,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v32i8,  MVT::v8f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v32i8,  MVT::v4f64,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v8f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v4f64,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v16i16, MVT::v8f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v16i16, MVT::v4f64,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v4f64,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i32,  MVT::v8f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i32,  MVT::v8f64,  { 5, 1, 1, 1 } },
+
+    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v8f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v4f64,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v32i8,  MVT::v8f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v32i8,  MVT::v4f64,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v8f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v4f64,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v16i16, MVT::v8f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v16i16, MVT::v4f64,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f32,  { 3, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v2f64,  { 4, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f64,  { 6, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v8f32,  { 7, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i32,  MVT::v4f64,  { 7, 1, 1, 1 } },
+
+    { ISD::FP_EXTEND,   MVT::v4f64,  MVT::v4f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_ROUND,    MVT::v4f32,  MVT::v4f64,  { 1, 1, 1, 1 } },
   };
 
-  static const TypeConversionCostTblEntry SSE41ConversionTbl[] = {
-    { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v16i8,   1 },
-    { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v16i8,   1 },
-    { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v16i8,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v16i8,   1 },
-    { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v16i8,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v16i8,   1 },
-    { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v8i16,   1 },
-    { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v8i16,   1 },
-    { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v8i16,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v8i16,   1 },
-    { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v4i32,   1 },
-    { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v4i32,   1 },
+  static const TypeConversionCostKindTblEntry SSE41ConversionTbl[] = {
+    { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v16i8,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v16i8,   { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v16i8,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v16i8,   { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v16i8,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v16i8,   { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v8i16,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v8i16,   { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v8i16,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v8i16,   { 1, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v4i32,   { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v4i32,   { 1, 1, 1, 1 } },
 
     // These truncates end up widening elements.
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i8,   1 }, // PMOVXZBQ
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i16,  1 }, // PMOVXZWQ
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i8,   1 }, // PMOVXZBD
-
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v4i32,  2 },
-    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v4i32,  2 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v2i64,  2 },
-
-    { ISD::SINT_TO_FP,  MVT::f32,    MVT::i32,    1 },
-    { ISD::SINT_TO_FP,  MVT::f64,    MVT::i32,    1 },
-    { ISD::SINT_TO_FP,  MVT::f32,    MVT::i64,    1 },
-    { ISD::SINT_TO_FP,  MVT::f64,    MVT::i64,    1 },
-    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v16i8,  1 },
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  1 },
-    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v8i16,  1 },
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  1 },
-    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  1 },
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v4i32,  1 },
-    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  2 },
-
-    { ISD::UINT_TO_FP,  MVT::f32,    MVT::i32,    1 },
-    { ISD::UINT_TO_FP,  MVT::f64,    MVT::i32,    1 },
-    { ISD::UINT_TO_FP,  MVT::f32,    MVT::i64,    4 },
-    { ISD::UINT_TO_FP,  MVT::f64,    MVT::i64,    4 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v16i8,  1 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  1 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v8i16,  1 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  1 },
-    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i32,  3 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  3 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v4i32,  2 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v2i64, 12 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i64, 22 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  4 },
-
-    { ISD::FP_TO_SINT,  MVT::i32,    MVT::f32,    1 },
-    { ISD::FP_TO_SINT,  MVT::i64,    MVT::f32,    1 },
-    { ISD::FP_TO_SINT,  MVT::i32,    MVT::f64,    1 },
-    { ISD::FP_TO_SINT,  MVT::i64,    MVT::f64,    1 },
-    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v4f32,  2 },
-    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v2f64,  2 },
-    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v4f32,  1 },
-    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v2f64,  1 },
-    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v4f32,  1 },
-    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v2f64,  1 },
-
-    { ISD::FP_TO_UINT,  MVT::i32,    MVT::f32,    1 },
-    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f32,    4 },
-    { ISD::FP_TO_UINT,  MVT::i32,    MVT::f64,    1 },
-    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f64,    4 },
-    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v4f32,  2 },
-    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v2f64,  2 },
-    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v4f32,  1 },
-    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v2f64,  1 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f32,  4 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v2f64,  4 },
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i8,   { 1, 1, 1, 1 } }, // PMOVXZBQ
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i16,  { 1, 1, 1, 1 } }, // PMOVXZWQ
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i8,   { 1, 1, 1, 1 } }, // PMOVXZBD
+
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v4i32,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v4i32,  { 2, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v2i64,  { 2, 1, 1, 1 } },
+
+    { ISD::SINT_TO_FP,  MVT::f32,    MVT::i32,    { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::f64,    MVT::i32,    { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::f32,    MVT::i64,    { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::f64,    MVT::i64,    { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v4i32,  { 1, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f64,  MVT::v4i32,  { 2, 1, 1, 1 } },
+
+    { ISD::UINT_TO_FP,  MVT::f32,    MVT::i32,    { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::f64,    MVT::i32,    { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::f32,    MVT::i64,    { 4, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::f64,    MVT::i64,    { 4, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i32,  { 3, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  { 3, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v4i32,  { 2, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v2i64,  {12, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i64,  {22, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  { 4, 1, 1, 1 } },
+
+    { ISD::FP_TO_SINT,  MVT::i32,    MVT::f32,    { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::i64,    MVT::f32,    { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::i32,    MVT::f64,    { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::i64,    MVT::f64,    { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v4f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v2f64,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v4f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v2f64,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v4f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v2f64,  { 1, 1, 1, 1 } },
+
+    { ISD::FP_TO_UINT,  MVT::i32,    MVT::f32,    { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f32,    { 4, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::i32,    MVT::f64,    { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f64,    { 4, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v4f32,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v2f64,  { 2, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v4f32,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v2f64,  { 1, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f32,  { 4, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v2f64,  { 4, 1, 1, 1 } },
   };
 
-  static const TypeConversionCostTblEntry SSE2ConversionTbl[] = {
+  static const TypeConversionCostKindTblEntry SSE2ConversionTbl[] = {
     // These are somewhat magic numbers justified by comparing the
     // output of llvm-mca for our various supported scheduler models
     // and basing it off the worst case scenario.
-    { ISD::SINT_TO_FP,  MVT::f32,    MVT::i32,    3 },
-    { ISD::SINT_TO_FP,  MVT::f64,    MVT::i32,    3 },
-    { ISD::SINT_TO_FP,  MVT::f32,    MVT::i64,    3 },
-    { ISD::SINT_TO_FP,  MVT::f64,    MVT::i64,    3 },
-    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v16i8,  3 },
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  4 },
-    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v8i16,  3 },
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  4 },
-    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  3 },
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v4i32,  4 },
-    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v2i64,  8 },
-    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  8 },
-
-    { ISD::UINT_TO_FP,  MVT::f32,    MVT::i32,    3 },
-    { ISD::UINT_TO_FP,  MVT::f64,    MVT::i32,    3 },
-    { ISD::UINT_TO_FP,  MVT::f32,    MVT::i64,    8 },
-    { ISD::UINT_TO_FP,  MVT::f64,    MVT::i64,    9 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  4 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v16i8,  4 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v8i16,  4 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  4 },
-    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i32,  7 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v4i32,  7 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  5 },
-    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i64, 15 },
-    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v2i64, 18 },
-
-    { ISD::FP_TO_SINT,  MVT::i32,    MVT::f32,    4 },
-    { ISD::FP_TO_SINT,  MVT::i64,    MVT::f32,    4 },
-    { ISD::FP_TO_SINT,  MVT::i32,    MVT::f64,    4 },
-    { ISD::FP_TO_SINT,  MVT::i64,    MVT::f64,    4 },
-    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v4f32,  6 },
-    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v2f64,  6 },
-    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v4f32,  5 },
-    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v2f64,  5 },
-    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v4f32,  4 },
-    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v2f64,  4 },
-
-    { ISD::FP_TO_UINT,  MVT::i32,    MVT::f32,    4 },
-    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f32,    4 },
-    { ISD::FP_TO_UINT,  MVT::i32,    MVT::f64,    4 },
-    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f64,   15 },
-    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v4f32,  6 },
-    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v2f64,  6 },
-    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v4f32,  5 },
-    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v2f64,  5 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f32,  8 },
-    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v2f64,  8 },
-
-    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v16i8,  4 },
-    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v16i8,  4 },
-    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v16i8,  2 },
-    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v16i8,  3 },
-    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v16i8,  1 },
-    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v16i8,  2 },
-    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v8i16,  2 },
-    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v8i16,  3 },
-    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v8i16,  1 },
-    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v8i16,  2 },
-    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v4i32,  1 },
-    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v4i32,  2 },
+    { ISD::SINT_TO_FP,  MVT::f32,    MVT::i32,    { 3, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::f64,    MVT::i32,    { 3, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::f32,    MVT::i64,    { 3, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::f64,    MVT::i64,    { 3, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v16i8,  { 3, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  { 4, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v8i16,  { 3, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  { 4, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  { 3, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v4i32,  { 4, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v4f32,  MVT::v2i64,  { 8, 1, 1, 1 } },
+    { ISD::SINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  { 8, 1, 1, 1 } },
+
+    { ISD::UINT_TO_FP,  MVT::f32,    MVT::i32,    { 3, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::f64,    MVT::i32,    { 3, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::f32,    MVT::i64,    { 8, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::f64,    MVT::i64,    { 9, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v16i8,  { 4, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v16i8,  { 4, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v8i16,  { 4, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v8i16,  { 4, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f32,  MVT::v2i32,  { 7, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v4i32,  { 7, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v4i32,  { 5, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v2f64,  MVT::v2i64,  {15, 1, 1, 1 } },
+    { ISD::UINT_TO_FP,  MVT::v4f32,  MVT::v2i64,  {18, 1, 1, 1 } },
+
+    { ISD::FP_TO_SINT,  MVT::i32,    MVT::f32,    { 4, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::i64,    MVT::f32,    { 4, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::i32,    MVT::f64,    { 4, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::i64,    MVT::f64,    { 4, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v4f32,  { 6, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v16i8,  MVT::v2f64,  { 6, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v4f32,  { 5, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v8i16,  MVT::v2f64,  { 5, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v4f32,  { 4, 1, 1, 1 } },
+    { ISD::FP_TO_SINT,  MVT::v4i32,  MVT::v2f64,  { 4, 1, 1, 1 } },
+
+    { ISD::FP_TO_UINT,  MVT::i32,    MVT::f32,    { 4, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f32,    { 4, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::i32,    MVT::f64,    { 4, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::i64,    MVT::f64,    {15, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v4f32,  { 6, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v2f64,  { 6, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v4f32,  { 5, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v8i16,  MVT::v2f64,  { 5, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v4f32,  { 8, 1, 1, 1 } },
+    { ISD::FP_TO_UINT,  MVT::v4i32,  MVT::v2f64,  { 8, 1, 1, 1 } },
+
+    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v16i8,  { 4, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v16i8,  { 4, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v16i8,  { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v8i16,  MVT::v16i8,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v8i16,  MVT::v16i8,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v8i16,  { 3, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v4i32,  MVT::v8i16,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v4i32,  MVT::v8i16,  { 2, 1, 1, 1 } },
+    { ISD::ZERO_EXTEND, MVT::v2i64,  MVT::v4i32,  { 1, 1, 1, 1 } },
+    { ISD::SIGN_EXTEND, MVT::v2i64,  MVT::v4i32,  { 2, 1, 1, 1 } },
 
     // These truncates are really widening elements.
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i32,  1 }, // PSHUFD
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i16,  2 }, // PUNPCKLWD+DQ
-    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i8,   3 }, // PUNPCKLBW+WD+PSHUFD
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i16,  1 }, // PUNPCKLWD
-    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i8,   2 }, // PUNPCKLBW+WD
-    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i8,   1 }, // PUNPCKLBW
-
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v8i16,  2 }, // PAND+PACKUSWB
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i16, 3 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v4i32,  3 }, // PAND+2*PACKUSWB
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i32, 7 },
-    { ISD::TRUNCATE,    MVT::v2i16,  MVT::v2i32,  1 },
-    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v4i32,  3 },
-    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v8i32,  5 },
-    { ISD::TRUNCATE,    MVT::v16i16, MVT::v16i32,10 },
-    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v2i64,  4 }, // PAND+3*PACKUSWB
-    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v2i64,  2 }, // PSHUFD+PSHUFLW
-    { ISD::TRUNCATE,    MVT::v4i32,  MVT::v2i64,  1 }, // PSHUFD
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i32,  { 1, 1, 1, 1 } }, // PSHUFD
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i16,  { 2, 1, 1, 1 } }, // PUNPCKLWD+DQ
+    { ISD::TRUNCATE,    MVT::v2i1,   MVT::v2i8,   { 3, 1, 1, 1 } }, // PUNPCKLBW+WD+PSHUFD
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i16,  { 1, 1, 1, 1 } }, // PUNPCKLWD
+    { ISD::TRUNCATE,    MVT::v4i1,   MVT::v4i8,   { 2, 1, 1, 1 } }, // PUNPCKLBW+WD
+    { ISD::TRUNCATE,    MVT::v8i1,   MVT::v8i8,   { 1, 1, 1, 1 } }, // PUNPCKLBW
+
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v8i16,  { 2, 1, 1, 1 } }, // PAND+PACKUSWB
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i16, { 3, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v4i32,  { 3, 1, 1, 1 } }, // PAND+2*PACKUSWB
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i32, { 7, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v2i16,  MVT::v2i32,  { 1, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v4i32,  { 3, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v8i32,  { 5, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i16, MVT::v16i32, {10, 1, 1, 1 } },
+    { ISD::TRUNCATE,    MVT::v16i8,  MVT::v2i64,  { 4, 1, 1, 1 } }, // PAND+3*PACKUSWB
+    { ISD::TRUNCATE,    MVT::v8i16,  MVT::v2i64,  { 2, 1, 1, 1 } }, // PSHUFD+PSHUFLW
+    { ISD::TRUNCATE,    MVT::v4i32,  MVT::v2i64,  { 1, 1, 1, 1 } }, // PSHUFD
   };
 
   // Attempt to map directly to (simple) MVT types to let us match custom entries.
@@ -2958,56 +2951,66 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
       if (ST->hasBWI())
         if (const auto *Entry = ConvertCostTableLookup(
                 AVX512BWConversionTbl, ISD, SimpleDstTy, SimpleSrcTy))
-          return AdjustCost(Entry->Cost);
+          if (auto KindCost = Entry->Cost[CostKind])
+            return *KindCost;
 
       if (ST->hasDQI())
         if (const auto *Entry = ConvertCostTableLookup(
                 AVX512DQConversionTbl, ISD, SimpleDstTy, SimpleSrcTy))
-          return AdjustCost(Entry->Cost);
+          if (auto KindCost = Entry->Cost[CostKind])
+            return *KindCost;
 
       if (ST->hasAVX512())
         if (const auto *Entry = ConvertCostTableLookup(
                 AVX512FConversionTbl, ISD, SimpleDstTy, SimpleSrcTy))
-          return AdjustCost(Entry->Cost);
+          if (auto KindCost = Entry->Cost[CostKind])
+            return *KindCost;
     }
 
     if (ST->hasBWI())
       if (const auto *Entry = ConvertCostTableLookup(
               AVX512BWVLConversionTbl, ISD, SimpleDstTy, SimpleSrcTy))
-        return AdjustCost(Entry->Cost);
+        if (auto KindCost = Entry->Cost[CostKind])
+          return *KindCost;
 
     if (ST->hasDQI())
       if (const auto *Entry = ConvertCostTableLookup(
               AVX512DQVLConversionTbl, ISD, SimpleDstTy, SimpleSrcTy))
-        return AdjustCost(Entry->Cost);
+        if (auto KindCost = Entry->Cost[CostKind])
+          return *KindCost;
 
     if (ST->hasAVX512())
       if (const auto *Entry = ConvertCostTableLookup(AVX512VLConversionTbl, ISD,
                                                      SimpleDstTy, SimpleSrcTy))
-        return AdjustCost(Entry->Cost);
+        if (auto KindCost = Entry->Cost[CostKind])
+          return *KindCost;
 
     if (ST->hasAVX2()) {
       if (const auto *Entry = ConvertCostTableLookup(AVX2ConversionTbl, ISD,
                                                      SimpleDstTy, SimpleSrcTy))
-        return AdjustCost(Entry->Cost);
+        if (auto KindCost = Entry->Cost[CostKind])
+          return *KindCost;
     }
 
     if (ST->hasAVX()) {
       if (const auto *Entry = ConvertCostTableLookup(AVXConversionTbl, ISD,
                                                      SimpleDstTy, SimpleSrcTy))
-        return AdjustCost(Entry->Cost);
+        if (auto KindCost = Entry->Cost[CostKind])
+          return *KindCost;
     }
 
     if (ST->hasSSE41()) {
       if (const auto *Entry = ConvertCostTableLookup(SSE41ConversionTbl, ISD,
                                                      SimpleDstTy, SimpleSrcTy))
-        return AdjustCost(Entry->Cost);
+        if (auto KindCost = Entry->Cost[CostKind])
+          return *KindCost;
     }
 
     if (ST->hasSSE2()) {
       if (const auto *Entry = ConvertCostTableLookup(SSE2ConversionTbl, ISD,
                                                      SimpleDstTy, SimpleSrcTy))
-        return AdjustCost(Entry->Cost);
+        if (auto KindCost = Entry->Cost[CostKind])
+          return *KindCost;
     }
   }
 
@@ -3023,53 +3026,63 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
     if (ST->hasBWI())
       if (const auto *Entry = ConvertCostTableLookup(
               AVX512BWConversionTbl, ISD, LTDest.second, LTSrc.second))
-        return AdjustCost(Entry->Cost, std::max(LTSrc.first, LTDest.first));
+        if (auto KindCost = Entry->Cost[CostKind])
+          return std::max(LTSrc.first, LTDest.first) * *KindCost;
 
     if (ST->hasDQI())
       if (const auto *Entry = ConvertCostTableLookup(
               AVX512DQConversionTbl, ISD, LTDest.second, LTSrc.second))
-        return AdjustCost(Entry->Cost, std::max(LTSrc.first, LTDest.first));
+        if (auto KindCost = Entry->Cost[CostKind])
+          return std::max(LTSrc.first, LTDest.first) * *KindCost;
 
     if (ST->hasAVX512())
       if (const auto *Entry = ConvertCostTableLookup(
               AVX512FConversionTbl, ISD, LTDest.second, LTSrc.second))
-        return AdjustCost(Entry->Cost, std::max(LTSrc.first, LTDest.first));
+        if (auto KindCost = Entry->Cost[CostKind])
+          return std::max(LTSrc.first, LTDest.first) * *KindCost;
   }
 
   if (ST->hasBWI())
     if (const auto *Entry = ConvertCostTableLookup(AVX512BWVLConversionTbl, ISD,
                                                    LTDest.second, LTSrc.second))
-      return AdjustCost(Entry->Cost, std::max(LTSrc.first, LTDest.first));
+      if (auto KindCost = Entry->Cost[CostKind])
+        return std::max(LTSrc.first, LTDest.first) * *KindCost;
 
   if (ST->hasDQI())
     if (const auto *Entry = ConvertCostTableLookup(AVX512DQVLConversionTbl, ISD,
                                                    LTDest.second, LTSrc.second))
-      return AdjustCost(Entry->Cost, std::max(LTSrc.first, LTDest.first));
+      if (auto KindCost = Entry->Cost[CostKind])
+        return std::max(LTSrc.first, LTDest.first) * *KindCost;
 
   if (ST->hasAVX512())
     if (const auto *Entry = ConvertCostTableLookup(AVX512VLConversionTbl, ISD,
                                                    LTDest.second, LTSrc.second))
-      return AdjustCost(Entry->Cost, std::max(LTSrc.first, LTDest.first));
+      if (auto KindCost = Entry->Cost[CostKind])
+        return std::max(LTSrc.first, LTDest.first) * *KindCost;
 
   if (ST->hasAVX2())
     if (const auto *Entry = ConvertCostTableLookup(AVX2ConversionTbl, ISD,
                                                    LTDest.second, LTSrc.second))
-      return AdjustCost(Entry->Cost, std::max(LTSrc.first, LTDest.first));
+      if (auto KindCost = Entry->Cost[CostKind])
+        return std::max(LTSrc.first, LTDest.first) * *KindCost;
 
   if (ST->hasAVX())
     if (const auto *Entry = ConvertCostTableLookup(AVXConversionTbl, ISD,
                                                    LTDest.second, LTSrc.second))
-      return AdjustCost(Entry->Cost, std::max(LTSrc.first, LTDest.first));
+      if (auto KindCost = Entry->Cost[CostKind])
+        return std::max(LTSrc.first, LTDest.first) * *KindCost;
 
   if (ST->hasSSE41())
     if (const auto *Entry = ConvertCostTableLookup(SSE41ConversionTbl, ISD,
                                                    LTDest.second, LTSrc.second))
-      return AdjustCost(Entry->Cost, std::max(LTSrc.first, LTDest.first));
+      if (auto KindCost = Entry->Cost[CostKind])
+        return std::max(LTSrc.first, LTDest.first) * *KindCost;
 
   if (ST->hasSSE2())
     if (const auto *Entry = ConvertCostTableLookup(SSE2ConversionTbl, ISD,
                                                    LTDest.second, LTSrc.second))
-      return AdjustCost(Entry->Cost, std::max(LTSrc.first, LTDest.first));
+      if (auto KindCost = Entry->Cost[CostKind])
+        return std::max(LTSrc.first, LTDest.first) * *KindCost;
 
   // Fallback, for i8/i16 sitofp/uitofp cases we need to extend to i32 for
   // sitofp.
@@ -3098,6 +3111,13 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                             TTI::CastContextHint::None, CostKind);
   }
 
+  // TODO: Allow non-throughput costs that aren't binary.
+  auto AdjustCost = [&CostKind](InstructionCost Cost,
+                                InstructionCost N = 1) -> InstructionCost {
+    if (CostKind != TTI::TCK_RecipThroughput)
+      return Cost == 0 ? 0 : N;
+    return Cost * N;
+  };
   return AdjustCost(
       BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
 }
-- 
GitLab


From f0590581aa01944a3bb502a622a412617d1efcbb Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project@meinersbur.de>
Date: Mon, 13 May 2024 14:55:34 +0200
Subject: [PATCH 0559/1206] [Clang][OpenMP] Fix unused lambda capture warning.

---
 clang/lib/Sema/SemaOpenMP.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index fff4c7350f0f..2475f962fd0d 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15207,8 +15207,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // Commonly used variables. One of the constraints of an AST is that every
     // node object must appear at most once, hence we define lamdas that create
     // a new AST node at every use.
-    auto MakeDimTileSize = [&SemaRef = this->SemaRef, &CopyTransformer, I,
-                            SizesClause]() -> Expr * {
+    auto MakeDimTileSize = [&CopyTransformer, I, SizesClause]() -> Expr * {
       Expr *DimTileSize = SizesClause->getSizesRefs()[I];
       return AssertSuccess(CopyTransformer.TransformExpr(DimTileSize));
     };
@@ -15298,8 +15297,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     QualType CntTy = OrigCntVar->getType();
 
     // Commonly used variables.
-    auto MakeDimTileSize = [&SemaRef = this->SemaRef, &CopyTransformer, I,
-                            SizesClause]() -> Expr * {
+    auto MakeDimTileSize = [&CopyTransformer, I, SizesClause]() -> Expr * {
       Expr *DimTileSize = SizesClause->getSizesRefs()[I];
       return AssertSuccess(CopyTransformer.TransformExpr(DimTileSize));
     };
-- 
GitLab


From ca1bd5995f6ed934f9187305190a5abfac049173 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Mon, 13 May 2024 08:09:24 -0500
Subject: [PATCH 0560/1206] [flang][OpenMP] Decompose compound constructs, do
 recursive lowering (#90098)

A compound construct with a list of clauses is broken up into individual
leaf/composite constructs. Each such construct has the list of clauses
that apply to it based on the OpenMP spec.

Each lowering function (i.e. a function that generates MLIR ops) is now
responsible for generating its body as described below.

Functions that receive AST nodes extract the construct, and the clauses
from the node. They then create a work queue consisting of individual
constructs, and invoke a common dispatch function to process (lower) the
queue.

The dispatch function examines the current position in the queue, and
invokes the appropriate lowering function. Each lowering function
receives the queue as well, and once it needs to generate its body, it
either invokes the dispatch function on the rest of the queue (if any),
or processes nested evaluations if the work queue is at the end.
---
 flang/lib/Lower/CMakeLists.txt                |    1 +
 flang/lib/Lower/OpenMP/Clauses.cpp            |   23 +
 flang/lib/Lower/OpenMP/Clauses.h              |   13 +-
 flang/lib/Lower/OpenMP/Decomposer.cpp         |  126 ++
 flang/lib/Lower/OpenMP/Decomposer.h           |   51 +
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  828 ++++++------
 flang/lib/Lower/OpenMP/Utils.cpp              |    6 -
 flang/lib/Lower/OpenMP/Utils.h                |    1 -
 .../Lower/OpenMP/default-clause-byref.f90     |    5 +-
 flang/test/Lower/OpenMP/default-clause.f90    |    4 +-
 .../parallel-lastprivate-clause-scalar.f90    |    4 +-
 llvm/include/llvm/Frontend/OpenMP/ClauseT.h   |   52 +-
 .../Frontend/OpenMP/ConstructCompositionT.h   |  403 ++++++
 .../Frontend/OpenMP/ConstructDecompositionT.h | 1161 +++++++++++++++++
 llvm/unittests/Frontend/CMakeLists.txt        |    1 +
 .../Frontend/OpenMPDecompositionTest.cpp      |  999 ++++++++++++++
 16 files changed, 3226 insertions(+), 452 deletions(-)
 create mode 100644 flang/lib/Lower/OpenMP/Decomposer.cpp
 create mode 100644 flang/lib/Lower/OpenMP/Decomposer.h
 create mode 100644 llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
 create mode 100644 llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
 create mode 100644 llvm/unittests/Frontend/OpenMPDecompositionTest.cpp

diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index f92d1a2bc7de..1546409752e7 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -27,6 +27,7 @@ add_flang_library(FortranLower
   OpenMP/ClauseProcessor.cpp
   OpenMP/Clauses.cpp
   OpenMP/DataSharingProcessor.cpp
+  OpenMP/Decomposer.cpp
   OpenMP/OpenMP.cpp
   OpenMP/ReductionProcessor.cpp
   OpenMP/Utils.cpp
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 97337cfc08c7..87370c92964a 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1227,4 +1227,27 @@ List<Clause> makeClauses(const parser::OmpClauseList &clauses,
     return makeClause(s, semaCtx);
   });
 }
+
+bool transferLocations(const List<Clause> &from, List<Clause> &to) {
+  bool allDone = true;
+
+  for (Clause &clause : to) {
+    if (!clause.source.empty())
+      continue;
+    auto found =
+        llvm::find_if(from, [&](const Clause &c) { return c.id == clause.id; });
+    // This is not completely accurate, but should be good enough for now.
+    // It can be improved in the future if necessary, but in cases of
+    // synthesized clauses getting accurate location may be impossible.
+    if (found != from.end()) {
+      clause.source = found->source;
+    } else {
+      // Found a clause that won't have "source".
+      allDone = false;
+    }
+  }
+
+  return allDone;
+}
+
 } // namespace Fortran::lower::omp
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index 3e776425c733..407579319279 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -23,11 +23,15 @@
 
 namespace Fortran::lower::omp {
 using namespace Fortran;
-using SomeType = evaluate::SomeType;
 using SomeExpr = semantics::SomeExpr;
 using MaybeExpr = semantics::MaybeExpr;
 
-using TypeTy = SomeType;
+// evaluate::SomeType doesn't provide == operation. It's not really used in
+// flang's clauses so far, so a trivial implementation is sufficient.
+struct TypeTy : public evaluate::SomeType {
+  bool operator==(const TypeTy &t) const { return true; }
+};
+
 using IdTy = semantics::Symbol *;
 using ExprTy = SomeExpr;
 
@@ -222,6 +226,8 @@ using When = tomp::clause::WhenT<TypeTy, IdTy, ExprTy>;
 using Write = tomp::clause::WriteT<TypeTy, IdTy, ExprTy>;
 } // namespace clause
 
+using tomp::type::operator==;
+
 struct CancellationConstructType {
   using EmptyTrait = std::true_type;
 };
@@ -244,6 +250,7 @@ using ClauseBase = tomp::ClauseT<TypeTy, IdTy, ExprTy,
                                  MemoryOrder, Threadprivate>;
 
 struct Clause : public ClauseBase {
+  // "source" will be ignored by tomp::type::operator==.
   parser::CharBlock source;
 };
 
@@ -258,6 +265,8 @@ Clause makeClause(const Fortran::parser::OmpClause &cls,
 
 List<Clause> makeClauses(const parser::OmpClauseList &clauses,
                          semantics::SemanticsContext &semaCtx);
+
+bool transferLocations(const List<Clause> &from, List<Clause> &to);
 } // namespace Fortran::lower::omp
 
 #endif // FORTRAN_LOWER_OPENMP_CLAUSES_H
diff --git a/flang/lib/Lower/OpenMP/Decomposer.cpp b/flang/lib/Lower/OpenMP/Decomposer.cpp
new file mode 100644
index 000000000000..e6897cb81e94
--- /dev/null
+++ b/flang/lib/Lower/OpenMP/Decomposer.cpp
@@ -0,0 +1,126 @@
+//===-- Decomposer.cpp -- Compound directive decomposition ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "Decomposer.h"
+
+#include "Clauses.h"
+#include "Utils.h"
+#include "flang/Lower/PFTBuilder.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Tools/CrossToolHelpers.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Frontend/OpenMP/ClauseT.h"
+#include "llvm/Frontend/OpenMP/ConstructCompositionT.h"
+#include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <optional>
+#include <utility>
+#include <variant>
+
+using namespace Fortran;
+
+namespace {
+using namespace Fortran::lower::omp;
+
+struct ConstructDecomposition {
+  ConstructDecomposition(mlir::ModuleOp modOp,
+                         semantics::SemanticsContext &semaCtx,
+                         lower::pft::Evaluation &ev,
+                         llvm::omp::Directive compound,
+                         const List<Clause> &clauses)
+      : semaCtx(semaCtx), mod(modOp), eval(ev) {
+    tomp::ConstructDecompositionT decompose(getOpenMPVersionAttribute(modOp),
+                                            *this, compound,
+                                            llvm::ArrayRef(clauses));
+    output = std::move(decompose.output);
+  }
+
+  // Given an object, return its base object if one exists.
+  std::optional<Object> getBaseObject(const Object &object) {
+    return lower::omp::getBaseObject(object, semaCtx);
+  }
+
+  // Return the iteration variable of the associated loop if any.
+  std::optional<Object> getLoopIterVar() {
+    if (semantics::Symbol *symbol = getIterationVariableSymbol(eval))
+      return Object{symbol, /*designator=*/{}};
+    return std::nullopt;
+  }
+
+  semantics::SemanticsContext &semaCtx;
+  mlir::ModuleOp mod;
+  lower::pft::Evaluation &eval;
+  List<UnitConstruct> output;
+};
+} // namespace
+
+static UnitConstruct mergeConstructs(uint32_t version,
+                                     llvm::ArrayRef<UnitConstruct> units) {
+  tomp::ConstructCompositionT compose(version, units);
+  return compose.merged;
+}
+
+namespace Fortran::lower::omp {
+LLVM_DUMP_METHOD llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
+                                               const UnitConstruct &uc) {
+  os << llvm::omp::getOpenMPDirectiveName(uc.id);
+  for (auto [index, clause] : llvm::enumerate(uc.clauses)) {
+    os << (index == 0 ? '\t' : ' ');
+    os << llvm::omp::getOpenMPClauseName(clause.id);
+  }
+  return os;
+}
+
+ConstructQueue buildConstructQueue(
+    mlir::ModuleOp modOp, Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::pft::Evaluation &eval, const parser::CharBlock &source,
+    llvm::omp::Directive compound, const List<Clause> &clauses) {
+
+  List<UnitConstruct> constructs;
+
+  ConstructDecomposition decompose(modOp, semaCtx, eval, compound, clauses);
+  assert(!decompose.output.empty() && "Construct decomposition failed");
+
+  llvm::SmallVector<llvm::omp::Directive> loweringUnits;
+  std::ignore =
+      llvm::omp::getLeafOrCompositeConstructs(compound, loweringUnits);
+  uint32_t version = getOpenMPVersionAttribute(modOp);
+
+  int leafIndex = 0;
+  for (llvm::omp::Directive dir_id : loweringUnits) {
+    llvm::ArrayRef<llvm::omp::Directive> leafsOrSelf =
+        llvm::omp::getLeafConstructsOrSelf(dir_id);
+    size_t numLeafs = leafsOrSelf.size();
+
+    llvm::ArrayRef<UnitConstruct> toMerge{&decompose.output[leafIndex],
+                                          numLeafs};
+    auto &uc = constructs.emplace_back(mergeConstructs(version, toMerge));
+
+    if (!transferLocations(clauses, uc.clauses)) {
+      // If some clauses are left without source information, use the
+      // directive's source.
+      for (auto &clause : uc.clauses) {
+        if (clause.source.empty())
+          clause.source = source;
+      }
+    }
+    leafIndex += numLeafs;
+  }
+
+  return constructs;
+}
+} // namespace Fortran::lower::omp
diff --git a/flang/lib/Lower/OpenMP/Decomposer.h b/flang/lib/Lower/OpenMP/Decomposer.h
new file mode 100644
index 000000000000..f42d8f5c1740
--- /dev/null
+++ b/flang/lib/Lower/OpenMP/Decomposer.h
@@ -0,0 +1,51 @@
+//===-- Decomposer.h -- Compound directive decomposition ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef FORTRAN_LOWER_OPENMP_DECOMPOSER_H
+#define FORTRAN_LOWER_OPENMP_DECOMPOSER_H
+
+#include "Clauses.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "llvm/Frontend/OpenMP/ConstructCompositionT.h"
+#include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+class raw_ostream;
+}
+
+namespace Fortran {
+namespace semantics {
+class SemanticsContext;
+}
+namespace lower::pft {
+struct Evaluation;
+}
+} // namespace Fortran
+
+namespace Fortran::lower::omp {
+using UnitConstruct = tomp::DirectiveWithClauses<lower::omp::Clause>;
+using ConstructQueue = List<UnitConstruct>;
+
+LLVM_DUMP_METHOD llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
+                                               const UnitConstruct &uc);
+
+// Given a potentially compound construct with a list of clauses that
+// apply to it, break it up into individual sub-constructs each with
+// the subset of applicable clauses (plus implicit clauses, if any).
+// From that create a work queue where each work item corresponds to
+// the sub-construct with its clauses.
+ConstructQueue buildConstructQueue(mlir::ModuleOp modOp,
+                                   semantics::SemanticsContext &semaCtx,
+                                   lower::pft::Evaluation &eval,
+                                   const parser::CharBlock &source,
+                                   llvm::omp::Directive compound,
+                                   const List<Clause> &clauses);
+} // namespace Fortran::lower::omp
+
+#endif // FORTRAN_LOWER_OPENMP_DECOMPOSER_H
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index f23902d6a823..eaf4b5f997ff 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -15,6 +15,7 @@
 #include "ClauseProcessor.h"
 #include "Clauses.h"
 #include "DataSharingProcessor.h"
+#include "Decomposer.h"
 #include "DirectivesCommon.h"
 #include "ReductionProcessor.h"
 #include "Utils.h"
@@ -44,6 +45,13 @@ using namespace Fortran::lower::omp;
 // Code generation helper functions
 //===----------------------------------------------------------------------===//
 
+static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
+                           Fortran::lower::SymMap &symTable,
+                           Fortran::semantics::SemanticsContext &semaCtx,
+                           Fortran::lower::pft::Evaluation &eval,
+                           mlir::Location loc, const ConstructQueue &queue,
+                           ConstructQueue::iterator item);
+
 static Fortran::lower::pft::Evaluation *
 getCollapsedLoopEval(Fortran::lower::pft::Evaluation &eval, int collapseValue) {
   // Return the Evaluation of the innermost collapsed loop, or the current one
@@ -460,81 +468,6 @@ markDeclareTarget(mlir::Operation *op,
   declareTargetOp.setDeclareTarget(deviceType, captureClause);
 }
 
-/// Split a combined directive into an outer leaf directive and the (possibly
-/// combined) rest of the combined directive. Composite directives and
-/// non-compound directives are not split, in which case it will return the
-/// input directive as its first output and an empty value as its second output.
-static std::pair<llvm::omp::Directive, std::optional<llvm::omp::Directive>>
-splitCombinedDirective(llvm::omp::Directive dir) {
-  using D = llvm::omp::Directive;
-  switch (dir) {
-  case D::OMPD_masked_taskloop:
-    return {D::OMPD_masked, D::OMPD_taskloop};
-  case D::OMPD_masked_taskloop_simd:
-    return {D::OMPD_masked, D::OMPD_taskloop_simd};
-  case D::OMPD_master_taskloop:
-    return {D::OMPD_master, D::OMPD_taskloop};
-  case D::OMPD_master_taskloop_simd:
-    return {D::OMPD_master, D::OMPD_taskloop_simd};
-  case D::OMPD_parallel_do:
-    return {D::OMPD_parallel, D::OMPD_do};
-  case D::OMPD_parallel_do_simd:
-    return {D::OMPD_parallel, D::OMPD_do_simd};
-  case D::OMPD_parallel_masked:
-    return {D::OMPD_parallel, D::OMPD_masked};
-  case D::OMPD_parallel_masked_taskloop:
-    return {D::OMPD_parallel, D::OMPD_masked_taskloop};
-  case D::OMPD_parallel_masked_taskloop_simd:
-    return {D::OMPD_parallel, D::OMPD_masked_taskloop_simd};
-  case D::OMPD_parallel_master:
-    return {D::OMPD_parallel, D::OMPD_master};
-  case D::OMPD_parallel_master_taskloop:
-    return {D::OMPD_parallel, D::OMPD_master_taskloop};
-  case D::OMPD_parallel_master_taskloop_simd:
-    return {D::OMPD_parallel, D::OMPD_master_taskloop_simd};
-  case D::OMPD_parallel_sections:
-    return {D::OMPD_parallel, D::OMPD_sections};
-  case D::OMPD_parallel_workshare:
-    return {D::OMPD_parallel, D::OMPD_workshare};
-  case D::OMPD_target_parallel:
-    return {D::OMPD_target, D::OMPD_parallel};
-  case D::OMPD_target_parallel_do:
-    return {D::OMPD_target, D::OMPD_parallel_do};
-  case D::OMPD_target_parallel_do_simd:
-    return {D::OMPD_target, D::OMPD_parallel_do_simd};
-  case D::OMPD_target_simd:
-    return {D::OMPD_target, D::OMPD_simd};
-  case D::OMPD_target_teams:
-    return {D::OMPD_target, D::OMPD_teams};
-  case D::OMPD_target_teams_distribute:
-    return {D::OMPD_target, D::OMPD_teams_distribute};
-  case D::OMPD_target_teams_distribute_parallel_do:
-    return {D::OMPD_target, D::OMPD_teams_distribute_parallel_do};
-  case D::OMPD_target_teams_distribute_parallel_do_simd:
-    return {D::OMPD_target, D::OMPD_teams_distribute_parallel_do_simd};
-  case D::OMPD_target_teams_distribute_simd:
-    return {D::OMPD_target, D::OMPD_teams_distribute_simd};
-  case D::OMPD_teams_distribute:
-    return {D::OMPD_teams, D::OMPD_distribute};
-  case D::OMPD_teams_distribute_parallel_do:
-    return {D::OMPD_teams, D::OMPD_distribute_parallel_do};
-  case D::OMPD_teams_distribute_parallel_do_simd:
-    return {D::OMPD_teams, D::OMPD_distribute_parallel_do_simd};
-  case D::OMPD_teams_distribute_simd:
-    return {D::OMPD_teams, D::OMPD_distribute_simd};
-  case D::OMPD_parallel_loop:
-    return {D::OMPD_parallel, D::OMPD_loop};
-  case D::OMPD_target_parallel_loop:
-    return {D::OMPD_target, D::OMPD_parallel_loop};
-  case D::OMPD_target_teams_loop:
-    return {D::OMPD_target, D::OMPD_teams_loop};
-  case D::OMPD_teams_loop:
-    return {D::OMPD_teams, D::OMPD_loop};
-  default:
-    return {dir, std::nullopt};
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Op body generation helper structures and functions
 //===----------------------------------------------------------------------===//
@@ -555,11 +488,6 @@ struct OpWithBodyGenInfo {
       : converter(converter), symTable(symTable), semaCtx(semaCtx), loc(loc),
         eval(eval), dir(dir) {}
 
-  OpWithBodyGenInfo &setGenNested(bool value) {
-    genNested = value;
-    return *this;
-  }
-
   OpWithBodyGenInfo &setOuterCombined(bool value) {
     outerCombined = value;
     return *this;
@@ -600,8 +528,6 @@ struct OpWithBodyGenInfo {
   Fortran::lower::pft::Evaluation &eval;
   /// [in] leaf directive for which to generate the op body.
   llvm::omp::Directive dir;
-  /// [in] whether to generate FIR for nested evaluations
-  bool genNested = true;
   /// [in] is this an outer operation - prevents privatization.
   bool outerCombined = false;
   /// [in] list of clauses to process.
@@ -620,9 +546,13 @@ struct OpWithBodyGenInfo {
 
 /// Create the body (block) for an OpenMP Operation.
 ///
-/// \param [in]   op - the operation the body belongs to.
-/// \param [in] info - options controlling code-gen for the construction.
-static void createBodyOfOp(mlir::Operation &op, OpWithBodyGenInfo &info) {
+/// \param [in]   op  - the operation the body belongs to.
+/// \param [in] info  - options controlling code-gen for the construction.
+/// \param [in] queue - work queue with nested constructs.
+/// \param [in] item  - item in the queue to generate body for.
+static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
+                           const ConstructQueue &queue,
+                           ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = info.converter.getFirOpBuilder();
 
   auto insertMarker = [](fir::FirOpBuilder &builder) {
@@ -678,7 +608,10 @@ static void createBodyOfOp(mlir::Operation &op, OpWithBodyGenInfo &info) {
     }
   }
 
-  if (info.genNested) {
+  if (ConstructQueue::iterator next = std::next(item); next != queue.end()) {
+    genOMPDispatch(info.converter, info.symTable, info.semaCtx, info.eval,
+                   info.loc, queue, next);
+  } else {
     // genFIR(Evaluation&) tries to patch up unterminated blocks, causing
     // a lot of complications for our approach if the terminator generation
     // is delayed past this point. Insert a temporary terminator here, then
@@ -769,11 +702,12 @@ static void genBodyOfTargetDataOp(
     Fortran::lower::AbstractConverter &converter,
     Fortran::lower::SymMap &symTable,
     Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, bool genNested,
-    mlir::omp::TargetDataOp &dataOp, llvm::ArrayRef<mlir::Type> useDeviceTypes,
+    Fortran::lower::pft::Evaluation &eval, mlir::omp::TargetDataOp &dataOp,
+    llvm::ArrayRef<mlir::Type> useDeviceTypes,
     llvm::ArrayRef<mlir::Location> useDeviceLocs,
     llvm::ArrayRef<const Fortran::semantics::Symbol *> useDeviceSymbols,
-    const mlir::Location &currentLocation) {
+    const mlir::Location &currentLocation, const ConstructQueue &queue,
+    ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::Region &region = dataOp.getRegion();
 
@@ -826,8 +760,13 @@ static void genBodyOfTargetDataOp(
 
   // Set the insertion point after the marker.
   firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
-  if (genNested)
+
+  if (ConstructQueue::iterator next = std::next(item); next != queue.end()) {
+    genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
+                   next);
+  } else {
     genNestedEvaluations(converter, eval);
+  }
 }
 
 // This functions creates a block for the body of the targetOp's region. It adds
@@ -836,12 +775,13 @@ static void
 genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
                   Fortran::lower::SymMap &symTable,
                   Fortran::semantics::SemanticsContext &semaCtx,
-                  Fortran::lower::pft::Evaluation &eval, bool genNested,
+                  Fortran::lower::pft::Evaluation &eval,
                   mlir::omp::TargetOp &targetOp,
                   llvm::ArrayRef<const Fortran::semantics::Symbol *> mapSyms,
                   llvm::ArrayRef<mlir::Location> mapSymLocs,
                   llvm::ArrayRef<mlir::Type> mapSymTypes,
-                  const mlir::Location &currentLocation) {
+                  const mlir::Location &currentLocation,
+                  const ConstructQueue &queue, ConstructQueue::iterator item) {
   assert(mapSymTypes.size() == mapSymLocs.size());
 
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
@@ -983,15 +923,22 @@ genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
 
   // Create the insertion point after the marker.
   firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
-  if (genNested)
+
+  if (ConstructQueue::iterator next = std::next(item); next != queue.end()) {
+    genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
+                   next);
+  } else {
     genNestedEvaluations(converter, eval);
+  }
 }
 
 template <typename OpTy, typename... Args>
-static OpTy genOpWithBody(OpWithBodyGenInfo &info, Args &&...args) {
+static OpTy genOpWithBody(const OpWithBodyGenInfo &info,
+                          const ConstructQueue &queue,
+                          ConstructQueue::iterator item, Args &&...args) {
   auto op = info.converter.getFirOpBuilder().create<OpTy>(
       info.loc, std::forward<Args>(args)...);
-  createBodyOfOp(*op, info);
+  createBodyOfOp(*op, info, queue, item);
   return op;
 }
 
@@ -1276,7 +1223,8 @@ static mlir::omp::BarrierOp
 genBarrierOp(Fortran::lower::AbstractConverter &converter,
              Fortran::lower::SymMap &symTable,
              Fortran::semantics::SemanticsContext &semaCtx,
-             Fortran::lower::pft::Evaluation &eval, mlir::Location loc) {
+             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+             const ConstructQueue &queue, ConstructQueue::iterator item) {
   return converter.getFirOpBuilder().create<mlir::omp::BarrierOp>(loc);
 }
 
@@ -1284,8 +1232,9 @@ static mlir::omp::CriticalOp
 genCriticalOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, bool genNested,
-              mlir::Location loc, const List<Clause> &clauses,
+              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+              const List<Clause> &clauses, const ConstructQueue &queue,
+              ConstructQueue::iterator item,
               const std::optional<Fortran::parser::Name> &name) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::FlatSymbolRefAttr nameAttr;
@@ -1308,17 +1257,17 @@ genCriticalOp(Fortran::lower::AbstractConverter &converter,
 
   return genOpWithBody<mlir::omp::CriticalOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                        llvm::omp::Directive::OMPD_critical)
-          .setGenNested(genNested),
-      nameAttr);
+                        llvm::omp::Directive::OMPD_critical),
+      queue, item, nameAttr);
 }
 
 static mlir::omp::DistributeOp
 genDistributeOp(Fortran::lower::AbstractConverter &converter,
                 Fortran::lower::SymMap &symTable,
                 Fortran::semantics::SemanticsContext &semaCtx,
-                Fortran::lower::pft::Evaluation &eval, bool genNested,
-                mlir::Location loc, const List<Clause> &clauses) {
+                Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+                const List<Clause> &clauses, const ConstructQueue &queue,
+                ConstructQueue::iterator item) {
   TODO(loc, "Distribute construct");
   return nullptr;
 }
@@ -1328,7 +1277,8 @@ genFlushOp(Fortran::lower::AbstractConverter &converter,
            Fortran::lower::SymMap &symTable,
            Fortran::semantics::SemanticsContext &semaCtx,
            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-           const ObjectList &objects, const List<Clause> &clauses) {
+           const ObjectList &objects, const List<Clause> &clauses,
+           const ConstructQueue &queue, ConstructQueue::iterator item) {
   llvm::SmallVector<mlir::Value> operandRange;
   genFlushClauses(converter, semaCtx, objects, clauses, loc, operandRange);
 
@@ -1340,12 +1290,13 @@ static mlir::omp::MasterOp
 genMasterOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, bool genNested,
-            mlir::Location loc) {
+            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+            const List<Clause> &clauses, const ConstructQueue &queue,
+            ConstructQueue::iterator item) {
   return genOpWithBody<mlir::omp::MasterOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                        llvm::omp::Directive::OMPD_master)
-          .setGenNested(genNested));
+                        llvm::omp::Directive::OMPD_master),
+      queue, item);
 }
 
 static mlir::omp::OrderedOp
@@ -1353,7 +1304,8 @@ genOrderedOp(Fortran::lower::AbstractConverter &converter,
              Fortran::lower::SymMap &symTable,
              Fortran::semantics::SemanticsContext &semaCtx,
              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-             const List<Clause> &clauses) {
+             const List<Clause> &clauses, const ConstructQueue &queue,
+             ConstructQueue::iterator item) {
   TODO(loc, "OMPD_ordered");
   return nullptr;
 }
@@ -1362,25 +1314,25 @@ static mlir::omp::OrderedRegionOp
 genOrderedRegionOp(Fortran::lower::AbstractConverter &converter,
                    Fortran::lower::SymMap &symTable,
                    Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval, bool genNested,
-                   mlir::Location loc, const List<Clause> &clauses) {
+                   Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+                   const List<Clause> &clauses, const ConstructQueue &queue,
+                   ConstructQueue::iterator item) {
   mlir::omp::OrderedRegionClauseOps clauseOps;
   genOrderedRegionClauses(converter, semaCtx, clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::OrderedRegionOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                        llvm::omp::Directive::OMPD_ordered)
-          .setGenNested(genNested),
-      clauseOps);
+                        llvm::omp::Directive::OMPD_ordered),
+      queue, item, clauseOps);
 }
 
 static mlir::omp::ParallelOp
 genParallelOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, bool genNested,
-              mlir::Location loc, const List<Clause> &clauses,
-              bool outerCombined = false) {
+              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+              const List<Clause> &clauses, const ConstructQueue &queue,
+              ConstructQueue::iterator item, bool outerCombined = false) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::ParallelClauseOps clauseOps;
@@ -1399,14 +1351,14 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
   OpWithBodyGenInfo genInfo =
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_parallel)
-          .setGenNested(genNested)
           .setOuterCombined(outerCombined)
           .setClauses(&clauses)
           .setReductions(&reductionSyms, &reductionTypes)
           .setGenRegionEntryCb(reductionCallback);
 
   if (!enableDelayedPrivatization)
-    return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
+    return genOpWithBody<mlir::omp::ParallelOp>(genInfo, queue, item,
+                                                clauseOps);
 
   bool privatize = !outerCombined;
   DataSharingProcessor dsp(converter, semaCtx, clauses, eval,
@@ -1454,19 +1406,23 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
   };
 
   genInfo.setGenRegionEntryCb(genRegionEntryCB).setDataSharingProcessor(&dsp);
-  return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
+  return genOpWithBody<mlir::omp::ParallelOp>(genInfo, queue, item, clauseOps);
 }
 
 static mlir::omp::SectionOp
 genSectionOp(Fortran::lower::AbstractConverter &converter,
              Fortran::lower::SymMap &symTable,
              Fortran::semantics::SemanticsContext &semaCtx,
-             Fortran::lower::pft::Evaluation &eval, bool genNested,
-             mlir::Location loc) {
+             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+             const List<Clause> &clauses, const ConstructQueue &queue,
+             ConstructQueue::iterator item) {
+  // Currently only private/firstprivate clause is handled, and
+  // all privatization is done within `omp.section` operations.
   return genOpWithBody<mlir::omp::SectionOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_section)
-          .setGenNested(genNested));
+          .setClauses(&clauses),
+      queue, item);
 }
 
 static mlir::omp::SectionsOp
@@ -1474,12 +1430,77 @@ genSectionsOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const mlir::omp::SectionsClauseOps &clauseOps) {
-  return genOpWithBody<mlir::omp::SectionsOp>(
+              const List<Clause> &clauses, const ConstructQueue &queue,
+              ConstructQueue::iterator item) {
+  mlir::omp::SectionsClauseOps clauseOps;
+  genSectionsClauses(converter, semaCtx, clauses, loc, clauseOps);
+
+  auto &builder = converter.getFirOpBuilder();
+
+  // Insert privatizations before SECTIONS
+  symTable.pushScope();
+  DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
+  dsp.processStep1();
+
+  List<Clause> nonDsaClauses;
+  List<const clause::Lastprivate *> lastprivates;
+
+  for (const Clause &clause : clauses) {
+    if (clause.id == llvm::omp::Clause::OMPC_lastprivate) {
+      lastprivates.push_back(&std::get<clause::Lastprivate>(clause.u));
+    } else {
+      switch (clause.id) {
+      case llvm::omp::Clause::OMPC_firstprivate:
+      case llvm::omp::Clause::OMPC_private:
+      case llvm::omp::Clause::OMPC_shared:
+        break;
+      default:
+        nonDsaClauses.push_back(clause);
+      }
+    }
+  }
+
+  // SECTIONS construct.
+  mlir::omp::SectionsOp sectionsOp = genOpWithBody<mlir::omp::SectionsOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_sections)
-          .setGenNested(false),
-      clauseOps);
+          .setClauses(&nonDsaClauses),
+      queue, item, clauseOps);
+
+  if (!lastprivates.empty()) {
+    mlir::Region &sectionsBody = sectionsOp.getRegion();
+    assert(sectionsBody.hasOneBlock());
+    mlir::Block &body = sectionsBody.front();
+
+    auto lastSectionOp = llvm::find_if(
+        llvm::reverse(body.getOperations()), [](const mlir::Operation &op) {
+          return llvm::isa<mlir::omp::SectionOp>(op);
+        });
+    assert(lastSectionOp != body.rend());
+
+    for (const clause::Lastprivate *lastp : lastprivates) {
+      builder.setInsertionPoint(
+          lastSectionOp->getRegion(0).back().getTerminator());
+      mlir::OpBuilder::InsertPoint insp = builder.saveInsertionPoint();
+      const auto &objList = std::get<ObjectList>(lastp->t);
+      for (const Object &object : objList) {
+        Fortran::semantics::Symbol *sym = object.id();
+        converter.copyHostAssociateVar(*sym, &insp);
+      }
+    }
+  }
+
+  // Perform DataSharingProcessor's step2 out of SECTIONS
+  builder.setInsertionPointAfter(sectionsOp.getOperation());
+  dsp.processStep2(sectionsOp, false);
+  // Emit implicit barrier to synchronize threads and avoid data
+  // races on post-update of lastprivate variables when `nowait`
+  // clause is present.
+  if (clauseOps.nowaitAttr && !lastprivates.empty())
+    builder.create<mlir::omp::BarrierOp>(loc);
+
+  symTable.popScope();
+  return sectionsOp;
 }
 
 static mlir::omp::SimdOp
@@ -1487,7 +1508,8 @@ genSimdOp(Fortran::lower::AbstractConverter &converter,
           Fortran::lower::SymMap &symTable,
           Fortran::semantics::SemanticsContext &semaCtx,
           Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-          const List<Clause> &clauses) {
+          const List<Clause> &clauses, const ConstructQueue &queue,
+          ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
   dsp.processStep1();
@@ -1522,7 +1544,8 @@ genSimdOp(Fortran::lower::AbstractConverter &converter,
                                    *nestedEval, llvm::omp::Directive::OMPD_simd)
                      .setClauses(&clauses)
                      .setDataSharingProcessor(&dsp)
-                     .setGenRegionEntryCb(ivCallback));
+                     .setGenRegionEntryCb(ivCallback),
+                 queue, item);
 
   return simdOp;
 }
@@ -1531,26 +1554,26 @@ static mlir::omp::SingleOp
 genSingleOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, bool genNested,
-            mlir::Location loc, const List<Clause> &clauses) {
+            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+            const List<Clause> &clauses, const ConstructQueue &queue,
+            ConstructQueue::iterator item) {
   mlir::omp::SingleClauseOps clauseOps;
   genSingleClauses(converter, semaCtx, clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::SingleOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_single)
-          .setGenNested(genNested)
           .setClauses(&clauses),
-      clauseOps);
+      queue, item, clauseOps);
 }
 
 static mlir::omp::TargetOp
 genTargetOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, bool genNested,
-            mlir::Location loc, const List<Clause> &clauses,
-            bool outerCombined = false) {
+            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+            const List<Clause> &clauses, const ConstructQueue &queue,
+            ConstructQueue::iterator item, bool outerCombined = false) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
 
@@ -1657,8 +1680,8 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
   Fortran::lower::pft::visitAllSymbols(eval, captureImplicitMap);
 
   auto targetOp = firOpBuilder.create<mlir::omp::TargetOp>(loc, clauseOps);
-  genBodyOfTargetOp(converter, symTable, semaCtx, eval, genNested, targetOp,
-                    mapSyms, mapLocs, mapTypes, loc);
+  genBodyOfTargetOp(converter, symTable, semaCtx, eval, targetOp, mapSyms,
+                    mapLocs, mapTypes, loc, queue, item);
   return targetOp;
 }
 
@@ -1666,8 +1689,9 @@ static mlir::omp::TargetDataOp
 genTargetDataOp(Fortran::lower::AbstractConverter &converter,
                 Fortran::lower::SymMap &symTable,
                 Fortran::semantics::SemanticsContext &semaCtx,
-                Fortran::lower::pft::Evaluation &eval, bool genNested,
-                mlir::Location loc, const List<Clause> &clauses) {
+                Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+                const List<Clause> &clauses, const ConstructQueue &queue,
+                ConstructQueue::iterator item) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TargetDataClauseOps clauseOps;
   llvm::SmallVector<mlir::Type> useDeviceTypes;
@@ -1679,9 +1703,9 @@ genTargetDataOp(Fortran::lower::AbstractConverter &converter,
   auto targetDataOp =
       converter.getFirOpBuilder().create<mlir::omp::TargetDataOp>(loc,
                                                                   clauseOps);
-  genBodyOfTargetDataOp(converter, symTable, semaCtx, eval, genNested,
-                        targetDataOp, useDeviceTypes, useDeviceLocs,
-                        useDeviceSyms, loc);
+  genBodyOfTargetDataOp(converter, symTable, semaCtx, eval, targetDataOp,
+                        useDeviceTypes, useDeviceLocs, useDeviceSyms, loc,
+                        queue, item);
   return targetDataOp;
 }
 
@@ -1690,8 +1714,9 @@ static OpTy
 genTargetEnterExitUpdateDataOp(Fortran::lower::AbstractConverter &converter,
                                Fortran::lower::SymMap &symTable,
                                Fortran::semantics::SemanticsContext &semaCtx,
-                               mlir::Location loc,
-                               const List<Clause> &clauses) {
+                               mlir::Location loc, const List<Clause> &clauses,
+                               const ConstructQueue &queue,
+                               ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
 
@@ -1718,8 +1743,9 @@ static mlir::omp::TaskOp
 genTaskOp(Fortran::lower::AbstractConverter &converter,
           Fortran::lower::SymMap &symTable,
           Fortran::semantics::SemanticsContext &semaCtx,
-          Fortran::lower::pft::Evaluation &eval, bool genNested,
-          mlir::Location loc, const List<Clause> &clauses) {
+          Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+          const List<Clause> &clauses, const ConstructQueue &queue,
+          ConstructQueue::iterator item) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TaskClauseOps clauseOps;
   genTaskClauses(converter, semaCtx, stmtCtx, clauses, loc, clauseOps);
@@ -1727,26 +1753,25 @@ genTaskOp(Fortran::lower::AbstractConverter &converter,
   return genOpWithBody<mlir::omp::TaskOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_task)
-          .setGenNested(genNested)
           .setClauses(&clauses),
-      clauseOps);
+      queue, item, clauseOps);
 }
 
 static mlir::omp::TaskgroupOp
 genTaskgroupOp(Fortran::lower::AbstractConverter &converter,
                Fortran::lower::SymMap &symTable,
                Fortran::semantics::SemanticsContext &semaCtx,
-               Fortran::lower::pft::Evaluation &eval, bool genNested,
-               mlir::Location loc, const List<Clause> &clauses) {
+               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+               const List<Clause> &clauses, const ConstructQueue &queue,
+               ConstructQueue::iterator item) {
   mlir::omp::TaskgroupClauseOps clauseOps;
   genTaskgroupClauses(converter, semaCtx, clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::TaskgroupOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_taskgroup)
-          .setGenNested(genNested)
           .setClauses(&clauses),
-      clauseOps);
+      queue, item, clauseOps);
 }
 
 static mlir::omp::TaskloopOp
@@ -1754,7 +1779,8 @@ genTaskloopOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses) {
+              const List<Clause> &clauses, const ConstructQueue &queue,
+              ConstructQueue::iterator item) {
   TODO(loc, "Taskloop construct");
 }
 
@@ -1763,7 +1789,8 @@ genTaskwaitOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses) {
+              const List<Clause> &clauses, const ConstructQueue &queue,
+              ConstructQueue::iterator item) {
   mlir::omp::TaskwaitClauseOps clauseOps;
   genTaskwaitClauses(converter, semaCtx, clauses, loc, clauseOps);
   return converter.getFirOpBuilder().create<mlir::omp::TaskwaitOp>(loc,
@@ -1774,7 +1801,8 @@ static mlir::omp::TaskyieldOp
 genTaskyieldOp(Fortran::lower::AbstractConverter &converter,
                Fortran::lower::SymMap &symTable,
                Fortran::semantics::SemanticsContext &semaCtx,
-               Fortran::lower::pft::Evaluation &eval, mlir::Location loc) {
+               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+               const ConstructQueue &queue, ConstructQueue::iterator item) {
   return converter.getFirOpBuilder().create<mlir::omp::TaskyieldOp>(loc);
 }
 
@@ -1782,9 +1810,9 @@ static mlir::omp::TeamsOp
 genTeamsOp(Fortran::lower::AbstractConverter &converter,
            Fortran::lower::SymMap &symTable,
            Fortran::semantics::SemanticsContext &semaCtx,
-           Fortran::lower::pft::Evaluation &eval, bool genNested,
-           mlir::Location loc, const List<Clause> &clauses,
-           bool outerCombined = false) {
+           Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+           const List<Clause> &clauses, const ConstructQueue &queue,
+           ConstructQueue::iterator item, bool outerCombined = false) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TeamsClauseOps clauseOps;
   genTeamsClauses(converter, semaCtx, stmtCtx, clauses, loc, clauseOps);
@@ -1792,10 +1820,9 @@ genTeamsOp(Fortran::lower::AbstractConverter &converter,
   return genOpWithBody<mlir::omp::TeamsOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_teams)
-          .setGenNested(genNested)
           .setOuterCombined(outerCombined)
           .setClauses(&clauses),
-      clauseOps);
+      queue, item, clauseOps);
 }
 
 static mlir::omp::WsloopOp
@@ -1803,7 +1830,8 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const List<Clause> &clauses) {
+            const List<Clause> &clauses, const ConstructQueue &queue,
+            ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
   dsp.processStep1();
@@ -1844,7 +1872,8 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
                      .setClauses(&clauses)
                      .setDataSharingProcessor(&dsp)
                      .setReductions(&reductionSyms, &reductionTypes)
-                     .setGenRegionEntryCb(ivCallback));
+                     .setGenRegionEntryCb(ivCallback),
+                 queue, item);
   return wsloopOp;
 }
 
@@ -1852,13 +1881,13 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
 // Code generation functions for composite constructs
 //===----------------------------------------------------------------------===//
 
-static void
-genCompositeDistributeParallelDo(Fortran::lower::AbstractConverter &converter,
-                                 Fortran::lower::SymMap &symTable,
-                                 Fortran::semantics::SemanticsContext &semaCtx,
-                                 Fortran::lower::pft::Evaluation &eval,
-                                 const List<Clause> &clauses,
-                                 mlir::Location loc) {
+static void genCompositeDistributeParallelDo(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::lower::SymMap &symTable,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+    const List<Clause> &clauses, const ConstructQueue &queue,
+    ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE PARALLEL DO");
 }
 
@@ -1866,8 +1895,9 @@ static void genCompositeDistributeParallelDoSimd(
     Fortran::lower::AbstractConverter &converter,
     Fortran::lower::SymMap &symTable,
     Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, const List<Clause> &clauses,
-    mlir::Location loc) {
+    Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+    const List<Clause> &clauses, const ConstructQueue &queue,
+    ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE PARALLEL DO SIMD");
 }
 
@@ -1876,7 +1906,9 @@ genCompositeDistributeSimd(Fortran::lower::AbstractConverter &converter,
                            Fortran::lower::SymMap &symTable,
                            Fortran::semantics::SemanticsContext &semaCtx,
                            Fortran::lower::pft::Evaluation &eval,
-                           const List<Clause> &clauses, mlir::Location loc) {
+                           mlir::Location loc, const List<Clause> &clauses,
+                           const ConstructQueue &queue,
+                           ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE SIMD");
 }
 
@@ -1884,8 +1916,9 @@ static void genCompositeDoSimd(Fortran::lower::AbstractConverter &converter,
                                Fortran::lower::SymMap &symTable,
                                Fortran::semantics::SemanticsContext &semaCtx,
                                Fortran::lower::pft::Evaluation &eval,
-                               const List<Clause> &clauses,
-                               mlir::Location loc) {
+                               mlir::Location loc, const List<Clause> &clauses,
+                               const ConstructQueue &queue,
+                               ConstructQueue::iterator item) {
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processTODO<clause::Aligned, clause::Allocate, clause::Linear,
                  clause::Order, clause::Safelen, clause::Simdlen>(
@@ -1898,7 +1931,7 @@ static void genCompositeDoSimd(Fortran::lower::AbstractConverter &converter,
   // When support for vectorization is enabled, then we need to add handling of
   // if clause. Currently if clause can be skipped because we always assume
   // SIMD length = 1.
-  genWsloopOp(converter, symTable, semaCtx, eval, loc, clauses);
+  genWsloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
 }
 
 static void
@@ -1906,10 +1939,128 @@ genCompositeTaskloopSimd(Fortran::lower::AbstractConverter &converter,
                          Fortran::lower::SymMap &symTable,
                          Fortran::semantics::SemanticsContext &semaCtx,
                          Fortran::lower::pft::Evaluation &eval,
-                         const List<Clause> &clauses, mlir::Location loc) {
+                         mlir::Location loc, const List<Clause> &clauses,
+                         const ConstructQueue &queue,
+                         ConstructQueue::iterator item) {
   TODO(loc, "Composite TASKLOOP SIMD");
 }
 
+//===----------------------------------------------------------------------===//
+// Dispatch
+//===----------------------------------------------------------------------===//
+
+static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
+                           Fortran::lower::SymMap &symTable,
+                           Fortran::semantics::SemanticsContext &semaCtx,
+                           Fortran::lower::pft::Evaluation &eval,
+                           mlir::Location loc, const ConstructQueue &queue,
+                           ConstructQueue::iterator item) {
+  assert(item != queue.end());
+  const List<Clause> &clauses = item->clauses;
+
+  switch (llvm::omp::Directive dir = item->id) {
+  case llvm::omp::Directive::OMPD_distribute:
+    genDistributeOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                    item);
+    break;
+  case llvm::omp::Directive::OMPD_do:
+    genWsloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_loop:
+  case llvm::omp::Directive::OMPD_masked:
+  case llvm::omp::Directive::OMPD_tile:
+  case llvm::omp::Directive::OMPD_unroll:
+    TODO(loc, "Unhandled loop directive (" +
+                  llvm::omp::getOpenMPDirectiveName(dir) + ")");
+    break;
+  case llvm::omp::Directive::OMPD_master:
+    genMasterOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_ordered:
+    genOrderedRegionOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                       item);
+    break;
+  case llvm::omp::Directive::OMPD_parallel:
+    genParallelOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item,
+                  /*outerCombined=*/false);
+    break;
+  case llvm::omp::Directive::OMPD_sections:
+    genSectionsOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                  item);
+    break;
+  case llvm::omp::Directive::OMPD_simd:
+    genSimdOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_single:
+    genSingleOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_target:
+    genTargetOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item,
+                /*outerCombined=*/false);
+    break;
+  case llvm::omp::Directive::OMPD_target_data:
+    genTargetDataOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                    item);
+    break;
+  case llvm::omp::Directive::OMPD_target_enter_data:
+    genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(
+        converter, symTable, semaCtx, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_target_exit_data:
+    genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(
+        converter, symTable, semaCtx, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_target_update:
+    genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(
+        converter, symTable, semaCtx, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_task:
+    genTaskOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_taskgroup:
+    genTaskgroupOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                   item);
+    break;
+  case llvm::omp::Directive::OMPD_taskloop:
+    genTaskloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                  item);
+    break;
+  case llvm::omp::Directive::OMPD_teams:
+    genTeamsOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  // case llvm::omp::Directive::OMPD_workdistribute:
+  case llvm::omp::Directive::OMPD_workshare:
+    // FIXME: Workshare is not a commonly used OpenMP construct, an
+    // implementation for this feature will come later. For the codes
+    // that use this construct, add a single construct for now.
+    genSingleOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  // Composite constructs
+  case llvm::omp::Directive::OMPD_distribute_parallel_do:
+    genCompositeDistributeParallelDo(converter, symTable, semaCtx, eval, loc,
+                                     clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_distribute_parallel_do_simd:
+    genCompositeDistributeParallelDoSimd(converter, symTable, semaCtx, eval,
+                                         loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_distribute_simd:
+    genCompositeDistributeSimd(converter, symTable, semaCtx, eval, loc, clauses,
+                               queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_do_simd:
+    genCompositeDoSimd(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                       item);
+    break;
+  case llvm::omp::Directive::OMPD_taskloop_simd:
+    genCompositeTaskloopSimd(converter, symTable, semaCtx, eval, loc, clauses,
+                             queue, item);
+    break;
+  default:
+    break;
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // OpenMPDeclarativeConstruct visitors
 //===----------------------------------------------------------------------===//
@@ -2020,36 +2171,47 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
       semaCtx);
   mlir::Location currentLocation = converter.genLocation(directive.source);
 
+  ConstructQueue queue{
+      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
+                          eval, directive.source, directive.v, clauses)};
+
   switch (directive.v) {
   default:
     break;
   case llvm::omp::Directive::OMPD_barrier:
-    genBarrierOp(converter, symTable, semaCtx, eval, currentLocation);
+    genBarrierOp(converter, symTable, semaCtx, eval, currentLocation, queue,
+                 queue.begin());
     break;
   case llvm::omp::Directive::OMPD_taskwait:
-    genTaskwaitOp(converter, symTable, semaCtx, eval, currentLocation, clauses);
+    genTaskwaitOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
+                  queue, queue.begin());
     break;
   case llvm::omp::Directive::OMPD_taskyield:
-    genTaskyieldOp(converter, symTable, semaCtx, eval, currentLocation);
+    genTaskyieldOp(converter, symTable, semaCtx, eval, currentLocation, queue,
+                   queue.begin());
     break;
   case llvm::omp::Directive::OMPD_target_data:
-    genTargetDataOp(converter, symTable, semaCtx, eval, /*genNested=*/true,
-                    currentLocation, clauses);
+    genTargetDataOp(converter, symTable, semaCtx, eval, currentLocation,
+                    clauses, queue, queue.begin());
     break;
   case llvm::omp::Directive::OMPD_target_enter_data:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(
-        converter, symTable, semaCtx, currentLocation, clauses);
+        converter, symTable, semaCtx, currentLocation, clauses, queue,
+        queue.begin());
     break;
   case llvm::omp::Directive::OMPD_target_exit_data:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(
-        converter, symTable, semaCtx, currentLocation, clauses);
+        converter, symTable, semaCtx, currentLocation, clauses, queue,
+        queue.begin());
     break;
   case llvm::omp::Directive::OMPD_target_update:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(
-        converter, symTable, semaCtx, currentLocation, clauses);
+        converter, symTable, semaCtx, currentLocation, clauses, queue,
+        queue.begin());
     break;
   case llvm::omp::Directive::OMPD_ordered:
-    genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, clauses);
+    genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
+                 queue, queue.begin());
     break;
   }
 }
@@ -2073,8 +2235,12 @@ genOMP(Fortran::lower::AbstractConverter &converter,
                             [&](auto &&s) { return makeClause(s.v, semaCtx); })
                  : List<Clause>{};
   mlir::Location currentLocation = converter.genLocation(verbatim.source);
+
+  ConstructQueue queue{buildConstructQueue(
+      converter.getFirOpBuilder().getModule(), semaCtx, eval, verbatim.source,
+      llvm::omp::Directive::OMPD_flush, clauses)};
   genFlushOp(converter, symTable, semaCtx, eval, currentLocation, objects,
-             clauses);
+             clauses, queue, queue.begin());
 }
 
 static void
@@ -2217,75 +2383,15 @@ genOMP(Fortran::lower::AbstractConverter &converter,
     }
   }
 
-  std::optional<llvm::omp::Directive> nextDir = origDirective;
-  bool outermostLeafConstruct = true;
-  while (nextDir) {
-    llvm::omp::Directive leafDir;
-    std::tie(leafDir, nextDir) = splitCombinedDirective(*nextDir);
-    const bool genNested = !nextDir;
-    const bool outerCombined = outermostLeafConstruct && nextDir.has_value();
-    switch (leafDir) {
-    case llvm::omp::Directive::OMPD_master:
-      // 2.16 MASTER construct.
-      genMasterOp(converter, symTable, semaCtx, eval, genNested,
-                  currentLocation);
-      break;
-    case llvm::omp::Directive::OMPD_ordered:
-      // 2.17.9 ORDERED construct.
-      genOrderedRegionOp(converter, symTable, semaCtx, eval, genNested,
-                         currentLocation, clauses);
-      break;
-    case llvm::omp::Directive::OMPD_parallel:
-      // 2.6 PARALLEL construct.
-      genParallelOp(converter, symTable, semaCtx, eval, genNested,
-                    currentLocation, clauses, outerCombined);
-      break;
-    case llvm::omp::Directive::OMPD_single:
-      // 2.8.2 SINGLE construct.
-      genSingleOp(converter, symTable, semaCtx, eval, genNested,
-                  currentLocation, clauses);
-      break;
-    case llvm::omp::Directive::OMPD_target:
-      // 2.12.5 TARGET construct.
-      genTargetOp(converter, symTable, semaCtx, eval, genNested,
-                  currentLocation, clauses, outerCombined);
-      break;
-    case llvm::omp::Directive::OMPD_target_data:
-      // 2.12.2 TARGET DATA construct.
-      genTargetDataOp(converter, symTable, semaCtx, eval, genNested,
-                      currentLocation, clauses);
-      break;
-    case llvm::omp::Directive::OMPD_task:
-      // 2.10.1 TASK construct.
-      genTaskOp(converter, symTable, semaCtx, eval, genNested, currentLocation,
-                clauses);
-      break;
-    case llvm::omp::Directive::OMPD_taskgroup:
-      // 2.17.6 TASKGROUP construct.
-      genTaskgroupOp(converter, symTable, semaCtx, eval, genNested,
-                     currentLocation, clauses);
-      break;
-    case llvm::omp::Directive::OMPD_teams:
-      // 2.7 TEAMS construct.
-      // FIXME Pass the outerCombined argument or rename it to better describe
-      // what it represents if it must always be `false` in this context.
-      genTeamsOp(converter, symTable, semaCtx, eval, genNested, currentLocation,
-                 clauses);
-      break;
-    case llvm::omp::Directive::OMPD_workshare:
-      // 2.8.3 WORKSHARE construct.
-      // FIXME: Workshare is not a commonly used OpenMP construct, an
-      // implementation for this feature will come later. For the codes
-      // that use this construct, add a single construct for now.
-      genSingleOp(converter, symTable, semaCtx, eval, genNested,
-                  currentLocation, clauses);
-      break;
-    default:
-      llvm_unreachable("Unexpected block construct");
-      break;
-    }
-    outermostLeafConstruct = false;
-  }
+  llvm::omp::Directive directive =
+      std::get<parser::OmpBlockDirective>(beginBlockDirective.t).v;
+  const parser::CharBlock &source =
+      std::get<parser::OmpBlockDirective>(beginBlockDirective.t).source;
+  ConstructQueue queue{
+      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
+                          eval, source, directive, clauses)};
+  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
+                 queue.begin());
 }
 
 static void
@@ -2298,10 +2404,15 @@ genOMP(Fortran::lower::AbstractConverter &converter,
       std::get<Fortran::parser::OmpCriticalDirective>(criticalConstruct.t);
   List<Clause> clauses =
       makeClauses(std::get<Fortran::parser::OmpClauseList>(cd.t), semaCtx);
+
+  ConstructQueue queue{buildConstructQueue(
+      converter.getFirOpBuilder().getModule(), semaCtx, eval, cd.source,
+      llvm::omp::Directive::OMPD_critical, clauses)};
+
   const auto &name = std::get<std::optional<Fortran::parser::Name>>(cd.t);
   mlir::Location currentLocation = converter.getCurrentLocation();
-  genCriticalOp(converter, symTable, semaCtx, eval, /*genNested=*/true,
-                currentLocation, clauses, name);
+  genCriticalOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
+                queue, queue.begin(), name);
 }
 
 static void
@@ -2322,14 +2433,6 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
       std::get<Fortran::parser::OmpBeginLoopDirective>(loopConstruct.t);
   List<Clause> clauses = makeClauses(
       std::get<Fortran::parser::OmpClauseList>(beginLoopDirective.t), semaCtx);
-  mlir::Location currentLocation =
-      converter.genLocation(beginLoopDirective.source);
-  const auto origDirective =
-      std::get<Fortran::parser::OmpLoopDirective>(beginLoopDirective.t).v;
-
-  assert(llvm::omp::loopConstructSet.test(origDirective) &&
-         "Expected loop construct");
-
   if (auto &endLoopDirective =
           std::get<std::optional<Fortran::parser::OmpEndLoopDirective>>(
               loopConstruct.t)) {
@@ -2338,101 +2441,18 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
         semaCtx));
   }
 
-  std::optional<llvm::omp::Directive> nextDir = origDirective;
-  while (nextDir) {
-    llvm::omp::Directive leafDir;
-    std::tie(leafDir, nextDir) = splitCombinedDirective(*nextDir);
-    if (llvm::omp::compositeConstructSet.test(leafDir)) {
-      assert(!nextDir && "Composite construct cannot be split");
-      switch (leafDir) {
-      case llvm::omp::Directive::OMPD_distribute_parallel_do:
-        // 2.9.4.3 DISTRIBUTE PARALLEL Worksharing-Loop construct.
-        genCompositeDistributeParallelDo(converter, symTable, semaCtx, eval,
-                                         clauses, currentLocation);
-        break;
-      case llvm::omp::Directive::OMPD_distribute_parallel_do_simd:
-        // 2.9.4.4 DISTRIBUTE PARALLEL Worksharing-Loop SIMD construct.
-        genCompositeDistributeParallelDoSimd(converter, symTable, semaCtx, eval,
-                                             clauses, currentLocation);
-        break;
-      case llvm::omp::Directive::OMPD_distribute_simd:
-        // 2.9.4.2 DISTRIBUTE SIMD construct.
-        genCompositeDistributeSimd(converter, symTable, semaCtx, eval, clauses,
-                                   currentLocation);
-        break;
-      case llvm::omp::Directive::OMPD_do_simd:
-        // 2.9.3.2 Worksharing-Loop SIMD construct.
-        genCompositeDoSimd(converter, symTable, semaCtx, eval, clauses,
-                           currentLocation);
-        break;
-      case llvm::omp::Directive::OMPD_taskloop_simd:
-        // 2.10.3 TASKLOOP SIMD construct.
-        genCompositeTaskloopSimd(converter, symTable, semaCtx, eval, clauses,
-                                 currentLocation);
-        break;
-      default:
-        llvm_unreachable("Unexpected composite construct");
-      }
-    } else {
-      const bool genNested = !nextDir;
-      switch (leafDir) {
-      case llvm::omp::Directive::OMPD_distribute:
-        // 2.9.4.1 DISTRIBUTE construct.
-        genDistributeOp(converter, symTable, semaCtx, eval, genNested,
-                        currentLocation, clauses);
-        break;
-      case llvm::omp::Directive::OMPD_do:
-        // 2.9.2 Worksharing-Loop construct.
-        genWsloopOp(converter, symTable, semaCtx, eval, currentLocation,
-                    clauses);
-        break;
-      case llvm::omp::Directive::OMPD_parallel:
-        // 2.6 PARALLEL construct.
-        // FIXME This is not necessarily always the outer leaf construct of a
-        // combined construct in this constext (e.g. distribute parallel do).
-        // Maybe rename the argument if it represents something else or
-        // initialize it properly.
-        genParallelOp(converter, symTable, semaCtx, eval, genNested,
-                      currentLocation, clauses,
-                      /*outerCombined=*/true);
-        break;
-      case llvm::omp::Directive::OMPD_simd:
-        // 2.9.3.1 SIMD construct.
-        genSimdOp(converter, symTable, semaCtx, eval, currentLocation, clauses);
-        break;
-      case llvm::omp::Directive::OMPD_target:
-        // 2.12.5 TARGET construct.
-        genTargetOp(converter, symTable, semaCtx, eval, genNested,
-                    currentLocation, clauses, /*outerCombined=*/true);
-        break;
-      case llvm::omp::Directive::OMPD_taskloop:
-        // 2.10.2 TASKLOOP construct.
-        genTaskloopOp(converter, symTable, semaCtx, eval, currentLocation,
-                      clauses);
-        break;
-      case llvm::omp::Directive::OMPD_teams:
-        // 2.7 TEAMS construct.
-        // FIXME This is not necessarily always the outer leaf construct of a
-        // combined construct in this constext (e.g. target teams distribute).
-        // Maybe rename the argument if it represents something else or
-        // initialize it properly.
-        genTeamsOp(converter, symTable, semaCtx, eval, genNested,
-                   currentLocation, clauses, /*outerCombined=*/true);
-        break;
-      case llvm::omp::Directive::OMPD_loop:
-      case llvm::omp::Directive::OMPD_masked:
-      case llvm::omp::Directive::OMPD_master:
-      case llvm::omp::Directive::OMPD_tile:
-      case llvm::omp::Directive::OMPD_unroll:
-        TODO(currentLocation, "Unhandled loop directive (" +
-                                  llvm::omp::getOpenMPDirectiveName(leafDir) +
-                                  ")");
-        break;
-      default:
-        llvm_unreachable("Unexpected loop construct");
-      }
-    }
-  }
+  mlir::Location currentLocation =
+      converter.genLocation(beginLoopDirective.source);
+
+  llvm::omp::Directive directive =
+      std::get<parser::OmpLoopDirective>(beginLoopDirective.t).v;
+  const parser::CharBlock &source =
+      std::get<parser::OmpLoopDirective>(beginLoopDirective.t).source;
+  ConstructQueue queue{
+      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
+                          eval, source, directive, clauses)};
+  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
+                 queue.begin());
 }
 
 static void
@@ -2441,8 +2461,12 @@ genOMP(Fortran::lower::AbstractConverter &converter,
        Fortran::semantics::SemanticsContext &semaCtx,
        Fortran::lower::pft::Evaluation &eval,
        const Fortran::parser::OpenMPSectionConstruct &sectionConstruct) {
-  // SECTION constructs are handled as a part of SECTIONS.
-  llvm_unreachable("Unexpected standalone OMP SECTION");
+  mlir::Location loc = converter.getCurrentLocation();
+  ConstructQueue queue{buildConstructQueue(
+      converter.getFirOpBuilder().getModule(), semaCtx, eval,
+      sectionConstruct.source, llvm::omp::Directive::OMPD_section, {})};
+  genSectionOp(converter, symTable, semaCtx, eval, loc,
+               /*clauses=*/{}, queue, queue.begin());
 }
 
 static void
@@ -2461,77 +2485,17 @@ genOMP(Fortran::lower::AbstractConverter &converter,
   clauses.append(makeClauses(
       std::get<Fortran::parser::OmpClauseList>(endSectionsDirective.t),
       semaCtx));
-
-  // Process clauses before optional omp.parallel, so that new variables are
-  // allocated outside of the parallel region
   mlir::Location currentLocation = converter.getCurrentLocation();
-  mlir::omp::SectionsClauseOps clauseOps;
-  genSectionsClauses(converter, semaCtx, clauses, currentLocation, clauseOps);
-
-  // Parallel wrapper of PARALLEL SECTIONS construct
-  llvm::omp::Directive dir =
-      std::get<Fortran::parser::OmpSectionsDirective>(beginSectionsDirective.t)
-          .v;
-  if (dir == llvm::omp::Directive::OMPD_parallel_sections) {
-    genParallelOp(converter, symTable, semaCtx, eval,
-                  /*genNested=*/false, currentLocation, clauses,
-                  /*outerCombined=*/true);
-  }
-
-  // Insert privatizations before SECTIONS
-  symTable.pushScope();
-  DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
-  dsp.processStep1();
-
-  // SECTIONS construct.
-  mlir::omp::SectionsOp sectionsOp = genSectionsOp(
-      converter, symTable, semaCtx, eval, currentLocation, clauseOps);
-
-  // Generate nested SECTION operations recursively.
-  const auto &sectionBlocks =
-      std::get<Fortran::parser::OmpSectionBlocks>(sectionsConstruct.t);
-  auto &firOpBuilder = converter.getFirOpBuilder();
-  auto ip = firOpBuilder.saveInsertionPoint();
-  mlir::omp::SectionOp lastSectionOp;
-  for (const auto &[nblock, neval] :
-       llvm::zip(sectionBlocks.v, eval.getNestedEvaluations())) {
-    symTable.pushScope();
-    lastSectionOp = genSectionOp(converter, symTable, semaCtx, neval,
-                                 /*genNested=*/true, currentLocation);
-    symTable.popScope();
-    firOpBuilder.restoreInsertionPoint(ip);
-  }
-
-  // For `omp.sections`, lastprivatized variables occur in
-  // lexically final `omp.section` operation.
-  bool hasLastPrivate = false;
-  if (lastSectionOp) {
-    for (const Clause &clause : clauses) {
-      if (const auto *lastPrivate =
-              std::get_if<clause::Lastprivate>(&clause.u)) {
-        hasLastPrivate = true;
-        firOpBuilder.setInsertionPoint(
-            lastSectionOp.getRegion().back().getTerminator());
-        mlir::OpBuilder::InsertPoint lastPrivIP =
-            converter.getFirOpBuilder().saveInsertionPoint();
-        const auto &objList = std::get<1>(lastPrivate->t);
-        for (const Object &obj : objList) {
-          Fortran::semantics::Symbol *sym = obj.id();
-          converter.copyHostAssociateVar(*sym, &lastPrivIP);
-        }
-      }
-    }
-  }
 
-  // Perform DataSharingProcessor's step2 out of SECTIONS
-  firOpBuilder.setInsertionPointAfter(sectionsOp.getOperation());
-  dsp.processStep2(sectionsOp, false);
-  // Emit implicit barrier to synchronize threads and avoid data
-  // races on post-update of lastprivate variables when `nowait`
-  // clause is present.
-  if (clauseOps.nowaitAttr && hasLastPrivate)
-    firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
-  symTable.popScope();
+  llvm::omp::Directive directive =
+      std::get<parser::OmpSectionsDirective>(beginSectionsDirective.t).v;
+  const parser::CharBlock &source =
+      std::get<parser::OmpSectionsDirective>(beginSectionsDirective.t).source;
+  ConstructQueue queue{
+      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
+                          eval, source, directive, clauses)};
+  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
+                 queue.begin());
 }
 
 static void genOMP(Fortran::lower::AbstractConverter &converter,
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index eed63b226133..cb1d1a5a7f3d 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -51,12 +51,6 @@ int64_t getCollapseValue(const List<Clause> &clauses) {
   return 1;
 }
 
-uint32_t getOpenMPVersion(mlir::ModuleOp mod) {
-  if (mlir::Attribute verAttr = mod->getAttr("omp.version"))
-    return llvm::cast<mlir::omp::VersionAttr>(verAttr).getVersion();
-  llvm_unreachable("Expecting OpenMP version attribute in module");
-}
-
 void genObjectList(const ObjectList &objects,
                    Fortran::lower::AbstractConverter &converter,
                    llvm::SmallVectorImpl<mlir::Value> &operands) {
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 8fbb18fa8656..345ce55620ee 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -93,7 +93,6 @@ void gatherFuncAndVarSyms(
     llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause);
 
 int64_t getCollapseValue(const List<Clause> &clauses);
-uint32_t getOpenMPVersion(mlir::ModuleOp mod);
 
 Fortran::semantics::Symbol *
 getOmpObjectSymbol(const Fortran::parser::OmpObject &ompObject);
diff --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90
index 62ba67e5962f..7cc2bc2e0c71 100644
--- a/flang/test/Lower/OpenMP/default-clause-byref.f90
+++ b/flang/test/Lower/OpenMP/default-clause-byref.f90
@@ -161,12 +161,12 @@ subroutine nested_default_clause_tests
 !CHECK: %[[Z:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFnested_default_clause_testsEz"}
 !CHECK: %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z]] {uniq_name = "_QFnested_default_clause_testsEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel   {
+!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"}
+!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
 !CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
-!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"}
-!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_default_clause_testsEz"}
 !CHECK: %[[PRIVATE_Z_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Z]] {uniq_name = "_QFnested_default_clause_testsEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_K:.*]] = fir.alloca i32 {bindc_name = "k", pinned, uniq_name = "_QFnested_default_clause_testsEk"}
@@ -221,6 +221,7 @@ subroutine nested_default_clause_tests
     
     
 !CHECK: omp.parallel {
+!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"}
 !CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90
index a90f0f4ef5f8..843ee6bb7910 100644
--- a/flang/test/Lower/OpenMP/default-clause.f90
+++ b/flang/test/Lower/OpenMP/default-clause.f90
@@ -160,12 +160,12 @@ end program default_clause_lowering
 !CHECK: %[[Z:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFnested_default_clause_test1Ez"}
 !CHECK: %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z]] {uniq_name = "_QFnested_default_clause_test1Ez"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel   {
+!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_test1Ey"}
+!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_test1Ey"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_test1Ex"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_test1Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
 !CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
-!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_test1Ey"}
-!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_test1Ey"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_default_clause_test1Ez"}
 !CHECK: %[[PRIVATE_Z_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Z]] {uniq_name = "_QFnested_default_clause_test1Ez"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_K:.*]] = fir.alloca i32 {bindc_name = "k", pinned, uniq_name = "_QFnested_default_clause_test1Ek"}
diff --git a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
index b7f11c8c722f..e6ee75c8a5be 100644
--- a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
@@ -145,10 +145,10 @@ end subroutine
 !CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
-!CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
-!CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK-DAG: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2"
 !CHECK-DAG: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
+!CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.wsloop {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index daef02bcfc9a..07c95497b7a4 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -178,6 +178,12 @@ template <typename T> using ListT = llvm::SmallVector<T, 0>;
 // provide their own specialization that conforms to the above requirements.
 template <typename IdType, typename ExprType> struct ObjectT;
 
+// By default, object equality is only determined by its identity.
+template <typename I, typename E>
+bool operator==(const ObjectT<I, E> &o1, const ObjectT<I, E> &o2) {
+  return o1.id() == o2.id();
+}
+
 template <typename I, typename E> using ObjectListT = ListT<ObjectT<I, E>>;
 
 using DirectiveName = llvm::omp::Directive;
@@ -264,6 +270,32 @@ struct ReductionIdentifierT {
 
 template <typename T, typename I, typename E> //
 using IteratorT = ListT<IteratorSpecifierT<T, I, E>>;
+
+template <typename T>
+std::enable_if_t<T::EmptyTrait::value, bool> operator==(const T &a,
+                                                        const T &b) {
+  return true;
+}
+template <typename T>
+std::enable_if_t<T::IncompleteTrait::value, bool> operator==(const T &a,
+                                                             const T &b) {
+  return true;
+}
+template <typename T>
+std::enable_if_t<T::WrapperTrait::value, bool> operator==(const T &a,
+                                                          const T &b) {
+  return a.v == b.v;
+}
+template <typename T>
+std::enable_if_t<T::TupleTrait::value, bool> operator==(const T &a,
+                                                        const T &b) {
+  return a.t == b.t;
+}
+template <typename T>
+std::enable_if_t<T::UnionTrait::value, bool> operator==(const T &a,
+                                                        const T &b) {
+  return a.u == b.u;
+}
 } // namespace type
 
 template <typename T> using ListT = type::ListT<T>;
@@ -285,6 +317,8 @@ ListT<ResultTy> makeList(ContainerTy &&container, FunctionTy &&func) {
 }
 
 namespace clause {
+using type::operator==;
+
 // V5.2: [8.3.1] `assumption` clauses
 template <typename T, typename I, typename E> //
 struct AbsentT {
@@ -726,7 +760,7 @@ struct LinearT {
   ENUM(LinearModifier, Ref, Val, Uval);
 
   using TupleTrait = std::true_type;
-  // Step == nullptr means 1.
+  // Step == nullopt means 1.
   std::tuple<OPT(StepSimpleModifier), OPT(StepComplexModifier),
              OPT(LinearModifier), List>
       t;
@@ -1142,9 +1176,11 @@ struct UsesAllocatorsT {
   using MemSpace = E;
   using TraitsArray = ObjectT<I, E>;
   using Allocator = E;
-  using AllocatorSpec =
-      std::tuple<OPT(MemSpace), OPT(TraitsArray), Allocator>; // Not a spec name
-  using Allocators = ListT<AllocatorSpec>;                    // Not a spec name
+  struct AllocatorSpec { // Not a spec name
+    using TupleTrait = std::true_type;
+    std::tuple<OPT(MemSpace), OPT(TraitsArray), Allocator> t;
+  };
+  using Allocators = ListT<AllocatorSpec>; // Not a spec name
   using WrapperTrait = std::true_type;
   Allocators v;
 };
@@ -1232,9 +1268,10 @@ using UnionOfAllClausesT = typename type::Union< //
     UnionClausesT<T, I, E>,                      //
     WrapperClausesT<T, I, E>                     //
     >::type;
-
 } // namespace clause
 
+using type::operator==;
+
 // The variant wrapper that encapsulates all possible specific clauses.
 // The `Extras` arguments are additional types representing local extensions
 // to the clause set, e.g.
@@ -1260,6 +1297,11 @@ struct ClauseT {
   VariantTy u;
 };
 
+template <typename ClauseType> struct DirectiveWithClauses {
+  llvm::omp::Directive id = llvm::omp::Directive::OMPD_unknown;
+  tomp::type::ListT<ClauseType> clauses;
+};
+
 } // namespace tomp
 
 #undef OPT
diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
new file mode 100644
index 000000000000..7a4ed92a1070
--- /dev/null
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
@@ -0,0 +1,403 @@
+//===- ConstructCompositionT.h -- Composing compound constructs -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Given a list of leaf construct, each with a set of clauses, generate the
+// compound construct whose leaf constructs are the given list, and whose clause
+// list is the merged lists of individual leaf clauses.
+//
+// *** At the moment it assumes that the individual constructs and their clauses
+// *** are a subset of those created by splitting a valid compound construct.
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_FRONTEND_OPENMP_CONSTRUCTCOMPOSITIONT_H
+#define LLVM_FRONTEND_OPENMP_CONSTRUCTCOMPOSITIONT_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Frontend/OpenMP/ClauseT.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <iterator>
+#include <optional>
+#include <tuple>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+
+namespace tomp {
+template <typename ClauseType> struct ConstructCompositionT {
+  using ClauseTy = ClauseType;
+
+  using TypeTy = typename ClauseTy::TypeTy;
+  using IdTy = typename ClauseTy::IdTy;
+  using ExprTy = typename ClauseTy::ExprTy;
+
+  ConstructCompositionT(uint32_t version,
+                        llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> leafs);
+
+  DirectiveWithClauses<ClauseTy> merged;
+
+private:
+  // Use an ordered container, since we beed to maintain the order in which
+  // clauses are added to it. This is to avoid non-deterministic output.
+  using ClauseSet = ListT<ClauseTy>;
+
+  enum class Presence {
+    All,  // Clause is preesnt on all leaf constructs that allow it.
+    Some, // Clause is present on some, but not on all constructs.
+    None, // Clause is absent on all constructs.
+  };
+
+  template <typename S>
+  ClauseTy makeClause(llvm::omp::Clause clauseId, S &&specific) {
+    return ClauseTy{clauseId, std::move(specific)};
+  }
+
+  llvm::omp::Directive
+  makeCompound(llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> parts);
+
+  Presence checkPresence(llvm::omp::Clause clauseId);
+
+  // There are clauses that need special handling:
+  // 1. "if": the "directive-name-modifier" on the merged clause may need
+  // to be set appropriately.
+  // 2. "reduction": implies "privateness" of all objects (incompatible
+  // with "shared"); there are rules for merging modifiers
+  void mergeIf();
+  void mergeReduction();
+  void mergeDSA();
+
+  uint32_t version;
+  llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> leafs;
+
+  // clause id -> set of leaf constructs that contain it
+  std::unordered_map<llvm::omp::Clause, llvm::BitVector> clausePresence;
+  // clause id -> set of instances of that clause
+  std::unordered_map<llvm::omp::Clause, ClauseSet> clauseSets;
+};
+
+template <typename C>
+ConstructCompositionT<C>::ConstructCompositionT(
+    uint32_t version, llvm::ArrayRef<DirectiveWithClauses<C>> leafs)
+    : version(version), leafs(leafs) {
+  // Merge the list of constructs with clauses into a compound construct
+  // with a single list of clauses.
+  // The intended use of this function is in splitting compound constructs,
+  // while preserving composite constituent constructs:
+  // Step 1: split compound construct into leaf constructs.
+  // Step 2: identify composite sub-construct, and merge the constituent leafs.
+  //
+  // *** At the moment it assumes that the individual constructs and their
+  // *** clauses are a subset of those created by splitting a valid compound
+  // *** construct.
+  //
+  // 1. Deduplicate clauses
+  //    - exact duplicates: e.g. shared(x) shared(x) -> shared(x)
+  //    - special cases of clauses differing in modifier:
+  //      (a) reduction: inscan + (none|default) = inscan
+  //      (b) reduction: task + (none|default) = task
+  //      (c) combine repeated "if" clauses if possible
+  // 2. Merge DSA clauses: e.g. private(x) private(y) -> private(x, y).
+  // 3. Resolve potential DSA conflicts (typically due to implied clauses).
+
+  if (leafs.empty())
+    return;
+
+  merged.id = makeCompound(leafs);
+
+  // Populate the two maps:
+  for (const auto &[index, leaf] : llvm::enumerate(leafs)) {
+    for (const auto &clause : leaf.clauses) {
+      // Update clausePresence.
+      auto &pset = clausePresence[clause.id];
+      if (pset.size() < leafs.size())
+        pset.resize(leafs.size());
+      pset.set(index);
+      // Update clauseSets.
+      ClauseSet &cset = clauseSets[clause.id];
+      if (!llvm::is_contained(cset, clause))
+        cset.push_back(clause);
+    }
+  }
+
+  mergeIf();
+  mergeReduction();
+  mergeDSA();
+
+  // Fir the rest of the clauses, just copy them.
+  for (auto &[id, clauses] : clauseSets) {
+    // Skip clauses we've already dealt with.
+    switch (id) {
+    case llvm::omp::Clause::OMPC_if:
+    case llvm::omp::Clause::OMPC_reduction:
+    case llvm::omp::Clause::OMPC_shared:
+    case llvm::omp::Clause::OMPC_private:
+    case llvm::omp::Clause::OMPC_firstprivate:
+    case llvm::omp::Clause::OMPC_lastprivate:
+      continue;
+    default:
+      break;
+    }
+    llvm::append_range(merged.clauses, clauses);
+  }
+}
+
+template <typename C>
+llvm::omp::Directive ConstructCompositionT<C>::makeCompound(
+    llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> parts) {
+  llvm::SmallVector<llvm::omp::Directive> dirIds;
+  llvm::transform(parts, std::back_inserter(dirIds),
+                  [](auto &&dwc) { return dwc.id; });
+
+  return llvm::omp::getCompoundConstruct(dirIds);
+}
+
+template <typename C>
+auto ConstructCompositionT<C>::checkPresence(llvm::omp::Clause clauseId)
+    -> Presence {
+  auto found = clausePresence.find(clauseId);
+  if (found == clausePresence.end())
+    return Presence::None;
+
+  bool OnAll = true, OnNone = true;
+  for (const auto &[index, leaf] : llvm::enumerate(leafs)) {
+    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, clauseId, version))
+      continue;
+
+    if (found->second.test(index))
+      OnNone = false;
+    else
+      OnAll = false;
+  }
+
+  if (OnNone)
+    return Presence::None;
+  if (OnAll)
+    return Presence::All;
+  return Presence::Some;
+}
+
+template <typename C> void ConstructCompositionT<C>::mergeIf() {
+  using IfTy = tomp::clause::IfT<TypeTy, IdTy, ExprTy>;
+  // Deal with the "if" clauses. If it's on all leafs that allow it, then it
+  // will apply to the compound construct. Otherwise it will apply to the
+  // single (assumed) leaf construct.
+  // This assumes that the "if" clauses have the same expression.
+  Presence presence = checkPresence(llvm::omp::Clause::OMPC_if);
+  if (presence == Presence::None)
+    return;
+
+  const ClauseTy &some = *clauseSets[llvm::omp::Clause::OMPC_if].begin();
+  const auto &someIf = std::get<IfTy>(some.u);
+
+  if (presence == Presence::All) {
+    // Create "if" without "directive-name-modifier".
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_if,
+                   IfTy{{/*DirectiveNameModifier=*/std::nullopt,
+                         /*IfExpression=*/std::get<typename IfTy::IfExpression>(
+                             someIf.t)}}));
+  } else {
+    // Find out where it's present and create "if" with the corresponding
+    // "directive-name-modifier".
+    int Idx = clausePresence[llvm::omp::Clause::OMPC_if].find_first();
+    assert(Idx >= 0);
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_if,
+                   IfTy{{/*DirectiveNameModifier=*/leafs[Idx].id,
+                         /*IfExpression=*/std::get<typename IfTy::IfExpression>(
+                             someIf.t)}}));
+  }
+}
+
+template <typename C> void ConstructCompositionT<C>::mergeReduction() {
+  Presence presence = checkPresence(llvm::omp::Clause::OMPC_reduction);
+  if (presence == Presence::None)
+    return;
+
+  using ReductionTy = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
+  using ModifierTy = typename ReductionTy::ReductionModifier;
+  using IdentifiersTy = typename ReductionTy::ReductionIdentifiers;
+  using ListTy = typename ReductionTy::List;
+  // There are exceptions on which constructs "reduction" may appear
+  // (specifically "parallel", and "teams"). Assume that if "reduction"
+  // is present, it can be applied to the compound construct.
+
+  // What's left is to see if there are any modifiers present. Again,
+  // assume that there are no conflicting modifiers.
+  // There can be, however, multiple reductions on different objects.
+  auto equal = [](const ClauseTy &red1, const ClauseTy &red2) {
+    // Extract actual reductions.
+    const auto r1 = std::get<ReductionTy>(red1.u);
+    const auto r2 = std::get<ReductionTy>(red2.u);
+    // Compare everything except modifiers.
+    if (std::get<IdentifiersTy>(r1.t) != std::get<IdentifiersTy>(r2.t))
+      return false;
+    if (std::get<ListTy>(r1.t) != std::get<ListTy>(r2.t))
+      return false;
+    return true;
+  };
+
+  auto getModifier = [](const ClauseTy &clause) {
+    const ReductionTy &red = std::get<ReductionTy>(clause.u);
+    return std::get<std::optional<ModifierTy>>(red.t);
+  };
+
+  const ClauseSet &reductions = clauseSets[llvm::omp::Clause::OMPC_reduction];
+  std::unordered_set<const ClauseTy *> visited;
+  while (reductions.size() != visited.size()) {
+    typename ClauseSet::const_iterator first;
+
+    // Find first non-visited reduction.
+    for (first = reductions.begin(); first != reductions.end(); ++first) {
+      if (visited.count(&*first))
+        continue;
+      visited.insert(&*first);
+      break;
+    }
+
+    std::optional<ModifierTy> modifier = getModifier(*first);
+
+    // Visit all other reductions that are "equal" (with respect to the
+    // definition above) to "first". Collect modifiers.
+    for (auto iter = std::next(first); iter != reductions.end(); ++iter) {
+      if (!equal(*first, *iter))
+        continue;
+      visited.insert(&*iter);
+      if (!modifier || *modifier == ModifierTy::Default)
+        modifier = getModifier(*iter);
+    }
+
+    const auto &firstRed = std::get<ReductionTy>(first->u);
+    merged.clauses.emplace_back(makeClause(
+        llvm::omp::Clause::OMPC_reduction,
+        ReductionTy{
+            {/*ReductionModifier=*/modifier,
+             /*ReductionIdentifiers=*/std::get<IdentifiersTy>(firstRed.t),
+             /*List=*/std::get<ListTy>(firstRed.t)}}));
+  }
+}
+
+template <typename C> void ConstructCompositionT<C>::mergeDSA() {
+  using ObjectTy = tomp::type::ObjectT<IdTy, ExprTy>;
+
+  // Resolve data-sharing attributes.
+  enum DSA : int {
+    None = 0,
+    Shared = 1 << 0,
+    Private = 1 << 1,
+    FirstPrivate = 1 << 2,
+    LastPrivate = 1 << 3,
+    LastPrivateConditional = 1 << 4,
+  };
+
+  // Use ordered containers to avoid non-deterministic output.
+  llvm::SmallVector<std::pair<ObjectTy, int>> objectDsa;
+
+  auto getDsa = [&](const ObjectTy &object) -> std::pair<ObjectTy, int> & {
+    auto found = llvm::find_if(objectDsa, [&](std::pair<ObjectTy, int> &p) {
+      return p.first.id() == object.id();
+    });
+    if (found != objectDsa.end())
+      return *found;
+    return objectDsa.emplace_back(object, DSA::None);
+  };
+
+  using SharedTy = tomp::clause::SharedT<TypeTy, IdTy, ExprTy>;
+  using PrivateTy = tomp::clause::PrivateT<TypeTy, IdTy, ExprTy>;
+  using FirstprivateTy = tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>;
+  using LastprivateTy = tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>;
+
+  // Visit clauses that affect DSA.
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_shared]) {
+    for (auto &object : std::get<SharedTy>(clause.u).v)
+      getDsa(object).second |= DSA::Shared;
+  }
+
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_private]) {
+    for (auto &object : std::get<PrivateTy>(clause.u).v)
+      getDsa(object).second |= DSA::Private;
+  }
+
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_firstprivate]) {
+    for (auto &object : std::get<FirstprivateTy>(clause.u).v)
+      getDsa(object).second |= DSA::FirstPrivate;
+  }
+
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_lastprivate]) {
+    using ModifierTy = typename LastprivateTy::LastprivateModifier;
+    using ListTy = typename LastprivateTy::List;
+    const auto &lastp = std::get<LastprivateTy>(clause.u);
+    for (auto &object : std::get<ListTy>(lastp.t)) {
+      auto &mod = std::get<std::optional<ModifierTy>>(lastp.t);
+      if (mod && *mod == ModifierTy::Conditional) {
+        getDsa(object).second |= DSA::LastPrivateConditional;
+      } else {
+        getDsa(object).second |= DSA::LastPrivate;
+      }
+    }
+  }
+
+  // Check reductions as well, clear "shared" if set.
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_reduction]) {
+    using ReductionTy = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
+    using ListTy = typename ReductionTy::List;
+    for (auto &object : std::get<ListTy>(std::get<ReductionTy>(clause.u).t))
+      getDsa(object).second &= ~DSA::Shared;
+  }
+
+  tomp::ListT<ObjectTy> privateObj, sharedObj, firstpObj, lastpObj, lastpcObj;
+  for (auto &[object, dsa] : objectDsa) {
+    if (dsa &
+        (DSA::FirstPrivate | DSA::LastPrivate | DSA::LastPrivateConditional)) {
+      if (dsa & DSA::FirstPrivate)
+        firstpObj.push_back(object); // no else
+      if (dsa & DSA::LastPrivateConditional)
+        lastpcObj.push_back(object);
+      else if (dsa & DSA::LastPrivate)
+        lastpObj.push_back(object);
+    } else if (dsa & DSA::Private) {
+      privateObj.push_back(object);
+    } else if (dsa & DSA::Shared) {
+      sharedObj.push_back(object);
+    }
+  }
+
+  // Materialize each clause.
+  if (!privateObj.empty()) {
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_private,
+                   PrivateTy{/*List=*/std::move(privateObj)}));
+  }
+  if (!sharedObj.empty()) {
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_shared,
+                   SharedTy{/*List=*/std::move(sharedObj)}));
+  }
+  if (!firstpObj.empty()) {
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_firstprivate,
+                   FirstprivateTy{/*List=*/std::move(firstpObj)}));
+  }
+  if (!lastpObj.empty()) {
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_lastprivate,
+                   LastprivateTy{{/*LastprivateModifier=*/std::nullopt,
+                                  /*List=*/std::move(lastpObj)}}));
+  }
+  if (!lastpcObj.empty()) {
+    auto conditional = LastprivateTy::LastprivateModifier::Conditional;
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_lastprivate,
+                   LastprivateTy{{/*LastprivateModifier=*/conditional,
+                                  /*List=*/std::move(lastpcObj)}}));
+  }
+}
+} // namespace tomp
+
+#endif // LLVM_FRONTEND_OPENMP_CONSTRUCTCOMPOSITIONT_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
new file mode 100644
index 000000000000..37c88f0fa07b
--- /dev/null
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
@@ -0,0 +1,1161 @@
+//===- ConstructDecompositionT.h -- Decomposing compound constructs -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Given a compound construct with a set of clauses, generate the list of
+// constituent leaf constructs, each with a list of clauses that apply to it.
+//
+// Note: Clauses that are not originally present, but that are implied by the
+// OpenMP spec are materialized, and are present in the output.
+//
+// Note: Composite constructs will also be broken up into leaf constructs.
+// If composite constructs require processing as a whole, the lists of clauses
+// for each leaf constituent should be merged.
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_FRONTEND_OPENMP_CONSTRUCTDECOMPOSITIONT_H
+#define LLVM_FRONTEND_OPENMP_CONSTRUCTDECOMPOSITIONT_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/Frontend/OpenMP/ClauseT.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <iterator>
+#include <list>
+#include <optional>
+#include <tuple>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <variant>
+
+static inline llvm::ArrayRef<llvm::omp::Directive> getWorksharing() {
+  static llvm::omp::Directive worksharing[] = {
+      llvm::omp::Directive::OMPD_do,     llvm::omp::Directive::OMPD_for,
+      llvm::omp::Directive::OMPD_scope,  llvm::omp::Directive::OMPD_sections,
+      llvm::omp::Directive::OMPD_single, llvm::omp::Directive::OMPD_workshare,
+  };
+  return worksharing;
+}
+
+static inline llvm::ArrayRef<llvm::omp::Directive> getWorksharingLoop() {
+  static llvm::omp::Directive worksharingLoop[] = {
+      llvm::omp::Directive::OMPD_do,
+      llvm::omp::Directive::OMPD_for,
+  };
+  return worksharingLoop;
+}
+
+namespace detail {
+template <typename Container, typename Predicate>
+typename std::remove_reference_t<Container>::iterator
+find_unique(Container &&container, Predicate &&pred) {
+  auto first = std::find_if(container.begin(), container.end(), pred);
+  if (first == container.end())
+    return first;
+  auto second = std::find_if(std::next(first), container.end(), pred);
+  if (second == container.end())
+    return first;
+  return container.end();
+}
+
+} // namespace detail
+
+namespace tomp {
+
+// ClauseType - Either instance of ClauseT, or a type derived from ClauseT.
+//
+// This is the clause representation in the code using this infrastructure.
+//
+// HelperType - A class that implements two member functions:
+//
+//   // Return the base object of the given object, if any.
+//   std::optional<Object> getBaseObject(const Object &object) const
+//   // Return the iteration variable of the outermost loop associated
+//   // with the construct being worked on, if any.
+//   std::optional<Object> getLoopIterVar() const
+template <typename ClauseType, typename HelperType>
+struct ConstructDecompositionT {
+  using ClauseTy = ClauseType;
+
+  using TypeTy = typename ClauseTy::TypeTy;
+  using IdTy = typename ClauseTy::IdTy;
+  using ExprTy = typename ClauseTy::ExprTy;
+  using HelperTy = HelperType;
+  using ObjectTy = tomp::ObjectT<IdTy, ExprTy>;
+
+  using ClauseSet = std::unordered_set<const ClauseTy *>;
+
+  ConstructDecompositionT(uint32_t ver, HelperType &helper,
+                          llvm::omp::Directive dir,
+                          llvm::ArrayRef<ClauseTy> clauses)
+      : version(ver), construct(dir), helper(helper) {
+    for (const ClauseTy &clause : clauses)
+      nodes.push_back(&clause);
+
+    bool success = split();
+    if (!success)
+      return;
+
+    // Copy the individual leaf directives with their clauses to the
+    // output list. Copy by value, since we don't own the storage
+    // with the input clauses, and the internal representation uses
+    // clause addresses.
+    for (auto &leaf : leafs) {
+      output.push_back({leaf.id});
+      auto &out = output.back();
+      for (const ClauseTy *c : leaf.clauses)
+        out.clauses.push_back(*c);
+    }
+  }
+
+  tomp::ListT<DirectiveWithClauses<ClauseType>> output;
+
+private:
+  bool split();
+
+  struct LeafReprInternal {
+    llvm::omp::Directive id = llvm::omp::Directive::OMPD_unknown;
+    tomp::type::ListT<const ClauseTy *> clauses;
+  };
+
+  LeafReprInternal *findDirective(llvm::omp::Directive dirId) {
+    auto found = llvm::find_if(
+        leafs, [&](const LeafReprInternal &leaf) { return leaf.id == dirId; });
+    return found != leafs.end() ? &*found : nullptr;
+  }
+
+  ClauseSet *findClausesWith(const ObjectTy &object) {
+    if (auto found = syms.find(object.id()); found != syms.end())
+      return &found->second;
+    return nullptr;
+  }
+
+  template <typename S>
+  ClauseTy *makeClause(llvm::omp::Clause clauseId, S &&specific) {
+    implicit.push_back(ClauseTy{clauseId, std::move(specific)});
+    return &implicit.back();
+  }
+
+  void addClauseSymsToMap(const ObjectTy &object, const ClauseTy *);
+  void addClauseSymsToMap(const tomp::ObjectListT<IdTy, ExprTy> &objects,
+                          const ClauseTy *);
+  void addClauseSymsToMap(const TypeTy &item, const ClauseTy *);
+  void addClauseSymsToMap(const ExprTy &item, const ClauseTy *);
+  void addClauseSymsToMap(const tomp::clause::MapT<TypeTy, IdTy, ExprTy> &item,
+                          const ClauseTy *);
+
+  template <typename U>
+  void addClauseSymsToMap(const std::optional<U> &item, const ClauseTy *);
+  template <typename U>
+  void addClauseSymsToMap(const tomp::ListT<U> &item, const ClauseTy *);
+  template <typename... U, size_t... Is>
+  void addClauseSymsToMap(const std::tuple<U...> &item, const ClauseTy *,
+                          std::index_sequence<Is...> = {});
+  template <typename U>
+  std::enable_if_t<std::is_enum_v<llvm::remove_cvref_t<U>>, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  template <typename U>
+  std::enable_if_t<llvm::remove_cvref_t<U>::EmptyTrait::value, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  template <typename U>
+  std::enable_if_t<llvm::remove_cvref_t<U>::IncompleteTrait::value, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  template <typename U>
+  std::enable_if_t<llvm::remove_cvref_t<U>::WrapperTrait::value, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  template <typename U>
+  std::enable_if_t<llvm::remove_cvref_t<U>::TupleTrait::value, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  template <typename U>
+  std::enable_if_t<llvm::remove_cvref_t<U>::UnionTrait::value, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  // Apply a clause to the only directive that allows it. If there are no
+  // directives that allow it, or if there is more that one, do not apply
+  // anything and return false, otherwise return true.
+  bool applyToUnique(const ClauseTy *node);
+
+  // Apply a clause to the first directive in given range that allows it.
+  // If such a directive does not exist, return false, otherwise return true.
+  template <typename Iterator>
+  bool applyToFirst(const ClauseTy *node, llvm::iterator_range<Iterator> range);
+
+  // Apply a clause to the innermost directive that allows it. If such a
+  // directive does not exist, return false, otherwise return true.
+  bool applyToInnermost(const ClauseTy *node);
+
+  // Apply a clause to the outermost directive that allows it. If such a
+  // directive does not exist, return false, otherwise return true.
+  bool applyToOutermost(const ClauseTy *node);
+
+  template <typename Predicate>
+  bool applyIf(const ClauseTy *node, Predicate shouldApply);
+
+  bool applyToAll(const ClauseTy *node);
+
+  template <typename Clause>
+  bool applyClause(Clause &&clause, const ClauseTy *node);
+
+  bool applyClause(const tomp::clause::CollapseT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::PrivateT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool
+  applyClause(const tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy> &clause,
+              const ClauseTy *);
+  bool
+  applyClause(const tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy> &clause,
+              const ClauseTy *);
+  bool applyClause(const tomp::clause::SharedT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::DefaultT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool
+  applyClause(const tomp::clause::ThreadLimitT<TypeTy, IdTy, ExprTy> &clause,
+              const ClauseTy *);
+  bool applyClause(const tomp::clause::OrderT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::AllocateT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::ReductionT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::IfT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::LinearT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::NowaitT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+
+  uint32_t version;
+  llvm::omp::Directive construct;
+  HelperType &helper;
+  ListT<LeafReprInternal> leafs;
+  tomp::ListT<const ClauseTy *> nodes;
+  std::list<ClauseTy> implicit; // Container for materialized implicit clauses.
+                                // Inserting must preserve element addresses.
+  std::unordered_map<IdTy, ClauseSet> syms;
+  std::unordered_set<IdTy> mapBases;
+};
+
+// Deduction guide
+template <typename ClauseType, typename HelperType>
+ConstructDecompositionT(uint32_t, HelperType &, llvm::omp::Directive,
+                        llvm::ArrayRef<ClauseType>)
+    -> ConstructDecompositionT<ClauseType, HelperType>;
+
+template <typename C, typename H>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(const ObjectTy &object,
+                                                       const ClauseTy *node) {
+  syms[object.id()].insert(node);
+}
+
+template <typename C, typename H>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(
+    const tomp::ObjectListT<IdTy, ExprTy> &objects, const ClauseTy *node) {
+  for (auto &object : objects)
+    syms[object.id()].insert(node);
+}
+
+template <typename C, typename H>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(const TypeTy &item,
+                                                       const ClauseTy *node) {
+  // Nothing to do for types.
+}
+
+template <typename C, typename H>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(const ExprTy &item,
+                                                       const ClauseTy *node) {
+  // Nothing to do for expressions.
+}
+
+template <typename C, typename H>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(
+    const tomp::clause::MapT<TypeTy, IdTy, ExprTy> &item,
+    const ClauseTy *node) {
+  auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(item.t);
+  addClauseSymsToMap(objects, node);
+  for (auto &object : objects) {
+    if (auto base = helper.getBaseObject(object))
+      mapBases.insert(base->id());
+  }
+}
+
+template <typename C, typename H>
+template <typename U>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(
+    const std::optional<U> &item, const ClauseTy *node) {
+  if (item)
+    addClauseSymsToMap(*item, node);
+}
+
+template <typename C, typename H>
+template <typename U>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(
+    const tomp::ListT<U> &item, const ClauseTy *node) {
+  for (auto &s : item)
+    addClauseSymsToMap(s, node);
+}
+
+template <typename C, typename H>
+template <typename... U, size_t... Is>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(
+    const std::tuple<U...> &item, const ClauseTy *node,
+    std::index_sequence<Is...>) {
+  (void)node; // Silence strange warning from GCC.
+  (addClauseSymsToMap(std::get<Is>(item), node), ...);
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<std::is_enum_v<llvm::remove_cvref_t<U>>, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  // Nothing to do for enums.
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<llvm::remove_cvref_t<U>::EmptyTrait::value, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  // Nothing to do for an empty class.
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<llvm::remove_cvref_t<U>::IncompleteTrait::value, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  // Nothing to do for an incomplete class (they're empty).
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<llvm::remove_cvref_t<U>::WrapperTrait::value, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  addClauseSymsToMap(item.v, node);
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<llvm::remove_cvref_t<U>::TupleTrait::value, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  constexpr size_t tuple_size =
+      std::tuple_size_v<llvm::remove_cvref_t<decltype(item.t)>>;
+  addClauseSymsToMap(item.t, node, std::make_index_sequence<tuple_size>{});
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<llvm::remove_cvref_t<U>::UnionTrait::value, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  std::visit([&](auto &&s) { addClauseSymsToMap(s, node); }, item.u);
+}
+
+// Apply a clause to the only directive that allows it. If there are no
+// directives that allow it, or if there is more that one, do not apply
+// anything and return false, otherwise return true.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyToUnique(const ClauseTy *node) {
+  auto unique = detail::find_unique(leafs, [=](const auto &dirInfo) {
+    return llvm::omp::isAllowedClauseForDirective(dirInfo.id, node->id,
+                                                  version);
+  });
+
+  if (unique != leafs.end()) {
+    unique->clauses.push_back(node);
+    return true;
+  }
+  return false;
+}
+
+// Apply a clause to the first directive in given range that allows it.
+// If such a directive does not exist, return false, otherwise return true.
+template <typename C, typename H>
+template <typename Iterator>
+bool ConstructDecompositionT<C, H>::applyToFirst(
+    const ClauseTy *node, llvm::iterator_range<Iterator> range) {
+  if (range.empty())
+    return false;
+
+  for (auto &leaf : range) {
+    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, node->id, version))
+      continue;
+    leaf.clauses.push_back(node);
+    return true;
+  }
+  return false;
+}
+
+// Apply a clause to the innermost directive that allows it. If such a
+// directive does not exist, return false, otherwise return true.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyToInnermost(const ClauseTy *node) {
+  return applyToFirst(node, llvm::reverse(leafs));
+}
+
+// Apply a clause to the outermost directive that allows it. If such a
+// directive does not exist, return false, otherwise return true.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyToOutermost(const ClauseTy *node) {
+  return applyToFirst(node, llvm::iterator_range(leafs));
+}
+
+template <typename C, typename H>
+template <typename Predicate>
+bool ConstructDecompositionT<C, H>::applyIf(const ClauseTy *node,
+                                            Predicate shouldApply) {
+  bool applied = false;
+  for (auto &leaf : leafs) {
+    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, node->id, version))
+      continue;
+    if (!shouldApply(leaf))
+      continue;
+    leaf.clauses.push_back(node);
+    applied = true;
+  }
+
+  return applied;
+}
+
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyToAll(const ClauseTy *node) {
+  return applyIf(node, [](auto) { return true; });
+}
+
+template <typename C, typename H>
+template <typename Clause>
+bool ConstructDecompositionT<C, H>::applyClause(Clause &&clause,
+                                                const ClauseTy *node) {
+  // The default behavior is to find the unique directive to which the
+  // given clause may be applied. If there are no such directives, or
+  // if there are multiple ones, flag an error.
+  // From "OpenMP Application Programming Interface", Version 5.2:
+  // S Some clauses are permitted only on a single leaf construct of the
+  // S combined or composite construct, in which case the effect is as if
+  // S the clause is applied to that specific construct. (p339, 31-33)
+  if (applyToUnique(node))
+    return true;
+
+  return false;
+}
+
+// COLLAPSE
+// [5.2:93:20-21]
+// Directives: distribute, do, for, loop, simd, taskloop
+//
+// [5.2:339:35]
+// (35) The collapse clause is applied once to the combined or composite
+// construct.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::CollapseT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // Apply "collapse" to the innermost directive. If it's not one that
+  // allows it flag an error.
+  if (!leafs.empty()) {
+    auto &last = leafs.back();
+
+    if (llvm::omp::isAllowedClauseForDirective(last.id, node->id, version)) {
+      last.clauses.push_back(node);
+      return true;
+    }
+  }
+
+  return false;
+}
+
+// PRIVATE
+// [5.2:111:5-7]
+// Directives: distribute, do, for, loop, parallel, scope, sections, simd,
+// single, target, task, taskloop, teams
+//
+// [5.2:340:1-2]
+// (1) The effect of the 1 private clause is as if it is applied only to the
+// innermost leaf construct that permits it.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::PrivateT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  return applyToInnermost(node);
+}
+
+// FIRSTPRIVATE
+// [5.2:112:5-7]
+// Directives: distribute, do, for, parallel, scope, sections, single, target,
+// task, taskloop, teams
+//
+// [5.2:340:3-20]
+// (3) The effect of the firstprivate clause is as if it is applied to one or
+// more leaf constructs as follows:
+//  (5) To the distribute construct if it is among the constituent constructs;
+//  (6) To the teams construct if it is among the constituent constructs and the
+//      distribute construct is not;
+//  (8) To a worksharing construct that accepts the clause if one is among the
+//      constituent constructs;
+//  (9) To the taskloop construct if it is among the constituent constructs;
+// (10) To the parallel construct if it is among the constituent constructs and
+//      neither a taskloop construct nor a worksharing construct that accepts
+//      the clause is among them;
+// (12) To the target construct if it is among the constituent constructs and
+//      the same list item neither appears in a lastprivate clause nor is the
+//      base variable or base pointer of a list item that appears in a map
+//      clause.
+//
+// (15) If the parallel construct is among the constituent constructs and the
+// effect is not as if the firstprivate clause is applied to it by the above
+// rules, then the effect is as if the shared clause with the same list item is
+// applied to the parallel construct.
+// (17) If the teams construct is among the constituent constructs and the
+// effect is not as if the firstprivate clause is applied to it by the above
+// rules, then the effect is as if the shared clause with the same list item is
+// applied to the teams construct.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  bool applied = false;
+
+  // [5.2:340:3-6]
+  auto dirDistribute = findDirective(llvm::omp::OMPD_distribute);
+  auto dirTeams = findDirective(llvm::omp::OMPD_teams);
+  if (dirDistribute != nullptr) {
+    dirDistribute->clauses.push_back(node);
+    applied = true;
+    // [5.2:340:17]
+    if (dirTeams != nullptr) {
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/clause.v});
+      dirTeams->clauses.push_back(shared);
+    }
+  } else if (dirTeams != nullptr) {
+    dirTeams->clauses.push_back(node);
+    applied = true;
+  }
+
+  // [5.2:340:8]
+  auto findWorksharing = [&]() {
+    auto worksharing = getWorksharing();
+    for (auto &leaf : leafs) {
+      auto found = llvm::find(worksharing, leaf.id);
+      if (found != std::end(worksharing))
+        return &leaf;
+    }
+    return static_cast<typename decltype(leafs)::value_type *>(nullptr);
+  };
+
+  auto dirWorksharing = findWorksharing();
+  if (dirWorksharing != nullptr) {
+    dirWorksharing->clauses.push_back(node);
+    applied = true;
+  }
+
+  // [5.2:340:9]
+  auto dirTaskloop = findDirective(llvm::omp::OMPD_taskloop);
+  if (dirTaskloop != nullptr) {
+    dirTaskloop->clauses.push_back(node);
+    applied = true;
+  }
+
+  // [5.2:340:10]
+  auto dirParallel = findDirective(llvm::omp::OMPD_parallel);
+  if (dirParallel != nullptr) {
+    if (dirTaskloop == nullptr && dirWorksharing == nullptr) {
+      dirParallel->clauses.push_back(node);
+      applied = true;
+    } else {
+      // [5.2:340:15]
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/clause.v});
+      dirParallel->clauses.push_back(shared);
+    }
+  }
+
+  // [5.2:340:12]
+  auto inLastprivate = [&](const ObjectTy &object) {
+    if (ClauseSet *set = findClausesWith(object)) {
+      return llvm::find_if(*set, [](const ClauseTy *c) {
+               return c->id == llvm::omp::Clause::OMPC_lastprivate;
+             }) != set->end();
+    }
+    return false;
+  };
+
+  auto dirTarget = findDirective(llvm::omp::OMPD_target);
+  if (dirTarget != nullptr) {
+    tomp::ObjectListT<IdTy, ExprTy> objects;
+    llvm::copy_if(
+        clause.v, std::back_inserter(objects), [&](const ObjectTy &object) {
+          return !inLastprivate(object) && !mapBases.count(object.id());
+        });
+    if (!objects.empty()) {
+      auto *firstp = makeClause(
+          llvm::omp::Clause::OMPC_firstprivate,
+          tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>{/*List=*/objects});
+      dirTarget->clauses.push_back(firstp);
+      applied = true;
+    }
+  }
+
+  // "task" is not handled by any of the cases above.
+  if (auto dirTask = findDirective(llvm::omp::OMPD_task)) {
+    dirTask->clauses.push_back(node);
+    applied = true;
+  }
+
+  return applied;
+}
+
+// LASTPRIVATE
+// [5.2:115:7-8]
+// Directives: distribute, do, for, loop, sections, simd, taskloop
+//
+// [5.2:340:21-30]
+// (21) The effect of the lastprivate clause is as if it is applied to all leaf
+// constructs that permit the clause.
+// (22) If the parallel construct is among the constituent constructs and the
+// list item is not also specified in the firstprivate clause, then the effect
+// of the lastprivate clause is as if the shared clause with the same list item
+// is applied to the parallel construct.
+// (24) If the teams construct is among the constituent constructs and the list
+// item is not also specified in the firstprivate clause, then the effect of the
+// lastprivate clause is as if the shared clause with the same list item is
+// applied to the teams construct.
+// (27) If the target construct is among the constituent constructs and the list
+// item is not the base variable or base pointer of a list item that appears in
+// a map clause, the effect of the lastprivate clause is as if the same list
+// item appears in a map clause with a map-type of tofrom.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  bool applied = false;
+
+  // [5.2:340:21]
+  applied = applyToAll(node);
+  if (!applied)
+    return false;
+
+  auto inFirstprivate = [&](const ObjectTy &object) {
+    if (ClauseSet *set = findClausesWith(object)) {
+      return llvm::find_if(*set, [](const ClauseTy *c) {
+               return c->id == llvm::omp::Clause::OMPC_firstprivate;
+             }) != set->end();
+    }
+    return false;
+  };
+
+  auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(clause.t);
+
+  // Prepare list of objects that could end up in a "shared" clause.
+  tomp::ObjectListT<IdTy, ExprTy> sharedObjects;
+  llvm::copy_if(
+      objects, std::back_inserter(sharedObjects),
+      [&](const ObjectTy &object) { return !inFirstprivate(object); });
+
+  if (!sharedObjects.empty()) {
+    // [5.2:340:22]
+    if (auto dirParallel = findDirective(llvm::omp::OMPD_parallel)) {
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
+      dirParallel->clauses.push_back(shared);
+      applied = true;
+    }
+
+    // [5.2:340:24]
+    if (auto dirTeams = findDirective(llvm::omp::OMPD_teams)) {
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
+      dirTeams->clauses.push_back(shared);
+      applied = true;
+    }
+  }
+
+  // [5.2:340:27]
+  if (auto dirTarget = findDirective(llvm::omp::OMPD_target)) {
+    tomp::ObjectListT<IdTy, ExprTy> tofrom;
+    llvm::copy_if(
+        objects, std::back_inserter(tofrom),
+        [&](const ObjectTy &object) { return !mapBases.count(object.id()); });
+
+    if (!tofrom.empty()) {
+      using MapType =
+          typename tomp::clause::MapT<TypeTy, IdTy, ExprTy>::MapType;
+      auto *map =
+          makeClause(llvm::omp::Clause::OMPC_map,
+                     tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
+                         {/*MapType=*/MapType::Tofrom,
+                          /*MapTypeModifier=*/std::nullopt,
+                          /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
+                          /*LocatorList=*/std::move(tofrom)}});
+      dirTarget->clauses.push_back(map);
+      applied = true;
+    }
+  }
+
+  return applied;
+}
+
+// SHARED
+// [5.2:110:5-6]
+// Directives: parallel, task, taskloop, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::SharedT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // [5.2:340:31]
+  return applyToAll(node);
+}
+
+// DEFAULT
+// [5.2:109:5-6]
+// Directives: parallel, task, taskloop, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::DefaultT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // [5.2:340:31]
+  return applyToAll(node);
+}
+
+// THREAD_LIMIT
+// [5.2:277:14-15]
+// Directives: target, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::ThreadLimitT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // [5.2:340:31]
+  return applyToAll(node);
+}
+
+// ORDER
+// [5.2:234:3-4]
+// Directives: distribute, do, for, loop, simd
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::OrderT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // [5.2:340:31]
+  return applyToAll(node);
+}
+
+// ALLOCATE
+// [5.2:178:7-9]
+// Directives: allocators, distribute, do, for, parallel, scope, sections,
+// single, target, task, taskgroup, taskloop, teams
+//
+// [5.2:340:33-35]
+// (33) The effect of the allocate clause is as if it is applied to all leaf
+// constructs that permit the clause and to which a data-sharing attribute
+// clause that may create a private copy of the same list item is applied.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::AllocateT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // This one needs to be applied at the end, once we know which clauses are
+  // assigned to which leaf constructs.
+
+  // [5.2:340:33]
+  auto canMakePrivateCopy = [](llvm::omp::Clause id) {
+    switch (id) {
+    case llvm::omp::Clause::OMPC_firstprivate:
+    case llvm::omp::Clause::OMPC_lastprivate:
+    case llvm::omp::Clause::OMPC_private:
+      return true;
+    default:
+      return false;
+    }
+  };
+
+  bool applied = applyIf(node, [&](const auto &leaf) {
+    return llvm::any_of(leaf.clauses, [&](const ClauseTy *n) {
+      return canMakePrivateCopy(n->id);
+    });
+  });
+
+  return applied;
+}
+
+// REDUCTION
+// [5.2:134:17-18]
+// Directives: do, for, loop, parallel, scope, sections, simd, taskloop, teams
+//
+// [5.2:340:36-37], [5.2:341:1-13]
+// (36) The effect of the reduction clause is as if it is applied to all leaf
+// constructs that permit the clause, except for the following constructs:
+//  (1) The parallel construct, when combined with the sections,
+//      worksharing-loop, loop, or taskloop construct; and
+//  (3) The teams construct, when combined with the loop construct.
+// (4) For the parallel and teams constructs above, the effect of the reduction
+// clause instead is as if each list item or, for any list item that is an array
+// item, its corresponding base array or base pointer appears in a shared clause
+// for the construct.
+// (6) If the task reduction-modifier is specified, the effect is as if it only
+// modifies the behavior of the reduction clause on the innermost leaf construct
+// that accepts the modifier (see Section 5.5.8).
+// (8) If the inscan reduction-modifier is specified, the effect is as if it
+// modifies the behavior of the reduction clause on all constructs of the
+// combined construct to which the clause is applied and that accept the
+// modifier.
+// (10) If a list item in a reduction clause on a combined target construct does
+// not have the same base variable or base pointer as a list item in a map
+// clause on the construct, then the effect is as if the list item in the
+// reduction clause appears as a list item in a map clause with a map-type of
+// tofrom.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::ReductionT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  using ReductionTy = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
+
+  // [5.2:340:36], [5.2:341:1], [5.2:341:3]
+  bool applyToParallel = true, applyToTeams = true;
+
+  auto dirParallel = findDirective(llvm::omp::Directive::OMPD_parallel);
+  if (dirParallel) {
+    auto exclusions = llvm::concat<const llvm::omp::Directive>(
+        getWorksharingLoop(), tomp::ListT<llvm::omp::Directive>{
+                                  llvm::omp::Directive::OMPD_loop,
+                                  llvm::omp::Directive::OMPD_sections,
+                                  llvm::omp::Directive::OMPD_taskloop,
+                              });
+    auto present = [&](llvm::omp::Directive id) {
+      return findDirective(id) != nullptr;
+    };
+
+    if (llvm::any_of(exclusions, present))
+      applyToParallel = false;
+  }
+
+  auto dirTeams = findDirective(llvm::omp::Directive::OMPD_teams);
+  if (dirTeams) {
+    // The only exclusion is OMPD_loop.
+    if (findDirective(llvm::omp::Directive::OMPD_loop))
+      applyToTeams = false;
+  }
+
+  using ReductionModifier = typename ReductionTy::ReductionModifier;
+  using ReductionIdentifiers = typename ReductionTy::ReductionIdentifiers;
+
+  auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(clause.t);
+  auto &modifier = std::get<std::optional<ReductionModifier>>(clause.t);
+
+  // Apply the reduction clause first to all directives according to the spec.
+  // If the reduction was applied at least once, proceed with the data sharing
+  // side-effects.
+  bool applied = false;
+
+  // [5.2:341:6], [5.2:341:8]
+  auto isValidModifier = [](llvm::omp::Directive dir, ReductionModifier mod,
+                            bool alreadyApplied) {
+    switch (mod) {
+    case ReductionModifier::Inscan:
+      // According to [5.2:135:11-13], "inscan" only applies to
+      // worksharing-loop, worksharing-loop-simd, or "simd" constructs.
+      return dir == llvm::omp::Directive::OMPD_simd ||
+             llvm::is_contained(getWorksharingLoop(), dir);
+    case ReductionModifier::Task:
+      if (alreadyApplied)
+        return false;
+      // According to [5.2:135:16-18], "task" only applies to "parallel" and
+      // worksharing constructs.
+      return dir == llvm::omp::Directive::OMPD_parallel ||
+             llvm::is_contained(getWorksharing(), dir);
+    case ReductionModifier::Default:
+      return true;
+    }
+    llvm_unreachable("Unexpected modifier");
+  };
+
+  auto *unmodified = makeClause(
+      llvm::omp::Clause::OMPC_reduction,
+      ReductionTy{
+          {/*ReductionModifier=*/std::nullopt,
+           /*ReductionIdentifiers=*/std::get<ReductionIdentifiers>(clause.t),
+           /*List=*/objects}});
+
+  ReductionModifier effective =
+      modifier.has_value() ? *modifier : ReductionModifier::Default;
+  bool effectiveApplied = false;
+  // Walk over the leaf constructs starting from the innermost, and apply
+  // the clause as required by the spec.
+  for (auto &leaf : llvm::reverse(leafs)) {
+    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, node->id, version))
+      continue;
+    if (!applyToParallel && &leaf == dirParallel)
+      continue;
+    if (!applyToTeams && &leaf == dirTeams)
+      continue;
+    // Some form of the clause will be applied past this point.
+    if (isValidModifier(leaf.id, effective, effectiveApplied)) {
+      // Apply clause with modifier.
+      leaf.clauses.push_back(node);
+      effectiveApplied = true;
+    } else {
+      // Apply clause without modifier.
+      leaf.clauses.push_back(unmodified);
+    }
+    applied = true;
+  }
+
+  if (!applied)
+    return false;
+
+  tomp::ObjectListT<IdTy, ExprTy> sharedObjects;
+  llvm::transform(objects, std::back_inserter(sharedObjects),
+                  [&](const ObjectTy &object) {
+                    auto maybeBase = helper.getBaseObject(object);
+                    return maybeBase ? *maybeBase : object;
+                  });
+
+  // [5.2:341:4]
+  if (!sharedObjects.empty()) {
+    if (dirParallel && !applyToParallel) {
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
+      dirParallel->clauses.push_back(shared);
+    }
+    if (dirTeams && !applyToTeams) {
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
+      dirTeams->clauses.push_back(shared);
+    }
+  }
+
+  // [5.2:341:10]
+  auto dirTarget = findDirective(llvm::omp::Directive::OMPD_target);
+  if (dirTarget && leafs.size() > 1) {
+    tomp::ObjectListT<IdTy, ExprTy> tofrom;
+    llvm::copy_if(objects, std::back_inserter(tofrom),
+                  [&](const ObjectTy &object) {
+                    if (auto maybeBase = helper.getBaseObject(object))
+                      return !mapBases.count(maybeBase->id());
+                    return !mapBases.count(object.id()); // XXX is this ok?
+                  });
+    if (!tofrom.empty()) {
+      using MapType =
+          typename tomp::clause::MapT<TypeTy, IdTy, ExprTy>::MapType;
+      auto *map = makeClause(
+          llvm::omp::Clause::OMPC_map,
+          tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
+              {/*MapType=*/MapType::Tofrom, /*MapTypeModifier=*/std::nullopt,
+               /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
+               /*LocatorList=*/std::move(tofrom)}});
+
+      dirTarget->clauses.push_back(map);
+      applied = true;
+    }
+  }
+
+  return applied;
+}
+
+// IF
+// [5.2:72:7-9]
+// Directives: cancel, parallel, simd, target, target data, target enter data,
+// target exit data, target update, task, taskloop
+//
+// [5.2:72:15-18]
+// (15) For combined or composite constructs, the if clause only applies to the
+// semantics of the construct named in the directive-name-modifier.
+// (16) For a combined or composite construct, if no directive-name-modifier is
+// specified then the if clause applies to all constituent constructs to which
+// an if clause can apply.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::IfT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  using DirectiveNameModifier =
+      typename clause::IfT<TypeTy, IdTy, ExprTy>::DirectiveNameModifier;
+  using IfExpression = typename clause::IfT<TypeTy, IdTy, ExprTy>::IfExpression;
+  auto &modifier = std::get<std::optional<DirectiveNameModifier>>(clause.t);
+
+  if (modifier) {
+    llvm::omp::Directive dirId = *modifier;
+    auto *unmodified =
+        makeClause(llvm::omp::Clause::OMPC_if,
+                   tomp::clause::IfT<TypeTy, IdTy, ExprTy>{
+                       {/*DirectiveNameModifier=*/std::nullopt,
+                        /*IfExpression=*/std::get<IfExpression>(clause.t)}});
+
+    if (auto *hasDir = findDirective(dirId)) {
+      hasDir->clauses.push_back(unmodified);
+      return true;
+    }
+    return false;
+  }
+
+  return applyToAll(node);
+}
+
+// LINEAR
+// [5.2:118:1-2]
+// Directives: declare simd, do, for, simd
+//
+// [5.2:341:15-22]
+// (15.1) The effect of the linear clause is as if it is applied to the
+// innermost leaf construct.
+// (15.2) Additionally, if the list item is not the iteration variable of a simd
+// or worksharing-loop SIMD construct, the effect on the outer leaf constructs
+// is as if the list item was specified in firstprivate and lastprivate clauses
+// on the combined or composite construct, with the rules specified above
+// applied.
+// (19) If a list item of the linear clause is the iteration variable of a simd
+// or worksharing-loop SIMD construct and it is not declared in the construct,
+// the effect on the outer leaf constructs is as if the list item was specified
+// in a lastprivate clause on the combined or composite construct with the rules
+// specified above applied.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::LinearT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // [5.2:341:15.1]
+  if (!applyToInnermost(node))
+    return false;
+
+  // [5.2:341:15.2], [5.2:341:19]
+  auto dirSimd = findDirective(llvm::omp::Directive::OMPD_simd);
+  std::optional<ObjectTy> iterVar = helper.getLoopIterVar();
+  const auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(clause.t);
+
+  // Lists of objects that will be used to construct "firstprivate" and
+  // "lastprivate" clauses.
+  tomp::ObjectListT<IdTy, ExprTy> first, last;
+
+  for (const ObjectTy &object : objects) {
+    last.push_back(object);
+    if (!dirSimd || !iterVar || object.id() != iterVar->id())
+      first.push_back(object);
+  }
+
+  if (!first.empty()) {
+    auto *firstp = makeClause(
+        llvm::omp::Clause::OMPC_firstprivate,
+        tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>{/*List=*/first});
+    nodes.push_back(firstp); // Appending to the main clause list.
+  }
+  if (!last.empty()) {
+    auto *lastp =
+        makeClause(llvm::omp::Clause::OMPC_lastprivate,
+                   tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>{
+                       {/*LastprivateModifier=*/std::nullopt, /*List=*/last}});
+    nodes.push_back(lastp); // Appending to the main clause list.
+  }
+  return true;
+}
+
+// NOWAIT
+// [5.2:308:11-13]
+// Directives: dispatch, do, for, interop, scope, sections, single, target,
+// target enter data, target exit data, target update, taskwait, workshare
+//
+// [5.2:341:23]
+// (23) The effect of the nowait clause is as if it is applied to the outermost
+// leaf construct that permits it.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::NowaitT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  return applyToOutermost(node);
+}
+
+template <typename C, typename H> bool ConstructDecompositionT<C, H>::split() {
+  bool success = true;
+
+  for (llvm::omp::Directive leaf :
+       llvm::omp::getLeafConstructsOrSelf(construct))
+    leafs.push_back(LeafReprInternal{leaf, /*clauses=*/{}});
+
+  for (const ClauseTy *node : nodes)
+    addClauseSymsToMap(*node, node);
+
+  // First we need to apply LINEAR, because it can generate additional
+  // "firstprivate" and "lastprivate" clauses that apply to the combined/
+  // composite construct.
+  // Collect them separately, because they may modify the clause list.
+  llvm::SmallVector<const ClauseTy *> linears;
+  for (const ClauseTy *node : nodes) {
+    if (node->id == llvm::omp::Clause::OMPC_linear)
+      linears.push_back(node);
+  }
+  for (const auto *node : linears) {
+    success = success &&
+              applyClause(std::get<tomp::clause::LinearT<TypeTy, IdTy, ExprTy>>(
+                              node->u),
+                          node);
+  }
+
+  // "allocate" clauses need to be applied last since they need to see
+  // which directives have data-privatizing clauses.
+  auto skip = [](const ClauseTy *node) {
+    switch (node->id) {
+    case llvm::omp::Clause::OMPC_allocate:
+    case llvm::omp::Clause::OMPC_linear:
+      return true;
+    default:
+      return false;
+    }
+  };
+
+  // Apply (almost) all clauses.
+  for (const ClauseTy *node : nodes) {
+    if (skip(node))
+      continue;
+    success =
+        success &&
+        std::visit([&](auto &&s) { return applyClause(s, node); }, node->u);
+  }
+
+  // Apply "allocate".
+  for (const ClauseTy *node : nodes) {
+    if (node->id != llvm::omp::Clause::OMPC_allocate)
+      continue;
+    success =
+        success &&
+        std::visit([&](auto &&s) { return applyClause(s, node); }, node->u);
+  }
+
+  return success;
+}
+
+} // namespace tomp
+
+#endif // LLVM_FRONTEND_OPENMP_CONSTRUCTDECOMPOSITIONT_H
diff --git a/llvm/unittests/Frontend/CMakeLists.txt b/llvm/unittests/Frontend/CMakeLists.txt
index 3f290b63ba64..85e113816e3b 100644
--- a/llvm/unittests/Frontend/CMakeLists.txt
+++ b/llvm/unittests/Frontend/CMakeLists.txt
@@ -15,6 +15,7 @@ add_llvm_unittest(LLVMFrontendTests
   OpenMPIRBuilderTest.cpp
   OpenMPParsingTest.cpp
   OpenMPCompositionTest.cpp
+  OpenMPDecompositionTest.cpp
 
   DEPENDS
   acc_gen
diff --git a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
new file mode 100644
index 000000000000..df48e9cc0ff4
--- /dev/null
+++ b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
@@ -0,0 +1,999 @@
+//===- llvm/unittests/Frontend/OpenMPDecompositionTest.cpp ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Frontend/OpenMP/ClauseT.h"
+#include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "gtest/gtest.h"
+
+#include <iterator>
+#include <optional>
+#include <sstream>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+// The actual tests start at comment "--- Test" below.
+
+// Create simple instantiations of all clauses to allow manual construction
+// of clauses, and implement emitting of a directive with clauses to a string.
+//
+// The tests then follow the pattern
+// 1. Create a list of clauses.
+// 2. Pass them, together with a construct, to the decomposition class.
+// 3. Extract individual resulting leaf constructs with clauses applied
+//    to them.
+// 4. Convert them to strings and compare with expected outputs.
+
+namespace omp {
+struct TypeTy {}; // placeholder
+struct ExprTy {}; // placeholder
+using IdTy = std::string;
+} // namespace omp
+
+namespace tomp::type {
+template <> struct ObjectT<omp::IdTy, omp::ExprTy> {
+  const omp::IdTy &id() const { return name; }
+  const std::optional<omp::ExprTy> ref() const { return omp::ExprTy{}; }
+
+  omp::IdTy name;
+};
+} // namespace tomp::type
+
+namespace omp {
+template <typename ElemTy> using List = tomp::type::ListT<ElemTy>;
+
+using Object = tomp::ObjectT<IdTy, ExprTy>;
+
+namespace clause {
+using DefinedOperator = tomp::type::DefinedOperatorT<IdTy, ExprTy>;
+using ProcedureDesignator = tomp::type::ProcedureDesignatorT<IdTy, ExprTy>;
+using ReductionOperator = tomp::type::ReductionIdentifierT<IdTy, ExprTy>;
+
+using AcqRel = tomp::clause::AcqRelT<TypeTy, IdTy, ExprTy>;
+using Acquire = tomp::clause::AcquireT<TypeTy, IdTy, ExprTy>;
+using AdjustArgs = tomp::clause::AdjustArgsT<TypeTy, IdTy, ExprTy>;
+using Affinity = tomp::clause::AffinityT<TypeTy, IdTy, ExprTy>;
+using Aligned = tomp::clause::AlignedT<TypeTy, IdTy, ExprTy>;
+using Align = tomp::clause::AlignT<TypeTy, IdTy, ExprTy>;
+using Allocate = tomp::clause::AllocateT<TypeTy, IdTy, ExprTy>;
+using Allocator = tomp::clause::AllocatorT<TypeTy, IdTy, ExprTy>;
+using AppendArgs = tomp::clause::AppendArgsT<TypeTy, IdTy, ExprTy>;
+using AtomicDefaultMemOrder =
+    tomp::clause::AtomicDefaultMemOrderT<TypeTy, IdTy, ExprTy>;
+using At = tomp::clause::AtT<TypeTy, IdTy, ExprTy>;
+using Bind = tomp::clause::BindT<TypeTy, IdTy, ExprTy>;
+using Capture = tomp::clause::CaptureT<TypeTy, IdTy, ExprTy>;
+using Collapse = tomp::clause::CollapseT<TypeTy, IdTy, ExprTy>;
+using Compare = tomp::clause::CompareT<TypeTy, IdTy, ExprTy>;
+using Copyin = tomp::clause::CopyinT<TypeTy, IdTy, ExprTy>;
+using Copyprivate = tomp::clause::CopyprivateT<TypeTy, IdTy, ExprTy>;
+using Defaultmap = tomp::clause::DefaultmapT<TypeTy, IdTy, ExprTy>;
+using Default = tomp::clause::DefaultT<TypeTy, IdTy, ExprTy>;
+using Depend = tomp::clause::DependT<TypeTy, IdTy, ExprTy>;
+using Destroy = tomp::clause::DestroyT<TypeTy, IdTy, ExprTy>;
+using Detach = tomp::clause::DetachT<TypeTy, IdTy, ExprTy>;
+using Device = tomp::clause::DeviceT<TypeTy, IdTy, ExprTy>;
+using DeviceType = tomp::clause::DeviceTypeT<TypeTy, IdTy, ExprTy>;
+using DistSchedule = tomp::clause::DistScheduleT<TypeTy, IdTy, ExprTy>;
+using Doacross = tomp::clause::DoacrossT<TypeTy, IdTy, ExprTy>;
+using DynamicAllocators =
+    tomp::clause::DynamicAllocatorsT<TypeTy, IdTy, ExprTy>;
+using Enter = tomp::clause::EnterT<TypeTy, IdTy, ExprTy>;
+using Exclusive = tomp::clause::ExclusiveT<TypeTy, IdTy, ExprTy>;
+using Fail = tomp::clause::FailT<TypeTy, IdTy, ExprTy>;
+using Filter = tomp::clause::FilterT<TypeTy, IdTy, ExprTy>;
+using Final = tomp::clause::FinalT<TypeTy, IdTy, ExprTy>;
+using Firstprivate = tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>;
+using From = tomp::clause::FromT<TypeTy, IdTy, ExprTy>;
+using Full = tomp::clause::FullT<TypeTy, IdTy, ExprTy>;
+using Grainsize = tomp::clause::GrainsizeT<TypeTy, IdTy, ExprTy>;
+using HasDeviceAddr = tomp::clause::HasDeviceAddrT<TypeTy, IdTy, ExprTy>;
+using Hint = tomp::clause::HintT<TypeTy, IdTy, ExprTy>;
+using If = tomp::clause::IfT<TypeTy, IdTy, ExprTy>;
+using Inbranch = tomp::clause::InbranchT<TypeTy, IdTy, ExprTy>;
+using Inclusive = tomp::clause::InclusiveT<TypeTy, IdTy, ExprTy>;
+using Indirect = tomp::clause::IndirectT<TypeTy, IdTy, ExprTy>;
+using Init = tomp::clause::InitT<TypeTy, IdTy, ExprTy>;
+using InReduction = tomp::clause::InReductionT<TypeTy, IdTy, ExprTy>;
+using IsDevicePtr = tomp::clause::IsDevicePtrT<TypeTy, IdTy, ExprTy>;
+using Lastprivate = tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>;
+using Linear = tomp::clause::LinearT<TypeTy, IdTy, ExprTy>;
+using Link = tomp::clause::LinkT<TypeTy, IdTy, ExprTy>;
+using Map = tomp::clause::MapT<TypeTy, IdTy, ExprTy>;
+using Match = tomp::clause::MatchT<TypeTy, IdTy, ExprTy>;
+using Mergeable = tomp::clause::MergeableT<TypeTy, IdTy, ExprTy>;
+using Message = tomp::clause::MessageT<TypeTy, IdTy, ExprTy>;
+using Nocontext = tomp::clause::NocontextT<TypeTy, IdTy, ExprTy>;
+using Nogroup = tomp::clause::NogroupT<TypeTy, IdTy, ExprTy>;
+using Nontemporal = tomp::clause::NontemporalT<TypeTy, IdTy, ExprTy>;
+using Notinbranch = tomp::clause::NotinbranchT<TypeTy, IdTy, ExprTy>;
+using Novariants = tomp::clause::NovariantsT<TypeTy, IdTy, ExprTy>;
+using Nowait = tomp::clause::NowaitT<TypeTy, IdTy, ExprTy>;
+using NumTasks = tomp::clause::NumTasksT<TypeTy, IdTy, ExprTy>;
+using NumTeams = tomp::clause::NumTeamsT<TypeTy, IdTy, ExprTy>;
+using NumThreads = tomp::clause::NumThreadsT<TypeTy, IdTy, ExprTy>;
+using OmpxAttribute = tomp::clause::OmpxAttributeT<TypeTy, IdTy, ExprTy>;
+using OmpxBare = tomp::clause::OmpxBareT<TypeTy, IdTy, ExprTy>;
+using OmpxDynCgroupMem = tomp::clause::OmpxDynCgroupMemT<TypeTy, IdTy, ExprTy>;
+using Ordered = tomp::clause::OrderedT<TypeTy, IdTy, ExprTy>;
+using Order = tomp::clause::OrderT<TypeTy, IdTy, ExprTy>;
+using Partial = tomp::clause::PartialT<TypeTy, IdTy, ExprTy>;
+using Priority = tomp::clause::PriorityT<TypeTy, IdTy, ExprTy>;
+using Private = tomp::clause::PrivateT<TypeTy, IdTy, ExprTy>;
+using ProcBind = tomp::clause::ProcBindT<TypeTy, IdTy, ExprTy>;
+using Read = tomp::clause::ReadT<TypeTy, IdTy, ExprTy>;
+using Reduction = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
+using Relaxed = tomp::clause::RelaxedT<TypeTy, IdTy, ExprTy>;
+using Release = tomp::clause::ReleaseT<TypeTy, IdTy, ExprTy>;
+using ReverseOffload = tomp::clause::ReverseOffloadT<TypeTy, IdTy, ExprTy>;
+using Safelen = tomp::clause::SafelenT<TypeTy, IdTy, ExprTy>;
+using Schedule = tomp::clause::ScheduleT<TypeTy, IdTy, ExprTy>;
+using SeqCst = tomp::clause::SeqCstT<TypeTy, IdTy, ExprTy>;
+using Severity = tomp::clause::SeverityT<TypeTy, IdTy, ExprTy>;
+using Shared = tomp::clause::SharedT<TypeTy, IdTy, ExprTy>;
+using Simdlen = tomp::clause::SimdlenT<TypeTy, IdTy, ExprTy>;
+using Simd = tomp::clause::SimdT<TypeTy, IdTy, ExprTy>;
+using Sizes = tomp::clause::SizesT<TypeTy, IdTy, ExprTy>;
+using TaskReduction = tomp::clause::TaskReductionT<TypeTy, IdTy, ExprTy>;
+using ThreadLimit = tomp::clause::ThreadLimitT<TypeTy, IdTy, ExprTy>;
+using Threads = tomp::clause::ThreadsT<TypeTy, IdTy, ExprTy>;
+using To = tomp::clause::ToT<TypeTy, IdTy, ExprTy>;
+using UnifiedAddress = tomp::clause::UnifiedAddressT<TypeTy, IdTy, ExprTy>;
+using UnifiedSharedMemory =
+    tomp::clause::UnifiedSharedMemoryT<TypeTy, IdTy, ExprTy>;
+using Uniform = tomp::clause::UniformT<TypeTy, IdTy, ExprTy>;
+using Unknown = tomp::clause::UnknownT<TypeTy, IdTy, ExprTy>;
+using Untied = tomp::clause::UntiedT<TypeTy, IdTy, ExprTy>;
+using Update = tomp::clause::UpdateT<TypeTy, IdTy, ExprTy>;
+using UseDeviceAddr = tomp::clause::UseDeviceAddrT<TypeTy, IdTy, ExprTy>;
+using UseDevicePtr = tomp::clause::UseDevicePtrT<TypeTy, IdTy, ExprTy>;
+using UsesAllocators = tomp::clause::UsesAllocatorsT<TypeTy, IdTy, ExprTy>;
+using Use = tomp::clause::UseT<TypeTy, IdTy, ExprTy>;
+using Weak = tomp::clause::WeakT<TypeTy, IdTy, ExprTy>;
+using When = tomp::clause::WhenT<TypeTy, IdTy, ExprTy>;
+using Write = tomp::clause::WriteT<TypeTy, IdTy, ExprTy>;
+} // namespace clause
+
+struct Helper {
+  std::optional<Object> getBaseObject(const Object &object) {
+    return std::nullopt;
+  }
+  std::optional<Object> getLoopIterVar() { return std::nullopt; }
+};
+
+using Clause = tomp::ClauseT<TypeTy, IdTy, ExprTy>;
+using ConstructDecomposition = tomp::ConstructDecompositionT<Clause, Helper>;
+using DirectiveWithClauses = tomp::DirectiveWithClauses<Clause>;
+} // namespace omp
+
+struct StringifyClause {
+  static std::string join(const omp::List<std::string> &Strings) {
+    std::stringstream Stream;
+    for (const auto &[Index, String] : llvm::enumerate(Strings)) {
+      if (Index != 0)
+        Stream << ", ";
+      Stream << String;
+    }
+    return Stream.str();
+  }
+
+  static std::string to_str(llvm::omp::Directive D) {
+    return getOpenMPDirectiveName(D).str();
+  }
+  static std::string to_str(llvm::omp::Clause C) {
+    return getOpenMPClauseName(C).str();
+  }
+  static std::string to_str(const omp::TypeTy &Type) { return "type"; }
+  static std::string to_str(const omp::ExprTy &Expr) { return "expr"; }
+  static std::string to_str(const omp::Object &Obj) { return Obj.id(); }
+
+  template <typename U>
+  static std::enable_if_t<std::is_enum_v<llvm::remove_cvref_t<U>>, std::string>
+  to_str(U &&Item) {
+    return std::to_string(llvm::to_underlying(Item));
+  }
+
+  template <typename U> static std::string to_str(const omp::List<U> &Items) {
+    omp::List<std::string> Names;
+    llvm::transform(Items, std::back_inserter(Names),
+                    [](auto &&S) { return to_str(S); });
+    return "(" + join(Names) + ")";
+  }
+
+  template <typename U>
+  static std::string to_str(const std::optional<U> &Item) {
+    if (Item)
+      return to_str(*Item);
+    return "";
+  }
+
+  template <typename... Us, size_t... Is>
+  static std::string to_str(const std::tuple<Us...> &Tuple,
+                            std::index_sequence<Is...>) {
+    omp::List<std::string> Strings;
+    (Strings.push_back(to_str(std::get<Is>(Tuple))), ...);
+    return "(" + join(Strings) + ")";
+  }
+
+  template <typename U>
+  static std::enable_if_t<llvm::remove_cvref_t<U>::EmptyTrait::value,
+                          std::string>
+  to_str(U &&Item) {
+    return "";
+  }
+
+  template <typename U>
+  static std::enable_if_t<llvm::remove_cvref_t<U>::IncompleteTrait::value,
+                          std::string>
+  to_str(U &&Item) {
+    return "";
+  }
+
+  template <typename U>
+  static std::enable_if_t<llvm::remove_cvref_t<U>::WrapperTrait::value,
+                          std::string>
+  to_str(U &&Item) {
+    // For a wrapper, stringify the wrappee, and only add parentheses if
+    // there aren't any already.
+    std::string Str = to_str(Item.v);
+    if (!Str.empty()) {
+      if (Str.front() == '(' && Str.back() == ')')
+        return Str;
+    }
+    return "(" + to_str(Item.v) + ")";
+  }
+
+  template <typename U>
+  static std::enable_if_t<llvm::remove_cvref_t<U>::TupleTrait::value,
+                          std::string>
+  to_str(U &&Item) {
+    constexpr size_t TupleSize =
+        std::tuple_size_v<llvm::remove_cvref_t<decltype(Item.t)>>;
+    return to_str(Item.t, std::make_index_sequence<TupleSize>{});
+  }
+
+  template <typename U>
+  static std::enable_if_t<llvm::remove_cvref_t<U>::UnionTrait::value,
+                          std::string>
+  to_str(U &&Item) {
+    return std::visit([](auto &&S) { return to_str(S); }, Item.u);
+  }
+
+  StringifyClause(const omp::Clause &C)
+      // Rely on content stringification to emit enclosing parentheses.
+      : Str(to_str(C.id) + to_str(C)) {}
+
+  std::string Str;
+};
+
+std::string stringify(const omp::DirectiveWithClauses &DWC) {
+  std::stringstream Stream;
+
+  Stream << getOpenMPDirectiveName(DWC.id).str();
+  for (const omp::Clause &C : DWC.clauses)
+    Stream << ' ' << StringifyClause(C).Str;
+
+  return Stream.str();
+}
+
+// --- Tests ----------------------------------------------------------
+
+namespace {
+using namespace llvm::omp;
+
+class OpenMPDecompositionTest : public testing::Test {
+protected:
+  void SetUp() override {}
+  void TearDown() override {}
+
+  omp::Helper Helper;
+  uint32_t AnyVersion = 999;
+};
+
+// PRIVATE
+// [5.2:111:5-7]
+// Directives: distribute, do, for, loop, parallel, scope, sections, simd,
+// single, target, task, taskloop, teams
+//
+// [5.2:340:1-2]
+// (1) The effect of the 1 private clause is as if it is applied only to the
+// innermost leaf construct that permits it.
+TEST_F(OpenMPDecompositionTest, Private1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_private, omp::clause::Private{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel");            // (1)
+  ASSERT_EQ(Dir1, "sections private(x)"); // (1)
+}
+
+TEST_F(OpenMPDecompositionTest, Private2) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_private, omp::clause::Private{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_masked,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel private(x)"); // (1)
+  ASSERT_EQ(Dir1, "masked");              // (1)
+}
+
+// FIRSTPRIVATE
+// [5.2:112:5-7]
+// Directives: distribute, do, for, parallel, scope, sections, single, target,
+// task, taskloop, teams
+//
+// [5.2:340:3-20]
+// (3) The effect of the firstprivate clause is as if it is applied to one or
+// more leaf constructs as follows:
+//  (5) To the distribute construct if it is among the constituent constructs;
+//  (6) To the teams construct if it is among the constituent constructs and the
+//      distribute construct is not;
+//  (8) To a worksharing construct that accepts the clause if one is among the
+//      constituent constructs;
+//  (9) To the taskloop construct if it is among the constituent constructs;
+// (10) To the parallel construct if it is among the constituent constructs and
+//      neither a taskloop construct nor a worksharing construct that accepts
+//      the clause is among them;
+// (12) To the target construct if it is among the constituent constructs and
+//      the same list item neither appears in a lastprivate clause nor is the
+//      base variable or base pointer of a list item that appears in a map
+//      clause.
+//
+// (15) If the parallel construct is among the constituent constructs and the
+// effect is not as if the firstprivate clause is applied to it by the above
+// rules, then the effect is as if the shared clause with the same list item is
+// applied to the parallel construct.
+// (17) If the teams construct is among the constituent constructs and the
+// effect is not as if the firstprivate clause is applied to it by the above
+// rules, then the effect is as if the shared clause with the same list item is
+// applied to the teams construct.
+TEST_F(OpenMPDecompositionTest, Firstprivate1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel shared(x)");       // (10), (15)
+  ASSERT_EQ(Dir1, "sections firstprivate(x)"); // (8)
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate2) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_target_teams_distribute, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "target firstprivate(x)");     // (12)
+  ASSERT_EQ(Dir1, "teams shared(x)");            // (6), (17)
+  ASSERT_EQ(Dir2, "distribute firstprivate(x)"); // (5)
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate3) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_target_teams_distribute, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (12), (27)
+  ASSERT_EQ(Dir1, "teams shared(x)");          // (6), (17)
+  ASSERT_EQ(Dir2, "distribute firstprivate(x) lastprivate(, (x))"); // (5), (21)
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate4) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_teams,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "target firstprivate(x)"); // (12)
+  ASSERT_EQ(Dir1, "teams firstprivate(x)");  // (6)
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate5) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_parallel_masked_taskloop, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "parallel shared(x)"); // (10)
+  ASSERT_EQ(Dir1, "masked");
+  ASSERT_EQ(Dir2, "taskloop firstprivate(x)"); // (9)
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate6) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_masked,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel firstprivate(x)"); // (10)
+  ASSERT_EQ(Dir1, "masked");
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate7) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  // Composite constructs are still decomposed.
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_teams_distribute,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "teams shared(x)");            // (17)
+  ASSERT_EQ(Dir1, "distribute firstprivate(x)"); // (5)
+}
+
+// LASTPRIVATE
+// [5.2:115:7-8]
+// Directives: distribute, do, for, loop, sections, simd, taskloop
+//
+// [5.2:340:21-30]
+// (21) The effect of the lastprivate clause is as if it is applied to all leaf
+// constructs that permit the clause.
+// (22) If the parallel construct is among the constituent constructs and the
+// list item is not also specified in the firstprivate clause, then the effect
+// of the lastprivate clause is as if the shared clause with the same list item
+// is applied to the parallel construct.
+// (24) If the teams construct is among the constituent constructs and the list
+// item is not also specified in the firstprivate clause, then the effect of the
+// lastprivate clause is as if the shared clause with the same list item is
+// applied to the teams construct.
+// (27) If the target construct is among the constituent constructs and the list
+// item is not the base variable or base pointer of a list item that appears in
+// a map clause, the effect of the lastprivate clause is as if the same list
+// item appears in a map clause with a map-type of tofrom.
+TEST_F(OpenMPDecompositionTest, Lastprivate1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel shared(x)");          // (21), (22)
+  ASSERT_EQ(Dir1, "sections lastprivate(, (x))"); // (21)
+}
+
+TEST_F(OpenMPDecompositionTest, Lastprivate2) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_teams_distribute,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "teams shared(x)");               // (21), (25)
+  ASSERT_EQ(Dir1, "distribute lastprivate(, (x))"); // (21)
+}
+
+TEST_F(OpenMPDecompositionTest, Lastprivate3) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_parallel_do,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (21), (27)
+  ASSERT_EQ(Dir1, "parallel shared(x)");       // (22)
+  ASSERT_EQ(Dir2, "do lastprivate(, (x))");    // (21)
+}
+
+// SHARED
+// [5.2:110:5-6]
+// Directives: parallel, task, taskloop, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+TEST_F(OpenMPDecompositionTest, Shared1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_shared, omp::clause::Shared{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_parallel_masked_taskloop, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "parallel shared(x)"); // (31)
+  ASSERT_EQ(Dir1, "masked");             // (31)
+  ASSERT_EQ(Dir2, "taskloop shared(x)"); // (31)
+}
+
+// DEFAULT
+// [5.2:109:5-6]
+// Directives: parallel, task, taskloop, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+TEST_F(OpenMPDecompositionTest, Default1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_default,
+       omp::clause::Default{
+           omp::clause::Default::DataSharingAttribute::Firstprivate}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_parallel_masked_taskloop, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "parallel default(0)"); // (31)
+  ASSERT_EQ(Dir1, "masked");              // (31)
+  ASSERT_EQ(Dir2, "taskloop default(0)"); // (31)
+}
+
+// THREAD_LIMIT
+// [5.2:277:14-15]
+// Directives: target, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+TEST_F(OpenMPDecompositionTest, ThreadLimit1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_thread_limit, omp::clause::ThreadLimit{omp::ExprTy{}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_target_teams_distribute, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "target thread_limit(expr)"); // (31)
+  ASSERT_EQ(Dir1, "teams thread_limit(expr)");  // (31)
+  ASSERT_EQ(Dir2, "distribute");                // (31)
+}
+
+// ORDER
+// [5.2:234:3-4]
+// Directives: distribute, do, for, loop, simd
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+TEST_F(OpenMPDecompositionTest, Order1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_order,
+       omp::clause::Order{{omp::clause::Order::OrderModifier::Unconstrained,
+                           omp::clause::Order::Ordering::Concurrent}}},
+  };
+
+  omp::ConstructDecomposition Dec(
+      AnyVersion, Helper, OMPD_target_teams_distribute_parallel_for_simd,
+      Clauses);
+  ASSERT_EQ(Dec.output.size(), 6u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  std::string Dir4 = stringify(Dec.output[4]);
+  std::string Dir5 = stringify(Dec.output[5]);
+  ASSERT_EQ(Dir0, "target"); // (31)
+  ASSERT_EQ(Dir1, "teams");  // (31)
+  // XXX OMP.td doesn't list "order" as allowed for "distribute"
+  ASSERT_EQ(Dir2, "distribute");       // (31)
+  ASSERT_EQ(Dir3, "parallel");         // (31)
+  ASSERT_EQ(Dir4, "for order(1, 0)");  // (31)
+  ASSERT_EQ(Dir5, "simd order(1, 0)"); // (31)
+}
+
+// ALLOCATE
+// [5.2:178:7-9]
+// Directives: allocators, distribute, do, for, parallel, scope, sections,
+// single, target, task, taskgroup, taskloop, teams
+//
+// [5.2:340:33-35]
+// (33) The effect of the allocate clause is as if it is applied to all leaf
+// constructs that permit the clause and to which a data-sharing attribute
+// clause that may create a private copy of the same list item is applied.
+TEST_F(OpenMPDecompositionTest, Allocate1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_allocate,
+       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_private, omp::clause::Private{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel");                                // (33)
+  ASSERT_EQ(Dir1, "sections private(x) allocate(, , , (x))"); // (33)
+}
+
+// REDUCTION
+// [5.2:134:17-18]
+// Directives: do, for, loop, parallel, scope, sections, simd, taskloop, teams
+//
+// [5.2:340-341:36-13]
+// (36) The effect of the reduction clause is as if it is applied to all leaf
+// constructs that permit the clause, except for the following constructs:
+//  (1) The parallel construct, when combined with the sections,
+//      worksharing-loop, loop, or taskloop construct; and
+//  (3) The teams construct, when combined with the loop construct.
+// (4) For the parallel and teams constructs above, the effect of the reduction
+// clause instead is as if each list item or, for any list item that is an array
+// item, its corresponding base array or base pointer appears in a shared clause
+// for the construct.
+// (6) If the task reduction-modifier is specified, the effect is as if it only
+// modifies the behavior of the reduction clause on the innermost leaf construct
+// that accepts the modifier (see Section 5.5.8).
+// (8) If the inscan reduction-modifier is specified, the effect is as if it
+// modifies the behavior of the reduction clause on all constructs of the
+// combined construct to which the clause is applied and that accept the
+// modifier.
+// (10) If a list item in a reduction clause on a combined target construct does
+// not have the same base variable or base pointer as a list item in a map
+// clause on the construct, then the effect is as if the list item in the
+// reduction clause appears as a list item in a map clause with a map-type of
+// tofrom.
+namespace red {
+// Make is easier to construct reduction operators from built-in intrinsics.
+omp::clause::ReductionOperator
+makeOp(omp::clause::DefinedOperator::IntrinsicOperator Op) {
+  return omp::clause::ReductionOperator{omp::clause::DefinedOperator{Op}};
+}
+} // namespace red
+
+TEST_F(OpenMPDecompositionTest, Reduction1) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel shared(x)");             // (36), (1), (4)
+  ASSERT_EQ(Dir1, "sections reduction(, (3), (x))"); // (36)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction2) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_masked,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel reduction(, (3), (x))"); // (36), (1), (4)
+  ASSERT_EQ(Dir1, "masked");                         // (36)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction3) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_teams_loop, Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "teams shared(x)");            // (36), (3), (4)
+  ASSERT_EQ(Dir1, "loop reduction(, (3), (x))"); // (36)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction4) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_teams_distribute_parallel_for, Clauses);
+  ASSERT_EQ(Dec.output.size(), 4u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  ASSERT_EQ(Dir0, "teams reduction(, (3), (x))"); // (36), (3)
+  ASSERT_EQ(Dir1, "distribute");                  // (36)
+  ASSERT_EQ(Dir2, "parallel shared(x)");          // (36), (1), (4)
+  ASSERT_EQ(Dir3, "for reduction(, (3), (x))");   // (36)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction5) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+  auto TaskMod = omp::clause::Reduction::ReductionModifier::Task;
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{TaskMod, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_teams_distribute_parallel_for, Clauses);
+  ASSERT_EQ(Dec.output.size(), 4u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  ASSERT_EQ(Dir0, "teams reduction(, (3), (x))"); // (36), (3), (6)
+  ASSERT_EQ(Dir1, "distribute");                  // (36)
+  ASSERT_EQ(Dir2, "parallel shared(x)");          // (36), (1), (4)
+  ASSERT_EQ(Dir3, "for reduction(2, (3), (x))");  // (36), (6)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction6) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+  auto InscanMod = omp::clause::Reduction::ReductionModifier::Inscan;
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{InscanMod, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_teams_distribute_parallel_for, Clauses);
+  ASSERT_EQ(Dec.output.size(), 4u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  ASSERT_EQ(Dir0, "teams reduction(, (3), (x))"); // (36), (3), (8)
+  ASSERT_EQ(Dir1, "distribute");                  // (36)
+  ASSERT_EQ(Dir2, "parallel shared(x)");          // (36), (1), (4)
+  ASSERT_EQ(Dir3, "for reduction(1, (3), (x))");  // (36), (8)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction7) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_parallel_do,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  // XXX Currently OMP.td allows "reduction" on "target".
+  ASSERT_EQ(Dir0,
+            "target reduction(, (3), (x)) map(2, , , , (x))"); // (36), (10)
+  ASSERT_EQ(Dir1, "parallel shared(x)");                       // (36), (1), (4)
+  ASSERT_EQ(Dir2, "do reduction(, (3), (x))");                 // (36)
+}
+
+// IF
+// [5.2:72:7-9]
+// Directives: cancel, parallel, simd, target, target data, target enter data,
+// target exit data, target update, task, taskloop
+//
+// [5.2:72:15-18]
+// (15) For combined or composite constructs, the if clause only applies to the
+// semantics of the construct named in the directive-name-modifier.
+// (16) For a combined or composite construct, if no directive-name-modifier is
+// specified then the if clause applies to all constituent constructs to which
+// an if clause can apply.
+TEST_F(OpenMPDecompositionTest, If1) {
+  omp::List<omp::Clause> Clauses{
+      {OMPC_if,
+       omp::clause::If{{llvm::omp::Directive::OMPD_parallel, omp::ExprTy{}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_target_parallel_for_simd, Clauses);
+  ASSERT_EQ(Dec.output.size(), 4u);
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  ASSERT_EQ(Dir0, "target");              // (15)
+  ASSERT_EQ(Dir1, "parallel if(, expr)"); // (15)
+  ASSERT_EQ(Dir2, "for");                 // (15)
+  ASSERT_EQ(Dir3, "simd");                // (15)
+}
+
+TEST_F(OpenMPDecompositionTest, If2) {
+  omp::List<omp::Clause> Clauses{
+      {OMPC_if, omp::clause::If{{std::nullopt, omp::ExprTy{}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_target_parallel_for_simd, Clauses);
+  ASSERT_EQ(Dec.output.size(), 4u);
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  ASSERT_EQ(Dir0, "target if(, expr)");   // (16)
+  ASSERT_EQ(Dir1, "parallel if(, expr)"); // (16)
+  ASSERT_EQ(Dir2, "for");                 // (16)
+  ASSERT_EQ(Dir3, "simd if(, expr)");     // (16)
+}
+
+// LINEAR
+// [5.2:118:1-2]
+// Directives: declare simd, do, for, simd
+//
+// [5.2:341:15-22]
+// (15.1) The effect of the linear clause is as if it is applied to the
+// innermost leaf construct.
+// (15.2) Additionally, if the list item is not the iteration variable of a simd
+// or worksharing-loop SIMD construct, the effect on the outer leaf constructs
+// is as if the list item was specified in firstprivate and lastprivate clauses
+// on the combined or composite construct, with the rules specified above
+// applied.
+// (19) If a list item of the linear clause is the iteration variable of a simd
+// or worksharing-loop SIMD construct and it is not declared in the construct,
+// the effect on the outer leaf constructs is as if the list item was specified
+// in a lastprivate clause on the combined or composite construct with the rules
+// specified above applied.
+TEST_F(OpenMPDecompositionTest, Linear1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_linear,
+       omp::clause::Linear{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_for_simd, Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "for firstprivate(x) lastprivate(, (x))"); // (15.1), (15.2)
+  ASSERT_EQ(Dir1, "simd linear(, , , (x)) lastprivate(, (x))"); // (15.1)
+}
+
+// NOWAIT
+// [5.2:308:11-13]
+// Directives: dispatch, do, for, interop, scope, sections, single, target,
+// target enter data, target exit data, target update, taskwait, workshare
+//
+// [5.2:341:23]
+// (23) The effect of the nowait clause is as if it is applied to the outermost
+// leaf construct that permits it.
+TEST_F(OpenMPDecompositionTest, Nowait1) {
+  omp::List<omp::Clause> Clauses{
+      {OMPC_nowait, omp::clause::Nowait{}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_parallel_for,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "target nowait"); // (23)
+  ASSERT_EQ(Dir1, "parallel");      // (23)
+  ASSERT_EQ(Dir2, "for");           // (23)
+}
+} // namespace
-- 
GitLab


From d2676a73336b83607565fb2e4ce61bd67d732b09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Hendrik=20H=C3=BCbner?=
 <117831077+HendrikHuebner@users.noreply.github.com>
Date: Mon, 13 May 2024 15:35:17 +0200
Subject: [PATCH 0561/1206] [libc][POSIX][pthreads] implemented missing
 pthread_rwlockattr functions (#90249)

Closes #89443

I added the two missing functions and respective test cases. Let me know
if anything needs changing.
---
 libc/config/linux/x86_64/entrypoints.txt      |  2 ++
 libc/include/pthread.h.def                    |  5 ++++
 libc/spec/posix.td                            | 10 +++++++
 libc/src/pthread/CMakeLists.txt               | 21 +++++++++++++
 .../pthread/pthread_rwlockattr_getkind_np.cpp | 24 +++++++++++++++
 .../pthread/pthread_rwlockattr_getkind_np.h   | 21 +++++++++++++
 libc/src/pthread/pthread_rwlockattr_init.cpp  |  1 +
 .../pthread/pthread_rwlockattr_setkind_np.cpp | 30 +++++++++++++++++++
 .../pthread/pthread_rwlockattr_setkind_np.h   | 20 +++++++++++++
 libc/test/src/pthread/CMakeLists.txt          |  2 ++
 .../src/pthread/pthread_rwlockattr_test.cpp   | 29 ++++++++++++++++--
 11 files changed, 162 insertions(+), 3 deletions(-)
 create mode 100644 libc/src/pthread/pthread_rwlockattr_getkind_np.cpp
 create mode 100644 libc/src/pthread/pthread_rwlockattr_getkind_np.h
 create mode 100644 libc/src/pthread/pthread_rwlockattr_setkind_np.cpp
 create mode 100644 libc/src/pthread/pthread_rwlockattr_setkind_np.h

diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 5e3ddd34fb4d..155deddaaade 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -673,8 +673,10 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.pthread.pthread_mutexattr_settype
     libc.src.pthread.pthread_once
     libc.src.pthread.pthread_rwlockattr_destroy
+    libc.src.pthread.pthread_rwlockattr_getkind_np
     libc.src.pthread.pthread_rwlockattr_getpshared
     libc.src.pthread.pthread_rwlockattr_init
+    libc.src.pthread.pthread_rwlockattr_setkind_np
     libc.src.pthread.pthread_rwlockattr_setpshared
     libc.src.pthread.pthread_setspecific
 
diff --git a/libc/include/pthread.h.def b/libc/include/pthread.h.def
index a94d770657e1..d41273b5590e 100644
--- a/libc/include/pthread.h.def
+++ b/libc/include/pthread.h.def
@@ -38,6 +38,11 @@ enum {
 #define PTHREAD_PROCESS_PRIVATE 0
 #define PTHREAD_PROCESS_SHARED 1
 
+#define PTHREAD_RWLOCK_PREFER_READER_NP 0
+#define PTHREAD_RWLOCK_PREFER_WRITER_NP 1
+#define PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP 2
+
+
 %%public_api()
 
 #endif // LLVM_LIBC_PTHREAD_H
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
index e7a0cf883c60..e16353b8142d 100644
--- a/libc/spec/posix.td
+++ b/libc/spec/posix.td
@@ -1234,6 +1234,11 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<IntType>,
           [ArgSpec<PThreadRWLockAttrTPtr>]
       >,
+      FunctionSpec<
+          "pthread_rwlockattr_getkind_np",
+          RetValSpec<IntType>,
+          [ArgSpec<PThreadRWLockAttrTPtr>, ArgSpec<IntPtr>]
+      >,
       FunctionSpec<
           "pthread_rwlockattr_getpshared",
           RetValSpec<IntType>,
@@ -1244,6 +1249,11 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<IntType>,
           [ArgSpec<PThreadRWLockAttrTPtr>]
       >,
+      FunctionSpec<
+          "pthread_rwlockattr_setkind_np",
+          RetValSpec<IntType>,
+          [ArgSpec<PThreadRWLockAttrTPtr>, ArgSpec<IntType>]
+      >,
       FunctionSpec<
           "pthread_rwlockattr_setpshared",
           RetValSpec<IntType>,
diff --git a/libc/src/pthread/CMakeLists.txt b/libc/src/pthread/CMakeLists.txt
index c57475c9114f..e5bebb63c640 100644
--- a/libc/src/pthread/CMakeLists.txt
+++ b/libc/src/pthread/CMakeLists.txt
@@ -470,6 +470,16 @@ add_entrypoint_object(
     libc.include.pthread
 )
 
+add_entrypoint_object(
+  pthread_rwlockattr_getkind_np
+  SRCS
+    pthread_rwlockattr_getkind_np.cpp
+  HDRS
+    pthread_rwlockattr_getkind_np.h
+  DEPENDS
+    libc.include.pthread
+)
+
 add_entrypoint_object(
   pthread_rwlockattr_getpshared
   SRCS
@@ -490,6 +500,17 @@ add_entrypoint_object(
     libc.include.pthread
 )
 
+add_entrypoint_object(
+  pthread_rwlockattr_setkind_np
+  SRCS
+    pthread_rwlockattr_setkind_np.cpp
+  HDRS
+    pthread_rwlockattr_setkind_np.h
+  DEPENDS
+    libc.include.pthread
+    libc.include.errno
+)
+
 add_entrypoint_object(
   pthread_rwlockattr_setpshared
   SRCS
diff --git a/libc/src/pthread/pthread_rwlockattr_getkind_np.cpp b/libc/src/pthread/pthread_rwlockattr_getkind_np.cpp
new file mode 100644
index 000000000000..0c821797b42c
--- /dev/null
+++ b/libc/src/pthread/pthread_rwlockattr_getkind_np.cpp
@@ -0,0 +1,24 @@
+//===-- Implementation of the pthread_rwlockattr_getkind_np ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "pthread_rwlockattr_getkind_np.h"
+
+#include "src/__support/common.h"
+
+#include <pthread.h> // pthread_rwlockattr_t
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, pthread_rwlockattr_getkind_np,
+                   (const pthread_rwlockattr_t *__restrict attr,
+                    int *__restrict pref)) {
+  *pref = attr->pref;
+  return 0;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/pthread/pthread_rwlockattr_getkind_np.h b/libc/src/pthread/pthread_rwlockattr_getkind_np.h
new file mode 100644
index 000000000000..51f633cd559d
--- /dev/null
+++ b/libc/src/pthread/pthread_rwlockattr_getkind_np.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for pthread_rwlockattr_getkind_np -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_GETKIND_NP_H
+#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_GETKIND_NP_H
+
+#include <pthread.h>
+
+namespace LIBC_NAMESPACE {
+
+int pthread_rwlockattr_getkind_np(const pthread_rwlockattr_t *__restrict attr,
+                                  int *__restrict pref);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_GETKIND_NP_H
diff --git a/libc/src/pthread/pthread_rwlockattr_init.cpp b/libc/src/pthread/pthread_rwlockattr_init.cpp
index 7971f1714db4..bbc89555c6c1 100644
--- a/libc/src/pthread/pthread_rwlockattr_init.cpp
+++ b/libc/src/pthread/pthread_rwlockattr_init.cpp
@@ -17,6 +17,7 @@ namespace LIBC_NAMESPACE {
 LLVM_LIBC_FUNCTION(int, pthread_rwlockattr_init,
                    (pthread_rwlockattr_t * attr)) {
   attr->pshared = PTHREAD_PROCESS_PRIVATE;
+  attr->pref = PTHREAD_RWLOCK_PREFER_READER_NP;
   return 0;
 }
 
diff --git a/libc/src/pthread/pthread_rwlockattr_setkind_np.cpp b/libc/src/pthread/pthread_rwlockattr_setkind_np.cpp
new file mode 100644
index 000000000000..47fbf2a851e5
--- /dev/null
+++ b/libc/src/pthread/pthread_rwlockattr_setkind_np.cpp
@@ -0,0 +1,30 @@
+//===-- Implementation of the pthread_rwlockattr_setkind_np ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "pthread_rwlockattr_setkind_np.h"
+
+#include "src/__support/common.h"
+
+#include <errno.h>
+#include <pthread.h> // pthread_rwlockattr_t
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, pthread_rwlockattr_setkind_np,
+                   (pthread_rwlockattr_t * attr, int pref)) {
+
+  if (pref != PTHREAD_RWLOCK_PREFER_READER_NP &&
+      pref != PTHREAD_RWLOCK_PREFER_WRITER_NP &&
+      pref != PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)
+    return EINVAL;
+
+  attr->pref = pref;
+  return 0;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/pthread/pthread_rwlockattr_setkind_np.h b/libc/src/pthread/pthread_rwlockattr_setkind_np.h
new file mode 100644
index 000000000000..00ef8e1bbe00
--- /dev/null
+++ b/libc/src/pthread/pthread_rwlockattr_setkind_np.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for pthread_rwlockattr_setkind_np -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_SETKIND_NP_H
+#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_SETKIND_NP_H
+
+#include <pthread.h>
+
+namespace LIBC_NAMESPACE {
+
+int pthread_rwlockattr_setkind_np(pthread_rwlockattr_t *attr, int pref);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_SETKIND_NP_H
diff --git a/libc/test/src/pthread/CMakeLists.txt b/libc/test/src/pthread/CMakeLists.txt
index ea75e65f57c9..0eeec445d5f4 100644
--- a/libc/test/src/pthread/CMakeLists.txt
+++ b/libc/test/src/pthread/CMakeLists.txt
@@ -68,7 +68,9 @@ add_libc_unittest(
     libc.include.errno
     libc.include.pthread
     libc.src.pthread.pthread_rwlockattr_destroy
+    libc.src.pthread.pthread_rwlockattr_getkind_np
     libc.src.pthread.pthread_rwlockattr_getpshared
     libc.src.pthread.pthread_rwlockattr_init
+    libc.src.pthread.pthread_rwlockattr_setkind_np
     libc.src.pthread.pthread_rwlockattr_setpshared
 )
diff --git a/libc/test/src/pthread/pthread_rwlockattr_test.cpp b/libc/test/src/pthread/pthread_rwlockattr_test.cpp
index 6e5ae70df734..3791f568e222 100644
--- a/libc/test/src/pthread/pthread_rwlockattr_test.cpp
+++ b/libc/test/src/pthread/pthread_rwlockattr_test.cpp
@@ -8,8 +8,10 @@
 
 #include "include/llvm-libc-macros/generic-error-number-macros.h" // EINVAL
 #include "src/pthread/pthread_rwlockattr_destroy.h"
+#include "src/pthread/pthread_rwlockattr_getkind_np.h"
 #include "src/pthread/pthread_rwlockattr_getpshared.h"
 #include "src/pthread/pthread_rwlockattr_init.h"
+#include "src/pthread/pthread_rwlockattr_setkind_np.h"
 #include "src/pthread/pthread_rwlockattr_setpshared.h"
 #include "test/UnitTest/Test.h"
 
@@ -25,40 +27,61 @@ TEST(LlvmLibcPThreadRWLockAttrTest, InitAndDestroy) {
 TEST(LlvmLibcPThreadRWLockAttrTest, GetDefaultValues) {
   pthread_rwlockattr_t attr;
 
-  // Invalid value.
+  // Invalid values.
   int pshared = 42;
+  int pref = 1337;
 
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_init(&attr), 0);
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getpshared(&attr, &pshared), 0);
+  ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getkind_np(&attr, &pref), 0);
+
   ASSERT_EQ(pshared, PTHREAD_PROCESS_PRIVATE);
+  ASSERT_EQ(pref, PTHREAD_RWLOCK_PREFER_READER_NP);
+
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_destroy(&attr), 0);
 }
 
 TEST(LlvmLibcPThreadRWLockAttrTest, SetGoodValues) {
   pthread_rwlockattr_t attr;
 
-  // Invalid value.
+  // Invalid values.
   int pshared = 42;
+  int pref = 1337;
 
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_init(&attr), 0);
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_setpshared(
                 &attr, PTHREAD_PROCESS_SHARED),
             0);
+  ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_setkind_np(
+                &attr, PTHREAD_RWLOCK_PREFER_WRITER_NP),
+            0);
+
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getpshared(&attr, &pshared), 0);
+  ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getkind_np(&attr, &pref), 0);
+
   ASSERT_EQ(pshared, PTHREAD_PROCESS_SHARED);
+  ASSERT_EQ(pref, PTHREAD_RWLOCK_PREFER_WRITER_NP);
+
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_destroy(&attr), 0);
 }
 
 TEST(LlvmLibcPThreadRWLockAttrTest, SetBadValues) {
   pthread_rwlockattr_t attr;
 
-  // Invalid value.
+  // Invalid values.
   int pshared = 42;
+  int pref = 1337;
 
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_init(&attr), 0);
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_setpshared(&attr, pshared),
             EINVAL);
+  ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_setkind_np(&attr, pref), EINVAL);
+
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getpshared(&attr, &pshared), 0);
+  ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getkind_np(&attr, &pref), 0);
+
   ASSERT_EQ(pshared, PTHREAD_PROCESS_PRIVATE);
+  ASSERT_EQ(pref, PTHREAD_RWLOCK_PREFER_READER_NP);
+
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_destroy(&attr), 0);
 }
-- 
GitLab


From 29a986bc0972c7d2deb85f8789ea3183c8ca06ca Mon Sep 17 00:00:00 2001
From: Matthias Gehre <matthias.gehre@amd.com>
Date: Mon, 13 May 2024 15:36:46 +0200
Subject: [PATCH 0562/1206] new-prs-labeler.yml: Include conversion passes of
 TOSA/emitc (#91921)

---
 .github/new-prs-labeler.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml
index d608ea449f1d..a57ba28faf16 100644
--- a/.github/new-prs-labeler.yml
+++ b/.github/new-prs-labeler.yml
@@ -239,7 +239,7 @@ mlir:dlti:
   - mlir/**/DLTI/**
 
 mlir:emitc:
-  - mlir/**/EmitC/**
+  - mlir/**/*EmitC*/**
   - mlir/lib/Target/Cpp/**
 
 mlir:func:
@@ -306,7 +306,7 @@ mlir:tensor:
   - mlir/**/Tensor/**
 
 mlir:tosa:
-  - mlir/**/Tosa/**
+  - mlir/**/*Tosa*/**
 
 mlir:ub:
   - mlir/**/UB/**
-- 
GitLab


From 27595c4befbbf42891ef7e13fe28926eb7fc825d Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Mon, 13 May 2024 09:43:14 -0400
Subject: [PATCH 0563/1206] Revert "[libc][POSIX][pthreads] implemented missing
 pthread_rwlockattr functions" (#91966)

Reverts llvm/llvm-project#90249

Fullbuild is broken:
https://lab.llvm.org/buildbot/#/builders/163/builds/56501
---
 libc/config/linux/x86_64/entrypoints.txt      |  2 --
 libc/include/pthread.h.def                    |  5 ----
 libc/spec/posix.td                            | 10 -------
 libc/src/pthread/CMakeLists.txt               | 21 -------------
 .../pthread/pthread_rwlockattr_getkind_np.cpp | 24 ---------------
 .../pthread/pthread_rwlockattr_getkind_np.h   | 21 -------------
 libc/src/pthread/pthread_rwlockattr_init.cpp  |  1 -
 .../pthread/pthread_rwlockattr_setkind_np.cpp | 30 -------------------
 .../pthread/pthread_rwlockattr_setkind_np.h   | 20 -------------
 libc/test/src/pthread/CMakeLists.txt          |  2 --
 .../src/pthread/pthread_rwlockattr_test.cpp   | 29 ++----------------
 11 files changed, 3 insertions(+), 162 deletions(-)
 delete mode 100644 libc/src/pthread/pthread_rwlockattr_getkind_np.cpp
 delete mode 100644 libc/src/pthread/pthread_rwlockattr_getkind_np.h
 delete mode 100644 libc/src/pthread/pthread_rwlockattr_setkind_np.cpp
 delete mode 100644 libc/src/pthread/pthread_rwlockattr_setkind_np.h

diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 155deddaaade..5e3ddd34fb4d 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -673,10 +673,8 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.pthread.pthread_mutexattr_settype
     libc.src.pthread.pthread_once
     libc.src.pthread.pthread_rwlockattr_destroy
-    libc.src.pthread.pthread_rwlockattr_getkind_np
     libc.src.pthread.pthread_rwlockattr_getpshared
     libc.src.pthread.pthread_rwlockattr_init
-    libc.src.pthread.pthread_rwlockattr_setkind_np
     libc.src.pthread.pthread_rwlockattr_setpshared
     libc.src.pthread.pthread_setspecific
 
diff --git a/libc/include/pthread.h.def b/libc/include/pthread.h.def
index d41273b5590e..a94d770657e1 100644
--- a/libc/include/pthread.h.def
+++ b/libc/include/pthread.h.def
@@ -38,11 +38,6 @@ enum {
 #define PTHREAD_PROCESS_PRIVATE 0
 #define PTHREAD_PROCESS_SHARED 1
 
-#define PTHREAD_RWLOCK_PREFER_READER_NP 0
-#define PTHREAD_RWLOCK_PREFER_WRITER_NP 1
-#define PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP 2
-
-
 %%public_api()
 
 #endif // LLVM_LIBC_PTHREAD_H
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
index e16353b8142d..e7a0cf883c60 100644
--- a/libc/spec/posix.td
+++ b/libc/spec/posix.td
@@ -1234,11 +1234,6 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<IntType>,
           [ArgSpec<PThreadRWLockAttrTPtr>]
       >,
-      FunctionSpec<
-          "pthread_rwlockattr_getkind_np",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadRWLockAttrTPtr>, ArgSpec<IntPtr>]
-      >,
       FunctionSpec<
           "pthread_rwlockattr_getpshared",
           RetValSpec<IntType>,
@@ -1249,11 +1244,6 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<IntType>,
           [ArgSpec<PThreadRWLockAttrTPtr>]
       >,
-      FunctionSpec<
-          "pthread_rwlockattr_setkind_np",
-          RetValSpec<IntType>,
-          [ArgSpec<PThreadRWLockAttrTPtr>, ArgSpec<IntType>]
-      >,
       FunctionSpec<
           "pthread_rwlockattr_setpshared",
           RetValSpec<IntType>,
diff --git a/libc/src/pthread/CMakeLists.txt b/libc/src/pthread/CMakeLists.txt
index e5bebb63c640..c57475c9114f 100644
--- a/libc/src/pthread/CMakeLists.txt
+++ b/libc/src/pthread/CMakeLists.txt
@@ -470,16 +470,6 @@ add_entrypoint_object(
     libc.include.pthread
 )
 
-add_entrypoint_object(
-  pthread_rwlockattr_getkind_np
-  SRCS
-    pthread_rwlockattr_getkind_np.cpp
-  HDRS
-    pthread_rwlockattr_getkind_np.h
-  DEPENDS
-    libc.include.pthread
-)
-
 add_entrypoint_object(
   pthread_rwlockattr_getpshared
   SRCS
@@ -500,17 +490,6 @@ add_entrypoint_object(
     libc.include.pthread
 )
 
-add_entrypoint_object(
-  pthread_rwlockattr_setkind_np
-  SRCS
-    pthread_rwlockattr_setkind_np.cpp
-  HDRS
-    pthread_rwlockattr_setkind_np.h
-  DEPENDS
-    libc.include.pthread
-    libc.include.errno
-)
-
 add_entrypoint_object(
   pthread_rwlockattr_setpshared
   SRCS
diff --git a/libc/src/pthread/pthread_rwlockattr_getkind_np.cpp b/libc/src/pthread/pthread_rwlockattr_getkind_np.cpp
deleted file mode 100644
index 0c821797b42c..000000000000
--- a/libc/src/pthread/pthread_rwlockattr_getkind_np.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-//===-- Implementation of the pthread_rwlockattr_getkind_np ---------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "pthread_rwlockattr_getkind_np.h"
-
-#include "src/__support/common.h"
-
-#include <pthread.h> // pthread_rwlockattr_t
-
-namespace LIBC_NAMESPACE {
-
-LLVM_LIBC_FUNCTION(int, pthread_rwlockattr_getkind_np,
-                   (const pthread_rwlockattr_t *__restrict attr,
-                    int *__restrict pref)) {
-  *pref = attr->pref;
-  return 0;
-}
-
-} // namespace LIBC_NAMESPACE
diff --git a/libc/src/pthread/pthread_rwlockattr_getkind_np.h b/libc/src/pthread/pthread_rwlockattr_getkind_np.h
deleted file mode 100644
index 51f633cd559d..000000000000
--- a/libc/src/pthread/pthread_rwlockattr_getkind_np.h
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- Implementation header for pthread_rwlockattr_getkind_np -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_GETKIND_NP_H
-#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_GETKIND_NP_H
-
-#include <pthread.h>
-
-namespace LIBC_NAMESPACE {
-
-int pthread_rwlockattr_getkind_np(const pthread_rwlockattr_t *__restrict attr,
-                                  int *__restrict pref);
-
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_GETKIND_NP_H
diff --git a/libc/src/pthread/pthread_rwlockattr_init.cpp b/libc/src/pthread/pthread_rwlockattr_init.cpp
index bbc89555c6c1..7971f1714db4 100644
--- a/libc/src/pthread/pthread_rwlockattr_init.cpp
+++ b/libc/src/pthread/pthread_rwlockattr_init.cpp
@@ -17,7 +17,6 @@ namespace LIBC_NAMESPACE {
 LLVM_LIBC_FUNCTION(int, pthread_rwlockattr_init,
                    (pthread_rwlockattr_t * attr)) {
   attr->pshared = PTHREAD_PROCESS_PRIVATE;
-  attr->pref = PTHREAD_RWLOCK_PREFER_READER_NP;
   return 0;
 }
 
diff --git a/libc/src/pthread/pthread_rwlockattr_setkind_np.cpp b/libc/src/pthread/pthread_rwlockattr_setkind_np.cpp
deleted file mode 100644
index 47fbf2a851e5..000000000000
--- a/libc/src/pthread/pthread_rwlockattr_setkind_np.cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-//===-- Implementation of the pthread_rwlockattr_setkind_np ---------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "pthread_rwlockattr_setkind_np.h"
-
-#include "src/__support/common.h"
-
-#include <errno.h>
-#include <pthread.h> // pthread_rwlockattr_t
-
-namespace LIBC_NAMESPACE {
-
-LLVM_LIBC_FUNCTION(int, pthread_rwlockattr_setkind_np,
-                   (pthread_rwlockattr_t * attr, int pref)) {
-
-  if (pref != PTHREAD_RWLOCK_PREFER_READER_NP &&
-      pref != PTHREAD_RWLOCK_PREFER_WRITER_NP &&
-      pref != PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)
-    return EINVAL;
-
-  attr->pref = pref;
-  return 0;
-}
-
-} // namespace LIBC_NAMESPACE
diff --git a/libc/src/pthread/pthread_rwlockattr_setkind_np.h b/libc/src/pthread/pthread_rwlockattr_setkind_np.h
deleted file mode 100644
index 00ef8e1bbe00..000000000000
--- a/libc/src/pthread/pthread_rwlockattr_setkind_np.h
+++ /dev/null
@@ -1,20 +0,0 @@
-//===-- Implementation header for pthread_rwlockattr_setkind_np -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_SETKIND_NP_H
-#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_SETKIND_NP_H
-
-#include <pthread.h>
-
-namespace LIBC_NAMESPACE {
-
-int pthread_rwlockattr_setkind_np(pthread_rwlockattr_t *attr, int pref);
-
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_RWLOCKATTR_SETKIND_NP_H
diff --git a/libc/test/src/pthread/CMakeLists.txt b/libc/test/src/pthread/CMakeLists.txt
index 0eeec445d5f4..ea75e65f57c9 100644
--- a/libc/test/src/pthread/CMakeLists.txt
+++ b/libc/test/src/pthread/CMakeLists.txt
@@ -68,9 +68,7 @@ add_libc_unittest(
     libc.include.errno
     libc.include.pthread
     libc.src.pthread.pthread_rwlockattr_destroy
-    libc.src.pthread.pthread_rwlockattr_getkind_np
     libc.src.pthread.pthread_rwlockattr_getpshared
     libc.src.pthread.pthread_rwlockattr_init
-    libc.src.pthread.pthread_rwlockattr_setkind_np
     libc.src.pthread.pthread_rwlockattr_setpshared
 )
diff --git a/libc/test/src/pthread/pthread_rwlockattr_test.cpp b/libc/test/src/pthread/pthread_rwlockattr_test.cpp
index 3791f568e222..6e5ae70df734 100644
--- a/libc/test/src/pthread/pthread_rwlockattr_test.cpp
+++ b/libc/test/src/pthread/pthread_rwlockattr_test.cpp
@@ -8,10 +8,8 @@
 
 #include "include/llvm-libc-macros/generic-error-number-macros.h" // EINVAL
 #include "src/pthread/pthread_rwlockattr_destroy.h"
-#include "src/pthread/pthread_rwlockattr_getkind_np.h"
 #include "src/pthread/pthread_rwlockattr_getpshared.h"
 #include "src/pthread/pthread_rwlockattr_init.h"
-#include "src/pthread/pthread_rwlockattr_setkind_np.h"
 #include "src/pthread/pthread_rwlockattr_setpshared.h"
 #include "test/UnitTest/Test.h"
 
@@ -27,61 +25,40 @@ TEST(LlvmLibcPThreadRWLockAttrTest, InitAndDestroy) {
 TEST(LlvmLibcPThreadRWLockAttrTest, GetDefaultValues) {
   pthread_rwlockattr_t attr;
 
-  // Invalid values.
+  // Invalid value.
   int pshared = 42;
-  int pref = 1337;
 
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_init(&attr), 0);
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getpshared(&attr, &pshared), 0);
-  ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getkind_np(&attr, &pref), 0);
-
   ASSERT_EQ(pshared, PTHREAD_PROCESS_PRIVATE);
-  ASSERT_EQ(pref, PTHREAD_RWLOCK_PREFER_READER_NP);
-
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_destroy(&attr), 0);
 }
 
 TEST(LlvmLibcPThreadRWLockAttrTest, SetGoodValues) {
   pthread_rwlockattr_t attr;
 
-  // Invalid values.
+  // Invalid value.
   int pshared = 42;
-  int pref = 1337;
 
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_init(&attr), 0);
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_setpshared(
                 &attr, PTHREAD_PROCESS_SHARED),
             0);
-  ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_setkind_np(
-                &attr, PTHREAD_RWLOCK_PREFER_WRITER_NP),
-            0);
-
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getpshared(&attr, &pshared), 0);
-  ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getkind_np(&attr, &pref), 0);
-
   ASSERT_EQ(pshared, PTHREAD_PROCESS_SHARED);
-  ASSERT_EQ(pref, PTHREAD_RWLOCK_PREFER_WRITER_NP);
-
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_destroy(&attr), 0);
 }
 
 TEST(LlvmLibcPThreadRWLockAttrTest, SetBadValues) {
   pthread_rwlockattr_t attr;
 
-  // Invalid values.
+  // Invalid value.
   int pshared = 42;
-  int pref = 1337;
 
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_init(&attr), 0);
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_setpshared(&attr, pshared),
             EINVAL);
-  ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_setkind_np(&attr, pref), EINVAL);
-
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getpshared(&attr, &pshared), 0);
-  ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_getkind_np(&attr, &pref), 0);
-
   ASSERT_EQ(pshared, PTHREAD_PROCESS_PRIVATE);
-  ASSERT_EQ(pref, PTHREAD_RWLOCK_PREFER_READER_NP);
-
   ASSERT_EQ(LIBC_NAMESPACE::pthread_rwlockattr_destroy(&attr), 0);
 }
-- 
GitLab


From 25a3ba33153e99c4614d404ba18b761d652e24de Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Mon, 13 May 2024 08:42:06 -0500
Subject: [PATCH 0564/1206] Revert "[flang][OpenMP] Decompose compound
 constructs, do recursive lowering (#90098)"

It breaks some builds, e.g.
https://lab.llvm.org/buildbot/#/builders/268/builds/13909

This reverts commit ca1bd5995f6ed934f9187305190a5abfac049173.
---
 flang/lib/Lower/CMakeLists.txt                |    1 -
 flang/lib/Lower/OpenMP/Clauses.cpp            |   23 -
 flang/lib/Lower/OpenMP/Clauses.h              |   13 +-
 flang/lib/Lower/OpenMP/Decomposer.cpp         |  126 --
 flang/lib/Lower/OpenMP/Decomposer.h           |   51 -
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  828 ++++++------
 flang/lib/Lower/OpenMP/Utils.cpp              |    6 +
 flang/lib/Lower/OpenMP/Utils.h                |    1 +
 .../Lower/OpenMP/default-clause-byref.f90     |    5 +-
 flang/test/Lower/OpenMP/default-clause.f90    |    4 +-
 .../parallel-lastprivate-clause-scalar.f90    |    4 +-
 llvm/include/llvm/Frontend/OpenMP/ClauseT.h   |   52 +-
 .../Frontend/OpenMP/ConstructCompositionT.h   |  403 ------
 .../Frontend/OpenMP/ConstructDecompositionT.h | 1161 -----------------
 llvm/unittests/Frontend/CMakeLists.txt        |    1 -
 .../Frontend/OpenMPDecompositionTest.cpp      |  999 --------------
 16 files changed, 452 insertions(+), 3226 deletions(-)
 delete mode 100644 flang/lib/Lower/OpenMP/Decomposer.cpp
 delete mode 100644 flang/lib/Lower/OpenMP/Decomposer.h
 delete mode 100644 llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
 delete mode 100644 llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
 delete mode 100644 llvm/unittests/Frontend/OpenMPDecompositionTest.cpp

diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index 1546409752e7..f92d1a2bc7de 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -27,7 +27,6 @@ add_flang_library(FortranLower
   OpenMP/ClauseProcessor.cpp
   OpenMP/Clauses.cpp
   OpenMP/DataSharingProcessor.cpp
-  OpenMP/Decomposer.cpp
   OpenMP/OpenMP.cpp
   OpenMP/ReductionProcessor.cpp
   OpenMP/Utils.cpp
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 87370c92964a..97337cfc08c7 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1227,27 +1227,4 @@ List<Clause> makeClauses(const parser::OmpClauseList &clauses,
     return makeClause(s, semaCtx);
   });
 }
-
-bool transferLocations(const List<Clause> &from, List<Clause> &to) {
-  bool allDone = true;
-
-  for (Clause &clause : to) {
-    if (!clause.source.empty())
-      continue;
-    auto found =
-        llvm::find_if(from, [&](const Clause &c) { return c.id == clause.id; });
-    // This is not completely accurate, but should be good enough for now.
-    // It can be improved in the future if necessary, but in cases of
-    // synthesized clauses getting accurate location may be impossible.
-    if (found != from.end()) {
-      clause.source = found->source;
-    } else {
-      // Found a clause that won't have "source".
-      allDone = false;
-    }
-  }
-
-  return allDone;
-}
-
 } // namespace Fortran::lower::omp
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index 407579319279..3e776425c733 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -23,15 +23,11 @@
 
 namespace Fortran::lower::omp {
 using namespace Fortran;
+using SomeType = evaluate::SomeType;
 using SomeExpr = semantics::SomeExpr;
 using MaybeExpr = semantics::MaybeExpr;
 
-// evaluate::SomeType doesn't provide == operation. It's not really used in
-// flang's clauses so far, so a trivial implementation is sufficient.
-struct TypeTy : public evaluate::SomeType {
-  bool operator==(const TypeTy &t) const { return true; }
-};
-
+using TypeTy = SomeType;
 using IdTy = semantics::Symbol *;
 using ExprTy = SomeExpr;
 
@@ -226,8 +222,6 @@ using When = tomp::clause::WhenT<TypeTy, IdTy, ExprTy>;
 using Write = tomp::clause::WriteT<TypeTy, IdTy, ExprTy>;
 } // namespace clause
 
-using tomp::type::operator==;
-
 struct CancellationConstructType {
   using EmptyTrait = std::true_type;
 };
@@ -250,7 +244,6 @@ using ClauseBase = tomp::ClauseT<TypeTy, IdTy, ExprTy,
                                  MemoryOrder, Threadprivate>;
 
 struct Clause : public ClauseBase {
-  // "source" will be ignored by tomp::type::operator==.
   parser::CharBlock source;
 };
 
@@ -265,8 +258,6 @@ Clause makeClause(const Fortran::parser::OmpClause &cls,
 
 List<Clause> makeClauses(const parser::OmpClauseList &clauses,
                          semantics::SemanticsContext &semaCtx);
-
-bool transferLocations(const List<Clause> &from, List<Clause> &to);
 } // namespace Fortran::lower::omp
 
 #endif // FORTRAN_LOWER_OPENMP_CLAUSES_H
diff --git a/flang/lib/Lower/OpenMP/Decomposer.cpp b/flang/lib/Lower/OpenMP/Decomposer.cpp
deleted file mode 100644
index e6897cb81e94..000000000000
--- a/flang/lib/Lower/OpenMP/Decomposer.cpp
+++ /dev/null
@@ -1,126 +0,0 @@
-//===-- Decomposer.cpp -- Compound directive decomposition ----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
-//
-//===----------------------------------------------------------------------===//
-
-#include "Decomposer.h"
-
-#include "Clauses.h"
-#include "Utils.h"
-#include "flang/Lower/PFTBuilder.h"
-#include "flang/Semantics/semantics.h"
-#include "flang/Tools/CrossToolHelpers.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Frontend/OpenMP/ClauseT.h"
-#include "llvm/Frontend/OpenMP/ConstructCompositionT.h"
-#include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
-#include "llvm/Frontend/OpenMP/OMP.h"
-#include "llvm/Support/raw_ostream.h"
-
-#include <optional>
-#include <utility>
-#include <variant>
-
-using namespace Fortran;
-
-namespace {
-using namespace Fortran::lower::omp;
-
-struct ConstructDecomposition {
-  ConstructDecomposition(mlir::ModuleOp modOp,
-                         semantics::SemanticsContext &semaCtx,
-                         lower::pft::Evaluation &ev,
-                         llvm::omp::Directive compound,
-                         const List<Clause> &clauses)
-      : semaCtx(semaCtx), mod(modOp), eval(ev) {
-    tomp::ConstructDecompositionT decompose(getOpenMPVersionAttribute(modOp),
-                                            *this, compound,
-                                            llvm::ArrayRef(clauses));
-    output = std::move(decompose.output);
-  }
-
-  // Given an object, return its base object if one exists.
-  std::optional<Object> getBaseObject(const Object &object) {
-    return lower::omp::getBaseObject(object, semaCtx);
-  }
-
-  // Return the iteration variable of the associated loop if any.
-  std::optional<Object> getLoopIterVar() {
-    if (semantics::Symbol *symbol = getIterationVariableSymbol(eval))
-      return Object{symbol, /*designator=*/{}};
-    return std::nullopt;
-  }
-
-  semantics::SemanticsContext &semaCtx;
-  mlir::ModuleOp mod;
-  lower::pft::Evaluation &eval;
-  List<UnitConstruct> output;
-};
-} // namespace
-
-static UnitConstruct mergeConstructs(uint32_t version,
-                                     llvm::ArrayRef<UnitConstruct> units) {
-  tomp::ConstructCompositionT compose(version, units);
-  return compose.merged;
-}
-
-namespace Fortran::lower::omp {
-LLVM_DUMP_METHOD llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                               const UnitConstruct &uc) {
-  os << llvm::omp::getOpenMPDirectiveName(uc.id);
-  for (auto [index, clause] : llvm::enumerate(uc.clauses)) {
-    os << (index == 0 ? '\t' : ' ');
-    os << llvm::omp::getOpenMPClauseName(clause.id);
-  }
-  return os;
-}
-
-ConstructQueue buildConstructQueue(
-    mlir::ModuleOp modOp, Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, const parser::CharBlock &source,
-    llvm::omp::Directive compound, const List<Clause> &clauses) {
-
-  List<UnitConstruct> constructs;
-
-  ConstructDecomposition decompose(modOp, semaCtx, eval, compound, clauses);
-  assert(!decompose.output.empty() && "Construct decomposition failed");
-
-  llvm::SmallVector<llvm::omp::Directive> loweringUnits;
-  std::ignore =
-      llvm::omp::getLeafOrCompositeConstructs(compound, loweringUnits);
-  uint32_t version = getOpenMPVersionAttribute(modOp);
-
-  int leafIndex = 0;
-  for (llvm::omp::Directive dir_id : loweringUnits) {
-    llvm::ArrayRef<llvm::omp::Directive> leafsOrSelf =
-        llvm::omp::getLeafConstructsOrSelf(dir_id);
-    size_t numLeafs = leafsOrSelf.size();
-
-    llvm::ArrayRef<UnitConstruct> toMerge{&decompose.output[leafIndex],
-                                          numLeafs};
-    auto &uc = constructs.emplace_back(mergeConstructs(version, toMerge));
-
-    if (!transferLocations(clauses, uc.clauses)) {
-      // If some clauses are left without source information, use the
-      // directive's source.
-      for (auto &clause : uc.clauses) {
-        if (clause.source.empty())
-          clause.source = source;
-      }
-    }
-    leafIndex += numLeafs;
-  }
-
-  return constructs;
-}
-} // namespace Fortran::lower::omp
diff --git a/flang/lib/Lower/OpenMP/Decomposer.h b/flang/lib/Lower/OpenMP/Decomposer.h
deleted file mode 100644
index f42d8f5c1740..000000000000
--- a/flang/lib/Lower/OpenMP/Decomposer.h
+++ /dev/null
@@ -1,51 +0,0 @@
-//===-- Decomposer.h -- Compound directive decomposition ------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-#ifndef FORTRAN_LOWER_OPENMP_DECOMPOSER_H
-#define FORTRAN_LOWER_OPENMP_DECOMPOSER_H
-
-#include "Clauses.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "llvm/Frontend/OpenMP/ConstructCompositionT.h"
-#include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
-#include "llvm/Frontend/OpenMP/OMP.h"
-#include "llvm/Support/Compiler.h"
-
-namespace llvm {
-class raw_ostream;
-}
-
-namespace Fortran {
-namespace semantics {
-class SemanticsContext;
-}
-namespace lower::pft {
-struct Evaluation;
-}
-} // namespace Fortran
-
-namespace Fortran::lower::omp {
-using UnitConstruct = tomp::DirectiveWithClauses<lower::omp::Clause>;
-using ConstructQueue = List<UnitConstruct>;
-
-LLVM_DUMP_METHOD llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
-                                               const UnitConstruct &uc);
-
-// Given a potentially compound construct with a list of clauses that
-// apply to it, break it up into individual sub-constructs each with
-// the subset of applicable clauses (plus implicit clauses, if any).
-// From that create a work queue where each work item corresponds to
-// the sub-construct with its clauses.
-ConstructQueue buildConstructQueue(mlir::ModuleOp modOp,
-                                   semantics::SemanticsContext &semaCtx,
-                                   lower::pft::Evaluation &eval,
-                                   const parser::CharBlock &source,
-                                   llvm::omp::Directive compound,
-                                   const List<Clause> &clauses);
-} // namespace Fortran::lower::omp
-
-#endif // FORTRAN_LOWER_OPENMP_DECOMPOSER_H
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index eaf4b5f997ff..f23902d6a823 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -15,7 +15,6 @@
 #include "ClauseProcessor.h"
 #include "Clauses.h"
 #include "DataSharingProcessor.h"
-#include "Decomposer.h"
 #include "DirectivesCommon.h"
 #include "ReductionProcessor.h"
 #include "Utils.h"
@@ -45,13 +44,6 @@ using namespace Fortran::lower::omp;
 // Code generation helper functions
 //===----------------------------------------------------------------------===//
 
-static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
-                           Fortran::lower::SymMap &symTable,
-                           Fortran::semantics::SemanticsContext &semaCtx,
-                           Fortran::lower::pft::Evaluation &eval,
-                           mlir::Location loc, const ConstructQueue &queue,
-                           ConstructQueue::iterator item);
-
 static Fortran::lower::pft::Evaluation *
 getCollapsedLoopEval(Fortran::lower::pft::Evaluation &eval, int collapseValue) {
   // Return the Evaluation of the innermost collapsed loop, or the current one
@@ -468,6 +460,81 @@ markDeclareTarget(mlir::Operation *op,
   declareTargetOp.setDeclareTarget(deviceType, captureClause);
 }
 
+/// Split a combined directive into an outer leaf directive and the (possibly
+/// combined) rest of the combined directive. Composite directives and
+/// non-compound directives are not split, in which case it will return the
+/// input directive as its first output and an empty value as its second output.
+static std::pair<llvm::omp::Directive, std::optional<llvm::omp::Directive>>
+splitCombinedDirective(llvm::omp::Directive dir) {
+  using D = llvm::omp::Directive;
+  switch (dir) {
+  case D::OMPD_masked_taskloop:
+    return {D::OMPD_masked, D::OMPD_taskloop};
+  case D::OMPD_masked_taskloop_simd:
+    return {D::OMPD_masked, D::OMPD_taskloop_simd};
+  case D::OMPD_master_taskloop:
+    return {D::OMPD_master, D::OMPD_taskloop};
+  case D::OMPD_master_taskloop_simd:
+    return {D::OMPD_master, D::OMPD_taskloop_simd};
+  case D::OMPD_parallel_do:
+    return {D::OMPD_parallel, D::OMPD_do};
+  case D::OMPD_parallel_do_simd:
+    return {D::OMPD_parallel, D::OMPD_do_simd};
+  case D::OMPD_parallel_masked:
+    return {D::OMPD_parallel, D::OMPD_masked};
+  case D::OMPD_parallel_masked_taskloop:
+    return {D::OMPD_parallel, D::OMPD_masked_taskloop};
+  case D::OMPD_parallel_masked_taskloop_simd:
+    return {D::OMPD_parallel, D::OMPD_masked_taskloop_simd};
+  case D::OMPD_parallel_master:
+    return {D::OMPD_parallel, D::OMPD_master};
+  case D::OMPD_parallel_master_taskloop:
+    return {D::OMPD_parallel, D::OMPD_master_taskloop};
+  case D::OMPD_parallel_master_taskloop_simd:
+    return {D::OMPD_parallel, D::OMPD_master_taskloop_simd};
+  case D::OMPD_parallel_sections:
+    return {D::OMPD_parallel, D::OMPD_sections};
+  case D::OMPD_parallel_workshare:
+    return {D::OMPD_parallel, D::OMPD_workshare};
+  case D::OMPD_target_parallel:
+    return {D::OMPD_target, D::OMPD_parallel};
+  case D::OMPD_target_parallel_do:
+    return {D::OMPD_target, D::OMPD_parallel_do};
+  case D::OMPD_target_parallel_do_simd:
+    return {D::OMPD_target, D::OMPD_parallel_do_simd};
+  case D::OMPD_target_simd:
+    return {D::OMPD_target, D::OMPD_simd};
+  case D::OMPD_target_teams:
+    return {D::OMPD_target, D::OMPD_teams};
+  case D::OMPD_target_teams_distribute:
+    return {D::OMPD_target, D::OMPD_teams_distribute};
+  case D::OMPD_target_teams_distribute_parallel_do:
+    return {D::OMPD_target, D::OMPD_teams_distribute_parallel_do};
+  case D::OMPD_target_teams_distribute_parallel_do_simd:
+    return {D::OMPD_target, D::OMPD_teams_distribute_parallel_do_simd};
+  case D::OMPD_target_teams_distribute_simd:
+    return {D::OMPD_target, D::OMPD_teams_distribute_simd};
+  case D::OMPD_teams_distribute:
+    return {D::OMPD_teams, D::OMPD_distribute};
+  case D::OMPD_teams_distribute_parallel_do:
+    return {D::OMPD_teams, D::OMPD_distribute_parallel_do};
+  case D::OMPD_teams_distribute_parallel_do_simd:
+    return {D::OMPD_teams, D::OMPD_distribute_parallel_do_simd};
+  case D::OMPD_teams_distribute_simd:
+    return {D::OMPD_teams, D::OMPD_distribute_simd};
+  case D::OMPD_parallel_loop:
+    return {D::OMPD_parallel, D::OMPD_loop};
+  case D::OMPD_target_parallel_loop:
+    return {D::OMPD_target, D::OMPD_parallel_loop};
+  case D::OMPD_target_teams_loop:
+    return {D::OMPD_target, D::OMPD_teams_loop};
+  case D::OMPD_teams_loop:
+    return {D::OMPD_teams, D::OMPD_loop};
+  default:
+    return {dir, std::nullopt};
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // Op body generation helper structures and functions
 //===----------------------------------------------------------------------===//
@@ -488,6 +555,11 @@ struct OpWithBodyGenInfo {
       : converter(converter), symTable(symTable), semaCtx(semaCtx), loc(loc),
         eval(eval), dir(dir) {}
 
+  OpWithBodyGenInfo &setGenNested(bool value) {
+    genNested = value;
+    return *this;
+  }
+
   OpWithBodyGenInfo &setOuterCombined(bool value) {
     outerCombined = value;
     return *this;
@@ -528,6 +600,8 @@ struct OpWithBodyGenInfo {
   Fortran::lower::pft::Evaluation &eval;
   /// [in] leaf directive for which to generate the op body.
   llvm::omp::Directive dir;
+  /// [in] whether to generate FIR for nested evaluations
+  bool genNested = true;
   /// [in] is this an outer operation - prevents privatization.
   bool outerCombined = false;
   /// [in] list of clauses to process.
@@ -546,13 +620,9 @@ struct OpWithBodyGenInfo {
 
 /// Create the body (block) for an OpenMP Operation.
 ///
-/// \param [in]   op  - the operation the body belongs to.
-/// \param [in] info  - options controlling code-gen for the construction.
-/// \param [in] queue - work queue with nested constructs.
-/// \param [in] item  - item in the queue to generate body for.
-static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
-                           const ConstructQueue &queue,
-                           ConstructQueue::iterator item) {
+/// \param [in]   op - the operation the body belongs to.
+/// \param [in] info - options controlling code-gen for the construction.
+static void createBodyOfOp(mlir::Operation &op, OpWithBodyGenInfo &info) {
   fir::FirOpBuilder &firOpBuilder = info.converter.getFirOpBuilder();
 
   auto insertMarker = [](fir::FirOpBuilder &builder) {
@@ -608,10 +678,7 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
     }
   }
 
-  if (ConstructQueue::iterator next = std::next(item); next != queue.end()) {
-    genOMPDispatch(info.converter, info.symTable, info.semaCtx, info.eval,
-                   info.loc, queue, next);
-  } else {
+  if (info.genNested) {
     // genFIR(Evaluation&) tries to patch up unterminated blocks, causing
     // a lot of complications for our approach if the terminator generation
     // is delayed past this point. Insert a temporary terminator here, then
@@ -702,12 +769,11 @@ static void genBodyOfTargetDataOp(
     Fortran::lower::AbstractConverter &converter,
     Fortran::lower::SymMap &symTable,
     Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, mlir::omp::TargetDataOp &dataOp,
-    llvm::ArrayRef<mlir::Type> useDeviceTypes,
+    Fortran::lower::pft::Evaluation &eval, bool genNested,
+    mlir::omp::TargetDataOp &dataOp, llvm::ArrayRef<mlir::Type> useDeviceTypes,
     llvm::ArrayRef<mlir::Location> useDeviceLocs,
     llvm::ArrayRef<const Fortran::semantics::Symbol *> useDeviceSymbols,
-    const mlir::Location &currentLocation, const ConstructQueue &queue,
-    ConstructQueue::iterator item) {
+    const mlir::Location &currentLocation) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::Region &region = dataOp.getRegion();
 
@@ -760,13 +826,8 @@ static void genBodyOfTargetDataOp(
 
   // Set the insertion point after the marker.
   firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
-
-  if (ConstructQueue::iterator next = std::next(item); next != queue.end()) {
-    genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
-                   next);
-  } else {
+  if (genNested)
     genNestedEvaluations(converter, eval);
-  }
 }
 
 // This functions creates a block for the body of the targetOp's region. It adds
@@ -775,13 +836,12 @@ static void
 genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
                   Fortran::lower::SymMap &symTable,
                   Fortran::semantics::SemanticsContext &semaCtx,
-                  Fortran::lower::pft::Evaluation &eval,
+                  Fortran::lower::pft::Evaluation &eval, bool genNested,
                   mlir::omp::TargetOp &targetOp,
                   llvm::ArrayRef<const Fortran::semantics::Symbol *> mapSyms,
                   llvm::ArrayRef<mlir::Location> mapSymLocs,
                   llvm::ArrayRef<mlir::Type> mapSymTypes,
-                  const mlir::Location &currentLocation,
-                  const ConstructQueue &queue, ConstructQueue::iterator item) {
+                  const mlir::Location &currentLocation) {
   assert(mapSymTypes.size() == mapSymLocs.size());
 
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
@@ -923,22 +983,15 @@ genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
 
   // Create the insertion point after the marker.
   firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
-
-  if (ConstructQueue::iterator next = std::next(item); next != queue.end()) {
-    genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
-                   next);
-  } else {
+  if (genNested)
     genNestedEvaluations(converter, eval);
-  }
 }
 
 template <typename OpTy, typename... Args>
-static OpTy genOpWithBody(const OpWithBodyGenInfo &info,
-                          const ConstructQueue &queue,
-                          ConstructQueue::iterator item, Args &&...args) {
+static OpTy genOpWithBody(OpWithBodyGenInfo &info, Args &&...args) {
   auto op = info.converter.getFirOpBuilder().create<OpTy>(
       info.loc, std::forward<Args>(args)...);
-  createBodyOfOp(*op, info, queue, item);
+  createBodyOfOp(*op, info);
   return op;
 }
 
@@ -1223,8 +1276,7 @@ static mlir::omp::BarrierOp
 genBarrierOp(Fortran::lower::AbstractConverter &converter,
              Fortran::lower::SymMap &symTable,
              Fortran::semantics::SemanticsContext &semaCtx,
-             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-             const ConstructQueue &queue, ConstructQueue::iterator item) {
+             Fortran::lower::pft::Evaluation &eval, mlir::Location loc) {
   return converter.getFirOpBuilder().create<mlir::omp::BarrierOp>(loc);
 }
 
@@ -1232,9 +1284,8 @@ static mlir::omp::CriticalOp
 genCriticalOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses, const ConstructQueue &queue,
-              ConstructQueue::iterator item,
+              Fortran::lower::pft::Evaluation &eval, bool genNested,
+              mlir::Location loc, const List<Clause> &clauses,
               const std::optional<Fortran::parser::Name> &name) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::FlatSymbolRefAttr nameAttr;
@@ -1257,17 +1308,17 @@ genCriticalOp(Fortran::lower::AbstractConverter &converter,
 
   return genOpWithBody<mlir::omp::CriticalOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                        llvm::omp::Directive::OMPD_critical),
-      queue, item, nameAttr);
+                        llvm::omp::Directive::OMPD_critical)
+          .setGenNested(genNested),
+      nameAttr);
 }
 
 static mlir::omp::DistributeOp
 genDistributeOp(Fortran::lower::AbstractConverter &converter,
                 Fortran::lower::SymMap &symTable,
                 Fortran::semantics::SemanticsContext &semaCtx,
-                Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-                const List<Clause> &clauses, const ConstructQueue &queue,
-                ConstructQueue::iterator item) {
+                Fortran::lower::pft::Evaluation &eval, bool genNested,
+                mlir::Location loc, const List<Clause> &clauses) {
   TODO(loc, "Distribute construct");
   return nullptr;
 }
@@ -1277,8 +1328,7 @@ genFlushOp(Fortran::lower::AbstractConverter &converter,
            Fortran::lower::SymMap &symTable,
            Fortran::semantics::SemanticsContext &semaCtx,
            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-           const ObjectList &objects, const List<Clause> &clauses,
-           const ConstructQueue &queue, ConstructQueue::iterator item) {
+           const ObjectList &objects, const List<Clause> &clauses) {
   llvm::SmallVector<mlir::Value> operandRange;
   genFlushClauses(converter, semaCtx, objects, clauses, loc, operandRange);
 
@@ -1290,13 +1340,12 @@ static mlir::omp::MasterOp
 genMasterOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const List<Clause> &clauses, const ConstructQueue &queue,
-            ConstructQueue::iterator item) {
+            Fortran::lower::pft::Evaluation &eval, bool genNested,
+            mlir::Location loc) {
   return genOpWithBody<mlir::omp::MasterOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                        llvm::omp::Directive::OMPD_master),
-      queue, item);
+                        llvm::omp::Directive::OMPD_master)
+          .setGenNested(genNested));
 }
 
 static mlir::omp::OrderedOp
@@ -1304,8 +1353,7 @@ genOrderedOp(Fortran::lower::AbstractConverter &converter,
              Fortran::lower::SymMap &symTable,
              Fortran::semantics::SemanticsContext &semaCtx,
              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-             const List<Clause> &clauses, const ConstructQueue &queue,
-             ConstructQueue::iterator item) {
+             const List<Clause> &clauses) {
   TODO(loc, "OMPD_ordered");
   return nullptr;
 }
@@ -1314,25 +1362,25 @@ static mlir::omp::OrderedRegionOp
 genOrderedRegionOp(Fortran::lower::AbstractConverter &converter,
                    Fortran::lower::SymMap &symTable,
                    Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-                   const List<Clause> &clauses, const ConstructQueue &queue,
-                   ConstructQueue::iterator item) {
+                   Fortran::lower::pft::Evaluation &eval, bool genNested,
+                   mlir::Location loc, const List<Clause> &clauses) {
   mlir::omp::OrderedRegionClauseOps clauseOps;
   genOrderedRegionClauses(converter, semaCtx, clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::OrderedRegionOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                        llvm::omp::Directive::OMPD_ordered),
-      queue, item, clauseOps);
+                        llvm::omp::Directive::OMPD_ordered)
+          .setGenNested(genNested),
+      clauseOps);
 }
 
 static mlir::omp::ParallelOp
 genParallelOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses, const ConstructQueue &queue,
-              ConstructQueue::iterator item, bool outerCombined = false) {
+              Fortran::lower::pft::Evaluation &eval, bool genNested,
+              mlir::Location loc, const List<Clause> &clauses,
+              bool outerCombined = false) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::ParallelClauseOps clauseOps;
@@ -1351,14 +1399,14 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
   OpWithBodyGenInfo genInfo =
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_parallel)
+          .setGenNested(genNested)
           .setOuterCombined(outerCombined)
           .setClauses(&clauses)
           .setReductions(&reductionSyms, &reductionTypes)
           .setGenRegionEntryCb(reductionCallback);
 
   if (!enableDelayedPrivatization)
-    return genOpWithBody<mlir::omp::ParallelOp>(genInfo, queue, item,
-                                                clauseOps);
+    return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
 
   bool privatize = !outerCombined;
   DataSharingProcessor dsp(converter, semaCtx, clauses, eval,
@@ -1406,23 +1454,19 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
   };
 
   genInfo.setGenRegionEntryCb(genRegionEntryCB).setDataSharingProcessor(&dsp);
-  return genOpWithBody<mlir::omp::ParallelOp>(genInfo, queue, item, clauseOps);
+  return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
 }
 
 static mlir::omp::SectionOp
 genSectionOp(Fortran::lower::AbstractConverter &converter,
              Fortran::lower::SymMap &symTable,
              Fortran::semantics::SemanticsContext &semaCtx,
-             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-             const List<Clause> &clauses, const ConstructQueue &queue,
-             ConstructQueue::iterator item) {
-  // Currently only private/firstprivate clause is handled, and
-  // all privatization is done within `omp.section` operations.
+             Fortran::lower::pft::Evaluation &eval, bool genNested,
+             mlir::Location loc) {
   return genOpWithBody<mlir::omp::SectionOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_section)
-          .setClauses(&clauses),
-      queue, item);
+          .setGenNested(genNested));
 }
 
 static mlir::omp::SectionsOp
@@ -1430,77 +1474,12 @@ genSectionsOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses, const ConstructQueue &queue,
-              ConstructQueue::iterator item) {
-  mlir::omp::SectionsClauseOps clauseOps;
-  genSectionsClauses(converter, semaCtx, clauses, loc, clauseOps);
-
-  auto &builder = converter.getFirOpBuilder();
-
-  // Insert privatizations before SECTIONS
-  symTable.pushScope();
-  DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
-  dsp.processStep1();
-
-  List<Clause> nonDsaClauses;
-  List<const clause::Lastprivate *> lastprivates;
-
-  for (const Clause &clause : clauses) {
-    if (clause.id == llvm::omp::Clause::OMPC_lastprivate) {
-      lastprivates.push_back(&std::get<clause::Lastprivate>(clause.u));
-    } else {
-      switch (clause.id) {
-      case llvm::omp::Clause::OMPC_firstprivate:
-      case llvm::omp::Clause::OMPC_private:
-      case llvm::omp::Clause::OMPC_shared:
-        break;
-      default:
-        nonDsaClauses.push_back(clause);
-      }
-    }
-  }
-
-  // SECTIONS construct.
-  mlir::omp::SectionsOp sectionsOp = genOpWithBody<mlir::omp::SectionsOp>(
+              const mlir::omp::SectionsClauseOps &clauseOps) {
+  return genOpWithBody<mlir::omp::SectionsOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_sections)
-          .setClauses(&nonDsaClauses),
-      queue, item, clauseOps);
-
-  if (!lastprivates.empty()) {
-    mlir::Region &sectionsBody = sectionsOp.getRegion();
-    assert(sectionsBody.hasOneBlock());
-    mlir::Block &body = sectionsBody.front();
-
-    auto lastSectionOp = llvm::find_if(
-        llvm::reverse(body.getOperations()), [](const mlir::Operation &op) {
-          return llvm::isa<mlir::omp::SectionOp>(op);
-        });
-    assert(lastSectionOp != body.rend());
-
-    for (const clause::Lastprivate *lastp : lastprivates) {
-      builder.setInsertionPoint(
-          lastSectionOp->getRegion(0).back().getTerminator());
-      mlir::OpBuilder::InsertPoint insp = builder.saveInsertionPoint();
-      const auto &objList = std::get<ObjectList>(lastp->t);
-      for (const Object &object : objList) {
-        Fortran::semantics::Symbol *sym = object.id();
-        converter.copyHostAssociateVar(*sym, &insp);
-      }
-    }
-  }
-
-  // Perform DataSharingProcessor's step2 out of SECTIONS
-  builder.setInsertionPointAfter(sectionsOp.getOperation());
-  dsp.processStep2(sectionsOp, false);
-  // Emit implicit barrier to synchronize threads and avoid data
-  // races on post-update of lastprivate variables when `nowait`
-  // clause is present.
-  if (clauseOps.nowaitAttr && !lastprivates.empty())
-    builder.create<mlir::omp::BarrierOp>(loc);
-
-  symTable.popScope();
-  return sectionsOp;
+          .setGenNested(false),
+      clauseOps);
 }
 
 static mlir::omp::SimdOp
@@ -1508,8 +1487,7 @@ genSimdOp(Fortran::lower::AbstractConverter &converter,
           Fortran::lower::SymMap &symTable,
           Fortran::semantics::SemanticsContext &semaCtx,
           Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-          const List<Clause> &clauses, const ConstructQueue &queue,
-          ConstructQueue::iterator item) {
+          const List<Clause> &clauses) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
   dsp.processStep1();
@@ -1544,8 +1522,7 @@ genSimdOp(Fortran::lower::AbstractConverter &converter,
                                    *nestedEval, llvm::omp::Directive::OMPD_simd)
                      .setClauses(&clauses)
                      .setDataSharingProcessor(&dsp)
-                     .setGenRegionEntryCb(ivCallback),
-                 queue, item);
+                     .setGenRegionEntryCb(ivCallback));
 
   return simdOp;
 }
@@ -1554,26 +1531,26 @@ static mlir::omp::SingleOp
 genSingleOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const List<Clause> &clauses, const ConstructQueue &queue,
-            ConstructQueue::iterator item) {
+            Fortran::lower::pft::Evaluation &eval, bool genNested,
+            mlir::Location loc, const List<Clause> &clauses) {
   mlir::omp::SingleClauseOps clauseOps;
   genSingleClauses(converter, semaCtx, clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::SingleOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_single)
+          .setGenNested(genNested)
           .setClauses(&clauses),
-      queue, item, clauseOps);
+      clauseOps);
 }
 
 static mlir::omp::TargetOp
 genTargetOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const List<Clause> &clauses, const ConstructQueue &queue,
-            ConstructQueue::iterator item, bool outerCombined = false) {
+            Fortran::lower::pft::Evaluation &eval, bool genNested,
+            mlir::Location loc, const List<Clause> &clauses,
+            bool outerCombined = false) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
 
@@ -1680,8 +1657,8 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
   Fortran::lower::pft::visitAllSymbols(eval, captureImplicitMap);
 
   auto targetOp = firOpBuilder.create<mlir::omp::TargetOp>(loc, clauseOps);
-  genBodyOfTargetOp(converter, symTable, semaCtx, eval, targetOp, mapSyms,
-                    mapLocs, mapTypes, loc, queue, item);
+  genBodyOfTargetOp(converter, symTable, semaCtx, eval, genNested, targetOp,
+                    mapSyms, mapLocs, mapTypes, loc);
   return targetOp;
 }
 
@@ -1689,9 +1666,8 @@ static mlir::omp::TargetDataOp
 genTargetDataOp(Fortran::lower::AbstractConverter &converter,
                 Fortran::lower::SymMap &symTable,
                 Fortran::semantics::SemanticsContext &semaCtx,
-                Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-                const List<Clause> &clauses, const ConstructQueue &queue,
-                ConstructQueue::iterator item) {
+                Fortran::lower::pft::Evaluation &eval, bool genNested,
+                mlir::Location loc, const List<Clause> &clauses) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TargetDataClauseOps clauseOps;
   llvm::SmallVector<mlir::Type> useDeviceTypes;
@@ -1703,9 +1679,9 @@ genTargetDataOp(Fortran::lower::AbstractConverter &converter,
   auto targetDataOp =
       converter.getFirOpBuilder().create<mlir::omp::TargetDataOp>(loc,
                                                                   clauseOps);
-  genBodyOfTargetDataOp(converter, symTable, semaCtx, eval, targetDataOp,
-                        useDeviceTypes, useDeviceLocs, useDeviceSyms, loc,
-                        queue, item);
+  genBodyOfTargetDataOp(converter, symTable, semaCtx, eval, genNested,
+                        targetDataOp, useDeviceTypes, useDeviceLocs,
+                        useDeviceSyms, loc);
   return targetDataOp;
 }
 
@@ -1714,9 +1690,8 @@ static OpTy
 genTargetEnterExitUpdateDataOp(Fortran::lower::AbstractConverter &converter,
                                Fortran::lower::SymMap &symTable,
                                Fortran::semantics::SemanticsContext &semaCtx,
-                               mlir::Location loc, const List<Clause> &clauses,
-                               const ConstructQueue &queue,
-                               ConstructQueue::iterator item) {
+                               mlir::Location loc,
+                               const List<Clause> &clauses) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
 
@@ -1743,9 +1718,8 @@ static mlir::omp::TaskOp
 genTaskOp(Fortran::lower::AbstractConverter &converter,
           Fortran::lower::SymMap &symTable,
           Fortran::semantics::SemanticsContext &semaCtx,
-          Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-          const List<Clause> &clauses, const ConstructQueue &queue,
-          ConstructQueue::iterator item) {
+          Fortran::lower::pft::Evaluation &eval, bool genNested,
+          mlir::Location loc, const List<Clause> &clauses) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TaskClauseOps clauseOps;
   genTaskClauses(converter, semaCtx, stmtCtx, clauses, loc, clauseOps);
@@ -1753,25 +1727,26 @@ genTaskOp(Fortran::lower::AbstractConverter &converter,
   return genOpWithBody<mlir::omp::TaskOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_task)
+          .setGenNested(genNested)
           .setClauses(&clauses),
-      queue, item, clauseOps);
+      clauseOps);
 }
 
 static mlir::omp::TaskgroupOp
 genTaskgroupOp(Fortran::lower::AbstractConverter &converter,
                Fortran::lower::SymMap &symTable,
                Fortran::semantics::SemanticsContext &semaCtx,
-               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-               const List<Clause> &clauses, const ConstructQueue &queue,
-               ConstructQueue::iterator item) {
+               Fortran::lower::pft::Evaluation &eval, bool genNested,
+               mlir::Location loc, const List<Clause> &clauses) {
   mlir::omp::TaskgroupClauseOps clauseOps;
   genTaskgroupClauses(converter, semaCtx, clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::TaskgroupOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_taskgroup)
+          .setGenNested(genNested)
           .setClauses(&clauses),
-      queue, item, clauseOps);
+      clauseOps);
 }
 
 static mlir::omp::TaskloopOp
@@ -1779,8 +1754,7 @@ genTaskloopOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses, const ConstructQueue &queue,
-              ConstructQueue::iterator item) {
+              const List<Clause> &clauses) {
   TODO(loc, "Taskloop construct");
 }
 
@@ -1789,8 +1763,7 @@ genTaskwaitOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses, const ConstructQueue &queue,
-              ConstructQueue::iterator item) {
+              const List<Clause> &clauses) {
   mlir::omp::TaskwaitClauseOps clauseOps;
   genTaskwaitClauses(converter, semaCtx, clauses, loc, clauseOps);
   return converter.getFirOpBuilder().create<mlir::omp::TaskwaitOp>(loc,
@@ -1801,8 +1774,7 @@ static mlir::omp::TaskyieldOp
 genTaskyieldOp(Fortran::lower::AbstractConverter &converter,
                Fortran::lower::SymMap &symTable,
                Fortran::semantics::SemanticsContext &semaCtx,
-               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-               const ConstructQueue &queue, ConstructQueue::iterator item) {
+               Fortran::lower::pft::Evaluation &eval, mlir::Location loc) {
   return converter.getFirOpBuilder().create<mlir::omp::TaskyieldOp>(loc);
 }
 
@@ -1810,9 +1782,9 @@ static mlir::omp::TeamsOp
 genTeamsOp(Fortran::lower::AbstractConverter &converter,
            Fortran::lower::SymMap &symTable,
            Fortran::semantics::SemanticsContext &semaCtx,
-           Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-           const List<Clause> &clauses, const ConstructQueue &queue,
-           ConstructQueue::iterator item, bool outerCombined = false) {
+           Fortran::lower::pft::Evaluation &eval, bool genNested,
+           mlir::Location loc, const List<Clause> &clauses,
+           bool outerCombined = false) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TeamsClauseOps clauseOps;
   genTeamsClauses(converter, semaCtx, stmtCtx, clauses, loc, clauseOps);
@@ -1820,9 +1792,10 @@ genTeamsOp(Fortran::lower::AbstractConverter &converter,
   return genOpWithBody<mlir::omp::TeamsOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_teams)
+          .setGenNested(genNested)
           .setOuterCombined(outerCombined)
           .setClauses(&clauses),
-      queue, item, clauseOps);
+      clauseOps);
 }
 
 static mlir::omp::WsloopOp
@@ -1830,8 +1803,7 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const List<Clause> &clauses, const ConstructQueue &queue,
-            ConstructQueue::iterator item) {
+            const List<Clause> &clauses) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
   dsp.processStep1();
@@ -1872,8 +1844,7 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
                      .setClauses(&clauses)
                      .setDataSharingProcessor(&dsp)
                      .setReductions(&reductionSyms, &reductionTypes)
-                     .setGenRegionEntryCb(ivCallback),
-                 queue, item);
+                     .setGenRegionEntryCb(ivCallback));
   return wsloopOp;
 }
 
@@ -1881,13 +1852,13 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
 // Code generation functions for composite constructs
 //===----------------------------------------------------------------------===//
 
-static void genCompositeDistributeParallelDo(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::lower::SymMap &symTable,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-    const List<Clause> &clauses, const ConstructQueue &queue,
-    ConstructQueue::iterator item) {
+static void
+genCompositeDistributeParallelDo(Fortran::lower::AbstractConverter &converter,
+                                 Fortran::lower::SymMap &symTable,
+                                 Fortran::semantics::SemanticsContext &semaCtx,
+                                 Fortran::lower::pft::Evaluation &eval,
+                                 const List<Clause> &clauses,
+                                 mlir::Location loc) {
   TODO(loc, "Composite DISTRIBUTE PARALLEL DO");
 }
 
@@ -1895,9 +1866,8 @@ static void genCompositeDistributeParallelDoSimd(
     Fortran::lower::AbstractConverter &converter,
     Fortran::lower::SymMap &symTable,
     Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-    const List<Clause> &clauses, const ConstructQueue &queue,
-    ConstructQueue::iterator item) {
+    Fortran::lower::pft::Evaluation &eval, const List<Clause> &clauses,
+    mlir::Location loc) {
   TODO(loc, "Composite DISTRIBUTE PARALLEL DO SIMD");
 }
 
@@ -1906,9 +1876,7 @@ genCompositeDistributeSimd(Fortran::lower::AbstractConverter &converter,
                            Fortran::lower::SymMap &symTable,
                            Fortran::semantics::SemanticsContext &semaCtx,
                            Fortran::lower::pft::Evaluation &eval,
-                           mlir::Location loc, const List<Clause> &clauses,
-                           const ConstructQueue &queue,
-                           ConstructQueue::iterator item) {
+                           const List<Clause> &clauses, mlir::Location loc) {
   TODO(loc, "Composite DISTRIBUTE SIMD");
 }
 
@@ -1916,9 +1884,8 @@ static void genCompositeDoSimd(Fortran::lower::AbstractConverter &converter,
                                Fortran::lower::SymMap &symTable,
                                Fortran::semantics::SemanticsContext &semaCtx,
                                Fortran::lower::pft::Evaluation &eval,
-                               mlir::Location loc, const List<Clause> &clauses,
-                               const ConstructQueue &queue,
-                               ConstructQueue::iterator item) {
+                               const List<Clause> &clauses,
+                               mlir::Location loc) {
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processTODO<clause::Aligned, clause::Allocate, clause::Linear,
                  clause::Order, clause::Safelen, clause::Simdlen>(
@@ -1931,7 +1898,7 @@ static void genCompositeDoSimd(Fortran::lower::AbstractConverter &converter,
   // When support for vectorization is enabled, then we need to add handling of
   // if clause. Currently if clause can be skipped because we always assume
   // SIMD length = 1.
-  genWsloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+  genWsloopOp(converter, symTable, semaCtx, eval, loc, clauses);
 }
 
 static void
@@ -1939,128 +1906,10 @@ genCompositeTaskloopSimd(Fortran::lower::AbstractConverter &converter,
                          Fortran::lower::SymMap &symTable,
                          Fortran::semantics::SemanticsContext &semaCtx,
                          Fortran::lower::pft::Evaluation &eval,
-                         mlir::Location loc, const List<Clause> &clauses,
-                         const ConstructQueue &queue,
-                         ConstructQueue::iterator item) {
+                         const List<Clause> &clauses, mlir::Location loc) {
   TODO(loc, "Composite TASKLOOP SIMD");
 }
 
-//===----------------------------------------------------------------------===//
-// Dispatch
-//===----------------------------------------------------------------------===//
-
-static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
-                           Fortran::lower::SymMap &symTable,
-                           Fortran::semantics::SemanticsContext &semaCtx,
-                           Fortran::lower::pft::Evaluation &eval,
-                           mlir::Location loc, const ConstructQueue &queue,
-                           ConstructQueue::iterator item) {
-  assert(item != queue.end());
-  const List<Clause> &clauses = item->clauses;
-
-  switch (llvm::omp::Directive dir = item->id) {
-  case llvm::omp::Directive::OMPD_distribute:
-    genDistributeOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                    item);
-    break;
-  case llvm::omp::Directive::OMPD_do:
-    genWsloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
-    break;
-  case llvm::omp::Directive::OMPD_loop:
-  case llvm::omp::Directive::OMPD_masked:
-  case llvm::omp::Directive::OMPD_tile:
-  case llvm::omp::Directive::OMPD_unroll:
-    TODO(loc, "Unhandled loop directive (" +
-                  llvm::omp::getOpenMPDirectiveName(dir) + ")");
-    break;
-  case llvm::omp::Directive::OMPD_master:
-    genMasterOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
-    break;
-  case llvm::omp::Directive::OMPD_ordered:
-    genOrderedRegionOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                       item);
-    break;
-  case llvm::omp::Directive::OMPD_parallel:
-    genParallelOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item,
-                  /*outerCombined=*/false);
-    break;
-  case llvm::omp::Directive::OMPD_sections:
-    genSectionsOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                  item);
-    break;
-  case llvm::omp::Directive::OMPD_simd:
-    genSimdOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
-    break;
-  case llvm::omp::Directive::OMPD_single:
-    genSingleOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
-    break;
-  case llvm::omp::Directive::OMPD_target:
-    genTargetOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item,
-                /*outerCombined=*/false);
-    break;
-  case llvm::omp::Directive::OMPD_target_data:
-    genTargetDataOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                    item);
-    break;
-  case llvm::omp::Directive::OMPD_target_enter_data:
-    genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(
-        converter, symTable, semaCtx, loc, clauses, queue, item);
-    break;
-  case llvm::omp::Directive::OMPD_target_exit_data:
-    genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(
-        converter, symTable, semaCtx, loc, clauses, queue, item);
-    break;
-  case llvm::omp::Directive::OMPD_target_update:
-    genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(
-        converter, symTable, semaCtx, loc, clauses, queue, item);
-    break;
-  case llvm::omp::Directive::OMPD_task:
-    genTaskOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
-    break;
-  case llvm::omp::Directive::OMPD_taskgroup:
-    genTaskgroupOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                   item);
-    break;
-  case llvm::omp::Directive::OMPD_taskloop:
-    genTaskloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                  item);
-    break;
-  case llvm::omp::Directive::OMPD_teams:
-    genTeamsOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
-    break;
-  // case llvm::omp::Directive::OMPD_workdistribute:
-  case llvm::omp::Directive::OMPD_workshare:
-    // FIXME: Workshare is not a commonly used OpenMP construct, an
-    // implementation for this feature will come later. For the codes
-    // that use this construct, add a single construct for now.
-    genSingleOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
-    break;
-  // Composite constructs
-  case llvm::omp::Directive::OMPD_distribute_parallel_do:
-    genCompositeDistributeParallelDo(converter, symTable, semaCtx, eval, loc,
-                                     clauses, queue, item);
-    break;
-  case llvm::omp::Directive::OMPD_distribute_parallel_do_simd:
-    genCompositeDistributeParallelDoSimd(converter, symTable, semaCtx, eval,
-                                         loc, clauses, queue, item);
-    break;
-  case llvm::omp::Directive::OMPD_distribute_simd:
-    genCompositeDistributeSimd(converter, symTable, semaCtx, eval, loc, clauses,
-                               queue, item);
-    break;
-  case llvm::omp::Directive::OMPD_do_simd:
-    genCompositeDoSimd(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                       item);
-    break;
-  case llvm::omp::Directive::OMPD_taskloop_simd:
-    genCompositeTaskloopSimd(converter, symTable, semaCtx, eval, loc, clauses,
-                             queue, item);
-    break;
-  default:
-    break;
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // OpenMPDeclarativeConstruct visitors
 //===----------------------------------------------------------------------===//
@@ -2171,47 +2020,36 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
       semaCtx);
   mlir::Location currentLocation = converter.genLocation(directive.source);
 
-  ConstructQueue queue{
-      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
-                          eval, directive.source, directive.v, clauses)};
-
   switch (directive.v) {
   default:
     break;
   case llvm::omp::Directive::OMPD_barrier:
-    genBarrierOp(converter, symTable, semaCtx, eval, currentLocation, queue,
-                 queue.begin());
+    genBarrierOp(converter, symTable, semaCtx, eval, currentLocation);
     break;
   case llvm::omp::Directive::OMPD_taskwait:
-    genTaskwaitOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
-                  queue, queue.begin());
+    genTaskwaitOp(converter, symTable, semaCtx, eval, currentLocation, clauses);
     break;
   case llvm::omp::Directive::OMPD_taskyield:
-    genTaskyieldOp(converter, symTable, semaCtx, eval, currentLocation, queue,
-                   queue.begin());
+    genTaskyieldOp(converter, symTable, semaCtx, eval, currentLocation);
     break;
   case llvm::omp::Directive::OMPD_target_data:
-    genTargetDataOp(converter, symTable, semaCtx, eval, currentLocation,
-                    clauses, queue, queue.begin());
+    genTargetDataOp(converter, symTable, semaCtx, eval, /*genNested=*/true,
+                    currentLocation, clauses);
     break;
   case llvm::omp::Directive::OMPD_target_enter_data:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(
-        converter, symTable, semaCtx, currentLocation, clauses, queue,
-        queue.begin());
+        converter, symTable, semaCtx, currentLocation, clauses);
     break;
   case llvm::omp::Directive::OMPD_target_exit_data:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(
-        converter, symTable, semaCtx, currentLocation, clauses, queue,
-        queue.begin());
+        converter, symTable, semaCtx, currentLocation, clauses);
     break;
   case llvm::omp::Directive::OMPD_target_update:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(
-        converter, symTable, semaCtx, currentLocation, clauses, queue,
-        queue.begin());
+        converter, symTable, semaCtx, currentLocation, clauses);
     break;
   case llvm::omp::Directive::OMPD_ordered:
-    genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
-                 queue, queue.begin());
+    genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, clauses);
     break;
   }
 }
@@ -2235,12 +2073,8 @@ genOMP(Fortran::lower::AbstractConverter &converter,
                             [&](auto &&s) { return makeClause(s.v, semaCtx); })
                  : List<Clause>{};
   mlir::Location currentLocation = converter.genLocation(verbatim.source);
-
-  ConstructQueue queue{buildConstructQueue(
-      converter.getFirOpBuilder().getModule(), semaCtx, eval, verbatim.source,
-      llvm::omp::Directive::OMPD_flush, clauses)};
   genFlushOp(converter, symTable, semaCtx, eval, currentLocation, objects,
-             clauses, queue, queue.begin());
+             clauses);
 }
 
 static void
@@ -2383,15 +2217,75 @@ genOMP(Fortran::lower::AbstractConverter &converter,
     }
   }
 
-  llvm::omp::Directive directive =
-      std::get<parser::OmpBlockDirective>(beginBlockDirective.t).v;
-  const parser::CharBlock &source =
-      std::get<parser::OmpBlockDirective>(beginBlockDirective.t).source;
-  ConstructQueue queue{
-      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
-                          eval, source, directive, clauses)};
-  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
-                 queue.begin());
+  std::optional<llvm::omp::Directive> nextDir = origDirective;
+  bool outermostLeafConstruct = true;
+  while (nextDir) {
+    llvm::omp::Directive leafDir;
+    std::tie(leafDir, nextDir) = splitCombinedDirective(*nextDir);
+    const bool genNested = !nextDir;
+    const bool outerCombined = outermostLeafConstruct && nextDir.has_value();
+    switch (leafDir) {
+    case llvm::omp::Directive::OMPD_master:
+      // 2.16 MASTER construct.
+      genMasterOp(converter, symTable, semaCtx, eval, genNested,
+                  currentLocation);
+      break;
+    case llvm::omp::Directive::OMPD_ordered:
+      // 2.17.9 ORDERED construct.
+      genOrderedRegionOp(converter, symTable, semaCtx, eval, genNested,
+                         currentLocation, clauses);
+      break;
+    case llvm::omp::Directive::OMPD_parallel:
+      // 2.6 PARALLEL construct.
+      genParallelOp(converter, symTable, semaCtx, eval, genNested,
+                    currentLocation, clauses, outerCombined);
+      break;
+    case llvm::omp::Directive::OMPD_single:
+      // 2.8.2 SINGLE construct.
+      genSingleOp(converter, symTable, semaCtx, eval, genNested,
+                  currentLocation, clauses);
+      break;
+    case llvm::omp::Directive::OMPD_target:
+      // 2.12.5 TARGET construct.
+      genTargetOp(converter, symTable, semaCtx, eval, genNested,
+                  currentLocation, clauses, outerCombined);
+      break;
+    case llvm::omp::Directive::OMPD_target_data:
+      // 2.12.2 TARGET DATA construct.
+      genTargetDataOp(converter, symTable, semaCtx, eval, genNested,
+                      currentLocation, clauses);
+      break;
+    case llvm::omp::Directive::OMPD_task:
+      // 2.10.1 TASK construct.
+      genTaskOp(converter, symTable, semaCtx, eval, genNested, currentLocation,
+                clauses);
+      break;
+    case llvm::omp::Directive::OMPD_taskgroup:
+      // 2.17.6 TASKGROUP construct.
+      genTaskgroupOp(converter, symTable, semaCtx, eval, genNested,
+                     currentLocation, clauses);
+      break;
+    case llvm::omp::Directive::OMPD_teams:
+      // 2.7 TEAMS construct.
+      // FIXME Pass the outerCombined argument or rename it to better describe
+      // what it represents if it must always be `false` in this context.
+      genTeamsOp(converter, symTable, semaCtx, eval, genNested, currentLocation,
+                 clauses);
+      break;
+    case llvm::omp::Directive::OMPD_workshare:
+      // 2.8.3 WORKSHARE construct.
+      // FIXME: Workshare is not a commonly used OpenMP construct, an
+      // implementation for this feature will come later. For the codes
+      // that use this construct, add a single construct for now.
+      genSingleOp(converter, symTable, semaCtx, eval, genNested,
+                  currentLocation, clauses);
+      break;
+    default:
+      llvm_unreachable("Unexpected block construct");
+      break;
+    }
+    outermostLeafConstruct = false;
+  }
 }
 
 static void
@@ -2404,15 +2298,10 @@ genOMP(Fortran::lower::AbstractConverter &converter,
       std::get<Fortran::parser::OmpCriticalDirective>(criticalConstruct.t);
   List<Clause> clauses =
       makeClauses(std::get<Fortran::parser::OmpClauseList>(cd.t), semaCtx);
-
-  ConstructQueue queue{buildConstructQueue(
-      converter.getFirOpBuilder().getModule(), semaCtx, eval, cd.source,
-      llvm::omp::Directive::OMPD_critical, clauses)};
-
   const auto &name = std::get<std::optional<Fortran::parser::Name>>(cd.t);
   mlir::Location currentLocation = converter.getCurrentLocation();
-  genCriticalOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
-                queue, queue.begin(), name);
+  genCriticalOp(converter, symTable, semaCtx, eval, /*genNested=*/true,
+                currentLocation, clauses, name);
 }
 
 static void
@@ -2433,6 +2322,14 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
       std::get<Fortran::parser::OmpBeginLoopDirective>(loopConstruct.t);
   List<Clause> clauses = makeClauses(
       std::get<Fortran::parser::OmpClauseList>(beginLoopDirective.t), semaCtx);
+  mlir::Location currentLocation =
+      converter.genLocation(beginLoopDirective.source);
+  const auto origDirective =
+      std::get<Fortran::parser::OmpLoopDirective>(beginLoopDirective.t).v;
+
+  assert(llvm::omp::loopConstructSet.test(origDirective) &&
+         "Expected loop construct");
+
   if (auto &endLoopDirective =
           std::get<std::optional<Fortran::parser::OmpEndLoopDirective>>(
               loopConstruct.t)) {
@@ -2441,18 +2338,101 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
         semaCtx));
   }
 
-  mlir::Location currentLocation =
-      converter.genLocation(beginLoopDirective.source);
-
-  llvm::omp::Directive directive =
-      std::get<parser::OmpLoopDirective>(beginLoopDirective.t).v;
-  const parser::CharBlock &source =
-      std::get<parser::OmpLoopDirective>(beginLoopDirective.t).source;
-  ConstructQueue queue{
-      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
-                          eval, source, directive, clauses)};
-  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
-                 queue.begin());
+  std::optional<llvm::omp::Directive> nextDir = origDirective;
+  while (nextDir) {
+    llvm::omp::Directive leafDir;
+    std::tie(leafDir, nextDir) = splitCombinedDirective(*nextDir);
+    if (llvm::omp::compositeConstructSet.test(leafDir)) {
+      assert(!nextDir && "Composite construct cannot be split");
+      switch (leafDir) {
+      case llvm::omp::Directive::OMPD_distribute_parallel_do:
+        // 2.9.4.3 DISTRIBUTE PARALLEL Worksharing-Loop construct.
+        genCompositeDistributeParallelDo(converter, symTable, semaCtx, eval,
+                                         clauses, currentLocation);
+        break;
+      case llvm::omp::Directive::OMPD_distribute_parallel_do_simd:
+        // 2.9.4.4 DISTRIBUTE PARALLEL Worksharing-Loop SIMD construct.
+        genCompositeDistributeParallelDoSimd(converter, symTable, semaCtx, eval,
+                                             clauses, currentLocation);
+        break;
+      case llvm::omp::Directive::OMPD_distribute_simd:
+        // 2.9.4.2 DISTRIBUTE SIMD construct.
+        genCompositeDistributeSimd(converter, symTable, semaCtx, eval, clauses,
+                                   currentLocation);
+        break;
+      case llvm::omp::Directive::OMPD_do_simd:
+        // 2.9.3.2 Worksharing-Loop SIMD construct.
+        genCompositeDoSimd(converter, symTable, semaCtx, eval, clauses,
+                           currentLocation);
+        break;
+      case llvm::omp::Directive::OMPD_taskloop_simd:
+        // 2.10.3 TASKLOOP SIMD construct.
+        genCompositeTaskloopSimd(converter, symTable, semaCtx, eval, clauses,
+                                 currentLocation);
+        break;
+      default:
+        llvm_unreachable("Unexpected composite construct");
+      }
+    } else {
+      const bool genNested = !nextDir;
+      switch (leafDir) {
+      case llvm::omp::Directive::OMPD_distribute:
+        // 2.9.4.1 DISTRIBUTE construct.
+        genDistributeOp(converter, symTable, semaCtx, eval, genNested,
+                        currentLocation, clauses);
+        break;
+      case llvm::omp::Directive::OMPD_do:
+        // 2.9.2 Worksharing-Loop construct.
+        genWsloopOp(converter, symTable, semaCtx, eval, currentLocation,
+                    clauses);
+        break;
+      case llvm::omp::Directive::OMPD_parallel:
+        // 2.6 PARALLEL construct.
+        // FIXME This is not necessarily always the outer leaf construct of a
+        // combined construct in this constext (e.g. distribute parallel do).
+        // Maybe rename the argument if it represents something else or
+        // initialize it properly.
+        genParallelOp(converter, symTable, semaCtx, eval, genNested,
+                      currentLocation, clauses,
+                      /*outerCombined=*/true);
+        break;
+      case llvm::omp::Directive::OMPD_simd:
+        // 2.9.3.1 SIMD construct.
+        genSimdOp(converter, symTable, semaCtx, eval, currentLocation, clauses);
+        break;
+      case llvm::omp::Directive::OMPD_target:
+        // 2.12.5 TARGET construct.
+        genTargetOp(converter, symTable, semaCtx, eval, genNested,
+                    currentLocation, clauses, /*outerCombined=*/true);
+        break;
+      case llvm::omp::Directive::OMPD_taskloop:
+        // 2.10.2 TASKLOOP construct.
+        genTaskloopOp(converter, symTable, semaCtx, eval, currentLocation,
+                      clauses);
+        break;
+      case llvm::omp::Directive::OMPD_teams:
+        // 2.7 TEAMS construct.
+        // FIXME This is not necessarily always the outer leaf construct of a
+        // combined construct in this constext (e.g. target teams distribute).
+        // Maybe rename the argument if it represents something else or
+        // initialize it properly.
+        genTeamsOp(converter, symTable, semaCtx, eval, genNested,
+                   currentLocation, clauses, /*outerCombined=*/true);
+        break;
+      case llvm::omp::Directive::OMPD_loop:
+      case llvm::omp::Directive::OMPD_masked:
+      case llvm::omp::Directive::OMPD_master:
+      case llvm::omp::Directive::OMPD_tile:
+      case llvm::omp::Directive::OMPD_unroll:
+        TODO(currentLocation, "Unhandled loop directive (" +
+                                  llvm::omp::getOpenMPDirectiveName(leafDir) +
+                                  ")");
+        break;
+      default:
+        llvm_unreachable("Unexpected loop construct");
+      }
+    }
+  }
 }
 
 static void
@@ -2461,12 +2441,8 @@ genOMP(Fortran::lower::AbstractConverter &converter,
        Fortran::semantics::SemanticsContext &semaCtx,
        Fortran::lower::pft::Evaluation &eval,
        const Fortran::parser::OpenMPSectionConstruct &sectionConstruct) {
-  mlir::Location loc = converter.getCurrentLocation();
-  ConstructQueue queue{buildConstructQueue(
-      converter.getFirOpBuilder().getModule(), semaCtx, eval,
-      sectionConstruct.source, llvm::omp::Directive::OMPD_section, {})};
-  genSectionOp(converter, symTable, semaCtx, eval, loc,
-               /*clauses=*/{}, queue, queue.begin());
+  // SECTION constructs are handled as a part of SECTIONS.
+  llvm_unreachable("Unexpected standalone OMP SECTION");
 }
 
 static void
@@ -2485,17 +2461,77 @@ genOMP(Fortran::lower::AbstractConverter &converter,
   clauses.append(makeClauses(
       std::get<Fortran::parser::OmpClauseList>(endSectionsDirective.t),
       semaCtx));
+
+  // Process clauses before optional omp.parallel, so that new variables are
+  // allocated outside of the parallel region
   mlir::Location currentLocation = converter.getCurrentLocation();
+  mlir::omp::SectionsClauseOps clauseOps;
+  genSectionsClauses(converter, semaCtx, clauses, currentLocation, clauseOps);
+
+  // Parallel wrapper of PARALLEL SECTIONS construct
+  llvm::omp::Directive dir =
+      std::get<Fortran::parser::OmpSectionsDirective>(beginSectionsDirective.t)
+          .v;
+  if (dir == llvm::omp::Directive::OMPD_parallel_sections) {
+    genParallelOp(converter, symTable, semaCtx, eval,
+                  /*genNested=*/false, currentLocation, clauses,
+                  /*outerCombined=*/true);
+  }
+
+  // Insert privatizations before SECTIONS
+  symTable.pushScope();
+  DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
+  dsp.processStep1();
+
+  // SECTIONS construct.
+  mlir::omp::SectionsOp sectionsOp = genSectionsOp(
+      converter, symTable, semaCtx, eval, currentLocation, clauseOps);
+
+  // Generate nested SECTION operations recursively.
+  const auto &sectionBlocks =
+      std::get<Fortran::parser::OmpSectionBlocks>(sectionsConstruct.t);
+  auto &firOpBuilder = converter.getFirOpBuilder();
+  auto ip = firOpBuilder.saveInsertionPoint();
+  mlir::omp::SectionOp lastSectionOp;
+  for (const auto &[nblock, neval] :
+       llvm::zip(sectionBlocks.v, eval.getNestedEvaluations())) {
+    symTable.pushScope();
+    lastSectionOp = genSectionOp(converter, symTable, semaCtx, neval,
+                                 /*genNested=*/true, currentLocation);
+    symTable.popScope();
+    firOpBuilder.restoreInsertionPoint(ip);
+  }
+
+  // For `omp.sections`, lastprivatized variables occur in
+  // lexically final `omp.section` operation.
+  bool hasLastPrivate = false;
+  if (lastSectionOp) {
+    for (const Clause &clause : clauses) {
+      if (const auto *lastPrivate =
+              std::get_if<clause::Lastprivate>(&clause.u)) {
+        hasLastPrivate = true;
+        firOpBuilder.setInsertionPoint(
+            lastSectionOp.getRegion().back().getTerminator());
+        mlir::OpBuilder::InsertPoint lastPrivIP =
+            converter.getFirOpBuilder().saveInsertionPoint();
+        const auto &objList = std::get<1>(lastPrivate->t);
+        for (const Object &obj : objList) {
+          Fortran::semantics::Symbol *sym = obj.id();
+          converter.copyHostAssociateVar(*sym, &lastPrivIP);
+        }
+      }
+    }
+  }
 
-  llvm::omp::Directive directive =
-      std::get<parser::OmpSectionsDirective>(beginSectionsDirective.t).v;
-  const parser::CharBlock &source =
-      std::get<parser::OmpSectionsDirective>(beginSectionsDirective.t).source;
-  ConstructQueue queue{
-      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
-                          eval, source, directive, clauses)};
-  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
-                 queue.begin());
+  // Perform DataSharingProcessor's step2 out of SECTIONS
+  firOpBuilder.setInsertionPointAfter(sectionsOp.getOperation());
+  dsp.processStep2(sectionsOp, false);
+  // Emit implicit barrier to synchronize threads and avoid data
+  // races on post-update of lastprivate variables when `nowait`
+  // clause is present.
+  if (clauseOps.nowaitAttr && hasLastPrivate)
+    firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
+  symTable.popScope();
 }
 
 static void genOMP(Fortran::lower::AbstractConverter &converter,
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index cb1d1a5a7f3d..eed63b226133 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -51,6 +51,12 @@ int64_t getCollapseValue(const List<Clause> &clauses) {
   return 1;
 }
 
+uint32_t getOpenMPVersion(mlir::ModuleOp mod) {
+  if (mlir::Attribute verAttr = mod->getAttr("omp.version"))
+    return llvm::cast<mlir::omp::VersionAttr>(verAttr).getVersion();
+  llvm_unreachable("Expecting OpenMP version attribute in module");
+}
+
 void genObjectList(const ObjectList &objects,
                    Fortran::lower::AbstractConverter &converter,
                    llvm::SmallVectorImpl<mlir::Value> &operands) {
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 345ce55620ee..8fbb18fa8656 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -93,6 +93,7 @@ void gatherFuncAndVarSyms(
     llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause);
 
 int64_t getCollapseValue(const List<Clause> &clauses);
+uint32_t getOpenMPVersion(mlir::ModuleOp mod);
 
 Fortran::semantics::Symbol *
 getOmpObjectSymbol(const Fortran::parser::OmpObject &ompObject);
diff --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90
index 7cc2bc2e0c71..62ba67e5962f 100644
--- a/flang/test/Lower/OpenMP/default-clause-byref.f90
+++ b/flang/test/Lower/OpenMP/default-clause-byref.f90
@@ -161,12 +161,12 @@ subroutine nested_default_clause_tests
 !CHECK: %[[Z:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFnested_default_clause_testsEz"}
 !CHECK: %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z]] {uniq_name = "_QFnested_default_clause_testsEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel   {
-!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"}
-!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
 !CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"}
+!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_default_clause_testsEz"}
 !CHECK: %[[PRIVATE_Z_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Z]] {uniq_name = "_QFnested_default_clause_testsEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_K:.*]] = fir.alloca i32 {bindc_name = "k", pinned, uniq_name = "_QFnested_default_clause_testsEk"}
@@ -221,7 +221,6 @@ subroutine nested_default_clause_tests
     
     
 !CHECK: omp.parallel {
-!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"}
 !CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90
index 843ee6bb7910..a90f0f4ef5f8 100644
--- a/flang/test/Lower/OpenMP/default-clause.f90
+++ b/flang/test/Lower/OpenMP/default-clause.f90
@@ -160,12 +160,12 @@ end program default_clause_lowering
 !CHECK: %[[Z:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFnested_default_clause_test1Ez"}
 !CHECK: %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z]] {uniq_name = "_QFnested_default_clause_test1Ez"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel   {
-!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_test1Ey"}
-!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_test1Ey"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_test1Ex"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_test1Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
 !CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_test1Ey"}
+!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_test1Ey"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_default_clause_test1Ez"}
 !CHECK: %[[PRIVATE_Z_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Z]] {uniq_name = "_QFnested_default_clause_test1Ez"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_K:.*]] = fir.alloca i32 {bindc_name = "k", pinned, uniq_name = "_QFnested_default_clause_test1Ek"}
diff --git a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
index e6ee75c8a5be..b7f11c8c722f 100644
--- a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
@@ -145,10 +145,10 @@ end subroutine
 !CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
-!CHECK-DAG: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2"
-!CHECK-DAG: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
 !CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK-DAG: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2"
+!CHECK-DAG: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.wsloop {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index 07c95497b7a4..daef02bcfc9a 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -178,12 +178,6 @@ template <typename T> using ListT = llvm::SmallVector<T, 0>;
 // provide their own specialization that conforms to the above requirements.
 template <typename IdType, typename ExprType> struct ObjectT;
 
-// By default, object equality is only determined by its identity.
-template <typename I, typename E>
-bool operator==(const ObjectT<I, E> &o1, const ObjectT<I, E> &o2) {
-  return o1.id() == o2.id();
-}
-
 template <typename I, typename E> using ObjectListT = ListT<ObjectT<I, E>>;
 
 using DirectiveName = llvm::omp::Directive;
@@ -270,32 +264,6 @@ struct ReductionIdentifierT {
 
 template <typename T, typename I, typename E> //
 using IteratorT = ListT<IteratorSpecifierT<T, I, E>>;
-
-template <typename T>
-std::enable_if_t<T::EmptyTrait::value, bool> operator==(const T &a,
-                                                        const T &b) {
-  return true;
-}
-template <typename T>
-std::enable_if_t<T::IncompleteTrait::value, bool> operator==(const T &a,
-                                                             const T &b) {
-  return true;
-}
-template <typename T>
-std::enable_if_t<T::WrapperTrait::value, bool> operator==(const T &a,
-                                                          const T &b) {
-  return a.v == b.v;
-}
-template <typename T>
-std::enable_if_t<T::TupleTrait::value, bool> operator==(const T &a,
-                                                        const T &b) {
-  return a.t == b.t;
-}
-template <typename T>
-std::enable_if_t<T::UnionTrait::value, bool> operator==(const T &a,
-                                                        const T &b) {
-  return a.u == b.u;
-}
 } // namespace type
 
 template <typename T> using ListT = type::ListT<T>;
@@ -317,8 +285,6 @@ ListT<ResultTy> makeList(ContainerTy &&container, FunctionTy &&func) {
 }
 
 namespace clause {
-using type::operator==;
-
 // V5.2: [8.3.1] `assumption` clauses
 template <typename T, typename I, typename E> //
 struct AbsentT {
@@ -760,7 +726,7 @@ struct LinearT {
   ENUM(LinearModifier, Ref, Val, Uval);
 
   using TupleTrait = std::true_type;
-  // Step == nullopt means 1.
+  // Step == nullptr means 1.
   std::tuple<OPT(StepSimpleModifier), OPT(StepComplexModifier),
              OPT(LinearModifier), List>
       t;
@@ -1176,11 +1142,9 @@ struct UsesAllocatorsT {
   using MemSpace = E;
   using TraitsArray = ObjectT<I, E>;
   using Allocator = E;
-  struct AllocatorSpec { // Not a spec name
-    using TupleTrait = std::true_type;
-    std::tuple<OPT(MemSpace), OPT(TraitsArray), Allocator> t;
-  };
-  using Allocators = ListT<AllocatorSpec>; // Not a spec name
+  using AllocatorSpec =
+      std::tuple<OPT(MemSpace), OPT(TraitsArray), Allocator>; // Not a spec name
+  using Allocators = ListT<AllocatorSpec>;                    // Not a spec name
   using WrapperTrait = std::true_type;
   Allocators v;
 };
@@ -1268,9 +1232,8 @@ using UnionOfAllClausesT = typename type::Union< //
     UnionClausesT<T, I, E>,                      //
     WrapperClausesT<T, I, E>                     //
     >::type;
-} // namespace clause
 
-using type::operator==;
+} // namespace clause
 
 // The variant wrapper that encapsulates all possible specific clauses.
 // The `Extras` arguments are additional types representing local extensions
@@ -1297,11 +1260,6 @@ struct ClauseT {
   VariantTy u;
 };
 
-template <typename ClauseType> struct DirectiveWithClauses {
-  llvm::omp::Directive id = llvm::omp::Directive::OMPD_unknown;
-  tomp::type::ListT<ClauseType> clauses;
-};
-
 } // namespace tomp
 
 #undef OPT
diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
deleted file mode 100644
index 7a4ed92a1070..000000000000
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
+++ /dev/null
@@ -1,403 +0,0 @@
-//===- ConstructCompositionT.h -- Composing compound constructs -----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// Given a list of leaf construct, each with a set of clauses, generate the
-// compound construct whose leaf constructs are the given list, and whose clause
-// list is the merged lists of individual leaf clauses.
-//
-// *** At the moment it assumes that the individual constructs and their clauses
-// *** are a subset of those created by splitting a valid compound construct.
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_FRONTEND_OPENMP_CONSTRUCTCOMPOSITIONT_H
-#define LLVM_FRONTEND_OPENMP_CONSTRUCTCOMPOSITIONT_H
-
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/BitVector.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Frontend/OpenMP/ClauseT.h"
-#include "llvm/Frontend/OpenMP/OMP.h"
-
-#include <iterator>
-#include <optional>
-#include <tuple>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
-
-namespace tomp {
-template <typename ClauseType> struct ConstructCompositionT {
-  using ClauseTy = ClauseType;
-
-  using TypeTy = typename ClauseTy::TypeTy;
-  using IdTy = typename ClauseTy::IdTy;
-  using ExprTy = typename ClauseTy::ExprTy;
-
-  ConstructCompositionT(uint32_t version,
-                        llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> leafs);
-
-  DirectiveWithClauses<ClauseTy> merged;
-
-private:
-  // Use an ordered container, since we beed to maintain the order in which
-  // clauses are added to it. This is to avoid non-deterministic output.
-  using ClauseSet = ListT<ClauseTy>;
-
-  enum class Presence {
-    All,  // Clause is preesnt on all leaf constructs that allow it.
-    Some, // Clause is present on some, but not on all constructs.
-    None, // Clause is absent on all constructs.
-  };
-
-  template <typename S>
-  ClauseTy makeClause(llvm::omp::Clause clauseId, S &&specific) {
-    return ClauseTy{clauseId, std::move(specific)};
-  }
-
-  llvm::omp::Directive
-  makeCompound(llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> parts);
-
-  Presence checkPresence(llvm::omp::Clause clauseId);
-
-  // There are clauses that need special handling:
-  // 1. "if": the "directive-name-modifier" on the merged clause may need
-  // to be set appropriately.
-  // 2. "reduction": implies "privateness" of all objects (incompatible
-  // with "shared"); there are rules for merging modifiers
-  void mergeIf();
-  void mergeReduction();
-  void mergeDSA();
-
-  uint32_t version;
-  llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> leafs;
-
-  // clause id -> set of leaf constructs that contain it
-  std::unordered_map<llvm::omp::Clause, llvm::BitVector> clausePresence;
-  // clause id -> set of instances of that clause
-  std::unordered_map<llvm::omp::Clause, ClauseSet> clauseSets;
-};
-
-template <typename C>
-ConstructCompositionT<C>::ConstructCompositionT(
-    uint32_t version, llvm::ArrayRef<DirectiveWithClauses<C>> leafs)
-    : version(version), leafs(leafs) {
-  // Merge the list of constructs with clauses into a compound construct
-  // with a single list of clauses.
-  // The intended use of this function is in splitting compound constructs,
-  // while preserving composite constituent constructs:
-  // Step 1: split compound construct into leaf constructs.
-  // Step 2: identify composite sub-construct, and merge the constituent leafs.
-  //
-  // *** At the moment it assumes that the individual constructs and their
-  // *** clauses are a subset of those created by splitting a valid compound
-  // *** construct.
-  //
-  // 1. Deduplicate clauses
-  //    - exact duplicates: e.g. shared(x) shared(x) -> shared(x)
-  //    - special cases of clauses differing in modifier:
-  //      (a) reduction: inscan + (none|default) = inscan
-  //      (b) reduction: task + (none|default) = task
-  //      (c) combine repeated "if" clauses if possible
-  // 2. Merge DSA clauses: e.g. private(x) private(y) -> private(x, y).
-  // 3. Resolve potential DSA conflicts (typically due to implied clauses).
-
-  if (leafs.empty())
-    return;
-
-  merged.id = makeCompound(leafs);
-
-  // Populate the two maps:
-  for (const auto &[index, leaf] : llvm::enumerate(leafs)) {
-    for (const auto &clause : leaf.clauses) {
-      // Update clausePresence.
-      auto &pset = clausePresence[clause.id];
-      if (pset.size() < leafs.size())
-        pset.resize(leafs.size());
-      pset.set(index);
-      // Update clauseSets.
-      ClauseSet &cset = clauseSets[clause.id];
-      if (!llvm::is_contained(cset, clause))
-        cset.push_back(clause);
-    }
-  }
-
-  mergeIf();
-  mergeReduction();
-  mergeDSA();
-
-  // Fir the rest of the clauses, just copy them.
-  for (auto &[id, clauses] : clauseSets) {
-    // Skip clauses we've already dealt with.
-    switch (id) {
-    case llvm::omp::Clause::OMPC_if:
-    case llvm::omp::Clause::OMPC_reduction:
-    case llvm::omp::Clause::OMPC_shared:
-    case llvm::omp::Clause::OMPC_private:
-    case llvm::omp::Clause::OMPC_firstprivate:
-    case llvm::omp::Clause::OMPC_lastprivate:
-      continue;
-    default:
-      break;
-    }
-    llvm::append_range(merged.clauses, clauses);
-  }
-}
-
-template <typename C>
-llvm::omp::Directive ConstructCompositionT<C>::makeCompound(
-    llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> parts) {
-  llvm::SmallVector<llvm::omp::Directive> dirIds;
-  llvm::transform(parts, std::back_inserter(dirIds),
-                  [](auto &&dwc) { return dwc.id; });
-
-  return llvm::omp::getCompoundConstruct(dirIds);
-}
-
-template <typename C>
-auto ConstructCompositionT<C>::checkPresence(llvm::omp::Clause clauseId)
-    -> Presence {
-  auto found = clausePresence.find(clauseId);
-  if (found == clausePresence.end())
-    return Presence::None;
-
-  bool OnAll = true, OnNone = true;
-  for (const auto &[index, leaf] : llvm::enumerate(leafs)) {
-    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, clauseId, version))
-      continue;
-
-    if (found->second.test(index))
-      OnNone = false;
-    else
-      OnAll = false;
-  }
-
-  if (OnNone)
-    return Presence::None;
-  if (OnAll)
-    return Presence::All;
-  return Presence::Some;
-}
-
-template <typename C> void ConstructCompositionT<C>::mergeIf() {
-  using IfTy = tomp::clause::IfT<TypeTy, IdTy, ExprTy>;
-  // Deal with the "if" clauses. If it's on all leafs that allow it, then it
-  // will apply to the compound construct. Otherwise it will apply to the
-  // single (assumed) leaf construct.
-  // This assumes that the "if" clauses have the same expression.
-  Presence presence = checkPresence(llvm::omp::Clause::OMPC_if);
-  if (presence == Presence::None)
-    return;
-
-  const ClauseTy &some = *clauseSets[llvm::omp::Clause::OMPC_if].begin();
-  const auto &someIf = std::get<IfTy>(some.u);
-
-  if (presence == Presence::All) {
-    // Create "if" without "directive-name-modifier".
-    merged.clauses.emplace_back(
-        makeClause(llvm::omp::Clause::OMPC_if,
-                   IfTy{{/*DirectiveNameModifier=*/std::nullopt,
-                         /*IfExpression=*/std::get<typename IfTy::IfExpression>(
-                             someIf.t)}}));
-  } else {
-    // Find out where it's present and create "if" with the corresponding
-    // "directive-name-modifier".
-    int Idx = clausePresence[llvm::omp::Clause::OMPC_if].find_first();
-    assert(Idx >= 0);
-    merged.clauses.emplace_back(
-        makeClause(llvm::omp::Clause::OMPC_if,
-                   IfTy{{/*DirectiveNameModifier=*/leafs[Idx].id,
-                         /*IfExpression=*/std::get<typename IfTy::IfExpression>(
-                             someIf.t)}}));
-  }
-}
-
-template <typename C> void ConstructCompositionT<C>::mergeReduction() {
-  Presence presence = checkPresence(llvm::omp::Clause::OMPC_reduction);
-  if (presence == Presence::None)
-    return;
-
-  using ReductionTy = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
-  using ModifierTy = typename ReductionTy::ReductionModifier;
-  using IdentifiersTy = typename ReductionTy::ReductionIdentifiers;
-  using ListTy = typename ReductionTy::List;
-  // There are exceptions on which constructs "reduction" may appear
-  // (specifically "parallel", and "teams"). Assume that if "reduction"
-  // is present, it can be applied to the compound construct.
-
-  // What's left is to see if there are any modifiers present. Again,
-  // assume that there are no conflicting modifiers.
-  // There can be, however, multiple reductions on different objects.
-  auto equal = [](const ClauseTy &red1, const ClauseTy &red2) {
-    // Extract actual reductions.
-    const auto r1 = std::get<ReductionTy>(red1.u);
-    const auto r2 = std::get<ReductionTy>(red2.u);
-    // Compare everything except modifiers.
-    if (std::get<IdentifiersTy>(r1.t) != std::get<IdentifiersTy>(r2.t))
-      return false;
-    if (std::get<ListTy>(r1.t) != std::get<ListTy>(r2.t))
-      return false;
-    return true;
-  };
-
-  auto getModifier = [](const ClauseTy &clause) {
-    const ReductionTy &red = std::get<ReductionTy>(clause.u);
-    return std::get<std::optional<ModifierTy>>(red.t);
-  };
-
-  const ClauseSet &reductions = clauseSets[llvm::omp::Clause::OMPC_reduction];
-  std::unordered_set<const ClauseTy *> visited;
-  while (reductions.size() != visited.size()) {
-    typename ClauseSet::const_iterator first;
-
-    // Find first non-visited reduction.
-    for (first = reductions.begin(); first != reductions.end(); ++first) {
-      if (visited.count(&*first))
-        continue;
-      visited.insert(&*first);
-      break;
-    }
-
-    std::optional<ModifierTy> modifier = getModifier(*first);
-
-    // Visit all other reductions that are "equal" (with respect to the
-    // definition above) to "first". Collect modifiers.
-    for (auto iter = std::next(first); iter != reductions.end(); ++iter) {
-      if (!equal(*first, *iter))
-        continue;
-      visited.insert(&*iter);
-      if (!modifier || *modifier == ModifierTy::Default)
-        modifier = getModifier(*iter);
-    }
-
-    const auto &firstRed = std::get<ReductionTy>(first->u);
-    merged.clauses.emplace_back(makeClause(
-        llvm::omp::Clause::OMPC_reduction,
-        ReductionTy{
-            {/*ReductionModifier=*/modifier,
-             /*ReductionIdentifiers=*/std::get<IdentifiersTy>(firstRed.t),
-             /*List=*/std::get<ListTy>(firstRed.t)}}));
-  }
-}
-
-template <typename C> void ConstructCompositionT<C>::mergeDSA() {
-  using ObjectTy = tomp::type::ObjectT<IdTy, ExprTy>;
-
-  // Resolve data-sharing attributes.
-  enum DSA : int {
-    None = 0,
-    Shared = 1 << 0,
-    Private = 1 << 1,
-    FirstPrivate = 1 << 2,
-    LastPrivate = 1 << 3,
-    LastPrivateConditional = 1 << 4,
-  };
-
-  // Use ordered containers to avoid non-deterministic output.
-  llvm::SmallVector<std::pair<ObjectTy, int>> objectDsa;
-
-  auto getDsa = [&](const ObjectTy &object) -> std::pair<ObjectTy, int> & {
-    auto found = llvm::find_if(objectDsa, [&](std::pair<ObjectTy, int> &p) {
-      return p.first.id() == object.id();
-    });
-    if (found != objectDsa.end())
-      return *found;
-    return objectDsa.emplace_back(object, DSA::None);
-  };
-
-  using SharedTy = tomp::clause::SharedT<TypeTy, IdTy, ExprTy>;
-  using PrivateTy = tomp::clause::PrivateT<TypeTy, IdTy, ExprTy>;
-  using FirstprivateTy = tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>;
-  using LastprivateTy = tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>;
-
-  // Visit clauses that affect DSA.
-  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_shared]) {
-    for (auto &object : std::get<SharedTy>(clause.u).v)
-      getDsa(object).second |= DSA::Shared;
-  }
-
-  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_private]) {
-    for (auto &object : std::get<PrivateTy>(clause.u).v)
-      getDsa(object).second |= DSA::Private;
-  }
-
-  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_firstprivate]) {
-    for (auto &object : std::get<FirstprivateTy>(clause.u).v)
-      getDsa(object).second |= DSA::FirstPrivate;
-  }
-
-  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_lastprivate]) {
-    using ModifierTy = typename LastprivateTy::LastprivateModifier;
-    using ListTy = typename LastprivateTy::List;
-    const auto &lastp = std::get<LastprivateTy>(clause.u);
-    for (auto &object : std::get<ListTy>(lastp.t)) {
-      auto &mod = std::get<std::optional<ModifierTy>>(lastp.t);
-      if (mod && *mod == ModifierTy::Conditional) {
-        getDsa(object).second |= DSA::LastPrivateConditional;
-      } else {
-        getDsa(object).second |= DSA::LastPrivate;
-      }
-    }
-  }
-
-  // Check reductions as well, clear "shared" if set.
-  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_reduction]) {
-    using ReductionTy = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
-    using ListTy = typename ReductionTy::List;
-    for (auto &object : std::get<ListTy>(std::get<ReductionTy>(clause.u).t))
-      getDsa(object).second &= ~DSA::Shared;
-  }
-
-  tomp::ListT<ObjectTy> privateObj, sharedObj, firstpObj, lastpObj, lastpcObj;
-  for (auto &[object, dsa] : objectDsa) {
-    if (dsa &
-        (DSA::FirstPrivate | DSA::LastPrivate | DSA::LastPrivateConditional)) {
-      if (dsa & DSA::FirstPrivate)
-        firstpObj.push_back(object); // no else
-      if (dsa & DSA::LastPrivateConditional)
-        lastpcObj.push_back(object);
-      else if (dsa & DSA::LastPrivate)
-        lastpObj.push_back(object);
-    } else if (dsa & DSA::Private) {
-      privateObj.push_back(object);
-    } else if (dsa & DSA::Shared) {
-      sharedObj.push_back(object);
-    }
-  }
-
-  // Materialize each clause.
-  if (!privateObj.empty()) {
-    merged.clauses.emplace_back(
-        makeClause(llvm::omp::Clause::OMPC_private,
-                   PrivateTy{/*List=*/std::move(privateObj)}));
-  }
-  if (!sharedObj.empty()) {
-    merged.clauses.emplace_back(
-        makeClause(llvm::omp::Clause::OMPC_shared,
-                   SharedTy{/*List=*/std::move(sharedObj)}));
-  }
-  if (!firstpObj.empty()) {
-    merged.clauses.emplace_back(
-        makeClause(llvm::omp::Clause::OMPC_firstprivate,
-                   FirstprivateTy{/*List=*/std::move(firstpObj)}));
-  }
-  if (!lastpObj.empty()) {
-    merged.clauses.emplace_back(
-        makeClause(llvm::omp::Clause::OMPC_lastprivate,
-                   LastprivateTy{{/*LastprivateModifier=*/std::nullopt,
-                                  /*List=*/std::move(lastpObj)}}));
-  }
-  if (!lastpcObj.empty()) {
-    auto conditional = LastprivateTy::LastprivateModifier::Conditional;
-    merged.clauses.emplace_back(
-        makeClause(llvm::omp::Clause::OMPC_lastprivate,
-                   LastprivateTy{{/*LastprivateModifier=*/conditional,
-                                  /*List=*/std::move(lastpcObj)}}));
-  }
-}
-} // namespace tomp
-
-#endif // LLVM_FRONTEND_OPENMP_CONSTRUCTCOMPOSITIONT_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
deleted file mode 100644
index 37c88f0fa07b..000000000000
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
+++ /dev/null
@@ -1,1161 +0,0 @@
-//===- ConstructDecompositionT.h -- Decomposing compound constructs -------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// Given a compound construct with a set of clauses, generate the list of
-// constituent leaf constructs, each with a list of clauses that apply to it.
-//
-// Note: Clauses that are not originally present, but that are implied by the
-// OpenMP spec are materialized, and are present in the output.
-//
-// Note: Composite constructs will also be broken up into leaf constructs.
-// If composite constructs require processing as a whole, the lists of clauses
-// for each leaf constituent should be merged.
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_FRONTEND_OPENMP_CONSTRUCTDECOMPOSITIONT_H
-#define LLVM_FRONTEND_OPENMP_CONSTRUCTDECOMPOSITIONT_H
-
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/iterator_range.h"
-#include "llvm/Frontend/OpenMP/ClauseT.h"
-#include "llvm/Frontend/OpenMP/OMP.h"
-
-#include <iterator>
-#include <list>
-#include <optional>
-#include <tuple>
-#include <type_traits>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
-#include <variant>
-
-static inline llvm::ArrayRef<llvm::omp::Directive> getWorksharing() {
-  static llvm::omp::Directive worksharing[] = {
-      llvm::omp::Directive::OMPD_do,     llvm::omp::Directive::OMPD_for,
-      llvm::omp::Directive::OMPD_scope,  llvm::omp::Directive::OMPD_sections,
-      llvm::omp::Directive::OMPD_single, llvm::omp::Directive::OMPD_workshare,
-  };
-  return worksharing;
-}
-
-static inline llvm::ArrayRef<llvm::omp::Directive> getWorksharingLoop() {
-  static llvm::omp::Directive worksharingLoop[] = {
-      llvm::omp::Directive::OMPD_do,
-      llvm::omp::Directive::OMPD_for,
-  };
-  return worksharingLoop;
-}
-
-namespace detail {
-template <typename Container, typename Predicate>
-typename std::remove_reference_t<Container>::iterator
-find_unique(Container &&container, Predicate &&pred) {
-  auto first = std::find_if(container.begin(), container.end(), pred);
-  if (first == container.end())
-    return first;
-  auto second = std::find_if(std::next(first), container.end(), pred);
-  if (second == container.end())
-    return first;
-  return container.end();
-}
-
-} // namespace detail
-
-namespace tomp {
-
-// ClauseType - Either instance of ClauseT, or a type derived from ClauseT.
-//
-// This is the clause representation in the code using this infrastructure.
-//
-// HelperType - A class that implements two member functions:
-//
-//   // Return the base object of the given object, if any.
-//   std::optional<Object> getBaseObject(const Object &object) const
-//   // Return the iteration variable of the outermost loop associated
-//   // with the construct being worked on, if any.
-//   std::optional<Object> getLoopIterVar() const
-template <typename ClauseType, typename HelperType>
-struct ConstructDecompositionT {
-  using ClauseTy = ClauseType;
-
-  using TypeTy = typename ClauseTy::TypeTy;
-  using IdTy = typename ClauseTy::IdTy;
-  using ExprTy = typename ClauseTy::ExprTy;
-  using HelperTy = HelperType;
-  using ObjectTy = tomp::ObjectT<IdTy, ExprTy>;
-
-  using ClauseSet = std::unordered_set<const ClauseTy *>;
-
-  ConstructDecompositionT(uint32_t ver, HelperType &helper,
-                          llvm::omp::Directive dir,
-                          llvm::ArrayRef<ClauseTy> clauses)
-      : version(ver), construct(dir), helper(helper) {
-    for (const ClauseTy &clause : clauses)
-      nodes.push_back(&clause);
-
-    bool success = split();
-    if (!success)
-      return;
-
-    // Copy the individual leaf directives with their clauses to the
-    // output list. Copy by value, since we don't own the storage
-    // with the input clauses, and the internal representation uses
-    // clause addresses.
-    for (auto &leaf : leafs) {
-      output.push_back({leaf.id});
-      auto &out = output.back();
-      for (const ClauseTy *c : leaf.clauses)
-        out.clauses.push_back(*c);
-    }
-  }
-
-  tomp::ListT<DirectiveWithClauses<ClauseType>> output;
-
-private:
-  bool split();
-
-  struct LeafReprInternal {
-    llvm::omp::Directive id = llvm::omp::Directive::OMPD_unknown;
-    tomp::type::ListT<const ClauseTy *> clauses;
-  };
-
-  LeafReprInternal *findDirective(llvm::omp::Directive dirId) {
-    auto found = llvm::find_if(
-        leafs, [&](const LeafReprInternal &leaf) { return leaf.id == dirId; });
-    return found != leafs.end() ? &*found : nullptr;
-  }
-
-  ClauseSet *findClausesWith(const ObjectTy &object) {
-    if (auto found = syms.find(object.id()); found != syms.end())
-      return &found->second;
-    return nullptr;
-  }
-
-  template <typename S>
-  ClauseTy *makeClause(llvm::omp::Clause clauseId, S &&specific) {
-    implicit.push_back(ClauseTy{clauseId, std::move(specific)});
-    return &implicit.back();
-  }
-
-  void addClauseSymsToMap(const ObjectTy &object, const ClauseTy *);
-  void addClauseSymsToMap(const tomp::ObjectListT<IdTy, ExprTy> &objects,
-                          const ClauseTy *);
-  void addClauseSymsToMap(const TypeTy &item, const ClauseTy *);
-  void addClauseSymsToMap(const ExprTy &item, const ClauseTy *);
-  void addClauseSymsToMap(const tomp::clause::MapT<TypeTy, IdTy, ExprTy> &item,
-                          const ClauseTy *);
-
-  template <typename U>
-  void addClauseSymsToMap(const std::optional<U> &item, const ClauseTy *);
-  template <typename U>
-  void addClauseSymsToMap(const tomp::ListT<U> &item, const ClauseTy *);
-  template <typename... U, size_t... Is>
-  void addClauseSymsToMap(const std::tuple<U...> &item, const ClauseTy *,
-                          std::index_sequence<Is...> = {});
-  template <typename U>
-  std::enable_if_t<std::is_enum_v<llvm::remove_cvref_t<U>>, void>
-  addClauseSymsToMap(U &&item, const ClauseTy *);
-
-  template <typename U>
-  std::enable_if_t<llvm::remove_cvref_t<U>::EmptyTrait::value, void>
-  addClauseSymsToMap(U &&item, const ClauseTy *);
-
-  template <typename U>
-  std::enable_if_t<llvm::remove_cvref_t<U>::IncompleteTrait::value, void>
-  addClauseSymsToMap(U &&item, const ClauseTy *);
-
-  template <typename U>
-  std::enable_if_t<llvm::remove_cvref_t<U>::WrapperTrait::value, void>
-  addClauseSymsToMap(U &&item, const ClauseTy *);
-
-  template <typename U>
-  std::enable_if_t<llvm::remove_cvref_t<U>::TupleTrait::value, void>
-  addClauseSymsToMap(U &&item, const ClauseTy *);
-
-  template <typename U>
-  std::enable_if_t<llvm::remove_cvref_t<U>::UnionTrait::value, void>
-  addClauseSymsToMap(U &&item, const ClauseTy *);
-
-  // Apply a clause to the only directive that allows it. If there are no
-  // directives that allow it, or if there is more that one, do not apply
-  // anything and return false, otherwise return true.
-  bool applyToUnique(const ClauseTy *node);
-
-  // Apply a clause to the first directive in given range that allows it.
-  // If such a directive does not exist, return false, otherwise return true.
-  template <typename Iterator>
-  bool applyToFirst(const ClauseTy *node, llvm::iterator_range<Iterator> range);
-
-  // Apply a clause to the innermost directive that allows it. If such a
-  // directive does not exist, return false, otherwise return true.
-  bool applyToInnermost(const ClauseTy *node);
-
-  // Apply a clause to the outermost directive that allows it. If such a
-  // directive does not exist, return false, otherwise return true.
-  bool applyToOutermost(const ClauseTy *node);
-
-  template <typename Predicate>
-  bool applyIf(const ClauseTy *node, Predicate shouldApply);
-
-  bool applyToAll(const ClauseTy *node);
-
-  template <typename Clause>
-  bool applyClause(Clause &&clause, const ClauseTy *node);
-
-  bool applyClause(const tomp::clause::CollapseT<TypeTy, IdTy, ExprTy> &clause,
-                   const ClauseTy *);
-  bool applyClause(const tomp::clause::PrivateT<TypeTy, IdTy, ExprTy> &clause,
-                   const ClauseTy *);
-  bool
-  applyClause(const tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy> &clause,
-              const ClauseTy *);
-  bool
-  applyClause(const tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy> &clause,
-              const ClauseTy *);
-  bool applyClause(const tomp::clause::SharedT<TypeTy, IdTy, ExprTy> &clause,
-                   const ClauseTy *);
-  bool applyClause(const tomp::clause::DefaultT<TypeTy, IdTy, ExprTy> &clause,
-                   const ClauseTy *);
-  bool
-  applyClause(const tomp::clause::ThreadLimitT<TypeTy, IdTy, ExprTy> &clause,
-              const ClauseTy *);
-  bool applyClause(const tomp::clause::OrderT<TypeTy, IdTy, ExprTy> &clause,
-                   const ClauseTy *);
-  bool applyClause(const tomp::clause::AllocateT<TypeTy, IdTy, ExprTy> &clause,
-                   const ClauseTy *);
-  bool applyClause(const tomp::clause::ReductionT<TypeTy, IdTy, ExprTy> &clause,
-                   const ClauseTy *);
-  bool applyClause(const tomp::clause::IfT<TypeTy, IdTy, ExprTy> &clause,
-                   const ClauseTy *);
-  bool applyClause(const tomp::clause::LinearT<TypeTy, IdTy, ExprTy> &clause,
-                   const ClauseTy *);
-  bool applyClause(const tomp::clause::NowaitT<TypeTy, IdTy, ExprTy> &clause,
-                   const ClauseTy *);
-
-  uint32_t version;
-  llvm::omp::Directive construct;
-  HelperType &helper;
-  ListT<LeafReprInternal> leafs;
-  tomp::ListT<const ClauseTy *> nodes;
-  std::list<ClauseTy> implicit; // Container for materialized implicit clauses.
-                                // Inserting must preserve element addresses.
-  std::unordered_map<IdTy, ClauseSet> syms;
-  std::unordered_set<IdTy> mapBases;
-};
-
-// Deduction guide
-template <typename ClauseType, typename HelperType>
-ConstructDecompositionT(uint32_t, HelperType &, llvm::omp::Directive,
-                        llvm::ArrayRef<ClauseType>)
-    -> ConstructDecompositionT<ClauseType, HelperType>;
-
-template <typename C, typename H>
-void ConstructDecompositionT<C, H>::addClauseSymsToMap(const ObjectTy &object,
-                                                       const ClauseTy *node) {
-  syms[object.id()].insert(node);
-}
-
-template <typename C, typename H>
-void ConstructDecompositionT<C, H>::addClauseSymsToMap(
-    const tomp::ObjectListT<IdTy, ExprTy> &objects, const ClauseTy *node) {
-  for (auto &object : objects)
-    syms[object.id()].insert(node);
-}
-
-template <typename C, typename H>
-void ConstructDecompositionT<C, H>::addClauseSymsToMap(const TypeTy &item,
-                                                       const ClauseTy *node) {
-  // Nothing to do for types.
-}
-
-template <typename C, typename H>
-void ConstructDecompositionT<C, H>::addClauseSymsToMap(const ExprTy &item,
-                                                       const ClauseTy *node) {
-  // Nothing to do for expressions.
-}
-
-template <typename C, typename H>
-void ConstructDecompositionT<C, H>::addClauseSymsToMap(
-    const tomp::clause::MapT<TypeTy, IdTy, ExprTy> &item,
-    const ClauseTy *node) {
-  auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(item.t);
-  addClauseSymsToMap(objects, node);
-  for (auto &object : objects) {
-    if (auto base = helper.getBaseObject(object))
-      mapBases.insert(base->id());
-  }
-}
-
-template <typename C, typename H>
-template <typename U>
-void ConstructDecompositionT<C, H>::addClauseSymsToMap(
-    const std::optional<U> &item, const ClauseTy *node) {
-  if (item)
-    addClauseSymsToMap(*item, node);
-}
-
-template <typename C, typename H>
-template <typename U>
-void ConstructDecompositionT<C, H>::addClauseSymsToMap(
-    const tomp::ListT<U> &item, const ClauseTy *node) {
-  for (auto &s : item)
-    addClauseSymsToMap(s, node);
-}
-
-template <typename C, typename H>
-template <typename... U, size_t... Is>
-void ConstructDecompositionT<C, H>::addClauseSymsToMap(
-    const std::tuple<U...> &item, const ClauseTy *node,
-    std::index_sequence<Is...>) {
-  (void)node; // Silence strange warning from GCC.
-  (addClauseSymsToMap(std::get<Is>(item), node), ...);
-}
-
-template <typename C, typename H>
-template <typename U>
-std::enable_if_t<std::is_enum_v<llvm::remove_cvref_t<U>>, void>
-ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
-                                                  const ClauseTy *node) {
-  // Nothing to do for enums.
-}
-
-template <typename C, typename H>
-template <typename U>
-std::enable_if_t<llvm::remove_cvref_t<U>::EmptyTrait::value, void>
-ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
-                                                  const ClauseTy *node) {
-  // Nothing to do for an empty class.
-}
-
-template <typename C, typename H>
-template <typename U>
-std::enable_if_t<llvm::remove_cvref_t<U>::IncompleteTrait::value, void>
-ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
-                                                  const ClauseTy *node) {
-  // Nothing to do for an incomplete class (they're empty).
-}
-
-template <typename C, typename H>
-template <typename U>
-std::enable_if_t<llvm::remove_cvref_t<U>::WrapperTrait::value, void>
-ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
-                                                  const ClauseTy *node) {
-  addClauseSymsToMap(item.v, node);
-}
-
-template <typename C, typename H>
-template <typename U>
-std::enable_if_t<llvm::remove_cvref_t<U>::TupleTrait::value, void>
-ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
-                                                  const ClauseTy *node) {
-  constexpr size_t tuple_size =
-      std::tuple_size_v<llvm::remove_cvref_t<decltype(item.t)>>;
-  addClauseSymsToMap(item.t, node, std::make_index_sequence<tuple_size>{});
-}
-
-template <typename C, typename H>
-template <typename U>
-std::enable_if_t<llvm::remove_cvref_t<U>::UnionTrait::value, void>
-ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
-                                                  const ClauseTy *node) {
-  std::visit([&](auto &&s) { addClauseSymsToMap(s, node); }, item.u);
-}
-
-// Apply a clause to the only directive that allows it. If there are no
-// directives that allow it, or if there is more that one, do not apply
-// anything and return false, otherwise return true.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyToUnique(const ClauseTy *node) {
-  auto unique = detail::find_unique(leafs, [=](const auto &dirInfo) {
-    return llvm::omp::isAllowedClauseForDirective(dirInfo.id, node->id,
-                                                  version);
-  });
-
-  if (unique != leafs.end()) {
-    unique->clauses.push_back(node);
-    return true;
-  }
-  return false;
-}
-
-// Apply a clause to the first directive in given range that allows it.
-// If such a directive does not exist, return false, otherwise return true.
-template <typename C, typename H>
-template <typename Iterator>
-bool ConstructDecompositionT<C, H>::applyToFirst(
-    const ClauseTy *node, llvm::iterator_range<Iterator> range) {
-  if (range.empty())
-    return false;
-
-  for (auto &leaf : range) {
-    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, node->id, version))
-      continue;
-    leaf.clauses.push_back(node);
-    return true;
-  }
-  return false;
-}
-
-// Apply a clause to the innermost directive that allows it. If such a
-// directive does not exist, return false, otherwise return true.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyToInnermost(const ClauseTy *node) {
-  return applyToFirst(node, llvm::reverse(leafs));
-}
-
-// Apply a clause to the outermost directive that allows it. If such a
-// directive does not exist, return false, otherwise return true.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyToOutermost(const ClauseTy *node) {
-  return applyToFirst(node, llvm::iterator_range(leafs));
-}
-
-template <typename C, typename H>
-template <typename Predicate>
-bool ConstructDecompositionT<C, H>::applyIf(const ClauseTy *node,
-                                            Predicate shouldApply) {
-  bool applied = false;
-  for (auto &leaf : leafs) {
-    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, node->id, version))
-      continue;
-    if (!shouldApply(leaf))
-      continue;
-    leaf.clauses.push_back(node);
-    applied = true;
-  }
-
-  return applied;
-}
-
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyToAll(const ClauseTy *node) {
-  return applyIf(node, [](auto) { return true; });
-}
-
-template <typename C, typename H>
-template <typename Clause>
-bool ConstructDecompositionT<C, H>::applyClause(Clause &&clause,
-                                                const ClauseTy *node) {
-  // The default behavior is to find the unique directive to which the
-  // given clause may be applied. If there are no such directives, or
-  // if there are multiple ones, flag an error.
-  // From "OpenMP Application Programming Interface", Version 5.2:
-  // S Some clauses are permitted only on a single leaf construct of the
-  // S combined or composite construct, in which case the effect is as if
-  // S the clause is applied to that specific construct. (p339, 31-33)
-  if (applyToUnique(node))
-    return true;
-
-  return false;
-}
-
-// COLLAPSE
-// [5.2:93:20-21]
-// Directives: distribute, do, for, loop, simd, taskloop
-//
-// [5.2:339:35]
-// (35) The collapse clause is applied once to the combined or composite
-// construct.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::CollapseT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  // Apply "collapse" to the innermost directive. If it's not one that
-  // allows it flag an error.
-  if (!leafs.empty()) {
-    auto &last = leafs.back();
-
-    if (llvm::omp::isAllowedClauseForDirective(last.id, node->id, version)) {
-      last.clauses.push_back(node);
-      return true;
-    }
-  }
-
-  return false;
-}
-
-// PRIVATE
-// [5.2:111:5-7]
-// Directives: distribute, do, for, loop, parallel, scope, sections, simd,
-// single, target, task, taskloop, teams
-//
-// [5.2:340:1-2]
-// (1) The effect of the 1 private clause is as if it is applied only to the
-// innermost leaf construct that permits it.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::PrivateT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  return applyToInnermost(node);
-}
-
-// FIRSTPRIVATE
-// [5.2:112:5-7]
-// Directives: distribute, do, for, parallel, scope, sections, single, target,
-// task, taskloop, teams
-//
-// [5.2:340:3-20]
-// (3) The effect of the firstprivate clause is as if it is applied to one or
-// more leaf constructs as follows:
-//  (5) To the distribute construct if it is among the constituent constructs;
-//  (6) To the teams construct if it is among the constituent constructs and the
-//      distribute construct is not;
-//  (8) To a worksharing construct that accepts the clause if one is among the
-//      constituent constructs;
-//  (9) To the taskloop construct if it is among the constituent constructs;
-// (10) To the parallel construct if it is among the constituent constructs and
-//      neither a taskloop construct nor a worksharing construct that accepts
-//      the clause is among them;
-// (12) To the target construct if it is among the constituent constructs and
-//      the same list item neither appears in a lastprivate clause nor is the
-//      base variable or base pointer of a list item that appears in a map
-//      clause.
-//
-// (15) If the parallel construct is among the constituent constructs and the
-// effect is not as if the firstprivate clause is applied to it by the above
-// rules, then the effect is as if the shared clause with the same list item is
-// applied to the parallel construct.
-// (17) If the teams construct is among the constituent constructs and the
-// effect is not as if the firstprivate clause is applied to it by the above
-// rules, then the effect is as if the shared clause with the same list item is
-// applied to the teams construct.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  bool applied = false;
-
-  // [5.2:340:3-6]
-  auto dirDistribute = findDirective(llvm::omp::OMPD_distribute);
-  auto dirTeams = findDirective(llvm::omp::OMPD_teams);
-  if (dirDistribute != nullptr) {
-    dirDistribute->clauses.push_back(node);
-    applied = true;
-    // [5.2:340:17]
-    if (dirTeams != nullptr) {
-      auto *shared = makeClause(
-          llvm::omp::Clause::OMPC_shared,
-          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/clause.v});
-      dirTeams->clauses.push_back(shared);
-    }
-  } else if (dirTeams != nullptr) {
-    dirTeams->clauses.push_back(node);
-    applied = true;
-  }
-
-  // [5.2:340:8]
-  auto findWorksharing = [&]() {
-    auto worksharing = getWorksharing();
-    for (auto &leaf : leafs) {
-      auto found = llvm::find(worksharing, leaf.id);
-      if (found != std::end(worksharing))
-        return &leaf;
-    }
-    return static_cast<typename decltype(leafs)::value_type *>(nullptr);
-  };
-
-  auto dirWorksharing = findWorksharing();
-  if (dirWorksharing != nullptr) {
-    dirWorksharing->clauses.push_back(node);
-    applied = true;
-  }
-
-  // [5.2:340:9]
-  auto dirTaskloop = findDirective(llvm::omp::OMPD_taskloop);
-  if (dirTaskloop != nullptr) {
-    dirTaskloop->clauses.push_back(node);
-    applied = true;
-  }
-
-  // [5.2:340:10]
-  auto dirParallel = findDirective(llvm::omp::OMPD_parallel);
-  if (dirParallel != nullptr) {
-    if (dirTaskloop == nullptr && dirWorksharing == nullptr) {
-      dirParallel->clauses.push_back(node);
-      applied = true;
-    } else {
-      // [5.2:340:15]
-      auto *shared = makeClause(
-          llvm::omp::Clause::OMPC_shared,
-          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/clause.v});
-      dirParallel->clauses.push_back(shared);
-    }
-  }
-
-  // [5.2:340:12]
-  auto inLastprivate = [&](const ObjectTy &object) {
-    if (ClauseSet *set = findClausesWith(object)) {
-      return llvm::find_if(*set, [](const ClauseTy *c) {
-               return c->id == llvm::omp::Clause::OMPC_lastprivate;
-             }) != set->end();
-    }
-    return false;
-  };
-
-  auto dirTarget = findDirective(llvm::omp::OMPD_target);
-  if (dirTarget != nullptr) {
-    tomp::ObjectListT<IdTy, ExprTy> objects;
-    llvm::copy_if(
-        clause.v, std::back_inserter(objects), [&](const ObjectTy &object) {
-          return !inLastprivate(object) && !mapBases.count(object.id());
-        });
-    if (!objects.empty()) {
-      auto *firstp = makeClause(
-          llvm::omp::Clause::OMPC_firstprivate,
-          tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>{/*List=*/objects});
-      dirTarget->clauses.push_back(firstp);
-      applied = true;
-    }
-  }
-
-  // "task" is not handled by any of the cases above.
-  if (auto dirTask = findDirective(llvm::omp::OMPD_task)) {
-    dirTask->clauses.push_back(node);
-    applied = true;
-  }
-
-  return applied;
-}
-
-// LASTPRIVATE
-// [5.2:115:7-8]
-// Directives: distribute, do, for, loop, sections, simd, taskloop
-//
-// [5.2:340:21-30]
-// (21) The effect of the lastprivate clause is as if it is applied to all leaf
-// constructs that permit the clause.
-// (22) If the parallel construct is among the constituent constructs and the
-// list item is not also specified in the firstprivate clause, then the effect
-// of the lastprivate clause is as if the shared clause with the same list item
-// is applied to the parallel construct.
-// (24) If the teams construct is among the constituent constructs and the list
-// item is not also specified in the firstprivate clause, then the effect of the
-// lastprivate clause is as if the shared clause with the same list item is
-// applied to the teams construct.
-// (27) If the target construct is among the constituent constructs and the list
-// item is not the base variable or base pointer of a list item that appears in
-// a map clause, the effect of the lastprivate clause is as if the same list
-// item appears in a map clause with a map-type of tofrom.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  bool applied = false;
-
-  // [5.2:340:21]
-  applied = applyToAll(node);
-  if (!applied)
-    return false;
-
-  auto inFirstprivate = [&](const ObjectTy &object) {
-    if (ClauseSet *set = findClausesWith(object)) {
-      return llvm::find_if(*set, [](const ClauseTy *c) {
-               return c->id == llvm::omp::Clause::OMPC_firstprivate;
-             }) != set->end();
-    }
-    return false;
-  };
-
-  auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(clause.t);
-
-  // Prepare list of objects that could end up in a "shared" clause.
-  tomp::ObjectListT<IdTy, ExprTy> sharedObjects;
-  llvm::copy_if(
-      objects, std::back_inserter(sharedObjects),
-      [&](const ObjectTy &object) { return !inFirstprivate(object); });
-
-  if (!sharedObjects.empty()) {
-    // [5.2:340:22]
-    if (auto dirParallel = findDirective(llvm::omp::OMPD_parallel)) {
-      auto *shared = makeClause(
-          llvm::omp::Clause::OMPC_shared,
-          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
-      dirParallel->clauses.push_back(shared);
-      applied = true;
-    }
-
-    // [5.2:340:24]
-    if (auto dirTeams = findDirective(llvm::omp::OMPD_teams)) {
-      auto *shared = makeClause(
-          llvm::omp::Clause::OMPC_shared,
-          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
-      dirTeams->clauses.push_back(shared);
-      applied = true;
-    }
-  }
-
-  // [5.2:340:27]
-  if (auto dirTarget = findDirective(llvm::omp::OMPD_target)) {
-    tomp::ObjectListT<IdTy, ExprTy> tofrom;
-    llvm::copy_if(
-        objects, std::back_inserter(tofrom),
-        [&](const ObjectTy &object) { return !mapBases.count(object.id()); });
-
-    if (!tofrom.empty()) {
-      using MapType =
-          typename tomp::clause::MapT<TypeTy, IdTy, ExprTy>::MapType;
-      auto *map =
-          makeClause(llvm::omp::Clause::OMPC_map,
-                     tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
-                         {/*MapType=*/MapType::Tofrom,
-                          /*MapTypeModifier=*/std::nullopt,
-                          /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
-                          /*LocatorList=*/std::move(tofrom)}});
-      dirTarget->clauses.push_back(map);
-      applied = true;
-    }
-  }
-
-  return applied;
-}
-
-// SHARED
-// [5.2:110:5-6]
-// Directives: parallel, task, taskloop, teams
-//
-// [5.2:340:31-32]
-// (31) The effect of the shared, default, thread_limit, or order clause is as
-// if it is applied to all leaf constructs that permit the clause.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::SharedT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  // [5.2:340:31]
-  return applyToAll(node);
-}
-
-// DEFAULT
-// [5.2:109:5-6]
-// Directives: parallel, task, taskloop, teams
-//
-// [5.2:340:31-32]
-// (31) The effect of the shared, default, thread_limit, or order clause is as
-// if it is applied to all leaf constructs that permit the clause.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::DefaultT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  // [5.2:340:31]
-  return applyToAll(node);
-}
-
-// THREAD_LIMIT
-// [5.2:277:14-15]
-// Directives: target, teams
-//
-// [5.2:340:31-32]
-// (31) The effect of the shared, default, thread_limit, or order clause is as
-// if it is applied to all leaf constructs that permit the clause.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::ThreadLimitT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  // [5.2:340:31]
-  return applyToAll(node);
-}
-
-// ORDER
-// [5.2:234:3-4]
-// Directives: distribute, do, for, loop, simd
-//
-// [5.2:340:31-32]
-// (31) The effect of the shared, default, thread_limit, or order clause is as
-// if it is applied to all leaf constructs that permit the clause.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::OrderT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  // [5.2:340:31]
-  return applyToAll(node);
-}
-
-// ALLOCATE
-// [5.2:178:7-9]
-// Directives: allocators, distribute, do, for, parallel, scope, sections,
-// single, target, task, taskgroup, taskloop, teams
-//
-// [5.2:340:33-35]
-// (33) The effect of the allocate clause is as if it is applied to all leaf
-// constructs that permit the clause and to which a data-sharing attribute
-// clause that may create a private copy of the same list item is applied.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::AllocateT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  // This one needs to be applied at the end, once we know which clauses are
-  // assigned to which leaf constructs.
-
-  // [5.2:340:33]
-  auto canMakePrivateCopy = [](llvm::omp::Clause id) {
-    switch (id) {
-    case llvm::omp::Clause::OMPC_firstprivate:
-    case llvm::omp::Clause::OMPC_lastprivate:
-    case llvm::omp::Clause::OMPC_private:
-      return true;
-    default:
-      return false;
-    }
-  };
-
-  bool applied = applyIf(node, [&](const auto &leaf) {
-    return llvm::any_of(leaf.clauses, [&](const ClauseTy *n) {
-      return canMakePrivateCopy(n->id);
-    });
-  });
-
-  return applied;
-}
-
-// REDUCTION
-// [5.2:134:17-18]
-// Directives: do, for, loop, parallel, scope, sections, simd, taskloop, teams
-//
-// [5.2:340:36-37], [5.2:341:1-13]
-// (36) The effect of the reduction clause is as if it is applied to all leaf
-// constructs that permit the clause, except for the following constructs:
-//  (1) The parallel construct, when combined with the sections,
-//      worksharing-loop, loop, or taskloop construct; and
-//  (3) The teams construct, when combined with the loop construct.
-// (4) For the parallel and teams constructs above, the effect of the reduction
-// clause instead is as if each list item or, for any list item that is an array
-// item, its corresponding base array or base pointer appears in a shared clause
-// for the construct.
-// (6) If the task reduction-modifier is specified, the effect is as if it only
-// modifies the behavior of the reduction clause on the innermost leaf construct
-// that accepts the modifier (see Section 5.5.8).
-// (8) If the inscan reduction-modifier is specified, the effect is as if it
-// modifies the behavior of the reduction clause on all constructs of the
-// combined construct to which the clause is applied and that accept the
-// modifier.
-// (10) If a list item in a reduction clause on a combined target construct does
-// not have the same base variable or base pointer as a list item in a map
-// clause on the construct, then the effect is as if the list item in the
-// reduction clause appears as a list item in a map clause with a map-type of
-// tofrom.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::ReductionT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  using ReductionTy = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
-
-  // [5.2:340:36], [5.2:341:1], [5.2:341:3]
-  bool applyToParallel = true, applyToTeams = true;
-
-  auto dirParallel = findDirective(llvm::omp::Directive::OMPD_parallel);
-  if (dirParallel) {
-    auto exclusions = llvm::concat<const llvm::omp::Directive>(
-        getWorksharingLoop(), tomp::ListT<llvm::omp::Directive>{
-                                  llvm::omp::Directive::OMPD_loop,
-                                  llvm::omp::Directive::OMPD_sections,
-                                  llvm::omp::Directive::OMPD_taskloop,
-                              });
-    auto present = [&](llvm::omp::Directive id) {
-      return findDirective(id) != nullptr;
-    };
-
-    if (llvm::any_of(exclusions, present))
-      applyToParallel = false;
-  }
-
-  auto dirTeams = findDirective(llvm::omp::Directive::OMPD_teams);
-  if (dirTeams) {
-    // The only exclusion is OMPD_loop.
-    if (findDirective(llvm::omp::Directive::OMPD_loop))
-      applyToTeams = false;
-  }
-
-  using ReductionModifier = typename ReductionTy::ReductionModifier;
-  using ReductionIdentifiers = typename ReductionTy::ReductionIdentifiers;
-
-  auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(clause.t);
-  auto &modifier = std::get<std::optional<ReductionModifier>>(clause.t);
-
-  // Apply the reduction clause first to all directives according to the spec.
-  // If the reduction was applied at least once, proceed with the data sharing
-  // side-effects.
-  bool applied = false;
-
-  // [5.2:341:6], [5.2:341:8]
-  auto isValidModifier = [](llvm::omp::Directive dir, ReductionModifier mod,
-                            bool alreadyApplied) {
-    switch (mod) {
-    case ReductionModifier::Inscan:
-      // According to [5.2:135:11-13], "inscan" only applies to
-      // worksharing-loop, worksharing-loop-simd, or "simd" constructs.
-      return dir == llvm::omp::Directive::OMPD_simd ||
-             llvm::is_contained(getWorksharingLoop(), dir);
-    case ReductionModifier::Task:
-      if (alreadyApplied)
-        return false;
-      // According to [5.2:135:16-18], "task" only applies to "parallel" and
-      // worksharing constructs.
-      return dir == llvm::omp::Directive::OMPD_parallel ||
-             llvm::is_contained(getWorksharing(), dir);
-    case ReductionModifier::Default:
-      return true;
-    }
-    llvm_unreachable("Unexpected modifier");
-  };
-
-  auto *unmodified = makeClause(
-      llvm::omp::Clause::OMPC_reduction,
-      ReductionTy{
-          {/*ReductionModifier=*/std::nullopt,
-           /*ReductionIdentifiers=*/std::get<ReductionIdentifiers>(clause.t),
-           /*List=*/objects}});
-
-  ReductionModifier effective =
-      modifier.has_value() ? *modifier : ReductionModifier::Default;
-  bool effectiveApplied = false;
-  // Walk over the leaf constructs starting from the innermost, and apply
-  // the clause as required by the spec.
-  for (auto &leaf : llvm::reverse(leafs)) {
-    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, node->id, version))
-      continue;
-    if (!applyToParallel && &leaf == dirParallel)
-      continue;
-    if (!applyToTeams && &leaf == dirTeams)
-      continue;
-    // Some form of the clause will be applied past this point.
-    if (isValidModifier(leaf.id, effective, effectiveApplied)) {
-      // Apply clause with modifier.
-      leaf.clauses.push_back(node);
-      effectiveApplied = true;
-    } else {
-      // Apply clause without modifier.
-      leaf.clauses.push_back(unmodified);
-    }
-    applied = true;
-  }
-
-  if (!applied)
-    return false;
-
-  tomp::ObjectListT<IdTy, ExprTy> sharedObjects;
-  llvm::transform(objects, std::back_inserter(sharedObjects),
-                  [&](const ObjectTy &object) {
-                    auto maybeBase = helper.getBaseObject(object);
-                    return maybeBase ? *maybeBase : object;
-                  });
-
-  // [5.2:341:4]
-  if (!sharedObjects.empty()) {
-    if (dirParallel && !applyToParallel) {
-      auto *shared = makeClause(
-          llvm::omp::Clause::OMPC_shared,
-          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
-      dirParallel->clauses.push_back(shared);
-    }
-    if (dirTeams && !applyToTeams) {
-      auto *shared = makeClause(
-          llvm::omp::Clause::OMPC_shared,
-          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
-      dirTeams->clauses.push_back(shared);
-    }
-  }
-
-  // [5.2:341:10]
-  auto dirTarget = findDirective(llvm::omp::Directive::OMPD_target);
-  if (dirTarget && leafs.size() > 1) {
-    tomp::ObjectListT<IdTy, ExprTy> tofrom;
-    llvm::copy_if(objects, std::back_inserter(tofrom),
-                  [&](const ObjectTy &object) {
-                    if (auto maybeBase = helper.getBaseObject(object))
-                      return !mapBases.count(maybeBase->id());
-                    return !mapBases.count(object.id()); // XXX is this ok?
-                  });
-    if (!tofrom.empty()) {
-      using MapType =
-          typename tomp::clause::MapT<TypeTy, IdTy, ExprTy>::MapType;
-      auto *map = makeClause(
-          llvm::omp::Clause::OMPC_map,
-          tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
-              {/*MapType=*/MapType::Tofrom, /*MapTypeModifier=*/std::nullopt,
-               /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
-               /*LocatorList=*/std::move(tofrom)}});
-
-      dirTarget->clauses.push_back(map);
-      applied = true;
-    }
-  }
-
-  return applied;
-}
-
-// IF
-// [5.2:72:7-9]
-// Directives: cancel, parallel, simd, target, target data, target enter data,
-// target exit data, target update, task, taskloop
-//
-// [5.2:72:15-18]
-// (15) For combined or composite constructs, the if clause only applies to the
-// semantics of the construct named in the directive-name-modifier.
-// (16) For a combined or composite construct, if no directive-name-modifier is
-// specified then the if clause applies to all constituent constructs to which
-// an if clause can apply.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::IfT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  using DirectiveNameModifier =
-      typename clause::IfT<TypeTy, IdTy, ExprTy>::DirectiveNameModifier;
-  using IfExpression = typename clause::IfT<TypeTy, IdTy, ExprTy>::IfExpression;
-  auto &modifier = std::get<std::optional<DirectiveNameModifier>>(clause.t);
-
-  if (modifier) {
-    llvm::omp::Directive dirId = *modifier;
-    auto *unmodified =
-        makeClause(llvm::omp::Clause::OMPC_if,
-                   tomp::clause::IfT<TypeTy, IdTy, ExprTy>{
-                       {/*DirectiveNameModifier=*/std::nullopt,
-                        /*IfExpression=*/std::get<IfExpression>(clause.t)}});
-
-    if (auto *hasDir = findDirective(dirId)) {
-      hasDir->clauses.push_back(unmodified);
-      return true;
-    }
-    return false;
-  }
-
-  return applyToAll(node);
-}
-
-// LINEAR
-// [5.2:118:1-2]
-// Directives: declare simd, do, for, simd
-//
-// [5.2:341:15-22]
-// (15.1) The effect of the linear clause is as if it is applied to the
-// innermost leaf construct.
-// (15.2) Additionally, if the list item is not the iteration variable of a simd
-// or worksharing-loop SIMD construct, the effect on the outer leaf constructs
-// is as if the list item was specified in firstprivate and lastprivate clauses
-// on the combined or composite construct, with the rules specified above
-// applied.
-// (19) If a list item of the linear clause is the iteration variable of a simd
-// or worksharing-loop SIMD construct and it is not declared in the construct,
-// the effect on the outer leaf constructs is as if the list item was specified
-// in a lastprivate clause on the combined or composite construct with the rules
-// specified above applied.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::LinearT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  // [5.2:341:15.1]
-  if (!applyToInnermost(node))
-    return false;
-
-  // [5.2:341:15.2], [5.2:341:19]
-  auto dirSimd = findDirective(llvm::omp::Directive::OMPD_simd);
-  std::optional<ObjectTy> iterVar = helper.getLoopIterVar();
-  const auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(clause.t);
-
-  // Lists of objects that will be used to construct "firstprivate" and
-  // "lastprivate" clauses.
-  tomp::ObjectListT<IdTy, ExprTy> first, last;
-
-  for (const ObjectTy &object : objects) {
-    last.push_back(object);
-    if (!dirSimd || !iterVar || object.id() != iterVar->id())
-      first.push_back(object);
-  }
-
-  if (!first.empty()) {
-    auto *firstp = makeClause(
-        llvm::omp::Clause::OMPC_firstprivate,
-        tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>{/*List=*/first});
-    nodes.push_back(firstp); // Appending to the main clause list.
-  }
-  if (!last.empty()) {
-    auto *lastp =
-        makeClause(llvm::omp::Clause::OMPC_lastprivate,
-                   tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>{
-                       {/*LastprivateModifier=*/std::nullopt, /*List=*/last}});
-    nodes.push_back(lastp); // Appending to the main clause list.
-  }
-  return true;
-}
-
-// NOWAIT
-// [5.2:308:11-13]
-// Directives: dispatch, do, for, interop, scope, sections, single, target,
-// target enter data, target exit data, target update, taskwait, workshare
-//
-// [5.2:341:23]
-// (23) The effect of the nowait clause is as if it is applied to the outermost
-// leaf construct that permits it.
-template <typename C, typename H>
-bool ConstructDecompositionT<C, H>::applyClause(
-    const tomp::clause::NowaitT<TypeTy, IdTy, ExprTy> &clause,
-    const ClauseTy *node) {
-  return applyToOutermost(node);
-}
-
-template <typename C, typename H> bool ConstructDecompositionT<C, H>::split() {
-  bool success = true;
-
-  for (llvm::omp::Directive leaf :
-       llvm::omp::getLeafConstructsOrSelf(construct))
-    leafs.push_back(LeafReprInternal{leaf, /*clauses=*/{}});
-
-  for (const ClauseTy *node : nodes)
-    addClauseSymsToMap(*node, node);
-
-  // First we need to apply LINEAR, because it can generate additional
-  // "firstprivate" and "lastprivate" clauses that apply to the combined/
-  // composite construct.
-  // Collect them separately, because they may modify the clause list.
-  llvm::SmallVector<const ClauseTy *> linears;
-  for (const ClauseTy *node : nodes) {
-    if (node->id == llvm::omp::Clause::OMPC_linear)
-      linears.push_back(node);
-  }
-  for (const auto *node : linears) {
-    success = success &&
-              applyClause(std::get<tomp::clause::LinearT<TypeTy, IdTy, ExprTy>>(
-                              node->u),
-                          node);
-  }
-
-  // "allocate" clauses need to be applied last since they need to see
-  // which directives have data-privatizing clauses.
-  auto skip = [](const ClauseTy *node) {
-    switch (node->id) {
-    case llvm::omp::Clause::OMPC_allocate:
-    case llvm::omp::Clause::OMPC_linear:
-      return true;
-    default:
-      return false;
-    }
-  };
-
-  // Apply (almost) all clauses.
-  for (const ClauseTy *node : nodes) {
-    if (skip(node))
-      continue;
-    success =
-        success &&
-        std::visit([&](auto &&s) { return applyClause(s, node); }, node->u);
-  }
-
-  // Apply "allocate".
-  for (const ClauseTy *node : nodes) {
-    if (node->id != llvm::omp::Clause::OMPC_allocate)
-      continue;
-    success =
-        success &&
-        std::visit([&](auto &&s) { return applyClause(s, node); }, node->u);
-  }
-
-  return success;
-}
-
-} // namespace tomp
-
-#endif // LLVM_FRONTEND_OPENMP_CONSTRUCTDECOMPOSITIONT_H
diff --git a/llvm/unittests/Frontend/CMakeLists.txt b/llvm/unittests/Frontend/CMakeLists.txt
index 85e113816e3b..3f290b63ba64 100644
--- a/llvm/unittests/Frontend/CMakeLists.txt
+++ b/llvm/unittests/Frontend/CMakeLists.txt
@@ -15,7 +15,6 @@ add_llvm_unittest(LLVMFrontendTests
   OpenMPIRBuilderTest.cpp
   OpenMPParsingTest.cpp
   OpenMPCompositionTest.cpp
-  OpenMPDecompositionTest.cpp
 
   DEPENDS
   acc_gen
diff --git a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
deleted file mode 100644
index df48e9cc0ff4..000000000000
--- a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
+++ /dev/null
@@ -1,999 +0,0 @@
-//===- llvm/unittests/Frontend/OpenMPDecompositionTest.cpp ----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Frontend/OpenMP/ClauseT.h"
-#include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
-#include "llvm/Frontend/OpenMP/OMP.h"
-#include "gtest/gtest.h"
-
-#include <iterator>
-#include <optional>
-#include <sstream>
-#include <string>
-#include <tuple>
-#include <type_traits>
-#include <utility>
-
-// The actual tests start at comment "--- Test" below.
-
-// Create simple instantiations of all clauses to allow manual construction
-// of clauses, and implement emitting of a directive with clauses to a string.
-//
-// The tests then follow the pattern
-// 1. Create a list of clauses.
-// 2. Pass them, together with a construct, to the decomposition class.
-// 3. Extract individual resulting leaf constructs with clauses applied
-//    to them.
-// 4. Convert them to strings and compare with expected outputs.
-
-namespace omp {
-struct TypeTy {}; // placeholder
-struct ExprTy {}; // placeholder
-using IdTy = std::string;
-} // namespace omp
-
-namespace tomp::type {
-template <> struct ObjectT<omp::IdTy, omp::ExprTy> {
-  const omp::IdTy &id() const { return name; }
-  const std::optional<omp::ExprTy> ref() const { return omp::ExprTy{}; }
-
-  omp::IdTy name;
-};
-} // namespace tomp::type
-
-namespace omp {
-template <typename ElemTy> using List = tomp::type::ListT<ElemTy>;
-
-using Object = tomp::ObjectT<IdTy, ExprTy>;
-
-namespace clause {
-using DefinedOperator = tomp::type::DefinedOperatorT<IdTy, ExprTy>;
-using ProcedureDesignator = tomp::type::ProcedureDesignatorT<IdTy, ExprTy>;
-using ReductionOperator = tomp::type::ReductionIdentifierT<IdTy, ExprTy>;
-
-using AcqRel = tomp::clause::AcqRelT<TypeTy, IdTy, ExprTy>;
-using Acquire = tomp::clause::AcquireT<TypeTy, IdTy, ExprTy>;
-using AdjustArgs = tomp::clause::AdjustArgsT<TypeTy, IdTy, ExprTy>;
-using Affinity = tomp::clause::AffinityT<TypeTy, IdTy, ExprTy>;
-using Aligned = tomp::clause::AlignedT<TypeTy, IdTy, ExprTy>;
-using Align = tomp::clause::AlignT<TypeTy, IdTy, ExprTy>;
-using Allocate = tomp::clause::AllocateT<TypeTy, IdTy, ExprTy>;
-using Allocator = tomp::clause::AllocatorT<TypeTy, IdTy, ExprTy>;
-using AppendArgs = tomp::clause::AppendArgsT<TypeTy, IdTy, ExprTy>;
-using AtomicDefaultMemOrder =
-    tomp::clause::AtomicDefaultMemOrderT<TypeTy, IdTy, ExprTy>;
-using At = tomp::clause::AtT<TypeTy, IdTy, ExprTy>;
-using Bind = tomp::clause::BindT<TypeTy, IdTy, ExprTy>;
-using Capture = tomp::clause::CaptureT<TypeTy, IdTy, ExprTy>;
-using Collapse = tomp::clause::CollapseT<TypeTy, IdTy, ExprTy>;
-using Compare = tomp::clause::CompareT<TypeTy, IdTy, ExprTy>;
-using Copyin = tomp::clause::CopyinT<TypeTy, IdTy, ExprTy>;
-using Copyprivate = tomp::clause::CopyprivateT<TypeTy, IdTy, ExprTy>;
-using Defaultmap = tomp::clause::DefaultmapT<TypeTy, IdTy, ExprTy>;
-using Default = tomp::clause::DefaultT<TypeTy, IdTy, ExprTy>;
-using Depend = tomp::clause::DependT<TypeTy, IdTy, ExprTy>;
-using Destroy = tomp::clause::DestroyT<TypeTy, IdTy, ExprTy>;
-using Detach = tomp::clause::DetachT<TypeTy, IdTy, ExprTy>;
-using Device = tomp::clause::DeviceT<TypeTy, IdTy, ExprTy>;
-using DeviceType = tomp::clause::DeviceTypeT<TypeTy, IdTy, ExprTy>;
-using DistSchedule = tomp::clause::DistScheduleT<TypeTy, IdTy, ExprTy>;
-using Doacross = tomp::clause::DoacrossT<TypeTy, IdTy, ExprTy>;
-using DynamicAllocators =
-    tomp::clause::DynamicAllocatorsT<TypeTy, IdTy, ExprTy>;
-using Enter = tomp::clause::EnterT<TypeTy, IdTy, ExprTy>;
-using Exclusive = tomp::clause::ExclusiveT<TypeTy, IdTy, ExprTy>;
-using Fail = tomp::clause::FailT<TypeTy, IdTy, ExprTy>;
-using Filter = tomp::clause::FilterT<TypeTy, IdTy, ExprTy>;
-using Final = tomp::clause::FinalT<TypeTy, IdTy, ExprTy>;
-using Firstprivate = tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>;
-using From = tomp::clause::FromT<TypeTy, IdTy, ExprTy>;
-using Full = tomp::clause::FullT<TypeTy, IdTy, ExprTy>;
-using Grainsize = tomp::clause::GrainsizeT<TypeTy, IdTy, ExprTy>;
-using HasDeviceAddr = tomp::clause::HasDeviceAddrT<TypeTy, IdTy, ExprTy>;
-using Hint = tomp::clause::HintT<TypeTy, IdTy, ExprTy>;
-using If = tomp::clause::IfT<TypeTy, IdTy, ExprTy>;
-using Inbranch = tomp::clause::InbranchT<TypeTy, IdTy, ExprTy>;
-using Inclusive = tomp::clause::InclusiveT<TypeTy, IdTy, ExprTy>;
-using Indirect = tomp::clause::IndirectT<TypeTy, IdTy, ExprTy>;
-using Init = tomp::clause::InitT<TypeTy, IdTy, ExprTy>;
-using InReduction = tomp::clause::InReductionT<TypeTy, IdTy, ExprTy>;
-using IsDevicePtr = tomp::clause::IsDevicePtrT<TypeTy, IdTy, ExprTy>;
-using Lastprivate = tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>;
-using Linear = tomp::clause::LinearT<TypeTy, IdTy, ExprTy>;
-using Link = tomp::clause::LinkT<TypeTy, IdTy, ExprTy>;
-using Map = tomp::clause::MapT<TypeTy, IdTy, ExprTy>;
-using Match = tomp::clause::MatchT<TypeTy, IdTy, ExprTy>;
-using Mergeable = tomp::clause::MergeableT<TypeTy, IdTy, ExprTy>;
-using Message = tomp::clause::MessageT<TypeTy, IdTy, ExprTy>;
-using Nocontext = tomp::clause::NocontextT<TypeTy, IdTy, ExprTy>;
-using Nogroup = tomp::clause::NogroupT<TypeTy, IdTy, ExprTy>;
-using Nontemporal = tomp::clause::NontemporalT<TypeTy, IdTy, ExprTy>;
-using Notinbranch = tomp::clause::NotinbranchT<TypeTy, IdTy, ExprTy>;
-using Novariants = tomp::clause::NovariantsT<TypeTy, IdTy, ExprTy>;
-using Nowait = tomp::clause::NowaitT<TypeTy, IdTy, ExprTy>;
-using NumTasks = tomp::clause::NumTasksT<TypeTy, IdTy, ExprTy>;
-using NumTeams = tomp::clause::NumTeamsT<TypeTy, IdTy, ExprTy>;
-using NumThreads = tomp::clause::NumThreadsT<TypeTy, IdTy, ExprTy>;
-using OmpxAttribute = tomp::clause::OmpxAttributeT<TypeTy, IdTy, ExprTy>;
-using OmpxBare = tomp::clause::OmpxBareT<TypeTy, IdTy, ExprTy>;
-using OmpxDynCgroupMem = tomp::clause::OmpxDynCgroupMemT<TypeTy, IdTy, ExprTy>;
-using Ordered = tomp::clause::OrderedT<TypeTy, IdTy, ExprTy>;
-using Order = tomp::clause::OrderT<TypeTy, IdTy, ExprTy>;
-using Partial = tomp::clause::PartialT<TypeTy, IdTy, ExprTy>;
-using Priority = tomp::clause::PriorityT<TypeTy, IdTy, ExprTy>;
-using Private = tomp::clause::PrivateT<TypeTy, IdTy, ExprTy>;
-using ProcBind = tomp::clause::ProcBindT<TypeTy, IdTy, ExprTy>;
-using Read = tomp::clause::ReadT<TypeTy, IdTy, ExprTy>;
-using Reduction = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
-using Relaxed = tomp::clause::RelaxedT<TypeTy, IdTy, ExprTy>;
-using Release = tomp::clause::ReleaseT<TypeTy, IdTy, ExprTy>;
-using ReverseOffload = tomp::clause::ReverseOffloadT<TypeTy, IdTy, ExprTy>;
-using Safelen = tomp::clause::SafelenT<TypeTy, IdTy, ExprTy>;
-using Schedule = tomp::clause::ScheduleT<TypeTy, IdTy, ExprTy>;
-using SeqCst = tomp::clause::SeqCstT<TypeTy, IdTy, ExprTy>;
-using Severity = tomp::clause::SeverityT<TypeTy, IdTy, ExprTy>;
-using Shared = tomp::clause::SharedT<TypeTy, IdTy, ExprTy>;
-using Simdlen = tomp::clause::SimdlenT<TypeTy, IdTy, ExprTy>;
-using Simd = tomp::clause::SimdT<TypeTy, IdTy, ExprTy>;
-using Sizes = tomp::clause::SizesT<TypeTy, IdTy, ExprTy>;
-using TaskReduction = tomp::clause::TaskReductionT<TypeTy, IdTy, ExprTy>;
-using ThreadLimit = tomp::clause::ThreadLimitT<TypeTy, IdTy, ExprTy>;
-using Threads = tomp::clause::ThreadsT<TypeTy, IdTy, ExprTy>;
-using To = tomp::clause::ToT<TypeTy, IdTy, ExprTy>;
-using UnifiedAddress = tomp::clause::UnifiedAddressT<TypeTy, IdTy, ExprTy>;
-using UnifiedSharedMemory =
-    tomp::clause::UnifiedSharedMemoryT<TypeTy, IdTy, ExprTy>;
-using Uniform = tomp::clause::UniformT<TypeTy, IdTy, ExprTy>;
-using Unknown = tomp::clause::UnknownT<TypeTy, IdTy, ExprTy>;
-using Untied = tomp::clause::UntiedT<TypeTy, IdTy, ExprTy>;
-using Update = tomp::clause::UpdateT<TypeTy, IdTy, ExprTy>;
-using UseDeviceAddr = tomp::clause::UseDeviceAddrT<TypeTy, IdTy, ExprTy>;
-using UseDevicePtr = tomp::clause::UseDevicePtrT<TypeTy, IdTy, ExprTy>;
-using UsesAllocators = tomp::clause::UsesAllocatorsT<TypeTy, IdTy, ExprTy>;
-using Use = tomp::clause::UseT<TypeTy, IdTy, ExprTy>;
-using Weak = tomp::clause::WeakT<TypeTy, IdTy, ExprTy>;
-using When = tomp::clause::WhenT<TypeTy, IdTy, ExprTy>;
-using Write = tomp::clause::WriteT<TypeTy, IdTy, ExprTy>;
-} // namespace clause
-
-struct Helper {
-  std::optional<Object> getBaseObject(const Object &object) {
-    return std::nullopt;
-  }
-  std::optional<Object> getLoopIterVar() { return std::nullopt; }
-};
-
-using Clause = tomp::ClauseT<TypeTy, IdTy, ExprTy>;
-using ConstructDecomposition = tomp::ConstructDecompositionT<Clause, Helper>;
-using DirectiveWithClauses = tomp::DirectiveWithClauses<Clause>;
-} // namespace omp
-
-struct StringifyClause {
-  static std::string join(const omp::List<std::string> &Strings) {
-    std::stringstream Stream;
-    for (const auto &[Index, String] : llvm::enumerate(Strings)) {
-      if (Index != 0)
-        Stream << ", ";
-      Stream << String;
-    }
-    return Stream.str();
-  }
-
-  static std::string to_str(llvm::omp::Directive D) {
-    return getOpenMPDirectiveName(D).str();
-  }
-  static std::string to_str(llvm::omp::Clause C) {
-    return getOpenMPClauseName(C).str();
-  }
-  static std::string to_str(const omp::TypeTy &Type) { return "type"; }
-  static std::string to_str(const omp::ExprTy &Expr) { return "expr"; }
-  static std::string to_str(const omp::Object &Obj) { return Obj.id(); }
-
-  template <typename U>
-  static std::enable_if_t<std::is_enum_v<llvm::remove_cvref_t<U>>, std::string>
-  to_str(U &&Item) {
-    return std::to_string(llvm::to_underlying(Item));
-  }
-
-  template <typename U> static std::string to_str(const omp::List<U> &Items) {
-    omp::List<std::string> Names;
-    llvm::transform(Items, std::back_inserter(Names),
-                    [](auto &&S) { return to_str(S); });
-    return "(" + join(Names) + ")";
-  }
-
-  template <typename U>
-  static std::string to_str(const std::optional<U> &Item) {
-    if (Item)
-      return to_str(*Item);
-    return "";
-  }
-
-  template <typename... Us, size_t... Is>
-  static std::string to_str(const std::tuple<Us...> &Tuple,
-                            std::index_sequence<Is...>) {
-    omp::List<std::string> Strings;
-    (Strings.push_back(to_str(std::get<Is>(Tuple))), ...);
-    return "(" + join(Strings) + ")";
-  }
-
-  template <typename U>
-  static std::enable_if_t<llvm::remove_cvref_t<U>::EmptyTrait::value,
-                          std::string>
-  to_str(U &&Item) {
-    return "";
-  }
-
-  template <typename U>
-  static std::enable_if_t<llvm::remove_cvref_t<U>::IncompleteTrait::value,
-                          std::string>
-  to_str(U &&Item) {
-    return "";
-  }
-
-  template <typename U>
-  static std::enable_if_t<llvm::remove_cvref_t<U>::WrapperTrait::value,
-                          std::string>
-  to_str(U &&Item) {
-    // For a wrapper, stringify the wrappee, and only add parentheses if
-    // there aren't any already.
-    std::string Str = to_str(Item.v);
-    if (!Str.empty()) {
-      if (Str.front() == '(' && Str.back() == ')')
-        return Str;
-    }
-    return "(" + to_str(Item.v) + ")";
-  }
-
-  template <typename U>
-  static std::enable_if_t<llvm::remove_cvref_t<U>::TupleTrait::value,
-                          std::string>
-  to_str(U &&Item) {
-    constexpr size_t TupleSize =
-        std::tuple_size_v<llvm::remove_cvref_t<decltype(Item.t)>>;
-    return to_str(Item.t, std::make_index_sequence<TupleSize>{});
-  }
-
-  template <typename U>
-  static std::enable_if_t<llvm::remove_cvref_t<U>::UnionTrait::value,
-                          std::string>
-  to_str(U &&Item) {
-    return std::visit([](auto &&S) { return to_str(S); }, Item.u);
-  }
-
-  StringifyClause(const omp::Clause &C)
-      // Rely on content stringification to emit enclosing parentheses.
-      : Str(to_str(C.id) + to_str(C)) {}
-
-  std::string Str;
-};
-
-std::string stringify(const omp::DirectiveWithClauses &DWC) {
-  std::stringstream Stream;
-
-  Stream << getOpenMPDirectiveName(DWC.id).str();
-  for (const omp::Clause &C : DWC.clauses)
-    Stream << ' ' << StringifyClause(C).Str;
-
-  return Stream.str();
-}
-
-// --- Tests ----------------------------------------------------------
-
-namespace {
-using namespace llvm::omp;
-
-class OpenMPDecompositionTest : public testing::Test {
-protected:
-  void SetUp() override {}
-  void TearDown() override {}
-
-  omp::Helper Helper;
-  uint32_t AnyVersion = 999;
-};
-
-// PRIVATE
-// [5.2:111:5-7]
-// Directives: distribute, do, for, loop, parallel, scope, sections, simd,
-// single, target, task, taskloop, teams
-//
-// [5.2:340:1-2]
-// (1) The effect of the 1 private clause is as if it is applied only to the
-// innermost leaf construct that permits it.
-TEST_F(OpenMPDecompositionTest, Private1) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_private, omp::clause::Private{{x}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel");            // (1)
-  ASSERT_EQ(Dir1, "sections private(x)"); // (1)
-}
-
-TEST_F(OpenMPDecompositionTest, Private2) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_private, omp::clause::Private{{x}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_masked,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel private(x)"); // (1)
-  ASSERT_EQ(Dir1, "masked");              // (1)
-}
-
-// FIRSTPRIVATE
-// [5.2:112:5-7]
-// Directives: distribute, do, for, parallel, scope, sections, single, target,
-// task, taskloop, teams
-//
-// [5.2:340:3-20]
-// (3) The effect of the firstprivate clause is as if it is applied to one or
-// more leaf constructs as follows:
-//  (5) To the distribute construct if it is among the constituent constructs;
-//  (6) To the teams construct if it is among the constituent constructs and the
-//      distribute construct is not;
-//  (8) To a worksharing construct that accepts the clause if one is among the
-//      constituent constructs;
-//  (9) To the taskloop construct if it is among the constituent constructs;
-// (10) To the parallel construct if it is among the constituent constructs and
-//      neither a taskloop construct nor a worksharing construct that accepts
-//      the clause is among them;
-// (12) To the target construct if it is among the constituent constructs and
-//      the same list item neither appears in a lastprivate clause nor is the
-//      base variable or base pointer of a list item that appears in a map
-//      clause.
-//
-// (15) If the parallel construct is among the constituent constructs and the
-// effect is not as if the firstprivate clause is applied to it by the above
-// rules, then the effect is as if the shared clause with the same list item is
-// applied to the parallel construct.
-// (17) If the teams construct is among the constituent constructs and the
-// effect is not as if the firstprivate clause is applied to it by the above
-// rules, then the effect is as if the shared clause with the same list item is
-// applied to the teams construct.
-TEST_F(OpenMPDecompositionTest, Firstprivate1) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel shared(x)");       // (10), (15)
-  ASSERT_EQ(Dir1, "sections firstprivate(x)"); // (8)
-}
-
-TEST_F(OpenMPDecompositionTest, Firstprivate2) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper,
-                                  OMPD_target_teams_distribute, Clauses);
-  ASSERT_EQ(Dec.output.size(), 3u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target firstprivate(x)");     // (12)
-  ASSERT_EQ(Dir1, "teams shared(x)");            // (6), (17)
-  ASSERT_EQ(Dir2, "distribute firstprivate(x)"); // (5)
-}
-
-TEST_F(OpenMPDecompositionTest, Firstprivate3) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
-      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper,
-                                  OMPD_target_teams_distribute, Clauses);
-  ASSERT_EQ(Dec.output.size(), 3u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (12), (27)
-  ASSERT_EQ(Dir1, "teams shared(x)");          // (6), (17)
-  ASSERT_EQ(Dir2, "distribute firstprivate(x) lastprivate(, (x))"); // (5), (21)
-}
-
-TEST_F(OpenMPDecompositionTest, Firstprivate4) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_teams,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "target firstprivate(x)"); // (12)
-  ASSERT_EQ(Dir1, "teams firstprivate(x)");  // (6)
-}
-
-TEST_F(OpenMPDecompositionTest, Firstprivate5) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper,
-                                  OMPD_parallel_masked_taskloop, Clauses);
-  ASSERT_EQ(Dec.output.size(), 3u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "parallel shared(x)"); // (10)
-  ASSERT_EQ(Dir1, "masked");
-  ASSERT_EQ(Dir2, "taskloop firstprivate(x)"); // (9)
-}
-
-TEST_F(OpenMPDecompositionTest, Firstprivate6) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_masked,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel firstprivate(x)"); // (10)
-  ASSERT_EQ(Dir1, "masked");
-}
-
-TEST_F(OpenMPDecompositionTest, Firstprivate7) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
-  };
-
-  // Composite constructs are still decomposed.
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_teams_distribute,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "teams shared(x)");            // (17)
-  ASSERT_EQ(Dir1, "distribute firstprivate(x)"); // (5)
-}
-
-// LASTPRIVATE
-// [5.2:115:7-8]
-// Directives: distribute, do, for, loop, sections, simd, taskloop
-//
-// [5.2:340:21-30]
-// (21) The effect of the lastprivate clause is as if it is applied to all leaf
-// constructs that permit the clause.
-// (22) If the parallel construct is among the constituent constructs and the
-// list item is not also specified in the firstprivate clause, then the effect
-// of the lastprivate clause is as if the shared clause with the same list item
-// is applied to the parallel construct.
-// (24) If the teams construct is among the constituent constructs and the list
-// item is not also specified in the firstprivate clause, then the effect of the
-// lastprivate clause is as if the shared clause with the same list item is
-// applied to the teams construct.
-// (27) If the target construct is among the constituent constructs and the list
-// item is not the base variable or base pointer of a list item that appears in
-// a map clause, the effect of the lastprivate clause is as if the same list
-// item appears in a map clause with a map-type of tofrom.
-TEST_F(OpenMPDecompositionTest, Lastprivate1) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel shared(x)");          // (21), (22)
-  ASSERT_EQ(Dir1, "sections lastprivate(, (x))"); // (21)
-}
-
-TEST_F(OpenMPDecompositionTest, Lastprivate2) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_teams_distribute,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "teams shared(x)");               // (21), (25)
-  ASSERT_EQ(Dir1, "distribute lastprivate(, (x))"); // (21)
-}
-
-TEST_F(OpenMPDecompositionTest, Lastprivate3) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_parallel_do,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 3u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (21), (27)
-  ASSERT_EQ(Dir1, "parallel shared(x)");       // (22)
-  ASSERT_EQ(Dir2, "do lastprivate(, (x))");    // (21)
-}
-
-// SHARED
-// [5.2:110:5-6]
-// Directives: parallel, task, taskloop, teams
-//
-// [5.2:340:31-32]
-// (31) The effect of the shared, default, thread_limit, or order clause is as
-// if it is applied to all leaf constructs that permit the clause.
-TEST_F(OpenMPDecompositionTest, Shared1) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_shared, omp::clause::Shared{{x}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper,
-                                  OMPD_parallel_masked_taskloop, Clauses);
-  ASSERT_EQ(Dec.output.size(), 3u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "parallel shared(x)"); // (31)
-  ASSERT_EQ(Dir1, "masked");             // (31)
-  ASSERT_EQ(Dir2, "taskloop shared(x)"); // (31)
-}
-
-// DEFAULT
-// [5.2:109:5-6]
-// Directives: parallel, task, taskloop, teams
-//
-// [5.2:340:31-32]
-// (31) The effect of the shared, default, thread_limit, or order clause is as
-// if it is applied to all leaf constructs that permit the clause.
-TEST_F(OpenMPDecompositionTest, Default1) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_default,
-       omp::clause::Default{
-           omp::clause::Default::DataSharingAttribute::Firstprivate}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper,
-                                  OMPD_parallel_masked_taskloop, Clauses);
-  ASSERT_EQ(Dec.output.size(), 3u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "parallel default(0)"); // (31)
-  ASSERT_EQ(Dir1, "masked");              // (31)
-  ASSERT_EQ(Dir2, "taskloop default(0)"); // (31)
-}
-
-// THREAD_LIMIT
-// [5.2:277:14-15]
-// Directives: target, teams
-//
-// [5.2:340:31-32]
-// (31) The effect of the shared, default, thread_limit, or order clause is as
-// if it is applied to all leaf constructs that permit the clause.
-TEST_F(OpenMPDecompositionTest, ThreadLimit1) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_thread_limit, omp::clause::ThreadLimit{omp::ExprTy{}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper,
-                                  OMPD_target_teams_distribute, Clauses);
-  ASSERT_EQ(Dec.output.size(), 3u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target thread_limit(expr)"); // (31)
-  ASSERT_EQ(Dir1, "teams thread_limit(expr)");  // (31)
-  ASSERT_EQ(Dir2, "distribute");                // (31)
-}
-
-// ORDER
-// [5.2:234:3-4]
-// Directives: distribute, do, for, loop, simd
-//
-// [5.2:340:31-32]
-// (31) The effect of the shared, default, thread_limit, or order clause is as
-// if it is applied to all leaf constructs that permit the clause.
-TEST_F(OpenMPDecompositionTest, Order1) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_order,
-       omp::clause::Order{{omp::clause::Order::OrderModifier::Unconstrained,
-                           omp::clause::Order::Ordering::Concurrent}}},
-  };
-
-  omp::ConstructDecomposition Dec(
-      AnyVersion, Helper, OMPD_target_teams_distribute_parallel_for_simd,
-      Clauses);
-  ASSERT_EQ(Dec.output.size(), 6u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  std::string Dir3 = stringify(Dec.output[3]);
-  std::string Dir4 = stringify(Dec.output[4]);
-  std::string Dir5 = stringify(Dec.output[5]);
-  ASSERT_EQ(Dir0, "target"); // (31)
-  ASSERT_EQ(Dir1, "teams");  // (31)
-  // XXX OMP.td doesn't list "order" as allowed for "distribute"
-  ASSERT_EQ(Dir2, "distribute");       // (31)
-  ASSERT_EQ(Dir3, "parallel");         // (31)
-  ASSERT_EQ(Dir4, "for order(1, 0)");  // (31)
-  ASSERT_EQ(Dir5, "simd order(1, 0)"); // (31)
-}
-
-// ALLOCATE
-// [5.2:178:7-9]
-// Directives: allocators, distribute, do, for, parallel, scope, sections,
-// single, target, task, taskgroup, taskloop, teams
-//
-// [5.2:340:33-35]
-// (33) The effect of the allocate clause is as if it is applied to all leaf
-// constructs that permit the clause and to which a data-sharing attribute
-// clause that may create a private copy of the same list item is applied.
-TEST_F(OpenMPDecompositionTest, Allocate1) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_allocate,
-       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
-      {OMPC_private, omp::clause::Private{{x}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel");                                // (33)
-  ASSERT_EQ(Dir1, "sections private(x) allocate(, , , (x))"); // (33)
-}
-
-// REDUCTION
-// [5.2:134:17-18]
-// Directives: do, for, loop, parallel, scope, sections, simd, taskloop, teams
-//
-// [5.2:340-341:36-13]
-// (36) The effect of the reduction clause is as if it is applied to all leaf
-// constructs that permit the clause, except for the following constructs:
-//  (1) The parallel construct, when combined with the sections,
-//      worksharing-loop, loop, or taskloop construct; and
-//  (3) The teams construct, when combined with the loop construct.
-// (4) For the parallel and teams constructs above, the effect of the reduction
-// clause instead is as if each list item or, for any list item that is an array
-// item, its corresponding base array or base pointer appears in a shared clause
-// for the construct.
-// (6) If the task reduction-modifier is specified, the effect is as if it only
-// modifies the behavior of the reduction clause on the innermost leaf construct
-// that accepts the modifier (see Section 5.5.8).
-// (8) If the inscan reduction-modifier is specified, the effect is as if it
-// modifies the behavior of the reduction clause on all constructs of the
-// combined construct to which the clause is applied and that accept the
-// modifier.
-// (10) If a list item in a reduction clause on a combined target construct does
-// not have the same base variable or base pointer as a list item in a map
-// clause on the construct, then the effect is as if the list item in the
-// reduction clause appears as a list item in a map clause with a map-type of
-// tofrom.
-namespace red {
-// Make is easier to construct reduction operators from built-in intrinsics.
-omp::clause::ReductionOperator
-makeOp(omp::clause::DefinedOperator::IntrinsicOperator Op) {
-  return omp::clause::ReductionOperator{omp::clause::DefinedOperator{Op}};
-}
-} // namespace red
-
-TEST_F(OpenMPDecompositionTest, Reduction1) {
-  omp::Object x{"x"};
-  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel shared(x)");             // (36), (1), (4)
-  ASSERT_EQ(Dir1, "sections reduction(, (3), (x))"); // (36)
-}
-
-TEST_F(OpenMPDecompositionTest, Reduction2) {
-  omp::Object x{"x"};
-  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_masked,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "parallel reduction(, (3), (x))"); // (36), (1), (4)
-  ASSERT_EQ(Dir1, "masked");                         // (36)
-}
-
-TEST_F(OpenMPDecompositionTest, Reduction3) {
-  omp::Object x{"x"};
-  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_teams_loop, Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "teams shared(x)");            // (36), (3), (4)
-  ASSERT_EQ(Dir1, "loop reduction(, (3), (x))"); // (36)
-}
-
-TEST_F(OpenMPDecompositionTest, Reduction4) {
-  omp::Object x{"x"};
-  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper,
-                                  OMPD_teams_distribute_parallel_for, Clauses);
-  ASSERT_EQ(Dec.output.size(), 4u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  std::string Dir3 = stringify(Dec.output[3]);
-  ASSERT_EQ(Dir0, "teams reduction(, (3), (x))"); // (36), (3)
-  ASSERT_EQ(Dir1, "distribute");                  // (36)
-  ASSERT_EQ(Dir2, "parallel shared(x)");          // (36), (1), (4)
-  ASSERT_EQ(Dir3, "for reduction(, (3), (x))");   // (36)
-}
-
-TEST_F(OpenMPDecompositionTest, Reduction5) {
-  omp::Object x{"x"};
-  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
-  auto TaskMod = omp::clause::Reduction::ReductionModifier::Task;
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_reduction, omp::clause::Reduction{{TaskMod, {Add}, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper,
-                                  OMPD_teams_distribute_parallel_for, Clauses);
-  ASSERT_EQ(Dec.output.size(), 4u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  std::string Dir3 = stringify(Dec.output[3]);
-  ASSERT_EQ(Dir0, "teams reduction(, (3), (x))"); // (36), (3), (6)
-  ASSERT_EQ(Dir1, "distribute");                  // (36)
-  ASSERT_EQ(Dir2, "parallel shared(x)");          // (36), (1), (4)
-  ASSERT_EQ(Dir3, "for reduction(2, (3), (x))");  // (36), (6)
-}
-
-TEST_F(OpenMPDecompositionTest, Reduction6) {
-  omp::Object x{"x"};
-  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
-  auto InscanMod = omp::clause::Reduction::ReductionModifier::Inscan;
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_reduction, omp::clause::Reduction{{InscanMod, {Add}, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper,
-                                  OMPD_teams_distribute_parallel_for, Clauses);
-  ASSERT_EQ(Dec.output.size(), 4u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  std::string Dir3 = stringify(Dec.output[3]);
-  ASSERT_EQ(Dir0, "teams reduction(, (3), (x))"); // (36), (3), (8)
-  ASSERT_EQ(Dir1, "distribute");                  // (36)
-  ASSERT_EQ(Dir2, "parallel shared(x)");          // (36), (1), (4)
-  ASSERT_EQ(Dir3, "for reduction(1, (3), (x))");  // (36), (8)
-}
-
-TEST_F(OpenMPDecompositionTest, Reduction7) {
-  omp::Object x{"x"};
-  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_parallel_do,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 3u);
-
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  // XXX Currently OMP.td allows "reduction" on "target".
-  ASSERT_EQ(Dir0,
-            "target reduction(, (3), (x)) map(2, , , , (x))"); // (36), (10)
-  ASSERT_EQ(Dir1, "parallel shared(x)");                       // (36), (1), (4)
-  ASSERT_EQ(Dir2, "do reduction(, (3), (x))");                 // (36)
-}
-
-// IF
-// [5.2:72:7-9]
-// Directives: cancel, parallel, simd, target, target data, target enter data,
-// target exit data, target update, task, taskloop
-//
-// [5.2:72:15-18]
-// (15) For combined or composite constructs, the if clause only applies to the
-// semantics of the construct named in the directive-name-modifier.
-// (16) For a combined or composite construct, if no directive-name-modifier is
-// specified then the if clause applies to all constituent constructs to which
-// an if clause can apply.
-TEST_F(OpenMPDecompositionTest, If1) {
-  omp::List<omp::Clause> Clauses{
-      {OMPC_if,
-       omp::clause::If{{llvm::omp::Directive::OMPD_parallel, omp::ExprTy{}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper,
-                                  OMPD_target_parallel_for_simd, Clauses);
-  ASSERT_EQ(Dec.output.size(), 4u);
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  std::string Dir3 = stringify(Dec.output[3]);
-  ASSERT_EQ(Dir0, "target");              // (15)
-  ASSERT_EQ(Dir1, "parallel if(, expr)"); // (15)
-  ASSERT_EQ(Dir2, "for");                 // (15)
-  ASSERT_EQ(Dir3, "simd");                // (15)
-}
-
-TEST_F(OpenMPDecompositionTest, If2) {
-  omp::List<omp::Clause> Clauses{
-      {OMPC_if, omp::clause::If{{std::nullopt, omp::ExprTy{}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper,
-                                  OMPD_target_parallel_for_simd, Clauses);
-  ASSERT_EQ(Dec.output.size(), 4u);
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  std::string Dir3 = stringify(Dec.output[3]);
-  ASSERT_EQ(Dir0, "target if(, expr)");   // (16)
-  ASSERT_EQ(Dir1, "parallel if(, expr)"); // (16)
-  ASSERT_EQ(Dir2, "for");                 // (16)
-  ASSERT_EQ(Dir3, "simd if(, expr)");     // (16)
-}
-
-// LINEAR
-// [5.2:118:1-2]
-// Directives: declare simd, do, for, simd
-//
-// [5.2:341:15-22]
-// (15.1) The effect of the linear clause is as if it is applied to the
-// innermost leaf construct.
-// (15.2) Additionally, if the list item is not the iteration variable of a simd
-// or worksharing-loop SIMD construct, the effect on the outer leaf constructs
-// is as if the list item was specified in firstprivate and lastprivate clauses
-// on the combined or composite construct, with the rules specified above
-// applied.
-// (19) If a list item of the linear clause is the iteration variable of a simd
-// or worksharing-loop SIMD construct and it is not declared in the construct,
-// the effect on the outer leaf constructs is as if the list item was specified
-// in a lastprivate clause on the combined or composite construct with the rules
-// specified above applied.
-TEST_F(OpenMPDecompositionTest, Linear1) {
-  omp::Object x{"x"};
-
-  omp::List<omp::Clause> Clauses{
-      {OMPC_linear,
-       omp::clause::Linear{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_for_simd, Clauses);
-  ASSERT_EQ(Dec.output.size(), 2u);
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  ASSERT_EQ(Dir0, "for firstprivate(x) lastprivate(, (x))"); // (15.1), (15.2)
-  ASSERT_EQ(Dir1, "simd linear(, , , (x)) lastprivate(, (x))"); // (15.1)
-}
-
-// NOWAIT
-// [5.2:308:11-13]
-// Directives: dispatch, do, for, interop, scope, sections, single, target,
-// target enter data, target exit data, target update, taskwait, workshare
-//
-// [5.2:341:23]
-// (23) The effect of the nowait clause is as if it is applied to the outermost
-// leaf construct that permits it.
-TEST_F(OpenMPDecompositionTest, Nowait1) {
-  omp::List<omp::Clause> Clauses{
-      {OMPC_nowait, omp::clause::Nowait{}},
-  };
-
-  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_parallel_for,
-                                  Clauses);
-  ASSERT_EQ(Dec.output.size(), 3u);
-  std::string Dir0 = stringify(Dec.output[0]);
-  std::string Dir1 = stringify(Dec.output[1]);
-  std::string Dir2 = stringify(Dec.output[2]);
-  ASSERT_EQ(Dir0, "target nowait"); // (23)
-  ASSERT_EQ(Dir1, "parallel");      // (23)
-  ASSERT_EQ(Dir2, "for");           // (23)
-}
-} // namespace
-- 
GitLab


From 1a25b723628e439d62dfb28ca5fa52e4b2a78e5a Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Mon, 13 May 2024 17:47:03 +0400
Subject: [PATCH 0565/1206] [lldb] Fixed the test TestPlatformProcessLaunch
 running on a remote target (#91923)

Transfer `stdio.log` from the remote target if necessary.
---
 .../launch/TestPlatformProcessLaunch.py       | 90 ++++++++++---------
 1 file changed, 46 insertions(+), 44 deletions(-)

diff --git a/lldb/test/API/commands/platform/process/launch/TestPlatformProcessLaunch.py b/lldb/test/API/commands/platform/process/launch/TestPlatformProcessLaunch.py
index 3fb7d00c93d2..7cbad03eeeea 100644
--- a/lldb/test/API/commands/platform/process/launch/TestPlatformProcessLaunch.py
+++ b/lldb/test/API/commands/platform/process/launch/TestPlatformProcessLaunch.py
@@ -3,6 +3,7 @@ Test platform process launch.
 """
 
 from textwrap import dedent
+from lldbsuite.test import lldbutil
 from lldbsuite.test.lldbtest import TestBase
 
 
@@ -11,9 +12,10 @@ class ProcessLaunchTestCase(TestBase):
 
     def setup(self):
         self.build()
-        exe = self.getBuildArtifact("a.out")
-        self.runCmd("file " + exe)
-        return (exe, self.getBuildArtifact("stdio.log"))
+        self.runCmd("file " + self.getBuildArtifact("a.out"))
+        exe = lldbutil.append_to_process_working_directory(self, "a.out")
+        outfile = lldbutil.append_to_process_working_directory(self, "stdio.log")
+        return (exe, outfile)
 
     def test_process_launch_no_args(self):
         # When there are no extra arguments we just have 0, the program name.
@@ -21,18 +23,18 @@ class ProcessLaunchTestCase(TestBase):
         self.runCmd("platform process launch --stdout {} -s".format(outfile))
         self.runCmd("continue")
 
-        with open(outfile) as f:
-            self.assertEqual(
-                dedent(
-                    """\
-                Got 1 argument(s).
-                [0]: {}
-                """.format(
-                        exe
-                    )
-                ),
-                f.read(),
-            )
+        stdio_log = lldbutil.read_file_on_target(self, outfile)
+        self.assertEqual(
+            dedent(
+                """\
+            Got 1 argument(s).
+            [0]: {}
+            """.format(
+                    exe
+                )
+            ),
+            stdio_log,
+        )
 
     def test_process_launch_command_args(self):
         exe, outfile = self.setup()
@@ -41,21 +43,21 @@ class ProcessLaunchTestCase(TestBase):
         self.runCmd("platform process launch --stdout {} -s -- A B C".format(outfile))
         self.runCmd("continue")
 
-        with open(outfile) as f:
-            self.assertEqual(
-                dedent(
-                    """\
-                Got 4 argument(s).
-                [0]: {}
-                [1]: A
-                [2]: B
-                [3]: C
-                """.format(
-                        exe
-                    )
-                ),
-                f.read(),
-            )
+        stdio_log = lldbutil.read_file_on_target(self, outfile)
+        self.assertEqual(
+            dedent(
+                """\
+            Got 4 argument(s).
+            [0]: {}
+            [1]: A
+            [2]: B
+            [3]: C
+            """.format(
+                    exe
+                )
+            ),
+            stdio_log,
+        )
 
     def test_process_launch_target_args(self):
         exe, outfile = self.setup()
@@ -64,17 +66,17 @@ class ProcessLaunchTestCase(TestBase):
         self.runCmd("platform process launch --stdout {} -s".format(outfile))
         self.runCmd("continue")
 
-        with open(outfile) as f:
-            self.assertEqual(
-                dedent(
-                    """\
-                Got 3 argument(s).
-                [0]: {}
-                [1]: D
-                [2]: E
-                """.format(
-                        exe
-                    )
-                ),
-                f.read(),
-            )
+        stdio_log = lldbutil.read_file_on_target(self, outfile)
+        self.assertEqual(
+            dedent(
+                """\
+            Got 3 argument(s).
+            [0]: {}
+            [1]: D
+            [2]: E
+            """.format(
+                    exe
+                )
+            ),
+            stdio_log,
+        )
-- 
GitLab


From 13cd88108f00fc97bcd1e4eb7cc9e4e388928677 Mon Sep 17 00:00:00 2001
From: Pranav Bhandarkar <pranav.bhandarkar@amd.com>
Date: Mon, 13 May 2024 08:54:23 -0500
Subject: [PATCH 0566/1206] [mlir][OpenMP] - Honor dependencies in
 code-generation of the if clause in `omp.task` correctly (#90891)

This patch fixes the code generation of the if clause, specifically when the
condition evaluates to false and when the task directive has the depend
clause on it. When the if clause of a task construct evaluates to false,
then the task is an undeferred task. This undeferred task still has to
honor dependencies. Previously, the OpenMPIRbuilder didn't honor
dependencies. This patch fixes that.

Fixes https://github.com/llvm/llvm-project/issues/90869
---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 13 +++++++++++++
 mlir/test/Target/LLVMIR/omptask_if_false.mlir | 17 +++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 mlir/test/Target/LLVMIR/omptask_if_false.mlir

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 42ea20919a5e..391a4947877a 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -1870,6 +1870,9 @@ OpenMPIRBuilder::createTask(const LocationDescription &Loc,
     //    call @__kmpc_omp_task(...)
     //    br label %exit
     //  else:
+    //    ;; Wait for resolution of dependencies, if any, before
+    //    ;; beginning the task
+    //    call @__kmpc_omp_wait_deps(...)
     //    call @__kmpc_omp_task_begin_if0(...)
     //    call @outlined_fn(...)
     //    call @__kmpc_omp_task_complete_if0(...)
@@ -1887,6 +1890,16 @@ OpenMPIRBuilder::createTask(const LocationDescription &Loc,
       SplitBlockAndInsertIfThenElse(IfCondition, IfTerminator, &ThenTI,
                                     &ElseTI);
       Builder.SetInsertPoint(ElseTI);
+
+      if (Dependencies.size()) {
+        Function *TaskWaitFn =
+            getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_wait_deps);
+        Builder.CreateCall(
+            TaskWaitFn,
+            {Ident, ThreadID, Builder.getInt32(Dependencies.size()), DepArray,
+             ConstantInt::get(Builder.getInt32Ty(), 0),
+             ConstantPointerNull::get(PointerType::getUnqual(M.getContext()))});
+      }
       Function *TaskBeginFn =
           getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_begin_if0);
       Function *TaskCompleteFn =
diff --git a/mlir/test/Target/LLVMIR/omptask_if_false.mlir b/mlir/test/Target/LLVMIR/omptask_if_false.mlir
new file mode 100644
index 000000000000..c6014a76add6
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptask_if_false.mlir
@@ -0,0 +1,17 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+llvm.func @foo_(%arg0: !llvm.ptr {fir.bindc_name = "n"}, %arg1: !llvm.ptr {fir.bindc_name = "r"}) attributes {fir.internal_name = "_QPfoo"} {
+  %0 = llvm.mlir.constant(false) : i1
+  omp.task if(%0) depend(taskdependin -> %arg0 : !llvm.ptr) {
+    %1 = llvm.load %arg0 : !llvm.ptr -> i32
+    llvm.store %1, %arg1 : i32, !llvm.ptr
+    omp.terminator
+  }
+  llvm.return
+}
+
+// CHECK: call void @__kmpc_omp_wait_deps
+// CHECK-NEXT: call void @__kmpc_omp_task_begin_if0
+// CHECK-NEXT: call void @foo_..omp_par
+// CHECK-NEXT: call void @__kmpc_omp_task_complete_if0
+
-- 
GitLab


From 4445ed4244cd00981a04b1f461128ed4c47c1dec Mon Sep 17 00:00:00 2001
From: Oleg Shyshkov <shyshkov@google.com>
Date: Mon, 13 May 2024 15:57:57 +0200
Subject: [PATCH 0567/1206] [OpenMP][MLIR] Fix llvm::sort comparator. (#91963)

The current comparator doesn't work correctly when two identical entries
with -1 are compared. The comparator returns `first` is case when
`aIndex == -1 && bIndex == -1`, but it should `continue` as those
indexes are the same.
---
 .../LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp  | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 282e640d3aaa..a7294632d666 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -2155,18 +2155,15 @@ getFirstOrLastMappedMemberPtr(mlir::omp::MapInfoOp mapInfo, bool first) {
           int aIndex = indexValues[a * shape[1] + i];
           int bIndex = indexValues[b * shape[1] + i];
 
+          if (aIndex == bIndex)
+            continue;
+
           if (aIndex != -1 && bIndex == -1)
             return false;
 
           if (aIndex == -1 && bIndex != -1)
             return true;
 
-          if (aIndex == -1)
-            return first;
-
-          if (bIndex == -1)
-            return !first;
-
           // A is earlier in the record type layout than B
           if (aIndex < bIndex)
             return first;
-- 
GitLab


From c4e9e41199127bb288e84e9477da99f28941edb3 Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Mon, 13 May 2024 16:04:20 +0200
Subject: [PATCH 0568/1206] [Clang] Ensure ``if consteval`` consititute an
 immediate function context (#91939)

We did not set the correct evaluation context for the compound statement
of an ``if consteval`` statement
in a templated entity in TreeTransform.

Fixes #91509
---
 clang/docs/ReleaseNotes.rst                   |  1 +
 clang/lib/Sema/TreeTransform.h                | 10 +++++++
 .../SemaCXX/cxx2b-consteval-propagate.cpp     | 26 +++++++++++++++++++
 3 files changed, 37 insertions(+)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 7c5dcc59c701..4702b8c10cdb 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -707,6 +707,7 @@ Bug Fixes to C++ Support
   initialized, rather than evaluating them as a part of the larger manifestly constant evaluated
   expression.
 - Fix a bug in access control checking due to dealyed checking of friend declaration. Fixes (#GH12361).
+- Correctly treat the compound statement of an ``if consteval`` as an immediate context. Fixes (#GH91509).
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 0b3cf566e3a7..126965088831 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -7964,6 +7964,11 @@ TreeTransform<Derived>::TransformIfStmt(IfStmt *S) {
   // Transform the "then" branch.
   StmtResult Then;
   if (!ConstexprConditionValue || *ConstexprConditionValue) {
+    EnterExpressionEvaluationContext Ctx(
+        getSema(), Sema::ExpressionEvaluationContext::ImmediateFunctionContext,
+        nullptr, Sema::ExpressionEvaluationContextRecord::EK_Other,
+        S->isNonNegatedConsteval());
+
     Then = getDerived().TransformStmt(S->getThen());
     if (Then.isInvalid())
       return StmtError();
@@ -7978,6 +7983,11 @@ TreeTransform<Derived>::TransformIfStmt(IfStmt *S) {
   // Transform the "else" branch.
   StmtResult Else;
   if (!ConstexprConditionValue || !*ConstexprConditionValue) {
+    EnterExpressionEvaluationContext Ctx(
+        getSema(), Sema::ExpressionEvaluationContext::ImmediateFunctionContext,
+        nullptr, Sema::ExpressionEvaluationContextRecord::EK_Other,
+        S->isNegatedConsteval());
+
     Else = getDerived().TransformStmt(S->getElse());
     if (Else.isInvalid())
       return StmtError();
diff --git a/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp b/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
index 37fa1f1bdf59..07937deb6673 100644
--- a/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
+++ b/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
@@ -420,3 +420,29 @@ int f = *fn().value + fn2();  // expected-error {{call to consteval function 'lv
                               // expected-note {{pointer to heap-allocated object}}
 }
 #endif
+
+
+#if __cplusplus >= 202302L
+
+namespace GH91509 {
+
+consteval int f(int) { return 0; }
+
+template<typename T>
+constexpr int g(int x) {
+    if consteval {
+        return f(x);
+    }
+    if !consteval {}
+    else {
+        return f(x);
+    }
+    return 1;
+}
+
+int h(int x) {
+    return g<void>(x);
+}
+}
+
+#endif
-- 
GitLab


From b0b6c16b470a7d5d9c63765058cca0ebe07ad57d Mon Sep 17 00:00:00 2001
From: Michael Kruse <llvm-project@meinersbur.de>
Date: Mon, 13 May 2024 16:10:58 +0200
Subject: [PATCH 0569/1206] [Clang][OpenMP][Tile] Allow non-constant tile
 sizes. (#91345)

Allow non-constants in the `sizes` clause such as
```
#pragma omp tile sizes(a)
for (int i = 0; i < n; ++i)
```
This is permitted since tile was introduced in [OpenMP
5.1](https://www.openmp.org/spec-html/5.1/openmpsu53.html#x78-860002.11.9).

It is possible to sneak-in negative numbers at runtime as in
```
int a = -1;
#pragma omp tile sizes(a)
```
Even though it is not well-formed, it should still result in every loop
iteration to be executed exactly once, an invariant of the tile
construct that we should ensure. `ParseOpenMPExprListClause` is
extracted-out to be reused by the `permutation` clause of the
`interchange` construct. Some care was put into ensuring correct behavior
in template contexts.
---
 clang/include/clang/Parse/Parser.h            |  17 ++
 clang/lib/Parse/ParseOpenMP.cpp               |  65 ++++--
 clang/lib/Sema/SemaOpenMP.cpp                 | 118 ++++++++--
 clang/test/OpenMP/tile_ast_print.cpp          |  17 ++
 clang/test/OpenMP/tile_codegen.cpp            | 216 ++++++++++++++++--
 clang/test/OpenMP/tile_messages.cpp           |  50 +++-
 openmp/runtime/test/transform/tile/intfor.c   | 191 ++++++++++++++++
 .../test/transform/tile/negtile_intfor.c      |  44 ++++
 .../tile/parallel-wsloop-collapse-intfor.cpp  | 100 ++++++++
 9 files changed, 748 insertions(+), 70 deletions(-)
 create mode 100644 openmp/runtime/test/transform/tile/intfor.c
 create mode 100644 openmp/runtime/test/transform/tile/negtile_intfor.c
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-intfor.cpp

diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 60d59732269b..61589fb7766f 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3553,6 +3553,23 @@ private:
   OMPClause *ParseOpenMPVarListClause(OpenMPDirectiveKind DKind,
                                       OpenMPClauseKind Kind, bool ParseOnly);
 
+  /// Parses a clause consisting of a list of expressions.
+  ///
+  /// \param Kind          The clause to parse.
+  /// \param ClauseNameLoc [out] The location of the clause name.
+  /// \param OpenLoc       [out] The location of '('.
+  /// \param CloseLoc      [out] The location of ')'.
+  /// \param Exprs         [out] The parsed expressions.
+  /// \param ReqIntConst   If true, each expression must be an integer constant.
+  ///
+  /// \return Whether the clause was parsed successfully.
+  bool ParseOpenMPExprListClause(OpenMPClauseKind Kind,
+                                 SourceLocation &ClauseNameLoc,
+                                 SourceLocation &OpenLoc,
+                                 SourceLocation &CloseLoc,
+                                 SmallVectorImpl<Expr *> &Exprs,
+                                 bool ReqIntConst = false);
+
   /// Parses and creates OpenMP 5.0 iterators expression:
   /// <iterators> = 'iterator' '(' { [ <iterator-type> ] identifier =
   /// <range-specification> }+ ')'
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 53eabe0c662e..03a64600bc2a 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3107,34 +3107,14 @@ bool Parser::ParseOpenMPSimpleVarList(
 }
 
 OMPClause *Parser::ParseOpenMPSizesClause() {
-  SourceLocation ClauseNameLoc = ConsumeToken();
+  SourceLocation ClauseNameLoc, OpenLoc, CloseLoc;
   SmallVector<Expr *, 4> ValExprs;
-
-  BalancedDelimiterTracker T(*this, tok::l_paren, tok::annot_pragma_openmp_end);
-  if (T.consumeOpen()) {
-    Diag(Tok, diag::err_expected) << tok::l_paren;
+  if (ParseOpenMPExprListClause(OMPC_sizes, ClauseNameLoc, OpenLoc, CloseLoc,
+                                ValExprs))
     return nullptr;
-  }
-
-  while (true) {
-    ExprResult Val = ParseConstantExpression();
-    if (!Val.isUsable()) {
-      T.skipToEnd();
-      return nullptr;
-    }
-
-    ValExprs.push_back(Val.get());
-
-    if (Tok.is(tok::r_paren) || Tok.is(tok::annot_pragma_openmp_end))
-      break;
-
-    ExpectAndConsume(tok::comma);
-  }
-
-  T.consumeClose();
 
-  return Actions.OpenMP().ActOnOpenMPSizesClause(
-      ValExprs, ClauseNameLoc, T.getOpenLocation(), T.getCloseLocation());
+  return Actions.OpenMP().ActOnOpenMPSizesClause(ValExprs, ClauseNameLoc,
+                                                 OpenLoc, CloseLoc);
 }
 
 OMPClause *Parser::ParseOpenMPUsesAllocatorClause(OpenMPDirectiveKind DKind) {
@@ -4991,3 +4971,38 @@ OMPClause *Parser::ParseOpenMPVarListClause(OpenMPDirectiveKind DKind,
   OMPVarListLocTy Locs(Loc, LOpen, Data.RLoc);
   return Actions.OpenMP().ActOnOpenMPVarListClause(Kind, Vars, Locs, Data);
 }
+
+bool Parser::ParseOpenMPExprListClause(OpenMPClauseKind Kind,
+                                       SourceLocation &ClauseNameLoc,
+                                       SourceLocation &OpenLoc,
+                                       SourceLocation &CloseLoc,
+                                       SmallVectorImpl<Expr *> &Exprs,
+                                       bool ReqIntConst) {
+  assert(getOpenMPClauseName(Kind) == PP.getSpelling(Tok) &&
+         "Expected parsing to start at clause name");
+  ClauseNameLoc = ConsumeToken();
+
+  // Parse inside of '(' and ')'.
+  BalancedDelimiterTracker T(*this, tok::l_paren, tok::annot_pragma_openmp_end);
+  if (T.consumeOpen()) {
+    Diag(Tok, diag::err_expected) << tok::l_paren;
+    return true;
+  }
+
+  // Parse the list with interleaved commas.
+  do {
+    ExprResult Val =
+        ReqIntConst ? ParseConstantExpression() : ParseAssignmentExpression();
+    if (!Val.isUsable()) {
+      // Encountered something other than an expression; abort to ')'.
+      T.skipToEnd();
+      return true;
+    }
+    Exprs.push_back(Val.get());
+  } while (TryConsumeToken(tok::comma));
+
+  bool Result = T.consumeClose();
+  OpenLoc = T.getOpenLocation();
+  CloseLoc = T.getCloseLocation();
+  return Result;
+}
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 2475f962fd0d..7d00cf6fb5b6 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15111,13 +15111,11 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   ASTContext &Context = getASTContext();
   Scope *CurScope = SemaRef.getCurScope();
 
-  auto SizesClauses =
-      OMPExecutableDirective::getClausesOfKind<OMPSizesClause>(Clauses);
-  if (SizesClauses.empty()) {
-    // A missing 'sizes' clause is already reported by the parser.
+  const auto *SizesClause =
+      OMPExecutableDirective::getSingleClause<OMPSizesClause>(Clauses);
+  if (!SizesClause ||
+      llvm::any_of(SizesClause->getSizesRefs(), [](Expr *E) { return !E; }))
     return StmtError();
-  }
-  const OMPSizesClause *SizesClause = *SizesClauses.begin();
   unsigned NumLoops = SizesClause->getNumSizes();
 
   // Empty statement should only be possible if there already was an error.
@@ -15138,6 +15136,13 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     return OMPTileDirective::Create(Context, StartLoc, EndLoc, Clauses,
                                     NumLoops, AStmt, nullptr, nullptr);
 
+  assert(LoopHelpers.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+
   SmallVector<Decl *, 4> PreInits;
   CaptureVars CopyTransformer(SemaRef);
 
@@ -15197,6 +15202,44 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // Once the original iteration values are set, append the innermost body.
   Stmt *Inner = Body;
 
+  auto MakeDimTileSize = [&SemaRef = this->SemaRef, &CopyTransformer, &Context,
+                          SizesClause, CurScope](int I) -> Expr * {
+    Expr *DimTileSizeExpr = SizesClause->getSizesRefs()[I];
+    if (isa<ConstantExpr>(DimTileSizeExpr))
+      return AssertSuccess(CopyTransformer.TransformExpr(DimTileSizeExpr));
+
+    // When the tile size is not a constant but a variable, it is possible to
+    // pass non-positive numbers. For instance:
+    // \code{c}
+    //   int a = 0;
+    //   #pragma omp tile sizes(a)
+    //   for (int i = 0; i < 42; ++i)
+    //     body(i);
+    // \endcode
+    // Although there is no meaningful interpretation of the tile size, the body
+    // should still be executed 42 times to avoid surprises. To preserve the
+    // invariant that every loop iteration is executed exactly once and not
+    // cause an infinite loop, apply a minimum tile size of one.
+    // Build expr:
+    // \code{c}
+    //   (TS <= 0) ? 1 : TS
+    // \endcode
+    QualType DimTy = DimTileSizeExpr->getType();
+    uint64_t DimWidth = Context.getTypeSize(DimTy);
+    IntegerLiteral *Zero = IntegerLiteral::Create(
+        Context, llvm::APInt::getZero(DimWidth), DimTy, {});
+    IntegerLiteral *One =
+        IntegerLiteral::Create(Context, llvm::APInt(DimWidth, 1), DimTy, {});
+    Expr *Cond = AssertSuccess(SemaRef.BuildBinOp(
+        CurScope, {}, BO_LE,
+        AssertSuccess(CopyTransformer.TransformExpr(DimTileSizeExpr)), Zero));
+    Expr *MinOne = new (Context) ConditionalOperator(
+        Cond, {}, One, {},
+        AssertSuccess(CopyTransformer.TransformExpr(DimTileSizeExpr)), DimTy,
+        VK_PRValue, OK_Ordinary);
+    return MinOne;
+  };
+
   // Create tile loops from the inside to the outside.
   for (int I = NumLoops - 1; I >= 0; --I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
@@ -15207,10 +15250,6 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // Commonly used variables. One of the constraints of an AST is that every
     // node object must appear at most once, hence we define lamdas that create
     // a new AST node at every use.
-    auto MakeDimTileSize = [&CopyTransformer, I, SizesClause]() -> Expr * {
-      Expr *DimTileSize = SizesClause->getSizesRefs()[I];
-      return AssertSuccess(CopyTransformer.TransformExpr(DimTileSize));
-    };
     auto MakeTileIVRef = [&SemaRef = this->SemaRef, &TileIndVars, I, CntTy,
                           OrigCntVar]() {
       return buildDeclRefExpr(SemaRef, TileIndVars[I], CntTy,
@@ -15237,7 +15276,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     //   .tile.iv < min(.floor.iv + DimTileSize, NumIterations)
     ExprResult EndOfTile =
         SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
-                           MakeFloorIVRef(), MakeDimTileSize());
+                           MakeFloorIVRef(), MakeDimTileSize(I));
     if (!EndOfTile.isUsable())
       return StmtError();
     ExprResult IsPartialTile =
@@ -15297,10 +15336,6 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     QualType CntTy = OrigCntVar->getType();
 
     // Commonly used variables.
-    auto MakeDimTileSize = [&CopyTransformer, I, SizesClause]() -> Expr * {
-      Expr *DimTileSize = SizesClause->getSizesRefs()[I];
-      return AssertSuccess(CopyTransformer.TransformExpr(DimTileSize));
-    };
     auto MakeFloorIVRef = [&SemaRef = this->SemaRef, &FloorIndVars, I, CntTy,
                            OrigCntVar]() {
       return buildDeclRefExpr(SemaRef, FloorIndVars[I], CntTy,
@@ -15329,7 +15364,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // For incr-statement: .floor.iv += DimTileSize
     ExprResult IncrStmt =
         SemaRef.BuildBinOp(CurScope, LoopHelper.Inc->getExprLoc(), BO_AddAssign,
-                           MakeFloorIVRef(), MakeDimTileSize());
+                           MakeFloorIVRef(), MakeDimTileSize(I));
     if (!IncrStmt.isUsable())
       return StmtError();
 
@@ -17430,16 +17465,53 @@ OMPClause *SemaOpenMP::ActOnOpenMPSizesClause(ArrayRef<Expr *> SizeExprs,
                                               SourceLocation StartLoc,
                                               SourceLocation LParenLoc,
                                               SourceLocation EndLoc) {
-  for (Expr *SizeExpr : SizeExprs) {
-    ExprResult NumForLoopsResult = VerifyPositiveIntegerConstantInClause(
-        SizeExpr, OMPC_sizes, /*StrictlyPositive=*/true);
-    if (!NumForLoopsResult.isUsable())
-      return nullptr;
+  SmallVector<Expr *> SanitizedSizeExprs(SizeExprs);
+
+  for (Expr *&SizeExpr : SanitizedSizeExprs) {
+    // Skip if already sanitized, e.g. during a partial template instantiation.
+    if (!SizeExpr)
+      continue;
+
+    bool IsValid = isNonNegativeIntegerValue(SizeExpr, SemaRef, OMPC_sizes,
+                                             /*StrictlyPositive=*/true);
+
+    // isNonNegativeIntegerValue returns true for non-integral types (but still
+    // emits error diagnostic), so check for the expected type explicitly.
+    QualType SizeTy = SizeExpr->getType();
+    if (!SizeTy->isIntegerType())
+      IsValid = false;
+
+    // Handling in templates is tricky. There are four possibilities to
+    // consider:
+    //
+    // 1a. The expression is valid and we are in a instantiated template or not
+    //     in a template:
+    //       Pass valid expression to be further analysed later in Sema.
+    // 1b. The expression is valid and we are in a template (including partial
+    //     instantiation):
+    //       isNonNegativeIntegerValue skipped any checks so there is no
+    //       guarantee it will be correct after instantiation.
+    //       ActOnOpenMPSizesClause will be called again at instantiation when
+    //       it is not in a dependent context anymore. This may cause warnings
+    //       to be emitted multiple times.
+    // 2a. The expression is invalid and we are in an instantiated template or
+    //     not in a template:
+    //       Invalidate the expression with a clearly wrong value (nullptr) so
+    //       later in Sema we do not have to do the same validity analysis again
+    //       or crash from unexpected data. Error diagnostics have already been
+    //       emitted.
+    // 2b. The expression is invalid and we are in a template (including partial
+    //     instantiation):
+    //       Pass the invalid expression as-is, template instantiation may
+    //       replace unexpected types/values with valid ones. The directives
+    //       with this clause must not try to use these expressions in dependent
+    //       contexts, but delay analysis until full instantiation.
+    if (!SizeExpr->isInstantiationDependent() && !IsValid)
+      SizeExpr = nullptr;
   }
 
-  DSAStack->setAssociatedLoops(SizeExprs.size());
   return OMPSizesClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
-                                SizeExprs);
+                                SanitizedSizeExprs);
 }
 
 OMPClause *SemaOpenMP::ActOnOpenMPFullClause(SourceLocation StartLoc,
diff --git a/clang/test/OpenMP/tile_ast_print.cpp b/clang/test/OpenMP/tile_ast_print.cpp
index afc8b34911e3..c4dff2c4be44 100644
--- a/clang/test/OpenMP/tile_ast_print.cpp
+++ b/clang/test/OpenMP/tile_ast_print.cpp
@@ -183,4 +183,21 @@ void tfoo7() {
 }
 
 
+// PRINT-LABEL: void foo8(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo8
+void foo8(int a) {
+  // PRINT:     #pragma omp tile sizes(a)
+  // DUMP:      OMPTileDirective
+  // DUMP-NEXT:   OMPSizesClause
+  // DUMP-NEXT:     ImplicitCastExpr
+  // DUMP-NEXT:       DeclRefExpr {{.*}} 'a'
+  #pragma omp tile sizes(a)
+  // PRINT-NEXT: for (int i = 7; i < 19; i += 3)
+  // DUMP-NEXT: ForStmt
+  for (int i = 7; i < 19; i += 3)
+    // PRINT: body(i);
+    // DUMP:  CallExpr
+    body(i);
+}
+
 #endif
diff --git a/clang/test/OpenMP/tile_codegen.cpp b/clang/test/OpenMP/tile_codegen.cpp
index 76cf2d8f1992..93a3a14133ab 100644
--- a/clang/test/OpenMP/tile_codegen.cpp
+++ b/clang/test/OpenMP/tile_codegen.cpp
@@ -83,6 +83,14 @@ extern "C" void tfoo7() {
   foo7<int,3,5>(0, 42);
 }
 
+
+extern "C" void foo8(int a) {
+#pragma omp tile sizes(a)
+  for (int i = 7; i < 17; i += 3)
+    body(i);
+}
+
+
 #endif /* HEADER */
 // CHECK1-LABEL: define {{[^@]+}}@body
 // CHECK1-SAME: (...) #[[ATTR0:[0-9]+]] {
@@ -98,7 +106,7 @@ extern "C" void tfoo7() {
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_ZN1SC1Ev
-// CHECK1-SAME: (ptr noundef nonnull align 4 dereferenceable(4) [[THIS:%.*]]) unnamed_addr #[[ATTR2:[0-9]+]] comdat align 2 {
+// CHECK1-SAME: (ptr noundef nonnull align 4 dereferenceable(4) [[THIS:%.*]]) unnamed_addr #[[ATTR0]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
@@ -108,7 +116,7 @@ extern "C" void tfoo7() {
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@_ZN1SC2Ev
-// CHECK1-SAME: (ptr noundef nonnull align 4 dereferenceable(4) [[THIS:%.*]]) unnamed_addr #[[ATTR2]] comdat align 2 {
+// CHECK1-SAME: (ptr noundef nonnull align 4 dereferenceable(4) [[THIS:%.*]]) unnamed_addr #[[ATTR0]] comdat align 2 {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[I:%.*]] = alloca ptr, align 8
@@ -885,7 +893,7 @@ extern "C" void tfoo7() {
 //
 //
 // CHECK1-LABEL: define {{[^@]+}}@foo6.omp_outlined
-// CHECK1-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]]) #[[ATTR5:[0-9]+]] {
+// CHECK1-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]]) #[[ATTR4:[0-9]+]] {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
@@ -1071,6 +1079,95 @@ extern "C" void tfoo7() {
 // CHECK1-NEXT:    ret void
 //
 //
+// CHECK1-LABEL: define {{[^@]+}}@foo8
+// CHECK1-SAME: (i32 noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK1-NEXT:  entry:
+// CHECK1-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK1:       for.cond:
+// CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
+// CHECK1:       for.cond1:
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK1:       cond.true4:
+// CHECK1-NEXT:    br label [[COND_END12:%.*]]
+// CHECK1:       cond.false5:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// CHECK1-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
+// CHECK1:       cond.true7:
+// CHECK1-NEXT:    br label [[COND_END9:%.*]]
+// CHECK1:       cond.false8:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END9]]
+// CHECK1:       cond.end9:
+// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// CHECK1-NEXT:    br label [[COND_END12]]
+// CHECK1:       cond.end12:
+// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
+// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// CHECK1-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body15:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
+// CHECK1-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK1:       for.inc:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP23:![0-9]+]]
+// CHECK1:       for.end:
+// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK1:       for.inc17:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// CHECK1-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
+// CHECK1:       cond.true19:
+// CHECK1-NEXT:    br label [[COND_END21:%.*]]
+// CHECK1:       cond.false20:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END21]]
+// CHECK1:       cond.end21:
+// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
+// CHECK1:       for.end24:
+// CHECK1-NEXT:    ret void
+//
+//
 // CHECK1-LABEL: define {{[^@]+}}@_GLOBAL__sub_I_tile_codegen.cpp
 // CHECK1-SAME: () #[[ATTR1]] section ".text.startup" {
 // CHECK1-NEXT:  entry:
@@ -1159,13 +1256,13 @@ extern "C" void tfoo7() {
 //
 //
 // CHECK2-LABEL: define {{[^@]+}}@body
-// CHECK2-SAME: (...) #[[ATTR2:[0-9]+]] {
+// CHECK2-SAME: (...) #[[ATTR1]] {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    ret void
 //
 //
 // CHECK2-LABEL: define {{[^@]+}}@foo1
-// CHECK2-SAME: (i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR2]] {
+// CHECK2-SAME: (i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR1]] {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
@@ -1255,7 +1352,7 @@ extern "C" void tfoo7() {
 //
 //
 // CHECK2-LABEL: define {{[^@]+}}@foo2
-// CHECK2-SAME: (i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR2]] {
+// CHECK2-SAME: (i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR1]] {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
@@ -1368,7 +1465,7 @@ extern "C" void tfoo7() {
 //
 //
 // CHECK2-LABEL: define {{[^@]+}}@foo3
-// CHECK2-SAME: () #[[ATTR2]] {
+// CHECK2-SAME: () #[[ATTR1]] {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
@@ -1510,7 +1607,7 @@ extern "C" void tfoo7() {
 //
 //
 // CHECK2-LABEL: define {{[^@]+}}@foo4
-// CHECK2-SAME: () #[[ATTR2]] {
+// CHECK2-SAME: () #[[ATTR1]] {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
@@ -1663,7 +1760,7 @@ extern "C" void tfoo7() {
 //
 //
 // CHECK2-LABEL: define {{[^@]+}}@foo5
-// CHECK2-SAME: () #[[ATTR2]] {
+// CHECK2-SAME: () #[[ATTR1]] {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
@@ -1872,14 +1969,14 @@ extern "C" void tfoo7() {
 //
 //
 // CHECK2-LABEL: define {{[^@]+}}@foo6
-// CHECK2-SAME: () #[[ATTR2]] {
+// CHECK2-SAME: () #[[ATTR1]] {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 0, ptr @foo6.omp_outlined)
 // CHECK2-NEXT:    ret void
 //
 //
 // CHECK2-LABEL: define {{[^@]+}}@foo6.omp_outlined
-// CHECK2-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]]) #[[ATTR5:[0-9]+]] {
+// CHECK2-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]]) #[[ATTR4:[0-9]+]] {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
@@ -1974,15 +2071,104 @@ extern "C" void tfoo7() {
 // CHECK2-NEXT:    ret void
 //
 //
+// CHECK2-LABEL: define {{[^@]+}}@foo8
+// CHECK2-SAME: (i32 noundef [[A:%.*]]) #[[ATTR1]] {
+// CHECK2-NEXT:  entry:
+// CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK2:       for.cond:
+// CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
+// CHECK2:       for.cond1:
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK2:       cond.true4:
+// CHECK2-NEXT:    br label [[COND_END12:%.*]]
+// CHECK2:       cond.false5:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// CHECK2-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
+// CHECK2:       cond.true7:
+// CHECK2-NEXT:    br label [[COND_END9:%.*]]
+// CHECK2:       cond.false8:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END9]]
+// CHECK2:       cond.end9:
+// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// CHECK2-NEXT:    br label [[COND_END12]]
+// CHECK2:       cond.end12:
+// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
+// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// CHECK2-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body15:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
+// CHECK2-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK2:       for.inc:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP21:![0-9]+]]
+// CHECK2:       for.end:
+// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK2:       for.inc17:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// CHECK2-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
+// CHECK2:       cond.true19:
+// CHECK2-NEXT:    br label [[COND_END21:%.*]]
+// CHECK2:       cond.false20:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END21]]
+// CHECK2:       cond.end21:
+// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
+// CHECK2:       for.end24:
+// CHECK2-NEXT:    ret void
+//
+//
 // CHECK2-LABEL: define {{[^@]+}}@tfoo7
-// CHECK2-SAME: () #[[ATTR2]] {
+// CHECK2-SAME: () #[[ATTR1]] {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    call void @_Z4foo7IiTnT_Li3ETnS0_Li5EEvS0_S0_(i32 noundef 0, i32 noundef 42)
 // CHECK2-NEXT:    ret void
 //
 //
 // CHECK2-LABEL: define {{[^@]+}}@_Z4foo7IiTnT_Li3ETnS0_Li5EEvS0_S0_
-// CHECK2-SAME: (i32 noundef [[START:%.*]], i32 noundef [[END:%.*]]) #[[ATTR2]] comdat {
+// CHECK2-SAME: (i32 noundef [[START:%.*]], i32 noundef [[END:%.*]]) #[[ATTR1]] comdat {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
@@ -2053,14 +2239,14 @@ extern "C" void tfoo7() {
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP16]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP21:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP23:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
 // CHECK2:       for.inc15:
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP17]], 5
 // CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
 // CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
diff --git a/clang/test/OpenMP/tile_messages.cpp b/clang/test/OpenMP/tile_messages.cpp
index adeef617b75c..5268dfe97e0c 100644
--- a/clang/test/OpenMP/tile_messages.cpp
+++ b/clang/test/OpenMP/tile_messages.cpp
@@ -43,13 +43,7 @@ void func() {
 
   // expected-error@+1 {{argument to 'sizes' clause must be a strictly positive integer value}}
   #pragma omp tile sizes(0)
-    ;
-
-  // expected-error@+4 {{expression is not an integral constant expression}}
-  // expected-note@+3 {{read of non-const variable 'a' is not allowed in a constant expression}}
-  // expected-note@+1 {{declared here}}
-  int a;
-  #pragma omp tile sizes(a)
+  for (int i = 0; i < 7; ++i)
     ;
 
   // expected-warning@+2 {{extra tokens at the end of '#pragma omp tile' are ignored}}
@@ -124,4 +118,46 @@ void func() {
   #pragma omp tile sizes(5)
   for (int i = 0; i/3<7; ++i)
     ;
+
+  // expected-error@+2 {{expression must have integral or unscoped enumeration type, not 'struct S'}}
+  struct S{} s;
+  #pragma omp tile sizes(s)
+  for (int i = 0; i < 7; ++i)
+    ;
+}
+
+
+template <typename T>
+static void templated_func() {
+  // In a template context, but expression itself not instantiation-dependent
+
+  // expected-error@+1 {{argument to 'sizes' clause must be a strictly positive integer value}}
+  #pragma omp tile sizes(0)
+  for (int i = 0; i < 7; ++i)
+    ;
+}
+
+template <int S>
+static void templated_func_value_dependent() {
+  // expected-error@+1 {{argument to 'sizes' clause must be a strictly positive integer value}}
+  #pragma omp tile sizes(S)
+  for (int i = 0; i < 7; ++i)
+    ;
+}
+
+template <typename T>
+static void templated_func_type_dependent() {
+  constexpr T s = 0;
+  // expected-error@+1 {{argument to 'sizes' clause must be a strictly positive integer value}}
+  #pragma omp tile sizes(s)
+  for (int i = 0; i < 7; ++i)
+    ;
+}
+
+void template_inst() {
+  templated_func<int>();
+  // expected-note@+1 {{in instantiation of function template specialization 'templated_func_value_dependent<0>' requested here}}
+  templated_func_value_dependent<0>();
+  // expected-note@+1 {{in instantiation of function template specialization 'templated_func_type_dependent<int>' requested here}}
+  templated_func_type_dependent<int>();
 }
diff --git a/openmp/runtime/test/transform/tile/intfor.c b/openmp/runtime/test/transform/tile/intfor.c
new file mode 100644
index 000000000000..4a930eab6730
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/intfor.c
@@ -0,0 +1,191 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+// TODO: The OpenMP specification explicitly does not define when and how often
+// expressions in the clause are evaluated. Currently Clang evaluates it again
+// whenever needed, but function calls in clauses are not common. A better
+// implementation would evaluate it just once and reuse the result.
+static int tilesize(int i) {
+  printf("tilesize(%d)\n", i);
+  return 3;
+}
+
+int main() {
+  printf("do\n");
+#pragma omp tile sizes(tilesize(1), tilesize(2))
+  for (int i = 7; i < 19; i += 3)
+    for (int j = 7; j < 20; j += 3)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=7
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=10
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=13
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=10 j=7
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=10 j=10
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=10 j=13
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=13 j=7
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=13 j=10
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=13 j=13
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=16
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=19
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=10 j=16
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=10 j=19
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=13 j=16
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=13 j=19
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=16 j=7
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=16 j=10
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=16 j=13
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=16 j=16
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=16 j=19
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: done
\ No newline at end of file
diff --git a/openmp/runtime/test/transform/tile/negtile_intfor.c b/openmp/runtime/test/transform/tile/negtile_intfor.c
new file mode 100644
index 000000000000..8784d9e9fa61
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/negtile_intfor.c
@@ -0,0 +1,44 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int tilesize = -2;
+
+int main() {
+  printf("do\n");
+#pragma omp tile sizes(tilesize, tilesize)
+  for (int i = 7; i < 19; i += 3)
+    for (int j = 7; j < 20; j += 3)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=7 j=7
+// CHECK-NEXT: i=7 j=10
+// CHECK-NEXT: i=7 j=13
+// CHECK-NEXT: i=7 j=16
+// CHECK-NEXT: i=7 j=19
+// CHECK-NEXT: i=10 j=7
+// CHECK-NEXT: i=10 j=10
+// CHECK-NEXT: i=10 j=13
+// CHECK-NEXT: i=10 j=16
+// CHECK-NEXT: i=10 j=19
+// CHECK-NEXT: i=13 j=7
+// CHECK-NEXT: i=13 j=10
+// CHECK-NEXT: i=13 j=13
+// CHECK-NEXT: i=13 j=16
+// CHECK-NEXT: i=13 j=19
+// CHECK-NEXT: i=16 j=7
+// CHECK-NEXT: i=16 j=10
+// CHECK-NEXT: i=16 j=13
+// CHECK-NEXT: i=16 j=16
+// CHECK-NEXT: i=16 j=19
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-intfor.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-intfor.cpp
new file mode 100644
index 000000000000..f4c2af610768
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-intfor.cpp
@@ -0,0 +1,100 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+  for (int i = 0; i < 3; ++i)
+#pragma omp tile sizes(3, 3)
+    for (int j = 0; j < 4; ++j)
+      for (int k = 0; k < 5; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+
+// Full tile
+// CHECK-NEXT: i=0 j=0 k=0
+// CHECK-NEXT: i=0 j=0 k=1
+// CHECK-NEXT: i=0 j=0 k=2
+// CHECK-NEXT: i=0 j=1 k=0
+// CHECK-NEXT: i=0 j=1 k=1
+// CHECK-NEXT: i=0 j=1 k=2
+// CHECK-NEXT: i=0 j=2 k=0
+// CHECK-NEXT: i=0 j=2 k=1
+// CHECK-NEXT: i=0 j=2 k=2
+
+// Partial tile
+// CHECK-NEXT: i=0 j=0 k=3
+// CHECK-NEXT: i=0 j=0 k=4
+// CHECK-NEXT: i=0 j=1 k=3
+// CHECK-NEXT: i=0 j=1 k=4
+// CHECK-NEXT: i=0 j=2 k=3
+// CHECK-NEXT: i=0 j=2 k=4
+
+// Partial tile
+// CHECK-NEXT: i=0 j=3 k=0
+// CHECK-NEXT: i=0 j=3 k=1
+// CHECK-NEXT: i=0 j=3 k=2
+
+// Partial tile
+// CHECK-NEXT: i=0 j=3 k=3
+// CHECK-NEXT: i=0 j=3 k=4
+
+// Full tile
+// CHECK-NEXT: i=1 j=0 k=0
+// CHECK-NEXT: i=1 j=0 k=1
+// CHECK-NEXT: i=1 j=0 k=2
+// CHECK-NEXT: i=1 j=1 k=0
+// CHECK-NEXT: i=1 j=1 k=1
+// CHECK-NEXT: i=1 j=1 k=2
+// CHECK-NEXT: i=1 j=2 k=0
+// CHECK-NEXT: i=1 j=2 k=1
+// CHECK-NEXT: i=1 j=2 k=2
+
+// Partial tiles
+// CHECK-NEXT: i=1 j=0 k=3
+// CHECK-NEXT: i=1 j=0 k=4
+// CHECK-NEXT: i=1 j=1 k=3
+// CHECK-NEXT: i=1 j=1 k=4
+// CHECK-NEXT: i=1 j=2 k=3
+// CHECK-NEXT: i=1 j=2 k=4
+// CHECK-NEXT: i=1 j=3 k=0
+// CHECK-NEXT: i=1 j=3 k=1
+// CHECK-NEXT: i=1 j=3 k=2
+// CHECK-NEXT: i=1 j=3 k=3
+// CHECK-NEXT: i=1 j=3 k=4
+
+// Full tile
+// CHECK-NEXT: i=2 j=0 k=0
+// CHECK-NEXT: i=2 j=0 k=1
+// CHECK-NEXT: i=2 j=0 k=2
+// CHECK-NEXT: i=2 j=1 k=0
+// CHECK-NEXT: i=2 j=1 k=1
+// CHECK-NEXT: i=2 j=1 k=2
+// CHECK-NEXT: i=2 j=2 k=0
+// CHECK-NEXT: i=2 j=2 k=1
+// CHECK-NEXT: i=2 j=2 k=2
+
+// Partial tiles
+// CHECK-NEXT: i=2 j=0 k=3
+// CHECK-NEXT: i=2 j=0 k=4
+// CHECK-NEXT: i=2 j=1 k=3
+// CHECK-NEXT: i=2 j=1 k=4
+// CHECK-NEXT: i=2 j=2 k=3
+// CHECK-NEXT: i=2 j=2 k=4
+// CHECK-NEXT: i=2 j=3 k=0
+// CHECK-NEXT: i=2 j=3 k=1
+// CHECK-NEXT: i=2 j=3 k=2
+// CHECK-NEXT: i=2 j=3 k=3
+// CHECK-NEXT: i=2 j=3 k=4
+// CHECK-NEXT: done
-- 
GitLab


From 69e13125af2511abd59499272c88fcb6f19b9300 Mon Sep 17 00:00:00 2001
From: chuongg3 <chuong.goh@arm.com>
Date: Mon, 13 May 2024 15:16:11 +0100
Subject: [PATCH 0570/1206] [AArch64][GlobalISel] Select G_ICMP instruction
 through TableGen (#89932)

G_ICMP NE => XOR(G_ICMP EQ, -1) moved to Legalizer to allow for combines
if they come up in following passes.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |  46 +++
 .../GISel/AArch64InstructionSelector.cpp      | 174 +--------
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |  47 ++-
 .../AArch64/GISel/AArch64LegalizerInfo.h      |   2 +
 .../CodeGen/AArch64/GlobalISel/select-cmp.mir |   4 +-
 .../AArch64/GlobalISel/select-vector-icmp.mir | 338 ------------------
 6 files changed, 86 insertions(+), 525 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 17d96370c04a..bb32280fe51f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -5403,6 +5403,52 @@ def : Pat<(AArch64bsp (v4i32 V128:$Rd), V128:$Rn, V128:$Rm),
 def : Pat<(AArch64bsp (v2i64 V128:$Rd), V128:$Rn, V128:$Rm),
           (BSPv16i8 V128:$Rd, V128:$Rn, V128:$Rm)>;
 
+// The following SetCC patterns are used for GlobalISel only
+multiclass SelectSetCC<PatFrags InFrag, string INST> {
+  def : Pat<(v8i8 (InFrag (v8i8 V64:$Rn), (v8i8 V64:$Rm))),
+            (v8i8 (!cast<Instruction>(INST # v8i8) (v8i8 V64:$Rn), (v8i8 V64:$Rm)))>;
+  def : Pat<(v16i8 (InFrag (v16i8 V128:$Rn), (v16i8 V128:$Rm))),
+            (v16i8 (!cast<Instruction>(INST # v16i8) (v16i8 V128:$Rn), (v16i8 V128:$Rm)))>;
+  def : Pat<(v4i16 (InFrag (v4i16 V64:$Rn), (v4i16 V64:$Rm))),
+            (v4i16 (!cast<Instruction>(INST # v4i16) (v4i16 V64:$Rn), (v4i16 V64:$Rm)))>;
+  def : Pat<(v8i16 (InFrag (v8i16 V128:$Rn), (v8i16 V128:$Rm))),
+            (v8i16 (!cast<Instruction>(INST # v8i16) (v8i16 V128:$Rn), (v8i16 V128:$Rm)))>;
+  def : Pat<(v2i32 (InFrag (v2i32 V64:$Rn), (v2i32 V64:$Rm))),
+            (v2i32 (!cast<Instruction>(INST # v2i32) (v2i32 V64:$Rn), (v2i32 V64:$Rm)))>;
+  def : Pat<(v4i32 (InFrag (v4i32 V128:$Rn), (v4i32 V128:$Rm))),
+            (v4i32 (!cast<Instruction>(INST # v4i32) (v4i32 V128:$Rn), (v4i32 V128:$Rm)))>;
+  def : Pat<(v2i64 (InFrag (v2i64 V128:$Rn), (v2i64 V128:$Rm))),
+            (v2i64 (!cast<Instruction>(INST # v2i64) (v2i64 V128:$Rn), (v2i64 V128:$Rm)))>;
+}
+
+defm : SelectSetCC<seteq, "CMEQ">;
+defm : SelectSetCC<setgt, "CMGT">;
+defm : SelectSetCC<setge, "CMGE">;
+defm : SelectSetCC<setugt, "CMHI">;
+defm : SelectSetCC<setuge, "CMHS">;
+
+multiclass SelectSetCCSwapOperands<PatFrags InFrag, string INST> {
+  def : Pat<(v8i8 (InFrag (v8i8 V64:$Rn), (v8i8 V64:$Rm))),
+            (v8i8 (!cast<Instruction>(INST # v8i8) (v8i8 V64:$Rm), (v8i8 V64:$Rn)))>;
+  def : Pat<(v16i8 (InFrag (v16i8 V128:$Rn), (v16i8 V128:$Rm))),
+            (v16i8 (!cast<Instruction>(INST # v16i8) (v16i8 V128:$Rm), (v16i8 V128:$Rn)))>;
+  def : Pat<(v4i16 (InFrag (v4i16 V64:$Rn), (v4i16 V64:$Rm))),
+            (v4i16 (!cast<Instruction>(INST # v4i16) (v4i16 V64:$Rm), (v4i16 V64:$Rn)))>;
+  def : Pat<(v8i16 (InFrag (v8i16 V128:$Rn), (v8i16 V128:$Rm))),
+            (v8i16 (!cast<Instruction>(INST # v8i16) (v8i16 V128:$Rm), (v8i16 V128:$Rn)))>;
+  def : Pat<(v2i32 (InFrag (v2i32 V64:$Rn), (v2i32 V64:$Rm))),
+            (v2i32 (!cast<Instruction>(INST # v2i32) (v2i32 V64:$Rm), (v2i32 V64:$Rn)))>;
+  def : Pat<(v4i32 (InFrag (v4i32 V128:$Rn), (v4i32 V128:$Rm))),
+            (v4i32 (!cast<Instruction>(INST # v4i32) (v4i32 V128:$Rm), (v4i32 V128:$Rn)))>;
+  def : Pat<(v2i64 (InFrag (v2i64 V128:$Rn), (v2i64 V128:$Rm))),
+            (v2i64 (!cast<Instruction>(INST # v2i64) (v2i64 V128:$Rm), (v2i64 V128:$Rn)))>;
+}
+
+defm : SelectSetCCSwapOperands<setlt, "CMGT">;
+defm : SelectSetCCSwapOperands<setle, "CMGE">;
+defm : SelectSetCCSwapOperands<setult, "CMHI">;
+defm : SelectSetCCSwapOperands<setule, "CMHS">;
+
 let Predicates = [HasNEON] in {
 def : InstAlias<"mov{\t$dst.16b, $src.16b|.16b\t$dst, $src}",
                 (ORRv16i8 V128:$dst, V128:$src, V128:$src), 1>;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 61f5bc2464ee..1b65ae7b4782 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -221,7 +221,6 @@ private:
   bool selectIntrinsicWithSideEffects(MachineInstr &I,
                                       MachineRegisterInfo &MRI);
   bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI);
-  bool selectVectorICmp(MachineInstr &I, MachineRegisterInfo &MRI);
   bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI);
   bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI);
   bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI);
@@ -3403,7 +3402,7 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
   }
   case TargetOpcode::G_ICMP: {
     if (Ty.isVector())
-      return selectVectorICmp(I, MRI);
+      return false;
 
     if (Ty != LLT::scalar(32)) {
       LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty
@@ -3652,177 +3651,6 @@ bool AArch64InstructionSelector::selectTLSGlobalValue(
   return true;
 }
 
-bool AArch64InstructionSelector::selectVectorICmp(
-    MachineInstr &I, MachineRegisterInfo &MRI) {
-  Register DstReg = I.getOperand(0).getReg();
-  LLT DstTy = MRI.getType(DstReg);
-  Register SrcReg = I.getOperand(2).getReg();
-  Register Src2Reg = I.getOperand(3).getReg();
-  LLT SrcTy = MRI.getType(SrcReg);
-
-  unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
-  unsigned NumElts = DstTy.getNumElements();
-
-  // First index is element size, 0 == 8b, 1 == 16b, 2 == 32b, 3 == 64b
-  // Second index is num elts, 0 == v2, 1 == v4, 2 == v8, 3 == v16
-  // Third index is cc opcode:
-  // 0 == eq
-  // 1 == ugt
-  // 2 == uge
-  // 3 == ult
-  // 4 == ule
-  // 5 == sgt
-  // 6 == sge
-  // 7 == slt
-  // 8 == sle
-  // ne is done by negating 'eq' result.
-
-  // This table below assumes that for some comparisons the operands will be
-  // commuted.
-  // ult op == commute + ugt op
-  // ule op == commute + uge op
-  // slt op == commute + sgt op
-  // sle op == commute + sge op
-  unsigned PredIdx = 0;
-  bool SwapOperands = false;
-  CmpInst::Predicate Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate();
-  switch (Pred) {
-  case CmpInst::ICMP_NE:
-  case CmpInst::ICMP_EQ:
-    PredIdx = 0;
-    break;
-  case CmpInst::ICMP_UGT:
-    PredIdx = 1;
-    break;
-  case CmpInst::ICMP_UGE:
-    PredIdx = 2;
-    break;
-  case CmpInst::ICMP_ULT:
-    PredIdx = 3;
-    SwapOperands = true;
-    break;
-  case CmpInst::ICMP_ULE:
-    PredIdx = 4;
-    SwapOperands = true;
-    break;
-  case CmpInst::ICMP_SGT:
-    PredIdx = 5;
-    break;
-  case CmpInst::ICMP_SGE:
-    PredIdx = 6;
-    break;
-  case CmpInst::ICMP_SLT:
-    PredIdx = 7;
-    SwapOperands = true;
-    break;
-  case CmpInst::ICMP_SLE:
-    PredIdx = 8;
-    SwapOperands = true;
-    break;
-  default:
-    llvm_unreachable("Unhandled icmp predicate");
-    return false;
-  }
-
-  // This table obviously should be tablegen'd when we have our GISel native
-  // tablegen selector.
-
-  static const unsigned OpcTable[4][4][9] = {
-      {
-          {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */},
-          {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */},
-          {AArch64::CMEQv8i8, AArch64::CMHIv8i8, AArch64::CMHSv8i8,
-           AArch64::CMHIv8i8, AArch64::CMHSv8i8, AArch64::CMGTv8i8,
-           AArch64::CMGEv8i8, AArch64::CMGTv8i8, AArch64::CMGEv8i8},
-          {AArch64::CMEQv16i8, AArch64::CMHIv16i8, AArch64::CMHSv16i8,
-           AArch64::CMHIv16i8, AArch64::CMHSv16i8, AArch64::CMGTv16i8,
-           AArch64::CMGEv16i8, AArch64::CMGTv16i8, AArch64::CMGEv16i8}
-      },
-      {
-          {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */},
-          {AArch64::CMEQv4i16, AArch64::CMHIv4i16, AArch64::CMHSv4i16,
-           AArch64::CMHIv4i16, AArch64::CMHSv4i16, AArch64::CMGTv4i16,
-           AArch64::CMGEv4i16, AArch64::CMGTv4i16, AArch64::CMGEv4i16},
-          {AArch64::CMEQv8i16, AArch64::CMHIv8i16, AArch64::CMHSv8i16,
-           AArch64::CMHIv8i16, AArch64::CMHSv8i16, AArch64::CMGTv8i16,
-           AArch64::CMGEv8i16, AArch64::CMGTv8i16, AArch64::CMGEv8i16},
-          {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */}
-      },
-      {
-          {AArch64::CMEQv2i32, AArch64::CMHIv2i32, AArch64::CMHSv2i32,
-           AArch64::CMHIv2i32, AArch64::CMHSv2i32, AArch64::CMGTv2i32,
-           AArch64::CMGEv2i32, AArch64::CMGTv2i32, AArch64::CMGEv2i32},
-          {AArch64::CMEQv4i32, AArch64::CMHIv4i32, AArch64::CMHSv4i32,
-           AArch64::CMHIv4i32, AArch64::CMHSv4i32, AArch64::CMGTv4i32,
-           AArch64::CMGEv4i32, AArch64::CMGTv4i32, AArch64::CMGEv4i32},
-          {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */},
-          {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */}
-      },
-      {
-          {AArch64::CMEQv2i64, AArch64::CMHIv2i64, AArch64::CMHSv2i64,
-           AArch64::CMHIv2i64, AArch64::CMHSv2i64, AArch64::CMGTv2i64,
-           AArch64::CMGEv2i64, AArch64::CMGTv2i64, AArch64::CMGEv2i64},
-          {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */},
-          {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */},
-          {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
-           0 /* invalid */}
-      },
-  };
-  unsigned EltIdx = Log2_32(SrcEltSize / 8);
-  unsigned NumEltsIdx = Log2_32(NumElts / 2);
-  unsigned Opc = OpcTable[EltIdx][NumEltsIdx][PredIdx];
-  if (!Opc) {
-    LLVM_DEBUG(dbgs() << "Could not map G_ICMP to cmp opcode");
-    return false;
-  }
-
-  const RegisterBank &VecRB = *RBI.getRegBank(SrcReg, MRI, TRI);
-  const TargetRegisterClass *SrcRC =
-      getRegClassForTypeOnBank(SrcTy, VecRB, true);
-  if (!SrcRC) {
-    LLVM_DEBUG(dbgs() << "Could not determine source register class.\n");
-    return false;
-  }
-
-  unsigned NotOpc = Pred == ICmpInst::ICMP_NE ? AArch64::NOTv8i8 : 0;
-  if (SrcTy.getSizeInBits() == 128)
-    NotOpc = NotOpc ? AArch64::NOTv16i8 : 0;
-
-  if (SwapOperands)
-    std::swap(SrcReg, Src2Reg);
-
-  auto Cmp = MIB.buildInstr(Opc, {SrcRC}, {SrcReg, Src2Reg});
-  constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
-
-  // Invert if we had a 'ne' cc.
-  if (NotOpc) {
-    Cmp = MIB.buildInstr(NotOpc, {DstReg}, {Cmp});
-    constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
-  } else {
-    MIB.buildCopy(DstReg, Cmp.getReg(0));
-  }
-  RBI.constrainGenericRegister(DstReg, *SrcRC, MRI);
-  I.eraseFromParent();
-  return true;
-}
-
 MachineInstr *AArch64InstructionSelector::emitScalarToVector(
     unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar,
     MachineIRBuilder &MIRBuilder) const {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index d4aac94d24f1..b8274f0f872c 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -495,17 +495,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
 
   // FIXME: fix moreElementsToNextPow2
   getActionDefinitionsBuilder(G_ICMP)
-      .legalFor({{s32, s32},
-                 {s32, s64},
-                 {s32, p0},
-                 {v4s32, v4s32},
-                 {v2s32, v2s32},
-                 {v2s64, v2s64},
-                 {v2s64, v2p0},
-                 {v4s16, v4s16},
-                 {v8s16, v8s16},
-                 {v8s8, v8s8},
-                 {v16s8, v16s8}})
+      .legalFor({{s32, s32}, {s32, s64}, {s32, p0}})
       .widenScalarOrEltToNextPow2(1)
       .clampScalar(1, s32, s64)
       .clampScalar(0, s32, s32)
@@ -527,7 +517,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .clampNumElements(1, v8s8, v16s8)
       .clampNumElements(1, v4s16, v8s16)
       .clampNumElements(1, v2s32, v4s32)
-      .clampNumElements(1, v2s64, v2s64);
+      .clampNumElements(1, v2s64, v2s64)
+      .customIf(isVector(0));
 
   getActionDefinitionsBuilder(G_FCMP)
       .legalFor({{s32, MinFPScalar},
@@ -1266,6 +1257,8 @@ bool AArch64LegalizerInfo::legalizeCustom(
     return legalizePrefetch(MI, Helper);
   case TargetOpcode::G_ABS:
     return Helper.lowerAbsToCNeg(MI);
+  case TargetOpcode::G_ICMP:
+    return legalizeICMP(MI, MRI, MIRBuilder);
   }
 
   llvm_unreachable("expected switch to return");
@@ -1324,6 +1317,36 @@ bool AArch64LegalizerInfo::legalizeFunnelShift(MachineInstr &MI,
   return true;
 }
 
+bool AArch64LegalizerInfo::legalizeICMP(MachineInstr &MI,
+                                        MachineRegisterInfo &MRI,
+                                        MachineIRBuilder &MIRBuilder) const {
+  Register DstReg = MI.getOperand(0).getReg();
+  Register SrcReg1 = MI.getOperand(2).getReg();
+  Register SrcReg2 = MI.getOperand(3).getReg();
+  LLT DstTy = MRI.getType(DstReg);
+  LLT SrcTy = MRI.getType(SrcReg1);
+
+  // Check the vector types are legal
+  if (DstTy.getScalarSizeInBits() != SrcTy.getScalarSizeInBits() ||
+      DstTy.getNumElements() != SrcTy.getNumElements() ||
+      (DstTy.getSizeInBits() != 64 && DstTy.getSizeInBits() != 128))
+    return false;
+
+  // Lowers G_ICMP NE => G_ICMP EQ to allow better pattern matching for
+  // following passes
+  CmpInst::Predicate Pred = (CmpInst::Predicate)MI.getOperand(1).getPredicate();
+  if (Pred != CmpInst::ICMP_NE)
+    return true;
+  Register CmpReg =
+      MIRBuilder
+          .buildICmp(CmpInst::ICMP_EQ, MRI.getType(DstReg), SrcReg1, SrcReg2)
+          .getReg(0);
+  MIRBuilder.buildNot(DstReg, CmpReg);
+
+  MI.eraseFromParent();
+  return true;
+}
+
 bool AArch64LegalizerInfo::legalizeRotate(MachineInstr &MI,
                                           MachineRegisterInfo &MRI,
                                           LegalizerHelper &Helper) const {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
index b69d9b015bd2..00d85a36e4b2 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
@@ -50,6 +50,8 @@ private:
                                LegalizerHelper &Helper) const;
   bool legalizeRotate(MachineInstr &MI, MachineRegisterInfo &MRI,
                       LegalizerHelper &Helper) const;
+  bool legalizeICMP(MachineInstr &MI, MachineRegisterInfo &MRI,
+                    MachineIRBuilder &MIRBuilder) const;
   bool legalizeFunnelShift(MachineInstr &MI, MachineRegisterInfo &MRI,
                            MachineIRBuilder &MIRBuilder,
                            GISelChangeObserver &Observer,
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-cmp.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-cmp.mir
index 4151f7ecb3ea..df4e7ddaac8b 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-cmp.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-cmp.mir
@@ -361,8 +361,8 @@ body:             |
     ; CHECK-NEXT: %cmp_lhs:fpr128 = COPY $q0
     ; CHECK-NEXT: %cmp_rhs:fpr128 = COPY $q1
     ; CHECK-NEXT: %add_lhs:fpr128 = COPY $q2
-    ; CHECK-NEXT: [[CMEQv4i32_:%[0-9]+]]:fpr128 = CMEQv4i32 %cmp_lhs, %cmp_rhs
-    ; CHECK-NEXT: %add:fpr128 = ADDv4i32 %add_lhs, [[CMEQv4i32_]]
+    ; CHECK-NEXT: %cmp:fpr128 = CMEQv4i32 %cmp_lhs, %cmp_rhs
+    ; CHECK-NEXT: %add:fpr128 = ADDv4i32 %add_lhs, %cmp
     ; CHECK-NEXT: $q0 = COPY %add
     ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %cmp_lhs:fpr(<4 x s32>) = COPY $q0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-icmp.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-icmp.mir
index 21e84ecaed32..7884d9e1b1d7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-icmp.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-icmp.mir
@@ -46,46 +46,6 @@
     ret <8 x i1> %cmp
   }
 
-  define <2 x i1> @test_v2i64_ne(<2 x i64> %v1, <2 x i64> %v2) {
-    %cmp = icmp ne <2 x i64> %v1, %v2
-    ret <2 x i1> %cmp
-  }
-
-  define <4 x i1> @test_v4i32_ne(<4 x i32> %v1, <4 x i32> %v2) {
-    %cmp = icmp ne <4 x i32> %v1, %v2
-    ret <4 x i1> %cmp
-  }
-
-  define <2 x i1> @test_v2i32_ne(<2 x i32> %v1, <2 x i32> %v2) {
-    %cmp = icmp ne <2 x i32> %v1, %v2
-    ret <2 x i1> %cmp
-  }
-
-  define <2 x i1> @test_v2i16_ne(<2 x i16> %v1, <2 x i16> %v2) {
-    %cmp = icmp ne <2 x i16> %v1, %v2
-    ret <2 x i1> %cmp
-  }
-
-  define <8 x i1> @test_v8i16_ne(<8 x i16> %v1, <8 x i16> %v2) {
-    %cmp = icmp ne <8 x i16> %v1, %v2
-    ret <8 x i1> %cmp
-  }
-
-  define <4 x i1> @test_v4i16_ne(<4 x i16> %v1, <4 x i16> %v2) {
-    %cmp = icmp ne <4 x i16> %v1, %v2
-    ret <4 x i1> %cmp
-  }
-
-  define <16 x i1> @test_v16i8_ne(<16 x i8> %v1, <16 x i8> %v2) {
-    %cmp = icmp ne <16 x i8> %v1, %v2
-    ret <16 x i1> %cmp
-  }
-
-  define <8 x i1> @test_v8i8_ne(<8 x i8> %v1, <8 x i8> %v2) {
-    %cmp = icmp ne <8 x i8> %v1, %v2
-    ret <8 x i1> %cmp
-  }
-
   define <2 x i1> @test_v2i64_ugt(<2 x i64> %v1, <2 x i64> %v2) {
     %cmp = icmp ugt <2 x i64> %v1, %v2
     ret <2 x i1> %cmp
@@ -696,304 +656,6 @@ body:             |
     $d0 = COPY %3(<8 x s8>)
     RET_ReallyLR implicit $d0
 
-...
----
-name:            test_v2i64_ne
-alignment:       4
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: fpr }
-  - { id: 1, class: fpr }
-  - { id: 2, class: _ }
-  - { id: 3, class: fpr }
-  - { id: 4, class: fpr }
-machineFunctionInfo: {}
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $q0, $q1
-
-    ; CHECK-LABEL: name: test_v2i64_ne
-    ; CHECK: liveins: $q0, $q1
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
-    ; CHECK-NEXT: [[CMEQv2i64_:%[0-9]+]]:fpr128 = CMEQv2i64 [[COPY]], [[COPY1]]
-    ; CHECK-NEXT: [[NOTv16i8_:%[0-9]+]]:fpr128 = NOTv16i8 [[CMEQv2i64_]]
-    ; CHECK-NEXT: [[XTNv2i32_:%[0-9]+]]:fpr64 = XTNv2i32 [[NOTv16i8_]]
-    ; CHECK-NEXT: $d0 = COPY [[XTNv2i32_]]
-    ; CHECK-NEXT: RET_ReallyLR implicit $d0
-    %0:fpr(<2 x s64>) = COPY $q0
-    %1:fpr(<2 x s64>) = COPY $q1
-    %4:fpr(<2 x s64>) = G_ICMP intpred(ne), %0(<2 x s64>), %1
-    %3:fpr(<2 x s32>) = G_TRUNC %4(<2 x s64>)
-    $d0 = COPY %3(<2 x s32>)
-    RET_ReallyLR implicit $d0
-
-...
----
-name:            test_v4i32_ne
-alignment:       4
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: fpr }
-  - { id: 1, class: fpr }
-  - { id: 2, class: _ }
-  - { id: 3, class: fpr }
-  - { id: 4, class: fpr }
-machineFunctionInfo: {}
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $q0, $q1
-
-    ; CHECK-LABEL: name: test_v4i32_ne
-    ; CHECK: liveins: $q0, $q1
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
-    ; CHECK-NEXT: [[CMEQv4i32_:%[0-9]+]]:fpr128 = CMEQv4i32 [[COPY]], [[COPY1]]
-    ; CHECK-NEXT: [[NOTv16i8_:%[0-9]+]]:fpr128 = NOTv16i8 [[CMEQv4i32_]]
-    ; CHECK-NEXT: [[XTNv4i16_:%[0-9]+]]:fpr64 = XTNv4i16 [[NOTv16i8_]]
-    ; CHECK-NEXT: $d0 = COPY [[XTNv4i16_]]
-    ; CHECK-NEXT: RET_ReallyLR implicit $d0
-    %0:fpr(<4 x s32>) = COPY $q0
-    %1:fpr(<4 x s32>) = COPY $q1
-    %4:fpr(<4 x s32>) = G_ICMP intpred(ne), %0(<4 x s32>), %1
-    %3:fpr(<4 x s16>) = G_TRUNC %4(<4 x s32>)
-    $d0 = COPY %3(<4 x s16>)
-    RET_ReallyLR implicit $d0
-
-...
----
-name:            test_v2i32_ne
-alignment:       4
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: fpr }
-  - { id: 1, class: fpr }
-  - { id: 2, class: _ }
-  - { id: 3, class: fpr }
-  - { id: 4, class: fpr }
-machineFunctionInfo: {}
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $d0, $d1
-
-    ; CHECK-LABEL: name: test_v2i32_ne
-    ; CHECK: liveins: $d0, $d1
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK-NEXT: [[CMEQv2i32_:%[0-9]+]]:fpr64 = CMEQv2i32 [[COPY]], [[COPY1]]
-    ; CHECK-NEXT: [[NOTv8i8_:%[0-9]+]]:fpr64 = NOTv8i8 [[CMEQv2i32_]]
-    ; CHECK-NEXT: $d0 = COPY [[NOTv8i8_]]
-    ; CHECK-NEXT: RET_ReallyLR implicit $d0
-    %0:fpr(<2 x s32>) = COPY $d0
-    %1:fpr(<2 x s32>) = COPY $d1
-    %4:fpr(<2 x s32>) = G_ICMP intpred(ne), %0(<2 x s32>), %1
-    %3:fpr(<2 x s32>) = COPY %4(<2 x s32>)
-    $d0 = COPY %3(<2 x s32>)
-    RET_ReallyLR implicit $d0
-
-...
----
-name:            test_v2i16_ne
-alignment:       4
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: fpr }
-  - { id: 3, class: fpr }
-  - { id: 4, class: _ }
-  - { id: 5, class: fpr }
-  - { id: 6, class: _ }
-  - { id: 7, class: fpr }
-  - { id: 8, class: fpr }
-  - { id: 9, class: fpr }
-  - { id: 10, class: gpr }
-  - { id: 11, class: fpr }
-  - { id: 12, class: fpr }
-  - { id: 13, class: gpr }
-  - { id: 14, class: fpr }
-  - { id: 15, class: fpr }
-machineFunctionInfo: {}
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $d0, $d1
-
-    ; CHECK-LABEL: name: test_v2i16_ne
-    ; CHECK: liveins: $d0, $d1
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK-NEXT: [[MOVID:%[0-9]+]]:fpr64 = MOVID 51
-    ; CHECK-NEXT: [[ANDv8i8_:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY]], [[MOVID]]
-    ; CHECK-NEXT: [[MOVID1:%[0-9]+]]:fpr64 = MOVID 51
-    ; CHECK-NEXT: [[ANDv8i8_1:%[0-9]+]]:fpr64 = ANDv8i8 [[COPY1]], [[MOVID1]]
-    ; CHECK-NEXT: [[CMEQv2i32_:%[0-9]+]]:fpr64 = CMEQv2i32 [[ANDv8i8_]], [[ANDv8i8_1]]
-    ; CHECK-NEXT: [[NOTv8i8_:%[0-9]+]]:fpr64 = NOTv8i8 [[CMEQv2i32_]]
-    ; CHECK-NEXT: $d0 = COPY [[NOTv8i8_]]
-    ; CHECK-NEXT: RET_ReallyLR implicit $d0
-    %2:fpr(<2 x s32>) = COPY $d0
-    %3:fpr(<2 x s32>) = COPY $d1
-    %13:gpr(s32) = G_CONSTANT i32 65535
-    %14:fpr(<2 x s32>) = G_BUILD_VECTOR %13(s32), %13(s32)
-    %15:fpr(<2 x s32>) = COPY %2(<2 x s32>)
-    %7:fpr(<2 x s32>) = G_AND %15, %14
-    %10:gpr(s32) = G_CONSTANT i32 65535
-    %11:fpr(<2 x s32>) = G_BUILD_VECTOR %10(s32), %10(s32)
-    %12:fpr(<2 x s32>) = COPY %3(<2 x s32>)
-    %8:fpr(<2 x s32>) = G_AND %12, %11
-    %9:fpr(<2 x s32>) = G_ICMP intpred(ne), %7(<2 x s32>), %8
-    %5:fpr(<2 x s32>) = COPY %9(<2 x s32>)
-    $d0 = COPY %5(<2 x s32>)
-    RET_ReallyLR implicit $d0
-
-...
----
-name:            test_v8i16_ne
-alignment:       4
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: fpr }
-  - { id: 1, class: fpr }
-  - { id: 2, class: _ }
-  - { id: 3, class: fpr }
-  - { id: 4, class: fpr }
-machineFunctionInfo: {}
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $q0, $q1
-
-    ; CHECK-LABEL: name: test_v8i16_ne
-    ; CHECK: liveins: $q0, $q1
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
-    ; CHECK-NEXT: [[CMEQv8i16_:%[0-9]+]]:fpr128 = CMEQv8i16 [[COPY]], [[COPY1]]
-    ; CHECK-NEXT: [[NOTv16i8_:%[0-9]+]]:fpr128 = NOTv16i8 [[CMEQv8i16_]]
-    ; CHECK-NEXT: [[XTNv8i8_:%[0-9]+]]:fpr64 = XTNv8i8 [[NOTv16i8_]]
-    ; CHECK-NEXT: $d0 = COPY [[XTNv8i8_]]
-    ; CHECK-NEXT: RET_ReallyLR implicit $d0
-    %0:fpr(<8 x s16>) = COPY $q0
-    %1:fpr(<8 x s16>) = COPY $q1
-    %4:fpr(<8 x s16>) = G_ICMP intpred(ne), %0(<8 x s16>), %1
-    %3:fpr(<8 x s8>) = G_TRUNC %4(<8 x s16>)
-    $d0 = COPY %3(<8 x s8>)
-    RET_ReallyLR implicit $d0
-
-...
----
-name:            test_v4i16_ne
-alignment:       4
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: fpr }
-  - { id: 1, class: fpr }
-  - { id: 2, class: _ }
-  - { id: 3, class: fpr }
-  - { id: 4, class: fpr }
-machineFunctionInfo: {}
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $d0, $d1
-
-    ; CHECK-LABEL: name: test_v4i16_ne
-    ; CHECK: liveins: $d0, $d1
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK-NEXT: [[CMEQv4i16_:%[0-9]+]]:fpr64 = CMEQv4i16 [[COPY]], [[COPY1]]
-    ; CHECK-NEXT: [[NOTv8i8_:%[0-9]+]]:fpr64 = NOTv8i8 [[CMEQv4i16_]]
-    ; CHECK-NEXT: $d0 = COPY [[NOTv8i8_]]
-    ; CHECK-NEXT: RET_ReallyLR implicit $d0
-    %0:fpr(<4 x s16>) = COPY $d0
-    %1:fpr(<4 x s16>) = COPY $d1
-    %4:fpr(<4 x s16>) = G_ICMP intpred(ne), %0(<4 x s16>), %1
-    %3:fpr(<4 x s16>) = COPY %4(<4 x s16>)
-    $d0 = COPY %3(<4 x s16>)
-    RET_ReallyLR implicit $d0
-
-...
----
-name:            test_v16i8_ne
-alignment:       4
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: fpr }
-  - { id: 1, class: fpr }
-  - { id: 2, class: _ }
-  - { id: 3, class: fpr }
-  - { id: 4, class: fpr }
-machineFunctionInfo: {}
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $q0, $q1
-
-    ; CHECK-LABEL: name: test_v16i8_ne
-    ; CHECK: liveins: $q0, $q1
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
-    ; CHECK-NEXT: [[CMEQv16i8_:%[0-9]+]]:fpr128 = CMEQv16i8 [[COPY]], [[COPY1]]
-    ; CHECK-NEXT: [[NOTv16i8_:%[0-9]+]]:fpr128 = NOTv16i8 [[CMEQv16i8_]]
-    ; CHECK-NEXT: $q0 = COPY [[NOTv16i8_]]
-    ; CHECK-NEXT: RET_ReallyLR implicit $q0
-    %0:fpr(<16 x s8>) = COPY $q0
-    %1:fpr(<16 x s8>) = COPY $q1
-    %4:fpr(<16 x s8>) = G_ICMP intpred(ne), %0(<16 x s8>), %1
-    %3:fpr(<16 x s8>) = COPY %4(<16 x s8>)
-    $q0 = COPY %3(<16 x s8>)
-    RET_ReallyLR implicit $q0
-
-...
----
-name:            test_v8i8_ne
-alignment:       4
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: fpr }
-  - { id: 1, class: fpr }
-  - { id: 2, class: _ }
-  - { id: 3, class: fpr }
-  - { id: 4, class: fpr }
-machineFunctionInfo: {}
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $d0, $d1
-
-    ; CHECK-LABEL: name: test_v8i8_ne
-    ; CHECK: liveins: $d0, $d1
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr64 = COPY $d1
-    ; CHECK-NEXT: [[CMEQv8i8_:%[0-9]+]]:fpr64 = CMEQv8i8 [[COPY]], [[COPY1]]
-    ; CHECK-NEXT: [[NOTv8i8_:%[0-9]+]]:fpr64 = NOTv8i8 [[CMEQv8i8_]]
-    ; CHECK-NEXT: $d0 = COPY [[NOTv8i8_]]
-    ; CHECK-NEXT: RET_ReallyLR implicit $d0
-    %0:fpr(<8 x s8>) = COPY $d0
-    %1:fpr(<8 x s8>) = COPY $d1
-    %4:fpr(<8 x s8>) = G_ICMP intpred(ne), %0(<8 x s8>), %1
-    %3:fpr(<8 x s8>) = COPY %4(<8 x s8>)
-    $d0 = COPY %3(<8 x s8>)
-    RET_ReallyLR implicit $d0
-
 ...
 ---
 name:            test_v2i64_ugt
-- 
GitLab


From ca051dfe9f0996e7fdad8fde5817e5b6ce758ab9 Mon Sep 17 00:00:00 2001
From: Ivan Butygin <ivan.butygin@gmail.com>
Date: Mon, 13 May 2024 17:24:22 +0300
Subject: [PATCH 0571/1206] [mlir][utils] Add script to verify
 canonicalizations against Alive2 (#91867)

This script takes IR before and after canonicalization, translates it
into llvm IR and converts it to format suitable for Alive2
https://alive2.llvm.org/ce/

This is primarily for arith canonicalizations verification, but
technically it can be adapted for any dialect translatable to llvm.

Usage `python verify_canon.py canonicalize.mlir -f func1 func2 ...`

Example output: https://alive2.llvm.org/ce/z/KhQs4J

Initial discussion:
https://github.com/llvm/llvm-project/pull/91646#pullrequestreview-2049342826
---
 mlir/utils/verify-canon/verify_canon.py | 77 +++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 mlir/utils/verify-canon/verify_canon.py

diff --git a/mlir/utils/verify-canon/verify_canon.py b/mlir/utils/verify-canon/verify_canon.py
new file mode 100644
index 000000000000..bfddba9577b9
--- /dev/null
+++ b/mlir/utils/verify-canon/verify_canon.py
@@ -0,0 +1,77 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# This script is a helper to verify canonicalization patterns using Alive2
+# https://alive2.llvm.org/ce/.
+# It performs the following steps:
+# - Filters out the provided test functions.
+# - Runs the canonicalization pass on the remaining functions.
+# - Lowers both the original and the canonicalized functions to LLVM IR.
+# - Prints the canonicalized and the original functions side-by-side in a format
+#   that can be copied into Alive2 for verification.
+# Example: `python verify_canon.py canonicalize.mlir -f func1 func2 func3`
+
+import subprocess
+import tempfile
+import sys
+from pathlib import Path
+from argparse import ArgumentParser
+
+
+def filter_funcs(ir, funcs):
+    if not funcs:
+        return ir
+
+    funcs_str = ",".join(funcs)
+    return subprocess.check_output(
+        ["mlir-opt", f"--symbol-privatize=exclude={funcs_str}", "--symbol-dce"],
+        input=ir,
+    )
+
+
+def add_func_prefix(src, prefix):
+    return src.replace("@", "@" + prefix)
+
+
+def merge_ir(chunks):
+    files = []
+    for chunk in chunks:
+        tmp = tempfile.NamedTemporaryFile(suffix=".ll")
+        tmp.write(chunk)
+        tmp.flush()
+        files.append(tmp)
+
+    return subprocess.check_output(["llvm-link", "-S"] + [f.name for f in files])
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("file")
+    parser.add_argument("-f", "--func-names", nargs="+", default=[])
+    args = parser.parse_args()
+
+    file = args.file
+    funcs = args.func_names
+
+    orig_ir = Path(file).read_bytes()
+    orig_ir = filter_funcs(orig_ir, funcs)
+
+    to_llvm_args = ["--convert-to-llvm"]
+    orig_args = ["mlir-opt"] + to_llvm_args
+    canon_args = ["mlir-opt", "-canonicalize"] + to_llvm_args
+    translate_args = ["mlir-translate", "-mlir-to-llvmir"]
+
+    orig = subprocess.check_output(orig_args, input=orig_ir)
+    canonicalized = subprocess.check_output(canon_args, input=orig_ir)
+
+    orig = subprocess.check_output(translate_args, input=orig)
+    canonicalized = subprocess.check_output(translate_args, input=canonicalized)
+
+    enc = "utf-8"
+    orig = bytes(add_func_prefix(orig.decode(enc), "src_"), enc)
+    canonicalized = bytes(add_func_prefix(canonicalized.decode(enc), "tgt_"), enc)
+
+    res = merge_ir([orig, canonicalized])
+
+    print(res.decode(enc))
-- 
GitLab


From 96ebed7c7481bb143c9d3db5f4c128bb32545229 Mon Sep 17 00:00:00 2001
From: JOSTAR <52376093+shenjunjiekoda@users.noreply.github.com>
Date: Mon, 13 May 2024 22:32:46 +0800
Subject: [PATCH 0572/1206] [analyzer][NFC] Move `CTUPhase1InliningMode` option
 to String analyzer options category (#91932)

The `CTUPhase1InliningMode`option was originally placed under Unsigned
analyzer options, but its value is a string.

This move aligns the option with its actual type.
---
 .../StaticAnalyzer/Core/AnalyzerOptions.def   | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
index 2fc825c2af9c..f008c9c581d9 100644
--- a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
+++ b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
@@ -413,22 +413,6 @@ ANALYZER_OPTION(
     "analysis is too low, it is meaningful to provide a minimum value that "
     "serves as an upper bound instead.", 10000)
 
-ANALYZER_OPTION(
-    StringRef, CTUPhase1InliningMode, "ctu-phase1-inlining",
-    "Controls which functions will be inlined during the first phase of the ctu "
-    "analysis. "
-    "If the value is set to 'all' then all foreign functions are inlinied "
-    "immediately during the first phase, thus rendering the second phase a noop. "
-    "The 'ctu-max-nodes-*' budge has no effect in this case. "
-    "If the value is 'small' then only functions with a linear CFG and with a "
-    "limited number of statements would be inlined during the first phase. The "
-    "long and/or nontrivial functions are handled in the second phase and are "
-    "controlled by the 'ctu-max-nodes-*' budge. "
-    "The value 'none' means that all foreign functions are inlined only in the "
-    "second phase, 'ctu-max-nodes-*' budge limits the second phase. "
-    "Value: \"none\", \"small\", \"all\".",
-    "small")
-
 ANALYZER_OPTION(
     unsigned, RegionStoreSmallStructLimit, "region-store-small-struct-limit",
     "The largest number of fields a struct can have and still be considered "
@@ -478,6 +462,22 @@ ANALYZER_OPTION(
     "where to look for those alternative implementations (called models).",
     "")
 
+ANALYZER_OPTION(
+    StringRef, CTUPhase1InliningMode, "ctu-phase1-inlining",
+    "Controls which functions will be inlined during the first phase of the ctu "
+    "analysis. "
+    "If the value is set to 'all' then all foreign functions are inlinied "
+    "immediately during the first phase, thus rendering the second phase a noop. "
+    "The 'ctu-max-nodes-*' budge has no effect in this case. "
+    "If the value is 'small' then only functions with a linear CFG and with a "
+    "limited number of statements would be inlined during the first phase. The "
+    "long and/or nontrivial functions are handled in the second phase and are "
+    "controlled by the 'ctu-max-nodes-*' budge. "
+    "The value 'none' means that all foreign functions are inlined only in the "
+    "second phase, 'ctu-max-nodes-*' budge limits the second phase. "
+    "Value: \"none\", \"small\", \"all\".",
+    "small")
+
 ANALYZER_OPTION(
     StringRef, CXXMemberInliningMode, "c++-inlining",
     "Controls which C++ member functions will be considered for inlining. "
-- 
GitLab


From 3acc6919109d91c1f047a02229a1785b461259d9 Mon Sep 17 00:00:00 2001
From: jyu2-git <jennifer.yu@intel.com>
Date: Mon, 13 May 2024 07:39:23 -0700
Subject: [PATCH 0573/1206] =?UTF-8?q?Revert=20"Revert=20"[OpenMP][TR12]=20?=
 =?UTF-8?q?change=20property=20of=20map-type=20modifier."=E2=80=A6=20(#918?=
 =?UTF-8?q?21)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… (#90885)"

This reverts commit eea81aa29848361eb5b24f24d2af643fdeb9adfd.
---
 .../clang/Basic/DiagnosticParseKinds.td       |   5 +
 clang/lib/Parse/ParseOpenMP.cpp               |  51 +++++++--
 clang/test/OpenMP/target_ast_print.cpp        |  58 ++++++++++
 clang/test/OpenMP/target_map_messages.cpp     | 105 ++++++++++--------
 4 files changed, 165 insertions(+), 54 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index bc9d7cacc50b..8316845844cb 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1438,6 +1438,9 @@ def err_omp_decl_in_declare_simd_variant : Error<
 def err_omp_sink_and_source_iteration_not_allowd: Error<" '%0 %select{sink:|source:}1' must be with '%select{omp_cur_iteration - 1|omp_cur_iteration}1'">;
 def err_omp_unknown_map_type : Error<
   "incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'">;
+def err_omp_more_one_map_type : Error<"map type is already specified">;
+def note_previous_map_type_specified_here
+    : Note<"map type '%0' is previous specified here">;
 def err_omp_unknown_map_type_modifier : Error<
   "incorrect map type modifier, expected one of: 'always', 'close', 'mapper'"
   "%select{|, 'present'|, 'present', 'iterator'}0%select{|, 'ompx_hold'}1">;
@@ -1445,6 +1448,8 @@ def err_omp_map_type_missing : Error<
   "missing map type">;
 def err_omp_map_type_modifier_missing : Error<
   "missing map type modifier">;
+def err_omp_map_modifier_specification_list : Error<
+  "empty modifier-specification-list is not allowed">;
 def err_omp_declare_simd_inbranch_notinbranch : Error<
   "unexpected '%0' clause, '%1' is specified already">;
 def err_omp_expected_clause_argument
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 03a64600bc2a..ca2c6d69eb98 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -4208,13 +4208,20 @@ bool Parser::parseMapperModifier(SemaOpenMP::OpenMPVarListDataTy &Data) {
   return T.consumeClose();
 }
 
+static OpenMPMapClauseKind isMapType(Parser &P);
+
 /// Parse map-type-modifiers in map clause.
-/// map([ [map-type-modifier[,] [map-type-modifier[,] ...] map-type : ] list)
+/// map([ [map-type-modifier[,] [map-type-modifier[,] ...] [map-type] : ] list)
 /// where, map-type-modifier ::= always | close | mapper(mapper-identifier) |
 /// present
+/// where, map-type ::= alloc | delete | from | release | to | tofrom
 bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
+  bool HasMapType = false;
+  SourceLocation PreMapLoc = Tok.getLocation();
+  StringRef PreMapName = "";
   while (getCurToken().isNot(tok::colon)) {
     OpenMPMapModifierKind TypeModifier = isMapModifier(*this);
+    OpenMPMapClauseKind MapKind = isMapType(*this);
     if (TypeModifier == OMPC_MAP_MODIFIER_always ||
         TypeModifier == OMPC_MAP_MODIFIER_close ||
         TypeModifier == OMPC_MAP_MODIFIER_present ||
@@ -4237,6 +4244,19 @@ bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
         Diag(Data.MapTypeModifiersLoc.back(), diag::err_omp_missing_comma)
             << "map type modifier";
 
+    } else if (getLangOpts().OpenMP >= 60 && MapKind != OMPC_MAP_unknown) {
+      if (!HasMapType) {
+        HasMapType = true;
+        Data.ExtraModifier = MapKind;
+        MapKind = OMPC_MAP_unknown;
+        PreMapLoc = Tok.getLocation();
+        PreMapName = Tok.getIdentifierInfo()->getName();
+      } else {
+        Diag(Tok, diag::err_omp_more_one_map_type);
+        Diag(PreMapLoc, diag::note_previous_map_type_specified_here)
+            << PreMapName;
+      }
+      ConsumeToken();
     } else {
       // For the case of unknown map-type-modifier or a map-type.
       // Map-type is followed by a colon; the function returns when it
@@ -4247,8 +4267,14 @@ bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
         continue;
       }
       // Potential map-type token as it is followed by a colon.
-      if (PP.LookAhead(0).is(tok::colon))
-        return false;
+      if (PP.LookAhead(0).is(tok::colon)) {
+        if (getLangOpts().OpenMP >= 60) {
+          break;
+        } else {
+          return false;
+        }
+      }
+
       Diag(Tok, diag::err_omp_unknown_map_type_modifier)
           << (getLangOpts().OpenMP >= 51 ? (getLangOpts().OpenMP >= 52 ? 2 : 1)
                                          : 0)
@@ -4258,6 +4284,14 @@ bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
     if (getCurToken().is(tok::comma))
       ConsumeToken();
   }
+  if (getLangOpts().OpenMP >= 60 && !HasMapType) {
+    if (!Tok.is(tok::colon)) {
+      Diag(Tok, diag::err_omp_unknown_map_type);
+      ConsumeToken();
+    } else {
+      Data.ExtraModifier = OMPC_MAP_unknown;
+    }
+  }
   return false;
 }
 
@@ -4269,13 +4303,12 @@ static OpenMPMapClauseKind isMapType(Parser &P) {
   if (!Tok.isOneOf(tok::identifier, tok::kw_delete))
     return OMPC_MAP_unknown;
   Preprocessor &PP = P.getPreprocessor();
-  OpenMPMapClauseKind MapType =
-      static_cast<OpenMPMapClauseKind>(getOpenMPSimpleClauseType(
-          OMPC_map, PP.getSpelling(Tok), P.getLangOpts()));
+  unsigned MapType =
+      getOpenMPSimpleClauseType(OMPC_map, PP.getSpelling(Tok), P.getLangOpts());
   if (MapType == OMPC_MAP_to || MapType == OMPC_MAP_from ||
       MapType == OMPC_MAP_tofrom || MapType == OMPC_MAP_alloc ||
       MapType == OMPC_MAP_delete || MapType == OMPC_MAP_release)
-    return MapType;
+    return static_cast<OpenMPMapClauseKind>(MapType);
   return OMPC_MAP_unknown;
 }
 
@@ -4659,8 +4692,10 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     // Only parse map-type-modifier[s] and map-type if a colon is present in
     // the map clause.
     if (ColonPresent) {
+      if (getLangOpts().OpenMP >= 60 && getCurToken().is(tok::colon))
+        Diag(Tok, diag::err_omp_map_modifier_specification_list);
       IsInvalidMapperModifier = parseMapTypeModifiers(Data);
-      if (!IsInvalidMapperModifier)
+      if (getLangOpts().OpenMP < 60 && !IsInvalidMapperModifier)
         parseMapType(*this, Data);
       else
         SkipUntil(tok::colon, tok::annot_pragma_openmp_end, StopBeforeMatch);
diff --git a/clang/test/OpenMP/target_ast_print.cpp b/clang/test/OpenMP/target_ast_print.cpp
index f4c10fe3a181..ec6cf2130d7a 100644
--- a/clang/test/OpenMP/target_ast_print.cpp
+++ b/clang/test/OpenMP/target_ast_print.cpp
@@ -1201,6 +1201,64 @@ foo();
 }
 #endif // OMP52
 
+#ifdef OMP60
+
+///==========================================================================///
+// RUN: %clang_cc1 -DOMP60 -verify -Wno-vla -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix OMP60
+// RUN: %clang_cc1 -DOMP60 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP60
+
+// RUN: %clang_cc1 -DOMP60 -verify -Wno-vla -fopenmp-simd -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix OMP60
+// RUN: %clang_cc1 -DOMP60 -fopenmp-simd -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp-simd -fopenmp-version=60 -std=c++11 -include-pch %t -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP60
+
+void foo() {}
+template <typename T, int C>
+T tmain(T argc, T *argv) {
+  T i;
+#pragma omp target map(from always: i)
+  foo();
+#pragma omp target map(from, close: i)
+  foo();
+#pragma omp target map(always,close: i)
+  foo();
+  return 0;
+}
+//OMP60: template <typename T, int C> T tmain(T argc, T *argv) {
+//OMP60-NEXT: T i;
+//OMP60-NEXT: #pragma omp target map(always,from: i)
+//OMP60-NEXT:     foo();
+//OMP60-NEXT: #pragma omp target map(close,from: i)
+//OMP60-NEXT:     foo();
+//OMP60-NEXT: #pragma omp target map(always,close,tofrom: i)
+//OMP60-NEXT:     foo();
+//OMP60-NEXT: return 0;
+//OMP60-NEXT:}
+//OMP60:  template<> int tmain<int, 5>(int argc, int *argv) {
+//OMP60-NEXT:  int i;
+//OMP60-NEXT:  #pragma omp target map(always,from: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  #pragma omp target map(close,from: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  #pragma omp target map(always,close,tofrom: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  return 0;
+//OMP60-NEXT:}
+//OMP60:  template<> char tmain<char, 1>(char argc, char *argv) {
+//OMP60-NEXT:  char i;
+//OMP60-NEXT:  #pragma omp target map(always,from: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  #pragma omp target map(close,from: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  #pragma omp target map(always,close,tofrom: i)
+//OMP60-NEXT:      foo();
+//OMP60-NEXT:  return 0;
+//OMP60-NEXT:}
+int main (int argc, char **argv) {
+  return tmain<int, 5>(argc, &argc) + tmain<char, 1>(argv[0][0], argv[0]);
+}
+#endif // OMP60
+
 #ifdef OMPX
 
 // RUN: %clang_cc1 -DOMPX -verify -Wno-vla -fopenmp -fopenmp-extensions -ast-print %s | FileCheck %s --check-prefix=OMPX
diff --git a/clang/test/OpenMP/target_map_messages.cpp b/clang/test/OpenMP/target_map_messages.cpp
index a6776ee12c0e..3bd432b47e63 100644
--- a/clang/test/OpenMP/target_map_messages.cpp
+++ b/clang/test/OpenMP/target_map_messages.cpp
@@ -1,34 +1,35 @@
 // -fopenmp, -fno-openmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,omp,ge51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=51 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,ge52,omp,ge52-omp,omp52 -fopenmp -fno-openmp-extensions -fopenmp-version=52 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,omp,ge51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=51 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,ge52,lt60,omp,ge52-omp,omp52 -fopenmp -fno-openmp-extensions -fopenmp-version=52 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge52,ge60,omp,ge60-omp,omp60 -fopenmp -fno-openmp-extensions -fopenmp-version=60 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized -Wno-vla
 
 // -fopenmp-simd, -fno-openmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,omp,ge51-omp -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,omp,ge51-omp -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
 
 // -fopenmp -fopenmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,ompx,ge51-ompx -fopenmp -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,ompx,ge51-ompx -fopenmp -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized -Wno-vla
 
 // -fopenmp-simd -fopenmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,ompx,ge51-ompx -fopenmp-simd -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,ompx,ge51-ompx -fopenmp-simd -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
 
 // Check
@@ -113,7 +114,7 @@ struct SA {
     #pragma omp target map(b[true:true])
     {}
 
-    #pragma omp target map(: c,f) // expected-error {{missing map type}}
+    #pragma omp target map(: c,f) // lt60-error {{missing map type}} // ge60-error {{empty modifier-specification-list is not allowed}}
     {}
     #pragma omp target map(always, tofrom: c,f)
     {}
@@ -159,28 +160,28 @@ struct SA {
     // expected-error@+1 {{use of undeclared identifier 'present'}}
     #pragma omp target map(present)
     {}
-    // ge52-omp-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c,f)
     {}
-    // ge52-omp-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c[1:2],f)
     {}
-    // ge52-omp-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c,f[1:2])
     {}
-    // ge52-omp-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+3 {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c[:],f)
     {}
-    // ge52-omp-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+3 {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
@@ -193,19 +194,19 @@ struct SA {
     {}
     #pragma omp target map(always, close, always, close, tofrom: a)   // expected-error 2 {{same map type modifier has been specified more than once}}
     {}
+    // ge60-error@+3 {{same map type modifier has been specified more than once}}
     // ge51-error@+2 {{same map type modifier has been specified more than once}}
     // lt51-error@+1 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(present, present, tofrom: a)
     {}
-    // ge52-omp-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
-    // ge52-omp-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge52-error@+4 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ompx-error@+3 {{same map type modifier has been specified more than once}}
     // ge51-omp-error@+2 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, ompx_hold, tofrom: a)
     {}
-    // ge52-omp-error@+9 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
-    // ge52-omp-error@+8 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
+    // ge60-error@+9 {{same map type modifier has been specified more than once}}
+    // ge52-error@+8 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+7 2 {{same map type modifier has been specified more than once}}
     // ge51-error@+6 {{same map type modifier has been specified more than once}}
     // lt51-ompx-error@+5 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'ompx_hold'}}
@@ -219,34 +220,45 @@ struct SA {
     {}
     #pragma omp target map( , , tofrom: a)   // expected-error {{missing map type modifier}} expected-error {{missing map type modifier}}
     {}
-    #pragma omp target map( , , : a)   // expected-error {{missing map type modifier}} expected-error {{missing map type modifier}} expected-error {{missing map type}}
+    #pragma omp target map( , , : a)   // expected-error {{missing map type modifier}} expected-error {{missing map type modifier}} lt60-error {{missing map type}}
     {}
+    // ge60-error@+4 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-error@+3 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+2 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     // expected-error@+1 {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
     #pragma omp target map( d, f, bf: a)
     {}
+    // ge60-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator}}
     // expected-error@+4 {{missing map type modifier}}
     // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-    // expected-error@+1 {{missing map type}}
+    // lt60-error@+1 {{missing map type}}
     #pragma omp target map( , f, : a)
     {}
-    #pragma omp target map(always close: a)   // expected-error {{missing map type}} omp52-error{{missing ',' after map type modifier}}
+    #pragma omp target map(always close: a)   // lt60-error {{missing map type}} ge52-error{{missing ',' after map type modifier}}
     {}
-    #pragma omp target map(always close bf: a)   // omp52-error 2 {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}} 
+    #pragma omp target map(always close bf: a)   // ge52-error 2 {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
     {}
-    // omp52-error@+4 {{missing ',' after map type modifier}}
+    // ge52-error@+4 {{missing ',' after map type modifier}}
     // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-    // expected-error@+1 {{missing map type}}
+    // lt60-error@+1 {{missing map type}}
     #pragma omp target map(always tofrom close: a)
     {}
+    // ge60-note@+4 {{map type 'tofrom' is previous specified here}}
+    // ge60-error@+3 {{map type is already specified}}
     // ge51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(tofrom from: a)
     {}
-    #pragma omp target map(close bf: a)   // omp52-error {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+    // ge60-note@+5 {{map type 'to' is previous specified here}}
+    // ge60-error@+4 {{map type is already specified}}
+    // ge52-error@+3 {{missing ',' after map type modifier}}
+    // ge51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
+    // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
+    #pragma omp target map(to always from: a)
+    {}
+    #pragma omp target map(close bf: a)   // ge52-error {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
     {}
     #pragma omp target map(([b[I]][bf])f)  // lt50-error {{expected ',' or ']' in lambda capture list}} lt50-error {{expected ')'}} lt50-note {{to match this '('}}
     {}
@@ -266,6 +278,7 @@ struct SA {
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(iterator(it=0:10, it=0:20), tofrom:a)
     {}
+    // ge60-error@+7 {{expected '(' after 'iterator'}}
     // ge51-ompx-error@+6 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'ompx_hold'}}
     // lt51-ompx-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'ompx_hold'}}
     // lt51-error@+4 {{expected '(' after 'iterator'}}
@@ -694,20 +707,20 @@ T tmain(T argc) {
   foo();
 
 #pragma omp target data map(always, tofrom: x)
-#pragma omp target data map(always: x) // expected-error {{missing map type}}
+#pragma omp target data map(always: x) // lt60-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// expected-error@+1 {{missing map type}}
+// lt60-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, always: x)
 #pragma omp target data map(always, tofrom: always, tofrom, x)
 #pragma omp target map(tofrom j) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
 
 #pragma omp target data map(close, tofrom: x)
-#pragma omp target data map(close: x) // expected-error {{missing map type}}
+#pragma omp target data map(close: x) // lt60-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// expected-error@+1 {{missing map type}}
+// lt60-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, close: x)
 #pragma omp target data map(close, tofrom: close, tofrom, x)
   foo();
@@ -829,19 +842,19 @@ int main(int argc, char **argv) {
   foo();
 
 #pragma omp target data map(always, tofrom: x)
-#pragma omp target data map(always: x) // expected-error {{missing map type}}
+#pragma omp target data map(always: x) // lt60-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// expected-error@+1 {{missing map type}}
+// lt60-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, always: x)
 #pragma omp target data map(always, tofrom: always, tofrom, x)
 #pragma omp target map(tofrom j) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
 #pragma omp target data map(close, tofrom: x)
-#pragma omp target data map(close: x) // expected-error {{missing map type}}
+#pragma omp target data map(close: x) // lt60-error {{missing map type}}
 // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
 // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-// expected-error@+1 {{missing map type}}
+// lt60-error@+1 {{missing map type}}
 #pragma omp target data map(tofrom, close: x)
   foo();
 // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
-- 
GitLab


From 8b7f178091b70d264c13b1f83b58e6c9dfe81c1b Mon Sep 17 00:00:00 2001
From: madanial0 <118996571+madanial0@users.noreply.github.com>
Date: Mon, 13 May 2024 10:40:14 -0400
Subject: [PATCH 0574/1206] [flang] Fix Failing rewrite-out_of_range testcase
 to only check for real 10 on supported platforms (#91629)

The real 10 tests fail on `AIX` on `PPC`, only check them on `x86_64`

Co-authored-by: Mark Danial <mark.danial@ibm.com>
---
 flang/test/Evaluate/rewrite-out_of_range.F90 | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/flang/test/Evaluate/rewrite-out_of_range.F90 b/flang/test/Evaluate/rewrite-out_of_range.F90
index a5cd09cb2853..b5df610ff2fb 100644
--- a/flang/test/Evaluate/rewrite-out_of_range.F90
+++ b/flang/test/Evaluate/rewrite-out_of_range.F90
@@ -1,5 +1,5 @@
 ! Tests rewriting of OUT_OF_RANGE()
-! RUN: %flang_fc1 -fdebug-unparse %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -fdebug-unparse -cpp %s 2>&1 | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-X86-64%}
 
 logical round
 
@@ -194,12 +194,12 @@ end
 !CHECK:   PRINT *, " real", 8_4, "real", 8_4, .false._4
 !CHECK:   PRINT *, " real", 8_4, "real", 10_4, .false._4
 !CHECK:   PRINT *, " real", 8_4, "real", 16_4, .false._4
-!CHECK:   PRINT *, " real", 10_4, "real", 2_4, blt(transfer(abs(x)-6.5504e4_10,0_16)-1_16,604444463063240877801471_16)
-!CHECK:   PRINT *, " real", 10_4, "real", 3_4, blt(transfer(abs(x)-3.3895313892515354759047080037148786688e38_10,0_16)-1_16,604444463063240877801471_16)
-!CHECK:   PRINT *, " real", 10_4, "real", 4_4, blt(transfer(abs(x)-3.4028234663852885981170418348451692544e38_10,0_16)-1_16,604444463063240877801471_16)
-!CHECK:   PRINT *, " real", 10_4, "real", 8_4, blt(transfer(abs(x)-1.79769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368e308_10,0_16)-1_16,604444463063240877801471_16)
-!CHECK:   PRINT *, " real", 10_4, "real", 10_4, .false._4
-!CHECK:   PRINT *, " real", 10_4, "real", 16_4, .false._4
+!CHECK-X86-64:   PRINT *, " real", 10_4, "real", 2_4, blt(transfer(abs(x)-6.5504e4_10,0_16)-1_16,604444463063240877801471_16)
+!CHECK-X86-64:   PRINT *, " real", 10_4, "real", 3_4, blt(transfer(abs(x)-3.3895313892515354759047080037148786688e38_10,0_16)-1_16,604444463063240877801471_16)
+!CHECK-X86-64:   PRINT *, " real", 10_4, "real", 4_4, blt(transfer(abs(x)-3.4028234663852885981170418348451692544e38_10,0_16)-1_16,604444463063240877801471_16)
+!CHECK-X86-64:   PRINT *, " real", 10_4, "real", 8_4, blt(transfer(abs(x)-1.79769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368e308_10,0_16)-1_16,604444463063240877801471_16)
+!CHECK-X86-64:   PRINT *, " real", 10_4, "real", 10_4, .false._4
+!CHECK-X86-64:   PRINT *, " real", 10_4, "real", 16_4, .false._4
 !CHECK:   PRINT *, " real", 16_4, "real", 2_4, blt(transfer(abs(x)-6.5504e4_16,0_16)-1_16,170135991163610696904058773219554885631_16)
 !CHECK:   PRINT *, " real", 16_4, "real", 3_4, blt(transfer(abs(x)-3.3895313892515354759047080037148786688e38_16,0_16)-1_16,170135991163610696904058773219554885631_16)
 !CHECK:   PRINT *, " real", 16_4, "real", 4_4, blt(transfer(abs(x)-3.4028234663852885981170418348451692544e38_16,0_16)-1_16,170135991163610696904058773219554885631_16)
-- 
GitLab


From d8197728145137e2aa00f3b2ed69914f48851d09 Mon Sep 17 00:00:00 2001
From: Tuan Chuong Goh <chuong.goh@arm.com>
Date: Mon, 13 May 2024 14:27:09 +0000
Subject: [PATCH 0575/1206] [AArch64][NFC] Pre-commit tests for Select G_ICMP
 Zero Instruction (#90054)

---
 llvm/test/CodeGen/AArch64/icmp.ll | 1114 +++++++++++++++++++++++++++++
 1 file changed, 1114 insertions(+)

diff --git a/llvm/test/CodeGen/AArch64/icmp.ll b/llvm/test/CodeGen/AArch64/icmp.ll
index 2292bc6d2038..88b2f279ec3e 100644
--- a/llvm/test/CodeGen/AArch64/icmp.ll
+++ b/llvm/test/CodeGen/AArch64/icmp.ll
@@ -1375,3 +1375,1117 @@ entry:
   %s = select <32 x i1> %c, <32 x i8> %d, <32 x i8> %e
   ret <32 x i8> %s
 }
+
+; ===== ICMP Zero RHS =====
+
+define <8 x i1> @icmp_eq_v8i8_Zero_RHS(<8 x i8> %a) {
+; CHECK-SD-LABEL: icmp_eq_v8i8_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.8b, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v8i8_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <8 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+    ret <8 x i1> %c
+}
+
+define <16 x i1> @icmp_eq_v16i8_Zero_RHS(<16 x i8> %a) {
+; CHECK-SD-LABEL: icmp_eq_v16i8_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.16b, v0.16b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v16i8_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <16 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+    ret <16 x i1> %c
+}
+
+define <4 x i1> @icmp_eq_v4i16_Zero_RHS(<4 x i16> %a) {
+; CHECK-SD-LABEL: icmp_eq_v4i16_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.4h, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v4i16_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
+    ret <4 x i1> %c
+}
+
+define <8 x i1> @icmp_eq_v8i16_Zero_RHS(<8 x i16> %a) {
+; CHECK-SD-LABEL: icmp_eq_v8i16_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.8h, v0.8h, #0
+; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v8i16_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <8 x i16> %a, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+    ret <8 x i1> %c
+}
+
+define <2 x i1> @icmp_eq_v2i32_Zero_RHS(<2 x i32> %a) {
+; CHECK-SD-LABEL: icmp_eq_v2i32_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.2s, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v2i32_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <2 x i32> %a, <i32 0, i32 0>
+    ret <2 x i1> %c
+}
+
+define <4 x i1> @icmp_eq_v4i32_Zero_RHS(<4 x i32> %a) {
+; CHECK-SD-LABEL: icmp_eq_v4i32_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v4i32_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <4 x i32> %a, <i32 0, i32 0, i32 0, i32 0>
+    ret <4 x i1> %c
+}
+
+define <2 x i1> @icmp_eq_v2i64_Zero_RHS(<2 x i64> %a) {
+; CHECK-SD-LABEL: icmp_eq_v2i64_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.2d, v0.2d, #0
+; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v2i64_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <2 x i64> %a, <i64 0, i64 0>
+    ret <2 x i1> %c
+}
+
+define <8 x i1> @icmp_sge_v8i8_Zero_RHS(<8 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sge_v8i8_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.8b, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v8i8_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <8 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+    ret <8 x i1> %c
+}
+
+define <16 x i1> @icmp_sge_v16i8_Zero_RHS(<16 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sge_v16i8_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.16b, v0.16b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v16i8_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <16 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+    ret <16 x i1> %c
+}
+
+define <4 x i1> @icmp_sge_v4i16_Zero_RHS(<4 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sge_v4i16_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.4h, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v4i16_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
+    ret <4 x i1> %c
+}
+
+define <8 x i1> @icmp_sge_v8i16_Zero_RHS(<8 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sge_v8i16_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.8h, v0.8h, #0
+; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v8i16_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <8 x i16> %a, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+    ret <8 x i1> %c
+}
+
+define <2 x i1> @icmp_sge_v2i32_Zero_RHS(<2 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sge_v2i32_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.2s, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v2i32_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <2 x i32> %a, <i32 0, i32 0>
+    ret <2 x i1> %c
+}
+
+define <4 x i1> @icmp_sge_v4i32_Zero_RHS(<4 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sge_v4i32_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v4i32_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <4 x i32> %a, <i32 0, i32 0, i32 0, i32 0>
+    ret <4 x i1> %c
+}
+
+define <2 x i1> @icmp_sge_v2i64_Zero_RHS(<2 x i64> %a) {
+; CHECK-SD-LABEL: icmp_sge_v2i64_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.2d, v0.2d, #0
+; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v2i64_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <2 x i64> %a, <i64 0, i64 0>
+    ret <2 x i1> %c
+}
+
+define <8 x i1> @icmp_sgt_v8i8_Zero_RHS(<8 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v8i8_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.8b, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v8i8_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <8 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+    ret <8 x i1> %c
+}
+
+define <16 x i1> @icmp_sgt_v16i8_Zero_RHS(<16 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v16i8_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.16b, v0.16b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v16i8_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <16 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+    ret <16 x i1> %c
+}
+
+define <4 x i1> @icmp_sgt_v4i16_Zero_RHS(<4 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v4i16_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.4h, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v4i16_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
+    ret <4 x i1> %c
+}
+
+define <8 x i1> @icmp_sgt_v8i16_Zero_RHS(<8 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v8i16_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.8h, v0.8h, #0
+; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v8i16_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <8 x i16> %a, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+    ret <8 x i1> %c
+}
+
+define <2 x i1> @icmp_sgt_v2i32_Zero_RHS(<2 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v2i32_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.2s, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v2i32_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <2 x i32> %a, <i32 0, i32 0>
+    ret <2 x i1> %c
+}
+
+define <4 x i1> @icmp_sgt_v4i32_Zero_RHS(<4 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v4i32_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.4s, v0.4s, #0
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v4i32_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <4 x i32> %a, <i32 0, i32 0, i32 0, i32 0>
+    ret <4 x i1> %c
+}
+
+define <2 x i1> @icmp_sgt_v2i64_Zero_RHS(<2 x i64> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v2i64_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.2d, v0.2d, #0
+; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v2i64_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <2 x i64> %a, <i64 0, i64 0>
+    ret <2 x i1> %c
+}
+
+define <8 x i1> @icmp_sle_v8i8_Zero_RHS(<8 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sle_v8i8_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.8b, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v8i8_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <8 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+    ret <8 x i1> %c
+}
+
+define <16 x i1> @icmp_sle_v16i8_Zero_RHS(<16 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sle_v16i8_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.16b, v0.16b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v16i8_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <16 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+    ret <16 x i1> %c
+}
+
+define <4 x i1> @icmp_sle_v4i16_Zero_RHS(<4 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sle_v4i16_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.4h, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v4i16_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
+    ret <4 x i1> %c
+}
+
+define <8 x i1> @icmp_sle_v8i16_Zero_RHS(<8 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sle_v8i16_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.8h, v0.8h, #0
+; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v8i16_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <8 x i16> %a, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+    ret <8 x i1> %c
+}
+
+define <2 x i1> @icmp_sle_v2i32_Zero_RHS(<2 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sle_v2i32_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.2s, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v2i32_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.2s, v1.2s, v0.2s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <2 x i32> %a, <i32 0, i32 0>
+    ret <2 x i1> %c
+}
+
+define <4 x i1> @icmp_sle_v4i32_Zero_RHS(<4 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sle_v4i32_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.4s, v0.4s, #0
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v4i32_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <4 x i32> %a, <i32 0, i32 0, i32 0, i32 0>
+    ret <4 x i1> %c
+}
+
+define <2 x i1> @icmp_sle_v2i64_Zero_RHS(<2 x i64> %a) {
+; CHECK-SD-LABEL: icmp_sle_v2i64_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.2d, v0.2d, #0
+; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v2i64_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.2d, v1.2d, v0.2d
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <2 x i64> %a, <i64 0, i64 0>
+    ret <2 x i1> %c
+}
+
+define <8 x i1> @icmp_slt_v8i8_Zero_RHS(<8 x i8> %a) {
+; CHECK-SD-LABEL: icmp_slt_v8i8_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.8b, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v8i8_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <8 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+    ret <8 x i1> %c
+}
+
+define <16 x i1> @icmp_slt_v16i8_Zero_RHS(<16 x i8> %a) {
+; CHECK-SD-LABEL: icmp_slt_v16i8_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.16b, v0.16b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v16i8_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <16 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+    ret <16 x i1> %c
+}
+
+define <4 x i1> @icmp_slt_v4i16_Zero_RHS(<4 x i16> %a) {
+; CHECK-SD-LABEL: icmp_slt_v4i16_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.4h, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v4i16_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
+    ret <4 x i1> %c
+}
+
+define <8 x i1> @icmp_slt_v8i16_Zero_RHS(<8 x i16> %a) {
+; CHECK-SD-LABEL: icmp_slt_v8i16_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v8i16_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <8 x i16> %a, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+    ret <8 x i1> %c
+}
+
+define <2 x i1> @icmp_slt_v2i32_Zero_RHS(<2 x i32> %a) {
+; CHECK-SD-LABEL: icmp_slt_v2i32_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.2s, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v2i32_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.2s, v1.2s, v0.2s
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <2 x i32> %a, <i32 0, i32 0>
+    ret <2 x i1> %c
+}
+
+define <4 x i1> @icmp_slt_v4i32_Zero_RHS(<4 x i32> %a) {
+; CHECK-SD-LABEL: icmp_slt_v4i32_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.4s, v0.4s, #0
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v4i32_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <4 x i32> %a, <i32 0, i32 0, i32 0, i32 0>
+    ret <4 x i1> %c
+}
+
+define <2 x i1> @icmp_slt_v2i64_Zero_RHS(<2 x i64> %a) {
+; CHECK-SD-LABEL: icmp_slt_v2i64_Zero_RHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.2d, v0.2d, #0
+; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v2i64_Zero_RHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.2d, v1.2d, v0.2d
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <2 x i64> %a, <i64 0, i64 0>
+    ret <2 x i1> %c
+}
+
+; ===== ICMP Zero LHS =====
+
+define <8 x i1> @icmp_eq_v8i8_Zero_LHS(<8 x i8> %a) {
+; CHECK-SD-LABEL: icmp_eq_v8i8_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.8b, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v8i8_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
+    ret <8 x i1> %c
+}
+
+define <16 x i1> @icmp_eq_v16i8_Zero_LHS(<16 x i8> %a) {
+; CHECK-SD-LABEL: icmp_eq_v16i8_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.16b, v0.16b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v16i8_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
+    ret <16 x i1> %c
+}
+
+define <4 x i1> @icmp_eq_v4i16_Zero_LHS(<4 x i16> %a) {
+; CHECK-SD-LABEL: icmp_eq_v4i16_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.4h, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v4i16_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <4 x i16> <i16 0, i16 0, i16 0, i16 0>, %a
+    ret <4 x i1> %c
+}
+
+define <8 x i1> @icmp_eq_v8i16_Zero_LHS(<8 x i16> %a) {
+; CHECK-SD-LABEL: icmp_eq_v8i16_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.8h, v0.8h, #0
+; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v8i16_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, %a
+    ret <8 x i1> %c
+}
+
+define <2 x i1> @icmp_eq_v2i32_Zero_LHS(<2 x i32> %a) {
+; CHECK-SD-LABEL: icmp_eq_v2i32_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.2s, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v2i32_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.2s, v1.2s, v0.2s
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <2 x i32> <i32 0, i32 0>, %a
+    ret <2 x i1> %c
+}
+
+define <4 x i1> @icmp_eq_v4i32_Zero_LHS(<4 x i32> %a) {
+; CHECK-SD-LABEL: icmp_eq_v4i32_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v4i32_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <4 x i32> <i32 0, i32 0, i32 0, i32 0>, %a
+    ret <4 x i1> %c
+}
+
+define <2 x i1> @icmp_eq_v2i64_Zero_LHS(<2 x i64> %a) {
+; CHECK-SD-LABEL: icmp_eq_v2i64_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmeq v0.2d, v0.2d, #0
+; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_eq_v2i64_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmeq v0.2d, v1.2d, v0.2d
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+    %c = icmp eq <2 x i64> <i64 0, i64 0>, %a
+    ret <2 x i1> %c
+}
+
+define <8 x i1> @icmp_sge_v8i8_Zero_LHS(<8 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sge_v8i8_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.8b, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v8i8_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
+    ret <8 x i1> %c
+}
+
+define <16 x i1> @icmp_sge_v16i8_Zero_LHS(<16 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sge_v16i8_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.16b, v0.16b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v16i8_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
+    ret <16 x i1> %c
+}
+
+define <4 x i1> @icmp_sge_v4i16_Zero_LHS(<4 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sge_v4i16_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.4h, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v4i16_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <4 x i16> <i16 0, i16 0, i16 0, i16 0>, %a
+    ret <4 x i1> %c
+}
+
+define <8 x i1> @icmp_sge_v8i16_Zero_LHS(<8 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sge_v8i16_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.8h, v0.8h, #0
+; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v8i16_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, %a
+    ret <8 x i1> %c
+}
+
+define <2 x i1> @icmp_sge_v2i32_Zero_LHS(<2 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sge_v2i32_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.2s, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v2i32_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.2s, v1.2s, v0.2s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <2 x i32> <i32 0, i32 0>, %a
+    ret <2 x i1> %c
+}
+
+define <4 x i1> @icmp_sge_v4i32_Zero_LHS(<4 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sge_v4i32_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.4s, v0.4s, #0
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v4i32_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <4 x i32> <i32 0, i32 0, i32 0, i32 0>, %a
+    ret <4 x i1> %c
+}
+
+define <2 x i1> @icmp_sge_v2i64_Zero_LHS(<2 x i64> %a) {
+; CHECK-SD-LABEL: icmp_sge_v2i64_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmle v0.2d, v0.2d, #0
+; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sge_v2i64_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.2d, v1.2d, v0.2d
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+    %c = icmp sge <2 x i64> <i64 0, i64 0>, %a
+    ret <2 x i1> %c
+}
+
+define <8 x i1> @icmp_sgt_v8i8_Zero_LHS(<8 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v8i8_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.8b, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v8i8_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
+    ret <8 x i1> %c
+}
+
+define <16 x i1> @icmp_sgt_v16i8_Zero_LHS(<16 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v16i8_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.16b, v0.16b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v16i8_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
+    ret <16 x i1> %c
+}
+
+define <4 x i1> @icmp_sgt_v4i16_Zero_LHS(<4 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v4i16_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.4h, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v4i16_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <4 x i16> <i16 0, i16 0, i16 0, i16 0>, %a
+    ret <4 x i1> %c
+}
+
+define <8 x i1> @icmp_sgt_v8i16_Zero_LHS(<8 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v8i16_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v8i16_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, %a
+    ret <8 x i1> %c
+}
+
+define <2 x i1> @icmp_sgt_v2i32_Zero_LHS(<2 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v2i32_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.2s, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v2i32_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.2s, v1.2s, v0.2s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <2 x i32> <i32 0, i32 0>, %a
+    ret <2 x i1> %c
+}
+
+define <4 x i1> @icmp_sgt_v4i32_Zero_LHS(<4 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v4i32_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.4s, v0.4s, #0
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v4i32_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <4 x i32> <i32 0, i32 0, i32 0, i32 0>, %a
+    ret <4 x i1> %c
+}
+
+define <2 x i1> @icmp_sgt_v2i64_Zero_LHS(<2 x i64> %a) {
+; CHECK-SD-LABEL: icmp_sgt_v2i64_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmlt v0.2d, v0.2d, #0
+; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sgt_v2i64_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.2d, v1.2d, v0.2d
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+    %c = icmp sgt <2 x i64> <i64 0, i64 0>, %a
+    ret <2 x i1> %c
+}
+
+define <8 x i1> @icmp_sle_v8i8_Zero_LHS(<8 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sle_v8i8_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.8b, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v8i8_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
+    ret <8 x i1> %c
+}
+
+define <16 x i1> @icmp_sle_v16i8_Zero_LHS(<16 x i8> %a) {
+; CHECK-SD-LABEL: icmp_sle_v16i8_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.16b, v0.16b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v16i8_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
+    ret <16 x i1> %c
+}
+
+define <4 x i1> @icmp_sle_v4i16_Zero_LHS(<4 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sle_v4i16_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.4h, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v4i16_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <4 x i16> <i16 0, i16 0, i16 0, i16 0>, %a
+    ret <4 x i1> %c
+}
+
+define <8 x i1> @icmp_sle_v8i16_Zero_LHS(<8 x i16> %a) {
+; CHECK-SD-LABEL: icmp_sle_v8i16_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.8h, v0.8h, #0
+; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v8i16_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, %a
+    ret <8 x i1> %c
+}
+
+define <2 x i1> @icmp_sle_v2i32_Zero_LHS(<2 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sle_v2i32_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.2s, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v2i32_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <2 x i32> <i32 0, i32 0>, %a
+    ret <2 x i1> %c
+}
+
+define <4 x i1> @icmp_sle_v4i32_Zero_LHS(<4 x i32> %a) {
+; CHECK-SD-LABEL: icmp_sle_v4i32_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v4i32_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <4 x i32> <i32 0, i32 0, i32 0, i32 0>, %a
+    ret <4 x i1> %c
+}
+
+define <2 x i1> @icmp_sle_v2i64_Zero_LHS(<2 x i64> %a) {
+; CHECK-SD-LABEL: icmp_sle_v2i64_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmge v0.2d, v0.2d, #0
+; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_sle_v2i64_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmge v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+    %c = icmp sle <2 x i64> <i64 0, i64 0>, %a
+    ret <2 x i1> %c
+}
+
+define <8 x i1> @icmp_slt_v8i8_Zero_LHS(<8 x i8> %a) {
+; CHECK-SD-LABEL: icmp_slt_v8i8_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.8b, v0.8b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v8i8_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
+    ret <8 x i1> %c
+}
+
+define <16 x i1> @icmp_slt_v16i8_Zero_LHS(<16 x i8> %a) {
+; CHECK-SD-LABEL: icmp_slt_v16i8_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.16b, v0.16b, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v16i8_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
+    ret <16 x i1> %c
+}
+
+define <4 x i1> @icmp_slt_v4i16_Zero_LHS(<4 x i16> %a) {
+; CHECK-SD-LABEL: icmp_slt_v4i16_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.4h, v0.4h, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v4i16_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <4 x i16> <i16 0, i16 0, i16 0, i16 0>, %a
+    ret <4 x i1> %c
+}
+
+define <8 x i1> @icmp_slt_v8i16_Zero_LHS(<8 x i16> %a) {
+; CHECK-SD-LABEL: icmp_slt_v8i16_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.8h, v0.8h, #0
+; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v8i16_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, %a
+    ret <8 x i1> %c
+}
+
+define <2 x i1> @icmp_slt_v2i32_Zero_LHS(<2 x i32> %a) {
+; CHECK-SD-LABEL: icmp_slt_v2i32_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.2s, v0.2s, #0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v2i32_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <2 x i32> <i32 0, i32 0>, %a
+    ret <2 x i1> %c
+}
+
+define <4 x i1> @icmp_slt_v4i32_Zero_LHS(<4 x i32> %a) {
+; CHECK-SD-LABEL: icmp_slt_v4i32_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.4s, v0.4s, #0
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v4i32_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <4 x i32> <i32 0, i32 0, i32 0, i32 0>, %a
+    ret <4 x i1> %c
+}
+
+define <2 x i1> @icmp_slt_v2i64_Zero_LHS(<2 x i64> %a) {
+; CHECK-SD-LABEL: icmp_slt_v2i64_Zero_LHS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    cmgt v0.2d, v0.2d, #0
+; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: icmp_slt_v2i64_Zero_LHS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GI-NEXT:    cmgt v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+    %c = icmp slt <2 x i64> <i64 0, i64 0>, %a
+    ret <2 x i1> %c
+}
-- 
GitLab


From c4a9a374749deb5f2a932a7d4ef9321be1b2ae5d Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Thu, 9 May 2024 09:44:35 -0700
Subject: [PATCH 0576/1206] [OpenACC] device_type clause Sema for Compute
 constructs

device_type, also spelled as dtype, specifies the applicability of the
clauses following it, and takes a series of identifiers representing the
architectures it applies to.  As we don't have a source for the valid
architectures yet, this patch just accepts all.

Semantically, this also limits the list of clauses that can be applied
after the device_type, so this implements that as well.
---
 clang/include/clang/AST/OpenACCClause.h       |  59 +++++
 .../clang/Basic/DiagnosticSemaKinds.td        |   4 +
 clang/include/clang/Basic/OpenACCClauses.def  |   2 +
 clang/include/clang/Parse/Parser.h            |   3 +-
 clang/include/clang/Sema/SemaOpenACC.h        |  23 +-
 clang/lib/AST/OpenACCClause.cpp               |  28 ++-
 clang/lib/AST/StmtProfile.cpp                 |   3 +
 clang/lib/AST/TextNodeDumper.cpp              |  13 ++
 clang/lib/Parse/ParseOpenACC.cpp              |  21 +-
 clang/lib/Sema/SemaOpenACC.cpp                |  55 +++++
 clang/lib/Sema/TreeTransform.h                |  10 +
 clang/lib/Serialization/ASTReader.cpp         |  17 +-
 clang/lib/Serialization/ASTWriter.cpp         |  15 +-
 .../ast-print-openacc-compute-construct.cpp   |  23 ++
 clang/test/ParserOpenACC/parse-clauses.c      |  28 +--
 .../compute-construct-device_type-ast.cpp     | 105 +++++++++
 .../compute-construct-device_type-clause.c    | 221 ++++++++++++++++++
 .../compute-construct-device_type-clause.cpp  |  25 ++
 clang/tools/libclang/CIndex.cpp               |   2 +
 19 files changed, 622 insertions(+), 35 deletions(-)
 create mode 100644 clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp
 create mode 100644 clang/test/SemaOpenACC/compute-construct-device_type-clause.c
 create mode 100644 clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp

diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 3d0b1ab9d31e..3998a3430ff9 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -17,6 +17,8 @@
 #include "clang/AST/StmtIterator.h"
 #include "clang/Basic/OpenACCKinds.h"
 
+#include <utility>
+
 namespace clang {
 /// This is the base type for all OpenACC Clauses.
 class OpenACCClause {
@@ -75,6 +77,63 @@ public:
   }
 };
 
+using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
+/// A 'device_type' or 'dtype' clause, takes a list of either an 'asterisk' or
+/// an identifier. The 'asterisk' means 'the rest'.
+class OpenACCDeviceTypeClause final
+    : public OpenACCClauseWithParams,
+      public llvm::TrailingObjects<OpenACCDeviceTypeClause,
+                                   DeviceTypeArgument> {
+  // Data stored in trailing objects as IdentifierInfo* /SourceLocation pairs. A
+  // nullptr IdentifierInfo* represents an asterisk.
+  unsigned NumArchs;
+  OpenACCDeviceTypeClause(OpenACCClauseKind K, SourceLocation BeginLoc,
+                          SourceLocation LParenLoc,
+                          ArrayRef<DeviceTypeArgument> Archs,
+                          SourceLocation EndLoc)
+      : OpenACCClauseWithParams(K, BeginLoc, LParenLoc, EndLoc),
+        NumArchs(Archs.size()) {
+    assert(
+        (K == OpenACCClauseKind::DeviceType || K == OpenACCClauseKind::DType) &&
+        "Invalid clause kind for device-type");
+
+    assert(!llvm::any_of(Archs, [](const DeviceTypeArgument &Arg) {
+      return Arg.second.isInvalid();
+    }) && "Invalid SourceLocation for an argument");
+
+    assert(Archs.size() == 1 ||
+           !llvm::any_of(Archs,
+                         [](const DeviceTypeArgument &Arg) {
+                           return Arg.first == nullptr;
+                         }) &&
+               "Only a single asterisk version is permitted, and must be the "
+               "only one");
+
+    std::uninitialized_copy(Archs.begin(), Archs.end(),
+                            getTrailingObjects<DeviceTypeArgument>());
+  }
+
+public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::DType ||
+           C->getClauseKind() == OpenACCClauseKind::DeviceType;
+  }
+  bool hasAsterisk() const {
+    return getArchitectures().size() > 0 &&
+           getArchitectures()[0].first == nullptr;
+  }
+
+  ArrayRef<DeviceTypeArgument> getArchitectures() const {
+    return ArrayRef<DeviceTypeArgument>(
+        getTrailingObjects<DeviceTypeArgument>(), NumArchs);
+  }
+
+  static OpenACCDeviceTypeClause *
+  Create(const ASTContext &C, OpenACCClauseKind K, SourceLocation BeginLoc,
+         SourceLocation LParenLoc, ArrayRef<DeviceTypeArgument> Archs,
+         SourceLocation EndLoc);
+};
+
 /// A 'default' clause, has the optional 'none' or 'present' argument.
 class OpenACCDefaultClause : public OpenACCClauseWithParams {
   friend class ASTReaderStmt;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 9e82130c9360..6100fba51005 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12344,4 +12344,8 @@ def warn_acc_deprecated_alias_name
 def err_acc_var_not_pointer_type
     : Error<"expected pointer in '%0' clause, type is %1">;
 def note_acc_expected_pointer_var : Note<"expected variable of pointer type">;
+def err_acc_clause_after_device_type
+    : Error<"OpenACC clause '%0' may not follow a '%1' clause in a "
+            "compute construct">;
+
 } // end of sema component.
diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def
index afb7b30b7465..7ecc51799468 100644
--- a/clang/include/clang/Basic/OpenACCClauses.def
+++ b/clang/include/clang/Basic/OpenACCClauses.def
@@ -37,6 +37,8 @@ CLAUSE_ALIAS(PCreate, Create)
 CLAUSE_ALIAS(PresentOrCreate, Create)
 VISIT_CLAUSE(Default)
 VISIT_CLAUSE(DevicePtr)
+VISIT_CLAUSE(DeviceType)
+CLAUSE_ALIAS(DType, DeviceType)
 VISIT_CLAUSE(FirstPrivate)
 VISIT_CLAUSE(If)
 VISIT_CLAUSE(NoCreate)
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 61589fb7766f..3910cba34a21 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3720,7 +3720,8 @@ private:
                                SourceLocation Loc,
                                llvm::SmallVectorImpl<Expr *> &IntExprs);
   /// Parses the 'device-type-list', which is a list of identifiers.
-  bool ParseOpenACCDeviceTypeList();
+  bool ParseOpenACCDeviceTypeList(
+      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs);
   /// Parses the 'async-argument', which is an integral value with two
   /// 'special' values that are likely negative (but come from Macros).
   OpenACCIntExprParseResult ParseOpenACCAsyncArgument(OpenACCDirectiveKind DK,
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index e684ee6b2be1..f838fa97d33a 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -26,6 +26,9 @@ class OpenACCClause;
 
 class SemaOpenACC : public SemaBase {
 public:
+  // Redeclaration of the version in OpenACCClause.h.
+  using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
+
   /// A type to represent all the data for an OpenACC Clause that has been
   /// parsed, but not yet created/semantically analyzed. This is effectively a
   /// discriminated union on the 'Clause Kind', with all of the individual
@@ -60,8 +63,12 @@ public:
       SmallVector<Expr *> QueueIdExprs;
     };
 
+    struct DeviceTypeDetails {
+      SmallVector<DeviceTypeArgument> Archs;
+    };
+
     std::variant<std::monostate, DefaultDetails, ConditionDetails,
-                 IntExprDetails, VarListDetails, WaitDetails>
+                 IntExprDetails, VarListDetails, WaitDetails, DeviceTypeDetails>
         Details = std::monostate{};
 
   public:
@@ -209,6 +216,13 @@ public:
       return std::get<VarListDetails>(Details).IsZero;
     }
 
+    ArrayRef<DeviceTypeArgument> getDeviceTypeArchitectures() const {
+      assert((ClauseKind == OpenACCClauseKind::DeviceType ||
+              ClauseKind == OpenACCClauseKind::DType) &&
+             "Only 'device_type'/'dtype' has a device-type-arg list");
+      return std::get<DeviceTypeDetails>(Details).Archs;
+    }
+
     void setLParenLoc(SourceLocation EndLoc) { LParenLoc = EndLoc; }
     void setEndLoc(SourceLocation EndLoc) { ClauseRange.setEnd(EndLoc); }
 
@@ -326,6 +340,13 @@ public:
              "Parsed clause kind does not have a wait-details");
       Details = WaitDetails{DevNum, QueuesLoc, std::move(IntExprs)};
     }
+
+    void setDeviceTypeDetails(llvm::SmallVector<DeviceTypeArgument> &&Archs) {
+      assert((ClauseKind == OpenACCClauseKind::DeviceType ||
+              ClauseKind == OpenACCClauseKind::DType) &&
+             "Only 'device_type'/'dtype' has a device-type-arg list");
+      Details = DeviceTypeDetails{std::move(Archs)};
+    }
   };
 
   SemaOpenACC(Sema &S);
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index ee13437b97b4..f80ecc90d396 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -18,7 +18,8 @@
 using namespace clang;
 
 bool OpenACCClauseWithParams::classof(const OpenACCClause *C) {
-  return OpenACCClauseWithCondition::classof(C) ||
+  return OpenACCDeviceTypeClause::classof(C) ||
+         OpenACCClauseWithCondition::classof(C) ||
          OpenACCClauseWithExprs::classof(C);
 }
 bool OpenACCClauseWithExprs::classof(const OpenACCClause *C) {
@@ -298,6 +299,17 @@ OpenACCCreateClause::Create(const ASTContext &C, OpenACCClauseKind Spelling,
                                        VarList, EndLoc);
 }
 
+OpenACCDeviceTypeClause *OpenACCDeviceTypeClause::Create(
+    const ASTContext &C, OpenACCClauseKind K, SourceLocation BeginLoc,
+    SourceLocation LParenLoc, ArrayRef<DeviceTypeArgument> Archs,
+    SourceLocation EndLoc) {
+  void *Mem =
+      C.Allocate(OpenACCDeviceTypeClause::totalSizeToAlloc<DeviceTypeArgument>(
+          Archs.size()));
+  return new (Mem)
+      OpenACCDeviceTypeClause(K, BeginLoc, LParenLoc, Archs, EndLoc);
+}
+
 //===----------------------------------------------------------------------===//
 //  OpenACC clauses printing methods
 //===----------------------------------------------------------------------===//
@@ -451,3 +463,17 @@ void OpenACCClausePrinter::VisitWaitClause(const OpenACCWaitClause &C) {
     OS << ")";
   }
 }
+
+void OpenACCClausePrinter::VisitDeviceTypeClause(
+    const OpenACCDeviceTypeClause &C) {
+  OS << C.getClauseKind();
+  OS << "(";
+  llvm::interleaveComma(C.getArchitectures(), OS,
+                        [&](const DeviceTypeArgument &Arch) {
+                          if (Arch.first == nullptr)
+                            OS << "*";
+                          else
+                            OS << Arch.first;
+                        });
+  OS << ")";
+}
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 8fb8940142eb..caab4ab0ef16 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2585,6 +2585,9 @@ void OpenACCClauseProfiler::VisitWaitClause(const OpenACCWaitClause &Clause) {
   for (auto *E : Clause.getQueueIdExprs())
     Profiler.VisitStmt(E);
 }
+/// Nothing to do here, there are no sub-statements.
+void OpenACCClauseProfiler::VisitDeviceTypeClause(
+    const OpenACCDeviceTypeClause &Clause) {}
 } // namespace
 
 void StmtProfiler::VisitOpenACCComputeConstruct(
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 12aa5858b798..efcd74717a4e 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -444,6 +444,19 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
       if (cast<OpenACCWaitClause>(C)->hasQueuesTag())
         OS << " has queues tag";
       break;
+    case OpenACCClauseKind::DeviceType:
+    case OpenACCClauseKind::DType:
+      OS << "(";
+      llvm::interleaveComma(
+          cast<OpenACCDeviceTypeClause>(C)->getArchitectures(), OS,
+          [&](const DeviceTypeArgument &Arch) {
+            if (Arch.first == nullptr)
+              OS << "*";
+            else
+              OS << Arch.first->getName();
+          });
+      OS << ")";
+      break;
     default:
       // Nothing to do here.
       break;
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 0e10632c8317..261c9cdc088b 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -711,14 +711,15 @@ bool Parser::ParseOpenACCIntExprList(OpenACCDirectiveKind DK,
 /// device_type( device-type-list )
 ///
 /// The device_type clause may be abbreviated to dtype.
-bool Parser::ParseOpenACCDeviceTypeList() {
+bool Parser::ParseOpenACCDeviceTypeList(
+    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs) {
 
   if (expectIdentifierOrKeyword(*this)) {
     SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
               Parser::StopBeforeMatch);
-    return false;
+    return true;
   }
-  ConsumeToken();
+  Archs.emplace_back(getCurToken().getIdentifierInfo(), ConsumeToken());
 
   while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) {
     ExpectAndConsume(tok::comma);
@@ -726,9 +727,9 @@ bool Parser::ParseOpenACCDeviceTypeList() {
     if (expectIdentifierOrKeyword(*this)) {
       SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
                 Parser::StopBeforeMatch);
-      return false;
+      return true;
     }
-    ConsumeToken();
+    Archs.emplace_back(getCurToken().getIdentifierInfo(), ConsumeToken());
   }
   return false;
 }
@@ -1021,16 +1022,20 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
       break;
     }
     case OpenACCClauseKind::DType:
-    case OpenACCClauseKind::DeviceType:
+    case OpenACCClauseKind::DeviceType: {
+      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> Archs;
       if (getCurToken().is(tok::star)) {
         // FIXME: We want to mark that this is an 'everything else' type of
         // device_type in Sema.
-        ConsumeToken();
-      } else if (ParseOpenACCDeviceTypeList()) {
+        ParsedClause.setDeviceTypeDetails({{nullptr, ConsumeToken()}});
+      } else if (!ParseOpenACCDeviceTypeList(Archs)) {
+        ParsedClause.setDeviceTypeDetails(std::move(Archs));
+      } else {
         Parens.skipToEnd();
         return OpenACCCanContinue();
       }
       break;
+    }
     case OpenACCClauseKind::Tile:
       if (ParseOpenACCSizeExprList()) {
         Parens.skipToEnd();
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index 656d30947a8d..f174b2fa63c6 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -255,6 +255,33 @@ bool checkAlreadyHasClauseOfKind(
   return false;
 }
 
+/// Implement check from OpenACC3.3: section 2.5.4:
+/// Only the async, wait, num_gangs, num_workers, and vector_length clauses may
+/// follow a device_type clause.
+bool checkValidAfterDeviceType(
+    SemaOpenACC &S, const OpenACCDeviceTypeClause &DeviceTypeClause,
+    const SemaOpenACC::OpenACCParsedClause &NewClause) {
+  // This is only a requirement on compute constructs so far, so this is fine
+  // otherwise.
+  if (!isOpenACCComputeDirectiveKind(NewClause.getDirectiveKind()))
+    return false;
+  switch (NewClause.getClauseKind()) {
+  case OpenACCClauseKind::Async:
+  case OpenACCClauseKind::Wait:
+  case OpenACCClauseKind::NumGangs:
+  case OpenACCClauseKind::NumWorkers:
+  case OpenACCClauseKind::VectorLength:
+  case OpenACCClauseKind::DType:
+  case OpenACCClauseKind::DeviceType:
+    return false;
+  default:
+    S.Diag(NewClause.getBeginLoc(), diag::err_acc_clause_after_device_type)
+        << NewClause.getClauseKind() << DeviceTypeClause.getClauseKind();
+    S.Diag(DeviceTypeClause.getBeginLoc(), diag::note_acc_previous_clause_here);
+    return true;
+  }
+}
+
 } // namespace
 
 SemaOpenACC::SemaOpenACC(Sema &S) : SemaBase(S) {}
@@ -273,6 +300,17 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
     return nullptr;
   }
 
+  if (const auto *DevTypeClause =
+          llvm::find_if(ExistingClauses,
+                        [&](const OpenACCClause *C) {
+                          return isa<OpenACCDeviceTypeClause>(C);
+                        });
+      DevTypeClause != ExistingClauses.end()) {
+    if (checkValidAfterDeviceType(
+            *this, *cast<OpenACCDeviceTypeClause>(*DevTypeClause), Clause))
+      return nullptr;
+  }
+
   switch (Clause.getClauseKind()) {
   case OpenACCClauseKind::Default: {
     // Restrictions only properly implemented on 'compute' constructs, and
@@ -651,6 +689,23 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
         Clause.getDevNumExpr(), Clause.getQueuesLoc(), Clause.getQueueIdExprs(),
         Clause.getEndLoc());
   }
+  case OpenACCClauseKind::DType:
+  case OpenACCClauseKind::DeviceType: {
+    // Restrictions only properly implemented on 'compute' constructs, and
+    // 'compute' constructs are the only construct that can do anything with
+    // this yet, so skip/treat as unimplemented in this case.
+    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+      break;
+
+    // TODO OpenACC: Once we get enough of the CodeGen implemented that we have
+    // a source for the list of valid architectures, we need to warn on unknown
+    // identifiers here.
+
+    return OpenACCDeviceTypeClause::Create(
+        getASTContext(), Clause.getClauseKind(), Clause.getBeginLoc(),
+        Clause.getLParenLoc(), Clause.getDeviceTypeArchitectures(),
+        Clause.getEndLoc());
+  }
   default:
     break;
   }
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 126965088831..ab26d1b1199a 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -11480,6 +11480,16 @@ void OpenACCClauseTransform<Derived>::VisitWaitClause(
       ParsedClause.getQueuesLoc(), ParsedClause.getQueueIdExprs(),
       ParsedClause.getEndLoc());
 }
+
+template <typename Derived>
+void OpenACCClauseTransform<Derived>::VisitDeviceTypeClause(
+    const OpenACCDeviceTypeClause &C) {
+  // Nothing to transform here, just create a new version of 'C'.
+  NewClause = OpenACCDeviceTypeClause::Create(
+      Self.getSema().getASTContext(), C.getClauseKind(),
+      ParsedClause.getBeginLoc(), ParsedClause.getLParenLoc(),
+      C.getArchitectures(), ParsedClause.getEndLoc());
+}
 } // namespace
 template <typename Derived>
 OpenACCClause *TreeTransform<Derived>::TransformOpenACCClause(
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 7627996d2c32..8f437a7c5f50 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11905,6 +11905,21 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
                                      DevNumExpr, QueuesLoc, QueueIdExprs,
                                      EndLoc);
   }
+  case OpenACCClauseKind::DeviceType:
+  case OpenACCClauseKind::DType: {
+    SourceLocation LParenLoc = readSourceLocation();
+    llvm::SmallVector<DeviceTypeArgument> Archs;
+    unsigned NumArchs = readInt();
+
+    for (unsigned I = 0; I < NumArchs; ++I) {
+      IdentifierInfo *Ident = readBool() ? readIdentifier() : nullptr;
+      SourceLocation Loc = readSourceLocation();
+      Archs.emplace_back(Ident, Loc);
+    }
+
+    return OpenACCDeviceTypeClause::Create(getContext(), ClauseKind, BeginLoc,
+                                           LParenLoc, Archs, EndLoc);
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -11926,8 +11941,6 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
   case OpenACCClauseKind::Bind:
   case OpenACCClauseKind::DeviceNum:
   case OpenACCClauseKind::DefaultAsync:
-  case OpenACCClauseKind::DeviceType:
-  case OpenACCClauseKind::DType:
   case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Invalid:
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 6154ead589d3..7a9d392889bb 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -7933,6 +7933,19 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeOpenACCIntExprList(WC->getQueueIdExprs());
     return;
   }
+  case OpenACCClauseKind::DeviceType:
+  case OpenACCClauseKind::DType: {
+    const auto *DTC = cast<OpenACCDeviceTypeClause>(C);
+    writeSourceLocation(DTC->getLParenLoc());
+    writeUInt32(DTC->getArchitectures().size());
+    for (const DeviceTypeArgument &Arg : DTC->getArchitectures()) {
+      writeBool(Arg.first);
+      if (Arg.first)
+        AddIdentifierRef(Arg.first);
+      writeSourceLocation(Arg.second);
+    }
+    return;
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -7954,8 +7967,6 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   case OpenACCClauseKind::Bind:
   case OpenACCClauseKind::DeviceNum:
   case OpenACCClauseKind::DefaultAsync:
-  case OpenACCClauseKind::DeviceType:
-  case OpenACCClauseKind::DType:
   case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Invalid:
diff --git a/clang/test/AST/ast-print-openacc-compute-construct.cpp b/clang/test/AST/ast-print-openacc-compute-construct.cpp
index 0bfb90bcb587..cdd9ab3377d0 100644
--- a/clang/test/AST/ast-print-openacc-compute-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-compute-construct.cpp
@@ -107,5 +107,28 @@ void foo() {
 // CHECK: #pragma acc parallel wait(devnum: i : queues: *iPtr, i)
 #pragma acc parallel wait(devnum:i:queues:*iPtr, i)
   while(true);
+
+  bool SomeB;
+  struct SomeStruct{} SomeStructImpl;
+
+//#pragma acc parallel dtype(SomeB)
+#pragma acc parallel dtype(SomeB)
+  while(true);
+
+//#pragma acc parallel device_type(SomeStruct)
+#pragma acc parallel device_type(SomeStruct)
+  while(true);
+
+//#pragma acc parallel device_type(int)
+#pragma acc parallel device_type(int)
+  while(true);
+
+//#pragma acc parallel dtype(bool)
+#pragma acc parallel dtype(bool)
+  while(true);
+
+//#pragma acc parallel device_type (SomeStructImpl)
+#pragma acc parallel device_type (SomeStructImpl)
+  while(true);
 }
 
diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c
index 51858b441e93..694f28b86ec9 100644
--- a/clang/test/ParserOpenACC/parse-clauses.c
+++ b/clang/test/ParserOpenACC/parse-clauses.c
@@ -1126,12 +1126,10 @@ void device_type() {
 #pragma acc parallel dtype(
   {}
 
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel device_type()
   {}
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel dtype()
   {}
 
@@ -1173,12 +1171,10 @@ void device_type() {
 #pragma acc parallel dtype(ident, ident2
   {}
 
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel device_type(ident, ident2,)
   {}
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel dtype(ident, ident2,)
   {}
 
@@ -1200,33 +1196,25 @@ void device_type() {
 #pragma acc parallel dtype(*,ident)
   {}
 
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel device_type(ident, *)
   {}
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel dtype(ident, *)
   {}
 
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel device_type("foo", 54)
   {}
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel dtype(31, "bar")
   {}
 
-  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
 #pragma acc parallel device_type(ident, auto, int, float)
   {}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
 #pragma acc parallel dtype(ident, auto, int, float)
   {}
 
-  // expected-warning@+2{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
 #pragma acc parallel device_type(ident, auto, int, float) dtype(ident, auto, int, float)
   {}
 }
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp b/clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp
new file mode 100644
index 000000000000..8a2423f4f542
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp
@@ -0,0 +1,105 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+struct SomeS{};
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+  SomeS SomeImpl;
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} SomeImpl 'SomeS'
+  // CHECK-NEXT: CXXConstructExpr
+  bool SomeVar;
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} SomeVar 'bool'
+
+#pragma acc parallel device_type(SomeS) dtype(SomeImpl)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(SomeS)
+  // CHECK-NEXT: dtype(SomeImpl)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(SomeVar) dtype(int)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(SomeVar)
+  // CHECK-NEXT: dtype(int)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(private) dtype(struct)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(private)
+  // CHECK-NEXT: dtype(struct)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(private) dtype(class)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(private)
+  // CHECK-NEXT: dtype(class)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(float) dtype(*)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(float)
+  // CHECK-NEXT: dtype(*)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(float, int) dtype(*)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(float, int)
+  // CHECK-NEXT: dtype(*)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+}
+
+template<typename T>
+void TemplUses() {
+  // CHECK-NEXT: FunctionTemplateDecl{{.*}}TemplUses
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}T
+  // CHECK-NEXT: FunctionDecl{{.*}}TemplUses
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(T) dtype(T)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(T)
+  // CHECK-NEXT: dtype(T)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+
+  // Instantiations
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void ()' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(T)
+  // CHECK-NEXT: dtype(T)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+}
+
+void Inst() {
+  TemplUses<int>();
+}
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
new file mode 100644
index 000000000000..15c9cf396c80
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
@@ -0,0 +1,221 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+#define MACRO +FOO
+
+void uses() {
+  typedef struct S{} STy;
+  STy SImpl;
+
+#pragma acc parallel device_type(I)
+  while(1);
+#pragma acc serial device_type(S) dtype(STy)
+  while(1);
+#pragma acc kernels dtype(SImpl)
+  while(1);
+#pragma acc kernels dtype(int) device_type(*)
+  while(1);
+#pragma acc kernels dtype(true) device_type(false)
+  while(1);
+
+  // expected-error@+1{{expected identifier}}
+#pragma acc kernels dtype(int, *)
+  while(1);
+
+#pragma acc parallel device_type(I, int)
+  while(1);
+  // expected-error@+2{{expected ','}}
+  // expected-error@+1{{expected identifier}}
+#pragma acc kernels dtype(int{})
+  while(1);
+  // expected-error@+1{{expected identifier}}
+#pragma acc kernels dtype(5)
+  while(1);
+  // expected-error@+1{{expected identifier}}
+#pragma acc kernels dtype(MACRO)
+  while(1);
+
+
+  // Only 'async', 'wait', num_gangs', 'num_workers', 'vector_length' allowed after 'device_type'.
+
+  // expected-error@+2{{OpenACC clause 'finalize' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) finalize
+  while(1);
+  // expected-error@+2{{OpenACC clause 'if_present' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) if_present
+  while(1);
+  // expected-error@+2{{OpenACC clause 'seq' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) seq
+  while(1);
+  // expected-error@+2{{OpenACC clause 'independent' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) independent
+  while(1);
+  // expected-error@+2{{OpenACC clause 'auto' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) auto
+  while(1);
+  // expected-error@+2{{OpenACC clause 'worker' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) worker
+  while(1);
+  // expected-error@+2{{OpenACC clause 'nohost' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) nohost
+  while(1);
+  // expected-error@+2{{OpenACC clause 'default' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) default(none)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'if' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) if(1)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'self' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) self
+  while(1);
+
+  int Var;
+  int *VarPtr;
+  // expected-error@+2{{OpenACC clause 'copy' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) copy(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'pcopy' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) pcopy(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'present_or_copy' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) present_or_copy(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'use_device' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) use_device(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'attach' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) attach(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'delete' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) delete(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'detach' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) detach(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'device' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) device(VarPtr)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'deviceptr' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) deviceptr(VarPtr)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'device_resident' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*)  device_resident(VarPtr)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'firstprivate' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel device_type(*) firstprivate(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'host' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) host(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'link' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) link(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'no_create' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) no_create(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'present' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) present(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'private' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel device_type(*) private(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'copyout' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) copyout(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'pcopyout' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) pcopyout(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'present_or_copyout' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) present_or_copyout(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'copyin' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) copyin(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'pcopyin' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) pcopyin(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'present_or_copyin' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) present_or_copyin(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'create' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) create(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'pcreate' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) pcreate(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'present_or_create' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) present_or_create(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'reduction' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) reduction(+:Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'collapse' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) collapse(1)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'bind' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) bind(Var)
+  while(1);
+#pragma acc kernels device_type(*) vector_length(1)
+  while(1);
+#pragma acc kernels device_type(*) num_gangs(1)
+  while(1);
+#pragma acc kernels device_type(*) num_workers(1)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'device_num' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) device_num(1)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'default_async' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) default_async(1)
+  while(1);
+#pragma acc kernels device_type(*) async
+  while(1);
+  // expected-error@+2{{OpenACC clause 'tile' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) tile(Var, 1)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'gang' may not follow a 'dtype' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels dtype(*) gang
+  while(1);
+#pragma acc kernels device_type(*) wait
+  while(1);
+}
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp b/clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp
new file mode 100644
index 000000000000..ed40e8bbceae
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+template<typename T>
+void TemplUses() {
+#pragma acc parallel device_type(I)
+  while(true);
+#pragma acc parallel dtype(*)
+  while(true);
+#pragma acc parallel device_type(class)
+  while(true);
+#pragma acc parallel device_type(private)
+  while(true);
+#pragma acc parallel device_type(bool)
+  while(true);
+#pragma acc kernels dtype(true) device_type(false)
+  while(true);
+  // expected-error@+2{{expected ','}}
+  // expected-error@+1{{expected identifier}}
+#pragma acc parallel device_type(T::value)
+  while(true);
+}
+
+void Inst() {
+  TemplUses<int>(); // #INST
+}
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index ae6659fe95e8..8b9417f985b5 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2857,6 +2857,8 @@ void OpenACCClauseEnqueue::VisitWaitClause(const OpenACCWaitClause &C) {
   for (Expr *QE : C.getQueueIdExprs())
     Visitor.AddStmt(QE);
 }
+void OpenACCClauseEnqueue::VisitDeviceTypeClause(
+    const OpenACCDeviceTypeClause &C) {}
 } // namespace
 
 void EnqueueVisitor::EnqueueChildren(const OpenACCClause *C) {
-- 
GitLab


From 67c18721eb2170a6cd7af461e16d994b1b83363a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Mon, 13 May 2024 14:47:21 +0200
Subject: [PATCH 0577/1206] [clang][Interp] Return false from visitExpr() if
 allocateLocal failed

---
 clang/lib/AST/Interp/ByteCodeExprGen.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 630fdb60c351..c0eae5ab20ed 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -2861,7 +2861,8 @@ bool ByteCodeExprGen<Emitter>::visitExpr(const Expr *E) {
     return this->emitRetValue(E) && RootScope.destroyLocals();
   }
 
-  return RootScope.destroyLocals();
+  RootScope.destroyLocals();
+  return false;
 }
 
 /// Toplevel visitDecl().
-- 
GitLab


From 2bd97ba390f9137f26eaf770a90c5f1cb72acbdb Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 13 May 2024 15:27:08 +0100
Subject: [PATCH 0578/1206] [X86] LowerGlobalOrExternal - cleanup SDLoc. NFC.

Don't create a new local SDLoc and then take a reference to it, just create the SDLoc directly.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 0410cc33ca33..bbea2befdcc6 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -18559,7 +18559,7 @@ X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
 SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
                                                  bool ForCall) const {
   // Unpack the global address or external symbol.
-  const SDLoc &dl = SDLoc(Op);
+  SDLoc dl(Op);
   const GlobalValue *GV = nullptr;
   int64_t Offset = 0;
   const char *ExternalSym = nullptr;
-- 
GitLab


From b4393c7d7ea538e92428c7769de590615509f8b4 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 13 May 2024 15:29:50 +0100
Subject: [PATCH 0579/1206] [X86] FP_TO_INTHelper - remove duplicate SDLoc.
 NFC.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index bbea2befdcc6..9d13c043c238 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -20113,7 +20113,7 @@ SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
                                          DAG.getVTList(MVT::Other),
                                          Ops, DstTy, MMO);
 
-  SDValue Res = DAG.getLoad(Op.getValueType(), SDLoc(Op), FIST, StackSlot, MPI);
+  SDValue Res = DAG.getLoad(Op.getValueType(), DL, FIST, StackSlot, MPI);
   Chain = Res.getValue(1);
 
   // If we need an unsigned fixup, XOR the result with adjust.
-- 
GitLab


From b5da0cd68287fa613052a3d3164f2e9de35bedd3 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 13 May 2024 15:36:29 +0100
Subject: [PATCH 0580/1206] [X86] LowerTruncateVecI1 - reuse existing SDLoc.
 NFC.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9d13c043c238..d1be60f18cab 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -20591,14 +20591,12 @@ static SDValue LowerTruncateVecPack(MVT DstVT, SDValue In, const SDLoc &DL,
   return SDValue();
 }
 
-static SDValue LowerTruncateVecI1(SDValue Op, SelectionDAG &DAG,
+static SDValue LowerTruncateVecI1(SDValue Op, const SDLoc &DL,
+                                  SelectionDAG &DAG,
                                   const X86Subtarget &Subtarget) {
-
-  SDLoc DL(Op);
   MVT VT = Op.getSimpleValueType();
   SDValue In = Op.getOperand(0);
   MVT InVT = In.getSimpleValueType();
-
   assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type.");
 
   // Shift LSB to MSB and use VPMOVB/W2M or TESTD/Q.
@@ -20717,7 +20715,7 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
   }
 
   if (VT.getVectorElementType() == MVT::i1)
-    return LowerTruncateVecI1(Op, DAG, Subtarget);
+    return LowerTruncateVecI1(Op, DL, DAG, Subtarget);
 
   // Attempt to truncate with PACKUS/PACKSS even on AVX512 if we'd have to
   // concat from subvectors to use VPTRUNC etc.
-- 
GitLab


From 0cc60235d178ebe18db14b9731aa01c24abd1c66 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 13 May 2024 15:40:55 +0100
Subject: [PATCH 0581/1206] [X86] LowerFP_TO_INT - remove duplicate SDLoc. NFC.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index d1be60f18cab..bf1f3f335771 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -21171,8 +21171,8 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
       LC = RTLIB::getFPTOUINT(SrcVT, VT);
 
     MakeLibCallOptions CallOptions;
-    std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, LC, VT, Src, CallOptions,
-                                                  SDLoc(Op), Chain);
+    std::pair<SDValue, SDValue> Tmp =
+        makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
 
     if (IsStrict)
       return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl);
-- 
GitLab


From b7e4a8a08ecf73d2b34dca1b71d08d622da58bdc Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 13 May 2024 15:51:03 +0100
Subject: [PATCH 0582/1206] [X86] LowerIntVSETCC_AVX512 - reuse existing SDLoc.
 NFC.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index bf1f3f335771..ecc5b3b3bf84 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -23104,14 +23104,12 @@ static SDValue splitIntVSETCC(EVT VT, SDValue LHS, SDValue RHS,
                      DAG.getNode(ISD::SETCC, dl, HiVT, LHS2, RHS2, CC));
 }
 
-static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) {
-
+static SDValue LowerIntVSETCC_AVX512(SDValue Op, const SDLoc &dl,
+                                     SelectionDAG &DAG) {
   SDValue Op0 = Op.getOperand(0);
   SDValue Op1 = Op.getOperand(1);
   SDValue CC = Op.getOperand(2);
   MVT VT = Op.getSimpleValueType();
-  SDLoc dl(Op);
-
   assert(VT.getVectorElementType() == MVT::i1 &&
          "Cannot set masked compare for this operation");
 
@@ -23387,7 +23385,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
     // But there is no compare instruction for i8 and i16 elements in KNL.
     assert((VTOp0.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()) &&
            "Unexpected operand type");
-    return LowerIntVSETCC_AVX512(Op, DAG);
+    return LowerIntVSETCC_AVX512(Op, dl, DAG);
   }
 
   // Lower using XOP integer comparisons.
-- 
GitLab


From 999fb097102f75feac1601c061b8bdf0a4cf6112 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 13 May 2024 15:02:49 +0000
Subject: [PATCH 0583/1206] [gn build] Port 05cc2d5fe10c

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 9dc06399877f..210b26e8f166 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -516,6 +516,7 @@ if (current_toolchain == default_toolchain) {
       "__ios/fpos.h",
       "__iterator/access.h",
       "__iterator/advance.h",
+      "__iterator/aliasing_iterator.h",
       "__iterator/back_insert_iterator.h",
       "__iterator/bounded_iter.h",
       "__iterator/common_iterator.h",
-- 
GitLab


From 9f858c7b79f9edff082050b930fee347887f8e6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Mon, 13 May 2024 16:12:09 +0100
Subject: [PATCH 0584/1206] [mlir][vector][test] Update tests for
 vector.xfter_{read|write} (#91943)

Updates tests in "vector-transfer-permutation-lowering.mlir" to make a
clearer split into cases for :

* xfer_read vs xfer_write
* fixed-width vs scalable tests

A new test case is added for fixed-width vectors for vector.transfer_read.
This is to complement an existing test for scalable vectors.

This is in preparation for #90835 and also for adding more tests for
scalable vectors.
---
 .../vector-transfer-permutation-lowering.mlir | 94 ++++++++++++++-----
 1 file changed, 68 insertions(+), 26 deletions(-)

diff --git a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir
index 31bd19c0be8e..e48af3cd7aac 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir
@@ -1,23 +1,84 @@
 // RUN: mlir-opt %s --transform-interpreter --split-input-file | FileCheck %s
 
-// CHECK-LABEL: func @lower_permutation_with_mask_fixed_width(
+///----------------------------------------------------------------------------------------
+/// vector.transfer_write
+///----------------------------------------------------------------------------------------
+/// Input: 
+///   * vector.transfer_write op with a map which _is not_ the permutation of a
+///     minor identity
+/// Output:
+///   * vector.broadcast + vector.transfer_write with a map which _is_ the permutation of a
+///     minor identity
+
+// CHECK-LABEL: func @permutation_with_mask_xfer_write_fixed_width(
 //       CHECK:   %[[vec:.*]] = arith.constant dense<-2.000000e+00> : vector<7x1xf32>
 //       CHECK:   %[[mask:.*]] = arith.constant dense<[true, false, true, false, true, true, true]> : vector<7xi1>
 //       CHECK:   %[[b:.*]] = vector.broadcast %[[mask]] : vector<7xi1> to vector<1x7xi1>
 //       CHECK:   %[[tp:.*]] = vector.transpose %[[b]], [1, 0] : vector<1x7xi1> to vector<7x1xi1>
 //       CHECK:   vector.transfer_write %[[vec]], %{{.*}}[%{{.*}}, %{{.*}}], %[[tp]] {in_bounds = [false, true]} : vector<7x1xf32>, memref<?x?xf32>
-func.func @lower_permutation_with_mask_fixed_width(%A : memref<?x?xf32>, %base1 : index,
-                                       %base2 : index) {
+func.func @permutation_with_mask_xfer_write_fixed_width(%mem : memref<?x?xf32>, %base1 : index,
+                                                   %base2 : index) {
+
   %fn1 = arith.constant -2.0 : f32
   %vf0 = vector.splat %fn1 : vector<7xf32>
   %mask = arith.constant dense<[1, 0, 1, 0, 1, 1, 1]> : vector<7xi1>
-  vector.transfer_write %vf0, %A[%base1, %base2], %mask
+  vector.transfer_write %vf0, %mem[%base1, %base2], %mask
     {permutation_map = affine_map<(d0, d1) -> (d0)>, in_bounds = [false]}
     : vector<7xf32>, memref<?x?xf32>
   return
 }
 
-// CHECK-LABEL:   func.func @permutation_with_mask_scalable(
+// CHECK:           func.func @permutation_with_mask_xfer_write_scalable(
+// CHECK-SAME:        %[[ARG_0:.*]]: vector<4x[8]xi16>,
+// CHECK-SAME:        %[[ARG_1:.*]]: memref<1x4x?x1xi16>,
+// CHECK-SAME:        %[[MASK:.*]]: vector<4x[8]xi1>) {
+// CHECK:             %[[C0:.*]] = arith.constant 0 : index
+// CHECK:             %[[BCAST_1:.*]] = vector.broadcast %[[ARG_0]] : vector<4x[8]xi16> to vector<1x4x[8]xi16>
+// CHECK:             %[[BCAST_2:.*]] = vector.broadcast %[[MASK]] : vector<4x[8]xi1> to vector<1x4x[8]xi1>
+// CHECK:             %[[TRANSPOSE_1:.*]] =  vector.transpose %[[BCAST_2]], [1, 2, 0] : vector<1x4x[8]xi1> to vector<4x[8]x1xi1>
+// CHECK:             %[[TRANSPOSE_2:.*]] =  vector.transpose %[[BCAST_1]], [1, 2, 0] : vector<1x4x[8]xi16> to vector<4x[8]x1xi16>
+// CHECK:             vector.transfer_write %[[TRANSPOSE_2]], %[[ARG_1]]{{.*}}, %[[TRANSPOSE_1]] {in_bounds = [true, true, true]} : vector<4x[8]x1xi16>, memref<1x4x?x1xi16>
+func.func @permutation_with_mask_xfer_write_scalable(%arg0: vector<4x[8]xi16>, %mem: memref<1x4x?x1xi16>, %mask:  vector<4x[8]xi1>){
+     %c0 = arith.constant 0 : index
+      vector.transfer_write %arg0, %mem[%c0, %c0, %c0, %c0], %mask {in_bounds = [true, true], permutation_map = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+} : vector<4x[8]xi16>, memref<1x4x?x1xi16>
+
+    return
+}
+
+///----------------------------------------------------------------------------------------
+/// vector.transfer_read
+///----------------------------------------------------------------------------------------
+/// Input: 
+///   * vector.transfer_read op with a permutation map
+/// Output:
+///   * vector.transfer_read with a permutation map composed of leading zeros followed by a minor identiy +
+///     vector.transpose op
+
+// CHECK-LABEL:   func.func @permutation_with_mask_xfer_read_fixed_width(
+// CHECK-SAME:      %[[ARG_0:.*]]: memref<?x?xf32>,
+// CHECK-SAME:      %[[IDX_1:.*]]: index,
+// CHECK-SAME:      %[[IDX_2:.*]]: index) -> vector<8x4x2xf32> {
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[PASS_THROUGH:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[MASK:.*]] = vector.create_mask %[[IDX_2]], %[[IDX_1]] : vector<2x4xi1>
+// CHECK:           %[[T_READ:.*]] = vector.transfer_read %[[ARG_0]]{{\[}}%[[C0]], %[[C0]]], %[[PASS_THROUGH]], %[[MASK]] {in_bounds = [true, true]} : memref<?x?xf32>, vector<2x4xf32>
+// CHECK:           %[[BCAST:.*]] = vector.broadcast %[[T_READ]] : vector<2x4xf32> to vector<8x2x4xf32>
+// CHECK:           %[[TRANSPOSE:.*]] = vector.transpose %[[BCAST]], [0, 2, 1] : vector<8x2x4xf32> to vector<8x4x2xf32>
+// CHECK:           return %[[TRANSPOSE]] : vector<8x4x2xf32>
+func.func @permutation_with_mask_xfer_read_fixed_width(%mem: memref<?x?xf32>, %dim_1: index, %dim_2: index) -> (vector<8x4x2xf32>) {
+
+  %c0 = arith.constant 0 : index
+  %cst_0 = arith.constant 0.000000e+00 : f32
+
+  %mask = vector.create_mask %dim_2, %dim_1 : vector<2x4xi1>
+  %1 = vector.transfer_read %mem[%c0, %c0], %cst_0, %mask
+    {in_bounds = [true, true, true], permutation_map = affine_map<(d0, d1) -> (0, d1, d0)>}
+    : memref<?x?xf32>, vector<8x4x2xf32>
+  return %1 : vector<8x4x2xf32>
+}
+
+// CHECK-LABEL:   func.func @permutation_with_mask_xfer_read_scalable(
 // CHECK-SAME:      %[[ARG_0:.*]]: memref<?x?xf32>,
 // CHECK-SAME:      %[[IDX_1:.*]]: index,
 // CHECK-SAME:      %[[IDX_2:.*]]: index) -> vector<8x[4]x2xf32> {
@@ -28,37 +89,18 @@ func.func @lower_permutation_with_mask_fixed_width(%A : memref<?x?xf32>, %base1
 // CHECK:           %[[BCAST:.*]] = vector.broadcast %[[T_READ]] : vector<2x[4]xf32> to vector<8x2x[4]xf32>
 // CHECK:           %[[TRANSPOSE:.*]] = vector.transpose %[[BCAST]], [0, 2, 1] : vector<8x2x[4]xf32> to vector<8x[4]x2xf32>
 // CHECK:           return %[[TRANSPOSE]] : vector<8x[4]x2xf32>
-// CHECK:         }
-func.func @permutation_with_mask_scalable(%2: memref<?x?xf32>, %dim_1: index, %dim_2: index) -> (vector<8x[4]x2xf32>) {
+func.func @permutation_with_mask_xfer_read_scalable(%mem: memref<?x?xf32>, %dim_1: index, %dim_2: index) -> (vector<8x[4]x2xf32>) {
 
   %c0 = arith.constant 0 : index
   %cst_0 = arith.constant 0.000000e+00 : f32
 
   %mask = vector.create_mask %dim_2, %dim_1 : vector<2x[4]xi1>
-  %1 = vector.transfer_read %2[%c0, %c0], %cst_0, %mask
+  %1 = vector.transfer_read %mem[%c0, %c0], %cst_0, %mask
     {in_bounds = [true, true, true], permutation_map = affine_map<(d0, d1) -> (0, d1, d0)>}
     : memref<?x?xf32>, vector<8x[4]x2xf32>
   return %1 : vector<8x[4]x2xf32>
 }
 
-// CHECK:           func.func @permutation_with_mask_transfer_write_scalable(
-// CHECK-SAME:        %[[ARG_0:.*]]: vector<4x[8]xi16>,
-// CHECK-SAME:        %[[ARG_1:.*]]: memref<1x4x?x1x1x1x1xi16>,
-// CHECK-SAME:        %[[MASK:.*]]: vector<4x[8]xi1>) {
-// CHECK:             %[[C0:.*]] = arith.constant 0 : index
-// CHECK:             %[[BCAST_1:.*]] = vector.broadcast %[[ARG_0]] : vector<4x[8]xi16> to vector<1x1x1x1x4x[8]xi16>
-// CHECK:             %[[BCAST_2:.*]] = vector.broadcast %[[MASK]] : vector<4x[8]xi1> to vector<1x1x1x1x4x[8]xi1>
-// CHECK:             %[[TRANSPOSE_1:.*]] = vector.transpose %[[BCAST_2]], [4, 5, 0, 1, 2, 3] : vector<1x1x1x1x4x[8]xi1> to vector<4x[8]x1x1x1x1xi1>
-// CHECK:             %[[TRANSPOSE_2:.*]] = vector.transpose %[[BCAST_1]], [4, 5, 0, 1, 2, 3] : vector<1x1x1x1x4x[8]xi16> to vector<4x[8]x1x1x1x1xi16>
-// CHECK:             vector.transfer_write %[[TRANSPOSE_2]], %[[ARG_1]]{{\[}}%[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]]], %[[TRANSPOSE_1]] {in_bounds = [true, true, true, true, true, true]} : vector<4x[8]x1x1x1x1xi16>, memref<1x4x?x1x1x1x1xi16>
-// CHECK:             return
-func.func @permutation_with_mask_transfer_write_scalable(%arg0: vector<4x[8]xi16>, %arg1: memref<1x4x?x1x1x1x1xi16>, %mask:  vector<4x[8]xi1>){
-     %c0 = arith.constant 0 : index
-      vector.transfer_write %arg0, %arg1[%c0, %c0, %c0, %c0, %c0, %c0, %c0], %mask {in_bounds = [true, true], permutation_map = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d1, d2)>
-} : vector<4x[8]xi16>, memref<1x4x?x1x1x1x1xi16>
-
-    return
-}
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
     %f = transform.structured.match ops{["func.func"]} in %module_op
-- 
GitLab


From 257013e4f5cbdf644646da9ec3d60d6209c9bf25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Mon, 13 May 2024 16:01:55 +0200
Subject: [PATCH 0585/1206] [clang][Interp] Handle VariableArrayTypes

---
 clang/lib/AST/Interp/ByteCodeExprGen.cpp | 4 +++-
 clang/lib/AST/Interp/Program.cpp         | 3 ++-
 clang/test/AST/Interp/arrays.cpp         | 5 +++++
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index c0eae5ab20ed..7b10482dff23 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -2159,7 +2159,9 @@ bool ByteCodeExprGen<Emitter>::VisitCXXConstructExpr(
   if (T->isArrayType()) {
     const ConstantArrayType *CAT =
         Ctx.getASTContext().getAsConstantArrayType(E->getType());
-    assert(CAT);
+    if (!CAT)
+      return false;
+
     size_t NumElems = CAT->getZExtSize();
     const Function *Func = getFunction(E->getConstructor());
     if (!Func || !Func->isConstexpr())
diff --git a/clang/lib/AST/Interp/Program.cpp b/clang/lib/AST/Interp/Program.cpp
index 87c767f85e79..6606149f1f69 100644
--- a/clang/lib/AST/Interp/Program.cpp
+++ b/clang/lib/AST/Interp/Program.cpp
@@ -386,7 +386,8 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
 
     // Array of unknown bounds - cannot be accessed and pointer arithmetic
     // is forbidden on pointers to such objects.
-    if (isa<IncompleteArrayType>(ArrayType)) {
+    if (isa<IncompleteArrayType>(ArrayType) ||
+        isa<VariableArrayType>(ArrayType)) {
       if (std::optional<PrimType> T = Ctx.classify(ElemTy)) {
         return allocateDescriptor(D, *T, MDSize, IsTemporary,
                                   Descriptor::UnknownSize{});
diff --git a/clang/test/AST/Interp/arrays.cpp b/clang/test/AST/Interp/arrays.cpp
index abfcd09338ca..70e87c4cd854 100644
--- a/clang/test/AST/Interp/arrays.cpp
+++ b/clang/test/AST/Interp/arrays.cpp
@@ -560,6 +560,11 @@ namespace LocalVLA {
      // both-note@-4 {{function parameter 'size' with unknown value}}
 #endif
   }
+
+  void f (unsigned int m) {
+    int e[2][m];
+    e[0][0] = 0;
+  }
 }
 
 char melchizedek[2];
-- 
GitLab


From 1f6f5bf9307033980121fc534815a7b59453e122 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Mon, 13 May 2024 08:23:19 -0700
Subject: [PATCH 0586/1206] Fix warning from c4a9a374

I wasn't able to reproduce the test crash, but I believe this might be a
different definition of 'assert' on some platforms, so I believe this
patch should fix it (and fixes the suggested warning).
---
 clang/include/clang/AST/OpenACCClause.h | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 3998a3430ff9..607a2b9d6536 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -101,13 +101,13 @@ class OpenACCDeviceTypeClause final
       return Arg.second.isInvalid();
     }) && "Invalid SourceLocation for an argument");
 
-    assert(Archs.size() == 1 ||
-           !llvm::any_of(Archs,
-                         [](const DeviceTypeArgument &Arg) {
-                           return Arg.first == nullptr;
-                         }) &&
-               "Only a single asterisk version is permitted, and must be the "
-               "only one");
+    assert(
+        (Archs.size() == 1 || !llvm::any_of(Archs,
+                                            [](const DeviceTypeArgument &Arg) {
+                                              return Arg.first == nullptr;
+                                            })) &&
+        "Only a single asterisk version is permitted, and must be the "
+        "only one");
 
     std::uninitialized_copy(Archs.begin(), Archs.end(),
                             getTrailingObjects<DeviceTypeArgument>());
-- 
GitLab


From be7c9e39572d876c16b6a8d7f4addaf9409071ff Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Mon, 13 May 2024 08:09:24 -0500
Subject: [PATCH 0587/1206] [flang][OpenMP] Decompose compound constructs, do
 recursive lowering (#90098)

A compound construct with a list of clauses is broken up into individual
leaf/composite constructs. Each such construct has the list of clauses
that apply to it based on the OpenMP spec.

Each lowering function (i.e. a function that generates MLIR ops) is now
responsible for generating its body as described below.

Functions that receive AST nodes extract the construct, and the clauses
from the node. They then create a work queue consisting of individual
constructs, and invoke a common dispatch function to process (lower) the
queue.

The dispatch function examines the current position in the queue, and
invokes the appropriate lowering function. Each lowering function
receives the queue as well, and once it needs to generate its body, it
either invokes the dispatch function on the rest of the queue (if any),
or processes nested evaluations if the work queue is at the end.

Re-application of ca1bd5995f6ed934f9187305190a5abfac049173 with fixes for
compilation errors.
---
 flang/lib/Lower/CMakeLists.txt                |    1 +
 flang/lib/Lower/OpenMP/Clauses.cpp            |   23 +
 flang/lib/Lower/OpenMP/Clauses.h              |   17 +-
 flang/lib/Lower/OpenMP/Decomposer.cpp         |  126 ++
 flang/lib/Lower/OpenMP/Decomposer.h           |   51 +
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  828 ++++++------
 flang/lib/Lower/OpenMP/Utils.cpp              |    6 -
 flang/lib/Lower/OpenMP/Utils.h                |    1 -
 .../Lower/OpenMP/default-clause-byref.f90     |    5 +-
 flang/test/Lower/OpenMP/default-clause.f90    |    4 +-
 .../parallel-lastprivate-clause-scalar.f90    |    4 +-
 llvm/include/llvm/Frontend/OpenMP/ClauseT.h   |   58 +-
 .../Frontend/OpenMP/ConstructCompositionT.h   |  403 ++++++
 .../Frontend/OpenMP/ConstructDecompositionT.h | 1160 +++++++++++++++++
 llvm/unittests/Frontend/CMakeLists.txt        |    1 +
 .../Frontend/OpenMPDecompositionTest.cpp      |  999 ++++++++++++++
 16 files changed, 3234 insertions(+), 453 deletions(-)
 create mode 100644 flang/lib/Lower/OpenMP/Decomposer.cpp
 create mode 100644 flang/lib/Lower/OpenMP/Decomposer.h
 create mode 100644 llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
 create mode 100644 llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
 create mode 100644 llvm/unittests/Frontend/OpenMPDecompositionTest.cpp

diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index f92d1a2bc7de..1546409752e7 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -27,6 +27,7 @@ add_flang_library(FortranLower
   OpenMP/ClauseProcessor.cpp
   OpenMP/Clauses.cpp
   OpenMP/DataSharingProcessor.cpp
+  OpenMP/Decomposer.cpp
   OpenMP/OpenMP.cpp
   OpenMP/ReductionProcessor.cpp
   OpenMP/Utils.cpp
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 97337cfc08c7..87370c92964a 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1227,4 +1227,27 @@ List<Clause> makeClauses(const parser::OmpClauseList &clauses,
     return makeClause(s, semaCtx);
   });
 }
+
+bool transferLocations(const List<Clause> &from, List<Clause> &to) {
+  bool allDone = true;
+
+  for (Clause &clause : to) {
+    if (!clause.source.empty())
+      continue;
+    auto found =
+        llvm::find_if(from, [&](const Clause &c) { return c.id == clause.id; });
+    // This is not completely accurate, but should be good enough for now.
+    // It can be improved in the future if necessary, but in cases of
+    // synthesized clauses getting accurate location may be impossible.
+    if (found != from.end()) {
+      clause.source = found->source;
+    } else {
+      // Found a clause that won't have "source".
+      allDone = false;
+    }
+  }
+
+  return allDone;
+}
+
 } // namespace Fortran::lower::omp
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index 3e776425c733..ca610c652896 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -23,11 +23,15 @@
 
 namespace Fortran::lower::omp {
 using namespace Fortran;
-using SomeType = evaluate::SomeType;
 using SomeExpr = semantics::SomeExpr;
 using MaybeExpr = semantics::MaybeExpr;
 
-using TypeTy = SomeType;
+// evaluate::SomeType doesn't provide == operation. It's not really used in
+// flang's clauses so far, so a trivial implementation is sufficient.
+struct TypeTy : public evaluate::SomeType {
+  bool operator==(const TypeTy &t) const { return true; }
+};
+
 using IdTy = semantics::Symbol *;
 using ExprTy = SomeExpr;
 
@@ -222,6 +226,8 @@ using When = tomp::clause::WhenT<TypeTy, IdTy, ExprTy>;
 using Write = tomp::clause::WriteT<TypeTy, IdTy, ExprTy>;
 } // namespace clause
 
+using tomp::type::operator==;
+
 struct CancellationConstructType {
   using EmptyTrait = std::true_type;
 };
@@ -244,13 +250,16 @@ using ClauseBase = tomp::ClauseT<TypeTy, IdTy, ExprTy,
                                  MemoryOrder, Threadprivate>;
 
 struct Clause : public ClauseBase {
+  Clause(ClauseBase &&base, const parser::CharBlock source = {})
+      : ClauseBase(std::move(base)), source(source) {}
+  // "source" will be ignored by tomp::type::operator==.
   parser::CharBlock source;
 };
 
 template <typename Specific>
 Clause makeClause(llvm::omp::Clause id, Specific &&specific,
                   parser::CharBlock source = {}) {
-  return Clause{{id, specific}, source};
+  return Clause(typename Clause::BaseT{id, specific}, source);
 }
 
 Clause makeClause(const Fortran::parser::OmpClause &cls,
@@ -258,6 +267,8 @@ Clause makeClause(const Fortran::parser::OmpClause &cls,
 
 List<Clause> makeClauses(const parser::OmpClauseList &clauses,
                          semantics::SemanticsContext &semaCtx);
+
+bool transferLocations(const List<Clause> &from, List<Clause> &to);
 } // namespace Fortran::lower::omp
 
 #endif // FORTRAN_LOWER_OPENMP_CLAUSES_H
diff --git a/flang/lib/Lower/OpenMP/Decomposer.cpp b/flang/lib/Lower/OpenMP/Decomposer.cpp
new file mode 100644
index 000000000000..e6897cb81e94
--- /dev/null
+++ b/flang/lib/Lower/OpenMP/Decomposer.cpp
@@ -0,0 +1,126 @@
+//===-- Decomposer.cpp -- Compound directive decomposition ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "Decomposer.h"
+
+#include "Clauses.h"
+#include "Utils.h"
+#include "flang/Lower/PFTBuilder.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Tools/CrossToolHelpers.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Frontend/OpenMP/ClauseT.h"
+#include "llvm/Frontend/OpenMP/ConstructCompositionT.h"
+#include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <optional>
+#include <utility>
+#include <variant>
+
+using namespace Fortran;
+
+namespace {
+using namespace Fortran::lower::omp;
+
+struct ConstructDecomposition {
+  ConstructDecomposition(mlir::ModuleOp modOp,
+                         semantics::SemanticsContext &semaCtx,
+                         lower::pft::Evaluation &ev,
+                         llvm::omp::Directive compound,
+                         const List<Clause> &clauses)
+      : semaCtx(semaCtx), mod(modOp), eval(ev) {
+    tomp::ConstructDecompositionT decompose(getOpenMPVersionAttribute(modOp),
+                                            *this, compound,
+                                            llvm::ArrayRef(clauses));
+    output = std::move(decompose.output);
+  }
+
+  // Given an object, return its base object if one exists.
+  std::optional<Object> getBaseObject(const Object &object) {
+    return lower::omp::getBaseObject(object, semaCtx);
+  }
+
+  // Return the iteration variable of the associated loop if any.
+  std::optional<Object> getLoopIterVar() {
+    if (semantics::Symbol *symbol = getIterationVariableSymbol(eval))
+      return Object{symbol, /*designator=*/{}};
+    return std::nullopt;
+  }
+
+  semantics::SemanticsContext &semaCtx;
+  mlir::ModuleOp mod;
+  lower::pft::Evaluation &eval;
+  List<UnitConstruct> output;
+};
+} // namespace
+
+static UnitConstruct mergeConstructs(uint32_t version,
+                                     llvm::ArrayRef<UnitConstruct> units) {
+  tomp::ConstructCompositionT compose(version, units);
+  return compose.merged;
+}
+
+namespace Fortran::lower::omp {
+LLVM_DUMP_METHOD llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
+                                               const UnitConstruct &uc) {
+  os << llvm::omp::getOpenMPDirectiveName(uc.id);
+  for (auto [index, clause] : llvm::enumerate(uc.clauses)) {
+    os << (index == 0 ? '\t' : ' ');
+    os << llvm::omp::getOpenMPClauseName(clause.id);
+  }
+  return os;
+}
+
+ConstructQueue buildConstructQueue(
+    mlir::ModuleOp modOp, Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::pft::Evaluation &eval, const parser::CharBlock &source,
+    llvm::omp::Directive compound, const List<Clause> &clauses) {
+
+  List<UnitConstruct> constructs;
+
+  ConstructDecomposition decompose(modOp, semaCtx, eval, compound, clauses);
+  assert(!decompose.output.empty() && "Construct decomposition failed");
+
+  llvm::SmallVector<llvm::omp::Directive> loweringUnits;
+  std::ignore =
+      llvm::omp::getLeafOrCompositeConstructs(compound, loweringUnits);
+  uint32_t version = getOpenMPVersionAttribute(modOp);
+
+  int leafIndex = 0;
+  for (llvm::omp::Directive dir_id : loweringUnits) {
+    llvm::ArrayRef<llvm::omp::Directive> leafsOrSelf =
+        llvm::omp::getLeafConstructsOrSelf(dir_id);
+    size_t numLeafs = leafsOrSelf.size();
+
+    llvm::ArrayRef<UnitConstruct> toMerge{&decompose.output[leafIndex],
+                                          numLeafs};
+    auto &uc = constructs.emplace_back(mergeConstructs(version, toMerge));
+
+    if (!transferLocations(clauses, uc.clauses)) {
+      // If some clauses are left without source information, use the
+      // directive's source.
+      for (auto &clause : uc.clauses) {
+        if (clause.source.empty())
+          clause.source = source;
+      }
+    }
+    leafIndex += numLeafs;
+  }
+
+  return constructs;
+}
+} // namespace Fortran::lower::omp
diff --git a/flang/lib/Lower/OpenMP/Decomposer.h b/flang/lib/Lower/OpenMP/Decomposer.h
new file mode 100644
index 000000000000..f42d8f5c1740
--- /dev/null
+++ b/flang/lib/Lower/OpenMP/Decomposer.h
@@ -0,0 +1,51 @@
+//===-- Decomposer.h -- Compound directive decomposition ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef FORTRAN_LOWER_OPENMP_DECOMPOSER_H
+#define FORTRAN_LOWER_OPENMP_DECOMPOSER_H
+
+#include "Clauses.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "llvm/Frontend/OpenMP/ConstructCompositionT.h"
+#include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+class raw_ostream;
+}
+
+namespace Fortran {
+namespace semantics {
+class SemanticsContext;
+}
+namespace lower::pft {
+struct Evaluation;
+}
+} // namespace Fortran
+
+namespace Fortran::lower::omp {
+using UnitConstruct = tomp::DirectiveWithClauses<lower::omp::Clause>;
+using ConstructQueue = List<UnitConstruct>;
+
+LLVM_DUMP_METHOD llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
+                                               const UnitConstruct &uc);
+
+// Given a potentially compound construct with a list of clauses that
+// apply to it, break it up into individual sub-constructs each with
+// the subset of applicable clauses (plus implicit clauses, if any).
+// From that create a work queue where each work item corresponds to
+// the sub-construct with its clauses.
+ConstructQueue buildConstructQueue(mlir::ModuleOp modOp,
+                                   semantics::SemanticsContext &semaCtx,
+                                   lower::pft::Evaluation &eval,
+                                   const parser::CharBlock &source,
+                                   llvm::omp::Directive compound,
+                                   const List<Clause> &clauses);
+} // namespace Fortran::lower::omp
+
+#endif // FORTRAN_LOWER_OPENMP_DECOMPOSER_H
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index f23902d6a823..eaf4b5f997ff 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -15,6 +15,7 @@
 #include "ClauseProcessor.h"
 #include "Clauses.h"
 #include "DataSharingProcessor.h"
+#include "Decomposer.h"
 #include "DirectivesCommon.h"
 #include "ReductionProcessor.h"
 #include "Utils.h"
@@ -44,6 +45,13 @@ using namespace Fortran::lower::omp;
 // Code generation helper functions
 //===----------------------------------------------------------------------===//
 
+static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
+                           Fortran::lower::SymMap &symTable,
+                           Fortran::semantics::SemanticsContext &semaCtx,
+                           Fortran::lower::pft::Evaluation &eval,
+                           mlir::Location loc, const ConstructQueue &queue,
+                           ConstructQueue::iterator item);
+
 static Fortran::lower::pft::Evaluation *
 getCollapsedLoopEval(Fortran::lower::pft::Evaluation &eval, int collapseValue) {
   // Return the Evaluation of the innermost collapsed loop, or the current one
@@ -460,81 +468,6 @@ markDeclareTarget(mlir::Operation *op,
   declareTargetOp.setDeclareTarget(deviceType, captureClause);
 }
 
-/// Split a combined directive into an outer leaf directive and the (possibly
-/// combined) rest of the combined directive. Composite directives and
-/// non-compound directives are not split, in which case it will return the
-/// input directive as its first output and an empty value as its second output.
-static std::pair<llvm::omp::Directive, std::optional<llvm::omp::Directive>>
-splitCombinedDirective(llvm::omp::Directive dir) {
-  using D = llvm::omp::Directive;
-  switch (dir) {
-  case D::OMPD_masked_taskloop:
-    return {D::OMPD_masked, D::OMPD_taskloop};
-  case D::OMPD_masked_taskloop_simd:
-    return {D::OMPD_masked, D::OMPD_taskloop_simd};
-  case D::OMPD_master_taskloop:
-    return {D::OMPD_master, D::OMPD_taskloop};
-  case D::OMPD_master_taskloop_simd:
-    return {D::OMPD_master, D::OMPD_taskloop_simd};
-  case D::OMPD_parallel_do:
-    return {D::OMPD_parallel, D::OMPD_do};
-  case D::OMPD_parallel_do_simd:
-    return {D::OMPD_parallel, D::OMPD_do_simd};
-  case D::OMPD_parallel_masked:
-    return {D::OMPD_parallel, D::OMPD_masked};
-  case D::OMPD_parallel_masked_taskloop:
-    return {D::OMPD_parallel, D::OMPD_masked_taskloop};
-  case D::OMPD_parallel_masked_taskloop_simd:
-    return {D::OMPD_parallel, D::OMPD_masked_taskloop_simd};
-  case D::OMPD_parallel_master:
-    return {D::OMPD_parallel, D::OMPD_master};
-  case D::OMPD_parallel_master_taskloop:
-    return {D::OMPD_parallel, D::OMPD_master_taskloop};
-  case D::OMPD_parallel_master_taskloop_simd:
-    return {D::OMPD_parallel, D::OMPD_master_taskloop_simd};
-  case D::OMPD_parallel_sections:
-    return {D::OMPD_parallel, D::OMPD_sections};
-  case D::OMPD_parallel_workshare:
-    return {D::OMPD_parallel, D::OMPD_workshare};
-  case D::OMPD_target_parallel:
-    return {D::OMPD_target, D::OMPD_parallel};
-  case D::OMPD_target_parallel_do:
-    return {D::OMPD_target, D::OMPD_parallel_do};
-  case D::OMPD_target_parallel_do_simd:
-    return {D::OMPD_target, D::OMPD_parallel_do_simd};
-  case D::OMPD_target_simd:
-    return {D::OMPD_target, D::OMPD_simd};
-  case D::OMPD_target_teams:
-    return {D::OMPD_target, D::OMPD_teams};
-  case D::OMPD_target_teams_distribute:
-    return {D::OMPD_target, D::OMPD_teams_distribute};
-  case D::OMPD_target_teams_distribute_parallel_do:
-    return {D::OMPD_target, D::OMPD_teams_distribute_parallel_do};
-  case D::OMPD_target_teams_distribute_parallel_do_simd:
-    return {D::OMPD_target, D::OMPD_teams_distribute_parallel_do_simd};
-  case D::OMPD_target_teams_distribute_simd:
-    return {D::OMPD_target, D::OMPD_teams_distribute_simd};
-  case D::OMPD_teams_distribute:
-    return {D::OMPD_teams, D::OMPD_distribute};
-  case D::OMPD_teams_distribute_parallel_do:
-    return {D::OMPD_teams, D::OMPD_distribute_parallel_do};
-  case D::OMPD_teams_distribute_parallel_do_simd:
-    return {D::OMPD_teams, D::OMPD_distribute_parallel_do_simd};
-  case D::OMPD_teams_distribute_simd:
-    return {D::OMPD_teams, D::OMPD_distribute_simd};
-  case D::OMPD_parallel_loop:
-    return {D::OMPD_parallel, D::OMPD_loop};
-  case D::OMPD_target_parallel_loop:
-    return {D::OMPD_target, D::OMPD_parallel_loop};
-  case D::OMPD_target_teams_loop:
-    return {D::OMPD_target, D::OMPD_teams_loop};
-  case D::OMPD_teams_loop:
-    return {D::OMPD_teams, D::OMPD_loop};
-  default:
-    return {dir, std::nullopt};
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Op body generation helper structures and functions
 //===----------------------------------------------------------------------===//
@@ -555,11 +488,6 @@ struct OpWithBodyGenInfo {
       : converter(converter), symTable(symTable), semaCtx(semaCtx), loc(loc),
         eval(eval), dir(dir) {}
 
-  OpWithBodyGenInfo &setGenNested(bool value) {
-    genNested = value;
-    return *this;
-  }
-
   OpWithBodyGenInfo &setOuterCombined(bool value) {
     outerCombined = value;
     return *this;
@@ -600,8 +528,6 @@ struct OpWithBodyGenInfo {
   Fortran::lower::pft::Evaluation &eval;
   /// [in] leaf directive for which to generate the op body.
   llvm::omp::Directive dir;
-  /// [in] whether to generate FIR for nested evaluations
-  bool genNested = true;
   /// [in] is this an outer operation - prevents privatization.
   bool outerCombined = false;
   /// [in] list of clauses to process.
@@ -620,9 +546,13 @@ struct OpWithBodyGenInfo {
 
 /// Create the body (block) for an OpenMP Operation.
 ///
-/// \param [in]   op - the operation the body belongs to.
-/// \param [in] info - options controlling code-gen for the construction.
-static void createBodyOfOp(mlir::Operation &op, OpWithBodyGenInfo &info) {
+/// \param [in]   op  - the operation the body belongs to.
+/// \param [in] info  - options controlling code-gen for the construction.
+/// \param [in] queue - work queue with nested constructs.
+/// \param [in] item  - item in the queue to generate body for.
+static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
+                           const ConstructQueue &queue,
+                           ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = info.converter.getFirOpBuilder();
 
   auto insertMarker = [](fir::FirOpBuilder &builder) {
@@ -678,7 +608,10 @@ static void createBodyOfOp(mlir::Operation &op, OpWithBodyGenInfo &info) {
     }
   }
 
-  if (info.genNested) {
+  if (ConstructQueue::iterator next = std::next(item); next != queue.end()) {
+    genOMPDispatch(info.converter, info.symTable, info.semaCtx, info.eval,
+                   info.loc, queue, next);
+  } else {
     // genFIR(Evaluation&) tries to patch up unterminated blocks, causing
     // a lot of complications for our approach if the terminator generation
     // is delayed past this point. Insert a temporary terminator here, then
@@ -769,11 +702,12 @@ static void genBodyOfTargetDataOp(
     Fortran::lower::AbstractConverter &converter,
     Fortran::lower::SymMap &symTable,
     Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, bool genNested,
-    mlir::omp::TargetDataOp &dataOp, llvm::ArrayRef<mlir::Type> useDeviceTypes,
+    Fortran::lower::pft::Evaluation &eval, mlir::omp::TargetDataOp &dataOp,
+    llvm::ArrayRef<mlir::Type> useDeviceTypes,
     llvm::ArrayRef<mlir::Location> useDeviceLocs,
     llvm::ArrayRef<const Fortran::semantics::Symbol *> useDeviceSymbols,
-    const mlir::Location &currentLocation) {
+    const mlir::Location &currentLocation, const ConstructQueue &queue,
+    ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::Region &region = dataOp.getRegion();
 
@@ -826,8 +760,13 @@ static void genBodyOfTargetDataOp(
 
   // Set the insertion point after the marker.
   firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
-  if (genNested)
+
+  if (ConstructQueue::iterator next = std::next(item); next != queue.end()) {
+    genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
+                   next);
+  } else {
     genNestedEvaluations(converter, eval);
+  }
 }
 
 // This functions creates a block for the body of the targetOp's region. It adds
@@ -836,12 +775,13 @@ static void
 genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
                   Fortran::lower::SymMap &symTable,
                   Fortran::semantics::SemanticsContext &semaCtx,
-                  Fortran::lower::pft::Evaluation &eval, bool genNested,
+                  Fortran::lower::pft::Evaluation &eval,
                   mlir::omp::TargetOp &targetOp,
                   llvm::ArrayRef<const Fortran::semantics::Symbol *> mapSyms,
                   llvm::ArrayRef<mlir::Location> mapSymLocs,
                   llvm::ArrayRef<mlir::Type> mapSymTypes,
-                  const mlir::Location &currentLocation) {
+                  const mlir::Location &currentLocation,
+                  const ConstructQueue &queue, ConstructQueue::iterator item) {
   assert(mapSymTypes.size() == mapSymLocs.size());
 
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
@@ -983,15 +923,22 @@ genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
 
   // Create the insertion point after the marker.
   firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
-  if (genNested)
+
+  if (ConstructQueue::iterator next = std::next(item); next != queue.end()) {
+    genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
+                   next);
+  } else {
     genNestedEvaluations(converter, eval);
+  }
 }
 
 template <typename OpTy, typename... Args>
-static OpTy genOpWithBody(OpWithBodyGenInfo &info, Args &&...args) {
+static OpTy genOpWithBody(const OpWithBodyGenInfo &info,
+                          const ConstructQueue &queue,
+                          ConstructQueue::iterator item, Args &&...args) {
   auto op = info.converter.getFirOpBuilder().create<OpTy>(
       info.loc, std::forward<Args>(args)...);
-  createBodyOfOp(*op, info);
+  createBodyOfOp(*op, info, queue, item);
   return op;
 }
 
@@ -1276,7 +1223,8 @@ static mlir::omp::BarrierOp
 genBarrierOp(Fortran::lower::AbstractConverter &converter,
              Fortran::lower::SymMap &symTable,
              Fortran::semantics::SemanticsContext &semaCtx,
-             Fortran::lower::pft::Evaluation &eval, mlir::Location loc) {
+             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+             const ConstructQueue &queue, ConstructQueue::iterator item) {
   return converter.getFirOpBuilder().create<mlir::omp::BarrierOp>(loc);
 }
 
@@ -1284,8 +1232,9 @@ static mlir::omp::CriticalOp
 genCriticalOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, bool genNested,
-              mlir::Location loc, const List<Clause> &clauses,
+              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+              const List<Clause> &clauses, const ConstructQueue &queue,
+              ConstructQueue::iterator item,
               const std::optional<Fortran::parser::Name> &name) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::FlatSymbolRefAttr nameAttr;
@@ -1308,17 +1257,17 @@ genCriticalOp(Fortran::lower::AbstractConverter &converter,
 
   return genOpWithBody<mlir::omp::CriticalOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                        llvm::omp::Directive::OMPD_critical)
-          .setGenNested(genNested),
-      nameAttr);
+                        llvm::omp::Directive::OMPD_critical),
+      queue, item, nameAttr);
 }
 
 static mlir::omp::DistributeOp
 genDistributeOp(Fortran::lower::AbstractConverter &converter,
                 Fortran::lower::SymMap &symTable,
                 Fortran::semantics::SemanticsContext &semaCtx,
-                Fortran::lower::pft::Evaluation &eval, bool genNested,
-                mlir::Location loc, const List<Clause> &clauses) {
+                Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+                const List<Clause> &clauses, const ConstructQueue &queue,
+                ConstructQueue::iterator item) {
   TODO(loc, "Distribute construct");
   return nullptr;
 }
@@ -1328,7 +1277,8 @@ genFlushOp(Fortran::lower::AbstractConverter &converter,
            Fortran::lower::SymMap &symTable,
            Fortran::semantics::SemanticsContext &semaCtx,
            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-           const ObjectList &objects, const List<Clause> &clauses) {
+           const ObjectList &objects, const List<Clause> &clauses,
+           const ConstructQueue &queue, ConstructQueue::iterator item) {
   llvm::SmallVector<mlir::Value> operandRange;
   genFlushClauses(converter, semaCtx, objects, clauses, loc, operandRange);
 
@@ -1340,12 +1290,13 @@ static mlir::omp::MasterOp
 genMasterOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, bool genNested,
-            mlir::Location loc) {
+            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+            const List<Clause> &clauses, const ConstructQueue &queue,
+            ConstructQueue::iterator item) {
   return genOpWithBody<mlir::omp::MasterOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                        llvm::omp::Directive::OMPD_master)
-          .setGenNested(genNested));
+                        llvm::omp::Directive::OMPD_master),
+      queue, item);
 }
 
 static mlir::omp::OrderedOp
@@ -1353,7 +1304,8 @@ genOrderedOp(Fortran::lower::AbstractConverter &converter,
              Fortran::lower::SymMap &symTable,
              Fortran::semantics::SemanticsContext &semaCtx,
              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-             const List<Clause> &clauses) {
+             const List<Clause> &clauses, const ConstructQueue &queue,
+             ConstructQueue::iterator item) {
   TODO(loc, "OMPD_ordered");
   return nullptr;
 }
@@ -1362,25 +1314,25 @@ static mlir::omp::OrderedRegionOp
 genOrderedRegionOp(Fortran::lower::AbstractConverter &converter,
                    Fortran::lower::SymMap &symTable,
                    Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval, bool genNested,
-                   mlir::Location loc, const List<Clause> &clauses) {
+                   Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+                   const List<Clause> &clauses, const ConstructQueue &queue,
+                   ConstructQueue::iterator item) {
   mlir::omp::OrderedRegionClauseOps clauseOps;
   genOrderedRegionClauses(converter, semaCtx, clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::OrderedRegionOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
-                        llvm::omp::Directive::OMPD_ordered)
-          .setGenNested(genNested),
-      clauseOps);
+                        llvm::omp::Directive::OMPD_ordered),
+      queue, item, clauseOps);
 }
 
 static mlir::omp::ParallelOp
 genParallelOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, bool genNested,
-              mlir::Location loc, const List<Clause> &clauses,
-              bool outerCombined = false) {
+              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+              const List<Clause> &clauses, const ConstructQueue &queue,
+              ConstructQueue::iterator item, bool outerCombined = false) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::ParallelClauseOps clauseOps;
@@ -1399,14 +1351,14 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
   OpWithBodyGenInfo genInfo =
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_parallel)
-          .setGenNested(genNested)
           .setOuterCombined(outerCombined)
           .setClauses(&clauses)
           .setReductions(&reductionSyms, &reductionTypes)
           .setGenRegionEntryCb(reductionCallback);
 
   if (!enableDelayedPrivatization)
-    return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
+    return genOpWithBody<mlir::omp::ParallelOp>(genInfo, queue, item,
+                                                clauseOps);
 
   bool privatize = !outerCombined;
   DataSharingProcessor dsp(converter, semaCtx, clauses, eval,
@@ -1454,19 +1406,23 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
   };
 
   genInfo.setGenRegionEntryCb(genRegionEntryCB).setDataSharingProcessor(&dsp);
-  return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
+  return genOpWithBody<mlir::omp::ParallelOp>(genInfo, queue, item, clauseOps);
 }
 
 static mlir::omp::SectionOp
 genSectionOp(Fortran::lower::AbstractConverter &converter,
              Fortran::lower::SymMap &symTable,
              Fortran::semantics::SemanticsContext &semaCtx,
-             Fortran::lower::pft::Evaluation &eval, bool genNested,
-             mlir::Location loc) {
+             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+             const List<Clause> &clauses, const ConstructQueue &queue,
+             ConstructQueue::iterator item) {
+  // Currently only private/firstprivate clause is handled, and
+  // all privatization is done within `omp.section` operations.
   return genOpWithBody<mlir::omp::SectionOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_section)
-          .setGenNested(genNested));
+          .setClauses(&clauses),
+      queue, item);
 }
 
 static mlir::omp::SectionsOp
@@ -1474,12 +1430,77 @@ genSectionsOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const mlir::omp::SectionsClauseOps &clauseOps) {
-  return genOpWithBody<mlir::omp::SectionsOp>(
+              const List<Clause> &clauses, const ConstructQueue &queue,
+              ConstructQueue::iterator item) {
+  mlir::omp::SectionsClauseOps clauseOps;
+  genSectionsClauses(converter, semaCtx, clauses, loc, clauseOps);
+
+  auto &builder = converter.getFirOpBuilder();
+
+  // Insert privatizations before SECTIONS
+  symTable.pushScope();
+  DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
+  dsp.processStep1();
+
+  List<Clause> nonDsaClauses;
+  List<const clause::Lastprivate *> lastprivates;
+
+  for (const Clause &clause : clauses) {
+    if (clause.id == llvm::omp::Clause::OMPC_lastprivate) {
+      lastprivates.push_back(&std::get<clause::Lastprivate>(clause.u));
+    } else {
+      switch (clause.id) {
+      case llvm::omp::Clause::OMPC_firstprivate:
+      case llvm::omp::Clause::OMPC_private:
+      case llvm::omp::Clause::OMPC_shared:
+        break;
+      default:
+        nonDsaClauses.push_back(clause);
+      }
+    }
+  }
+
+  // SECTIONS construct.
+  mlir::omp::SectionsOp sectionsOp = genOpWithBody<mlir::omp::SectionsOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_sections)
-          .setGenNested(false),
-      clauseOps);
+          .setClauses(&nonDsaClauses),
+      queue, item, clauseOps);
+
+  if (!lastprivates.empty()) {
+    mlir::Region &sectionsBody = sectionsOp.getRegion();
+    assert(sectionsBody.hasOneBlock());
+    mlir::Block &body = sectionsBody.front();
+
+    auto lastSectionOp = llvm::find_if(
+        llvm::reverse(body.getOperations()), [](const mlir::Operation &op) {
+          return llvm::isa<mlir::omp::SectionOp>(op);
+        });
+    assert(lastSectionOp != body.rend());
+
+    for (const clause::Lastprivate *lastp : lastprivates) {
+      builder.setInsertionPoint(
+          lastSectionOp->getRegion(0).back().getTerminator());
+      mlir::OpBuilder::InsertPoint insp = builder.saveInsertionPoint();
+      const auto &objList = std::get<ObjectList>(lastp->t);
+      for (const Object &object : objList) {
+        Fortran::semantics::Symbol *sym = object.id();
+        converter.copyHostAssociateVar(*sym, &insp);
+      }
+    }
+  }
+
+  // Perform DataSharingProcessor's step2 out of SECTIONS
+  builder.setInsertionPointAfter(sectionsOp.getOperation());
+  dsp.processStep2(sectionsOp, false);
+  // Emit implicit barrier to synchronize threads and avoid data
+  // races on post-update of lastprivate variables when `nowait`
+  // clause is present.
+  if (clauseOps.nowaitAttr && !lastprivates.empty())
+    builder.create<mlir::omp::BarrierOp>(loc);
+
+  symTable.popScope();
+  return sectionsOp;
 }
 
 static mlir::omp::SimdOp
@@ -1487,7 +1508,8 @@ genSimdOp(Fortran::lower::AbstractConverter &converter,
           Fortran::lower::SymMap &symTable,
           Fortran::semantics::SemanticsContext &semaCtx,
           Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-          const List<Clause> &clauses) {
+          const List<Clause> &clauses, const ConstructQueue &queue,
+          ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
   dsp.processStep1();
@@ -1522,7 +1544,8 @@ genSimdOp(Fortran::lower::AbstractConverter &converter,
                                    *nestedEval, llvm::omp::Directive::OMPD_simd)
                      .setClauses(&clauses)
                      .setDataSharingProcessor(&dsp)
-                     .setGenRegionEntryCb(ivCallback));
+                     .setGenRegionEntryCb(ivCallback),
+                 queue, item);
 
   return simdOp;
 }
@@ -1531,26 +1554,26 @@ static mlir::omp::SingleOp
 genSingleOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, bool genNested,
-            mlir::Location loc, const List<Clause> &clauses) {
+            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+            const List<Clause> &clauses, const ConstructQueue &queue,
+            ConstructQueue::iterator item) {
   mlir::omp::SingleClauseOps clauseOps;
   genSingleClauses(converter, semaCtx, clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::SingleOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_single)
-          .setGenNested(genNested)
           .setClauses(&clauses),
-      clauseOps);
+      queue, item, clauseOps);
 }
 
 static mlir::omp::TargetOp
 genTargetOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, bool genNested,
-            mlir::Location loc, const List<Clause> &clauses,
-            bool outerCombined = false) {
+            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+            const List<Clause> &clauses, const ConstructQueue &queue,
+            ConstructQueue::iterator item, bool outerCombined = false) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
 
@@ -1657,8 +1680,8 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
   Fortran::lower::pft::visitAllSymbols(eval, captureImplicitMap);
 
   auto targetOp = firOpBuilder.create<mlir::omp::TargetOp>(loc, clauseOps);
-  genBodyOfTargetOp(converter, symTable, semaCtx, eval, genNested, targetOp,
-                    mapSyms, mapLocs, mapTypes, loc);
+  genBodyOfTargetOp(converter, symTable, semaCtx, eval, targetOp, mapSyms,
+                    mapLocs, mapTypes, loc, queue, item);
   return targetOp;
 }
 
@@ -1666,8 +1689,9 @@ static mlir::omp::TargetDataOp
 genTargetDataOp(Fortran::lower::AbstractConverter &converter,
                 Fortran::lower::SymMap &symTable,
                 Fortran::semantics::SemanticsContext &semaCtx,
-                Fortran::lower::pft::Evaluation &eval, bool genNested,
-                mlir::Location loc, const List<Clause> &clauses) {
+                Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+                const List<Clause> &clauses, const ConstructQueue &queue,
+                ConstructQueue::iterator item) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TargetDataClauseOps clauseOps;
   llvm::SmallVector<mlir::Type> useDeviceTypes;
@@ -1679,9 +1703,9 @@ genTargetDataOp(Fortran::lower::AbstractConverter &converter,
   auto targetDataOp =
       converter.getFirOpBuilder().create<mlir::omp::TargetDataOp>(loc,
                                                                   clauseOps);
-  genBodyOfTargetDataOp(converter, symTable, semaCtx, eval, genNested,
-                        targetDataOp, useDeviceTypes, useDeviceLocs,
-                        useDeviceSyms, loc);
+  genBodyOfTargetDataOp(converter, symTable, semaCtx, eval, targetDataOp,
+                        useDeviceTypes, useDeviceLocs, useDeviceSyms, loc,
+                        queue, item);
   return targetDataOp;
 }
 
@@ -1690,8 +1714,9 @@ static OpTy
 genTargetEnterExitUpdateDataOp(Fortran::lower::AbstractConverter &converter,
                                Fortran::lower::SymMap &symTable,
                                Fortran::semantics::SemanticsContext &semaCtx,
-                               mlir::Location loc,
-                               const List<Clause> &clauses) {
+                               mlir::Location loc, const List<Clause> &clauses,
+                               const ConstructQueue &queue,
+                               ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
 
@@ -1718,8 +1743,9 @@ static mlir::omp::TaskOp
 genTaskOp(Fortran::lower::AbstractConverter &converter,
           Fortran::lower::SymMap &symTable,
           Fortran::semantics::SemanticsContext &semaCtx,
-          Fortran::lower::pft::Evaluation &eval, bool genNested,
-          mlir::Location loc, const List<Clause> &clauses) {
+          Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+          const List<Clause> &clauses, const ConstructQueue &queue,
+          ConstructQueue::iterator item) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TaskClauseOps clauseOps;
   genTaskClauses(converter, semaCtx, stmtCtx, clauses, loc, clauseOps);
@@ -1727,26 +1753,25 @@ genTaskOp(Fortran::lower::AbstractConverter &converter,
   return genOpWithBody<mlir::omp::TaskOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_task)
-          .setGenNested(genNested)
           .setClauses(&clauses),
-      clauseOps);
+      queue, item, clauseOps);
 }
 
 static mlir::omp::TaskgroupOp
 genTaskgroupOp(Fortran::lower::AbstractConverter &converter,
                Fortran::lower::SymMap &symTable,
                Fortran::semantics::SemanticsContext &semaCtx,
-               Fortran::lower::pft::Evaluation &eval, bool genNested,
-               mlir::Location loc, const List<Clause> &clauses) {
+               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+               const List<Clause> &clauses, const ConstructQueue &queue,
+               ConstructQueue::iterator item) {
   mlir::omp::TaskgroupClauseOps clauseOps;
   genTaskgroupClauses(converter, semaCtx, clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::TaskgroupOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_taskgroup)
-          .setGenNested(genNested)
           .setClauses(&clauses),
-      clauseOps);
+      queue, item, clauseOps);
 }
 
 static mlir::omp::TaskloopOp
@@ -1754,7 +1779,8 @@ genTaskloopOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses) {
+              const List<Clause> &clauses, const ConstructQueue &queue,
+              ConstructQueue::iterator item) {
   TODO(loc, "Taskloop construct");
 }
 
@@ -1763,7 +1789,8 @@ genTaskwaitOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses) {
+              const List<Clause> &clauses, const ConstructQueue &queue,
+              ConstructQueue::iterator item) {
   mlir::omp::TaskwaitClauseOps clauseOps;
   genTaskwaitClauses(converter, semaCtx, clauses, loc, clauseOps);
   return converter.getFirOpBuilder().create<mlir::omp::TaskwaitOp>(loc,
@@ -1774,7 +1801,8 @@ static mlir::omp::TaskyieldOp
 genTaskyieldOp(Fortran::lower::AbstractConverter &converter,
                Fortran::lower::SymMap &symTable,
                Fortran::semantics::SemanticsContext &semaCtx,
-               Fortran::lower::pft::Evaluation &eval, mlir::Location loc) {
+               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+               const ConstructQueue &queue, ConstructQueue::iterator item) {
   return converter.getFirOpBuilder().create<mlir::omp::TaskyieldOp>(loc);
 }
 
@@ -1782,9 +1810,9 @@ static mlir::omp::TeamsOp
 genTeamsOp(Fortran::lower::AbstractConverter &converter,
            Fortran::lower::SymMap &symTable,
            Fortran::semantics::SemanticsContext &semaCtx,
-           Fortran::lower::pft::Evaluation &eval, bool genNested,
-           mlir::Location loc, const List<Clause> &clauses,
-           bool outerCombined = false) {
+           Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+           const List<Clause> &clauses, const ConstructQueue &queue,
+           ConstructQueue::iterator item, bool outerCombined = false) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TeamsClauseOps clauseOps;
   genTeamsClauses(converter, semaCtx, stmtCtx, clauses, loc, clauseOps);
@@ -1792,10 +1820,9 @@ genTeamsOp(Fortran::lower::AbstractConverter &converter,
   return genOpWithBody<mlir::omp::TeamsOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_teams)
-          .setGenNested(genNested)
           .setOuterCombined(outerCombined)
           .setClauses(&clauses),
-      clauseOps);
+      queue, item, clauseOps);
 }
 
 static mlir::omp::WsloopOp
@@ -1803,7 +1830,8 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const List<Clause> &clauses) {
+            const List<Clause> &clauses, const ConstructQueue &queue,
+            ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
   dsp.processStep1();
@@ -1844,7 +1872,8 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
                      .setClauses(&clauses)
                      .setDataSharingProcessor(&dsp)
                      .setReductions(&reductionSyms, &reductionTypes)
-                     .setGenRegionEntryCb(ivCallback));
+                     .setGenRegionEntryCb(ivCallback),
+                 queue, item);
   return wsloopOp;
 }
 
@@ -1852,13 +1881,13 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
 // Code generation functions for composite constructs
 //===----------------------------------------------------------------------===//
 
-static void
-genCompositeDistributeParallelDo(Fortran::lower::AbstractConverter &converter,
-                                 Fortran::lower::SymMap &symTable,
-                                 Fortran::semantics::SemanticsContext &semaCtx,
-                                 Fortran::lower::pft::Evaluation &eval,
-                                 const List<Clause> &clauses,
-                                 mlir::Location loc) {
+static void genCompositeDistributeParallelDo(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::lower::SymMap &symTable,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+    const List<Clause> &clauses, const ConstructQueue &queue,
+    ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE PARALLEL DO");
 }
 
@@ -1866,8 +1895,9 @@ static void genCompositeDistributeParallelDoSimd(
     Fortran::lower::AbstractConverter &converter,
     Fortran::lower::SymMap &symTable,
     Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, const List<Clause> &clauses,
-    mlir::Location loc) {
+    Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+    const List<Clause> &clauses, const ConstructQueue &queue,
+    ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE PARALLEL DO SIMD");
 }
 
@@ -1876,7 +1906,9 @@ genCompositeDistributeSimd(Fortran::lower::AbstractConverter &converter,
                            Fortran::lower::SymMap &symTable,
                            Fortran::semantics::SemanticsContext &semaCtx,
                            Fortran::lower::pft::Evaluation &eval,
-                           const List<Clause> &clauses, mlir::Location loc) {
+                           mlir::Location loc, const List<Clause> &clauses,
+                           const ConstructQueue &queue,
+                           ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE SIMD");
 }
 
@@ -1884,8 +1916,9 @@ static void genCompositeDoSimd(Fortran::lower::AbstractConverter &converter,
                                Fortran::lower::SymMap &symTable,
                                Fortran::semantics::SemanticsContext &semaCtx,
                                Fortran::lower::pft::Evaluation &eval,
-                               const List<Clause> &clauses,
-                               mlir::Location loc) {
+                               mlir::Location loc, const List<Clause> &clauses,
+                               const ConstructQueue &queue,
+                               ConstructQueue::iterator item) {
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processTODO<clause::Aligned, clause::Allocate, clause::Linear,
                  clause::Order, clause::Safelen, clause::Simdlen>(
@@ -1898,7 +1931,7 @@ static void genCompositeDoSimd(Fortran::lower::AbstractConverter &converter,
   // When support for vectorization is enabled, then we need to add handling of
   // if clause. Currently if clause can be skipped because we always assume
   // SIMD length = 1.
-  genWsloopOp(converter, symTable, semaCtx, eval, loc, clauses);
+  genWsloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
 }
 
 static void
@@ -1906,10 +1939,128 @@ genCompositeTaskloopSimd(Fortran::lower::AbstractConverter &converter,
                          Fortran::lower::SymMap &symTable,
                          Fortran::semantics::SemanticsContext &semaCtx,
                          Fortran::lower::pft::Evaluation &eval,
-                         const List<Clause> &clauses, mlir::Location loc) {
+                         mlir::Location loc, const List<Clause> &clauses,
+                         const ConstructQueue &queue,
+                         ConstructQueue::iterator item) {
   TODO(loc, "Composite TASKLOOP SIMD");
 }
 
+//===----------------------------------------------------------------------===//
+// Dispatch
+//===----------------------------------------------------------------------===//
+
+static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
+                           Fortran::lower::SymMap &symTable,
+                           Fortran::semantics::SemanticsContext &semaCtx,
+                           Fortran::lower::pft::Evaluation &eval,
+                           mlir::Location loc, const ConstructQueue &queue,
+                           ConstructQueue::iterator item) {
+  assert(item != queue.end());
+  const List<Clause> &clauses = item->clauses;
+
+  switch (llvm::omp::Directive dir = item->id) {
+  case llvm::omp::Directive::OMPD_distribute:
+    genDistributeOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                    item);
+    break;
+  case llvm::omp::Directive::OMPD_do:
+    genWsloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_loop:
+  case llvm::omp::Directive::OMPD_masked:
+  case llvm::omp::Directive::OMPD_tile:
+  case llvm::omp::Directive::OMPD_unroll:
+    TODO(loc, "Unhandled loop directive (" +
+                  llvm::omp::getOpenMPDirectiveName(dir) + ")");
+    break;
+  case llvm::omp::Directive::OMPD_master:
+    genMasterOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_ordered:
+    genOrderedRegionOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                       item);
+    break;
+  case llvm::omp::Directive::OMPD_parallel:
+    genParallelOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item,
+                  /*outerCombined=*/false);
+    break;
+  case llvm::omp::Directive::OMPD_sections:
+    genSectionsOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                  item);
+    break;
+  case llvm::omp::Directive::OMPD_simd:
+    genSimdOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_single:
+    genSingleOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_target:
+    genTargetOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item,
+                /*outerCombined=*/false);
+    break;
+  case llvm::omp::Directive::OMPD_target_data:
+    genTargetDataOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                    item);
+    break;
+  case llvm::omp::Directive::OMPD_target_enter_data:
+    genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(
+        converter, symTable, semaCtx, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_target_exit_data:
+    genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(
+        converter, symTable, semaCtx, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_target_update:
+    genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(
+        converter, symTable, semaCtx, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_task:
+    genTaskOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_taskgroup:
+    genTaskgroupOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                   item);
+    break;
+  case llvm::omp::Directive::OMPD_taskloop:
+    genTaskloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                  item);
+    break;
+  case llvm::omp::Directive::OMPD_teams:
+    genTeamsOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  // case llvm::omp::Directive::OMPD_workdistribute:
+  case llvm::omp::Directive::OMPD_workshare:
+    // FIXME: Workshare is not a commonly used OpenMP construct, an
+    // implementation for this feature will come later. For the codes
+    // that use this construct, add a single construct for now.
+    genSingleOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    break;
+  // Composite constructs
+  case llvm::omp::Directive::OMPD_distribute_parallel_do:
+    genCompositeDistributeParallelDo(converter, symTable, semaCtx, eval, loc,
+                                     clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_distribute_parallel_do_simd:
+    genCompositeDistributeParallelDoSimd(converter, symTable, semaCtx, eval,
+                                         loc, clauses, queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_distribute_simd:
+    genCompositeDistributeSimd(converter, symTable, semaCtx, eval, loc, clauses,
+                               queue, item);
+    break;
+  case llvm::omp::Directive::OMPD_do_simd:
+    genCompositeDoSimd(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                       item);
+    break;
+  case llvm::omp::Directive::OMPD_taskloop_simd:
+    genCompositeTaskloopSimd(converter, symTable, semaCtx, eval, loc, clauses,
+                             queue, item);
+    break;
+  default:
+    break;
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // OpenMPDeclarativeConstruct visitors
 //===----------------------------------------------------------------------===//
@@ -2020,36 +2171,47 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
       semaCtx);
   mlir::Location currentLocation = converter.genLocation(directive.source);
 
+  ConstructQueue queue{
+      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
+                          eval, directive.source, directive.v, clauses)};
+
   switch (directive.v) {
   default:
     break;
   case llvm::omp::Directive::OMPD_barrier:
-    genBarrierOp(converter, symTable, semaCtx, eval, currentLocation);
+    genBarrierOp(converter, symTable, semaCtx, eval, currentLocation, queue,
+                 queue.begin());
     break;
   case llvm::omp::Directive::OMPD_taskwait:
-    genTaskwaitOp(converter, symTable, semaCtx, eval, currentLocation, clauses);
+    genTaskwaitOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
+                  queue, queue.begin());
     break;
   case llvm::omp::Directive::OMPD_taskyield:
-    genTaskyieldOp(converter, symTable, semaCtx, eval, currentLocation);
+    genTaskyieldOp(converter, symTable, semaCtx, eval, currentLocation, queue,
+                   queue.begin());
     break;
   case llvm::omp::Directive::OMPD_target_data:
-    genTargetDataOp(converter, symTable, semaCtx, eval, /*genNested=*/true,
-                    currentLocation, clauses);
+    genTargetDataOp(converter, symTable, semaCtx, eval, currentLocation,
+                    clauses, queue, queue.begin());
     break;
   case llvm::omp::Directive::OMPD_target_enter_data:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(
-        converter, symTable, semaCtx, currentLocation, clauses);
+        converter, symTable, semaCtx, currentLocation, clauses, queue,
+        queue.begin());
     break;
   case llvm::omp::Directive::OMPD_target_exit_data:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(
-        converter, symTable, semaCtx, currentLocation, clauses);
+        converter, symTable, semaCtx, currentLocation, clauses, queue,
+        queue.begin());
     break;
   case llvm::omp::Directive::OMPD_target_update:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(
-        converter, symTable, semaCtx, currentLocation, clauses);
+        converter, symTable, semaCtx, currentLocation, clauses, queue,
+        queue.begin());
     break;
   case llvm::omp::Directive::OMPD_ordered:
-    genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, clauses);
+    genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
+                 queue, queue.begin());
     break;
   }
 }
@@ -2073,8 +2235,12 @@ genOMP(Fortran::lower::AbstractConverter &converter,
                             [&](auto &&s) { return makeClause(s.v, semaCtx); })
                  : List<Clause>{};
   mlir::Location currentLocation = converter.genLocation(verbatim.source);
+
+  ConstructQueue queue{buildConstructQueue(
+      converter.getFirOpBuilder().getModule(), semaCtx, eval, verbatim.source,
+      llvm::omp::Directive::OMPD_flush, clauses)};
   genFlushOp(converter, symTable, semaCtx, eval, currentLocation, objects,
-             clauses);
+             clauses, queue, queue.begin());
 }
 
 static void
@@ -2217,75 +2383,15 @@ genOMP(Fortran::lower::AbstractConverter &converter,
     }
   }
 
-  std::optional<llvm::omp::Directive> nextDir = origDirective;
-  bool outermostLeafConstruct = true;
-  while (nextDir) {
-    llvm::omp::Directive leafDir;
-    std::tie(leafDir, nextDir) = splitCombinedDirective(*nextDir);
-    const bool genNested = !nextDir;
-    const bool outerCombined = outermostLeafConstruct && nextDir.has_value();
-    switch (leafDir) {
-    case llvm::omp::Directive::OMPD_master:
-      // 2.16 MASTER construct.
-      genMasterOp(converter, symTable, semaCtx, eval, genNested,
-                  currentLocation);
-      break;
-    case llvm::omp::Directive::OMPD_ordered:
-      // 2.17.9 ORDERED construct.
-      genOrderedRegionOp(converter, symTable, semaCtx, eval, genNested,
-                         currentLocation, clauses);
-      break;
-    case llvm::omp::Directive::OMPD_parallel:
-      // 2.6 PARALLEL construct.
-      genParallelOp(converter, symTable, semaCtx, eval, genNested,
-                    currentLocation, clauses, outerCombined);
-      break;
-    case llvm::omp::Directive::OMPD_single:
-      // 2.8.2 SINGLE construct.
-      genSingleOp(converter, symTable, semaCtx, eval, genNested,
-                  currentLocation, clauses);
-      break;
-    case llvm::omp::Directive::OMPD_target:
-      // 2.12.5 TARGET construct.
-      genTargetOp(converter, symTable, semaCtx, eval, genNested,
-                  currentLocation, clauses, outerCombined);
-      break;
-    case llvm::omp::Directive::OMPD_target_data:
-      // 2.12.2 TARGET DATA construct.
-      genTargetDataOp(converter, symTable, semaCtx, eval, genNested,
-                      currentLocation, clauses);
-      break;
-    case llvm::omp::Directive::OMPD_task:
-      // 2.10.1 TASK construct.
-      genTaskOp(converter, symTable, semaCtx, eval, genNested, currentLocation,
-                clauses);
-      break;
-    case llvm::omp::Directive::OMPD_taskgroup:
-      // 2.17.6 TASKGROUP construct.
-      genTaskgroupOp(converter, symTable, semaCtx, eval, genNested,
-                     currentLocation, clauses);
-      break;
-    case llvm::omp::Directive::OMPD_teams:
-      // 2.7 TEAMS construct.
-      // FIXME Pass the outerCombined argument or rename it to better describe
-      // what it represents if it must always be `false` in this context.
-      genTeamsOp(converter, symTable, semaCtx, eval, genNested, currentLocation,
-                 clauses);
-      break;
-    case llvm::omp::Directive::OMPD_workshare:
-      // 2.8.3 WORKSHARE construct.
-      // FIXME: Workshare is not a commonly used OpenMP construct, an
-      // implementation for this feature will come later. For the codes
-      // that use this construct, add a single construct for now.
-      genSingleOp(converter, symTable, semaCtx, eval, genNested,
-                  currentLocation, clauses);
-      break;
-    default:
-      llvm_unreachable("Unexpected block construct");
-      break;
-    }
-    outermostLeafConstruct = false;
-  }
+  llvm::omp::Directive directive =
+      std::get<parser::OmpBlockDirective>(beginBlockDirective.t).v;
+  const parser::CharBlock &source =
+      std::get<parser::OmpBlockDirective>(beginBlockDirective.t).source;
+  ConstructQueue queue{
+      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
+                          eval, source, directive, clauses)};
+  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
+                 queue.begin());
 }
 
 static void
@@ -2298,10 +2404,15 @@ genOMP(Fortran::lower::AbstractConverter &converter,
       std::get<Fortran::parser::OmpCriticalDirective>(criticalConstruct.t);
   List<Clause> clauses =
       makeClauses(std::get<Fortran::parser::OmpClauseList>(cd.t), semaCtx);
+
+  ConstructQueue queue{buildConstructQueue(
+      converter.getFirOpBuilder().getModule(), semaCtx, eval, cd.source,
+      llvm::omp::Directive::OMPD_critical, clauses)};
+
   const auto &name = std::get<std::optional<Fortran::parser::Name>>(cd.t);
   mlir::Location currentLocation = converter.getCurrentLocation();
-  genCriticalOp(converter, symTable, semaCtx, eval, /*genNested=*/true,
-                currentLocation, clauses, name);
+  genCriticalOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
+                queue, queue.begin(), name);
 }
 
 static void
@@ -2322,14 +2433,6 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
       std::get<Fortran::parser::OmpBeginLoopDirective>(loopConstruct.t);
   List<Clause> clauses = makeClauses(
       std::get<Fortran::parser::OmpClauseList>(beginLoopDirective.t), semaCtx);
-  mlir::Location currentLocation =
-      converter.genLocation(beginLoopDirective.source);
-  const auto origDirective =
-      std::get<Fortran::parser::OmpLoopDirective>(beginLoopDirective.t).v;
-
-  assert(llvm::omp::loopConstructSet.test(origDirective) &&
-         "Expected loop construct");
-
   if (auto &endLoopDirective =
           std::get<std::optional<Fortran::parser::OmpEndLoopDirective>>(
               loopConstruct.t)) {
@@ -2338,101 +2441,18 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
         semaCtx));
   }
 
-  std::optional<llvm::omp::Directive> nextDir = origDirective;
-  while (nextDir) {
-    llvm::omp::Directive leafDir;
-    std::tie(leafDir, nextDir) = splitCombinedDirective(*nextDir);
-    if (llvm::omp::compositeConstructSet.test(leafDir)) {
-      assert(!nextDir && "Composite construct cannot be split");
-      switch (leafDir) {
-      case llvm::omp::Directive::OMPD_distribute_parallel_do:
-        // 2.9.4.3 DISTRIBUTE PARALLEL Worksharing-Loop construct.
-        genCompositeDistributeParallelDo(converter, symTable, semaCtx, eval,
-                                         clauses, currentLocation);
-        break;
-      case llvm::omp::Directive::OMPD_distribute_parallel_do_simd:
-        // 2.9.4.4 DISTRIBUTE PARALLEL Worksharing-Loop SIMD construct.
-        genCompositeDistributeParallelDoSimd(converter, symTable, semaCtx, eval,
-                                             clauses, currentLocation);
-        break;
-      case llvm::omp::Directive::OMPD_distribute_simd:
-        // 2.9.4.2 DISTRIBUTE SIMD construct.
-        genCompositeDistributeSimd(converter, symTable, semaCtx, eval, clauses,
-                                   currentLocation);
-        break;
-      case llvm::omp::Directive::OMPD_do_simd:
-        // 2.9.3.2 Worksharing-Loop SIMD construct.
-        genCompositeDoSimd(converter, symTable, semaCtx, eval, clauses,
-                           currentLocation);
-        break;
-      case llvm::omp::Directive::OMPD_taskloop_simd:
-        // 2.10.3 TASKLOOP SIMD construct.
-        genCompositeTaskloopSimd(converter, symTable, semaCtx, eval, clauses,
-                                 currentLocation);
-        break;
-      default:
-        llvm_unreachable("Unexpected composite construct");
-      }
-    } else {
-      const bool genNested = !nextDir;
-      switch (leafDir) {
-      case llvm::omp::Directive::OMPD_distribute:
-        // 2.9.4.1 DISTRIBUTE construct.
-        genDistributeOp(converter, symTable, semaCtx, eval, genNested,
-                        currentLocation, clauses);
-        break;
-      case llvm::omp::Directive::OMPD_do:
-        // 2.9.2 Worksharing-Loop construct.
-        genWsloopOp(converter, symTable, semaCtx, eval, currentLocation,
-                    clauses);
-        break;
-      case llvm::omp::Directive::OMPD_parallel:
-        // 2.6 PARALLEL construct.
-        // FIXME This is not necessarily always the outer leaf construct of a
-        // combined construct in this constext (e.g. distribute parallel do).
-        // Maybe rename the argument if it represents something else or
-        // initialize it properly.
-        genParallelOp(converter, symTable, semaCtx, eval, genNested,
-                      currentLocation, clauses,
-                      /*outerCombined=*/true);
-        break;
-      case llvm::omp::Directive::OMPD_simd:
-        // 2.9.3.1 SIMD construct.
-        genSimdOp(converter, symTable, semaCtx, eval, currentLocation, clauses);
-        break;
-      case llvm::omp::Directive::OMPD_target:
-        // 2.12.5 TARGET construct.
-        genTargetOp(converter, symTable, semaCtx, eval, genNested,
-                    currentLocation, clauses, /*outerCombined=*/true);
-        break;
-      case llvm::omp::Directive::OMPD_taskloop:
-        // 2.10.2 TASKLOOP construct.
-        genTaskloopOp(converter, symTable, semaCtx, eval, currentLocation,
-                      clauses);
-        break;
-      case llvm::omp::Directive::OMPD_teams:
-        // 2.7 TEAMS construct.
-        // FIXME This is not necessarily always the outer leaf construct of a
-        // combined construct in this constext (e.g. target teams distribute).
-        // Maybe rename the argument if it represents something else or
-        // initialize it properly.
-        genTeamsOp(converter, symTable, semaCtx, eval, genNested,
-                   currentLocation, clauses, /*outerCombined=*/true);
-        break;
-      case llvm::omp::Directive::OMPD_loop:
-      case llvm::omp::Directive::OMPD_masked:
-      case llvm::omp::Directive::OMPD_master:
-      case llvm::omp::Directive::OMPD_tile:
-      case llvm::omp::Directive::OMPD_unroll:
-        TODO(currentLocation, "Unhandled loop directive (" +
-                                  llvm::omp::getOpenMPDirectiveName(leafDir) +
-                                  ")");
-        break;
-      default:
-        llvm_unreachable("Unexpected loop construct");
-      }
-    }
-  }
+  mlir::Location currentLocation =
+      converter.genLocation(beginLoopDirective.source);
+
+  llvm::omp::Directive directive =
+      std::get<parser::OmpLoopDirective>(beginLoopDirective.t).v;
+  const parser::CharBlock &source =
+      std::get<parser::OmpLoopDirective>(beginLoopDirective.t).source;
+  ConstructQueue queue{
+      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
+                          eval, source, directive, clauses)};
+  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
+                 queue.begin());
 }
 
 static void
@@ -2441,8 +2461,12 @@ genOMP(Fortran::lower::AbstractConverter &converter,
        Fortran::semantics::SemanticsContext &semaCtx,
        Fortran::lower::pft::Evaluation &eval,
        const Fortran::parser::OpenMPSectionConstruct &sectionConstruct) {
-  // SECTION constructs are handled as a part of SECTIONS.
-  llvm_unreachable("Unexpected standalone OMP SECTION");
+  mlir::Location loc = converter.getCurrentLocation();
+  ConstructQueue queue{buildConstructQueue(
+      converter.getFirOpBuilder().getModule(), semaCtx, eval,
+      sectionConstruct.source, llvm::omp::Directive::OMPD_section, {})};
+  genSectionOp(converter, symTable, semaCtx, eval, loc,
+               /*clauses=*/{}, queue, queue.begin());
 }
 
 static void
@@ -2461,77 +2485,17 @@ genOMP(Fortran::lower::AbstractConverter &converter,
   clauses.append(makeClauses(
       std::get<Fortran::parser::OmpClauseList>(endSectionsDirective.t),
       semaCtx));
-
-  // Process clauses before optional omp.parallel, so that new variables are
-  // allocated outside of the parallel region
   mlir::Location currentLocation = converter.getCurrentLocation();
-  mlir::omp::SectionsClauseOps clauseOps;
-  genSectionsClauses(converter, semaCtx, clauses, currentLocation, clauseOps);
-
-  // Parallel wrapper of PARALLEL SECTIONS construct
-  llvm::omp::Directive dir =
-      std::get<Fortran::parser::OmpSectionsDirective>(beginSectionsDirective.t)
-          .v;
-  if (dir == llvm::omp::Directive::OMPD_parallel_sections) {
-    genParallelOp(converter, symTable, semaCtx, eval,
-                  /*genNested=*/false, currentLocation, clauses,
-                  /*outerCombined=*/true);
-  }
-
-  // Insert privatizations before SECTIONS
-  symTable.pushScope();
-  DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
-  dsp.processStep1();
-
-  // SECTIONS construct.
-  mlir::omp::SectionsOp sectionsOp = genSectionsOp(
-      converter, symTable, semaCtx, eval, currentLocation, clauseOps);
-
-  // Generate nested SECTION operations recursively.
-  const auto &sectionBlocks =
-      std::get<Fortran::parser::OmpSectionBlocks>(sectionsConstruct.t);
-  auto &firOpBuilder = converter.getFirOpBuilder();
-  auto ip = firOpBuilder.saveInsertionPoint();
-  mlir::omp::SectionOp lastSectionOp;
-  for (const auto &[nblock, neval] :
-       llvm::zip(sectionBlocks.v, eval.getNestedEvaluations())) {
-    symTable.pushScope();
-    lastSectionOp = genSectionOp(converter, symTable, semaCtx, neval,
-                                 /*genNested=*/true, currentLocation);
-    symTable.popScope();
-    firOpBuilder.restoreInsertionPoint(ip);
-  }
-
-  // For `omp.sections`, lastprivatized variables occur in
-  // lexically final `omp.section` operation.
-  bool hasLastPrivate = false;
-  if (lastSectionOp) {
-    for (const Clause &clause : clauses) {
-      if (const auto *lastPrivate =
-              std::get_if<clause::Lastprivate>(&clause.u)) {
-        hasLastPrivate = true;
-        firOpBuilder.setInsertionPoint(
-            lastSectionOp.getRegion().back().getTerminator());
-        mlir::OpBuilder::InsertPoint lastPrivIP =
-            converter.getFirOpBuilder().saveInsertionPoint();
-        const auto &objList = std::get<1>(lastPrivate->t);
-        for (const Object &obj : objList) {
-          Fortran::semantics::Symbol *sym = obj.id();
-          converter.copyHostAssociateVar(*sym, &lastPrivIP);
-        }
-      }
-    }
-  }
 
-  // Perform DataSharingProcessor's step2 out of SECTIONS
-  firOpBuilder.setInsertionPointAfter(sectionsOp.getOperation());
-  dsp.processStep2(sectionsOp, false);
-  // Emit implicit barrier to synchronize threads and avoid data
-  // races on post-update of lastprivate variables when `nowait`
-  // clause is present.
-  if (clauseOps.nowaitAttr && hasLastPrivate)
-    firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
-  symTable.popScope();
+  llvm::omp::Directive directive =
+      std::get<parser::OmpSectionsDirective>(beginSectionsDirective.t).v;
+  const parser::CharBlock &source =
+      std::get<parser::OmpSectionsDirective>(beginSectionsDirective.t).source;
+  ConstructQueue queue{
+      buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
+                          eval, source, directive, clauses)};
+  genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
+                 queue.begin());
 }
 
 static void genOMP(Fortran::lower::AbstractConverter &converter,
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index eed63b226133..cb1d1a5a7f3d 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -51,12 +51,6 @@ int64_t getCollapseValue(const List<Clause> &clauses) {
   return 1;
 }
 
-uint32_t getOpenMPVersion(mlir::ModuleOp mod) {
-  if (mlir::Attribute verAttr = mod->getAttr("omp.version"))
-    return llvm::cast<mlir::omp::VersionAttr>(verAttr).getVersion();
-  llvm_unreachable("Expecting OpenMP version attribute in module");
-}
-
 void genObjectList(const ObjectList &objects,
                    Fortran::lower::AbstractConverter &converter,
                    llvm::SmallVectorImpl<mlir::Value> &operands) {
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 8fbb18fa8656..345ce55620ee 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -93,7 +93,6 @@ void gatherFuncAndVarSyms(
     llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause);
 
 int64_t getCollapseValue(const List<Clause> &clauses);
-uint32_t getOpenMPVersion(mlir::ModuleOp mod);
 
 Fortran::semantics::Symbol *
 getOmpObjectSymbol(const Fortran::parser::OmpObject &ompObject);
diff --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90
index 62ba67e5962f..7cc2bc2e0c71 100644
--- a/flang/test/Lower/OpenMP/default-clause-byref.f90
+++ b/flang/test/Lower/OpenMP/default-clause-byref.f90
@@ -161,12 +161,12 @@ subroutine nested_default_clause_tests
 !CHECK: %[[Z:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFnested_default_clause_testsEz"}
 !CHECK: %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z]] {uniq_name = "_QFnested_default_clause_testsEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel   {
+!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"}
+!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
 !CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
-!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"}
-!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_default_clause_testsEz"}
 !CHECK: %[[PRIVATE_Z_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Z]] {uniq_name = "_QFnested_default_clause_testsEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_K:.*]] = fir.alloca i32 {bindc_name = "k", pinned, uniq_name = "_QFnested_default_clause_testsEk"}
@@ -221,6 +221,7 @@ subroutine nested_default_clause_tests
     
     
 !CHECK: omp.parallel {
+!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_testsEx"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_testsEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_testsEy"}
 !CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_testsEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
diff --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90
index a90f0f4ef5f8..843ee6bb7910 100644
--- a/flang/test/Lower/OpenMP/default-clause.f90
+++ b/flang/test/Lower/OpenMP/default-clause.f90
@@ -160,12 +160,12 @@ end program default_clause_lowering
 !CHECK: %[[Z:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFnested_default_clause_test1Ez"}
 !CHECK: %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z]] {uniq_name = "_QFnested_default_clause_test1Ez"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel   {
+!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_test1Ey"}
+!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_test1Ey"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_test1Ex"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFnested_default_clause_test1Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
 !CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
-!CHECK: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_test1Ey"}
-!CHECK: %[[PRIVATE_Y_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Y]] {uniq_name = "_QFnested_default_clause_test1Ey"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_default_clause_test1Ez"}
 !CHECK: %[[PRIVATE_Z_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_Z]] {uniq_name = "_QFnested_default_clause_test1Ez"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[PRIVATE_K:.*]] = fir.alloca i32 {bindc_name = "k", pinned, uniq_name = "_QFnested_default_clause_test1Ek"}
diff --git a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
index b7f11c8c722f..e6ee75c8a5be 100644
--- a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
@@ -145,10 +145,10 @@ end subroutine
 !CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
-!CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
-!CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK-DAG: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2"
 !CHECK-DAG: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
+!CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.wsloop {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index daef02bcfc9a..13a37265762a 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -178,6 +178,12 @@ template <typename T> using ListT = llvm::SmallVector<T, 0>;
 // provide their own specialization that conforms to the above requirements.
 template <typename IdType, typename ExprType> struct ObjectT;
 
+// By default, object equality is only determined by its identity.
+template <typename I, typename E>
+bool operator==(const ObjectT<I, E> &o1, const ObjectT<I, E> &o2) {
+  return o1.id() == o2.id();
+}
+
 template <typename I, typename E> using ObjectListT = ListT<ObjectT<I, E>>;
 
 using DirectiveName = llvm::omp::Directive;
@@ -264,6 +270,32 @@ struct ReductionIdentifierT {
 
 template <typename T, typename I, typename E> //
 using IteratorT = ListT<IteratorSpecifierT<T, I, E>>;
+
+template <typename T>
+std::enable_if_t<T::EmptyTrait::value, bool> operator==(const T &a,
+                                                        const T &b) {
+  return true;
+}
+template <typename T>
+std::enable_if_t<T::IncompleteTrait::value, bool> operator==(const T &a,
+                                                             const T &b) {
+  return true;
+}
+template <typename T>
+std::enable_if_t<T::WrapperTrait::value, bool> operator==(const T &a,
+                                                          const T &b) {
+  return a.v == b.v;
+}
+template <typename T>
+std::enable_if_t<T::TupleTrait::value, bool> operator==(const T &a,
+                                                        const T &b) {
+  return a.t == b.t;
+}
+template <typename T>
+std::enable_if_t<T::UnionTrait::value, bool> operator==(const T &a,
+                                                        const T &b) {
+  return a.u == b.u;
+}
 } // namespace type
 
 template <typename T> using ListT = type::ListT<T>;
@@ -285,6 +317,8 @@ ListT<ResultTy> makeList(ContainerTy &&container, FunctionTy &&func) {
 }
 
 namespace clause {
+using type::operator==;
+
 // V5.2: [8.3.1] `assumption` clauses
 template <typename T, typename I, typename E> //
 struct AbsentT {
@@ -726,7 +760,7 @@ struct LinearT {
   ENUM(LinearModifier, Ref, Val, Uval);
 
   using TupleTrait = std::true_type;
-  // Step == nullptr means 1.
+  // Step == nullopt means 1.
   std::tuple<OPT(StepSimpleModifier), OPT(StepComplexModifier),
              OPT(LinearModifier), List>
       t;
@@ -1142,9 +1176,11 @@ struct UsesAllocatorsT {
   using MemSpace = E;
   using TraitsArray = ObjectT<I, E>;
   using Allocator = E;
-  using AllocatorSpec =
-      std::tuple<OPT(MemSpace), OPT(TraitsArray), Allocator>; // Not a spec name
-  using Allocators = ListT<AllocatorSpec>;                    // Not a spec name
+  struct AllocatorSpec { // Not a spec name
+    using TupleTrait = std::true_type;
+    std::tuple<OPT(MemSpace), OPT(TraitsArray), Allocator> t;
+  };
+  using Allocators = ListT<AllocatorSpec>; // Not a spec name
   using WrapperTrait = std::true_type;
   Allocators v;
 };
@@ -1232,9 +1268,10 @@ using UnionOfAllClausesT = typename type::Union< //
     UnionClausesT<T, I, E>,                      //
     WrapperClausesT<T, I, E>                     //
     >::type;
-
 } // namespace clause
 
+using type::operator==;
+
 // The variant wrapper that encapsulates all possible specific clauses.
 // The `Extras` arguments are additional types representing local extensions
 // to the clause set, e.g.
@@ -1244,6 +1281,9 @@ using UnionOfAllClausesT = typename type::Union< //
 //
 // The member Clause::u will be a variant containing all specific clauses
 // defined above, plus MyClause1 and MyClause2.
+//
+// Note: Any derived class must be constructible from the base class
+// ClauseT<...>.
 template <typename TypeType, typename IdType, typename ExprType,
           typename... Extras>
 struct ClauseT {
@@ -1251,6 +1291,9 @@ struct ClauseT {
   using IdTy = IdType;
   using ExprTy = ExprType;
 
+  // Type of "self" to specify this type given a derived class type.
+  using BaseT = ClauseT<TypeType, IdType, ExprType, Extras...>;
+
   using VariantTy = typename type::Union<
       clause::UnionOfAllClausesT<TypeType, IdType, ExprType>,
       std::variant<Extras...>>::type;
@@ -1260,6 +1303,11 @@ struct ClauseT {
   VariantTy u;
 };
 
+template <typename ClauseType> struct DirectiveWithClauses {
+  llvm::omp::Directive id = llvm::omp::Directive::OMPD_unknown;
+  tomp::type::ListT<ClauseType> clauses;
+};
+
 } // namespace tomp
 
 #undef OPT
diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
new file mode 100644
index 000000000000..edb65ed6d324
--- /dev/null
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
@@ -0,0 +1,403 @@
+//===- ConstructCompositionT.h -- Composing compound constructs -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Given a list of leaf construct, each with a set of clauses, generate the
+// compound construct whose leaf constructs are the given list, and whose clause
+// list is the merged lists of individual leaf clauses.
+//
+// *** At the moment it assumes that the individual constructs and their clauses
+// *** are a subset of those created by splitting a valid compound construct.
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_FRONTEND_OPENMP_CONSTRUCTCOMPOSITIONT_H
+#define LLVM_FRONTEND_OPENMP_CONSTRUCTCOMPOSITIONT_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Frontend/OpenMP/ClauseT.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <iterator>
+#include <optional>
+#include <tuple>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+
+namespace tomp {
+template <typename ClauseType> struct ConstructCompositionT {
+  using ClauseTy = ClauseType;
+
+  using TypeTy = typename ClauseTy::TypeTy;
+  using IdTy = typename ClauseTy::IdTy;
+  using ExprTy = typename ClauseTy::ExprTy;
+
+  ConstructCompositionT(uint32_t version,
+                        llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> leafs);
+
+  DirectiveWithClauses<ClauseTy> merged;
+
+private:
+  // Use an ordered container, since we beed to maintain the order in which
+  // clauses are added to it. This is to avoid non-deterministic output.
+  using ClauseSet = ListT<ClauseTy>;
+
+  enum class Presence {
+    All,  // Clause is preesnt on all leaf constructs that allow it.
+    Some, // Clause is present on some, but not on all constructs.
+    None, // Clause is absent on all constructs.
+  };
+
+  template <typename S>
+  ClauseTy makeClause(llvm::omp::Clause clauseId, S &&specific) {
+    return typename ClauseTy::BaseT{clauseId, std::move(specific)};
+  }
+
+  llvm::omp::Directive
+  makeCompound(llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> parts);
+
+  Presence checkPresence(llvm::omp::Clause clauseId);
+
+  // There are clauses that need special handling:
+  // 1. "if": the "directive-name-modifier" on the merged clause may need
+  // to be set appropriately.
+  // 2. "reduction": implies "privateness" of all objects (incompatible
+  // with "shared"); there are rules for merging modifiers
+  void mergeIf();
+  void mergeReduction();
+  void mergeDSA();
+
+  uint32_t version;
+  llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> leafs;
+
+  // clause id -> set of leaf constructs that contain it
+  std::unordered_map<llvm::omp::Clause, llvm::BitVector> clausePresence;
+  // clause id -> set of instances of that clause
+  std::unordered_map<llvm::omp::Clause, ClauseSet> clauseSets;
+};
+
+template <typename C>
+ConstructCompositionT<C>::ConstructCompositionT(
+    uint32_t version, llvm::ArrayRef<DirectiveWithClauses<C>> leafs)
+    : version(version), leafs(leafs) {
+  // Merge the list of constructs with clauses into a compound construct
+  // with a single list of clauses.
+  // The intended use of this function is in splitting compound constructs,
+  // while preserving composite constituent constructs:
+  // Step 1: split compound construct into leaf constructs.
+  // Step 2: identify composite sub-construct, and merge the constituent leafs.
+  //
+  // *** At the moment it assumes that the individual constructs and their
+  // *** clauses are a subset of those created by splitting a valid compound
+  // *** construct.
+  //
+  // 1. Deduplicate clauses
+  //    - exact duplicates: e.g. shared(x) shared(x) -> shared(x)
+  //    - special cases of clauses differing in modifier:
+  //      (a) reduction: inscan + (none|default) = inscan
+  //      (b) reduction: task + (none|default) = task
+  //      (c) combine repeated "if" clauses if possible
+  // 2. Merge DSA clauses: e.g. private(x) private(y) -> private(x, y).
+  // 3. Resolve potential DSA conflicts (typically due to implied clauses).
+
+  if (leafs.empty())
+    return;
+
+  merged.id = makeCompound(leafs);
+
+  // Populate the two maps:
+  for (const auto &[index, leaf] : llvm::enumerate(leafs)) {
+    for (const auto &clause : leaf.clauses) {
+      // Update clausePresence.
+      auto &pset = clausePresence[clause.id];
+      if (pset.size() < leafs.size())
+        pset.resize(leafs.size());
+      pset.set(index);
+      // Update clauseSets.
+      ClauseSet &cset = clauseSets[clause.id];
+      if (!llvm::is_contained(cset, clause))
+        cset.push_back(clause);
+    }
+  }
+
+  mergeIf();
+  mergeReduction();
+  mergeDSA();
+
+  // Fir the rest of the clauses, just copy them.
+  for (auto &[id, clauses] : clauseSets) {
+    // Skip clauses we've already dealt with.
+    switch (id) {
+    case llvm::omp::Clause::OMPC_if:
+    case llvm::omp::Clause::OMPC_reduction:
+    case llvm::omp::Clause::OMPC_shared:
+    case llvm::omp::Clause::OMPC_private:
+    case llvm::omp::Clause::OMPC_firstprivate:
+    case llvm::omp::Clause::OMPC_lastprivate:
+      continue;
+    default:
+      break;
+    }
+    llvm::append_range(merged.clauses, clauses);
+  }
+}
+
+template <typename C>
+llvm::omp::Directive ConstructCompositionT<C>::makeCompound(
+    llvm::ArrayRef<DirectiveWithClauses<ClauseTy>> parts) {
+  llvm::SmallVector<llvm::omp::Directive> dirIds;
+  llvm::transform(parts, std::back_inserter(dirIds),
+                  [](auto &&dwc) { return dwc.id; });
+
+  return llvm::omp::getCompoundConstruct(dirIds);
+}
+
+template <typename C>
+auto ConstructCompositionT<C>::checkPresence(llvm::omp::Clause clauseId)
+    -> Presence {
+  auto found = clausePresence.find(clauseId);
+  if (found == clausePresence.end())
+    return Presence::None;
+
+  bool OnAll = true, OnNone = true;
+  for (const auto &[index, leaf] : llvm::enumerate(leafs)) {
+    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, clauseId, version))
+      continue;
+
+    if (found->second.test(index))
+      OnNone = false;
+    else
+      OnAll = false;
+  }
+
+  if (OnNone)
+    return Presence::None;
+  if (OnAll)
+    return Presence::All;
+  return Presence::Some;
+}
+
+template <typename C> void ConstructCompositionT<C>::mergeIf() {
+  using IfTy = tomp::clause::IfT<TypeTy, IdTy, ExprTy>;
+  // Deal with the "if" clauses. If it's on all leafs that allow it, then it
+  // will apply to the compound construct. Otherwise it will apply to the
+  // single (assumed) leaf construct.
+  // This assumes that the "if" clauses have the same expression.
+  Presence presence = checkPresence(llvm::omp::Clause::OMPC_if);
+  if (presence == Presence::None)
+    return;
+
+  const ClauseTy &some = *clauseSets[llvm::omp::Clause::OMPC_if].begin();
+  const auto &someIf = std::get<IfTy>(some.u);
+
+  if (presence == Presence::All) {
+    // Create "if" without "directive-name-modifier".
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_if,
+                   IfTy{{/*DirectiveNameModifier=*/std::nullopt,
+                         /*IfExpression=*/std::get<typename IfTy::IfExpression>(
+                             someIf.t)}}));
+  } else {
+    // Find out where it's present and create "if" with the corresponding
+    // "directive-name-modifier".
+    int Idx = clausePresence[llvm::omp::Clause::OMPC_if].find_first();
+    assert(Idx >= 0);
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_if,
+                   IfTy{{/*DirectiveNameModifier=*/leafs[Idx].id,
+                         /*IfExpression=*/std::get<typename IfTy::IfExpression>(
+                             someIf.t)}}));
+  }
+}
+
+template <typename C> void ConstructCompositionT<C>::mergeReduction() {
+  Presence presence = checkPresence(llvm::omp::Clause::OMPC_reduction);
+  if (presence == Presence::None)
+    return;
+
+  using ReductionTy = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
+  using ModifierTy = typename ReductionTy::ReductionModifier;
+  using IdentifiersTy = typename ReductionTy::ReductionIdentifiers;
+  using ListTy = typename ReductionTy::List;
+  // There are exceptions on which constructs "reduction" may appear
+  // (specifically "parallel", and "teams"). Assume that if "reduction"
+  // is present, it can be applied to the compound construct.
+
+  // What's left is to see if there are any modifiers present. Again,
+  // assume that there are no conflicting modifiers.
+  // There can be, however, multiple reductions on different objects.
+  auto equal = [](const ClauseTy &red1, const ClauseTy &red2) {
+    // Extract actual reductions.
+    const auto r1 = std::get<ReductionTy>(red1.u);
+    const auto r2 = std::get<ReductionTy>(red2.u);
+    // Compare everything except modifiers.
+    if (std::get<IdentifiersTy>(r1.t) != std::get<IdentifiersTy>(r2.t))
+      return false;
+    if (std::get<ListTy>(r1.t) != std::get<ListTy>(r2.t))
+      return false;
+    return true;
+  };
+
+  auto getModifier = [](const ClauseTy &clause) {
+    const ReductionTy &red = std::get<ReductionTy>(clause.u);
+    return std::get<std::optional<ModifierTy>>(red.t);
+  };
+
+  const ClauseSet &reductions = clauseSets[llvm::omp::Clause::OMPC_reduction];
+  std::unordered_set<const ClauseTy *> visited;
+  while (reductions.size() != visited.size()) {
+    typename ClauseSet::const_iterator first;
+
+    // Find first non-visited reduction.
+    for (first = reductions.begin(); first != reductions.end(); ++first) {
+      if (visited.count(&*first))
+        continue;
+      visited.insert(&*first);
+      break;
+    }
+
+    std::optional<ModifierTy> modifier = getModifier(*first);
+
+    // Visit all other reductions that are "equal" (with respect to the
+    // definition above) to "first". Collect modifiers.
+    for (auto iter = std::next(first); iter != reductions.end(); ++iter) {
+      if (!equal(*first, *iter))
+        continue;
+      visited.insert(&*iter);
+      if (!modifier || *modifier == ModifierTy::Default)
+        modifier = getModifier(*iter);
+    }
+
+    const auto &firstRed = std::get<ReductionTy>(first->u);
+    merged.clauses.emplace_back(makeClause(
+        llvm::omp::Clause::OMPC_reduction,
+        ReductionTy{
+            {/*ReductionModifier=*/modifier,
+             /*ReductionIdentifiers=*/std::get<IdentifiersTy>(firstRed.t),
+             /*List=*/std::get<ListTy>(firstRed.t)}}));
+  }
+}
+
+template <typename C> void ConstructCompositionT<C>::mergeDSA() {
+  using ObjectTy = tomp::type::ObjectT<IdTy, ExprTy>;
+
+  // Resolve data-sharing attributes.
+  enum DSA : int {
+    None = 0,
+    Shared = 1 << 0,
+    Private = 1 << 1,
+    FirstPrivate = 1 << 2,
+    LastPrivate = 1 << 3,
+    LastPrivateConditional = 1 << 4,
+  };
+
+  // Use ordered containers to avoid non-deterministic output.
+  llvm::SmallVector<std::pair<ObjectTy, int>> objectDsa;
+
+  auto getDsa = [&](const ObjectTy &object) -> std::pair<ObjectTy, int> & {
+    auto found = llvm::find_if(objectDsa, [&](std::pair<ObjectTy, int> &p) {
+      return p.first.id() == object.id();
+    });
+    if (found != objectDsa.end())
+      return *found;
+    return objectDsa.emplace_back(object, DSA::None);
+  };
+
+  using SharedTy = tomp::clause::SharedT<TypeTy, IdTy, ExprTy>;
+  using PrivateTy = tomp::clause::PrivateT<TypeTy, IdTy, ExprTy>;
+  using FirstprivateTy = tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>;
+  using LastprivateTy = tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>;
+
+  // Visit clauses that affect DSA.
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_shared]) {
+    for (auto &object : std::get<SharedTy>(clause.u).v)
+      getDsa(object).second |= DSA::Shared;
+  }
+
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_private]) {
+    for (auto &object : std::get<PrivateTy>(clause.u).v)
+      getDsa(object).second |= DSA::Private;
+  }
+
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_firstprivate]) {
+    for (auto &object : std::get<FirstprivateTy>(clause.u).v)
+      getDsa(object).second |= DSA::FirstPrivate;
+  }
+
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_lastprivate]) {
+    using ModifierTy = typename LastprivateTy::LastprivateModifier;
+    using ListTy = typename LastprivateTy::List;
+    const auto &lastp = std::get<LastprivateTy>(clause.u);
+    for (auto &object : std::get<ListTy>(lastp.t)) {
+      auto &mod = std::get<std::optional<ModifierTy>>(lastp.t);
+      if (mod && *mod == ModifierTy::Conditional) {
+        getDsa(object).second |= DSA::LastPrivateConditional;
+      } else {
+        getDsa(object).second |= DSA::LastPrivate;
+      }
+    }
+  }
+
+  // Check reductions as well, clear "shared" if set.
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_reduction]) {
+    using ReductionTy = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
+    using ListTy = typename ReductionTy::List;
+    for (auto &object : std::get<ListTy>(std::get<ReductionTy>(clause.u).t))
+      getDsa(object).second &= ~DSA::Shared;
+  }
+
+  tomp::ListT<ObjectTy> privateObj, sharedObj, firstpObj, lastpObj, lastpcObj;
+  for (auto &[object, dsa] : objectDsa) {
+    if (dsa &
+        (DSA::FirstPrivate | DSA::LastPrivate | DSA::LastPrivateConditional)) {
+      if (dsa & DSA::FirstPrivate)
+        firstpObj.push_back(object); // no else
+      if (dsa & DSA::LastPrivateConditional)
+        lastpcObj.push_back(object);
+      else if (dsa & DSA::LastPrivate)
+        lastpObj.push_back(object);
+    } else if (dsa & DSA::Private) {
+      privateObj.push_back(object);
+    } else if (dsa & DSA::Shared) {
+      sharedObj.push_back(object);
+    }
+  }
+
+  // Materialize each clause.
+  if (!privateObj.empty()) {
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_private,
+                   PrivateTy{/*List=*/std::move(privateObj)}));
+  }
+  if (!sharedObj.empty()) {
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_shared,
+                   SharedTy{/*List=*/std::move(sharedObj)}));
+  }
+  if (!firstpObj.empty()) {
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_firstprivate,
+                   FirstprivateTy{/*List=*/std::move(firstpObj)}));
+  }
+  if (!lastpObj.empty()) {
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_lastprivate,
+                   LastprivateTy{{/*LastprivateModifier=*/std::nullopt,
+                                  /*List=*/std::move(lastpObj)}}));
+  }
+  if (!lastpcObj.empty()) {
+    auto conditional = LastprivateTy::LastprivateModifier::Conditional;
+    merged.clauses.emplace_back(
+        makeClause(llvm::omp::Clause::OMPC_lastprivate,
+                   LastprivateTy{{/*LastprivateModifier=*/conditional,
+                                  /*List=*/std::move(lastpcObj)}}));
+  }
+}
+} // namespace tomp
+
+#endif // LLVM_FRONTEND_OPENMP_CONSTRUCTCOMPOSITIONT_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
new file mode 100644
index 000000000000..5f12c62b832f
--- /dev/null
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
@@ -0,0 +1,1160 @@
+//===- ConstructDecompositionT.h -- Decomposing compound constructs -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Given a compound construct with a set of clauses, generate the list of
+// constituent leaf constructs, each with a list of clauses that apply to it.
+//
+// Note: Clauses that are not originally present, but that are implied by the
+// OpenMP spec are materialized, and are present in the output.
+//
+// Note: Composite constructs will also be broken up into leaf constructs.
+// If composite constructs require processing as a whole, the lists of clauses
+// for each leaf constituent should be merged.
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_FRONTEND_OPENMP_CONSTRUCTDECOMPOSITIONT_H
+#define LLVM_FRONTEND_OPENMP_CONSTRUCTDECOMPOSITIONT_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/Frontend/OpenMP/ClauseT.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <iterator>
+#include <list>
+#include <optional>
+#include <tuple>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <variant>
+
+static inline llvm::ArrayRef<llvm::omp::Directive> getWorksharing() {
+  static llvm::omp::Directive worksharing[] = {
+      llvm::omp::Directive::OMPD_do,     llvm::omp::Directive::OMPD_for,
+      llvm::omp::Directive::OMPD_scope,  llvm::omp::Directive::OMPD_sections,
+      llvm::omp::Directive::OMPD_single, llvm::omp::Directive::OMPD_workshare,
+  };
+  return worksharing;
+}
+
+static inline llvm::ArrayRef<llvm::omp::Directive> getWorksharingLoop() {
+  static llvm::omp::Directive worksharingLoop[] = {
+      llvm::omp::Directive::OMPD_do,
+      llvm::omp::Directive::OMPD_for,
+  };
+  return worksharingLoop;
+}
+
+namespace detail {
+template <typename Container, typename Predicate>
+typename std::remove_reference_t<Container>::iterator
+find_unique(Container &&container, Predicate &&pred) {
+  auto first = std::find_if(container.begin(), container.end(), pred);
+  if (first == container.end())
+    return first;
+  auto second = std::find_if(std::next(first), container.end(), pred);
+  if (second == container.end())
+    return first;
+  return container.end();
+}
+} // namespace detail
+
+namespace tomp {
+
+// ClauseType - Either instance of ClauseT, or a type derived from ClauseT.
+//
+// This is the clause representation in the code using this infrastructure.
+//
+// HelperType - A class that implements two member functions:
+//
+//   // Return the base object of the given object, if any.
+//   std::optional<Object> getBaseObject(const Object &object) const
+//   // Return the iteration variable of the outermost loop associated
+//   // with the construct being worked on, if any.
+//   std::optional<Object> getLoopIterVar() const
+template <typename ClauseType, typename HelperType>
+struct ConstructDecompositionT {
+  using ClauseTy = ClauseType;
+
+  using TypeTy = typename ClauseTy::TypeTy;
+  using IdTy = typename ClauseTy::IdTy;
+  using ExprTy = typename ClauseTy::ExprTy;
+  using HelperTy = HelperType;
+  using ObjectTy = tomp::ObjectT<IdTy, ExprTy>;
+
+  using ClauseSet = std::unordered_set<const ClauseTy *>;
+
+  ConstructDecompositionT(uint32_t ver, HelperType &helper,
+                          llvm::omp::Directive dir,
+                          llvm::ArrayRef<ClauseTy> clauses)
+      : version(ver), construct(dir), helper(helper) {
+    for (const ClauseTy &clause : clauses)
+      nodes.push_back(&clause);
+
+    bool success = split();
+    if (!success)
+      return;
+
+    // Copy the individual leaf directives with their clauses to the
+    // output list. Copy by value, since we don't own the storage
+    // with the input clauses, and the internal representation uses
+    // clause addresses.
+    for (auto &leaf : leafs) {
+      output.push_back({leaf.id, {}});
+      auto &out = output.back();
+      for (const ClauseTy *c : leaf.clauses)
+        out.clauses.push_back(*c);
+    }
+  }
+
+  tomp::ListT<DirectiveWithClauses<ClauseType>> output;
+
+private:
+  bool split();
+
+  struct LeafReprInternal {
+    llvm::omp::Directive id = llvm::omp::Directive::OMPD_unknown;
+    tomp::type::ListT<const ClauseTy *> clauses;
+  };
+
+  LeafReprInternal *findDirective(llvm::omp::Directive dirId) {
+    auto found = llvm::find_if(
+        leafs, [&](const LeafReprInternal &leaf) { return leaf.id == dirId; });
+    return found != leafs.end() ? &*found : nullptr;
+  }
+
+  ClauseSet *findClausesWith(const ObjectTy &object) {
+    if (auto found = syms.find(object.id()); found != syms.end())
+      return &found->second;
+    return nullptr;
+  }
+
+  template <typename S>
+  ClauseTy *makeClause(llvm::omp::Clause clauseId, S &&specific) {
+    implicit.push_back(typename ClauseTy::BaseT{clauseId, std::move(specific)});
+    return &implicit.back();
+  }
+
+  void addClauseSymsToMap(const ObjectTy &object, const ClauseTy *);
+  void addClauseSymsToMap(const tomp::ObjectListT<IdTy, ExprTy> &objects,
+                          const ClauseTy *);
+  void addClauseSymsToMap(const TypeTy &item, const ClauseTy *);
+  void addClauseSymsToMap(const ExprTy &item, const ClauseTy *);
+  void addClauseSymsToMap(const tomp::clause::MapT<TypeTy, IdTy, ExprTy> &item,
+                          const ClauseTy *);
+
+  template <typename U>
+  void addClauseSymsToMap(const std::optional<U> &item, const ClauseTy *);
+  template <typename U>
+  void addClauseSymsToMap(const tomp::ListT<U> &item, const ClauseTy *);
+  template <typename... U, size_t... Is>
+  void addClauseSymsToMap(const std::tuple<U...> &item, const ClauseTy *,
+                          std::index_sequence<Is...> = {});
+  template <typename U>
+  std::enable_if_t<std::is_enum_v<llvm::remove_cvref_t<U>>, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  template <typename U>
+  std::enable_if_t<llvm::remove_cvref_t<U>::EmptyTrait::value, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  template <typename U>
+  std::enable_if_t<llvm::remove_cvref_t<U>::IncompleteTrait::value, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  template <typename U>
+  std::enable_if_t<llvm::remove_cvref_t<U>::WrapperTrait::value, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  template <typename U>
+  std::enable_if_t<llvm::remove_cvref_t<U>::TupleTrait::value, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  template <typename U>
+  std::enable_if_t<llvm::remove_cvref_t<U>::UnionTrait::value, void>
+  addClauseSymsToMap(U &&item, const ClauseTy *);
+
+  // Apply a clause to the only directive that allows it. If there are no
+  // directives that allow it, or if there is more that one, do not apply
+  // anything and return false, otherwise return true.
+  bool applyToUnique(const ClauseTy *node);
+
+  // Apply a clause to the first directive in given range that allows it.
+  // If such a directive does not exist, return false, otherwise return true.
+  template <typename Iterator>
+  bool applyToFirst(const ClauseTy *node, llvm::iterator_range<Iterator> range);
+
+  // Apply a clause to the innermost directive that allows it. If such a
+  // directive does not exist, return false, otherwise return true.
+  bool applyToInnermost(const ClauseTy *node);
+
+  // Apply a clause to the outermost directive that allows it. If such a
+  // directive does not exist, return false, otherwise return true.
+  bool applyToOutermost(const ClauseTy *node);
+
+  template <typename Predicate>
+  bool applyIf(const ClauseTy *node, Predicate shouldApply);
+
+  bool applyToAll(const ClauseTy *node);
+
+  template <typename Clause>
+  bool applyClause(Clause &&clause, const ClauseTy *node);
+
+  bool applyClause(const tomp::clause::CollapseT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::PrivateT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool
+  applyClause(const tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy> &clause,
+              const ClauseTy *);
+  bool
+  applyClause(const tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy> &clause,
+              const ClauseTy *);
+  bool applyClause(const tomp::clause::SharedT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::DefaultT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool
+  applyClause(const tomp::clause::ThreadLimitT<TypeTy, IdTy, ExprTy> &clause,
+              const ClauseTy *);
+  bool applyClause(const tomp::clause::OrderT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::AllocateT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::ReductionT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::IfT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::LinearT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+  bool applyClause(const tomp::clause::NowaitT<TypeTy, IdTy, ExprTy> &clause,
+                   const ClauseTy *);
+
+  uint32_t version;
+  llvm::omp::Directive construct;
+  HelperType &helper;
+  ListT<LeafReprInternal> leafs;
+  tomp::ListT<const ClauseTy *> nodes;
+  std::list<ClauseTy> implicit; // Container for materialized implicit clauses.
+                                // Inserting must preserve element addresses.
+  std::unordered_map<IdTy, ClauseSet> syms;
+  std::unordered_set<IdTy> mapBases;
+};
+
+// Deduction guide
+template <typename ClauseType, typename HelperType>
+ConstructDecompositionT(uint32_t, HelperType &, llvm::omp::Directive,
+                        llvm::ArrayRef<ClauseType>)
+    -> ConstructDecompositionT<ClauseType, HelperType>;
+
+template <typename C, typename H>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(const ObjectTy &object,
+                                                       const ClauseTy *node) {
+  syms[object.id()].insert(node);
+}
+
+template <typename C, typename H>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(
+    const tomp::ObjectListT<IdTy, ExprTy> &objects, const ClauseTy *node) {
+  for (auto &object : objects)
+    syms[object.id()].insert(node);
+}
+
+template <typename C, typename H>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(const TypeTy &item,
+                                                       const ClauseTy *node) {
+  // Nothing to do for types.
+}
+
+template <typename C, typename H>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(const ExprTy &item,
+                                                       const ClauseTy *node) {
+  // Nothing to do for expressions.
+}
+
+template <typename C, typename H>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(
+    const tomp::clause::MapT<TypeTy, IdTy, ExprTy> &item,
+    const ClauseTy *node) {
+  auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(item.t);
+  addClauseSymsToMap(objects, node);
+  for (auto &object : objects) {
+    if (auto base = helper.getBaseObject(object))
+      mapBases.insert(base->id());
+  }
+}
+
+template <typename C, typename H>
+template <typename U>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(
+    const std::optional<U> &item, const ClauseTy *node) {
+  if (item)
+    addClauseSymsToMap(*item, node);
+}
+
+template <typename C, typename H>
+template <typename U>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(
+    const tomp::ListT<U> &item, const ClauseTy *node) {
+  for (auto &s : item)
+    addClauseSymsToMap(s, node);
+}
+
+template <typename C, typename H>
+template <typename... U, size_t... Is>
+void ConstructDecompositionT<C, H>::addClauseSymsToMap(
+    const std::tuple<U...> &item, const ClauseTy *node,
+    std::index_sequence<Is...>) {
+  (void)node; // Silence strange warning from GCC.
+  (addClauseSymsToMap(std::get<Is>(item), node), ...);
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<std::is_enum_v<llvm::remove_cvref_t<U>>, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  // Nothing to do for enums.
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<llvm::remove_cvref_t<U>::EmptyTrait::value, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  // Nothing to do for an empty class.
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<llvm::remove_cvref_t<U>::IncompleteTrait::value, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  // Nothing to do for an incomplete class (they're empty).
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<llvm::remove_cvref_t<U>::WrapperTrait::value, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  addClauseSymsToMap(item.v, node);
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<llvm::remove_cvref_t<U>::TupleTrait::value, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  constexpr size_t tuple_size =
+      std::tuple_size_v<llvm::remove_cvref_t<decltype(item.t)>>;
+  addClauseSymsToMap(item.t, node, std::make_index_sequence<tuple_size>{});
+}
+
+template <typename C, typename H>
+template <typename U>
+std::enable_if_t<llvm::remove_cvref_t<U>::UnionTrait::value, void>
+ConstructDecompositionT<C, H>::addClauseSymsToMap(U &&item,
+                                                  const ClauseTy *node) {
+  std::visit([&](auto &&s) { addClauseSymsToMap(s, node); }, item.u);
+}
+
+// Apply a clause to the only directive that allows it. If there are no
+// directives that allow it, or if there is more that one, do not apply
+// anything and return false, otherwise return true.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyToUnique(const ClauseTy *node) {
+  auto unique = detail::find_unique(leafs, [=](const auto &dirInfo) {
+    return llvm::omp::isAllowedClauseForDirective(dirInfo.id, node->id,
+                                                  version);
+  });
+
+  if (unique != leafs.end()) {
+    unique->clauses.push_back(node);
+    return true;
+  }
+  return false;
+}
+
+// Apply a clause to the first directive in given range that allows it.
+// If such a directive does not exist, return false, otherwise return true.
+template <typename C, typename H>
+template <typename Iterator>
+bool ConstructDecompositionT<C, H>::applyToFirst(
+    const ClauseTy *node, llvm::iterator_range<Iterator> range) {
+  if (range.empty())
+    return false;
+
+  for (auto &leaf : range) {
+    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, node->id, version))
+      continue;
+    leaf.clauses.push_back(node);
+    return true;
+  }
+  return false;
+}
+
+// Apply a clause to the innermost directive that allows it. If such a
+// directive does not exist, return false, otherwise return true.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyToInnermost(const ClauseTy *node) {
+  return applyToFirst(node, llvm::reverse(leafs));
+}
+
+// Apply a clause to the outermost directive that allows it. If such a
+// directive does not exist, return false, otherwise return true.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyToOutermost(const ClauseTy *node) {
+  return applyToFirst(node, llvm::iterator_range(leafs));
+}
+
+template <typename C, typename H>
+template <typename Predicate>
+bool ConstructDecompositionT<C, H>::applyIf(const ClauseTy *node,
+                                            Predicate shouldApply) {
+  bool applied = false;
+  for (auto &leaf : leafs) {
+    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, node->id, version))
+      continue;
+    if (!shouldApply(leaf))
+      continue;
+    leaf.clauses.push_back(node);
+    applied = true;
+  }
+
+  return applied;
+}
+
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyToAll(const ClauseTy *node) {
+  return applyIf(node, [](auto) { return true; });
+}
+
+template <typename C, typename H>
+template <typename Clause>
+bool ConstructDecompositionT<C, H>::applyClause(Clause &&clause,
+                                                const ClauseTy *node) {
+  // The default behavior is to find the unique directive to which the
+  // given clause may be applied. If there are no such directives, or
+  // if there are multiple ones, flag an error.
+  // From "OpenMP Application Programming Interface", Version 5.2:
+  // S Some clauses are permitted only on a single leaf construct of the
+  // S combined or composite construct, in which case the effect is as if
+  // S the clause is applied to that specific construct. (p339, 31-33)
+  if (applyToUnique(node))
+    return true;
+
+  return false;
+}
+
+// COLLAPSE
+// [5.2:93:20-21]
+// Directives: distribute, do, for, loop, simd, taskloop
+//
+// [5.2:339:35]
+// (35) The collapse clause is applied once to the combined or composite
+// construct.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::CollapseT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // Apply "collapse" to the innermost directive. If it's not one that
+  // allows it flag an error.
+  if (!leafs.empty()) {
+    auto &last = leafs.back();
+
+    if (llvm::omp::isAllowedClauseForDirective(last.id, node->id, version)) {
+      last.clauses.push_back(node);
+      return true;
+    }
+  }
+
+  return false;
+}
+
+// PRIVATE
+// [5.2:111:5-7]
+// Directives: distribute, do, for, loop, parallel, scope, sections, simd,
+// single, target, task, taskloop, teams
+//
+// [5.2:340:1-2]
+// (1) The effect of the 1 private clause is as if it is applied only to the
+// innermost leaf construct that permits it.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::PrivateT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  return applyToInnermost(node);
+}
+
+// FIRSTPRIVATE
+// [5.2:112:5-7]
+// Directives: distribute, do, for, parallel, scope, sections, single, target,
+// task, taskloop, teams
+//
+// [5.2:340:3-20]
+// (3) The effect of the firstprivate clause is as if it is applied to one or
+// more leaf constructs as follows:
+//  (5) To the distribute construct if it is among the constituent constructs;
+//  (6) To the teams construct if it is among the constituent constructs and the
+//      distribute construct is not;
+//  (8) To a worksharing construct that accepts the clause if one is among the
+//      constituent constructs;
+//  (9) To the taskloop construct if it is among the constituent constructs;
+// (10) To the parallel construct if it is among the constituent constructs and
+//      neither a taskloop construct nor a worksharing construct that accepts
+//      the clause is among them;
+// (12) To the target construct if it is among the constituent constructs and
+//      the same list item neither appears in a lastprivate clause nor is the
+//      base variable or base pointer of a list item that appears in a map
+//      clause.
+//
+// (15) If the parallel construct is among the constituent constructs and the
+// effect is not as if the firstprivate clause is applied to it by the above
+// rules, then the effect is as if the shared clause with the same list item is
+// applied to the parallel construct.
+// (17) If the teams construct is among the constituent constructs and the
+// effect is not as if the firstprivate clause is applied to it by the above
+// rules, then the effect is as if the shared clause with the same list item is
+// applied to the teams construct.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  bool applied = false;
+
+  // [5.2:340:3-6]
+  auto dirDistribute = findDirective(llvm::omp::OMPD_distribute);
+  auto dirTeams = findDirective(llvm::omp::OMPD_teams);
+  if (dirDistribute != nullptr) {
+    dirDistribute->clauses.push_back(node);
+    applied = true;
+    // [5.2:340:17]
+    if (dirTeams != nullptr) {
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/clause.v});
+      dirTeams->clauses.push_back(shared);
+    }
+  } else if (dirTeams != nullptr) {
+    dirTeams->clauses.push_back(node);
+    applied = true;
+  }
+
+  // [5.2:340:8]
+  auto findWorksharing = [&]() {
+    auto worksharing = getWorksharing();
+    for (auto &leaf : leafs) {
+      auto found = llvm::find(worksharing, leaf.id);
+      if (found != std::end(worksharing))
+        return &leaf;
+    }
+    return static_cast<typename decltype(leafs)::value_type *>(nullptr);
+  };
+
+  auto dirWorksharing = findWorksharing();
+  if (dirWorksharing != nullptr) {
+    dirWorksharing->clauses.push_back(node);
+    applied = true;
+  }
+
+  // [5.2:340:9]
+  auto dirTaskloop = findDirective(llvm::omp::OMPD_taskloop);
+  if (dirTaskloop != nullptr) {
+    dirTaskloop->clauses.push_back(node);
+    applied = true;
+  }
+
+  // [5.2:340:10]
+  auto dirParallel = findDirective(llvm::omp::OMPD_parallel);
+  if (dirParallel != nullptr) {
+    if (dirTaskloop == nullptr && dirWorksharing == nullptr) {
+      dirParallel->clauses.push_back(node);
+      applied = true;
+    } else {
+      // [5.2:340:15]
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/clause.v});
+      dirParallel->clauses.push_back(shared);
+    }
+  }
+
+  // [5.2:340:12]
+  auto inLastprivate = [&](const ObjectTy &object) {
+    if (ClauseSet *set = findClausesWith(object)) {
+      return llvm::find_if(*set, [](const ClauseTy *c) {
+               return c->id == llvm::omp::Clause::OMPC_lastprivate;
+             }) != set->end();
+    }
+    return false;
+  };
+
+  auto dirTarget = findDirective(llvm::omp::OMPD_target);
+  if (dirTarget != nullptr) {
+    tomp::ObjectListT<IdTy, ExprTy> objects;
+    llvm::copy_if(
+        clause.v, std::back_inserter(objects), [&](const ObjectTy &object) {
+          return !inLastprivate(object) && !mapBases.count(object.id());
+        });
+    if (!objects.empty()) {
+      auto *firstp = makeClause(
+          llvm::omp::Clause::OMPC_firstprivate,
+          tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>{/*List=*/objects});
+      dirTarget->clauses.push_back(firstp);
+      applied = true;
+    }
+  }
+
+  // "task" is not handled by any of the cases above.
+  if (auto dirTask = findDirective(llvm::omp::OMPD_task)) {
+    dirTask->clauses.push_back(node);
+    applied = true;
+  }
+
+  return applied;
+}
+
+// LASTPRIVATE
+// [5.2:115:7-8]
+// Directives: distribute, do, for, loop, sections, simd, taskloop
+//
+// [5.2:340:21-30]
+// (21) The effect of the lastprivate clause is as if it is applied to all leaf
+// constructs that permit the clause.
+// (22) If the parallel construct is among the constituent constructs and the
+// list item is not also specified in the firstprivate clause, then the effect
+// of the lastprivate clause is as if the shared clause with the same list item
+// is applied to the parallel construct.
+// (24) If the teams construct is among the constituent constructs and the list
+// item is not also specified in the firstprivate clause, then the effect of the
+// lastprivate clause is as if the shared clause with the same list item is
+// applied to the teams construct.
+// (27) If the target construct is among the constituent constructs and the list
+// item is not the base variable or base pointer of a list item that appears in
+// a map clause, the effect of the lastprivate clause is as if the same list
+// item appears in a map clause with a map-type of tofrom.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  bool applied = false;
+
+  // [5.2:340:21]
+  applied = applyToAll(node);
+  if (!applied)
+    return false;
+
+  auto inFirstprivate = [&](const ObjectTy &object) {
+    if (ClauseSet *set = findClausesWith(object)) {
+      return llvm::find_if(*set, [](const ClauseTy *c) {
+               return c->id == llvm::omp::Clause::OMPC_firstprivate;
+             }) != set->end();
+    }
+    return false;
+  };
+
+  auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(clause.t);
+
+  // Prepare list of objects that could end up in a "shared" clause.
+  tomp::ObjectListT<IdTy, ExprTy> sharedObjects;
+  llvm::copy_if(
+      objects, std::back_inserter(sharedObjects),
+      [&](const ObjectTy &object) { return !inFirstprivate(object); });
+
+  if (!sharedObjects.empty()) {
+    // [5.2:340:22]
+    if (auto dirParallel = findDirective(llvm::omp::OMPD_parallel)) {
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
+      dirParallel->clauses.push_back(shared);
+      applied = true;
+    }
+
+    // [5.2:340:24]
+    if (auto dirTeams = findDirective(llvm::omp::OMPD_teams)) {
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
+      dirTeams->clauses.push_back(shared);
+      applied = true;
+    }
+  }
+
+  // [5.2:340:27]
+  if (auto dirTarget = findDirective(llvm::omp::OMPD_target)) {
+    tomp::ObjectListT<IdTy, ExprTy> tofrom;
+    llvm::copy_if(
+        objects, std::back_inserter(tofrom),
+        [&](const ObjectTy &object) { return !mapBases.count(object.id()); });
+
+    if (!tofrom.empty()) {
+      using MapType =
+          typename tomp::clause::MapT<TypeTy, IdTy, ExprTy>::MapType;
+      auto *map =
+          makeClause(llvm::omp::Clause::OMPC_map,
+                     tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
+                         {/*MapType=*/MapType::Tofrom,
+                          /*MapTypeModifier=*/std::nullopt,
+                          /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
+                          /*LocatorList=*/std::move(tofrom)}});
+      dirTarget->clauses.push_back(map);
+      applied = true;
+    }
+  }
+
+  return applied;
+}
+
+// SHARED
+// [5.2:110:5-6]
+// Directives: parallel, task, taskloop, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::SharedT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // [5.2:340:31]
+  return applyToAll(node);
+}
+
+// DEFAULT
+// [5.2:109:5-6]
+// Directives: parallel, task, taskloop, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::DefaultT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // [5.2:340:31]
+  return applyToAll(node);
+}
+
+// THREAD_LIMIT
+// [5.2:277:14-15]
+// Directives: target, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::ThreadLimitT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // [5.2:340:31]
+  return applyToAll(node);
+}
+
+// ORDER
+// [5.2:234:3-4]
+// Directives: distribute, do, for, loop, simd
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::OrderT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // [5.2:340:31]
+  return applyToAll(node);
+}
+
+// ALLOCATE
+// [5.2:178:7-9]
+// Directives: allocators, distribute, do, for, parallel, scope, sections,
+// single, target, task, taskgroup, taskloop, teams
+//
+// [5.2:340:33-35]
+// (33) The effect of the allocate clause is as if it is applied to all leaf
+// constructs that permit the clause and to which a data-sharing attribute
+// clause that may create a private copy of the same list item is applied.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::AllocateT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // This one needs to be applied at the end, once we know which clauses are
+  // assigned to which leaf constructs.
+
+  // [5.2:340:33]
+  auto canMakePrivateCopy = [](llvm::omp::Clause id) {
+    switch (id) {
+    case llvm::omp::Clause::OMPC_firstprivate:
+    case llvm::omp::Clause::OMPC_lastprivate:
+    case llvm::omp::Clause::OMPC_private:
+      return true;
+    default:
+      return false;
+    }
+  };
+
+  bool applied = applyIf(node, [&](const auto &leaf) {
+    return llvm::any_of(leaf.clauses, [&](const ClauseTy *n) {
+      return canMakePrivateCopy(n->id);
+    });
+  });
+
+  return applied;
+}
+
+// REDUCTION
+// [5.2:134:17-18]
+// Directives: do, for, loop, parallel, scope, sections, simd, taskloop, teams
+//
+// [5.2:340:36-37], [5.2:341:1-13]
+// (36) The effect of the reduction clause is as if it is applied to all leaf
+// constructs that permit the clause, except for the following constructs:
+//  (1) The parallel construct, when combined with the sections,
+//      worksharing-loop, loop, or taskloop construct; and
+//  (3) The teams construct, when combined with the loop construct.
+// (4) For the parallel and teams constructs above, the effect of the reduction
+// clause instead is as if each list item or, for any list item that is an array
+// item, its corresponding base array or base pointer appears in a shared clause
+// for the construct.
+// (6) If the task reduction-modifier is specified, the effect is as if it only
+// modifies the behavior of the reduction clause on the innermost leaf construct
+// that accepts the modifier (see Section 5.5.8).
+// (8) If the inscan reduction-modifier is specified, the effect is as if it
+// modifies the behavior of the reduction clause on all constructs of the
+// combined construct to which the clause is applied and that accept the
+// modifier.
+// (10) If a list item in a reduction clause on a combined target construct does
+// not have the same base variable or base pointer as a list item in a map
+// clause on the construct, then the effect is as if the list item in the
+// reduction clause appears as a list item in a map clause with a map-type of
+// tofrom.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::ReductionT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  using ReductionTy = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
+
+  // [5.2:340:36], [5.2:341:1], [5.2:341:3]
+  bool applyToParallel = true, applyToTeams = true;
+
+  auto dirParallel = findDirective(llvm::omp::Directive::OMPD_parallel);
+  if (dirParallel) {
+    auto exclusions = llvm::concat<const llvm::omp::Directive>(
+        getWorksharingLoop(), tomp::ListT<llvm::omp::Directive>{
+                                  llvm::omp::Directive::OMPD_loop,
+                                  llvm::omp::Directive::OMPD_sections,
+                                  llvm::omp::Directive::OMPD_taskloop,
+                              });
+    auto present = [&](llvm::omp::Directive id) {
+      return findDirective(id) != nullptr;
+    };
+
+    if (llvm::any_of(exclusions, present))
+      applyToParallel = false;
+  }
+
+  auto dirTeams = findDirective(llvm::omp::Directive::OMPD_teams);
+  if (dirTeams) {
+    // The only exclusion is OMPD_loop.
+    if (findDirective(llvm::omp::Directive::OMPD_loop))
+      applyToTeams = false;
+  }
+
+  using ReductionModifier = typename ReductionTy::ReductionModifier;
+  using ReductionIdentifiers = typename ReductionTy::ReductionIdentifiers;
+
+  auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(clause.t);
+  auto &modifier = std::get<std::optional<ReductionModifier>>(clause.t);
+
+  // Apply the reduction clause first to all directives according to the spec.
+  // If the reduction was applied at least once, proceed with the data sharing
+  // side-effects.
+  bool applied = false;
+
+  // [5.2:341:6], [5.2:341:8]
+  auto isValidModifier = [](llvm::omp::Directive dir, ReductionModifier mod,
+                            bool alreadyApplied) {
+    switch (mod) {
+    case ReductionModifier::Inscan:
+      // According to [5.2:135:11-13], "inscan" only applies to
+      // worksharing-loop, worksharing-loop-simd, or "simd" constructs.
+      return dir == llvm::omp::Directive::OMPD_simd ||
+             llvm::is_contained(getWorksharingLoop(), dir);
+    case ReductionModifier::Task:
+      if (alreadyApplied)
+        return false;
+      // According to [5.2:135:16-18], "task" only applies to "parallel" and
+      // worksharing constructs.
+      return dir == llvm::omp::Directive::OMPD_parallel ||
+             llvm::is_contained(getWorksharing(), dir);
+    case ReductionModifier::Default:
+      return true;
+    }
+    llvm_unreachable("Unexpected modifier");
+  };
+
+  auto *unmodified = makeClause(
+      llvm::omp::Clause::OMPC_reduction,
+      ReductionTy{
+          {/*ReductionModifier=*/std::nullopt,
+           /*ReductionIdentifiers=*/std::get<ReductionIdentifiers>(clause.t),
+           /*List=*/objects}});
+
+  ReductionModifier effective =
+      modifier.has_value() ? *modifier : ReductionModifier::Default;
+  bool effectiveApplied = false;
+  // Walk over the leaf constructs starting from the innermost, and apply
+  // the clause as required by the spec.
+  for (auto &leaf : llvm::reverse(leafs)) {
+    if (!llvm::omp::isAllowedClauseForDirective(leaf.id, node->id, version))
+      continue;
+    if (!applyToParallel && &leaf == dirParallel)
+      continue;
+    if (!applyToTeams && &leaf == dirTeams)
+      continue;
+    // Some form of the clause will be applied past this point.
+    if (isValidModifier(leaf.id, effective, effectiveApplied)) {
+      // Apply clause with modifier.
+      leaf.clauses.push_back(node);
+      effectiveApplied = true;
+    } else {
+      // Apply clause without modifier.
+      leaf.clauses.push_back(unmodified);
+    }
+    applied = true;
+  }
+
+  if (!applied)
+    return false;
+
+  tomp::ObjectListT<IdTy, ExprTy> sharedObjects;
+  llvm::transform(objects, std::back_inserter(sharedObjects),
+                  [&](const ObjectTy &object) {
+                    auto maybeBase = helper.getBaseObject(object);
+                    return maybeBase ? *maybeBase : object;
+                  });
+
+  // [5.2:341:4]
+  if (!sharedObjects.empty()) {
+    if (dirParallel && !applyToParallel) {
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
+      dirParallel->clauses.push_back(shared);
+    }
+    if (dirTeams && !applyToTeams) {
+      auto *shared = makeClause(
+          llvm::omp::Clause::OMPC_shared,
+          tomp::clause::SharedT<TypeTy, IdTy, ExprTy>{/*List=*/sharedObjects});
+      dirTeams->clauses.push_back(shared);
+    }
+  }
+
+  // [5.2:341:10]
+  auto dirTarget = findDirective(llvm::omp::Directive::OMPD_target);
+  if (dirTarget && leafs.size() > 1) {
+    tomp::ObjectListT<IdTy, ExprTy> tofrom;
+    llvm::copy_if(objects, std::back_inserter(tofrom),
+                  [&](const ObjectTy &object) {
+                    if (auto maybeBase = helper.getBaseObject(object))
+                      return !mapBases.count(maybeBase->id());
+                    return !mapBases.count(object.id()); // XXX is this ok?
+                  });
+    if (!tofrom.empty()) {
+      using MapType =
+          typename tomp::clause::MapT<TypeTy, IdTy, ExprTy>::MapType;
+      auto *map = makeClause(
+          llvm::omp::Clause::OMPC_map,
+          tomp::clause::MapT<TypeTy, IdTy, ExprTy>{
+              {/*MapType=*/MapType::Tofrom, /*MapTypeModifier=*/std::nullopt,
+               /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
+               /*LocatorList=*/std::move(tofrom)}});
+
+      dirTarget->clauses.push_back(map);
+      applied = true;
+    }
+  }
+
+  return applied;
+}
+
+// IF
+// [5.2:72:7-9]
+// Directives: cancel, parallel, simd, target, target data, target enter data,
+// target exit data, target update, task, taskloop
+//
+// [5.2:72:15-18]
+// (15) For combined or composite constructs, the if clause only applies to the
+// semantics of the construct named in the directive-name-modifier.
+// (16) For a combined or composite construct, if no directive-name-modifier is
+// specified then the if clause applies to all constituent constructs to which
+// an if clause can apply.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::IfT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  using DirectiveNameModifier =
+      typename clause::IfT<TypeTy, IdTy, ExprTy>::DirectiveNameModifier;
+  using IfExpression = typename clause::IfT<TypeTy, IdTy, ExprTy>::IfExpression;
+  auto &modifier = std::get<std::optional<DirectiveNameModifier>>(clause.t);
+
+  if (modifier) {
+    llvm::omp::Directive dirId = *modifier;
+    auto *unmodified =
+        makeClause(llvm::omp::Clause::OMPC_if,
+                   tomp::clause::IfT<TypeTy, IdTy, ExprTy>{
+                       {/*DirectiveNameModifier=*/std::nullopt,
+                        /*IfExpression=*/std::get<IfExpression>(clause.t)}});
+
+    if (auto *hasDir = findDirective(dirId)) {
+      hasDir->clauses.push_back(unmodified);
+      return true;
+    }
+    return false;
+  }
+
+  return applyToAll(node);
+}
+
+// LINEAR
+// [5.2:118:1-2]
+// Directives: declare simd, do, for, simd
+//
+// [5.2:341:15-22]
+// (15.1) The effect of the linear clause is as if it is applied to the
+// innermost leaf construct.
+// (15.2) Additionally, if the list item is not the iteration variable of a simd
+// or worksharing-loop SIMD construct, the effect on the outer leaf constructs
+// is as if the list item was specified in firstprivate and lastprivate clauses
+// on the combined or composite construct, with the rules specified above
+// applied.
+// (19) If a list item of the linear clause is the iteration variable of a simd
+// or worksharing-loop SIMD construct and it is not declared in the construct,
+// the effect on the outer leaf constructs is as if the list item was specified
+// in a lastprivate clause on the combined or composite construct with the rules
+// specified above applied.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::LinearT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  // [5.2:341:15.1]
+  if (!applyToInnermost(node))
+    return false;
+
+  // [5.2:341:15.2], [5.2:341:19]
+  auto dirSimd = findDirective(llvm::omp::Directive::OMPD_simd);
+  std::optional<ObjectTy> iterVar = helper.getLoopIterVar();
+  const auto &objects = std::get<tomp::ObjectListT<IdTy, ExprTy>>(clause.t);
+
+  // Lists of objects that will be used to construct "firstprivate" and
+  // "lastprivate" clauses.
+  tomp::ObjectListT<IdTy, ExprTy> first, last;
+
+  for (const ObjectTy &object : objects) {
+    last.push_back(object);
+    if (!dirSimd || !iterVar || object.id() != iterVar->id())
+      first.push_back(object);
+  }
+
+  if (!first.empty()) {
+    auto *firstp = makeClause(
+        llvm::omp::Clause::OMPC_firstprivate,
+        tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>{/*List=*/first});
+    nodes.push_back(firstp); // Appending to the main clause list.
+  }
+  if (!last.empty()) {
+    auto *lastp =
+        makeClause(llvm::omp::Clause::OMPC_lastprivate,
+                   tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>{
+                       {/*LastprivateModifier=*/std::nullopt, /*List=*/last}});
+    nodes.push_back(lastp); // Appending to the main clause list.
+  }
+  return true;
+}
+
+// NOWAIT
+// [5.2:308:11-13]
+// Directives: dispatch, do, for, interop, scope, sections, single, target,
+// target enter data, target exit data, target update, taskwait, workshare
+//
+// [5.2:341:23]
+// (23) The effect of the nowait clause is as if it is applied to the outermost
+// leaf construct that permits it.
+template <typename C, typename H>
+bool ConstructDecompositionT<C, H>::applyClause(
+    const tomp::clause::NowaitT<TypeTy, IdTy, ExprTy> &clause,
+    const ClauseTy *node) {
+  return applyToOutermost(node);
+}
+
+template <typename C, typename H> bool ConstructDecompositionT<C, H>::split() {
+  bool success = true;
+
+  for (llvm::omp::Directive leaf :
+       llvm::omp::getLeafConstructsOrSelf(construct))
+    leafs.push_back(LeafReprInternal{leaf, /*clauses=*/{}});
+
+  for (const ClauseTy *node : nodes)
+    addClauseSymsToMap(*node, node);
+
+  // First we need to apply LINEAR, because it can generate additional
+  // "firstprivate" and "lastprivate" clauses that apply to the combined/
+  // composite construct.
+  // Collect them separately, because they may modify the clause list.
+  llvm::SmallVector<const ClauseTy *> linears;
+  for (const ClauseTy *node : nodes) {
+    if (node->id == llvm::omp::Clause::OMPC_linear)
+      linears.push_back(node);
+  }
+  for (const auto *node : linears) {
+    success = success &&
+              applyClause(std::get<tomp::clause::LinearT<TypeTy, IdTy, ExprTy>>(
+                              node->u),
+                          node);
+  }
+
+  // "allocate" clauses need to be applied last since they need to see
+  // which directives have data-privatizing clauses.
+  auto skip = [](const ClauseTy *node) {
+    switch (node->id) {
+    case llvm::omp::Clause::OMPC_allocate:
+    case llvm::omp::Clause::OMPC_linear:
+      return true;
+    default:
+      return false;
+    }
+  };
+
+  // Apply (almost) all clauses.
+  for (const ClauseTy *node : nodes) {
+    if (skip(node))
+      continue;
+    success =
+        success &&
+        std::visit([&](auto &&s) { return applyClause(s, node); }, node->u);
+  }
+
+  // Apply "allocate".
+  for (const ClauseTy *node : nodes) {
+    if (node->id != llvm::omp::Clause::OMPC_allocate)
+      continue;
+    success =
+        success &&
+        std::visit([&](auto &&s) { return applyClause(s, node); }, node->u);
+  }
+
+  return success;
+}
+
+} // namespace tomp
+
+#endif // LLVM_FRONTEND_OPENMP_CONSTRUCTDECOMPOSITIONT_H
diff --git a/llvm/unittests/Frontend/CMakeLists.txt b/llvm/unittests/Frontend/CMakeLists.txt
index 3f290b63ba64..85e113816e3b 100644
--- a/llvm/unittests/Frontend/CMakeLists.txt
+++ b/llvm/unittests/Frontend/CMakeLists.txt
@@ -15,6 +15,7 @@ add_llvm_unittest(LLVMFrontendTests
   OpenMPIRBuilderTest.cpp
   OpenMPParsingTest.cpp
   OpenMPCompositionTest.cpp
+  OpenMPDecompositionTest.cpp
 
   DEPENDS
   acc_gen
diff --git a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
new file mode 100644
index 000000000000..df48e9cc0ff4
--- /dev/null
+++ b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
@@ -0,0 +1,999 @@
+//===- llvm/unittests/Frontend/OpenMPDecompositionTest.cpp ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Frontend/OpenMP/ClauseT.h"
+#include "llvm/Frontend/OpenMP/ConstructDecompositionT.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "gtest/gtest.h"
+
+#include <iterator>
+#include <optional>
+#include <sstream>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+// The actual tests start at comment "--- Test" below.
+
+// Create simple instantiations of all clauses to allow manual construction
+// of clauses, and implement emitting of a directive with clauses to a string.
+//
+// The tests then follow the pattern
+// 1. Create a list of clauses.
+// 2. Pass them, together with a construct, to the decomposition class.
+// 3. Extract individual resulting leaf constructs with clauses applied
+//    to them.
+// 4. Convert them to strings and compare with expected outputs.
+
+namespace omp {
+struct TypeTy {}; // placeholder
+struct ExprTy {}; // placeholder
+using IdTy = std::string;
+} // namespace omp
+
+namespace tomp::type {
+template <> struct ObjectT<omp::IdTy, omp::ExprTy> {
+  const omp::IdTy &id() const { return name; }
+  const std::optional<omp::ExprTy> ref() const { return omp::ExprTy{}; }
+
+  omp::IdTy name;
+};
+} // namespace tomp::type
+
+namespace omp {
+template <typename ElemTy> using List = tomp::type::ListT<ElemTy>;
+
+using Object = tomp::ObjectT<IdTy, ExprTy>;
+
+namespace clause {
+using DefinedOperator = tomp::type::DefinedOperatorT<IdTy, ExprTy>;
+using ProcedureDesignator = tomp::type::ProcedureDesignatorT<IdTy, ExprTy>;
+using ReductionOperator = tomp::type::ReductionIdentifierT<IdTy, ExprTy>;
+
+using AcqRel = tomp::clause::AcqRelT<TypeTy, IdTy, ExprTy>;
+using Acquire = tomp::clause::AcquireT<TypeTy, IdTy, ExprTy>;
+using AdjustArgs = tomp::clause::AdjustArgsT<TypeTy, IdTy, ExprTy>;
+using Affinity = tomp::clause::AffinityT<TypeTy, IdTy, ExprTy>;
+using Aligned = tomp::clause::AlignedT<TypeTy, IdTy, ExprTy>;
+using Align = tomp::clause::AlignT<TypeTy, IdTy, ExprTy>;
+using Allocate = tomp::clause::AllocateT<TypeTy, IdTy, ExprTy>;
+using Allocator = tomp::clause::AllocatorT<TypeTy, IdTy, ExprTy>;
+using AppendArgs = tomp::clause::AppendArgsT<TypeTy, IdTy, ExprTy>;
+using AtomicDefaultMemOrder =
+    tomp::clause::AtomicDefaultMemOrderT<TypeTy, IdTy, ExprTy>;
+using At = tomp::clause::AtT<TypeTy, IdTy, ExprTy>;
+using Bind = tomp::clause::BindT<TypeTy, IdTy, ExprTy>;
+using Capture = tomp::clause::CaptureT<TypeTy, IdTy, ExprTy>;
+using Collapse = tomp::clause::CollapseT<TypeTy, IdTy, ExprTy>;
+using Compare = tomp::clause::CompareT<TypeTy, IdTy, ExprTy>;
+using Copyin = tomp::clause::CopyinT<TypeTy, IdTy, ExprTy>;
+using Copyprivate = tomp::clause::CopyprivateT<TypeTy, IdTy, ExprTy>;
+using Defaultmap = tomp::clause::DefaultmapT<TypeTy, IdTy, ExprTy>;
+using Default = tomp::clause::DefaultT<TypeTy, IdTy, ExprTy>;
+using Depend = tomp::clause::DependT<TypeTy, IdTy, ExprTy>;
+using Destroy = tomp::clause::DestroyT<TypeTy, IdTy, ExprTy>;
+using Detach = tomp::clause::DetachT<TypeTy, IdTy, ExprTy>;
+using Device = tomp::clause::DeviceT<TypeTy, IdTy, ExprTy>;
+using DeviceType = tomp::clause::DeviceTypeT<TypeTy, IdTy, ExprTy>;
+using DistSchedule = tomp::clause::DistScheduleT<TypeTy, IdTy, ExprTy>;
+using Doacross = tomp::clause::DoacrossT<TypeTy, IdTy, ExprTy>;
+using DynamicAllocators =
+    tomp::clause::DynamicAllocatorsT<TypeTy, IdTy, ExprTy>;
+using Enter = tomp::clause::EnterT<TypeTy, IdTy, ExprTy>;
+using Exclusive = tomp::clause::ExclusiveT<TypeTy, IdTy, ExprTy>;
+using Fail = tomp::clause::FailT<TypeTy, IdTy, ExprTy>;
+using Filter = tomp::clause::FilterT<TypeTy, IdTy, ExprTy>;
+using Final = tomp::clause::FinalT<TypeTy, IdTy, ExprTy>;
+using Firstprivate = tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>;
+using From = tomp::clause::FromT<TypeTy, IdTy, ExprTy>;
+using Full = tomp::clause::FullT<TypeTy, IdTy, ExprTy>;
+using Grainsize = tomp::clause::GrainsizeT<TypeTy, IdTy, ExprTy>;
+using HasDeviceAddr = tomp::clause::HasDeviceAddrT<TypeTy, IdTy, ExprTy>;
+using Hint = tomp::clause::HintT<TypeTy, IdTy, ExprTy>;
+using If = tomp::clause::IfT<TypeTy, IdTy, ExprTy>;
+using Inbranch = tomp::clause::InbranchT<TypeTy, IdTy, ExprTy>;
+using Inclusive = tomp::clause::InclusiveT<TypeTy, IdTy, ExprTy>;
+using Indirect = tomp::clause::IndirectT<TypeTy, IdTy, ExprTy>;
+using Init = tomp::clause::InitT<TypeTy, IdTy, ExprTy>;
+using InReduction = tomp::clause::InReductionT<TypeTy, IdTy, ExprTy>;
+using IsDevicePtr = tomp::clause::IsDevicePtrT<TypeTy, IdTy, ExprTy>;
+using Lastprivate = tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>;
+using Linear = tomp::clause::LinearT<TypeTy, IdTy, ExprTy>;
+using Link = tomp::clause::LinkT<TypeTy, IdTy, ExprTy>;
+using Map = tomp::clause::MapT<TypeTy, IdTy, ExprTy>;
+using Match = tomp::clause::MatchT<TypeTy, IdTy, ExprTy>;
+using Mergeable = tomp::clause::MergeableT<TypeTy, IdTy, ExprTy>;
+using Message = tomp::clause::MessageT<TypeTy, IdTy, ExprTy>;
+using Nocontext = tomp::clause::NocontextT<TypeTy, IdTy, ExprTy>;
+using Nogroup = tomp::clause::NogroupT<TypeTy, IdTy, ExprTy>;
+using Nontemporal = tomp::clause::NontemporalT<TypeTy, IdTy, ExprTy>;
+using Notinbranch = tomp::clause::NotinbranchT<TypeTy, IdTy, ExprTy>;
+using Novariants = tomp::clause::NovariantsT<TypeTy, IdTy, ExprTy>;
+using Nowait = tomp::clause::NowaitT<TypeTy, IdTy, ExprTy>;
+using NumTasks = tomp::clause::NumTasksT<TypeTy, IdTy, ExprTy>;
+using NumTeams = tomp::clause::NumTeamsT<TypeTy, IdTy, ExprTy>;
+using NumThreads = tomp::clause::NumThreadsT<TypeTy, IdTy, ExprTy>;
+using OmpxAttribute = tomp::clause::OmpxAttributeT<TypeTy, IdTy, ExprTy>;
+using OmpxBare = tomp::clause::OmpxBareT<TypeTy, IdTy, ExprTy>;
+using OmpxDynCgroupMem = tomp::clause::OmpxDynCgroupMemT<TypeTy, IdTy, ExprTy>;
+using Ordered = tomp::clause::OrderedT<TypeTy, IdTy, ExprTy>;
+using Order = tomp::clause::OrderT<TypeTy, IdTy, ExprTy>;
+using Partial = tomp::clause::PartialT<TypeTy, IdTy, ExprTy>;
+using Priority = tomp::clause::PriorityT<TypeTy, IdTy, ExprTy>;
+using Private = tomp::clause::PrivateT<TypeTy, IdTy, ExprTy>;
+using ProcBind = tomp::clause::ProcBindT<TypeTy, IdTy, ExprTy>;
+using Read = tomp::clause::ReadT<TypeTy, IdTy, ExprTy>;
+using Reduction = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
+using Relaxed = tomp::clause::RelaxedT<TypeTy, IdTy, ExprTy>;
+using Release = tomp::clause::ReleaseT<TypeTy, IdTy, ExprTy>;
+using ReverseOffload = tomp::clause::ReverseOffloadT<TypeTy, IdTy, ExprTy>;
+using Safelen = tomp::clause::SafelenT<TypeTy, IdTy, ExprTy>;
+using Schedule = tomp::clause::ScheduleT<TypeTy, IdTy, ExprTy>;
+using SeqCst = tomp::clause::SeqCstT<TypeTy, IdTy, ExprTy>;
+using Severity = tomp::clause::SeverityT<TypeTy, IdTy, ExprTy>;
+using Shared = tomp::clause::SharedT<TypeTy, IdTy, ExprTy>;
+using Simdlen = tomp::clause::SimdlenT<TypeTy, IdTy, ExprTy>;
+using Simd = tomp::clause::SimdT<TypeTy, IdTy, ExprTy>;
+using Sizes = tomp::clause::SizesT<TypeTy, IdTy, ExprTy>;
+using TaskReduction = tomp::clause::TaskReductionT<TypeTy, IdTy, ExprTy>;
+using ThreadLimit = tomp::clause::ThreadLimitT<TypeTy, IdTy, ExprTy>;
+using Threads = tomp::clause::ThreadsT<TypeTy, IdTy, ExprTy>;
+using To = tomp::clause::ToT<TypeTy, IdTy, ExprTy>;
+using UnifiedAddress = tomp::clause::UnifiedAddressT<TypeTy, IdTy, ExprTy>;
+using UnifiedSharedMemory =
+    tomp::clause::UnifiedSharedMemoryT<TypeTy, IdTy, ExprTy>;
+using Uniform = tomp::clause::UniformT<TypeTy, IdTy, ExprTy>;
+using Unknown = tomp::clause::UnknownT<TypeTy, IdTy, ExprTy>;
+using Untied = tomp::clause::UntiedT<TypeTy, IdTy, ExprTy>;
+using Update = tomp::clause::UpdateT<TypeTy, IdTy, ExprTy>;
+using UseDeviceAddr = tomp::clause::UseDeviceAddrT<TypeTy, IdTy, ExprTy>;
+using UseDevicePtr = tomp::clause::UseDevicePtrT<TypeTy, IdTy, ExprTy>;
+using UsesAllocators = tomp::clause::UsesAllocatorsT<TypeTy, IdTy, ExprTy>;
+using Use = tomp::clause::UseT<TypeTy, IdTy, ExprTy>;
+using Weak = tomp::clause::WeakT<TypeTy, IdTy, ExprTy>;
+using When = tomp::clause::WhenT<TypeTy, IdTy, ExprTy>;
+using Write = tomp::clause::WriteT<TypeTy, IdTy, ExprTy>;
+} // namespace clause
+
+struct Helper {
+  std::optional<Object> getBaseObject(const Object &object) {
+    return std::nullopt;
+  }
+  std::optional<Object> getLoopIterVar() { return std::nullopt; }
+};
+
+using Clause = tomp::ClauseT<TypeTy, IdTy, ExprTy>;
+using ConstructDecomposition = tomp::ConstructDecompositionT<Clause, Helper>;
+using DirectiveWithClauses = tomp::DirectiveWithClauses<Clause>;
+} // namespace omp
+
+struct StringifyClause {
+  static std::string join(const omp::List<std::string> &Strings) {
+    std::stringstream Stream;
+    for (const auto &[Index, String] : llvm::enumerate(Strings)) {
+      if (Index != 0)
+        Stream << ", ";
+      Stream << String;
+    }
+    return Stream.str();
+  }
+
+  static std::string to_str(llvm::omp::Directive D) {
+    return getOpenMPDirectiveName(D).str();
+  }
+  static std::string to_str(llvm::omp::Clause C) {
+    return getOpenMPClauseName(C).str();
+  }
+  static std::string to_str(const omp::TypeTy &Type) { return "type"; }
+  static std::string to_str(const omp::ExprTy &Expr) { return "expr"; }
+  static std::string to_str(const omp::Object &Obj) { return Obj.id(); }
+
+  template <typename U>
+  static std::enable_if_t<std::is_enum_v<llvm::remove_cvref_t<U>>, std::string>
+  to_str(U &&Item) {
+    return std::to_string(llvm::to_underlying(Item));
+  }
+
+  template <typename U> static std::string to_str(const omp::List<U> &Items) {
+    omp::List<std::string> Names;
+    llvm::transform(Items, std::back_inserter(Names),
+                    [](auto &&S) { return to_str(S); });
+    return "(" + join(Names) + ")";
+  }
+
+  template <typename U>
+  static std::string to_str(const std::optional<U> &Item) {
+    if (Item)
+      return to_str(*Item);
+    return "";
+  }
+
+  template <typename... Us, size_t... Is>
+  static std::string to_str(const std::tuple<Us...> &Tuple,
+                            std::index_sequence<Is...>) {
+    omp::List<std::string> Strings;
+    (Strings.push_back(to_str(std::get<Is>(Tuple))), ...);
+    return "(" + join(Strings) + ")";
+  }
+
+  template <typename U>
+  static std::enable_if_t<llvm::remove_cvref_t<U>::EmptyTrait::value,
+                          std::string>
+  to_str(U &&Item) {
+    return "";
+  }
+
+  template <typename U>
+  static std::enable_if_t<llvm::remove_cvref_t<U>::IncompleteTrait::value,
+                          std::string>
+  to_str(U &&Item) {
+    return "";
+  }
+
+  template <typename U>
+  static std::enable_if_t<llvm::remove_cvref_t<U>::WrapperTrait::value,
+                          std::string>
+  to_str(U &&Item) {
+    // For a wrapper, stringify the wrappee, and only add parentheses if
+    // there aren't any already.
+    std::string Str = to_str(Item.v);
+    if (!Str.empty()) {
+      if (Str.front() == '(' && Str.back() == ')')
+        return Str;
+    }
+    return "(" + to_str(Item.v) + ")";
+  }
+
+  template <typename U>
+  static std::enable_if_t<llvm::remove_cvref_t<U>::TupleTrait::value,
+                          std::string>
+  to_str(U &&Item) {
+    constexpr size_t TupleSize =
+        std::tuple_size_v<llvm::remove_cvref_t<decltype(Item.t)>>;
+    return to_str(Item.t, std::make_index_sequence<TupleSize>{});
+  }
+
+  template <typename U>
+  static std::enable_if_t<llvm::remove_cvref_t<U>::UnionTrait::value,
+                          std::string>
+  to_str(U &&Item) {
+    return std::visit([](auto &&S) { return to_str(S); }, Item.u);
+  }
+
+  StringifyClause(const omp::Clause &C)
+      // Rely on content stringification to emit enclosing parentheses.
+      : Str(to_str(C.id) + to_str(C)) {}
+
+  std::string Str;
+};
+
+std::string stringify(const omp::DirectiveWithClauses &DWC) {
+  std::stringstream Stream;
+
+  Stream << getOpenMPDirectiveName(DWC.id).str();
+  for (const omp::Clause &C : DWC.clauses)
+    Stream << ' ' << StringifyClause(C).Str;
+
+  return Stream.str();
+}
+
+// --- Tests ----------------------------------------------------------
+
+namespace {
+using namespace llvm::omp;
+
+class OpenMPDecompositionTest : public testing::Test {
+protected:
+  void SetUp() override {}
+  void TearDown() override {}
+
+  omp::Helper Helper;
+  uint32_t AnyVersion = 999;
+};
+
+// PRIVATE
+// [5.2:111:5-7]
+// Directives: distribute, do, for, loop, parallel, scope, sections, simd,
+// single, target, task, taskloop, teams
+//
+// [5.2:340:1-2]
+// (1) The effect of the 1 private clause is as if it is applied only to the
+// innermost leaf construct that permits it.
+TEST_F(OpenMPDecompositionTest, Private1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_private, omp::clause::Private{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel");            // (1)
+  ASSERT_EQ(Dir1, "sections private(x)"); // (1)
+}
+
+TEST_F(OpenMPDecompositionTest, Private2) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_private, omp::clause::Private{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_masked,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel private(x)"); // (1)
+  ASSERT_EQ(Dir1, "masked");              // (1)
+}
+
+// FIRSTPRIVATE
+// [5.2:112:5-7]
+// Directives: distribute, do, for, parallel, scope, sections, single, target,
+// task, taskloop, teams
+//
+// [5.2:340:3-20]
+// (3) The effect of the firstprivate clause is as if it is applied to one or
+// more leaf constructs as follows:
+//  (5) To the distribute construct if it is among the constituent constructs;
+//  (6) To the teams construct if it is among the constituent constructs and the
+//      distribute construct is not;
+//  (8) To a worksharing construct that accepts the clause if one is among the
+//      constituent constructs;
+//  (9) To the taskloop construct if it is among the constituent constructs;
+// (10) To the parallel construct if it is among the constituent constructs and
+//      neither a taskloop construct nor a worksharing construct that accepts
+//      the clause is among them;
+// (12) To the target construct if it is among the constituent constructs and
+//      the same list item neither appears in a lastprivate clause nor is the
+//      base variable or base pointer of a list item that appears in a map
+//      clause.
+//
+// (15) If the parallel construct is among the constituent constructs and the
+// effect is not as if the firstprivate clause is applied to it by the above
+// rules, then the effect is as if the shared clause with the same list item is
+// applied to the parallel construct.
+// (17) If the teams construct is among the constituent constructs and the
+// effect is not as if the firstprivate clause is applied to it by the above
+// rules, then the effect is as if the shared clause with the same list item is
+// applied to the teams construct.
+TEST_F(OpenMPDecompositionTest, Firstprivate1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel shared(x)");       // (10), (15)
+  ASSERT_EQ(Dir1, "sections firstprivate(x)"); // (8)
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate2) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_target_teams_distribute, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "target firstprivate(x)");     // (12)
+  ASSERT_EQ(Dir1, "teams shared(x)");            // (6), (17)
+  ASSERT_EQ(Dir2, "distribute firstprivate(x)"); // (5)
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate3) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_target_teams_distribute, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (12), (27)
+  ASSERT_EQ(Dir1, "teams shared(x)");          // (6), (17)
+  ASSERT_EQ(Dir2, "distribute firstprivate(x) lastprivate(, (x))"); // (5), (21)
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate4) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_teams,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "target firstprivate(x)"); // (12)
+  ASSERT_EQ(Dir1, "teams firstprivate(x)");  // (6)
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate5) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_parallel_masked_taskloop, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "parallel shared(x)"); // (10)
+  ASSERT_EQ(Dir1, "masked");
+  ASSERT_EQ(Dir2, "taskloop firstprivate(x)"); // (9)
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate6) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_masked,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel firstprivate(x)"); // (10)
+  ASSERT_EQ(Dir1, "masked");
+}
+
+TEST_F(OpenMPDecompositionTest, Firstprivate7) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  // Composite constructs are still decomposed.
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_teams_distribute,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "teams shared(x)");            // (17)
+  ASSERT_EQ(Dir1, "distribute firstprivate(x)"); // (5)
+}
+
+// LASTPRIVATE
+// [5.2:115:7-8]
+// Directives: distribute, do, for, loop, sections, simd, taskloop
+//
+// [5.2:340:21-30]
+// (21) The effect of the lastprivate clause is as if it is applied to all leaf
+// constructs that permit the clause.
+// (22) If the parallel construct is among the constituent constructs and the
+// list item is not also specified in the firstprivate clause, then the effect
+// of the lastprivate clause is as if the shared clause with the same list item
+// is applied to the parallel construct.
+// (24) If the teams construct is among the constituent constructs and the list
+// item is not also specified in the firstprivate clause, then the effect of the
+// lastprivate clause is as if the shared clause with the same list item is
+// applied to the teams construct.
+// (27) If the target construct is among the constituent constructs and the list
+// item is not the base variable or base pointer of a list item that appears in
+// a map clause, the effect of the lastprivate clause is as if the same list
+// item appears in a map clause with a map-type of tofrom.
+TEST_F(OpenMPDecompositionTest, Lastprivate1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel shared(x)");          // (21), (22)
+  ASSERT_EQ(Dir1, "sections lastprivate(, (x))"); // (21)
+}
+
+TEST_F(OpenMPDecompositionTest, Lastprivate2) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_teams_distribute,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "teams shared(x)");               // (21), (25)
+  ASSERT_EQ(Dir1, "distribute lastprivate(, (x))"); // (21)
+}
+
+TEST_F(OpenMPDecompositionTest, Lastprivate3) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_parallel_do,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "target map(2, , , , (x))"); // (21), (27)
+  ASSERT_EQ(Dir1, "parallel shared(x)");       // (22)
+  ASSERT_EQ(Dir2, "do lastprivate(, (x))");    // (21)
+}
+
+// SHARED
+// [5.2:110:5-6]
+// Directives: parallel, task, taskloop, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+TEST_F(OpenMPDecompositionTest, Shared1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_shared, omp::clause::Shared{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_parallel_masked_taskloop, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "parallel shared(x)"); // (31)
+  ASSERT_EQ(Dir1, "masked");             // (31)
+  ASSERT_EQ(Dir2, "taskloop shared(x)"); // (31)
+}
+
+// DEFAULT
+// [5.2:109:5-6]
+// Directives: parallel, task, taskloop, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+TEST_F(OpenMPDecompositionTest, Default1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_default,
+       omp::clause::Default{
+           omp::clause::Default::DataSharingAttribute::Firstprivate}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_parallel_masked_taskloop, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "parallel default(0)"); // (31)
+  ASSERT_EQ(Dir1, "masked");              // (31)
+  ASSERT_EQ(Dir2, "taskloop default(0)"); // (31)
+}
+
+// THREAD_LIMIT
+// [5.2:277:14-15]
+// Directives: target, teams
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+TEST_F(OpenMPDecompositionTest, ThreadLimit1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_thread_limit, omp::clause::ThreadLimit{omp::ExprTy{}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_target_teams_distribute, Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "target thread_limit(expr)"); // (31)
+  ASSERT_EQ(Dir1, "teams thread_limit(expr)");  // (31)
+  ASSERT_EQ(Dir2, "distribute");                // (31)
+}
+
+// ORDER
+// [5.2:234:3-4]
+// Directives: distribute, do, for, loop, simd
+//
+// [5.2:340:31-32]
+// (31) The effect of the shared, default, thread_limit, or order clause is as
+// if it is applied to all leaf constructs that permit the clause.
+TEST_F(OpenMPDecompositionTest, Order1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_order,
+       omp::clause::Order{{omp::clause::Order::OrderModifier::Unconstrained,
+                           omp::clause::Order::Ordering::Concurrent}}},
+  };
+
+  omp::ConstructDecomposition Dec(
+      AnyVersion, Helper, OMPD_target_teams_distribute_parallel_for_simd,
+      Clauses);
+  ASSERT_EQ(Dec.output.size(), 6u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  std::string Dir4 = stringify(Dec.output[4]);
+  std::string Dir5 = stringify(Dec.output[5]);
+  ASSERT_EQ(Dir0, "target"); // (31)
+  ASSERT_EQ(Dir1, "teams");  // (31)
+  // XXX OMP.td doesn't list "order" as allowed for "distribute"
+  ASSERT_EQ(Dir2, "distribute");       // (31)
+  ASSERT_EQ(Dir3, "parallel");         // (31)
+  ASSERT_EQ(Dir4, "for order(1, 0)");  // (31)
+  ASSERT_EQ(Dir5, "simd order(1, 0)"); // (31)
+}
+
+// ALLOCATE
+// [5.2:178:7-9]
+// Directives: allocators, distribute, do, for, parallel, scope, sections,
+// single, target, task, taskgroup, taskloop, teams
+//
+// [5.2:340:33-35]
+// (33) The effect of the allocate clause is as if it is applied to all leaf
+// constructs that permit the clause and to which a data-sharing attribute
+// clause that may create a private copy of the same list item is applied.
+TEST_F(OpenMPDecompositionTest, Allocate1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_allocate,
+       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_private, omp::clause::Private{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel");                                // (33)
+  ASSERT_EQ(Dir1, "sections private(x) allocate(, , , (x))"); // (33)
+}
+
+// REDUCTION
+// [5.2:134:17-18]
+// Directives: do, for, loop, parallel, scope, sections, simd, taskloop, teams
+//
+// [5.2:340-341:36-13]
+// (36) The effect of the reduction clause is as if it is applied to all leaf
+// constructs that permit the clause, except for the following constructs:
+//  (1) The parallel construct, when combined with the sections,
+//      worksharing-loop, loop, or taskloop construct; and
+//  (3) The teams construct, when combined with the loop construct.
+// (4) For the parallel and teams constructs above, the effect of the reduction
+// clause instead is as if each list item or, for any list item that is an array
+// item, its corresponding base array or base pointer appears in a shared clause
+// for the construct.
+// (6) If the task reduction-modifier is specified, the effect is as if it only
+// modifies the behavior of the reduction clause on the innermost leaf construct
+// that accepts the modifier (see Section 5.5.8).
+// (8) If the inscan reduction-modifier is specified, the effect is as if it
+// modifies the behavior of the reduction clause on all constructs of the
+// combined construct to which the clause is applied and that accept the
+// modifier.
+// (10) If a list item in a reduction clause on a combined target construct does
+// not have the same base variable or base pointer as a list item in a map
+// clause on the construct, then the effect is as if the list item in the
+// reduction clause appears as a list item in a map clause with a map-type of
+// tofrom.
+namespace red {
+// Make is easier to construct reduction operators from built-in intrinsics.
+omp::clause::ReductionOperator
+makeOp(omp::clause::DefinedOperator::IntrinsicOperator Op) {
+  return omp::clause::ReductionOperator{omp::clause::DefinedOperator{Op}};
+}
+} // namespace red
+
+TEST_F(OpenMPDecompositionTest, Reduction1) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel shared(x)");             // (36), (1), (4)
+  ASSERT_EQ(Dir1, "sections reduction(, (3), (x))"); // (36)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction2) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_masked,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel reduction(, (3), (x))"); // (36), (1), (4)
+  ASSERT_EQ(Dir1, "masked");                         // (36)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction3) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_teams_loop, Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "teams shared(x)");            // (36), (3), (4)
+  ASSERT_EQ(Dir1, "loop reduction(, (3), (x))"); // (36)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction4) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_teams_distribute_parallel_for, Clauses);
+  ASSERT_EQ(Dec.output.size(), 4u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  ASSERT_EQ(Dir0, "teams reduction(, (3), (x))"); // (36), (3)
+  ASSERT_EQ(Dir1, "distribute");                  // (36)
+  ASSERT_EQ(Dir2, "parallel shared(x)");          // (36), (1), (4)
+  ASSERT_EQ(Dir3, "for reduction(, (3), (x))");   // (36)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction5) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+  auto TaskMod = omp::clause::Reduction::ReductionModifier::Task;
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{TaskMod, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_teams_distribute_parallel_for, Clauses);
+  ASSERT_EQ(Dec.output.size(), 4u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  ASSERT_EQ(Dir0, "teams reduction(, (3), (x))"); // (36), (3), (6)
+  ASSERT_EQ(Dir1, "distribute");                  // (36)
+  ASSERT_EQ(Dir2, "parallel shared(x)");          // (36), (1), (4)
+  ASSERT_EQ(Dir3, "for reduction(2, (3), (x))");  // (36), (6)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction6) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+  auto InscanMod = omp::clause::Reduction::ReductionModifier::Inscan;
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{InscanMod, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_teams_distribute_parallel_for, Clauses);
+  ASSERT_EQ(Dec.output.size(), 4u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  ASSERT_EQ(Dir0, "teams reduction(, (3), (x))"); // (36), (3), (8)
+  ASSERT_EQ(Dir1, "distribute");                  // (36)
+  ASSERT_EQ(Dir2, "parallel shared(x)");          // (36), (1), (4)
+  ASSERT_EQ(Dir3, "for reduction(1, (3), (x))");  // (36), (8)
+}
+
+TEST_F(OpenMPDecompositionTest, Reduction7) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_parallel_do,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  // XXX Currently OMP.td allows "reduction" on "target".
+  ASSERT_EQ(Dir0,
+            "target reduction(, (3), (x)) map(2, , , , (x))"); // (36), (10)
+  ASSERT_EQ(Dir1, "parallel shared(x)");                       // (36), (1), (4)
+  ASSERT_EQ(Dir2, "do reduction(, (3), (x))");                 // (36)
+}
+
+// IF
+// [5.2:72:7-9]
+// Directives: cancel, parallel, simd, target, target data, target enter data,
+// target exit data, target update, task, taskloop
+//
+// [5.2:72:15-18]
+// (15) For combined or composite constructs, the if clause only applies to the
+// semantics of the construct named in the directive-name-modifier.
+// (16) For a combined or composite construct, if no directive-name-modifier is
+// specified then the if clause applies to all constituent constructs to which
+// an if clause can apply.
+TEST_F(OpenMPDecompositionTest, If1) {
+  omp::List<omp::Clause> Clauses{
+      {OMPC_if,
+       omp::clause::If{{llvm::omp::Directive::OMPD_parallel, omp::ExprTy{}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_target_parallel_for_simd, Clauses);
+  ASSERT_EQ(Dec.output.size(), 4u);
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  ASSERT_EQ(Dir0, "target");              // (15)
+  ASSERT_EQ(Dir1, "parallel if(, expr)"); // (15)
+  ASSERT_EQ(Dir2, "for");                 // (15)
+  ASSERT_EQ(Dir3, "simd");                // (15)
+}
+
+TEST_F(OpenMPDecompositionTest, If2) {
+  omp::List<omp::Clause> Clauses{
+      {OMPC_if, omp::clause::If{{std::nullopt, omp::ExprTy{}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper,
+                                  OMPD_target_parallel_for_simd, Clauses);
+  ASSERT_EQ(Dec.output.size(), 4u);
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  std::string Dir3 = stringify(Dec.output[3]);
+  ASSERT_EQ(Dir0, "target if(, expr)");   // (16)
+  ASSERT_EQ(Dir1, "parallel if(, expr)"); // (16)
+  ASSERT_EQ(Dir2, "for");                 // (16)
+  ASSERT_EQ(Dir3, "simd if(, expr)");     // (16)
+}
+
+// LINEAR
+// [5.2:118:1-2]
+// Directives: declare simd, do, for, simd
+//
+// [5.2:341:15-22]
+// (15.1) The effect of the linear clause is as if it is applied to the
+// innermost leaf construct.
+// (15.2) Additionally, if the list item is not the iteration variable of a simd
+// or worksharing-loop SIMD construct, the effect on the outer leaf constructs
+// is as if the list item was specified in firstprivate and lastprivate clauses
+// on the combined or composite construct, with the rules specified above
+// applied.
+// (19) If a list item of the linear clause is the iteration variable of a simd
+// or worksharing-loop SIMD construct and it is not declared in the construct,
+// the effect on the outer leaf constructs is as if the list item was specified
+// in a lastprivate clause on the combined or composite construct with the rules
+// specified above applied.
+TEST_F(OpenMPDecompositionTest, Linear1) {
+  omp::Object x{"x"};
+
+  omp::List<omp::Clause> Clauses{
+      {OMPC_linear,
+       omp::clause::Linear{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_for_simd, Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "for firstprivate(x) lastprivate(, (x))"); // (15.1), (15.2)
+  ASSERT_EQ(Dir1, "simd linear(, , , (x)) lastprivate(, (x))"); // (15.1)
+}
+
+// NOWAIT
+// [5.2:308:11-13]
+// Directives: dispatch, do, for, interop, scope, sections, single, target,
+// target enter data, target exit data, target update, taskwait, workshare
+//
+// [5.2:341:23]
+// (23) The effect of the nowait clause is as if it is applied to the outermost
+// leaf construct that permits it.
+TEST_F(OpenMPDecompositionTest, Nowait1) {
+  omp::List<omp::Clause> Clauses{
+      {OMPC_nowait, omp::clause::Nowait{}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_parallel_for,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 3u);
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  std::string Dir2 = stringify(Dec.output[2]);
+  ASSERT_EQ(Dir0, "target nowait"); // (23)
+  ASSERT_EQ(Dir1, "parallel");      // (23)
+  ASSERT_EQ(Dir2, "for");           // (23)
+}
+} // namespace
-- 
GitLab


From e8eb52d167eb2bf972b3cfa67ff1028b86cd209d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Mon, 13 May 2024 08:34:36 -0700
Subject: [PATCH 0588/1206] [flang][cuda] Extends matching distance computation
 (#91810)

Extends the computation of the matching distance in the generic
resolution to support options described in the table:
https://docs.nvidia.com/hpc-sdk/archive/24.3/compilers/cuda-fortran-prog-guide/index.html#cfref-var-attr-unified-data

Options are added as language features in the `SemanticsContext` and a
flag is added in bbc for testing purpose.
---
 flang/include/flang/Common/Fortran-features.h |  4 +-
 flang/include/flang/Common/Fortran.h          |  4 +-
 flang/lib/Common/Fortran.cpp                  | 37 +++++++++----
 flang/lib/Semantics/check-call.cpp            |  2 +-
 flang/lib/Semantics/expression.cpp            | 34 +++++++++---
 flang/test/Semantics/cuf14.cuf                | 55 +++++++++++++++++++
 flang/test/Semantics/cuf15.cuf                | 55 +++++++++++++++++++
 flang/tools/bbc/bbc.cpp                       | 10 ++++
 8 files changed, 180 insertions(+), 21 deletions(-)
 create mode 100644 flang/test/Semantics/cuf14.cuf
 create mode 100644 flang/test/Semantics/cuf15.cuf

diff --git a/flang/include/flang/Common/Fortran-features.h b/flang/include/flang/Common/Fortran-features.h
index 07ed7f43c1e7..f57fcdc895ad 100644
--- a/flang/include/flang/Common/Fortran-features.h
+++ b/flang/include/flang/Common/Fortran-features.h
@@ -49,7 +49,7 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     IndistinguishableSpecifics, SubroutineAndFunctionSpecifics,
     EmptySequenceType, NonSequenceCrayPointee, BranchIntoConstruct,
     BadBranchTarget, ConvertedArgument, HollerithPolymorphic, ListDirectedSize,
-    NonBindCInteroperability)
+    NonBindCInteroperability, CudaManaged, CudaUnified)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
@@ -81,6 +81,8 @@ public:
     disable_.set(LanguageFeature::OpenACC);
     disable_.set(LanguageFeature::OpenMP);
     disable_.set(LanguageFeature::CUDA); // !@cuf
+    disable_.set(LanguageFeature::CudaManaged);
+    disable_.set(LanguageFeature::CudaUnified);
     disable_.set(LanguageFeature::ImplicitNoneTypeNever);
     disable_.set(LanguageFeature::ImplicitNoneTypeAlways);
     disable_.set(LanguageFeature::DefaultSave);
diff --git a/flang/include/flang/Common/Fortran.h b/flang/include/flang/Common/Fortran.h
index 3b965fe60c2f..0701e3e8b64c 100644
--- a/flang/include/flang/Common/Fortran.h
+++ b/flang/include/flang/Common/Fortran.h
@@ -19,6 +19,7 @@
 #include <string>
 
 namespace Fortran::common {
+class LanguageFeatureControl;
 
 // Fortran has five kinds of intrinsic data types, plus the derived types.
 ENUM_CLASS(TypeCategory, Integer, Real, Complex, Character, Logical, Derived)
@@ -115,7 +116,8 @@ static constexpr IgnoreTKRSet ignoreTKRAll{IgnoreTKR::Type, IgnoreTKR::Kind,
 std::string AsFortran(IgnoreTKRSet);
 
 bool AreCompatibleCUDADataAttrs(std::optional<CUDADataAttr>,
-    std::optional<CUDADataAttr>, IgnoreTKRSet, bool allowUnifiedMatchingRule);
+    std::optional<CUDADataAttr>, IgnoreTKRSet, bool allowUnifiedMatchingRule,
+    const LanguageFeatureControl *features = nullptr);
 
 static constexpr char blankCommonObjectName[] = "__BLNK__";
 
diff --git a/flang/lib/Common/Fortran.cpp b/flang/lib/Common/Fortran.cpp
index 170ce8c22509..c014b1263a67 100644
--- a/flang/lib/Common/Fortran.cpp
+++ b/flang/lib/Common/Fortran.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Common/Fortran.h"
+#include "flang/Common/Fortran-features.h"
 
 namespace Fortran::common {
 
@@ -102,7 +103,13 @@ std::string AsFortran(IgnoreTKRSet tkr) {
 /// dummy argument attribute while `y` represents the actual argument attribute.
 bool AreCompatibleCUDADataAttrs(std::optional<CUDADataAttr> x,
     std::optional<CUDADataAttr> y, IgnoreTKRSet ignoreTKR,
-    bool allowUnifiedMatchingRule) {
+    bool allowUnifiedMatchingRule, const LanguageFeatureControl *features) {
+  bool isCudaManaged{features
+          ? features->IsEnabled(common::LanguageFeature::CudaManaged)
+          : false};
+  bool isCudaUnified{features
+          ? features->IsEnabled(common::LanguageFeature::CudaUnified)
+          : false};
   if (!x && !y) {
     return true;
   } else if (x && y && *x == *y) {
@@ -120,19 +127,27 @@ bool AreCompatibleCUDADataAttrs(std::optional<CUDADataAttr> x,
     return true;
   } else if (allowUnifiedMatchingRule) {
     if (!x) { // Dummy argument has no attribute -> host
-      if (y && (*y == CUDADataAttr::Managed || *y == CUDADataAttr::Unified)) {
+      if ((y && (*y == CUDADataAttr::Managed || *y == CUDADataAttr::Unified)) ||
+          (!y && (isCudaUnified || isCudaManaged))) {
         return true;
       }
     } else {
-      if (*x == CUDADataAttr::Device && y &&
-          (*y == CUDADataAttr::Managed || *y == CUDADataAttr::Unified)) {
-        return true;
-      } else if (*x == CUDADataAttr::Managed && y &&
-          *y == CUDADataAttr::Unified) {
-        return true;
-      } else if (*x == CUDADataAttr::Unified && y &&
-          *y == CUDADataAttr::Managed) {
-        return true;
+      if (*x == CUDADataAttr::Device) {
+        if ((y &&
+                (*y == CUDADataAttr::Managed || *y == CUDADataAttr::Unified)) ||
+            (!y && (isCudaUnified || isCudaManaged))) {
+          return true;
+        }
+      } else if (*x == CUDADataAttr::Managed) {
+        if ((y && *y == CUDADataAttr::Unified) ||
+            (!y && (isCudaUnified || isCudaManaged))) {
+          return true;
+        }
+      } else if (*x == CUDADataAttr::Unified) {
+        if ((y && *y == CUDADataAttr::Managed) ||
+            (!y && (isCudaUnified || isCudaManaged))) {
+          return true;
+        }
       }
     }
     return false;
diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index 94afcbb68b34..8f51ef5ebeba 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -914,7 +914,7 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
     }
     if (!common::AreCompatibleCUDADataAttrs(dummyDataAttr, actualDataAttr,
             dummy.ignoreTKR,
-            /*allowUnifiedMatchingRule=*/true)) {
+            /*allowUnifiedMatchingRule=*/true, &context.languageFeatures())) {
       auto toStr{[](std::optional<common::CUDADataAttr> x) {
         return x ? "ATTRIBUTES("s +
                 parser::ToUpperCaseLetters(common::EnumToString(*x)) + ")"s
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index c503ea3f0246..06e38da6626a 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -2501,8 +2501,13 @@ static constexpr int cudaInfMatchingValue{std::numeric_limits<int>::max()};
 
 // Compute the matching distance as described in section 3.2.3 of the CUDA
 // Fortran references.
-static int GetMatchingDistance(const characteristics::DummyArgument &dummy,
+static int GetMatchingDistance(const common::LanguageFeatureControl &features,
+    const characteristics::DummyArgument &dummy,
     const std::optional<ActualArgument> &actual) {
+  bool isCudaManaged{features.IsEnabled(common::LanguageFeature::CudaManaged)};
+  bool isCudaUnified{features.IsEnabled(common::LanguageFeature::CudaUnified)};
+  CHECK(!(isCudaUnified && isCudaManaged) && "expect only one enabled.");
+
   std::optional<common::CUDADataAttr> actualDataAttr, dummyDataAttr;
   if (actual) {
     if (auto *expr{actual->UnwrapExpr()}) {
@@ -2529,6 +2534,9 @@ static int GetMatchingDistance(const characteristics::DummyArgument &dummy,
 
   if (!dummyDataAttr) {
     if (!actualDataAttr) {
+      if (isCudaUnified || isCudaManaged) {
+        return 3;
+      }
       return 0;
     } else if (*actualDataAttr == common::CUDADataAttr::Device) {
       return cudaInfMatchingValue;
@@ -2538,6 +2546,9 @@ static int GetMatchingDistance(const characteristics::DummyArgument &dummy,
     }
   } else if (*dummyDataAttr == common::CUDADataAttr::Device) {
     if (!actualDataAttr) {
+      if (isCudaUnified || isCudaManaged) {
+        return 2;
+      }
       return cudaInfMatchingValue;
     } else if (*actualDataAttr == common::CUDADataAttr::Device) {
       return 0;
@@ -2546,7 +2557,10 @@ static int GetMatchingDistance(const characteristics::DummyArgument &dummy,
       return 2;
     }
   } else if (*dummyDataAttr == common::CUDADataAttr::Managed) {
-    if (!actualDataAttr || *actualDataAttr == common::CUDADataAttr::Device) {
+    if (!actualDataAttr) {
+      return isCudaUnified ? 1 : isCudaManaged ? 0 : cudaInfMatchingValue;
+    }
+    if (*actualDataAttr == common::CUDADataAttr::Device) {
       return cudaInfMatchingValue;
     } else if (*actualDataAttr == common::CUDADataAttr::Managed) {
       return 0;
@@ -2554,7 +2568,10 @@ static int GetMatchingDistance(const characteristics::DummyArgument &dummy,
       return 1;
     }
   } else if (*dummyDataAttr == common::CUDADataAttr::Unified) {
-    if (!actualDataAttr || *actualDataAttr == common::CUDADataAttr::Device) {
+    if (!actualDataAttr) {
+      return isCudaUnified ? 0 : isCudaManaged ? 1 : cudaInfMatchingValue;
+    }
+    if (*actualDataAttr == common::CUDADataAttr::Device) {
       return cudaInfMatchingValue;
     } else if (*actualDataAttr == common::CUDADataAttr::Managed) {
       return 1;
@@ -2566,6 +2583,7 @@ static int GetMatchingDistance(const characteristics::DummyArgument &dummy,
 }
 
 static int ComputeCudaMatchingDistance(
+    const common::LanguageFeatureControl &features,
     const characteristics::Procedure &procedure,
     const ActualArguments &actuals) {
   const auto &dummies{procedure.dummyArguments};
@@ -2574,7 +2592,7 @@ static int ComputeCudaMatchingDistance(
   for (std::size_t i{0}; i < dummies.size(); ++i) {
     const characteristics::DummyArgument &dummy{dummies[i]};
     const std::optional<ActualArgument> &actual{actuals[i]};
-    int d{GetMatchingDistance(dummy, actual)};
+    int d{GetMatchingDistance(features, dummy, actual)};
     if (d == cudaInfMatchingValue)
       return d;
     distance += d;
@@ -2666,7 +2684,9 @@ std::pair<const Symbol *, bool> ExpressionAnalyzer::ResolveGeneric(
             CheckCompatibleArguments(*procedure, localActuals)) {
           if ((procedure->IsElemental() && elemental) ||
               (!procedure->IsElemental() && nonElemental)) {
-            int d{ComputeCudaMatchingDistance(*procedure, localActuals)};
+            int d{ComputeCudaMatchingDistance(
+                context_.languageFeatures(), *procedure, localActuals)};
+            llvm::errs() << "matching distance: " << d << "\n";
             if (d != crtMatchingDistance) {
               if (d > crtMatchingDistance) {
                 continue;
@@ -2688,8 +2708,8 @@ std::pair<const Symbol *, bool> ExpressionAnalyzer::ResolveGeneric(
           } else {
             elemental = &specific;
           }
-          crtMatchingDistance =
-              ComputeCudaMatchingDistance(*procedure, localActuals);
+          crtMatchingDistance = ComputeCudaMatchingDistance(
+              context_.languageFeatures(), *procedure, localActuals);
         }
       }
     }
diff --git a/flang/test/Semantics/cuf14.cuf b/flang/test/Semantics/cuf14.cuf
new file mode 100644
index 000000000000..29c9ecf90677
--- /dev/null
+++ b/flang/test/Semantics/cuf14.cuf
@@ -0,0 +1,55 @@
+! RUN: bbc -emit-hlfir -fcuda -gpu=unified %s -o - | FileCheck %s
+
+module matching
+  interface host_and_device
+    module procedure sub_host
+    module procedure sub_device
+  end interface
+
+  interface all
+    module procedure sub_host
+    module procedure sub_device
+    module procedure sub_managed
+    module procedure sub_unified
+  end interface
+
+  interface all_without_unified
+    module procedure sub_host
+    module procedure sub_device
+    module procedure sub_managed
+  end interface
+
+contains
+  subroutine sub_host(a)
+    integer :: a(:)
+  end
+
+  subroutine sub_device(a)
+    integer, device :: a(:)
+  end
+
+  subroutine sub_managed(a)
+    integer, managed :: a(:)
+  end
+
+  subroutine sub_unified(a)
+    integer, unified :: a(:)
+  end
+end module
+
+program m
+  use matching
+
+  integer, allocatable :: actual_host(:)
+
+  allocate(actual_host(10))
+
+  call host_and_device(actual_host)     ! Should resolve to sub_device
+  call all(actual_host)                 ! Should resolved to unified
+  call all_without_unified(actual_host) ! Should resolved to managed
+end
+
+! CHECK: fir.call @_QMmatchingPsub_device
+! CHECK: fir.call @_QMmatchingPsub_unified
+! CHECK: fir.call @_QMmatchingPsub_managed
+
diff --git a/flang/test/Semantics/cuf15.cuf b/flang/test/Semantics/cuf15.cuf
new file mode 100644
index 000000000000..030dd6ff8ffe
--- /dev/null
+++ b/flang/test/Semantics/cuf15.cuf
@@ -0,0 +1,55 @@
+! RUN: bbc -emit-hlfir -fcuda -gpu=managed %s -o - | FileCheck %s
+
+module matching
+  interface host_and_device
+    module procedure sub_host
+    module procedure sub_device
+  end interface
+
+  interface all
+    module procedure sub_host
+    module procedure sub_device
+    module procedure sub_managed
+    module procedure sub_unified
+  end interface
+
+  interface all_without_managed
+    module procedure sub_host
+    module procedure sub_device
+    module procedure sub_unified
+  end interface
+
+contains
+  subroutine sub_host(a)
+    integer :: a(:)
+  end
+
+  subroutine sub_device(a)
+    integer, device :: a(:)
+  end
+
+  subroutine sub_managed(a)
+    integer, managed :: a(:)
+  end
+
+  subroutine sub_unified(a)
+    integer, unified :: a(:)
+  end
+end module
+
+program m
+  use matching
+
+  integer, allocatable :: actual_host(:)
+
+  allocate(actual_host(10))
+
+  call host_and_device(actual_host)     ! Should resolve to sub_device
+  call all(actual_host)                 ! Should resolved to unified
+  call all_without_managed(actual_host) ! Should resolved to managed
+end
+
+! CHECK: fir.call @_QMmatchingPsub_device
+! CHECK: fir.call @_QMmatchingPsub_managed
+! CHECK: fir.call @_QMmatchingPsub_unified
+
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index ee2ff8562e9f..f7092d35eeb5 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -204,6 +204,10 @@ static llvm::cl::opt<bool> enableCUDA("fcuda",
                                       llvm::cl::desc("enable CUDA Fortran"),
                                       llvm::cl::init(false));
 
+static llvm::cl::opt<std::string>
+    enableGPUMode("gpu", llvm::cl::desc("Enable GPU Mode managed|unified"),
+                  llvm::cl::init(""));
+
 static llvm::cl::opt<bool> fixedForm("ffixed-form",
                                      llvm::cl::desc("enable fixed form"),
                                      llvm::cl::init(false));
@@ -495,6 +499,12 @@ int main(int argc, char **argv) {
     options.features.Enable(Fortran::common::LanguageFeature::CUDA);
   }
 
+  if (enableGPUMode == "managed") {
+    options.features.Enable(Fortran::common::LanguageFeature::CudaManaged);
+  } else if (enableGPUMode == "unified") {
+    options.features.Enable(Fortran::common::LanguageFeature::CudaUnified);
+  }
+
   if (fixedForm) {
     options.isFixedForm = fixedForm;
   }
-- 
GitLab


From 06f04b2e27f2586d3db2204ed4e54f8b78fea74e Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Mon, 13 May 2024 08:40:43 -0700
Subject: [PATCH 0589/1206] Revert "[OpenACC] device_type clause Sema for
 Compute constructs"

This reverts commit c4a9a374749deb5f2a932a7d4ef9321be1b2ae5d.

This and the followup patch keep hitting an assert I wrote on the build
bots in a way that isn't clear.  Reverting so I can fix it without a
rush.
---
 clang/include/clang/AST/OpenACCClause.h       |  59 -----
 .../clang/Basic/DiagnosticSemaKinds.td        |   4 -
 clang/include/clang/Basic/OpenACCClauses.def  |   2 -
 clang/include/clang/Parse/Parser.h            |   3 +-
 clang/include/clang/Sema/SemaOpenACC.h        |  23 +-
 clang/lib/AST/OpenACCClause.cpp               |  28 +--
 clang/lib/AST/StmtProfile.cpp                 |   3 -
 clang/lib/AST/TextNodeDumper.cpp              |  13 --
 clang/lib/Parse/ParseOpenACC.cpp              |  21 +-
 clang/lib/Sema/SemaOpenACC.cpp                |  55 -----
 clang/lib/Sema/TreeTransform.h                |  10 -
 clang/lib/Serialization/ASTReader.cpp         |  17 +-
 clang/lib/Serialization/ASTWriter.cpp         |  15 +-
 .../ast-print-openacc-compute-construct.cpp   |  23 --
 clang/test/ParserOpenACC/parse-clauses.c      |  28 ++-
 .../compute-construct-device_type-ast.cpp     | 105 ---------
 .../compute-construct-device_type-clause.c    | 221 ------------------
 .../compute-construct-device_type-clause.cpp  |  25 --
 clang/tools/libclang/CIndex.cpp               |   2 -
 19 files changed, 35 insertions(+), 622 deletions(-)
 delete mode 100644 clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp
 delete mode 100644 clang/test/SemaOpenACC/compute-construct-device_type-clause.c
 delete mode 100644 clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp

diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 607a2b9d6536..3d0b1ab9d31e 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -17,8 +17,6 @@
 #include "clang/AST/StmtIterator.h"
 #include "clang/Basic/OpenACCKinds.h"
 
-#include <utility>
-
 namespace clang {
 /// This is the base type for all OpenACC Clauses.
 class OpenACCClause {
@@ -77,63 +75,6 @@ public:
   }
 };
 
-using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
-/// A 'device_type' or 'dtype' clause, takes a list of either an 'asterisk' or
-/// an identifier. The 'asterisk' means 'the rest'.
-class OpenACCDeviceTypeClause final
-    : public OpenACCClauseWithParams,
-      public llvm::TrailingObjects<OpenACCDeviceTypeClause,
-                                   DeviceTypeArgument> {
-  // Data stored in trailing objects as IdentifierInfo* /SourceLocation pairs. A
-  // nullptr IdentifierInfo* represents an asterisk.
-  unsigned NumArchs;
-  OpenACCDeviceTypeClause(OpenACCClauseKind K, SourceLocation BeginLoc,
-                          SourceLocation LParenLoc,
-                          ArrayRef<DeviceTypeArgument> Archs,
-                          SourceLocation EndLoc)
-      : OpenACCClauseWithParams(K, BeginLoc, LParenLoc, EndLoc),
-        NumArchs(Archs.size()) {
-    assert(
-        (K == OpenACCClauseKind::DeviceType || K == OpenACCClauseKind::DType) &&
-        "Invalid clause kind for device-type");
-
-    assert(!llvm::any_of(Archs, [](const DeviceTypeArgument &Arg) {
-      return Arg.second.isInvalid();
-    }) && "Invalid SourceLocation for an argument");
-
-    assert(
-        (Archs.size() == 1 || !llvm::any_of(Archs,
-                                            [](const DeviceTypeArgument &Arg) {
-                                              return Arg.first == nullptr;
-                                            })) &&
-        "Only a single asterisk version is permitted, and must be the "
-        "only one");
-
-    std::uninitialized_copy(Archs.begin(), Archs.end(),
-                            getTrailingObjects<DeviceTypeArgument>());
-  }
-
-public:
-  static bool classof(const OpenACCClause *C) {
-    return C->getClauseKind() == OpenACCClauseKind::DType ||
-           C->getClauseKind() == OpenACCClauseKind::DeviceType;
-  }
-  bool hasAsterisk() const {
-    return getArchitectures().size() > 0 &&
-           getArchitectures()[0].first == nullptr;
-  }
-
-  ArrayRef<DeviceTypeArgument> getArchitectures() const {
-    return ArrayRef<DeviceTypeArgument>(
-        getTrailingObjects<DeviceTypeArgument>(), NumArchs);
-  }
-
-  static OpenACCDeviceTypeClause *
-  Create(const ASTContext &C, OpenACCClauseKind K, SourceLocation BeginLoc,
-         SourceLocation LParenLoc, ArrayRef<DeviceTypeArgument> Archs,
-         SourceLocation EndLoc);
-};
-
 /// A 'default' clause, has the optional 'none' or 'present' argument.
 class OpenACCDefaultClause : public OpenACCClauseWithParams {
   friend class ASTReaderStmt;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 6100fba51005..9e82130c9360 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12344,8 +12344,4 @@ def warn_acc_deprecated_alias_name
 def err_acc_var_not_pointer_type
     : Error<"expected pointer in '%0' clause, type is %1">;
 def note_acc_expected_pointer_var : Note<"expected variable of pointer type">;
-def err_acc_clause_after_device_type
-    : Error<"OpenACC clause '%0' may not follow a '%1' clause in a "
-            "compute construct">;
-
 } // end of sema component.
diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def
index 7ecc51799468..afb7b30b7465 100644
--- a/clang/include/clang/Basic/OpenACCClauses.def
+++ b/clang/include/clang/Basic/OpenACCClauses.def
@@ -37,8 +37,6 @@ CLAUSE_ALIAS(PCreate, Create)
 CLAUSE_ALIAS(PresentOrCreate, Create)
 VISIT_CLAUSE(Default)
 VISIT_CLAUSE(DevicePtr)
-VISIT_CLAUSE(DeviceType)
-CLAUSE_ALIAS(DType, DeviceType)
 VISIT_CLAUSE(FirstPrivate)
 VISIT_CLAUSE(If)
 VISIT_CLAUSE(NoCreate)
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 3910cba34a21..61589fb7766f 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3720,8 +3720,7 @@ private:
                                SourceLocation Loc,
                                llvm::SmallVectorImpl<Expr *> &IntExprs);
   /// Parses the 'device-type-list', which is a list of identifiers.
-  bool ParseOpenACCDeviceTypeList(
-      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs);
+  bool ParseOpenACCDeviceTypeList();
   /// Parses the 'async-argument', which is an integral value with two
   /// 'special' values that are likely negative (but come from Macros).
   OpenACCIntExprParseResult ParseOpenACCAsyncArgument(OpenACCDirectiveKind DK,
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index f838fa97d33a..e684ee6b2be1 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -26,9 +26,6 @@ class OpenACCClause;
 
 class SemaOpenACC : public SemaBase {
 public:
-  // Redeclaration of the version in OpenACCClause.h.
-  using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
-
   /// A type to represent all the data for an OpenACC Clause that has been
   /// parsed, but not yet created/semantically analyzed. This is effectively a
   /// discriminated union on the 'Clause Kind', with all of the individual
@@ -63,12 +60,8 @@ public:
       SmallVector<Expr *> QueueIdExprs;
     };
 
-    struct DeviceTypeDetails {
-      SmallVector<DeviceTypeArgument> Archs;
-    };
-
     std::variant<std::monostate, DefaultDetails, ConditionDetails,
-                 IntExprDetails, VarListDetails, WaitDetails, DeviceTypeDetails>
+                 IntExprDetails, VarListDetails, WaitDetails>
         Details = std::monostate{};
 
   public:
@@ -216,13 +209,6 @@ public:
       return std::get<VarListDetails>(Details).IsZero;
     }
 
-    ArrayRef<DeviceTypeArgument> getDeviceTypeArchitectures() const {
-      assert((ClauseKind == OpenACCClauseKind::DeviceType ||
-              ClauseKind == OpenACCClauseKind::DType) &&
-             "Only 'device_type'/'dtype' has a device-type-arg list");
-      return std::get<DeviceTypeDetails>(Details).Archs;
-    }
-
     void setLParenLoc(SourceLocation EndLoc) { LParenLoc = EndLoc; }
     void setEndLoc(SourceLocation EndLoc) { ClauseRange.setEnd(EndLoc); }
 
@@ -340,13 +326,6 @@ public:
              "Parsed clause kind does not have a wait-details");
       Details = WaitDetails{DevNum, QueuesLoc, std::move(IntExprs)};
     }
-
-    void setDeviceTypeDetails(llvm::SmallVector<DeviceTypeArgument> &&Archs) {
-      assert((ClauseKind == OpenACCClauseKind::DeviceType ||
-              ClauseKind == OpenACCClauseKind::DType) &&
-             "Only 'device_type'/'dtype' has a device-type-arg list");
-      Details = DeviceTypeDetails{std::move(Archs)};
-    }
   };
 
   SemaOpenACC(Sema &S);
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index f80ecc90d396..ee13437b97b4 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -18,8 +18,7 @@
 using namespace clang;
 
 bool OpenACCClauseWithParams::classof(const OpenACCClause *C) {
-  return OpenACCDeviceTypeClause::classof(C) ||
-         OpenACCClauseWithCondition::classof(C) ||
+  return OpenACCClauseWithCondition::classof(C) ||
          OpenACCClauseWithExprs::classof(C);
 }
 bool OpenACCClauseWithExprs::classof(const OpenACCClause *C) {
@@ -299,17 +298,6 @@ OpenACCCreateClause::Create(const ASTContext &C, OpenACCClauseKind Spelling,
                                        VarList, EndLoc);
 }
 
-OpenACCDeviceTypeClause *OpenACCDeviceTypeClause::Create(
-    const ASTContext &C, OpenACCClauseKind K, SourceLocation BeginLoc,
-    SourceLocation LParenLoc, ArrayRef<DeviceTypeArgument> Archs,
-    SourceLocation EndLoc) {
-  void *Mem =
-      C.Allocate(OpenACCDeviceTypeClause::totalSizeToAlloc<DeviceTypeArgument>(
-          Archs.size()));
-  return new (Mem)
-      OpenACCDeviceTypeClause(K, BeginLoc, LParenLoc, Archs, EndLoc);
-}
-
 //===----------------------------------------------------------------------===//
 //  OpenACC clauses printing methods
 //===----------------------------------------------------------------------===//
@@ -463,17 +451,3 @@ void OpenACCClausePrinter::VisitWaitClause(const OpenACCWaitClause &C) {
     OS << ")";
   }
 }
-
-void OpenACCClausePrinter::VisitDeviceTypeClause(
-    const OpenACCDeviceTypeClause &C) {
-  OS << C.getClauseKind();
-  OS << "(";
-  llvm::interleaveComma(C.getArchitectures(), OS,
-                        [&](const DeviceTypeArgument &Arch) {
-                          if (Arch.first == nullptr)
-                            OS << "*";
-                          else
-                            OS << Arch.first;
-                        });
-  OS << ")";
-}
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index caab4ab0ef16..8fb8940142eb 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2585,9 +2585,6 @@ void OpenACCClauseProfiler::VisitWaitClause(const OpenACCWaitClause &Clause) {
   for (auto *E : Clause.getQueueIdExprs())
     Profiler.VisitStmt(E);
 }
-/// Nothing to do here, there are no sub-statements.
-void OpenACCClauseProfiler::VisitDeviceTypeClause(
-    const OpenACCDeviceTypeClause &Clause) {}
 } // namespace
 
 void StmtProfiler::VisitOpenACCComputeConstruct(
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index efcd74717a4e..12aa5858b798 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -444,19 +444,6 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
       if (cast<OpenACCWaitClause>(C)->hasQueuesTag())
         OS << " has queues tag";
       break;
-    case OpenACCClauseKind::DeviceType:
-    case OpenACCClauseKind::DType:
-      OS << "(";
-      llvm::interleaveComma(
-          cast<OpenACCDeviceTypeClause>(C)->getArchitectures(), OS,
-          [&](const DeviceTypeArgument &Arch) {
-            if (Arch.first == nullptr)
-              OS << "*";
-            else
-              OS << Arch.first->getName();
-          });
-      OS << ")";
-      break;
     default:
       // Nothing to do here.
       break;
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 261c9cdc088b..0e10632c8317 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -711,15 +711,14 @@ bool Parser::ParseOpenACCIntExprList(OpenACCDirectiveKind DK,
 /// device_type( device-type-list )
 ///
 /// The device_type clause may be abbreviated to dtype.
-bool Parser::ParseOpenACCDeviceTypeList(
-    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs) {
+bool Parser::ParseOpenACCDeviceTypeList() {
 
   if (expectIdentifierOrKeyword(*this)) {
     SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
               Parser::StopBeforeMatch);
-    return true;
+    return false;
   }
-  Archs.emplace_back(getCurToken().getIdentifierInfo(), ConsumeToken());
+  ConsumeToken();
 
   while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) {
     ExpectAndConsume(tok::comma);
@@ -727,9 +726,9 @@ bool Parser::ParseOpenACCDeviceTypeList(
     if (expectIdentifierOrKeyword(*this)) {
       SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
                 Parser::StopBeforeMatch);
-      return true;
+      return false;
     }
-    Archs.emplace_back(getCurToken().getIdentifierInfo(), ConsumeToken());
+    ConsumeToken();
   }
   return false;
 }
@@ -1022,20 +1021,16 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
       break;
     }
     case OpenACCClauseKind::DType:
-    case OpenACCClauseKind::DeviceType: {
-      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> Archs;
+    case OpenACCClauseKind::DeviceType:
       if (getCurToken().is(tok::star)) {
         // FIXME: We want to mark that this is an 'everything else' type of
         // device_type in Sema.
-        ParsedClause.setDeviceTypeDetails({{nullptr, ConsumeToken()}});
-      } else if (!ParseOpenACCDeviceTypeList(Archs)) {
-        ParsedClause.setDeviceTypeDetails(std::move(Archs));
-      } else {
+        ConsumeToken();
+      } else if (ParseOpenACCDeviceTypeList()) {
         Parens.skipToEnd();
         return OpenACCCanContinue();
       }
       break;
-    }
     case OpenACCClauseKind::Tile:
       if (ParseOpenACCSizeExprList()) {
         Parens.skipToEnd();
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index f174b2fa63c6..656d30947a8d 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -255,33 +255,6 @@ bool checkAlreadyHasClauseOfKind(
   return false;
 }
 
-/// Implement check from OpenACC3.3: section 2.5.4:
-/// Only the async, wait, num_gangs, num_workers, and vector_length clauses may
-/// follow a device_type clause.
-bool checkValidAfterDeviceType(
-    SemaOpenACC &S, const OpenACCDeviceTypeClause &DeviceTypeClause,
-    const SemaOpenACC::OpenACCParsedClause &NewClause) {
-  // This is only a requirement on compute constructs so far, so this is fine
-  // otherwise.
-  if (!isOpenACCComputeDirectiveKind(NewClause.getDirectiveKind()))
-    return false;
-  switch (NewClause.getClauseKind()) {
-  case OpenACCClauseKind::Async:
-  case OpenACCClauseKind::Wait:
-  case OpenACCClauseKind::NumGangs:
-  case OpenACCClauseKind::NumWorkers:
-  case OpenACCClauseKind::VectorLength:
-  case OpenACCClauseKind::DType:
-  case OpenACCClauseKind::DeviceType:
-    return false;
-  default:
-    S.Diag(NewClause.getBeginLoc(), diag::err_acc_clause_after_device_type)
-        << NewClause.getClauseKind() << DeviceTypeClause.getClauseKind();
-    S.Diag(DeviceTypeClause.getBeginLoc(), diag::note_acc_previous_clause_here);
-    return true;
-  }
-}
-
 } // namespace
 
 SemaOpenACC::SemaOpenACC(Sema &S) : SemaBase(S) {}
@@ -300,17 +273,6 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
     return nullptr;
   }
 
-  if (const auto *DevTypeClause =
-          llvm::find_if(ExistingClauses,
-                        [&](const OpenACCClause *C) {
-                          return isa<OpenACCDeviceTypeClause>(C);
-                        });
-      DevTypeClause != ExistingClauses.end()) {
-    if (checkValidAfterDeviceType(
-            *this, *cast<OpenACCDeviceTypeClause>(*DevTypeClause), Clause))
-      return nullptr;
-  }
-
   switch (Clause.getClauseKind()) {
   case OpenACCClauseKind::Default: {
     // Restrictions only properly implemented on 'compute' constructs, and
@@ -689,23 +651,6 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
         Clause.getDevNumExpr(), Clause.getQueuesLoc(), Clause.getQueueIdExprs(),
         Clause.getEndLoc());
   }
-  case OpenACCClauseKind::DType:
-  case OpenACCClauseKind::DeviceType: {
-    // Restrictions only properly implemented on 'compute' constructs, and
-    // 'compute' constructs are the only construct that can do anything with
-    // this yet, so skip/treat as unimplemented in this case.
-    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
-      break;
-
-    // TODO OpenACC: Once we get enough of the CodeGen implemented that we have
-    // a source for the list of valid architectures, we need to warn on unknown
-    // identifiers here.
-
-    return OpenACCDeviceTypeClause::Create(
-        getASTContext(), Clause.getClauseKind(), Clause.getBeginLoc(),
-        Clause.getLParenLoc(), Clause.getDeviceTypeArchitectures(),
-        Clause.getEndLoc());
-  }
   default:
     break;
   }
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index ab26d1b1199a..126965088831 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -11480,16 +11480,6 @@ void OpenACCClauseTransform<Derived>::VisitWaitClause(
       ParsedClause.getQueuesLoc(), ParsedClause.getQueueIdExprs(),
       ParsedClause.getEndLoc());
 }
-
-template <typename Derived>
-void OpenACCClauseTransform<Derived>::VisitDeviceTypeClause(
-    const OpenACCDeviceTypeClause &C) {
-  // Nothing to transform here, just create a new version of 'C'.
-  NewClause = OpenACCDeviceTypeClause::Create(
-      Self.getSema().getASTContext(), C.getClauseKind(),
-      ParsedClause.getBeginLoc(), ParsedClause.getLParenLoc(),
-      C.getArchitectures(), ParsedClause.getEndLoc());
-}
 } // namespace
 template <typename Derived>
 OpenACCClause *TreeTransform<Derived>::TransformOpenACCClause(
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 8f437a7c5f50..7627996d2c32 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11905,21 +11905,6 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
                                      DevNumExpr, QueuesLoc, QueueIdExprs,
                                      EndLoc);
   }
-  case OpenACCClauseKind::DeviceType:
-  case OpenACCClauseKind::DType: {
-    SourceLocation LParenLoc = readSourceLocation();
-    llvm::SmallVector<DeviceTypeArgument> Archs;
-    unsigned NumArchs = readInt();
-
-    for (unsigned I = 0; I < NumArchs; ++I) {
-      IdentifierInfo *Ident = readBool() ? readIdentifier() : nullptr;
-      SourceLocation Loc = readSourceLocation();
-      Archs.emplace_back(Ident, Loc);
-    }
-
-    return OpenACCDeviceTypeClause::Create(getContext(), ClauseKind, BeginLoc,
-                                           LParenLoc, Archs, EndLoc);
-  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -11941,6 +11926,8 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
   case OpenACCClauseKind::Bind:
   case OpenACCClauseKind::DeviceNum:
   case OpenACCClauseKind::DefaultAsync:
+  case OpenACCClauseKind::DeviceType:
+  case OpenACCClauseKind::DType:
   case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Invalid:
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 7a9d392889bb..6154ead589d3 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -7933,19 +7933,6 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeOpenACCIntExprList(WC->getQueueIdExprs());
     return;
   }
-  case OpenACCClauseKind::DeviceType:
-  case OpenACCClauseKind::DType: {
-    const auto *DTC = cast<OpenACCDeviceTypeClause>(C);
-    writeSourceLocation(DTC->getLParenLoc());
-    writeUInt32(DTC->getArchitectures().size());
-    for (const DeviceTypeArgument &Arg : DTC->getArchitectures()) {
-      writeBool(Arg.first);
-      if (Arg.first)
-        AddIdentifierRef(Arg.first);
-      writeSourceLocation(Arg.second);
-    }
-    return;
-  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -7967,6 +7954,8 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   case OpenACCClauseKind::Bind:
   case OpenACCClauseKind::DeviceNum:
   case OpenACCClauseKind::DefaultAsync:
+  case OpenACCClauseKind::DeviceType:
+  case OpenACCClauseKind::DType:
   case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Invalid:
diff --git a/clang/test/AST/ast-print-openacc-compute-construct.cpp b/clang/test/AST/ast-print-openacc-compute-construct.cpp
index cdd9ab3377d0..0bfb90bcb587 100644
--- a/clang/test/AST/ast-print-openacc-compute-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-compute-construct.cpp
@@ -107,28 +107,5 @@ void foo() {
 // CHECK: #pragma acc parallel wait(devnum: i : queues: *iPtr, i)
 #pragma acc parallel wait(devnum:i:queues:*iPtr, i)
   while(true);
-
-  bool SomeB;
-  struct SomeStruct{} SomeStructImpl;
-
-//#pragma acc parallel dtype(SomeB)
-#pragma acc parallel dtype(SomeB)
-  while(true);
-
-//#pragma acc parallel device_type(SomeStruct)
-#pragma acc parallel device_type(SomeStruct)
-  while(true);
-
-//#pragma acc parallel device_type(int)
-#pragma acc parallel device_type(int)
-  while(true);
-
-//#pragma acc parallel dtype(bool)
-#pragma acc parallel dtype(bool)
-  while(true);
-
-//#pragma acc parallel device_type (SomeStructImpl)
-#pragma acc parallel device_type (SomeStructImpl)
-  while(true);
 }
 
diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c
index 694f28b86ec9..51858b441e93 100644
--- a/clang/test/ParserOpenACC/parse-clauses.c
+++ b/clang/test/ParserOpenACC/parse-clauses.c
@@ -1126,10 +1126,12 @@ void device_type() {
 #pragma acc parallel dtype(
   {}
 
-  // expected-error@+1{{expected identifier}}
+  // expected-error@+2{{expected identifier}}
+  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
 #pragma acc parallel device_type()
   {}
-  // expected-error@+1{{expected identifier}}
+  // expected-error@+2{{expected identifier}}
+  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
 #pragma acc parallel dtype()
   {}
 
@@ -1171,10 +1173,12 @@ void device_type() {
 #pragma acc parallel dtype(ident, ident2
   {}
 
-  // expected-error@+1{{expected identifier}}
+  // expected-error@+2{{expected identifier}}
+  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
 #pragma acc parallel device_type(ident, ident2,)
   {}
-  // expected-error@+1{{expected identifier}}
+  // expected-error@+2{{expected identifier}}
+  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
 #pragma acc parallel dtype(ident, ident2,)
   {}
 
@@ -1196,25 +1200,33 @@ void device_type() {
 #pragma acc parallel dtype(*,ident)
   {}
 
-  // expected-error@+1{{expected identifier}}
+  // expected-error@+2{{expected identifier}}
+  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
 #pragma acc parallel device_type(ident, *)
   {}
-  // expected-error@+1{{expected identifier}}
+  // expected-error@+2{{expected identifier}}
+  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
 #pragma acc parallel dtype(ident, *)
   {}
 
-  // expected-error@+1{{expected identifier}}
+  // expected-error@+2{{expected identifier}}
+  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
 #pragma acc parallel device_type("foo", 54)
   {}
-  // expected-error@+1{{expected identifier}}
+  // expected-error@+2{{expected identifier}}
+  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
 #pragma acc parallel dtype(31, "bar")
   {}
 
+  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
 #pragma acc parallel device_type(ident, auto, int, float)
   {}
+  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
 #pragma acc parallel dtype(ident, auto, int, float)
   {}
 
+  // expected-warning@+2{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
+  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
 #pragma acc parallel device_type(ident, auto, int, float) dtype(ident, auto, int, float)
   {}
 }
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp b/clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp
deleted file mode 100644
index 8a2423f4f542..000000000000
--- a/clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp
+++ /dev/null
@@ -1,105 +0,0 @@
-// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
-
-// Test this with PCH.
-// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
-// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
-#ifndef PCH_HELPER
-#define PCH_HELPER
-
-struct SomeS{};
-void NormalUses() {
-  // CHECK: FunctionDecl{{.*}}NormalUses
-  // CHECK-NEXT: CompoundStmt
-
-  SomeS SomeImpl;
-  // CHECK-NEXT: DeclStmt
-  // CHECK-NEXT: VarDecl{{.*}} SomeImpl 'SomeS'
-  // CHECK-NEXT: CXXConstructExpr
-  bool SomeVar;
-  // CHECK-NEXT: DeclStmt
-  // CHECK-NEXT: VarDecl{{.*}} SomeVar 'bool'
-
-#pragma acc parallel device_type(SomeS) dtype(SomeImpl)
-  while(true){}
-  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
-  // CHECK-NEXT: device_type(SomeS)
-  // CHECK-NEXT: dtype(SomeImpl)
-  // CHECK-NEXT: WhileStmt
-  // CHECK-NEXT: CXXBoolLiteralExpr
-  // CHECK-NEXT: CompoundStmt
-#pragma acc parallel device_type(SomeVar) dtype(int)
-  while(true){}
-  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
-  // CHECK-NEXT: device_type(SomeVar)
-  // CHECK-NEXT: dtype(int)
-  // CHECK-NEXT: WhileStmt
-  // CHECK-NEXT: CXXBoolLiteralExpr
-  // CHECK-NEXT: CompoundStmt
-#pragma acc parallel device_type(private) dtype(struct)
-  while(true){}
-  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
-  // CHECK-NEXT: device_type(private)
-  // CHECK-NEXT: dtype(struct)
-  // CHECK-NEXT: WhileStmt
-  // CHECK-NEXT: CXXBoolLiteralExpr
-  // CHECK-NEXT: CompoundStmt
-#pragma acc parallel device_type(private) dtype(class)
-  while(true){}
-  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
-  // CHECK-NEXT: device_type(private)
-  // CHECK-NEXT: dtype(class)
-  // CHECK-NEXT: WhileStmt
-  // CHECK-NEXT: CXXBoolLiteralExpr
-  // CHECK-NEXT: CompoundStmt
-#pragma acc parallel device_type(float) dtype(*)
-  while(true){}
-  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
-  // CHECK-NEXT: device_type(float)
-  // CHECK-NEXT: dtype(*)
-  // CHECK-NEXT: WhileStmt
-  // CHECK-NEXT: CXXBoolLiteralExpr
-  // CHECK-NEXT: CompoundStmt
-#pragma acc parallel device_type(float, int) dtype(*)
-  while(true){}
-  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
-  // CHECK-NEXT: device_type(float, int)
-  // CHECK-NEXT: dtype(*)
-  // CHECK-NEXT: WhileStmt
-  // CHECK-NEXT: CXXBoolLiteralExpr
-  // CHECK-NEXT: CompoundStmt
-}
-
-template<typename T>
-void TemplUses() {
-  // CHECK-NEXT: FunctionTemplateDecl{{.*}}TemplUses
-  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}T
-  // CHECK-NEXT: FunctionDecl{{.*}}TemplUses
-  // CHECK-NEXT: CompoundStmt
-#pragma acc parallel device_type(T) dtype(T)
-  while(true){}
-  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
-  // CHECK-NEXT: device_type(T)
-  // CHECK-NEXT: dtype(T)
-  // CHECK-NEXT: WhileStmt
-  // CHECK-NEXT: CXXBoolLiteralExpr
-  // CHECK-NEXT: CompoundStmt
-
-
-  // Instantiations
-  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void ()' implicit_instantiation
-  // CHECK-NEXT: TemplateArgument type 'int'
-  // CHECK-NEXT: BuiltinType{{.*}} 'int'
-  // CHECK-NEXT: CompoundStmt
-
-  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
-  // CHECK-NEXT: device_type(T)
-  // CHECK-NEXT: dtype(T)
-  // CHECK-NEXT: WhileStmt
-  // CHECK-NEXT: CXXBoolLiteralExpr
-  // CHECK-NEXT: CompoundStmt
-}
-
-void Inst() {
-  TemplUses<int>();
-}
-#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
deleted file mode 100644
index 15c9cf396c80..000000000000
--- a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
+++ /dev/null
@@ -1,221 +0,0 @@
-// RUN: %clang_cc1 %s -fopenacc -verify
-
-#define MACRO +FOO
-
-void uses() {
-  typedef struct S{} STy;
-  STy SImpl;
-
-#pragma acc parallel device_type(I)
-  while(1);
-#pragma acc serial device_type(S) dtype(STy)
-  while(1);
-#pragma acc kernels dtype(SImpl)
-  while(1);
-#pragma acc kernels dtype(int) device_type(*)
-  while(1);
-#pragma acc kernels dtype(true) device_type(false)
-  while(1);
-
-  // expected-error@+1{{expected identifier}}
-#pragma acc kernels dtype(int, *)
-  while(1);
-
-#pragma acc parallel device_type(I, int)
-  while(1);
-  // expected-error@+2{{expected ','}}
-  // expected-error@+1{{expected identifier}}
-#pragma acc kernels dtype(int{})
-  while(1);
-  // expected-error@+1{{expected identifier}}
-#pragma acc kernels dtype(5)
-  while(1);
-  // expected-error@+1{{expected identifier}}
-#pragma acc kernels dtype(MACRO)
-  while(1);
-
-
-  // Only 'async', 'wait', num_gangs', 'num_workers', 'vector_length' allowed after 'device_type'.
-
-  // expected-error@+2{{OpenACC clause 'finalize' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) finalize
-  while(1);
-  // expected-error@+2{{OpenACC clause 'if_present' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) if_present
-  while(1);
-  // expected-error@+2{{OpenACC clause 'seq' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) seq
-  while(1);
-  // expected-error@+2{{OpenACC clause 'independent' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) independent
-  while(1);
-  // expected-error@+2{{OpenACC clause 'auto' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) auto
-  while(1);
-  // expected-error@+2{{OpenACC clause 'worker' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) worker
-  while(1);
-  // expected-error@+2{{OpenACC clause 'nohost' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) nohost
-  while(1);
-  // expected-error@+2{{OpenACC clause 'default' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) default(none)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'if' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) if(1)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'self' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) self
-  while(1);
-
-  int Var;
-  int *VarPtr;
-  // expected-error@+2{{OpenACC clause 'copy' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) copy(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'pcopy' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) pcopy(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'present_or_copy' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) present_or_copy(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'use_device' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) use_device(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'attach' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) attach(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'delete' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) delete(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'detach' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) detach(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'device' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) device(VarPtr)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'deviceptr' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) deviceptr(VarPtr)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'device_resident' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*)  device_resident(VarPtr)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'firstprivate' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc parallel device_type(*) firstprivate(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'host' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) host(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'link' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) link(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'no_create' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) no_create(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'present' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) present(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'private' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc parallel device_type(*) private(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'copyout' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) copyout(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'pcopyout' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) pcopyout(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'present_or_copyout' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) present_or_copyout(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'copyin' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) copyin(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'pcopyin' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) pcopyin(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'present_or_copyin' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) present_or_copyin(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'create' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) create(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'pcreate' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) pcreate(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'present_or_create' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) present_or_create(Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'reduction' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) reduction(+:Var)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'collapse' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) collapse(1)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'bind' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) bind(Var)
-  while(1);
-#pragma acc kernels device_type(*) vector_length(1)
-  while(1);
-#pragma acc kernels device_type(*) num_gangs(1)
-  while(1);
-#pragma acc kernels device_type(*) num_workers(1)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'device_num' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) device_num(1)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'default_async' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) default_async(1)
-  while(1);
-#pragma acc kernels device_type(*) async
-  while(1);
-  // expected-error@+2{{OpenACC clause 'tile' may not follow a 'device_type' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels device_type(*) tile(Var, 1)
-  while(1);
-  // expected-error@+2{{OpenACC clause 'gang' may not follow a 'dtype' clause in a compute construct}}
-  // expected-note@+1{{previous clause is here}}
-#pragma acc kernels dtype(*) gang
-  while(1);
-#pragma acc kernels device_type(*) wait
-  while(1);
-}
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp b/clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp
deleted file mode 100644
index ed40e8bbceae..000000000000
--- a/clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp
+++ /dev/null
@@ -1,25 +0,0 @@
-// RUN: %clang_cc1 %s -fopenacc -verify
-
-template<typename T>
-void TemplUses() {
-#pragma acc parallel device_type(I)
-  while(true);
-#pragma acc parallel dtype(*)
-  while(true);
-#pragma acc parallel device_type(class)
-  while(true);
-#pragma acc parallel device_type(private)
-  while(true);
-#pragma acc parallel device_type(bool)
-  while(true);
-#pragma acc kernels dtype(true) device_type(false)
-  while(true);
-  // expected-error@+2{{expected ','}}
-  // expected-error@+1{{expected identifier}}
-#pragma acc parallel device_type(T::value)
-  while(true);
-}
-
-void Inst() {
-  TemplUses<int>(); // #INST
-}
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 8b9417f985b5..ae6659fe95e8 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2857,8 +2857,6 @@ void OpenACCClauseEnqueue::VisitWaitClause(const OpenACCWaitClause &C) {
   for (Expr *QE : C.getQueueIdExprs())
     Visitor.AddStmt(QE);
 }
-void OpenACCClauseEnqueue::VisitDeviceTypeClause(
-    const OpenACCDeviceTypeClause &C) {}
 } // namespace
 
 void EnqueueVisitor::EnqueueChildren(const OpenACCClause *C) {
-- 
GitLab


From 1934e4afd6276cd2440357b788d7efe6a4abdebe Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Mon, 13 May 2024 10:41:36 -0500
Subject: [PATCH 0590/1206] [flang][OpenMP] Add explicit N to SmallVector to
 avoid size error

This fixes https://lab.llvm.org/buildbot/#/builders/268/builds/13925,
which somehow doesn't show in any of my local builds.
---
 llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
index edb65ed6d324..9dcb115a0c51 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
@@ -297,7 +297,7 @@ template <typename C> void ConstructCompositionT<C>::mergeDSA() {
   };
 
   // Use ordered containers to avoid non-deterministic output.
-  llvm::SmallVector<std::pair<ObjectTy, int>> objectDsa;
+  llvm::SmallVector<std::pair<ObjectTy, int>, 8> objectDsa;
 
   auto getDsa = [&](const ObjectTy &object) -> std::pair<ObjectTy, int> & {
     auto found = llvm::find_if(objectDsa, [&](std::pair<ObjectTy, int> &p) {
-- 
GitLab


From 2d511cdc10be611999d2a3c8983a992dd90f892c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Mon, 13 May 2024 17:44:14 +0200
Subject: [PATCH 0591/1206] [clang][Interp][NFC] Add missing test diagnostic
 output

This was left out from https://github.com/llvm/llvm-project/commit/257013e4f5cbdf644646da9ec3d60d6209c9bf25
---
 clang/test/AST/Interp/arrays.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/clang/test/AST/Interp/arrays.cpp b/clang/test/AST/Interp/arrays.cpp
index 70e87c4cd854..f6d265d4b3d1 100644
--- a/clang/test/AST/Interp/arrays.cpp
+++ b/clang/test/AST/Interp/arrays.cpp
@@ -563,6 +563,11 @@ namespace LocalVLA {
 
   void f (unsigned int m) {
     int e[2][m];
+#if __cplusplus >= 202002L
+     // both-note@-3 {{declared here}}
+     // both-warning@-3 2{{variable length array}}
+     // both-note@-4 {{function parameter 'm' with unknown value}}
+#endif
     e[0][0] = 0;
   }
 }
-- 
GitLab


From d95f7c9cabf493ffdc615df47a420a80d4be8e5c Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Mon, 13 May 2024 08:52:59 -0700
Subject: [PATCH 0592/1206] [RISCV] Use the thread local stack protector for
 Android targets (#87672)

Android supports per thread stack protectors that are individually
managed and
initialized, which can provide stronger protections than using the
global stack
protector cookie. This patch matches the convention for other
architectures
targeting Android platforms.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp     |  6 ++++++
 .../CodeGen/RISCV/stack-protector-target.ll     | 17 +++++++----------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 60985edd9420..d0f62b1d5414 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -21052,6 +21052,12 @@ Value *RISCVTargetLowering::getIRStackGuard(IRBuilderBase &IRB) const {
   if (Subtarget.isTargetFuchsia())
     return useTpOffset(IRB, -0x10);
 
+  // Android provides a fixed TLS slot for the stack cookie. See the definition
+  // of TLS_SLOT_STACK_GUARD in
+  // https://android.googlesource.com/platform/bionic/+/main/libc/platform/bionic/tls_defines.h
+  if (Subtarget.isTargetAndroid())
+    return useTpOffset(IRB, -0x18);
+
   return TargetLowering::getIRStackGuard(IRB);
 }
 
diff --git a/llvm/test/CodeGen/RISCV/stack-protector-target.ll b/llvm/test/CodeGen/RISCV/stack-protector-target.ll
index 50531d384982..a4bd0e9ceac9 100644
--- a/llvm/test/CodeGen/RISCV/stack-protector-target.ll
+++ b/llvm/test/CodeGen/RISCV/stack-protector-target.ll
@@ -50,21 +50,18 @@ define void @func() sspreq nounwind {
 ; ANDROID-RISCV64:       # %bb.0:
 ; ANDROID-RISCV64-NEXT:    addi sp, sp, -32
 ; ANDROID-RISCV64-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; ANDROID-RISCV64-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; ANDROID-RISCV64-NEXT:    lui s0, %hi(__stack_chk_guard)
-; ANDROID-RISCV64-NEXT:    ld a0, %lo(__stack_chk_guard)(s0)
-; ANDROID-RISCV64-NEXT:    sd a0, 8(sp)
-; ANDROID-RISCV64-NEXT:    addi a0, sp, 4
+; ANDROID-RISCV64-NEXT:    ld a0, -24(tp)
+; ANDROID-RISCV64-NEXT:    sd a0, 16(sp)
+; ANDROID-RISCV64-NEXT:    addi a0, sp, 12
 ; ANDROID-RISCV64-NEXT:    call capture
-; ANDROID-RISCV64-NEXT:    ld a0, %lo(__stack_chk_guard)(s0)
-; ANDROID-RISCV64-NEXT:    ld a1, 8(sp)
+; ANDROID-RISCV64-NEXT:    ld a0, -24(tp)
+; ANDROID-RISCV64-NEXT:    ld a1, 16(sp)
 ; ANDROID-RISCV64-NEXT:    bne a0, a1, .LBB0_2
-; ANDROID-RISCV64-NEXT:  # %bb.1:
+; ANDROID-RISCV64-NEXT:  # %bb.1: # %SP_return
 ; ANDROID-RISCV64-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; ANDROID-RISCV64-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; ANDROID-RISCV64-NEXT:    addi sp, sp, 32
 ; ANDROID-RISCV64-NEXT:    ret
-; ANDROID-RISCV64-NEXT:  .LBB0_2:
+; ANDROID-RISCV64-NEXT:  .LBB0_2: # %CallStackCheckFailBlk
 ; ANDROID-RISCV64-NEXT:    call __stack_chk_fail
   %1 = alloca i32, align 4
   call void @capture(ptr %1)
-- 
GitLab


From d94e0a1005ee1cfeeda829d9deb1f299deb7b10a Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 13 May 2024 15:53:36 +0000
Subject: [PATCH 0593/1206] [gn build] Port be7c9e39572d

---
 llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
index 80ac77feec9a..6070a6f00419 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
@@ -15,6 +15,7 @@ unittest("LLVMFrontendTests") {
     "OpenACCTest.cpp",
     "OpenMPCompositionTest.cpp",
     "OpenMPContextTest.cpp",
+    "OpenMPDecompositionTest.cpp",
     "OpenMPIRBuilderTest.cpp",
     "OpenMPParsingTest.cpp",
   ]
-- 
GitLab


From 754ff0f54a4b09a8e4b00783475c51f66b949b66 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Mon, 13 May 2024 17:05:28 +0100
Subject: [PATCH 0594/1206] [TableGen][RISCV] Use
 getAllDerivedDefinitionsIfDefined in RISCVTargetDefEmitter (#91941)

getAllDerivedDefinitions produces a fatal error if there are no
definitions. In practice this isn't much of a problem for
llvm/lib/Target/RISCV/*.td where it's hard to imagine not having at
least one of the required defitions. But it limits our ability to
structure and maintain tests (which is how I came across this issue).

This commit moves to using getAllDerivedDefinitionsIfDefined and aims to
skip emission of data structures that make no sense if no definitions
were found.
---
 llvm/utils/TableGen/RISCVTargetDefEmitter.cpp | 60 ++++++++++---------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
index 6784514032eb..bb409ea6ea69 100644
--- a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
@@ -48,37 +48,41 @@ static void emitRISCVExtensions(RecordKeeper &Records, raw_ostream &OS) {
   OS << "#undef GET_SUPPORTED_EXTENSIONS\n\n";
 
   std::vector<Record *> Extensions =
-      Records.getAllDerivedDefinitions("RISCVExtension");
+      Records.getAllDerivedDefinitionsIfDefined("RISCVExtension");
   llvm::sort(Extensions, [](const Record *Rec1, const Record *Rec2) {
     return getExtensionName(Rec1) < getExtensionName(Rec2);
   });
 
-  printExtensionTable(OS, Extensions, /*Experimental=*/false);
-  printExtensionTable(OS, Extensions, /*Experimental=*/true);
+  if (!Extensions.empty()) {
+    printExtensionTable(OS, Extensions, /*Experimental=*/false);
+    printExtensionTable(OS, Extensions, /*Experimental=*/true);
+  }
 
   OS << "#endif // GET_SUPPORTED_EXTENSIONS\n\n";
 
   OS << "#ifdef GET_IMPLIED_EXTENSIONS\n";
   OS << "#undef GET_IMPLIED_EXTENSIONS\n\n";
 
-  OS << "\nstatic constexpr ImpliedExtsEntry ImpliedExts[] = {\n";
-  for (Record *Ext : Extensions) {
-    auto ImpliesList = Ext->getValueAsListOfDefs("Implies");
-    if (ImpliesList.empty())
-      continue;
+  if (!Extensions.empty()) {
+    OS << "\nstatic constexpr ImpliedExtsEntry ImpliedExts[] = {\n";
+    for (Record *Ext : Extensions) {
+      auto ImpliesList = Ext->getValueAsListOfDefs("Implies");
+      if (ImpliesList.empty())
+        continue;
 
-    StringRef Name = getExtensionName(Ext);
+      StringRef Name = getExtensionName(Ext);
 
-    for (auto *ImpliedExt : ImpliesList) {
-      if (!ImpliedExt->isSubClassOf("RISCVExtension"))
-        continue;
+      for (auto *ImpliedExt : ImpliesList) {
+        if (!ImpliedExt->isSubClassOf("RISCVExtension"))
+          continue;
 
-      OS << "    { {\"" << Name << "\"}, \"" << getExtensionName(ImpliedExt)
-         << "\"},\n";
+        OS << "    { {\"" << Name << "\"}, \"" << getExtensionName(ImpliedExt)
+           << "\"},\n";
+      }
     }
-  }
 
-  OS << "};\n\n";
+    OS << "};\n\n";
+  }
 
   OS << "#endif // GET_IMPLIED_EXTENSIONS\n\n";
 }
@@ -122,19 +126,20 @@ static void emitRISCVProfiles(RecordKeeper &Records, raw_ostream &OS) {
   OS << "#ifdef GET_SUPPORTED_PROFILES\n";
   OS << "#undef GET_SUPPORTED_PROFILES\n\n";
 
-  OS << "static constexpr RISCVProfile SupportedProfiles[] = {\n";
+  auto Profiles = Records.getAllDerivedDefinitionsIfDefined("RISCVProfile");
 
-  auto Profiles = Records.getAllDerivedDefinitions("RISCVProfile");
-  llvm::sort(Profiles, LessRecordFieldName());
+  if (!Profiles.empty()) {
+    llvm::sort(Profiles, LessRecordFieldName());
+    OS << "static constexpr RISCVProfile SupportedProfiles[] = {\n";
+    for (const Record *Rec : Profiles) {
+      OS.indent(4) << "{\"" << Rec->getValueAsString("Name") << "\",\"";
+      printMArch(OS, Rec->getValueAsListOfDefs("Implies"));
+      OS << "\"},\n";
+    }
 
-  for (const Record *Rec : Profiles) {
-    OS.indent(4) << "{\"" << Rec->getValueAsString("Name") << "\",\"";
-    printMArch(OS, Rec->getValueAsListOfDefs("Implies"));
-    OS << "\"},\n";
+    OS << "};\n\n";
   }
 
-  OS << "};\n\n";
-
   OS << "#endif // GET_SUPPORTED_PROFILES\n\n";
 }
 
@@ -144,7 +149,8 @@ static void emitRISCVProcs(RecordKeeper &RK, raw_ostream &OS) {
      << "#endif\n\n";
 
   // Iterate on all definition records.
-  for (const Record *Rec : RK.getAllDerivedDefinitions("RISCVProcessorModel")) {
+  for (const Record *Rec :
+       RK.getAllDerivedDefinitionsIfDefined("RISCVProcessorModel")) {
     const std::vector<Record *> &Features =
         Rec->getValueAsListOfDefs("Features");
     bool FastScalarUnalignedAccess = any_of(Features, [&](auto &Feature) {
@@ -177,7 +183,7 @@ static void emitRISCVProcs(RecordKeeper &RK, raw_ostream &OS) {
      << "#endif\n\n";
 
   for (const Record *Rec :
-       RK.getAllDerivedDefinitions("RISCVTuneProcessorModel")) {
+       RK.getAllDerivedDefinitionsIfDefined("RISCVTuneProcessorModel")) {
     OS << "TUNE_PROC(" << Rec->getName() << ", "
        << "\"" << Rec->getValueAsString("Name") << "\")\n";
   }
-- 
GitLab


From 89a080cb79972abae240c226090af9a3094e2269 Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Mon, 25 Mar 2024 21:02:15 +0000
Subject: [PATCH 0595/1206] [llvm][NFC] Document cl::opt MisExpectTolerance and
 fix typo

Pull Request: https://github.com/llvm/llvm-project/pull/90670
---
 llvm/lib/Transforms/Utils/MisExpect.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Utils/MisExpect.cpp b/llvm/lib/Transforms/Utils/MisExpect.cpp
index 6f5a25a26821..759289384ee0 100644
--- a/llvm/lib/Transforms/Utils/MisExpect.cpp
+++ b/llvm/lib/Transforms/Utils/MisExpect.cpp
@@ -59,9 +59,10 @@ static cl::opt<bool> PGOWarnMisExpect(
     cl::desc("Use this option to turn on/off "
              "warnings about incorrect usage of llvm.expect intrinsics."));
 
+// Command line option for setting the diagnostic tolerance threshold
 static cl::opt<uint32_t> MisExpectTolerance(
     "misexpect-tolerance", cl::init(0),
-    cl::desc("Prevents emiting diagnostics when profile counts are "
+    cl::desc("Prevents emitting diagnostics when profile counts are "
              "within N% of the threshold.."));
 
 } // namespace llvm
-- 
GitLab


From 596a9c1f9b3179b3c77cbde1e96619292ce2a10a Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Mon, 13 May 2024 12:24:46 -0400
Subject: [PATCH 0596/1206] [Clang][Sema] Fix bug where operator-> typo
 corrects in the current instantiation (#91972)

#90152 introduced a bug that occurs when typo-correction attempts to fix a reference to a
non-existent member of the current instantiation (even though
`operator->` may return a different type than the object type). This
patch fixes it by simply considering the object expression to be of type
`ASTContext::DependentTy` when the arrow operator is used with a
dependent non-pointer non-function operand (after any implicit
conversions).
---
 clang/lib/Sema/SemaExprMember.cpp             | 49 ++++++++++---------
 .../temp.res/temp.dep/temp.dep.type/p4.cpp    | 27 ++++++++++
 2 files changed, 52 insertions(+), 24 deletions(-)

diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp
index 9fa69da4f968..244488a0b562 100644
--- a/clang/lib/Sema/SemaExprMember.cpp
+++ b/clang/lib/Sema/SemaExprMember.cpp
@@ -995,8 +995,6 @@ Sema::BuildMemberReferenceExpr(Expr *BaseExpr, QualType BaseExprType,
   // arrow operator was used with a dependent non-pointer object expression,
   // build a CXXDependentScopeMemberExpr.
   if (R.wasNotFoundInCurrentInstantiation() ||
-      (IsArrow && !BaseExprType->isPointerType() &&
-       BaseExprType->isDependentType()) ||
       (R.getLookupName().getCXXOverloadedOperator() == OO_Equal &&
        (SS.isSet() ? SS.getScopeRep()->isDependent()
                    : BaseExprType->isDependentType())))
@@ -1322,28 +1320,28 @@ static ExprResult LookupMemberExpr(Sema &S, LookupResult &R,
     else if (const ObjCObjectPointerType *Ptr =
                  BaseType->getAs<ObjCObjectPointerType>())
       BaseType = Ptr->getPointeeType();
-    else if (!BaseType->isDependentType()) {
-      if (BaseType->isRecordType()) {
-        // Recover from arrow accesses to records, e.g.:
-        //   struct MyRecord foo;
-        //   foo->bar
-        // This is actually well-formed in C++ if MyRecord has an
-        // overloaded operator->, but that should have been dealt with
-        // by now--or a diagnostic message already issued if a problem
-        // was encountered while looking for the overloaded operator->.
-        if (!S.getLangOpts().CPlusPlus) {
-          S.Diag(OpLoc, diag::err_typecheck_member_reference_suggestion)
-              << BaseType << int(IsArrow) << BaseExpr.get()->getSourceRange()
-              << FixItHint::CreateReplacement(OpLoc, ".");
-        }
-        IsArrow = false;
-      } else if (BaseType->isFunctionType()) {
-        goto fail;
-      } else {
-        S.Diag(MemberLoc, diag::err_typecheck_member_reference_arrow)
-            << BaseType << BaseExpr.get()->getSourceRange();
-        return ExprError();
+    else if (BaseType->isFunctionType())
+      goto fail;
+    else if (BaseType->isDependentType())
+      BaseType = S.Context.DependentTy;
+    else if (BaseType->isRecordType()) {
+      // Recover from arrow accesses to records, e.g.:
+      //   struct MyRecord foo;
+      //   foo->bar
+      // This is actually well-formed in C++ if MyRecord has an
+      // overloaded operator->, but that should have been dealt with
+      // by now--or a diagnostic message already issued if a problem
+      // was encountered while looking for the overloaded operator->.
+      if (!S.getLangOpts().CPlusPlus) {
+        S.Diag(OpLoc, diag::err_typecheck_member_reference_suggestion)
+            << BaseType << int(IsArrow) << BaseExpr.get()->getSourceRange()
+            << FixItHint::CreateReplacement(OpLoc, ".");
       }
+      IsArrow = false;
+    } else {
+      S.Diag(MemberLoc, diag::err_typecheck_member_reference_arrow)
+          << BaseType << BaseExpr.get()->getSourceRange();
+      return ExprError();
     }
   }
 
@@ -1363,7 +1361,7 @@ static ExprResult LookupMemberExpr(Sema &S, LookupResult &R,
   }
 
   // Handle field access to simple records.
-  if (BaseType->getAsRecordDecl() || BaseType->isDependentType()) {
+  if (BaseType->getAsRecordDecl()) {
     TypoExpr *TE = nullptr;
     if (LookupMemberExprInRecord(S, R, BaseExpr.get(), BaseType, OpLoc, IsArrow,
                                  SS, HasTemplateArgs, TemplateKWLoc, TE))
@@ -1374,6 +1372,9 @@ static ExprResult LookupMemberExpr(Sema &S, LookupResult &R,
     // failed, the lookup result will have been cleared--that combined with the
     // valid-but-null ExprResult will trigger the appropriate diagnostics.
     return ExprResult(TE);
+  } else if (BaseType->isDependentType()) {
+    R.setNotFoundInCurrentInstantiation();
+    return ExprEmpty();
   }
 
   // Handle ivar access to Objective-C objects.
diff --git a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
index 1adbc33a701c..3ca7c6c7eb8e 100644
--- a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
@@ -539,6 +539,17 @@ namespace N4 {
       a->y;
       a->f();
       a->g();
+
+      a->T::x;
+      a->T::y;
+      a->T::f();
+      a->T::g();
+
+      // FIXME: 'U' should be a dependent name, and its lookup context should be 'a.operator->()'!
+      a->U::x; // expected-error {{use of undeclared identifier 'U'}}
+      a->U::y; // expected-error {{use of undeclared identifier 'U'}}
+      a->U::f(); // expected-error {{use of undeclared identifier 'U'}}
+      a->U::g(); // expected-error {{use of undeclared identifier 'U'}}
     }
 
     void instantiated(D a) {
@@ -546,9 +557,25 @@ namespace N4 {
       a->y; // expected-error {{no member named 'y' in 'N4::B'}}
       a->f();
       a->g(); // expected-error {{no member named 'g' in 'N4::B'}}
+
+      a->T::x;
+      a->T::y; // expected-error {{no member named 'y' in 'N4::B'}}
+      a->T::f();
+      a->T::g(); // expected-error {{no member named 'g' in 'N4::B'}}
     }
   };
 
   template void D<B>::instantiated(D); // expected-note {{in instantiation of}}
 
+  template<typename T>
+  struct Typo {
+    T *operator->();
+
+    void not_instantiated(Typo a) {
+      a->Not_instantiated;
+      a->typo;
+      a->T::Not_instantiated;
+      a->T::typo;
+    }
+  };
 } // namespace N4
-- 
GitLab


From 1066eb55477044a3a92f3a40471375194dfcdbc8 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Mon, 13 May 2024 09:33:43 -0700
Subject: [PATCH 0597/1206] [flang] Fix a warning

This patch fixes:

  flang/lib/Lower/OpenMP/OpenMP.cpp:2346:14: error: unused variable
  'origDirective' [-Werror,-Wunused-variable]
---
 flang/lib/Lower/OpenMP/OpenMP.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index eaf4b5f997ff..f9ba2fcbbca7 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2352,6 +2352,7 @@ genOMP(Fortran::lower::AbstractConverter &converter,
 
   assert(llvm::omp::blockConstructSet.test(origDirective) &&
          "Expected block construct");
+  (void)origDirective;
 
   for (const Clause &clause : clauses) {
     mlir::Location clauseLocation = converter.genLocation(clause.source);
-- 
GitLab


From 79643565a890d29560782fea23877ae61ea5b987 Mon Sep 17 00:00:00 2001
From: chenlin <chenlin138@huawei.com>
Date: Tue, 14 May 2024 00:42:04 +0800
Subject: [PATCH 0598/1206] [LoopUnroll] Remove redundant debug instructions
 after blocks have been merged (#91246)

Remove redundant debug instructions after blocks have been merged into
the predecessor, It can reduce some compile time in some cases.

This change only fixes the situation of loop unrolling, and other
situations are not considered. "RemoveRedundantDbgInstrs" seems to be
very time-consuming. Thus, we just add here after the "Dest" has been
merged into the "Fold", this may be a more targeted solution!!!

fixes: https://github.com/llvm/llvm-project/issues/89073
---
 llvm/lib/Transforms/Utils/LoopUnroll.cpp      |  4 ++
 .../LoopUnroll/unroll-remove-redundant-dbg.ll | 50 +++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-remove-redundant-dbg.ll

diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 20978cf2e748..1216538195fb 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -377,6 +377,10 @@ void llvm::simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI,
   const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
   SmallVector<WeakTrackingVH, 16> DeadInsts;
   for (BasicBlock *BB : L->getBlocks()) {
+    // Remove repeated debug instructions after loop unrolling.
+    if (BB->getParent()->getSubprogram())
+      RemoveRedundantDbgInstrs(BB);
+
     for (Instruction &Inst : llvm::make_early_inc_range(*BB)) {
       if (Value *V = simplifyInstruction(&Inst, {DL, nullptr, DT, AC}))
         if (LI->replacementPreservesLCSSAForm(&Inst, V))
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-remove-redundant-dbg.ll b/llvm/test/Transforms/LoopUnroll/unroll-remove-redundant-dbg.ll
new file mode 100644
index 000000000000..8e348281dc61
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-remove-redundant-dbg.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes=loop-unroll | FileCheck %s
+
+define i64 @d(i1 %tobool.not, i32 %add, i64 %conv23) !dbg !14{
+; There should be only one "llvm.dbg.vale" after loop unrolling
+; CHECK-LABEL: @d(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    tail call void @llvm.dbg.value(metadata i32 0, metadata [[META16:![0-9]+]], metadata !DIExpression()), !dbg [[DBG17:![0-9]+]]
+; CHECK-NEXT:    ret i64 5
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %k.045 = phi i64 [ 0, %entry ], [ %k.046, %for.body ]
+  tail call void @llvm.dbg.value(metadata i32 0, metadata !13, metadata !DIExpression()), !dbg !17
+  %k.046 = add nuw nsw i64 %k.045, 1
+  %exitcond = icmp ne i64 %k.046, 5
+  br i1 %exitcond, label %for.body, label %for.end22
+
+for.end22:                                        ; preds = %for.body
+  ret i64 %k.046
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C89, file: !1, producer: "clang version 19.0.0git (https://github.com/llvm/llvm-project.git ec062f5b33ed22c61742e3c1486f6cba915801e0)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "unroll-remove-redundant-dbg.c", directory: "", checksumkind: CSK_MD5, checksum: "aa30a1d8c04deb9b0f3885c258d2b674")
+!2 = !{!3, !8, !10}
+!3 = !DIGlobalVariableExpression(var: !4, expr: !DIExpression())
+!4 = distinct !DIGlobalVariable(name: "a", scope: !0, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true)
+!5 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint32_t", file: !6, line: 198, baseType: !7)
+!6 = !DIFile(filename: "/usr/include/stdint.h", directory: "", checksumkind: CSK_MD5, checksum: "da031bcff2d0c1d65aa92e7e68a44ef3")
+!7 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!8 = !DIGlobalVariableExpression(var: !9, expr: !DIExpression())
+!9 = distinct !DIGlobalVariable(name: "c", scope: !0, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true)
+!10 = !DIGlobalVariableExpression(var: !11, expr: !DIExpression())
+!11 = distinct !DIGlobalVariable(name: "b", scope: !0, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true)
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !DILocalVariable(name: "f", scope: !14, file: !1, line: 4, type: !5)
+!14 = distinct !DISubprogram(name: "d", scope: !1, file: !1, line: 3, type: !15, scopeLine: 3, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !16)
+!15 = !DISubroutineType(types: !16)
+!16 = !{}
+!17 = !DILocation(line: 0, scope: !14)
-- 
GitLab


From 91a14dbf825b79ff143d1b16124763a4a80facab Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Mon, 13 May 2024 09:47:34 -0700
Subject: [PATCH 0599/1206] Support polynomial attributes with floating point
 coefficients (#91137)

In summary:

- `Monomial` -> `MonomialBase` with two inheriting `IntMonomial` and
`FloatMonomial` for the different coefficient types
- `Polynomial` -> `PolynomialBase` with `IntPolynomial` and
`FloatPolynomial` inheriting
- `PolynomialAttr` -> `IntPolynomialAttr`, and new `FloatPolynomialAttr`
attribute, both of which may be input to `polynomial.constant`
- Refactoring common parts of attribute parsers.

---------

Co-authored-by: Jeremy Kun <j2kun@users.noreply.github.com>
---
 .../mlir/Dialect/Polynomial/IR/Polynomial.h   | 193 ++++++++++++++----
 .../mlir/Dialect/Polynomial/IR/Polynomial.td  | 139 ++++++++-----
 mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp |  78 +++----
 .../Polynomial/IR/PolynomialAttributes.cpp    | 172 +++++++---------
 mlir/test/Dialect/Polynomial/attributes.mlir  |  22 +-
 mlir/test/Dialect/Polynomial/ops.mlir         |  64 +++---
 mlir/test/Dialect/Polynomial/ops_errors.mlir  |  66 +++---
 mlir/test/Dialect/Polynomial/types.mlir       |  65 +++---
 8 files changed, 451 insertions(+), 348 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
index 3325a6fa3f9f..2b3f0e105c6c 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
@@ -11,10 +11,13 @@
 
 #include "mlir/Support/LLVM.h"
 #include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Hashing.h"
-#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/raw_ostream.h"
 
 namespace mlir {
 
@@ -27,98 +30,202 @@ namespace polynomial {
 /// would want to specify 128-bit polynomials statically in the source code.
 constexpr unsigned apintBitWidth = 64;
 
-/// A class representing a monomial of a single-variable polynomial with integer
-/// coefficients.
-class Monomial {
+template <typename CoefficientType>
+class MonomialBase {
 public:
-  Monomial(int64_t coeff, uint64_t expo)
-      : coefficient(apintBitWidth, coeff), exponent(apintBitWidth, expo) {}
-
-  Monomial(const APInt &coeff, const APInt &expo)
+  MonomialBase(const CoefficientType &coeff, const APInt &expo)
       : coefficient(coeff), exponent(expo) {}
+  virtual ~MonomialBase() = 0;
 
-  Monomial() : coefficient(apintBitWidth, 0), exponent(apintBitWidth, 0) {}
+  const CoefficientType &getCoefficient() const { return coefficient; }
+  CoefficientType &getMutableCoefficient() { return coefficient; }
+  const APInt &getExponent() const { return exponent; }
+  void setCoefficient(const CoefficientType &coeff) { coefficient = coeff; }
+  void setExponent(const APInt &exp) { exponent = exp; }
 
-  bool operator==(const Monomial &other) const {
+  bool operator==(const MonomialBase &other) const {
     return other.coefficient == coefficient && other.exponent == exponent;
   }
-  bool operator!=(const Monomial &other) const {
+  bool operator!=(const MonomialBase &other) const {
     return other.coefficient != coefficient || other.exponent != exponent;
   }
 
   /// Monomials are ordered by exponent.
-  bool operator<(const Monomial &other) const {
+  bool operator<(const MonomialBase &other) const {
     return (exponent.ult(other.exponent));
   }
 
-  friend ::llvm::hash_code hash_value(const Monomial &arg);
+  virtual bool isMonic() const = 0;
+  virtual void
+  coefficientToString(llvm::SmallString<16> &coeffString) const = 0;
 
-public:
-  APInt coefficient;
+  template <typename T>
+  friend ::llvm::hash_code hash_value(const MonomialBase<T> &arg);
 
-  // Always unsigned
+protected:
+  CoefficientType coefficient;
   APInt exponent;
 };
 
-/// A single-variable polynomial with integer coefficients.
-///
-/// Eg: x^1024 + x + 1
-///
-/// The symbols used as the polynomial's indeterminate don't matter, so long as
-/// it is used consistently throughout the polynomial.
-class Polynomial {
+/// A class representing a monomial of a single-variable polynomial with integer
+/// coefficients.
+class IntMonomial : public MonomialBase<APInt> {
 public:
-  Polynomial() = delete;
+  IntMonomial(int64_t coeff, uint64_t expo)
+      : MonomialBase(APInt(apintBitWidth, coeff), APInt(apintBitWidth, expo)) {}
 
-  explicit Polynomial(ArrayRef<Monomial> terms) : terms(terms){};
+  IntMonomial()
+      : MonomialBase(APInt(apintBitWidth, 0), APInt(apintBitWidth, 0)) {}
 
-  // Returns a Polynomial from a list of monomials.
-  // Fails if two monomials have the same exponent.
-  static FailureOr<Polynomial> fromMonomials(ArrayRef<Monomial> monomials);
+  ~IntMonomial() = default;
 
-  /// Returns a polynomial with coefficients given by `coeffs`. The value
-  /// coeffs[i] is converted to a monomial with exponent i.
-  static Polynomial fromCoefficients(ArrayRef<int64_t> coeffs);
+  bool isMonic() const override { return coefficient == 1; }
+
+  void coefficientToString(llvm::SmallString<16> &coeffString) const override {
+    coefficient.toStringSigned(coeffString);
+  }
+};
+
+/// A class representing a monomial of a single-variable polynomial with integer
+/// coefficients.
+class FloatMonomial : public MonomialBase<APFloat> {
+public:
+  FloatMonomial(double coeff, uint64_t expo)
+      : MonomialBase(APFloat(coeff), APInt(apintBitWidth, expo)) {}
+
+  FloatMonomial() : MonomialBase(APFloat((double)0), APInt(apintBitWidth, 0)) {}
+
+  ~FloatMonomial() = default;
+
+  bool isMonic() const override { return coefficient == APFloat(1.0); }
+
+  void coefficientToString(llvm::SmallString<16> &coeffString) const override {
+    coefficient.toString(coeffString);
+  }
+};
+
+template <typename Monomial>
+class PolynomialBase {
+public:
+  PolynomialBase() = delete;
+
+  explicit PolynomialBase(ArrayRef<Monomial> terms) : terms(terms){};
 
   explicit operator bool() const { return !terms.empty(); }
-  bool operator==(const Polynomial &other) const {
+  bool operator==(const PolynomialBase &other) const {
     return other.terms == terms;
   }
-  bool operator!=(const Polynomial &other) const {
+  bool operator!=(const PolynomialBase &other) const {
     return !(other.terms == terms);
   }
 
-  // Prints polynomial to 'os'.
-  void print(raw_ostream &os) const;
   void print(raw_ostream &os, ::llvm::StringRef separator,
-             ::llvm::StringRef exponentiation) const;
+             ::llvm::StringRef exponentiation) const {
+    bool first = true;
+    for (const Monomial &term : getTerms()) {
+      if (first) {
+        first = false;
+      } else {
+        os << separator;
+      }
+      std::string coeffToPrint;
+      if (term.isMonic() && term.getExponent().uge(1)) {
+        coeffToPrint = "";
+      } else {
+        llvm::SmallString<16> coeffString;
+        term.coefficientToString(coeffString);
+        coeffToPrint = coeffString.str();
+      }
+
+      if (term.getExponent() == 0) {
+        os << coeffToPrint;
+      } else if (term.getExponent() == 1) {
+        os << coeffToPrint << "x";
+      } else {
+        llvm::SmallString<16> expString;
+        term.getExponent().toStringSigned(expString);
+        os << coeffToPrint << "x" << exponentiation << expString;
+      }
+    }
+  }
+
+  // Prints polynomial to 'os'.
+  void print(raw_ostream &os) const { print(os, " + ", "**"); }
+
   void dump() const;
 
   // Prints polynomial so that it can be used as a valid identifier
-  std::string toIdentifier() const;
+  std::string toIdentifier() const {
+    std::string result;
+    llvm::raw_string_ostream os(result);
+    print(os, "_", "");
+    return os.str();
+  }
 
-  unsigned getDegree() const;
+  unsigned getDegree() const {
+    return terms.back().getExponent().getZExtValue();
+  }
 
   ArrayRef<Monomial> getTerms() const { return terms; }
 
-  friend ::llvm::hash_code hash_value(const Polynomial &arg);
+  template <typename T>
+  friend ::llvm::hash_code hash_value(const PolynomialBase<T> &arg);
 
 private:
   // The monomial terms for this polynomial.
   SmallVector<Monomial> terms;
 };
 
-// Make Polynomial hashable.
-inline ::llvm::hash_code hash_value(const Polynomial &arg) {
+/// A single-variable polynomial with integer coefficients.
+///
+/// Eg: x^1024 + x + 1
+class IntPolynomial : public PolynomialBase<IntMonomial> {
+public:
+  explicit IntPolynomial(ArrayRef<IntMonomial> terms) : PolynomialBase(terms) {}
+
+  // Returns a Polynomial from a list of monomials.
+  // Fails if two monomials have the same exponent.
+  static FailureOr<IntPolynomial>
+  fromMonomials(ArrayRef<IntMonomial> monomials);
+
+  /// Returns a polynomial with coefficients given by `coeffs`. The value
+  /// coeffs[i] is converted to a monomial with exponent i.
+  static IntPolynomial fromCoefficients(ArrayRef<int64_t> coeffs);
+};
+
+/// A single-variable polynomial with double coefficients.
+///
+/// Eg: 1.0 x^1024 + 3.5 x + 1e-05
+class FloatPolynomial : public PolynomialBase<FloatMonomial> {
+public:
+  explicit FloatPolynomial(ArrayRef<FloatMonomial> terms)
+      : PolynomialBase(terms) {}
+
+  // Returns a Polynomial from a list of monomials.
+  // Fails if two monomials have the same exponent.
+  static FailureOr<FloatPolynomial>
+  fromMonomials(ArrayRef<FloatMonomial> monomials);
+
+  /// Returns a polynomial with coefficients given by `coeffs`. The value
+  /// coeffs[i] is converted to a monomial with exponent i.
+  static FloatPolynomial fromCoefficients(ArrayRef<double> coeffs);
+};
+
+// Make Polynomials hashable.
+template <typename T>
+inline ::llvm::hash_code hash_value(const PolynomialBase<T> &arg) {
   return ::llvm::hash_combine_range(arg.terms.begin(), arg.terms.end());
 }
 
-inline ::llvm::hash_code hash_value(const Monomial &arg) {
+template <typename T>
+inline ::llvm::hash_code hash_value(const MonomialBase<T> &arg) {
   return llvm::hash_combine(::llvm::hash_value(arg.coefficient),
                             ::llvm::hash_value(arg.exponent));
 }
 
-inline raw_ostream &operator<<(raw_ostream &os, const Polynomial &polynomial) {
+template <typename T>
+inline raw_ostream &operator<<(raw_ostream &os,
+                               const PolynomialBase<T> &polynomial) {
   polynomial.print(os);
   return os;
 }
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
index ed1f4ce8b7e5..ae8484501a50 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
@@ -39,14 +39,14 @@ def Polynomial_Dialect : Dialect {
     %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
 
     // A constant polynomial in a ring with i32 coefficients, modulo (x^1024 + 1)
-    #modulus = #polynomial.polynomial<1 + x**1024>
+    #modulus = #polynomial.int_polynomial<1 + x**1024>
     #ring = #polynomial.ring<coefficientType=i32, polynomialModulus=#modulus>
     %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
 
     // A constant polynomial in a ring with i32 coefficients, with a polynomial
     // modulus of (x^1024 + 1) and a coefficient modulus of 17.
-    #modulus = #polynomial.polynomial<1 + x**1024>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=17, polynomialModulus=#modulus>
+    #modulus = #polynomial.int_polynomial<1 + x**1024>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=17:i32, polynomialModulus=#modulus>
     %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
     ```
   }];
@@ -60,12 +60,12 @@ class Polynomial_Attr<string name, string attrMnemonic, list<Trait> traits = []>
   let mnemonic = attrMnemonic;
 }
 
-def Polynomial_PolynomialAttr : Polynomial_Attr<"Polynomial", "polynomial"> {
-  let summary = "An attribute containing a single-variable polynomial.";
+def Polynomial_IntPolynomialAttr : Polynomial_Attr<"IntPolynomial", "int_polynomial"> {
+  let summary = "An attribute containing a single-variable polynomial with integer coefficients.";
   let description = [{
-    A polynomial attribute represents a single-variable polynomial, which
-    is used to define the modulus of a `RingAttr`, as well as to define constants
-    and perform constant folding for `polynomial` ops.
+    A polynomial attribute represents a single-variable polynomial with integer
+    coefficients, which is used to define the modulus of a `RingAttr`, as well
+    as to define constants and perform constant folding for `polynomial` ops.
 
     The polynomial must be expressed as a list of monomial terms, with addition
     or subtraction between them. The choice of variable name is arbitrary, but
@@ -76,10 +76,32 @@ def Polynomial_PolynomialAttr : Polynomial_Attr<"Polynomial", "polynomial"> {
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 + 1>
+    #poly = #polynomial.int_polynomial<x**1024 + 1>
     ```
   }];
-  let parameters = (ins "::mlir::polynomial::Polynomial":$polynomial);
+  let parameters = (ins "::mlir::polynomial::IntPolynomial":$polynomial);
+  let hasCustomAssemblyFormat = 1;
+}
+
+def Polynomial_FloatPolynomialAttr : Polynomial_Attr<"FloatPolynomial", "float_polynomial"> {
+  let summary = "An attribute containing a single-variable polynomial with double precision floating point coefficients.";
+  let description = [{
+    A polynomial attribute represents a single-variable polynomial with double
+    precision floating point coefficients.
+
+    The polynomial must be expressed as a list of monomial terms, with addition
+    or subtraction between them. The choice of variable name is arbitrary, but
+    must be consistent across all the monomials used to define a single
+    attribute. The order of monomial terms is arbitrary, each monomial degree
+    must occur at most once.
+
+    Example:
+
+    ```mlir
+    #poly = #polynomial.float_polynomial<0.5 x**7 + 1.5>
+    ```
+  }];
+  let parameters = (ins "FloatPolynomial":$polynomial);
   let hasCustomAssemblyFormat = 1;
 }
 
@@ -104,9 +126,9 @@ def Polynomial_RingAttr : Polynomial_Attr<"Ring", "ring"> {
     `x**1024 - 1`.
 
     ```mlir
-    #poly_mod = #polynomial.polynomial<-1 + x**1024>
+    #poly_mod = #polynomial.int_polynomial<-1 + x**1024>
     #ring = #polynomial.ring<coefficientType=i32,
-                             coefficientModulus=4294967291,
+                             coefficientModulus=4294967291:i32,
                              polynomialModulus=#poly_mod>
 
     %0 = ... : polynomial.polynomial<#ring>
@@ -123,19 +145,24 @@ def Polynomial_RingAttr : Polynomial_Attr<"Ring", "ring"> {
   let parameters = (ins
     "Type": $coefficientType,
     OptionalParameter<"::mlir::IntegerAttr">: $coefficientModulus,
-    OptionalParameter<"::mlir::polynomial::PolynomialAttr">: $polynomialModulus,
+    OptionalParameter<"::mlir::polynomial::IntPolynomialAttr">: $polynomialModulus,
     OptionalParameter<"::mlir::IntegerAttr">: $primitiveRoot
   );
-
+  let assemblyFormat = "`<` struct(params) `>`";
   let builders = [
-    AttrBuilder<
+    AttrBuilderWithInferredContext<
         (ins "::mlir::Type":$coefficientTy,
-             "::mlir::IntegerAttr":$coefficientModulusAttr,
-             "::mlir::polynomial::PolynomialAttr":$polynomialModulusAttr), [{
-      return $_get($_ctxt, coefficientTy, coefficientModulusAttr, polynomialModulusAttr, nullptr);
-    }]>
+              CArg<"::mlir::IntegerAttr", "nullptr"> :$coefficientModulusAttr,
+              CArg<"::mlir::polynomial::IntPolynomialAttr", "nullptr"> :$polynomialModulusAttr,
+              CArg<"::mlir::IntegerAttr", "nullptr"> :$primitiveRootAttr), [{
+      return $_get(
+        coefficientTy.getContext(),
+        coefficientTy,
+        coefficientModulusAttr,
+        polynomialModulusAttr,
+        primitiveRootAttr);
+    }]>,
   ];
-  let hasCustomAssemblyFormat = 1;
 }
 
 class Polynomial_Type<string name, string typeMnemonic>
@@ -149,7 +176,7 @@ def Polynomial_PolynomialType : Polynomial_Type<"Polynomial", "polynomial"> {
     A type for polynomials in a polynomial quotient ring.
   }];
   let parameters = (ins Polynomial_RingAttr:$ring);
-  let assemblyFormat = "`<` $ring `>`";
+  let assemblyFormat = "`<` struct(params) `>`";
 }
 
 def PolynomialLike: TypeOrContainer<Polynomial_PolynomialType, "polynomial-like">;
@@ -187,10 +214,10 @@ def Polynomial_AddOp : Polynomial_BinaryOp<"add", [Commutative]> {
 
     ```mlir
     // add two polynomials modulo x^1024 - 1
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
-    %1 = polynomial.constant #polynomial.polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    %1 = polynomial.constant #polynomial.int_polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
     %2 = polynomial.add %0, %1 : !polynomial.polynomial<#ring>
     ```
   }];
@@ -211,10 +238,10 @@ def Polynomial_SubOp : Polynomial_BinaryOp<"sub"> {
 
     ```mlir
     // subtract two polynomials modulo x^1024 - 1
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
-    %1 = polynomial.constant #polynomial.polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    %1 = polynomial.constant #polynomial.int_polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
     %2 = polynomial.sub %0, %1 : !polynomial.polynomial<#ring>
     ```
   }];
@@ -235,10 +262,10 @@ def Polynomial_MulOp : Polynomial_BinaryOp<"mul", [Commutative]> {
 
     ```mlir
     // multiply two polynomials modulo x^1024 - 1
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
-    %1 = polynomial.constant #polynomial.polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    %1 = polynomial.constant #polynomial.int_polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
     %2 = polynomial.mul %0, %1 : !polynomial.polynomial<#ring>
     ```
   }];
@@ -260,9 +287,9 @@ def Polynomial_MulScalarOp : Polynomial_Op<"mul_scalar", [
 
     ```mlir
     // multiply two polynomials modulo x^1024 - 1
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
     %1 = arith.constant 3 : i32
     %2 = polynomial.mul_scalar %0, %1 : !polynomial.polynomial<#ring>, i32
     ```
@@ -291,9 +318,9 @@ def Polynomial_LeadingTermOp: Polynomial_Op<"leading_term"> {
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
     %1, %2 = polynomial.leading_term %0 : !polynomial.polynomial<#ring> -> (index, i32)
     ```
   }];
@@ -314,8 +341,8 @@ def Polynomial_MonomialOp: Polynomial_Op<"monomial"> {
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
     %deg = arith.constant 1023 : index
     %five = arith.constant 5 : i32
     %0 = polynomial.monomial %five, %deg : (i32, index) -> !polynomial.polynomial<#ring>
@@ -354,8 +381,8 @@ def Polynomial_FromTensorOp : Polynomial_Op<"from_tensor", [Pure]> {
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
     %two = arith.constant 2 : i32
     %five = arith.constant 5 : i32
     %coeffs = tensor.from_elements %two, %two, %five : tensor<3xi32>
@@ -393,8 +420,8 @@ def Polynomial_ToTensorOp : Polynomial_Op<"to_tensor", [Pure]> {
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
     %two = arith.constant 2 : i32
     %five = arith.constant 5 : i32
     %coeffs = tensor.from_elements %two, %two, %five : tensor<3xi32>
@@ -405,24 +432,32 @@ def Polynomial_ToTensorOp : Polynomial_Op<"to_tensor", [Pure]> {
   let arguments = (ins Polynomial_PolynomialType:$input);
   let results = (outs RankedTensorOf<[AnyInteger]>:$output);
   let assemblyFormat = "$input attr-dict `:` type($input) `->` type($output)";
-
   let hasVerifier = 1;
 }
 
-def Polynomial_ConstantOp : Polynomial_Op<"constant", [Pure]> {
+def Polynomial_AnyPolynomialAttr : AnyAttrOf<[
+  Polynomial_FloatPolynomialAttr,
+  Polynomial_IntPolynomialAttr
+]>;
+
+// Not deriving from Polynomial_Op due to need for custom assembly format
+def Polynomial_ConstantOp : Op<Polynomial_Dialect, "constant", [Pure]> {
   let summary = "Define a constant polynomial via an attribute.";
   let description = [{
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+
+    #float_ring = #polynomial.ring<coefficientType=f32>
+    %0 = polynomial.constant #polynomial.float_polynomial<0.5 + 1.3e06 x**2> : !polynomial.polynomial<#float_ring>
     ```
   }];
-  let arguments = (ins Polynomial_PolynomialAttr:$input);
+  let arguments = (ins Polynomial_AnyPolynomialAttr:$value);
   let results = (outs Polynomial_PolynomialType:$output);
-  let assemblyFormat = "$input attr-dict `:` type($output)";
+  let assemblyFormat = "attr-dict `:` type($output)";
 }
 
 def Polynomial_NTTOp : Polynomial_Op<"ntt", [Pure]> {
diff --git a/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp b/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
index 5916ffba78e2..42e678fad060 100644
--- a/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
+++ b/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
@@ -9,87 +9,63 @@
 #include "mlir/Dialect/Polynomial/IR/Polynomial.h"
 
 #include "mlir/Support/LogicalResult.h"
-#include "llvm/ADT/APInt.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Support/raw_ostream.h"
 
 namespace mlir {
 namespace polynomial {
 
-FailureOr<Polynomial> Polynomial::fromMonomials(ArrayRef<Monomial> monomials) {
+template <typename T>
+MonomialBase<T>::~MonomialBase() {}
+
+template <typename PolyT, typename MonomialT>
+FailureOr<PolyT> fromMonomialsImpl(ArrayRef<MonomialT> monomials) {
   // A polynomial's terms are canonically stored in order of increasing degree.
-  auto monomialsCopy = llvm::SmallVector<Monomial>(monomials);
+  auto monomialsCopy = llvm::SmallVector<MonomialT>(monomials);
   std::sort(monomialsCopy.begin(), monomialsCopy.end());
 
   // Ensure non-unique exponents are not present. Since we sorted the list by
   // exponent, a linear scan of adjancent monomials suffices.
   if (std::adjacent_find(monomialsCopy.begin(), monomialsCopy.end(),
-                         [](const Monomial &lhs, const Monomial &rhs) {
-                           return lhs.exponent == rhs.exponent;
+                         [](const MonomialT &lhs, const MonomialT &rhs) {
+                           return lhs.getExponent() == rhs.getExponent();
                          }) != monomialsCopy.end()) {
     return failure();
   }
 
-  return Polynomial(monomialsCopy);
+  return PolyT(monomialsCopy);
+}
+
+FailureOr<IntPolynomial>
+IntPolynomial::fromMonomials(ArrayRef<IntMonomial> monomials) {
+  return fromMonomialsImpl<IntPolynomial, IntMonomial>(monomials);
+}
+
+FailureOr<FloatPolynomial>
+FloatPolynomial::fromMonomials(ArrayRef<FloatMonomial> monomials) {
+  return fromMonomialsImpl<FloatPolynomial, FloatMonomial>(monomials);
 }
 
-Polynomial Polynomial::fromCoefficients(ArrayRef<int64_t> coeffs) {
-  llvm::SmallVector<Monomial> monomials;
+template <typename PolyT, typename MonomialT, typename CoeffT>
+PolyT fromCoefficientsImpl(ArrayRef<CoeffT> coeffs) {
+  llvm::SmallVector<MonomialT> monomials;
   auto size = coeffs.size();
   monomials.reserve(size);
   for (size_t i = 0; i < size; i++) {
     monomials.emplace_back(coeffs[i], i);
   }
-  auto result = Polynomial::fromMonomials(monomials);
+  auto result = PolyT::fromMonomials(monomials);
   // Construction guarantees unique exponents, so the failure mode of
   // fromMonomials can be bypassed.
   assert(succeeded(result));
   return result.value();
 }
 
-void Polynomial::print(raw_ostream &os, ::llvm::StringRef separator,
-                       ::llvm::StringRef exponentiation) const {
-  bool first = true;
-  for (const Monomial &term : terms) {
-    if (first) {
-      first = false;
-    } else {
-      os << separator;
-    }
-    std::string coeffToPrint;
-    if (term.coefficient == 1 && term.exponent.uge(1)) {
-      coeffToPrint = "";
-    } else {
-      llvm::SmallString<16> coeffString;
-      term.coefficient.toStringSigned(coeffString);
-      coeffToPrint = coeffString.str();
-    }
-
-    if (term.exponent == 0) {
-      os << coeffToPrint;
-    } else if (term.exponent == 1) {
-      os << coeffToPrint << "x";
-    } else {
-      llvm::SmallString<16> expString;
-      term.exponent.toStringSigned(expString);
-      os << coeffToPrint << "x" << exponentiation << expString;
-    }
-  }
-}
-
-void Polynomial::print(raw_ostream &os) const { print(os, " + ", "**"); }
-
-std::string Polynomial::toIdentifier() const {
-  std::string result;
-  llvm::raw_string_ostream os(result);
-  print(os, "_", "");
-  return os.str();
+IntPolynomial IntPolynomial::fromCoefficients(ArrayRef<int64_t> coeffs) {
+  return fromCoefficientsImpl<IntPolynomial, IntMonomial, int64_t>(coeffs);
 }
 
-unsigned Polynomial::getDegree() const {
-  return terms.back().exponent.getZExtValue();
+FloatPolynomial FloatPolynomial::fromCoefficients(ArrayRef<double> coeffs) {
+  return fromCoefficientsImpl<FloatPolynomial, FloatMonomial, double>(coeffs);
 }
 
 } // namespace polynomial
diff --git a/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp b/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
index 236bb7896635..890ce5226c30 100644
--- a/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
+++ b/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/Polynomial/IR/Polynomial.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
@@ -17,22 +18,31 @@
 namespace mlir {
 namespace polynomial {
 
-void PolynomialAttr::print(AsmPrinter &p) const {
-  p << '<';
-  p << getPolynomial();
-  p << '>';
+void IntPolynomialAttr::print(AsmPrinter &p) const {
+  p << '<' << getPolynomial() << '>';
 }
 
+void FloatPolynomialAttr::print(AsmPrinter &p) const {
+  p << '<' << getPolynomial() << '>';
+}
+
+/// A callable that parses the coefficient using the appropriate method for the
+/// given monomial type, and stores the parsed coefficient value on the
+/// monomial.
+template <typename MonomialType>
+using ParseCoefficientFn = std::function<OptionalParseResult(MonomialType &)>;
+
 /// Try to parse a monomial. If successful, populate the fields of the outparam
 /// `monomial` with the results, and the `variable` outparam with the parsed
 /// variable name. Sets shouldParseMore to true if the monomial is followed by
 /// a '+'.
-ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
-                          llvm::StringRef &variable, bool &isConstantTerm,
-                          bool &shouldParseMore) {
-  APInt parsedCoeff(apintBitWidth, 1);
-  auto parsedCoeffResult = parser.parseOptionalInteger(parsedCoeff);
-  monomial.coefficient = parsedCoeff;
+///
+template <typename Monomial>
+ParseResult
+parseMonomial(AsmParser &parser, Monomial &monomial, llvm::StringRef &variable,
+              bool &isConstantTerm, bool &shouldParseMore,
+              ParseCoefficientFn<Monomial> parseAndStoreCoefficient) {
+  OptionalParseResult parsedCoeffResult = parseAndStoreCoefficient(monomial);
 
   isConstantTerm = false;
   shouldParseMore = false;
@@ -44,7 +54,7 @@ ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
     if (!parsedCoeffResult.has_value()) {
       return failure();
     }
-    monomial.exponent = APInt(apintBitWidth, 0);
+    monomial.setExponent(APInt(apintBitWidth, 0));
     isConstantTerm = true;
     shouldParseMore = true;
     return success();
@@ -58,7 +68,7 @@ ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
       return failure();
     }
 
-    monomial.exponent = APInt(apintBitWidth, 0);
+    monomial.setExponent(APInt(apintBitWidth, 0));
     isConstantTerm = true;
     return success();
   }
@@ -80,9 +90,9 @@ ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
       return failure();
     }
 
-    monomial.exponent = parsedExponent;
+    monomial.setExponent(parsedExponent);
   } else {
-    monomial.exponent = APInt(apintBitWidth, 1);
+    monomial.setExponent(APInt(apintBitWidth, 1));
   }
 
   if (succeeded(parser.parseOptionalPlus())) {
@@ -91,22 +101,21 @@ ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
   return success();
 }
 
-Attribute PolynomialAttr::parse(AsmParser &parser, Type type) {
-  if (failed(parser.parseLess()))
-    return {};
-
-  llvm::SmallVector<Monomial> monomials;
-  llvm::StringSet<> variables;
-
+template <typename PolynoimalAttrTy, typename Monomial>
+LogicalResult
+parsePolynomialAttr(AsmParser &parser, llvm::SmallVector<Monomial> &monomials,
+                    llvm::StringSet<> &variables,
+                    ParseCoefficientFn<Monomial> parseAndStoreCoefficient) {
   while (true) {
     Monomial parsedMonomial;
     llvm::StringRef parsedVariableRef;
     bool isConstantTerm;
     bool shouldParseMore;
-    if (failed(parseMonomial(parser, parsedMonomial, parsedVariableRef,
-                             isConstantTerm, shouldParseMore))) {
+    if (failed(parseMonomial<Monomial>(
+            parser, parsedMonomial, parsedVariableRef, isConstantTerm,
+            shouldParseMore, parseAndStoreCoefficient))) {
       parser.emitError(parser.getCurrentLocation(), "expected a monomial");
-      return {};
+      return failure();
     }
 
     if (!isConstantTerm) {
@@ -124,7 +133,7 @@ Attribute PolynomialAttr::parse(AsmParser &parser, Type type) {
     parser.emitError(
         parser.getCurrentLocation(),
         "expected + and more monomials, or > to end polynomial attribute");
-    return {};
+    return failure();
   }
 
   if (variables.size() > 1) {
@@ -133,96 +142,67 @@ Attribute PolynomialAttr::parse(AsmParser &parser, Type type) {
         parser.getCurrentLocation(),
         "polynomials must have one indeterminate, but there were multiple: " +
             vars);
+    return failure();
   }
 
-  auto result = Polynomial::fromMonomials(monomials);
-  if (failed(result)) {
-    parser.emitError(parser.getCurrentLocation())
-        << "parsed polynomial must have unique exponents among monomials";
-    return {};
-  }
-  return PolynomialAttr::get(parser.getContext(), result.value());
-}
-
-void RingAttr::print(AsmPrinter &p) const {
-  p << "#polynomial.ring<coefficientType=" << getCoefficientType()
-    << ", coefficientModulus=" << getCoefficientModulus()
-    << ", polynomialModulus=" << getPolynomialModulus() << '>';
+  return success();
 }
 
-Attribute RingAttr::parse(AsmParser &parser, Type type) {
+Attribute IntPolynomialAttr::parse(AsmParser &parser, Type type) {
   if (failed(parser.parseLess()))
     return {};
 
-  if (failed(parser.parseKeyword("coefficientType")))
-    return {};
+  llvm::SmallVector<IntMonomial> monomials;
+  llvm::StringSet<> variables;
 
-  if (failed(parser.parseEqual()))
+  if (failed(parsePolynomialAttr<IntPolynomialAttr, IntMonomial>(
+          parser, monomials, variables,
+          [&](IntMonomial &monomial) -> OptionalParseResult {
+            APInt parsedCoeff(apintBitWidth, 1);
+            OptionalParseResult result =
+                parser.parseOptionalInteger(parsedCoeff);
+            monomial.setCoefficient(parsedCoeff);
+            return result;
+          }))) {
     return {};
+  }
 
-  Type ty;
-  if (failed(parser.parseType(ty)))
+  auto result = IntPolynomial::fromMonomials(monomials);
+  if (failed(result)) {
+    parser.emitError(parser.getCurrentLocation())
+        << "parsed polynomial must have unique exponents among monomials";
     return {};
+  }
+  return IntPolynomialAttr::get(parser.getContext(), result.value());
+}
 
-  if (failed(parser.parseComma()))
+Attribute FloatPolynomialAttr::parse(AsmParser &parser, Type type) {
+  if (failed(parser.parseLess()))
     return {};
 
-  IntegerAttr coefficientModulusAttr = nullptr;
-  if (succeeded(parser.parseKeyword("coefficientModulus"))) {
-    if (failed(parser.parseEqual()))
-      return {};
-
-    IntegerType iType = mlir::dyn_cast<IntegerType>(ty);
-    if (!iType) {
-      parser.emitError(parser.getCurrentLocation(),
-                       "coefficientType must specify an integer type");
-      return {};
-    }
-    APInt coefficientModulus(iType.getWidth(), 0);
-    auto result = parser.parseInteger(coefficientModulus);
-    if (failed(result)) {
-      parser.emitError(parser.getCurrentLocation(),
-                       "invalid coefficient modulus");
-      return {};
-    }
-    coefficientModulusAttr = IntegerAttr::get(iType, coefficientModulus);
-
-    if (failed(parser.parseComma()))
-      return {};
-  }
-
-  PolynomialAttr polyAttr = nullptr;
-  if (succeeded(parser.parseKeyword("polynomialModulus"))) {
-    if (failed(parser.parseEqual()))
-      return {};
+  llvm::SmallVector<FloatMonomial> monomials;
+  llvm::StringSet<> variables;
 
-    PolynomialAttr attr;
-    if (failed(parser.parseAttribute<PolynomialAttr>(attr)))
-      return {};
-    polyAttr = attr;
-  }
+  ParseCoefficientFn<FloatMonomial> parseAndStoreCoefficient =
+      [&](FloatMonomial &monomial) -> OptionalParseResult {
+    double coeffValue = 1.0;
+    ParseResult result = parser.parseFloat(coeffValue);
+    monomial.setCoefficient(APFloat(coeffValue));
+    return OptionalParseResult(result);
+  };
 
-  Polynomial poly = polyAttr.getPolynomial();
-  APInt root(coefficientModulusAttr.getValue().getBitWidth(), 0);
-  IntegerAttr rootAttr = nullptr;
-  if (succeeded(parser.parseOptionalComma())) {
-    if (failed(parser.parseKeyword("primitiveRoot")) ||
-        failed(parser.parseEqual()))
-      return {};
-
-    ParseResult result = parser.parseInteger(root);
-    if (failed(result)) {
-      parser.emitError(parser.getCurrentLocation(), "invalid primitiveRoot");
-      return {};
-    }
-    rootAttr = IntegerAttr::get(coefficientModulusAttr.getType(), root);
+  if (failed(parsePolynomialAttr<FloatPolynomialAttr, FloatMonomial>(
+          parser, monomials, variables, parseAndStoreCoefficient))) {
+    return {};
   }
 
-  if (failed(parser.parseGreater()))
+  auto result = FloatPolynomial::fromMonomials(monomials);
+  if (failed(result)) {
+    parser.emitError(parser.getCurrentLocation())
+        << "parsed polynomial must have unique exponents among monomials";
     return {};
-
-  return RingAttr::get(parser.getContext(), ty, coefficientModulusAttr,
-                       polyAttr, rootAttr);
+  }
+  return FloatPolynomialAttr::get(parser.getContext(), result.value());
 }
 
 } // namespace polynomial
diff --git a/mlir/test/Dialect/Polynomial/attributes.mlir b/mlir/test/Dialect/Polynomial/attributes.mlir
index 3973ae394433..4bdfd44fd4d1 100644
--- a/mlir/test/Dialect/Polynomial/attributes.mlir
+++ b/mlir/test/Dialect/Polynomial/attributes.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s --split-input-file --verify-diagnostics
 
-#my_poly = #polynomial.polynomial<y + x**1024>
+#my_poly = #polynomial.int_polynomial<y + x**1024>
 // expected-error@below {{polynomials must have one indeterminate, but there were multiple: x, y}}
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
@@ -9,37 +9,31 @@
 // expected-error@below {{expected integer value}}
 // expected-error@below {{expected a monomial}}
 // expected-error@below {{found invalid integer exponent}}
-#my_poly = #polynomial.polynomial<5 + x**f>
+#my_poly = #polynomial.int_polynomial<5 + x**f>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 // -----
 
-#my_poly = #polynomial.polynomial<5 + x**2 + 3x**2>
+#my_poly = #polynomial.int_polynomial<5 + x**2 + 3x**2>
 // expected-error@below {{parsed polynomial must have unique exponents among monomials}}
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 // -----
 
 // expected-error@below {{expected + and more monomials, or > to end polynomial attribute}}
-#my_poly = #polynomial.polynomial<5 + x**2 7>
+#my_poly = #polynomial.int_polynomial<5 + x**2 7>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 // -----
 
 // expected-error@below {{expected a monomial}}
-#my_poly = #polynomial.polynomial<5 + x**2 +>
+#my_poly = #polynomial.int_polynomial<5 + x**2 +>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 
 // -----
 
-#my_poly = #polynomial.polynomial<5 + x**2>
-// expected-error@below {{coefficientType must specify an integer type}}
-#ring1 = #polynomial.ring<coefficientType=f64, coefficientModulus=2837465, polynomialModulus=#my_poly>
-
-// -----
-
-#my_poly = #polynomial.polynomial<5 + x**2>
-// expected-error@below {{expected integer value}}
-// expected-error@below {{invalid coefficient modulus}}
+#my_poly = #polynomial.int_polynomial<5 + x**2>
+// expected-error@below {{failed to parse Polynomial_RingAttr parameter 'coefficientModulus' which is to be a `::mlir::IntegerAttr`}}
+// expected-error@below {{expected attribute value}}
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=x, polynomialModulus=#my_poly>
diff --git a/mlir/test/Dialect/Polynomial/ops.mlir b/mlir/test/Dialect/Polynomial/ops.mlir
index a29cfc2e9cc5..ff709960c50e 100644
--- a/mlir/test/Dialect/Polynomial/ops.mlir
+++ b/mlir/test/Dialect/Polynomial/ops.mlir
@@ -2,85 +2,87 @@
 
 // This simply tests for syntax.
 
-#my_poly = #polynomial.polynomial<1 + x**1024>
-#my_poly_2 = #polynomial.polynomial<2>
-#my_poly_3 = #polynomial.polynomial<3x>
-#my_poly_4 = #polynomial.polynomial<t**3 + 4t + 2>
+#my_poly = #polynomial.int_polynomial<1 + x**1024>
+#my_poly_2 = #polynomial.int_polynomial<2>
+#my_poly_3 = #polynomial.int_polynomial<3x>
+#my_poly_4 = #polynomial.int_polynomial<t**3 + 4t + 2>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
-#one_plus_x_squared = #polynomial.polynomial<1 + x**2>
+#ring2 = #polynomial.ring<coefficientType=f32>
+#one_plus_x_squared = #polynomial.int_polynomial<1 + x**2>
 
-#ideal = #polynomial.polynomial<-1 + x**1024>
+#ideal = #polynomial.int_polynomial<-1 + x**1024>
 #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#ideal, primitiveRoot=193>
-!poly_ty = !polynomial.polynomial<#ring>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
-#ntt_poly = #polynomial.polynomial<-1 + x**8>
+#ntt_poly = #polynomial.int_polynomial<-1 + x**8>
 #ntt_ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#ntt_poly, primitiveRoot=31>
-!ntt_poly_ty = !polynomial.polynomial<#ntt_ring>
+!ntt_poly_ty = !polynomial.polynomial<ring=#ntt_ring>
 
 module {
-  func.func @test_multiply() -> !polynomial.polynomial<#ring1> {
+  func.func @test_multiply() -> !polynomial.polynomial<ring=#ring1> {
     %c0 = arith.constant 0 : index
     %two = arith.constant 2 : i16
     %five = arith.constant 5 : i16
     %coeffs1 = tensor.from_elements %two, %two, %five : tensor<3xi16>
     %coeffs2 = tensor.from_elements %five, %five, %two : tensor<3xi16>
 
-    %poly1 = polynomial.from_tensor %coeffs1 : tensor<3xi16> -> !polynomial.polynomial<#ring1>
-    %poly2 = polynomial.from_tensor %coeffs2 : tensor<3xi16> -> !polynomial.polynomial<#ring1>
+    %poly1 = polynomial.from_tensor %coeffs1 : tensor<3xi16> -> !polynomial.polynomial<ring=#ring1>
+    %poly2 = polynomial.from_tensor %coeffs2 : tensor<3xi16> -> !polynomial.polynomial<ring=#ring1>
 
-    %3 = polynomial.mul %poly1, %poly2 : !polynomial.polynomial<#ring1>
+    %3 = polynomial.mul %poly1, %poly2 : !polynomial.polynomial<ring=#ring1>
 
-    return %3 : !polynomial.polynomial<#ring1>
+    return %3 : !polynomial.polynomial<ring=#ring1>
   }
 
-  func.func @test_elementwise(%p0 : !polynomial.polynomial<#ring1>, %p1: !polynomial.polynomial<#ring1>) {
-    %tp0 = tensor.from_elements %p0, %p1 : tensor<2x!polynomial.polynomial<#ring1>>
-    %tp1 = tensor.from_elements %p1, %p0 : tensor<2x!polynomial.polynomial<#ring1>>
+  func.func @test_elementwise(%p0 : !polynomial.polynomial<ring=#ring1>, %p1: !polynomial.polynomial<ring=#ring1>) {
+    %tp0 = tensor.from_elements %p0, %p1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
+    %tp1 = tensor.from_elements %p1, %p0 : tensor<2x!polynomial.polynomial<ring=#ring1>>
 
     %c = arith.constant 2 : i32
-    %mul_const_sclr = polynomial.mul_scalar %tp0, %c : tensor<2x!polynomial.polynomial<#ring1>>, i32
+    %mul_const_sclr = polynomial.mul_scalar %tp0, %c : tensor<2x!polynomial.polynomial<ring=#ring1>>, i32
 
-    %add = polynomial.add %tp0, %tp1 : tensor<2x!polynomial.polynomial<#ring1>>
-    %sub = polynomial.sub %tp0, %tp1 : tensor<2x!polynomial.polynomial<#ring1>>
-    %mul = polynomial.mul %tp0, %tp1 : tensor<2x!polynomial.polynomial<#ring1>>
+    %add = polynomial.add %tp0, %tp1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
+    %sub = polynomial.sub %tp0, %tp1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
+    %mul = polynomial.mul %tp0, %tp1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
 
     return
   }
 
-  func.func @test_to_from_tensor(%p0 : !polynomial.polynomial<#ring1>) {
+  func.func @test_to_from_tensor(%p0 : !polynomial.polynomial<ring=#ring1>) {
     %c0 = arith.constant 0 : index
     %two = arith.constant 2 : i16
     %coeffs1 = tensor.from_elements %two, %two : tensor<2xi16>
     // CHECK: from_tensor
-    %poly = polynomial.from_tensor %coeffs1 : tensor<2xi16> -> !polynomial.polynomial<#ring1>
+    %poly = polynomial.from_tensor %coeffs1 : tensor<2xi16> -> !polynomial.polynomial<ring=#ring1>
     // CHECK: to_tensor
-    %tensor = polynomial.to_tensor %poly : !polynomial.polynomial<#ring1> -> tensor<1024xi16>
+    %tensor = polynomial.to_tensor %poly : !polynomial.polynomial<ring=#ring1> -> tensor<1024xi16>
 
     return
   }
 
-  func.func @test_degree(%p0 : !polynomial.polynomial<#ring1>) {
-    %0, %1 = polynomial.leading_term %p0 : !polynomial.polynomial<#ring1> -> (index, i32)
+  func.func @test_degree(%p0 : !polynomial.polynomial<ring=#ring1>) {
+    %0, %1 = polynomial.leading_term %p0 : !polynomial.polynomial<ring=#ring1> -> (index, i32)
     return
   }
 
   func.func @test_monomial() {
     %deg = arith.constant 1023 : index
     %five = arith.constant 5 : i16
-    %0 = polynomial.monomial %five, %deg : (i16, index) -> !polynomial.polynomial<#ring1>
+    %0 = polynomial.monomial %five, %deg : (i16, index) -> !polynomial.polynomial<ring=#ring1>
     return
   }
 
   func.func @test_monic_monomial_mul() {
     %five = arith.constant 5 : index
-    %0 = polynomial.constant #one_plus_x_squared : !polynomial.polynomial<#ring1>
-    %1 = polynomial.monic_monomial_mul %0, %five : (!polynomial.polynomial<#ring1>, index) -> !polynomial.polynomial<#ring1>
+    %0 = polynomial.constant {value=#one_plus_x_squared} : !polynomial.polynomial<ring=#ring1>
+    %1 = polynomial.monic_monomial_mul %0, %five : (!polynomial.polynomial<ring=#ring1>, index) -> !polynomial.polynomial<ring=#ring1>
     return
   }
 
   func.func @test_constant() {
-    %0 = polynomial.constant #one_plus_x_squared : !polynomial.polynomial<#ring1>
-    %1 = polynomial.constant <1 + x**2> : !polynomial.polynomial<#ring1>
+    %0 = polynomial.constant {value=#one_plus_x_squared} : !polynomial.polynomial<ring=#ring1>
+    %1 = polynomial.constant {value=#polynomial.int_polynomial<1 + x**2>} : !polynomial.polynomial<ring=#ring1>
+    %2 = polynomial.constant {value=#polynomial.float_polynomial<1.5 + 0.5 x**2>} : !polynomial.polynomial<ring=#ring2>
     return
   }
 
diff --git a/mlir/test/Dialect/Polynomial/ops_errors.mlir b/mlir/test/Dialect/Polynomial/ops_errors.mlir
index 2c20e7bcbf1d..af8e4aa5da86 100644
--- a/mlir/test/Dialect/Polynomial/ops_errors.mlir
+++ b/mlir/test/Dialect/Polynomial/ops_errors.mlir
@@ -1,8 +1,8 @@
 // RUN: mlir-opt --split-input-file --verify-diagnostics %s
 
-#my_poly = #polynomial.polynomial<1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i32, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<ring=#ring>
 
 func.func @test_from_tensor_too_large_coeffs() {
   %two = arith.constant 2 : i32
@@ -15,13 +15,13 @@ func.func @test_from_tensor_too_large_coeffs() {
 
 // -----
 
-#my_poly = #polynomial.polynomial<1 + x**4>
-#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<1 + x**4>
+#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256:i32, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<ring=#ring>
 func.func @test_from_tensor_wrong_tensor_type() {
   %two = arith.constant 2 : i32
   %coeffs1 = tensor.from_elements %two, %two, %two, %two, %two : tensor<5xi32>
-  // expected-error@below {{input type 'tensor<5xi32>' does not match output type '!polynomial.polynomial<#polynomial.ring<coefficientType=i32, coefficientModulus=256 : i32, polynomialModulus=#polynomial.polynomial<1 + x**4>>>'}}
+  // expected-error@below {{input type 'tensor<5xi32>' does not match output type '!polynomial.polynomial<ring = <coefficientType = i32, coefficientModulus = 256 : i32, polynomialModulus = <1 + x**4>>>'}}
   // expected-note@below {{at most the degree of the polynomialModulus of the output type's ring attribute}}
   %poly = polynomial.from_tensor %coeffs1 : tensor<5xi32> -> !ty
   return
@@ -29,11 +29,11 @@ func.func @test_from_tensor_wrong_tensor_type() {
 
 // -----
 
-#my_poly = #polynomial.polynomial<1 + x**4>
-#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<1 + x**4>
+#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256:i32, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<ring=#ring>
 func.func @test_to_tensor_wrong_output_tensor_type(%arg0 : !ty) {
-  // expected-error@below {{input type '!polynomial.polynomial<#polynomial.ring<coefficientType=i32, coefficientModulus=256 : i32, polynomialModulus=#polynomial.polynomial<1 + x**4>>>' does not match output type 'tensor<5xi32>'}}
+  // expected-error@below {{input type '!polynomial.polynomial<ring = <coefficientType = i32, coefficientModulus = 256 : i32, polynomialModulus = <1 + x**4>>>' does not match output type 'tensor<5xi32>'}}
   // expected-note@below {{at most the degree of the polynomialModulus of the input type's ring attribute}}
   %tensor = polynomial.to_tensor %arg0 : !ty -> tensor<5xi32>
   return
@@ -41,9 +41,9 @@ func.func @test_to_tensor_wrong_output_tensor_type(%arg0 : !ty) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i32, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<ring=#ring>
 
 func.func @test_mul_scalar_wrong_type(%arg0: !ty) -> !ty {
   %scalar = arith.constant 2 : i32  // should be i16
@@ -54,9 +54,9 @@ func.func @test_mul_scalar_wrong_type(%arg0: !ty) -> !ty {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=31>
-!poly_ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_ntt
 // CHECK-NOT: polynomial.ntt
@@ -68,9 +68,9 @@ func.func @test_invalid_ntt(%0 : !poly_ty) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=31>
-!poly_ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_ntt
 // CHECK-NOT: polynomial.ntt
@@ -82,10 +82,10 @@ func.func @test_invalid_ntt(%0 : !poly_ty) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
-#ring1 = #polynomial.ring<coefficientType=i16, coefficientModulus=257, polynomialModulus=#my_poly, primitiveRoot=31>
-!poly_ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly>
+#ring1 = #polynomial.ring<coefficientType=i16, coefficientModulus=257:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_intt
 // CHECK-NOT: polynomial.intt
@@ -97,9 +97,9 @@ func.func @test_invalid_intt(%0 : tensor<1024xi32, #ring1>) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=31>
-!poly_ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_intt
 // CHECK-NOT: polynomial.intt
@@ -112,9 +112,9 @@ func.func @test_invalid_intt(%0 : tensor<1025xi32, #ring>) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
-!poly_ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_ntt
 // CHECK-NOT: polynomial.ntt
@@ -126,10 +126,10 @@ func.func @test_invalid_ntt(%0 : !poly_ty) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**8>
+#my_poly = #polynomial.int_polynomial<-1 + x**8>
 // A valid root is 31
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=32>
-!poly_ty = !polynomial.polynomial<#ring>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=32:i16>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_intt
 // CHECK-NOT: polynomial.intt
diff --git a/mlir/test/Dialect/Polynomial/types.mlir b/mlir/test/Dialect/Polynomial/types.mlir
index 00296a36e890..dcc5663ceb84 100644
--- a/mlir/test/Dialect/Polynomial/types.mlir
+++ b/mlir/test/Dialect/Polynomial/types.mlir
@@ -2,13 +2,13 @@
 
 // CHECK-LABEL: func @test_types
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    #polynomial.ring<
-// CHECK-SAME:       coefficientType=i32,
-// CHECK-SAME:       coefficientModulus=2837465 : i32,
-// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<1 + x**1024>>>
-#my_poly = #polynomial.polynomial<1 + x**1024>
-#ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<#ring1>
+// CHECK-SAME:    ring = <
+// CHECK-SAME:       coefficientType = i32,
+// CHECK-SAME:       coefficientModulus = 2837465 : i32,
+// CHECK-SAME:       polynomialModulus = <1 + x**1024>>>
+#my_poly = #polynomial.int_polynomial<1 + x**1024>
+#ring1 = #polynomial.ring<coefficientType = i32, coefficientModulus = 2837465 : i32, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<ring=#ring1>
 func.func @test_types(%0: !ty) -> !ty {
   return %0 : !ty
 }
@@ -16,13 +16,13 @@ func.func @test_types(%0: !ty) -> !ty {
 
 // CHECK-LABEL: func @test_non_x_variable_64_bit
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    #polynomial.ring<
-// CHECK-SAME:       coefficientType=i64,
-// CHECK-SAME:       coefficientModulus=2837465 : i64,
-// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<2 + 4x + x**3>>>
-#my_poly_2 = #polynomial.polynomial<t**3 + 4t + 2>
-#ring2 = #polynomial.ring<coefficientType=i64, coefficientModulus=2837465, polynomialModulus=#my_poly_2>
-!ty2 = !polynomial.polynomial<#ring2>
+// CHECK-SAME:    ring = <
+// CHECK-SAME:       coefficientType = i64,
+// CHECK-SAME:       coefficientModulus = 2837465 : i64,
+// CHECK-SAME:       polynomialModulus = <2 + 4x + x**3>>>
+#my_poly_2 = #polynomial.int_polynomial<t**3 + 4t + 2>
+#ring2 = #polynomial.ring<coefficientType = i64, coefficientModulus = 2837465 : i64, polynomialModulus=#my_poly_2>
+!ty2 = !polynomial.polynomial<ring=#ring2>
 func.func @test_non_x_variable_64_bit(%0: !ty2) -> !ty2 {
   return %0 : !ty2
 }
@@ -30,27 +30,36 @@ func.func @test_non_x_variable_64_bit(%0: !ty2) -> !ty2 {
 
 // CHECK-LABEL: func @test_linear_poly
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    #polynomial.ring<
-// CHECK-SAME:       coefficientType=i32,
-// CHECK-SAME:       coefficientModulus=12 : i32,
-// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<4x>>
-#my_poly_3 = #polynomial.polynomial<4x>
-#ring3 = #polynomial.ring<coefficientType=i32, coefficientModulus=12, polynomialModulus=#my_poly_3>
-!ty3 = !polynomial.polynomial<#ring3>
+// CHECK-SAME:    ring = <
+// CHECK-SAME:       coefficientType = i32,
+// CHECK-SAME:       coefficientModulus = 12 : i32,
+// CHECK-SAME:       polynomialModulus = <4x>>
+#my_poly_3 = #polynomial.int_polynomial<4x>
+#ring3 = #polynomial.ring<coefficientType = i32, coefficientModulus=12 : i32, polynomialModulus=#my_poly_3>
+!ty3 = !polynomial.polynomial<ring=#ring3>
 func.func @test_linear_poly(%0: !ty3) -> !ty3 {
   return %0 : !ty3
 }
 
 // CHECK-LABEL: func @test_negative_leading_1
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    #polynomial.ring<
-// CHECK-SAME:       coefficientType=i32,
-// CHECK-SAME:       coefficientModulus=2837465 : i32,
-// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<-1 + x**1024>>>
-#my_poly_4 = #polynomial.polynomial<-1 + x**1024>
-#ring4 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly_4>
-!ty4 = !polynomial.polynomial<#ring4>
+// CHECK-SAME:    ring = <
+// CHECK-SAME:       coefficientType = i32,
+// CHECK-SAME:       coefficientModulus = 2837465 : i32,
+// CHECK-SAME:       polynomialModulus = <-1 + x**1024>>>
+#my_poly_4 = #polynomial.int_polynomial<-1 + x**1024>
+#ring4 = #polynomial.ring<coefficientType = i32, coefficientModulus = 2837465 : i32, polynomialModulus=#my_poly_4>
+!ty4 = !polynomial.polynomial<ring=#ring4>
 func.func @test_negative_leading_1(%0: !ty4) -> !ty4 {
   return %0 : !ty4
 }
 
+// CHECK-LABEL: func @test_float_coefficients
+// CHECK-SAME:  !polynomial.polynomial<ring = <coefficientType = f32>>
+#my_poly_5 = #polynomial.float_polynomial<0.5 + 1.6e03 x**1024>
+#ring5 = #polynomial.ring<coefficientType=f32>
+!ty5 = !polynomial.polynomial<ring=#ring5>
+func.func @test_float_coefficients(%0: !ty5) -> !ty5 {
+  return %0 : !ty5
+}
+
-- 
GitLab


From 6140b5bae475069f958f90a81fb9d69c969daab6 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Mon, 13 May 2024 09:48:46 -0700
Subject: [PATCH 0600/1206] [RISCV] Use RISCVISD::SHL_ADD in transformAddShlImm
 (#89832)

Doing so avoids negative interactions with other combines which don't
know the shl_add is a single instruction. From the commit log, we've had
several combine loops already.

This was originally posted as part of #88791, where a bug was pointed
out. That bug was fixed by #89789 which hits the same issue from another
angle. To confirm the fix, I included the reduced test case here.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 18 +++++++-------
 llvm/test/CodeGen/RISCV/addimm-mulimm.ll    | 26 +++++++++++++++++++++
 2 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index d0f62b1d5414..5a84ad4d436b 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12987,10 +12987,9 @@ static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
   SDLoc DL(N);
   SDValue NS = (C0 < C1) ? N0->getOperand(0) : N1->getOperand(0);
   SDValue NL = (C0 > C1) ? N0->getOperand(0) : N1->getOperand(0);
-  SDValue NA0 =
-      DAG.getNode(ISD::SHL, DL, VT, NL, DAG.getConstant(Diff, DL, VT));
-  SDValue NA1 = DAG.getNode(ISD::ADD, DL, VT, NA0, NS);
-  return DAG.getNode(ISD::SHL, DL, VT, NA1, DAG.getConstant(Bits, DL, VT));
+  SDValue SHADD = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, NL,
+                              DAG.getConstant(Diff, DL, VT), NS);
+  return DAG.getNode(ISD::SHL, DL, VT, SHADD, DAG.getConstant(Bits, DL, VT));
 }
 
 // Combine a constant select operand into its use:
@@ -13226,14 +13225,17 @@ static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) {
                      N0.getOperand(0));
 }
 
-static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
+static SDValue performADDCombine(SDNode *N,
+                                 TargetLowering::DAGCombinerInfo &DCI,
                                  const RISCVSubtarget &Subtarget) {
+  SelectionDAG &DAG = DCI.DAG;
   if (SDValue V = combineAddOfBooleanXor(N, DAG))
     return V;
   if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
     return V;
-  if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
-    return V;
+  if (!DCI.isBeforeLegalize() && !DCI.isCalledByLegalizer())
+    if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
+      return V;
   if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
     return V;
   if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
@@ -16230,7 +16232,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
       return V;
     if (SDValue V = combineToVWMACC(N, DAG, Subtarget))
       return V;
-    return performADDCombine(N, DAG, Subtarget);
+    return performADDCombine(N, DCI, Subtarget);
   }
   case ISD::SUB: {
     if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget))
diff --git a/llvm/test/CodeGen/RISCV/addimm-mulimm.ll b/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
index e2f7be2e6d7f..a18526718461 100644
--- a/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
+++ b/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
@@ -944,3 +944,29 @@ define i1 @pr53831(i32 %x) {
   %tmp5 = icmp eq i32 %tmp4, %tmp2
   ret i1 %tmp5
 }
+
+define i64 @sh2add_uw(i64 signext %0, i32 signext %1) {
+; RV32IMB-LABEL: sh2add_uw:
+; RV32IMB:       # %bb.0: # %entry
+; RV32IMB-NEXT:    srli a3, a2, 27
+; RV32IMB-NEXT:    slli a2, a2, 5
+; RV32IMB-NEXT:    srli a4, a0, 29
+; RV32IMB-NEXT:    sh3add a1, a1, a4
+; RV32IMB-NEXT:    sh3add a0, a0, a2
+; RV32IMB-NEXT:    sltu a2, a0, a2
+; RV32IMB-NEXT:    add a1, a3, a1
+; RV32IMB-NEXT:    add a1, a1, a2
+; RV32IMB-NEXT:    ret
+;
+; RV64IMB-LABEL: sh2add_uw:
+; RV64IMB:       # %bb.0: # %entry
+; RV64IMB-NEXT:    sh2add.uw a0, a1, a0
+; RV64IMB-NEXT:    slli a0, a0, 3
+; RV64IMB-NEXT:    ret
+entry:
+  %2 = zext i32 %1 to i64
+  %3 = shl i64 %2, 5
+  %4 = shl i64 %0, 3
+  %5 = add i64 %3, %4
+  ret i64 %5
+}
-- 
GitLab


From 37ffbbb19576a884c5bb93b9ac0ae97f89523b6b Mon Sep 17 00:00:00 2001
From: Peiming Liu <peiming@google.com>
Date: Mon, 13 May 2024 09:53:15 -0700
Subject: [PATCH 0601/1206] [mlir][tensor][sparse] don't drop encoding when
 infer result type (#91817)

A general question is: is it possible to support hooks here to infer the
encoding? E.g., when the extracted tensor slice is rank-reduced, the
encoding need to be updated accordingly as well.
---
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      |  3 ++-
 .../Dialect/SparseTensor/canonicalize.mlir    | 23 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 mlir/test/Dialect/SparseTensor/canonicalize.mlir

diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 1f94397e823f..e41d59a0e0b9 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -2020,7 +2020,8 @@ RankedTensorType ExtractSliceOp::inferResultType(
   assert(static_cast<int64_t>(staticSizes.size()) ==
              sourceTensorType.getRank() &&
          "unexpected staticSizes not equal to rank of source");
-  return RankedTensorType::get(staticSizes, sourceTensorType.getElementType());
+  return RankedTensorType::get(staticSizes, sourceTensorType.getElementType(),
+                               sourceTensorType.getEncoding());
 }
 
 RankedTensorType ExtractSliceOp::inferResultType(
diff --git a/mlir/test/Dialect/SparseTensor/canonicalize.mlir b/mlir/test/Dialect/SparseTensor/canonicalize.mlir
new file mode 100644
index 000000000000..b1d3d7916c14
--- /dev/null
+++ b/mlir/test/Dialect/SparseTensor/canonicalize.mlir
@@ -0,0 +1,23 @@
+// RUN: mlir-opt %s -split-input-file -canonicalize="test-convergence" | FileCheck %s
+
+#BCOO = #sparse_tensor.encoding<{
+  map = (d0, d1, d2) -> (d0 : dense, d1 : loose_compressed(nonunique), d2 : singleton)
+}>
+
+// CHECK-DAG: #[[$BCOO:.*]] = #sparse_tensor.encoding<{ map = (d0, d1, d2) -> (d0 : dense, d1 : loose_compressed(nonunique), d2 : singleton) }>
+// CHECK-LABEL: func @sparse_slice_canonicalize
+//  CHECK-SAME:   %[[ARG0:.+]]: tensor<?x?x?xf32, #[[$BCOO]]>
+//       CHECK:   %[[SLICE:.+]] = tensor.extract_slice %[[ARG0]][0, %{{[a-zA-Z0-9_]+}}, 1]
+//  CHECK-SAME:      [4, 1, %{{[a-zA-Z0-9_]+}}] [1, 1, 1]
+//  CHECK-SAME:      : tensor<?x?x?xf32, #[[$BCOO]]> to tensor<4x1x?xf32, #[[$BCOO]]>
+//       CHECK:   %[[RESULT:.+]] = tensor.cast %[[SLICE]]
+//       CHECK:   return %[[RESULT]]
+func.func @sparse_slice_canonicalize(%arg0 : tensor<?x?x?xf32, #BCOO>, %arg1 : index,
+    %arg2 : index) -> tensor<?x?x?xf32, #BCOO>
+{
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %0 = tensor.extract_slice %arg0[%c0, %arg1, %c1] [%c4, %c1, %arg2] [%c1, %c1, %c1] : tensor<?x?x?xf32, #BCOO> to tensor<?x?x?xf32, #BCOO>
+  return %0 : tensor<?x?x?xf32, #BCOO>
+}
-- 
GitLab


From 2a114d171d4c1e6c742d039a289bf509b86668ac Mon Sep 17 00:00:00 2001
From: Jover <joverzh@gmail.com>
Date: Tue, 14 May 2024 01:12:29 +0800
Subject: [PATCH 0602/1206] [clang-tidy] Ignore `if consteval` in
 else-after-return (#91588)

---
 .../readability/ElseAfterReturnCheck.cpp        |  4 ++--
 clang-tools-extra/docs/ReleaseNotes.rst         |  4 ++++
 .../else-after-return-if-consteval.cpp          | 17 +++++++++++++++++
 3 files changed, 23 insertions(+), 2 deletions(-)
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability/else-after-return-if-consteval.cpp

diff --git a/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp b/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp
index 1e85caf68835..2b185e7594ad 100644
--- a/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ElseAfterReturnCheck.cpp
@@ -113,7 +113,7 @@ static bool containsDeclInScope(const Stmt *Node) {
 }
 
 static void removeElseAndBrackets(DiagnosticBuilder &Diag, ASTContext &Context,
-                           const Stmt *Else, SourceLocation ElseLoc) {
+                                  const Stmt *Else, SourceLocation ElseLoc) {
   auto Remap = [&](SourceLocation Loc) {
     return Context.getSourceManager().getExpansionLoc(Loc);
   };
@@ -172,7 +172,7 @@ void ElseAfterReturnCheck::registerMatchers(MatchFinder *Finder) {
       breakStmt().bind(InterruptingStr), cxxThrowExpr().bind(InterruptingStr)));
   Finder->addMatcher(
       compoundStmt(
-          forEach(ifStmt(unless(isConstexpr()),
+          forEach(ifStmt(unless(isConstexpr()), unless(isConsteval()),
                          hasThen(stmt(
                              anyOf(InterruptsControlFlow,
                                    compoundStmt(has(InterruptsControlFlow))))),
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index fc976ce3a33d..8183d394cf42 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -339,6 +339,10 @@ Changes in existing checks
   <clang-tidy/checks/readability/duplicate-include>` check by excluding include
   directives that form the filename using macro.
 
+- Improved :doc:`readability-else-after-return
+  <clang-tidy/checks/readability/else-after-return>` check to ignore
+  `if consteval` statements, for which the `else` branch must not be removed.
+
 - Improved :doc:`readability-identifier-naming
   <clang-tidy/checks/readability/identifier-naming>` check in `GetConfigPerFile`
   mode by resolving symbolic links to header files. Fixed handling of Hungarian
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/else-after-return-if-consteval.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/else-after-return-if-consteval.cpp
new file mode 100644
index 000000000000..8810d215ee97
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/else-after-return-if-consteval.cpp
@@ -0,0 +1,17 @@
+// RUN: %check_clang_tidy -std=c++23 %s readability-else-after-return %t 
+
+// Consteval if is an exception to the rule, we cannot remove the else.
+void f() {
+  if (sizeof(int) > 4) {
+    return;
+  } else {
+    return;
+  }
+  // CHECK-MESSAGES: [[@LINE-3]]:5: warning: do not use 'else' after 'return'
+
+  if consteval {
+    return;
+  } else {
+    return;
+  }
+}
-- 
GitLab


From ad727b1a6757c4c23b4121192b121fd67219820e Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Mon, 13 May 2024 10:19:59 -0700
Subject: [PATCH 0603/1206] Revert "Support polynomial attributes with floating
 point coefficients (#91137)" (#92001)

This reverts commit 91a14dbf825b79ff143d1b16124763a4a80facab.

Not sure how to fix the build error this introduced, so reverting until
I can figure it out

https://lab.llvm.org/buildbot/#/builders/264/builds/10468

Co-authored-by: Jeremy Kun <j2kun@users.noreply.github.com>
---
 .../mlir/Dialect/Polynomial/IR/Polynomial.h   | 193 ++++--------------
 .../mlir/Dialect/Polynomial/IR/Polynomial.td  | 139 +++++--------
 mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp |  78 ++++---
 .../Polynomial/IR/PolynomialAttributes.cpp    | 172 +++++++++-------
 mlir/test/Dialect/Polynomial/attributes.mlir  |  22 +-
 mlir/test/Dialect/Polynomial/ops.mlir         |  64 +++---
 mlir/test/Dialect/Polynomial/ops_errors.mlir  |  66 +++---
 mlir/test/Dialect/Polynomial/types.mlir       |  65 +++---
 8 files changed, 348 insertions(+), 451 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
index 2b3f0e105c6c..3325a6fa3f9f 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
@@ -11,13 +11,10 @@
 
 #include "mlir/Support/LLVM.h"
 #include "mlir/Support/LogicalResult.h"
-#include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Hashing.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/SmallVector.h"
 
 namespace mlir {
 
@@ -30,202 +27,98 @@ namespace polynomial {
 /// would want to specify 128-bit polynomials statically in the source code.
 constexpr unsigned apintBitWidth = 64;
 
-template <typename CoefficientType>
-class MonomialBase {
+/// A class representing a monomial of a single-variable polynomial with integer
+/// coefficients.
+class Monomial {
 public:
-  MonomialBase(const CoefficientType &coeff, const APInt &expo)
+  Monomial(int64_t coeff, uint64_t expo)
+      : coefficient(apintBitWidth, coeff), exponent(apintBitWidth, expo) {}
+
+  Monomial(const APInt &coeff, const APInt &expo)
       : coefficient(coeff), exponent(expo) {}
-  virtual ~MonomialBase() = 0;
 
-  const CoefficientType &getCoefficient() const { return coefficient; }
-  CoefficientType &getMutableCoefficient() { return coefficient; }
-  const APInt &getExponent() const { return exponent; }
-  void setCoefficient(const CoefficientType &coeff) { coefficient = coeff; }
-  void setExponent(const APInt &exp) { exponent = exp; }
+  Monomial() : coefficient(apintBitWidth, 0), exponent(apintBitWidth, 0) {}
 
-  bool operator==(const MonomialBase &other) const {
+  bool operator==(const Monomial &other) const {
     return other.coefficient == coefficient && other.exponent == exponent;
   }
-  bool operator!=(const MonomialBase &other) const {
+  bool operator!=(const Monomial &other) const {
     return other.coefficient != coefficient || other.exponent != exponent;
   }
 
   /// Monomials are ordered by exponent.
-  bool operator<(const MonomialBase &other) const {
+  bool operator<(const Monomial &other) const {
     return (exponent.ult(other.exponent));
   }
 
-  virtual bool isMonic() const = 0;
-  virtual void
-  coefficientToString(llvm::SmallString<16> &coeffString) const = 0;
-
-  template <typename T>
-  friend ::llvm::hash_code hash_value(const MonomialBase<T> &arg);
+  friend ::llvm::hash_code hash_value(const Monomial &arg);
 
-protected:
-  CoefficientType coefficient;
-  APInt exponent;
-};
-
-/// A class representing a monomial of a single-variable polynomial with integer
-/// coefficients.
-class IntMonomial : public MonomialBase<APInt> {
 public:
-  IntMonomial(int64_t coeff, uint64_t expo)
-      : MonomialBase(APInt(apintBitWidth, coeff), APInt(apintBitWidth, expo)) {}
-
-  IntMonomial()
-      : MonomialBase(APInt(apintBitWidth, 0), APInt(apintBitWidth, 0)) {}
-
-  ~IntMonomial() = default;
-
-  bool isMonic() const override { return coefficient == 1; }
+  APInt coefficient;
 
-  void coefficientToString(llvm::SmallString<16> &coeffString) const override {
-    coefficient.toStringSigned(coeffString);
-  }
+  // Always unsigned
+  APInt exponent;
 };
 
-/// A class representing a monomial of a single-variable polynomial with integer
-/// coefficients.
-class FloatMonomial : public MonomialBase<APFloat> {
+/// A single-variable polynomial with integer coefficients.
+///
+/// Eg: x^1024 + x + 1
+///
+/// The symbols used as the polynomial's indeterminate don't matter, so long as
+/// it is used consistently throughout the polynomial.
+class Polynomial {
 public:
-  FloatMonomial(double coeff, uint64_t expo)
-      : MonomialBase(APFloat(coeff), APInt(apintBitWidth, expo)) {}
-
-  FloatMonomial() : MonomialBase(APFloat((double)0), APInt(apintBitWidth, 0)) {}
+  Polynomial() = delete;
 
-  ~FloatMonomial() = default;
+  explicit Polynomial(ArrayRef<Monomial> terms) : terms(terms){};
 
-  bool isMonic() const override { return coefficient == APFloat(1.0); }
-
-  void coefficientToString(llvm::SmallString<16> &coeffString) const override {
-    coefficient.toString(coeffString);
-  }
-};
-
-template <typename Monomial>
-class PolynomialBase {
-public:
-  PolynomialBase() = delete;
+  // Returns a Polynomial from a list of monomials.
+  // Fails if two monomials have the same exponent.
+  static FailureOr<Polynomial> fromMonomials(ArrayRef<Monomial> monomials);
 
-  explicit PolynomialBase(ArrayRef<Monomial> terms) : terms(terms){};
+  /// Returns a polynomial with coefficients given by `coeffs`. The value
+  /// coeffs[i] is converted to a monomial with exponent i.
+  static Polynomial fromCoefficients(ArrayRef<int64_t> coeffs);
 
   explicit operator bool() const { return !terms.empty(); }
-  bool operator==(const PolynomialBase &other) const {
+  bool operator==(const Polynomial &other) const {
     return other.terms == terms;
   }
-  bool operator!=(const PolynomialBase &other) const {
+  bool operator!=(const Polynomial &other) const {
     return !(other.terms == terms);
   }
 
-  void print(raw_ostream &os, ::llvm::StringRef separator,
-             ::llvm::StringRef exponentiation) const {
-    bool first = true;
-    for (const Monomial &term : getTerms()) {
-      if (first) {
-        first = false;
-      } else {
-        os << separator;
-      }
-      std::string coeffToPrint;
-      if (term.isMonic() && term.getExponent().uge(1)) {
-        coeffToPrint = "";
-      } else {
-        llvm::SmallString<16> coeffString;
-        term.coefficientToString(coeffString);
-        coeffToPrint = coeffString.str();
-      }
-
-      if (term.getExponent() == 0) {
-        os << coeffToPrint;
-      } else if (term.getExponent() == 1) {
-        os << coeffToPrint << "x";
-      } else {
-        llvm::SmallString<16> expString;
-        term.getExponent().toStringSigned(expString);
-        os << coeffToPrint << "x" << exponentiation << expString;
-      }
-    }
-  }
-
   // Prints polynomial to 'os'.
-  void print(raw_ostream &os) const { print(os, " + ", "**"); }
-
+  void print(raw_ostream &os) const;
+  void print(raw_ostream &os, ::llvm::StringRef separator,
+             ::llvm::StringRef exponentiation) const;
   void dump() const;
 
   // Prints polynomial so that it can be used as a valid identifier
-  std::string toIdentifier() const {
-    std::string result;
-    llvm::raw_string_ostream os(result);
-    print(os, "_", "");
-    return os.str();
-  }
+  std::string toIdentifier() const;
 
-  unsigned getDegree() const {
-    return terms.back().getExponent().getZExtValue();
-  }
+  unsigned getDegree() const;
 
   ArrayRef<Monomial> getTerms() const { return terms; }
 
-  template <typename T>
-  friend ::llvm::hash_code hash_value(const PolynomialBase<T> &arg);
+  friend ::llvm::hash_code hash_value(const Polynomial &arg);
 
 private:
   // The monomial terms for this polynomial.
   SmallVector<Monomial> terms;
 };
 
-/// A single-variable polynomial with integer coefficients.
-///
-/// Eg: x^1024 + x + 1
-class IntPolynomial : public PolynomialBase<IntMonomial> {
-public:
-  explicit IntPolynomial(ArrayRef<IntMonomial> terms) : PolynomialBase(terms) {}
-
-  // Returns a Polynomial from a list of monomials.
-  // Fails if two monomials have the same exponent.
-  static FailureOr<IntPolynomial>
-  fromMonomials(ArrayRef<IntMonomial> monomials);
-
-  /// Returns a polynomial with coefficients given by `coeffs`. The value
-  /// coeffs[i] is converted to a monomial with exponent i.
-  static IntPolynomial fromCoefficients(ArrayRef<int64_t> coeffs);
-};
-
-/// A single-variable polynomial with double coefficients.
-///
-/// Eg: 1.0 x^1024 + 3.5 x + 1e-05
-class FloatPolynomial : public PolynomialBase<FloatMonomial> {
-public:
-  explicit FloatPolynomial(ArrayRef<FloatMonomial> terms)
-      : PolynomialBase(terms) {}
-
-  // Returns a Polynomial from a list of monomials.
-  // Fails if two monomials have the same exponent.
-  static FailureOr<FloatPolynomial>
-  fromMonomials(ArrayRef<FloatMonomial> monomials);
-
-  /// Returns a polynomial with coefficients given by `coeffs`. The value
-  /// coeffs[i] is converted to a monomial with exponent i.
-  static FloatPolynomial fromCoefficients(ArrayRef<double> coeffs);
-};
-
-// Make Polynomials hashable.
-template <typename T>
-inline ::llvm::hash_code hash_value(const PolynomialBase<T> &arg) {
+// Make Polynomial hashable.
+inline ::llvm::hash_code hash_value(const Polynomial &arg) {
   return ::llvm::hash_combine_range(arg.terms.begin(), arg.terms.end());
 }
 
-template <typename T>
-inline ::llvm::hash_code hash_value(const MonomialBase<T> &arg) {
+inline ::llvm::hash_code hash_value(const Monomial &arg) {
   return llvm::hash_combine(::llvm::hash_value(arg.coefficient),
                             ::llvm::hash_value(arg.exponent));
 }
 
-template <typename T>
-inline raw_ostream &operator<<(raw_ostream &os,
-                               const PolynomialBase<T> &polynomial) {
+inline raw_ostream &operator<<(raw_ostream &os, const Polynomial &polynomial) {
   polynomial.print(os);
   return os;
 }
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
index ae8484501a50..ed1f4ce8b7e5 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
@@ -39,14 +39,14 @@ def Polynomial_Dialect : Dialect {
     %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
 
     // A constant polynomial in a ring with i32 coefficients, modulo (x^1024 + 1)
-    #modulus = #polynomial.int_polynomial<1 + x**1024>
+    #modulus = #polynomial.polynomial<1 + x**1024>
     #ring = #polynomial.ring<coefficientType=i32, polynomialModulus=#modulus>
     %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
 
     // A constant polynomial in a ring with i32 coefficients, with a polynomial
     // modulus of (x^1024 + 1) and a coefficient modulus of 17.
-    #modulus = #polynomial.int_polynomial<1 + x**1024>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=17:i32, polynomialModulus=#modulus>
+    #modulus = #polynomial.polynomial<1 + x**1024>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=17, polynomialModulus=#modulus>
     %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
     ```
   }];
@@ -60,12 +60,12 @@ class Polynomial_Attr<string name, string attrMnemonic, list<Trait> traits = []>
   let mnemonic = attrMnemonic;
 }
 
-def Polynomial_IntPolynomialAttr : Polynomial_Attr<"IntPolynomial", "int_polynomial"> {
-  let summary = "An attribute containing a single-variable polynomial with integer coefficients.";
+def Polynomial_PolynomialAttr : Polynomial_Attr<"Polynomial", "polynomial"> {
+  let summary = "An attribute containing a single-variable polynomial.";
   let description = [{
-    A polynomial attribute represents a single-variable polynomial with integer
-    coefficients, which is used to define the modulus of a `RingAttr`, as well
-    as to define constants and perform constant folding for `polynomial` ops.
+    A polynomial attribute represents a single-variable polynomial, which
+    is used to define the modulus of a `RingAttr`, as well as to define constants
+    and perform constant folding for `polynomial` ops.
 
     The polynomial must be expressed as a list of monomial terms, with addition
     or subtraction between them. The choice of variable name is arbitrary, but
@@ -76,32 +76,10 @@ def Polynomial_IntPolynomialAttr : Polynomial_Attr<"IntPolynomial", "int_polynom
     Example:
 
     ```mlir
-    #poly = #polynomial.int_polynomial<x**1024 + 1>
+    #poly = #polynomial.polynomial<x**1024 + 1>
     ```
   }];
-  let parameters = (ins "::mlir::polynomial::IntPolynomial":$polynomial);
-  let hasCustomAssemblyFormat = 1;
-}
-
-def Polynomial_FloatPolynomialAttr : Polynomial_Attr<"FloatPolynomial", "float_polynomial"> {
-  let summary = "An attribute containing a single-variable polynomial with double precision floating point coefficients.";
-  let description = [{
-    A polynomial attribute represents a single-variable polynomial with double
-    precision floating point coefficients.
-
-    The polynomial must be expressed as a list of monomial terms, with addition
-    or subtraction between them. The choice of variable name is arbitrary, but
-    must be consistent across all the monomials used to define a single
-    attribute. The order of monomial terms is arbitrary, each monomial degree
-    must occur at most once.
-
-    Example:
-
-    ```mlir
-    #poly = #polynomial.float_polynomial<0.5 x**7 + 1.5>
-    ```
-  }];
-  let parameters = (ins "FloatPolynomial":$polynomial);
+  let parameters = (ins "::mlir::polynomial::Polynomial":$polynomial);
   let hasCustomAssemblyFormat = 1;
 }
 
@@ -126,9 +104,9 @@ def Polynomial_RingAttr : Polynomial_Attr<"Ring", "ring"> {
     `x**1024 - 1`.
 
     ```mlir
-    #poly_mod = #polynomial.int_polynomial<-1 + x**1024>
+    #poly_mod = #polynomial.polynomial<-1 + x**1024>
     #ring = #polynomial.ring<coefficientType=i32,
-                             coefficientModulus=4294967291:i32,
+                             coefficientModulus=4294967291,
                              polynomialModulus=#poly_mod>
 
     %0 = ... : polynomial.polynomial<#ring>
@@ -145,24 +123,19 @@ def Polynomial_RingAttr : Polynomial_Attr<"Ring", "ring"> {
   let parameters = (ins
     "Type": $coefficientType,
     OptionalParameter<"::mlir::IntegerAttr">: $coefficientModulus,
-    OptionalParameter<"::mlir::polynomial::IntPolynomialAttr">: $polynomialModulus,
+    OptionalParameter<"::mlir::polynomial::PolynomialAttr">: $polynomialModulus,
     OptionalParameter<"::mlir::IntegerAttr">: $primitiveRoot
   );
-  let assemblyFormat = "`<` struct(params) `>`";
+
   let builders = [
-    AttrBuilderWithInferredContext<
+    AttrBuilder<
         (ins "::mlir::Type":$coefficientTy,
-              CArg<"::mlir::IntegerAttr", "nullptr"> :$coefficientModulusAttr,
-              CArg<"::mlir::polynomial::IntPolynomialAttr", "nullptr"> :$polynomialModulusAttr,
-              CArg<"::mlir::IntegerAttr", "nullptr"> :$primitiveRootAttr), [{
-      return $_get(
-        coefficientTy.getContext(),
-        coefficientTy,
-        coefficientModulusAttr,
-        polynomialModulusAttr,
-        primitiveRootAttr);
-    }]>,
+             "::mlir::IntegerAttr":$coefficientModulusAttr,
+             "::mlir::polynomial::PolynomialAttr":$polynomialModulusAttr), [{
+      return $_get($_ctxt, coefficientTy, coefficientModulusAttr, polynomialModulusAttr, nullptr);
+    }]>
   ];
+  let hasCustomAssemblyFormat = 1;
 }
 
 class Polynomial_Type<string name, string typeMnemonic>
@@ -176,7 +149,7 @@ def Polynomial_PolynomialType : Polynomial_Type<"Polynomial", "polynomial"> {
     A type for polynomials in a polynomial quotient ring.
   }];
   let parameters = (ins Polynomial_RingAttr:$ring);
-  let assemblyFormat = "`<` struct(params) `>`";
+  let assemblyFormat = "`<` $ring `>`";
 }
 
 def PolynomialLike: TypeOrContainer<Polynomial_PolynomialType, "polynomial-like">;
@@ -214,10 +187,10 @@ def Polynomial_AddOp : Polynomial_BinaryOp<"add", [Commutative]> {
 
     ```mlir
     // add two polynomials modulo x^1024 - 1
-    #poly = #polynomial.int_polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
-    %1 = polynomial.constant #polynomial.int_polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    %1 = polynomial.constant #polynomial.polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
     %2 = polynomial.add %0, %1 : !polynomial.polynomial<#ring>
     ```
   }];
@@ -238,10 +211,10 @@ def Polynomial_SubOp : Polynomial_BinaryOp<"sub"> {
 
     ```mlir
     // subtract two polynomials modulo x^1024 - 1
-    #poly = #polynomial.int_polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
-    %1 = polynomial.constant #polynomial.int_polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    %1 = polynomial.constant #polynomial.polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
     %2 = polynomial.sub %0, %1 : !polynomial.polynomial<#ring>
     ```
   }];
@@ -262,10 +235,10 @@ def Polynomial_MulOp : Polynomial_BinaryOp<"mul", [Commutative]> {
 
     ```mlir
     // multiply two polynomials modulo x^1024 - 1
-    #poly = #polynomial.int_polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
-    %1 = polynomial.constant #polynomial.int_polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    %1 = polynomial.constant #polynomial.polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
     %2 = polynomial.mul %0, %1 : !polynomial.polynomial<#ring>
     ```
   }];
@@ -287,9 +260,9 @@ def Polynomial_MulScalarOp : Polynomial_Op<"mul_scalar", [
 
     ```mlir
     // multiply two polynomials modulo x^1024 - 1
-    #poly = #polynomial.int_polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
     %1 = arith.constant 3 : i32
     %2 = polynomial.mul_scalar %0, %1 : !polynomial.polynomial<#ring>, i32
     ```
@@ -318,9 +291,9 @@ def Polynomial_LeadingTermOp: Polynomial_Op<"leading_term"> {
     Example:
 
     ```mlir
-    #poly = #polynomial.int_polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
     %1, %2 = polynomial.leading_term %0 : !polynomial.polynomial<#ring> -> (index, i32)
     ```
   }];
@@ -341,8 +314,8 @@ def Polynomial_MonomialOp: Polynomial_Op<"monomial"> {
     Example:
 
     ```mlir
-    #poly = #polynomial.int_polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    #poly = #polynomial.polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
     %deg = arith.constant 1023 : index
     %five = arith.constant 5 : i32
     %0 = polynomial.monomial %five, %deg : (i32, index) -> !polynomial.polynomial<#ring>
@@ -381,8 +354,8 @@ def Polynomial_FromTensorOp : Polynomial_Op<"from_tensor", [Pure]> {
     Example:
 
     ```mlir
-    #poly = #polynomial.int_polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    #poly = #polynomial.polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
     %two = arith.constant 2 : i32
     %five = arith.constant 5 : i32
     %coeffs = tensor.from_elements %two, %two, %five : tensor<3xi32>
@@ -420,8 +393,8 @@ def Polynomial_ToTensorOp : Polynomial_Op<"to_tensor", [Pure]> {
     Example:
 
     ```mlir
-    #poly = #polynomial.int_polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    #poly = #polynomial.polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
     %two = arith.constant 2 : i32
     %five = arith.constant 5 : i32
     %coeffs = tensor.from_elements %two, %two, %five : tensor<3xi32>
@@ -432,32 +405,24 @@ def Polynomial_ToTensorOp : Polynomial_Op<"to_tensor", [Pure]> {
   let arguments = (ins Polynomial_PolynomialType:$input);
   let results = (outs RankedTensorOf<[AnyInteger]>:$output);
   let assemblyFormat = "$input attr-dict `:` type($input) `->` type($output)";
+
   let hasVerifier = 1;
 }
 
-def Polynomial_AnyPolynomialAttr : AnyAttrOf<[
-  Polynomial_FloatPolynomialAttr,
-  Polynomial_IntPolynomialAttr
-]>;
-
-// Not deriving from Polynomial_Op due to need for custom assembly format
-def Polynomial_ConstantOp : Op<Polynomial_Dialect, "constant", [Pure]> {
+def Polynomial_ConstantOp : Polynomial_Op<"constant", [Pure]> {
   let summary = "Define a constant polynomial via an attribute.";
   let description = [{
     Example:
 
     ```mlir
-    #poly = #polynomial.int_polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
-
-    #float_ring = #polynomial.ring<coefficientType=f32>
-    %0 = polynomial.constant #polynomial.float_polynomial<0.5 + 1.3e06 x**2> : !polynomial.polynomial<#float_ring>
+    #poly = #polynomial.polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
     ```
   }];
-  let arguments = (ins Polynomial_AnyPolynomialAttr:$value);
+  let arguments = (ins Polynomial_PolynomialAttr:$input);
   let results = (outs Polynomial_PolynomialType:$output);
-  let assemblyFormat = "attr-dict `:` type($output)";
+  let assemblyFormat = "$input attr-dict `:` type($output)";
 }
 
 def Polynomial_NTTOp : Polynomial_Op<"ntt", [Pure]> {
diff --git a/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp b/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
index 42e678fad060..5916ffba78e2 100644
--- a/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
+++ b/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
@@ -9,63 +9,87 @@
 #include "mlir/Dialect/Polynomial/IR/Polynomial.h"
 
 #include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/raw_ostream.h"
 
 namespace mlir {
 namespace polynomial {
 
-template <typename T>
-MonomialBase<T>::~MonomialBase() {}
-
-template <typename PolyT, typename MonomialT>
-FailureOr<PolyT> fromMonomialsImpl(ArrayRef<MonomialT> monomials) {
+FailureOr<Polynomial> Polynomial::fromMonomials(ArrayRef<Monomial> monomials) {
   // A polynomial's terms are canonically stored in order of increasing degree.
-  auto monomialsCopy = llvm::SmallVector<MonomialT>(monomials);
+  auto monomialsCopy = llvm::SmallVector<Monomial>(monomials);
   std::sort(monomialsCopy.begin(), monomialsCopy.end());
 
   // Ensure non-unique exponents are not present. Since we sorted the list by
   // exponent, a linear scan of adjancent monomials suffices.
   if (std::adjacent_find(monomialsCopy.begin(), monomialsCopy.end(),
-                         [](const MonomialT &lhs, const MonomialT &rhs) {
-                           return lhs.getExponent() == rhs.getExponent();
+                         [](const Monomial &lhs, const Monomial &rhs) {
+                           return lhs.exponent == rhs.exponent;
                          }) != monomialsCopy.end()) {
     return failure();
   }
 
-  return PolyT(monomialsCopy);
-}
-
-FailureOr<IntPolynomial>
-IntPolynomial::fromMonomials(ArrayRef<IntMonomial> monomials) {
-  return fromMonomialsImpl<IntPolynomial, IntMonomial>(monomials);
-}
-
-FailureOr<FloatPolynomial>
-FloatPolynomial::fromMonomials(ArrayRef<FloatMonomial> monomials) {
-  return fromMonomialsImpl<FloatPolynomial, FloatMonomial>(monomials);
+  return Polynomial(monomialsCopy);
 }
 
-template <typename PolyT, typename MonomialT, typename CoeffT>
-PolyT fromCoefficientsImpl(ArrayRef<CoeffT> coeffs) {
-  llvm::SmallVector<MonomialT> monomials;
+Polynomial Polynomial::fromCoefficients(ArrayRef<int64_t> coeffs) {
+  llvm::SmallVector<Monomial> monomials;
   auto size = coeffs.size();
   monomials.reserve(size);
   for (size_t i = 0; i < size; i++) {
     monomials.emplace_back(coeffs[i], i);
   }
-  auto result = PolyT::fromMonomials(monomials);
+  auto result = Polynomial::fromMonomials(monomials);
   // Construction guarantees unique exponents, so the failure mode of
   // fromMonomials can be bypassed.
   assert(succeeded(result));
   return result.value();
 }
 
-IntPolynomial IntPolynomial::fromCoefficients(ArrayRef<int64_t> coeffs) {
-  return fromCoefficientsImpl<IntPolynomial, IntMonomial, int64_t>(coeffs);
+void Polynomial::print(raw_ostream &os, ::llvm::StringRef separator,
+                       ::llvm::StringRef exponentiation) const {
+  bool first = true;
+  for (const Monomial &term : terms) {
+    if (first) {
+      first = false;
+    } else {
+      os << separator;
+    }
+    std::string coeffToPrint;
+    if (term.coefficient == 1 && term.exponent.uge(1)) {
+      coeffToPrint = "";
+    } else {
+      llvm::SmallString<16> coeffString;
+      term.coefficient.toStringSigned(coeffString);
+      coeffToPrint = coeffString.str();
+    }
+
+    if (term.exponent == 0) {
+      os << coeffToPrint;
+    } else if (term.exponent == 1) {
+      os << coeffToPrint << "x";
+    } else {
+      llvm::SmallString<16> expString;
+      term.exponent.toStringSigned(expString);
+      os << coeffToPrint << "x" << exponentiation << expString;
+    }
+  }
+}
+
+void Polynomial::print(raw_ostream &os) const { print(os, " + ", "**"); }
+
+std::string Polynomial::toIdentifier() const {
+  std::string result;
+  llvm::raw_string_ostream os(result);
+  print(os, "_", "");
+  return os.str();
 }
 
-FloatPolynomial FloatPolynomial::fromCoefficients(ArrayRef<double> coeffs) {
-  return fromCoefficientsImpl<FloatPolynomial, FloatMonomial, double>(coeffs);
+unsigned Polynomial::getDegree() const {
+  return terms.back().exponent.getZExtValue();
 }
 
 } // namespace polynomial
diff --git a/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp b/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
index 890ce5226c30..236bb7896635 100644
--- a/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
+++ b/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
@@ -10,7 +10,6 @@
 #include "mlir/Dialect/Polynomial/IR/Polynomial.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Support/LogicalResult.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
@@ -18,31 +17,22 @@
 namespace mlir {
 namespace polynomial {
 
-void IntPolynomialAttr::print(AsmPrinter &p) const {
-  p << '<' << getPolynomial() << '>';
+void PolynomialAttr::print(AsmPrinter &p) const {
+  p << '<';
+  p << getPolynomial();
+  p << '>';
 }
 
-void FloatPolynomialAttr::print(AsmPrinter &p) const {
-  p << '<' << getPolynomial() << '>';
-}
-
-/// A callable that parses the coefficient using the appropriate method for the
-/// given monomial type, and stores the parsed coefficient value on the
-/// monomial.
-template <typename MonomialType>
-using ParseCoefficientFn = std::function<OptionalParseResult(MonomialType &)>;
-
 /// Try to parse a monomial. If successful, populate the fields of the outparam
 /// `monomial` with the results, and the `variable` outparam with the parsed
 /// variable name. Sets shouldParseMore to true if the monomial is followed by
 /// a '+'.
-///
-template <typename Monomial>
-ParseResult
-parseMonomial(AsmParser &parser, Monomial &monomial, llvm::StringRef &variable,
-              bool &isConstantTerm, bool &shouldParseMore,
-              ParseCoefficientFn<Monomial> parseAndStoreCoefficient) {
-  OptionalParseResult parsedCoeffResult = parseAndStoreCoefficient(monomial);
+ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
+                          llvm::StringRef &variable, bool &isConstantTerm,
+                          bool &shouldParseMore) {
+  APInt parsedCoeff(apintBitWidth, 1);
+  auto parsedCoeffResult = parser.parseOptionalInteger(parsedCoeff);
+  monomial.coefficient = parsedCoeff;
 
   isConstantTerm = false;
   shouldParseMore = false;
@@ -54,7 +44,7 @@ parseMonomial(AsmParser &parser, Monomial &monomial, llvm::StringRef &variable,
     if (!parsedCoeffResult.has_value()) {
       return failure();
     }
-    monomial.setExponent(APInt(apintBitWidth, 0));
+    monomial.exponent = APInt(apintBitWidth, 0);
     isConstantTerm = true;
     shouldParseMore = true;
     return success();
@@ -68,7 +58,7 @@ parseMonomial(AsmParser &parser, Monomial &monomial, llvm::StringRef &variable,
       return failure();
     }
 
-    monomial.setExponent(APInt(apintBitWidth, 0));
+    monomial.exponent = APInt(apintBitWidth, 0);
     isConstantTerm = true;
     return success();
   }
@@ -90,9 +80,9 @@ parseMonomial(AsmParser &parser, Monomial &monomial, llvm::StringRef &variable,
       return failure();
     }
 
-    monomial.setExponent(parsedExponent);
+    monomial.exponent = parsedExponent;
   } else {
-    monomial.setExponent(APInt(apintBitWidth, 1));
+    monomial.exponent = APInt(apintBitWidth, 1);
   }
 
   if (succeeded(parser.parseOptionalPlus())) {
@@ -101,21 +91,22 @@ parseMonomial(AsmParser &parser, Monomial &monomial, llvm::StringRef &variable,
   return success();
 }
 
-template <typename PolynoimalAttrTy, typename Monomial>
-LogicalResult
-parsePolynomialAttr(AsmParser &parser, llvm::SmallVector<Monomial> &monomials,
-                    llvm::StringSet<> &variables,
-                    ParseCoefficientFn<Monomial> parseAndStoreCoefficient) {
+Attribute PolynomialAttr::parse(AsmParser &parser, Type type) {
+  if (failed(parser.parseLess()))
+    return {};
+
+  llvm::SmallVector<Monomial> monomials;
+  llvm::StringSet<> variables;
+
   while (true) {
     Monomial parsedMonomial;
     llvm::StringRef parsedVariableRef;
     bool isConstantTerm;
     bool shouldParseMore;
-    if (failed(parseMonomial<Monomial>(
-            parser, parsedMonomial, parsedVariableRef, isConstantTerm,
-            shouldParseMore, parseAndStoreCoefficient))) {
+    if (failed(parseMonomial(parser, parsedMonomial, parsedVariableRef,
+                             isConstantTerm, shouldParseMore))) {
       parser.emitError(parser.getCurrentLocation(), "expected a monomial");
-      return failure();
+      return {};
     }
 
     if (!isConstantTerm) {
@@ -133,7 +124,7 @@ parsePolynomialAttr(AsmParser &parser, llvm::SmallVector<Monomial> &monomials,
     parser.emitError(
         parser.getCurrentLocation(),
         "expected + and more monomials, or > to end polynomial attribute");
-    return failure();
+    return {};
   }
 
   if (variables.size() > 1) {
@@ -142,67 +133,96 @@ parsePolynomialAttr(AsmParser &parser, llvm::SmallVector<Monomial> &monomials,
         parser.getCurrentLocation(),
         "polynomials must have one indeterminate, but there were multiple: " +
             vars);
-    return failure();
   }
 
-  return success();
+  auto result = Polynomial::fromMonomials(monomials);
+  if (failed(result)) {
+    parser.emitError(parser.getCurrentLocation())
+        << "parsed polynomial must have unique exponents among monomials";
+    return {};
+  }
+  return PolynomialAttr::get(parser.getContext(), result.value());
 }
 
-Attribute IntPolynomialAttr::parse(AsmParser &parser, Type type) {
+void RingAttr::print(AsmPrinter &p) const {
+  p << "#polynomial.ring<coefficientType=" << getCoefficientType()
+    << ", coefficientModulus=" << getCoefficientModulus()
+    << ", polynomialModulus=" << getPolynomialModulus() << '>';
+}
+
+Attribute RingAttr::parse(AsmParser &parser, Type type) {
   if (failed(parser.parseLess()))
     return {};
 
-  llvm::SmallVector<IntMonomial> monomials;
-  llvm::StringSet<> variables;
+  if (failed(parser.parseKeyword("coefficientType")))
+    return {};
 
-  if (failed(parsePolynomialAttr<IntPolynomialAttr, IntMonomial>(
-          parser, monomials, variables,
-          [&](IntMonomial &monomial) -> OptionalParseResult {
-            APInt parsedCoeff(apintBitWidth, 1);
-            OptionalParseResult result =
-                parser.parseOptionalInteger(parsedCoeff);
-            monomial.setCoefficient(parsedCoeff);
-            return result;
-          }))) {
+  if (failed(parser.parseEqual()))
     return {};
-  }
 
-  auto result = IntPolynomial::fromMonomials(monomials);
-  if (failed(result)) {
-    parser.emitError(parser.getCurrentLocation())
-        << "parsed polynomial must have unique exponents among monomials";
+  Type ty;
+  if (failed(parser.parseType(ty)))
     return {};
-  }
-  return IntPolynomialAttr::get(parser.getContext(), result.value());
-}
 
-Attribute FloatPolynomialAttr::parse(AsmParser &parser, Type type) {
-  if (failed(parser.parseLess()))
+  if (failed(parser.parseComma()))
     return {};
 
-  llvm::SmallVector<FloatMonomial> monomials;
-  llvm::StringSet<> variables;
+  IntegerAttr coefficientModulusAttr = nullptr;
+  if (succeeded(parser.parseKeyword("coefficientModulus"))) {
+    if (failed(parser.parseEqual()))
+      return {};
 
-  ParseCoefficientFn<FloatMonomial> parseAndStoreCoefficient =
-      [&](FloatMonomial &monomial) -> OptionalParseResult {
-    double coeffValue = 1.0;
-    ParseResult result = parser.parseFloat(coeffValue);
-    monomial.setCoefficient(APFloat(coeffValue));
-    return OptionalParseResult(result);
-  };
+    IntegerType iType = mlir::dyn_cast<IntegerType>(ty);
+    if (!iType) {
+      parser.emitError(parser.getCurrentLocation(),
+                       "coefficientType must specify an integer type");
+      return {};
+    }
+    APInt coefficientModulus(iType.getWidth(), 0);
+    auto result = parser.parseInteger(coefficientModulus);
+    if (failed(result)) {
+      parser.emitError(parser.getCurrentLocation(),
+                       "invalid coefficient modulus");
+      return {};
+    }
+    coefficientModulusAttr = IntegerAttr::get(iType, coefficientModulus);
 
-  if (failed(parsePolynomialAttr<FloatPolynomialAttr, FloatMonomial>(
-          parser, monomials, variables, parseAndStoreCoefficient))) {
-    return {};
+    if (failed(parser.parseComma()))
+      return {};
   }
 
-  auto result = FloatPolynomial::fromMonomials(monomials);
-  if (failed(result)) {
-    parser.emitError(parser.getCurrentLocation())
-        << "parsed polynomial must have unique exponents among monomials";
-    return {};
+  PolynomialAttr polyAttr = nullptr;
+  if (succeeded(parser.parseKeyword("polynomialModulus"))) {
+    if (failed(parser.parseEqual()))
+      return {};
+
+    PolynomialAttr attr;
+    if (failed(parser.parseAttribute<PolynomialAttr>(attr)))
+      return {};
+    polyAttr = attr;
   }
-  return FloatPolynomialAttr::get(parser.getContext(), result.value());
+
+  Polynomial poly = polyAttr.getPolynomial();
+  APInt root(coefficientModulusAttr.getValue().getBitWidth(), 0);
+  IntegerAttr rootAttr = nullptr;
+  if (succeeded(parser.parseOptionalComma())) {
+    if (failed(parser.parseKeyword("primitiveRoot")) ||
+        failed(parser.parseEqual()))
+      return {};
+
+    ParseResult result = parser.parseInteger(root);
+    if (failed(result)) {
+      parser.emitError(parser.getCurrentLocation(), "invalid primitiveRoot");
+      return {};
+    }
+    rootAttr = IntegerAttr::get(coefficientModulusAttr.getType(), root);
+  }
+
+  if (failed(parser.parseGreater()))
+    return {};
+
+  return RingAttr::get(parser.getContext(), ty, coefficientModulusAttr,
+                       polyAttr, rootAttr);
 }
 
 } // namespace polynomial
diff --git a/mlir/test/Dialect/Polynomial/attributes.mlir b/mlir/test/Dialect/Polynomial/attributes.mlir
index 4bdfd44fd4d1..3973ae394433 100644
--- a/mlir/test/Dialect/Polynomial/attributes.mlir
+++ b/mlir/test/Dialect/Polynomial/attributes.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s --split-input-file --verify-diagnostics
 
-#my_poly = #polynomial.int_polynomial<y + x**1024>
+#my_poly = #polynomial.polynomial<y + x**1024>
 // expected-error@below {{polynomials must have one indeterminate, but there were multiple: x, y}}
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
@@ -9,31 +9,37 @@
 // expected-error@below {{expected integer value}}
 // expected-error@below {{expected a monomial}}
 // expected-error@below {{found invalid integer exponent}}
-#my_poly = #polynomial.int_polynomial<5 + x**f>
+#my_poly = #polynomial.polynomial<5 + x**f>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 // -----
 
-#my_poly = #polynomial.int_polynomial<5 + x**2 + 3x**2>
+#my_poly = #polynomial.polynomial<5 + x**2 + 3x**2>
 // expected-error@below {{parsed polynomial must have unique exponents among monomials}}
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 // -----
 
 // expected-error@below {{expected + and more monomials, or > to end polynomial attribute}}
-#my_poly = #polynomial.int_polynomial<5 + x**2 7>
+#my_poly = #polynomial.polynomial<5 + x**2 7>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 // -----
 
 // expected-error@below {{expected a monomial}}
-#my_poly = #polynomial.int_polynomial<5 + x**2 +>
+#my_poly = #polynomial.polynomial<5 + x**2 +>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 
 // -----
 
-#my_poly = #polynomial.int_polynomial<5 + x**2>
-// expected-error@below {{failed to parse Polynomial_RingAttr parameter 'coefficientModulus' which is to be a `::mlir::IntegerAttr`}}
-// expected-error@below {{expected attribute value}}
+#my_poly = #polynomial.polynomial<5 + x**2>
+// expected-error@below {{coefficientType must specify an integer type}}
+#ring1 = #polynomial.ring<coefficientType=f64, coefficientModulus=2837465, polynomialModulus=#my_poly>
+
+// -----
+
+#my_poly = #polynomial.polynomial<5 + x**2>
+// expected-error@below {{expected integer value}}
+// expected-error@below {{invalid coefficient modulus}}
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=x, polynomialModulus=#my_poly>
diff --git a/mlir/test/Dialect/Polynomial/ops.mlir b/mlir/test/Dialect/Polynomial/ops.mlir
index ff709960c50e..a29cfc2e9cc5 100644
--- a/mlir/test/Dialect/Polynomial/ops.mlir
+++ b/mlir/test/Dialect/Polynomial/ops.mlir
@@ -2,87 +2,85 @@
 
 // This simply tests for syntax.
 
-#my_poly = #polynomial.int_polynomial<1 + x**1024>
-#my_poly_2 = #polynomial.int_polynomial<2>
-#my_poly_3 = #polynomial.int_polynomial<3x>
-#my_poly_4 = #polynomial.int_polynomial<t**3 + 4t + 2>
+#my_poly = #polynomial.polynomial<1 + x**1024>
+#my_poly_2 = #polynomial.polynomial<2>
+#my_poly_3 = #polynomial.polynomial<3x>
+#my_poly_4 = #polynomial.polynomial<t**3 + 4t + 2>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
-#ring2 = #polynomial.ring<coefficientType=f32>
-#one_plus_x_squared = #polynomial.int_polynomial<1 + x**2>
+#one_plus_x_squared = #polynomial.polynomial<1 + x**2>
 
-#ideal = #polynomial.int_polynomial<-1 + x**1024>
+#ideal = #polynomial.polynomial<-1 + x**1024>
 #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#ideal, primitiveRoot=193>
-!poly_ty = !polynomial.polynomial<ring=#ring>
+!poly_ty = !polynomial.polynomial<#ring>
 
-#ntt_poly = #polynomial.int_polynomial<-1 + x**8>
+#ntt_poly = #polynomial.polynomial<-1 + x**8>
 #ntt_ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#ntt_poly, primitiveRoot=31>
-!ntt_poly_ty = !polynomial.polynomial<ring=#ntt_ring>
+!ntt_poly_ty = !polynomial.polynomial<#ntt_ring>
 
 module {
-  func.func @test_multiply() -> !polynomial.polynomial<ring=#ring1> {
+  func.func @test_multiply() -> !polynomial.polynomial<#ring1> {
     %c0 = arith.constant 0 : index
     %two = arith.constant 2 : i16
     %five = arith.constant 5 : i16
     %coeffs1 = tensor.from_elements %two, %two, %five : tensor<3xi16>
     %coeffs2 = tensor.from_elements %five, %five, %two : tensor<3xi16>
 
-    %poly1 = polynomial.from_tensor %coeffs1 : tensor<3xi16> -> !polynomial.polynomial<ring=#ring1>
-    %poly2 = polynomial.from_tensor %coeffs2 : tensor<3xi16> -> !polynomial.polynomial<ring=#ring1>
+    %poly1 = polynomial.from_tensor %coeffs1 : tensor<3xi16> -> !polynomial.polynomial<#ring1>
+    %poly2 = polynomial.from_tensor %coeffs2 : tensor<3xi16> -> !polynomial.polynomial<#ring1>
 
-    %3 = polynomial.mul %poly1, %poly2 : !polynomial.polynomial<ring=#ring1>
+    %3 = polynomial.mul %poly1, %poly2 : !polynomial.polynomial<#ring1>
 
-    return %3 : !polynomial.polynomial<ring=#ring1>
+    return %3 : !polynomial.polynomial<#ring1>
   }
 
-  func.func @test_elementwise(%p0 : !polynomial.polynomial<ring=#ring1>, %p1: !polynomial.polynomial<ring=#ring1>) {
-    %tp0 = tensor.from_elements %p0, %p1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
-    %tp1 = tensor.from_elements %p1, %p0 : tensor<2x!polynomial.polynomial<ring=#ring1>>
+  func.func @test_elementwise(%p0 : !polynomial.polynomial<#ring1>, %p1: !polynomial.polynomial<#ring1>) {
+    %tp0 = tensor.from_elements %p0, %p1 : tensor<2x!polynomial.polynomial<#ring1>>
+    %tp1 = tensor.from_elements %p1, %p0 : tensor<2x!polynomial.polynomial<#ring1>>
 
     %c = arith.constant 2 : i32
-    %mul_const_sclr = polynomial.mul_scalar %tp0, %c : tensor<2x!polynomial.polynomial<ring=#ring1>>, i32
+    %mul_const_sclr = polynomial.mul_scalar %tp0, %c : tensor<2x!polynomial.polynomial<#ring1>>, i32
 
-    %add = polynomial.add %tp0, %tp1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
-    %sub = polynomial.sub %tp0, %tp1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
-    %mul = polynomial.mul %tp0, %tp1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
+    %add = polynomial.add %tp0, %tp1 : tensor<2x!polynomial.polynomial<#ring1>>
+    %sub = polynomial.sub %tp0, %tp1 : tensor<2x!polynomial.polynomial<#ring1>>
+    %mul = polynomial.mul %tp0, %tp1 : tensor<2x!polynomial.polynomial<#ring1>>
 
     return
   }
 
-  func.func @test_to_from_tensor(%p0 : !polynomial.polynomial<ring=#ring1>) {
+  func.func @test_to_from_tensor(%p0 : !polynomial.polynomial<#ring1>) {
     %c0 = arith.constant 0 : index
     %two = arith.constant 2 : i16
     %coeffs1 = tensor.from_elements %two, %two : tensor<2xi16>
     // CHECK: from_tensor
-    %poly = polynomial.from_tensor %coeffs1 : tensor<2xi16> -> !polynomial.polynomial<ring=#ring1>
+    %poly = polynomial.from_tensor %coeffs1 : tensor<2xi16> -> !polynomial.polynomial<#ring1>
     // CHECK: to_tensor
-    %tensor = polynomial.to_tensor %poly : !polynomial.polynomial<ring=#ring1> -> tensor<1024xi16>
+    %tensor = polynomial.to_tensor %poly : !polynomial.polynomial<#ring1> -> tensor<1024xi16>
 
     return
   }
 
-  func.func @test_degree(%p0 : !polynomial.polynomial<ring=#ring1>) {
-    %0, %1 = polynomial.leading_term %p0 : !polynomial.polynomial<ring=#ring1> -> (index, i32)
+  func.func @test_degree(%p0 : !polynomial.polynomial<#ring1>) {
+    %0, %1 = polynomial.leading_term %p0 : !polynomial.polynomial<#ring1> -> (index, i32)
     return
   }
 
   func.func @test_monomial() {
     %deg = arith.constant 1023 : index
     %five = arith.constant 5 : i16
-    %0 = polynomial.monomial %five, %deg : (i16, index) -> !polynomial.polynomial<ring=#ring1>
+    %0 = polynomial.monomial %five, %deg : (i16, index) -> !polynomial.polynomial<#ring1>
     return
   }
 
   func.func @test_monic_monomial_mul() {
     %five = arith.constant 5 : index
-    %0 = polynomial.constant {value=#one_plus_x_squared} : !polynomial.polynomial<ring=#ring1>
-    %1 = polynomial.monic_monomial_mul %0, %five : (!polynomial.polynomial<ring=#ring1>, index) -> !polynomial.polynomial<ring=#ring1>
+    %0 = polynomial.constant #one_plus_x_squared : !polynomial.polynomial<#ring1>
+    %1 = polynomial.monic_monomial_mul %0, %five : (!polynomial.polynomial<#ring1>, index) -> !polynomial.polynomial<#ring1>
     return
   }
 
   func.func @test_constant() {
-    %0 = polynomial.constant {value=#one_plus_x_squared} : !polynomial.polynomial<ring=#ring1>
-    %1 = polynomial.constant {value=#polynomial.int_polynomial<1 + x**2>} : !polynomial.polynomial<ring=#ring1>
-    %2 = polynomial.constant {value=#polynomial.float_polynomial<1.5 + 0.5 x**2>} : !polynomial.polynomial<ring=#ring2>
+    %0 = polynomial.constant #one_plus_x_squared : !polynomial.polynomial<#ring1>
+    %1 = polynomial.constant <1 + x**2> : !polynomial.polynomial<#ring1>
     return
   }
 
diff --git a/mlir/test/Dialect/Polynomial/ops_errors.mlir b/mlir/test/Dialect/Polynomial/ops_errors.mlir
index af8e4aa5da86..2c20e7bcbf1d 100644
--- a/mlir/test/Dialect/Polynomial/ops_errors.mlir
+++ b/mlir/test/Dialect/Polynomial/ops_errors.mlir
@@ -1,8 +1,8 @@
 // RUN: mlir-opt --split-input-file --verify-diagnostics %s
 
-#my_poly = #polynomial.int_polynomial<1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i32, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<ring=#ring>
+#my_poly = #polynomial.polynomial<1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<#ring>
 
 func.func @test_from_tensor_too_large_coeffs() {
   %two = arith.constant 2 : i32
@@ -15,13 +15,13 @@ func.func @test_from_tensor_too_large_coeffs() {
 
 // -----
 
-#my_poly = #polynomial.int_polynomial<1 + x**4>
-#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256:i32, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<ring=#ring>
+#my_poly = #polynomial.polynomial<1 + x**4>
+#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<#ring>
 func.func @test_from_tensor_wrong_tensor_type() {
   %two = arith.constant 2 : i32
   %coeffs1 = tensor.from_elements %two, %two, %two, %two, %two : tensor<5xi32>
-  // expected-error@below {{input type 'tensor<5xi32>' does not match output type '!polynomial.polynomial<ring = <coefficientType = i32, coefficientModulus = 256 : i32, polynomialModulus = <1 + x**4>>>'}}
+  // expected-error@below {{input type 'tensor<5xi32>' does not match output type '!polynomial.polynomial<#polynomial.ring<coefficientType=i32, coefficientModulus=256 : i32, polynomialModulus=#polynomial.polynomial<1 + x**4>>>'}}
   // expected-note@below {{at most the degree of the polynomialModulus of the output type's ring attribute}}
   %poly = polynomial.from_tensor %coeffs1 : tensor<5xi32> -> !ty
   return
@@ -29,11 +29,11 @@ func.func @test_from_tensor_wrong_tensor_type() {
 
 // -----
 
-#my_poly = #polynomial.int_polynomial<1 + x**4>
-#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256:i32, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<ring=#ring>
+#my_poly = #polynomial.polynomial<1 + x**4>
+#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<#ring>
 func.func @test_to_tensor_wrong_output_tensor_type(%arg0 : !ty) {
-  // expected-error@below {{input type '!polynomial.polynomial<ring = <coefficientType = i32, coefficientModulus = 256 : i32, polynomialModulus = <1 + x**4>>>' does not match output type 'tensor<5xi32>'}}
+  // expected-error@below {{input type '!polynomial.polynomial<#polynomial.ring<coefficientType=i32, coefficientModulus=256 : i32, polynomialModulus=#polynomial.polynomial<1 + x**4>>>' does not match output type 'tensor<5xi32>'}}
   // expected-note@below {{at most the degree of the polynomialModulus of the input type's ring attribute}}
   %tensor = polynomial.to_tensor %arg0 : !ty -> tensor<5xi32>
   return
@@ -41,9 +41,9 @@ func.func @test_to_tensor_wrong_output_tensor_type(%arg0 : !ty) {
 
 // -----
 
-#my_poly = #polynomial.int_polynomial<1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i32, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<ring=#ring>
+#my_poly = #polynomial.polynomial<1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<#ring>
 
 func.func @test_mul_scalar_wrong_type(%arg0: !ty) -> !ty {
   %scalar = arith.constant 2 : i32  // should be i16
@@ -54,9 +54,9 @@ func.func @test_mul_scalar_wrong_type(%arg0: !ty) -> !ty {
 
 // -----
 
-#my_poly = #polynomial.int_polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
-!poly_ty = !polynomial.polynomial<ring=#ring>
+#my_poly = #polynomial.polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=31>
+!poly_ty = !polynomial.polynomial<#ring>
 
 // CHECK-NOT: @test_invalid_ntt
 // CHECK-NOT: polynomial.ntt
@@ -68,9 +68,9 @@ func.func @test_invalid_ntt(%0 : !poly_ty) {
 
 // -----
 
-#my_poly = #polynomial.int_polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
-!poly_ty = !polynomial.polynomial<ring=#ring>
+#my_poly = #polynomial.polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=31>
+!poly_ty = !polynomial.polynomial<#ring>
 
 // CHECK-NOT: @test_invalid_ntt
 // CHECK-NOT: polynomial.ntt
@@ -82,10 +82,10 @@ func.func @test_invalid_ntt(%0 : !poly_ty) {
 
 // -----
 
-#my_poly = #polynomial.int_polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly>
-#ring1 = #polynomial.ring<coefficientType=i16, coefficientModulus=257:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
-!poly_ty = !polynomial.polynomial<ring=#ring>
+#my_poly = #polynomial.polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
+#ring1 = #polynomial.ring<coefficientType=i16, coefficientModulus=257, polynomialModulus=#my_poly, primitiveRoot=31>
+!poly_ty = !polynomial.polynomial<#ring>
 
 // CHECK-NOT: @test_invalid_intt
 // CHECK-NOT: polynomial.intt
@@ -97,9 +97,9 @@ func.func @test_invalid_intt(%0 : tensor<1024xi32, #ring1>) {
 
 // -----
 
-#my_poly = #polynomial.int_polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
-!poly_ty = !polynomial.polynomial<ring=#ring>
+#my_poly = #polynomial.polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=31>
+!poly_ty = !polynomial.polynomial<#ring>
 
 // CHECK-NOT: @test_invalid_intt
 // CHECK-NOT: polynomial.intt
@@ -112,9 +112,9 @@ func.func @test_invalid_intt(%0 : tensor<1025xi32, #ring>) {
 
 // -----
 
-#my_poly = #polynomial.int_polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly>
-!poly_ty = !polynomial.polynomial<ring=#ring>
+#my_poly = #polynomial.polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
+!poly_ty = !polynomial.polynomial<#ring>
 
 // CHECK-NOT: @test_invalid_ntt
 // CHECK-NOT: polynomial.ntt
@@ -126,10 +126,10 @@ func.func @test_invalid_ntt(%0 : !poly_ty) {
 
 // -----
 
-#my_poly = #polynomial.int_polynomial<-1 + x**8>
+#my_poly = #polynomial.polynomial<-1 + x**8>
 // A valid root is 31
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=32:i16>
-!poly_ty = !polynomial.polynomial<ring=#ring>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=32>
+!poly_ty = !polynomial.polynomial<#ring>
 
 // CHECK-NOT: @test_invalid_intt
 // CHECK-NOT: polynomial.intt
diff --git a/mlir/test/Dialect/Polynomial/types.mlir b/mlir/test/Dialect/Polynomial/types.mlir
index dcc5663ceb84..00296a36e890 100644
--- a/mlir/test/Dialect/Polynomial/types.mlir
+++ b/mlir/test/Dialect/Polynomial/types.mlir
@@ -2,13 +2,13 @@
 
 // CHECK-LABEL: func @test_types
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    ring = <
-// CHECK-SAME:       coefficientType = i32,
-// CHECK-SAME:       coefficientModulus = 2837465 : i32,
-// CHECK-SAME:       polynomialModulus = <1 + x**1024>>>
-#my_poly = #polynomial.int_polynomial<1 + x**1024>
-#ring1 = #polynomial.ring<coefficientType = i32, coefficientModulus = 2837465 : i32, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<ring=#ring1>
+// CHECK-SAME:    #polynomial.ring<
+// CHECK-SAME:       coefficientType=i32,
+// CHECK-SAME:       coefficientModulus=2837465 : i32,
+// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<1 + x**1024>>>
+#my_poly = #polynomial.polynomial<1 + x**1024>
+#ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<#ring1>
 func.func @test_types(%0: !ty) -> !ty {
   return %0 : !ty
 }
@@ -16,13 +16,13 @@ func.func @test_types(%0: !ty) -> !ty {
 
 // CHECK-LABEL: func @test_non_x_variable_64_bit
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    ring = <
-// CHECK-SAME:       coefficientType = i64,
-// CHECK-SAME:       coefficientModulus = 2837465 : i64,
-// CHECK-SAME:       polynomialModulus = <2 + 4x + x**3>>>
-#my_poly_2 = #polynomial.int_polynomial<t**3 + 4t + 2>
-#ring2 = #polynomial.ring<coefficientType = i64, coefficientModulus = 2837465 : i64, polynomialModulus=#my_poly_2>
-!ty2 = !polynomial.polynomial<ring=#ring2>
+// CHECK-SAME:    #polynomial.ring<
+// CHECK-SAME:       coefficientType=i64,
+// CHECK-SAME:       coefficientModulus=2837465 : i64,
+// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<2 + 4x + x**3>>>
+#my_poly_2 = #polynomial.polynomial<t**3 + 4t + 2>
+#ring2 = #polynomial.ring<coefficientType=i64, coefficientModulus=2837465, polynomialModulus=#my_poly_2>
+!ty2 = !polynomial.polynomial<#ring2>
 func.func @test_non_x_variable_64_bit(%0: !ty2) -> !ty2 {
   return %0 : !ty2
 }
@@ -30,36 +30,27 @@ func.func @test_non_x_variable_64_bit(%0: !ty2) -> !ty2 {
 
 // CHECK-LABEL: func @test_linear_poly
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    ring = <
-// CHECK-SAME:       coefficientType = i32,
-// CHECK-SAME:       coefficientModulus = 12 : i32,
-// CHECK-SAME:       polynomialModulus = <4x>>
-#my_poly_3 = #polynomial.int_polynomial<4x>
-#ring3 = #polynomial.ring<coefficientType = i32, coefficientModulus=12 : i32, polynomialModulus=#my_poly_3>
-!ty3 = !polynomial.polynomial<ring=#ring3>
+// CHECK-SAME:    #polynomial.ring<
+// CHECK-SAME:       coefficientType=i32,
+// CHECK-SAME:       coefficientModulus=12 : i32,
+// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<4x>>
+#my_poly_3 = #polynomial.polynomial<4x>
+#ring3 = #polynomial.ring<coefficientType=i32, coefficientModulus=12, polynomialModulus=#my_poly_3>
+!ty3 = !polynomial.polynomial<#ring3>
 func.func @test_linear_poly(%0: !ty3) -> !ty3 {
   return %0 : !ty3
 }
 
 // CHECK-LABEL: func @test_negative_leading_1
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    ring = <
-// CHECK-SAME:       coefficientType = i32,
-// CHECK-SAME:       coefficientModulus = 2837465 : i32,
-// CHECK-SAME:       polynomialModulus = <-1 + x**1024>>>
-#my_poly_4 = #polynomial.int_polynomial<-1 + x**1024>
-#ring4 = #polynomial.ring<coefficientType = i32, coefficientModulus = 2837465 : i32, polynomialModulus=#my_poly_4>
-!ty4 = !polynomial.polynomial<ring=#ring4>
+// CHECK-SAME:    #polynomial.ring<
+// CHECK-SAME:       coefficientType=i32,
+// CHECK-SAME:       coefficientModulus=2837465 : i32,
+// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<-1 + x**1024>>>
+#my_poly_4 = #polynomial.polynomial<-1 + x**1024>
+#ring4 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly_4>
+!ty4 = !polynomial.polynomial<#ring4>
 func.func @test_negative_leading_1(%0: !ty4) -> !ty4 {
   return %0 : !ty4
 }
 
-// CHECK-LABEL: func @test_float_coefficients
-// CHECK-SAME:  !polynomial.polynomial<ring = <coefficientType = f32>>
-#my_poly_5 = #polynomial.float_polynomial<0.5 + 1.6e03 x**1024>
-#ring5 = #polynomial.ring<coefficientType=f32>
-!ty5 = !polynomial.polynomial<ring=#ring5>
-func.func @test_float_coefficients(%0: !ty5) -> !ty5 {
-  return %0 : !ty5
-}
-
-- 
GitLab


From cf40c93b5be5cd0011ebbf3a9eead224f7b7079a Mon Sep 17 00:00:00 2001
From: Benoit Jacob <jacob.benoit.1@gmail.com>
Date: Mon, 13 May 2024 13:20:30 -0400
Subject: [PATCH 0604/1206] [mlir][vector] Add Vector-dialect
 interleave-to-shuffle pattern, enable in VectorToSPIRV (#91800)

Context: https://github.com/iree-org/iree/issues/17346.

Test IREE integrate showing it's fixing the problem it's intended to
fix, i.e. it allows IREE to drop its local revert of
https://github.com/llvm/llvm-project/pull/89131:

https://github.com/iree-org/iree/pull/17359

This is added to VectorToSPIRV because SPIRV doesn't currently handle
`vector.interleave` (see motivating context above).

This is limited to 1D, non-scalable vectors.
---
 .../Vector/TransformOps/VectorTransformOps.td | 14 +++++++
 .../Vector/Transforms/LoweringPatterns.h      |  3 ++
 .../VectorToSPIRV/VectorToSPIRV.cpp           |  4 ++
 .../TransformOps/VectorTransformOps.cpp       |  5 +++
 .../Transforms/LowerVectorInterleave.cpp      | 41 +++++++++++++++++++
 .../Vector/vector-interleave-to-shuffle.mlir  | 21 ++++++++++
 6 files changed, 88 insertions(+)
 create mode 100644 mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir

diff --git a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
index f6371f39c394..bc3c16d40520 100644
--- a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
+++ b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
@@ -306,6 +306,20 @@ def ApplyLowerInterleavePatternsOp : Op<Transform_Dialect,
   let assemblyFormat = "attr-dict";
 }
 
+def ApplyInterleaveToShufflePatternsOp : Op<Transform_Dialect,
+    "apply_patterns.vector.interleave_to_shuffle",
+    [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
+  let description = [{
+    Indicates that 1D vector interleave operations should be rewritten as
+    vector shuffle operations.
+
+    This is motivated by some current codegen backends not handling vector
+    interleave operations.
+  }];
+
+  let assemblyFormat = "attr-dict";
+}
+
 def ApplyRewriteNarrowTypePatternsOp : Op<Transform_Dialect,
     "apply_patterns.vector.rewrite_narrow_types",
     [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
index 350d2777cadf..8fd9904fabc0 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
@@ -273,6 +273,9 @@ void populateVectorInterleaveLoweringPatterns(RewritePatternSet &patterns,
                                               int64_t targetRank = 1,
                                               PatternBenefit benefit = 1);
 
+void populateVectorInterleaveToShufflePatterns(RewritePatternSet &patterns,
+                                               PatternBenefit benefit = 1);
+
 } // namespace vector
 } // namespace mlir
 #endif // MLIR_DIALECT_VECTOR_TRANSFORMS_LOWERINGPATTERNS_H
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index 868a3521e7a0..c2dd37f48146 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
@@ -828,6 +829,9 @@ void mlir::populateVectorToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
   // than the generic one that extracts all elements.
   patterns.add<VectorReductionToFPDotProd>(typeConverter, patterns.getContext(),
                                            PatternBenefit(2));
+
+  // Need this until vector.interleave is handled.
+  vector::populateVectorInterleaveToShufflePatterns(patterns);
 }
 
 void mlir::populateVectorReductionToSPIRVDotProductPatterns(
diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
index 885644864c0f..61fd6bd972e3 100644
--- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
+++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
@@ -164,6 +164,11 @@ void transform::ApplyLowerInterleavePatternsOp::populatePatterns(
   vector::populateVectorInterleaveLoweringPatterns(patterns);
 }
 
+void transform::ApplyInterleaveToShufflePatternsOp::populatePatterns(
+    RewritePatternSet &patterns) {
+  vector::populateVectorInterleaveToShufflePatterns(patterns);
+}
+
 void transform::ApplyRewriteNarrowTypePatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
   populateVectorNarrowTypeRewritePatterns(patterns);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
index 3a456076f8fb..5326760c9b4e 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/PatternMatch.h"
+#include "mlir/Support/LogicalResult.h"
 
 #define DEBUG_TYPE "vector-interleave-lowering"
 
@@ -77,9 +78,49 @@ private:
   int64_t targetRank = 1;
 };
 
+/// Rewrite vector.interleave op into an equivalent vector.shuffle op, when
+/// applicable: `sourceType` must be 1D and non-scalable.
+///
+/// Example:
+///
+/// ```mlir
+/// vector.interleave %a, %b : vector<7xi16>
+/// ```
+///
+/// Is rewritten into:
+///
+/// ```mlir
+/// vector.shuffle %arg0, %arg1 [0, 7, 1, 8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13]
+///   : vector<7xi16>, vector<7xi16>
+/// ```
+class InterleaveToShuffle : public OpRewritePattern<vector::InterleaveOp> {
+public:
+  InterleaveToShuffle(MLIRContext *context, PatternBenefit benefit = 1)
+      : OpRewritePattern(context, benefit) {};
+
+  LogicalResult matchAndRewrite(vector::InterleaveOp op,
+                                PatternRewriter &rewriter) const override {
+    VectorType sourceType = op.getSourceVectorType();
+    if (sourceType.getRank() != 1 || sourceType.isScalable()) {
+      return failure();
+    }
+    int64_t n = sourceType.getNumElements();
+    auto seq = llvm::seq<int64_t>(2 * n);
+    auto zip = llvm::to_vector(llvm::map_range(
+        seq, [n](int64_t i) { return (i % 2 ? n : 0) + i / 2; }));
+    rewriter.replaceOpWithNewOp<ShuffleOp>(op, op.getLhs(), op.getRhs(), zip);
+    return success();
+  }
+};
+
 } // namespace
 
 void mlir::vector::populateVectorInterleaveLoweringPatterns(
     RewritePatternSet &patterns, int64_t targetRank, PatternBenefit benefit) {
   patterns.add<UnrollInterleaveOp>(targetRank, patterns.getContext(), benefit);
 }
+
+void mlir::vector::populateVectorInterleaveToShufflePatterns(
+    RewritePatternSet &patterns, PatternBenefit benefit) {
+  patterns.add<InterleaveToShuffle>(patterns.getContext(), benefit);
+}
diff --git a/mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir b/mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir
new file mode 100644
index 000000000000..ed3b3396bf3e
--- /dev/null
+++ b/mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir
@@ -0,0 +1,21 @@
+// RUN: mlir-opt %s --transform-interpreter | FileCheck %s
+
+// CHECK-LABEL: @vector_interleave_to_shuffle
+func.func @vector_interleave_to_shuffle(%a: vector<7xi16>, %b: vector<7xi16>) -> vector<14xi16>
+{
+  %0 = vector.interleave %a, %b : vector<7xi16>
+  return %0 : vector<14xi16>
+}
+// CHECK: vector.shuffle %arg0, %arg1 [0, 7, 1, 8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13] : vector<7xi16>, vector<7xi16>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+    %f = transform.structured.match ops{["func.func"]} in %module_op
+      : (!transform.any_op) -> !transform.any_op
+
+    transform.apply_patterns to %f {
+      transform.apply_patterns.vector.interleave_to_shuffle
+    } : !transform.any_op
+    transform.yield
+  }
+}
-- 
GitLab


From 0f7906645d18a38a6b80a1e8e1d425396f6ab353 Mon Sep 17 00:00:00 2001
From: Felix Schneider <fx.schn@gmail.com>
Date: Mon, 13 May 2024 19:27:38 +0200
Subject: [PATCH 0605/1206] [mlir][intrange] Fix `arith.shl` inference in case
 of overflow (#91737)

When an overflow happens during shift left, i.e. the last sign bit or
the most significant data bit gets shifted out, the current approach of
inferring the range of results does not work anymore.

This patch checks for possible overflow and returns the max range in
that case.

Fix https://github.com/llvm/llvm-project/issues/82158
---
 .../Interfaces/Utils/InferIntRangeCommon.cpp  | 19 ++++++++++--
 mlir/test/Dialect/Arith/int-range-opts.mlir   | 29 +++++++++++++++++++
 2 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp b/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
index 2b2d937d55d8..6af229cae10a 100644
--- a/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
+++ b/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
@@ -544,15 +544,30 @@ mlir::intrange::inferXor(ArrayRef<ConstantIntRanges> argRanges) {
 ConstantIntRanges
 mlir::intrange::inferShl(ArrayRef<ConstantIntRanges> argRanges) {
   const ConstantIntRanges &lhs = argRanges[0], &rhs = argRanges[1];
+  const APInt &lhsSMin = lhs.smin(), &lhsSMax = lhs.smax(),
+              &lhsUMax = lhs.umax(), &rhsUMin = rhs.umin(),
+              &rhsUMax = rhs.umax();
+
   ConstArithFn shl = [](const APInt &l,
                         const APInt &r) -> std::optional<APInt> {
     return r.uge(r.getBitWidth()) ? std::optional<APInt>() : l.shl(r);
   };
+
+  // The minMax inference does not work when there is danger of overflow. In the
+  // signed case, this leads to the obvious problem that the sign bit might
+  // change. In the unsigned case, it also leads to problems because the largest
+  // LHS shifted by the largest RHS does not necessarily result in the largest
+  // result anymore.
+  assert(rhsUMax.isNonNegative() && "Unexpected negative shift count");
+  if (rhsUMax.uge(lhsSMin.getNumSignBits()) ||
+      rhsUMax.uge(lhsSMax.getNumSignBits()))
+    return ConstantIntRanges::maxRange(lhsUMax.getBitWidth());
+
   ConstantIntRanges urange =
-      minMaxBy(shl, {lhs.umin(), lhs.umax()}, {rhs.umin(), rhs.umax()},
+      minMaxBy(shl, {lhs.umin(), lhsUMax}, {rhsUMin, rhsUMax},
                /*isSigned=*/false);
   ConstantIntRanges srange =
-      minMaxBy(shl, {lhs.smin(), lhs.smax()}, {rhs.umin(), rhs.umax()},
+      minMaxBy(shl, {lhsSMin, lhsSMax}, {rhsUMin, rhsUMax},
                /*isSigned=*/true);
   return urange.intersection(srange);
 }
diff --git a/mlir/test/Dialect/Arith/int-range-opts.mlir b/mlir/test/Dialect/Arith/int-range-opts.mlir
index be0a7e8ccd70..4c3c0854ed02 100644
--- a/mlir/test/Dialect/Arith/int-range-opts.mlir
+++ b/mlir/test/Dialect/Arith/int-range-opts.mlir
@@ -71,3 +71,32 @@ func.func @test() -> i1 {
   %1 = arith.cmpi sle, %0, %cst1 : index
   return %1: i1
 }
+
+// -----
+
+// CHECK-LABEL: func @test
+// CHECK: test.reflect_bounds {smax = 24 : index, smin = 0 : index, umax = 24 : index, umin = 0 : index}
+func.func @test() -> index {
+  %cst1 = arith.constant 1 : i8
+  %0 = test.with_bounds { umin = 0 : index, umax = 12 : index, smin = 0 : index, smax = 12 : index }
+  %i8val = arith.index_cast %0 : index to i8
+  %shifted = arith.shli %i8val, %cst1 : i8
+  %si = arith.index_cast %shifted : i8 to index
+  %1 = test.reflect_bounds %si
+  return %1: index
+}
+
+// -----
+
+// CHECK-LABEL: func @test
+// CHECK: test.reflect_bounds {smax = 127 : index, smin = -128 : index, umax = -1 : index, umin = 0 : index}
+func.func @test() -> index {
+  %cst1 = arith.constant 1 : i8
+  %0 = test.with_bounds { umin = 0 : index, umax = 127 : index, smin = 0 : index, smax = 127 : index }
+  %i8val = arith.index_cast %0 : index to i8
+  %shifted = arith.shli %i8val, %cst1 : i8
+  %si = arith.index_cast %shifted : i8 to index
+  %1 = test.reflect_bounds %si
+  return %1: index
+}
+
-- 
GitLab


From b8f802f783db481ecfd953c9abe74212a8348aff Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Mon, 13 May 2024 14:28:05 -0300
Subject: [PATCH 0606/1206] [clang] Allow pack expansions when partial ordering
 against template template parameters (#91833)

When partial ordering alias templates against template template
parameters, allow pack expansions when the alias has a fixed-size
parameter list.

These expansions were generally disallowed by proposed resolution for
CWG1430.

By previously diagnosing these when checking template template
parameters, we would be too strict in trying to prevent any potential
invalid use.

This flows against the more general idea that template template
parameters are weakly typed, that we would rather allow an argument that
might be possibly misused, and only diagnose the actual misuses during
instantiation.

Since this interaction between P0522R0 and CWG1430 is also a
backwards-compat breaking change, we implement provisional wording to
allow these.

Fixes https://github.com/llvm/llvm-project/issues/62529
---
 clang/docs/ReleaseNotes.rst                        |  2 ++
 clang/include/clang/Sema/Sema.h                    |  8 +++++++-
 clang/lib/Sema/SemaTemplate.cpp                    | 14 ++++++++++----
 clang/lib/Sema/SemaTemplateDeduction.cpp           |  4 +++-
 ...plate_cxx1z.cpp => temp_arg_template_p0522.cpp} | 12 ++++++++++--
 5 files changed, 32 insertions(+), 8 deletions(-)
 rename clang/test/SemaTemplate/{temp_arg_template_cxx1z.cpp => temp_arg_template_p0522.cpp} (91%)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 4702b8c10cdb..28ac54127383 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -708,6 +708,8 @@ Bug Fixes to C++ Support
   expression.
 - Fix a bug in access control checking due to dealyed checking of friend declaration. Fixes (#GH12361).
 - Correctly treat the compound statement of an ``if consteval`` as an immediate context. Fixes (#GH91509).
+- When partial ordering alias templates against template template parameters,
+  allow pack expansions when the alias has a fixed-size parameter list. Fixes (#GH62529).
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 4efd3878e861..869769f95fd7 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -9216,6 +9216,12 @@ public:
   /// receive true if the cause for the error is the associated constraints of
   /// the template not being satisfied by the template arguments.
   ///
+  /// \param PartialOrderingTTP If true, assume these template arguments are
+  /// the injected template arguments for a template template parameter.
+  /// This will relax the requirement that all its possible uses are valid:
+  /// TTP checking is loose, and assumes that invalid uses will be diagnosed
+  /// during instantiation.
+  ///
   /// \returns true if an error occurred, false otherwise.
   bool CheckTemplateArgumentList(
       TemplateDecl *Template, SourceLocation TemplateLoc,
@@ -9223,7 +9229,7 @@ public:
       SmallVectorImpl<TemplateArgument> &SugaredConverted,
       SmallVectorImpl<TemplateArgument> &CanonicalConverted,
       bool UpdateArgsWithConversions = true,
-      bool *ConstraintsNotSatisfied = nullptr);
+      bool *ConstraintsNotSatisfied = nullptr, bool PartialOrderingTTP = false);
 
   bool CheckTemplateTypeArgument(
       TemplateTypeParmDecl *Param, TemplateArgumentLoc &Arg,
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index bae00c629270..8219d5eed8db 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -6556,7 +6556,8 @@ bool Sema::CheckTemplateArgumentList(
     TemplateArgumentListInfo &TemplateArgs, bool PartialTemplateArgs,
     SmallVectorImpl<TemplateArgument> &SugaredConverted,
     SmallVectorImpl<TemplateArgument> &CanonicalConverted,
-    bool UpdateArgsWithConversions, bool *ConstraintsNotSatisfied) {
+    bool UpdateArgsWithConversions, bool *ConstraintsNotSatisfied,
+    bool PartialOrderingTTP) {
 
   if (ConstraintsNotSatisfied)
     *ConstraintsNotSatisfied = false;
@@ -6627,9 +6628,14 @@ bool Sema::CheckTemplateArgumentList(
       bool PackExpansionIntoNonPack =
           NewArgs[ArgIdx].getArgument().isPackExpansion() &&
           (!(*Param)->isTemplateParameterPack() || getExpandedPackSize(*Param));
-      if (PackExpansionIntoNonPack && (isa<TypeAliasTemplateDecl>(Template) ||
-                                       isa<ConceptDecl>(Template))) {
-        // Core issue 1430: we have a pack expansion as an argument to an
+      // CWG1430: Don't diagnose this pack expansion when partial
+      // ordering template template parameters. Some uses of the template could
+      // be valid, and invalid uses will be diagnosed later during
+      // instantiation.
+      if (PackExpansionIntoNonPack && !PartialOrderingTTP &&
+          (isa<TypeAliasTemplateDecl>(Template) ||
+           isa<ConceptDecl>(Template))) {
+        // CWG1430: we have a pack expansion as an argument to an
         // alias template, and it's not part of a parameter pack. This
         // can't be canonicalized, so reject it now.
         // As for concepts - we cannot normalize constraints where this
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index fe7e35d84151..853c0e1b5061 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -6243,7 +6243,9 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
     //   specialized as A.
     SmallVector<TemplateArgument, 4> SugaredPArgs;
     if (CheckTemplateArgumentList(AArg, Loc, PArgList, false, SugaredPArgs,
-                                  PArgs) ||
+                                  PArgs, /*UpdateArgsWithConversions=*/true,
+                                  /*ConstraintsNotSatisfied=*/nullptr,
+                                  /*PartialOrderTTP=*/true) ||
         Trap.hasErrorOccurred())
       return false;
   }
diff --git a/clang/test/SemaTemplate/temp_arg_template_cxx1z.cpp b/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
similarity index 91%
rename from clang/test/SemaTemplate/temp_arg_template_cxx1z.cpp
rename to clang/test/SemaTemplate/temp_arg_template_p0522.cpp
index 372a00efc601..251b6fc7d2be 100644
--- a/clang/test/SemaTemplate/temp_arg_template_cxx1z.cpp
+++ b/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -std=c++1z %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++20 %s
 
-// expected-note@temp_arg_template_cxx1z.cpp:* 1+{{}}
+// expected-note@temp_arg_template_p0522.cpp:* 1+{{}}
 
 template<template<int> typename> struct Ti;
 template<template<int...> typename> struct TPi;
@@ -118,3 +118,11 @@ namespace Auto {
   TInt<SubstFailure> isf; // FIXME: this should be ill-formed
   TIntPtr<SubstFailure> ipsf;
 }
+
+namespace GH62529 {
+  // Note: the constraint here is just for bypassing a fast-path.
+  template<class T1> requires(true) using A = int;
+  template<template<class ...T2s> class TT1, class T3> struct B {};
+  template<class T4> B<A, T4> f();
+  auto t = f<int>();
+} // namespace GH62529
-- 
GitLab


From 8ef2011b2cd3a8fc2ef8d6ea0facb1a39a0dd621 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Mon, 13 May 2024 08:43:15 -0700
Subject: [PATCH 0607/1206] Reapply "[OpenACC] device_type clause Sema for
 Compute constructs"

device_type, also spelled as dtype, specifies the applicability of the
clauses following it, and takes a series of identifiers representing the
architectures it applies to.  As we don't have a source for the valid
architectures yet, this patch just accepts all.

Semantically, this also limits the list of clauses that can be applied
after the device_type, so this implements that as well.

This reverts commit 06f04b2e27f2586d3db2204ed4e54f8b78fea74e.
This reapplies commit c4a9a374749deb5f2a932a7d4ef9321be1b2ae5d.
The build failures were caused by the patch depending on the order of
evaluation of arguments to a function. This reapplication separates out
the capture of one of the values.
---
 clang/include/clang/AST/OpenACCClause.h       |  59 +++++
 .../clang/Basic/DiagnosticSemaKinds.td        |   4 +
 clang/include/clang/Basic/OpenACCClauses.def  |   2 +
 clang/include/clang/Parse/Parser.h            |   3 +-
 clang/include/clang/Sema/SemaOpenACC.h        |  23 +-
 clang/lib/AST/OpenACCClause.cpp               |  28 ++-
 clang/lib/AST/StmtProfile.cpp                 |   3 +
 clang/lib/AST/TextNodeDumper.cpp              |  13 ++
 clang/lib/Parse/ParseOpenACC.cpp              |  23 +-
 clang/lib/Sema/SemaOpenACC.cpp                |  55 +++++
 clang/lib/Sema/TreeTransform.h                |  10 +
 clang/lib/Serialization/ASTReader.cpp         |  17 +-
 clang/lib/Serialization/ASTWriter.cpp         |  15 +-
 .../ast-print-openacc-compute-construct.cpp   |  23 ++
 clang/test/ParserOpenACC/parse-clauses.c      |  28 +--
 .../compute-construct-device_type-ast.cpp     | 105 +++++++++
 .../compute-construct-device_type-clause.c    | 221 ++++++++++++++++++
 .../compute-construct-device_type-clause.cpp  |  25 ++
 clang/tools/libclang/CIndex.cpp               |   2 +
 19 files changed, 624 insertions(+), 35 deletions(-)
 create mode 100644 clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp
 create mode 100644 clang/test/SemaOpenACC/compute-construct-device_type-clause.c
 create mode 100644 clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp

diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 3d0b1ab9d31e..607a2b9d6536 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -17,6 +17,8 @@
 #include "clang/AST/StmtIterator.h"
 #include "clang/Basic/OpenACCKinds.h"
 
+#include <utility>
+
 namespace clang {
 /// This is the base type for all OpenACC Clauses.
 class OpenACCClause {
@@ -75,6 +77,63 @@ public:
   }
 };
 
+using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
+/// A 'device_type' or 'dtype' clause, takes a list of either an 'asterisk' or
+/// an identifier. The 'asterisk' means 'the rest'.
+class OpenACCDeviceTypeClause final
+    : public OpenACCClauseWithParams,
+      public llvm::TrailingObjects<OpenACCDeviceTypeClause,
+                                   DeviceTypeArgument> {
+  // Data stored in trailing objects as IdentifierInfo* /SourceLocation pairs. A
+  // nullptr IdentifierInfo* represents an asterisk.
+  unsigned NumArchs;
+  OpenACCDeviceTypeClause(OpenACCClauseKind K, SourceLocation BeginLoc,
+                          SourceLocation LParenLoc,
+                          ArrayRef<DeviceTypeArgument> Archs,
+                          SourceLocation EndLoc)
+      : OpenACCClauseWithParams(K, BeginLoc, LParenLoc, EndLoc),
+        NumArchs(Archs.size()) {
+    assert(
+        (K == OpenACCClauseKind::DeviceType || K == OpenACCClauseKind::DType) &&
+        "Invalid clause kind for device-type");
+
+    assert(!llvm::any_of(Archs, [](const DeviceTypeArgument &Arg) {
+      return Arg.second.isInvalid();
+    }) && "Invalid SourceLocation for an argument");
+
+    assert(
+        (Archs.size() == 1 || !llvm::any_of(Archs,
+                                            [](const DeviceTypeArgument &Arg) {
+                                              return Arg.first == nullptr;
+                                            })) &&
+        "Only a single asterisk version is permitted, and must be the "
+        "only one");
+
+    std::uninitialized_copy(Archs.begin(), Archs.end(),
+                            getTrailingObjects<DeviceTypeArgument>());
+  }
+
+public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::DType ||
+           C->getClauseKind() == OpenACCClauseKind::DeviceType;
+  }
+  bool hasAsterisk() const {
+    return getArchitectures().size() > 0 &&
+           getArchitectures()[0].first == nullptr;
+  }
+
+  ArrayRef<DeviceTypeArgument> getArchitectures() const {
+    return ArrayRef<DeviceTypeArgument>(
+        getTrailingObjects<DeviceTypeArgument>(), NumArchs);
+  }
+
+  static OpenACCDeviceTypeClause *
+  Create(const ASTContext &C, OpenACCClauseKind K, SourceLocation BeginLoc,
+         SourceLocation LParenLoc, ArrayRef<DeviceTypeArgument> Archs,
+         SourceLocation EndLoc);
+};
+
 /// A 'default' clause, has the optional 'none' or 'present' argument.
 class OpenACCDefaultClause : public OpenACCClauseWithParams {
   friend class ASTReaderStmt;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 9e82130c9360..6100fba51005 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12344,4 +12344,8 @@ def warn_acc_deprecated_alias_name
 def err_acc_var_not_pointer_type
     : Error<"expected pointer in '%0' clause, type is %1">;
 def note_acc_expected_pointer_var : Note<"expected variable of pointer type">;
+def err_acc_clause_after_device_type
+    : Error<"OpenACC clause '%0' may not follow a '%1' clause in a "
+            "compute construct">;
+
 } // end of sema component.
diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def
index afb7b30b7465..7ecc51799468 100644
--- a/clang/include/clang/Basic/OpenACCClauses.def
+++ b/clang/include/clang/Basic/OpenACCClauses.def
@@ -37,6 +37,8 @@ CLAUSE_ALIAS(PCreate, Create)
 CLAUSE_ALIAS(PresentOrCreate, Create)
 VISIT_CLAUSE(Default)
 VISIT_CLAUSE(DevicePtr)
+VISIT_CLAUSE(DeviceType)
+CLAUSE_ALIAS(DType, DeviceType)
 VISIT_CLAUSE(FirstPrivate)
 VISIT_CLAUSE(If)
 VISIT_CLAUSE(NoCreate)
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 61589fb7766f..3910cba34a21 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3720,7 +3720,8 @@ private:
                                SourceLocation Loc,
                                llvm::SmallVectorImpl<Expr *> &IntExprs);
   /// Parses the 'device-type-list', which is a list of identifiers.
-  bool ParseOpenACCDeviceTypeList();
+  bool ParseOpenACCDeviceTypeList(
+      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs);
   /// Parses the 'async-argument', which is an integral value with two
   /// 'special' values that are likely negative (but come from Macros).
   OpenACCIntExprParseResult ParseOpenACCAsyncArgument(OpenACCDirectiveKind DK,
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index e684ee6b2be1..f838fa97d33a 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -26,6 +26,9 @@ class OpenACCClause;
 
 class SemaOpenACC : public SemaBase {
 public:
+  // Redeclaration of the version in OpenACCClause.h.
+  using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
+
   /// A type to represent all the data for an OpenACC Clause that has been
   /// parsed, but not yet created/semantically analyzed. This is effectively a
   /// discriminated union on the 'Clause Kind', with all of the individual
@@ -60,8 +63,12 @@ public:
       SmallVector<Expr *> QueueIdExprs;
     };
 
+    struct DeviceTypeDetails {
+      SmallVector<DeviceTypeArgument> Archs;
+    };
+
     std::variant<std::monostate, DefaultDetails, ConditionDetails,
-                 IntExprDetails, VarListDetails, WaitDetails>
+                 IntExprDetails, VarListDetails, WaitDetails, DeviceTypeDetails>
         Details = std::monostate{};
 
   public:
@@ -209,6 +216,13 @@ public:
       return std::get<VarListDetails>(Details).IsZero;
     }
 
+    ArrayRef<DeviceTypeArgument> getDeviceTypeArchitectures() const {
+      assert((ClauseKind == OpenACCClauseKind::DeviceType ||
+              ClauseKind == OpenACCClauseKind::DType) &&
+             "Only 'device_type'/'dtype' has a device-type-arg list");
+      return std::get<DeviceTypeDetails>(Details).Archs;
+    }
+
     void setLParenLoc(SourceLocation EndLoc) { LParenLoc = EndLoc; }
     void setEndLoc(SourceLocation EndLoc) { ClauseRange.setEnd(EndLoc); }
 
@@ -326,6 +340,13 @@ public:
              "Parsed clause kind does not have a wait-details");
       Details = WaitDetails{DevNum, QueuesLoc, std::move(IntExprs)};
     }
+
+    void setDeviceTypeDetails(llvm::SmallVector<DeviceTypeArgument> &&Archs) {
+      assert((ClauseKind == OpenACCClauseKind::DeviceType ||
+              ClauseKind == OpenACCClauseKind::DType) &&
+             "Only 'device_type'/'dtype' has a device-type-arg list");
+      Details = DeviceTypeDetails{std::move(Archs)};
+    }
   };
 
   SemaOpenACC(Sema &S);
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index ee13437b97b4..f80ecc90d396 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -18,7 +18,8 @@
 using namespace clang;
 
 bool OpenACCClauseWithParams::classof(const OpenACCClause *C) {
-  return OpenACCClauseWithCondition::classof(C) ||
+  return OpenACCDeviceTypeClause::classof(C) ||
+         OpenACCClauseWithCondition::classof(C) ||
          OpenACCClauseWithExprs::classof(C);
 }
 bool OpenACCClauseWithExprs::classof(const OpenACCClause *C) {
@@ -298,6 +299,17 @@ OpenACCCreateClause::Create(const ASTContext &C, OpenACCClauseKind Spelling,
                                        VarList, EndLoc);
 }
 
+OpenACCDeviceTypeClause *OpenACCDeviceTypeClause::Create(
+    const ASTContext &C, OpenACCClauseKind K, SourceLocation BeginLoc,
+    SourceLocation LParenLoc, ArrayRef<DeviceTypeArgument> Archs,
+    SourceLocation EndLoc) {
+  void *Mem =
+      C.Allocate(OpenACCDeviceTypeClause::totalSizeToAlloc<DeviceTypeArgument>(
+          Archs.size()));
+  return new (Mem)
+      OpenACCDeviceTypeClause(K, BeginLoc, LParenLoc, Archs, EndLoc);
+}
+
 //===----------------------------------------------------------------------===//
 //  OpenACC clauses printing methods
 //===----------------------------------------------------------------------===//
@@ -451,3 +463,17 @@ void OpenACCClausePrinter::VisitWaitClause(const OpenACCWaitClause &C) {
     OS << ")";
   }
 }
+
+void OpenACCClausePrinter::VisitDeviceTypeClause(
+    const OpenACCDeviceTypeClause &C) {
+  OS << C.getClauseKind();
+  OS << "(";
+  llvm::interleaveComma(C.getArchitectures(), OS,
+                        [&](const DeviceTypeArgument &Arch) {
+                          if (Arch.first == nullptr)
+                            OS << "*";
+                          else
+                            OS << Arch.first;
+                        });
+  OS << ")";
+}
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 8fb8940142eb..caab4ab0ef16 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2585,6 +2585,9 @@ void OpenACCClauseProfiler::VisitWaitClause(const OpenACCWaitClause &Clause) {
   for (auto *E : Clause.getQueueIdExprs())
     Profiler.VisitStmt(E);
 }
+/// Nothing to do here, there are no sub-statements.
+void OpenACCClauseProfiler::VisitDeviceTypeClause(
+    const OpenACCDeviceTypeClause &Clause) {}
 } // namespace
 
 void StmtProfiler::VisitOpenACCComputeConstruct(
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 12aa5858b798..efcd74717a4e 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -444,6 +444,19 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
       if (cast<OpenACCWaitClause>(C)->hasQueuesTag())
         OS << " has queues tag";
       break;
+    case OpenACCClauseKind::DeviceType:
+    case OpenACCClauseKind::DType:
+      OS << "(";
+      llvm::interleaveComma(
+          cast<OpenACCDeviceTypeClause>(C)->getArchitectures(), OS,
+          [&](const DeviceTypeArgument &Arch) {
+            if (Arch.first == nullptr)
+              OS << "*";
+            else
+              OS << Arch.first->getName();
+          });
+      OS << ")";
+      break;
     default:
       // Nothing to do here.
       break;
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 0e10632c8317..5db3036b0003 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -711,14 +711,16 @@ bool Parser::ParseOpenACCIntExprList(OpenACCDirectiveKind DK,
 /// device_type( device-type-list )
 ///
 /// The device_type clause may be abbreviated to dtype.
-bool Parser::ParseOpenACCDeviceTypeList() {
+bool Parser::ParseOpenACCDeviceTypeList(
+    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> &Archs) {
 
   if (expectIdentifierOrKeyword(*this)) {
     SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
               Parser::StopBeforeMatch);
-    return false;
+    return true;
   }
-  ConsumeToken();
+  IdentifierInfo *Ident = getCurToken().getIdentifierInfo();
+  Archs.emplace_back(Ident, ConsumeToken());
 
   while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) {
     ExpectAndConsume(tok::comma);
@@ -726,9 +728,10 @@ bool Parser::ParseOpenACCDeviceTypeList() {
     if (expectIdentifierOrKeyword(*this)) {
       SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
                 Parser::StopBeforeMatch);
-      return false;
+      return true;
     }
-    ConsumeToken();
+    Ident = getCurToken().getIdentifierInfo();
+    Archs.emplace_back(Ident, ConsumeToken());
   }
   return false;
 }
@@ -1021,16 +1024,20 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
       break;
     }
     case OpenACCClauseKind::DType:
-    case OpenACCClauseKind::DeviceType:
+    case OpenACCClauseKind::DeviceType: {
+      llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>> Archs;
       if (getCurToken().is(tok::star)) {
         // FIXME: We want to mark that this is an 'everything else' type of
         // device_type in Sema.
-        ConsumeToken();
-      } else if (ParseOpenACCDeviceTypeList()) {
+        ParsedClause.setDeviceTypeDetails({{nullptr, ConsumeToken()}});
+      } else if (!ParseOpenACCDeviceTypeList(Archs)) {
+        ParsedClause.setDeviceTypeDetails(std::move(Archs));
+      } else {
         Parens.skipToEnd();
         return OpenACCCanContinue();
       }
       break;
+    }
     case OpenACCClauseKind::Tile:
       if (ParseOpenACCSizeExprList()) {
         Parens.skipToEnd();
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index 656d30947a8d..f174b2fa63c6 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -255,6 +255,33 @@ bool checkAlreadyHasClauseOfKind(
   return false;
 }
 
+/// Implement check from OpenACC3.3: section 2.5.4:
+/// Only the async, wait, num_gangs, num_workers, and vector_length clauses may
+/// follow a device_type clause.
+bool checkValidAfterDeviceType(
+    SemaOpenACC &S, const OpenACCDeviceTypeClause &DeviceTypeClause,
+    const SemaOpenACC::OpenACCParsedClause &NewClause) {
+  // This is only a requirement on compute constructs so far, so this is fine
+  // otherwise.
+  if (!isOpenACCComputeDirectiveKind(NewClause.getDirectiveKind()))
+    return false;
+  switch (NewClause.getClauseKind()) {
+  case OpenACCClauseKind::Async:
+  case OpenACCClauseKind::Wait:
+  case OpenACCClauseKind::NumGangs:
+  case OpenACCClauseKind::NumWorkers:
+  case OpenACCClauseKind::VectorLength:
+  case OpenACCClauseKind::DType:
+  case OpenACCClauseKind::DeviceType:
+    return false;
+  default:
+    S.Diag(NewClause.getBeginLoc(), diag::err_acc_clause_after_device_type)
+        << NewClause.getClauseKind() << DeviceTypeClause.getClauseKind();
+    S.Diag(DeviceTypeClause.getBeginLoc(), diag::note_acc_previous_clause_here);
+    return true;
+  }
+}
+
 } // namespace
 
 SemaOpenACC::SemaOpenACC(Sema &S) : SemaBase(S) {}
@@ -273,6 +300,17 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
     return nullptr;
   }
 
+  if (const auto *DevTypeClause =
+          llvm::find_if(ExistingClauses,
+                        [&](const OpenACCClause *C) {
+                          return isa<OpenACCDeviceTypeClause>(C);
+                        });
+      DevTypeClause != ExistingClauses.end()) {
+    if (checkValidAfterDeviceType(
+            *this, *cast<OpenACCDeviceTypeClause>(*DevTypeClause), Clause))
+      return nullptr;
+  }
+
   switch (Clause.getClauseKind()) {
   case OpenACCClauseKind::Default: {
     // Restrictions only properly implemented on 'compute' constructs, and
@@ -651,6 +689,23 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
         Clause.getDevNumExpr(), Clause.getQueuesLoc(), Clause.getQueueIdExprs(),
         Clause.getEndLoc());
   }
+  case OpenACCClauseKind::DType:
+  case OpenACCClauseKind::DeviceType: {
+    // Restrictions only properly implemented on 'compute' constructs, and
+    // 'compute' constructs are the only construct that can do anything with
+    // this yet, so skip/treat as unimplemented in this case.
+    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+      break;
+
+    // TODO OpenACC: Once we get enough of the CodeGen implemented that we have
+    // a source for the list of valid architectures, we need to warn on unknown
+    // identifiers here.
+
+    return OpenACCDeviceTypeClause::Create(
+        getASTContext(), Clause.getClauseKind(), Clause.getBeginLoc(),
+        Clause.getLParenLoc(), Clause.getDeviceTypeArchitectures(),
+        Clause.getEndLoc());
+  }
   default:
     break;
   }
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 126965088831..ab26d1b1199a 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -11480,6 +11480,16 @@ void OpenACCClauseTransform<Derived>::VisitWaitClause(
       ParsedClause.getQueuesLoc(), ParsedClause.getQueueIdExprs(),
       ParsedClause.getEndLoc());
 }
+
+template <typename Derived>
+void OpenACCClauseTransform<Derived>::VisitDeviceTypeClause(
+    const OpenACCDeviceTypeClause &C) {
+  // Nothing to transform here, just create a new version of 'C'.
+  NewClause = OpenACCDeviceTypeClause::Create(
+      Self.getSema().getASTContext(), C.getClauseKind(),
+      ParsedClause.getBeginLoc(), ParsedClause.getLParenLoc(),
+      C.getArchitectures(), ParsedClause.getEndLoc());
+}
 } // namespace
 template <typename Derived>
 OpenACCClause *TreeTransform<Derived>::TransformOpenACCClause(
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 7627996d2c32..8f437a7c5f50 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11905,6 +11905,21 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
                                      DevNumExpr, QueuesLoc, QueueIdExprs,
                                      EndLoc);
   }
+  case OpenACCClauseKind::DeviceType:
+  case OpenACCClauseKind::DType: {
+    SourceLocation LParenLoc = readSourceLocation();
+    llvm::SmallVector<DeviceTypeArgument> Archs;
+    unsigned NumArchs = readInt();
+
+    for (unsigned I = 0; I < NumArchs; ++I) {
+      IdentifierInfo *Ident = readBool() ? readIdentifier() : nullptr;
+      SourceLocation Loc = readSourceLocation();
+      Archs.emplace_back(Ident, Loc);
+    }
+
+    return OpenACCDeviceTypeClause::Create(getContext(), ClauseKind, BeginLoc,
+                                           LParenLoc, Archs, EndLoc);
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -11926,8 +11941,6 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
   case OpenACCClauseKind::Bind:
   case OpenACCClauseKind::DeviceNum:
   case OpenACCClauseKind::DefaultAsync:
-  case OpenACCClauseKind::DeviceType:
-  case OpenACCClauseKind::DType:
   case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Invalid:
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 6154ead589d3..7a9d392889bb 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -7933,6 +7933,19 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeOpenACCIntExprList(WC->getQueueIdExprs());
     return;
   }
+  case OpenACCClauseKind::DeviceType:
+  case OpenACCClauseKind::DType: {
+    const auto *DTC = cast<OpenACCDeviceTypeClause>(C);
+    writeSourceLocation(DTC->getLParenLoc());
+    writeUInt32(DTC->getArchitectures().size());
+    for (const DeviceTypeArgument &Arg : DTC->getArchitectures()) {
+      writeBool(Arg.first);
+      if (Arg.first)
+        AddIdentifierRef(Arg.first);
+      writeSourceLocation(Arg.second);
+    }
+    return;
+  }
 
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
@@ -7954,8 +7967,6 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   case OpenACCClauseKind::Bind:
   case OpenACCClauseKind::DeviceNum:
   case OpenACCClauseKind::DefaultAsync:
-  case OpenACCClauseKind::DeviceType:
-  case OpenACCClauseKind::DType:
   case OpenACCClauseKind::Tile:
   case OpenACCClauseKind::Gang:
   case OpenACCClauseKind::Invalid:
diff --git a/clang/test/AST/ast-print-openacc-compute-construct.cpp b/clang/test/AST/ast-print-openacc-compute-construct.cpp
index 0bfb90bcb587..cdd9ab3377d0 100644
--- a/clang/test/AST/ast-print-openacc-compute-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-compute-construct.cpp
@@ -107,5 +107,28 @@ void foo() {
 // CHECK: #pragma acc parallel wait(devnum: i : queues: *iPtr, i)
 #pragma acc parallel wait(devnum:i:queues:*iPtr, i)
   while(true);
+
+  bool SomeB;
+  struct SomeStruct{} SomeStructImpl;
+
+//#pragma acc parallel dtype(SomeB)
+#pragma acc parallel dtype(SomeB)
+  while(true);
+
+//#pragma acc parallel device_type(SomeStruct)
+#pragma acc parallel device_type(SomeStruct)
+  while(true);
+
+//#pragma acc parallel device_type(int)
+#pragma acc parallel device_type(int)
+  while(true);
+
+//#pragma acc parallel dtype(bool)
+#pragma acc parallel dtype(bool)
+  while(true);
+
+//#pragma acc parallel device_type (SomeStructImpl)
+#pragma acc parallel device_type (SomeStructImpl)
+  while(true);
 }
 
diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c
index 51858b441e93..694f28b86ec9 100644
--- a/clang/test/ParserOpenACC/parse-clauses.c
+++ b/clang/test/ParserOpenACC/parse-clauses.c
@@ -1126,12 +1126,10 @@ void device_type() {
 #pragma acc parallel dtype(
   {}
 
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel device_type()
   {}
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel dtype()
   {}
 
@@ -1173,12 +1171,10 @@ void device_type() {
 #pragma acc parallel dtype(ident, ident2
   {}
 
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel device_type(ident, ident2,)
   {}
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel dtype(ident, ident2,)
   {}
 
@@ -1200,33 +1196,25 @@ void device_type() {
 #pragma acc parallel dtype(*,ident)
   {}
 
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel device_type(ident, *)
   {}
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel dtype(ident, *)
   {}
 
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel device_type("foo", 54)
   {}
-  // expected-error@+2{{expected identifier}}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
+  // expected-error@+1{{expected identifier}}
 #pragma acc parallel dtype(31, "bar")
   {}
 
-  // expected-warning@+1{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
 #pragma acc parallel device_type(ident, auto, int, float)
   {}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
 #pragma acc parallel dtype(ident, auto, int, float)
   {}
 
-  // expected-warning@+2{{OpenACC clause 'device_type' not yet implemented, clause ignored}}
-  // expected-warning@+1{{OpenACC clause 'dtype' not yet implemented, clause ignored}}
 #pragma acc parallel device_type(ident, auto, int, float) dtype(ident, auto, int, float)
   {}
 }
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp b/clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp
new file mode 100644
index 000000000000..8a2423f4f542
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-device_type-ast.cpp
@@ -0,0 +1,105 @@
+// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
+struct SomeS{};
+void NormalUses() {
+  // CHECK: FunctionDecl{{.*}}NormalUses
+  // CHECK-NEXT: CompoundStmt
+
+  SomeS SomeImpl;
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} SomeImpl 'SomeS'
+  // CHECK-NEXT: CXXConstructExpr
+  bool SomeVar;
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}} SomeVar 'bool'
+
+#pragma acc parallel device_type(SomeS) dtype(SomeImpl)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(SomeS)
+  // CHECK-NEXT: dtype(SomeImpl)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(SomeVar) dtype(int)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(SomeVar)
+  // CHECK-NEXT: dtype(int)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(private) dtype(struct)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(private)
+  // CHECK-NEXT: dtype(struct)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(private) dtype(class)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(private)
+  // CHECK-NEXT: dtype(class)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(float) dtype(*)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(float)
+  // CHECK-NEXT: dtype(*)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(float, int) dtype(*)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(float, int)
+  // CHECK-NEXT: dtype(*)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+}
+
+template<typename T>
+void TemplUses() {
+  // CHECK-NEXT: FunctionTemplateDecl{{.*}}TemplUses
+  // CHECK-NEXT: TemplateTypeParmDecl{{.*}}T
+  // CHECK-NEXT: FunctionDecl{{.*}}TemplUses
+  // CHECK-NEXT: CompoundStmt
+#pragma acc parallel device_type(T) dtype(T)
+  while(true){}
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(T)
+  // CHECK-NEXT: dtype(T)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+
+
+  // Instantiations
+  // CHECK-NEXT: FunctionDecl{{.*}} TemplUses 'void ()' implicit_instantiation
+  // CHECK-NEXT: TemplateArgument type 'int'
+  // CHECK-NEXT: BuiltinType{{.*}} 'int'
+  // CHECK-NEXT: CompoundStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: device_type(T)
+  // CHECK-NEXT: dtype(T)
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: CompoundStmt
+}
+
+void Inst() {
+  TemplUses<int>();
+}
+#endif // PCH_HELPER
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
new file mode 100644
index 000000000000..15c9cf396c80
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
@@ -0,0 +1,221 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+#define MACRO +FOO
+
+void uses() {
+  typedef struct S{} STy;
+  STy SImpl;
+
+#pragma acc parallel device_type(I)
+  while(1);
+#pragma acc serial device_type(S) dtype(STy)
+  while(1);
+#pragma acc kernels dtype(SImpl)
+  while(1);
+#pragma acc kernels dtype(int) device_type(*)
+  while(1);
+#pragma acc kernels dtype(true) device_type(false)
+  while(1);
+
+  // expected-error@+1{{expected identifier}}
+#pragma acc kernels dtype(int, *)
+  while(1);
+
+#pragma acc parallel device_type(I, int)
+  while(1);
+  // expected-error@+2{{expected ','}}
+  // expected-error@+1{{expected identifier}}
+#pragma acc kernels dtype(int{})
+  while(1);
+  // expected-error@+1{{expected identifier}}
+#pragma acc kernels dtype(5)
+  while(1);
+  // expected-error@+1{{expected identifier}}
+#pragma acc kernels dtype(MACRO)
+  while(1);
+
+
+  // Only 'async', 'wait', num_gangs', 'num_workers', 'vector_length' allowed after 'device_type'.
+
+  // expected-error@+2{{OpenACC clause 'finalize' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) finalize
+  while(1);
+  // expected-error@+2{{OpenACC clause 'if_present' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) if_present
+  while(1);
+  // expected-error@+2{{OpenACC clause 'seq' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) seq
+  while(1);
+  // expected-error@+2{{OpenACC clause 'independent' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) independent
+  while(1);
+  // expected-error@+2{{OpenACC clause 'auto' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) auto
+  while(1);
+  // expected-error@+2{{OpenACC clause 'worker' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) worker
+  while(1);
+  // expected-error@+2{{OpenACC clause 'nohost' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) nohost
+  while(1);
+  // expected-error@+2{{OpenACC clause 'default' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) default(none)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'if' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) if(1)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'self' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) self
+  while(1);
+
+  int Var;
+  int *VarPtr;
+  // expected-error@+2{{OpenACC clause 'copy' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) copy(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'pcopy' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) pcopy(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'present_or_copy' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) present_or_copy(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'use_device' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) use_device(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'attach' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) attach(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'delete' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) delete(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'detach' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) detach(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'device' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) device(VarPtr)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'deviceptr' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) deviceptr(VarPtr)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'device_resident' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*)  device_resident(VarPtr)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'firstprivate' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel device_type(*) firstprivate(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'host' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) host(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'link' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) link(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'no_create' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) no_create(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'present' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) present(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'private' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel device_type(*) private(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'copyout' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) copyout(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'pcopyout' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) pcopyout(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'present_or_copyout' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) present_or_copyout(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'copyin' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) copyin(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'pcopyin' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) pcopyin(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'present_or_copyin' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) present_or_copyin(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'create' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) create(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'pcreate' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) pcreate(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'present_or_create' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) present_or_create(Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'reduction' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) reduction(+:Var)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'collapse' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) collapse(1)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'bind' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) bind(Var)
+  while(1);
+#pragma acc kernels device_type(*) vector_length(1)
+  while(1);
+#pragma acc kernels device_type(*) num_gangs(1)
+  while(1);
+#pragma acc kernels device_type(*) num_workers(1)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'device_num' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) device_num(1)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'default_async' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) default_async(1)
+  while(1);
+#pragma acc kernels device_type(*) async
+  while(1);
+  // expected-error@+2{{OpenACC clause 'tile' may not follow a 'device_type' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels device_type(*) tile(Var, 1)
+  while(1);
+  // expected-error@+2{{OpenACC clause 'gang' may not follow a 'dtype' clause in a compute construct}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc kernels dtype(*) gang
+  while(1);
+#pragma acc kernels device_type(*) wait
+  while(1);
+}
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp b/clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp
new file mode 100644
index 000000000000..ed40e8bbceae
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-device_type-clause.cpp
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+template<typename T>
+void TemplUses() {
+#pragma acc parallel device_type(I)
+  while(true);
+#pragma acc parallel dtype(*)
+  while(true);
+#pragma acc parallel device_type(class)
+  while(true);
+#pragma acc parallel device_type(private)
+  while(true);
+#pragma acc parallel device_type(bool)
+  while(true);
+#pragma acc kernels dtype(true) device_type(false)
+  while(true);
+  // expected-error@+2{{expected ','}}
+  // expected-error@+1{{expected identifier}}
+#pragma acc parallel device_type(T::value)
+  while(true);
+}
+
+void Inst() {
+  TemplUses<int>(); // #INST
+}
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index ae6659fe95e8..8b9417f985b5 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2857,6 +2857,8 @@ void OpenACCClauseEnqueue::VisitWaitClause(const OpenACCWaitClause &C) {
   for (Expr *QE : C.getQueueIdExprs())
     Visitor.AddStmt(QE);
 }
+void OpenACCClauseEnqueue::VisitDeviceTypeClause(
+    const OpenACCDeviceTypeClause &C) {}
 } // namespace
 
 void EnqueueVisitor::EnqueueChildren(const OpenACCClause *C) {
-- 
GitLab


From 673114447b66335268467396a073cdaaadd0b601 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn@google.com>
Date: Mon, 13 May 2024 12:30:50 -0500
Subject: [PATCH 0608/1206] [LLD] Implement --enable-non-contiguous-regions
 (#90007)

When enabled, input sections that would otherwise overflow a memory
region are instead spilled to the next matching output section.

This feature parallels the one in GNU LD, but there are some differences
from its documented behavior:

- /DISCARD/ only matches previously-unmatched sections (i.e., the flag
does not affect it).

- If a section fails to fit at any of its matches, the link fails
instead of discarding the section.

- The flag --enable-non-contiguous-regions-warnings is not implemented,
as it exists to warn about such occurrences.

The implementation places stubs at possible spill locations, and
replaces them with the original input section when effecting spills.
Spilling decisions occur after address assignment. Sections are spilled
in reverse order of assignment, with each spill naively decreasing the
size of the affected memory regions. This continues until the memory
regions are brought back under size. Spilling anything causes another
pass of address assignment, and this continues to fixed point.

Spilling after rather than during assignment allows the algorithm to
consider the size effects of unspillable input sections that appear
later in the assignment. Otherwise, such sections (e.g. thunks) may
force an overflow, even if spilling something earlier could have avoided
it.

A few notable feature interactions occur:

- Stubs affect alignment, ONLY_IF_RO, etc, broadly as if a copy of the
input section were actually placed there.

- SHF_MERGE synthetic sections use the spill list of their first
contained input section (the one that gives the section its name).

- ICF occurs oblivious to spill sections; spill lists for merged-away
sections become inert and are removed after assignment.

- SHF_LINK_ORDER and .ARM.exidx are ordered according to the final
section ordering, after all spilling has completed.

- INSERT BEFORE/AFTER and OVERWRITE_SECTIONS are explicitly disallowed.
---
 lld/ELF/Config.h                              |   1 +
 lld/ELF/Driver.cpp                            |   4 +-
 lld/ELF/InputSection.cpp                      |   7 +
 lld/ELF/InputSection.h                        |  25 +-
 lld/ELF/LinkerScript.cpp                      | 181 +++++++++++-
 lld/ELF/LinkerScript.h                        |  15 +-
 lld/ELF/Options.td                            |   3 +
 lld/ELF/OutputSections.cpp                    |   7 +-
 lld/ELF/OutputSections.h                      |   2 +-
 lld/ELF/SyntheticSections.cpp                 |   7 +
 lld/ELF/SyntheticSections.h                   |   4 +
 lld/ELF/Writer.cpp                            |  22 +-
 lld/docs/ELF/linker_script.rst                |  11 +
 lld/docs/ReleaseNotes.rst                     |   6 +
 lld/docs/ld.lld.1                             |   2 +
 ...able-non-contiguous-regions-arm-exidx.test |  55 ++++
 .../enable-non-contiguous-regions.test        | 265 ++++++++++++++++++
 17 files changed, 600 insertions(+), 17 deletions(-)
 create mode 100644 lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test
 create mode 100644 lld/test/ELF/linkerscript/enable-non-contiguous-regions.test

diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index c55b547a733c..dbb81412453a 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -238,6 +238,7 @@ struct Config {
   bool emitLLVM;
   bool emitRelocs;
   bool enableNewDtags;
+  bool enableNonContiguousRegions;
   bool executeOnly;
   bool exportDynamic;
   bool fixCortexA53Errata843419;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index dd33f4bd772f..828499fa05a3 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1250,6 +1250,8 @@ static void readConfigs(opt::InputArgList &args) {
   config->emitRelocs = args.hasArg(OPT_emit_relocs);
   config->enableNewDtags =
       args.hasFlag(OPT_enable_new_dtags, OPT_disable_new_dtags, true);
+  config->enableNonContiguousRegions =
+      args.hasArg(OPT_enable_non_contiguous_regions);
   config->entry = args.getLastArgValue(OPT_entry);
 
   errorHandler().errorHandlingScript =
@@ -3085,7 +3087,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
     // sectionBases.
     for (SectionCommand *cmd : script->sectionCommands)
       if (auto *osd = dyn_cast<OutputDesc>(cmd))
-        osd->osec.finalizeInputSections();
+        osd->osec.finalizeInputSections(script.get());
   }
 
   // Two input sections with different output sections should not be folded.
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index fa81611e7c9e..2a1ccd997f8b 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -161,6 +161,7 @@ uint64_t SectionBase::getOffset(uint64_t offset) const {
   }
   case Regular:
   case Synthetic:
+  case Spill:
     return cast<InputSection>(this)->outSecOff + offset;
   case EHFrame: {
     // Two code paths may reach here. First, clang_rt.crtbegin.o and GCC
@@ -309,6 +310,12 @@ std::string InputSectionBase::getObjMsg(uint64_t off) const {
       .str();
 }
 
+PotentialSpillSection::PotentialSpillSection(const InputSectionBase &source,
+                                             InputSectionDescription &isd)
+    : InputSection(source.file, source.flags, source.type, source.addralign, {},
+                   source.name, SectionBase::Spill),
+      isd(&isd) {}
+
 InputSection InputSection::discarded(nullptr, 0, 0, 0, ArrayRef<uint8_t>(), "");
 
 InputSection::InputSection(InputFile *f, uint64_t flags, uint32_t type,
diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h
index 1fb7077ca435..58e5306fd6dc 100644
--- a/lld/ELF/InputSection.h
+++ b/lld/ELF/InputSection.h
@@ -48,7 +48,7 @@ template <class ELFT> struct RelsOrRelas {
 // sections.
 class SectionBase {
 public:
-  enum Kind { Regular, Synthetic, EHFrame, Merge, Output };
+  enum Kind { Regular, Synthetic, Spill, EHFrame, Merge, Output };
 
   Kind kind() const { return (Kind)sectionKind; }
 
@@ -382,7 +382,8 @@ public:
 
   static bool classof(const SectionBase *s) {
     return s->kind() == SectionBase::Regular ||
-           s->kind() == SectionBase::Synthetic;
+           s->kind() == SectionBase::Synthetic ||
+           s->kind() == SectionBase::Spill;
   }
 
   // Write this section to a mmap'ed file, assuming Buf is pointing to
@@ -425,6 +426,26 @@ private:
   template <class ELFT> void copyShtGroup(uint8_t *buf);
 };
 
+// A marker for a potential spill location for another input section. This
+// broadly acts as if it were the original section until address assignment.
+// Then it is either replaced with the real input section or removed.
+class PotentialSpillSection : public InputSection {
+public:
+  // The containing input section description; used to quickly replace this stub
+  // with the actual section.
+  InputSectionDescription *isd;
+
+  // Next potential spill location for the same source input section.
+  PotentialSpillSection *next = nullptr;
+
+  PotentialSpillSection(const InputSectionBase &source,
+                        InputSectionDescription &isd);
+
+  static bool classof(const SectionBase *sec) {
+    return sec->kind() == InputSectionBase::Spill;
+  }
+};
+
 static_assert(sizeof(InputSection) <= 160, "InputSection is too big");
 
 class SyntheticSection : public InputSection {
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index c0a5014817b9..3ba59c112b8a 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -304,6 +304,9 @@ getChangedSymbolAssignment(const SymbolAssignmentMap &oldValues) {
 void LinkerScript::processInsertCommands() {
   SmallVector<OutputDesc *, 0> moves;
   for (const InsertCommand &cmd : insertCommands) {
+    if (config->enableNonContiguousRegions)
+      error("INSERT cannot be used with --enable-non-contiguous-regions");
+
     for (StringRef name : cmd.names) {
       // If base is empty, it may have been discarded by
       // adjustOutputSections(). We do not handle such output sections.
@@ -486,10 +489,12 @@ static void sortInputSections(MutableArrayRef<InputSectionBase *> vec,
 // Compute and remember which sections the InputSectionDescription matches.
 SmallVector<InputSectionBase *, 0>
 LinkerScript::computeInputSections(const InputSectionDescription *cmd,
-                                   ArrayRef<InputSectionBase *> sections) {
+                                   ArrayRef<InputSectionBase *> sections,
+                                   const OutputSection &outCmd) {
   SmallVector<InputSectionBase *, 0> ret;
   SmallVector<size_t, 0> indexes;
   DenseSet<size_t> seen;
+  DenseSet<InputSectionBase *> spills;
   auto sortByPositionThenCommandLine = [&](size_t begin, size_t end) {
     llvm::sort(MutableArrayRef<size_t>(indexes).slice(begin, end - begin));
     for (size_t i = begin; i != end; ++i)
@@ -505,10 +510,10 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
     size_t sizeBeforeCurrPat = ret.size();
 
     for (size_t i = 0, e = sections.size(); i != e; ++i) {
-      // Skip if the section is dead or has been matched by a previous input
-      // section description or a previous pattern.
+      // Skip if the section is dead or has been matched by a previous pattern
+      // in this input section description.
       InputSectionBase *sec = sections[i];
-      if (!sec->isLive() || sec->parent || seen.contains(i))
+      if (!sec->isLive() || seen.contains(i))
         continue;
 
       // For --emit-relocs we have to ignore entries like
@@ -529,6 +534,29 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
           (sec->flags & cmd->withoutFlags) != 0)
         continue;
 
+      if (sec->parent) {
+        // Skip if not allowing multiple matches.
+        if (!config->enableNonContiguousRegions)
+          continue;
+
+        // Disallow spilling into /DISCARD/; special handling would be needed
+        // for this in address assignment, and the semantics are nebulous.
+        if (outCmd.name == "/DISCARD/")
+          continue;
+
+        // Skip if the section's first match was /DISCARD/; such sections are
+        // always discarded.
+        if (sec->parent->name == "/DISCARD/")
+          continue;
+
+        // Skip if the section was already matched by a different input section
+        // description within this output section.
+        if (sec->parent == &outCmd)
+          continue;
+
+        spills.insert(sec);
+      }
+
       ret.push_back(sec);
       indexes.push_back(i);
       seen.insert(i);
@@ -555,6 +583,30 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
   // Matched sections after the last SORT* are sorted by (--sort-alignment,
   // input order).
   sortByPositionThenCommandLine(sizeAfterPrevSort, ret.size());
+
+  // The flag --enable-non-contiguous-regions may cause sections to match an
+  // InputSectionDescription in more than one OutputSection. Matches after the
+  // first were collected in the spills set, so replace these with potential
+  // spill sections.
+  if (!spills.empty()) {
+    for (InputSectionBase *&sec : ret) {
+      if (!spills.contains(sec))
+        continue;
+
+      // Append the spill input section to the list for the input section,
+      // creating it if necessary.
+      PotentialSpillSection *pss = make<PotentialSpillSection>(
+          *sec, const_cast<InputSectionDescription &>(*cmd));
+      auto [it, inserted] =
+          potentialSpillLists.try_emplace(sec, PotentialSpillList{pss, pss});
+      if (!inserted) {
+        PotentialSpillSection *&tail = it->second.tail;
+        tail = tail->next = pss;
+      }
+      sec = pss;
+    }
+  }
+
   return ret;
 }
 
@@ -577,7 +629,7 @@ void LinkerScript::discardSynthetic(OutputSection &outCmd) {
         part.armExidx->exidxSections.end());
     for (SectionCommand *cmd : outCmd.commands)
       if (auto *isd = dyn_cast<InputSectionDescription>(cmd))
-        for (InputSectionBase *s : computeInputSections(isd, secs))
+        for (InputSectionBase *s : computeInputSections(isd, secs, outCmd))
           discard(*s);
   }
 }
@@ -588,7 +640,7 @@ LinkerScript::createInputSectionList(OutputSection &outCmd) {
 
   for (SectionCommand *cmd : outCmd.commands) {
     if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) {
-      isd->sectionBases = computeInputSections(isd, ctx.inputSections);
+      isd->sectionBases = computeInputSections(isd, ctx.inputSections, outCmd);
       for (InputSectionBase *s : isd->sectionBases)
         s->parent = &outCmd;
       ret.insert(ret.end(), isd->sectionBases.begin(), isd->sectionBases.end());
@@ -644,6 +696,9 @@ void LinkerScript::processSectionCommands() {
 
   // Process OVERWRITE_SECTIONS first so that it can overwrite the main script
   // or orphans.
+  if (config->enableNonContiguousRegions && !overwriteSections.empty())
+    error("OVERWRITE_SECTIONS cannot be used with "
+          "--enable-non-contiguous-regions");
   DenseMap<CachedHashStringRef, OutputDesc *> map;
   size_t i = 0;
   for (OutputDesc *osd : overwriteSections) {
@@ -1066,8 +1121,12 @@ void LinkerScript::assignOffsets(OutputSection *sec) {
     // Handle a single input section description command.
     // It calculates and assigns the offsets for each section and also
     // updates the output section size.
-    for (InputSection *isec : cast<InputSectionDescription>(cmd)->sections) {
+
+    auto &sections = cast<InputSectionDescription>(cmd)->sections;
+    for (InputSection *isec : sections) {
       assert(isec->getParent() == sec);
+      if (isa<PotentialSpillSection>(isec))
+        continue;
       const uint64_t pos = dot;
       dot = alignToPowerOf2(dot, isec->addralign);
       isec->outSecOff = dot - sec->addr;
@@ -1364,6 +1423,114 @@ const Defined *LinkerScript::assignAddresses() {
   return getChangedSymbolAssignment(oldValues);
 }
 
+static bool hasRegionOverflowed(MemoryRegion *mr) {
+  if (!mr)
+    return false;
+  return mr->curPos - mr->getOrigin() > mr->getLength();
+}
+
+// Spill input sections in reverse order of address assignment to (potentially)
+// bring memory regions out of overflow. The size savings of a spill can only be
+// estimated, since general linker script arithmetic may occur afterwards.
+// Under-estimates may cause unnecessary spills, but over-estimates can always
+// be corrected on the next pass.
+bool LinkerScript::spillSections() {
+  if (!config->enableNonContiguousRegions)
+    return false;
+
+  bool spilled = false;
+  for (SectionCommand *cmd : reverse(sectionCommands)) {
+    auto *od = dyn_cast<OutputDesc>(cmd);
+    if (!od)
+      continue;
+    OutputSection *osec = &od->osec;
+    if (!osec->memRegion)
+      continue;
+
+    // Input sections that have replaced a potential spill and should be removed
+    // from their input section description.
+    DenseSet<InputSection *> spilledInputSections;
+
+    for (SectionCommand *cmd : reverse(osec->commands)) {
+      if (!hasRegionOverflowed(osec->memRegion) &&
+          !hasRegionOverflowed(osec->lmaRegion))
+        break;
+
+      auto *isd = dyn_cast<InputSectionDescription>(cmd);
+      if (!isd)
+        continue;
+      for (InputSection *isec : reverse(isd->sections)) {
+        // Potential spill locations cannot be spilled.
+        if (isa<PotentialSpillSection>(isec))
+          continue;
+
+        // Find the next potential spill location and remove it from the list.
+        auto it = potentialSpillLists.find(isec);
+        if (it == potentialSpillLists.end())
+          continue;
+        PotentialSpillList &list = it->second;
+        PotentialSpillSection *spill = list.head;
+        if (spill->next)
+          list.head = spill->next;
+        else
+          potentialSpillLists.erase(isec);
+
+        // Replace the next spill location with the spilled section and adjust
+        // its properties to match the new location. Note that the alignment of
+        // the spill section may have diverged from the original due to e.g. a
+        // SUBALIGN. Correct assignment requires the spill's alignment to be
+        // used, not the original.
+        spilledInputSections.insert(isec);
+        *llvm::find(spill->isd->sections, spill) = isec;
+        isec->parent = spill->parent;
+        isec->addralign = spill->addralign;
+
+        // Record the (potential) reduction in the region's end position.
+        osec->memRegion->curPos -= isec->getSize();
+        if (osec->lmaRegion)
+          osec->lmaRegion->curPos -= isec->getSize();
+
+        // Spilling continues until the end position no longer overflows the
+        // region. Then, another round of address assignment will either confirm
+        // the spill's success or lead to yet more spilling.
+        if (!hasRegionOverflowed(osec->memRegion) &&
+            !hasRegionOverflowed(osec->lmaRegion))
+          break;
+      }
+
+      // Remove any spilled input sections to complete their move.
+      if (!spilledInputSections.empty()) {
+        spilled = true;
+        llvm::erase_if(isd->sections, [&](InputSection *isec) {
+          return spilledInputSections.contains(isec);
+        });
+      }
+    }
+  }
+
+  return spilled;
+}
+
+// Erase any potential spill sections that were not used.
+void LinkerScript::erasePotentialSpillSections() {
+  if (potentialSpillLists.empty())
+    return;
+
+  // Collect the set of input section descriptions that contain potential
+  // spills.
+  DenseSet<InputSectionDescription *> isds;
+  for (const auto &[_, list] : potentialSpillLists)
+    for (PotentialSpillSection *s = list.head; s; s = s->next)
+      isds.insert(s->isd);
+
+  for (InputSectionDescription *isd : isds)
+    llvm::erase_if(isd->sections, [](InputSection *s) {
+      return isa<PotentialSpillSection>(s);
+    });
+
+  potentialSpillLists.clear();
+}
+
 // Creates program headers as instructed by PHDRS linker script command.
 SmallVector<PhdrEntry *, 0> LinkerScript::createPhdrs() {
   SmallVector<PhdrEntry *, 0> ret;
diff --git a/lld/ELF/LinkerScript.h b/lld/ELF/LinkerScript.h
index b09cd12c46f9..734d4e7498aa 100644
--- a/lld/ELF/LinkerScript.h
+++ b/lld/ELF/LinkerScript.h
@@ -10,6 +10,7 @@
 #define LLD_ELF_LINKER_SCRIPT_H
 
 #include "Config.h"
+#include "InputSection.h"
 #include "Writer.h"
 #include "lld/Common/LLVM.h"
 #include "lld/Common/Strings.h"
@@ -287,7 +288,8 @@ class LinkerScript final {
 
   SmallVector<InputSectionBase *, 0>
   computeInputSections(const InputSectionDescription *,
-                       ArrayRef<InputSectionBase *>);
+                       ArrayRef<InputSectionBase *>,
+                       const OutputSection &outCmd);
 
   SmallVector<InputSectionBase *, 0> createInputSectionList(OutputSection &cmd);
 
@@ -333,6 +335,8 @@ public:
 
   bool shouldKeep(InputSectionBase *s);
   const Defined *assignAddresses();
+  bool spillSections();
+  void erasePotentialSpillSections();
   void allocateHeaders(SmallVector<PhdrEntry *, 0> &phdrs);
   void processSectionCommands();
   void processSymbolAssignments();
@@ -400,6 +404,15 @@ public:
   //
   // then provideMap should contain the mapping: 'v' -> ['a', 'b', 'c']
   llvm::MapVector<StringRef, SmallVector<StringRef, 0>> provideMap;
+
+  // List of potential spill locations (PotentialSpillSection) for an input
+  // section.
+  struct PotentialSpillList {
+    // Never nullptr.
+    PotentialSpillSection *head;
+    PotentialSpillSection *tail;
+  };
+  llvm::DenseMap<InputSectionBase *, PotentialSpillList> potentialSpillLists;
 };
 
 struct ScriptWrapper {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index b9e05a4b1fd5..883a6079bf50 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -197,6 +197,9 @@ def emit_relocs: F<"emit-relocs">, HelpText<"Generate relocations in output">;
 def enable_new_dtags: F<"enable-new-dtags">,
   HelpText<"Enable new dynamic tags (default)">;
 
+def enable_non_contiguous_regions : FF<"enable-non-contiguous-regions">,
+  HelpText<"Spill input sections to later matching output sections to avoid memory region overflow">;
+
 def end_group: F<"end-group">,
   HelpText<"Ignored for compatibility with GNU unless you pass --warn-backrefs">;
 
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 9c667241360f..fcb4c4387aa9 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -186,7 +186,7 @@ static MergeSyntheticSection *createMergeSynthetic(StringRef name,
 // new synthetic sections at the location of the first input section
 // that it replaces. It then finalizes each synthetic section in order
 // to compute an output offset for each piece of each input section.
-void OutputSection::finalizeInputSections() {
+void OutputSection::finalizeInputSections(LinkerScript *script) {
   std::vector<MergeSyntheticSection *> mergeSections;
   for (SectionCommand *cmd : commands) {
     auto *isd = dyn_cast<InputSectionDescription>(cmd);
@@ -226,6 +226,11 @@ void OutputSection::finalizeInputSections() {
         i = std::prev(mergeSections.end());
         syn->entsize = ms->entsize;
         isd->sections.push_back(syn);
+        // The merge synthetic section inherits the potential spill locations of
+        // its first contained section.
+        auto it = script->potentialSpillLists.find(ms);
+        if (it != script->potentialSpillLists.end())
+          script->potentialSpillLists.try_emplace(syn, it->second);
       }
       (*i)->addSection(ms);
     }
diff --git a/lld/ELF/OutputSections.h b/lld/ELF/OutputSections.h
index 421a0181feb5..78fede48a23f 100644
--- a/lld/ELF/OutputSections.h
+++ b/lld/ELF/OutputSections.h
@@ -75,7 +75,7 @@ public:
 
   void recordSection(InputSectionBase *isec);
   void commitSection(InputSection *isec);
-  void finalizeInputSections();
+  void finalizeInputSections(LinkerScript *script = nullptr);
 
   // The following members are normally only used in linker scripts.
   MemoryRegion *memRegion = nullptr;
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 7b9ada40c0f6..298c714adb3b 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -4074,6 +4074,13 @@ static bool isDuplicateArmExidxSec(InputSection *prev, InputSection *cur) {
 // InputSection with the highest address and any InputSections that have
 // mergeable .ARM.exidx table entries are removed from it.
 void ARMExidxSyntheticSection::finalizeContents() {
+  // Ensure that any fixed-point iterations after the first see the original set
+  // of sections.
+  if (!originalExecutableSections.empty())
+    executableSections = originalExecutableSections;
+  else if (config->enableNonContiguousRegions)
+    originalExecutableSections = executableSections;
+
   // The executableSections and exidxSections that we use to derive the final
   // contents of this SyntheticSection are populated before
   // processSectionCommands() and ICF. A /DISCARD/ entry in SECTIONS command or
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index 995fd4b344b0..34949025a45f 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -1255,6 +1255,10 @@ private:
   // either find the .ARM.exidx section or know that we need to generate one.
   SmallVector<InputSection *, 0> executableSections;
 
+  // Value of executableSecitons before finalizeContents(), so that it can be
+  // run repeateadly during fixed point iteration.
+  SmallVector<InputSection *, 0> originalExecutableSections;
+
   // The executable InputSection with the highest address to use for the
   // sentinel. We store separately from ExecutableSections as merging of
   // duplicate entries may mean this InputSection is removed from
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index e400ed2ae945..8d529f2bdb9f 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -1403,13 +1403,18 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
   AArch64Err843419Patcher a64p;
   ARMErr657417Patcher a32p;
   script->assignAddresses();
+
   // .ARM.exidx and SHF_LINK_ORDER do not require precise addresses, but they
   // do require the relative addresses of OutputSections because linker scripts
   // can assign Virtual Addresses to OutputSections that are not monotonically
-  // increasing.
-  for (Partition &part : partitions)
-    finalizeSynthetic(part.armExidx.get());
-  resolveShfLinkOrder();
+  // increasing. Anything here must be repeatable, since spilling may change
+  // section order.
+  const auto finalizeOrderDependentContent = [this] {
+    for (Partition &part : partitions)
+      finalizeSynthetic(part.armExidx.get());
+    resolveShfLinkOrder();
+  };
+  finalizeOrderDependentContent();
 
   // Converts call x@GDPLT to call __tls_get_addr
   if (config->emachine == EM_HEXAGON)
@@ -1419,6 +1424,8 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
   for (;;) {
     bool changed = target->needsThunks ? tc.createThunks(pass, outputSections)
                                        : target->relaxOnce(pass);
+    bool spilled = script->spillSections();
+    changed |= spilled;
     ++pass;
 
     // With Thunk Size much smaller than branch range we expect to
@@ -1464,6 +1471,9 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
                     " does not converge");
         break;
       }
+    } else if (spilled) {
+      // Spilling can change relative section order.
+      finalizeOrderDependentContent();
     }
   }
   if (!config->relocatable)
@@ -1483,6 +1493,10 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
              osec->name + " is not a multiple of alignment (" +
              Twine(osec->addralign) + ")");
     }
+
+  // Sizes are no longer allowed to grow, so all allowable spills have been
+  // taken. Remove any leftover potential spills.
+  script->erasePotentialSpillSections();
 }
 
 // If Input Sections have been shrunk (basic block sections) then
diff --git a/lld/docs/ELF/linker_script.rst b/lld/docs/ELF/linker_script.rst
index 3606ef4fe4b8..7a35534be096 100644
--- a/lld/docs/ELF/linker_script.rst
+++ b/lld/docs/ELF/linker_script.rst
@@ -197,3 +197,14 @@ the current location to a max-page-size boundary, ensuring that the next
 
 LLD will insert ``.relro_padding`` immediately before the symbol assignment
 using ``DATA_SEGMENT_RELRO_END``.
+
+Non-contiguous regions
+~~~~~~~~~~~~~~~~~~~~~~
+
+The flag ``--enable-non-contiguous-regions`` allows input sections to spill to
+later matches rather than causing the link to fail by overflowing a memory
+region. Unlike GNU ld, ``/DISCARD/`` only matches previously-unmatched sections
+(i.e., the flag does not affect it). Also, if a section fails to fit at any of
+its matches, the link fails instead of discarding the section. Accordingly, the
+GNU flag ``--enable-non-contiguous-regions-warnings`` is not implemented, as it
+exists to warn about such occurrences.
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index f8fdebfeaecf..e7a913e025da 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -38,6 +38,12 @@ ELF Improvements
 * ``--debug-names`` is added to create a merged ``.debug_names`` index
   from input ``.debug_names`` sections. Type units are not handled yet.
   (`#86508 <https://github.com/llvm/llvm-project/pull/86508>`_)
+* ``--enable-non-contiguous-regions`` option allows automatically packing input
+  sections into memory regions by automatically spilling to later matches if a
+  region would overflow. This reduces the toil of manually packing regions
+  (typical for embedded). It also makes full LTO feasible in such cases, since
+  IR merging currently prevents the linker script from referring to input
+  files. (`#90007 <https://github.com/llvm/llvm-project/pull/90007>`_)
 
 Breaking changes
 ----------------
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 9ea1a9c52f2a..0df13f07f560 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -222,6 +222,8 @@ segment header.
 Generate relocations in the output.
 .It Fl -enable-new-dtags
 Enable new dynamic tags.
+.It Fl -enable-non-contiguous-regions
+Spill input sections to later matching output sections to avoid memory region overflow.
 .It Fl -end-lib
 End a grouping of objects that should be treated as if they were together
 in an archive.
diff --git a/lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test b/lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test
new file mode 100644
index 000000000000..3f7b9c4e5f8b
--- /dev/null
+++ b/lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test
@@ -0,0 +1,55 @@
+## When spilling reorders input sections, the .ARM.exidx table is rebuilt using
+## the new order.
+
+# REQUIRES: arm
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=armv7a-none-linux-gnueabi test.s -o test.o
+# RUN: ld.lld -T test.ld test.o -o test --enable-non-contiguous-regions
+# RUN: llvm-readobj -x .ARM.exidx test | FileCheck %s
+
+# CHECK:      20000000 08849780 1c000000 10849880
+# CHECK-NEXT: 1c000000 01000000
+
+#--- test.ld
+MEMORY {
+  exidx : ORIGIN = 0, LENGTH = 32
+  a : ORIGIN = 32, LENGTH = 4
+  b : ORIGIN = 36, LENGTH = 4
+  c : ORIGIN = 40, LENGTH = 4
+}
+
+SECTIONS {
+  .ARM.exidx : { *(.ARM.exidx) } >exidx
+  .first_chance : { *(.text .text.f2) } >a
+  .text.f1 : { *(.text.f1) } >b
+  .last_chance : { *(.text.f2) } >c
+}
+
+#--- test.s
+ .syntax unified
+ .section .text, "ax",%progbits
+ .globl _start
+_start:
+ .fnstart
+ bx lr
+ .save {r7, lr}
+ .setfp r7, sp, #0
+ .fnend
+
+ .section .text.f1, "ax", %progbits
+ .globl f1
+f1:
+ .fnstart
+ bx lr
+ .save {r8, lr}
+ .setfp r8, sp, #0
+ .fnend
+
+ .section .text.f2, "ax", %progbits
+ .globl f2
+f2:
+ .fnstart
+ bx lr
+ .save {r8, lr}
+ .setfp r8, sp, #0
+ .fnend
diff --git a/lld/test/ELF/linkerscript/enable-non-contiguous-regions.test b/lld/test/ELF/linkerscript/enable-non-contiguous-regions.test
new file mode 100644
index 000000000000..392106fd476f
--- /dev/null
+++ b/lld/test/ELF/linkerscript/enable-non-contiguous-regions.test
@@ -0,0 +1,265 @@
+# REQUIRES: x86
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -n -filetype=obj -triple=x86_64 spill.s -o spill.o
+
+## An input section spills to a later match when the region of its first match
+## would overflow. The spill uses the alignment of the later match.
+
+# RUN: ld.lld -T spill.ld spill.o -o spill --enable-non-contiguous-regions
+# RUN: llvm-readelf -S spill | FileCheck %s --check-prefix=SPILL
+
+# SPILL:      Name          Type     Address          Off    Size
+# SPILL:      .first_chance PROGBITS 0000000000000000 001000 000001
+# SPILL-NEXT: .last_chance  PROGBITS 0000000000000008 001008 000002
+
+## A spill off the end still fails the link.
+
+# RUN: not ld.lld -T spill-fail.ld spill.o --enable-non-contiguous-regions 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=SPILL-FAIL --implicit-check-not=error:
+
+# SPILL-FAIL: error: section '.last_chance' will not fit in region 'b': overflowed by 2 bytes
+
+## The above spill still occurs when the LMA would overflow, even though the
+## VMA would fit.
+
+# RUN: ld.lld -T spill-lma.ld spill.o -o spill-lma --enable-non-contiguous-regions
+# RUN: llvm-readelf -S spill-lma | FileCheck %s --check-prefix=SPILL-LMA
+
+# SPILL-LMA:      Name          Type     Address          Off    Size
+# SPILL-LMA:      .first_chance PROGBITS 0000000000000000 001000 000001
+# SPILL-LMA-NEXT: .last_chance  PROGBITS 0000000000000003 001003 000002
+
+## A spill occurs to an additional match after the first.
+
+# RUN: ld.lld -T spill-later.ld spill.o -o spill-later --enable-non-contiguous-regions
+# RUN: llvm-readelf -S spill-later | FileCheck %s --check-prefix=SPILL-LATER
+
+# SPILL-LATER:      Name            Type     Address          Off    Size
+# SPILL-LATER:      .first_chance   PROGBITS 0000000000000000 001000 000001
+# SPILL-LATER-NEXT: .second_chance  PROGBITS 0000000000000002 001001 000000
+# SPILL-LATER-NEXT: .last_chance    PROGBITS 0000000000000003 001003 000002
+
+## A later overflow causes an earlier section to spill.
+
+# RUN: ld.lld -T spill-earlier.ld spill.o -o spill-earlier --enable-non-contiguous-regions
+# RUN: llvm-readelf -S spill-earlier | FileCheck %s --check-prefix=SPILL-EARLIER
+
+# SPILL-EARLIER:      Name          Type     Address          Off    Size
+# SPILL-EARLIER:      .first_chance PROGBITS 0000000000000000 001000 000002
+# SPILL-EARLIER-NEXT: .last_chance  PROGBITS 0000000000000002 001002 000001
+
+## An additional match in /DISCARD/ has no effect.
+
+# RUN: not ld.lld -T no-spill-into-discard.ld spill.o --enable-non-contiguous-regions 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=NO-SPILL-INTO-DISCARD --implicit-check-not=error:
+
+# NO-SPILL-INTO-DISCARD: error: section '.osec' will not fit in region 'a': overflowed by 1 bytes
+
+## An additional match after /DISCARD/ has no effect.
+
+# RUN: ld.lld -T no-spill-from-discard.ld spill.o -o no-spill-from-discard --enable-non-contiguous-regions
+# RUN: llvm-readelf -S no-spill-from-discard | FileCheck %s --check-prefix=NO-SPILL-FROM-DISCARD
+
+# NO-SPILL-FROM-DISCARD: Name          Type     Address          Off    Size
+# NO-SPILL-FROM-DISCARD-NOT: .osec
+
+## SHF_MERGEd sections are spilled according to the matches of the first merged
+## input section (the one giving the resulting section its name).
+
+# RUN: llvm-mc -n -filetype=obj -triple=x86_64 merge.s -o merge.o
+# RUN: ld.lld -T spill-merge.ld merge.o -o spill-merge --enable-non-contiguous-regions
+# RUN: llvm-readelf -S spill-merge | FileCheck %s --check-prefix=SPILL-MERGE
+
+# SPILL-MERGE:      Name          Type     Address          Off    Size
+# SPILL-MERGE:      .first  PROGBITS 0000000000000000 000190 000000
+# SPILL-MERGE-NEXT: .second PROGBITS 0000000000000001 001001 000002
+# SPILL-MERGE-NEXT: .third  PROGBITS 0000000000000003 001003 000000
+
+## An error is reported for INSERT.
+
+# RUN: not ld.lld -T insert.ld spill.o --enable-non-contiguous-regions 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=INSERT
+
+# INSERT: error: INSERT cannot be used with --enable-non-contiguous-regions
+
+## An error is reported for OVERWRITE_SECTIONS.
+
+# RUN: not ld.lld -T overwrite-sections.ld spill.o --enable-non-contiguous-regions 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=OVERWRITE_SECTIONS
+
+# OVERWRITE_SECTIONS: error: OVERWRITE_SECTIONS cannot be used with --enable-non-contiguous-regions
+
+## SHF_LINK_ORDER is reordered when spilling changes relative section order.
+
+# RUN: llvm-mc -n -filetype=obj -triple=x86_64 link-order.s -o link-order.o
+# RUN: ld.lld -T link-order.ld link-order.o -o link-order --enable-non-contiguous-regions
+# RUN: llvm-readobj -x .order link-order | FileCheck %s --check-prefix=LINK-ORDER
+
+# LINK-ORDER: 020301
+
+#--- spill.s
+.section .one_byte_section,"a",@progbits
+.fill 1
+
+.section .two_byte_section,"a",@progbits
+.fill 2
+
+#--- spill.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 2
+  b : ORIGIN = 2, LENGTH = 16
+}
+
+SECTIONS {
+  .first_chance : SUBALIGN(1) { *(.one_byte_section) *(.two_byte_section) } >a
+  .last_chance : SUBALIGN(8) { *(.two_byte_section) } >b
+}
+
+#--- spill-fail.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 1
+  b : ORIGIN = 2, LENGTH = 0
+}
+
+SECTIONS {
+  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >a
+  .last_chance : { *(.two_byte_section) } >b
+}
+
+#--- spill-lma.ld
+MEMORY {
+  vma_a : ORIGIN = 0, LENGTH = 3
+  vma_b : ORIGIN = 3, LENGTH = 3
+  lma_a : ORIGIN = 6, LENGTH = 2
+  lma_b : ORIGIN = 8, LENGTH = 2
+}
+
+SECTIONS {
+  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >vma_a AT>lma_a
+  .last_chance : { *(.two_byte_section) } >vma_b AT>lma_b
+}
+
+#--- spill-later.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 2
+  b : ORIGIN = 2, LENGTH = 1
+  c : ORIGIN = 3, LENGTH = 2
+}
+
+SECTIONS {
+  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >a
+  .second_chance : { *(.two_byte_section) } >b
+  .last_chance : { *(.two_byte_section) } >c
+}
+
+#--- spill-earlier.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 2
+  b : ORIGIN = 2, LENGTH = 1
+}
+
+SECTIONS {
+  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >a
+  .last_chance : { *(.one_byte_section) } >b
+}
+
+#--- no-spill-into-discard.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 1
+}
+
+SECTIONS {
+  .osec : { *(.two_byte_section) } >a
+  /DISCARD/ : { *(.one_byte_section) *(.two_byte_section) }
+}
+
+#--- no-spill-from-discard.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 2
+}
+
+SECTIONS {
+  /DISCARD/ : { *(.one_byte_section) *(.two_byte_section) }
+  .osec : { *(.two_byte_section) } >a
+}
+
+#--- merge.s
+.section .a,"aM",@progbits,1
+.byte 0x12, 0x34
+
+.section .b,"aM",@progbits,1
+.byte 0x12
+
+#--- spill-merge.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 1
+  b : ORIGIN = 1, LENGTH = 2
+  c : ORIGIN = 3, LENGTH = 2
+}
+
+SECTIONS {
+  .first : { *(.a) *(.b) } >a
+  .second : { *(.a) } >b
+  .third : { *(.b) } >c
+}
+
+#--- insert.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 1
+}
+
+SECTIONS {
+  .a : { *(.two_byte_section) } >a
+}
+
+SECTIONS {
+  .b : { *(.one_byte_section) } >a
+} INSERT AFTER .a;
+
+#--- overwrite-sections.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 1
+}
+
+SECTIONS {
+  .a : { *(.two_byte_section) } >a
+}
+
+OVERWRITE_SECTIONS {
+  .b : { *(.one_byte_section) } >a
+}
+
+#--- link-order.s
+.section .a,"a",@progbits
+.fill 1
+
+.section .b,"a",@progbits
+.fill 1
+
+.section .c,"a",@progbits
+.fill 1
+
+.section .link_order.a,"ao",@progbits,.a
+.byte 1
+
+.section .link_order.b,"ao",@progbits,.b
+.byte 2
+
+.section .link_order.c,"ao",@progbits,.c
+.byte 3
+
+#--- link-order.ld
+MEMORY {
+  order : ORIGIN = 0, LENGTH = 3
+  potential_a : ORIGIN = 3, LENGTH = 0
+  bc : ORIGIN = 3, LENGTH = 2
+  actual_a : ORIGIN = 5, LENGTH = 1
+}
+
+SECTIONS {
+  .order :  { *(.link_order.*) } > order
+  .potential_a : { *(.a) } >potential_a
+  .bc : { *(.b) *(.c) } >bc
+  .actual_a : { *(.a) } >actual_a
+}
-- 
GitLab


From 81f34afa5c39584c2721000e2bcf3b8ec02a4f4d Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn@google.com>
Date: Mon, 13 May 2024 12:38:40 -0500
Subject: [PATCH 0609/1206] Revert "[LLD] Implement
 --enable-non-contiguous-regions" (#92005)

Reverts llvm/llvm-project#90007

Broke in merging I think.
---
 lld/ELF/Config.h                              |   1 -
 lld/ELF/Driver.cpp                            |   4 +-
 lld/ELF/InputSection.cpp                      |   7 -
 lld/ELF/InputSection.h                        |  25 +-
 lld/ELF/LinkerScript.cpp                      | 181 +-----------
 lld/ELF/LinkerScript.h                        |  15 +-
 lld/ELF/Options.td                            |   3 -
 lld/ELF/OutputSections.cpp                    |   7 +-
 lld/ELF/OutputSections.h                      |   2 +-
 lld/ELF/SyntheticSections.cpp                 |   7 -
 lld/ELF/SyntheticSections.h                   |   4 -
 lld/ELF/Writer.cpp                            |  22 +-
 lld/docs/ELF/linker_script.rst                |  11 -
 lld/docs/ReleaseNotes.rst                     |   6 -
 lld/docs/ld.lld.1                             |   2 -
 ...able-non-contiguous-regions-arm-exidx.test |  55 ----
 .../enable-non-contiguous-regions.test        | 265 ------------------
 17 files changed, 17 insertions(+), 600 deletions(-)
 delete mode 100644 lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test
 delete mode 100644 lld/test/ELF/linkerscript/enable-non-contiguous-regions.test

diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index dbb81412453a..c55b547a733c 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -238,7 +238,6 @@ struct Config {
   bool emitLLVM;
   bool emitRelocs;
   bool enableNewDtags;
-  bool enableNonContiguousRegions;
   bool executeOnly;
   bool exportDynamic;
   bool fixCortexA53Errata843419;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 828499fa05a3..dd33f4bd772f 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1250,8 +1250,6 @@ static void readConfigs(opt::InputArgList &args) {
   config->emitRelocs = args.hasArg(OPT_emit_relocs);
   config->enableNewDtags =
       args.hasFlag(OPT_enable_new_dtags, OPT_disable_new_dtags, true);
-  config->enableNonContiguousRegions =
-      args.hasArg(OPT_enable_non_contiguous_regions);
   config->entry = args.getLastArgValue(OPT_entry);
 
   errorHandler().errorHandlingScript =
@@ -3087,7 +3085,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
     // sectionBases.
     for (SectionCommand *cmd : script->sectionCommands)
       if (auto *osd = dyn_cast<OutputDesc>(cmd))
-        osd->osec.finalizeInputSections(script.get());
+        osd->osec.finalizeInputSections();
   }
 
   // Two input sections with different output sections should not be folded.
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 2a1ccd997f8b..fa81611e7c9e 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -161,7 +161,6 @@ uint64_t SectionBase::getOffset(uint64_t offset) const {
   }
   case Regular:
   case Synthetic:
-  case Spill:
     return cast<InputSection>(this)->outSecOff + offset;
   case EHFrame: {
     // Two code paths may reach here. First, clang_rt.crtbegin.o and GCC
@@ -310,12 +309,6 @@ std::string InputSectionBase::getObjMsg(uint64_t off) const {
       .str();
 }
 
-PotentialSpillSection::PotentialSpillSection(const InputSectionBase &source,
-                                             InputSectionDescription &isd)
-    : InputSection(source.file, source.flags, source.type, source.addralign, {},
-                   source.name, SectionBase::Spill),
-      isd(&isd) {}
-
 InputSection InputSection::discarded(nullptr, 0, 0, 0, ArrayRef<uint8_t>(), "");
 
 InputSection::InputSection(InputFile *f, uint64_t flags, uint32_t type,
diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h
index 58e5306fd6dc..1fb7077ca435 100644
--- a/lld/ELF/InputSection.h
+++ b/lld/ELF/InputSection.h
@@ -48,7 +48,7 @@ template <class ELFT> struct RelsOrRelas {
 // sections.
 class SectionBase {
 public:
-  enum Kind { Regular, Synthetic, Spill, EHFrame, Merge, Output };
+  enum Kind { Regular, Synthetic, EHFrame, Merge, Output };
 
   Kind kind() const { return (Kind)sectionKind; }
 
@@ -382,8 +382,7 @@ public:
 
   static bool classof(const SectionBase *s) {
     return s->kind() == SectionBase::Regular ||
-           s->kind() == SectionBase::Synthetic ||
-           s->kind() == SectionBase::Spill;
+           s->kind() == SectionBase::Synthetic;
   }
 
   // Write this section to a mmap'ed file, assuming Buf is pointing to
@@ -426,26 +425,6 @@ private:
   template <class ELFT> void copyShtGroup(uint8_t *buf);
 };
 
-// A marker for a potential spill location for another input section. This
-// broadly acts as if it were the original section until address assignment.
-// Then it is either replaced with the real input section or removed.
-class PotentialSpillSection : public InputSection {
-public:
-  // The containing input section description; used to quickly replace this stub
-  // with the actual section.
-  InputSectionDescription *isd;
-
-  // Next potential spill location for the same source input section.
-  PotentialSpillSection *next = nullptr;
-
-  PotentialSpillSection(const InputSectionBase &source,
-                        InputSectionDescription &isd);
-
-  static bool classof(const SectionBase *sec) {
-    return sec->kind() == InputSectionBase::Spill;
-  }
-};
-
 static_assert(sizeof(InputSection) <= 160, "InputSection is too big");
 
 class SyntheticSection : public InputSection {
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index 3ba59c112b8a..c0a5014817b9 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -304,9 +304,6 @@ getChangedSymbolAssignment(const SymbolAssignmentMap &oldValues) {
 void LinkerScript::processInsertCommands() {
   SmallVector<OutputDesc *, 0> moves;
   for (const InsertCommand &cmd : insertCommands) {
-    if (config->enableNonContiguousRegions)
-      error("INSERT cannot be used with --enable-non-contiguous-regions");
-
     for (StringRef name : cmd.names) {
       // If base is empty, it may have been discarded by
       // adjustOutputSections(). We do not handle such output sections.
@@ -489,12 +486,10 @@ static void sortInputSections(MutableArrayRef<InputSectionBase *> vec,
 // Compute and remember which sections the InputSectionDescription matches.
 SmallVector<InputSectionBase *, 0>
 LinkerScript::computeInputSections(const InputSectionDescription *cmd,
-                                   ArrayRef<InputSectionBase *> sections,
-                                   const OutputSection &outCmd) {
+                                   ArrayRef<InputSectionBase *> sections) {
   SmallVector<InputSectionBase *, 0> ret;
   SmallVector<size_t, 0> indexes;
   DenseSet<size_t> seen;
-  DenseSet<InputSectionBase *> spills;
   auto sortByPositionThenCommandLine = [&](size_t begin, size_t end) {
     llvm::sort(MutableArrayRef<size_t>(indexes).slice(begin, end - begin));
     for (size_t i = begin; i != end; ++i)
@@ -510,10 +505,10 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
     size_t sizeBeforeCurrPat = ret.size();
 
     for (size_t i = 0, e = sections.size(); i != e; ++i) {
-      // Skip if the section is dead or has been matched by a previous pattern
-      // in this input section description.
+      // Skip if the section is dead or has been matched by a previous input
+      // section description or a previous pattern.
       InputSectionBase *sec = sections[i];
-      if (!sec->isLive() || seen.contains(i))
+      if (!sec->isLive() || sec->parent || seen.contains(i))
         continue;
 
       // For --emit-relocs we have to ignore entries like
@@ -534,29 +529,6 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
           (sec->flags & cmd->withoutFlags) != 0)
         continue;
 
-      if (sec->parent) {
-        // Skip if not allowing multiple matches.
-        if (!config->enableNonContiguousRegions)
-          continue;
-
-        // Disallow spilling into /DISCARD/; special handling would be needed
-        // for this in address assignment, and the semantics are nebulous.
-        if (outCmd.name == "/DISCARD/")
-          continue;
-
-        // Skip if the section's first match was /DISCARD/; such sections are
-        // always discarded.
-        if (sec->parent->name == "/DISCARD/")
-          continue;
-
-        // Skip if the section was already matched by a different input section
-        // description within this output section.
-        if (sec->parent == &outCmd)
-          continue;
-
-        spills.insert(sec);
-      }
-
       ret.push_back(sec);
       indexes.push_back(i);
       seen.insert(i);
@@ -583,30 +555,6 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
   // Matched sections after the last SORT* are sorted by (--sort-alignment,
   // input order).
   sortByPositionThenCommandLine(sizeAfterPrevSort, ret.size());
-
-  // The flag --enable-non-contiguous-regions may cause sections to match an
-  // InputSectionDescription in more than one OutputSection. Matches after the
-  // first were collected in the spills set, so replace these with potential
-  // spill sections.
-  if (!spills.empty()) {
-    for (InputSectionBase *&sec : ret) {
-      if (!spills.contains(sec))
-        continue;
-
-      // Append the spill input section to the list for the input section,
-      // creating it if necessary.
-      PotentialSpillSection *pss = make<PotentialSpillSection>(
-          *sec, const_cast<InputSectionDescription &>(*cmd));
-      auto [it, inserted] =
-          potentialSpillLists.try_emplace(sec, PotentialSpillList{pss, pss});
-      if (!inserted) {
-        PotentialSpillSection *&tail = it->second.tail;
-        tail = tail->next = pss;
-      }
-      sec = pss;
-    }
-  }
-
   return ret;
 }
 
@@ -629,7 +577,7 @@ void LinkerScript::discardSynthetic(OutputSection &outCmd) {
         part.armExidx->exidxSections.end());
     for (SectionCommand *cmd : outCmd.commands)
       if (auto *isd = dyn_cast<InputSectionDescription>(cmd))
-        for (InputSectionBase *s : computeInputSections(isd, secs, outCmd))
+        for (InputSectionBase *s : computeInputSections(isd, secs))
           discard(*s);
   }
 }
@@ -640,7 +588,7 @@ LinkerScript::createInputSectionList(OutputSection &outCmd) {
 
   for (SectionCommand *cmd : outCmd.commands) {
     if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) {
-      isd->sectionBases = computeInputSections(isd, ctx.inputSections, outCmd);
+      isd->sectionBases = computeInputSections(isd, ctx.inputSections);
       for (InputSectionBase *s : isd->sectionBases)
         s->parent = &outCmd;
       ret.insert(ret.end(), isd->sectionBases.begin(), isd->sectionBases.end());
@@ -696,9 +644,6 @@ void LinkerScript::processSectionCommands() {
 
   // Process OVERWRITE_SECTIONS first so that it can overwrite the main script
   // or orphans.
-  if (config->enableNonContiguousRegions && !overwriteSections.empty())
-    error("OVERWRITE_SECTIONS cannot be used with "
-          "--enable-non-contiguous-regions");
   DenseMap<CachedHashStringRef, OutputDesc *> map;
   size_t i = 0;
   for (OutputDesc *osd : overwriteSections) {
@@ -1121,12 +1066,8 @@ void LinkerScript::assignOffsets(OutputSection *sec) {
     // Handle a single input section description command.
     // It calculates and assigns the offsets for each section and also
     // updates the output section size.
-
-    auto &sections = cast<InputSectionDescription>(cmd)->sections;
-    for (InputSection *isec : sections) {
+    for (InputSection *isec : cast<InputSectionDescription>(cmd)->sections) {
       assert(isec->getParent() == sec);
-      if (isa<PotentialSpillSection>(isec))
-        continue;
       const uint64_t pos = dot;
       dot = alignToPowerOf2(dot, isec->addralign);
       isec->outSecOff = dot - sec->addr;
@@ -1423,114 +1364,6 @@ const Defined *LinkerScript::assignAddresses() {
   return getChangedSymbolAssignment(oldValues);
 }
 
-static bool hasRegionOverflowed(MemoryRegion *mr) {
-  if (!mr)
-    return false;
-  return mr->curPos - mr->getOrigin() > mr->getLength();
-}
-
-// Spill input sections in reverse order of address assignment to (potentially)
-// bring memory regions out of overflow. The size savings of a spill can only be
-// estimated, since general linker script arithmetic may occur afterwards.
-// Under-estimates may cause unnecessary spills, but over-estimates can always
-// be corrected on the next pass.
-bool LinkerScript::spillSections() {
-  if (!config->enableNonContiguousRegions)
-    return false;
-
-  bool spilled = false;
-  for (SectionCommand *cmd : reverse(sectionCommands)) {
-    auto *od = dyn_cast<OutputDesc>(cmd);
-    if (!od)
-      continue;
-    OutputSection *osec = &od->osec;
-    if (!osec->memRegion)
-      continue;
-
-    // Input sections that have replaced a potential spill and should be removed
-    // from their input section description.
-    DenseSet<InputSection *> spilledInputSections;
-
-    for (SectionCommand *cmd : reverse(osec->commands)) {
-      if (!hasRegionOverflowed(osec->memRegion) &&
-          !hasRegionOverflowed(osec->lmaRegion))
-        break;
-
-      auto *isd = dyn_cast<InputSectionDescription>(cmd);
-      if (!isd)
-        continue;
-      for (InputSection *isec : reverse(isd->sections)) {
-        // Potential spill locations cannot be spilled.
-        if (isa<PotentialSpillSection>(isec))
-          continue;
-
-        // Find the next potential spill location and remove it from the list.
-        auto it = potentialSpillLists.find(isec);
-        if (it == potentialSpillLists.end())
-          continue;
-        PotentialSpillList &list = it->second;
-        PotentialSpillSection *spill = list.head;
-        if (spill->next)
-          list.head = spill->next;
-        else
-          potentialSpillLists.erase(isec);
-
-        // Replace the next spill location with the spilled section and adjust
-        // its properties to match the new location. Note that the alignment of
-        // the spill section may have diverged from the original due to e.g. a
-        // SUBALIGN. Correct assignment requires the spill's alignment to be
-        // used, not the original.
-        spilledInputSections.insert(isec);
-        *llvm::find(spill->isd->sections, spill) = isec;
-        isec->parent = spill->parent;
-        isec->addralign = spill->addralign;
-
-        // Record the (potential) reduction in the region's end position.
-        osec->memRegion->curPos -= isec->getSize();
-        if (osec->lmaRegion)
-          osec->lmaRegion->curPos -= isec->getSize();
-
-        // Spilling continues until the end position no longer overflows the
-        // region. Then, another round of address assignment will either confirm
-        // the spill's success or lead to yet more spilling.
-        if (!hasRegionOverflowed(osec->memRegion) &&
-            !hasRegionOverflowed(osec->lmaRegion))
-          break;
-      }
-
-      // Remove any spilled input sections to complete their move.
-      if (!spilledInputSections.empty()) {
-        spilled = true;
-        llvm::erase_if(isd->sections, [&](InputSection *isec) {
-          return spilledInputSections.contains(isec);
-        });
-      }
-    }
-  }
-
-  return spilled;
-}
-
-// Erase any potential spill sections that were not used.
-void LinkerScript::erasePotentialSpillSections() {
-  if (potentialSpillLists.empty())
-    return;
-
-  // Collect the set of input section descriptions that contain potential
-  // spills.
-  DenseSet<InputSectionDescription *> isds;
-  for (const auto &[_, list] : potentialSpillLists)
-    for (PotentialSpillSection *s = list.head; s; s = s->next)
-      isds.insert(s->isd);
-
-  for (InputSectionDescription *isd : isds)
-    llvm::erase_if(isd->sections, [](InputSection *s) {
-      return isa<PotentialSpillSection>(s);
-    });
-
-  potentialSpillLists.clear();
-}
-
 // Creates program headers as instructed by PHDRS linker script command.
 SmallVector<PhdrEntry *, 0> LinkerScript::createPhdrs() {
   SmallVector<PhdrEntry *, 0> ret;
diff --git a/lld/ELF/LinkerScript.h b/lld/ELF/LinkerScript.h
index 734d4e7498aa..b09cd12c46f9 100644
--- a/lld/ELF/LinkerScript.h
+++ b/lld/ELF/LinkerScript.h
@@ -10,7 +10,6 @@
 #define LLD_ELF_LINKER_SCRIPT_H
 
 #include "Config.h"
-#include "InputSection.h"
 #include "Writer.h"
 #include "lld/Common/LLVM.h"
 #include "lld/Common/Strings.h"
@@ -288,8 +287,7 @@ class LinkerScript final {
 
   SmallVector<InputSectionBase *, 0>
   computeInputSections(const InputSectionDescription *,
-                       ArrayRef<InputSectionBase *>,
-                       const OutputSection &outCmd);
+                       ArrayRef<InputSectionBase *>);
 
   SmallVector<InputSectionBase *, 0> createInputSectionList(OutputSection &cmd);
 
@@ -335,8 +333,6 @@ public:
 
   bool shouldKeep(InputSectionBase *s);
   const Defined *assignAddresses();
-  bool spillSections();
-  void erasePotentialSpillSections();
   void allocateHeaders(SmallVector<PhdrEntry *, 0> &phdrs);
   void processSectionCommands();
   void processSymbolAssignments();
@@ -404,15 +400,6 @@ public:
   //
   // then provideMap should contain the mapping: 'v' -> ['a', 'b', 'c']
   llvm::MapVector<StringRef, SmallVector<StringRef, 0>> provideMap;
-
-  // List of potential spill locations (PotentialSpillSection) for an input
-  // section.
-  struct PotentialSpillList {
-    // Never nullptr.
-    PotentialSpillSection *head;
-    PotentialSpillSection *tail;
-  };
-  llvm::DenseMap<InputSectionBase *, PotentialSpillList> potentialSpillLists;
 };
 
 struct ScriptWrapper {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 883a6079bf50..b9e05a4b1fd5 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -197,9 +197,6 @@ def emit_relocs: F<"emit-relocs">, HelpText<"Generate relocations in output">;
 def enable_new_dtags: F<"enable-new-dtags">,
   HelpText<"Enable new dynamic tags (default)">;
 
-def enable_non_contiguous_regions : FF<"enable-non-contiguous-regions">,
-  HelpText<"Spill input sections to later matching output sections to avoid memory region overflow">;
-
 def end_group: F<"end-group">,
   HelpText<"Ignored for compatibility with GNU unless you pass --warn-backrefs">;
 
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index fcb4c4387aa9..9c667241360f 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -186,7 +186,7 @@ static MergeSyntheticSection *createMergeSynthetic(StringRef name,
 // new synthetic sections at the location of the first input section
 // that it replaces. It then finalizes each synthetic section in order
 // to compute an output offset for each piece of each input section.
-void OutputSection::finalizeInputSections(LinkerScript *script) {
+void OutputSection::finalizeInputSections() {
   std::vector<MergeSyntheticSection *> mergeSections;
   for (SectionCommand *cmd : commands) {
     auto *isd = dyn_cast<InputSectionDescription>(cmd);
@@ -226,11 +226,6 @@ void OutputSection::finalizeInputSections(LinkerScript *script) {
         i = std::prev(mergeSections.end());
         syn->entsize = ms->entsize;
         isd->sections.push_back(syn);
-        // The merge synthetic section inherits the potential spill locations of
-        // its first contained section.
-        auto it = script->potentialSpillLists.find(ms);
-        if (it != script->potentialSpillLists.end())
-          script->potentialSpillLists.try_emplace(syn, it->second);
       }
       (*i)->addSection(ms);
     }
diff --git a/lld/ELF/OutputSections.h b/lld/ELF/OutputSections.h
index 78fede48a23f..421a0181feb5 100644
--- a/lld/ELF/OutputSections.h
+++ b/lld/ELF/OutputSections.h
@@ -75,7 +75,7 @@ public:
 
   void recordSection(InputSectionBase *isec);
   void commitSection(InputSection *isec);
-  void finalizeInputSections(LinkerScript *script = nullptr);
+  void finalizeInputSections();
 
   // The following members are normally only used in linker scripts.
   MemoryRegion *memRegion = nullptr;
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 298c714adb3b..7b9ada40c0f6 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -4074,13 +4074,6 @@ static bool isDuplicateArmExidxSec(InputSection *prev, InputSection *cur) {
 // InputSection with the highest address and any InputSections that have
 // mergeable .ARM.exidx table entries are removed from it.
 void ARMExidxSyntheticSection::finalizeContents() {
-  // Ensure that any fixed-point iterations after the first see the original set
-  // of sections.
-  if (!originalExecutableSections.empty())
-    executableSections = originalExecutableSections;
-  else if (config->enableNonContiguousRegions)
-    originalExecutableSections = executableSections;
-
   // The executableSections and exidxSections that we use to derive the final
   // contents of this SyntheticSection are populated before
   // processSectionCommands() and ICF. A /DISCARD/ entry in SECTIONS command or
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index 34949025a45f..995fd4b344b0 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -1255,10 +1255,6 @@ private:
   // either find the .ARM.exidx section or know that we need to generate one.
   SmallVector<InputSection *, 0> executableSections;
 
-  // Value of executableSecitons before finalizeContents(), so that it can be
-  // run repeateadly during fixed point iteration.
-  SmallVector<InputSection *, 0> originalExecutableSections;
-
   // The executable InputSection with the highest address to use for the
   // sentinel. We store separately from ExecutableSections as merging of
   // duplicate entries may mean this InputSection is removed from
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 8d529f2bdb9f..e400ed2ae945 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -1403,18 +1403,13 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
   AArch64Err843419Patcher a64p;
   ARMErr657417Patcher a32p;
   script->assignAddresses();
-
   // .ARM.exidx and SHF_LINK_ORDER do not require precise addresses, but they
   // do require the relative addresses of OutputSections because linker scripts
   // can assign Virtual Addresses to OutputSections that are not monotonically
-  // increasing. Anything here must be repeatable, since spilling may change
-  // section order.
-  const auto finalizeOrderDependentContent = [this] {
-    for (Partition &part : partitions)
-      finalizeSynthetic(part.armExidx.get());
-    resolveShfLinkOrder();
-  };
-  finalizeOrderDependentContent();
+  // increasing.
+  for (Partition &part : partitions)
+    finalizeSynthetic(part.armExidx.get());
+  resolveShfLinkOrder();
 
   // Converts call x@GDPLT to call __tls_get_addr
   if (config->emachine == EM_HEXAGON)
@@ -1424,8 +1419,6 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
   for (;;) {
     bool changed = target->needsThunks ? tc.createThunks(pass, outputSections)
                                        : target->relaxOnce(pass);
-    bool spilled = script->spillSections();
-    changed |= spilled;
     ++pass;
 
     // With Thunk Size much smaller than branch range we expect to
@@ -1471,9 +1464,6 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
                     " does not converge");
         break;
       }
-    } else if (spilled) {
-      // Spilling can change relative section order.
-      finalizeOrderDependentContent();
     }
   }
   if (!config->relocatable)
@@ -1493,10 +1483,6 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
              osec->name + " is not a multiple of alignment (" +
              Twine(osec->addralign) + ")");
     }
-
-  // Sizes are no longer allowed to grow, so all allowable spills have been
-  // taken. Remove any leftover potential spills.
-  script->erasePotentialSpillSections();
 }
 
 // If Input Sections have been shrunk (basic block sections) then
diff --git a/lld/docs/ELF/linker_script.rst b/lld/docs/ELF/linker_script.rst
index 7a35534be096..3606ef4fe4b8 100644
--- a/lld/docs/ELF/linker_script.rst
+++ b/lld/docs/ELF/linker_script.rst
@@ -197,14 +197,3 @@ the current location to a max-page-size boundary, ensuring that the next
 
 LLD will insert ``.relro_padding`` immediately before the symbol assignment
 using ``DATA_SEGMENT_RELRO_END``.
-
-Non-contiguous regions
-~~~~~~~~~~~~~~~~~~~~~~
-
-The flag ``--enable-non-contiguous-regions`` allows input sections to spill to
-later matches rather than causing the link to fail by overflowing a memory
-region. Unlike GNU ld, ``/DISCARD/`` only matches previously-unmatched sections
-(i.e., the flag does not affect it). Also, if a section fails to fit at any of
-its matches, the link fails instead of discarding the section. Accordingly, the
-GNU flag ``--enable-non-contiguous-regions-warnings`` is not implemented, as it
-exists to warn about such occurrences.
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index e7a913e025da..f8fdebfeaecf 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -38,12 +38,6 @@ ELF Improvements
 * ``--debug-names`` is added to create a merged ``.debug_names`` index
   from input ``.debug_names`` sections. Type units are not handled yet.
   (`#86508 <https://github.com/llvm/llvm-project/pull/86508>`_)
-* ``--enable-non-contiguous-regions`` option allows automatically packing input
-  sections into memory regions by automatically spilling to later matches if a
-  region would overflow. This reduces the toil of manually packing regions
-  (typical for embedded). It also makes full LTO feasible in such cases, since
-  IR merging currently prevents the linker script from referring to input
-  files. (`#90007 <https://github.com/llvm/llvm-project/pull/90007>`_)
 
 Breaking changes
 ----------------
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 0df13f07f560..9ea1a9c52f2a 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -222,8 +222,6 @@ segment header.
 Generate relocations in the output.
 .It Fl -enable-new-dtags
 Enable new dynamic tags.
-.It Fl -enable-non-contiguous-regions
-Spill input sections to later matching output sections to avoid memory region overflow.
 .It Fl -end-lib
 End a grouping of objects that should be treated as if they were together
 in an archive.
diff --git a/lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test b/lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test
deleted file mode 100644
index 3f7b9c4e5f8b..000000000000
--- a/lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test
+++ /dev/null
@@ -1,55 +0,0 @@
-## When spilling reorders input sections, the .ARM.exidx table is rebuilt using
-## the new order.
-
-# REQUIRES: arm
-# RUN: rm -rf %t && split-file %s %t && cd %t
-# RUN: llvm-mc -filetype=obj -triple=armv7a-none-linux-gnueabi test.s -o test.o
-# RUN: ld.lld -T test.ld test.o -o test --enable-non-contiguous-regions
-# RUN: llvm-readobj -x .ARM.exidx test | FileCheck %s
-
-# CHECK:      20000000 08849780 1c000000 10849880
-# CHECK-NEXT: 1c000000 01000000
-
-#--- test.ld
-MEMORY {
-  exidx : ORIGIN = 0, LENGTH = 32
-  a : ORIGIN = 32, LENGTH = 4
-  b : ORIGIN = 36, LENGTH = 4
-  c : ORIGIN = 40, LENGTH = 4
-}
-
-SECTIONS {
-  .ARM.exidx : { *(.ARM.exidx) } >exidx
-  .first_chance : { *(.text .text.f2) } >a
-  .text.f1 : { *(.text.f1) } >b
-  .last_chance : { *(.text.f2) } >c
-}
-
-#--- test.s
- .syntax unified
- .section .text, "ax",%progbits
- .globl _start
-_start:
- .fnstart
- bx lr
- .save {r7, lr}
- .setfp r7, sp, #0
- .fnend
-
- .section .text.f1, "ax", %progbits
- .globl f1
-f1:
- .fnstart
- bx lr
- .save {r8, lr}
- .setfp r8, sp, #0
- .fnend
-
- .section .text.f2, "ax", %progbits
- .globl f2
-f2:
- .fnstart
- bx lr
- .save {r8, lr}
- .setfp r8, sp, #0
- .fnend
diff --git a/lld/test/ELF/linkerscript/enable-non-contiguous-regions.test b/lld/test/ELF/linkerscript/enable-non-contiguous-regions.test
deleted file mode 100644
index 392106fd476f..000000000000
--- a/lld/test/ELF/linkerscript/enable-non-contiguous-regions.test
+++ /dev/null
@@ -1,265 +0,0 @@
-# REQUIRES: x86
-
-# RUN: rm -rf %t && split-file %s %t && cd %t
-# RUN: llvm-mc -n -filetype=obj -triple=x86_64 spill.s -o spill.o
-
-## An input section spills to a later match when the region of its first match
-## would overflow. The spill uses the alignment of the later match.
-
-# RUN: ld.lld -T spill.ld spill.o -o spill --enable-non-contiguous-regions
-# RUN: llvm-readelf -S spill | FileCheck %s --check-prefix=SPILL
-
-# SPILL:      Name          Type     Address          Off    Size
-# SPILL:      .first_chance PROGBITS 0000000000000000 001000 000001
-# SPILL-NEXT: .last_chance  PROGBITS 0000000000000008 001008 000002
-
-## A spill off the end still fails the link.
-
-# RUN: not ld.lld -T spill-fail.ld spill.o --enable-non-contiguous-regions 2>&1 |\
-# RUN:   FileCheck %s --check-prefix=SPILL-FAIL --implicit-check-not=error:
-
-# SPILL-FAIL: error: section '.last_chance' will not fit in region 'b': overflowed by 2 bytes
-
-## The above spill still occurs when the LMA would overflow, even though the
-## VMA would fit.
-
-# RUN: ld.lld -T spill-lma.ld spill.o -o spill-lma --enable-non-contiguous-regions
-# RUN: llvm-readelf -S spill-lma | FileCheck %s --check-prefix=SPILL-LMA
-
-# SPILL-LMA:      Name          Type     Address          Off    Size
-# SPILL-LMA:      .first_chance PROGBITS 0000000000000000 001000 000001
-# SPILL-LMA-NEXT: .last_chance  PROGBITS 0000000000000003 001003 000002
-
-## A spill occurs to an additional match after the first.
-
-# RUN: ld.lld -T spill-later.ld spill.o -o spill-later --enable-non-contiguous-regions
-# RUN: llvm-readelf -S spill-later | FileCheck %s --check-prefix=SPILL-LATER
-
-# SPILL-LATER:      Name            Type     Address          Off    Size
-# SPILL-LATER:      .first_chance   PROGBITS 0000000000000000 001000 000001
-# SPILL-LATER-NEXT: .second_chance  PROGBITS 0000000000000002 001001 000000
-# SPILL-LATER-NEXT: .last_chance    PROGBITS 0000000000000003 001003 000002
-
-## A later overflow causes an earlier section to spill.
-
-# RUN: ld.lld -T spill-earlier.ld spill.o -o spill-earlier --enable-non-contiguous-regions
-# RUN: llvm-readelf -S spill-earlier | FileCheck %s --check-prefix=SPILL-EARLIER
-
-# SPILL-EARLIER:      Name          Type     Address          Off    Size
-# SPILL-EARLIER:      .first_chance PROGBITS 0000000000000000 001000 000002
-# SPILL-EARLIER-NEXT: .last_chance  PROGBITS 0000000000000002 001002 000001
-
-## An additional match in /DISCARD/ has no effect.
-
-# RUN: not ld.lld -T no-spill-into-discard.ld spill.o --enable-non-contiguous-regions 2>&1 |\
-# RUN:   FileCheck %s --check-prefix=NO-SPILL-INTO-DISCARD --implicit-check-not=error:
-
-# NO-SPILL-INTO-DISCARD: error: section '.osec' will not fit in region 'a': overflowed by 1 bytes
-
-## An additional match after /DISCARD/ has no effect.
-
-# RUN: ld.lld -T no-spill-from-discard.ld spill.o -o no-spill-from-discard --enable-non-contiguous-regions
-# RUN: llvm-readelf -S no-spill-from-discard | FileCheck %s --check-prefix=NO-SPILL-FROM-DISCARD
-
-# NO-SPILL-FROM-DISCARD: Name          Type     Address          Off    Size
-# NO-SPILL-FROM-DISCARD-NOT: .osec
-
-## SHF_MERGEd sections are spilled according to the matches of the first merged
-## input section (the one giving the resulting section its name).
-
-# RUN: llvm-mc -n -filetype=obj -triple=x86_64 merge.s -o merge.o
-# RUN: ld.lld -T spill-merge.ld merge.o -o spill-merge --enable-non-contiguous-regions
-# RUN: llvm-readelf -S spill-merge | FileCheck %s --check-prefix=SPILL-MERGE
-
-# SPILL-MERGE:      Name          Type     Address          Off    Size
-# SPILL-MERGE:      .first  PROGBITS 0000000000000000 000190 000000
-# SPILL-MERGE-NEXT: .second PROGBITS 0000000000000001 001001 000002
-# SPILL-MERGE-NEXT: .third  PROGBITS 0000000000000003 001003 000000
-
-## An error is reported for INSERT.
-
-# RUN: not ld.lld -T insert.ld spill.o --enable-non-contiguous-regions 2>&1 |\
-# RUN:   FileCheck %s --check-prefix=INSERT
-
-# INSERT: error: INSERT cannot be used with --enable-non-contiguous-regions
-
-## An error is reported for OVERWRITE_SECTIONS.
-
-# RUN: not ld.lld -T overwrite-sections.ld spill.o --enable-non-contiguous-regions 2>&1 |\
-# RUN:   FileCheck %s --check-prefix=OVERWRITE_SECTIONS
-
-# OVERWRITE_SECTIONS: error: OVERWRITE_SECTIONS cannot be used with --enable-non-contiguous-regions
-
-## SHF_LINK_ORDER is reordered when spilling changes relative section order.
-
-# RUN: llvm-mc -n -filetype=obj -triple=x86_64 link-order.s -o link-order.o
-# RUN: ld.lld -T link-order.ld link-order.o -o link-order --enable-non-contiguous-regions
-# RUN: llvm-readobj -x .order link-order | FileCheck %s --check-prefix=LINK-ORDER
-
-# LINK-ORDER: 020301
-
-#--- spill.s
-.section .one_byte_section,"a",@progbits
-.fill 1
-
-.section .two_byte_section,"a",@progbits
-.fill 2
-
-#--- spill.ld
-MEMORY {
-  a : ORIGIN = 0, LENGTH = 2
-  b : ORIGIN = 2, LENGTH = 16
-}
-
-SECTIONS {
-  .first_chance : SUBALIGN(1) { *(.one_byte_section) *(.two_byte_section) } >a
-  .last_chance : SUBALIGN(8) { *(.two_byte_section) } >b
-}
-
-#--- spill-fail.ld
-MEMORY {
-  a : ORIGIN = 0, LENGTH = 1
-  b : ORIGIN = 2, LENGTH = 0
-}
-
-SECTIONS {
-  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >a
-  .last_chance : { *(.two_byte_section) } >b
-}
-
-#--- spill-lma.ld
-MEMORY {
-  vma_a : ORIGIN = 0, LENGTH = 3
-  vma_b : ORIGIN = 3, LENGTH = 3
-  lma_a : ORIGIN = 6, LENGTH = 2
-  lma_b : ORIGIN = 8, LENGTH = 2
-}
-
-SECTIONS {
-  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >vma_a AT>lma_a
-  .last_chance : { *(.two_byte_section) } >vma_b AT>lma_b
-}
-
-#--- spill-later.ld
-MEMORY {
-  a : ORIGIN = 0, LENGTH = 2
-  b : ORIGIN = 2, LENGTH = 1
-  c : ORIGIN = 3, LENGTH = 2
-}
-
-SECTIONS {
-  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >a
-  .second_chance : { *(.two_byte_section) } >b
-  .last_chance : { *(.two_byte_section) } >c
-}
-
-#--- spill-earlier.ld
-MEMORY {
-  a : ORIGIN = 0, LENGTH = 2
-  b : ORIGIN = 2, LENGTH = 1
-}
-
-SECTIONS {
-  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >a
-  .last_chance : { *(.one_byte_section) } >b
-}
-
-#--- no-spill-into-discard.ld
-MEMORY {
-  a : ORIGIN = 0, LENGTH = 1
-}
-
-SECTIONS {
-  .osec : { *(.two_byte_section) } >a
-  /DISCARD/ : { *(.one_byte_section) *(.two_byte_section) }
-}
-
-#--- no-spill-from-discard.ld
-MEMORY {
-  a : ORIGIN = 0, LENGTH = 2
-}
-
-SECTIONS {
-  /DISCARD/ : { *(.one_byte_section) *(.two_byte_section) }
-  .osec : { *(.two_byte_section) } >a
-}
-
-#--- merge.s
-.section .a,"aM",@progbits,1
-.byte 0x12, 0x34
-
-.section .b,"aM",@progbits,1
-.byte 0x12
-
-#--- spill-merge.ld
-MEMORY {
-  a : ORIGIN = 0, LENGTH = 1
-  b : ORIGIN = 1, LENGTH = 2
-  c : ORIGIN = 3, LENGTH = 2
-}
-
-SECTIONS {
-  .first : { *(.a) *(.b) } >a
-  .second : { *(.a) } >b
-  .third : { *(.b) } >c
-}
-
-#--- insert.ld
-MEMORY {
-  a : ORIGIN = 0, LENGTH = 1
-}
-
-SECTIONS {
-  .a : { *(.two_byte_section) } >a
-}
-
-SECTIONS {
-  .b : { *(.one_byte_section) } >a
-} INSERT AFTER .a;
-
-#--- overwrite-sections.ld
-MEMORY {
-  a : ORIGIN = 0, LENGTH = 1
-}
-
-SECTIONS {
-  .a : { *(.two_byte_section) } >a
-}
-
-OVERWRITE_SECTIONS {
-  .b : { *(.one_byte_section) } >a
-}
-
-#--- link-order.s
-.section .a,"a",@progbits
-.fill 1
-
-.section .b,"a",@progbits
-.fill 1
-
-.section .c,"a",@progbits
-.fill 1
-
-.section .link_order.a,"ao",@progbits,.a
-.byte 1
-
-.section .link_order.b,"ao",@progbits,.b
-.byte 2
-
-.section .link_order.c,"ao",@progbits,.c
-.byte 3
-
-#--- link-order.ld
-MEMORY {
-  order : ORIGIN = 0, LENGTH = 3
-  potential_a : ORIGIN = 3, LENGTH = 0
-  bc : ORIGIN = 3, LENGTH = 2
-  actual_a : ORIGIN = 5, LENGTH = 1
-}
-
-SECTIONS {
-  .order :  { *(.link_order.*) } > order
-  .potential_a : { *(.a) } >potential_a
-  .bc : { *(.b) *(.c) } >bc
-  .actual_a : { *(.a) } >actual_a
-}
-- 
GitLab


From 276c0bd4b386cc6b9d91e5e44ca7b053c11f13b5 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Mon, 13 May 2024 18:47:39 +0100
Subject: [PATCH 0610/1206] [lldb][ExpressionParser][NFCI] Log pointers as hex
 (#91989)

This ensures that we log pointers as lower-case hex. E.g., instead of:
```
 LayoutRecordType on (ASTContext*)0x000000010E78D600 'scratch ASTContext' for (RecordDecl*)0x000000010E797
```
we now log:
```
 LayoutRecordType on (ASTContext*)0x000000010e78d600 'scratch ASTContext' for (RecordDecl*)0x000000010e797
```

Which is consistent with how the AST dump gets emitted into the log.
This makes it easier to correlate pointers we log from LLDB and pointers
that are part of any AST dumps in the same `expr` log.
---
 .../Clang/ClangASTImporter.cpp                | 62 ++++++++++---------
 .../ExpressionParser/Clang/ClangASTSource.cpp | 44 ++++++-------
 2 files changed, 54 insertions(+), 52 deletions(-)

diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
index 30b50df79da9..44071d1ea71c 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
@@ -313,8 +313,8 @@ CompilerType ClangASTImporter::DeportType(TypeSystemClang &dst,
     return {};
 
   LLDB_LOG(log,
-           "    [ClangASTImporter] DeportType called on ({0}Type*){1} "
-           "from (ASTContext*){2} to (ASTContext*){3}",
+           "    [ClangASTImporter] DeportType called on ({0}Type*){1:x} "
+           "from (ASTContext*){2:x} to (ASTContext*){3:x}",
            src_type.GetTypeName(), src_type.GetOpaqueQualType(),
            &src_ctxt->getASTContext(), &dst.getASTContext());
 
@@ -334,8 +334,8 @@ clang::Decl *ClangASTImporter::DeportDecl(clang::ASTContext *dst_ctx,
 
   clang::ASTContext *src_ctx = &decl->getASTContext();
   LLDB_LOG(log,
-           "    [ClangASTImporter] DeportDecl called on ({0}Decl*){1} from "
-           "(ASTContext*){2} to (ASTContext*){3}",
+           "    [ClangASTImporter] DeportDecl called on ({0}Decl*){1:x} from "
+           "(ASTContext*){2:x} to (ASTContext*){3:x}",
            decl->getDeclKindName(), decl, src_ctx, dst_ctx);
 
   DeclContextOverride decl_context_override;
@@ -352,8 +352,8 @@ clang::Decl *ClangASTImporter::DeportDecl(clang::ASTContext *dst_ctx,
     return nullptr;
 
   LLDB_LOG(log,
-           "    [ClangASTImporter] DeportDecl deported ({0}Decl*){1} to "
-           "({2}Decl*){3}",
+           "    [ClangASTImporter] DeportDecl deported ({0}Decl*){1:x} to "
+           "({2}Decl*){3:x}",
            decl->getDeclKindName(), decl, result->getDeclKindName(), result);
 
   return result;
@@ -637,8 +637,8 @@ bool ClangASTImporter::importRecordLayoutFromOrigin(
 
   clang::ASTContext &dest_ctx = record->getASTContext();
   LLDB_LOG(log,
-           "LayoutRecordType on (ASTContext*){0} '{1}' for (RecordDecl*)"
-           "{2} [name = '{3}']",
+           "LayoutRecordType on (ASTContext*){0:x} '{1}' for (RecordDecl*)"
+           "{2:x} [name = '{3}']",
            &dest_ctx,
            TypeSystemClang::GetASTContext(&dest_ctx)->getDisplayName(), record,
            record->getName());
@@ -703,7 +703,7 @@ bool ClangASTImporter::importRecordLayoutFromOrigin(
 
   if (log) {
     LLDB_LOG(log, "LRT returned:");
-    LLDB_LOG(log, "LRT   Original = (RecordDecl*){0}",
+    LLDB_LOG(log, "LRT   Original = (RecordDecl*){0:x}",
              static_cast<const void *>(origin_record.decl));
     LLDB_LOG(log, "LRT   Size = {0}", size);
     LLDB_LOG(log, "LRT   Alignment = {0}", alignment);
@@ -711,11 +711,11 @@ bool ClangASTImporter::importRecordLayoutFromOrigin(
     for (RecordDecl::field_iterator fi = record->field_begin(),
                                     fe = record->field_end();
          fi != fe; ++fi) {
-      LLDB_LOG(log,
-               "LRT     (FieldDecl*){0}, Name = '{1}', Type = '{2}', Offset = "
-               "{3} bits",
-               *fi, fi->getName(), fi->getType().getAsString(),
-               field_offsets[*fi]);
+      LLDB_LOG(
+          log,
+          "LRT     (FieldDecl*){0:x}, Name = '{1}', Type = '{2}', Offset = "
+          "{3} bits",
+          *fi, fi->getName(), fi->getType().getAsString(), field_offsets[*fi]);
     }
     DeclFromParser<const CXXRecordDecl> parser_cxx_record =
         DynCast<const CXXRecordDecl>(parser_record);
@@ -734,7 +734,7 @@ bool ClangASTImporter::importRecordLayoutFromOrigin(
             DynCast<CXXRecordDecl>(base_record);
 
         LLDB_LOG(log,
-                 "LRT     {0}(CXXRecordDecl*){1}, Name = '{2}', Offset = "
+                 "LRT     {0}(CXXRecordDecl*){1:x}, Name = '{2}', Offset = "
                  "{3} chars",
                  (is_virtual ? "Virtual " : ""), base_cxx_record.decl,
                  base_cxx_record.decl->getName(),
@@ -1025,7 +1025,7 @@ void ClangASTImporter::ForgetDestination(clang::ASTContext *dst_ast) {
   Log *log = GetLog(LLDBLog::Expressions);
 
   LLDB_LOG(log,
-           "    [ClangASTImporter] Forgetting destination (ASTContext*){0}",
+           "    [ClangASTImporter] Forgetting destination (ASTContext*){0:x}",
            dst_ast);
 
   m_metadata_map.erase(dst_ast);
@@ -1039,7 +1039,7 @@ void ClangASTImporter::ForgetSource(clang::ASTContext *dst_ast,
 
   LLDB_LOG(log,
            "    [ClangASTImporter] Forgetting source->dest "
-           "(ASTContext*){0}->(ASTContext*){1}",
+           "(ASTContext*){0:x}->(ASTContext*){1:x}",
            src_ast, dst_ast);
 
   if (!md)
@@ -1164,9 +1164,10 @@ void ClangASTImporter::ASTImporterDelegate::ImportDefinitionTo(
           from_named_decl->printName(name_stream);
           name_stream.flush();
         }
-        LLDB_LOG(log_ast, "==== [ClangASTImporter][TUDecl: {0}] Imported "
-                          "({1}Decl*){2}, named {3} (from "
-                          "(Decl*){4})",
+        LLDB_LOG(log_ast,
+                 "==== [ClangASTImporter][TUDecl: {0:x}] Imported "
+                 "({1}Decl*){2:x}, named {3} (from "
+                 "(Decl*){4:x})",
                  static_cast<void *>(to->getTranslationUnitDecl()),
                  from->getDeclKindName(), static_cast<void *>(to), name_string,
                  static_cast<void *>(from));
@@ -1294,14 +1295,15 @@ void ClangASTImporter::ASTImporterDelegate::Imported(clang::Decl *from,
       from_named_decl->printName(name_stream);
       name_stream.flush();
 
-      LLDB_LOG(log,
-               "    [ClangASTImporter] Imported ({0}Decl*){1}, named {2} (from "
-               "(Decl*){3}), metadata {4}",
-               from->getDeclKindName(), to, name_string, from, user_id);
+      LLDB_LOG(
+          log,
+          "    [ClangASTImporter] Imported ({0}Decl*){1:x}, named {2} (from "
+          "(Decl*){3:x}), metadata {4}",
+          from->getDeclKindName(), to, name_string, from, user_id);
     } else {
       LLDB_LOG(log,
-               "    [ClangASTImporter] Imported ({0}Decl*){1} (from "
-               "(Decl*){2}), metadata {3}",
+               "    [ClangASTImporter] Imported ({0}Decl*){1:x} (from "
+               "(Decl*){2:x}), metadata {3}",
                from->getDeclKindName(), to, from, user_id);
     }
   }
@@ -1321,8 +1323,8 @@ void ClangASTImporter::ASTImporterDelegate::Imported(clang::Decl *from,
 
         LLDB_LOG(log,
                  "    [ClangASTImporter] Propagated origin "
-                 "(Decl*){0}/(ASTContext*){1} from (ASTContext*){2} to "
-                 "(ASTContext*){3}",
+                 "(Decl*){0:x}/(ASTContext*){1:x} from (ASTContext*){2:x} to "
+                 "(ASTContext*){3:x}",
                  origin.decl, origin.ctx, &from->getASTContext(),
                  &to->getASTContext());
       }
@@ -1335,7 +1337,7 @@ void ClangASTImporter::ASTImporterDelegate::Imported(clang::Decl *from,
 
       LLDB_LOG(log,
                "    [ClangASTImporter] Decl has no origin information in "
-               "(ASTContext*){0}",
+               "(ASTContext*){0:x}",
                &from->getASTContext());
     }
 
@@ -1356,7 +1358,7 @@ void ClangASTImporter::ASTImporterDelegate::Imported(clang::Decl *from,
 
     LLDB_LOG(log,
              "    [ClangASTImporter] Sourced origin "
-             "(Decl*){0}/(ASTContext*){1} into (ASTContext*){2}",
+             "(Decl*){0:x}/(ASTContext*){1:x} into (ASTContext*){2:x}",
              from, m_source_ctx, &to->getASTContext());
   }
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
index 75493eb10d73..82a7a2cc3f1e 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
@@ -193,7 +193,7 @@ TagDecl *ClangASTSource::FindCompleteType(const TagDecl *decl) {
     if (!namespace_map)
       return nullptr;
 
-    LLDB_LOGV(log, "      CTD Inspecting namespace map{0} ({1} entries)",
+    LLDB_LOGV(log, "      CTD Inspecting namespace map{0:x} ({1} entries)",
               namespace_map.get(), namespace_map->size());
 
     for (const ClangASTImporter::NamespaceMapItem &item : *namespace_map) {
@@ -265,7 +265,7 @@ void ClangASTSource::CompleteType(TagDecl *tag_decl) {
   if (log) {
     LLDB_LOG(log,
              "    CompleteTagDecl on (ASTContext*){0} Completing "
-             "(TagDecl*){1} named {2}",
+             "(TagDecl*){1:x} named {2}",
              m_clang_ast_context->getDisplayName(), tag_decl,
              tag_decl->getName());
 
@@ -292,7 +292,7 @@ void ClangASTSource::CompleteType(clang::ObjCInterfaceDecl *interface_decl) {
   Log *log = GetLog(LLDBLog::Expressions);
 
   LLDB_LOG(log,
-           "    [CompleteObjCInterfaceDecl] on (ASTContext*){0} '{1}' "
+           "    [CompleteObjCInterfaceDecl] on (ASTContext*){0:x} '{1}' "
            "Completing an ObjCInterfaceDecl named {1}",
            m_ast_context, m_clang_ast_context->getDisplayName(),
            interface_decl->getName());
@@ -385,7 +385,7 @@ void ClangASTSource::FindExternalLexicalDecls(
   if (log) {
     if (const NamedDecl *context_named_decl = dyn_cast<NamedDecl>(context_decl))
       LLDB_LOG(log,
-               "FindExternalLexicalDecls on (ASTContext*){0} '{1}' in "
+               "FindExternalLexicalDecls on (ASTContext*){0:x} '{1}' in "
                "'{2}' ({3}Decl*){4}",
                m_ast_context, m_clang_ast_context->getDisplayName(),
                context_named_decl->getNameAsString().c_str(),
@@ -393,14 +393,14 @@ void ClangASTSource::FindExternalLexicalDecls(
                static_cast<const void *>(context_decl));
     else if (context_decl)
       LLDB_LOG(log,
-               "FindExternalLexicalDecls on (ASTContext*){0} '{1}' in "
+               "FindExternalLexicalDecls on (ASTContext*){0:x} '{1}' in "
                "({2}Decl*){3}",
                m_ast_context, m_clang_ast_context->getDisplayName(),
                context_decl->getDeclKindName(),
                static_cast<const void *>(context_decl));
     else
       LLDB_LOG(log,
-               "FindExternalLexicalDecls on (ASTContext*){0} '{1}' in a "
+               "FindExternalLexicalDecls on (ASTContext*){0:x} '{1}' in a "
                "NULL context",
                m_ast_context, m_clang_ast_context->getDisplayName());
   }
@@ -410,7 +410,7 @@ void ClangASTSource::FindExternalLexicalDecls(
   if (!original.Valid())
     return;
 
-  LLDB_LOG(log, "  FELD Original decl {0} (Decl*){1:x}:\n{2}",
+  LLDB_LOG(log, "  FELD Original decl (ASTContext*){0:x} (Decl*){1:x}:\n{2}",
            static_cast<void *>(original.ctx),
            static_cast<void *>(original.decl),
            ClangUtil::DumpDecl(original.decl));
@@ -508,19 +508,19 @@ void ClangASTSource::FindExternalVisibleDecls(NameSearchContext &context) {
     if (!context.m_decl_context)
       LLDB_LOG(log,
                "ClangASTSource::FindExternalVisibleDecls on "
-               "(ASTContext*){0} '{1}' for '{2}' in a NULL DeclContext",
+               "(ASTContext*){0:x} '{1}' for '{2}' in a NULL DeclContext",
                m_ast_context, m_clang_ast_context->getDisplayName(), name);
     else if (const NamedDecl *context_named_decl =
                  dyn_cast<NamedDecl>(context.m_decl_context))
       LLDB_LOG(log,
                "ClangASTSource::FindExternalVisibleDecls on "
-               "(ASTContext*){0} '{1}' for '{2}' in '{3}'",
+               "(ASTContext*){0:x} '{1}' for '{2}' in '{3}'",
                m_ast_context, m_clang_ast_context->getDisplayName(), name,
                context_named_decl->getName());
     else
       LLDB_LOG(log,
                "ClangASTSource::FindExternalVisibleDecls on "
-               "(ASTContext*){0} '{1}' for '{2}' in a '{3}'",
+               "(ASTContext*){0:x} '{1}' for '{2}' in a '{3}'",
                m_ast_context, m_clang_ast_context->getDisplayName(), name,
                context.m_decl_context->getDeclKindName());
   }
@@ -542,7 +542,7 @@ void ClangASTSource::FindExternalVisibleDecls(NameSearchContext &context) {
 
   if (!context.m_namespace_map->empty()) {
     if (log && log->GetVerbose())
-      LLDB_LOG(log, "  CAS::FEVD Registering namespace map {0} ({1} entries)",
+      LLDB_LOG(log, "  CAS::FEVD Registering namespace map {0:x} ({1} entries)",
                context.m_namespace_map.get(), context.m_namespace_map->size());
 
     NamespaceDecl *clang_namespace_decl =
@@ -918,7 +918,7 @@ void ClangASTSource::FindObjCMethodDecls(NameSearchContext &context) {
   ConstString selector_name(ss.GetString());
 
   LLDB_LOG(log,
-           "ClangASTSource::FindObjCMethodDecls on (ASTContext*){0} '{1}' "
+           "ClangASTSource::FindObjCMethodDecls on (ASTContext*){0:x} '{1}' "
            "for selector [{2} {3}]",
            m_ast_context, m_clang_ast_context->getDisplayName(),
            interface_decl->getName(), selector_name);
@@ -1062,7 +1062,7 @@ void ClangASTSource::FindObjCMethodDecls(NameSearchContext &context) {
 
     LLDB_LOG(log,
              "CAS::FOPD trying origin "
-             "(ObjCInterfaceDecl*){0}/(ASTContext*){1}...",
+             "(ObjCInterfaceDecl*){0:x}/(ASTContext*){1:x}...",
              complete_interface_decl, &complete_iface_decl->getASTContext());
 
     FindObjCMethodDeclsWithOrigin(context, complete_interface_decl,
@@ -1199,7 +1199,7 @@ void ClangASTSource::FindObjCPropertyAndIvarDecls(NameSearchContext &context) {
 
   LLDB_LOG(log,
            "ClangASTSource::FindObjCPropertyAndIvarDecls on "
-           "(ASTContext*){0} '{1}' for '{2}.{3}'",
+           "(ASTContext*){0:x} '{1}' for '{2}.{3}'",
            m_ast_context, m_clang_ast_context->getDisplayName(),
            parser_iface_decl->getName(), context.m_decl_name.getAsString());
 
@@ -1208,7 +1208,7 @@ void ClangASTSource::FindObjCPropertyAndIvarDecls(NameSearchContext &context) {
 
   LLDB_LOG(log,
            "CAS::FOPD couldn't find the property on origin "
-           "(ObjCInterfaceDecl*){0}/(ASTContext*){1}, searching "
+           "(ObjCInterfaceDecl*){0:x}/(ASTContext*){1:x}, searching "
            "elsewhere...",
            origin_iface_decl.decl, &origin_iface_decl->getASTContext());
 
@@ -1233,7 +1233,7 @@ void ClangASTSource::FindObjCPropertyAndIvarDecls(NameSearchContext &context) {
 
     LLDB_LOG(log,
              "CAS::FOPD trying origin "
-             "(ObjCInterfaceDecl*){0}/(ASTContext*){1}...",
+             "(ObjCInterfaceDecl*){0:x}/(ASTContext*){1:x}...",
              complete_iface_decl.decl, &complete_iface_decl->getASTContext());
 
     FindObjCPropertyAndIvarDeclsWithOrigin(context, complete_iface_decl);
@@ -1265,8 +1265,8 @@ void ClangASTSource::FindObjCPropertyAndIvarDecls(NameSearchContext &context) {
       break;
 
     LLDB_LOG(log,
-             "CAS::FOPD[{0}] trying module "
-             "(ObjCInterfaceDecl*){0}/(ASTContext*){1}...",
+             "CAS::FOPD[{0:x}] trying module "
+             "(ObjCInterfaceDecl*){0:x}/(ASTContext*){1:x}...",
              interface_decl_from_modules.decl,
              &interface_decl_from_modules->getASTContext());
 
@@ -1309,8 +1309,8 @@ void ClangASTSource::FindObjCPropertyAndIvarDecls(NameSearchContext &context) {
       break;
 
     LLDB_LOG(log,
-             "CAS::FOPD[{0}] trying runtime "
-             "(ObjCInterfaceDecl*){0}/(ASTContext*){1}...",
+             "CAS::FOPD[{0:x}] trying runtime "
+             "(ObjCInterfaceDecl*){0:x}/(ASTContext*){1:x}...",
              interface_decl_from_runtime.decl,
              &interface_decl_from_runtime->getASTContext());
 
@@ -1329,7 +1329,7 @@ void ClangASTSource::LookupInNamespace(NameSearchContext &context) {
   ClangASTImporter::NamespaceMapSP namespace_map =
       m_ast_importer_sp->GetNamespaceMap(namespace_context);
 
-  LLDB_LOGV(log, "  CAS::FEVD Inspecting namespace map {0} ({1} entries)",
+  LLDB_LOGV(log, "  CAS::FEVD Inspecting namespace map {0:x} ({1} entries)",
             namespace_map.get(), namespace_map->size());
 
   if (!namespace_map)
@@ -1366,7 +1366,7 @@ void ClangASTSource::CompleteNamespaceMap(
   if (log) {
     if (parent_map && parent_map->size())
       LLDB_LOG(log,
-               "CompleteNamespaceMap on (ASTContext*){0} '{1}' Searching "
+               "CompleteNamespaceMap on (ASTContext*){0:x} '{1}' Searching "
                "for namespace {2} in namespace {3}",
                m_ast_context, m_clang_ast_context->getDisplayName(), name,
                parent_map->begin()->second.GetName());
-- 
GitLab


From 0df67fa212feb3cb503cd05c440afac446bb9457 Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Mon, 13 May 2024 14:50:08 -0300
Subject: [PATCH 0611/1206] Revert "[clang] Revert default behavior change of
 P0522R0 implementation (#91811)" (#91837)

With blocking issues fixed, re-enable relaxed template template argument
matching by reverting these commits.

This reverts commit 4198aebc96cb0236fc63e29a92d886e6a2e3fedb.
This reverts commit 2d5634a4b39d8e5497b6a67caa54049b3cfade8e.
---
 clang/lib/Driver/ToolChains/Clang.cpp              | 14 +++++++-------
 .../Driver/frelaxed-template-template-args.cpp     |  6 +++---
 clang/test/Driver/rewrite-legacy-objc.m            |  6 +++---
 clang/test/Driver/rewrite-objc.m                   |  2 +-
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index f0cc018b6668..42feb1650574 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -7249,15 +7249,15 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.addOptOutFlag(CmdArgs, options::OPT_fassume_unique_vtables,
                      options::OPT_fno_assume_unique_vtables);
 
-  // -fno-relaxed-template-template-args is deprecated.
-  if (Arg *A = Args.getLastArg(options::OPT_frelaxed_template_template_args,
-                               options::OPT_fno_relaxed_template_template_args);
-      A &&
-      A->getOption().matches(options::OPT_fno_relaxed_template_template_args))
+  // -frelaxed-template-template-args is deprecated.
+  if (Arg *A =
+          Args.getLastArg(options::OPT_frelaxed_template_template_args,
+                          options::OPT_fno_relaxed_template_template_args)) {
     D.Diag(diag::warn_drv_deprecated_arg)
         << A->getAsString(Args) << /*hasReplacement=*/false;
-  else
-    CmdArgs.push_back("-fno-relaxed-template-template-args");
+    if (A->getOption().matches(options::OPT_fno_relaxed_template_template_args))
+      CmdArgs.push_back("-fno-relaxed-template-template-args");
+  }
 
   // -fsized-deallocation is off by default, as it is an ABI-breaking change for
   // most platforms.
diff --git a/clang/test/Driver/frelaxed-template-template-args.cpp b/clang/test/Driver/frelaxed-template-template-args.cpp
index 136c360276a1..57fc4e3da6e5 100644
--- a/clang/test/Driver/frelaxed-template-template-args.cpp
+++ b/clang/test/Driver/frelaxed-template-template-args.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang -fsyntax-only -### %s 2>&1 | FileCheck --check-prefix=CHECK-DEF %s
-// RUN: %clang -fsyntax-only -frelaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-ON --allow-empty %s
+// RUN: %clang -fsyntax-only -frelaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-ON %s
 // RUN: %clang -fsyntax-only -fno-relaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-OFF %s
 
-// CHECK-DEF: "-cc1"{{.*}} "-fno-relaxed-template-template-args"
-// CHECK-ON-NOT: warning: argument '-frelaxed-template-template-args' is deprecated [-Wdeprecated]
+// CHECK-DEF-NOT: "-cc1"{{.*}} "-fno-relaxed-template-template-args"
+// CHECK-ON:  warning: argument '-frelaxed-template-template-args' is deprecated [-Wdeprecated]
 // CHECK-OFF: warning: argument '-fno-relaxed-template-template-args' is deprecated [-Wdeprecated]
diff --git a/clang/test/Driver/rewrite-legacy-objc.m b/clang/test/Driver/rewrite-legacy-objc.m
index d45fb8c405c5..413a7a7a61f0 100644
--- a/clang/test/Driver/rewrite-legacy-objc.m
+++ b/clang/test/Driver/rewrite-legacy-objc.m
@@ -3,11 +3,11 @@
 // TEST0: "-cc1"
 // TEST0: "-rewrite-objc"
 // FIXME: CHECK-NOT is broken somehow, it doesn't work here. Check adjacency instead.
-// TEST0: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fno-relaxed-template-template-args" "-fmax-type-align=16"
+// TEST0: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fmax-type-align=16"
 // TEST0: rewrite-legacy-objc.m"
 // RUN: %clang --target=i386-apple-macosx10.9.0 -rewrite-legacy-objc %s -o - -### 2>&1 | \
 // RUN:   FileCheck -check-prefix=TEST1 %s
 // RUN: %clang --target=i386-apple-macosx10.6.0 -rewrite-legacy-objc %s -o - -### 2>&1 | \
 // RUN:   FileCheck -check-prefix=TEST2 %s
-// TEST1: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fno-relaxed-template-template-args" "-fmax-type-align=16"
-// TEST2: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fno-relaxed-template-template-args" "-fmax-type-align=16"
+// TEST1: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fmax-type-align=16"
+// TEST2: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fmax-type-align=16"
diff --git a/clang/test/Driver/rewrite-objc.m b/clang/test/Driver/rewrite-objc.m
index d19d38d8ab83..de3577a770df 100644
--- a/clang/test/Driver/rewrite-objc.m
+++ b/clang/test/Driver/rewrite-objc.m
@@ -2,4 +2,4 @@
 // RUN:   FileCheck -check-prefix=TEST0 %s
 // TEST0: "-cc1" {{.*}} "-rewrite-objc"
 // FIXME: CHECK-NOT is broken somehow, it doesn't work here. Check adjacency instead.
-// TEST0: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fno-relaxed-template-template-args" "-fmax-type-align=16"
+// TEST0: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1"{{.*}} "-fobjc-runtime=macosx" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fmax-type-align=16"
-- 
GitLab


From 5df01ed79c47b0b10ea6be87251ec6e0484515a5 Mon Sep 17 00:00:00 2001
From: Benoit Jacob <jacob.benoit.1@gmail.com>
Date: Mon, 13 May 2024 13:50:27 -0400
Subject: [PATCH 0612/1206] Revert "[mlir][vector] Add Vector-dialect
 interleave-to-shuffle pattern, enable in VectorToSPIRV" (#92006)

Reverts llvm/llvm-project#91800

Reason: https://lab.llvm.org/buildbot/#/builders/268/builds/13935
---
 .../Vector/TransformOps/VectorTransformOps.td | 14 -------
 .../Vector/Transforms/LoweringPatterns.h      |  3 --
 .../VectorToSPIRV/VectorToSPIRV.cpp           |  4 --
 .../TransformOps/VectorTransformOps.cpp       |  5 ---
 .../Transforms/LowerVectorInterleave.cpp      | 41 -------------------
 .../Vector/vector-interleave-to-shuffle.mlir  | 21 ----------
 6 files changed, 88 deletions(-)
 delete mode 100644 mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir

diff --git a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
index bc3c16d40520..f6371f39c394 100644
--- a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
+++ b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
@@ -306,20 +306,6 @@ def ApplyLowerInterleavePatternsOp : Op<Transform_Dialect,
   let assemblyFormat = "attr-dict";
 }
 
-def ApplyInterleaveToShufflePatternsOp : Op<Transform_Dialect,
-    "apply_patterns.vector.interleave_to_shuffle",
-    [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
-  let description = [{
-    Indicates that 1D vector interleave operations should be rewritten as
-    vector shuffle operations.
-
-    This is motivated by some current codegen backends not handling vector
-    interleave operations.
-  }];
-
-  let assemblyFormat = "attr-dict";
-}
-
 def ApplyRewriteNarrowTypePatternsOp : Op<Transform_Dialect,
     "apply_patterns.vector.rewrite_narrow_types",
     [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
index 8fd9904fabc0..350d2777cadf 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
@@ -273,9 +273,6 @@ void populateVectorInterleaveLoweringPatterns(RewritePatternSet &patterns,
                                               int64_t targetRank = 1,
                                               PatternBenefit benefit = 1);
 
-void populateVectorInterleaveToShufflePatterns(RewritePatternSet &patterns,
-                                               PatternBenefit benefit = 1);
-
 } // namespace vector
 } // namespace mlir
 #endif // MLIR_DIALECT_VECTOR_TRANSFORMS_LOWERINGPATTERNS_H
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index c2dd37f48146..868a3521e7a0 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -18,7 +18,6 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
-#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
@@ -829,9 +828,6 @@ void mlir::populateVectorToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
   // than the generic one that extracts all elements.
   patterns.add<VectorReductionToFPDotProd>(typeConverter, patterns.getContext(),
                                            PatternBenefit(2));
-
-  // Need this until vector.interleave is handled.
-  vector::populateVectorInterleaveToShufflePatterns(patterns);
 }
 
 void mlir::populateVectorReductionToSPIRVDotProductPatterns(
diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
index 61fd6bd972e3..885644864c0f 100644
--- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
+++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
@@ -164,11 +164,6 @@ void transform::ApplyLowerInterleavePatternsOp::populatePatterns(
   vector::populateVectorInterleaveLoweringPatterns(patterns);
 }
 
-void transform::ApplyInterleaveToShufflePatternsOp::populatePatterns(
-    RewritePatternSet &patterns) {
-  vector::populateVectorInterleaveToShufflePatterns(patterns);
-}
-
 void transform::ApplyRewriteNarrowTypePatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
   populateVectorNarrowTypeRewritePatterns(patterns);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
index 5326760c9b4e..3a456076f8fb 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
@@ -16,7 +16,6 @@
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Support/LogicalResult.h"
 
 #define DEBUG_TYPE "vector-interleave-lowering"
 
@@ -78,49 +77,9 @@ private:
   int64_t targetRank = 1;
 };
 
-/// Rewrite vector.interleave op into an equivalent vector.shuffle op, when
-/// applicable: `sourceType` must be 1D and non-scalable.
-///
-/// Example:
-///
-/// ```mlir
-/// vector.interleave %a, %b : vector<7xi16>
-/// ```
-///
-/// Is rewritten into:
-///
-/// ```mlir
-/// vector.shuffle %arg0, %arg1 [0, 7, 1, 8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13]
-///   : vector<7xi16>, vector<7xi16>
-/// ```
-class InterleaveToShuffle : public OpRewritePattern<vector::InterleaveOp> {
-public:
-  InterleaveToShuffle(MLIRContext *context, PatternBenefit benefit = 1)
-      : OpRewritePattern(context, benefit) {};
-
-  LogicalResult matchAndRewrite(vector::InterleaveOp op,
-                                PatternRewriter &rewriter) const override {
-    VectorType sourceType = op.getSourceVectorType();
-    if (sourceType.getRank() != 1 || sourceType.isScalable()) {
-      return failure();
-    }
-    int64_t n = sourceType.getNumElements();
-    auto seq = llvm::seq<int64_t>(2 * n);
-    auto zip = llvm::to_vector(llvm::map_range(
-        seq, [n](int64_t i) { return (i % 2 ? n : 0) + i / 2; }));
-    rewriter.replaceOpWithNewOp<ShuffleOp>(op, op.getLhs(), op.getRhs(), zip);
-    return success();
-  }
-};
-
 } // namespace
 
 void mlir::vector::populateVectorInterleaveLoweringPatterns(
     RewritePatternSet &patterns, int64_t targetRank, PatternBenefit benefit) {
   patterns.add<UnrollInterleaveOp>(targetRank, patterns.getContext(), benefit);
 }
-
-void mlir::vector::populateVectorInterleaveToShufflePatterns(
-    RewritePatternSet &patterns, PatternBenefit benefit) {
-  patterns.add<InterleaveToShuffle>(patterns.getContext(), benefit);
-}
diff --git a/mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir b/mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir
deleted file mode 100644
index ed3b3396bf3e..000000000000
--- a/mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir
+++ /dev/null
@@ -1,21 +0,0 @@
-// RUN: mlir-opt %s --transform-interpreter | FileCheck %s
-
-// CHECK-LABEL: @vector_interleave_to_shuffle
-func.func @vector_interleave_to_shuffle(%a: vector<7xi16>, %b: vector<7xi16>) -> vector<14xi16>
-{
-  %0 = vector.interleave %a, %b : vector<7xi16>
-  return %0 : vector<14xi16>
-}
-// CHECK: vector.shuffle %arg0, %arg1 [0, 7, 1, 8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13] : vector<7xi16>, vector<7xi16>
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
-    %f = transform.structured.match ops{["func.func"]} in %module_op
-      : (!transform.any_op) -> !transform.any_op
-
-    transform.apply_patterns to %f {
-      transform.apply_patterns.vector.interleave_to_shuffle
-    } : !transform.any_op
-    transform.yield
-  }
-}
-- 
GitLab


From d03a1a6e5838c7c2c0836d71507dfdf7840ade49 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Tue, 14 May 2024 01:58:45 +0800
Subject: [PATCH 0613/1206] [ValueTracking] Compute knownbits from known fp
 classes (#86409)

This patch calculates knownbits from fp instructions/dominating fcmp
conditions. It will enable more optimizations with signbit idioms.
---
 llvm/include/llvm/IR/PatternMatch.h           |   2 +-
 llvm/lib/Analysis/ValueTracking.cpp           |  36 +++
 .../AMDGPU/amdgpu-simplify-libcall-pow.ll     |  14 +-
 .../AMDGPU/amdgpu-simplify-libcall-pown.ll    |  12 +-
 llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll |  10 +-
 .../test/Transforms/InstCombine/known-bits.ll | 264 ++++++++++++++++++
 6 files changed, 319 insertions(+), 19 deletions(-)

diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 171ddab977de..0d6d86cb47e6 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -1904,7 +1904,7 @@ template <typename Op_t> struct ElementWiseBitCast_match {
   ElementWiseBitCast_match(const Op_t &OpMatch) : Op(OpMatch) {}
 
   template <typename OpTy> bool match(OpTy *V) {
-    BitCastInst *I = dyn_cast<BitCastInst>(V);
+    auto *I = dyn_cast<BitCastInst>(V);
     if (!I)
       return false;
     Type *SrcType = I->getSrcTy();
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 375385aca7a3..2fdbb6e3ef84 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1118,6 +1118,42 @@ static void computeKnownBitsFromOperator(const Operator *I,
       break;
     }
 
+    const Value *V;
+    // Handle bitcast from floating point to integer.
+    if (match(I, m_ElementWiseBitCast(m_Value(V))) &&
+        V->getType()->isFPOrFPVectorTy()) {
+      Type *FPType = V->getType()->getScalarType();
+      KnownFPClass Result = computeKnownFPClass(V, fcAllFlags, Depth + 1, Q);
+      FPClassTest FPClasses = Result.KnownFPClasses;
+
+      if (Result.isKnownNever(fcNormal | fcSubnormal | fcNan)) {
+        Known.Zero.setAllBits();
+        Known.One.setAllBits();
+
+        if (FPClasses & fcInf)
+          Known = Known.intersectWith(KnownBits::makeConstant(
+              APFloat::getInf(FPType->getFltSemantics()).bitcastToAPInt()));
+
+        if (FPClasses & fcZero)
+          Known = Known.intersectWith(KnownBits::makeConstant(
+              APInt::getZero(FPType->getScalarSizeInBits())));
+      }
+
+      if (Result.SignBit) {
+        if (*Result.SignBit)
+          Known.makeNegative();
+        else
+          Known.makeNonNegative();
+      } else {
+        Known.Zero.clearSignBit();
+        Known.One.clearSignBit();
+      }
+
+      assert(!Known.hasConflict() && "Bits known to be one AND zero?");
+
+      break;
+    }
+
     // Handle cast from vector integer type to scalar or vector integer.
     auto *SrcVecTy = dyn_cast<FixedVectorType>(SrcTy);
     if (!SrcVecTy || !SrcVecTy->getElementType()->isIntegerTy() ||
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
index c4bd4bc126f7..5db25a59d33f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
@@ -2216,7 +2216,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_sitofp(float %x, i32 %y)
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2304,7 +2304,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_uitofp(float %x, i32 %y)
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2353,7 +2353,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_uitofp_i256(float %x, i2
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2376,7 +2376,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_sitofp_i256(float %x, i2
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2399,7 +2399,7 @@ define <2 x float> @test_pow_afn_nnan_ninf_v2f32_known_integral_sitofp(<2 x floa
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x float> [[X]] to <2 x i32>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i32> [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x float> [[__EXP2]] to <2 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i32> [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint <2 x i32> [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP4]] to <2 x float>
 ; CHECK-NEXT:    ret <2 x float> [[TMP5]]
 ;
@@ -2448,7 +2448,7 @@ define <2 x float> @test_pow_afn_nnan_ninf_v2f32_known_integral_uitofp(<2 x floa
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x float> [[X]] to <2 x i32>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i32> [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x float> [[__EXP2]] to <2 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i32> [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint <2 x i32> [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP4]] to <2 x float>
 ; CHECK-NEXT:    ret <2 x float> [[TMP5]]
 ;
@@ -2560,7 +2560,7 @@ define float @test_pow_afn_f32_nnan_ninf__y_known_integral_trunc(float %x, float
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll
index 8ddaf243db92..e298226ee7cc 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll
@@ -680,7 +680,7 @@ define float @test_pown_afn_nnan_ninf_f32(float %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
@@ -703,7 +703,7 @@ define <2 x float> @test_pown_afn_nnan_ninf_v2f32(<2 x float> %x, <2 x i32> %y)
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[X]] to <2 x i32>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i32> [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x float> [[__EXP2]] to <2 x i32>
-; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i32> [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint <2 x i32> [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
 ; CHECK-NEXT:    ret <2 x float> [[TMP3]]
 ;
@@ -772,7 +772,7 @@ define half @test_pown_afn_nnan_ninf_f16(half %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast half [[X]] to i16
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i16 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[__EXP2]] to i16
-; CHECK-NEXT:    [[TMP2:%.*]] = or i16 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i16 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP2]] to half
 ; CHECK-NEXT:    ret half [[TMP3]]
 ;
@@ -795,7 +795,7 @@ define <2 x half> @test_pown_afn_nnan_ninf_v2f16(<2 x half> %x, <2 x i32> %y) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x half> [[X]] to <2 x i16>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i16> [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x half> [[__EXP2]] to <2 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i16> [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint <2 x i16> [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i16> [[TMP2]] to <2 x half>
 ; CHECK-NEXT:    ret <2 x half> [[TMP3]]
 ;
@@ -829,7 +829,7 @@ define float @test_pown_fast_f32_strictfp(float %x, i32 %y) #1 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
@@ -1075,7 +1075,7 @@ define float @test_pown_afn_ninf_nnan_f32__x_known_positive(float nofpclass(ninf
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
index 204c8140d3f1..54ca33401ccf 100644
--- a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
+++ b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
@@ -360,7 +360,7 @@ declare half @_Z4pownDhi(half, i32)
 ; GCN-NATIVE: %0 = bitcast half %x to i16
 ; GCN-NATIVE: %__pow_sign = and i16 %__yeven, %0
 ; GCN-NATIVE: %1 = bitcast half %__exp2 to i16
-; GCN-NATIVE: %2 = or i16 %__pow_sign, %1
+; GCN-NATIVE: %2 = or disjoint i16 %__pow_sign, %1
 ; GCN-NATIVE: %3 = bitcast i16 %2 to half
 define half @test_pown_f16(half %x, i32 %y) {
 entry:
@@ -378,7 +378,7 @@ declare float @_Z4pownfi(float, i32)
 ; GCN: %[[r0:.*]] = bitcast float %tmp to i32
 ; GCN: %__pow_sign = and i32 %[[r0]], -2147483648
 ; GCN: %[[r1:.*]] = bitcast float %__exp2 to i32
-; GCN: %[[r2:.*]] = or i32 %__pow_sign, %[[r1]]
+; GCN: %[[r2:.*]] = or disjoint i32 %__pow_sign, %[[r1]]
 ; GCN: store i32 %[[r2]], ptr addrspace(1) %a, align 4
 define amdgpu_kernel void @test_pow(ptr addrspace(1) nocapture %a) {
 entry:
@@ -414,7 +414,7 @@ entry:
 ; GCN: %[[r0:.*]] = bitcast float %tmp to i32
 ; GCN: %__pow_sign = and i32 %__yeven, %[[r0]]
 ; GCN: %[[r1:.*]] = bitcast float %__exp2 to i32
-; GCN: %[[r2:.*]] = or i32 %__pow_sign, %[[r1]]
+; GCN: %[[r2:.*]] = or disjoint i32 %__pow_sign, %[[r1]]
 ; GCN: store i32 %[[r2]], ptr addrspace(1) %a, align 4
 define amdgpu_kernel void @test_pown(ptr addrspace(1) nocapture %a) {
 entry:
@@ -438,7 +438,7 @@ declare <2 x half> @_Z3powDv2_DhS_(<2 x half>, <2 x half>)
 ; GCN: %1 = bitcast half %x to i16
 ; GCN: %__pow_sign = and i16 %1, -32768
 ; GCN: %2 = bitcast half %__exp2 to i16
-; GCN: %3 = or i16 %__pow_sign, %2
+; GCN: %3 = or disjoint i16 %__pow_sign, %2
 ; GCN: %4 = bitcast i16 %3 to half
 define half @test_pow_fast_f16__y_13(half %x) {
   %powr = tail call fast half @_Z3powDhDh(half %x, half 13.0)
@@ -453,7 +453,7 @@ define half @test_pow_fast_f16__y_13(half %x) {
 ; GCN: %1 = bitcast <2 x half> %x to <2 x i16>
 ; GCN: %__pow_sign = and <2 x i16> %1, <i16 -32768, i16 -32768>
 ; GCN: %2 = bitcast <2 x half> %__exp2 to <2 x i16>
-; GCN: %3 = or <2 x i16> %__pow_sign, %2
+; GCN: %3 = or disjoint <2 x i16> %__pow_sign, %2
 ; GCN: %4 = bitcast <2 x i16> %3 to <2 x half>
 define <2 x half> @test_pow_fast_v2f16__y_13(<2 x half> %x) {
   %powr = tail call fast <2 x half> @_Z3powDv2_DhS_(<2 x half> %x, <2 x half> <half 13.0, half 13.0>)
diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index 8b4249b2c25a..816bd6f352df 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1374,5 +1374,269 @@ define i8 @nonzero_reduce_xor_vscale_odd(<vscale x 3 x i8> %xx) {
   ret i8 %r
 }
 
+define i1 @test_sign_pos(float %x) {
+; CHECK-LABEL: @test_sign_pos(
+; CHECK-NEXT:    ret i1 true
+;
+  %fabs = call float @llvm.fabs.f32(float %x)
+  %y = bitcast float %fabs to i32
+  %sign = icmp sgt i32 %y, -1
+  ret i1 %sign
+}
+
+define i1 @test_sign_pos_half(half %x) {
+; CHECK-LABEL: @test_sign_pos_half(
+; CHECK-NEXT:    ret i1 true
+;
+  %fabs = call half @llvm.fabs.f16(half %x)
+  %y = bitcast half %fabs to i16
+  %sign = icmp sgt i16 %y, -1
+  ret i1 %sign
+}
+
+define i1 @test_sign_pos_half_non_elementwise(<2 x half> %x) {
+; CHECK-LABEL: @test_sign_pos_half_non_elementwise(
+; CHECK-NEXT:    [[FABS:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[X:%.*]])
+; CHECK-NEXT:    [[Y:%.*]] = bitcast <2 x half> [[FABS]] to i32
+; CHECK-NEXT:    [[SIGN:%.*]] = icmp sgt i32 [[Y]], -1
+; CHECK-NEXT:    ret i1 [[SIGN]]
+;
+  %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %x)
+  %y = bitcast <2 x half> %fabs to i32
+  %sign = icmp sgt i32 %y, -1
+  ret i1 %sign
+}
+
+define i1 @test_sign_neg(float %x) {
+; CHECK-LABEL: @test_sign_neg(
+; CHECK-NEXT:    ret i1 true
+;
+  %fabs = call float @llvm.fabs.f32(float %x)
+  %fnabs = fneg float %fabs
+  %y = bitcast float %fnabs to i32
+  %sign = icmp slt i32 %y, 0
+  ret i1 %sign
+}
+
+define <2 x i1> @test_sign_pos_vec(<2 x float> %x) {
+; CHECK-LABEL: @test_sign_pos_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %x)
+  %y = bitcast <2 x float> %fabs to <2 x i32>
+  %sign = icmp slt <2 x i32> %y, zeroinitializer
+  ret <2 x i1> %sign
+}
+
+define i32 @test_inf_only(float nofpclass(nan sub norm zero) %x) {
+; CHECK-LABEL: @test_inf_only(
+; CHECK-NEXT:    ret i32 2139095040
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 2147483647
+  ret i32 %and
+}
+
+define i16 @test_inf_only_bfloat(bfloat nofpclass(nan sub norm zero) %x) {
+; CHECK-LABEL: @test_inf_only_bfloat(
+; CHECK-NEXT:    ret i16 32640
+;
+  %y = bitcast bfloat %x to i16
+  %and = and i16 %y, 32767
+  ret i16 %and
+}
+
+define i128 @test_inf_only_ppc_fp128(ppc_fp128 nofpclass(nan sub norm zero) %x) {
+; CHECK-LABEL: @test_inf_only_ppc_fp128(
+; CHECK-NEXT:    ret i128 9218868437227405312
+;
+  %y = bitcast ppc_fp128 %x to i128
+  %and = and i128 %y, 170141183460469231731687303715884105727
+  ret i128 %and
+}
+
+define i32 @test_zero_only(float nofpclass(nan sub norm inf) %x) {
+; CHECK-LABEL: @test_zero_only(
+; CHECK-NEXT:    ret i32 0
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 2147483647
+  ret i32 %and
+}
+
+define i80 @test_zero_only_non_ieee(x86_fp80 nofpclass(nan sub norm inf) %x) {
+; CHECK-LABEL: @test_zero_only_non_ieee(
+; CHECK-NEXT:    ret i80 0
+;
+  %y = bitcast x86_fp80 %x to i80
+  %and = and i80 %y, 604462909807314587353087
+  ret i80 %and
+}
+
+define i32 @test_inf_nan_only(float nofpclass(sub norm zero) %x) {
+; CHECK-LABEL: @test_inf_nan_only(
+; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 2130706432
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 2130706432
+  ret i32 %and
+}
+
+define i32 @test_sub_zero_only(float nofpclass(nan norm inf) %x) {
+; CHECK-LABEL: @test_sub_zero_only(
+; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 2130706432
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 2130706432
+  ret i32 %and
+}
+
+define i32 @test_inf_zero_only(float nofpclass(nan norm sub) %x) {
+; CHECK-LABEL: @test_inf_zero_only(
+; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 8388608
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 16777215
+  ret i32 %and
+}
+
+
+
+define i1 @test_simplify_icmp(i32 %x) {
+; CHECK-LABEL: @test_simplify_icmp(
+; CHECK-NEXT:    ret i1 false
+;
+  %cast1 = uitofp i32 %x to double
+  %cast2 = bitcast double %cast1 to i64
+  %mask = and i64 %cast2, -140737488355328
+  %cmp = icmp eq i64 %mask, -1970324836974592
+  ret i1 %cmp
+}
+
+define i32 @test_snan_quiet_bit1(float nofpclass(sub norm inf qnan) %x) {
+; CHECK-LABEL: @test_snan_quiet_bit1(
+; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 4194304
+; CHECK-NEXT:    ret i32 [[MASKED]]
+;
+  %bits = bitcast float %x to i32
+  %masked = and i32 %bits, 4194304
+  ret i32 %masked
+}
+
+define i32 @test_snan_quiet_bit2(float nofpclass(sub norm inf qnan) %x) {
+; CHECK-LABEL: @test_snan_quiet_bit2(
+; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 2097152
+; CHECK-NEXT:    ret i32 [[MASKED]]
+;
+  %bits = bitcast float %x to i32
+  %masked = and i32 %bits, 2097152
+  ret i32 %masked
+}
+
+define i32 @test_qnan_quiet_bit1(float nofpclass(sub norm inf snan) %x) {
+; CHECK-LABEL: @test_qnan_quiet_bit1(
+; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 4194304
+; CHECK-NEXT:    ret i32 [[MASKED]]
+;
+  %bits = bitcast float %x to i32
+  %masked = and i32 %bits, 4194304
+  ret i32 %masked
+}
+
+define i32 @test_qnan_quiet_bit2(float nofpclass(sub norm inf snan) %x) {
+; CHECK-LABEL: @test_qnan_quiet_bit2(
+; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 2097152
+; CHECK-NEXT:    ret i32 [[MASKED]]
+;
+  %bits = bitcast float %x to i32
+  %masked = and i32 %bits, 2097152
+  ret i32 %masked
+}
+
+define i16 @test_simplify_mask(i32 %ui, float %x) {
+; CHECK-LABEL: @test_simplify_mask(
+; CHECK-NEXT:    [[CONV:%.*]] = uitofp i32 [[UI:%.*]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[CONV]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_ELSE:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i16 31744
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i16 0
+;
+  %conv = uitofp i32 %ui to float
+  %cmp = fcmp olt float %x, %conv
+  br i1 %cmp, label %if.else, label %if.end
+
+if.end:
+  %cast = bitcast float %conv to i32
+  %shr = lshr i32 %cast, 16
+  %trunc = trunc i32 %shr to i16
+  %and = and i16 %trunc, -32768
+  %or = or disjoint i16 %and, 31744
+  ret i16 %or
+
+if.else:
+  ret i16 0
+}
+
+; TODO: %cmp always evaluates to false
+
+define i1 @test_simplify_icmp2(double %x) {
+; CHECK-LABEL: @test_simplify_icmp2(
+; CHECK-NEXT:    [[ABS:%.*]] = tail call double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[COND:%.*]] = fcmp oeq double [[ABS]], 0x7FF0000000000000
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast double [[X]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[CAST]], 3458764513820540928
+; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i1 false
+;
+  %abs = tail call double @llvm.fabs.f64(double %x)
+  %cond = fcmp oeq double %abs, 0x7FF0000000000000
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:
+  %cast = bitcast double %x to i64
+  %cmp = icmp eq i64 %cast, 3458764513820540928
+  ret i1 %cmp
+
+if.else:
+  ret i1 false
+}
+
+define i32 @test_snan_only(float nofpclass(qnan sub norm zero inf) %x) {
+; CHECK-LABEL: @test_snan_only(
+; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 4194304
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 4194304
+  ret i32 %and
+}
+
+define i32 @test_qnan_only(float nofpclass(snan sub norm zero inf) %x) {
+; CHECK-LABEL: @test_qnan_only(
+; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 4194304
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 4194304
+  ret i32 %and
+}
+
 declare void @use(i1)
 declare void @sink(i8)
-- 
GitLab


From 08177541267fff84a96701fc5fc232eb4f12f9d9 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 13 May 2024 10:37:12 -0700
Subject: [PATCH 0614/1206] [RISCV] Improve testing of loads with offset in
 local-stack-slot-allocation.ll. NFC

The test we had didn't match it's description.

Now we have one test with a large offset that requires a virtual base register
and a test with a smaller offset that should not.

There is currently a bug that causes the offset to double counted
leading to the small case also using a virtual base register.
---
 .../RISCV/local-stack-slot-allocation.ll      | 87 ++++++++++++-------
 1 file changed, 54 insertions(+), 33 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll b/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
index 40d08513e3cf..c34e2bfdca08 100644
--- a/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
+++ b/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
@@ -47,48 +47,69 @@ define void @use_frame_base_reg() {
 }
 
 ; Test containing a load with its own local offset. Make sure isFrameOffsetLegal
-; considers it and does not create a virtual base register.
+; considers it and creates a virtual base register.
 define void @load_with_offset() {
 ; RV32I-LABEL: load_with_offset:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a0, 25
-; RV32I-NEXT:    addi a0, a0, -1792
-; RV32I-NEXT:    sub sp, sp, a0
-; RV32I-NEXT:    .cfi_def_cfa_offset 100608
-; RV32I-NEXT:    lui a0, 25
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    lbu zero, -292(a0)
-; RV32I-NEXT:    lui a0, 24
-; RV32I-NEXT:    add a0, sp, a0
-; RV32I-NEXT:    lbu zero, 1704(a0)
-; RV32I-NEXT:    lui a0, 25
-; RV32I-NEXT:    addi a0, a0, -1792
-; RV32I-NEXT:    add sp, sp, a0
+; RV32I-NEXT:    addi sp, sp, -2048
+; RV32I-NEXT:    addi sp, sp, -464
+; RV32I-NEXT:    .cfi_def_cfa_offset 2512
+; RV32I-NEXT:    addi a0, sp, 2012
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    sb a1, 0(a0)
+; RV32I-NEXT:    addi sp, sp, 2032
+; RV32I-NEXT:    addi sp, sp, 480
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: load_with_offset:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lui a0, 25
-; RV64I-NEXT:    addiw a0, a0, -1792
-; RV64I-NEXT:    sub sp, sp, a0
-; RV64I-NEXT:    .cfi_def_cfa_offset 100608
-; RV64I-NEXT:    lui a0, 25
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    lbu zero, -292(a0)
-; RV64I-NEXT:    lui a0, 24
-; RV64I-NEXT:    add a0, sp, a0
-; RV64I-NEXT:    lbu zero, 1704(a0)
-; RV64I-NEXT:    lui a0, 25
-; RV64I-NEXT:    addiw a0, a0, -1792
-; RV64I-NEXT:    add sp, sp, a0
+; RV64I-NEXT:    addi sp, sp, -2048
+; RV64I-NEXT:    addi sp, sp, -464
+; RV64I-NEXT:    .cfi_def_cfa_offset 2512
+; RV64I-NEXT:    addi a0, sp, 2012
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    sb a1, 0(a0)
+; RV64I-NEXT:    addi sp, sp, 2032
+; RV64I-NEXT:    addi sp, sp, 480
+; RV64I-NEXT:    ret
+
+  %va = alloca [2500 x i8], align 4
+  %va_gep = getelementptr [2000 x i8], ptr %va, i64 0, i64 2000
+  %load = load volatile i8, ptr %va_gep, align 4
+  store volatile i8 %load, ptr %va_gep, align 4
+  ret void
+}
+
+; Test containing a load with its own local offset that is smaller than the
+; previous test case. Make sure we don't create a virtual base register.
+define void @load_with_offset2() {
+; RV32I-LABEL: load_with_offset2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -2048
+; RV32I-NEXT:    addi sp, sp, -464
+; RV32I-NEXT:    .cfi_def_cfa_offset 2512
+; RV32I-NEXT:    addi a0, sp, 1412
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    sb a1, 0(a0)
+; RV32I-NEXT:    addi sp, sp, 2032
+; RV32I-NEXT:    addi sp, sp, 480
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_with_offset2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -2048
+; RV64I-NEXT:    addi sp, sp, -464
+; RV64I-NEXT:    .cfi_def_cfa_offset 2512
+; RV64I-NEXT:    addi a0, sp, 1412
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    sb a1, 0(a0)
+; RV64I-NEXT:    addi sp, sp, 2032
+; RV64I-NEXT:    addi sp, sp, 480
 ; RV64I-NEXT:    ret
 
-  %va = alloca [100 x i8], align 4
-  %va1 = alloca [500 x i8], align 4
-  %large = alloca [100000 x i8]
-  %va_gep = getelementptr [100 x i8], ptr %va, i64 16
-  %va1_gep = getelementptr [100 x i8], ptr %va1, i64 0
+  %va = alloca [2500 x i8], align 4
+  %va_gep = getelementptr [2000 x i8], ptr %va, i64 0, i64 1400
   %load = load volatile i8, ptr %va_gep, align 4
-  %load1 = load volatile i8, ptr %va1_gep, align 4
+  store volatile i8 %load, ptr %va_gep, align 4
   ret void
 }
-- 
GitLab


From 026686bac606ac03d688a44f2ea4cb829d7b08bc Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 13 May 2024 10:51:17 -0700
Subject: [PATCH 0615/1206] [RISCV] Don't add getFrameIndexInstrOffset in
 RISCVRegisterInfo::needsFrameBaseReg.

It's already added in isFrameOffsetLegal so adding it in needsFrameBaseReg
causes it to be double counted.
---
 llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp            |  1 -
 llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll | 10 ++++------
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index 6a48848e2022..c3281e409653 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -607,7 +607,6 @@ bool RISCVRegisterInfo::needsFrameBaseReg(MachineInstr *MI,
   const MachineFrameInfo &MFI = MF.getFrameInfo();
   const RISCVFrameLowering *TFI = getFrameLowering(MF);
   const MachineRegisterInfo &MRI = MF.getRegInfo();
-  Offset += getFrameIndexInstrOffset(MI, FIOperandNum);
 
   if (TFI->hasFP(MF) && !shouldRealignStack(MF)) {
     // Estimate the stack size used to store callee saved registers(
diff --git a/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll b/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
index c34e2bfdca08..18e7992f30a3 100644
--- a/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
+++ b/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
@@ -88,9 +88,8 @@ define void @load_with_offset2() {
 ; RV32I-NEXT:    addi sp, sp, -2048
 ; RV32I-NEXT:    addi sp, sp, -464
 ; RV32I-NEXT:    .cfi_def_cfa_offset 2512
-; RV32I-NEXT:    addi a0, sp, 1412
-; RV32I-NEXT:    lbu a1, 0(a0)
-; RV32I-NEXT:    sb a1, 0(a0)
+; RV32I-NEXT:    lbu a0, 1412(sp)
+; RV32I-NEXT:    sb a0, 1412(sp)
 ; RV32I-NEXT:    addi sp, sp, 2032
 ; RV32I-NEXT:    addi sp, sp, 480
 ; RV32I-NEXT:    ret
@@ -100,9 +99,8 @@ define void @load_with_offset2() {
 ; RV64I-NEXT:    addi sp, sp, -2048
 ; RV64I-NEXT:    addi sp, sp, -464
 ; RV64I-NEXT:    .cfi_def_cfa_offset 2512
-; RV64I-NEXT:    addi a0, sp, 1412
-; RV64I-NEXT:    lbu a1, 0(a0)
-; RV64I-NEXT:    sb a1, 0(a0)
+; RV64I-NEXT:    lbu a0, 1412(sp)
+; RV64I-NEXT:    sb a0, 1412(sp)
 ; RV64I-NEXT:    addi sp, sp, 2032
 ; RV64I-NEXT:    addi sp, sp, 480
 ; RV64I-NEXT:    ret
-- 
GitLab


From 66466ff151bd71fa64c63d951173efc589df2860 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <dthorn@google.com>
Date: Mon, 13 May 2024 12:30:50 -0500
Subject: [PATCH 0616/1206] Reland: [LLD] Implement
 --enable-non-contiguous-regions (#90007)

When enabled, input sections that would otherwise overflow a memory
region are instead spilled to the next matching output section.

This feature parallels the one in GNU LD, but there are some differences
from its documented behavior:

- /DISCARD/ only matches previously-unmatched sections (i.e., the flag
does not affect it).

- If a section fails to fit at any of its matches, the link fails
instead of discarding the section.

- The flag --enable-non-contiguous-regions-warnings is not implemented,
as it exists to warn about such occurrences.

The implementation places stubs at possible spill locations, and
replaces them with the original input section when effecting spills.
Spilling decisions occur after address assignment. Sections are spilled
in reverse order of assignment, with each spill naively decreasing the
size of the affected memory regions. This continues until the memory
regions are brought back under size. Spilling anything causes another
pass of address assignment, and this continues to fixed point.

Spilling after rather than during assignment allows the algorithm to
consider the size effects of unspillable input sections that appear
later in the assignment. Otherwise, such sections (e.g. thunks) may
force an overflow, even if spilling something earlier could have avoided
it.

A few notable feature interactions occur:

- Stubs affect alignment, ONLY_IF_RO, etc, broadly as if a copy of the
input section were actually placed there.

- SHF_MERGE synthetic sections use the spill list of their first
contained input section (the one that gives the section its name).

- ICF occurs oblivious to spill sections; spill lists for merged-away
sections become inert and are removed after assignment.

- SHF_LINK_ORDER and .ARM.exidx are ordered according to the final
section ordering, after all spilling has completed.

- INSERT BEFORE/AFTER and OVERWRITE_SECTIONS are explicitly disallowed.
---
 lld/ELF/Config.h                              |   1 +
 lld/ELF/Driver.cpp                            |   4 +-
 lld/ELF/InputSection.cpp                      |   7 +
 lld/ELF/InputSection.h                        |  25 +-
 lld/ELF/LinkerScript.cpp                      | 181 +++++++++++-
 lld/ELF/LinkerScript.h                        |  15 +-
 lld/ELF/Options.td                            |   3 +
 lld/ELF/OutputSections.cpp                    |   7 +-
 lld/ELF/OutputSections.h                      |   2 +-
 lld/ELF/SyntheticSections.cpp                 |   7 +
 lld/ELF/SyntheticSections.h                   |   4 +
 lld/ELF/Writer.cpp                            |  22 +-
 lld/docs/ELF/linker_script.rst                |  11 +
 lld/docs/ReleaseNotes.rst                     |   6 +
 lld/docs/ld.lld.1                             |   2 +
 ...able-non-contiguous-regions-arm-exidx.test |  55 ++++
 .../enable-non-contiguous-regions.test        | 265 ++++++++++++++++++
 17 files changed, 600 insertions(+), 17 deletions(-)
 create mode 100644 lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test
 create mode 100644 lld/test/ELF/linkerscript/enable-non-contiguous-regions.test

diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index c55b547a733c..dbb81412453a 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -238,6 +238,7 @@ struct Config {
   bool emitLLVM;
   bool emitRelocs;
   bool enableNewDtags;
+  bool enableNonContiguousRegions;
   bool executeOnly;
   bool exportDynamic;
   bool fixCortexA53Errata843419;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index dd33f4bd772f..028cdcc83d2f 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1250,6 +1250,8 @@ static void readConfigs(opt::InputArgList &args) {
   config->emitRelocs = args.hasArg(OPT_emit_relocs);
   config->enableNewDtags =
       args.hasFlag(OPT_enable_new_dtags, OPT_disable_new_dtags, true);
+  config->enableNonContiguousRegions =
+      args.hasArg(OPT_enable_non_contiguous_regions);
   config->entry = args.getLastArgValue(OPT_entry);
 
   errorHandler().errorHandlingScript =
@@ -3085,7 +3087,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
     // sectionBases.
     for (SectionCommand *cmd : script->sectionCommands)
       if (auto *osd = dyn_cast<OutputDesc>(cmd))
-        osd->osec.finalizeInputSections();
+        osd->osec.finalizeInputSections(&script.s);
   }
 
   // Two input sections with different output sections should not be folded.
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index fa81611e7c9e..2a1ccd997f8b 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -161,6 +161,7 @@ uint64_t SectionBase::getOffset(uint64_t offset) const {
   }
   case Regular:
   case Synthetic:
+  case Spill:
     return cast<InputSection>(this)->outSecOff + offset;
   case EHFrame: {
     // Two code paths may reach here. First, clang_rt.crtbegin.o and GCC
@@ -309,6 +310,12 @@ std::string InputSectionBase::getObjMsg(uint64_t off) const {
       .str();
 }
 
+PotentialSpillSection::PotentialSpillSection(const InputSectionBase &source,
+                                             InputSectionDescription &isd)
+    : InputSection(source.file, source.flags, source.type, source.addralign, {},
+                   source.name, SectionBase::Spill),
+      isd(&isd) {}
+
 InputSection InputSection::discarded(nullptr, 0, 0, 0, ArrayRef<uint8_t>(), "");
 
 InputSection::InputSection(InputFile *f, uint64_t flags, uint32_t type,
diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h
index 1fb7077ca435..58e5306fd6dc 100644
--- a/lld/ELF/InputSection.h
+++ b/lld/ELF/InputSection.h
@@ -48,7 +48,7 @@ template <class ELFT> struct RelsOrRelas {
 // sections.
 class SectionBase {
 public:
-  enum Kind { Regular, Synthetic, EHFrame, Merge, Output };
+  enum Kind { Regular, Synthetic, Spill, EHFrame, Merge, Output };
 
   Kind kind() const { return (Kind)sectionKind; }
 
@@ -382,7 +382,8 @@ public:
 
   static bool classof(const SectionBase *s) {
     return s->kind() == SectionBase::Regular ||
-           s->kind() == SectionBase::Synthetic;
+           s->kind() == SectionBase::Synthetic ||
+           s->kind() == SectionBase::Spill;
   }
 
   // Write this section to a mmap'ed file, assuming Buf is pointing to
@@ -425,6 +426,26 @@ private:
   template <class ELFT> void copyShtGroup(uint8_t *buf);
 };
 
+// A marker for a potential spill location for another input section. This
+// broadly acts as if it were the original section until address assignment.
+// Then it is either replaced with the real input section or removed.
+class PotentialSpillSection : public InputSection {
+public:
+  // The containing input section description; used to quickly replace this stub
+  // with the actual section.
+  InputSectionDescription *isd;
+
+  // Next potential spill location for the same source input section.
+  PotentialSpillSection *next = nullptr;
+
+  PotentialSpillSection(const InputSectionBase &source,
+                        InputSectionDescription &isd);
+
+  static bool classof(const SectionBase *sec) {
+    return sec->kind() == InputSectionBase::Spill;
+  }
+};
+
 static_assert(sizeof(InputSection) <= 160, "InputSection is too big");
 
 class SyntheticSection : public InputSection {
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index c0a5014817b9..3ba59c112b8a 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -304,6 +304,9 @@ getChangedSymbolAssignment(const SymbolAssignmentMap &oldValues) {
 void LinkerScript::processInsertCommands() {
   SmallVector<OutputDesc *, 0> moves;
   for (const InsertCommand &cmd : insertCommands) {
+    if (config->enableNonContiguousRegions)
+      error("INSERT cannot be used with --enable-non-contiguous-regions");
+
     for (StringRef name : cmd.names) {
       // If base is empty, it may have been discarded by
       // adjustOutputSections(). We do not handle such output sections.
@@ -486,10 +489,12 @@ static void sortInputSections(MutableArrayRef<InputSectionBase *> vec,
 // Compute and remember which sections the InputSectionDescription matches.
 SmallVector<InputSectionBase *, 0>
 LinkerScript::computeInputSections(const InputSectionDescription *cmd,
-                                   ArrayRef<InputSectionBase *> sections) {
+                                   ArrayRef<InputSectionBase *> sections,
+                                   const OutputSection &outCmd) {
   SmallVector<InputSectionBase *, 0> ret;
   SmallVector<size_t, 0> indexes;
   DenseSet<size_t> seen;
+  DenseSet<InputSectionBase *> spills;
   auto sortByPositionThenCommandLine = [&](size_t begin, size_t end) {
     llvm::sort(MutableArrayRef<size_t>(indexes).slice(begin, end - begin));
     for (size_t i = begin; i != end; ++i)
@@ -505,10 +510,10 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
     size_t sizeBeforeCurrPat = ret.size();
 
     for (size_t i = 0, e = sections.size(); i != e; ++i) {
-      // Skip if the section is dead or has been matched by a previous input
-      // section description or a previous pattern.
+      // Skip if the section is dead or has been matched by a previous pattern
+      // in this input section description.
       InputSectionBase *sec = sections[i];
-      if (!sec->isLive() || sec->parent || seen.contains(i))
+      if (!sec->isLive() || seen.contains(i))
         continue;
 
       // For --emit-relocs we have to ignore entries like
@@ -529,6 +534,29 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
           (sec->flags & cmd->withoutFlags) != 0)
         continue;
 
+      if (sec->parent) {
+        // Skip if not allowing multiple matches.
+        if (!config->enableNonContiguousRegions)
+          continue;
+
+        // Disallow spilling into /DISCARD/; special handling would be needed
+        // for this in address assignment, and the semantics are nebulous.
+        if (outCmd.name == "/DISCARD/")
+          continue;
+
+        // Skip if the section's first match was /DISCARD/; such sections are
+        // always discarded.
+        if (sec->parent->name == "/DISCARD/")
+          continue;
+
+        // Skip if the section was already matched by a different input section
+        // description within this output section.
+        if (sec->parent == &outCmd)
+          continue;
+
+        spills.insert(sec);
+      }
+
       ret.push_back(sec);
       indexes.push_back(i);
       seen.insert(i);
@@ -555,6 +583,30 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
   // Matched sections after the last SORT* are sorted by (--sort-alignment,
   // input order).
   sortByPositionThenCommandLine(sizeAfterPrevSort, ret.size());
+
+  // The flag --enable-non-contiguous-regions may cause sections to match an
+  // InputSectionDescription in more than one OutputSection. Matches after the
+  // first were collected in the spills set, so replace these with potential
+  // spill sections.
+  if (!spills.empty()) {
+    for (InputSectionBase *&sec : ret) {
+      if (!spills.contains(sec))
+        continue;
+
+      // Append the spill input section to the list for the input section,
+      // creating it if necessary.
+      PotentialSpillSection *pss = make<PotentialSpillSection>(
+          *sec, const_cast<InputSectionDescription &>(*cmd));
+      auto [it, inserted] =
+          potentialSpillLists.try_emplace(sec, PotentialSpillList{pss, pss});
+      if (!inserted) {
+        PotentialSpillSection *&tail = it->second.tail;
+        tail = tail->next = pss;
+      }
+      sec = pss;
+    }
+  }
+
   return ret;
 }
 
@@ -577,7 +629,7 @@ void LinkerScript::discardSynthetic(OutputSection &outCmd) {
         part.armExidx->exidxSections.end());
     for (SectionCommand *cmd : outCmd.commands)
       if (auto *isd = dyn_cast<InputSectionDescription>(cmd))
-        for (InputSectionBase *s : computeInputSections(isd, secs))
+        for (InputSectionBase *s : computeInputSections(isd, secs, outCmd))
           discard(*s);
   }
 }
@@ -588,7 +640,7 @@ LinkerScript::createInputSectionList(OutputSection &outCmd) {
 
   for (SectionCommand *cmd : outCmd.commands) {
     if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) {
-      isd->sectionBases = computeInputSections(isd, ctx.inputSections);
+      isd->sectionBases = computeInputSections(isd, ctx.inputSections, outCmd);
       for (InputSectionBase *s : isd->sectionBases)
         s->parent = &outCmd;
       ret.insert(ret.end(), isd->sectionBases.begin(), isd->sectionBases.end());
@@ -644,6 +696,9 @@ void LinkerScript::processSectionCommands() {
 
   // Process OVERWRITE_SECTIONS first so that it can overwrite the main script
   // or orphans.
+  if (config->enableNonContiguousRegions && !overwriteSections.empty())
+    error("OVERWRITE_SECTIONS cannot be used with "
+          "--enable-non-contiguous-regions");
   DenseMap<CachedHashStringRef, OutputDesc *> map;
   size_t i = 0;
   for (OutputDesc *osd : overwriteSections) {
@@ -1066,8 +1121,12 @@ void LinkerScript::assignOffsets(OutputSection *sec) {
     // Handle a single input section description command.
     // It calculates and assigns the offsets for each section and also
     // updates the output section size.
-    for (InputSection *isec : cast<InputSectionDescription>(cmd)->sections) {
+
+    auto &sections = cast<InputSectionDescription>(cmd)->sections;
+    for (InputSection *isec : sections) {
       assert(isec->getParent() == sec);
+      if (isa<PotentialSpillSection>(isec))
+        continue;
       const uint64_t pos = dot;
       dot = alignToPowerOf2(dot, isec->addralign);
       isec->outSecOff = dot - sec->addr;
@@ -1364,6 +1423,114 @@ const Defined *LinkerScript::assignAddresses() {
   return getChangedSymbolAssignment(oldValues);
 }
 
+static bool hasRegionOverflowed(MemoryRegion *mr) {
+  if (!mr)
+    return false;
+  return mr->curPos - mr->getOrigin() > mr->getLength();
+}
+
+// Spill input sections in reverse order of address assignment to (potentially)
+// bring memory regions out of overflow. The size savings of a spill can only be
+// estimated, since general linker script arithmetic may occur afterwards.
+// Under-estimates may cause unnecessary spills, but over-estimates can always
+// be corrected on the next pass.
+bool LinkerScript::spillSections() {
+  if (!config->enableNonContiguousRegions)
+    return false;
+
+  bool spilled = false;
+  for (SectionCommand *cmd : reverse(sectionCommands)) {
+    auto *od = dyn_cast<OutputDesc>(cmd);
+    if (!od)
+      continue;
+    OutputSection *osec = &od->osec;
+    if (!osec->memRegion)
+      continue;
+
+    // Input sections that have replaced a potential spill and should be removed
+    // from their input section description.
+    DenseSet<InputSection *> spilledInputSections;
+
+    for (SectionCommand *cmd : reverse(osec->commands)) {
+      if (!hasRegionOverflowed(osec->memRegion) &&
+          !hasRegionOverflowed(osec->lmaRegion))
+        break;
+
+      auto *isd = dyn_cast<InputSectionDescription>(cmd);
+      if (!isd)
+        continue;
+      for (InputSection *isec : reverse(isd->sections)) {
+        // Potential spill locations cannot be spilled.
+        if (isa<PotentialSpillSection>(isec))
+          continue;
+
+        // Find the next potential spill location and remove it from the list.
+        auto it = potentialSpillLists.find(isec);
+        if (it == potentialSpillLists.end())
+          continue;
+        PotentialSpillList &list = it->second;
+        PotentialSpillSection *spill = list.head;
+        if (spill->next)
+          list.head = spill->next;
+        else
+          potentialSpillLists.erase(isec);
+
+        // Replace the next spill location with the spilled section and adjust
+        // its properties to match the new location. Note that the alignment of
+        // the spill section may have diverged from the original due to e.g. a
+        // SUBALIGN. Correct assignment requires the spill's alignment to be
+        // used, not the original.
+        spilledInputSections.insert(isec);
+        *llvm::find(spill->isd->sections, spill) = isec;
+        isec->parent = spill->parent;
+        isec->addralign = spill->addralign;
+
+        // Record the (potential) reduction in the region's end position.
+        osec->memRegion->curPos -= isec->getSize();
+        if (osec->lmaRegion)
+          osec->lmaRegion->curPos -= isec->getSize();
+
+        // Spilling continues until the end position no longer overflows the
+        // region. Then, another round of address assignment will either confirm
+        // the spill's success or lead to yet more spilling.
+        if (!hasRegionOverflowed(osec->memRegion) &&
+            !hasRegionOverflowed(osec->lmaRegion))
+          break;
+      }
+
+      // Remove any spilled input sections to complete their move.
+      if (!spilledInputSections.empty()) {
+        spilled = true;
+        llvm::erase_if(isd->sections, [&](InputSection *isec) {
+          return spilledInputSections.contains(isec);
+        });
+      }
+    }
+  }
+
+  return spilled;
+}
+
+// Erase any potential spill sections that were not used.
+void LinkerScript::erasePotentialSpillSections() {
+  if (potentialSpillLists.empty())
+    return;
+
+  // Collect the set of input section descriptions that contain potential
+  // spills.
+  DenseSet<InputSectionDescription *> isds;
+  for (const auto &[_, list] : potentialSpillLists)
+    for (PotentialSpillSection *s = list.head; s; s = s->next)
+      isds.insert(s->isd);
+
+  for (InputSectionDescription *isd : isds)
+    llvm::erase_if(isd->sections, [](InputSection *s) {
+      return isa<PotentialSpillSection>(s);
+    });
+
+  potentialSpillLists.clear();
+}
+
 // Creates program headers as instructed by PHDRS linker script command.
 SmallVector<PhdrEntry *, 0> LinkerScript::createPhdrs() {
   SmallVector<PhdrEntry *, 0> ret;
diff --git a/lld/ELF/LinkerScript.h b/lld/ELF/LinkerScript.h
index b09cd12c46f9..734d4e7498aa 100644
--- a/lld/ELF/LinkerScript.h
+++ b/lld/ELF/LinkerScript.h
@@ -10,6 +10,7 @@
 #define LLD_ELF_LINKER_SCRIPT_H
 
 #include "Config.h"
+#include "InputSection.h"
 #include "Writer.h"
 #include "lld/Common/LLVM.h"
 #include "lld/Common/Strings.h"
@@ -287,7 +288,8 @@ class LinkerScript final {
 
   SmallVector<InputSectionBase *, 0>
   computeInputSections(const InputSectionDescription *,
-                       ArrayRef<InputSectionBase *>);
+                       ArrayRef<InputSectionBase *>,
+                       const OutputSection &outCmd);
 
   SmallVector<InputSectionBase *, 0> createInputSectionList(OutputSection &cmd);
 
@@ -333,6 +335,8 @@ public:
 
   bool shouldKeep(InputSectionBase *s);
   const Defined *assignAddresses();
+  bool spillSections();
+  void erasePotentialSpillSections();
   void allocateHeaders(SmallVector<PhdrEntry *, 0> &phdrs);
   void processSectionCommands();
   void processSymbolAssignments();
@@ -400,6 +404,15 @@ public:
   //
   // then provideMap should contain the mapping: 'v' -> ['a', 'b', 'c']
   llvm::MapVector<StringRef, SmallVector<StringRef, 0>> provideMap;
+
+  // List of potential spill locations (PotentialSpillSection) for an input
+  // section.
+  struct PotentialSpillList {
+    // Never nullptr.
+    PotentialSpillSection *head;
+    PotentialSpillSection *tail;
+  };
+  llvm::DenseMap<InputSectionBase *, PotentialSpillList> potentialSpillLists;
 };
 
 struct ScriptWrapper {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index b9e05a4b1fd5..883a6079bf50 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -197,6 +197,9 @@ def emit_relocs: F<"emit-relocs">, HelpText<"Generate relocations in output">;
 def enable_new_dtags: F<"enable-new-dtags">,
   HelpText<"Enable new dynamic tags (default)">;
 
+def enable_non_contiguous_regions : FF<"enable-non-contiguous-regions">,
+  HelpText<"Spill input sections to later matching output sections to avoid memory region overflow">;
+
 def end_group: F<"end-group">,
   HelpText<"Ignored for compatibility with GNU unless you pass --warn-backrefs">;
 
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 9c667241360f..fcb4c4387aa9 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -186,7 +186,7 @@ static MergeSyntheticSection *createMergeSynthetic(StringRef name,
 // new synthetic sections at the location of the first input section
 // that it replaces. It then finalizes each synthetic section in order
 // to compute an output offset for each piece of each input section.
-void OutputSection::finalizeInputSections() {
+void OutputSection::finalizeInputSections(LinkerScript *script) {
   std::vector<MergeSyntheticSection *> mergeSections;
   for (SectionCommand *cmd : commands) {
     auto *isd = dyn_cast<InputSectionDescription>(cmd);
@@ -226,6 +226,11 @@ void OutputSection::finalizeInputSections() {
         i = std::prev(mergeSections.end());
         syn->entsize = ms->entsize;
         isd->sections.push_back(syn);
+        // The merge synthetic section inherits the potential spill locations of
+        // its first contained section.
+        auto it = script->potentialSpillLists.find(ms);
+        if (it != script->potentialSpillLists.end())
+          script->potentialSpillLists.try_emplace(syn, it->second);
       }
       (*i)->addSection(ms);
     }
diff --git a/lld/ELF/OutputSections.h b/lld/ELF/OutputSections.h
index 421a0181feb5..78fede48a23f 100644
--- a/lld/ELF/OutputSections.h
+++ b/lld/ELF/OutputSections.h
@@ -75,7 +75,7 @@ public:
 
   void recordSection(InputSectionBase *isec);
   void commitSection(InputSection *isec);
-  void finalizeInputSections();
+  void finalizeInputSections(LinkerScript *script = nullptr);
 
   // The following members are normally only used in linker scripts.
   MemoryRegion *memRegion = nullptr;
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 7b9ada40c0f6..298c714adb3b 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -4074,6 +4074,13 @@ static bool isDuplicateArmExidxSec(InputSection *prev, InputSection *cur) {
 // InputSection with the highest address and any InputSections that have
 // mergeable .ARM.exidx table entries are removed from it.
 void ARMExidxSyntheticSection::finalizeContents() {
+  // Ensure that any fixed-point iterations after the first see the original set
+  // of sections.
+  if (!originalExecutableSections.empty())
+    executableSections = originalExecutableSections;
+  else if (config->enableNonContiguousRegions)
+    originalExecutableSections = executableSections;
+
   // The executableSections and exidxSections that we use to derive the final
   // contents of this SyntheticSection are populated before
   // processSectionCommands() and ICF. A /DISCARD/ entry in SECTIONS command or
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index 995fd4b344b0..34949025a45f 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -1255,6 +1255,10 @@ private:
   // either find the .ARM.exidx section or know that we need to generate one.
   SmallVector<InputSection *, 0> executableSections;
 
+  // Value of executableSecitons before finalizeContents(), so that it can be
+  // run repeateadly during fixed point iteration.
+  SmallVector<InputSection *, 0> originalExecutableSections;
+
   // The executable InputSection with the highest address to use for the
   // sentinel. We store separately from ExecutableSections as merging of
   // duplicate entries may mean this InputSection is removed from
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index e400ed2ae945..8d529f2bdb9f 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -1403,13 +1403,18 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
   AArch64Err843419Patcher a64p;
   ARMErr657417Patcher a32p;
   script->assignAddresses();
+
   // .ARM.exidx and SHF_LINK_ORDER do not require precise addresses, but they
   // do require the relative addresses of OutputSections because linker scripts
   // can assign Virtual Addresses to OutputSections that are not monotonically
-  // increasing.
-  for (Partition &part : partitions)
-    finalizeSynthetic(part.armExidx.get());
-  resolveShfLinkOrder();
+  // increasing. Anything here must be repeatable, since spilling may change
+  // section order.
+  const auto finalizeOrderDependentContent = [this] {
+    for (Partition &part : partitions)
+      finalizeSynthetic(part.armExidx.get());
+    resolveShfLinkOrder();
+  };
+  finalizeOrderDependentContent();
 
   // Converts call x@GDPLT to call __tls_get_addr
   if (config->emachine == EM_HEXAGON)
@@ -1419,6 +1424,8 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
   for (;;) {
     bool changed = target->needsThunks ? tc.createThunks(pass, outputSections)
                                        : target->relaxOnce(pass);
+    bool spilled = script->spillSections();
+    changed |= spilled;
     ++pass;
 
     // With Thunk Size much smaller than branch range we expect to
@@ -1464,6 +1471,9 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
                     " does not converge");
         break;
       }
+    } else if (spilled) {
+      // Spilling can change relative section order.
+      finalizeOrderDependentContent();
     }
   }
   if (!config->relocatable)
@@ -1483,6 +1493,10 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
              osec->name + " is not a multiple of alignment (" +
              Twine(osec->addralign) + ")");
     }
+
+  // Sizes are no longer allowed to grow, so all allowable spills have been
+  // taken. Remove any leftover potential spills.
+  script->erasePotentialSpillSections();
 }
 
 // If Input Sections have been shrunk (basic block sections) then
diff --git a/lld/docs/ELF/linker_script.rst b/lld/docs/ELF/linker_script.rst
index 3606ef4fe4b8..7a35534be096 100644
--- a/lld/docs/ELF/linker_script.rst
+++ b/lld/docs/ELF/linker_script.rst
@@ -197,3 +197,14 @@ the current location to a max-page-size boundary, ensuring that the next
 
 LLD will insert ``.relro_padding`` immediately before the symbol assignment
 using ``DATA_SEGMENT_RELRO_END``.
+
+Non-contiguous regions
+~~~~~~~~~~~~~~~~~~~~~~
+
+The flag ``--enable-non-contiguous-regions`` allows input sections to spill to
+later matches rather than causing the link to fail by overflowing a memory
+region. Unlike GNU ld, ``/DISCARD/`` only matches previously-unmatched sections
+(i.e., the flag does not affect it). Also, if a section fails to fit at any of
+its matches, the link fails instead of discarding the section. Accordingly, the
+GNU flag ``--enable-non-contiguous-regions-warnings`` is not implemented, as it
+exists to warn about such occurrences.
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index f8fdebfeaecf..e7a913e025da 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -38,6 +38,12 @@ ELF Improvements
 * ``--debug-names`` is added to create a merged ``.debug_names`` index
   from input ``.debug_names`` sections. Type units are not handled yet.
   (`#86508 <https://github.com/llvm/llvm-project/pull/86508>`_)
+* ``--enable-non-contiguous-regions`` option allows automatically packing input
+  sections into memory regions by automatically spilling to later matches if a
+  region would overflow. This reduces the toil of manually packing regions
+  (typical for embedded). It also makes full LTO feasible in such cases, since
+  IR merging currently prevents the linker script from referring to input
+  files. (`#90007 <https://github.com/llvm/llvm-project/pull/90007>`_)
 
 Breaking changes
 ----------------
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 9ea1a9c52f2a..0df13f07f560 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -222,6 +222,8 @@ segment header.
 Generate relocations in the output.
 .It Fl -enable-new-dtags
 Enable new dynamic tags.
+.It Fl -enable-non-contiguous-regions
+Spill input sections to later matching output sections to avoid memory region overflow.
 .It Fl -end-lib
 End a grouping of objects that should be treated as if they were together
 in an archive.
diff --git a/lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test b/lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test
new file mode 100644
index 000000000000..3f7b9c4e5f8b
--- /dev/null
+++ b/lld/test/ELF/linkerscript/enable-non-contiguous-regions-arm-exidx.test
@@ -0,0 +1,55 @@
+## When spilling reorders input sections, the .ARM.exidx table is rebuilt using
+## the new order.
+
+# REQUIRES: arm
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=armv7a-none-linux-gnueabi test.s -o test.o
+# RUN: ld.lld -T test.ld test.o -o test --enable-non-contiguous-regions
+# RUN: llvm-readobj -x .ARM.exidx test | FileCheck %s
+
+# CHECK:      20000000 08849780 1c000000 10849880
+# CHECK-NEXT: 1c000000 01000000
+
+#--- test.ld
+MEMORY {
+  exidx : ORIGIN = 0, LENGTH = 32
+  a : ORIGIN = 32, LENGTH = 4
+  b : ORIGIN = 36, LENGTH = 4
+  c : ORIGIN = 40, LENGTH = 4
+}
+
+SECTIONS {
+  .ARM.exidx : { *(.ARM.exidx) } >exidx
+  .first_chance : { *(.text .text.f2) } >a
+  .text.f1 : { *(.text.f1) } >b
+  .last_chance : { *(.text.f2) } >c
+}
+
+#--- test.s
+ .syntax unified
+ .section .text, "ax",%progbits
+ .globl _start
+_start:
+ .fnstart
+ bx lr
+ .save {r7, lr}
+ .setfp r7, sp, #0
+ .fnend
+
+ .section .text.f1, "ax", %progbits
+ .globl f1
+f1:
+ .fnstart
+ bx lr
+ .save {r8, lr}
+ .setfp r8, sp, #0
+ .fnend
+
+ .section .text.f2, "ax", %progbits
+ .globl f2
+f2:
+ .fnstart
+ bx lr
+ .save {r8, lr}
+ .setfp r8, sp, #0
+ .fnend
diff --git a/lld/test/ELF/linkerscript/enable-non-contiguous-regions.test b/lld/test/ELF/linkerscript/enable-non-contiguous-regions.test
new file mode 100644
index 000000000000..392106fd476f
--- /dev/null
+++ b/lld/test/ELF/linkerscript/enable-non-contiguous-regions.test
@@ -0,0 +1,265 @@
+# REQUIRES: x86
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -n -filetype=obj -triple=x86_64 spill.s -o spill.o
+
+## An input section spills to a later match when the region of its first match
+## would overflow. The spill uses the alignment of the later match.
+
+# RUN: ld.lld -T spill.ld spill.o -o spill --enable-non-contiguous-regions
+# RUN: llvm-readelf -S spill | FileCheck %s --check-prefix=SPILL
+
+# SPILL:      Name          Type     Address          Off    Size
+# SPILL:      .first_chance PROGBITS 0000000000000000 001000 000001
+# SPILL-NEXT: .last_chance  PROGBITS 0000000000000008 001008 000002
+
+## A spill off the end still fails the link.
+
+# RUN: not ld.lld -T spill-fail.ld spill.o --enable-non-contiguous-regions 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=SPILL-FAIL --implicit-check-not=error:
+
+# SPILL-FAIL: error: section '.last_chance' will not fit in region 'b': overflowed by 2 bytes
+
+## The above spill still occurs when the LMA would overflow, even though the
+## VMA would fit.
+
+# RUN: ld.lld -T spill-lma.ld spill.o -o spill-lma --enable-non-contiguous-regions
+# RUN: llvm-readelf -S spill-lma | FileCheck %s --check-prefix=SPILL-LMA
+
+# SPILL-LMA:      Name          Type     Address          Off    Size
+# SPILL-LMA:      .first_chance PROGBITS 0000000000000000 001000 000001
+# SPILL-LMA-NEXT: .last_chance  PROGBITS 0000000000000003 001003 000002
+
+## A spill occurs to an additional match after the first.
+
+# RUN: ld.lld -T spill-later.ld spill.o -o spill-later --enable-non-contiguous-regions
+# RUN: llvm-readelf -S spill-later | FileCheck %s --check-prefix=SPILL-LATER
+
+# SPILL-LATER:      Name            Type     Address          Off    Size
+# SPILL-LATER:      .first_chance   PROGBITS 0000000000000000 001000 000001
+# SPILL-LATER-NEXT: .second_chance  PROGBITS 0000000000000002 001001 000000
+# SPILL-LATER-NEXT: .last_chance    PROGBITS 0000000000000003 001003 000002
+
+## A later overflow causes an earlier section to spill.
+
+# RUN: ld.lld -T spill-earlier.ld spill.o -o spill-earlier --enable-non-contiguous-regions
+# RUN: llvm-readelf -S spill-earlier | FileCheck %s --check-prefix=SPILL-EARLIER
+
+# SPILL-EARLIER:      Name          Type     Address          Off    Size
+# SPILL-EARLIER:      .first_chance PROGBITS 0000000000000000 001000 000002
+# SPILL-EARLIER-NEXT: .last_chance  PROGBITS 0000000000000002 001002 000001
+
+## An additional match in /DISCARD/ has no effect.
+
+# RUN: not ld.lld -T no-spill-into-discard.ld spill.o --enable-non-contiguous-regions 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=NO-SPILL-INTO-DISCARD --implicit-check-not=error:
+
+# NO-SPILL-INTO-DISCARD: error: section '.osec' will not fit in region 'a': overflowed by 1 bytes
+
+## An additional match after /DISCARD/ has no effect.
+
+# RUN: ld.lld -T no-spill-from-discard.ld spill.o -o no-spill-from-discard --enable-non-contiguous-regions
+# RUN: llvm-readelf -S no-spill-from-discard | FileCheck %s --check-prefix=NO-SPILL-FROM-DISCARD
+
+# NO-SPILL-FROM-DISCARD: Name          Type     Address          Off    Size
+# NO-SPILL-FROM-DISCARD-NOT: .osec
+
+## SHF_MERGEd sections are spilled according to the matches of the first merged
+## input section (the one giving the resulting section its name).
+
+# RUN: llvm-mc -n -filetype=obj -triple=x86_64 merge.s -o merge.o
+# RUN: ld.lld -T spill-merge.ld merge.o -o spill-merge --enable-non-contiguous-regions
+# RUN: llvm-readelf -S spill-merge | FileCheck %s --check-prefix=SPILL-MERGE
+
+# SPILL-MERGE:      Name          Type     Address          Off    Size
+# SPILL-MERGE:      .first  PROGBITS 0000000000000000 000190 000000
+# SPILL-MERGE-NEXT: .second PROGBITS 0000000000000001 001001 000002
+# SPILL-MERGE-NEXT: .third  PROGBITS 0000000000000003 001003 000000
+
+## An error is reported for INSERT.
+
+# RUN: not ld.lld -T insert.ld spill.o --enable-non-contiguous-regions 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=INSERT
+
+# INSERT: error: INSERT cannot be used with --enable-non-contiguous-regions
+
+## An error is reported for OVERWRITE_SECTIONS.
+
+# RUN: not ld.lld -T overwrite-sections.ld spill.o --enable-non-contiguous-regions 2>&1 |\
+# RUN:   FileCheck %s --check-prefix=OVERWRITE_SECTIONS
+
+# OVERWRITE_SECTIONS: error: OVERWRITE_SECTIONS cannot be used with --enable-non-contiguous-regions
+
+## SHF_LINK_ORDER is reordered when spilling changes relative section order.
+
+# RUN: llvm-mc -n -filetype=obj -triple=x86_64 link-order.s -o link-order.o
+# RUN: ld.lld -T link-order.ld link-order.o -o link-order --enable-non-contiguous-regions
+# RUN: llvm-readobj -x .order link-order | FileCheck %s --check-prefix=LINK-ORDER
+
+# LINK-ORDER: 020301
+
+#--- spill.s
+.section .one_byte_section,"a",@progbits
+.fill 1
+
+.section .two_byte_section,"a",@progbits
+.fill 2
+
+#--- spill.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 2
+  b : ORIGIN = 2, LENGTH = 16
+}
+
+SECTIONS {
+  .first_chance : SUBALIGN(1) { *(.one_byte_section) *(.two_byte_section) } >a
+  .last_chance : SUBALIGN(8) { *(.two_byte_section) } >b
+}
+
+#--- spill-fail.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 1
+  b : ORIGIN = 2, LENGTH = 0
+}
+
+SECTIONS {
+  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >a
+  .last_chance : { *(.two_byte_section) } >b
+}
+
+#--- spill-lma.ld
+MEMORY {
+  vma_a : ORIGIN = 0, LENGTH = 3
+  vma_b : ORIGIN = 3, LENGTH = 3
+  lma_a : ORIGIN = 6, LENGTH = 2
+  lma_b : ORIGIN = 8, LENGTH = 2
+}
+
+SECTIONS {
+  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >vma_a AT>lma_a
+  .last_chance : { *(.two_byte_section) } >vma_b AT>lma_b
+}
+
+#--- spill-later.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 2
+  b : ORIGIN = 2, LENGTH = 1
+  c : ORIGIN = 3, LENGTH = 2
+}
+
+SECTIONS {
+  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >a
+  .second_chance : { *(.two_byte_section) } >b
+  .last_chance : { *(.two_byte_section) } >c
+}
+
+#--- spill-earlier.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 2
+  b : ORIGIN = 2, LENGTH = 1
+}
+
+SECTIONS {
+  .first_chance : { *(.one_byte_section) *(.two_byte_section) } >a
+  .last_chance : { *(.one_byte_section) } >b
+}
+
+#--- no-spill-into-discard.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 1
+}
+
+SECTIONS {
+  .osec : { *(.two_byte_section) } >a
+  /DISCARD/ : { *(.one_byte_section) *(.two_byte_section) }
+}
+
+#--- no-spill-from-discard.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 2
+}
+
+SECTIONS {
+  /DISCARD/ : { *(.one_byte_section) *(.two_byte_section) }
+  .osec : { *(.two_byte_section) } >a
+}
+
+#--- merge.s
+.section .a,"aM",@progbits,1
+.byte 0x12, 0x34
+
+.section .b,"aM",@progbits,1
+.byte 0x12
+
+#--- spill-merge.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 1
+  b : ORIGIN = 1, LENGTH = 2
+  c : ORIGIN = 3, LENGTH = 2
+}
+
+SECTIONS {
+  .first : { *(.a) *(.b) } >a
+  .second : { *(.a) } >b
+  .third : { *(.b) } >c
+}
+
+#--- insert.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 1
+}
+
+SECTIONS {
+  .a : { *(.two_byte_section) } >a
+}
+
+SECTIONS {
+  .b : { *(.one_byte_section) } >a
+} INSERT AFTER .a;
+
+#--- overwrite-sections.ld
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 1
+}
+
+SECTIONS {
+  .a : { *(.two_byte_section) } >a
+}
+
+OVERWRITE_SECTIONS {
+  .b : { *(.one_byte_section) } >a
+}
+
+#--- link-order.s
+.section .a,"a",@progbits
+.fill 1
+
+.section .b,"a",@progbits
+.fill 1
+
+.section .c,"a",@progbits
+.fill 1
+
+.section .link_order.a,"ao",@progbits,.a
+.byte 1
+
+.section .link_order.b,"ao",@progbits,.b
+.byte 2
+
+.section .link_order.c,"ao",@progbits,.c
+.byte 3
+
+#--- link-order.ld
+MEMORY {
+  order : ORIGIN = 0, LENGTH = 3
+  potential_a : ORIGIN = 3, LENGTH = 0
+  bc : ORIGIN = 3, LENGTH = 2
+  actual_a : ORIGIN = 5, LENGTH = 1
+}
+
+SECTIONS {
+  .order :  { *(.link_order.*) } > order
+  .potential_a : { *(.a) } >potential_a
+  .bc : { *(.b) *(.c) } >bc
+  .actual_a : { *(.a) } >actual_a
+}
-- 
GitLab


From dc7ce3b41c936c4cc189b4bbf6a2e3b5475d9fc5 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Mon, 13 May 2024 19:12:49 +0100
Subject: [PATCH 0617/1206] [lldb][TypeSystem][NFCI] Log creation of new
 TypeSystem instances to expression log (#91985)

We emit `ASTContext` and `TypeSystem` pointers into the `expr` log but
there is no easy way (that I know of) to correlate the pointer value
back to an easily readible form. This patch simply logs the name of the
`TypeSystem` and the associated `ASTContext` into the `expr` channel
whenever we create a new `TypeSystemClang`.

The following is an example of the new log entries:
```
$ grep Created /tmp/lldb.log
 Created new TypeSystem for (ASTContext*)0x0000000101a2e200 'ASTContext for '/Users/michaelbuch/a.out''
 Created new TypeSystem for (ASTContext*)0x0000000102512a00 'scratch ASTContext'
 Created new TypeSystem for (ASTContext*)0x0000000102116a00 'ClangModulesDeclVendor ASTContext'
 Created new TypeSystem for (ASTContext*)0x00000001022e8c00 'Expression ASTContext for '<user expression 0>''
 Created new TypeSystem for (ASTContext*)0x00000001103e7200 'AppleObjCTypeEncodingParser ASTContext'
 Created new TypeSystem for (ASTContext*)0x00000001103f7000 'AppleObjCDeclVendor AST'
 Created new TypeSystem for (ASTContext*)0x00000001104bfe00 'Expression ASTContext for '<clang expression>''
 Created new TypeSystem for (ASTContext*)0x0000000101f01000 'Expression ASTContext for '<clang expression>''
 Created new TypeSystem for (ASTContext*)0x00000001025d3c00 'Expression ASTContext for '<clang expression>''
 Created new TypeSystem for (ASTContext*)0x0000000110422400 'Expression ASTContext for '<clang expression>''
 Created new TypeSystem for (ASTContext*)0x000000011602c200 'Expression ASTContext for '<user expression 1>''
 Created new TypeSystem for (ASTContext*)0x0000000110641600 'Expression ASTContext for '<clang expression>''
 Created new TypeSystem for (ASTContext*)0x0000000110617400 'Expression ASTContext for '<clang expression>''
```
---
 .../Plugins/TypeSystem/Clang/TypeSystemClang.cpp     | 12 +++++++++++-
 .../Plugins/TypeSystem/Clang/TypeSystemClang.h       |  8 +++++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index d0033fcd9cdf..17a9c675fbba 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -501,6 +501,8 @@ TypeSystemClang::TypeSystemClang(llvm::StringRef name,
   // The caller didn't pass an ASTContext so create a new one for this
   // TypeSystemClang.
   CreateASTContext();
+
+  LogCreation();
 }
 
 TypeSystemClang::TypeSystemClang(llvm::StringRef name,
@@ -510,6 +512,8 @@ TypeSystemClang::TypeSystemClang(llvm::StringRef name,
 
   m_ast_up.reset(&existing_ctxt);
   GetASTMap().Insert(&existing_ctxt, this);
+
+  LogCreation();
 }
 
 // Destructor
@@ -630,7 +634,7 @@ void TypeSystemClang::SetExternalSource(
   ast.setExternalSource(ast_source_up);
 }
 
-ASTContext &TypeSystemClang::getASTContext() {
+ASTContext &TypeSystemClang::getASTContext() const {
   assert(m_ast_up);
   return *m_ast_up;
 }
@@ -9750,3 +9754,9 @@ bool TypeSystemClang::SetDeclIsForcefullyCompleted(const clang::TagDecl *td) {
   metadata->SetIsForcefullyCompleted();
   return true;
 }
+
+void TypeSystemClang::LogCreation() const {
+  if (auto *log = GetLog(LLDBLog::Expressions))
+    LLDB_LOG(log, "Created new TypeSystem for (ASTContext*){0:x} '{1}'",
+             &getASTContext(), getDisplayName());
+}
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index 59ca69622d9e..042379d40bcb 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -162,7 +162,7 @@ public:
   llvm::StringRef getDisplayName() const { return m_display_name; }
 
   /// Returns the clang::ASTContext instance managed by this TypeSystemClang.
-  clang::ASTContext &getASTContext();
+  clang::ASTContext &getASTContext() const;
 
   clang::MangleContext *getMangleContext();
 
@@ -1166,6 +1166,12 @@ private:
   bool IsTypeImpl(lldb::opaque_compiler_type_t type,
                   llvm::function_ref<bool(clang::QualType)> predicate) const;
 
+  /// Emits information about this TypeSystem into the expression log.
+  ///
+  /// Helper method that is used in \ref TypeSystemClang::TypeSystemClang
+  /// on creation of a new instance.
+  void LogCreation() const;
+
   // Classes that inherit from TypeSystemClang can see and modify these
   std::string m_target_triple;
   std::unique_ptr<clang::ASTContext> m_ast_up;
-- 
GitLab


From 4a67f809828e11988c6a097cb400fd7cbbf47628 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 13 May 2024 11:25:12 -0700
Subject: [PATCH 0618/1206] [test] Fix check prefixes

---
 .../CodeGen/AArch64/aarch64_tree_tests.ll     | 40 +++++++++----------
 llvm/test/CodeGen/SPARC/inlineasm-bad.ll      |  5 +--
 llvm/test/MC/AArch64/SVE/index.s              | 16 ++++----
 llvm/test/MC/XCOFF/inlineasm.s                |  2 +-
 .../PhaseOrdering/lifetime-sanitizer.ll       | 20 +++++-----
 .../test/tools/dsymutil/ARM/dwarf5-macho.test | 26 ++++++------
 6 files changed, 54 insertions(+), 55 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/aarch64_tree_tests.ll b/llvm/test/CodeGen/AArch64/aarch64_tree_tests.ll
index 0a06765a8f75..f66f96cf463b 100644
--- a/llvm/test/CodeGen/AArch64/aarch64_tree_tests.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64_tree_tests.ll
@@ -1,19 +1,18 @@
-; RUN: llc < %s | FileCheck %s 
+; RUN: llc < %s | FileCheck %s
 
 ; ModuleID = 'aarch64_tree_tests.bc'
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "arm64--linux-gnu"
 
-; FIXME: Misspelled CHECK-LABEL
-; CHECK-LABLE: @aarch64_tree_tests_and
-; CHECK: .hword	32768                   
-; CHECK: .hword	32767                   
-; CHECK: .hword	4664                    
-; CHECK: .hword	32767                   
-; CHECK: .hword	32768                   
-; CHECK: .hword	32768                   
-; CHECK: .hword	0                       
-; CHECK: .hword	0                      
+; CHECK-LABEL: .LCPI0_0:
+; CHECK-NEXT:    .hword 32768
+; CHECK-NEXT:    .hword 32767
+; CHECK-NEXT:    .hword 4664
+; CHECK-NEXT:    .hword 32767
+; CHECK-NEXT:    .hword 32768
+; CHECK-NEXT:    .hword 32768
+; CHECK-NEXT:    .hword 0
+; CHECK-NEXT:    .hword 0
 
 ; Function Attrs: nounwind readnone
 define <8 x i16> @aarch64_tree_tests_and(<8 x i16> %a) {
@@ -23,16 +22,15 @@ entry:
   ret <8 x i16> %ret
 }
 
-; FIXME: Misspelled CHECK-LABEL
-; CHECK-LABLE: @aarch64_tree_tests_or
-; CHECK: .hword	32768                 
-; CHECK: .hword	32766
-; CHECK: .hword	4664     
-; CHECK: .hword	32766                
-; CHECK: .hword	32768 
-; CHECK: .hword	32768
-; CHECK: .hword	65535            
-; CHECK: .hword	65535
+; CHECK-LABEL: .LCPI1_0:
+; CHECK-NEXT:    .hword 32768
+; CHECK-NEXT:    .hword 32766
+; CHECK-NEXT:    .hword 4664
+; CHECK-NEXT:    .hword 32766
+; CHECK-NEXT:    .hword 32768
+; CHECK-NEXT:    .hword 32768
+; CHECK-NEXT:    .hword 65535
+; CHECK-NEXT:    .hword 65535
 
 ; Function Attrs: nounwind readnone
 define <8 x i16> @aarch64_tree_tests_or(<8 x i16> %a) {
diff --git a/llvm/test/CodeGen/SPARC/inlineasm-bad.ll b/llvm/test/CodeGen/SPARC/inlineasm-bad.ll
index 07eb67df6e5f..bfcf98ed7fd5 100644
--- a/llvm/test/CodeGen/SPARC/inlineasm-bad.ll
+++ b/llvm/test/CodeGen/SPARC/inlineasm-bad.ll
@@ -12,9 +12,8 @@ entry:
   ret void
 }
 
-; CHECK-label:test_twinword_error
-; CHECK: error: Hi part of pair should point to an even-numbered register
-; CHECK: error: (note that in some cases it might be necessary to manually bind the input/output registers instead of relying on automatic allocation)
+; CHECK: <unknown>:0: error: Hi part of pair should point to an even-numbered register
+; CHECK: <unknown>:0: error: (note that in some cases it might be necessary to manually bind the input/output registers instead of relying on automatic allocation)
 
 define i64 @test_twinword_error(){
   %1 = tail call i64 asm sideeffect "rd %asr5, ${0:L} \0A\09 srlx ${0:L}, 32, ${0:H}", "={i1}"()
diff --git a/llvm/test/MC/AArch64/SVE/index.s b/llvm/test/MC/AArch64/SVE/index.s
index c06fab25a82f..d4c106b9f0bf 100644
--- a/llvm/test/MC/AArch64/SVE/index.s
+++ b/llvm/test/MC/AArch64/SVE/index.s
@@ -180,16 +180,16 @@ index   z21.b, w10, w21
 // CHECK-UNKNOWN: 04354d55 <unknown>
 
 index   z31.h, wzr, wzr
-// check-inst: index   z31.h, wzr, wzr
-// check-encoding: [0xff,0x4f,0x7f,0x04]
-// check-error: instruction requires: sve or sme
-// check-unknown: ff 4f 7f 04 <unknown>
+// CHECK-INST: index   z31.h, wzr, wzr
+// CHECK-ENCODING: [0xff,0x4f,0x7f,0x04]
+// CHECK-ERROR: instruction requires: sve or sme
+// CHECK-UNKNOWN: 047f4fff <unknown>
 
 index   z0.h, w0, w0
-// check-inst: index   z0.h, w0, w0
-// check-encoding: [0x00,0x4c,0x60,0x04]
-// check-error: instruction requires: sve or sme
-// check-unknown: 00 4c 60 04 <unknown>
+// CHECK-INST: index   z0.h, w0, w0
+// CHECK-ENCODING: [0x00,0x4c,0x60,0x04]
+// CHECK-ERROR: instruction requires: sve or sme
+// CHECK-UNKNOWN: 04604c00 <unknown>
 
 index   z31.s, wzr, wzr
 // CHECK-INST: index   z31.s, wzr, wzr
diff --git a/llvm/test/MC/XCOFF/inlineasm.s b/llvm/test/MC/XCOFF/inlineasm.s
index 85a40024711a..e92d403f45a3 100644
--- a/llvm/test/MC/XCOFF/inlineasm.s
+++ b/llvm/test/MC/XCOFF/inlineasm.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -filetype=asm -triple powerpc-ibm-aix-xcoff %s | FileCheck %s
 
-// CHECK-label:       .csect .text[PR],2
+// CHECK-LABEL:       .csect ..text..[PR],5
 // CHECK:L..tmp0:
 // CHECK-NEXT:        lwarx 3, 0, 4
 // CHECK-NEXT:        cmpw    5, 3
diff --git a/llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll b/llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll
index 5f4d389265a8..21fa234535ca 100644
--- a/llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll
+++ b/llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll
@@ -1,11 +1,11 @@
-; RUN: opt < %s -O0 -S | FileCheck %s
-; RUN: opt < %s -O1 -S | FileCheck %s
-; RUN: opt < %s -O2 -S | FileCheck %s
-; RUN: opt < %s -O3 -S | FileCheck %s
-; RUN: opt < %s -passes='default<O0>' -S | FileCheck %s
-; RUN: opt < %s -passes='default<O1>' -S | FileCheck %s
-; RUN: opt < %s -passes='default<O2>' -S | FileCheck %s
-; RUN: opt < %s -passes='default<O3>' -S | FileCheck %s
+; RUN: opt < %s -O0 -S | FileCheck %s --check-prefixes=CHECK,NOOPT
+; RUN: opt < %s -O1 -S | FileCheck %s --check-prefixes=CHECK,OPT
+; RUN: opt < %s -O2 -S | FileCheck %s --check-prefixes=CHECK,OPT
+; RUN: opt < %s -O3 -S | FileCheck %s --check-prefixes=CHECK,OPT
+; RUN: opt < %s -passes='default<O0>' -S | FileCheck %s --check-prefixes=CHECK,NOOPT
+; RUN: opt < %s -passes='default<O1>' -S | FileCheck %s --check-prefixes=CHECK,OPT
+; RUN: opt < %s -passes='default<O2>' -S | FileCheck %s --check-prefixes=CHECK,OPT
+; RUN: opt < %s -passes='default<O3>' -S | FileCheck %s --check-prefixes=CHECK,OPT
 
 declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
 declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
@@ -63,7 +63,9 @@ entry:
 
   call void @llvm.lifetime.start.p0(i64 1, ptr %text)
   call void @llvm.lifetime.end.p0(i64 1, ptr %text)
-  ; CHECK-NO: call void @llvm.lifetime
+  ; OPT-NOT:    call void @llvm.lifetime
+  ; NOOPT:      call void @llvm.lifetime.start
+  ; NOOPT-NEXT: call void @llvm.lifetime.end
 
   call void @foo(ptr %text) ; Keep alloca alive
 
diff --git a/llvm/test/tools/dsymutil/ARM/dwarf5-macho.test b/llvm/test/tools/dsymutil/ARM/dwarf5-macho.test
index 5268324c2e10..08c8bba73928 100644
--- a/llvm/test/tools/dsymutil/ARM/dwarf5-macho.test
+++ b/llvm/test/tools/dsymutil/ARM/dwarf5-macho.test
@@ -109,16 +109,16 @@ CHECK-NEXT: 0x0000000e: [DW_RLE_offset_pair  ]: {{.*}}[0x[[RANGELIST_OFFSET_STAR
 CHECK-NEXT: 0x00000011: [DW_RLE_end_of_list  ]
 
 CHECK: .debug_names contents:
-CHECK-NEX:T Name Index @ 0x0 {
-CHECK-NEX:T   Header {
-CHECK-NEX:T     Length: 0x7C
-CHECK-NEX:T     Format: DWARF32
-CHECK-NEX:T     Version: 5
-CHECK-NEX:T     CU count: 1
-CHECK-NEX:T     Local TU count: 0
-CHECK-NEX:T     Foreign TU count: 0
-CHECK-NEX:T     Bucket count: 3
-CHECK-NEX:T     Name count: 3
-CHECK-NEX:T     Abbreviations table size: 0xD
-CHECK-NEX:T     Augmentation: 'LLVM0700'
-CHECK-NEX:T   }
+CHECK-NEXT: Name Index @ 0x0 {
+CHECK-NEXT:   Header {
+CHECK-NEXT:     Length:
+CHECK-NEXT:     Format: DWARF32
+CHECK-NEXT:     Version: 5
+CHECK-NEXT:     CU count: 1
+CHECK-NEXT:     Local TU count: 0
+CHECK-NEXT:     Foreign TU count: 0
+CHECK-NEXT:     Bucket count: 3
+CHECK-NEXT:     Name count: 3
+CHECK-NEXT:     Abbreviations table size:
+CHECK-NEXT:     Augmentation: 'LLVM0700'
+CHECK-NEXT:   }
-- 
GitLab


From bd679865c05b29450428ad460e59e2dcd07fe974 Mon Sep 17 00:00:00 2001
From: Leon Clark <PeddleSpam@users.noreply.github.com>
Date: Mon, 13 May 2024 19:39:26 +0100
Subject: [PATCH 0619/1206] [AMDGPU] Add tests for vector rebroadcast. (#91322)

Co-authored-by: Leon Clark <leoclark@amd.com>
---
 .../test/CodeGen/AMDGPU/vector_rebroadcast.ll | 1871 +++++++++++++++++
 1 file changed, 1871 insertions(+)
 create mode 100644 llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll

diff --git a/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll b/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll
new file mode 100644
index 000000000000..b079a94b5fcc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll
@@ -0,0 +1,1871 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
+
+define <2 x i8> @shuffle_v2i8_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v2i8_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_ushort v0, v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v2i8_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_ushort v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_lshrrev_b16 v0, 8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v2i8_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_u16 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_lshrrev_b16 v0, 8, v0
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <2 x i8>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <2 x i8> %val0, <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+  ret <2 x i8> %val1
+}
+
+define <4 x i8> @shuffle_v4i8_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v4i8_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v4i8_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v4i8_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <4 x i8>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <4 x i8> %val0, <4 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i8> %val1
+}
+
+define <8 x i8> @shuffle_v8i8_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v8i8_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v8i8_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v8i8_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <8 x i8>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <8 x i8> %val0, <8 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <8 x i8> %val1
+}
+
+define <16 x i8> @shuffle_v16i8_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v16i8_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  v_mov_b32_e32 v8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v9, v0
+; GFX9-NEXT:  v_mov_b32_e32 v10, v0
+; GFX9-NEXT:  v_mov_b32_e32 v11, v0
+; GFX9-NEXT:  v_mov_b32_e32 v12, v0
+; GFX9-NEXT:  v_mov_b32_e32 v13, v0
+; GFX9-NEXT:  v_mov_b32_e32 v14, v0
+; GFX9-NEXT:  v_mov_b32_e32 v15, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v16i8_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  v_mov_b32_e32 v8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v9, v0
+; GFX10-NEXT:  v_mov_b32_e32 v10, v0
+; GFX10-NEXT:  v_mov_b32_e32 v11, v0
+; GFX10-NEXT:  v_mov_b32_e32 v12, v0
+; GFX10-NEXT:  v_mov_b32_e32 v13, v0
+; GFX10-NEXT:  v_mov_b32_e32 v14, v0
+; GFX10-NEXT:  v_mov_b32_e32 v15, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v16i8_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  v_mov_b32_e32 v8, v0
+; GFX11-NEXT:  v_mov_b32_e32 v9, v0
+; GFX11-NEXT:  v_mov_b32_e32 v10, v0
+; GFX11-NEXT:  v_mov_b32_e32 v11, v0
+; GFX11-NEXT:  v_mov_b32_e32 v12, v0
+; GFX11-NEXT:  v_mov_b32_e32 v13, v0
+; GFX11-NEXT:  v_mov_b32_e32 v14, v0
+; GFX11-NEXT:  v_mov_b32_e32 v15, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <16 x i8>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <16 x i8> %val0, <16 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <16 x i8> %val1
+}
+
+define <32 x i8> @shuffle_v32i8_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v32i8_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  v_mov_b32_e32 v8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v9, v0
+; GFX9-NEXT:  v_mov_b32_e32 v10, v0
+; GFX9-NEXT:  v_mov_b32_e32 v11, v0
+; GFX9-NEXT:  v_mov_b32_e32 v12, v0
+; GFX9-NEXT:  v_mov_b32_e32 v13, v0
+; GFX9-NEXT:  v_mov_b32_e32 v14, v0
+; GFX9-NEXT:  v_mov_b32_e32 v15, v0
+; GFX9-NEXT:  v_mov_b32_e32 v16, v0
+; GFX9-NEXT:  v_mov_b32_e32 v17, v0
+; GFX9-NEXT:  v_mov_b32_e32 v18, v0
+; GFX9-NEXT:  v_mov_b32_e32 v19, v0
+; GFX9-NEXT:  v_mov_b32_e32 v20, v0
+; GFX9-NEXT:  v_mov_b32_e32 v21, v0
+; GFX9-NEXT:  v_mov_b32_e32 v22, v0
+; GFX9-NEXT:  v_mov_b32_e32 v23, v0
+; GFX9-NEXT:  v_mov_b32_e32 v24, v0
+; GFX9-NEXT:  v_mov_b32_e32 v25, v0
+; GFX9-NEXT:  v_mov_b32_e32 v26, v0
+; GFX9-NEXT:  v_mov_b32_e32 v27, v0
+; GFX9-NEXT:  v_mov_b32_e32 v28, v0
+; GFX9-NEXT:  v_mov_b32_e32 v29, v0
+; GFX9-NEXT:  v_mov_b32_e32 v30, v0
+; GFX9-NEXT:  v_mov_b32_e32 v31, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v32i8_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  v_mov_b32_e32 v8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v9, v0
+; GFX10-NEXT:  v_mov_b32_e32 v10, v0
+; GFX10-NEXT:  v_mov_b32_e32 v11, v0
+; GFX10-NEXT:  v_mov_b32_e32 v12, v0
+; GFX10-NEXT:  v_mov_b32_e32 v13, v0
+; GFX10-NEXT:  v_mov_b32_e32 v14, v0
+; GFX10-NEXT:  v_mov_b32_e32 v15, v0
+; GFX10-NEXT:  v_mov_b32_e32 v16, v0
+; GFX10-NEXT:  v_mov_b32_e32 v17, v0
+; GFX10-NEXT:  v_mov_b32_e32 v18, v0
+; GFX10-NEXT:  v_mov_b32_e32 v19, v0
+; GFX10-NEXT:  v_mov_b32_e32 v20, v0
+; GFX10-NEXT:  v_mov_b32_e32 v21, v0
+; GFX10-NEXT:  v_mov_b32_e32 v22, v0
+; GFX10-NEXT:  v_mov_b32_e32 v23, v0
+; GFX10-NEXT:  v_mov_b32_e32 v24, v0
+; GFX10-NEXT:  v_mov_b32_e32 v25, v0
+; GFX10-NEXT:  v_mov_b32_e32 v26, v0
+; GFX10-NEXT:  v_mov_b32_e32 v27, v0
+; GFX10-NEXT:  v_mov_b32_e32 v28, v0
+; GFX10-NEXT:  v_mov_b32_e32 v29, v0
+; GFX10-NEXT:  v_mov_b32_e32 v30, v0
+; GFX10-NEXT:  v_mov_b32_e32 v31, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v32i8_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_lshrrev_b32_e32 v0, 8, v0
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  v_mov_b32_e32 v8, v0
+; GFX11-NEXT:  v_mov_b32_e32 v9, v0
+; GFX11-NEXT:  v_mov_b32_e32 v10, v0
+; GFX11-NEXT:  v_mov_b32_e32 v11, v0
+; GFX11-NEXT:  v_mov_b32_e32 v12, v0
+; GFX11-NEXT:  v_mov_b32_e32 v13, v0
+; GFX11-NEXT:  v_mov_b32_e32 v14, v0
+; GFX11-NEXT:  v_mov_b32_e32 v15, v0
+; GFX11-NEXT:  v_mov_b32_e32 v16, v0
+; GFX11-NEXT:  v_mov_b32_e32 v17, v0
+; GFX11-NEXT:  v_mov_b32_e32 v18, v0
+; GFX11-NEXT:  v_mov_b32_e32 v19, v0
+; GFX11-NEXT:  v_mov_b32_e32 v20, v0
+; GFX11-NEXT:  v_mov_b32_e32 v21, v0
+; GFX11-NEXT:  v_mov_b32_e32 v22, v0
+; GFX11-NEXT:  v_mov_b32_e32 v23, v0
+; GFX11-NEXT:  v_mov_b32_e32 v24, v0
+; GFX11-NEXT:  v_mov_b32_e32 v25, v0
+; GFX11-NEXT:  v_mov_b32_e32 v26, v0
+; GFX11-NEXT:  v_mov_b32_e32 v27, v0
+; GFX11-NEXT:  v_mov_b32_e32 v28, v0
+; GFX11-NEXT:  v_mov_b32_e32 v29, v0
+; GFX11-NEXT:  v_mov_b32_e32 v30, v0
+; GFX11-NEXT:  v_mov_b32_e32 v31, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <32 x i8>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <32 x i8> %val0, <32 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <32 x i8> %val1
+}
+
+define <2 x i16> @shuffle_v2i16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v2i16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v2i16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v2i16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <2 x i16>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <2 x i16> %val0, <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+  ret <2 x i16> %val1
+}
+
+define <4 x i16> @shuffle_v4i16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v4i16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v4i16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v4i16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <4 x i16>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <4 x i16> %val0, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i16> %val1
+}
+
+define <8 x i16> @shuffle_v8i16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v8i16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v8i16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v8i16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <8 x i16>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <8 x i16> %val0, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <8 x i16> %val1
+}
+
+define <16 x i16> @shuffle_v16i16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v16i16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v16i16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v16i16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <16 x i16>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <16 x i16> %val0, <16 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <16 x i16> %val1
+}
+
+define <32 x i16> @shuffle_v32i16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v32i16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  v_mov_b32_e32 v8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v9, v0
+; GFX9-NEXT:  v_mov_b32_e32 v10, v0
+; GFX9-NEXT:  v_mov_b32_e32 v11, v0
+; GFX9-NEXT:  v_mov_b32_e32 v12, v0
+; GFX9-NEXT:  v_mov_b32_e32 v13, v0
+; GFX9-NEXT:  v_mov_b32_e32 v14, v0
+; GFX9-NEXT:  v_mov_b32_e32 v15, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v32i16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  v_mov_b32_e32 v8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v9, v0
+; GFX10-NEXT:  v_mov_b32_e32 v10, v0
+; GFX10-NEXT:  v_mov_b32_e32 v11, v0
+; GFX10-NEXT:  v_mov_b32_e32 v12, v0
+; GFX10-NEXT:  v_mov_b32_e32 v13, v0
+; GFX10-NEXT:  v_mov_b32_e32 v14, v0
+; GFX10-NEXT:  v_mov_b32_e32 v15, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v32i16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  v_mov_b32_e32 v8, v0
+; GFX11-NEXT:  v_mov_b32_e32 v9, v0
+; GFX11-NEXT:  v_mov_b32_e32 v10, v0
+; GFX11-NEXT:  v_mov_b32_e32 v11, v0
+; GFX11-NEXT:  v_mov_b32_e32 v12, v0
+; GFX11-NEXT:  v_mov_b32_e32 v13, v0
+; GFX11-NEXT:  v_mov_b32_e32 v14, v0
+; GFX11-NEXT:  v_mov_b32_e32 v15, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <32 x i16>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <32 x i16> %val0, <32 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <32 x i16> %val1
+}
+
+define <2 x i32> @shuffle_v2i32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v2i32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off offset:4
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v2i32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off offset:4
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v2i32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <2 x i32>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <2 x i32> %val0, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %val1
+}
+
+define <4 x i32> @shuffle_v4i32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v4i32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off offset:4
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v4i32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off offset:4
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v4i32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <4 x i32>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <4 x i32> %val0, <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %val1
+}
+
+define <8 x i32> @shuffle_v8i32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v8i32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off offset:4
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v8i32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off offset:4
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v8i32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <8 x i32>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <8 x i32> %val0, <8 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <8 x i32> %val1
+}
+
+define <16 x i32> @shuffle_v16i32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v16i32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off offset:4
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  v_mov_b32_e32 v8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v9, v0
+; GFX9-NEXT:  v_mov_b32_e32 v10, v0
+; GFX9-NEXT:  v_mov_b32_e32 v11, v0
+; GFX9-NEXT:  v_mov_b32_e32 v12, v0
+; GFX9-NEXT:  v_mov_b32_e32 v13, v0
+; GFX9-NEXT:  v_mov_b32_e32 v14, v0
+; GFX9-NEXT:  v_mov_b32_e32 v15, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v16i32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off offset:4
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  v_mov_b32_e32 v8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v9, v0
+; GFX10-NEXT:  v_mov_b32_e32 v10, v0
+; GFX10-NEXT:  v_mov_b32_e32 v11, v0
+; GFX10-NEXT:  v_mov_b32_e32 v12, v0
+; GFX10-NEXT:  v_mov_b32_e32 v13, v0
+; GFX10-NEXT:  v_mov_b32_e32 v14, v0
+; GFX10-NEXT:  v_mov_b32_e32 v15, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v16i32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  v_mov_b32_e32 v8, v0
+; GFX11-NEXT:  v_mov_b32_e32 v9, v0
+; GFX11-NEXT:  v_mov_b32_e32 v10, v0
+; GFX11-NEXT:  v_mov_b32_e32 v11, v0
+; GFX11-NEXT:  v_mov_b32_e32 v12, v0
+; GFX11-NEXT:  v_mov_b32_e32 v13, v0
+; GFX11-NEXT:  v_mov_b32_e32 v14, v0
+; GFX11-NEXT:  v_mov_b32_e32 v15, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <16 x i32>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <16 x i32> %val0, <16 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <16 x i32> %val1
+}
+
+define <32 x i32> @shuffle_v32i32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v32i32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off offset:4
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  v_mov_b32_e32 v8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v9, v0
+; GFX9-NEXT:  v_mov_b32_e32 v10, v0
+; GFX9-NEXT:  v_mov_b32_e32 v11, v0
+; GFX9-NEXT:  v_mov_b32_e32 v12, v0
+; GFX9-NEXT:  v_mov_b32_e32 v13, v0
+; GFX9-NEXT:  v_mov_b32_e32 v14, v0
+; GFX9-NEXT:  v_mov_b32_e32 v15, v0
+; GFX9-NEXT:  v_mov_b32_e32 v16, v0
+; GFX9-NEXT:  v_mov_b32_e32 v17, v0
+; GFX9-NEXT:  v_mov_b32_e32 v18, v0
+; GFX9-NEXT:  v_mov_b32_e32 v19, v0
+; GFX9-NEXT:  v_mov_b32_e32 v20, v0
+; GFX9-NEXT:  v_mov_b32_e32 v21, v0
+; GFX9-NEXT:  v_mov_b32_e32 v22, v0
+; GFX9-NEXT:  v_mov_b32_e32 v23, v0
+; GFX9-NEXT:  v_mov_b32_e32 v24, v0
+; GFX9-NEXT:  v_mov_b32_e32 v25, v0
+; GFX9-NEXT:  v_mov_b32_e32 v26, v0
+; GFX9-NEXT:  v_mov_b32_e32 v27, v0
+; GFX9-NEXT:  v_mov_b32_e32 v28, v0
+; GFX9-NEXT:  v_mov_b32_e32 v29, v0
+; GFX9-NEXT:  v_mov_b32_e32 v30, v0
+; GFX9-NEXT:  v_mov_b32_e32 v31, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v32i32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off offset:4
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  v_mov_b32_e32 v8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v9, v0
+; GFX10-NEXT:  v_mov_b32_e32 v10, v0
+; GFX10-NEXT:  v_mov_b32_e32 v11, v0
+; GFX10-NEXT:  v_mov_b32_e32 v12, v0
+; GFX10-NEXT:  v_mov_b32_e32 v13, v0
+; GFX10-NEXT:  v_mov_b32_e32 v14, v0
+; GFX10-NEXT:  v_mov_b32_e32 v15, v0
+; GFX10-NEXT:  v_mov_b32_e32 v16, v0
+; GFX10-NEXT:  v_mov_b32_e32 v17, v0
+; GFX10-NEXT:  v_mov_b32_e32 v18, v0
+; GFX10-NEXT:  v_mov_b32_e32 v19, v0
+; GFX10-NEXT:  v_mov_b32_e32 v20, v0
+; GFX10-NEXT:  v_mov_b32_e32 v21, v0
+; GFX10-NEXT:  v_mov_b32_e32 v22, v0
+; GFX10-NEXT:  v_mov_b32_e32 v23, v0
+; GFX10-NEXT:  v_mov_b32_e32 v24, v0
+; GFX10-NEXT:  v_mov_b32_e32 v25, v0
+; GFX10-NEXT:  v_mov_b32_e32 v26, v0
+; GFX10-NEXT:  v_mov_b32_e32 v27, v0
+; GFX10-NEXT:  v_mov_b32_e32 v28, v0
+; GFX10-NEXT:  v_mov_b32_e32 v29, v0
+; GFX10-NEXT:  v_mov_b32_e32 v30, v0
+; GFX10-NEXT:  v_mov_b32_e32 v31, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v32i32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  v_mov_b32_e32 v8, v0
+; GFX11-NEXT:  v_mov_b32_e32 v9, v0
+; GFX11-NEXT:  v_mov_b32_e32 v10, v0
+; GFX11-NEXT:  v_mov_b32_e32 v11, v0
+; GFX11-NEXT:  v_mov_b32_e32 v12, v0
+; GFX11-NEXT:  v_mov_b32_e32 v13, v0
+; GFX11-NEXT:  v_mov_b32_e32 v14, v0
+; GFX11-NEXT:  v_mov_b32_e32 v15, v0
+; GFX11-NEXT:  v_mov_b32_e32 v16, v0
+; GFX11-NEXT:  v_mov_b32_e32 v17, v0
+; GFX11-NEXT:  v_mov_b32_e32 v18, v0
+; GFX11-NEXT:  v_mov_b32_e32 v19, v0
+; GFX11-NEXT:  v_mov_b32_e32 v20, v0
+; GFX11-NEXT:  v_mov_b32_e32 v21, v0
+; GFX11-NEXT:  v_mov_b32_e32 v22, v0
+; GFX11-NEXT:  v_mov_b32_e32 v23, v0
+; GFX11-NEXT:  v_mov_b32_e32 v24, v0
+; GFX11-NEXT:  v_mov_b32_e32 v25, v0
+; GFX11-NEXT:  v_mov_b32_e32 v26, v0
+; GFX11-NEXT:  v_mov_b32_e32 v27, v0
+; GFX11-NEXT:  v_mov_b32_e32 v28, v0
+; GFX11-NEXT:  v_mov_b32_e32 v29, v0
+; GFX11-NEXT:  v_mov_b32_e32 v30, v0
+; GFX11-NEXT:  v_mov_b32_e32 v31, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <32 x i32>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <32 x i32> %val0, <32 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <32 x i32> %val1
+}
+
+define <2 x bfloat> @shuffle_v2bf16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v2bf16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v2bf16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v2bf16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <2 x bfloat>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <2 x bfloat> %val0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+  ret <2 x bfloat> %val1
+}
+
+define <3 x bfloat> @shuffle_v3bf16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v3bf16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v1, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v1, v1, s4
+; GFX9-NEXT:  v_alignbit_b32 v1, s4, v1, 16
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v3bf16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v1, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v1, v1, 0x7060302
+; GFX10-NEXT:  v_alignbit_b32 v1, s4, v1, 16
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v3bf16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v1, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v1, v1, 0x7060302
+; GFX11-NEXT:  v_alignbit_b32 v1, s0, v1, 16
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <3 x bfloat>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <3 x bfloat> %val0, <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  ret <3 x bfloat> %val1
+}
+
+define <4 x bfloat> @shuffle_v4bf16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v4bf16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v4bf16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v4bf16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <4 x bfloat> %val0, <4 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x bfloat> %val1
+}
+
+define <6 x bfloat> @shuffle_v6bf16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v6bf16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v6bf16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v6bf16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <6 x bfloat>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <6 x bfloat> %val0, <6 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <6 x bfloat> %val1
+}
+
+define <8 x bfloat> @shuffle_v8bf16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v8bf16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v8bf16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v8bf16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <8 x bfloat>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <8 x bfloat> %val0, <8 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <8 x bfloat> %val1
+}
+
+define <16 x bfloat> @shuffle_v16bf16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v16bf16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v16bf16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v16bf16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <16 x bfloat>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <16 x bfloat> %val0, <16 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <16 x bfloat> %val1
+}
+
+define <32 x bfloat> @shuffle_v32bf16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v32bf16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  v_mov_b32_e32 v8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v9, v0
+; GFX9-NEXT:  v_mov_b32_e32 v10, v0
+; GFX9-NEXT:  v_mov_b32_e32 v11, v0
+; GFX9-NEXT:  v_mov_b32_e32 v12, v0
+; GFX9-NEXT:  v_mov_b32_e32 v13, v0
+; GFX9-NEXT:  v_mov_b32_e32 v14, v0
+; GFX9-NEXT:  v_mov_b32_e32 v15, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v32bf16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  v_mov_b32_e32 v8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v9, v0
+; GFX10-NEXT:  v_mov_b32_e32 v10, v0
+; GFX10-NEXT:  v_mov_b32_e32 v11, v0
+; GFX10-NEXT:  v_mov_b32_e32 v12, v0
+; GFX10-NEXT:  v_mov_b32_e32 v13, v0
+; GFX10-NEXT:  v_mov_b32_e32 v14, v0
+; GFX10-NEXT:  v_mov_b32_e32 v15, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v32bf16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  v_mov_b32_e32 v8, v0
+; GFX11-NEXT:  v_mov_b32_e32 v9, v0
+; GFX11-NEXT:  v_mov_b32_e32 v10, v0
+; GFX11-NEXT:  v_mov_b32_e32 v11, v0
+; GFX11-NEXT:  v_mov_b32_e32 v12, v0
+; GFX11-NEXT:  v_mov_b32_e32 v13, v0
+; GFX11-NEXT:  v_mov_b32_e32 v14, v0
+; GFX11-NEXT:  v_mov_b32_e32 v15, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <32 x bfloat>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <32 x bfloat> %val0, <32 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <32 x bfloat> %val1
+}
+
+define <2 x half> @shuffle_v2f16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v2f16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v2f16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v2f16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <2 x half>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <2 x half> %val0, <2 x half> poison, <2 x i32> <i32 1, i32 1>
+  ret <2 x half> %val1
+}
+
+define <3 x half> @shuffle_v3f16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v3f16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v1, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v1, v1, s4
+; GFX9-NEXT:  v_alignbit_b32 v1, s4, v1, 16
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v3f16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v1, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v1, v1, 0x7060302
+; GFX10-NEXT:  v_alignbit_b32 v1, s4, v1, 16
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v3f16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v1, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v1, v1, 0x7060302
+; GFX11-NEXT:  v_alignbit_b32 v1, s0, v1, 16
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <3 x half>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <3 x half> %val0, <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  ret <3 x half> %val1
+}
+
+define <4 x half> @shuffle_v4f16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v4f16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v4f16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v4f16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <4 x half>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <4 x half> %val0, <4 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x half> %val1
+}
+
+define <6 x half> @shuffle_v6f16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v6f16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v6f16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v6f16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <6 x half>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <6 x half> %val0, <6 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <6 x half> %val1
+}
+
+define <8 x half> @shuffle_v8f16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v8f16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v8f16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v8f16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <8 x half>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <8 x half> %val0, <8 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <8 x half> %val1
+}
+
+define <16 x half> @shuffle_v16f16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v16f16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v16f16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v16f16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <16 x half>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <16 x half> %val0, <16 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <16 x half> %val1
+}
+
+define <32 x half> @shuffle_v32f16_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v32f16_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dword v0, v[0:1], off
+; GFX9-NEXT:  s_mov_b32 s4, 0x7060302
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:  v_mov_b32_e32 v1, v0
+; GFX9-NEXT:  v_mov_b32_e32 v2, v0
+; GFX9-NEXT:  v_mov_b32_e32 v3, v0
+; GFX9-NEXT:  v_mov_b32_e32 v4, v0
+; GFX9-NEXT:  v_mov_b32_e32 v5, v0
+; GFX9-NEXT:  v_mov_b32_e32 v6, v0
+; GFX9-NEXT:  v_mov_b32_e32 v7, v0
+; GFX9-NEXT:  v_mov_b32_e32 v8, v0
+; GFX9-NEXT:  v_mov_b32_e32 v9, v0
+; GFX9-NEXT:  v_mov_b32_e32 v10, v0
+; GFX9-NEXT:  v_mov_b32_e32 v11, v0
+; GFX9-NEXT:  v_mov_b32_e32 v12, v0
+; GFX9-NEXT:  v_mov_b32_e32 v13, v0
+; GFX9-NEXT:  v_mov_b32_e32 v14, v0
+; GFX9-NEXT:  v_mov_b32_e32 v15, v0
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v32f16_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dword v0, v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX10-NEXT:  v_mov_b32_e32 v1, v0
+; GFX10-NEXT:  v_mov_b32_e32 v2, v0
+; GFX10-NEXT:  v_mov_b32_e32 v3, v0
+; GFX10-NEXT:  v_mov_b32_e32 v4, v0
+; GFX10-NEXT:  v_mov_b32_e32 v5, v0
+; GFX10-NEXT:  v_mov_b32_e32 v6, v0
+; GFX10-NEXT:  v_mov_b32_e32 v7, v0
+; GFX10-NEXT:  v_mov_b32_e32 v8, v0
+; GFX10-NEXT:  v_mov_b32_e32 v9, v0
+; GFX10-NEXT:  v_mov_b32_e32 v10, v0
+; GFX10-NEXT:  v_mov_b32_e32 v11, v0
+; GFX10-NEXT:  v_mov_b32_e32 v12, v0
+; GFX10-NEXT:  v_mov_b32_e32 v13, v0
+; GFX10-NEXT:  v_mov_b32_e32 v14, v0
+; GFX10-NEXT:  v_mov_b32_e32 v15, v0
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v32f16_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b32 v0, v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_perm_b32 v0, v0, v0, 0x7060302
+; GFX11-NEXT:  s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:  v_mov_b32_e32 v1, v0
+; GFX11-NEXT:  v_mov_b32_e32 v2, v0
+; GFX11-NEXT:  v_mov_b32_e32 v3, v0
+; GFX11-NEXT:  v_mov_b32_e32 v4, v0
+; GFX11-NEXT:  v_mov_b32_e32 v5, v0
+; GFX11-NEXT:  v_mov_b32_e32 v6, v0
+; GFX11-NEXT:  v_mov_b32_e32 v7, v0
+; GFX11-NEXT:  v_mov_b32_e32 v8, v0
+; GFX11-NEXT:  v_mov_b32_e32 v9, v0
+; GFX11-NEXT:  v_mov_b32_e32 v10, v0
+; GFX11-NEXT:  v_mov_b32_e32 v11, v0
+; GFX11-NEXT:  v_mov_b32_e32 v12, v0
+; GFX11-NEXT:  v_mov_b32_e32 v13, v0
+; GFX11-NEXT:  v_mov_b32_e32 v14, v0
+; GFX11-NEXT:  v_mov_b32_e32 v15, v0
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <32 x half>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <32 x half> %val0, <32 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <32 x half> %val1
+}
+
+define <2 x float> @shuffle_v2f32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v2f32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v0, v1
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v2f32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v0, v1
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v2f32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b64 v[0:1], v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v0, v1
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <2 x float>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <2 x float> %val0, <2 x float> poison, <2 x i32> <i32 1, i32 1>
+  ret <2 x float> %val1
+}
+
+define <3 x float> @shuffle_v3f32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v3f32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dwordx3 v[0:2], v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v0, v1
+; GFX9-NEXT:  v_mov_b32_e32 v2, v1
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v3f32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dwordx3 v[0:2], v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v0, v1
+; GFX10-NEXT:  v_mov_b32_e32 v2, v1
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v3f32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b96 v[0:2], v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v0, v1
+; GFX11-NEXT:  v_mov_b32_e32 v2, v1
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <3 x float>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <3 x float> %val0, <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  ret <3 x float> %val1
+}
+
+define <4 x float> @shuffle_v4f32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v4f32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v0, v1
+; GFX9-NEXT:  v_mov_b32_e32 v2, v1
+; GFX9-NEXT:  v_mov_b32_e32 v3, v1
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v4f32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dwordx4 v[0:3], v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v0, v1
+; GFX10-NEXT:  v_mov_b32_e32 v2, v1
+; GFX10-NEXT:  v_mov_b32_e32 v3, v1
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v4f32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b128 v[0:3], v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v0, v1
+; GFX11-NEXT:  v_mov_b32_e32 v2, v1
+; GFX11-NEXT:  v_mov_b32_e32 v3, v1
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <4 x float>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <4 x float> %val0, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x float> %val1
+}
+
+define <6 x float> @shuffle_v6f32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v6f32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v0, v1
+; GFX9-NEXT:  v_mov_b32_e32 v2, v1
+; GFX9-NEXT:  v_mov_b32_e32 v3, v1
+; GFX9-NEXT:  v_mov_b32_e32 v4, v1
+; GFX9-NEXT:  v_mov_b32_e32 v5, v1
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v6f32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dwordx4 v[0:3], v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v0, v1
+; GFX10-NEXT:  v_mov_b32_e32 v2, v1
+; GFX10-NEXT:  v_mov_b32_e32 v3, v1
+; GFX10-NEXT:  v_mov_b32_e32 v4, v1
+; GFX10-NEXT:  v_mov_b32_e32 v5, v1
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v6f32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b128 v[0:3], v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v0, v1
+; GFX11-NEXT:  v_mov_b32_e32 v2, v1
+; GFX11-NEXT:  v_mov_b32_e32 v3, v1
+; GFX11-NEXT:  v_mov_b32_e32 v4, v1
+; GFX11-NEXT:  v_mov_b32_e32 v5, v1
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <6 x float>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <6 x float> %val0, <6 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <6 x float> %val1
+}
+
+define <8 x float> @shuffle_v8f32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v8f32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v0, v1
+; GFX9-NEXT:  v_mov_b32_e32 v2, v1
+; GFX9-NEXT:  v_mov_b32_e32 v3, v1
+; GFX9-NEXT:  v_mov_b32_e32 v4, v1
+; GFX9-NEXT:  v_mov_b32_e32 v5, v1
+; GFX9-NEXT:  v_mov_b32_e32 v6, v1
+; GFX9-NEXT:  v_mov_b32_e32 v7, v1
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v8f32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dwordx4 v[0:3], v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v0, v1
+; GFX10-NEXT:  v_mov_b32_e32 v2, v1
+; GFX10-NEXT:  v_mov_b32_e32 v3, v1
+; GFX10-NEXT:  v_mov_b32_e32 v4, v1
+; GFX10-NEXT:  v_mov_b32_e32 v5, v1
+; GFX10-NEXT:  v_mov_b32_e32 v6, v1
+; GFX10-NEXT:  v_mov_b32_e32 v7, v1
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v8f32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b128 v[0:3], v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v0, v1
+; GFX11-NEXT:  v_mov_b32_e32 v2, v1
+; GFX11-NEXT:  v_mov_b32_e32 v3, v1
+; GFX11-NEXT:  v_mov_b32_e32 v4, v1
+; GFX11-NEXT:  v_mov_b32_e32 v5, v1
+; GFX11-NEXT:  v_mov_b32_e32 v6, v1
+; GFX11-NEXT:  v_mov_b32_e32 v7, v1
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <8 x float>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <8 x float> %val0, <8 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <8 x float> %val1
+}
+
+define <16 x float> @shuffle_v16f32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v16f32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v0, v1
+; GFX9-NEXT:  v_mov_b32_e32 v2, v1
+; GFX9-NEXT:  v_mov_b32_e32 v3, v1
+; GFX9-NEXT:  v_mov_b32_e32 v4, v1
+; GFX9-NEXT:  v_mov_b32_e32 v5, v1
+; GFX9-NEXT:  v_mov_b32_e32 v6, v1
+; GFX9-NEXT:  v_mov_b32_e32 v7, v1
+; GFX9-NEXT:  v_mov_b32_e32 v8, v1
+; GFX9-NEXT:  v_mov_b32_e32 v9, v1
+; GFX9-NEXT:  v_mov_b32_e32 v10, v1
+; GFX9-NEXT:  v_mov_b32_e32 v11, v1
+; GFX9-NEXT:  v_mov_b32_e32 v12, v1
+; GFX9-NEXT:  v_mov_b32_e32 v13, v1
+; GFX9-NEXT:  v_mov_b32_e32 v14, v1
+; GFX9-NEXT:  v_mov_b32_e32 v15, v1
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v16f32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dwordx4 v[0:3], v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v0, v1
+; GFX10-NEXT:  v_mov_b32_e32 v2, v1
+; GFX10-NEXT:  v_mov_b32_e32 v3, v1
+; GFX10-NEXT:  v_mov_b32_e32 v4, v1
+; GFX10-NEXT:  v_mov_b32_e32 v5, v1
+; GFX10-NEXT:  v_mov_b32_e32 v6, v1
+; GFX10-NEXT:  v_mov_b32_e32 v7, v1
+; GFX10-NEXT:  v_mov_b32_e32 v8, v1
+; GFX10-NEXT:  v_mov_b32_e32 v9, v1
+; GFX10-NEXT:  v_mov_b32_e32 v10, v1
+; GFX10-NEXT:  v_mov_b32_e32 v11, v1
+; GFX10-NEXT:  v_mov_b32_e32 v12, v1
+; GFX10-NEXT:  v_mov_b32_e32 v13, v1
+; GFX10-NEXT:  v_mov_b32_e32 v14, v1
+; GFX10-NEXT:  v_mov_b32_e32 v15, v1
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v16f32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b128 v[0:3], v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v0, v1
+; GFX11-NEXT:  v_mov_b32_e32 v2, v1
+; GFX11-NEXT:  v_mov_b32_e32 v3, v1
+; GFX11-NEXT:  v_mov_b32_e32 v4, v1
+; GFX11-NEXT:  v_mov_b32_e32 v5, v1
+; GFX11-NEXT:  v_mov_b32_e32 v6, v1
+; GFX11-NEXT:  v_mov_b32_e32 v7, v1
+; GFX11-NEXT:  v_mov_b32_e32 v8, v1
+; GFX11-NEXT:  v_mov_b32_e32 v9, v1
+; GFX11-NEXT:  v_mov_b32_e32 v10, v1
+; GFX11-NEXT:  v_mov_b32_e32 v11, v1
+; GFX11-NEXT:  v_mov_b32_e32 v12, v1
+; GFX11-NEXT:  v_mov_b32_e32 v13, v1
+; GFX11-NEXT:  v_mov_b32_e32 v14, v1
+; GFX11-NEXT:  v_mov_b32_e32 v15, v1
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <16 x float>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <16 x float> %val0, <16 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <16 x float> %val1
+}
+
+define <32 x float> @shuffle_v32f32_rebroadcast(ptr addrspace(1) %arg0) {
+; GFX9-LABEL: shuffle_v32f32_rebroadcast:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:  global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:  s_waitcnt vmcnt(0)
+; GFX9-NEXT:  v_mov_b32_e32 v0, v1
+; GFX9-NEXT:  v_mov_b32_e32 v2, v1
+; GFX9-NEXT:  v_mov_b32_e32 v3, v1
+; GFX9-NEXT:  v_mov_b32_e32 v4, v1
+; GFX9-NEXT:  v_mov_b32_e32 v5, v1
+; GFX9-NEXT:  v_mov_b32_e32 v6, v1
+; GFX9-NEXT:  v_mov_b32_e32 v7, v1
+; GFX9-NEXT:  v_mov_b32_e32 v8, v1
+; GFX9-NEXT:  v_mov_b32_e32 v9, v1
+; GFX9-NEXT:  v_mov_b32_e32 v10, v1
+; GFX9-NEXT:  v_mov_b32_e32 v11, v1
+; GFX9-NEXT:  v_mov_b32_e32 v12, v1
+; GFX9-NEXT:  v_mov_b32_e32 v13, v1
+; GFX9-NEXT:  v_mov_b32_e32 v14, v1
+; GFX9-NEXT:  v_mov_b32_e32 v15, v1
+; GFX9-NEXT:  v_mov_b32_e32 v16, v1
+; GFX9-NEXT:  v_mov_b32_e32 v17, v1
+; GFX9-NEXT:  v_mov_b32_e32 v18, v1
+; GFX9-NEXT:  v_mov_b32_e32 v19, v1
+; GFX9-NEXT:  v_mov_b32_e32 v20, v1
+; GFX9-NEXT:  v_mov_b32_e32 v21, v1
+; GFX9-NEXT:  v_mov_b32_e32 v22, v1
+; GFX9-NEXT:  v_mov_b32_e32 v23, v1
+; GFX9-NEXT:  v_mov_b32_e32 v24, v1
+; GFX9-NEXT:  v_mov_b32_e32 v25, v1
+; GFX9-NEXT:  v_mov_b32_e32 v26, v1
+; GFX9-NEXT:  v_mov_b32_e32 v27, v1
+; GFX9-NEXT:  v_mov_b32_e32 v28, v1
+; GFX9-NEXT:  v_mov_b32_e32 v29, v1
+; GFX9-NEXT:  v_mov_b32_e32 v30, v1
+; GFX9-NEXT:  v_mov_b32_e32 v31, v1
+; GFX9-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: shuffle_v32f32_rebroadcast:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:  global_load_dwordx4 v[0:3], v[0:1], off
+; GFX10-NEXT:  s_waitcnt vmcnt(0)
+; GFX10-NEXT:  v_mov_b32_e32 v0, v1
+; GFX10-NEXT:  v_mov_b32_e32 v2, v1
+; GFX10-NEXT:  v_mov_b32_e32 v3, v1
+; GFX10-NEXT:  v_mov_b32_e32 v4, v1
+; GFX10-NEXT:  v_mov_b32_e32 v5, v1
+; GFX10-NEXT:  v_mov_b32_e32 v6, v1
+; GFX10-NEXT:  v_mov_b32_e32 v7, v1
+; GFX10-NEXT:  v_mov_b32_e32 v8, v1
+; GFX10-NEXT:  v_mov_b32_e32 v9, v1
+; GFX10-NEXT:  v_mov_b32_e32 v10, v1
+; GFX10-NEXT:  v_mov_b32_e32 v11, v1
+; GFX10-NEXT:  v_mov_b32_e32 v12, v1
+; GFX10-NEXT:  v_mov_b32_e32 v13, v1
+; GFX10-NEXT:  v_mov_b32_e32 v14, v1
+; GFX10-NEXT:  v_mov_b32_e32 v15, v1
+; GFX10-NEXT:  v_mov_b32_e32 v16, v1
+; GFX10-NEXT:  v_mov_b32_e32 v17, v1
+; GFX10-NEXT:  v_mov_b32_e32 v18, v1
+; GFX10-NEXT:  v_mov_b32_e32 v19, v1
+; GFX10-NEXT:  v_mov_b32_e32 v20, v1
+; GFX10-NEXT:  v_mov_b32_e32 v21, v1
+; GFX10-NEXT:  v_mov_b32_e32 v22, v1
+; GFX10-NEXT:  v_mov_b32_e32 v23, v1
+; GFX10-NEXT:  v_mov_b32_e32 v24, v1
+; GFX10-NEXT:  v_mov_b32_e32 v25, v1
+; GFX10-NEXT:  v_mov_b32_e32 v26, v1
+; GFX10-NEXT:  v_mov_b32_e32 v27, v1
+; GFX10-NEXT:  v_mov_b32_e32 v28, v1
+; GFX10-NEXT:  v_mov_b32_e32 v29, v1
+; GFX10-NEXT:  v_mov_b32_e32 v30, v1
+; GFX10-NEXT:  v_mov_b32_e32 v31, v1
+; GFX10-NEXT:  s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: shuffle_v32f32_rebroadcast:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:  s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:  global_load_b128 v[0:3], v[0:1], off
+; GFX11-NEXT:  s_waitcnt vmcnt(0)
+; GFX11-NEXT:  v_mov_b32_e32 v0, v1
+; GFX11-NEXT:  v_mov_b32_e32 v2, v1
+; GFX11-NEXT:  v_mov_b32_e32 v3, v1
+; GFX11-NEXT:  v_mov_b32_e32 v4, v1
+; GFX11-NEXT:  v_mov_b32_e32 v5, v1
+; GFX11-NEXT:  v_mov_b32_e32 v6, v1
+; GFX11-NEXT:  v_mov_b32_e32 v7, v1
+; GFX11-NEXT:  v_mov_b32_e32 v8, v1
+; GFX11-NEXT:  v_mov_b32_e32 v9, v1
+; GFX11-NEXT:  v_mov_b32_e32 v10, v1
+; GFX11-NEXT:  v_mov_b32_e32 v11, v1
+; GFX11-NEXT:  v_mov_b32_e32 v12, v1
+; GFX11-NEXT:  v_mov_b32_e32 v13, v1
+; GFX11-NEXT:  v_mov_b32_e32 v14, v1
+; GFX11-NEXT:  v_mov_b32_e32 v15, v1
+; GFX11-NEXT:  v_mov_b32_e32 v16, v1
+; GFX11-NEXT:  v_mov_b32_e32 v17, v1
+; GFX11-NEXT:  v_mov_b32_e32 v18, v1
+; GFX11-NEXT:  v_mov_b32_e32 v19, v1
+; GFX11-NEXT:  v_mov_b32_e32 v20, v1
+; GFX11-NEXT:  v_mov_b32_e32 v21, v1
+; GFX11-NEXT:  v_mov_b32_e32 v22, v1
+; GFX11-NEXT:  v_mov_b32_e32 v23, v1
+; GFX11-NEXT:  v_mov_b32_e32 v24, v1
+; GFX11-NEXT:  v_mov_b32_e32 v25, v1
+; GFX11-NEXT:  v_mov_b32_e32 v26, v1
+; GFX11-NEXT:  v_mov_b32_e32 v27, v1
+; GFX11-NEXT:  v_mov_b32_e32 v28, v1
+; GFX11-NEXT:  v_mov_b32_e32 v29, v1
+; GFX11-NEXT:  v_mov_b32_e32 v30, v1
+; GFX11-NEXT:  v_mov_b32_e32 v31, v1
+; GFX11-NEXT:  s_setpc_b64 s[30:31]
+entry:
+  %val0 = load <32 x float>, ptr addrspace(1) %arg0
+  %val1 = shufflevector <32 x float> %val0, <32 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  ret <32 x float> %val1
+}
-- 
GitLab


From 69937982dbdd73172ec06580f6f93616edca8e9e Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Mon, 13 May 2024 20:41:42 +0200
Subject: [PATCH 0620/1206] [clang-tidy] Ignore unevaluated context in
 bugprone-optional-value-conversion (#90410)

Ignore optionals in unevaluated context, like static_assert or decltype.

Closes #89593
---
 .../clang-tidy/bugprone/OptionalValueConversionCheck.cpp      | 4 +++-
 clang-tools-extra/docs/ReleaseNotes.rst                       | 4 ++++
 .../checkers/bugprone/optional-value-conversion.cpp           | 2 ++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp
index 9ab59e6b0474..600eab375527 100644
--- a/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp
@@ -71,7 +71,9 @@ void OptionalValueConversionCheck::registerMatchers(MatchFinder *Finder) {
               ofClass(matchers::matchesAnyListedName(OptionalTypes)))),
           hasType(ConstructTypeMatcher),
           hasArgument(0U, ignoringImpCasts(anyOf(OptionalDereferenceMatcher,
-                                                 StdMoveCallMatcher))))
+                                                 StdMoveCallMatcher))),
+          unless(anyOf(hasAncestor(typeLoc()),
+                       hasAncestor(expr(matchers::hasUnevaluatedContext())))))
           .bind("expr"),
       this);
 }
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 8183d394cf42..8c76f5f60ee3 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -204,6 +204,10 @@ Changes in existing checks
   eliminating false positives resulting from direct usage of bitwise operators
   within parentheses.
 
+- Improved :doc:`bugprone-optional-value-conversion
+  <clang-tidy/checks/bugprone/optional-value-conversion>` check by eliminating
+  false positives resulting from use of optionals in unevaluated context.
+
 - Improved :doc:`bugprone-suspicious-include
   <clang-tidy/checks/bugprone/suspicious-include>` check by replacing the local
   options `HeaderFileExtensions` and `ImplementationFileExtensions` by the
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/optional-value-conversion.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/optional-value-conversion.cpp
index 72ef35c956d2..1228d64bb690 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/optional-value-conversion.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/optional-value-conversion.cpp
@@ -210,4 +210,6 @@ void correct(std::optional<int> param)
   std::optional<long>* p2 = &p;
   takeOptionalValue(p2->value_or(5U));
   takeOptionalRef(p2->value_or(5U));
+
+  using Type = decltype(takeOptionalValue(*param));
 }
-- 
GitLab


From af79372d6349cfba6beff26d54b7ad1b798fc4d5 Mon Sep 17 00:00:00 2001
From: Mike Crowe <mac@mcrowe.com>
Date: Mon, 13 May 2024 19:42:44 +0100
Subject: [PATCH 0621/1206] [clang-tidy] Add modernize-use-std-format check
 (#90397)

Add a new clang-tidy check that converts absl::StrFormat (and similar
functions) to std::format (and similar functions.)

Split the configuration of FormatStringConverter out to a separate
Configuration class so that we don't risk confusion by passing two
boolean configuration parameters into the constructor. Add
AllowTrailingNewlineRemoval option since we never want to remove
trailing newlines in this check.
---
 .../clang-tidy/modernize/CMakeLists.txt       |   1 +
 .../modernize/ModernizeTidyModule.cpp         |   2 +
 .../modernize/UseStdFormatCheck.cpp           | 107 ++++++++++++++++
 .../clang-tidy/modernize/UseStdFormatCheck.h  |  51 ++++++++
 .../clang-tidy/modernize/UseStdPrintCheck.cpp |   5 +-
 .../utils/FormatStringConverter.cpp           |  16 ++-
 .../clang-tidy/utils/FormatStringConverter.h  |   9 +-
 clang-tools-extra/docs/ReleaseNotes.rst       |   9 ++
 .../docs/clang-tidy/checks/list.rst           |   1 +
 .../checks/modernize/use-std-format.rst       |  84 ++++++++++++
 .../modernize/use-std-format-custom.cpp       |  52 ++++++++
 .../checkers/modernize/use-std-format-fmt.cpp |  24 ++++
 .../checkers/modernize/use-std-format.cpp     | 120 ++++++++++++++++++
 13 files changed, 473 insertions(+), 8 deletions(-)
 create mode 100644 clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp
 create mode 100644 clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.h
 create mode 100644 clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-custom.cpp
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-fmt.cpp
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format.cpp

diff --git a/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt b/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
index 8005d6e91c06..576805c4c7f1 100644
--- a/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
@@ -41,6 +41,7 @@ add_clang_library(clangTidyModernizeModule
   UseNullptrCheck.cpp
   UseOverrideCheck.cpp
   UseStartsEndsWithCheck.cpp
+  UseStdFormatCheck.cpp
   UseStdNumbersCheck.cpp
   UseStdPrintCheck.cpp
   UseTrailingReturnTypeCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp b/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
index 776558433c5b..b9c7a2dc383e 100644
--- a/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
@@ -42,6 +42,7 @@
 #include "UseNullptrCheck.h"
 #include "UseOverrideCheck.h"
 #include "UseStartsEndsWithCheck.h"
+#include "UseStdFormatCheck.h"
 #include "UseStdNumbersCheck.h"
 #include "UseStdPrintCheck.h"
 #include "UseTrailingReturnTypeCheck.h"
@@ -76,6 +77,7 @@ public:
         "modernize-use-designated-initializers");
     CheckFactories.registerCheck<UseStartsEndsWithCheck>(
         "modernize-use-starts-ends-with");
+    CheckFactories.registerCheck<UseStdFormatCheck>("modernize-use-std-format");
     CheckFactories.registerCheck<UseStdNumbersCheck>(
         "modernize-use-std-numbers");
     CheckFactories.registerCheck<UseStdPrintCheck>("modernize-use-std-print");
diff --git a/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp
new file mode 100644
index 000000000000..6cef21f1318a
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp
@@ -0,0 +1,107 @@
+//===--- UseStdFormatCheck.cpp - clang-tidy -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UseStdFormatCheck.h"
+#include "../utils/FormatStringConverter.h"
+#include "../utils/Matchers.h"
+#include "../utils/OptionsUtils.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Tooling/FixIt.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang::tidy::modernize {
+
+namespace {
+AST_MATCHER(StringLiteral, isOrdinary) { return Node.isOrdinary(); }
+} // namespace
+
+UseStdFormatCheck::UseStdFormatCheck(StringRef Name, ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      StrictMode(Options.getLocalOrGlobal("StrictMode", false)),
+      StrFormatLikeFunctions(utils::options::parseStringList(
+          Options.get("StrFormatLikeFunctions", ""))),
+      ReplacementFormatFunction(
+          Options.get("ReplacementFormatFunction", "std::format")),
+      IncludeInserter(Options.getLocalOrGlobal("IncludeStyle",
+                                               utils::IncludeSorter::IS_LLVM),
+                      areDiagsSelfContained()),
+      MaybeHeaderToInclude(Options.get("FormatHeader")) {
+  if (StrFormatLikeFunctions.empty())
+    StrFormatLikeFunctions.push_back("absl::StrFormat");
+
+  if (!MaybeHeaderToInclude && ReplacementFormatFunction == "std::format")
+    MaybeHeaderToInclude = "<format>";
+}
+
+void UseStdFormatCheck::registerPPCallbacks(const SourceManager &SM,
+                                            Preprocessor *PP,
+                                            Preprocessor *ModuleExpanderPP) {
+  IncludeInserter.registerPreprocessor(PP);
+}
+
+void UseStdFormatCheck::registerMatchers(MatchFinder *Finder) {
+  Finder->addMatcher(
+      callExpr(argumentCountAtLeast(1),
+               hasArgument(0, stringLiteral(isOrdinary())),
+               callee(functionDecl(unless(cxxMethodDecl()),
+                                   matchers::matchesAnyListedName(
+                                       StrFormatLikeFunctions))
+                          .bind("func_decl")))
+          .bind("strformat"),
+      this);
+}
+
+void UseStdFormatCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
+  using utils::options::serializeStringList;
+  Options.store(Opts, "StrictMode", StrictMode);
+  Options.store(Opts, "StrFormatLikeFunctions",
+                serializeStringList(StrFormatLikeFunctions));
+  Options.store(Opts, "ReplacementFormatFunction", ReplacementFormatFunction);
+  Options.store(Opts, "IncludeStyle", IncludeInserter.getStyle());
+  if (MaybeHeaderToInclude)
+    Options.store(Opts, "FormatHeader", *MaybeHeaderToInclude);
+}
+
+void UseStdFormatCheck::check(const MatchFinder::MatchResult &Result) {
+  const unsigned FormatArgOffset = 0;
+  const auto *OldFunction = Result.Nodes.getNodeAs<FunctionDecl>("func_decl");
+  const auto *StrFormat = Result.Nodes.getNodeAs<CallExpr>("strformat");
+
+  utils::FormatStringConverter::Configuration ConverterConfig;
+  ConverterConfig.StrictMode = StrictMode;
+  utils::FormatStringConverter Converter(Result.Context, StrFormat,
+                                         FormatArgOffset, ConverterConfig,
+                                         getLangOpts());
+  const Expr *StrFormatCall = StrFormat->getCallee();
+  if (!Converter.canApply()) {
+    diag(StrFormat->getBeginLoc(),
+         "unable to use '%0' instead of %1 because %2")
+        << StrFormatCall->getSourceRange() << ReplacementFormatFunction
+        << OldFunction->getIdentifier()
+        << Converter.conversionNotPossibleReason();
+    return;
+  }
+
+  DiagnosticBuilder Diag =
+      diag(StrFormatCall->getBeginLoc(), "use '%0' instead of %1")
+      << ReplacementFormatFunction << OldFunction->getIdentifier();
+  Diag << FixItHint::CreateReplacement(
+      CharSourceRange::getTokenRange(StrFormatCall->getSourceRange()),
+      ReplacementFormatFunction);
+  Converter.applyFixes(Diag, *Result.SourceManager);
+
+  if (MaybeHeaderToInclude)
+    Diag << IncludeInserter.createIncludeInsertion(
+        Result.Context->getSourceManager().getFileID(
+            StrFormatCall->getBeginLoc()),
+        *MaybeHeaderToInclude);
+}
+
+} // namespace clang::tidy::modernize
diff --git a/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.h b/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.h
new file mode 100644
index 000000000000..b59a4708c6e4
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.h
@@ -0,0 +1,51 @@
+//===--- UseStdFormatCheck.h - clang-tidy -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MODERNIZE_USESTDFORMATCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MODERNIZE_USESTDFORMATCHECK_H
+
+#include "../ClangTidyCheck.h"
+#include "../utils/IncludeInserter.h"
+
+namespace clang::tidy::modernize {
+
+/// Converts calls to absl::StrFormat, or other functions via configuration
+/// options, to C++20's std::format, or another function via a configuration
+/// option, modifying the format string appropriately and removing
+/// now-unnecessary calls to std::string::c_str() and std::string::data().
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/modernize/use-std-format.html
+class UseStdFormatCheck : public ClangTidyCheck {
+public:
+  UseStdFormatCheck(StringRef Name, ClangTidyContext *Context);
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    if (ReplacementFormatFunction == "std::format")
+      return LangOpts.CPlusPlus20;
+    return LangOpts.CPlusPlus;
+  }
+  void registerPPCallbacks(const SourceManager &SM, Preprocessor *PP,
+                           Preprocessor *ModuleExpanderPP) override;
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  std::optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
+
+private:
+  bool StrictMode;
+  std::vector<StringRef> StrFormatLikeFunctions;
+  StringRef ReplacementFormatFunction;
+  utils::IncludeInserter IncludeInserter;
+  std::optional<StringRef> MaybeHeaderToInclude;
+};
+
+} // namespace clang::tidy::modernize
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MODERNIZE_USESTDFORMATCHECK_H
diff --git a/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp
index aa60c904a363..ff990feadc0c 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp
@@ -129,8 +129,11 @@ void UseStdPrintCheck::check(const MatchFinder::MatchResult &Result) {
     FormatArgOffset = 1;
   }
 
+  utils::FormatStringConverter::Configuration ConverterConfig;
+  ConverterConfig.StrictMode = StrictMode;
+  ConverterConfig.AllowTrailingNewlineRemoval = true;
   utils::FormatStringConverter Converter(
-      Result.Context, Printf, FormatArgOffset, StrictMode, getLangOpts());
+      Result.Context, Printf, FormatArgOffset, ConverterConfig, getLangOpts());
   const Expr *PrintfCall = Printf->getCallee();
   const StringRef ReplacementFunction = Converter.usePrintNewlineFunction()
                                             ? ReplacementPrintlnFunction
diff --git a/clang-tools-extra/clang-tidy/utils/FormatStringConverter.cpp b/clang-tools-extra/clang-tidy/utils/FormatStringConverter.cpp
index ad10f745b6ac..845e71c5003b 100644
--- a/clang-tools-extra/clang-tidy/utils/FormatStringConverter.cpp
+++ b/clang-tools-extra/clang-tidy/utils/FormatStringConverter.cpp
@@ -198,10 +198,11 @@ static bool castMismatchedIntegerTypes(const CallExpr *Call, bool StrictMode) {
 FormatStringConverter::FormatStringConverter(ASTContext *ContextIn,
                                              const CallExpr *Call,
                                              unsigned FormatArgOffset,
-                                             bool StrictMode,
+                                             const Configuration ConfigIn,
                                              const LangOptions &LO)
-    : Context(ContextIn),
-      CastMismatchedIntegerTypes(castMismatchedIntegerTypes(Call, StrictMode)),
+    : Context(ContextIn), Config(ConfigIn),
+      CastMismatchedIntegerTypes(
+          castMismatchedIntegerTypes(Call, ConfigIn.StrictMode)),
       Args(Call->getArgs()), NumArgs(Call->getNumArgs()),
       ArgsOffset(FormatArgOffset + 1), LangOpts(LO) {
   assert(ArgsOffset <= NumArgs);
@@ -627,9 +628,12 @@ void FormatStringConverter::finalizeFormatText() {
 
   // It's clearer to convert printf("Hello\r\n"); to std::print("Hello\r\n")
   // than to std::println("Hello\r");
-  if (StringRef(StandardFormatString).ends_with("\\n") &&
-      !StringRef(StandardFormatString).ends_with("\\\\n") &&
-      !StringRef(StandardFormatString).ends_with("\\r\\n")) {
+  // Use StringRef until C++20 std::string::ends_with() is available.
+  const auto StandardFormatStringRef = StringRef(StandardFormatString);
+  if (Config.AllowTrailingNewlineRemoval &&
+      StandardFormatStringRef.ends_with("\\n") &&
+      !StandardFormatStringRef.ends_with("\\\\n") &&
+      !StandardFormatStringRef.ends_with("\\r\\n")) {
     UsePrintNewlineFunction = true;
     FormatStringNeededRewriting = true;
     StandardFormatString.erase(StandardFormatString.end() - 2,
diff --git a/clang-tools-extra/clang-tidy/utils/FormatStringConverter.h b/clang-tools-extra/clang-tidy/utils/FormatStringConverter.h
index 1949870f62ed..1109a0b60226 100644
--- a/clang-tools-extra/clang-tidy/utils/FormatStringConverter.h
+++ b/clang-tools-extra/clang-tidy/utils/FormatStringConverter.h
@@ -32,8 +32,14 @@ class FormatStringConverter
 public:
   using ConversionSpecifier = clang::analyze_format_string::ConversionSpecifier;
   using PrintfSpecifier = analyze_printf::PrintfSpecifier;
+
+  struct Configuration {
+    bool StrictMode = false;
+    bool AllowTrailingNewlineRemoval = false;
+  };
+
   FormatStringConverter(ASTContext *Context, const CallExpr *Call,
-                        unsigned FormatArgOffset, bool StrictMode,
+                        unsigned FormatArgOffset, Configuration Config,
                         const LangOptions &LO);
 
   bool canApply() const { return ConversionNotPossibleReason.empty(); }
@@ -45,6 +51,7 @@ public:
 
 private:
   ASTContext *Context;
+  const Configuration Config;
   const bool CastMismatchedIntegerTypes;
   const Expr *const *Args;
   const unsigned NumArgs;
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 8c76f5f60ee3..898c7acc1310 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -150,6 +150,15 @@ New checks
   Finds initializer lists for aggregate types that could be
   written as designated initializers instead.
 
+- New :doc:`modernize-use-std-format
+  <clang-tidy/checks/modernize/use-std-format>` check.
+
+  Converts calls to ``absl::StrFormat``, or other functions via
+  configuration options, to C++20's ``std::format``, or another function
+  via a configuration option, modifying the format string appropriately and
+  removing now-unnecessary calls to ``std::string::c_str()`` and
+  ``std::string::data()``.
+
 - New :doc:`readability-enum-initial-value
   <clang-tidy/checks/readability/enum-initial-value>` check.
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index 046a5ff57ad1..85e4f0352ac2 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -300,6 +300,7 @@ Clang-Tidy Checks
    :doc:`modernize-use-nullptr <modernize/use-nullptr>`, "Yes"
    :doc:`modernize-use-override <modernize/use-override>`, "Yes"
    :doc:`modernize-use-starts-ends-with <modernize/use-starts-ends-with>`, "Yes"
+   :doc:`modernize-use-std-format <modernize/use-std-format>`, "Yes"
    :doc:`modernize-use-std-numbers <modernize/use-std-numbers>`, "Yes"
    :doc:`modernize-use-std-print <modernize/use-std-print>`, "Yes"
    :doc:`modernize-use-trailing-return-type <modernize/use-trailing-return-type>`, "Yes"
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
new file mode 100644
index 000000000000..a1599f0fc58f
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
@@ -0,0 +1,84 @@
+.. title:: clang-tidy - modernize-use-std-format
+
+modernize-use-std-format
+========================
+
+Converts calls to ``absl::StrFormat``, or other functions via
+configuration options, to C++20's ``std::format``, or another function
+via a configuration option, modifying the format string appropriately and
+removing now-unnecessary calls to ``std::string::c_str()`` and
+``std::string::data()``.
+
+For example, it turns lines like
+
+.. code-block:: c++
+
+  return absl::StrFormat("The %s is %3d", description.c_str(), value);
+
+into:
+
+.. code-block:: c++
+
+  return std::format("The {} is {:3}", description, value);
+
+The check uses the same format-string-conversion algorithm as
+`modernize-use-std-print <../modernize/use-std-print.html>`_ and its
+shortcomings are described in the documentation for that check.
+
+Options
+-------
+
+.. option:: StrictMode
+
+   When `true`, the check will add casts when converting from variadic
+   functions and printing signed or unsigned integer types (including
+   fixed-width integer types from ``<cstdint>``, ``ptrdiff_t``, ``size_t``
+   and ``ssize_t``) as the opposite signedness to ensure that the output
+   would matches that of a simple wrapper for ``std::sprintf`` that
+   accepted a C-style variable argument list. For example, with
+   `StrictMode` enabled,
+
+  .. code-block:: c++
+
+    extern std::string strprintf(const char *format, ...);
+    int i = -42;
+    unsigned int u = 0xffffffff;
+    return strprintf("%d %u\n", i, u);
+
+  would be converted to
+
+  .. code-block:: c++
+
+    return std::format("{} {}\n", static_cast<unsigned int>(i), static_cast<int>(u));
+
+  to ensure that the output will continue to be the unsigned representation
+  of -42 and the signed representation of 0xffffffff (often 4294967254
+  and -1 respectively). When `false` (which is the default), these casts
+  will not be added which may cause a change in the output. Note that this
+  option makes no difference for the default value of
+  `StrFormatLikeFunctions` since ``absl::StrFormat`` takes a function
+  parameter pack and is not a variadic function.
+
+.. option:: StrFormatLikeFunctions
+
+   A semicolon-separated list of (fully qualified) function names to
+   replace, with the requirement that the first parameter contains the
+   printf-style format string and the arguments to be formatted follow
+   immediately afterwards. The default value for this option is
+   `absl::StrFormat`.
+
+.. option:: ReplacementFormatFunction
+
+   The function that will be used to replace the function set by the
+   `StrFormatLikeFunctions` option rather than the default
+   `std::format`. It is expected that the function provides an interface
+   that is compatible with ``std::format``. A suitable candidate would be
+   `fmt::format`.
+
+.. option:: FormatHeader
+
+   The header that must be included for the declaration of
+   `ReplacementFormatFunction` so that a ``#include`` directive can be added if
+   required. If `ReplacementFormatFunction` is `std::format` then this option will
+   default to ``<format>``, otherwise this option will default to nothing
+   and no ``#include`` directive will be added.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-custom.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-custom.cpp
new file mode 100644
index 000000000000..815e22b29155
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-custom.cpp
@@ -0,0 +1,52 @@
+// RUN: %check_clang_tidy -check-suffixes=,STRICT                       \
+// RUN:   -std=c++20 %s modernize-use-std-format %t --                  \
+// RUN:   -config="{CheckOptions: {                                     \
+// RUN:              modernize-use-std-format.StrictMode: true,         \
+// RUN:              modernize-use-std-format.StrFormatLikeFunctions: '::strprintf; mynamespace::strprintf2', \
+// RUN:              modernize-use-std-format.ReplacementFormatFunction: 'fmt::format', \
+// RUN:              modernize-use-std-format.FormatHeader: '<fmt/core.h>' \
+// RUN:            }}"                                                  \
+// RUN:   -- -isystem %clang_tidy_headers
+// RUN: %check_clang_tidy -check-suffixes=,NOTSTRICT                    \
+// RUN:   -std=c++20 %s modernize-use-std-format %t --                  \
+// RUN:   -config="{CheckOptions: {                                     \
+// RUN:              modernize-use-std-format.StrFormatLikeFunctions: '::strprintf; mynamespace::strprintf2', \
+// RUN:              modernize-use-std-format.ReplacementFormatFunction: 'fmt::format', \
+// RUN:              modernize-use-std-format.FormatHeader: '<fmt/core.h>' \
+// RUN:            }}"                                                  \
+// RUN:   -- -isystem %clang_tidy_headers
+
+#include <cstdio>
+#include <string>
+// CHECK-FIXES: #include <fmt/core.h>
+
+std::string strprintf(const char *, ...);
+
+namespace mynamespace {
+  std::string strprintf2(const char *, ...);
+}
+
+std::string strprintf_test(const std::string &name, double value) {
+  return strprintf("'%s'='%f'\n", name.c_str(), value);
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'fmt::format' instead of 'strprintf' [modernize-use-std-format]
+  // CHECK-FIXES: return fmt::format("'{}'='{:f}'\n", name, value);
+
+  return mynamespace::strprintf2("'%s'='%f'\n", name.c_str(), value);
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'fmt::format' instead of 'strprintf2' [modernize-use-std-format]
+  // CHECK-FIXES: return fmt::format("'{}'='{:f}'\n", name, value);
+}
+
+std::string StrFormat_strict_conversion() {
+  const unsigned char uc = 'A';
+  return strprintf("Integer %hhd from unsigned char\n", uc);
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'fmt::format' instead of 'strprintf' [modernize-use-std-format]
+  // CHECK-FIXES-NOTSTRICT: return fmt::format("Integer {} from unsigned char\n", uc);
+  // CHECK-FIXES-STRICT: return fmt::format("Integer {} from unsigned char\n", static_cast<signed char>(uc));
+}
+
+// Ensure that MatchesAnyListedNameMatcher::NameMatcher::match() can cope with a
+// NamedDecl that has no name when we're trying to match unqualified_strprintf.
+std::string A(const std::string &in)
+{
+    return "_" + in;
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-fmt.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-fmt.cpp
new file mode 100644
index 000000000000..9d136cf30916
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-fmt.cpp
@@ -0,0 +1,24 @@
+// RUN: %check_clang_tidy %s modernize-use-std-format %t -- \
+// RUN:   -config="{CheckOptions: { \
+// RUN:              StrictMode: true, \
+// RUN:              modernize-use-std-format.StrFormatLikeFunctions: 'fmt::sprintf', \
+// RUN:              modernize-use-std-format.ReplacementFormatFunction: 'fmt::format', \
+// RUN:              modernize-use-std-format.FormatHeader: '<fmt/core.h>' \
+// RUN:            }}" \
+// RUN:   -- -isystem %clang_tidy_headers
+
+// CHECK-FIXES: #include <fmt/core.h>
+#include <string>
+
+namespace fmt
+{
+// Use const char * for the format since the real type is hard to mock up.
+template <typename... Args>
+std::string sprintf(const char *format, const Args&... args);
+} // namespace fmt
+
+std::string fmt_sprintf_simple() {
+  return fmt::sprintf("Hello %s %d", "world", 42);
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'fmt::format' instead of 'sprintf' [modernize-use-std-format]
+  // CHECK-FIXES: fmt::format("Hello {} {}", "world", 42);
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format.cpp
new file mode 100644
index 000000000000..e8dea1dce2c9
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format.cpp
@@ -0,0 +1,120 @@
+// RUN: %check_clang_tidy \
+// RUN:   -std=c++20 %s modernize-use-std-format %t -- \
+// RUN:   -config="{CheckOptions: {StrictMode: true}}" \
+// RUN:   -- -isystem %clang_tidy_headers
+// RUN: %check_clang_tidy \
+// RUN:   -std=c++20 %s modernize-use-std-format %t -- \
+// RUN:   -config="{CheckOptions: {StrictMode: false}}" \
+// RUN:   -- -isystem %clang_tidy_headers
+#include <string>
+// CHECK-FIXES: #include <format>
+
+namespace absl
+{
+// Use const char * for the format since the real type is hard to mock up.
+template <typename... Args>
+std::string StrFormat(const char *format, const Args&... args);
+} // namespace absl
+
+template <typename T>
+struct iterator {
+  T *operator->();
+  T &operator*();
+};
+
+std::string StrFormat_simple() {
+  return absl::StrFormat("Hello");
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: return std::format("Hello");
+}
+
+std::string StrFormat_complex(const char *name, double value) {
+  return absl::StrFormat("'%s'='%f'", name, value);
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: return std::format("'{}'='{:f}'", name, value);
+}
+
+std::string StrFormat_integer_conversions() {
+  return absl::StrFormat("int:%d int:%d char:%c char:%c", 65, 'A', 66, 'B');
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: return std::format("int:{} int:{:d} char:{:c} char:{}", 65, 'A', 66, 'B');
+}
+
+// FormatConverter is capable of removing newlines from the end of the format
+// string. Ensure that isn't incorrectly happening for std::format.
+std::string StrFormat_no_newline_removal() {
+  return absl::StrFormat("a line\n");
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: return std::format("a line\n");
+}
+
+// FormatConverter is capable of removing newlines from the end of the format
+// string. Ensure that isn't incorrectly happening for std::format.
+std::string StrFormat_cstr_removal(const std::string &s1, const std::string *s2) {
+  return absl::StrFormat("%s %s %s %s", s1.c_str(), s1.data(), s2->c_str(), s2->data());
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: return std::format("{} {} {} {}", s1, s1, *s2, *s2);
+}
+
+std::string StrFormat_strict_conversion() {
+  const unsigned char uc = 'A';
+  return absl::StrFormat("Integer %hhd from unsigned char\n", uc);
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: return std::format("Integer {} from unsigned char\n", uc);
+}
+
+std::string StrFormat_field_width_and_precision() {
+  auto s1 = absl::StrFormat("width only:%*d width and precision:%*.*f precision only:%.*f", 3, 42, 4, 2, 3.14159265358979323846, 5, 2.718);
+  // CHECK-MESSAGES: [[@LINE-1]]:13: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: std::format("width only:{:{}} width and precision:{:{}.{}f} precision only:{:.{}f}", 42, 3, 3.14159265358979323846, 4, 2, 2.718, 5);
+
+  auto s2 = absl::StrFormat("width and precision positional:%1$*2$.*3$f after", 3.14159265358979323846, 4, 2);
+  // CHECK-MESSAGES: [[@LINE-1]]:13: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: std::format("width and precision positional:{0:{1}.{2}f} after", 3.14159265358979323846, 4, 2);
+
+  const int width = 10, precision = 3;
+  const unsigned int ui1 = 42, ui2 = 43, ui3 = 44;
+  auto s3 = absl::StrFormat("casts width only:%*d width and precision:%*.*d precision only:%.*d\n", 3, ui1, 4, 2, ui2, 5, ui3);
+  // CHECK-MESSAGES: [[@LINE-1]]:13: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES-NOTSTRICT: std::format("casts width only:{:{}} width and precision:{:{}.{}} precision only:{:.{}}", ui1, 3, ui2, 4, 2, ui3, 5);
+  // CHECK-FIXES-STRICT: std::format("casts width only:{:{}} width and precision:{:{}.{}} precision only:{:.{}}", static_cast<int>(ui1), 3, static_cast<int>(ui2), 4, 2, static_cast<int>(ui3), 5);
+
+  auto s4 = absl::StrFormat("c_str removal width only:%*s width and precision:%*.*s precision only:%.*s", 3, s1.c_str(), 4, 2, s2.c_str(), 5, s3.c_str());
+  // CHECK-MESSAGES: [[@LINE-1]]:13: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: std::format("c_str removal width only:{:>{}} width and precision:{:>{}.{}} precision only:{:.{}}", s1, 3, s2, 4, 2, s3, 5);
+
+  const std::string *ps1 = &s1, *ps2 = &s2, *ps3 = &s3;
+  auto s5 = absl::StrFormat("c_str() removal pointer width only:%-*s width and precision:%-*.*s precision only:%-.*s", 3, ps1->c_str(), 4, 2, ps2->c_str(), 5, ps3->c_str());
+  // CHECK-MESSAGES: [[@LINE-1]]:13: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: std::format("c_str() removal pointer width only:{:{}} width and precision:{:{}.{}} precision only:{:.{}}", *ps1, 3, *ps2, 4, 2, *ps3, 5);
+
+  iterator<std::string> is1, is2, is3;
+  auto s6 = absl::StrFormat("c_str() removal iterator width only:%-*s width and precision:%-*.*s precision only:%-.*s", 3, is1->c_str(), 4, 2, is2->c_str(), 5, is3->c_str());
+  // CHECK-MESSAGES: [[@LINE-1]]:13: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: std::format("c_str() removal iterator width only:{:{}} width and precision:{:{}.{}} precision only:{:.{}}", *is1, 3, *is2, 4, 2, *is3, 5);
+
+  return s1 + s2 + s3 + s4 + s5 + s6;
+}
+
+std::string StrFormat_macros() {
+  // The function call is replaced even though it comes from a macro.
+#define FORMAT absl::StrFormat
+  auto s1 = FORMAT("Hello %d", 42);
+  // CHECK-MESSAGES: [[@LINE-1]]:13: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: std::format("Hello {}", 42);
+
+  // The format string is replaced even though it comes from a macro, this
+  // behaviour is required so that that <inttypes.h> macros are replaced.
+#define FORMAT_STRING "Hello %s"
+  auto s2 = absl::StrFormat(FORMAT_STRING, 42);
+  // CHECK-MESSAGES: [[@LINE-1]]:13: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: std::format("Hello {}", 42);
+
+  // Arguments that are macros aren't replaced with their value, even if they are rearranged.
+#define VALUE 3.14159265358979323846
+#define WIDTH 10
+#define PRECISION 4
+  auto s3 = absl::StrFormat("Hello %*.*f", WIDTH, PRECISION, VALUE);
+  // CHECK-MESSAGES: [[@LINE-1]]:13: warning: use 'std::format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: std::format("Hello {:{}.{}f}", VALUE, WIDTH, PRECISION);
+}
-- 
GitLab


From 5944579ab20cfcb6d1a9d1a2fe3d4b478ea24c64 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 13 May 2024 18:43:29 +0000
Subject: [PATCH 0622/1206] [gn build] Port af79372d6349

---
 .../gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn
index 0d27b786da1f..9b5e157385dd 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn
@@ -50,6 +50,7 @@ static_library("modernize") {
     "UseNullptrCheck.cpp",
     "UseOverrideCheck.cpp",
     "UseStartsEndsWithCheck.cpp",
+    "UseStdFormatCheck.cpp",
     "UseStdNumbersCheck.cpp",
     "UseStdPrintCheck.cpp",
     "UseTrailingReturnTypeCheck.cpp",
-- 
GitLab


From a037d88929460ff9571927c56d6db215be086149 Mon Sep 17 00:00:00 2001
From: Lei Zhang <antiagainst@gmail.com>
Date: Mon, 13 May 2024 15:10:25 -0400
Subject: [PATCH 0623/1206] [mlir][gpu] Support extf before contract when
 converting to MMA ops (#91988)

This commit allows `inferFragType` to see through all arith.ext op
and other elementwise users before reaching contract op for
figuring out the fragment type.
---
 .../Conversion/VectorToGPU/VectorToGPU.cpp    | 15 ++++++++---
 .../VectorToGPU/vector-to-mma-ops.mlir        | 27 +++++++++++++++++++
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
index 782cc92f83fe..332f0a2eecfc 100644
--- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
+++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
@@ -515,6 +515,14 @@ struct CombineTransferReadOpTranspose final
 // TODO: Change the GPU dialect to abstract the layout at the this level and
 // only care about it during lowering to NVVM.
 static const char *inferFragType(Operation *op) {
+  // We can have arith.ext ops before reaching contract ops. See through them
+  // and other kinds of elementwise ops.
+  if (op->hasOneUse()) {
+    Operation *userOp = *op->user_begin();
+    if (userOp->hasTrait<OpTrait::Elementwise>())
+      return inferFragType(userOp);
+  }
+
   for (Operation *users : op->getUsers()) {
     auto contract = dyn_cast<vector::ContractionOp>(users);
     if (!contract)
@@ -560,13 +568,12 @@ convertTransferReadOp(RewriterBase &rewriter, vector::TransferReadOp op,
   if (op->hasOneUse()) {
     auto *user = *op->user_begin();
     // Infer the signedness of the mma type from the integer extend.
-    bool isSignedExtend = isa<arith::ExtSIOp>(user);
-    if (isSignedExtend || isa<arith::ExtUIOp>(user)) {
+    if (isa<arith::ExtSIOp, arith::ExtUIOp>(user)) {
       elType = IntegerType::get(
           op.getContext(), cast<IntegerType>(elType).getWidth(),
-          isSignedExtend ? IntegerType::Signed : IntegerType::Unsigned);
+          isa<arith::ExtSIOp>(user) ? IntegerType::Signed
+                                    : IntegerType::Unsigned);
       mappingResult = user->getResult(0);
-      fragType = inferFragType(user);
     }
   }
   gpu::MMAMatrixType type =
diff --git a/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir b/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir
index 962ed7de584a..8526ff139259 100644
--- a/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir
+++ b/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir
@@ -490,3 +490,30 @@ func.func @fold_transpose_into_transfer_read(%alloc: memref<64x128xf16>, %vector
 }
 
 // -----
+
+#map1 = affine_map<(d0, d1, d2) -> (d0, d2)>
+#map2 = affine_map<(d0, d1, d2) -> (d1, d2)>
+#map3 = affine_map<(d0, d1, d2) -> (d0, d1)>
+
+// CHECK-LABEL: func @cast_f16_to_f32_read
+//       CHECK:    %[[A:.+]] = gpu.subgroup_mma_load_matrix {{.+}} {leadDimension = 16 : index} : memref<16x16xf16> -> !gpu.mma_matrix<16x16xf16, "AOp">
+//       CHECK:    %[[C:.+]] = gpu.subgroup_mma_load_matrix {{.+}} {leadDimension = 16 : index} : memref<16x16xf16> -> !gpu.mma_matrix<16x16xf16, "COp">
+//       CHECK:    %[[AE:.+]] = gpu.subgroup_mma_elementwise  extf %[[A]] : (!gpu.mma_matrix<16x16xf16, "AOp">) -> !gpu.mma_matrix<16x16xf32, "AOp">
+//       CHECK:    %[[CE:.+]] = gpu.subgroup_mma_elementwise  extf %[[C]] : (!gpu.mma_matrix<16x16xf16, "COp">) -> !gpu.mma_matrix<16x16xf32, "COp">
+//       CHECK:    %[[B:.+]] = gpu.subgroup_mma_load_matrix {{.+}} {leadDimension = 16 : index, transpose} : memref<16x16xf16> -> !gpu.mma_matrix<16x16xf16, "BOp">
+//       CHECK:    %[[BE:.+]] = gpu.subgroup_mma_elementwise  extf %[[B]] : (!gpu.mma_matrix<16x16xf16, "BOp">) -> !gpu.mma_matrix<16x16xf32, "BOp">
+//       CHECK:    gpu.subgroup_mma_compute %[[AE]], %[[BE]], %[[CE]]
+func.func @cast_f16_to_f32_read(%arg0: memref<16x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<16x16xf16>, %arg3: memref<16x16xf32>) {
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant 0.000000e+00 : f16
+  %A = vector.transfer_read %arg0[%c0, %c0], %cst {in_bounds = [true, true]} : memref<16x16xf16>, vector<16x16xf16>
+  %B = vector.transfer_read %arg1[%c0, %c0], %cst {in_bounds = [true, true]} : memref<16x16xf16>, vector<16x16xf16>
+  %C = vector.transfer_read %arg2[%c0, %c0], %cst {in_bounds = [true, true]} : memref<16x16xf16>, vector<16x16xf16>
+  %Aext = arith.extf %A : vector<16x16xf16> to vector<16x16xf32>
+  %Bext = arith.extf %B : vector<16x16xf16> to vector<16x16xf32>
+  %Cext = arith.extf %C : vector<16x16xf16> to vector<16x16xf32>
+  %D = vector.contract {indexing_maps = [#map1, #map2, #map3], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>}
+                        %Aext, %Bext, %Cext : vector<16x16xf32>, vector<16x16xf32> into vector<16x16xf32>
+  vector.transfer_write %D, %arg3[%c0, %c0] {in_bounds = [true, true]} : vector<16x16xf32>, memref<16x16xf32>
+  return
+}
-- 
GitLab


From 54b17fa4eefb7ccb00fba8189e92fac2d46cc481 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Sun, 12 May 2024 14:26:41 -0700
Subject: [PATCH 0624/1206] [BOLT] Preserve Offset annotation in fixDoubleJumps
 (#91898)

Offset annotation was missed when optimizing an unconditional branch to
a tail call.

Test Plan: update bb-with-two-tail-calls.s
---
 bolt/lib/Passes/BinaryPasses.cpp       | 3 +++
 bolt/test/X86/bb-with-two-tail-calls.s | 6 +++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp
index df6dbcddeed5..867f977cebca 100644
--- a/bolt/lib/Passes/BinaryPasses.cpp
+++ b/bolt/lib/Passes/BinaryPasses.cpp
@@ -715,6 +715,9 @@ static uint64_t fixDoubleJumps(BinaryFunction &Function, bool MarkInvalid) {
           Pred->removeSuccessor(&BB);
           Pred->eraseInstruction(Pred->findInstruction(Branch));
           Pred->addTailCallInstruction(SuccSym);
+          MCInst *TailCall = Pred->getLastNonPseudoInstr();
+          assert(TailCall);
+          MIB->setOffset(*TailCall, BB.getOffset());
         } else {
           return false;
         }
diff --git a/bolt/test/X86/bb-with-two-tail-calls.s b/bolt/test/X86/bb-with-two-tail-calls.s
index caad7b3d735f..bb2b0cd4cc23 100644
--- a/bolt/test/X86/bb-with-two-tail-calls.s
+++ b/bolt/test/X86/bb-with-two-tail-calls.s
@@ -9,11 +9,11 @@
 # RUN: llvm-strip --strip-unneeded %t.o
 # RUN: %clang %cflags %t.o -o %t.exe -Wl,-q -nostdlib
 # RUN: llvm-bolt %t.exe -o %t.out --data %t.fdata --lite=0 --dyno-stats \
-# RUN:    --print-sctc --print-only=_start 2>&1 | FileCheck %s
+# RUN:    --print-sctc --print-only=_start -enable-bat 2>&1 | FileCheck %s
 # CHECK-NOT: Assertion `BranchInfo.size() == 2 && "could only be called for blocks with 2 successors"' failed.
 # Two tail calls in the same basic block after SCTC:
-# CHECK:         {{.*}}:   ja      {{.*}} # TAILCALL  # CTCTakenCount: {{.*}}
-# CHECK-NEXT:    {{.*}}:   jmp     {{.*}} # TAILCALL
+# CHECK:         {{.*}}:   ja      {{.*}} # TAILCALL # Offset: 7 # CTCTakenCount: 4
+# CHECK-NEXT:    {{.*}}:   jmp     {{.*}} # TAILCALL # Offset: 12
 
   .globl _start
 _start:
-- 
GitLab


From fd4b5f4b52e414bb6fee7c906a22867891fb46b5 Mon Sep 17 00:00:00 2001
From: Alex Langford <alangford@apple.com>
Date: Mon, 13 May 2024 12:32:16 -0700
Subject: [PATCH 0625/1206] [lldb] Add CMake dependency tracking for
 SBLanguages generation script (#91686)

If you change the generation script and re-run ninja (or whatever drives
your build), it currently will not regenerate SBLanguages.h. With
dependency tracking, it should re-run when the script changes.
---
 lldb/source/API/CMakeLists.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lldb/source/API/CMakeLists.txt b/lldb/source/API/CMakeLists.txt
index aa31caddfde3..76b42ecf63f9 100644
--- a/lldb/source/API/CMakeLists.txt
+++ b/lldb/source/API/CMakeLists.txt
@@ -23,14 +23,17 @@ endif()
 # Generate SBLanguages.h from Dwarf.def.
 set(sb_languages_file
   ${CMAKE_CURRENT_BINARY_DIR}/../../include/lldb/API/SBLanguages.h)
+set(sb_languages_generator
+  ${LLDB_SOURCE_DIR}/scripts/generate-sbapi-dwarf-enum.py)
 add_custom_command(
   COMMENT "Generating SBLanguages.h from Dwarf.def"
   COMMAND "${Python3_EXECUTABLE}"
-      ${LLDB_SOURCE_DIR}/scripts/generate-sbapi-dwarf-enum.py
+      ${sb_languages_generator}
       ${LLVM_MAIN_INCLUDE_DIR}/llvm/BinaryFormat/Dwarf.def
       -o ${sb_languages_file}
   OUTPUT ${sb_languages_file}
   DEPENDS ${LLVM_MAIN_INCLUDE_DIR}/llvm/BinaryFormat/Dwarf.def
+          ${sb_languages_generator}
   WORKING_DIRECTORY ${LLVM_LIBRARY_OUTPUT_INTDIR}
 )
 add_custom_target(lldb-sbapi-dwarf-enums
-- 
GitLab


From a1d43c14d8a672730af48d946acc41fa01cf301e Mon Sep 17 00:00:00 2001
From: Benoit Jacob <jacob.benoit.1@gmail.com>
Date: Mon, 13 May 2024 15:36:28 -0400
Subject: [PATCH 0626/1206] [mlir][vector] Add Vector-dialect
 interleave-to-shuffle pattern, enable in VectorToSPIRV (#92012)

This is the second attempt at merging #91800, which bounced due to a
linker error apparently caused by an undeclared dependency.
`MLIRVectorToSPIRV` needed to depend on `MLIRVectorTransforms`. In fact
that was a preexisting issue already flagged by the tool in
https://discourse.llvm.org/t/ninja-can-now-check-for-missing-cmake-dependencies-on-generated-files/74344.

Context: https://github.com/iree-org/iree/issues/17346.

Test IREE integrate showing it's fixing the problem it's intended to
fix, i.e. it allows IREE to drop its local revert of
https://github.com/llvm/llvm-project/pull/89131:

https://github.com/iree-org/iree/pull/17359

This is added to VectorToSPIRV because SPIRV doesn't currently handle
`vector.interleave` (see motivating context above).

This is limited to 1D, non-scalable vectors.
---
 .../Vector/TransformOps/VectorTransformOps.td | 14 +++++++
 .../Vector/Transforms/LoweringPatterns.h      |  3 ++
 .../Conversion/VectorToSPIRV/CMakeLists.txt   |  1 +
 .../VectorToSPIRV/VectorToSPIRV.cpp           |  4 ++
 .../TransformOps/VectorTransformOps.cpp       |  5 +++
 .../Transforms/LowerVectorInterleave.cpp      | 41 +++++++++++++++++++
 .../Vector/vector-interleave-to-shuffle.mlir  | 21 ++++++++++
 .../llvm-project-overlay/mlir/BUILD.bazel     |  1 +
 8 files changed, 90 insertions(+)
 create mode 100644 mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir

diff --git a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
index f6371f39c394..bc3c16d40520 100644
--- a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
+++ b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
@@ -306,6 +306,20 @@ def ApplyLowerInterleavePatternsOp : Op<Transform_Dialect,
   let assemblyFormat = "attr-dict";
 }
 
+def ApplyInterleaveToShufflePatternsOp : Op<Transform_Dialect,
+    "apply_patterns.vector.interleave_to_shuffle",
+    [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
+  let description = [{
+    Indicates that 1D vector interleave operations should be rewritten as
+    vector shuffle operations.
+
+    This is motivated by some current codegen backends not handling vector
+    interleave operations.
+  }];
+
+  let assemblyFormat = "attr-dict";
+}
+
 def ApplyRewriteNarrowTypePatternsOp : Op<Transform_Dialect,
     "apply_patterns.vector.rewrite_narrow_types",
     [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
index 350d2777cadf..8fd9904fabc0 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
@@ -273,6 +273,9 @@ void populateVectorInterleaveLoweringPatterns(RewritePatternSet &patterns,
                                               int64_t targetRank = 1,
                                               PatternBenefit benefit = 1);
 
+void populateVectorInterleaveToShufflePatterns(RewritePatternSet &patterns,
+                                               PatternBenefit benefit = 1);
+
 } // namespace vector
 } // namespace mlir
 #endif // MLIR_DIALECT_VECTOR_TRANSFORMS_LOWERINGPATTERNS_H
diff --git a/mlir/lib/Conversion/VectorToSPIRV/CMakeLists.txt b/mlir/lib/Conversion/VectorToSPIRV/CMakeLists.txt
index bb9f793d7fe0..113983146f5b 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/CMakeLists.txt
+++ b/mlir/lib/Conversion/VectorToSPIRV/CMakeLists.txt
@@ -14,5 +14,6 @@ add_mlir_conversion_library(MLIRVectorToSPIRV
   MLIRSPIRVDialect
   MLIRSPIRVConversion
   MLIRVectorDialect
+  MLIRVectorTransforms
   MLIRTransforms
   )
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index 868a3521e7a0..c2dd37f48146 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
@@ -828,6 +829,9 @@ void mlir::populateVectorToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
   // than the generic one that extracts all elements.
   patterns.add<VectorReductionToFPDotProd>(typeConverter, patterns.getContext(),
                                            PatternBenefit(2));
+
+  // Need this until vector.interleave is handled.
+  vector::populateVectorInterleaveToShufflePatterns(patterns);
 }
 
 void mlir::populateVectorReductionToSPIRVDotProductPatterns(
diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
index 885644864c0f..61fd6bd972e3 100644
--- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
+++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
@@ -164,6 +164,11 @@ void transform::ApplyLowerInterleavePatternsOp::populatePatterns(
   vector::populateVectorInterleaveLoweringPatterns(patterns);
 }
 
+void transform::ApplyInterleaveToShufflePatternsOp::populatePatterns(
+    RewritePatternSet &patterns) {
+  vector::populateVectorInterleaveToShufflePatterns(patterns);
+}
+
 void transform::ApplyRewriteNarrowTypePatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
   populateVectorNarrowTypeRewritePatterns(patterns);
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
index 3a456076f8fb..5326760c9b4e 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorInterleave.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/PatternMatch.h"
+#include "mlir/Support/LogicalResult.h"
 
 #define DEBUG_TYPE "vector-interleave-lowering"
 
@@ -77,9 +78,49 @@ private:
   int64_t targetRank = 1;
 };
 
+/// Rewrite vector.interleave op into an equivalent vector.shuffle op, when
+/// applicable: `sourceType` must be 1D and non-scalable.
+///
+/// Example:
+///
+/// ```mlir
+/// vector.interleave %a, %b : vector<7xi16>
+/// ```
+///
+/// Is rewritten into:
+///
+/// ```mlir
+/// vector.shuffle %arg0, %arg1 [0, 7, 1, 8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13]
+///   : vector<7xi16>, vector<7xi16>
+/// ```
+class InterleaveToShuffle : public OpRewritePattern<vector::InterleaveOp> {
+public:
+  InterleaveToShuffle(MLIRContext *context, PatternBenefit benefit = 1)
+      : OpRewritePattern(context, benefit) {};
+
+  LogicalResult matchAndRewrite(vector::InterleaveOp op,
+                                PatternRewriter &rewriter) const override {
+    VectorType sourceType = op.getSourceVectorType();
+    if (sourceType.getRank() != 1 || sourceType.isScalable()) {
+      return failure();
+    }
+    int64_t n = sourceType.getNumElements();
+    auto seq = llvm::seq<int64_t>(2 * n);
+    auto zip = llvm::to_vector(llvm::map_range(
+        seq, [n](int64_t i) { return (i % 2 ? n : 0) + i / 2; }));
+    rewriter.replaceOpWithNewOp<ShuffleOp>(op, op.getLhs(), op.getRhs(), zip);
+    return success();
+  }
+};
+
 } // namespace
 
 void mlir::vector::populateVectorInterleaveLoweringPatterns(
     RewritePatternSet &patterns, int64_t targetRank, PatternBenefit benefit) {
   patterns.add<UnrollInterleaveOp>(targetRank, patterns.getContext(), benefit);
 }
+
+void mlir::vector::populateVectorInterleaveToShufflePatterns(
+    RewritePatternSet &patterns, PatternBenefit benefit) {
+  patterns.add<InterleaveToShuffle>(patterns.getContext(), benefit);
+}
diff --git a/mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir b/mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir
new file mode 100644
index 000000000000..ed3b3396bf3e
--- /dev/null
+++ b/mlir/test/Dialect/Vector/vector-interleave-to-shuffle.mlir
@@ -0,0 +1,21 @@
+// RUN: mlir-opt %s --transform-interpreter | FileCheck %s
+
+// CHECK-LABEL: @vector_interleave_to_shuffle
+func.func @vector_interleave_to_shuffle(%a: vector<7xi16>, %b: vector<7xi16>) -> vector<14xi16>
+{
+  %0 = vector.interleave %a, %b : vector<7xi16>
+  return %0 : vector<14xi16>
+}
+// CHECK: vector.shuffle %arg0, %arg1 [0, 7, 1, 8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13] : vector<7xi16>, vector<7xi16>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+    %f = transform.structured.match ops{["func.func"]} in %module_op
+      : (!transform.any_op) -> !transform.any_op
+
+    transform.apply_patterns to %f {
+      transform.apply_patterns.vector.interleave_to_shuffle
+    } : !transform.any_op
+    transform.yield
+  }
+}
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 6304b7b548d8..debd8daf5549 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -5877,6 +5877,7 @@ cc_library(
         ":Support",
         ":TransformUtils",
         ":VectorDialect",
+        ":VectorTransforms",
         "//llvm:Support",
     ],
 )
-- 
GitLab


From 31a203fa8af47a8b2e8e357857b114cf90638b2e Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Mon, 13 May 2024 23:37:59 +0400
Subject: [PATCH 0627/1206] [clang] Introduce `SemaObjC` (#89086)

This is continuation of efforts to split `Sema` up, following the
example of OpenMP, OpenACC, etc. Context can be found in
https://github.com/llvm/llvm-project/pull/82217 and
https://github.com/llvm/llvm-project/pull/84184.

I split formatting changes into a separate commit to help reviewing the
actual changes.
---
 clang/include/clang/Parse/Parser.h            |   11 +-
 clang/include/clang/Sema/Sema.h               | 1002 +----------
 clang/include/clang/Sema/SemaObjC.h           | 1014 +++++++++++
 clang/lib/ARCMigrate/Transforms.cpp           |    5 +-
 clang/lib/Parse/ParseDecl.cpp                 |    7 +-
 clang/lib/Parse/ParseExpr.cpp                 |   17 +-
 clang/lib/Parse/ParseInit.cpp                 |    9 +-
 clang/lib/Parse/ParseObjc.cpp                 |  255 ++-
 clang/lib/Parse/ParseStmt.cpp                 |   11 +-
 clang/lib/Sema/CMakeLists.txt                 |    1 +
 clang/lib/Sema/Sema.cpp                       |   14 +-
 clang/lib/Sema/SemaAPINotes.cpp               |    3 +-
 clang/lib/Sema/SemaAttr.cpp                   |   16 -
 clang/lib/Sema/SemaAvailability.cpp           |    7 +-
 clang/lib/Sema/SemaCast.cpp                   |   13 +-
 clang/lib/Sema/SemaChecking.cpp               |  528 +-----
 clang/lib/Sema/SemaCodeComplete.cpp           |   40 +-
 clang/lib/Sema/SemaDecl.cpp                   |  292 +---
 clang/lib/Sema/SemaDeclAttr.cpp               |   11 +-
 clang/lib/Sema/SemaDeclCXX.cpp                |   58 +-
 clang/lib/Sema/SemaDeclObjC.cpp               | 1444 +++++++++-------
 clang/lib/Sema/SemaExpr.cpp                   |  559 +------
 clang/lib/Sema/SemaExprCXX.cpp                |   17 +-
 clang/lib/Sema/SemaExprMember.cpp             |   12 +-
 clang/lib/Sema/SemaExprObjC.cpp               | 1342 ++++++++++-----
 clang/lib/Sema/SemaInit.cpp                   |   17 +-
 clang/lib/Sema/SemaLookup.cpp                 |    9 -
 clang/lib/Sema/SemaObjC.cpp                   | 1486 +++++++++++++++++
 clang/lib/Sema/SemaObjCProperty.cpp           |  290 ++--
 clang/lib/Sema/SemaOverload.cpp               |   78 +-
 clang/lib/Sema/SemaPseudoObject.cpp           |  186 +--
 clang/lib/Sema/SemaStmt.cpp                   |  307 +---
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |    3 +-
 clang/lib/Sema/SemaType.cpp                   |  465 +-----
 clang/lib/Sema/TreeTransform.h                |  113 +-
 clang/lib/Serialization/ASTCommon.cpp         |    2 +-
 clang/lib/Serialization/ASTReader.cpp         |   15 +-
 clang/lib/Serialization/ASTWriter.cpp         |   14 +-
 38 files changed, 4897 insertions(+), 4776 deletions(-)
 create mode 100644 clang/include/clang/Sema/SemaObjC.h
 create mode 100644 clang/lib/Sema/SemaObjC.cpp

diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 3910cba34a21..7a8c2bcde804 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -18,6 +18,7 @@
 #include "clang/Lex/CodeCompletionHandler.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Frontend/OpenMP/OMPContext.h"
@@ -445,8 +446,8 @@ class Parser : public CodeCompletionHandler {
   /// True if we are within an Objective-C container while parsing C-like decls.
   ///
   /// This is necessary because Sema thinks we have left the container
-  /// to parse the C-like decls, meaning Actions.getObjCDeclContext() will
-  /// be NULL.
+  /// to parse the C-like decls, meaning Actions.ObjC().getObjCDeclContext()
+  /// will be NULL.
   bool ParsingInObjCContainer;
 
   /// Whether to skip parsing of function bodies.
@@ -497,7 +498,7 @@ public:
   }
 
   ObjCContainerDecl *getObjCDeclContext() const {
-    return Actions.getObjCDeclContext();
+    return Actions.ObjC().getObjCDeclContext();
   }
 
   // Type forwarding.  All of these are statically 'void*', but they may all be
@@ -1083,11 +1084,11 @@ private:
       : P(p), DC(p.getObjCDeclContext()),
         WithinObjCContainer(P.ParsingInObjCContainer, DC != nullptr) {
       if (DC)
-        P.Actions.ActOnObjCTemporaryExitContainerContext(DC);
+        P.Actions.ObjC().ActOnObjCTemporaryExitContainerContext(DC);
     }
     ~ObjCDeclContextSwitch() {
       if (DC)
-        P.Actions.ActOnObjCReenterContainerContext(DC);
+        P.Actions.ObjC().ActOnObjCReenterContainerContext(DC);
     }
   };
 
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 869769f95fd7..6a414aa57f32 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -152,18 +152,9 @@ typedef ArrayRef<std::pair<IdentifierInfo *, SourceLocation>> ModuleIdPath;
 class ModuleLoader;
 class MultiLevelTemplateArgumentList;
 class NamedDecl;
-class ObjCCategoryDecl;
-class ObjCCategoryImplDecl;
-class ObjCCompatibleAliasDecl;
-class ObjCContainerDecl;
-class ObjCImplDecl;
 class ObjCImplementationDecl;
 class ObjCInterfaceDecl;
-class ObjCIvarDecl;
-template <class T> class ObjCList;
-class ObjCMessageExpr;
 class ObjCMethodDecl;
-class ObjCPropertyDecl;
 class ObjCProtocolDecl;
 struct OverloadCandidate;
 enum class OverloadCandidateParamOrder : char;
@@ -178,6 +169,7 @@ class PseudoObjectExpr;
 class QualType;
 class SemaCUDA;
 class SemaHLSL;
+class SemaObjC;
 class SemaOpenACC;
 class SemaOpenMP;
 class SemaSYCL;
@@ -489,12 +481,9 @@ class Sema final : public SemaBase {
   // 29. C++ Variadic Templates (SemaTemplateVariadic.cpp)
   // 30. Constraints and Concepts (SemaConcept.cpp)
   // 31. Types (SemaType.cpp)
-  // 32. ObjC Declarations (SemaDeclObjC.cpp)
-  // 33. ObjC Expressions (SemaExprObjC.cpp)
-  // 34. ObjC @property and @synthesize (SemaObjCProperty.cpp)
-  // 35. Code Completion (SemaCodeComplete.cpp)
-  // 36. FixIt Helpers (SemaFixItUtils.cpp)
-  // 37. Name Lookup for RISC-V Vector Intrinsic (SemaRISCVVectorLookup.cpp)
+  // 32. Code Completion (SemaCodeComplete.cpp)
+  // 33. FixIt Helpers (SemaFixItUtils.cpp)
+  // 34. Name Lookup for RISC-V Vector Intrinsic (SemaRISCVVectorLookup.cpp)
 
   /// \name Semantic Analysis
   /// Implementations are in Sema.cpp
@@ -1005,6 +994,11 @@ public:
     return *HLSLPtr;
   }
 
+  SemaObjC &ObjC() {
+    assert(ObjCPtr);
+    return *ObjCPtr;
+  }
+
   SemaOpenACC &OpenACC() {
     assert(OpenACCPtr);
     return *OpenACCPtr;
@@ -1020,6 +1014,9 @@ public:
     return *SYCLPtr;
   }
 
+  /// Source of additional semantic information.
+  IntrusiveRefCntPtr<ExternalSemaSource> ExternalSource;
+
 protected:
   friend class Parser;
   friend class InitializationSequence;
@@ -1034,9 +1031,6 @@ private:
   Sema(const Sema &) = delete;
   void operator=(const Sema &) = delete;
 
-  /// Source of additional semantic information.
-  IntrusiveRefCntPtr<ExternalSemaSource> ExternalSource;
-
   /// The handler for the FileChanged preprocessor events.
   ///
   /// Used for diagnostics that implement custom semantic analysis for #include
@@ -1052,6 +1046,7 @@ private:
 
   std::unique_ptr<SemaCUDA> CUDAPtr;
   std::unique_ptr<SemaHLSL> HLSLPtr;
+  std::unique_ptr<SemaObjC> ObjCPtr;
   std::unique_ptr<SemaOpenACC> OpenACCPtr;
   std::unique_ptr<SemaOpenMP> OpenMPPtr;
   std::unique_ptr<SemaSYCL> SYCLPtr;
@@ -1634,11 +1629,6 @@ public:
   void ActOnPragmaUnused(const Token &Identifier, Scope *curScope,
                          SourceLocation PragmaLoc);
 
-  /// AddCFAuditedAttribute - Check whether we're currently within
-  /// '\#pragma clang arc_cf_code_audited' and, if so, consider adding
-  /// the appropriate attribute.
-  void AddCFAuditedAttribute(Decl *D);
-
   void ActOnPragmaAttributeAttribute(ParsedAttr &Attribute,
                                      SourceLocation PragmaLoc,
                                      attr::ParsedSubjectMatchRuleSet Rules);
@@ -1978,12 +1968,6 @@ public:
 
   void CheckCastAlign(Expr *Op, QualType T, SourceRange TRange);
 
-  /// checkRetainCycles - Check whether an Objective-C message send
-  /// might create an obvious retain cycle.
-  void checkRetainCycles(ObjCMessageExpr *msg);
-  void checkRetainCycles(Expr *receiver, Expr *argument);
-  void checkRetainCycles(VarDecl *Var, Expr *Init);
-
   /// checkUnsafeAssigns - Check whether +1 expr is being assigned
   /// to weak/__unsafe_unretained type.
   bool checkUnsafeAssigns(SourceLocation Loc, QualType LHS, Expr *RHS);
@@ -2030,14 +2014,20 @@ public:
 
   bool CheckHLSLBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
 
+  void checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto,
+                 const Expr *ThisArg, ArrayRef<const Expr *> Args,
+                 bool IsMemberFunction, SourceLocation Loc, SourceRange Range,
+                 VariadicCallType CallType);
+
+  void CheckTCBEnforcement(const SourceLocation CallExprLoc,
+                           const NamedDecl *Callee);
+
 private:
   void CheckArrayAccess(const Expr *BaseExpr, const Expr *IndexExpr,
                         const ArraySubscriptExpr *ASE = nullptr,
                         bool AllowOnePastEnd = true, bool IndexNegated = false);
   void CheckArrayAccess(const Expr *E);
 
-  bool CheckObjCMethodCall(ObjCMethodDecl *Method, SourceLocation loc,
-                           ArrayRef<const Expr *> Args);
   bool CheckPointerCall(NamedDecl *NDecl, CallExpr *TheCall,
                         const FunctionProtoType *Proto);
   bool CheckOtherCall(CallExpr *TheCall, const FunctionProtoType *Proto);
@@ -2050,12 +2040,6 @@ private:
   void CheckArgAlignment(SourceLocation Loc, NamedDecl *FDecl,
                          StringRef ParamName, QualType ArgTy, QualType ParamTy);
 
-  void checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto,
-                 const Expr *ThisArg, ArrayRef<const Expr *> Args,
-                 bool IsMemberFunction, SourceLocation Loc, SourceRange Range,
-                 VariadicCallType CallType);
-
-  bool CheckObjCString(Expr *Arg);
   ExprResult CheckOSLogFormatStringArg(Expr *Arg);
 
   ExprResult CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
@@ -2228,12 +2212,6 @@ private:
 
   void CheckBitFieldInitialization(SourceLocation InitLoc, FieldDecl *Field,
                                    Expr *Init);
-  /// Check whether receiver is mutable ObjC container which
-  /// attempts to add itself into the container
-  void CheckObjCCircularContainer(ObjCMessageExpr *Message);
-
-  void CheckTCBEnforcement(const SourceLocation CallExprLoc,
-                           const NamedDecl *Callee);
 
   /// A map from magic value to type information.
   std::unique_ptr<llvm::DenseMap<TypeTagMagicValue, TypeTagData>>
@@ -2636,7 +2614,7 @@ public:
   SmallVector<VarDecl *, 4> ExternalDeclarations;
 
   /// Generally null except when we temporarily switch decl contexts,
-  /// like in \see ActOnObjCTemporaryExitContainerContext.
+  /// like in \see SemaObjC::ActOnObjCTemporaryExitContainerContext.
   DeclContext *OriginalLexicalContext;
 
   /// Is the module scope we are in a C++ Header Unit?
@@ -2985,9 +2963,6 @@ public:
                              SourceLocation ExplicitThisLoc = {});
   ParmVarDecl *BuildParmVarDeclForTypedef(DeclContext *DC, SourceLocation Loc,
                                           QualType T);
-  QualType AdjustParameterTypeForObjCAutoRefCount(QualType T,
-                                                  SourceLocation NameLoc,
-                                                  TypeSourceInfo *TSInfo);
   ParmVarDecl *CheckParameter(DeclContext *DC, SourceLocation StartLoc,
                               SourceLocation NameLoc,
                               const IdentifierInfo *Name, QualType T,
@@ -3225,8 +3200,6 @@ public:
 
   void ActOnLastBitfield(SourceLocation DeclStart,
                          SmallVectorImpl<Decl *> &AllIvarDecls);
-  Decl *ActOnIvar(Scope *S, SourceLocation DeclStart, Declarator &D,
-                  Expr *BitWidth, tok::ObjCKeywordKind visibility);
 
   // This is used for both record definitions and ObjC interface declarations.
   void ActOnFields(Scope *S, SourceLocation RecLoc, Decl *TagDecl,
@@ -3248,8 +3221,6 @@ public:
   /// Invoked when we enter a tag definition that we're skipping.
   SkippedDefinitionContext ActOnTagStartSkippedDefinition(Scope *S, Decl *TD);
 
-  void ActOnObjCContainerStartDefinition(ObjCContainerDecl *IDecl);
-
   /// ActOnStartCXXMemberDeclarations - Invoked when we have parsed a
   /// C++ record definition's base-specifiers clause and are starting its
   /// member declarations.
@@ -3266,15 +3237,6 @@ public:
 
   void ActOnTagFinishSkippedDefinition(SkippedDefinitionContext Context);
 
-  void ActOnObjCContainerFinishDefinition();
-
-  /// Invoked when we must temporarily exit the objective-c container
-  /// scope for parsing/looking-up C constructs.
-  ///
-  /// Must be followed by a call to \see ActOnObjCReenterContainerContext
-  void ActOnObjCTemporaryExitContainerContext(ObjCContainerDecl *ObjCCtx);
-  void ActOnObjCReenterContainerContext(ObjCContainerDecl *ObjCCtx);
-
   /// ActOnTagDefinitionError - Invoked when there was an unrecoverable
   /// error parsing the definition of a tag.
   void ActOnTagDefinitionError(Scope *S, Decl *TagDecl);
@@ -3401,10 +3363,6 @@ public:
   /// variable.
   void DiagnoseUnusedButSetDecl(const VarDecl *VD, DiagReceiverTy DiagReceiver);
 
-  ObjCInterfaceDecl *getObjCInterfaceDecl(const IdentifierInfo *&Id,
-                                          SourceLocation IdLoc,
-                                          bool TypoCorrection = false);
-
   Scope *getNonFieldDeclScope(Scope *S);
 
   FunctionDecl *CreateBuiltin(IdentifierInfo *II, QualType Type, unsigned ID,
@@ -3426,8 +3384,6 @@ public:
   /// Look for a locally scoped extern "C" declaration by the given name.
   NamedDecl *findLocallyScopedExternCDecl(DeclarationName Name);
 
-  bool inferObjCARCLifetime(ValueDecl *decl);
-
   void deduceOpenCLAddressSpace(ValueDecl *decl);
 
   static bool adjustContextForLocalExternDecl(DeclContext *&DC);
@@ -3499,8 +3455,6 @@ public:
                             SourceLocation WeakNameLoc,
                             SourceLocation AliasNameLoc);
 
-  ObjCContainerDecl *getObjCDeclContext() const;
-
   /// Status of the function emission on the CUDA/HIP/OpenMP host/device attrs.
   enum class FunctionEmissionStatus {
     Emitted,
@@ -4339,8 +4293,6 @@ public:
       CXXConstructorDecl *Constructor, bool AnyErrors,
       ArrayRef<CXXCtorInitializer *> Initializers = std::nullopt);
 
-  void SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation);
-
   /// MarkBaseAndMemberDestructorsReferenced - Given a record decl,
   /// mark all the non-trivial destructors of its members and bases as
   /// referenced.
@@ -5390,14 +5342,6 @@ public:
                       DeclContext *LookupCtx = nullptr,
                       TypoExpr **Out = nullptr);
 
-  DeclResult LookupIvarInObjCMethod(LookupResult &Lookup, Scope *S,
-                                    IdentifierInfo *II);
-  ExprResult BuildIvarRefExpr(Scope *S, SourceLocation Loc, ObjCIvarDecl *IV);
-
-  ExprResult LookupInObjCMethod(LookupResult &LookUp, Scope *S,
-                                IdentifierInfo *II,
-                                bool AllowBuiltinCreation = false);
-
   /// If \p D cannot be odr-used in the current expression evaluation context,
   /// return a reason explaining why. Otherwise, return NOUR_None.
   NonOdrUseReason getNonOdrUseReasonInCurrentContext(ValueDecl *D);
@@ -5715,19 +5659,6 @@ public:
                                 ArrayRef<Expr *> SubExprs,
                                 QualType T = QualType());
 
-  // Note that LK_String is intentionally after the other literals, as
-  // this is used for diagnostics logic.
-  enum ObjCLiteralKind {
-    LK_Array,
-    LK_Dictionary,
-    LK_Numeric,
-    LK_Boxed,
-    LK_String,
-    LK_Block,
-    LK_None
-  };
-  ObjCLiteralKind CheckLiteralKind(Expr *FromE);
-
   ExprResult PerformObjectMemberConversion(Expr *From,
                                            NestedNameSpecifier *Qualifier,
                                            NamedDecl *FoundDecl,
@@ -5763,14 +5694,6 @@ public:
 
   bool IsInvalidSMECallConversion(QualType FromType, QualType ToType);
 
-  const DeclContext *getCurObjCLexicalContext() const {
-    const DeclContext *DC = getCurLexicalContext();
-    // A category implicitly has the attribute of the interface.
-    if (const ObjCCategoryDecl *CatD = dyn_cast<ObjCCategoryDecl>(DC))
-      DC = CatD->getClassInterface();
-    return DC;
-  }
-
   /// Abstract base class used for diagnosing integer constant
   /// expression violations.
   class VerifyICEDiagnoser {
@@ -5908,9 +5831,6 @@ public:
       ExprResult &Cond, ExprResult &LHS, ExprResult &RHS, ExprValueKind &VK,
       ExprObjectKind &OK, SourceLocation QuestionLoc);
 
-  QualType FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS,
-                                        SourceLocation QuestionLoc);
-
   bool DiagnoseConditionalForNull(const Expr *LHSExpr, const Expr *RHSExpr,
                                   SourceLocation QuestionLoc);
 
@@ -6258,9 +6178,6 @@ public:
       Expr *LHSExpr, ExprResult &RHS, SourceLocation Loc, QualType CompoundType,
       BinaryOperatorKind Opc);
 
-  bool CheckConversionToObjCLiteral(QualType DstType, Expr *&SrcExpr,
-                                    bool Diagnose = true);
-
   /// To be used for checking whether the arguments being passed to
   /// function exceeds the number of parameters expected for it.
   static bool TooManyArguments(size_t NumParams, size_t NumArgs,
@@ -6593,13 +6510,6 @@ public:
   /// ActOnCXXBoolLiteral - Parse {true,false} literals.
   ExprResult ActOnCXXBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind);
 
-  /// ActOnObjCBoolLiteral - Parse {__objc_yes,__objc_no} literals.
-  ExprResult ActOnObjCBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind);
-
-  ExprResult
-  ActOnObjCAvailabilityCheckExpr(llvm::ArrayRef<AvailabilitySpec> AvailSpecs,
-                                 SourceLocation AtLoc, SourceLocation RParen);
-
   /// ActOnCXXNullPtrLiteral - Parse 'nullptr'.
   ExprResult ActOnCXXNullPtrLiteral(SourceLocation Loc);
 
@@ -7468,9 +7378,6 @@ public:
   bool LookupParsedName(LookupResult &R, Scope *S, CXXScopeSpec *SS,
                         QualType ObjectType, bool AllowBuiltinCreation = false,
                         bool EnteringContext = false);
-  ObjCProtocolDecl *LookupProtocol(
-      IdentifierInfo *II, SourceLocation IdLoc,
-      RedeclarationKind Redecl = RedeclarationKind::NotForRedeclaration);
   bool LookupInSuper(LookupResult &R, CXXRecordDecl *Class);
 
   void LookupOverloadedOperatorName(OverloadedOperatorKind Op, Scope *S,
@@ -7499,6 +7406,20 @@ public:
   /// visible at the specified location.
   void makeMergedDefinitionVisible(NamedDecl *ND);
 
+  /// Check ODR hashes for C/ObjC when merging types from modules.
+  /// Differently from C++, actually parse the body and reject in case
+  /// of a mismatch.
+  template <typename T,
+            typename = std::enable_if_t<std::is_base_of<NamedDecl, T>::value>>
+  bool ActOnDuplicateODRHashDefinition(T *Duplicate, T *Previous) {
+    if (Duplicate->getODRHash() != Previous->getODRHash())
+      return false;
+
+    // Make the previous decl visible.
+    makeMergedDefinitionVisible(Previous);
+    return true;
+  }
+
   /// Get the set of additional modules that should be checked during
   /// name lookup. A module and its imports become visible when instanting a
   /// template defined within it.
@@ -7964,8 +7885,6 @@ public:
                            bool &IncompatibleObjC);
   bool isObjCPointerConversion(QualType FromType, QualType ToType,
                                QualType &ConvertedType, bool &IncompatibleObjC);
-  bool isObjCWritebackConversion(QualType FromType, QualType ToType,
-                                 QualType &ConvertedType);
   bool IsBlockPointerConversion(QualType FromType, QualType ToType,
                                 QualType &ConvertedType);
 
@@ -8427,7 +8346,6 @@ public:
                                             DeclAccessPair FoundDecl,
                                             FunctionDecl *Fn);
 
-private:
   /// - Returns a selector which best matches given argument list or
   /// nullptr if none could be found
   ObjCMethodDecl *SelectBestMethod(Selector Sel, MultiExprArg Args,
@@ -8448,10 +8366,6 @@ private:
 
 public:
   void maybeExtendBlockObject(ExprResult &E);
-  CastKind PrepareCastToObjCObjectPointer(ExprResult &E);
-
-  enum ObjCSubscriptKind { OS_Array, OS_Dictionary, OS_Error };
-  ObjCSubscriptKind CheckSubscriptingKind(Expr *FromE);
 
   ExprResult checkPseudoObjectIncDec(Scope *S, SourceLocation OpLoc,
                                      UnaryOperatorKind Opcode, Expr *Op);
@@ -8559,13 +8473,6 @@ public:
 
   StmtResult ActOnForEachLValueExpr(Expr *E);
 
-  ExprResult CheckObjCForCollectionOperand(SourceLocation forLoc,
-                                           Expr *collection);
-  StmtResult ActOnObjCForCollectionStmt(SourceLocation ForColLoc, Stmt *First,
-                                        Expr *collection,
-                                        SourceLocation RParenLoc);
-  StmtResult FinishObjCForCollectionStmt(Stmt *ForCollection, Stmt *Body);
-
   enum BuildForRangeKind {
     /// Initial building of a for-range statement.
     BFRK_Build,
@@ -8632,24 +8539,6 @@ public:
                                      NamedReturnInfo &NRInfo,
                                      bool SupressSimplerImplicitMoves);
 
-  StmtResult ActOnObjCAtCatchStmt(SourceLocation AtLoc, SourceLocation RParen,
-                                  Decl *Parm, Stmt *Body);
-
-  StmtResult ActOnObjCAtFinallyStmt(SourceLocation AtLoc, Stmt *Body);
-
-  StmtResult ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try,
-                                MultiStmtArg Catch, Stmt *Finally);
-
-  StmtResult BuildObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw);
-  StmtResult ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw,
-                                  Scope *CurScope);
-  ExprResult ActOnObjCAtSynchronizedOperand(SourceLocation atLoc,
-                                            Expr *operand);
-  StmtResult ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc, Expr *SynchExpr,
-                                         Stmt *SynchBody);
-
-  StmtResult ActOnObjCAutoreleasePoolStmt(SourceLocation AtLoc, Stmt *Body);
-
   StmtResult ActOnCXXCatchBlock(SourceLocation CatchLoc, Decl *ExDecl,
                                 Stmt *HandlerBlock);
   StmtResult ActOnCXXTryBlock(SourceLocation TryLoc, Stmt *TryBlock,
@@ -11471,36 +11360,6 @@ public:
     Default = AcceptSizeless
   };
 
-  /// Build a an Objective-C protocol-qualified 'id' type where no
-  /// base type was specified.
-  TypeResult actOnObjCProtocolQualifierType(
-      SourceLocation lAngleLoc, ArrayRef<Decl *> protocols,
-      ArrayRef<SourceLocation> protocolLocs, SourceLocation rAngleLoc);
-
-  /// Build a specialized and/or protocol-qualified Objective-C type.
-  TypeResult actOnObjCTypeArgsAndProtocolQualifiers(
-      Scope *S, SourceLocation Loc, ParsedType BaseType,
-      SourceLocation TypeArgsLAngleLoc, ArrayRef<ParsedType> TypeArgs,
-      SourceLocation TypeArgsRAngleLoc, SourceLocation ProtocolLAngleLoc,
-      ArrayRef<Decl *> Protocols, ArrayRef<SourceLocation> ProtocolLocs,
-      SourceLocation ProtocolRAngleLoc);
-
-  /// Build an Objective-C type parameter type.
-  QualType BuildObjCTypeParamType(const ObjCTypeParamDecl *Decl,
-                                  SourceLocation ProtocolLAngleLoc,
-                                  ArrayRef<ObjCProtocolDecl *> Protocols,
-                                  ArrayRef<SourceLocation> ProtocolLocs,
-                                  SourceLocation ProtocolRAngleLoc,
-                                  bool FailOnError = false);
-
-  /// Build an Objective-C object pointer type.
-  QualType BuildObjCObjectType(
-      QualType BaseType, SourceLocation Loc, SourceLocation TypeArgsLAngleLoc,
-      ArrayRef<TypeSourceInfo *> TypeArgs, SourceLocation TypeArgsRAngleLoc,
-      SourceLocation ProtocolLAngleLoc, ArrayRef<ObjCProtocolDecl *> Protocols,
-      ArrayRef<SourceLocation> ProtocolLocs, SourceLocation ProtocolRAngleLoc,
-      bool FailOnError, bool Rebuilding);
-
   QualType BuildQualifiedType(QualType T, SourceLocation Loc, Qualifiers Qs,
                               const DeclSpec *DS = nullptr);
   QualType BuildQualifiedType(QualType T, SourceLocation Loc, unsigned CVRA,
@@ -11582,10 +11441,6 @@ public:
 
   TypeResult ActOnTypeName(Declarator &D);
 
-  /// The parser has parsed the context-sensitive type 'instancetype'
-  /// in an Objective-C message declaration. Return the appropriate type.
-  ParsedType ActOnObjCInstanceType(SourceLocation Loc);
-
   // Check whether the size of array element of type \p EltTy is a multiple of
   // its alignment and return false if it isn't.
   bool checkArrayElementAlignment(QualType EltTy, SourceLocation Loc);
@@ -11602,13 +11457,6 @@ public:
   /// Retrieve the keyword associated
   IdentifierInfo *getNullabilityKeyword(NullabilityKind nullability);
 
-  /// The struct behind the CFErrorRef pointer.
-  RecordDecl *CFError = nullptr;
-  bool isCFError(RecordDecl *D);
-
-  /// Retrieve the identifier "NSError".
-  IdentifierInfo *getNSErrorIdent();
-
   /// Adjust the calling convention of a method to be the ABI default if it
   /// wasn't specified explicitly.  This handles method types formed from
   /// function type typedefs and typename template arguments.
@@ -11795,784 +11643,6 @@ private:
   IdentifierInfo *Ident__Nullable_result = nullptr;
   IdentifierInfo *Ident__Null_unspecified = nullptr;
 
-  IdentifierInfo *Ident_NSError = nullptr;
-
-  ///@}
-
-  //
-  //
-  // -------------------------------------------------------------------------
-  //
-  //
-
-  /// \name ObjC Declarations
-  /// Implementations are in SemaDeclObjC.cpp
-  ///@{
-
-public:
-  enum ObjCSpecialMethodKind {
-    OSMK_None,
-    OSMK_Alloc,
-    OSMK_New,
-    OSMK_Copy,
-    OSMK_RetainingInit,
-    OSMK_NonRetainingInit
-  };
-
-  /// Method selectors used in a \@selector expression. Used for implementation
-  /// of -Wselector.
-  llvm::MapVector<Selector, SourceLocation> ReferencedSelectors;
-
-  class GlobalMethodPool {
-  public:
-    using Lists = std::pair<ObjCMethodList, ObjCMethodList>;
-    using iterator = llvm::DenseMap<Selector, Lists>::iterator;
-    iterator begin() { return Methods.begin(); }
-    iterator end() { return Methods.end(); }
-    iterator find(Selector Sel) { return Methods.find(Sel); }
-    std::pair<iterator, bool> insert(std::pair<Selector, Lists> &&Val) {
-      return Methods.insert(Val);
-    }
-    int count(Selector Sel) const { return Methods.count(Sel); }
-    bool empty() const { return Methods.empty(); }
-
-  private:
-    llvm::DenseMap<Selector, Lists> Methods;
-  };
-
-  /// Method Pool - allows efficient lookup when typechecking messages to "id".
-  /// We need to maintain a list, since selectors can have differing signatures
-  /// across classes. In Cocoa, this happens to be extremely uncommon (only 1%
-  /// of selectors are "overloaded").
-  /// At the head of the list it is recorded whether there were 0, 1, or >= 2
-  /// methods inside categories with a particular selector.
-  GlobalMethodPool MethodPool;
-
-  /// Check ODR hashes for C/ObjC when merging types from modules.
-  /// Differently from C++, actually parse the body and reject in case
-  /// of a mismatch.
-  template <typename T,
-            typename = std::enable_if_t<std::is_base_of<NamedDecl, T>::value>>
-  bool ActOnDuplicateODRHashDefinition(T *Duplicate, T *Previous) {
-    if (Duplicate->getODRHash() != Previous->getODRHash())
-      return false;
-
-    // Make the previous decl visible.
-    makeMergedDefinitionVisible(Previous);
-    return true;
-  }
-
-  typedef llvm::SmallPtrSet<Selector, 8> SelectorSet;
-
-  enum MethodMatchStrategy { MMS_loose, MMS_strict };
-
-  enum ObjCContainerKind {
-    OCK_None = -1,
-    OCK_Interface = 0,
-    OCK_Protocol,
-    OCK_Category,
-    OCK_ClassExtension,
-    OCK_Implementation,
-    OCK_CategoryImplementation
-  };
-  ObjCContainerKind getObjCContainerKind() const;
-
-  DeclResult actOnObjCTypeParam(Scope *S, ObjCTypeParamVariance variance,
-                                SourceLocation varianceLoc, unsigned index,
-                                IdentifierInfo *paramName,
-                                SourceLocation paramLoc,
-                                SourceLocation colonLoc, ParsedType typeBound);
-
-  ObjCTypeParamList *actOnObjCTypeParamList(Scope *S, SourceLocation lAngleLoc,
-                                            ArrayRef<Decl *> typeParams,
-                                            SourceLocation rAngleLoc);
-  void popObjCTypeParamList(Scope *S, ObjCTypeParamList *typeParamList);
-
-  ObjCInterfaceDecl *ActOnStartClassInterface(
-      Scope *S, SourceLocation AtInterfaceLoc, IdentifierInfo *ClassName,
-      SourceLocation ClassLoc, ObjCTypeParamList *typeParamList,
-      IdentifierInfo *SuperName, SourceLocation SuperLoc,
-      ArrayRef<ParsedType> SuperTypeArgs, SourceRange SuperTypeArgsRange,
-      Decl *const *ProtoRefs, unsigned NumProtoRefs,
-      const SourceLocation *ProtoLocs, SourceLocation EndProtoLoc,
-      const ParsedAttributesView &AttrList, SkipBodyInfo *SkipBody);
-
-  void ActOnSuperClassOfClassInterface(
-      Scope *S, SourceLocation AtInterfaceLoc, ObjCInterfaceDecl *IDecl,
-      IdentifierInfo *ClassName, SourceLocation ClassLoc,
-      IdentifierInfo *SuperName, SourceLocation SuperLoc,
-      ArrayRef<ParsedType> SuperTypeArgs, SourceRange SuperTypeArgsRange);
-
-  void ActOnTypedefedProtocols(SmallVectorImpl<Decl *> &ProtocolRefs,
-                               SmallVectorImpl<SourceLocation> &ProtocolLocs,
-                               IdentifierInfo *SuperName,
-                               SourceLocation SuperLoc);
-
-  Decl *ActOnCompatibilityAlias(SourceLocation AtCompatibilityAliasLoc,
-                                IdentifierInfo *AliasName,
-                                SourceLocation AliasLocation,
-                                IdentifierInfo *ClassName,
-                                SourceLocation ClassLocation);
-
-  bool CheckForwardProtocolDeclarationForCircularDependency(
-      IdentifierInfo *PName, SourceLocation &PLoc, SourceLocation PrevLoc,
-      const ObjCList<ObjCProtocolDecl> &PList);
-
-  ObjCProtocolDecl *ActOnStartProtocolInterface(
-      SourceLocation AtProtoInterfaceLoc, IdentifierInfo *ProtocolName,
-      SourceLocation ProtocolLoc, Decl *const *ProtoRefNames,
-      unsigned NumProtoRefs, const SourceLocation *ProtoLocs,
-      SourceLocation EndProtoLoc, const ParsedAttributesView &AttrList,
-      SkipBodyInfo *SkipBody);
-
-  ObjCCategoryDecl *ActOnStartCategoryInterface(
-      SourceLocation AtInterfaceLoc, const IdentifierInfo *ClassName,
-      SourceLocation ClassLoc, ObjCTypeParamList *typeParamList,
-      const IdentifierInfo *CategoryName, SourceLocation CategoryLoc,
-      Decl *const *ProtoRefs, unsigned NumProtoRefs,
-      const SourceLocation *ProtoLocs, SourceLocation EndProtoLoc,
-      const ParsedAttributesView &AttrList);
-
-  ObjCImplementationDecl *ActOnStartClassImplementation(
-      SourceLocation AtClassImplLoc, const IdentifierInfo *ClassName,
-      SourceLocation ClassLoc, const IdentifierInfo *SuperClassname,
-      SourceLocation SuperClassLoc, const ParsedAttributesView &AttrList);
-
-  ObjCCategoryImplDecl *ActOnStartCategoryImplementation(
-      SourceLocation AtCatImplLoc, const IdentifierInfo *ClassName,
-      SourceLocation ClassLoc, const IdentifierInfo *CatName,
-      SourceLocation CatLoc, const ParsedAttributesView &AttrList);
-
-  DeclGroupPtrTy ActOnFinishObjCImplementation(Decl *ObjCImpDecl,
-                                               ArrayRef<Decl *> Decls);
-
-  DeclGroupPtrTy
-  ActOnForwardProtocolDeclaration(SourceLocation AtProtoclLoc,
-                                  ArrayRef<IdentifierLocPair> IdentList,
-                                  const ParsedAttributesView &attrList);
-
-  void FindProtocolDeclaration(bool WarnOnDeclarations, bool ForObjCContainer,
-                               ArrayRef<IdentifierLocPair> ProtocolId,
-                               SmallVectorImpl<Decl *> &Protocols);
-
-  void DiagnoseTypeArgsAndProtocols(IdentifierInfo *ProtocolId,
-                                    SourceLocation ProtocolLoc,
-                                    IdentifierInfo *TypeArgId,
-                                    SourceLocation TypeArgLoc,
-                                    bool SelectProtocolFirst = false);
-
-  /// Given a list of identifiers (and their locations), resolve the
-  /// names to either Objective-C protocol qualifiers or type
-  /// arguments, as appropriate.
-  void actOnObjCTypeArgsOrProtocolQualifiers(
-      Scope *S, ParsedType baseType, SourceLocation lAngleLoc,
-      ArrayRef<IdentifierInfo *> identifiers,
-      ArrayRef<SourceLocation> identifierLocs, SourceLocation rAngleLoc,
-      SourceLocation &typeArgsLAngleLoc, SmallVectorImpl<ParsedType> &typeArgs,
-      SourceLocation &typeArgsRAngleLoc, SourceLocation &protocolLAngleLoc,
-      SmallVectorImpl<Decl *> &protocols, SourceLocation &protocolRAngleLoc,
-      bool warnOnIncompleteProtocols);
-
-  void DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT,
-                                        ObjCInterfaceDecl *ID);
-
-  Decl *ActOnAtEnd(Scope *S, SourceRange AtEnd,
-                   ArrayRef<Decl *> allMethods = std::nullopt,
-                   ArrayRef<DeclGroupPtrTy> allTUVars = std::nullopt);
-
-  struct ObjCArgInfo {
-    IdentifierInfo *Name;
-    SourceLocation NameLoc;
-    // The Type is null if no type was specified, and the DeclSpec is invalid
-    // in this case.
-    ParsedType Type;
-    ObjCDeclSpec DeclSpec;
-
-    /// ArgAttrs - Attribute list for this argument.
-    ParsedAttributesView ArgAttrs;
-  };
-
-  Decl *ActOnMethodDeclaration(
-      Scope *S,
-      SourceLocation BeginLoc, // location of the + or -.
-      SourceLocation EndLoc,   // location of the ; or {.
-      tok::TokenKind MethodType, ObjCDeclSpec &ReturnQT, ParsedType ReturnType,
-      ArrayRef<SourceLocation> SelectorLocs, Selector Sel,
-      // optional arguments. The number of types/arguments is obtained
-      // from the Sel.getNumArgs().
-      ObjCArgInfo *ArgInfo, DeclaratorChunk::ParamInfo *CParamInfo,
-      unsigned CNumArgs, // c-style args
-      const ParsedAttributesView &AttrList, tok::ObjCKeywordKind MethodImplKind,
-      bool isVariadic, bool MethodDefinition);
-
-  bool CheckARCMethodDecl(ObjCMethodDecl *method);
-
-  bool checkInitMethod(ObjCMethodDecl *method, QualType receiverTypeIfCall);
-
-  /// Check whether the given new method is a valid override of the
-  /// given overridden method, and set any properties that should be inherited.
-  void CheckObjCMethodOverride(ObjCMethodDecl *NewMethod,
-                               const ObjCMethodDecl *Overridden);
-
-  /// Describes the compatibility of a result type with its method.
-  enum ResultTypeCompatibilityKind {
-    RTC_Compatible,
-    RTC_Incompatible,
-    RTC_Unknown
-  };
-
-  void CheckObjCMethodDirectOverrides(ObjCMethodDecl *method,
-                                      ObjCMethodDecl *overridden);
-
-  void CheckObjCMethodOverrides(ObjCMethodDecl *ObjCMethod,
-                                ObjCInterfaceDecl *CurrentClass,
-                                ResultTypeCompatibilityKind RTC);
-
-  /// AddAnyMethodToGlobalPool - Add any method, instance or factory to global
-  /// pool.
-  void AddAnyMethodToGlobalPool(Decl *D);
-
-  void ActOnStartOfObjCMethodDef(Scope *S, Decl *D);
-  bool isObjCMethodDecl(Decl *D) { return D && isa<ObjCMethodDecl>(D); }
-
-  /// CheckImplementationIvars - This routine checks if the instance variables
-  /// listed in the implelementation match those listed in the interface.
-  void CheckImplementationIvars(ObjCImplementationDecl *ImpDecl,
-                                ObjCIvarDecl **Fields, unsigned nIvars,
-                                SourceLocation Loc);
-
-  void WarnConflictingTypedMethods(ObjCMethodDecl *Method,
-                                   ObjCMethodDecl *MethodDecl,
-                                   bool IsProtocolMethodDecl);
-
-  void CheckConflictingOverridingMethod(ObjCMethodDecl *Method,
-                                        ObjCMethodDecl *Overridden,
-                                        bool IsProtocolMethodDecl);
-
-  /// WarnExactTypedMethods - This routine issues a warning if method
-  /// implementation declaration matches exactly that of its declaration.
-  void WarnExactTypedMethods(ObjCMethodDecl *Method, ObjCMethodDecl *MethodDecl,
-                             bool IsProtocolMethodDecl);
-
-  /// MatchAllMethodDeclarations - Check methods declaraed in interface or
-  /// or protocol against those declared in their implementations.
-  void MatchAllMethodDeclarations(
-      const SelectorSet &InsMap, const SelectorSet &ClsMap,
-      SelectorSet &InsMapSeen, SelectorSet &ClsMapSeen, ObjCImplDecl *IMPDecl,
-      ObjCContainerDecl *IDecl, bool &IncompleteImpl, bool ImmediateClass,
-      bool WarnCategoryMethodImpl = false);
-
-  /// CheckCategoryVsClassMethodMatches - Checks that methods implemented in
-  /// category matches with those implemented in its primary class and
-  /// warns each time an exact match is found.
-  void CheckCategoryVsClassMethodMatches(ObjCCategoryImplDecl *CatIMP);
-
-  /// ImplMethodsVsClassMethods - This is main routine to warn if any method
-  /// remains unimplemented in the class or category \@implementation.
-  void ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl *IMPDecl,
-                                 ObjCContainerDecl *IDecl,
-                                 bool IncompleteImpl = false);
-
-  DeclGroupPtrTy ActOnForwardClassDeclaration(
-      SourceLocation Loc, IdentifierInfo **IdentList, SourceLocation *IdentLocs,
-      ArrayRef<ObjCTypeParamList *> TypeParamLists, unsigned NumElts);
-
-  /// MatchTwoMethodDeclarations - Checks if two methods' type match and returns
-  /// true, or false, accordingly.
-  bool MatchTwoMethodDeclarations(const ObjCMethodDecl *Method,
-                                  const ObjCMethodDecl *PrevMethod,
-                                  MethodMatchStrategy strategy = MMS_strict);
-
-  /// Add the given method to the list of globally-known methods.
-  void addMethodToGlobalList(ObjCMethodList *List, ObjCMethodDecl *Method);
-
-  void ReadMethodPool(Selector Sel);
-  void updateOutOfDateSelector(Selector Sel);
-
-  /// - Returns instance or factory methods in global method pool for
-  /// given selector. It checks the desired kind first, if none is found, and
-  /// parameter checkTheOther is set, it then checks the other kind. If no such
-  /// method or only one method is found, function returns false; otherwise, it
-  /// returns true.
-  bool
-  CollectMultipleMethodsInGlobalPool(Selector Sel,
-                                     SmallVectorImpl<ObjCMethodDecl *> &Methods,
-                                     bool InstanceFirst, bool CheckTheOther,
-                                     const ObjCObjectType *TypeBound = nullptr);
-
-  bool
-  AreMultipleMethodsInGlobalPool(Selector Sel, ObjCMethodDecl *BestMethod,
-                                 SourceRange R, bool receiverIdOrClass,
-                                 SmallVectorImpl<ObjCMethodDecl *> &Methods);
-
-  void
-  DiagnoseMultipleMethodInGlobalPool(SmallVectorImpl<ObjCMethodDecl *> &Methods,
-                                     Selector Sel, SourceRange R,
-                                     bool receiverIdOrClass);
-
-  const ObjCMethodDecl *
-  SelectorsForTypoCorrection(Selector Sel, QualType ObjectType = QualType());
-  /// LookupImplementedMethodInGlobalPool - Returns the method which has an
-  /// implementation.
-  ObjCMethodDecl *LookupImplementedMethodInGlobalPool(Selector Sel);
-
-  void DiagnoseDuplicateIvars(ObjCInterfaceDecl *ID, ObjCInterfaceDecl *SID);
-
-  /// Checks that the Objective-C declaration is declared in the global scope.
-  /// Emits an error and marks the declaration as invalid if it's not declared
-  /// in the global scope.
-  bool CheckObjCDeclScope(Decl *D);
-
-  void ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart,
-                 const IdentifierInfo *ClassName,
-                 SmallVectorImpl<Decl *> &Decls);
-
-  VarDecl *BuildObjCExceptionDecl(TypeSourceInfo *TInfo, QualType ExceptionType,
-                                  SourceLocation StartLoc, SourceLocation IdLoc,
-                                  const IdentifierInfo *Id,
-                                  bool Invalid = false);
-
-  Decl *ActOnObjCExceptionDecl(Scope *S, Declarator &D);
-
-  /// CollectIvarsToConstructOrDestruct - Collect those ivars which require
-  /// initialization.
-  void
-  CollectIvarsToConstructOrDestruct(ObjCInterfaceDecl *OI,
-                                    SmallVectorImpl<ObjCIvarDecl *> &Ivars);
-
-  void DiagnoseUseOfUnimplementedSelectors();
-
-  /// DiagnoseUnusedBackingIvarInAccessor - Issue an 'unused' warning if ivar
-  /// which backs the property is not used in the property's accessor.
-  void DiagnoseUnusedBackingIvarInAccessor(Scope *S,
-                                           const ObjCImplementationDecl *ImplD);
-
-  /// GetIvarBackingPropertyAccessor - If method is a property setter/getter and
-  /// it property has a backing ivar, returns this ivar; otherwise, returns
-  /// NULL. It also returns ivar's property on success.
-  ObjCIvarDecl *
-  GetIvarBackingPropertyAccessor(const ObjCMethodDecl *Method,
-                                 const ObjCPropertyDecl *&PDecl) const;
-
-  /// AddInstanceMethodToGlobalPool - All instance methods in a translation
-  /// unit are added to a global pool. This allows us to efficiently associate
-  /// a selector with a method declaraation for purposes of typechecking
-  /// messages sent to "id" (where the class of the object is unknown).
-  void AddInstanceMethodToGlobalPool(ObjCMethodDecl *Method,
-                                     bool impl = false) {
-    AddMethodToGlobalPool(Method, impl, /*instance*/ true);
-  }
-
-  /// AddFactoryMethodToGlobalPool - Same as above, but for factory methods.
-  void AddFactoryMethodToGlobalPool(ObjCMethodDecl *Method, bool impl = false) {
-    AddMethodToGlobalPool(Method, impl, /*instance*/ false);
-  }
-
-private:
-  /// AddMethodToGlobalPool - Add an instance or factory method to the global
-  /// pool. See descriptoin of AddInstanceMethodToGlobalPool.
-  void AddMethodToGlobalPool(ObjCMethodDecl *Method, bool impl, bool instance);
-
-  /// LookupMethodInGlobalPool - Returns the instance or factory method and
-  /// optionally warns if there are multiple signatures.
-  ObjCMethodDecl *LookupMethodInGlobalPool(Selector Sel, SourceRange R,
-                                           bool receiverIdOrClass,
-                                           bool instance);
-
-  ///@}
-
-  //
-  //
-  // -------------------------------------------------------------------------
-  //
-  //
-
-  /// \name ObjC Expressions
-  /// Implementations are in SemaExprObjC.cpp
-  ///@{
-
-public:
-  /// Caches identifiers/selectors for NSFoundation APIs.
-  std::unique_ptr<NSAPI> NSAPIObj;
-
-  /// The declaration of the Objective-C NSNumber class.
-  ObjCInterfaceDecl *NSNumberDecl;
-
-  /// The declaration of the Objective-C NSValue class.
-  ObjCInterfaceDecl *NSValueDecl;
-
-  /// Pointer to NSNumber type (NSNumber *).
-  QualType NSNumberPointer;
-
-  /// Pointer to NSValue type (NSValue *).
-  QualType NSValuePointer;
-
-  /// The Objective-C NSNumber methods used to create NSNumber literals.
-  ObjCMethodDecl *NSNumberLiteralMethods[NSAPI::NumNSNumberLiteralMethods];
-
-  /// The declaration of the Objective-C NSString class.
-  ObjCInterfaceDecl *NSStringDecl;
-
-  /// Pointer to NSString type (NSString *).
-  QualType NSStringPointer;
-
-  /// The declaration of the stringWithUTF8String: method.
-  ObjCMethodDecl *StringWithUTF8StringMethod;
-
-  /// The declaration of the valueWithBytes:objCType: method.
-  ObjCMethodDecl *ValueWithBytesObjCTypeMethod;
-
-  /// The declaration of the Objective-C NSArray class.
-  ObjCInterfaceDecl *NSArrayDecl;
-
-  /// The declaration of the arrayWithObjects:count: method.
-  ObjCMethodDecl *ArrayWithObjectsMethod;
-
-  /// The declaration of the Objective-C NSDictionary class.
-  ObjCInterfaceDecl *NSDictionaryDecl;
-
-  /// The declaration of the dictionaryWithObjects:forKeys:count: method.
-  ObjCMethodDecl *DictionaryWithObjectsMethod;
-
-  /// id<NSCopying> type.
-  QualType QIDNSCopying;
-
-  /// will hold 'respondsToSelector:'
-  Selector RespondsToSelectorSel;
-
-  ExprResult HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
-                                       Expr *BaseExpr, SourceLocation OpLoc,
-                                       DeclarationName MemberName,
-                                       SourceLocation MemberLoc,
-                                       SourceLocation SuperLoc,
-                                       QualType SuperType, bool Super);
-
-  ExprResult ActOnClassPropertyRefExpr(const IdentifierInfo &receiverName,
-                                       const IdentifierInfo &propertyName,
-                                       SourceLocation receiverNameLoc,
-                                       SourceLocation propertyNameLoc);
-
-  // ParseObjCStringLiteral - Parse Objective-C string literals.
-  ExprResult ParseObjCStringLiteral(SourceLocation *AtLocs,
-                                    ArrayRef<Expr *> Strings);
-
-  ExprResult BuildObjCStringLiteral(SourceLocation AtLoc, StringLiteral *S);
-
-  /// BuildObjCNumericLiteral - builds an ObjCBoxedExpr AST node for the
-  /// numeric literal expression. Type of the expression will be "NSNumber *"
-  /// or "id" if NSNumber is unavailable.
-  ExprResult BuildObjCNumericLiteral(SourceLocation AtLoc, Expr *Number);
-  ExprResult ActOnObjCBoolLiteral(SourceLocation AtLoc, SourceLocation ValueLoc,
-                                  bool Value);
-  ExprResult BuildObjCArrayLiteral(SourceRange SR, MultiExprArg Elements);
-
-  /// BuildObjCBoxedExpr - builds an ObjCBoxedExpr AST node for the
-  /// '@' prefixed parenthesized expression. The type of the expression will
-  /// either be "NSNumber *", "NSString *" or "NSValue *" depending on the type
-  /// of ValueType, which is allowed to be a built-in numeric type, "char *",
-  /// "const char *" or C structure with attribute 'objc_boxable'.
-  ExprResult BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr);
-
-  ExprResult BuildObjCSubscriptExpression(SourceLocation RB, Expr *BaseExpr,
-                                          Expr *IndexExpr,
-                                          ObjCMethodDecl *getterMethod,
-                                          ObjCMethodDecl *setterMethod);
-
-  ExprResult
-  BuildObjCDictionaryLiteral(SourceRange SR,
-                             MutableArrayRef<ObjCDictionaryElement> Elements);
-
-  ExprResult BuildObjCEncodeExpression(SourceLocation AtLoc,
-                                       TypeSourceInfo *EncodedTypeInfo,
-                                       SourceLocation RParenLoc);
-
-  ExprResult ParseObjCEncodeExpression(SourceLocation AtLoc,
-                                       SourceLocation EncodeLoc,
-                                       SourceLocation LParenLoc, ParsedType Ty,
-                                       SourceLocation RParenLoc);
-
-  /// ParseObjCSelectorExpression - Build selector expression for \@selector
-  ExprResult ParseObjCSelectorExpression(Selector Sel, SourceLocation AtLoc,
-                                         SourceLocation SelLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation RParenLoc,
-                                         bool WarnMultipleSelectors);
-
-  /// ParseObjCProtocolExpression - Build protocol expression for \@protocol
-  ExprResult ParseObjCProtocolExpression(IdentifierInfo *ProtocolName,
-                                         SourceLocation AtLoc,
-                                         SourceLocation ProtoLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation ProtoIdLoc,
-                                         SourceLocation RParenLoc);
-
-  ObjCMethodDecl *tryCaptureObjCSelf(SourceLocation Loc);
-
-  /// Describes the kind of message expression indicated by a message
-  /// send that starts with an identifier.
-  enum ObjCMessageKind {
-    /// The message is sent to 'super'.
-    ObjCSuperMessage,
-    /// The message is an instance message.
-    ObjCInstanceMessage,
-    /// The message is a class message, and the identifier is a type
-    /// name.
-    ObjCClassMessage
-  };
-
-  ObjCMessageKind getObjCMessageKind(Scope *S, IdentifierInfo *Name,
-                                     SourceLocation NameLoc, bool IsSuper,
-                                     bool HasTrailingDot,
-                                     ParsedType &ReceiverType);
-
-  ExprResult ActOnSuperMessage(Scope *S, SourceLocation SuperLoc, Selector Sel,
-                               SourceLocation LBracLoc,
-                               ArrayRef<SourceLocation> SelectorLocs,
-                               SourceLocation RBracLoc, MultiExprArg Args);
-
-  ExprResult BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
-                               QualType ReceiverType, SourceLocation SuperLoc,
-                               Selector Sel, ObjCMethodDecl *Method,
-                               SourceLocation LBracLoc,
-                               ArrayRef<SourceLocation> SelectorLocs,
-                               SourceLocation RBracLoc, MultiExprArg Args,
-                               bool isImplicit = false);
-
-  ExprResult BuildClassMessageImplicit(QualType ReceiverType,
-                                       bool isSuperReceiver, SourceLocation Loc,
-                                       Selector Sel, ObjCMethodDecl *Method,
-                                       MultiExprArg Args);
-
-  ExprResult ActOnClassMessage(Scope *S, ParsedType Receiver, Selector Sel,
-                               SourceLocation LBracLoc,
-                               ArrayRef<SourceLocation> SelectorLocs,
-                               SourceLocation RBracLoc, MultiExprArg Args);
-
-  ExprResult BuildInstanceMessage(Expr *Receiver, QualType ReceiverType,
-                                  SourceLocation SuperLoc, Selector Sel,
-                                  ObjCMethodDecl *Method,
-                                  SourceLocation LBracLoc,
-                                  ArrayRef<SourceLocation> SelectorLocs,
-                                  SourceLocation RBracLoc, MultiExprArg Args,
-                                  bool isImplicit = false);
-
-  ExprResult BuildInstanceMessageImplicit(Expr *Receiver, QualType ReceiverType,
-                                          SourceLocation Loc, Selector Sel,
-                                          ObjCMethodDecl *Method,
-                                          MultiExprArg Args);
-
-  ExprResult ActOnInstanceMessage(Scope *S, Expr *Receiver, Selector Sel,
-                                  SourceLocation LBracLoc,
-                                  ArrayRef<SourceLocation> SelectorLocs,
-                                  SourceLocation RBracLoc, MultiExprArg Args);
-
-  ExprResult BuildObjCBridgedCast(SourceLocation LParenLoc,
-                                  ObjCBridgeCastKind Kind,
-                                  SourceLocation BridgeKeywordLoc,
-                                  TypeSourceInfo *TSInfo, Expr *SubExpr);
-
-  ExprResult ActOnObjCBridgedCast(Scope *S, SourceLocation LParenLoc,
-                                  ObjCBridgeCastKind Kind,
-                                  SourceLocation BridgeKeywordLoc,
-                                  ParsedType Type, SourceLocation RParenLoc,
-                                  Expr *SubExpr);
-
-  void CheckTollFreeBridgeCast(QualType castType, Expr *castExpr);
-
-  void CheckObjCBridgeRelatedCast(QualType castType, Expr *castExpr);
-
-  bool CheckTollFreeBridgeStaticCast(QualType castType, Expr *castExpr,
-                                     CastKind &Kind);
-
-  bool checkObjCBridgeRelatedComponents(SourceLocation Loc, QualType DestType,
-                                        QualType SrcType,
-                                        ObjCInterfaceDecl *&RelatedClass,
-                                        ObjCMethodDecl *&ClassMethod,
-                                        ObjCMethodDecl *&InstanceMethod,
-                                        TypedefNameDecl *&TDNDecl, bool CfToNs,
-                                        bool Diagnose = true);
-
-  bool CheckObjCBridgeRelatedConversions(SourceLocation Loc, QualType DestType,
-                                         QualType SrcType, Expr *&SrcExpr,
-                                         bool Diagnose = true);
-
-  /// Private Helper predicate to check for 'self'.
-  bool isSelfExpr(Expr *RExpr);
-  bool isSelfExpr(Expr *RExpr, const ObjCMethodDecl *Method);
-
-  ObjCMethodDecl *LookupMethodInQualifiedType(Selector Sel,
-                                              const ObjCObjectPointerType *OPT,
-                                              bool IsInstance);
-  ObjCMethodDecl *LookupMethodInObjectType(Selector Sel, QualType Ty,
-                                           bool IsInstance);
-
-  bool isKnownName(StringRef name);
-
-  enum ARCConversionResult { ACR_okay, ACR_unbridged, ACR_error };
-
-  /// Checks for invalid conversions and casts between
-  /// retainable pointers and other pointer kinds for ARC and Weak.
-  ARCConversionResult CheckObjCConversion(SourceRange castRange,
-                                          QualType castType, Expr *&op,
-                                          CheckedConversionKind CCK,
-                                          bool Diagnose = true,
-                                          bool DiagnoseCFAudited = false,
-                                          BinaryOperatorKind Opc = BO_PtrMemD);
-
-  Expr *stripARCUnbridgedCast(Expr *e);
-  void diagnoseARCUnbridgedCast(Expr *e);
-
-  bool CheckObjCARCUnavailableWeakConversion(QualType castType,
-                                             QualType ExprType);
-
-  /// CheckMessageArgumentTypes - Check types in an Obj-C message send.
-  /// \param Method - May be null.
-  /// \param [out] ReturnType - The return type of the send.
-  /// \return true iff there were any incompatible types.
-  bool CheckMessageArgumentTypes(const Expr *Receiver, QualType ReceiverType,
-                                 MultiExprArg Args, Selector Sel,
-                                 ArrayRef<SourceLocation> SelectorLocs,
-                                 ObjCMethodDecl *Method, bool isClassMessage,
-                                 bool isSuperMessage, SourceLocation lbrac,
-                                 SourceLocation rbrac, SourceRange RecRange,
-                                 QualType &ReturnType, ExprValueKind &VK);
-
-  /// Determine the result of a message send expression based on
-  /// the type of the receiver, the method expected to receive the message,
-  /// and the form of the message send.
-  QualType getMessageSendResultType(const Expr *Receiver, QualType ReceiverType,
-                                    ObjCMethodDecl *Method, bool isClassMessage,
-                                    bool isSuperMessage);
-
-  /// If the given expression involves a message send to a method
-  /// with a related result type, emit a note describing what happened.
-  void EmitRelatedResultTypeNote(const Expr *E);
-
-  /// Given that we had incompatible pointer types in a return
-  /// statement, check whether we're in a method with a related result
-  /// type, and if so, emit a note describing what happened.
-  void EmitRelatedResultTypeNoteForReturn(QualType destType);
-
-  /// LookupInstanceMethodInGlobalPool - Returns the method and warns if
-  /// there are multiple signatures.
-  ObjCMethodDecl *
-  LookupInstanceMethodInGlobalPool(Selector Sel, SourceRange R,
-                                   bool receiverIdOrClass = false) {
-    return LookupMethodInGlobalPool(Sel, R, receiverIdOrClass,
-                                    /*instance*/ true);
-  }
-
-  /// LookupFactoryMethodInGlobalPool - Returns the method and warns if
-  /// there are multiple signatures.
-  ObjCMethodDecl *
-  LookupFactoryMethodInGlobalPool(Selector Sel, SourceRange R,
-                                  bool receiverIdOrClass = false) {
-    return LookupMethodInGlobalPool(Sel, R, receiverIdOrClass,
-                                    /*instance*/ false);
-  }
-
-  ///@}
-
-  //
-  //
-  // -------------------------------------------------------------------------
-  //
-  //
-
-  /// \name ObjC @property and @synthesize
-  /// Implementations are in SemaObjCProperty.cpp
-  ///@{
-
-public:
-  /// Ensure attributes are consistent with type.
-  /// \param [in, out] Attributes The attributes to check; they will
-  /// be modified to be consistent with \p PropertyTy.
-  void CheckObjCPropertyAttributes(Decl *PropertyPtrTy, SourceLocation Loc,
-                                   unsigned &Attributes,
-                                   bool propertyInPrimaryClass);
-
-  /// Process the specified property declaration and create decls for the
-  /// setters and getters as needed.
-  /// \param property The property declaration being processed
-  void ProcessPropertyDecl(ObjCPropertyDecl *property);
-
-  Decl *ActOnProperty(Scope *S, SourceLocation AtLoc, SourceLocation LParenLoc,
-                      FieldDeclarator &FD, ObjCDeclSpec &ODS,
-                      Selector GetterSel, Selector SetterSel,
-                      tok::ObjCKeywordKind MethodImplKind,
-                      DeclContext *lexicalDC = nullptr);
-
-  Decl *ActOnPropertyImplDecl(Scope *S, SourceLocation AtLoc,
-                              SourceLocation PropertyLoc, bool ImplKind,
-                              IdentifierInfo *PropertyId,
-                              IdentifierInfo *PropertyIvar,
-                              SourceLocation PropertyIvarLoc,
-                              ObjCPropertyQueryKind QueryKind);
-
-  /// Called by ActOnProperty to handle \@property declarations in
-  /// class extensions.
-  ObjCPropertyDecl *HandlePropertyInClassExtension(
-      Scope *S, SourceLocation AtLoc, SourceLocation LParenLoc,
-      FieldDeclarator &FD, Selector GetterSel, SourceLocation GetterNameLoc,
-      Selector SetterSel, SourceLocation SetterNameLoc, const bool isReadWrite,
-      unsigned &Attributes, const unsigned AttributesAsWritten, QualType T,
-      TypeSourceInfo *TSI, tok::ObjCKeywordKind MethodImplKind);
-
-  /// Called by ActOnProperty and HandlePropertyInClassExtension to
-  /// handle creating the ObjcPropertyDecl for a category or \@interface.
-  ObjCPropertyDecl *
-  CreatePropertyDecl(Scope *S, ObjCContainerDecl *CDecl, SourceLocation AtLoc,
-                     SourceLocation LParenLoc, FieldDeclarator &FD,
-                     Selector GetterSel, SourceLocation GetterNameLoc,
-                     Selector SetterSel, SourceLocation SetterNameLoc,
-                     const bool isReadWrite, const unsigned Attributes,
-                     const unsigned AttributesAsWritten, QualType T,
-                     TypeSourceInfo *TSI, tok::ObjCKeywordKind MethodImplKind,
-                     DeclContext *lexicalDC = nullptr);
-
-  void DiagnosePropertyMismatch(ObjCPropertyDecl *Property,
-                                ObjCPropertyDecl *SuperProperty,
-                                const IdentifierInfo *Name,
-                                bool OverridingProtocolProperty);
-
-  bool DiagnosePropertyAccessorMismatch(ObjCPropertyDecl *PD,
-                                        ObjCMethodDecl *Getter,
-                                        SourceLocation Loc);
-
-  /// DiagnoseUnimplementedProperties - This routine warns on those properties
-  /// which must be implemented by this implementation.
-  void DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl *IMPDecl,
-                                       ObjCContainerDecl *CDecl,
-                                       bool SynthesizeProperties);
-
-  /// Diagnose any null-resettable synthesized setters.
-  void diagnoseNullResettableSynthesizedSetters(const ObjCImplDecl *impDecl);
-
-  /// DefaultSynthesizeProperties - This routine default synthesizes all
-  /// properties which must be synthesized in the class's \@implementation.
-  void DefaultSynthesizeProperties(Scope *S, ObjCImplDecl *IMPDecl,
-                                   ObjCInterfaceDecl *IDecl,
-                                   SourceLocation AtEnd);
-  void DefaultSynthesizeProperties(Scope *S, Decl *D, SourceLocation AtEnd);
-
-  /// IvarBacksCurrentMethodAccessor - This routine returns 'true' if 'IV' is
-  /// an ivar synthesized for 'Method' and 'Method' is a property accessor
-  /// declared in class 'IFace'.
-  bool IvarBacksCurrentMethodAccessor(ObjCInterfaceDecl *IFace,
-                                      ObjCMethodDecl *Method, ObjCIvarDecl *IV);
-
-  void DiagnoseOwningPropertyGetterSynthesis(const ObjCImplementationDecl *D);
-
-  void
-  DiagnoseMissingDesignatedInitOverrides(const ObjCImplementationDecl *ImplD,
-                                         const ObjCInterfaceDecl *IFD);
-
-  /// AtomicPropertySetterGetterRules - This routine enforces the rule (via
-  /// warning) when atomic property has one but not the other user-declared
-  /// setter or getter.
-  void AtomicPropertySetterGetterRules(ObjCImplDecl *IMPDecl,
-                                       ObjCInterfaceDecl *IDecl);
-
   ///@}
 
   //
diff --git a/clang/include/clang/Sema/SemaObjC.h b/clang/include/clang/Sema/SemaObjC.h
new file mode 100644
index 000000000000..a9a0d1678095
--- /dev/null
+++ b/clang/include/clang/Sema/SemaObjC.h
@@ -0,0 +1,1014 @@
+//===----- SemaObjC.h ------ Semantic Analysis for Objective-C ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file declares semantic analysis for Objective-C.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SEMA_SEMAOBJC_H
+#define LLVM_CLANG_SEMA_SEMAOBJC_H
+
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclBase.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprObjC.h"
+#include "clang/AST/NSAPI.h"
+#include "clang/AST/OperationKinds.h"
+#include "clang/AST/Type.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/Specifiers.h"
+#include "clang/Basic/TokenKinds.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/ObjCMethodList.h"
+#include "clang/Sema/Ownership.h"
+#include "clang/Sema/ParsedAttr.h"
+#include "clang/Sema/Redeclaration.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaBase.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include <memory>
+#include <optional>
+#include <type_traits>
+#include <utility>
+
+namespace clang {
+
+enum class CheckedConversionKind;
+struct SkipBodyInfo;
+
+class SemaObjC : public SemaBase {
+public:
+  SemaObjC(Sema &S);
+
+  ExprResult CheckObjCForCollectionOperand(SourceLocation forLoc,
+                                           Expr *collection);
+  StmtResult ActOnObjCForCollectionStmt(SourceLocation ForColLoc, Stmt *First,
+                                        Expr *collection,
+                                        SourceLocation RParenLoc);
+  /// FinishObjCForCollectionStmt - Attach the body to a objective-C foreach
+  /// statement.
+  StmtResult FinishObjCForCollectionStmt(Stmt *ForCollection, Stmt *Body);
+
+  StmtResult ActOnObjCAtCatchStmt(SourceLocation AtLoc, SourceLocation RParen,
+                                  Decl *Parm, Stmt *Body);
+
+  StmtResult ActOnObjCAtFinallyStmt(SourceLocation AtLoc, Stmt *Body);
+
+  StmtResult ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try,
+                                MultiStmtArg Catch, Stmt *Finally);
+
+  StmtResult BuildObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw);
+  StmtResult ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw,
+                                  Scope *CurScope);
+  ExprResult ActOnObjCAtSynchronizedOperand(SourceLocation atLoc,
+                                            Expr *operand);
+  StmtResult ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc, Expr *SynchExpr,
+                                         Stmt *SynchBody);
+
+  StmtResult ActOnObjCAutoreleasePoolStmt(SourceLocation AtLoc, Stmt *Body);
+
+  /// Build a an Objective-C protocol-qualified 'id' type where no
+  /// base type was specified.
+  TypeResult actOnObjCProtocolQualifierType(
+      SourceLocation lAngleLoc, ArrayRef<Decl *> protocols,
+      ArrayRef<SourceLocation> protocolLocs, SourceLocation rAngleLoc);
+
+  /// Build a specialized and/or protocol-qualified Objective-C type.
+  TypeResult actOnObjCTypeArgsAndProtocolQualifiers(
+      Scope *S, SourceLocation Loc, ParsedType BaseType,
+      SourceLocation TypeArgsLAngleLoc, ArrayRef<ParsedType> TypeArgs,
+      SourceLocation TypeArgsRAngleLoc, SourceLocation ProtocolLAngleLoc,
+      ArrayRef<Decl *> Protocols, ArrayRef<SourceLocation> ProtocolLocs,
+      SourceLocation ProtocolRAngleLoc);
+
+  /// Build an Objective-C type parameter type.
+  QualType BuildObjCTypeParamType(const ObjCTypeParamDecl *Decl,
+                                  SourceLocation ProtocolLAngleLoc,
+                                  ArrayRef<ObjCProtocolDecl *> Protocols,
+                                  ArrayRef<SourceLocation> ProtocolLocs,
+                                  SourceLocation ProtocolRAngleLoc,
+                                  bool FailOnError = false);
+
+  /// Build an Objective-C object pointer type.
+  QualType BuildObjCObjectType(
+      QualType BaseType, SourceLocation Loc, SourceLocation TypeArgsLAngleLoc,
+      ArrayRef<TypeSourceInfo *> TypeArgs, SourceLocation TypeArgsRAngleLoc,
+      SourceLocation ProtocolLAngleLoc, ArrayRef<ObjCProtocolDecl *> Protocols,
+      ArrayRef<SourceLocation> ProtocolLocs, SourceLocation ProtocolRAngleLoc,
+      bool FailOnError, bool Rebuilding);
+
+  /// The parser has parsed the context-sensitive type 'instancetype'
+  /// in an Objective-C message declaration. Return the appropriate type.
+  ParsedType ActOnObjCInstanceType(SourceLocation Loc);
+
+  /// checkRetainCycles - Check whether an Objective-C message send
+  /// might create an obvious retain cycle.
+  void checkRetainCycles(ObjCMessageExpr *msg);
+  void checkRetainCycles(Expr *receiver, Expr *argument);
+  void checkRetainCycles(VarDecl *Var, Expr *Init);
+
+  bool CheckObjCString(Expr *Arg);
+  bool CheckObjCMethodCall(ObjCMethodDecl *Method, SourceLocation loc,
+                           ArrayRef<const Expr *> Args);
+  /// Check whether receiver is mutable ObjC container which
+  /// attempts to add itself into the container
+  void CheckObjCCircularContainer(ObjCMessageExpr *Message);
+
+  void ActOnObjCContainerStartDefinition(ObjCContainerDecl *IDecl);
+  void ActOnObjCContainerFinishDefinition();
+
+  /// Invoked when we must temporarily exit the objective-c container
+  /// scope for parsing/looking-up C constructs.
+  ///
+  /// Must be followed by a call to \see ActOnObjCReenterContainerContext
+  void ActOnObjCTemporaryExitContainerContext(ObjCContainerDecl *ObjCCtx);
+  void ActOnObjCReenterContainerContext(ObjCContainerDecl *ObjCCtx);
+
+  const DeclContext *getCurObjCLexicalContext() const;
+
+  ObjCProtocolDecl *LookupProtocol(
+      IdentifierInfo *II, SourceLocation IdLoc,
+      RedeclarationKind Redecl = RedeclarationKind::NotForRedeclaration);
+
+  bool isObjCWritebackConversion(QualType FromType, QualType ToType,
+                                 QualType &ConvertedType);
+
+  enum ObjCSubscriptKind { OS_Array, OS_Dictionary, OS_Error };
+  ObjCSubscriptKind CheckSubscriptingKind(Expr *FromE);
+
+  /// AddCFAuditedAttribute - Check whether we're currently within
+  /// '\#pragma clang arc_cf_code_audited' and, if so, consider adding
+  /// the appropriate attribute.
+  void AddCFAuditedAttribute(Decl *D);
+
+  /// The struct behind the CFErrorRef pointer.
+  RecordDecl *CFError = nullptr;
+  bool isCFError(RecordDecl *D);
+
+  IdentifierInfo *getNSErrorIdent();
+
+private:
+  IdentifierInfo *Ident_NSError = nullptr;
+
+  //
+  //
+  // -------------------------------------------------------------------------
+  //
+  //
+
+  /// \name ObjC Declarations
+  /// Implementations are in SemaDeclObjC.cpp
+  ///@{
+
+public:
+  enum ObjCSpecialMethodKind {
+    OSMK_None,
+    OSMK_Alloc,
+    OSMK_New,
+    OSMK_Copy,
+    OSMK_RetainingInit,
+    OSMK_NonRetainingInit
+  };
+
+  /// Method selectors used in a \@selector expression. Used for implementation
+  /// of -Wselector.
+  llvm::MapVector<Selector, SourceLocation> ReferencedSelectors;
+
+  class GlobalMethodPool {
+  public:
+    using Lists = std::pair<ObjCMethodList, ObjCMethodList>;
+    using iterator = llvm::DenseMap<Selector, Lists>::iterator;
+    iterator begin() { return Methods.begin(); }
+    iterator end() { return Methods.end(); }
+    iterator find(Selector Sel) { return Methods.find(Sel); }
+    std::pair<iterator, bool> insert(std::pair<Selector, Lists> &&Val) {
+      return Methods.insert(Val);
+    }
+    int count(Selector Sel) const { return Methods.count(Sel); }
+    bool empty() const { return Methods.empty(); }
+
+  private:
+    llvm::DenseMap<Selector, Lists> Methods;
+  };
+
+  /// Method Pool - allows efficient lookup when typechecking messages to "id".
+  /// We need to maintain a list, since selectors can have differing signatures
+  /// across classes. In Cocoa, this happens to be extremely uncommon (only 1%
+  /// of selectors are "overloaded").
+  /// At the head of the list it is recorded whether there were 0, 1, or >= 2
+  /// methods inside categories with a particular selector.
+  GlobalMethodPool MethodPool;
+
+  typedef llvm::SmallPtrSet<Selector, 8> SelectorSet;
+
+  enum MethodMatchStrategy { MMS_loose, MMS_strict };
+
+  enum ObjCContainerKind {
+    OCK_None = -1,
+    OCK_Interface = 0,
+    OCK_Protocol,
+    OCK_Category,
+    OCK_ClassExtension,
+    OCK_Implementation,
+    OCK_CategoryImplementation
+  };
+  ObjCContainerKind getObjCContainerKind() const;
+
+  DeclResult actOnObjCTypeParam(Scope *S, ObjCTypeParamVariance variance,
+                                SourceLocation varianceLoc, unsigned index,
+                                IdentifierInfo *paramName,
+                                SourceLocation paramLoc,
+                                SourceLocation colonLoc, ParsedType typeBound);
+
+  ObjCTypeParamList *actOnObjCTypeParamList(Scope *S, SourceLocation lAngleLoc,
+                                            ArrayRef<Decl *> typeParams,
+                                            SourceLocation rAngleLoc);
+  void popObjCTypeParamList(Scope *S, ObjCTypeParamList *typeParamList);
+
+  ObjCInterfaceDecl *ActOnStartClassInterface(
+      Scope *S, SourceLocation AtInterfaceLoc, IdentifierInfo *ClassName,
+      SourceLocation ClassLoc, ObjCTypeParamList *typeParamList,
+      IdentifierInfo *SuperName, SourceLocation SuperLoc,
+      ArrayRef<ParsedType> SuperTypeArgs, SourceRange SuperTypeArgsRange,
+      Decl *const *ProtoRefs, unsigned NumProtoRefs,
+      const SourceLocation *ProtoLocs, SourceLocation EndProtoLoc,
+      const ParsedAttributesView &AttrList, SkipBodyInfo *SkipBody);
+
+  void ActOnSuperClassOfClassInterface(
+      Scope *S, SourceLocation AtInterfaceLoc, ObjCInterfaceDecl *IDecl,
+      IdentifierInfo *ClassName, SourceLocation ClassLoc,
+      IdentifierInfo *SuperName, SourceLocation SuperLoc,
+      ArrayRef<ParsedType> SuperTypeArgs, SourceRange SuperTypeArgsRange);
+
+  void ActOnTypedefedProtocols(SmallVectorImpl<Decl *> &ProtocolRefs,
+                               SmallVectorImpl<SourceLocation> &ProtocolLocs,
+                               IdentifierInfo *SuperName,
+                               SourceLocation SuperLoc);
+
+  Decl *ActOnCompatibilityAlias(SourceLocation AtCompatibilityAliasLoc,
+                                IdentifierInfo *AliasName,
+                                SourceLocation AliasLocation,
+                                IdentifierInfo *ClassName,
+                                SourceLocation ClassLocation);
+
+  bool CheckForwardProtocolDeclarationForCircularDependency(
+      IdentifierInfo *PName, SourceLocation &PLoc, SourceLocation PrevLoc,
+      const ObjCList<ObjCProtocolDecl> &PList);
+
+  ObjCProtocolDecl *ActOnStartProtocolInterface(
+      SourceLocation AtProtoInterfaceLoc, IdentifierInfo *ProtocolName,
+      SourceLocation ProtocolLoc, Decl *const *ProtoRefNames,
+      unsigned NumProtoRefs, const SourceLocation *ProtoLocs,
+      SourceLocation EndProtoLoc, const ParsedAttributesView &AttrList,
+      SkipBodyInfo *SkipBody);
+
+  ObjCCategoryDecl *ActOnStartCategoryInterface(
+      SourceLocation AtInterfaceLoc, const IdentifierInfo *ClassName,
+      SourceLocation ClassLoc, ObjCTypeParamList *typeParamList,
+      const IdentifierInfo *CategoryName, SourceLocation CategoryLoc,
+      Decl *const *ProtoRefs, unsigned NumProtoRefs,
+      const SourceLocation *ProtoLocs, SourceLocation EndProtoLoc,
+      const ParsedAttributesView &AttrList);
+
+  ObjCImplementationDecl *ActOnStartClassImplementation(
+      SourceLocation AtClassImplLoc, const IdentifierInfo *ClassName,
+      SourceLocation ClassLoc, const IdentifierInfo *SuperClassname,
+      SourceLocation SuperClassLoc, const ParsedAttributesView &AttrList);
+
+  ObjCCategoryImplDecl *ActOnStartCategoryImplementation(
+      SourceLocation AtCatImplLoc, const IdentifierInfo *ClassName,
+      SourceLocation ClassLoc, const IdentifierInfo *CatName,
+      SourceLocation CatLoc, const ParsedAttributesView &AttrList);
+
+  using DeclGroupPtrTy = OpaquePtr<DeclGroupRef>;
+
+  DeclGroupPtrTy ActOnFinishObjCImplementation(Decl *ObjCImpDecl,
+                                               ArrayRef<Decl *> Decls);
+
+  DeclGroupPtrTy
+  ActOnForwardProtocolDeclaration(SourceLocation AtProtoclLoc,
+                                  ArrayRef<IdentifierLocPair> IdentList,
+                                  const ParsedAttributesView &attrList);
+
+  void FindProtocolDeclaration(bool WarnOnDeclarations, bool ForObjCContainer,
+                               ArrayRef<IdentifierLocPair> ProtocolId,
+                               SmallVectorImpl<Decl *> &Protocols);
+
+  void DiagnoseTypeArgsAndProtocols(IdentifierInfo *ProtocolId,
+                                    SourceLocation ProtocolLoc,
+                                    IdentifierInfo *TypeArgId,
+                                    SourceLocation TypeArgLoc,
+                                    bool SelectProtocolFirst = false);
+
+  /// Given a list of identifiers (and their locations), resolve the
+  /// names to either Objective-C protocol qualifiers or type
+  /// arguments, as appropriate.
+  void actOnObjCTypeArgsOrProtocolQualifiers(
+      Scope *S, ParsedType baseType, SourceLocation lAngleLoc,
+      ArrayRef<IdentifierInfo *> identifiers,
+      ArrayRef<SourceLocation> identifierLocs, SourceLocation rAngleLoc,
+      SourceLocation &typeArgsLAngleLoc, SmallVectorImpl<ParsedType> &typeArgs,
+      SourceLocation &typeArgsRAngleLoc, SourceLocation &protocolLAngleLoc,
+      SmallVectorImpl<Decl *> &protocols, SourceLocation &protocolRAngleLoc,
+      bool warnOnIncompleteProtocols);
+
+  void DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT,
+                                        ObjCInterfaceDecl *ID);
+
+  Decl *ActOnAtEnd(Scope *S, SourceRange AtEnd,
+                   ArrayRef<Decl *> allMethods = std::nullopt,
+                   ArrayRef<DeclGroupPtrTy> allTUVars = std::nullopt);
+
+  struct ObjCArgInfo {
+    IdentifierInfo *Name;
+    SourceLocation NameLoc;
+    // The Type is null if no type was specified, and the DeclSpec is invalid
+    // in this case.
+    ParsedType Type;
+    ObjCDeclSpec DeclSpec;
+
+    /// ArgAttrs - Attribute list for this argument.
+    ParsedAttributesView ArgAttrs;
+  };
+
+  Decl *ActOnMethodDeclaration(
+      Scope *S,
+      SourceLocation BeginLoc, // location of the + or -.
+      SourceLocation EndLoc,   // location of the ; or {.
+      tok::TokenKind MethodType, ObjCDeclSpec &ReturnQT, ParsedType ReturnType,
+      ArrayRef<SourceLocation> SelectorLocs, Selector Sel,
+      // optional arguments. The number of types/arguments is obtained
+      // from the Sel.getNumArgs().
+      ObjCArgInfo *ArgInfo, DeclaratorChunk::ParamInfo *CParamInfo,
+      unsigned CNumArgs, // c-style args
+      const ParsedAttributesView &AttrList, tok::ObjCKeywordKind MethodImplKind,
+      bool isVariadic, bool MethodDefinition);
+
+  bool CheckARCMethodDecl(ObjCMethodDecl *method);
+
+  bool checkInitMethod(ObjCMethodDecl *method, QualType receiverTypeIfCall);
+
+  /// Check whether the given new method is a valid override of the
+  /// given overridden method, and set any properties that should be inherited.
+  void CheckObjCMethodOverride(ObjCMethodDecl *NewMethod,
+                               const ObjCMethodDecl *Overridden);
+
+  /// Describes the compatibility of a result type with its method.
+  enum ResultTypeCompatibilityKind {
+    RTC_Compatible,
+    RTC_Incompatible,
+    RTC_Unknown
+  };
+
+  void CheckObjCMethodDirectOverrides(ObjCMethodDecl *method,
+                                      ObjCMethodDecl *overridden);
+
+  void CheckObjCMethodOverrides(ObjCMethodDecl *ObjCMethod,
+                                ObjCInterfaceDecl *CurrentClass,
+                                ResultTypeCompatibilityKind RTC);
+
+  /// AddAnyMethodToGlobalPool - Add any method, instance or factory to global
+  /// pool.
+  void AddAnyMethodToGlobalPool(Decl *D);
+
+  void ActOnStartOfObjCMethodDef(Scope *S, Decl *D);
+  bool isObjCMethodDecl(Decl *D) { return D && isa<ObjCMethodDecl>(D); }
+
+  /// CheckImplementationIvars - This routine checks if the instance variables
+  /// listed in the implelementation match those listed in the interface.
+  void CheckImplementationIvars(ObjCImplementationDecl *ImpDecl,
+                                ObjCIvarDecl **Fields, unsigned nIvars,
+                                SourceLocation Loc);
+
+  void WarnConflictingTypedMethods(ObjCMethodDecl *Method,
+                                   ObjCMethodDecl *MethodDecl,
+                                   bool IsProtocolMethodDecl);
+
+  void CheckConflictingOverridingMethod(ObjCMethodDecl *Method,
+                                        ObjCMethodDecl *Overridden,
+                                        bool IsProtocolMethodDecl);
+
+  /// WarnExactTypedMethods - This routine issues a warning if method
+  /// implementation declaration matches exactly that of its declaration.
+  void WarnExactTypedMethods(ObjCMethodDecl *Method, ObjCMethodDecl *MethodDecl,
+                             bool IsProtocolMethodDecl);
+
+  /// MatchAllMethodDeclarations - Check methods declaraed in interface or
+  /// or protocol against those declared in their implementations.
+  void MatchAllMethodDeclarations(
+      const SelectorSet &InsMap, const SelectorSet &ClsMap,
+      SelectorSet &InsMapSeen, SelectorSet &ClsMapSeen, ObjCImplDecl *IMPDecl,
+      ObjCContainerDecl *IDecl, bool &IncompleteImpl, bool ImmediateClass,
+      bool WarnCategoryMethodImpl = false);
+
+  /// CheckCategoryVsClassMethodMatches - Checks that methods implemented in
+  /// category matches with those implemented in its primary class and
+  /// warns each time an exact match is found.
+  void CheckCategoryVsClassMethodMatches(ObjCCategoryImplDecl *CatIMP);
+
+  /// ImplMethodsVsClassMethods - This is main routine to warn if any method
+  /// remains unimplemented in the class or category \@implementation.
+  void ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl *IMPDecl,
+                                 ObjCContainerDecl *IDecl,
+                                 bool IncompleteImpl = false);
+
+  DeclGroupPtrTy ActOnForwardClassDeclaration(
+      SourceLocation Loc, IdentifierInfo **IdentList, SourceLocation *IdentLocs,
+      ArrayRef<ObjCTypeParamList *> TypeParamLists, unsigned NumElts);
+
+  /// MatchTwoMethodDeclarations - Checks if two methods' type match and returns
+  /// true, or false, accordingly.
+  bool MatchTwoMethodDeclarations(const ObjCMethodDecl *Method,
+                                  const ObjCMethodDecl *PrevMethod,
+                                  MethodMatchStrategy strategy = MMS_strict);
+
+  /// Add the given method to the list of globally-known methods.
+  void addMethodToGlobalList(ObjCMethodList *List, ObjCMethodDecl *Method);
+
+  void ReadMethodPool(Selector Sel);
+  void updateOutOfDateSelector(Selector Sel);
+
+  /// - Returns instance or factory methods in global method pool for
+  /// given selector. It checks the desired kind first, if none is found, and
+  /// parameter checkTheOther is set, it then checks the other kind. If no such
+  /// method or only one method is found, function returns false; otherwise, it
+  /// returns true.
+  bool
+  CollectMultipleMethodsInGlobalPool(Selector Sel,
+                                     SmallVectorImpl<ObjCMethodDecl *> &Methods,
+                                     bool InstanceFirst, bool CheckTheOther,
+                                     const ObjCObjectType *TypeBound = nullptr);
+
+  bool
+  AreMultipleMethodsInGlobalPool(Selector Sel, ObjCMethodDecl *BestMethod,
+                                 SourceRange R, bool receiverIdOrClass,
+                                 SmallVectorImpl<ObjCMethodDecl *> &Methods);
+
+  void
+  DiagnoseMultipleMethodInGlobalPool(SmallVectorImpl<ObjCMethodDecl *> &Methods,
+                                     Selector Sel, SourceRange R,
+                                     bool receiverIdOrClass);
+
+  const ObjCMethodDecl *
+  SelectorsForTypoCorrection(Selector Sel, QualType ObjectType = QualType());
+  /// LookupImplementedMethodInGlobalPool - Returns the method which has an
+  /// implementation.
+  ObjCMethodDecl *LookupImplementedMethodInGlobalPool(Selector Sel);
+
+  void DiagnoseDuplicateIvars(ObjCInterfaceDecl *ID, ObjCInterfaceDecl *SID);
+
+  /// Checks that the Objective-C declaration is declared in the global scope.
+  /// Emits an error and marks the declaration as invalid if it's not declared
+  /// in the global scope.
+  bool CheckObjCDeclScope(Decl *D);
+
+  void ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart,
+                 const IdentifierInfo *ClassName,
+                 SmallVectorImpl<Decl *> &Decls);
+
+  VarDecl *BuildObjCExceptionDecl(TypeSourceInfo *TInfo, QualType ExceptionType,
+                                  SourceLocation StartLoc, SourceLocation IdLoc,
+                                  const IdentifierInfo *Id,
+                                  bool Invalid = false);
+
+  Decl *ActOnObjCExceptionDecl(Scope *S, Declarator &D);
+
+  /// CollectIvarsToConstructOrDestruct - Collect those ivars which require
+  /// initialization.
+  void
+  CollectIvarsToConstructOrDestruct(ObjCInterfaceDecl *OI,
+                                    SmallVectorImpl<ObjCIvarDecl *> &Ivars);
+
+  void DiagnoseUseOfUnimplementedSelectors();
+
+  /// DiagnoseUnusedBackingIvarInAccessor - Issue an 'unused' warning if ivar
+  /// which backs the property is not used in the property's accessor.
+  void DiagnoseUnusedBackingIvarInAccessor(Scope *S,
+                                           const ObjCImplementationDecl *ImplD);
+
+  /// GetIvarBackingPropertyAccessor - If method is a property setter/getter and
+  /// it property has a backing ivar, returns this ivar; otherwise, returns
+  /// NULL. It also returns ivar's property on success.
+  ObjCIvarDecl *
+  GetIvarBackingPropertyAccessor(const ObjCMethodDecl *Method,
+                                 const ObjCPropertyDecl *&PDecl) const;
+
+  /// AddInstanceMethodToGlobalPool - All instance methods in a translation
+  /// unit are added to a global pool. This allows us to efficiently associate
+  /// a selector with a method declaraation for purposes of typechecking
+  /// messages sent to "id" (where the class of the object is unknown).
+  void AddInstanceMethodToGlobalPool(ObjCMethodDecl *Method,
+                                     bool impl = false) {
+    AddMethodToGlobalPool(Method, impl, /*instance*/ true);
+  }
+
+  /// AddFactoryMethodToGlobalPool - Same as above, but for factory methods.
+  void AddFactoryMethodToGlobalPool(ObjCMethodDecl *Method, bool impl = false) {
+    AddMethodToGlobalPool(Method, impl, /*instance*/ false);
+  }
+
+  QualType AdjustParameterTypeForObjCAutoRefCount(QualType T,
+                                                  SourceLocation NameLoc,
+                                                  TypeSourceInfo *TSInfo);
+
+  /// Look for an Objective-C class in the translation unit.
+  ///
+  /// \param Id The name of the Objective-C class we're looking for. If
+  /// typo-correction fixes this name, the Id will be updated
+  /// to the fixed name.
+  ///
+  /// \param IdLoc The location of the name in the translation unit.
+  ///
+  /// \param DoTypoCorrection If true, this routine will attempt typo correction
+  /// if there is no class with the given name.
+  ///
+  /// \returns The declaration of the named Objective-C class, or NULL if the
+  /// class could not be found.
+  ObjCInterfaceDecl *getObjCInterfaceDecl(const IdentifierInfo *&Id,
+                                          SourceLocation IdLoc,
+                                          bool TypoCorrection = false);
+
+  bool inferObjCARCLifetime(ValueDecl *decl);
+
+  /// SetIvarInitializers - This routine builds initialization ASTs for the
+  /// Objective-C implementation whose ivars need be initialized.
+  void SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation);
+
+  Decl *ActOnIvar(Scope *S, SourceLocation DeclStart, Declarator &D,
+                  Expr *BitWidth, tok::ObjCKeywordKind visibility);
+
+  ObjCContainerDecl *getObjCDeclContext() const;
+
+private:
+  /// AddMethodToGlobalPool - Add an instance or factory method to the global
+  /// pool. See descriptoin of AddInstanceMethodToGlobalPool.
+  void AddMethodToGlobalPool(ObjCMethodDecl *Method, bool impl, bool instance);
+
+  /// LookupMethodInGlobalPool - Returns the instance or factory method and
+  /// optionally warns if there are multiple signatures.
+  ObjCMethodDecl *LookupMethodInGlobalPool(Selector Sel, SourceRange R,
+                                           bool receiverIdOrClass,
+                                           bool instance);
+
+  ///@}
+
+  //
+  //
+  // -------------------------------------------------------------------------
+  //
+  //
+
+  /// \name ObjC Expressions
+  /// Implementations are in SemaExprObjC.cpp
+  ///@{
+
+public:
+  /// Caches identifiers/selectors for NSFoundation APIs.
+  std::unique_ptr<NSAPI> NSAPIObj;
+
+  /// The declaration of the Objective-C NSNumber class.
+  ObjCInterfaceDecl *NSNumberDecl;
+
+  /// The declaration of the Objective-C NSValue class.
+  ObjCInterfaceDecl *NSValueDecl;
+
+  /// Pointer to NSNumber type (NSNumber *).
+  QualType NSNumberPointer;
+
+  /// Pointer to NSValue type (NSValue *).
+  QualType NSValuePointer;
+
+  /// The Objective-C NSNumber methods used to create NSNumber literals.
+  ObjCMethodDecl *NSNumberLiteralMethods[NSAPI::NumNSNumberLiteralMethods];
+
+  /// The declaration of the Objective-C NSString class.
+  ObjCInterfaceDecl *NSStringDecl;
+
+  /// Pointer to NSString type (NSString *).
+  QualType NSStringPointer;
+
+  /// The declaration of the stringWithUTF8String: method.
+  ObjCMethodDecl *StringWithUTF8StringMethod;
+
+  /// The declaration of the valueWithBytes:objCType: method.
+  ObjCMethodDecl *ValueWithBytesObjCTypeMethod;
+
+  /// The declaration of the Objective-C NSArray class.
+  ObjCInterfaceDecl *NSArrayDecl;
+
+  /// The declaration of the arrayWithObjects:count: method.
+  ObjCMethodDecl *ArrayWithObjectsMethod;
+
+  /// The declaration of the Objective-C NSDictionary class.
+  ObjCInterfaceDecl *NSDictionaryDecl;
+
+  /// The declaration of the dictionaryWithObjects:forKeys:count: method.
+  ObjCMethodDecl *DictionaryWithObjectsMethod;
+
+  /// id<NSCopying> type.
+  QualType QIDNSCopying;
+
+  /// will hold 'respondsToSelector:'
+  Selector RespondsToSelectorSel;
+
+  ExprResult HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
+                                       Expr *BaseExpr, SourceLocation OpLoc,
+                                       DeclarationName MemberName,
+                                       SourceLocation MemberLoc,
+                                       SourceLocation SuperLoc,
+                                       QualType SuperType, bool Super);
+
+  ExprResult ActOnClassPropertyRefExpr(const IdentifierInfo &receiverName,
+                                       const IdentifierInfo &propertyName,
+                                       SourceLocation receiverNameLoc,
+                                       SourceLocation propertyNameLoc);
+
+  // ParseObjCStringLiteral - Parse Objective-C string literals.
+  ExprResult ParseObjCStringLiteral(SourceLocation *AtLocs,
+                                    ArrayRef<Expr *> Strings);
+
+  ExprResult BuildObjCStringLiteral(SourceLocation AtLoc, StringLiteral *S);
+
+  /// BuildObjCNumericLiteral - builds an ObjCBoxedExpr AST node for the
+  /// numeric literal expression. Type of the expression will be "NSNumber *"
+  /// or "id" if NSNumber is unavailable.
+  ExprResult BuildObjCNumericLiteral(SourceLocation AtLoc, Expr *Number);
+  ExprResult ActOnObjCBoolLiteral(SourceLocation AtLoc, SourceLocation ValueLoc,
+                                  bool Value);
+  ExprResult BuildObjCArrayLiteral(SourceRange SR, MultiExprArg Elements);
+
+  /// BuildObjCBoxedExpr - builds an ObjCBoxedExpr AST node for the
+  /// '@' prefixed parenthesized expression. The type of the expression will
+  /// either be "NSNumber *", "NSString *" or "NSValue *" depending on the type
+  /// of ValueType, which is allowed to be a built-in numeric type, "char *",
+  /// "const char *" or C structure with attribute 'objc_boxable'.
+  ExprResult BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr);
+
+  ExprResult BuildObjCSubscriptExpression(SourceLocation RB, Expr *BaseExpr,
+                                          Expr *IndexExpr,
+                                          ObjCMethodDecl *getterMethod,
+                                          ObjCMethodDecl *setterMethod);
+
+  ExprResult
+  BuildObjCDictionaryLiteral(SourceRange SR,
+                             MutableArrayRef<ObjCDictionaryElement> Elements);
+
+  ExprResult BuildObjCEncodeExpression(SourceLocation AtLoc,
+                                       TypeSourceInfo *EncodedTypeInfo,
+                                       SourceLocation RParenLoc);
+
+  ExprResult ParseObjCEncodeExpression(SourceLocation AtLoc,
+                                       SourceLocation EncodeLoc,
+                                       SourceLocation LParenLoc, ParsedType Ty,
+                                       SourceLocation RParenLoc);
+
+  /// ParseObjCSelectorExpression - Build selector expression for \@selector
+  ExprResult ParseObjCSelectorExpression(Selector Sel, SourceLocation AtLoc,
+                                         SourceLocation SelLoc,
+                                         SourceLocation LParenLoc,
+                                         SourceLocation RParenLoc,
+                                         bool WarnMultipleSelectors);
+
+  /// ParseObjCProtocolExpression - Build protocol expression for \@protocol
+  ExprResult ParseObjCProtocolExpression(IdentifierInfo *ProtocolName,
+                                         SourceLocation AtLoc,
+                                         SourceLocation ProtoLoc,
+                                         SourceLocation LParenLoc,
+                                         SourceLocation ProtoIdLoc,
+                                         SourceLocation RParenLoc);
+
+  ObjCMethodDecl *tryCaptureObjCSelf(SourceLocation Loc);
+
+  /// Describes the kind of message expression indicated by a message
+  /// send that starts with an identifier.
+  enum ObjCMessageKind {
+    /// The message is sent to 'super'.
+    ObjCSuperMessage,
+    /// The message is an instance message.
+    ObjCInstanceMessage,
+    /// The message is a class message, and the identifier is a type
+    /// name.
+    ObjCClassMessage
+  };
+
+  ObjCMessageKind getObjCMessageKind(Scope *S, IdentifierInfo *Name,
+                                     SourceLocation NameLoc, bool IsSuper,
+                                     bool HasTrailingDot,
+                                     ParsedType &ReceiverType);
+
+  ExprResult ActOnSuperMessage(Scope *S, SourceLocation SuperLoc, Selector Sel,
+                               SourceLocation LBracLoc,
+                               ArrayRef<SourceLocation> SelectorLocs,
+                               SourceLocation RBracLoc, MultiExprArg Args);
+
+  ExprResult BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
+                               QualType ReceiverType, SourceLocation SuperLoc,
+                               Selector Sel, ObjCMethodDecl *Method,
+                               SourceLocation LBracLoc,
+                               ArrayRef<SourceLocation> SelectorLocs,
+                               SourceLocation RBracLoc, MultiExprArg Args,
+                               bool isImplicit = false);
+
+  ExprResult BuildClassMessageImplicit(QualType ReceiverType,
+                                       bool isSuperReceiver, SourceLocation Loc,
+                                       Selector Sel, ObjCMethodDecl *Method,
+                                       MultiExprArg Args);
+
+  ExprResult ActOnClassMessage(Scope *S, ParsedType Receiver, Selector Sel,
+                               SourceLocation LBracLoc,
+                               ArrayRef<SourceLocation> SelectorLocs,
+                               SourceLocation RBracLoc, MultiExprArg Args);
+
+  ExprResult BuildInstanceMessage(Expr *Receiver, QualType ReceiverType,
+                                  SourceLocation SuperLoc, Selector Sel,
+                                  ObjCMethodDecl *Method,
+                                  SourceLocation LBracLoc,
+                                  ArrayRef<SourceLocation> SelectorLocs,
+                                  SourceLocation RBracLoc, MultiExprArg Args,
+                                  bool isImplicit = false);
+
+  ExprResult BuildInstanceMessageImplicit(Expr *Receiver, QualType ReceiverType,
+                                          SourceLocation Loc, Selector Sel,
+                                          ObjCMethodDecl *Method,
+                                          MultiExprArg Args);
+
+  ExprResult ActOnInstanceMessage(Scope *S, Expr *Receiver, Selector Sel,
+                                  SourceLocation LBracLoc,
+                                  ArrayRef<SourceLocation> SelectorLocs,
+                                  SourceLocation RBracLoc, MultiExprArg Args);
+
+  ExprResult BuildObjCBridgedCast(SourceLocation LParenLoc,
+                                  ObjCBridgeCastKind Kind,
+                                  SourceLocation BridgeKeywordLoc,
+                                  TypeSourceInfo *TSInfo, Expr *SubExpr);
+
+  ExprResult ActOnObjCBridgedCast(Scope *S, SourceLocation LParenLoc,
+                                  ObjCBridgeCastKind Kind,
+                                  SourceLocation BridgeKeywordLoc,
+                                  ParsedType Type, SourceLocation RParenLoc,
+                                  Expr *SubExpr);
+
+  void CheckTollFreeBridgeCast(QualType castType, Expr *castExpr);
+
+  void CheckObjCBridgeRelatedCast(QualType castType, Expr *castExpr);
+
+  bool CheckTollFreeBridgeStaticCast(QualType castType, Expr *castExpr,
+                                     CastKind &Kind);
+
+  bool checkObjCBridgeRelatedComponents(SourceLocation Loc, QualType DestType,
+                                        QualType SrcType,
+                                        ObjCInterfaceDecl *&RelatedClass,
+                                        ObjCMethodDecl *&ClassMethod,
+                                        ObjCMethodDecl *&InstanceMethod,
+                                        TypedefNameDecl *&TDNDecl, bool CfToNs,
+                                        bool Diagnose = true);
+
+  bool CheckObjCBridgeRelatedConversions(SourceLocation Loc, QualType DestType,
+                                         QualType SrcType, Expr *&SrcExpr,
+                                         bool Diagnose = true);
+
+  /// Private Helper predicate to check for 'self'.
+  bool isSelfExpr(Expr *RExpr);
+  bool isSelfExpr(Expr *RExpr, const ObjCMethodDecl *Method);
+
+  ObjCMethodDecl *LookupMethodInQualifiedType(Selector Sel,
+                                              const ObjCObjectPointerType *OPT,
+                                              bool IsInstance);
+  ObjCMethodDecl *LookupMethodInObjectType(Selector Sel, QualType Ty,
+                                           bool IsInstance);
+
+  bool isKnownName(StringRef name);
+
+  enum ARCConversionResult { ACR_okay, ACR_unbridged, ACR_error };
+
+  /// Checks for invalid conversions and casts between
+  /// retainable pointers and other pointer kinds for ARC and Weak.
+  ARCConversionResult CheckObjCConversion(SourceRange castRange,
+                                          QualType castType, Expr *&op,
+                                          CheckedConversionKind CCK,
+                                          bool Diagnose = true,
+                                          bool DiagnoseCFAudited = false,
+                                          BinaryOperatorKind Opc = BO_PtrMemD);
+
+  Expr *stripARCUnbridgedCast(Expr *e);
+  void diagnoseARCUnbridgedCast(Expr *e);
+
+  bool CheckObjCARCUnavailableWeakConversion(QualType castType,
+                                             QualType ExprType);
+
+  /// CheckMessageArgumentTypes - Check types in an Obj-C message send.
+  /// \param Method - May be null.
+  /// \param [out] ReturnType - The return type of the send.
+  /// \return true iff there were any incompatible types.
+  bool CheckMessageArgumentTypes(const Expr *Receiver, QualType ReceiverType,
+                                 MultiExprArg Args, Selector Sel,
+                                 ArrayRef<SourceLocation> SelectorLocs,
+                                 ObjCMethodDecl *Method, bool isClassMessage,
+                                 bool isSuperMessage, SourceLocation lbrac,
+                                 SourceLocation rbrac, SourceRange RecRange,
+                                 QualType &ReturnType, ExprValueKind &VK);
+
+  /// Determine the result of a message send expression based on
+  /// the type of the receiver, the method expected to receive the message,
+  /// and the form of the message send.
+  QualType getMessageSendResultType(const Expr *Receiver, QualType ReceiverType,
+                                    ObjCMethodDecl *Method, bool isClassMessage,
+                                    bool isSuperMessage);
+
+  /// If the given expression involves a message send to a method
+  /// with a related result type, emit a note describing what happened.
+  void EmitRelatedResultTypeNote(const Expr *E);
+
+  /// Given that we had incompatible pointer types in a return
+  /// statement, check whether we're in a method with a related result
+  /// type, and if so, emit a note describing what happened.
+  void EmitRelatedResultTypeNoteForReturn(QualType destType);
+
+  /// LookupInstanceMethodInGlobalPool - Returns the method and warns if
+  /// there are multiple signatures.
+  ObjCMethodDecl *
+  LookupInstanceMethodInGlobalPool(Selector Sel, SourceRange R,
+                                   bool receiverIdOrClass = false) {
+    return LookupMethodInGlobalPool(Sel, R, receiverIdOrClass,
+                                    /*instance*/ true);
+  }
+
+  /// LookupFactoryMethodInGlobalPool - Returns the method and warns if
+  /// there are multiple signatures.
+  ObjCMethodDecl *
+  LookupFactoryMethodInGlobalPool(Selector Sel, SourceRange R,
+                                  bool receiverIdOrClass = false) {
+    return LookupMethodInGlobalPool(Sel, R, receiverIdOrClass,
+                                    /*instance*/ false);
+  }
+
+  /// The parser has read a name in, and Sema has detected that we're currently
+  /// inside an ObjC method. Perform some additional checks and determine if we
+  /// should form a reference to an ivar.
+  ///
+  /// Ideally, most of this would be done by lookup, but there's
+  /// actually quite a lot of extra work involved.
+  DeclResult LookupIvarInObjCMethod(LookupResult &Lookup, Scope *S,
+                                    IdentifierInfo *II);
+
+  /// The parser has read a name in, and Sema has detected that we're currently
+  /// inside an ObjC method. Perform some additional checks and determine if we
+  /// should form a reference to an ivar. If so, build an expression referencing
+  /// that ivar.
+  ExprResult LookupInObjCMethod(LookupResult &LookUp, Scope *S,
+                                IdentifierInfo *II,
+                                bool AllowBuiltinCreation = false);
+
+  ExprResult BuildIvarRefExpr(Scope *S, SourceLocation Loc, ObjCIvarDecl *IV);
+
+  /// FindCompositeObjCPointerType - Helper method to find composite type of
+  /// two objective-c pointer types of the two input expressions.
+  QualType FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS,
+                                        SourceLocation QuestionLoc);
+
+  bool CheckConversionToObjCLiteral(QualType DstType, Expr *&SrcExpr,
+                                    bool Diagnose = true);
+
+  /// ActOnObjCBoolLiteral - Parse {__objc_yes,__objc_no} literals.
+  ExprResult ActOnObjCBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind);
+
+  ExprResult
+  ActOnObjCAvailabilityCheckExpr(llvm::ArrayRef<AvailabilitySpec> AvailSpecs,
+                                 SourceLocation AtLoc, SourceLocation RParen);
+
+  /// Prepare a conversion of the given expression to an ObjC object
+  /// pointer type.
+  CastKind PrepareCastToObjCObjectPointer(ExprResult &E);
+
+  // Note that LK_String is intentionally after the other literals, as
+  // this is used for diagnostics logic.
+  enum ObjCLiteralKind {
+    LK_Array,
+    LK_Dictionary,
+    LK_Numeric,
+    LK_Boxed,
+    LK_String,
+    LK_Block,
+    LK_None
+  };
+  ObjCLiteralKind CheckLiteralKind(Expr *FromE);
+
+  ///@}
+
+  //
+  //
+  // -------------------------------------------------------------------------
+  //
+  //
+
+  /// \name ObjC @property and @synthesize
+  /// Implementations are in SemaObjCProperty.cpp
+  ///@{
+
+public:
+  /// Ensure attributes are consistent with type.
+  /// \param [in, out] Attributes The attributes to check; they will
+  /// be modified to be consistent with \p PropertyTy.
+  void CheckObjCPropertyAttributes(Decl *PropertyPtrTy, SourceLocation Loc,
+                                   unsigned &Attributes,
+                                   bool propertyInPrimaryClass);
+
+  /// Process the specified property declaration and create decls for the
+  /// setters and getters as needed.
+  /// \param property The property declaration being processed
+  void ProcessPropertyDecl(ObjCPropertyDecl *property);
+
+  Decl *ActOnProperty(Scope *S, SourceLocation AtLoc, SourceLocation LParenLoc,
+                      FieldDeclarator &FD, ObjCDeclSpec &ODS,
+                      Selector GetterSel, Selector SetterSel,
+                      tok::ObjCKeywordKind MethodImplKind,
+                      DeclContext *lexicalDC = nullptr);
+
+  Decl *ActOnPropertyImplDecl(Scope *S, SourceLocation AtLoc,
+                              SourceLocation PropertyLoc, bool ImplKind,
+                              IdentifierInfo *PropertyId,
+                              IdentifierInfo *PropertyIvar,
+                              SourceLocation PropertyIvarLoc,
+                              ObjCPropertyQueryKind QueryKind);
+
+  /// Called by ActOnProperty to handle \@property declarations in
+  /// class extensions.
+  ObjCPropertyDecl *HandlePropertyInClassExtension(
+      Scope *S, SourceLocation AtLoc, SourceLocation LParenLoc,
+      FieldDeclarator &FD, Selector GetterSel, SourceLocation GetterNameLoc,
+      Selector SetterSel, SourceLocation SetterNameLoc, const bool isReadWrite,
+      unsigned &Attributes, const unsigned AttributesAsWritten, QualType T,
+      TypeSourceInfo *TSI, tok::ObjCKeywordKind MethodImplKind);
+
+  /// Called by ActOnProperty and HandlePropertyInClassExtension to
+  /// handle creating the ObjcPropertyDecl for a category or \@interface.
+  ObjCPropertyDecl *
+  CreatePropertyDecl(Scope *S, ObjCContainerDecl *CDecl, SourceLocation AtLoc,
+                     SourceLocation LParenLoc, FieldDeclarator &FD,
+                     Selector GetterSel, SourceLocation GetterNameLoc,
+                     Selector SetterSel, SourceLocation SetterNameLoc,
+                     const bool isReadWrite, const unsigned Attributes,
+                     const unsigned AttributesAsWritten, QualType T,
+                     TypeSourceInfo *TSI, tok::ObjCKeywordKind MethodImplKind,
+                     DeclContext *lexicalDC = nullptr);
+
+  void DiagnosePropertyMismatch(ObjCPropertyDecl *Property,
+                                ObjCPropertyDecl *SuperProperty,
+                                const IdentifierInfo *Name,
+                                bool OverridingProtocolProperty);
+
+  bool DiagnosePropertyAccessorMismatch(ObjCPropertyDecl *PD,
+                                        ObjCMethodDecl *Getter,
+                                        SourceLocation Loc);
+
+  /// DiagnoseUnimplementedProperties - This routine warns on those properties
+  /// which must be implemented by this implementation.
+  void DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl *IMPDecl,
+                                       ObjCContainerDecl *CDecl,
+                                       bool SynthesizeProperties);
+
+  /// Diagnose any null-resettable synthesized setters.
+  void diagnoseNullResettableSynthesizedSetters(const ObjCImplDecl *impDecl);
+
+  /// DefaultSynthesizeProperties - This routine default synthesizes all
+  /// properties which must be synthesized in the class's \@implementation.
+  void DefaultSynthesizeProperties(Scope *S, ObjCImplDecl *IMPDecl,
+                                   ObjCInterfaceDecl *IDecl,
+                                   SourceLocation AtEnd);
+  void DefaultSynthesizeProperties(Scope *S, Decl *D, SourceLocation AtEnd);
+
+  /// IvarBacksCurrentMethodAccessor - This routine returns 'true' if 'IV' is
+  /// an ivar synthesized for 'Method' and 'Method' is a property accessor
+  /// declared in class 'IFace'.
+  bool IvarBacksCurrentMethodAccessor(ObjCInterfaceDecl *IFace,
+                                      ObjCMethodDecl *Method, ObjCIvarDecl *IV);
+
+  void DiagnoseOwningPropertyGetterSynthesis(const ObjCImplementationDecl *D);
+
+  void
+  DiagnoseMissingDesignatedInitOverrides(const ObjCImplementationDecl *ImplD,
+                                         const ObjCInterfaceDecl *IFD);
+
+  /// AtomicPropertySetterGetterRules - This routine enforces the rule (via
+  /// warning) when atomic property has one but not the other user-declared
+  /// setter or getter.
+  void AtomicPropertySetterGetterRules(ObjCImplDecl *IMPDecl,
+                                       ObjCInterfaceDecl *IDecl);
+
+  ///@}
+};
+
+} // namespace clang
+
+#endif // LLVM_CLANG_SEMA_SEMAOBJC_H
diff --git a/clang/lib/ARCMigrate/Transforms.cpp b/clang/lib/ARCMigrate/Transforms.cpp
index 2808e35135dc..fda0e1c932fc 100644
--- a/clang/lib/ARCMigrate/Transforms.cpp
+++ b/clang/lib/ARCMigrate/Transforms.cpp
@@ -17,6 +17,7 @@
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaObjC.h"
 
 using namespace clang;
 using namespace arcmt;
@@ -26,8 +27,8 @@ ASTTraverser::~ASTTraverser() { }
 
 bool MigrationPass::CFBridgingFunctionsDefined() {
   if (!EnableCFBridgeFns)
-    EnableCFBridgeFns = SemaRef.isKnownName("CFBridgingRetain") &&
-                        SemaRef.isKnownName("CFBridgingRelease");
+    EnableCFBridgeFns = SemaRef.ObjC().isKnownName("CFBridgingRetain") &&
+                        SemaRef.ObjC().isKnownName("CFBridgingRelease");
   return *EnableCFBridgeFns;
 }
 
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 7fbaee5690bd..6d026878d327 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -28,6 +28,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaDiagnostic.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallString.h"
@@ -3945,7 +3946,7 @@ void Parser::ParseDeclarationSpecifiers(
 
       if (DSContext == DeclSpecContext::DSC_objc_method_result &&
           isObjCInstancetype()) {
-        ParsedType TypeRep = Actions.ActOnObjCInstanceType(Loc);
+        ParsedType TypeRep = Actions.ObjC().ActOnObjCInstanceType(Loc);
         assert(TypeRep);
         isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec,
                                        DiagID, TypeRep, Policy);
@@ -5002,8 +5003,8 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
         continue;
       }
       SmallVector<Decl *, 16> Fields;
-      Actions.ActOnDefs(getCurScope(), TagDecl, Tok.getLocation(),
-                        Tok.getIdentifierInfo(), Fields);
+      Actions.ObjC().ActOnDefs(getCurScope(), TagDecl, Tok.getLocation(),
+                               Tok.getIdentifierInfo(), Fields);
       ConsumeToken();
       ExpectAndConsume(tok::r_paren);
     }
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 0551b8314f9f..9cd7d20fc4d5 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -31,6 +31,7 @@
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaCUDA.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenACC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/SemaSYCL.h"
@@ -1227,8 +1228,8 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
       IdentifierInfo &PropertyName = *Tok.getIdentifierInfo();
       SourceLocation PropertyLoc = ConsumeToken();
 
-      Res = Actions.ActOnClassPropertyRefExpr(II, PropertyName,
-                                              ILoc, PropertyLoc);
+      Res = Actions.ObjC().ActOnClassPropertyRefExpr(II, PropertyName, ILoc,
+                                                     PropertyLoc);
       break;
     }
 
@@ -3093,9 +3094,9 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     if (Ty.isInvalid() || SubExpr.isInvalid())
       return ExprError();
 
-    return Actions.ActOnObjCBridgedCast(getCurScope(), OpenLoc, Kind,
-                                        BridgeKeywordLoc, Ty.get(),
-                                        RParenLoc, SubExpr.get());
+    return Actions.ObjC().ActOnObjCBridgedCast(getCurScope(), OpenLoc, Kind,
+                                               BridgeKeywordLoc, Ty.get(),
+                                               RParenLoc, SubExpr.get());
   } else if (ExprType >= CompoundLiteral &&
              isTypeIdInParens(isAmbiguousTypeId)) {
 
@@ -3811,7 +3812,7 @@ ExprResult Parser::ParseBlockLiteralExpression() {
 ///         '__objc_no'
 ExprResult Parser::ParseObjCBoolLiteral() {
   tok::TokenKind Kind = Tok.getKind();
-  return Actions.ActOnObjCBoolLiteral(ConsumeToken(), Kind);
+  return Actions.ObjC().ActOnObjCBoolLiteral(ConsumeToken(), Kind);
 }
 
 /// Validate availability spec list, emitting diagnostics if necessary. Returns
@@ -3932,6 +3933,6 @@ ExprResult Parser::ParseAvailabilityCheckExpr(SourceLocation BeginLoc) {
   if (Parens.consumeClose())
     return ExprError();
 
-  return Actions.ActOnObjCAvailabilityCheckExpr(AvailSpecs, BeginLoc,
-                                                Parens.getCloseLocation());
+  return Actions.ObjC().ActOnObjCAvailabilityCheckExpr(
+      AvailSpecs, BeginLoc, Parens.getCloseLocation());
 }
diff --git a/clang/lib/Parse/ParseInit.cpp b/clang/lib/Parse/ParseInit.cpp
index 423497bfcb66..04e4419f4d45 100644
--- a/clang/lib/Parse/ParseInit.cpp
+++ b/clang/lib/Parse/ParseInit.cpp
@@ -18,6 +18,7 @@
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 using namespace clang;
@@ -290,15 +291,15 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator(
       // Three cases. This is a message send to a type: [type foo]
       // This is a message send to super:  [super foo]
       // This is a message sent to an expr:  [super.bar foo]
-      switch (Actions.getObjCMessageKind(
+      switch (Actions.ObjC().getObjCMessageKind(
           getCurScope(), II, IILoc, II == Ident_super,
           NextToken().is(tok::period), ReceiverType)) {
-      case Sema::ObjCSuperMessage:
+      case SemaObjC::ObjCSuperMessage:
         CheckArrayDesignatorSyntax(*this, StartLoc, Desig);
         return ParseAssignmentExprWithObjCMessageExprStart(
             StartLoc, ConsumeToken(), nullptr, nullptr);
 
-      case Sema::ObjCClassMessage:
+      case SemaObjC::ObjCClassMessage:
         CheckArrayDesignatorSyntax(*this, StartLoc, Desig);
         ConsumeToken(); // the identifier
         if (!ReceiverType) {
@@ -326,7 +327,7 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator(
                                                            ReceiverType,
                                                            nullptr);
 
-      case Sema::ObjCInstanceMessage:
+      case SemaObjC::ObjCInstanceMessage:
         // Fall through; we'll just parse the expression and
         // (possibly) treat this like an Objective-C message send
         // later.
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index 8e54fe012c55..4cb04b353473 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -20,6 +20,7 @@
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 
@@ -132,7 +133,7 @@ public:
 
   void leave() {
     if (Params)
-      Actions.popObjCTypeParamList(S, Params);
+      Actions.ObjC().popObjCTypeParamList(S, Params);
     Params = nullptr;
   }
 };
@@ -179,23 +180,22 @@ Parser::ParseObjCAtClassDeclaration(SourceLocation atLoc) {
   if (ExpectAndConsume(tok::semi, diag::err_expected_after, "@class"))
     return Actions.ConvertDeclToDeclGroup(nullptr);
 
-  return Actions.ActOnForwardClassDeclaration(atLoc, ClassNames.data(),
-                                              ClassLocs.data(),
-                                              ClassTypeParams,
-                                              ClassNames.size());
+  return Actions.ObjC().ActOnForwardClassDeclaration(
+      atLoc, ClassNames.data(), ClassLocs.data(), ClassTypeParams,
+      ClassNames.size());
 }
 
 void Parser::CheckNestedObjCContexts(SourceLocation AtLoc)
 {
-  Sema::ObjCContainerKind ock = Actions.getObjCContainerKind();
-  if (ock == Sema::OCK_None)
+  SemaObjC::ObjCContainerKind ock = Actions.ObjC().getObjCContainerKind();
+  if (ock == SemaObjC::OCK_None)
     return;
 
-  Decl *Decl = Actions.getObjCDeclContext();
+  Decl *Decl = Actions.ObjC().getObjCDeclContext();
   if (CurParsedObjCImpl) {
     CurParsedObjCImpl->finish(AtLoc);
   } else {
-    Actions.ActOnAtEnd(getCurScope(), AtLoc);
+    Actions.ObjC().ActOnAtEnd(getCurScope(), AtLoc);
   }
   Diag(AtLoc, diag::err_objc_missing_end)
       << FixItHint::CreateInsertion(AtLoc, "@end\n");
@@ -305,7 +305,7 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
                                     /*consumeLastToken=*/true))
       return nullptr;
 
-    ObjCCategoryDecl *CategoryType = Actions.ActOnStartCategoryInterface(
+    ObjCCategoryDecl *CategoryType = Actions.ObjC().ActOnStartCategoryInterface(
         AtLoc, nameId, nameLoc, typeParameterList, categoryId, categoryLoc,
         ProtocolRefs.data(), ProtocolRefs.size(), ProtocolLocs.data(),
         EndProtoLoc, attrs);
@@ -360,9 +360,9 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
       for (const auto &pair : ProtocolIdents) {
         protocolLocs.push_back(pair.second);
       }
-      Actions.FindProtocolDeclaration(/*WarnOnDeclarations=*/true,
-                                      /*ForObjCContainer=*/true,
-                                      ProtocolIdents, protocols);
+      Actions.ObjC().FindProtocolDeclaration(/*WarnOnDeclarations=*/true,
+                                             /*ForObjCContainer=*/true,
+                                             ProtocolIdents, protocols);
     }
   } else if (protocols.empty() && Tok.is(tok::less) &&
              ParseObjCProtocolReferences(protocols, protocolLocs, true, true,
@@ -372,11 +372,11 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
   }
 
   if (Tok.isNot(tok::less))
-    Actions.ActOnTypedefedProtocols(protocols, protocolLocs,
-                                    superClassId, superClassLoc);
+    Actions.ObjC().ActOnTypedefedProtocols(protocols, protocolLocs,
+                                           superClassId, superClassLoc);
 
   SkipBodyInfo SkipBody;
-  ObjCInterfaceDecl *ClsType = Actions.ActOnStartClassInterface(
+  ObjCInterfaceDecl *ClsType = Actions.ObjC().ActOnStartClassInterface(
       getCurScope(), AtLoc, nameId, nameLoc, typeParameterList, superClassId,
       superClassLoc, typeArgs,
       SourceRange(typeArgsLAngleLoc, typeArgsRAngleLoc), protocols.data(),
@@ -468,7 +468,7 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
   auto makeProtocolIdentsIntoTypeParameters = [&]() {
     unsigned index = 0;
     for (const auto &pair : protocolIdents) {
-      DeclResult typeParam = Actions.actOnObjCTypeParam(
+      DeclResult typeParam = Actions.ObjC().actOnObjCTypeParam(
           getCurScope(), ObjCTypeParamVariance::Invariant, SourceLocation(),
           index++, pair.first, pair.second, SourceLocation(), nullptr);
       if (typeParam.isUsable())
@@ -546,7 +546,7 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
     }
 
     // Create the type parameter.
-    DeclResult typeParam = Actions.actOnObjCTypeParam(
+    DeclResult typeParam = Actions.ObjC().actOnObjCTypeParam(
         getCurScope(), variance, varianceLoc, typeParams.size(), paramName,
         paramLoc, colonLoc, boundType.isUsable() ? boundType.get() : nullptr);
     if (typeParam.isUsable())
@@ -587,11 +587,8 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
   }
 
   // Form the type parameter list and enter its scope.
-  ObjCTypeParamList *list = Actions.actOnObjCTypeParamList(
-                              getCurScope(),
-                              lAngleLoc,
-                              typeParams,
-                              rAngleLoc);
+  ObjCTypeParamList *list = Actions.ObjC().actOnObjCTypeParamList(
+      getCurScope(), lAngleLoc, typeParams, rAngleLoc);
   Scope.enter(list);
 
   // Clear out the angle locations; they're used by the caller to indicate
@@ -811,7 +808,7 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
           SetterSel = SelectorTable::constructSetterSelector(
               PP.getIdentifierTable(), PP.getSelectorTable(),
               FD.D.getIdentifier());
-        Decl *Property = Actions.ActOnProperty(
+        Decl *Property = Actions.ObjC().ActOnProperty(
             getCurScope(), AtLoc, LParenLoc, FD, OCDS, GetterSel, SetterSel,
             MethodImplKind);
 
@@ -836,14 +833,14 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
     Diag(Tok, diag::err_objc_missing_end)
         << FixItHint::CreateInsertion(Tok.getLocation(), "\n@end\n");
     Diag(CDecl->getBeginLoc(), diag::note_objc_container_start)
-        << (int)Actions.getObjCContainerKind();
+        << (int)Actions.ObjC().getObjCContainerKind();
     AtEnd.setBegin(Tok.getLocation());
     AtEnd.setEnd(Tok.getLocation());
   }
 
   // Insert collected methods declarations into the @interface object.
   // This passes in an invalid SourceLocation for AtEndLoc when EOF is hit.
-  Actions.ActOnAtEnd(getCurScope(), AtEnd, allMethods, allTUVariables);
+  Actions.ObjC().ActOnAtEnd(getCurScope(), AtEnd, allMethods, allTUVariables);
 }
 
 /// Diagnose redundant or conflicting nullability information.
@@ -1437,7 +1434,7 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
                          methodAttrs);
 
     Selector Sel = PP.getSelectorTable().getNullarySelector(SelIdent);
-    Decl *Result = Actions.ActOnMethodDeclaration(
+    Decl *Result = Actions.ObjC().ActOnMethodDeclaration(
         getCurScope(), mLoc, Tok.getLocation(), mType, DSRet, ReturnType,
         selLoc, Sel, nullptr, CParamInfo.data(), CParamInfo.size(), methodAttrs,
         MethodImplKind, false, MethodDefinition);
@@ -1447,14 +1444,14 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
   SmallVector<const IdentifierInfo *, 12> KeyIdents;
   SmallVector<SourceLocation, 12> KeyLocs;
-  SmallVector<Sema::ObjCArgInfo, 12> ArgInfos;
+  SmallVector<SemaObjC::ObjCArgInfo, 12> ArgInfos;
   ParseScope PrototypeScope(this, Scope::FunctionPrototypeScope |
                             Scope::FunctionDeclarationScope | Scope::DeclScope);
 
   AttributePool allParamAttrs(AttrFactory);
   while (true) {
     ParsedAttributes paramAttrs(AttrFactory);
-    Sema::ObjCArgInfo ArgInfo;
+    SemaObjC::ObjCArgInfo ArgInfo;
 
     // Each iteration parses a single keyword argument.
     if (ExpectAndConsume(tok::colon))
@@ -1559,7 +1556,7 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
   Selector Sel = PP.getSelectorTable().getSelector(KeyIdents.size(),
                                                    &KeyIdents[0]);
-  Decl *Result = Actions.ActOnMethodDeclaration(
+  Decl *Result = Actions.ObjC().ActOnMethodDeclaration(
       getCurScope(), mLoc, Tok.getLocation(), mType, DSRet, ReturnType, KeyLocs,
       Sel, &ArgInfos[0], CParamInfo.data(), CParamInfo.size(), methodAttrs,
       MethodImplKind, isVariadic, MethodDefinition);
@@ -1609,8 +1606,8 @@ ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
     return true;
 
   // Convert the list of protocols identifiers into a list of protocol decls.
-  Actions.FindProtocolDeclaration(WarnOnDeclarations, ForObjCContainer,
-                                  ProtocolIdents, Protocols);
+  Actions.ObjC().FindProtocolDeclaration(WarnOnDeclarations, ForObjCContainer,
+                                         ProtocolIdents, Protocols);
   return false;
 }
 
@@ -1624,10 +1621,8 @@ TypeResult Parser::parseObjCProtocolQualifierType(SourceLocation &rAngleLoc) {
   (void)ParseObjCProtocolReferences(protocols, protocolLocs, false, false,
                                     lAngleLoc, rAngleLoc,
                                     /*consumeLastToken=*/true);
-  TypeResult result = Actions.actOnObjCProtocolQualifierType(lAngleLoc,
-                                                             protocols,
-                                                             protocolLocs,
-                                                             rAngleLoc);
+  TypeResult result = Actions.ObjC().actOnObjCProtocolQualifierType(
+      lAngleLoc, protocols, protocolLocs, rAngleLoc);
   if (result.isUsable()) {
     Diag(lAngleLoc, diag::warn_objc_protocol_qualifier_missing_id)
       << FixItHint::CreateInsertion(lAngleLoc, "id")
@@ -1706,19 +1701,11 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers(
                                          /*ObjCGenericList=*/true);
 
     // Let Sema figure out what we parsed.
-    Actions.actOnObjCTypeArgsOrProtocolQualifiers(getCurScope(),
-                                                  baseType,
-                                                  lAngleLoc,
-                                                  identifiers,
-                                                  identifierLocs,
-                                                  rAngleLoc,
-                                                  typeArgsLAngleLoc,
-                                                  typeArgs,
-                                                  typeArgsRAngleLoc,
-                                                  protocolLAngleLoc,
-                                                  protocols,
-                                                  protocolRAngleLoc,
-                                                  warnOnIncompleteProtocols);
+    Actions.ObjC().actOnObjCTypeArgsOrProtocolQualifiers(
+        getCurScope(), baseType, lAngleLoc, identifiers, identifierLocs,
+        rAngleLoc, typeArgsLAngleLoc, typeArgs, typeArgsRAngleLoc,
+        protocolLAngleLoc, protocols, protocolRAngleLoc,
+        warnOnIncompleteProtocols);
     return;
   }
 
@@ -1761,7 +1748,7 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers(
       }
     } else {
       invalid = true;
-      if (!Actions.LookupProtocol(identifiers[i], identifierLocs[i])) {
+      if (!Actions.ObjC().LookupProtocol(identifiers[i], identifierLocs[i])) {
         unknownTypeArgs.push_back(identifiers[i]);
         unknownTypeArgsLoc.push_back(identifierLocs[i]);
       } else if (!foundProtocolId) {
@@ -1796,9 +1783,9 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers(
 
   // Diagnose the mix between type args and protocols.
   if (foundProtocolId && foundValidTypeId)
-    Actions.DiagnoseTypeArgsAndProtocols(foundProtocolId, foundProtocolSrcLoc,
-                                         foundValidTypeId,
-                                         foundValidTypeSrcLoc);
+    Actions.ObjC().DiagnoseTypeArgsAndProtocols(
+        foundProtocolId, foundProtocolSrcLoc, foundValidTypeId,
+        foundValidTypeSrcLoc);
 
   // Diagnose unknown arg types.
   ParsedType T;
@@ -1904,17 +1891,9 @@ TypeResult Parser::parseObjCTypeArgsAndProtocolQualifiers(
   else
     endLoc = Tok.getLocation();
 
-  return Actions.actOnObjCTypeArgsAndProtocolQualifiers(
-           getCurScope(),
-           loc,
-           type,
-           typeArgsLAngleLoc,
-           typeArgs,
-           typeArgsRAngleLoc,
-           protocolLAngleLoc,
-           protocols,
-           protocolLocs,
-           protocolRAngleLoc);
+  return Actions.ObjC().actOnObjCTypeArgsAndProtocolQualifiers(
+      getCurScope(), loc, type, typeArgsLAngleLoc, typeArgs, typeArgsRAngleLoc,
+      protocolLAngleLoc, protocols, protocolLocs, protocolRAngleLoc);
 }
 
 void Parser::HelperActionsForIvarDeclarations(
@@ -2029,7 +2008,7 @@ void Parser::ParseObjCClassInstanceVariables(ObjCContainerDecl *interfaceDecl,
              "Ivar should have interfaceDecl as its decl context");
       // Install the declarator into the interface decl.
       FD.D.setObjCIvar(true);
-      Decl *Field = Actions.ActOnIvar(
+      Decl *Field = Actions.ObjC().ActOnIvar(
           getCurScope(), FD.D.getDeclSpec().getSourceRange().getBegin(), FD.D,
           FD.BitfieldSize, visibility);
       if (Field)
@@ -2092,7 +2071,8 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
 
   if (TryConsumeToken(tok::semi)) { // forward declaration of one protocol.
     IdentifierLocPair ProtoInfo(protocolName, nameLoc);
-    return Actions.ActOnForwardProtocolDeclaration(AtLoc, ProtoInfo, attrs);
+    return Actions.ObjC().ActOnForwardProtocolDeclaration(AtLoc, ProtoInfo,
+                                                          attrs);
   }
 
   CheckNestedObjCContexts(AtLoc);
@@ -2119,7 +2099,8 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
     if (ExpectAndConsume(tok::semi, diag::err_expected_after, "@protocol"))
       return nullptr;
 
-    return Actions.ActOnForwardProtocolDeclaration(AtLoc, ProtocolRefs, attrs);
+    return Actions.ObjC().ActOnForwardProtocolDeclaration(AtLoc, ProtocolRefs,
+                                                          attrs);
   }
 
   // Last, and definitely not least, parse a protocol declaration.
@@ -2134,7 +2115,7 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
     return nullptr;
 
   SkipBodyInfo SkipBody;
-  ObjCProtocolDecl *ProtoType = Actions.ActOnStartProtocolInterface(
+  ObjCProtocolDecl *ProtoType = Actions.ObjC().ActOnStartProtocolInterface(
       AtLoc, protocolName, nameLoc, ProtocolRefs.data(), ProtocolRefs.size(),
       ProtocolLocs.data(), EndProtoLoc, attrs, &SkipBody);
 
@@ -2241,7 +2222,7 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc,
                                         protocolLAngleLoc, protocolRAngleLoc,
                                         /*consumeLastToken=*/true);
     }
-    ObjCImpDecl = Actions.ActOnStartCategoryImplementation(
+    ObjCImpDecl = Actions.ObjC().ActOnStartCategoryImplementation(
         AtLoc, nameId, nameLoc, categoryId, categoryLoc, Attrs);
 
   } else {
@@ -2255,7 +2236,7 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc,
       superClassId = Tok.getIdentifierInfo();
       superClassLoc = ConsumeToken(); // Consume super class name
     }
-    ObjCImpDecl = Actions.ActOnStartClassImplementation(
+    ObjCImpDecl = Actions.ObjC().ActOnStartClassImplementation(
         AtLoc, nameId, nameLoc, superClassId, superClassLoc, Attrs);
 
     if (Tok.is(tok::l_brace)) // we have ivars
@@ -2291,7 +2272,8 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc,
     }
   }
 
-  return Actions.ActOnFinishObjCImplementation(ObjCImpDecl, DeclsInGroup);
+  return Actions.ObjC().ActOnFinishObjCImplementation(ObjCImpDecl,
+                                                      DeclsInGroup);
 }
 
 Parser::DeclGroupPtrTy
@@ -2314,7 +2296,7 @@ Parser::ObjCImplParsingDataRAII::~ObjCImplParsingDataRAII() {
       P.Diag(P.Tok, diag::err_objc_missing_end)
           << FixItHint::CreateInsertion(P.Tok.getLocation(), "\n@end\n");
       P.Diag(Dcl->getBeginLoc(), diag::note_objc_container_start)
-          << Sema::OCK_Implementation;
+          << SemaObjC::OCK_Implementation;
     }
   }
   P.CurParsedObjCImpl = nullptr;
@@ -2323,12 +2305,13 @@ Parser::ObjCImplParsingDataRAII::~ObjCImplParsingDataRAII() {
 
 void Parser::ObjCImplParsingDataRAII::finish(SourceRange AtEnd) {
   assert(!Finished);
-  P.Actions.DefaultSynthesizeProperties(P.getCurScope(), Dcl, AtEnd.getBegin());
+  P.Actions.ObjC().DefaultSynthesizeProperties(P.getCurScope(), Dcl,
+                                               AtEnd.getBegin());
   for (size_t i = 0; i < LateParsedObjCMethods.size(); ++i)
     P.ParseLexedObjCMethodDefs(*LateParsedObjCMethods[i],
                                true/*Methods*/);
 
-  P.Actions.ActOnAtEnd(P.getCurScope(), AtEnd);
+  P.Actions.ObjC().ActOnAtEnd(P.getCurScope(), AtEnd);
 
   if (HasCFunction)
     for (size_t i = 0; i < LateParsedObjCMethods.size(); ++i)
@@ -2361,8 +2344,8 @@ Decl *Parser::ParseObjCAtAliasDeclaration(SourceLocation atLoc) {
   IdentifierInfo *classId = Tok.getIdentifierInfo();
   SourceLocation classLoc = ConsumeToken(); // consume class-name;
   ExpectAndConsume(tok::semi, diag::err_expected_after, "@compatibility_alias");
-  return Actions.ActOnCompatibilityAlias(atLoc, aliasId, aliasLoc,
-                                         classId, classLoc);
+  return Actions.ObjC().ActOnCompatibilityAlias(atLoc, aliasId, aliasLoc,
+                                                classId, classLoc);
 }
 
 ///   property-synthesis:
@@ -2411,10 +2394,9 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
       propertyIvar = Tok.getIdentifierInfo();
       propertyIvarLoc = ConsumeToken(); // consume ivar-name
     }
-    Actions.ActOnPropertyImplDecl(
-        getCurScope(), atLoc, propertyLoc, true,
-        propertyId, propertyIvar, propertyIvarLoc,
-        ObjCPropertyQueryKind::OBJC_PR_query_unknown);
+    Actions.ObjC().ActOnPropertyImplDecl(
+        getCurScope(), atLoc, propertyLoc, true, propertyId, propertyIvar,
+        propertyIvarLoc, ObjCPropertyQueryKind::OBJC_PR_query_unknown);
     if (Tok.isNot(tok::comma))
       break;
     ConsumeToken(); // consume ','
@@ -2473,11 +2455,11 @@ Decl *Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
 
     IdentifierInfo *propertyId = Tok.getIdentifierInfo();
     SourceLocation propertyLoc = ConsumeToken(); // consume property name
-    Actions.ActOnPropertyImplDecl(
-        getCurScope(), atLoc, propertyLoc, false,
-        propertyId, nullptr, SourceLocation(),
-        isClassProperty ? ObjCPropertyQueryKind::OBJC_PR_query_class :
-        ObjCPropertyQueryKind::OBJC_PR_query_unknown);
+    Actions.ObjC().ActOnPropertyImplDecl(
+        getCurScope(), atLoc, propertyLoc, false, propertyId, nullptr,
+        SourceLocation(),
+        isClassProperty ? ObjCPropertyQueryKind::OBJC_PR_query_class
+                        : ObjCPropertyQueryKind::OBJC_PR_query_unknown);
 
     if (Tok.isNot(tok::comma))
       break;
@@ -2502,7 +2484,7 @@ StmtResult Parser::ParseObjCThrowStmt(SourceLocation atLoc) {
   }
   // consume ';'
   ExpectAndConsume(tok::semi, diag::err_expected_after, "@throw");
-  return Actions.ActOnObjCAtThrowStmt(atLoc, Res.get(), getCurScope());
+  return Actions.ObjC().ActOnObjCAtThrowStmt(atLoc, Res.get(), getCurScope());
 }
 
 /// objc-synchronized-statement:
@@ -2539,7 +2521,8 @@ Parser::ParseObjCSynchronizedStmt(SourceLocation atLoc) {
 
   // Check the @synchronized operand now.
   if (!operand.isInvalid())
-    operand = Actions.ActOnObjCAtSynchronizedOperand(atLoc, operand.get());
+    operand =
+        Actions.ObjC().ActOnObjCAtSynchronizedOperand(atLoc, operand.get());
 
   // Parse the compound statement within a new scope.
   ParseScope bodyScope(this, Scope::DeclScope | Scope::CompoundStmtScope);
@@ -2554,7 +2537,8 @@ Parser::ParseObjCSynchronizedStmt(SourceLocation atLoc) {
   if (body.isInvalid())
     body = Actions.ActOnNullStmt(Tok.getLocation());
 
-  return Actions.ActOnObjCAtSynchronizedStmt(atLoc, operand.get(), body.get());
+  return Actions.ObjC().ActOnObjCAtSynchronizedStmt(atLoc, operand.get(),
+                                                    body.get());
 }
 
 ///  objc-try-catch-statement:
@@ -2611,7 +2595,8 @@ StmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
 
           // Inform the actions module about the declarator, so it
           // gets added to the current scope.
-          FirstPart = Actions.ActOnObjCExceptionDecl(getCurScope(), ParmDecl);
+          FirstPart =
+              Actions.ObjC().ActOnObjCExceptionDecl(getCurScope(), ParmDecl);
         } else
           ConsumeToken(); // consume '...'
 
@@ -2630,10 +2615,8 @@ StmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
         if (CatchBody.isInvalid())
           CatchBody = Actions.ActOnNullStmt(Tok.getLocation());
 
-        StmtResult Catch = Actions.ActOnObjCAtCatchStmt(AtCatchFinallyLoc,
-                                                              RParenLoc,
-                                                              FirstPart,
-                                                              CatchBody.get());
+        StmtResult Catch = Actions.ObjC().ActOnObjCAtCatchStmt(
+            AtCatchFinallyLoc, RParenLoc, FirstPart, CatchBody.get());
         if (!Catch.isInvalid())
           CatchStmts.push_back(Catch.get());
 
@@ -2669,8 +2652,8 @@ StmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
         FinallyBody = Actions.ActOnCapturedRegionEnd(FinallyBody.get());
       }
 
-      FinallyStmt = Actions.ActOnObjCAtFinallyStmt(AtCatchFinallyLoc,
-                                                   FinallyBody.get());
+      FinallyStmt = Actions.ObjC().ActOnObjCAtFinallyStmt(AtCatchFinallyLoc,
+                                                          FinallyBody.get());
       catch_or_finally_seen = true;
       break;
     }
@@ -2680,9 +2663,8 @@ StmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
     return StmtError();
   }
 
-  return Actions.ActOnObjCAtTryStmt(atLoc, TryBody.get(),
-                                    CatchStmts,
-                                    FinallyStmt.get());
+  return Actions.ObjC().ActOnObjCAtTryStmt(atLoc, TryBody.get(), CatchStmts,
+                                           FinallyStmt.get());
 }
 
 /// objc-autoreleasepool-statement:
@@ -2704,8 +2686,8 @@ Parser::ParseObjCAutoreleasePoolStmt(SourceLocation atLoc) {
   BodyScope.Exit();
   if (AutoreleasePoolBody.isInvalid())
     AutoreleasePoolBody = Actions.ActOnNullStmt(Tok.getLocation());
-  return Actions.ActOnObjCAutoreleasePoolStmt(atLoc,
-                                                AutoreleasePoolBody.get());
+  return Actions.ObjC().ActOnObjCAutoreleasePoolStmt(atLoc,
+                                                     AutoreleasePoolBody.get());
 }
 
 /// StashAwayMethodOrFunctionBodyTokens -  Consume the tokens and store them
@@ -2788,7 +2770,7 @@ Decl *Parser::ParseObjCMethodDefinition() {
   }
 
   // Allow the rest of sema to find private method decl implementations.
-  Actions.AddAnyMethodToGlobalPool(MDecl);
+  Actions.ObjC().AddAnyMethodToGlobalPool(MDecl);
   assert (CurParsedObjCImpl
           && "ParseObjCMethodDefinition - Method out of @implementation");
   // Consume the tokens and store them for later parsing.
@@ -2872,7 +2854,7 @@ ExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) {
       return Lit;
 
     return ParsePostfixExpressionSuffix(
-             Actions.BuildObjCNumericLiteral(AtLoc, Lit.get()));
+        Actions.ObjC().BuildObjCNumericLiteral(AtLoc, Lit.get()));
   }
 
   case tok::string_literal:    // primary-expression: string-literal
@@ -3128,15 +3110,14 @@ ExprResult Parser::ParseObjCMessageExpression() {
     IdentifierInfo *Name = Tok.getIdentifierInfo();
     SourceLocation NameLoc = Tok.getLocation();
     ParsedType ReceiverType;
-    switch (Actions.getObjCMessageKind(getCurScope(), Name, NameLoc,
-                                       Name == Ident_super,
-                                       NextToken().is(tok::period),
-                                       ReceiverType)) {
-    case Sema::ObjCSuperMessage:
+    switch (Actions.ObjC().getObjCMessageKind(
+        getCurScope(), Name, NameLoc, Name == Ident_super,
+        NextToken().is(tok::period), ReceiverType)) {
+    case SemaObjC::ObjCSuperMessage:
       return ParseObjCMessageExpressionBody(LBracLoc, ConsumeToken(), nullptr,
                                             nullptr);
 
-    case Sema::ObjCClassMessage:
+    case SemaObjC::ObjCClassMessage:
       if (!ReceiverType) {
         SkipUntil(tok::r_square, StopAtSemi);
         return ExprError();
@@ -3162,7 +3143,7 @@ ExprResult Parser::ParseObjCMessageExpression() {
       return ParseObjCMessageExpressionBody(LBracLoc, SourceLocation(),
                                             ReceiverType, nullptr);
 
-    case Sema::ObjCInstanceMessage:
+    case SemaObjC::ObjCInstanceMessage:
       // Fall through to parse an expression.
       break;
     }
@@ -3375,13 +3356,14 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
   Selector Sel = PP.getSelectorTable().getSelector(nKeys, &KeyIdents[0]);
 
   if (SuperLoc.isValid())
-    return Actions.ActOnSuperMessage(getCurScope(), SuperLoc, Sel,
-                                     LBracLoc, KeyLocs, RBracLoc, KeyExprs);
+    return Actions.ObjC().ActOnSuperMessage(
+        getCurScope(), SuperLoc, Sel, LBracLoc, KeyLocs, RBracLoc, KeyExprs);
   else if (ReceiverType)
-    return Actions.ActOnClassMessage(getCurScope(), ReceiverType, Sel,
-                                     LBracLoc, KeyLocs, RBracLoc, KeyExprs);
-  return Actions.ActOnInstanceMessage(getCurScope(), ReceiverExpr, Sel,
-                                      LBracLoc, KeyLocs, RBracLoc, KeyExprs);
+    return Actions.ObjC().ActOnClassMessage(getCurScope(), ReceiverType, Sel,
+                                            LBracLoc, KeyLocs, RBracLoc,
+                                            KeyExprs);
+  return Actions.ObjC().ActOnInstanceMessage(
+      getCurScope(), ReceiverExpr, Sel, LBracLoc, KeyLocs, RBracLoc, KeyExprs);
 }
 
 ExprResult Parser::ParseObjCStringLiteral(SourceLocation AtLoc) {
@@ -3410,7 +3392,7 @@ ExprResult Parser::ParseObjCStringLiteral(SourceLocation AtLoc) {
     AtStrings.push_back(Lit.get());
   }
 
-  return Actions.ParseObjCStringLiteral(AtLocs.data(), AtStrings);
+  return Actions.ObjC().ParseObjCStringLiteral(AtLocs.data(), AtStrings);
 }
 
 /// ParseObjCBooleanLiteral -
@@ -3421,7 +3403,7 @@ ExprResult Parser::ParseObjCStringLiteral(SourceLocation AtLoc) {
 ExprResult Parser::ParseObjCBooleanLiteral(SourceLocation AtLoc,
                                            bool ArgValue) {
   SourceLocation EndLoc = ConsumeToken();             // consume the keyword.
-  return Actions.ActOnObjCBoolLiteral(AtLoc, EndLoc, ArgValue);
+  return Actions.ObjC().ActOnObjCBoolLiteral(AtLoc, EndLoc, ArgValue);
 }
 
 /// ParseObjCCharacterLiteral -
@@ -3433,7 +3415,7 @@ ExprResult Parser::ParseObjCCharacterLiteral(SourceLocation AtLoc) {
     return Lit;
   }
   ConsumeToken(); // Consume the literal token.
-  return Actions.BuildObjCNumericLiteral(AtLoc, Lit.get());
+  return Actions.ObjC().BuildObjCNumericLiteral(AtLoc, Lit.get());
 }
 
 /// ParseObjCNumericLiteral -
@@ -3447,7 +3429,7 @@ ExprResult Parser::ParseObjCNumericLiteral(SourceLocation AtLoc) {
     return Lit;
   }
   ConsumeToken(); // Consume the literal token.
-  return Actions.BuildObjCNumericLiteral(AtLoc, Lit.get());
+  return Actions.ObjC().BuildObjCNumericLiteral(AtLoc, Lit.get());
 }
 
 /// ParseObjCBoxedExpr -
@@ -3471,8 +3453,8 @@ Parser::ParseObjCBoxedExpr(SourceLocation AtLoc) {
   // a boxed expression from a literal.
   SourceLocation LPLoc = T.getOpenLocation(), RPLoc = T.getCloseLocation();
   ValueExpr = Actions.ActOnParenExpr(LPLoc, RPLoc, ValueExpr.get());
-  return Actions.BuildObjCBoxedExpr(SourceRange(AtLoc, RPLoc),
-                                    ValueExpr.get());
+  return Actions.ObjC().BuildObjCBoxedExpr(SourceRange(AtLoc, RPLoc),
+                                           ValueExpr.get());
 }
 
 ExprResult Parser::ParseObjCArrayLiteral(SourceLocation AtLoc) {
@@ -3515,7 +3497,7 @@ ExprResult Parser::ParseObjCArrayLiteral(SourceLocation AtLoc) {
     return ExprError();
 
   MultiExprArg Args(ElementExprs);
-  return Actions.BuildObjCArrayLiteral(SourceRange(AtLoc, EndLoc), Args);
+  return Actions.ObjC().BuildObjCArrayLiteral(SourceRange(AtLoc, EndLoc), Args);
 }
 
 ExprResult Parser::ParseObjCDictionaryLiteral(SourceLocation AtLoc) {
@@ -3580,8 +3562,8 @@ ExprResult Parser::ParseObjCDictionaryLiteral(SourceLocation AtLoc) {
     return ExprError();
 
   // Create the ObjCDictionaryLiteral.
-  return Actions.BuildObjCDictionaryLiteral(SourceRange(AtLoc, EndLoc),
-                                            Elements);
+  return Actions.ObjC().BuildObjCDictionaryLiteral(SourceRange(AtLoc, EndLoc),
+                                                   Elements);
 }
 
 ///    objc-encode-expression:
@@ -3605,8 +3587,8 @@ Parser::ParseObjCEncodeExpression(SourceLocation AtLoc) {
   if (Ty.isInvalid())
     return ExprError();
 
-  return Actions.ParseObjCEncodeExpression(AtLoc, EncLoc, T.getOpenLocation(),
-                                           Ty.get(), T.getCloseLocation());
+  return Actions.ObjC().ParseObjCEncodeExpression(
+      AtLoc, EncLoc, T.getOpenLocation(), Ty.get(), T.getCloseLocation());
 }
 
 ///     objc-protocol-expression
@@ -3629,9 +3611,9 @@ Parser::ParseObjCProtocolExpression(SourceLocation AtLoc) {
 
   T.consumeClose();
 
-  return Actions.ParseObjCProtocolExpression(protocolId, AtLoc, ProtoLoc,
-                                             T.getOpenLocation(), ProtoIdLoc,
-                                             T.getCloseLocation());
+  return Actions.ObjC().ParseObjCProtocolExpression(
+      protocolId, AtLoc, ProtoLoc, T.getOpenLocation(), ProtoIdLoc,
+      T.getCloseLocation());
 }
 
 ///     objc-selector-expression
@@ -3695,18 +3677,17 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
     ConsumeParen(); // ')'
   T.consumeClose();
   Selector Sel = PP.getSelectorTable().getSelector(nColons, &KeyIdents[0]);
-  return Actions.ParseObjCSelectorExpression(Sel, AtLoc, SelectorLoc,
-                                             T.getOpenLocation(),
-                                             T.getCloseLocation(),
-                                             !HasOptionalParen);
+  return Actions.ObjC().ParseObjCSelectorExpression(
+      Sel, AtLoc, SelectorLoc, T.getOpenLocation(), T.getCloseLocation(),
+      !HasOptionalParen);
 }
 
 void Parser::ParseLexedObjCMethodDefs(LexedMethod &LM, bool parseMethod) {
   // MCDecl might be null due to error in method or c-function  prototype, etc.
   Decl *MCDecl = LM.D;
-  bool skip = MCDecl &&
-              ((parseMethod && !Actions.isObjCMethodDecl(MCDecl)) ||
-              (!parseMethod && Actions.isObjCMethodDecl(MCDecl)));
+  bool skip =
+      MCDecl && ((parseMethod && !Actions.ObjC().isObjCMethodDecl(MCDecl)) ||
+                 (!parseMethod && Actions.ObjC().isObjCMethodDecl(MCDecl)));
   if (skip)
     return;
 
@@ -3741,7 +3722,7 @@ void Parser::ParseLexedObjCMethodDefs(LexedMethod &LM, bool parseMethod) {
   // Tell the actions module that we have entered a method or c-function definition
   // with the specified Declarator for the method/function.
   if (parseMethod)
-    Actions.ActOnStartOfObjCMethodDef(getCurScope(), MCDecl);
+    Actions.ObjC().ActOnStartOfObjCMethodDef(getCurScope(), MCDecl);
   else
     Actions.ActOnStartOfFunctionDef(getCurScope(), MCDecl);
   if (Tok.is(tok::kw_try))
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index 629421c01d17..e0116d300310 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -22,6 +22,7 @@
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "llvm/ADT/STLExtras.h"
@@ -2294,10 +2295,8 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
   } else if (ForEach) {
     // Similarly, we need to do the semantic analysis for a for-range
     // statement immediately in order to close over temporaries correctly.
-    ForEachStmt = Actions.ActOnObjCForCollectionStmt(ForLoc,
-                                                     FirstPart.get(),
-                                                     Collection.get(),
-                                                     T.getCloseLocation());
+    ForEachStmt = Actions.ObjC().ActOnObjCForCollectionStmt(
+        ForLoc, FirstPart.get(), Collection.get(), T.getCloseLocation());
   } else {
     // In OpenMP loop region loop control variable must be captured and be
     // private. Perform analysis of first part (if any).
@@ -2345,8 +2344,8 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
     return StmtError();
 
   if (ForEach)
-   return Actions.FinishObjCForCollectionStmt(ForEachStmt.get(),
-                                              Body.get());
+    return Actions.ObjC().FinishObjCForCollectionStmt(ForEachStmt.get(),
+                                                      Body.get());
 
   if (ForRangeInfo.ParsedForRangeDecl())
     return Actions.FinishCXXForRangeStmt(ForRangeStmt.get(), Body.get());
diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index a96439df6642..58e0a3b9679b 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -54,6 +54,7 @@ add_clang_library(clangSema
   SemaLambda.cpp
   SemaLookup.cpp
   SemaModule.cpp
+  SemaObjC.cpp
   SemaObjCProperty.cpp
   SemaOpenACC.cpp
   SemaOpenMP.cpp
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index a1e32d391ed0..7585f1c367be 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -45,6 +45,7 @@
 #include "clang/Sema/SemaConsumer.h"
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenACC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/SemaSYCL.h"
@@ -203,6 +204,7 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
       CurScope(nullptr), Ident_super(nullptr),
       CUDAPtr(std::make_unique<SemaCUDA>(*this)),
       HLSLPtr(std::make_unique<SemaHLSL>(*this)),
+      ObjCPtr(std::make_unique<SemaObjC>(*this)),
       OpenACCPtr(std::make_unique<SemaOpenACC>(*this)),
       OpenMPPtr(std::make_unique<SemaOpenMP>(*this)),
       SYCLPtr(std::make_unique<SemaSYCL>(*this)),
@@ -224,20 +226,16 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
       AccessCheckingSFINAE(false), CurrentInstantiationScope(nullptr),
       InNonInstantiationSFINAEContext(false), NonInstantiationEntries(0),
       ArgumentPackSubstitutionIndex(-1), SatisfactionCache(Context),
-      NSNumberDecl(nullptr), NSValueDecl(nullptr), NSStringDecl(nullptr),
-      StringWithUTF8StringMethod(nullptr),
-      ValueWithBytesObjCTypeMethod(nullptr), NSArrayDecl(nullptr),
-      ArrayWithObjectsMethod(nullptr), NSDictionaryDecl(nullptr),
-      DictionaryWithObjectsMethod(nullptr), CodeCompleter(CodeCompleter) {
+      CodeCompleter(CodeCompleter) {
   assert(pp.TUKind == TUKind);
   TUScope = nullptr;
 
   LoadedExternalKnownNamespaces = false;
   for (unsigned I = 0; I != NSAPI::NumNSNumberLiteralMethods; ++I)
-    NSNumberLiteralMethods[I] = nullptr;
+    ObjC().NSNumberLiteralMethods[I] = nullptr;
 
   if (getLangOpts().ObjC)
-    NSAPIObj.reset(new NSAPI(Context));
+    ObjC().NSAPIObj.reset(new NSAPI(Context));
 
   if (getLangOpts().CPlusPlus)
     FieldCollector.reset(new CXXFieldCollector());
@@ -1129,7 +1127,7 @@ void Sema::ActOnEndOfTranslationUnit() {
   // Complete translation units and modules define vtables and perform implicit
   // instantiations. PCH files do not.
   if (TUKind != TU_Prefix) {
-    DiagnoseUseOfUnimplementedSelectors();
+    ObjC().DiagnoseUseOfUnimplementedSelectors();
 
     ActOnEndOfTranslationUnitFragment(
         !ModuleScopes.empty() && ModuleScopes.back().Module->Kind ==
diff --git a/clang/lib/Sema/SemaAPINotes.cpp b/clang/lib/Sema/SemaAPINotes.cpp
index c5998aca0d72..443bf162044f 100644
--- a/clang/lib/Sema/SemaAPINotes.cpp
+++ b/clang/lib/Sema/SemaAPINotes.cpp
@@ -16,6 +16,7 @@
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 
 using namespace clang;
 
@@ -372,7 +373,7 @@ static void ProcessAPINotes(Sema &S, Decl *D,
       if (auto Var = dyn_cast<VarDecl>(D)) {
         // Make adjustments to parameter types.
         if (isa<ParmVarDecl>(Var)) {
-          Type = S.AdjustParameterTypeForObjCAutoRefCount(
+          Type = S.ObjC().AdjustParameterTypeForObjCAutoRefCount(
               Type, D->getLocation(), TypeInfo);
           Type = S.Context.getAdjustedParameterType(Type);
         }
diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index a83b1e8afadb..bb44531495a5 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -860,22 +860,6 @@ void Sema::ActOnPragmaUnused(const Token &IdTok, Scope *curScope,
                                          UnusedAttr::GNU_unused));
 }
 
-void Sema::AddCFAuditedAttribute(Decl *D) {
-  IdentifierInfo *Ident;
-  SourceLocation Loc;
-  std::tie(Ident, Loc) = PP.getPragmaARCCFCodeAuditedInfo();
-  if (!Loc.isValid()) return;
-
-  // Don't add a redundant or conflicting attribute.
-  if (D->hasAttr<CFAuditedTransferAttr>() ||
-      D->hasAttr<CFUnknownTransferAttr>())
-    return;
-
-  AttributeCommonInfo Info(Ident, SourceRange(Loc),
-                           AttributeCommonInfo::Form::Pragma());
-  D->addAttr(CFAuditedTransferAttr::CreateImplicit(Context, Info));
-}
-
 namespace {
 
 std::optional<attr::SubjectMatchRule>
diff --git a/clang/lib/Sema/SemaAvailability.cpp b/clang/lib/Sema/SemaAvailability.cpp
index 846a31a79673..5ebc25317bf3 100644
--- a/clang/lib/Sema/SemaAvailability.cpp
+++ b/clang/lib/Sema/SemaAvailability.cpp
@@ -19,6 +19,7 @@
 #include "clang/Sema/DelayedDiagnostic.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaObjC.h"
 #include <optional>
 
 using namespace clang;
@@ -98,11 +99,11 @@ ShouldDiagnoseAvailabilityOfDecl(Sema &S, const NamedDecl *D,
 
   // For +new, infer availability from -init.
   if (const auto *MD = dyn_cast<ObjCMethodDecl>(D)) {
-    if (S.NSAPIObj && ClassReceiver) {
+    if (S.ObjC().NSAPIObj && ClassReceiver) {
       ObjCMethodDecl *Init = ClassReceiver->lookupInstanceMethod(
-          S.NSAPIObj->getInitSelector());
+          S.ObjC().NSAPIObj->getInitSelector());
       if (Init && Result == AR_Available && MD->isClassMethod() &&
-          MD->getSelector() == S.NSAPIObj->getNewSelector() &&
+          MD->getSelector() == S.ObjC().NSAPIObj->getNewSelector() &&
           MD->definedInNSObject(S.getASTContext())) {
         Result = Init->getAvailability(Message);
         D = Init;
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index 126fd3797417..483ec7e36eae 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -24,6 +24,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include <set>
@@ -159,8 +160,8 @@ namespace {
       assert(Self.getLangOpts().allowsNonTrivialObjCLifetimeQualifiers());
 
       Expr *src = SrcExpr.get();
-      if (Self.CheckObjCConversion(OpRange, DestType, src, CCK) ==
-          Sema::ACR_unbridged)
+      if (Self.ObjC().CheckObjCConversion(OpRange, DestType, src, CCK) ==
+          SemaObjC::ACR_unbridged)
         IsARCUnbridgedCast = true;
       SrcExpr = src;
     }
@@ -1499,7 +1500,7 @@ static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
   // Allow ns-pointer to cf-pointer conversion in either direction
   // with static casts.
   if (!CStyle &&
-      Self.CheckTollFreeBridgeStaticCast(DestType, SrcExpr.get(), Kind))
+      Self.ObjC().CheckTollFreeBridgeStaticCast(DestType, SrcExpr.get(), Kind))
     return TC_Success;
 
   // See if it looks like the user is trying to convert between
@@ -2524,7 +2525,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
   } else if (IsLValueCast) {
     Kind = CK_LValueBitCast;
   } else if (DestType->isObjCObjectPointerType()) {
-    Kind = Self.PrepareCastToObjCObjectPointer(SrcExpr);
+    Kind = Self.ObjC().PrepareCastToObjCObjectPointer(SrcExpr);
   } else if (DestType->isBlockPointerType()) {
     if (!SrcType->isBlockPointerType()) {
       Kind = CK_AnyPointerToBlockPointerCast;
@@ -3217,8 +3218,8 @@ void CastOperation::CheckCStyleCast() {
           return;
         }
       }
-    }
-    else if (!Self.CheckObjCARCUnavailableWeakConversion(DestType, SrcType)) {
+    } else if (!Self.ObjC().CheckObjCARCUnavailableWeakConversion(DestType,
+                                                                  SrcType)) {
       Self.Diag(SrcExpr.get()->getBeginLoc(),
                 diag::err_arc_convesion_of_weak_unavailable)
           << 1 << SrcType << DestType << SrcExpr.get()->getSourceRange();
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 54789dde5069..ecd182165114 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -62,6 +62,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/APSInt.h"
@@ -2488,7 +2489,7 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
       return ExprError();
     assert(TheCall->getNumArgs() == 1 &&
            "Wrong # arguments to builtin CFStringMakeConstantString");
-    if (CheckObjCString(TheCall->getArg(0)))
+    if (ObjC().CheckObjCString(TheCall->getArg(0)))
       return ExprError();
     break;
   case Builtin::BI__builtin_ms_va_start:
@@ -8195,20 +8196,6 @@ bool Sema::CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall,
   return false;
 }
 
-bool Sema::CheckObjCMethodCall(ObjCMethodDecl *Method, SourceLocation lbrac,
-                               ArrayRef<const Expr *> Args) {
-  VariadicCallType CallType =
-      Method->isVariadic() ? VariadicMethod : VariadicDoesNotApply;
-
-  checkCall(Method, nullptr, /*ThisArg=*/nullptr, Args,
-            /*IsMemberFunction=*/false, lbrac, Method->getSourceRange(),
-            CallType);
-
-  CheckTCBEnforcement(lbrac, Method);
-
-  return false;
-}
-
 bool Sema::CheckPointerCall(NamedDecl *NDecl, CallExpr *TheCall,
                             const FunctionProtoType *Proto) {
   QualType Ty;
@@ -9362,38 +9349,6 @@ ExprResult Sema::BuiltinNontemporalOverloaded(ExprResult TheCallResult) {
   return TheCallResult;
 }
 
-/// CheckObjCString - Checks that the argument to the builtin
-/// CFString constructor is correct
-/// Note: It might also make sense to do the UTF-16 conversion here (would
-/// simplify the backend).
-bool Sema::CheckObjCString(Expr *Arg) {
-  Arg = Arg->IgnoreParenCasts();
-  StringLiteral *Literal = dyn_cast<StringLiteral>(Arg);
-
-  if (!Literal || !Literal->isOrdinary()) {
-    Diag(Arg->getBeginLoc(), diag::err_cfstring_literal_not_string_constant)
-        << Arg->getSourceRange();
-    return true;
-  }
-
-  if (Literal->containsNonAsciiOrNull()) {
-    StringRef String = Literal->getString();
-    unsigned NumBytes = String.size();
-    SmallVector<llvm::UTF16, 128> ToBuf(NumBytes);
-    const llvm::UTF8 *FromPtr = (const llvm::UTF8 *)String.data();
-    llvm::UTF16 *ToPtr = &ToBuf[0];
-
-    llvm::ConversionResult Result =
-        llvm::ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes, &ToPtr,
-                                 ToPtr + NumBytes, llvm::strictConversion);
-    // Check for conversion failure.
-    if (Result != llvm::conversionOK)
-      Diag(Arg->getBeginLoc(), diag::warn_cfstring_truncated)
-          << Arg->getSourceRange();
-  }
-  return false;
-}
-
 /// CheckObjCString - Checks that the format string argument to the os_log()
 /// and os_trace() functions is correct, and converts it to const char *.
 ExprResult Sema::CheckOSLogFormatStringArg(Expr *Arg) {
@@ -15279,7 +15234,7 @@ static bool CheckTautologicalComparison(Sema &S, BinaryOperator *E,
   // Special case for ObjC BOOL on targets where its a typedef for a signed char
   // (Namely, macOS). FIXME: IntRange::forValueOfType should do this.
   bool IsObjCSignedCharBool = S.getLangOpts().ObjC &&
-                              S.NSAPIObj->isObjCBOOLType(OtherT) &&
+                              S.ObjC().NSAPIObj->isObjCBOOLType(OtherT) &&
                               OtherT->isSpecificBuiltinType(BuiltinType::SChar);
 
   // Whether we're treating Other as being a bool because of the form of
@@ -15709,7 +15664,7 @@ static void DiagnoseImpCast(Sema &S, Expr *E, QualType T,
 
 static bool isObjCSignedCharBool(Sema &S, QualType Ty) {
   return Ty->isSpecificBuiltinType(BuiltinType::SChar) &&
-      S.getLangOpts().ObjC && S.NSAPIObj->isObjCBOOLType(Ty);
+         S.getLangOpts().ObjC && S.ObjC().NSAPIObj->isObjCBOOLType(Ty);
 }
 
 static void adornObjCBoolConversionDiagWithTernaryFixit(
@@ -16021,7 +15976,7 @@ static void checkObjCCollectionLiteralElement(Sema &S,
 /// target type.
 static void checkObjCArrayLiteral(Sema &S, QualType TargetType,
                                   ObjCArrayLiteral *ArrayLiteral) {
-  if (!S.NSArrayDecl)
+  if (!S.ObjC().NSArrayDecl)
     return;
 
   const auto *TargetObjCPtr = TargetType->getAs<ObjCObjectPointerType>();
@@ -16029,8 +15984,8 @@ static void checkObjCArrayLiteral(Sema &S, QualType TargetType,
     return;
 
   if (TargetObjCPtr->isUnspecialized() ||
-      TargetObjCPtr->getInterfaceDecl()->getCanonicalDecl()
-        != S.NSArrayDecl->getCanonicalDecl())
+      TargetObjCPtr->getInterfaceDecl()->getCanonicalDecl() !=
+          S.ObjC().NSArrayDecl->getCanonicalDecl())
     return;
 
   auto TypeArgs = TargetObjCPtr->getTypeArgs();
@@ -16050,7 +16005,7 @@ static void checkObjCArrayLiteral(Sema &S, QualType TargetType,
 static void
 checkObjCDictionaryLiteral(Sema &S, QualType TargetType,
                            ObjCDictionaryLiteral *DictionaryLiteral) {
-  if (!S.NSDictionaryDecl)
+  if (!S.ObjC().NSDictionaryDecl)
     return;
 
   const auto *TargetObjCPtr = TargetType->getAs<ObjCObjectPointerType>();
@@ -16058,8 +16013,8 @@ checkObjCDictionaryLiteral(Sema &S, QualType TargetType,
     return;
 
   if (TargetObjCPtr->isUnspecialized() ||
-      TargetObjCPtr->getInterfaceDecl()->getCanonicalDecl()
-        != S.NSDictionaryDecl->getCanonicalDecl())
+      TargetObjCPtr->getInterfaceDecl()->getCanonicalDecl() !=
+          S.ObjC().NSDictionaryDecl->getCanonicalDecl())
     return;
 
   auto TypeArgs = TargetObjCPtr->getTypeArgs();
@@ -18794,465 +18749,6 @@ void Sema::CheckArrayAccess(const Expr *expr) {
   }
 }
 
-//===--- CHECK: Objective-C retain cycles ----------------------------------//
-
-namespace {
-
-struct RetainCycleOwner {
-  VarDecl *Variable = nullptr;
-  SourceRange Range;
-  SourceLocation Loc;
-  bool Indirect = false;
-
-  RetainCycleOwner() = default;
-
-  void setLocsFrom(Expr *e) {
-    Loc = e->getExprLoc();
-    Range = e->getSourceRange();
-  }
-};
-
-} // namespace
-
-/// Consider whether capturing the given variable can possibly lead to
-/// a retain cycle.
-static bool considerVariable(VarDecl *var, Expr *ref, RetainCycleOwner &owner) {
-  // In ARC, it's captured strongly iff the variable has __strong
-  // lifetime.  In MRR, it's captured strongly if the variable is
-  // __block and has an appropriate type.
-  if (var->getType().getObjCLifetime() != Qualifiers::OCL_Strong)
-    return false;
-
-  owner.Variable = var;
-  if (ref)
-    owner.setLocsFrom(ref);
-  return true;
-}
-
-static bool findRetainCycleOwner(Sema &S, Expr *e, RetainCycleOwner &owner) {
-  while (true) {
-    e = e->IgnoreParens();
-    if (CastExpr *cast = dyn_cast<CastExpr>(e)) {
-      switch (cast->getCastKind()) {
-      case CK_BitCast:
-      case CK_LValueBitCast:
-      case CK_LValueToRValue:
-      case CK_ARCReclaimReturnedObject:
-        e = cast->getSubExpr();
-        continue;
-
-      default:
-        return false;
-      }
-    }
-
-    if (ObjCIvarRefExpr *ref = dyn_cast<ObjCIvarRefExpr>(e)) {
-      ObjCIvarDecl *ivar = ref->getDecl();
-      if (ivar->getType().getObjCLifetime() != Qualifiers::OCL_Strong)
-        return false;
-
-      // Try to find a retain cycle in the base.
-      if (!findRetainCycleOwner(S, ref->getBase(), owner))
-        return false;
-
-      if (ref->isFreeIvar()) owner.setLocsFrom(ref);
-      owner.Indirect = true;
-      return true;
-    }
-
-    if (DeclRefExpr *ref = dyn_cast<DeclRefExpr>(e)) {
-      VarDecl *var = dyn_cast<VarDecl>(ref->getDecl());
-      if (!var) return false;
-      return considerVariable(var, ref, owner);
-    }
-
-    if (MemberExpr *member = dyn_cast<MemberExpr>(e)) {
-      if (member->isArrow()) return false;
-
-      // Don't count this as an indirect ownership.
-      e = member->getBase();
-      continue;
-    }
-
-    if (PseudoObjectExpr *pseudo = dyn_cast<PseudoObjectExpr>(e)) {
-      // Only pay attention to pseudo-objects on property references.
-      ObjCPropertyRefExpr *pre
-        = dyn_cast<ObjCPropertyRefExpr>(pseudo->getSyntacticForm()
-                                              ->IgnoreParens());
-      if (!pre) return false;
-      if (pre->isImplicitProperty()) return false;
-      ObjCPropertyDecl *property = pre->getExplicitProperty();
-      if (!property->isRetaining() &&
-          !(property->getPropertyIvarDecl() &&
-            property->getPropertyIvarDecl()->getType()
-              .getObjCLifetime() == Qualifiers::OCL_Strong))
-          return false;
-
-      owner.Indirect = true;
-      if (pre->isSuperReceiver()) {
-        owner.Variable = S.getCurMethodDecl()->getSelfDecl();
-        if (!owner.Variable)
-          return false;
-        owner.Loc = pre->getLocation();
-        owner.Range = pre->getSourceRange();
-        return true;
-      }
-      e = const_cast<Expr*>(cast<OpaqueValueExpr>(pre->getBase())
-                              ->getSourceExpr());
-      continue;
-    }
-
-    // Array ivars?
-
-    return false;
-  }
-}
-
-namespace {
-
-  struct FindCaptureVisitor : EvaluatedExprVisitor<FindCaptureVisitor> {
-    VarDecl *Variable;
-    Expr *Capturer = nullptr;
-    bool VarWillBeReased = false;
-
-    FindCaptureVisitor(ASTContext &Context, VarDecl *variable)
-        : EvaluatedExprVisitor<FindCaptureVisitor>(Context),
-          Variable(variable) {}
-
-    void VisitDeclRefExpr(DeclRefExpr *ref) {
-      if (ref->getDecl() == Variable && !Capturer)
-        Capturer = ref;
-    }
-
-    void VisitObjCIvarRefExpr(ObjCIvarRefExpr *ref) {
-      if (Capturer) return;
-      Visit(ref->getBase());
-      if (Capturer && ref->isFreeIvar())
-        Capturer = ref;
-    }
-
-    void VisitBlockExpr(BlockExpr *block) {
-      // Look inside nested blocks
-      if (block->getBlockDecl()->capturesVariable(Variable))
-        Visit(block->getBlockDecl()->getBody());
-    }
-
-    void VisitOpaqueValueExpr(OpaqueValueExpr *OVE) {
-      if (Capturer) return;
-      if (OVE->getSourceExpr())
-        Visit(OVE->getSourceExpr());
-    }
-
-    void VisitBinaryOperator(BinaryOperator *BinOp) {
-      if (!Variable || VarWillBeReased || BinOp->getOpcode() != BO_Assign)
-        return;
-      Expr *LHS = BinOp->getLHS();
-      if (const DeclRefExpr *DRE = dyn_cast_or_null<DeclRefExpr>(LHS)) {
-        if (DRE->getDecl() != Variable)
-          return;
-        if (Expr *RHS = BinOp->getRHS()) {
-          RHS = RHS->IgnoreParenCasts();
-          std::optional<llvm::APSInt> Value;
-          VarWillBeReased =
-              (RHS && (Value = RHS->getIntegerConstantExpr(Context)) &&
-               *Value == 0);
-        }
-      }
-    }
-  };
-
-} // namespace
-
-/// Check whether the given argument is a block which captures a
-/// variable.
-static Expr *findCapturingExpr(Sema &S, Expr *e, RetainCycleOwner &owner) {
-  assert(owner.Variable && owner.Loc.isValid());
-
-  e = e->IgnoreParenCasts();
-
-  // Look through [^{...} copy] and Block_copy(^{...}).
-  if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(e)) {
-    Selector Cmd = ME->getSelector();
-    if (Cmd.isUnarySelector() && Cmd.getNameForSlot(0) == "copy") {
-      e = ME->getInstanceReceiver();
-      if (!e)
-        return nullptr;
-      e = e->IgnoreParenCasts();
-    }
-  } else if (CallExpr *CE = dyn_cast<CallExpr>(e)) {
-    if (CE->getNumArgs() == 1) {
-      FunctionDecl *Fn = dyn_cast_or_null<FunctionDecl>(CE->getCalleeDecl());
-      if (Fn) {
-        const IdentifierInfo *FnI = Fn->getIdentifier();
-        if (FnI && FnI->isStr("_Block_copy")) {
-          e = CE->getArg(0)->IgnoreParenCasts();
-        }
-      }
-    }
-  }
-
-  BlockExpr *block = dyn_cast<BlockExpr>(e);
-  if (!block || !block->getBlockDecl()->capturesVariable(owner.Variable))
-    return nullptr;
-
-  FindCaptureVisitor visitor(S.Context, owner.Variable);
-  visitor.Visit(block->getBlockDecl()->getBody());
-  return visitor.VarWillBeReased ? nullptr : visitor.Capturer;
-}
-
-static void diagnoseRetainCycle(Sema &S, Expr *capturer,
-                                RetainCycleOwner &owner) {
-  assert(capturer);
-  assert(owner.Variable && owner.Loc.isValid());
-
-  S.Diag(capturer->getExprLoc(), diag::warn_arc_retain_cycle)
-    << owner.Variable << capturer->getSourceRange();
-  S.Diag(owner.Loc, diag::note_arc_retain_cycle_owner)
-    << owner.Indirect << owner.Range;
-}
-
-/// Check for a keyword selector that starts with the word 'add' or
-/// 'set'.
-static bool isSetterLikeSelector(Selector sel) {
-  if (sel.isUnarySelector()) return false;
-
-  StringRef str = sel.getNameForSlot(0);
-  str = str.ltrim('_');
-  if (str.starts_with("set"))
-    str = str.substr(3);
-  else if (str.starts_with("add")) {
-    // Specially allow 'addOperationWithBlock:'.
-    if (sel.getNumArgs() == 1 && str.starts_with("addOperationWithBlock"))
-      return false;
-    str = str.substr(3);
-  } else
-    return false;
-
-  if (str.empty()) return true;
-  return !isLowercase(str.front());
-}
-
-static std::optional<int>
-GetNSMutableArrayArgumentIndex(Sema &S, ObjCMessageExpr *Message) {
-  bool IsMutableArray = S.NSAPIObj->isSubclassOfNSClass(
-                                                Message->getReceiverInterface(),
-                                                NSAPI::ClassId_NSMutableArray);
-  if (!IsMutableArray) {
-    return std::nullopt;
-  }
-
-  Selector Sel = Message->getSelector();
-
-  std::optional<NSAPI::NSArrayMethodKind> MKOpt =
-      S.NSAPIObj->getNSArrayMethodKind(Sel);
-  if (!MKOpt) {
-    return std::nullopt;
-  }
-
-  NSAPI::NSArrayMethodKind MK = *MKOpt;
-
-  switch (MK) {
-    case NSAPI::NSMutableArr_addObject:
-    case NSAPI::NSMutableArr_insertObjectAtIndex:
-    case NSAPI::NSMutableArr_setObjectAtIndexedSubscript:
-      return 0;
-    case NSAPI::NSMutableArr_replaceObjectAtIndex:
-      return 1;
-
-    default:
-      return std::nullopt;
-  }
-
-  return std::nullopt;
-}
-
-static std::optional<int>
-GetNSMutableDictionaryArgumentIndex(Sema &S, ObjCMessageExpr *Message) {
-  bool IsMutableDictionary = S.NSAPIObj->isSubclassOfNSClass(
-                                            Message->getReceiverInterface(),
-                                            NSAPI::ClassId_NSMutableDictionary);
-  if (!IsMutableDictionary) {
-    return std::nullopt;
-  }
-
-  Selector Sel = Message->getSelector();
-
-  std::optional<NSAPI::NSDictionaryMethodKind> MKOpt =
-      S.NSAPIObj->getNSDictionaryMethodKind(Sel);
-  if (!MKOpt) {
-    return std::nullopt;
-  }
-
-  NSAPI::NSDictionaryMethodKind MK = *MKOpt;
-
-  switch (MK) {
-    case NSAPI::NSMutableDict_setObjectForKey:
-    case NSAPI::NSMutableDict_setValueForKey:
-    case NSAPI::NSMutableDict_setObjectForKeyedSubscript:
-      return 0;
-
-    default:
-      return std::nullopt;
-  }
-
-  return std::nullopt;
-}
-
-static std::optional<int> GetNSSetArgumentIndex(Sema &S,
-                                                ObjCMessageExpr *Message) {
-  bool IsMutableSet = S.NSAPIObj->isSubclassOfNSClass(
-                                                Message->getReceiverInterface(),
-                                                NSAPI::ClassId_NSMutableSet);
-
-  bool IsMutableOrderedSet = S.NSAPIObj->isSubclassOfNSClass(
-                                            Message->getReceiverInterface(),
-                                            NSAPI::ClassId_NSMutableOrderedSet);
-  if (!IsMutableSet && !IsMutableOrderedSet) {
-    return std::nullopt;
-  }
-
-  Selector Sel = Message->getSelector();
-
-  std::optional<NSAPI::NSSetMethodKind> MKOpt =
-      S.NSAPIObj->getNSSetMethodKind(Sel);
-  if (!MKOpt) {
-    return std::nullopt;
-  }
-
-  NSAPI::NSSetMethodKind MK = *MKOpt;
-
-  switch (MK) {
-    case NSAPI::NSMutableSet_addObject:
-    case NSAPI::NSOrderedSet_setObjectAtIndex:
-    case NSAPI::NSOrderedSet_setObjectAtIndexedSubscript:
-    case NSAPI::NSOrderedSet_insertObjectAtIndex:
-      return 0;
-    case NSAPI::NSOrderedSet_replaceObjectAtIndexWithObject:
-      return 1;
-  }
-
-  return std::nullopt;
-}
-
-void Sema::CheckObjCCircularContainer(ObjCMessageExpr *Message) {
-  if (!Message->isInstanceMessage()) {
-    return;
-  }
-
-  std::optional<int> ArgOpt;
-
-  if (!(ArgOpt = GetNSMutableArrayArgumentIndex(*this, Message)) &&
-      !(ArgOpt = GetNSMutableDictionaryArgumentIndex(*this, Message)) &&
-      !(ArgOpt = GetNSSetArgumentIndex(*this, Message))) {
-    return;
-  }
-
-  int ArgIndex = *ArgOpt;
-
-  Expr *Arg = Message->getArg(ArgIndex)->IgnoreImpCasts();
-  if (OpaqueValueExpr *OE = dyn_cast<OpaqueValueExpr>(Arg)) {
-    Arg = OE->getSourceExpr()->IgnoreImpCasts();
-  }
-
-  if (Message->getReceiverKind() == ObjCMessageExpr::SuperInstance) {
-    if (DeclRefExpr *ArgRE = dyn_cast<DeclRefExpr>(Arg)) {
-      if (ArgRE->isObjCSelfExpr()) {
-        Diag(Message->getSourceRange().getBegin(),
-             diag::warn_objc_circular_container)
-          << ArgRE->getDecl() << StringRef("'super'");
-      }
-    }
-  } else {
-    Expr *Receiver = Message->getInstanceReceiver()->IgnoreImpCasts();
-
-    if (OpaqueValueExpr *OE = dyn_cast<OpaqueValueExpr>(Receiver)) {
-      Receiver = OE->getSourceExpr()->IgnoreImpCasts();
-    }
-
-    if (DeclRefExpr *ReceiverRE = dyn_cast<DeclRefExpr>(Receiver)) {
-      if (DeclRefExpr *ArgRE = dyn_cast<DeclRefExpr>(Arg)) {
-        if (ReceiverRE->getDecl() == ArgRE->getDecl()) {
-          ValueDecl *Decl = ReceiverRE->getDecl();
-          Diag(Message->getSourceRange().getBegin(),
-               diag::warn_objc_circular_container)
-            << Decl << Decl;
-          if (!ArgRE->isObjCSelfExpr()) {
-            Diag(Decl->getLocation(),
-                 diag::note_objc_circular_container_declared_here)
-              << Decl;
-          }
-        }
-      }
-    } else if (ObjCIvarRefExpr *IvarRE = dyn_cast<ObjCIvarRefExpr>(Receiver)) {
-      if (ObjCIvarRefExpr *IvarArgRE = dyn_cast<ObjCIvarRefExpr>(Arg)) {
-        if (IvarRE->getDecl() == IvarArgRE->getDecl()) {
-          ObjCIvarDecl *Decl = IvarRE->getDecl();
-          Diag(Message->getSourceRange().getBegin(),
-               diag::warn_objc_circular_container)
-            << Decl << Decl;
-          Diag(Decl->getLocation(),
-               diag::note_objc_circular_container_declared_here)
-            << Decl;
-        }
-      }
-    }
-  }
-}
-
-/// Check a message send to see if it's likely to cause a retain cycle.
-void Sema::checkRetainCycles(ObjCMessageExpr *msg) {
-  // Only check instance methods whose selector looks like a setter.
-  if (!msg->isInstanceMessage() || !isSetterLikeSelector(msg->getSelector()))
-    return;
-
-  // Try to find a variable that the receiver is strongly owned by.
-  RetainCycleOwner owner;
-  if (msg->getReceiverKind() == ObjCMessageExpr::Instance) {
-    if (!findRetainCycleOwner(*this, msg->getInstanceReceiver(), owner))
-      return;
-  } else {
-    assert(msg->getReceiverKind() == ObjCMessageExpr::SuperInstance);
-    owner.Variable = getCurMethodDecl()->getSelfDecl();
-    owner.Loc = msg->getSuperLoc();
-    owner.Range = msg->getSuperLoc();
-  }
-
-  // Check whether the receiver is captured by any of the arguments.
-  const ObjCMethodDecl *MD = msg->getMethodDecl();
-  for (unsigned i = 0, e = msg->getNumArgs(); i != e; ++i) {
-    if (Expr *capturer = findCapturingExpr(*this, msg->getArg(i), owner)) {
-      // noescape blocks should not be retained by the method.
-      if (MD && MD->parameters()[i]->hasAttr<NoEscapeAttr>())
-        continue;
-      return diagnoseRetainCycle(*this, capturer, owner);
-    }
-  }
-}
-
-/// Check a property assign to see if it's likely to cause a retain cycle.
-void Sema::checkRetainCycles(Expr *receiver, Expr *argument) {
-  RetainCycleOwner owner;
-  if (!findRetainCycleOwner(*this, receiver, owner))
-    return;
-
-  if (Expr *capturer = findCapturingExpr(*this, argument, owner))
-    diagnoseRetainCycle(*this, capturer, owner);
-}
-
-void Sema::checkRetainCycles(VarDecl *Var, Expr *Init) {
-  RetainCycleOwner Owner;
-  if (!considerVariable(Var, /*DeclRefExpr=*/nullptr, Owner))
-    return;
-
-  // Because we don't have an expression for the variable, we have to set the
-  // location explicitly here.
-  Owner.Loc = Var->getLocation();
-  Owner.Range = Var->getSourceRange();
-
-  if (Expr *Capturer = findCapturingExpr(*this, Init, Owner))
-    diagnoseRetainCycle(*this, Capturer, Owner);
-}
-
 static bool checkUnsafeAssignLiteral(Sema &S, SourceLocation Loc,
                                      Expr *RHS, bool isProperty) {
   // Check if RHS is an Objective-C object literal, which also can get
@@ -19262,8 +18758,8 @@ static bool checkUnsafeAssignLiteral(Sema &S, SourceLocation Loc,
 
   // This enum needs to match with the 'select' in
   // warn_objc_arc_literal_assign (off-by-1).
-  Sema::ObjCLiteralKind Kind = S.CheckLiteralKind(RHS);
-  if (Kind == Sema::LK_String || Kind == Sema::LK_None)
+  SemaObjC::ObjCLiteralKind Kind = S.ObjC().CheckLiteralKind(RHS);
+  if (Kind == SemaObjC::LK_String || Kind == SemaObjC::LK_None)
     return false;
 
   S.Diag(Loc, diag::warn_arc_literal_assign)
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 87aa0cacc249..328641ed9488 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -42,6 +42,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallBitVector.h"
@@ -5862,7 +5863,8 @@ void Sema::CodeCompleteObjCClassPropertyRefExpr(Scope *S,
                                                 SourceLocation ClassNameLoc,
                                                 bool IsBaseExprStatement) {
   const IdentifierInfo *ClassNamePtr = &ClassName;
-  ObjCInterfaceDecl *IFace = getObjCInterfaceDecl(ClassNamePtr, ClassNameLoc);
+  ObjCInterfaceDecl *IFace =
+      ObjC().getObjCInterfaceDecl(ClassNamePtr, ClassNameLoc);
   if (!IFace)
     return;
   CodeCompletionContext CCContext(
@@ -8176,15 +8178,16 @@ AddClassMessageCompletions(Sema &SemaRef, Scope *S, ParsedType Receiver,
                     N = SemaRef.getExternalSource()->GetNumExternalSelectors();
            I != N; ++I) {
         Selector Sel = SemaRef.getExternalSource()->GetExternalSelector(I);
-        if (Sel.isNull() || SemaRef.MethodPool.count(Sel))
+        if (Sel.isNull() || SemaRef.ObjC().MethodPool.count(Sel))
           continue;
 
-        SemaRef.ReadMethodPool(Sel);
+        SemaRef.ObjC().ReadMethodPool(Sel);
       }
     }
 
-    for (Sema::GlobalMethodPool::iterator M = SemaRef.MethodPool.begin(),
-                                          MEnd = SemaRef.MethodPool.end();
+    for (SemaObjC::GlobalMethodPool::iterator
+             M = SemaRef.ObjC().MethodPool.begin(),
+             MEnd = SemaRef.ObjC().MethodPool.end();
          M != MEnd; ++M) {
       for (ObjCMethodList *MethList = &M->second.second;
            MethList && MethList->getMethod(); MethList = MethList->getNext()) {
@@ -8346,15 +8349,15 @@ void Sema::CodeCompleteObjCInstanceMessage(
       for (uint32_t I = 0, N = ExternalSource->GetNumExternalSelectors();
            I != N; ++I) {
         Selector Sel = ExternalSource->GetExternalSelector(I);
-        if (Sel.isNull() || MethodPool.count(Sel))
+        if (Sel.isNull() || ObjC().MethodPool.count(Sel))
           continue;
 
-        ReadMethodPool(Sel);
+        ObjC().ReadMethodPool(Sel);
       }
     }
 
-    for (GlobalMethodPool::iterator M = MethodPool.begin(),
-                                    MEnd = MethodPool.end();
+    for (SemaObjC::GlobalMethodPool::iterator M = ObjC().MethodPool.begin(),
+                                              MEnd = ObjC().MethodPool.end();
          M != MEnd; ++M) {
       for (ObjCMethodList *MethList = &M->second.first;
            MethList && MethList->getMethod(); MethList = MethList->getNext()) {
@@ -8417,10 +8420,10 @@ void Sema::CodeCompleteObjCSelector(
     for (uint32_t I = 0, N = ExternalSource->GetNumExternalSelectors(); I != N;
          ++I) {
       Selector Sel = ExternalSource->GetExternalSelector(I);
-      if (Sel.isNull() || MethodPool.count(Sel))
+      if (Sel.isNull() || ObjC().MethodPool.count(Sel))
         continue;
 
-      ReadMethodPool(Sel);
+      ObjC().ReadMethodPool(Sel);
     }
   }
 
@@ -8428,8 +8431,8 @@ void Sema::CodeCompleteObjCSelector(
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_SelectorName);
   Results.EnterNewScope();
-  for (GlobalMethodPool::iterator M = MethodPool.begin(),
-                                  MEnd = MethodPool.end();
+  for (SemaObjC::GlobalMethodPool::iterator M = ObjC().MethodPool.begin(),
+                                            MEnd = ObjC().MethodPool.end();
        M != MEnd; ++M) {
 
     Selector Sel = M->first;
@@ -8497,7 +8500,8 @@ void Sema::CodeCompleteObjCProtocolReferences(
     // already seen.
     // FIXME: This doesn't work when caching code-completion results.
     for (const IdentifierLocPair &Pair : Protocols)
-      if (ObjCProtocolDecl *Protocol = LookupProtocol(Pair.first, Pair.second))
+      if (ObjCProtocolDecl *Protocol =
+              ObjC().LookupProtocol(Pair.first, Pair.second))
         Results.Ignore(Protocol);
 
     // Add all protocols.
@@ -9755,10 +9759,10 @@ void Sema::CodeCompleteObjCMethodDeclSelector(
     for (uint32_t I = 0, N = ExternalSource->GetNumExternalSelectors(); I != N;
          ++I) {
       Selector Sel = ExternalSource->GetExternalSelector(I);
-      if (Sel.isNull() || MethodPool.count(Sel))
+      if (Sel.isNull() || ObjC().MethodPool.count(Sel))
         continue;
 
-      ReadMethodPool(Sel);
+      ObjC().ReadMethodPool(Sel);
     }
   }
 
@@ -9772,8 +9776,8 @@ void Sema::CodeCompleteObjCMethodDeclSelector(
     Results.setPreferredType(GetTypeFromParser(ReturnTy).getNonReferenceType());
 
   Results.EnterNewScope();
-  for (GlobalMethodPool::iterator M = MethodPool.begin(),
-                                  MEnd = MethodPool.end();
+  for (SemaObjC::GlobalMethodPool::iterator M = ObjC().MethodPool.begin(),
+                                            MEnd = ObjC().MethodPool.end();
        M != MEnd; ++M) {
     for (ObjCMethodList *MethList = IsInstanceMethod ? &M->second.first
                                                      : &M->second.second;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index fb913034bd83..0dbdf923df95 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -48,6 +48,7 @@
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/STLForwardCompat.h"
@@ -915,7 +916,7 @@ Sema::NameClassification Sema::ClassifyName(Scope *S, CXXScopeSpec &SS,
   // FIXME: This lookup really, really needs to be folded in to the normal
   // unqualified lookup mechanism.
   if (SS.isEmpty() && CurMethod && !isResultTypeOrTemplate(Result, NextToken)) {
-    DeclResult Ivar = LookupIvarInObjCMethod(Result, S, Name);
+    DeclResult Ivar = ObjC().LookupIvarInObjCMethod(Result, S, Name);
     if (Ivar.isInvalid())
       return NameClassification::Error();
     if (Ivar.isUsable())
@@ -1033,7 +1034,7 @@ Corrected:
         // FIXME: This is a gross hack.
         if (ObjCIvarDecl *Ivar = Result.getAsSingle<ObjCIvarDecl>()) {
           DeclResult R =
-              LookupIvarInObjCMethod(Result, S, Ivar->getIdentifier());
+              ObjC().LookupIvarInObjCMethod(Result, S, Ivar->getIdentifier());
           if (R.isInvalid())
             return NameClassification::Error();
           if (R.isUsable())
@@ -1271,7 +1272,7 @@ ExprResult Sema::ActOnNameClassifiedAsNonType(Scope *S, const CXXScopeSpec &SS,
                                               const Token &NextToken) {
   if (getCurMethodDecl() && SS.isEmpty())
     if (auto *Ivar = dyn_cast<ObjCIvarDecl>(Found->getUnderlyingDecl()))
-      return BuildIvarRefExpr(S, NameLoc, Ivar);
+      return ObjC().BuildIvarRefExpr(S, NameLoc, Ivar);
 
   // Reconstruct the lookup result.
   LookupResult Result(*this, Found->getDeclName(), NameLoc, LookupOrdinaryName);
@@ -2310,45 +2311,6 @@ void Sema::ActOnPopScope(SourceLocation Loc, Scope *S) {
   }
 }
 
-/// Look for an Objective-C class in the translation unit.
-///
-/// \param Id The name of the Objective-C class we're looking for. If
-/// typo-correction fixes this name, the Id will be updated
-/// to the fixed name.
-///
-/// \param IdLoc The location of the name in the translation unit.
-///
-/// \param DoTypoCorrection If true, this routine will attempt typo correction
-/// if there is no class with the given name.
-///
-/// \returns The declaration of the named Objective-C class, or NULL if the
-/// class could not be found.
-ObjCInterfaceDecl *Sema::getObjCInterfaceDecl(const IdentifierInfo *&Id,
-                                              SourceLocation IdLoc,
-                                              bool DoTypoCorrection) {
-  // The third "scope" argument is 0 since we aren't enabling lazy built-in
-  // creation from this context.
-  NamedDecl *IDecl = LookupSingleName(TUScope, Id, IdLoc, LookupOrdinaryName);
-
-  if (!IDecl && DoTypoCorrection) {
-    // Perform typo correction at the given location, but only if we
-    // find an Objective-C class name.
-    DeclFilterCCC<ObjCInterfaceDecl> CCC{};
-    if (TypoCorrection C =
-            CorrectTypo(DeclarationNameInfo(Id, IdLoc), LookupOrdinaryName,
-                        TUScope, nullptr, CCC, CTK_ErrorRecovery)) {
-      diagnoseTypo(C, PDiag(diag::err_undef_interface_suggest) << Id);
-      IDecl = C.getCorrectionDeclAs<ObjCInterfaceDecl>();
-      Id = IDecl->getIdentifier();
-    }
-  }
-  ObjCInterfaceDecl *Def = dyn_cast_or_null<ObjCInterfaceDecl>(IDecl);
-  // This routine must always return a class definition, if any.
-  if (Def && Def->getDefinition())
-      Def = Def->getDefinition();
-  return Def;
-}
-
 /// getNonFieldDeclScope - Retrieves the innermost scope, starting
 /// from S, where a non-field would be declared. This routine copes
 /// with the difference between C and C++ scoping rules in structs and
@@ -4415,7 +4377,7 @@ void Sema::mergeObjCMethodDecls(ObjCMethodDecl *newMethod,
        ni != ne && oi != oe; ++ni, ++oi)
     mergeParamDeclAttributes(*ni, *oi, *this);
 
-  CheckObjCMethodOverride(newMethod, oldMethod);
+  ObjC().CheckObjCMethodOverride(newMethod, oldMethod);
 }
 
 static void diagnoseVarDeclTypeMismatch(Sema &S, VarDecl *New, VarDecl* Old) {
@@ -6982,50 +6944,6 @@ static void SetNestedNameSpecifier(Sema &S, DeclaratorDecl *DD, Declarator &D) {
   DD->setQualifierInfo(SS.getWithLocInContext(S.Context));
 }
 
-bool Sema::inferObjCARCLifetime(ValueDecl *decl) {
-  QualType type = decl->getType();
-  Qualifiers::ObjCLifetime lifetime = type.getObjCLifetime();
-  if (lifetime == Qualifiers::OCL_Autoreleasing) {
-    // Various kinds of declaration aren't allowed to be __autoreleasing.
-    unsigned kind = -1U;
-    if (VarDecl *var = dyn_cast<VarDecl>(decl)) {
-      if (var->hasAttr<BlocksAttr>())
-        kind = 0; // __block
-      else if (!var->hasLocalStorage())
-        kind = 1; // global
-    } else if (isa<ObjCIvarDecl>(decl)) {
-      kind = 3; // ivar
-    } else if (isa<FieldDecl>(decl)) {
-      kind = 2; // field
-    }
-
-    if (kind != -1U) {
-      Diag(decl->getLocation(), diag::err_arc_autoreleasing_var)
-        << kind;
-    }
-  } else if (lifetime == Qualifiers::OCL_None) {
-    // Try to infer lifetime.
-    if (!type->isObjCLifetimeType())
-      return false;
-
-    lifetime = type->getObjCARCImplicitLifetime();
-    type = Context.getLifetimeQualifiedType(type, lifetime);
-    decl->setType(type);
-  }
-
-  if (VarDecl *var = dyn_cast<VarDecl>(decl)) {
-    // Thread-local variables cannot have lifetime.
-    if (lifetime && lifetime != Qualifiers::OCL_ExplicitNone &&
-        var->getTLSKind()) {
-      Diag(var->getLocation(), diag::err_arc_thread_ownership)
-        << var->getType();
-      return true;
-    }
-  }
-
-  return false;
-}
-
 void Sema::deduceOpenCLAddressSpace(ValueDecl *Decl) {
   if (Decl->getType().hasAddressSpace())
     return;
@@ -8064,7 +7982,7 @@ NamedDecl *Sema::ActOnVariableDeclarator(
 
   // In auto-retain/release, infer strong retension for variables of
   // retainable type.
-  if (getLangOpts().ObjCAutoRefCount && inferObjCARCLifetime(NewVD))
+  if (getLangOpts().ObjCAutoRefCount && ObjC().inferObjCARCLifetime(NewVD))
     NewVD->setInvalidDecl();
 
   // Handle GNU asm-label extension (encoded as an attribute).
@@ -10878,7 +10796,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
 
   // If there's a #pragma clang arc_cf_code_audited in scope, consider
   // marking the function.
-  AddCFAuditedAttribute(NewFD);
+  ObjC().AddCFAuditedAttribute(NewFD);
 
   // If this is a function definition, check if we have to apply any
   // attributes (i.e. optnone and no_builtin) due to a pragma.
@@ -13208,7 +13126,7 @@ bool Sema::DeduceVariableDeclarationType(VarDecl *VDecl, bool DirectInit,
   assert(VDecl->isLinkageValid());
 
   // In ARC, infer lifetime.
-  if (getLangOpts().ObjCAutoRefCount && inferObjCARCLifetime(VDecl))
+  if (getLangOpts().ObjCAutoRefCount && ObjC().inferObjCARCLifetime(VDecl))
     VDecl->setInvalidDecl();
 
   if (getLangOpts().OpenCL)
@@ -13783,7 +13701,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     checkUnsafeAssigns(VDecl->getLocation(), VDecl->getType(), Init);
 
     if (VDecl->hasAttr<BlocksAttr>())
-      checkRetainCycles(VDecl, Init);
+      ObjC().checkRetainCycles(VDecl, Init);
 
     // It is safe to assign a weak reference into a strong variable.
     // Although this code can still have problems:
@@ -15337,37 +15255,6 @@ void Sema::DiagnoseSizeOfParametersAndReturnValue(
   }
 }
 
-QualType Sema::AdjustParameterTypeForObjCAutoRefCount(QualType T,
-                                                      SourceLocation NameLoc,
-                                                      TypeSourceInfo *TSInfo) {
-  // In ARC, infer a lifetime qualifier for appropriate parameter types.
-  if (!getLangOpts().ObjCAutoRefCount ||
-      T.getObjCLifetime() != Qualifiers::OCL_None || !T->isObjCLifetimeType())
-    return T;
-
-  Qualifiers::ObjCLifetime Lifetime;
-
-  // Special cases for arrays:
-  //   - if it's const, use __unsafe_unretained
-  //   - otherwise, it's an error
-  if (T->isArrayType()) {
-    if (!T.isConstQualified()) {
-      if (DelayedDiagnostics.shouldDelayDiagnostics())
-        DelayedDiagnostics.add(sema::DelayedDiagnostic::makeForbiddenType(
-            NameLoc, diag::err_arc_array_param_no_ownership, T, false));
-      else
-        Diag(NameLoc, diag::err_arc_array_param_no_ownership)
-            << TSInfo->getTypeLoc().getSourceRange();
-    }
-    Lifetime = Qualifiers::OCL_ExplicitNone;
-  } else {
-    Lifetime = T->getObjCARCImplicitLifetime();
-  }
-  T = Context.getLifetimeQualifiedType(T, Lifetime);
-
-  return T;
-}
-
 ParmVarDecl *Sema::CheckParameter(DeclContext *DC, SourceLocation StartLoc,
                                   SourceLocation NameLoc,
                                   const IdentifierInfo *Name, QualType T,
@@ -16401,7 +16288,7 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
           if (!SuperD)
             return false;
           return SuperD->getIdentifier() ==
-                 NSAPIObj->getNSClassId(NSAPI::ClassId_NSObject);
+                 ObjC().NSAPIObj->getNSClassId(NSAPI::ClassId_NSObject);
         };
         // Don't issue this warning for unavailable inits or direct subclasses
         // of NSObject.
@@ -18348,12 +18235,6 @@ bool Sema::ActOnDuplicateDefinition(Decl *Prev, SkipBodyInfo &SkipBody) {
   return true;
 }
 
-void Sema::ActOnObjCContainerStartDefinition(ObjCContainerDecl *IDecl) {
-  assert(IDecl->getLexicalParent() == CurContext &&
-      "The next DeclContext should be lexically contained in the current one.");
-  CurContext = IDecl;
-}
-
 void Sema::ActOnStartCXXMemberDeclarations(Scope *S, Decl *TagD,
                                            SourceLocation FinalLoc,
                                            bool IsFinalSpelledSealed,
@@ -18455,22 +18336,6 @@ void Sema::ActOnTagFinishDefinition(Scope *S, Decl *TagD,
   }
 }
 
-void Sema::ActOnObjCContainerFinishDefinition() {
-  // Exit this scope of this interface definition.
-  PopDeclContext();
-}
-
-void Sema::ActOnObjCTemporaryExitContainerContext(ObjCContainerDecl *ObjCCtx) {
-  assert(ObjCCtx == CurContext && "Mismatch of container contexts");
-  OriginalLexicalContext = ObjCCtx;
-  ActOnObjCContainerFinishDefinition();
-}
-
-void Sema::ActOnObjCReenterContainerContext(ObjCContainerDecl *ObjCCtx) {
-  ActOnObjCContainerStartDefinition(ObjCCtx);
-  OriginalLexicalContext = nullptr;
-}
-
 void Sema::ActOnTagDefinitionError(Scope *S, Decl *TagD) {
   AdjustDeclIfTemplate(TagD);
   TagDecl *Tag = cast<TagDecl>(TagD);
@@ -18870,7 +18735,7 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
 
   // In auto-retain/release, infer strong retension for fields of
   // retainable type.
-  if (getLangOpts().ObjCAutoRefCount && inferObjCARCLifetime(NewFD))
+  if (getLangOpts().ObjCAutoRefCount && ObjC().inferObjCARCLifetime(NewFD))
     NewFD->setInvalidDecl();
 
   if (T.isObjCGCWeak())
@@ -18948,132 +18813,6 @@ bool Sema::CheckNontrivialField(FieldDecl *FD) {
   return false;
 }
 
-/// TranslateIvarVisibility - Translate visibility from a token ID to an
-///  AST enum value.
-static ObjCIvarDecl::AccessControl
-TranslateIvarVisibility(tok::ObjCKeywordKind ivarVisibility) {
-  switch (ivarVisibility) {
-  default: llvm_unreachable("Unknown visitibility kind");
-  case tok::objc_private: return ObjCIvarDecl::Private;
-  case tok::objc_public: return ObjCIvarDecl::Public;
-  case tok::objc_protected: return ObjCIvarDecl::Protected;
-  case tok::objc_package: return ObjCIvarDecl::Package;
-  }
-}
-
-/// ActOnIvar - Each ivar field of an objective-c class is passed into this
-/// in order to create an IvarDecl object for it.
-Decl *Sema::ActOnIvar(Scope *S, SourceLocation DeclStart, Declarator &D,
-                      Expr *BitWidth, tok::ObjCKeywordKind Visibility) {
-
-  const IdentifierInfo *II = D.getIdentifier();
-  SourceLocation Loc = DeclStart;
-  if (II) Loc = D.getIdentifierLoc();
-
-  // FIXME: Unnamed fields can be handled in various different ways, for
-  // example, unnamed unions inject all members into the struct namespace!
-
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
-  QualType T = TInfo->getType();
-
-  if (BitWidth) {
-    // 6.7.2.1p3, 6.7.2.1p4
-    BitWidth = VerifyBitField(Loc, II, T, /*IsMsStruct*/false, BitWidth).get();
-    if (!BitWidth)
-      D.setInvalidType();
-  } else {
-    // Not a bitfield.
-
-    // validate II.
-
-  }
-  if (T->isReferenceType()) {
-    Diag(Loc, diag::err_ivar_reference_type);
-    D.setInvalidType();
-  }
-  // C99 6.7.2.1p8: A member of a structure or union may have any type other
-  // than a variably modified type.
-  else if (T->isVariablyModifiedType()) {
-    if (!tryToFixVariablyModifiedVarType(
-            TInfo, T, Loc, diag::err_typecheck_ivar_variable_size))
-      D.setInvalidType();
-  }
-
-  // Get the visibility (access control) for this ivar.
-  ObjCIvarDecl::AccessControl ac =
-    Visibility != tok::objc_not_keyword ? TranslateIvarVisibility(Visibility)
-                                        : ObjCIvarDecl::None;
-  // Must set ivar's DeclContext to its enclosing interface.
-  ObjCContainerDecl *EnclosingDecl = cast<ObjCContainerDecl>(CurContext);
-  if (!EnclosingDecl || EnclosingDecl->isInvalidDecl())
-    return nullptr;
-  ObjCContainerDecl *EnclosingContext;
-  if (ObjCImplementationDecl *IMPDecl =
-      dyn_cast<ObjCImplementationDecl>(EnclosingDecl)) {
-    if (LangOpts.ObjCRuntime.isFragile()) {
-    // Case of ivar declared in an implementation. Context is that of its class.
-      EnclosingContext = IMPDecl->getClassInterface();
-      assert(EnclosingContext && "Implementation has no class interface!");
-    }
-    else
-      EnclosingContext = EnclosingDecl;
-  } else {
-    if (ObjCCategoryDecl *CDecl =
-        dyn_cast<ObjCCategoryDecl>(EnclosingDecl)) {
-      if (LangOpts.ObjCRuntime.isFragile() || !CDecl->IsClassExtension()) {
-        Diag(Loc, diag::err_misplaced_ivar) << CDecl->IsClassExtension();
-        return nullptr;
-      }
-    }
-    EnclosingContext = EnclosingDecl;
-  }
-
-  // Construct the decl.
-  ObjCIvarDecl *NewID = ObjCIvarDecl::Create(
-      Context, EnclosingContext, DeclStart, Loc, II, T, TInfo, ac, BitWidth);
-
-  if (T->containsErrors())
-    NewID->setInvalidDecl();
-
-  if (II) {
-    NamedDecl *PrevDecl =
-        LookupSingleName(S, II, Loc, LookupMemberName,
-                         RedeclarationKind::ForVisibleRedeclaration);
-    if (PrevDecl && isDeclInScope(PrevDecl, EnclosingContext, S)
-        && !isa<TagDecl>(PrevDecl)) {
-      Diag(Loc, diag::err_duplicate_member) << II;
-      Diag(PrevDecl->getLocation(), diag::note_previous_declaration);
-      NewID->setInvalidDecl();
-    }
-  }
-
-  // Process attributes attached to the ivar.
-  ProcessDeclAttributes(S, NewID, D);
-
-  if (D.isInvalidType())
-    NewID->setInvalidDecl();
-
-  // In ARC, infer 'retaining' for ivars of retainable type.
-  if (getLangOpts().ObjCAutoRefCount && inferObjCARCLifetime(NewID))
-    NewID->setInvalidDecl();
-
-  if (D.getDeclSpec().isModulePrivateSpecified())
-    NewID->setModulePrivate();
-
-  if (II) {
-    // FIXME: When interfaces are DeclContexts, we'll need to add
-    // these to the interface.
-    S->AddDecl(NewID);
-    IdResolver.AddDecl(NewID);
-  }
-
-  if (LangOpts.ObjCRuntime.isNonFragile() &&
-      !NewID->isInvalidDecl() && isa<ObjCInterfaceDecl>(EnclosingDecl))
-    Diag(Loc, diag::warn_ivars_in_interface);
-
-  return NewID;
-}
-
 /// ActOnLastBitfield - This routine handles synthesized bitfields rules for
 /// class and class extensions. For every class \@interface and class
 /// extension \@interface, if the last ivar is a bitfield of any type,
@@ -19777,7 +19516,7 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
       // Must enforce the rule that ivars in the base classes may not be
       // duplicates.
       if (ID->getSuperClass())
-        DiagnoseDuplicateIvars(ID, ID->getSuperClass());
+        ObjC().DiagnoseDuplicateIvars(ID, ID->getSuperClass());
     } else if (ObjCImplementationDecl *IMPDecl =
                   dyn_cast<ObjCImplementationDecl>(EnclosingDecl)) {
       assert(IMPDecl && "ActOnFields - missing ObjCImplementationDecl");
@@ -19785,7 +19524,8 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
         // Ivar declared in @implementation never belongs to the implementation.
         // Only it is in implementation's lexical context.
         ClsFields[I]->setLexicalDeclContext(IMPDecl);
-      CheckImplementationIvars(IMPDecl, ClsFields, RecFields.size(), RBrac);
+      ObjC().CheckImplementationIvars(IMPDecl, ClsFields, RecFields.size(),
+                                      RBrac);
       IMPDecl->setIvarLBraceLoc(LBrac);
       IMPDecl->setIvarRBraceLoc(RBrac);
     } else if (ObjCCategoryDecl *CDecl =
@@ -20659,10 +20399,6 @@ void Sema::ActOnPragmaWeakAlias(IdentifierInfo* Name,
   }
 }
 
-ObjCContainerDecl *Sema::getObjCDeclContext() const {
-  return (dyn_cast_or_null<ObjCContainerDecl>(CurContext));
-}
-
 Sema::FunctionEmissionStatus Sema::getEmissionStatus(const FunctionDecl *FD,
                                                      bool Final) {
   assert(FD && "Expected non-null FunctionDecl");
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 6d957ac09e1c..777171f4f15f 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -42,6 +42,7 @@
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/StringExtras.h"
@@ -6560,13 +6561,13 @@ static bool isErrorParameter(Sema &S, QualType QT) {
   // Check for NSError**.
   if (const auto *OPT = Pointee->getAs<ObjCObjectPointerType>())
     if (const auto *ID = OPT->getInterfaceDecl())
-      if (ID->getIdentifier() == S.getNSErrorIdent())
+      if (ID->getIdentifier() == S.ObjC().getNSErrorIdent())
         return true;
 
   // Check for CFError**.
   if (const auto *PT = Pointee->getAs<PointerType>())
     if (const auto *RT = PT->getPointeeType()->getAs<RecordType>())
-      if (S.isCFError(RT->getDecl()))
+      if (S.ObjC().isCFError(RT->getDecl()))
         return true;
 
   return false;
@@ -6697,7 +6698,7 @@ static void checkSwiftAsyncErrorBlock(Sema &S, Decl *D,
       // Check for NSError *.
       if (const auto *ObjCPtrTy = Param->getAs<ObjCObjectPointerType>()) {
         if (const auto *ID = ObjCPtrTy->getInterfaceDecl()) {
-          if (ID->getIdentifier() == S.getNSErrorIdent()) {
+          if (ID->getIdentifier() == S.ObjC().getNSErrorIdent()) {
             AnyErrorParams = true;
             break;
           }
@@ -6706,7 +6707,7 @@ static void checkSwiftAsyncErrorBlock(Sema &S, Decl *D,
       // Check for CFError *.
       if (const auto *PtrTy = Param->getAs<PointerType>()) {
         if (const auto *RT = PtrTy->getPointeeType()->getAs<RecordType>()) {
-          if (S.isCFError(RT->getDecl())) {
+          if (S.ObjC().isCFError(RT->getDecl())) {
             AnyErrorParams = true;
             break;
           }
@@ -8837,7 +8838,7 @@ static bool tryMakeVariablePseudoStrong(Sema &S, VarDecl *VD,
 
   Qualifiers::ObjCLifetime LifetimeQual = Ty.getQualifiers().getObjCLifetime();
 
-  // Sema::inferObjCARCLifetime must run after processing decl attributes
+  // SemaObjC::inferObjCARCLifetime must run after processing decl attributes
   // (because __block lowers to an attribute), so if the lifetime hasn't been
   // explicitly specified, infer it locally now.
   if (LifetimeQual == Qualifiers::OCL_None)
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 53238d355ea0..822538198505 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -44,6 +44,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -17054,7 +17055,7 @@ VarDecl *Sema::BuildExceptionDeclaration(Scope *S, TypeSourceInfo *TInfo,
   ExDecl->setExceptionVariable(true);
 
   // In ARC, infer 'retaining' for variables of retainable type.
-  if (getLangOpts().ObjCAutoRefCount && inferObjCARCLifetime(ExDecl))
+  if (getLangOpts().ObjCAutoRefCount && ObjC().inferObjCARCLifetime(ExDecl))
     Invalid = true;
 
   if (!Invalid && !ExDeclType->isDependentType()) {
@@ -18853,61 +18854,6 @@ void Sema::MarkVirtualMembersReferenced(SourceLocation Loc,
   }
 }
 
-/// SetIvarInitializers - This routine builds initialization ASTs for the
-/// Objective-C implementation whose ivars need be initialized.
-void Sema::SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation) {
-  if (!getLangOpts().CPlusPlus)
-    return;
-  if (ObjCInterfaceDecl *OID = ObjCImplementation->getClassInterface()) {
-    SmallVector<ObjCIvarDecl*, 8> ivars;
-    CollectIvarsToConstructOrDestruct(OID, ivars);
-    if (ivars.empty())
-      return;
-    SmallVector<CXXCtorInitializer*, 32> AllToInit;
-    for (unsigned i = 0; i < ivars.size(); i++) {
-      FieldDecl *Field = ivars[i];
-      if (Field->isInvalidDecl())
-        continue;
-
-      CXXCtorInitializer *Member;
-      InitializedEntity InitEntity = InitializedEntity::InitializeMember(Field);
-      InitializationKind InitKind =
-        InitializationKind::CreateDefault(ObjCImplementation->getLocation());
-
-      InitializationSequence InitSeq(*this, InitEntity, InitKind, std::nullopt);
-      ExprResult MemberInit =
-          InitSeq.Perform(*this, InitEntity, InitKind, std::nullopt);
-      MemberInit = MaybeCreateExprWithCleanups(MemberInit);
-      // Note, MemberInit could actually come back empty if no initialization
-      // is required (e.g., because it would call a trivial default constructor)
-      if (!MemberInit.get() || MemberInit.isInvalid())
-        continue;
-
-      Member =
-        new (Context) CXXCtorInitializer(Context, Field, SourceLocation(),
-                                         SourceLocation(),
-                                         MemberInit.getAs<Expr>(),
-                                         SourceLocation());
-      AllToInit.push_back(Member);
-
-      // Be sure that the destructor is accessible and is marked as referenced.
-      if (const RecordType *RecordTy =
-              Context.getBaseElementType(Field->getType())
-                  ->getAs<RecordType>()) {
-        CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getDecl());
-        if (CXXDestructorDecl *Destructor = LookupDestructor(RD)) {
-          MarkFunctionReferenced(Field->getLocation(), Destructor);
-          CheckDestructorAccess(Field->getLocation(), Destructor,
-                            PDiag(diag::err_access_dtor_ivar)
-                              << Context.getBaseElementType(Field->getType()));
-        }
-      }
-    }
-    ObjCImplementation->setIvarInitializers(Context,
-                                            AllToInit.data(), AllToInit.size());
-  }
-}
-
 static
 void DelegatingCycleHelper(CXXConstructorDecl* Ctor,
                            llvm::SmallPtrSet<CXXConstructorDecl*, 4> &Valid,
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index 934ba174a426..6d4379283f19 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -21,10 +21,13 @@
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/DelayedDiagnostic.h"
+#include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 
@@ -39,8 +42,9 @@ using namespace clang;
 ///
 /// \return true to indicate that there was an error and appropriate
 ///   actions were taken
-bool Sema::checkInitMethod(ObjCMethodDecl *method,
-                           QualType receiverTypeIfCall) {
+bool SemaObjC::checkInitMethod(ObjCMethodDecl *method,
+                               QualType receiverTypeIfCall) {
+  ASTContext &Context = getASTContext();
   if (method->isInvalidDecl()) return true;
 
   // This castAs is safe: methods that don't return an object
@@ -97,7 +101,8 @@ bool Sema::checkInitMethod(ObjCMethodDecl *method,
 
   // If we're in a system header, and this is not a call, just make
   // the method unusable.
-  if (receiverTypeIfCall.isNull() && getSourceManager().isInSystemHeader(loc)) {
+  if (receiverTypeIfCall.isNull() &&
+      SemaRef.getSourceManager().isInSystemHeader(loc)) {
     method->addAttr(UnavailableAttr::CreateImplicit(Context, "",
                       UnavailableAttr::IR_ARCInitReturnsUnrelated, loc));
     return true;
@@ -133,8 +138,9 @@ static void diagnoseNoescape(const ParmVarDecl *NewD, const ParmVarDecl *OldD,
         << cast<ObjCMethodDecl>(NewD->getDeclContext());
 }
 
-void Sema::CheckObjCMethodOverride(ObjCMethodDecl *NewMethod,
-                                   const ObjCMethodDecl *Overridden) {
+void SemaObjC::CheckObjCMethodOverride(ObjCMethodDecl *NewMethod,
+                                       const ObjCMethodDecl *Overridden) {
+  ASTContext &Context = getASTContext();
   if (Overridden->hasRelatedResultType() &&
       !NewMethod->hasRelatedResultType()) {
     // This can only happen when the method follows a naming convention that
@@ -216,13 +222,14 @@ void Sema::CheckObjCMethodOverride(ObjCMethodDecl *NewMethod,
       Diag(oldDecl->getLocation(), diag::note_previous_decl) << "parameter";
     }
 
-    diagnoseNoescape(newDecl, oldDecl, *this);
+    diagnoseNoescape(newDecl, oldDecl, SemaRef);
   }
 }
 
 /// Check a method declaration for compatibility with the Objective-C
 /// ARC conventions.
-bool Sema::CheckARCMethodDecl(ObjCMethodDecl *method) {
+bool SemaObjC::CheckARCMethodDecl(ObjCMethodDecl *method) {
+  ASTContext &Context = getASTContext();
   ObjCMethodFamily family = method->getMethodFamily();
   switch (family) {
   case OMF_None:
@@ -326,7 +333,7 @@ static void DiagnoseObjCImplementedDeprecations(Sema &S, const NamedDecl *ND,
 
 /// AddAnyMethodToGlobalPool - Add any method, instance or factory to global
 /// pool.
-void Sema::AddAnyMethodToGlobalPool(Decl *D) {
+void SemaObjC::AddAnyMethodToGlobalPool(Decl *D) {
   ObjCMethodDecl *MDecl = dyn_cast_or_null<ObjCMethodDecl>(D);
 
   // If we don't have a valid method decl, simply return.
@@ -359,12 +366,14 @@ HasExplicitOwnershipAttr(Sema &S, ParmVarDecl *Param) {
 
 /// ActOnStartOfObjCMethodDef - This routine sets up parameters; invisible
 /// and user declared, in the method definition's AST.
-void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
-  ImplicitlyRetainedSelfLocs.clear();
-  assert((getCurMethodDecl() == nullptr) && "Methodparsing confused");
+void SemaObjC::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
+  ASTContext &Context = getASTContext();
+  SemaRef.ImplicitlyRetainedSelfLocs.clear();
+  assert((SemaRef.getCurMethodDecl() == nullptr) && "Methodparsing confused");
   ObjCMethodDecl *MDecl = dyn_cast_or_null<ObjCMethodDecl>(D);
 
-  PushExpressionEvaluationContext(ExprEvalContexts.back().Context);
+  SemaRef.PushExpressionEvaluationContext(
+      SemaRef.ExprEvalContexts.back().Context);
 
   // If we don't have a valid method decl, simply return.
   if (!MDecl)
@@ -373,13 +382,13 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
   QualType ResultType = MDecl->getReturnType();
   if (!ResultType->isDependentType() && !ResultType->isVoidType() &&
       !MDecl->isInvalidDecl() &&
-      RequireCompleteType(MDecl->getLocation(), ResultType,
-                          diag::err_func_def_incomplete_result))
+      SemaRef.RequireCompleteType(MDecl->getLocation(), ResultType,
+                                  diag::err_func_def_incomplete_result))
     MDecl->setInvalidDecl();
 
   // Allow all of Sema to see that we are entering a method definition.
-  PushDeclContext(FnBodyScope, MDecl);
-  PushFunctionScope();
+  SemaRef.PushDeclContext(FnBodyScope, MDecl);
+  SemaRef.PushFunctionScope();
 
   // Create Decl objects for each parameter, entrring them in the scope for
   // binding to their use.
@@ -387,23 +396,22 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
   // Insert the invisible arguments, self and _cmd!
   MDecl->createImplicitParams(Context, MDecl->getClassInterface());
 
-  PushOnScopeChains(MDecl->getSelfDecl(), FnBodyScope);
-  PushOnScopeChains(MDecl->getCmdDecl(), FnBodyScope);
+  SemaRef.PushOnScopeChains(MDecl->getSelfDecl(), FnBodyScope);
+  SemaRef.PushOnScopeChains(MDecl->getCmdDecl(), FnBodyScope);
 
   // The ObjC parser requires parameter names so there's no need to check.
-  CheckParmsForFunctionDef(MDecl->parameters(),
-                           /*CheckParameterNames=*/false);
+  SemaRef.CheckParmsForFunctionDef(MDecl->parameters(),
+                                   /*CheckParameterNames=*/false);
 
   // Introduce all of the other parameters into this scope.
   for (auto *Param : MDecl->parameters()) {
-    if (!Param->isInvalidDecl() &&
-        getLangOpts().ObjCAutoRefCount &&
-        !HasExplicitOwnershipAttr(*this, Param))
+    if (!Param->isInvalidDecl() && getLangOpts().ObjCAutoRefCount &&
+        !HasExplicitOwnershipAttr(SemaRef, Param))
       Diag(Param->getLocation(), diag::warn_arc_strong_pointer_objc_pointer) <<
             Param->getType();
 
     if (Param->getIdentifier())
-      PushOnScopeChains(Param, FnBodyScope);
+      SemaRef.PushOnScopeChains(Param, FnBodyScope);
   }
 
   // In ARC, disallow definition of retain/release/autorelease/retainCount
@@ -456,17 +464,17 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
       // No need to issue deprecated warning if deprecated mehod in class/category
       // is being implemented in its own implementation (no overriding is involved).
       if (!ImplDeclOfMethodDecl || ImplDeclOfMethodDecl != ImplDeclOfMethodDef)
-        DiagnoseObjCImplementedDeprecations(*this, IMD, MDecl->getLocation());
+        DiagnoseObjCImplementedDeprecations(SemaRef, IMD, MDecl->getLocation());
     }
 
     if (MDecl->getMethodFamily() == OMF_init) {
       if (MDecl->isDesignatedInitializerForTheInterface()) {
-        getCurFunction()->ObjCIsDesignatedInit = true;
-        getCurFunction()->ObjCWarnForNoDesignatedInitChain =
+        SemaRef.getCurFunction()->ObjCIsDesignatedInit = true;
+        SemaRef.getCurFunction()->ObjCWarnForNoDesignatedInitChain =
             IC->getSuperClass() != nullptr;
       } else if (IC->hasDesignatedInitializers()) {
-        getCurFunction()->ObjCIsSecondaryInit = true;
-        getCurFunction()->ObjCWarnForNoInitDelegation = true;
+        SemaRef.getCurFunction()->ObjCIsSecondaryInit = true;
+        SemaRef.getCurFunction()->ObjCWarnForNoInitDelegation = true;
       }
     }
 
@@ -479,25 +487,25 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
       if (Family == OMF_dealloc) {
         if (!(getLangOpts().ObjCAutoRefCount ||
               getLangOpts().getGC() == LangOptions::GCOnly))
-          getCurFunction()->ObjCShouldCallSuper = true;
+          SemaRef.getCurFunction()->ObjCShouldCallSuper = true;
 
       } else if (Family == OMF_finalize) {
         if (Context.getLangOpts().getGC() != LangOptions::NonGC)
-          getCurFunction()->ObjCShouldCallSuper = true;
+          SemaRef.getCurFunction()->ObjCShouldCallSuper = true;
 
       } else {
         const ObjCMethodDecl *SuperMethod =
           SuperClass->lookupMethod(MDecl->getSelector(),
                                    MDecl->isInstanceMethod());
-        getCurFunction()->ObjCShouldCallSuper =
-          (SuperMethod && SuperMethod->hasAttr<ObjCRequiresSuperAttr>());
+        SemaRef.getCurFunction()->ObjCShouldCallSuper =
+            (SuperMethod && SuperMethod->hasAttr<ObjCRequiresSuperAttr>());
       }
     }
   }
 
   // Some function attributes (like OptimizeNoneAttr) need actions before
   // parsing body started.
-  applyFunctionAttributesBeforeParsingBody(D);
+  SemaRef.applyFunctionAttributesBeforeParsingBody(D);
 }
 
 namespace {
@@ -542,29 +550,26 @@ static void diagnoseUseOfProtocols(Sema &TheSema,
   }
 }
 
-void Sema::
-ActOnSuperClassOfClassInterface(Scope *S,
-                                SourceLocation AtInterfaceLoc,
-                                ObjCInterfaceDecl *IDecl,
-                                IdentifierInfo *ClassName,
-                                SourceLocation ClassLoc,
-                                IdentifierInfo *SuperName,
-                                SourceLocation SuperLoc,
-                                ArrayRef<ParsedType> SuperTypeArgs,
-                                SourceRange SuperTypeArgsRange) {
+void SemaObjC::ActOnSuperClassOfClassInterface(
+    Scope *S, SourceLocation AtInterfaceLoc, ObjCInterfaceDecl *IDecl,
+    IdentifierInfo *ClassName, SourceLocation ClassLoc,
+    IdentifierInfo *SuperName, SourceLocation SuperLoc,
+    ArrayRef<ParsedType> SuperTypeArgs, SourceRange SuperTypeArgsRange) {
+  ASTContext &Context = getASTContext();
   // Check if a different kind of symbol declared in this scope.
-  NamedDecl *PrevDecl = LookupSingleName(TUScope, SuperName, SuperLoc,
-                                         LookupOrdinaryName);
+  NamedDecl *PrevDecl = SemaRef.LookupSingleName(
+      SemaRef.TUScope, SuperName, SuperLoc, Sema::LookupOrdinaryName);
 
   if (!PrevDecl) {
     // Try to correct for a typo in the superclass name without correcting
     // to the class we're defining.
     ObjCInterfaceValidatorCCC CCC(IDecl);
-    if (TypoCorrection Corrected = CorrectTypo(
-            DeclarationNameInfo(SuperName, SuperLoc), LookupOrdinaryName,
-            TUScope, nullptr, CCC, CTK_ErrorRecovery)) {
-      diagnoseTypo(Corrected, PDiag(diag::err_undef_superclass_suggest)
-                   << SuperName << ClassName);
+    if (TypoCorrection Corrected = SemaRef.CorrectTypo(
+            DeclarationNameInfo(SuperName, SuperLoc), Sema::LookupOrdinaryName,
+            SemaRef.TUScope, nullptr, CCC, Sema::CTK_ErrorRecovery)) {
+      SemaRef.diagnoseTypo(Corrected,
+                           SemaRef.PDiag(diag::err_undef_superclass_suggest)
+                               << SuperName << ClassName);
       PrevDecl = Corrected.getCorrectionDeclAs<ObjCInterfaceDecl>();
     }
   }
@@ -580,7 +585,7 @@ ActOnSuperClassOfClassInterface(Scope *S,
 
     // Diagnose classes that inherit from deprecated classes.
     if (SuperClassDecl) {
-      (void)DiagnoseUseOfDecl(SuperClassDecl, SuperLoc);
+      (void)SemaRef.DiagnoseUseOfDecl(SuperClassDecl, SuperLoc);
       SuperClassType = Context.getObjCInterfaceType(SuperClassDecl);
     }
 
@@ -599,7 +604,8 @@ ActOnSuperClassOfClassInterface(Scope *S,
             // @interface NewI @end
             // typedef NewI DeprI __attribute__((deprecated("blah")))
             // @interface SI : DeprI /* warn here */ @end
-            (void)DiagnoseUseOfDecl(const_cast<TypedefNameDecl*>(TDecl), SuperLoc);
+            (void)SemaRef.DiagnoseUseOfDecl(
+                const_cast<TypedefNameDecl *>(TDecl), SuperLoc);
           }
         }
       }
@@ -619,12 +625,10 @@ ActOnSuperClassOfClassInterface(Scope *S,
       if (!SuperClassDecl)
         Diag(SuperLoc, diag::err_undef_superclass)
           << SuperName << ClassName << SourceRange(AtInterfaceLoc, ClassLoc);
-      else if (RequireCompleteType(SuperLoc,
-                                   SuperClassType,
-                                   diag::err_forward_superclass,
-                                   SuperClassDecl->getDeclName(),
-                                   ClassName,
-                                   SourceRange(AtInterfaceLoc, ClassLoc))) {
+      else if (SemaRef.RequireCompleteType(
+                   SuperLoc, SuperClassType, diag::err_forward_superclass,
+                   SuperClassDecl->getDeclName(), ClassName,
+                   SourceRange(AtInterfaceLoc, ClassLoc))) {
         SuperClassDecl = nullptr;
         SuperClassType = QualType();
       }
@@ -639,22 +643,15 @@ ActOnSuperClassOfClassInterface(Scope *S,
     TypeSourceInfo *SuperClassTInfo = nullptr;
     if (!SuperTypeArgs.empty()) {
       TypeResult fullSuperClassType = actOnObjCTypeArgsAndProtocolQualifiers(
-                                        S,
-                                        SuperLoc,
-                                        CreateParsedType(SuperClassType,
-                                                         nullptr),
-                                        SuperTypeArgsRange.getBegin(),
-                                        SuperTypeArgs,
-                                        SuperTypeArgsRange.getEnd(),
-                                        SourceLocation(),
-                                        { },
-                                        { },
-                                        SourceLocation());
+          S, SuperLoc, SemaRef.CreateParsedType(SuperClassType, nullptr),
+          SuperTypeArgsRange.getBegin(), SuperTypeArgs,
+          SuperTypeArgsRange.getEnd(), SourceLocation(), {}, {},
+          SourceLocation());
       if (!fullSuperClassType.isUsable())
         return;
 
-      SuperClassType = GetTypeFromParser(fullSuperClassType.get(),
-                                         &SuperClassTInfo);
+      SuperClassType =
+          SemaRef.GetTypeFromParser(fullSuperClassType.get(), &SuperClassTInfo);
     }
 
     if (!SuperClassTInfo) {
@@ -667,26 +664,24 @@ ActOnSuperClassOfClassInterface(Scope *S,
   }
 }
 
-DeclResult Sema::actOnObjCTypeParam(Scope *S,
-                                    ObjCTypeParamVariance variance,
-                                    SourceLocation varianceLoc,
-                                    unsigned index,
-                                    IdentifierInfo *paramName,
-                                    SourceLocation paramLoc,
-                                    SourceLocation colonLoc,
-                                    ParsedType parsedTypeBound) {
+DeclResult SemaObjC::actOnObjCTypeParam(
+    Scope *S, ObjCTypeParamVariance variance, SourceLocation varianceLoc,
+    unsigned index, IdentifierInfo *paramName, SourceLocation paramLoc,
+    SourceLocation colonLoc, ParsedType parsedTypeBound) {
+  ASTContext &Context = getASTContext();
   // If there was an explicitly-provided type bound, check it.
   TypeSourceInfo *typeBoundInfo = nullptr;
   if (parsedTypeBound) {
     // The type bound can be any Objective-C pointer type.
-    QualType typeBound = GetTypeFromParser(parsedTypeBound, &typeBoundInfo);
+    QualType typeBound =
+        SemaRef.GetTypeFromParser(parsedTypeBound, &typeBoundInfo);
     if (typeBound->isObjCObjectPointerType()) {
       // okay
     } else if (typeBound->isObjCObjectType()) {
       // The user forgot the * on an Objective-C pointer type, e.g.,
       // "T : NSView".
-      SourceLocation starLoc = getLocForEndOfToken(
-                                 typeBoundInfo->getTypeLoc().getEndLoc());
+      SourceLocation starLoc =
+          SemaRef.getLocForEndOfToken(typeBoundInfo->getTypeLoc().getEndLoc());
       Diag(typeBoundInfo->getTypeLoc().getBeginLoc(),
            diag::err_objc_type_param_bound_missing_pointer)
         << typeBound << paramName
@@ -766,15 +761,16 @@ DeclResult Sema::actOnObjCTypeParam(Scope *S,
   }
 
   // Create the type parameter.
-  return ObjCTypeParamDecl::Create(Context, CurContext, variance, varianceLoc,
-                                   index, paramLoc, paramName, colonLoc,
-                                   typeBoundInfo);
+  return ObjCTypeParamDecl::Create(Context, SemaRef.CurContext, variance,
+                                   varianceLoc, index, paramLoc, paramName,
+                                   colonLoc, typeBoundInfo);
 }
 
-ObjCTypeParamList *Sema::actOnObjCTypeParamList(Scope *S,
-                                                SourceLocation lAngleLoc,
-                                                ArrayRef<Decl *> typeParamsIn,
-                                                SourceLocation rAngleLoc) {
+ObjCTypeParamList *
+SemaObjC::actOnObjCTypeParamList(Scope *S, SourceLocation lAngleLoc,
+                                 ArrayRef<Decl *> typeParamsIn,
+                                 SourceLocation rAngleLoc) {
+  ASTContext &Context = getASTContext();
   // We know that the array only contains Objective-C type parameters.
   ArrayRef<ObjCTypeParamDecl *>
     typeParams(
@@ -798,7 +794,7 @@ ObjCTypeParamList *Sema::actOnObjCTypeParamList(Scope *S,
       knownParams.insert(std::make_pair(typeParam->getIdentifier(), typeParam));
 
       // Push the type parameter into scope.
-      PushOnScopeChains(typeParam, S, /*AddToContext=*/false);
+      SemaRef.PushOnScopeChains(typeParam, S, /*AddToContext=*/false);
     }
   }
 
@@ -806,11 +802,12 @@ ObjCTypeParamList *Sema::actOnObjCTypeParamList(Scope *S,
   return ObjCTypeParamList::create(Context, lAngleLoc, typeParams, rAngleLoc);
 }
 
-void Sema::popObjCTypeParamList(Scope *S, ObjCTypeParamList *typeParamList) {
+void SemaObjC::popObjCTypeParamList(Scope *S,
+                                    ObjCTypeParamList *typeParamList) {
   for (auto *typeParam : *typeParamList) {
     if (!typeParam->isInvalidDecl()) {
       S->RemoveDecl(typeParam);
-      IdResolver.RemoveDecl(typeParam);
+      SemaRef.IdResolver.RemoveDecl(typeParam);
     }
   }
 }
@@ -975,7 +972,7 @@ static bool checkTypeParamListConsistency(Sema &S,
   return false;
 }
 
-ObjCInterfaceDecl *Sema::ActOnStartClassInterface(
+ObjCInterfaceDecl *SemaObjC::ActOnStartClassInterface(
     Scope *S, SourceLocation AtInterfaceLoc, IdentifierInfo *ClassName,
     SourceLocation ClassLoc, ObjCTypeParamList *typeParamList,
     IdentifierInfo *SuperName, SourceLocation SuperLoc,
@@ -985,10 +982,11 @@ ObjCInterfaceDecl *Sema::ActOnStartClassInterface(
     const ParsedAttributesView &AttrList, SkipBodyInfo *SkipBody) {
   assert(ClassName && "Missing class identifier");
 
+  ASTContext &Context = getASTContext();
   // Check for another declaration kind with the same name.
-  NamedDecl *PrevDecl =
-      LookupSingleName(TUScope, ClassName, ClassLoc, LookupOrdinaryName,
-                       forRedeclarationInCurContext());
+  NamedDecl *PrevDecl = SemaRef.LookupSingleName(
+      SemaRef.TUScope, ClassName, ClassLoc, Sema::LookupOrdinaryName,
+      SemaRef.forRedeclarationInCurContext());
 
   if (PrevDecl && !isa<ObjCInterfaceDecl>(PrevDecl)) {
     Diag(ClassLoc, diag::err_redefinition_different_kind) << ClassName;
@@ -1020,7 +1018,7 @@ ObjCInterfaceDecl *Sema::ActOnStartClassInterface(
     if (ObjCTypeParamList *prevTypeParamList = PrevIDecl->getTypeParamList()) {
       if (typeParamList) {
         // Both have type parameter lists; check for consistency.
-        if (checkTypeParamListConsistency(*this, prevTypeParamList,
+        if (checkTypeParamListConsistency(SemaRef, prevTypeParamList,
                                           typeParamList,
                                           TypeParamListContext::Definition)) {
           typeParamList = nullptr;
@@ -1034,17 +1032,12 @@ ObjCInterfaceDecl *Sema::ActOnStartClassInterface(
         // Clone the type parameter list.
         SmallVector<ObjCTypeParamDecl *, 4> clonedTypeParams;
         for (auto *typeParam : *prevTypeParamList) {
-          clonedTypeParams.push_back(
-            ObjCTypeParamDecl::Create(
-              Context,
-              CurContext,
-              typeParam->getVariance(),
-              SourceLocation(),
-              typeParam->getIndex(),
-              SourceLocation(),
-              typeParam->getIdentifier(),
-              SourceLocation(),
-              Context.getTrivialTypeSourceInfo(typeParam->getUnderlyingType())));
+          clonedTypeParams.push_back(ObjCTypeParamDecl::Create(
+              Context, SemaRef.CurContext, typeParam->getVariance(),
+              SourceLocation(), typeParam->getIndex(), SourceLocation(),
+              typeParam->getIdentifier(), SourceLocation(),
+              Context.getTrivialTypeSourceInfo(
+                  typeParam->getUnderlyingType())));
         }
 
         typeParamList = ObjCTypeParamList::create(Context,
@@ -1055,13 +1048,13 @@ ObjCInterfaceDecl *Sema::ActOnStartClassInterface(
     }
   }
 
-  ObjCInterfaceDecl *IDecl
-    = ObjCInterfaceDecl::Create(Context, CurContext, AtInterfaceLoc, ClassName,
-                                typeParamList, PrevIDecl, ClassLoc);
+  ObjCInterfaceDecl *IDecl =
+      ObjCInterfaceDecl::Create(Context, SemaRef.CurContext, AtInterfaceLoc,
+                                ClassName, typeParamList, PrevIDecl, ClassLoc);
   if (PrevIDecl) {
     // Class already seen. Was it a definition?
     if (ObjCInterfaceDecl *Def = PrevIDecl->getDefinition()) {
-      if (SkipBody && !hasVisibleDefinition(Def)) {
+      if (SkipBody && !SemaRef.hasVisibleDefinition(Def)) {
         SkipBody->CheckSameAsPrevious = true;
         SkipBody->New = IDecl;
         SkipBody->Previous = Def;
@@ -1074,15 +1067,15 @@ ObjCInterfaceDecl *Sema::ActOnStartClassInterface(
     }
   }
 
-  ProcessDeclAttributeList(TUScope, IDecl, AttrList);
-  AddPragmaAttributes(TUScope, IDecl);
-  ProcessAPINotes(IDecl);
+  SemaRef.ProcessDeclAttributeList(SemaRef.TUScope, IDecl, AttrList);
+  SemaRef.AddPragmaAttributes(SemaRef.TUScope, IDecl);
+  SemaRef.ProcessAPINotes(IDecl);
 
   // Merge attributes from previous declarations.
   if (PrevIDecl)
-    mergeDeclAttributes(IDecl, PrevIDecl);
+    SemaRef.mergeDeclAttributes(IDecl, PrevIDecl);
 
-  PushOnScopeChains(IDecl, TUScope);
+  SemaRef.PushOnScopeChains(IDecl, SemaRef.TUScope);
 
   // Start the definition of this class. If we're in a redefinition case, there
   // may already be a definition, so we'll end up adding to it.
@@ -1093,7 +1086,7 @@ ObjCInterfaceDecl *Sema::ActOnStartClassInterface(
 
   if (SuperName) {
     // Diagnose availability in the context of the @interface.
-    ContextRAII SavedContext(*this, IDecl);
+    Sema::ContextRAII SavedContext(SemaRef, IDecl);
 
     ActOnSuperClassOfClassInterface(S, AtInterfaceLoc, IDecl,
                                     ClassName, ClassLoc,
@@ -1105,7 +1098,7 @@ ObjCInterfaceDecl *Sema::ActOnStartClassInterface(
 
   // Check then save referenced protocols.
   if (NumProtoRefs) {
-    diagnoseUseOfProtocols(*this, IDecl, (ObjCProtocolDecl*const*)ProtoRefs,
+    diagnoseUseOfProtocols(SemaRef, IDecl, (ObjCProtocolDecl *const *)ProtoRefs,
                            NumProtoRefs, ProtoLocs);
     IDecl->setProtocolList((ObjCProtocolDecl*const*)ProtoRefs, NumProtoRefs,
                            ProtoLocs, Context);
@@ -1120,14 +1113,14 @@ ObjCInterfaceDecl *Sema::ActOnStartClassInterface(
 /// ActOnTypedefedProtocols - this action finds protocol list as part of the
 /// typedef'ed use for a qualified super class and adds them to the list
 /// of the protocols.
-void Sema::ActOnTypedefedProtocols(SmallVectorImpl<Decl *> &ProtocolRefs,
-                                  SmallVectorImpl<SourceLocation> &ProtocolLocs,
-                                   IdentifierInfo *SuperName,
-                                   SourceLocation SuperLoc) {
+void SemaObjC::ActOnTypedefedProtocols(
+    SmallVectorImpl<Decl *> &ProtocolRefs,
+    SmallVectorImpl<SourceLocation> &ProtocolLocs, IdentifierInfo *SuperName,
+    SourceLocation SuperLoc) {
   if (!SuperName)
     return;
-  NamedDecl* IDecl = LookupSingleName(TUScope, SuperName, SuperLoc,
-                                      LookupOrdinaryName);
+  NamedDecl *IDecl = SemaRef.LookupSingleName(
+      SemaRef.TUScope, SuperName, SuperLoc, Sema::LookupOrdinaryName);
   if (!IDecl)
     return;
 
@@ -1147,33 +1140,34 @@ void Sema::ActOnTypedefedProtocols(SmallVectorImpl<Decl *> &ProtocolRefs,
 
 /// ActOnCompatibilityAlias - this action is called after complete parsing of
 /// a \@compatibility_alias declaration. It sets up the alias relationships.
-Decl *Sema::ActOnCompatibilityAlias(SourceLocation AtLoc,
-                                    IdentifierInfo *AliasName,
-                                    SourceLocation AliasLocation,
-                                    IdentifierInfo *ClassName,
-                                    SourceLocation ClassLocation) {
+Decl *SemaObjC::ActOnCompatibilityAlias(SourceLocation AtLoc,
+                                        IdentifierInfo *AliasName,
+                                        SourceLocation AliasLocation,
+                                        IdentifierInfo *ClassName,
+                                        SourceLocation ClassLocation) {
+  ASTContext &Context = getASTContext();
   // Look for previous declaration of alias name
-  NamedDecl *ADecl =
-      LookupSingleName(TUScope, AliasName, AliasLocation, LookupOrdinaryName,
-                       forRedeclarationInCurContext());
+  NamedDecl *ADecl = SemaRef.LookupSingleName(
+      SemaRef.TUScope, AliasName, AliasLocation, Sema::LookupOrdinaryName,
+      SemaRef.forRedeclarationInCurContext());
   if (ADecl) {
     Diag(AliasLocation, diag::err_conflicting_aliasing_type) << AliasName;
     Diag(ADecl->getLocation(), diag::note_previous_declaration);
     return nullptr;
   }
   // Check for class declaration
-  NamedDecl *CDeclU =
-      LookupSingleName(TUScope, ClassName, ClassLocation, LookupOrdinaryName,
-                       forRedeclarationInCurContext());
+  NamedDecl *CDeclU = SemaRef.LookupSingleName(
+      SemaRef.TUScope, ClassName, ClassLocation, Sema::LookupOrdinaryName,
+      SemaRef.forRedeclarationInCurContext());
   if (const TypedefNameDecl *TDecl =
         dyn_cast_or_null<TypedefNameDecl>(CDeclU)) {
     QualType T = TDecl->getUnderlyingType();
     if (T->isObjCObjectType()) {
       if (NamedDecl *IDecl = T->castAs<ObjCObjectType>()->getInterface()) {
         ClassName = IDecl->getIdentifier();
-        CDeclU = LookupSingleName(TUScope, ClassName, ClassLocation,
-                                  LookupOrdinaryName,
-                                  forRedeclarationInCurContext());
+        CDeclU = SemaRef.LookupSingleName(
+            SemaRef.TUScope, ClassName, ClassLocation, Sema::LookupOrdinaryName,
+            SemaRef.forRedeclarationInCurContext());
       }
     }
   }
@@ -1186,25 +1180,23 @@ Decl *Sema::ActOnCompatibilityAlias(SourceLocation AtLoc,
   }
 
   // Everything checked out, instantiate a new alias declaration AST.
-  ObjCCompatibleAliasDecl *AliasDecl =
-    ObjCCompatibleAliasDecl::Create(Context, CurContext, AtLoc, AliasName, CDecl);
+  ObjCCompatibleAliasDecl *AliasDecl = ObjCCompatibleAliasDecl::Create(
+      Context, SemaRef.CurContext, AtLoc, AliasName, CDecl);
 
   if (!CheckObjCDeclScope(AliasDecl))
-    PushOnScopeChains(AliasDecl, TUScope);
+    SemaRef.PushOnScopeChains(AliasDecl, SemaRef.TUScope);
 
   return AliasDecl;
 }
 
-bool Sema::CheckForwardProtocolDeclarationForCircularDependency(
-  IdentifierInfo *PName,
-  SourceLocation &Ploc, SourceLocation PrevLoc,
-  const ObjCList<ObjCProtocolDecl> &PList) {
+bool SemaObjC::CheckForwardProtocolDeclarationForCircularDependency(
+    IdentifierInfo *PName, SourceLocation &Ploc, SourceLocation PrevLoc,
+    const ObjCList<ObjCProtocolDecl> &PList) {
 
   bool res = false;
   for (ObjCList<ObjCProtocolDecl>::iterator I = PList.begin(),
        E = PList.end(); I != E; ++I) {
-    if (ObjCProtocolDecl *PDecl = LookupProtocol((*I)->getIdentifier(),
-                                                 Ploc)) {
+    if (ObjCProtocolDecl *PDecl = LookupProtocol((*I)->getIdentifier(), Ploc)) {
       if (PDecl->getIdentifier() == PName) {
         Diag(Ploc, diag::err_protocol_has_circular_dependency);
         Diag(PrevLoc, diag::note_previous_definition);
@@ -1222,27 +1214,28 @@ bool Sema::CheckForwardProtocolDeclarationForCircularDependency(
   return res;
 }
 
-ObjCProtocolDecl *Sema::ActOnStartProtocolInterface(
+ObjCProtocolDecl *SemaObjC::ActOnStartProtocolInterface(
     SourceLocation AtProtoInterfaceLoc, IdentifierInfo *ProtocolName,
     SourceLocation ProtocolLoc, Decl *const *ProtoRefs, unsigned NumProtoRefs,
     const SourceLocation *ProtoLocs, SourceLocation EndProtoLoc,
     const ParsedAttributesView &AttrList, SkipBodyInfo *SkipBody) {
+  ASTContext &Context = getASTContext();
   bool err = false;
   // FIXME: Deal with AttrList.
   assert(ProtocolName && "Missing protocol identifier");
-  ObjCProtocolDecl *PrevDecl = LookupProtocol(ProtocolName, ProtocolLoc,
-                                              forRedeclarationInCurContext());
+  ObjCProtocolDecl *PrevDecl = LookupProtocol(
+      ProtocolName, ProtocolLoc, SemaRef.forRedeclarationInCurContext());
   ObjCProtocolDecl *PDecl = nullptr;
   if (ObjCProtocolDecl *Def = PrevDecl? PrevDecl->getDefinition() : nullptr) {
     // Create a new protocol that is completely distinct from previous
     // declarations, and do not make this protocol available for name lookup.
     // That way, we'll end up completely ignoring the duplicate.
     // FIXME: Can we turn this into an error?
-    PDecl = ObjCProtocolDecl::Create(Context, CurContext, ProtocolName,
+    PDecl = ObjCProtocolDecl::Create(Context, SemaRef.CurContext, ProtocolName,
                                      ProtocolLoc, AtProtoInterfaceLoc,
                                      /*PrevDecl=*/Def);
 
-    if (SkipBody && !hasVisibleDefinition(Def)) {
+    if (SkipBody && !SemaRef.hasVisibleDefinition(Def)) {
       SkipBody->CheckSameAsPrevious = true;
       SkipBody->New = PDecl;
       SkipBody->Previous = Def;
@@ -1255,7 +1248,7 @@ ObjCProtocolDecl *Sema::ActOnStartProtocolInterface(
     // If we are using modules, add the decl to the context in order to
     // serialize something meaningful.
     if (getLangOpts().Modules)
-      PushOnScopeChains(PDecl, TUScope);
+      SemaRef.PushOnScopeChains(PDecl, SemaRef.TUScope);
     PDecl->startDuplicateDefinitionForComparison();
   } else {
     if (PrevDecl) {
@@ -1268,25 +1261,25 @@ ObjCProtocolDecl *Sema::ActOnStartProtocolInterface(
     }
 
     // Create the new declaration.
-    PDecl = ObjCProtocolDecl::Create(Context, CurContext, ProtocolName,
+    PDecl = ObjCProtocolDecl::Create(Context, SemaRef.CurContext, ProtocolName,
                                      ProtocolLoc, AtProtoInterfaceLoc,
                                      /*PrevDecl=*/PrevDecl);
 
-    PushOnScopeChains(PDecl, TUScope);
+    SemaRef.PushOnScopeChains(PDecl, SemaRef.TUScope);
     PDecl->startDefinition();
   }
 
-  ProcessDeclAttributeList(TUScope, PDecl, AttrList);
-  AddPragmaAttributes(TUScope, PDecl);
-  ProcessAPINotes(PDecl);
+  SemaRef.ProcessDeclAttributeList(SemaRef.TUScope, PDecl, AttrList);
+  SemaRef.AddPragmaAttributes(SemaRef.TUScope, PDecl);
+  SemaRef.ProcessAPINotes(PDecl);
 
   // Merge attributes from previous declarations.
   if (PrevDecl)
-    mergeDeclAttributes(PDecl, PrevDecl);
+    SemaRef.mergeDeclAttributes(PDecl, PrevDecl);
 
   if (!err && NumProtoRefs ) {
     /// Check then save referenced protocols.
-    diagnoseUseOfProtocols(*this, PDecl, (ObjCProtocolDecl*const*)ProtoRefs,
+    diagnoseUseOfProtocols(SemaRef, PDecl, (ObjCProtocolDecl *const *)ProtoRefs,
                            NumProtoRefs, ProtoLocs);
     PDecl->setProtocolList((ObjCProtocolDecl*const*)ProtoRefs, NumProtoRefs,
                            ProtoLocs, Context);
@@ -1316,20 +1309,22 @@ static bool NestedProtocolHasNoDefinition(ObjCProtocolDecl *PDecl,
 /// FindProtocolDeclaration - This routine looks up protocols and
 /// issues an error if they are not declared. It returns list of
 /// protocol declarations in its 'Protocols' argument.
-void
-Sema::FindProtocolDeclaration(bool WarnOnDeclarations, bool ForObjCContainer,
-                              ArrayRef<IdentifierLocPair> ProtocolId,
-                              SmallVectorImpl<Decl *> &Protocols) {
+void SemaObjC::FindProtocolDeclaration(bool WarnOnDeclarations,
+                                       bool ForObjCContainer,
+                                       ArrayRef<IdentifierLocPair> ProtocolId,
+                                       SmallVectorImpl<Decl *> &Protocols) {
   for (const IdentifierLocPair &Pair : ProtocolId) {
     ObjCProtocolDecl *PDecl = LookupProtocol(Pair.first, Pair.second);
     if (!PDecl) {
       DeclFilterCCC<ObjCProtocolDecl> CCC{};
-      TypoCorrection Corrected = CorrectTypo(
-          DeclarationNameInfo(Pair.first, Pair.second), LookupObjCProtocolName,
-          TUScope, nullptr, CCC, CTK_ErrorRecovery);
+      TypoCorrection Corrected =
+          SemaRef.CorrectTypo(DeclarationNameInfo(Pair.first, Pair.second),
+                              Sema::LookupObjCProtocolName, SemaRef.TUScope,
+                              nullptr, CCC, Sema::CTK_ErrorRecovery);
       if ((PDecl = Corrected.getCorrectionDeclAs<ObjCProtocolDecl>()))
-        diagnoseTypo(Corrected, PDiag(diag::err_undeclared_protocol_suggest)
-                                    << Pair.first);
+        SemaRef.diagnoseTypo(
+            Corrected, SemaRef.PDiag(diag::err_undeclared_protocol_suggest)
+                           << Pair.first);
     }
 
     if (!PDecl) {
@@ -1343,7 +1338,7 @@ Sema::FindProtocolDeclaration(bool WarnOnDeclarations, bool ForObjCContainer,
     // For an objc container, delay protocol reference checking until after we
     // can set the objc decl as the availability context, otherwise check now.
     if (!ForObjCContainer) {
-      (void)DiagnoseUseOfDecl(PDecl, Pair.second);
+      (void)SemaRef.DiagnoseUseOfDecl(PDecl, Pair.second);
     }
 
     // If this is a forward declaration and we are supposed to warn in this
@@ -1419,30 +1414,25 @@ class ObjCTypeArgOrProtocolValidatorCCC final
 };
 } // end anonymous namespace
 
-void Sema::DiagnoseTypeArgsAndProtocols(IdentifierInfo *ProtocolId,
-                                        SourceLocation ProtocolLoc,
-                                        IdentifierInfo *TypeArgId,
-                                        SourceLocation TypeArgLoc,
-                                        bool SelectProtocolFirst) {
+void SemaObjC::DiagnoseTypeArgsAndProtocols(IdentifierInfo *ProtocolId,
+                                            SourceLocation ProtocolLoc,
+                                            IdentifierInfo *TypeArgId,
+                                            SourceLocation TypeArgLoc,
+                                            bool SelectProtocolFirst) {
   Diag(TypeArgLoc, diag::err_objc_type_args_and_protocols)
       << SelectProtocolFirst << TypeArgId << ProtocolId
       << SourceRange(ProtocolLoc);
 }
 
-void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
-       Scope *S,
-       ParsedType baseType,
-       SourceLocation lAngleLoc,
-       ArrayRef<IdentifierInfo *> identifiers,
-       ArrayRef<SourceLocation> identifierLocs,
-       SourceLocation rAngleLoc,
-       SourceLocation &typeArgsLAngleLoc,
-       SmallVectorImpl<ParsedType> &typeArgs,
-       SourceLocation &typeArgsRAngleLoc,
-       SourceLocation &protocolLAngleLoc,
-       SmallVectorImpl<Decl *> &protocols,
-       SourceLocation &protocolRAngleLoc,
-       bool warnOnIncompleteProtocols) {
+void SemaObjC::actOnObjCTypeArgsOrProtocolQualifiers(
+    Scope *S, ParsedType baseType, SourceLocation lAngleLoc,
+    ArrayRef<IdentifierInfo *> identifiers,
+    ArrayRef<SourceLocation> identifierLocs, SourceLocation rAngleLoc,
+    SourceLocation &typeArgsLAngleLoc, SmallVectorImpl<ParsedType> &typeArgs,
+    SourceLocation &typeArgsRAngleLoc, SourceLocation &protocolLAngleLoc,
+    SmallVectorImpl<Decl *> &protocols, SourceLocation &protocolRAngleLoc,
+    bool warnOnIncompleteProtocols) {
+  ASTContext &Context = getASTContext();
   // Local function that updates the declaration specifiers with
   // protocol information.
   unsigned numProtocolsResolved = 0;
@@ -1453,7 +1443,7 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
     // which case we want to warn about typos such as
     // "NSArray<NSObject>" (that should be NSArray<NSObject *>).
     ObjCInterfaceDecl *baseClass = nullptr;
-    QualType base = GetTypeFromParser(baseType, nullptr);
+    QualType base = SemaRef.GetTypeFromParser(baseType, nullptr);
     bool allAreTypeNames = false;
     SourceLocation firstClassNameLoc;
     if (!base.isNull()) {
@@ -1476,7 +1466,7 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
       // For an objc container, delay protocol reference checking until after we
       // can set the objc decl as the availability context, otherwise check now.
       if (!warnOnIncompleteProtocols) {
-        (void)DiagnoseUseOfDecl(proto, identifierLocs[i]);
+        (void)SemaRef.DiagnoseUseOfDecl(proto, identifierLocs[i]);
       }
 
       // If this is a forward protocol declaration, get its definition.
@@ -1499,8 +1489,9 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
       // about such things), check whether this name refers to a type
       // as well.
       if (allAreTypeNames) {
-        if (auto *decl = LookupSingleName(S, identifiers[i], identifierLocs[i],
-                                          LookupOrdinaryName)) {
+        if (auto *decl =
+                SemaRef.LookupSingleName(S, identifiers[i], identifierLocs[i],
+                                         Sema::LookupOrdinaryName)) {
           if (isa<ObjCInterfaceDecl>(decl)) {
             if (firstClassNameLoc.isInvalid())
               firstClassNameLoc = identifierLocs[i];
@@ -1531,9 +1522,9 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
 
       if (allProtocolsDeclared) {
         Diag(firstClassNameLoc, diag::warn_objc_redundant_qualified_class_type)
-          << baseClass->getDeclName() << SourceRange(lAngleLoc, rAngleLoc)
-          << FixItHint::CreateInsertion(getLocForEndOfToken(firstClassNameLoc),
-                                        " *");
+            << baseClass->getDeclName() << SourceRange(lAngleLoc, rAngleLoc)
+            << FixItHint::CreateInsertion(
+                   SemaRef.getLocForEndOfToken(firstClassNameLoc), " *");
       }
     }
 
@@ -1562,8 +1553,8 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
   SmallVector<TypeOrClassDecl, 4> typeDecls;
   unsigned numTypeDeclsResolved = 0;
   for (unsigned i = 0, n = identifiers.size(); i != n; ++i) {
-    NamedDecl *decl = LookupSingleName(S, identifiers[i], identifierLocs[i],
-                                       LookupOrdinaryName);
+    NamedDecl *decl = SemaRef.LookupSingleName(
+        S, identifiers[i], identifierLocs[i], Sema::LookupOrdinaryName);
     if (!decl) {
       typeDecls.push_back(TypeOrClassDecl());
       continue;
@@ -1600,7 +1591,7 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
     else
       type = Context.getObjCInterfaceType(typeDecl.get<ObjCInterfaceDecl *>());
     TypeSourceInfo *parsedTSInfo = Context.getTrivialTypeSourceInfo(type, loc);
-    ParsedType parsedType = CreateParsedType(type, parsedTSInfo);
+    ParsedType parsedType = SemaRef.CreateParsedType(type, parsedTSInfo);
     DS.SetTypeSpecType(DeclSpec::TST_typename, loc, prevSpec, diagID,
                        parsedType, Context.getPrintingPolicy());
     // Use the identifier location for the type source range.
@@ -1613,7 +1604,7 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
     // If we have a typedef of an Objective-C class type that is missing a '*',
     // add the '*'.
     if (type->getAs<ObjCInterfaceType>()) {
-      SourceLocation starLoc = getLocForEndOfToken(loc);
+      SourceLocation starLoc = SemaRef.getLocForEndOfToken(loc);
       D.AddTypeInfo(DeclaratorChunk::getPointer(/*TypeQuals=*/0, starLoc,
                                                 SourceLocation(),
                                                 SourceLocation(),
@@ -1629,7 +1620,7 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
     }
 
     // Convert this to a type.
-    return ActOnTypeName(D);
+    return SemaRef.ActOnTypeName(D);
   };
 
   // Local function that updates the declaration specifiers with
@@ -1663,14 +1654,14 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
   // Error recovery: some names weren't found, or we have a mix of
   // type and protocol names. Go resolve all of the unresolved names
   // and complain if we can't find a consistent answer.
-  LookupNameKind lookupKind = LookupAnyName;
+  Sema::LookupNameKind lookupKind = Sema::LookupAnyName;
   for (unsigned i = 0, n = identifiers.size(); i != n; ++i) {
     // If we already have a protocol or type. Check whether it is the
     // right thing.
     if (protocols[i] || typeDecls[i]) {
       // If we haven't figured out whether we want types or protocols
       // yet, try to figure it out from this name.
-      if (lookupKind == LookupAnyName) {
+      if (lookupKind == Sema::LookupAnyName) {
         // If this name refers to both a protocol and a type (e.g., \c
         // NSObject), don't conclude anything yet.
         if (protocols[i] && typeDecls[i])
@@ -1678,19 +1669,19 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
 
         // Otherwise, let this name decide whether we'll be correcting
         // toward types or protocols.
-        lookupKind = protocols[i] ? LookupObjCProtocolName
-                                  : LookupOrdinaryName;
+        lookupKind = protocols[i] ? Sema::LookupObjCProtocolName
+                                  : Sema::LookupOrdinaryName;
         continue;
       }
 
       // If we want protocols and we have a protocol, there's nothing
       // more to do.
-      if (lookupKind == LookupObjCProtocolName && protocols[i])
+      if (lookupKind == Sema::LookupObjCProtocolName && protocols[i])
         continue;
 
       // If we want types and we have a type declaration, there's
       // nothing more to do.
-      if (lookupKind == LookupOrdinaryName && typeDecls[i])
+      if (lookupKind == Sema::LookupOrdinaryName && typeDecls[i])
         continue;
 
       // We have a conflict: some names refer to protocols and others
@@ -1706,16 +1697,16 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
 
     // Perform typo correction on the name.
     ObjCTypeArgOrProtocolValidatorCCC CCC(Context, lookupKind);
-    TypoCorrection corrected =
-        CorrectTypo(DeclarationNameInfo(identifiers[i], identifierLocs[i]),
-                    lookupKind, S, nullptr, CCC, CTK_ErrorRecovery);
+    TypoCorrection corrected = SemaRef.CorrectTypo(
+        DeclarationNameInfo(identifiers[i], identifierLocs[i]), lookupKind, S,
+        nullptr, CCC, Sema::CTK_ErrorRecovery);
     if (corrected) {
       // Did we find a protocol?
       if (auto proto = corrected.getCorrectionDeclAs<ObjCProtocolDecl>()) {
-        diagnoseTypo(corrected,
-                     PDiag(diag::err_undeclared_protocol_suggest)
-                       << identifiers[i]);
-        lookupKind = LookupObjCProtocolName;
+        SemaRef.diagnoseTypo(
+            corrected, SemaRef.PDiag(diag::err_undeclared_protocol_suggest)
+                           << identifiers[i]);
+        lookupKind = Sema::LookupObjCProtocolName;
         protocols[i] = proto;
         ++numProtocolsResolved;
         continue;
@@ -1723,10 +1714,10 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
 
       // Did we find a type?
       if (auto typeDecl = corrected.getCorrectionDeclAs<TypeDecl>()) {
-        diagnoseTypo(corrected,
-                     PDiag(diag::err_unknown_typename_suggest)
-                       << identifiers[i]);
-        lookupKind = LookupOrdinaryName;
+        SemaRef.diagnoseTypo(corrected,
+                             SemaRef.PDiag(diag::err_unknown_typename_suggest)
+                                 << identifiers[i]);
+        lookupKind = Sema::LookupOrdinaryName;
         typeDecls[i] = typeDecl;
         ++numTypeDeclsResolved;
         continue;
@@ -1734,10 +1725,11 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
 
       // Did we find an Objective-C class?
       if (auto objcClass = corrected.getCorrectionDeclAs<ObjCInterfaceDecl>()) {
-        diagnoseTypo(corrected,
-                     PDiag(diag::err_unknown_type_or_class_name_suggest)
-                       << identifiers[i] << true);
-        lookupKind = LookupOrdinaryName;
+        SemaRef.diagnoseTypo(
+            corrected,
+            SemaRef.PDiag(diag::err_unknown_type_or_class_name_suggest)
+                << identifiers[i] << true);
+        lookupKind = Sema::LookupOrdinaryName;
         typeDecls[i] = objcClass;
         ++numTypeDeclsResolved;
         continue;
@@ -1746,10 +1738,11 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
 
     // We couldn't find anything.
     Diag(identifierLocs[i],
-         (lookupKind == LookupAnyName ? diag::err_objc_type_arg_missing
-          : lookupKind == LookupObjCProtocolName ? diag::err_undeclared_protocol
-          : diag::err_unknown_typename))
-      << identifiers[i];
+         (lookupKind == Sema::LookupAnyName ? diag::err_objc_type_arg_missing
+          : lookupKind == Sema::LookupObjCProtocolName
+              ? diag::err_undeclared_protocol
+              : diag::err_unknown_typename))
+        << identifiers[i];
     protocols.clear();
     typeArgs.clear();
     return;
@@ -1768,8 +1761,8 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
 /// DiagnoseClassExtensionDupMethods - Check for duplicate declaration of
 /// a class method in its extension.
 ///
-void Sema::DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT,
-                                            ObjCInterfaceDecl *ID) {
+void SemaObjC::DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT,
+                                                ObjCInterfaceDecl *ID) {
   if (!ID)
     return;  // Possibly due to previous error
 
@@ -1792,59 +1785,59 @@ void Sema::DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT,
 }
 
 /// ActOnForwardProtocolDeclaration - Handle \@protocol foo;
-Sema::DeclGroupPtrTy
-Sema::ActOnForwardProtocolDeclaration(SourceLocation AtProtocolLoc,
-                                      ArrayRef<IdentifierLocPair> IdentList,
-                                      const ParsedAttributesView &attrList) {
+SemaObjC::DeclGroupPtrTy SemaObjC::ActOnForwardProtocolDeclaration(
+    SourceLocation AtProtocolLoc, ArrayRef<IdentifierLocPair> IdentList,
+    const ParsedAttributesView &attrList) {
+  ASTContext &Context = getASTContext();
   SmallVector<Decl *, 8> DeclsInGroup;
   for (const IdentifierLocPair &IdentPair : IdentList) {
     IdentifierInfo *Ident = IdentPair.first;
-    ObjCProtocolDecl *PrevDecl = LookupProtocol(Ident, IdentPair.second,
-                                                forRedeclarationInCurContext());
-    ObjCProtocolDecl *PDecl
-      = ObjCProtocolDecl::Create(Context, CurContext, Ident,
-                                 IdentPair.second, AtProtocolLoc,
-                                 PrevDecl);
-
-    PushOnScopeChains(PDecl, TUScope);
+    ObjCProtocolDecl *PrevDecl = LookupProtocol(
+        Ident, IdentPair.second, SemaRef.forRedeclarationInCurContext());
+    ObjCProtocolDecl *PDecl =
+        ObjCProtocolDecl::Create(Context, SemaRef.CurContext, Ident,
+                                 IdentPair.second, AtProtocolLoc, PrevDecl);
+
+    SemaRef.PushOnScopeChains(PDecl, SemaRef.TUScope);
     CheckObjCDeclScope(PDecl);
 
-    ProcessDeclAttributeList(TUScope, PDecl, attrList);
-    AddPragmaAttributes(TUScope, PDecl);
+    SemaRef.ProcessDeclAttributeList(SemaRef.TUScope, PDecl, attrList);
+    SemaRef.AddPragmaAttributes(SemaRef.TUScope, PDecl);
 
     if (PrevDecl)
-      mergeDeclAttributes(PDecl, PrevDecl);
+      SemaRef.mergeDeclAttributes(PDecl, PrevDecl);
 
     DeclsInGroup.push_back(PDecl);
   }
 
-  return BuildDeclaratorGroup(DeclsInGroup);
+  return SemaRef.BuildDeclaratorGroup(DeclsInGroup);
 }
 
-ObjCCategoryDecl *Sema::ActOnStartCategoryInterface(
+ObjCCategoryDecl *SemaObjC::ActOnStartCategoryInterface(
     SourceLocation AtInterfaceLoc, const IdentifierInfo *ClassName,
     SourceLocation ClassLoc, ObjCTypeParamList *typeParamList,
     const IdentifierInfo *CategoryName, SourceLocation CategoryLoc,
     Decl *const *ProtoRefs, unsigned NumProtoRefs,
     const SourceLocation *ProtoLocs, SourceLocation EndProtoLoc,
     const ParsedAttributesView &AttrList) {
+  ASTContext &Context = getASTContext();
   ObjCCategoryDecl *CDecl;
   ObjCInterfaceDecl *IDecl = getObjCInterfaceDecl(ClassName, ClassLoc, true);
 
   /// Check that class of this category is already completely declared.
 
-  if (!IDecl
-      || RequireCompleteType(ClassLoc, Context.getObjCInterfaceType(IDecl),
-                             diag::err_category_forward_interface,
-                             CategoryName == nullptr)) {
+  if (!IDecl ||
+      SemaRef.RequireCompleteType(ClassLoc, Context.getObjCInterfaceType(IDecl),
+                                  diag::err_category_forward_interface,
+                                  CategoryName == nullptr)) {
     // Create an invalid ObjCCategoryDecl to serve as context for
     // the enclosing method declarations.  We mark the decl invalid
     // to make it clear that this isn't a valid AST.
-    CDecl = ObjCCategoryDecl::Create(Context, CurContext, AtInterfaceLoc,
-                                     ClassLoc, CategoryLoc, CategoryName,
-                                     IDecl, typeParamList);
+    CDecl = ObjCCategoryDecl::Create(Context, SemaRef.CurContext,
+                                     AtInterfaceLoc, ClassLoc, CategoryLoc,
+                                     CategoryName, IDecl, typeParamList);
     CDecl->setInvalidDecl();
-    CurContext->addDecl(CDecl);
+    SemaRef.CurContext->addDecl(CDecl);
 
     if (!IDecl)
       Diag(ClassLoc, diag::err_undef_interface) << ClassName;
@@ -1872,10 +1865,10 @@ ObjCCategoryDecl *Sema::ActOnStartCategoryInterface(
   // If we have a type parameter list, check it.
   if (typeParamList) {
     if (auto prevTypeParamList = IDecl->getTypeParamList()) {
-      if (checkTypeParamListConsistency(*this, prevTypeParamList, typeParamList,
-                                        CategoryName
-                                          ? TypeParamListContext::Category
-                                          : TypeParamListContext::Extension))
+      if (checkTypeParamListConsistency(
+              SemaRef, prevTypeParamList, typeParamList,
+              CategoryName ? TypeParamListContext::Category
+                           : TypeParamListContext::Extension))
         typeParamList = nullptr;
     } else {
       Diag(typeParamList->getLAngleLoc(),
@@ -1888,20 +1881,20 @@ ObjCCategoryDecl *Sema::ActOnStartCategoryInterface(
     }
   }
 
-  CDecl = ObjCCategoryDecl::Create(Context, CurContext, AtInterfaceLoc,
+  CDecl = ObjCCategoryDecl::Create(Context, SemaRef.CurContext, AtInterfaceLoc,
                                    ClassLoc, CategoryLoc, CategoryName, IDecl,
                                    typeParamList);
   // FIXME: PushOnScopeChains?
-  CurContext->addDecl(CDecl);
+  SemaRef.CurContext->addDecl(CDecl);
 
   // Process the attributes before looking at protocols to ensure that the
   // availability attribute is attached to the category to provide availability
   // checking for protocol uses.
-  ProcessDeclAttributeList(TUScope, CDecl, AttrList);
-  AddPragmaAttributes(TUScope, CDecl);
+  SemaRef.ProcessDeclAttributeList(SemaRef.TUScope, CDecl, AttrList);
+  SemaRef.AddPragmaAttributes(SemaRef.TUScope, CDecl);
 
   if (NumProtoRefs) {
-    diagnoseUseOfProtocols(*this, CDecl, (ObjCProtocolDecl*const*)ProtoRefs,
+    diagnoseUseOfProtocols(SemaRef, CDecl, (ObjCProtocolDecl *const *)ProtoRefs,
                            NumProtoRefs, ProtoLocs);
     CDecl->setProtocolList((ObjCProtocolDecl*const*)ProtoRefs, NumProtoRefs,
                            ProtoLocs, Context);
@@ -1919,10 +1912,11 @@ ObjCCategoryDecl *Sema::ActOnStartCategoryInterface(
 /// ActOnStartCategoryImplementation - Perform semantic checks on the
 /// category implementation declaration and build an ObjCCategoryImplDecl
 /// object.
-ObjCCategoryImplDecl *Sema::ActOnStartCategoryImplementation(
+ObjCCategoryImplDecl *SemaObjC::ActOnStartCategoryImplementation(
     SourceLocation AtCatImplLoc, const IdentifierInfo *ClassName,
     SourceLocation ClassLoc, const IdentifierInfo *CatName,
     SourceLocation CatLoc, const ParsedAttributesView &Attrs) {
+  ASTContext &Context = getASTContext();
   ObjCInterfaceDecl *IDecl = getObjCInterfaceDecl(ClassName, ClassLoc, true);
   ObjCCategoryDecl *CatIDecl = nullptr;
   if (IDecl && IDecl->hasDefinition()) {
@@ -1930,31 +1924,32 @@ ObjCCategoryImplDecl *Sema::ActOnStartCategoryImplementation(
     if (!CatIDecl) {
       // Category @implementation with no corresponding @interface.
       // Create and install one.
-      CatIDecl = ObjCCategoryDecl::Create(Context, CurContext, AtCatImplLoc,
-                                          ClassLoc, CatLoc,
-                                          CatName, IDecl,
-                                          /*typeParamList=*/nullptr);
+      CatIDecl =
+          ObjCCategoryDecl::Create(Context, SemaRef.CurContext, AtCatImplLoc,
+                                   ClassLoc, CatLoc, CatName, IDecl,
+                                   /*typeParamList=*/nullptr);
       CatIDecl->setImplicit();
     }
   }
 
   ObjCCategoryImplDecl *CDecl =
-    ObjCCategoryImplDecl::Create(Context, CurContext, CatName, IDecl,
-                                 ClassLoc, AtCatImplLoc, CatLoc);
+      ObjCCategoryImplDecl::Create(Context, SemaRef.CurContext, CatName, IDecl,
+                                   ClassLoc, AtCatImplLoc, CatLoc);
   /// Check that class of this category is already completely declared.
   if (!IDecl) {
     Diag(ClassLoc, diag::err_undef_interface) << ClassName;
     CDecl->setInvalidDecl();
-  } else if (RequireCompleteType(ClassLoc, Context.getObjCInterfaceType(IDecl),
-                                 diag::err_undef_interface)) {
+  } else if (SemaRef.RequireCompleteType(ClassLoc,
+                                         Context.getObjCInterfaceType(IDecl),
+                                         diag::err_undef_interface)) {
     CDecl->setInvalidDecl();
   }
 
-  ProcessDeclAttributeList(TUScope, CDecl, Attrs);
-  AddPragmaAttributes(TUScope, CDecl);
+  SemaRef.ProcessDeclAttributeList(SemaRef.TUScope, CDecl, Attrs);
+  SemaRef.AddPragmaAttributes(SemaRef.TUScope, CDecl);
 
   // FIXME: PushOnScopeChains?
-  CurContext->addDecl(CDecl);
+  SemaRef.CurContext->addDecl(CDecl);
 
   // If the interface has the objc_runtime_visible attribute, we
   // cannot implement a category for it.
@@ -1975,7 +1970,7 @@ ObjCCategoryImplDecl *Sema::ActOnStartCategoryImplementation(
       CatIDecl->setImplementation(CDecl);
       // Warn on implementating category of deprecated class under
       // -Wdeprecated-implementations flag.
-      DiagnoseObjCImplementedDeprecations(*this, CatIDecl,
+      DiagnoseObjCImplementedDeprecations(SemaRef, CatIDecl,
                                           CDecl->getLocation());
     }
   }
@@ -1985,37 +1980,39 @@ ObjCCategoryImplDecl *Sema::ActOnStartCategoryImplementation(
   return CDecl;
 }
 
-ObjCImplementationDecl *Sema::ActOnStartClassImplementation(
+ObjCImplementationDecl *SemaObjC::ActOnStartClassImplementation(
     SourceLocation AtClassImplLoc, const IdentifierInfo *ClassName,
     SourceLocation ClassLoc, const IdentifierInfo *SuperClassname,
     SourceLocation SuperClassLoc, const ParsedAttributesView &Attrs) {
+  ASTContext &Context = getASTContext();
   ObjCInterfaceDecl *IDecl = nullptr;
   // Check for another declaration kind with the same name.
-  NamedDecl *PrevDecl
-    = LookupSingleName(TUScope, ClassName, ClassLoc, LookupOrdinaryName,
-                       forRedeclarationInCurContext());
+  NamedDecl *PrevDecl = SemaRef.LookupSingleName(
+      SemaRef.TUScope, ClassName, ClassLoc, Sema::LookupOrdinaryName,
+      SemaRef.forRedeclarationInCurContext());
   if (PrevDecl && !isa<ObjCInterfaceDecl>(PrevDecl)) {
     Diag(ClassLoc, diag::err_redefinition_different_kind) << ClassName;
     Diag(PrevDecl->getLocation(), diag::note_previous_definition);
   } else if ((IDecl = dyn_cast_or_null<ObjCInterfaceDecl>(PrevDecl))) {
     // FIXME: This will produce an error if the definition of the interface has
     // been imported from a module but is not visible.
-    RequireCompleteType(ClassLoc, Context.getObjCInterfaceType(IDecl),
-                        diag::warn_undef_interface);
+    SemaRef.RequireCompleteType(ClassLoc, Context.getObjCInterfaceType(IDecl),
+                                diag::warn_undef_interface);
   } else {
     // We did not find anything with the name ClassName; try to correct for
     // typos in the class name.
     ObjCInterfaceValidatorCCC CCC{};
-    TypoCorrection Corrected =
-        CorrectTypo(DeclarationNameInfo(ClassName, ClassLoc),
-                    LookupOrdinaryName, TUScope, nullptr, CCC, CTK_NonError);
+    TypoCorrection Corrected = SemaRef.CorrectTypo(
+        DeclarationNameInfo(ClassName, ClassLoc), Sema::LookupOrdinaryName,
+        SemaRef.TUScope, nullptr, CCC, Sema::CTK_NonError);
     if (Corrected.getCorrectionDeclAs<ObjCInterfaceDecl>()) {
       // Suggest the (potentially) correct interface name. Don't provide a
       // code-modification hint or use the typo name for recovery, because
       // this is just a warning. The program may actually be correct.
-      diagnoseTypo(Corrected,
-                   PDiag(diag::warn_undef_interface_suggest) << ClassName,
-                   /*ErrorRecovery*/false);
+      SemaRef.diagnoseTypo(Corrected,
+                           SemaRef.PDiag(diag::warn_undef_interface_suggest)
+                               << ClassName,
+                           /*ErrorRecovery*/ false);
     } else {
       Diag(ClassLoc, diag::warn_undef_interface) << ClassName;
     }
@@ -2025,8 +2022,9 @@ ObjCImplementationDecl *Sema::ActOnStartClassImplementation(
   ObjCInterfaceDecl *SDecl = nullptr;
   if (SuperClassname) {
     // Check if a different kind of symbol declared in this scope.
-    PrevDecl = LookupSingleName(TUScope, SuperClassname, SuperClassLoc,
-                                LookupOrdinaryName);
+    PrevDecl =
+        SemaRef.LookupSingleName(SemaRef.TUScope, SuperClassname, SuperClassLoc,
+                                 Sema::LookupOrdinaryName);
     if (PrevDecl && !isa<ObjCInterfaceDecl>(PrevDecl)) {
       Diag(SuperClassLoc, diag::err_redefinition_different_kind)
         << SuperClassname;
@@ -2054,11 +2052,11 @@ ObjCImplementationDecl *Sema::ActOnStartClassImplementation(
 
     // FIXME: Do we support attributes on the @implementation? If so we should
     // copy them over.
-    IDecl = ObjCInterfaceDecl::Create(Context, CurContext, AtClassImplLoc,
-                                      ClassName, /*typeParamList=*/nullptr,
-                                      /*PrevDecl=*/nullptr, ClassLoc,
-                                      true);
-    AddPragmaAttributes(TUScope, IDecl);
+    IDecl =
+        ObjCInterfaceDecl::Create(Context, SemaRef.CurContext, AtClassImplLoc,
+                                  ClassName, /*typeParamList=*/nullptr,
+                                  /*PrevDecl=*/nullptr, ClassLoc, true);
+    SemaRef.AddPragmaAttributes(SemaRef.TUScope, IDecl);
     IDecl->startDefinition();
     if (SDecl) {
       IDecl->setSuperClass(Context.getTrivialTypeSourceInfo(
@@ -2069,7 +2067,7 @@ ObjCImplementationDecl *Sema::ActOnStartClassImplementation(
       IDecl->setEndOfDefinitionLoc(ClassLoc);
     }
 
-    PushOnScopeChains(IDecl, TUScope);
+    SemaRef.PushOnScopeChains(IDecl, SemaRef.TUScope);
   } else {
     // Mark the interface as being completed, even if it was just as
     //   @class ....;
@@ -2078,12 +2076,12 @@ ObjCImplementationDecl *Sema::ActOnStartClassImplementation(
       IDecl->startDefinition();
   }
 
-  ObjCImplementationDecl* IMPDecl =
-    ObjCImplementationDecl::Create(Context, CurContext, IDecl, SDecl,
-                                   ClassLoc, AtClassImplLoc, SuperClassLoc);
+  ObjCImplementationDecl *IMPDecl =
+      ObjCImplementationDecl::Create(Context, SemaRef.CurContext, IDecl, SDecl,
+                                     ClassLoc, AtClassImplLoc, SuperClassLoc);
 
-  ProcessDeclAttributeList(TUScope, IMPDecl, Attrs);
-  AddPragmaAttributes(TUScope, IMPDecl);
+  SemaRef.ProcessDeclAttributeList(SemaRef.TUScope, IMPDecl, Attrs);
+  SemaRef.AddPragmaAttributes(SemaRef.TUScope, IMPDecl);
 
   if (CheckObjCDeclScope(IMPDecl)) {
     ActOnObjCContainerStartDefinition(IMPDecl);
@@ -2099,10 +2097,10 @@ ObjCImplementationDecl *Sema::ActOnStartClassImplementation(
     IMPDecl->setInvalidDecl();
   } else { // add it to the list.
     IDecl->setImplementation(IMPDecl);
-    PushOnScopeChains(IMPDecl, TUScope);
+    SemaRef.PushOnScopeChains(IMPDecl, SemaRef.TUScope);
     // Warn on implementating deprecated class under
     // -Wdeprecated-implementations flag.
-    DiagnoseObjCImplementedDeprecations(*this, IDecl, IMPDecl->getLocation());
+    DiagnoseObjCImplementedDeprecations(SemaRef, IDecl, IMPDecl->getLocation());
   }
 
   // If the superclass has the objc_runtime_visible attribute, we
@@ -2118,8 +2116,9 @@ ObjCImplementationDecl *Sema::ActOnStartClassImplementation(
   return IMPDecl;
 }
 
-Sema::DeclGroupPtrTy
-Sema::ActOnFinishObjCImplementation(Decl *ObjCImpDecl, ArrayRef<Decl *> Decls) {
+SemaObjC::DeclGroupPtrTy
+SemaObjC::ActOnFinishObjCImplementation(Decl *ObjCImpDecl,
+                                        ArrayRef<Decl *> Decls) {
   SmallVector<Decl *, 64> DeclsInGroup;
   DeclsInGroup.reserve(Decls.size() + 1);
 
@@ -2134,13 +2133,14 @@ Sema::ActOnFinishObjCImplementation(Decl *ObjCImpDecl, ArrayRef<Decl *> Decls) {
 
   DeclsInGroup.push_back(ObjCImpDecl);
 
-  return BuildDeclaratorGroup(DeclsInGroup);
+  return SemaRef.BuildDeclaratorGroup(DeclsInGroup);
 }
 
-void Sema::CheckImplementationIvars(ObjCImplementationDecl *ImpDecl,
-                                    ObjCIvarDecl **ivars, unsigned numIvars,
-                                    SourceLocation RBrace) {
+void SemaObjC::CheckImplementationIvars(ObjCImplementationDecl *ImpDecl,
+                                        ObjCIvarDecl **ivars, unsigned numIvars,
+                                        SourceLocation RBrace) {
   assert(ImpDecl && "missing implementation decl");
+  ASTContext &Context = getASTContext();
   ObjCInterfaceDecl* IDecl = ImpDecl->getClassInterface();
   if (!IDecl)
     return;
@@ -2156,7 +2156,7 @@ void Sema::CheckImplementationIvars(ObjCImplementationDecl *ImpDecl,
       // ObjCInterfaceDecl while in a 'non-fragile' runtime the ivar is
       // only in the ObjCImplementationDecl. In the non-fragile case the ivar
       // therefore also needs to be propagated to the ObjCInterfaceDecl.
-      if (!LangOpts.ObjCRuntime.isFragile())
+      if (!getLangOpts().ObjCRuntime.isFragile())
         IDecl->makeDeclVisibleInContext(ivars[i]);
       ImpDecl->addDecl(ivars[i]);
     }
@@ -2168,7 +2168,7 @@ void Sema::CheckImplementationIvars(ObjCImplementationDecl *ImpDecl,
     return;
 
   assert(ivars && "missing @implementation ivars");
-  if (LangOpts.ObjCRuntime.isNonFragile()) {
+  if (getLangOpts().ObjCRuntime.isNonFragile()) {
     if (ImpDecl->getSuperClass())
       Diag(ImpDecl->getLocation(), diag::warn_on_superclass_use);
     for (unsigned i = 0; i < numIvars; i++) {
@@ -2255,7 +2255,8 @@ static void WarnUndefinedMethod(Sema &S, ObjCImplDecl *Impl,
   // separate warnings.  We will give that approach a try, as that
   // matches what we do with protocols.
   {
-    const Sema::SemaDiagnosticBuilder &B = S.Diag(Impl->getLocation(), DiagID);
+    const SemaBase::SemaDiagnosticBuilder &B =
+        S.Diag(Impl->getLocation(), DiagID);
     B << method;
     if (NeededFor)
       B << NeededFor;
@@ -2585,22 +2586,21 @@ static bool checkMethodFamilyMismatch(Sema &S, ObjCMethodDecl *impl,
   return true;
 }
 
-void Sema::WarnConflictingTypedMethods(ObjCMethodDecl *ImpMethodDecl,
-                                       ObjCMethodDecl *MethodDecl,
-                                       bool IsProtocolMethodDecl) {
+void SemaObjC::WarnConflictingTypedMethods(ObjCMethodDecl *ImpMethodDecl,
+                                           ObjCMethodDecl *MethodDecl,
+                                           bool IsProtocolMethodDecl) {
   if (getLangOpts().ObjCAutoRefCount &&
-      checkMethodFamilyMismatch(*this, ImpMethodDecl, MethodDecl))
+      checkMethodFamilyMismatch(SemaRef, ImpMethodDecl, MethodDecl))
     return;
 
-  CheckMethodOverrideReturn(*this, ImpMethodDecl, MethodDecl,
-                            IsProtocolMethodDecl, false,
-                            true);
+  CheckMethodOverrideReturn(SemaRef, ImpMethodDecl, MethodDecl,
+                            IsProtocolMethodDecl, false, true);
 
   for (ObjCMethodDecl::param_iterator IM = ImpMethodDecl->param_begin(),
        IF = MethodDecl->param_begin(), EM = ImpMethodDecl->param_end(),
        EF = MethodDecl->param_end();
        IM != EM && IF != EF; ++IM, ++IF) {
-    CheckMethodOverrideParam(*this, ImpMethodDecl, MethodDecl, *IM, *IF,
+    CheckMethodOverrideParam(SemaRef, ImpMethodDecl, MethodDecl, *IM, *IF,
                              IsProtocolMethodDecl, false, true);
   }
 
@@ -2611,19 +2611,18 @@ void Sema::WarnConflictingTypedMethods(ObjCMethodDecl *ImpMethodDecl,
   }
 }
 
-void Sema::CheckConflictingOverridingMethod(ObjCMethodDecl *Method,
-                                       ObjCMethodDecl *Overridden,
-                                       bool IsProtocolMethodDecl) {
+void SemaObjC::CheckConflictingOverridingMethod(ObjCMethodDecl *Method,
+                                                ObjCMethodDecl *Overridden,
+                                                bool IsProtocolMethodDecl) {
 
-  CheckMethodOverrideReturn(*this, Method, Overridden,
-                            IsProtocolMethodDecl, true,
-                            true);
+  CheckMethodOverrideReturn(SemaRef, Method, Overridden, IsProtocolMethodDecl,
+                            true, true);
 
   for (ObjCMethodDecl::param_iterator IM = Method->param_begin(),
        IF = Overridden->param_begin(), EM = Method->param_end(),
        EF = Overridden->param_end();
        IM != EM && IF != EF; ++IM, ++IF) {
-    CheckMethodOverrideParam(*this, Method, Overridden, *IM, *IF,
+    CheckMethodOverrideParam(SemaRef, Method, Overridden, *IM, *IF,
                              IsProtocolMethodDecl, true, true);
   }
 
@@ -2636,9 +2635,10 @@ void Sema::CheckConflictingOverridingMethod(ObjCMethodDecl *Method,
 
 /// WarnExactTypedMethods - This routine issues a warning if method
 /// implementation declaration matches exactly that of its declaration.
-void Sema::WarnExactTypedMethods(ObjCMethodDecl *ImpMethodDecl,
-                                 ObjCMethodDecl *MethodDecl,
-                                 bool IsProtocolMethodDecl) {
+void SemaObjC::WarnExactTypedMethods(ObjCMethodDecl *ImpMethodDecl,
+                                     ObjCMethodDecl *MethodDecl,
+                                     bool IsProtocolMethodDecl) {
+  ASTContext &Context = getASTContext();
   // don't issue warning when protocol method is optional because primary
   // class is not required to implement it and it is safe for protocol
   // to implement it.
@@ -2651,16 +2651,15 @@ void Sema::WarnExactTypedMethods(ObjCMethodDecl *ImpMethodDecl,
       MethodDecl->hasAttr<DeprecatedAttr>())
     return;
 
-  bool match = CheckMethodOverrideReturn(*this, ImpMethodDecl, MethodDecl,
-                                      IsProtocolMethodDecl, false, false);
+  bool match = CheckMethodOverrideReturn(SemaRef, ImpMethodDecl, MethodDecl,
+                                         IsProtocolMethodDecl, false, false);
   if (match)
     for (ObjCMethodDecl::param_iterator IM = ImpMethodDecl->param_begin(),
          IF = MethodDecl->param_begin(), EM = ImpMethodDecl->param_end(),
          EF = MethodDecl->param_end();
          IM != EM && IF != EF; ++IM, ++IF) {
-      match = CheckMethodOverrideParam(*this, ImpMethodDecl, MethodDecl,
-                                       *IM, *IF,
-                                       IsProtocolMethodDecl, false, false);
+      match = CheckMethodOverrideParam(SemaRef, ImpMethodDecl, MethodDecl, *IM,
+                                       *IF, IsProtocolMethodDecl, false, false);
       if (!match)
         break;
     }
@@ -2713,7 +2712,7 @@ static void findProtocolsWithExplicitImpls(const ObjCInterfaceDecl *Super,
 /// Declared in protocol, and those referenced by it.
 static void CheckProtocolMethodDefs(
     Sema &S, ObjCImplDecl *Impl, ObjCProtocolDecl *PDecl, bool &IncompleteImpl,
-    const Sema::SelectorSet &InsMap, const Sema::SelectorSet &ClsMap,
+    const SemaObjC::SelectorSet &InsMap, const SemaObjC::SelectorSet &ClsMap,
     ObjCContainerDecl *CDecl, LazyProtocolNameSet &ProtocolsExplictImpl) {
   ObjCCategoryDecl *C = dyn_cast<ObjCCategoryDecl>(CDecl);
   ObjCInterfaceDecl *IDecl = C ? C->getClassInterface()
@@ -2835,15 +2834,11 @@ static void CheckProtocolMethodDefs(
 /// MatchAllMethodDeclarations - Check methods declared in interface
 /// or protocol against those declared in their implementations.
 ///
-void Sema::MatchAllMethodDeclarations(const SelectorSet &InsMap,
-                                      const SelectorSet &ClsMap,
-                                      SelectorSet &InsMapSeen,
-                                      SelectorSet &ClsMapSeen,
-                                      ObjCImplDecl* IMPDecl,
-                                      ObjCContainerDecl* CDecl,
-                                      bool &IncompleteImpl,
-                                      bool ImmediateClass,
-                                      bool WarnCategoryMethodImpl) {
+void SemaObjC::MatchAllMethodDeclarations(
+    const SelectorSet &InsMap, const SelectorSet &ClsMap,
+    SelectorSet &InsMapSeen, SelectorSet &ClsMapSeen, ObjCImplDecl *IMPDecl,
+    ObjCContainerDecl *CDecl, bool &IncompleteImpl, bool ImmediateClass,
+    bool WarnCategoryMethodImpl) {
   // Check and see if instance methods in class interface have been
   // implemented in the implementation class. If so, their types match.
   for (auto *I : CDecl->instance_methods()) {
@@ -2852,7 +2847,7 @@ void Sema::MatchAllMethodDeclarations(const SelectorSet &InsMap,
     if (!I->isPropertyAccessor() &&
         !InsMap.count(I->getSelector())) {
       if (ImmediateClass)
-        WarnUndefinedMethod(*this, IMPDecl, I, IncompleteImpl,
+        WarnUndefinedMethod(SemaRef, IMPDecl, I, IncompleteImpl,
                             diag::warn_undef_method_impl);
       continue;
     } else {
@@ -2882,7 +2877,7 @@ void Sema::MatchAllMethodDeclarations(const SelectorSet &InsMap,
     if (!I->isPropertyAccessor() &&
         !ClsMap.count(I->getSelector())) {
       if (ImmediateClass)
-        WarnUndefinedMethod(*this, IMPDecl, I, IncompleteImpl,
+        WarnUndefinedMethod(SemaRef, IMPDecl, I, IncompleteImpl,
                             diag::warn_undef_method_impl);
     } else {
       ObjCMethodDecl *ImpMethodDecl =
@@ -2948,8 +2943,8 @@ void Sema::MatchAllMethodDeclarations(const SelectorSet &InsMap,
 /// CheckCategoryVsClassMethodMatches - Checks that methods implemented in
 /// category matches with those implemented in its primary class and
 /// warns each time an exact match is found.
-void Sema::CheckCategoryVsClassMethodMatches(
-                                  ObjCCategoryImplDecl *CatIMPDecl) {
+void SemaObjC::CheckCategoryVsClassMethodMatches(
+    ObjCCategoryImplDecl *CatIMPDecl) {
   // Get category's primary class.
   ObjCCategoryDecl *CatDecl = CatIMPDecl->getCategoryDecl();
   if (!CatDecl)
@@ -2987,9 +2982,9 @@ void Sema::CheckCategoryVsClassMethodMatches(
                              true /*WarnCategoryMethodImpl*/);
 }
 
-void Sema::ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
-                                     ObjCContainerDecl* CDecl,
-                                     bool IncompleteImpl) {
+void SemaObjC::ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl *IMPDecl,
+                                         ObjCContainerDecl *CDecl,
+                                         bool IncompleteImpl) {
   SelectorSet InsMap;
   // Check and see if instance methods in class interface have been
   // implemented in the implementation class.
@@ -3014,8 +3009,8 @@ void Sema::ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
   // an implementation or 2) there is a @synthesize/@dynamic implementation
   // of the property in the @implementation.
   if (const ObjCInterfaceDecl *IDecl = dyn_cast<ObjCInterfaceDecl>(CDecl)) {
-    bool SynthesizeProperties = LangOpts.ObjCDefaultSynthProperties &&
-                                LangOpts.ObjCRuntime.isNonFragile() &&
+    bool SynthesizeProperties = getLangOpts().ObjCDefaultSynthProperties &&
+                                getLangOpts().ObjCRuntime.isNonFragile() &&
                                 !IDecl->isObjCRequiresPropertyDefs();
     DiagnoseUnimplementedProperties(S, IMPDecl, CDecl, SynthesizeProperties);
   }
@@ -3049,14 +3044,14 @@ void Sema::ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
 
   if (ObjCInterfaceDecl *I = dyn_cast<ObjCInterfaceDecl> (CDecl)) {
     for (auto *PI : I->all_referenced_protocols())
-      CheckProtocolMethodDefs(*this, IMPDecl, PI, IncompleteImpl, InsMap,
+      CheckProtocolMethodDefs(SemaRef, IMPDecl, PI, IncompleteImpl, InsMap,
                               ClsMap, I, ExplicitImplProtocols);
   } else if (ObjCCategoryDecl *C = dyn_cast<ObjCCategoryDecl>(CDecl)) {
     // For extended class, unimplemented methods in its protocols will
     // be reported in the primary class.
     if (!C->IsClassExtension()) {
       for (auto *P : C->protocols())
-        CheckProtocolMethodDefs(*this, IMPDecl, P, IncompleteImpl, InsMap,
+        CheckProtocolMethodDefs(SemaRef, IMPDecl, P, IncompleteImpl, InsMap,
                                 ClsMap, CDecl, ExplicitImplProtocols);
       DiagnoseUnimplementedProperties(S, IMPDecl, CDecl,
                                       /*SynthesizeProperties=*/false);
@@ -3065,18 +3060,17 @@ void Sema::ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
     llvm_unreachable("invalid ObjCContainerDecl type.");
 }
 
-Sema::DeclGroupPtrTy
-Sema::ActOnForwardClassDeclaration(SourceLocation AtClassLoc,
-                                   IdentifierInfo **IdentList,
-                                   SourceLocation *IdentLocs,
-                                   ArrayRef<ObjCTypeParamList *> TypeParamLists,
-                                   unsigned NumElts) {
+SemaObjC::DeclGroupPtrTy SemaObjC::ActOnForwardClassDeclaration(
+    SourceLocation AtClassLoc, IdentifierInfo **IdentList,
+    SourceLocation *IdentLocs, ArrayRef<ObjCTypeParamList *> TypeParamLists,
+    unsigned NumElts) {
+  ASTContext &Context = getASTContext();
   SmallVector<Decl *, 8> DeclsInGroup;
   for (unsigned i = 0; i != NumElts; ++i) {
     // Check for another declaration kind with the same name.
-    NamedDecl *PrevDecl
-      = LookupSingleName(TUScope, IdentList[i], IdentLocs[i],
-                         LookupOrdinaryName, forRedeclarationInCurContext());
+    NamedDecl *PrevDecl = SemaRef.LookupSingleName(
+        SemaRef.TUScope, IdentList[i], IdentLocs[i], Sema::LookupOrdinaryName,
+        SemaRef.forRedeclarationInCurContext());
     if (PrevDecl && !isa<ObjCInterfaceDecl>(PrevDecl)) {
       // GCC apparently allows the following idiom:
       //
@@ -3131,8 +3125,8 @@ Sema::ActOnForwardClassDeclaration(SourceLocation AtClassLoc,
       if (ObjCTypeParamList *PrevTypeParams = PrevIDecl->getTypeParamList()) {
         // Check for consistency with the previous declaration.
         if (checkTypeParamListConsistency(
-              *this, PrevTypeParams, TypeParams,
-              TypeParamListContext::ForwardDeclaration)) {
+                SemaRef, PrevTypeParams, TypeParams,
+                TypeParamListContext::ForwardDeclaration)) {
           TypeParams = nullptr;
         }
       } else if (ObjCInterfaceDecl *Def = PrevIDecl->getDefinition()) {
@@ -3147,29 +3141,29 @@ Sema::ActOnForwardClassDeclaration(SourceLocation AtClassLoc,
       }
     }
 
-    ObjCInterfaceDecl *IDecl
-      = ObjCInterfaceDecl::Create(Context, CurContext, AtClassLoc,
-                                  ClassName, TypeParams, PrevIDecl,
-                                  IdentLocs[i]);
+    ObjCInterfaceDecl *IDecl = ObjCInterfaceDecl::Create(
+        Context, SemaRef.CurContext, AtClassLoc, ClassName, TypeParams,
+        PrevIDecl, IdentLocs[i]);
     IDecl->setAtEndRange(IdentLocs[i]);
 
     if (PrevIDecl)
-      mergeDeclAttributes(IDecl, PrevIDecl);
+      SemaRef.mergeDeclAttributes(IDecl, PrevIDecl);
 
-    PushOnScopeChains(IDecl, TUScope);
+    SemaRef.PushOnScopeChains(IDecl, SemaRef.TUScope);
     CheckObjCDeclScope(IDecl);
     DeclsInGroup.push_back(IDecl);
   }
 
-  return BuildDeclaratorGroup(DeclsInGroup);
+  return SemaRef.BuildDeclaratorGroup(DeclsInGroup);
 }
 
 static bool tryMatchRecordTypes(ASTContext &Context,
-                                Sema::MethodMatchStrategy strategy,
+                                SemaObjC::MethodMatchStrategy strategy,
                                 const Type *left, const Type *right);
 
-static bool matchTypes(ASTContext &Context, Sema::MethodMatchStrategy strategy,
-                       QualType leftQT, QualType rightQT) {
+static bool matchTypes(ASTContext &Context,
+                       SemaObjC::MethodMatchStrategy strategy, QualType leftQT,
+                       QualType rightQT) {
   const Type *left =
     Context.getCanonicalType(leftQT).getUnqualifiedType().getTypePtr();
   const Type *right =
@@ -3178,7 +3172,8 @@ static bool matchTypes(ASTContext &Context, Sema::MethodMatchStrategy strategy,
   if (left == right) return true;
 
   // If we're doing a strict match, the types have to match exactly.
-  if (strategy == Sema::MMS_strict) return false;
+  if (strategy == SemaObjC::MMS_strict)
+    return false;
 
   if (left->isIncompleteType() || right->isIncompleteType()) return false;
 
@@ -3226,7 +3221,7 @@ static bool matchTypes(ASTContext &Context, Sema::MethodMatchStrategy strategy,
 }
 
 static bool tryMatchRecordTypes(ASTContext &Context,
-                                Sema::MethodMatchStrategy strategy,
+                                SemaObjC::MethodMatchStrategy strategy,
                                 const Type *lt, const Type *rt) {
   assert(lt && rt && lt != rt);
 
@@ -3264,9 +3259,10 @@ static bool tryMatchRecordTypes(ASTContext &Context,
 /// MatchTwoMethodDeclarations - Checks that two methods have matching type and
 /// returns true, or false, accordingly.
 /// TODO: Handle protocol list; such as id<p1,p2> in type comparisons
-bool Sema::MatchTwoMethodDeclarations(const ObjCMethodDecl *left,
-                                      const ObjCMethodDecl *right,
-                                      MethodMatchStrategy strategy) {
+bool SemaObjC::MatchTwoMethodDeclarations(const ObjCMethodDecl *left,
+                                          const ObjCMethodDecl *right,
+                                          MethodMatchStrategy strategy) {
+  ASTContext &Context = getASTContext();
   if (!matchTypes(Context, strategy, left->getReturnType(),
                   right->getReturnType()))
     return false;
@@ -3323,8 +3319,8 @@ static bool isMethodContextSameForKindofLookup(ObjCMethodDecl *Method,
   return MethodInterface == MethodInListInterface;
 }
 
-void Sema::addMethodToGlobalList(ObjCMethodList *List,
-                                 ObjCMethodDecl *Method) {
+void SemaObjC::addMethodToGlobalList(ObjCMethodList *List,
+                                     ObjCMethodDecl *Method) {
   // Record at the head of the list whether there were 0, 1, or >= 2 methods
   // inside categories.
   if (ObjCCategoryDecl *CD =
@@ -3412,7 +3408,7 @@ void Sema::addMethodToGlobalList(ObjCMethodList *List,
 
   // We have a new signature for an existing method - add it.
   // This is extremely rare. Only 1% of Cocoa selectors are "overloaded".
-  ObjCMethodList *Mem = BumpAlloc.Allocate<ObjCMethodList>();
+  ObjCMethodList *Mem = SemaRef.BumpAlloc.Allocate<ObjCMethodList>();
 
   // We insert it right before ListWithSameDeclaration.
   if (ListWithSameDeclaration) {
@@ -3428,24 +3424,24 @@ void Sema::addMethodToGlobalList(ObjCMethodList *List,
 
 /// Read the contents of the method pool for a given selector from
 /// external storage.
-void Sema::ReadMethodPool(Selector Sel) {
-  assert(ExternalSource && "We need an external AST source");
-  ExternalSource->ReadMethodPool(Sel);
+void SemaObjC::ReadMethodPool(Selector Sel) {
+  assert(SemaRef.ExternalSource && "We need an external AST source");
+  SemaRef.ExternalSource->ReadMethodPool(Sel);
 }
 
-void Sema::updateOutOfDateSelector(Selector Sel) {
-  if (!ExternalSource)
+void SemaObjC::updateOutOfDateSelector(Selector Sel) {
+  if (!SemaRef.ExternalSource)
     return;
-  ExternalSource->updateOutOfDateSelector(Sel);
+  SemaRef.ExternalSource->updateOutOfDateSelector(Sel);
 }
 
-void Sema::AddMethodToGlobalPool(ObjCMethodDecl *Method, bool impl,
-                                 bool instance) {
+void SemaObjC::AddMethodToGlobalPool(ObjCMethodDecl *Method, bool impl,
+                                     bool instance) {
   // Ignore methods of invalid containers.
   if (cast<Decl>(Method->getDeclContext())->isInvalidDecl())
     return;
 
-  if (ExternalSource)
+  if (SemaRef.ExternalSource)
     ReadMethodPool(Method->getSelector());
 
   GlobalMethodPool::iterator Pos = MethodPool.find(Method->getSelector());
@@ -3518,11 +3514,10 @@ static bool FilterMethodsByTypeBound(ObjCMethodDecl *Method,
 
 /// We first select the type of the method: Instance or Factory, then collect
 /// all methods with that type.
-bool Sema::CollectMultipleMethodsInGlobalPool(
+bool SemaObjC::CollectMultipleMethodsInGlobalPool(
     Selector Sel, SmallVectorImpl<ObjCMethodDecl *> &Methods,
-    bool InstanceFirst, bool CheckTheOther,
-    const ObjCObjectType *TypeBound) {
-  if (ExternalSource)
+    bool InstanceFirst, bool CheckTheOther, const ObjCObjectType *TypeBound) {
+  if (SemaRef.ExternalSource)
     ReadMethodPool(Sel);
 
   GlobalMethodPool::iterator Pos = MethodPool.find(Sel);
@@ -3557,7 +3552,7 @@ bool Sema::CollectMultipleMethodsInGlobalPool(
   return Methods.size() > 1;
 }
 
-bool Sema::AreMultipleMethodsInGlobalPool(
+bool SemaObjC::AreMultipleMethodsInGlobalPool(
     Selector Sel, ObjCMethodDecl *BestMethod, SourceRange R,
     bool receiverIdOrClass, SmallVectorImpl<ObjCMethodDecl *> &Methods) {
   // Diagnose finding more than one method in global pool.
@@ -3582,10 +3577,10 @@ bool Sema::AreMultipleMethodsInGlobalPool(
   return MethList.hasMoreThanOneDecl();
 }
 
-ObjCMethodDecl *Sema::LookupMethodInGlobalPool(Selector Sel, SourceRange R,
-                                               bool receiverIdOrClass,
-                                               bool instance) {
-  if (ExternalSource)
+ObjCMethodDecl *SemaObjC::LookupMethodInGlobalPool(Selector Sel, SourceRange R,
+                                                   bool receiverIdOrClass,
+                                                   bool instance) {
+  if (SemaRef.ExternalSource)
     ReadMethodPool(Sel);
 
   GlobalMethodPool::iterator Pos = MethodPool.find(Sel);
@@ -3602,17 +3597,18 @@ ObjCMethodDecl *Sema::LookupMethodInGlobalPool(Selector Sel, SourceRange R,
   return nullptr;
 }
 
-void Sema::DiagnoseMultipleMethodInGlobalPool(SmallVectorImpl<ObjCMethodDecl*> &Methods,
-                                              Selector Sel, SourceRange R,
-                                              bool receiverIdOrClass) {
+void SemaObjC::DiagnoseMultipleMethodInGlobalPool(
+    SmallVectorImpl<ObjCMethodDecl *> &Methods, Selector Sel, SourceRange R,
+    bool receiverIdOrClass) {
   // We found multiple methods, so we may have to complain.
   bool issueDiagnostic = false, issueError = false;
 
   // We support a warning which complains about *any* difference in
   // method signature.
   bool strictSelectorMatch =
-  receiverIdOrClass &&
-  !Diags.isIgnored(diag::warn_strict_multiple_method_decl, R.getBegin());
+      receiverIdOrClass &&
+      !getDiagnostics().isIgnored(diag::warn_strict_multiple_method_decl,
+                                  R.getBegin());
   if (strictSelectorMatch) {
     for (unsigned I = 1, N = Methods.size(); I != N; ++I) {
       if (!MatchTwoMethodDeclarations(Methods[0], Methods[I], MMS_strict)) {
@@ -3656,7 +3652,7 @@ void Sema::DiagnoseMultipleMethodInGlobalPool(SmallVectorImpl<ObjCMethodDecl*> &
   }
 }
 
-ObjCMethodDecl *Sema::LookupImplementedMethodInGlobalPool(Selector Sel) {
+ObjCMethodDecl *SemaObjC::LookupImplementedMethodInGlobalPool(Selector Sel) {
   GlobalMethodPool::iterator Pos = MethodPool.find(Sel);
   if (Pos == MethodPool.end())
     return nullptr;
@@ -3705,15 +3701,15 @@ static bool HelperIsMethodInObjCType(Sema &S, Selector Sel,
                                      QualType ObjectType) {
   if (ObjectType.isNull())
     return true;
-  if (S.LookupMethodInObjectType(Sel, ObjectType, true/*Instance method*/))
+  if (S.ObjC().LookupMethodInObjectType(Sel, ObjectType,
+                                        true /*Instance method*/))
     return true;
-  return S.LookupMethodInObjectType(Sel, ObjectType, false/*Class method*/) !=
-         nullptr;
+  return S.ObjC().LookupMethodInObjectType(Sel, ObjectType,
+                                           false /*Class method*/) != nullptr;
 }
 
 const ObjCMethodDecl *
-Sema::SelectorsForTypoCorrection(Selector Sel,
-                                 QualType ObjectType) {
+SemaObjC::SelectorsForTypoCorrection(Selector Sel, QualType ObjectType) {
   unsigned NumArgs = Sel.getNumArgs();
   SmallVector<const ObjCMethodDecl *, 8> Methods;
   bool ObjectIsId = true, ObjectIsClass = true;
@@ -3743,8 +3739,8 @@ Sema::SelectorsForTypoCorrection(Selector Sel,
         if (ObjectIsId)
           Methods.push_back(M->getMethod());
         else if (!ObjectIsClass &&
-                 HelperIsMethodInObjCType(*this, M->getMethod()->getSelector(),
-                                          ObjectType))
+                 HelperIsMethodInObjCType(
+                     SemaRef, M->getMethod()->getSelector(), ObjectType))
           Methods.push_back(M->getMethod());
       }
     // class methods
@@ -3755,8 +3751,8 @@ Sema::SelectorsForTypoCorrection(Selector Sel,
         if (ObjectIsClass)
           Methods.push_back(M->getMethod());
         else if (!ObjectIsId &&
-                 HelperIsMethodInObjCType(*this, M->getMethod()->getSelector(),
-                                          ObjectType))
+                 HelperIsMethodInObjCType(
+                     SemaRef, M->getMethod()->getSelector(), ObjectType))
           Methods.push_back(M->getMethod());
       }
   }
@@ -3774,8 +3770,8 @@ Sema::SelectorsForTypoCorrection(Selector Sel,
 /// \@implementation. This becomes necessary because class extension can
 /// add ivars to a class in random order which will not be known until
 /// class's \@implementation is seen.
-void Sema::DiagnoseDuplicateIvars(ObjCInterfaceDecl *ID,
-                                  ObjCInterfaceDecl *SID) {
+void SemaObjC::DiagnoseDuplicateIvars(ObjCInterfaceDecl *ID,
+                                      ObjCInterfaceDecl *SID) {
   for (auto *Ivar : ID->ivars()) {
     if (Ivar->isInvalidDecl())
       continue;
@@ -3827,23 +3823,23 @@ static void DiagnoseRetainableFlexibleArrayMember(Sema &S,
   }
 }
 
-Sema::ObjCContainerKind Sema::getObjCContainerKind() const {
-  switch (CurContext->getDeclKind()) {
-    case Decl::ObjCInterface:
-      return Sema::OCK_Interface;
-    case Decl::ObjCProtocol:
-      return Sema::OCK_Protocol;
-    case Decl::ObjCCategory:
-      if (cast<ObjCCategoryDecl>(CurContext)->IsClassExtension())
-        return Sema::OCK_ClassExtension;
-      return Sema::OCK_Category;
-    case Decl::ObjCImplementation:
-      return Sema::OCK_Implementation;
-    case Decl::ObjCCategoryImpl:
-      return Sema::OCK_CategoryImplementation;
-
-    default:
-      return Sema::OCK_None;
+SemaObjC::ObjCContainerKind SemaObjC::getObjCContainerKind() const {
+  switch (SemaRef.CurContext->getDeclKind()) {
+  case Decl::ObjCInterface:
+    return SemaObjC::OCK_Interface;
+  case Decl::ObjCProtocol:
+    return SemaObjC::OCK_Protocol;
+  case Decl::ObjCCategory:
+    if (cast<ObjCCategoryDecl>(SemaRef.CurContext)->IsClassExtension())
+      return SemaObjC::OCK_ClassExtension;
+    return SemaObjC::OCK_Category;
+  case Decl::ObjCImplementation:
+    return SemaObjC::OCK_Implementation;
+  case Decl::ObjCCategoryImpl:
+    return SemaObjC::OCK_CategoryImplementation;
+
+  default:
+    return SemaObjC::OCK_None;
   }
 }
 
@@ -3987,14 +3983,16 @@ static void DiagnoseCategoryDirectMembersProtocolConformance(
 }
 
 // Note: For class/category implementations, allMethods is always null.
-Decl *Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd, ArrayRef<Decl *> allMethods,
-                       ArrayRef<DeclGroupPtrTy> allTUVars) {
-  if (getObjCContainerKind() == Sema::OCK_None)
+Decl *SemaObjC::ActOnAtEnd(Scope *S, SourceRange AtEnd,
+                           ArrayRef<Decl *> allMethods,
+                           ArrayRef<DeclGroupPtrTy> allTUVars) {
+  ASTContext &Context = getASTContext();
+  if (getObjCContainerKind() == SemaObjC::OCK_None)
     return nullptr;
 
   assert(AtEnd.isValid() && "Invalid location for '@end'");
 
-  auto *OCD = cast<ObjCContainerDecl>(CurContext);
+  auto *OCD = cast<ObjCContainerDecl>(SemaRef.CurContext);
   Decl *ClassDecl = OCD;
 
   bool isInterfaceDeclKind =
@@ -4006,7 +4004,7 @@ Decl *Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd, ArrayRef<Decl *> allMethods,
   // ActOnPropertyImplDecl() creates them as not visible in case
   // they are overridden by an explicit method that is encountered
   // later.
-  if (auto *OID = dyn_cast<ObjCImplementationDecl>(CurContext)) {
+  if (auto *OID = dyn_cast<ObjCImplementationDecl>(SemaRef.CurContext)) {
     for (auto *PropImpl : OID->property_impls()) {
       if (auto *Getter = PropImpl->getGetterMethodDecl())
         if (Getter->isSynthesizedAccessorStub())
@@ -4087,7 +4085,8 @@ Decl *Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd, ArrayRef<Decl *> allMethods,
       DiagnoseClassExtensionDupMethods(C, CCPrimary);
     }
 
-    DiagnoseCategoryDirectMembersProtocolConformance(*this, C, C->protocols());
+    DiagnoseCategoryDirectMembersProtocolConformance(SemaRef, C,
+                                                     C->protocols());
   }
   if (ObjCContainerDecl *CDecl = dyn_cast<ObjCContainerDecl>(ClassDecl)) {
     if (CDecl->getIdentifier())
@@ -4133,8 +4132,8 @@ Decl *Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd, ArrayRef<Decl *> allMethods,
       DiagnoseUnusedBackingIvarInAccessor(S, IC);
       if (IDecl->hasDesignatedInitializers())
         DiagnoseMissingDesignatedInitOverrides(IC, IDecl);
-      DiagnoseWeakIvars(*this, IC);
-      DiagnoseRetainableFlexibleArrayMember(*this, IDecl);
+      DiagnoseWeakIvars(SemaRef, IC);
+      DiagnoseRetainableFlexibleArrayMember(SemaRef, IDecl);
 
       bool HasRootClassAttr = IDecl->hasAttr<ObjCRootClassAttr>();
       if (IDecl->getSuperClass() == nullptr) {
@@ -4142,14 +4141,14 @@ Decl *Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd, ArrayRef<Decl *> allMethods,
         // __attribute((objc_root_class)).
         if (!HasRootClassAttr) {
           SourceLocation DeclLoc(IDecl->getLocation());
-          SourceLocation SuperClassLoc(getLocForEndOfToken(DeclLoc));
+          SourceLocation SuperClassLoc(SemaRef.getLocForEndOfToken(DeclLoc));
           Diag(DeclLoc, diag::warn_objc_root_class_missing)
             << IDecl->getIdentifier();
           // See if NSObject is in the current scope, and if it is, suggest
           // adding " : NSObject " to the class declaration.
-          NamedDecl *IF = LookupSingleName(TUScope,
-                                           NSAPIObj->getNSClassId(NSAPI::ClassId_NSObject),
-                                           DeclLoc, LookupOrdinaryName);
+          NamedDecl *IF = SemaRef.LookupSingleName(
+              SemaRef.TUScope, NSAPIObj->getNSClassId(NSAPI::ClassId_NSObject),
+              DeclLoc, Sema::LookupOrdinaryName);
           ObjCInterfaceDecl *NSObjectDecl = dyn_cast_or_null<ObjCInterfaceDecl>(IF);
           if (NSObjectDecl && NSObjectDecl->getDefinition()) {
             Diag(SuperClassLoc, diag::note_objc_needs_superclass)
@@ -4178,7 +4177,7 @@ Decl *Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd, ArrayRef<Decl *> allMethods,
       if (IDecl->hasAttr<ObjCClassStubAttr>())
         Diag(IC->getLocation(), diag::err_implementation_of_class_stub);
 
-      if (LangOpts.ObjCRuntime.isNonFragile()) {
+      if (getLangOpts().ObjCRuntime.isNonFragile()) {
         while (IDecl->getSuperClass()) {
           DiagnoseDuplicateIvars(IDecl, IDecl->getSuperClass());
           IDecl = IDecl->getSuperClass();
@@ -4211,7 +4210,7 @@ Decl *Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd, ArrayRef<Decl *> allMethods,
         !IntfDecl->hasAttr<ObjCSubclassingRestrictedAttr>())
       Diag(IntfDecl->getLocation(), diag::err_class_stub_subclassing_mismatch);
   }
-  DiagnoseVariableSizedIvars(*this, OCD);
+  DiagnoseVariableSizedIvars(SemaRef, OCD);
   if (isInterfaceDeclKind) {
     // Reject invalid vardecls.
     for (unsigned i = 0, e = allTUVars.size(); i != e; i++) {
@@ -4229,10 +4228,10 @@ Decl *Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd, ArrayRef<Decl *> allMethods,
     DeclGroupRef DG = allTUVars[i].get();
     for (DeclGroupRef::iterator I = DG.begin(), E = DG.end(); I != E; ++I)
       (*I)->setTopLevelDeclInObjCContainer();
-    Consumer.HandleTopLevelDeclInObjCContainer(DG);
+    SemaRef.Consumer.HandleTopLevelDeclInObjCContainer(DG);
   }
 
-  ActOnDocumentableDecl(ClassDecl);
+  SemaRef.ActOnDocumentableDecl(ClassDecl);
   return ClassDecl;
 }
 
@@ -4246,7 +4245,7 @@ CvtQTToAstBitMask(ObjCDeclSpec::ObjCDeclQualifier PQTVal) {
 /// Check whether the declared result type of the given Objective-C
 /// method declaration is compatible with the method's class.
 ///
-static Sema::ResultTypeCompatibilityKind
+static SemaObjC::ResultTypeCompatibilityKind
 CheckRelatedResultTypeCompatibility(Sema &S, ObjCMethodDecl *Method,
                                     ObjCInterfaceDecl *CurrentClass) {
   QualType ResultType = Method->getReturnType();
@@ -4259,27 +4258,27 @@ CheckRelatedResultTypeCompatibility(Sema &S, ObjCMethodDecl *Method,
     //   - it is id or qualified id, or
     if (ResultObjectType->isObjCIdType() ||
         ResultObjectType->isObjCQualifiedIdType())
-      return Sema::RTC_Compatible;
+      return SemaObjC::RTC_Compatible;
 
     if (CurrentClass) {
       if (ObjCInterfaceDecl *ResultClass
                                       = ResultObjectType->getInterfaceDecl()) {
         //   - it is the same as the method's class type, or
         if (declaresSameEntity(CurrentClass, ResultClass))
-          return Sema::RTC_Compatible;
+          return SemaObjC::RTC_Compatible;
 
         //   - it is a superclass of the method's class type
         if (ResultClass->isSuperClassOf(CurrentClass))
-          return Sema::RTC_Compatible;
+          return SemaObjC::RTC_Compatible;
       }
     } else {
       // Any Objective-C pointer type might be acceptable for a protocol
       // method; we just don't know.
-      return Sema::RTC_Unknown;
+      return SemaObjC::RTC_Unknown;
     }
   }
 
-  return Sema::RTC_Incompatible;
+  return SemaObjC::RTC_Incompatible;
 }
 
 namespace {
@@ -4297,13 +4296,14 @@ public:
 
     // Bypass this search if we've never seen an instance/class method
     // with this selector before.
-    Sema::GlobalMethodPool::iterator it = S.MethodPool.find(selector);
-    if (it == S.MethodPool.end()) {
+    SemaObjC::GlobalMethodPool::iterator it =
+        S.ObjC().MethodPool.find(selector);
+    if (it == S.ObjC().MethodPool.end()) {
       if (!S.getExternalSource()) return;
-      S.ReadMethodPool(selector);
+      S.ObjC().ReadMethodPool(selector);
 
-      it = S.MethodPool.find(selector);
-      if (it == S.MethodPool.end())
+      it = S.ObjC().MethodPool.find(selector);
+      if (it == S.ObjC().MethodPool.end())
         return;
     }
     const ObjCMethodList &list =
@@ -4430,8 +4430,8 @@ private:
 };
 } // end anonymous namespace
 
-void Sema::CheckObjCMethodDirectOverrides(ObjCMethodDecl *method,
-                                          ObjCMethodDecl *overridden) {
+void SemaObjC::CheckObjCMethodDirectOverrides(ObjCMethodDecl *method,
+                                              ObjCMethodDecl *overridden) {
   if (overridden->isDirectMethod()) {
     const auto *attr = overridden->getAttr<ObjCDirectAttr>();
     Diag(method->getLocation(), diag::err_objc_override_direct_method);
@@ -4444,9 +4444,10 @@ void Sema::CheckObjCMethodDirectOverrides(ObjCMethodDecl *method,
   }
 }
 
-void Sema::CheckObjCMethodOverrides(ObjCMethodDecl *ObjCMethod,
-                                    ObjCInterfaceDecl *CurrentClass,
-                                    ResultTypeCompatibilityKind RTC) {
+void SemaObjC::CheckObjCMethodOverrides(ObjCMethodDecl *ObjCMethod,
+                                        ObjCInterfaceDecl *CurrentClass,
+                                        ResultTypeCompatibilityKind RTC) {
+  ASTContext &Context = getASTContext();
   if (!ObjCMethod)
     return;
   auto IsMethodInCurrentClass = [CurrentClass](const ObjCMethodDecl *M) {
@@ -4455,7 +4456,7 @@ void Sema::CheckObjCMethodOverrides(ObjCMethodDecl *ObjCMethod,
            CurrentClass->getCanonicalDecl();
   };
   // Search for overridden methods and merge information down from them.
-  OverrideSearch overrides(*this, ObjCMethod);
+  OverrideSearch overrides(SemaRef, ObjCMethod);
   // Keep track if the method overrides any method in the class's base classes,
   // its protocols, or its categories' protocols; we will keep that info
   // in the ObjCMethodDecl.
@@ -4487,7 +4488,7 @@ void Sema::CheckObjCMethodOverrides(ObjCMethodDecl *ObjCMethod,
             // least 2 category methods recorded, otherwise only one will do.
             if (CategCount > 1 ||
                 !isa<ObjCCategoryImplDecl>(overridden->getDeclContext())) {
-              OverrideSearch overrides(*this, overridden);
+              OverrideSearch overrides(SemaRef, overridden);
               for (ObjCMethodDecl *SuperOverridden : overrides) {
                 if (isa<ObjCProtocolDecl>(SuperOverridden->getDeclContext()) ||
                     !IsMethodInCurrentClass(SuperOverridden)) {
@@ -4504,11 +4505,11 @@ void Sema::CheckObjCMethodOverrides(ObjCMethodDecl *ObjCMethod,
     }
 
     // Propagate down the 'related result type' bit from overridden methods.
-    if (RTC != Sema::RTC_Incompatible && overridden->hasRelatedResultType())
+    if (RTC != SemaObjC::RTC_Incompatible && overridden->hasRelatedResultType())
       ObjCMethod->setRelatedResultType();
 
     // Then merge the declarations.
-    mergeObjCMethodDecls(ObjCMethod, overridden);
+    SemaRef.mergeObjCMethodDecls(ObjCMethod, overridden);
 
     if (ObjCMethod->isImplicit() && overridden->isImplicit())
       continue; // Conflicting properties are detected elsewhere.
@@ -4727,7 +4728,7 @@ static void checkObjCDirectMethodClashes(Sema &S, ObjCInterfaceDecl *IDecl,
           diagClash(IMD);
 }
 
-Decl *Sema::ActOnMethodDeclaration(
+Decl *SemaObjC::ActOnMethodDeclaration(
     Scope *S, SourceLocation MethodLoc, SourceLocation EndLoc,
     tok::TokenKind MethodType, ObjCDeclSpec &ReturnQT, ParsedType ReturnType,
     ArrayRef<SourceLocation> SelectorLocs, Selector Sel,
@@ -4737,21 +4738,22 @@ Decl *Sema::ActOnMethodDeclaration(
     unsigned CNumArgs, // c-style args
     const ParsedAttributesView &AttrList, tok::ObjCKeywordKind MethodDeclKind,
     bool isVariadic, bool MethodDefinition) {
+  ASTContext &Context = getASTContext();
   // Make sure we can establish a context for the method.
-  if (!CurContext->isObjCContainer()) {
+  if (!SemaRef.CurContext->isObjCContainer()) {
     Diag(MethodLoc, diag::err_missing_method_context);
     return nullptr;
   }
 
-  Decl *ClassDecl = cast<ObjCContainerDecl>(CurContext);
+  Decl *ClassDecl = cast<ObjCContainerDecl>(SemaRef.CurContext);
   QualType resultDeclType;
 
   bool HasRelatedResultType = false;
   TypeSourceInfo *ReturnTInfo = nullptr;
   if (ReturnType) {
-    resultDeclType = GetTypeFromParser(ReturnType, &ReturnTInfo);
+    resultDeclType = SemaRef.GetTypeFromParser(ReturnType, &ReturnTInfo);
 
-    if (CheckFunctionReturnType(resultDeclType, MethodLoc))
+    if (SemaRef.CheckFunctionReturnType(resultDeclType, MethodLoc))
       return nullptr;
 
     QualType bareResultType = resultDeclType;
@@ -4764,8 +4766,8 @@ Decl *Sema::ActOnMethodDeclaration(
   }
 
   ObjCMethodDecl *ObjCMethod = ObjCMethodDecl::Create(
-      Context, MethodLoc, EndLoc, Sel, resultDeclType, ReturnTInfo, CurContext,
-      MethodType == tok::minus, isVariadic,
+      Context, MethodLoc, EndLoc, Sel, resultDeclType, ReturnTInfo,
+      SemaRef.CurContext, MethodType == tok::minus, isVariadic,
       /*isPropertyAccessor=*/false, /*isSynthesizedAccessorStub=*/false,
       /*isImplicitlyDeclared=*/false, /*isDefined=*/false,
       MethodDeclKind == tok::objc_optional
@@ -4783,12 +4785,13 @@ Decl *Sema::ActOnMethodDeclaration(
       ArgType = Context.getObjCIdType();
       DI = nullptr;
     } else {
-      ArgType = GetTypeFromParser(ArgInfo[i].Type, &DI);
+      ArgType = SemaRef.GetTypeFromParser(ArgInfo[i].Type, &DI);
     }
 
-    LookupResult R(*this, ArgInfo[i].Name, ArgInfo[i].NameLoc,
-                   LookupOrdinaryName, forRedeclarationInCurContext());
-    LookupName(R, S);
+    LookupResult R(SemaRef, ArgInfo[i].Name, ArgInfo[i].NameLoc,
+                   Sema::LookupOrdinaryName,
+                   SemaRef.forRedeclarationInCurContext());
+    SemaRef.LookupName(R, S);
     if (R.isSingleResult()) {
       NamedDecl *PrevDecl = R.getFoundDecl();
       if (S->isDeclScope(PrevDecl)) {
@@ -4805,9 +4808,9 @@ Decl *Sema::ActOnMethodDeclaration(
       ? DI->getTypeLoc().getBeginLoc()
       : ArgInfo[i].NameLoc;
 
-    ParmVarDecl* Param = CheckParameter(ObjCMethod, StartLoc,
-                                        ArgInfo[i].NameLoc, ArgInfo[i].Name,
-                                        ArgType, DI, SC_None);
+    ParmVarDecl *Param =
+        SemaRef.CheckParameter(ObjCMethod, StartLoc, ArgInfo[i].NameLoc,
+                               ArgInfo[i].Name, ArgType, DI, SC_None);
 
     Param->setObjCMethodScopeInfo(i);
 
@@ -4815,16 +4818,17 @@ Decl *Sema::ActOnMethodDeclaration(
       CvtQTToAstBitMask(ArgInfo[i].DeclSpec.getObjCDeclQualifier()));
 
     // Apply the attributes to the parameter.
-    ProcessDeclAttributeList(TUScope, Param, ArgInfo[i].ArgAttrs);
-    AddPragmaAttributes(TUScope, Param);
-    ProcessAPINotes(Param);
+    SemaRef.ProcessDeclAttributeList(SemaRef.TUScope, Param,
+                                     ArgInfo[i].ArgAttrs);
+    SemaRef.AddPragmaAttributes(SemaRef.TUScope, Param);
+    SemaRef.ProcessAPINotes(Param);
 
     if (Param->hasAttr<BlocksAttr>()) {
       Diag(Param->getLocation(), diag::err_block_on_nonlocal);
       Param->setInvalidDecl();
     }
     S->AddDecl(Param);
-    IdResolver.AddDecl(Param);
+    SemaRef.IdResolver.AddDecl(Param);
 
     Params.push_back(Param);
   }
@@ -4846,9 +4850,9 @@ Decl *Sema::ActOnMethodDeclaration(
   ObjCMethod->setObjCDeclQualifier(
     CvtQTToAstBitMask(ReturnQT.getObjCDeclQualifier()));
 
-  ProcessDeclAttributeList(TUScope, ObjCMethod, AttrList);
-  AddPragmaAttributes(TUScope, ObjCMethod);
-  ProcessAPINotes(ObjCMethod);
+  SemaRef.ProcessDeclAttributeList(SemaRef.TUScope, ObjCMethod, AttrList);
+  SemaRef.AddPragmaAttributes(SemaRef.TUScope, ObjCMethod);
+  SemaRef.ProcessAPINotes(ObjCMethod);
 
   // Add the method now.
   const ObjCMethodDecl *PrevMethod = nullptr;
@@ -4902,7 +4906,7 @@ Decl *Sema::ActOnMethodDeclaration(
     if (ObjCInterfaceDecl *IDecl = ImpDecl->getClassInterface()) {
       if (auto *IMD = IDecl->lookupMethod(ObjCMethod->getSelector(),
                                           ObjCMethod->isInstanceMethod())) {
-        mergeInterfaceMethodToImpl(*this, ObjCMethod, IMD);
+        mergeInterfaceMethodToImpl(SemaRef, ObjCMethod, IMD);
 
         // The Idecl->lookupMethod() above will find declarations for ObjCMethod
         // in one of these places:
@@ -4962,8 +4966,8 @@ Decl *Sema::ActOnMethodDeclaration(
             << ObjCMethod->getDeclName();
         }
       } else {
-        mergeObjCDirectMembers(*this, ClassDecl, ObjCMethod);
-        checkObjCDirectMethodClashes(*this, IDecl, ObjCMethod, ImpDecl);
+        mergeObjCDirectMembers(SemaRef, ClassDecl, ObjCMethod);
+        checkObjCDirectMethodClashes(SemaRef, IDecl, ObjCMethod, ImpDecl);
       }
 
       // Warn if a method declared in a protocol to which a category or
@@ -4979,12 +4983,12 @@ Decl *Sema::ActOnMethodDeclaration(
             auto OI = IMD->param_begin(), OE = IMD->param_end();
             auto NI = ObjCMethod->param_begin();
             for (; OI != OE; ++OI, ++NI)
-              diagnoseNoescape(*NI, *OI, C, P, *this);
+              diagnoseNoescape(*NI, *OI, C, P, SemaRef);
           }
     }
   } else {
     if (!isa<ObjCProtocolDecl>(ClassDecl)) {
-      mergeObjCDirectMembers(*this, ClassDecl, ObjCMethod);
+      mergeObjCDirectMembers(SemaRef, ClassDecl, ObjCMethod);
 
       ObjCInterfaceDecl *IDecl = dyn_cast<ObjCInterfaceDecl>(ClassDecl);
       if (!IDecl)
@@ -4993,7 +4997,7 @@ Decl *Sema::ActOnMethodDeclaration(
       // declaration by now, however for invalid code we'll keep parsing
       // but we won't find the primary interface and IDecl will be nil.
       if (IDecl)
-        checkObjCDirectMethodClashes(*this, IDecl, ObjCMethod);
+        checkObjCDirectMethodClashes(SemaRef, IDecl, ObjCMethod);
     }
 
     cast<DeclContext>(ClassDecl)->addDecl(ObjCMethod);
@@ -5022,8 +5026,8 @@ Decl *Sema::ActOnMethodDeclaration(
       CurrentClass = CatImpl->getClassInterface();
   }
 
-  ResultTypeCompatibilityKind RTC
-    = CheckRelatedResultTypeCompatibility(*this, ObjCMethod, CurrentClass);
+  ResultTypeCompatibilityKind RTC =
+      CheckRelatedResultTypeCompatibility(SemaRef, ObjCMethod, CurrentClass);
 
   CheckObjCMethodOverrides(ObjCMethod, CurrentClass, RTC);
 
@@ -5032,9 +5036,9 @@ Decl *Sema::ActOnMethodDeclaration(
     ARCError = CheckARCMethodDecl(ObjCMethod);
 
   // Infer the related result type when possible.
-  if (!ARCError && RTC == Sema::RTC_Compatible &&
+  if (!ARCError && RTC == SemaObjC::RTC_Compatible &&
       !ObjCMethod->hasRelatedResultType() &&
-      LangOpts.ObjCInferRelatedResultType) {
+      getLangOpts().ObjCInferRelatedResultType) {
     bool InferRelatedResultType = false;
     switch (ObjCMethod->getMethodFamily()) {
     case OMF_None:
@@ -5068,7 +5072,7 @@ Decl *Sema::ActOnMethodDeclaration(
 
   if (MethodDefinition &&
       Context.getTargetInfo().getTriple().getArch() == llvm::Triple::x86)
-    checkObjCMethodX86VectorTypes(*this, ObjCMethod);
+    checkObjCMethodX86VectorTypes(SemaRef, ObjCMethod);
 
   // + load method cannot have availability attributes. It get called on
   // startup, so it has to have the availability of the deployment target.
@@ -5084,20 +5088,21 @@ Decl *Sema::ActOnMethodDeclaration(
   // Insert the invisible arguments, self and _cmd!
   ObjCMethod->createImplicitParams(Context, ObjCMethod->getClassInterface());
 
-  ActOnDocumentableDecl(ObjCMethod);
+  SemaRef.ActOnDocumentableDecl(ObjCMethod);
 
   return ObjCMethod;
 }
 
-bool Sema::CheckObjCDeclScope(Decl *D) {
+bool SemaObjC::CheckObjCDeclScope(Decl *D) {
   // Following is also an error. But it is caused by a missing @end
   // and diagnostic is issued elsewhere.
-  if (isa<ObjCContainerDecl>(CurContext->getRedeclContext()))
+  if (isa<ObjCContainerDecl>(SemaRef.CurContext->getRedeclContext()))
     return false;
 
   // If we switched context to translation unit while we are still lexically in
   // an objc container, it means the parser missed emitting an error.
-  if (isa<TranslationUnitDecl>(getCurLexicalContext()->getRedeclContext()))
+  if (isa<TranslationUnitDecl>(
+          SemaRef.getCurLexicalContext()->getRedeclContext()))
     return false;
 
   Diag(D->getLocation(), diag::err_objc_decls_may_only_appear_in_global_scope);
@@ -5108,16 +5113,17 @@ bool Sema::CheckObjCDeclScope(Decl *D) {
 
 /// Called whenever \@defs(ClassName) is encountered in the source.  Inserts the
 /// instance variables of ClassName into Decls.
-void Sema::ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart,
-                     const IdentifierInfo *ClassName,
-                     SmallVectorImpl<Decl *> &Decls) {
+void SemaObjC::ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart,
+                         const IdentifierInfo *ClassName,
+                         SmallVectorImpl<Decl *> &Decls) {
+  ASTContext &Context = getASTContext();
   // Check that ClassName is a valid class
   ObjCInterfaceDecl *Class = getObjCInterfaceDecl(ClassName, DeclStart);
   if (!Class) {
     Diag(DeclStart, diag::err_undef_interface) << ClassName;
     return;
   }
-  if (LangOpts.ObjCRuntime.isNonFragile()) {
+  if (getLangOpts().ObjCRuntime.isNonFragile()) {
     Diag(DeclStart, diag::err_atdef_nonfragile_interface);
     return;
   }
@@ -5142,17 +5148,19 @@ void Sema::ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart,
        D != Decls.end(); ++D) {
     FieldDecl *FD = cast<FieldDecl>(*D);
     if (getLangOpts().CPlusPlus)
-      PushOnScopeChains(FD, S);
+      SemaRef.PushOnScopeChains(FD, S);
     else if (RecordDecl *Record = dyn_cast<RecordDecl>(TagD))
       Record->addDecl(FD);
   }
 }
 
 /// Build a type-check a new Objective-C exception variable declaration.
-VarDecl *Sema::BuildObjCExceptionDecl(TypeSourceInfo *TInfo, QualType T,
-                                      SourceLocation StartLoc,
-                                      SourceLocation IdLoc,
-                                      const IdentifierInfo *Id, bool Invalid) {
+VarDecl *SemaObjC::BuildObjCExceptionDecl(TypeSourceInfo *TInfo, QualType T,
+                                          SourceLocation StartLoc,
+                                          SourceLocation IdLoc,
+                                          const IdentifierInfo *Id,
+                                          bool Invalid) {
+  ASTContext &Context = getASTContext();
   // ISO/IEC TR 18037 S6.7.3: "The type of an object with automatic storage
   // duration shall not be qualified by an address-space qualifier."
   // Since all parameters have automatic store duration, they can not have
@@ -5181,8 +5189,8 @@ VarDecl *Sema::BuildObjCExceptionDecl(TypeSourceInfo *TInfo, QualType T,
     Diag(IdLoc, diag::err_catch_param_not_objc_type);
   }
 
-  VarDecl *New = VarDecl::Create(Context, CurContext, StartLoc, IdLoc, Id,
-                                 T, TInfo, SC_None);
+  VarDecl *New = VarDecl::Create(Context, SemaRef.CurContext, StartLoc, IdLoc,
+                                 Id, T, TInfo, SC_None);
   New->setExceptionVariable(true);
 
   // In ARC, infer 'retaining' for variables of retainable type.
@@ -5194,7 +5202,7 @@ VarDecl *Sema::BuildObjCExceptionDecl(TypeSourceInfo *TInfo, QualType T,
   return New;
 }
 
-Decl *Sema::ActOnObjCExceptionDecl(Scope *S, Declarator &D) {
+Decl *SemaObjC::ActOnObjCExceptionDecl(Scope *S, Declarator &D) {
   const DeclSpec &DS = D.getDeclSpec();
 
   // We allow the "register" storage class on exception variables because
@@ -5215,14 +5223,14 @@ Decl *Sema::ActOnObjCExceptionDecl(Scope *S, Declarator &D) {
      << DeclSpec::getSpecifierName(TSCS);
   D.getMutableDeclSpec().ClearStorageClassSpecs();
 
-  DiagnoseFunctionSpecifiers(D.getDeclSpec());
+  SemaRef.DiagnoseFunctionSpecifiers(D.getDeclSpec());
 
   // Check that there are no default arguments inside the type of this
   // exception object (C++ only).
   if (getLangOpts().CPlusPlus)
-    CheckExtraCXXDefaultArguments(D);
+    SemaRef.CheckExtraCXXDefaultArguments(D);
 
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
+  TypeSourceInfo *TInfo = SemaRef.GetTypeForDeclarator(D);
   QualType ExceptionType = TInfo->getType();
 
   VarDecl *New = BuildObjCExceptionDecl(TInfo, ExceptionType,
@@ -5241,9 +5249,9 @@ Decl *Sema::ActOnObjCExceptionDecl(Scope *S, Declarator &D) {
   // Add the parameter declaration into this scope.
   S->AddDecl(New);
   if (D.getIdentifier())
-    IdResolver.AddDecl(New);
+    SemaRef.IdResolver.AddDecl(New);
 
-  ProcessDeclAttributes(S, New, D);
+  SemaRef.ProcessDeclAttributes(S, New, D);
 
   if (New->hasAttr<BlocksAttr>())
     Diag(New->getLocation(), diag::err_block_on_nonlocal);
@@ -5252,8 +5260,9 @@ Decl *Sema::ActOnObjCExceptionDecl(Scope *S, Declarator &D) {
 
 /// CollectIvarsToConstructOrDestruct - Collect those ivars which require
 /// initialization.
-void Sema::CollectIvarsToConstructOrDestruct(ObjCInterfaceDecl *OI,
-                                SmallVectorImpl<ObjCIvarDecl*> &Ivars) {
+void SemaObjC::CollectIvarsToConstructOrDestruct(
+    ObjCInterfaceDecl *OI, SmallVectorImpl<ObjCIvarDecl *> &Ivars) {
+  ASTContext &Context = getASTContext();
   for (ObjCIvarDecl *Iv = OI->all_declared_ivar_begin(); Iv;
        Iv= Iv->getNextIvar()) {
     QualType QT = Context.getBaseElementType(Iv->getType());
@@ -5262,11 +5271,12 @@ void Sema::CollectIvarsToConstructOrDestruct(ObjCInterfaceDecl *OI,
   }
 }
 
-void Sema::DiagnoseUseOfUnimplementedSelectors() {
+void SemaObjC::DiagnoseUseOfUnimplementedSelectors() {
+  ASTContext &Context = getASTContext();
   // Load referenced selectors from the external source.
-  if (ExternalSource) {
+  if (SemaRef.ExternalSource) {
     SmallVector<std::pair<Selector, SourceLocation>, 4> Sels;
-    ExternalSource->ReadReferencedSelectors(Sels);
+    SemaRef.ExternalSource->ReadReferencedSelectors(Sels);
     for (unsigned I = 0, N = Sels.size(); I != N; ++I)
       ReferencedSelectors[Sels[I].first] = Sels[I].second;
   }
@@ -5286,8 +5296,8 @@ void Sema::DiagnoseUseOfUnimplementedSelectors() {
 }
 
 ObjCIvarDecl *
-Sema::GetIvarBackingPropertyAccessor(const ObjCMethodDecl *Method,
-                                     const ObjCPropertyDecl *&PDecl) const {
+SemaObjC::GetIvarBackingPropertyAccessor(const ObjCMethodDecl *Method,
+                                         const ObjCPropertyDecl *&PDecl) const {
   if (Method->isClassMethod())
     return nullptr;
   const ObjCInterfaceDecl *IDecl = Method->getClassInterface();
@@ -5311,51 +5321,51 @@ Sema::GetIvarBackingPropertyAccessor(const ObjCMethodDecl *Method,
 }
 
 namespace {
-  /// Used by Sema::DiagnoseUnusedBackingIvarInAccessor to check if a property
-  /// accessor references the backing ivar.
-  class UnusedBackingIvarChecker :
-      public RecursiveASTVisitor<UnusedBackingIvarChecker> {
-  public:
-    Sema &S;
-    const ObjCMethodDecl *Method;
-    const ObjCIvarDecl *IvarD;
-    bool AccessedIvar;
-    bool InvokedSelfMethod;
-
-    UnusedBackingIvarChecker(Sema &S, const ObjCMethodDecl *Method,
-                             const ObjCIvarDecl *IvarD)
-      : S(S), Method(Method), IvarD(IvarD),
-        AccessedIvar(false), InvokedSelfMethod(false) {
-      assert(IvarD);
-    }
-
-    bool VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
-      if (E->getDecl() == IvarD) {
-        AccessedIvar = true;
-        return false;
-      }
-      return true;
+/// Used by SemaObjC::DiagnoseUnusedBackingIvarInAccessor to check if a property
+/// accessor references the backing ivar.
+class UnusedBackingIvarChecker
+    : public RecursiveASTVisitor<UnusedBackingIvarChecker> {
+public:
+  Sema &S;
+  const ObjCMethodDecl *Method;
+  const ObjCIvarDecl *IvarD;
+  bool AccessedIvar;
+  bool InvokedSelfMethod;
+
+  UnusedBackingIvarChecker(Sema &S, const ObjCMethodDecl *Method,
+                           const ObjCIvarDecl *IvarD)
+      : S(S), Method(Method), IvarD(IvarD), AccessedIvar(false),
+        InvokedSelfMethod(false) {
+    assert(IvarD);
+  }
+
+  bool VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
+    if (E->getDecl() == IvarD) {
+      AccessedIvar = true;
+      return false;
     }
+    return true;
+  }
 
-    bool VisitObjCMessageExpr(ObjCMessageExpr *E) {
-      if (E->getReceiverKind() == ObjCMessageExpr::Instance &&
-          S.isSelfExpr(E->getInstanceReceiver(), Method)) {
-        InvokedSelfMethod = true;
-      }
-      return true;
+  bool VisitObjCMessageExpr(ObjCMessageExpr *E) {
+    if (E->getReceiverKind() == ObjCMessageExpr::Instance &&
+        S.ObjC().isSelfExpr(E->getInstanceReceiver(), Method)) {
+      InvokedSelfMethod = true;
     }
-  };
+    return true;
+  }
+};
 } // end anonymous namespace
 
-void Sema::DiagnoseUnusedBackingIvarInAccessor(Scope *S,
-                                          const ObjCImplementationDecl *ImplD) {
+void SemaObjC::DiagnoseUnusedBackingIvarInAccessor(
+    Scope *S, const ObjCImplementationDecl *ImplD) {
   if (S->hasUnrecoverableErrorOccurred())
     return;
 
   for (const auto *CurMethod : ImplD->instance_methods()) {
     unsigned DIAG = diag::warn_unused_property_backing_ivar;
     SourceLocation Loc = CurMethod->getLocation();
-    if (Diags.isIgnored(DIAG, Loc))
+    if (getDiagnostics().isIgnored(DIAG, Loc))
       continue;
 
     const ObjCPropertyDecl *PDecl;
@@ -5366,7 +5376,7 @@ void Sema::DiagnoseUnusedBackingIvarInAccessor(Scope *S,
     if (CurMethod->isSynthesizedAccessorStub())
       continue;
 
-    UnusedBackingIvarChecker Checker(*this, CurMethod, IV);
+    UnusedBackingIvarChecker Checker(SemaRef, CurMethod, IV);
     Checker.TraverseStmt(CurMethod->getBody());
     if (Checker.AccessedIvar)
       continue;
@@ -5381,3 +5391,299 @@ void Sema::DiagnoseUnusedBackingIvarInAccessor(Scope *S,
     }
   }
 }
+
+QualType SemaObjC::AdjustParameterTypeForObjCAutoRefCount(
+    QualType T, SourceLocation NameLoc, TypeSourceInfo *TSInfo) {
+  ASTContext &Context = getASTContext();
+  // In ARC, infer a lifetime qualifier for appropriate parameter types.
+  if (!getLangOpts().ObjCAutoRefCount ||
+      T.getObjCLifetime() != Qualifiers::OCL_None || !T->isObjCLifetimeType())
+    return T;
+
+  Qualifiers::ObjCLifetime Lifetime;
+
+  // Special cases for arrays:
+  //   - if it's const, use __unsafe_unretained
+  //   - otherwise, it's an error
+  if (T->isArrayType()) {
+    if (!T.isConstQualified()) {
+      if (SemaRef.DelayedDiagnostics.shouldDelayDiagnostics())
+        SemaRef.DelayedDiagnostics.add(
+            sema::DelayedDiagnostic::makeForbiddenType(
+                NameLoc, diag::err_arc_array_param_no_ownership, T, false));
+      else
+        Diag(NameLoc, diag::err_arc_array_param_no_ownership)
+            << TSInfo->getTypeLoc().getSourceRange();
+    }
+    Lifetime = Qualifiers::OCL_ExplicitNone;
+  } else {
+    Lifetime = T->getObjCARCImplicitLifetime();
+  }
+  T = Context.getLifetimeQualifiedType(T, Lifetime);
+
+  return T;
+}
+
+ObjCInterfaceDecl *SemaObjC::getObjCInterfaceDecl(const IdentifierInfo *&Id,
+                                                  SourceLocation IdLoc,
+                                                  bool DoTypoCorrection) {
+  // The third "scope" argument is 0 since we aren't enabling lazy built-in
+  // creation from this context.
+  NamedDecl *IDecl = SemaRef.LookupSingleName(SemaRef.TUScope, Id, IdLoc,
+                                              Sema::LookupOrdinaryName);
+
+  if (!IDecl && DoTypoCorrection) {
+    // Perform typo correction at the given location, but only if we
+    // find an Objective-C class name.
+    DeclFilterCCC<ObjCInterfaceDecl> CCC{};
+    if (TypoCorrection C = SemaRef.CorrectTypo(
+            DeclarationNameInfo(Id, IdLoc), Sema::LookupOrdinaryName,
+            SemaRef.TUScope, nullptr, CCC, Sema::CTK_ErrorRecovery)) {
+      SemaRef.diagnoseTypo(C, SemaRef.PDiag(diag::err_undef_interface_suggest)
+                                  << Id);
+      IDecl = C.getCorrectionDeclAs<ObjCInterfaceDecl>();
+      Id = IDecl->getIdentifier();
+    }
+  }
+  ObjCInterfaceDecl *Def = dyn_cast_or_null<ObjCInterfaceDecl>(IDecl);
+  // This routine must always return a class definition, if any.
+  if (Def && Def->getDefinition())
+    Def = Def->getDefinition();
+  return Def;
+}
+
+bool SemaObjC::inferObjCARCLifetime(ValueDecl *decl) {
+  ASTContext &Context = getASTContext();
+  QualType type = decl->getType();
+  Qualifiers::ObjCLifetime lifetime = type.getObjCLifetime();
+  if (lifetime == Qualifiers::OCL_Autoreleasing) {
+    // Various kinds of declaration aren't allowed to be __autoreleasing.
+    unsigned kind = -1U;
+    if (VarDecl *var = dyn_cast<VarDecl>(decl)) {
+      if (var->hasAttr<BlocksAttr>())
+        kind = 0; // __block
+      else if (!var->hasLocalStorage())
+        kind = 1; // global
+    } else if (isa<ObjCIvarDecl>(decl)) {
+      kind = 3; // ivar
+    } else if (isa<FieldDecl>(decl)) {
+      kind = 2; // field
+    }
+
+    if (kind != -1U) {
+      Diag(decl->getLocation(), diag::err_arc_autoreleasing_var) << kind;
+    }
+  } else if (lifetime == Qualifiers::OCL_None) {
+    // Try to infer lifetime.
+    if (!type->isObjCLifetimeType())
+      return false;
+
+    lifetime = type->getObjCARCImplicitLifetime();
+    type = Context.getLifetimeQualifiedType(type, lifetime);
+    decl->setType(type);
+  }
+
+  if (VarDecl *var = dyn_cast<VarDecl>(decl)) {
+    // Thread-local variables cannot have lifetime.
+    if (lifetime && lifetime != Qualifiers::OCL_ExplicitNone &&
+        var->getTLSKind()) {
+      Diag(var->getLocation(), diag::err_arc_thread_ownership)
+          << var->getType();
+      return true;
+    }
+  }
+
+  return false;
+}
+
+ObjCContainerDecl *SemaObjC::getObjCDeclContext() const {
+  return (dyn_cast_or_null<ObjCContainerDecl>(SemaRef.CurContext));
+}
+
+void SemaObjC::SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation) {
+  if (!getLangOpts().CPlusPlus)
+    return;
+  if (ObjCInterfaceDecl *OID = ObjCImplementation->getClassInterface()) {
+    ASTContext &Context = getASTContext();
+    SmallVector<ObjCIvarDecl *, 8> ivars;
+    CollectIvarsToConstructOrDestruct(OID, ivars);
+    if (ivars.empty())
+      return;
+    SmallVector<CXXCtorInitializer *, 32> AllToInit;
+    for (unsigned i = 0; i < ivars.size(); i++) {
+      FieldDecl *Field = ivars[i];
+      if (Field->isInvalidDecl())
+        continue;
+
+      CXXCtorInitializer *Member;
+      InitializedEntity InitEntity = InitializedEntity::InitializeMember(Field);
+      InitializationKind InitKind =
+          InitializationKind::CreateDefault(ObjCImplementation->getLocation());
+
+      InitializationSequence InitSeq(SemaRef, InitEntity, InitKind,
+                                     std::nullopt);
+      ExprResult MemberInit =
+          InitSeq.Perform(SemaRef, InitEntity, InitKind, std::nullopt);
+      MemberInit = SemaRef.MaybeCreateExprWithCleanups(MemberInit);
+      // Note, MemberInit could actually come back empty if no initialization
+      // is required (e.g., because it would call a trivial default constructor)
+      if (!MemberInit.get() || MemberInit.isInvalid())
+        continue;
+
+      Member = new (Context)
+          CXXCtorInitializer(Context, Field, SourceLocation(), SourceLocation(),
+                             MemberInit.getAs<Expr>(), SourceLocation());
+      AllToInit.push_back(Member);
+
+      // Be sure that the destructor is accessible and is marked as referenced.
+      if (const RecordType *RecordTy =
+              Context.getBaseElementType(Field->getType())
+                  ->getAs<RecordType>()) {
+        CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getDecl());
+        if (CXXDestructorDecl *Destructor = SemaRef.LookupDestructor(RD)) {
+          SemaRef.MarkFunctionReferenced(Field->getLocation(), Destructor);
+          SemaRef.CheckDestructorAccess(
+              Field->getLocation(), Destructor,
+              SemaRef.PDiag(diag::err_access_dtor_ivar)
+                  << Context.getBaseElementType(Field->getType()));
+        }
+      }
+    }
+    ObjCImplementation->setIvarInitializers(Context, AllToInit.data(),
+                                            AllToInit.size());
+  }
+}
+
+/// TranslateIvarVisibility - Translate visibility from a token ID to an
+///  AST enum value.
+static ObjCIvarDecl::AccessControl
+TranslateIvarVisibility(tok::ObjCKeywordKind ivarVisibility) {
+  switch (ivarVisibility) {
+  default:
+    llvm_unreachable("Unknown visitibility kind");
+  case tok::objc_private:
+    return ObjCIvarDecl::Private;
+  case tok::objc_public:
+    return ObjCIvarDecl::Public;
+  case tok::objc_protected:
+    return ObjCIvarDecl::Protected;
+  case tok::objc_package:
+    return ObjCIvarDecl::Package;
+  }
+}
+
+/// ActOnIvar - Each ivar field of an objective-c class is passed into this
+/// in order to create an IvarDecl object for it.
+Decl *SemaObjC::ActOnIvar(Scope *S, SourceLocation DeclStart, Declarator &D,
+                          Expr *BitWidth, tok::ObjCKeywordKind Visibility) {
+
+  const IdentifierInfo *II = D.getIdentifier();
+  SourceLocation Loc = DeclStart;
+  if (II)
+    Loc = D.getIdentifierLoc();
+
+  // FIXME: Unnamed fields can be handled in various different ways, for
+  // example, unnamed unions inject all members into the struct namespace!
+
+  TypeSourceInfo *TInfo = SemaRef.GetTypeForDeclarator(D);
+  QualType T = TInfo->getType();
+
+  if (BitWidth) {
+    // 6.7.2.1p3, 6.7.2.1p4
+    BitWidth =
+        SemaRef.VerifyBitField(Loc, II, T, /*IsMsStruct*/ false, BitWidth)
+            .get();
+    if (!BitWidth)
+      D.setInvalidType();
+  } else {
+    // Not a bitfield.
+
+    // validate II.
+  }
+  if (T->isReferenceType()) {
+    Diag(Loc, diag::err_ivar_reference_type);
+    D.setInvalidType();
+  }
+  // C99 6.7.2.1p8: A member of a structure or union may have any type other
+  // than a variably modified type.
+  else if (T->isVariablyModifiedType()) {
+    if (!SemaRef.tryToFixVariablyModifiedVarType(
+            TInfo, T, Loc, diag::err_typecheck_ivar_variable_size))
+      D.setInvalidType();
+  }
+
+  // Get the visibility (access control) for this ivar.
+  ObjCIvarDecl::AccessControl ac = Visibility != tok::objc_not_keyword
+                                       ? TranslateIvarVisibility(Visibility)
+                                       : ObjCIvarDecl::None;
+  // Must set ivar's DeclContext to its enclosing interface.
+  ObjCContainerDecl *EnclosingDecl =
+      cast<ObjCContainerDecl>(SemaRef.CurContext);
+  if (!EnclosingDecl || EnclosingDecl->isInvalidDecl())
+    return nullptr;
+  ObjCContainerDecl *EnclosingContext;
+  if (ObjCImplementationDecl *IMPDecl =
+          dyn_cast<ObjCImplementationDecl>(EnclosingDecl)) {
+    if (getLangOpts().ObjCRuntime.isFragile()) {
+      // Case of ivar declared in an implementation. Context is that of its
+      // class.
+      EnclosingContext = IMPDecl->getClassInterface();
+      assert(EnclosingContext && "Implementation has no class interface!");
+    } else
+      EnclosingContext = EnclosingDecl;
+  } else {
+    if (ObjCCategoryDecl *CDecl = dyn_cast<ObjCCategoryDecl>(EnclosingDecl)) {
+      if (getLangOpts().ObjCRuntime.isFragile() || !CDecl->IsClassExtension()) {
+        Diag(Loc, diag::err_misplaced_ivar) << CDecl->IsClassExtension();
+        return nullptr;
+      }
+    }
+    EnclosingContext = EnclosingDecl;
+  }
+
+  // Construct the decl.
+  ObjCIvarDecl *NewID =
+      ObjCIvarDecl::Create(getASTContext(), EnclosingContext, DeclStart, Loc,
+                           II, T, TInfo, ac, BitWidth);
+
+  if (T->containsErrors())
+    NewID->setInvalidDecl();
+
+  if (II) {
+    NamedDecl *PrevDecl =
+        SemaRef.LookupSingleName(S, II, Loc, Sema::LookupMemberName,
+                                 RedeclarationKind::ForVisibleRedeclaration);
+    if (PrevDecl && SemaRef.isDeclInScope(PrevDecl, EnclosingContext, S) &&
+        !isa<TagDecl>(PrevDecl)) {
+      Diag(Loc, diag::err_duplicate_member) << II;
+      Diag(PrevDecl->getLocation(), diag::note_previous_declaration);
+      NewID->setInvalidDecl();
+    }
+  }
+
+  // Process attributes attached to the ivar.
+  SemaRef.ProcessDeclAttributes(S, NewID, D);
+
+  if (D.isInvalidType())
+    NewID->setInvalidDecl();
+
+  // In ARC, infer 'retaining' for ivars of retainable type.
+  if (getLangOpts().ObjCAutoRefCount && inferObjCARCLifetime(NewID))
+    NewID->setInvalidDecl();
+
+  if (D.getDeclSpec().isModulePrivateSpecified())
+    NewID->setModulePrivate();
+
+  if (II) {
+    // FIXME: When interfaces are DeclContexts, we'll need to add
+    // these to the interface.
+    S->AddDecl(NewID);
+    SemaRef.IdResolver.AddDecl(NewID);
+  }
+
+  if (getLangOpts().ObjCRuntime.isNonFragile() && !NewID->isInvalidDecl() &&
+      isa<ObjCInterfaceDecl>(EnclosingDecl))
+    Diag(Loc, diag::warn_ivars_in_interface);
+
+  return NewID;
+}
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index bb4b116fd73c..e6c3fa51d54d 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -52,6 +52,7 @@
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaFixItUtils.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/STLExtras.h"
@@ -109,7 +110,7 @@ static void DiagnoseUnusedOfDecl(Sema &S, NamedDecl *D, SourceLocation Loc) {
     // should diagnose them.
     if (A->getSemanticSpelling() != UnusedAttr::CXX11_maybe_unused &&
         A->getSemanticSpelling() != UnusedAttr::C23_maybe_unused) {
-      const Decl *DC = cast_or_null<Decl>(S.getCurObjCLexicalContext());
+      const Decl *DC = cast_or_null<Decl>(S.ObjC().getCurObjCLexicalContext());
       if (DC && !DC->hasAttr<UnusedAttr>())
         S.Diag(Loc, diag::warn_used_but_marked_unused) << D;
     }
@@ -1051,9 +1052,9 @@ ExprResult Sema::DefaultVariadicArgumentPromotion(Expr *E, VariadicCallType CT,
     if (PlaceholderTy->getKind() == BuiltinType::ARCUnbridgedCast &&
         (CT == VariadicMethod ||
          (FDecl && FDecl->hasAttr<CFAuditedTransferAttr>()))) {
-      E = stripARCUnbridgedCast(E);
+      E = ObjC().stripARCUnbridgedCast(E);
 
-    // Otherwise, do normal placeholder checking.
+      // Otherwise, do normal placeholder checking.
     } else {
       ExprResult ExprRes = CheckPlaceholderExpr(E);
       if (ExprRes.isInvalid())
@@ -2797,7 +2798,7 @@ Sema::ActOnIdExpression(Scope *S, CXXScopeSpec &SS,
     // If this reference is in an Objective-C method, then we need to do
     // some special Objective-C lookup, too.
     if (IvarLookupFollowUp) {
-      ExprResult E(LookupInObjCMethod(R, S, II, true));
+      ExprResult E(ObjC().LookupInObjCMethod(R, S, II, true));
       if (E.isInvalid())
         return ExprError();
 
@@ -2882,7 +2883,7 @@ Sema::ActOnIdExpression(Scope *S, CXXScopeSpec &SS,
     // reference the ivar.
     if (ObjCIvarDecl *Ivar = R.getAsSingle<ObjCIvarDecl>()) {
       R.clear();
-      ExprResult E(LookupInObjCMethod(R, S, Ivar->getIdentifier()));
+      ExprResult E(ObjC().LookupInObjCMethod(R, S, Ivar->getIdentifier()));
       // In a hopelessly buggy code, Objective-C instance variable
       // lookup fails and no expression will be built to reference it.
       if (!E.isInvalid() && !E.get())
@@ -3028,166 +3029,6 @@ ExprResult Sema::BuildQualifiedDeclarationNameExpr(
   return BuildDeclarationNameExpr(SS, R, /* ADL */ false);
 }
 
-/// The parser has read a name in, and Sema has detected that we're currently
-/// inside an ObjC method. Perform some additional checks and determine if we
-/// should form a reference to an ivar.
-///
-/// Ideally, most of this would be done by lookup, but there's
-/// actually quite a lot of extra work involved.
-DeclResult Sema::LookupIvarInObjCMethod(LookupResult &Lookup, Scope *S,
-                                        IdentifierInfo *II) {
-  SourceLocation Loc = Lookup.getNameLoc();
-  ObjCMethodDecl *CurMethod = getCurMethodDecl();
-
-  // Check for error condition which is already reported.
-  if (!CurMethod)
-    return DeclResult(true);
-
-  // There are two cases to handle here.  1) scoped lookup could have failed,
-  // in which case we should look for an ivar.  2) scoped lookup could have
-  // found a decl, but that decl is outside the current instance method (i.e.
-  // a global variable).  In these two cases, we do a lookup for an ivar with
-  // this name, if the lookup sucedes, we replace it our current decl.
-
-  // If we're in a class method, we don't normally want to look for
-  // ivars.  But if we don't find anything else, and there's an
-  // ivar, that's an error.
-  bool IsClassMethod = CurMethod->isClassMethod();
-
-  bool LookForIvars;
-  if (Lookup.empty())
-    LookForIvars = true;
-  else if (IsClassMethod)
-    LookForIvars = false;
-  else
-    LookForIvars = (Lookup.isSingleResult() &&
-                    Lookup.getFoundDecl()->isDefinedOutsideFunctionOrMethod());
-  ObjCInterfaceDecl *IFace = nullptr;
-  if (LookForIvars) {
-    IFace = CurMethod->getClassInterface();
-    ObjCInterfaceDecl *ClassDeclared;
-    ObjCIvarDecl *IV = nullptr;
-    if (IFace && (IV = IFace->lookupInstanceVariable(II, ClassDeclared))) {
-      // Diagnose using an ivar in a class method.
-      if (IsClassMethod) {
-        Diag(Loc, diag::err_ivar_use_in_class_method) << IV->getDeclName();
-        return DeclResult(true);
-      }
-
-      // Diagnose the use of an ivar outside of the declaring class.
-      if (IV->getAccessControl() == ObjCIvarDecl::Private &&
-          !declaresSameEntity(ClassDeclared, IFace) &&
-          !getLangOpts().DebuggerSupport)
-        Diag(Loc, diag::err_private_ivar_access) << IV->getDeclName();
-
-      // Success.
-      return IV;
-    }
-  } else if (CurMethod->isInstanceMethod()) {
-    // We should warn if a local variable hides an ivar.
-    if (ObjCInterfaceDecl *IFace = CurMethod->getClassInterface()) {
-      ObjCInterfaceDecl *ClassDeclared;
-      if (ObjCIvarDecl *IV = IFace->lookupInstanceVariable(II, ClassDeclared)) {
-        if (IV->getAccessControl() != ObjCIvarDecl::Private ||
-            declaresSameEntity(IFace, ClassDeclared))
-          Diag(Loc, diag::warn_ivar_use_hidden) << IV->getDeclName();
-      }
-    }
-  } else if (Lookup.isSingleResult() &&
-             Lookup.getFoundDecl()->isDefinedOutsideFunctionOrMethod()) {
-    // If accessing a stand-alone ivar in a class method, this is an error.
-    if (const ObjCIvarDecl *IV =
-            dyn_cast<ObjCIvarDecl>(Lookup.getFoundDecl())) {
-      Diag(Loc, diag::err_ivar_use_in_class_method) << IV->getDeclName();
-      return DeclResult(true);
-    }
-  }
-
-  // Didn't encounter an error, didn't find an ivar.
-  return DeclResult(false);
-}
-
-ExprResult Sema::BuildIvarRefExpr(Scope *S, SourceLocation Loc,
-                                  ObjCIvarDecl *IV) {
-  ObjCMethodDecl *CurMethod = getCurMethodDecl();
-  assert(CurMethod && CurMethod->isInstanceMethod() &&
-         "should not reference ivar from this context");
-
-  ObjCInterfaceDecl *IFace = CurMethod->getClassInterface();
-  assert(IFace && "should not reference ivar from this context");
-
-  // If we're referencing an invalid decl, just return this as a silent
-  // error node.  The error diagnostic was already emitted on the decl.
-  if (IV->isInvalidDecl())
-    return ExprError();
-
-  // Check if referencing a field with __attribute__((deprecated)).
-  if (DiagnoseUseOfDecl(IV, Loc))
-    return ExprError();
-
-  // FIXME: This should use a new expr for a direct reference, don't
-  // turn this into Self->ivar, just return a BareIVarExpr or something.
-  IdentifierInfo &II = Context.Idents.get("self");
-  UnqualifiedId SelfName;
-  SelfName.setImplicitSelfParam(&II);
-  CXXScopeSpec SelfScopeSpec;
-  SourceLocation TemplateKWLoc;
-  ExprResult SelfExpr =
-      ActOnIdExpression(S, SelfScopeSpec, TemplateKWLoc, SelfName,
-                        /*HasTrailingLParen=*/false,
-                        /*IsAddressOfOperand=*/false);
-  if (SelfExpr.isInvalid())
-    return ExprError();
-
-  SelfExpr = DefaultLvalueConversion(SelfExpr.get());
-  if (SelfExpr.isInvalid())
-    return ExprError();
-
-  MarkAnyDeclReferenced(Loc, IV, true);
-
-  ObjCMethodFamily MF = CurMethod->getMethodFamily();
-  if (MF != OMF_init && MF != OMF_dealloc && MF != OMF_finalize &&
-      !IvarBacksCurrentMethodAccessor(IFace, CurMethod, IV))
-    Diag(Loc, diag::warn_direct_ivar_access) << IV->getDeclName();
-
-  ObjCIvarRefExpr *Result = new (Context)
-      ObjCIvarRefExpr(IV, IV->getUsageType(SelfExpr.get()->getType()), Loc,
-                      IV->getLocation(), SelfExpr.get(), true, true);
-
-  if (IV->getType().getObjCLifetime() == Qualifiers::OCL_Weak) {
-    if (!isUnevaluatedContext() &&
-        !Diags.isIgnored(diag::warn_arc_repeated_use_of_weak, Loc))
-      getCurFunction()->recordUseOfWeak(Result);
-  }
-  if (getLangOpts().ObjCAutoRefCount && !isUnevaluatedContext())
-    if (const BlockDecl *BD = CurContext->getInnermostBlockDecl())
-      ImplicitlyRetainedSelfLocs.push_back({Loc, BD});
-
-  return Result;
-}
-
-/// The parser has read a name in, and Sema has detected that we're currently
-/// inside an ObjC method. Perform some additional checks and determine if we
-/// should form a reference to an ivar. If so, build an expression referencing
-/// that ivar.
-ExprResult
-Sema::LookupInObjCMethod(LookupResult &Lookup, Scope *S,
-                         IdentifierInfo *II, bool AllowBuiltinCreation) {
-  // FIXME: Integrate this lookup step into LookupParsedName.
-  DeclResult Ivar = LookupIvarInObjCMethod(Lookup, S, II);
-  if (Ivar.isInvalid())
-    return ExprError();
-  if (Ivar.isUsable())
-    return BuildIvarRefExpr(S, Lookup.getNameLoc(),
-                            cast<ObjCIvarDecl>(Ivar.get()));
-
-  if (Lookup.empty() && II && AllowBuiltinCreation)
-    LookupBuiltin(Lookup);
-
-  // Sentinel value saying that we didn't do anything special.
-  return ExprResult(false);
-}
-
 /// Cast a base object to a member's actual type.
 ///
 /// There are two relevant checks:
@@ -5419,8 +5260,8 @@ Sema::CreateBuiltinArraySubscriptExpr(Expr *Base, SourceLocation LLoc,
     // Use custom logic if this should be the pseudo-object subscript
     // expression.
     if (!LangOpts.isSubscriptPointerArithmetic())
-      return BuildObjCSubscriptExpression(RLoc, BaseExpr, IndexExpr, nullptr,
-                                          nullptr);
+      return ObjC().BuildObjCSubscriptExpression(RLoc, BaseExpr, IndexExpr,
+                                                 nullptr, nullptr);
 
     ResultType = PTy->getPointeeType();
   } else if (const PointerType *PTy = RHSTy->getAs<PointerType>()) {
@@ -6188,7 +6029,7 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc, FunctionDecl *FDecl,
       if (Arg->getType() == Context.ARCUnbridgedCastTy &&
           FDecl && FDecl->hasAttr<CFAuditedTransferAttr>() &&
           (!Param || !Param->hasAttr<CFConsumedAttr>()))
-        Arg = stripARCUnbridgedCast(Arg);
+        Arg = ObjC().stripARCUnbridgedCast(Arg);
       else if (getLangOpts().ObjCAutoRefCount &&
                FDecl && FDecl->hasAttr<CFAuditedTransferAttr>() &&
                (!Param || !Param->hasAttr<CFConsumedAttr>()))
@@ -7500,21 +7341,6 @@ void Sema::maybeExtendBlockObject(ExprResult &E) {
   Cleanup.setExprNeedsCleanups(true);
 }
 
-/// Prepare a conversion of the given expression to an ObjC object
-/// pointer type.
-CastKind Sema::PrepareCastToObjCObjectPointer(ExprResult &E) {
-  QualType type = E.get()->getType();
-  if (type->isObjCObjectPointerType()) {
-    return CK_BitCast;
-  } else if (type->isBlockPointerType()) {
-    maybeExtendBlockObject(E);
-    return CK_BlockPointerToObjCPointerCast;
-  } else {
-    assert(type->isPointerType());
-    return CK_CPointerToObjCPointerCast;
-  }
-}
-
 /// Prepares for a scalar cast, performing all the necessary stages
 /// except the final cast and returning the kind required.
 CastKind Sema::PrepareScalarCast(ExprResult &Src, QualType DestTy) {
@@ -8065,9 +7891,9 @@ Sema::ActOnCastExpr(Scope *S, SourceLocation LParenLoc,
   if (getLangOpts().CPlusPlus && !castType->isVoidType())
     Diag(LParenLoc, diag::warn_old_style_cast) << CastExpr->getSourceRange();
 
-  CheckTollFreeBridgeCast(castType, CastExpr);
+  ObjC().CheckTollFreeBridgeCast(castType, CastExpr);
 
-  CheckObjCBridgeRelatedCast(castType, CastExpr);
+  ObjC().CheckObjCBridgeRelatedCast(castType, CastExpr);
 
   DiscardMisalignedMemberAddress(castType.getTypePtr(), CastExpr);
 
@@ -8817,8 +8643,8 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
   if (!checkConditionalNullPointer(*this, LHS, RHSTy)) return RHSTy;
 
   // All objective-c pointer type analysis is done here.
-  QualType compositeType = FindCompositeObjCPointerType(LHS, RHS,
-                                                        QuestionLoc);
+  QualType compositeType =
+      ObjC().FindCompositeObjCPointerType(LHS, RHS, QuestionLoc);
   if (LHS.isInvalid() || RHS.isInvalid())
     return QualType();
   if (!compositeType.isNull())
@@ -8862,148 +8688,6 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
   return QualType();
 }
 
-/// FindCompositeObjCPointerType - Helper method to find composite type of
-/// two objective-c pointer types of the two input expressions.
-QualType Sema::FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS,
-                                            SourceLocation QuestionLoc) {
-  QualType LHSTy = LHS.get()->getType();
-  QualType RHSTy = RHS.get()->getType();
-
-  // Handle things like Class and struct objc_class*.  Here we case the result
-  // to the pseudo-builtin, because that will be implicitly cast back to the
-  // redefinition type if an attempt is made to access its fields.
-  if (LHSTy->isObjCClassType() &&
-      (Context.hasSameType(RHSTy, Context.getObjCClassRedefinitionType()))) {
-    RHS = ImpCastExprToType(RHS.get(), LHSTy, CK_CPointerToObjCPointerCast);
-    return LHSTy;
-  }
-  if (RHSTy->isObjCClassType() &&
-      (Context.hasSameType(LHSTy, Context.getObjCClassRedefinitionType()))) {
-    LHS = ImpCastExprToType(LHS.get(), RHSTy, CK_CPointerToObjCPointerCast);
-    return RHSTy;
-  }
-  // And the same for struct objc_object* / id
-  if (LHSTy->isObjCIdType() &&
-      (Context.hasSameType(RHSTy, Context.getObjCIdRedefinitionType()))) {
-    RHS = ImpCastExprToType(RHS.get(), LHSTy, CK_CPointerToObjCPointerCast);
-    return LHSTy;
-  }
-  if (RHSTy->isObjCIdType() &&
-      (Context.hasSameType(LHSTy, Context.getObjCIdRedefinitionType()))) {
-    LHS = ImpCastExprToType(LHS.get(), RHSTy, CK_CPointerToObjCPointerCast);
-    return RHSTy;
-  }
-  // And the same for struct objc_selector* / SEL
-  if (Context.isObjCSelType(LHSTy) &&
-      (Context.hasSameType(RHSTy, Context.getObjCSelRedefinitionType()))) {
-    RHS = ImpCastExprToType(RHS.get(), LHSTy, CK_BitCast);
-    return LHSTy;
-  }
-  if (Context.isObjCSelType(RHSTy) &&
-      (Context.hasSameType(LHSTy, Context.getObjCSelRedefinitionType()))) {
-    LHS = ImpCastExprToType(LHS.get(), RHSTy, CK_BitCast);
-    return RHSTy;
-  }
-  // Check constraints for Objective-C object pointers types.
-  if (LHSTy->isObjCObjectPointerType() && RHSTy->isObjCObjectPointerType()) {
-
-    if (Context.getCanonicalType(LHSTy) == Context.getCanonicalType(RHSTy)) {
-      // Two identical object pointer types are always compatible.
-      return LHSTy;
-    }
-    const ObjCObjectPointerType *LHSOPT = LHSTy->castAs<ObjCObjectPointerType>();
-    const ObjCObjectPointerType *RHSOPT = RHSTy->castAs<ObjCObjectPointerType>();
-    QualType compositeType = LHSTy;
-
-    // If both operands are interfaces and either operand can be
-    // assigned to the other, use that type as the composite
-    // type. This allows
-    //   xxx ? (A*) a : (B*) b
-    // where B is a subclass of A.
-    //
-    // Additionally, as for assignment, if either type is 'id'
-    // allow silent coercion. Finally, if the types are
-    // incompatible then make sure to use 'id' as the composite
-    // type so the result is acceptable for sending messages to.
-
-    // FIXME: Consider unifying with 'areComparableObjCPointerTypes'.
-    // It could return the composite type.
-    if (!(compositeType =
-          Context.areCommonBaseCompatible(LHSOPT, RHSOPT)).isNull()) {
-      // Nothing more to do.
-    } else if (Context.canAssignObjCInterfaces(LHSOPT, RHSOPT)) {
-      compositeType = RHSOPT->isObjCBuiltinType() ? RHSTy : LHSTy;
-    } else if (Context.canAssignObjCInterfaces(RHSOPT, LHSOPT)) {
-      compositeType = LHSOPT->isObjCBuiltinType() ? LHSTy : RHSTy;
-    } else if ((LHSOPT->isObjCQualifiedIdType() ||
-                RHSOPT->isObjCQualifiedIdType()) &&
-               Context.ObjCQualifiedIdTypesAreCompatible(LHSOPT, RHSOPT,
-                                                         true)) {
-      // Need to handle "id<xx>" explicitly.
-      // GCC allows qualified id and any Objective-C type to devolve to
-      // id. Currently localizing to here until clear this should be
-      // part of ObjCQualifiedIdTypesAreCompatible.
-      compositeType = Context.getObjCIdType();
-    } else if (LHSTy->isObjCIdType() || RHSTy->isObjCIdType()) {
-      compositeType = Context.getObjCIdType();
-    } else {
-      Diag(QuestionLoc, diag::ext_typecheck_cond_incompatible_operands)
-      << LHSTy << RHSTy
-      << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
-      QualType incompatTy = Context.getObjCIdType();
-      LHS = ImpCastExprToType(LHS.get(), incompatTy, CK_BitCast);
-      RHS = ImpCastExprToType(RHS.get(), incompatTy, CK_BitCast);
-      return incompatTy;
-    }
-    // The object pointer types are compatible.
-    LHS = ImpCastExprToType(LHS.get(), compositeType, CK_BitCast);
-    RHS = ImpCastExprToType(RHS.get(), compositeType, CK_BitCast);
-    return compositeType;
-  }
-  // Check Objective-C object pointer types and 'void *'
-  if (LHSTy->isVoidPointerType() && RHSTy->isObjCObjectPointerType()) {
-    if (getLangOpts().ObjCAutoRefCount) {
-      // ARC forbids the implicit conversion of object pointers to 'void *',
-      // so these types are not compatible.
-      Diag(QuestionLoc, diag::err_cond_voidptr_arc) << LHSTy << RHSTy
-          << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
-      LHS = RHS = true;
-      return QualType();
-    }
-    QualType lhptee = LHSTy->castAs<PointerType>()->getPointeeType();
-    QualType rhptee = RHSTy->castAs<ObjCObjectPointerType>()->getPointeeType();
-    QualType destPointee
-    = Context.getQualifiedType(lhptee, rhptee.getQualifiers());
-    QualType destType = Context.getPointerType(destPointee);
-    // Add qualifiers if necessary.
-    LHS = ImpCastExprToType(LHS.get(), destType, CK_NoOp);
-    // Promote to void*.
-    RHS = ImpCastExprToType(RHS.get(), destType, CK_BitCast);
-    return destType;
-  }
-  if (LHSTy->isObjCObjectPointerType() && RHSTy->isVoidPointerType()) {
-    if (getLangOpts().ObjCAutoRefCount) {
-      // ARC forbids the implicit conversion of object pointers to 'void *',
-      // so these types are not compatible.
-      Diag(QuestionLoc, diag::err_cond_voidptr_arc) << LHSTy << RHSTy
-          << LHS.get()->getSourceRange() << RHS.get()->getSourceRange();
-      LHS = RHS = true;
-      return QualType();
-    }
-    QualType lhptee = LHSTy->castAs<ObjCObjectPointerType>()->getPointeeType();
-    QualType rhptee = RHSTy->castAs<PointerType>()->getPointeeType();
-    QualType destPointee
-    = Context.getQualifiedType(rhptee, lhptee.getQualifiers());
-    QualType destType = Context.getPointerType(destPointee);
-    // Add qualifiers if necessary.
-    RHS = ImpCastExprToType(RHS.get(), destType, CK_NoOp);
-    // Promote to void*.
-    LHS = ImpCastExprToType(LHS.get(), destType, CK_BitCast);
-    return destType;
-  }
-  return QualType();
-}
-
 /// SuggestParentheses - Emit a note with a fixit hint that wraps
 /// ParenRange in parentheses.
 static void SuggestParentheses(Sema &Self, SourceLocation Loc,
@@ -9868,7 +9552,7 @@ Sema::CheckAssignmentConstraints(QualType LHSType, ExprResult &RHS,
         checkObjCPointerTypesForAssignment(*this, LHSType, RHSType);
       if (getLangOpts().allowsNonTrivialObjCLifetimeQualifiers() &&
           result == Compatible &&
-          !CheckObjCARCUnavailableWeakConversion(OrigLHSType, RHSType))
+          !ObjC().CheckObjCARCUnavailableWeakConversion(OrigLHSType, RHSType))
         result = IncompatibleObjCWeakRef;
       return result;
     }
@@ -10094,7 +9778,7 @@ Sema::CheckSingleAssignmentConstraints(QualType LHSType, ExprResult &CallerRHS,
         return Incompatible;
       Sema::AssignConvertType result = Compatible;
       if (getLangOpts().allowsNonTrivialObjCLifetimeQualifiers() &&
-          !CheckObjCARCUnavailableWeakConversion(LHSType, RHSType))
+          !ObjC().CheckObjCARCUnavailableWeakConversion(LHSType, RHSType))
         result = IncompatibleObjCWeakRef;
       return result;
     }
@@ -10200,16 +9884,16 @@ Sema::CheckSingleAssignmentConstraints(QualType LHSType, ExprResult &CallerRHS,
     // diagnostics and just checking for errors, e.g., during overload
     // resolution, return Incompatible to indicate the failure.
     if (getLangOpts().allowsNonTrivialObjCLifetimeQualifiers() &&
-        CheckObjCConversion(SourceRange(), Ty, E,
-                            CheckedConversionKind::Implicit, Diagnose,
-                            DiagnoseCFAudited) != ACR_okay) {
+        ObjC().CheckObjCConversion(SourceRange(), Ty, E,
+                                   CheckedConversionKind::Implicit, Diagnose,
+                                   DiagnoseCFAudited) != SemaObjC::ACR_okay) {
       if (!Diagnose)
         return Incompatible;
     }
     if (getLangOpts().ObjC &&
-        (CheckObjCBridgeRelatedConversions(E->getBeginLoc(), LHSType,
-                                           E->getType(), E, Diagnose) ||
-         CheckConversionToObjCLiteral(LHSType, E, Diagnose))) {
+        (ObjC().CheckObjCBridgeRelatedConversions(E->getBeginLoc(), LHSType,
+                                                  E->getType(), E, Diagnose) ||
+         ObjC().CheckConversionToObjCLiteral(LHSType, E, Diagnose))) {
       if (!Diagnose)
         return Incompatible;
       // Replace the expression with a corrected version and continue so we
@@ -12026,19 +11710,20 @@ static bool hasIsEqualMethod(Sema &S, const Expr *LHS, const Expr *RHS) {
     return false;
 
   // Try to find the -isEqual: method.
-  Selector IsEqualSel = S.NSAPIObj->getIsEqualSelector();
-  ObjCMethodDecl *Method = S.LookupMethodInObjectType(IsEqualSel,
-                                                      InterfaceType,
-                                                      /*IsInstance=*/true);
+  Selector IsEqualSel = S.ObjC().NSAPIObj->getIsEqualSelector();
+  ObjCMethodDecl *Method =
+      S.ObjC().LookupMethodInObjectType(IsEqualSel, InterfaceType,
+                                        /*IsInstance=*/true);
   if (!Method) {
     if (Type->isObjCIdType()) {
       // For 'id', just check the global pool.
-      Method = S.LookupInstanceMethodInGlobalPool(IsEqualSel, SourceRange(),
-                                                  /*receiverId=*/true);
+      Method =
+          S.ObjC().LookupInstanceMethodInGlobalPool(IsEqualSel, SourceRange(),
+                                                    /*receiverId=*/true);
     } else {
       // Check protocols.
-      Method = S.LookupMethodInQualifiedType(IsEqualSel, Type,
-                                             /*IsInstance=*/true);
+      Method = S.ObjC().LookupMethodInQualifiedType(IsEqualSel, Type,
+                                                    /*IsInstance=*/true);
     }
   }
 
@@ -12056,48 +11741,6 @@ static bool hasIsEqualMethod(Sema &S, const Expr *LHS, const Expr *RHS) {
   return true;
 }
 
-Sema::ObjCLiteralKind Sema::CheckLiteralKind(Expr *FromE) {
-  FromE = FromE->IgnoreParenImpCasts();
-  switch (FromE->getStmtClass()) {
-    default:
-      break;
-    case Stmt::ObjCStringLiteralClass:
-      // "string literal"
-      return LK_String;
-    case Stmt::ObjCArrayLiteralClass:
-      // "array literal"
-      return LK_Array;
-    case Stmt::ObjCDictionaryLiteralClass:
-      // "dictionary literal"
-      return LK_Dictionary;
-    case Stmt::BlockExprClass:
-      return LK_Block;
-    case Stmt::ObjCBoxedExprClass: {
-      Expr *Inner = cast<ObjCBoxedExpr>(FromE)->getSubExpr()->IgnoreParens();
-      switch (Inner->getStmtClass()) {
-        case Stmt::IntegerLiteralClass:
-        case Stmt::FloatingLiteralClass:
-        case Stmt::CharacterLiteralClass:
-        case Stmt::ObjCBoolLiteralExprClass:
-        case Stmt::CXXBoolLiteralExprClass:
-          // "numeric literal"
-          return LK_Numeric;
-        case Stmt::ImplicitCastExprClass: {
-          CastKind CK = cast<CastExpr>(Inner)->getCastKind();
-          // Boolean literals can be represented by implicit casts.
-          if (CK == CK_IntegralToBoolean || CK == CK_IntegralCast)
-            return LK_Numeric;
-          break;
-        }
-        default:
-          break;
-      }
-      return LK_Boxed;
-    }
-  }
-  return LK_None;
-}
-
 static void diagnoseObjCLiteralComparison(Sema &S, SourceLocation Loc,
                                           ExprResult &LHS, ExprResult &RHS,
                                           BinaryOperator::Opcode Opc){
@@ -12120,13 +11763,13 @@ static void diagnoseObjCLiteralComparison(Sema &S, SourceLocation Loc,
   // This should be kept in sync with warn_objc_literal_comparison.
   // LK_String should always be after the other literals, since it has its own
   // warning flag.
-  Sema::ObjCLiteralKind LiteralKind = S.CheckLiteralKind(Literal);
-  assert(LiteralKind != Sema::LK_Block);
-  if (LiteralKind == Sema::LK_None) {
+  SemaObjC::ObjCLiteralKind LiteralKind = S.ObjC().CheckLiteralKind(Literal);
+  assert(LiteralKind != SemaObjC::LK_Block);
+  if (LiteralKind == SemaObjC::LK_None) {
     llvm_unreachable("Unknown Objective-C object literal kind");
   }
 
-  if (LiteralKind == Sema::LK_String)
+  if (LiteralKind == SemaObjC::LK_String)
     S.Diag(Loc, diag::warn_objc_string_literal_comparison)
       << Literal->getSourceRange();
   else
@@ -12922,18 +12565,18 @@ QualType Sema::CheckCompareOperands(ExprResult &LHS, ExprResult &RHS,
       if (LHSIsNull && !RHSIsNull) {
         Expr *E = LHS.get();
         if (getLangOpts().ObjCAutoRefCount)
-          CheckObjCConversion(SourceRange(), RHSType, E,
-                              CheckedConversionKind::Implicit);
+          ObjC().CheckObjCConversion(SourceRange(), RHSType, E,
+                                     CheckedConversionKind::Implicit);
         LHS = ImpCastExprToType(E, RHSType,
                                 RPT ? CK_BitCast :CK_CPointerToObjCPointerCast);
       }
       else {
         Expr *E = RHS.get();
         if (getLangOpts().ObjCAutoRefCount)
-          CheckObjCConversion(SourceRange(), LHSType, E,
-                              CheckedConversionKind::Implicit,
-                              /*Diagnose=*/true,
-                              /*DiagnoseCFAudited=*/false, Opc);
+          ObjC().CheckObjCConversion(SourceRange(), LHSType, E,
+                                     CheckedConversionKind::Implicit,
+                                     /*Diagnose=*/true,
+                                     /*DiagnoseCFAudited=*/false, Opc);
         RHS = ImpCastExprToType(E, LHSType,
                                 LPT ? CK_BitCast :CK_CPointerToObjCPointerCast);
       }
@@ -14144,7 +13787,7 @@ QualType Sema::CheckAssignmentOperands(Expr *LHSExpr, ExprResult &RHS,
         const Expr *InnerLHS = LHSExpr->IgnoreParenCasts();
         const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(InnerLHS);
         if (!DRE || DRE->getDecl()->hasAttr<BlocksAttr>())
-          checkRetainCycles(LHSExpr, RHS.get());
+          ObjC().checkRetainCycles(LHSExpr, RHS.get());
       }
 
       if (LHSType.getObjCLifetime() == Qualifiers::OCL_Strong ||
@@ -17091,61 +16734,6 @@ ExprResult Sema::BuildSourceLocExpr(SourceLocIdentKind Kind, QualType ResultTy,
       SourceLocExpr(Context, Kind, ResultTy, BuiltinLoc, RPLoc, ParentContext);
 }
 
-bool Sema::CheckConversionToObjCLiteral(QualType DstType, Expr *&Exp,
-                                        bool Diagnose) {
-  if (!getLangOpts().ObjC)
-    return false;
-
-  const ObjCObjectPointerType *PT = DstType->getAs<ObjCObjectPointerType>();
-  if (!PT)
-    return false;
-  const ObjCInterfaceDecl *ID = PT->getInterfaceDecl();
-
-  // Ignore any parens, implicit casts (should only be
-  // array-to-pointer decays), and not-so-opaque values.  The last is
-  // important for making this trigger for property assignments.
-  Expr *SrcExpr = Exp->IgnoreParenImpCasts();
-  if (OpaqueValueExpr *OV = dyn_cast<OpaqueValueExpr>(SrcExpr))
-    if (OV->getSourceExpr())
-      SrcExpr = OV->getSourceExpr()->IgnoreParenImpCasts();
-
-  if (auto *SL = dyn_cast<StringLiteral>(SrcExpr)) {
-    if (!PT->isObjCIdType() &&
-        !(ID && ID->getIdentifier()->isStr("NSString")))
-      return false;
-    if (!SL->isOrdinary())
-      return false;
-
-    if (Diagnose) {
-      Diag(SL->getBeginLoc(), diag::err_missing_atsign_prefix)
-          << /*string*/0 << FixItHint::CreateInsertion(SL->getBeginLoc(), "@");
-      Exp = BuildObjCStringLiteral(SL->getBeginLoc(), SL).get();
-    }
-    return true;
-  }
-
-  if ((isa<IntegerLiteral>(SrcExpr) || isa<CharacterLiteral>(SrcExpr) ||
-      isa<FloatingLiteral>(SrcExpr) || isa<ObjCBoolLiteralExpr>(SrcExpr) ||
-      isa<CXXBoolLiteralExpr>(SrcExpr)) &&
-      !SrcExpr->isNullPointerConstant(
-          getASTContext(), Expr::NPC_NeverValueDependent)) {
-    if (!ID || !ID->getIdentifier()->isStr("NSNumber"))
-      return false;
-    if (Diagnose) {
-      Diag(SrcExpr->getBeginLoc(), diag::err_missing_atsign_prefix)
-          << /*number*/1
-          << FixItHint::CreateInsertion(SrcExpr->getBeginLoc(), "@");
-      Expr *NumLit =
-          BuildObjCNumericLiteral(SrcExpr->getBeginLoc(), SrcExpr).get();
-      if (NumLit)
-        Exp = NumLit;
-    }
-    return true;
-  }
-
-  return false;
-}
-
 static bool maybeDiagnoseAssignmentToFunction(Sema &S, QualType DstType,
                                               const Expr *SrcExpr) {
   if (!DstType->isFunctionPointerType() ||
@@ -17439,10 +17027,10 @@ bool Sema::DiagnoseAssignmentResult(AssignConvertType ConvTy,
                               FirstType, /*TakingAddress=*/true);
 
   if (CheckInferredResultType)
-    EmitRelatedResultTypeNote(SrcExpr);
+    ObjC().EmitRelatedResultTypeNote(SrcExpr);
 
   if (Action == AA_Returning && ConvTy == IncompatiblePointer)
-    EmitRelatedResultTypeNoteForReturn(DstType);
+    ObjC().EmitRelatedResultTypeNoteForReturn(DstType);
 
   if (Complained)
     *Complained = true;
@@ -20611,7 +20199,7 @@ void Sema::DiagnoseAssignmentAsCondition(Expr *E) {
       Selector Sel = ME->getSelector();
 
       // self = [<foo> init...]
-      if (isSelfExpr(Op->getLHS()) && ME->getMethodFamily() == OMF_init)
+      if (ObjC().isSelfExpr(Op->getLHS()) && ME->getMethodFamily() == OMF_init)
         diagnostic = diag::warn_condition_is_idiomatic_assignment;
 
       // <foo> = [<bar> nextObject]
@@ -21342,8 +20930,8 @@ ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
 
   // ARC unbridged casts.
   case BuiltinType::ARCUnbridgedCast: {
-    Expr *realCast = stripARCUnbridgedCast(E);
-    diagnoseARCUnbridgedCast(realCast);
+    Expr *realCast = ObjC().stripARCUnbridgedCast(E);
+    ObjC().diagnoseARCUnbridgedCast(realCast);
     return realCast;
   }
 
@@ -21459,61 +21047,6 @@ bool Sema::CheckCaseExpression(Expr *E) {
   return false;
 }
 
-/// ActOnObjCBoolLiteral - Parse {__objc_yes,__objc_no} literals.
-ExprResult
-Sema::ActOnObjCBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind) {
-  assert((Kind == tok::kw___objc_yes || Kind == tok::kw___objc_no) &&
-         "Unknown Objective-C Boolean value!");
-  QualType BoolT = Context.ObjCBuiltinBoolTy;
-  if (!Context.getBOOLDecl()) {
-    LookupResult Result(*this, &Context.Idents.get("BOOL"), OpLoc,
-                        Sema::LookupOrdinaryName);
-    if (LookupName(Result, getCurScope()) && Result.isSingleResult()) {
-      NamedDecl *ND = Result.getFoundDecl();
-      if (TypedefDecl *TD = dyn_cast<TypedefDecl>(ND))
-        Context.setBOOLDecl(TD);
-    }
-  }
-  if (Context.getBOOLDecl())
-    BoolT = Context.getBOOLType();
-  return new (Context)
-      ObjCBoolLiteralExpr(Kind == tok::kw___objc_yes, BoolT, OpLoc);
-}
-
-ExprResult Sema::ActOnObjCAvailabilityCheckExpr(
-    llvm::ArrayRef<AvailabilitySpec> AvailSpecs, SourceLocation AtLoc,
-    SourceLocation RParen) {
-  auto FindSpecVersion =
-      [&](StringRef Platform) -> std::optional<VersionTuple> {
-    auto Spec = llvm::find_if(AvailSpecs, [&](const AvailabilitySpec &Spec) {
-      return Spec.getPlatform() == Platform;
-    });
-    // Transcribe the "ios" availability check to "maccatalyst" when compiling
-    // for "maccatalyst" if "maccatalyst" is not specified.
-    if (Spec == AvailSpecs.end() && Platform == "maccatalyst") {
-      Spec = llvm::find_if(AvailSpecs, [&](const AvailabilitySpec &Spec) {
-        return Spec.getPlatform() == "ios";
-      });
-    }
-    if (Spec == AvailSpecs.end())
-      return std::nullopt;
-    return Spec->getVersion();
-  };
-
-  VersionTuple Version;
-  if (auto MaybeVersion =
-          FindSpecVersion(Context.getTargetInfo().getPlatformName()))
-    Version = *MaybeVersion;
-
-  // The use of `@available` in the enclosing context should be analyzed to
-  // warn when it's used inappropriately (i.e. not if(@available)).
-  if (FunctionScopeInfo *Context = getCurFunctionAvailabilityContext())
-    Context->HasPotentialAvailabilityViolations = true;
-
-  return new (Context)
-      ObjCAvailabilityCheckExpr(Version, AtLoc, RParen, Context.BoolTy);
-}
-
 ExprResult Sema::CreateRecoveryExpr(SourceLocation Begin, SourceLocation End,
                                     ArrayRef<Expr *> SubExprs, QualType T) {
   if (!Context.getLangOpts().RecoveryAST)
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index c181092113e1..34e12078a8c9 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -41,6 +41,7 @@
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/SemaLambda.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateDeduction.h"
 #include "llvm/ADT/APInt.h"
@@ -4619,10 +4620,10 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
 
       if (From->getType()->isObjCObjectPointerType() &&
           ToType->isObjCObjectPointerType())
-        EmitRelatedResultTypeNote(From);
+        ObjC().EmitRelatedResultTypeNote(From);
     } else if (getLangOpts().allowsNonTrivialObjCLifetimeQualifiers() &&
-               !CheckObjCARCUnavailableWeakConversion(ToType,
-                                                      From->getType())) {
+               !ObjC().CheckObjCARCUnavailableWeakConversion(ToType,
+                                                             From->getType())) {
       if (Action == AA_Initializing)
         Diag(From->getBeginLoc(), diag::err_arc_weak_unavailable_assign);
       else
@@ -4657,11 +4658,11 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     // FIXME: doing this here is really ugly.
     if (Kind == CK_BlockPointerToObjCPointerCast) {
       ExprResult E = From;
-      (void) PrepareCastToObjCObjectPointer(E);
+      (void)ObjC().PrepareCastToObjCObjectPointer(E);
       From = E.get();
     }
     if (getLangOpts().allowsNonTrivialObjCLifetimeQualifiers())
-      CheckObjCConversion(SourceRange(), NewToType, From, CCK);
+      ObjC().CheckObjCConversion(SourceRange(), NewToType, From, CCK);
     From = ImpCastExprToType(From, NewToType, Kind, VK_PRValue, &BasePath, CCK)
                .get();
     break;
@@ -7098,7 +7099,7 @@ QualType Sema::CXXCheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
     return Composite;
 
   // Similarly, attempt to find composite type of two objective-c pointers.
-  Composite = FindCompositeObjCPointerType(LHS, RHS, QuestionLoc);
+  Composite = ObjC().FindCompositeObjCPointerType(LHS, RHS, QuestionLoc);
   if (LHS.isInvalid() || RHS.isInvalid())
     return QualType();
   if (!Composite.isNull())
@@ -8691,8 +8692,8 @@ static ExprResult attemptRecovery(Sema &SemaRef,
             NewSS, /*TemplateKWLoc*/ SourceLocation(), R,
             /*TemplateArgs*/ nullptr, /*S*/ nullptr);
     } else if (auto *Ivar = dyn_cast<ObjCIvarDecl>(ND)) {
-      return SemaRef.LookupInObjCMethod(R, Consumer.getScope(),
-                                        Ivar->getIdentifier());
+      return SemaRef.ObjC().LookupInObjCMethod(R, Consumer.getScope(),
+                                               Ivar->getIdentifier());
     }
   }
 
diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp
index 244488a0b562..9aa60204bf29 100644
--- a/clang/lib/Sema/SemaExprMember.cpp
+++ b/clang/lib/Sema/SemaExprMember.cpp
@@ -21,6 +21,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 
 using namespace clang;
@@ -1518,9 +1519,8 @@ static ExprResult LookupMemberExpr(Sema &S, LookupResult &R,
     if (warn) {
       if (ObjCMethodDecl *MD = S.getCurMethodDecl()) {
         ObjCMethodFamily MF = MD->getMethodFamily();
-        warn = (MF != OMF_init && MF != OMF_dealloc &&
-                MF != OMF_finalize &&
-                !S.IvarBacksCurrentMethodAccessor(IDecl, MD, IV));
+        warn = (MF != OMF_init && MF != OMF_dealloc && MF != OMF_finalize &&
+                !S.ObjC().IvarBacksCurrentMethodAccessor(IDecl, MD, IV));
       }
       if (warn)
         S.Diag(MemberLoc, diag::warn_direct_ivar_access) << IV->getDeclName();
@@ -1658,9 +1658,9 @@ static ExprResult LookupMemberExpr(Sema &S, LookupResult &R,
     }
 
     // Normal property access.
-    return S.HandleExprPropertyRefExpr(OPT, BaseExpr.get(), OpLoc, MemberName,
-                                       MemberLoc, SourceLocation(), QualType(),
-                                       false);
+    return S.ObjC().HandleExprPropertyRefExpr(
+        OPT, BaseExpr.get(), OpLoc, MemberName, MemberLoc, SourceLocation(),
+        QualType(), false);
   }
 
   if (BaseType->isExtVectorBoolType()) {
diff --git a/clang/lib/Sema/SemaExprObjC.cpp b/clang/lib/Sema/SemaExprObjC.cpp
index b13a9d426983..462ab2c952b6 100644
--- a/clang/lib/Sema/SemaExprObjC.cpp
+++ b/clang/lib/Sema/SemaExprObjC.cpp
@@ -17,6 +17,7 @@
 #include "clang/AST/TypeLoc.h"
 #include "clang/Analysis/DomainSpecific/CocoaConventions.h"
 #include "clang/Basic/Builtins.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Edit/Commit.h"
 #include "clang/Edit/Rewriters.h"
 #include "clang/Lex/Preprocessor.h"
@@ -25,6 +26,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/ConvertUTF.h"
 #include <optional>
@@ -33,8 +35,9 @@ using namespace clang;
 using namespace sema;
 using llvm::ArrayRef;
 
-ExprResult Sema::ParseObjCStringLiteral(SourceLocation *AtLocs,
-                                        ArrayRef<Expr *> Strings) {
+ExprResult SemaObjC::ParseObjCStringLiteral(SourceLocation *AtLocs,
+                                            ArrayRef<Expr *> Strings) {
+  ASTContext &Context = getASTContext();
   // Most ObjC strings are formed out of a single piece.  However, we *can*
   // have strings formed out of multiple @ strings with multiple pptokens in
   // each one, e.g. @"foo" "bar" @"baz" "qux"   which need to be turned into one
@@ -79,7 +82,9 @@ ExprResult Sema::ParseObjCStringLiteral(SourceLocation *AtLocs,
   return BuildObjCStringLiteral(AtLocs[0], S);
 }
 
-ExprResult Sema::BuildObjCStringLiteral(SourceLocation AtLoc, StringLiteral *S){
+ExprResult SemaObjC::BuildObjCStringLiteral(SourceLocation AtLoc,
+                                            StringLiteral *S) {
+  ASTContext &Context = getASTContext();
   // Verify that this composite string is acceptable for ObjC strings.
   if (CheckObjCString(S))
     return true;
@@ -100,8 +105,8 @@ ExprResult Sema::BuildObjCStringLiteral(SourceLocation AtLoc, StringLiteral *S){
     else
       NSIdent = &Context.Idents.get(StringClass);
 
-    NamedDecl *IF = LookupSingleName(TUScope, NSIdent, AtLoc,
-                                     LookupOrdinaryName);
+    NamedDecl *IF = SemaRef.LookupSingleName(SemaRef.TUScope, NSIdent, AtLoc,
+                                             Sema::LookupOrdinaryName);
     if (ObjCInterfaceDecl *StrIF = dyn_cast_or_null<ObjCInterfaceDecl>(IF)) {
       Context.setObjCConstantStringInterface(StrIF);
       Ty = Context.getObjCConstantStringInterface();
@@ -115,8 +120,8 @@ ExprResult Sema::BuildObjCStringLiteral(SourceLocation AtLoc, StringLiteral *S){
     }
   } else {
     IdentifierInfo *NSIdent = NSAPIObj->getNSClassId(NSAPI::ClassId_NSString);
-    NamedDecl *IF = LookupSingleName(TUScope, NSIdent, AtLoc,
-                                     LookupOrdinaryName);
+    NamedDecl *IF = SemaRef.LookupSingleName(SemaRef.TUScope, NSIdent, AtLoc,
+                                             Sema::LookupOrdinaryName);
     if (ObjCInterfaceDecl *StrIF = dyn_cast_or_null<ObjCInterfaceDecl>(IF)) {
       Context.setObjCConstantStringInterface(StrIF);
       Ty = Context.getObjCConstantStringInterface();
@@ -168,25 +173,25 @@ static bool validateBoxingMethod(Sema &S, SourceLocation Loc,
 }
 
 /// Maps ObjCLiteralKind to NSClassIdKindKind
-static NSAPI::NSClassIdKindKind ClassKindFromLiteralKind(
-                                            Sema::ObjCLiteralKind LiteralKind) {
+static NSAPI::NSClassIdKindKind
+ClassKindFromLiteralKind(SemaObjC::ObjCLiteralKind LiteralKind) {
   switch (LiteralKind) {
-    case Sema::LK_Array:
-      return NSAPI::ClassId_NSArray;
-    case Sema::LK_Dictionary:
-      return NSAPI::ClassId_NSDictionary;
-    case Sema::LK_Numeric:
-      return NSAPI::ClassId_NSNumber;
-    case Sema::LK_String:
-      return NSAPI::ClassId_NSString;
-    case Sema::LK_Boxed:
-      return NSAPI::ClassId_NSValue;
-
-    // there is no corresponding matching
-    // between LK_None/LK_Block and NSClassIdKindKind
-    case Sema::LK_Block:
-    case Sema::LK_None:
-      break;
+  case SemaObjC::LK_Array:
+    return NSAPI::ClassId_NSArray;
+  case SemaObjC::LK_Dictionary:
+    return NSAPI::ClassId_NSDictionary;
+  case SemaObjC::LK_Numeric:
+    return NSAPI::ClassId_NSNumber;
+  case SemaObjC::LK_String:
+    return NSAPI::ClassId_NSString;
+  case SemaObjC::LK_Boxed:
+    return NSAPI::ClassId_NSValue;
+
+  // there is no corresponding matching
+  // between LK_None/LK_Block and NSClassIdKindKind
+  case SemaObjC::LK_Block:
+  case SemaObjC::LK_None:
+    break;
   }
   llvm_unreachable("LiteralKind can't be converted into a ClassKind");
 }
@@ -194,12 +199,13 @@ static NSAPI::NSClassIdKindKind ClassKindFromLiteralKind(
 /// Validates ObjCInterfaceDecl availability.
 /// ObjCInterfaceDecl, used to create ObjC literals, should be defined
 /// if clang not in a debugger mode.
-static bool ValidateObjCLiteralInterfaceDecl(Sema &S, ObjCInterfaceDecl *Decl,
-                                            SourceLocation Loc,
-                                            Sema::ObjCLiteralKind LiteralKind) {
+static bool
+ValidateObjCLiteralInterfaceDecl(Sema &S, ObjCInterfaceDecl *Decl,
+                                 SourceLocation Loc,
+                                 SemaObjC::ObjCLiteralKind LiteralKind) {
   if (!Decl) {
     NSAPI::NSClassIdKindKind Kind = ClassKindFromLiteralKind(LiteralKind);
-    IdentifierInfo *II = S.NSAPIObj->getNSClassId(Kind);
+    IdentifierInfo *II = S.ObjC().NSAPIObj->getNSClassId(Kind);
     S.Diag(Loc, diag::err_undeclared_objc_literal_class)
       << II->getName() << LiteralKind;
     return false;
@@ -216,11 +222,11 @@ static bool ValidateObjCLiteralInterfaceDecl(Sema &S, ObjCInterfaceDecl *Decl,
 /// Looks up ObjCInterfaceDecl of a given NSClassIdKindKind.
 /// Used to create ObjC literals, such as NSDictionary (@{}),
 /// NSArray (@[]) and Boxed Expressions (@())
-static ObjCInterfaceDecl *LookupObjCInterfaceDeclForLiteral(Sema &S,
-                                            SourceLocation Loc,
-                                            Sema::ObjCLiteralKind LiteralKind) {
+static ObjCInterfaceDecl *
+LookupObjCInterfaceDeclForLiteral(Sema &S, SourceLocation Loc,
+                                  SemaObjC::ObjCLiteralKind LiteralKind) {
   NSAPI::NSClassIdKindKind ClassKind = ClassKindFromLiteralKind(LiteralKind);
-  IdentifierInfo *II = S.NSAPIObj->getNSClassId(ClassKind);
+  IdentifierInfo *II = S.ObjC().NSAPIObj->getNSClassId(ClassKind);
   NamedDecl *IF = S.LookupSingleName(S.TUScope, II, Loc,
                                      Sema::LookupOrdinaryName);
   ObjCInterfaceDecl *ID = dyn_cast_or_null<ObjCInterfaceDecl>(IF);
@@ -240,7 +246,7 @@ static ObjCInterfaceDecl *LookupObjCInterfaceDeclForLiteral(Sema &S,
 
 /// Retrieve the NSNumber factory method that should be used to create
 /// an Objective-C literal for the given type.
-static ObjCMethodDecl *getNSNumberFactoryMethod(Sema &S, SourceLocation Loc,
+static ObjCMethodDecl *getNSNumberFactoryMethod(SemaObjC &S, SourceLocation Loc,
                                                 QualType NumberType,
                                                 bool isLiteral = false,
                                                 SourceRange R = SourceRange()) {
@@ -262,13 +268,13 @@ static ObjCMethodDecl *getNSNumberFactoryMethod(Sema &S, SourceLocation Loc,
   Selector Sel = S.NSAPIObj->getNSNumberLiteralSelector(*Kind,
                                                         /*Instance=*/false);
 
-  ASTContext &CX = S.Context;
+  ASTContext &CX = S.SemaRef.Context;
 
   // Look up the NSNumber class, if we haven't done so already. It's cached
   // in the Sema instance.
   if (!S.NSNumberDecl) {
-    S.NSNumberDecl = LookupObjCInterfaceDeclForLiteral(S, Loc,
-                                                       Sema::LK_Numeric);
+    S.NSNumberDecl =
+        LookupObjCInterfaceDeclForLiteral(S.SemaRef, Loc, SemaObjC::LK_Numeric);
     if (!S.NSNumberDecl) {
       return nullptr;
     }
@@ -294,15 +300,14 @@ static ObjCMethodDecl *getNSNumberFactoryMethod(Sema &S, SourceLocation Loc,
         /*isImplicitlyDeclared=*/true,
         /*isDefined=*/false, ObjCImplementationControl::Required,
         /*HasRelatedResultType=*/false);
-    ParmVarDecl *value = ParmVarDecl::Create(S.Context, Method,
-                                             SourceLocation(), SourceLocation(),
-                                             &CX.Idents.get("value"),
-                                             NumberType, /*TInfo=*/nullptr,
-                                             SC_None, nullptr);
-    Method->setMethodParams(S.Context, value, std::nullopt);
+    ParmVarDecl *value =
+        ParmVarDecl::Create(S.SemaRef.Context, Method, SourceLocation(),
+                            SourceLocation(), &CX.Idents.get("value"),
+                            NumberType, /*TInfo=*/nullptr, SC_None, nullptr);
+    Method->setMethodParams(S.SemaRef.Context, value, std::nullopt);
   }
 
-  if (!validateBoxingMethod(S, Loc, S.NSNumberDecl, Sel, Method))
+  if (!validateBoxingMethod(S.SemaRef, Loc, S.NSNumberDecl, Sel, Method))
     return nullptr;
 
   // Note: if the parameter type is out-of-line, we'll catch it later in the
@@ -314,7 +319,9 @@ static ObjCMethodDecl *getNSNumberFactoryMethod(Sema &S, SourceLocation Loc,
 
 /// BuildObjCNumericLiteral - builds an ObjCBoxedExpr AST node for the
 /// numeric literal expression. Type of the expression will be "NSNumber *".
-ExprResult Sema::BuildObjCNumericLiteral(SourceLocation AtLoc, Expr *Number) {
+ExprResult SemaObjC::BuildObjCNumericLiteral(SourceLocation AtLoc,
+                                             Expr *Number) {
+  ASTContext &Context = getASTContext();
   // Determine the type of the literal.
   QualType NumberType = Number->getType();
   if (CharacterLiteral *Char = dyn_cast<CharacterLiteral>(Number)) {
@@ -352,31 +359,30 @@ ExprResult Sema::BuildObjCNumericLiteral(SourceLocation AtLoc, Expr *Number) {
   ParmVarDecl *ParamDecl = Method->parameters()[0];
   InitializedEntity Entity = InitializedEntity::InitializeParameter(Context,
                                                                     ParamDecl);
-  ExprResult ConvertedNumber = PerformCopyInitialization(Entity,
-                                                         SourceLocation(),
-                                                         Number);
+  ExprResult ConvertedNumber =
+      SemaRef.PerformCopyInitialization(Entity, SourceLocation(), Number);
   if (ConvertedNumber.isInvalid())
     return ExprError();
   Number = ConvertedNumber.get();
 
   // Use the effective source range of the literal, including the leading '@'.
-  return MaybeBindToTemporary(
-           new (Context) ObjCBoxedExpr(Number, NSNumberPointer, Method,
-                                       SourceRange(AtLoc, NR.getEnd())));
+  return SemaRef.MaybeBindToTemporary(new (Context) ObjCBoxedExpr(
+      Number, NSNumberPointer, Method, SourceRange(AtLoc, NR.getEnd())));
 }
 
-ExprResult Sema::ActOnObjCBoolLiteral(SourceLocation AtLoc,
-                                      SourceLocation ValueLoc,
-                                      bool Value) {
+ExprResult SemaObjC::ActOnObjCBoolLiteral(SourceLocation AtLoc,
+                                          SourceLocation ValueLoc, bool Value) {
+  ASTContext &Context = getASTContext();
   ExprResult Inner;
   if (getLangOpts().CPlusPlus) {
-    Inner = ActOnCXXBoolLiteral(ValueLoc, Value? tok::kw_true : tok::kw_false);
+    Inner = SemaRef.ActOnCXXBoolLiteral(ValueLoc,
+                                        Value ? tok::kw_true : tok::kw_false);
   } else {
     // C doesn't actually have a way to represent literal values of type
     // _Bool. So, we'll use 0/1 and implicit cast to _Bool.
-    Inner = ActOnIntegerConstant(ValueLoc, Value? 1 : 0);
-    Inner = ImpCastExprToType(Inner.get(), Context.BoolTy,
-                              CK_IntegralToBoolean);
+    Inner = SemaRef.ActOnIntegerConstant(ValueLoc, Value ? 1 : 0);
+    Inner = SemaRef.ImpCastExprToType(Inner.get(), Context.BoolTy,
+                                      CK_IntegralToBoolean);
   }
 
   return BuildObjCNumericLiteral(AtLoc, Inner.get());
@@ -428,7 +434,8 @@ static ExprResult CheckObjCCollectionLiteralElement(Sema &S, Expr *Element,
         isa<FloatingLiteral>(OrigElement) ||
         isa<ObjCBoolLiteralExpr>(OrigElement) ||
         isa<CXXBoolLiteralExpr>(OrigElement)) {
-      if (S.NSAPIObj->getNSNumberFactoryMethodKind(OrigElement->getType())) {
+      if (S.ObjC().NSAPIObj->getNSNumberFactoryMethodKind(
+              OrigElement->getType())) {
         int Which = isa<CharacterLiteral>(OrigElement) ? 1
                   : (isa<CXXBoolLiteralExpr>(OrigElement) ||
                      isa<ObjCBoolLiteralExpr>(OrigElement)) ? 2
@@ -438,8 +445,8 @@ static ExprResult CheckObjCCollectionLiteralElement(Sema &S, Expr *Element,
             << Which << OrigElement->getSourceRange()
             << FixItHint::CreateInsertion(OrigElement->getBeginLoc(), "@");
 
-        Result =
-            S.BuildObjCNumericLiteral(OrigElement->getBeginLoc(), OrigElement);
+        Result = S.ObjC().BuildObjCNumericLiteral(OrigElement->getBeginLoc(),
+                                                  OrigElement);
         if (Result.isInvalid())
           return ExprError();
 
@@ -454,7 +461,8 @@ static ExprResult CheckObjCCollectionLiteralElement(Sema &S, Expr *Element,
             << 0 << OrigElement->getSourceRange()
             << FixItHint::CreateInsertion(OrigElement->getBeginLoc(), "@");
 
-        Result = S.BuildObjCStringLiteral(OrigElement->getBeginLoc(), String);
+        Result =
+            S.ObjC().BuildObjCStringLiteral(OrigElement->getBeginLoc(), String);
         if (Result.isInvalid())
           return ExprError();
 
@@ -498,7 +506,8 @@ static ExprResult CheckObjCCollectionLiteralElement(Sema &S, Expr *Element,
       Element->getBeginLoc(), Element);
 }
 
-ExprResult Sema::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
+ExprResult SemaObjC::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
+  ASTContext &Context = getASTContext();
   if (ValueExpr->isTypeDependent()) {
     ObjCBoxedExpr *BoxedExpr =
       new (Context) ObjCBoxedExpr(ValueExpr, Context.DependentTy, nullptr, SR);
@@ -507,7 +516,7 @@ ExprResult Sema::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
   ObjCMethodDecl *BoxingMethod = nullptr;
   QualType BoxedType;
   // Convert the expression to an RValue, so we can check for pointer types...
-  ExprResult RValue = DefaultFunctionArrayLvalueConversion(ValueExpr);
+  ExprResult RValue = SemaRef.DefaultFunctionArrayLvalueConversion(ValueExpr);
   if (RValue.isInvalid()) {
     return ExprError();
   }
@@ -519,8 +528,8 @@ ExprResult Sema::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
     if (Context.hasSameUnqualifiedType(PointeeType, Context.CharTy)) {
 
       if (!NSStringDecl) {
-        NSStringDecl = LookupObjCInterfaceDeclForLiteral(*this, Loc,
-                                                         Sema::LK_String);
+        NSStringDecl =
+            LookupObjCInterfaceDeclForLiteral(SemaRef, Loc, LK_String);
         if (!NSStringDecl) {
           return ExprError();
         }
@@ -582,9 +591,9 @@ ExprResult Sema::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
           BoxingMethod = M;
         }
 
-        if (!validateBoxingMethod(*this, Loc, NSStringDecl,
+        if (!validateBoxingMethod(SemaRef, Loc, NSStringDecl,
                                   stringWithUTF8String, BoxingMethod))
-           return ExprError();
+          return ExprError();
 
         StringWithUTF8StringMethod = BoxingMethod;
       }
@@ -651,8 +660,7 @@ ExprResult Sema::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
     // Look up the NSValue class, if we haven't done so already. It's cached
     // in the Sema instance.
     if (!NSValueDecl) {
-      NSValueDecl = LookupObjCInterfaceDeclForLiteral(*this, Loc,
-                                                      Sema::LK_Boxed);
+      NSValueDecl = LookupObjCInterfaceDeclForLiteral(SemaRef, Loc, LK_Boxed);
       if (!NSValueDecl) {
         return ExprError();
       }
@@ -708,7 +716,7 @@ ExprResult Sema::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
         BoxingMethod = M;
       }
 
-      if (!validateBoxingMethod(*this, Loc, NSValueDecl,
+      if (!validateBoxingMethod(SemaRef, Loc, NSValueDecl,
                                 ValueWithBytesObjCType, BoxingMethod))
         return ExprError();
 
@@ -731,20 +739,20 @@ ExprResult Sema::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
     return ExprError();
   }
 
-  DiagnoseUseOfDecl(BoxingMethod, Loc);
+  SemaRef.DiagnoseUseOfDecl(BoxingMethod, Loc);
 
   ExprResult ConvertedValueExpr;
   if (ValueType->isObjCBoxableRecordType()) {
     InitializedEntity IE = InitializedEntity::InitializeTemporary(ValueType);
-    ConvertedValueExpr = PerformCopyInitialization(IE, ValueExpr->getExprLoc(),
-                                                   ValueExpr);
+    ConvertedValueExpr = SemaRef.PerformCopyInitialization(
+        IE, ValueExpr->getExprLoc(), ValueExpr);
   } else {
     // Convert the expression to the type that the parameter requires.
     ParmVarDecl *ParamDecl = BoxingMethod->parameters()[0];
     InitializedEntity IE = InitializedEntity::InitializeParameter(Context,
                                                                   ParamDecl);
-    ConvertedValueExpr = PerformCopyInitialization(IE, SourceLocation(),
-                                                   ValueExpr);
+    ConvertedValueExpr =
+        SemaRef.PerformCopyInitialization(IE, SourceLocation(), ValueExpr);
   }
 
   if (ConvertedValueExpr.isInvalid())
@@ -754,16 +762,16 @@ ExprResult Sema::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
   ObjCBoxedExpr *BoxedExpr =
     new (Context) ObjCBoxedExpr(ValueExpr, BoxedType,
                                       BoxingMethod, SR);
-  return MaybeBindToTemporary(BoxedExpr);
+  return SemaRef.MaybeBindToTemporary(BoxedExpr);
 }
 
 /// Build an ObjC subscript pseudo-object expression, given that
 /// that's supported by the runtime.
-ExprResult Sema::BuildObjCSubscriptExpression(SourceLocation RB, Expr *BaseExpr,
-                                        Expr *IndexExpr,
-                                        ObjCMethodDecl *getterMethod,
-                                        ObjCMethodDecl *setterMethod) {
-  assert(!LangOpts.isSubscriptPointerArithmetic());
+ExprResult SemaObjC::BuildObjCSubscriptExpression(
+    SourceLocation RB, Expr *BaseExpr, Expr *IndexExpr,
+    ObjCMethodDecl *getterMethod, ObjCMethodDecl *setterMethod) {
+  assert(!getLangOpts().isSubscriptPointerArithmetic());
+  ASTContext &Context = getASTContext();
 
   // We can't get dependent types here; our callers should have
   // filtered them out.
@@ -772,13 +780,13 @@ ExprResult Sema::BuildObjCSubscriptExpression(SourceLocation RB, Expr *BaseExpr,
 
   // Filter out placeholders in the index.  In theory, overloads could
   // be preserved here, although that might not actually work correctly.
-  ExprResult Result = CheckPlaceholderExpr(IndexExpr);
+  ExprResult Result = SemaRef.CheckPlaceholderExpr(IndexExpr);
   if (Result.isInvalid())
     return ExprError();
   IndexExpr = Result.get();
 
   // Perform lvalue-to-rvalue conversion on the base.
-  Result = DefaultLvalueConversion(BaseExpr);
+  Result = SemaRef.DefaultLvalueConversion(BaseExpr);
   if (Result.isInvalid())
     return ExprError();
   BaseExpr = Result.get();
@@ -789,12 +797,14 @@ ExprResult Sema::BuildObjCSubscriptExpression(SourceLocation RB, Expr *BaseExpr,
       getterMethod, setterMethod, RB);
 }
 
-ExprResult Sema::BuildObjCArrayLiteral(SourceRange SR, MultiExprArg Elements) {
+ExprResult SemaObjC::BuildObjCArrayLiteral(SourceRange SR,
+                                           MultiExprArg Elements) {
+  ASTContext &Context = getASTContext();
   SourceLocation Loc = SR.getBegin();
 
   if (!NSArrayDecl) {
-    NSArrayDecl = LookupObjCInterfaceDeclForLiteral(*this, Loc,
-                                                    Sema::LK_Array);
+    NSArrayDecl =
+        LookupObjCInterfaceDeclForLiteral(SemaRef, Loc, SemaObjC::LK_Array);
     if (!NSArrayDecl) {
       return ExprError();
     }
@@ -835,7 +845,7 @@ ExprResult Sema::BuildObjCArrayLiteral(SourceRange SR, MultiExprArg Elements) {
       Method->setMethodParams(Context, Params, std::nullopt);
     }
 
-    if (!validateBoxingMethod(*this, Loc, NSArrayDecl, Sel, Method))
+    if (!validateBoxingMethod(SemaRef, Loc, NSArrayDecl, Sel, Method))
       return ExprError();
 
     // Dig out the type that all elements should be converted to.
@@ -875,9 +885,8 @@ ExprResult Sema::BuildObjCArrayLiteral(SourceRange SR, MultiExprArg Elements) {
   // performing conversions as necessary.
   Expr **ElementsBuffer = Elements.data();
   for (unsigned I = 0, N = Elements.size(); I != N; ++I) {
-    ExprResult Converted = CheckObjCCollectionLiteralElement(*this,
-                                                             ElementsBuffer[I],
-                                                             RequiredType, true);
+    ExprResult Converted = CheckObjCCollectionLiteralElement(
+        SemaRef, ElementsBuffer[I], RequiredType, true);
     if (Converted.isInvalid())
       return ExprError();
 
@@ -888,9 +897,8 @@ ExprResult Sema::BuildObjCArrayLiteral(SourceRange SR, MultiExprArg Elements) {
     = Context.getObjCObjectPointerType(
                                     Context.getObjCInterfaceType(NSArrayDecl));
 
-  return MaybeBindToTemporary(
-           ObjCArrayLiteral::Create(Context, Elements, Ty,
-                                    ArrayWithObjectsMethod, SR));
+  return SemaRef.MaybeBindToTemporary(ObjCArrayLiteral::Create(
+      Context, Elements, Ty, ArrayWithObjectsMethod, SR));
 }
 
 /// Check for duplicate keys in an ObjC dictionary literal. For instance:
@@ -949,13 +957,14 @@ CheckObjCDictionaryLiteralDuplicateKeys(Sema &S,
   }
 }
 
-ExprResult Sema::BuildObjCDictionaryLiteral(SourceRange SR,
-                              MutableArrayRef<ObjCDictionaryElement> Elements) {
+ExprResult SemaObjC::BuildObjCDictionaryLiteral(
+    SourceRange SR, MutableArrayRef<ObjCDictionaryElement> Elements) {
+  ASTContext &Context = getASTContext();
   SourceLocation Loc = SR.getBegin();
 
   if (!NSDictionaryDecl) {
-    NSDictionaryDecl = LookupObjCInterfaceDeclForLiteral(*this, Loc,
-                                                         Sema::LK_Dictionary);
+    NSDictionaryDecl = LookupObjCInterfaceDeclForLiteral(
+        SemaRef, Loc, SemaObjC::LK_Dictionary);
     if (!NSDictionaryDecl) {
       return ExprError();
     }
@@ -1005,9 +1014,9 @@ ExprResult Sema::BuildObjCDictionaryLiteral(SourceRange SR,
       Method->setMethodParams(Context, Params, std::nullopt);
     }
 
-    if (!validateBoxingMethod(*this, SR.getBegin(), NSDictionaryDecl, Sel,
+    if (!validateBoxingMethod(SemaRef, SR.getBegin(), NSDictionaryDecl, Sel,
                               Method))
-       return ExprError();
+      return ExprError();
 
     // Dig out the type that all values should be converted to.
     QualType ValueT = Method->parameters()[0]->getType();
@@ -1084,14 +1093,14 @@ ExprResult Sema::BuildObjCDictionaryLiteral(SourceRange SR,
   bool HasPackExpansions = false;
   for (ObjCDictionaryElement &Element : Elements) {
     // Check the key.
-    ExprResult Key = CheckObjCCollectionLiteralElement(*this, Element.Key,
-                                                       KeyT);
+    ExprResult Key =
+        CheckObjCCollectionLiteralElement(SemaRef, Element.Key, KeyT);
     if (Key.isInvalid())
       return ExprError();
 
     // Check the value.
-    ExprResult Value
-      = CheckObjCCollectionLiteralElement(*this, Element.Value, ValueT);
+    ExprResult Value =
+        CheckObjCCollectionLiteralElement(SemaRef, Element.Value, ValueT);
     if (Value.isInvalid())
       return ExprError();
 
@@ -1119,13 +1128,14 @@ ExprResult Sema::BuildObjCDictionaryLiteral(SourceRange SR,
   auto *Literal =
       ObjCDictionaryLiteral::Create(Context, Elements, HasPackExpansions, Ty,
                                     DictionaryWithObjectsMethod, SR);
-  CheckObjCDictionaryLiteralDuplicateKeys(*this, Literal);
-  return MaybeBindToTemporary(Literal);
+  CheckObjCDictionaryLiteralDuplicateKeys(SemaRef, Literal);
+  return SemaRef.MaybeBindToTemporary(Literal);
 }
 
-ExprResult Sema::BuildObjCEncodeExpression(SourceLocation AtLoc,
-                                      TypeSourceInfo *EncodedTypeInfo,
-                                      SourceLocation RParenLoc) {
+ExprResult SemaObjC::BuildObjCEncodeExpression(SourceLocation AtLoc,
+                                               TypeSourceInfo *EncodedTypeInfo,
+                                               SourceLocation RParenLoc) {
+  ASTContext &Context = getASTContext();
   QualType EncodedType = EncodedTypeInfo->getType();
   QualType StrTy;
   if (EncodedType->isDependentType())
@@ -1133,9 +1143,9 @@ ExprResult Sema::BuildObjCEncodeExpression(SourceLocation AtLoc,
   else {
     if (!EncodedType->getAsArrayTypeUnsafe() && //// Incomplete array is handled.
         !EncodedType->isVoidType()) // void is handled too.
-      if (RequireCompleteType(AtLoc, EncodedType,
-                              diag::err_incomplete_type_objc_at_encode,
-                              EncodedTypeInfo->getTypeLoc()))
+      if (SemaRef.RequireCompleteType(AtLoc, EncodedType,
+                                      diag::err_incomplete_type_objc_at_encode,
+                                      EncodedTypeInfo->getTypeLoc()))
         return ExprError();
 
     std::string Str;
@@ -1153,17 +1163,18 @@ ExprResult Sema::BuildObjCEncodeExpression(SourceLocation AtLoc,
   return new (Context) ObjCEncodeExpr(StrTy, EncodedTypeInfo, AtLoc, RParenLoc);
 }
 
-ExprResult Sema::ParseObjCEncodeExpression(SourceLocation AtLoc,
-                                           SourceLocation EncodeLoc,
-                                           SourceLocation LParenLoc,
-                                           ParsedType ty,
-                                           SourceLocation RParenLoc) {
+ExprResult SemaObjC::ParseObjCEncodeExpression(SourceLocation AtLoc,
+                                               SourceLocation EncodeLoc,
+                                               SourceLocation LParenLoc,
+                                               ParsedType ty,
+                                               SourceLocation RParenLoc) {
+  ASTContext &Context = getASTContext();
   // FIXME: Preserve type source info ?
   TypeSourceInfo *TInfo;
-  QualType EncodedType = GetTypeFromParser(ty, &TInfo);
+  QualType EncodedType = SemaRef.GetTypeFromParser(ty, &TInfo);
   if (!TInfo)
-    TInfo = Context.getTrivialTypeSourceInfo(EncodedType,
-                                             getLocForEndOfToken(LParenLoc));
+    TInfo = Context.getTrivialTypeSourceInfo(
+        EncodedType, SemaRef.getLocForEndOfToken(LParenLoc));
 
   return BuildObjCEncodeExpression(AtLoc, TInfo, RParenLoc);
 }
@@ -1182,8 +1193,8 @@ static bool HelperToDiagnoseMismatchedMethodsInGlobalPool(Sema &S,
         isa<ObjCImplDecl>(MatchingMethodDecl->getDeclContext()) ||
         MatchingMethodDecl->getSelector() != Method->getSelector())
       continue;
-    if (!S.MatchTwoMethodDeclarations(Method,
-                                      MatchingMethodDecl, Sema::MMS_loose)) {
+    if (!S.ObjC().MatchTwoMethodDeclarations(Method, MatchingMethodDecl,
+                                             SemaObjC::MMS_loose)) {
       if (!Warned) {
         Warned = true;
         S.Diag(AtLoc, diag::warn_multiple_selectors)
@@ -1208,8 +1219,9 @@ static void DiagnoseMismatchedSelectors(Sema &S, SourceLocation AtLoc,
       S.Diags.isIgnored(diag::warn_multiple_selectors, SourceLocation()))
     return;
   bool Warned = false;
-  for (Sema::GlobalMethodPool::iterator b = S.MethodPool.begin(),
-       e = S.MethodPool.end(); b != e; b++) {
+  for (SemaObjC::GlobalMethodPool::iterator b = S.ObjC().MethodPool.begin(),
+                                            e = S.ObjC().MethodPool.end();
+       b != e; b++) {
     // first, instance methods
     ObjCMethodList &InstMethList = b->second.first;
     if (HelperToDiagnoseMismatchedMethodsInGlobalPool(S, AtLoc, LParenLoc, RParenLoc,
@@ -1253,8 +1265,8 @@ static ObjCMethodDecl *LookupDirectMethodInMethodList(Sema &S, Selector Sel,
 static ObjCMethodDecl *LookupDirectMethodInGlobalPool(Sema &S, Selector Sel,
                                                       bool &onlyDirect,
                                                       bool &anyDirect) {
-  auto Iter = S.MethodPool.find(Sel);
-  if (Iter == S.MethodPool.end())
+  auto Iter = S.ObjC().MethodPool.find(Sel);
+  if (Iter == S.ObjC().MethodPool.end())
     return nullptr;
 
   ObjCMethodDecl *DirectInstance = LookupDirectMethodInMethodList(
@@ -1286,12 +1298,13 @@ static ObjCMethodDecl *findMethodInCurrentClass(Sema &S, Selector Sel) {
   return nullptr;
 }
 
-ExprResult Sema::ParseObjCSelectorExpression(Selector Sel,
-                                             SourceLocation AtLoc,
-                                             SourceLocation SelLoc,
-                                             SourceLocation LParenLoc,
-                                             SourceLocation RParenLoc,
-                                             bool WarnMultipleSelectors) {
+ExprResult SemaObjC::ParseObjCSelectorExpression(Selector Sel,
+                                                 SourceLocation AtLoc,
+                                                 SourceLocation SelLoc,
+                                                 SourceLocation LParenLoc,
+                                                 SourceLocation RParenLoc,
+                                                 bool WarnMultipleSelectors) {
+  ASTContext &Context = getASTContext();
   ObjCMethodDecl *Method = LookupInstanceMethodInGlobalPool(Sel,
                              SourceRange(LParenLoc, RParenLoc));
   if (!Method)
@@ -1309,13 +1322,13 @@ ExprResult Sema::ParseObjCSelectorExpression(Selector Sel,
     } else
         Diag(SelLoc, diag::warn_undeclared_selector) << Sel;
   } else {
-    DiagnoseMismatchedSelectors(*this, AtLoc, Method, LParenLoc, RParenLoc,
+    DiagnoseMismatchedSelectors(SemaRef, AtLoc, Method, LParenLoc, RParenLoc,
                                 WarnMultipleSelectors);
 
     bool onlyDirect = true;
     bool anyDirect = false;
     ObjCMethodDecl *GlobalDirectMethod =
-        LookupDirectMethodInGlobalPool(*this, Sel, onlyDirect, anyDirect);
+        LookupDirectMethodInGlobalPool(SemaRef, Sel, onlyDirect, anyDirect);
 
     if (onlyDirect) {
       Diag(AtLoc, diag::err_direct_selector_expression)
@@ -1326,7 +1339,8 @@ ExprResult Sema::ParseObjCSelectorExpression(Selector Sel,
       // If we saw any direct methods, see if we see a direct member of the
       // current class. If so, the @selector will likely be used to refer to
       // this direct method.
-      ObjCMethodDecl *LikelyTargetMethod = findMethodInCurrentClass(*this, Sel);
+      ObjCMethodDecl *LikelyTargetMethod =
+          findMethodInCurrentClass(SemaRef, Sel);
       if (LikelyTargetMethod && LikelyTargetMethod->isDirectMethod()) {
         Diag(AtLoc, diag::warn_potentially_direct_selector_expression) << Sel;
         Diag(LikelyTargetMethod->getLocation(),
@@ -1347,7 +1361,7 @@ ExprResult Sema::ParseObjCSelectorExpression(Selector Sel,
   if (Method &&
       Method->getImplementationControl() !=
           ObjCImplementationControl::Optional &&
-      !getSourceManager().isInSystemHeader(Method->getLocation()))
+      !SemaRef.getSourceManager().isInSystemHeader(Method->getLocation()))
     ReferencedSelectors.insert(std::make_pair(Sel, AtLoc));
 
   // In ARC, forbid the user from using @selector for
@@ -1380,12 +1394,13 @@ ExprResult Sema::ParseObjCSelectorExpression(Selector Sel,
   return new (Context) ObjCSelectorExpr(Ty, Sel, AtLoc, RParenLoc);
 }
 
-ExprResult Sema::ParseObjCProtocolExpression(IdentifierInfo *ProtocolId,
-                                             SourceLocation AtLoc,
-                                             SourceLocation ProtoLoc,
-                                             SourceLocation LParenLoc,
-                                             SourceLocation ProtoIdLoc,
-                                             SourceLocation RParenLoc) {
+ExprResult SemaObjC::ParseObjCProtocolExpression(IdentifierInfo *ProtocolId,
+                                                 SourceLocation AtLoc,
+                                                 SourceLocation ProtoLoc,
+                                                 SourceLocation LParenLoc,
+                                                 SourceLocation ProtoIdLoc,
+                                                 SourceLocation RParenLoc) {
+  ASTContext &Context = getASTContext();
   ObjCProtocolDecl* PDecl = LookupProtocol(ProtocolId, ProtoIdLoc);
   if (!PDecl) {
     Diag(ProtoLoc, diag::err_undeclared_protocol) << ProtocolId;
@@ -1409,8 +1424,8 @@ ExprResult Sema::ParseObjCProtocolExpression(IdentifierInfo *ProtocolId,
 }
 
 /// Try to capture an implicit reference to 'self'.
-ObjCMethodDecl *Sema::tryCaptureObjCSelf(SourceLocation Loc) {
-  DeclContext *DC = getFunctionLevelDeclContext();
+ObjCMethodDecl *SemaObjC::tryCaptureObjCSelf(SourceLocation Loc) {
+  DeclContext *DC = SemaRef.getFunctionLevelDeclContext();
 
   // If we're not in an ObjC method, error out.  Note that, unlike the
   // C++ case, we don't require an instance method --- class methods
@@ -1419,7 +1434,7 @@ ObjCMethodDecl *Sema::tryCaptureObjCSelf(SourceLocation Loc) {
   if (!method)
     return nullptr;
 
-  tryCaptureVariable(method->getSelfDecl(), Loc);
+  SemaRef.tryCaptureVariable(method->getSelfDecl(), Loc);
 
   return method;
 }
@@ -1513,16 +1528,15 @@ static QualType getBaseMessageSendResultType(Sema &S,
   return transferNullability(ReceiverType);
 }
 
-QualType Sema::getMessageSendResultType(const Expr *Receiver,
-                                        QualType ReceiverType,
-                                        ObjCMethodDecl *Method,
-                                        bool isClassMessage,
-                                        bool isSuperMessage) {
+QualType SemaObjC::getMessageSendResultType(const Expr *Receiver,
+                                            QualType ReceiverType,
+                                            ObjCMethodDecl *Method,
+                                            bool isClassMessage,
+                                            bool isSuperMessage) {
+  ASTContext &Context = getASTContext();
   // Produce the result type.
-  QualType resultType = getBaseMessageSendResultType(*this, ReceiverType,
-                                                     Method,
-                                                     isClassMessage,
-                                                     isSuperMessage);
+  QualType resultType = getBaseMessageSendResultType(
+      SemaRef, ReceiverType, Method, isClassMessage, isSuperMessage);
 
   // If this is a class message, ignore the nullability of the receiver.
   if (isClassMessage) {
@@ -1651,10 +1665,11 @@ findExplicitInstancetypeDeclarer(const ObjCMethodDecl *MD,
   return nullptr;
 }
 
-void Sema::EmitRelatedResultTypeNoteForReturn(QualType destType) {
+void SemaObjC::EmitRelatedResultTypeNoteForReturn(QualType destType) {
+  ASTContext &Context = getASTContext();
   // Only complain if we're in an ObjC method and the required return
   // type doesn't match the method's declared return type.
-  ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(CurContext);
+  ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(SemaRef.CurContext);
   if (!MD || !MD->hasRelatedResultType() ||
       Context.hasSameUnqualifiedType(destType, MD->getReturnType()))
     return;
@@ -1680,7 +1695,8 @@ void Sema::EmitRelatedResultTypeNoteForReturn(QualType destType) {
       << family;
 }
 
-void Sema::EmitRelatedResultTypeNote(const Expr *E) {
+void SemaObjC::EmitRelatedResultTypeNote(const Expr *E) {
+  ASTContext &Context = getASTContext();
   E = E->IgnoreParenImpCasts();
   const ObjCMessageExpr *MsgSend = dyn_cast<ObjCMessageExpr>(E);
   if (!MsgSend)
@@ -1706,12 +1722,13 @@ void Sema::EmitRelatedResultTypeNote(const Expr *E) {
     << MsgSend->getType();
 }
 
-bool Sema::CheckMessageArgumentTypes(
+bool SemaObjC::CheckMessageArgumentTypes(
     const Expr *Receiver, QualType ReceiverType, MultiExprArg Args,
     Selector Sel, ArrayRef<SourceLocation> SelectorLocs, ObjCMethodDecl *Method,
     bool isClassMessage, bool isSuperMessage, SourceLocation lbrac,
     SourceLocation rbrac, SourceRange RecRange, QualType &ReturnType,
     ExprValueKind &VK) {
+  ASTContext &Context = getASTContext();
   SourceLocation SelLoc;
   if (!SelectorLocs.empty() && SelectorLocs.front().isValid())
     SelLoc = SelectorLocs.front();
@@ -1727,9 +1744,9 @@ bool Sema::CheckMessageArgumentTypes(
       ExprResult result;
       if (getLangOpts().DebuggerSupport) {
         QualType paramTy; // ignored
-        result = checkUnknownAnyArg(SelLoc, Args[i], paramTy);
+        result = SemaRef.checkUnknownAnyArg(SelLoc, Args[i], paramTy);
       } else {
-        result = DefaultArgumentPromotion(Args[i]);
+        result = SemaRef.DefaultArgumentPromotion(Args[i]);
       }
       if (result.isInvalid())
         return true;
@@ -1835,7 +1852,7 @@ bool Sema::CheckMessageArgumentTypes(
     // from the argument.
     if (param->getType() == Context.UnknownAnyTy) {
       QualType paramType;
-      ExprResult argE = checkUnknownAnyArg(SelLoc, argExpr, paramType);
+      ExprResult argE = SemaRef.checkUnknownAnyArg(SelLoc, argExpr, paramType);
       if (argE.isInvalid()) {
         IsError = true;
       } else {
@@ -1855,14 +1872,15 @@ bool Sema::CheckMessageArgumentTypes(
                     *typeArgs,
                     ObjCSubstitutionContext::Parameter);
 
-    if (RequireCompleteType(argExpr->getSourceRange().getBegin(),
-                            paramType,
-                            diag::err_call_incomplete_argument, argExpr))
+    if (SemaRef.RequireCompleteType(
+            argExpr->getSourceRange().getBegin(), paramType,
+            diag::err_call_incomplete_argument, argExpr))
       return true;
 
     InitializedEntity Entity
       = InitializedEntity::InitializeParameter(Context, param, paramType);
-    ExprResult ArgE = PerformCopyInitialization(Entity, SourceLocation(), argExpr);
+    ExprResult ArgE =
+        SemaRef.PerformCopyInitialization(Entity, SourceLocation(), argExpr);
     if (ArgE.isInvalid())
       IsError = true;
     else {
@@ -1875,7 +1893,7 @@ bool Sema::CheckMessageArgumentTypes(
           Args[i]->getType()->isBlockPointerType() &&
           origParamType->isObjCObjectPointerType()) {
         ExprResult arg = Args[i];
-        maybeExtendBlockObject(arg);
+        SemaRef.maybeExtendBlockObject(arg);
         Args[i] = arg.get();
       }
     }
@@ -1887,8 +1905,8 @@ bool Sema::CheckMessageArgumentTypes(
       if (Args[i]->isTypeDependent())
         continue;
 
-      ExprResult Arg = DefaultVariadicArgumentPromotion(Args[i], VariadicMethod,
-                                                        nullptr);
+      ExprResult Arg = SemaRef.DefaultVariadicArgumentPromotion(
+          Args[i], Sema::VariadicMethod, nullptr);
       IsError |= Arg.isInvalid();
       Args[i] = Arg.get();
     }
@@ -1904,7 +1922,7 @@ bool Sema::CheckMessageArgumentTypes(
     }
   }
 
-  DiagnoseSentinelCalls(Method, SelLoc, Args);
+  SemaRef.DiagnoseSentinelCalls(Method, SelLoc, Args);
 
   // Do additional checkings on method.
   IsError |=
@@ -1913,14 +1931,14 @@ bool Sema::CheckMessageArgumentTypes(
   return IsError;
 }
 
-bool Sema::isSelfExpr(Expr *RExpr) {
+bool SemaObjC::isSelfExpr(Expr *RExpr) {
   // 'self' is objc 'self' in an objc method only.
-  ObjCMethodDecl *Method =
-      dyn_cast_or_null<ObjCMethodDecl>(CurContext->getNonClosureAncestor());
+  ObjCMethodDecl *Method = dyn_cast_or_null<ObjCMethodDecl>(
+      SemaRef.CurContext->getNonClosureAncestor());
   return isSelfExpr(RExpr, Method);
 }
 
-bool Sema::isSelfExpr(Expr *receiver, const ObjCMethodDecl *method) {
+bool SemaObjC::isSelfExpr(Expr *receiver, const ObjCMethodDecl *method) {
   if (!method) return false;
 
   receiver = receiver->IgnoreParenLValueCasts();
@@ -1931,8 +1949,8 @@ bool Sema::isSelfExpr(Expr *receiver, const ObjCMethodDecl *method) {
 }
 
 /// LookupMethodInType - Look up a method in an ObjCObjectType.
-ObjCMethodDecl *Sema::LookupMethodInObjectType(Selector sel, QualType type,
-                                               bool isInstance) {
+ObjCMethodDecl *SemaObjC::LookupMethodInObjectType(Selector sel, QualType type,
+                                                   bool isInstance) {
   const ObjCObjectType *objType = type->castAs<ObjCObjectType>();
   if (ObjCInterfaceDecl *iface = objType->getInterface()) {
     // Look it up in the main interface (and categories, etc.)
@@ -1955,10 +1973,8 @@ ObjCMethodDecl *Sema::LookupMethodInObjectType(Selector sel, QualType type,
 
 /// LookupMethodInQualifiedType - Lookups up a method in protocol qualifier
 /// list of a qualified objective pointer type.
-ObjCMethodDecl *Sema::LookupMethodInQualifiedType(Selector Sel,
-                                              const ObjCObjectPointerType *OPT,
-                                              bool Instance)
-{
+ObjCMethodDecl *SemaObjC::LookupMethodInQualifiedType(
+    Selector Sel, const ObjCObjectPointerType *OPT, bool Instance) {
   ObjCMethodDecl *MD = nullptr;
   for (const auto *PROTO : OPT->quals()) {
     if ((MD = PROTO->lookupMethod(Sel, Instance))) {
@@ -1970,13 +1986,11 @@ ObjCMethodDecl *Sema::LookupMethodInQualifiedType(Selector Sel,
 
 /// HandleExprPropertyRefExpr - Handle foo.bar where foo is a pointer to an
 /// objective C interface.  This is a property reference expression.
-ExprResult Sema::
-HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
-                          Expr *BaseExpr, SourceLocation OpLoc,
-                          DeclarationName MemberName,
-                          SourceLocation MemberLoc,
-                          SourceLocation SuperLoc, QualType SuperType,
-                          bool Super) {
+ExprResult SemaObjC::HandleExprPropertyRefExpr(
+    const ObjCObjectPointerType *OPT, Expr *BaseExpr, SourceLocation OpLoc,
+    DeclarationName MemberName, SourceLocation MemberLoc,
+    SourceLocation SuperLoc, QualType SuperType, bool Super) {
+  ASTContext &Context = getASTContext();
   const ObjCInterfaceType *IFaceT = OPT->getInterfaceType();
   ObjCInterfaceDecl *IFace = IFaceT->getDecl();
 
@@ -1990,15 +2004,15 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
 
   SourceRange BaseRange = Super? SourceRange(SuperLoc)
                                : BaseExpr->getSourceRange();
-  if (RequireCompleteType(MemberLoc, OPT->getPointeeType(),
-                          diag::err_property_not_found_forward_class,
-                          MemberName, BaseRange))
+  if (SemaRef.RequireCompleteType(MemberLoc, OPT->getPointeeType(),
+                                  diag::err_property_not_found_forward_class,
+                                  MemberName, BaseRange))
     return ExprError();
 
   if (ObjCPropertyDecl *PD = IFace->FindPropertyDeclaration(
           Member, ObjCPropertyQueryKind::OBJC_PR_query_instance)) {
     // Check whether we can reference this property.
-    if (DiagnoseUseOfDecl(PD, MemberLoc))
+    if (SemaRef.DiagnoseUseOfDecl(PD, MemberLoc))
       return ExprError();
     if (Super)
       return new (Context)
@@ -2014,7 +2028,7 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
     if (ObjCPropertyDecl *PD = I->FindPropertyDeclaration(
             Member, ObjCPropertyQueryKind::OBJC_PR_query_instance)) {
       // Check whether we can reference this property.
-      if (DiagnoseUseOfDecl(PD, MemberLoc))
+      if (SemaRef.DiagnoseUseOfDecl(PD, MemberLoc))
         return ExprError();
 
       if (Super)
@@ -2032,7 +2046,7 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
   // FIXME: The logic for looking up nullary and unary selectors should be
   // shared with the code in ActOnInstanceMessage.
 
-  Selector Sel = PP.getSelectorTable().getNullarySelector(Member);
+  Selector Sel = SemaRef.PP.getSelectorTable().getNullarySelector(Member);
   ObjCMethodDecl *Getter = IFace->lookupInstanceMethod(Sel);
 
   // May be found in property's qualified list.
@@ -2045,14 +2059,13 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
 
   if (Getter) {
     // Check if we can reference this property.
-    if (DiagnoseUseOfDecl(Getter, MemberLoc))
+    if (SemaRef.DiagnoseUseOfDecl(Getter, MemberLoc))
       return ExprError();
   }
   // If we found a getter then this may be a valid dot-reference, we
   // will look for the matching setter, in case it is needed.
-  Selector SetterSel =
-    SelectorTable::constructSetterSelector(PP.getIdentifierTable(),
-                                           PP.getSelectorTable(), Member);
+  Selector SetterSel = SelectorTable::constructSetterSelector(
+      SemaRef.PP.getIdentifierTable(), SemaRef.PP.getSelectorTable(), Member);
   ObjCMethodDecl *Setter = IFace->lookupInstanceMethod(SetterSel);
 
   // May be found in property's qualified list.
@@ -2065,7 +2078,7 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
     Setter = IFace->lookupPrivateMethod(SetterSel);
   }
 
-  if (Setter && DiagnoseUseOfDecl(Setter, MemberLoc))
+  if (Setter && SemaRef.DiagnoseUseOfDecl(Setter, MemberLoc))
     return ExprError();
 
   // Special warning if member name used in a property-dot for a setter accessor
@@ -2100,9 +2113,9 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
 
   // Attempt to correct for typos in property names.
   DeclFilterCCC<ObjCPropertyDecl> CCC{};
-  if (TypoCorrection Corrected = CorrectTypo(
-          DeclarationNameInfo(MemberName, MemberLoc), LookupOrdinaryName,
-          nullptr, nullptr, CCC, CTK_ErrorRecovery, IFace, false, OPT)) {
+  if (TypoCorrection Corrected = SemaRef.CorrectTypo(
+          DeclarationNameInfo(MemberName, MemberLoc), Sema::LookupOrdinaryName,
+          nullptr, nullptr, CCC, Sema::CTK_ErrorRecovery, IFace, false, OPT)) {
     DeclarationName TypoResult = Corrected.getCorrection();
     if (TypoResult.isIdentifier() &&
         TypoResult.getAsIdentifierInfo() == Member) {
@@ -2120,8 +2133,9 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
           return ExprError();
         }
     } else {
-      diagnoseTypo(Corrected, PDiag(diag::err_property_not_found_suggest)
-                                << MemberName << QualType(OPT, 0));
+      SemaRef.diagnoseTypo(Corrected,
+                           SemaRef.PDiag(diag::err_property_not_found_suggest)
+                               << MemberName << QualType(OPT, 0));
       return HandleExprPropertyRefExpr(OPT, BaseExpr, OpLoc,
                                        TypoResult, MemberLoc,
                                        SuperLoc, SuperType, Super);
@@ -2133,9 +2147,9 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
     QualType T = Ivar->getType();
     if (const ObjCObjectPointerType * OBJPT =
         T->getAsObjCInterfacePointerType()) {
-      if (RequireCompleteType(MemberLoc, OBJPT->getPointeeType(),
-                              diag::err_property_not_as_forward_class,
-                              MemberName, BaseExpr))
+      if (SemaRef.RequireCompleteType(MemberLoc, OBJPT->getPointeeType(),
+                                      diag::err_property_not_as_forward_class,
+                                      MemberName, BaseExpr))
         return ExprError();
     }
     Diag(MemberLoc,
@@ -2153,11 +2167,10 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
   return ExprError();
 }
 
-ExprResult Sema::ActOnClassPropertyRefExpr(const IdentifierInfo &receiverName,
-                                           const IdentifierInfo &propertyName,
-                                           SourceLocation receiverNameLoc,
-                                           SourceLocation propertyNameLoc) {
-
+ExprResult SemaObjC::ActOnClassPropertyRefExpr(
+    const IdentifierInfo &receiverName, const IdentifierInfo &propertyName,
+    SourceLocation receiverNameLoc, SourceLocation propertyNameLoc) {
+  ASTContext &Context = getASTContext();
   const IdentifierInfo *receiverNamePtr = &receiverName;
   ObjCInterfaceDecl *IFace = getObjCInterfaceDecl(receiverNamePtr,
                                                   receiverNameLoc);
@@ -2208,9 +2221,10 @@ ExprResult Sema::ActOnClassPropertyRefExpr(const IdentifierInfo &receiverName,
     GetterSel = PD->getGetterName();
     SetterSel = PD->getSetterName();
   } else {
-    GetterSel = PP.getSelectorTable().getNullarySelector(&propertyName);
+    GetterSel = SemaRef.PP.getSelectorTable().getNullarySelector(&propertyName);
     SetterSel = SelectorTable::constructSetterSelector(
-        PP.getIdentifierTable(), PP.getSelectorTable(), &propertyName);
+        SemaRef.PP.getIdentifierTable(), SemaRef.PP.getSelectorTable(),
+        &propertyName);
   }
 
   // Search for a declared property first.
@@ -2223,7 +2237,7 @@ ExprResult Sema::ActOnClassPropertyRefExpr(const IdentifierInfo &receiverName,
   if (Getter) {
     // FIXME: refactor/share with ActOnMemberReference().
     // Check if we can reference this property.
-    if (DiagnoseUseOfDecl(Getter, propertyNameLoc))
+    if (SemaRef.DiagnoseUseOfDecl(Getter, propertyNameLoc))
       return ExprError();
   }
 
@@ -2238,7 +2252,7 @@ ExprResult Sema::ActOnClassPropertyRefExpr(const IdentifierInfo &receiverName,
   if (!Setter)
     Setter = IFace->getCategoryClassMethod(SetterSel);
 
-  if (Setter && DiagnoseUseOfDecl(Setter, propertyNameLoc))
+  if (Setter && SemaRef.DiagnoseUseOfDecl(Setter, propertyNameLoc))
     return ExprError();
 
   if (Getter || Setter) {
@@ -2278,12 +2292,11 @@ class ObjCInterfaceOrSuperCCC final : public CorrectionCandidateCallback {
 
 } // end anonymous namespace
 
-Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
-                                               IdentifierInfo *Name,
-                                               SourceLocation NameLoc,
-                                               bool IsSuper,
-                                               bool HasTrailingDot,
-                                               ParsedType &ReceiverType) {
+SemaObjC::ObjCMessageKind
+SemaObjC::getObjCMessageKind(Scope *S, IdentifierInfo *Name,
+                             SourceLocation NameLoc, bool IsSuper,
+                             bool HasTrailingDot, ParsedType &ReceiverType) {
+  ASTContext &Context = getASTContext();
   ReceiverType = nullptr;
 
   // If the identifier is "super" and there is no trailing dot, we're
@@ -2292,8 +2305,8 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
   if (IsSuper && S->isInObjcMethodScope())
     return HasTrailingDot? ObjCInstanceMessage : ObjCSuperMessage;
 
-  LookupResult Result(*this, Name, NameLoc, LookupOrdinaryName);
-  LookupName(Result, S);
+  LookupResult Result(SemaRef, Name, NameLoc, Sema::LookupOrdinaryName);
+  SemaRef.LookupName(Result, S);
 
   switch (Result.getResultKind()) {
   case LookupResult::NotFound:
@@ -2301,7 +2314,7 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
     // Objective-C method, look for ivars. If we find one, we're done!
     // FIXME: This is a hack. Ivar lookup should be part of normal
     // lookup.
-    if (ObjCMethodDecl *Method = getCurMethodDecl()) {
+    if (ObjCMethodDecl *Method = SemaRef.getCurMethodDecl()) {
       if (!Method->getClassInterface()) {
         // Fall back: let the parser try to parse it as an instance message.
         return ObjCInstanceMessage;
@@ -2336,7 +2349,7 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
       T = Context.getObjCInterfaceType(Class);
     else if (TypeDecl *Type = dyn_cast<TypeDecl>(ND)) {
       T = Context.getTypeDeclType(Type);
-      DiagnoseUseOfDecl(Type, NameLoc);
+      SemaRef.DiagnoseUseOfDecl(Type, NameLoc);
     }
     else
       return ObjCInstanceMessage;
@@ -2344,30 +2357,30 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
     //  We have a class message, and T is the type we're
     //  messaging. Build source-location information for it.
     TypeSourceInfo *TSInfo = Context.getTrivialTypeSourceInfo(T, NameLoc);
-    ReceiverType = CreateParsedType(T, TSInfo);
+    ReceiverType = SemaRef.CreateParsedType(T, TSInfo);
     return ObjCClassMessage;
   }
   }
 
-  ObjCInterfaceOrSuperCCC CCC(getCurMethodDecl());
-  if (TypoCorrection Corrected = CorrectTypo(
+  ObjCInterfaceOrSuperCCC CCC(SemaRef.getCurMethodDecl());
+  if (TypoCorrection Corrected = SemaRef.CorrectTypo(
           Result.getLookupNameInfo(), Result.getLookupKind(), S, nullptr, CCC,
-          CTK_ErrorRecovery, nullptr, false, nullptr, false)) {
+          Sema::CTK_ErrorRecovery, nullptr, false, nullptr, false)) {
     if (Corrected.isKeyword()) {
       // If we've found the keyword "super" (the only keyword that would be
       // returned by CorrectTypo), this is a send to super.
-      diagnoseTypo(Corrected,
-                   PDiag(diag::err_unknown_receiver_suggest) << Name);
+      SemaRef.diagnoseTypo(
+          Corrected, SemaRef.PDiag(diag::err_unknown_receiver_suggest) << Name);
       return ObjCSuperMessage;
     } else if (ObjCInterfaceDecl *Class =
                    Corrected.getCorrectionDeclAs<ObjCInterfaceDecl>()) {
       // If we found a declaration, correct when it refers to an Objective-C
       // class.
-      diagnoseTypo(Corrected,
-                   PDiag(diag::err_unknown_receiver_suggest) << Name);
+      SemaRef.diagnoseTypo(
+          Corrected, SemaRef.PDiag(diag::err_unknown_receiver_suggest) << Name);
       QualType T = Context.getObjCInterfaceType(Class);
       TypeSourceInfo *TSInfo = Context.getTrivialTypeSourceInfo(T, NameLoc);
-      ReceiverType = CreateParsedType(T, TSInfo);
+      ReceiverType = SemaRef.CreateParsedType(T, TSInfo);
       return ObjCClassMessage;
     }
   }
@@ -2376,13 +2389,12 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
   return ObjCInstanceMessage;
 }
 
-ExprResult Sema::ActOnSuperMessage(Scope *S,
-                                   SourceLocation SuperLoc,
-                                   Selector Sel,
-                                   SourceLocation LBracLoc,
-                                   ArrayRef<SourceLocation> SelectorLocs,
-                                   SourceLocation RBracLoc,
-                                   MultiExprArg Args) {
+ExprResult SemaObjC::ActOnSuperMessage(Scope *S, SourceLocation SuperLoc,
+                                       Selector Sel, SourceLocation LBracLoc,
+                                       ArrayRef<SourceLocation> SelectorLocs,
+                                       SourceLocation RBracLoc,
+                                       MultiExprArg Args) {
+  ASTContext &Context = getASTContext();
   // Determine whether we are inside a method or not.
   ObjCMethodDecl *Method = tryCaptureObjCSelf(SuperLoc);
   if (!Method) {
@@ -2408,7 +2420,7 @@ ExprResult Sema::ActOnSuperMessage(Scope *S,
   // We are in a method whose class has a superclass, so 'super'
   // is acting as a keyword.
   if (Method->getSelector() == Sel)
-    getCurFunction()->ObjCShouldCallSuper = false;
+    SemaRef.getCurFunction()->ObjCShouldCallSuper = false;
 
   if (Method->isInstanceMethod()) {
     // Since we are in an instance method, this is an instance
@@ -2427,12 +2439,12 @@ ExprResult Sema::ActOnSuperMessage(Scope *S,
                            LBracLoc, SelectorLocs, RBracLoc, Args);
 }
 
-ExprResult Sema::BuildClassMessageImplicit(QualType ReceiverType,
-                                           bool isSuperReceiver,
-                                           SourceLocation Loc,
-                                           Selector Sel,
-                                           ObjCMethodDecl *Method,
-                                           MultiExprArg Args) {
+ExprResult SemaObjC::BuildClassMessageImplicit(QualType ReceiverType,
+                                               bool isSuperReceiver,
+                                               SourceLocation Loc, Selector Sel,
+                                               ObjCMethodDecl *Method,
+                                               MultiExprArg Args) {
+  ASTContext &Context = getASTContext();
   TypeSourceInfo *receiverTypeInfo = nullptr;
   if (!ReceiverType.isNull())
     receiverTypeInfo = Context.getTrivialTypeSourceInfo(ReceiverType);
@@ -2456,7 +2468,7 @@ static void applyCocoaAPICheck(Sema &S, const ObjCMessageExpr *Msg,
 
   SourceManager &SM = S.SourceMgr;
   edit::Commit ECommit(SM, S.LangOpts);
-  if (refactor(Msg,*S.NSAPIObj, ECommit)) {
+  if (refactor(Msg, *S.ObjC().NSAPIObj, ECommit)) {
     auto Builder = S.Diag(MsgLoc, DiagID)
                    << Msg->getSelector() << Msg->getSourceRange();
     // FIXME: Don't emit diagnostic at all if fixits are non-commitable.
@@ -2603,16 +2615,12 @@ DiagnoseCStringFormatDirectiveInObjCAPI(Sema &S,
 /// \param RBracLoc The location of the closing square bracket ']'.
 ///
 /// \param ArgsIn The message arguments.
-ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
-                                   QualType ReceiverType,
-                                   SourceLocation SuperLoc,
-                                   Selector Sel,
-                                   ObjCMethodDecl *Method,
-                                   SourceLocation LBracLoc,
-                                   ArrayRef<SourceLocation> SelectorLocs,
-                                   SourceLocation RBracLoc,
-                                   MultiExprArg ArgsIn,
-                                   bool isImplicit) {
+ExprResult SemaObjC::BuildClassMessage(
+    TypeSourceInfo *ReceiverTypeInfo, QualType ReceiverType,
+    SourceLocation SuperLoc, Selector Sel, ObjCMethodDecl *Method,
+    SourceLocation LBracLoc, ArrayRef<SourceLocation> SelectorLocs,
+    SourceLocation RBracLoc, MultiExprArg ArgsIn, bool isImplicit) {
+  ASTContext &Context = getASTContext();
   SourceLocation Loc = SuperLoc.isValid()? SuperLoc
     : ReceiverTypeInfo->getTypeLoc().getSourceRange().getBegin();
   if (LBracLoc.isInvalid()) {
@@ -2650,17 +2658,17 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
   assert(Class && "We don't know which class we're messaging?");
   // objc++ diagnoses during typename annotation.
   if (!getLangOpts().CPlusPlus)
-    (void)DiagnoseUseOfDecl(Class, SelectorSlotLocs);
+    (void)SemaRef.DiagnoseUseOfDecl(Class, SelectorSlotLocs);
   // Find the method we are messaging.
   if (!Method) {
     SourceRange TypeRange
       = SuperLoc.isValid()? SourceRange(SuperLoc)
                           : ReceiverTypeInfo->getTypeLoc().getSourceRange();
-    if (RequireCompleteType(Loc, Context.getObjCInterfaceType(Class),
-                            (getLangOpts().ObjCAutoRefCount
-                               ? diag::err_arc_receiver_forward_class
-                               : diag::warn_receiver_forward_class),
-                            TypeRange)) {
+    if (SemaRef.RequireCompleteType(Loc, Context.getObjCInterfaceType(Class),
+                                    (getLangOpts().ObjCAutoRefCount
+                                         ? diag::err_arc_receiver_forward_class
+                                         : diag::warn_receiver_forward_class),
+                                    TypeRange)) {
       // A forward class used in messaging is treated as a 'Class'
       Method = LookupFactoryMethodInGlobalPool(Sel,
                                                SourceRange(LBracLoc, RBracLoc));
@@ -2675,8 +2683,8 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
     if (!Method)
       Method = Class->lookupPrivateClassMethod(Sel);
 
-    if (Method && DiagnoseUseOfDecl(Method, SelectorSlotLocs,
-                                    nullptr, false, false, Class))
+    if (Method && SemaRef.DiagnoseUseOfDecl(Method, SelectorSlotLocs, nullptr,
+                                            false, false, Class))
       return ExprError();
   }
 
@@ -2693,8 +2701,9 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
     return ExprError();
 
   if (Method && !Method->getReturnType()->isVoidType() &&
-      RequireCompleteType(LBracLoc, Method->getReturnType(),
-                          diag::err_illegal_message_expr_incomplete_type))
+      SemaRef.RequireCompleteType(
+          LBracLoc, Method->getReturnType(),
+          diag::err_illegal_message_expr_incomplete_type))
     return ExprError();
 
   if (Method && Method->isDirectMethod() && SuperLoc.isValid()) {
@@ -2717,8 +2726,7 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
         Diag(Method->getLocation(), diag::note_method_declared_at)
           << Method->getDeclName();
       }
-    }
-    else if (ObjCMethodDecl *CurMeth = getCurMethodDecl()) {
+    } else if (ObjCMethodDecl *CurMeth = SemaRef.getCurMethodDecl()) {
       // [super initialize] is allowed only within an +initialize implementation
       if (CurMeth->getMethodFamily() != OMF_initialize) {
         Diag(Loc, diag::warn_direct_super_initialize_call);
@@ -2730,7 +2738,7 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
     }
   }
 
-  DiagnoseCStringFormatDirectiveInObjCAPI(*this, Method, Sel, Args, NumArgs);
+  DiagnoseCStringFormatDirectiveInObjCAPI(SemaRef, Method, Sel, Args, NumArgs);
 
   // Construct the appropriate ObjCMessageExpr.
   ObjCMessageExpr *Result;
@@ -2744,26 +2752,26 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
         Context, ReturnType, VK, LBracLoc, ReceiverTypeInfo, Sel, SelectorLocs,
         Method, ArrayRef(Args, NumArgs), RBracLoc, isImplicit);
     if (!isImplicit)
-      checkCocoaAPI(*this, Result);
+      checkCocoaAPI(SemaRef, Result);
   }
   if (Method)
-    checkFoundationAPI(*this, SelLoc, Method, ArrayRef(Args, NumArgs),
+    checkFoundationAPI(SemaRef, SelLoc, Method, ArrayRef(Args, NumArgs),
                        ReceiverType, /*IsClassObjectCall=*/true);
-  return MaybeBindToTemporary(Result);
+  return SemaRef.MaybeBindToTemporary(Result);
 }
 
 // ActOnClassMessage - used for both unary and keyword messages.
 // ArgExprs is optional - if it is present, the number of expressions
 // is obtained from Sel.getNumArgs().
-ExprResult Sema::ActOnClassMessage(Scope *S,
-                                   ParsedType Receiver,
-                                   Selector Sel,
-                                   SourceLocation LBracLoc,
-                                   ArrayRef<SourceLocation> SelectorLocs,
-                                   SourceLocation RBracLoc,
-                                   MultiExprArg Args) {
+ExprResult SemaObjC::ActOnClassMessage(Scope *S, ParsedType Receiver,
+                                       Selector Sel, SourceLocation LBracLoc,
+                                       ArrayRef<SourceLocation> SelectorLocs,
+                                       SourceLocation RBracLoc,
+                                       MultiExprArg Args) {
+  ASTContext &Context = getASTContext();
   TypeSourceInfo *ReceiverTypeInfo;
-  QualType ReceiverType = GetTypeFromParser(Receiver, &ReceiverTypeInfo);
+  QualType ReceiverType =
+      SemaRef.GetTypeFromParser(Receiver, &ReceiverTypeInfo);
   if (ReceiverType.isNull())
     return ExprError();
 
@@ -2776,12 +2784,9 @@ ExprResult Sema::ActOnClassMessage(Scope *S,
                            Args);
 }
 
-ExprResult Sema::BuildInstanceMessageImplicit(Expr *Receiver,
-                                              QualType ReceiverType,
-                                              SourceLocation Loc,
-                                              Selector Sel,
-                                              ObjCMethodDecl *Method,
-                                              MultiExprArg Args) {
+ExprResult SemaObjC::BuildInstanceMessageImplicit(
+    Expr *Receiver, QualType ReceiverType, SourceLocation Loc, Selector Sel,
+    ObjCMethodDecl *Method, MultiExprArg Args) {
   return BuildInstanceMessage(Receiver, ReceiverType,
                               /*SuperLoc=*/!Receiver ? Loc : SourceLocation(),
                               Sel, Method, Loc, Loc, Loc, Args,
@@ -2789,12 +2794,13 @@ ExprResult Sema::BuildInstanceMessageImplicit(Expr *Receiver,
 }
 
 static bool isMethodDeclaredInRootProtocol(Sema &S, const ObjCMethodDecl *M) {
-  if (!S.NSAPIObj)
+  if (!S.ObjC().NSAPIObj)
     return false;
   const auto *Protocol = dyn_cast<ObjCProtocolDecl>(M->getDeclContext());
   if (!Protocol)
     return false;
-  const IdentifierInfo *II = S.NSAPIObj->getNSClassId(NSAPI::ClassId_NSObject);
+  const IdentifierInfo *II =
+      S.ObjC().NSAPIObj->getNSClassId(NSAPI::ClassId_NSObject);
   if (const auto *RootClass = dyn_cast_or_null<ObjCInterfaceDecl>(
           S.LookupSingleName(S.TUScope, II, Protocol->getBeginLoc(),
                              Sema::LookupOrdinaryName))) {
@@ -2834,19 +2840,15 @@ static bool isMethodDeclaredInRootProtocol(Sema &S, const ObjCMethodDecl *M) {
 /// \param RBracLoc The location of the closing square bracket ']'.
 ///
 /// \param ArgsIn The message arguments.
-ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
-                                      QualType ReceiverType,
-                                      SourceLocation SuperLoc,
-                                      Selector Sel,
-                                      ObjCMethodDecl *Method,
-                                      SourceLocation LBracLoc,
-                                      ArrayRef<SourceLocation> SelectorLocs,
-                                      SourceLocation RBracLoc,
-                                      MultiExprArg ArgsIn,
-                                      bool isImplicit) {
+ExprResult SemaObjC::BuildInstanceMessage(
+    Expr *Receiver, QualType ReceiverType, SourceLocation SuperLoc,
+    Selector Sel, ObjCMethodDecl *Method, SourceLocation LBracLoc,
+    ArrayRef<SourceLocation> SelectorLocs, SourceLocation RBracLoc,
+    MultiExprArg ArgsIn, bool isImplicit) {
   assert((Receiver || SuperLoc.isValid()) && "If the Receiver is null, the "
                                              "SuperLoc must be valid so we can "
                                              "use it instead.");
+  ASTContext &Context = getASTContext();
 
   // The location of the receiver.
   SourceLocation Loc = SuperLoc.isValid() ? SuperLoc : Receiver->getBeginLoc();
@@ -2871,9 +2873,10 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
     if (Receiver->hasPlaceholderType()) {
       ExprResult Result;
       if (Receiver->getType() == Context.UnknownAnyTy)
-        Result = forceUnknownAnyToType(Receiver, Context.getObjCIdType());
+        Result =
+            SemaRef.forceUnknownAnyToType(Receiver, Context.getObjCIdType());
       else
-        Result = CheckPlaceholderExpr(Receiver);
+        Result = SemaRef.CheckPlaceholderExpr(Receiver);
       if (Result.isInvalid()) return ExprError();
       Receiver = Result.get();
     }
@@ -2892,7 +2895,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
 
     // If necessary, apply function/array conversion to the receiver.
     // C99 6.7.5.3p[7,8].
-    ExprResult Result = DefaultFunctionArrayLvalueConversion(Receiver);
+    ExprResult Result = SemaRef.DefaultFunctionArrayLvalueConversion(Receiver);
     if (Result.isInvalid())
       return ExprError();
     Receiver = Result.get();
@@ -2911,24 +2914,28 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
       // But not in ARC.
       Diag(Loc, diag::warn_bad_receiver_type) << ReceiverType << RecRange;
       if (ReceiverType->isPointerType()) {
-        Receiver = ImpCastExprToType(Receiver, Context.getObjCIdType(),
-                                     CK_CPointerToObjCPointerCast).get();
+        Receiver = SemaRef
+                       .ImpCastExprToType(Receiver, Context.getObjCIdType(),
+                                          CK_CPointerToObjCPointerCast)
+                       .get();
       } else {
         // TODO: specialized warning on null receivers?
         bool IsNull = Receiver->isNullPointerConstant(Context,
                                               Expr::NPC_ValueDependentIsNull);
         CastKind Kind = IsNull ? CK_NullToPointer : CK_IntegralToPointer;
-        Receiver = ImpCastExprToType(Receiver, Context.getObjCIdType(),
-                                     Kind).get();
+        Receiver =
+            SemaRef.ImpCastExprToType(Receiver, Context.getObjCIdType(), Kind)
+                .get();
       }
       ReceiverType = Receiver->getType();
     } else if (getLangOpts().CPlusPlus) {
       // The receiver must be a complete type.
-      if (RequireCompleteType(Loc, Receiver->getType(),
-                              diag::err_incomplete_receiver_type))
+      if (SemaRef.RequireCompleteType(Loc, Receiver->getType(),
+                                      diag::err_incomplete_receiver_type))
         return ExprError();
 
-      ExprResult result = PerformContextuallyConvertToObjCPointer(Receiver);
+      ExprResult result =
+          SemaRef.PerformContextuallyConvertToObjCPointer(Receiver);
       if (result.isUsable()) {
         Receiver = result.get();
         ReceiverType = Receiver->getType();
@@ -2957,14 +2964,14 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
         // select a better one.
         Method = Methods[0];
 
-        if (ObjCMethodDecl *BestMethod =
-            SelectBestMethod(Sel, ArgsIn, Method->isInstanceMethod(), Methods))
+        if (ObjCMethodDecl *BestMethod = SemaRef.SelectBestMethod(
+                Sel, ArgsIn, Method->isInstanceMethod(), Methods))
           Method = BestMethod;
 
         if (!AreMultipleMethodsInGlobalPool(Sel, Method,
                                             SourceRange(LBracLoc, RBracLoc),
                                             receiverIsIdLike, Methods))
-          DiagnoseUseOfDecl(Method, SelectorSlotLocs);
+          SemaRef.DiagnoseUseOfDecl(Method, SelectorSlotLocs);
       }
     } else if (ReceiverType->isObjCClassOrClassKindOfType() ||
                ReceiverType->isObjCQualifiedClassType()) {
@@ -2980,7 +2987,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
         if (!Method) {
           Method = LookupMethodInQualifiedType(Sel, QClassTy, true);
           // warn if instance method found for a Class message.
-          if (Method && !isMethodDeclaredInRootProtocol(*this, Method)) {
+          if (Method && !isMethodDeclaredInRootProtocol(SemaRef, Method)) {
             Diag(SelLoc, diag::warn_instance_method_on_class_found)
               << Method->getSelector() << Sel;
             Diag(Method->getLocation(), diag::note_method_declared_at)
@@ -2988,7 +2995,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
           }
         }
       } else {
-        if (ObjCMethodDecl *CurMeth = getCurMethodDecl()) {
+        if (ObjCMethodDecl *CurMeth = SemaRef.getCurMethodDecl()) {
           if (ObjCInterfaceDecl *ClassDecl = CurMeth->getClassInterface()) {
             // As a guess, try looking for the method in the current interface.
             // This very well may not produce the "right" method.
@@ -2999,7 +3006,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
             if (!Method)
               Method = ClassDecl->lookupPrivateClassMethod(Sel);
 
-            if (Method && DiagnoseUseOfDecl(Method, SelectorSlotLocs))
+            if (Method && SemaRef.DiagnoseUseOfDecl(Method, SelectorSlotLocs))
               return ExprError();
           }
         }
@@ -3027,10 +3034,9 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
                 }
               }
 
-             if (ObjCMethodDecl *BestMethod =
-                 SelectBestMethod(Sel, ArgsIn, Method->isInstanceMethod(),
-                                  Methods))
-               Method = BestMethod;
+              if (ObjCMethodDecl *BestMethod = SemaRef.SelectBestMethod(
+                      Sel, ArgsIn, Method->isInstanceMethod(), Methods))
+                Method = BestMethod;
             }
           }
         }
@@ -3047,7 +3053,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
         Method = LookupMethodInQualifiedType(Sel, QIdTy, true);
         if (!Method)
           Method = LookupMethodInQualifiedType(Sel, QIdTy, false);
-        if (Method && DiagnoseUseOfDecl(Method, SelectorSlotLocs))
+        if (Method && SemaRef.DiagnoseUseOfDecl(Method, SelectorSlotLocs))
           return ExprError();
       } else if (const ObjCObjectPointerType *OCIType
                    = ReceiverType->getAsObjCInterfacePointerType()) {
@@ -3059,11 +3065,12 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
         // FIXME: In the non-ARC case, this will still be a hard error if the
         // definition is found in a module that's not visible.
         const ObjCInterfaceDecl *forwardClass = nullptr;
-        if (RequireCompleteType(Loc, OCIType->getPointeeType(),
-                                getLangOpts().ObjCAutoRefCount
-                                    ? diag::err_arc_receiver_forward_instance
-                                    : diag::warn_receiver_forward_instance,
-                                RecRange)) {
+        if (SemaRef.RequireCompleteType(
+                Loc, OCIType->getPointeeType(),
+                getLangOpts().ObjCAutoRefCount
+                    ? diag::err_arc_receiver_forward_instance
+                    : diag::warn_receiver_forward_instance,
+                RecRange)) {
           if (getLangOpts().ObjCAutoRefCount)
             return ExprError();
 
@@ -3104,9 +3111,8 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
                 // to select a better one.
                 Method = Methods[0];
 
-                if (ObjCMethodDecl *BestMethod =
-                    SelectBestMethod(Sel, ArgsIn, Method->isInstanceMethod(),
-                                     Methods))
+                if (ObjCMethodDecl *BestMethod = SemaRef.SelectBestMethod(
+                        Sel, ArgsIn, Method->isInstanceMethod(), Methods))
                   Method = BestMethod;
 
                 AreMultipleMethodsInGlobalPool(Sel, Method,
@@ -3121,7 +3127,8 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
             }
           }
         }
-        if (Method && DiagnoseUseOfDecl(Method, SelectorSlotLocs, forwardClass))
+        if (Method &&
+            SemaRef.DiagnoseUseOfDecl(Method, SelectorSlotLocs, forwardClass))
           return ExprError();
       } else {
         // Reject other random receiver types (e.g. structs).
@@ -3132,8 +3139,9 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
   }
 
   FunctionScopeInfo *DIFunctionScopeInfo =
-    (Method && Method->getMethodFamily() == OMF_init)
-      ? getEnclosingFunction() : nullptr;
+      (Method && Method->getMethodFamily() == OMF_init)
+          ? SemaRef.getEnclosingFunction()
+          : nullptr;
 
   if (Method && Method->isDirectMethod()) {
     if (ReceiverType->isObjCIdType() && !isImplicit) {
@@ -3199,7 +3207,8 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
     if (!isDesignatedInitChain) {
       const ObjCMethodDecl *InitMethod = nullptr;
       bool isDesignated =
-        getCurMethodDecl()->isDesignatedInitializerForTheInterface(&InitMethod);
+          SemaRef.getCurMethodDecl()->isDesignatedInitializerForTheInterface(
+              &InitMethod);
       assert(isDesignated && InitMethod);
       (void)isDesignated;
       Diag(SelLoc, SuperLoc.isValid() ?
@@ -3234,8 +3243,9 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
     return ExprError();
 
   if (Method && !Method->getReturnType()->isVoidType() &&
-      RequireCompleteType(LBracLoc, Method->getReturnType(),
-                          diag::err_illegal_message_expr_incomplete_type))
+      SemaRef.RequireCompleteType(
+          LBracLoc, Method->getReturnType(),
+          diag::err_illegal_message_expr_incomplete_type))
     return ExprError();
 
   // In ARC, forbid the user from sending messages to
@@ -3319,7 +3329,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
     }
   }
 
-  DiagnoseCStringFormatDirectiveInObjCAPI(*this, Method, Sel, Args, NumArgs);
+  DiagnoseCStringFormatDirectiveInObjCAPI(SemaRef, Method, Sel, Args, NumArgs);
 
   // Construct the appropriate ObjCMessageExpr instance.
   ObjCMessageExpr *Result;
@@ -3333,7 +3343,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
         Context, ReturnType, VK, LBracLoc, Receiver, Sel, SelectorLocs, Method,
         ArrayRef(Args, NumArgs), RBracLoc, isImplicit);
     if (!isImplicit)
-      checkCocoaAPI(*this, Result);
+      checkCocoaAPI(SemaRef, Result);
   }
   if (Method) {
     bool IsClassObjectCall = ClassMessage;
@@ -3344,7 +3354,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
     if (Receiver && isSelfExpr(Receiver)) {
       if (const auto *OPT = ReceiverType->getAs<ObjCObjectPointerType>()) {
         if (OPT->getObjectType()->isObjCClass()) {
-          if (const auto *CurMeth = getCurMethodDecl()) {
+          if (const auto *CurMeth = SemaRef.getCurMethodDecl()) {
             IsClassObjectCall = true;
             ReceiverType =
                 Context.getObjCInterfaceType(CurMeth->getClassInterface());
@@ -3352,7 +3362,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
         }
       }
     }
-    checkFoundationAPI(*this, SelLoc, Method, ArrayRef(Args, NumArgs),
+    checkFoundationAPI(SemaRef, SelLoc, Method, ArrayRef(Args, NumArgs),
                        ReceiverType, IsClassObjectCall);
   }
 
@@ -3362,7 +3372,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
         (SuperLoc.isValid() || isSelfExpr(Receiver))) {
       // Only consider init calls *directly* in init implementations,
       // not within blocks.
-      ObjCMethodDecl *method = dyn_cast<ObjCMethodDecl>(CurContext);
+      ObjCMethodDecl *method = dyn_cast<ObjCMethodDecl>(SemaRef.CurContext);
       if (method && method->getMethodFamily() == OMF_init) {
         // The implicit assignment to self means we also don't want to
         // consume the result.
@@ -3383,19 +3393,20 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
             Prop->getPropertyAttributes() & ObjCPropertyAttribute::kind_weak;
         if (!IsWeak && Sel.isUnarySelector())
           IsWeak = ReturnType.getObjCLifetime() & Qualifiers::OCL_Weak;
-        if (IsWeak && !isUnevaluatedContext() &&
-            !Diags.isIgnored(diag::warn_arc_repeated_use_of_weak, LBracLoc))
-          getCurFunction()->recordUseOfWeak(Result, Prop);
+        if (IsWeak && !SemaRef.isUnevaluatedContext() &&
+            !getDiagnostics().isIgnored(diag::warn_arc_repeated_use_of_weak,
+                                        LBracLoc))
+          SemaRef.getCurFunction()->recordUseOfWeak(Result, Prop);
       }
     }
   }
 
   CheckObjCCircularContainer(Result);
 
-  return MaybeBindToTemporary(Result);
+  return SemaRef.MaybeBindToTemporary(Result);
 }
 
-static void RemoveSelectorFromWarningCache(Sema &S, Expr* Arg) {
+static void RemoveSelectorFromWarningCache(SemaObjC &S, Expr *Arg) {
   if (ObjCSelectorExpr *OSE =
       dyn_cast<ObjCSelectorExpr>(Arg->IgnoreParenCasts())) {
     Selector Sel = OSE->getSelector();
@@ -3409,19 +3420,19 @@ static void RemoveSelectorFromWarningCache(Sema &S, Expr* Arg) {
 // ActOnInstanceMessage - used for both unary and keyword messages.
 // ArgExprs is optional - if it is present, the number of expressions
 // is obtained from Sel.getNumArgs().
-ExprResult Sema::ActOnInstanceMessage(Scope *S,
-                                      Expr *Receiver,
-                                      Selector Sel,
-                                      SourceLocation LBracLoc,
-                                      ArrayRef<SourceLocation> SelectorLocs,
-                                      SourceLocation RBracLoc,
-                                      MultiExprArg Args) {
+ExprResult SemaObjC::ActOnInstanceMessage(Scope *S, Expr *Receiver,
+                                          Selector Sel, SourceLocation LBracLoc,
+                                          ArrayRef<SourceLocation> SelectorLocs,
+                                          SourceLocation RBracLoc,
+                                          MultiExprArg Args) {
+  ASTContext &Context = getASTContext();
   if (!Receiver)
     return ExprError();
 
   // A ParenListExpr can show up while doing error recovery with invalid code.
   if (isa<ParenListExpr>(Receiver)) {
-    ExprResult Result = MaybeConvertParenListExprToParenExpr(S, Receiver);
+    ExprResult Result =
+        SemaRef.MaybeConvertParenListExprToParenExpr(S, Receiver);
     if (Result.isInvalid()) return ExprError();
     Receiver = Result.get();
   }
@@ -3735,12 +3746,13 @@ namespace {
   };
 } // end anonymous namespace
 
-bool Sema::isKnownName(StringRef name) {
+bool SemaObjC::isKnownName(StringRef name) {
+  ASTContext &Context = getASTContext();
   if (name.empty())
     return false;
-  LookupResult R(*this, &Context.Idents.get(name), SourceLocation(),
+  LookupResult R(SemaRef, &Context.Idents.get(name), SourceLocation(),
                  Sema::LookupOrdinaryName);
-  return LookupName(R, TUScope, false);
+  return SemaRef.LookupName(R, SemaRef.TUScope, false);
 }
 
 template <typename DiagBuilderT>
@@ -3921,7 +3933,7 @@ static void diagnoseObjCARCConversion(Sema &S, SourceRange castRange,
       << castType
       << castRange
       << castExpr->getSourceRange();
-    bool br = S.isKnownName("CFBridgingRelease");
+    bool br = S.ObjC().isKnownName("CFBridgingRelease");
     ACCResult CreateRule =
       ARCCastChecker(S.Context, exprACTC, castACTC, true).Visit(castExpr);
     assert(CreateRule != ACC_bottom && "This cast should already be accepted.");
@@ -3954,7 +3966,7 @@ static void diagnoseObjCARCConversion(Sema &S, SourceRange castRange,
 
   // Bridge from a CF type to an ARC type.
   if (exprACTC == ACTC_retainable && isAnyRetainable(castACTC)) {
-    bool br = S.isKnownName("CFBridgingRetain");
+    bool br = S.ObjC().isKnownName("CFBridgingRetain");
     S.Diag(loc, diag::err_arc_cast_requires_bridge)
       << convKindForDiag
       << unsigned(castExprType->isBlockPointerType()) // of ObjC|block type
@@ -4130,7 +4142,7 @@ static bool CheckObjCBridgeCFCast(Sema &S, QualType castType, Expr *castExpr,
   return true;
 }
 
-void Sema::CheckTollFreeBridgeCast(QualType castType, Expr *castExpr) {
+void SemaObjC::CheckTollFreeBridgeCast(QualType castType, Expr *castExpr) {
   if (!getLangOpts().ObjC)
     return;
   // warn in presence of __bridge casting to or from a toll free bridge cast.
@@ -4138,49 +4150,47 @@ void Sema::CheckTollFreeBridgeCast(QualType castType, Expr *castExpr) {
   ARCConversionTypeClass castACTC = classifyTypeForARCConversion(castType);
   if (castACTC == ACTC_retainable && exprACTC == ACTC_coreFoundation) {
     bool HasObjCBridgeAttr;
-    bool ObjCBridgeAttrWillNotWarn =
-      CheckObjCBridgeNSCast<ObjCBridgeAttr>(*this, castType, castExpr, HasObjCBridgeAttr,
-                                            false);
+    bool ObjCBridgeAttrWillNotWarn = CheckObjCBridgeNSCast<ObjCBridgeAttr>(
+        SemaRef, castType, castExpr, HasObjCBridgeAttr, false);
     if (ObjCBridgeAttrWillNotWarn && HasObjCBridgeAttr)
       return;
     bool HasObjCBridgeMutableAttr;
     bool ObjCBridgeMutableAttrWillNotWarn =
-      CheckObjCBridgeNSCast<ObjCBridgeMutableAttr>(*this, castType, castExpr,
-                                                   HasObjCBridgeMutableAttr, false);
+        CheckObjCBridgeNSCast<ObjCBridgeMutableAttr>(
+            SemaRef, castType, castExpr, HasObjCBridgeMutableAttr, false);
     if (ObjCBridgeMutableAttrWillNotWarn && HasObjCBridgeMutableAttr)
       return;
 
     if (HasObjCBridgeAttr)
-      CheckObjCBridgeNSCast<ObjCBridgeAttr>(*this, castType, castExpr, HasObjCBridgeAttr,
-                                            true);
+      CheckObjCBridgeNSCast<ObjCBridgeAttr>(SemaRef, castType, castExpr,
+                                            HasObjCBridgeAttr, true);
     else if (HasObjCBridgeMutableAttr)
-      CheckObjCBridgeNSCast<ObjCBridgeMutableAttr>(*this, castType, castExpr,
-                                                   HasObjCBridgeMutableAttr, true);
+      CheckObjCBridgeNSCast<ObjCBridgeMutableAttr>(
+          SemaRef, castType, castExpr, HasObjCBridgeMutableAttr, true);
   }
   else if (castACTC == ACTC_coreFoundation && exprACTC == ACTC_retainable) {
     bool HasObjCBridgeAttr;
-    bool ObjCBridgeAttrWillNotWarn =
-      CheckObjCBridgeCFCast<ObjCBridgeAttr>(*this, castType, castExpr, HasObjCBridgeAttr,
-                                            false);
+    bool ObjCBridgeAttrWillNotWarn = CheckObjCBridgeCFCast<ObjCBridgeAttr>(
+        SemaRef, castType, castExpr, HasObjCBridgeAttr, false);
     if (ObjCBridgeAttrWillNotWarn && HasObjCBridgeAttr)
       return;
     bool HasObjCBridgeMutableAttr;
     bool ObjCBridgeMutableAttrWillNotWarn =
-      CheckObjCBridgeCFCast<ObjCBridgeMutableAttr>(*this, castType, castExpr,
-                                                   HasObjCBridgeMutableAttr, false);
+        CheckObjCBridgeCFCast<ObjCBridgeMutableAttr>(
+            SemaRef, castType, castExpr, HasObjCBridgeMutableAttr, false);
     if (ObjCBridgeMutableAttrWillNotWarn && HasObjCBridgeMutableAttr)
       return;
 
     if (HasObjCBridgeAttr)
-      CheckObjCBridgeCFCast<ObjCBridgeAttr>(*this, castType, castExpr, HasObjCBridgeAttr,
-                                            true);
+      CheckObjCBridgeCFCast<ObjCBridgeAttr>(SemaRef, castType, castExpr,
+                                            HasObjCBridgeAttr, true);
     else if (HasObjCBridgeMutableAttr)
-      CheckObjCBridgeCFCast<ObjCBridgeMutableAttr>(*this, castType, castExpr,
-                                                   HasObjCBridgeMutableAttr, true);
+      CheckObjCBridgeCFCast<ObjCBridgeMutableAttr>(
+          SemaRef, castType, castExpr, HasObjCBridgeMutableAttr, true);
   }
 }
 
-void Sema::CheckObjCBridgeRelatedCast(QualType castType, Expr *castExpr) {
+void SemaObjC::CheckObjCBridgeRelatedCast(QualType castType, Expr *castExpr) {
   QualType SrcType = castExpr->getType();
   if (ObjCPropertyRefExpr *PRE = dyn_cast<ObjCPropertyRefExpr>(castExpr)) {
     if (PRE->isExplicitProperty()) {
@@ -4201,8 +4211,8 @@ void Sema::CheckObjCBridgeRelatedCast(QualType castType, Expr *castExpr) {
                                     castExpr);
 }
 
-bool Sema::CheckTollFreeBridgeStaticCast(QualType castType, Expr *castExpr,
-                                         CastKind &Kind) {
+bool SemaObjC::CheckTollFreeBridgeStaticCast(QualType castType, Expr *castExpr,
+                                             CastKind &Kind) {
   if (!getLangOpts().ObjC)
     return false;
   ARCConversionTypeClass exprACTC =
@@ -4218,13 +4228,12 @@ bool Sema::CheckTollFreeBridgeStaticCast(QualType castType, Expr *castExpr,
   return false;
 }
 
-bool Sema::checkObjCBridgeRelatedComponents(SourceLocation Loc,
-                                            QualType DestType, QualType SrcType,
-                                            ObjCInterfaceDecl *&RelatedClass,
-                                            ObjCMethodDecl *&ClassMethod,
-                                            ObjCMethodDecl *&InstanceMethod,
-                                            TypedefNameDecl *&TDNDecl,
-                                            bool CfToNs, bool Diagnose) {
+bool SemaObjC::checkObjCBridgeRelatedComponents(
+    SourceLocation Loc, QualType DestType, QualType SrcType,
+    ObjCInterfaceDecl *&RelatedClass, ObjCMethodDecl *&ClassMethod,
+    ObjCMethodDecl *&InstanceMethod, TypedefNameDecl *&TDNDecl, bool CfToNs,
+    bool Diagnose) {
+  ASTContext &Context = getASTContext();
   QualType T = CfToNs ? SrcType : DestType;
   ObjCBridgeRelatedAttr *ObjCBAttr = ObjCBridgeRelatedAttrFromType(T, TDNDecl);
   if (!ObjCBAttr)
@@ -4237,9 +4246,9 @@ bool Sema::checkObjCBridgeRelatedComponents(SourceLocation Loc,
     return false;
   NamedDecl *Target = nullptr;
   // Check for an existing type with this name.
-  LookupResult R(*this, DeclarationName(RCId), SourceLocation(),
+  LookupResult R(SemaRef, DeclarationName(RCId), SourceLocation(),
                  Sema::LookupOrdinaryName);
-  if (!LookupName(R, TUScope)) {
+  if (!SemaRef.LookupName(R, SemaRef.TUScope)) {
     if (Diagnose) {
       Diag(Loc, diag::err_objc_bridged_related_invalid_class) << RCId
             << SrcType << DestType;
@@ -4291,10 +4300,12 @@ bool Sema::checkObjCBridgeRelatedComponents(SourceLocation Loc,
   return true;
 }
 
-bool
-Sema::CheckObjCBridgeRelatedConversions(SourceLocation Loc,
-                                        QualType DestType, QualType SrcType,
-                                        Expr *&SrcExpr, bool Diagnose) {
+bool SemaObjC::CheckObjCBridgeRelatedConversions(SourceLocation Loc,
+                                                 QualType DestType,
+                                                 QualType SrcType,
+                                                 Expr *&SrcExpr,
+                                                 bool Diagnose) {
+  ASTContext &Context = getASTContext();
   ARCConversionTypeClass rhsExprACTC = classifyTypeForARCConversion(SrcType);
   ARCConversionTypeClass lhsExprACTC = classifyTypeForARCConversion(DestType);
   bool CfToNs = (rhsExprACTC == ACTC_coreFoundation && lhsExprACTC == ACTC_retainable);
@@ -4320,7 +4331,7 @@ Sema::CheckObjCBridgeRelatedConversions(SourceLocation Loc,
         ExpressionString += " ";
         ExpressionString += ClassMethod->getSelector().getAsString();
         SourceLocation SrcExprEndLoc =
-            getLocForEndOfToken(SrcExpr->getEndLoc());
+            SemaRef.getLocForEndOfToken(SrcExpr->getEndLoc());
         // Provide a fixit: [RelatedClass ClassMethod SrcExpr]
         Diag(Loc, diag::err_objc_bridged_related_known_method)
             << SrcType << DestType << ClassMethod->getSelector() << false
@@ -4348,7 +4359,7 @@ Sema::CheckObjCBridgeRelatedConversions(SourceLocation Loc,
       if (Diagnose) {
         std::string ExpressionString;
         SourceLocation SrcExprEndLoc =
-            getLocForEndOfToken(SrcExpr->getEndLoc());
+            SemaRef.getLocForEndOfToken(SrcExpr->getEndLoc());
         if (InstanceMethod->isPropertyAccessor())
           if (const ObjCPropertyDecl *PDecl =
                   InstanceMethod->findPropertyDecl()) {
@@ -4384,11 +4395,12 @@ Sema::CheckObjCBridgeRelatedConversions(SourceLocation Loc,
   return false;
 }
 
-Sema::ARCConversionResult
-Sema::CheckObjCConversion(SourceRange castRange, QualType castType,
-                          Expr *&castExpr, CheckedConversionKind CCK,
-                          bool Diagnose, bool DiagnoseCFAudited,
-                          BinaryOperatorKind Opc) {
+SemaObjC::ARCConversionResult
+SemaObjC::CheckObjCConversion(SourceRange castRange, QualType castType,
+                              Expr *&castExpr, CheckedConversionKind CCK,
+                              bool Diagnose, bool DiagnoseCFAudited,
+                              BinaryOperatorKind Opc) {
+  ASTContext &Context = getASTContext();
   QualType castExprType = castExpr->getType();
 
   // For the purposes of the classification, we assume reference types
@@ -4449,11 +4461,11 @@ Sema::CheckObjCConversion(SourceRange castRange, QualType castType,
   // pointers too, but only when the conversions are explicit.
   if (exprACTC == ACTC_indirectRetainable &&
       (castACTC == ACTC_voidPtr ||
-       (castACTC == ACTC_coreFoundation && isCast(CCK))))
+       (castACTC == ACTC_coreFoundation && SemaRef.isCast(CCK))))
     return ACR_okay;
   if (castACTC == ACTC_indirectRetainable &&
       (exprACTC == ACTC_voidPtr || exprACTC == ACTC_coreFoundation) &&
-      isCast(CCK))
+      SemaRef.isCast(CCK))
     return ACR_okay;
 
   switch (ARCCastChecker(Context, exprACTC, castACTC, false).Visit(castExpr)) {
@@ -4471,14 +4483,15 @@ Sema::CheckObjCConversion(SourceRange castRange, QualType castType,
     castExpr = ImplicitCastExpr::Create(Context, castExpr->getType(),
                                         CK_ARCConsumeObject, castExpr, nullptr,
                                         VK_PRValue, FPOptionsOverride());
-    Cleanup.setExprNeedsCleanups(true);
+    SemaRef.Cleanup.setExprNeedsCleanups(true);
     return ACR_okay;
   }
 
   // If this is a non-implicit cast from id or block type to a
   // CoreFoundation type, delay complaining in case the cast is used
   // in an acceptable context.
-  if (exprACTC == ACTC_retainable && isAnyRetainable(castACTC) && isCast(CCK))
+  if (exprACTC == ACTC_retainable && isAnyRetainable(castACTC) &&
+      SemaRef.isCast(CCK))
     return ACR_unbridged;
 
   // Issue a diagnostic about a missing @-sign when implicit casting a cstring
@@ -4497,8 +4510,8 @@ Sema::CheckObjCConversion(SourceRange castRange, QualType castType,
       !(exprACTC == ACTC_voidPtr && castACTC == ACTC_retainable &&
         (Opc == BO_NE || Opc == BO_EQ))) {
     if (Diagnose)
-      diagnoseObjCARCConversion(*this, castRange, castType, castACTC, castExpr,
-                                castExpr, exprACTC, CCK);
+      diagnoseObjCARCConversion(SemaRef, castRange, castType, castACTC,
+                                castExpr, castExpr, exprACTC, CCK);
     return ACR_error;
   }
   return ACR_okay;
@@ -4506,7 +4519,7 @@ Sema::CheckObjCConversion(SourceRange castRange, QualType castType,
 
 /// Given that we saw an expression with the ARCUnbridgedCastTy
 /// placeholder type, complain bitterly.
-void Sema::diagnoseARCUnbridgedCast(Expr *e) {
+void SemaObjC::diagnoseARCUnbridgedCast(Expr *e) {
   // We expect the spurious ImplicitCastExpr to already have been stripped.
   assert(!e->hasPlaceholderType(BuiltinType::ARCUnbridgedCast));
   CastExpr *realCast = cast<CastExpr>(e->IgnoreParens());
@@ -4533,14 +4546,15 @@ void Sema::diagnoseARCUnbridgedCast(Expr *e) {
   Expr *castExpr = realCast->getSubExpr();
   assert(classifyTypeForARCConversion(castExpr->getType()) == ACTC_retainable);
 
-  diagnoseObjCARCConversion(*this, castRange, castType, castACTC,
-                            castExpr, realCast, ACTC_retainable, CCK);
+  diagnoseObjCARCConversion(SemaRef, castRange, castType, castACTC, castExpr,
+                            realCast, ACTC_retainable, CCK);
 }
 
 /// stripARCUnbridgedCast - Given an expression of ARCUnbridgedCast
 /// type, remove the placeholder cast.
-Expr *Sema::stripARCUnbridgedCast(Expr *e) {
+Expr *SemaObjC::stripARCUnbridgedCast(Expr *e) {
   assert(e->hasPlaceholderType(BuiltinType::ARCUnbridgedCast));
+  ASTContext &Context = getASTContext();
 
   if (ParenExpr *pe = dyn_cast<ParenExpr>(e)) {
     Expr *sub = stripARCUnbridgedCast(pe->getSubExpr());
@@ -4551,7 +4565,7 @@ Expr *Sema::stripARCUnbridgedCast(Expr *e) {
     return UnaryOperator::Create(Context, sub, UO_Extension, sub->getType(),
                                  sub->getValueKind(), sub->getObjectKind(),
                                  uo->getOperatorLoc(), false,
-                                 CurFPFeatureOverrides());
+                                 SemaRef.CurFPFeatureOverrides());
   } else if (GenericSelectionExpr *gse = dyn_cast<GenericSelectionExpr>(e)) {
     assert(!gse->isResultDependent());
     assert(!gse->isTypePredicate());
@@ -4579,8 +4593,9 @@ Expr *Sema::stripARCUnbridgedCast(Expr *e) {
   }
 }
 
-bool Sema::CheckObjCARCUnavailableWeakConversion(QualType castType,
-                                                 QualType exprType) {
+bool SemaObjC::CheckObjCARCUnavailableWeakConversion(QualType castType,
+                                                     QualType exprType) {
+  ASTContext &Context = getASTContext();
   QualType canCastType =
     Context.getCanonicalType(castType).getUnqualifiedType();
   QualType canExprType =
@@ -4633,12 +4648,13 @@ static Expr *maybeUndoReclaimObject(Expr *e) {
   return e;
 }
 
-ExprResult Sema::BuildObjCBridgedCast(SourceLocation LParenLoc,
-                                      ObjCBridgeCastKind Kind,
-                                      SourceLocation BridgeKeywordLoc,
-                                      TypeSourceInfo *TSInfo,
-                                      Expr *SubExpr) {
-  ExprResult SubResult = UsualUnaryConversions(SubExpr);
+ExprResult SemaObjC::BuildObjCBridgedCast(SourceLocation LParenLoc,
+                                          ObjCBridgeCastKind Kind,
+                                          SourceLocation BridgeKeywordLoc,
+                                          TypeSourceInfo *TSInfo,
+                                          Expr *SubExpr) {
+  ASTContext &Context = getASTContext();
+  ExprResult SubResult = SemaRef.UsualUnaryConversions(SubExpr);
   if (SubResult.isInvalid()) return ExprError();
   SubExpr = SubResult.get();
 
@@ -4736,7 +4752,7 @@ ExprResult Sema::BuildObjCBridgedCast(SourceLocation LParenLoc,
                                                    TSInfo, SubExpr);
 
   if (MustConsume) {
-    Cleanup.setExprNeedsCleanups(true);
+    SemaRef.Cleanup.setExprNeedsCleanups(true);
     Result = ImplicitCastExpr::Create(Context, T, CK_ARCConsumeObject, Result,
                                       nullptr, VK_PRValue, FPOptionsOverride());
   }
@@ -4744,15 +4760,15 @@ ExprResult Sema::BuildObjCBridgedCast(SourceLocation LParenLoc,
   return Result;
 }
 
-ExprResult Sema::ActOnObjCBridgedCast(Scope *S,
-                                      SourceLocation LParenLoc,
-                                      ObjCBridgeCastKind Kind,
-                                      SourceLocation BridgeKeywordLoc,
-                                      ParsedType Type,
-                                      SourceLocation RParenLoc,
-                                      Expr *SubExpr) {
+ExprResult SemaObjC::ActOnObjCBridgedCast(Scope *S, SourceLocation LParenLoc,
+                                          ObjCBridgeCastKind Kind,
+                                          SourceLocation BridgeKeywordLoc,
+                                          ParsedType Type,
+                                          SourceLocation RParenLoc,
+                                          Expr *SubExpr) {
+  ASTContext &Context = getASTContext();
   TypeSourceInfo *TSInfo = nullptr;
-  QualType T = GetTypeFromParser(Type, &TSInfo);
+  QualType T = SemaRef.GetTypeFromParser(Type, &TSInfo);
   if (Kind == OBC_Bridge)
     CheckTollFreeBridgeCast(T, SubExpr);
   if (!TSInfo)
@@ -4760,3 +4776,473 @@ ExprResult Sema::ActOnObjCBridgedCast(Scope *S,
   return BuildObjCBridgedCast(LParenLoc, Kind, BridgeKeywordLoc, TSInfo,
                               SubExpr);
 }
+
+DeclResult SemaObjC::LookupIvarInObjCMethod(LookupResult &Lookup, Scope *S,
+                                            IdentifierInfo *II) {
+  SourceLocation Loc = Lookup.getNameLoc();
+  ObjCMethodDecl *CurMethod = SemaRef.getCurMethodDecl();
+
+  // Check for error condition which is already reported.
+  if (!CurMethod)
+    return DeclResult(true);
+
+  // There are two cases to handle here.  1) scoped lookup could have failed,
+  // in which case we should look for an ivar.  2) scoped lookup could have
+  // found a decl, but that decl is outside the current instance method (i.e.
+  // a global variable).  In these two cases, we do a lookup for an ivar with
+  // this name, if the lookup sucedes, we replace it our current decl.
+
+  // If we're in a class method, we don't normally want to look for
+  // ivars.  But if we don't find anything else, and there's an
+  // ivar, that's an error.
+  bool IsClassMethod = CurMethod->isClassMethod();
+
+  bool LookForIvars;
+  if (Lookup.empty())
+    LookForIvars = true;
+  else if (IsClassMethod)
+    LookForIvars = false;
+  else
+    LookForIvars = (Lookup.isSingleResult() &&
+                    Lookup.getFoundDecl()->isDefinedOutsideFunctionOrMethod());
+  ObjCInterfaceDecl *IFace = nullptr;
+  if (LookForIvars) {
+    IFace = CurMethod->getClassInterface();
+    ObjCInterfaceDecl *ClassDeclared;
+    ObjCIvarDecl *IV = nullptr;
+    if (IFace && (IV = IFace->lookupInstanceVariable(II, ClassDeclared))) {
+      // Diagnose using an ivar in a class method.
+      if (IsClassMethod) {
+        Diag(Loc, diag::err_ivar_use_in_class_method) << IV->getDeclName();
+        return DeclResult(true);
+      }
+
+      // Diagnose the use of an ivar outside of the declaring class.
+      if (IV->getAccessControl() == ObjCIvarDecl::Private &&
+          !declaresSameEntity(ClassDeclared, IFace) &&
+          !getLangOpts().DebuggerSupport)
+        Diag(Loc, diag::err_private_ivar_access) << IV->getDeclName();
+
+      // Success.
+      return IV;
+    }
+  } else if (CurMethod->isInstanceMethod()) {
+    // We should warn if a local variable hides an ivar.
+    if (ObjCInterfaceDecl *IFace = CurMethod->getClassInterface()) {
+      ObjCInterfaceDecl *ClassDeclared;
+      if (ObjCIvarDecl *IV = IFace->lookupInstanceVariable(II, ClassDeclared)) {
+        if (IV->getAccessControl() != ObjCIvarDecl::Private ||
+            declaresSameEntity(IFace, ClassDeclared))
+          Diag(Loc, diag::warn_ivar_use_hidden) << IV->getDeclName();
+      }
+    }
+  } else if (Lookup.isSingleResult() &&
+             Lookup.getFoundDecl()->isDefinedOutsideFunctionOrMethod()) {
+    // If accessing a stand-alone ivar in a class method, this is an error.
+    if (const ObjCIvarDecl *IV =
+            dyn_cast<ObjCIvarDecl>(Lookup.getFoundDecl())) {
+      Diag(Loc, diag::err_ivar_use_in_class_method) << IV->getDeclName();
+      return DeclResult(true);
+    }
+  }
+
+  // Didn't encounter an error, didn't find an ivar.
+  return DeclResult(false);
+}
+
+ExprResult SemaObjC::LookupInObjCMethod(LookupResult &Lookup, Scope *S,
+                                        IdentifierInfo *II,
+                                        bool AllowBuiltinCreation) {
+  // FIXME: Integrate this lookup step into LookupParsedName.
+  DeclResult Ivar = LookupIvarInObjCMethod(Lookup, S, II);
+  if (Ivar.isInvalid())
+    return ExprError();
+  if (Ivar.isUsable())
+    return BuildIvarRefExpr(S, Lookup.getNameLoc(),
+                            cast<ObjCIvarDecl>(Ivar.get()));
+
+  if (Lookup.empty() && II && AllowBuiltinCreation)
+    SemaRef.LookupBuiltin(Lookup);
+
+  // Sentinel value saying that we didn't do anything special.
+  return ExprResult(false);
+}
+
+ExprResult SemaObjC::BuildIvarRefExpr(Scope *S, SourceLocation Loc,
+                                      ObjCIvarDecl *IV) {
+  ASTContext &Context = getASTContext();
+  ObjCMethodDecl *CurMethod = SemaRef.getCurMethodDecl();
+  assert(CurMethod && CurMethod->isInstanceMethod() &&
+         "should not reference ivar from this context");
+
+  ObjCInterfaceDecl *IFace = CurMethod->getClassInterface();
+  assert(IFace && "should not reference ivar from this context");
+
+  // If we're referencing an invalid decl, just return this as a silent
+  // error node.  The error diagnostic was already emitted on the decl.
+  if (IV->isInvalidDecl())
+    return ExprError();
+
+  // Check if referencing a field with __attribute__((deprecated)).
+  if (SemaRef.DiagnoseUseOfDecl(IV, Loc))
+    return ExprError();
+
+  // FIXME: This should use a new expr for a direct reference, don't
+  // turn this into Self->ivar, just return a BareIVarExpr or something.
+  IdentifierInfo &II = Context.Idents.get("self");
+  UnqualifiedId SelfName;
+  SelfName.setImplicitSelfParam(&II);
+  CXXScopeSpec SelfScopeSpec;
+  SourceLocation TemplateKWLoc;
+  ExprResult SelfExpr =
+      SemaRef.ActOnIdExpression(S, SelfScopeSpec, TemplateKWLoc, SelfName,
+                                /*HasTrailingLParen=*/false,
+                                /*IsAddressOfOperand=*/false);
+  if (SelfExpr.isInvalid())
+    return ExprError();
+
+  SelfExpr = SemaRef.DefaultLvalueConversion(SelfExpr.get());
+  if (SelfExpr.isInvalid())
+    return ExprError();
+
+  SemaRef.MarkAnyDeclReferenced(Loc, IV, true);
+
+  ObjCMethodFamily MF = CurMethod->getMethodFamily();
+  if (MF != OMF_init && MF != OMF_dealloc && MF != OMF_finalize &&
+      !IvarBacksCurrentMethodAccessor(IFace, CurMethod, IV))
+    Diag(Loc, diag::warn_direct_ivar_access) << IV->getDeclName();
+
+  ObjCIvarRefExpr *Result = new (Context)
+      ObjCIvarRefExpr(IV, IV->getUsageType(SelfExpr.get()->getType()), Loc,
+                      IV->getLocation(), SelfExpr.get(), true, true);
+
+  if (IV->getType().getObjCLifetime() == Qualifiers::OCL_Weak) {
+    if (!SemaRef.isUnevaluatedContext() &&
+        !getDiagnostics().isIgnored(diag::warn_arc_repeated_use_of_weak, Loc))
+      SemaRef.getCurFunction()->recordUseOfWeak(Result);
+  }
+  if (getLangOpts().ObjCAutoRefCount && !SemaRef.isUnevaluatedContext())
+    if (const BlockDecl *BD = SemaRef.CurContext->getInnermostBlockDecl())
+      SemaRef.ImplicitlyRetainedSelfLocs.push_back({Loc, BD});
+
+  return Result;
+}
+
+QualType SemaObjC::FindCompositeObjCPointerType(ExprResult &LHS,
+                                                ExprResult &RHS,
+                                                SourceLocation QuestionLoc) {
+  ASTContext &Context = getASTContext();
+  QualType LHSTy = LHS.get()->getType();
+  QualType RHSTy = RHS.get()->getType();
+
+  // Handle things like Class and struct objc_class*.  Here we case the result
+  // to the pseudo-builtin, because that will be implicitly cast back to the
+  // redefinition type if an attempt is made to access its fields.
+  if (LHSTy->isObjCClassType() &&
+      (Context.hasSameType(RHSTy, Context.getObjCClassRedefinitionType()))) {
+    RHS = SemaRef.ImpCastExprToType(RHS.get(), LHSTy,
+                                    CK_CPointerToObjCPointerCast);
+    return LHSTy;
+  }
+  if (RHSTy->isObjCClassType() &&
+      (Context.hasSameType(LHSTy, Context.getObjCClassRedefinitionType()))) {
+    LHS = SemaRef.ImpCastExprToType(LHS.get(), RHSTy,
+                                    CK_CPointerToObjCPointerCast);
+    return RHSTy;
+  }
+  // And the same for struct objc_object* / id
+  if (LHSTy->isObjCIdType() &&
+      (Context.hasSameType(RHSTy, Context.getObjCIdRedefinitionType()))) {
+    RHS = SemaRef.ImpCastExprToType(RHS.get(), LHSTy,
+                                    CK_CPointerToObjCPointerCast);
+    return LHSTy;
+  }
+  if (RHSTy->isObjCIdType() &&
+      (Context.hasSameType(LHSTy, Context.getObjCIdRedefinitionType()))) {
+    LHS = SemaRef.ImpCastExprToType(LHS.get(), RHSTy,
+                                    CK_CPointerToObjCPointerCast);
+    return RHSTy;
+  }
+  // And the same for struct objc_selector* / SEL
+  if (Context.isObjCSelType(LHSTy) &&
+      (Context.hasSameType(RHSTy, Context.getObjCSelRedefinitionType()))) {
+    RHS = SemaRef.ImpCastExprToType(RHS.get(), LHSTy, CK_BitCast);
+    return LHSTy;
+  }
+  if (Context.isObjCSelType(RHSTy) &&
+      (Context.hasSameType(LHSTy, Context.getObjCSelRedefinitionType()))) {
+    LHS = SemaRef.ImpCastExprToType(LHS.get(), RHSTy, CK_BitCast);
+    return RHSTy;
+  }
+  // Check constraints for Objective-C object pointers types.
+  if (LHSTy->isObjCObjectPointerType() && RHSTy->isObjCObjectPointerType()) {
+
+    if (Context.getCanonicalType(LHSTy) == Context.getCanonicalType(RHSTy)) {
+      // Two identical object pointer types are always compatible.
+      return LHSTy;
+    }
+    const ObjCObjectPointerType *LHSOPT =
+        LHSTy->castAs<ObjCObjectPointerType>();
+    const ObjCObjectPointerType *RHSOPT =
+        RHSTy->castAs<ObjCObjectPointerType>();
+    QualType compositeType = LHSTy;
+
+    // If both operands are interfaces and either operand can be
+    // assigned to the other, use that type as the composite
+    // type. This allows
+    //   xxx ? (A*) a : (B*) b
+    // where B is a subclass of A.
+    //
+    // Additionally, as for assignment, if either type is 'id'
+    // allow silent coercion. Finally, if the types are
+    // incompatible then make sure to use 'id' as the composite
+    // type so the result is acceptable for sending messages to.
+
+    // FIXME: Consider unifying with 'areComparableObjCPointerTypes'.
+    // It could return the composite type.
+    if (!(compositeType = Context.areCommonBaseCompatible(LHSOPT, RHSOPT))
+             .isNull()) {
+      // Nothing more to do.
+    } else if (Context.canAssignObjCInterfaces(LHSOPT, RHSOPT)) {
+      compositeType = RHSOPT->isObjCBuiltinType() ? RHSTy : LHSTy;
+    } else if (Context.canAssignObjCInterfaces(RHSOPT, LHSOPT)) {
+      compositeType = LHSOPT->isObjCBuiltinType() ? LHSTy : RHSTy;
+    } else if ((LHSOPT->isObjCQualifiedIdType() ||
+                RHSOPT->isObjCQualifiedIdType()) &&
+               Context.ObjCQualifiedIdTypesAreCompatible(LHSOPT, RHSOPT,
+                                                         true)) {
+      // Need to handle "id<xx>" explicitly.
+      // GCC allows qualified id and any Objective-C type to devolve to
+      // id. Currently localizing to here until clear this should be
+      // part of ObjCQualifiedIdTypesAreCompatible.
+      compositeType = Context.getObjCIdType();
+    } else if (LHSTy->isObjCIdType() || RHSTy->isObjCIdType()) {
+      compositeType = Context.getObjCIdType();
+    } else {
+      Diag(QuestionLoc, diag::ext_typecheck_cond_incompatible_operands)
+          << LHSTy << RHSTy << LHS.get()->getSourceRange()
+          << RHS.get()->getSourceRange();
+      QualType incompatTy = Context.getObjCIdType();
+      LHS = SemaRef.ImpCastExprToType(LHS.get(), incompatTy, CK_BitCast);
+      RHS = SemaRef.ImpCastExprToType(RHS.get(), incompatTy, CK_BitCast);
+      return incompatTy;
+    }
+    // The object pointer types are compatible.
+    LHS = SemaRef.ImpCastExprToType(LHS.get(), compositeType, CK_BitCast);
+    RHS = SemaRef.ImpCastExprToType(RHS.get(), compositeType, CK_BitCast);
+    return compositeType;
+  }
+  // Check Objective-C object pointer types and 'void *'
+  if (LHSTy->isVoidPointerType() && RHSTy->isObjCObjectPointerType()) {
+    if (getLangOpts().ObjCAutoRefCount) {
+      // ARC forbids the implicit conversion of object pointers to 'void *',
+      // so these types are not compatible.
+      Diag(QuestionLoc, diag::err_cond_voidptr_arc)
+          << LHSTy << RHSTy << LHS.get()->getSourceRange()
+          << RHS.get()->getSourceRange();
+      LHS = RHS = true;
+      return QualType();
+    }
+    QualType lhptee = LHSTy->castAs<PointerType>()->getPointeeType();
+    QualType rhptee = RHSTy->castAs<ObjCObjectPointerType>()->getPointeeType();
+    QualType destPointee =
+        Context.getQualifiedType(lhptee, rhptee.getQualifiers());
+    QualType destType = Context.getPointerType(destPointee);
+    // Add qualifiers if necessary.
+    LHS = SemaRef.ImpCastExprToType(LHS.get(), destType, CK_NoOp);
+    // Promote to void*.
+    RHS = SemaRef.ImpCastExprToType(RHS.get(), destType, CK_BitCast);
+    return destType;
+  }
+  if (LHSTy->isObjCObjectPointerType() && RHSTy->isVoidPointerType()) {
+    if (getLangOpts().ObjCAutoRefCount) {
+      // ARC forbids the implicit conversion of object pointers to 'void *',
+      // so these types are not compatible.
+      Diag(QuestionLoc, diag::err_cond_voidptr_arc)
+          << LHSTy << RHSTy << LHS.get()->getSourceRange()
+          << RHS.get()->getSourceRange();
+      LHS = RHS = true;
+      return QualType();
+    }
+    QualType lhptee = LHSTy->castAs<ObjCObjectPointerType>()->getPointeeType();
+    QualType rhptee = RHSTy->castAs<PointerType>()->getPointeeType();
+    QualType destPointee =
+        Context.getQualifiedType(rhptee, lhptee.getQualifiers());
+    QualType destType = Context.getPointerType(destPointee);
+    // Add qualifiers if necessary.
+    RHS = SemaRef.ImpCastExprToType(RHS.get(), destType, CK_NoOp);
+    // Promote to void*.
+    LHS = SemaRef.ImpCastExprToType(LHS.get(), destType, CK_BitCast);
+    return destType;
+  }
+  return QualType();
+}
+
+bool SemaObjC::CheckConversionToObjCLiteral(QualType DstType, Expr *&Exp,
+                                            bool Diagnose) {
+  if (!getLangOpts().ObjC)
+    return false;
+
+  const ObjCObjectPointerType *PT = DstType->getAs<ObjCObjectPointerType>();
+  if (!PT)
+    return false;
+  const ObjCInterfaceDecl *ID = PT->getInterfaceDecl();
+
+  // Ignore any parens, implicit casts (should only be
+  // array-to-pointer decays), and not-so-opaque values.  The last is
+  // important for making this trigger for property assignments.
+  Expr *SrcExpr = Exp->IgnoreParenImpCasts();
+  if (OpaqueValueExpr *OV = dyn_cast<OpaqueValueExpr>(SrcExpr))
+    if (OV->getSourceExpr())
+      SrcExpr = OV->getSourceExpr()->IgnoreParenImpCasts();
+
+  if (auto *SL = dyn_cast<StringLiteral>(SrcExpr)) {
+    if (!PT->isObjCIdType() && !(ID && ID->getIdentifier()->isStr("NSString")))
+      return false;
+    if (!SL->isOrdinary())
+      return false;
+
+    if (Diagnose) {
+      Diag(SL->getBeginLoc(), diag::err_missing_atsign_prefix)
+          << /*string*/ 0 << FixItHint::CreateInsertion(SL->getBeginLoc(), "@");
+      Exp = BuildObjCStringLiteral(SL->getBeginLoc(), SL).get();
+    }
+    return true;
+  }
+
+  if ((isa<IntegerLiteral>(SrcExpr) || isa<CharacterLiteral>(SrcExpr) ||
+       isa<FloatingLiteral>(SrcExpr) || isa<ObjCBoolLiteralExpr>(SrcExpr) ||
+       isa<CXXBoolLiteralExpr>(SrcExpr)) &&
+      !SrcExpr->isNullPointerConstant(getASTContext(),
+                                      Expr::NPC_NeverValueDependent)) {
+    if (!ID || !ID->getIdentifier()->isStr("NSNumber"))
+      return false;
+    if (Diagnose) {
+      Diag(SrcExpr->getBeginLoc(), diag::err_missing_atsign_prefix)
+          << /*number*/ 1
+          << FixItHint::CreateInsertion(SrcExpr->getBeginLoc(), "@");
+      Expr *NumLit =
+          BuildObjCNumericLiteral(SrcExpr->getBeginLoc(), SrcExpr).get();
+      if (NumLit)
+        Exp = NumLit;
+    }
+    return true;
+  }
+
+  return false;
+}
+
+/// ActOnObjCBoolLiteral - Parse {__objc_yes,__objc_no} literals.
+ExprResult SemaObjC::ActOnObjCBoolLiteral(SourceLocation OpLoc,
+                                          tok::TokenKind Kind) {
+  assert((Kind == tok::kw___objc_yes || Kind == tok::kw___objc_no) &&
+         "Unknown Objective-C Boolean value!");
+  ASTContext &Context = getASTContext();
+  QualType BoolT = Context.ObjCBuiltinBoolTy;
+  if (!Context.getBOOLDecl()) {
+    LookupResult Result(SemaRef, &Context.Idents.get("BOOL"), OpLoc,
+                        Sema::LookupOrdinaryName);
+    if (SemaRef.LookupName(Result, SemaRef.getCurScope()) &&
+        Result.isSingleResult()) {
+      NamedDecl *ND = Result.getFoundDecl();
+      if (TypedefDecl *TD = dyn_cast<TypedefDecl>(ND))
+        Context.setBOOLDecl(TD);
+    }
+  }
+  if (Context.getBOOLDecl())
+    BoolT = Context.getBOOLType();
+  return new (Context)
+      ObjCBoolLiteralExpr(Kind == tok::kw___objc_yes, BoolT, OpLoc);
+}
+
+ExprResult SemaObjC::ActOnObjCAvailabilityCheckExpr(
+    llvm::ArrayRef<AvailabilitySpec> AvailSpecs, SourceLocation AtLoc,
+    SourceLocation RParen) {
+  ASTContext &Context = getASTContext();
+  auto FindSpecVersion =
+      [&](StringRef Platform) -> std::optional<VersionTuple> {
+    auto Spec = llvm::find_if(AvailSpecs, [&](const AvailabilitySpec &Spec) {
+      return Spec.getPlatform() == Platform;
+    });
+    // Transcribe the "ios" availability check to "maccatalyst" when compiling
+    // for "maccatalyst" if "maccatalyst" is not specified.
+    if (Spec == AvailSpecs.end() && Platform == "maccatalyst") {
+      Spec = llvm::find_if(AvailSpecs, [&](const AvailabilitySpec &Spec) {
+        return Spec.getPlatform() == "ios";
+      });
+    }
+    if (Spec == AvailSpecs.end())
+      return std::nullopt;
+    return Spec->getVersion();
+  };
+
+  VersionTuple Version;
+  if (auto MaybeVersion =
+          FindSpecVersion(Context.getTargetInfo().getPlatformName()))
+    Version = *MaybeVersion;
+
+  // The use of `@available` in the enclosing context should be analyzed to
+  // warn when it's used inappropriately (i.e. not if(@available)).
+  if (FunctionScopeInfo *Context = SemaRef.getCurFunctionAvailabilityContext())
+    Context->HasPotentialAvailabilityViolations = true;
+
+  return new (Context)
+      ObjCAvailabilityCheckExpr(Version, AtLoc, RParen, Context.BoolTy);
+}
+
+/// Prepare a conversion of the given expression to an ObjC object
+/// pointer type.
+CastKind SemaObjC::PrepareCastToObjCObjectPointer(ExprResult &E) {
+  QualType type = E.get()->getType();
+  if (type->isObjCObjectPointerType()) {
+    return CK_BitCast;
+  } else if (type->isBlockPointerType()) {
+    SemaRef.maybeExtendBlockObject(E);
+    return CK_BlockPointerToObjCPointerCast;
+  } else {
+    assert(type->isPointerType());
+    return CK_CPointerToObjCPointerCast;
+  }
+}
+
+SemaObjC::ObjCLiteralKind SemaObjC::CheckLiteralKind(Expr *FromE) {
+  FromE = FromE->IgnoreParenImpCasts();
+  switch (FromE->getStmtClass()) {
+  default:
+    break;
+  case Stmt::ObjCStringLiteralClass:
+    // "string literal"
+    return LK_String;
+  case Stmt::ObjCArrayLiteralClass:
+    // "array literal"
+    return LK_Array;
+  case Stmt::ObjCDictionaryLiteralClass:
+    // "dictionary literal"
+    return LK_Dictionary;
+  case Stmt::BlockExprClass:
+    return LK_Block;
+  case Stmt::ObjCBoxedExprClass: {
+    Expr *Inner = cast<ObjCBoxedExpr>(FromE)->getSubExpr()->IgnoreParens();
+    switch (Inner->getStmtClass()) {
+    case Stmt::IntegerLiteralClass:
+    case Stmt::FloatingLiteralClass:
+    case Stmt::CharacterLiteralClass:
+    case Stmt::ObjCBoolLiteralExprClass:
+    case Stmt::CXXBoolLiteralExprClass:
+      // "numeric literal"
+      return LK_Numeric;
+    case Stmt::ImplicitCastExprClass: {
+      CastKind CK = cast<CastExpr>(Inner)->getCastKind();
+      // Boolean literals can be represented by implicit casts.
+      if (CK == CK_IntegralToBoolean || CK == CK_IntegralCast)
+        return LK_Numeric;
+      break;
+    }
+    default:
+      break;
+    }
+    return LK_Boxed;
+  }
+  }
+  return LK_None;
+}
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index fe4a698a612e..708286e192f9 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -28,6 +28,7 @@
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/PointerIntPair.h"
@@ -6018,8 +6019,8 @@ static bool tryObjCWritebackConversion(Sema &S,
 
   // Handle write-back conversion.
   QualType ConvertedArgType;
-  if (!S.isObjCWritebackConversion(ArgType, Entity.getType(),
-                                   ConvertedArgType))
+  if (!S.ObjC().isObjCWritebackConversion(ArgType, Entity.getType(),
+                                          ConvertedArgType))
     return false;
 
   // We should copy unless we're passing to an argument explicitly
@@ -6211,10 +6212,10 @@ void InitializationSequence::InitializeFrom(Sema &S,
   if (Args.size() == 1) {
     Initializer = Args[0];
     if (S.getLangOpts().ObjC) {
-      if (S.CheckObjCBridgeRelatedConversions(Initializer->getBeginLoc(),
-                                              DestType, Initializer->getType(),
-                                              Initializer) ||
-          S.CheckConversionToObjCLiteral(DestType, Initializer))
+      if (S.ObjC().CheckObjCBridgeRelatedConversions(
+              Initializer->getBeginLoc(), DestType, Initializer->getType(),
+              Initializer) ||
+          S.ObjC().CheckConversionToObjCLiteral(DestType, Initializer))
         Args[0] = Initializer;
     }
     if (!isa<InitListExpr>(Initializer))
@@ -9558,12 +9559,12 @@ static void emitBadConversionNotes(Sema &S, const InitializedEntity &entity,
 
     // Emit a possible note about the conversion failing because the
     // operand is a message send with a related result type.
-    S.EmitRelatedResultTypeNote(op);
+    S.ObjC().EmitRelatedResultTypeNote(op);
 
     // Emit a possible note about a return failing because we're
     // expecting a related result type.
     if (entity.getKind() == InitializedEntity::EK_Result)
-      S.EmitRelatedResultTypeNoteForReturn(destType);
+      S.ObjC().EmitRelatedResultTypeNoteForReturn(destType);
   }
   QualType fromType = op->getType();
   QualType fromPointeeType = fromType.getCanonicalType()->getPointeeType();
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 7251aabc6af2..0834db95d42a 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -3365,15 +3365,6 @@ NamedDecl *Sema::LookupSingleName(Scope *S, DeclarationName Name,
   return R.getAsSingle<NamedDecl>();
 }
 
-/// Find the protocol with the given name, if any.
-ObjCProtocolDecl *Sema::LookupProtocol(IdentifierInfo *II,
-                                       SourceLocation IdLoc,
-                                       RedeclarationKind Redecl) {
-  Decl *D = LookupSingleName(TUScope, II, IdLoc,
-                             LookupObjCProtocolName, Redecl);
-  return cast_or_null<ObjCProtocolDecl>(D);
-}
-
 void Sema::LookupOverloadedOperatorName(OverloadedOperatorKind Op, Scope *S,
                                         UnresolvedSetImpl &Functions) {
   // C++ [over.match.oper]p3:
diff --git a/clang/lib/Sema/SemaObjC.cpp b/clang/lib/Sema/SemaObjC.cpp
new file mode 100644
index 000000000000..1e6cc21a4870
--- /dev/null
+++ b/clang/lib/Sema/SemaObjC.cpp
@@ -0,0 +1,1486 @@
+//===----- SemaObjC.cpp ---- Semantic Analysis for Objective-C ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file implements semantic analysis for Objective-C.
+///
+//===----------------------------------------------------------------------===//
+
+#include "clang/Sema/SemaObjC.h"
+#include "clang/AST/EvaluatedExprVisitor.h"
+#include "clang/AST/StmtObjC.h"
+#include "clang/Basic/DiagnosticSema.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/Sema.h"
+#include "clang/Sema/TemplateDeduction.h"
+#include "llvm/Support/ConvertUTF.h"
+
+namespace clang {
+
+SemaObjC::SemaObjC(Sema &S)
+    : SemaBase(S), NSNumberDecl(nullptr), NSValueDecl(nullptr),
+      NSStringDecl(nullptr), StringWithUTF8StringMethod(nullptr),
+      ValueWithBytesObjCTypeMethod(nullptr), NSArrayDecl(nullptr),
+      ArrayWithObjectsMethod(nullptr), NSDictionaryDecl(nullptr),
+      DictionaryWithObjectsMethod(nullptr) {}
+
+StmtResult SemaObjC::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
+                                                Stmt *First, Expr *collection,
+                                                SourceLocation RParenLoc) {
+  ASTContext &Context = getASTContext();
+  SemaRef.setFunctionHasBranchProtectedScope();
+
+  ExprResult CollectionExprResult =
+      CheckObjCForCollectionOperand(ForLoc, collection);
+
+  if (First) {
+    QualType FirstType;
+    if (DeclStmt *DS = dyn_cast<DeclStmt>(First)) {
+      if (!DS->isSingleDecl())
+        return StmtError(Diag((*DS->decl_begin())->getLocation(),
+                              diag::err_toomany_element_decls));
+
+      VarDecl *D = dyn_cast<VarDecl>(DS->getSingleDecl());
+      if (!D || D->isInvalidDecl())
+        return StmtError();
+
+      FirstType = D->getType();
+      // C99 6.8.5p3: The declaration part of a 'for' statement shall only
+      // declare identifiers for objects having storage class 'auto' or
+      // 'register'.
+      if (!D->hasLocalStorage())
+        return StmtError(
+            Diag(D->getLocation(), diag::err_non_local_variable_decl_in_for));
+
+      // If the type contained 'auto', deduce the 'auto' to 'id'.
+      if (FirstType->getContainedAutoType()) {
+        SourceLocation Loc = D->getLocation();
+        OpaqueValueExpr OpaqueId(Loc, Context.getObjCIdType(), VK_PRValue);
+        Expr *DeducedInit = &OpaqueId;
+        sema::TemplateDeductionInfo Info(Loc);
+        FirstType = QualType();
+        TemplateDeductionResult Result = SemaRef.DeduceAutoType(
+            D->getTypeSourceInfo()->getTypeLoc(), DeducedInit, FirstType, Info);
+        if (Result != TemplateDeductionResult::Success &&
+            Result != TemplateDeductionResult::AlreadyDiagnosed)
+          SemaRef.DiagnoseAutoDeductionFailure(D, DeducedInit);
+        if (FirstType.isNull()) {
+          D->setInvalidDecl();
+          return StmtError();
+        }
+
+        D->setType(FirstType);
+
+        if (!SemaRef.inTemplateInstantiation()) {
+          SourceLocation Loc =
+              D->getTypeSourceInfo()->getTypeLoc().getBeginLoc();
+          Diag(Loc, diag::warn_auto_var_is_id) << D->getDeclName();
+        }
+      }
+
+    } else {
+      Expr *FirstE = cast<Expr>(First);
+      if (!FirstE->isTypeDependent() && !FirstE->isLValue())
+        return StmtError(
+            Diag(First->getBeginLoc(), diag::err_selector_element_not_lvalue)
+            << First->getSourceRange());
+
+      FirstType = static_cast<Expr *>(First)->getType();
+      if (FirstType.isConstQualified())
+        Diag(ForLoc, diag::err_selector_element_const_type)
+            << FirstType << First->getSourceRange();
+    }
+    if (!FirstType->isDependentType() &&
+        !FirstType->isObjCObjectPointerType() &&
+        !FirstType->isBlockPointerType())
+      return StmtError(Diag(ForLoc, diag::err_selector_element_type)
+                       << FirstType << First->getSourceRange());
+  }
+
+  if (CollectionExprResult.isInvalid())
+    return StmtError();
+
+  CollectionExprResult = SemaRef.ActOnFinishFullExpr(CollectionExprResult.get(),
+                                                     /*DiscardedValue*/ false);
+  if (CollectionExprResult.isInvalid())
+    return StmtError();
+
+  return new (Context) ObjCForCollectionStmt(First, CollectionExprResult.get(),
+                                             nullptr, ForLoc, RParenLoc);
+}
+
+ExprResult SemaObjC::CheckObjCForCollectionOperand(SourceLocation forLoc,
+                                                   Expr *collection) {
+  ASTContext &Context = getASTContext();
+  if (!collection)
+    return ExprError();
+
+  ExprResult result = SemaRef.CorrectDelayedTyposInExpr(collection);
+  if (!result.isUsable())
+    return ExprError();
+  collection = result.get();
+
+  // Bail out early if we've got a type-dependent expression.
+  if (collection->isTypeDependent())
+    return collection;
+
+  // Perform normal l-value conversion.
+  result = SemaRef.DefaultFunctionArrayLvalueConversion(collection);
+  if (result.isInvalid())
+    return ExprError();
+  collection = result.get();
+
+  // The operand needs to have object-pointer type.
+  // TODO: should we do a contextual conversion?
+  const ObjCObjectPointerType *pointerType =
+      collection->getType()->getAs<ObjCObjectPointerType>();
+  if (!pointerType)
+    return Diag(forLoc, diag::err_collection_expr_type)
+           << collection->getType() << collection->getSourceRange();
+
+  // Check that the operand provides
+  //   - countByEnumeratingWithState:objects:count:
+  const ObjCObjectType *objectType = pointerType->getObjectType();
+  ObjCInterfaceDecl *iface = objectType->getInterface();
+
+  // If we have a forward-declared type, we can't do this check.
+  // Under ARC, it is an error not to have a forward-declared class.
+  if (iface &&
+      (getLangOpts().ObjCAutoRefCount
+           ? SemaRef.RequireCompleteType(forLoc, QualType(objectType, 0),
+                                         diag::err_arc_collection_forward,
+                                         collection)
+           : !SemaRef.isCompleteType(forLoc, QualType(objectType, 0)))) {
+    // Otherwise, if we have any useful type information, check that
+    // the type declares the appropriate method.
+  } else if (iface || !objectType->qual_empty()) {
+    const IdentifierInfo *selectorIdents[] = {
+        &Context.Idents.get("countByEnumeratingWithState"),
+        &Context.Idents.get("objects"), &Context.Idents.get("count")};
+    Selector selector = Context.Selectors.getSelector(3, &selectorIdents[0]);
+
+    ObjCMethodDecl *method = nullptr;
+
+    // If there's an interface, look in both the public and private APIs.
+    if (iface) {
+      method = iface->lookupInstanceMethod(selector);
+      if (!method)
+        method = iface->lookupPrivateMethod(selector);
+    }
+
+    // Also check protocol qualifiers.
+    if (!method)
+      method = LookupMethodInQualifiedType(selector, pointerType,
+                                           /*instance*/ true);
+
+    // If we didn't find it anywhere, give up.
+    if (!method) {
+      Diag(forLoc, diag::warn_collection_expr_type)
+          << collection->getType() << selector << collection->getSourceRange();
+    }
+
+    // TODO: check for an incompatible signature?
+  }
+
+  // Wrap up any cleanups in the expression.
+  return collection;
+}
+
+StmtResult SemaObjC::FinishObjCForCollectionStmt(Stmt *S, Stmt *B) {
+  if (!S || !B)
+    return StmtError();
+  ObjCForCollectionStmt *ForStmt = cast<ObjCForCollectionStmt>(S);
+
+  ForStmt->setBody(B);
+  return S;
+}
+
+StmtResult SemaObjC::ActOnObjCAtCatchStmt(SourceLocation AtLoc,
+                                          SourceLocation RParen, Decl *Parm,
+                                          Stmt *Body) {
+  ASTContext &Context = getASTContext();
+  VarDecl *Var = cast_or_null<VarDecl>(Parm);
+  if (Var && Var->isInvalidDecl())
+    return StmtError();
+
+  return new (Context) ObjCAtCatchStmt(AtLoc, RParen, Var, Body);
+}
+
+StmtResult SemaObjC::ActOnObjCAtFinallyStmt(SourceLocation AtLoc, Stmt *Body) {
+  ASTContext &Context = getASTContext();
+  return new (Context) ObjCAtFinallyStmt(AtLoc, Body);
+}
+
+StmtResult SemaObjC::ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try,
+                                        MultiStmtArg CatchStmts,
+                                        Stmt *Finally) {
+  ASTContext &Context = getASTContext();
+  if (!getLangOpts().ObjCExceptions)
+    Diag(AtLoc, diag::err_objc_exceptions_disabled) << "@try";
+
+  // Objective-C try is incompatible with SEH __try.
+  sema::FunctionScopeInfo *FSI = SemaRef.getCurFunction();
+  if (FSI->FirstSEHTryLoc.isValid()) {
+    Diag(AtLoc, diag::err_mixing_cxx_try_seh_try) << 1;
+    Diag(FSI->FirstSEHTryLoc, diag::note_conflicting_try_here) << "'__try'";
+  }
+
+  FSI->setHasObjCTry(AtLoc);
+  unsigned NumCatchStmts = CatchStmts.size();
+  return ObjCAtTryStmt::Create(Context, AtLoc, Try, CatchStmts.data(),
+                               NumCatchStmts, Finally);
+}
+
+StmtResult SemaObjC::BuildObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw) {
+  ASTContext &Context = getASTContext();
+  if (Throw) {
+    ExprResult Result = SemaRef.DefaultLvalueConversion(Throw);
+    if (Result.isInvalid())
+      return StmtError();
+
+    Result =
+        SemaRef.ActOnFinishFullExpr(Result.get(), /*DiscardedValue*/ false);
+    if (Result.isInvalid())
+      return StmtError();
+    Throw = Result.get();
+
+    QualType ThrowType = Throw->getType();
+    // Make sure the expression type is an ObjC pointer or "void *".
+    if (!ThrowType->isDependentType() &&
+        !ThrowType->isObjCObjectPointerType()) {
+      const PointerType *PT = ThrowType->getAs<PointerType>();
+      if (!PT || !PT->getPointeeType()->isVoidType())
+        return StmtError(Diag(AtLoc, diag::err_objc_throw_expects_object)
+                         << Throw->getType() << Throw->getSourceRange());
+    }
+  }
+
+  return new (Context) ObjCAtThrowStmt(AtLoc, Throw);
+}
+
+StmtResult SemaObjC::ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw,
+                                          Scope *CurScope) {
+  if (!getLangOpts().ObjCExceptions)
+    Diag(AtLoc, diag::err_objc_exceptions_disabled) << "@throw";
+
+  if (!Throw) {
+    // @throw without an expression designates a rethrow (which must occur
+    // in the context of an @catch clause).
+    Scope *AtCatchParent = CurScope;
+    while (AtCatchParent && !AtCatchParent->isAtCatchScope())
+      AtCatchParent = AtCatchParent->getParent();
+    if (!AtCatchParent)
+      return StmtError(Diag(AtLoc, diag::err_rethrow_used_outside_catch));
+  }
+  return BuildObjCAtThrowStmt(AtLoc, Throw);
+}
+
+ExprResult SemaObjC::ActOnObjCAtSynchronizedOperand(SourceLocation atLoc,
+                                                    Expr *operand) {
+  ExprResult result = SemaRef.DefaultLvalueConversion(operand);
+  if (result.isInvalid())
+    return ExprError();
+  operand = result.get();
+
+  // Make sure the expression type is an ObjC pointer or "void *".
+  QualType type = operand->getType();
+  if (!type->isDependentType() && !type->isObjCObjectPointerType()) {
+    const PointerType *pointerType = type->getAs<PointerType>();
+    if (!pointerType || !pointerType->getPointeeType()->isVoidType()) {
+      if (getLangOpts().CPlusPlus) {
+        if (SemaRef.RequireCompleteType(atLoc, type,
+                                        diag::err_incomplete_receiver_type))
+          return Diag(atLoc, diag::err_objc_synchronized_expects_object)
+                 << type << operand->getSourceRange();
+
+        ExprResult result =
+            SemaRef.PerformContextuallyConvertToObjCPointer(operand);
+        if (result.isInvalid())
+          return ExprError();
+        if (!result.isUsable())
+          return Diag(atLoc, diag::err_objc_synchronized_expects_object)
+                 << type << operand->getSourceRange();
+
+        operand = result.get();
+      } else {
+        return Diag(atLoc, diag::err_objc_synchronized_expects_object)
+               << type << operand->getSourceRange();
+      }
+    }
+  }
+
+  // The operand to @synchronized is a full-expression.
+  return SemaRef.ActOnFinishFullExpr(operand, /*DiscardedValue*/ false);
+}
+
+StmtResult SemaObjC::ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc,
+                                                 Expr *SyncExpr,
+                                                 Stmt *SyncBody) {
+  ASTContext &Context = getASTContext();
+  // We can't jump into or indirect-jump out of a @synchronized block.
+  SemaRef.setFunctionHasBranchProtectedScope();
+  return new (Context) ObjCAtSynchronizedStmt(AtLoc, SyncExpr, SyncBody);
+}
+
+StmtResult SemaObjC::ActOnObjCAutoreleasePoolStmt(SourceLocation AtLoc,
+                                                  Stmt *Body) {
+  ASTContext &Context = getASTContext();
+  SemaRef.setFunctionHasBranchProtectedScope();
+  return new (Context) ObjCAutoreleasePoolStmt(AtLoc, Body);
+}
+
+TypeResult SemaObjC::actOnObjCProtocolQualifierType(
+    SourceLocation lAngleLoc, ArrayRef<Decl *> protocols,
+    ArrayRef<SourceLocation> protocolLocs, SourceLocation rAngleLoc) {
+  ASTContext &Context = getASTContext();
+  // Form id<protocol-list>.
+  QualType Result = Context.getObjCObjectType(
+      Context.ObjCBuiltinIdTy, {},
+      llvm::ArrayRef((ObjCProtocolDecl *const *)protocols.data(),
+                     protocols.size()),
+      false);
+  Result = Context.getObjCObjectPointerType(Result);
+
+  TypeSourceInfo *ResultTInfo = Context.CreateTypeSourceInfo(Result);
+  TypeLoc ResultTL = ResultTInfo->getTypeLoc();
+
+  auto ObjCObjectPointerTL = ResultTL.castAs<ObjCObjectPointerTypeLoc>();
+  ObjCObjectPointerTL.setStarLoc(SourceLocation()); // implicit
+
+  auto ObjCObjectTL =
+      ObjCObjectPointerTL.getPointeeLoc().castAs<ObjCObjectTypeLoc>();
+  ObjCObjectTL.setHasBaseTypeAsWritten(false);
+  ObjCObjectTL.getBaseLoc().initialize(Context, SourceLocation());
+
+  // No type arguments.
+  ObjCObjectTL.setTypeArgsLAngleLoc(SourceLocation());
+  ObjCObjectTL.setTypeArgsRAngleLoc(SourceLocation());
+
+  // Fill in protocol qualifiers.
+  ObjCObjectTL.setProtocolLAngleLoc(lAngleLoc);
+  ObjCObjectTL.setProtocolRAngleLoc(rAngleLoc);
+  for (unsigned i = 0, n = protocols.size(); i != n; ++i)
+    ObjCObjectTL.setProtocolLoc(i, protocolLocs[i]);
+
+  // We're done. Return the completed type to the parser.
+  return SemaRef.CreateParsedType(Result, ResultTInfo);
+}
+
+TypeResult SemaObjC::actOnObjCTypeArgsAndProtocolQualifiers(
+    Scope *S, SourceLocation Loc, ParsedType BaseType,
+    SourceLocation TypeArgsLAngleLoc, ArrayRef<ParsedType> TypeArgs,
+    SourceLocation TypeArgsRAngleLoc, SourceLocation ProtocolLAngleLoc,
+    ArrayRef<Decl *> Protocols, ArrayRef<SourceLocation> ProtocolLocs,
+    SourceLocation ProtocolRAngleLoc) {
+  ASTContext &Context = getASTContext();
+  TypeSourceInfo *BaseTypeInfo = nullptr;
+  QualType T = SemaRef.GetTypeFromParser(BaseType, &BaseTypeInfo);
+  if (T.isNull())
+    return true;
+
+  // Handle missing type-source info.
+  if (!BaseTypeInfo)
+    BaseTypeInfo = Context.getTrivialTypeSourceInfo(T, Loc);
+
+  // Extract type arguments.
+  SmallVector<TypeSourceInfo *, 4> ActualTypeArgInfos;
+  for (unsigned i = 0, n = TypeArgs.size(); i != n; ++i) {
+    TypeSourceInfo *TypeArgInfo = nullptr;
+    QualType TypeArg = SemaRef.GetTypeFromParser(TypeArgs[i], &TypeArgInfo);
+    if (TypeArg.isNull()) {
+      ActualTypeArgInfos.clear();
+      break;
+    }
+
+    assert(TypeArgInfo && "No type source info?");
+    ActualTypeArgInfos.push_back(TypeArgInfo);
+  }
+
+  // Build the object type.
+  QualType Result = BuildObjCObjectType(
+      T, BaseTypeInfo->getTypeLoc().getSourceRange().getBegin(),
+      TypeArgsLAngleLoc, ActualTypeArgInfos, TypeArgsRAngleLoc,
+      ProtocolLAngleLoc,
+      llvm::ArrayRef((ObjCProtocolDecl *const *)Protocols.data(),
+                     Protocols.size()),
+      ProtocolLocs, ProtocolRAngleLoc,
+      /*FailOnError=*/false,
+      /*Rebuilding=*/false);
+
+  if (Result == T)
+    return BaseType;
+
+  // Create source information for this type.
+  TypeSourceInfo *ResultTInfo = Context.CreateTypeSourceInfo(Result);
+  TypeLoc ResultTL = ResultTInfo->getTypeLoc();
+
+  // For id<Proto1, Proto2> or Class<Proto1, Proto2>, we'll have an
+  // object pointer type. Fill in source information for it.
+  if (auto ObjCObjectPointerTL = ResultTL.getAs<ObjCObjectPointerTypeLoc>()) {
+    // The '*' is implicit.
+    ObjCObjectPointerTL.setStarLoc(SourceLocation());
+    ResultTL = ObjCObjectPointerTL.getPointeeLoc();
+  }
+
+  if (auto OTPTL = ResultTL.getAs<ObjCTypeParamTypeLoc>()) {
+    // Protocol qualifier information.
+    if (OTPTL.getNumProtocols() > 0) {
+      assert(OTPTL.getNumProtocols() == Protocols.size());
+      OTPTL.setProtocolLAngleLoc(ProtocolLAngleLoc);
+      OTPTL.setProtocolRAngleLoc(ProtocolRAngleLoc);
+      for (unsigned i = 0, n = Protocols.size(); i != n; ++i)
+        OTPTL.setProtocolLoc(i, ProtocolLocs[i]);
+    }
+
+    // We're done. Return the completed type to the parser.
+    return SemaRef.CreateParsedType(Result, ResultTInfo);
+  }
+
+  auto ObjCObjectTL = ResultTL.castAs<ObjCObjectTypeLoc>();
+
+  // Type argument information.
+  if (ObjCObjectTL.getNumTypeArgs() > 0) {
+    assert(ObjCObjectTL.getNumTypeArgs() == ActualTypeArgInfos.size());
+    ObjCObjectTL.setTypeArgsLAngleLoc(TypeArgsLAngleLoc);
+    ObjCObjectTL.setTypeArgsRAngleLoc(TypeArgsRAngleLoc);
+    for (unsigned i = 0, n = ActualTypeArgInfos.size(); i != n; ++i)
+      ObjCObjectTL.setTypeArgTInfo(i, ActualTypeArgInfos[i]);
+  } else {
+    ObjCObjectTL.setTypeArgsLAngleLoc(SourceLocation());
+    ObjCObjectTL.setTypeArgsRAngleLoc(SourceLocation());
+  }
+
+  // Protocol qualifier information.
+  if (ObjCObjectTL.getNumProtocols() > 0) {
+    assert(ObjCObjectTL.getNumProtocols() == Protocols.size());
+    ObjCObjectTL.setProtocolLAngleLoc(ProtocolLAngleLoc);
+    ObjCObjectTL.setProtocolRAngleLoc(ProtocolRAngleLoc);
+    for (unsigned i = 0, n = Protocols.size(); i != n; ++i)
+      ObjCObjectTL.setProtocolLoc(i, ProtocolLocs[i]);
+  } else {
+    ObjCObjectTL.setProtocolLAngleLoc(SourceLocation());
+    ObjCObjectTL.setProtocolRAngleLoc(SourceLocation());
+  }
+
+  // Base type.
+  ObjCObjectTL.setHasBaseTypeAsWritten(true);
+  if (ObjCObjectTL.getType() == T)
+    ObjCObjectTL.getBaseLoc().initializeFullCopy(BaseTypeInfo->getTypeLoc());
+  else
+    ObjCObjectTL.getBaseLoc().initialize(Context, Loc);
+
+  // We're done. Return the completed type to the parser.
+  return SemaRef.CreateParsedType(Result, ResultTInfo);
+}
+
+QualType SemaObjC::BuildObjCTypeParamType(
+    const ObjCTypeParamDecl *Decl, SourceLocation ProtocolLAngleLoc,
+    ArrayRef<ObjCProtocolDecl *> Protocols,
+    ArrayRef<SourceLocation> ProtocolLocs, SourceLocation ProtocolRAngleLoc,
+    bool FailOnError) {
+  ASTContext &Context = getASTContext();
+  QualType Result = QualType(Decl->getTypeForDecl(), 0);
+  if (!Protocols.empty()) {
+    bool HasError;
+    Result = Context.applyObjCProtocolQualifiers(Result, Protocols, HasError);
+    if (HasError) {
+      Diag(SourceLocation(), diag::err_invalid_protocol_qualifiers)
+          << SourceRange(ProtocolLAngleLoc, ProtocolRAngleLoc);
+      if (FailOnError)
+        Result = QualType();
+    }
+    if (FailOnError && Result.isNull())
+      return QualType();
+  }
+
+  return Result;
+}
+
+/// Apply Objective-C type arguments to the given type.
+static QualType applyObjCTypeArgs(Sema &S, SourceLocation loc, QualType type,
+                                  ArrayRef<TypeSourceInfo *> typeArgs,
+                                  SourceRange typeArgsRange, bool failOnError,
+                                  bool rebuilding) {
+  // We can only apply type arguments to an Objective-C class type.
+  const auto *objcObjectType = type->getAs<ObjCObjectType>();
+  if (!objcObjectType || !objcObjectType->getInterface()) {
+    S.Diag(loc, diag::err_objc_type_args_non_class) << type << typeArgsRange;
+
+    if (failOnError)
+      return QualType();
+    return type;
+  }
+
+  // The class type must be parameterized.
+  ObjCInterfaceDecl *objcClass = objcObjectType->getInterface();
+  ObjCTypeParamList *typeParams = objcClass->getTypeParamList();
+  if (!typeParams) {
+    S.Diag(loc, diag::err_objc_type_args_non_parameterized_class)
+        << objcClass->getDeclName() << FixItHint::CreateRemoval(typeArgsRange);
+
+    if (failOnError)
+      return QualType();
+
+    return type;
+  }
+
+  // The type must not already be specialized.
+  if (objcObjectType->isSpecialized()) {
+    S.Diag(loc, diag::err_objc_type_args_specialized_class)
+        << type << FixItHint::CreateRemoval(typeArgsRange);
+
+    if (failOnError)
+      return QualType();
+
+    return type;
+  }
+
+  // Check the type arguments.
+  SmallVector<QualType, 4> finalTypeArgs;
+  unsigned numTypeParams = typeParams->size();
+  bool anyPackExpansions = false;
+  for (unsigned i = 0, n = typeArgs.size(); i != n; ++i) {
+    TypeSourceInfo *typeArgInfo = typeArgs[i];
+    QualType typeArg = typeArgInfo->getType();
+
+    // Type arguments cannot have explicit qualifiers or nullability.
+    // We ignore indirect sources of these, e.g. behind typedefs or
+    // template arguments.
+    if (TypeLoc qual = typeArgInfo->getTypeLoc().findExplicitQualifierLoc()) {
+      bool diagnosed = false;
+      SourceRange rangeToRemove;
+      if (auto attr = qual.getAs<AttributedTypeLoc>()) {
+        rangeToRemove = attr.getLocalSourceRange();
+        if (attr.getTypePtr()->getImmediateNullability()) {
+          typeArg = attr.getTypePtr()->getModifiedType();
+          S.Diag(attr.getBeginLoc(),
+                 diag::err_objc_type_arg_explicit_nullability)
+              << typeArg << FixItHint::CreateRemoval(rangeToRemove);
+          diagnosed = true;
+        }
+      }
+
+      // When rebuilding, qualifiers might have gotten here through a
+      // final substitution.
+      if (!rebuilding && !diagnosed) {
+        S.Diag(qual.getBeginLoc(), diag::err_objc_type_arg_qualified)
+            << typeArg << typeArg.getQualifiers().getAsString()
+            << FixItHint::CreateRemoval(rangeToRemove);
+      }
+    }
+
+    // Remove qualifiers even if they're non-local.
+    typeArg = typeArg.getUnqualifiedType();
+
+    finalTypeArgs.push_back(typeArg);
+
+    if (typeArg->getAs<PackExpansionType>())
+      anyPackExpansions = true;
+
+    // Find the corresponding type parameter, if there is one.
+    ObjCTypeParamDecl *typeParam = nullptr;
+    if (!anyPackExpansions) {
+      if (i < numTypeParams) {
+        typeParam = typeParams->begin()[i];
+      } else {
+        // Too many arguments.
+        S.Diag(loc, diag::err_objc_type_args_wrong_arity)
+            << false << objcClass->getDeclName() << (unsigned)typeArgs.size()
+            << numTypeParams;
+        S.Diag(objcClass->getLocation(), diag::note_previous_decl) << objcClass;
+
+        if (failOnError)
+          return QualType();
+
+        return type;
+      }
+    }
+
+    // Objective-C object pointer types must be substitutable for the bounds.
+    if (const auto *typeArgObjC = typeArg->getAs<ObjCObjectPointerType>()) {
+      // If we don't have a type parameter to match against, assume
+      // everything is fine. There was a prior pack expansion that
+      // means we won't be able to match anything.
+      if (!typeParam) {
+        assert(anyPackExpansions && "Too many arguments?");
+        continue;
+      }
+
+      // Retrieve the bound.
+      QualType bound = typeParam->getUnderlyingType();
+      const auto *boundObjC = bound->castAs<ObjCObjectPointerType>();
+
+      // Determine whether the type argument is substitutable for the bound.
+      if (typeArgObjC->isObjCIdType()) {
+        // When the type argument is 'id', the only acceptable type
+        // parameter bound is 'id'.
+        if (boundObjC->isObjCIdType())
+          continue;
+      } else if (S.Context.canAssignObjCInterfaces(boundObjC, typeArgObjC)) {
+        // Otherwise, we follow the assignability rules.
+        continue;
+      }
+
+      // Diagnose the mismatch.
+      S.Diag(typeArgInfo->getTypeLoc().getBeginLoc(),
+             diag::err_objc_type_arg_does_not_match_bound)
+          << typeArg << bound << typeParam->getDeclName();
+      S.Diag(typeParam->getLocation(), diag::note_objc_type_param_here)
+          << typeParam->getDeclName();
+
+      if (failOnError)
+        return QualType();
+
+      return type;
+    }
+
+    // Block pointer types are permitted for unqualified 'id' bounds.
+    if (typeArg->isBlockPointerType()) {
+      // If we don't have a type parameter to match against, assume
+      // everything is fine. There was a prior pack expansion that
+      // means we won't be able to match anything.
+      if (!typeParam) {
+        assert(anyPackExpansions && "Too many arguments?");
+        continue;
+      }
+
+      // Retrieve the bound.
+      QualType bound = typeParam->getUnderlyingType();
+      if (bound->isBlockCompatibleObjCPointerType(S.Context))
+        continue;
+
+      // Diagnose the mismatch.
+      S.Diag(typeArgInfo->getTypeLoc().getBeginLoc(),
+             diag::err_objc_type_arg_does_not_match_bound)
+          << typeArg << bound << typeParam->getDeclName();
+      S.Diag(typeParam->getLocation(), diag::note_objc_type_param_here)
+          << typeParam->getDeclName();
+
+      if (failOnError)
+        return QualType();
+
+      return type;
+    }
+
+    // Types that have __attribute__((NSObject)) are permitted.
+    if (typeArg->isObjCNSObjectType()) {
+      continue;
+    }
+
+    // Dependent types will be checked at instantiation time.
+    if (typeArg->isDependentType()) {
+      continue;
+    }
+
+    // Diagnose non-id-compatible type arguments.
+    S.Diag(typeArgInfo->getTypeLoc().getBeginLoc(),
+           diag::err_objc_type_arg_not_id_compatible)
+        << typeArg << typeArgInfo->getTypeLoc().getSourceRange();
+
+    if (failOnError)
+      return QualType();
+
+    return type;
+  }
+
+  // Make sure we didn't have the wrong number of arguments.
+  if (!anyPackExpansions && finalTypeArgs.size() != numTypeParams) {
+    S.Diag(loc, diag::err_objc_type_args_wrong_arity)
+        << (typeArgs.size() < typeParams->size()) << objcClass->getDeclName()
+        << (unsigned)finalTypeArgs.size() << (unsigned)numTypeParams;
+    S.Diag(objcClass->getLocation(), diag::note_previous_decl) << objcClass;
+
+    if (failOnError)
+      return QualType();
+
+    return type;
+  }
+
+  // Success. Form the specialized type.
+  return S.Context.getObjCObjectType(type, finalTypeArgs, {}, false);
+}
+
+QualType SemaObjC::BuildObjCObjectType(
+    QualType BaseType, SourceLocation Loc, SourceLocation TypeArgsLAngleLoc,
+    ArrayRef<TypeSourceInfo *> TypeArgs, SourceLocation TypeArgsRAngleLoc,
+    SourceLocation ProtocolLAngleLoc, ArrayRef<ObjCProtocolDecl *> Protocols,
+    ArrayRef<SourceLocation> ProtocolLocs, SourceLocation ProtocolRAngleLoc,
+    bool FailOnError, bool Rebuilding) {
+  ASTContext &Context = getASTContext();
+  QualType Result = BaseType;
+  if (!TypeArgs.empty()) {
+    Result =
+        applyObjCTypeArgs(SemaRef, Loc, Result, TypeArgs,
+                          SourceRange(TypeArgsLAngleLoc, TypeArgsRAngleLoc),
+                          FailOnError, Rebuilding);
+    if (FailOnError && Result.isNull())
+      return QualType();
+  }
+
+  if (!Protocols.empty()) {
+    bool HasError;
+    Result = Context.applyObjCProtocolQualifiers(Result, Protocols, HasError);
+    if (HasError) {
+      Diag(Loc, diag::err_invalid_protocol_qualifiers)
+          << SourceRange(ProtocolLAngleLoc, ProtocolRAngleLoc);
+      if (FailOnError)
+        Result = QualType();
+    }
+    if (FailOnError && Result.isNull())
+      return QualType();
+  }
+
+  return Result;
+}
+
+ParsedType SemaObjC::ActOnObjCInstanceType(SourceLocation Loc) {
+  ASTContext &Context = getASTContext();
+  QualType T = Context.getObjCInstanceType();
+  TypeSourceInfo *TInfo = Context.getTrivialTypeSourceInfo(T, Loc);
+  return SemaRef.CreateParsedType(T, TInfo);
+}
+
+//===--- CHECK: Objective-C retain cycles ----------------------------------//
+
+namespace {
+
+struct RetainCycleOwner {
+  VarDecl *Variable = nullptr;
+  SourceRange Range;
+  SourceLocation Loc;
+  bool Indirect = false;
+
+  RetainCycleOwner() = default;
+
+  void setLocsFrom(Expr *e) {
+    Loc = e->getExprLoc();
+    Range = e->getSourceRange();
+  }
+};
+
+} // namespace
+
+/// Consider whether capturing the given variable can possibly lead to
+/// a retain cycle.
+static bool considerVariable(VarDecl *var, Expr *ref, RetainCycleOwner &owner) {
+  // In ARC, it's captured strongly iff the variable has __strong
+  // lifetime.  In MRR, it's captured strongly if the variable is
+  // __block and has an appropriate type.
+  if (var->getType().getObjCLifetime() != Qualifiers::OCL_Strong)
+    return false;
+
+  owner.Variable = var;
+  if (ref)
+    owner.setLocsFrom(ref);
+  return true;
+}
+
+static bool findRetainCycleOwner(Sema &S, Expr *e, RetainCycleOwner &owner) {
+  while (true) {
+    e = e->IgnoreParens();
+    if (CastExpr *cast = dyn_cast<CastExpr>(e)) {
+      switch (cast->getCastKind()) {
+      case CK_BitCast:
+      case CK_LValueBitCast:
+      case CK_LValueToRValue:
+      case CK_ARCReclaimReturnedObject:
+        e = cast->getSubExpr();
+        continue;
+
+      default:
+        return false;
+      }
+    }
+
+    if (ObjCIvarRefExpr *ref = dyn_cast<ObjCIvarRefExpr>(e)) {
+      ObjCIvarDecl *ivar = ref->getDecl();
+      if (ivar->getType().getObjCLifetime() != Qualifiers::OCL_Strong)
+        return false;
+
+      // Try to find a retain cycle in the base.
+      if (!findRetainCycleOwner(S, ref->getBase(), owner))
+        return false;
+
+      if (ref->isFreeIvar())
+        owner.setLocsFrom(ref);
+      owner.Indirect = true;
+      return true;
+    }
+
+    if (DeclRefExpr *ref = dyn_cast<DeclRefExpr>(e)) {
+      VarDecl *var = dyn_cast<VarDecl>(ref->getDecl());
+      if (!var)
+        return false;
+      return considerVariable(var, ref, owner);
+    }
+
+    if (MemberExpr *member = dyn_cast<MemberExpr>(e)) {
+      if (member->isArrow())
+        return false;
+
+      // Don't count this as an indirect ownership.
+      e = member->getBase();
+      continue;
+    }
+
+    if (PseudoObjectExpr *pseudo = dyn_cast<PseudoObjectExpr>(e)) {
+      // Only pay attention to pseudo-objects on property references.
+      ObjCPropertyRefExpr *pre = dyn_cast<ObjCPropertyRefExpr>(
+          pseudo->getSyntacticForm()->IgnoreParens());
+      if (!pre)
+        return false;
+      if (pre->isImplicitProperty())
+        return false;
+      ObjCPropertyDecl *property = pre->getExplicitProperty();
+      if (!property->isRetaining() &&
+          !(property->getPropertyIvarDecl() &&
+            property->getPropertyIvarDecl()->getType().getObjCLifetime() ==
+                Qualifiers::OCL_Strong))
+        return false;
+
+      owner.Indirect = true;
+      if (pre->isSuperReceiver()) {
+        owner.Variable = S.getCurMethodDecl()->getSelfDecl();
+        if (!owner.Variable)
+          return false;
+        owner.Loc = pre->getLocation();
+        owner.Range = pre->getSourceRange();
+        return true;
+      }
+      e = const_cast<Expr *>(
+          cast<OpaqueValueExpr>(pre->getBase())->getSourceExpr());
+      continue;
+    }
+
+    // Array ivars?
+
+    return false;
+  }
+}
+
+namespace {
+
+struct FindCaptureVisitor : EvaluatedExprVisitor<FindCaptureVisitor> {
+  VarDecl *Variable;
+  Expr *Capturer = nullptr;
+  bool VarWillBeReased = false;
+
+  FindCaptureVisitor(ASTContext &Context, VarDecl *variable)
+      : EvaluatedExprVisitor<FindCaptureVisitor>(Context), Variable(variable) {}
+
+  void VisitDeclRefExpr(DeclRefExpr *ref) {
+    if (ref->getDecl() == Variable && !Capturer)
+      Capturer = ref;
+  }
+
+  void VisitObjCIvarRefExpr(ObjCIvarRefExpr *ref) {
+    if (Capturer)
+      return;
+    Visit(ref->getBase());
+    if (Capturer && ref->isFreeIvar())
+      Capturer = ref;
+  }
+
+  void VisitBlockExpr(BlockExpr *block) {
+    // Look inside nested blocks
+    if (block->getBlockDecl()->capturesVariable(Variable))
+      Visit(block->getBlockDecl()->getBody());
+  }
+
+  void VisitOpaqueValueExpr(OpaqueValueExpr *OVE) {
+    if (Capturer)
+      return;
+    if (OVE->getSourceExpr())
+      Visit(OVE->getSourceExpr());
+  }
+
+  void VisitBinaryOperator(BinaryOperator *BinOp) {
+    if (!Variable || VarWillBeReased || BinOp->getOpcode() != BO_Assign)
+      return;
+    Expr *LHS = BinOp->getLHS();
+    if (const DeclRefExpr *DRE = dyn_cast_or_null<DeclRefExpr>(LHS)) {
+      if (DRE->getDecl() != Variable)
+        return;
+      if (Expr *RHS = BinOp->getRHS()) {
+        RHS = RHS->IgnoreParenCasts();
+        std::optional<llvm::APSInt> Value;
+        VarWillBeReased =
+            (RHS && (Value = RHS->getIntegerConstantExpr(Context)) &&
+             *Value == 0);
+      }
+    }
+  }
+};
+
+} // namespace
+
+/// Check whether the given argument is a block which captures a
+/// variable.
+static Expr *findCapturingExpr(Sema &S, Expr *e, RetainCycleOwner &owner) {
+  assert(owner.Variable && owner.Loc.isValid());
+
+  e = e->IgnoreParenCasts();
+
+  // Look through [^{...} copy] and Block_copy(^{...}).
+  if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(e)) {
+    Selector Cmd = ME->getSelector();
+    if (Cmd.isUnarySelector() && Cmd.getNameForSlot(0) == "copy") {
+      e = ME->getInstanceReceiver();
+      if (!e)
+        return nullptr;
+      e = e->IgnoreParenCasts();
+    }
+  } else if (CallExpr *CE = dyn_cast<CallExpr>(e)) {
+    if (CE->getNumArgs() == 1) {
+      FunctionDecl *Fn = dyn_cast_or_null<FunctionDecl>(CE->getCalleeDecl());
+      if (Fn) {
+        const IdentifierInfo *FnI = Fn->getIdentifier();
+        if (FnI && FnI->isStr("_Block_copy")) {
+          e = CE->getArg(0)->IgnoreParenCasts();
+        }
+      }
+    }
+  }
+
+  BlockExpr *block = dyn_cast<BlockExpr>(e);
+  if (!block || !block->getBlockDecl()->capturesVariable(owner.Variable))
+    return nullptr;
+
+  FindCaptureVisitor visitor(S.Context, owner.Variable);
+  visitor.Visit(block->getBlockDecl()->getBody());
+  return visitor.VarWillBeReased ? nullptr : visitor.Capturer;
+}
+
+static void diagnoseRetainCycle(Sema &S, Expr *capturer,
+                                RetainCycleOwner &owner) {
+  assert(capturer);
+  assert(owner.Variable && owner.Loc.isValid());
+
+  S.Diag(capturer->getExprLoc(), diag::warn_arc_retain_cycle)
+      << owner.Variable << capturer->getSourceRange();
+  S.Diag(owner.Loc, diag::note_arc_retain_cycle_owner)
+      << owner.Indirect << owner.Range;
+}
+
+/// Check for a keyword selector that starts with the word 'add' or
+/// 'set'.
+static bool isSetterLikeSelector(Selector sel) {
+  if (sel.isUnarySelector())
+    return false;
+
+  StringRef str = sel.getNameForSlot(0);
+  str = str.ltrim('_');
+  if (str.starts_with("set"))
+    str = str.substr(3);
+  else if (str.starts_with("add")) {
+    // Specially allow 'addOperationWithBlock:'.
+    if (sel.getNumArgs() == 1 && str.starts_with("addOperationWithBlock"))
+      return false;
+    str = str.substr(3);
+  } else
+    return false;
+
+  if (str.empty())
+    return true;
+  return !isLowercase(str.front());
+}
+
+static std::optional<int>
+GetNSMutableArrayArgumentIndex(SemaObjC &S, ObjCMessageExpr *Message) {
+  bool IsMutableArray = S.NSAPIObj->isSubclassOfNSClass(
+      Message->getReceiverInterface(), NSAPI::ClassId_NSMutableArray);
+  if (!IsMutableArray) {
+    return std::nullopt;
+  }
+
+  Selector Sel = Message->getSelector();
+
+  std::optional<NSAPI::NSArrayMethodKind> MKOpt =
+      S.NSAPIObj->getNSArrayMethodKind(Sel);
+  if (!MKOpt) {
+    return std::nullopt;
+  }
+
+  NSAPI::NSArrayMethodKind MK = *MKOpt;
+
+  switch (MK) {
+  case NSAPI::NSMutableArr_addObject:
+  case NSAPI::NSMutableArr_insertObjectAtIndex:
+  case NSAPI::NSMutableArr_setObjectAtIndexedSubscript:
+    return 0;
+  case NSAPI::NSMutableArr_replaceObjectAtIndex:
+    return 1;
+
+  default:
+    return std::nullopt;
+  }
+
+  return std::nullopt;
+}
+
+static std::optional<int>
+GetNSMutableDictionaryArgumentIndex(SemaObjC &S, ObjCMessageExpr *Message) {
+  bool IsMutableDictionary = S.NSAPIObj->isSubclassOfNSClass(
+      Message->getReceiverInterface(), NSAPI::ClassId_NSMutableDictionary);
+  if (!IsMutableDictionary) {
+    return std::nullopt;
+  }
+
+  Selector Sel = Message->getSelector();
+
+  std::optional<NSAPI::NSDictionaryMethodKind> MKOpt =
+      S.NSAPIObj->getNSDictionaryMethodKind(Sel);
+  if (!MKOpt) {
+    return std::nullopt;
+  }
+
+  NSAPI::NSDictionaryMethodKind MK = *MKOpt;
+
+  switch (MK) {
+  case NSAPI::NSMutableDict_setObjectForKey:
+  case NSAPI::NSMutableDict_setValueForKey:
+  case NSAPI::NSMutableDict_setObjectForKeyedSubscript:
+    return 0;
+
+  default:
+    return std::nullopt;
+  }
+
+  return std::nullopt;
+}
+
+static std::optional<int> GetNSSetArgumentIndex(SemaObjC &S,
+                                                ObjCMessageExpr *Message) {
+  bool IsMutableSet = S.NSAPIObj->isSubclassOfNSClass(
+      Message->getReceiverInterface(), NSAPI::ClassId_NSMutableSet);
+
+  bool IsMutableOrderedSet = S.NSAPIObj->isSubclassOfNSClass(
+      Message->getReceiverInterface(), NSAPI::ClassId_NSMutableOrderedSet);
+  if (!IsMutableSet && !IsMutableOrderedSet) {
+    return std::nullopt;
+  }
+
+  Selector Sel = Message->getSelector();
+
+  std::optional<NSAPI::NSSetMethodKind> MKOpt =
+      S.NSAPIObj->getNSSetMethodKind(Sel);
+  if (!MKOpt) {
+    return std::nullopt;
+  }
+
+  NSAPI::NSSetMethodKind MK = *MKOpt;
+
+  switch (MK) {
+  case NSAPI::NSMutableSet_addObject:
+  case NSAPI::NSOrderedSet_setObjectAtIndex:
+  case NSAPI::NSOrderedSet_setObjectAtIndexedSubscript:
+  case NSAPI::NSOrderedSet_insertObjectAtIndex:
+    return 0;
+  case NSAPI::NSOrderedSet_replaceObjectAtIndexWithObject:
+    return 1;
+  }
+
+  return std::nullopt;
+}
+
+void SemaObjC::CheckObjCCircularContainer(ObjCMessageExpr *Message) {
+  if (!Message->isInstanceMessage()) {
+    return;
+  }
+
+  std::optional<int> ArgOpt;
+
+  if (!(ArgOpt = GetNSMutableArrayArgumentIndex(*this, Message)) &&
+      !(ArgOpt = GetNSMutableDictionaryArgumentIndex(*this, Message)) &&
+      !(ArgOpt = GetNSSetArgumentIndex(*this, Message))) {
+    return;
+  }
+
+  int ArgIndex = *ArgOpt;
+
+  Expr *Arg = Message->getArg(ArgIndex)->IgnoreImpCasts();
+  if (OpaqueValueExpr *OE = dyn_cast<OpaqueValueExpr>(Arg)) {
+    Arg = OE->getSourceExpr()->IgnoreImpCasts();
+  }
+
+  if (Message->getReceiverKind() == ObjCMessageExpr::SuperInstance) {
+    if (DeclRefExpr *ArgRE = dyn_cast<DeclRefExpr>(Arg)) {
+      if (ArgRE->isObjCSelfExpr()) {
+        Diag(Message->getSourceRange().getBegin(),
+             diag::warn_objc_circular_container)
+            << ArgRE->getDecl() << StringRef("'super'");
+      }
+    }
+  } else {
+    Expr *Receiver = Message->getInstanceReceiver()->IgnoreImpCasts();
+
+    if (OpaqueValueExpr *OE = dyn_cast<OpaqueValueExpr>(Receiver)) {
+      Receiver = OE->getSourceExpr()->IgnoreImpCasts();
+    }
+
+    if (DeclRefExpr *ReceiverRE = dyn_cast<DeclRefExpr>(Receiver)) {
+      if (DeclRefExpr *ArgRE = dyn_cast<DeclRefExpr>(Arg)) {
+        if (ReceiverRE->getDecl() == ArgRE->getDecl()) {
+          ValueDecl *Decl = ReceiverRE->getDecl();
+          Diag(Message->getSourceRange().getBegin(),
+               diag::warn_objc_circular_container)
+              << Decl << Decl;
+          if (!ArgRE->isObjCSelfExpr()) {
+            Diag(Decl->getLocation(),
+                 diag::note_objc_circular_container_declared_here)
+                << Decl;
+          }
+        }
+      }
+    } else if (ObjCIvarRefExpr *IvarRE = dyn_cast<ObjCIvarRefExpr>(Receiver)) {
+      if (ObjCIvarRefExpr *IvarArgRE = dyn_cast<ObjCIvarRefExpr>(Arg)) {
+        if (IvarRE->getDecl() == IvarArgRE->getDecl()) {
+          ObjCIvarDecl *Decl = IvarRE->getDecl();
+          Diag(Message->getSourceRange().getBegin(),
+               diag::warn_objc_circular_container)
+              << Decl << Decl;
+          Diag(Decl->getLocation(),
+               diag::note_objc_circular_container_declared_here)
+              << Decl;
+        }
+      }
+    }
+  }
+}
+
+/// Check a message send to see if it's likely to cause a retain cycle.
+void SemaObjC::checkRetainCycles(ObjCMessageExpr *msg) {
+  // Only check instance methods whose selector looks like a setter.
+  if (!msg->isInstanceMessage() || !isSetterLikeSelector(msg->getSelector()))
+    return;
+
+  // Try to find a variable that the receiver is strongly owned by.
+  RetainCycleOwner owner;
+  if (msg->getReceiverKind() == ObjCMessageExpr::Instance) {
+    if (!findRetainCycleOwner(SemaRef, msg->getInstanceReceiver(), owner))
+      return;
+  } else {
+    assert(msg->getReceiverKind() == ObjCMessageExpr::SuperInstance);
+    owner.Variable = SemaRef.getCurMethodDecl()->getSelfDecl();
+    owner.Loc = msg->getSuperLoc();
+    owner.Range = msg->getSuperLoc();
+  }
+
+  // Check whether the receiver is captured by any of the arguments.
+  const ObjCMethodDecl *MD = msg->getMethodDecl();
+  for (unsigned i = 0, e = msg->getNumArgs(); i != e; ++i) {
+    if (Expr *capturer = findCapturingExpr(SemaRef, msg->getArg(i), owner)) {
+      // noescape blocks should not be retained by the method.
+      if (MD && MD->parameters()[i]->hasAttr<NoEscapeAttr>())
+        continue;
+      return diagnoseRetainCycle(SemaRef, capturer, owner);
+    }
+  }
+}
+
+/// Check a property assign to see if it's likely to cause a retain cycle.
+void SemaObjC::checkRetainCycles(Expr *receiver, Expr *argument) {
+  RetainCycleOwner owner;
+  if (!findRetainCycleOwner(SemaRef, receiver, owner))
+    return;
+
+  if (Expr *capturer = findCapturingExpr(SemaRef, argument, owner))
+    diagnoseRetainCycle(SemaRef, capturer, owner);
+}
+
+void SemaObjC::checkRetainCycles(VarDecl *Var, Expr *Init) {
+  RetainCycleOwner Owner;
+  if (!considerVariable(Var, /*DeclRefExpr=*/nullptr, Owner))
+    return;
+
+  // Because we don't have an expression for the variable, we have to set the
+  // location explicitly here.
+  Owner.Loc = Var->getLocation();
+  Owner.Range = Var->getSourceRange();
+
+  if (Expr *Capturer = findCapturingExpr(SemaRef, Init, Owner))
+    diagnoseRetainCycle(SemaRef, Capturer, Owner);
+}
+
+/// CheckObjCString - Checks that the argument to the builtin
+/// CFString constructor is correct
+/// Note: It might also make sense to do the UTF-16 conversion here (would
+/// simplify the backend).
+bool SemaObjC::CheckObjCString(Expr *Arg) {
+  Arg = Arg->IgnoreParenCasts();
+  StringLiteral *Literal = dyn_cast<StringLiteral>(Arg);
+
+  if (!Literal || !Literal->isOrdinary()) {
+    Diag(Arg->getBeginLoc(), diag::err_cfstring_literal_not_string_constant)
+        << Arg->getSourceRange();
+    return true;
+  }
+
+  if (Literal->containsNonAsciiOrNull()) {
+    StringRef String = Literal->getString();
+    unsigned NumBytes = String.size();
+    SmallVector<llvm::UTF16, 128> ToBuf(NumBytes);
+    const llvm::UTF8 *FromPtr = (const llvm::UTF8 *)String.data();
+    llvm::UTF16 *ToPtr = &ToBuf[0];
+
+    llvm::ConversionResult Result =
+        llvm::ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes, &ToPtr,
+                                 ToPtr + NumBytes, llvm::strictConversion);
+    // Check for conversion failure.
+    if (Result != llvm::conversionOK)
+      Diag(Arg->getBeginLoc(), diag::warn_cfstring_truncated)
+          << Arg->getSourceRange();
+  }
+  return false;
+}
+
+bool SemaObjC::CheckObjCMethodCall(ObjCMethodDecl *Method, SourceLocation lbrac,
+                                   ArrayRef<const Expr *> Args) {
+  Sema::VariadicCallType CallType =
+      Method->isVariadic() ? Sema::VariadicMethod : Sema::VariadicDoesNotApply;
+
+  SemaRef.checkCall(Method, nullptr, /*ThisArg=*/nullptr, Args,
+                    /*IsMemberFunction=*/false, lbrac, Method->getSourceRange(),
+                    CallType);
+
+  SemaRef.CheckTCBEnforcement(lbrac, Method);
+
+  return false;
+}
+
+const DeclContext *SemaObjC::getCurObjCLexicalContext() const {
+  const DeclContext *DC = SemaRef.getCurLexicalContext();
+  // A category implicitly has the attribute of the interface.
+  if (const ObjCCategoryDecl *CatD = dyn_cast<ObjCCategoryDecl>(DC))
+    DC = CatD->getClassInterface();
+  return DC;
+}
+
+/// Retrieve the identifier "NSError".
+IdentifierInfo *SemaObjC::getNSErrorIdent() {
+  if (!Ident_NSError)
+    Ident_NSError = SemaRef.PP.getIdentifierInfo("NSError");
+
+  return Ident_NSError;
+}
+
+void SemaObjC::ActOnObjCContainerStartDefinition(ObjCContainerDecl *IDecl) {
+  assert(
+      IDecl->getLexicalParent() == SemaRef.CurContext &&
+      "The next DeclContext should be lexically contained in the current one.");
+  SemaRef.CurContext = IDecl;
+}
+
+void SemaObjC::ActOnObjCContainerFinishDefinition() {
+  // Exit this scope of this interface definition.
+  SemaRef.PopDeclContext();
+}
+
+void SemaObjC::ActOnObjCTemporaryExitContainerContext(
+    ObjCContainerDecl *ObjCCtx) {
+  assert(ObjCCtx == SemaRef.CurContext && "Mismatch of container contexts");
+  SemaRef.OriginalLexicalContext = ObjCCtx;
+  ActOnObjCContainerFinishDefinition();
+}
+
+void SemaObjC::ActOnObjCReenterContainerContext(ObjCContainerDecl *ObjCCtx) {
+  ActOnObjCContainerStartDefinition(ObjCCtx);
+  SemaRef.OriginalLexicalContext = nullptr;
+}
+
+/// Find the protocol with the given name, if any.
+ObjCProtocolDecl *SemaObjC::LookupProtocol(IdentifierInfo *II,
+                                           SourceLocation IdLoc,
+                                           RedeclarationKind Redecl) {
+  Decl *D = SemaRef.LookupSingleName(SemaRef.TUScope, II, IdLoc,
+                                     Sema::LookupObjCProtocolName, Redecl);
+  return cast_or_null<ObjCProtocolDecl>(D);
+}
+
+/// Determine whether this is an Objective-C writeback conversion,
+/// used for parameter passing when performing automatic reference counting.
+///
+/// \param FromType The type we're converting form.
+///
+/// \param ToType The type we're converting to.
+///
+/// \param ConvertedType The type that will be produced after applying
+/// this conversion.
+bool SemaObjC::isObjCWritebackConversion(QualType FromType, QualType ToType,
+                                         QualType &ConvertedType) {
+  ASTContext &Context = getASTContext();
+  if (!getLangOpts().ObjCAutoRefCount ||
+      Context.hasSameUnqualifiedType(FromType, ToType))
+    return false;
+
+  // Parameter must be a pointer to __autoreleasing (with no other qualifiers).
+  QualType ToPointee;
+  if (const PointerType *ToPointer = ToType->getAs<PointerType>())
+    ToPointee = ToPointer->getPointeeType();
+  else
+    return false;
+
+  Qualifiers ToQuals = ToPointee.getQualifiers();
+  if (!ToPointee->isObjCLifetimeType() ||
+      ToQuals.getObjCLifetime() != Qualifiers::OCL_Autoreleasing ||
+      !ToQuals.withoutObjCLifetime().empty())
+    return false;
+
+  // Argument must be a pointer to __strong to __weak.
+  QualType FromPointee;
+  if (const PointerType *FromPointer = FromType->getAs<PointerType>())
+    FromPointee = FromPointer->getPointeeType();
+  else
+    return false;
+
+  Qualifiers FromQuals = FromPointee.getQualifiers();
+  if (!FromPointee->isObjCLifetimeType() ||
+      (FromQuals.getObjCLifetime() != Qualifiers::OCL_Strong &&
+       FromQuals.getObjCLifetime() != Qualifiers::OCL_Weak))
+    return false;
+
+  // Make sure that we have compatible qualifiers.
+  FromQuals.setObjCLifetime(Qualifiers::OCL_Autoreleasing);
+  if (!ToQuals.compatiblyIncludes(FromQuals))
+    return false;
+
+  // Remove qualifiers from the pointee type we're converting from; they
+  // aren't used in the compatibility check belong, and we'll be adding back
+  // qualifiers (with __autoreleasing) if the compatibility check succeeds.
+  FromPointee = FromPointee.getUnqualifiedType();
+
+  // The unqualified form of the pointee types must be compatible.
+  ToPointee = ToPointee.getUnqualifiedType();
+  bool IncompatibleObjC;
+  if (Context.typesAreCompatible(FromPointee, ToPointee))
+    FromPointee = ToPointee;
+  else if (!SemaRef.isObjCPointerConversion(FromPointee, ToPointee, FromPointee,
+                                            IncompatibleObjC))
+    return false;
+
+  /// Construct the type we're converting to, which is a pointer to
+  /// __autoreleasing pointee.
+  FromPointee = Context.getQualifiedType(FromPointee, FromQuals);
+  ConvertedType = Context.getPointerType(FromPointee);
+  return true;
+}
+
+/// CheckSubscriptingKind - This routine decide what type
+/// of indexing represented by "FromE" is being done.
+SemaObjC::ObjCSubscriptKind SemaObjC::CheckSubscriptingKind(Expr *FromE) {
+  // If the expression already has integral or enumeration type, we're golden.
+  QualType T = FromE->getType();
+  if (T->isIntegralOrEnumerationType())
+    return SemaObjC::OS_Array;
+
+  // If we don't have a class type in C++, there's no way we can get an
+  // expression of integral or enumeration type.
+  const RecordType *RecordTy = T->getAs<RecordType>();
+  if (!RecordTy && (T->isObjCObjectPointerType() || T->isVoidPointerType()))
+    // All other scalar cases are assumed to be dictionary indexing which
+    // caller handles, with diagnostics if needed.
+    return SemaObjC::OS_Dictionary;
+  if (!getLangOpts().CPlusPlus || !RecordTy || RecordTy->isIncompleteType()) {
+    // No indexing can be done. Issue diagnostics and quit.
+    const Expr *IndexExpr = FromE->IgnoreParenImpCasts();
+    if (isa<StringLiteral>(IndexExpr))
+      Diag(FromE->getExprLoc(), diag::err_objc_subscript_pointer)
+          << T << FixItHint::CreateInsertion(FromE->getExprLoc(), "@");
+    else
+      Diag(FromE->getExprLoc(), diag::err_objc_subscript_type_conversion) << T;
+    return SemaObjC::OS_Error;
+  }
+
+  // We must have a complete class type.
+  if (SemaRef.RequireCompleteType(FromE->getExprLoc(), T,
+                                  diag::err_objc_index_incomplete_class_type,
+                                  FromE))
+    return SemaObjC::OS_Error;
+
+  // Look for a conversion to an integral, enumeration type, or
+  // objective-C pointer type.
+  int NoIntegrals = 0, NoObjCIdPointers = 0;
+  SmallVector<CXXConversionDecl *, 4> ConversionDecls;
+
+  for (NamedDecl *D : cast<CXXRecordDecl>(RecordTy->getDecl())
+                          ->getVisibleConversionFunctions()) {
+    if (CXXConversionDecl *Conversion =
+            dyn_cast<CXXConversionDecl>(D->getUnderlyingDecl())) {
+      QualType CT = Conversion->getConversionType().getNonReferenceType();
+      if (CT->isIntegralOrEnumerationType()) {
+        ++NoIntegrals;
+        ConversionDecls.push_back(Conversion);
+      } else if (CT->isObjCIdType() || CT->isBlockPointerType()) {
+        ++NoObjCIdPointers;
+        ConversionDecls.push_back(Conversion);
+      }
+    }
+  }
+  if (NoIntegrals == 1 && NoObjCIdPointers == 0)
+    return SemaObjC::OS_Array;
+  if (NoIntegrals == 0 && NoObjCIdPointers == 1)
+    return SemaObjC::OS_Dictionary;
+  if (NoIntegrals == 0 && NoObjCIdPointers == 0) {
+    // No conversion function was found. Issue diagnostic and return.
+    Diag(FromE->getExprLoc(), diag::err_objc_subscript_type_conversion)
+        << FromE->getType();
+    return SemaObjC::OS_Error;
+  }
+  Diag(FromE->getExprLoc(), diag::err_objc_multiple_subscript_type_conversion)
+      << FromE->getType();
+  for (unsigned int i = 0; i < ConversionDecls.size(); i++)
+    Diag(ConversionDecls[i]->getLocation(),
+         diag::note_conv_function_declared_at);
+
+  return SemaObjC::OS_Error;
+}
+
+void SemaObjC::AddCFAuditedAttribute(Decl *D) {
+  ASTContext &Context = getASTContext();
+  IdentifierInfo *Ident;
+  SourceLocation Loc;
+  std::tie(Ident, Loc) = SemaRef.PP.getPragmaARCCFCodeAuditedInfo();
+  if (!Loc.isValid())
+    return;
+
+  // Don't add a redundant or conflicting attribute.
+  if (D->hasAttr<CFAuditedTransferAttr>() ||
+      D->hasAttr<CFUnknownTransferAttr>())
+    return;
+
+  AttributeCommonInfo Info(Ident, SourceRange(Loc),
+                           AttributeCommonInfo::Form::Pragma());
+  D->addAttr(CFAuditedTransferAttr::CreateImplicit(Context, Info));
+}
+
+bool SemaObjC::isCFError(RecordDecl *RD) {
+  // If we already know about CFError, test it directly.
+  if (CFError)
+    return CFError == RD;
+
+  // Check whether this is CFError, which we identify based on its bridge to
+  // NSError. CFErrorRef used to be declared with "objc_bridge" but is now
+  // declared with "objc_bridge_mutable", so look for either one of the two
+  // attributes.
+  if (RD->getTagKind() == TagTypeKind::Struct) {
+    IdentifierInfo *bridgedType = nullptr;
+    if (auto bridgeAttr = RD->getAttr<ObjCBridgeAttr>())
+      bridgedType = bridgeAttr->getBridgedType();
+    else if (auto bridgeAttr = RD->getAttr<ObjCBridgeMutableAttr>())
+      bridgedType = bridgeAttr->getBridgedType();
+
+    if (bridgedType == getNSErrorIdent()) {
+      CFError = RD;
+      return true;
+    }
+  }
+
+  return false;
+}
+
+} // namespace clang
diff --git a/clang/lib/Sema/SemaObjCProperty.cpp b/clang/lib/Sema/SemaObjCProperty.cpp
index 222a65a13dd0..031f2a6af877 100644
--- a/clang/lib/Sema/SemaObjCProperty.cpp
+++ b/clang/lib/Sema/SemaObjCProperty.cpp
@@ -11,7 +11,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Sema/SemaInternal.h"
 #include "clang/AST/ASTMutationListener.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/ExprCXX.h"
@@ -20,6 +19,8 @@
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Initialization.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallString.h"
 
@@ -114,7 +115,8 @@ CheckPropertyAgainstProtocol(Sema &S, ObjCPropertyDecl *Prop,
   // Look for a property with the same name.
   if (ObjCPropertyDecl *ProtoProp = Proto->getProperty(
           Prop->getIdentifier(), Prop->isInstanceProperty())) {
-    S.DiagnosePropertyMismatch(Prop, ProtoProp, Proto->getIdentifier(), true);
+    S.ObjC().DiagnosePropertyMismatch(Prop, ProtoProp, Proto->getIdentifier(),
+                                      true);
     return;
   }
 
@@ -169,28 +171,26 @@ static unsigned getOwnershipRule(unsigned attr) {
   return result;
 }
 
-Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
-                          SourceLocation LParenLoc,
-                          FieldDeclarator &FD,
-                          ObjCDeclSpec &ODS,
-                          Selector GetterSel,
-                          Selector SetterSel,
-                          tok::ObjCKeywordKind MethodImplKind,
-                          DeclContext *lexicalDC) {
+Decl *SemaObjC::ActOnProperty(Scope *S, SourceLocation AtLoc,
+                              SourceLocation LParenLoc, FieldDeclarator &FD,
+                              ObjCDeclSpec &ODS, Selector GetterSel,
+                              Selector SetterSel,
+                              tok::ObjCKeywordKind MethodImplKind,
+                              DeclContext *lexicalDC) {
   unsigned Attributes = ODS.getPropertyAttributes();
   FD.D.setObjCWeakProperty((Attributes & ObjCPropertyAttribute::kind_weak) !=
                            0);
-  TypeSourceInfo *TSI = GetTypeForDeclarator(FD.D);
+  TypeSourceInfo *TSI = SemaRef.GetTypeForDeclarator(FD.D);
   QualType T = TSI->getType();
   if (!getOwnershipRule(Attributes)) {
-    Attributes |= deducePropertyOwnershipFromType(*this, T);
+    Attributes |= deducePropertyOwnershipFromType(SemaRef, T);
   }
   bool isReadWrite = ((Attributes & ObjCPropertyAttribute::kind_readwrite) ||
                       // default is readwrite!
                       !(Attributes & ObjCPropertyAttribute::kind_readonly));
 
   // Proceed with constructing the ObjCPropertyDecls.
-  ObjCContainerDecl *ClassDecl = cast<ObjCContainerDecl>(CurContext);
+  ObjCContainerDecl *ClassDecl = cast<ObjCContainerDecl>(SemaRef.CurContext);
   ObjCPropertyDecl *Res = nullptr;
   if (ObjCCategoryDecl *CDecl = dyn_cast<ObjCCategoryDecl>(ClassDecl)) {
     if (CDecl->IsClassExtension()) {
@@ -223,7 +223,7 @@ Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
 
   // Check consistency if the type has explicit ownership qualification.
   if (Res->getType().getObjCLifetime())
-    checkPropertyDeclWithOwnership(*this, Res);
+    checkPropertyDeclWithOwnership(SemaRef, Res);
 
   llvm::SmallPtrSet<ObjCProtocolDecl *, 16> KnownProtos;
   if (ObjCInterfaceDecl *IFace = dyn_cast<ObjCInterfaceDecl>(ClassDecl)) {
@@ -243,12 +243,12 @@ Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
     if (FoundInSuper) {
       // Also compare the property against a property in our protocols.
       for (auto *P : CurrentInterfaceDecl->protocols()) {
-        CheckPropertyAgainstProtocol(*this, Res, P, KnownProtos);
+        CheckPropertyAgainstProtocol(SemaRef, Res, P, KnownProtos);
       }
     } else {
       // Slower path: look in all protocols we referenced.
       for (auto *P : IFace->all_referenced_protocols()) {
-        CheckPropertyAgainstProtocol(*this, Res, P, KnownProtos);
+        CheckPropertyAgainstProtocol(SemaRef, Res, P, KnownProtos);
       }
     }
   } else if (ObjCCategoryDecl *Cat = dyn_cast<ObjCCategoryDecl>(ClassDecl)) {
@@ -257,14 +257,14 @@ Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
     // when property in class extension is constructed.
     if (!Cat->IsClassExtension())
       for (auto *P : Cat->protocols())
-        CheckPropertyAgainstProtocol(*this, Res, P, KnownProtos);
+        CheckPropertyAgainstProtocol(SemaRef, Res, P, KnownProtos);
   } else {
     ObjCProtocolDecl *Proto = cast<ObjCProtocolDecl>(ClassDecl);
     for (auto *P : Proto->protocols())
-      CheckPropertyAgainstProtocol(*this, Res, P, KnownProtos);
+      CheckPropertyAgainstProtocol(SemaRef, Res, P, KnownProtos);
   }
 
-  ActOnDocumentableDecl(Res);
+  SemaRef.ActOnDocumentableDecl(Res);
   return Res;
 }
 
@@ -401,24 +401,15 @@ static void checkAtomicPropertyMismatch(Sema &S,
   S.Diag(OldProperty->getLocation(), diag::note_property_declare);
 }
 
-ObjCPropertyDecl *
-Sema::HandlePropertyInClassExtension(Scope *S,
-                                     SourceLocation AtLoc,
-                                     SourceLocation LParenLoc,
-                                     FieldDeclarator &FD,
-                                     Selector GetterSel,
-                                     SourceLocation GetterNameLoc,
-                                     Selector SetterSel,
-                                     SourceLocation SetterNameLoc,
-                                     const bool isReadWrite,
-                                     unsigned &Attributes,
-                                     const unsigned AttributesAsWritten,
-                                     QualType T,
-                                     TypeSourceInfo *TSI,
-                                     tok::ObjCKeywordKind MethodImplKind) {
-  ObjCCategoryDecl *CDecl = cast<ObjCCategoryDecl>(CurContext);
+ObjCPropertyDecl *SemaObjC::HandlePropertyInClassExtension(
+    Scope *S, SourceLocation AtLoc, SourceLocation LParenLoc,
+    FieldDeclarator &FD, Selector GetterSel, SourceLocation GetterNameLoc,
+    Selector SetterSel, SourceLocation SetterNameLoc, const bool isReadWrite,
+    unsigned &Attributes, const unsigned AttributesAsWritten, QualType T,
+    TypeSourceInfo *TSI, tok::ObjCKeywordKind MethodImplKind) {
+  ObjCCategoryDecl *CDecl = cast<ObjCCategoryDecl>(SemaRef.CurContext);
   // Diagnose if this property is already in continuation class.
-  DeclContext *DC = CurContext;
+  DeclContext *DC = SemaRef.CurContext;
   const IdentifierInfo *PropertyId = FD.D.getIdentifier();
   ObjCInterfaceDecl *CCPrimary = CDecl->getClassInterface();
 
@@ -515,7 +506,7 @@ Sema::HandlePropertyInClassExtension(Scope *S,
                                                isReadWrite,
                                                Attributes, AttributesAsWritten,
                                                T, TSI, MethodImplKind, DC);
-
+  ASTContext &Context = getASTContext();
   // If there was no declaration of a property with the same name in
   // the primary class, we're done.
   if (!PIDecl) {
@@ -536,9 +527,10 @@ Sema::HandlePropertyInClassExtension(Scope *S,
     QualType ClassExtPropertyT = Context.getCanonicalType(PDecl->getType());
     if (!isa<ObjCObjectPointerType>(PrimaryClassPropertyT) ||
         !isa<ObjCObjectPointerType>(ClassExtPropertyT) ||
-        (!isObjCPointerConversion(ClassExtPropertyT, PrimaryClassPropertyT,
-                                  ConvertedType, IncompatibleObjC))
-        || IncompatibleObjC) {
+        (!SemaRef.isObjCPointerConversion(ClassExtPropertyT,
+                                          PrimaryClassPropertyT, ConvertedType,
+                                          IncompatibleObjC)) ||
+        IncompatibleObjC) {
       Diag(AtLoc,
           diag::err_type_mismatch_continuation_class) << PDecl->getType();
       Diag(PIDecl->getLocation(), diag::note_property_declare);
@@ -548,29 +540,22 @@ Sema::HandlePropertyInClassExtension(Scope *S,
 
   // Check that atomicity of property in class extension matches the previous
   // declaration.
-  checkAtomicPropertyMismatch(*this, PIDecl, PDecl, true);
+  checkAtomicPropertyMismatch(SemaRef, PIDecl, PDecl, true);
 
   // Make sure getter/setter are appropriately synthesized.
   ProcessPropertyDecl(PDecl);
   return PDecl;
 }
 
-ObjCPropertyDecl *Sema::CreatePropertyDecl(Scope *S,
-                                           ObjCContainerDecl *CDecl,
-                                           SourceLocation AtLoc,
-                                           SourceLocation LParenLoc,
-                                           FieldDeclarator &FD,
-                                           Selector GetterSel,
-                                           SourceLocation GetterNameLoc,
-                                           Selector SetterSel,
-                                           SourceLocation SetterNameLoc,
-                                           const bool isReadWrite,
-                                           const unsigned Attributes,
-                                           const unsigned AttributesAsWritten,
-                                           QualType T,
-                                           TypeSourceInfo *TInfo,
-                                           tok::ObjCKeywordKind MethodImplKind,
-                                           DeclContext *lexicalDC){
+ObjCPropertyDecl *SemaObjC::CreatePropertyDecl(
+    Scope *S, ObjCContainerDecl *CDecl, SourceLocation AtLoc,
+    SourceLocation LParenLoc, FieldDeclarator &FD, Selector GetterSel,
+    SourceLocation GetterNameLoc, Selector SetterSel,
+    SourceLocation SetterNameLoc, const bool isReadWrite,
+    const unsigned Attributes, const unsigned AttributesAsWritten, QualType T,
+    TypeSourceInfo *TInfo, tok::ObjCKeywordKind MethodImplKind,
+    DeclContext *lexicalDC) {
+  ASTContext &Context = getASTContext();
   const IdentifierInfo *PropertyId = FD.D.getIdentifier();
 
   // Property defaults to 'assign' if it is readwrite, unless this is ARC
@@ -603,7 +588,7 @@ ObjCPropertyDecl *Sema::CreatePropertyDecl(Scope *S,
 
   if (T->isObjCObjectType()) {
     SourceLocation StarLoc = TInfo->getTypeLoc().getEndLoc();
-    StarLoc = getLocForEndOfToken(StarLoc);
+    StarLoc = SemaRef.getLocForEndOfToken(StarLoc);
     Diag(FD.D.getIdentifierLoc(), diag::err_statically_allocated_object)
       << FixItHint::CreateInsertion(StarLoc, "*");
     T = Context.getObjCObjectPointerType(T);
@@ -645,7 +630,7 @@ ObjCPropertyDecl *Sema::CreatePropertyDecl(Scope *S,
   PDecl->setPropertyAttributesAsWritten(
                           makePropertyAttributesAsWritten(AttributesAsWritten));
 
-  ProcessDeclAttributes(S, PDecl, FD.D);
+  SemaRef.ProcessDeclAttributes(S, PDecl, FD.D);
 
   if (Attributes & ObjCPropertyAttribute::kind_readonly)
     PDecl->setPropertyAttributes(ObjCPropertyAttribute::kind_readonly);
@@ -1074,16 +1059,13 @@ RedeclarePropertyAccessor(ASTContext &Context, ObjCImplementationDecl *Impl,
 /// builds the AST node for a property implementation declaration; declared
 /// as \@synthesize or \@dynamic.
 ///
-Decl *Sema::ActOnPropertyImplDecl(Scope *S,
-                                  SourceLocation AtLoc,
-                                  SourceLocation PropertyLoc,
-                                  bool Synthesize,
-                                  IdentifierInfo *PropertyId,
-                                  IdentifierInfo *PropertyIvar,
-                                  SourceLocation PropertyIvarLoc,
-                                  ObjCPropertyQueryKind QueryKind) {
+Decl *SemaObjC::ActOnPropertyImplDecl(
+    Scope *S, SourceLocation AtLoc, SourceLocation PropertyLoc, bool Synthesize,
+    IdentifierInfo *PropertyId, IdentifierInfo *PropertyIvar,
+    SourceLocation PropertyIvarLoc, ObjCPropertyQueryKind QueryKind) {
+  ASTContext &Context = getASTContext();
   ObjCContainerDecl *ClassImpDecl =
-    dyn_cast<ObjCContainerDecl>(CurContext);
+      dyn_cast<ObjCContainerDecl>(SemaRef.CurContext);
   // Make sure we have a context for the property implementation declaration.
   if (!ClassImpDecl) {
     Diag(AtLoc, diag::err_missing_property_context);
@@ -1167,7 +1149,7 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
       }
     }
     if (Synthesize && isa<ObjCProtocolDecl>(property->getDeclContext()))
-      property = SelectPropertyForSynthesisFromProtocols(*this, AtLoc, IDecl,
+      property = SelectPropertyForSynthesisFromProtocols(SemaRef, AtLoc, IDecl,
                                                          property);
 
   } else if ((CatImplClass = dyn_cast<ObjCCategoryImplDecl>(ClassImpDecl))) {
@@ -1212,9 +1194,9 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
     QualType PropType = property->getType();
     QualType PropertyIvarType = PropType.getNonReferenceType();
 
-    if (RequireCompleteType(PropertyDiagLoc, PropertyIvarType,
-                            diag::err_incomplete_synthesized_property,
-                            property->getDeclName())) {
+    if (SemaRef.RequireCompleteType(PropertyDiagLoc, PropertyIvarType,
+                                    diag::err_incomplete_synthesized_property,
+                                    property->getDeclName())) {
       Diag(property->getLocation(), diag::note_property_declare);
       CompleteTypeErr = true;
     }
@@ -1320,10 +1302,9 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
                                   PropertyIvarType, /*TInfo=*/nullptr,
                                   ObjCIvarDecl::Private,
                                   (Expr *)nullptr, true);
-      if (RequireNonAbstractType(PropertyIvarLoc,
-                                 PropertyIvarType,
-                                 diag::err_abstract_type_in_decl,
-                                 AbstractSynthesizedIvarType)) {
+      if (SemaRef.RequireNonAbstractType(PropertyIvarLoc, PropertyIvarType,
+                                         diag::err_abstract_type_in_decl,
+                                         Sema::AbstractSynthesizedIvarType)) {
         Diag(property->getLocation(), diag::note_property_declare);
         // An abstract type is as bad as an incomplete type.
         CompleteTypeErr = true;
@@ -1367,9 +1348,9 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
             PropertyIvarType->castAs<ObjCObjectPointerType>(),
             IvarType->castAs<ObjCObjectPointerType>());
       else {
-        compat = (CheckAssignmentConstraints(PropertyIvarLoc, PropertyIvarType,
-                                             IvarType)
-                    == Compatible);
+        compat = (SemaRef.CheckAssignmentConstraints(
+                      PropertyIvarLoc, PropertyIvarType, IvarType) ==
+                  Sema::Compatible);
       }
       if (!compat) {
         Diag(PropertyDiagLoc, diag::err_property_ivar_type)
@@ -1413,19 +1394,17 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
     }
     if (getLangOpts().ObjCAutoRefCount || isARCWeak ||
         Ivar->getType().getObjCLifetime())
-      checkARCPropertyImpl(*this, PropertyLoc, property, Ivar);
+      checkARCPropertyImpl(SemaRef, PropertyLoc, property, Ivar);
   } else if (PropertyIvar)
     // @dynamic
     Diag(PropertyDiagLoc, diag::err_dynamic_property_ivar_decl);
 
   assert (property && "ActOnPropertyImplDecl - property declaration missing");
-  ObjCPropertyImplDecl *PIDecl =
-  ObjCPropertyImplDecl::Create(Context, CurContext, AtLoc, PropertyLoc,
-                               property,
-                               (Synthesize ?
-                                ObjCPropertyImplDecl::Synthesize
-                                : ObjCPropertyImplDecl::Dynamic),
-                               Ivar, PropertyIvarLoc);
+  ObjCPropertyImplDecl *PIDecl = ObjCPropertyImplDecl::Create(
+      Context, SemaRef.CurContext, AtLoc, PropertyLoc, property,
+      (Synthesize ? ObjCPropertyImplDecl::Synthesize
+                  : ObjCPropertyImplDecl::Dynamic),
+      Ivar, PropertyIvarLoc);
 
   if (CompleteTypeErr || !compat)
     PIDecl->setInvalidDecl();
@@ -1449,12 +1428,12 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
       // For Objective-C++, need to synthesize the AST for the IVAR object to be
       // returned by the getter as it must conform to C++'s copy-return rules.
       // FIXME. Eventually we want to do this for Objective-C as well.
-      SynthesizedFunctionScope Scope(*this, getterMethod);
+      Sema::SynthesizedFunctionScope Scope(SemaRef, getterMethod);
       ImplicitParamDecl *SelfDecl = getterMethod->getSelfDecl();
       DeclRefExpr *SelfExpr = new (Context)
           DeclRefExpr(Context, SelfDecl, false, SelfDecl->getType(), VK_LValue,
                       PropertyDiagLoc);
-      MarkDeclRefReferenced(SelfExpr);
+      SemaRef.MarkDeclRefReferenced(SelfExpr);
       Expr *LoadSelfExpr = ImplicitCastExpr::Create(
           Context, SelfDecl->getType(), CK_LValueToRValue, SelfExpr, nullptr,
           VK_PRValue, FPOptionsOverride());
@@ -1464,14 +1443,14 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
                                       PropertyDiagLoc,
                                       Ivar->getLocation(),
                                       LoadSelfExpr, true, true);
-      ExprResult Res = PerformCopyInitialization(
+      ExprResult Res = SemaRef.PerformCopyInitialization(
           InitializedEntity::InitializeResult(PropertyDiagLoc,
                                               getterMethod->getReturnType()),
           PropertyDiagLoc, IvarRefExpr);
       if (!Res.isInvalid()) {
         Expr *ResExpr = Res.getAs<Expr>();
         if (ResExpr)
-          ResExpr = MaybeCreateExprWithCleanups(ResExpr);
+          ResExpr = SemaRef.MaybeCreateExprWithCleanups(ResExpr);
         PIDecl->setGetterCXXConstructor(ResExpr);
       }
     }
@@ -1511,12 +1490,12 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
     if (getLangOpts().CPlusPlus && Synthesize && !CompleteTypeErr &&
         Ivar->getType()->isRecordType()) {
       // FIXME. Eventually we want to do this for Objective-C as well.
-      SynthesizedFunctionScope Scope(*this, setterMethod);
+      Sema::SynthesizedFunctionScope Scope(SemaRef, setterMethod);
       ImplicitParamDecl *SelfDecl = setterMethod->getSelfDecl();
       DeclRefExpr *SelfExpr = new (Context)
           DeclRefExpr(Context, SelfDecl, false, SelfDecl->getType(), VK_LValue,
                       PropertyDiagLoc);
-      MarkDeclRefReferenced(SelfExpr);
+      SemaRef.MarkDeclRefReferenced(SelfExpr);
       Expr *LoadSelfExpr = ImplicitCastExpr::Create(
           Context, SelfDecl->getType(), CK_LValueToRValue, SelfExpr, nullptr,
           VK_PRValue, FPOptionsOverride());
@@ -1531,9 +1510,9 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
       QualType T = Param->getType().getNonReferenceType();
       DeclRefExpr *rhs = new (Context)
           DeclRefExpr(Context, Param, false, T, VK_LValue, PropertyDiagLoc);
-      MarkDeclRefReferenced(rhs);
-      ExprResult Res = BuildBinOp(S, PropertyDiagLoc,
-                                  BO_Assign, lhs, rhs);
+      SemaRef.MarkDeclRefReferenced(rhs);
+      ExprResult Res =
+          SemaRef.BuildBinOp(S, PropertyDiagLoc, BO_Assign, lhs, rhs);
       if (property->getPropertyAttributes() &
           ObjCPropertyAttribute::kind_atomic) {
         Expr *callExpr = Res.getAs<Expr>();
@@ -1630,11 +1609,11 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
 /// DiagnosePropertyMismatch - Compares two properties for their
 /// attributes and types and warns on a variety of inconsistencies.
 ///
-void
-Sema::DiagnosePropertyMismatch(ObjCPropertyDecl *Property,
-                               ObjCPropertyDecl *SuperProperty,
-                               const IdentifierInfo *inheritedName,
-                               bool OverridingProtocolProperty) {
+void SemaObjC::DiagnosePropertyMismatch(ObjCPropertyDecl *Property,
+                                        ObjCPropertyDecl *SuperProperty,
+                                        const IdentifierInfo *inheritedName,
+                                        bool OverridingProtocolProperty) {
+  ASTContext &Context = getASTContext();
   ObjCPropertyAttribute::Kind CAttr = Property->getPropertyAttributes();
   ObjCPropertyAttribute::Kind SAttr = SuperProperty->getPropertyAttributes();
 
@@ -1669,7 +1648,7 @@ Sema::DiagnosePropertyMismatch(ObjCPropertyDecl *Property,
   // Check for nonatomic; note that nonatomic is effectively
   // meaningless for readonly properties, so don't diagnose if the
   // atomic property is 'readonly'.
-  checkAtomicPropertyMismatch(*this, SuperProperty, Property, false);
+  checkAtomicPropertyMismatch(SemaRef, SuperProperty, Property, false);
   // Readonly properties from protocols can be implemented as "readwrite"
   // with a custom setter name.
   if (Property->getSetterName() != SuperProperty->getSetterName() &&
@@ -1695,19 +1674,20 @@ Sema::DiagnosePropertyMismatch(ObjCPropertyDecl *Property,
     // FIXME. For future support of covariant property types, revisit this.
     bool IncompatibleObjC = false;
     QualType ConvertedType;
-    if (!isObjCPointerConversion(RHSType, LHSType,
-                                 ConvertedType, IncompatibleObjC) ||
+    if (!SemaRef.isObjCPointerConversion(RHSType, LHSType, ConvertedType,
+                                         IncompatibleObjC) ||
         IncompatibleObjC) {
-        Diag(Property->getLocation(), diag::warn_property_types_are_incompatible)
-        << Property->getType() << SuperProperty->getType() << inheritedName;
+      Diag(Property->getLocation(), diag::warn_property_types_are_incompatible)
+          << Property->getType() << SuperProperty->getType() << inheritedName;
       Diag(SuperProperty->getLocation(), diag::note_property_declare);
     }
   }
 }
 
-bool Sema::DiagnosePropertyAccessorMismatch(ObjCPropertyDecl *property,
-                                            ObjCMethodDecl *GetterMethod,
-                                            SourceLocation Loc) {
+bool SemaObjC::DiagnosePropertyAccessorMismatch(ObjCPropertyDecl *property,
+                                                ObjCMethodDecl *GetterMethod,
+                                                SourceLocation Loc) {
+  ASTContext &Context = getASTContext();
   if (!GetterMethod)
     return false;
   QualType GetterType = GetterMethod->getReturnType().getNonReferenceType();
@@ -1721,13 +1701,13 @@ bool Sema::DiagnosePropertyAccessorMismatch(ObjCPropertyDecl *property,
              PropertyRValueType->getAs<ObjCObjectPointerType>()) &&
         (getterObjCPtr = GetterType->getAs<ObjCObjectPointerType>()))
       compat = Context.canAssignObjCInterfaces(getterObjCPtr, propertyObjCPtr);
-    else if (CheckAssignmentConstraints(Loc, GetterType, PropertyRValueType)
-              != Compatible) {
-          Diag(Loc, diag::err_property_accessor_type)
-            << property->getDeclName() << PropertyRValueType
-            << GetterMethod->getSelector() << GetterType;
-          Diag(GetterMethod->getLocation(), diag::note_declared_at);
-          return true;
+    else if (SemaRef.CheckAssignmentConstraints(
+                 Loc, GetterType, PropertyRValueType) != Sema::Compatible) {
+      Diag(Loc, diag::err_property_accessor_type)
+          << property->getDeclName() << PropertyRValueType
+          << GetterMethod->getSelector() << GetterType;
+      Diag(GetterMethod->getLocation(), diag::note_declared_at);
+      return true;
     } else {
       compat = true;
       QualType lhsType = Context.getCanonicalType(PropertyRValueType);
@@ -1831,9 +1811,9 @@ static void CollectSuperClassPropertyImplementations(ObjCInterfaceDecl *CDecl,
 /// IvarBacksCurrentMethodAccessor - This routine returns 'true' if 'IV' is
 /// an ivar synthesized for 'Method' and 'Method' is a property accessor
 /// declared in class 'IFace'.
-bool
-Sema::IvarBacksCurrentMethodAccessor(ObjCInterfaceDecl *IFace,
-                                     ObjCMethodDecl *Method, ObjCIvarDecl *IV) {
+bool SemaObjC::IvarBacksCurrentMethodAccessor(ObjCInterfaceDecl *IFace,
+                                              ObjCMethodDecl *Method,
+                                              ObjCIvarDecl *IV) {
   if (!IV->getSynthesize())
     return false;
   ObjCMethodDecl *IMD = IFace->lookupMethod(Method->getSelector(),
@@ -1883,9 +1863,10 @@ static bool SuperClassImplementsProperty(ObjCInterfaceDecl *IDecl,
 
 /// Default synthesizes all properties which must be synthesized
 /// in class's \@implementation.
-void Sema::DefaultSynthesizeProperties(Scope *S, ObjCImplDecl *IMPDecl,
-                                       ObjCInterfaceDecl *IDecl,
-                                       SourceLocation AtEnd) {
+void SemaObjC::DefaultSynthesizeProperties(Scope *S, ObjCImplDecl *IMPDecl,
+                                           ObjCInterfaceDecl *IDecl,
+                                           SourceLocation AtEnd) {
+  ASTContext &Context = getASTContext();
   ObjCInterfaceDecl::PropertyMap PropMap;
   IDecl->collectPropertiesToImplement(PropMap);
   if (PropMap.empty())
@@ -1977,9 +1958,10 @@ void Sema::DefaultSynthesizeProperties(Scope *S, ObjCImplDecl *IMPDecl,
   }
 }
 
-void Sema::DefaultSynthesizeProperties(Scope *S, Decl *D,
-                                       SourceLocation AtEnd) {
-  if (!LangOpts.ObjCDefaultSynthProperties || LangOpts.ObjCRuntime.isFragile())
+void SemaObjC::DefaultSynthesizeProperties(Scope *S, Decl *D,
+                                           SourceLocation AtEnd) {
+  if (!getLangOpts().ObjCDefaultSynthProperties ||
+      getLangOpts().ObjCRuntime.isFragile())
     return;
   ObjCImplementationDecl *IC=dyn_cast_or_null<ObjCImplementationDecl>(D);
   if (!IC)
@@ -2026,9 +2008,9 @@ static void DiagnoseUnimplementedAccessor(
   }
 }
 
-void Sema::DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl* IMPDecl,
-                                           ObjCContainerDecl *CDecl,
-                                           bool SynthesizeProperties) {
+void SemaObjC::DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl *IMPDecl,
+                                               ObjCContainerDecl *CDecl,
+                                               bool SynthesizeProperties) {
   ObjCContainerDecl::PropertyMap PropMap;
   ObjCInterfaceDecl *IDecl = dyn_cast<ObjCInterfaceDecl>(CDecl);
 
@@ -2124,16 +2106,17 @@ void Sema::DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl* IMPDecl,
       continue;
 
     // Diagnose unimplemented getters and setters.
-    DiagnoseUnimplementedAccessor(*this,
-          PrimaryClass, Prop->getGetterName(), IMPDecl, CDecl, C, Prop, InsMap);
+    DiagnoseUnimplementedAccessor(SemaRef, PrimaryClass, Prop->getGetterName(),
+                                  IMPDecl, CDecl, C, Prop, InsMap);
     if (!Prop->isReadOnly())
-      DiagnoseUnimplementedAccessor(*this,
-                                    PrimaryClass, Prop->getSetterName(),
-                                    IMPDecl, CDecl, C, Prop, InsMap);
+      DiagnoseUnimplementedAccessor(SemaRef, PrimaryClass,
+                                    Prop->getSetterName(), IMPDecl, CDecl, C,
+                                    Prop, InsMap);
   }
 }
 
-void Sema::diagnoseNullResettableSynthesizedSetters(const ObjCImplDecl *impDecl) {
+void SemaObjC::diagnoseNullResettableSynthesizedSetters(
+    const ObjCImplDecl *impDecl) {
   for (const auto *propertyImpl : impDecl->property_impls()) {
     const auto *property = propertyImpl->getPropertyDecl();
     // Warn about null_resettable properties with synthesized setters,
@@ -2158,9 +2141,8 @@ void Sema::diagnoseNullResettableSynthesizedSetters(const ObjCImplDecl *impDecl)
   }
 }
 
-void
-Sema::AtomicPropertySetterGetterRules (ObjCImplDecl* IMPDecl,
-                                       ObjCInterfaceDecl* IDecl) {
+void SemaObjC::AtomicPropertySetterGetterRules(ObjCImplDecl *IMPDecl,
+                                               ObjCInterfaceDecl *IDecl) {
   // Rules apply in non-GC mode only
   if (getLangOpts().getGC() != LangOptions::NonGC)
     return;
@@ -2232,7 +2214,7 @@ Sema::AtomicPropertySetterGetterRules (ObjCImplDecl* IMPDecl,
             !(AttributesAsWritten & ObjCPropertyAttribute::kind_atomic)) {
           // @property () ... case.
           SourceLocation AfterLParen =
-            getLocForEndOfToken(Property->getLParenLoc());
+              SemaRef.getLocForEndOfToken(Property->getLParenLoc());
           StringRef NonatomicStr = AttributesAsWritten? "nonatomic, "
                                                       : "nonatomic";
           Diag(Property->getLocation(),
@@ -2253,7 +2235,8 @@ Sema::AtomicPropertySetterGetterRules (ObjCImplDecl* IMPDecl,
   }
 }
 
-void Sema::DiagnoseOwningPropertyGetterSynthesis(const ObjCImplementationDecl *D) {
+void SemaObjC::DiagnoseOwningPropertyGetterSynthesis(
+    const ObjCImplementationDecl *D) {
   if (getLangOpts().getGC() == LangOptions::GCOnly)
     return;
 
@@ -2288,7 +2271,7 @@ void Sema::DiagnoseOwningPropertyGetterSynthesis(const ObjCImplementationDecl *D
           fixItLoc = getterRedecl->getEndLoc();
         }
 
-        Preprocessor &PP = getPreprocessor();
+        Preprocessor &PP = SemaRef.getPreprocessor();
         TokenValue tokens[] = {
           tok::kw___attribute, tok::l_paren, tok::l_paren,
           PP.getIdentifierInfo("objc_method_family"), tok::l_paren,
@@ -2312,9 +2295,8 @@ void Sema::DiagnoseOwningPropertyGetterSynthesis(const ObjCImplementationDecl *D
   }
 }
 
-void Sema::DiagnoseMissingDesignatedInitOverrides(
-                                            const ObjCImplementationDecl *ImplD,
-                                            const ObjCInterfaceDecl *IFD) {
+void SemaObjC::DiagnoseMissingDesignatedInitOverrides(
+    const ObjCImplementationDecl *ImplD, const ObjCInterfaceDecl *IFD) {
   assert(IFD->hasDesignatedInitializers());
   const ObjCInterfaceDecl *SuperD = IFD->getSuperClass();
   if (!SuperD)
@@ -2371,7 +2353,8 @@ static void AddPropertyAttrs(Sema &S, ObjCMethodDecl *PropertyMethod,
 /// have the property type and issue diagnostics if they don't.
 /// Also synthesize a getter/setter method if none exist (and update the
 /// appropriate lookup tables.
-void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property) {
+void SemaObjC::ProcessPropertyDecl(ObjCPropertyDecl *property) {
+  ASTContext &Context = getASTContext();
   ObjCMethodDecl *GetterMethod, *SetterMethod;
   ObjCContainerDecl *CD = cast<ObjCContainerDecl>(property->getDeclContext());
   if (CD->isInvalidDecl())
@@ -2492,7 +2475,7 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property) {
             : ObjCImplementationControl::Required);
     CD->addDecl(GetterMethod);
 
-    AddPropertyAttrs(*this, GetterMethod, property);
+    AddPropertyAttrs(SemaRef, GetterMethod, property);
 
     if (property->isDirectProperty())
       GetterMethod->addAttr(ObjCDirectAttr::CreateImplicit(Context, Loc));
@@ -2509,7 +2492,7 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property) {
       GetterMethod->addAttr(SectionAttr::CreateImplicit(
           Context, SA->getName(), Loc, SectionAttr::GNU_section));
 
-    ProcessAPINotes(GetterMethod);
+    SemaRef.ProcessAPINotes(GetterMethod);
 
     if (getLangOpts().ObjCAutoRefCount)
       CheckARCMethodDecl(GetterMethod);
@@ -2571,7 +2554,7 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property) {
                                                   nullptr);
       SetterMethod->setMethodParams(Context, Argument, std::nullopt);
 
-      AddPropertyAttrs(*this, SetterMethod, property);
+      AddPropertyAttrs(SemaRef, SetterMethod, property);
 
       if (property->isDirectProperty())
         SetterMethod->addAttr(ObjCDirectAttr::CreateImplicit(Context, Loc));
@@ -2581,7 +2564,7 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property) {
         SetterMethod->addAttr(SectionAttr::CreateImplicit(
             Context, SA->getName(), Loc, SectionAttr::GNU_section));
 
-      ProcessAPINotes(SetterMethod);
+      SemaRef.ProcessAPINotes(SetterMethod);
 
       // It's possible for the user to have set a very odd custom
       // setter selector that causes it to have a method family.
@@ -2628,15 +2611,14 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property) {
       CurrentClass = Impl->getClassInterface();
   }
   if (GetterMethod)
-    CheckObjCMethodOverrides(GetterMethod, CurrentClass, Sema::RTC_Unknown);
+    CheckObjCMethodOverrides(GetterMethod, CurrentClass, SemaObjC::RTC_Unknown);
   if (SetterMethod)
-    CheckObjCMethodOverrides(SetterMethod, CurrentClass, Sema::RTC_Unknown);
+    CheckObjCMethodOverrides(SetterMethod, CurrentClass, SemaObjC::RTC_Unknown);
 }
 
-void Sema::CheckObjCPropertyAttributes(Decl *PDecl,
-                                       SourceLocation Loc,
-                                       unsigned &Attributes,
-                                       bool propertyInPrimaryClass) {
+void SemaObjC::CheckObjCPropertyAttributes(Decl *PDecl, SourceLocation Loc,
+                                           unsigned &Attributes,
+                                           bool propertyInPrimaryClass) {
   // FIXME: Improve the reported location.
   if (!PDecl || PDecl->isInvalidDecl())
     return;
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index f173300b5c96..2eb25237a0de 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -33,6 +33,7 @@
 #include "clang/Sema/Overload.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateDeduction.h"
 #include "llvm/ADT/DenseSet.h"
@@ -1086,7 +1087,7 @@ namespace {
       assert(E->hasPlaceholderType(BuiltinType::ARCUnbridgedCast));
       Entry entry = { &E, E };
       Entries.push_back(entry);
-      E = S.stripARCUnbridgedCast(E);
+      E = S.ObjC().stripARCUnbridgedCast(E);
     }
 
     void restore() {
@@ -1778,8 +1779,8 @@ ExprResult Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     = getLangOpts().ObjCAutoRefCount &&
       (Action == AA_Passing || Action == AA_Sending);
   if (getLangOpts().ObjC)
-    CheckObjCBridgeRelatedConversions(From->getBeginLoc(), ToType,
-                                      From->getType(), From);
+    ObjC().CheckObjCBridgeRelatedConversions(From->getBeginLoc(), ToType,
+                                             From->getType(), From);
   ImplicitConversionSequence ICS = ::TryImplicitConversion(
       *this, From, ToType,
       /*SuppressUserConversions=*/false,
@@ -2272,7 +2273,7 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
   } else if (S.IsBlockPointerConversion(FromType, ToType, FromType)) {
     SCS.Second = ICK_Block_Pointer_Conversion;
   } else if (AllowObjCWritebackConversion &&
-             S.isObjCWritebackConversion(FromType, ToType, FromType)) {
+             S.ObjC().isObjCWritebackConversion(FromType, ToType, FromType)) {
     SCS.Second = ICK_Writeback_Conversion;
   } else if (S.IsPointerConversion(From, FromType, ToType, InOverloadResolution,
                                    FromType, IncompatibleObjC)) {
@@ -3060,73 +3061,6 @@ bool Sema::isObjCPointerConversion(QualType FromType, QualType ToType,
   return false;
 }
 
-/// Determine whether this is an Objective-C writeback conversion,
-/// used for parameter passing when performing automatic reference counting.
-///
-/// \param FromType The type we're converting form.
-///
-/// \param ToType The type we're converting to.
-///
-/// \param ConvertedType The type that will be produced after applying
-/// this conversion.
-bool Sema::isObjCWritebackConversion(QualType FromType, QualType ToType,
-                                     QualType &ConvertedType) {
-  if (!getLangOpts().ObjCAutoRefCount ||
-      Context.hasSameUnqualifiedType(FromType, ToType))
-    return false;
-
-  // Parameter must be a pointer to __autoreleasing (with no other qualifiers).
-  QualType ToPointee;
-  if (const PointerType *ToPointer = ToType->getAs<PointerType>())
-    ToPointee = ToPointer->getPointeeType();
-  else
-    return false;
-
-  Qualifiers ToQuals = ToPointee.getQualifiers();
-  if (!ToPointee->isObjCLifetimeType() ||
-      ToQuals.getObjCLifetime() != Qualifiers::OCL_Autoreleasing ||
-      !ToQuals.withoutObjCLifetime().empty())
-    return false;
-
-  // Argument must be a pointer to __strong to __weak.
-  QualType FromPointee;
-  if (const PointerType *FromPointer = FromType->getAs<PointerType>())
-    FromPointee = FromPointer->getPointeeType();
-  else
-    return false;
-
-  Qualifiers FromQuals = FromPointee.getQualifiers();
-  if (!FromPointee->isObjCLifetimeType() ||
-      (FromQuals.getObjCLifetime() != Qualifiers::OCL_Strong &&
-       FromQuals.getObjCLifetime() != Qualifiers::OCL_Weak))
-    return false;
-
-  // Make sure that we have compatible qualifiers.
-  FromQuals.setObjCLifetime(Qualifiers::OCL_Autoreleasing);
-  if (!ToQuals.compatiblyIncludes(FromQuals))
-    return false;
-
-  // Remove qualifiers from the pointee type we're converting from; they
-  // aren't used in the compatibility check belong, and we'll be adding back
-  // qualifiers (with __autoreleasing) if the compatibility check succeeds.
-  FromPointee = FromPointee.getUnqualifiedType();
-
-  // The unqualified form of the pointee types must be compatible.
-  ToPointee = ToPointee.getUnqualifiedType();
-  bool IncompatibleObjC;
-  if (Context.typesAreCompatible(FromPointee, ToPointee))
-    FromPointee = ToPointee;
-  else if (!isObjCPointerConversion(FromPointee, ToPointee, FromPointee,
-                                    IncompatibleObjC))
-    return false;
-
-  /// Construct the type we're converting to, which is a pointer to
-  /// __autoreleasing pointee.
-  FromPointee = Context.getQualifiedType(FromPointee, FromQuals);
-  ConvertedType = Context.getPointerType(FromPointee);
-  return true;
-}
-
 bool Sema::IsBlockPointerConversion(QualType FromType, QualType ToType,
                                     QualType& ConvertedType) {
   QualType ToPointeeType;
@@ -7241,7 +7175,7 @@ Sema::SelectBestMethod(Selector Sel, MultiExprArg Args, bool IsInstance,
       // a consumed argument.
       if (argExpr->hasPlaceholderType(BuiltinType::ARCUnbridgedCast) &&
           !param->hasAttr<CFConsumedAttr>())
-        argExpr = stripARCUnbridgedCast(argExpr);
+        argExpr = ObjC().stripARCUnbridgedCast(argExpr);
 
       // If the parameter is __unknown_anytype, move on to the next method.
       if (param->getType() == Context.UnknownAnyTy) {
diff --git a/clang/lib/Sema/SemaPseudoObject.cpp b/clang/lib/Sema/SemaPseudoObject.cpp
index c6a0a182d358..14ed9590afc6 100644
--- a/clang/lib/Sema/SemaPseudoObject.cpp
+++ b/clang/lib/Sema/SemaPseudoObject.cpp
@@ -29,13 +29,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Sema/SemaInternal.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/SmallString.h"
 
 using namespace clang;
@@ -557,30 +558,31 @@ static ObjCMethodDecl *LookupMethodInReceiverType(Sema &S, Selector sel,
 
     // Special case for 'self' in class method implementations.
     if (PT->isObjCClassType() &&
-        S.isSelfExpr(const_cast<Expr*>(PRE->getBase()))) {
+        S.ObjC().isSelfExpr(const_cast<Expr *>(PRE->getBase()))) {
       // This cast is safe because isSelfExpr is only true within
       // methods.
       ObjCMethodDecl *method =
         cast<ObjCMethodDecl>(S.CurContext->getNonClosureAncestor());
-      return S.LookupMethodInObjectType(sel,
-                 S.Context.getObjCInterfaceType(method->getClassInterface()),
-                                        /*instance*/ false);
+      return S.ObjC().LookupMethodInObjectType(
+          sel, S.Context.getObjCInterfaceType(method->getClassInterface()),
+          /*instance*/ false);
     }
 
-    return S.LookupMethodInObjectType(sel, PT->getPointeeType(), true);
+    return S.ObjC().LookupMethodInObjectType(sel, PT->getPointeeType(), true);
   }
 
   if (PRE->isSuperReceiver()) {
     if (const ObjCObjectPointerType *PT =
         PRE->getSuperReceiverType()->getAs<ObjCObjectPointerType>())
-      return S.LookupMethodInObjectType(sel, PT->getPointeeType(), true);
+      return S.ObjC().LookupMethodInObjectType(sel, PT->getPointeeType(), true);
 
-    return S.LookupMethodInObjectType(sel, PRE->getSuperReceiverType(), false);
+    return S.ObjC().LookupMethodInObjectType(sel, PRE->getSuperReceiverType(),
+                                             false);
   }
 
   assert(PRE->isClassReceiver() && "Invalid expression");
   QualType IT = S.Context.getObjCInterfaceType(PRE->getClassReceiver());
-  return S.LookupMethodInObjectType(sel, IT, false);
+  return S.ObjC().LookupMethodInObjectType(sel, IT, false);
 }
 
 bool ObjCPropertyOpBuilder::isWeakProperty() const {
@@ -741,13 +743,13 @@ ExprResult ObjCPropertyOpBuilder::buildGet() {
   if ((Getter->isInstanceMethod() && !RefExpr->isClassReceiver()) ||
       RefExpr->isObjectReceiver()) {
     assert(InstanceReceiver || RefExpr->isSuperReceiver());
-    msg = S.BuildInstanceMessageImplicit(InstanceReceiver, receiverType,
-                                         GenericLoc, Getter->getSelector(),
-                                         Getter, std::nullopt);
+    msg = S.ObjC().BuildInstanceMessageImplicit(
+        InstanceReceiver, receiverType, GenericLoc, Getter->getSelector(),
+        Getter, std::nullopt);
   } else {
-    msg = S.BuildClassMessageImplicit(receiverType, RefExpr->isSuperReceiver(),
-                                      GenericLoc, Getter->getSelector(), Getter,
-                                      std::nullopt);
+    msg = S.ObjC().BuildClassMessageImplicit(
+        receiverType, RefExpr->isSuperReceiver(), GenericLoc,
+        Getter->getSelector(), Getter, std::nullopt);
   }
   return msg;
 }
@@ -801,14 +803,13 @@ ExprResult ObjCPropertyOpBuilder::buildSet(Expr *op, SourceLocation opcLoc,
     S.DiagnoseUseOfDecl(Setter, GenericLoc, nullptr, true);
   if ((Setter->isInstanceMethod() && !RefExpr->isClassReceiver()) ||
       RefExpr->isObjectReceiver()) {
-    msg = S.BuildInstanceMessageImplicit(InstanceReceiver, receiverType,
-                                         GenericLoc, SetterSelector, Setter,
-                                         MultiExprArg(args, 1));
+    msg = S.ObjC().BuildInstanceMessageImplicit(InstanceReceiver, receiverType,
+                                                GenericLoc, SetterSelector,
+                                                Setter, MultiExprArg(args, 1));
   } else {
-    msg = S.BuildClassMessageImplicit(receiverType, RefExpr->isSuperReceiver(),
-                                      GenericLoc,
-                                      SetterSelector, Setter,
-                                      MultiExprArg(args, 1));
+    msg = S.ObjC().BuildClassMessageImplicit(
+        receiverType, RefExpr->isSuperReceiver(), GenericLoc, SetterSelector,
+        Setter, MultiExprArg(args, 1));
   }
 
   if (!msg.isInvalid() && captureSetValueAsResult) {
@@ -836,8 +837,8 @@ ExprResult ObjCPropertyOpBuilder::buildRValueOperation(Expr *op) {
   if (result.isInvalid()) return ExprError();
 
   if (RefExpr->isExplicitProperty() && !Getter->hasRelatedResultType())
-    S.DiagnosePropertyAccessorMismatch(RefExpr->getExplicitProperty(),
-                                       Getter, RefExpr->getLocation());
+    S.ObjC().DiagnosePropertyAccessorMismatch(RefExpr->getExplicitProperty(),
+                                              Getter, RefExpr->getLocation());
 
   // As a special case, if the method returns 'id', try to get
   // a better type from the property.
@@ -925,7 +926,7 @@ ObjCPropertyOpBuilder::buildAssignmentOperation(Scope *Sc,
 
   // Various warnings about property assignments in ARC.
   if (S.getLangOpts().ObjCAutoRefCount && InstanceReceiver) {
-    S.checkRetainCycles(InstanceReceiver->getSourceExpr(), RHS);
+    S.ObjC().checkRetainCycles(InstanceReceiver->getSourceExpr(), RHS);
     S.checkUnsafeExprAssigns(opcLoc, LHS, RHS);
   }
 
@@ -1014,7 +1015,7 @@ ObjCSubscriptOpBuilder::buildAssignmentOperation(Scope *Sc,
 
   // Various warnings about objc Index'ed assignments in ARC.
   if (S.getLangOpts().ObjCAutoRefCount && InstanceBase) {
-    S.checkRetainCycles(InstanceBase->getSourceExpr(), RHS);
+    S.ObjC().checkRetainCycles(InstanceBase->getSourceExpr(), RHS);
     S.checkUnsafeExprAssigns(opcLoc, LHS, RHS);
   }
 
@@ -1045,80 +1046,6 @@ Expr *ObjCSubscriptOpBuilder::rebuildAndCaptureObject(Expr *syntacticBase) {
   return syntacticBase;
 }
 
-/// CheckSubscriptingKind - This routine decide what type
-/// of indexing represented by "FromE" is being done.
-Sema::ObjCSubscriptKind
-  Sema::CheckSubscriptingKind(Expr *FromE) {
-  // If the expression already has integral or enumeration type, we're golden.
-  QualType T = FromE->getType();
-  if (T->isIntegralOrEnumerationType())
-    return OS_Array;
-
-  // If we don't have a class type in C++, there's no way we can get an
-  // expression of integral or enumeration type.
-  const RecordType *RecordTy = T->getAs<RecordType>();
-  if (!RecordTy &&
-      (T->isObjCObjectPointerType() || T->isVoidPointerType()))
-    // All other scalar cases are assumed to be dictionary indexing which
-    // caller handles, with diagnostics if needed.
-    return OS_Dictionary;
-  if (!getLangOpts().CPlusPlus ||
-      !RecordTy || RecordTy->isIncompleteType()) {
-    // No indexing can be done. Issue diagnostics and quit.
-    const Expr *IndexExpr = FromE->IgnoreParenImpCasts();
-    if (isa<StringLiteral>(IndexExpr))
-      Diag(FromE->getExprLoc(), diag::err_objc_subscript_pointer)
-        << T << FixItHint::CreateInsertion(FromE->getExprLoc(), "@");
-    else
-      Diag(FromE->getExprLoc(), diag::err_objc_subscript_type_conversion)
-        << T;
-    return OS_Error;
-  }
-
-  // We must have a complete class type.
-  if (RequireCompleteType(FromE->getExprLoc(), T,
-                          diag::err_objc_index_incomplete_class_type, FromE))
-    return OS_Error;
-
-  // Look for a conversion to an integral, enumeration type, or
-  // objective-C pointer type.
-  int NoIntegrals=0, NoObjCIdPointers=0;
-  SmallVector<CXXConversionDecl *, 4> ConversionDecls;
-
-  for (NamedDecl *D : cast<CXXRecordDecl>(RecordTy->getDecl())
-                          ->getVisibleConversionFunctions()) {
-    if (CXXConversionDecl *Conversion =
-            dyn_cast<CXXConversionDecl>(D->getUnderlyingDecl())) {
-      QualType CT = Conversion->getConversionType().getNonReferenceType();
-      if (CT->isIntegralOrEnumerationType()) {
-        ++NoIntegrals;
-        ConversionDecls.push_back(Conversion);
-      }
-      else if (CT->isObjCIdType() ||CT->isBlockPointerType()) {
-        ++NoObjCIdPointers;
-        ConversionDecls.push_back(Conversion);
-      }
-    }
-  }
-  if (NoIntegrals ==1 && NoObjCIdPointers == 0)
-    return OS_Array;
-  if (NoIntegrals == 0 && NoObjCIdPointers == 1)
-    return OS_Dictionary;
-  if (NoIntegrals == 0 && NoObjCIdPointers == 0) {
-    // No conversion function was found. Issue diagnostic and return.
-    Diag(FromE->getExprLoc(), diag::err_objc_subscript_type_conversion)
-      << FromE->getType();
-    return OS_Error;
-  }
-  Diag(FromE->getExprLoc(), diag::err_objc_multiple_subscript_type_conversion)
-      << FromE->getType();
-  for (unsigned int i = 0; i < ConversionDecls.size(); i++)
-    Diag(ConversionDecls[i]->getLocation(),
-         diag::note_conv_function_declared_at);
-
-  return OS_Error;
-}
-
 /// CheckKeyForObjCARCConversion - This routine suggests bridge casting of CF
 /// objects used as dictionary subscript key objects.
 static void CheckKeyForObjCARCConversion(Sema &S, QualType ContainerT,
@@ -1130,13 +1057,13 @@ static void CheckKeyForObjCARCConversion(Sema &S, QualType ContainerT,
   const IdentifierInfo *KeyIdents[] = {
       &S.Context.Idents.get("objectForKeyedSubscript")};
   Selector GetterSelector = S.Context.Selectors.getSelector(1, KeyIdents);
-  ObjCMethodDecl *Getter = S.LookupMethodInObjectType(GetterSelector, ContainerT,
-                                                      true /*instance*/);
+  ObjCMethodDecl *Getter = S.ObjC().LookupMethodInObjectType(
+      GetterSelector, ContainerT, true /*instance*/);
   if (!Getter)
     return;
   QualType T = Getter->parameters()[0]->getType();
-  S.CheckObjCConversion(Key->getSourceRange(), T, Key,
-                        CheckedConversionKind::Implicit);
+  S.ObjC().CheckObjCConversion(Key->getSourceRange(), T, Key,
+                               CheckedConversionKind::Implicit);
 }
 
 bool ObjCSubscriptOpBuilder::findAtIndexGetter() {
@@ -1151,15 +1078,15 @@ bool ObjCSubscriptOpBuilder::findAtIndexGetter() {
       BaseT->getAs<ObjCObjectPointerType>()) {
     ResultType = PTy->getPointeeType();
   }
-  Sema::ObjCSubscriptKind Res =
-    S.CheckSubscriptingKind(RefExpr->getKeyExpr());
-  if (Res == Sema::OS_Error) {
+  SemaObjC::ObjCSubscriptKind Res =
+      S.ObjC().CheckSubscriptingKind(RefExpr->getKeyExpr());
+  if (Res == SemaObjC::OS_Error) {
     if (S.getLangOpts().ObjCAutoRefCount)
       CheckKeyForObjCARCConversion(S, ResultType,
                                    RefExpr->getKeyExpr());
     return false;
   }
-  bool arrayRef = (Res == Sema::OS_Array);
+  bool arrayRef = (Res == SemaObjC::OS_Array);
 
   if (ResultType.isNull()) {
     S.Diag(BaseExpr->getExprLoc(), diag::err_objc_subscript_base_type)
@@ -1181,8 +1108,8 @@ bool ObjCSubscriptOpBuilder::findAtIndexGetter() {
     AtIndexGetterSelector = S.Context.Selectors.getSelector(1, KeyIdents);
   }
 
-  AtIndexGetter = S.LookupMethodInObjectType(AtIndexGetterSelector, ResultType,
-                                             true /*instance*/);
+  AtIndexGetter = S.ObjC().LookupMethodInObjectType(
+      AtIndexGetterSelector, ResultType, true /*instance*/);
 
   if (!AtIndexGetter && S.getLangOpts().DebuggerObjCLiteral) {
     AtIndexGetter = ObjCMethodDecl::Create(
@@ -1212,10 +1139,8 @@ bool ObjCSubscriptOpBuilder::findAtIndexGetter() {
       << BaseExpr->getType() << 0 << arrayRef;
       return false;
     }
-    AtIndexGetter =
-      S.LookupInstanceMethodInGlobalPool(AtIndexGetterSelector,
-                                         RefExpr->getSourceRange(),
-                                         true);
+    AtIndexGetter = S.ObjC().LookupInstanceMethodInGlobalPool(
+        AtIndexGetterSelector, RefExpr->getSourceRange(), true);
   }
 
   if (AtIndexGetter) {
@@ -1253,15 +1178,15 @@ bool ObjCSubscriptOpBuilder::findAtIndexSetter() {
     ResultType = PTy->getPointeeType();
   }
 
-  Sema::ObjCSubscriptKind Res =
-    S.CheckSubscriptingKind(RefExpr->getKeyExpr());
-  if (Res == Sema::OS_Error) {
+  SemaObjC::ObjCSubscriptKind Res =
+      S.ObjC().CheckSubscriptingKind(RefExpr->getKeyExpr());
+  if (Res == SemaObjC::OS_Error) {
     if (S.getLangOpts().ObjCAutoRefCount)
       CheckKeyForObjCARCConversion(S, ResultType,
                                    RefExpr->getKeyExpr());
     return false;
   }
-  bool arrayRef = (Res == Sema::OS_Array);
+  bool arrayRef = (Res == SemaObjC::OS_Array);
 
   if (ResultType.isNull()) {
     S.Diag(BaseExpr->getExprLoc(), diag::err_objc_subscript_base_type)
@@ -1284,8 +1209,8 @@ bool ObjCSubscriptOpBuilder::findAtIndexSetter() {
         &S.Context.Idents.get("atIndexedSubscript")};
     AtIndexSetterSelector = S.Context.Selectors.getSelector(2, KeyIdents);
   }
-  AtIndexSetter = S.LookupMethodInObjectType(AtIndexSetterSelector, ResultType,
-                                             true /*instance*/);
+  AtIndexSetter = S.ObjC().LookupMethodInObjectType(
+      AtIndexSetterSelector, ResultType, true /*instance*/);
 
   if (!AtIndexSetter && S.getLangOpts().DebuggerObjCLiteral) {
     TypeSourceInfo *ReturnTInfo = nullptr;
@@ -1327,10 +1252,8 @@ bool ObjCSubscriptOpBuilder::findAtIndexSetter() {
       << BaseExpr->getType() << 1 << arrayRef;
       return false;
     }
-    AtIndexSetter =
-      S.LookupInstanceMethodInGlobalPool(AtIndexSetterSelector,
-                                         RefExpr->getSourceRange(),
-                                         true);
+    AtIndexSetter = S.ObjC().LookupInstanceMethodInGlobalPool(
+        AtIndexSetterSelector, RefExpr->getSourceRange(), true);
   }
 
   bool err = false;
@@ -1388,10 +1311,9 @@ ExprResult ObjCSubscriptOpBuilder::buildGet() {
   assert(InstanceBase);
   if (AtIndexGetter)
     S.DiagnoseUseOfDecl(AtIndexGetter, GenericLoc);
-  msg = S.BuildInstanceMessageImplicit(InstanceBase, receiverType,
-                                       GenericLoc,
-                                       AtIndexGetterSelector, AtIndexGetter,
-                                       MultiExprArg(args, 1));
+  msg = S.ObjC().BuildInstanceMessageImplicit(
+      InstanceBase, receiverType, GenericLoc, AtIndexGetterSelector,
+      AtIndexGetter, MultiExprArg(args, 1));
   return msg;
 }
 
@@ -1413,11 +1335,9 @@ ExprResult ObjCSubscriptOpBuilder::buildSet(Expr *op, SourceLocation opcLoc,
   Expr *args[] = { op, Index };
 
   // Build a message-send.
-  ExprResult msg = S.BuildInstanceMessageImplicit(InstanceBase, receiverType,
-                                                  GenericLoc,
-                                                  AtIndexSetterSelector,
-                                                  AtIndexSetter,
-                                                  MultiExprArg(args, 2));
+  ExprResult msg = S.ObjC().BuildInstanceMessageImplicit(
+      InstanceBase, receiverType, GenericLoc, AtIndexSetterSelector,
+      AtIndexSetter, MultiExprArg(args, 2));
 
   if (!msg.isInvalid() && captureSetValueAsResult) {
     ObjCMessageExpr *msgExpr =
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index a7b33f0db047..57465d4a77ac 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -35,6 +35,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
@@ -2236,165 +2237,6 @@ StmtResult Sema::ActOnForEachLValueExpr(Expr *E) {
   return StmtResult(static_cast<Stmt*>(FullExpr.get()));
 }
 
-ExprResult
-Sema::CheckObjCForCollectionOperand(SourceLocation forLoc, Expr *collection) {
-  if (!collection)
-    return ExprError();
-
-  ExprResult result = CorrectDelayedTyposInExpr(collection);
-  if (!result.isUsable())
-    return ExprError();
-  collection = result.get();
-
-  // Bail out early if we've got a type-dependent expression.
-  if (collection->isTypeDependent()) return collection;
-
-  // Perform normal l-value conversion.
-  result = DefaultFunctionArrayLvalueConversion(collection);
-  if (result.isInvalid())
-    return ExprError();
-  collection = result.get();
-
-  // The operand needs to have object-pointer type.
-  // TODO: should we do a contextual conversion?
-  const ObjCObjectPointerType *pointerType =
-    collection->getType()->getAs<ObjCObjectPointerType>();
-  if (!pointerType)
-    return Diag(forLoc, diag::err_collection_expr_type)
-             << collection->getType() << collection->getSourceRange();
-
-  // Check that the operand provides
-  //   - countByEnumeratingWithState:objects:count:
-  const ObjCObjectType *objectType = pointerType->getObjectType();
-  ObjCInterfaceDecl *iface = objectType->getInterface();
-
-  // If we have a forward-declared type, we can't do this check.
-  // Under ARC, it is an error not to have a forward-declared class.
-  if (iface &&
-      (getLangOpts().ObjCAutoRefCount
-           ? RequireCompleteType(forLoc, QualType(objectType, 0),
-                                 diag::err_arc_collection_forward, collection)
-           : !isCompleteType(forLoc, QualType(objectType, 0)))) {
-    // Otherwise, if we have any useful type information, check that
-    // the type declares the appropriate method.
-  } else if (iface || !objectType->qual_empty()) {
-    const IdentifierInfo *selectorIdents[] = {
-        &Context.Idents.get("countByEnumeratingWithState"),
-        &Context.Idents.get("objects"), &Context.Idents.get("count")};
-    Selector selector = Context.Selectors.getSelector(3, &selectorIdents[0]);
-
-    ObjCMethodDecl *method = nullptr;
-
-    // If there's an interface, look in both the public and private APIs.
-    if (iface) {
-      method = iface->lookupInstanceMethod(selector);
-      if (!method) method = iface->lookupPrivateMethod(selector);
-    }
-
-    // Also check protocol qualifiers.
-    if (!method)
-      method = LookupMethodInQualifiedType(selector, pointerType,
-                                           /*instance*/ true);
-
-    // If we didn't find it anywhere, give up.
-    if (!method) {
-      Diag(forLoc, diag::warn_collection_expr_type)
-        << collection->getType() << selector << collection->getSourceRange();
-    }
-
-    // TODO: check for an incompatible signature?
-  }
-
-  // Wrap up any cleanups in the expression.
-  return collection;
-}
-
-StmtResult
-Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
-                                 Stmt *First, Expr *collection,
-                                 SourceLocation RParenLoc) {
-  setFunctionHasBranchProtectedScope();
-
-  ExprResult CollectionExprResult =
-    CheckObjCForCollectionOperand(ForLoc, collection);
-
-  if (First) {
-    QualType FirstType;
-    if (DeclStmt *DS = dyn_cast<DeclStmt>(First)) {
-      if (!DS->isSingleDecl())
-        return StmtError(Diag((*DS->decl_begin())->getLocation(),
-                         diag::err_toomany_element_decls));
-
-      VarDecl *D = dyn_cast<VarDecl>(DS->getSingleDecl());
-      if (!D || D->isInvalidDecl())
-        return StmtError();
-
-      FirstType = D->getType();
-      // C99 6.8.5p3: The declaration part of a 'for' statement shall only
-      // declare identifiers for objects having storage class 'auto' or
-      // 'register'.
-      if (!D->hasLocalStorage())
-        return StmtError(Diag(D->getLocation(),
-                              diag::err_non_local_variable_decl_in_for));
-
-      // If the type contained 'auto', deduce the 'auto' to 'id'.
-      if (FirstType->getContainedAutoType()) {
-        SourceLocation Loc = D->getLocation();
-        OpaqueValueExpr OpaqueId(Loc, Context.getObjCIdType(), VK_PRValue);
-        Expr *DeducedInit = &OpaqueId;
-        TemplateDeductionInfo Info(Loc);
-        FirstType = QualType();
-        TemplateDeductionResult Result = DeduceAutoType(
-            D->getTypeSourceInfo()->getTypeLoc(), DeducedInit, FirstType, Info);
-        if (Result != TemplateDeductionResult::Success &&
-            Result != TemplateDeductionResult::AlreadyDiagnosed)
-          DiagnoseAutoDeductionFailure(D, DeducedInit);
-        if (FirstType.isNull()) {
-          D->setInvalidDecl();
-          return StmtError();
-        }
-
-        D->setType(FirstType);
-
-        if (!inTemplateInstantiation()) {
-          SourceLocation Loc =
-              D->getTypeSourceInfo()->getTypeLoc().getBeginLoc();
-          Diag(Loc, diag::warn_auto_var_is_id)
-            << D->getDeclName();
-        }
-      }
-
-    } else {
-      Expr *FirstE = cast<Expr>(First);
-      if (!FirstE->isTypeDependent() && !FirstE->isLValue())
-        return StmtError(
-            Diag(First->getBeginLoc(), diag::err_selector_element_not_lvalue)
-            << First->getSourceRange());
-
-      FirstType = static_cast<Expr*>(First)->getType();
-      if (FirstType.isConstQualified())
-        Diag(ForLoc, diag::err_selector_element_const_type)
-          << FirstType << First->getSourceRange();
-    }
-    if (!FirstType->isDependentType() &&
-        !FirstType->isObjCObjectPointerType() &&
-        !FirstType->isBlockPointerType())
-        return StmtError(Diag(ForLoc, diag::err_selector_element_type)
-                           << FirstType << First->getSourceRange());
-  }
-
-  if (CollectionExprResult.isInvalid())
-    return StmtError();
-
-  CollectionExprResult =
-      ActOnFinishFullExpr(CollectionExprResult.get(), /*DiscardedValue*/ false);
-  if (CollectionExprResult.isInvalid())
-    return StmtError();
-
-  return new (Context) ObjCForCollectionStmt(First, CollectionExprResult.get(),
-                                             nullptr, ForLoc, RParenLoc);
-}
-
 /// Finish building a variable declaration for a for-range statement.
 /// \return true if an error occurs.
 static bool FinishForRangeVarDecl(Sema &SemaRef, VarDecl *Decl, Expr *Init,
@@ -2432,7 +2274,7 @@ static bool FinishForRangeVarDecl(Sema &SemaRef, VarDecl *Decl, Expr *Init,
   // FIXME: ARC may want to turn this into 'const __unsafe_unretained' if
   // we're doing the equivalent of fast iteration.
   if (SemaRef.getLangOpts().ObjCAutoRefCount &&
-      SemaRef.inferObjCARCLifetime(Decl))
+      SemaRef.ObjC().inferObjCARCLifetime(Decl))
     Decl->setInvalidDecl();
 
   SemaRef.AddInitializerToDecl(Decl, Init, /*DirectInit=*/false);
@@ -2526,7 +2368,7 @@ StmtResult Sema::ActOnCXXForRangeStmt(
     if (InitStmt)
       return Diag(InitStmt->getBeginLoc(), diag::err_objc_for_range_init_stmt)
                  << InitStmt->getSourceRange();
-    return ActOnObjCForCollectionStmt(ForLoc, First, Range, RParenLoc);
+    return ObjC().ActOnObjCForCollectionStmt(ForLoc, First, Range, RParenLoc);
   }
 
   DeclStmt *DS = dyn_cast<DeclStmt>(First);
@@ -3107,17 +2949,6 @@ StmtResult Sema::BuildCXXForRangeStmt(
       ColonLoc, RParenLoc);
 }
 
-/// FinishObjCForCollectionStmt - Attach the body to a objective-C foreach
-/// statement.
-StmtResult Sema::FinishObjCForCollectionStmt(Stmt *S, Stmt *B) {
-  if (!S || !B)
-    return StmtError();
-  ObjCForCollectionStmt * ForStmt = cast<ObjCForCollectionStmt>(S);
-
-  ForStmt->setBody(B);
-  return S;
-}
-
 // Warn when the loop variable is a const reference that creates a copy.
 // Suggest using the non-reference type for copies.  If a copy can be prevented
 // suggest the const reference type that would do so.
@@ -3307,7 +3138,7 @@ StmtResult Sema::FinishCXXForRangeStmt(Stmt *S, Stmt *B) {
     return StmtError();
 
   if (isa<ObjCForCollectionStmt>(S))
-    return FinishObjCForCollectionStmt(S, B);
+    return ObjC().FinishObjCForCollectionStmt(S, B);
 
   CXXForRangeStmt *ForStmt = cast<CXXForRangeStmt>(S);
   ForStmt->setBody(B);
@@ -4301,130 +4132,6 @@ StmtResult Sema::BuildReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp,
   return Result;
 }
 
-StmtResult
-Sema::ActOnObjCAtCatchStmt(SourceLocation AtLoc,
-                           SourceLocation RParen, Decl *Parm,
-                           Stmt *Body) {
-  VarDecl *Var = cast_or_null<VarDecl>(Parm);
-  if (Var && Var->isInvalidDecl())
-    return StmtError();
-
-  return new (Context) ObjCAtCatchStmt(AtLoc, RParen, Var, Body);
-}
-
-StmtResult
-Sema::ActOnObjCAtFinallyStmt(SourceLocation AtLoc, Stmt *Body) {
-  return new (Context) ObjCAtFinallyStmt(AtLoc, Body);
-}
-
-StmtResult
-Sema::ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try,
-                         MultiStmtArg CatchStmts, Stmt *Finally) {
-  if (!getLangOpts().ObjCExceptions)
-    Diag(AtLoc, diag::err_objc_exceptions_disabled) << "@try";
-
-  // Objective-C try is incompatible with SEH __try.
-  sema::FunctionScopeInfo *FSI = getCurFunction();
-  if (FSI->FirstSEHTryLoc.isValid()) {
-    Diag(AtLoc, diag::err_mixing_cxx_try_seh_try) << 1;
-    Diag(FSI->FirstSEHTryLoc, diag::note_conflicting_try_here) << "'__try'";
-  }
-
-  FSI->setHasObjCTry(AtLoc);
-  unsigned NumCatchStmts = CatchStmts.size();
-  return ObjCAtTryStmt::Create(Context, AtLoc, Try, CatchStmts.data(),
-                               NumCatchStmts, Finally);
-}
-
-StmtResult Sema::BuildObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw) {
-  if (Throw) {
-    ExprResult Result = DefaultLvalueConversion(Throw);
-    if (Result.isInvalid())
-      return StmtError();
-
-    Result = ActOnFinishFullExpr(Result.get(), /*DiscardedValue*/ false);
-    if (Result.isInvalid())
-      return StmtError();
-    Throw = Result.get();
-
-    QualType ThrowType = Throw->getType();
-    // Make sure the expression type is an ObjC pointer or "void *".
-    if (!ThrowType->isDependentType() &&
-        !ThrowType->isObjCObjectPointerType()) {
-      const PointerType *PT = ThrowType->getAs<PointerType>();
-      if (!PT || !PT->getPointeeType()->isVoidType())
-        return StmtError(Diag(AtLoc, diag::err_objc_throw_expects_object)
-                         << Throw->getType() << Throw->getSourceRange());
-    }
-  }
-
-  return new (Context) ObjCAtThrowStmt(AtLoc, Throw);
-}
-
-StmtResult
-Sema::ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw,
-                           Scope *CurScope) {
-  if (!getLangOpts().ObjCExceptions)
-    Diag(AtLoc, diag::err_objc_exceptions_disabled) << "@throw";
-
-  if (!Throw) {
-    // @throw without an expression designates a rethrow (which must occur
-    // in the context of an @catch clause).
-    Scope *AtCatchParent = CurScope;
-    while (AtCatchParent && !AtCatchParent->isAtCatchScope())
-      AtCatchParent = AtCatchParent->getParent();
-    if (!AtCatchParent)
-      return StmtError(Diag(AtLoc, diag::err_rethrow_used_outside_catch));
-  }
-  return BuildObjCAtThrowStmt(AtLoc, Throw);
-}
-
-ExprResult
-Sema::ActOnObjCAtSynchronizedOperand(SourceLocation atLoc, Expr *operand) {
-  ExprResult result = DefaultLvalueConversion(operand);
-  if (result.isInvalid())
-    return ExprError();
-  operand = result.get();
-
-  // Make sure the expression type is an ObjC pointer or "void *".
-  QualType type = operand->getType();
-  if (!type->isDependentType() &&
-      !type->isObjCObjectPointerType()) {
-    const PointerType *pointerType = type->getAs<PointerType>();
-    if (!pointerType || !pointerType->getPointeeType()->isVoidType()) {
-      if (getLangOpts().CPlusPlus) {
-        if (RequireCompleteType(atLoc, type,
-                                diag::err_incomplete_receiver_type))
-          return Diag(atLoc, diag::err_objc_synchronized_expects_object)
-                   << type << operand->getSourceRange();
-
-        ExprResult result = PerformContextuallyConvertToObjCPointer(operand);
-        if (result.isInvalid())
-          return ExprError();
-        if (!result.isUsable())
-          return Diag(atLoc, diag::err_objc_synchronized_expects_object)
-                   << type << operand->getSourceRange();
-
-        operand = result.get();
-      } else {
-          return Diag(atLoc, diag::err_objc_synchronized_expects_object)
-                   << type << operand->getSourceRange();
-      }
-    }
-  }
-
-  // The operand to @synchronized is a full-expression.
-  return ActOnFinishFullExpr(operand, /*DiscardedValue*/ false);
-}
-
-StmtResult
-Sema::ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc, Expr *SyncExpr,
-                                  Stmt *SyncBody) {
-  // We can't jump into or indirect-jump out of a @synchronized block.
-  setFunctionHasBranchProtectedScope();
-  return new (Context) ObjCAtSynchronizedStmt(AtLoc, SyncExpr, SyncBody);
-}
-
 /// ActOnCXXCatchBlock - Takes an exception declaration and a handler block
 /// and creates a proper catch handler from them.
 StmtResult
@@ -4435,12 +4142,6 @@ Sema::ActOnCXXCatchBlock(SourceLocation CatchLoc, Decl *ExDecl,
       CXXCatchStmt(CatchLoc, cast_or_null<VarDecl>(ExDecl), HandlerBlock);
 }
 
-StmtResult
-Sema::ActOnObjCAutoreleasePoolStmt(SourceLocation AtLoc, Stmt *Body) {
-  setFunctionHasBranchProtectedScope();
-  return new (Context) ObjCAutoreleasePoolStmt(AtLoc, Body);
-}
-
 namespace {
 class CatchHandlerType {
   QualType QT;
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index fde2d920c785..e0c1f814f852 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -28,6 +28,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateInstCallback.h"
@@ -1217,7 +1218,7 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D,
 
   // In ARC, infer 'retaining' for variables of retainable type.
   if (SemaRef.getLangOpts().ObjCAutoRefCount &&
-      SemaRef.inferObjCARCLifetime(Var))
+      SemaRef.ObjC().inferObjCARCLifetime(Var))
     Var->setInvalidDecl();
 
   if (SemaRef.getLangOpts().OpenCL)
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index fddc3545ecb6..bfa3799bda06 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -35,6 +35,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateInstCallback.h"
@@ -849,424 +850,6 @@ static bool checkOmittedBlockReturnType(Sema &S, Declarator &declarator,
   return true;
 }
 
-/// Apply Objective-C type arguments to the given type.
-static QualType applyObjCTypeArgs(Sema &S, SourceLocation loc, QualType type,
-                                  ArrayRef<TypeSourceInfo *> typeArgs,
-                                  SourceRange typeArgsRange, bool failOnError,
-                                  bool rebuilding) {
-  // We can only apply type arguments to an Objective-C class type.
-  const auto *objcObjectType = type->getAs<ObjCObjectType>();
-  if (!objcObjectType || !objcObjectType->getInterface()) {
-    S.Diag(loc, diag::err_objc_type_args_non_class)
-      << type
-      << typeArgsRange;
-
-    if (failOnError)
-      return QualType();
-    return type;
-  }
-
-  // The class type must be parameterized.
-  ObjCInterfaceDecl *objcClass = objcObjectType->getInterface();
-  ObjCTypeParamList *typeParams = objcClass->getTypeParamList();
-  if (!typeParams) {
-    S.Diag(loc, diag::err_objc_type_args_non_parameterized_class)
-      << objcClass->getDeclName()
-      << FixItHint::CreateRemoval(typeArgsRange);
-
-    if (failOnError)
-      return QualType();
-
-    return type;
-  }
-
-  // The type must not already be specialized.
-  if (objcObjectType->isSpecialized()) {
-    S.Diag(loc, diag::err_objc_type_args_specialized_class)
-      << type
-      << FixItHint::CreateRemoval(typeArgsRange);
-
-    if (failOnError)
-      return QualType();
-
-    return type;
-  }
-
-  // Check the type arguments.
-  SmallVector<QualType, 4> finalTypeArgs;
-  unsigned numTypeParams = typeParams->size();
-  bool anyPackExpansions = false;
-  for (unsigned i = 0, n = typeArgs.size(); i != n; ++i) {
-    TypeSourceInfo *typeArgInfo = typeArgs[i];
-    QualType typeArg = typeArgInfo->getType();
-
-    // Type arguments cannot have explicit qualifiers or nullability.
-    // We ignore indirect sources of these, e.g. behind typedefs or
-    // template arguments.
-    if (TypeLoc qual = typeArgInfo->getTypeLoc().findExplicitQualifierLoc()) {
-      bool diagnosed = false;
-      SourceRange rangeToRemove;
-      if (auto attr = qual.getAs<AttributedTypeLoc>()) {
-        rangeToRemove = attr.getLocalSourceRange();
-        if (attr.getTypePtr()->getImmediateNullability()) {
-          typeArg = attr.getTypePtr()->getModifiedType();
-          S.Diag(attr.getBeginLoc(),
-                 diag::err_objc_type_arg_explicit_nullability)
-              << typeArg << FixItHint::CreateRemoval(rangeToRemove);
-          diagnosed = true;
-        }
-      }
-
-      // When rebuilding, qualifiers might have gotten here through a
-      // final substitution.
-      if (!rebuilding && !diagnosed) {
-        S.Diag(qual.getBeginLoc(), diag::err_objc_type_arg_qualified)
-            << typeArg << typeArg.getQualifiers().getAsString()
-            << FixItHint::CreateRemoval(rangeToRemove);
-      }
-    }
-
-    // Remove qualifiers even if they're non-local.
-    typeArg = typeArg.getUnqualifiedType();
-
-    finalTypeArgs.push_back(typeArg);
-
-    if (typeArg->getAs<PackExpansionType>())
-      anyPackExpansions = true;
-
-    // Find the corresponding type parameter, if there is one.
-    ObjCTypeParamDecl *typeParam = nullptr;
-    if (!anyPackExpansions) {
-      if (i < numTypeParams) {
-        typeParam = typeParams->begin()[i];
-      } else {
-        // Too many arguments.
-        S.Diag(loc, diag::err_objc_type_args_wrong_arity)
-          << false
-          << objcClass->getDeclName()
-          << (unsigned)typeArgs.size()
-          << numTypeParams;
-        S.Diag(objcClass->getLocation(), diag::note_previous_decl)
-          << objcClass;
-
-        if (failOnError)
-          return QualType();
-
-        return type;
-      }
-    }
-
-    // Objective-C object pointer types must be substitutable for the bounds.
-    if (const auto *typeArgObjC = typeArg->getAs<ObjCObjectPointerType>()) {
-      // If we don't have a type parameter to match against, assume
-      // everything is fine. There was a prior pack expansion that
-      // means we won't be able to match anything.
-      if (!typeParam) {
-        assert(anyPackExpansions && "Too many arguments?");
-        continue;
-      }
-
-      // Retrieve the bound.
-      QualType bound = typeParam->getUnderlyingType();
-      const auto *boundObjC = bound->castAs<ObjCObjectPointerType>();
-
-      // Determine whether the type argument is substitutable for the bound.
-      if (typeArgObjC->isObjCIdType()) {
-        // When the type argument is 'id', the only acceptable type
-        // parameter bound is 'id'.
-        if (boundObjC->isObjCIdType())
-          continue;
-      } else if (S.Context.canAssignObjCInterfaces(boundObjC, typeArgObjC)) {
-        // Otherwise, we follow the assignability rules.
-        continue;
-      }
-
-      // Diagnose the mismatch.
-      S.Diag(typeArgInfo->getTypeLoc().getBeginLoc(),
-             diag::err_objc_type_arg_does_not_match_bound)
-          << typeArg << bound << typeParam->getDeclName();
-      S.Diag(typeParam->getLocation(), diag::note_objc_type_param_here)
-        << typeParam->getDeclName();
-
-      if (failOnError)
-        return QualType();
-
-      return type;
-    }
-
-    // Block pointer types are permitted for unqualified 'id' bounds.
-    if (typeArg->isBlockPointerType()) {
-      // If we don't have a type parameter to match against, assume
-      // everything is fine. There was a prior pack expansion that
-      // means we won't be able to match anything.
-      if (!typeParam) {
-        assert(anyPackExpansions && "Too many arguments?");
-        continue;
-      }
-
-      // Retrieve the bound.
-      QualType bound = typeParam->getUnderlyingType();
-      if (bound->isBlockCompatibleObjCPointerType(S.Context))
-        continue;
-
-      // Diagnose the mismatch.
-      S.Diag(typeArgInfo->getTypeLoc().getBeginLoc(),
-             diag::err_objc_type_arg_does_not_match_bound)
-          << typeArg << bound << typeParam->getDeclName();
-      S.Diag(typeParam->getLocation(), diag::note_objc_type_param_here)
-        << typeParam->getDeclName();
-
-      if (failOnError)
-        return QualType();
-
-      return type;
-    }
-
-    // Types that have __attribute__((NSObject)) are permitted.
-    if (typeArg->isObjCNSObjectType()) {
-      continue;
-    }
-
-    // Dependent types will be checked at instantiation time.
-    if (typeArg->isDependentType()) {
-      continue;
-    }
-
-    // Diagnose non-id-compatible type arguments.
-    S.Diag(typeArgInfo->getTypeLoc().getBeginLoc(),
-           diag::err_objc_type_arg_not_id_compatible)
-        << typeArg << typeArgInfo->getTypeLoc().getSourceRange();
-
-    if (failOnError)
-      return QualType();
-
-    return type;
-  }
-
-  // Make sure we didn't have the wrong number of arguments.
-  if (!anyPackExpansions && finalTypeArgs.size() != numTypeParams) {
-    S.Diag(loc, diag::err_objc_type_args_wrong_arity)
-      << (typeArgs.size() < typeParams->size())
-      << objcClass->getDeclName()
-      << (unsigned)finalTypeArgs.size()
-      << (unsigned)numTypeParams;
-    S.Diag(objcClass->getLocation(), diag::note_previous_decl)
-      << objcClass;
-
-    if (failOnError)
-      return QualType();
-
-    return type;
-  }
-
-  // Success. Form the specialized type.
-  return S.Context.getObjCObjectType(type, finalTypeArgs, { }, false);
-}
-
-QualType Sema::BuildObjCTypeParamType(const ObjCTypeParamDecl *Decl,
-                                      SourceLocation ProtocolLAngleLoc,
-                                      ArrayRef<ObjCProtocolDecl *> Protocols,
-                                      ArrayRef<SourceLocation> ProtocolLocs,
-                                      SourceLocation ProtocolRAngleLoc,
-                                      bool FailOnError) {
-  QualType Result = QualType(Decl->getTypeForDecl(), 0);
-  if (!Protocols.empty()) {
-    bool HasError;
-    Result = Context.applyObjCProtocolQualifiers(Result, Protocols,
-                                                 HasError);
-    if (HasError) {
-      Diag(SourceLocation(), diag::err_invalid_protocol_qualifiers)
-        << SourceRange(ProtocolLAngleLoc, ProtocolRAngleLoc);
-      if (FailOnError) Result = QualType();
-    }
-    if (FailOnError && Result.isNull())
-      return QualType();
-  }
-
-  return Result;
-}
-
-QualType Sema::BuildObjCObjectType(
-    QualType BaseType, SourceLocation Loc, SourceLocation TypeArgsLAngleLoc,
-    ArrayRef<TypeSourceInfo *> TypeArgs, SourceLocation TypeArgsRAngleLoc,
-    SourceLocation ProtocolLAngleLoc, ArrayRef<ObjCProtocolDecl *> Protocols,
-    ArrayRef<SourceLocation> ProtocolLocs, SourceLocation ProtocolRAngleLoc,
-    bool FailOnError, bool Rebuilding) {
-  QualType Result = BaseType;
-  if (!TypeArgs.empty()) {
-    Result =
-        applyObjCTypeArgs(*this, Loc, Result, TypeArgs,
-                          SourceRange(TypeArgsLAngleLoc, TypeArgsRAngleLoc),
-                          FailOnError, Rebuilding);
-    if (FailOnError && Result.isNull())
-      return QualType();
-  }
-
-  if (!Protocols.empty()) {
-    bool HasError;
-    Result = Context.applyObjCProtocolQualifiers(Result, Protocols,
-                                                 HasError);
-    if (HasError) {
-      Diag(Loc, diag::err_invalid_protocol_qualifiers)
-        << SourceRange(ProtocolLAngleLoc, ProtocolRAngleLoc);
-      if (FailOnError) Result = QualType();
-    }
-    if (FailOnError && Result.isNull())
-      return QualType();
-  }
-
-  return Result;
-}
-
-TypeResult Sema::actOnObjCProtocolQualifierType(
-             SourceLocation lAngleLoc,
-             ArrayRef<Decl *> protocols,
-             ArrayRef<SourceLocation> protocolLocs,
-             SourceLocation rAngleLoc) {
-  // Form id<protocol-list>.
-  QualType Result = Context.getObjCObjectType(
-      Context.ObjCBuiltinIdTy, {},
-      llvm::ArrayRef((ObjCProtocolDecl *const *)protocols.data(),
-                     protocols.size()),
-      false);
-  Result = Context.getObjCObjectPointerType(Result);
-
-  TypeSourceInfo *ResultTInfo = Context.CreateTypeSourceInfo(Result);
-  TypeLoc ResultTL = ResultTInfo->getTypeLoc();
-
-  auto ObjCObjectPointerTL = ResultTL.castAs<ObjCObjectPointerTypeLoc>();
-  ObjCObjectPointerTL.setStarLoc(SourceLocation()); // implicit
-
-  auto ObjCObjectTL = ObjCObjectPointerTL.getPointeeLoc()
-                        .castAs<ObjCObjectTypeLoc>();
-  ObjCObjectTL.setHasBaseTypeAsWritten(false);
-  ObjCObjectTL.getBaseLoc().initialize(Context, SourceLocation());
-
-  // No type arguments.
-  ObjCObjectTL.setTypeArgsLAngleLoc(SourceLocation());
-  ObjCObjectTL.setTypeArgsRAngleLoc(SourceLocation());
-
-  // Fill in protocol qualifiers.
-  ObjCObjectTL.setProtocolLAngleLoc(lAngleLoc);
-  ObjCObjectTL.setProtocolRAngleLoc(rAngleLoc);
-  for (unsigned i = 0, n = protocols.size(); i != n; ++i)
-    ObjCObjectTL.setProtocolLoc(i, protocolLocs[i]);
-
-  // We're done. Return the completed type to the parser.
-  return CreateParsedType(Result, ResultTInfo);
-}
-
-TypeResult Sema::actOnObjCTypeArgsAndProtocolQualifiers(
-             Scope *S,
-             SourceLocation Loc,
-             ParsedType BaseType,
-             SourceLocation TypeArgsLAngleLoc,
-             ArrayRef<ParsedType> TypeArgs,
-             SourceLocation TypeArgsRAngleLoc,
-             SourceLocation ProtocolLAngleLoc,
-             ArrayRef<Decl *> Protocols,
-             ArrayRef<SourceLocation> ProtocolLocs,
-             SourceLocation ProtocolRAngleLoc) {
-  TypeSourceInfo *BaseTypeInfo = nullptr;
-  QualType T = GetTypeFromParser(BaseType, &BaseTypeInfo);
-  if (T.isNull())
-    return true;
-
-  // Handle missing type-source info.
-  if (!BaseTypeInfo)
-    BaseTypeInfo = Context.getTrivialTypeSourceInfo(T, Loc);
-
-  // Extract type arguments.
-  SmallVector<TypeSourceInfo *, 4> ActualTypeArgInfos;
-  for (unsigned i = 0, n = TypeArgs.size(); i != n; ++i) {
-    TypeSourceInfo *TypeArgInfo = nullptr;
-    QualType TypeArg = GetTypeFromParser(TypeArgs[i], &TypeArgInfo);
-    if (TypeArg.isNull()) {
-      ActualTypeArgInfos.clear();
-      break;
-    }
-
-    assert(TypeArgInfo && "No type source info?");
-    ActualTypeArgInfos.push_back(TypeArgInfo);
-  }
-
-  // Build the object type.
-  QualType Result = BuildObjCObjectType(
-      T, BaseTypeInfo->getTypeLoc().getSourceRange().getBegin(),
-      TypeArgsLAngleLoc, ActualTypeArgInfos, TypeArgsRAngleLoc,
-      ProtocolLAngleLoc,
-      llvm::ArrayRef((ObjCProtocolDecl *const *)Protocols.data(),
-                     Protocols.size()),
-      ProtocolLocs, ProtocolRAngleLoc,
-      /*FailOnError=*/false,
-      /*Rebuilding=*/false);
-
-  if (Result == T)
-    return BaseType;
-
-  // Create source information for this type.
-  TypeSourceInfo *ResultTInfo = Context.CreateTypeSourceInfo(Result);
-  TypeLoc ResultTL = ResultTInfo->getTypeLoc();
-
-  // For id<Proto1, Proto2> or Class<Proto1, Proto2>, we'll have an
-  // object pointer type. Fill in source information for it.
-  if (auto ObjCObjectPointerTL = ResultTL.getAs<ObjCObjectPointerTypeLoc>()) {
-    // The '*' is implicit.
-    ObjCObjectPointerTL.setStarLoc(SourceLocation());
-    ResultTL = ObjCObjectPointerTL.getPointeeLoc();
-  }
-
-  if (auto OTPTL = ResultTL.getAs<ObjCTypeParamTypeLoc>()) {
-    // Protocol qualifier information.
-    if (OTPTL.getNumProtocols() > 0) {
-      assert(OTPTL.getNumProtocols() == Protocols.size());
-      OTPTL.setProtocolLAngleLoc(ProtocolLAngleLoc);
-      OTPTL.setProtocolRAngleLoc(ProtocolRAngleLoc);
-      for (unsigned i = 0, n = Protocols.size(); i != n; ++i)
-        OTPTL.setProtocolLoc(i, ProtocolLocs[i]);
-    }
-
-    // We're done. Return the completed type to the parser.
-    return CreateParsedType(Result, ResultTInfo);
-  }
-
-  auto ObjCObjectTL = ResultTL.castAs<ObjCObjectTypeLoc>();
-
-  // Type argument information.
-  if (ObjCObjectTL.getNumTypeArgs() > 0) {
-    assert(ObjCObjectTL.getNumTypeArgs() == ActualTypeArgInfos.size());
-    ObjCObjectTL.setTypeArgsLAngleLoc(TypeArgsLAngleLoc);
-    ObjCObjectTL.setTypeArgsRAngleLoc(TypeArgsRAngleLoc);
-    for (unsigned i = 0, n = ActualTypeArgInfos.size(); i != n; ++i)
-      ObjCObjectTL.setTypeArgTInfo(i, ActualTypeArgInfos[i]);
-  } else {
-    ObjCObjectTL.setTypeArgsLAngleLoc(SourceLocation());
-    ObjCObjectTL.setTypeArgsRAngleLoc(SourceLocation());
-  }
-
-  // Protocol qualifier information.
-  if (ObjCObjectTL.getNumProtocols() > 0) {
-    assert(ObjCObjectTL.getNumProtocols() == Protocols.size());
-    ObjCObjectTL.setProtocolLAngleLoc(ProtocolLAngleLoc);
-    ObjCObjectTL.setProtocolRAngleLoc(ProtocolRAngleLoc);
-    for (unsigned i = 0, n = Protocols.size(); i != n; ++i)
-      ObjCObjectTL.setProtocolLoc(i, ProtocolLocs[i]);
-  } else {
-    ObjCObjectTL.setProtocolLAngleLoc(SourceLocation());
-    ObjCObjectTL.setProtocolRAngleLoc(SourceLocation());
-  }
-
-  // Base type.
-  ObjCObjectTL.setHasBaseTypeAsWritten(true);
-  if (ObjCObjectTL.getType() == T)
-    ObjCObjectTL.getBaseLoc().initializeFullCopy(BaseTypeInfo->getTypeLoc());
-  else
-    ObjCObjectTL.getBaseLoc().initialize(Context, Loc);
-
-  // We're done. Return the completed type to the parser.
-  return CreateParsedType(Result, ResultTInfo);
-}
-
 static OpenCLAccessAttr::Spelling
 getImageAccess(const ParsedAttributesView &Attrs) {
   for (const ParsedAttr &AL : Attrs)
@@ -4259,14 +3842,6 @@ IdentifierInfo *Sema::getNullabilityKeyword(NullabilityKind nullability) {
   llvm_unreachable("Unknown nullability kind.");
 }
 
-/// Retrieve the identifier "NSError".
-IdentifierInfo *Sema::getNSErrorIdent() {
-  if (!Ident_NSError)
-    Ident_NSError = PP.getIdentifierInfo("NSError");
-
-  return Ident_NSError;
-}
-
 /// Check whether there is a nullability attribute of any kind in the given
 /// attribute list.
 static bool hasNullabilityAttr(const ParsedAttributesView &attrs) {
@@ -4392,7 +3967,7 @@ classifyPointerDeclarator(Sema &S, QualType type, Declarator &declarator,
 
       // If this is NSError**, report that.
       if (auto objcClassDecl = objcObjectPtr->getInterfaceDecl()) {
-        if (objcClassDecl->getIdentifier() == S.getNSErrorIdent() &&
+        if (objcClassDecl->getIdentifier() == S.ObjC().getNSErrorIdent() &&
             numNormalPointers == 2 && numTypeSpecifierPointers < 2) {
           return PointerDeclaratorKind::NSErrorPointerPointer;
         }
@@ -4403,7 +3978,8 @@ classifyPointerDeclarator(Sema &S, QualType type, Declarator &declarator,
 
     // Look at Objective-C class types.
     if (auto objcClass = type->getAs<ObjCInterfaceType>()) {
-      if (objcClass->getInterface()->getIdentifier() == S.getNSErrorIdent()) {
+      if (objcClass->getInterface()->getIdentifier() ==
+          S.ObjC().getNSErrorIdent()) {
         if (numNormalPointers == 2 && numTypeSpecifierPointers < 2)
           return PointerDeclaratorKind::NSErrorPointerPointer;
       }
@@ -4420,7 +3996,7 @@ classifyPointerDeclarator(Sema &S, QualType type, Declarator &declarator,
 
       // If this is CFErrorRef*, report it as such.
       if (numNormalPointers == 2 && numTypeSpecifierPointers < 2 &&
-          S.isCFError(recordDecl)) {
+          S.ObjC().isCFError(recordDecl)) {
         return PointerDeclaratorKind::CFErrorRefPointer;
       }
       break;
@@ -4444,31 +4020,6 @@ classifyPointerDeclarator(Sema &S, QualType type, Declarator &declarator,
   }
 }
 
-bool Sema::isCFError(RecordDecl *RD) {
-  // If we already know about CFError, test it directly.
-  if (CFError)
-    return CFError == RD;
-
-  // Check whether this is CFError, which we identify based on its bridge to
-  // NSError. CFErrorRef used to be declared with "objc_bridge" but is now
-  // declared with "objc_bridge_mutable", so look for either one of the two
-  // attributes.
-  if (RD->getTagKind() == TagTypeKind::Struct) {
-    IdentifierInfo *bridgedType = nullptr;
-    if (auto bridgeAttr = RD->getAttr<ObjCBridgeAttr>())
-      bridgedType = bridgeAttr->getBridgedType();
-    else if (auto bridgeAttr = RD->getAttr<ObjCBridgeMutableAttr>())
-      bridgedType = bridgeAttr->getBridgedType();
-
-    if (bridgedType == getNSErrorIdent()) {
-      CFError = RD;
-      return true;
-    }
-  }
-
-  return false;
-}
-
 static FileID getNullabilityCompletenessCheckFileID(Sema &S,
                                                     SourceLocation loc) {
   // If we're anywhere in a function, method, or closure context, don't perform
@@ -6839,12 +6390,6 @@ TypeResult Sema::ActOnTypeName(Declarator &D) {
   return CreateParsedType(T, TInfo);
 }
 
-ParsedType Sema::ActOnObjCInstanceType(SourceLocation Loc) {
-  QualType T = Context.getObjCInstanceType();
-  TypeSourceInfo *TInfo = Context.getTrivialTypeSourceInfo(T, Loc);
-  return CreateParsedType(T, TInfo);
-}
-
 //===----------------------------------------------------------------------===//
 // Type Attribute Processing
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index ab26d1b1199a..2d903dc52556 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -39,6 +39,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaDiagnostic.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenACC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/SemaSYCL.h"
@@ -1609,8 +1610,8 @@ public:
                                         Stmt *TryBody,
                                         MultiStmtArg CatchStmts,
                                         Stmt *Finally) {
-    return getSema().ActOnObjCAtTryStmt(AtLoc, TryBody, CatchStmts,
-                                        Finally);
+    return getSema().ObjC().ActOnObjCAtTryStmt(AtLoc, TryBody, CatchStmts,
+                                               Finally);
   }
 
   /// Rebuild an Objective-C exception declaration.
@@ -1619,10 +1620,9 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   VarDecl *RebuildObjCExceptionDecl(VarDecl *ExceptionDecl,
                                     TypeSourceInfo *TInfo, QualType T) {
-    return getSema().BuildObjCExceptionDecl(TInfo, T,
-                                            ExceptionDecl->getInnerLocStart(),
-                                            ExceptionDecl->getLocation(),
-                                            ExceptionDecl->getIdentifier());
+    return getSema().ObjC().BuildObjCExceptionDecl(
+        TInfo, T, ExceptionDecl->getInnerLocStart(),
+        ExceptionDecl->getLocation(), ExceptionDecl->getIdentifier());
   }
 
   /// Build a new Objective-C \@catch statement.
@@ -1633,8 +1633,7 @@ public:
                                           SourceLocation RParenLoc,
                                           VarDecl *Var,
                                           Stmt *Body) {
-    return getSema().ActOnObjCAtCatchStmt(AtLoc, RParenLoc,
-                                          Var, Body);
+    return getSema().ObjC().ActOnObjCAtCatchStmt(AtLoc, RParenLoc, Var, Body);
   }
 
   /// Build a new Objective-C \@finally statement.
@@ -1643,7 +1642,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildObjCAtFinallyStmt(SourceLocation AtLoc,
                                             Stmt *Body) {
-    return getSema().ActOnObjCAtFinallyStmt(AtLoc, Body);
+    return getSema().ObjC().ActOnObjCAtFinallyStmt(AtLoc, Body);
   }
 
   /// Build a new Objective-C \@throw statement.
@@ -1652,7 +1651,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildObjCAtThrowStmt(SourceLocation AtLoc,
                                           Expr *Operand) {
-    return getSema().BuildObjCAtThrowStmt(AtLoc, Operand);
+    return getSema().ObjC().BuildObjCAtThrowStmt(AtLoc, Operand);
   }
 
   /// Build a new OpenMP Canonical loop.
@@ -2492,7 +2491,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildObjCAtSynchronizedOperand(SourceLocation atLoc,
                                               Expr *object) {
-    return getSema().ActOnObjCAtSynchronizedOperand(atLoc, object);
+    return getSema().ObjC().ActOnObjCAtSynchronizedOperand(atLoc, object);
   }
 
   /// Build a new Objective-C \@synchronized statement.
@@ -2501,7 +2500,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildObjCAtSynchronizedStmt(SourceLocation AtLoc,
                                            Expr *Object, Stmt *Body) {
-    return getSema().ActOnObjCAtSynchronizedStmt(AtLoc, Object, Body);
+    return getSema().ObjC().ActOnObjCAtSynchronizedStmt(AtLoc, Object, Body);
   }
 
   /// Build a new Objective-C \@autoreleasepool statement.
@@ -2510,7 +2509,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildObjCAutoreleasePoolStmt(SourceLocation AtLoc,
                                             Stmt *Body) {
-    return getSema().ActOnObjCAutoreleasePoolStmt(AtLoc, Body);
+    return getSema().ObjC().ActOnObjCAutoreleasePoolStmt(AtLoc, Body);
   }
 
   /// Build a new Objective-C fast enumeration statement.
@@ -2522,14 +2521,13 @@ public:
                                           Expr *Collection,
                                           SourceLocation RParenLoc,
                                           Stmt *Body) {
-    StmtResult ForEachStmt = getSema().ActOnObjCForCollectionStmt(ForLoc,
-                                                Element,
-                                                Collection,
-                                                RParenLoc);
+    StmtResult ForEachStmt = getSema().ObjC().ActOnObjCForCollectionStmt(
+        ForLoc, Element, Collection, RParenLoc);
     if (ForEachStmt.isInvalid())
       return StmtError();
 
-    return getSema().FinishObjCForCollectionStmt(ForEachStmt.get(), Body);
+    return getSema().ObjC().FinishObjCForCollectionStmt(ForEachStmt.get(),
+                                                        Body);
   }
 
   /// Build a new C++ exception declaration.
@@ -2595,8 +2593,8 @@ public:
                                   diag::err_objc_for_range_init_stmt)
                          << Init->getSourceRange();
             }
-            return getSema().ActOnObjCForCollectionStmt(ForLoc, LoopVar,
-                                                        RangeExpr, RParenLoc);
+            return getSema().ObjC().ActOnObjCForCollectionStmt(
+                ForLoc, LoopVar, RangeExpr, RParenLoc);
           }
         }
       }
@@ -3720,7 +3718,7 @@ public:
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
-    return getSema().BuildObjCBoxedExpr(SR, ValueExpr);
+    return getSema().ObjC().BuildObjCBoxedExpr(SR, ValueExpr);
   }
 
   /// Build a new Objective-C array literal.
@@ -3729,16 +3727,16 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildObjCArrayLiteral(SourceRange Range,
                                      Expr **Elements, unsigned NumElements) {
-    return getSema().BuildObjCArrayLiteral(Range,
-                                           MultiExprArg(Elements, NumElements));
+    return getSema().ObjC().BuildObjCArrayLiteral(
+        Range, MultiExprArg(Elements, NumElements));
   }
 
   ExprResult RebuildObjCSubscriptRefExpr(SourceLocation RB,
                                          Expr *Base, Expr *Key,
                                          ObjCMethodDecl *getterMethod,
                                          ObjCMethodDecl *setterMethod) {
-    return  getSema().BuildObjCSubscriptExpression(RB, Base, Key,
-                                                   getterMethod, setterMethod);
+    return getSema().ObjC().BuildObjCSubscriptExpression(
+        RB, Base, Key, getterMethod, setterMethod);
   }
 
   /// Build a new Objective-C dictionary literal.
@@ -3747,7 +3745,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildObjCDictionaryLiteral(SourceRange Range,
                               MutableArrayRef<ObjCDictionaryElement> Elements) {
-    return getSema().BuildObjCDictionaryLiteral(Range, Elements);
+    return getSema().ObjC().BuildObjCDictionaryLiteral(Range, Elements);
   }
 
   /// Build a new Objective-C \@encode expression.
@@ -3757,7 +3755,8 @@ public:
   ExprResult RebuildObjCEncodeExpr(SourceLocation AtLoc,
                                          TypeSourceInfo *EncodeTypeInfo,
                                          SourceLocation RParenLoc) {
-    return SemaRef.BuildObjCEncodeExpression(AtLoc, EncodeTypeInfo, RParenLoc);
+    return SemaRef.ObjC().BuildObjCEncodeExpression(AtLoc, EncodeTypeInfo,
+                                                    RParenLoc);
   }
 
   /// Build a new Objective-C class message.
@@ -3768,11 +3767,10 @@ public:
                                           SourceLocation LBracLoc,
                                           MultiExprArg Args,
                                           SourceLocation RBracLoc) {
-    return SemaRef.BuildClassMessage(ReceiverTypeInfo,
-                                     ReceiverTypeInfo->getType(),
-                                     /*SuperLoc=*/SourceLocation(),
-                                     Sel, Method, LBracLoc, SelectorLocs,
-                                     RBracLoc, Args);
+    return SemaRef.ObjC().BuildClassMessage(
+        ReceiverTypeInfo, ReceiverTypeInfo->getType(),
+        /*SuperLoc=*/SourceLocation(), Sel, Method, LBracLoc, SelectorLocs,
+        RBracLoc, Args);
   }
 
   /// Build a new Objective-C instance message.
@@ -3783,11 +3781,10 @@ public:
                                           SourceLocation LBracLoc,
                                           MultiExprArg Args,
                                           SourceLocation RBracLoc) {
-    return SemaRef.BuildInstanceMessage(Receiver,
-                                        Receiver->getType(),
-                                        /*SuperLoc=*/SourceLocation(),
-                                        Sel, Method, LBracLoc, SelectorLocs,
-                                        RBracLoc, Args);
+    return SemaRef.ObjC().BuildInstanceMessage(Receiver, Receiver->getType(),
+                                               /*SuperLoc=*/SourceLocation(),
+                                               Sel, Method, LBracLoc,
+                                               SelectorLocs, RBracLoc, Args);
   }
 
   /// Build a new Objective-C instance/class message to 'super'.
@@ -3799,18 +3796,13 @@ public:
                                     SourceLocation LBracLoc,
                                     MultiExprArg Args,
                                     SourceLocation RBracLoc) {
-    return Method->isInstanceMethod() ? SemaRef.BuildInstanceMessage(nullptr,
-                                          SuperType,
-                                          SuperLoc,
-                                          Sel, Method, LBracLoc, SelectorLocs,
-                                          RBracLoc, Args)
-                                      : SemaRef.BuildClassMessage(nullptr,
-                                          SuperType,
-                                          SuperLoc,
-                                          Sel, Method, LBracLoc, SelectorLocs,
-                                          RBracLoc, Args);
-
-
+    return Method->isInstanceMethod()
+               ? SemaRef.ObjC().BuildInstanceMessage(
+                     nullptr, SuperType, SuperLoc, Sel, Method, LBracLoc,
+                     SelectorLocs, RBracLoc, Args)
+               : SemaRef.ObjC().BuildClassMessage(nullptr, SuperType, SuperLoc,
+                                                  Sel, Method, LBracLoc,
+                                                  SelectorLocs, RBracLoc, Args);
   }
 
   /// Build a new Objective-C ivar reference expression.
@@ -15450,9 +15442,9 @@ TransformObjCBridgedCastExpr(ObjCBridgedCastExpr *E) {
       Result.get() == E->getSubExpr())
     return E;
 
-  return SemaRef.BuildObjCBridgedCast(E->getLParenLoc(), E->getBridgeKind(),
-                                      E->getBridgeKeywordLoc(), TSInfo,
-                                      Result.get());
+  return SemaRef.ObjC().BuildObjCBridgedCast(
+      E->getLParenLoc(), E->getBridgeKind(), E->getBridgeKeywordLoc(), TSInfo,
+      Result.get());
 }
 
 template <typename Derived>
@@ -15840,10 +15832,9 @@ QualType TreeTransform<Derived>::RebuildObjCTypeParamType(
            ArrayRef<ObjCProtocolDecl *> Protocols,
            ArrayRef<SourceLocation> ProtocolLocs,
            SourceLocation ProtocolRAngleLoc) {
-  return SemaRef.BuildObjCTypeParamType(Decl,
-                                        ProtocolLAngleLoc, Protocols,
-                                        ProtocolLocs, ProtocolRAngleLoc,
-                                        /*FailOnError=*/true);
+  return SemaRef.ObjC().BuildObjCTypeParamType(
+      Decl, ProtocolLAngleLoc, Protocols, ProtocolLocs, ProtocolRAngleLoc,
+      /*FailOnError=*/true);
 }
 
 template<typename Derived>
@@ -15857,11 +15848,11 @@ QualType TreeTransform<Derived>::RebuildObjCObjectType(
            ArrayRef<ObjCProtocolDecl *> Protocols,
            ArrayRef<SourceLocation> ProtocolLocs,
            SourceLocation ProtocolRAngleLoc) {
-  return SemaRef.BuildObjCObjectType(BaseType, Loc, TypeArgsLAngleLoc, TypeArgs,
-                                     TypeArgsRAngleLoc, ProtocolLAngleLoc,
-                                     Protocols, ProtocolLocs, ProtocolRAngleLoc,
-                                     /*FailOnError=*/true,
-                                     /*Rebuilding=*/true);
+  return SemaRef.ObjC().BuildObjCObjectType(
+      BaseType, Loc, TypeArgsLAngleLoc, TypeArgs, TypeArgsRAngleLoc,
+      ProtocolLAngleLoc, Protocols, ProtocolLocs, ProtocolRAngleLoc,
+      /*FailOnError=*/true,
+      /*Rebuilding=*/true);
 }
 
 template<typename Derived>
diff --git a/clang/lib/Serialization/ASTCommon.cpp b/clang/lib/Serialization/ASTCommon.cpp
index 63c5140086d8..bc662a87a7bf 100644
--- a/clang/lib/Serialization/ASTCommon.cpp
+++ b/clang/lib/Serialization/ASTCommon.cpp
@@ -341,7 +341,7 @@ serialization::getDefinitiveDeclContext(const DeclContext *DC) {
 
   // FIXME: These are defined in one place, but properties in class extensions
   // end up being back-patched into the main interface. See
-  // Sema::HandlePropertyInClassExtension for the offending code.
+  // SemaObjC::HandlePropertyInClassExtension for the offending code.
   case Decl::ObjCInterface:
     return nullptr;
 
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 8f437a7c5f50..510f61d9cccd 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -78,6 +78,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaCUDA.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/Weak.h"
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ASTDeserializationListener.h"
@@ -4233,9 +4234,9 @@ ASTReader::ReadModuleMapFileBlock(RecordData &Record, ModuleFile &F,
 /// Move the given method to the back of the global list of methods.
 static void moveMethodToBackOfGlobalList(Sema &S, ObjCMethodDecl *Method) {
   // Find the entry for this selector in the method pool.
-  Sema::GlobalMethodPool::iterator Known
-    = S.MethodPool.find(Method->getSelector());
-  if (Known == S.MethodPool.end())
+  SemaObjC::GlobalMethodPool::iterator Known =
+      S.ObjC().MethodPool.find(Method->getSelector());
+  if (Known == S.ObjC().MethodPool.end())
     return;
 
   // Retrieve the appropriate method list.
@@ -8551,7 +8552,7 @@ namespace serialization {
 static void addMethodsToPool(Sema &S, ArrayRef<ObjCMethodDecl *> Methods,
                              ObjCMethodList &List) {
   for (ObjCMethodDecl *M : llvm::reverse(Methods))
-    S.addMethodToGlobalList(&List, M);
+    S.ObjC().addMethodToGlobalList(&List, M);
 }
 
 void ASTReader::ReadMethodPool(Selector Sel) {
@@ -8576,8 +8577,10 @@ void ASTReader::ReadMethodPool(Selector Sel) {
     return;
 
   Sema &S = *getSema();
-  Sema::GlobalMethodPool::iterator Pos =
-      S.MethodPool.insert(std::make_pair(Sel, Sema::GlobalMethodPool::Lists()))
+  SemaObjC::GlobalMethodPool::iterator Pos =
+      S.ObjC()
+          .MethodPool
+          .insert(std::make_pair(Sel, SemaObjC::GlobalMethodPool::Lists()))
           .first;
 
   Pos->second.first.setBits(Visitor.getInstanceBits());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 7a9d392889bb..129bc337c892 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -66,6 +66,7 @@
 #include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaCUDA.h"
+#include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/Weak.h"
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ASTReader.h"
@@ -3547,7 +3548,7 @@ void ASTWriter::WriteSelectors(Sema &SemaRef) {
   using namespace llvm;
 
   // Do we have to do anything at all?
-  if (SemaRef.MethodPool.empty() && SelectorIDs.empty())
+  if (SemaRef.ObjC().MethodPool.empty() && SelectorIDs.empty())
     return;
   unsigned NumTableEntries = 0;
   // Create and write out the blob that contains selectors and the method pool.
@@ -3561,13 +3562,14 @@ void ASTWriter::WriteSelectors(Sema &SemaRef) {
     for (auto &SelectorAndID : SelectorIDs) {
       Selector S = SelectorAndID.first;
       SelectorID ID = SelectorAndID.second;
-      Sema::GlobalMethodPool::iterator F = SemaRef.MethodPool.find(S);
+      SemaObjC::GlobalMethodPool::iterator F =
+          SemaRef.ObjC().MethodPool.find(S);
       ASTMethodPoolTrait::data_type Data = {
         ID,
         ObjCMethodList(),
         ObjCMethodList()
       };
-      if (F != SemaRef.MethodPool.end()) {
+      if (F != SemaRef.ObjC().MethodPool.end()) {
         Data.Instance = F->second.first;
         Data.Factory = F->second.second;
       }
@@ -3652,7 +3654,7 @@ void ASTWriter::WriteSelectors(Sema &SemaRef) {
 void ASTWriter::WriteReferencedSelectorsPool(Sema &SemaRef) {
   using namespace llvm;
 
-  if (SemaRef.ReferencedSelectors.empty())
+  if (SemaRef.ObjC().ReferencedSelectors.empty())
     return;
 
   RecordData Record;
@@ -3661,7 +3663,7 @@ void ASTWriter::WriteReferencedSelectorsPool(Sema &SemaRef) {
   // Note: this writes out all references even for a dependent AST. But it is
   // very tricky to fix, and given that @selector shouldn't really appear in
   // headers, probably not worth it. It's not a correctness issue.
-  for (auto &SelectorAndLocation : SemaRef.ReferencedSelectors) {
+  for (auto &SelectorAndLocation : SemaRef.ObjC().ReferencedSelectors) {
     Selector Sel = SelectorAndLocation.first;
     SourceLocation Loc = SelectorAndLocation.second;
     Writer.AddSelectorRef(Sel);
@@ -5346,7 +5348,7 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
   for (auto &SelectorAndID : SelectorIDs)
     AllSelectors.push_back(SelectorAndID.first);
   for (auto &Selector : AllSelectors)
-    SemaRef.updateOutOfDateSelector(Selector);
+    SemaRef.ObjC().updateOutOfDateSelector(Selector);
 
   // Form the record of special types.
   RecordData SpecialTypes;
-- 
GitLab


From bb867b59b86ec48bb8a4e399cbb7ee99c7981d32 Mon Sep 17 00:00:00 2001
From: Daniel Chen <cdchen@ca.ibm.com>
Date: Mon, 13 May 2024 15:39:35 -0400
Subject: [PATCH 0628/1206] [MLIR] NFC: change variable name to lowercase to
 follow mlir convention. (#91974)

---
 mlir/tools/mlir-src-sharder/mlir-src-sharder.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/tools/mlir-src-sharder/mlir-src-sharder.cpp b/mlir/tools/mlir-src-sharder/mlir-src-sharder.cpp
index 5bfc24ef3b47..b15f2ce54405 100644
--- a/mlir/tools/mlir-src-sharder/mlir-src-sharder.cpp
+++ b/mlir/tools/mlir-src-sharder/mlir-src-sharder.cpp
@@ -66,7 +66,7 @@ int main(int argc, char **argv) {
   // of `llvm-tblgen`, which caused `TestOps.cpp` to fail due to
   // "Unknnown command line argument '-D...`" when a macros name is
   // present. The following is a workaround to re-register it again.
-  llvm::cl::list<std::string> MacroNames(
+  llvm::cl::list<std::string> macroNames(
       "D",
       llvm::cl::desc(
           "Name of the macro to be defined -- ignored by mlir-src-sharder"),
-- 
GitLab


From e1223804458a99f82dc110d41647410bdac420df Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 13 May 2024 20:30:09 +0100
Subject: [PATCH 0629/1206] [LV] Use VPBuilder to create Select (NFCI).

---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 261933966b74..8b4cb5a6658d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8377,10 +8377,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
       VPValue *Mask = getBlockInMask(I->getParent());
       VPValue *One =
           Plan.getOrAddLiveIn(ConstantInt::get(I->getType(), 1u, false));
-      auto *SafeRHS =
-         new VPInstruction(Instruction::Select, {Mask, Ops[1], One},
-                           I->getDebugLoc());
-      VPBB->appendRecipe(SafeRHS);
+      auto *SafeRHS = Builder.createSelect(Mask, Ops[1], One, I->getDebugLoc());
       Ops[1] = SafeRHS;
       return new VPWidenRecipe(*I, make_range(Ops.begin(), Ops.end()));
     }
-- 
GitLab


From a4b3422536c712dcb198223822424ff15fb9e45b Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 13 May 2024 19:45:57 +0000
Subject: [PATCH 0630/1206] [gn build] Port 31a203fa8af4

---
 llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
index e9ba5fb132b0..f6c9526278dd 100644
--- a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
@@ -78,6 +78,7 @@ static_library("Sema") {
     "SemaLambda.cpp",
     "SemaLookup.cpp",
     "SemaModule.cpp",
+    "SemaObjC.cpp",
     "SemaObjCProperty.cpp",
     "SemaOpenACC.cpp",
     "SemaOpenMP.cpp",
-- 
GitLab


From a6d7828f4c50c1ec7b0b5f61fe59d7a768175dcc Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 13 May 2024 12:53:16 -0700
Subject: [PATCH 0631/1206] [test] Use conventional -emit-llvm-only

This avoids 'Permission denied' when PWD is read-only.
While here, change the triple from a Linux one to a generic ELF one.
---
 clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c   | 4 ++--
 .../Sema/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c
index b1582569971d..66aba36ff68a 100644
--- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_fmlas16.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -verify -emit-llvm %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -verify -emit-llvm-only %s
 
 // REQUIRES: aarch64-registered-target
 
@@ -87,4 +87,4 @@ void test_imm(uint32_t slice, svfloat16_t zm, svfloat16x2_t zn2,svfloat16x4_t zn
   svmls_lane_za16_bf16_vg1x2(slice, bzn2, bzm, -1);
   // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
   svmls_lane_za16_bf16_vg1x4(slice, bzn4, bzm, -1);
-}
\ No newline at end of file
+}
diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c
index 201ad4b8ff7f..1331bf2050b7 100644
--- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_mopa_nonwide.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -verify -emit-llvm %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -verify -emit-llvm-only %s
 
 // REQUIRES: aarch64-registered-target
 
-- 
GitLab


From c3028a230557405b0f10bdd7d450f7f92747bbe3 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 13 May 2024 12:33:30 -0700
Subject: [PATCH 0632/1206] [RISCV] Don't exlude the frame pointer from the
 callee saved registers in RISCVRegisterInfo::needsFrameBaseReg.

Instead of using getReservedRegs, just check the subtarget reserved
list. getReservedRegs considers the frame pointer to be reserved when
it is being used, but we do need to save/restore it so it should be
counted as a callee saved register. AArch64 hardcodes their callee
saved size, but the comment mentions the Frame Pointer being counted.
---
 llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp   |  4 +-
 .../RISCV/local-stack-slot-allocation.ll      | 47 +++++++++++++++++++
 2 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index c3281e409653..7ac73b59f36c 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -609,13 +609,13 @@ bool RISCVRegisterInfo::needsFrameBaseReg(MachineInstr *MI,
   const MachineRegisterInfo &MRI = MF.getRegInfo();
 
   if (TFI->hasFP(MF) && !shouldRealignStack(MF)) {
+    auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
     // Estimate the stack size used to store callee saved registers(
     // excludes reserved registers).
     unsigned CalleeSavedSize = 0;
-    BitVector ReservedRegs = getReservedRegs(MF);
     for (const MCPhysReg *R = MRI.getCalleeSavedRegs(); MCPhysReg Reg = *R;
          ++R) {
-      if (!ReservedRegs.test(Reg))
+      if (!Subtarget.isRegisterReservedByUser(Reg))
         CalleeSavedSize += getSpillSize(*getMinimalPhysRegClass(Reg));
     }
 
diff --git a/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll b/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
index 18e7992f30a3..1d5487e19e89 100644
--- a/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
+++ b/llvm/test/CodeGen/RISCV/local-stack-slot-allocation.ll
@@ -111,3 +111,50 @@ define void @load_with_offset2() {
   store volatile i8 %load, ptr %va_gep, align 4
   ret void
 }
+
+define void @frame_pointer() "frame-pointer"="all" {
+; RV32I-LABEL: frame_pointer:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -2032
+; RV32I-NEXT:    .cfi_def_cfa_offset 2032
+; RV32I-NEXT:    sw ra, 2028(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 2024(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset ra, -4
+; RV32I-NEXT:    .cfi_offset s0, -8
+; RV32I-NEXT:    addi s0, sp, 2032
+; RV32I-NEXT:    .cfi_def_cfa s0, 0
+; RV32I-NEXT:    addi sp, sp, -480
+; RV32I-NEXT:    lbu a0, -1960(s0)
+; RV32I-NEXT:    sb a0, -1960(s0)
+; RV32I-NEXT:    addi sp, sp, 480
+; RV32I-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 2032
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: frame_pointer:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -2032
+; RV64I-NEXT:    .cfi_def_cfa_offset 2032
+; RV64I-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 2016(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset ra, -8
+; RV64I-NEXT:    .cfi_offset s0, -16
+; RV64I-NEXT:    addi s0, sp, 2032
+; RV64I-NEXT:    .cfi_def_cfa s0, 0
+; RV64I-NEXT:    addi sp, sp, -496
+; RV64I-NEXT:    addi a0, s0, -1972
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    sb a1, 0(a0)
+; RV64I-NEXT:    addi sp, sp, 496
+; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 2032
+; RV64I-NEXT:    ret
+
+  %va = alloca [2500 x i8], align 4
+  %va_gep = getelementptr [2000 x i8], ptr %va, i64 0, i64 552
+  %load = load volatile i8, ptr %va_gep, align 4
+  store volatile i8 %load, ptr %va_gep, align 4
+  ret void
+}
-- 
GitLab


From 55e59083cb610f30ad21fe8c8cdb9900534937ec Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 13 May 2024 13:26:17 -0700
Subject: [PATCH 0633/1206] [RISCV] Exclude vector callee saved registers from
 RISCVRegisterInfo::needsFrameBaseReg

The vector callee saved registers shouldn't affect the frame pointer
offset so we don't want to consider them.

I've listed the GPR, FPR32, and FPR64 register classes explicitly
because getMinimalPhysRegClass is slow and this function is called
frequently. So explicitly listing the interesting classs should be
a compile time improvement.
---
 llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp   | 12 +++++--
 .../CodeGen/RISCV/rvv/callee-saved-regs.ll    | 31 +++++++++++++++++++
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index 7ac73b59f36c..129b4cb4e8cb 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -615,8 +615,16 @@ bool RISCVRegisterInfo::needsFrameBaseReg(MachineInstr *MI,
     unsigned CalleeSavedSize = 0;
     for (const MCPhysReg *R = MRI.getCalleeSavedRegs(); MCPhysReg Reg = *R;
          ++R) {
-      if (!Subtarget.isRegisterReservedByUser(Reg))
-        CalleeSavedSize += getSpillSize(*getMinimalPhysRegClass(Reg));
+      if (Subtarget.isRegisterReservedByUser(Reg))
+        continue;
+
+      if (RISCV::GPRRegClass.contains(Reg))
+        CalleeSavedSize += getSpillSize(RISCV::GPRRegClass);
+      else if (RISCV::FPR64RegClass.contains(Reg))
+        CalleeSavedSize += getSpillSize(RISCV::FPR64RegClass);
+      else if (RISCV::FPR32RegClass.contains(Reg))
+        CalleeSavedSize += getSpillSize(RISCV::FPR32RegClass);
+      // Ignore vector registers.
     }
 
     int64_t MaxFPOffset = Offset - CalleeSavedSize;
diff --git a/llvm/test/CodeGen/RISCV/rvv/callee-saved-regs.ll b/llvm/test/CodeGen/RISCV/rvv/callee-saved-regs.ll
index 84936d88e187..2177bbfe5b2a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/callee-saved-regs.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/callee-saved-regs.ll
@@ -93,3 +93,34 @@ entry:
 
   ret <vscale x 1 x i32> %va
 }
+
+; Make sure the local stack allocation pass doesn't count vector registers. The
+; sizes are chosen to be on the edge of what RISCVRegister::needsFrameBaseReg
+; considers to need a virtual base register.
+define riscv_vector_cc void @local_stack_allocation_frame_pointer() "frame-pointer"="all" {
+; SPILL-O2-LABEL: local_stack_allocation_frame_pointer:
+; SPILL-O2:       # %bb.0:
+; SPILL-O2-NEXT:    addi sp, sp, -2032
+; SPILL-O2-NEXT:    .cfi_def_cfa_offset 2032
+; SPILL-O2-NEXT:    sw ra, 2028(sp) # 4-byte Folded Spill
+; SPILL-O2-NEXT:    sw s0, 2024(sp) # 4-byte Folded Spill
+; SPILL-O2-NEXT:    .cfi_offset ra, -4
+; SPILL-O2-NEXT:    .cfi_offset s0, -8
+; SPILL-O2-NEXT:    addi s0, sp, 2032
+; SPILL-O2-NEXT:    .cfi_def_cfa s0, 0
+; SPILL-O2-NEXT:    addi sp, sp, -480
+; SPILL-O2-NEXT:    lbu a0, -1912(s0)
+; SPILL-O2-NEXT:    sb a0, -1912(s0)
+; SPILL-O2-NEXT:    addi sp, s0, -2048
+; SPILL-O2-NEXT:    addi sp, sp, -464
+; SPILL-O2-NEXT:    addi sp, sp, 480
+; SPILL-O2-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
+; SPILL-O2-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
+; SPILL-O2-NEXT:    addi sp, sp, 2032
+; SPILL-O2-NEXT:    ret
+  %va = alloca [2500 x i8], align 4
+  %va_gep = getelementptr [2000 x i8], ptr %va, i64 0, i64 600
+  %load = load volatile i8, ptr %va_gep, align 4
+  store volatile i8 %load, ptr %va_gep, align 4
+  ret void
+}
-- 
GitLab


From 34de2151e2328db800bcd226f31cb6b0cdcf08bb Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 13 May 2024 21:58:41 +0100
Subject: [PATCH 0634/1206] [AArch64][GlobalISel] Improve legalization of
 G_PTR_ADD (#91763)

The testing we have for vector ptradd was a bit lacking. In adding tests
this patch found a couple of issues mostly with the way v3 vectors of
ptrs were sometimes legalized via i64, and with non-i64 additions. It
does not attempt to fix the issue with mergevalues from returning vector
ptrs.
---
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |   7 +-
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   5 +-
 llvm/test/CodeGen/AArch64/ptradd.ll           | 221 ++++++++++++++++++
 3 files changed, 227 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/ptradd.ll

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 6a76ad7f5db7..40507845d8d8 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -69,8 +69,9 @@ getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
     unsigned EltSize = OrigTy.getScalarSizeInBits();
     if (LeftoverSize % EltSize != 0)
       return {-1, -1};
-    LeftoverTy = LLT::scalarOrVector(
-        ElementCount::getFixed(LeftoverSize / EltSize), EltSize);
+    LeftoverTy =
+        LLT::scalarOrVector(ElementCount::getFixed(LeftoverSize / EltSize),
+                            OrigTy.getElementType());
   } else {
     LeftoverTy = LLT::scalar(LeftoverSize);
   }
@@ -212,7 +213,7 @@ void LegalizerHelper::mergeMixedSubvectors(Register DstReg,
     appendVectorElts(AllElts, PartRegs[i]);
 
   Register Leftover = PartRegs[PartRegs.size() - 1];
-  if (MRI.getType(Leftover).isScalar())
+  if (!MRI.getType(Leftover).isVector())
     AllElts.push_back(Leftover);
   else
     appendVectorElts(AllElts, Leftover);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index b8274f0f872c..a21be7de6f42 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -177,9 +177,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
 
   getActionDefinitionsBuilder(G_PTR_ADD)
       .legalFor({{p0, s64}, {v2p0, v2s64}})
-      .clampScalar(1, s64, s64)
-      .clampNumElements(0, v2p0, v2p0)
-      .clampNumElements(1, v2s64, v2s64);
+      .clampScalarOrElt(1, s64, s64)
+      .clampNumElements(0, v2p0, v2p0);
 
   getActionDefinitionsBuilder(G_PTRMASK).legalFor({{p0, s64}});
 
diff --git a/llvm/test/CodeGen/AArch64/ptradd.ll b/llvm/test/CodeGen/AArch64/ptradd.ll
new file mode 100644
index 000000000000..107db8723c64
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ptradd.ll
@@ -0,0 +1,221 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-none-eabi | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple=aarch64-none-eabi -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; Note: these tests use stores instead of returns as the return handling for
+; vector ptrs is currently sometimes create invalid unmerge values.
+
+define void @vector_gep_i32(ptr %b, i32 %off, ptr %p) {
+; CHECK-LABEL: vector_gep_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add x8, x0, w1, sxtw
+; CHECK-NEXT:    str x8, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %g = getelementptr i8, ptr %b, i32 %off
+  store ptr %g, ptr %p
+  ret void
+}
+
+define void @vector_gep_i64(ptr %b, i64 %off, ptr %p) {
+; CHECK-LABEL: vector_gep_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add x8, x0, x1
+; CHECK-NEXT:    str x8, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %g = getelementptr i8, ptr %b, i64 %off
+  store ptr %g, ptr %p
+  ret void
+}
+
+define void @vector_gep_v1i32(<1 x ptr> %b, <1 x i32> %off, ptr %p) {
+; CHECK-SD-LABEL: vector_gep_v1i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl d1, d1, #32
+; CHECK-SD-NEXT:    ssra d0, d1, #32
+; CHECK-SD-NEXT:    str d0, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: vector_gep_v1i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    add x8, x9, w8, sxtw
+; CHECK-GI-NEXT:    str x8, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %g = getelementptr i8, <1 x ptr> %b, <1 x i32> %off
+  store <1 x ptr> %g, ptr %p
+  ret void
+}
+
+define void @vector_gep_v2i32(<2 x ptr> %b, <2 x i32> %off, ptr %p) {
+; CHECK-LABEL: vector_gep_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    saddw v0.2d, v0.2d, v1.2s
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %g = getelementptr i8, <2 x ptr> %b, <2 x i32> %off
+  store <2 x ptr> %g, ptr %p
+  ret void
+}
+
+define void @vector_gep_v3i32(<3 x ptr> %b, <3 x i32> %off, ptr %p) {
+; CHECK-SD-LABEL: vector_gep_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    saddw2 v2.2d, v2.2d, v3.4s
+; CHECK-SD-NEXT:    str d2, [x0, #16]
+; CHECK-SD-NEXT:    saddw v0.2d, v0.2d, v3.2s
+; CHECK-SD-NEXT:    str q0, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: vector_gep_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    smov x8, v3.s[0]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    smov x9, v3.s[1]
+; CHECK-GI-NEXT:    mov s3, v3.s[2]
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    fmov d1, x8
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    mov v1.d[1], x9
+; CHECK-GI-NEXT:    fmov w9, s3
+; CHECK-GI-NEXT:    add x8, x8, w9, sxtw
+; CHECK-GI-NEXT:    add v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    str x8, [x0, #16]
+; CHECK-GI-NEXT:    str q0, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %g = getelementptr i8, <3 x ptr> %b, <3 x i32> %off
+  store <3 x ptr> %g, ptr %p
+  ret void
+}
+
+define void @vector_gep_v4i32(<4 x ptr> %b, <4 x i32> %off, ptr %p) {
+; CHECK-SD-LABEL: vector_gep_v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    saddw2 v1.2d, v1.2d, v2.4s
+; CHECK-SD-NEXT:    saddw v0.2d, v0.2d, v2.2s
+; CHECK-SD-NEXT:    stp q0, q1, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: vector_gep_v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    saddw v0.2d, v0.2d, v2.2s
+; CHECK-GI-NEXT:    saddw2 v1.2d, v1.2d, v2.4s
+; CHECK-GI-NEXT:    stp q0, q1, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %g = getelementptr i8, <4 x ptr> %b, <4 x i32> %off
+  store <4 x ptr> %g, ptr %p
+  ret void
+}
+
+define void @vector_gep_v1i64(<1 x ptr> %b, <1 x i64> %off, ptr %p) {
+; CHECK-SD-LABEL: vector_gep_v1i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    add d0, d0, d1
+; CHECK-SD-NEXT:    str d0, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: vector_gep_v1i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    add x8, x8, x9
+; CHECK-GI-NEXT:    str x8, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %g = getelementptr i8, <1 x ptr> %b, <1 x i64> %off
+  store <1 x ptr> %g, ptr %p
+  ret void
+}
+
+define void @vector_gep_v2i64(<2 x ptr> %b, <2 x i64> %off, ptr %p) {
+; CHECK-LABEL: vector_gep_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %g = getelementptr i8, <2 x ptr> %b, <2 x i64> %off
+  store <2 x ptr> %g, ptr %p
+  ret void
+}
+
+define void @vector_gep_v3i64(<3 x ptr> %b, <3 x i64> %off, ptr %p) {
+; CHECK-SD-LABEL: vector_gep_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    add d1, d2, d5
+; CHECK-SD-NEXT:    str d1, [x0, #16]
+; CHECK-SD-NEXT:    add v0.2d, v0.2d, v3.2d
+; CHECK-SD-NEXT:    str q0, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: vector_gep_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    fmov x9, d5
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-GI-NEXT:    add x8, x8, x9
+; CHECK-GI-NEXT:    str x8, [x0, #16]
+; CHECK-GI-NEXT:    add v0.2d, v0.2d, v3.2d
+; CHECK-GI-NEXT:    str q0, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %g = getelementptr i8, <3 x ptr> %b, <3 x i64> %off
+  store <3 x ptr> %g, ptr %p
+  ret void
+}
+
+define void @vector_gep_v4i64(<4 x ptr> %b, <4 x i64> %off, ptr %p) {
+; CHECK-SD-LABEL: vector_gep_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    add v1.2d, v1.2d, v3.2d
+; CHECK-SD-NEXT:    add v0.2d, v0.2d, v2.2d
+; CHECK-SD-NEXT:    stp q0, q1, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: vector_gep_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    add v0.2d, v0.2d, v2.2d
+; CHECK-GI-NEXT:    add v1.2d, v1.2d, v3.2d
+; CHECK-GI-NEXT:    stp q0, q1, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %g = getelementptr i8, <4 x ptr> %b, <4 x i64> %off
+  store <4 x ptr> %g, ptr %p
+  ret void
+}
+
+define void @vector_gep_v4i128(<2 x ptr> %b, <2 x i128> %off, ptr %p) {
+; CHECK-LABEL: vector_gep_v4i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov d1, x0
+; CHECK-NEXT:    mov v1.d[1], x2
+; CHECK-NEXT:    add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    str q0, [x4]
+; CHECK-NEXT:    ret
+entry:
+  %g = getelementptr i8, <2 x ptr> %b, <2 x i128> %off
+  store <2 x ptr> %g, ptr %p
+  ret void
+}
-- 
GitLab


From ef9090fcb5b8d1c9f56c11d567987ffa1000a486 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 13 May 2024 14:01:00 -0700
Subject: [PATCH 0635/1206] [test] Fix check prefixes

---
 .../AArch64/vec3-loads-ext-trunc-stores.ll    | 36 -------------------
 .../test/CodeGen/NVPTX/global-variable-big.ll |  5 +--
 .../MemorySanitizer/X86/msan_x86_bts_asm.ll   |  5 ---
 .../first-order-recurrence-complex.ll         |  4 ---
 4 files changed, 1 insertion(+), 49 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
index 71d55df66517..66c884e95fa4 100644
--- a/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
+++ b/llvm/test/CodeGen/AArch64/vec3-loads-ext-trunc-stores.ll
@@ -217,42 +217,6 @@ define <4 x i32> @load_v3i8_to_4xi32_const_offset_3(ptr %src) {
 }
 
 define <4 x i32> @volatile_load_v3i8_to_4xi32(ptr %src) {
-; check-label: volatile_load_v3i8_to_4xi32:
-; check:       ; %bb.0:
-; check-next:    sub sp, sp, #16
-; check-next:    .cfi_def_cfa_offset 16
-; check-next:    ldrh w8, [x0]
-; check-next:    movi.2d v1, #0x0000ff000000ff
-; check-next:    strh w8, [sp, #12]
-; check-next:    ldr s0, [sp, #12]
-; check-next:    ldrsb w8, [x0, #2]
-; check-next:    ushll.8h v0, v0, #0
-; check-next:    mov.h v0[1], v0[1]
-; check-next:    mov.h v0[2], w8
-; check-next:    ushll.4s v0, v0, #0
-; check-next:    and.16b v0, v0, v1
-; check-next:    add sp, sp, #16
-; check-next:    ret
-;
-; be-label: volatile_load_v3i8_to_4xi32:
-; be:       // %bb.0:
-; be-next:    sub sp, sp, #16
-; be-next:    .cfi_def_cfa_offset 16
-; be-next:    ldrh w8, [x0]
-; be-next:    movi v1.2d, #0x0000ff000000ff
-; be-next:    strh w8, [sp, #12]
-; be-next:    ldr s0, [sp, #12]
-; be-next:    ldrsb w8, [x0, #2]
-; be-next:    rev32 v0.8b, v0.8b
-; be-next:    ushll v0.8h, v0.8b, #0
-; be-next:    mov v0.h[1], v0.h[1]
-; be-next:    mov v0.h[2], w8
-; be-next:    ushll v0.4s, v0.4h, #0
-; be-next:    and v0.16b, v0.16b, v1.16b
-; be-next:    rev64 v0.4s, v0.4s
-; be-next:    ext v0.16b, v0.16b, v0.16b, #8
-; be-next:    add sp, sp, #16
-; be-next:    ret
 ; CHECK-LABEL: volatile_load_v3i8_to_4xi32:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #16
diff --git a/llvm/test/CodeGen/NVPTX/global-variable-big.ll b/llvm/test/CodeGen/NVPTX/global-variable-big.ll
index f4194df4434d..e8d7fb3815b7 100644
--- a/llvm/test/CodeGen/NVPTX/global-variable-big.ll
+++ b/llvm/test/CodeGen/NVPTX/global-variable-big.ll
@@ -11,8 +11,5 @@ target triple = "nvptx64-nvidia-cuda"
 ; CHECK: .visible .global .align 16 .b8 gv[16] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
 
 ; Make sure that we do not overflow on large number of elements.
-; CHECK-LABEL large_data
-; CHECK: .visible .global .align 1 .b8 large_data[4831838208]
+; CHECK: .visible .global .align 1 .b8 large_data[4831838208];
 @large_data = global [4831838208 x i8] zeroinitializer
-
-
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/msan_x86_bts_asm.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/msan_x86_bts_asm.ll
index 5eeba197c763..dd2fecb081be 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/msan_x86_bts_asm.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/msan_x86_bts_asm.ll
@@ -73,11 +73,6 @@ if.else:                                          ; preds = %entry
 
 ; CHECK: call void asm "btsq $2, $1; setc $0"
 
-; Calculating the shadow offset of %bit.
-; CHECKz: [[PTR:%.*]] = ptrtoint {{.*}} %bit to i64
-; CHECKz: [[SH_NUM:%.*]] = xor i64 [[PTR]]
-; CHECKz: [[SHADOW:%.*]] = inttoptr i64 [[SH_NUM]] {{.*}}
-
 ; CHECK: [[META:%.*]] = call {{.*}} @__msan_metadata_ptr_for_load_1(ptr %bit)
 ; CHECK: [[SHADOW:%.*]] = extractvalue { ptr, ptr } [[META]], 0
 
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
index 95671078a776..149e4705885b 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-complex.ll
@@ -332,10 +332,6 @@ define void @cannot_sink_reduction(i32 %x, ptr %ptr, i64 %tc) {
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
-
-
-
-; CHECK-NET:     ret void
 entry:
   br label %preheader
 
-- 
GitLab


From 70e227a404e51f9248c7ad5d79953805b2afacb4 Mon Sep 17 00:00:00 2001
From: Aart Bik <ajcbik@google.com>
Date: Mon, 13 May 2024 14:02:29 -0700
Subject: [PATCH 0636/1206] [mlir][sparse] recognize ReLu operation during
 sparsification (#92016)

This is a proof of concept recognition of the most basic forms of ReLu
operations, used to show-case sparsification of end-to-end PyTorch
models. In the long run, we must avoid lowering such constructs too
early (with this need for raising them back).

See discussion at

https://discourse.llvm.org/t/min-max-abs-relu-recognition-starter-project/78918
---
 .../mlir/Dialect/SparseTensor/Utils/Merger.h  |   3 +-
 .../lib/Dialect/SparseTensor/Utils/Merger.cpp | 100 ++++++++++++++++--
 .../Dialect/SparseTensor/sparse_relu.mlir     |  34 ++++++
 .../Dialect/SparseTensor/MergerTest.cpp       |   1 +
 4 files changed, 127 insertions(+), 11 deletions(-)
 create mode 100644 mlir/test/Dialect/SparseTensor/sparse_relu.mlir

diff --git a/mlir/include/mlir/Dialect/SparseTensor/Utils/Merger.h b/mlir/include/mlir/Dialect/SparseTensor/Utils/Merger.h
index 7f9820df984b..b8d278152dc0 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/Utils/Merger.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/Utils/Merger.h
@@ -144,6 +144,7 @@ enum class TensorExp::Kind {
   kExpm1C,
   kLog1pF,
   kLog1pC,
+  kRelu,
   kSinF,
   kSinC,
   kTanhF,
@@ -316,7 +317,7 @@ public:
   /// lattice point on an expression E is simply copied over, but with OP E
   /// as new expression. Returns the identifier of the new set.
   LatSetId mapSet(TensorExp::Kind kind, LatSetId s, Value v = Value(),
-                  Operation *op = nullptr);
+                  Operation *op = nullptr, Attribute attr = nullptr);
 
   /// Maps the binary operator to the same operation but with one of its operand
   /// set to zero, i.e. each lattice point on an expression E is simply copied
diff --git a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
index 308fbd965259..0258f797143c 100644
--- a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp
@@ -44,6 +44,7 @@ static ExpArity getExpArity(TensorExp::Kind k) {
   case TensorExp::Kind::kExpm1C:
   case TensorExp::Kind::kLog1pF:
   case TensorExp::Kind::kLog1pC:
+  case TensorExp::Kind::kRelu:
   case TensorExp::Kind::kSinF:
   case TensorExp::Kind::kSinC:
   case TensorExp::Kind::kTanhF:
@@ -104,7 +105,7 @@ static ExpArity getExpArity(TensorExp::Kind k) {
 
 TensorExp::TensorExp(TensorExp::Kind k, unsigned x, ExprId y, Value v,
                      Operation *o, Attribute a)
-    : kind(k), val(v), op(o) {
+    : kind(k), val(v), op(o), attr(a) {
   switch (kind) {
   // Leaf.
   case TensorExp::Kind::kTensor:
@@ -133,6 +134,7 @@ TensorExp::TensorExp(TensorExp::Kind k, unsigned x, ExprId y, Value v,
   case TensorExp::Kind::kExpm1C:
   case TensorExp::Kind::kLog1pF:
   case TensorExp::Kind::kLog1pC:
+  case TensorExp::Kind::kRelu:
   case TensorExp::Kind::kSinF:
   case TensorExp::Kind::kSinC:
   case TensorExp::Kind::kTanhF:
@@ -201,7 +203,6 @@ TensorExp::TensorExp(TensorExp::Kind k, unsigned x, ExprId y, Value v,
   case TensorExp::Kind::kCmpF:
   case TensorExp::Kind::kCmpI:
     assert(x != detail::kInvalidId && y != detail::kInvalidId && !v && !o);
-    attr = a;
     children.e0 = x;
     children.e1 = y;
     return;
@@ -337,7 +338,6 @@ LatSetId Merger::conjSet(ExprId e, LatSetId s0, LatSetId s1, Operation *op) {
 LatSetId Merger::disjSet(ExprId e, LatSetId s0, LatSetId s1, Operation *op) {
   const LatSetId sNew = conjSet(e, s0, s1, op);
   TensorExp::Kind kind = exp(e).kind;
-
   // Followed by all in s0.
   latSets[sNew].append(latSets[s0]);
   // Map binary 0-y to unary -y.
@@ -381,31 +381,32 @@ LatSetId Merger::combiSet(ExprId e, LatSetId s0, LatSetId s1, Operation *orig,
                           bool includeLeft, TensorExp::Kind ltrans,
                           Operation *opleft, bool includeRight,
                           TensorExp::Kind rtrans, Operation *opright) {
+  Attribute a = exp(e).attr;
   const LatSetId sNew = conjSet(e, s0, s1, orig);
   // Left Region.
   if (includeLeft) {
     if (opleft)
-      s0 = mapSet(ltrans, s0, Value(), opleft);
+      s0 = mapSet(ltrans, s0, Value(), opleft, a);
     latSets[sNew].append(latSets[s0]);
   }
   // Right Region.
   if (includeRight) {
     if (opright)
-      s1 = mapSet(rtrans, s1, Value(), opright);
+      s1 = mapSet(rtrans, s1, Value(), opright, a);
     latSets[sNew].append(latSets[s1]);
   }
   return sNew;
 }
 
 LatSetId Merger::mapSet(TensorExp::Kind kind, LatSetId s0, Value v,
-                        Operation *op) {
+                        Operation *op, Attribute a) {
   assert((TensorExp::Kind::kAbsF <= kind && kind <= TensorExp::Kind::kSelect) ||
          TensorExp::Kind::kDenseOp == kind);
   const LatSetId sNew = addSet();
   auto &setNew = latSets[sNew];
   for (const LatPointId p : set(s0)) {
     const auto &point = latPoints[p];
-    setNew.push_back(addLat(point.bits, addExp(kind, point.exp, v, op)));
+    setNew.push_back(addLat(point.bits, addExp(kind, point.exp, v, op, a)));
   }
   return sNew;
 }
@@ -596,6 +597,7 @@ bool Merger::isSingleCondition(TensorId t, ExprId e) const {
   case TensorExp::Kind::kExpm1C:
   case TensorExp::Kind::kLog1pF:
   case TensorExp::Kind::kLog1pC:
+  case TensorExp::Kind::kRelu:
   case TensorExp::Kind::kSinF:
   case TensorExp::Kind::kSinC:
   case TensorExp::Kind::kTanhF:
@@ -717,6 +719,8 @@ static const char *kindToOpSymbol(TensorExp::Kind kind) {
   case TensorExp::Kind::kLog1pF:
   case TensorExp::Kind::kLog1pC:
     return "log1p";
+  case TensorExp::Kind::kRelu:
+    return "relu";
   case TensorExp::Kind::kSinF:
   case TensorExp::Kind::kSinC:
     return "sin";
@@ -824,6 +828,7 @@ void Merger::dumpExp(ExprId e) const {
   case TensorExp::Kind::kExpm1C:
   case TensorExp::Kind::kLog1pF:
   case TensorExp::Kind::kLog1pC:
+  case TensorExp::Kind::kRelu:
   case TensorExp::Kind::kSinF:
   case TensorExp::Kind::kSinC:
   case TensorExp::Kind::kTanhF:
@@ -972,6 +977,7 @@ LatSetId Merger::buildLattices(ExprId e, LoopId i) {
   case TensorExp::Kind::kExpm1C:
   case TensorExp::Kind::kLog1pF:
   case TensorExp::Kind::kLog1pC:
+  case TensorExp::Kind::kRelu:
   case TensorExp::Kind::kSinF:
   case TensorExp::Kind::kSinC:
   case TensorExp::Kind::kTanhF:
@@ -1001,7 +1007,8 @@ LatSetId Merger::buildLattices(ExprId e, LoopId i) {
     {
       const ExprId e0 = expr.children.e0;
       const Value v = expr.val;
-      return mapSet(kind, buildLattices(e0, i), v);
+      Attribute a = expr.attr;
+      return mapSet(kind, buildLattices(e0, i), v, nullptr, a);
     }
   case TensorExp::Kind::kBinaryBranch:
   case TensorExp::Kind::kSelect:
@@ -1190,10 +1197,26 @@ std::optional<ExprId> Merger::buildTensorExpFromLinalg(linalg::GenericOp op) {
   return buildTensorExp(op, yield->getOperand(0)).first;
 }
 
+/// Only returns true if we are certain this is a zero.
+static bool isCertainZero(Value val) {
+  if (auto c = val.getDefiningOp<complex::ConstantOp>()) {
+    ArrayAttr arrayAttr = c.getValue();
+    return cast<FloatAttr>(arrayAttr[0]).getValue().isZero() &&
+           cast<FloatAttr>(arrayAttr[1]).getValue().isZero();
+  }
+  if (auto c = val.getDefiningOp<arith::ConstantIntOp>())
+    return c.value() == 0;
+  if (auto c = val.getDefiningOp<arith::ConstantFloatOp>())
+    return c.value().isZero();
+  return false;
+}
+
 /// Only returns false if we are certain this is a nonzero.
 bool Merger::maybeZero(ExprId e) const {
   const auto &expr = exp(e);
   if (expr.kind == TensorExp::Kind::kInvariant) {
+    // Note that this is different from isCertainZero() in a subtle
+    // way by always returning true for non-constants.
     if (auto c = expr.val.getDefiningOp<complex::ConstantOp>()) {
       ArrayAttr arrayAttr = c.getValue();
       return cast<FloatAttr>(arrayAttr[0]).getValue().isZero() &&
@@ -1247,6 +1270,21 @@ static bool isAdmissibleBranch(Operation *op, Region &region) {
   return isAdmissibleBranchExp(op, &region.front(), yield->getOperand(0));
 }
 
+// Recognizes a direct GT comparison.
+static bool isGreater(TensorExp::Kind kind, Attribute attr) {
+  if (kind == TensorExp::Kind::kCmpI) {
+    auto pred = llvm::cast<arith::CmpIPredicateAttr>(attr).getValue();
+    return pred == arith::CmpIPredicate::ugt ||
+           pred == arith::CmpIPredicate::sgt;
+  }
+  if (kind == TensorExp::Kind::kCmpF) {
+    auto pred = llvm::cast<arith::CmpFPredicateAttr>(attr).getValue();
+    return pred == arith::CmpFPredicate::UGT ||
+           pred == arith::CmpFPredicate::OGT;
+  }
+  return false;
+}
+
 std::pair<std::optional<ExprId>, bool>
 Merger::buildTensorExp(linalg::GenericOp op, Value v) {
   // Recursion leaves.
@@ -1266,6 +1304,7 @@ Merger::buildTensorExp(linalg::GenericOp op, Value v) {
     // or belonging to an enveloping op) is considered invariant.
     return {addInvariantExp(v), /*hasSpDep=*/false};
   }
+
   // Something defined outside is invariant.
   Operation *def = v.getDefiningOp();
   if (def->getBlock() != &op.getRegion().front())
@@ -1352,6 +1391,7 @@ Merger::buildTensorExp(linalg::GenericOp op, Value v) {
       }
     }
   }
+
   // Construct binary operations if subexpressions can be built.
   // See buildLattices() for an explanation of rejecting certain
   // division and shift operations.
@@ -1447,6 +1487,7 @@ Merger::buildTensorExp(linalg::GenericOp op, Value v) {
       }
     }
   }
+
   // Construct ternary operations if subexpressions can be built.
   if (def->getNumOperands() == 3) {
     const auto [x, xDepSp] = buildTensorExp(op, def->getOperand(0));
@@ -1460,6 +1501,26 @@ Merger::buildTensorExp(linalg::GenericOp op, Value v) {
         if (isAdmissibleBranch(redop, redop.getRegion()))
           return {addExp(TensorExp::Kind::kReduce, e0, e1, def), hasSpDep};
       }
+      if (auto selop = dyn_cast<arith::SelectOp>(def)) {
+        // Recognize an integral or floating-point ReLu(x) = Max(x, 0)
+        // operation inside a very specific ternary select operation.
+        // TODO: capture MIN/MAX/ABS/RELU structure in a more generic way
+        const auto &cnd = exp(*x);
+        if (isGreater(cnd.kind, cnd.attr) &&
+            exp(*y).kind == TensorExp::Kind::kTensor &&
+            exp(*z).kind == TensorExp::Kind::kInvariant &&
+            isCertainZero(exp(*z).val)) {
+          const auto &a = exp(cnd.children.e0);
+          const auto &b = exp(cnd.children.e1);
+          if (a.kind == TensorExp::Kind::kTensor &&
+              a.tensor == exp(*y).tensor &&
+              b.kind == TensorExp::Kind::kInvariant && isCertainZero(b.val)) {
+            return {addExp(TensorExp::Kind::kRelu, *y, detail::kInvalidId,
+                           nullptr, cnd.attr),
+                    yDepSp};
+          }
+        }
+      }
     }
   }
 
@@ -1469,7 +1530,6 @@ Merger::buildTensorExp(linalg::GenericOp op, Value v) {
   // tensors).
   if (def->getNumResults() != 1) // only handle single result operation.
     return {std::nullopt, false};
-
   SmallVector<std::pair<std::optional<ExprId>, bool>, 2> subExp;
   // Builds all the sub-expressions
   for (Value operand : def->getOperands())
@@ -1489,6 +1549,7 @@ Merger::buildTensorExp(linalg::GenericOp op, Value v) {
       return {e, false};
     }
   }
+
   // Cannot build.
   return {std::nullopt, false};
 }
@@ -1538,6 +1599,22 @@ static Value buildBinaryOverlap(RewriterBase &rewriter, Location loc,
   return insertYieldOp(rewriter, loc, overlapRegion, {v0, v1});
 }
 
+static Value buildRelu(RewriterBase &rewriter, Location loc, Value v0,
+                       Attribute attr) {
+  Type tp = v0.getType();
+  auto zero =
+      rewriter.create<arith::ConstantOp>(loc, tp, rewriter.getZeroAttr(tp));
+  Value cmp;
+  if (isa<FloatType>(tp)) {
+    auto pred = llvm::cast<arith::CmpFPredicateAttr>(attr);
+    cmp = rewriter.create<arith::CmpFOp>(loc, pred, v0, zero);
+  } else {
+    auto pred = llvm::cast<arith::CmpIPredicateAttr>(attr);
+    cmp = rewriter.create<arith::CmpIOp>(loc, pred, v0, zero);
+  }
+  return rewriter.create<arith::SelectOp>(loc, cmp, v0, zero);
+}
+
 Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
                        Value v1) const {
   const auto &expr = exp(e);
@@ -1574,6 +1651,8 @@ Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
     return rewriter.create<math::Log1pOp>(loc, v0);
   case TensorExp::Kind::kLog1pC:
     return rewriter.create<complex::Log1pOp>(loc, v0);
+  case TensorExp::Kind::kRelu:
+    return buildRelu(rewriter, loc, v0, expr.attr);
   case TensorExp::Kind::kSinF:
     return rewriter.create<math::SinOp>(loc, v0);
   case TensorExp::Kind::kSinC:
@@ -1677,7 +1756,8 @@ Value Merger::buildExp(RewriterBase &rewriter, Location loc, ExprId e, Value v0,
   case TensorExp::Kind::kUnary:
     return buildUnaryPresent(rewriter, loc, expr.op, v0);
   case TensorExp::Kind::kSelect:
-    return insertYieldOp(rewriter, loc, cast<SelectOp>(expr.op).getRegion(),
+    return insertYieldOp(rewriter, loc,
+                         cast<sparse_tensor::SelectOp>(expr.op).getRegion(),
                          {v0});
   case TensorExp::Kind::kBinary:
     return buildBinaryOverlap(rewriter, loc, expr.op, v0, v1);
diff --git a/mlir/test/Dialect/SparseTensor/sparse_relu.mlir b/mlir/test/Dialect/SparseTensor/sparse_relu.mlir
new file mode 100644
index 000000000000..25f0c790b43d
--- /dev/null
+++ b/mlir/test/Dialect/SparseTensor/sparse_relu.mlir
@@ -0,0 +1,34 @@
+// RUN: mlir-opt %s --sparsification-and-bufferization | FileCheck %s
+
+#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+
+#sparse = #sparse_tensor.encoding<{
+    map = (d0, d1, d2) -> (d0 : dense, d1 : dense, d2 : compressed)
+}>
+
+//
+// Make sure a simple ReLU passes the sparsifier
+//
+// CHECK-LABEL: func.func @relu
+// CHECK:       scf.for
+// CHECK:         scf.for
+// CHECK:           scf.for
+// CHECK:             arith.cmpf ugt
+// CHECK:             arith.select
+//
+func.func @relu(%arg0: tensor<10x20x30xf64, #sparse>) -> tensor<10x20x30xf64, #sparse> {
+  %cst = arith.constant 0.000000e+00 : f64
+  %0 = tensor.empty() : tensor<10x20x30xf64>
+  %1 = linalg.generic {
+      indexing_maps = [#map, #map],
+      iterator_types = ["parallel", "parallel", "parallel"]}
+      ins(%arg0 : tensor<10x20x30xf64, #sparse>)
+      outs(%0 : tensor<10x20x30xf64>) {
+  ^bb0(%in: f64, %out: f64):
+      %2 = arith.cmpf ugt, %in, %cst : f64
+      %3 = arith.select %2, %in, %cst : f64
+      linalg.yield %3 : f64
+  } -> tensor<10x20x30xf64>
+  %cast = tensor.cast %1 : tensor<10x20x30xf64> to tensor<10x20x30xf64, #sparse>
+  return %cast : tensor<10x20x30xf64, #sparse>
+}
diff --git a/mlir/unittests/Dialect/SparseTensor/MergerTest.cpp b/mlir/unittests/Dialect/SparseTensor/MergerTest.cpp
index 943e7d5c120b..abc6c7076694 100644
--- a/mlir/unittests/Dialect/SparseTensor/MergerTest.cpp
+++ b/mlir/unittests/Dialect/SparseTensor/MergerTest.cpp
@@ -236,6 +236,7 @@ protected:
     case TensorExp::Kind::kExpm1C:
     case TensorExp::Kind::kLog1pF:
     case TensorExp::Kind::kLog1pC:
+    case TensorExp::Kind::kRelu:
     case TensorExp::Kind::kSinF:
     case TensorExp::Kind::kSinC:
     case TensorExp::Kind::kTanhF:
-- 
GitLab


From 4ecf2caf687014a63f0434a63fe9a522ec9be445 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Mon, 13 May 2024 14:23:32 -0700
Subject: [PATCH 0637/1206] [BOLT] Use aggregated FuncBranchData in
 writeBATYAML

Switch from FuncBranchData intermediate maps (Intra/InterIndex)
to aggregated Data, same as one used by DataReader:
https://github.com/llvm/llvm-project/blob/e62ce1f8842cca36eb14126d79dcca0a85bf6d36/bolt/lib/Profile/DataReader.cpp#L385-L389
This aligns the order of the output between YAMLProfileWriter and
writeBATYAML.

Test Plan: updated bolt-address-translation-yaml.test

Reviewers: rafaelauler, dcci, ayermolo, maksfb

Reviewed By: ayermolo, maksfb

Pull Request: https://github.com/llvm/llvm-project/pull/91289
---
 bolt/lib/Profile/DataAggregator.cpp           | 63 ++++++-------------
 .../Inputs/blarge_new_bat_order.preagg.txt    |  2 +
 .../X86/bolt-address-translation-yaml.test    | 11 ++++
 3 files changed, 33 insertions(+), 43 deletions(-)
 create mode 100644 bolt/test/X86/Inputs/blarge_new_bat_order.preagg.txt

diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 302bcf1f2d87..167899ccba12 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -2355,30 +2355,6 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
       for (auto BI = BlockMap.begin(), BE = BlockMap.end(); BI != BE; ++BI)
         YamlBF.Blocks[BI->second.getBBIndex()].Hash = BI->second.getBBHash();
 
-      auto getSuccessorInfo = [&](uint32_t SuccOffset, unsigned SuccDataIdx) {
-        const llvm::bolt::BranchInfo &BI = Branches.Data.at(SuccDataIdx);
-        yaml::bolt::SuccessorInfo SI;
-        SI.Index = BlockMap.getBBIndex(SuccOffset);
-        SI.Count = BI.Branches;
-        SI.Mispreds = BI.Mispreds;
-        return SI;
-      };
-
-      auto getCallSiteInfo = [&](Location CallToLoc, unsigned CallToIdx,
-                                 uint32_t Offset) {
-        const llvm::bolt::BranchInfo &BI = Branches.Data.at(CallToIdx);
-        yaml::bolt::CallSiteInfo CSI;
-        CSI.DestId = 0; // designated for unknown functions
-        CSI.EntryDiscriminator = 0;
-        CSI.Count = BI.Branches;
-        CSI.Mispreds = BI.Mispreds;
-        CSI.Offset = Offset;
-        if (BinaryData *BD = BC.getBinaryDataByName(CallToLoc.Name))
-          YAMLProfileWriter::setCSIDestination(BC, CSI, BD->getSymbol(), BAT,
-                                               CallToLoc.Offset);
-        return CSI;
-      };
-
       // Lookup containing basic block offset and index
       auto getBlock = [&BlockMap](uint32_t Offset) {
         auto BlockIt = BlockMap.upper_bound(Offset);
@@ -2390,25 +2366,26 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
         return std::pair(BlockIt->first, BlockIt->second.getBBIndex());
       };
 
-      for (const auto &[FromOffset, SuccKV] : Branches.IntraIndex) {
-        const auto &[_, Index] = getBlock(FromOffset);
-        yaml::bolt::BinaryBasicBlockProfile &YamlBB = YamlBF.Blocks[Index];
-        for (const auto &[SuccOffset, SuccDataIdx] : SuccKV)
-          if (BlockMap.isInputBlock(SuccOffset))
-            YamlBB.Successors.emplace_back(
-                getSuccessorInfo(SuccOffset, SuccDataIdx));
-      }
-      for (const auto &[FromOffset, CallTo] : Branches.InterIndex) {
-        const auto &[BlockOffset, BlockIndex] = getBlock(FromOffset);
-        yaml::bolt::BinaryBasicBlockProfile &YamlBB = YamlBF.Blocks[BlockIndex];
-        const uint32_t Offset = FromOffset - BlockOffset;
-        for (const auto &[CallToLoc, CallToIdx] : CallTo)
-          YamlBB.CallSites.emplace_back(
-              getCallSiteInfo(CallToLoc, CallToIdx, Offset));
-        llvm::sort(YamlBB.CallSites, [](yaml::bolt::CallSiteInfo &A,
-                                        yaml::bolt::CallSiteInfo &B) {
-          return A.Offset < B.Offset;
-        });
+      for (const llvm::bolt::BranchInfo &BI : Branches.Data) {
+        using namespace yaml::bolt;
+        const auto &[BlockOffset, BlockIndex] = getBlock(BI.From.Offset);
+        BinaryBasicBlockProfile &YamlBB = YamlBF.Blocks[BlockIndex];
+        if (BI.To.IsSymbol && BI.To.Name == BI.From.Name && BI.To.Offset != 0) {
+          // Internal branch
+          const unsigned SuccIndex = getBlock(BI.To.Offset).second;
+          auto &SI = YamlBB.Successors.emplace_back(SuccessorInfo{SuccIndex});
+          SI.Count = BI.Branches;
+          SI.Mispreds = BI.Mispreds;
+        } else {
+          // Call
+          const uint32_t Offset = BI.From.Offset - BlockOffset;
+          auto &CSI = YamlBB.CallSites.emplace_back(CallSiteInfo{Offset});
+          CSI.Count = BI.Branches;
+          CSI.Mispreds = BI.Mispreds;
+          if (const BinaryData *BD = BC.getBinaryDataByName(BI.To.Name))
+            YAMLProfileWriter::setCSIDestination(BC, CSI, BD->getSymbol(), BAT,
+                                                 BI.To.Offset);
+        }
       }
       // Set entry counts, similar to DataReader::readProfile.
       for (const llvm::bolt::BranchInfo &BI : Branches.EntryData) {
diff --git a/bolt/test/X86/Inputs/blarge_new_bat_order.preagg.txt b/bolt/test/X86/Inputs/blarge_new_bat_order.preagg.txt
new file mode 100644
index 000000000000..e4e1f170343c
--- /dev/null
+++ b/bolt/test/X86/Inputs/blarge_new_bat_order.preagg.txt
@@ -0,0 +1,2 @@
+B 800154 401050 20 0
+F 800159 800193 7
diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test
index c15d6ce15ed0..e21513b7dfe5 100644
--- a/bolt/test/X86/bolt-address-translation-yaml.test
+++ b/bolt/test/X86/bolt-address-translation-yaml.test
@@ -15,6 +15,17 @@ BRANCHENTRY-YAML-CHECK:    - name: SolveCubic
 BRANCHENTRY-YAML-CHECK:      bid: 0
 BRANCHENTRY-YAML-CHECK:      hash: 0x700F19D24600000
 BRANCHENTRY-YAML-CHECK-NEXT: succ: [ { bid: 7, cnt: 1 }
+# Check that the order is correct between BAT YAML and FDATA->YAML.
+RUN: perf2bolt %t.out --pa -p %p/Inputs/blarge_new_bat_order.preagg.txt \
+RUN:   -w %t.yaml -o %t.fdata
+RUN: llvm-bolt %t.exe -data %t.fdata -w %t.yaml-fdata -o %t.null
+RUN: FileCheck --input-file %t.yaml --check-prefix ORDER-YAML-CHECK %s
+RUN: FileCheck --input-file %t.yaml-fdata --check-prefix ORDER-YAML-CHECK %s
+ORDER-YAML-CHECK:    - name: SolveCubic
+ORDER-YAML-CHECK:      bid: 3
+ORDER-YAML-CHECK:      hash: 0xDDA1DC5F69F900AC
+ORDER-YAML-CHECK-NEXT: calls: [ { off: 0x26, fid: [[#]], cnt: 20 } ]
+ORDER-YAML-CHECK-NEXT: succ: [ { bid: 5, cnt: 7 }
 # Large profile test
 RUN: perf2bolt %t.out --pa -p %p/Inputs/blarge_new_bat.preagg.txt -w %t.yaml -o %t.fdata \
 RUN:   2>&1 | FileCheck --check-prefix READ-BAT-CHECK %s
-- 
GitLab


From efc7bbb917428393f543b09eecddf6e4bb5fce08 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Mon, 13 May 2024 14:53:26 -0700
Subject: [PATCH 0638/1206] [AMDGPU] Make v2bf16 BUILD_VECTOR legal (#92022)

There is nothing specific here and it is not different from i16 or f16.
---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |    8 +-
 llvm/lib/Target/AMDGPU/SIInstructions.td      |    4 +-
 llvm/test/CodeGen/AMDGPU/bf16-conversions.ll  |    6 +-
 .../AMDGPU/insert_vector_elt.v2bf16.ll        | 1690 +++++++++++++++++
 4 files changed, 1698 insertions(+), 10 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0a3a56e9b3a0..8645f560d997 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -233,9 +233,6 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
     // sources.
     setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
     setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
-
-    setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Promote);
-    AddPromotedToType(ISD::BUILD_VECTOR, MVT::v2bf16, MVT::v2i16);
   }
 
   setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
@@ -744,9 +741,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
     setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
                        MVT::v8i32, Expand);
 
-    if (!Subtarget->hasVOP3PInsts())
-      setOperationAction(ISD::BUILD_VECTOR,
-                         {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Custom);
+    setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
+                       Subtarget->hasVOP3PInsts() ? Legal : Custom);
 
     setOperationAction(ISD::FNEG, MVT::v2f16, Legal);
     // This isn't really legal, but this avoids the legalizer unrolling it (and
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index f9e811f54d05..e7aeaa017306 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3166,7 +3166,7 @@ def : GCNPat <
   (v2f16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), VGPR_32:$src1))
 >;
 
-foreach vecTy = [v2i16, v2f16] in {
+foreach vecTy = [v2i16, v2f16, v2bf16] in {
 
 defvar Ty = vecTy.ElementType;
 
@@ -3212,7 +3212,7 @@ def : GCNPat <
 >;
 
 
-foreach vecTy = [v2i16, v2f16] in {
+foreach vecTy = [v2i16, v2f16, v2bf16] in {
 
 defvar Ty = vecTy.ElementType;
 defvar immzeroTy = !if(!eq(Ty, i16), immzero, fpimmzero);
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index 7108f3d65768..1c9f35dd45fe 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -55,7 +55,8 @@ define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_s(<2 x float> inreg %src) {
 ; GCN-NEXT:    s_add_i32 s5, s2, 0x7fff
 ; GCN-NEXT:    v_cmp_u_f32_e64 s[2:3], s1, s1
 ; GCN-NEXT:    s_and_b64 s[2:3], s[2:3], exec
-; GCN-NEXT:    s_cselect_b32 s2, s4, s5
+; GCN-NEXT:    s_cselect_b32 s1, s4, s5
+; GCN-NEXT:    s_lshr_b32 s2, s1, 16
 ; GCN-NEXT:    s_bfe_u32 s1, s0, 0x10010
 ; GCN-NEXT:    s_add_i32 s1, s1, s0
 ; GCN-NEXT:    s_or_b32 s3, s0, 0x400000
@@ -63,7 +64,8 @@ define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_s(<2 x float> inreg %src) {
 ; GCN-NEXT:    v_cmp_u_f32_e64 s[0:1], s0, s0
 ; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], exec
 ; GCN-NEXT:    s_cselect_b32 s0, s3, s4
-; GCN-NEXT:    s_pack_hh_b32_b16 s0, s0, s2
+; GCN-NEXT:    s_lshr_b32 s0, s0, 16
+; GCN-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    ; return to shader part epilog
   %res = fptrunc <2 x float> %src to <2 x bfloat>
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
new file mode 100644
index 000000000000..c9b01eb5a972
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
@@ -0,0 +1,1690 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=tonga < %s | FileCheck -check-prefix=VI %s
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX900 %s
+; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 < %s | FileCheck -check-prefix=GFX940 %s
+
+define amdgpu_kernel void @s_insertelement_v2bf16_0(ptr addrspace(1) %out, ptr addrspace(4) %vec.ptr) #0 {
+; SI-LABEL: s_insertelement_v2bf16_0:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s4, s[2:3], 0x0
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_and_b32 s4, s4, 0xffff0000
+; SI-NEXT:    s_or_b32 s4, s4, 0x40a0
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_insertelement_v2bf16_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s0, s2, 0xffff0000
+; VI-NEXT:    s_or_b32 s0, s0, 0x40a0
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: s_insertelement_v2bf16_0:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX900-NEXT:    s_pack_ll_b32_b16 s2, 0x40a0, s2
+; GFX900-NEXT:    v_mov_b32_e32 v1, s2
+; GFX900-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: s_insertelement_v2bf16_0:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX940-NEXT:    s_pack_ll_b32_b16 s2, 0x40a0, s2
+; GFX940-NEXT:    v_mov_b32_e32 v1, s2
+; GFX940-NEXT:    global_store_dword v0, v1, s[0:1] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %vec = load <2 x bfloat>, ptr addrspace(4) %vec.ptr
+  %vecins = insertelement <2 x bfloat> %vec, bfloat 5.000000e+00, i32 0
+  store <2 x bfloat> %vecins, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @s_insertelement_v2bf16_1(ptr addrspace(1) %out, ptr addrspace(4) %vec.ptr) #0 {
+; SI-LABEL: s_insertelement_v2bf16_1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s4, s[2:3], 0x0
+; SI-NEXT:    s_mov_b32 s3, 0x100f000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_and_b32 s4, s4, 0xffff
+; SI-NEXT:    s_or_b32 s4, s4, 0x40a00000
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_insertelement_v2bf16_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s2, s[2:3], 0x0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s0, s2, 0xffff
+; VI-NEXT:    s_or_b32 s0, s0, 0x40a00000
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: s_insertelement_v2bf16_1:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    s_pack_ll_b32_b16 s2, s2, 0x40a0
+; GFX900-NEXT:    v_mov_b32_e32 v1, s2
+; GFX900-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: s_insertelement_v2bf16_1:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_load_dword s2, s[2:3], 0x0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_pack_ll_b32_b16 s2, s2, 0x40a0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s2
+; GFX940-NEXT:    global_store_dword v0, v1, s[0:1] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %vec = load <2 x bfloat>, ptr addrspace(4) %vec.ptr
+  %vecins = insertelement <2 x bfloat> %vec, bfloat 5.000000e+00, i32 1
+  store <2 x bfloat> %vecins, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v2bf16_0(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: v_insertelement_v2bf16_0:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; SI-NEXT:    v_or_b32_e32 v2, 0x40a0, v2
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2bf16_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; VI-NEXT:    v_or_b32_e32 v2, 0x40a0, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v2bf16_0:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX900-NEXT:    v_mov_b32_e32 v2, 0x40a0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX900-NEXT:    s_mov_b32 s2, 0xffff
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_bfi_b32 v1, s2, v2, v1
+; GFX900-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v2bf16_0:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0x40a0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX940-NEXT:    s_mov_b32 s2, 0xffff
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_bfi_b32 v1, s2, v2, v1
+; GFX940-NEXT:    global_store_dword v0, v1, s[0:1] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <2 x bfloat>, ptr addrspace(1) %in.gep
+  %vecins = insertelement <2 x bfloat> %vec, bfloat 5.000000e+00, i32 0
+  store <2 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v2bf16_0_inlineimm(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: v_insertelement_v2bf16_0_inlineimm:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; SI-NEXT:    v_or_b32_e32 v2, 53, v2
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2bf16_0_inlineimm:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v2, 0xffff0000, v3
+; VI-NEXT:    v_or_b32_e32 v2, 53, v2
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v2bf16_0_inlineimm:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX900-NEXT:    s_mov_b32 s2, 0xffff
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_bfi_b32 v1, s2, 53, v1
+; GFX900-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v2bf16_0_inlineimm:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX940-NEXT:    s_mov_b32 s2, 0xffff
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_bfi_b32 v1, s2, 53, v1
+; GFX940-NEXT:    global_store_dword v0, v1, s[0:1] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <2 x bfloat>, ptr addrspace(1) %in.gep
+  %vecins = insertelement <2 x bfloat> %vec, bfloat 0xR0035, i32 0
+  store <2 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v2bf16_1(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: v_insertelement_v2bf16_1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    v_or_b32_e32 v2, 0x40a00000, v2
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2bf16_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v2, 0x40a00000
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v2bf16_1:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX900-NEXT:    v_mov_b32_e32 v2, 0x5040100
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX900-NEXT:    s_movk_i32 s2, 0x40a0
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_perm_b32 v1, s2, v1, v2
+; GFX900-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v2bf16_1:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0x5040100
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX940-NEXT:    s_movk_i32 s2, 0x40a0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_perm_b32 v1, s2, v1, v2
+; GFX940-NEXT:    global_store_dword v0, v1, s[0:1] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <2 x bfloat>, ptr addrspace(1) %in.gep
+  %vecins = insertelement <2 x bfloat> %vec, bfloat 5.000000e+00, i32 1
+  store <2 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v2bf16_1_inlineimm(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: v_insertelement_v2bf16_1_inlineimm:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    v_or_b32_e32 v2, 0x230000, v2
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2bf16_1_inlineimm:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v2, 0x230000
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v2bf16_1_inlineimm:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX900-NEXT:    v_mov_b32_e32 v2, 0x5040100
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_perm_b32 v1, 35, v1, v2
+; GFX900-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v2bf16_1_inlineimm:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0x5040100
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_perm_b32 v1, 35, v1, v2
+; GFX940-NEXT:    global_store_dword v0, v1, s[0:1] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <2 x bfloat>, ptr addrspace(1) %in.gep
+  %vecins = insertelement <2 x bfloat> %vec, bfloat 0xR0023, i32 1
+  store <2 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v2bf16_dynamic_vgpr(ptr addrspace(1) %out, ptr addrspace(1) %in, ptr addrspace(1) %idx.ptr) #0 {
+; SI-LABEL: v_insertelement_v2bf16_dynamic_vgpr:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s11, 0x100f000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_mov_b32 s4, 0x12341234
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v2, 4, v2
+; SI-NEXT:    v_lshl_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_bfi_b32 v2, v2, s4, v3
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v2bf16_dynamic_vgpr:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x10
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v3, s3
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b32 s2, 0xffff
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    s_mov_b32 s0, 0x12341234
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 4, v4
+; VI-NEXT:    v_lshlrev_b32_e64 v2, v2, s2
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_bfi_b32 v2, v2, s0, v3
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v2bf16_dynamic_vgpr:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dword v1, v0, s[6:7]
+; GFX900-NEXT:    global_load_dword v2, v0, s[2:3]
+; GFX900-NEXT:    s_mov_b32 s2, 0xffff
+; GFX900-NEXT:    s_waitcnt vmcnt(1)
+; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 4, v1
+; GFX900-NEXT:    v_lshlrev_b32_e64 v1, v1, s2
+; GFX900-NEXT:    s_mov_b32 s2, 0x12341234
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_bfi_b32 v1, v1, s2, v2
+; GFX900-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v2bf16_dynamic_vgpr:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x10
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX940-NEXT:    s_mov_b32 s0, 0xffff
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX940-NEXT:    global_load_dword v2, v0, s[6:7]
+; GFX940-NEXT:    s_waitcnt vmcnt(1)
+; GFX940-NEXT:    v_lshlrev_b32_e32 v1, 4, v1
+; GFX940-NEXT:    v_lshlrev_b32_e64 v1, v1, s0
+; GFX940-NEXT:    s_mov_b32 s0, 0x12341234
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_bfi_b32 v1, v1, s0, v2
+; GFX940-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %idx.gep = getelementptr inbounds i32, ptr addrspace(1) %idx.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %idx = load i32, ptr addrspace(1) %idx.gep
+  %vec = load <2 x bfloat>, ptr addrspace(1) %in.gep
+  %vecins = insertelement <2 x bfloat> %vec, bfloat 0xR1234, i32 %idx
+  store <2 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v4bf16_0(ptr addrspace(1) %out, ptr addrspace(1) %in, [8 x i32], i32 %val) #0 {
+; SI-LABEL: v_insertelement_v4bf16_0:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xc
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b32 s4, 0xffff
+; SI-NEXT:    v_mov_b32_e32 v4, s8
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_bfi_b32 v2, s4, v4, v2
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4bf16_0:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x30
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_perm_b32 v0, s4, v0, v4
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v4bf16_0:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    s_load_dword s6, s[4:5], 0x30
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX900-NEXT:    s_mov_b32 s2, 0xffff
+; GFX900-NEXT:    v_mov_b32_e32 v3, s6
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_bfi_b32 v0, s2, v3, v0
+; GFX900-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v4bf16_0:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    s_load_dword s2, s[0:1], 0x30
+; GFX940-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX940-NEXT:    s_mov_b32 s0, 0xffff
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dwordx2 v[0:1], v2, s[6:7]
+; GFX940-NEXT:    v_mov_b32_e32 v3, s2
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_bfi_b32 v0, s0, v3, v0
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <4 x bfloat>, ptr addrspace(1) %in.gep
+  %val.trunc = trunc i32 %val to i16
+  %val.cvt = bitcast i16 %val.trunc to bfloat
+  %vecins = insertelement <4 x bfloat> %vec, bfloat %val.cvt, i32 0
+  store <4 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v4bf16_1(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val) #0 {
+; SI-LABEL: v_insertelement_v4bf16_1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_load_dword s8, s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_lshl_b32 s4, s8, 16
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    v_or_b32_e32 v2, s4, v2
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4bf16_1:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    v_mov_b32_e32 v4, 0x1000504
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_perm_b32 v0, v0, s4, v4
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v4bf16_1:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    s_load_dword s6, s[4:5], 0x10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX900-NEXT:    v_mov_b32_e32 v3, 0x5040100
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_perm_b32 v0, s6, v0, v3
+; GFX900-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v4bf16_1:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    s_load_dword s2, s[0:1], 0x10
+; GFX940-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0x5040100
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dwordx2 v[0:1], v2, s[6:7]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_perm_b32 v0, s2, v0, v3
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <4 x bfloat>, ptr addrspace(1) %in.gep
+  %val.trunc = trunc i32 %val to i16
+  %val.cvt = bitcast i16 %val.trunc to bfloat
+  %vecins = insertelement <4 x bfloat> %vec, bfloat %val.cvt, i32 1
+  store <4 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v4bf16_2(ptr addrspace(1) %out, ptr addrspace(1) %in, [8 x i32], i32 %val) #0 {
+; SI-LABEL: v_insertelement_v4bf16_2:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xc
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b32 s4, 0xffff
+; SI-NEXT:    v_mov_b32_e32 v4, s8
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_bfi_b32 v3, s4, v4, v3
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4bf16_2:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x30
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_perm_b32 v1, s4, v1, v4
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v4bf16_2:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    s_load_dword s6, s[4:5], 0x30
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX900-NEXT:    s_mov_b32 s2, 0xffff
+; GFX900-NEXT:    v_mov_b32_e32 v3, s6
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_bfi_b32 v1, s2, v3, v1
+; GFX900-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v4bf16_2:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    s_load_dword s2, s[0:1], 0x30
+; GFX940-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX940-NEXT:    s_mov_b32 s0, 0xffff
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dwordx2 v[0:1], v2, s[6:7]
+; GFX940-NEXT:    v_mov_b32_e32 v3, s2
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_bfi_b32 v1, s0, v3, v1
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <4 x bfloat>, ptr addrspace(1) %in.gep
+  %val.trunc = trunc i32 %val to i16
+  %val.cvt = bitcast i16 %val.trunc to bfloat
+  %vecins = insertelement <4 x bfloat> %vec, bfloat %val.cvt, i32 2
+  store <4 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v4bf16_3(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val) #0 {
+; SI-LABEL: v_insertelement_v4bf16_3:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_load_dword s8, s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_lshl_b32 s4, s8, 16
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT:    v_or_b32_e32 v3, s4, v3
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4bf16_3:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    v_mov_b32_e32 v4, 0x1000504
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_perm_b32 v1, v1, s4, v4
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v4bf16_3:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    s_load_dword s6, s[4:5], 0x10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX900-NEXT:    v_mov_b32_e32 v3, 0x5040100
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_perm_b32 v1, s6, v1, v3
+; GFX900-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v4bf16_3:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    s_load_dword s2, s[0:1], 0x10
+; GFX940-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0x5040100
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dwordx2 v[0:1], v2, s[6:7]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_perm_b32 v1, s2, v1, v3
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <4 x bfloat>, ptr addrspace(1) %in.gep
+  %val.trunc = trunc i32 %val to i16
+  %val.cvt = bitcast i16 %val.trunc to bfloat
+  %vecins = insertelement <4 x bfloat> %vec, bfloat %val.cvt, i32 3
+  store <4 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v4bf16_dynamic_sgpr(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val, i32 %idxval) #0 {
+; SI-LABEL: v_insertelement_v4bf16_dynamic_sgpr:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_lshl_b32 s4, s8, 16
+; SI-NEXT:    s_and_b32 s5, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_lshl_b32 s6, s9, 4
+; SI-NEXT:    s_or_b32 s7, s5, s4
+; SI-NEXT:    s_lshl_b64 s[4:5], 0xffff, s6
+; SI-NEXT:    v_mov_b32_e32 v4, s7
+; SI-NEXT:    v_mov_b32_e32 v5, s7
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_bfi_b32 v3, s5, v4, v3
+; SI-NEXT:    v_bfi_b32 v2, s4, v5, v2
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v4bf16_dynamic_sgpr:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x10
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    s_lshl_b32 s1, s4, 16
+; VI-NEXT:    s_and_b32 s2, s4, 0xffff
+; VI-NEXT:    s_lshl_b32 s3, s5, 4
+; VI-NEXT:    s_or_b32 s2, s2, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    s_lshl_b64 s[0:1], 0xffff, s3
+; VI-NEXT:    v_mov_b32_e32 v4, s2
+; VI-NEXT:    v_mov_b32_e32 v5, s2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_bfi_b32 v1, s1, v4, v1
+; VI-NEXT:    v_bfi_b32 v0, s0, v5, v0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v4bf16_dynamic_sgpr:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX900-NEXT:    s_lshl_b32 s2, s7, 4
+; GFX900-NEXT:    s_pack_ll_b32_b16 s4, s6, s6
+; GFX900-NEXT:    s_lshl_b64 s[2:3], 0xffff, s2
+; GFX900-NEXT:    v_mov_b32_e32 v3, s4
+; GFX900-NEXT:    v_mov_b32_e32 v4, s4
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_bfi_b32 v1, s3, v3, v1
+; GFX900-NEXT:    v_bfi_b32 v0, s2, v4, v0
+; GFX900-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v4bf16_dynamic_sgpr:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x10
+; GFX940-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dwordx2 v[0:1], v2, s[6:7]
+; GFX940-NEXT:    s_lshl_b32 s0, s3, 4
+; GFX940-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
+; GFX940-NEXT:    s_lshl_b64 s[0:1], 0xffff, s0
+; GFX940-NEXT:    v_mov_b32_e32 v3, s2
+; GFX940-NEXT:    v_mov_b32_e32 v4, s2
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_bfi_b32 v1, s1, v3, v1
+; GFX940-NEXT:    v_bfi_b32 v0, s0, v4, v0
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <4 x bfloat>, ptr addrspace(1) %in.gep
+  %val.trunc = trunc i32 %val to i16
+  %val.cvt = bitcast i16 %val.trunc to bfloat
+  %vecins = insertelement <4 x bfloat> %vec, bfloat %val.cvt, i32 %idxval
+  store <4 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v8bf16_3(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val) {
+; SI-LABEL: v_insertelement_v8bf16_3:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_load_dword s8, s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    v_mov_b32_e32 v5, 0
+; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64
+; SI-NEXT:    s_lshl_b32 s4, s8, 16
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_or_b32_e32 v1, s4, v1
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], v[4:5], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v8bf16_3:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; VI-NEXT:    v_add_u32_e32 v4, vcc, s0, v4
+; VI-NEXT:    s_lshl_b32 s0, s4, 16
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_mov_b32_e32 v6, s0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v8bf16_3:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    s_load_dword s6, s[4:5], 0x10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; GFX900-NEXT:    v_mov_b32_e32 v5, 0x5040100
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3]
+; GFX900-NEXT:    s_mov_b32 s2, 0xffff
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_bfi_b32 v3, s2, v3, v3
+; GFX900-NEXT:    v_bfi_b32 v2, s2, v2, v2
+; GFX900-NEXT:    v_bfi_b32 v0, s2, v0, v0
+; GFX900-NEXT:    v_perm_b32 v1, s6, v1, v5
+; GFX900-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v8bf16_3:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    s_load_dword s2, s[0:1], 0x10
+; GFX940-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; GFX940-NEXT:    s_mov_b32 s0, 0xffff
+; GFX940-NEXT:    v_mov_b32_e32 v5, 0x5040100
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dwordx4 v[0:3], v4, s[6:7]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_bfi_b32 v3, s0, v3, v3
+; GFX940-NEXT:    v_bfi_b32 v2, s0, v2, v2
+; GFX940-NEXT:    v_bfi_b32 v0, s0, v0, v0
+; GFX940-NEXT:    v_perm_b32 v1, s2, v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <8 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <8 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <8 x bfloat>, ptr addrspace(1) %in.gep
+  %val.trunc = trunc i32 %val to i16
+  %val.cvt = bitcast i16 %val.trunc to bfloat
+  %vecins = insertelement <8 x bfloat> %vec, bfloat %val.cvt, i32 3
+  store <8 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v8bf16_dynamic(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val, i32 %n) {
+; SI-LABEL: v_insertelement_v8bf16_dynamic:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    v_mov_b32_e32 v5, 0
+; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64
+; SI-NEXT:    s_cmp_eq_u32 s9, 6
+; SI-NEXT:    v_mov_b32_e32 v6, s8
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s9, 7
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cndmask_b32_e32 v7, v3, v6, vcc
+; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s9, 4
+; SI-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s9, 5
+; SI-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
+; SI-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s9, 2
+; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT:    v_cndmask_b32_e32 v8, v8, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s9, 3
+; SI-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    v_or_b32_e32 v3, v7, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v8
+; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s9, 0
+; SI-NEXT:    v_or_b32_e32 v2, v2, v7
+; SI-NEXT:    v_cndmask_b32_e32 v7, v9, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s9, 1
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT:    v_or_b32_e32 v1, v1, v7
+; SI-NEXT:    v_or_b32_e32 v0, v0, v6
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], v[4:5], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v8bf16_dynamic:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x10
+; VI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_add_u32_e32 v4, vcc, s0, v4
+; VI-NEXT:    s_cmp_eq_u32 s5, 6
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    v_mov_b32_e32 v6, s4
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 7
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cndmask_b32_e32 v7, v3, v6, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 4
+; VI-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 5
+; VI-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
+; VI-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 2
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; VI-NEXT:    v_cndmask_b32_e32 v8, v8, v6, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 3
+; VI-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; VI-NEXT:    v_or_b32_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v7, 16, v8
+; VI-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 0
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_cndmask_b32_e32 v7, v9, v6, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 1
+; VI-NEXT:    v_lshrrev_b32_e32 v10, 16, v0
+; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
+; VI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; VI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v8bf16_dynamic:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3]
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 6
+; GFX900-NEXT:    v_mov_b32_e32 v5, s6
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 7
+; GFX900-NEXT:    s_mov_b32 s2, 0x5040100
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v3, v5, vcc
+; GFX900-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 4
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 5
+; GFX900-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 2
+; GFX900-NEXT:    v_perm_b32 v3, v3, v6, s2
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v5, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 3
+; GFX900-NEXT:    v_lshrrev_b32_e32 v8, 16, v1
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 0
+; GFX900-NEXT:    v_perm_b32 v2, v6, v2, s2
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v8, v5, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 1
+; GFX900-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
+; GFX900-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; GFX900-NEXT:    v_perm_b32 v1, v6, v1, s2
+; GFX900-NEXT:    v_perm_b32 v0, v5, v0, s2
+; GFX900-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v8bf16_dynamic:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x10
+; GFX940-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dwordx4 v[0:3], v4, s[6:7]
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 6
+; GFX940-NEXT:    v_mov_b32_e32 v5, s2
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 7
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_cndmask_b32_e32 v6, v3, v5, vcc
+; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 4
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 5
+; GFX940-NEXT:    v_lshrrev_b32_e32 v7, 16, v2
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 2
+; GFX940-NEXT:    v_perm_b32 v3, v3, v6, s0
+; GFX940-NEXT:    v_cndmask_b32_e32 v6, v7, v5, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 3
+; GFX940-NEXT:    v_lshrrev_b32_e32 v8, 16, v1
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 0
+; GFX940-NEXT:    v_perm_b32 v2, v6, v2, s0
+; GFX940-NEXT:    v_cndmask_b32_e32 v6, v8, v5, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX940-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; GFX940-NEXT:    v_perm_b32 v1, v6, v1, s0
+; GFX940-NEXT:    v_perm_b32 v0, v5, v0, s0
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <8 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <8 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <8 x bfloat>, ptr addrspace(1) %in.gep
+  %val.trunc = trunc i32 %val to i16
+  %val.cvt = bitcast i16 %val.trunc to bfloat
+  %vecins = insertelement <8 x bfloat> %vec, bfloat %val.cvt, i32 %n
+  store <8 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v16bf16_3(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val) {
+; SI-LABEL: v_insertelement_v16bf16_3:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_load_dword s8, s[4:5], 0x4
+; SI-NEXT:    s_mov_b32 s7, 0x100f000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    v_mov_b32_e32 v9, 0
+; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[8:9], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dwordx4 v[4:7], v[8:9], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_lshl_b32 s4, s8, 16
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_or_b32_e32 v1, s4, v1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dwordx4 v[4:7], v[8:9], s[0:3], 0 addr64 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], v[8:9], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v16bf16_3:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
+; VI-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v8
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 16, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; VI-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; VI-NEXT:    v_mov_b32_e32 v9, s1
+; VI-NEXT:    v_add_u32_e32 v8, vcc, s0, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    s_lshl_b32 s1, s4, 16
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 16, v8
+; VI-NEXT:    v_mov_b32_e32 v12, s1
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v9, vcc
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
+; VI-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v16bf16_3:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    s_load_dword s6, s[4:5], 0x10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; GFX900-NEXT:    v_mov_b32_e32 v9, 0x5040100
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[0:3], v8, s[2:3]
+; GFX900-NEXT:    global_load_dwordx4 v[4:7], v8, s[2:3] offset:16
+; GFX900-NEXT:    s_waitcnt vmcnt(1)
+; GFX900-NEXT:    v_perm_b32 v1, s6, v1, v9
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
+; GFX900-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v16bf16_3:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    s_load_dword s2, s[0:1], 0x10
+; GFX940-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; GFX940-NEXT:    v_mov_b32_e32 v9, 0x5040100
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dwordx4 v[0:3], v8, s[6:7]
+; GFX940-NEXT:    global_load_dwordx4 v[4:7], v8, s[6:7] offset:16
+; GFX940-NEXT:    s_waitcnt vmcnt(1)
+; GFX940-NEXT:    v_perm_b32 v1, s2, v1, v9
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[4:5] offset:16 sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <16 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <16 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <16 x bfloat>, ptr addrspace(1) %in.gep
+  %val.trunc = trunc i32 %val to i16
+  %val.cvt = bitcast i16 %val.trunc to bfloat
+  %vecins = insertelement <16 x bfloat> %vec, bfloat %val.cvt, i32 3
+  store <16 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val, i32 %n) {
+; SI-LABEL: v_insertelement_v16bf16_dynamic:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; SI-NEXT:    s_mov_b32 s11, 0x100f000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 5, v0
+; SI-NEXT:    v_mov_b32_e32 v5, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_dwordx4 v[7:10], v[4:5], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[4:5], s[8:11], 0 addr64 offset:16
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x4
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_cmp_eq_u32 s5, 6
+; SI-NEXT:    v_mov_b32_e32 v6, s4
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 7
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cndmask_b32_e32 v11, v10, v6, vcc
+; SI-NEXT:    v_lshrrev_b32_e32 v10, 16, v10
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 4
+; SI-NEXT:    v_cndmask_b32_e32 v10, v10, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 5
+; SI-NEXT:    v_lshrrev_b32_e32 v12, 16, v9
+; SI-NEXT:    v_cndmask_b32_e32 v9, v9, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 2
+; SI-NEXT:    v_and_b32_e32 v11, 0xffff, v11
+; SI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT:    v_cndmask_b32_e32 v12, v12, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 3
+; SI-NEXT:    v_lshrrev_b32_e32 v13, 16, v8
+; SI-NEXT:    v_and_b32_e32 v9, 0xffff, v9
+; SI-NEXT:    v_or_b32_e32 v10, v11, v10
+; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v12
+; SI-NEXT:    v_cndmask_b32_e32 v8, v8, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 0
+; SI-NEXT:    v_or_b32_e32 v9, v9, v11
+; SI-NEXT:    v_cndmask_b32_e32 v11, v13, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 1
+; SI-NEXT:    v_lshrrev_b32_e32 v14, 16, v7
+; SI-NEXT:    v_and_b32_e32 v8, 0xffff, v8
+; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT:    v_cndmask_b32_e32 v7, v7, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 14
+; SI-NEXT:    v_or_b32_e32 v8, v8, v11
+; SI-NEXT:    v_cndmask_b32_e32 v11, v14, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 15
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v15, 16, v3
+; SI-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT:    v_cndmask_b32_e32 v3, v3, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 12
+; SI-NEXT:    v_or_b32_e32 v7, v7, v11
+; SI-NEXT:    v_cndmask_b32_e32 v11, v15, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 13
+; SI-NEXT:    v_lshrrev_b32_e32 v16, 16, v2
+; SI-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 10
+; SI-NEXT:    v_or_b32_e32 v3, v3, v11
+; SI-NEXT:    v_cndmask_b32_e32 v11, v16, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 11
+; SI-NEXT:    v_lshrrev_b32_e32 v17, 16, v1
+; SI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 8
+; SI-NEXT:    v_or_b32_e32 v2, v2, v11
+; SI-NEXT:    v_cndmask_b32_e32 v11, v17, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    s_cmp_eq_u32 s5, 9
+; SI-NEXT:    v_lshrrev_b32_e32 v18, 16, v0
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; SI-NEXT:    s_cselect_b64 vcc, -1, 0
+; SI-NEXT:    v_cndmask_b32_e32 v6, v18, v6, vcc
+; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT:    v_or_b32_e32 v1, v1, v11
+; SI-NEXT:    v_or_b32_e32 v0, v0, v6
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], v[4:5], s[0:3], 0 addr64 offset:16
+; SI-NEXT:    buffer_store_dwordx4 v[7:10], v[4:5], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_insertelement_v16bf16_dynamic:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x10
+; VI-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s3
+; VI-NEXT:    v_add_u32_e32 v4, vcc, s2, v8
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v0, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v4
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; VI-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; VI-NEXT:    v_mov_b32_e32 v9, s1
+; VI-NEXT:    v_add_u32_e32 v8, vcc, s0, v8
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; VI-NEXT:    v_add_u32_e32 v10, vcc, 16, v8
+; VI-NEXT:    s_cmp_eq_u32 s5, 14
+; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v9, vcc
+; VI-NEXT:    v_mov_b32_e32 v12, s4
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 15
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_cndmask_b32_e32 v13, v3, v12, vcc
+; VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 12
+; VI-NEXT:    v_cndmask_b32_e32 v3, v3, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 13
+; VI-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
+; VI-NEXT:    v_cndmask_b32_e32 v2, v2, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 10
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; VI-NEXT:    v_cndmask_b32_e32 v14, v14, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 11
+; VI-NEXT:    v_lshrrev_b32_e32 v15, 16, v1
+; VI-NEXT:    v_or_b32_sdwa v3, v13, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_lshlrev_b32_e32 v13, 16, v14
+; VI-NEXT:    v_cndmask_b32_e32 v1, v1, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 8
+; VI-NEXT:    v_or_b32_sdwa v2, v2, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_cndmask_b32_e32 v13, v15, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 9
+; VI-NEXT:    v_lshrrev_b32_e32 v16, 16, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 6
+; VI-NEXT:    v_or_b32_sdwa v1, v1, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_cndmask_b32_e32 v13, v16, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 7
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v17, 16, v7
+; VI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; VI-NEXT:    v_cndmask_b32_e32 v7, v7, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 4
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_cndmask_b32_e32 v13, v17, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 5
+; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
+; VI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; VI-NEXT:    v_cndmask_b32_e32 v6, v6, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 2
+; VI-NEXT:    v_or_b32_sdwa v7, v7, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_cndmask_b32_e32 v13, v18, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 3
+; VI-NEXT:    v_lshrrev_b32_e32 v19, 16, v5
+; VI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; VI-NEXT:    v_cndmask_b32_e32 v5, v5, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 0
+; VI-NEXT:    v_or_b32_sdwa v6, v6, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_cndmask_b32_e32 v13, v19, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    s_cmp_eq_u32 s5, 1
+; VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v4
+; VI-NEXT:    v_cndmask_b32_e32 v4, v4, v12, vcc
+; VI-NEXT:    s_cselect_b64 vcc, -1, 0
+; VI-NEXT:    v_cndmask_b32_e32 v12, v20, v12, vcc
+; VI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
+; VI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
+; VI-NEXT:    v_or_b32_sdwa v5, v5, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    v_or_b32_sdwa v4, v4, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-NEXT:    flat_store_dwordx4 v[8:9], v[4:7]
+; VI-NEXT:    flat_store_dwordx4 v[10:11], v[0:3]
+; VI-NEXT:    s_endpgm
+;
+; GFX900-LABEL: v_insertelement_v16bf16_dynamic:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX900-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX900-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
+; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[1:4], v0, s[2:3]
+; GFX900-NEXT:    global_load_dwordx4 v[5:8], v0, s[2:3] offset:16
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 6
+; GFX900-NEXT:    v_mov_b32_e32 v9, s6
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 7
+; GFX900-NEXT:    s_mov_b32 s2, 0x5040100
+; GFX900-NEXT:    s_waitcnt vmcnt(1)
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v4, v9, vcc
+; GFX900-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 4
+; GFX900-NEXT:    v_cndmask_b32_e32 v4, v4, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 5
+; GFX900-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
+; GFX900-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 2
+; GFX900-NEXT:    v_perm_b32 v4, v4, v10, s2
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v11, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 3
+; GFX900-NEXT:    v_lshrrev_b32_e32 v12, 16, v2
+; GFX900-NEXT:    v_cndmask_b32_e32 v2, v2, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 0
+; GFX900-NEXT:    v_perm_b32 v3, v10, v3, s2
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v12, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 1
+; GFX900-NEXT:    v_lshrrev_b32_e32 v13, 16, v1
+; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 14
+; GFX900-NEXT:    v_perm_b32 v2, v10, v2, s2
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v13, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 15
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_lshrrev_b32_e32 v14, 16, v8
+; GFX900-NEXT:    v_cndmask_b32_e32 v8, v8, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 12
+; GFX900-NEXT:    v_perm_b32 v1, v10, v1, s2
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v14, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 13
+; GFX900-NEXT:    v_lshrrev_b32_e32 v15, 16, v7
+; GFX900-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 10
+; GFX900-NEXT:    v_perm_b32 v8, v10, v8, s2
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v15, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 11
+; GFX900-NEXT:    v_lshrrev_b32_e32 v16, 16, v6
+; GFX900-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 8
+; GFX900-NEXT:    v_perm_b32 v7, v10, v7, s2
+; GFX900-NEXT:    v_cndmask_b32_e32 v10, v16, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    s_cmp_eq_u32 s7, 9
+; GFX900-NEXT:    v_lshrrev_b32_e32 v17, 16, v5
+; GFX900-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
+; GFX900-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX900-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
+; GFX900-NEXT:    v_perm_b32 v6, v10, v6, s2
+; GFX900-NEXT:    v_perm_b32 v5, v9, v5, s2
+; GFX900-NEXT:    global_store_dwordx4 v0, v[5:8], s[0:1] offset:16
+; GFX900-NEXT:    global_store_dwordx4 v0, v[1:4], s[0:1]
+; GFX900-NEXT:    s_endpgm
+;
+; GFX940-LABEL: v_insertelement_v16bf16_dynamic:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x10
+; GFX940-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; GFX940-NEXT:    s_mov_b32 s0, 0x5040100
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    global_load_dwordx4 v[0:3], v8, s[6:7]
+; GFX940-NEXT:    global_load_dwordx4 v[4:7], v8, s[6:7] offset:16
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 6
+; GFX940-NEXT:    v_mov_b32_e32 v9, s2
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 7
+; GFX940-NEXT:    s_waitcnt vmcnt(1)
+; GFX940-NEXT:    v_cndmask_b32_e32 v10, v3, v9, vcc
+; GFX940-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 4
+; GFX940-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 5
+; GFX940-NEXT:    v_lshrrev_b32_e32 v11, 16, v2
+; GFX940-NEXT:    v_cndmask_b32_e32 v2, v2, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 2
+; GFX940-NEXT:    v_perm_b32 v3, v3, v10, s0
+; GFX940-NEXT:    v_cndmask_b32_e32 v10, v11, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 3
+; GFX940-NEXT:    v_lshrrev_b32_e32 v12, 16, v1
+; GFX940-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 0
+; GFX940-NEXT:    v_perm_b32 v2, v10, v2, s0
+; GFX940-NEXT:    v_cndmask_b32_e32 v10, v12, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX940-NEXT:    v_lshrrev_b32_e32 v13, 16, v0
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 14
+; GFX940-NEXT:    v_perm_b32 v1, v10, v1, s0
+; GFX940-NEXT:    v_cndmask_b32_e32 v10, v13, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 15
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    v_lshrrev_b32_e32 v14, 16, v7
+; GFX940-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 12
+; GFX940-NEXT:    v_perm_b32 v0, v10, v0, s0
+; GFX940-NEXT:    v_cndmask_b32_e32 v10, v14, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 13
+; GFX940-NEXT:    v_lshrrev_b32_e32 v15, 16, v6
+; GFX940-NEXT:    v_cndmask_b32_e32 v6, v6, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 10
+; GFX940-NEXT:    v_perm_b32 v7, v10, v7, s0
+; GFX940-NEXT:    v_cndmask_b32_e32 v10, v15, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 11
+; GFX940-NEXT:    v_lshrrev_b32_e32 v16, 16, v5
+; GFX940-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 8
+; GFX940-NEXT:    v_perm_b32 v6, v10, v6, s0
+; GFX940-NEXT:    v_cndmask_b32_e32 v10, v16, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    s_cmp_eq_u32 s3, 9
+; GFX940-NEXT:    v_lshrrev_b32_e32 v17, 16, v4
+; GFX940-NEXT:    v_cndmask_b32_e32 v4, v4, v9, vcc
+; GFX940-NEXT:    s_cselect_b64 vcc, -1, 0
+; GFX940-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
+; GFX940-NEXT:    v_perm_b32 v5, v10, v5, s0
+; GFX940-NEXT:    v_perm_b32 v4, v9, v4, s0
+; GFX940-NEXT:    global_store_dwordx4 v8, v[4:7], s[4:5] offset:16 sc0 sc1
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tid.ext = sext i32 %tid to i64
+  %in.gep = getelementptr inbounds <16 x bfloat>, ptr addrspace(1) %in, i64 %tid.ext
+  %out.gep = getelementptr inbounds <16 x bfloat>, ptr addrspace(1) %out, i64 %tid.ext
+  %vec = load <16 x bfloat>, ptr addrspace(1) %in.gep
+  %val.trunc = trunc i32 %val to i16
+  %val.cvt = bitcast i16 %val.trunc to bfloat
+  %vecins = insertelement <16 x bfloat> %vec, bfloat %val.cvt, i32 %n
+  store <16 x bfloat> %vecins, ptr addrspace(1) %out.gep
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
-- 
GitLab


From 561c42df5712c346d4de2e6499b06712403d3164 Mon Sep 17 00:00:00 2001
From: Robin Caloudis <robin.caloudis@gmx.de>
Date: Mon, 13 May 2024 23:56:01 +0200
Subject: [PATCH 0639/1206] [libc][errno] Use macro instead of system header
 (#91150)

## Why
Currently, the system header `errno.h` is included in `libc_errno.h`,
which is supposed to be consumed by internal implementations only. As
unit and hermetic tests should never use `#include <errno.h>` but
instead use `#include "src/errno/libc_errno.h"`, we do not want to
implicitly include `errno.h`. In order to have a clear seperation
between those two, we want to pull out the definitions of errno numbers
from `errno.h`.

## What
* Extract the definitions of errno numbers from
[include/errno.h.def](https://github.com/llvm/llvm-project/pull/91150/files#diff-ed38ed463ed50571b498a5b69039cab58dc9d145da7f751a24da9d77f07781cd)
and place it under
[include/llvm-libc-macros/linux/error-number-macros.h](https://github.com/llvm/llvm-project/pull/91150/files#diff-d6192866629690ebb7cefa1f0a90b6675073e9642f3279df08a04dcdb05fd892)
* Provide mips-specific errno numbers in
[include/llvm-libc-macros/linux/mips/error-number-macros.h](https://github.com/llvm/llvm-project/pull/91150/files#diff-3fd35a4c94e0cc359933e497b10311d857857b2e173e8afebc421b04b7527743)
* Find definition of mips errno numbers in glibc
[here](https://github.com/bminor/glibc/blob/ea73eb5f581ef5931fd67005aa0c526ba43366c9/sysdeps/unix/sysv/linux/mips/bits/errno.h#L32-L50)
(equally defined in the Linux kernel)
* Provide sparc-specific errno numbers in
[include/llvm-libc-macros/linux/sparc/error-number-macros.h](https://github.com/llvm/llvm-project/pull/91150/files#diff-5f16ffb2a51a6f72ebd4403aca7e1edea48289c99dd5978a1c84385bec4f226b)
* Find definition of sparc errno numbers in glibc
[here](https://github.com/bminor/glibc/blob/ea73eb5f581ef5931fd67005aa0c526ba43366c9/sysdeps/unix/sysv/linux/sparc/bits/errno.h#L33-L51)
(equally defined in the Linux kernel)
* Include proxy header `errno_macros.h` instead of the system header
`errno.h` in `libc_errno.h`/`libc_errno.cpp`

Closes https://github.com/llvm/llvm-project/issues/80172
---
 libc/hdr/CMakeLists.txt                       | 10 ++++++
 libc/hdr/errno_macros.h                       | 26 +++++++++++++++
 libc/include/errno.h.def                      | 20 +-----------
 libc/include/llvm-libc-macros/CMakeLists.txt  |  6 ++++
 .../llvm-libc-macros/error-number-macros.h    |  8 +++++
 .../generic-error-number-macros.h             |  2 ++
 .../llvm-libc-macros/linux/CMakeLists.txt     | 12 +++++++
 .../linux/error-number-macros.h               | 32 +++++++++++++++++++
 .../linux/mips/CMakeLists.txt                 |  5 +++
 .../linux/mips/error-number-macros.h          | 24 ++++++++++++++
 .../linux/sparc/CMakeLists.txt                |  5 +++
 .../linux/sparc/error-number-macros.h         | 24 ++++++++++++++
 libc/src/errno/CMakeLists.txt                 |  2 +-
 libc/src/errno/libc_errno.cpp                 |  2 +-
 libc/src/errno/libc_errno.h                   |  6 +---
 15 files changed, 158 insertions(+), 26 deletions(-)
 create mode 100644 libc/hdr/errno_macros.h
 create mode 100644 libc/include/llvm-libc-macros/error-number-macros.h
 create mode 100644 libc/include/llvm-libc-macros/linux/error-number-macros.h
 create mode 100644 libc/include/llvm-libc-macros/linux/mips/CMakeLists.txt
 create mode 100644 libc/include/llvm-libc-macros/linux/mips/error-number-macros.h
 create mode 100644 libc/include/llvm-libc-macros/linux/sparc/CMakeLists.txt
 create mode 100644 libc/include/llvm-libc-macros/linux/sparc/error-number-macros.h

diff --git a/libc/hdr/CMakeLists.txt b/libc/hdr/CMakeLists.txt
index 754934251430..91b8cb71552a 100644
--- a/libc/hdr/CMakeLists.txt
+++ b/libc/hdr/CMakeLists.txt
@@ -32,6 +32,16 @@ add_proxy_header_library(
     libc.include.math
 )
 
+add_proxy_header_library(
+  errno_macros
+  HDRS
+    errno_macros.h
+  FULL_BUILD_DEPENDS
+    libc.include.errno
+    libc.include.llvm-libc-macros.error_number_macros
+    libc.include.llvm-libc-macros.generic_error_number_macros
+)
+
 add_proxy_header_library(
   fcntl_macros
   HDRS
diff --git a/libc/hdr/errno_macros.h b/libc/hdr/errno_macros.h
new file mode 100644
index 000000000000..b5ef7dc2a207
--- /dev/null
+++ b/libc/hdr/errno_macros.h
@@ -0,0 +1,26 @@
+//===-- Definition of macros from errno.h ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_ERRNO_MACROS_H
+#define LLVM_LIBC_HDR_ERRNO_MACROS_H
+
+#ifdef LIBC_FULL_BUILD
+
+#ifdef __linux__
+#include "llvm-libc-macros/error-number-macros.h"
+#else // __linux__
+#include "llvm-libc-macros/generic-error-number-macros.h"
+#endif
+
+#else // Overlay mode
+
+#include <errno.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_ERRNO_MACROS_H
diff --git a/libc/include/errno.h.def b/libc/include/errno.h.def
index d7ae90ad4524..3ffcd3fe4c72 100644
--- a/libc/include/errno.h.def
+++ b/libc/include/errno.h.def
@@ -15,29 +15,11 @@
 
 #include <linux/errno.h>
 
-#ifndef ERFKILL
-#define ERFKILL 132
-#endif // ERFKILL
-
-#ifndef EOWNERDEAD
-#define EOWNERDEAD 130
-#endif // EOWNERDEAD
-
-#ifndef EHWPOISON
-#define EHWPOISON 133
-#endif // EHWPOISON
-
-#ifndef ECANCELED
-#define ECANCELED 125
-#endif // ECANCELED
-
 #ifndef ENOTSUP
 #define ENOTSUP EOPNOTSUPP
 #endif // ENOTSUP
 
-#ifndef ENOTRECOVERABLE
-#define ENOTRECOVERABLE 131
-#endif // ENOTRECOVERABLE
+#include "llvm-libc-macros/linux/error-number-macros.h"
 
 #else // __linux__
 #include "llvm-libc-macros/generic-error-number-macros.h"
diff --git a/libc/include/llvm-libc-macros/CMakeLists.txt b/libc/include/llvm-libc-macros/CMakeLists.txt
index 68ba110aec80..961830ef9766 100644
--- a/libc/include/llvm-libc-macros/CMakeLists.txt
+++ b/libc/include/llvm-libc-macros/CMakeLists.txt
@@ -37,6 +37,12 @@ add_macro_header(
     assert-macros.h
 )
 
+add_macro_header(
+  error_number_macros
+  HDR
+    error-number-macros.h
+)
+
 add_macro_header(
   generic_error_number_macros
   HDR
diff --git a/libc/include/llvm-libc-macros/error-number-macros.h b/libc/include/llvm-libc-macros/error-number-macros.h
new file mode 100644
index 000000000000..29bd54d07f2e
--- /dev/null
+++ b/libc/include/llvm-libc-macros/error-number-macros.h
@@ -0,0 +1,8 @@
+#ifndef LLVM_LIBC_MACROS_ERROR_NUMBER_MACROS_H
+#define LLVM_LIBC_MACROS_ERROR_NUMBER_MACROS_H
+
+#ifdef __linux__
+#include "linux/error-number-macros.h"
+#endif
+
+#endif // LLVM_LIBC_MACROS_ERROR_NUMBER_MACROS_H
diff --git a/libc/include/llvm-libc-macros/generic-error-number-macros.h b/libc/include/llvm-libc-macros/generic-error-number-macros.h
index 7ee0352669b8..b5b1b676dacc 100644
--- a/libc/include/llvm-libc-macros/generic-error-number-macros.h
+++ b/libc/include/llvm-libc-macros/generic-error-number-macros.h
@@ -44,5 +44,7 @@
 #define EDOM 33
 #define ERANGE 34
 #define EILSEQ 35
+#define ENAMETOOLONG 36
+#define EOVERFLOW 75
 
 #endif // LLVM_LIBC_MACROS_GENERIC_ERROR_NUMBER_MACROS_H
diff --git a/libc/include/llvm-libc-macros/linux/CMakeLists.txt b/libc/include/llvm-libc-macros/linux/CMakeLists.txt
index 4ee429d1db16..a07803103eef 100644
--- a/libc/include/llvm-libc-macros/linux/CMakeLists.txt
+++ b/libc/include/llvm-libc-macros/linux/CMakeLists.txt
@@ -1,3 +1,15 @@
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/mips)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/sparc)
+
+add_header(
+  error_number_macros
+  HDR
+    error-number-macros.h
+  DEPENDS
+    .mips.error_number_macros
+    .sparc.error_number_macros
+)
+
 add_header(
   fcntl_macros
   HDR
diff --git a/libc/include/llvm-libc-macros/linux/error-number-macros.h b/libc/include/llvm-libc-macros/linux/error-number-macros.h
new file mode 100644
index 000000000000..4c8b3feb3dc3
--- /dev/null
+++ b/libc/include/llvm-libc-macros/linux/error-number-macros.h
@@ -0,0 +1,32 @@
+#ifndef LLVM_LIBC_MACROS_LINUX_ERROR_NUMBER_MACROS_H
+#define LLVM_LIBC_MACROS_LINUX_ERROR_NUMBER_MACROS_H
+
+#if defined(__mips__)
+#include "mips/error-number-macros.h"
+
+#elif defined(__sparc__)
+#include "sparc/error-number-macros.h"
+
+#else
+#ifndef ECANCELED
+#define ECANCELED 125
+#endif // ECANCELED
+
+#ifndef EOWNERDEAD
+#define EOWNERDEAD 130
+#endif // EOWNERDEAD
+
+#ifndef ENOTRECOVERABLE
+#define ENOTRECOVERABLE 131
+#endif // ENOTRECOVERABLE
+
+#ifndef ERFKILL
+#define ERFKILL 132
+#endif // ERFKILL
+
+#ifndef EHWPOISON
+#define EHWPOISON 133
+#endif // EHWPOISON
+#endif
+
+#endif // LLVM_LIBC_MACROS_LINUX_ERROR_NUMBER_MACROS_H
diff --git a/libc/include/llvm-libc-macros/linux/mips/CMakeLists.txt b/libc/include/llvm-libc-macros/linux/mips/CMakeLists.txt
new file mode 100644
index 000000000000..eee4cfd19396
--- /dev/null
+++ b/libc/include/llvm-libc-macros/linux/mips/CMakeLists.txt
@@ -0,0 +1,5 @@
+add_header(
+  error_number_macros
+  HDR
+    error-number-macros.h
+)
diff --git a/libc/include/llvm-libc-macros/linux/mips/error-number-macros.h b/libc/include/llvm-libc-macros/linux/mips/error-number-macros.h
new file mode 100644
index 000000000000..af2a4243e3ce
--- /dev/null
+++ b/libc/include/llvm-libc-macros/linux/mips/error-number-macros.h
@@ -0,0 +1,24 @@
+#ifndef LLVM_LIBC_MACROS_LINUX_MIPS_ERROR_NUMBER_MACROS_H
+#define LLVM_LIBC_MACROS_LINUX_MIPS_ERROR_NUMBER_MACROS_H
+
+#ifndef ECANCELED
+#define ECANCELED 158
+#endif // ECANCELED
+
+#ifndef EOWNERDEAD
+#define EOWNERDEAD 165
+#endif // EOWNERDEAD
+
+#ifndef ENOTRECOVERABLE
+#define ENOTRECOVERABLE 166
+#endif // ENOTRECOVERABLE
+
+#ifndef ERFKILL
+#define ERFKILL 167
+#endif // ERFKILL
+
+#ifndef EHWPOISON
+#define EHWPOISON 168
+#endif // EHWPOISON
+
+#endif // LLVM_LIBC_MACROS_LINUX_MIPS_ERROR_NUMBER_MACROS_H
diff --git a/libc/include/llvm-libc-macros/linux/sparc/CMakeLists.txt b/libc/include/llvm-libc-macros/linux/sparc/CMakeLists.txt
new file mode 100644
index 000000000000..eee4cfd19396
--- /dev/null
+++ b/libc/include/llvm-libc-macros/linux/sparc/CMakeLists.txt
@@ -0,0 +1,5 @@
+add_header(
+  error_number_macros
+  HDR
+    error-number-macros.h
+)
diff --git a/libc/include/llvm-libc-macros/linux/sparc/error-number-macros.h b/libc/include/llvm-libc-macros/linux/sparc/error-number-macros.h
new file mode 100644
index 000000000000..76a1408bf760
--- /dev/null
+++ b/libc/include/llvm-libc-macros/linux/sparc/error-number-macros.h
@@ -0,0 +1,24 @@
+#ifndef LLVM_LIBC_MACROS_LINUX_SPARC_ERROR_NUMBER_MACROS_H
+#define LLVM_LIBC_MACROS_LINUX_SPARC_ERROR_NUMBER_MACROS_H
+
+#ifndef ECANCELED
+#define ECANCELED 127
+#endif // ECANCELED
+
+#ifndef EOWNERDEAD
+#define EOWNERDEAD 132
+#endif // EOWNERDEAD
+
+#ifndef ENOTRECOVERABLE
+#define ENOTRECOVERABLE 133
+#endif // ENOTRECOVERABLE
+
+#ifndef ERFKILL
+#define ERFKILL 134
+#endif // ERFKILL
+
+#ifndef EHWPOISON
+#define EHWPOISON 135
+#endif // EHWPOISON
+
+#endif // LLVM_LIBC_MACROS_LINUX_SPARC_ERROR_NUMBER_MACROS_H
diff --git a/libc/src/errno/CMakeLists.txt b/libc/src/errno/CMakeLists.txt
index d9b8d9957c17..2622e51261cc 100644
--- a/libc/src/errno/CMakeLists.txt
+++ b/libc/src/errno/CMakeLists.txt
@@ -18,6 +18,6 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     ${full_build_flag}
   DEPENDS
-    libc.include.errno
+    libc.hdr.errno_macros
     libc.src.__support.common
 )
diff --git a/libc/src/errno/libc_errno.cpp b/libc/src/errno/libc_errno.cpp
index 30b0a67a3241..a59e6c34029d 100644
--- a/libc/src/errno/libc_errno.cpp
+++ b/libc/src/errno/libc_errno.cpp
@@ -37,7 +37,7 @@ LIBC_NAMESPACE::Errno::operator int() { return __llvmlibc_errno; }
 
 #else
 // In overlay mode, we simply use the system errno.
-#include <errno.h>
+#include "hdr/errno_macros.h"
 
 void LIBC_NAMESPACE::Errno::operator=(int a) { errno = a; }
 LIBC_NAMESPACE::Errno::operator int() { return errno; }
diff --git a/libc/src/errno/libc_errno.h b/libc/src/errno/libc_errno.h
index 5afc0a41d348..df67ea3b42fa 100644
--- a/libc/src/errno/libc_errno.h
+++ b/libc/src/errno/libc_errno.h
@@ -12,11 +12,7 @@
 #include "src/__support/macros/attributes.h"
 #include "src/__support/macros/properties/architectures.h"
 
-// TODO: https://github.com/llvm/llvm-project/issues/80172
-// Separate just the definition of errno numbers in
-// include/llvm-libc-macros/* and only include that instead of the system
-// <errno.h>.
-#include <errno.h>
+#include "hdr/errno_macros.h"
 
 // This header is to be consumed by internal implementations, in which all of
 // them should refer to `libc_errno` instead of using `errno` directly from
-- 
GitLab


From c5e67b86ef6718585120e3cabb04c1fc2d292cfb Mon Sep 17 00:00:00 2001
From: Sayan Saha <sayans@mathworks.com>
Date: Mon, 13 May 2024 18:18:20 -0400
Subject: [PATCH 0640/1206] [mlir] [tensor] Crash in getPackOpResultTypeShape
 for tensor.pack/unpack ops. (#90641)

Windows build of `mlir` with Visual Studio (19.36.32538 for x64) using
with the following command:

`cmake.exe -GNinja -DCMAKE_BUILD_TYPE=Release
-DLLVM_ENABLE_PROJECTS=mlir -DLLVM_ENABLE_EH=ON -DLLVM_ENABLE_RTTI=1
-DLLVM_TARGETS_TO_BUILD=host ../llvm`

is leading to a crash when calling canonicalization on
`tensor.pack`/`tensor.unpack` ops `mlir-opt --canonicalize input.mlir`
where the `input.mlir` is as follows (this is taken from one of the
filecheck tests for `tensor.pack`):

```
func.func @pack_unpack(%arg0: tensor<128x256xf32>) -> tensor<128x256xf32> {
          %pack_dest = tensor.empty() : tensor<8x16x8x32xf32>
          %unpack_dest = tensor.empty() : tensor<128x256xf32>
          %tp = tensor.pack %arg0 outer_dims_perm = [1, 0] inner_dims_pos = [0, 1] inner_tiles = [8, 32] into %pack_dest : tensor<128x256xf32> -> tensor<8x16x8x32xf32>
          %tup = tensor.unpack %tp outer_dims_perm = [1, 0] inner_dims_pos = [0, 1] inner_tiles = [8, 32] into %unpack_dest : tensor<8x16x8x32xf32> -> tensor<128x256xf32>
          return %tup : tensor<128x256xf32>
        }
```

The crash is seemingly coming from invalid memory access during
iterating over `innerDimsPos` within `getPackOpResultTypeShape`.

This crash is also causing the following tests to fail:


```
MLIR :: Dialect/Linalg/canonicalize.mlir
MLIR :: Dialect/Linalg/data-layout-propagation.mlir
MLIR :: Dialect/Linalg/generalize-tensor-pack-tile.mlir
MLIR :: Dialect/Linalg/generalize-tensor-pack.mlir
MLIR :: Dialect/Linalg/generalize-tensor-unpack-tile.mlir
MLIR :: Dialect/Linalg/generalize-tensor-unpack.mlir
MLIR :: Dialect/Linalg/transform-lower-pack.mlir
MLIR :: Dialect/Linalg/transform-op-fuse.mlir
MLIR :: Dialect/Linalg/transform-op-pack.mlir
MLIR :: Dialect/Linalg/transform-pack-greedily.mlir
MLIR :: Dialect/Tensor/canonicalize.mlir
MLIR :: Dialect/Tensor/fold-into-pack-and-unpack.mlir
MLIR :: Dialect/Tensor/invalid.mlir
MLIR :: Dialect/Tensor/ops.mlir
MLIR :: Dialect/Tensor/simplify-pack-unpack.mlir
MLIR :: Dialect/Tensor/tiling.mlir
```
---
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index e41d59a0e0b9..414bd7459af8 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -3963,7 +3963,7 @@ static SmallVector<int64_t> getPackOpResultTypeShape(
     ArrayRef<int64_t> sourceShape, ArrayRef<int64_t> innerTileSizes,
     ArrayRef<int64_t> innerDimsPos, ArrayRef<int64_t> outerDimsPerm) {
   SmallVector<int64_t> resultShape = llvm::to_vector(sourceShape);
-  for (auto tiledDim : llvm::enumerate(innerDimsPos)) {
+  for (auto tiledDim : llvm::enumerate(llvm::to_vector(innerDimsPos))) {
     if (ShapedType::isDynamic(resultShape[tiledDim.value()]))
       continue;
     if (ShapedType::isDynamic(innerTileSizes[tiledDim.index()])) {
@@ -3992,7 +3992,7 @@ SmallVector<OpFoldResult> PackOp::getResultShape(
   AffineExpr s0, s1;
   bindSymbols(builder.getContext(), s0, s1);
   AffineExpr ceilDivExpr = s0.ceilDiv(s1);
-  for (auto tiledDim : llvm::enumerate(innerDimsPos)) {
+  for (auto tiledDim : llvm::enumerate(llvm::to_vector(innerDimsPos))) {
     resultDims[tiledDim.value()] = affine::makeComposedFoldedAffineApply(
         builder, loc, ceilDivExpr,
         {resultDims[tiledDim.value()], innerTileSizes[tiledDim.index()]});
-- 
GitLab


From 8960078765f141c770f70629a205b3ea88cd9781 Mon Sep 17 00:00:00 2001
From: lntue <35648136+lntue@users.noreply.github.com>
Date: Mon, 13 May 2024 18:31:35 -0400
Subject: [PATCH 0641/1206] [libc][errno] Include <linux/errno.h> for Linux in
 full build mode. (#92041)

---
 libc/hdr/errno_macros.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libc/hdr/errno_macros.h b/libc/hdr/errno_macros.h
index b5ef7dc2a207..198b5233d440 100644
--- a/libc/hdr/errno_macros.h
+++ b/libc/hdr/errno_macros.h
@@ -12,6 +12,8 @@
 #ifdef LIBC_FULL_BUILD
 
 #ifdef __linux__
+#include <linux/errno.h>
+
 #include "llvm-libc-macros/error-number-macros.h"
 #else // __linux__
 #include "llvm-libc-macros/generic-error-number-macros.h"
-- 
GitLab


From 943baf327409fdcb01c9d02aa3c3368f2fca114b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 13 May 2024 15:47:35 -0700
Subject: [PATCH 0642/1206] [ELF] Make compareByFilePosition a strict weak
 order

This fixes the new test linkerscript/enable-non-contiguous-regions.test
from #90007 in -stdlib=libc++ -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG builds.

adjustOutputSections does not discard the output section .potential_a
because it contained .a (which would be spilled to .actual_a).
.potential_a and .bc have the same address and will cause an assertion
failure.
---
 lld/ELF/Writer.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 8d529f2bdb9f..e8a7b19a95ee 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -1338,9 +1338,11 @@ static bool compareByFilePosition(InputSection *a, InputSection *b) {
   OutputSection *aOut = la->getParent();
   OutputSection *bOut = lb->getParent();
 
-  if (aOut != bOut)
-    return aOut->addr < bOut->addr;
-  return la->outSecOff < lb->outSecOff;
+  if (aOut == bOut)
+    return la->outSecOff < lb->outSecOff;
+  if (aOut->addr == bOut->addr)
+    return aOut->sectionIndex < bOut->sectionIndex;
+  return aOut->addr < bOut->addr;
 }
 
 template <class ELFT> void Writer<ELFT>::resolveShfLinkOrder() {
-- 
GitLab


From 531a0b67ea1ad65ea4d1a99c67fee280beeb8fbb Mon Sep 17 00:00:00 2001
From: Xiang Li <python3kgae@outlook.com>
Date: Mon, 13 May 2024 15:50:16 -0700
Subject: [PATCH 0643/1206] [DirectX] Reapply Fix DXIL part header version
 encoding (#91956)

This reapplies
https://github.com/llvm/llvm-project/commit/195d8ac26d91ca798733c3a5f58d67992d43503d
[DirectX] Fix DXIL part header version encoding. The endian issue was
fixed by
https://github.com/llvm/llvm-project/commit/f42117c8517cc928c6373bad35ebf75d94fe865b.

Move MinorVersion be the lower 8 bit.
Set DXIL version in DXContainerObjectWriter::writeObject.

Fixes #89952
---
 llvm/include/llvm/BinaryFormat/DXContainer.h |  2 +-
 llvm/include/llvm/TargetParser/Triple.h      |  2 +-
 llvm/lib/MC/MCDXContainerWriter.cpp          |  3 +++
 llvm/lib/TargetParser/Triple.cpp             |  2 ++
 llvm/test/CodeGen/DirectX/embed-dxil.ll      |  4 ++--
 llvm/unittests/Object/DXContainerTest.cpp    | 16 +++++++++-------
 6 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index 847d8103e681..013431faff27 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -103,8 +103,8 @@ struct PartHeader {
 
 struct BitcodeHeader {
   uint8_t Magic[4];     // ACSII "DXIL".
-  uint8_t MajorVersion; // DXIL version.
   uint8_t MinorVersion; // DXIL version.
+  uint8_t MajorVersion; // DXIL version.
   uint16_t Unused;
   uint32_t Offset; // Offset to LLVM bitcode (from start of header).
   uint32_t Size;   // Size of LLVM bitcode (in bytes).
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 8f9d99816931..b3bb354b38ff 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -429,7 +429,7 @@ public:
   /// (SubArch).  This should only be called with Vulkan SPIR-V triples.
   VersionTuple getVulkanVersion() const;
 
-  /// Parse the DXIL version number from the DXIL version
+  /// Parse the DXIL version number from the OSVersion and DXIL version
   /// (SubArch).  This should only be called with DXIL triples.
   VersionTuple getDXILVersion() const;
 
diff --git a/llvm/lib/MC/MCDXContainerWriter.cpp b/llvm/lib/MC/MCDXContainerWriter.cpp
index ff64c6e538ac..1d82a7ec849f 100644
--- a/llvm/lib/MC/MCDXContainerWriter.cpp
+++ b/llvm/lib/MC/MCDXContainerWriter.cpp
@@ -129,6 +129,9 @@ uint64_t DXContainerObjectWriter::writeObject(MCAssembler &Asm,
       // The program header's size field is in 32-bit words.
       Header.Size = (SectionSize + sizeof(dxbc::ProgramHeader) + 3) / 4;
       memcpy(Header.Bitcode.Magic, "DXIL", 4);
+      VersionTuple DXILVersion = TT.getDXILVersion();
+      Header.Bitcode.MajorVersion = DXILVersion.getMajor();
+      Header.Bitcode.MinorVersion = DXILVersion.getMinor().value_or(0);
       Header.Bitcode.Offset = sizeof(dxbc::BitcodeHeader);
       Header.Bitcode.Size = SectionSize;
       if (sys::IsBigEndianHost)
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index f8269a51dc0b..4fc1ff5aaa05 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -1510,6 +1510,8 @@ VersionTuple Triple::getDXILVersion() const {
   if (getArch() != dxil || getOS() != ShaderModel)
     llvm_unreachable("invalid DXIL triple");
   StringRef Arch = getArchName();
+  if (getSubArch() == NoSubArch)
+    Arch = getDXILArchNameFromShaderModel(getOSName());
   Arch.consume_front("dxilv");
   VersionTuple DXILVersion = parseVersionFromName(Arch);
   // FIXME: validate DXIL version against Shader Model version.
diff --git a/llvm/test/CodeGen/DirectX/embed-dxil.ll b/llvm/test/CodeGen/DirectX/embed-dxil.ll
index 306e5c385b5a..9f4fb19d86fa 100644
--- a/llvm/test/CodeGen/DirectX/embed-dxil.ll
+++ b/llvm/test/CodeGen/DirectX/embed-dxil.ll
@@ -42,8 +42,8 @@ define i32 @add(i32 %a, i32 %b) {
 ; DXC-NEXT:       MinorVersion:    5
 ; DXC-NEXT:       ShaderKind:      6
 ; DXC-NEXT:       Size:            [[#div(SIZE,4)]]
-; DXC-NEXT:       DXILMajorVersion: [[#]]
-; DXC-NEXT:       DXILMinorVersion: [[#]]
+; DXC-NEXT:       DXILMajorVersion: 1
+; DXC-NEXT:       DXILMinorVersion: 5
 ; DXC-NEXT:       DXILSize:        [[#SIZE - 24]]
 ; DXC-NEXT:       DXIL:            [ 0x42, 0x43, 0xC0, 0xDE,
 ; DXC:      - Name:            SFI0
diff --git a/llvm/unittests/Object/DXContainerTest.cpp b/llvm/unittests/Object/DXContainerTest.cpp
index 115de47a3cef..9da6543c520c 100644
--- a/llvm/unittests/Object/DXContainerTest.cpp
+++ b/llvm/unittests/Object/DXContainerTest.cpp
@@ -215,6 +215,8 @@ TEST(DXCFile, ParseDXILPart) {
   EXPECT_EQ(Header.getMinorVersion(), 5u);
   EXPECT_EQ(Header.ShaderKind, 5u);
   EXPECT_EQ(Header.Size, 8u);
+  EXPECT_EQ(Header.Bitcode.MajorVersion, 1u);
+  EXPECT_EQ(Header.Bitcode.MinorVersion, 5u);
 }
 
 static Expected<DXContainer>
@@ -283,8 +285,8 @@ Parts:
       MinorVersion:    0
       ShaderKind:      14
       Size:            6
-      DXILMajorVersion: 0
-      DXILMinorVersion: 1
+      DXILMajorVersion: 1
+      DXILMinorVersion: 0
       DXILSize:        0
 ...
 )";
@@ -404,8 +406,8 @@ Parts:
 //       MinorVersion:    0
 //       ShaderKind:      14
 //       Size:            6
-//       DXILMajorVersion: 0
-//       DXILMinorVersion: 1
+//       DXILMajorVersion: 1
+//       DXILMinorVersion: 0
 //       DXILSize:        0
 //   - Name:            PSV0
 //     Size:            36
@@ -520,7 +522,7 @@ TEST(DXCFile, MaliciousFiles) {
 //
 // --- !dxcontainer
 // Header:
-//   Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+//   Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
 //                      0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
 //   Version:
 //     Major:           1
@@ -534,8 +536,8 @@ TEST(DXCFile, MaliciousFiles) {
 //       MinorVersion:    0
 //       ShaderKind:      14
 //       Size:            6
-//       DXILMajorVersion: 0
-//       DXILMinorVersion: 1
+//       DXILMajorVersion: 1
+//       DXILMinorVersion: 0
 //       DXILSize:        0
 //   - Name:            PSV0
 //     Size:            100
-- 
GitLab


From 6cfac497e96978f2bfc50a00b51c198f2ed50f82 Mon Sep 17 00:00:00 2001
From: Muhammad Omair Javaid <omair.javaid@linaro.org>
Date: Tue, 14 May 2024 03:52:30 +0500
Subject: [PATCH 0644/1206] [lldb][DWARF] Mark
 delayed-definition-die-searching.test unsupported on Windows

This marks delayed-definition-die-searching.test as unsupported on
Windows. Clang uses link.exe as default linker if not marked explicitly
to use lld. When used with link.exe clang produces PDB format debug info
even when -gdwarf is specified.
This test will be unsupported until we make lldb-aarch64-windows buildbot
to use lld.
---
 .../SymbolFile/DWARF/delayed-definition-die-searching.test      | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test b/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
index 836fcd7b587b..d253981b498c 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
+++ b/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
@@ -1,5 +1,7 @@
 # Test definition DIE searching is delayed until complete type is required.
 
+# UNSUPPORTED: system-windows
+
 # RUN: split-file %s %t
 # RUN: %clangxx_host %t/main.cpp %t/t1_def.cpp -gdwarf -o %t.out
 # RUN: %lldb -b %t.out -s %t/lldb.cmd | FileCheck %s
-- 
GitLab


From ec3bc2fbbf73c834697283a7066a8efe88bd0058 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 13 May 2024 15:51:10 -0700
Subject: [PATCH 0645/1206] [RISCV] Use printRegName in
 RISCVInstPrinter::printRlist. NFC

Instead of hardcoding all of the register name strings.
---
 .../RISCV/MCTargetDesc/RISCVInstPrinter.cpp   | 35 +++++++++----------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
index 2f127238a97f..663d4bad767d 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
@@ -222,30 +222,30 @@ void RISCVInstPrinter::printRlist(const MCInst *MI, unsigned OpNo,
   O << "{";
   switch (Imm) {
   case RISCVZC::RLISTENCODE::RA:
-    markup(O, Markup::Register) << (ArchRegNames ? "x1" : "ra");
+    printRegName(O, RISCV::X1);
     break;
   case RISCVZC::RLISTENCODE::RA_S0:
-    markup(O, Markup::Register) << (ArchRegNames ? "x1" : "ra");
+    printRegName(O, RISCV::X1);
     O << ", ";
-    markup(O, Markup::Register) << (ArchRegNames ? "x8" : "s0");
+    printRegName(O, RISCV::X8);
     break;
   case RISCVZC::RLISTENCODE::RA_S0_S1:
-    markup(O, Markup::Register) << (ArchRegNames ? "x1" : "ra");
+    printRegName(O, RISCV::X1);
     O << ", ";
-    markup(O, Markup::Register) << (ArchRegNames ? "x8" : "s0");
+    printRegName(O, RISCV::X8);
     O << '-';
-    markup(O, Markup::Register) << (ArchRegNames ? "x9" : "s1");
+    printRegName(O, RISCV::X9);
     break;
   case RISCVZC::RLISTENCODE::RA_S0_S2:
-    markup(O, Markup::Register) << (ArchRegNames ? "x1" : "ra");
+    printRegName(O, RISCV::X1);
     O << ", ";
-    markup(O, Markup::Register) << (ArchRegNames ? "x8" : "s0");
+    printRegName(O, RISCV::X8);
     O << '-';
-    markup(O, Markup::Register) << (ArchRegNames ? "x9" : "s2");
     if (ArchRegNames) {
+      printRegName(O, RISCV::X9);
       O << ", ";
-      markup(O, Markup::Register) << "x18";
     }
+    printRegName(O, RISCV::X18);
     break;
   case RISCVZC::RLISTENCODE::RA_S0_S3:
   case RISCVZC::RLISTENCODE::RA_S0_S4:
@@ -255,20 +255,19 @@ void RISCVInstPrinter::printRlist(const MCInst *MI, unsigned OpNo,
   case RISCVZC::RLISTENCODE::RA_S0_S8:
   case RISCVZC::RLISTENCODE::RA_S0_S9:
   case RISCVZC::RLISTENCODE::RA_S0_S11:
-    markup(O, Markup::Register) << (ArchRegNames ? "x1" : "ra");
+    printRegName(O, RISCV::X1);
     O << ", ";
-    markup(O, Markup::Register) << (ArchRegNames ? "x8" : "s0");
+    printRegName(O, RISCV::X8);
     O << '-';
     if (ArchRegNames) {
-      markup(O, Markup::Register) << "x9";
+      printRegName(O, RISCV::X9);
       O << ", ";
-      markup(O, Markup::Register) << "x18";
+      printRegName(O, RISCV::X18);
       O << '-';
     }
-    markup(O, Markup::Register) << getRegisterName(
-        RISCV::X19 + (Imm == RISCVZC::RLISTENCODE::RA_S0_S11
-                          ? 8
-                          : Imm - RISCVZC::RLISTENCODE::RA_S0_S3));
+    printRegName(O, RISCV::X19 + (Imm == RISCVZC::RLISTENCODE::RA_S0_S11
+                                      ? 8
+                                      : Imm - RISCVZC::RLISTENCODE::RA_S0_S3));
     break;
   default:
     llvm_unreachable("invalid register list");
-- 
GitLab


From b342d18a8f0240342ea5c461145e78c6e3af92cc Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Mon, 13 May 2024 19:00:19 -0400
Subject: [PATCH 0646/1206] [libc] add timeout and clock conversion utilities
 (#91905)

This PR:

- Make `clock_gettime` a header-only library
- Add `clock_conversion` header library to allow conversion between
clocks relative to the time of call
- Add `timeout` header library to manage the absolute timeout used in
POSIX's timed locking/waiting APIs
---
 .../__support/threads/linux/CMakeLists.txt    |  2 +-
 .../src/__support/threads/linux/futex_utils.h | 11 ++--
 libc/src/__support/time/linux/CMakeLists.txt  | 33 +++++++++-
 libc/src/__support/time/linux/abs_timeout.h   | 49 +++++++++++++++
 .../__support/time/linux/clock_conversion.h   | 42 +++++++++++++
 .../__support/time/linux/clock_gettime.cpp    | 35 -----------
 libc/src/__support/time/linux/clock_gettime.h | 25 +++++++-
 libc/src/__support/time/linux/monotonicity.h  | 43 +++++++++++++
 libc/test/src/__support/CMakeLists.txt        |  1 +
 libc/test/src/__support/time/CMakeLists.txt   |  5 ++
 .../src/__support/time/linux/CMakeLists.txt   |  9 +++
 .../src/__support/time/linux/timeout_test.cpp | 60 +++++++++++++++++++
 12 files changed, 267 insertions(+), 48 deletions(-)
 create mode 100644 libc/src/__support/time/linux/abs_timeout.h
 create mode 100644 libc/src/__support/time/linux/clock_conversion.h
 delete mode 100644 libc/src/__support/time/linux/clock_gettime.cpp
 create mode 100644 libc/src/__support/time/linux/monotonicity.h
 create mode 100644 libc/test/src/__support/time/CMakeLists.txt
 create mode 100644 libc/test/src/__support/time/linux/CMakeLists.txt
 create mode 100644 libc/test/src/__support/time/linux/timeout_test.cpp

diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index 9bee30206f1b..d3353f6b3ff8 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -19,7 +19,7 @@ add_header_library(
     libc.src.__support.CPP.atomic
     libc.src.__support.CPP.limits
     libc.src.__support.CPP.optional
-    libc.hdr.types.struct_timespec
+    libc.src.__support.time.linux.abs_timeout
 )
 
 add_header_library(
diff --git a/libc/src/__support/threads/linux/futex_utils.h b/libc/src/__support/threads/linux/futex_utils.h
index 1fbce4f7bf43..e40ade8e709b 100644
--- a/libc/src/__support/threads/linux/futex_utils.h
+++ b/libc/src/__support/threads/linux/futex_utils.h
@@ -9,23 +9,20 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_UTILS_H
 #define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_UTILS_H
 
-#include "hdr/types/struct_timespec.h"
 #include "src/__support/CPP/atomic.h"
 #include "src/__support/CPP/limits.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/OSUtil/syscall.h"
 #include "src/__support/macros/attributes.h"
 #include "src/__support/threads/linux/futex_word.h"
+#include "src/__support/time/linux/abs_timeout.h"
 #include <linux/errno.h>
 #include <linux/futex.h>
 
 namespace LIBC_NAMESPACE {
 class Futex : public cpp::Atomic<FutexWordType> {
 public:
-  struct Timeout {
-    timespec abs_time;
-    bool is_realtime;
-  };
+  using Timeout = internal::AbsTimeout;
   LIBC_INLINE constexpr Futex(FutexWordType value)
       : cpp::Atomic<FutexWordType>(value) {}
   LIBC_INLINE Futex &operator=(FutexWordType value) {
@@ -37,7 +34,7 @@ public:
                         bool is_shared = false) {
     // use bitset variants to enforce abs_time
     uint32_t op = is_shared ? FUTEX_WAIT_BITSET : FUTEX_WAIT_BITSET_PRIVATE;
-    if (timeout && timeout->is_realtime) {
+    if (timeout && timeout->is_realtime()) {
       op |= FUTEX_CLOCK_REALTIME;
     }
     for (;;) {
@@ -49,7 +46,7 @@ public:
           /* futex address */ this,
           /* futex operation  */ op,
           /* expected value */ expected,
-          /* timeout */ timeout ? &timeout->abs_time : nullptr,
+          /* timeout */ timeout ? &timeout->get_timespec() : nullptr,
           /* ignored */ nullptr,
           /* bitset */ FUTEX_BITSET_MATCH_ANY);
 
diff --git a/libc/src/__support/time/linux/CMakeLists.txt b/libc/src/__support/time/linux/CMakeLists.txt
index f04d550555e1..1b41c7cb0a98 100644
--- a/libc/src/__support/time/linux/CMakeLists.txt
+++ b/libc/src/__support/time/linux/CMakeLists.txt
@@ -1,9 +1,7 @@
-add_object_library(
+add_header_library(
   clock_gettime
   HDRS
     clock_gettime.h
-  SRCS
-    clock_gettime.cpp
   DEPENDS
     libc.include.sys_syscall
     libc.hdr.types.struct_timespec
@@ -12,3 +10,32 @@ add_object_library(
     libc.src.__support.error_or
     libc.src.__support.OSUtil.osutil
 )
+
+add_header_library(
+  clock_conversion
+  HDRS
+    clock_conversion.h
+  DEPENDS
+    .clock_gettime
+    libc.src.__support.time.units
+)
+
+add_header_library(
+  abs_timeout
+  HDRS
+    abs_timeout.h
+  DEPENDS
+    libc.hdr.types.struct_timespec
+    libc.src.__support.time.units
+    libc.src.__support.CPP.expected
+)
+
+add_header_library(
+  monotonicity
+  HDRS
+    monotonicity.h
+  DEPENDS
+    .clock_conversion
+    .abs_timeout
+    libc.hdr.time_macros
+)
diff --git a/libc/src/__support/time/linux/abs_timeout.h b/libc/src/__support/time/linux/abs_timeout.h
new file mode 100644
index 000000000000..6e5e59b32b7a
--- /dev/null
+++ b/libc/src/__support/time/linux/abs_timeout.h
@@ -0,0 +1,49 @@
+//===--- Linux absolute timeout ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_ABS_TIMEOUT_H
+#define LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_ABS_TIMEOUT_H
+
+#include "hdr/time_macros.h"
+#include "hdr/types/struct_timespec.h"
+#include "src/__support/CPP/expected.h"
+#include "src/__support/time/units.h"
+
+namespace LIBC_NAMESPACE {
+namespace internal {
+// We use AbsTimeout to remind ourselves that the timeout is an absolute time.
+// This is a simple wrapper around the timespec struct that also keeps track of
+// whether the time is in realtime or monotonic time.
+class AbsTimeout {
+  timespec timeout;
+  bool realtime_flag;
+  LIBC_INLINE constexpr explicit AbsTimeout(timespec ts, bool realtime)
+      : timeout(ts), realtime_flag(realtime) {}
+
+public:
+  enum class Error { Invalid, BeforeEpoch };
+  LIBC_INLINE const timespec &get_timespec() const { return timeout; }
+  LIBC_INLINE bool is_realtime() const { return realtime_flag; }
+  LIBC_INLINE static constexpr cpp::expected<AbsTimeout, Error>
+  from_timespec(timespec ts, bool realtime) {
+    using namespace time_units;
+    if (ts.tv_nsec < 0 || ts.tv_nsec >= 1_s_ns)
+      return cpp::unexpected<Error>(Error::Invalid);
+
+    // POSIX allows tv_sec to be negative. We interpret this as an expired
+    // timeout.
+    if (ts.tv_sec < 0)
+      return cpp::unexpected<Error>(Error::BeforeEpoch);
+
+    return AbsTimeout{ts, realtime};
+  }
+};
+} // namespace internal
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_ABS_TIMEOUT_H
diff --git a/libc/src/__support/time/linux/clock_conversion.h b/libc/src/__support/time/linux/clock_conversion.h
new file mode 100644
index 000000000000..4a7c8ff28484
--- /dev/null
+++ b/libc/src/__support/time/linux/clock_conversion.h
@@ -0,0 +1,42 @@
+//===--- clock conversion linux implementation ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_CONVERSION_H
+#define LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_CONVERSION_H
+
+#include "src/__support/time/linux/clock_gettime.h"
+#include "src/__support/time/units.h"
+
+namespace LIBC_NAMESPACE {
+namespace internal {
+
+LIBC_INLINE timespec convert_clock(timespec input, clockid_t from,
+                                   clockid_t to) {
+  using namespace time_units;
+  timespec from_time;
+  timespec to_time;
+  timespec output;
+  internal::clock_gettime(from, &from_time);
+  internal::clock_gettime(to, &to_time);
+  output.tv_sec = input.tv_sec - from_time.tv_sec + to_time.tv_sec;
+  output.tv_nsec = input.tv_nsec - from_time.tv_nsec + to_time.tv_nsec;
+
+  if (output.tv_nsec > 1_s_ns) {
+    output.tv_sec++;
+    output.tv_nsec -= 1_s_ns;
+  } else if (output.tv_nsec < 0) {
+    output.tv_sec--;
+    output.tv_nsec += 1_s_ns;
+  }
+  return output;
+}
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_CONVERSION_H
diff --git a/libc/src/__support/time/linux/clock_gettime.cpp b/libc/src/__support/time/linux/clock_gettime.cpp
deleted file mode 100644
index 7f266b282a39..000000000000
--- a/libc/src/__support/time/linux/clock_gettime.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-//===--- clock_gettime linux implementation ---------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/__support/time/linux/clock_gettime.h"
-#include "src/__support/OSUtil/syscall.h"
-#include <sys/syscall.h>
-namespace LIBC_NAMESPACE {
-namespace internal {
-ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts) {
-#if SYS_clock_gettime
-  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_clock_gettime,
-                                              static_cast<long>(clockid),
-                                              reinterpret_cast<long>(ts));
-#elif defined(SYS_clock_gettime64)
-  static_assert(
-      sizeof(time_t) == sizeof(int64_t),
-      "SYS_clock_gettime64 requires struct timespec with 64-bit members.");
-  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_clock_gettime64,
-                                              static_cast<long>(clockid),
-                                              reinterpret_cast<long>(ts));
-#else
-#error "SYS_clock_gettime and SYS_clock_gettime64 syscalls not available."
-#endif
-  if (ret < 0)
-    return Error(-ret);
-  return ret;
-}
-
-} // namespace internal
-} // namespace LIBC_NAMESPACE
diff --git a/libc/src/__support/time/linux/clock_gettime.h b/libc/src/__support/time/linux/clock_gettime.h
index b1572726f630..bbdde98551ab 100644
--- a/libc/src/__support/time/linux/clock_gettime.h
+++ b/libc/src/__support/time/linux/clock_gettime.h
@@ -8,16 +8,37 @@
 
 #ifndef LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
 #define LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
+
 #include "hdr/types/clockid_t.h"
 #include "hdr/types/struct_timespec.h"
+#include "src/__support/OSUtil/syscall.h"
 #include "src/__support/common.h"
-
 #include "src/__support/error_or.h"
+#include <sys/syscall.h>
 
 namespace LIBC_NAMESPACE {
 namespace internal {
-ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts);
+LIBC_INLINE ErrorOr<int> clock_gettime(clockid_t clockid, timespec *ts) {
+#if SYS_clock_gettime
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_clock_gettime,
+                                              static_cast<long>(clockid),
+                                              reinterpret_cast<long>(ts));
+#elif defined(SYS_clock_gettime64)
+  static_assert(
+      sizeof(time_t) == sizeof(int64_t),
+      "SYS_clock_gettime64 requires struct timespec with 64-bit members.");
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_clock_gettime64,
+                                              static_cast<long>(clockid),
+                                              reinterpret_cast<long>(ts));
+#else
+#error "SYS_clock_gettime and SYS_clock_gettime64 syscalls not available."
+#endif
+  if (ret < 0)
+    return Error(-ret);
+  return ret;
 }
+
+} // namespace internal
 } // namespace LIBC_NAMESPACE
 
 #endif // LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_CLOCK_GETTIME_H
diff --git a/libc/src/__support/time/linux/monotonicity.h b/libc/src/__support/time/linux/monotonicity.h
new file mode 100644
index 000000000000..e413275430dd
--- /dev/null
+++ b/libc/src/__support/time/linux/monotonicity.h
@@ -0,0 +1,43 @@
+//===--- timeout linux implementation ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_MONOTONICITY_H
+#define LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_MONOTONICITY_H
+
+#include "hdr/time_macros.h"
+#include "src/__support/libc_assert.h"
+#include "src/__support/time/linux/abs_timeout.h"
+#include "src/__support/time/linux/clock_conversion.h"
+namespace LIBC_NAMESPACE {
+namespace internal {
+// This function is separated from abs_timeout.
+// This function pulls in the dependency to clock_conversion.h,
+// which may transitively depend on vDSO hence futex. However, this structure
+// would be passed to futex, so we need to avoid cyclic dependencies.
+// This function is going to be used in timed locks. Pthread generally uses
+// realtime clocks for timeouts. However, due to non-monotoncity, realtime
+// clocks reportedly lead to undesired behaviors. Therefore, we also provide a
+// method to convert the timespec to a monotonic clock relative to the time of
+// function call.
+LIBC_INLINE void ensure_monotonicity(AbsTimeout &timeout) {
+  if (timeout.is_realtime()) {
+    auto res = AbsTimeout::from_timespec(
+        convert_clock(timeout.get_timespec(), CLOCK_REALTIME, CLOCK_MONOTONIC),
+        false);
+
+    LIBC_ASSERT(res.has_value());
+    if (!res.has_value())
+      __builtin_unreachable();
+
+    timeout = *res;
+  }
+}
+} // namespace internal
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_TIME_LINUX_MONOTONICITY_H
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 5d1230f5f3a7..8bdc56ee59cc 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -206,3 +206,4 @@ add_subdirectory(OSUtil)
 add_subdirectory(FPUtil)
 add_subdirectory(fixed_point)
 add_subdirectory(HashTable)
+add_subdirectory(time)
diff --git a/libc/test/src/__support/time/CMakeLists.txt b/libc/test/src/__support/time/CMakeLists.txt
new file mode 100644
index 000000000000..37062e131acf
--- /dev/null
+++ b/libc/test/src/__support/time/CMakeLists.txt
@@ -0,0 +1,5 @@
+add_custom_target(libc-support-time-tests)
+
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${LIBC_TARGET_OS})
+endif()
diff --git a/libc/test/src/__support/time/linux/CMakeLists.txt b/libc/test/src/__support/time/linux/CMakeLists.txt
new file mode 100644
index 000000000000..3174986283dd
--- /dev/null
+++ b/libc/test/src/__support/time/linux/CMakeLists.txt
@@ -0,0 +1,9 @@
+add_libc_test(
+  timeout_test
+  SUITE libc-support-time-tests
+  SRCS timeout_test.cpp
+  DEPENDS
+    libc.src.__support.time.linux.abs_timeout
+    libc.src.__support.time.linux.monotonicity
+    libc.src.__support.CPP.expected
+)
diff --git a/libc/test/src/__support/time/linux/timeout_test.cpp b/libc/test/src/__support/time/linux/timeout_test.cpp
new file mode 100644
index 000000000000..886d4389709e
--- /dev/null
+++ b/libc/test/src/__support/time/linux/timeout_test.cpp
@@ -0,0 +1,60 @@
+//===-- unit tests for linux's timeout utilities --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/expected.h"
+#include "src/__support/time/linux/abs_timeout.h"
+#include "src/__support/time/linux/monotonicity.h"
+#include "test/UnitTest/Test.h"
+
+template <class T, class E>
+using expected = LIBC_NAMESPACE::cpp::expected<T, E>;
+using AbsTimeout = LIBC_NAMESPACE::internal::AbsTimeout;
+
+TEST(LlvmLibcSupportLinuxTimeoutTest, NegativeSecond) {
+  timespec ts = {-1, 0};
+  expected<AbsTimeout, AbsTimeout::Error> result =
+      AbsTimeout::from_timespec(ts, false);
+  ASSERT_FALSE(result.has_value());
+  ASSERT_EQ(result.error(), AbsTimeout::Error::BeforeEpoch);
+}
+TEST(LlvmLibcSupportLinuxTimeoutTest, OverflowNano) {
+  using namespace LIBC_NAMESPACE::time_units;
+  timespec ts = {0, 2_s_ns};
+  expected<AbsTimeout, AbsTimeout::Error> result =
+      AbsTimeout::from_timespec(ts, false);
+  ASSERT_FALSE(result.has_value());
+  ASSERT_EQ(result.error(), AbsTimeout::Error::Invalid);
+}
+TEST(LlvmLibcSupportLinuxTimeoutTest, UnderflowNano) {
+  timespec ts = {0, -1};
+  expected<AbsTimeout, AbsTimeout::Error> result =
+      AbsTimeout::from_timespec(ts, false);
+  ASSERT_FALSE(result.has_value());
+  ASSERT_EQ(result.error(), AbsTimeout::Error::Invalid);
+}
+TEST(LlvmLibcSupportLinuxTimeoutTest, NoChangeIfClockIsMonotonic) {
+  timespec ts = {10000, 0};
+  expected<AbsTimeout, AbsTimeout::Error> result =
+      AbsTimeout::from_timespec(ts, false);
+  ASSERT_TRUE(result.has_value());
+  ensure_monotonicity(*result);
+  ASSERT_FALSE(result->is_realtime());
+  ASSERT_EQ(result->get_timespec().tv_sec, static_cast<time_t>(10000));
+  ASSERT_EQ(result->get_timespec().tv_nsec, static_cast<time_t>(0));
+}
+TEST(LlvmLibcSupportLinuxTimeoutTest, ValidAfterConversion) {
+  timespec ts;
+  LIBC_NAMESPACE::internal::clock_gettime(CLOCK_REALTIME, &ts);
+  expected<AbsTimeout, AbsTimeout::Error> result =
+      AbsTimeout::from_timespec(ts, true);
+  ASSERT_TRUE(result.has_value());
+  ensure_monotonicity(*result);
+  ASSERT_FALSE(result->is_realtime());
+  ASSERT_TRUE(
+      AbsTimeout::from_timespec(result->get_timespec(), false).has_value());
+}
-- 
GitLab


From 5b6f15110422f4955212bd26a96057972e3304ad Mon Sep 17 00:00:00 2001
From: Lei Wang <wlei@fb.com>
Date: Mon, 13 May 2024 16:01:29 -0700
Subject: [PATCH 0647/1206] [SampleFDO] Improve stale profile matching by diff
 algorithm (#87375)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This change improves the matching algorithm by using the diff algorithm,
the current matching algorithm only processes the callsites grouped by
the same name functions, it doesn't consider the order relationships
between different name functions, this sometimes fails to handle this
ambiguous anchor case. For example. (`Foo:1` means a
calliste[callee_name: callsite_location])
```
IR :      foo:1  bar:2  foo:4  bar:5
Profile :        bar:3  foo:5  bar:6
```
The `foo:1` is matched to the 2nd `foo:5` and using the diff
algorithm(finding longest common subsequence ) can help on this issue.
One well-known diff algorithm is the Myers diff algorithm(paper "An
O(ND) Difference Algorithm and Its Variations∗" Eugene W. Myers), its
variations have been implemented and used in many famous tools, like the
GNU diff or git diff. It provides an efficient way to find the longest
common subsequence or the shortest edit script through graph searching.
There are several variations/refinements for the algorithm, but as in
our case, the num of function callsites is usually very small, so we
implemented the basic greedy version in this change which should be good
enough.
We observed better matchings and positive perf improvement on our
internal services.
---
 .../Transforms/IPO/SampleProfileMatcher.h     |  46 ++-
 .../Transforms/IPO/SampleProfileMatcher.cpp   | 290 +++++++++++-------
 ...eudo-probe-stale-profile-matching-LCS.prof |  26 ++
 ...pseudo-probe-stale-profile-matching-LCS.ll | 219 +++++++++++++
 .../pseudo-probe-stale-profile-matching.ll    |   2 +-
 5 files changed, 457 insertions(+), 126 deletions(-)
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-stale-profile-matching-LCS.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll

diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfileMatcher.h b/llvm/include/llvm/Transforms/IPO/SampleProfileMatcher.h
index 7ae6194da7c9..b6feca5d4703 100644
--- a/llvm/include/llvm/Transforms/IPO/SampleProfileMatcher.h
+++ b/llvm/include/llvm/Transforms/IPO/SampleProfileMatcher.h
@@ -19,6 +19,9 @@
 
 namespace llvm {
 
+using AnchorList = std::vector<std::pair<LineLocation, FunctionId>>;
+using AnchorMap = std::map<LineLocation, FunctionId>;
+
 // Sample profile matching - fuzzy match.
 class SampleProfileMatcher {
   Module &M;
@@ -27,8 +30,8 @@ class SampleProfileMatcher {
   const ThinOrFullLTOPhase LTOPhase;
   SampleProfileMap FlattenedProfiles;
   // For each function, the matcher generates a map, of which each entry is a
-  // mapping from the source location of current build to the source location in
-  // the profile.
+  // mapping from the source location of current build to the source location
+  // in the profile.
   StringMap<LocToLocMap> FuncMappings;
 
   // Match state for an anchor/callsite.
@@ -95,18 +98,13 @@ private:
     return nullptr;
   }
   void runOnFunction(Function &F);
-  void findIRAnchors(const Function &F,
-                     std::map<LineLocation, StringRef> &IRAnchors);
-  void findProfileAnchors(
-      const FunctionSamples &FS,
-      std::map<LineLocation, std::unordered_set<FunctionId>> &ProfileAnchors);
+  void findIRAnchors(const Function &F, AnchorMap &IRAnchors);
+  void findProfileAnchors(const FunctionSamples &FS, AnchorMap &ProfileAnchors);
   // Record the callsite match states for profile staleness report, the result
   // is saved in FuncCallsiteMatchStates.
-  void recordCallsiteMatchStates(
-      const Function &F, const std::map<LineLocation, StringRef> &IRAnchors,
-      const std::map<LineLocation, std::unordered_set<FunctionId>>
-          &ProfileAnchors,
-      const LocToLocMap *IRToProfileLocationMap);
+  void recordCallsiteMatchStates(const Function &F, const AnchorMap &IRAnchors,
+                                 const AnchorMap &ProfileAnchors,
+                                 const LocToLocMap *IRToProfileLocationMap);
 
   bool isMismatchState(const enum MatchState &State) {
     return State == MatchState::InitialMismatch ||
@@ -143,11 +141,25 @@ private:
   }
   void distributeIRToProfileLocationMap();
   void distributeIRToProfileLocationMap(FunctionSamples &FS);
-  void runStaleProfileMatching(
-      const Function &F, const std::map<LineLocation, StringRef> &IRAnchors,
-      const std::map<LineLocation, std::unordered_set<FunctionId>>
-          &ProfileAnchors,
-      LocToLocMap &IRToProfileLocationMap);
+  // This function implements the Myers diff algorithm used for stale profile
+  // matching. The algorithm provides a simple and efficient way to find the
+  // Longest Common Subsequence(LCS) or the Shortest Edit Script(SES) of two
+  // sequences. For more details, refer to the paper 'An O(ND) Difference
+  // Algorithm and Its Variations' by Eugene W. Myers.
+  // In the scenario of profile fuzzy matching, the two sequences are the IR
+  // callsite anchors and profile callsite anchors. The subsequence equivalent
+  // parts from the resulting SES are used to remap the IR locations to the
+  // profile locations. As the number of function callsite is usually not big,
+  // we currently just implements the basic greedy version(page 6 of the paper).
+  LocToLocMap
+  longestCommonSequence(const AnchorList &IRCallsiteAnchors,
+                        const AnchorList &ProfileCallsiteAnchors) const;
+  void matchNonCallsiteLocs(const LocToLocMap &AnchorMatchings,
+                            const AnchorMap &IRAnchors,
+                            LocToLocMap &IRToProfileLocationMap);
+  void runStaleProfileMatching(const Function &F, const AnchorMap &IRAnchors,
+                               const AnchorMap &ProfileAnchors,
+                               LocToLocMap &IRToProfileLocationMap);
   void reportOrPersistProfileStats();
 };
 } // end namespace llvm
diff --git a/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp b/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
index 142660bcc58e..d7613bce4c52 100644
--- a/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfileMatcher.cpp
@@ -24,8 +24,8 @@ extern cl::opt<bool> SalvageStaleProfile;
 extern cl::opt<bool> PersistProfileStaleness;
 extern cl::opt<bool> ReportProfileStaleness;
 
-void SampleProfileMatcher::findIRAnchors(
-    const Function &F, std::map<LineLocation, StringRef> &IRAnchors) {
+void SampleProfileMatcher::findIRAnchors(const Function &F,
+                                         AnchorMap &IRAnchors) {
   // For inlined code, recover the original callsite and callee by finding the
   // top-level inline frame. e.g. For frame stack "main:1 @ foo:2 @ bar:3", the
   // top-level frame is "main:1", the callsite is "1" and the callee is "foo".
@@ -40,7 +40,7 @@ void SampleProfileMatcher::findIRAnchors(
     LineLocation Callsite = FunctionSamples::getCallSiteIdentifier(
         DIL, FunctionSamples::ProfileIsFS);
     StringRef CalleeName = PrevDIL->getSubprogramLinkageName();
-    return std::make_pair(Callsite, CalleeName);
+    return std::make_pair(Callsite, FunctionId(CalleeName));
   };
 
   auto GetCanonicalCalleeName = [](const CallBase *CB) {
@@ -70,7 +70,8 @@ void SampleProfileMatcher::findIRAnchors(
               if (!isa<IntrinsicInst>(&I))
                 CalleeName = GetCanonicalCalleeName(CB);
             }
-            IRAnchors.emplace(LineLocation(Probe->Id, 0), CalleeName);
+            LineLocation Loc = LineLocation(Probe->Id, 0);
+            IRAnchors.emplace(Loc, FunctionId(CalleeName));
           }
         }
       } else {
@@ -86,84 +87,127 @@ void SampleProfileMatcher::findIRAnchors(
           LineLocation Callsite = FunctionSamples::getCallSiteIdentifier(
               DIL, FunctionSamples::ProfileIsFS);
           StringRef CalleeName = GetCanonicalCalleeName(dyn_cast<CallBase>(&I));
-          IRAnchors.emplace(Callsite, CalleeName);
+          IRAnchors.emplace(Callsite, FunctionId(CalleeName));
         }
       }
     }
   }
 }
 
-void SampleProfileMatcher::findProfileAnchors(
-    const FunctionSamples &FS,
-    std::map<LineLocation, std::unordered_set<FunctionId>> &ProfileAnchors) {
+void SampleProfileMatcher::findProfileAnchors(const FunctionSamples &FS,
+                                              AnchorMap &ProfileAnchors) {
   auto isInvalidLineOffset = [](uint32_t LineOffset) {
     return LineOffset & 0x8000;
   };
 
+  auto InsertAnchor = [](const LineLocation &Loc, const FunctionId &CalleeName,
+                         AnchorMap &ProfileAnchors) {
+    auto Ret = ProfileAnchors.try_emplace(Loc, CalleeName);
+    if (!Ret.second) {
+      // For multiple callees, which indicates it's an indirect call, we use a
+      // dummy name(UnknownIndirectCallee) as the indrect callee name.
+      Ret.first->second = FunctionId(UnknownIndirectCallee);
+    }
+  };
+
   for (const auto &I : FS.getBodySamples()) {
     const LineLocation &Loc = I.first;
     if (isInvalidLineOffset(Loc.LineOffset))
       continue;
-    for (const auto &I : I.second.getCallTargets()) {
-      auto Ret =
-          ProfileAnchors.try_emplace(Loc, std::unordered_set<FunctionId>());
-      Ret.first->second.insert(I.first);
-    }
+    for (const auto &C : I.second.getCallTargets())
+      InsertAnchor(Loc, C.first, ProfileAnchors);
   }
 
   for (const auto &I : FS.getCallsiteSamples()) {
     const LineLocation &Loc = I.first;
     if (isInvalidLineOffset(Loc.LineOffset))
       continue;
-    const auto &CalleeMap = I.second;
-    for (const auto &I : CalleeMap) {
-      auto Ret =
-          ProfileAnchors.try_emplace(Loc, std::unordered_set<FunctionId>());
-      Ret.first->second.insert(I.first);
-    }
+    for (const auto &C : I.second)
+      InsertAnchor(Loc, C.first, ProfileAnchors);
   }
 }
 
-// Call target name anchor based profile fuzzy matching.
-// Input:
-// For IR locations, the anchor is the callee name of direct callsite; For
-// profile locations, it's the call target name for BodySamples or inlinee's
-// profile name for CallsiteSamples.
-// Matching heuristic:
-// First match all the anchors in lexical order, then split the non-anchor
-// locations between the two anchors evenly, first half are matched based on the
-// start anchor, second half are matched based on the end anchor.
-// For example, given:
-// IR locations:      [1, 2(foo), 3, 5, 6(bar), 7]
-// Profile locations: [1, 2, 3(foo), 4, 7, 8(bar), 9]
-// The matching gives:
-//   [1,    2(foo), 3,  5,  6(bar), 7]
-//    |     |       |   |     |     |
-//   [1, 2, 3(foo), 4,  7,  8(bar), 9]
-// The output mapping: [2->3, 3->4, 5->7, 6->8, 7->9].
-void SampleProfileMatcher::runStaleProfileMatching(
-    const Function &F, const std::map<LineLocation, StringRef> &IRAnchors,
-    const std::map<LineLocation, std::unordered_set<FunctionId>>
-        &ProfileAnchors,
-    LocToLocMap &IRToProfileLocationMap) {
-  LLVM_DEBUG(dbgs() << "Run stale profile matching for " << F.getName()
-                    << "\n");
-  assert(IRToProfileLocationMap.empty() &&
-         "Run stale profile matching only once per function");
+LocToLocMap SampleProfileMatcher::longestCommonSequence(
+    const AnchorList &AnchorList1, const AnchorList &AnchorList2) const {
+  int32_t Size1 = AnchorList1.size(), Size2 = AnchorList2.size(),
+          MaxDepth = Size1 + Size2;
+  auto Index = [&](int32_t I) { return I + MaxDepth; };
+
+  LocToLocMap EqualLocations;
+  if (MaxDepth == 0)
+    return EqualLocations;
+
+  // Backtrack the SES result.
+  auto Backtrack = [&](const std::vector<std::vector<int32_t>> &Trace,
+                       const AnchorList &AnchorList1,
+                       const AnchorList &AnchorList2,
+                       LocToLocMap &EqualLocations) {
+    int32_t X = Size1, Y = Size2;
+    for (int32_t Depth = Trace.size() - 1; X > 0 || Y > 0; Depth--) {
+      const auto &P = Trace[Depth];
+      int32_t K = X - Y;
+      int32_t PrevK = K;
+      if (K == -Depth || (K != Depth && P[Index(K - 1)] < P[Index(K + 1)]))
+        PrevK = K + 1;
+      else
+        PrevK = K - 1;
+
+      int32_t PrevX = P[Index(PrevK)];
+      int32_t PrevY = PrevX - PrevK;
+      while (X > PrevX && Y > PrevY) {
+        X--;
+        Y--;
+        EqualLocations.insert({AnchorList1[X].first, AnchorList2[Y].first});
+      }
 
-  std::unordered_map<FunctionId, std::set<LineLocation>> CalleeToCallsitesMap;
-  for (const auto &I : ProfileAnchors) {
-    const auto &Loc = I.first;
-    const auto &Callees = I.second;
-    // Filter out possible indirect calls, use direct callee name as anchor.
-    if (Callees.size() == 1) {
-      FunctionId CalleeName = *Callees.begin();
-      const auto &Candidates = CalleeToCallsitesMap.try_emplace(
-          CalleeName, std::set<LineLocation>());
-      Candidates.first->second.insert(Loc);
+      if (Depth == 0)
+        break;
+
+      if (Y == PrevY)
+        X--;
+      else if (X == PrevX)
+        Y--;
+      X = PrevX;
+      Y = PrevY;
+    }
+  };
+
+  // The greedy LCS/SES algorithm.
+
+  // An array contains the endpoints of the furthest reaching D-paths.
+  std::vector<int32_t> V(2 * MaxDepth + 1, -1);
+  V[Index(1)] = 0;
+  // Trace is used to backtrack the SES result.
+  std::vector<std::vector<int32_t>> Trace;
+  for (int32_t Depth = 0; Depth <= MaxDepth; Depth++) {
+    Trace.push_back(V);
+    for (int32_t K = -Depth; K <= Depth; K += 2) {
+      int32_t X = 0, Y = 0;
+      if (K == -Depth || (K != Depth && V[Index(K - 1)] < V[Index(K + 1)]))
+        X = V[Index(K + 1)];
+      else
+        X = V[Index(K - 1)] + 1;
+      Y = X - K;
+      while (X < Size1 && Y < Size2 &&
+             AnchorList1[X].second == AnchorList2[Y].second)
+        X++, Y++;
+
+      V[Index(K)] = X;
+
+      if (X >= Size1 && Y >= Size2) {
+        // Length of an SES is D.
+        Backtrack(Trace, AnchorList1, AnchorList2, EqualLocations);
+        return EqualLocations;
+      }
     }
   }
+  // Length of an SES is greater than MaxDepth.
+  return EqualLocations;
+}
 
+void SampleProfileMatcher::matchNonCallsiteLocs(
+    const LocToLocMap &MatchedAnchors, const AnchorMap &IRAnchors,
+    LocToLocMap &IRToProfileLocationMap) {
   auto InsertMatching = [&](const LineLocation &From, const LineLocation &To) {
     // Skip the unchanged location mapping to save memory.
     if (From != To)
@@ -173,43 +217,35 @@ void SampleProfileMatcher::runStaleProfileMatching(
   // Use function's beginning location as the initial anchor.
   int32_t LocationDelta = 0;
   SmallVector<LineLocation> LastMatchedNonAnchors;
-
   for (const auto &IR : IRAnchors) {
     const auto &Loc = IR.first;
-    auto CalleeName = IR.second;
     bool IsMatchedAnchor = false;
     // Match the anchor location in lexical order.
-    if (!CalleeName.empty()) {
-      auto CandidateAnchors =
-          CalleeToCallsitesMap.find(getRepInFormat(CalleeName));
-      if (CandidateAnchors != CalleeToCallsitesMap.end() &&
-          !CandidateAnchors->second.empty()) {
-        auto CI = CandidateAnchors->second.begin();
-        const auto Candidate = *CI;
-        CandidateAnchors->second.erase(CI);
-        InsertMatching(Loc, Candidate);
-        LLVM_DEBUG(dbgs() << "Callsite with callee:" << CalleeName
-                          << " is matched from " << Loc << " to " << Candidate
-                          << "\n");
-        LocationDelta = Candidate.LineOffset - Loc.LineOffset;
-
-        // Match backwards for non-anchor locations.
-        // The locations in LastMatchedNonAnchors have been matched forwards
-        // based on the previous anchor, spilt it evenly and overwrite the
-        // second half based on the current anchor.
-        for (size_t I = (LastMatchedNonAnchors.size() + 1) / 2;
-             I < LastMatchedNonAnchors.size(); I++) {
-          const auto &L = LastMatchedNonAnchors[I];
-          uint32_t CandidateLineOffset = L.LineOffset + LocationDelta;
-          LineLocation Candidate(CandidateLineOffset, L.Discriminator);
-          InsertMatching(L, Candidate);
-          LLVM_DEBUG(dbgs() << "Location is rematched backwards from " << L
-                            << " to " << Candidate << "\n");
-        }
-
-        IsMatchedAnchor = true;
-        LastMatchedNonAnchors.clear();
+    auto R = MatchedAnchors.find(Loc);
+    if (R != MatchedAnchors.end()) {
+      const auto &Candidate = R->second;
+      InsertMatching(Loc, Candidate);
+      LLVM_DEBUG(dbgs() << "Callsite with callee:" << IR.second.stringRef()
+                        << " is matched from " << Loc << " to " << Candidate
+                        << "\n");
+      LocationDelta = Candidate.LineOffset - Loc.LineOffset;
+
+      // Match backwards for non-anchor locations.
+      // The locations in LastMatchedNonAnchors have been matched forwards
+      // based on the previous anchor, spilt it evenly and overwrite the
+      // second half based on the current anchor.
+      for (size_t I = (LastMatchedNonAnchors.size() + 1) / 2;
+           I < LastMatchedNonAnchors.size(); I++) {
+        const auto &L = LastMatchedNonAnchors[I];
+        uint32_t CandidateLineOffset = L.LineOffset + LocationDelta;
+        LineLocation Candidate(CandidateLineOffset, L.Discriminator);
+        InsertMatching(L, Candidate);
+        LLVM_DEBUG(dbgs() << "Location is rematched backwards from " << L
+                          << " to " << Candidate << "\n");
       }
+
+      IsMatchedAnchor = true;
+      LastMatchedNonAnchors.clear();
     }
 
     // Match forwards for non-anchor locations.
@@ -224,6 +260,57 @@ void SampleProfileMatcher::runStaleProfileMatching(
   }
 }
 
+// Call target name anchor based profile fuzzy matching.
+// Input:
+// For IR locations, the anchor is the callee name of direct callsite; For
+// profile locations, it's the call target name for BodySamples or inlinee's
+// profile name for CallsiteSamples.
+// Matching heuristic:
+// First match all the anchors using the diff algorithm, then split the
+// non-anchor locations between the two anchors evenly, first half are matched
+// based on the start anchor, second half are matched based on the end anchor.
+// For example, given:
+// IR locations:      [1, 2(foo), 3, 5, 6(bar), 7]
+// Profile locations: [1, 2, 3(foo), 4, 7, 8(bar), 9]
+// The matching gives:
+//   [1,    2(foo), 3,  5,  6(bar), 7]
+//    |     |       |   |     |     |
+//   [1, 2, 3(foo), 4,  7,  8(bar), 9]
+// The output mapping: [2->3, 3->4, 5->7, 6->8, 7->9].
+void SampleProfileMatcher::runStaleProfileMatching(
+    const Function &F, const AnchorMap &IRAnchors,
+    const AnchorMap &ProfileAnchors, LocToLocMap &IRToProfileLocationMap) {
+  LLVM_DEBUG(dbgs() << "Run stale profile matching for " << F.getName()
+                    << "\n");
+  assert(IRToProfileLocationMap.empty() &&
+         "Run stale profile matching only once per function");
+
+  AnchorList FilteredProfileAnchorList;
+  for (const auto &I : ProfileAnchors)
+    FilteredProfileAnchorList.emplace_back(I);
+
+  AnchorList FilteredIRAnchorsList;
+  // Filter the non-callsite from IRAnchors.
+  for (const auto &I : IRAnchors) {
+    if (I.second.stringRef().empty())
+      continue;
+    FilteredIRAnchorsList.emplace_back(I);
+  }
+
+  if (FilteredIRAnchorsList.empty() || FilteredProfileAnchorList.empty())
+    return;
+
+  // Match the callsite anchors by finding the longest common subsequence
+  // between IR and profile. Note that we need to use IR anchor as base(A side)
+  // to align with the order of IRToProfileLocationMap.
+  LocToLocMap MatchedAnchors =
+      longestCommonSequence(FilteredIRAnchorsList, FilteredProfileAnchorList);
+
+  // Match the non-callsite locations and write the result to
+  // IRToProfileLocationMap.
+  matchNonCallsiteLocs(MatchedAnchors, IRAnchors, IRToProfileLocationMap);
+}
+
 void SampleProfileMatcher::runOnFunction(Function &F) {
   // We need to use flattened function samples for matching.
   // Unlike IR, which includes all callsites from the source code, the callsites
@@ -238,11 +325,11 @@ void SampleProfileMatcher::runOnFunction(Function &F) {
   // Anchors for IR. It's a map from IR location to callee name, callee name is
   // empty for non-call instruction and use a dummy name(UnknownIndirectCallee)
   // for unknown indrect callee name.
-  std::map<LineLocation, StringRef> IRAnchors;
+  AnchorMap IRAnchors;
   findIRAnchors(F, IRAnchors);
   // Anchors for profile. It's a map from callsite location to a set of callee
   // name.
-  std::map<LineLocation, std::unordered_set<FunctionId>> ProfileAnchors;
+  AnchorMap ProfileAnchors;
   findProfileAnchors(*FSFlattened, ProfileAnchors);
 
   // Compute the callsite match states for profile staleness report.
@@ -274,9 +361,8 @@ void SampleProfileMatcher::runOnFunction(Function &F) {
 }
 
 void SampleProfileMatcher::recordCallsiteMatchStates(
-    const Function &F, const std::map<LineLocation, StringRef> &IRAnchors,
-    const std::map<LineLocation, std::unordered_set<FunctionId>>
-        &ProfileAnchors,
+    const Function &F, const AnchorMap &IRAnchors,
+    const AnchorMap &ProfileAnchors,
     const LocToLocMap *IRToProfileLocationMap) {
   bool IsPostMatch = IRToProfileLocationMap != nullptr;
   auto &CallsiteMatchStates =
@@ -297,23 +383,12 @@ void SampleProfileMatcher::recordCallsiteMatchStates(
     // After fuzzy profile matching, use the matching result to remap the
     // current IR callsite.
     const auto &ProfileLoc = MapIRLocToProfileLoc(I.first);
-    const auto &IRCalleeName = I.second;
+    const auto &IRCalleeId = I.second;
     const auto &It = ProfileAnchors.find(ProfileLoc);
     if (It == ProfileAnchors.end())
       continue;
-    const auto &Callees = It->second;
-
-    bool IsCallsiteMatched = false;
-    // Since indirect call does not have CalleeName, check conservatively if
-    // callsite in the profile is a callsite location. This is to reduce num of
-    // false positive since otherwise all the indirect call samples will be
-    // reported as mismatching.
-    if (IRCalleeName == SampleProfileMatcher::UnknownIndirectCallee)
-      IsCallsiteMatched = true;
-    else if (Callees.size() == 1 && Callees.count(getRepInFormat(IRCalleeName)))
-      IsCallsiteMatched = true;
-
-    if (IsCallsiteMatched) {
+    const auto &ProfCalleeId = It->second;
+    if (IRCalleeId == ProfCalleeId) {
       auto It = CallsiteMatchStates.find(ProfileLoc);
       if (It == CallsiteMatchStates.end())
         CallsiteMatchStates.emplace(ProfileLoc, MatchState::InitialMatch);
@@ -330,8 +405,7 @@ void SampleProfileMatcher::recordCallsiteMatchStates(
   // IR callsites.
   for (const auto &I : ProfileAnchors) {
     const auto &Loc = I.first;
-    [[maybe_unused]] const auto &Callees = I.second;
-    assert(!Callees.empty() && "Callees should not be empty");
+    assert(!I.second.stringRef().empty() && "Callees should not be empty");
     auto It = CallsiteMatchStates.find(Loc);
     if (It == CallsiteMatchStates.end())
       CallsiteMatchStates.emplace(Loc, MatchState::InitialMismatch);
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-stale-profile-matching-LCS.prof b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-stale-profile-matching-LCS.prof
new file mode 100644
index 000000000000..e56c7c01865d
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-stale-profile-matching-LCS.prof
@@ -0,0 +1,26 @@
+test_direct_call:606:83
+ 1: 83
+ 2: 83 C:83
+ 3: 90 B:90
+ 4: 83 A:83
+ 5: 92 B:92
+ 6: 83 A:83
+ 7: 97 C:97
+ !CFGChecksum: 123456
+test_indirect_call:589:86
+ 1: 86
+ 2: 86 C:86
+ 3: 83 A:43 B:40
+ 4: 84 B:84
+ 6: 82 B:62 A:20
+ 7: 91 C:91
+ !CFGChecksum: 123456
+main:403:0
+ 1: 0
+ 2: 80
+ 3: 80
+ 4: 86 test_indirect_call:86
+ 5: 83 test_direct_call:83
+ 6: 83
+ 7: 0
+ !CFGChecksum: 563036051115663
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll
new file mode 100644
index 000000000000..ecf8484d98e5
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll
@@ -0,0 +1,219 @@
+; REQUIRES: x86_64-linux
+; REQUIRES: asserts
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-stale-profile-matching-LCS.prof --salvage-stale-profile -S --debug-only=sample-profile,sample-profile-matcher,sample-profile-impl 2>&1 | FileCheck %s
+
+; CHECK: Run stale profile matching for test_direct_call
+; CHECK: Location is matched from 1 to 1
+; CHECK: Location is matched from 2 to 2
+; CHECK: Location is matched from 3 to 3
+; CHECK: Callsite with callee:C is matched from 4 to 2
+; CHECK: Location is rematched backwards from 3 to 1
+; CHECK: Callsite with callee:A is matched from 5 to 4
+; CHECK: Callsite with callee:B is matched from 6 to 5
+; CHECK: Location is matched from 7 to 6
+; CHECK: Callsite with callee:A is matched from 8 to 6
+
+; CHECK: Run stale profile matching for test_indirect_call
+; CHECK: Location is matched from 1 to 1
+; CHECK: Location is matched from 2 to 2
+; CHECK: Location is matched from 3 to 3
+; CHECK: Location is matched from 4 to 4
+; CHECK: Callsite with callee:C is matched from 5 to 2
+; CHECK: Location is rematched backwards from 3 to 0
+; CHECK: Location is rematched backwards from 4 to 1
+; CHECK: Callsite with callee:unknown.indirect.callee is matched from 6 to 3
+; CHECK:Callsite with callee:B is matched from 7 to 4
+; CHECK: Location is matched from 8 to 5
+; CHECK: Callsite with callee:unknown.indirect.callee is matched from 9 to 6
+; CHECK: Callsite with callee:C is matched from 10 to 7
+
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@c = external global i32, align 4
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @test_direct_call(i32 noundef %x) #0 !dbg !12 {
+entry:
+    #dbg_value(i32 %x, !17, !DIExpression(), !18)
+  call void @llvm.pseudoprobe(i64 -4364451034228175269, i64 1, i32 0, i64 -1), !dbg !19
+  %call = call i32 @A(i32 noundef %x), !dbg !20
+  %add = add nsw i32 %x, %call, !dbg !22
+    #dbg_value(i32 %add, !17, !DIExpression(), !18)
+  %call1 = call i32 @B(i32 noundef %add), !dbg !23
+  %add2 = add nsw i32 %add, %call1, !dbg !25
+    #dbg_value(i32 %add2, !17, !DIExpression(), !18)
+  %call3 = call i32 @C(i32 noundef %add2), !dbg !26
+  %add4 = add nsw i32 %add2, %call3, !dbg !28
+    #dbg_value(i32 %add4, !17, !DIExpression(), !18)
+  %call5 = call i32 @A(i32 noundef %add4), !dbg !29
+  %add6 = add nsw i32 %add4, %call5, !dbg !31
+    #dbg_value(i32 %add6, !17, !DIExpression(), !18)
+  %call7 = call i32 @B(i32 noundef %add6), !dbg !32
+  %add8 = add nsw i32 %add6, %call7, !dbg !34
+    #dbg_value(i32 %add8, !17, !DIExpression(), !18)
+  %call9 = call i32 @B(i32 noundef %add8), !dbg !35
+  %add10 = add nsw i32 %add8, %call9, !dbg !37
+    #dbg_value(i32 %add10, !17, !DIExpression(), !18)
+  %call11 = call i32 @A(i32 noundef %add10), !dbg !38
+  %add12 = add nsw i32 %add10, %call11, !dbg !40
+    #dbg_value(i32 %add12, !17, !DIExpression(), !18)
+  ret i32 %add12, !dbg !41
+}
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+declare !dbg !42 i32 @A(i32 noundef) #2
+
+declare !dbg !43 i32 @B(i32 noundef) #2
+
+declare !dbg !44 i32 @C(i32 noundef) #2
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @test_indirect_call(i32 noundef %x) #0 !dbg !45 {
+entry:
+    #dbg_value(i32 %x, !47, !DIExpression(), !50)
+  call void @llvm.pseudoprobe(i64 -8563147518712133441, i64 1, i32 0, i64 -1), !dbg !51
+  %0 = load i32, ptr @c, align 4, !dbg !51, !tbaa !53
+  %tobool = icmp ne i32 %0, 0, !dbg !51
+  br i1 %tobool, label %if.then, label %if.else, !dbg !57
+
+if.then:                                          ; preds = %entry
+  call void @llvm.pseudoprobe(i64 -8563147518712133441, i64 2, i32 0, i64 -1), !dbg !58
+    #dbg_value(ptr @A, !48, !DIExpression(), !50)
+  br label %if.end, !dbg !59
+
+if.else:                                          ; preds = %entry
+  call void @llvm.pseudoprobe(i64 -8563147518712133441, i64 3, i32 0, i64 -1), !dbg !60
+    #dbg_value(ptr @B, !48, !DIExpression(), !50)
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %fp.0 = phi ptr [ @A, %if.then ], [ @B, %if.else ], !dbg !61
+    #dbg_value(ptr %fp.0, !48, !DIExpression(), !50)
+  call void @llvm.pseudoprobe(i64 -8563147518712133441, i64 4, i32 0, i64 -1), !dbg !62
+  %call = call i32 @C(i32 noundef %x), !dbg !63
+  %add = add nsw i32 %x, %call, !dbg !65
+    #dbg_value(i32 %add, !47, !DIExpression(), !50)
+  %call1 = call i32 %fp.0(i32 noundef %add), !dbg !66
+  %add2 = add nsw i32 %add, %call1, !dbg !68
+    #dbg_value(i32 %add2, !47, !DIExpression(), !50)
+  %call3 = call i32 @B(i32 noundef %add2), !dbg !69
+  %add4 = add nsw i32 %add2, %call3, !dbg !71
+    #dbg_value(i32 %add4, !47, !DIExpression(), !50)
+  %call5 = call i32 @C(i32 noundef %add4), !dbg !72
+  %add6 = add nsw i32 %add4, %call5, !dbg !74
+    #dbg_value(i32 %add6, !47, !DIExpression(), !50)
+  %call7 = call i32 %fp.0(i32 noundef %add6), !dbg !75
+  %add8 = add nsw i32 %add6, %call7, !dbg !77
+    #dbg_value(i32 %add8, !47, !DIExpression(), !50)
+  %call9 = call i32 @C(i32 noundef %add8), !dbg !78
+  %add10 = add nsw i32 %add8, %call9, !dbg !80
+    #dbg_value(i32 %add10, !47, !DIExpression(), !50)
+  ret i32 %add10, !dbg !81
+}
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #3
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #3
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
+declare void @llvm.pseudoprobe(i64, i64, i32, i64) #4
+
+attributes #0 = { nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "use-sample-profile" }
+attributes #1 = { mustprogress nocallback nofree nosync nounwind speculatable willreturn }
+attributes #2 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
+attributes #3 = { mustprogress nocallback nofree nosync nounwind willreturn }
+attributes #4 = { mustprogress nocallback nofree nosync nounwind willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8}
+!llvm.ident = !{!9}
+!llvm.pseudo_probe_desc = !{!10, !11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 19.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, debugInfoForProfiling: true, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/home/", checksumkind: CSK_MD5, checksum: "be98aa946f37f0ad8d307c9121efe101")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 8, !"PIC Level", i32 2}
+!6 = !{i32 7, !"PIE Level", i32 2}
+!7 = !{i32 7, !"uwtable", i32 2}
+!8 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
+!9 = !{!"clang version 19.0.0"}
+!10 = !{i64 -4364451034228175269, i64 1970329131941887, !"test_direct_call"}
+!11 = !{i64 -8563147518712133441, i64 1688922477484692, !"test_indirect_call"}
+!12 = distinct !DISubprogram(name: "test_direct_call", scope: !1, file: !1, line: 10, type: !13, scopeLine: 10, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !16)
+!13 = !DISubroutineType(types: !14)
+!14 = !{!15, !15}
+!15 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!16 = !{!17}
+!17 = !DILocalVariable(name: "x", arg: 1, scope: !12, file: !1, line: 10, type: !15)
+!18 = !DILocation(line: 0, scope: !12)
+!19 = !DILocation(line: 11, column: 10, scope: !12)
+!20 = !DILocation(line: 11, column: 8, scope: !21)
+!21 = !DILexicalBlockFile(scope: !12, file: !1, discriminator: 186646551)
+!22 = !DILocation(line: 11, column: 5, scope: !12)
+!23 = !DILocation(line: 12, column: 8, scope: !24)
+!24 = !DILexicalBlockFile(scope: !12, file: !1, discriminator: 186646559)
+!25 = !DILocation(line: 12, column: 5, scope: !12)
+!26 = !DILocation(line: 13, column: 8, scope: !27)
+!27 = !DILexicalBlockFile(scope: !12, file: !1, discriminator: 186646567)
+!28 = !DILocation(line: 13, column: 5, scope: !12)
+!29 = !DILocation(line: 14, column: 8, scope: !30)
+!30 = !DILexicalBlockFile(scope: !12, file: !1, discriminator: 186646575)
+!31 = !DILocation(line: 14, column: 5, scope: !12)
+!32 = !DILocation(line: 15, column: 8, scope: !33)
+!33 = !DILexicalBlockFile(scope: !12, file: !1, discriminator: 186646583)
+!34 = !DILocation(line: 15, column: 5, scope: !12)
+!35 = !DILocation(line: 16, column: 8, scope: !36)
+!36 = !DILexicalBlockFile(scope: !12, file: !1, discriminator: 186646591)
+!37 = !DILocation(line: 16, column: 5, scope: !12)
+!38 = !DILocation(line: 17, column: 8, scope: !39)
+!39 = !DILexicalBlockFile(scope: !12, file: !1, discriminator: 186646599)
+!40 = !DILocation(line: 17, column: 5, scope: !12)
+!41 = !DILocation(line: 18, column: 3, scope: !12)
+!42 = !DISubprogram(name: "A", scope: !1, file: !1, line: 2, type: !13, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized)
+!43 = !DISubprogram(name: "B", scope: !1, file: !1, line: 3, type: !13, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized)
+!44 = !DISubprogram(name: "C", scope: !1, file: !1, line: 4, type: !13, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized)
+!45 = distinct !DISubprogram(name: "test_indirect_call", scope: !1, file: !1, line: 21, type: !13, scopeLine: 21, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !46)
+!46 = !{!47, !48}
+!47 = !DILocalVariable(name: "x", arg: 1, scope: !45, file: !1, line: 21, type: !15)
+!48 = !DILocalVariable(name: "fp", scope: !45, file: !1, line: 22, type: !49)
+!49 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!50 = !DILocation(line: 0, scope: !45)
+!51 = !DILocation(line: 23, column: 6, scope: !52)
+!52 = distinct !DILexicalBlock(scope: !45, file: !1, line: 23, column: 6)
+!53 = !{!54, !54, i64 0}
+!54 = !{!"int", !55, i64 0}
+!55 = !{!"omnipotent char", !56, i64 0}
+!56 = !{!"Simple C/C++ TBAA"}
+!57 = !DILocation(line: 23, column: 6, scope: !45)
+!58 = !DILocation(line: 24, column: 8, scope: !52)
+!59 = !DILocation(line: 24, column: 5, scope: !52)
+!60 = !DILocation(line: 26, column: 8, scope: !52)
+!61 = !DILocation(line: 0, scope: !52)
+!62 = !DILocation(line: 27, column: 10, scope: !45)
+!63 = !DILocation(line: 27, column: 8, scope: !64)
+!64 = !DILexicalBlockFile(scope: !45, file: !1, discriminator: 186646575)
+!65 = !DILocation(line: 27, column: 5, scope: !45)
+!66 = !DILocation(line: 28, column: 8, scope: !67)
+!67 = !DILexicalBlockFile(scope: !45, file: !1, discriminator: 119537719)
+!68 = !DILocation(line: 28, column: 5, scope: !45)
+!69 = !DILocation(line: 29, column: 8, scope: !70)
+!70 = !DILexicalBlockFile(scope: !45, file: !1, discriminator: 186646591)
+!71 = !DILocation(line: 29, column: 5, scope: !45)
+!72 = !DILocation(line: 30, column: 8, scope: !73)
+!73 = !DILexicalBlockFile(scope: !45, file: !1, discriminator: 186646599)
+!74 = !DILocation(line: 30, column: 5, scope: !45)
+!75 = !DILocation(line: 31, column: 8, scope: !76)
+!76 = !DILexicalBlockFile(scope: !45, file: !1, discriminator: 119537743)
+!77 = !DILocation(line: 31, column: 5, scope: !45)
+!78 = !DILocation(line: 32, column: 8, scope: !79)
+!79 = !DILexicalBlockFile(scope: !45, file: !1, discriminator: 186646615)
+!80 = !DILocation(line: 32, column: 5, scope: !45)
+!81 = !DILocation(line: 33, column: 3, scope: !45)
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll
index 0d471e43d2a7..20be0c2fec7f 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll
@@ -86,7 +86,7 @@
 ; CHECK:    3:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 3, i32 0, i64 -1), !dbg ![[#]] - weight: 13 - factor: 1.00)
 ; CHECK:    6:  %call1.i5 = call i32 @bar(i32 noundef %add.i4), !dbg ![[#]] - weight: 13 - factor: 1.00)
 ; CHECK:    4:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 4, i32 0, i64 -1), !dbg ![[#]] - weight: 112 - factor: 1.00)
-; CHECK:    14:  %call2 = call i32 @bar(i32 noundef %3), !dbg ![[#]] - weight: 124 - factor: 1.00)
+; CHECK:    14: %call2 = call i32 @bar(i32 noundef %3), !dbg ![[#]] - weight: 124 - factor: 1.00)
 ; CHECK:    8:  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 8, i32 0, i64 -1), !dbg ![[#]] - weight: 0 - factor: 1.00)
 ; CHECK:    1:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1), !dbg ![[#]] - weight: 117 - factor: 1.00)
 ; CHECK:    2:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 2, i32 0, i64 -1), !dbg ![[#]] - weight: 104 - factor: 1.00)
-- 
GitLab


From 435771228caf77cce35406ecf57a49a06e227fe4 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 13 May 2024 16:03:26 -0700
Subject: [PATCH 0648/1206] [RISCV] Inogre CallingConv::RISCV_VectorCall in
 getCalleeSavedRegs if V/Zve is not enabled.

We can't save vector registers without V/Zve.
---
 llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index 129b4cb4e8cb..caa5dbc15f8b 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -72,7 +72,8 @@ RISCVRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   }
 
   bool HasVectorCSR =
-      MF->getFunction().getCallingConv() == CallingConv::RISCV_VectorCall;
+      MF->getFunction().getCallingConv() == CallingConv::RISCV_VectorCall &&
+      Subtarget.hasVInstructions();
 
   switch (Subtarget.getTargetABI()) {
   default:
-- 
GitLab


From 4c79d38f82e1f6fe8575d88d8c74f2f1806b19ce Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj@google.com>
Date: Mon, 13 May 2024 16:05:09 -0700
Subject: [PATCH 0649/1206] [libc] add errno_macro header to bazel build
 (#92044)

Patch #91150 added a proxy header for errno macros. This patch fixes the
bazel build since it needs to be added as a dependency.
---
 utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 6255ac998db1..ce61c432c2ed 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -117,6 +117,11 @@ libc_support_library(
     hdrs = ["hdr/sys_epoll_macros.h"],
 )
 
+libc_support_library(
+    name = "hdr_errno_macros",
+    hdrs = ["hdr/errno_macros.h"],
+)
+
 ############################ Type Proxy Header Files ###########################
 
 libc_support_library(
@@ -1144,6 +1149,7 @@ libc_function(
         ":__support_common",
         ":__support_macros_attributes",
         ":__support_macros_properties_architectures",
+        ":hdr_errno_macros",
     ],
 )
 
-- 
GitLab


From c99d1156c28dfed67a8479dd97608d1f0d6cd593 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar@redhat.com>
Date: Mon, 13 May 2024 16:31:21 -0700
Subject: [PATCH 0650/1206] [workflows] Add a job for requesting a release note
 on release branch PRs (#91826)

We have been collecting release notes from the PRs for most of the
18.1.x releases and this just helps automate the process.
---
 .github/workflows/pr-request-release-note.yml | 43 +++++++++++++++++++
 llvm/utils/git/github-automation.py           | 33 ++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 .github/workflows/pr-request-release-note.yml

diff --git a/.github/workflows/pr-request-release-note.yml b/.github/workflows/pr-request-release-note.yml
new file mode 100644
index 000000000000..0fcb95f1fe29
--- /dev/null
+++ b/.github/workflows/pr-request-release-note.yml
@@ -0,0 +1,43 @@
+name: PR Request Release Note
+
+permissions:
+  contents: read
+  pull-requests: write
+
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  request-release-note:
+    if: >-
+      github.repository_owner == 'llvm' &&
+      startsWith(github.ref, 'refs/heads/release')
+
+    runs-on: ubuntu-latest
+    steps:
+      # We need to pull the script from the main branch, so that we ensure
+      # we get the latest version of this script.
+      - name: Checkout Scripts
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          sparse-checkout: |
+            llvm/utils/git/requirements.txt
+            llvm/utils/git/github-automation.py
+          sparse-checkout-cone-mode: false
+
+      - name: Install Dependencies
+        run: |
+          pip install -r llvm/utils/git/requirements.txt
+
+      - name: Request Release Note
+        env:
+          # We need to use an llvmbot token here, because we are mentioning a user.
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          python3 llvm/utils/git/github-automation.py \
+            --repo "$GITHUB_REPOSITORY" \
+            --token "$GITHUB_TOKEN" \
+            request-release-note \
+            --pr-number ${{ github.event.pull_request.number}}
diff --git a/llvm/utils/git/github-automation.py b/llvm/utils/git/github-automation.py
index 1b5141e42594..1766ccb38ba2 100755
--- a/llvm/utils/git/github-automation.py
+++ b/llvm/utils/git/github-automation.py
@@ -637,6 +637,25 @@ class ReleaseWorkflow:
         return False
 
 
+def request_release_note(token: str, repo_name: str, pr_number: int):
+    repo = github.Github(token).get_repo(repo_name)
+    pr = repo.get_issue(pr_number).as_pull_request()
+    submitter = pr.user.login
+    if submitter == "llvmbot":
+        m = re.search("Requested by: @(.+)$", pr.body)
+        if not m:
+            submitter = None
+            print("Warning could not determine user who requested backport.")
+        submitter = m.group(1)
+
+    mention = ""
+    if submitter:
+        mention = f"@{submitter}"
+
+    comment = f"{mention} (or anyone else). If you would like to add a note about this fix in the release notes (completely optional). Please reply to this comment with a one or two sentence description of the fix.  When you are done, please add the release:note label to this PR. "
+    pr.as_issue().create_comment(comment)
+
+
 parser = argparse.ArgumentParser()
 parser.add_argument(
     "--token", type=str, required=True, help="GitHub authentication token"
@@ -703,6 +722,18 @@ release_workflow_parser.add_argument(
     help="The user that requested this backport",
 )
 
+request_release_note_parser = subparsers.add_parser(
+    "request-release-note",
+    help="Request a release note for a pull request",
+)
+request_release_note_parser.add_argument(
+    "--pr-number",
+    type=int,
+    required=True,
+    help="The pull request to request the release note",
+)
+
+
 args = parser.parse_args()
 
 if args.command == "issue-subscriber":
@@ -743,3 +774,5 @@ elif args.command == "release-workflow":
             sys.exit(1)
 elif args.command == "setup-llvmbot-git":
     setup_llvmbot_git()
+elif args.command == "request-release-note":
+    request_release_note(args.token, args.repo, args.pr_number)
-- 
GitLab


From 23f8fac745bdde70ed4f9c585d19c4913734f1b8 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 13 May 2024 16:37:39 -0700
Subject: [PATCH 0651/1206] Revert "Repply#2 "[RemoveDIs] Load into new debug
 info format by default in LLVM (#89799)""

This reverts commit 91446e2aa687ec57ad88dc0df793d0c6e694a7c9 and
a unittest followup 1530f319311908b06fe935c89fca692d3e53184f (#90476).

In a stage-2 -flto=thin -gsplit-dwarf -g -fdebug-info-for-profiling
-fprofile-sample-use= build of clang, a ThinLTO backend compile has
assertion failures:

    Global is external, but doesn't have external or weak linkage!
    ptr @_ZN5clang12ast_matchers8internal18makeAllOfCompositeINS_8QualTypeEEENS1_15BindableMatcherIT_EEN4llvm8ArrayRefIPKNS1_7MatcherIS5_EEEE
    function declaration may only have a unique !dbg attachment
    ptr @_ZN5clang12ast_matchers8internal18makeAllOfCompositeINS_8QualTypeEEENS1_15BindableMatcherIT_EEN4llvm8ArrayRefIPKNS1_7MatcherIS5_EEEE

The failures somehow go away if -fprofile-sample-use= is removed.
---
 llvm/docs/ReleaseNotes.rst                    |   7 -
 llvm/include/llvm/AsmParser/LLParser.h        |   1 +
 llvm/lib/AsmParser/LLParser.cpp               |  34 +--
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp     |   2 +-
 llvm/lib/IR/BasicBlock.cpp                    |   2 +-
 llvm/lib/IR/DebugProgramInstruction.cpp       |   4 +-
 llvm/lib/IR/Function.cpp                      |   4 +-
 llvm/lib/IR/Module.cpp                        |   4 +-
 llvm/tools/llvm-as/llvm-as.cpp                |   7 +-
 llvm/tools/llvm-dis/llvm-dis.cpp              |   2 +-
 llvm/tools/llvm-link/llvm-link.cpp            |   8 +-
 .../Analysis/IRSimilarityIdentifierTest.cpp   |  22 +-
 llvm/unittests/IR/BasicBlockDbgInfoTest.cpp   |  68 ++++++
 llvm/unittests/IR/DebugInfoTest.cpp           |  73 +++---
 llvm/unittests/IR/IRBuilderTest.cpp           |  12 +-
 llvm/unittests/IR/InstructionsTest.cpp        |   6 -
 llvm/unittests/IR/ValueTest.cpp               |   9 +-
 .../Transforms/Utils/CloningTest.cpp          |   5 +-
 llvm/unittests/Transforms/Utils/LocalTest.cpp | 211 ++++++++----------
 19 files changed, 248 insertions(+), 233 deletions(-)

diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index f2577e1684f5..84320461fa9e 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -189,13 +189,6 @@ Changes to the Metadata Info
 Changes to the Debug Info
 ---------------------------------
 
-* LLVM has switched from using debug intrinsics internally to using debug
-  records by default. This should happen transparently when using the DIBuilder
-  to construct debug variable information, but will require changes for any code
-  that interacts with debug intrinsics directly. Debug intrinsics will only be
-  supported on a best-effort basis from here onwards; for more information, see
-  the `migration docs <https://llvm.org/docs/RemoveDIsDebugInfo.html>`_.
-
 Changes to the LLVM tools
 ---------------------------------
 * llvm-nm and llvm-objdump can now print symbol information from linked
diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h
index e687254f6c4c..b2dcdfad0a04 100644
--- a/llvm/include/llvm/AsmParser/LLParser.h
+++ b/llvm/include/llvm/AsmParser/LLParser.h
@@ -337,6 +337,7 @@ namespace llvm {
 
     // Top-Level Entities
     bool parseTopLevelEntities();
+    bool finalizeDebugInfoFormat(Module *M);
     void dropUnknownMetadataReferences();
     bool validateEndOfModule(bool UpgradeDebugInfo);
     bool validateEndOfIndex();
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 34053a5ca9c8..2902bd9fe17c 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -74,6 +74,23 @@ static std::string getTypeString(Type *T) {
   return Tmp.str();
 }
 
+// Whatever debug info format we parsed, we should convert to the expected debug
+// info format immediately afterwards.
+bool LLParser::finalizeDebugInfoFormat(Module *M) {
+  // We should have already returned an error if we observed both intrinsics and
+  // records in this IR.
+  assert(!(SeenNewDbgInfoFormat && SeenOldDbgInfoFormat) &&
+         "Mixed debug intrinsics/records seen without a parsing error?");
+  if (PreserveInputDbgFormat == cl::boolOrDefault::BOU_TRUE) {
+    UseNewDbgInfoFormat = SeenNewDbgInfoFormat;
+    WriteNewDbgInfoFormatToBitcode = SeenNewDbgInfoFormat;
+    WriteNewDbgInfoFormat = SeenNewDbgInfoFormat;
+  } else if (M) {
+    M->setIsNewDbgInfoFormat(false);
+  }
+  return false;
+}
+
 /// Run: module ::= toplevelentity*
 bool LLParser::Run(bool UpgradeDebugInfo,
                    DataLayoutCallbackTy DataLayoutCallback) {
@@ -91,7 +108,7 @@ bool LLParser::Run(bool UpgradeDebugInfo,
   }
 
   return parseTopLevelEntities() || validateEndOfModule(UpgradeDebugInfo) ||
-         validateEndOfIndex();
+         validateEndOfIndex() || finalizeDebugInfoFormat(M);
 }
 
 bool LLParser::parseStandaloneConstantValue(Constant *&C,
@@ -190,18 +207,6 @@ void LLParser::dropUnknownMetadataReferences() {
 bool LLParser::validateEndOfModule(bool UpgradeDebugInfo) {
   if (!M)
     return false;
-
-  // We should have already returned an error if we observed both intrinsics and
-  // records in this IR.
-  assert(!(SeenNewDbgInfoFormat && SeenOldDbgInfoFormat) &&
-         "Mixed debug intrinsics/records seen without a parsing error?");
-  if (PreserveInputDbgFormat == cl::boolOrDefault::BOU_TRUE) {
-    UseNewDbgInfoFormat = SeenNewDbgInfoFormat;
-    WriteNewDbgInfoFormatToBitcode = SeenNewDbgInfoFormat;
-    WriteNewDbgInfoFormat = SeenNewDbgInfoFormat;
-    M->setNewDbgInfoFormatFlag(SeenNewDbgInfoFormat);
-  }
-
   // Handle any function attribute group forward references.
   for (const auto &RAG : ForwardRefAttrGroups) {
     Value *V = RAG.first;
@@ -434,9 +439,6 @@ bool LLParser::validateEndOfModule(bool UpgradeDebugInfo) {
   UpgradeModuleFlags(*M);
   UpgradeSectionAttributes(*M);
 
-  if (PreserveInputDbgFormat != cl::boolOrDefault::BOU_TRUE)
-    M->setIsNewDbgInfoFormat(UseNewDbgInfoFormat);
-
   if (!Slots)
     return false;
   // Initialize the slot mapping.
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index be2381cd7d77..19a15209f8b6 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -4319,7 +4319,7 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
   if (PreserveInputDbgFormat != cl::boolOrDefault::BOU_TRUE) {
     TheModule->IsNewDbgInfoFormat =
         UseNewDbgInfoFormat &&
-        LoadBitcodeIntoNewDbgInfoFormat != cl::boolOrDefault::BOU_FALSE;
+        LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_TRUE;
   }
 
   this->ValueTypeCallback = std::move(Callbacks.ValueType);
diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp
index aea9425ebeba..29f2cbf611fa 100644
--- a/llvm/lib/IR/BasicBlock.cpp
+++ b/llvm/lib/IR/BasicBlock.cpp
@@ -181,7 +181,7 @@ template class llvm::SymbolTableListTraits<Instruction,
 BasicBlock::BasicBlock(LLVMContext &C, const Twine &Name, Function *NewParent,
                        BasicBlock *InsertBefore)
     : Value(Type::getLabelTy(C), Value::BasicBlockVal),
-      IsNewDbgInfoFormat(UseNewDbgInfoFormat), Parent(nullptr) {
+      IsNewDbgInfoFormat(false), Parent(nullptr) {
 
   if (NewParent)
     insertInto(NewParent, InsertBefore);
diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp
index 9a4926c81dca..fbca7cdfcf3f 100644
--- a/llvm/lib/IR/DebugProgramInstruction.cpp
+++ b/llvm/lib/IR/DebugProgramInstruction.cpp
@@ -366,8 +366,8 @@ void DbgVariableRecord::setKillLocation() {
 }
 
 bool DbgVariableRecord::isKillLocation() const {
-  return (!hasArgList() && isa<MDNode>(getRawLocation())) ||
-         (getNumVariableLocationOps() == 0 && !getExpression()->isComplex()) ||
+  return (getNumVariableLocationOps() == 0 &&
+          !getExpression()->isComplex()) ||
          any_of(location_ops(), [](Value *V) { return isa<UndefValue>(V); });
 }
 
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 7f1e832f8597..bd06ff82a15a 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -83,8 +83,6 @@ static cl::opt<unsigned> NonGlobalValueMaxNameSize(
     "non-global-value-max-name-size", cl::Hidden, cl::init(1024),
     cl::desc("Maximum size for the name of non-global values."));
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 void Function::convertToNewDbgValues() {
   IsNewDbgInfoFormat = true;
   for (auto &BB : *this) {
@@ -440,7 +438,7 @@ Function::Function(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace,
     : GlobalObject(Ty, Value::FunctionVal,
                    OperandTraits<Function>::op_begin(this), 0, Linkage, name,
                    computeAddrSpace(AddrSpace, ParentModule)),
-      NumArgs(Ty->getNumParams()), IsNewDbgInfoFormat(UseNewDbgInfoFormat) {
+      NumArgs(Ty->getNumParams()), IsNewDbgInfoFormat(false) {
   assert(FunctionType::isValidReturnType(getReturnType()) &&
          "invalid return type");
   setGlobalObjectSubClassData(0);
diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp
index 915fa5097383..a8696ed9e3ce 100644
--- a/llvm/lib/IR/Module.cpp
+++ b/llvm/lib/IR/Module.cpp
@@ -54,8 +54,6 @@
 
 using namespace llvm;
 
-extern cl::opt<bool> UseNewDbgInfoFormat;
-
 //===----------------------------------------------------------------------===//
 // Methods to implement the globals and functions lists.
 //
@@ -74,7 +72,7 @@ template class llvm::SymbolTableListTraits<GlobalIFunc>;
 Module::Module(StringRef MID, LLVMContext &C)
     : Context(C), ValSymTab(std::make_unique<ValueSymbolTable>(-1)),
       ModuleID(std::string(MID)), SourceFileName(std::string(MID)), DL(""),
-      IsNewDbgInfoFormat(UseNewDbgInfoFormat) {
+      IsNewDbgInfoFormat(false) {
   Context.addModule(this);
 }
 
diff --git a/llvm/tools/llvm-as/llvm-as.cpp b/llvm/tools/llvm-as/llvm-as.cpp
index 0958e16c2197..e48e3f4d22c1 100644
--- a/llvm/tools/llvm-as/llvm-as.cpp
+++ b/llvm/tools/llvm-as/llvm-as.cpp
@@ -142,10 +142,11 @@ int main(int argc, char **argv) {
   }
 
   // Convert to new debug format if requested.
-  M->setIsNewDbgInfoFormat(UseNewDbgInfoFormat &&
-                           WriteNewDbgInfoFormatToBitcode);
-  if (M->IsNewDbgInfoFormat)
+  assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug mode");
+  if (UseNewDbgInfoFormat && WriteNewDbgInfoFormatToBitcode) {
+    M->convertToNewDbgValues();
     M->removeDebugIntrinsicDeclarations();
+  }
 
   std::unique_ptr<ModuleSummaryIndex> Index = std::move(ModuleAndIndex.Index);
 
diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp
index d28af85bc739..fbbb5506e43e 100644
--- a/llvm/tools/llvm-dis/llvm-dis.cpp
+++ b/llvm/tools/llvm-dis/llvm-dis.cpp
@@ -258,7 +258,7 @@ int main(int argc, char **argv) {
       // All that llvm-dis does is write the assembly to a file.
       if (!DontPrint) {
         if (M) {
-          M->setIsNewDbgInfoFormat(WriteNewDbgInfoFormat);
+          ScopedDbgInfoFormatSetter FormatSetter(*M, WriteNewDbgInfoFormat);
           if (WriteNewDbgInfoFormat)
             M->removeDebugIntrinsicDeclarations();
           M->print(Out->os(), Annotator.get(), PreserveAssemblyUseListOrder);
diff --git a/llvm/tools/llvm-link/llvm-link.cpp b/llvm/tools/llvm-link/llvm-link.cpp
index b84469d1c757..7794f2d81ed0 100644
--- a/llvm/tools/llvm-link/llvm-link.cpp
+++ b/llvm/tools/llvm-link/llvm-link.cpp
@@ -489,6 +489,12 @@ int main(int argc, char **argv) {
   if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET)
     LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE;
 
+  // RemoveDIs debug-info transition: tests may request that we /try/ to use the
+  // new debug-info format.
+  if (TryUseNewDbgInfoFormat) {
+    // Turn the new debug-info format on.
+    UseNewDbgInfoFormat = true;
+  }
   // Since llvm-link collects multiple IR modules together, for simplicity's
   // sake we disable the "PreserveInputDbgFormat" flag to enforce a single
   // debug info format.
@@ -550,7 +556,7 @@ int main(int argc, char **argv) {
     SetFormat(WriteNewDbgInfoFormat);
     Composite->print(Out.os(), nullptr, PreserveAssemblyUseListOrder);
   } else if (Force || !CheckBitcodeOutputToConsole(Out.os())) {
-    SetFormat(UseNewDbgInfoFormat && WriteNewDbgInfoFormatToBitcode);
+    SetFormat(WriteNewDbgInfoFormatToBitcode);
     WriteBitcodeToFile(*Composite, Out.os(), PreserveBitcodeUseListOrder);
   }
 
diff --git a/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp b/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
index 24f4f11db9a8..f6a053792f85 100644
--- a/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
+++ b/llvm/unittests/Analysis/IRSimilarityIdentifierTest.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/IRSimilarityIdentifier.h"
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
@@ -23,11 +22,6 @@
 using namespace llvm;
 using namespace IRSimilarity;
 
-extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
-extern cl::opt<cl::boolOrDefault> PreserveInputDbgFormat;
-extern bool WriteNewDbgInfoFormatToBitcode;
-extern cl::opt<bool> WriteNewDbgInfoFormat;
-
 static std::unique_ptr<Module> makeLLVMModule(LLVMContext &Context,
                                               StringRef ModuleStr) {
   SMDiagnostic Err;
@@ -1312,18 +1306,19 @@ TEST(IRInstructionMapper, CallBrInstIllegal) {
   ASSERT_GT(UnsignedVec[0], Mapper.IllegalInstrNumber);
 }
 
-// Checks that an debuginfo records are mapped to be invisible. Since they
+// Checks that an debuginfo intrinsics are mapped to be invisible.  Since they
 // do not semantically change the program, they can be recognized as similar.
 TEST(IRInstructionMapper, DebugInfoInvisible) {
   StringRef ModuleString = R"(
                           define i32 @f(i32 %a, i32 %b) {
                           then:
-                            %0 = add i32 %a, %b
-                              #dbg_value(i32 0, !0, !0, !0)
-                            %1 = add i32 %a, %b
+                            %0 = add i32 %a, %b                    
+                            call void @llvm.dbg.value(metadata !0)
+                            %1 = add i32 %a, %b     
                             ret i32 0
                           }
 
+                          declare void @llvm.dbg.value(metadata)
                           !0 = distinct !{!"test\00", i32 10})";
   LLVMContext Context;
   std::unique_ptr<Module> M = makeLLVMModule(Context, ModuleString);
@@ -1919,19 +1914,19 @@ TEST(IRSimilarityCandidate, CheckRegionsDifferentTypes) {
   ASSERT_FALSE(longSimCandCompare(InstrList));
 }
 
-// Check that debug records do not impact similarity. They are marked as
+// Check that debug instructions do not impact similarity. They are marked as
 // invisible.
 TEST(IRSimilarityCandidate, IdenticalWithDebug) {
   StringRef ModuleString = R"(
                           define i32 @f(i32 %a, i32 %b) {
                           bb0:
                              %0 = add i32 %a, %b
-                               #dbg_value(i32 0, !0, !0, !0)
+                             call void @llvm.dbg.value(metadata !0)
                              %1 = add i32 %b, %a
                              ret i32 0
                           bb1:
                              %2 = add i32 %a, %b
-                               #dbg_value(i32 1, !1, !1, !1)
+                             call void @llvm.dbg.value(metadata !1)
                              %3 = add i32 %b, %a
                              ret i32 0
                           bb2:
@@ -1940,6 +1935,7 @@ TEST(IRSimilarityCandidate, IdenticalWithDebug) {
                              ret i32 0       
                           }
 
+                          declare void @llvm.dbg.value(metadata)
                           !0 = distinct !{!"test\00", i32 10}
                           !1 = distinct !{!"test\00", i32 11})";
   LLVMContext Context;
diff --git a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
index 91a0745a0cc7..f873bbd4293a 100644
--- a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
+++ b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
@@ -25,6 +25,8 @@
 
 using namespace llvm;
 
+extern cl::opt<bool> UseNewDbgInfoFormat;
+
 static std::unique_ptr<Module> parseIR(LLVMContext &C, const char *IR) {
   SMDiagnostic Err;
   std::unique_ptr<Module> Mod = parseAssemblyString(IR, Err, C);
@@ -42,6 +44,8 @@ namespace {
 // by DbgVariableRecords, the dbg.value replacement.
 TEST(BasicBlockDbgInfoTest, InsertAfterSelf) {
   LLVMContext C;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       call void @llvm.dbg.value(metadata i16 %a, metadata !9, metadata !DIExpression()), !dbg !11
@@ -68,6 +72,8 @@ TEST(BasicBlockDbgInfoTest, InsertAfterSelf) {
     !11 = !DILocation(line: 1, column: 1, scope: !6)
 )");
 
+  // Convert the module to "new" form debug-info.
+  M->convertToNewDbgValues();
   // Fetch the entry block.
   BasicBlock &BB = M->getFunction("f")->getEntryBlock();
 
@@ -97,10 +103,16 @@ TEST(BasicBlockDbgInfoTest, InsertAfterSelf) {
   EXPECT_TRUE(RetInst->hasDbgRecords());
   auto Range2 = RetInst->getDbgRecordRange();
   EXPECT_EQ(std::distance(Range2.begin(), Range2.end()), 1u);
+
+  M->convertFromNewDbgValues();
+
+  UseNewDbgInfoFormat = false;
 }
 
 TEST(BasicBlockDbgInfoTest, SplitBasicBlockBefore) {
   LLVMContext C;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"---(
     define dso_local void @func() #0 !dbg !10 {
       %1 = alloca i32, align 4
@@ -138,6 +150,8 @@ TEST(BasicBlockDbgInfoTest, SplitBasicBlockBefore) {
   )---");
   ASSERT_TRUE(M);
 
+  M->convertToNewDbgValues();
+
   Function *F = M->getFunction("func");
 
   BasicBlock &BB = F->getEntryBlock();
@@ -147,10 +161,14 @@ TEST(BasicBlockDbgInfoTest, SplitBasicBlockBefore) {
   BasicBlock &BBBefore = F->getEntryBlock();
   auto I2 = std::prev(BBBefore.end(), 2);
   ASSERT_TRUE(I2->hasDbgRecords());
+
+  UseNewDbgInfoFormat = false;
 }
 
 TEST(BasicBlockDbgInfoTest, MarkerOperations) {
   LLVMContext C;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       call void @llvm.dbg.value(metadata i16 %a, metadata !9, metadata !DIExpression()), !dbg !11
@@ -178,6 +196,8 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) {
 
   // Fetch the entry block,
   BasicBlock &BB = M->getFunction("f")->getEntryBlock();
+  // Convert the module to "new" form debug-info.
+  M->convertToNewDbgValues();
   EXPECT_EQ(BB.size(), 2u);
 
   // Fetch out our two markers,
@@ -275,10 +295,14 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) {
 
   // Teardown,
   Instr1->insertBefore(BB, BB.begin());
+
+  UseNewDbgInfoFormat = false;
 }
 
 TEST(BasicBlockDbgInfoTest, HeadBitOperations) {
   LLVMContext C;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       %b = add i16 %a, 1, !dbg !11
@@ -308,6 +332,8 @@ TEST(BasicBlockDbgInfoTest, HeadBitOperations) {
   // Test that the movement of debug-data when using moveBefore etc and
   // insertBefore etc are governed by the "head" bit of iterators.
   BasicBlock &BB = M->getFunction("f")->getEntryBlock();
+  // Convert the module to "new" form debug-info.
+  M->convertToNewDbgValues();
 
   // Test that the head bit behaves as expected: it should be set when the
   // code wants the _start_ of the block, but not otherwise.
@@ -378,10 +404,14 @@ TEST(BasicBlockDbgInfoTest, HeadBitOperations) {
               DInst->DebugMarker->StoredDbgRecords.empty());
   EXPECT_FALSE(CInst->DebugMarker->StoredDbgRecords.empty());
   EXPECT_EQ(&*BB.begin(), CInst);
+
+  UseNewDbgInfoFormat = false;
 }
 
 TEST(BasicBlockDbgInfoTest, InstrDbgAccess) {
   LLVMContext C;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       %b = add i16 %a, 1, !dbg !11
@@ -411,6 +441,8 @@ TEST(BasicBlockDbgInfoTest, InstrDbgAccess) {
   // Check that DbgVariableRecords can be accessed from Instructions without
   // digging into the depths of DbgMarkers.
   BasicBlock &BB = M->getFunction("f")->getEntryBlock();
+  // Convert the module to "new" form debug-info.
+  M->convertToNewDbgValues();
 
   Instruction *BInst = &*BB.begin();
   Instruction *CInst = BInst->getNextNode();
@@ -451,6 +483,8 @@ TEST(BasicBlockDbgInfoTest, InstrDbgAccess) {
   CInst->dropOneDbgRecord(DVR1);
   EXPECT_FALSE(CInst->hasDbgRecords());
   EXPECT_EQ(CInst->DebugMarker->StoredDbgRecords.size(), 0u);
+
+  UseNewDbgInfoFormat = false;
 }
 
 /* Let's recall the big illustration from BasicBlock::spliceDebugInfo:
@@ -543,7 +577,9 @@ protected:
   DbgVariableRecord *DVRA, *DVRB, *DVRConst;
 
   void SetUp() override {
+    UseNewDbgInfoFormat = true;
     M = parseIR(C, SpliceTestIR.c_str());
+    M->convertToNewDbgValues();
 
     BBEntry = &M->getFunction("f")->getEntryBlock();
     BBExit = BBEntry->getNextNode();
@@ -563,6 +599,8 @@ protected:
         cast<DbgVariableRecord>(&*CInst->DebugMarker->StoredDbgRecords.begin());
   }
 
+  void TearDown() override { UseNewDbgInfoFormat = false; }
+
   bool InstContainsDbgVariableRecord(Instruction *I, DbgVariableRecord *DVR) {
     for (DbgRecord &D : I->getDbgRecordRange()) {
       if (&D == DVR) {
@@ -1149,6 +1187,8 @@ metadata !9, metadata !DIExpression()), !dbg !11 Dest      %c = add i16 %b, 1,
 // then the trailing DbgVariableRecords should get flushed back out.
 TEST(BasicBlockDbgInfoTest, DbgSpliceTrailing) {
   LLVMContext C;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1179,6 +1219,7 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceTrailing) {
 
   BasicBlock &Entry = M->getFunction("f")->getEntryBlock();
   BasicBlock &Exit = *Entry.getNextNode();
+  M->convertToNewDbgValues();
 
   // Begin by forcing entry block to have dangling DbgVariableRecord.
   Entry.getTerminator()->eraseFromParent();
@@ -1193,6 +1234,8 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceTrailing) {
   Instruction *BInst = &*Entry.begin();
   ASSERT_TRUE(BInst->DebugMarker);
   EXPECT_EQ(BInst->DebugMarker->StoredDbgRecords.size(), 1u);
+
+  UseNewDbgInfoFormat = false;
 }
 
 // When we remove instructions from the program, adjacent DbgVariableRecords
@@ -1201,6 +1244,8 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceTrailing) {
 // dbg.values. Test that this can be replicated correctly by DbgVariableRecords
 TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsert) {
   LLVMContext C;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1228,6 +1273,7 @@ TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsert) {
 )");
 
   BasicBlock &Entry = M->getFunction("f")->getEntryBlock();
+  M->convertToNewDbgValues();
 
   // Fetch the relevant instructions from the converted function.
   Instruction *SubInst = &*Entry.begin();
@@ -1270,12 +1316,16 @@ TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsert) {
   EXPECT_EQ(std::distance(R4.begin(), R4.end()), 1u);
   auto R5 = RetInst->getDbgRecordRange();
   EXPECT_EQ(std::distance(R5.begin(), R5.end()), 1u);
+
+  UseNewDbgInfoFormat = false;
 }
 
 // Test instruction removal and re-insertion, this time with one
 // DbgVariableRecord that should hop up one instruction.
 TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsertForOneDbgVariableRecord) {
   LLVMContext C;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1302,6 +1352,7 @@ TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsertForOneDbgVariableRecord) {
 )");
 
   BasicBlock &Entry = M->getFunction("f")->getEntryBlock();
+  M->convertToNewDbgValues();
 
   // Fetch the relevant instructions from the converted function.
   Instruction *SubInst = &*Entry.begin();
@@ -1340,6 +1391,8 @@ TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsertForOneDbgVariableRecord) {
   EXPECT_FALSE(RetInst->hasDbgRecords());
   auto R3 = AddInst->getDbgRecordRange();
   EXPECT_EQ(std::distance(R3.begin(), R3.end()), 1u);
+
+  UseNewDbgInfoFormat = false;
 }
 
 // Similar to the above, what if we splice into an empty block with debug-info,
@@ -1348,6 +1401,8 @@ TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsertForOneDbgVariableRecord) {
 // of the i16 0 dbg.value.
 TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty1) {
   LLVMContext C;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1381,6 +1436,7 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty1) {
   Function &F = *M->getFunction("f");
   BasicBlock &Entry = F.getEntryBlock();
   BasicBlock &Exit = *Entry.getNextNode();
+  M->convertToNewDbgValues();
 
   // Begin by forcing entry block to have dangling DbgVariableRecord.
   Entry.getTerminator()->eraseFromParent();
@@ -1407,12 +1463,16 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty1) {
 
   // No trailing DbgVariableRecords in the entry block now.
   EXPECT_EQ(Entry.getTrailingDbgRecords(), nullptr);
+
+  UseNewDbgInfoFormat = false;
 }
 
 // Similar test again, but this time: splice the contents of exit into entry,
 // with the intention of leaving the first dbg.value (i16 0) behind.
 TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty2) {
   LLVMContext C;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1446,6 +1506,7 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty2) {
   Function &F = *M->getFunction("f");
   BasicBlock &Entry = F.getEntryBlock();
   BasicBlock &Exit = *Entry.getNextNode();
+  M->convertToNewDbgValues();
 
   // Begin by forcing entry block to have dangling DbgVariableRecord.
   Entry.getTerminator()->eraseFromParent();
@@ -1476,12 +1537,16 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty2) {
   EXPECT_FALSE(Exit.getTrailingDbgRecords()->empty());
   Exit.getTrailingDbgRecords()->eraseFromParent();
   Exit.deleteTrailingDbgRecords();
+
+  UseNewDbgInfoFormat = false;
 }
 
 // What if we moveBefore end() -- there might be no debug-info there, in which
 // case we shouldn't crash.
 TEST(BasicBlockDbgInfoTest, DbgMoveToEnd) {
   LLVMContext C;
+  UseNewDbgInfoFormat = true;
+
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
     entry:
@@ -1511,6 +1576,7 @@ TEST(BasicBlockDbgInfoTest, DbgMoveToEnd) {
   Function &F = *M->getFunction("f");
   BasicBlock &Entry = F.getEntryBlock();
   BasicBlock &Exit = *Entry.getNextNode();
+  M->convertToNewDbgValues();
 
   // Move the return to the end of the entry block.
   Instruction *Br = Entry.getTerminator();
@@ -1523,6 +1589,8 @@ TEST(BasicBlockDbgInfoTest, DbgMoveToEnd) {
   EXPECT_EQ(Entry.getTrailingDbgRecords(), nullptr);
   EXPECT_EQ(Exit.getTrailingDbgRecords(), nullptr);
   EXPECT_FALSE(Ret->hasDbgRecords());
+
+  UseNewDbgInfoFormat = false;
 }
 
 } // End anonymous namespace.
diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index cac8acbe15a7..ec3f33318f8c 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -156,7 +156,7 @@ TEST(StripTest, LoopMetadata) {
   EXPECT_FALSE(BrokenDebugInfo);
 }
 
-TEST(MetadataTest, DeleteInstUsedByDbgRecord) {
+TEST(MetadataTest, DeleteInstUsedByDbgValue) {
   LLVMContext C;
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
@@ -187,13 +187,12 @@ TEST(MetadataTest, DeleteInstUsedByDbgRecord) {
 
   // Find the dbg.value using %b.
   SmallVector<DbgValueInst *, 1> DVIs;
-  SmallVector<DbgVariableRecord *, 1> DVRs;
-  findDbgValues(DVIs, &I, &DVRs);
+  findDbgValues(DVIs, &I);
 
   // Delete %b. The dbg.value should now point to undef.
   I.eraseFromParent();
-  EXPECT_EQ(DVRs[0]->getNumVariableLocationOps(), 1u);
-  EXPECT_TRUE(isa<UndefValue>(DVRs[0]->getValue(0)));
+  EXPECT_EQ(DVIs[0]->getNumVariableLocationOps(), 1u);
+  EXPECT_TRUE(isa<UndefValue>(DVIs[0]->getValue(0)));
 }
 
 TEST(DbgVariableIntrinsic, EmptyMDIsKillLocation) {
@@ -231,8 +230,8 @@ TEST(DbgVariableIntrinsic, EmptyMDIsKillLocation) {
 
   // Get the dbg.declare.
   Function &F = *cast<Function>(M->getNamedValue("fun"));
-  DbgVariableRecord *DbgDeclare =
-      cast<DbgVariableRecord>(&*F.front().front().getDbgRecordRange().begin());
+  DbgVariableIntrinsic *DbgDeclare =
+      cast<DbgVariableIntrinsic>(&F.front().front());
   // Check that this form counts as a "no location" marker.
   EXPECT_TRUE(DbgDeclare->isKillLocation());
 }
@@ -240,9 +239,6 @@ TEST(DbgVariableIntrinsic, EmptyMDIsKillLocation) {
 // Duplicate of above test, but in DbgVariableRecord representation.
 TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
   LLVMContext C;
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = true;
-
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       %b = add i16 %a, 1, !dbg !11
@@ -268,7 +264,10 @@ TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
     !11 = !DILocation(line: 1, column: 1, scope: !6)
 )");
 
+  bool OldDbgValueMode = UseNewDbgInfoFormat;
+  UseNewDbgInfoFormat = true;
   Instruction &I = *M->getFunction("f")->getEntryBlock().getFirstNonPHI();
+  M->convertToNewDbgValues();
 
   // Find the DbgVariableRecords using %b.
   SmallVector<DbgValueInst *, 2> DVIs;
@@ -290,8 +289,6 @@ TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
 // Ensure that the order of dbg.value intrinsics returned by findDbgValues, and
 // their corresponding DbgVariableRecord representation, are consistent.
 TEST(MetadataTest, OrderingOfDbgVariableRecords) {
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = false;
   LLVMContext C;
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
@@ -319,6 +316,8 @@ TEST(MetadataTest, OrderingOfDbgVariableRecords) {
     !12 = !DILocalVariable(name: "bar", scope: !6, file: !1, line: 1, type: !10)
 )");
 
+  bool OldDbgValueMode = UseNewDbgInfoFormat;
+  UseNewDbgInfoFormat = true;
   Instruction &I = *M->getFunction("f")->getEntryBlock().getFirstNonPHI();
 
   SmallVector<DbgValueInst *, 2> DVIs;
@@ -516,15 +515,14 @@ TEST(DbgAssignIntrinsicTest, replaceVariableLocationOp) {
   Value *V1 = Fun.getArg(0);
   Value *P1 = Fun.getArg(1);
   Value *P2 = Fun.getArg(2);
-  DbgVariableRecord *DbgAssign = cast<DbgVariableRecord>(
-      &*Fun.front().front().getDbgRecordRange().begin());
-  ASSERT_TRUE(V1 == DbgAssign->getVariableLocationOp(0));
-  ASSERT_TRUE(P1 == DbgAssign->getAddress());
+  DbgAssignIntrinsic *DAI = cast<DbgAssignIntrinsic>(Fun.begin()->begin());
+  ASSERT_TRUE(V1 == DAI->getVariableLocationOp(0));
+  ASSERT_TRUE(P1 == DAI->getAddress());
 
 #define TEST_REPLACE(Old, New, ExpectedValue, ExpectedAddr)                    \
-  DbgAssign->replaceVariableLocationOp(Old, New);                              \
-  EXPECT_EQ(DbgAssign->getVariableLocationOp(0), ExpectedValue);               \
-  EXPECT_EQ(DbgAssign->getAddress(), ExpectedAddr);
+  DAI->replaceVariableLocationOp(Old, New);                                    \
+  EXPECT_EQ(DAI->getVariableLocationOp(0), ExpectedValue);                     \
+  EXPECT_EQ(DAI->getAddress(), ExpectedAddr);
 
   // Replace address only.
   TEST_REPLACE(/*Old*/ P1, /*New*/ P2, /*Value*/ V1, /*Address*/ P2);
@@ -535,8 +533,8 @@ TEST(DbgAssignIntrinsicTest, replaceVariableLocationOp) {
 
   // Replace address only, value uses a DIArgList.
   // Value = {DIArgList(V1)}, Addr = P1.
-  DbgAssign->setRawLocation(DIArgList::get(C, ValueAsMetadata::get(V1)));
-  DbgAssign->setExpression(DIExpression::get(
+  DAI->setRawLocation(DIArgList::get(C, ValueAsMetadata::get(V1)));
+  DAI->setExpression(DIExpression::get(
       C, {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_stack_value}));
   TEST_REPLACE(/*Old*/ P1, /*New*/ P2, /*Value*/ V1, /*Address*/ P2);
 #undef TEST_REPLACE
@@ -622,11 +620,11 @@ TEST(AssignmentTrackingTest, Utils) {
   //
   // Check there are two llvm.dbg.assign intrinsics linked to Alloca.
   auto CheckFun1Mapping = [&Alloca]() {
-    auto Markers = at::getDVRAssignmentMarkers(&Alloca);
+    auto Markers = at::getAssignmentMarkers(&Alloca);
     EXPECT_TRUE(std::distance(Markers.begin(), Markers.end()) == 2);
     // Check those two entries are distinct.
-    DbgVariableRecord *First = *Markers.begin();
-    DbgVariableRecord *Second = *std::next(Markers.begin());
+    DbgAssignIntrinsic *First = *Markers.begin();
+    DbgAssignIntrinsic *Second = *std::next(Markers.begin());
     EXPECT_NE(First, Second);
 
     // Check that we can get back to Alloca from each llvm.dbg.assign.
@@ -662,7 +660,7 @@ TEST(AssignmentTrackingTest, Utils) {
   DIAssignID *Fun2ID = cast_or_null<DIAssignID>(
       Fun2Alloca.getMetadata(LLVMContext::MD_DIAssignID));
   EXPECT_NE(New, Fun2ID);
-  auto Fun2Markers = at::getDVRAssignmentMarkers(&Fun2Alloca);
+  auto Fun2Markers = at::getAssignmentMarkers(&Fun2Alloca);
   ASSERT_TRUE(std::distance(Fun2Markers.begin(), Fun2Markers.end()) == 1);
   auto Fun2Insts = at::getAssignmentInsts(*Fun2Markers.begin());
   ASSERT_TRUE(std::distance(Fun2Insts.begin(), Fun2Insts.end()) == 1);
@@ -671,10 +669,10 @@ TEST(AssignmentTrackingTest, Utils) {
   // 3. Check that deleting dbg.assigns from a specific instruction works.
   Instruction &Fun3Alloca =
       *M->getFunction("fun3")->getEntryBlock().getFirstNonPHIOrDbg();
-  auto Fun3Markers = at::getDVRAssignmentMarkers(&Fun3Alloca);
+  auto Fun3Markers = at::getAssignmentMarkers(&Fun3Alloca);
   ASSERT_TRUE(std::distance(Fun3Markers.begin(), Fun3Markers.end()) == 1);
   at::deleteAssignmentMarkers(&Fun3Alloca);
-  Fun3Markers = at::getDVRAssignmentMarkers(&Fun3Alloca);
+  Fun3Markers = at::getAssignmentMarkers(&Fun3Alloca);
   EXPECT_EQ(Fun3Markers.empty(), true);
 
   // 4. Check that deleting works and applies only to the target function.
@@ -685,7 +683,7 @@ TEST(AssignmentTrackingTest, Utils) {
   // llvm.dbg.assign.
   EXPECT_EQ(Fun2ID, cast_or_null<DIAssignID>(
                         Fun2Alloca.getMetadata(LLVMContext::MD_DIAssignID)));
-  EXPECT_FALSE(at::getDVRAssignmentMarkers(&Fun2Alloca).empty());
+  EXPECT_FALSE(at::getAssignmentMarkers(&Fun2Alloca).empty());
 }
 
 TEST(IRBuilder, GetSetInsertionPointWithEmptyBasicBlock) {
@@ -771,12 +769,12 @@ TEST(AssignmentTrackingTest, InstrMethods) {
   // Use SetVectors to check that the attachments and markers are unique
   // (another test requirement).
   SetVector<Metadata *> OrigIDs;
-  SetVector<DbgVariableRecord *> Markers;
+  SetVector<DbgAssignIntrinsic *> Markers;
   for (const Instruction *SI : Stores) {
     Metadata *ID = SI->getMetadata(LLVMContext::MD_DIAssignID);
     ASSERT_TRUE(OrigIDs.insert(ID));
     ASSERT_TRUE(ID != nullptr);
-    auto Range = at::getDVRAssignmentMarkers(SI);
+    auto Range = at::getAssignmentMarkers(SI);
     ASSERT_TRUE(std::distance(Range.begin(), Range.end()) == 1);
     ASSERT_TRUE(Markers.insert(*Range.begin()));
   }
@@ -869,8 +867,6 @@ TEST(AssignmentTrackingTest, InstrMethods) {
 // dbg.values that have been converted to a non-instruction format.
 TEST(MetadataTest, ConvertDbgToDbgVariableRecord) {
   LLVMContext C;
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = false;
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
       call void @llvm.dbg.value(metadata i16 %a, metadata !9, metadata !DIExpression()), !dbg !11
@@ -1045,14 +1041,14 @@ TEST(MetadataTest, ConvertDbgToDbgVariableRecord) {
   // The record of those trailing DbgVariableRecords would dangle and cause an
   // assertion failure if it lived until the end of the LLVMContext.
   ExitBlock->deleteTrailingDbgRecords();
-  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
   LLVMContext C;
 
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = false;
+  // For the purpose of this test, set and un-set the command line option
+  // corresponding to UseNewDbgInfoFormat.
+  UseNewDbgInfoFormat = true;
 
   std::unique_ptr<Module> M = parseIR(C, R"(
     define i16 @f(i16 %a) !dbg !6 {
@@ -1083,11 +1079,6 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
     !11 = !DILocation(line: 1, column: 1, scope: !6)
 )");
 
-  // For the purpose of this test, set and un-set the command line option
-  // corresponding to UseNewDbgInfoFormat, but only after parsing, to ensure
-  // that the IR starts off in the old format.
-  UseNewDbgInfoFormat = true;
-
   // Check that the conversion routines and utilities between dbg.value
   // debug-info format and DbgVariableRecords works.
   Function *F = M->getFunction("f");
@@ -1192,7 +1183,7 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) {
   EXPECT_EQ(DVI2->getVariable(), DLV2);
   EXPECT_EQ(DVI2->getExpression(), Expr2);
 
-  UseNewDbgInfoFormat = OldDbgValueMode;
+  UseNewDbgInfoFormat = false;
 }
 
 // Test that the hashing function for DISubprograms representing methods produce
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index ff96df858120..2001df090aed 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -994,17 +994,17 @@ TEST_F(IRBuilderTest, DIBuilder) {
     EXPECT_TRUE(verifyModule(*M));
   };
 
-  // Test in new-debug mode.
-  EXPECT_TRUE(M->IsNewDbgInfoFormat);
+  // Test in old-debug mode.
+  EXPECT_FALSE(M->IsNewDbgInfoFormat);
   RunTest();
 
-  // Test in old-debug mode.
-  // Reset the test then call convertFromNewDbgValues to flip the flag
+  // Test in new-debug mode.
+  // Reset the test then call convertToNewDbgValues to flip the flag
   // on the test's Module, Function and BasicBlock.
   TearDown();
   SetUp();
-  M->convertFromNewDbgValues();
-  EXPECT_FALSE(M->IsNewDbgInfoFormat);
+  M->convertToNewDbgValues();
+  EXPECT_TRUE(M->IsNewDbgInfoFormat);
   RunTest();
 }
 
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index b6044b286292..b47c73f0b329 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -25,15 +25,12 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/NoFolder.h"
 #include "llvm/IR/Operator.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm-c/Core.h"
 #include "gmock/gmock-matchers.h"
 #include "gtest/gtest.h"
 #include <memory>
 
-extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
-
 namespace llvm {
 namespace {
 
@@ -1463,8 +1460,6 @@ TEST(InstructionsTest, GetSplat) {
 
 TEST(InstructionsTest, SkipDebug) {
   LLVMContext C;
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = false;
   std::unique_ptr<Module> M = parseIR(C,
                                       R"(
       declare void @llvm.dbg.value(metadata, metadata, metadata)
@@ -1500,7 +1495,6 @@ TEST(InstructionsTest, SkipDebug) {
 
   // After the terminator, there are no non-debug instructions.
   EXPECT_EQ(nullptr, Term->getNextNonDebugInstruction());
-  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 TEST(InstructionsTest, PhiMightNotBeFPMathOperator) {
diff --git a/llvm/unittests/IR/ValueTest.cpp b/llvm/unittests/IR/ValueTest.cpp
index 33a86d510d45..246c2fc7fe40 100644
--- a/llvm/unittests/IR/ValueTest.cpp
+++ b/llvm/unittests/IR/ValueTest.cpp
@@ -13,7 +13,6 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ModuleSlotTracker.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
 using namespace llvm;
@@ -256,8 +255,6 @@ TEST(ValueTest, getLocalSlotDeath) {
 TEST(ValueTest, replaceUsesOutsideBlock) {
   // Check that Value::replaceUsesOutsideBlock(New, BB) replaces uses outside
   // BB, including dbg.* uses of MetadataAsValue(ValueAsMetadata(this)).
-  bool OldDbgValueMode = UseNewDbgInfoFormat;
-  UseNewDbgInfoFormat = false;
   const auto *IR = R"(
     define i32 @f() !dbg !6 {
     entry:
@@ -318,7 +315,6 @@ TEST(ValueTest, replaceUsesOutsideBlock) {
   // These users are outside Entry so should be changed.
   ASSERT_TRUE(ExitDbg->getValue(0) == cast<Value>(B));
   ASSERT_TRUE(Ret->getOperand(0) == cast<Value>(B));
-  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 TEST(ValueTest, replaceUsesOutsideBlockDbgVariableRecord) {
@@ -363,6 +359,10 @@ TEST(ValueTest, replaceUsesOutsideBlockDbgVariableRecord) {
   if (!M)
     Err.print("ValueTest", errs());
 
+  bool OldDbgValueMode = UseNewDbgInfoFormat;
+  UseNewDbgInfoFormat = true;
+  M->convertToNewDbgValues();
+
   auto GetNext = [](auto *I) { return &*++I->getIterator(); };
 
   Function *F = M->getFunction("f");
@@ -389,6 +389,7 @@ TEST(ValueTest, replaceUsesOutsideBlockDbgVariableRecord) {
   EXPECT_TRUE(DVR1->getVariableLocationOp(0) == cast<Value>(A));
   // These users are outside Entry so should be changed.
   EXPECT_TRUE(DVR2->getVariableLocationOp(0) == cast<Value>(B));
+  UseNewDbgInfoFormat = OldDbgValueMode;
 }
 
 } // end anonymous namespace
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index 1d0d56a2099c..5e302d9c0a0d 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -844,9 +844,8 @@ TEST(CloneFunction, CloneFunctionWithInlinedSubprograms) {
   EXPECT_FALSE(verifyModule(*ImplModule, &errs()));
 
   // Check that DILexicalBlock of inlined function was not cloned.
-  auto DbgDeclareI = Func->begin()->begin()->getDbgRecordRange().begin();
-  auto ClonedDbgDeclareI =
-      ClonedFunc->begin()->begin()->getDbgRecordRange().begin();
+  auto DbgDeclareI = Func->begin()->begin();
+  auto ClonedDbgDeclareI = ClonedFunc->begin()->begin();
   const DebugLoc &DbgLoc = DbgDeclareI->getDebugLoc();
   const DebugLoc &ClonedDbgLoc = ClonedDbgDeclareI->getDebugLoc();
   EXPECT_NE(DbgLoc.get(), ClonedDbgLoc.get());
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index 6052e58b697d..32c5244d3ff5 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Utils/Local.h"
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/PostDominators.h"
@@ -27,27 +26,6 @@
 
 using namespace llvm;
 
-extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
-extern cl::opt<cl::boolOrDefault> PreserveInputDbgFormat;
-extern bool WriteNewDbgInfoFormatToBitcode;
-extern cl::opt<bool> WriteNewDbgInfoFormat;
-
-// Backup all of the existing settings that may be modified when
-// PreserveInputDbgFormat=true, so that when the test is finished we return them
-// (and the "preserve" setting) to their original values.
-static auto SaveDbgInfoFormat() {
-  return make_scope_exit(
-      [OldPreserveInputDbgFormat = PreserveInputDbgFormat.getValue(),
-       OldUseNewDbgInfoFormat = UseNewDbgInfoFormat.getValue(),
-       OldWriteNewDbgInfoFormatToBitcode = WriteNewDbgInfoFormatToBitcode,
-       OldWriteNewDbgInfoFormat = WriteNewDbgInfoFormat.getValue()] {
-        PreserveInputDbgFormat = OldPreserveInputDbgFormat;
-        UseNewDbgInfoFormat = OldUseNewDbgInfoFormat;
-        WriteNewDbgInfoFormatToBitcode = OldWriteNewDbgInfoFormatToBitcode;
-        WriteNewDbgInfoFormat = OldWriteNewDbgInfoFormat;
-      });
-}
-
 TEST(Local, RecursivelyDeleteDeadPHINodes) {
   LLVMContext C;
 
@@ -138,6 +116,7 @@ static std::unique_ptr<Module> parseIR(LLVMContext &C, const char *IR) {
 
 TEST(Local, ReplaceDbgDeclare) {
   LLVMContext C;
+
   // Original C source to get debug info for a local variable:
   // void f() { int x; }
   std::unique_ptr<Module> M = parseIR(C,
@@ -145,11 +124,11 @@ TEST(Local, ReplaceDbgDeclare) {
       define void @f() !dbg !8 {
       entry:
         %x = alloca i32, align 4
-          #dbg_declare(ptr %x, !11, !DIExpression(), !13)
-          #dbg_declare(ptr %x, !11, !DIExpression(), !13)
+        call void @llvm.dbg.declare(metadata i32* %x, metadata !11, metadata !DIExpression()), !dbg !13
+        call void @llvm.dbg.declare(metadata i32* %x, metadata !11, metadata !DIExpression()), !dbg !13
         ret void, !dbg !14
       }
-
+      declare void @llvm.dbg.declare(metadata, metadata, metadata)
       !llvm.dbg.cu = !{!0}
       !llvm.module.flags = !{!3, !4}
       !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
@@ -172,18 +151,20 @@ TEST(Local, ReplaceDbgDeclare) {
   Instruction *Inst = &F->front().front();
   auto *AI = dyn_cast<AllocaInst>(Inst);
   ASSERT_TRUE(AI);
-
+  Inst = Inst->getNextNode()->getNextNode();
+  ASSERT_TRUE(Inst);
+  auto *DII = dyn_cast<DbgDeclareInst>(Inst);
+  ASSERT_TRUE(DII);
   Value *NewBase = Constant::getNullValue(PointerType::getUnqual(C));
   DIBuilder DIB(*M);
   replaceDbgDeclare(AI, NewBase, DIB, DIExpression::ApplyOffset, 0);
 
-  // There should be exactly two dbg.declares, attached to the terminator.
-  Inst = F->front().getTerminator();
-  ASSERT_TRUE(Inst);
-  EXPECT_TRUE(Inst->hasDbgRecords());
-  EXPECT_EQ(range_size(Inst->getDbgRecordRange()), 2u);
-  for (DbgVariableRecord &DVR : filterDbgVars(Inst->getDbgRecordRange()))
-    EXPECT_EQ(DVR.getAddress(), NewBase);
+  // There should be exactly two dbg.declares.
+  int Declares = 0;
+  for (const Instruction &I : F->front())
+    if (isa<DbgDeclareInst>(I))
+      Declares++;
+  EXPECT_EQ(2, Declares);
 }
 
 /// Build the dominator tree for the function and run the Test.
@@ -518,10 +499,11 @@ struct SalvageDebugInfoTest : ::testing::Test {
       entry:
         %x = add i32 0, 1
         %y = add i32 %x, 2
-          #dbg_value(i32 %x, !11, !DIExpression(), !13)
-          #dbg_value(i32 %y, !11, !DIExpression(), !13)
+        call void @llvm.dbg.value(metadata i32 %x, metadata !11, metadata !DIExpression()), !dbg !13
+        call void @llvm.dbg.value(metadata i32 %y, metadata !11, metadata !DIExpression()), !dbg !13
         ret void, !dbg !14
       }
+      declare void @llvm.dbg.value(metadata, metadata, metadata)
       !llvm.dbg.cu = !{!0}
       !llvm.module.flags = !{!3, !4}
       !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
@@ -544,48 +526,49 @@ struct SalvageDebugInfoTest : ::testing::Test {
     ASSERT_TRUE(F);
   }
 
-  bool doesDebugValueDescribeX(const DbgVariableRecord &DVR) {
-    if (DVR.getNumVariableLocationOps() != 1u)
+  bool doesDebugValueDescribeX(const DbgValueInst &DI) {
+    if (DI.getNumVariableLocationOps() != 1u)
       return false;
-    const auto &CI = *cast<ConstantInt>(DVR.getValue(0));
+    const auto &CI = *cast<ConstantInt>(DI.getValue(0));
     if (CI.isZero())
-      return DVR.getExpression()->getElements().equals(
+      return DI.getExpression()->getElements().equals(
           {dwarf::DW_OP_plus_uconst, 1, dwarf::DW_OP_stack_value});
     else if (CI.isOneValue())
-      return DVR.getExpression()->getElements().empty();
+      return DI.getExpression()->getElements().empty();
     return false;
   }
 
-  bool doesDebugValueDescribeY(const DbgVariableRecord &DVR) {
-    if (DVR.getNumVariableLocationOps() != 1u)
+  bool doesDebugValueDescribeY(const DbgValueInst &DI) {
+    if (DI.getNumVariableLocationOps() != 1u)
       return false;
-    const auto &CI = *cast<ConstantInt>(DVR.getVariableLocationOp(0));
+    const auto &CI = *cast<ConstantInt>(DI.getVariableLocationOp(0));
     if (CI.isZero())
-      return DVR.getExpression()->getElements().equals(
+      return DI.getExpression()->getElements().equals(
           {dwarf::DW_OP_plus_uconst, 1, dwarf::DW_OP_plus_uconst, 2,
            dwarf::DW_OP_stack_value});
     else if (CI.isOneValue())
-      return DVR.getExpression()->getElements().equals(
+      return DI.getExpression()->getElements().equals(
           {dwarf::DW_OP_plus_uconst, 2, dwarf::DW_OP_stack_value});
     return false;
   }
 
   void verifyDebugValuesAreSalvaged() {
-    // The function should only contain debug values and a terminator.
-    EXPECT_EQ(F->size(), 1u);
-    EXPECT_TRUE(F->begin()->begin()->isTerminator());
-
     // Check that the debug values for %x and %y are preserved.
     bool FoundX = false;
     bool FoundY = false;
-    for (DbgVariableRecord &DVR :
-         filterDbgVars(F->begin()->begin()->getDbgRecordRange())) {
-      EXPECT_EQ(DVR.getVariable()->getName(), "x");
-      FoundX |= doesDebugValueDescribeX(DVR);
-      FoundY |= doesDebugValueDescribeY(DVR);
+    for (const Instruction &I : F->front()) {
+      auto DI = dyn_cast<DbgValueInst>(&I);
+      if (!DI) {
+        // The function should only contain debug values and a terminator.
+        ASSERT_TRUE(I.isTerminator());
+        continue;
+      }
+      EXPECT_EQ(DI->getVariable()->getName(), "x");
+      FoundX |= doesDebugValueDescribeX(*DI);
+      FoundY |= doesDebugValueDescribeY(*DI);
     }
-    EXPECT_TRUE(FoundX);
-    EXPECT_TRUE(FoundY);
+    ASSERT_TRUE(FoundX);
+    ASSERT_TRUE(FoundY);
   }
 };
 
@@ -608,12 +591,6 @@ TEST_F(SalvageDebugInfoTest, RecursiveBlockSimplification) {
 
 TEST(Local, wouldInstructionBeTriviallyDead) {
   LLVMContext Ctx;
-  // FIXME: PreserveInputDbgFormat is set to true because this test has
-  // been written to expect debug intrinsics rather than debug records.
-  // TODO: This test doesn't have a DbgRecord equivalent form so delete
-  // it when debug intrinsics are removed.
-  auto SettingGuard = SaveDbgInfoFormat();
-  PreserveInputDbgFormat = cl::boolOrDefault::BOU_TRUE;
   std::unique_ptr<Module> M = parseIR(Ctx,
                                       R"(
     define dso_local void @fun() local_unnamed_addr #0 !dbg !9 {
@@ -707,10 +684,12 @@ TEST(Local, FindDbgUsers) {
                                       R"(
   define dso_local void @fun(ptr %a) #0 !dbg !11 {
   entry:
-      #dbg_assign(ptr %a, !16, !DIExpression(), !15, ptr %a, !DIExpression(), !19)
+    call void @llvm.dbg.assign(metadata ptr %a, metadata !16, metadata !DIExpression(), metadata !15, metadata ptr %a, metadata !DIExpression()), !dbg !19
     ret void
   }
 
+  declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata)
+
   !llvm.dbg.cu = !{!0}
   !llvm.module.flags = !{!2, !3, !9}
   !llvm.ident = !{!10}
@@ -737,13 +716,9 @@ TEST(Local, FindDbgUsers) {
   verifyModule(*M, &errs(), &BrokenDebugInfo);
   ASSERT_FALSE(BrokenDebugInfo);
 
-  // Convert to debug intrinsics as we want to test findDbgUsers and
-  // findDbgValue's debug-intrinsic-finding code here.
-  // TODO: Remove this test when debug intrinsics are removed.
-  M->convertFromNewDbgValues();
-
   Function &Fun = *cast<Function>(M->getNamedValue("fun"));
   Value *Arg = Fun.getArg(0);
+
   SmallVector<DbgVariableIntrinsic *> Users;
   // Arg (%a) is used twice by a single dbg.assign. Check findDbgUsers returns
   // only 1 pointer to it rather than 2.
@@ -764,7 +739,7 @@ TEST(Local, FindDbgRecords) {
                                       R"(
   define dso_local void @fun(ptr %a) #0 !dbg !11 {
   entry:
-      #dbg_assign(ptr %a, !16, !DIExpression(), !15, ptr %a, !DIExpression(), !19)
+    call void @llvm.dbg.assign(metadata ptr %a, metadata !16, metadata !DIExpression(), metadata !15, metadata ptr %a, metadata !DIExpression()), !dbg !19
     ret void
   }
 
@@ -793,6 +768,9 @@ TEST(Local, FindDbgRecords) {
   bool BrokenDebugInfo = true;
   verifyModule(*M, &errs(), &BrokenDebugInfo);
   ASSERT_FALSE(BrokenDebugInfo);
+  bool NewDbgInfoFormat = UseNewDbgInfoFormat;
+  UseNewDbgInfoFormat = true;
+  M->convertToNewDbgValues();
 
   Function &Fun = *cast<Function>(M->getNamedValue("fun"));
   Value *Arg = Fun.getArg(0);
@@ -812,10 +790,12 @@ TEST(Local, FindDbgRecords) {
   findDbgValues(Vals, Arg, &Records);
   EXPECT_EQ(Vals.size(), 0u);
   EXPECT_EQ(Records.size(), 1u);
+  UseNewDbgInfoFormat = NewDbgInfoFormat;
 }
 
 TEST(Local, ReplaceAllDbgUsesWith) {
   using namespace llvm::dwarf;
+
   LLVMContext Ctx;
 
   // Note: The datalayout simulates Darwin/x86_64.
@@ -828,36 +808,39 @@ TEST(Local, ReplaceAllDbgUsesWith) {
     define void @f() !dbg !6 {
     entry:
       %a = add i32 0, 1, !dbg !15
+      call void @llvm.dbg.value(metadata i32 %a, metadata !9, metadata !DIExpression()), !dbg !15
 
-        #dbg_value(i32 %a, !9, !DIExpression(), !15)
       %b = add i64 0, 1, !dbg !16
+      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression()), !dbg !16
+      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul)), !dbg !16
+      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value)), !dbg !16
+      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !16
+      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_LLVM_fragment, 0, 8)), !dbg !16
+      call void @llvm.dbg.value(metadata i64 %b, metadata !11, metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value, DW_OP_LLVM_fragment, 0, 8)), !dbg !16
 
-        #dbg_value(i64 %b, !11, !DIExpression(), !16)
-        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul), !16)
-        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value), !16)
-        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_LLVM_fragment, 0, 8), !16)
-        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_LLVM_fragment, 0, 8), !16)
-        #dbg_value(i64 %b, !11, !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_stack_value, DW_OP_LLVM_fragment, 0, 8), !16)
-      %c = inttoptr i64 0 to ptr, !dbg !17
+      %c = inttoptr i64 0 to i64*, !dbg !17
+      call void @llvm.dbg.declare(metadata i64* %c, metadata !13, metadata !DIExpression()), !dbg !17
 
-        #dbg_declare(ptr %c, !13, !DIExpression(), !17)
-      %d = inttoptr i64 0 to ptr, !dbg !18
+      %d = inttoptr i64 0 to i32*, !dbg !18
+      call void @llvm.dbg.declare(metadata i32* %d, metadata !20, metadata !DIExpression()), !dbg !18
 
-        #dbg_declare(ptr %d,  !20,  !DIExpression(), !18)
       %e = add <2 x i16> zeroinitializer, zeroinitializer
+      call void @llvm.dbg.value(metadata <2 x i16> %e, metadata !14, metadata !DIExpression()), !dbg !18
 
-        #dbg_value(<2 x i16> %e, !14, !DIExpression(), !18)
       %f = call i32 @escape(i32 0)
+      call void @llvm.dbg.value(metadata i32 %f, metadata !9, metadata !DIExpression()), !dbg !15
 
-        #dbg_value(i32 %f, !9, !DIExpression(), !15)
       %barrier = call i32 @escape(i32 0)
 
       %g = call i32 @escape(i32 %f)
+      call void @llvm.dbg.value(metadata i32 %g, metadata !9, metadata !DIExpression()), !dbg !15
 
-        #dbg_value(i32 %g, !9, !DIExpression(), !15)
       ret void, !dbg !19
     }
 
+    declare void @llvm.dbg.declare(metadata, metadata, metadata)
+    declare void @llvm.dbg.value(metadata, metadata, metadata)
+
     !llvm.dbg.cu = !{!0}
     !llvm.module.flags = !{!5}
 
@@ -912,47 +895,38 @@ TEST(Local, ReplaceAllDbgUsesWith) {
   EXPECT_TRUE(replaceAllDbgUsesWith(D, C, C, DT));
 
   SmallVector<DbgVariableIntrinsic *, 2> CDbgVals;
-  SmallVector<DbgVariableRecord *, 2> CDbgRecords;
-  findDbgUsers(CDbgVals, &C, &CDbgRecords);
-  EXPECT_EQ(0U, CDbgVals.size());
-  EXPECT_EQ(2U, CDbgRecords.size());
-  EXPECT_TRUE(all_of(
-      CDbgRecords, [](DbgVariableRecord *DVR) { return DVR->isDbgDeclare(); }));
+  findDbgUsers(CDbgVals, &C);
+  EXPECT_EQ(2U, CDbgVals.size());
+  EXPECT_TRUE(all_of(CDbgVals, [](DbgVariableIntrinsic *DII) {
+    return isa<DbgDeclareInst>(DII);
+  }));
 
   EXPECT_TRUE(replaceAllDbgUsesWith(C, D, D, DT));
 
   SmallVector<DbgVariableIntrinsic *, 2> DDbgVals;
-  SmallVector<DbgVariableRecord *, 2> DDbgRecords;
-  findDbgUsers(DDbgVals, &D, &DDbgRecords);
-  EXPECT_EQ(0U, DDbgVals.size());
-  EXPECT_EQ(2U, DDbgRecords.size());
-  EXPECT_TRUE(all_of(
-      DDbgRecords, [](DbgVariableRecord *DVR) { return DVR->isDbgDeclare(); }));
+  findDbgUsers(DDbgVals, &D);
+  EXPECT_EQ(2U, DDbgVals.size());
+  EXPECT_TRUE(all_of(DDbgVals, [](DbgVariableIntrinsic *DII) {
+    return isa<DbgDeclareInst>(DII);
+  }));
 
   // Introduce a use-before-def. Check that the dbg.value for %a is salvaged.
   EXPECT_TRUE(replaceAllDbgUsesWith(A, F_, F_, DT));
 
-  EXPECT_FALSE(A.hasDbgRecords());
-  EXPECT_TRUE(B.hasDbgRecords());
-  DbgVariableRecord *BDbgVal =
-      cast<DbgVariableRecord>(&*B.getDbgRecordRange().begin());
-  EXPECT_EQ(BDbgVal->getNumVariableLocationOps(), 1u);
-  EXPECT_EQ(ConstantInt::get(A.getType(), 0),
-            BDbgVal->getVariableLocationOp(0));
+  auto *ADbgVal = cast<DbgValueInst>(A.getNextNode());
+  EXPECT_EQ(ADbgVal->getNumVariableLocationOps(), 1u);
+  EXPECT_EQ(ConstantInt::get(A.getType(), 0), ADbgVal->getVariableLocationOp(0));
 
   // Introduce a use-before-def. Check that the dbg.values for %f become undef.
   EXPECT_TRUE(replaceAllDbgUsesWith(F_, G, G, DT));
 
-  DbgVariableRecord *BarrierDbgVal =
-      cast<DbgVariableRecord>(&*Barrier.getDbgRecordRange().begin());
-  EXPECT_EQ(BarrierDbgVal->getNumVariableLocationOps(), 1u);
-  EXPECT_TRUE(BarrierDbgVal->isKillLocation());
+  auto *FDbgVal = cast<DbgValueInst>(F_.getNextNode());
+  EXPECT_EQ(FDbgVal->getNumVariableLocationOps(), 1u);
+  EXPECT_TRUE(FDbgVal->isKillLocation());
 
-  SmallVector<DbgValueInst *, 1> BarrierDbgVals;
-  SmallVector<DbgVariableRecord *, 8> BarrierDbgRecs;
-  findDbgValues(BarrierDbgVals, &F_, &BarrierDbgRecs);
-  EXPECT_EQ(0U, BarrierDbgVals.size());
-  EXPECT_EQ(0U, BarrierDbgRecs.size());
+  SmallVector<DbgValueInst *, 1> FDbgVals;
+  findDbgValues(FDbgVals, &F_);
+  EXPECT_EQ(0U, FDbgVals.size());
 
   // Simulate i32 -> i64 conversion to test sign-extension. Here are some
   // interesting cases to handle:
@@ -962,15 +936,13 @@ TEST(Local, ReplaceAllDbgUsesWith) {
   //  4-6) like (1-3), but with a fragment
   EXPECT_TRUE(replaceAllDbgUsesWith(B, A, A, DT));
 
-  SmallVector<DbgValueInst *, 8> BDbgVals;
-  SmallVector<DbgVariableRecord *, 8> BDbgRecs;
-  findDbgValues(BDbgVals, &A, &BDbgRecs);
-  EXPECT_EQ(0U, BDbgVals.size());
-  EXPECT_EQ(6U, BDbgRecs.size());
+  SmallVector<DbgValueInst *, 8> ADbgVals;
+  findDbgValues(ADbgVals, &A);
+  EXPECT_EQ(6U, ADbgVals.size());
 
   // Check that %a has a dbg.value with a DIExpression matching \p Ops.
   auto hasADbgVal = [&](ArrayRef<uint64_t> Ops) {
-    return any_of(BDbgRecs, [&](DbgVariableRecord *DVI) {
+    return any_of(ADbgVals, [&](DbgValueInst *DVI) {
       assert(DVI->getVariable()->getName() == "2");
       return DVI->getExpression()->getElements() == Ops;
     });
@@ -1373,11 +1345,6 @@ TEST(Local, ExpressionForConstant) {
 
 TEST(Local, ReplaceDbgVariableRecord) {
   LLVMContext C;
-  // FIXME: PreserveInputDbgFormat is set to true because this test has
-  // been written to expect debug intrinsics rather than debug records; use the
-  // intrinsic format until we update the test checks.
-  auto SettingGuard = SaveDbgInfoFormat();
-  PreserveInputDbgFormat = cl::boolOrDefault::BOU_TRUE;
 
   // Test that RAUW also replaces the operands of DbgVariableRecord objects,
   // i.e. non-instruction stored debugging information.
-- 
GitLab


From ad1083dce4f664265c5489ecd2e46649cd978683 Mon Sep 17 00:00:00 2001
From: Peiming Liu <peiming@google.com>
Date: Mon, 13 May 2024 17:29:01 -0700
Subject: [PATCH 0652/1206] [mlir][sparse] introduce new pass to propagate
 sparse encodings. (#92052)

---
 .../Dialect/SparseTensor/Transforms/Passes.h  |  6 ++++
 .../Dialect/SparseTensor/Transforms/Passes.td | 36 +++++++++++++++++++
 .../Transforms/SparseTensorPasses.cpp         | 13 +++++++
 3 files changed, 55 insertions(+)

diff --git a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
index d6d038ef65bd..bb49d6c256f2 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
@@ -65,6 +65,12 @@ void populateSparseAssembler(RewritePatternSet &patterns, bool directOut);
 std::unique_ptr<Pass> createSparseAssembler();
 std::unique_ptr<Pass> createSparseAssembler(bool directOut);
 
+//===----------------------------------------------------------------------===//
+// The SparseEncodingPropagation pass.
+//===----------------------------------------------------------------------===//
+
+std::unique_ptr<Pass> createSparseEncodingPropagationPass();
+
 //===----------------------------------------------------------------------===//
 // The SparseReinterpretMap pass.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td
index 2f844cee5ff5..94c3ca60030e 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td
@@ -40,6 +40,42 @@ def SparseAssembler : Pass<"sparse-assembler", "ModuleOp"> {
   ];
 }
 
+def SparseEncodingPropagation : Pass<"sparse-encoding-propagation", "func::FuncOp"> {
+  let summary = "Propagate sparse tensor encodings";
+  let description = [{
+    A pass that propagates sparse tensor encodings.
+
+    Background: To avoid introducing repetitive operations, sparse tensors
+    in MLIR try to reuse tensor operations whenever available. However, most
+    tensor operations are canonicalized/transformed without the knowledge
+    of sparsity. The pass tries to propagate missing sparse encodings.
+
+    For example:
+    ```mlir
+    %s = tensor.extract_slice %input[0, 0,] [2, 1] [1, 1]
+       : tensor<2x3xf32, #sparse> to tensor<2x1xf32, #sparse>
+
+    // After rank reducing (by tensor dialect transformation)
+    %t = tensor.extract_slice %input[0, 0,] [2, 1] [1, 1]
+       : tensor<2x3xf32, #sparse> to tensor<2xf32>
+    %s = tensor.expand_shape [[0, 1]] %t
+       : tensor<2xf32> to tensor<2x1xf32, #sparse>
+
+    // After sparsity propagation
+    %t = tensor.extract_slice %input[0, 0,] [2, 1] [1, 1]
+       : tensor<2x3xf32, #sparse> to tensor<2xf32, #sparse1>
+    %s = tensor.expand_shape [[0, 1]] %t
+       : tensor<2xf32, #sparse1> to tensor<2x1xf32, #sparse>
+    ```
+  }];
+
+  let constructor = "mlir::createSparseEncodingPropagationPass()";
+  let dependentDialects = [
+    "sparse_tensor::SparseTensorDialect",
+    "tensor::TensorDialect",
+  ];
+}
+
 def SparseReinterpretMap : Pass<"sparse-reinterpret-map", "ModuleOp"> {
   let summary = "Reinterprets sparse tensor type mappings";
   let description = [{
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp
index b42d58634a36..f57353b5892b 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp
@@ -23,6 +23,7 @@
 
 namespace mlir {
 #define GEN_PASS_DEF_SPARSEASSEMBLER
+#define GEN_PASS_DEF_SPARSEENCODINGPROPAGATION
 #define GEN_PASS_DEF_SPARSEREINTERPRETMAP
 #define GEN_PASS_DEF_PRESPARSIFICATIONREWRITE
 #define GEN_PASS_DEF_SPARSIFICATIONPASS
@@ -60,6 +61,14 @@ struct SparseAssembler : public impl::SparseAssemblerBase<SparseAssembler> {
   }
 };
 
+struct SparseEncodingPropagation
+    : public impl::SparseEncodingPropagationBase<SparseEncodingPropagation> {
+  SparseEncodingPropagation() = default;
+  SparseEncodingPropagation(const SparseEncodingPropagation &pass) = default;
+
+  void runOnOperation() override {}
+};
+
 struct SparseReinterpretMap
     : public impl::SparseReinterpretMapBase<SparseReinterpretMap> {
   SparseReinterpretMap() = default;
@@ -398,6 +407,10 @@ std::unique_ptr<Pass> mlir::createSparseAssembler() {
   return std::make_unique<SparseAssembler>();
 }
 
+std::unique_ptr<Pass> mlir::createSparseEncodingPropagationPass() {
+  return std::make_unique<SparseEncodingPropagation>();
+}
+
 std::unique_ptr<Pass> mlir::createSparseReinterpretMapPass() {
   return std::make_unique<SparseReinterpretMap>();
 }
-- 
GitLab


From 595de12ff307f3f06f4ccd2acafc400cc1262bc6 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Tue, 14 May 2024 09:44:49 +0900
Subject: [PATCH 0653/1206] [APFloat] Replace partsCount array with single
 variable (NFC) (#91910)

We only ever use the last element of this array, so there shouldn't be a
need to store the preceding elements as well.
---
 llvm/lib/Support/APFloat.cpp | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Support/APFloat.cpp b/llvm/lib/Support/APFloat.cpp
index 64a7e0c7223f..2a9b3903720b 100644
--- a/llvm/lib/Support/APFloat.cpp
+++ b/llvm/lib/Support/APFloat.cpp
@@ -732,7 +732,7 @@ powerOf5(APFloatBase::integerPart *dst, unsigned int power) {
   APFloatBase::integerPart pow5s[maxPowerOfFiveParts * 2 + 5];
   pow5s[0] = 78125 * 5;
 
-  unsigned int partsCount[16] = { 1 };
+  unsigned int partsCount = 1;
   APFloatBase::integerPart scratch[maxPowerOfFiveParts], *p1, *p2, *pow5;
   unsigned int result;
   assert(power <= maxExponent);
@@ -747,25 +747,20 @@ powerOf5(APFloatBase::integerPart *dst, unsigned int power) {
   pow5 = pow5s;
 
   for (unsigned int n = 0; power; power >>= 1, n++) {
-    unsigned int pc;
-
-    pc = partsCount[n];
-
     /* Calculate pow(5,pow(2,n+3)) if we haven't yet.  */
-    if (pc == 0) {
-      pc = partsCount[n - 1];
-      APInt::tcFullMultiply(pow5, pow5 - pc, pow5 - pc, pc, pc);
-      pc *= 2;
-      if (pow5[pc - 1] == 0)
-        pc--;
-      partsCount[n] = pc;
+    if (n != 0) {
+      APInt::tcFullMultiply(pow5, pow5 - partsCount, pow5 - partsCount,
+                            partsCount, partsCount);
+      partsCount *= 2;
+      if (pow5[partsCount - 1] == 0)
+        partsCount--;
     }
 
     if (power & 1) {
       APFloatBase::integerPart *tmp;
 
-      APInt::tcFullMultiply(p2, p1, pow5, result, pc);
-      result += pc;
+      APInt::tcFullMultiply(p2, p1, pow5, result, partsCount);
+      result += partsCount;
       if (p2[result - 1] == 0)
         result--;
 
@@ -776,7 +771,7 @@ powerOf5(APFloatBase::integerPart *dst, unsigned int power) {
       p2 = tmp;
     }
 
-    pow5 += pc;
+    pow5 += partsCount;
   }
 
   if (p1 != dst)
-- 
GitLab


From 37b8e5feb1d065a7c474e6595bac6d2f65faeb51 Mon Sep 17 00:00:00 2001
From: Jim Ingham <jingham@apple.com>
Date: Mon, 13 May 2024 17:12:58 -0700
Subject: [PATCH 0654/1206] Revert "[lldb][DWARF] Delay struct/class/union
 definition DIE searching when parsing declaration DIEs. (#90663)"

This reverts commit 9a7262c2601874e5aa64c5db19746770212d4b44.
---
 .../Plugins/SymbolFile/DWARF/DWARFASTParser.h |   2 -
 .../SymbolFile/DWARF/DWARFASTParserClang.cpp  | 397 ++++++++----------
 .../SymbolFile/DWARF/DWARFASTParserClang.h    | 197 +++++----
 .../SymbolFile/DWARF/SymbolFileDWARF.cpp      |  44 +-
 .../SymbolFile/DWARF/SymbolFileDWARF.h        |   4 -
 .../SymbolFile/DWARF/UniqueDWARFASTType.cpp   | 107 +++--
 .../SymbolFile/DWARF/UniqueDWARFASTType.h     |  36 +-
 7 files changed, 383 insertions(+), 404 deletions(-)

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
index e144cf0f9bd9..66db396279e0 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
@@ -60,8 +60,6 @@ public:
 
   virtual ConstString GetDIEClassTemplateParams(const DWARFDIE &die) = 0;
 
-  virtual lldb_private::Type *FindDefinitionTypeForDIE(const DWARFDIE &die) = 0;
-
   static std::optional<SymbolFile::ArrayInfo>
   ParseChildArrayInfo(const DWARFDIE &parent_die,
                       const ExecutionContext *exe_ctx = nullptr);
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 2a46be921612..034817c3b4fa 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -154,26 +154,6 @@ static bool TagIsRecordType(dw_tag_t tag) {
   }
 }
 
-static bool IsForwardDeclaration(const DWARFDIE &die,
-                                 const ParsedDWARFTypeAttributes &attrs,
-                                 LanguageType cu_language) {
-  if (attrs.is_forward_declaration)
-    return true;
-
-  // Work around an issue with clang at the moment where forward
-  // declarations for objective C classes are emitted as:
-  //  DW_TAG_structure_type [2]
-  //  DW_AT_name( "ForwardObjcClass" )
-  //  DW_AT_byte_size( 0x00 )
-  //  DW_AT_decl_file( "..." )
-  //  DW_AT_decl_line( 1 )
-  //
-  // Note that there is no DW_AT_declaration and there are no children,
-  // and the byte size is zero.
-  return attrs.byte_size && *attrs.byte_size == 0 && attrs.name &&
-         !die.HasChildren() && cu_language == eLanguageTypeObjC;
-}
-
 TypeSP DWARFASTParserClang::ParseTypeFromClangModule(const SymbolContext &sc,
                                                      const DWARFDIE &die,
                                                      Log *log) {
@@ -269,9 +249,11 @@ static void ForcefullyCompleteType(CompilerType type) {
 /// This function serves a similar purpose as RequireCompleteType above, but it
 /// avoids completing the type if it is not immediately necessary. It only
 /// ensures we _can_ complete the type later.
-void DWARFASTParserClang::PrepareContextToReceiveMembers(
-    clang::DeclContext *decl_ctx, const DWARFDIE &decl_ctx_die,
-    const DWARFDIE &die, const char *type_name_cstr) {
+static void PrepareContextToReceiveMembers(TypeSystemClang &ast,
+                                           ClangASTImporter &ast_importer,
+                                           clang::DeclContext *decl_ctx,
+                                           DWARFDIE die,
+                                           const char *type_name_cstr) {
   auto *tag_decl_ctx = clang::dyn_cast<clang::TagDecl>(decl_ctx);
   if (!tag_decl_ctx)
     return; // Non-tag context are always ready.
@@ -286,8 +268,7 @@ void DWARFASTParserClang::PrepareContextToReceiveMembers(
   // gmodules case), we can complete the type by doing a full import.
 
   // If this type was not imported from an external AST, there's nothing to do.
-  CompilerType type = m_ast.GetTypeForDecl(tag_decl_ctx);
-  ClangASTImporter &ast_importer = GetClangASTImporter();
+  CompilerType type = ast.GetTypeForDecl(tag_decl_ctx);
   if (type && ast_importer.CanImport(type)) {
     auto qual_type = ClangUtil::GetQualType(type);
     if (ast_importer.RequireCompleteType(qual_type))
@@ -298,13 +279,6 @@ void DWARFASTParserClang::PrepareContextToReceiveMembers(
         type_name_cstr ? type_name_cstr : "", die.GetOffset());
   }
 
-  // By searching for the definition DIE of the decl_ctx type, we will either:
-  // 1. Found the the definition DIE and start its definition with
-  // TypeSystemClang::StartTagDeclarationDefinition.
-  // 2. Unable to find it, then need to forcefully complete it.
-  FindDefinitionTypeForDIE(decl_ctx_die);
-  if (tag_decl_ctx->isCompleteDefinition() || tag_decl_ctx->isBeingDefined())
-    return;
   // We don't have a type definition and/or the import failed. We must
   // forcefully complete the type to avoid crashes.
   ForcefullyCompleteType(type);
@@ -646,11 +620,10 @@ DWARFASTParserClang::ParseTypeModifier(const SymbolContext &sc,
   if (tag == DW_TAG_typedef) {
     // DeclContext will be populated when the clang type is materialized in
     // Type::ResolveCompilerType.
-    DWARFDIE decl_ctx_die;
-    clang::DeclContext *decl_ctx =
-        GetClangDeclContextContainingDIE(die, &decl_ctx_die);
-    PrepareContextToReceiveMembers(decl_ctx, decl_ctx_die, die,
-                                   attrs.name.GetCString());
+    PrepareContextToReceiveMembers(
+        m_ast, GetClangASTImporter(),
+        GetClangDeclContextContainingDIE(die, nullptr), die,
+        attrs.name.GetCString());
 
     if (attrs.type.IsValid()) {
       // Try to parse a typedef from the (DWARF embedded in the) Clang
@@ -1130,6 +1103,32 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
         // struct and see if this is actually a C++ method
         Type *class_type = dwarf->ResolveType(decl_ctx_die);
         if (class_type) {
+          if (class_type->GetID() != decl_ctx_die.GetID() ||
+              IsClangModuleFwdDecl(decl_ctx_die)) {
+
+            // We uniqued the parent class of this function to another
+            // class so we now need to associate all dies under
+            // "decl_ctx_die" to DIEs in the DIE for "class_type"...
+            DWARFDIE class_type_die = dwarf->GetDIE(class_type->GetID());
+
+            if (class_type_die) {
+              std::vector<DWARFDIE> failures;
+
+              CopyUniqueClassMethodTypes(decl_ctx_die, class_type_die,
+                                         class_type, failures);
+
+              // FIXME do something with these failures that's
+              // smarter than just dropping them on the ground.
+              // Unfortunately classes don't like having stuff added
+              // to them after their definitions are complete...
+
+              Type *type_ptr = dwarf->GetDIEToType()[die.GetDIE()];
+              if (type_ptr && type_ptr != DIE_IS_BEING_PARSED) {
+                return type_ptr->shared_from_this();
+              }
+            }
+          }
+
           if (attrs.specification.IsValid()) {
             // We have a specification which we are going to base our
             // function prototype off of, so we need this type to be
@@ -1264,39 +1263,6 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
               }
             }
           }
-          // By here, we should have already completed the c++ class_type
-          // because if either specification or abstract_origin is present, we
-          // call GetClangDeclContextForDIE to resolve the DW_TAG_subprogram
-          // refered by this one until we reached the DW_TAG_subprogram without
-          // specification or abstract_origin (the else branch above). Then the
-          // above GetFullCompilerType() will complete the class_type if it's
-          // not completed yet. After that, we will have the mapping from DIEs
-          // in class_type_die to DeclContexts in m_die_to_decl_ctx.
-          if (class_type->GetID() != decl_ctx_die.GetID() ||
-              IsClangModuleFwdDecl(decl_ctx_die)) {
-
-            // We uniqued the parent class of this function to another
-            // class so we now need to associate all dies under
-            // "decl_ctx_die" to DIEs in the DIE for "class_type"...
-            DWARFDIE class_type_die = dwarf->GetDIE(class_type->GetID());
-
-            if (class_type_die) {
-              std::vector<DWARFDIE> failures;
-
-              CopyUniqueClassMethodTypes(decl_ctx_die, class_type_die,
-                                         class_type, failures);
-
-              // FIXME do something with these failures that's
-              // smarter than just dropping them on the ground.
-              // Unfortunately classes don't like having stuff added
-              // to them after their definitions are complete...
-
-              Type *type_ptr = dwarf->GetDIEToType()[die.GetDIE()];
-              if (type_ptr && type_ptr != DIE_IS_BEING_PARSED) {
-                return type_ptr->shared_from_this();
-              }
-            }
-          }
         }
       }
     }
@@ -1669,93 +1635,6 @@ DWARFASTParserClang::GetCPlusPlusQualifiedName(const DWARFDIE &die) {
   return qualified_name;
 }
 
-lldb_private::Type *
-DWARFASTParserClang::FindDefinitionTypeForDIE(const DWARFDIE &die) {
-  SymbolFileDWARF *dwarf = die.GetDWARF();
-  ParsedDWARFTypeAttributes attrs(die);
-  bool is_forward_declaration = IsForwardDeclaration(
-      die, attrs, SymbolFileDWARF::GetLanguage(*die.GetCU()));
-  if (!is_forward_declaration)
-    return dwarf->GetDIEToType()[die.GetDIE()];
-
-  const dw_tag_t tag = die.Tag();
-  TypeSP type_sp;
-  Log *log = GetLog(DWARFLog::TypeCompletion | DWARFLog::Lookups);
-  if (log) {
-    dwarf->GetObjectFile()->GetModule()->LogMessage(
-        log,
-        "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a "
-        "forward declaration DIE, trying to find definition DIE",
-        static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
-        attrs.name.GetCString());
-  }
-  // We haven't parse definition die for this type, starting to search for it.
-  // After we found the definition die, the GetDeclarationDIEToDefinitionDIE()
-  // map will have the new mapping from this declaration die to definition die.
-  if (attrs.class_language == eLanguageTypeObjC ||
-      attrs.class_language == eLanguageTypeObjC_plus_plus) {
-    if (!attrs.is_complete_objc_class &&
-        die.Supports_DW_AT_APPLE_objc_complete_type()) {
-      // We have a valid eSymbolTypeObjCClass class symbol whose name
-      // matches the current objective C class that we are trying to find
-      // and this DIE isn't the complete definition (we checked
-      // is_complete_objc_class above and know it is false), so the real
-      // definition is in here somewhere
-      type_sp =
-          dwarf->FindCompleteObjCDefinitionTypeForDIE(die, attrs.name, true);
-
-      if (!type_sp) {
-        SymbolFileDWARFDebugMap *debug_map_symfile =
-            dwarf->GetDebugMapSymfile();
-        if (debug_map_symfile) {
-          // We weren't able to find a full declaration in this DWARF,
-          // see if we have a declaration anywhere else...
-          type_sp = debug_map_symfile->FindCompleteObjCDefinitionTypeForDIE(
-              die, attrs.name, true);
-        }
-      }
-
-      if (type_sp && log) {
-        dwarf->GetObjectFile()->GetModule()->LogMessage(
-            log,
-            "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is an "
-            "incomplete objc type, complete type is {5:x8}",
-            static_cast<void *>(this), die.GetOffset(),
-            DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(),
-            type_sp->GetID());
-      }
-    }
-  }
-
-  type_sp = dwarf->FindDefinitionTypeForDWARFDeclContext(die);
-  if (!type_sp) {
-    SymbolFileDWARFDebugMap *debug_map_symfile = dwarf->GetDebugMapSymfile();
-    if (debug_map_symfile) {
-      // We weren't able to find a full declaration in this DWARF, see
-      // if we have a declaration anywhere else...
-      type_sp = debug_map_symfile->FindDefinitionTypeForDWARFDeclContext(die);
-    }
-    if (type_sp && log) {
-      dwarf->GetObjectFile()->GetModule()->LogMessage(
-          log,
-          "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a "
-          "forward declaration, complete type is {4:x8}",
-          static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
-          attrs.name.GetCString(), type_sp->GetID());
-    }
-  }
-
-  if (!type_sp && log) {
-    dwarf->GetObjectFile()->GetModule()->LogMessage(
-        log,
-        "SymbolFileDWARF({0:p}) - {1:x16}: {2} type \"{3}\" is a "
-        "forward declaration, unable to find definition DIE for it",
-        static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
-        attrs.name.GetCString());
-  }
-  return type_sp.get();
-}
-
 TypeSP
 DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
                                            const DWARFDIE &die,
@@ -1767,10 +1646,14 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
   LanguageType cu_language = SymbolFileDWARF::GetLanguage(*die.GetCU());
   Log *log = GetLog(DWARFLog::TypeCompletion | DWARFLog::Lookups);
 
+  // UniqueDWARFASTType is large, so don't create a local variables on the
+  // stack, put it on the heap. This function is often called recursively and
+  // clang isn't good at sharing the stack space for variables in different
+  // blocks.
+  auto unique_ast_entry_up = std::make_unique<UniqueDWARFASTType>();
+
   ConstString unique_typename(attrs.name);
   Declaration unique_decl(attrs.decl);
-  uint64_t byte_size = attrs.byte_size.value_or(0);
-  attrs.is_forward_declaration = IsForwardDeclaration(die, attrs, cu_language);
 
   if (attrs.name) {
     if (Language::LanguageIsCPlusPlus(cu_language)) {
@@ -1783,42 +1666,14 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
       unique_decl.Clear();
     }
 
-    if (UniqueDWARFASTType *unique_ast_entry_type =
-            dwarf->GetUniqueDWARFASTTypeMap().Find(
-                unique_typename, die, unique_decl, byte_size,
-                attrs.is_forward_declaration)) {
-      type_sp = unique_ast_entry_type->m_type_sp;
+    if (dwarf->GetUniqueDWARFASTTypeMap().Find(
+            unique_typename, die, unique_decl, attrs.byte_size.value_or(-1),
+            *unique_ast_entry_up)) {
+      type_sp = unique_ast_entry_up->m_type_sp;
       if (type_sp) {
         dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
         LinkDeclContextToDIE(
-            GetCachedClangDeclContextForDIE(unique_ast_entry_type->m_die), die);
-        if (!attrs.is_forward_declaration) {
-          // If the DIE being parsed in this function is a definition and the
-          // entry in the map is a declaration, then we need to update the entry
-          // to point to the definition DIE.
-          if (unique_ast_entry_type->m_is_forward_declaration) {
-            unique_ast_entry_type->m_die = die;
-            unique_ast_entry_type->m_byte_size = byte_size;
-            unique_ast_entry_type->m_declaration = unique_decl;
-            unique_ast_entry_type->m_is_forward_declaration = false;
-            // Need to update Type ID to refer to the definition DIE. because
-            // it's used in ParseSubroutine to determine if we need to copy cxx
-            // method types from a declaration DIE to this definition DIE.
-            type_sp->SetID(die.GetID());
-            clang_type = type_sp->GetForwardCompilerType();
-            if (attrs.class_language != eLanguageTypeObjC &&
-                attrs.class_language != eLanguageTypeObjC_plus_plus)
-              TypeSystemClang::StartTagDeclarationDefinition(clang_type);
-
-            CompilerType compiler_type_no_qualifiers =
-                ClangUtil::RemoveFastQualifiers(clang_type);
-            auto result = dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace(
-                compiler_type_no_qualifiers.GetOpaqueQualType(),
-                *die.GetDIERef());
-            if (!result.second)
-              result.first->second = *die.GetDIERef();
-          }
-        }
+            GetCachedClangDeclContextForDIE(unique_ast_entry_up->m_die), die);
         return type_sp;
       }
     }
@@ -1840,21 +1695,125 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
     default_accessibility = eAccessPrivate;
   }
 
+  if (attrs.byte_size && *attrs.byte_size == 0 && attrs.name &&
+      !die.HasChildren() && cu_language == eLanguageTypeObjC) {
+    // Work around an issue with clang at the moment where forward
+    // declarations for objective C classes are emitted as:
+    //  DW_TAG_structure_type [2]
+    //  DW_AT_name( "ForwardObjcClass" )
+    //  DW_AT_byte_size( 0x00 )
+    //  DW_AT_decl_file( "..." )
+    //  DW_AT_decl_line( 1 )
+    //
+    // Note that there is no DW_AT_declaration and there are no children,
+    // and the byte size is zero.
+    attrs.is_forward_declaration = true;
+  }
+
+  if (attrs.class_language == eLanguageTypeObjC ||
+      attrs.class_language == eLanguageTypeObjC_plus_plus) {
+    if (!attrs.is_complete_objc_class &&
+        die.Supports_DW_AT_APPLE_objc_complete_type()) {
+      // We have a valid eSymbolTypeObjCClass class symbol whose name
+      // matches the current objective C class that we are trying to find
+      // and this DIE isn't the complete definition (we checked
+      // is_complete_objc_class above and know it is false), so the real
+      // definition is in here somewhere
+      type_sp =
+          dwarf->FindCompleteObjCDefinitionTypeForDIE(die, attrs.name, true);
+
+      if (!type_sp) {
+        SymbolFileDWARFDebugMap *debug_map_symfile =
+            dwarf->GetDebugMapSymfile();
+        if (debug_map_symfile) {
+          // We weren't able to find a full declaration in this DWARF,
+          // see if we have a declaration anywhere else...
+          type_sp = debug_map_symfile->FindCompleteObjCDefinitionTypeForDIE(
+              die, attrs.name, true);
+        }
+      }
+
+      if (type_sp) {
+        if (log) {
+          dwarf->GetObjectFile()->GetModule()->LogMessage(
+              log,
+              "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is an "
+              "incomplete objc type, complete type is {5:x8}",
+              static_cast<void *>(this), die.GetOffset(),
+              DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(),
+              type_sp->GetID());
+        }
+
+        // We found a real definition for this type elsewhere so lets use
+        // it and cache the fact that we found a complete type for this
+        // die
+        dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
+        return type_sp;
+      }
+    }
+  }
+
   if (attrs.is_forward_declaration) {
+    // We have a forward declaration to a type and we need to try and
+    // find a full declaration. We look in the current type index just in
+    // case we have a forward declaration followed by an actual
+    // declarations in the DWARF. If this fails, we need to look
+    // elsewhere...
+    if (log) {
+      dwarf->GetObjectFile()->GetModule()->LogMessage(
+          log,
+          "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is a "
+          "forward declaration, trying to find complete type",
+          static_cast<void *>(this), die.GetOffset(), DW_TAG_value_to_name(tag),
+          tag, attrs.name.GetCString());
+    }
+
     // See if the type comes from a Clang module and if so, track down
     // that type.
     type_sp = ParseTypeFromClangModule(sc, die, log);
     if (type_sp)
       return type_sp;
-  }
 
+    // type_sp = FindDefinitionTypeForDIE (dwarf_cu, die,
+    // type_name_const_str);
+    type_sp = dwarf->FindDefinitionTypeForDWARFDeclContext(die);
+
+    if (!type_sp) {
+      SymbolFileDWARFDebugMap *debug_map_symfile = dwarf->GetDebugMapSymfile();
+      if (debug_map_symfile) {
+        // We weren't able to find a full declaration in this DWARF, see
+        // if we have a declaration anywhere else...
+        type_sp = debug_map_symfile->FindDefinitionTypeForDWARFDeclContext(die);
+      }
+    }
+
+    if (type_sp) {
+      if (log) {
+        dwarf->GetObjectFile()->GetModule()->LogMessage(
+            log,
+            "SymbolFileDWARF({0:p}) - {1:x16}: {2} ({3}) type \"{4}\" is a "
+            "forward declaration, complete type is {5:x8}",
+            static_cast<void *>(this), die.GetOffset(),
+            DW_TAG_value_to_name(tag), tag, attrs.name.GetCString(),
+            type_sp->GetID());
+      }
+
+      // We found a real definition for this type elsewhere so lets use
+      // it and cache the fact that we found a complete type for this die
+      dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
+      clang::DeclContext *defn_decl_ctx =
+          GetCachedClangDeclContextForDIE(dwarf->GetDIE(type_sp->GetID()));
+      if (defn_decl_ctx)
+        LinkDeclContextToDIE(defn_decl_ctx, die);
+      return type_sp;
+    }
+  }
   assert(tag_decl_kind != -1);
   UNUSED_IF_ASSERT_DISABLED(tag_decl_kind);
-  DWARFDIE decl_ctx_die;
-  clang::DeclContext *decl_ctx =
-      GetClangDeclContextContainingDIE(die, &decl_ctx_die);
+  bool clang_type_was_created = false;
+  clang::DeclContext *decl_ctx = GetClangDeclContextContainingDIE(die, nullptr);
 
-  PrepareContextToReceiveMembers(decl_ctx, decl_ctx_die, die,
+  PrepareContextToReceiveMembers(m_ast, GetClangASTImporter(), decl_ctx, die,
                                  attrs.name.GetCString());
 
   if (attrs.accessibility == eAccessNone && decl_ctx) {
@@ -1893,17 +1852,20 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
             tag_decl_kind, template_param_infos);
     clang_type =
         m_ast.CreateClassTemplateSpecializationType(class_specialization_decl);
+    clang_type_was_created = true;
 
     m_ast.SetMetadata(class_template_decl, metadata);
     m_ast.SetMetadata(class_specialization_decl, metadata);
   }
 
-  if (!clang_type) {
+  if (!clang_type_was_created) {
+    clang_type_was_created = true;
     clang_type = m_ast.CreateRecordType(
         decl_ctx, GetOwningClangModule(die), attrs.accessibility,
         attrs.name.GetCString(), tag_decl_kind, attrs.class_language, &metadata,
         attrs.exports_symbols);
   }
+
   // Store a forward declaration to this class type in case any
   // parameters in any class methods need it for the clang types for
   // function prototypes.
@@ -1914,19 +1876,13 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
       Type::ResolveState::Forward,
       TypePayloadClang(OptionalClangModuleID(), attrs.is_complete_objc_class));
 
-  // UniqueDWARFASTType is large, so don't create a local variables on the
-  // stack, put it on the heap. This function is often called recursively and
-  // clang isn't good at sharing the stack space for variables in different
-  // blocks.
-  auto unique_ast_entry_up = std::make_unique<UniqueDWARFASTType>();
   // Add our type to the unique type map so we don't end up creating many
   // copies of the same type over and over in the ASTContext for our
   // module
   unique_ast_entry_up->m_type_sp = type_sp;
   unique_ast_entry_up->m_die = die;
   unique_ast_entry_up->m_declaration = unique_decl;
-  unique_ast_entry_up->m_byte_size = byte_size;
-  unique_ast_entry_up->m_is_forward_declaration = attrs.is_forward_declaration;
+  unique_ast_entry_up->m_byte_size = attrs.byte_size.value_or(0);
   dwarf->GetUniqueDWARFASTTypeMap().Insert(unique_typename,
                                            *unique_ast_entry_up);
 
@@ -1967,7 +1923,7 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
           GetClangASTImporter().SetRecordLayout(record_decl, layout);
         }
       }
-    } else {
+    } else if (clang_type_was_created) {
       // Start the definition if the class is not objective C since the
       // underlying decls respond to isCompleteDefinition(). Objective
       // C decls don't respond to isCompleteDefinition() so we can't
@@ -1979,21 +1935,26 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
       if (attrs.class_language != eLanguageTypeObjC &&
           attrs.class_language != eLanguageTypeObjC_plus_plus)
         TypeSystemClang::StartTagDeclarationDefinition(clang_type);
+
+      // Leave this as a forward declaration until we need to know the
+      // details of the type. lldb_private::Type will automatically call
+      // the SymbolFile virtual function
+      // "SymbolFileDWARF::CompleteType(Type *)" When the definition
+      // needs to be defined.
+      assert(!dwarf->GetForwardDeclCompilerTypeToDIE().count(
+                 ClangUtil::RemoveFastQualifiers(clang_type)
+                     .GetOpaqueQualType()) &&
+             "Type already in the forward declaration map!");
+      // Can't assume m_ast.GetSymbolFile() is actually a
+      // SymbolFileDWARF, it can be a SymbolFileDWARFDebugMap for Apple
+      // binaries.
+      dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace(
+          ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType(),
+          *die.GetDIERef());
+      m_ast.SetHasExternalStorage(clang_type.GetOpaqueQualType(), true);
     }
   }
 
-  // If this is a declaration DIE, leave this as a forward declaration until we
-  // need to know the details of the type. lldb_private::Type will automatically
-  // call the SymbolFile virtual function "SymbolFileDWARF::CompleteType(Type
-  // *)" When the definition needs to be defined.
-  assert(!dwarf->GetForwardDeclCompilerTypeToDIE().count(
-             ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType()) &&
-         "Type already in the forward declaration map!");
-  dwarf->GetForwardDeclCompilerTypeToDIE().try_emplace(
-      ClangUtil::RemoveFastQualifiers(clang_type).GetOpaqueQualType(),
-      *die.GetDIERef());
-  m_ast.SetHasExternalStorage(clang_type.GetOpaqueQualType(), true);
-
   // If we made a clang type, set the trivial abi if applicable: We only
   // do this for pass by value - which implies the Trivial ABI. There
   // isn't a way to assert that something that would normally be pass by
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
index 853b8ccc3036..8d4af203bb28 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
@@ -42,40 +42,40 @@ struct ParsedDWARFTypeAttributes;
 
 class DWARFASTParserClang : public lldb_private::plugin::dwarf::DWARFASTParser {
 public:
-  typedef lldb_private::plugin::dwarf::DWARFDIE DWARFDIE;
-
   DWARFASTParserClang(lldb_private::TypeSystemClang &ast);
 
   ~DWARFASTParserClang() override;
 
   // DWARFASTParser interface.
-  lldb::TypeSP ParseTypeFromDWARF(const lldb_private::SymbolContext &sc,
-                                  const DWARFDIE &die,
-                                  bool *type_is_new_ptr) override;
+  lldb::TypeSP
+  ParseTypeFromDWARF(const lldb_private::SymbolContext &sc,
+                     const lldb_private::plugin::dwarf::DWARFDIE &die,
+                     bool *type_is_new_ptr) override;
 
-  lldb_private::ConstString
-  ConstructDemangledNameFromDWARF(const DWARFDIE &die) override;
+  lldb_private::ConstString ConstructDemangledNameFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
   lldb_private::Function *
   ParseFunctionFromDWARF(lldb_private::CompileUnit &comp_unit,
-                         const DWARFDIE &die,
+                         const lldb_private::plugin::dwarf::DWARFDIE &die,
                          const lldb_private::AddressRange &func_range) override;
 
   bool
-  CompleteTypeFromDWARF(const DWARFDIE &die, lldb_private::Type *type,
+  CompleteTypeFromDWARF(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                        lldb_private::Type *type,
                         lldb_private::CompilerType &compiler_type) override;
 
-  lldb_private::CompilerDecl
-  GetDeclForUIDFromDWARF(const DWARFDIE &die) override;
+  lldb_private::CompilerDecl GetDeclForUIDFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
   void EnsureAllDIEsInDeclContextHaveBeenParsed(
       lldb_private::CompilerDeclContext decl_context) override;
 
-  lldb_private::CompilerDeclContext
-  GetDeclContextForUIDFromDWARF(const DWARFDIE &die) override;
+  lldb_private::CompilerDeclContext GetDeclContextForUIDFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
-  lldb_private::CompilerDeclContext
-  GetDeclContextContainingUIDFromDWARF(const DWARFDIE &die) override;
+  lldb_private::CompilerDeclContext GetDeclContextContainingUIDFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
   lldb_private::ClangASTImporter &GetClangASTImporter();
 
@@ -105,13 +105,8 @@ public:
   /// \return A string, including surrounding '<>', of the template parameters.
   /// If the DIE's name already has '<>', returns an empty ConstString because
   /// it's assumed that the caller is using the DIE name anyway.
-  lldb_private::ConstString
-  GetDIEClassTemplateParams(const DWARFDIE &die) override;
-
-  // Searching for definition DIE for the given DIE and return the type
-  // associated with the definition DIE, or nullptr if definition DIE is not
-  // found.
-  lldb_private::Type *FindDefinitionTypeForDIE(const DWARFDIE &die) override;
+  lldb_private::ConstString GetDIEClassTemplateParams(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
 protected:
   /// Protected typedefs and members.
@@ -123,7 +118,8 @@ protected:
       const lldb_private::plugin::dwarf::DWARFDebugInfoEntry *,
       clang::DeclContext *>
       DIEToDeclContextMap;
-  typedef std::multimap<const clang::DeclContext *, const DWARFDIE>
+  typedef std::multimap<const clang::DeclContext *,
+                        const lldb_private::plugin::dwarf::DWARFDIE>
       DeclContextToDIEMap;
   typedef llvm::DenseMap<
       const lldb_private::plugin::dwarf::DWARFDebugInfoEntry *,
@@ -141,11 +137,14 @@ protected:
   std::unique_ptr<lldb_private::ClangASTImporter> m_clang_ast_importer_up;
   /// @}
 
-  clang::DeclContext *GetDeclContextForBlock(const DWARFDIE &die);
+  clang::DeclContext *
+  GetDeclContextForBlock(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::BlockDecl *ResolveBlockDIE(const DWARFDIE &die);
+  clang::BlockDecl *
+  ResolveBlockDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::NamespaceDecl *ResolveNamespaceDIE(const DWARFDIE &die);
+  clang::NamespaceDecl *
+  ResolveNamespaceDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   /// Returns the namespace decl that a DW_TAG_imported_declaration imports.
   ///
@@ -156,86 +155,96 @@ protected:
   ///          'die' imports. If the imported entity is not a namespace
   ///          or another import declaration, returns nullptr. If an error
   ///          occurs, returns nullptr.
-  clang::NamespaceDecl *ResolveImportedDeclarationDIE(const DWARFDIE &die);
+  clang::NamespaceDecl *ResolveImportedDeclarationDIE(
+      const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  bool ParseTemplateDIE(const DWARFDIE &die,
+  bool ParseTemplateDIE(const lldb_private::plugin::dwarf::DWARFDIE &die,
                         lldb_private::TypeSystemClang::TemplateParameterInfos
                             &template_param_infos);
 
   bool ParseTemplateParameterInfos(
-      const DWARFDIE &parent_die,
+      const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
       lldb_private::TypeSystemClang::TemplateParameterInfos
           &template_param_infos);
 
-  std::string GetCPlusPlusQualifiedName(const DWARFDIE &die);
+  std::string
+  GetCPlusPlusQualifiedName(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   bool ParseChildMembers(
-      const DWARFDIE &die, lldb_private::CompilerType &class_compiler_type,
+      const lldb_private::plugin::dwarf::DWARFDIE &die,
+      lldb_private::CompilerType &class_compiler_type,
       std::vector<std::unique_ptr<clang::CXXBaseSpecifier>> &base_classes,
-      std::vector<DWARFDIE> &member_function_dies,
-      std::vector<DWARFDIE> &contained_type_dies,
+      std::vector<lldb_private::plugin::dwarf::DWARFDIE> &member_function_dies,
+      std::vector<lldb_private::plugin::dwarf::DWARFDIE> &contained_type_dies,
       DelayedPropertyList &delayed_properties,
       const lldb::AccessType default_accessibility,
       lldb_private::ClangASTImporter::LayoutInfo &layout_info);
 
   size_t
   ParseChildParameters(clang::DeclContext *containing_decl_ctx,
-                       const DWARFDIE &parent_die, bool skip_artificial,
-                       bool &is_static, bool &is_variadic,
+                       const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+                       bool skip_artificial, bool &is_static, bool &is_variadic,
                        bool &has_template_params,
                        std::vector<lldb_private::CompilerType> &function_args,
                        std::vector<clang::ParmVarDecl *> &function_param_decls,
                        unsigned &type_quals);
 
-  size_t ParseChildEnumerators(lldb_private::CompilerType &compiler_type,
-                               bool is_signed, uint32_t enumerator_byte_size,
-                               const DWARFDIE &parent_die);
+  size_t ParseChildEnumerators(
+      lldb_private::CompilerType &compiler_type, bool is_signed,
+      uint32_t enumerator_byte_size,
+      const lldb_private::plugin::dwarf::DWARFDIE &parent_die);
 
   /// Parse a structure, class, or union type DIE.
-  lldb::TypeSP ParseStructureLikeDIE(const lldb_private::SymbolContext &sc,
-                                     const DWARFDIE &die,
-                                     ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP
+  ParseStructureLikeDIE(const lldb_private::SymbolContext &sc,
+                        const lldb_private::plugin::dwarf::DWARFDIE &die,
+                        ParsedDWARFTypeAttributes &attrs);
 
-  clang::Decl *GetClangDeclForDIE(const DWARFDIE &die);
+  clang::Decl *
+  GetClangDeclForDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::DeclContext *GetClangDeclContextForDIE(const DWARFDIE &die);
+  clang::DeclContext *
+  GetClangDeclContextForDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::DeclContext *GetClangDeclContextContainingDIE(const DWARFDIE &die,
-                                                       DWARFDIE *decl_ctx_die);
-  lldb_private::OptionalClangModuleID GetOwningClangModule(const DWARFDIE &die);
+  clang::DeclContext *GetClangDeclContextContainingDIE(
+      const lldb_private::plugin::dwarf::DWARFDIE &die,
+      lldb_private::plugin::dwarf::DWARFDIE *decl_ctx_die);
+  lldb_private::OptionalClangModuleID
+  GetOwningClangModule(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  bool CopyUniqueClassMethodTypes(const DWARFDIE &src_class_die,
-                                  const DWARFDIE &dst_class_die,
-                                  lldb_private::Type *class_type,
-                                  std::vector<DWARFDIE> &failures);
+  bool CopyUniqueClassMethodTypes(
+      const lldb_private::plugin::dwarf::DWARFDIE &src_class_die,
+      const lldb_private::plugin::dwarf::DWARFDIE &dst_class_die,
+      lldb_private::Type *class_type,
+      std::vector<lldb_private::plugin::dwarf::DWARFDIE> &failures);
 
-  clang::DeclContext *GetCachedClangDeclContextForDIE(const DWARFDIE &die);
+  clang::DeclContext *GetCachedClangDeclContextForDIE(
+      const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  void LinkDeclContextToDIE(clang::DeclContext *decl_ctx, const DWARFDIE &die);
+  void LinkDeclContextToDIE(clang::DeclContext *decl_ctx,
+                            const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  void LinkDeclToDIE(clang::Decl *decl, const DWARFDIE &die);
+  void LinkDeclToDIE(clang::Decl *decl,
+                     const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   /// If \p type_sp is valid, calculate and set its symbol context scope, and
   /// update the type list for its backing symbol file.
   ///
   /// Returns \p type_sp.
-  lldb::TypeSP
-  UpdateSymbolContextScopeForType(const lldb_private::SymbolContext &sc,
-                                  const DWARFDIE &die, lldb::TypeSP type_sp);
+  lldb::TypeSP UpdateSymbolContextScopeForType(
+      const lldb_private::SymbolContext &sc,
+      const lldb_private::plugin::dwarf::DWARFDIE &die, lldb::TypeSP type_sp);
 
   /// Follow Clang Module Skeleton CU references to find a type definition.
-  lldb::TypeSP ParseTypeFromClangModule(const lldb_private::SymbolContext &sc,
-                                        const DWARFDIE &die,
-                                        lldb_private::Log *log);
+  lldb::TypeSP
+  ParseTypeFromClangModule(const lldb_private::SymbolContext &sc,
+                           const lldb_private::plugin::dwarf::DWARFDIE &die,
+                           lldb_private::Log *log);
 
   // Return true if this type is a declaration to a type in an external
   // module.
-  lldb::ModuleSP GetModuleForType(const DWARFDIE &die);
-
-  void PrepareContextToReceiveMembers(clang::DeclContext *decl_ctx,
-                                      const DWARFDIE &decl_ctx_die,
-                                      const DWARFDIE &die,
-                                      const char *type_name_cstr);
+  lldb::ModuleSP
+  GetModuleForType(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   static bool classof(const DWARFASTParser *Parser) {
     return Parser->GetKind() == Kind::DWARFASTParserClang;
@@ -265,8 +274,10 @@ private:
 
   /// Parsed form of all attributes that are relevant for parsing type members.
   struct MemberAttributes {
-    explicit MemberAttributes(const DWARFDIE &die, const DWARFDIE &parent_die,
-                              lldb::ModuleSP module_sp);
+    explicit MemberAttributes(
+        const lldb_private::plugin::dwarf::DWARFDIE &die,
+        const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+        lldb::ModuleSP module_sp);
     const char *name = nullptr;
     /// Indicates how many bits into the word (according to the host endianness)
     /// the low-order bit of the field starts. Can be negative.
@@ -313,12 +324,15 @@ private:
   /// created property.
   /// \param delayed_properties The list of delayed properties that the result
   /// will be appended to.
-  void ParseObjCProperty(const DWARFDIE &die, const DWARFDIE &parent_die,
-                         const lldb_private::CompilerType &class_clang_type,
-                         DelayedPropertyList &delayed_properties);
+  void
+  ParseObjCProperty(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                    const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+                    const lldb_private::CompilerType &class_clang_type,
+                    DelayedPropertyList &delayed_properties);
 
   void
-  ParseSingleMember(const DWARFDIE &die, const DWARFDIE &parent_die,
+  ParseSingleMember(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                    const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
                     const lldb_private::CompilerType &class_clang_type,
                     lldb::AccessType default_accessibility,
                     lldb_private::ClangASTImporter::LayoutInfo &layout_info,
@@ -336,25 +350,31 @@ private:
   /// \param[in] class_clang_type The parent RecordType of the static
   ///                             member this function will create.
   void CreateStaticMemberVariable(
-      const DWARFDIE &die, const MemberAttributes &attrs,
+      const lldb_private::plugin::dwarf::DWARFDIE &die,
+      const MemberAttributes &attrs,
       const lldb_private::CompilerType &class_clang_type);
 
-  bool CompleteRecordType(const DWARFDIE &die, lldb_private::Type *type,
+  bool CompleteRecordType(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                          lldb_private::Type *type,
                           lldb_private::CompilerType &clang_type);
-  bool CompleteEnumType(const DWARFDIE &die, lldb_private::Type *type,
+  bool CompleteEnumType(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                        lldb_private::Type *type,
                         lldb_private::CompilerType &clang_type);
 
-  lldb::TypeSP ParseTypeModifier(const lldb_private::SymbolContext &sc,
-                                 const DWARFDIE &die,
-                                 ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP
+  ParseTypeModifier(const lldb_private::SymbolContext &sc,
+                    const lldb_private::plugin::dwarf::DWARFDIE &die,
+                    ParsedDWARFTypeAttributes &attrs);
   lldb::TypeSP ParseEnum(const lldb_private::SymbolContext &sc,
-                         const DWARFDIE &die, ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP ParseSubroutine(const DWARFDIE &die,
+                         const lldb_private::plugin::dwarf::DWARFDIE &die,
+                         ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP ParseSubroutine(const lldb_private::plugin::dwarf::DWARFDIE &die,
                                const ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP ParseArrayType(const DWARFDIE &die,
+  lldb::TypeSP ParseArrayType(const lldb_private::plugin::dwarf::DWARFDIE &die,
                               const ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP ParsePointerToMemberType(const DWARFDIE &die,
-                                        const ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP
+  ParsePointerToMemberType(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                           const ParsedDWARFTypeAttributes &attrs);
 
   /// Parses a DW_TAG_inheritance DIE into a base/super class.
   ///
@@ -371,7 +391,8 @@ private:
   /// \param layout_info The layout information that will be updated for C++
   /// base classes with the base offset.
   void ParseInheritance(
-      const DWARFDIE &die, const DWARFDIE &parent_die,
+      const lldb_private::plugin::dwarf::DWARFDIE &die,
+      const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
       const lldb_private::CompilerType class_clang_type,
       const lldb::AccessType default_accessibility,
       const lldb::ModuleSP &module_sp,
@@ -388,7 +409,8 @@ private:
   /// \param layout_info The layout information that will be updated for
   //   base classes with the base offset
   void
-  ParseRustVariantPart(DWARFDIE &die, const DWARFDIE &parent_die,
+  ParseRustVariantPart(lldb_private::plugin::dwarf::DWARFDIE &die,
+                       const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
                        lldb_private::CompilerType &class_clang_type,
                        const lldb::AccessType default_accesibility,
                        lldb_private::ClangASTImporter::LayoutInfo &layout_info);
@@ -398,9 +420,8 @@ private:
 /// Some attributes are relevant for all kinds of types (declaration), while
 /// others are only meaningful to a specific type (is_virtual)
 struct ParsedDWARFTypeAttributes {
-  typedef lldb_private::plugin::dwarf::DWARFDIE DWARFDIE;
-
-  explicit ParsedDWARFTypeAttributes(const DWARFDIE &die);
+  explicit ParsedDWARFTypeAttributes(
+      const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   lldb::AccessType accessibility = lldb::eAccessNone;
   bool is_artificial = false;
@@ -417,7 +438,7 @@ struct ParsedDWARFTypeAttributes {
   const char *mangled_name = nullptr;
   lldb_private::ConstString name;
   lldb_private::Declaration decl;
-  DWARFDIE object_pointer;
+  lldb_private::plugin::dwarf::DWARFDIE object_pointer;
   lldb_private::plugin::dwarf::DWARFFormValue abstract_origin;
   lldb_private::plugin::dwarf::DWARFFormValue containing_type;
   lldb_private::plugin::dwarf::DWARFFormValue signature;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index 5a07fd30fbf7..f6f152726bf7 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -1632,33 +1632,27 @@ bool SymbolFileDWARF::CompleteType(CompilerType &compiler_type) {
     return true;
   }
 
-  // Once we start resolving this type, remove it from the forward
-  // declaration map in case anyone's child members or other types require this
-  // type to get resolved.
-  DWARFDIE dwarf_die = GetDIE(die_it->second);
-  GetForwardDeclCompilerTypeToDIE().erase(die_it);
-  Type *type = nullptr;
-  if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU()))
-    type = dwarf_ast->FindDefinitionTypeForDIE(dwarf_die);
-  if (!type)
-    return false;
-
-  die_it = GetForwardDeclCompilerTypeToDIE().find(
-      compiler_type_no_qualifiers.GetOpaqueQualType());
-  if (die_it != GetForwardDeclCompilerTypeToDIE().end()) {
-    dwarf_die = GetDIE(die_it->getSecond());
+  DWARFDIE dwarf_die = GetDIE(die_it->getSecond());
+  if (dwarf_die) {
+    // Once we start resolving this type, remove it from the forward
+    // declaration map in case anyone child members or other types require this
+    // type to get resolved. The type will get resolved when all of the calls
+    // to SymbolFileDWARF::ResolveClangOpaqueTypeDefinition are done.
     GetForwardDeclCompilerTypeToDIE().erase(die_it);
-  }
 
-  if (Log *log = GetLog(DWARFLog::DebugInfo | DWARFLog::TypeCompletion))
-    GetObjectFile()->GetModule()->LogMessageVerboseBacktrace(
-        log, "{0:x8}: {1} ({2}) '{3}' resolving forward declaration...",
-        dwarf_die.GetID(), DW_TAG_value_to_name(dwarf_die.Tag()),
-        dwarf_die.Tag(), type->GetName().AsCString());
-  assert(compiler_type);
-  if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU()))
-    return dwarf_ast->CompleteTypeFromDWARF(dwarf_die, type, compiler_type);
-  return true;
+    Type *type = GetDIEToType().lookup(dwarf_die.GetDIE());
+
+    Log *log = GetLog(DWARFLog::DebugInfo | DWARFLog::TypeCompletion);
+    if (log)
+      GetObjectFile()->GetModule()->LogMessageVerboseBacktrace(
+          log, "{0:x8}: {1} ({2}) '{3}' resolving forward declaration...",
+          dwarf_die.GetID(), DW_TAG_value_to_name(dwarf_die.Tag()),
+          dwarf_die.Tag(), type->GetName().AsCString());
+    assert(compiler_type);
+    if (DWARFASTParser *dwarf_ast = GetDWARFParser(*dwarf_die.GetCU()))
+      return dwarf_ast->CompleteTypeFromDWARF(dwarf_die, type, compiler_type);
+  }
+  return false;
 }
 
 Type *SymbolFileDWARF::ResolveType(const DWARFDIE &die,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index 94aa810680c5..7282c08c6857 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -533,12 +533,8 @@ protected:
   NameToOffsetMap m_function_scope_qualified_name_map;
   std::unique_ptr<DWARFDebugRanges> m_ranges;
   UniqueDWARFASTTypeMap m_unique_ast_type_map;
-  // A map from DIE to lldb_private::Type. For record type, the key might be
-  // either declaration DIE or definition DIE.
   DIEToTypePtr m_die_to_type;
   DIEToVariableSP m_die_to_variable_sp;
-  // A map from CompilerType to the struct/class/union/enum DIE (might be a
-  // declaration or a definition) that is used to construct it.
   CompilerTypeToDIE m_forward_decl_compiler_type_to_die;
   llvm::DenseMap<dw_offset_t, std::unique_ptr<SupportFileList>>
       m_type_unit_support_files;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
index 4762356034ca..223518f0ae82 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
@@ -13,67 +13,66 @@
 using namespace lldb_private::dwarf;
 using namespace lldb_private::plugin::dwarf;
 
-UniqueDWARFASTType *UniqueDWARFASTTypeList::Find(
-    const DWARFDIE &die, const lldb_private::Declaration &decl,
-    const int32_t byte_size, bool is_forward_declaration) {
-  for (UniqueDWARFASTType &udt : m_collection) {
+bool UniqueDWARFASTTypeList::Find(const DWARFDIE &die,
+                                  const lldb_private::Declaration &decl,
+                                  const int32_t byte_size,
+                                  UniqueDWARFASTType &entry) const {
+  for (const UniqueDWARFASTType &udt : m_collection) {
     // Make sure the tags match
     if (udt.m_die.Tag() == die.Tag()) {
-      // If they are not both definition DIEs or both declaration DIEs, then
-      // don't check for byte size and declaration location, because declaration
-      // DIEs usually don't have those info.
-      bool matching_size_declaration =
-          udt.m_is_forward_declaration != is_forward_declaration
-              ? true
-              : (udt.m_byte_size < 0 || byte_size < 0 ||
-                 udt.m_byte_size == byte_size) &&
-                    udt.m_declaration == decl;
-      if (!matching_size_declaration)
-        continue;
-      // The type has the same name, and was defined on the same file and
-      // line. Now verify all of the parent DIEs match.
-      DWARFDIE parent_arg_die = die.GetParent();
-      DWARFDIE parent_pos_die = udt.m_die.GetParent();
-      bool match = true;
-      bool done = false;
-      while (!done && match && parent_arg_die && parent_pos_die) {
-        const dw_tag_t parent_arg_tag = parent_arg_die.Tag();
-        const dw_tag_t parent_pos_tag = parent_pos_die.Tag();
-        if (parent_arg_tag == parent_pos_tag) {
-          switch (parent_arg_tag) {
-          case DW_TAG_class_type:
-          case DW_TAG_structure_type:
-          case DW_TAG_union_type:
-          case DW_TAG_namespace: {
-            const char *parent_arg_die_name = parent_arg_die.GetName();
-            if (parent_arg_die_name == nullptr) {
-              // Anonymous (i.e. no-name) struct
-              match = false;
-            } else {
-              const char *parent_pos_die_name = parent_pos_die.GetName();
-              if (parent_pos_die_name == nullptr ||
-                  ((parent_arg_die_name != parent_pos_die_name) &&
-                   strcmp(parent_arg_die_name, parent_pos_die_name)))
-                match = false;
+      // Validate byte sizes of both types only if both are valid.
+      if (udt.m_byte_size < 0 || byte_size < 0 ||
+          udt.m_byte_size == byte_size) {
+        // Make sure the file and line match
+        if (udt.m_declaration == decl) {
+          // The type has the same name, and was defined on the same file and
+          // line. Now verify all of the parent DIEs match.
+          DWARFDIE parent_arg_die = die.GetParent();
+          DWARFDIE parent_pos_die = udt.m_die.GetParent();
+          bool match = true;
+          bool done = false;
+          while (!done && match && parent_arg_die && parent_pos_die) {
+            const dw_tag_t parent_arg_tag = parent_arg_die.Tag();
+            const dw_tag_t parent_pos_tag = parent_pos_die.Tag();
+            if (parent_arg_tag == parent_pos_tag) {
+              switch (parent_arg_tag) {
+              case DW_TAG_class_type:
+              case DW_TAG_structure_type:
+              case DW_TAG_union_type:
+              case DW_TAG_namespace: {
+                const char *parent_arg_die_name = parent_arg_die.GetName();
+                if (parent_arg_die_name ==
+                    nullptr) // Anonymous (i.e. no-name) struct
+                {
+                  match = false;
+                } else {
+                  const char *parent_pos_die_name = parent_pos_die.GetName();
+                  if (parent_pos_die_name == nullptr ||
+                      ((parent_arg_die_name != parent_pos_die_name) &&
+                       strcmp(parent_arg_die_name, parent_pos_die_name)))
+                    match = false;
+                }
+              } break;
+
+              case DW_TAG_compile_unit:
+              case DW_TAG_partial_unit:
+                done = true;
+                break;
+              default:
+                break;
+              }
             }
-          } break;
+            parent_arg_die = parent_arg_die.GetParent();
+            parent_pos_die = parent_pos_die.GetParent();
+          }
 
-          case DW_TAG_compile_unit:
-          case DW_TAG_partial_unit:
-            done = true;
-            break;
-          default:
-            break;
+          if (match) {
+            entry = udt;
+            return true;
           }
         }
-        parent_arg_die = parent_arg_die.GetParent();
-        parent_pos_die = parent_pos_die.GetParent();
-      }
-
-      if (match) {
-        return &udt;
       }
     }
   }
-  return nullptr;
+  return false;
 }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
index 29e5c02dcbe1..bf3cbae55e5c 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
@@ -23,19 +23,31 @@ public:
   // Constructors and Destructors
   UniqueDWARFASTType() : m_type_sp(), m_die(), m_declaration() {}
 
+  UniqueDWARFASTType(lldb::TypeSP &type_sp, const DWARFDIE &die,
+                     const Declaration &decl, int32_t byte_size)
+      : m_type_sp(type_sp), m_die(die), m_declaration(decl),
+        m_byte_size(byte_size) {}
+
   UniqueDWARFASTType(const UniqueDWARFASTType &rhs)
       : m_type_sp(rhs.m_type_sp), m_die(rhs.m_die),
-        m_declaration(rhs.m_declaration), m_byte_size(rhs.m_byte_size),
-        m_is_forward_declaration(rhs.m_is_forward_declaration) {}
+        m_declaration(rhs.m_declaration), m_byte_size(rhs.m_byte_size) {}
 
   ~UniqueDWARFASTType() = default;
 
+  UniqueDWARFASTType &operator=(const UniqueDWARFASTType &rhs) {
+    if (this != &rhs) {
+      m_type_sp = rhs.m_type_sp;
+      m_die = rhs.m_die;
+      m_declaration = rhs.m_declaration;
+      m_byte_size = rhs.m_byte_size;
+    }
+    return *this;
+  }
+
   lldb::TypeSP m_type_sp;
   DWARFDIE m_die;
   Declaration m_declaration;
   int32_t m_byte_size = -1;
-  // True if the m_die is a forward declaration DIE.
-  bool m_is_forward_declaration = true;
 };
 
 class UniqueDWARFASTTypeList {
@@ -50,9 +62,8 @@ public:
     m_collection.push_back(entry);
   }
 
-  UniqueDWARFASTType *Find(const DWARFDIE &die, const Declaration &decl,
-                           const int32_t byte_size,
-                           bool is_forward_declaration);
+  bool Find(const DWARFDIE &die, const Declaration &decl,
+            const int32_t byte_size, UniqueDWARFASTType &entry) const;
 
 protected:
   typedef std::vector<UniqueDWARFASTType> collection;
@@ -69,15 +80,14 @@ public:
     m_collection[name.GetCString()].Append(entry);
   }
 
-  UniqueDWARFASTType *Find(ConstString name, const DWARFDIE &die,
-                           const Declaration &decl, const int32_t byte_size,
-                           bool is_forward_declaration) {
+  bool Find(ConstString name, const DWARFDIE &die, const Declaration &decl,
+            const int32_t byte_size, UniqueDWARFASTType &entry) const {
     const char *unique_name_cstr = name.GetCString();
-    collection::iterator pos = m_collection.find(unique_name_cstr);
+    collection::const_iterator pos = m_collection.find(unique_name_cstr);
     if (pos != m_collection.end()) {
-      return pos->second.Find(die, decl, byte_size, is_forward_declaration);
+      return pos->second.Find(die, decl, byte_size, entry);
     }
-    return nullptr;
+    return false;
   }
 
 protected:
-- 
GitLab


From 70de9b21cbdeb1297108c4ee520b8f6dbd6496a7 Mon Sep 17 00:00:00 2001
From: Jim Ingham <jingham@apple.com>
Date: Mon, 13 May 2024 17:13:12 -0700
Subject: [PATCH 0655/1206] Revert "[lldb][DWARF] Do not complete type from
 declaration die. (#91799)"

This reverts commit a7eff59f78f08f8ef0487dfe2a136fb311af4fd0.
---
 lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 034817c3b4fa..f8101aba5c62 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -2306,11 +2306,6 @@ bool DWARFASTParserClang::CompleteTypeFromDWARF(const DWARFDIE &die,
 
   if (!die)
     return false;
-  ParsedDWARFTypeAttributes attrs(die);
-  bool is_forward_declaration = IsForwardDeclaration(
-      die, attrs, SymbolFileDWARF::GetLanguage(*die.GetCU()));
-  if (is_forward_declaration)
-    return false;
 
   const dw_tag_t tag = die.Tag();
 
-- 
GitLab


From b06f97b039b3a4f2397281609069b2aaad86dd59 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Mon, 13 May 2024 18:02:38 -0700
Subject: [PATCH 0656/1206] [BOLT] Allow pass-through blocks in
 YAMLProfileReader (#91828)

---
 bolt/lib/Profile/DataReader.cpp              |  1 +
 bolt/lib/Profile/YAMLProfileReader.cpp       | 29 ++++++---
 bolt/test/X86/profile-passthrough-block.test | 67 ++++++++++++++++++++
 3 files changed, 88 insertions(+), 9 deletions(-)
 create mode 100644 bolt/test/X86/profile-passthrough-block.test

diff --git a/bolt/lib/Profile/DataReader.cpp b/bolt/lib/Profile/DataReader.cpp
index b2511ba10399..06c5e96b7806 100644
--- a/bolt/lib/Profile/DataReader.cpp
+++ b/bolt/lib/Profile/DataReader.cpp
@@ -775,6 +775,7 @@ bool DataReader::recordBranch(BinaryFunction &BF, uint64_t From, uint64_t To,
     if (collectedInBoltedBinary() && FromBB == ToBB)
       return true;
 
+    // Allow passthrough blocks.
     BinaryBasicBlock *FTSuccessor = FromBB->getConditionalSuccessor(false);
     if (FTSuccessor && FTSuccessor->succ_size() == 1 &&
         FTSuccessor->getSuccessor(ToBB->getLabel())) {
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index e4673f6e3c30..978a7cadfe79 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -218,17 +218,28 @@ bool YAMLProfileReader::parseFunctionProfile(
         continue;
       }
 
-      BinaryBasicBlock &SuccessorBB = *Order[YamlSI.Index];
-      if (!BB.getSuccessor(SuccessorBB.getLabel())) {
-        if (opts::Verbosity >= 1)
-          errs() << "BOLT-WARNING: no successor for block " << BB.getName()
-                 << " that matches index " << YamlSI.Index << " or block "
-                 << SuccessorBB.getName() << '\n';
-        ++MismatchedEdges;
-        continue;
+      BinaryBasicBlock *ToBB = Order[YamlSI.Index];
+      if (!BB.getSuccessor(ToBB->getLabel())) {
+        // Allow passthrough blocks.
+        BinaryBasicBlock *FTSuccessor = BB.getConditionalSuccessor(false);
+        if (FTSuccessor && FTSuccessor->succ_size() == 1 &&
+            FTSuccessor->getSuccessor(ToBB->getLabel())) {
+          BinaryBasicBlock::BinaryBranchInfo &FTBI =
+              FTSuccessor->getBranchInfo(*ToBB);
+          FTBI.Count += YamlSI.Count;
+          FTBI.MispredictedCount += YamlSI.Mispreds;
+          ToBB = FTSuccessor;
+        } else {
+          if (opts::Verbosity >= 1)
+            errs() << "BOLT-WARNING: no successor for block " << BB.getName()
+                   << " that matches index " << YamlSI.Index << " or block "
+                   << ToBB->getName() << '\n';
+          ++MismatchedEdges;
+          continue;
+        }
       }
 
-      BinaryBasicBlock::BinaryBranchInfo &BI = BB.getBranchInfo(SuccessorBB);
+      BinaryBasicBlock::BinaryBranchInfo &BI = BB.getBranchInfo(*ToBB);
       BI.Count += YamlSI.Count;
       BI.MispredictedCount += YamlSI.Mispreds;
     }
diff --git a/bolt/test/X86/profile-passthrough-block.test b/bolt/test/X86/profile-passthrough-block.test
new file mode 100644
index 000000000000..1b875885260d
--- /dev/null
+++ b/bolt/test/X86/profile-passthrough-block.test
@@ -0,0 +1,67 @@
+## Test YAMLProfileReader support for pass-through blocks in non-matching edges:
+## match the profile edge A -> C to the CFG with blocks A -> B -> C.
+
+# REQUIRES: system-linux
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %t/main.s -o %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q -nostdlib
+# RUN: llvm-bolt %t.exe -o %t.out --data %t/yaml --profile-ignore-hash -v=1 \
+# RUN:   --print-cfg 2>&1 | FileCheck %s
+
+# CHECK: Binary Function "main" after building cfg
+# CHECK: Profile Acc : 100.0%
+# CHECK-NOT: BOLT-WARNING: no successor for block .LFT0 that matches index 3 or block .Ltmp0
+
+#--- main.s
+.globl main
+.type	main, @function
+main:
+  .cfi_startproc
+.LBB00:
+  pushq   %rbp
+  movq    %rsp, %rbp
+  subq    $16, %rsp
+  testq   %rax, %rax
+  js      .LBB03
+.LBB01:
+  jne     .LBB04
+.LBB02:
+  nop
+.LBB03:
+  xorl    %eax, %eax
+  addq    $16, %rsp
+  popq    %rbp
+  retq
+.LBB04:
+  xorl    %eax, %eax
+  addq    $16, %rsp
+  popq    %rbp
+  retq
+## For relocations against .text
+.LBB05:
+  call exit
+  .cfi_endproc
+  .size	main, .-main
+
+#--- yaml
+---
+header:
+  profile-version: 1
+  binary-name:     'profile-passthrough-block.s.tmp.exe'
+  binary-build-id: '<unknown>'
+  profile-flags:   [ lbr ]
+  profile-origin:  branch profile reader
+  profile-events:  ''
+  dfs-order:       false
+  hash-func:       xxh3
+functions:
+  - name:            main
+    fid:             0
+    hash:            0x0000000000000000
+    exec:            1
+    nblocks:         6
+    blocks:
+      - bid:             1
+        insns:           1
+        succ:            [ { bid: 3, cnt: 1} ]
+...
-- 
GitLab


From b1c958e50e5d58040c53e2aa822f4dfbcbf9c273 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Tue, 14 May 2024 10:06:10 +0900
Subject: [PATCH 0657/1206] [AArch64] Clarify atomic load/store size condition
 (NFCI) (#91907)

This is currently bailing out on MemSizeInBytes larger than 64 bytes.
However, the following code can only handle sizes up to 8 bytes.
Possibly there was confusion here between MemSizeInBytes and
MemSizeInBits.

I *think* that this can't actually result in an out of bounds read of
the opcode table because we'll only ever mark loads/stores of up to 8
bytes as legal (16 byte atomics are custom-legalized earlier). As such,
I've changed this condition to an assert.
---
 llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 1b65ae7b4782..3b3c1fc8b27b 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2861,8 +2861,8 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
         Order != AtomicOrdering::Unordered &&
         Order != AtomicOrdering::Monotonic) {
       assert(!isa<GZExtLoad>(LdSt));
-      if (MemSizeInBytes > 64)
-        return false;
+      assert(MemSizeInBytes <= 8 &&
+             "128-bit atomics should already be custom-legalized");
 
       if (isa<GLoad>(LdSt)) {
         static constexpr unsigned LDAPROpcodes[] = {
-- 
GitLab


From 4420ea7a4971eadad528c0cd609da471a7614422 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Tue, 14 May 2024 10:06:41 +0900
Subject: [PATCH 0658/1206] [StringMap] Move free into StringMapImpl dtor (NFC)
 (#91908)

StringMapImpl allocates the memory for the table, but does not have a
dtor that free it. Instead, StringMap (which inherits from
StringMapImpl) contains the free call. I don't really see a good reason
why this free is performed in the "wrong" class, so move it into
StringMapImpl.
---
 llvm/include/llvm/ADT/StringMap.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/ADT/StringMap.h b/llvm/include/llvm/ADT/StringMap.h
index daaf82654e09..9b58af732739 100644
--- a/llvm/include/llvm/ADT/StringMap.h
+++ b/llvm/include/llvm/ADT/StringMap.h
@@ -53,6 +53,7 @@ protected:
   }
 
   StringMapImpl(unsigned InitSize, unsigned ItemSize);
+  ~StringMapImpl() { free(TheTable); }
   unsigned RehashTable(unsigned BucketNo = 0);
 
   /// LookupBucketFor - Look up the bucket that the specified string should end
@@ -203,7 +204,6 @@ public:
         }
       }
     }
-    free(TheTable);
   }
 
   using AllocTy::getAllocator;
-- 
GitLab


From 22cc4488c9dde0f0d27c0cfc58f6e82517c83f7f Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 13 May 2024 18:18:30 -0700
Subject: [PATCH 0659/1206] [bazel] Port libc #91905

---
 .../llvm-project-overlay/libc/BUILD.bazel     | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index ce61c432c2ed..9cdcc7577b46 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -122,6 +122,11 @@ libc_support_library(
     hdrs = ["hdr/errno_macros.h"],
 )
 
+libc_support_library(
+    name = "hdr_time_macros",
+    hdrs = ["hdr/time_macros.h"],
+)
+
 ############################ Type Proxy Header Files ###########################
 
 libc_support_library(
@@ -149,6 +154,11 @@ libc_support_library(
     hdrs = ["hdr/types/struct_timespec.h"],
 )
 
+libc_support_library(
+    name = "types_time_t",
+    hdrs = ["hdr/types/time_t.h"],
+)
+
 ############################### Support libraries ##############################
 
 libc_support_library(
@@ -1115,6 +1125,7 @@ libc_support_library(
         ":__support_cpp_optional",
         ":__support_osutil_syscall",
         ":__support_threads_linux_futex_word_type",
+        ":__support_time_linux",
         ":types_struct_timespec",
     ],
 )
@@ -1139,6 +1150,28 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_time",
+    hdrs = glob(["src/__support/time/*.h"]),
+    deps = [
+        ":hdr_time_macros",
+        ":types_time_t",
+    ],
+)
+
+libc_support_library(
+    name = "__support_time_linux",
+    hdrs = glob(["src/__support/time/linux/**/*.h"]),
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    deps = [
+        ":__support_time",
+        ":hdr_time_macros",
+    ],
+)
+
 ############################### errno targets ################################
 
 libc_function(
-- 
GitLab


From e6b2197a89f5d6d0f56a03c03b8afda561eee899 Mon Sep 17 00:00:00 2001
From: Jim Ingham <jingham@apple.com>
Date: Mon, 13 May 2024 18:16:47 -0700
Subject: [PATCH 0660/1206] Revert a test that was failing after a previous
 reversion.

This test was modified as part of the commit:

9a7262c2601874e5aa64c5db19746770212d4b44

but without that patch this test is failing.  Remove the test for now
till the issue with the original patch can be sorted out.
---
 .../delayed-definition-die-searching.test     | 36 -------------------
 1 file changed, 36 deletions(-)
 delete mode 100644 lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test

diff --git a/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test b/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
deleted file mode 100644
index d253981b498c..000000000000
--- a/lldb/test/Shell/SymbolFile/DWARF/delayed-definition-die-searching.test
+++ /dev/null
@@ -1,36 +0,0 @@
-# Test definition DIE searching is delayed until complete type is required.
-
-# UNSUPPORTED: system-windows
-
-# RUN: split-file %s %t
-# RUN: %clangxx_host %t/main.cpp %t/t1_def.cpp -gdwarf -o %t.out
-# RUN: %lldb -b %t.out -s %t/lldb.cmd | FileCheck %s
-
-# CHECK: (lldb) p v1
-# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't2<t1>'
-# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't1'
-# CHECK: DW_TAG_structure_type (DW_TAG_structure_type) 't2<t1>' resolving forward declaration...
-# CHECK: (t2<t1>)  {}
-# CHECK: (lldb) p v2
-# CHECK: DWARFASTParserClang::ParseTypeFromDWARF{{.*}}DW_TAG_structure_type (DW_TAG_structure_type) name = 't1'
-# CHECK: DW_TAG_structure_type (DW_TAG_structure_type) 't1' resolving forward declaration...
-
-#--- lldb.cmd
-log enable dwarf comp
-p v1
-p v2
-
-#--- main.cpp
-template<typename T>
-struct t2 {
-};
-struct t1;
-t2<t1> v1; // this CU doesn't have definition DIE for t1, but only declaration DIE for it.
-int main() {
-}
-
-#--- t1_def.cpp
-struct t1 { // this CU contains definition DIE for t1.
-  int x;
-};
-t1 v2;
-- 
GitLab


From f12018eba11f8d4b74cf67dbc416c429c870a5f4 Mon Sep 17 00:00:00 2001
From: Congcong Cai <congcongcai0907@163.com>
Date: Tue, 14 May 2024 09:48:57 +0800
Subject: [PATCH 0661/1206] [clang-tidy] support expect no diagnosis test
 (#91293)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When someone wants to declare a test case without any diagnosis.
check-clang-tidy will failed with error message

```
CHECK-FIXES, CHECK-MESSAGES or CHECK-NOTES not found in the input
```

This PR want to check there are no diagnosis from clang-tidy when
CHECK-FIXES, CHECK-MESSAGES or CHECK-NOTES are not found.

It also changes the extension of a test case. `hxx` is not a valid test
case extension and won't be tested.

---------

Co-authored-by: Danny Mösch <danny.moesch@icloud.com>
---
 .../test/clang-tidy/check_clang_tidy.py       | 28 +++++++++++++++----
 .../checkers/misc/unused-using-decls.hpp      |  6 ++++
 .../checkers/misc/unused-using-decls.hxx      |  6 ----
 3 files changed, 28 insertions(+), 12 deletions(-)
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.hpp
 delete mode 100644 clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.hxx

diff --git a/clang-tools-extra/test/clang-tidy/check_clang_tidy.py b/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
index 6d4b466afa69..e92179ac82c6 100755
--- a/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
+++ b/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
@@ -99,6 +99,7 @@ class CheckRunner:
         self.has_check_fixes = False
         self.has_check_messages = False
         self.has_check_notes = False
+        self.expect_no_diagnosis = False
         self.export_fixes = args.export_fixes
         self.fixes = MessagePrefix("CHECK-FIXES")
         self.messages = MessagePrefix("CHECK-MESSAGES")
@@ -172,12 +173,21 @@ class CheckRunner:
                 )
 
             if not has_check_fix and not has_check_message and not has_check_note:
-                sys.exit(
-                    "%s, %s or %s not found in the input"
-                    % (self.fixes.prefix, self.messages.prefix, self.notes.prefix)
-                )
+                self.expect_no_diagnosis = True
 
-        assert self.has_check_fixes or self.has_check_messages or self.has_check_notes
+        expect_diagnosis = (
+            self.has_check_fixes or self.has_check_messages or self.has_check_notes
+        )
+        if self.expect_no_diagnosis and expect_diagnosis:
+            sys.exit(
+                "%s, %s or %s not found in the input"
+                % (
+                    self.fixes.prefix,
+                    self.messages.prefix,
+                    self.notes.prefix,
+                )
+            )
+        assert expect_diagnosis or self.expect_no_diagnosis
 
     def prepare_test_inputs(self):
         # Remove the contents of the CHECK lines to avoid CHECKs matching on
@@ -226,6 +236,10 @@ class CheckRunner:
         print("------------------------------------------------------------------")
         return clang_tidy_output
 
+    def check_no_diagnosis(self, clang_tidy_output):
+        if clang_tidy_output != "":
+            sys.exit("No diagnostics were expected, but found the ones above")
+
     def check_fixes(self):
         if self.has_check_fixes:
             try_run(
@@ -277,7 +291,9 @@ class CheckRunner:
             self.get_prefixes()
         self.prepare_test_inputs()
         clang_tidy_output = self.run_clang_tidy()
-        if self.export_fixes is None:
+        if self.expect_no_diagnosis:
+            self.check_no_diagnosis(clang_tidy_output)
+        elif self.export_fixes is None:
             self.check_fixes()
             self.check_messages(clang_tidy_output)
             self.check_notes(clang_tidy_output)
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.hpp b/clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.hpp
new file mode 100644
index 000000000000..4918aae16cb9
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.hpp
@@ -0,0 +1,6 @@
+// RUN: %check_clang_tidy %s misc-unused-using-decls %t
+
+// Verify that we don't generate the warnings on header files.
+namespace foo { class Foo {}; }
+
+using foo::Foo;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.hxx b/clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.hxx
deleted file mode 100644
index f15e4fae80c0..000000000000
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/unused-using-decls.hxx
+++ /dev/null
@@ -1,6 +0,0 @@
-// RUN: %check_clang_tidy %s misc-unused-using-decls %t -- --fix-notes -- -fno-delayed-template-parsing -isystem %S/Inputs
-
-// Verify that we don't generate the warnings on header files.
-namespace foo { class Foo {}; }
-
-using foo::Foo;
-- 
GitLab


From 881d45cd7d3c5ea97f0d409bab5c57ae7bd43ab2 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Tue, 14 May 2024 10:18:10 +0800
Subject: [PATCH 0662/1206] [X86][BF16] Do not combine FP_TRUNC + FP_EXTEND if
 they come from user (#91420)

As discussed in
https://github.com/llvm/llvm-project/commit/3cf8535dbf0bf5fafa99ea1f300e2384a7254fba
We are not allowed to combine explicit fptrunc/fpext from user.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp |  7 ++--
 llvm/test/CodeGen/X86/bfloat.ll         | 44 +++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index ecc5b3b3bf84..a57c10e784d9 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56741,6 +56741,7 @@ static SDValue combineFP16_TO_FP(SDNode *N, SelectionDAG &DAG,
 }
 
 static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
+                                TargetLowering::DAGCombinerInfo &DCI,
                                 const X86Subtarget &Subtarget) {
   EVT VT = N->getValueType(0);
   bool IsStrict = N->isStrictFPOpcode();
@@ -56749,8 +56750,8 @@ static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
 
   SDLoc dl(N);
   if (SrcVT.getScalarType() == MVT::bf16) {
-    if (!IsStrict && Src.getOpcode() == ISD::FP_ROUND &&
-        Src.getOperand(0).getValueType() == VT)
+    if (DCI.isAfterLegalizeDAG() && Src.getOpcode() == ISD::FP_ROUND &&
+        !IsStrict && Src.getOperand(0).getValueType() == VT)
       return Src.getOperand(0);
 
     if (!SrcVT.isVector())
@@ -57168,7 +57169,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case X86ISD::KSHIFTR:     return combineKSHIFT(N, DAG, DCI);
   case ISD::FP16_TO_FP:     return combineFP16_TO_FP(N, DAG, Subtarget);
   case ISD::STRICT_FP_EXTEND:
-  case ISD::FP_EXTEND:      return combineFP_EXTEND(N, DAG, Subtarget);
+  case ISD::FP_EXTEND:      return combineFP_EXTEND(N, DAG, DCI, Subtarget);
   case ISD::STRICT_FP_ROUND:
   case ISD::FP_ROUND:       return combineFP_ROUND(N, DAG, Subtarget);
   case X86ISD::VBROADCAST_LOAD:
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index 39d8e2d50c91..b3e04590075f 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -2420,3 +2420,47 @@ define <16 x bfloat> @concat_dup_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
   %a = shufflevector <8 x bfloat> %x, <8 x bfloat> %y, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x bfloat> %a
 }
+
+define float @trunc_ext(float %a) nounwind {
+; X86-LABEL: trunc_ext:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vcvtneps2bf16 %xmm0, %xmm0
+; X86-NEXT:    vmovw %xmm0, %eax
+; X86-NEXT:    shll $16, %eax
+; X86-NEXT:    vmovd %eax, %xmm0
+; X86-NEXT:    vmovd %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+;
+; SSE2-LABEL: trunc_ext:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pushq %rax
+; SSE2-NEXT:    callq __truncsfbf2@PLT
+; SSE2-NEXT:    pextrw $0, %xmm0, %eax
+; SSE2-NEXT:    shll $16, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    popq %rax
+; SSE2-NEXT:    retq
+;
+; FP16-LABEL: trunc_ext:
+; FP16:       # %bb.0:
+; FP16-NEXT:    vcvtneps2bf16 %xmm0, %xmm0
+; FP16-NEXT:    vmovw %xmm0, %eax
+; FP16-NEXT:    shll $16, %eax
+; FP16-NEXT:    vmovd %eax, %xmm0
+; FP16-NEXT:    retq
+;
+; AVXNC-LABEL: trunc_ext:
+; AVXNC:       # %bb.0:
+; AVXNC-NEXT:    {vex} vcvtneps2bf16 %xmm0, %xmm0
+; AVXNC-NEXT:    vmovd %xmm0, %eax
+; AVXNC-NEXT:    shll $16, %eax
+; AVXNC-NEXT:    vmovd %eax, %xmm0
+; AVXNC-NEXT:    retq
+  %b = fptrunc float %a to bfloat
+  %c = fpext bfloat %b to float
+  ret float %c
+}
-- 
GitLab


From c72e94382c21db2f5ff066d72103ac55eb8d2874 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Mon, 13 May 2024 19:19:15 -0700
Subject: [PATCH 0663/1206] [clang-format][NFC] Move
 LeftRightQualifierAlignmentFixer::is...() (#91930)

Move static member functions LeftRightQualifierAlignmentFixer::is...()
out of the class so that #91712 can reland.
---
 clang/lib/Format/QualifierAlignmentFixer.cpp  | 11 +--
 clang/lib/Format/QualifierAlignmentFixer.h    | 19 ++--
 clang/unittests/Format/QualifierFixerTest.cpp | 98 +++++++------------
 3 files changed, 52 insertions(+), 76 deletions(-)

diff --git a/clang/lib/Format/QualifierAlignmentFixer.cpp b/clang/lib/Format/QualifierAlignmentFixer.cpp
index c26353045672..36d0639041c6 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.cpp
+++ b/clang/lib/Format/QualifierAlignmentFixer.cpp
@@ -614,22 +614,21 @@ void prepareLeftRightOrderingForQualifierAlignmentFixer(
   }
 }
 
-bool LeftRightQualifierAlignmentFixer::isQualifierOrType(const FormatToken *Tok,
-                                                         bool IsCpp) {
+bool isQualifierOrType(const FormatToken *Tok, bool IsCpp) {
   return Tok &&
          (Tok->isTypeName(IsCpp) || Tok->is(tok::kw_auto) || isQualifier(Tok));
 }
 
-bool LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-    const FormatToken *Tok, const std::vector<tok::TokenKind> &Qualifiers,
-    bool IsCpp) {
+bool isConfiguredQualifierOrType(const FormatToken *Tok,
+                                 const std::vector<tok::TokenKind> &Qualifiers,
+                                 bool IsCpp) {
   return Tok && (Tok->isTypeName(IsCpp) || Tok->is(tok::kw_auto) ||
                  isConfiguredQualifier(Tok, Qualifiers));
 }
 
 // If a token is an identifier and it's upper case, it could
 // be a macro and hence we need to be able to ignore it.
-bool LeftRightQualifierAlignmentFixer::isPossibleMacro(const FormatToken *Tok) {
+bool isPossibleMacro(const FormatToken *Tok) {
   if (!Tok)
     return false;
   if (Tok->isNot(tok::identifier))
diff --git a/clang/lib/Format/QualifierAlignmentFixer.h b/clang/lib/Format/QualifierAlignmentFixer.h
index e1cc27e62b13..e31d525da164 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.h
+++ b/clang/lib/Format/QualifierAlignmentFixer.h
@@ -32,6 +32,15 @@ void prepareLeftRightOrderingForQualifierAlignmentFixer(
     std::vector<std::string> &RightOrder,
     std::vector<tok::TokenKind> &Qualifiers);
 
+// Is the Token a simple or qualifier type
+bool isQualifierOrType(const FormatToken *Tok, bool IsCpp = true);
+bool isConfiguredQualifierOrType(const FormatToken *Tok,
+                                 const std::vector<tok::TokenKind> &Qualifiers,
+                                 bool IsCpp = true);
+
+// Is the Token likely a Macro
+bool isPossibleMacro(const FormatToken *Tok);
+
 class LeftRightQualifierAlignmentFixer : public TokenAnalyzer {
   std::string Qualifier;
   bool RightAlign;
@@ -69,16 +78,6 @@ public:
                                  const FormatToken *Tok,
                                  const std::string &Qualifier,
                                  tok::TokenKind QualifierType);
-
-  // Is the Token a simple or qualifier type
-  static bool isQualifierOrType(const FormatToken *Tok, bool IsCpp = true);
-  static bool
-  isConfiguredQualifierOrType(const FormatToken *Tok,
-                              const std::vector<tok::TokenKind> &Qualifiers,
-                              bool IsCpp = true);
-
-  // Is the Token likely a Macro
-  static bool isPossibleMacro(const FormatToken *Tok);
 };
 
 } // end namespace format
diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index 792d8f3c3a98..1e997bb06b86 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -1059,66 +1059,44 @@ TEST_F(QualifierFixerTest, IsQualifierType) {
       "const static inline auto restrict int double long constexpr friend");
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
 
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[0], ConfiguredTokens));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[1], ConfiguredTokens));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[2], ConfiguredTokens));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[3], ConfiguredTokens));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[4], ConfiguredTokens));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[5], ConfiguredTokens));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[6], ConfiguredTokens));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[7], ConfiguredTokens));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[8], ConfiguredTokens));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      Tokens[9], ConfiguredTokens));
-
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[0]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[1]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[2]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[3]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[4]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[5]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[6]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[7]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[8]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isQualifierOrType(Tokens[9]));
+  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[0], ConfiguredTokens));
+  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[1], ConfiguredTokens));
+  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[2], ConfiguredTokens));
+  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[3], ConfiguredTokens));
+  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[4], ConfiguredTokens));
+  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[5], ConfiguredTokens));
+  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[6], ConfiguredTokens));
+  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[7], ConfiguredTokens));
+  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[8], ConfiguredTokens));
+  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[9], ConfiguredTokens));
+
+  EXPECT_TRUE(isQualifierOrType(Tokens[0]));
+  EXPECT_TRUE(isQualifierOrType(Tokens[1]));
+  EXPECT_TRUE(isQualifierOrType(Tokens[2]));
+  EXPECT_TRUE(isQualifierOrType(Tokens[3]));
+  EXPECT_TRUE(isQualifierOrType(Tokens[4]));
+  EXPECT_TRUE(isQualifierOrType(Tokens[5]));
+  EXPECT_TRUE(isQualifierOrType(Tokens[6]));
+  EXPECT_TRUE(isQualifierOrType(Tokens[7]));
+  EXPECT_TRUE(isQualifierOrType(Tokens[8]));
+  EXPECT_TRUE(isQualifierOrType(Tokens[9]));
 
   auto NotTokens = annotate("for while do Foo Bar ");
   ASSERT_EQ(NotTokens.size(), 6u) << Tokens;
 
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[0], ConfiguredTokens));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[1], ConfiguredTokens));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[2], ConfiguredTokens));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[3], ConfiguredTokens));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[4], ConfiguredTokens));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isConfiguredQualifierOrType(
-      NotTokens[5], ConfiguredTokens));
-
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[0]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[1]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[2]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[3]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[4]));
-  EXPECT_FALSE(
-      LeftRightQualifierAlignmentFixer::isQualifierOrType(NotTokens[5]));
+  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[0], ConfiguredTokens));
+  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[1], ConfiguredTokens));
+  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[2], ConfiguredTokens));
+  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[3], ConfiguredTokens));
+  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[4], ConfiguredTokens));
+  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[5], ConfiguredTokens));
+
+  EXPECT_FALSE(isQualifierOrType(NotTokens[0]));
+  EXPECT_FALSE(isQualifierOrType(NotTokens[1]));
+  EXPECT_FALSE(isQualifierOrType(NotTokens[2]));
+  EXPECT_FALSE(isQualifierOrType(NotTokens[3]));
+  EXPECT_FALSE(isQualifierOrType(NotTokens[4]));
+  EXPECT_FALSE(isQualifierOrType(NotTokens[5]));
 }
 
 TEST_F(QualifierFixerTest, IsMacro) {
@@ -1126,10 +1104,10 @@ TEST_F(QualifierFixerTest, IsMacro) {
   auto Tokens = annotate("INT INTPR Foo int");
   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
 
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isPossibleMacro(Tokens[0]));
-  EXPECT_TRUE(LeftRightQualifierAlignmentFixer::isPossibleMacro(Tokens[1]));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isPossibleMacro(Tokens[2]));
-  EXPECT_FALSE(LeftRightQualifierAlignmentFixer::isPossibleMacro(Tokens[3]));
+  EXPECT_TRUE(isPossibleMacro(Tokens[0]));
+  EXPECT_TRUE(isPossibleMacro(Tokens[1]));
+  EXPECT_FALSE(isPossibleMacro(Tokens[2]));
+  EXPECT_FALSE(isPossibleMacro(Tokens[3]));
 }
 
 TEST_F(QualifierFixerTest, OverlappingQualifier) {
-- 
GitLab


From e20800c16f0570562fea31e9a02d65ba56e6858a Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Mon, 13 May 2024 19:45:11 -0700
Subject: [PATCH 0664/1206] [clang-format][NFC] Test IsQualifier only needs to
 call the lexer

---
 clang/unittests/Format/QualifierFixerTest.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index 1e997bb06b86..4ddeef50f5f7 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -1055,7 +1055,9 @@ TEST_F(QualifierFixerTest, IsQualifierType) {
   ConfiguredTokens.push_back(tok::kw_constexpr);
   ConfiguredTokens.push_back(tok::kw_friend);
 
-  auto Tokens = annotate(
+  TestLexer lexer{Allocator, Buffers};
+
+  auto Tokens = lexer.lex(
       "const static inline auto restrict int double long constexpr friend");
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
 
@@ -1081,7 +1083,7 @@ TEST_F(QualifierFixerTest, IsQualifierType) {
   EXPECT_TRUE(isQualifierOrType(Tokens[8]));
   EXPECT_TRUE(isQualifierOrType(Tokens[9]));
 
-  auto NotTokens = annotate("for while do Foo Bar ");
+  auto NotTokens = lexer.lex("for while do Foo Bar ");
   ASSERT_EQ(NotTokens.size(), 6u) << Tokens;
 
   EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[0], ConfiguredTokens));
-- 
GitLab


From 05a97a1a5143d0af60f2dc1e452c3f4ab7409b4c Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Tue, 14 May 2024 12:21:55 +0900
Subject: [PATCH 0665/1206] [MustExec] Drop duplicate RUN line (NFC)

---
 llvm/test/Analysis/MustExecute/const-cond.ll | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/test/Analysis/MustExecute/const-cond.ll b/llvm/test/Analysis/MustExecute/const-cond.ll
index e829db349ca6..97358f670b88 100644
--- a/llvm/test/Analysis/MustExecute/const-cond.ll
+++ b/llvm/test/Analysis/MustExecute/const-cond.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -disable-output -passes=print-mustexecute %s 2>&1 | FileCheck %s
-; RUN: opt -disable-output -passes=print-mustexecute %s 2>&1 | FileCheck %s
 
 ; In general the CFG below is easily simplified but this is useful for
 ; pass ordering issue elimination.
-- 
GitLab


From 3a25e358e2957cce912e701a544fb6163f572575 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Tue, 14 May 2024 12:26:15 +0900
Subject: [PATCH 0666/1206] [LICM] Generate test checks (NFC)

---
 llvm/test/Transforms/LICM/hoist-mustexec.ll | 480 +++++++++++++++++---
 1 file changed, 411 insertions(+), 69 deletions(-)

diff --git a/llvm/test/Transforms/LICM/hoist-mustexec.ll b/llvm/test/Transforms/LICM/hoist-mustexec.ll
index d47209941298..9f3c2f9b4e8c 100644
--- a/llvm/test/Transforms/LICM/hoist-mustexec.ll
+++ b/llvm/test/Transforms/LICM/hoist-mustexec.ll
@@ -1,18 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; REQUIRES: asserts
 ; RUN: opt -S -passes=licm -ipt-expensive-asserts=true < %s | FileCheck %s
-; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop-mssa(licm)' -ipt-expensive-asserts=true -S %s | FileCheck %s
+
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
 
 declare void @f() nounwind
 declare void @llvm.experimental.guard(i1,...)
 
 ; constant fold on first ieration
 define i32 @test1(ptr noalias nocapture readonly %a) nounwind uwtable {
-; CHECK-LABEL: @test1(
+; CHECK-LABEL: define i32 @test1(
+; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[CONTINUE:%.*]] ]
+; CHECK-NEXT:    [[ACC:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[CONTINUE]] ]
+; CHECK-NEXT:    [[R_CHK:%.*]] = icmp ult i32 [[IV]], 2000
+; CHECK-NEXT:    br i1 [[R_CHK]], label [[CONTINUE]], label [[FAIL:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[I1]], [[ACC]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[CONTINUE]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+; CHECK:       fail:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 -1
+;
 entry:
-; CHECK: %i1 = load i32, ptr %a, align 4
-; CHECK-NEXT: br label %for.body
   br label %for.body
 
 for.body:
@@ -37,10 +56,29 @@ fail:
 
 ; Same as test1, but with a floating point IR and fcmp
 define i32 @test_fcmp(ptr noalias nocapture readonly %a) nounwind uwtable {
-; CHECK-LABEL: @test_fcmp(
+; CHECK-LABEL: define i32 @test_fcmp(
+; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[CONTINUE:%.*]] ]
+; CHECK-NEXT:    [[ACC:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[CONTINUE]] ]
+; CHECK-NEXT:    [[R_CHK:%.*]] = fcmp olt float [[IV]], 2.000000e+03
+; CHECK-NEXT:    br i1 [[R_CHK]], label [[CONTINUE]], label [[FAIL:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[I1]], [[ACC]]
+; CHECK-NEXT:    [[INC]] = fadd float [[IV]], 1.000000e+00
+; CHECK-NEXT:    [[EXITCOND:%.*]] = fcmp ogt float [[INC]], 1.000000e+03
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[CONTINUE]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+; CHECK:       fail:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 -1
+;
 entry:
-; CHECK: %i1 = load i32, ptr %a, align 4
-; CHECK-NEXT: br label %for.body
   br label %for.body
 
 for.body:
@@ -67,7 +105,35 @@ fail:
 ; TODO: currently unable to prove the following:
 ; ule i32 (add nsw i32 %len, -1), %len where len is [0, 512]
 define i32 @test2(ptr noalias nocapture readonly %a) nounwind uwtable {
-; CHECK-LABEL: @test2(
+; CHECK-LABEL: define i32 @test2(
+; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[A]], align 4, !range [[RNG0:![0-9]+]]
+; CHECK-NEXT:    [[IS_NON_POS:%.*]] = icmp eq i32 [[LEN]], 0
+; CHECK-NEXT:    br i1 [[IS_NON_POS]], label [[FAIL:%.*]], label [[PREHEADER:%.*]]
+; CHECK:       preheader:
+; CHECK-NEXT:    [[LENMINUSONE:%.*]] = add nsw i32 [[LEN]], -1
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[LENMINUSONE]], [[PREHEADER]] ], [ [[DEC:%.*]], [[CONTINUE:%.*]] ]
+; CHECK-NEXT:    [[ACC:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[ADD:%.*]], [[CONTINUE]] ]
+; CHECK-NEXT:    [[R_CHK:%.*]] = icmp ule i32 [[IV]], [[LEN]]
+; CHECK-NEXT:    br i1 [[R_CHK]], label [[CONTINUE]], label [[FAIL_LOOPEXIT:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[I1]], [[ACC]]
+; CHECK-NEXT:    [[DEC]] = add nsw i32 [[IV]], -1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[DEC]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[CONTINUE]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+; CHECK:       fail.loopexit:
+; CHECK-NEXT:    br label [[FAIL]]
+; CHECK:       fail:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 -1
+;
 entry:
   %len = load i32, ptr %a, align 4, !range !{i32 0, i32 512}
   %is.non.pos = icmp eq i32 %len, 0
@@ -81,8 +147,6 @@ for.body:
   %r.chk = icmp ule i32 %iv, %len
   br i1 %r.chk, label %continue, label %fail
 continue:
-; CHECK-LABEL: continue
-; CHECK: %i1 = load i32, ptr %a, align 4
   %i1 = load i32, ptr %a, align 4
   %add = add nsw i32 %i1, %acc
   %dec = add nsw i32 %iv, -1
@@ -99,14 +163,39 @@ fail:
 
 ; trivially true for zero
 define i32 @test3(ptr noalias nocapture readonly %a) nounwind uwtable {
-; CHECK-LABEL: @test3(
+; CHECK-LABEL: define i32 @test3(
+; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[A]], align 4, !range [[RNG0]]
+; CHECK-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0
+; CHECK-NEXT:    br i1 [[IS_ZERO]], label [[FAIL:%.*]], label [[PREHEADER:%.*]]
+; CHECK:       preheader:
+; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[INC:%.*]], [[CONTINUE:%.*]] ]
+; CHECK-NEXT:    [[ACC:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[ADD:%.*]], [[CONTINUE]] ]
+; CHECK-NEXT:    [[R_CHK:%.*]] = icmp ule i32 [[IV]], [[LEN]]
+; CHECK-NEXT:    br i1 [[R_CHK]], label [[CONTINUE]], label [[FAIL_LOOPEXIT:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[I1]], [[ACC]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[CONTINUE]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+; CHECK:       fail.loopexit:
+; CHECK-NEXT:    br label [[FAIL]]
+; CHECK:       fail:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 -1
+;
 entry:
   %len = load i32, ptr %a, align 4, !range !{i32 0, i32 512}
   %is.zero = icmp eq i32 %len, 0
   br i1 %is.zero, label %fail, label %preheader
 preheader:
-; CHECK: %i1 = load i32, ptr %a, align 4
-; CHECK-NEXT: br label %for.body
   br label %for.body
 for.body:
   %iv = phi i32 [ 0, %preheader ], [ %inc, %continue ]
@@ -130,9 +219,10 @@ fail:
 
 ; requires fact length is non-zero
 define i32 @test4(ptr noalias nocapture readonly %a) nounwind uwtable {
-; CHECK-LABEL: @test4(
+; CHECK-LABEL: define i32 @test4(
+; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[A:%.*]], align 4, !range !0
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[A]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0
 ; CHECK-NEXT:    br i1 [[IS_ZERO]], label [[FAIL:%.*]], label [[PREHEADER:%.*]]
 ; CHECK:       preheader:
@@ -185,10 +275,29 @@ fail:
 
 ; variation on test1 with branch swapped
 define i32 @test-brswap(ptr noalias nocapture readonly %a) nounwind uwtable {
-; CHECK-LABEL: @test-brswap(
+; CHECK-LABEL: define i32 @test-brswap(
+; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[CONTINUE:%.*]] ]
+; CHECK-NEXT:    [[ACC:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[CONTINUE]] ]
+; CHECK-NEXT:    [[R_CHK:%.*]] = icmp ugt i32 [[IV]], 2000
+; CHECK-NEXT:    br i1 [[R_CHK]], label [[FAIL:%.*]], label [[CONTINUE]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[I1]], [[ACC]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[CONTINUE]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+; CHECK:       fail:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 -1
+;
 entry:
-; CHECK: %i1 = load i32, ptr %a, align 4
-; CHECK-NEXT: br label %for.body
   br label %for.body
 
 for.body:
@@ -212,13 +321,33 @@ fail:
 }
 
 define i32 @test-nonphi(ptr noalias nocapture readonly %a) nounwind uwtable {
-; CHECK-LABEL: @test-nonphi(
+; CHECK-LABEL: define i32 @test-nonphi(
+; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[CONTINUE:%.*]] ]
+; CHECK-NEXT:    [[ACC:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[CONTINUE]] ]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[IV]], 72
+; CHECK-NEXT:    [[R_CHK:%.*]] = icmp ugt i32 [[XOR]], 2000
+; CHECK-NEXT:    br i1 [[R_CHK]], label [[FAIL:%.*]], label [[CONTINUE]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[I1]], [[ACC]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[CONTINUE]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+; CHECK:       fail:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 -1
+;
 entry:
   br label %for.body
 
 for.body:
-; CHECK-LABEL: continue
-; CHECK: %i1 = load i32, ptr %a, align 4
   %iv = phi i32 [ 0, %entry ], [ %inc, %continue ]
   %acc = phi i32 [ 0, %entry ], [ %add, %continue ]
   %xor = xor i32 %iv, 72
@@ -240,7 +369,34 @@ fail:
 }
 
 define i32 @test-wrongphi(ptr noalias nocapture readonly %a) nounwind uwtable {
-; CHECK-LABEL: @test-wrongphi(
+; CHECK-LABEL: define i32 @test-wrongphi(
+; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[CONTINUE:%.*]] ]
+; CHECK-NEXT:    [[ACC:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[CONTINUE]] ]
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[IV]], 500
+; CHECK-NEXT:    br i1 [[COND]], label [[DUMMY_BLOCK1:%.*]], label [[DUMMY_BLOCK2:%.*]]
+; CHECK:       dummy_block1:
+; CHECK-NEXT:    br label [[DUMMY_BLOCK2]]
+; CHECK:       dummy_block2:
+; CHECK-NEXT:    [[WRONGPHI:%.*]] = phi i32 [ 11, [[FOR_BODY]] ], [ 12, [[DUMMY_BLOCK1]] ]
+; CHECK-NEXT:    [[R_CHK:%.*]] = icmp ugt i32 [[WRONGPHI]], 2000
+; CHECK-NEXT:    br i1 [[R_CHK]], label [[FAIL:%.*]], label [[CONTINUE]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[I1]], [[ACC]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[CONTINUE]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+; CHECK:       fail:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 -1
+;
 entry:
   br label %for.body
 
@@ -258,8 +414,6 @@ dummy_block2:
   %r.chk = icmp ugt i32 %wrongphi, 2000
   br i1 %r.chk, label %fail, label %continue
 continue:
-; CHECK-LABEL: continue
-; CHECK: %i1 = load i32, ptr %a, align 4
   %i1 = load i32, ptr %a, align 4
   %add = add nsw i32 %i1, %acc
   %inc = add nuw nsw i32 %iv, 1
@@ -276,10 +430,34 @@ fail:
 
 ; This works because loop-simplify is run implicitly, but test for it anyways
 define i32 @test-multiple-latch(ptr noalias nocapture readonly %a) nounwind uwtable {
-; CHECK-LABEL: @test-multiple-latch(
+; CHECK-LABEL: define i32 @test-multiple-latch(
+; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[ACC:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY_BACKEDGE]] ]
+; CHECK-NEXT:    [[R_CHK:%.*]] = icmp ult i32 [[IV]], 2000
+; CHECK-NEXT:    br i1 [[R_CHK]], label [[CONTINUE1:%.*]], label [[FAIL:%.*]]
+; CHECK:       continue1:
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[I1]], [[ACC]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[ADD]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[CONTINUE2:%.*]], label [[FOR_BODY_BACKEDGE]]
+; CHECK:       for.body.backedge:
+; CHECK-NEXT:    br label [[FOR_BODY]]
+; CHECK:       continue2:
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_BACKEDGE]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[CONTINUE2]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+; CHECK:       fail:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 -1
+;
 entry:
-; CHECK: %i1 = load i32, ptr %a, align 4
-; CHECK-NEXT: br label %for.body
   br label %for.body
 
 for.body:
@@ -306,12 +484,21 @@ fail:
 }
 
 define void @test-hoisting-in-presence-of-guards(i1 %c, ptr %p) {
-
-; CHECK-LABEL: @test-hoisting-in-presence-of-guards
-; CHECK:       entry:
-; CHECK:         %a = load i32, ptr %p
-; CHECK:         %invariant_cond = icmp ne i32 %a, 100
+; CHECK-LABEL: define void @test-hoisting-in-presence-of-guards(
+; CHECK-SAME: i1 [[C:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[INVARIANT_COND:%.*]] = icmp ne i32 [[A]], 100
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[INVARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %loop
@@ -335,11 +522,30 @@ declare void @may_throw() inaccessiblememonly
 ; Test that we can sink a mustexecute load from loop header even in presence of
 ; throwing instructions after it.
 define void @test_hoist_from_header_01(ptr %p, i32 %n) {
-
-; CHECK-LABEL: @test_hoist_from_header_01(
-; CHECK:       entry:
-; CHECK-NEXT:  %load = load i32, ptr %p
-; CHECK-NOT:   load i32
+; CHECK-LABEL: define void @test_hoist_from_header_01(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    call void @may_throw()
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[A]], [[IF_TRUE]] ], [ [[B]], [[IF_FALSE]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[MERGE]]
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], [[LOAD]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %loop
@@ -371,11 +577,30 @@ exit:
 }
 
 define void @test_hoist_from_header_02(ptr %p, i32 %n) {
-
-; CHECK-LABEL: @test_hoist_from_header_02(
-; CHECK:       entry:
-; CHECK-NEXT:  %load = load i32, ptr %p
-; CHECK-NOT:   load i32
+; CHECK-LABEL: define void @test_hoist_from_header_02(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @may_throw()
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[A]], [[IF_TRUE]] ], [ [[B]], [[IF_FALSE]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[MERGE]]
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], [[LOAD]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %loop
@@ -407,11 +632,30 @@ exit:
 }
 
 define void @test_hoist_from_header_03(ptr %p, i32 %n) {
-
-; CHECK-LABEL: @test_hoist_from_header_03(
-; CHECK:       entry:
-; CHECK-NEXT:  %load = load i32, ptr %p
-; CHECK-NOT:   load i32
+; CHECK-LABEL: define void @test_hoist_from_header_03(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[A]], [[IF_TRUE]] ], [ [[B]], [[IF_FALSE]] ]
+; CHECK-NEXT:    call void @may_throw()
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[MERGE]]
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], [[LOAD]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %loop
@@ -444,11 +688,30 @@ exit:
 
 ; Check that a throwing instruction prohibits hoisting across it.
 define void @test_hoist_from_header_04(ptr %p, i32 %n) {
-
-; CHECK-LABEL: @test_hoist_from_header_04(
-; CHECK:       entry:
+; CHECK-LABEL: define void @test_hoist_from_header_04(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK:       %load = load i32, ptr %p
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    call void @may_throw()
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[A]], [[IF_TRUE]] ], [ [[B]], [[IF_FALSE]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[MERGE]]
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], [[LOAD]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %loop
@@ -482,11 +745,30 @@ exit:
 ; Check that we can hoist a mustexecute load from backedge even if something
 ; throws after it.
 define void @test_hoist_from_backedge_01(ptr %p, i32 %n) {
-
-; CHECK-LABEL: @test_hoist_from_backedge_01(
-; CHECK:       entry:
-; CHECK-NEXT:  %load = load i32, ptr %p
-; CHECK-NOT:   load i32
+; CHECK-LABEL: define void @test_hoist_from_backedge_01(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[A]], [[IF_TRUE]] ], [ [[B]], [[IF_FALSE]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[MERGE]]
+; CHECK-NEXT:    call void @may_throw()
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], [[LOAD]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %loop
@@ -519,11 +801,30 @@ exit:
 
 ; Check that we don't hoist the load if something before it can throw.
 define void @test_hoist_from_backedge_02(ptr %p, i32 %n) {
-
-; CHECK-LABEL: @test_hoist_from_backedge_02(
-; CHECK:       entry:
+; CHECK-LABEL: define void @test_hoist_from_backedge_02(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK:       %load = load i32, ptr %p
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[A]], [[IF_TRUE]] ], [ [[B]], [[IF_FALSE]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[MERGE]]
+; CHECK-NEXT:    call void @may_throw()
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], [[LOAD]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %loop
@@ -555,11 +856,30 @@ exit:
 }
 
 define void @test_hoist_from_backedge_03(ptr %p, i32 %n) {
-
-; CHECK-LABEL: @test_hoist_from_backedge_03(
-; CHECK:       entry:
+; CHECK-LABEL: define void @test_hoist_from_backedge_03(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK:       %load = load i32, ptr %p
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[IV]], [[IV]]
+; CHECK-NEXT:    call void @may_throw()
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[A]], [[IF_TRUE]] ], [ [[B]], [[IF_FALSE]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[MERGE]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], [[LOAD]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %loop
@@ -591,11 +911,30 @@ exit:
 }
 
 define void @test_hoist_from_backedge_04(ptr %p, i32 %n) {
-
-; CHECK-LABEL: @test_hoist_from_backedge_04(
-; CHECK:       entry:
+; CHECK-LABEL: define void @test_hoist_from_backedge_04(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK:       %load = load i32, ptr %p
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    call void @may_throw()
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[IV]], [[IV]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[A]], [[IF_TRUE]] ], [ [[B]], [[IF_FALSE]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[MERGE]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], [[LOAD]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %loop
@@ -625,3 +964,6 @@ backedge:
 exit:
   ret void
 }
+;.
+; CHECK: [[RNG0]] = !{i32 0, i32 512}
+;.
-- 
GitLab


From b7adba8e78662b901099dac5e06fd9f4beda22b4 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Tue, 14 May 2024 12:28:01 +0900
Subject: [PATCH 0667/1206] [LICM] Add must exec hoisting test with commuted
 operands (NFC)

---
 llvm/test/Transforms/LICM/hoist-mustexec.ll | 57 +++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/llvm/test/Transforms/LICM/hoist-mustexec.ll b/llvm/test/Transforms/LICM/hoist-mustexec.ll
index 9f3c2f9b4e8c..81e0815053ff 100644
--- a/llvm/test/Transforms/LICM/hoist-mustexec.ll
+++ b/llvm/test/Transforms/LICM/hoist-mustexec.ll
@@ -217,6 +217,63 @@ fail:
   ret i32 -1
 }
 
+; Same as previous case, with commuted icmp.
+; FIXME: The load should get hoisted here as well.
+define i32 @test3_commuted(ptr noalias nocapture readonly %a) nounwind uwtable {
+; CHECK-LABEL: define i32 @test3_commuted(
+; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[A]], align 4, !range [[RNG0]]
+; CHECK-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0
+; CHECK-NEXT:    br i1 [[IS_ZERO]], label [[FAIL:%.*]], label [[PREHEADER:%.*]]
+; CHECK:       preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[INC:%.*]], [[CONTINUE:%.*]] ]
+; CHECK-NEXT:    [[ACC:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[ADD:%.*]], [[CONTINUE]] ]
+; CHECK-NEXT:    [[R_CHK:%.*]] = icmp uge i32 [[LEN]], [[IV]]
+; CHECK-NEXT:    br i1 [[R_CHK]], label [[CONTINUE]], label [[FAIL_LOOPEXIT:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[I1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[I1]], [[ACC]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[CONTINUE]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+; CHECK:       fail.loopexit:
+; CHECK-NEXT:    br label [[FAIL]]
+; CHECK:       fail:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    ret i32 -1
+;
+entry:
+  %len = load i32, ptr %a, align 4, !range !{i32 0, i32 512}
+  %is.zero = icmp eq i32 %len, 0
+  br i1 %is.zero, label %fail, label %preheader
+preheader:
+  br label %for.body
+for.body:
+  %iv = phi i32 [ 0, %preheader ], [ %inc, %continue ]
+  %acc = phi i32 [ 0, %preheader ], [ %add, %continue ]
+  %r.chk = icmp uge i32 %len, %iv
+  br i1 %r.chk, label %continue, label %fail
+continue:
+  %i1 = load i32, ptr %a, align 4
+  %add = add nsw i32 %i1, %acc
+  %inc = add nuw nsw i32 %iv, 1
+  %exitcond = icmp eq i32 %inc, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+
+fail:
+  call void @f()
+  ret i32 -1
+}
+
 ; requires fact length is non-zero
 define i32 @test4(ptr noalias nocapture readonly %a) nounwind uwtable {
 ; CHECK-LABEL: define i32 @test4(
-- 
GitLab


From 7198c3d613f1087c78124928cbe2cbc4e03a0e5a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 13 May 2024 20:33:09 -0700
Subject: [PATCH 0668/1206] [RISCV] Reduce the amount of similar code in
 RISCVInstPrinter::printRlist. NFC (#92053)

Remove the switch statement and instead do range checks to know which
pieces we need to print.
---
 .../RISCV/MCTargetDesc/RISCVInstPrinter.cpp   | 70 +++++++------------
 1 file changed, 26 insertions(+), 44 deletions(-)

diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
index 663d4bad767d..48b669c78cad 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
@@ -216,62 +216,44 @@ void RISCVInstPrinter::printVTypeI(const MCInst *MI, unsigned OpNo,
   RISCVVType::printVType(Imm, O);
 }
 
+// Print a Zcmp RList. If we are printing architectural register names rather
+// than ABI register names, we need to print "{x1, x8-x9, x18-x27}" for all
+// registers. Otherwise, we print "{ra, s0-s11}".
 void RISCVInstPrinter::printRlist(const MCInst *MI, unsigned OpNo,
                                   const MCSubtargetInfo &STI, raw_ostream &O) {
   unsigned Imm = MI->getOperand(OpNo).getImm();
   O << "{";
-  switch (Imm) {
-  case RISCVZC::RLISTENCODE::RA:
-    printRegName(O, RISCV::X1);
-    break;
-  case RISCVZC::RLISTENCODE::RA_S0:
-    printRegName(O, RISCV::X1);
-    O << ", ";
-    printRegName(O, RISCV::X8);
-    break;
-  case RISCVZC::RLISTENCODE::RA_S0_S1:
-    printRegName(O, RISCV::X1);
-    O << ", ";
-    printRegName(O, RISCV::X8);
-    O << '-';
-    printRegName(O, RISCV::X9);
-    break;
-  case RISCVZC::RLISTENCODE::RA_S0_S2:
-    printRegName(O, RISCV::X1);
-    O << ", ";
-    printRegName(O, RISCV::X8);
-    O << '-';
-    if (ArchRegNames) {
-      printRegName(O, RISCV::X9);
-      O << ", ";
-    }
-    printRegName(O, RISCV::X18);
-    break;
-  case RISCVZC::RLISTENCODE::RA_S0_S3:
-  case RISCVZC::RLISTENCODE::RA_S0_S4:
-  case RISCVZC::RLISTENCODE::RA_S0_S5:
-  case RISCVZC::RLISTENCODE::RA_S0_S6:
-  case RISCVZC::RLISTENCODE::RA_S0_S7:
-  case RISCVZC::RLISTENCODE::RA_S0_S8:
-  case RISCVZC::RLISTENCODE::RA_S0_S9:
-  case RISCVZC::RLISTENCODE::RA_S0_S11:
-    printRegName(O, RISCV::X1);
+  printRegName(O, RISCV::X1);
+
+  if (Imm >= RISCVZC::RLISTENCODE::RA_S0) {
     O << ", ";
     printRegName(O, RISCV::X8);
+  }
+
+  if (Imm >= RISCVZC::RLISTENCODE::RA_S0_S1) {
     O << '-';
-    if (ArchRegNames) {
+    if (Imm == RISCVZC::RLISTENCODE::RA_S0_S1 || ArchRegNames)
       printRegName(O, RISCV::X9);
+  }
+
+  if (Imm >= RISCVZC::RLISTENCODE::RA_S0_S2) {
+    if (ArchRegNames)
       O << ", ";
+    if (Imm == RISCVZC::RLISTENCODE::RA_S0_S2 || ArchRegNames)
       printRegName(O, RISCV::X18);
+  }
+
+  if (Imm >= RISCVZC::RLISTENCODE::RA_S0_S3) {
+    if (ArchRegNames)
       O << '-';
-    }
-    printRegName(O, RISCV::X19 + (Imm == RISCVZC::RLISTENCODE::RA_S0_S11
-                                      ? 8
-                                      : Imm - RISCVZC::RLISTENCODE::RA_S0_S3));
-    break;
-  default:
-    llvm_unreachable("invalid register list");
+    unsigned Offset = (Imm - RISCVZC::RLISTENCODE::RA_S0_S3);
+    // Encodings for S3-S9 are contiguous. There is no encoding for S10, so we
+    // must skip to S11(X27).
+    if (Imm == RISCVZC::RLISTENCODE::RA_S0_S11)
+      ++Offset;
+    printRegName(O, RISCV::X19 + Offset);
   }
+
   O << "}";
 }
 
-- 
GitLab


From f608ac261781b7707b2721563765e07c57366619 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 14 May 2024 03:54:24 +0000
Subject: [PATCH 0669/1206] [bazel] Actually port libc #91905

---
 utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 9cdcc7577b46..446499cf15d7 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -129,6 +129,11 @@ libc_support_library(
 
 ############################ Type Proxy Header Files ###########################
 
+libc_support_library(
+    name = "types_clockid_t",
+    hdrs = ["hdr/types/clockid_t.h"],
+)
+
 libc_support_library(
     name = "types_fenv_t",
     hdrs = ["hdr/types/fenv_t.h"],
@@ -1154,6 +1159,7 @@ libc_support_library(
     name = "__support_time",
     hdrs = glob(["src/__support/time/*.h"]),
     deps = [
+        ":__support_common",
         ":hdr_time_macros",
         ":types_time_t",
     ],
@@ -1167,8 +1173,14 @@ libc_support_library(
         "//conditions:default": ["@platforms//:incompatible"],
     }),
     deps = [
+        ":__support_common",
+        ":__support_cpp_expected",
+        ":__support_error_or",
+        ":__support_libc_assert",
         ":__support_time",
         ":hdr_time_macros",
+        ":types_clockid_t",
+        ":types_struct_timespec",
     ],
 )
 
-- 
GitLab


From cf8b93d8234c13b6a6baf7f8a1e5bdb8c1c8cca1 Mon Sep 17 00:00:00 2001
From: Ryosuke Niwa <rniwa@webkit.org>
Date: Mon, 13 May 2024 21:05:41 -0700
Subject: [PATCH 0670/1206] [alpha.webkit.UncountedCallArgsChecker] Allow
 explicit instantiation of Ref/RefPtr on call arguments. (#91875)

Co-authored-by: Ryosuke Niwa <rniwa@apple.com>
---
 clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp | 7 +++++++
 clang/test/Analysis/Checkers/WebKit/call-args.cpp     | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
index f81db0e67d83..be07cf51eefb 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
@@ -28,6 +28,13 @@ bool tryToFindPtrOrigin(
       E = tempExpr->getSubExpr();
       continue;
     }
+    if (auto *tempExpr = dyn_cast<CXXTemporaryObjectExpr>(E)) {
+      if (auto *C = tempExpr->getConstructor()) {
+        if (auto *Class = C->getParent(); Class && isRefCounted(Class))
+          return callback(E, true);
+        break;
+      }
+    }
     if (auto *tempExpr = dyn_cast<ParenExpr>(E)) {
       E = tempExpr->getSubExpr();
       continue;
diff --git a/clang/test/Analysis/Checkers/WebKit/call-args.cpp b/clang/test/Analysis/Checkers/WebKit/call-args.cpp
index e1bee8a23a25..94efddeaf66c 100644
--- a/clang/test/Analysis/Checkers/WebKit/call-args.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/call-args.cpp
@@ -358,3 +358,10 @@ namespace call_with_ptr_on_ref {
     // expected-warning@-1{{Call argument for parameter 'bad' is uncounted and unsafe}}
   }
 }
+
+namespace call_with_explicit_temporary_obj {
+  void foo() {
+    Ref { *provide() }->method();
+    RefPtr { provide() }->method();
+  }
+}
-- 
GitLab


From cff9e77783aceb52da705b7e2b2e45bfbc86c628 Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Tue, 14 May 2024 06:15:01 +0200
Subject: [PATCH 0671/1206] [Clang][NFC] Mark P2552 as implemented. (#92007)

wg21.link/P2552 suggest that __has_cpp_attribute
should return a non-zero value for all attributes that the
implementation does something interesting with.

Clang does something meaninful with all attributes except for:

- no_unique_address which we do not support for msvc target
- carries_dependency which arguably does nothing interesting. P2552
shies away from specifying a behavior for that attribute (despite being
the only one for which a recommandation would have been interesting,
arguably)

As such, we have nothing to change for this paper. This paper is a DR
and clang always behaved reasonably.
---
 clang/test/SemaCXX/cxx2c-attributes.cpp | 20 ++++++++++++++++++++
 clang/www/cxx_status.html               |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/SemaCXX/cxx2c-attributes.cpp

diff --git a/clang/test/SemaCXX/cxx2c-attributes.cpp b/clang/test/SemaCXX/cxx2c-attributes.cpp
new file mode 100644
index 000000000000..c20a1b606a0f
--- /dev/null
+++ b/clang/test/SemaCXX/cxx2c-attributes.cpp
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple x86_64-pc-linux -fsyntax-only -verify -Wno-c++17-extensions
+// RUN: %clang_cc1 %s -x c++ -std=c++11 -triple x86_64-windows-msvc -fsyntax-only -verify=msvc -Wno-c++17-extensions
+// expected-no-diagnostics
+
+// Check we return non-zero values for supported attributes as per
+// wg21.link/P2552
+static_assert(__has_cpp_attribute(assume));
+
+// The standard does not prescribe a behavior for [[carries_dependency]]
+
+static_assert(__has_cpp_attribute(deprecated));
+static_assert(__has_cpp_attribute(fallthrough));
+static_assert(__has_cpp_attribute(likely));
+static_assert(__has_cpp_attribute(unlikely));
+static_assert(__has_cpp_attribute(maybe_unused));
+static_assert(__has_cpp_attribute(nodiscard));
+static_assert(__has_cpp_attribute(noreturn));
+
+// We do not support [[no_unique_address]] in MSVC emulation mode
+static_assert(__has_cpp_attribute(no_unique_address)); // msvc-error {{static assertion failed}}
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index 1338f544ffcb..06777eaa6df6 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -130,7 +130,7 @@ C++23, informally referred to as C++26.</p>
  <tr>
   <td>On the ignorability of standard attributes</td>
   <td><a href="https://wg21.link/P2552R3">P2552R3</a> (<a href="#dr">DR</a>)</td>
-  <td class="none" align="center">No</td>
+  <td class="full" align="center">Yes</td>
  </tr>
  <tr>
   <td>Static storage for braced initializers</td>
-- 
GitLab


From e5a277b16755ad273d6c1caa5dd29b4a3ae29078 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Tue, 14 May 2024 05:34:39 +0100
Subject: [PATCH 0672/1206] [TableGen][RISCV] Add initial support for marking
 profiles as experimental (#91993)

This is just the TableGen-side changes, split out as the minimal
testable unit. It doesn't yet transition RVA23 and friends to be
experimental (and add the necessary other changes for this to work).

Although choosing not to emit the SupportedExperimentalProfiles array if
no experimental profiles are present isn't consistent with what we do
for experimental extensions, we need to do this in order to avoid adding
a warning for the empty array when building LLVM for as long as we don't
have any experimental profiles defined.
---
 llvm/lib/Target/RISCV/RISCVProfiles.td        |  6 +++-
 llvm/test/TableGen/riscv-target-def.td        | 13 ++++++-
 llvm/utils/TableGen/RISCVTargetDefEmitter.cpp | 35 ++++++++++++++-----
 3 files changed, 43 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVProfiles.td b/llvm/lib/Target/RISCV/RISCVProfiles.td
index 5c13710faf65..e56df33bd8cb 100644
--- a/llvm/lib/Target/RISCV/RISCVProfiles.td
+++ b/llvm/lib/Target/RISCV/RISCVProfiles.td
@@ -8,7 +8,11 @@
 
 class RISCVProfile<string name, list<SubtargetFeature> features>
     : SubtargetFeature<name, "Is" # NAME, "true",
-                       "RISC-V " # name # " profile", features>;
+                       "RISC-V " # name # " profile", features> {
+  // Indicates if the profile is not yet ratified, so should be treated as
+  // experimental.
+  bit Experimental = false;
+}
 
 defvar RVI20U32Features = [Feature32Bit, FeatureStdExtI];
 defvar RVI20U64Features = [Feature64Bit, FeatureStdExtI];
diff --git a/llvm/test/TableGen/riscv-target-def.td b/llvm/test/TableGen/riscv-target-def.td
index 7f3d9bdb278c..e328a6e3b583 100644
--- a/llvm/test/TableGen/riscv-target-def.td
+++ b/llvm/test/TableGen/riscv-target-def.td
@@ -53,12 +53,19 @@ def FeatureDummy
 
 class RISCVProfile<string name, list<SubtargetFeature> features>
     : SubtargetFeature<name, "Is" # NAME, "true",
-                       "RISC-V " # name # " profile", features>;
+                       "RISC-V " # name # " profile", features> {
+  bit Experimental = false;
+}
+class RISCVExperimentalProfile<string name, list<SubtargetFeature> features>
+    : RISCVProfile<"experimental-"#name, features> {
+  let Experimental = true;
+}
 
 def RVI20U32 : RISCVProfile<"rvi20u32", [Feature32Bit, FeatureStdExtI]>;
 def RVI20U64 : RISCVProfile<"rvi20u64", [Feature64Bit, FeatureStdExtI]>;
 def ProfileDummy : RISCVProfile<"dummy", [Feature64Bit, FeatureStdExtI,
                                           FeatureStdExtF, FeatureStdExtZidummy]>;
+def RVI99U64 : RISCVExperimentalProfile<"rvi99u64", [Feature64Bit, FeatureStdExtI]>;
 
 class RISCVProcessorModel<string n,
                           SchedMachineModel m,
@@ -139,6 +146,10 @@ def ROCKET : RISCVTuneProcessorModel<"rocket",
 // CHECK-NEXT:     {"rvi20u64","rv64i2p1"},
 // CHECK-NEXT: };
 
+// CHECK:      static constexpr RISCVProfile SupportedExperimentalProfiles[] = {
+// CHECK-NEXT:     {"experimental-rvi99u64","rv64i2p1"},
+// CHECK-NEXT: };
+
 // CHECK:      #endif // GET_SUPPORTED_PROFILES
 
 // CHECK:      #ifndef PROC
diff --git a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
index bb409ea6ea69..d174b1961344 100644
--- a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
@@ -122,6 +122,26 @@ static void printMArch(raw_ostream &OS, const std::vector<Record *> &Features) {
     OS << LS << Ext.first << Ext.second.Major << 'p' << Ext.second.Minor;
 }
 
+static void printProfileTable(raw_ostream &OS,
+                              const std::vector<Record *> &Profiles,
+                              bool Experimental) {
+  OS << "static constexpr RISCVProfile Supported";
+  if (Experimental)
+    OS << "Experimental";
+  OS << "Profiles[] = {\n";
+
+  for (const Record *Rec : Profiles) {
+    if (Rec->getValueAsBit("Experimental") != Experimental)
+      continue;
+
+    OS.indent(4) << "{\"" << Rec->getValueAsString("Name") << "\",\"";
+    printMArch(OS, Rec->getValueAsListOfDefs("Implies"));
+    OS << "\"},\n";
+  }
+
+  OS << "};\n\n";
+}
+
 static void emitRISCVProfiles(RecordKeeper &Records, raw_ostream &OS) {
   OS << "#ifdef GET_SUPPORTED_PROFILES\n";
   OS << "#undef GET_SUPPORTED_PROFILES\n\n";
@@ -129,15 +149,12 @@ static void emitRISCVProfiles(RecordKeeper &Records, raw_ostream &OS) {
   auto Profiles = Records.getAllDerivedDefinitionsIfDefined("RISCVProfile");
 
   if (!Profiles.empty()) {
-    llvm::sort(Profiles, LessRecordFieldName());
-    OS << "static constexpr RISCVProfile SupportedProfiles[] = {\n";
-    for (const Record *Rec : Profiles) {
-      OS.indent(4) << "{\"" << Rec->getValueAsString("Name") << "\",\"";
-      printMArch(OS, Rec->getValueAsListOfDefs("Implies"));
-      OS << "\"},\n";
-    }
-
-    OS << "};\n\n";
+    printProfileTable(OS, Profiles, /*Experimental=*/false);
+    bool HasExperimentalProfiles = any_of(Profiles, [&](auto &Rec) {
+      return Rec->getValueAsBit("Experimental");
+    });
+    if (HasExperimentalProfiles)
+      printProfileTable(OS, Profiles, /*Experimental=*/true);
   }
 
   OS << "#endif // GET_SUPPORTED_PROFILES\n\n";
-- 
GitLab


From f0a681640e012356974024dd2971d74fc18f5b48 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 13 May 2024 21:34:06 -0700
Subject: [PATCH 0673/1206] [RISCV] Remove AllPopRegs array from
 RISCVFrameLowering.cpp. NFC

The same registers are listed in the same order in FixedCSRFIMap.
---
 llvm/lib/Target/RISCV/RISCVFrameLowering.cpp | 35 +++++++++-----------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index cb41577c5d94..316f6a90893a 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -42,10 +42,16 @@ RISCVFrameLowering::RISCVFrameLowering(const RISCVSubtarget &STI)
           /*TransientStackAlignment=*/getABIStackAlignment(STI.getTargetABI())),
       STI(STI) {}
 
-static const MCPhysReg AllPopRegs[] = {
-    RISCV::X1,  RISCV::X8,  RISCV::X9,  RISCV::X18, RISCV::X19,
-    RISCV::X20, RISCV::X21, RISCV::X22, RISCV::X23, RISCV::X24,
-    RISCV::X25, RISCV::X26, RISCV::X27};
+// Offsets which need to be scale by XLen representing locations of CSRs which
+// are given a fixed location by save/restore libcalls or Zcmp Push/Pop.
+static const std::pair<MCPhysReg, int8_t> FixedCSRFIMap[] = {
+    {/*ra*/ RISCV::X1, -1},   {/*s0*/ RISCV::X8, -2},
+    {/*s1*/ RISCV::X9, -3},   {/*s2*/ RISCV::X18, -4},
+    {/*s3*/ RISCV::X19, -5},  {/*s4*/ RISCV::X20, -6},
+    {/*s5*/ RISCV::X21, -7},  {/*s6*/ RISCV::X22, -8},
+    {/*s7*/ RISCV::X23, -9},  {/*s8*/ RISCV::X24, -10},
+    {/*s9*/ RISCV::X25, -11}, {/*s10*/ RISCV::X26, -12},
+    {/*s11*/ RISCV::X27, -13}};
 
 // For now we use x3, a.k.a gp, as pointer to shadow call stack.
 // User should not use x3 in their asm.
@@ -170,7 +176,7 @@ static int getLibCallID(const MachineFunction &MF,
 
   Register MaxReg = RISCV::NoRegister;
   for (auto &CS : CSI)
-    // RISCVRegisterInfo::hasReservedSpillSlot assigns negative frame indexes to
+    // assignCalleeSavedSpillSlots assigns negative frame indexes to
     // registers which can be saved by libcall.
     if (CS.getFrameIdx() < 0)
       MaxReg = std::max(MaxReg.id(), CS.getReg().id());
@@ -291,7 +297,9 @@ static Register getMaxPushPopReg(const MachineFunction &MF,
                                  const std::vector<CalleeSavedInfo> &CSI) {
   Register MaxPushPopReg = RISCV::NoRegister;
   for (auto &CS : CSI) {
-    if (llvm::is_contained(AllPopRegs, CS.getReg().id()))
+    if (llvm::find_if(FixedCSRFIMap, [&](auto P) {
+          return P.first == CS.getReg();
+        }) != std::end(FixedCSRFIMap))
       MaxPushPopReg = std::max(MaxPushPopReg.id(), CS.getReg().id());
   }
   // if rlist is {rs, s0-s10}, then s11 will also be included
@@ -1385,17 +1393,6 @@ RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const {
   return 0;
 }
 
-// Offsets which need to be scale by XLen representing locations of CSRs which
-// are given a fixed location by save/restore libcalls or Zcmp Push/Pop.
-static const std::pair<MCPhysReg, int8_t> FixedCSRFIMap[] = {
-    {/*ra*/ RISCV::X1, -1},   {/*s0*/ RISCV::X8, -2},
-    {/*s1*/ RISCV::X9, -3},   {/*s2*/ RISCV::X18, -4},
-    {/*s3*/ RISCV::X19, -5},  {/*s4*/ RISCV::X20, -6},
-    {/*s5*/ RISCV::X21, -7},  {/*s6*/ RISCV::X22, -8},
-    {/*s7*/ RISCV::X23, -9},  {/*s8*/ RISCV::X24, -10},
-    {/*s9*/ RISCV::X25, -11}, {/*s10*/ RISCV::X26, -12},
-    {/*s11*/ RISCV::X27, -13}};
-
 bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
     MachineFunction &MF, const TargetRegisterInfo *TRI,
     std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
@@ -1498,7 +1495,7 @@ bool RISCVFrameLowering::spillCalleeSavedRegisters(
       PushBuilder.addImm(0);
 
       for (unsigned i = 0; i < PushedRegNum; i++)
-        PushBuilder.addUse(AllPopRegs[i], RegState::Implicit);
+        PushBuilder.addUse(FixedCSRFIMap[i].first, RegState::Implicit);
     }
   } else if (const char *SpillLibCall = getSpillLibCallName(*MF, CSI)) {
     // Add spill libcall via non-callee-saved register t0.
@@ -1611,7 +1608,7 @@ bool RISCVFrameLowering::restoreCalleeSavedRegisters(
       PopBuilder.addImm(0);
 
       for (unsigned i = 0; i < RVFI->getRVPushRegs(); i++)
-        PopBuilder.addDef(AllPopRegs[i], RegState::ImplicitDefine);
+        PopBuilder.addDef(FixedCSRFIMap[i].first, RegState::ImplicitDefine);
     }
   } else {
     const char *RestoreLibCall = getRestoreLibCallName(*MF, CSI);
-- 
GitLab


From 96c23af8b39a222ce1facd2ec621fbe661e072b7 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Tue, 14 May 2024 00:46:17 -0400
Subject: [PATCH 0674/1206] [libc] fix 32bit arm build (casting time_t)
 (#92065)

---
 libc/src/__support/time/units.h | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/libc/src/__support/time/units.h b/libc/src/__support/time/units.h
index f6bd19f9b139..ee74af70efdf 100644
--- a/libc/src/__support/time/units.h
+++ b/libc/src/__support/time/units.h
@@ -15,22 +15,22 @@
 namespace LIBC_NAMESPACE {
 namespace time_units {
 LIBC_INLINE constexpr time_t operator""_s_ns(unsigned long long s) {
-  return s * 1'000'000'000;
+  return static_cast<time_t>(s * 1'000'000'000);
 }
 LIBC_INLINE constexpr time_t operator""_s_us(unsigned long long s) {
-  return s * 1'000'000;
+  return static_cast<time_t>(s * 1'000'000);
 }
 LIBC_INLINE constexpr time_t operator""_s_ms(unsigned long long s) {
-  return s * 1'000;
+  return static_cast<time_t>(s * 1'000);
 }
 LIBC_INLINE constexpr time_t operator""_ms_ns(unsigned long long ms) {
-  return ms * 1'000'000;
+  return static_cast<time_t>(ms * 1'000'000);
 }
 LIBC_INLINE constexpr time_t operator""_ms_us(unsigned long long ms) {
-  return ms * 1'000;
+  return static_cast<time_t>(ms * 1'000);
 }
 LIBC_INLINE constexpr time_t operator""_us_ns(unsigned long long us) {
-  return us * 1'000;
+  return static_cast<time_t>(us * 1'000);
 }
 } // namespace time_units
 } // namespace LIBC_NAMESPACE
-- 
GitLab


From 364f988d3feb46ead8fdb657c9eab78d93425a28 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Mon, 13 May 2024 21:52:50 -0700
Subject: [PATCH 0675/1206] Reland "[clang-format] Fix
 FormatToken::isSimpleTypeSpecifier() (#91712)"

Remove FormatToken::isSimpleTypeSpecifier() and call
Token::isSimpleTypeSpecifier(LangOpts) instead.
---
 clang/lib/Format/Format.cpp                   |  3 +-
 clang/lib/Format/FormatToken.cpp              | 46 ++--------
 clang/lib/Format/FormatToken.h                |  8 +-
 clang/lib/Format/FormatTokenLexer.cpp         |  1 -
 clang/lib/Format/QualifierAlignmentFixer.cpp  | 22 +++--
 clang/lib/Format/QualifierAlignmentFixer.h    |  4 +-
 clang/lib/Format/TokenAnalyzer.cpp            |  4 +-
 clang/lib/Format/TokenAnalyzer.h              |  1 +
 clang/lib/Format/TokenAnnotator.cpp           | 43 ++++++----
 clang/lib/Format/TokenAnnotator.h             |  6 +-
 clang/lib/Format/UnwrappedLineParser.cpp      | 19 +++--
 clang/lib/Format/UnwrappedLineParser.h        |  1 +
 clang/unittests/Format/QualifierFixerTest.cpp | 85 +++++++++++--------
 13 files changed, 116 insertions(+), 127 deletions(-)

diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 8f027ffa20cc..52005a6c881f 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -3858,8 +3858,7 @@ LangOptions getFormattingLangOpts(const FormatStyle &Style) {
   LangOpts.Digraphs = LexingStd >= FormatStyle::LS_Cpp11;
 
   LangOpts.LineComment = 1;
-  bool AlternativeOperators = Style.isCpp();
-  LangOpts.CXXOperatorNames = AlternativeOperators ? 1 : 0;
+  LangOpts.CXXOperatorNames = Style.isCpp();
   LangOpts.Bool = 1;
   LangOpts.ObjC = 1;
   LangOpts.MicrosoftExt = 1;    // To get kw___try, kw___finally.
diff --git a/clang/lib/Format/FormatToken.cpp b/clang/lib/Format/FormatToken.cpp
index 4fb70ffac706..85bec71ffbbc 100644
--- a/clang/lib/Format/FormatToken.cpp
+++ b/clang/lib/Format/FormatToken.cpp
@@ -34,43 +34,6 @@ const char *getTokenTypeName(TokenType Type) {
   return nullptr;
 }
 
-// FIXME: This is copy&pasted from Sema. Put it in a common place and remove
-// duplication.
-bool FormatToken::isSimpleTypeSpecifier() const {
-  switch (Tok.getKind()) {
-  case tok::kw_short:
-  case tok::kw_long:
-  case tok::kw___int64:
-  case tok::kw___int128:
-  case tok::kw_signed:
-  case tok::kw_unsigned:
-  case tok::kw_void:
-  case tok::kw_char:
-  case tok::kw_int:
-  case tok::kw_half:
-  case tok::kw_float:
-  case tok::kw_double:
-  case tok::kw___bf16:
-  case tok::kw__Float16:
-  case tok::kw___float128:
-  case tok::kw___ibm128:
-  case tok::kw_wchar_t:
-  case tok::kw_bool:
-#define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) case tok::kw___##Trait:
-#include "clang/Basic/TransformTypeTraits.def"
-  case tok::annot_typename:
-  case tok::kw_char8_t:
-  case tok::kw_char16_t:
-  case tok::kw_char32_t:
-  case tok::kw_typeof:
-  case tok::kw_decltype:
-  case tok::kw__Atomic:
-    return true;
-  default:
-    return false;
-  }
-}
-
 // Sorted common C++ non-keyword types.
 static SmallVector<StringRef> CppNonKeywordTypes = {
     "clock_t",  "int16_t",   "int32_t", "int64_t",   "int8_t",
@@ -78,15 +41,16 @@ static SmallVector<StringRef> CppNonKeywordTypes = {
     "uint32_t", "uint64_t",  "uint8_t", "uintptr_t",
 };
 
-bool FormatToken::isTypeName(bool IsCpp) const {
-  return is(TT_TypeName) || isSimpleTypeSpecifier() ||
+bool FormatToken::isTypeName(const LangOptions &LangOpts) const {
+  const bool IsCpp = LangOpts.CXXOperatorNames;
+  return is(TT_TypeName) || Tok.isSimpleTypeSpecifier(LangOpts) ||
          (IsCpp && is(tok::identifier) &&
           std::binary_search(CppNonKeywordTypes.begin(),
                              CppNonKeywordTypes.end(), TokenText));
 }
 
-bool FormatToken::isTypeOrIdentifier(bool IsCpp) const {
-  return isTypeName(IsCpp) || isOneOf(tok::kw_auto, tok::identifier);
+bool FormatToken::isTypeOrIdentifier(const LangOptions &LangOpts) const {
+  return isTypeName(LangOpts) || isOneOf(tok::kw_auto, tok::identifier);
 }
 
 bool FormatToken::isBlockIndentedInitRBrace(const FormatStyle &Style) const {
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 95f16fde5005..8792f4c75074 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -684,12 +684,8 @@ public:
            isAttribute();
   }
 
-  /// Determine whether the token is a simple-type-specifier.
-  [[nodiscard]] bool isSimpleTypeSpecifier() const;
-
-  [[nodiscard]] bool isTypeName(bool IsCpp) const;
-
-  [[nodiscard]] bool isTypeOrIdentifier(bool IsCpp) const;
+  [[nodiscard]] bool isTypeName(const LangOptions &LangOpts) const;
+  [[nodiscard]] bool isTypeOrIdentifier(const LangOptions &LangOpts) const;
 
   bool isObjCAccessSpecifier() const {
     return is(tok::at) && Next &&
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index f430d3764bab..e21b5a882b77 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -1442,7 +1442,6 @@ void FormatTokenLexer::readRawToken(FormatToken &Tok) {
 
 void FormatTokenLexer::resetLexer(unsigned Offset) {
   StringRef Buffer = SourceMgr.getBufferData(ID);
-  LangOpts = getFormattingLangOpts(Style);
   Lex.reset(new Lexer(SourceMgr.getLocForStartOfFile(ID), LangOpts,
                       Buffer.begin(), Buffer.begin() + Offset, Buffer.end()));
   Lex->SetKeepWhitespaceMode(true);
diff --git a/clang/lib/Format/QualifierAlignmentFixer.cpp b/clang/lib/Format/QualifierAlignmentFixer.cpp
index 36d0639041c6..593f8efff25a 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.cpp
+++ b/clang/lib/Format/QualifierAlignmentFixer.cpp
@@ -268,13 +268,11 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeRight(
   if (isPossibleMacro(TypeToken))
     return Tok;
 
-  const bool IsCpp = Style.isCpp();
-
   // The case `const long long int volatile` -> `long long int const volatile`
   // The case `long const long int volatile` -> `long long int const volatile`
   // The case `long long volatile int const` -> `long long int const volatile`
   // The case `const long long volatile int` -> `long long int const volatile`
-  if (TypeToken->isTypeName(IsCpp)) {
+  if (TypeToken->isTypeName(LangOpts)) {
     // The case `const decltype(foo)` -> `const decltype(foo)`
     // The case `const typeof(foo)` -> `const typeof(foo)`
     // The case `const _Atomic(foo)` -> `const _Atomic(foo)`
@@ -283,7 +281,7 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeRight(
 
     const FormatToken *LastSimpleTypeSpecifier = TypeToken;
     while (isQualifierOrType(LastSimpleTypeSpecifier->getNextNonComment(),
-                             IsCpp)) {
+                             LangOpts)) {
       LastSimpleTypeSpecifier = LastSimpleTypeSpecifier->getNextNonComment();
     }
 
@@ -295,7 +293,7 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeRight(
   // The case  `unsigned short const` -> `unsigned short const`
   // The case:
   // `unsigned short volatile const` -> `unsigned short const volatile`
-  if (PreviousCheck && PreviousCheck->isTypeName(IsCpp)) {
+  if (PreviousCheck && PreviousCheck->isTypeName(LangOpts)) {
     if (LastQual != Tok)
       rotateTokens(SourceMgr, Fixes, Tok, LastQual, /*Left=*/false);
     return Tok;
@@ -412,11 +410,11 @@ const FormatToken *LeftRightQualifierAlignmentFixer::analyzeLeft(
   // The case `volatile long long const int` -> `const volatile long long int`
   // The case `const long long volatile int` -> `const volatile long long int`
   // The case `long volatile long int const` -> `const volatile long long int`
-  if (const bool IsCpp = Style.isCpp(); TypeToken->isTypeName(IsCpp)) {
+  if (TypeToken->isTypeName(LangOpts)) {
     const FormatToken *LastSimpleTypeSpecifier = TypeToken;
     while (isConfiguredQualifierOrType(
         LastSimpleTypeSpecifier->getPreviousNonComment(),
-        ConfiguredQualifierTokens, IsCpp)) {
+        ConfiguredQualifierTokens, LangOpts)) {
       LastSimpleTypeSpecifier =
           LastSimpleTypeSpecifier->getPreviousNonComment();
     }
@@ -614,15 +612,15 @@ void prepareLeftRightOrderingForQualifierAlignmentFixer(
   }
 }
 
-bool isQualifierOrType(const FormatToken *Tok, bool IsCpp) {
-  return Tok &&
-         (Tok->isTypeName(IsCpp) || Tok->is(tok::kw_auto) || isQualifier(Tok));
+bool isQualifierOrType(const FormatToken *Tok, const LangOptions &LangOpts) {
+  return Tok && (Tok->isTypeName(LangOpts) || Tok->is(tok::kw_auto) ||
+                 isQualifier(Tok));
 }
 
 bool isConfiguredQualifierOrType(const FormatToken *Tok,
                                  const std::vector<tok::TokenKind> &Qualifiers,
-                                 bool IsCpp) {
-  return Tok && (Tok->isTypeName(IsCpp) || Tok->is(tok::kw_auto) ||
+                                 const LangOptions &LangOpts) {
+  return Tok && (Tok->isTypeName(LangOpts) || Tok->is(tok::kw_auto) ||
                  isConfiguredQualifier(Tok, Qualifiers));
 }
 
diff --git a/clang/lib/Format/QualifierAlignmentFixer.h b/clang/lib/Format/QualifierAlignmentFixer.h
index e31d525da164..a0a0d597ebf3 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.h
+++ b/clang/lib/Format/QualifierAlignmentFixer.h
@@ -33,10 +33,10 @@ void prepareLeftRightOrderingForQualifierAlignmentFixer(
     std::vector<tok::TokenKind> &Qualifiers);
 
 // Is the Token a simple or qualifier type
-bool isQualifierOrType(const FormatToken *Tok, bool IsCpp = true);
+bool isQualifierOrType(const FormatToken *Tok, const LangOptions &LangOpts);
 bool isConfiguredQualifierOrType(const FormatToken *Tok,
                                  const std::vector<tok::TokenKind> &Qualifiers,
-                                 bool IsCpp = true);
+                                 const LangOptions &LangOpts);
 
 // Is the Token likely a Macro
 bool isPossibleMacro(const FormatToken *Tok);
diff --git a/clang/lib/Format/TokenAnalyzer.cpp b/clang/lib/Format/TokenAnalyzer.cpp
index bd648c430f9b..804a2b0f5e8c 100644
--- a/clang/lib/Format/TokenAnalyzer.cpp
+++ b/clang/lib/Format/TokenAnalyzer.cpp
@@ -84,7 +84,7 @@ Environment::Environment(StringRef Code, StringRef FileName,
       NextStartColumn(NextStartColumn), LastStartColumn(LastStartColumn) {}
 
 TokenAnalyzer::TokenAnalyzer(const Environment &Env, const FormatStyle &Style)
-    : Style(Style), Env(Env),
+    : Style(Style), LangOpts(getFormattingLangOpts(Style)), Env(Env),
       AffectedRangeMgr(Env.getSourceManager(), Env.getCharRanges()),
       UnwrappedLines(1),
       Encoding(encoding::detectEncoding(
@@ -101,7 +101,7 @@ std::pair<tooling::Replacements, unsigned>
 TokenAnalyzer::process(bool SkipAnnotation) {
   tooling::Replacements Result;
   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
-  IdentifierTable IdentTable(getFormattingLangOpts(Style));
+  IdentifierTable IdentTable(LangOpts);
   FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(),
                        Env.getFirstStartColumn(), Style, Encoding, Allocator,
                        IdentTable);
diff --git a/clang/lib/Format/TokenAnalyzer.h b/clang/lib/Format/TokenAnalyzer.h
index b7494c395c8a..ef559099d325 100644
--- a/clang/lib/Format/TokenAnalyzer.h
+++ b/clang/lib/Format/TokenAnalyzer.h
@@ -92,6 +92,7 @@ protected:
   void finishRun() override;
 
   FormatStyle Style;
+  LangOptions LangOpts;
   // Stores Style, FileID and SourceManager etc.
   const Environment &Env;
   // AffectedRangeMgr stores ranges to be fixed.
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index e935d3e2709c..478cae23d3c8 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -126,7 +126,9 @@ public:
                    const AdditionalKeywords &Keywords,
                    SmallVector<ScopeType> &Scopes)
       : Style(Style), Line(Line), CurrentToken(Line.First), AutoFound(false),
-        IsCpp(Style.isCpp()), Keywords(Keywords), Scopes(Scopes) {
+        IsCpp(Style.isCpp()), LangOpts(getFormattingLangOpts(Style)),
+        Keywords(Keywords), Scopes(Scopes) {
+    assert(IsCpp == LangOpts.CXXOperatorNames);
     Contexts.push_back(Context(tok::unknown, 1, /*IsExpression=*/false));
     resetTokenMetadata();
   }
@@ -562,7 +564,7 @@ private:
           (CurrentToken->is(tok::l_paren) && CurrentToken->Next &&
            CurrentToken->Next->isOneOf(tok::star, tok::amp, tok::caret));
       if ((CurrentToken->Previous->isOneOf(tok::kw_const, tok::kw_auto) ||
-           CurrentToken->Previous->isTypeName(IsCpp)) &&
+           CurrentToken->Previous->isTypeName(LangOpts)) &&
           !(CurrentToken->is(tok::l_brace) ||
             (CurrentToken->is(tok::l_paren) && !ProbablyFunctionTypeLParen))) {
         Contexts.back().IsExpression = false;
@@ -2624,7 +2626,7 @@ private:
       return true;
 
     // MyClass a;
-    if (PreviousNotConst->isTypeName(IsCpp))
+    if (PreviousNotConst->isTypeName(LangOpts))
       return true;
 
     // type[] a in Java
@@ -2728,7 +2730,7 @@ private:
     }
 
     if (Tok.Next->is(tok::question) ||
-        (Tok.Next->is(tok::ampamp) && !Tok.Previous->isTypeName(IsCpp))) {
+        (Tok.Next->is(tok::ampamp) && !Tok.Previous->isTypeName(LangOpts))) {
       return false;
     }
 
@@ -2757,9 +2759,10 @@ private:
     }
 
     // Heuristically try to determine whether the parentheses contain a type.
-    auto IsQualifiedPointerOrReference = [](FormatToken *T, bool IsCpp) {
+    auto IsQualifiedPointerOrReference = [](FormatToken *T,
+                                            const LangOptions &LangOpts) {
       // This is used to handle cases such as x = (foo *const)&y;
-      assert(!T->isTypeName(IsCpp) && "Should have already been checked");
+      assert(!T->isTypeName(LangOpts) && "Should have already been checked");
       // Strip trailing qualifiers such as const or volatile when checking
       // whether the parens could be a cast to a pointer/reference type.
       while (T) {
@@ -2791,8 +2794,8 @@ private:
     bool ParensAreType =
         !Tok.Previous ||
         Tok.Previous->isOneOf(TT_TemplateCloser, TT_TypeDeclarationParen) ||
-        Tok.Previous->isTypeName(IsCpp) ||
-        IsQualifiedPointerOrReference(Tok.Previous, IsCpp);
+        Tok.Previous->isTypeName(LangOpts) ||
+        IsQualifiedPointerOrReference(Tok.Previous, LangOpts);
     bool ParensCouldEndDecl =
         Tok.Next->isOneOf(tok::equal, tok::semi, tok::l_brace, tok::greater);
     if (ParensAreType && !ParensCouldEndDecl)
@@ -3065,6 +3068,7 @@ private:
   FormatToken *CurrentToken;
   bool AutoFound;
   bool IsCpp;
+  LangOptions LangOpts;
   const AdditionalKeywords &Keywords;
 
   SmallVector<ScopeType> &Scopes;
@@ -3639,7 +3643,8 @@ void TokenAnnotator::annotate(AnnotatedLine &Line) {
 
 // This function heuristically determines whether 'Current' starts the name of a
 // function declaration.
-static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
+static bool isFunctionDeclarationName(const LangOptions &LangOpts,
+                                      const FormatToken &Current,
                                       const AnnotatedLine &Line,
                                       FormatToken *&ClosingParen) {
   assert(Current.Previous);
@@ -3658,7 +3663,7 @@ static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
   }
 
   auto skipOperatorName =
-      [IsCpp](const FormatToken *Next) -> const FormatToken * {
+      [&LangOpts](const FormatToken *Next) -> const FormatToken * {
     for (; Next; Next = Next->Next) {
       if (Next->is(TT_OverloadedOperatorLParen))
         return Next;
@@ -3677,7 +3682,7 @@ static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
         Next = Next->Next;
         continue;
       }
-      if ((Next->isTypeName(IsCpp) || Next->is(tok::identifier)) &&
+      if ((Next->isTypeName(LangOpts) || Next->is(tok::identifier)) &&
           Next->Next && Next->Next->isPointerOrReference()) {
         // For operator void*(), operator char*(), operator Foo*().
         Next = Next->Next;
@@ -3693,8 +3698,10 @@ static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
     return nullptr;
   };
 
+  const auto *Next = Current.Next;
+  const bool IsCpp = LangOpts.CXXOperatorNames;
+
   // Find parentheses of parameter list.
-  const FormatToken *Next = Current.Next;
   if (Current.is(tok::kw_operator)) {
     if (Previous.Tok.getIdentifierInfo() &&
         !Previous.isOneOf(tok::kw_return, tok::kw_co_return)) {
@@ -3774,7 +3781,7 @@ static bool isFunctionDeclarationName(bool IsCpp, const FormatToken &Current,
       Tok = Tok->MatchingParen;
       continue;
     }
-    if (Tok->is(tok::kw_const) || Tok->isTypeName(IsCpp) ||
+    if (Tok->is(tok::kw_const) || Tok->isTypeName(LangOpts) ||
         Tok->isOneOf(TT_PointerOrReference, TT_StartOfName, tok::ellipsis)) {
       return true;
     }
@@ -3837,7 +3844,7 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) const {
       AfterLastAttribute = Tok;
     if (const bool IsCtorOrDtor = Tok->is(TT_CtorDtorDeclName);
         IsCtorOrDtor ||
-        isFunctionDeclarationName(IsCpp, *Tok, Line, ClosingParen)) {
+        isFunctionDeclarationName(LangOpts, *Tok, Line, ClosingParen)) {
       if (!IsCtorOrDtor)
         Tok->setFinalizedType(TT_FunctionDeclarationName);
       LineIsFunctionDeclaration = true;
@@ -4447,7 +4454,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     if (Left.Tok.isLiteral())
       return true;
     // for (auto a = 0, b = 0; const auto & c : {1, 2, 3})
-    if (Left.isTypeOrIdentifier(IsCpp) && Right.Next && Right.Next->Next &&
+    if (Left.isTypeOrIdentifier(LangOpts) && Right.Next && Right.Next->Next &&
         Right.Next->Next->is(TT_RangeBasedForLoopColon)) {
       return getTokenPointerOrReferenceAlignment(Right) !=
              FormatStyle::PAS_Left;
@@ -4490,7 +4497,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     if (Right.is(tok::l_brace) && Right.is(BK_Block))
       return true;
     // for (auto a = 0, b = 0; const auto& c : {1, 2, 3})
-    if (BeforeLeft && BeforeLeft->isTypeOrIdentifier(IsCpp) && Right.Next &&
+    if (BeforeLeft && BeforeLeft->isTypeOrIdentifier(LangOpts) && Right.Next &&
         Right.Next->is(TT_RangeBasedForLoopColon)) {
       return getTokenPointerOrReferenceAlignment(Left) !=
              FormatStyle::PAS_Right;
@@ -4534,7 +4541,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   if (Right.isPointerOrReference()) {
     const FormatToken *Previous = &Left;
     while (Previous && Previous->isNot(tok::kw_operator)) {
-      if (Previous->is(tok::identifier) || Previous->isTypeName(IsCpp)) {
+      if (Previous->is(tok::identifier) || Previous->isTypeName(LangOpts)) {
         Previous = Previous->getPreviousNonComment();
         continue;
       }
@@ -4723,7 +4730,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   if (!Style.isVerilog() &&
       (Left.isOneOf(tok::identifier, tok::greater, tok::r_square,
                     tok::r_paren) ||
-       Left.isTypeName(IsCpp)) &&
+       Left.isTypeName(LangOpts)) &&
       Right.is(tok::l_brace) && Right.getNextNonComment() &&
       Right.isNot(BK_Block)) {
     return false;
diff --git a/clang/lib/Format/TokenAnnotator.h b/clang/lib/Format/TokenAnnotator.h
index 25a24dccb1b8..d19d3d061e40 100644
--- a/clang/lib/Format/TokenAnnotator.h
+++ b/clang/lib/Format/TokenAnnotator.h
@@ -211,7 +211,10 @@ private:
 class TokenAnnotator {
 public:
   TokenAnnotator(const FormatStyle &Style, const AdditionalKeywords &Keywords)
-      : Style(Style), IsCpp(Style.isCpp()), Keywords(Keywords) {}
+      : Style(Style), IsCpp(Style.isCpp()),
+        LangOpts(getFormattingLangOpts(Style)), Keywords(Keywords) {
+    assert(IsCpp == LangOpts.CXXOperatorNames);
+  }
 
   /// Adapts the indent levels of comment lines to the indent of the
   /// subsequent line.
@@ -260,6 +263,7 @@ private:
   const FormatStyle &Style;
 
   bool IsCpp;
+  LangOptions LangOpts;
 
   const AdditionalKeywords &Keywords;
 
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 310b75485e08..4f1c2c5114e9 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -160,13 +160,16 @@ UnwrappedLineParser::UnwrappedLineParser(
     IdentifierTable &IdentTable)
     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
       CurrentLines(&Lines), Style(Style), IsCpp(Style.isCpp()),
-      Keywords(Keywords), CommentPragmasRegex(Style.CommentPragmas),
-      Tokens(nullptr), Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1),
+      LangOpts(getFormattingLangOpts(Style)), Keywords(Keywords),
+      CommentPragmasRegex(Style.CommentPragmas), Tokens(nullptr),
+      Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1),
       IncludeGuard(Style.IndentPPDirectives == FormatStyle::PPDIS_None
                        ? IG_Rejected
                        : IG_Inited),
       IncludeGuardToken(nullptr), FirstStartColumn(FirstStartColumn),
-      Macros(Style.Macros, SourceMgr, Style, Allocator, IdentTable) {}
+      Macros(Style.Macros, SourceMgr, Style, Allocator, IdentTable) {
+  assert(IsCpp == LangOpts.CXXOperatorNames);
+}
 
 void UnwrappedLineParser::reset() {
   PPBranchLevel = -1;
@@ -1870,7 +1873,7 @@ void UnwrappedLineParser::parseStructuralElement(
     case tok::caret:
       nextToken();
       // Block return type.
-      if (FormatTok->Tok.isAnyIdentifier() || FormatTok->isTypeName(IsCpp)) {
+      if (FormatTok->Tok.isAnyIdentifier() || FormatTok->isTypeName(LangOpts)) {
         nextToken();
         // Return types: pointers are ok too.
         while (FormatTok->is(tok::star))
@@ -2231,7 +2234,7 @@ bool UnwrappedLineParser::tryToParseLambda() {
   bool InTemplateParameterList = false;
 
   while (FormatTok->isNot(tok::l_brace)) {
-    if (FormatTok->isTypeName(IsCpp)) {
+    if (FormatTok->isTypeName(LangOpts)) {
       nextToken();
       continue;
     }
@@ -3448,7 +3451,7 @@ bool UnwrappedLineParser::parseRequires() {
     break;
   }
   default:
-    if (PreviousNonComment->isTypeOrIdentifier(IsCpp)) {
+    if (PreviousNonComment->isTypeOrIdentifier(LangOpts)) {
       // This is a requires clause.
       parseRequiresClause(RequiresToken);
       return true;
@@ -3511,7 +3514,7 @@ bool UnwrappedLineParser::parseRequires() {
       --OpenAngles;
       break;
     default:
-      if (NextToken->isTypeName(IsCpp)) {
+      if (NextToken->isTypeName(LangOpts)) {
         FormatTok = Tokens->setPosition(StoredPosition);
         parseRequiresExpression(RequiresToken);
         return false;
@@ -4027,7 +4030,7 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
       if (FormatTok->is(tok::l_square)) {
         FormatToken *Previous = FormatTok->Previous;
         if (!Previous || (Previous->isNot(tok::r_paren) &&
-                          !Previous->isTypeOrIdentifier(IsCpp))) {
+                          !Previous->isTypeOrIdentifier(LangOpts))) {
           // Don't try parsing a lambda if we had a closing parenthesis before,
           // it was probably a pointer to an array: int (*)[].
           if (!tryToParseLambda())
diff --git a/clang/lib/Format/UnwrappedLineParser.h b/clang/lib/Format/UnwrappedLineParser.h
index 2a0fe19d0957..d7963a4211bb 100644
--- a/clang/lib/Format/UnwrappedLineParser.h
+++ b/clang/lib/Format/UnwrappedLineParser.h
@@ -316,6 +316,7 @@ private:
 
   const FormatStyle &Style;
   bool IsCpp;
+  LangOptions LangOpts;
   const AdditionalKeywords &Keywords;
 
   llvm::Regex CommentPragmasRegex;
diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index 4ddeef50f5f7..1f21fc0e0b42 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -1056,49 +1056,66 @@ TEST_F(QualifierFixerTest, IsQualifierType) {
   ConfiguredTokens.push_back(tok::kw_friend);
 
   TestLexer lexer{Allocator, Buffers};
+  const auto LangOpts = getFormattingLangOpts();
 
   auto Tokens = lexer.lex(
       "const static inline auto restrict int double long constexpr friend");
   ASSERT_EQ(Tokens.size(), 11u) << Tokens;
 
-  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[0], ConfiguredTokens));
-  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[1], ConfiguredTokens));
-  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[2], ConfiguredTokens));
-  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[3], ConfiguredTokens));
-  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[4], ConfiguredTokens));
-  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[5], ConfiguredTokens));
-  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[6], ConfiguredTokens));
-  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[7], ConfiguredTokens));
-  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[8], ConfiguredTokens));
-  EXPECT_TRUE(isConfiguredQualifierOrType(Tokens[9], ConfiguredTokens));
-
-  EXPECT_TRUE(isQualifierOrType(Tokens[0]));
-  EXPECT_TRUE(isQualifierOrType(Tokens[1]));
-  EXPECT_TRUE(isQualifierOrType(Tokens[2]));
-  EXPECT_TRUE(isQualifierOrType(Tokens[3]));
-  EXPECT_TRUE(isQualifierOrType(Tokens[4]));
-  EXPECT_TRUE(isQualifierOrType(Tokens[5]));
-  EXPECT_TRUE(isQualifierOrType(Tokens[6]));
-  EXPECT_TRUE(isQualifierOrType(Tokens[7]));
-  EXPECT_TRUE(isQualifierOrType(Tokens[8]));
-  EXPECT_TRUE(isQualifierOrType(Tokens[9]));
+  EXPECT_TRUE(
+      isConfiguredQualifierOrType(Tokens[0], ConfiguredTokens, LangOpts));
+  EXPECT_TRUE(
+      isConfiguredQualifierOrType(Tokens[1], ConfiguredTokens, LangOpts));
+  EXPECT_TRUE(
+      isConfiguredQualifierOrType(Tokens[2], ConfiguredTokens, LangOpts));
+  EXPECT_TRUE(
+      isConfiguredQualifierOrType(Tokens[3], ConfiguredTokens, LangOpts));
+  EXPECT_TRUE(
+      isConfiguredQualifierOrType(Tokens[4], ConfiguredTokens, LangOpts));
+  EXPECT_TRUE(
+      isConfiguredQualifierOrType(Tokens[5], ConfiguredTokens, LangOpts));
+  EXPECT_TRUE(
+      isConfiguredQualifierOrType(Tokens[6], ConfiguredTokens, LangOpts));
+  EXPECT_TRUE(
+      isConfiguredQualifierOrType(Tokens[7], ConfiguredTokens, LangOpts));
+  EXPECT_TRUE(
+      isConfiguredQualifierOrType(Tokens[8], ConfiguredTokens, LangOpts));
+  EXPECT_TRUE(
+      isConfiguredQualifierOrType(Tokens[9], ConfiguredTokens, LangOpts));
+
+  EXPECT_TRUE(isQualifierOrType(Tokens[0], LangOpts));
+  EXPECT_TRUE(isQualifierOrType(Tokens[1], LangOpts));
+  EXPECT_TRUE(isQualifierOrType(Tokens[2], LangOpts));
+  EXPECT_TRUE(isQualifierOrType(Tokens[3], LangOpts));
+  EXPECT_TRUE(isQualifierOrType(Tokens[4], LangOpts));
+  EXPECT_TRUE(isQualifierOrType(Tokens[5], LangOpts));
+  EXPECT_TRUE(isQualifierOrType(Tokens[6], LangOpts));
+  EXPECT_TRUE(isQualifierOrType(Tokens[7], LangOpts));
+  EXPECT_TRUE(isQualifierOrType(Tokens[8], LangOpts));
+  EXPECT_TRUE(isQualifierOrType(Tokens[9], LangOpts));
 
   auto NotTokens = lexer.lex("for while do Foo Bar ");
   ASSERT_EQ(NotTokens.size(), 6u) << Tokens;
 
-  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[0], ConfiguredTokens));
-  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[1], ConfiguredTokens));
-  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[2], ConfiguredTokens));
-  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[3], ConfiguredTokens));
-  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[4], ConfiguredTokens));
-  EXPECT_FALSE(isConfiguredQualifierOrType(NotTokens[5], ConfiguredTokens));
-
-  EXPECT_FALSE(isQualifierOrType(NotTokens[0]));
-  EXPECT_FALSE(isQualifierOrType(NotTokens[1]));
-  EXPECT_FALSE(isQualifierOrType(NotTokens[2]));
-  EXPECT_FALSE(isQualifierOrType(NotTokens[3]));
-  EXPECT_FALSE(isQualifierOrType(NotTokens[4]));
-  EXPECT_FALSE(isQualifierOrType(NotTokens[5]));
+  EXPECT_FALSE(
+      isConfiguredQualifierOrType(NotTokens[0], ConfiguredTokens, LangOpts));
+  EXPECT_FALSE(
+      isConfiguredQualifierOrType(NotTokens[1], ConfiguredTokens, LangOpts));
+  EXPECT_FALSE(
+      isConfiguredQualifierOrType(NotTokens[2], ConfiguredTokens, LangOpts));
+  EXPECT_FALSE(
+      isConfiguredQualifierOrType(NotTokens[3], ConfiguredTokens, LangOpts));
+  EXPECT_FALSE(
+      isConfiguredQualifierOrType(NotTokens[4], ConfiguredTokens, LangOpts));
+  EXPECT_FALSE(
+      isConfiguredQualifierOrType(NotTokens[5], ConfiguredTokens, LangOpts));
+
+  EXPECT_FALSE(isQualifierOrType(NotTokens[0], LangOpts));
+  EXPECT_FALSE(isQualifierOrType(NotTokens[1], LangOpts));
+  EXPECT_FALSE(isQualifierOrType(NotTokens[2], LangOpts));
+  EXPECT_FALSE(isQualifierOrType(NotTokens[3], LangOpts));
+  EXPECT_FALSE(isQualifierOrType(NotTokens[4], LangOpts));
+  EXPECT_FALSE(isQualifierOrType(NotTokens[5], LangOpts));
 }
 
 TEST_F(QualifierFixerTest, IsMacro) {
-- 
GitLab


From cd45bb2e435b8e648ac528ed5f5fc1dc2bba48fe Mon Sep 17 00:00:00 2001
From: Robin Caloudis <robin.caloudis@gmx.de>
Date: Tue, 14 May 2024 06:58:13 +0200
Subject: [PATCH 0676/1206] [libc][errno] Remove unnecessary include (#92063)

Since https://github.com/llvm/llvm-project/pull/91150, a proxy header
for the errno macros is available and gets included in `libc_errno.h`
since then.

As `libc_errno.cpp` includes `libc_errno.h`, which already includes the
proxy header `hdr/errno_macros.h`, there's no need to include it in
`libc_errno.cpp` if we are in overlay mode, because the proxy header
takes care to either include our header from libc/include/ (fullbuild)
or the corresponding underlying system header (overlay).
---
 libc/src/errno/libc_errno.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/libc/src/errno/libc_errno.cpp b/libc/src/errno/libc_errno.cpp
index a59e6c34029d..64f9f522ca29 100644
--- a/libc/src/errno/libc_errno.cpp
+++ b/libc/src/errno/libc_errno.cpp
@@ -36,9 +36,6 @@ void LIBC_NAMESPACE::Errno::operator=(int a) { __llvmlibc_errno = a; }
 LIBC_NAMESPACE::Errno::operator int() { return __llvmlibc_errno; }
 
 #else
-// In overlay mode, we simply use the system errno.
-#include "hdr/errno_macros.h"
-
 void LIBC_NAMESPACE::Errno::operator=(int a) { errno = a; }
 LIBC_NAMESPACE::Errno::operator int() { return errno; }
 
-- 
GitLab


From f3b8d91ca885744925ce775026df40660d9a4d4e Mon Sep 17 00:00:00 2001
From: appujee <124090381+appujee@users.noreply.github.com>
Date: Mon, 13 May 2024 22:39:51 -0700
Subject: [PATCH 0677/1206] LLVM vectorizer working group (#92068)

Recurring meeting at 3rd Thursday of every month.
---
 llvm/docs/GettingInvolved.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/docs/GettingInvolved.rst b/llvm/docs/GettingInvolved.rst
index 89df6cbe31c4..3588ef14db15 100644
--- a/llvm/docs/GettingInvolved.rst
+++ b/llvm/docs/GettingInvolved.rst
@@ -209,6 +209,10 @@ what to add to your calendar invite.
      - `ics <https://calendar.google.com/calendar/ical/02582507bac79d186900712566ec3fc69b33ac24d7de0a8c76c7b19976f190c0%40group.calendar.google.com/private-6e35506dbfe13812e92e9aa8cd5d761d/basic.ics>`__
        `gcal <https://calendar.google.com/calendar/u/0?cid=MDI1ODI1MDdiYWM3OWQxODY5MDA3MTI1NjZlYzNmYzY5YjMzYWMyNGQ3ZGUwYThjNzZjN2IxOTk3NmYxOTBjMEBncm91cC5jYWxlbmRhci5nb29nbGUuY29t>`__
      - `Meeting details/agenda: <https://docs.google.com/document/d/1QcmUlWftPlBi-Wz6b6PipqJfvjpJ-OuRMRnN9Dm2t0c>`__
+   * - Vectorizer Improvement Working Group
+     - Every 3rd Thursday of the month
+     - `ics <https://drive.google.com/file/d/1ten-u-4yjOcCoONUtR4_AxsFxRDTUp1b/view?usp=sharing>`__
+     - `Meeting details/agenda: <https://docs.google.com/document/d/1Glzy2JiWuysbD-HBWGUOkZqT09GJ4_Ljodr0lXD5XfQ/edit>`__
 
 Past online sync-ups
 ^^^^^^^^^^^^^^^^^^^^
-- 
GitLab


From 12c0024d196189bd38a140512c0bfbda85d8d75e Mon Sep 17 00:00:00 2001
From: Tomas Matheson <Tomas.Matheson@arm.com>
Date: Tue, 14 May 2024 07:04:29 +0100
Subject: [PATCH 0678/1206] [AArch64][TargetParser] Move extension aliases into
 tablegen (#91970)

---
 .../llvm/TargetParser/AArch64TargetParser.h      | 14 ++++++--------
 llvm/lib/Target/AArch64/AArch64Features.td       |  5 +++++
 llvm/lib/TargetParser/AArch64TargetParser.cpp    | 16 +++-------------
 llvm/utils/TableGen/ARMTargetDefEmitter.cpp      |  6 +++++-
 4 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index 20c3f95173c2..b662a74e5fc6 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -114,11 +114,12 @@ using ExtensionBitset = Bitset<AEK_NUM_EXTENSIONS>;
 // SubtargetFeature which may represent either an actual extension or some
 // internal LLVM property.
 struct ExtensionInfo {
-  StringRef Name;              // Human readable name, e.g. "profile".
-  ArchExtKind ID;              // Corresponding to the ArchExtKind, this
-                               // extensions representation in the bitfield.
-  StringRef Feature;           // -mattr enable string, e.g. "+spe"
-  StringRef NegFeature;        // -mattr disable string, e.g. "-spe"
+  StringRef Name;                 // Human readable name, e.g. "profile".
+  std::optional<StringRef> Alias; // An alias for this extension, if one exists.
+  ArchExtKind ID;                 // Corresponding to the ArchExtKind, this
+                                  // extensions representation in the bitfield.
+  StringRef Feature;              // -mattr enable string, e.g. "+spe"
+  StringRef NegFeature;           // -mattr disable string, e.g. "-spe"
   CPUFeatures CPUFeature;      // Function Multi Versioning (FMV) bitfield value
                                // set in __aarch64_cpu_features
   StringRef DependentFeatures; // FMV enabled features string,
@@ -674,8 +675,6 @@ struct Alias {
 inline constexpr Alias CpuAliases[] = {{"cobalt-100", "neoverse-n2"},
                                        {"grace", "neoverse-v2"}};
 
-inline constexpr Alias ExtAliases[] = {{"rdma", "rdm"}};
-
 const ExtensionInfo &getExtensionByID(ArchExtKind(ExtID));
 
 bool getExtensionFeatures(
@@ -684,7 +683,6 @@ bool getExtensionFeatures(
 
 StringRef getArchExtFeature(StringRef ArchExt);
 StringRef resolveCPUAlias(StringRef CPU);
-StringRef resolveExtAlias(StringRef ArchExt);
 
 // Information by Name
 const ArchInfo *getArchForCpu(StringRef CPU);
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 920ca7f4fbfc..b9c26e99ae03 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -39,6 +39,10 @@ class Extension<
     // not doing so.
     string MArchName = TargetFeatureName;
 
+    // An alias that can be used on the command line, if the extension has one.
+    // Used for correcting historical names while remaining backwards compatible.
+    string MArchAlias = "";
+
     // Function MultiVersioning (FMV) properties
 
     // A C++ expression giving the number of the bit in the FMV ABI.
@@ -163,6 +167,7 @@ def FeatureOutlineAtomics : SubtargetFeature<"outline-atomics", "OutlineAtomics"
 def FeatureFMV : SubtargetFeature<"fmv", "HasFMV", "true",
   "Enable Function Multi Versioning support.">;
 
+let MArchAlias = "rdma" in
 def FeatureRDM : Extension<"rdm", "RDM",
   "Enable ARMv8.1 Rounding Double Multiply Add/Subtract instructions (FEAT_RDM)",
   [FeatureNEON],
diff --git a/llvm/lib/TargetParser/AArch64TargetParser.cpp b/llvm/lib/TargetParser/AArch64TargetParser.cpp
index 026214e7e2ea..c10b4be4eded 100644
--- a/llvm/lib/TargetParser/AArch64TargetParser.cpp
+++ b/llvm/lib/TargetParser/AArch64TargetParser.cpp
@@ -74,13 +74,6 @@ StringRef AArch64::resolveCPUAlias(StringRef Name) {
   return Name;
 }
 
-StringRef AArch64::resolveExtAlias(StringRef Name) {
-  for (const auto &A : ExtAliases)
-    if (A.AltName == Name)
-      return A.Name;
-  return Name;
-}
-
 StringRef AArch64::getArchExtFeature(StringRef ArchExt) {
   bool IsNegated = ArchExt.starts_with("no");
   StringRef ArchExtBase = IsNegated ? ArchExt.drop_front(2) : ArchExt;
@@ -120,13 +113,10 @@ const AArch64::ArchInfo *AArch64::parseArch(StringRef Arch) {
   return {};
 }
 
-std::optional<AArch64::ExtensionInfo> AArch64::parseArchExtension(StringRef ArchExt) {
-  // Resolve aliases first.
-  ArchExt = resolveExtAlias(ArchExt);
-
-  // Then find the Extension name.
+std::optional<AArch64::ExtensionInfo>
+AArch64::parseArchExtension(StringRef ArchExt) {
   for (const auto &A : Extensions) {
-    if (ArchExt == A.Name)
+    if (ArchExt == A.Name || ArchExt == A.Alias)
       return A;
   }
   return {};
diff --git a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
index 4a46f2ea9586..0e90f57af493 100644
--- a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
@@ -89,6 +89,10 @@ static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
     auto AEK = Rec->getValueAsString("ArchExtKindSpelling").upper();
     OS << "  ";
     OS << "{\"" << Rec->getValueAsString("MArchName") << "\"";
+    if (auto Alias = Rec->getValueAsString("MArchAlias"); Alias.empty())
+      OS << ", {}";
+    else
+      OS << ", \"" << Alias << "\"";
     OS << ", AArch64::" << AEK;
     if (AEK == "AEK_NONE") {
       // HACK: don't emit posfeat/negfeat strings for FMVOnlyExtensions.
@@ -102,7 +106,7 @@ static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
     OS << ", " << (uint64_t)Rec->getValueAsInt("FMVPriority");
     OS << "},\n";
   };
-  OS << "  {\"none\", AArch64::AEK_NONE, {}, {}, FEAT_INIT, \"\", "
+  OS << "  {\"none\", {}, AArch64::AEK_NONE, {}, {}, FEAT_INIT, \"\", "
         "ExtensionInfo::MaxFMVPriority},\n";
   OS << "};\n"
      << "#undef EMIT_EXTENSIONS\n"
-- 
GitLab


From 922fafaff83319e33e8a890a692df073d3ce55c9 Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Tue, 14 May 2024 08:26:00 +0200
Subject: [PATCH 0679/1206] [GlobalISel] Micro-optimize
 getConstantVRegValWithLookThrough (#91969)

I was benchmarking the MatchTable when I found that
`getConstantVRegValWithLookThrough` took a non-negligible amount of
time, about 7.5% of all of
`AArch64PreLegalizerCombinerImpl::tryCombineAll`.

I decided to take a closer look to see if I could squeeze some
performance out of it, and I landed on a few changes that:
- Avoid copying APint unnecessarily, especially returning
std::optional<APInt> can be expensive when a out parameter also works.
- Avoid indirect call by using templated function pointers instead of
function_ref/std::function

Both of those changes seem to speedup this function by about 50%, but my
benchmarking (`perf record`) seems inconsistent (so take measurements
with a grain of salt), I saw as high as 4.5% and as low as 2% for this
function on the exact same input after the changes, but it never got
close again to 7% in a few runs so this looks like a stable improvement.
---
 llvm/lib/CodeGen/GlobalISel/Utils.cpp | 76 ++++++++++++++++-----------
 1 file changed, 44 insertions(+), 32 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 4e3781cb4e9d..cd5dc0e01ed0 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -313,13 +313,22 @@ llvm::getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI) {
 
 namespace {
 
-typedef std::function<bool(const MachineInstr *)> IsOpcodeFn;
-typedef std::function<std::optional<APInt>(const MachineInstr *MI)> GetAPCstFn;
-
-std::optional<ValueAndVReg> getConstantVRegValWithLookThrough(
-    Register VReg, const MachineRegisterInfo &MRI, IsOpcodeFn IsConstantOpcode,
-    GetAPCstFn getAPCstValue, bool LookThroughInstrs = true,
-    bool LookThroughAnyExt = false) {
+// This function is used in many places, and as such, it has some
+// micro-optimizations to try and make it as fast as it can be.
+//
+// - We use template arguments to avoid an indirect call caused by passing a
+// function_ref/std::function
+// - GetAPCstValue does not return std::optional<APInt> as that's expensive.
+// Instead it returns true/false and places the result in a pre-constructed
+// APInt.
+//
+// Please change this function carefully and benchmark your changes.
+template <bool (*IsConstantOpcode)(const MachineInstr *),
+          bool (*GetAPCstValue)(const MachineInstr *MI, APInt &)>
+std::optional<ValueAndVReg>
+getConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI,
+                                  bool LookThroughInstrs = true,
+                                  bool LookThroughAnyExt = false) {
   SmallVector<std::pair<unsigned, unsigned>, 4> SeenOpcodes;
   MachineInstr *MI;
 
@@ -353,26 +362,25 @@ std::optional<ValueAndVReg> getConstantVRegValWithLookThrough(
   if (!MI || !IsConstantOpcode(MI))
     return std::nullopt;
 
-  std::optional<APInt> MaybeVal = getAPCstValue(MI);
-  if (!MaybeVal)
+  APInt Val;
+  if (!GetAPCstValue(MI, Val))
     return std::nullopt;
-  APInt &Val = *MaybeVal;
-  for (auto [Opcode, Size] : reverse(SeenOpcodes)) {
-    switch (Opcode) {
+  for (auto &Pair : reverse(SeenOpcodes)) {
+    switch (Pair.first) {
     case TargetOpcode::G_TRUNC:
-      Val = Val.trunc(Size);
+      Val = Val.trunc(Pair.second);
       break;
     case TargetOpcode::G_ANYEXT:
     case TargetOpcode::G_SEXT:
-      Val = Val.sext(Size);
+      Val = Val.sext(Pair.second);
       break;
     case TargetOpcode::G_ZEXT:
-      Val = Val.zext(Size);
+      Val = Val.zext(Pair.second);
       break;
     }
   }
 
-  return ValueAndVReg{Val, VReg};
+  return ValueAndVReg{std::move(Val), VReg};
 }
 
 bool isIConstant(const MachineInstr *MI) {
@@ -394,42 +402,46 @@ bool isAnyConstant(const MachineInstr *MI) {
   return Opc == TargetOpcode::G_CONSTANT || Opc == TargetOpcode::G_FCONSTANT;
 }
 
-std::optional<APInt> getCImmAsAPInt(const MachineInstr *MI) {
+bool getCImmAsAPInt(const MachineInstr *MI, APInt &Result) {
   const MachineOperand &CstVal = MI->getOperand(1);
-  if (CstVal.isCImm())
-    return CstVal.getCImm()->getValue();
-  return std::nullopt;
+  if (!CstVal.isCImm())
+    return false;
+  Result = CstVal.getCImm()->getValue();
+  return true;
 }
 
-std::optional<APInt> getCImmOrFPImmAsAPInt(const MachineInstr *MI) {
+bool getCImmOrFPImmAsAPInt(const MachineInstr *MI, APInt &Result) {
   const MachineOperand &CstVal = MI->getOperand(1);
   if (CstVal.isCImm())
-    return CstVal.getCImm()->getValue();
-  if (CstVal.isFPImm())
-    return CstVal.getFPImm()->getValueAPF().bitcastToAPInt();
-  return std::nullopt;
+    Result = CstVal.getCImm()->getValue();
+  else if (CstVal.isFPImm())
+    Result = CstVal.getFPImm()->getValueAPF().bitcastToAPInt();
+  else
+    return false;
+  return true;
 }
 
 } // end anonymous namespace
 
 std::optional<ValueAndVReg> llvm::getIConstantVRegValWithLookThrough(
     Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {
-  return getConstantVRegValWithLookThrough(VReg, MRI, isIConstant,
-                                           getCImmAsAPInt, LookThroughInstrs);
+  return getConstantVRegValWithLookThrough<isIConstant, getCImmAsAPInt>(
+      VReg, MRI, LookThroughInstrs);
 }
 
 std::optional<ValueAndVReg> llvm::getAnyConstantVRegValWithLookThrough(
     Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs,
     bool LookThroughAnyExt) {
-  return getConstantVRegValWithLookThrough(
-      VReg, MRI, isAnyConstant, getCImmOrFPImmAsAPInt, LookThroughInstrs,
-      LookThroughAnyExt);
+  return getConstantVRegValWithLookThrough<isAnyConstant,
+                                           getCImmOrFPImmAsAPInt>(
+      VReg, MRI, LookThroughInstrs, LookThroughAnyExt);
 }
 
 std::optional<FPValueAndVReg> llvm::getFConstantVRegValWithLookThrough(
     Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) {
-  auto Reg = getConstantVRegValWithLookThrough(
-      VReg, MRI, isFConstant, getCImmOrFPImmAsAPInt, LookThroughInstrs);
+  auto Reg =
+      getConstantVRegValWithLookThrough<isFConstant, getCImmOrFPImmAsAPInt>(
+          VReg, MRI, LookThroughInstrs);
   if (!Reg)
     return std::nullopt;
   return FPValueAndVReg{getConstantFPVRegVal(Reg->VReg, MRI)->getValueAPF(),
-- 
GitLab


From ea238974e1b5f2243b6753c2d737c1f04dd1f17b Mon Sep 17 00:00:00 2001
From: Matthias Gehre <matthias.gehre@amd.com>
Date: Tue, 14 May 2024 08:28:27 +0200
Subject: [PATCH 0680/1206] [mlir] [TOSA] Allow any floating point type
 (#91745)

After #86509 allowed all integer types in TOSA ops, this PR allows TOSA
ops on all floating point types.
This helps to experiment with `f64` and 8-bit float types when spec
conformance is not required.
---
 mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td  |  6 +++---
 .../mlir/Dialect/Tosa/IR/TosaTypesBase.td     | 21 ++++---------------
 .../Tosa/Transforms/TosaValidation.cpp        |  9 ++++----
 mlir/test/Dialect/Tosa/invalid.mlir           |  2 +-
 mlir/test/Dialect/Tosa/level_check.mlir       |  8 +++++++
 5 files changed, 20 insertions(+), 26 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index 97a36c49d01b..7871b46724a0 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -1857,11 +1857,11 @@ def Tosa_CastOp: Tosa_Op<"cast", [Pure,
   }];
 
   let arguments = (ins
-    Tosa_Tensor_Plus_F64:$input
+    Tosa_Tensor:$input
   );
 
   let results = (outs
-    Tosa_Tensor_Plus_F64:$output
+    Tosa_Tensor:$output
   );
 
   let assemblyFormat = "operands attr-dict `:` functional-type(operands, results)";
@@ -1944,7 +1944,7 @@ def Tosa_ConstOp : Tosa_Op<"const", [ConstantLike, Pure,
   );
 
   let results = (outs
-    TensorOf<[AnyTypeOf<[Tosa_AnyNumber_Plus_F64]>]>:$output
+    TensorOf<[AnyTypeOf<[Tosa_AnyNumber]>]>:$output
   );
 
   let hasFolder = 1;
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
index 3687891fe4b7..14fc9c7a6730 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
@@ -71,28 +71,16 @@ def Tosa_QuantizedInt	: AnyTypeOf<[ Tosa_QuantizedType<"uint8", [8], 0>,
                                      Tosa_QuantizedType<"int16", [16, 0], 1>,
                                      Tosa_QuantizedType<"int32", [32, 0], 1>]>;
 
-//===----------------------------------------------------------------------===//
-// Floating-point types.
-//===----------------------------------------------------------------------===//
-def Tosa_Float : AnyTypeOf<[
-                            F32,
-			    F16,
-			    BF16]>;
-
 //===----------------------------------------------------------------------===//
 // Multi-category types.
 //===----------------------------------------------------------------------===//
-def Tosa_AnyNumber : AnyTypeOf<[Tosa_Int, Tosa_QuantizedInt, Tosa_Float],
+def Tosa_AnyNumber : AnyTypeOf<[Tosa_Int, Tosa_QuantizedInt, AnyFloat],
                                "number">;
 
-// Add F64 type support just for tosa::CastOp and tosa::ConstOp
-def Tosa_AnyNumber_Plus_F64 : AnyTypeOf<[Tosa_Int, Tosa_QuantizedInt, Tosa_Float, F64],
-                               "number_plus_f64">;
-
 // For weight tensors from tosa::Conv2DOp, tosa::Conv3DOp,
 // tosa::DepthwiseConv2DOp, tosa::TransposeConv2DOp, tosa::FullyConnectedOp
 def Tosa_Weight : AnyTypeOf<[Tosa_Int4, Tosa_Int8,
-                             Tosa_QuantizedInt, Tosa_Float]>;
+                             Tosa_QuantizedInt, AnyFloat]>;
 
 //===----------------------------------------------------------------------===//
 // Tensor types
@@ -101,18 +89,17 @@ def Tosa_Weight : AnyTypeOf<[Tosa_Int4, Tosa_Int8,
 def Tosa_Int32Tensor : TensorOf<[Tosa_Int32]>;
 def Tosa_Int32Or64Tensor : TensorOf<[Tosa_Int32Or64]>;
 
-def Tosa_FloatTensor : TensorOf<[Tosa_Float]>;
+def Tosa_FloatTensor : TensorOf<[AnyFloat]>;
 
 // Either ranked or unranked tensor of TOSA supported element types.
 def Tosa_Tensor : TensorOf<[Tosa_AnyNumber]>;
-def Tosa_Tensor_Plus_F64 : TensorOf<[Tosa_AnyNumber_Plus_F64]>;
 
 // Must be ranked but no further constraints
 def Tosa_RankedTensor : RankedTensorOf<[Tosa_AnyNumber]>;
 
 // Any tensor element type allowed in Tosa ops.
 def Tosa_ElementType : Type<Or<[Tosa_Int.predicate, Tosa_QuantizedInt.predicate,
-                                Tosa_Float.predicate]>, "tosa.dtype">;
+                                AnyFloat.predicate]>, "tosa.dtype">;
 
 class Tosa_TensorOfOrNone<list<Type> allowedTypes, string description = ""> :
   AnyTypeOf<[TensorOf<allowedTypes>, NoneType], description>;
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
index 539501082fd3..b78c372af77e 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
@@ -506,11 +506,10 @@ LogicalResult TosaValidation::applyVariableCheck(Operation *op) {
 }
 
 bool TosaValidation::isValidElementType(Type type) {
-  if ((profile == TosaProfileEnum::BaseInference) && isa<FloatType>(type)) {
-    return false;
-  }
-  if (type.isF64()) {
-    return false;
+  if (isa<FloatType>(type)) {
+    if (profile == TosaProfileEnum::BaseInference)
+      return false;
+    return type.isF32() || type.isF16() || type.isBF16();
   }
   if (auto intTy = dyn_cast<IntegerType>(type)) {
     if (intTy.isUnsigned()) {
diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir
index 730ac41dd7a8..cb38d4d81ca2 100644
--- a/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/mlir/test/Dialect/Tosa/invalid.mlir
@@ -20,7 +20,7 @@ func.func @test_conv2d(%arg0: tensor<*xi8>, %arg1: tensor<16x3x3x4xi8>, %arg2: t
 // -----
 
 func.func @test_conv2d(%arg0: tensor<1x29x29x4xi8>, %arg1: tensor<*xi8>, %arg2: tensor<16xi8>) -> tensor<1x27x27x16xi8> {
-  // expected-error@+1 {{'tosa.conv2d' op operand #1 must be 4D tensor of 4-bit signless integer or 8-bit signless integer or Quint8 type or Qint4 type or Qint8 type or Qint16 type or Qint32 type or 32-bit float or 16-bit float or bfloat16 type values, but got 'tensor<*xi8>'}}
+  // expected-error@+1 {{'tosa.conv2d' op operand #1 must be 4D tensor of 4-bit signless integer or 8-bit signless integer or Quint8 type or Qint4 type or Qint8 type or Qint16 type or Qint32 type or floating-point values, but got 'tensor<*xi8>'}}
   %0 = tosa.conv2d %arg0, %arg1, %arg2 {dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
            : (tensor<1x29x29x4xi8>, tensor<*xi8>, tensor<16xi8>) -> tensor<1x27x27x16xi8>
   return %0 : tensor<1x27x27x16xi8>
diff --git a/mlir/test/Dialect/Tosa/level_check.mlir b/mlir/test/Dialect/Tosa/level_check.mlir
index d8dd878051f1..9b652f2d0bd1 100644
--- a/mlir/test/Dialect/Tosa/level_check.mlir
+++ b/mlir/test/Dialect/Tosa/level_check.mlir
@@ -131,6 +131,14 @@ func.func @test_const_ui32(%arg0 : tensor<1xui32>) {
 
 // -----
 
+func.func @test_const_f64(%arg0 : tensor<1xf64>) {
+  // expected-error@+1 {{'tosa.const' op is not profile-aligned: element type 'f64' is not legal}}
+  %0 = "tosa.const"() {value = dense<0.0> : tensor<1xf64>} : () -> tensor<1xf64>
+  return
+}
+
+// -----
+
 func.func @test_avgpool2d_kernel_y(%arg0: tensor<1x32x32x8xf32>) -> tensor<1x32x32x8xf32> {
   // expected-error@+1 {{'tosa.avg_pool2d' op failed level check: kernel <= MAX_KERNEL}}
   %0 = "tosa.avg_pool2d"(%arg0) {kernel = array<i64: 8193, 1>, pad = array<i64: 4, 4, 4, 4>, stride = array<i64: 1, 1>, acc_type = f32} :
-- 
GitLab


From ecce5ccdd5725bd0669c24742bfd46dbf043fec2 Mon Sep 17 00:00:00 2001
From: Matthias Gehre <matthias.gehre@amd.com>
Date: Tue, 14 May 2024 08:28:41 +0200
Subject: [PATCH 0681/1206] TosaToLinalg: Allow to skip the TOSA validation
 pass (#91742)

Allow to skip running the TOSA validation pass when spec conformance is
not required.
---
 .../mlir/Conversion/TosaToLinalg/TosaToLinalg.h    |  5 +++--
 .../Conversion/TosaToLinalg/TosaToLinalgPass.cpp   | 14 ++++++++------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h b/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h
index 5fd77c8a0211..67965e34d8a3 100644
--- a/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h
+++ b/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h
@@ -38,8 +38,9 @@ void addTosaToLinalgPasses(
     const TosaToLinalgNamedOptions &tosaToLinalgNamedOptions =
         TosaToLinalgNamedOptions(),
     // Note: Default to 'none' level unless otherwise specified.
-    tosa::TosaValidationOptions const &validationOptions = {
-        tosa::TosaProfileEnum::Undefined, false, tosa::TosaLevelEnum::None});
+    std::optional<tosa::TosaValidationOptions> validationOptions =
+        tosa::TosaValidationOptions{tosa::TosaProfileEnum::Undefined, false,
+                                    tosa::TosaLevelEnum::None});
 
 /// Populates TOSA to linalg pipelines
 /// Currently, this includes only the "tosa-to-linalg-pipeline".
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
index ad7f6cf84e5e..8904e3253922 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
@@ -78,7 +78,7 @@ std::unique_ptr<Pass> mlir::tosa::createTosaToLinalg() {
 void mlir::tosa::addTosaToLinalgPasses(
     OpPassManager &pm, const TosaToLinalgOptions &options,
     const TosaToLinalgNamedOptions &tosaToLinalgNamedOptions,
-    tosa::TosaValidationOptions const &validationOptions) {
+    std::optional<tosa::TosaValidationOptions> validationOptions) {
   // Optional decompositions are designed to benefit linalg.
   if (!options.disableTosaDecompositions)
     pm.addNestedPass<func::FuncOp>(tosa::createTosaOptionalDecompositions());
@@ -93,7 +93,8 @@ void mlir::tosa::addTosaToLinalgPasses(
   pm.addNestedPass<func::FuncOp>(tosa::createTosaLayerwiseConstantFoldPass(
       {options.aggressiveReduceConstant}));
   pm.addNestedPass<func::FuncOp>(tosa::createTosaMakeBroadcastablePass());
-  pm.addPass(tosa::createTosaValidation(validationOptions));
+  if (validationOptions)
+    pm.addPass(tosa::createTosaValidation(*validationOptions));
   pm.addNestedPass<func::FuncOp>(tosa::createTosaToLinalg());
 }
 
@@ -110,11 +111,12 @@ void mlir::tosa::registerTosaToLinalgPipelines() {
       [](OpPassManager &pm) {
         TosaToLinalgOptions tosaToLinalgOptions;
         TosaToLinalgNamedOptions tosaToLinalgNamedOptions;
+        TosaValidationOptions validationOptions;
+        validationOptions.profile = tosa::TosaProfileEnum::BaseInference;
+        validationOptions.StrictOperationSpecAlignment = true;
+        validationOptions.level = tosa::TosaLevelEnum::EightK;
         tosa::addTosaToLinalgPasses(pm, tosaToLinalgOptions,
                                     tosaToLinalgNamedOptions,
-                                    /* validationOptions = */
-                                    {tosa::TosaProfileEnum::BaseInference,
-                                     /* StrictOperationSpecAlignment = */ true,
-                                     tosa::TosaLevelEnum::EightK});
+                                    validationOptions);
       });
 }
-- 
GitLab


From 4014e2e045f5160ce9cbb9562d151f540d61c0bb Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Tue, 14 May 2024 07:19:40 +0100
Subject: [PATCH 0682/1206] [TableGen][RISCV] Strip experimental- prefix in
 profile names in SupportedExperimentalProfiles

This matches what we do for extensions, and saves us having to do it in
RISCVISAInfo.

This is a minor tweak to what I added in #91993.
---
 llvm/test/TableGen/riscv-target-def.td        | 2 +-
 llvm/utils/TableGen/RISCVTargetDefEmitter.cpp | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/test/TableGen/riscv-target-def.td b/llvm/test/TableGen/riscv-target-def.td
index e328a6e3b583..fb58448d7ce8 100644
--- a/llvm/test/TableGen/riscv-target-def.td
+++ b/llvm/test/TableGen/riscv-target-def.td
@@ -147,7 +147,7 @@ def ROCKET : RISCVTuneProcessorModel<"rocket",
 // CHECK-NEXT: };
 
 // CHECK:      static constexpr RISCVProfile SupportedExperimentalProfiles[] = {
-// CHECK-NEXT:     {"experimental-rvi99u64","rv64i2p1"},
+// CHECK-NEXT:     {"rvi99u64","rv64i2p1"},
 // CHECK-NEXT: };
 
 // CHECK:      #endif // GET_SUPPORTED_PROFILES
diff --git a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
index d174b1961344..b76ba05954aa 100644
--- a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
@@ -134,7 +134,9 @@ static void printProfileTable(raw_ostream &OS,
     if (Rec->getValueAsBit("Experimental") != Experimental)
       continue;
 
-    OS.indent(4) << "{\"" << Rec->getValueAsString("Name") << "\",\"";
+    StringRef Name = Rec->getValueAsString("Name");
+    Name.consume_front("experimental-");
+    OS.indent(4) << "{\"" << Name << "\",\"";
     printMArch(OS, Rec->getValueAsListOfDefs("Implies"));
     OS << "\"},\n";
   }
-- 
GitLab


From d488a54b408046eb4286727053cd44166dcd4daa Mon Sep 17 00:00:00 2001
From: Yeting Kuo <46629943+yetingk@users.noreply.github.com>
Date: Tue, 14 May 2024 14:44:25 +0800
Subject: [PATCH 0683/1206] [RISCV] Use software guarded branch for indirect
 jump table branch. (#66762)

When Zicfilp enabled, indirect jump table branch should be a software
guarded branch.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  14 +++
 llvm/lib/Target/RISCV/RISCVISelLowering.h     |   7 ++
 llvm/lib/Target/RISCV/RISCVInstrInfo.td       |  11 +-
 llvm/lib/Target/RISCV/RISCVRegisterInfo.td    |   2 +
 .../test/CodeGen/RISCV/jumptable-swguarded.ll | 105 ++++++++++++++++++
 5 files changed, 138 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/RISCV/jumptable-swguarded.ll

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 5a84ad4d436b..d3e8a86f8766 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -21660,6 +21660,20 @@ MCPhysReg RVVArgDispatcher::getNextPhysReg() {
   return AllocatedPhysRegs[CurIdx++];
 }
 
+SDValue RISCVTargetLowering::expandIndirectJTBranch(const SDLoc &dl,
+                                                    SDValue Value, SDValue Addr,
+                                                    int JTI,
+                                                    SelectionDAG &DAG) const {
+  if (Subtarget.hasStdExtZicfilp()) {
+    // When Zicfilp enabled, we need to use software guarded branch for jump
+    // table branch.
+    SDValue JTInfo = DAG.getJumpTableDebugInfo(JTI, Value, dl);
+    return DAG.getNode(RISCVISD::SW_GUARDED_BRIND, dl, MVT::Other, JTInfo,
+                       Addr);
+  }
+  return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, JTI, DAG);
+}
+
 namespace llvm::RISCVVIntrinsicsTable {
 
 #define GET_RISCVVIntrinsicsTable_IMPL
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 78f99e70c083..afc317f94dae 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -400,6 +400,10 @@ enum NodeType : unsigned {
   CZERO_EQZ, // vt.maskc for XVentanaCondOps.
   CZERO_NEZ, // vt.maskcn for XVentanaCondOps.
 
+  /// Software guarded BRIND node. Operand 0 is the chain operand and
+  /// operand 1 is the target address.
+  SW_GUARDED_BRIND,
+
   // FP to 32 bit int conversions for RV64. These are used to keep track of the
   // result being sign extended to 64 bit. These saturate out of range inputs.
   STRICT_FCVT_W_RV64 = ISD::FIRST_TARGET_STRICTFP_OPCODE,
@@ -869,6 +873,9 @@ public:
 
   bool supportKCFIBundles() const override { return true; }
 
+  SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr,
+                                 int JTI, SelectionDAG &DAG) const override;
+
   MachineInstr *EmitKCFICheck(MachineBasicBlock &MBB,
                               MachineBasicBlock::instr_iterator &MBBI,
                               const TargetInstrInfo *TII) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index b867eccf4266..9d574edb4e6d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -69,6 +69,8 @@ def riscv_brcc      : SDNode<"RISCVISD::BR_CC", SDT_RISCVBrCC,
 def riscv_tail      : SDNode<"RISCVISD::TAIL", SDT_RISCVCall,
                              [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
                               SDNPVariadic]>;
+def riscv_sw_guarded_brind : SDNode<"RISCVISD::SW_GUARDED_BRIND",
+                                    SDTBrind, [SDNPHasChain]>;
 def riscv_sllw      : SDNode<"RISCVISD::SLLW", SDT_RISCVIntBinOpW>;
 def riscv_sraw      : SDNode<"RISCVISD::SRAW", SDT_RISCVIntBinOpW>;
 def riscv_srlw      : SDNode<"RISCVISD::SRLW", SDT_RISCVIntBinOpW>;
@@ -1454,9 +1456,12 @@ def PseudoBRIND : Pseudo<(outs), (ins GPRJALR:$rs1, simm12:$imm12), []>,
                   PseudoInstExpansion<(JALR X0, GPR:$rs1, simm12:$imm12)>;
 
 let Predicates = [HasStdExtZicfilp],
-    isBarrier = 1, isBranch = 1, isIndirectBranch = 1, isTerminator = 1 in
+    isBarrier = 1, isBranch = 1, isIndirectBranch = 1, isTerminator = 1 in {
 def PseudoBRINDNonX7 : Pseudo<(outs), (ins GPRJALRNonX7:$rs1, simm12:$imm12), []>,
                        PseudoInstExpansion<(JALR X0, GPR:$rs1, simm12:$imm12)>;
+def PseudoBRINDX7 : Pseudo<(outs), (ins GPRX7:$rs1, simm12:$imm12), []>,
+                    PseudoInstExpansion<(JALR X0, GPR:$rs1, simm12:$imm12)>;
+}
 
 // For Zicfilp, need to avoid using X7/T2 for indirect branches which need
 // landing pad.
@@ -1464,6 +1469,10 @@ let Predicates = [HasStdExtZicfilp] in {
 def : Pat<(brind GPRJALRNonX7:$rs1), (PseudoBRINDNonX7 GPRJALRNonX7:$rs1, 0)>;
 def : Pat<(brind (add GPRJALRNonX7:$rs1, simm12:$imm12)),
           (PseudoBRINDNonX7 GPRJALRNonX7:$rs1, simm12:$imm12)>;
+
+def : Pat<(riscv_sw_guarded_brind GPRX7:$rs1), (PseudoBRINDX7 GPRX7:$rs1, 0)>;
+def : Pat<(riscv_sw_guarded_brind (add GPRX7:$rs1, simm12:$imm12)),
+          (PseudoBRINDX7 GPRX7:$rs1, simm12:$imm12)>;
 }
 
 let Predicates = [NoStdExtZicfilp] in {
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
index 90e62dc39e6a..b12634c24622 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -167,6 +167,8 @@ def GPRNoX0 : GPRRegisterClass<(sub GPR, X0)>;
 
 def GPRNoX0X2 : GPRRegisterClass<(sub GPR, X0, X2)>;
 
+def GPRX7 : GPRRegisterClass<(add X7)>;
+
 // Don't use X1 or X5 for JALR since that is a hint to pop the return address
 // stack on some microarchitectures. Also remove the reserved registers X0, X2,
 // X3, and X4 as it reduces the number of register classes that get synthesized
diff --git a/llvm/test/CodeGen/RISCV/jumptable-swguarded.ll b/llvm/test/CodeGen/RISCV/jumptable-swguarded.ll
new file mode 100644
index 000000000000..9d57ca74cd78
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/jumptable-swguarded.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple riscv32 -mattr=+experimental-zicfilp < %s | FileCheck %s
+; RUN: llc -mtriple riscv64 -mattr=+experimental-zicfilp < %s | FileCheck %s
+; RUN: llc -mtriple riscv32 < %s | FileCheck %s --check-prefix=NO-ZICFILP
+; RUN: llc -mtriple riscv64 < %s | FileCheck %s --check-prefix=NO-ZICFILP
+
+; Test using t2 to jump table branch.
+define void @above_threshold(i32 signext %in, ptr %out) nounwind {
+; CHECK-LABEL: above_threshold:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    li a2, 5
+; CHECK-NEXT:    bltu a2, a0, .LBB0_9
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    lui a2, %hi(.LJTI0_0)
+; CHECK-NEXT:    addi a2, a2, %lo(.LJTI0_0)
+; CHECK-NEXT:    add a0, a0, a2
+; CHECK-NEXT:    lw t2, 0(a0)
+; CHECK-NEXT:    jr t2
+; CHECK-NEXT:  .LBB0_2: # %bb1
+; CHECK-NEXT:    li a0, 4
+; CHECK-NEXT:    j .LBB0_8
+; CHECK-NEXT:  .LBB0_3: # %bb5
+; CHECK-NEXT:    li a0, 100
+; CHECK-NEXT:    j .LBB0_8
+; CHECK-NEXT:  .LBB0_4: # %bb3
+; CHECK-NEXT:    li a0, 2
+; CHECK-NEXT:    j .LBB0_8
+; CHECK-NEXT:  .LBB0_5: # %bb4
+; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    j .LBB0_8
+; CHECK-NEXT:  .LBB0_6: # %bb2
+; CHECK-NEXT:    li a0, 3
+; CHECK-NEXT:    j .LBB0_8
+; CHECK-NEXT:  .LBB0_7: # %bb6
+; CHECK-NEXT:    li a0, 200
+; CHECK-NEXT:  .LBB0_8: # %exit
+; CHECK-NEXT:    sw a0, 0(a1)
+; CHECK-NEXT:  .LBB0_9: # %exit
+; CHECK-NEXT:    ret
+;
+; NO-ZICFILP-LABEL: above_threshold:
+; NO-ZICFILP:       # %bb.0: # %entry
+; NO-ZICFILP-NEXT:    addi a0, a0, -1
+; NO-ZICFILP-NEXT:    li a2, 5
+; NO-ZICFILP-NEXT:    bltu a2, a0, .LBB0_9
+; NO-ZICFILP-NEXT:  # %bb.1: # %entry
+; NO-ZICFILP-NEXT:    slli a0, a0, 2
+; NO-ZICFILP-NEXT:    lui a2, %hi(.LJTI0_0)
+; NO-ZICFILP-NEXT:    addi a2, a2, %lo(.LJTI0_0)
+; NO-ZICFILP-NEXT:    add a0, a0, a2
+; NO-ZICFILP-NEXT:    lw a0, 0(a0)
+; NO-ZICFILP-NEXT:    jr a0
+; NO-ZICFILP-NEXT:  .LBB0_2: # %bb1
+; NO-ZICFILP-NEXT:    li a0, 4
+; NO-ZICFILP-NEXT:    j .LBB0_8
+; NO-ZICFILP-NEXT:  .LBB0_3: # %bb5
+; NO-ZICFILP-NEXT:    li a0, 100
+; NO-ZICFILP-NEXT:    j .LBB0_8
+; NO-ZICFILP-NEXT:  .LBB0_4: # %bb3
+; NO-ZICFILP-NEXT:    li a0, 2
+; NO-ZICFILP-NEXT:    j .LBB0_8
+; NO-ZICFILP-NEXT:  .LBB0_5: # %bb4
+; NO-ZICFILP-NEXT:    li a0, 1
+; NO-ZICFILP-NEXT:    j .LBB0_8
+; NO-ZICFILP-NEXT:  .LBB0_6: # %bb2
+; NO-ZICFILP-NEXT:    li a0, 3
+; NO-ZICFILP-NEXT:    j .LBB0_8
+; NO-ZICFILP-NEXT:  .LBB0_7: # %bb6
+; NO-ZICFILP-NEXT:    li a0, 200
+; NO-ZICFILP-NEXT:  .LBB0_8: # %exit
+; NO-ZICFILP-NEXT:    sw a0, 0(a1)
+; NO-ZICFILP-NEXT:  .LBB0_9: # %exit
+; NO-ZICFILP-NEXT:    ret
+entry:
+  switch i32 %in, label %exit [
+    i32 1, label %bb1
+    i32 2, label %bb2
+    i32 3, label %bb3
+    i32 4, label %bb4
+    i32 5, label %bb5
+    i32 6, label %bb6
+  ]
+bb1:
+  store i32 4, ptr %out
+  br label %exit
+bb2:
+  store i32 3, ptr %out
+  br label %exit
+bb3:
+  store i32 2, ptr %out
+  br label %exit
+bb4:
+  store i32 1, ptr %out
+  br label %exit
+bb5:
+  store i32 100, ptr %out
+  br label %exit
+bb6:
+  store i32 200, ptr %out
+  br label %exit
+exit:
+  ret void
+}
-- 
GitLab


From 9837a1cb53e94609c005ed44f937a99f24208452 Mon Sep 17 00:00:00 2001
From: Jacob Lambert <jacob.lambert@amd.com>
Date: Mon, 13 May 2024 23:50:31 -0700
Subject: [PATCH 0684/1206] [NFC] Add missing spaces in BoolOption for apinotes
 (#92027)

---
 clang/include/clang/Driver/Options.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index ed3f1b8b2981..c9d8a1f50fec 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1858,13 +1858,13 @@ defm apinotes : BoolOption<"f", "apinotes",
     LangOpts<"APINotes">, DefaultFalse,
     PosFlag<SetTrue, [], [ClangOption], "Enable">,
     NegFlag<SetFalse, [], [ClangOption], "Disable">,
-    BothFlags<[], [ClangOption, CC1Option], "external API notes support">>,
+    BothFlags<[], [ClangOption, CC1Option], " external API notes support">>,
     Group<f_clang_Group>;
 defm apinotes_modules : BoolOption<"f", "apinotes-modules",
     LangOpts<"APINotesModules">, DefaultFalse,
     PosFlag<SetTrue, [], [ClangOption], "Enable">,
     NegFlag<SetFalse, [], [ClangOption], "Disable">,
-    BothFlags<[], [ClangOption, CC1Option], "module-based external API notes support">>,
+    BothFlags<[], [ClangOption, CC1Option], " module-based external API notes support">>,
     Group<f_clang_Group>;
 def fapinotes_swift_version : Joined<["-"], "fapinotes-swift-version=">,
   Group<f_clang_Group>, Visibility<[ClangOption, CC1Option]>,
-- 
GitLab


From c441aa51e16e2fa5f407191287f48d2b7c302ceb Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Tue, 14 May 2024 10:59:16 +0400
Subject: [PATCH 0685/1206] [lldb] Add lldbutil.install_to_target() helper
 (#91944)

It can be used in tests #91918, #91931 and such.
---
 lldb/packages/Python/lldbsuite/test/lldbutil.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/lldb/packages/Python/lldbsuite/test/lldbutil.py b/lldb/packages/Python/lldbsuite/test/lldbutil.py
index 58eb37fd742d..1ec036f885e7 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbutil.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbutil.py
@@ -1654,6 +1654,22 @@ def skip_if_library_missing(test, target, library):
     )
 
 
+def install_to_target(test, path):
+    if lldb.remote_platform:
+        filename = os.path.basename(path)
+        remote_path = append_to_process_working_directory(test, filename)
+        err = lldb.remote_platform.Install(
+            lldb.SBFileSpec(path, True), lldb.SBFileSpec(remote_path, False)
+        )
+        if err.Fail():
+            raise Exception(
+                "remote_platform.Install('%s', '%s') failed: %s"
+                % (path, remote_path, err)
+            )
+        path = remote_path
+    return path
+
+
 def read_file_on_target(test, remote):
     if lldb.remote_platform:
         local = test.getBuildArtifact("file_from_target")
-- 
GitLab


From 2df06e42d733a1f7a1cdf715894921a5bbbc2956 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm@amd.com>
Date: Tue, 14 May 2024 09:01:51 +0200
Subject: [PATCH 0686/1206] [Flang][Driver] Add -print-resource-dir command
 line flag to emit Flang's resource directory (#90886)

This should be a NFC change for all drivers, but Flang.
---
 clang/include/clang/Driver/Driver.h      |  3 +++
 clang/include/clang/Driver/Options.td    |  5 ++++-
 clang/lib/Driver/Driver.cpp              | 22 +++++++++++++++++++---
 flang/test/Driver/print-resource-dir.F90 |  4 ++++
 4 files changed, 30 insertions(+), 4 deletions(-)
 create mode 100644 flang/test/Driver/print-resource-dir.F90

diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index cc1538372d5f..084c3ffe69ae 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -747,6 +747,9 @@ private:
   /// option.
   void setDriverMode(StringRef DriverModeValue);
 
+  /// Set the resource directory, depending on which driver is being used.
+  void setResourceDirectory();
+
   /// Parse the \p Args list for LTO options and record the type of LTO
   /// compilation based on which -f(no-)?lto(=.*)? option occurs last.
   void setLTOMode(const llvm::opt::ArgList &Args);
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index c9d8a1f50fec..c54eb543d658 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5491,7 +5491,10 @@ def print_prog_name_EQ : Joined<["-", "--"], "print-prog-name=">,
   Visibility<[ClangOption, CLOption]>;
 def print_resource_dir : Flag<["-", "--"], "print-resource-dir">,
   HelpText<"Print the resource directory pathname">,
-  Visibility<[ClangOption, CLOption]>;
+  HelpTextForVariants<[FlangOption],
+                      "Print the resource directory pathname that contains lib and "
+                      "include directories with the runtime libraries and MODULE files.">,
+  Visibility<[ClangOption, CLOption, FlangOption]>;
 def print_search_dirs : Flag<["-", "--"], "print-search-dirs">,
   HelpText<"Print the paths used for finding libraries and programs">,
   Visibility<[ClangOption, CLOption]>;
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 7b36d8e5084c..2868b4f2b02e 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -229,9 +229,6 @@ Driver::Driver(StringRef ClangExecutable, StringRef TargetTriple,
     UserConfigDir = static_cast<std::string>(P);
   }
 #endif
-
-  // Compute the path to the resource directory.
-  ResourceDir = GetResourcesPath(ClangExecutable, CLANG_RESOURCE_DIR);
 }
 
 void Driver::setDriverMode(StringRef Value) {
@@ -250,6 +247,24 @@ void Driver::setDriverMode(StringRef Value) {
     Diag(diag::err_drv_unsupported_option_argument) << OptName << Value;
 }
 
+void Driver::setResourceDirectory() {
+  // Compute the path to the resource directory, depending on the driver mode.
+  switch (Mode) {
+  case GCCMode:
+  case GXXMode:
+  case CPPMode:
+  case CLMode:
+  case DXCMode:
+    ResourceDir = GetResourcesPath(ClangExecutable, CLANG_RESOURCE_DIR);
+    break;
+  case FlangMode:
+    SmallString<64> customResourcePathRelativeToDriver{".."};
+    ResourceDir =
+        GetResourcesPath(ClangExecutable, customResourcePathRelativeToDriver);
+    break;
+  }
+}
+
 InputArgList Driver::ParseArgStrings(ArrayRef<const char *> ArgStrings,
                                      bool UseDriverMode, bool &ContainsError) {
   llvm::PrettyStackTraceString CrashInfo("Command line argument parsing");
@@ -1202,6 +1217,7 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
   if (!DriverMode.empty())
     setDriverMode(DriverMode);
 
+  setResourceDirectory();
   // FIXME: What are we going to do with -V and -b?
 
   // Arguments specified in command line.
diff --git a/flang/test/Driver/print-resource-dir.F90 b/flang/test/Driver/print-resource-dir.F90
new file mode 100644
index 000000000000..8fd35f1800df
--- /dev/null
+++ b/flang/test/Driver/print-resource-dir.F90
@@ -0,0 +1,4 @@
+! DEFINE: %{resource_dir} = %S/Inputs/resource_dir
+! RUN: %flang -print-resource-dir -resource-dir=%{resource_dir}.. \
+! RUN:  | FileCheck -check-prefix=PRINT-RESOURCE-DIR -DFILE=%{resource_dir} %s
+! PRINT-RESOURCE-DIR: [[FILE]]
-- 
GitLab


From 3ae63430aae52b260ce7ea99e5d586c77963b94a Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy@intel.com>
Date: Tue, 14 May 2024 09:22:24 +0200
Subject: [PATCH 0687/1206] [SPIR-V] Set non-kernel function linkage type via
 OpDecorate for all linkage types except for static functions (#91598)

This PR fixes the issue
https://github.com/llvm/llvm-project/issues/91595 by setting non-kernel
function linkage type via OpDecorate for all linkage types except for
static functions. A new test case is added.
---
 llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp      |  4 ++--
 .../CodeGen/SPIRV/{ => linkage}/LinkOnceODR.ll   |  0
 .../SPIRV/{ => linkage}/LinkOnceODRFun.ll        |  0
 .../CodeGen/SPIRV/linkage/extern-weak-linkage.ll | 16 ++++++++++++++++
 .../SPIRV/{ => linkage}/link-attribute.ll        |  0
 .../CodeGen/SPIRV/{ => linkage}/linkage-types.ll |  0
 6 files changed, 18 insertions(+), 2 deletions(-)
 rename llvm/test/CodeGen/SPIRV/{ => linkage}/LinkOnceODR.ll (100%)
 rename llvm/test/CodeGen/SPIRV/{ => linkage}/LinkOnceODRFun.ll (100%)
 create mode 100644 llvm/test/CodeGen/SPIRV/linkage/extern-weak-linkage.ll
 rename llvm/test/CodeGen/SPIRV/{ => linkage}/link-attribute.ll (100%)
 rename llvm/test/CodeGen/SPIRV/{ => linkage}/linkage-types.ll (100%)

diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index c107b99cf4cb..727e4e584c05 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -430,8 +430,8 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
                    .addImm(static_cast<uint32_t>(executionModel))
                    .addUse(FuncVReg);
     addStringImm(F.getName(), MIB);
-  } else if (F.getLinkage() == GlobalValue::LinkageTypes::ExternalLinkage ||
-             F.getLinkage() == GlobalValue::LinkOnceODRLinkage) {
+  } else if (F.getLinkage() != GlobalValue::InternalLinkage &&
+             F.getLinkage() != GlobalValue::PrivateLinkage) {
     SPIRV::LinkageType::LinkageType LnkTy =
         F.isDeclaration()
             ? SPIRV::LinkageType::Import
diff --git a/llvm/test/CodeGen/SPIRV/LinkOnceODR.ll b/llvm/test/CodeGen/SPIRV/linkage/LinkOnceODR.ll
similarity index 100%
rename from llvm/test/CodeGen/SPIRV/LinkOnceODR.ll
rename to llvm/test/CodeGen/SPIRV/linkage/LinkOnceODR.ll
diff --git a/llvm/test/CodeGen/SPIRV/LinkOnceODRFun.ll b/llvm/test/CodeGen/SPIRV/linkage/LinkOnceODRFun.ll
similarity index 100%
rename from llvm/test/CodeGen/SPIRV/LinkOnceODRFun.ll
rename to llvm/test/CodeGen/SPIRV/linkage/LinkOnceODRFun.ll
diff --git a/llvm/test/CodeGen/SPIRV/linkage/extern-weak-linkage.ll b/llvm/test/CodeGen/SPIRV/linkage/extern-weak-linkage.ll
new file mode 100644
index 000000000000..e742de4bc1e2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/linkage/extern-weak-linkage.ll
@@ -0,0 +1,16 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-SPIRV: Capability Linkage
+; CHECK-SPIRV-DAG: OpName %[[#AbsFun:]] "abs"
+; CHECK-SPIRV-DAG: OpName %[[#ExternalFun:]] "__devicelib_abs"
+; CHECK-SPIRV-DAG: OpDecorate %[[#AbsFun]] LinkageAttributes "abs" Export
+; CHECK-SPIRV-DAG: OpDecorate %[[#ExternalFun]] LinkageAttributes "__devicelib_abs" Import
+
+define weak dso_local spir_func i32 @abs(i32 noundef %x) {
+entry:
+  %call = tail call spir_func i32 @__devicelib_abs(i32 noundef %x) #11
+  ret i32 %call
+}
+
+declare extern_weak dso_local spir_func i32 @__devicelib_abs(i32 noundef)
diff --git a/llvm/test/CodeGen/SPIRV/link-attribute.ll b/llvm/test/CodeGen/SPIRV/linkage/link-attribute.ll
similarity index 100%
rename from llvm/test/CodeGen/SPIRV/link-attribute.ll
rename to llvm/test/CodeGen/SPIRV/linkage/link-attribute.ll
diff --git a/llvm/test/CodeGen/SPIRV/linkage-types.ll b/llvm/test/CodeGen/SPIRV/linkage/linkage-types.ll
similarity index 100%
rename from llvm/test/CodeGen/SPIRV/linkage-types.ll
rename to llvm/test/CodeGen/SPIRV/linkage/linkage-types.ll
-- 
GitLab


From cf9a5a162b701b4c27eda1ddf823137ed16ca235 Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy@intel.com>
Date: Tue, 14 May 2024 09:22:37 +0200
Subject: [PATCH 0688/1206] [SPIR-V] Support saturation arithmetic intrinsics
 in SPIR-V Backend (#91722)

This PR is to support saturation arithmetic intrinsics in SPIR-V
Backend.
---
 .../Target/SPIRV/SPIRVInstructionSelector.cpp |  9 +++++
 llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp  |  4 ++
 .../SPIRV/llvm-intrinsics/satur-arith.ll      | 37 +++++++++++++++++++
 3 files changed, 50 insertions(+)
 create mode 100644 llvm/test/CodeGen/SPIRV/llvm-intrinsics/satur-arith.ll

diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 2051cdc7e01f..517a9b490eba 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -495,6 +495,15 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
   case TargetOpcode::G_UMULH:
     return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi);
 
+  case TargetOpcode::G_SADDSAT:
+    return selectExtInst(ResVReg, ResType, I, CL::s_add_sat);
+  case TargetOpcode::G_UADDSAT:
+    return selectExtInst(ResVReg, ResType, I, CL::u_add_sat);
+  case TargetOpcode::G_SSUBSAT:
+    return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat);
+  case TargetOpcode::G_USUBSAT:
+    return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat);
+
   case TargetOpcode::G_SEXT:
     return selectExt(ResVReg, ResType, I, true);
   case TargetOpcode::G_ANYEXT:
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index e7b35555293a..42d36fd30ed6 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -304,6 +304,10 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
 
     // Struct return types become a single scalar, so cannot easily legalize.
     getActionDefinitionsBuilder({G_SMULH, G_UMULH}).alwaysLegal();
+
+    // supported saturation arithmetic
+    getActionDefinitionsBuilder({G_SADDSAT, G_UADDSAT, G_SSUBSAT, G_USUBSAT})
+        .legalFor(allIntScalarsAndVectors);
   }
 
   getLegacyLegalizerInfo().computeTables();
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/satur-arith.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/satur-arith.ll
new file mode 100644
index 000000000000..5b59206ff7f2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/satur-arith.ll
@@ -0,0 +1,37 @@
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpExtInstImport "OpenCL.std"
+; CHECK-DAG: OpName %[[#Foo:]] "foo"
+; CHECK-DAG: OpName %[[#Bar:]] "bar"
+; CHECK: %[[#Foo]] = OpFunction
+; CHECK: %[[#]] = OpExtInst %[[#]] %[[#]] u_add_sat
+; CHECK-NEXT: %[[#]] = OpExtInst %[[#]] %[[#]] u_sub_sat
+; CHECK-NEXT: %[[#]] = OpExtInst %[[#]] %[[#]] s_add_sat
+; CHECK-NEXT: %[[#]] = OpExtInst %[[#]] %[[#]] s_sub_sat
+; CHECK: %[[#Bar]] = OpFunction
+; CHECK: %[[#]] = OpExtInst %[[#]] %[[#]] u_add_sat
+; CHECK-NEXT: %[[#]] = OpExtInst %[[#]] %[[#]] u_sub_sat
+; CHECK-NEXT: %[[#]] = OpExtInst %[[#]] %[[#]] s_add_sat
+; CHECK-NEXT: %[[#]] = OpExtInst %[[#]] %[[#]] s_sub_sat
+
+define spir_func void @foo(i16 %x, i16 %y) {
+entry:
+  %r1 = tail call i16 @llvm.uadd.sat.i16(i16 %x, i16 %y)
+  %r2 = tail call i16 @llvm.usub.sat.i16(i16 %x, i16 %y)
+  %r3 = tail call i16 @llvm.sadd.sat.i16(i16 %x, i16 %y)
+  %r4 = tail call i16 @llvm.ssub.sat.i16(i16 %x, i16 %y)
+  ret void
+}
+
+define spir_func void @bar(<4 x i32> %x, <4 x i32> %y) {
+entry:
+  %r1 = tail call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %r2 = tail call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %r3 = tail call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %r4 = tail call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  ret void
+}
-- 
GitLab


From e2f079cc6c3689fa5a6f64550b2d4fdc628dad6f Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Tue, 14 May 2024 11:26:14 +0400
Subject: [PATCH 0689/1206] [lldb] Fixed the test TestGdbRemoteLaunch (#91931)

Install `a.out` to the remote target (after handshake) if necessary and
use the remote path to call `vRun`.
---
 .../tools/lldb-server/TestGdbRemoteLaunch.py  | 41 +++++++++----------
 1 file changed, 20 insertions(+), 21 deletions(-)

diff --git a/lldb/test/API/tools/lldb-server/TestGdbRemoteLaunch.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteLaunch.py
index 78a4d326c12d..ad84a40932c6 100644
--- a/lldb/test/API/tools/lldb-server/TestGdbRemoteLaunch.py
+++ b/lldb/test/API/tools/lldb-server/TestGdbRemoteLaunch.py
@@ -12,13 +12,13 @@ class GdbRemoteLaunchTestCase(gdbremote_testcase.GdbRemoteTestCaseBase):
     @add_test_categories(["llgs"])
     def test_launch_via_A(self):
         self.build()
-        exe_path = self.getBuildArtifact("a.out")
-        args = [exe_path, "stderr:arg1", "stderr:arg2", "stderr:arg3"]
-        hex_args = [seven.hexlify(x) for x in args]
-
         server = self.connect_to_debug_monitor()
         self.assertIsNotNone(server)
         self.do_handshake()
+        exe_path = lldbutil.install_to_target(self, self.getBuildArtifact("a.out"))
+        args = [exe_path, "stderr:arg1", "stderr:arg2", "stderr:arg3"]
+        hex_args = [seven.hexlify(x) for x in args]
+
         # NB: strictly speaking we should use %x here but this packet
         # is deprecated, so no point in changing lldb-server's expectations
         self.test_sequence.add_log_lines(
@@ -38,13 +38,13 @@ class GdbRemoteLaunchTestCase(gdbremote_testcase.GdbRemoteTestCaseBase):
     @add_test_categories(["llgs"])
     def test_launch_via_vRun(self):
         self.build()
-        exe_path = self.getBuildArtifact("a.out")
-        args = [exe_path, "stderr:arg1", "stderr:arg2", "stderr:arg3"]
-        hex_args = [seven.hexlify(x) for x in args]
-
         server = self.connect_to_debug_monitor()
         self.assertIsNotNone(server)
         self.do_handshake()
+        exe_path = lldbutil.install_to_target(self, self.getBuildArtifact("a.out"))
+        args = [exe_path, "stderr:arg1", "stderr:arg2", "stderr:arg3"]
+        hex_args = [seven.hexlify(x) for x in args]
+
         self.test_sequence.add_log_lines(
             [
                 "read packet: $vRun;%s;%s;%s;%s#00" % tuple(hex_args),
@@ -60,12 +60,12 @@ class GdbRemoteLaunchTestCase(gdbremote_testcase.GdbRemoteTestCaseBase):
     @add_test_categories(["llgs"])
     def test_launch_via_vRun_no_args(self):
         self.build()
-        exe_path = self.getBuildArtifact("a.out")
-        hex_path = seven.hexlify(exe_path)
-
         server = self.connect_to_debug_monitor()
         self.assertIsNotNone(server)
         self.do_handshake()
+        exe_path = lldbutil.install_to_target(self, self.getBuildArtifact("a.out"))
+        hex_path = seven.hexlify(exe_path)
+
         self.test_sequence.add_log_lines(
             [
                 "read packet: $vRun;%s#00" % (hex_path,),
@@ -78,6 +78,7 @@ class GdbRemoteLaunchTestCase(gdbremote_testcase.GdbRemoteTestCaseBase):
         self.expect_gdbremote_sequence()
 
     @add_test_categories(["llgs"])
+    @skipIfRemote
     def test_launch_failure_via_vRun(self):
         self.build()
         exe_path = self.getBuildArtifact("a.out")
@@ -110,14 +111,13 @@ class GdbRemoteLaunchTestCase(gdbremote_testcase.GdbRemoteTestCaseBase):
     @add_test_categories(["llgs"])
     def test_QEnvironment(self):
         self.build()
-        exe_path = self.getBuildArtifact("a.out")
-        env = {"FOO": "test", "BAR": "a=z"}
-        args = [exe_path, "print-env:FOO", "print-env:BAR"]
-        hex_args = [seven.hexlify(x) for x in args]
-
         server = self.connect_to_debug_monitor()
         self.assertIsNotNone(server)
         self.do_handshake()
+        exe_path = lldbutil.install_to_target(self, self.getBuildArtifact("a.out"))
+        env = {"FOO": "test", "BAR": "a=z"}
+        args = [exe_path, "print-env:FOO", "print-env:BAR"]
+        hex_args = [seven.hexlify(x) for x in args]
 
         for key, value in env.items():
             self.test_sequence.add_log_lines(
@@ -143,14 +143,13 @@ class GdbRemoteLaunchTestCase(gdbremote_testcase.GdbRemoteTestCaseBase):
     @add_test_categories(["llgs"])
     def test_QEnvironmentHexEncoded(self):
         self.build()
-        exe_path = self.getBuildArtifact("a.out")
-        env = {"FOO": "test", "BAR": "a=z", "BAZ": "a*}#z"}
-        args = [exe_path, "print-env:FOO", "print-env:BAR", "print-env:BAZ"]
-        hex_args = [seven.hexlify(x) for x in args]
-
         server = self.connect_to_debug_monitor()
         self.assertIsNotNone(server)
         self.do_handshake()
+        exe_path = lldbutil.install_to_target(self, self.getBuildArtifact("a.out"))
+        env = {"FOO": "test", "BAR": "a=z", "BAZ": "a*}#z"}
+        args = [exe_path, "print-env:FOO", "print-env:BAR", "print-env:BAZ"]
+        hex_args = [seven.hexlify(x) for x in args]
 
         for key, value in env.items():
             hex_enc = seven.hexlify("%s=%s" % (key, value))
-- 
GitLab


From 2e165a2c4b2a0e9a9d34a721d756f9006d1502df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Tue, 14 May 2024 10:25:38 +0300
Subject: [PATCH 0690/1206] Revert "[ValueTracking] Compute knownbits from
 known fp classes (#86409)"

This reverts commit d03a1a6e5838c7c2c0836d71507dfdf7840ade49.

This change caused failed assertions, see
https://github.com/llvm/llvm-project/pull/86409#issuecomment-2109469845
for details.
---
 llvm/include/llvm/IR/PatternMatch.h           |   2 +-
 llvm/lib/Analysis/ValueTracking.cpp           |  36 ---
 .../AMDGPU/amdgpu-simplify-libcall-pow.ll     |  14 +-
 .../AMDGPU/amdgpu-simplify-libcall-pown.ll    |  12 +-
 llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll |  10 +-
 .../test/Transforms/InstCombine/known-bits.ll | 264 ------------------
 6 files changed, 19 insertions(+), 319 deletions(-)

diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 0d6d86cb47e6..171ddab977de 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -1904,7 +1904,7 @@ template <typename Op_t> struct ElementWiseBitCast_match {
   ElementWiseBitCast_match(const Op_t &OpMatch) : Op(OpMatch) {}
 
   template <typename OpTy> bool match(OpTy *V) {
-    auto *I = dyn_cast<BitCastInst>(V);
+    BitCastInst *I = dyn_cast<BitCastInst>(V);
     if (!I)
       return false;
     Type *SrcType = I->getSrcTy();
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 2fdbb6e3ef84..375385aca7a3 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1118,42 +1118,6 @@ static void computeKnownBitsFromOperator(const Operator *I,
       break;
     }
 
-    const Value *V;
-    // Handle bitcast from floating point to integer.
-    if (match(I, m_ElementWiseBitCast(m_Value(V))) &&
-        V->getType()->isFPOrFPVectorTy()) {
-      Type *FPType = V->getType()->getScalarType();
-      KnownFPClass Result = computeKnownFPClass(V, fcAllFlags, Depth + 1, Q);
-      FPClassTest FPClasses = Result.KnownFPClasses;
-
-      if (Result.isKnownNever(fcNormal | fcSubnormal | fcNan)) {
-        Known.Zero.setAllBits();
-        Known.One.setAllBits();
-
-        if (FPClasses & fcInf)
-          Known = Known.intersectWith(KnownBits::makeConstant(
-              APFloat::getInf(FPType->getFltSemantics()).bitcastToAPInt()));
-
-        if (FPClasses & fcZero)
-          Known = Known.intersectWith(KnownBits::makeConstant(
-              APInt::getZero(FPType->getScalarSizeInBits())));
-      }
-
-      if (Result.SignBit) {
-        if (*Result.SignBit)
-          Known.makeNegative();
-        else
-          Known.makeNonNegative();
-      } else {
-        Known.Zero.clearSignBit();
-        Known.One.clearSignBit();
-      }
-
-      assert(!Known.hasConflict() && "Bits known to be one AND zero?");
-
-      break;
-    }
-
     // Handle cast from vector integer type to scalar or vector integer.
     auto *SrcVecTy = dyn_cast<FixedVectorType>(SrcTy);
     if (!SrcVecTy || !SrcVecTy->getElementType()->isIntegerTy() ||
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
index 5db25a59d33f..c4bd4bc126f7 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
@@ -2216,7 +2216,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_sitofp(float %x, i32 %y)
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2304,7 +2304,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_uitofp(float %x, i32 %y)
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2353,7 +2353,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_uitofp_i256(float %x, i2
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2376,7 +2376,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_sitofp_i256(float %x, i2
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2399,7 +2399,7 @@ define <2 x float> @test_pow_afn_nnan_ninf_v2f32_known_integral_sitofp(<2 x floa
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x float> [[X]] to <2 x i32>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i32> [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x float> [[__EXP2]] to <2 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint <2 x i32> [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i32> [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP4]] to <2 x float>
 ; CHECK-NEXT:    ret <2 x float> [[TMP5]]
 ;
@@ -2448,7 +2448,7 @@ define <2 x float> @test_pow_afn_nnan_ninf_v2f32_known_integral_uitofp(<2 x floa
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x float> [[X]] to <2 x i32>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i32> [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x float> [[__EXP2]] to <2 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint <2 x i32> [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i32> [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP4]] to <2 x float>
 ; CHECK-NEXT:    ret <2 x float> [[TMP5]]
 ;
@@ -2560,7 +2560,7 @@ define float @test_pow_afn_f32_nnan_ninf__y_known_integral_trunc(float %x, float
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll
index e298226ee7cc..8ddaf243db92 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll
@@ -680,7 +680,7 @@ define float @test_pown_afn_nnan_ninf_f32(float %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
@@ -703,7 +703,7 @@ define <2 x float> @test_pown_afn_nnan_ninf_v2f32(<2 x float> %x, <2 x i32> %y)
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[X]] to <2 x i32>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i32> [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x float> [[__EXP2]] to <2 x i32>
-; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint <2 x i32> [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i32> [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
 ; CHECK-NEXT:    ret <2 x float> [[TMP3]]
 ;
@@ -772,7 +772,7 @@ define half @test_pown_afn_nnan_ninf_f16(half %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast half [[X]] to i16
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i16 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[__EXP2]] to i16
-; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i16 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i16 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP2]] to half
 ; CHECK-NEXT:    ret half [[TMP3]]
 ;
@@ -795,7 +795,7 @@ define <2 x half> @test_pown_afn_nnan_ninf_v2f16(<2 x half> %x, <2 x i32> %y) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x half> [[X]] to <2 x i16>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i16> [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x half> [[__EXP2]] to <2 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint <2 x i16> [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i16> [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i16> [[TMP2]] to <2 x half>
 ; CHECK-NEXT:    ret <2 x half> [[TMP3]]
 ;
@@ -829,7 +829,7 @@ define float @test_pown_fast_f32_strictfp(float %x, i32 %y) #1 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
@@ -1075,7 +1075,7 @@ define float @test_pown_afn_ninf_nnan_f32__x_known_positive(float nofpclass(ninf
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
index 54ca33401ccf..204c8140d3f1 100644
--- a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
+++ b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
@@ -360,7 +360,7 @@ declare half @_Z4pownDhi(half, i32)
 ; GCN-NATIVE: %0 = bitcast half %x to i16
 ; GCN-NATIVE: %__pow_sign = and i16 %__yeven, %0
 ; GCN-NATIVE: %1 = bitcast half %__exp2 to i16
-; GCN-NATIVE: %2 = or disjoint i16 %__pow_sign, %1
+; GCN-NATIVE: %2 = or i16 %__pow_sign, %1
 ; GCN-NATIVE: %3 = bitcast i16 %2 to half
 define half @test_pown_f16(half %x, i32 %y) {
 entry:
@@ -378,7 +378,7 @@ declare float @_Z4pownfi(float, i32)
 ; GCN: %[[r0:.*]] = bitcast float %tmp to i32
 ; GCN: %__pow_sign = and i32 %[[r0]], -2147483648
 ; GCN: %[[r1:.*]] = bitcast float %__exp2 to i32
-; GCN: %[[r2:.*]] = or disjoint i32 %__pow_sign, %[[r1]]
+; GCN: %[[r2:.*]] = or i32 %__pow_sign, %[[r1]]
 ; GCN: store i32 %[[r2]], ptr addrspace(1) %a, align 4
 define amdgpu_kernel void @test_pow(ptr addrspace(1) nocapture %a) {
 entry:
@@ -414,7 +414,7 @@ entry:
 ; GCN: %[[r0:.*]] = bitcast float %tmp to i32
 ; GCN: %__pow_sign = and i32 %__yeven, %[[r0]]
 ; GCN: %[[r1:.*]] = bitcast float %__exp2 to i32
-; GCN: %[[r2:.*]] = or disjoint i32 %__pow_sign, %[[r1]]
+; GCN: %[[r2:.*]] = or i32 %__pow_sign, %[[r1]]
 ; GCN: store i32 %[[r2]], ptr addrspace(1) %a, align 4
 define amdgpu_kernel void @test_pown(ptr addrspace(1) nocapture %a) {
 entry:
@@ -438,7 +438,7 @@ declare <2 x half> @_Z3powDv2_DhS_(<2 x half>, <2 x half>)
 ; GCN: %1 = bitcast half %x to i16
 ; GCN: %__pow_sign = and i16 %1, -32768
 ; GCN: %2 = bitcast half %__exp2 to i16
-; GCN: %3 = or disjoint i16 %__pow_sign, %2
+; GCN: %3 = or i16 %__pow_sign, %2
 ; GCN: %4 = bitcast i16 %3 to half
 define half @test_pow_fast_f16__y_13(half %x) {
   %powr = tail call fast half @_Z3powDhDh(half %x, half 13.0)
@@ -453,7 +453,7 @@ define half @test_pow_fast_f16__y_13(half %x) {
 ; GCN: %1 = bitcast <2 x half> %x to <2 x i16>
 ; GCN: %__pow_sign = and <2 x i16> %1, <i16 -32768, i16 -32768>
 ; GCN: %2 = bitcast <2 x half> %__exp2 to <2 x i16>
-; GCN: %3 = or disjoint <2 x i16> %__pow_sign, %2
+; GCN: %3 = or <2 x i16> %__pow_sign, %2
 ; GCN: %4 = bitcast <2 x i16> %3 to <2 x half>
 define <2 x half> @test_pow_fast_v2f16__y_13(<2 x half> %x) {
   %powr = tail call fast <2 x half> @_Z3powDv2_DhS_(<2 x half> %x, <2 x half> <half 13.0, half 13.0>)
diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index 816bd6f352df..8b4249b2c25a 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1374,269 +1374,5 @@ define i8 @nonzero_reduce_xor_vscale_odd(<vscale x 3 x i8> %xx) {
   ret i8 %r
 }
 
-define i1 @test_sign_pos(float %x) {
-; CHECK-LABEL: @test_sign_pos(
-; CHECK-NEXT:    ret i1 true
-;
-  %fabs = call float @llvm.fabs.f32(float %x)
-  %y = bitcast float %fabs to i32
-  %sign = icmp sgt i32 %y, -1
-  ret i1 %sign
-}
-
-define i1 @test_sign_pos_half(half %x) {
-; CHECK-LABEL: @test_sign_pos_half(
-; CHECK-NEXT:    ret i1 true
-;
-  %fabs = call half @llvm.fabs.f16(half %x)
-  %y = bitcast half %fabs to i16
-  %sign = icmp sgt i16 %y, -1
-  ret i1 %sign
-}
-
-define i1 @test_sign_pos_half_non_elementwise(<2 x half> %x) {
-; CHECK-LABEL: @test_sign_pos_half_non_elementwise(
-; CHECK-NEXT:    [[FABS:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[X:%.*]])
-; CHECK-NEXT:    [[Y:%.*]] = bitcast <2 x half> [[FABS]] to i32
-; CHECK-NEXT:    [[SIGN:%.*]] = icmp sgt i32 [[Y]], -1
-; CHECK-NEXT:    ret i1 [[SIGN]]
-;
-  %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %x)
-  %y = bitcast <2 x half> %fabs to i32
-  %sign = icmp sgt i32 %y, -1
-  ret i1 %sign
-}
-
-define i1 @test_sign_neg(float %x) {
-; CHECK-LABEL: @test_sign_neg(
-; CHECK-NEXT:    ret i1 true
-;
-  %fabs = call float @llvm.fabs.f32(float %x)
-  %fnabs = fneg float %fabs
-  %y = bitcast float %fnabs to i32
-  %sign = icmp slt i32 %y, 0
-  ret i1 %sign
-}
-
-define <2 x i1> @test_sign_pos_vec(<2 x float> %x) {
-; CHECK-LABEL: @test_sign_pos_vec(
-; CHECK-NEXT:    ret <2 x i1> zeroinitializer
-;
-  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %x)
-  %y = bitcast <2 x float> %fabs to <2 x i32>
-  %sign = icmp slt <2 x i32> %y, zeroinitializer
-  ret <2 x i1> %sign
-}
-
-define i32 @test_inf_only(float nofpclass(nan sub norm zero) %x) {
-; CHECK-LABEL: @test_inf_only(
-; CHECK-NEXT:    ret i32 2139095040
-;
-  %y = bitcast float %x to i32
-  %and = and i32 %y, 2147483647
-  ret i32 %and
-}
-
-define i16 @test_inf_only_bfloat(bfloat nofpclass(nan sub norm zero) %x) {
-; CHECK-LABEL: @test_inf_only_bfloat(
-; CHECK-NEXT:    ret i16 32640
-;
-  %y = bitcast bfloat %x to i16
-  %and = and i16 %y, 32767
-  ret i16 %and
-}
-
-define i128 @test_inf_only_ppc_fp128(ppc_fp128 nofpclass(nan sub norm zero) %x) {
-; CHECK-LABEL: @test_inf_only_ppc_fp128(
-; CHECK-NEXT:    ret i128 9218868437227405312
-;
-  %y = bitcast ppc_fp128 %x to i128
-  %and = and i128 %y, 170141183460469231731687303715884105727
-  ret i128 %and
-}
-
-define i32 @test_zero_only(float nofpclass(nan sub norm inf) %x) {
-; CHECK-LABEL: @test_zero_only(
-; CHECK-NEXT:    ret i32 0
-;
-  %y = bitcast float %x to i32
-  %and = and i32 %y, 2147483647
-  ret i32 %and
-}
-
-define i80 @test_zero_only_non_ieee(x86_fp80 nofpclass(nan sub norm inf) %x) {
-; CHECK-LABEL: @test_zero_only_non_ieee(
-; CHECK-NEXT:    ret i80 0
-;
-  %y = bitcast x86_fp80 %x to i80
-  %and = and i80 %y, 604462909807314587353087
-  ret i80 %and
-}
-
-define i32 @test_inf_nan_only(float nofpclass(sub norm zero) %x) {
-; CHECK-LABEL: @test_inf_nan_only(
-; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 2130706432
-; CHECK-NEXT:    ret i32 [[AND]]
-;
-  %y = bitcast float %x to i32
-  %and = and i32 %y, 2130706432
-  ret i32 %and
-}
-
-define i32 @test_sub_zero_only(float nofpclass(nan norm inf) %x) {
-; CHECK-LABEL: @test_sub_zero_only(
-; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 2130706432
-; CHECK-NEXT:    ret i32 [[AND]]
-;
-  %y = bitcast float %x to i32
-  %and = and i32 %y, 2130706432
-  ret i32 %and
-}
-
-define i32 @test_inf_zero_only(float nofpclass(nan norm sub) %x) {
-; CHECK-LABEL: @test_inf_zero_only(
-; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 8388608
-; CHECK-NEXT:    ret i32 [[AND]]
-;
-  %y = bitcast float %x to i32
-  %and = and i32 %y, 16777215
-  ret i32 %and
-}
-
-
-
-define i1 @test_simplify_icmp(i32 %x) {
-; CHECK-LABEL: @test_simplify_icmp(
-; CHECK-NEXT:    ret i1 false
-;
-  %cast1 = uitofp i32 %x to double
-  %cast2 = bitcast double %cast1 to i64
-  %mask = and i64 %cast2, -140737488355328
-  %cmp = icmp eq i64 %mask, -1970324836974592
-  ret i1 %cmp
-}
-
-define i32 @test_snan_quiet_bit1(float nofpclass(sub norm inf qnan) %x) {
-; CHECK-LABEL: @test_snan_quiet_bit1(
-; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
-; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 4194304
-; CHECK-NEXT:    ret i32 [[MASKED]]
-;
-  %bits = bitcast float %x to i32
-  %masked = and i32 %bits, 4194304
-  ret i32 %masked
-}
-
-define i32 @test_snan_quiet_bit2(float nofpclass(sub norm inf qnan) %x) {
-; CHECK-LABEL: @test_snan_quiet_bit2(
-; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
-; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 2097152
-; CHECK-NEXT:    ret i32 [[MASKED]]
-;
-  %bits = bitcast float %x to i32
-  %masked = and i32 %bits, 2097152
-  ret i32 %masked
-}
-
-define i32 @test_qnan_quiet_bit1(float nofpclass(sub norm inf snan) %x) {
-; CHECK-LABEL: @test_qnan_quiet_bit1(
-; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
-; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 4194304
-; CHECK-NEXT:    ret i32 [[MASKED]]
-;
-  %bits = bitcast float %x to i32
-  %masked = and i32 %bits, 4194304
-  ret i32 %masked
-}
-
-define i32 @test_qnan_quiet_bit2(float nofpclass(sub norm inf snan) %x) {
-; CHECK-LABEL: @test_qnan_quiet_bit2(
-; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
-; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 2097152
-; CHECK-NEXT:    ret i32 [[MASKED]]
-;
-  %bits = bitcast float %x to i32
-  %masked = and i32 %bits, 2097152
-  ret i32 %masked
-}
-
-define i16 @test_simplify_mask(i32 %ui, float %x) {
-; CHECK-LABEL: @test_simplify_mask(
-; CHECK-NEXT:    [[CONV:%.*]] = uitofp i32 [[UI:%.*]] to float
-; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[CONV]], [[X:%.*]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF_ELSE:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.end:
-; CHECK-NEXT:    ret i16 31744
-; CHECK:       if.else:
-; CHECK-NEXT:    ret i16 0
-;
-  %conv = uitofp i32 %ui to float
-  %cmp = fcmp olt float %x, %conv
-  br i1 %cmp, label %if.else, label %if.end
-
-if.end:
-  %cast = bitcast float %conv to i32
-  %shr = lshr i32 %cast, 16
-  %trunc = trunc i32 %shr to i16
-  %and = and i16 %trunc, -32768
-  %or = or disjoint i16 %and, 31744
-  ret i16 %or
-
-if.else:
-  ret i16 0
-}
-
-; TODO: %cmp always evaluates to false
-
-define i1 @test_simplify_icmp2(double %x) {
-; CHECK-LABEL: @test_simplify_icmp2(
-; CHECK-NEXT:    [[ABS:%.*]] = tail call double @llvm.fabs.f64(double [[X:%.*]])
-; CHECK-NEXT:    [[COND:%.*]] = fcmp oeq double [[ABS]], 0x7FF0000000000000
-; CHECK-NEXT:    br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    [[CAST:%.*]] = bitcast double [[X]] to i64
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[CAST]], 3458764513820540928
-; CHECK-NEXT:    ret i1 [[CMP]]
-; CHECK:       if.else:
-; CHECK-NEXT:    ret i1 false
-;
-  %abs = tail call double @llvm.fabs.f64(double %x)
-  %cond = fcmp oeq double %abs, 0x7FF0000000000000
-  br i1 %cond, label %if.then, label %if.else
-
-if.then:
-  %cast = bitcast double %x to i64
-  %cmp = icmp eq i64 %cast, 3458764513820540928
-  ret i1 %cmp
-
-if.else:
-  ret i1 false
-}
-
-define i32 @test_snan_only(float nofpclass(qnan sub norm zero inf) %x) {
-; CHECK-LABEL: @test_snan_only(
-; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 4194304
-; CHECK-NEXT:    ret i32 [[AND]]
-;
-  %y = bitcast float %x to i32
-  %and = and i32 %y, 4194304
-  ret i32 %and
-}
-
-define i32 @test_qnan_only(float nofpclass(snan sub norm zero inf) %x) {
-; CHECK-LABEL: @test_qnan_only(
-; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 4194304
-; CHECK-NEXT:    ret i32 [[AND]]
-;
-  %y = bitcast float %x to i32
-  %and = and i32 %y, 4194304
-  ret i32 %and
-}
-
 declare void @use(i1)
 declare void @sink(i8)
-- 
GitLab


From fcd020d561f28a2b33b6cc12a5a0164a6d5e4172 Mon Sep 17 00:00:00 2001
From: martinboehme <mboehme@google.com>
Date: Tue, 14 May 2024 09:45:54 +0200
Subject: [PATCH 0691/1206] [Clang][Sema] Fix malformed AST for anonymous class
 access in template. (#90842)

# Observed erroneous behavior

Prior to this change, a `MemberExpr` that accesses an anonymous class
might have a prvalue as its base (even though C++ mandates that the base
of a `MemberExpr` must be a glvalue), if the code containing the
`MemberExpr` was in a template.

Here's an example on [godbolt](https://godbolt.org/z/Gz1Mer9oz) (that is
essentially identical to the new test this patch adds).

This example sets up a struct containing an anonymous struct:

```cxx
struct S {
  struct {
    int i;
  };
};
```

It then accesses the member `i` using the expression `S().i`.

When we do this in a non-template function, we get the following AST:

```
`-ExprWithCleanups <col:10, col:14> 'int'
  `-ImplicitCastExpr <col:10, col:14> 'int' <LValueToRValue>
    `-MemberExpr <col:10, col:14> 'int' xvalue .i 0xbdcb3c0
      `-MemberExpr <col:10, col:14> 'S::(anonymous struct at line:2:3)' xvalue .S::(anonymous struct at line:2:3) 0xbdcb488
        `-MaterializeTemporaryExpr <col:10, col:12> 'S' xvalue
          `-CXXTemporaryObjectExpr <col:10, col:12> 'S' 'void () noexcept' zeroing
```

As expected, the AST contains a `MaterializeTemporarExpr` to materialize
the prvalue `S()` before accessing its members.

When we perform this access in a function template (that doesn't
actually even use its template parameter), the AST for the template
itself looks the same as above. However, the AST for an instantiation of
the template looks different:

```
`-ExprWithCleanups <col:10, col:14> 'int'
  `-ImplicitCastExpr <col:10, col:14> 'int' <LValueToRValue>
    `-MemberExpr <col:10, col:14> 'int' xvalue .i 0xbdcb3c0
      `-MaterializeTemporaryExpr <col:10, col:14> 'S::(anonymous struct at line:2:3)' xvalue
        `-MemberExpr <col:10, col:14> 'S::(anonymous struct at line:2:3)' .S::(anonymous struct at line:2:3) 0xbdcb488
          `-CXXTemporaryObjectExpr <col:10, col:12> 'S' 'void () noexcept' zeroing
```

Note how the inner `MemberExpr` (the one accessing the anonymous struct)
acts on a prvalue.

Interestingly, this does not appear to cause any problems for CodeGen,
probably because CodeGen is set up to deal with `MemberExpr`s on rvalues
in C. However, it does cause issues in the dataflow framework, which
only supports C++ today and expects the base of a `MemberExpr` to be a
glvalue.

Beyond the issues with the dataflow framework, I think this issue should
be fixed because it goes contrary to what the C++ standard mandates, and
the AST produced for the non-template case indicates that we want to
follow the C++ rules here.

# Reasons for erroneous behavior

Here's why we're getting this malformed AST.

First of all, `TreeTransform` [strips any
`MaterializeTemporaryExpr`s](https://github.com/llvm/llvm-project/blob/cd132dcbeb0fc79fd657bd5e0a8e9244c3fb5da6/clang/lib/Sema/TreeTransform.h#L14853)
from the AST.

It is therefore up to
[`TreeTransform::RebuildMemberExpr()`](https://github.com/llvm/llvm-project/blob/cd132dcbeb0fc79fd657bd5e0a8e9244c3fb5da6/clang/lib/Sema/TreeTransform.h#L2853)
to recreate a `MaterializeTemporaryExpr` if needed. In the [general
case](https://github.com/llvm/llvm-project/blob/cd132dcbeb0fc79fd657bd5e0a8e9244c3fb5da6/clang/lib/Sema/TreeTransform.h#L2915),
it does this: It calls `Sema::BuildMemberReferenceExpr()`, which ensures
that the base is a glvalue by [materializing a
temporary](https://github.com/llvm/llvm-project/blob/cd132dcbeb0fc79fd657bd5e0a8e9244c3fb5da6/clang/lib/Sema/SemaExprMember.cpp#L1016)
if needed. However, when `TreeTransform::RebuildMemberExpr()` encounters
an anonymous class, it [calls
`Sema::BuildFieldReferenceExpr()`](https://github.com/llvm/llvm-project/blob/cd132dcbeb0fc79fd657bd5e0a8e9244c3fb5da6/clang/lib/Sema/TreeTransform.h#L2880),
which, unlike `Sema::BuildMemberReferenceExpr()`, does not make sure
that the base is a glvalue.

# Proposed fix

I considered several possible ways to fix this issue:

- Add logic to `Sema::BuildFieldReferenceExpr()` that materializes a
temporary if needed. This appears to work, but it feels like the fix is
in the wrong place:
- AFAIU, other callers of `Sema::BuildFieldReferenceExpr()` don't need
this logic.
- The issue is caused by `TreeTransform` removing the
`MaterializeTemporaryExpr`, so it seems the fix should also be in
`TreeTransform`

- Materialize the temporary directly in
`TreeTransform::RebuildMemberExpr()` if needed (within the case that
deals with anonymous classes).

This would work, too, but it would duplicate logic that already exists
in `Sema::BuildMemberReferenceExpr()` (which we leverage for the general
case).

- Use `Sema::BuildMemberReferenceExpr()` instead of
`Sema::BuildFieldReferenceExpr()` for the anonymous class case, so that
it also uses the existing logic for materializing the temporary.

This is the option I've decided to go with here. There's a slight
wrinkle in that we create a `LookupResult` that claims we looked up the
unnamed field for the anonymous class -- even though we would obviously
never be able to look up an unnamed field. I think this is defensible
and still better than the other alternatives, but I would welcome
feedback on this from others who know the code better.
---
 clang/docs/ReleaseNotes.rst                   |  1 +
 clang/lib/Sema/TreeTransform.h                | 13 ++++-
 clang/test/AST/ast-dump-anonymous-class.cpp   | 49 +++++++++++++++++++
 .../Analysis/FlowSensitive/TransferTest.cpp   | 35 +++++++++++++
 4 files changed, 97 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/AST/ast-dump-anonymous-class.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 28ac54127383..49ab222bec40 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -715,6 +715,7 @@ Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 - Clang now properly preserves ``FoundDecls`` within a ``ConceptReference``. (#GH82628)
 - The presence of the ``typename`` keyword is now stored in ``TemplateTemplateParmDecl``.
+- Fixed malformed AST generated for anonymous union access in templates. (#GH90842)
 
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 2d903dc52556..c039b95293af 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -2874,10 +2874,21 @@ public:
         return ExprError();
       Base = BaseResult.get();
 
+      // `TranformMaterializeTemporaryExpr()` removes materialized temporaries
+      // from the AST, so we need to re-insert them if needed (since
+      // `BuildFieldRefereneExpr()` doesn't do this).
+      if (!isArrow && Base->isPRValue()) {
+        BaseResult = getSema().TemporaryMaterializationConversion(Base);
+        if (BaseResult.isInvalid())
+          return ExprError();
+        Base = BaseResult.get();
+      }
+
       CXXScopeSpec EmptySS;
       return getSema().BuildFieldReferenceExpr(
           Base, isArrow, OpLoc, EmptySS, cast<FieldDecl>(Member),
-          DeclAccessPair::make(FoundDecl, FoundDecl->getAccess()), MemberNameInfo);
+          DeclAccessPair::make(FoundDecl, FoundDecl->getAccess()),
+          MemberNameInfo);
     }
 
     CXXScopeSpec SS;
diff --git a/clang/test/AST/ast-dump-anonymous-class.cpp b/clang/test/AST/ast-dump-anonymous-class.cpp
new file mode 100644
index 000000000000..393c084c913d
--- /dev/null
+++ b/clang/test/AST/ast-dump-anonymous-class.cpp
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown -ast-dump %s \
+// RUN: | FileCheck -strict-whitespace %s
+
+struct S {
+  struct {
+    int i;
+  };
+};
+
+int accessInRegularFunction() {
+  return S().i;
+  // CHECK: FunctionDecl {{.*}} accessInRegularFunction 'int ()'
+  // CHECK:      |   `-ReturnStmt {{.*}}
+  // CHECK-NEXT: |     `-ExprWithCleanups {{.*}} 'int'
+  // CHECK-NEXT: |       `-ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: |         `-MemberExpr {{.*}} 'int' xvalue .i
+  // CHECK-NEXT: |           `-MemberExpr {{.*}} 'S::(anonymous struct at {{.*}})
+  // CHECK-NEXT: |             `-MaterializeTemporaryExpr {{.*}} 'S' xvalue
+  // CHECK-NEXT: |               `-CXXTemporaryObjectExpr {{.*}} 'S' 'void () noexcept' zeroing
+}
+
+// AST should look the same in a function template with an unused template
+// parameter.
+template <class>
+int accessInFunctionTemplate() {
+  return S().i;
+  // CHECK: FunctionDecl {{.*}} accessInFunctionTemplate 'int ()'
+  // CHECK:      |   `-ReturnStmt {{.*}}
+  // CHECK-NEXT: |     `-ExprWithCleanups {{.*}} 'int'
+  // CHECK-NEXT: |       `-ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: |         `-MemberExpr {{.*}} 'int' xvalue .i
+  // CHECK-NEXT: |           `-MemberExpr {{.*}} 'S::(anonymous struct at {{.*}})
+  // CHECK-NEXT: |             `-MaterializeTemporaryExpr {{.*}} 'S' xvalue
+  // CHECK-NEXT: |               `-CXXTemporaryObjectExpr {{.*}} 'S' 'void () noexcept' zeroing
+}
+
+// AST should look the same in an instantiation of the function template.
+// This is a regression test: The AST used to contain the
+// `MaterializeTemporaryExpr` in the wrong place, causing a `MemberExpr` to have
+// a prvalue base (which is not allowed in C++).
+template int accessInFunctionTemplate<int>();
+  // CHECK: FunctionDecl {{.*}} accessInFunctionTemplate 'int ()' explicit_instantiation_definition
+  // CHECK:          `-ReturnStmt {{.*}}
+  // CHECK-NEXT:       `-ExprWithCleanups {{.*}} 'int'
+  // CHECK-NEXT:         `-ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT:           `-MemberExpr {{.*}} 'int' xvalue .i
+  // CHECK-NEXT:             `-MemberExpr {{.*}} 'S::(anonymous struct at {{.*}})
+  // CHECK-NEXT:               `-MaterializeTemporaryExpr {{.*}} 'S' xvalue
+  // CHECK-NEXT:                 `-CXXTemporaryObjectExpr {{.*}} 'S' 'void () noexcept' zeroing
diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
index e1fb16b64fd6..5c0582e872eb 100644
--- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
@@ -27,6 +27,14 @@
 #include <string>
 #include <utility>
 
+namespace clang {
+namespace dataflow {
+namespace {
+AST_MATCHER(FunctionDecl, isTemplated) { return Node.isTemplated(); }
+} // namespace
+} // namespace dataflow
+} // namespace clang
+
 namespace {
 
 using namespace clang;
@@ -7416,4 +7424,31 @@ TEST(TransferTest, ConditionalRelation) {
       });
 }
 
+// This is a crash repro.
+// We used to crash while transferring `S().i` because Clang contained a bug
+// causing the AST to be malformed.
+TEST(TransferTest, AnonymousUnionMemberExprInTemplate) {
+  using ast_matchers::functionDecl;
+  using ast_matchers::hasName;
+  using ast_matchers::unless;
+
+  std::string Code = R"cc(
+    struct S {
+      struct {
+        int i;
+      };
+    };
+
+    template <class>
+    void target() {
+        S().i;
+    }
+
+    template void target<int>();
+  )cc";
+  auto Matcher = functionDecl(hasName("target"), unless(isTemplated()));
+  ASSERT_THAT_ERROR(checkDataflowWithNoopAnalysis(Code, Matcher),
+                    llvm::Succeeded());
+}
+
 } // namespace
-- 
GitLab


From 79a6a7e28fffd14e54a9a208af12d724b6eeb2d4 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 14 May 2024 00:48:56 -0700
Subject: [PATCH 0692/1206] [RISCV] Fix a warning

This patch fixes:

  llvm/lib/Target/RISCV/RISCVISelLowering.cpp:19848:11: error:
  enumeration value 'SW_GUARDED_BRIND' not handled in switch
  [-Werror,-Wswitch]
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index d3e8a86f8766..8d9b0f2acc5f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -20067,6 +20067,7 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(SWAP_CSR)
   NODE_NAME_CASE(CZERO_EQZ)
   NODE_NAME_CASE(CZERO_NEZ)
+  NODE_NAME_CASE(SW_GUARDED_BRIND)
   NODE_NAME_CASE(SF_VC_XV_SE)
   NODE_NAME_CASE(SF_VC_IV_SE)
   NODE_NAME_CASE(SF_VC_VV_SE)
-- 
GitLab


From 11e5d1cfee399cfaba373078879c1ac3e1109b11 Mon Sep 17 00:00:00 2001
From: Sameer Sahasrabuddhe <sameer.sahasrabuddhe@amd.com>
Date: Tue, 14 May 2024 13:39:37 +0530
Subject: [PATCH 0693/1206] [AMDGPU] Respect existing glue when lowering
 convergence tokens (#90834)

---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8645f560d997..8f741ffc58a8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3859,20 +3859,20 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
   assert(Mask && "Missing call preserved mask for calling convention");
   Ops.push_back(DAG.getRegisterMask(Mask));
 
-  if (InGlue.getNode())
-    Ops.push_back(InGlue);
-
-  // NOTE: This potentially results in *two* glue operands, and the wrong one
-  // might possibly show up where the other was intended. In particular,
-  // Emitter::EmitMachineNode() expects only the glued convergence token if it
-  // exists. Similarly, the selection of the call expects to match only the
-  // InGlue operand if it exists.
   if (SDValue Token = CLI.ConvergenceControlToken) {
-    Ops.push_back(SDValue(DAG.getMachineNode(TargetOpcode::CONVERGENCECTRL_GLUE,
-                                             DL, MVT::Glue, Token),
-                          0));
+    SmallVector<SDValue, 2> GlueOps;
+    GlueOps.push_back(Token);
+    if (InGlue)
+      GlueOps.push_back(InGlue);
+
+    InGlue = SDValue(DAG.getMachineNode(TargetOpcode::CONVERGENCECTRL_GLUE, DL,
+                                        MVT::Glue, GlueOps),
+                     0);
   }
 
+  if (InGlue)
+    Ops.push_back(InGlue);
+
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
 
   // If we're doing a tall call, use a TC_RETURN here rather than an
-- 
GitLab


From c28529788955dbfada9f8a5092432f09eec2c3ab Mon Sep 17 00:00:00 2001
From: aabhinavg <78288544+aabhinavg@users.noreply.github.com>
Date: Tue, 14 May 2024 13:44:17 +0530
Subject: [PATCH 0694/1206] [lldb] Fix redundant condition in Target.cpp
 (#91882)

This commit addresses issue #87244, where a redundant condition was
found in the Target.cpp file. Static analyzer cppcheck flagged the issue
in the Target.cpp file

fix #87244
---
 lldb/source/Target/Target.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 82f3040e539a..77731167995e 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -841,7 +841,7 @@ static bool CheckIfWatchpointsSupported(Target *target, Status &error) {
   if (!num_supported_hardware_watchpoints)
     return true;
 
-  if (num_supported_hardware_watchpoints == 0) {
+  if (*num_supported_hardware_watchpoints == 0) {
     error.SetErrorStringWithFormat(
         "Target supports (%u) hardware watchpoint slots.\n",
         *num_supported_hardware_watchpoints);
-- 
GitLab


From 0b5b2027f94c60c73d6871cf64d3f580c27c5a53 Mon Sep 17 00:00:00 2001
From: Christian Ulmann <christianulmann@gmail.com>
Date: Tue, 14 May 2024 10:26:27 +0200
Subject: [PATCH 0695/1206] [MLIR][SROA] Reuse allocators to avoid rewalking
 the IR (#91971)

This commit extends the SROA interfaces to ensure the interface
instantiations can communicate newly created allocators to the
algorithm. This ensures that the SROA implementation does no longer
require re-walking the IR to find new allocators.
---
 .../mlir/Interfaces/MemorySlotInterfaces.td   | 10 ++-
 mlir/include/mlir/Transforms/SROA.h           |  6 +-
 mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp | 13 +--
 .../Dialect/MemRef/IR/MemRefMemorySlot.cpp    | 13 +--
 mlir/lib/Transforms/SROA.cpp                  | 86 ++++++++++++-------
 mlir/test/Transforms/sroa.mlir                | 31 +++++++
 mlir/test/lib/Dialect/Test/TestOpDefs.cpp     | 78 ++++++++++++++---
 mlir/test/lib/Dialect/Test/TestOps.td         |  5 +-
 8 files changed, 184 insertions(+), 58 deletions(-)
 create mode 100644 mlir/test/Transforms/sroa.mlir

diff --git a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td
index e2409cbec5fd..6f023f0c5263 100644
--- a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td
+++ b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td
@@ -298,14 +298,20 @@ def DestructurableAllocationOpInterface
       "destructure",
       (ins "const ::mlir::DestructurableMemorySlot &":$slot,
            "const ::llvm::SmallPtrSetImpl<::mlir::Attribute> &":$usedIndices,
-           "::mlir::OpBuilder &":$builder)
+           "::mlir::OpBuilder &":$builder,
+           "::mlir::SmallVectorImpl<::mlir::DestructurableAllocationOpInterface> &":
+             $newAllocators)
     >,
     InterfaceMethod<[{
         Hook triggered once the destructuring of a slot is complete, meaning the
         original slot is no longer being refered to and could be deleted.
         This will only be called for slots declared by this operation.
+
+        Must return a new destructurable allocation op if this hook creates
+        a new destructurable op, nullopt otherwise. 
       }],
-      "void", "handleDestructuringComplete",
+      "::std::optional<::mlir::DestructurableAllocationOpInterface>",
+      "handleDestructuringComplete",
       (ins "const ::mlir::DestructurableMemorySlot &":$slot,
            "::mlir::OpBuilder &":$builder)
     >,
diff --git a/mlir/include/mlir/Transforms/SROA.h b/mlir/include/mlir/Transforms/SROA.h
index fa84fb1eae73..d48f809d3076 100644
--- a/mlir/include/mlir/Transforms/SROA.h
+++ b/mlir/include/mlir/Transforms/SROA.h
@@ -27,8 +27,10 @@ struct SROAStatistics {
   llvm::Statistic *maxSubelementAmount = nullptr;
 };
 
-/// Attempts to destructure the slots of destructurable allocators. Returns
-/// failure if no slot was destructured.
+/// Attempts to destructure the slots of destructurable allocators. Iteratively
+/// retries the destructuring of all slots as destructuring one slot might
+/// enable subsequent destructuring. Returns failure if no slot was
+/// destructured.
 LogicalResult tryToDestructureMemorySlots(
     ArrayRef<DestructurableAllocationOpInterface> allocators,
     OpBuilder &builder, const DataLayout &dataLayout,
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index 4fdf847a559c..3f1e5b1773bf 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -77,10 +77,10 @@ SmallVector<DestructurableMemorySlot> LLVM::AllocaOp::getDestructurableSlots() {
                                    *destructuredType}};
 }
 
-DenseMap<Attribute, MemorySlot>
-LLVM::AllocaOp::destructure(const DestructurableMemorySlot &slot,
-                            const SmallPtrSetImpl<Attribute> &usedIndices,
-                            OpBuilder &builder) {
+DenseMap<Attribute, MemorySlot> LLVM::AllocaOp::destructure(
+    const DestructurableMemorySlot &slot,
+    const SmallPtrSetImpl<Attribute> &usedIndices, OpBuilder &builder,
+    SmallVectorImpl<DestructurableAllocationOpInterface> &newAllocators) {
   assert(slot.ptr == getResult());
   builder.setInsertionPointAfter(*this);
 
@@ -92,16 +92,19 @@ LLVM::AllocaOp::destructure(const DestructurableMemorySlot &slot,
     auto subAlloca = builder.create<LLVM::AllocaOp>(
         getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType,
         getArraySize());
+    newAllocators.push_back(subAlloca);
     slotMap.try_emplace<MemorySlot>(index, {subAlloca.getResult(), elemType});
   }
 
   return slotMap;
 }
 
-void LLVM::AllocaOp::handleDestructuringComplete(
+std::optional<DestructurableAllocationOpInterface>
+LLVM::AllocaOp::handleDestructuringComplete(
     const DestructurableMemorySlot &slot, OpBuilder &builder) {
   assert(slot.ptr == getResult());
   this->erase();
+  return std::nullopt;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
index e30598e6878f..631dee2d4053 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp
@@ -126,10 +126,10 @@ memref::AllocaOp::getDestructurableSlots() {
       DestructurableMemorySlot{{getMemref(), memrefType}, *destructuredType}};
 }
 
-DenseMap<Attribute, MemorySlot>
-memref::AllocaOp::destructure(const DestructurableMemorySlot &slot,
-                              const SmallPtrSetImpl<Attribute> &usedIndices,
-                              OpBuilder &builder) {
+DenseMap<Attribute, MemorySlot> memref::AllocaOp::destructure(
+    const DestructurableMemorySlot &slot,
+    const SmallPtrSetImpl<Attribute> &usedIndices, OpBuilder &builder,
+    SmallVectorImpl<DestructurableAllocationOpInterface> &newAllocators) {
   builder.setInsertionPointAfter(*this);
 
   DenseMap<Attribute, MemorySlot> slotMap;
@@ -139,6 +139,7 @@ memref::AllocaOp::destructure(const DestructurableMemorySlot &slot,
     Type elemType = memrefType.getTypeAtIndex(usedIndex);
     MemRefType elemPtr = MemRefType::get({}, elemType);
     auto subAlloca = builder.create<memref::AllocaOp>(getLoc(), elemPtr);
+    newAllocators.push_back(subAlloca);
     slotMap.try_emplace<MemorySlot>(usedIndex,
                                     {subAlloca.getResult(), elemType});
   }
@@ -146,10 +147,12 @@ memref::AllocaOp::destructure(const DestructurableMemorySlot &slot,
   return slotMap;
 }
 
-void memref::AllocaOp::handleDestructuringComplete(
+std::optional<DestructurableAllocationOpInterface>
+memref::AllocaOp::handleDestructuringComplete(
     const DestructurableMemorySlot &slot, OpBuilder &builder) {
   assert(slot.ptr == getResult());
   this->erase();
+  return std::nullopt;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Transforms/SROA.cpp b/mlir/lib/Transforms/SROA.cpp
index 4e28fa687ffd..67cbade07bc9 100644
--- a/mlir/lib/Transforms/SROA.cpp
+++ b/mlir/lib/Transforms/SROA.cpp
@@ -132,16 +132,17 @@ computeDestructuringInfo(DestructurableMemorySlot &slot,
 /// Performs the destructuring of a destructible slot given associated
 /// destructuring information. The provided slot will be destructured in
 /// subslots as specified by its allocator.
-static void destructureSlot(DestructurableMemorySlot &slot,
-                            DestructurableAllocationOpInterface allocator,
-                            OpBuilder &builder, const DataLayout &dataLayout,
-                            MemorySlotDestructuringInfo &info,
-                            const SROAStatistics &statistics) {
+static void destructureSlot(
+    DestructurableMemorySlot &slot,
+    DestructurableAllocationOpInterface allocator, OpBuilder &builder,
+    const DataLayout &dataLayout, MemorySlotDestructuringInfo &info,
+    SmallVectorImpl<DestructurableAllocationOpInterface> &newAllocators,
+    const SROAStatistics &statistics) {
   OpBuilder::InsertionGuard guard(builder);
 
   builder.setInsertionPointToStart(slot.ptr.getParentBlock());
   DenseMap<Attribute, MemorySlot> subslots =
-      allocator.destructure(slot, info.usedIndices, builder);
+      allocator.destructure(slot, info.usedIndices, builder, newAllocators);
 
   if (statistics.slotsWithMemoryBenefit &&
       slot.elementPtrs.size() != info.usedIndices.size())
@@ -185,7 +186,11 @@ static void destructureSlot(DestructurableMemorySlot &slot,
   if (statistics.destructuredAmount)
     (*statistics.destructuredAmount)++;
 
-  allocator.handleDestructuringComplete(slot, builder);
+  std::optional<DestructurableAllocationOpInterface> newAllocator =
+      allocator.handleDestructuringComplete(slot, builder);
+  // Add newly created allocators to the worklist for further processing.
+  if (newAllocator)
+    newAllocators.push_back(*newAllocator);
 }
 
 LogicalResult mlir::tryToDestructureMemorySlots(
@@ -194,16 +199,44 @@ LogicalResult mlir::tryToDestructureMemorySlots(
     SROAStatistics statistics) {
   bool destructuredAny = false;
 
-  for (DestructurableAllocationOpInterface allocator : allocators) {
-    for (DestructurableMemorySlot slot : allocator.getDestructurableSlots()) {
-      std::optional<MemorySlotDestructuringInfo> info =
-          computeDestructuringInfo(slot, dataLayout);
-      if (!info)
-        continue;
+  SmallVector<DestructurableAllocationOpInterface> workList(allocators.begin(),
+                                                            allocators.end());
+  SmallVector<DestructurableAllocationOpInterface> newWorkList;
+  newWorkList.reserve(allocators.size());
+  // Destructuring a slot can allow for further destructuring of other
+  // slots, destructuring is tried until no destructuring succeeds.
+  while (true) {
+    bool changesInThisRound = false;
+
+    for (DestructurableAllocationOpInterface allocator : workList) {
+      bool destructuredAnySlot = false;
+      for (DestructurableMemorySlot slot : allocator.getDestructurableSlots()) {
+        std::optional<MemorySlotDestructuringInfo> info =
+            computeDestructuringInfo(slot, dataLayout);
+        if (!info)
+          continue;
 
-      destructureSlot(slot, allocator, builder, dataLayout, *info, statistics);
-      destructuredAny = true;
+        destructureSlot(slot, allocator, builder, dataLayout, *info,
+                        newWorkList, statistics);
+        destructuredAnySlot = true;
+
+        // A break is required, since destructuring a slot may invalidate the
+        // remaning slots of an allocator.
+        break;
+      }
+      if (!destructuredAnySlot)
+        newWorkList.push_back(allocator);
+      changesInThisRound |= destructuredAnySlot;
     }
+
+    if (!changesInThisRound)
+      break;
+    destructuredAny |= changesInThisRound;
+
+    // Swap the vector's backing memory and clear the entries in newWorkList
+    // afterwards. This ensures that additional heap allocations can be avoided.
+    workList.swap(newWorkList);
+    newWorkList.clear();
   }
 
   return success(destructuredAny);
@@ -230,23 +263,16 @@ struct SROA : public impl::SROABase<SROA> {
 
       OpBuilder builder(&region.front(), region.front().begin());
 
-      // Destructuring a slot can allow for further destructuring of other
-      // slots, destructuring is tried until no destructuring succeeds.
-      while (true) {
-        SmallVector<DestructurableAllocationOpInterface> allocators;
-        // Build a list of allocators to attempt to destructure the slots of.
-        // TODO: Update list on the fly to avoid repeated visiting of the same
-        // allocators.
-        region.walk([&](DestructurableAllocationOpInterface allocator) {
-          allocators.emplace_back(allocator);
-        });
-
-        if (failed(tryToDestructureMemorySlots(allocators, builder, dataLayout,
-                                               statistics)))
-          break;
+      SmallVector<DestructurableAllocationOpInterface> allocators;
+      // Build a list of allocators to attempt to destructure the slots of.
+      region.walk([&](DestructurableAllocationOpInterface allocator) {
+        allocators.emplace_back(allocator);
+      });
 
+      // Attempt to destructure as many slots as possible.
+      if (succeeded(tryToDestructureMemorySlots(allocators, builder, dataLayout,
+                                                statistics)))
         changed = true;
-      }
     }
     if (!changed)
       markAllAnalysesPreserved();
diff --git a/mlir/test/Transforms/sroa.mlir b/mlir/test/Transforms/sroa.mlir
new file mode 100644
index 000000000000..c9e80a6cf8dd
--- /dev/null
+++ b/mlir/test/Transforms/sroa.mlir
@@ -0,0 +1,31 @@
+// RUN: mlir-opt %s --pass-pipeline='builtin.module(func.func(sroa))' --split-input-file | FileCheck %s
+
+// Verifies that allocators with mutliple slots are handled properly.
+
+// CHECK-LABEL: func.func @multi_slot_alloca
+func.func @multi_slot_alloca() -> (i32, i32) {
+  %0 = arith.constant 0 : index
+  %1, %2 = test.multi_slot_alloca : () -> (memref<2xi32>, memref<4xi32>)
+  // CHECK-COUNT-2: test.multi_slot_alloca : () -> memref<i32>
+  %3 = memref.load %1[%0] {first}: memref<2xi32>
+  %4 = memref.load %2[%0] {second} : memref<4xi32>
+  return %3, %4 : i32, i32
+}
+
+// -----
+
+// Verifies that a multi slot allocator can be partially destructured.
+
+func.func private @consumer(memref<2xi32>)
+
+// CHECK-LABEL: func.func @multi_slot_alloca_only_second
+func.func @multi_slot_alloca_only_second() -> (i32, i32) {
+  %0 = arith.constant 0 : index
+  // CHECK: test.multi_slot_alloca : () -> memref<2xi32>
+  // CHECK: test.multi_slot_alloca : () -> memref<i32>
+  %1, %2 = test.multi_slot_alloca : () -> (memref<2xi32>, memref<4xi32>)
+  func.call @consumer(%1) : (memref<2xi32>) -> ()
+  %3 = memref.load %1[%0] : memref<2xi32>
+  %4 = memref.load %2[%0] : memref<4xi32>
+  return %3, %4 : i32, i32
+}
diff --git a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
index d22d48b139a0..0b676db18af4 100644
--- a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
+++ b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
@@ -1199,22 +1199,20 @@ void TestMultiSlotAlloca::handleBlockArgument(const MemorySlot &slot,
   // Not relevant for testing.
 }
 
-std::optional<PromotableAllocationOpInterface>
-TestMultiSlotAlloca::handlePromotionComplete(const MemorySlot &slot,
-                                             Value defaultValue,
-                                             OpBuilder &builder) {
-  if (defaultValue && defaultValue.use_empty())
-    defaultValue.getDefiningOp()->erase();
+/// Creates a new TestMultiSlotAlloca operation, just without the `slot`.
+static std::optional<TestMultiSlotAlloca>
+createNewMultiAllocaWithoutSlot(const MemorySlot &slot, OpBuilder &builder,
+                                TestMultiSlotAlloca oldOp) {
 
-  if (getNumResults() == 1) {
-    erase();
+  if (oldOp.getNumResults() == 1) {
+    oldOp.erase();
     return std::nullopt;
   }
 
   SmallVector<Type> newTypes;
   SmallVector<Value> remainingValues;
 
-  for (Value oldResult : getResults()) {
+  for (Value oldResult : oldOp.getResults()) {
     if (oldResult == slot.ptr)
       continue;
     remainingValues.push_back(oldResult);
@@ -1222,12 +1220,68 @@ TestMultiSlotAlloca::handlePromotionComplete(const MemorySlot &slot,
   }
 
   OpBuilder::InsertionGuard guard(builder);
-  builder.setInsertionPoint(*this);
-  auto replacement = builder.create<TestMultiSlotAlloca>(getLoc(), newTypes);
+  builder.setInsertionPoint(oldOp);
+  auto replacement =
+      builder.create<TestMultiSlotAlloca>(oldOp->getLoc(), newTypes);
   for (auto [oldResult, newResult] :
        llvm::zip_equal(remainingValues, replacement.getResults()))
     oldResult.replaceAllUsesWith(newResult);
 
-  erase();
+  oldOp.erase();
   return replacement;
 }
+
+std::optional<PromotableAllocationOpInterface>
+TestMultiSlotAlloca::handlePromotionComplete(const MemorySlot &slot,
+                                             Value defaultValue,
+                                             OpBuilder &builder) {
+  if (defaultValue && defaultValue.use_empty())
+    defaultValue.getDefiningOp()->erase();
+  return createNewMultiAllocaWithoutSlot(slot, builder, *this);
+}
+
+SmallVector<DestructurableMemorySlot>
+TestMultiSlotAlloca::getDestructurableSlots() {
+  SmallVector<DestructurableMemorySlot> slots;
+  for (Value result : getResults()) {
+    auto memrefType = cast<MemRefType>(result.getType());
+    auto destructurable = dyn_cast<DestructurableTypeInterface>(memrefType);
+    if (!destructurable)
+      continue;
+
+    std::optional<DenseMap<Attribute, Type>> destructuredType =
+        destructurable.getSubelementIndexMap();
+    if (!destructuredType)
+      continue;
+    slots.emplace_back(
+        DestructurableMemorySlot{{result, memrefType}, *destructuredType});
+  }
+  return slots;
+}
+
+DenseMap<Attribute, MemorySlot> TestMultiSlotAlloca::destructure(
+    const DestructurableMemorySlot &slot,
+    const SmallPtrSetImpl<Attribute> &usedIndices, OpBuilder &builder,
+    SmallVectorImpl<DestructurableAllocationOpInterface> &newAllocators) {
+  OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPointAfter(*this);
+
+  DenseMap<Attribute, MemorySlot> slotMap;
+
+  for (Attribute usedIndex : usedIndices) {
+    Type elemType = slot.elementPtrs.lookup(usedIndex);
+    MemRefType elemPtr = MemRefType::get({}, elemType);
+    auto subAlloca = builder.create<TestMultiSlotAlloca>(getLoc(), elemPtr);
+    newAllocators.push_back(subAlloca);
+    slotMap.try_emplace<MemorySlot>(usedIndex,
+                                    {subAlloca.getResult(0), elemType});
+  }
+
+  return slotMap;
+}
+
+std::optional<DestructurableAllocationOpInterface>
+TestMultiSlotAlloca::handleDestructuringComplete(
+    const DestructurableMemorySlot &slot, OpBuilder &builder) {
+  return createNewMultiAllocaWithoutSlot(slot, builder, *this);
+}
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index e16ea2407314..7fc3d22d1895 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -3169,11 +3169,12 @@ def TestOpOptionallyImplementingInterface
 }
 
 //===----------------------------------------------------------------------===//
-// Test Mem2Reg
+// Test Mem2Reg & SROA
 //===----------------------------------------------------------------------===//
 
 def TestMultiSlotAlloca : TEST_Op<"multi_slot_alloca",
-    [DeclareOpInterfaceMethods<PromotableAllocationOpInterface>]> {
+    [DeclareOpInterfaceMethods<PromotableAllocationOpInterface>,
+     DeclareOpInterfaceMethods<DestructurableAllocationOpInterface>]> {
   let results = (outs Variadic<MemRefOf<[I32]>>:$results);
   let assemblyFormat = "attr-dict `:` functional-type(operands, results)";
 }
-- 
GitLab


From 346f2b76246a46d5e634dfcf0004d72ac5127f8e Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Tue, 14 May 2024 12:30:34 +0400
Subject: [PATCH 0696/1206] [lldb] Fix the test TestGdbRemotePlatformFile when
 run with a remote target (#91918)

It is necessary to transfer the test file to/from the really remote
target (for example Windows host and Linux target). Also ignore chmod
check in case of the Windows host.
---
 .../lldb-server/TestGdbRemotePlatformFile.py  | 33 +++++++++++++++++--
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/lldb/test/API/tools/lldb-server/TestGdbRemotePlatformFile.py b/lldb/test/API/tools/lldb-server/TestGdbRemotePlatformFile.py
index 4c8ce01e8ba3..2e1c72ee56d7 100644
--- a/lldb/test/API/tools/lldb-server/TestGdbRemotePlatformFile.py
+++ b/lldb/test/API/tools/lldb-server/TestGdbRemotePlatformFile.py
@@ -1,6 +1,7 @@
 # lldb test suite imports
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import TestBase
+from lldbsuite.test import lldbutil
 
 # gdb-remote-specific imports
 import lldbgdbserverutils
@@ -117,6 +118,7 @@ class TestGdbRemotePlatformFile(GdbRemoteTestCaseBase):
         temp_file = self.getBuildArtifact("test")
         with open(temp_file, "wb"):
             pass
+        temp_file = lldbutil.install_to_target(self, temp_file)
 
         # attempt to open the file with O_CREAT|O_EXCL
         self.do_handshake()
@@ -140,6 +142,7 @@ class TestGdbRemotePlatformFile(GdbRemoteTestCaseBase):
         test_data = b"test data of some length"
         with open(temp_path, "wb") as temp_file:
             temp_file.write(test_data)
+        temp_path = lldbutil.install_to_target(self, temp_path)
 
         self.do_handshake()
         self.test_sequence.add_log_lines(
@@ -167,7 +170,11 @@ class TestGdbRemotePlatformFile(GdbRemoteTestCaseBase):
         test_mode = 0o751
 
         with open(temp_path, "wb") as temp_file:
-            os.chmod(temp_file.fileno(), test_mode)
+            if lldbplatformutil.getHostPlatform() == "windows":
+                test_mode = 0o700
+            else:
+                os.chmod(temp_file.fileno(), test_mode)
+        temp_path = lldbutil.install_to_target(self, temp_path)
 
         self.do_handshake()
         self.test_sequence.add_log_lines(
@@ -213,6 +220,7 @@ class TestGdbRemotePlatformFile(GdbRemoteTestCaseBase):
         temp_path = self.getBuildArtifact("test")
         with open(temp_path, "wb"):
             pass
+        temp_path = lldbutil.install_to_target(self, temp_path)
 
         self.do_handshake()
         self.test_sequence.add_log_lines(
@@ -244,6 +252,10 @@ class TestGdbRemotePlatformFile(GdbRemoteTestCaseBase):
         self.expect_gdbremote_sequence()
 
     @skipIfWindows
+    # FIXME: lldb.remote_platform.Install() cannot copy opened temp file on Windows.
+    # It is possible to use tempfile.NamedTemporaryFile(..., delete=False) and
+    # delete the temp file manually at the end.
+    @skipIf(hostoslist=["windows"])
     @add_test_categories(["llgs"])
     def test_platform_file_fstat(self):
         server = self.connect_to_debug_monitor()
@@ -252,12 +264,13 @@ class TestGdbRemotePlatformFile(GdbRemoteTestCaseBase):
         with tempfile.NamedTemporaryFile() as temp_file:
             temp_file.write(b"some test data for stat")
             temp_file.flush()
+            temp_path = lldbutil.install_to_target(self, temp_file.name)
 
             self.do_handshake()
             self.test_sequence.add_log_lines(
                 [
                     "read packet: $vFile:open:%s,0,0#00"
-                    % (binascii.b2a_hex(temp_file.name.encode()).decode(),),
+                    % (binascii.b2a_hex(temp_path.encode()).decode(),),
                     {
                         "direction": "send",
                         "regex": r"^\$F([0-9a-fA-F]+)#[0-9a-fA-F]{2}$",
@@ -359,9 +372,12 @@ class TestGdbRemotePlatformFile(GdbRemoteTestCaseBase):
 
         if creat:
             self.assertFalse(os.path.exists(temp_path))
+            if lldb.remote_platform:
+                temp_path = lldbutil.append_to_process_working_directory(self, "test")
         else:
             with open(temp_path, "wb") as temp_file:
                 temp_file.write(test_data.encode())
+            temp_path = lldbutil.install_to_target(self, temp_path)
 
         # open the file for reading
         self.do_handshake()
@@ -448,8 +464,19 @@ class TestGdbRemotePlatformFile(GdbRemoteTestCaseBase):
 
         if write:
             # check if the data was actually written
+            if lldb.remote_platform:
+                local_path = self.getBuildArtifact("file_from_target")
+                error = lldb.remote_platform.Get(
+                    lldb.SBFileSpec(temp_path, False), lldb.SBFileSpec(local_path, True)
+                )
+                self.assertTrue(
+                    error.Success(),
+                    "Reading file {0} failed: {1}".format(temp_path, error),
+                )
+                temp_path = local_path
+
             with open(temp_path, "rb") as temp_file:
-                if creat:
+                if creat and lldbplatformutil.getHostPlatform() != "windows":
                     self.assertEqual(
                         os.fstat(temp_file.fileno()).st_mode & 0o7777, 0o640
                     )
-- 
GitLab


From 632317e9ab5548e991d8974954353033bea62a5b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 14 May 2024 09:42:49 +0100
Subject: [PATCH 0697/1206] [VPlan] Add non-poison propagating LogicalAnd
 VPInstruction opcode. (#91897)

Add a new opcode to mode non-poison propagating logical AND operations
used when generating edge masks. This follows the similar decision to
model Not as dedicated opcode as well, to improve clarity.

This also helps to simplify the matchers for
https://github.com/llvm/llvm-project/pull/89386.


PR: https://github.com/llvm/llvm-project/pull/91897
---
 .../Vectorize/LoopVectorizationPlanner.h      |  6 ++++++
 .../Transforms/Vectorize/LoopVectorize.cpp    | 15 +++++++-------
 llvm/lib/Transforms/Vectorize/VPlan.h         |  1 +
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |  9 +++++++++
 .../LoopVectorize/vplan-printing.ll           |  4 ++--
 .../vplan-sink-scalars-and-merge.ll           | 20 +++++++++----------
 6 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index ece2a34f180c..c03c278fcebe 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -179,6 +179,12 @@ public:
         VPRecipeWithIRFlags::DisjointFlagsTy(false), DL, Name));
   }
 
+  VPValue *createLogicalAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
+                            const Twine &Name = "") {
+    return tryInsertInstruction(
+        new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name));
+  }
+
   VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
                         DebugLoc DL = {}, const Twine &Name = "",
                         std::optional<FastMathFlags> FMFs = std::nullopt) {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 8b4cb5a6658d..ba02c98285c3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8011,14 +8011,13 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) {
     EdgeMask = Builder.createNot(EdgeMask, BI->getDebugLoc());
 
   if (SrcMask) { // Otherwise block in-mask is all-one, no need to AND.
-    // The condition is 'SrcMask && EdgeMask', which is equivalent to
-    // 'select i1 SrcMask, i1 EdgeMask, i1 false'.
-    // The select version does not introduce new UB if SrcMask is false and
-    // EdgeMask is poison. Using 'and' here introduces undefined behavior.
-    VPValue *False = Plan.getOrAddLiveIn(
-        ConstantInt::getFalse(BI->getCondition()->getType()));
-    EdgeMask =
-        Builder.createSelect(SrcMask, EdgeMask, False, BI->getDebugLoc());
+    // Use LogicalAnd as it does not propagate poison, i.e. does not introduce
+    // new UB if SrcMask is false and EdgeMask is poison. Using 'and' here
+    // introduces undefined behavior.
+    // The bitwise 'And' of SrcMask and EdgeMask introduces new UB if SrcMask
+    // is false and EdgeMask is poison. Avoid that by using 'LogicalAnd'
+    // instead which generates 'select i1 SrcMask, i1 EdgeMask, i1 false'.
+    EdgeMask = Builder.createLogicalAnd(SrcMask, EdgeMask, BI->getDebugLoc());
   }
 
   return EdgeMaskCache[Edge] = EdgeMask;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 0784665efd14..4b3cb15b5e1e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1177,6 +1177,7 @@ public:
     BranchOnCount,
     BranchOnCond,
     ComputeReductionResult,
+    LogicalAnd, // Non-poison propagating logical And.
     // Add an offset in bytes (second operand) to a base pointer (first
     // operand). Only generates scalar values (either for the first lane only or
     // for all lanes, depending on its uses).
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 140516e08e79..fa634e774b5c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -137,6 +137,7 @@ bool VPRecipeBase::mayHaveSideEffects() const {
     case VPInstruction::Not:
     case VPInstruction::CalculateTripCountMinusVF:
     case VPInstruction::CanonicalIVIncrementForPart:
+    case VPInstruction::LogicalAnd:
     case VPInstruction::PtrAdd:
       return false;
     default:
@@ -557,6 +558,11 @@ Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
 
     return ReducedPartRdx;
   }
+  case VPInstruction::LogicalAnd: {
+    Value *A = State.get(getOperand(0), Part);
+    Value *B = State.get(getOperand(1), Part);
+    return Builder.CreateLogicalAnd(A, B, Name);
+  }
   case VPInstruction::PtrAdd: {
     assert(vputils::onlyFirstLaneUsed(this) &&
            "can only generate first lane for PtrAdd");
@@ -689,6 +695,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::ComputeReductionResult:
     O << "compute-reduction-result";
     break;
+  case VPInstruction::LogicalAnd:
+    O << "logical-and";
+    break;
   case VPInstruction::PtrAdd:
     O << "ptradd";
     break;
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 7056bbe6ba1b..c95f94bddf5e 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -432,7 +432,7 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
 ; CHECK-NEXT:    WIDEN ir<%cmp1> = icmp slt ir<%lsd>, ir<100>
 ; CHECK-NEXT:    EMIT vp<[[NOT1:%.+]]> = not ir<%cmp1>, !dbg /tmp/s.c:5:3
 ; CHECK-NEXT:    WIDEN ir<%cmp2> = icmp sge ir<%lsd>, ir<200>
-; CHECK-NEXT:    EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]>, ir<%cmp2>, ir<false>, !dbg /tmp/s.c:5:21
+; CHECK-NEXT:    EMIT vp<[[SEL1:%.+]]> = logical-and vp<[[NOT1]]>, ir<%cmp2>, !dbg /tmp/s.c:5:21
 ; CHECK-NEXT:    EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]>, ir<%cmp1>
 ; CHECK-NEXT:  Successor(s): pred.sdiv
 ; CHECK-EMPTY:
@@ -453,7 +453,7 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  if.then.0:
 ; CHECK-NEXT:    EMIT vp<[[NOT2:%.+]]> = not ir<%cmp2>
-; CHECK-NEXT:    EMIT vp<[[SEL2:%.+]]> = select vp<[[NOT1]]>, vp<[[NOT2]]>, ir<false>
+; CHECK-NEXT:    EMIT vp<[[SEL2:%.+]]> = logical-and vp<[[NOT1]]>, vp<[[NOT2]]>
 ; CHECK-NEXT:    BLEND ir<%ysd.0> = vp<[[PHI]]> ir<%psd>/vp<[[SEL2]]>
 ; CHECK-NEXT:    vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%isd>
 ; CHECK-NEXT:    WIDEN store vp<[[VEC_PTR2]]>, ir<%ysd.0>
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 108b78a70fa1..1e60e57a5409 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -269,7 +269,7 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
 ; CHECK-NEXT:   CLONE ir<%lv> = load ir<%gep.A.uniform>
 ; CHECK-NEXT:   WIDEN ir<%cmp> = icmp ult ir<%iv>, ir<%k>
 ; CHECK-NEXT:   EMIT vp<[[NOT2:%.+]]> = not ir<%cmp>
-; CHECK-NEXT:   EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, vp<[[NOT2]]>, ir<false>
+; CHECK-NEXT:   EMIT vp<[[MASK2:%.+]]> = logical-and vp<[[MASK]]>, vp<[[NOT2]]>
 ; CHECK-NEXT: Successor(s): pred.store
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -340,7 +340,7 @@ define void @pred_cfg1(i32 %k, i32 %j) {
 ; CHECK-NEXT:   EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
 ; CHECK-NEXT:   WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j>
 ; CHECK-NEXT:   WIDEN ir<%mul> = mul ir<%iv>, ir<10>
-; CHECK-NEXT:   EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.1>, ir<false>
+; CHECK-NEXT:   EMIT vp<[[MASK2:%.+]]> = logical-and vp<[[MASK1]]>, ir<%c.1>
 ; CHECK-NEXT: Successor(s): pred.load
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -362,7 +362,7 @@ define void @pred_cfg1(i32 %k, i32 %j) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: then.0.0:
 ; CHECK-NEXT:   EMIT vp<[[NOT:%.+]]> = not ir<%c.1>
-; CHECK-NEXT:   EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
+; CHECK-NEXT:   EMIT vp<[[MASK3:%.+]]> = logical-and vp<[[MASK1]]>, vp<[[NOT]]>
 ; CHECK-NEXT:   EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
 ; CHECK-NEXT:   BLEND ir<%p> = ir<0> vp<[[PRED]]>/vp<[[MASK2]]>
 ; CHECK-NEXT: Successor(s): pred.store
@@ -441,7 +441,7 @@ define void @pred_cfg2(i32 %k, i32 %j) {
 ; CHECK-NEXT:   WIDEN ir<%mul> = mul ir<%iv>, ir<10>
 ; CHECK-NEXT:   WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
 ; CHECK-NEXT:   WIDEN ir<%c.1> = icmp ugt ir<%iv>, ir<%j>
-; CHECK-NEXT:   EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.0>, ir<false>
+; CHECK-NEXT:   EMIT vp<[[MASK2:%.+]]> = logical-and vp<[[MASK1]]>, ir<%c.0>
 ; CHECK-NEXT: Successor(s): pred.load
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -463,10 +463,10 @@ define void @pred_cfg2(i32 %k, i32 %j) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: then.0.0:
 ; CHECK-NEXT:   EMIT vp<[[NOT:%.+]]> = not ir<%c.0>
-; CHECK-NEXT:   EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
+; CHECK-NEXT:   EMIT vp<[[MASK3:%.+]]> = logical-and vp<[[MASK1]]>, vp<[[NOT]]>
 ; CHECK-NEXT:   EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
 ; CHECK-NEXT:   BLEND ir<%p> = ir<0> vp<[[PRED]]>/vp<[[MASK2]]>
-; CHECK-NEXT:   EMIT vp<[[MASK4:%.+]]> = select vp<[[OR]]>, ir<%c.1>, ir<false>
+; CHECK-NEXT:   EMIT vp<[[MASK4:%.+]]> = logical-and vp<[[OR]]>, ir<%c.1>
 ; CHECK-NEXT: Successor(s): pred.store
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -549,7 +549,7 @@ define void @pred_cfg3(i32 %k, i32 %j) {
 ; CHECK-NEXT:   EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>
 ; CHECK-NEXT:   WIDEN ir<%mul> = mul ir<%iv>, ir<10>
 ; CHECK-NEXT:   WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT:   EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]>, ir<%c.0>, ir<false>
+; CHECK-NEXT:   EMIT vp<[[MASK2:%.+]]> = logical-and vp<[[MASK1:%.+]]>, ir<%c.0>
 ; CHECK-NEXT: Successor(s): pred.load
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -571,10 +571,10 @@ define void @pred_cfg3(i32 %k, i32 %j) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: then.0.0:
 ; CHECK-NEXT:   EMIT vp<[[NOT:%.+]]> = not ir<%c.0>
-; CHECK-NEXT:   EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir<false>
+; CHECK-NEXT:   EMIT vp<[[MASK3:%.+]]> = logical-and vp<[[MASK1]]>, vp<[[NOT]]>
 ; CHECK-NEXT:   EMIT vp<[[MASK4:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]>
 ; CHECK-NEXT:   BLEND ir<%p> = ir<0> vp<[[PRED]]>/vp<[[MASK2]]>
-; CHECK-NEXT:   EMIT vp<[[MASK5:%.+]]> = select vp<[[MASK4]]>, ir<%c.0>, ir<false>
+; CHECK-NEXT:   EMIT vp<[[MASK5:%.+]]> = logical-and vp<[[MASK4]]>, ir<%c.0>
 ; CHECK-NEXT: Successor(s): pred.store
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -683,7 +683,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: loop.3:
 ; CHECK-NEXT:   WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT:   EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, ir<%c.0>, ir<false>
+; CHECK-NEXT:   EMIT vp<[[MASK2:%.+]]> = logical-and vp<[[MASK]]>, ir<%c.0>
 ; CHECK-NEXT:   WIDEN ir<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]>
 ; CHECK-NEXT: Successor(s): pred.store
 ; CHECK-EMPTY:
-- 
GitLab


From d9be51ce68b743bde4d73b6858c454e09df341c5 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye@arm.com>
Date: Tue, 14 May 2024 09:49:53 +0100
Subject: [PATCH 0698/1206] [AArch64] Improve code generation for
 experimental.cttz.elts (#91505)

This patch extends support for lowering the experimental.cttz.elts
intrinsic to BRKB + CNTP instruction sequences, using this lowering for
all legal predicate types. An unused parameter is also removed from some
of the related regression tests.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |   7 +-
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  36 +++
 .../Analysis/CostModel/AArch64/cttz_elts.ll   |  72 +++---
 .../AArch64/intrinsic-cttz-elts-sve.ll        | 212 +++++++++++++-----
 4 files changed, 237 insertions(+), 90 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2aa328e0a127..33cc8ffaf85d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1868,7 +1868,12 @@ bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
 }
 
 bool AArch64TargetLowering::shouldExpandCttzElements(EVT VT) const {
-  return !Subtarget->hasSVEorSME() || VT != MVT::nxv16i1;
+  if (!Subtarget->hasSVEorSME())
+    return true;
+
+  // We can only use the BRKB + CNTP sequence with legal predicate types.
+  return VT != MVT::nxv16i1 && VT != MVT::nxv8i1 && VT != MVT::nxv4i1 &&
+         VT != MVT::nxv2i1;
 }
 
 void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index d4405a230613..bd5de628d852 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2082,6 +2082,18 @@ let Predicates = [HasSVEorSME] in {
   def : Pat<(i64 (AArch64CttzElts nxv16i1:$Op1)),
             (CNTP_XPP_B (BRKB_PPzP (PTRUE_B 31), PPR:$Op1),
                         (BRKB_PPzP (PTRUE_B 31), PPR:$Op1))>;
+
+  def : Pat<(i64 (AArch64CttzElts nxv8i1:$Op1)),
+            (CNTP_XPP_H (BRKB_PPzP (PTRUE_H 31), PPR:$Op1),
+                        (BRKB_PPzP (PTRUE_H 31), PPR:$Op1))>;
+
+  def : Pat<(i64 (AArch64CttzElts nxv4i1:$Op1)),
+            (CNTP_XPP_S (BRKB_PPzP (PTRUE_S 31), PPR:$Op1),
+                        (BRKB_PPzP (PTRUE_S 31), PPR:$Op1))>;
+
+  def : Pat<(i64 (AArch64CttzElts nxv2i1:$Op1)),
+            (CNTP_XPP_D (BRKB_PPzP (PTRUE_D 31), PPR:$Op1),
+                        (BRKB_PPzP (PTRUE_D 31), PPR:$Op1))>;
 }
 
   defm INCB_XPiI : sve_int_pred_pattern_a<0b000, "incb", add, int_aarch64_sve_cntb>;
@@ -2175,6 +2187,30 @@ let Predicates = [HasSVEorSME] in {
                                        (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
                             sub_32)>;
 
+  def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv8i1:$Op2)))),
+            (INCP_XP_H (BRKB_PPzP (PTRUE_H 31), PPR:$Op2), GPR64:$Op1)>;
+
+  def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv8i1:$Op2))))),
+            (EXTRACT_SUBREG (INCP_XP_H (BRKB_PPzP (PTRUE_H 31), PPR:$Op2),
+                                       (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
+                            sub_32)>;
+
+  def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv4i1:$Op2)))),
+            (INCP_XP_S (BRKB_PPzP (PTRUE_S 31), PPR:$Op2), GPR64:$Op1)>;
+
+  def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv4i1:$Op2))))),
+            (EXTRACT_SUBREG (INCP_XP_S (BRKB_PPzP (PTRUE_S 31), PPR:$Op2),
+                                       (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
+                            sub_32)>;
+
+  def : Pat<(i64 (add GPR64:$Op1, (i64 (AArch64CttzElts nxv2i1:$Op2)))),
+            (INCP_XP_D (BRKB_PPzP (PTRUE_D 31), PPR:$Op2), GPR64:$Op1)>;
+
+  def : Pat<(i32 (add GPR32:$Op1, (trunc (i64 (AArch64CttzElts nxv2i1:$Op2))))),
+            (EXTRACT_SUBREG (INCP_XP_D (BRKB_PPzP (PTRUE_D 31), PPR:$Op2),
+                                       (INSERT_SUBREG (IMPLICIT_DEF), GPR32:$Op1, sub_32)),
+                            sub_32)>;
+
   defm INDEX_RR : sve_int_index_rr<"index", AArch64mul_p_oneuse>;
   defm INDEX_IR : sve_int_index_ir<"index", AArch64mul_p, AArch64mul_p_oneuse>;
   defm INDEX_RI : sve_int_index_ri<"index">;
diff --git a/llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll b/llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll
index 01dc086d9385..cc1532ee33dc 100644
--- a/llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/cttz_elts.ll
@@ -3,14 +3,14 @@
 
 define void @foo_no_vscale_range() {
 ; CHECK-LABEL: 'foo_no_vscale_range'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.v2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> undef, i1 true)
@@ -23,14 +23,14 @@ define void @foo_no_vscale_range() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.v8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.v16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %res.i32.v32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.v32i1(<32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.v2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> undef, i1 false)
@@ -95,24 +95,24 @@ define void @foo_no_vscale_range() {
 
 define void @foo_vscale_range_1_16() vscale_range(1,16) {
 ; CHECK-LABEL: 'foo_vscale_range_1_16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
@@ -144,24 +144,24 @@ define void @foo_vscale_range_1_16() vscale_range(1,16) {
 
 define void @foo_vscale_range_1_16384() vscale_range(1,16384) {
 ; CHECK-LABEL: 'foo_vscale_range_1_16384'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
index 9bd2ed240810..211237542a15 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
@@ -4,25 +4,6 @@
 
 ; WITH VSCALE RANGE
 
-define i64 @ctz_nxv8i1(<vscale x 8 x i1> %a) #0 {
-; CHECK-LABEL: ctz_nxv8i1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    index z0.h, #0, #-1
-; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    cnth x9
-; CHECK-NEXT:    inch z0.h
-; CHECK-NEXT:    and z0.d, z0.d, z1.d
-; CHECK-NEXT:    and z0.h, z0.h, #0xff
-; CHECK-NEXT:    umaxv h0, p0, z0.h
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    sub w8, w9, w8
-; CHECK-NEXT:    and x0, x8, #0xff
-; CHECK-NEXT:    ret
-  %res = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> %a, i1 0)
-  ret i64 %res
-}
-
 define i32 @ctz_nxv32i1(<vscale x 32 x i1> %a) #0 {
 ; CHECK-LABEL: ctz_nxv32i1:
 ; CHECK:       // %bb.0:
@@ -156,41 +137,166 @@ define i64 @vscale_4096_poison(<vscale x 16 x i8> %a) #1 {
   ret i64 %res
 }
 
-; NO VSCALE RANGE
+; EFFICIENT LOWERING USING BRKB
 
-define i32 @ctz_nxv8i1_no_range(<vscale x 8 x i1> %a) {
-; CHECK-LABEL: ctz_nxv8i1_no_range:
+define i32 @ctz_nxv2i1(<vscale x 2 x i1> %a) {
+; CHECK-LABEL: ctz_nxv2i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    index z0.s, #0, #-1
-; CHECK-NEXT:    cntw x8
-; CHECK-NEXT:    punpklo p1.h, p0.b
-; CHECK-NEXT:    neg x8, x8
-; CHECK-NEXT:    punpkhi p0.h, p0.b
-; CHECK-NEXT:    cnth x9
-; CHECK-NEXT:    mov z1.s, w8
-; CHECK-NEXT:    mov z2.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z3.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    incw z0.s, all, mul #2
-; CHECK-NEXT:    add z1.s, z0.s, z1.s
-; CHECK-NEXT:    and z0.d, z0.d, z2.d
-; CHECK-NEXT:    and z1.d, z1.d, z3.d
-; CHECK-NEXT:    umax z0.s, p0/m, z0.s, z1.s
-; CHECK-NEXT:    umaxv s0, p0, z0.s
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    sub w0, w9, w8
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    cntp x0, p0, p0.d
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> %a, i1 0)
+  ret i32 %res
+}
+
+define i32 @ctz_nxv2i1_poison(<vscale x 2 x i1> %a) {
+; CHECK-LABEL: ctz_nxv2i1_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    cntp x0, p0, p0.d
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> %a, i1 1)
+  ret i32 %res
+}
+
+define i64 @add_i64_ctz_nxv2i1_poison(<vscale x 2 x i1> %a, i64 %b) {
+; CHECK-LABEL: add_i64_ctz_nxv2i1_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    incp x0, p0.d
+; CHECK-NEXT:    ret
+  %res = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> %a, i1 1)
+  %add = add i64 %res, %b
+  ret i64 %add
+}
+
+define i32 @add_i32_ctz_nxv2i1_poison(<vscale x 2 x i1> %a, i32 %b) {
+; CHECK-LABEL: add_i32_ctz_nxv2i1_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    incp x0, p0.d
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %res = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> %a, i1 1)
+  %trunc = trunc i64 %res to i32
+  %add = add i32 %trunc, %b
+  ret i32 %add
+}
+
+define i32 @ctz_nxv4i1(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: ctz_nxv4i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    cntp x0, p0, p0.s
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> %a, i1 0)
+  ret i32 %res
+}
+
+define i32 @ctz_nxv4i1_poison(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: ctz_nxv4i1_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    cntp x0, p0, p0.s
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> %a, i1 1)
+  ret i32 %res
+}
+
+define i64 @add_i64_ctz_nxv4i1_poison(<vscale x 4 x i1> %a, i64 %b) {
+; CHECK-LABEL: add_i64_ctz_nxv4i1_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    incp x0, p0.s
+; CHECK-NEXT:    ret
+  %res = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> %a, i1 1)
+  %add = add i64 %res, %b
+  ret i64 %add
+}
+
+define i32 @add_i32_ctz_nxv4i1_poison(<vscale x 4 x i1> %a, i32 %b) {
+; CHECK-LABEL: add_i32_ctz_nxv4i1_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    incp x0, p0.s
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %res = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> %a, i1 1)
+  %trunc = trunc i64 %res to i32
+  %add = add i32 %trunc, %b
+  ret i32 %add
+}
+
+define i32 @ctz_nxv8i1(<vscale x 8 x i1> %a) {
+; CHECK-LABEL: ctz_nxv8i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    cntp x0, p0, p0.h
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
   %res = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> %a, i1 0)
   ret i32 %res
 }
 
-; MATCH WITH BRKB + CNTP
+define i32 @ctz_nxv8i1_poison(<vscale x 8 x i1> %a) {
+; CHECK-LABEL: ctz_nxv8i1_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    cntp x0, p0, p0.h
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> %a, i1 1)
+  ret i32 %res
+}
+
+define i64 @add_i64_ctz_nxv8i1_poison(<vscale x 8 x i1> %a, i64 %b) {
+; CHECK-LABEL: add_i64_ctz_nxv8i1_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    incp x0, p0.h
+; CHECK-NEXT:    ret
+  %res = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> %a, i1 1)
+  %add = add i64 %res, %b
+  ret i64 %add
+}
 
-define i32 @ctz_nxv16i1(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
+define i32 @add_i32_ctz_nxv8i1_poison(<vscale x 8 x i1> %a, i32 %b) {
+; CHECK-LABEL: add_i32_ctz_nxv8i1_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
+; CHECK-NEXT:    incp x0, p0.h
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %res = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> %a, i1 1)
+  %trunc = trunc i64 %res to i32
+  %add = add i32 %trunc, %b
+  ret i32 %add
+}
+
+define i32 @ctz_nxv16i1(<vscale x 16 x i1> %a) {
 ; CHECK-LABEL: ctz_nxv16i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    brkb p0.b, p0/z, p1.b
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
 ; CHECK-NEXT:    cntp x0, p0, p0.b
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
@@ -198,11 +304,11 @@ define i32 @ctz_nxv16i1(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
   ret i32 %res
 }
 
-define i32 @ctz_nxv16i1_poison(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
+define i32 @ctz_nxv16i1_poison(<vscale x 16 x i1> %a) {
 ; CHECK-LABEL: ctz_nxv16i1_poison:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    brkb p0.b, p0/z, p1.b
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
 ; CHECK-NEXT:    cntp x0, p0, p0.b
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
@@ -226,11 +332,11 @@ define i32 @ctz_and_nxv16i1(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vsca
   ret i32 %res
 }
 
-define i64 @add_i64_ctz_nxv16i1_poison(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a, i64 %b) {
+define i64 @add_i64_ctz_nxv16i1_poison(<vscale x 16 x i1> %a, i64 %b) {
 ; CHECK-LABEL: add_i64_ctz_nxv16i1_poison:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    brkb p0.b, p0/z, p1.b
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
 ; CHECK-NEXT:    incp x0, p0.b
 ; CHECK-NEXT:    ret
   %res = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> %a, i1 1)
@@ -238,12 +344,12 @@ define i64 @add_i64_ctz_nxv16i1_poison(<vscale x 16 x i1> %pg, <vscale x 16 x i1
   ret i64 %add
 }
 
-define i32 @add_i32_ctz_nxv16i1_poison(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a, i32 %b) {
+define i32 @add_i32_ctz_nxv16i1_poison(<vscale x 16 x i1> %a, i32 %b) {
 ; CHECK-LABEL: add_i32_ctz_nxv16i1_poison:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    ptrue p1.b
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    brkb p0.b, p0/z, p1.b
+; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
 ; CHECK-NEXT:    incp x0, p0.b
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
-- 
GitLab


From 0bc23f10328e9f61200c33c02391a44abde59b27 Mon Sep 17 00:00:00 2001
From: chuongg3 <chuong.goh@arm.com>
Date: Tue, 14 May 2024 10:04:21 +0100
Subject: [PATCH 0699/1206] [AArch64][GlobalISel] Select G_ICMP Zero
 Instruction (#90054)

---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |   46 +
 llvm/test/CodeGen/AArch64/aarch64-addv.ll     |   25 +-
 llvm/test/CodeGen/AArch64/arm64-vabs.ll       |  227 ++--
 llvm/test/CodeGen/AArch64/icmp.ll             | 1070 +++++------------
 .../AArch64/neon-bitwise-instructions.ll      |   43 +-
 .../AArch64/neon-compare-instructions.ll      |  532 +++-----
 6 files changed, 640 insertions(+), 1303 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index bb32280fe51f..a39e3b7be76d 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -5449,6 +5449,52 @@ defm : SelectSetCCSwapOperands<setle, "CMGE">;
 defm : SelectSetCCSwapOperands<setult, "CMHI">;
 defm : SelectSetCCSwapOperands<setule, "CMHS">;
 
+multiclass SelectSetCCZeroRHS<PatFrags InFrag, string INST> {
+  def : Pat<(v8i8 (InFrag (v8i8 V64:$Rn), immAllZerosV)),
+            (v8i8 (!cast<Instruction>(INST # v8i8rz) (v8i8 V64:$Rn)))>;
+  def : Pat<(v16i8 (InFrag (v16i8 V128:$Rn), immAllZerosV)),
+            (v16i8 (!cast<Instruction>(INST # v16i8rz) (v16i8 V128:$Rn)))>;
+  def : Pat<(v4i16 (InFrag (v4i16 V64:$Rn), immAllZerosV)),
+            (v4i16 (!cast<Instruction>(INST # v4i16rz) (v4i16 V64:$Rn)))>;
+  def : Pat<(v8i16 (InFrag (v8i16 V128:$Rn), immAllZerosV)),
+            (v8i16 (!cast<Instruction>(INST # v8i16rz) (v8i16 V128:$Rn)))>;
+  def : Pat<(v2i32 (InFrag (v2i32 V64:$Rn), immAllZerosV)),
+            (v2i32 (!cast<Instruction>(INST # v2i32rz) (v2i32 V64:$Rn)))>;
+  def : Pat<(v4i32 (InFrag (v4i32 V128:$Rn), immAllZerosV)),
+            (v4i32 (!cast<Instruction>(INST # v4i32rz) (v4i32 V128:$Rn)))>;
+  def : Pat<(v2i64 (InFrag (v2i64 V128:$Rn), immAllZerosV)),
+            (v2i64 (!cast<Instruction>(INST # v2i64rz) (v2i64 V128:$Rn)))>;
+}
+
+defm : SelectSetCCZeroRHS<seteq, "CMEQ">;
+defm : SelectSetCCZeroRHS<setgt, "CMGT">;
+defm : SelectSetCCZeroRHS<setge, "CMGE">;
+defm : SelectSetCCZeroRHS<setlt, "CMLT">;
+defm : SelectSetCCZeroRHS<setle, "CMLE">;
+
+multiclass SelectSetCCZeroLHS<PatFrags InFrag, string INST> {
+  def : Pat<(v8i8 (InFrag immAllZerosV, (v8i8 V64:$Rn))),
+            (v8i8 (!cast<Instruction>(INST # v8i8rz) (v8i8 V64:$Rn)))>;
+  def : Pat<(v16i8 (InFrag immAllZerosV, (v16i8 V128:$Rn))),
+            (v16i8 (!cast<Instruction>(INST # v16i8rz) (v16i8 V128:$Rn)))>;
+  def : Pat<(v4i16 (InFrag immAllZerosV, (v4i16 V64:$Rn))),
+            (v4i16 (!cast<Instruction>(INST # v4i16rz) (v4i16 V64:$Rn)))>;
+  def : Pat<(v8i16 (InFrag immAllZerosV, (v8i16 V128:$Rn))),
+            (v8i16 (!cast<Instruction>(INST # v8i16rz) (v8i16 V128:$Rn)))>;
+  def : Pat<(v2i32 (InFrag immAllZerosV, (v2i32 V64:$Rn))),
+            (v2i32 (!cast<Instruction>(INST # v2i32rz) (v2i32 V64:$Rn)))>;
+  def : Pat<(v4i32 (InFrag immAllZerosV, (v4i32 V128:$Rn))),
+            (v4i32 (!cast<Instruction>(INST # v4i32rz) (v4i32 V128:$Rn)))>;
+  def : Pat<(v2i64 (InFrag immAllZerosV, (v2i64 V128:$Rn))),
+            (v2i64 (!cast<Instruction>(INST # v2i64rz) (v2i64 V128:$Rn)))>;
+}
+
+defm : SelectSetCCZeroLHS<seteq, "CMEQ">;
+defm : SelectSetCCZeroLHS<setgt, "CMLT">;
+defm : SelectSetCCZeroLHS<setge, "CMLE">;
+defm : SelectSetCCZeroLHS<setlt, "CMGT">;
+defm : SelectSetCCZeroLHS<setle, "CMGE">;
+
 let Predicates = [HasNEON] in {
 def : InstAlias<"mov{\t$dst.16b, $src.16b|.16b\t$dst, $src}",
                 (ORRv16i8 V128:$dst, V128:$src, V128:$src), 1>;
diff --git a/llvm/test/CodeGen/AArch64/aarch64-addv.ll b/llvm/test/CodeGen/AArch64/aarch64-addv.ll
index ee035ec1941d..94b792b887eb 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-addv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-addv.ll
@@ -94,20 +94,19 @@ define i32 @oversized_ADDV_256(ptr noalias nocapture readonly %arg1, ptr noalias
 ;
 ; GISEL-LABEL: oversized_ADDV_256:
 ; GISEL:       // %bb.0: // %entry
-; GISEL-NEXT:    ldr d1, [x0]
-; GISEL-NEXT:    ldr d2, [x1]
-; GISEL-NEXT:    movi v0.2d, #0000000000000000
+; GISEL-NEXT:    ldr d0, [x0]
+; GISEL-NEXT:    ldr d1, [x1]
+; GISEL-NEXT:    ushll v0.8h, v0.8b, #0
 ; GISEL-NEXT:    ushll v1.8h, v1.8b, #0
-; GISEL-NEXT:    ushll v2.8h, v2.8b, #0
-; GISEL-NEXT:    usubl v3.4s, v1.4h, v2.4h
-; GISEL-NEXT:    usubl2 v1.4s, v1.8h, v2.8h
-; GISEL-NEXT:    cmgt v2.4s, v0.4s, v3.4s
-; GISEL-NEXT:    cmgt v0.4s, v0.4s, v1.4s
-; GISEL-NEXT:    neg v4.4s, v3.4s
-; GISEL-NEXT:    neg v5.4s, v1.4s
-; GISEL-NEXT:    bsl v2.16b, v4.16b, v3.16b
-; GISEL-NEXT:    bsl v0.16b, v5.16b, v1.16b
-; GISEL-NEXT:    add v0.4s, v2.4s, v0.4s
+; GISEL-NEXT:    usubl v2.4s, v0.4h, v1.4h
+; GISEL-NEXT:    usubl2 v0.4s, v0.8h, v1.8h
+; GISEL-NEXT:    cmlt v1.4s, v2.4s, #0
+; GISEL-NEXT:    cmlt v3.4s, v0.4s, #0
+; GISEL-NEXT:    neg v4.4s, v2.4s
+; GISEL-NEXT:    neg v5.4s, v0.4s
+; GISEL-NEXT:    bsl v1.16b, v4.16b, v2.16b
+; GISEL-NEXT:    bit v0.16b, v5.16b, v3.16b
+; GISEL-NEXT:    add v0.4s, v1.4s, v0.4s
 ; GISEL-NEXT:    addv s0, v0.4s
 ; GISEL-NEXT:    fmov w0, s0
 ; GISEL-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/arm64-vabs.ll b/llvm/test/CodeGen/AArch64/arm64-vabs.ll
index d64327656a9e..f7d31a214563 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vabs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vabs.ll
@@ -252,18 +252,17 @@ define i16 @uabd16b_rdx(ptr %a, ptr %b) {
 ;
 ; CHECK-GI-LABEL: uabd16b_rdx:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    ldr q1, [x0]
-; CHECK-GI-NEXT:    ldr q2, [x1]
-; CHECK-GI-NEXT:    movi.2d v0, #0000000000000000
-; CHECK-GI-NEXT:    usubl.8h v3, v1, v2
-; CHECK-GI-NEXT:    usubl2.8h v1, v1, v2
-; CHECK-GI-NEXT:    cmgt.8h v2, v0, v3
-; CHECK-GI-NEXT:    cmgt.8h v0, v0, v1
-; CHECK-GI-NEXT:    neg.8h v4, v3
-; CHECK-GI-NEXT:    neg.8h v5, v1
-; CHECK-GI-NEXT:    bsl.16b v2, v4, v3
-; CHECK-GI-NEXT:    bsl.16b v0, v5, v1
-; CHECK-GI-NEXT:    add.8h v0, v2, v0
+; CHECK-GI-NEXT:    ldr q0, [x0]
+; CHECK-GI-NEXT:    ldr q1, [x1]
+; CHECK-GI-NEXT:    usubl.8h v2, v0, v1
+; CHECK-GI-NEXT:    usubl2.8h v0, v0, v1
+; CHECK-GI-NEXT:    cmlt.8h v1, v2, #0
+; CHECK-GI-NEXT:    cmlt.8h v3, v0, #0
+; CHECK-GI-NEXT:    neg.8h v4, v2
+; CHECK-GI-NEXT:    neg.8h v5, v0
+; CHECK-GI-NEXT:    bsl.16b v1, v4, v2
+; CHECK-GI-NEXT:    bit.16b v0, v5, v3
+; CHECK-GI-NEXT:    add.8h v0, v1, v0
 ; CHECK-GI-NEXT:    addv.8h h0, v0
 ; CHECK-GI-NEXT:    fmov w0, s0
 ; CHECK-GI-NEXT:    ret
@@ -290,29 +289,28 @@ define i32 @uabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) {
 ;
 ; CHECK-GI-LABEL: uabd16b_rdx_i32:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    ushll.8h v3, v0, #0
-; CHECK-GI-NEXT:    ushll.8h v4, v1, #0
+; CHECK-GI-NEXT:    ushll.8h v2, v0, #0
+; CHECK-GI-NEXT:    ushll.8h v3, v1, #0
 ; CHECK-GI-NEXT:    ushll2.8h v0, v0, #0
 ; CHECK-GI-NEXT:    ushll2.8h v1, v1, #0
-; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
-; CHECK-GI-NEXT:    usubl.4s v5, v3, v4
-; CHECK-GI-NEXT:    usubl2.4s v3, v3, v4
-; CHECK-GI-NEXT:    usubl.4s v4, v0, v1
+; CHECK-GI-NEXT:    usubl.4s v4, v2, v3
+; CHECK-GI-NEXT:    usubl2.4s v2, v2, v3
+; CHECK-GI-NEXT:    usubl.4s v3, v0, v1
 ; CHECK-GI-NEXT:    usubl2.4s v0, v0, v1
-; CHECK-GI-NEXT:    cmgt.4s v1, v2, v5
-; CHECK-GI-NEXT:    cmgt.4s v6, v2, v3
-; CHECK-GI-NEXT:    neg.4s v16, v5
-; CHECK-GI-NEXT:    cmgt.4s v7, v2, v4
-; CHECK-GI-NEXT:    cmgt.4s v2, v2, v0
-; CHECK-GI-NEXT:    neg.4s v17, v3
-; CHECK-GI-NEXT:    neg.4s v18, v4
+; CHECK-GI-NEXT:    cmlt.4s v1, v4, #0
+; CHECK-GI-NEXT:    cmlt.4s v5, v2, #0
+; CHECK-GI-NEXT:    neg.4s v16, v4
+; CHECK-GI-NEXT:    cmlt.4s v6, v3, #0
+; CHECK-GI-NEXT:    cmlt.4s v7, v0, #0
+; CHECK-GI-NEXT:    neg.4s v17, v2
+; CHECK-GI-NEXT:    neg.4s v18, v3
 ; CHECK-GI-NEXT:    neg.4s v19, v0
-; CHECK-GI-NEXT:    bsl.16b v1, v16, v5
-; CHECK-GI-NEXT:    bit.16b v3, v17, v6
-; CHECK-GI-NEXT:    bit.16b v4, v18, v7
-; CHECK-GI-NEXT:    bit.16b v0, v19, v2
-; CHECK-GI-NEXT:    add.4s v1, v1, v3
-; CHECK-GI-NEXT:    add.4s v0, v4, v0
+; CHECK-GI-NEXT:    bsl.16b v1, v16, v4
+; CHECK-GI-NEXT:    bit.16b v2, v17, v5
+; CHECK-GI-NEXT:    bit.16b v3, v18, v6
+; CHECK-GI-NEXT:    bit.16b v0, v19, v7
+; CHECK-GI-NEXT:    add.4s v1, v1, v2
+; CHECK-GI-NEXT:    add.4s v0, v3, v0
 ; CHECK-GI-NEXT:    add.4s v0, v1, v0
 ; CHECK-GI-NEXT:    addv.4s s0, v0
 ; CHECK-GI-NEXT:    fmov w0, s0
@@ -338,29 +336,28 @@ define i32 @sabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) {
 ;
 ; CHECK-GI-LABEL: sabd16b_rdx_i32:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    sshll.8h v3, v0, #0
-; CHECK-GI-NEXT:    sshll.8h v4, v1, #0
+; CHECK-GI-NEXT:    sshll.8h v2, v0, #0
+; CHECK-GI-NEXT:    sshll.8h v3, v1, #0
 ; CHECK-GI-NEXT:    sshll2.8h v0, v0, #0
 ; CHECK-GI-NEXT:    sshll2.8h v1, v1, #0
-; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
-; CHECK-GI-NEXT:    ssubl.4s v5, v3, v4
-; CHECK-GI-NEXT:    ssubl2.4s v3, v3, v4
-; CHECK-GI-NEXT:    ssubl.4s v4, v0, v1
+; CHECK-GI-NEXT:    ssubl.4s v4, v2, v3
+; CHECK-GI-NEXT:    ssubl2.4s v2, v2, v3
+; CHECK-GI-NEXT:    ssubl.4s v3, v0, v1
 ; CHECK-GI-NEXT:    ssubl2.4s v0, v0, v1
-; CHECK-GI-NEXT:    cmgt.4s v1, v2, v5
-; CHECK-GI-NEXT:    cmgt.4s v6, v2, v3
-; CHECK-GI-NEXT:    neg.4s v16, v5
-; CHECK-GI-NEXT:    cmgt.4s v7, v2, v4
-; CHECK-GI-NEXT:    cmgt.4s v2, v2, v0
-; CHECK-GI-NEXT:    neg.4s v17, v3
-; CHECK-GI-NEXT:    neg.4s v18, v4
+; CHECK-GI-NEXT:    cmlt.4s v1, v4, #0
+; CHECK-GI-NEXT:    cmlt.4s v5, v2, #0
+; CHECK-GI-NEXT:    neg.4s v16, v4
+; CHECK-GI-NEXT:    cmlt.4s v6, v3, #0
+; CHECK-GI-NEXT:    cmlt.4s v7, v0, #0
+; CHECK-GI-NEXT:    neg.4s v17, v2
+; CHECK-GI-NEXT:    neg.4s v18, v3
 ; CHECK-GI-NEXT:    neg.4s v19, v0
-; CHECK-GI-NEXT:    bsl.16b v1, v16, v5
-; CHECK-GI-NEXT:    bit.16b v3, v17, v6
-; CHECK-GI-NEXT:    bit.16b v4, v18, v7
-; CHECK-GI-NEXT:    bit.16b v0, v19, v2
-; CHECK-GI-NEXT:    add.4s v1, v1, v3
-; CHECK-GI-NEXT:    add.4s v0, v4, v0
+; CHECK-GI-NEXT:    bsl.16b v1, v16, v4
+; CHECK-GI-NEXT:    bit.16b v2, v17, v5
+; CHECK-GI-NEXT:    bit.16b v3, v18, v6
+; CHECK-GI-NEXT:    bit.16b v0, v19, v7
+; CHECK-GI-NEXT:    add.4s v1, v1, v2
+; CHECK-GI-NEXT:    add.4s v0, v3, v0
 ; CHECK-GI-NEXT:    add.4s v0, v1, v0
 ; CHECK-GI-NEXT:    addv.4s s0, v0
 ; CHECK-GI-NEXT:    fmov w0, s0
@@ -391,18 +388,17 @@ define i32 @uabd8h_rdx(ptr %a, ptr %b) {
 ;
 ; CHECK-GI-LABEL: uabd8h_rdx:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    ldr q1, [x0]
-; CHECK-GI-NEXT:    ldr q2, [x1]
-; CHECK-GI-NEXT:    movi.2d v0, #0000000000000000
-; CHECK-GI-NEXT:    usubl.4s v3, v1, v2
-; CHECK-GI-NEXT:    usubl2.4s v1, v1, v2
-; CHECK-GI-NEXT:    cmgt.4s v2, v0, v3
-; CHECK-GI-NEXT:    cmgt.4s v0, v0, v1
-; CHECK-GI-NEXT:    neg.4s v4, v3
-; CHECK-GI-NEXT:    neg.4s v5, v1
-; CHECK-GI-NEXT:    bsl.16b v2, v4, v3
-; CHECK-GI-NEXT:    bsl.16b v0, v5, v1
-; CHECK-GI-NEXT:    add.4s v0, v2, v0
+; CHECK-GI-NEXT:    ldr q0, [x0]
+; CHECK-GI-NEXT:    ldr q1, [x1]
+; CHECK-GI-NEXT:    usubl.4s v2, v0, v1
+; CHECK-GI-NEXT:    usubl2.4s v0, v0, v1
+; CHECK-GI-NEXT:    cmlt.4s v1, v2, #0
+; CHECK-GI-NEXT:    cmlt.4s v3, v0, #0
+; CHECK-GI-NEXT:    neg.4s v4, v2
+; CHECK-GI-NEXT:    neg.4s v5, v0
+; CHECK-GI-NEXT:    bsl.16b v1, v4, v2
+; CHECK-GI-NEXT:    bit.16b v0, v5, v3
+; CHECK-GI-NEXT:    add.4s v0, v1, v0
 ; CHECK-GI-NEXT:    addv.4s s0, v0
 ; CHECK-GI-NEXT:    fmov w0, s0
 ; CHECK-GI-NEXT:    ret
@@ -428,15 +424,14 @@ define i32 @sabd8h_rdx(<8 x i16> %a, <8 x i16> %b) {
 ;
 ; CHECK-GI-LABEL: sabd8h_rdx:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
-; CHECK-GI-NEXT:    ssubl.4s v3, v0, v1
+; CHECK-GI-NEXT:    ssubl.4s v2, v0, v1
 ; CHECK-GI-NEXT:    ssubl2.4s v0, v0, v1
-; CHECK-GI-NEXT:    neg.4s v4, v3
+; CHECK-GI-NEXT:    cmlt.4s v1, v2, #0
+; CHECK-GI-NEXT:    cmlt.4s v3, v0, #0
+; CHECK-GI-NEXT:    neg.4s v4, v2
 ; CHECK-GI-NEXT:    neg.4s v5, v0
-; CHECK-GI-NEXT:    cmgt.4s v1, v2, v3
-; CHECK-GI-NEXT:    cmgt.4s v2, v2, v0
-; CHECK-GI-NEXT:    bsl.16b v1, v4, v3
-; CHECK-GI-NEXT:    bit.16b v0, v5, v2
+; CHECK-GI-NEXT:    bsl.16b v1, v4, v2
+; CHECK-GI-NEXT:    bit.16b v0, v5, v3
 ; CHECK-GI-NEXT:    add.4s v0, v1, v0
 ; CHECK-GI-NEXT:    addv.4s s0, v0
 ; CHECK-GI-NEXT:    fmov w0, s0
@@ -461,9 +456,8 @@ define i32 @uabdl4s_rdx_i32(<4 x i16> %a, <4 x i16> %b) {
 ;
 ; CHECK-GI-LABEL: uabdl4s_rdx_i32:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
 ; CHECK-GI-NEXT:    usubl.4s v0, v0, v1
-; CHECK-GI-NEXT:    cmgt.4s v1, v2, v0
+; CHECK-GI-NEXT:    cmlt.4s v1, v0, #0
 ; CHECK-GI-NEXT:    neg.4s v2, v0
 ; CHECK-GI-NEXT:    bit.16b v0, v2, v1
 ; CHECK-GI-NEXT:    addv.4s s0, v0
@@ -494,18 +488,17 @@ define i64 @uabd4s_rdx(ptr %a, ptr %b, i32 %h) {
 ;
 ; CHECK-GI-LABEL: uabd4s_rdx:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    ldr q1, [x0]
-; CHECK-GI-NEXT:    ldr q2, [x1]
-; CHECK-GI-NEXT:    movi.2d v0, #0000000000000000
-; CHECK-GI-NEXT:    usubl.2d v3, v1, v2
-; CHECK-GI-NEXT:    usubl2.2d v1, v1, v2
-; CHECK-GI-NEXT:    cmgt.2d v2, v0, v3
-; CHECK-GI-NEXT:    cmgt.2d v0, v0, v1
-; CHECK-GI-NEXT:    neg.2d v4, v3
-; CHECK-GI-NEXT:    neg.2d v5, v1
-; CHECK-GI-NEXT:    bsl.16b v2, v4, v3
-; CHECK-GI-NEXT:    bsl.16b v0, v5, v1
-; CHECK-GI-NEXT:    add.2d v0, v2, v0
+; CHECK-GI-NEXT:    ldr q0, [x0]
+; CHECK-GI-NEXT:    ldr q1, [x1]
+; CHECK-GI-NEXT:    usubl.2d v2, v0, v1
+; CHECK-GI-NEXT:    usubl2.2d v0, v0, v1
+; CHECK-GI-NEXT:    cmlt.2d v1, v2, #0
+; CHECK-GI-NEXT:    cmlt.2d v3, v0, #0
+; CHECK-GI-NEXT:    neg.2d v4, v2
+; CHECK-GI-NEXT:    neg.2d v5, v0
+; CHECK-GI-NEXT:    bsl.16b v1, v4, v2
+; CHECK-GI-NEXT:    bit.16b v0, v5, v3
+; CHECK-GI-NEXT:    add.2d v0, v1, v0
 ; CHECK-GI-NEXT:    addp.2d d0, v0
 ; CHECK-GI-NEXT:    fmov x0, d0
 ; CHECK-GI-NEXT:    ret
@@ -531,15 +524,14 @@ define i64 @sabd4s_rdx(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; CHECK-GI-LABEL: sabd4s_rdx:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
-; CHECK-GI-NEXT:    ssubl.2d v3, v0, v1
+; CHECK-GI-NEXT:    ssubl.2d v2, v0, v1
 ; CHECK-GI-NEXT:    ssubl2.2d v0, v0, v1
-; CHECK-GI-NEXT:    neg.2d v4, v3
+; CHECK-GI-NEXT:    cmlt.2d v1, v2, #0
+; CHECK-GI-NEXT:    cmlt.2d v3, v0, #0
+; CHECK-GI-NEXT:    neg.2d v4, v2
 ; CHECK-GI-NEXT:    neg.2d v5, v0
-; CHECK-GI-NEXT:    cmgt.2d v1, v2, v3
-; CHECK-GI-NEXT:    cmgt.2d v2, v2, v0
-; CHECK-GI-NEXT:    bsl.16b v1, v4, v3
-; CHECK-GI-NEXT:    bit.16b v0, v5, v2
+; CHECK-GI-NEXT:    bsl.16b v1, v4, v2
+; CHECK-GI-NEXT:    bit.16b v0, v5, v3
 ; CHECK-GI-NEXT:    add.2d v0, v1, v0
 ; CHECK-GI-NEXT:    addp.2d d0, v0
 ; CHECK-GI-NEXT:    fmov x0, d0
@@ -564,9 +556,8 @@ define i64 @uabdl2d_rdx_i64(<2 x i32> %a, <2 x i32> %b) {
 ;
 ; CHECK-GI-LABEL: uabdl2d_rdx_i64:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
 ; CHECK-GI-NEXT:    usubl.2d v0, v0, v1
-; CHECK-GI-NEXT:    cmgt.2d v1, v2, v0
+; CHECK-GI-NEXT:    cmlt.2d v1, v0, #0
 ; CHECK-GI-NEXT:    neg.2d v2, v0
 ; CHECK-GI-NEXT:    bit.16b v0, v2, v1
 ; CHECK-GI-NEXT:    addp.2d d0, v0
@@ -1662,10 +1653,9 @@ define <2 x i32> @abspattern1(<2 x i32> %a) nounwind {
 ;
 ; CHECK-GI-LABEL: abspattern1:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v1, #0000000000000000
-; CHECK-GI-NEXT:    neg.2s v2, v0
-; CHECK-GI-NEXT:    cmge.2s v1, v0, v1
-; CHECK-GI-NEXT:    bif.8b v0, v2, v1
+; CHECK-GI-NEXT:    neg.2s v1, v0
+; CHECK-GI-NEXT:    cmge.2s v2, v0, #0
+; CHECK-GI-NEXT:    bif.8b v0, v1, v2
 ; CHECK-GI-NEXT:    ret
   %tmp1neg = sub <2 x i32> zeroinitializer, %a
   %b = icmp sge <2 x i32> %a, zeroinitializer
@@ -1682,10 +1672,9 @@ define <4 x i16> @abspattern2(<4 x i16> %a) nounwind {
 ;
 ; CHECK-GI-LABEL: abspattern2:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v1, #0000000000000000
-; CHECK-GI-NEXT:    neg.4h v2, v0
-; CHECK-GI-NEXT:    cmgt.4h v1, v0, v1
-; CHECK-GI-NEXT:    bif.8b v0, v2, v1
+; CHECK-GI-NEXT:    neg.4h v1, v0
+; CHECK-GI-NEXT:    cmgt.4h v2, v0, #0
+; CHECK-GI-NEXT:    bif.8b v0, v1, v2
 ; CHECK-GI-NEXT:    ret
   %tmp1neg = sub <4 x i16> zeroinitializer, %a
   %b = icmp sgt <4 x i16> %a, zeroinitializer
@@ -1701,10 +1690,9 @@ define <8 x i8> @abspattern3(<8 x i8> %a) nounwind {
 ;
 ; CHECK-GI-LABEL: abspattern3:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v1, #0000000000000000
-; CHECK-GI-NEXT:    neg.8b v2, v0
-; CHECK-GI-NEXT:    cmgt.8b v1, v1, v0
-; CHECK-GI-NEXT:    bit.8b v0, v2, v1
+; CHECK-GI-NEXT:    neg.8b v1, v0
+; CHECK-GI-NEXT:    cmlt.8b v2, v0, #0
+; CHECK-GI-NEXT:    bit.8b v0, v1, v2
 ; CHECK-GI-NEXT:    ret
   %tmp1neg = sub <8 x i8> zeroinitializer, %a
   %b = icmp slt <8 x i8> %a, zeroinitializer
@@ -1720,10 +1708,9 @@ define <4 x i32> @abspattern4(<4 x i32> %a) nounwind {
 ;
 ; CHECK-GI-LABEL: abspattern4:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v1, #0000000000000000
-; CHECK-GI-NEXT:    neg.4s v2, v0
-; CHECK-GI-NEXT:    cmge.4s v1, v0, v1
-; CHECK-GI-NEXT:    bif.16b v0, v2, v1
+; CHECK-GI-NEXT:    neg.4s v1, v0
+; CHECK-GI-NEXT:    cmge.4s v2, v0, #0
+; CHECK-GI-NEXT:    bif.16b v0, v1, v2
 ; CHECK-GI-NEXT:    ret
   %tmp1neg = sub <4 x i32> zeroinitializer, %a
   %b = icmp sge <4 x i32> %a, zeroinitializer
@@ -1739,10 +1726,9 @@ define <8 x i16> @abspattern5(<8 x i16> %a) nounwind {
 ;
 ; CHECK-GI-LABEL: abspattern5:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v1, #0000000000000000
-; CHECK-GI-NEXT:    neg.8h v2, v0
-; CHECK-GI-NEXT:    cmgt.8h v1, v0, v1
-; CHECK-GI-NEXT:    bif.16b v0, v2, v1
+; CHECK-GI-NEXT:    neg.8h v1, v0
+; CHECK-GI-NEXT:    cmgt.8h v2, v0, #0
+; CHECK-GI-NEXT:    bif.16b v0, v1, v2
 ; CHECK-GI-NEXT:    ret
   %tmp1neg = sub <8 x i16> zeroinitializer, %a
   %b = icmp sgt <8 x i16> %a, zeroinitializer
@@ -1758,10 +1744,9 @@ define <16 x i8> @abspattern6(<16 x i8> %a) nounwind {
 ;
 ; CHECK-GI-LABEL: abspattern6:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v1, #0000000000000000
-; CHECK-GI-NEXT:    neg.16b v2, v0
-; CHECK-GI-NEXT:    cmgt.16b v1, v1, v0
-; CHECK-GI-NEXT:    bit.16b v0, v2, v1
+; CHECK-GI-NEXT:    neg.16b v1, v0
+; CHECK-GI-NEXT:    cmlt.16b v2, v0, #0
+; CHECK-GI-NEXT:    bit.16b v0, v1, v2
 ; CHECK-GI-NEXT:    ret
   %tmp1neg = sub <16 x i8> zeroinitializer, %a
   %b = icmp slt <16 x i8> %a, zeroinitializer
@@ -1777,10 +1762,9 @@ define <2 x i64> @abspattern7(<2 x i64> %a) nounwind {
 ;
 ; CHECK-GI-LABEL: abspattern7:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v1, #0000000000000000
-; CHECK-GI-NEXT:    neg.2d v2, v0
-; CHECK-GI-NEXT:    cmge.2d v1, v1, v0
-; CHECK-GI-NEXT:    bit.16b v0, v2, v1
+; CHECK-GI-NEXT:    neg.2d v1, v0
+; CHECK-GI-NEXT:    cmle.2d v2, v0, #0
+; CHECK-GI-NEXT:    bit.16b v0, v1, v2
 ; CHECK-GI-NEXT:    ret
   %tmp1neg = sub <2 x i64> zeroinitializer, %a
   %b = icmp sle <2 x i64> %a, zeroinitializer
@@ -1796,9 +1780,8 @@ define <2 x i64> @uabd_i32(<2 x i32> %a, <2 x i32> %b) {
 ;
 ; CHECK-GI-LABEL: uabd_i32:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi.2d v2, #0000000000000000
 ; CHECK-GI-NEXT:    ssubl.2d v0, v0, v1
-; CHECK-GI-NEXT:    cmgt.2d v1, v2, v0
+; CHECK-GI-NEXT:    cmlt.2d v1, v0, #0
 ; CHECK-GI-NEXT:    neg.2d v2, v0
 ; CHECK-GI-NEXT:    bit.16b v0, v2, v1
 ; CHECK-GI-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/icmp.ll b/llvm/test/CodeGen/AArch64/icmp.ll
index 88b2f279ec3e..6baf1a84d407 100644
--- a/llvm/test/CodeGen/AArch64/icmp.ll
+++ b/llvm/test/CodeGen/AArch64/icmp.ll
@@ -1379,556 +1379,331 @@ entry:
 ; ===== ICMP Zero RHS =====
 
 define <8 x i1> @icmp_eq_v8i8_Zero_RHS(<8 x i8> %a) {
-; CHECK-SD-LABEL: icmp_eq_v8i8_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v8i8_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v8i8_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
     %c = icmp eq <8 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
     ret <8 x i1> %c
 }
 
 define <16 x i1> @icmp_eq_v16i8_Zero_RHS(<16 x i8> %a) {
-; CHECK-SD-LABEL: icmp_eq_v16i8_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v16i8_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v16i8_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
     %c = icmp eq <16 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
     ret <16 x i1> %c
 }
 
 define <4 x i1> @icmp_eq_v4i16_Zero_RHS(<4 x i16> %a) {
-; CHECK-SD-LABEL: icmp_eq_v4i16_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v4i16_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.4h, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v4i16_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
     %c = icmp eq <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
     ret <4 x i1> %c
 }
 
 define <8 x i1> @icmp_eq_v8i16_Zero_RHS(<8 x i16> %a) {
-; CHECK-SD-LABEL: icmp_eq_v8i16_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v8i16_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v8i16_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
     %c = icmp eq <8 x i16> %a, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
     ret <8 x i1> %c
 }
 
 define <2 x i1> @icmp_eq_v2i32_Zero_RHS(<2 x i32> %a) {
-; CHECK-SD-LABEL: icmp_eq_v2i32_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v2i32_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v2i32_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
     %c = icmp eq <2 x i32> %a, <i32 0, i32 0>
     ret <2 x i1> %c
 }
 
 define <4 x i1> @icmp_eq_v4i32_Zero_RHS(<4 x i32> %a) {
-; CHECK-SD-LABEL: icmp_eq_v4i32_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v4i32_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v4i32_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %c = icmp eq <4 x i32> %a, <i32 0, i32 0, i32 0, i32 0>
     ret <4 x i1> %c
 }
 
 define <2 x i1> @icmp_eq_v2i64_Zero_RHS(<2 x i64> %a) {
-; CHECK-SD-LABEL: icmp_eq_v2i64_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v2i64_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v2i64_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
     %c = icmp eq <2 x i64> %a, <i64 0, i64 0>
     ret <2 x i1> %c
 }
 
 define <8 x i1> @icmp_sge_v8i8_Zero_RHS(<8 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sge_v8i8_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v8i8_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v8i8_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
     %c = icmp sge <8 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
     ret <8 x i1> %c
 }
 
 define <16 x i1> @icmp_sge_v16i8_Zero_RHS(<16 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sge_v16i8_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v16i8_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v16i8_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
     %c = icmp sge <16 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
     ret <16 x i1> %c
 }
 
 define <4 x i1> @icmp_sge_v4i16_Zero_RHS(<4 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sge_v4i16_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v4i16_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4h, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v4i16_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
     %c = icmp sge <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
     ret <4 x i1> %c
 }
 
 define <8 x i1> @icmp_sge_v8i16_Zero_RHS(<8 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sge_v8i16_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v8i16_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v8i16_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
     %c = icmp sge <8 x i16> %a, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
     ret <8 x i1> %c
 }
 
 define <2 x i1> @icmp_sge_v2i32_Zero_RHS(<2 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sge_v2i32_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v2i32_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v2i32_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
     %c = icmp sge <2 x i32> %a, <i32 0, i32 0>
     ret <2 x i1> %c
 }
 
 define <4 x i1> @icmp_sge_v4i32_Zero_RHS(<4 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sge_v4i32_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v4i32_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v4i32_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %c = icmp sge <4 x i32> %a, <i32 0, i32 0, i32 0, i32 0>
     ret <4 x i1> %c
 }
 
 define <2 x i1> @icmp_sge_v2i64_Zero_RHS(<2 x i64> %a) {
-; CHECK-SD-LABEL: icmp_sge_v2i64_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v2i64_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v2i64_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
     %c = icmp sge <2 x i64> %a, <i64 0, i64 0>
     ret <2 x i1> %c
 }
 
 define <8 x i1> @icmp_sgt_v8i8_Zero_RHS(<8 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v8i8_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v8i8_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v8i8_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
     %c = icmp sgt <8 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
     ret <8 x i1> %c
 }
 
 define <16 x i1> @icmp_sgt_v16i8_Zero_RHS(<16 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v16i8_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v16i8_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v16i8_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
     %c = icmp sgt <16 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
     ret <16 x i1> %c
 }
 
 define <4 x i1> @icmp_sgt_v4i16_Zero_RHS(<4 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v4i16_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v4i16_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4h, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v4i16_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
     %c = icmp sgt <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
     ret <4 x i1> %c
 }
 
 define <8 x i1> @icmp_sgt_v8i16_Zero_RHS(<8 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v8i16_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v8i16_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v8i16_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
     %c = icmp sgt <8 x i16> %a, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
     ret <8 x i1> %c
 }
 
 define <2 x i1> @icmp_sgt_v2i32_Zero_RHS(<2 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v2i32_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v2i32_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v2i32_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
     %c = icmp sgt <2 x i32> %a, <i32 0, i32 0>
     ret <2 x i1> %c
 }
 
 define <4 x i1> @icmp_sgt_v4i32_Zero_RHS(<4 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v4i32_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v4i32_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v4i32_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %c = icmp sgt <4 x i32> %a, <i32 0, i32 0, i32 0, i32 0>
     ret <4 x i1> %c
 }
 
 define <2 x i1> @icmp_sgt_v2i64_Zero_RHS(<2 x i64> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v2i64_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v2i64_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v2i64_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
     %c = icmp sgt <2 x i64> %a, <i64 0, i64 0>
     ret <2 x i1> %c
 }
 
 define <8 x i1> @icmp_sle_v8i8_Zero_RHS(<8 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sle_v8i8_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v8i8_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v8i8_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
     %c = icmp sle <8 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
     ret <8 x i1> %c
 }
 
 define <16 x i1> @icmp_sle_v16i8_Zero_RHS(<16 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sle_v16i8_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v16i8_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.16b, v1.16b, v0.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v16i8_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
     %c = icmp sle <16 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
     ret <16 x i1> %c
 }
 
 define <4 x i1> @icmp_sle_v4i16_Zero_RHS(<4 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sle_v4i16_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v4i16_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v4i16_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
     %c = icmp sle <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
     ret <4 x i1> %c
 }
 
 define <8 x i1> @icmp_sle_v8i16_Zero_RHS(<8 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sle_v8i16_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v8i16_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8h, v1.8h, v0.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v8i16_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
     %c = icmp sle <8 x i16> %a, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
     ret <8 x i1> %c
 }
 
 define <2 x i1> @icmp_sle_v2i32_Zero_RHS(<2 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sle_v2i32_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v2i32_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v2i32_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
     %c = icmp sle <2 x i32> %a, <i32 0, i32 0>
     ret <2 x i1> %c
 }
 
 define <4 x i1> @icmp_sle_v4i32_Zero_RHS(<4 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sle_v4i32_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v4i32_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v4i32_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %c = icmp sle <4 x i32> %a, <i32 0, i32 0, i32 0, i32 0>
     ret <4 x i1> %c
 }
 
 define <2 x i1> @icmp_sle_v2i64_Zero_RHS(<2 x i64> %a) {
-; CHECK-SD-LABEL: icmp_sle_v2i64_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v2i64_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2d, v1.2d, v0.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v2i64_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
     %c = icmp sle <2 x i64> %a, <i64 0, i64 0>
     ret <2 x i1> %c
 }
 
 define <8 x i1> @icmp_slt_v8i8_Zero_RHS(<8 x i8> %a) {
-; CHECK-SD-LABEL: icmp_slt_v8i8_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v8i8_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v8i8_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
     %c = icmp slt <8 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
     ret <8 x i1> %c
 }
 
 define <16 x i1> @icmp_slt_v16i8_Zero_RHS(<16 x i8> %a) {
-; CHECK-SD-LABEL: icmp_slt_v16i8_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v16i8_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.16b, v1.16b, v0.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v16i8_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
     %c = icmp slt <16 x i8> %a, <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
     ret <16 x i1> %c
 }
 
 define <4 x i1> @icmp_slt_v4i16_Zero_RHS(<4 x i16> %a) {
-; CHECK-SD-LABEL: icmp_slt_v4i16_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v4i16_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v4i16_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
     %c = icmp slt <4 x i16> %a, <i16 0, i16 0, i16 0, i16 0>
     ret <4 x i1> %c
 }
 
 define <8 x i1> @icmp_slt_v8i16_Zero_RHS(<8 x i16> %a) {
-; CHECK-SD-LABEL: icmp_slt_v8i16_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v8i16_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8h, v1.8h, v0.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v8i16_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
     %c = icmp slt <8 x i16> %a, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
     ret <8 x i1> %c
 }
 
 define <2 x i1> @icmp_slt_v2i32_Zero_RHS(<2 x i32> %a) {
-; CHECK-SD-LABEL: icmp_slt_v2i32_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v2i32_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v2i32_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
     %c = icmp slt <2 x i32> %a, <i32 0, i32 0>
     ret <2 x i1> %c
 }
 
 define <4 x i1> @icmp_slt_v4i32_Zero_RHS(<4 x i32> %a) {
-; CHECK-SD-LABEL: icmp_slt_v4i32_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v4i32_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v4i32_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %c = icmp slt <4 x i32> %a, <i32 0, i32 0, i32 0, i32 0>
     ret <4 x i1> %c
 }
 
 define <2 x i1> @icmp_slt_v2i64_Zero_RHS(<2 x i64> %a) {
-; CHECK-SD-LABEL: icmp_slt_v2i64_Zero_RHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v2i64_Zero_RHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2d, v1.2d, v0.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v2i64_Zero_RHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
     %c = icmp slt <2 x i64> %a, <i64 0, i64 0>
     ret <2 x i1> %c
 }
@@ -1936,556 +1711,331 @@ define <2 x i1> @icmp_slt_v2i64_Zero_RHS(<2 x i64> %a) {
 ; ===== ICMP Zero LHS =====
 
 define <8 x i1> @icmp_eq_v8i8_Zero_LHS(<8 x i8> %a) {
-; CHECK-SD-LABEL: icmp_eq_v8i8_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v8i8_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v8i8_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
     %c = icmp eq <8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
     ret <8 x i1> %c
 }
 
 define <16 x i1> @icmp_eq_v16i8_Zero_LHS(<16 x i8> %a) {
-; CHECK-SD-LABEL: icmp_eq_v16i8_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v16i8_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.16b, v1.16b, v0.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v16i8_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
     %c = icmp eq <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
     ret <16 x i1> %c
 }
 
 define <4 x i1> @icmp_eq_v4i16_Zero_LHS(<4 x i16> %a) {
-; CHECK-SD-LABEL: icmp_eq_v4i16_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v4i16_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v4i16_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
     %c = icmp eq <4 x i16> <i16 0, i16 0, i16 0, i16 0>, %a
     ret <4 x i1> %c
 }
 
 define <8 x i1> @icmp_eq_v8i16_Zero_LHS(<8 x i16> %a) {
-; CHECK-SD-LABEL: icmp_eq_v8i16_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v8i16_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.8h, v1.8h, v0.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v8i16_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
     %c = icmp eq <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, %a
     ret <8 x i1> %c
 }
 
 define <2 x i1> @icmp_eq_v2i32_Zero_LHS(<2 x i32> %a) {
-; CHECK-SD-LABEL: icmp_eq_v2i32_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v2i32_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v2i32_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
     %c = icmp eq <2 x i32> <i32 0, i32 0>, %a
     ret <2 x i1> %c
 }
 
 define <4 x i1> @icmp_eq_v4i32_Zero_LHS(<4 x i32> %a) {
-; CHECK-SD-LABEL: icmp_eq_v4i32_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v4i32_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v4i32_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %c = icmp eq <4 x i32> <i32 0, i32 0, i32 0, i32 0>, %a
     ret <4 x i1> %c
 }
 
 define <2 x i1> @icmp_eq_v2i64_Zero_LHS(<2 x i64> %a) {
-; CHECK-SD-LABEL: icmp_eq_v2i64_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_eq_v2i64_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.2d, v1.2d, v0.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_eq_v2i64_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
     %c = icmp eq <2 x i64> <i64 0, i64 0>, %a
     ret <2 x i1> %c
 }
 
 define <8 x i1> @icmp_sge_v8i8_Zero_LHS(<8 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sge_v8i8_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v8i8_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v8i8_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
     %c = icmp sge <8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
     ret <8 x i1> %c
 }
 
 define <16 x i1> @icmp_sge_v16i8_Zero_LHS(<16 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sge_v16i8_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v16i8_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.16b, v1.16b, v0.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v16i8_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
     %c = icmp sge <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
     ret <16 x i1> %c
 }
 
 define <4 x i1> @icmp_sge_v4i16_Zero_LHS(<4 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sge_v4i16_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v4i16_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v4i16_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
     %c = icmp sge <4 x i16> <i16 0, i16 0, i16 0, i16 0>, %a
     ret <4 x i1> %c
 }
 
 define <8 x i1> @icmp_sge_v8i16_Zero_LHS(<8 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sge_v8i16_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v8i16_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8h, v1.8h, v0.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v8i16_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
     %c = icmp sge <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, %a
     ret <8 x i1> %c
 }
 
 define <2 x i1> @icmp_sge_v2i32_Zero_LHS(<2 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sge_v2i32_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v2i32_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v2i32_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
     %c = icmp sge <2 x i32> <i32 0, i32 0>, %a
     ret <2 x i1> %c
 }
 
 define <4 x i1> @icmp_sge_v4i32_Zero_LHS(<4 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sge_v4i32_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v4i32_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v4i32_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %c = icmp sge <4 x i32> <i32 0, i32 0, i32 0, i32 0>, %a
     ret <4 x i1> %c
 }
 
 define <2 x i1> @icmp_sge_v2i64_Zero_LHS(<2 x i64> %a) {
-; CHECK-SD-LABEL: icmp_sge_v2i64_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sge_v2i64_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2d, v1.2d, v0.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sge_v2i64_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
     %c = icmp sge <2 x i64> <i64 0, i64 0>, %a
     ret <2 x i1> %c
 }
 
 define <8 x i1> @icmp_sgt_v8i8_Zero_LHS(<8 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v8i8_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v8i8_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v8i8_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
     %c = icmp sgt <8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
     ret <8 x i1> %c
 }
 
 define <16 x i1> @icmp_sgt_v16i8_Zero_LHS(<16 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v16i8_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v16i8_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.16b, v1.16b, v0.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v16i8_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
     %c = icmp sgt <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
     ret <16 x i1> %c
 }
 
 define <4 x i1> @icmp_sgt_v4i16_Zero_LHS(<4 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v4i16_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v4i16_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v4i16_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
     %c = icmp sgt <4 x i16> <i16 0, i16 0, i16 0, i16 0>, %a
     ret <4 x i1> %c
 }
 
 define <8 x i1> @icmp_sgt_v8i16_Zero_LHS(<8 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v8i16_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v8i16_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8h, v1.8h, v0.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v8i16_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
     %c = icmp sgt <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, %a
     ret <8 x i1> %c
 }
 
 define <2 x i1> @icmp_sgt_v2i32_Zero_LHS(<2 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v2i32_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v2i32_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v2i32_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
     %c = icmp sgt <2 x i32> <i32 0, i32 0>, %a
     ret <2 x i1> %c
 }
 
 define <4 x i1> @icmp_sgt_v4i32_Zero_LHS(<4 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v4i32_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v4i32_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v4i32_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %c = icmp sgt <4 x i32> <i32 0, i32 0, i32 0, i32 0>, %a
     ret <4 x i1> %c
 }
 
 define <2 x i1> @icmp_sgt_v2i64_Zero_LHS(<2 x i64> %a) {
-; CHECK-SD-LABEL: icmp_sgt_v2i64_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sgt_v2i64_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2d, v1.2d, v0.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sgt_v2i64_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
     %c = icmp sgt <2 x i64> <i64 0, i64 0>, %a
     ret <2 x i1> %c
 }
 
 define <8 x i1> @icmp_sle_v8i8_Zero_LHS(<8 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sle_v8i8_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v8i8_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v8i8_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
     %c = icmp sle <8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
     ret <8 x i1> %c
 }
 
 define <16 x i1> @icmp_sle_v16i8_Zero_LHS(<16 x i8> %a) {
-; CHECK-SD-LABEL: icmp_sle_v16i8_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v16i8_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v16i8_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
     %c = icmp sle <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
     ret <16 x i1> %c
 }
 
 define <4 x i1> @icmp_sle_v4i16_Zero_LHS(<4 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sle_v4i16_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v4i16_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4h, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v4i16_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
     %c = icmp sle <4 x i16> <i16 0, i16 0, i16 0, i16 0>, %a
     ret <4 x i1> %c
 }
 
 define <8 x i1> @icmp_sle_v8i16_Zero_LHS(<8 x i16> %a) {
-; CHECK-SD-LABEL: icmp_sle_v8i16_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v8i16_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v8i16_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
     %c = icmp sle <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, %a
     ret <8 x i1> %c
 }
 
 define <2 x i1> @icmp_sle_v2i32_Zero_LHS(<2 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sle_v2i32_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v2i32_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v2i32_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
     %c = icmp sle <2 x i32> <i32 0, i32 0>, %a
     ret <2 x i1> %c
 }
 
 define <4 x i1> @icmp_sle_v4i32_Zero_LHS(<4 x i32> %a) {
-; CHECK-SD-LABEL: icmp_sle_v4i32_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v4i32_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v4i32_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %c = icmp sle <4 x i32> <i32 0, i32 0, i32 0, i32 0>, %a
     ret <4 x i1> %c
 }
 
 define <2 x i1> @icmp_sle_v2i64_Zero_LHS(<2 x i64> %a) {
-; CHECK-SD-LABEL: icmp_sle_v2i64_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_sle_v2i64_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_sle_v2i64_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
     %c = icmp sle <2 x i64> <i64 0, i64 0>, %a
     ret <2 x i1> %c
 }
 
 define <8 x i1> @icmp_slt_v8i8_Zero_LHS(<8 x i8> %a) {
-; CHECK-SD-LABEL: icmp_slt_v8i8_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v8i8_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v8i8_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
     %c = icmp slt <8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
     ret <8 x i1> %c
 }
 
 define <16 x i1> @icmp_slt_v16i8_Zero_LHS(<16 x i8> %a) {
-; CHECK-SD-LABEL: icmp_slt_v16i8_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v16i8_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v16i8_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
     %c = icmp slt <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, %a
     ret <16 x i1> %c
 }
 
 define <4 x i1> @icmp_slt_v4i16_Zero_LHS(<4 x i16> %a) {
-; CHECK-SD-LABEL: icmp_slt_v4i16_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v4i16_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4h, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v4i16_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
     %c = icmp slt <4 x i16> <i16 0, i16 0, i16 0, i16 0>, %a
     ret <4 x i1> %c
 }
 
 define <8 x i1> @icmp_slt_v8i16_Zero_LHS(<8 x i16> %a) {
-; CHECK-SD-LABEL: icmp_slt_v8i16_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v8i16_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v8i16_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, #0
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
     %c = icmp slt <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, %a
     ret <8 x i1> %c
 }
 
 define <2 x i1> @icmp_slt_v2i32_Zero_LHS(<2 x i32> %a) {
-; CHECK-SD-LABEL: icmp_slt_v2i32_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v2i32_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v2i32_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
     %c = icmp slt <2 x i32> <i32 0, i32 0>, %a
     ret <2 x i1> %c
 }
 
 define <4 x i1> @icmp_slt_v4i32_Zero_LHS(<4 x i32> %a) {
-; CHECK-SD-LABEL: icmp_slt_v4i32_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v4i32_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v4i32_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, #0
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %c = icmp slt <4 x i32> <i32 0, i32 0, i32 0, i32 0>, %a
     ret <4 x i1> %c
 }
 
 define <2 x i1> @icmp_slt_v2i64_Zero_LHS(<2 x i64> %a) {
-; CHECK-SD-LABEL: icmp_slt_v2i64_Zero_LHS:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: icmp_slt_v2i64_Zero_LHS:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: icmp_slt_v2i64_Zero_LHS:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.2d, v0.2d, #0
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
     %c = icmp slt <2 x i64> <i64 0, i64 0>, %a
     ret <2 x i1> %c
 }
diff --git a/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll b/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
index 57f220f621cf..50c0c8b11e75 100644
--- a/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
@@ -1513,8 +1513,7 @@ define <8 x i8> @vselect_cmpz_ne(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) {
 ;
 ; CHECK-GI-LABEL: vselect_cmpz_ne:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, v3.8b
+; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, #0
 ; CHECK-GI-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-GI-NEXT:    bsl v0.8b, v1.8b, v2.8b
 ; CHECK-GI-NEXT:    ret
@@ -1524,38 +1523,23 @@ define <8 x i8> @vselect_cmpz_ne(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) {
 }
 
 define <8 x i8> @vselect_cmpz_eq(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) {
-; CHECK-SD-LABEL: vselect_cmpz_eq:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    bsl v0.8b, v1.8b, v2.8b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: vselect_cmpz_eq:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, v3.8b
-; CHECK-GI-NEXT:    bsl v0.8b, v1.8b, v2.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: vselect_cmpz_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.8b, v0.8b, #0
+; CHECK-NEXT:    bsl v0.8b, v1.8b, v2.8b
+; CHECK-NEXT:    ret
   %cmp = icmp eq <8 x i8> %a, zeroinitializer
   %d = select <8 x i1> %cmp, <8 x i8> %b, <8 x i8> %c
   ret <8 x i8> %d
 }
 
 define <8 x i8> @vselect_tst(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) {
-; CHECK-SD-LABEL: vselect_tst:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    cmeq v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    bsl v0.8b, v2.8b, v1.8b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: vselect_tst:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, v3.8b
-; CHECK-GI-NEXT:    bsl v0.8b, v2.8b, v1.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: vselect_tst:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    cmeq v0.8b, v0.8b, #0
+; CHECK-NEXT:    bsl v0.8b, v2.8b, v1.8b
+; CHECK-NEXT:    ret
   %tmp3 = and <8 x i8> %a, %b
   %tmp4 = icmp eq <8 x i8> %tmp3, zeroinitializer
   %d = select <8 x i1> %tmp4, <8 x i8> %c, <8 x i8> %b
@@ -1570,9 +1554,8 @@ define <8 x i8> @sext_tst(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) {
 ;
 ; CHECK-GI-LABEL: sext_tst:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, #0
 ; CHECK-GI-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = and <8 x i8> %a, %b
diff --git a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
index dbb5dfebd44a..59958afdd0d1 100644
--- a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
@@ -745,9 +745,8 @@ define <8 x i8> @cmtst8xi8(<8 x i8> %A, <8 x i8> %B) {
 ;
 ; CHECK-GI-LABEL: cmtst8xi8:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, #0
 ; CHECK-GI-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = and <8 x i8> %A, %B
@@ -764,9 +763,8 @@ define <16 x i8> @cmtst16xi8(<16 x i8> %A, <16 x i8> %B) {
 ;
 ; CHECK-GI-LABEL: cmtst16xi8:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    cmeq v0.16b, v0.16b, v2.16b
+; CHECK-GI-NEXT:    cmeq v0.16b, v0.16b, #0
 ; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = and <16 x i8> %A, %B
@@ -783,9 +781,8 @@ define <4 x i16> @cmtst4xi16(<4 x i16> %A, <4 x i16> %B) {
 ;
 ; CHECK-GI-LABEL: cmtst4xi16:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    cmeq v0.4h, v0.4h, v2.4h
+; CHECK-GI-NEXT:    cmeq v0.4h, v0.4h, #0
 ; CHECK-GI-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = and <4 x i16> %A, %B
@@ -802,9 +799,8 @@ define <8 x i16> @cmtst8xi16(<8 x i16> %A, <8 x i16> %B) {
 ;
 ; CHECK-GI-LABEL: cmtst8xi16:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    cmeq v0.8h, v0.8h, v2.8h
+; CHECK-GI-NEXT:    cmeq v0.8h, v0.8h, #0
 ; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = and <8 x i16> %A, %B
@@ -821,9 +817,8 @@ define <2 x i32> @cmtst2xi32(<2 x i32> %A, <2 x i32> %B) {
 ;
 ; CHECK-GI-LABEL: cmtst2xi32:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    cmeq v0.2s, v0.2s, v2.2s
+; CHECK-GI-NEXT:    cmeq v0.2s, v0.2s, #0
 ; CHECK-GI-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = and <2 x i32> %A, %B
@@ -840,9 +835,8 @@ define <4 x i32> @cmtst4xi32(<4 x i32> %A, <4 x i32> %B) {
 ;
 ; CHECK-GI-LABEL: cmtst4xi32:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    cmeq v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = and <4 x i32> %A, %B
@@ -859,9 +853,8 @@ define <2 x i64> @cmtst2xi64(<2 x i64> %A, <2 x i64> %B) {
 ;
 ; CHECK-GI-LABEL: cmtst2xi64:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    cmeq v0.2d, v0.2d, v2.2d
+; CHECK-GI-NEXT:    cmeq v0.2d, v0.2d, #0
 ; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = and <2 x i64> %A, %B
@@ -873,112 +866,70 @@ define <2 x i64> @cmtst2xi64(<2 x i64> %A, <2 x i64> %B) {
 
 
 define <8 x i8> @cmeqz8xi8(<8 x i8> %A) {
-; CHECK-SD-LABEL: cmeqz8xi8:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmeqz8xi8:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmeqz8xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp eq <8 x i8> %A, zeroinitializer
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmeqz16xi8(<16 x i8> %A) {
-; CHECK-SD-LABEL: cmeqz16xi8:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmeqz16xi8:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmeqz16xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp eq <16 x i8> %A, zeroinitializer
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmeqz4xi16(<4 x i16> %A) {
-; CHECK-SD-LABEL: cmeqz4xi16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmeqz4xi16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.4h, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmeqz4xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp eq <4 x i16> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmeqz8xi16(<8 x i16> %A) {
-; CHECK-SD-LABEL: cmeqz8xi16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmeqz8xi16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmeqz8xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.8h, v0.8h, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp eq <8 x i16> %A, zeroinitializer
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmeqz2xi32(<2 x i32> %A) {
-; CHECK-SD-LABEL: cmeqz2xi32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmeqz2xi32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmeqz2xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp eq <2 x i32> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmeqz4xi32(<4 x i32> %A) {
-; CHECK-SD-LABEL: cmeqz4xi32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmeqz4xi32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmeqz4xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp eq <4 x i32> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmeqz2xi64(<2 x i64> %A) {
-; CHECK-SD-LABEL: cmeqz2xi64:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmeq v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmeqz2xi64:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmeqz2xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.2d, v0.2d, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp eq <2 x i64> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -986,112 +937,70 @@ define <2 x i64> @cmeqz2xi64(<2 x i64> %A) {
 
 
 define <8 x i8> @cmgez8xi8(<8 x i8> %A) {
-; CHECK-SD-LABEL: cmgez8xi8:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgez8xi8:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgez8xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sge <8 x i8> %A, zeroinitializer
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmgez16xi8(<16 x i8> %A) {
-; CHECK-SD-LABEL: cmgez16xi8:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgez16xi8:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgez16xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sge <16 x i8> %A, zeroinitializer
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmgez4xi16(<4 x i16> %A) {
-; CHECK-SD-LABEL: cmgez4xi16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgez4xi16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4h, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgez4xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sge <4 x i16> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmgez8xi16(<8 x i16> %A) {
-; CHECK-SD-LABEL: cmgez8xi16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgez8xi16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgez8xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sge <8 x i16> %A, zeroinitializer
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmgez2xi32(<2 x i32> %A) {
-; CHECK-SD-LABEL: cmgez2xi32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgez2xi32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgez2xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sge <2 x i32> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmgez4xi32(<4 x i32> %A) {
-; CHECK-SD-LABEL: cmgez4xi32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgez4xi32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgez4xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sge <4 x i32> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmgez2xi64(<2 x i64> %A) {
-; CHECK-SD-LABEL: cmgez2xi64:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgez2xi64:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgez2xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sge <2 x i64> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1324,224 +1233,140 @@ define <2 x i64> @cmgez2xi64_alt2(<2 x i64> %A) {
 
 
 define <8 x i8> @cmgtz8xi8(<8 x i8> %A) {
-; CHECK-SD-LABEL: cmgtz8xi8:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgtz8xi8:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgtz8xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sgt <8 x i8> %A, zeroinitializer
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmgtz16xi8(<16 x i8> %A) {
-; CHECK-SD-LABEL: cmgtz16xi8:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgtz16xi8:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgtz16xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sgt <16 x i8> %A, zeroinitializer
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmgtz4xi16(<4 x i16> %A) {
-; CHECK-SD-LABEL: cmgtz4xi16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgtz4xi16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4h, v0.4h, v1.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgtz4xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sgt <4 x i16> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmgtz8xi16(<8 x i16> %A) {
-; CHECK-SD-LABEL: cmgtz8xi16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgtz8xi16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgtz8xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sgt <8 x i16> %A, zeroinitializer
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmgtz2xi32(<2 x i32> %A) {
-; CHECK-SD-LABEL: cmgtz2xi32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgtz2xi32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgtz2xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sgt <2 x i32> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmgtz4xi32(<4 x i32> %A) {
-; CHECK-SD-LABEL: cmgtz4xi32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgtz4xi32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgtz4xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sgt <4 x i32> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmgtz2xi64(<2 x i64> %A) {
-; CHECK-SD-LABEL: cmgtz2xi64:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmgt v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmgtz2xi64:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2d, v0.2d, v1.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmgtz2xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmgt v0.2d, v0.2d, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sgt <2 x i64> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <8 x i8> @cmlez8xi8(<8 x i8> %A) {
-; CHECK-SD-LABEL: cmlez8xi8:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmlez8xi8:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmlez8xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sle <8 x i8> %A, zeroinitializer
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmlez16xi8(<16 x i8> %A) {
-; CHECK-SD-LABEL: cmlez16xi8:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmlez16xi8:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.16b, v1.16b, v0.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmlez16xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sle <16 x i8> %A, zeroinitializer
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmlez4xi16(<4 x i16> %A) {
-; CHECK-SD-LABEL: cmlez4xi16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmlez4xi16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmlez4xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sle <4 x i16> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmlez8xi16(<8 x i16> %A) {
-; CHECK-SD-LABEL: cmlez8xi16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmlez8xi16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.8h, v1.8h, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmlez8xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.8h, v0.8h, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sle <8 x i16> %A, zeroinitializer
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmlez2xi32(<2 x i32> %A) {
-; CHECK-SD-LABEL: cmlez2xi32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmlez2xi32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmlez2xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sle <2 x i32> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmlez4xi32(<4 x i32> %A) {
-; CHECK-SD-LABEL: cmlez4xi32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmlez4xi32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmlez4xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.4s, v0.4s, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sle <4 x i32> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmlez2xi64(<2 x i64> %A) {
-; CHECK-SD-LABEL: cmlez2xi64:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmle v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmlez2xi64:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmge v0.2d, v1.2d, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmlez2xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmle v0.2d, v0.2d, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp sle <2 x i64> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1661,112 +1486,70 @@ define <2 x i64> @cmlez2xi64_alt(<2 x i64> %A) {
 }
 
 define <8 x i8> @cmltz8xi8(<8 x i8> %A) {
-; CHECK-SD-LABEL: cmltz8xi8:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.8b, v0.8b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmltz8xi8:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8b, v1.8b, v0.8b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmltz8xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp slt <8 x i8> %A, zeroinitializer
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmltz16xi8(<16 x i8> %A) {
-; CHECK-SD-LABEL: cmltz16xi8:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmltz16xi8:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.16b, v1.16b, v0.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmltz16xi8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp slt <16 x i8> %A, zeroinitializer
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmltz4xi16(<4 x i16> %A) {
-; CHECK-SD-LABEL: cmltz4xi16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.4h, v0.4h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmltz4xi16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4h, v1.4h, v0.4h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmltz4xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp slt <4 x i16> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmltz8xi16(<8 x i16> %A) {
-; CHECK-SD-LABEL: cmltz8xi16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.8h, v0.8h, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmltz8xi16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.8h, v1.8h, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmltz8xi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp slt <8 x i16> %A, zeroinitializer
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmltz2xi32(<2 x i32> %A) {
-; CHECK-SD-LABEL: cmltz2xi32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.2s, v0.2s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmltz2xi32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmltz2xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp slt <2 x i32> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmltz4xi32(<4 x i32> %A) {
-; CHECK-SD-LABEL: cmltz4xi32:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.4s, v0.4s, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmltz4xi32:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmltz4xi32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp slt <4 x i32> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmltz2xi64(<2 x i64> %A) {
-; CHECK-SD-LABEL: cmltz2xi64:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    cmlt v0.2d, v0.2d, #0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: cmltz2xi64:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmgt v0.2d, v1.2d, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: cmltz2xi64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmlt v0.2d, v0.2d, #0
+; CHECK-NEXT:    ret
   %tmp3 = icmp slt <2 x i64> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1916,8 +1699,7 @@ define <8 x i8> @cmneqz8xi8(<8 x i8> %A) {
 ;
 ; CHECK-GI-LABEL: cmneqz8xi8:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, #0
 ; CHECK-GI-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = icmp ne <8 x i8> %A, zeroinitializer
@@ -1933,8 +1715,7 @@ define <16 x i8> @cmneqz16xi8(<16 x i8> %A) {
 ;
 ; CHECK-GI-LABEL: cmneqz16xi8:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    cmeq v0.16b, v0.16b, #0
 ; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = icmp ne <16 x i8> %A, zeroinitializer
@@ -1950,8 +1731,7 @@ define <4 x i16> @cmneqz4xi16(<4 x i16> %A) {
 ;
 ; CHECK-GI-LABEL: cmneqz4xi16:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    cmeq v0.4h, v0.4h, #0
 ; CHECK-GI-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = icmp ne <4 x i16> %A, zeroinitializer
@@ -1967,8 +1747,7 @@ define <8 x i16> @cmneqz8xi16(<8 x i16> %A) {
 ;
 ; CHECK-GI-LABEL: cmneqz8xi16:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    cmeq v0.8h, v0.8h, #0
 ; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = icmp ne <8 x i16> %A, zeroinitializer
@@ -1984,8 +1763,7 @@ define <2 x i32> @cmneqz2xi32(<2 x i32> %A) {
 ;
 ; CHECK-GI-LABEL: cmneqz2xi32:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    cmeq v0.2s, v0.2s, #0
 ; CHECK-GI-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = icmp ne <2 x i32> %A, zeroinitializer
@@ -2001,8 +1779,7 @@ define <4 x i32> @cmneqz4xi32(<4 x i32> %A) {
 ;
 ; CHECK-GI-LABEL: cmneqz4xi32:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = icmp ne <4 x i32> %A, zeroinitializer
@@ -2018,8 +1795,7 @@ define <2 x i64> @cmneqz2xi64(<2 x i64> %A) {
 ;
 ; CHECK-GI-LABEL: cmneqz2xi64:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    cmeq v0.2d, v0.2d, #0
 ; CHECK-GI-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = icmp ne <2 x i64> %A, zeroinitializer
-- 
GitLab


From 429ce59bd0a7d93ef833939d4a92b56aae103a5a Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Tue, 14 May 2024 13:15:03 +0400
Subject: [PATCH 0700/1206] [lldb][Windows] Fixed the test
 TestGdbRemoteMemoryTagging (#92077)

Windows path is case insensitive. Tests `test_QMemTags_packets` and
`test_qMemTags_packets` will use the same build dir and conflict. Added
a suffix to resolve conflicts.
---
 .../lldb-server/memory-tagging/TestGdbRemoteMemoryTagging.py  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/test/API/tools/lldb-server/memory-tagging/TestGdbRemoteMemoryTagging.py b/lldb/test/API/tools/lldb-server/memory-tagging/TestGdbRemoteMemoryTagging.py
index 584de5e2ef49..6ddd264057c3 100644
--- a/lldb/test/API/tools/lldb-server/memory-tagging/TestGdbRemoteMemoryTagging.py
+++ b/lldb/test/API/tools/lldb-server/memory-tagging/TestGdbRemoteMemoryTagging.py
@@ -84,7 +84,7 @@ class TestGdbRemoteMemoryTagging(gdbremote_testcase.GdbRemoteTestCaseBase):
     @skipUnlessArch("aarch64")
     @skipUnlessPlatform(["linux"])
     @skipUnlessAArch64MTELinuxCompiler
-    def test_qMemTags_packets(self):
+    def test_tag_read_qMemTags_packets(self):
         """Test that qMemTags packets are parsed correctly and/or rejected."""
         buf_address, page_size = self.prep_memtags_test()
 
@@ -154,7 +154,7 @@ class TestGdbRemoteMemoryTagging(gdbremote_testcase.GdbRemoteTestCaseBase):
     @skipUnlessArch("aarch64")
     @skipUnlessPlatform(["linux"])
     @skipUnlessAArch64MTELinuxCompiler
-    def test_QMemTags_packets(self):
+    def test_tag_write_QMemTags_packets(self):
         """Test that QMemTags packets are parsed correctly and/or rejected."""
         buf_address, page_size = self.prep_memtags_test()
 
-- 
GitLab


From be9b4dab40c36a3d3d9be26498b24efedd8253bf Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy@intel.com>
Date: Tue, 14 May 2024 11:35:11 +0200
Subject: [PATCH 0701/1206] [SPIR-V] Introduce support for 'spirv.Decorations'
 metadata node in SPIR-V Backend (#91736)

This PR is to introduce support for 'spirv.Decorations' metadata node in
SPIR-V Backend.
See also
https://github.com/KhronosGroup/SPIRV-LLVM-Translator/blob/main/docs/SPIRVRepresentationInLLVM.rst
that describes `spirv.Decorations` as an important part of
SPIRV-friendly LLVM IR.
---
 llvm/docs/SPIRVUsage.rst                      |  4 +++
 llvm/include/llvm/IR/IntrinsicsSPIRV.td       |  1 +
 llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 11 +++++++
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp |  7 +++++
 llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp   | 17 ++++++++++
 llvm/lib/Target/SPIRV/SPIRVUtils.cpp          | 28 +++++++++++++++++
 llvm/lib/Target/SPIRV/SPIRVUtils.h            |  4 +++
 llvm/test/CodeGen/SPIRV/spirv-decoration.ll   | 31 +++++++++++++++++++
 8 files changed, 103 insertions(+)
 create mode 100644 llvm/test/CodeGen/SPIRV/spirv-decoration.ll

diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index a183877c48cb..589ee7646ce1 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -253,6 +253,10 @@ SPIR-V backend, along with their descriptions and argument details.
      - None
      - `[Type, Vararg]`
      - Assigns names to types or values, enhancing readability and debuggability of SPIR-V code. Not emitted directly but used for metadata enrichment.
+   * - `int_spv_assign_decoration`
+     - None
+     - `[Type, Metadata]`
+     - Assigns decoration to values by associating them with metadatas. Not emitted directly but used to support SPIR-V representation in LLVM IR.
    * - `int_spv_track_constant`
      - Type
      - `[Type, Metadata]`
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 8660782d71d9..931786ab9647 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -14,6 +14,7 @@ let TargetPrefix = "spv" in {
   def int_spv_assign_type : Intrinsic<[], [llvm_any_ty, llvm_metadata_ty]>;
   def int_spv_assign_ptr_type : Intrinsic<[], [llvm_any_ty, llvm_metadata_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>;
   def int_spv_assign_name : Intrinsic<[], [llvm_any_ty, llvm_vararg_ty]>;
+  def int_spv_assign_decoration : Intrinsic<[], [llvm_any_ty, llvm_metadata_ty]>;
 
   def int_spv_track_constant : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_metadata_ty]>;
   def int_spv_init_global : Intrinsic<[], [llvm_any_ty, llvm_any_ty]>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 0d539b1ed9a8..c00066f5dca6 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -111,6 +111,7 @@ class SPIRVEmitIntrinsics
                                         unsigned OperandToReplace,
                                         IRBuilder<> &B);
   void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
+  void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
   void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
   void processParamTypes(Function *F, IRBuilder<> &B);
   void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
@@ -1116,6 +1117,15 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
   }
 }
 
+void SPIRVEmitIntrinsics::insertSpirvDecorations(Instruction *I,
+                                                 IRBuilder<> &B) {
+  if (MDNode *MD = I->getMetadata("spirv.Decorations")) {
+    B.SetInsertPoint(I->getNextNode());
+    B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {I->getType()},
+                      {I, MetadataAsValue::get(I->getContext(), MD)});
+  }
+}
+
 void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
                                                  IRBuilder<> &B) {
   auto *II = dyn_cast<IntrinsicInst>(I);
@@ -1287,6 +1297,7 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
     insertAssignPtrTypeIntrs(I, B);
     insertAssignTypeIntrs(I, B);
     insertPtrCastOrAssignTypeInstr(I, B);
+    insertSpirvDecorations(I, B);
   }
 
   for (auto &I : instructions(Func))
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index cebe230d3e8c..0a4e44e2dac7 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -691,6 +691,13 @@ Register SPIRVGlobalRegistry::buildGlobalVariable(
     buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::BuiltIn,
                     {static_cast<uint32_t>(BuiltInId)});
 
+  // If it's a global variable with "spirv.Decorations" metadata node
+  // recognize it as a SPIR-V friendly LLVM IR and parse "spirv.Decorations"
+  // arguments.
+  MDNode *GVarMD = nullptr;
+  if (GVar && (GVarMD = GVar->getMetadata("spirv.Decorations")) != nullptr)
+    buildOpSpirvDecorations(Reg, MIRBuilder, GVarMD);
+
   return Reg;
 }
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 84508fb5fe09..6ee5d90d9afe 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -519,6 +519,22 @@ static void processInstrsWithTypeFolding(MachineFunction &MF,
   }
 }
 
+static void insertSpirvDecorations(MachineFunction &MF, MachineIRBuilder MIB) {
+  SmallVector<MachineInstr *, 10> ToErase;
+  for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : MBB) {
+      if (!isSpvIntrinsic(MI, Intrinsic::spv_assign_decoration))
+        continue;
+      MIB.setInsertPt(*MI.getParent(), MI);
+      buildOpSpirvDecorations(MI.getOperand(1).getReg(), MIB,
+                              MI.getOperand(2).getMetadata());
+      ToErase.push_back(&MI);
+    }
+  }
+  for (MachineInstr *MI : ToErase)
+    MI->eraseFromParent();
+}
+
 // Find basic blocks of the switch and replace registers in spv_switch() by its
 // MBB equivalent.
 static void processSwitches(MachineFunction &MF, SPIRVGlobalRegistry *GR,
@@ -639,6 +655,7 @@ bool SPIRVPreLegalizer::runOnMachineFunction(MachineFunction &MF) {
   processSwitches(MF, GR, MIB);
   processInstrsWithTypeFolding(MF, GR, MIB);
   removeImplicitFallthroughs(MF, MIB);
+  insertSpirvDecorations(MF, MIB);
 
   return true;
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 871b95a28068..c20f3546a3e5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -133,6 +133,34 @@ void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII,
   finishBuildOpDecorate(MIB, DecArgs, StrImm);
 }
 
+void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
+                             const MDNode *GVarMD) {
+  for (unsigned I = 0, E = GVarMD->getNumOperands(); I != E; ++I) {
+    auto *OpMD = dyn_cast<MDNode>(GVarMD->getOperand(I));
+    if (!OpMD)
+      report_fatal_error("Invalid decoration");
+    if (OpMD->getNumOperands() == 0)
+      report_fatal_error("Expect operand(s) of the decoration");
+    ConstantInt *DecorationId =
+        mdconst::dyn_extract<ConstantInt>(OpMD->getOperand(0));
+    if (!DecorationId)
+      report_fatal_error("Expect SPIR-V <Decoration> operand to be the first "
+                         "element of the decoration");
+    auto MIB = MIRBuilder.buildInstr(SPIRV::OpDecorate)
+                   .addUse(Reg)
+                   .addImm(static_cast<uint32_t>(DecorationId->getZExtValue()));
+    for (unsigned OpI = 1, OpE = OpMD->getNumOperands(); OpI != OpE; ++OpI) {
+      if (ConstantInt *OpV =
+              mdconst::dyn_extract<ConstantInt>(OpMD->getOperand(OpI)))
+        MIB.addImm(static_cast<uint32_t>(OpV->getZExtValue()));
+      else if (MDString *OpV = dyn_cast<MDString>(OpMD->getOperand(OpI)))
+        addStringImm(OpV->getString(), MIB);
+      else
+        report_fatal_error("Unexpected operand of the decoration");
+    }
+  }
+}
+
 // TODO: maybe the following two functions should be handled in the subtarget
 // to allow for different OpenCL vs Vulkan handling.
 unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index 6a91b6e576f9..33cb509dc4a5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -59,6 +59,10 @@ void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII,
                      const std::vector<uint32_t> &DecArgs,
                      StringRef StrImm = "");
 
+// Add an OpDecorate instruction by "spirv.Decorations" metadata node.
+void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
+                             const MDNode *GVarMD);
+
 // Convert a SPIR-V storage class to the corresponding LLVM IR address space.
 unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC);
 
diff --git a/llvm/test/CodeGen/SPIRV/spirv-decoration.ll b/llvm/test/CodeGen/SPIRV/spirv-decoration.ll
new file mode 100644
index 000000000000..783a2e916c22
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/spirv-decoration.ll
@@ -0,0 +1,31 @@
+; RUN: llc -O0 -mtriple=spirv64v1.4-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64v1.4-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: OpName %[[#GV:]] "v"
+; CHECK-DAG: OpName %[[#FunBar:]] "bar"
+; CHECK-DAG: OpDecorate %[[#GV]] LinkageAttributes "v" Export
+; CHECK-DAG: OpDecorate %[[#GV]] Constant
+; CHECK-DAG: OpDecorate %[[#Idx:]] UserSemantic "SemanticValue"
+; CHECK: %[[#FunBar]] = OpFunction
+; CHECK: %[[#Idx]] = OpInBoundsPtrAccessChain
+
+@v = addrspace(1) global i32 0, !spirv.Decorations !0
+
+define spir_kernel void @foo() {
+entry:
+  %pv = load ptr addrspace(1), ptr addrspace(1) @v
+  store i32 3, ptr addrspace(1) %pv
+  ret void
+}
+
+define spir_kernel void @bar(ptr addrspace(1) %arg) {
+entry:
+  %idx = getelementptr inbounds i32, ptr addrspace(1) %arg, i64 1, !spirv.Decorations !3
+  ret void
+}
+
+!0 = !{!1, !2}
+!1 = !{i32 22}                     ; 22 is Constant decoration
+!2 = !{i32 41, !"v", i32 0}        ; 41 is LinkageAttributes decoration with 2 extra operands
+!3 = !{!4}
+!4 = !{i32 5635, !"SemanticValue"} ; 5635 is UserSemantic decoration
-- 
GitLab


From 3b8b1022684175e988f043f14596f2dc9b31c6c7 Mon Sep 17 00:00:00 2001
From: aengelke <engelke@in.tum.de>
Date: Tue, 14 May 2024 11:38:38 +0200
Subject: [PATCH 0702/1206] [MC] Make ELFEntrySizeMap a DenseMap (#91728)

There is no need for an ordered std::map and also no need to duplicate
the section name, which is owned by the ELFSectionKey. Therefore, use a
DenseMap instead and don't copy the string. As a further, minor
performance optimization, avoid the hash table lookup in
isELFGenericMergeableSection when the section name was just added.

This slightly improves compilation performance in our application, where
we occasionally compile many small object files.
---
 llvm/include/llvm/MC/MCContext.h | 22 +++-------------------
 llvm/lib/MC/MCContext.cpp        | 12 ++++++++----
 2 files changed, 11 insertions(+), 23 deletions(-)

diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index ef9833cdf2b0..3de41b6a6ea7 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -391,29 +391,13 @@ private:
   /// Map of currently defined macros.
   StringMap<MCAsmMacro> MacroMap;
 
-  struct ELFEntrySizeKey {
-    std::string SectionName;
-    unsigned Flags;
-    unsigned EntrySize;
-
-    ELFEntrySizeKey(StringRef SectionName, unsigned Flags, unsigned EntrySize)
-        : SectionName(SectionName), Flags(Flags), EntrySize(EntrySize) {}
-
-    bool operator<(const ELFEntrySizeKey &Other) const {
-      if (SectionName != Other.SectionName)
-        return SectionName < Other.SectionName;
-      if (Flags != Other.Flags)
-        return Flags < Other.Flags;
-      return EntrySize < Other.EntrySize;
-    }
-  };
-
   // Symbols must be assigned to a section with a compatible entry size and
   // flags. This map is used to assign unique IDs to sections to distinguish
   // between sections with identical names but incompatible entry sizes and/or
   // flags. This can occur when a symbol is explicitly assigned to a section,
-  // e.g. via __attribute__((section("myname"))).
-  std::map<ELFEntrySizeKey, unsigned> ELFEntrySizeMap;
+  // e.g. via __attribute__((section("myname"))). The map key is the tuple
+  // (section name, flags, entry size).
+  DenseMap<std::tuple<StringRef, unsigned, unsigned>, unsigned> ELFEntrySizeMap;
 
   // This set is used to record the generic mergeable section names seen.
   // These are sections that are created as mergeable e.g. .debug_str. We need
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index 3aee96fdf57f..f027eb65d700 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -620,15 +620,20 @@ void MCContext::recordELFMergeableSectionInfo(StringRef SectionName,
                                               unsigned Flags, unsigned UniqueID,
                                               unsigned EntrySize) {
   bool IsMergeable = Flags & ELF::SHF_MERGE;
-  if (UniqueID == GenericSectionID)
+  if (UniqueID == GenericSectionID) {
     ELFSeenGenericMergeableSections.insert(SectionName);
+    // Minor performance optimization: avoid hash map lookup in
+    // isELFGenericMergeableSection, which will return true for SectionName.
+    IsMergeable = true;
+  }
 
   // For mergeable sections or non-mergeable sections with a generic mergeable
   // section name we enter their Unique ID into the ELFEntrySizeMap so that
   // compatible globals can be assigned to the same section.
+
   if (IsMergeable || isELFGenericMergeableSection(SectionName)) {
     ELFEntrySizeMap.insert(std::make_pair(
-        ELFEntrySizeKey{SectionName, Flags, EntrySize}, UniqueID));
+        std::make_tuple(SectionName, Flags, EntrySize), UniqueID));
   }
 }
 
@@ -645,8 +650,7 @@ bool MCContext::isELFGenericMergeableSection(StringRef SectionName) {
 std::optional<unsigned>
 MCContext::getELFUniqueIDForEntsize(StringRef SectionName, unsigned Flags,
                                     unsigned EntrySize) {
-  auto I = ELFEntrySizeMap.find(
-      MCContext::ELFEntrySizeKey{SectionName, Flags, EntrySize});
+  auto I = ELFEntrySizeMap.find(std::make_tuple(SectionName, Flags, EntrySize));
   return (I != ELFEntrySizeMap.end()) ? std::optional<unsigned>(I->second)
                                       : std::nullopt;
 }
-- 
GitLab


From 9f80f437c0b698478c6396c8c44ba094f7199144 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 14 May 2024 10:49:27 +0100
Subject: [PATCH 0703/1206] [Hexagon] Regenerate asr-rnd.ll + asr-rnd64.ll to
 show all test checks

These are affected by upcoming support for AVG legalization
---
 llvm/test/CodeGen/Hexagon/asr-rnd.ll   | 27 ++++++++++++++++++++++--
 llvm/test/CodeGen/Hexagon/asr-rnd64.ll | 29 ++++++++++++++++++++++++--
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/llvm/test/CodeGen/Hexagon/asr-rnd.ll b/llvm/test/CodeGen/Hexagon/asr-rnd.ll
index bc77e2a7a3ad..cd088dd8f013 100644
--- a/llvm/test/CodeGen/Hexagon/asr-rnd.ll
+++ b/llvm/test/CodeGen/Hexagon/asr-rnd.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -march=hexagon < %s | FileCheck %s
 ;
 ; Check if we generate rounding-asr instruction.  It is equivalent to
@@ -6,8 +7,19 @@ target triple = "hexagon"
 
 ; Function Attrs: nounwind
 define i32 @f0(i32 %a0) #0 {
+; CHECK-LABEL: f0:
+; CHECK:       // %bb.0: // %b0
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = asr(r0,#10):rnd
+; CHECK-NEXT:     r1 = r0
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memw(r29+#4) = r1
+; CHECK-NEXT:    }
 b0:
-; CHECK: asr{{.*}}:rnd
   %v0 = alloca i32, align 4
   store i32 %a0, ptr %v0, align 4
   %v1 = load i32, ptr %v0, align 4
@@ -19,8 +31,19 @@ b0:
 
 ; Function Attrs: nounwind
 define i64 @f1(i64 %a0) #0 {
+; CHECK-LABEL: f1:
+; CHECK:       // %bb.0: // %b0
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r1:0 = asr(r1:0,#17):rnd
+; CHECK-NEXT:     r3:2 = combine(r1,r0)
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memd(r29+#0) = r3:2
+; CHECK-NEXT:    }
 b0:
-; CHECK: asr{{.*}}:rnd
   %v0 = alloca i64, align 8
   store i64 %a0, ptr %v0, align 8
   %v1 = load i64, ptr %v0, align 8
diff --git a/llvm/test/CodeGen/Hexagon/asr-rnd64.ll b/llvm/test/CodeGen/Hexagon/asr-rnd64.ll
index 4928483e5be6..e32bdff7d764 100644
--- a/llvm/test/CodeGen/Hexagon/asr-rnd64.ll
+++ b/llvm/test/CodeGen/Hexagon/asr-rnd64.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -march=hexagon < %s | FileCheck %s
 ;
 ; Check if we generate rounding-asr instruction.  It is equivalent to
@@ -6,8 +7,20 @@
 target triple = "hexagon"
 
 define i32 @f0(i32 %a0) {
+; CHECK-LABEL: f0:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  // %bb.0: // %b0
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = asr(r0,#10):rnd
+; CHECK-NEXT:     r1 = r0
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memw(r29+#4) = r1
+; CHECK-NEXT:    }
 b0:
-; CHECK: asr{{.*}}:rnd
   %v0 = alloca i32, align 4
   store i32 %a0, ptr %v0, align 4
   %v1 = load i32, ptr %v0, align 4
@@ -18,8 +31,20 @@ b0:
 }
 
 define i64 @f1(i64 %a0) {
+; CHECK-LABEL: f1:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  // %bb.0: // %b0
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r1:0 = asr(r1:0,#17):rnd
+; CHECK-NEXT:     r3:2 = combine(r1,r0)
+; CHECK-NEXT:     r29 = add(r29,#-8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r29 = add(r29,#8)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memd(r29+#0) = r3:2
+; CHECK-NEXT:    }
 b0:
-; CHECK: asr{{.*}}:rnd
   %v0 = alloca i64, align 8
   store i64 %a0, ptr %v0, align 8
   %v1 = load i64, ptr %v0, align 8
-- 
GitLab


From 995a8af81dc10140109cdd95c44bfb093e8a4c02 Mon Sep 17 00:00:00 2001
From: Dominik Steenken <dost@de.ibm.com>
Date: Tue, 14 May 2024 11:50:16 +0200
Subject: [PATCH 0704/1206] [SystemZ] Add extended mnemonics tests. (#91733)

This adds tests for some extended mnemonics of load, branch, and
compare-and-trap instructions.
---
 llvm/test/MC/SystemZ/insn-good-z13.s   |  39 +++++++
 llvm/test/MC/SystemZ/insn-good-z14.s   |  12 +++
 llvm/test/MC/SystemZ/insn-good-zEC12.s |  24 +++++
 llvm/test/MC/SystemZ/insn-good.s       | 143 +++++++++++++++++++++++++
 4 files changed, 218 insertions(+)

diff --git a/llvm/test/MC/SystemZ/insn-good-z13.s b/llvm/test/MC/SystemZ/insn-good-z13.s
index 37d80c34289b..709a95512c52 100644
--- a/llvm/test/MC/SystemZ/insn-good-z13.s
+++ b/llvm/test/MC/SystemZ/insn-good-z13.s
@@ -156,6 +156,19 @@
         lochino %r11, 32512
         lochi %r11, 32512, 15
 
+#CHECK: lochim %r10, 2766   # encoding: [0xec,0xa4,0x0a,0xce,0x00,0x42]
+#CHECK: lochinm %r11, 6862  # encoding: [0xec,0xbb,0x1a,0xce,0x00,0x42]
+#CHECK: lochip %r12, 10958  # encoding: [0xec,0xc2,0x2a,0xce,0x00,0x42]
+#CHECK: lochiz %r13, 15054  # encoding: [0xec,0xd8,0x3a,0xce,0x00,0x42]
+#CHECK: lochinp %r14, 19150 # encoding: [0xec,0xed,0x4a,0xce,0x00,0x42]
+#CHECK: lochinz %r15, 23246 # encoding: [0xec,0xf7,0x5a,0xce,0x00,0x42]
+        lochim %r10, 2766
+        lochinm %r11, 6862
+        lochip %r12, 10958
+        lochiz %r13, 15054
+        lochinp %r14, 19150
+        lochinz %r15, 23246
+
 #CHECK: locghi  %r11, 42, 0    # encoding: [0xec,0xb0,0x00,0x2a,0x00,0x46]
 #CHECK: locghio %r11, 42       # encoding: [0xec,0xb1,0x00,0x2a,0x00,0x46]
 #CHECK: locghih %r11, 42       # encoding: [0xec,0xb2,0x00,0x2a,0x00,0x46]
@@ -190,6 +203,19 @@
         locghino %r11, 32512
         locghi %r11, 32512, 15
 
+#CHECK: locghim %r10, 2766   # encoding: [0xec,0xa4,0x0a,0xce,0x00,0x46]
+#CHECK: locghinm %r11, 6862  # encoding: [0xec,0xbb,0x1a,0xce,0x00,0x46]
+#CHECK: locghip %r12, 10958  # encoding: [0xec,0xc2,0x2a,0xce,0x00,0x46]
+#CHECK: locghiz %r13, 15054  # encoding: [0xec,0xd8,0x3a,0xce,0x00,0x46]
+#CHECK: locghinp %r14, 19150 # encoding: [0xec,0xed,0x4a,0xce,0x00,0x46]
+#CHECK: locghinz %r15, 23246 # encoding: [0xec,0xf7,0x5a,0xce,0x00,0x46]
+        locghim %r10, 2766
+        locghinm %r11, 6862
+        locghip %r12, 10958
+        locghiz %r13, 15054
+        locghinp %r14, 19150
+        locghinz %r15, 23246
+
 #CHECK: lochhi  %r11, 42, 0    # encoding: [0xec,0xb0,0x00,0x2a,0x00,0x4e]
 #CHECK: lochhio %r11, 42       # encoding: [0xec,0xb1,0x00,0x2a,0x00,0x4e]
 #CHECK: lochhih %r11, 42       # encoding: [0xec,0xb2,0x00,0x2a,0x00,0x4e]
@@ -224,6 +250,19 @@
         lochhino %r11, 32512
         lochhi %r11, 32512, 15
 
+#CHECK: lochhim %r10, 2766   # encoding: [0xec,0xa4,0x0a,0xce,0x00,0x4e]
+#CHECK: lochhinm %r11, 6862  # encoding: [0xec,0xbb,0x1a,0xce,0x00,0x4e]
+#CHECK: lochhip %r12, 10958  # encoding: [0xec,0xc2,0x2a,0xce,0x00,0x4e]
+#CHECK: lochhiz %r13, 15054  # encoding: [0xec,0xd8,0x3a,0xce,0x00,0x4e]
+#CHECK: lochhinp %r14, 19150 # encoding: [0xec,0xed,0x4a,0xce,0x00,0x4e]
+#CHECK: lochhinz %r15, 23246 # encoding: [0xec,0xf7,0x5a,0xce,0x00,0x4e]
+        lochhim %r10, 2766
+        lochhinm %r11, 6862
+        lochhip %r12, 10958
+        lochhiz %r13, 15054
+        lochhinp %r14, 19150
+        lochhinz %r15, 23246
+
 #CHECK: locfh   %r0, 0, 0               # encoding: [0xeb,0x00,0x00,0x00,0x00,0xe0]
 #CHECK: locfh   %r0, 0, 15              # encoding: [0xeb,0x0f,0x00,0x00,0x00,0xe0]
 #CHECK: locfh   %r0, -524288, 0         # encoding: [0xeb,0x00,0x00,0x00,0x80,0xe0]
diff --git a/llvm/test/MC/SystemZ/insn-good-z14.s b/llvm/test/MC/SystemZ/insn-good-z14.s
index 385fd3ce7d42..af2663bc50c2 100644
--- a/llvm/test/MC/SystemZ/insn-good-z14.s
+++ b/llvm/test/MC/SystemZ/insn-good-z14.s
@@ -96,9 +96,11 @@
 
 #CHECK:	bic	2, 0(%r7)		# encoding: [0xe3,0x20,0x70,0x00,0x00,0x47]
 #CHECK:	bih	0(%r15)			# encoding: [0xe3,0x20,0xf0,0x00,0x00,0x47]
+#CHECK:	bip	0(%r14)			# encoding: [0xe3,0x20,0xe0,0x00,0x00,0x47]
 
 	bic	2, 0(%r7)
 	bih	0(%r15)
+	bip	0(%r14)
 
 #CHECK:	bic	3, 0(%r7)		# encoding: [0xe3,0x30,0x70,0x00,0x00,0x47]
 #CHECK:	binle	0(%r15)			# encoding: [0xe3,0x30,0xf0,0x00,0x00,0x47]
@@ -108,9 +110,11 @@
 
 #CHECK:	bic	4, 0(%r7)		# encoding: [0xe3,0x40,0x70,0x00,0x00,0x47]
 #CHECK:	bil	0(%r15)			# encoding: [0xe3,0x40,0xf0,0x00,0x00,0x47]
+#CHECK:	bim	0(%r13)			# encoding: [0xe3,0x40,0xd0,0x00,0x00,0x47]
 
 	bic	4, 0(%r7)
 	bil	0(%r15)
+	bim	0(%r13)
 
 #CHECK:	bic	5, 0(%r7)		# encoding: [0xe3,0x50,0x70,0x00,0x00,0x47]
 #CHECK:	binhe	0(%r15)			# encoding: [0xe3,0x50,0xf0,0x00,0x00,0x47]
@@ -126,15 +130,19 @@
 
 #CHECK:	bic	7, 0(%r7)		# encoding: [0xe3,0x70,0x70,0x00,0x00,0x47]
 #CHECK:	bine	0(%r15)			# encoding: [0xe3,0x70,0xf0,0x00,0x00,0x47]
+#CHECK:	binz	0(%r12)			# encoding: [0xe3,0x70,0xc0,0x00,0x00,0x47]
 
 	bic	7, 0(%r7)
 	bine	0(%r15)
+	binz	0(%r12)
 
 #CHECK:	bic	8, 0(%r7)		# encoding: [0xe3,0x80,0x70,0x00,0x00,0x47]
 #CHECK:	bie	0(%r15)			# encoding: [0xe3,0x80,0xf0,0x00,0x00,0x47]
+#CHECK:	biz	0(%r11)			# encoding: [0xe3,0x80,0xb0,0x00,0x00,0x47]
 
 	bic	8, 0(%r7)
 	bie	0(%r15)
+	biz	0(%r11)
 
 #CHECK:	bic	9, 0(%r7)		# encoding: [0xe3,0x90,0x70,0x00,0x00,0x47]
 #CHECK:	binlh	0(%r15)			# encoding: [0xe3,0x90,0xf0,0x00,0x00,0x47]
@@ -150,9 +158,11 @@
 
 #CHECK:	bic	11, 0(%r7)		# encoding: [0xe3,0xb0,0x70,0x00,0x00,0x47]
 #CHECK:	binl	0(%r15)			# encoding: [0xe3,0xb0,0xf0,0x00,0x00,0x47]
+#CHECK:	binm	0(%r10)			# encoding: [0xe3,0xb0,0xa0,0x00,0x00,0x47]
 
 	bic	11, 0(%r7)
 	binl	0(%r15)
+	binm	0(%r10)
 
 #CHECK:	bic	12, 0(%r7)		# encoding: [0xe3,0xc0,0x70,0x00,0x00,0x47]
 #CHECK:	bile	0(%r15)			# encoding: [0xe3,0xc0,0xf0,0x00,0x00,0x47]
@@ -162,9 +172,11 @@
 
 #CHECK:	bic	13, 0(%r7)		# encoding: [0xe3,0xd0,0x70,0x00,0x00,0x47]
 #CHECK:	binh	0(%r15)			# encoding: [0xe3,0xd0,0xf0,0x00,0x00,0x47]
+#CHECK:	binp	0(%r9)			# encoding: [0xe3,0xd0,0x90,0x00,0x00,0x47]
 
 	bic	13, 0(%r7)
 	binh	0(%r15)
+	binp	0(%r9)
 
 #CHECK:	bic	14, 0(%r7)		# encoding: [0xe3,0xe0,0x70,0x00,0x00,0x47]
 #CHECK:	bino	0(%r15)			# encoding: [0xe3,0xe0,0xf0,0x00,0x00,0x47]
diff --git a/llvm/test/MC/SystemZ/insn-good-zEC12.s b/llvm/test/MC/SystemZ/insn-good-zEC12.s
index c0ff72298fa6..db37d28686e9 100644
--- a/llvm/test/MC/SystemZ/insn-good-zEC12.s
+++ b/llvm/test/MC/SystemZ/insn-good-zEC12.s
@@ -149,6 +149,12 @@
 #CHECK: cltne	%r0, 0(%r15)                # encoding: [0xeb,0x06,0xf0,0x00,0x00,0x23]
 #CHECK: cltnl	%r0, 0(%r15)                # encoding: [0xeb,0x0a,0xf0,0x00,0x00,0x23]
 #CHECK: cltnh	%r0, 0(%r15)                # encoding: [0xeb,0x0c,0xf0,0x00,0x00,0x23]
+#CHECK: cltnle  %r0, 0(%r15)                # encoding: [0xeb,0x02,0xf0,0x00,0x00,0x23]
+#CHECK: cltnhe  %r0, 0(%r15)                # encoding: [0xeb,0x04,0xf0,0x00,0x00,0x23]
+#CHECK: cltnlh  %r0, 0(%r15)                # encoding: [0xeb,0x08,0xf0,0x00,0x00,0x23]
+#CHECK: cltlh   %r0, 0(%r15)                # encoding: [0xeb,0x06,0xf0,0x00,0x00,0x23]
+#CHECK: clthe   %r0, 0(%r15)                # encoding: [0xeb,0x0a,0xf0,0x00,0x00,0x23]
+#CHECK: cltle   %r0, 0(%r15)                # encoding: [0xeb,0x0c,0xf0,0x00,0x00,0x23]
 
 	clt	%r0, 12, -524288
 	clt	%r0, 12, -1
@@ -165,6 +171,12 @@
 	cltne	%r0, 0(%r15)
 	cltnl	%r0, 0(%r15)
 	cltnh	%r0, 0(%r15)
+	cltnle  %r0, 0(%r15)
+	cltnhe  %r0, 0(%r15)
+	cltnlh  %r0, 0(%r15)
+	cltlh   %r0, 0(%r15)
+	clthe   %r0, 0(%r15)
+	cltle   %r0, 0(%r15)
 
 #CHECK: clgt	%r0, 12, -524288            # encoding: [0xeb,0x0c,0x00,0x00,0x80,0x2b]
 #CHECK: clgt	%r0, 12, -1                 # encoding: [0xeb,0x0c,0x0f,0xff,0xff,0x2b]
@@ -181,6 +193,12 @@
 #CHECK: clgtne	%r0, 0(%r15)                # encoding: [0xeb,0x06,0xf0,0x00,0x00,0x2b]
 #CHECK: clgtnl	%r0, 0(%r15)                # encoding: [0xeb,0x0a,0xf0,0x00,0x00,0x2b]
 #CHECK: clgtnh	%r0, 0(%r15)                # encoding: [0xeb,0x0c,0xf0,0x00,0x00,0x2b]
+#CHECK: clgtnle %r0, 0(%r15)                # encoding: [0xeb,0x02,0xf0,0x00,0x00,0x2b]
+#CHECK: clgtnhe %r0, 0(%r15)                # encoding: [0xeb,0x04,0xf0,0x00,0x00,0x2b]
+#CHECK: clgtnlh %r0, 0(%r15)                # encoding: [0xeb,0x08,0xf0,0x00,0x00,0x2b]
+#CHECK: clgtlh  %r0, 0(%r15)                # encoding: [0xeb,0x06,0xf0,0x00,0x00,0x2b]
+#CHECK: clgthe  %r0, 0(%r15)                # encoding: [0xeb,0x0a,0xf0,0x00,0x00,0x2b]
+#CHECK: clgtle  %r0, 0(%r15)                # encoding: [0xeb,0x0c,0xf0,0x00,0x00,0x2b]
 
 	clgt	%r0, 12, -524288
 	clgt	%r0, 12, -1
@@ -197,6 +215,12 @@
 	clgtne	%r0, 0(%r15)
 	clgtnl	%r0, 0(%r15)
 	clgtnh	%r0, 0(%r15)
+	clgtnle %r0, 0(%r15)
+	clgtnhe %r0, 0(%r15)
+	clgtnlh %r0, 0(%r15)
+	clgtlh  %r0, 0(%r15)
+	clgthe  %r0, 0(%r15)
+	clgtle  %r0, 0(%r15)
 
 #CHECK: crdte	%r0, %r0, %r0               # encoding: [0xb9,0x8f,0x00,0x00]
 #CHECK: crdte	%r0, %r0, %r14              # encoding: [0xb9,0x8f,0x00,0x0e]
diff --git a/llvm/test/MC/SystemZ/insn-good.s b/llvm/test/MC/SystemZ/insn-good.s
index f5dd672f9dd8..2add4a108319 100644
--- a/llvm/test/MC/SystemZ/insn-good.s
+++ b/llvm/test/MC/SystemZ/insn-good.s
@@ -970,9 +970,11 @@
 
 #CHECK:	bc	2, 0(%r7)		# encoding: [0x47,0x20,0x70,0x00]
 #CHECK:	bh	0(%r15)			# encoding: [0x47,0x20,0xf0,0x00]
+#CHECK:	bp	0(%r10)			# encoding: [0x47,0x20,0xa0,0x00]
 
 	bc	2, 0(%r7)
 	bh	0(%r15)
+	bp	0(%r10)
 
 #CHECK:	bc	3, 0(%r7)		# encoding: [0x47,0x30,0x70,0x00]
 #CHECK:	bnle	0(%r15)			# encoding: [0x47,0x30,0xf0,0x00]
@@ -982,9 +984,11 @@
 
 #CHECK:	bc	4, 0(%r7)		# encoding: [0x47,0x40,0x70,0x00]
 #CHECK:	bl	0(%r15)			# encoding: [0x47,0x40,0xf0,0x00]
+#CHECK:	bm	0(%r14)			# encoding: [0x47,0x40,0xe0,0x00]
 
 	bc	4, 0(%r7)
 	bl	0(%r15)
+	bm	0(%r14)
 
 #CHECK:	bc	5, 0(%r7)		# encoding: [0x47,0x50,0x70,0x00]
 #CHECK:	bnhe	0(%r15)			# encoding: [0x47,0x50,0xf0,0x00]
@@ -1000,9 +1004,11 @@
 
 #CHECK:	bc	7, 0(%r7)		# encoding: [0x47,0x70,0x70,0x00]
 #CHECK:	bne	0(%r15)			# encoding: [0x47,0x70,0xf0,0x00]
+#CHECK:	bnz	0(%r11)			# encoding: [0x47,0x70,0xb0,0x00]
 
 	bc	7, 0(%r7)
 	bne	0(%r15)
+	bnz	0(%r11)
 
 #CHECK:	bc	8, 0(%r7)		# encoding: [0x47,0x80,0x70,0x00]
 #CHECK:	be	0(%r15)			# encoding: [0x47,0x80,0xf0,0x00]
@@ -1024,9 +1030,11 @@
 
 #CHECK:	bc	11, 0(%r7)		# encoding: [0x47,0xb0,0x70,0x00]
 #CHECK:	bnl	0(%r15)			# encoding: [0x47,0xb0,0xf0,0x00]
+#CHECK:	bnm	0(%r13)			# encoding: [0x47,0xb0,0xd0,0x00]
 
 	bc	11, 0(%r7)
 	bnl	0(%r15)
+	bnm	0(%r13)
 
 #CHECK:	bc	12, 0(%r7)		# encoding: [0x47,0xc0,0x70,0x00]
 #CHECK:	ble	0(%r15)			# encoding: [0x47,0xc0,0xf0,0x00]
@@ -1036,9 +1044,11 @@
 
 #CHECK:	bc	13, 0(%r7)		# encoding: [0x47,0xd0,0x70,0x00]
 #CHECK:	bnh	0(%r15)			# encoding: [0x47,0xd0,0xf0,0x00]
+#CHECK:	bnp	0(%r12)			# encoding: [0x47,0xd0,0xc0,0x00]
 
 	bc	13, 0(%r7)
 	bnh	0(%r15)
+	bnp	0(%r12)
 
 #CHECK:	bc	14, 0(%r7)		# encoding: [0x47,0xe0,0x70,0x00]
 #CHECK:	bno	0(%r15)			# encoding: [0x47,0xe0,0xf0,0x00]
@@ -1046,6 +1056,13 @@
 	bc	14, 0(%r7)
 	bno	0(%r15)
 
+#CHECK:	bc	8, 0(%r13)			# encoding: [0x47,0x80,0xd0,0x00]
+#CHECK:	bz	0(%r6)			# encoding: [0x47,0x80,0x60,0x00]
+
+	bc	8, 0(%r13)
+	bz	0(%r6)
+
+
 #CHECK: bcr	0, %r0			# encoding: [0x07,0x00]
 #CHECK:	bcr	0, %r15			# encoding: [0x07,0x0f]
 
@@ -1176,6 +1193,36 @@
 	br	%r14
 	br	%r15
 
+#CHECK:	bcr	4, %r7			# encoding: [0x07,0x47]
+#CHECK:	bmr	%r0			# encoding: [0x07,0x40]
+	bcr	4, %r7
+	bmr	%r0
+
+#CHECK:	bcr	11, %r8			# encoding: [0x07,0xb8]
+#CHECK:	bnmr	%r1			# encoding: [0x07,0xb1]
+	bcr	11, %r8
+	bnmr	%r1
+
+#CHECK:	bcr	13, %r9			# encoding: [0x07,0xd9]
+#CHECK:	bnpr	%r2			# encoding: [0x07,0xd2]
+	bcr	13, %r9
+	bnpr	%r2
+
+#CHECK:	bcr	7, %r10			# encoding: [0x07,0x7a]
+#CHECK:	bnzr	%r3			# encoding: [0x07,0x73]
+	bcr	7, %r10
+	bnzr	%r3
+
+#CHECK:	bcr	2, %r11			# encoding: [0x07,0x2b]
+#CHECK:	bpr	%r4			# encoding: [0x07,0x24]
+	bcr	2, %r11
+	bpr	%r4
+
+#CHECK:	bcr	8, %r12			# encoding: [0x07,0x8c]
+#CHECK:	bzr	%r5			# encoding: [0x07,0x85]
+	bcr	8, %r12
+	bzr	%r5
+
 #CHECK: bras	%r0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x05,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 #CHECK: bras	%r0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x05,A,A]
@@ -3823,6 +3870,12 @@
 #CHECK: cgitne   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x60,0x70]
 #CHECK: cgitnl   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xa0,0x70]
 #CHECK: cgitnh   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xc0,0x70]
+#CHECK: cgitnle  %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x20,0x70]
+#CHECK: cgitnhe  %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x40,0x70]
+#CHECK: cgitnlh  %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x80,0x70]
+#CHECK: cgitlh   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x60,0x70]
+#CHECK: cgithe   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xa0,0x70]
+#CHECK: cgitle   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xc0,0x70]
 
         cgit     %r0, 0, 12
         cgit     %r0, -1, 12
@@ -3834,6 +3887,12 @@
         cgitne   %r15, 1
         cgitnl   %r15, 1
         cgitnh   %r15, 1
+        cgitnle  %r15, 1
+        cgitnhe  %r15, 1
+        cgitnlh  %r15, 1
+        cgitlh   %r15, 1
+        cgithe   %r15, 1
+        cgitle   %r15, 1
 
 #CHECK: cgr	%r0, %r0                # encoding: [0xb9,0x20,0x00,0x00]
 #CHECK: cgr	%r0, %r15               # encoding: [0xb9,0x20,0x00,0x0f]
@@ -4198,6 +4257,12 @@
 #CHECK: cgrtne   %r0, %r15              # encoding: [0xb9,0x60,0x60,0x0f]
 #CHECK: cgrtnl   %r0, %r15              # encoding: [0xb9,0x60,0xa0,0x0f]
 #CHECK: cgrtnh   %r0, %r15              # encoding: [0xb9,0x60,0xc0,0x0f]
+#CHECK: cgrtnle  %r0, %r15              # encoding: [0xb9,0x60,0x20,0x0f]
+#CHECK: cgrtnhe  %r0, %r15              # encoding: [0xb9,0x60,0x40,0x0f]
+#CHECK: cgrtnlh  %r0, %r15              # encoding: [0xb9,0x60,0x80,0x0f]
+#CHECK: cgrtlh   %r0, %r15              # encoding: [0xb9,0x60,0x60,0x0f]
+#CHECK: cgrthe   %r0, %r15              # encoding: [0xb9,0x60,0xa0,0x0f]
+#CHECK: cgrtle   %r0, %r15              # encoding: [0xb9,0x60,0xc0,0x0f]
 
         cgrt     %r0, %r1, 12
         cgrt     %r0, %r1, 12
@@ -4209,6 +4274,12 @@
         cgrtne   %r0, %r15
         cgrtnl   %r0, %r15
         cgrtnh   %r0, %r15
+        cgrtnle  %r0, %r15
+        cgrtnhe  %r0, %r15
+        cgrtnlh  %r0, %r15
+        cgrtlh   %r0, %r15
+        cgrthe   %r0, %r15
+        cgrtle   %r0, %r15
 
 #CHECK: cgxbr	%r0, 0, %f0             # encoding: [0xb3,0xaa,0x00,0x00]
 #CHECK: cgxbr	%r0, 0, %f13            # encoding: [0xb3,0xaa,0x00,0x0d]
@@ -4708,6 +4779,12 @@
 #CHECK: citne   %r15, 1                 # encoding: [0xec,0xf0,0x00,0x01,0x60,0x72]
 #CHECK: citnl   %r15, 1                 # encoding: [0xec,0xf0,0x00,0x01,0xa0,0x72]
 #CHECK: citnh   %r15, 1                 # encoding: [0xec,0xf0,0x00,0x01,0xc0,0x72]
+#CHECK: citnle  %r15, 1                 # encoding: [0xec,0xf0,0x00,0x01,0x20,0x72]
+#CHECK: citnhe  %r15, 1                 # encoding: [0xec,0xf0,0x00,0x01,0x40,0x72]
+#CHECK: citnlh  %r15, 1                 # encoding: [0xec,0xf0,0x00,0x01,0x80,0x72]
+#CHECK: citlh   %r15, 1                 # encoding: [0xec,0xf0,0x00,0x01,0x60,0x72]
+#CHECK: cithe   %r15, 1                 # encoding: [0xec,0xf0,0x00,0x01,0xa0,0x72]
+#CHECK: citle   %r15, 1                 # encoding: [0xec,0xf0,0x00,0x01,0xc0,0x72]
 
         cit     %r0, 0, 12
         cit     %r0, -1, 12
@@ -4719,6 +4796,12 @@
         citne   %r15, 1
         citnl   %r15, 1
         citnh   %r15, 1
+        citnle  %r15, 1
+        citnhe  %r15, 1
+        citnlh  %r15, 1
+        citlh   %r15, 1
+        cithe   %r15, 1
+        citle   %r15, 1
 
 #CHECK: cksm	%r0, %r8                # encoding: [0xb2,0x41,0x00,0x08]
 #CHECK: cksm	%r0, %r14               # encoding: [0xb2,0x41,0x00,0x0e]
@@ -4853,6 +4936,12 @@
 #CHECK: clfitne   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x60,0x73]
 #CHECK: clfitnl   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xa0,0x73]
 #CHECK: clfitnh   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xc0,0x73]
+#CHECK: clfitnle  %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x20,0x73]
+#CHECK: clfitnhe  %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x40,0x73]
+#CHECK: clfitnlh  %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x80,0x73]
+#CHECK: clfitlh   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x60,0x73]
+#CHECK: clfithe   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xa0,0x73]
+#CHECK: clfitle   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xc0,0x73]
 
         clfit     %r0, 0, 12
         clfit     %r0, 65535, 12
@@ -4863,6 +4952,12 @@
         clfitne   %r15, 1
         clfitnl   %r15, 1
         clfitnh   %r15, 1
+        clfitnle  %r15, 1
+        clfitnhe  %r15, 1
+        clfitnlh  %r15, 1
+        clfitlh   %r15, 1
+        clfithe   %r15, 1
+        clfitle   %r15, 1
 
 #CHECK: clg	%r0, -524288            # encoding: [0xe3,0x00,0x00,0x00,0x80,0x21]
 #CHECK: clg	%r0, -1                 # encoding: [0xe3,0x00,0x0f,0xff,0xff,0x21]
@@ -5332,6 +5427,12 @@
 #CHECK: clgitne   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x60,0x71]
 #CHECK: clgitnl   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xa0,0x71]
 #CHECK: clgitnh   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xc0,0x71]
+#CHECK: clgitnle  %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x20,0x71]
+#CHECK: clgitnhe  %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x40,0x71]
+#CHECK: clgitnlh  %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x80,0x71]
+#CHECK: clgitlh   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0x60,0x71]
+#CHECK: clgithe   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xa0,0x71]
+#CHECK: clgitle   %r15, 1                # encoding: [0xec,0xf0,0x00,0x01,0xc0,0x71]
 
         clgit     %r0, 0, 12
         clgit     %r0, 65535, 12
@@ -5342,6 +5443,12 @@
         clgitne   %r15, 1
         clgitnl   %r15, 1
         clgitnh   %r15, 1
+        clgitnle  %r15, 1
+        clgitnhe  %r15, 1
+        clgitnlh  %r15, 1
+        clgitlh   %r15, 1
+        clgithe   %r15, 1
+        clgitle   %r15, 1
 
 #CHECK: clgr	%r0, %r0                # encoding: [0xb9,0x21,0x00,0x00]
 #CHECK: clgr	%r0, %r15               # encoding: [0xb9,0x21,0x00,0x0f]
@@ -6250,6 +6357,12 @@
 #CHECK: clgrtne   %r0, %r15             # encoding: [0xb9,0x61,0x60,0x0f]
 #CHECK: clgrtnl   %r0, %r15             # encoding: [0xb9,0x61,0xa0,0x0f]
 #CHECK: clgrtnh   %r0, %r15             # encoding: [0xb9,0x61,0xc0,0x0f]
+#CHECK: clgrtnle  %r0, %r15             # encoding: [0xb9,0x61,0x20,0x0f]
+#CHECK: clgrtnhe  %r0, %r15             # encoding: [0xb9,0x61,0x40,0x0f]
+#CHECK: clgrtnlh  %r0, %r15             # encoding: [0xb9,0x61,0x80,0x0f]
+#CHECK: clgrtlh   %r0, %r15             # encoding: [0xb9,0x61,0x60,0x0f]
+#CHECK: clgrthe   %r0, %r15             # encoding: [0xb9,0x61,0xa0,0x0f]
+#CHECK: clgrtle   %r0, %r15             # encoding: [0xb9,0x61,0xc0,0x0f]
 
         clgrt     %r0, %r1, 12
         clgrt     %r0, %r1, 12
@@ -6261,6 +6374,12 @@
         clgrtne   %r0, %r15
         clgrtnl   %r0, %r15
         clgrtnh   %r0, %r15
+        clgrtnle  %r0, %r15
+        clgrtnhe  %r0, %r15
+        clgrtnlh  %r0, %r15
+        clgrtlh   %r0, %r15
+        clgrthe   %r0, %r15
+        clgrtle   %r0, %r15
 
 #CHECK: clrj	%r0, %r0, 0, .[[LAB:L.*]]	# encoding: [0xec,0x00,A,A,0x00,0x77]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
@@ -6539,6 +6658,12 @@
 #CHECK: clrtne   %r0, %r15              # encoding: [0xb9,0x73,0x60,0x0f]
 #CHECK: clrtnl   %r0, %r15              # encoding: [0xb9,0x73,0xa0,0x0f]
 #CHECK: clrtnh   %r0, %r15              # encoding: [0xb9,0x73,0xc0,0x0f]
+#CHECK: clrtnle  %r0, %r15              # encoding: [0xb9,0x73,0x20,0x0f]
+#CHECK: clrtnhe  %r0, %r15              # encoding: [0xb9,0x73,0x40,0x0f]
+#CHECK: clrtnlh  %r0, %r15              # encoding: [0xb9,0x73,0x80,0x0f]
+#CHECK: clrtlh   %r0, %r15              # encoding: [0xb9,0x73,0x60,0x0f]
+#CHECK: clrthe   %r0, %r15              # encoding: [0xb9,0x73,0xa0,0x0f]
+#CHECK: clrtle   %r0, %r15              # encoding: [0xb9,0x73,0xc0,0x0f]
 
         clrt     %r0, %r1, 12
         clrt     %r0, %r1, 12
@@ -6550,6 +6675,12 @@
         clrtne   %r0, %r15
         clrtnl   %r0, %r15
         clrtnh   %r0, %r15
+        clrtnle  %r0, %r15
+        clrtnhe  %r0, %r15
+        clrtnlh  %r0, %r15
+        clrtlh   %r0, %r15
+        clrthe   %r0, %r15
+        clrtle   %r0, %r15
 
 #CHECK: clst	%r0, %r0                # encoding: [0xb2,0x5d,0x00,0x00]
 #CHECK: clst	%r0, %r15               # encoding: [0xb2,0x5d,0x00,0x0f]
@@ -7012,6 +7143,12 @@
 #CHECK: crtne   %r0, %r15               # encoding: [0xb9,0x72,0x60,0x0f]
 #CHECK: crtnl   %r0, %r15               # encoding: [0xb9,0x72,0xa0,0x0f]
 #CHECK: crtnh   %r0, %r15               # encoding: [0xb9,0x72,0xc0,0x0f]
+#CHECK: crtnle  %r0, %r15               # encoding: [0xb9,0x72,0x20,0x0f]
+#CHECK: crtnhe  %r0, %r15               # encoding: [0xb9,0x72,0x40,0x0f]
+#CHECK: crtnlh  %r0, %r15               # encoding: [0xb9,0x72,0x80,0x0f]
+#CHECK: crtlh   %r0, %r15               # encoding: [0xb9,0x72,0x60,0x0f]
+#CHECK: crthe   %r0, %r15               # encoding: [0xb9,0x72,0xa0,0x0f]
+#CHECK: crtle   %r0, %r15               # encoding: [0xb9,0x72,0xc0,0x0f]
 
         crt     %r0, %r1, 12
         crt     %r0, %r1, 12
@@ -7023,6 +7160,12 @@
         crtne   %r0, %r15
         crtnl   %r0, %r15
         crtnh   %r0, %r15
+        crtnle  %r0, %r15
+        crtnhe  %r0, %r15
+        crtnlh  %r0, %r15
+        crtlh   %r0, %r15
+        crthe   %r0, %r15
+        crtle   %r0, %r15
 
 #CHECK: cs	%r0, %r0, 0             # encoding: [0xba,0x00,0x00,0x00]
 #CHECK: cs	%r0, %r0, 4095          # encoding: [0xba,0x00,0x0f,0xff]
-- 
GitLab


From e7d09cecc9123f89ace1712a617e252d78b179e9 Mon Sep 17 00:00:00 2001
From: Petr Kurapov <petr.kurapov@gmail.com>
Date: Tue, 14 May 2024 11:50:35 +0200
Subject: [PATCH 0705/1206] [MLIR][Linalg] Ternary Op & Linalg select (#91461)

Following #90236, adding `select` to linalg as `arith.select`. No
implicit type casting.
OpDSL doesn't expose a type restriction for bool, but I saw no reason in
adding it (put a separate symbolic type and check the semantics in the
builder).

---------

Co-authored-by: Renato Golin <rengolin@systemcall.eu>
Co-authored-by: Maksim Levental <maksim.levental@gmail.com>
---
 .../mlir/Dialect/Linalg/IR/LinalgBase.td      |  3 +
 .../mlir/Dialect/Linalg/IR/LinalgEnums.td     |  6 ++
 .../Linalg/IR/LinalgNamedStructuredOps.yaml   | 57 +++++++++++++++++
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      | 19 ++++++
 .../linalg/opdsl/lang/comprehension.py        | 61 ++++++++++++++++++-
 .../dialects/linalg/opdsl/lang/emitter.py     |  7 +++
 .../linalg/opdsl/ops/core_named_ops.py        | 20 ++++++
 .../Dialect/Linalg/generalize-named-ops.mlir  | 25 ++++++++
 mlir/test/Dialect/Linalg/named-ops-fail.mlir  | 16 +++++
 mlir/test/Dialect/Linalg/named-ops.mlir       | 34 +++++++++++
 .../mlir-linalg-ods-yaml-gen.cpp              | 10 ++-
 11 files changed, 255 insertions(+), 3 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
index e87e8b560010..73f984dc072d 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
@@ -68,6 +68,9 @@ def UnaryFnAttr : EnumAttr<Linalg_Dialect, UnaryFn, "unary_fn"> {
 def BinaryFnAttr : EnumAttr<Linalg_Dialect, BinaryFn, "binary_fn"> {
   let assemblyFormat = "`<` $value `>`";
 }
+def TernaryFnAttr : EnumAttr<Linalg_Dialect, TernaryFn, "ternary_fn"> {
+  let assemblyFormat = "`<` $value `>`";
+}
 def TypeFnAttr : EnumAttr<Linalg_Dialect, TypeFn, "type_fn"> {
   let assemblyFormat = "`<` $value `>`";
 }
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgEnums.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgEnums.td
index 6b4b073fc672..e615876a95d0 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgEnums.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgEnums.td
@@ -49,6 +49,12 @@ def BinaryFn : I32EnumAttr<"BinaryFn", "", [
   let genSpecializedAttr = 0;
   let cppNamespace = "::mlir::linalg";
 }
+def TernaryFn : I32EnumAttr<"TernaryFn", "", [
+  I32EnumAttrCase<"select", 0>
+]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::mlir::linalg";
+}
 def TypeFn : I32EnumAttr<"TypeFn", "", [
   I32EnumAttrCase<"cast_signed", 0>,
   I32EnumAttrCase<"cast_unsigned", 1>
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
index 584bfcd8b59d..eb7dd37010a6 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
@@ -1008,6 +1008,63 @@ structured_op: !LinalgStructuredOpConfig
         - !ScalarExpression
           scalar_arg: rhs
 --- !LinalgOpConfig
+metadata: !LinalgOpMetadata
+  name: select
+  cpp_class_name: SelectOp
+  doc: |-
+    Chooses one value based on a binary condition supplied as its first operand.
+
+    The shapes and element types must be identical. The appropriate casts,
+    broadcasts and reductions should be done previously to calling this op.
+
+    This means reduction/broadcast/element cast semantics is explicit. Further
+    passes can take that into account when lowering this code. For example,
+    a `linalg.broadcast` + `linalg.select` sequence can be lowered to a
+    `linalg.generic` with different affine maps for the two operands.
+structured_op: !LinalgStructuredOpConfig
+  args:
+  - !LinalgOperandDefConfig
+    name: cond
+    kind: input_tensor
+    type_var: U
+    shape_map: affine_map<() -> ()>
+  - !LinalgOperandDefConfig
+    name: lhs
+    kind: input_tensor
+    type_var: T1
+    shape_map: affine_map<() -> ()>
+  - !LinalgOperandDefConfig
+    name: rhs
+    kind: input_tensor
+    type_var: T1
+    shape_map: affine_map<() -> ()>
+  - !LinalgOperandDefConfig
+    name: O
+    kind: output_tensor
+    type_var: T1
+    shape_map: affine_map<() -> ()>
+  indexing_maps: !LinalgIndexingMapsConfig
+    static_indexing_maps:
+    - affine_map<() -> ()>
+    - affine_map<() -> ()>
+    - affine_map<() -> ()>
+    - affine_map<() -> ()>
+  iterator_types: []
+  assignments:
+  - !ScalarAssign
+    arg: O
+    value: !ScalarExpression
+      scalar_fn:
+        kind: ternary
+        fn_name: select
+        operands:
+        - !ScalarExpression
+          scalar_arg: cond
+        - !ScalarExpression
+          scalar_arg: lhs
+        - !ScalarExpression
+          scalar_arg: rhs
+--- !LinalgOpConfig
 metadata: !LinalgOpMetadata
   name: matmul
   cpp_class_name: MatmulOp
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index e5f83331baf8..6a5f25a7605f 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -492,6 +492,25 @@ public:
     llvm_unreachable("unsupported binary function");
   }
 
+  // Build the ternary functions defined by OpDSL.
+  Value buildTernaryFn(TernaryFn ternaryFn, Value arg0, Value arg1,
+                       Value arg2) {
+    bool headBool =
+        isInteger(arg0) && arg0.getType().getIntOrFloatBitWidth() == 1;
+    bool tailFloatingPoint =
+        isFloatingPoint(arg0) && isFloatingPoint(arg1) && isFloatingPoint(arg2);
+    bool tailInteger = isInteger(arg0) && isInteger(arg1) && isInteger(arg1);
+    OpBuilder::InsertionGuard g(builder);
+    builder.setInsertionPointToEnd(&block);
+    switch (ternaryFn) {
+    case TernaryFn::select:
+      if (!headBool && !(tailFloatingPoint || tailInteger))
+        llvm_unreachable("unsupported non numeric type");
+      return builder.create<arith::SelectOp>(arg0.getLoc(), arg0, arg1, arg2);
+    }
+    llvm_unreachable("unsupported ternary function");
+  }
+
   // Build the type functions defined by OpDSL.
   Value buildTypeFn(TypeFn typeFn, Type toType, Value operand) {
     switch (typeFn) {
diff --git a/mlir/python/mlir/dialects/linalg/opdsl/lang/comprehension.py b/mlir/python/mlir/dialects/linalg/opdsl/lang/comprehension.py
index bb43ebf2b692..1a198fc5ec6f 100644
--- a/mlir/python/mlir/dialects/linalg/opdsl/lang/comprehension.py
+++ b/mlir/python/mlir/dialects/linalg/opdsl/lang/comprehension.py
@@ -262,7 +262,8 @@ class index(TensorExpression):
 class FunctionKind(Enum):
     UNARY = 0
     BINARY = 1
-    TYPE = 2
+    TERNARY = 2
+    TYPE = 3
 
 
 class UnaryFnType:
@@ -339,6 +340,33 @@ class BinaryFn:
     powf = BinaryFnType("powf")
 
 
+class TernaryFnType:
+    """Ternary function.
+
+    A ternary function takes three tensor expressions and returns the
+    function evaluation result.
+    """
+
+    def __init__(self, fn_name: str):
+        self.fn_name = fn_name
+
+    def __call__(
+        self, arg0: TensorExpression, arg1: TensorExpression, arg2: TensorExpression
+    ) -> "TensorFn":
+        return TensorFn(
+            FunctionKind.TERNARY, self.fn_name, None, None, [arg0, arg1, arg2]
+        )
+
+    def __repr__(self):
+        return f"{self.fn_name}"
+
+
+class TernaryFn:
+    """Ternary function namespace."""
+
+    select = TernaryFnType("select")
+
+
 class TypeFnType:
     """Type conversion function.
 
@@ -437,7 +465,8 @@ class OperandKind(Enum):
     INDEX_ATTR = 3
     UNARY_FN_ATTR = 4
     BINARY_FN_ATTR = 5
-    TYPE_FN_ATTR = 6
+    TERNARY_FN_ATTR = 6
+    TYPE_FN_ATTR = 7
 
 
 class OperandDef:
@@ -489,6 +518,7 @@ class OperandDef:
             self.kind == OperandKind.INDEX_ATTR
             or self.kind == OperandKind.UNARY_FN_ATTR
             or self.kind == OperandKind.BINARY_FN_ATTR
+            or self.kind == OperandKind.TERNARY_FN_ATTR
             or self.kind == OperandKind.TYPE_FN_ATTR
         )
 
@@ -670,6 +700,33 @@ class BinaryFnAttrDef:
         return ReduceFnUse(None, self, *reduce_dims)
 
 
+class TernaryFnAttrDef:
+    """Ternary function attribute definition.
+
+    Ternary function attributes provide a way to make the arithmetic computation
+    parametrizable. Every attribute specifies a default Ternary function
+    that may be overwritten at operation instantiation time.
+    """
+
+    def __init__(self, default: "TernaryFnType"):
+        if not isinstance(default, TernaryFnType):
+            raise ValueError(
+                f"TernaryFnAttrDef requires default of type TernaryFnType "
+                f"but got {default}"
+            )
+        self.operand_def = OperandDef(
+            OperandKind.TERNARY_FN_ATTR, default_fn=default.fn_name
+        )
+
+    def __call__(self, arg0: TensorExpression, arg1: TensorExpression) -> TensorFn:
+        return TensorFn(
+            FunctionKind.TERNARY, None, self.operand_def, None, [arg0, arg1]
+        )
+
+    def __getitem__(self, reduce_dims: Tuple[DimDef]) -> ReduceFnUse:
+        return ReduceFnUse(None, self, *reduce_dims)
+
+
 class TypeFnAttrDef:
     """Type conversion function attribute definition.
 
diff --git a/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py b/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py
index f91fc8b71600..845b533db52a 100644
--- a/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py
+++ b/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py
@@ -60,6 +60,7 @@ def prepare_common_structured_op(
         in [
             OperandKind.UNARY_FN_ATTR,
             OperandKind.BINARY_FN_ATTR,
+            OperandKind.TERNARY_FN_ATTR,
             OperandKind.TYPE_FN_ATTR,
         ]
     ]
@@ -180,6 +181,12 @@ def prepare_common_structured_op(
                         f"Attribute {fn_attr.name} needs to be of type "
                         f"BinaryFnType but got {type(attr_val)}"
                     )
+            elif attr_kind == OperandKind.TERNARY_FN_ATTR:
+                if not isinstance(fn, TernaryFnType):
+                    raise ValueError(
+                        f"Attribute {fn_attr.name} needs to be of type "
+                        f"TernaryFnType but got {type(attr_val)}"
+                    )
             else:
                 if not isinstance(fn, TypeFnType):
                     raise ValueError(
diff --git a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
index ca2bb0c5f7f8..d73428a0f4df 100644
--- a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
+++ b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
@@ -351,6 +351,26 @@ def powf(
     O[None] = BinaryFn.powf(lhs[None], rhs[None])
 
 
+@linalg_structured_op
+def select(
+    cond=TensorDef(U),
+    lhs=TensorDef(T1),
+    rhs=TensorDef(T1),
+    O=TensorDef(T1, output=True),
+):
+    """Chooses one value based on a binary condition supplied as its first operand.
+
+    The shapes and element types must be identical. The appropriate casts,
+    broadcasts and reductions should be done previously to calling this op.
+
+    This means reduction/broadcast/element cast semantics is explicit. Further
+    passes can take that into account when lowering this code. For example,
+    a `linalg.broadcast` + `linalg.select` sequence can be lowered to a
+    `linalg.generic` with different affine maps for the two operands.
+    """
+    O[None] = TernaryFn.select(cond[None], lhs[None], rhs[None])
+
+
 @linalg_structured_op
 def matmul(
     A=TensorDef(T1, S.M, S.K),
diff --git a/mlir/test/Dialect/Linalg/generalize-named-ops.mlir b/mlir/test/Dialect/Linalg/generalize-named-ops.mlir
index 667ea3c18c8a..4f43ec2c9e1c 100644
--- a/mlir/test/Dialect/Linalg/generalize-named-ops.mlir
+++ b/mlir/test/Dialect/Linalg/generalize-named-ops.mlir
@@ -791,6 +791,31 @@ func.func @generalize_powf(%lhs: memref<7x14x21xf32>, %rhs: memref<7x14x21xf32>,
 
 // -----
 
+func.func @generalize_select(%cond: memref<7x14x21xi1>, %lhs: memref<7x14x21xf32>, %rhs: memref<7x14x21xf32>,
+                              %out: memref<7x14x21xf32>) {
+  linalg.select ins(%cond, %lhs, %rhs: memref<7x14x21xi1>, memref<7x14x21xf32>, memref<7x14x21xf32>)
+                outs(%out: memref<7x14x21xf32>)
+  return
+}
+
+// CHECK: #[[MAP:.+]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+
+// CHECK: func @generalize_select
+// CHECK-SAME: (%[[COND:.+]]: memref<7x14x21xi1>, %[[LHS:.+]]: memref<7x14x21xf32>, %[[RHS:.+]]: memref<7x14x21xf32>,
+// CHECK-SAME:  %[[OUT:.+]]: memref<7x14x21xf32>)
+
+// CHECK: linalg.generic
+// CHECK-SAME: indexing_maps = [#[[MAP]], #[[MAP]], #[[MAP]], #[[MAP]]]
+// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel"]}
+// CHECK-SAME:  ins(%[[COND]], %[[LHS]], %[[RHS]] : memref<7x14x21xi1>, memref<7x14x21xf32>, memref<7x14x21xf32>)
+// CHECK-SAME: outs(%[[OUT]] : memref<7x14x21xf32>)
+
+// CHECK:         ^{{.+}}(%[[BBARG0:.+]]: i1, %[[BBARG1:.+]]: f32, %[[BBARG2:.+]]: f32, %[[BBARG3:.+]]: f32)
+// CHECK-NEXT:      %[[select:.+]] = arith.select %[[BBARG0]], %[[BBARG1]], %[[BBARG2]] : f32
+// CHECK-NEXT:      linalg.yield %[[select]] : f32
+
+
+// -----
 
 // CHECK-LABEL: func @fill_tensor
 func.func @fill_tensor(%f: f32, %v: vector<2x4xf32>) -> (tensor<f32>, tensor<vector<2x4xf32>>) {
diff --git a/mlir/test/Dialect/Linalg/named-ops-fail.mlir b/mlir/test/Dialect/Linalg/named-ops-fail.mlir
index e92a77aa7ad0..552a0abaa797 100644
--- a/mlir/test/Dialect/Linalg/named-ops-fail.mlir
+++ b/mlir/test/Dialect/Linalg/named-ops-fail.mlir
@@ -334,3 +334,19 @@ func.func @powf_broadcast(%arg0: memref<8x16xf32>, %arg1: memref<4x8x16xf32>, %a
   return
 }
 
+// -----
+
+func.func @select_type_cast(%arg0: memref<4x8x16xi1>, %arg1: memref<4x8x16xf16>, %arg2: memref<4x8x16xf32>, %arg3: memref<4x8x16xf32>) {
+  // CHECK: op failed to verify that all of {true_value, false_value, result} have same type
+  linalg.select ins(%arg0, %arg1, %arg2 : memref<4x8x16xi1>, memref<4x8x16xf16>, memref<4x8x16xf32>) outs(%arg3: memref<4x8x16xf32>)
+  return
+}
+
+// -----
+
+func.func @select_wrong_condition_type(%arg0: memref<4x8x16xf32>, %arg1: memref<4x8x16xf32>, %arg2: memref<4x8x16xf32>, %arg3: memref<4x8x16xf32>) {
+  // CHECK: op operand #0 must be bool-like, but got 'f32'
+  linalg.select ins(%arg0, %arg1, %arg2 : memref<4x8x16xf32>, memref<4x8x16xf32>, memref<4x8x16xf32>) outs(%arg3: memref<4x8x16xf32>)
+  return
+}
+
diff --git a/mlir/test/Dialect/Linalg/named-ops.mlir b/mlir/test/Dialect/Linalg/named-ops.mlir
index fefe5578947f..051054e67edf 100644
--- a/mlir/test/Dialect/Linalg/named-ops.mlir
+++ b/mlir/test/Dialect/Linalg/named-ops.mlir
@@ -1924,3 +1924,37 @@ func.func @fill_tensor(%f: f32, %v: vector<2x4xf32>) -> (tensor<f32>, tensor<vec
   %1 = linalg.fill ins(%v : vector<2x4xf32>) outs(%e1 : tensor<vector<2x4xf32>>) -> tensor<vector<2x4xf32>>
   return %0, %1: tensor<f32>, tensor<vector<2x4xf32>>
 }
+
+// -----
+
+// CHECK-LABEL: func @select_dynamic
+func.func @select_dynamic(%arg0: memref<?x?x?xi1>, %arg1: memref<?x?x?xf32>, %arg2: memref<?x?x?xf32>, %arg3: memref<?x?x?xf32>) {
+  // CHECK: linalg.select
+  // CHECK-SAME: ins(%{{.+}}, %{{.+}}, %{{.+}} : memref<?x?x?xi1>, memref<?x?x?xf32>, memref<?x?x?xf32>)
+  // CHECK-SAME: outs(%{{.+}} : memref<?x?x?xf32>)
+  linalg.select ins(%arg0, %arg1, %arg2 : memref<?x?x?xi1>, memref<?x?x?xf32>, memref<?x?x?xf32>) outs(%arg3: memref<?x?x?xf32>)
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func @select_static
+func.func @select_static(%arg0: memref<4x8x16xi1>, %arg1: memref<4x8x16xf32>, %arg2: memref<4x8x16xf32>, %arg3: memref<4x8x16xf32>) {
+  // CHECK: linalg.select
+  // CHECK-SAME: ins(%{{.+}}, %{{.+}}, %{{.+}} : memref<4x8x16xi1>, memref<4x8x16xf32>, memref<4x8x16xf32>)
+  // CHECK-SAME: outs(%{{.+}} : memref<4x8x16xf32>)
+  linalg.select ins(%arg0, %arg1, %arg2 : memref<4x8x16xi1>, memref<4x8x16xf32>, memref<4x8x16xf32>) outs(%arg3: memref<4x8x16xf32>)
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func @select_tensor
+func.func @select_tensor(%arg0: tensor<4x8x16xi1>, %arg1: tensor<4x8x16xf32>, %arg2: tensor<4x8x16xf32>) -> tensor<4x8x16xf32> {
+  %0 = tensor.empty() : tensor<4x8x16xf32>
+  // CHECK: linalg.select
+  // CHECK-SAME: ins(%{{.+}}, %{{.+}}, %{{.+}} : tensor<4x8x16xi1>, tensor<4x8x16xf32>, tensor<4x8x16xf32>)
+  // CHECK-SAME: outs(%{{.+}} : tensor<4x8x16xf32>)
+  %1 = linalg.select ins(%arg0, %arg1, %arg2 : tensor<4x8x16xi1>, tensor<4x8x16xf32>, tensor<4x8x16xf32>) outs(%0: tensor<4x8x16xf32>) -> tensor<4x8x16xf32>
+  return %1 : tensor<4x8x16xf32>
+}
diff --git a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
index fe6ad1504112..37240164c377 100644
--- a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
+++ b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
@@ -70,6 +70,7 @@ enum class LinalgOperandDefKind {
   IndexAttr,
   UnaryFnAttr,
   BinaryFnAttr,
+  TernaryFnAttr,
   TypeFnAttr
 };
 
@@ -94,7 +95,7 @@ struct LinalgIndexingMapsConfig {
 
 struct ScalarExpression;
 
-enum class ScalarFnKind { Unary, Binary, Type };
+enum class ScalarFnKind { Unary, Binary, Ternary, Type };
 
 struct ScalarFn {
   ScalarFnKind kind;
@@ -214,6 +215,7 @@ struct ScalarEnumerationTraits<LinalgOperandDefKind> {
     io.enumCase(value, "index_attr", LinalgOperandDefKind::IndexAttr);
     io.enumCase(value, "unary_fn_attr", LinalgOperandDefKind::UnaryFnAttr);
     io.enumCase(value, "binary_fn_attr", LinalgOperandDefKind::BinaryFnAttr);
+    io.enumCase(value, "ternary_fn_attr", LinalgOperandDefKind::TernaryFnAttr);
     io.enumCase(value, "type_fn_attr", LinalgOperandDefKind::TypeFnAttr);
   }
 };
@@ -284,6 +286,7 @@ struct ScalarEnumerationTraits<ScalarFnKind> {
   static void enumeration(IO &io, ScalarFnKind &value) {
     io.enumCase(value, "unary", ScalarFnKind::Unary);
     io.enumCase(value, "binary", ScalarFnKind::Binary);
+    io.enumCase(value, "ternary", ScalarFnKind::Ternary);
     io.enumCase(value, "type", ScalarFnKind::Type);
   }
 };
@@ -441,6 +444,7 @@ static ScalarAssign *findAssignment(StringRef name,
 static bool isFunctionAttribute(LinalgOperandDefKind kind) {
   return kind == LinalgOperandDefKind::UnaryFnAttr ||
          kind == LinalgOperandDefKind::BinaryFnAttr ||
+         kind == LinalgOperandDefKind::TernaryFnAttr ||
          kind == LinalgOperandDefKind::TypeFnAttr;
 }
 
@@ -456,6 +460,8 @@ std::string convertOperandKindToEnumName(LinalgOperandDefKind kind) {
     return std::string("UnaryFn");
   case LinalgOperandDefKind::BinaryFnAttr:
     return std::string("BinaryFn");
+  case LinalgOperandDefKind::TernaryFnAttr:
+    return std::string("TernaryFn");
   case LinalgOperandDefKind::TypeFnAttr:
     return std::string("TypeFn");
   default:
@@ -471,6 +477,8 @@ std::string convertFunctionKindToEnumName(ScalarFnKind kind) {
     return std::string("UnaryFn");
   case ScalarFnKind::Binary:
     return std::string("BinaryFn");
+  case ScalarFnKind::Ternary:
+    return std::string("TernaryFn");
   case ScalarFnKind::Type:
     return std::string("TypeFn");
   }
-- 
GitLab


From 8fe21fda7469f2fdf83980a2720a15baad74ae4f Mon Sep 17 00:00:00 2001
From: Andrew Sukach <134116196+soukatch@users.noreply.github.com>
Date: Tue, 14 May 2024 05:57:10 -0400
Subject: [PATCH 0706/1206] [clang][analyzer] Ignore try-statements in
 UnreachableCode checker (#91675)

Fixes #90162
---
 .../Checkers/UnreachableCodeChecker.cpp             |  4 +++-
 clang/test/Analysis/unreachable-code-exceptions.cpp | 13 +++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/Analysis/unreachable-code-exceptions.cpp

diff --git a/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
index d24a124f5ffe..7ce9a5b5bb6d 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
@@ -159,6 +159,8 @@ void UnreachableCodeChecker::checkEndAnalysis(ExplodedGraph &G,
       SL = DL.asLocation();
       if (SR.isInvalid() || !SL.isValid())
         continue;
+      if (isa<CXXTryStmt>(S))
+        continue;
     }
     else
       continue;
@@ -254,4 +256,4 @@ void ento::registerUnreachableCodeChecker(CheckerManager &mgr) {
 
 bool ento::shouldRegisterUnreachableCodeChecker(const CheckerManager &mgr) {
   return true;
-}
+}
\ No newline at end of file
diff --git a/clang/test/Analysis/unreachable-code-exceptions.cpp b/clang/test/Analysis/unreachable-code-exceptions.cpp
new file mode 100644
index 000000000000..f47674ea8097
--- /dev/null
+++ b/clang/test/Analysis/unreachable-code-exceptions.cpp
@@ -0,0 +1,13 @@
+// RUN: %clang_analyze_cc1 -verify %s -fcxx-exceptions -fexceptions -analyzer-checker=core,alpha.deadcode.UnreachableCode
+
+// expected-no-diagnostics
+
+void foo();
+
+void fp_90162() {
+  try { // no-warning: The TryStmt shouldn't be unreachable.
+    foo();
+  } catch (int) {
+    foo(); // We assume that catch handlers are reachable.
+  }
+}
-- 
GitLab


From 7b1b1279414217ea7f2402a03dfb5a18ea5a5367 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Tue, 14 May 2024 13:57:37 +0400
Subject: [PATCH 0707/1206] [lldb][Windows] Enforce exec permission using
 Platform::Install() from Windows host (#91887)

Target::Install() set 0700 permissions for the main executable file.
Platform::Install() just copies permissions from the source. But the
permission eFilePermissionsUserExecute is missing on the Windows host. A
lot of tests failed in case of Windows host and Linux target because of
this issue. There is no API to provide the exec flag. This patch set the
permission eFilePermissionsUserExecute for all files installed via
Platform::Install() from the Windows host. It fixes a lot of tests in
case of Windows host and Linux target.
---
 lldb/source/Target/Platform.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/source/Target/Platform.cpp b/lldb/source/Target/Platform.cpp
index 4af4aa68ccd0..ee1f92470e16 100644
--- a/lldb/source/Target/Platform.cpp
+++ b/lldb/source/Target/Platform.cpp
@@ -1225,7 +1225,7 @@ Status Platform::PutFile(const FileSpec &source, const FileSpec &destination,
 
   uint32_t permissions = source_file.get()->GetPermissions(error);
   if (permissions == 0)
-    permissions = lldb::eFilePermissionsFileDefault;
+    permissions = lldb::eFilePermissionsUserRWX;
 
   lldb::user_id_t dest_file = OpenFile(
       destination, File::eOpenOptionCanCreate | File::eOpenOptionWriteOnly |
-- 
GitLab


From d7ef34bfe3d432ffd66a05fc9fcc87fd6c3db2ee Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <r@artagnon.com>
Date: Tue, 14 May 2024 10:59:26 +0100
Subject: [PATCH 0708/1206] [LV] update comment following 63d8058 (NFC)
 (#91120)

Address a review comment post landing 63d8058 (LoopVectorize: guard
appending InstsToScalarize; fix bug) to update a comment.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ba02c98285c3..9353666e417c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5813,10 +5813,11 @@ void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
     for (Instruction &I : *BB)
       if (isScalarWithPredication(&I, VF)) {
         ScalarCostsTy ScalarCosts;
-        // Do not apply discount if scalable, because that would lead to
-        // invalid scalarization costs.
-        // Do not apply discount logic if hacked cost is needed
-        // for emulated masked memrefs.
+        // Do not apply discount logic for:
+        // 1. Scalars after vectorization, as there will only be a single copy
+        // of the instruction.
+        // 2. Scalable VF, as that would lead to invalid scalarization costs.
+        // 3. Emulated masked memrefs, if a hacked cost is needed.
         if (!isScalarAfterVectorization(&I, VF) && !VF.isScalable() &&
             !useEmulatedMaskMemRefHack(&I, VF) &&
             computePredInstDiscount(&I, ScalarCosts, VF) >= 0)
-- 
GitLab


From ac42f7689d741feda2badc438101e7952db048f3 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Tue, 14 May 2024 14:00:16 +0400
Subject: [PATCH 0709/1206] [lldb] Fixed the test TestDyldLaunchLinux (#92080)

Install a.out and libsignal_file.so to the remote target if necessary.
---
 .../dyld-launch-linux/TestDyldLaunchLinux.py              | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lldb/test/API/functionalities/dyld-launch-linux/TestDyldLaunchLinux.py b/lldb/test/API/functionalities/dyld-launch-linux/TestDyldLaunchLinux.py
index 26360c20db1e..c4eba023ea72 100644
--- a/lldb/test/API/functionalities/dyld-launch-linux/TestDyldLaunchLinux.py
+++ b/lldb/test/API/functionalities/dyld-launch-linux/TestDyldLaunchLinux.py
@@ -7,6 +7,7 @@ import os
 
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
 
 
 class TestLinux64LaunchingViaDynamicLoader(TestBase):
@@ -39,11 +40,16 @@ class TestLinux64LaunchingViaDynamicLoader(TestBase):
         breakpoint_shared_library = target.BreakpointCreateBySourceRegex(
             "get_signal_crash", lldb.SBFileSpec("signal_file.cpp")
         )
+        inferior_exe_path = lldbutil.install_to_target(
+            self, self.getBuildArtifact("a.out")
+        )
+        lldbutil.install_to_target(self, self.getBuildArtifact("libsignal_file.so"))
+
         launch_info = lldb.SBLaunchInfo(
             [
                 "--library-path",
                 self.get_process_working_directory(),
-                self.getBuildArtifact("a.out"),
+                inferior_exe_path,
             ]
         )
         launch_info.SetWorkingDirectory(self.get_process_working_directory())
-- 
GitLab


From f658d84e01bcdd49e27dc9ef80e1a6cc5f9417fe Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Tue, 14 May 2024 14:02:31 +0400
Subject: [PATCH 0710/1206] [lldb] Fixed the test TestExec (#92082)

Install `secondprog` to the remote target if necessary.
---
 lldb/test/API/functionalities/exec/TestExec.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lldb/test/API/functionalities/exec/TestExec.py b/lldb/test/API/functionalities/exec/TestExec.py
index aab1f5c9455f..968b879c4cd4 100644
--- a/lldb/test/API/functionalities/exec/TestExec.py
+++ b/lldb/test/API/functionalities/exec/TestExec.py
@@ -45,6 +45,8 @@ class ExecTestCase(TestBase):
         # Create the target
         target = self.dbg.CreateTarget(exe)
 
+        lldbutil.install_to_target(self, secondprog)
+
         # Create any breakpoints we need
         breakpoint1 = target.BreakpointCreateBySourceRegex(
             "Set breakpoint 1 here", lldb.SBFileSpec("main.c", False)
@@ -143,6 +145,8 @@ class ExecTestCase(TestBase):
         exe = self.getBuildArtifact("a.out")
         target = self.dbg.CreateTarget(exe)
 
+        lldbutil.install_to_target(self, self.getBuildArtifact("secondprog"))
+
         (target, process, thread, breakpoint1) = lldbutil.run_to_source_breakpoint(
             self, "Set breakpoint 1 here", lldb.SBFileSpec("main.c", False)
         )
-- 
GitLab


From 3aae916ff7fe9d0953aa63b2ba1d0e871f6f76fc Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Tue, 14 May 2024 18:10:28 +0800
Subject: [PATCH 0711/1206] Reland "[ValueTracking] Compute knownbits from
 known fp classes" (#92084)

This patch relands https://github.com/llvm/llvm-project/pull/86409.

I mistakenly thought that `Known.makeNegative()` clears the sign bit of
`Known.Zero`. This patch fixes the assertion failure by explicitly
clearing the sign bit.
---
 llvm/include/llvm/IR/PatternMatch.h           |   2 +-
 llvm/lib/Analysis/ValueTracking.cpp           |  35 +++
 .../AMDGPU/amdgpu-simplify-libcall-pow.ll     |  14 +-
 .../AMDGPU/amdgpu-simplify-libcall-pown.ll    |  12 +-
 llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll |  10 +-
 .../test/Transforms/InstCombine/known-bits.ll | 284 ++++++++++++++++++
 6 files changed, 338 insertions(+), 19 deletions(-)

diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 171ddab977de..0d6d86cb47e6 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -1904,7 +1904,7 @@ template <typename Op_t> struct ElementWiseBitCast_match {
   ElementWiseBitCast_match(const Op_t &OpMatch) : Op(OpMatch) {}
 
   template <typename OpTy> bool match(OpTy *V) {
-    BitCastInst *I = dyn_cast<BitCastInst>(V);
+    auto *I = dyn_cast<BitCastInst>(V);
     if (!I)
       return false;
     Type *SrcType = I->getSrcTy();
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 375385aca7a3..c8c527a2d4d2 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1118,6 +1118,41 @@ static void computeKnownBitsFromOperator(const Operator *I,
       break;
     }
 
+    const Value *V;
+    // Handle bitcast from floating point to integer.
+    if (match(I, m_ElementWiseBitCast(m_Value(V))) &&
+        V->getType()->isFPOrFPVectorTy()) {
+      Type *FPType = V->getType()->getScalarType();
+      KnownFPClass Result = computeKnownFPClass(V, fcAllFlags, Depth + 1, Q);
+      FPClassTest FPClasses = Result.KnownFPClasses;
+
+      if (Result.isKnownNever(fcNormal | fcSubnormal | fcNan)) {
+        Known.Zero.setAllBits();
+        Known.One.setAllBits();
+
+        if (FPClasses & fcInf)
+          Known = Known.intersectWith(KnownBits::makeConstant(
+              APFloat::getInf(FPType->getFltSemantics()).bitcastToAPInt()));
+
+        if (FPClasses & fcZero)
+          Known = Known.intersectWith(KnownBits::makeConstant(
+              APInt::getZero(FPType->getScalarSizeInBits())));
+
+        Known.Zero.clearSignBit();
+        Known.One.clearSignBit();
+      }
+
+      if (Result.SignBit) {
+        if (*Result.SignBit)
+          Known.makeNegative();
+        else
+          Known.makeNonNegative();
+      }
+
+      assert(!Known.hasConflict() && "Bits known to be one AND zero?");
+      break;
+    }
+
     // Handle cast from vector integer type to scalar or vector integer.
     auto *SrcVecTy = dyn_cast<FixedVectorType>(SrcTy);
     if (!SrcVecTy || !SrcVecTy->getElementType()->isIntegerTy() ||
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
index c4bd4bc126f7..5db25a59d33f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
@@ -2216,7 +2216,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_sitofp(float %x, i32 %y)
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2304,7 +2304,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_uitofp(float %x, i32 %y)
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2353,7 +2353,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_uitofp_i256(float %x, i2
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2376,7 +2376,7 @@ define float @test_pow_afn_nnan_ninf_f32_known_integral_sitofp_i256(float %x, i2
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
@@ -2399,7 +2399,7 @@ define <2 x float> @test_pow_afn_nnan_ninf_v2f32_known_integral_sitofp(<2 x floa
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x float> [[X]] to <2 x i32>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i32> [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x float> [[__EXP2]] to <2 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i32> [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint <2 x i32> [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP4]] to <2 x float>
 ; CHECK-NEXT:    ret <2 x float> [[TMP5]]
 ;
@@ -2448,7 +2448,7 @@ define <2 x float> @test_pow_afn_nnan_ninf_v2f32_known_integral_uitofp(<2 x floa
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x float> [[X]] to <2 x i32>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i32> [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x float> [[__EXP2]] to <2 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i32> [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint <2 x i32> [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i32> [[TMP4]] to <2 x float>
 ; CHECK-NEXT:    ret <2 x float> [[TMP5]]
 ;
@@ -2560,7 +2560,7 @@ define float @test_pow_afn_f32_nnan_ninf__y_known_integral_trunc(float %x, float
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[__POW_SIGN]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
 ; CHECK-NEXT:    ret float [[TMP5]]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll
index 8ddaf243db92..e298226ee7cc 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pown.ll
@@ -680,7 +680,7 @@ define float @test_pown_afn_nnan_ninf_f32(float %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
@@ -703,7 +703,7 @@ define <2 x float> @test_pown_afn_nnan_ninf_v2f32(<2 x float> %x, <2 x i32> %y)
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[X]] to <2 x i32>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i32> [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x float> [[__EXP2]] to <2 x i32>
-; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i32> [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint <2 x i32> [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
 ; CHECK-NEXT:    ret <2 x float> [[TMP3]]
 ;
@@ -772,7 +772,7 @@ define half @test_pown_afn_nnan_ninf_f16(half %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast half [[X]] to i16
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i16 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half [[__EXP2]] to i16
-; CHECK-NEXT:    [[TMP2:%.*]] = or i16 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i16 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP2]] to half
 ; CHECK-NEXT:    ret half [[TMP3]]
 ;
@@ -795,7 +795,7 @@ define <2 x half> @test_pown_afn_nnan_ninf_v2f16(<2 x half> %x, <2 x i32> %y) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x half> [[X]] to <2 x i16>
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and <2 x i16> [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x half> [[__EXP2]] to <2 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i16> [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint <2 x i16> [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i16> [[TMP2]] to <2 x half>
 ; CHECK-NEXT:    ret <2 x half> [[TMP3]]
 ;
@@ -829,7 +829,7 @@ define float @test_pown_fast_f32_strictfp(float %x, i32 %y) #1 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
@@ -1075,7 +1075,7 @@ define float @test_pown_afn_ninf_nnan_f32__x_known_positive(float nofpclass(ninf
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[__POW_SIGN:%.*]] = and i32 [[__YEVEN]], [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[__EXP2]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[__POW_SIGN]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or disjoint i32 [[__POW_SIGN]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
index 204c8140d3f1..54ca33401ccf 100644
--- a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
+++ b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
@@ -360,7 +360,7 @@ declare half @_Z4pownDhi(half, i32)
 ; GCN-NATIVE: %0 = bitcast half %x to i16
 ; GCN-NATIVE: %__pow_sign = and i16 %__yeven, %0
 ; GCN-NATIVE: %1 = bitcast half %__exp2 to i16
-; GCN-NATIVE: %2 = or i16 %__pow_sign, %1
+; GCN-NATIVE: %2 = or disjoint i16 %__pow_sign, %1
 ; GCN-NATIVE: %3 = bitcast i16 %2 to half
 define half @test_pown_f16(half %x, i32 %y) {
 entry:
@@ -378,7 +378,7 @@ declare float @_Z4pownfi(float, i32)
 ; GCN: %[[r0:.*]] = bitcast float %tmp to i32
 ; GCN: %__pow_sign = and i32 %[[r0]], -2147483648
 ; GCN: %[[r1:.*]] = bitcast float %__exp2 to i32
-; GCN: %[[r2:.*]] = or i32 %__pow_sign, %[[r1]]
+; GCN: %[[r2:.*]] = or disjoint i32 %__pow_sign, %[[r1]]
 ; GCN: store i32 %[[r2]], ptr addrspace(1) %a, align 4
 define amdgpu_kernel void @test_pow(ptr addrspace(1) nocapture %a) {
 entry:
@@ -414,7 +414,7 @@ entry:
 ; GCN: %[[r0:.*]] = bitcast float %tmp to i32
 ; GCN: %__pow_sign = and i32 %__yeven, %[[r0]]
 ; GCN: %[[r1:.*]] = bitcast float %__exp2 to i32
-; GCN: %[[r2:.*]] = or i32 %__pow_sign, %[[r1]]
+; GCN: %[[r2:.*]] = or disjoint i32 %__pow_sign, %[[r1]]
 ; GCN: store i32 %[[r2]], ptr addrspace(1) %a, align 4
 define amdgpu_kernel void @test_pown(ptr addrspace(1) nocapture %a) {
 entry:
@@ -438,7 +438,7 @@ declare <2 x half> @_Z3powDv2_DhS_(<2 x half>, <2 x half>)
 ; GCN: %1 = bitcast half %x to i16
 ; GCN: %__pow_sign = and i16 %1, -32768
 ; GCN: %2 = bitcast half %__exp2 to i16
-; GCN: %3 = or i16 %__pow_sign, %2
+; GCN: %3 = or disjoint i16 %__pow_sign, %2
 ; GCN: %4 = bitcast i16 %3 to half
 define half @test_pow_fast_f16__y_13(half %x) {
   %powr = tail call fast half @_Z3powDhDh(half %x, half 13.0)
@@ -453,7 +453,7 @@ define half @test_pow_fast_f16__y_13(half %x) {
 ; GCN: %1 = bitcast <2 x half> %x to <2 x i16>
 ; GCN: %__pow_sign = and <2 x i16> %1, <i16 -32768, i16 -32768>
 ; GCN: %2 = bitcast <2 x half> %__exp2 to <2 x i16>
-; GCN: %3 = or <2 x i16> %__pow_sign, %2
+; GCN: %3 = or disjoint <2 x i16> %__pow_sign, %2
 ; GCN: %4 = bitcast <2 x i16> %3 to <2 x half>
 define <2 x half> @test_pow_fast_v2f16__y_13(<2 x half> %x) {
   %powr = tail call fast <2 x half> @_Z3powDv2_DhS_(<2 x half> %x, <2 x half> <half 13.0, half 13.0>)
diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index 8b4249b2c25a..7a3802050051 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1374,5 +1374,289 @@ define i8 @nonzero_reduce_xor_vscale_odd(<vscale x 3 x i8> %xx) {
   ret i8 %r
 }
 
+define i1 @test_sign_pos(float %x) {
+; CHECK-LABEL: @test_sign_pos(
+; CHECK-NEXT:    ret i1 true
+;
+  %fabs = call float @llvm.fabs.f32(float %x)
+  %y = bitcast float %fabs to i32
+  %sign = icmp sgt i32 %y, -1
+  ret i1 %sign
+}
+
+define i1 @test_sign_pos_half(half %x) {
+; CHECK-LABEL: @test_sign_pos_half(
+; CHECK-NEXT:    ret i1 true
+;
+  %fabs = call half @llvm.fabs.f16(half %x)
+  %y = bitcast half %fabs to i16
+  %sign = icmp sgt i16 %y, -1
+  ret i1 %sign
+}
+
+define i1 @test_sign_pos_half_non_elementwise(<2 x half> %x) {
+; CHECK-LABEL: @test_sign_pos_half_non_elementwise(
+; CHECK-NEXT:    [[FABS:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[X:%.*]])
+; CHECK-NEXT:    [[Y:%.*]] = bitcast <2 x half> [[FABS]] to i32
+; CHECK-NEXT:    [[SIGN:%.*]] = icmp sgt i32 [[Y]], -1
+; CHECK-NEXT:    ret i1 [[SIGN]]
+;
+  %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %x)
+  %y = bitcast <2 x half> %fabs to i32
+  %sign = icmp sgt i32 %y, -1
+  ret i1 %sign
+}
+
+define i1 @test_sign_neg(float %x) {
+; CHECK-LABEL: @test_sign_neg(
+; CHECK-NEXT:    ret i1 true
+;
+  %fabs = call float @llvm.fabs.f32(float %x)
+  %fnabs = fneg float %fabs
+  %y = bitcast float %fnabs to i32
+  %sign = icmp slt i32 %y, 0
+  ret i1 %sign
+}
+
+define <2 x i1> @test_sign_pos_vec(<2 x float> %x) {
+; CHECK-LABEL: @test_sign_pos_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %x)
+  %y = bitcast <2 x float> %fabs to <2 x i32>
+  %sign = icmp slt <2 x i32> %y, zeroinitializer
+  ret <2 x i1> %sign
+}
+
+define i32 @test_inf_only(float nofpclass(nan sub norm zero) %x) {
+; CHECK-LABEL: @test_inf_only(
+; CHECK-NEXT:    ret i32 2139095040
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 2147483647
+  ret i32 %and
+}
+
+define i16 @test_inf_only_bfloat(bfloat nofpclass(nan sub norm zero) %x) {
+; CHECK-LABEL: @test_inf_only_bfloat(
+; CHECK-NEXT:    ret i16 32640
+;
+  %y = bitcast bfloat %x to i16
+  %and = and i16 %y, 32767
+  ret i16 %and
+}
+
+define i128 @test_inf_only_ppc_fp128(ppc_fp128 nofpclass(nan sub norm zero) %x) {
+; CHECK-LABEL: @test_inf_only_ppc_fp128(
+; CHECK-NEXT:    ret i128 9218868437227405312
+;
+  %y = bitcast ppc_fp128 %x to i128
+  %and = and i128 %y, 170141183460469231731687303715884105727
+  ret i128 %and
+}
+
+define i32 @test_zero_only(float nofpclass(nan sub norm inf) %x) {
+; CHECK-LABEL: @test_zero_only(
+; CHECK-NEXT:    ret i32 0
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 2147483647
+  ret i32 %and
+}
+
+define i80 @test_zero_only_non_ieee(x86_fp80 nofpclass(nan sub norm inf) %x) {
+; CHECK-LABEL: @test_zero_only_non_ieee(
+; CHECK-NEXT:    ret i80 0
+;
+  %y = bitcast x86_fp80 %x to i80
+  %and = and i80 %y, 604462909807314587353087
+  ret i80 %and
+}
+
+define i32 @test_inf_nan_only(float nofpclass(sub norm zero) %x) {
+; CHECK-LABEL: @test_inf_nan_only(
+; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 2130706432
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 2130706432
+  ret i32 %and
+}
+
+define i32 @test_sub_zero_only(float nofpclass(nan norm inf) %x) {
+; CHECK-LABEL: @test_sub_zero_only(
+; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 2130706432
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 2130706432
+  ret i32 %and
+}
+
+define i32 @test_inf_zero_only(float nofpclass(nan norm sub) %x) {
+; CHECK-LABEL: @test_inf_zero_only(
+; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 8388608
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 16777215
+  ret i32 %and
+}
+
+; Make sure that the signbit is cleared.
+define i32 @test_ninf_only(double %x) {
+; CHECK-LABEL: @test_ninf_only(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[X:%.*]], 0xFFF0000000000000
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i32 0
+;
+  %cmp = fcmp oeq double %x, 0xFFF0000000000000
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %cast = bitcast double %x to i64
+  %trunc = trunc i64 %cast to i32
+  ret i32 %trunc
+
+if.else:
+  ret i32 0
+}
+
+define i1 @test_simplify_icmp(i32 %x) {
+; CHECK-LABEL: @test_simplify_icmp(
+; CHECK-NEXT:    ret i1 false
+;
+  %cast1 = uitofp i32 %x to double
+  %cast2 = bitcast double %cast1 to i64
+  %mask = and i64 %cast2, -140737488355328
+  %cmp = icmp eq i64 %mask, -1970324836974592
+  ret i1 %cmp
+}
+
+define i32 @test_snan_quiet_bit1(float nofpclass(sub norm inf qnan) %x) {
+; CHECK-LABEL: @test_snan_quiet_bit1(
+; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 4194304
+; CHECK-NEXT:    ret i32 [[MASKED]]
+;
+  %bits = bitcast float %x to i32
+  %masked = and i32 %bits, 4194304
+  ret i32 %masked
+}
+
+define i32 @test_snan_quiet_bit2(float nofpclass(sub norm inf qnan) %x) {
+; CHECK-LABEL: @test_snan_quiet_bit2(
+; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 2097152
+; CHECK-NEXT:    ret i32 [[MASKED]]
+;
+  %bits = bitcast float %x to i32
+  %masked = and i32 %bits, 2097152
+  ret i32 %masked
+}
+
+define i32 @test_qnan_quiet_bit1(float nofpclass(sub norm inf snan) %x) {
+; CHECK-LABEL: @test_qnan_quiet_bit1(
+; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 4194304
+; CHECK-NEXT:    ret i32 [[MASKED]]
+;
+  %bits = bitcast float %x to i32
+  %masked = and i32 %bits, 4194304
+  ret i32 %masked
+}
+
+define i32 @test_qnan_quiet_bit2(float nofpclass(sub norm inf snan) %x) {
+; CHECK-LABEL: @test_qnan_quiet_bit2(
+; CHECK-NEXT:    [[BITS:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[BITS]], 2097152
+; CHECK-NEXT:    ret i32 [[MASKED]]
+;
+  %bits = bitcast float %x to i32
+  %masked = and i32 %bits, 2097152
+  ret i32 %masked
+}
+
+define i16 @test_simplify_mask(i32 %ui, float %x) {
+; CHECK-LABEL: @test_simplify_mask(
+; CHECK-NEXT:    [[CONV:%.*]] = uitofp i32 [[UI:%.*]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[CONV]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_ELSE:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i16 31744
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i16 0
+;
+  %conv = uitofp i32 %ui to float
+  %cmp = fcmp olt float %x, %conv
+  br i1 %cmp, label %if.else, label %if.end
+
+if.end:
+  %cast = bitcast float %conv to i32
+  %shr = lshr i32 %cast, 16
+  %trunc = trunc i32 %shr to i16
+  %and = and i16 %trunc, -32768
+  %or = or disjoint i16 %and, 31744
+  ret i16 %or
+
+if.else:
+  ret i16 0
+}
+
+; TODO: %cmp always evaluates to false
+
+define i1 @test_simplify_icmp2(double %x) {
+; CHECK-LABEL: @test_simplify_icmp2(
+; CHECK-NEXT:    [[ABS:%.*]] = tail call double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[COND:%.*]] = fcmp oeq double [[ABS]], 0x7FF0000000000000
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast double [[X]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[CAST]], 3458764513820540928
+; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i1 false
+;
+  %abs = tail call double @llvm.fabs.f64(double %x)
+  %cond = fcmp oeq double %abs, 0x7FF0000000000000
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:
+  %cast = bitcast double %x to i64
+  %cmp = icmp eq i64 %cast, 3458764513820540928
+  ret i1 %cmp
+
+if.else:
+  ret i1 false
+}
+
+define i32 @test_snan_only(float nofpclass(qnan sub norm zero inf) %x) {
+; CHECK-LABEL: @test_snan_only(
+; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 4194304
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 4194304
+  ret i32 %and
+}
+
+define i32 @test_qnan_only(float nofpclass(snan sub norm zero inf) %x) {
+; CHECK-LABEL: @test_qnan_only(
+; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 4194304
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 4194304
+  ret i32 %and
+}
+
 declare void @use(i1)
 declare void @sink(i8)
-- 
GitLab


From 58b9564d5d12063bb9c662039802ede8df615374 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Tue, 14 May 2024 10:47:12 +0200
Subject: [PATCH 0712/1206] [clang][Interp][NFC] Add some assertions

Make sure we pass a non-null Descriptor when creating a new Block.
---
 clang/lib/AST/Interp/InterpBlock.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/Interp/InterpBlock.h b/clang/lib/AST/Interp/InterpBlock.h
index 6d5856fbd4ea..506034e880d0 100644
--- a/clang/lib/AST/Interp/InterpBlock.h
+++ b/clang/lib/AST/Interp/InterpBlock.h
@@ -51,11 +51,15 @@ public:
   /// Creates a new block.
   Block(const std::optional<unsigned> &DeclID, const Descriptor *Desc,
         bool IsStatic = false, bool IsExtern = false)
-      : DeclID(DeclID), IsStatic(IsStatic), IsExtern(IsExtern), Desc(Desc) {}
+      : DeclID(DeclID), IsStatic(IsStatic), IsExtern(IsExtern), Desc(Desc) {
+        assert(Desc);
+      }
 
   Block(const Descriptor *Desc, bool IsStatic = false, bool IsExtern = false)
       : DeclID((unsigned)-1), IsStatic(IsStatic), IsExtern(IsExtern),
-        Desc(Desc) {}
+        Desc(Desc) {
+          assert(Desc);
+        }
 
   /// Returns the block's descriptor.
   const Descriptor *getDescriptor() const { return Desc; }
-- 
GitLab


From 0aa5fa9630d0f4ea707c5b8d5cfa2f4bc8d06a14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Tue, 14 May 2024 10:47:57 +0200
Subject: [PATCH 0713/1206] [clang][Interp][NFC] Improve Pointer::print()

---
 clang/lib/AST/Interp/Pointer.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp
index 12bef73f7e21..d2e34f2c7f09 100644
--- a/clang/lib/AST/Interp/Pointer.cpp
+++ b/clang/lib/AST/Interp/Pointer.cpp
@@ -181,12 +181,12 @@ void Pointer::print(llvm::raw_ostream &OS) const {
   if (isBlockPointer()) {
     OS << "Block) {";
 
-    if (PointeeStorage.BS.Base == RootPtrMark)
-      OS << "rootptr, ";
+    if (isRoot())
+      OS << "rootptr(" << PointeeStorage.BS.Base << "), ";
     else
       OS << PointeeStorage.BS.Base << ", ";
 
-    if (Offset == PastEndMark)
+    if (isElementPastEnd())
       OS << "pastend, ";
     else
       OS << Offset << ", ";
-- 
GitLab


From 5865482049872d3ae52ea5559abb9e8f4a1e55e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Tue, 14 May 2024 10:48:26 +0200
Subject: [PATCH 0714/1206] [clang][Interp][NFC] Don't pass on metadata size
 for composite arrays

We don't need the metadata size for every element, just for the topmost
descriptor.
---
 clang/lib/AST/Interp/Program.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/AST/Interp/Program.cpp b/clang/lib/AST/Interp/Program.cpp
index 6606149f1f69..0b95db849269 100644
--- a/clang/lib/AST/Interp/Program.cpp
+++ b/clang/lib/AST/Interp/Program.cpp
@@ -372,7 +372,7 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
         // Arrays of composites. In this case, the array is a list of pointers,
         // followed by the actual elements.
         const Descriptor *ElemDesc = createDescriptor(
-            D, ElemTy.getTypePtr(), MDSize, IsConst, IsTemporary);
+            D, ElemTy.getTypePtr(), std::nullopt, IsConst, IsTemporary);
         if (!ElemDesc)
           return nullptr;
         unsigned ElemSize =
-- 
GitLab


From c1bd68867497cf6e2f2afdba1a3a2993a47b5856 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Tue, 14 May 2024 11:17:49 +0200
Subject: [PATCH 0715/1206] [clang][Interp] Fix some dummy-related FIXME
 comments

---
 clang/lib/AST/Interp/Interp.h    |  9 +++++----
 clang/lib/AST/Interp/Pointer.h   |  7 -------
 clang/lib/AST/Interp/Program.cpp |  6 +++++-
 clang/test/AST/Interp/arrays.cpp | 15 +++++++++++++++
 4 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index a0bf87430012..d9f23a4b8c96 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -1569,9 +1569,7 @@ bool OffsetHelper(InterpState &S, CodePtr OpPC, const T &Offset,
     APSInt NewIndex =
         (Op == ArithOp::Add) ? (APIndex + APOffset) : (APIndex - APOffset);
     S.CCEDiag(S.Current->getSource(OpPC), diag::note_constexpr_array_index)
-        << NewIndex
-        << /*array*/ static_cast<int>(!Ptr.inArray())
-        << static_cast<unsigned>(MaxIndex);
+        << NewIndex << /*array*/ static_cast<int>(!Ptr.inArray()) << MaxIndex;
     Invalid = true;
   };
 
@@ -1598,7 +1596,7 @@ bool OffsetHelper(InterpState &S, CodePtr OpPC, const T &Offset,
     }
   }
 
-  if (Invalid && !Ptr.isDummy() && S.getLangOpts().CPlusPlus)
+  if (Invalid && S.getLangOpts().CPlusPlus)
     return false;
 
   // Offset is valid - compute it on unsigned.
@@ -2110,6 +2108,9 @@ inline bool ArrayDecay(InterpState &S, CodePtr OpPC) {
     return true;
   }
 
+  if (!CheckRange(S, OpPC, Ptr, CSK_ArrayToPointer))
+    return false;
+
   if (!Ptr.isUnknownSizeArray() || Ptr.isDummy()) {
     S.Stk.push<Pointer>(Ptr.atIndex(0));
     return true;
diff --git a/clang/lib/AST/Interp/Pointer.h b/clang/lib/AST/Interp/Pointer.h
index 79fab05670e9..9900f37e60d4 100644
--- a/clang/lib/AST/Interp/Pointer.h
+++ b/clang/lib/AST/Interp/Pointer.h
@@ -384,11 +384,6 @@ public:
   bool isUnknownSizeArray() const {
     if (!isBlockPointer())
       return false;
-    // If this points inside a dummy block, return true.
-    // FIXME: This might change in the future. If it does, we need
-    // to set the proper Ctor/Dtor functions for dummy Descriptors.
-    if (!isRoot() && isDummy())
-      return true;
     return getFieldDesc()->isUnknownSizeArray();
   }
   /// Checks if the pointer points to an array.
@@ -560,8 +555,6 @@ public:
 
     if (!asBlockPointer().Pointee)
       return false;
-    if (isDummy())
-      return false;
 
     return isElementPastEnd() || getSize() == getOffset();
   }
diff --git a/clang/lib/AST/Interp/Program.cpp b/clang/lib/AST/Interp/Program.cpp
index 0b95db849269..31a64e13d2b1 100644
--- a/clang/lib/AST/Interp/Program.cpp
+++ b/clang/lib/AST/Interp/Program.cpp
@@ -144,8 +144,12 @@ std::optional<unsigned> Program::getOrCreateDummy(const ValueDecl *VD) {
   if (auto It = DummyVariables.find(VD); It != DummyVariables.end())
     return It->second;
 
+  QualType QT = VD->getType();
+  if (const auto *RT = QT->getAs<ReferenceType>())
+    QT = RT->getPointeeType();
+
   Descriptor *Desc;
-  if (std::optional<PrimType> T = Ctx.classify(VD->getType()))
+  if (std::optional<PrimType> T = Ctx.classify(QT))
     Desc = createDescriptor(VD, *T, std::nullopt, true, false);
   else
     Desc = createDescriptor(VD, VD->getType().getTypePtr(), std::nullopt, true,
diff --git a/clang/test/AST/Interp/arrays.cpp b/clang/test/AST/Interp/arrays.cpp
index f6d265d4b3d1..929f25b95fa1 100644
--- a/clang/test/AST/Interp/arrays.cpp
+++ b/clang/test/AST/Interp/arrays.cpp
@@ -580,3 +580,18 @@ constexpr ptrdiff_t d3 = &melchizedek[0] - &melchizedek[1]; // ok
 /// GH#88018
 const int SZA[] = {};
 void testZeroSizedArrayAccess() { unsigned c = SZA[4]; }
+
+#if __cplusplus >= 202002L
+constexpr int test_multiarray2() { // both-error {{never produces a constant expression}}
+  int multi2[2][1]; // both-note {{declared here}}
+  return multi2[2][0]; // both-note {{cannot access array element of pointer past the end of object}} \
+                       // both-warning {{array index 2 is past the end of the array (that has type 'int[2][1]')}}
+}
+
+/// Same but with a dummy pointer.
+int multi22[2][2]; // both-note {{declared here}}
+int test_multiarray22() {
+  return multi22[2][0]; // both-warning {{array index 2 is past the end of the array (that has type 'int[2][2]')}}
+}
+
+#endif
-- 
GitLab


From 31fb0ae23d3d1a1b90198a68c80c9116d844a01f Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 14 May 2024 11:54:38 +0100
Subject: [PATCH 0716/1206] [PowerPC] Regenerate and_sext.ll with test checks

I've kept the grep checks for extsh/extsb instructions, but we can now see the actual codegen as well
---
 llvm/test/CodeGen/PowerPC/and_sext.ll | 45 ++++++++++++++++++---------
 1 file changed, 30 insertions(+), 15 deletions(-)

diff --git a/llvm/test/CodeGen/PowerPC/and_sext.ll b/llvm/test/CodeGen/PowerPC/and_sext.ll
index 3b576ca18ee7..b67b86bd5132 100644
--- a/llvm/test/CodeGen/PowerPC/and_sext.ll
+++ b/llvm/test/CodeGen/PowerPC/and_sext.ll
@@ -1,28 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; These tests should not contain a sign extend.
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- | FileCheck %s
 ; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- | not grep extsh
 ; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- | not grep extsb
 
 define i32 @test1(i32 %mode.0.i.0) {
-        %tmp.79 = trunc i32 %mode.0.i.0 to i16
-        %tmp.80 = sext i16 %tmp.79 to i32
-        %tmp.81 = and i32 %tmp.80, 24
-        ret i32 %tmp.81
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    rlwinm 3, 3, 0, 27, 28
+; CHECK-NEXT:    blr
+  %tmp.79 = trunc i32 %mode.0.i.0 to i16
+  %tmp.80 = sext i16 %tmp.79 to i32
+  %tmp.81 = and i32 %tmp.80, 24
+  ret i32 %tmp.81
 }
 
 define signext i16 @test2(i16 signext %X, i16 signext %x)  {
-        %tmp = sext i16 %X to i32
-        %tmp1 = sext i16 %x to i32
-        %tmp2 = add i32 %tmp, %tmp1
-        %tmp4 = ashr i32 %tmp2, 1
-        %tmp5 = trunc i32 %tmp4 to i16
-        %tmp45 = sext i16 %tmp5 to i32
-        %retval = trunc i32 %tmp45 to i16
-        ret i16 %retval
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    srawi 3, 3, 1
+; CHECK-NEXT:    blr
+  %tmp = sext i16 %X to i32
+  %tmp1 = sext i16 %x to i32
+  %tmp2 = add i32 %tmp, %tmp1
+  %tmp4 = ashr i32 %tmp2, 1
+  %tmp5 = trunc i32 %tmp4 to i16
+  %tmp45 = sext i16 %tmp5 to i32
+  %retval = trunc i32 %tmp45 to i16
+  ret i16 %retval
 }
 
 define signext i16 @test3(i32 zeroext %X)  {
-        %tmp1 = lshr i32 %X, 16
-        %tmp2 = trunc i32 %tmp1 to i16
-        ret i16 %tmp2
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srawi 3, 3, 16
+; CHECK-NEXT:    blr
+  %tmp1 = lshr i32 %X, 16
+  %tmp2 = trunc i32 %tmp1 to i16
+  ret i16 %tmp2
 }
 
-- 
GitLab


From c34d1893cb8b485e6871512ef4e743bfa2d462f8 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 14 May 2024 06:00:23 -0500
Subject: [PATCH 0717/1206] [Offload] Remove support for old "BUILD_PLUGIN"
 options. (#91644)

Summary:
Since the move to the statically linked plugins, we added a new way to
directly control which plugins will be added. Delete these old ones as
they will cause the build to fail and suggest the new format.
---
 offload/CMakeLists.txt                        | 6 ++++++
 offload/plugins-nextgen/amdgpu/CMakeLists.txt | 7 -------
 offload/plugins-nextgen/cuda/CMakeLists.txt   | 7 -------
 3 files changed, 6 insertions(+), 14 deletions(-)

diff --git a/offload/CMakeLists.txt b/offload/CMakeLists.txt
index 3f77583ffa3b..626df8125063 100644
--- a/offload/CMakeLists.txt
+++ b/offload/CMakeLists.txt
@@ -151,6 +151,11 @@ if (NOT LIBOMPTARGET_LLVM_INCLUDE_DIRS)
   message(FATAL_ERROR "Missing definition for LIBOMPTARGET_LLVM_INCLUDE_DIRS")
 endif()
 
+if(DEFINED LIBOMPTARGET_BUILD_CUDA_PLUGIN OR 
+   DEFINED LIBOMPTARGET_BUILD_AMDGPU_PLUGIN)
+  message(WARNING "Option removed, use 'LIBOMPTARGET_PLUGINS_TO_BUILD' instead")
+endif()
+
 set(LIBOMPTARGET_ALL_PLUGIN_TARGETS amdgpu cuda host)
 set(LIBOMPTARGET_PLUGINS_TO_BUILD "all" CACHE STRING
     "Semicolon-separated list of plugins to use: cuda, amdgpu, host or \"all\".")
@@ -158,6 +163,7 @@ set(LIBOMPTARGET_PLUGINS_TO_BUILD "all" CACHE STRING
 if(LIBOMPTARGET_PLUGINS_TO_BUILD STREQUAL "all")
   set(LIBOMPTARGET_PLUGINS_TO_BUILD ${LIBOMPTARGET_ALL_PLUGIN_TARGETS})
 endif()
+message("Building with support for ${LIBOMPTARGET_PLUGINS_TO_BUILD} plugins")
 
 set(LIBOMPTARGET_ENUM_PLUGIN_TARGETS "")
 foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD)
diff --git a/offload/plugins-nextgen/amdgpu/CMakeLists.txt b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
index 738183f8945e..280cb57e22e9 100644
--- a/offload/plugins-nextgen/amdgpu/CMakeLists.txt
+++ b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
@@ -12,13 +12,6 @@
 ##===----------------------------------------------------------------------===##
 
 ################################################################################
-set(LIBOMPTARGET_BUILD_AMDGPU_PLUGIN TRUE CACHE BOOL
-  "Whether to build AMDGPU plugin")
-if (NOT LIBOMPTARGET_BUILD_AMDGPU_PLUGIN)
-  libomptarget_say("Not building AMDGPU NextGen offloading plugin: LIBOMPTARGET_BUILD_AMDGPU_PLUGIN is false")
-  return()
-endif()
-
 # as of rocm-3.7, hsa is installed with cmake packages and kmt is found via hsa
 find_package(hsa-runtime64 QUIET 1.2.0 HINTS ${CMAKE_INSTALL_PREFIX} PATHS /opt/rocm)
 
diff --git a/offload/plugins-nextgen/cuda/CMakeLists.txt b/offload/plugins-nextgen/cuda/CMakeLists.txt
index dd684bb22343..3f3388e56c5f 100644
--- a/offload/plugins-nextgen/cuda/CMakeLists.txt
+++ b/offload/plugins-nextgen/cuda/CMakeLists.txt
@@ -9,13 +9,6 @@
 # Build a plugin for a CUDA machine if available.
 #
 ##===----------------------------------------------------------------------===##
-set(LIBOMPTARGET_BUILD_CUDA_PLUGIN TRUE CACHE BOOL
-  "Whether to build CUDA plugin")
-if (NOT LIBOMPTARGET_BUILD_CUDA_PLUGIN)
-  libomptarget_say("Not building CUDA NextGen offloading plugin: LIBOMPTARGET_BUILD_CUDA_PLUGIN is false")
-  return()
-endif()
-
 if (NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)|(aarch64)$" AND CMAKE_SYSTEM_NAME MATCHES "Linux"))
   libomptarget_say("Not building CUDA NextGen offloading plugin: only support CUDA in Linux x86_64, ppc64le, or aarch64 hosts.")
   return()
-- 
GitLab


From 363258a3ccbb752ec23f681d19b6a874c4db99ab Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 14 May 2024 06:00:34 -0500
Subject: [PATCH 0718/1206] [Offload] Remove old references to `isCtor`
 (#91766)

Summary:
These have long since been removed, support for ctors / dtors now
happens through special kernels the backend creates.
---
 offload/plugins-nextgen/common/include/PluginInterface.h | 6 ------
 offload/plugins-nextgen/common/src/PluginInterface.cpp   | 5 +----
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index e7a008f3a857..c396099ac625 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -270,12 +270,6 @@ struct GenericKernelTy {
   /// Get the kernel name.
   const char *getName() const { return Name; }
 
-  /// Return true if this kernel is a constructor or destructor.
-  bool isCtorOrDtor() const {
-    // TODO: This is not a great solution and should be revisited.
-    return StringRef(Name).ends_with("tor");
-  }
-
   /// Get the kernel image.
   DeviceImageTy &getImage() const {
     assert(ImagePtr && "Kernel is not initialized!");
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index fae197527850..a5c8cce63fac 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -470,7 +470,7 @@ GenericKernelTy::getKernelLaunchEnvironment(
   // Ctor/Dtor have no arguments, replaying uses the original kernel launch
   // environment. Older versions of the compiler do not generate a kernel
   // launch environment.
-  if (isCtorOrDtor() || RecordReplay.isReplaying() ||
+  if (RecordReplay.isReplaying() ||
       Version < OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR)
     return nullptr;
 
@@ -579,9 +579,6 @@ void *GenericKernelTy::prepareArgs(
     uint32_t &NumArgs, llvm::SmallVectorImpl<void *> &Args,
     llvm::SmallVectorImpl<void *> &Ptrs,
     KernelLaunchEnvironmentTy *KernelLaunchEnvironment) const {
-  if (isCtorOrDtor())
-    return nullptr;
-
   uint32_t KLEOffset = !!KernelLaunchEnvironment;
   NumArgs += KLEOffset;
 
-- 
GitLab


From cbd72cb0deec31a5c3063cf1f1af759761115eee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Tue, 14 May 2024 12:08:57 +0100
Subject: [PATCH 0719/1206] [mlir][vector] Split `TransposeOpLowering` into 2
 patterns (#91935)

Splits `TransposeOpLowering` into two patterns:
1. `Transpose2DWithUnitDimToShapeCast` - rewrites 2D `vector.transpose`
    as `vector.shape_cast` (there has to be at least one unit dim),
  2. `TransposeOpLowering` - the original pattern without the part
    extracted into `Transpose2DWithUnitDimToShapeCast`.

The rationale behind the split:
  * the output generated by `Transpose2DWithUnitDimToShapeCast` doesn't
    really match the intended output from `TransposeOpLowering` as
    documented in the source file - it doesn't make much sense to keep
    it embedded inside `TransposeOpLowering`,
* `Transpose2DWithUnitDimToShapeCast` _does_ work for scalable vectors,
    `TransposeOpLowering` _does_ not.
---
 .../Transforms/LowerVectorTranspose.cpp       | 86 ++++++++++++++-----
 1 file changed, 64 insertions(+), 22 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
index 7011c478fefb..ca8a6f6d82a6 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
@@ -326,6 +326,10 @@ public:
     VectorType inputType = op.getSourceVectorType();
     VectorType resType = op.getResultVectorType();
 
+    if (inputType.isScalable())
+      return rewriter.notifyMatchFailure(
+          op, "This lowering does not support scalable vectors");
+
     // Set up convenience transposition table.
     ArrayRef<int64_t> transp = op.getPermutation();
 
@@ -334,28 +338,6 @@ public:
       return rewriter.notifyMatchFailure(
           op, "Options specifies lowering to shuffle");
 
-    // Replace:
-    //   vector.transpose %0, [1, 0] : vector<nx1x<eltty>> to
-    //                                 vector<1xnxelty>
-    // with:
-    //   vector.shape_cast %0 : vector<nx1x<eltty>> to vector<1xnxelty>
-    //
-    // Source with leading unit dim (inverse) is also replaced. Unit dim must
-    // be fixed. Non-unit can be scalable.
-    if (resType.getRank() == 2 &&
-        ((resType.getShape().front() == 1 &&
-          !resType.getScalableDims().front()) ||
-         (resType.getShape().back() == 1 &&
-          !resType.getScalableDims().back())) &&
-        transp == ArrayRef<int64_t>({1, 0})) {
-      rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resType, input);
-      return success();
-    }
-
-    // TODO: Add support for scalable vectors
-    if (inputType.isScalable())
-      return failure();
-
     // Handle a true 2-D matrix transpose differently when requested.
     if (vectorTransformOptions.vectorTransposeLowering ==
             vector::VectorTransposeLowering::Flat &&
@@ -411,6 +393,64 @@ private:
   vector::VectorTransformsOptions vectorTransformOptions;
 };
 
+/// Rewrites vector.transpose as vector.shape_cast. This pattern is only applied
+/// to 2D vectors with at least one unit dim. For example:
+///
+/// Replace:
+///   vector.transpose %0, [1, 0] : vector<4x1xi32>> to
+///                                 vector<1x4xi32>
+/// with:
+///   vector.shape_cast %0 : vector<4x1xi32> to vector<1x4xi32>
+///
+/// Source with leading unit dim (inverse) is also replaced. Unit dim must
+/// be fixed. Non-unit dim can be scalable.
+///
+/// TODO: This pattern was introduced specifically to help lower scalable
+/// vectors. In hindsight, a more specialised canonicalization (for shape_cast's
+/// to cancel out) would be preferable:
+///
+///  BEFORE:
+///     %0 = some_op
+///     %1 = vector.shape_cast %0 : vector<[4]xf32> to vector<[4]x1xf32>
+///     %2 = vector.transpose %1 [1, 0] : vector<[4]x1xf32> to vector<1x[4]xf32>
+///  AFTER:
+///     %0 = some_op
+///     %1 = vector.shape_cast %0 : vector<[4]xf32> to vector<1x[4]xf32>
+///
+/// Given the context above, we may want to consider (re-)moving this pattern
+/// at some later time. I am leaving it for now in case there are other users
+/// that I am not aware of.
+class Transpose2DWithUnitDimToShapeCast
+    : public OpRewritePattern<vector::TransposeOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+
+  Transpose2DWithUnitDimToShapeCast(MLIRContext *context,
+                                    PatternBenefit benefit = 1)
+      : OpRewritePattern<vector::TransposeOp>(context, benefit) {}
+
+  LogicalResult matchAndRewrite(vector::TransposeOp op,
+                                PatternRewriter &rewriter) const override {
+    Value input = op.getVector();
+    VectorType resType = op.getResultVectorType();
+
+    // Set up convenience transposition table.
+    ArrayRef<int64_t> transp = op.getPermutation();
+
+    if (resType.getRank() == 2 &&
+        ((resType.getShape().front() == 1 &&
+          !resType.getScalableDims().front()) ||
+         (resType.getShape().back() == 1 &&
+          !resType.getScalableDims().back())) &&
+        transp == ArrayRef<int64_t>({1, 0})) {
+      rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, resType, input);
+      return success();
+    }
+
+    return failure();
+  }
+};
+
 /// Rewrite a 2-D vector.transpose as a sequence of shuffle ops.
 /// If the strategy is Shuffle1D, it will be lowered to:
 ///   vector.shape_cast 2D -> 1D
@@ -483,6 +523,8 @@ private:
 void mlir::vector::populateVectorTransposeLoweringPatterns(
     RewritePatternSet &patterns, VectorTransformsOptions options,
     PatternBenefit benefit) {
+  patterns.add<Transpose2DWithUnitDimToShapeCast>(patterns.getContext(),
+                                                  benefit);
   patterns.add<TransposeOpLowering, TransposeOp2DToShuffleLowering>(
       options, patterns.getContext(), benefit);
 }
-- 
GitLab


From e6d3a4212d20b49a8e63f11fedea79cccf261479 Mon Sep 17 00:00:00 2001
From: aengelke <engelke@in.tum.de>
Date: Tue, 14 May 2024 13:13:24 +0200
Subject: [PATCH 0720/1206] [CodeGen] Use SmallVector for FixedStackPSVs
 (#91760)

Frame indices are dense and consecutive, so use a vector instead of a
std::map. Due to possibly negative frame indices, use zig-zag encoding.
IndexedMap was not usable, as it attempted to copy the null value, which
is not possible with a std::unique_ptr.

This is just a minor performance improvement, but a low-hanging fruit.
---
 llvm/include/llvm/CodeGen/PseudoSourceValueManager.h | 4 ++--
 llvm/lib/CodeGen/PseudoSourceValue.cpp               | 7 ++++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/PseudoSourceValueManager.h b/llvm/include/llvm/CodeGen/PseudoSourceValueManager.h
index 4be6ae0b60cb..8ea043bf0327 100644
--- a/llvm/include/llvm/CodeGen/PseudoSourceValueManager.h
+++ b/llvm/include/llvm/CodeGen/PseudoSourceValueManager.h
@@ -13,10 +13,10 @@
 #ifndef LLVM_CODEGEN_PSEUDOSOURCEVALUEMANAGER_H
 #define LLVM_CODEGEN_PSEUDOSOURCEVALUEMANAGER_H
 
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/IR/ValueMap.h"
-#include <map>
 
 namespace llvm {
 
@@ -27,7 +27,7 @@ class TargetMachine;
 class PseudoSourceValueManager {
   const TargetMachine &TM;
   const PseudoSourceValue StackPSV, GOTPSV, JumpTablePSV, ConstantPoolPSV;
-  std::map<int, std::unique_ptr<FixedStackPseudoSourceValue>> FSValues;
+  SmallVector<std::unique_ptr<FixedStackPseudoSourceValue>> FSValues;
   StringMap<std::unique_ptr<const ExternalSymbolPseudoSourceValue>>
       ExternalCallEntries;
   ValueMap<const GlobalValue *,
diff --git a/llvm/lib/CodeGen/PseudoSourceValue.cpp b/llvm/lib/CodeGen/PseudoSourceValue.cpp
index 0e1a2c921c5c..242889d31784 100644
--- a/llvm/lib/CodeGen/PseudoSourceValue.cpp
+++ b/llvm/lib/CodeGen/PseudoSourceValue.cpp
@@ -122,7 +122,12 @@ const PseudoSourceValue *PseudoSourceValueManager::getJumpTable() {
 
 const PseudoSourceValue *
 PseudoSourceValueManager::getFixedStack(int FI) {
-  std::unique_ptr<FixedStackPseudoSourceValue> &V = FSValues[FI];
+  // Frame index is often continuously positive, but can be negative. Use
+  // zig-zag encoding for dense index into FSValues vector.
+  unsigned Idx = (2 * unsigned(FI)) ^ (FI >> (sizeof(FI) * 8 - 1));
+  if (FSValues.size() <= Idx)
+    FSValues.resize(Idx + 1);
+  std::unique_ptr<FixedStackPseudoSourceValue> &V = FSValues[Idx];
   if (!V)
     V = std::make_unique<FixedStackPseudoSourceValue>(FI, TM);
   return V.get();
-- 
GitLab


From c7c5666aac543a49b485a133f4a94865e2613a43 Mon Sep 17 00:00:00 2001
From: jeanPerier <jperier@nvidia.com>
Date: Tue, 14 May 2024 13:34:46 +0200
Subject: [PATCH 0721/1206] [flang] Do not hoist all scalar sub-expressions
 from WHERE constructs (#91395)

The HLFIR pass lowering WHERE (hlfir.where op) was too aggressive in its
hoisting of scalar sub-expressions from LHS/RHS/MASKS outside of the
loops generated for the WHERE construct.
This violated F'2023 10.2.3.2 point 10 that stipulated that elemental
operations must be evaluated only for elements corresponding to true
values, because scalar operations are still elemental, and hoisting them
is invalid if they could have side effects (e.g, division by zero) and
if the MASK is always false (i.e., the loop body is never evaluated).

The difficulty is that 10.2.3.2 point 9 mandates that nonelemental
function must be evaluated before the loops. So it is not possible to
simply stop hoisting non hlfir.elemental operations.
Marking calls with an elemental/nonelemental attribute would not allow
the pass to be correct if inlining is run before and drops this
information, beside, extracting the argument tree that may have been
CSE-ed with the rest of the expression evaluation would be a bit
combursome.

Instead, lower nonelemental calls into a new hlfir.exactly_once
operation that will allow retaining the information that the operations
contained inside its region must be hoisted. This allows inlining to
operate before if desired in order to improve alias analysis.

The LowerHLFIROrderedAssignments pass is updated to only hoist the
operations contained inside hlfir.exactly_once bodies.
---
 flang/include/flang/Lower/StatementContext.h  |  14 ++
 .../include/flang/Optimizer/HLFIR/HLFIROps.td |  24 ++-
 flang/lib/Lower/Bridge.cpp                    |  45 ++--
 flang/lib/Lower/ConvertCall.cpp               |  38 ++++
 .../LowerHLFIROrderedAssignments.cpp          | 130 ++++++++++--
 .../HLFIR/order_assignments/impure-where.fir  |   9 +-
 .../order_assignments/inlined-stack-temp.fir  |   2 +-
 .../user-defined-assignment-finalization.fir  |  31 +--
 .../HLFIR/order_assignments/where-cleanup.f90 |  44 ++++
 .../where-codegen-no-conflict.fir             |   4 +-
 .../order_assignments/where-hoisting.f90      |  50 +++++
 flang/test/Lower/HLFIR/where-nonelemental.f90 | 198 ++++++++++++++++++
 12 files changed, 525 insertions(+), 64 deletions(-)
 create mode 100644 flang/test/HLFIR/order_assignments/where-cleanup.f90
 create mode 100644 flang/test/HLFIR/order_assignments/where-hoisting.f90
 create mode 100644 flang/test/Lower/HLFIR/where-nonelemental.f90

diff --git a/flang/include/flang/Lower/StatementContext.h b/flang/include/flang/Lower/StatementContext.h
index cec9641d43a0..7776edc93ed7 100644
--- a/flang/include/flang/Lower/StatementContext.h
+++ b/flang/include/flang/Lower/StatementContext.h
@@ -18,6 +18,15 @@
 #include <functional>
 #include <optional>
 
+namespace mlir {
+class Location;
+class Region;
+} // namespace mlir
+
+namespace fir {
+class FirOpBuilder;
+}
+
 namespace Fortran::lower {
 
 /// When lowering a statement, temporaries for intermediate results may be
@@ -105,6 +114,11 @@ private:
   llvm::SmallVector<std::optional<CleanupFunction>> cufs;
 };
 
+/// If \p context contains any cleanups, ensure \p region has a block, and
+/// generate the cleanup inside that block.
+void genCleanUpInRegionIfAny(mlir::Location loc, fir::FirOpBuilder &builder,
+                             mlir::Region &region, StatementContext &context);
+
 } // namespace Fortran::lower
 
 #endif // FORTRAN_LOWER_STATEMENTCONTEXT_H
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
index 9558a6832972..376417e3c353 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
@@ -1330,7 +1330,8 @@ def hlfir_RegionAssignOp : hlfir_Op<"region_assign", [hlfir_OrderedAssignmentTre
 }
 
 def hlfir_YieldOp : hlfir_Op<"yield", [Terminator, ParentOneOf<["RegionAssignOp",
-    "ElementalAddrOp", "ForallOp", "ForallMaskOp", "WhereOp", "ElseWhereOp"]>,
+    "ElementalAddrOp", "ForallOp", "ForallMaskOp", "WhereOp", "ElseWhereOp",
+    "ExactlyOnceOp"]>,
     SingleBlockImplicitTerminator<"fir::FirEndOp">, RecursivelySpeculatable,
         RecursiveMemoryEffects]> {
 
@@ -1595,6 +1596,27 @@ def hlfir_ForallMaskOp : hlfir_AssignmentMaskOp<"forall_mask"> {
   let hasVerifier = 1;
 }
 
+def hlfir_ExactlyOnceOp : hlfir_Op<"exactly_once", [RecursiveMemoryEffects]> {
+  let summary = "Execute exactly once its region in a WhereOp";
+  let description = [{
+    Inside a Where assignment, Fortran requires a non elemental call and its
+    arguments to be executed exactly once, regardless of the mask values.
+    This operation allows holding these evaluations that cannot be hoisted
+    until potential parent Forall loops have been created.
+    It also allows inlining the calls without losing the information that
+    these calls must be hoisted.
+  }];
+
+  let regions = (region SizedRegion<1>:$body);
+
+  let results = (outs AnyFortranEntity:$result);
+
+  let assemblyFormat = [{
+    attr-dict `:` type($result)
+    $body
+  }];
+}
+
 def hlfir_WhereOp : hlfir_AssignmentMaskOp<"where"> {
   let summary = "Represent a Fortran where construct or statement";
   let description = [{
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 79d6bbf65cbf..596049fcfc92 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -3687,22 +3687,6 @@ private:
     return hlfir::Entity{valueAndPair.first};
   }
 
-  static void
-  genCleanUpInRegionIfAny(mlir::Location loc, fir::FirOpBuilder &builder,
-                          mlir::Region &region,
-                          Fortran::lower::StatementContext &context) {
-    if (!context.hasCode())
-      return;
-    mlir::OpBuilder::InsertPoint insertPt = builder.saveInsertionPoint();
-    if (region.empty())
-      builder.createBlock(&region);
-    else
-      builder.setInsertionPointToEnd(&region.front());
-    context.finalizeAndPop();
-    hlfir::YieldOp::ensureTerminator(region, builder, loc);
-    builder.restoreInsertionPoint(insertPt);
-  }
-
   bool firstDummyIsPointerOrAllocatable(
       const Fortran::evaluate::ProcedureRef &userDefinedAssignment) {
     using DummyAttr = Fortran::evaluate::characteristics::DummyDataObject::Attr;
@@ -3928,7 +3912,8 @@ private:
     Fortran::lower::StatementContext rhsContext;
     hlfir::Entity rhs = evaluateRhs(rhsContext);
     auto rhsYieldOp = builder.create<hlfir::YieldOp>(loc, rhs);
-    genCleanUpInRegionIfAny(loc, builder, rhsYieldOp.getCleanup(), rhsContext);
+    Fortran::lower::genCleanUpInRegionIfAny(
+        loc, builder, rhsYieldOp.getCleanup(), rhsContext);
     // Lower LHS in its own region.
     builder.createBlock(&regionAssignOp.getLhsRegion());
     Fortran::lower::StatementContext lhsContext;
@@ -3936,15 +3921,15 @@ private:
     if (!lhsHasVectorSubscripts) {
       hlfir::Entity lhs = evaluateLhs(lhsContext);
       auto lhsYieldOp = builder.create<hlfir::YieldOp>(loc, lhs);
-      genCleanUpInRegionIfAny(loc, builder, lhsYieldOp.getCleanup(),
-                              lhsContext);
+      Fortran::lower::genCleanUpInRegionIfAny(
+          loc, builder, lhsYieldOp.getCleanup(), lhsContext);
       lhsYield = lhs;
     } else {
       hlfir::ElementalAddrOp elementalAddr =
           Fortran::lower::convertVectorSubscriptedExprToElementalAddr(
               loc, *this, assign.lhs, localSymbols, lhsContext);
-      genCleanUpInRegionIfAny(loc, builder, elementalAddr.getCleanup(),
-                              lhsContext);
+      Fortran::lower::genCleanUpInRegionIfAny(
+          loc, builder, elementalAddr.getCleanup(), lhsContext);
       lhsYield = elementalAddr.getYieldOp().getEntity();
     }
     assert(lhsYield && "must have been set");
@@ -4299,7 +4284,8 @@ private:
         loc, *this, *maskExpr, localSymbols, maskContext);
     mask = hlfir::loadTrivialScalar(loc, *builder, mask);
     auto yieldOp = builder->create<hlfir::YieldOp>(loc, mask);
-    genCleanUpInRegionIfAny(loc, *builder, yieldOp.getCleanup(), maskContext);
+    Fortran::lower::genCleanUpInRegionIfAny(loc, *builder, yieldOp.getCleanup(),
+                                            maskContext);
   }
   void genFIR(const Fortran::parser::WhereConstructStmt &stmt) {
     const Fortran::semantics::SomeExpr *maskExpr = Fortran::semantics::GetExpr(
@@ -5599,3 +5585,18 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::support::setMLIRDataLayout(*module.get(),
                                   targetMachine.createDataLayout());
 }
+
+void Fortran::lower::genCleanUpInRegionIfAny(
+    mlir::Location loc, fir::FirOpBuilder &builder, mlir::Region &region,
+    Fortran::lower::StatementContext &context) {
+  if (!context.hasCode())
+    return;
+  mlir::OpBuilder::InsertPoint insertPt = builder.saveInsertionPoint();
+  if (region.empty())
+    builder.createBlock(&region);
+  else
+    builder.setInsertionPointToEnd(&region.front());
+  context.finalizeAndPop();
+  hlfir::YieldOp::ensureTerminator(region, builder, loc);
+  builder.restoreInsertionPoint(insertPt);
+}
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 3659dad367b4..c6bfe3592169 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -2682,10 +2682,48 @@ bool Fortran::lower::isIntrinsicModuleProcRef(
   return module && module->attrs().test(Fortran::semantics::Attr::INTRINSIC);
 }
 
+static bool isInWhereMaskedExpression(fir::FirOpBuilder &builder) {
+  // The MASK of the outer WHERE is not masked itself.
+  mlir::Operation *op = builder.getRegion().getParentOp();
+  return op && op->getParentOfType<hlfir::WhereOp>();
+}
+
 std::optional<hlfir::EntityWithAttributes> Fortran::lower::convertCallToHLFIR(
     mlir::Location loc, Fortran::lower::AbstractConverter &converter,
     const evaluate::ProcedureRef &procRef, std::optional<mlir::Type> resultType,
     Fortran::lower::SymMap &symMap, Fortran::lower::StatementContext &stmtCtx) {
+  auto &builder = converter.getFirOpBuilder();
+  if (resultType && !procRef.IsElemental() &&
+      isInWhereMaskedExpression(builder) &&
+      !builder.getRegion().getParentOfType<hlfir::ExactlyOnceOp>()) {
+    // Non elemental calls inside a where-assignment-stmt must be executed
+    // exactly once without mask control. Lower them in a special region so that
+    // this can be enforced whenscheduling forall/where expression evaluations.
+    Fortran::lower::StatementContext localStmtCtx;
+    mlir::Type bogusType = builder.getIndexType();
+    auto exactlyOnce = builder.create<hlfir::ExactlyOnceOp>(loc, bogusType);
+    mlir::Block *block = builder.createBlock(&exactlyOnce.getBody());
+    builder.setInsertionPointToStart(block);
+    CallContext callContext(procRef, resultType, loc, converter, symMap,
+                            localStmtCtx);
+    std::optional<hlfir::EntityWithAttributes> res =
+        genProcedureRef(callContext);
+    assert(res.has_value() && "must be a function");
+    auto yield = builder.create<hlfir::YieldOp>(loc, *res);
+    Fortran::lower::genCleanUpInRegionIfAny(loc, builder, yield.getCleanup(),
+                                            localStmtCtx);
+    builder.setInsertionPointAfter(exactlyOnce);
+    exactlyOnce->getResult(0).setType(res->getType());
+    if (hlfir::isFortranValue(exactlyOnce.getResult()))
+      return hlfir::EntityWithAttributes{exactlyOnce.getResult()};
+    // Create hlfir.declare for the result to satisfy
+    // hlfir::EntityWithAttributes requirements.
+    auto [exv, cleanup] = hlfir::translateToExtendedValue(
+        loc, builder, hlfir::Entity{exactlyOnce});
+    assert(!cleanup && "resut is a variable");
+    return hlfir::genDeclare(loc, builder, exv, ".func.pointer.result",
+                             fir::FortranVariableFlagsAttr{});
+  }
   CallContext callContext(procRef, resultType, loc, converter, symMap, stmtCtx);
   return genProcedureRef(callContext);
 }
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
index 63b52c0cd0bc..c9ff4b1c3374 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -56,7 +56,8 @@ namespace {
 /// expression and allows splitting the generation of the none elemental part
 /// from the elemental part.
 struct MaskedArrayExpr {
-  MaskedArrayExpr(mlir::Location loc, mlir::Region &region);
+  MaskedArrayExpr(mlir::Location loc, mlir::Region &region,
+                  bool isOuterMaskExpr);
 
   /// Generate the none elemental part. Must be called outside of the
   /// loops created for the WHERE construct.
@@ -79,16 +80,25 @@ struct MaskedArrayExpr {
   void generateNoneElementalCleanupIfAny(fir::FirOpBuilder &builder,
                                          mlir::IRMapping &mapper);
 
+  /// Helper to clone the clean-ups of the masked expr region terminator.
+  /// This is called outside of the loops for the initial mask, and inside
+  /// the loops for the other masked expressions.
+  mlir::Operation *generateMaskedExprCleanUps(fir::FirOpBuilder &builder,
+                                              mlir::IRMapping &mapper);
+
   mlir::Location loc;
   mlir::Region &region;
-  /// Was generateNoneElementalPart called?
-  bool noneElementalPartWasGenerated = false;
   /// Set of operations that form the elemental parts of the
   /// expression evaluation. These are the hlfir.elemental and
   /// hlfir.elemental_addr that form the elemental tree producing
   /// the expression value. hlfir.elemental that produce values
   /// used inside transformational operations are not part of this set.
   llvm::SmallSet<mlir::Operation *, 4> elementalParts{};
+  /// Was generateNoneElementalPart called?
+  bool noneElementalPartWasGenerated = false;
+  /// Is this expression the mask expression of the outer where statement?
+  /// It is special because its evaluation is not masked by anything yet.
+  bool isOuterMaskExpr = false;
 };
 } // namespace
 
@@ -202,7 +212,7 @@ private:
   /// This method returns the scalar element (that may have been previously
   /// saved) for the current indices inside the where loop.
   mlir::Value generateMaskedEntity(mlir::Location loc, mlir::Region &region) {
-    MaskedArrayExpr maskedExpr(loc, region);
+    MaskedArrayExpr maskedExpr(loc, region, /*isOuterMaskExpr=*/!whereLoopNest);
     return generateMaskedEntity(maskedExpr);
   }
   mlir::Value generateMaskedEntity(MaskedArrayExpr &maskedExpr);
@@ -524,7 +534,8 @@ void OrderedAssignmentRewriter::pre(hlfir::WhereOp whereOp) {
       return;
     }
     // The mask was not evaluated yet or can be safely re-evaluated.
-    MaskedArrayExpr mask(loc, whereOp.getMaskRegion());
+    MaskedArrayExpr mask(loc, whereOp.getMaskRegion(),
+                         /*isOuterMaskExpr=*/true);
     mask.generateNoneElementalPart(builder, mapper);
     mlir::Value shape = mask.generateShape(builder, mapper);
     whereLoopNest = hlfir::genLoopNest(loc, builder, shape);
@@ -628,6 +639,13 @@ OrderedAssignmentRewriter::getIfSaved(mlir::Region &region) {
   return std::nullopt;
 }
 
+static hlfir::YieldOp getYield(mlir::Region &region) {
+  auto yield = mlir::dyn_cast_or_null<hlfir::YieldOp>(
+      region.back().getOperations().back());
+  assert(yield && "region computing entities must end with a YieldOp");
+  return yield;
+}
+
 OrderedAssignmentRewriter::ValueAndCleanUp
 OrderedAssignmentRewriter::generateYieldedEntity(
     mlir::Region &region, std::optional<mlir::Type> castToType) {
@@ -644,9 +662,7 @@ OrderedAssignmentRewriter::generateYieldedEntity(
   }
 
   assert(region.hasOneBlock() && "region must contain one block");
-  auto oldYield = mlir::dyn_cast_or_null<hlfir::YieldOp>(
-      region.back().getOperations().back());
-  assert(oldYield && "region computing entities must end with a YieldOp");
+  auto oldYield = getYield(region);
   mlir::Block::OpListType &ops = region.back().getOperations();
 
   // Inside Forall, scalars that do not depend on forall indices can be hoisted
@@ -792,8 +808,15 @@ OrderedAssignmentRewriter::generateMaskedEntity(MaskedArrayExpr &maskedExpr) {
   // at the current insertion point (inside the where loops, and any fir.if
   // generated for previous masks).
   builder.restoreInsertionPoint(insertionPoint);
-  return maskedExpr.generateElementalParts(
+  mlir::Value scalar = maskedExpr.generateElementalParts(
       builder, whereLoopNest->oneBasedIndices, mapper);
+  /// Generate cleanups for the elemental parts inside the loops (setting the
+  /// location so that the assignment will be generated before the cleanups).
+  if (!maskedExpr.isOuterMaskExpr)
+    if (mlir::Operation *firstCleanup =
+            maskedExpr.generateMaskedExprCleanUps(builder, mapper))
+      builder.setInsertionPoint(firstCleanup);
+  return scalar;
 }
 
 void OrderedAssignmentRewriter::generateCleanupIfAny(
@@ -887,8 +910,9 @@ gatherElementalTree(hlfir::ElementalOpInterface elemental,
   }
 }
 
-MaskedArrayExpr::MaskedArrayExpr(mlir::Location loc, mlir::Region &region)
-    : loc{loc}, region{region} {
+MaskedArrayExpr::MaskedArrayExpr(mlir::Location loc, mlir::Region &region,
+                                 bool isOuterMaskExpr)
+    : loc{loc}, region{region}, isOuterMaskExpr{isOuterMaskExpr} {
   mlir::Operation &terminator = region.back().back();
   if (auto elementalAddr =
           mlir::dyn_cast<hlfir::ElementalOpInterface>(terminator)) {
@@ -907,13 +931,36 @@ void MaskedArrayExpr::generateNoneElementalPart(fir::FirOpBuilder &builder,
                                                 mlir::IRMapping &mapper) {
   assert(!noneElementalPartWasGenerated &&
          "none elemental parts already generated");
-  // Clone all operations, except the elemental and the final yield.
-  mlir::Block::OpListType &ops = region.back().getOperations();
-  assert(!ops.empty() && "yield block cannot be empty");
-  auto end = ops.end();
-  for (auto opIt = ops.begin(); std::next(opIt) != end; ++opIt)
-    if (!elementalParts.contains(&*opIt))
-      (void)builder.clone(*opIt, mapper);
+  if (isOuterMaskExpr) {
+    // The outer mask expression is actually not masked, it is dealt as
+    // such so that its elemental part, if any, can be inlined in the WHERE
+    // loops. But all of the operations outside of hlfir.elemental/
+    // hlfir.elemental_addr must be emitted now because their value may be
+    // required to deduce the mask shape and the WHERE loop bounds.
+    for (mlir::Operation &op : region.back().without_terminator())
+      if (!elementalParts.contains(&op))
+        (void)builder.clone(op, mapper);
+  } else {
+    // For actual masked expressions, Fortran requires elemental expressions,
+    // even the scalar ones that are not encoded with hlfir.elemental, to be
+    // evaluated only when the mask is true. Blindly hoisting all scalar SSA
+    // tree could be wrong if the scalar computation has side effects and
+    // would never have been evaluated (e.g. division by zero) if the mask
+    // is fully false. See F'2023 10.2.3.2 point 10.
+    // Clone only the bodies of all hlfir.exactly_once operations, which contain
+    // the evaluation of sub-expression tree whose root was a non elemental
+    // function call at the Fortran level (the call itself may have been inlined
+    // since). These must be evaluated only once as per F'2023 10.2.3.2 point 9.
+    for (mlir::Operation &op : region.back().without_terminator())
+      if (auto exactlyOnce = mlir::dyn_cast<hlfir::ExactlyOnceOp>(op)) {
+        for (mlir::Operation &subOp :
+             exactlyOnce.getBody().back().without_terminator())
+          (void)builder.clone(subOp, mapper);
+        mlir::Value oldYield = getYield(exactlyOnce.getBody()).getEntity();
+        auto newYield = mapper.lookupOrDefault(oldYield);
+        mapper.map(exactlyOnce.getResult(), newYield);
+      }
+  }
   noneElementalPartWasGenerated = true;
 }
 
@@ -942,6 +989,15 @@ MaskedArrayExpr::generateElementalParts(fir::FirOpBuilder &builder,
                                         mlir::IRMapping &mapper) {
   assert(noneElementalPartWasGenerated &&
          "non elemental part must have been generated");
+  if (!isOuterMaskExpr) {
+    // Clone all operations that are not hlfir.exactly_once and that are not
+    // hlfir.elemental/hlfir.elemental_addr.
+    for (mlir::Operation &op : region.back().without_terminator())
+      if (!mlir::isa<hlfir::ExactlyOnceOp>(op) && !elementalParts.contains(&op))
+        (void)builder.clone(op, mapper);
+    // For the outer mask, this was already done outside of the loop.
+  }
+  // Clone and "index" bodies of hlfir.elemental/hlfir.elemental_addr.
   mlir::Operation &terminator = region.back().back();
   hlfir::ElementalOpInterface elemental =
       mlir::dyn_cast<hlfir::ElementalAddrOp>(terminator);
@@ -966,8 +1022,11 @@ MaskedArrayExpr::generateElementalParts(fir::FirOpBuilder &builder,
                            mustRecursivelyInline);
 }
 
-void MaskedArrayExpr::generateNoneElementalCleanupIfAny(
-    fir::FirOpBuilder &builder, mlir::IRMapping &mapper) {
+mlir::Operation *
+MaskedArrayExpr::generateMaskedExprCleanUps(fir::FirOpBuilder &builder,
+                                            mlir::IRMapping &mapper) {
+  // Clone the clean-ups from the region itself, except for the destroy
+  // of the hlfir.elemental that have been inlined.
   mlir::Operation &terminator = region.back().back();
   mlir::Region *cleanupRegion = nullptr;
   if (auto elementalAddr = mlir::dyn_cast<hlfir::ElementalAddrOp>(terminator)) {
@@ -977,12 +1036,39 @@ void MaskedArrayExpr::generateNoneElementalCleanupIfAny(
     cleanupRegion = &yieldOp.getCleanup();
   }
   if (cleanupRegion->empty())
-    return;
+    return nullptr;
+  mlir::Operation *firstNewCleanup = nullptr;
   for (mlir::Operation &op : cleanupRegion->front().without_terminator()) {
     if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(op))
       if (elementalParts.contains(destroy.getExpr().getDefiningOp()))
         continue;
-    (void)builder.clone(op, mapper);
+    mlir::Operation *cleanup = builder.clone(op, mapper);
+    if (!firstNewCleanup)
+      firstNewCleanup = cleanup;
+  }
+  return firstNewCleanup;
+}
+
+void MaskedArrayExpr::generateNoneElementalCleanupIfAny(
+    fir::FirOpBuilder &builder, mlir::IRMapping &mapper) {
+  if (!isOuterMaskExpr) {
+    // Clone clean-ups of hlfir.exactly_once operations (in reverse order
+    // to properly deal with stack restores).
+    for (mlir::Operation &op :
+         llvm::reverse(region.back().without_terminator()))
+      if (auto exactlyOnce = mlir::dyn_cast<hlfir::ExactlyOnceOp>(op)) {
+        mlir::Region &cleanupRegion =
+            getYield(exactlyOnce.getBody()).getCleanup();
+        if (!cleanupRegion.empty())
+          for (mlir::Operation &cleanupOp :
+               cleanupRegion.front().without_terminator())
+            (void)builder.clone(cleanupOp, mapper);
+      }
+  } else {
+    // For the outer mask, the region clean-ups must be generated
+    // outside of the loops since the mask non hlfir.elemental part
+    // is generated before the loops.
+    generateMaskedExprCleanUps(builder, mapper);
   }
 }
 
diff --git a/flang/test/HLFIR/order_assignments/impure-where.fir b/flang/test/HLFIR/order_assignments/impure-where.fir
index 9399ea83d182..011a486b2baf 100644
--- a/flang/test/HLFIR/order_assignments/impure-where.fir
+++ b/flang/test/HLFIR/order_assignments/impure-where.fir
@@ -13,10 +13,13 @@ func.func @test_elsewhere_impure_mask(%x: !fir.ref<!fir.array<10xi32>>, %y: !fir
     hlfir.yield %mask : !fir.ref<!fir.array<10x!fir.logical<4>>>
   } do {
     hlfir.elsewhere mask {
-      %mask2 = fir.call @impure() : () -> !fir.heap<!fir.array<10x!fir.logical<4>>>
-      hlfir.yield %mask2 : !fir.heap<!fir.array<10x!fir.logical<4>>> cleanup {
-        fir.freemem %mask2 : !fir.heap<!fir.array<10x!fir.logical<4>>>
+      %mask2 = hlfir.exactly_once : !fir.heap<!fir.array<10x!fir.logical<4>>> {
+        %imp = fir.call @impure() : () -> !fir.heap<!fir.array<10x!fir.logical<4>>>
+        hlfir.yield %imp : !fir.heap<!fir.array<10x!fir.logical<4>>> cleanup {
+          fir.freemem %imp : !fir.heap<!fir.array<10x!fir.logical<4>>>
+        }
       }
+      hlfir.yield %mask2 : !fir.heap<!fir.array<10x!fir.logical<4>>>
     } do {
       hlfir.region_assign {
         hlfir.yield %y : !fir.ref<!fir.array<10xi32>>
diff --git a/flang/test/HLFIR/order_assignments/inlined-stack-temp.fir b/flang/test/HLFIR/order_assignments/inlined-stack-temp.fir
index 66ff55558ea6..0724d019537c 100644
--- a/flang/test/HLFIR/order_assignments/inlined-stack-temp.fir
+++ b/flang/test/HLFIR/order_assignments/inlined-stack-temp.fir
@@ -282,7 +282,6 @@ func.func @test_where_rhs_save(%x: !fir.ref<!fir.array<10xi32>>, %mask: !fir.ref
 // CHECK:           %[[VAL_7:.*]] = arith.constant 10 : index
 // CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>
 // CHECK:           %[[VAL_9:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_10:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_5]]:%[[VAL_4]]:%[[VAL_3]])  shape %[[VAL_6]] : (!fir.ref<!fir.array<10xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<10xi32>>
 // CHECK:           %[[VAL_11:.*]] = arith.constant 0 : index
 // CHECK:           %[[VAL_12:.*]] = arith.subi %[[VAL_7]], %[[VAL_9]] : index
 // CHECK:           %[[VAL_13:.*]] = arith.addi %[[VAL_12]], %[[VAL_9]] : index
@@ -300,6 +299,7 @@ func.func @test_where_rhs_save(%x: !fir.ref<!fir.array<10xi32>>, %mask: !fir.ref
 // CHECK:             %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<!fir.logical<4>>
 // CHECK:             %[[VAL_25:.*]] = fir.convert %[[VAL_24]] : (!fir.logical<4>) -> i1
 // CHECK:             fir.if %[[VAL_25]] {
+// CHECK:           %[[VAL_10:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_5]]:%[[VAL_4]]:%[[VAL_3]])  shape %[[VAL_6]] : (!fir.ref<!fir.array<10xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<10xi32>>
 // CHECK:               %[[VAL_26:.*]] = hlfir.designate %[[VAL_10]] (%[[VAL_22]])  : (!fir.ref<!fir.array<10xi32>>, index) -> !fir.ref<i32>
 // CHECK:               %[[VAL_27:.*]] = fir.load %[[VAL_26]] : !fir.ref<i32>
 // CHECK:               %[[VAL_28:.*]] = fir.load %[[VAL_2]] : !fir.ref<index>
diff --git a/flang/test/HLFIR/order_assignments/user-defined-assignment-finalization.fir b/flang/test/HLFIR/order_assignments/user-defined-assignment-finalization.fir
index bbb589169a80..ae5329a2d243 100644
--- a/flang/test/HLFIR/order_assignments/user-defined-assignment-finalization.fir
+++ b/flang/test/HLFIR/order_assignments/user-defined-assignment-finalization.fir
@@ -59,7 +59,7 @@ func.func @_QPtest1() {
       %7 = fir.call @_FortranADestroy(%6) fastmath<contract> : (!fir.box<none>) -> none
     }
   } to {
-    hlfir.yield %2#0 : !fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>> 
+    hlfir.yield %2#0 : !fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
   } user_defined_assign  (%arg0: !fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) to (%arg1: !fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) {
     %3 = fir.embox %arg1 : (!fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) -> !fir.box<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
     %4 = fir.convert %3 : (!fir.box<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) -> !fir.class<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
@@ -119,7 +119,7 @@ func.func @_QPtest2() {
       fir.call @llvm.stackrestore.p0(%4) fastmath<contract> : (!fir.ref<i8>) -> ()
     }
   } to {
-    hlfir.yield %3#0 : !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>> 
+    hlfir.yield %3#0 : !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>
   } user_defined_assign  (%arg0: !fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) to (%arg1: !fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) {
     %4 = fir.embox %arg1 : (!fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) -> !fir.box<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
     %5 = fir.convert %4 : (!fir.box<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) -> !fir.class<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
@@ -193,18 +193,22 @@ func.func @_QPtest3(%arg0: !fir.ref<!fir.array<2xi32>> {fir.bindc_name = "y"}) {
     }
   } do {
     hlfir.region_assign {
-      %5 = fir.call @llvm.stacksave.p0() fastmath<contract> : () -> !fir.ref<i8>
-      %6 = fir.call @_QPnew_obja() fastmath<contract> : () -> !fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
-      fir.save_result %6 to %0(%2) : !fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>, !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, !fir.shape<1>
-      %7:2 = hlfir.declare %0(%2) {uniq_name = ".tmp.func_result"} : (!fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>)
-      hlfir.yield %7#0 : !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>> cleanup {
-        %8 = fir.embox %0(%2) : (!fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>
-        %9 = fir.convert %8 : (!fir.box<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>) -> !fir.box<none>
-        %10 = fir.call @_FortranADestroy(%9) fastmath<contract> : (!fir.box<none>) -> none
-        fir.call @llvm.stackrestore.p0(%5) fastmath<contract> : (!fir.ref<i8>) -> ()
+      %5 = hlfir.exactly_once : !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>> {
+        %7 = fir.call @llvm.stacksave.p0() fastmath<contract> : () -> !fir.ref<i8>
+        %8 = fir.call @_QPnew_obja() fastmath<contract> : () -> !fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
+        fir.save_result %8 to %0(%2) : !fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>, !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, !fir.shape<1>
+        %9:2 = hlfir.declare %0(%2) {uniq_name = ".tmp.func_result"} : (!fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>)
+        hlfir.yield %9#0 : !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>> cleanup {
+          %10 = fir.embox %0(%2) : (!fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>
+          %11 = fir.convert %10 : (!fir.box<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>) -> !fir.box<none>
+          %12 = fir.call @_FortranADestroy(%11) fastmath<contract> : (!fir.box<none>) -> none
+          fir.call @llvm.stackrestore.p0(%7) fastmath<contract> : (!fir.ref<i8>) -> ()
+        }
       }
+      %6:2 = hlfir.declare %5(%2) {uniq_name = ".func.pointer.result"} : (!fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>)
+      hlfir.yield %6#0 : !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>
     } to {
-      hlfir.yield %3#0 : !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>> 
+      hlfir.yield %3#0 : !fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>
     } user_defined_assign  (%arg1: !fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) to (%arg2: !fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) {
       %5 = fir.embox %arg2 : (!fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) -> !fir.box<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
       %6 = fir.convert %5 : (!fir.box<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) -> !fir.class<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
@@ -246,7 +250,8 @@ func.func @_QPtest3(%arg0: !fir.ref<!fir.array<2xi32>> {fir.bindc_name = "y"}) {
 // CHECK:             %[[VAL_30:.*]] = fir.load %[[VAL_29]] : !fir.ref<!fir.logical<4>>
 // CHECK:             %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (!fir.logical<4>) -> i1
 // CHECK:             fir.if %[[VAL_31]] {
-// CHECK:               %[[VAL_32:.*]] = hlfir.designate %[[VAL_20]]#0 (%[[VAL_28]])  : (!fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, index) -> !fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
+// CHECK:               %[[VAL_20B:.*]]:2 = hlfir.declare %[[VAL_20]]#0(%[[VAL_7]]) {uniq_name = ".func.pointer.result"}
+// CHECK:               %[[VAL_32:.*]] = hlfir.designate %[[VAL_20B]]#0 (%[[VAL_28]])  : (!fir.ref<!fir.array<2x!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>>, index) -> !fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
 // CHECK:               %[[VAL_33:.*]] = fir.embox %[[VAL_32]] : (!fir.ref<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) -> !fir.box<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>
 // CHECK:               %[[VAL_34:.*]] = fir.convert %[[VAL_33]] : (!fir.box<!fir.type<_QMtypesTud_assign{x:!fir.box<!fir.ptr<i32>>}>>) -> !fir.box<none>
 // CHECK:               %[[VAL_35:.*]] = fir.call @_FortranAPushValue(%[[VAL_27]], %[[VAL_34]]) : (!fir.llvm_ptr<i8>, !fir.box<none>) -> none
diff --git a/flang/test/HLFIR/order_assignments/where-cleanup.f90 b/flang/test/HLFIR/order_assignments/where-cleanup.f90
new file mode 100644
index 000000000000..b4d16452bd2c
--- /dev/null
+++ b/flang/test/HLFIR/order_assignments/where-cleanup.f90
@@ -0,0 +1,44 @@
+// Test hlfir.where masked region cleanup lowering (the freemem in the tests).
+// RUN: fir-opt %s --lower-hlfir-ordered-assignments | FileCheck %s
+
+func.func @loop_cleanup(%mask : !fir.ref<!fir.array<2x!fir.logical<4>>>, %x : !fir.ref<!fir.array<2xf32>>, %y : !fir.ref<!fir.array<2xf32>>) {
+    hlfir.where {
+        %1 = fir.allocmem !fir.array<10xi32>
+        hlfir.yield %mask : !fir.ref<!fir.array<2x!fir.logical<4>>> cleanup {
+          fir.freemem %1 : !fir.heap<!fir.array<10xi32>>
+        }
+    } do {
+      hlfir.region_assign {
+        %1 = fir.allocmem !fir.array<1xi32>
+        %2 = fir.allocmem !fir.array<2xi32>
+        hlfir.yield %x : !fir.ref<!fir.array<2xf32>> cleanup {
+          fir.freemem %2 : !fir.heap<!fir.array<2xi32>>
+          fir.freemem %1 : !fir.heap<!fir.array<1xi32>>
+        }
+      } to {
+        %1 = fir.allocmem !fir.array<3xi32>
+        %2 = fir.allocmem !fir.array<4xi32>
+        hlfir.yield %y : !fir.ref<!fir.array<2xf32>> cleanup {
+          fir.freemem %2 : !fir.heap<!fir.array<4xi32>>
+          fir.freemem %1 : !fir.heap<!fir.array<3xi32>>
+        }
+      }
+    }
+  return
+}
+// CHECK-LABEL:   func.func @loop_cleanup(
+// CHECK:           %[[VAL_3:.*]] = fir.allocmem !fir.array<10xi32>
+// CHECK:           fir.do_loop
+// CHECK:             fir.if
+// CHECK:               %[[VAL_11:.*]] = fir.allocmem !fir.array<1xi32>
+// CHECK:               %[[VAL_12:.*]] = fir.allocmem !fir.array<2xi32>
+// CHECK:               %[[VAL_14:.*]] = fir.allocmem !fir.array<3xi32>
+// CHECK:               %[[VAL_15:.*]] = fir.allocmem !fir.array<4xi32>
+// CHECK:               hlfir.assign
+// CHECK:               fir.freemem %[[VAL_15]] : !fir.heap<!fir.array<4xi32>>
+// CHECK:               fir.freemem %[[VAL_14]] : !fir.heap<!fir.array<3xi32>>
+// CHECK:               fir.freemem %[[VAL_12]] : !fir.heap<!fir.array<2xi32>>
+// CHECK:               fir.freemem %[[VAL_11]] : !fir.heap<!fir.array<1xi32>>
+// CHECK:             }
+// CHECK:           }
+// CHECK:           fir.freemem %[[VAL_3]] : !fir.heap<!fir.array<10xi32>>
diff --git a/flang/test/HLFIR/order_assignments/where-codegen-no-conflict.fir b/flang/test/HLFIR/order_assignments/where-codegen-no-conflict.fir
index ac93e6828096..a1a357b45a64 100644
--- a/flang/test/HLFIR/order_assignments/where-codegen-no-conflict.fir
+++ b/flang/test/HLFIR/order_assignments/where-codegen-no-conflict.fir
@@ -290,8 +290,6 @@ func.func @inside_forall(%arg0: !fir.ref<!fir.array<10x20xf32>>, %arg1: !fir.ref
 // CHECK:           fir.do_loop %[[VAL_15:.*]] = %[[VAL_12]] to %[[VAL_13]] step %[[VAL_14]] {
 // CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (index) -> i32
 // CHECK:             %[[VAL_17:.*]] = arith.constant 1 : index
-// CHECK:             %[[VAL_18:.*]] = fir.convert %[[VAL_16]] : (i32) -> i64
-// CHECK:             %[[VAL_19:.*]] = hlfir.designate %[[VAL_9]]#0 (%[[VAL_18]], %[[VAL_2]]:%[[VAL_7]]:%[[VAL_2]])  shape %[[VAL_10]] : (!fir.ref<!fir.array<10x20xf32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<20xf32>>
 // CHECK:             fir.do_loop %[[VAL_20:.*]] = %[[VAL_17]] to %[[VAL_7]] step %[[VAL_17]] {
 // CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_11]]#0 (%[[VAL_20]])  : (!fir.ref<!fir.array<20xf32>>, index) -> !fir.ref<f32>
 // CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<f32>
@@ -300,6 +298,8 @@ func.func @inside_forall(%arg0: !fir.ref<!fir.array<10x20xf32>>, %arg1: !fir.ref
 // CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_24]] : (!fir.logical<4>) -> i1
 // CHECK:               fir.if %[[VAL_25]] {
 // CHECK:                 %[[VAL_26:.*]] = hlfir.designate %[[VAL_11]]#0 (%[[VAL_20]])  : (!fir.ref<!fir.array<20xf32>>, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_18:.*]] = fir.convert %[[VAL_16]] : (i32) -> i64
+// CHECK:                 %[[VAL_19:.*]] = hlfir.designate %[[VAL_9]]#0 (%[[VAL_18]], %[[VAL_2]]:%[[VAL_7]]:%[[VAL_2]])  shape %[[VAL_10]] : (!fir.ref<!fir.array<10x20xf32>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<20xf32>>
 // CHECK:                 %[[VAL_27:.*]] = hlfir.designate %[[VAL_19]] (%[[VAL_20]])  : (!fir.box<!fir.array<20xf32>>, index) -> !fir.ref<f32>
 // CHECK:                 hlfir.assign %[[VAL_26]] to %[[VAL_27]] : !fir.ref<f32>, !fir.ref<f32>
 // CHECK:               }
diff --git a/flang/test/HLFIR/order_assignments/where-hoisting.f90 b/flang/test/HLFIR/order_assignments/where-hoisting.f90
new file mode 100644
index 000000000000..6ed2ecb3624b
--- /dev/null
+++ b/flang/test/HLFIR/order_assignments/where-hoisting.f90
@@ -0,0 +1,50 @@
+! Test that scalar expressions are not hoisted from WHERE loops
+! when they do not appear
+! RUN: bbc -hlfir -o - -pass-pipeline="builtin.module(lower-hlfir-ordered-assignments)" %s | FileCheck %s
+
+subroutine do_not_hoist_div(n, mask, a)
+  integer :: a(10), n
+  logical :: mask(10)
+  where(mask) a=1/n
+end subroutine
+! CHECK-LABEL:   func.func @_QPdo_not_hoist_div(
+! CHECK-NOT:       arith.divsi
+! CHECK:           fir.do_loop {{.*}} {
+! CHECK:             fir.if {{.*}} {
+! CHECK:               arith.divsi
+! CHECK:             }
+! CHECK:           }
+
+subroutine do_not_hoist_optional(n, mask, a)
+  integer :: a(10)
+  integer, optional :: n
+  logical :: mask(10)
+  where(mask) a=n
+end subroutine
+! CHECK-LABEL:   func.func @_QPdo_not_hoist_optional(
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare {{.*}}"_QFdo_not_hoist_optionalEn"
+! CHECK-NOT:       fir.load %[[VAL_9]]
+! CHECK:           fir.do_loop {{.*}} {
+! CHECK:             fir.if {{.*}} {
+! CHECK:               %[[VAL_15:.*]] = fir.load %[[VAL_9]]#0 : !fir.ref<i32>
+! CHECK:             }
+! CHECK:           }
+
+subroutine hoist_function(n, mask, a)
+  integer :: a(10, 10)
+  integer, optional :: n
+  logical :: mask(10, 10)
+  forall (i=1:10)
+  where(mask(i, :)) a(i,:)=ihoist_me(i)
+  end forall
+end subroutine
+! CHECK-LABEL:   func.func @_QPhoist_function(
+! CHECK:           fir.do_loop {{.*}} {
+! CHECK:             fir.call @_QPihoist_me
+! CHECK:             fir.do_loop {{.*}} {
+! CHECK:               fir.if %{{.*}} {
+! CHECK-NOT:             fir.call @_QPihoist_me
+! CHECK:               }
+! CHECK:             }
+! CHECK:           }
+! CHECK-NOT:       fir.call @_QPihoist_me
diff --git a/flang/test/Lower/HLFIR/where-nonelemental.f90 b/flang/test/Lower/HLFIR/where-nonelemental.f90
new file mode 100644
index 000000000000..f0a6857f0f4b
--- /dev/null
+++ b/flang/test/Lower/HLFIR/where-nonelemental.f90
@@ -0,0 +1,198 @@
+! Test lowering of non elemental calls and there inputs inside WHERE
+! constructs. These must be lowered inside hlfir.exactly_once so that
+! they are properly hoisted once the loops are materialized and
+! expression evaluations are scheduled.
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
+
+subroutine test_where(a, b, c)
+ real, dimension(:) :: a, b, c
+ interface
+  function logical_func1()
+    logical :: logical_func1(100)
+  end function
+  function logical_func2()
+    logical :: logical_func2(100)
+  end function
+  real elemental function elem_func(x)
+    real, intent(in) :: x
+  end function
+ end interface
+ where (logical_func1())
+  a = b + real_func(a+b+real_func2()) + elem_func(a)
+ elsewhere(logical_func2())
+  a(1:ifoo()) = c
+ end where
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest_where(
+! CHECK:           hlfir.where {
+! CHECK-NOT: hlfir.exactly_once
+! CHECK:             %[[VAL_17:.*]] = fir.call @llvm.stacksave.p0() fastmath<contract> : () -> !fir.ref<i8>
+! CHECK:             %[[VAL_19:.*]] = fir.call @_QPlogical_func1() fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
+! CHECK:             hlfir.yield %{{.*}} : !hlfir.expr<100x!fir.logical<4>> cleanup {
+! CHECK:               fir.call @llvm.stackrestore.p0(%[[VAL_17]]) fastmath<contract> : (!fir.ref<i8>) -> ()
+! CHECK:             }
+! CHECK:           } do {
+! CHECK:             hlfir.region_assign {
+! CHECK:               %[[VAL_24:.*]] = hlfir.exactly_once : f32 {
+! CHECK:                 %[[VAL_28:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:                 }
+! CHECK-NOT: hlfir.exactly_once
+! CHECK:                 %[[VAL_35:.*]] = fir.call @_QPreal_func2() fastmath<contract> : () -> f32
+! CHECK:                 %[[VAL_36:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:                 ^bb0(%[[VAL_37:.*]]: index):
+! CHECK:                   %[[VAL_38:.*]] = hlfir.apply %[[VAL_28]], %[[VAL_37]] : (!hlfir.expr<?xf32>, index) -> f32
+! CHECK:                   %[[VAL_39:.*]] = arith.addf %[[VAL_38]], %[[VAL_35]] fastmath<contract> : f32
+! CHECK:                   hlfir.yield_element %[[VAL_39]] : f32
+! CHECK:                 }
+! CHECK:                 %[[VAL_41:.*]] = fir.call @_QPreal_func
+! CHECK:                 hlfir.yield %[[VAL_41]] : f32 cleanup {
+! CHECK:                   hlfir.destroy %[[VAL_36]] : !hlfir.expr<?xf32>
+! CHECK:                   hlfir.destroy %[[VAL_28]] : !hlfir.expr<?xf32>
+! CHECK:                 }
+! CHECK:               }
+! CHECK:               %[[VAL_45:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:                 arith.addf
+! CHECK-NOT: hlfir.exactly_once
+! CHECK:               }
+! CHECK:               %[[VAL_53:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:                 fir.call @_QPelem_func
+! CHECK:               }
+! CHECK:               %[[VAL_57:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:                 arith.addf
+! CHECK:               }
+! CHECK:               hlfir.yield %[[VAL_57]] : !hlfir.expr<?xf32> cleanup {
+! CHECK:                 hlfir.destroy %[[VAL_57]] : !hlfir.expr<?xf32>
+! CHECK:                 hlfir.destroy %[[VAL_53]] : !hlfir.expr<?xf32>
+! CHECK:                 hlfir.destroy %[[VAL_45]] : !hlfir.expr<?xf32>
+! CHECK:               }
+! CHECK:             } to {
+! CHECK:               hlfir.yield %{{.*}} : !fir.box<!fir.array<?xf32>>
+! CHECK:             }
+! CHECK:             hlfir.elsewhere mask {
+! CHECK:               %[[VAL_62:.*]] = hlfir.exactly_once : !hlfir.expr<100x!fir.logical<4>> {
+! CHECK:                 %[[VAL_72:.*]] = fir.call @llvm.stacksave.p0() fastmath<contract> : () -> !fir.ref<i8>
+! CHECK:                 fir.call @_QPlogical_func2() fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
+! CHECK:                 hlfir.yield %{{.*}} : !hlfir.expr<100x!fir.logical<4>> cleanup {
+! CHECK:                   fir.call @llvm.stackrestore.p0(%[[VAL_72]]) fastmath<contract> : (!fir.ref<i8>) -> ()
+! CHECK:                 }
+! CHECK:               }
+! CHECK:               hlfir.yield %[[VAL_62]] : !hlfir.expr<100x!fir.logical<4>>
+! CHECK:             } do {
+! CHECK:               hlfir.region_assign {
+! CHECK:                 hlfir.yield %{{.*}} : !fir.box<!fir.array<?xf32>>
+! CHECK:               } to {
+! CHECK:                 %[[VAL_80:.*]] = hlfir.exactly_once : i32 {
+! CHECK:                   %[[VAL_81:.*]] = fir.call @_QPifoo() fastmath<contract> : () -> i32
+! CHECK:                   hlfir.yield %[[VAL_81]] : i32
+! CHECK:                 }
+! CHECK:                 hlfir.yield %{{.*}} : !fir.box<!fir.array<?xf32>>
+! CHECK:               }
+! CHECK:             }
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
+
+subroutine test_where_in_forall(a, b, c)
+ real, dimension(:, :) :: a, b, c
+ interface
+  pure function pure_logical_func1()
+    logical :: pure_logical_func1(100)
+  end function
+  pure function pure_logical_func2()
+    logical :: pure_logical_func2(100)
+  end function
+  real pure elemental function pure_elem_func(x)
+    real, intent(in) :: x
+  end function
+  integer pure function pure_ifoo()
+  end function
+ end interface
+ forall(i=1:10)
+   where (pure_logical_func1())
+    a(2*i, :) = b(i, :) + pure_real_func(a(i,:)+b(i,:)+pure_real_func2()) + pure_elem_func(a(i,:))
+   elsewhere(pure_logical_func2())
+    a(2*i, 1:pure_ifoo()) = c(i, :)
+   end where
+ end forall
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest_where_in_forall(
+! CHECK:           hlfir.forall lb {
+! CHECK:             hlfir.yield %{{.*}} : i32
+! CHECK:           } ub {
+! CHECK:             hlfir.yield %{{.*}} : i32
+! CHECK:           }  (%[[VAL_10:.*]]: i32) {
+! CHECK:             %[[VAL_11:.*]] = hlfir.forall_index "i" %[[VAL_10]] : (i32) -> !fir.ref<i32>
+! CHECK:             hlfir.where {
+! CHECK:               %[[VAL_21:.*]] = fir.call @llvm.stacksave.p0() fastmath<contract> : () -> !fir.ref<i8>
+! CHECK-NOT: hlfir.exactly_once
+! CHECK:               %[[VAL_23:.*]] = fir.call @_QPpure_logical_func1() fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
+! CHECK:               hlfir.yield %{{.*}} : !hlfir.expr<100x!fir.logical<4>> cleanup {
+! CHECK:                 fir.call @llvm.stackrestore.p0(%[[VAL_21]]) fastmath<contract> : (!fir.ref<i8>) -> ()
+! CHECK:               }
+! CHECK:             } do {
+! CHECK:               hlfir.region_assign {
+! CHECK:                 %[[VAL_41:.*]] = hlfir.designate
+! CHECK:                 %[[VAL_42:.*]] = hlfir.exactly_once : f32 {
+! CHECK:                                    hlfir.designate
+! CHECK:                                    hlfir.designate
+! CHECK:                   %[[VAL_71:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:                     arith.addf
+! CHECK:                   }
+! CHECK-NOT: hlfir.exactly_once
+! CHECK:                   %[[VAL_78:.*]] = fir.call @_QPpure_real_func2() fastmath<contract> : () -> f32
+! CHECK:                   %[[VAL_79:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:                     arith.addf
+! CHECK:                   }
+! CHECK:                   %[[VAL_84:.*]] = fir.call @_QPpure_real_func(
+! CHECK:                   hlfir.yield %[[VAL_84]] : f32 cleanup {
+! CHECK:                     hlfir.destroy %[[VAL_79]] : !hlfir.expr<?xf32>
+! CHECK:                     hlfir.destroy %[[VAL_71]] : !hlfir.expr<?xf32>
+! CHECK:                   }
+! CHECK:                 }
+! CHECK:                 %[[VAL_85:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:                     arith.addf
+! CHECK:                 }
+! CHECK-NOT: hlfir.exactly_once
+! CHECK:                 %[[VAL_104:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:                 ^bb0(%[[VAL_105:.*]]: index):
+! CHECK-NOT: hlfir.exactly_once
+! CHECK:                   fir.call @_QPpure_elem_func
+! CHECK:                 }
+! CHECK:                 %[[VAL_108:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+! CHECK:                   arith.addf
+! CHECK:                 }
+! CHECK:                 hlfir.yield %[[VAL_108]] : !hlfir.expr<?xf32> cleanup {
+! CHECK:                   hlfir.destroy %[[VAL_108]] : !hlfir.expr<?xf32>
+! CHECK:                   hlfir.destroy %[[VAL_104]] : !hlfir.expr<?xf32>
+! CHECK:                   hlfir.destroy %[[VAL_85]] : !hlfir.expr<?xf32>
+! CHECK:                 }
+! CHECK:               } to {
+! CHECK:                 hlfir.designate
+! CHECK:                 hlfir.yield %{{.*}} : !fir.box<!fir.array<?xf32>>
+! CHECK:               }
+! CHECK:               hlfir.elsewhere mask {
+! CHECK:                 %[[VAL_129:.*]] = hlfir.exactly_once : !hlfir.expr<100x!fir.logical<4>> {
+! CHECK:                   %[[VAL_139:.*]] = fir.call @llvm.stacksave.p0() fastmath<contract> : () -> !fir.ref<i8>
+! CHECK:                   %[[VAL_141:.*]] = fir.call @_QPpure_logical_func2() fastmath<contract> : () -> !fir.array<100x!fir.logical<4>>
+! CHECK:                   hlfir.yield %{{.*}} : !hlfir.expr<100x!fir.logical<4>> cleanup {
+! CHECK:                     fir.call @llvm.stackrestore.p0(%[[VAL_139]]) fastmath<contract> : (!fir.ref<i8>) -> ()
+! CHECK:                   }
+! CHECK:                 }
+! CHECK:                 hlfir.yield %[[VAL_129]] : !hlfir.expr<100x!fir.logical<4>>
+! CHECK:               } do {
+! CHECK:                 hlfir.region_assign {
+! CHECK:                   hlfir.designate
+! CHECK:                   hlfir.yield %{{.*}} : !fir.box<!fir.array<?xf32>>
+! CHECK:                 } to {
+! CHECK:                   %[[VAL_165:.*]] = hlfir.exactly_once : i32 {
+! CHECK:                     %[[VAL_166:.*]] = fir.call @_QPpure_ifoo() fastmath<contract> : () -> i32
+! CHECK:                     hlfir.yield %[[VAL_166]] : i32
+! CHECK:                   }
+! CHECK:                   hlfir.designate
+! CHECK:                   hlfir.yield %{{.*}} : !fir.box<!fir.array<?xf32>>
+! CHECK:                 }
+! CHECK:               }
+! CHECK:             }
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
-- 
GitLab


From b1e99a699db02f3a61d5b66f5d6dd68bae3b9a69 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 14 May 2024 12:43:47 +0100
Subject: [PATCH 0722/1206] [LV] Drop redundant comment from createEdgeMask
 (NFC).

Follow-up to remove a redundant comment post-commit
https://github.com/llvm/llvm-project/pull/91897
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9353666e417c..f2a541f5167a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8012,9 +8012,6 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) {
     EdgeMask = Builder.createNot(EdgeMask, BI->getDebugLoc());
 
   if (SrcMask) { // Otherwise block in-mask is all-one, no need to AND.
-    // Use LogicalAnd as it does not propagate poison, i.e. does not introduce
-    // new UB if SrcMask is false and EdgeMask is poison. Using 'and' here
-    // introduces undefined behavior.
     // The bitwise 'And' of SrcMask and EdgeMask introduces new UB if SrcMask
     // is false and EdgeMask is poison. Avoid that by using 'LogicalAnd'
     // instead which generates 'select i1 SrcMask, i1 EdgeMask, i1 false'.
-- 
GitLab


From 292b300c5131e54b9977305bb4aca9a03e1b4fed Mon Sep 17 00:00:00 2001
From: Guillaume Chatelet <gchatelet@google.com>
Date: Tue, 14 May 2024 13:55:24 +0200
Subject: [PATCH 0723/1206] [libc][bug] Fix out of bound write in memcpy w/
 software prefetching (#90591)

This patch adds tests for `memcpy` and `memset` making sure that we
don't access buffers out of bounds. It relies on POSIX `mmap` /
`mprotect` and works only when FULL_BUILD_MODE is disabled.

The bug showed up while enabling software prefetching.
`loop_and_tail_offset` is always running at least one iteration but in
some configurations loop unrolled prefetching was actually needing only
the tail operation and no loop iterations at all.
---
 .../memory_utils/x86_64/inline_memcpy.h       | 16 ++-
 libc/test/src/string/memcpy_test.cpp          | 41 ++++++++
 .../src/string/memory_utils/protected_pages.h | 99 +++++++++++++++++++
 libc/test/src/string/memset_test.cpp          | 31 ++++++
 4 files changed, 185 insertions(+), 2 deletions(-)
 create mode 100644 libc/test/src/string/memory_utils/protected_pages.h

diff --git a/libc/src/string/memory_utils/x86_64/inline_memcpy.h b/libc/src/string/memory_utils/x86_64/inline_memcpy.h
index ae61b1235bd0..507b3e25199e 100644
--- a/libc/src/string/memory_utils/x86_64/inline_memcpy.h
+++ b/libc/src/string/memory_utils/x86_64/inline_memcpy.h
@@ -107,7 +107,13 @@ inline_memcpy_x86_sse2_ge64_sw_prefetching(Ptr __restrict dst,
       offset += K_THREE_CACHELINES;
     }
   }
-  return builtin::Memcpy<32>::loop_and_tail_offset(dst, src, count, offset);
+  // We don't use 'loop_and_tail_offset' because it assumes at least one
+  // iteration of the loop.
+  while (offset + 32 <= count) {
+    builtin::Memcpy<32>::block_offset(dst, src, offset);
+    offset += 32;
+  }
+  return builtin::Memcpy<32>::tail(dst, src, count);
 }
 
 [[maybe_unused]] LIBC_INLINE void
@@ -139,7 +145,13 @@ inline_memcpy_x86_avx_ge64_sw_prefetching(Ptr __restrict dst,
     builtin::Memcpy<K_THREE_CACHELINES>::block_offset(dst, src, offset);
     offset += K_THREE_CACHELINES;
   }
-  return builtin::Memcpy<64>::loop_and_tail_offset(dst, src, count, offset);
+  // We don't use 'loop_and_tail_offset' because it assumes at least one
+  // iteration of the loop.
+  while (offset + 64 <= count) {
+    builtin::Memcpy<64>::block_offset(dst, src, offset);
+    offset += 64;
+  }
+  return builtin::Memcpy<64>::tail(dst, src, count);
 }
 
 [[maybe_unused]] LIBC_INLINE void
diff --git a/libc/test/src/string/memcpy_test.cpp b/libc/test/src/string/memcpy_test.cpp
index b05d4202ea31..1ac963114844 100644
--- a/libc/test/src/string/memcpy_test.cpp
+++ b/libc/test/src/string/memcpy_test.cpp
@@ -7,9 +7,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "memory_utils/memory_check_utils.h"
+#include "src/__support/macros/properties/os.h" // LIBC_TARGET_OS_IS_LINUX
 #include "src/string/memcpy.h"
 #include "test/UnitTest/Test.h"
 
+#if !defined(LIBC_FULL_BUILD) && defined(LIBC_TARGET_OS_IS_LINUX)
+#include "memory_utils/protected_pages.h"
+#endif // !defined(LIBC_FULL_BUILD) && defined(LIBC_TARGET_OS_IS_LINUX)
+
 namespace LIBC_NAMESPACE {
 
 // Adapt CheckMemcpy signature to memcpy.
@@ -30,4 +35,40 @@ TEST(LlvmLibcMemcpyTest, SizeSweep) {
   }
 }
 
+#if !defined(LIBC_FULL_BUILD) && defined(LIBC_TARGET_OS_IS_LINUX)
+
+TEST(LlvmLibcMemcpyTest, CheckAccess) {
+  static constexpr size_t MAX_SIZE = 1024;
+  LIBC_ASSERT(MAX_SIZE < GetPageSize());
+  ProtectedPages pages;
+  const Page write_buffer = pages.GetPageA().WithAccess(PROT_WRITE);
+  const Page read_buffer = [&]() {
+    // We fetch page B in write mode.
+    auto page = pages.GetPageB().WithAccess(PROT_WRITE);
+    // And fill it with random numbers.
+    for (size_t i = 0; i < page.page_size; ++i)
+      page.page_ptr[i] = rand();
+    // Then return it in read mode.
+    return page.WithAccess(PROT_READ);
+  }();
+  for (size_t size = 0; size < MAX_SIZE; ++size) {
+    // We cross-check the function with two sources and two destinations.
+    //  - The first of them (bottom) is always page aligned and faults when
+    //    accessing bytes before it.
+    //  - The second one (top) is not necessarily aligned and faults when
+    //    accessing bytes after it.
+    const uint8_t *sources[2] = {read_buffer.bottom(size),
+                                 read_buffer.top(size)};
+    uint8_t *destinations[2] = {write_buffer.bottom(size),
+                                write_buffer.top(size)};
+    for (const uint8_t *src : sources) {
+      for (uint8_t *dst : destinations) {
+        LIBC_NAMESPACE::memcpy(dst, src, size);
+      }
+    }
+  }
+}
+
+#endif // !defined(LIBC_FULL_BUILD) && defined(LIBC_TARGET_OS_IS_LINUX)
+
 } // namespace LIBC_NAMESPACE
diff --git a/libc/test/src/string/memory_utils/protected_pages.h b/libc/test/src/string/memory_utils/protected_pages.h
new file mode 100644
index 000000000000..50cfbaa37937
--- /dev/null
+++ b/libc/test/src/string/memory_utils/protected_pages.h
@@ -0,0 +1,99 @@
+//===-- protected_pages.h -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This file provides protected pages that fault when accessing prior or past
+// it. This is useful to check memory functions that must not access outside of
+// the provided size limited buffer.
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBC_TEST_SRC_STRING_MEMORY_UTILS_PROTECTED_PAGES_H
+#define LIBC_TEST_SRC_STRING_MEMORY_UTILS_PROTECTED_PAGES_H
+
+#include "src/__support/macros/properties/os.h" // LIBC_TARGET_OS_IS_LINUX
+#if defined(LIBC_FULL_BUILD) || !defined(LIBC_TARGET_OS_IS_LINUX)
+#error "Protected pages requires mmap and cannot be used in full build mode."
+#endif // defined(LIBC_FULL_BUILD) || !defined(LIBC_TARGET_OS_IS_LINUX)
+
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include <stddef.h>                          // size_t
+#include <stdint.h>                          // uint8_t
+#include <sys/mman.h>                        // mmap, munmap
+#include <unistd.h>                          // sysconf, _SC_PAGESIZE
+
+// Returns mmap page size.
+LIBC_INLINE size_t GetPageSize() {
+  static const size_t PAGE_SIZE = sysconf(_SC_PAGESIZE);
+  return PAGE_SIZE;
+}
+
+// Represents a page of memory whose access can be configured throught the
+// 'WithAccess' function. Accessing data above or below this page will trap as
+// it is sandwiched between two pages with no read / write access.
+struct Page {
+  // Returns an aligned pointer that can be accessed up to page_size. Accessing
+  // data at ptr[-1] will fault.
+  LIBC_INLINE uint8_t *bottom(size_t size) const {
+    if (size >= page_size)
+      __builtin_trap();
+    return page_ptr;
+  }
+  // Returns a pointer to a buffer that can be accessed up to size. Accessing
+  // data at ptr[size] will trap.
+  LIBC_INLINE uint8_t *top(size_t size) const {
+    return page_ptr + page_size - size;
+  }
+
+  // protection is one of PROT_READ / PROT_WRITE.
+  LIBC_INLINE Page &WithAccess(int protection) {
+    if (mprotect(page_ptr, page_size, protection) != 0)
+      __builtin_trap();
+    return *this;
+  }
+
+  const size_t page_size;
+  uint8_t *const page_ptr;
+};
+
+// Allocates 5 consecutive pages that will trap if accessed.
+// | page layout | access | page name |
+// |-------------|--------|:---------:|
+// | 0           | trap   |           |
+// | 1           | custom |     A     |
+// | 2           | trap   |           |
+// | 3           | custom |     B     |
+// | 4           | trap   |           |
+//
+// The pages A and B can be retrieved as with 'GetPageA' / 'GetPageB' and their
+// accesses can be customized through the 'WithAccess' function.
+struct ProtectedPages {
+  static constexpr size_t PAGES = 5;
+
+  ProtectedPages()
+      : page_size(GetPageSize()),
+        ptr(mmap(/*address*/ nullptr, /*length*/ PAGES * page_size,
+                 /*protection*/ PROT_NONE,
+                 /*flags*/ MAP_PRIVATE | MAP_ANONYMOUS, /*fd*/ -1,
+                 /*offset*/ 0)) {
+    if (reinterpret_cast<intptr_t>(ptr) == -1)
+      __builtin_trap();
+  }
+  ~ProtectedPages() { munmap(ptr, PAGES * page_size); }
+
+  LIBC_INLINE Page GetPageA() const { return Page{page_size, page<1>()}; }
+  LIBC_INLINE Page GetPageB() const { return Page{page_size, page<3>()}; }
+
+private:
+  template <size_t index> LIBC_INLINE uint8_t *page() const {
+    static_assert(index < PAGES);
+    return static_cast<uint8_t *>(ptr) + (index * page_size);
+  }
+
+  const size_t page_size;
+  void *const ptr = nullptr;
+};
+
+#endif // LIBC_TEST_SRC_STRING_MEMORY_UTILS_PROTECTED_PAGES_H
diff --git a/libc/test/src/string/memset_test.cpp b/libc/test/src/string/memset_test.cpp
index 3a54498498f6..d78b579a0edb 100644
--- a/libc/test/src/string/memset_test.cpp
+++ b/libc/test/src/string/memset_test.cpp
@@ -7,9 +7,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "memory_utils/memory_check_utils.h"
+#include "src/__support/macros/properties/os.h" // LIBC_TARGET_OS_IS_LINUX
 #include "src/string/memset.h"
 #include "test/UnitTest/Test.h"
 
+#if !defined(LIBC_FULL_BUILD) && defined(LIBC_TARGET_OS_IS_LINUX)
+#include "memory_utils/protected_pages.h"
+#endif // !defined(LIBC_FULL_BUILD) && defined(LIBC_TARGET_OS_IS_LINUX)
+
 namespace LIBC_NAMESPACE {
 
 // Adapt CheckMemset signature to memset.
@@ -27,4 +32,30 @@ TEST(LlvmLibcMemsetTest, SizeSweep) {
   }
 }
 
+#if !defined(LIBC_FULL_BUILD) && defined(LIBC_TARGET_OS_IS_LINUX)
+
+TEST(LlvmLibcMemsetTest, CheckAccess) {
+  static constexpr size_t MAX_SIZE = 1024;
+  LIBC_ASSERT(MAX_SIZE < GetPageSize());
+  ProtectedPages pages;
+  const Page write_buffer = pages.GetPageA().WithAccess(PROT_WRITE);
+  const cpp::array<int, 2> fill_chars = {0, 0x7F};
+  for (int fill_char : fill_chars) {
+    for (size_t size = 0; size < MAX_SIZE; ++size) {
+      // We cross-check the function with two destinations.
+      // - The first of them (bottom) is always page aligned and faults when
+      //   accessing bytes before it.
+      // - The second one (top) is not necessarily aligned and faults when
+      //   accessing bytes after it.
+      uint8_t *destinations[2] = {write_buffer.bottom(size),
+                                  write_buffer.top(size)};
+      for (uint8_t *dst : destinations) {
+        LIBC_NAMESPACE::memset(dst, fill_char, size);
+      }
+    }
+  }
+}
+
+#endif // !defined(LIBC_FULL_BUILD) && defined(LIBC_TARGET_OS_IS_LINUX)
+
 } // namespace LIBC_NAMESPACE
-- 
GitLab


From 9d4f7f44b64d87d1068859906f43b7ce03a7388b Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Tue, 14 May 2024 19:39:47 +0800
Subject: [PATCH 0724/1206] [test][LoongArch] Add -mattr=+d option. NFC

Because most of tests assume target-abi=`lp64d`, adding the
corresponding feature is reasonable.

rg -l loongarch -g '!*.s' | xargs sed -i '/mtriple=loongarch/ {/-mattr=/!{/target-abi/! s/mtriple=loongarch.. /&-mattr=+d /}}'
---
 llvm/test/CodeGen/LoongArch/O0-pipeline.ll     |  4 ++--
 llvm/test/CodeGen/LoongArch/addrspacecast.ll   |  4 ++--
 llvm/test/CodeGen/LoongArch/alloca.ll          |  4 ++--
 llvm/test/CodeGen/LoongArch/alsl.ll            |  4 ++--
 llvm/test/CodeGen/LoongArch/analyze-branch.ll  |  2 +-
 llvm/test/CodeGen/LoongArch/andn-icmp.ll       |  4 ++--
 .../LoongArch/atomicrmw-uinc-udec-wrap.ll      |  2 +-
 llvm/test/CodeGen/LoongArch/bitreverse.ll      |  4 ++--
 llvm/test/CodeGen/LoongArch/block-address.ll   |  4 ++--
 .../CodeGen/LoongArch/blockaddress-symbol.ll   |  8 ++++----
 llvm/test/CodeGen/LoongArch/bnez-beqz.ll       |  4 ++--
 .../LoongArch/branch-relaxation-spill-32.ll    |  4 ++--
 .../LoongArch/branch-relaxation-spill-64.ll    |  4 ++--
 .../CodeGen/LoongArch/branch-relaxation.ll     |  8 ++++----
 llvm/test/CodeGen/LoongArch/bstrins_d.ll       |  2 +-
 llvm/test/CodeGen/LoongArch/bstrins_w.ll       |  2 +-
 llvm/test/CodeGen/LoongArch/bstrpick_d.ll      |  2 +-
 llvm/test/CodeGen/LoongArch/bstrpick_w.ll      |  2 +-
 .../test/CodeGen/LoongArch/bswap-bitreverse.ll |  4 ++--
 llvm/test/CodeGen/LoongArch/bswap.ll           |  4 ++--
 llvm/test/CodeGen/LoongArch/bytepick.ll        |  4 ++--
 llvm/test/CodeGen/LoongArch/code-models.ll     |  6 +++---
 .../test/CodeGen/LoongArch/cpu-name-generic.ll |  8 ++++----
 llvm/test/CodeGen/LoongArch/cpus.ll            |  6 +++---
 llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll |  4 ++--
 .../duplicate-returns-for-tailcall.ll          |  2 +-
 llvm/test/CodeGen/LoongArch/dwarf-eh.ll        |  8 ++++----
 llvm/test/CodeGen/LoongArch/e_flags.ll         |  4 ++--
 llvm/test/CodeGen/LoongArch/eh-dwarf-cfa.ll    |  4 ++--
 .../CodeGen/LoongArch/emergency-spill-slot.ll  |  2 +-
 .../LoongArch/exception-pointer-register.ll    |  4 ++--
 llvm/test/CodeGen/LoongArch/expand-call.ll     |  4 ++--
 llvm/test/CodeGen/LoongArch/frame.ll           |  2 +-
 .../CodeGen/LoongArch/frameaddr-returnaddr.ll  |  4 ++--
 llvm/test/CodeGen/LoongArch/gep-imm.ll         |  2 +-
 .../CodeGen/LoongArch/get-reg-error-la32.ll    |  2 +-
 .../CodeGen/LoongArch/get-reg-error-la64.ll    |  2 +-
 llvm/test/CodeGen/LoongArch/get-reg.ll         |  2 +-
 .../CodeGen/LoongArch/get-setcc-result-type.ll |  2 +-
 llvm/test/CodeGen/LoongArch/global-address.ll  | 12 ++++++------
 .../LoongArch/global-variable-code-model.ll    |  2 +-
 llvm/test/CodeGen/LoongArch/imm.ll             |  2 +-
 .../LoongArch/inline-asm-constraint-ZB.ll      |  4 ++--
 .../LoongArch/inline-asm-constraint-ZC.ll      |  4 ++--
 .../LoongArch/inline-asm-constraint-k.ll       |  4 ++--
 .../LoongArch/inline-asm-constraint-m.ll       |  4 ++--
 .../CodeGen/LoongArch/inline-asm-constraint.ll |  4 ++--
 .../LoongArch/inline-asm-operand-modifiers.ll  |  4 ++--
 .../LoongArch/inline-asm-reg-names-error.ll    |  4 ++--
 .../CodeGen/LoongArch/inline-asm-reg-names.ll  |  4 ++--
 .../LoongArch/intrinsic-csr-side-effects.ll    |  4 ++--
 .../LoongArch/intrinsic-iocsr-side-effects.ll  |  2 +-
 .../CodeGen/LoongArch/intrinsic-la32-error.ll  |  2 +-
 llvm/test/CodeGen/LoongArch/intrinsic-la32.ll  |  2 +-
 .../CodeGen/LoongArch/intrinsic-la64-error.ll  |  2 +-
 llvm/test/CodeGen/LoongArch/intrinsic-la64.ll  |  2 +-
 .../test/CodeGen/LoongArch/intrinsic-memcpy.ll |  2 +-
 .../LoongArch/intrinsic-not-constant-error.ll  |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/add.ll    |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/and.ll    |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/ashr.ll   |  4 ++--
 .../LoongArch/ir-instruction/atomic-cmpxchg.ll |  2 +-
 .../ir-instruction/atomicrmw-minmax.ll         |  2 +-
 .../LoongArch/ir-instruction/atomicrmw.ll      |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/br.ll     |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/call.ll   |  4 ++--
 .../ir-instruction/fence-singlethread.ll       |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/fence.ll  |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/icmp.ll   |  4 ++--
 .../LoongArch/ir-instruction/indirectbr.ll     |  2 +-
 .../ir-instruction/load-store-atomic.ll        |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/lshr.ll   |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/mul.ll    |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/or.ll     |  4 ++--
 .../ir-instruction/sdiv-udiv-srem-urem.ll      |  8 ++++----
 .../ir-instruction/select-bare-int.ll          |  4 ++--
 .../LoongArch/ir-instruction/select-icc-int.ll |  4 ++--
 .../ir-instruction/sext-zext-trunc.ll          |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/shl.ll    |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/sub.ll    |  4 ++--
 .../CodeGen/LoongArch/ir-instruction/xor.ll    |  4 ++--
 llvm/test/CodeGen/LoongArch/jump-table.ll      |  8 ++++----
 llvm/test/CodeGen/LoongArch/ldptr.ll           |  4 ++--
 llvm/test/CodeGen/LoongArch/ldx-stx-sp-2.ll    |  4 ++--
 llvm/test/CodeGen/LoongArch/ldx-stx-sp-3.ll    |  4 ++--
 llvm/test/CodeGen/LoongArch/legalicmpimm.ll    |  4 ++--
 .../CodeGen/LoongArch/load-store-offset.ll     |  4 ++--
 llvm/test/CodeGen/LoongArch/memcmp.ll          |  2 +-
 .../test/CodeGen/LoongArch/mir-target-flags.ll |  4 ++--
 llvm/test/CodeGen/LoongArch/nomerge.ll         |  2 +-
 llvm/test/CodeGen/LoongArch/not.ll             |  4 ++--
 .../CodeGen/LoongArch/numeric-reg-names.ll     |  4 ++--
 llvm/test/CodeGen/LoongArch/opt-pipeline.ll    | 12 ++++++------
 .../LoongArch/patchable-function-entry.ll      |  4 ++--
 llvm/test/CodeGen/LoongArch/prefer-w-inst.ll   |  4 ++--
 .../CodeGen/LoongArch/preferred-alignments.ll  |  4 ++--
 .../LoongArch/psabi-restricted-scheduling.ll   |  8 ++++----
 .../register-coalescer-crash-pr79718.mir       |  2 +-
 .../test/CodeGen/LoongArch/returnaddr-error.ll |  2 +-
 llvm/test/CodeGen/LoongArch/rotl-rotr.ll       |  4 ++--
 llvm/test/CodeGen/LoongArch/select-const.ll    |  4 ++--
 .../CodeGen/LoongArch/select-to-shiftand.ll    |  4 ++--
 .../LoongArch/sext-cheaper-than-zext.ll        |  2 +-
 .../CodeGen/LoongArch/shift-masked-shamt.ll    |  4 ++--
 llvm/test/CodeGen/LoongArch/shrinkwrap.ll      |  4 ++--
 .../CodeGen/LoongArch/smul-with-overflow.ll    |  4 ++--
 .../CodeGen/LoongArch/spill-ra-without-kill.ll |  2 +-
 llvm/test/CodeGen/LoongArch/split-sp-adjust.ll |  2 +-
 ...-realignment-with-variable-sized-objects.ll |  4 ++--
 .../CodeGen/LoongArch/stack-realignment.ll     |  4 ++--
 llvm/test/CodeGen/LoongArch/stptr.ll           |  4 ++--
 llvm/test/CodeGen/LoongArch/tail-calls.ll      |  2 +-
 .../CodeGen/LoongArch/test_bl_fixupkind.mir    |  2 +-
 llvm/test/CodeGen/LoongArch/thread-pointer.ll  |  4 ++--
 llvm/test/CodeGen/LoongArch/tls-models.ll      | 18 +++++++++---------
 llvm/test/CodeGen/LoongArch/trap.ll            |  4 ++--
 .../test/CodeGen/LoongArch/unaligned-access.ll |  4 ++--
 .../xray-attribute-instrumentation.ll          |  4 ++--
 .../LoongArch/zext-with-load-is-free.ll        |  4 ++--
 .../LoongArch/load-store-atomic.ll             |  4 ++--
 .../CodeGenPrepare/LoongArch/splitgep.ll       |  2 +-
 .../LoopDataPrefetch/LoongArch/basic.ll        |  2 +-
 122 files changed, 239 insertions(+), 239 deletions(-)

diff --git a/llvm/test/CodeGen/LoongArch/O0-pipeline.ll b/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
index 84d235d78eb9..38c3291b6367 100644
--- a/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
+++ b/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
@@ -1,8 +1,8 @@
 ;; When EXPENSIVE_CHECKS are enabled, the machine verifier appears between each
 ;; pass. Ignore it with 'grep -v'.
-; RUN: llc --mtriple=loongarch32 -O0 --debug-pass=Structure %s -o /dev/null 2>&1 | \
+; RUN: llc --mtriple=loongarch32 -mattr=+d -O0 --debug-pass=Structure %s -o /dev/null 2>&1 | \
 ; RUN:   grep -v "Verify generated machine code" | FileCheck %s
-; RUN: llc --mtriple=loongarch64 -O0 --debug-pass=Structure %s -o /dev/null 2>&1 | \
+; RUN: llc --mtriple=loongarch64 -mattr=+d -O0 --debug-pass=Structure %s -o /dev/null 2>&1 | \
 ; RUN:   grep -v "Verify generated machine code" | FileCheck %s
 
 ; REQUIRES: asserts
diff --git a/llvm/test/CodeGen/LoongArch/addrspacecast.ll b/llvm/test/CodeGen/LoongArch/addrspacecast.ll
index 7875562331be..1ca41705f974 100644
--- a/llvm/test/CodeGen/LoongArch/addrspacecast.ll
+++ b/llvm/test/CodeGen/LoongArch/addrspacecast.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64
 
 define void @cast0(ptr addrspace(1) %ptr) {
 ; LA32-LABEL: cast0:
diff --git a/llvm/test/CodeGen/LoongArch/alloca.ll b/llvm/test/CodeGen/LoongArch/alloca.ll
index 75a05689e417..d298beaaa766 100644
--- a/llvm/test/CodeGen/LoongArch/alloca.ll
+++ b/llvm/test/CodeGen/LoongArch/alloca.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
 
 declare void @notdead(ptr)
diff --git a/llvm/test/CodeGen/LoongArch/alsl.ll b/llvm/test/CodeGen/LoongArch/alsl.ll
index 177e37de0952..6db9a179d65d 100644
--- a/llvm/test/CodeGen/LoongArch/alsl.ll
+++ b/llvm/test/CodeGen/LoongArch/alsl.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define i8 @alsl_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: alsl_i8:
diff --git a/llvm/test/CodeGen/LoongArch/analyze-branch.ll b/llvm/test/CodeGen/LoongArch/analyze-branch.ll
index fb89964af838..d15229a8c9e1 100644
--- a/llvm/test/CodeGen/LoongArch/analyze-branch.ll
+++ b/llvm/test/CodeGen/LoongArch/analyze-branch.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 ;; This test checks that LLVM can do basic stripping and reapplying of branches
 ;; to basic blocks.
diff --git a/llvm/test/CodeGen/LoongArch/andn-icmp.ll b/llvm/test/CodeGen/LoongArch/andn-icmp.ll
index ff6935e2e23c..46bae6a9b70c 100644
--- a/llvm/test/CodeGen/LoongArch/andn-icmp.ll
+++ b/llvm/test/CodeGen/LoongArch/andn-icmp.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define i1 @andn_icmp_eq_i8(i8 signext %a, i8 signext %b) nounwind {
 ; LA32-LABEL: andn_icmp_eq_i8:
diff --git a/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
index b95c2e24737a..5ca6d8699135 100644
--- a/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck --check-prefix=LA64 %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck --check-prefix=LA64 %s
 
 define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 ; LA64-LABEL: atomicrmw_uinc_wrap_i8:
diff --git a/llvm/test/CodeGen/LoongArch/bitreverse.ll b/llvm/test/CodeGen/LoongArch/bitreverse.ll
index fcf523aa3c88..78d5c7e4a797 100644
--- a/llvm/test/CodeGen/LoongArch/bitreverse.ll
+++ b/llvm/test/CodeGen/LoongArch/bitreverse.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
 
 declare i7 @llvm.bitreverse.i7(i7)
diff --git a/llvm/test/CodeGen/LoongArch/block-address.ll b/llvm/test/CodeGen/LoongArch/block-address.ll
index 63d310dd9bea..eaba81f3563d 100644
--- a/llvm/test/CodeGen/LoongArch/block-address.ll
+++ b/llvm/test/CodeGen/LoongArch/block-address.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 @addr = dso_local global ptr null
 
diff --git a/llvm/test/CodeGen/LoongArch/blockaddress-symbol.ll b/llvm/test/CodeGen/LoongArch/blockaddress-symbol.ll
index d07092230a4d..88864087ee4b 100644
--- a/llvm/test/CodeGen/LoongArch/blockaddress-symbol.ll
+++ b/llvm/test/CodeGen/LoongArch/blockaddress-symbol.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
-; RUN: llc --mtriple=loongarch32 --no-integrated-as < %s | FileCheck %s
-; RUN: llc --mtriple=loongarch64 --no-integrated-as < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d --no-integrated-as < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d --no-integrated-as < %s | FileCheck %s
 
 ;; This regression test is for ensuring the AsmParser does not use the
 ;; getOrCreateSymbol interface to create blockaddress symbols.
diff --git a/llvm/test/CodeGen/LoongArch/bnez-beqz.ll b/llvm/test/CodeGen/LoongArch/bnez-beqz.ll
index d1652c73c25e..b2d7f3fe4173 100644
--- a/llvm/test/CodeGen/LoongArch/bnez-beqz.ll
+++ b/llvm/test/CodeGen/LoongArch/bnez-beqz.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 declare void @bar()
 
diff --git a/llvm/test/CodeGen/LoongArch/branch-relaxation-spill-32.ll b/llvm/test/CodeGen/LoongArch/branch-relaxation-spill-32.ll
index 1ed9386310eb..5c96d5c416b9 100644
--- a/llvm/test/CodeGen/LoongArch/branch-relaxation-spill-32.ll
+++ b/llvm/test/CodeGen/LoongArch/branch-relaxation-spill-32.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --filetype=obj --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --filetype=obj --verify-machineinstrs < %s \
 ; RUN:   -o /dev/null 2>&1
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s | FileCheck %s
 
 define void @relax_b28_spill() {
 ; CHECK-LABEL: relax_b28_spill:
diff --git a/llvm/test/CodeGen/LoongArch/branch-relaxation-spill-64.ll b/llvm/test/CodeGen/LoongArch/branch-relaxation-spill-64.ll
index 1c4ef48a9761..a161fe97da9b 100644
--- a/llvm/test/CodeGen/LoongArch/branch-relaxation-spill-64.ll
+++ b/llvm/test/CodeGen/LoongArch/branch-relaxation-spill-64.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 --filetype=obj --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --filetype=obj --verify-machineinstrs < %s \
 ; RUN:   -o /dev/null 2>&1
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s | FileCheck %s
 
 define void @relax_b28_spill() {
 ; CHECK-LABEL: relax_b28_spill:
diff --git a/llvm/test/CodeGen/LoongArch/branch-relaxation.ll b/llvm/test/CodeGen/LoongArch/branch-relaxation.ll
index 296f543e18d9..6037ef9337dc 100644
--- a/llvm/test/CodeGen/LoongArch/branch-relaxation.ll
+++ b/llvm/test/CodeGen/LoongArch/branch-relaxation.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --filetype=obj --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --filetype=obj --verify-machineinstrs < %s \
 ; RUN:   -o /dev/null 2>&1
-; RUN: llc --mtriple=loongarch64 --filetype=obj --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --filetype=obj --verify-machineinstrs < %s \
 ; RUN:   -o /dev/null 2>&1
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64
 
 define i32 @relax_b18(i32 signext %a, i32 signext %b) {
 ; LA32-LABEL: relax_b18:
diff --git a/llvm/test/CodeGen/LoongArch/bstrins_d.ll b/llvm/test/CodeGen/LoongArch/bstrins_d.ll
index fe1f6270f966..ba43b63af6d6 100644
--- a/llvm/test/CodeGen/LoongArch/bstrins_d.ll
+++ b/llvm/test/CodeGen/LoongArch/bstrins_d.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 ;; Test generation of the bstrins.d instruction.
 ;; There are 8 patterns that can be matched to bstrins.d. See performORCombine
diff --git a/llvm/test/CodeGen/LoongArch/bstrins_w.ll b/llvm/test/CodeGen/LoongArch/bstrins_w.ll
index e008caacad2a..92b98df21564 100644
--- a/llvm/test/CodeGen/LoongArch/bstrins_w.ll
+++ b/llvm/test/CodeGen/LoongArch/bstrins_w.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s
 
 ;; Test generation of the bstrins.w instruction.
 ;; There are 8 patterns that can be matched to bstrins.w. See performORCombine
diff --git a/llvm/test/CodeGen/LoongArch/bstrpick_d.ll b/llvm/test/CodeGen/LoongArch/bstrpick_d.ll
index e93c1391d463..e6fc385b610d 100644
--- a/llvm/test/CodeGen/LoongArch/bstrpick_d.ll
+++ b/llvm/test/CodeGen/LoongArch/bstrpick_d.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 define i64 @lshr40_and255(i64 %a) {
 ; CHECK-LABEL: lshr40_and255:
diff --git a/llvm/test/CodeGen/LoongArch/bstrpick_w.ll b/llvm/test/CodeGen/LoongArch/bstrpick_w.ll
index f9027e1fb32d..8d5c9d0df44c 100644
--- a/llvm/test/CodeGen/LoongArch/bstrpick_w.ll
+++ b/llvm/test/CodeGen/LoongArch/bstrpick_w.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s
 
 define i32 @lshr10_and255(i32 %a) {
 ; CHECK-LABEL: lshr10_and255:
diff --git a/llvm/test/CodeGen/LoongArch/bswap-bitreverse.ll b/llvm/test/CodeGen/LoongArch/bswap-bitreverse.ll
index 828fb933bf3c..c8f9596b9b0c 100644
--- a/llvm/test/CodeGen/LoongArch/bswap-bitreverse.ll
+++ b/llvm/test/CodeGen/LoongArch/bswap-bitreverse.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
 
 declare i16 @llvm.bitreverse.i16(i16)
diff --git a/llvm/test/CodeGen/LoongArch/bswap.ll b/llvm/test/CodeGen/LoongArch/bswap.ll
index 71095ab972e3..122dab7fb496 100644
--- a/llvm/test/CodeGen/LoongArch/bswap.ll
+++ b/llvm/test/CodeGen/LoongArch/bswap.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
 
 declare i16 @llvm.bswap.i16(i16)
diff --git a/llvm/test/CodeGen/LoongArch/bytepick.ll b/llvm/test/CodeGen/LoongArch/bytepick.ll
index 1a2cd48448ba..22a78bcd5611 100644
--- a/llvm/test/CodeGen/LoongArch/bytepick.ll
+++ b/llvm/test/CodeGen/LoongArch/bytepick.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
 
 ;; a=00112233 b=44556677
diff --git a/llvm/test/CodeGen/LoongArch/code-models.ll b/llvm/test/CodeGen/LoongArch/code-models.ll
index f93c31670928..4b2b72afaee1 100644
--- a/llvm/test/CodeGen/LoongArch/code-models.ll
+++ b/llvm/test/CodeGen/LoongArch/code-models.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 --code-model=small < %s | \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --code-model=small < %s | \
 ; RUN:    FileCheck --check-prefix=SMALL %s
-; RUN: llc --mtriple=loongarch64 --code-model=medium < %s | \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --code-model=medium < %s | \
 ; RUN:    FileCheck --check-prefix=MEDIUM %s
-; RUN: llc --mtriple=loongarch64 --code-model=large < %s | \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --code-model=large < %s | \
 ; RUN:    FileCheck --check-prefix=LARGE %s
 
 declare void @llvm.memset.p0.i64(ptr, i8, i64, i1)
diff --git a/llvm/test/CodeGen/LoongArch/cpu-name-generic.ll b/llvm/test/CodeGen/LoongArch/cpu-name-generic.ll
index 1129d9fcb254..7472c1cc4f57 100644
--- a/llvm/test/CodeGen/LoongArch/cpu-name-generic.ll
+++ b/llvm/test/CodeGen/LoongArch/cpu-name-generic.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --mcpu=generic < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --mcpu=generic < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch32 --mcpu=generic-la32 < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --mcpu=generic-la32 < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --mcpu=generic < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --mcpu=generic < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
-; RUN: llc --mtriple=loongarch64 --mcpu=generic-la64 < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --mcpu=generic-la64 < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
 
 ;; The CPU name "generic" should map to the corresponding concrete names
diff --git a/llvm/test/CodeGen/LoongArch/cpus.ll b/llvm/test/CodeGen/LoongArch/cpus.ll
index 35945ae4de71..41ff1be496e3 100644
--- a/llvm/test/CodeGen/LoongArch/cpus.ll
+++ b/llvm/test/CodeGen/LoongArch/cpus.ll
@@ -1,9 +1,9 @@
 ;; This tests that llc accepts all valid LoongArch CPUs.
 ;; Note the 'generic' names have been tested in cpu-name-generic.ll.
 
-; RUN: llc < %s --mtriple=loongarch64 --mcpu=loongarch64 2>&1 | FileCheck %s
-; RUN: llc < %s --mtriple=loongarch64 --mcpu=la464 2>&1 | FileCheck %s
-; RUN: llc < %s --mtriple=loongarch64 2>&1 | FileCheck %s
+; RUN: llc < %s --mtriple=loongarch64 -mattr=+d --mcpu=loongarch64 2>&1 | FileCheck %s
+; RUN: llc < %s --mtriple=loongarch64 -mattr=+d --mcpu=la464 2>&1 | FileCheck %s
+; RUN: llc < %s --mtriple=loongarch64 -mattr=+d 2>&1 | FileCheck %s
 
 ; CHECK-NOT: {{.*}} is not a recognized processor for this target
 
diff --git a/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll b/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll
index 9fa3f5076bb2..f17cec231f32 100644
--- a/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll
+++ b/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 declare i8 @llvm.ctlz.i8(i8, i1)
 declare i16 @llvm.ctlz.i16(i16, i1)
diff --git a/llvm/test/CodeGen/LoongArch/duplicate-returns-for-tailcall.ll b/llvm/test/CodeGen/LoongArch/duplicate-returns-for-tailcall.ll
index 80e55ef9e21f..ac9d885c7ce0 100644
--- a/llvm/test/CodeGen/LoongArch/duplicate-returns-for-tailcall.ll
+++ b/llvm/test/CodeGen/LoongArch/duplicate-returns-for-tailcall.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 ;; Perform tail call optimization for duplicate returns.
 declare i32 @test()
diff --git a/llvm/test/CodeGen/LoongArch/dwarf-eh.ll b/llvm/test/CodeGen/LoongArch/dwarf-eh.ll
index f4e347e07de5..cdeb6d81711d 100644
--- a/llvm/test/CodeGen/LoongArch/dwarf-eh.ll
+++ b/llvm/test/CodeGen/LoongArch/dwarf-eh.ll
@@ -1,7 +1,7 @@
-; RUN: llc --mtriple=loongarch32 --relocation-model=static < %s | FileCheck %s
-; RUN: llc --mtriple=loongarch32 --relocation-model=pic < %s | FileCheck %s
-; RUN: llc --mtriple=loongarch64 --relocation-model=static < %s | FileCheck %s
-; RUN: llc --mtriple=loongarch64 --relocation-model=pic < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d --relocation-model=static < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d --relocation-model=pic < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d --relocation-model=static < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d --relocation-model=pic < %s | FileCheck %s
 
 declare void @throw_exception()
 
diff --git a/llvm/test/CodeGen/LoongArch/e_flags.ll b/llvm/test/CodeGen/LoongArch/e_flags.ll
index c004d1f9cdf4..25f3285bdf32 100644
--- a/llvm/test/CodeGen/LoongArch/e_flags.ll
+++ b/llvm/test/CodeGen/LoongArch/e_flags.ll
@@ -1,4 +1,4 @@
-; RUN: llc --mtriple=loongarch32 --filetype=obj %s -o %t-la32
+; RUN: llc --mtriple=loongarch32 -mattr=+d --filetype=obj %s -o %t-la32
 ; RUN: llvm-readelf -h %t-la32 | FileCheck %s --check-prefixes=ILP32,ABI-D --match-full-lines
 
 ; RUN: llc --mtriple=loongarch32 --filetype=obj %s --target-abi=ilp32s -o %t-ilp32s
@@ -10,7 +10,7 @@
 ; RUN: llc --mtriple=loongarch32 --filetype=obj %s --target-abi=ilp32d -o %t-ilp32d
 ; RUN: llvm-readelf -h %t-ilp32d | FileCheck %s --check-prefixes=ILP32,ABI-D --match-full-lines
 
-; RUN: llc --mtriple=loongarch64 --filetype=obj %s -o %t-la64
+; RUN: llc --mtriple=loongarch64 -mattr=+d --filetype=obj %s -o %t-la64
 ; RUN: llvm-readelf -h %t-la64 | FileCheck %s --check-prefixes=LP64,ABI-D --match-full-lines
 
 ; RUN: llc --mtriple=loongarch64 --filetype=obj %s --target-abi=lp64s -o %t-lp64s
diff --git a/llvm/test/CodeGen/LoongArch/eh-dwarf-cfa.ll b/llvm/test/CodeGen/LoongArch/eh-dwarf-cfa.ll
index 796ada3a1a02..f00cf9491c08 100644
--- a/llvm/test/CodeGen/LoongArch/eh-dwarf-cfa.ll
+++ b/llvm/test/CodeGen/LoongArch/eh-dwarf-cfa.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck --check-prefix=LA32 %s
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck --check-prefix=LA64 %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck --check-prefix=LA32 %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck --check-prefix=LA64 %s
 
 define void @dwarf() {
 ; LA32-LABEL: dwarf:
diff --git a/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll b/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
index 4565c63f08d9..ccc5c703e71e 100644
--- a/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
+++ b/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 -O0 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d -O0 < %s | FileCheck %s
 
 @var = external global i32
 
diff --git a/llvm/test/CodeGen/LoongArch/exception-pointer-register.ll b/llvm/test/CodeGen/LoongArch/exception-pointer-register.ll
index 797c7e520f5b..530d97ff4bab 100644
--- a/llvm/test/CodeGen/LoongArch/exception-pointer-register.ll
+++ b/llvm/test/CodeGen/LoongArch/exception-pointer-register.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:    | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:    | FileCheck %s --check-prefix=LA64
 
 declare void @foo(ptr %p);
diff --git a/llvm/test/CodeGen/LoongArch/expand-call.ll b/llvm/test/CodeGen/LoongArch/expand-call.ll
index e0d179f92de6..8c21adbcbb55 100644
--- a/llvm/test/CodeGen/LoongArch/expand-call.ll
+++ b/llvm/test/CodeGen/LoongArch/expand-call.ll
@@ -1,6 +1,6 @@
-; RUN: llc --mtriple=loongarch64 --stop-before loongarch-prera-expand-pseudo \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --stop-before loongarch-prera-expand-pseudo \
 ; RUN:     --verify-machineinstrs < %s | FileCheck %s --check-prefix=NOEXPAND
-; RUN: llc --mtriple=loongarch64 --stop-before machine-opt-remark-emitter \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --stop-before machine-opt-remark-emitter \
 ; RUN:     --verify-machineinstrs < %s | FileCheck %s --check-prefix=EXPAND
 
 declare void @callee()
diff --git a/llvm/test/CodeGen/LoongArch/frame.ll b/llvm/test/CodeGen/LoongArch/frame.ll
index 8d3133316c43..ac5cb3c7e721 100644
--- a/llvm/test/CodeGen/LoongArch/frame.ll
+++ b/llvm/test/CodeGen/LoongArch/frame.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 %struct.key_t = type { i32, [16 x i8] }
 
diff --git a/llvm/test/CodeGen/LoongArch/frameaddr-returnaddr.ll b/llvm/test/CodeGen/LoongArch/frameaddr-returnaddr.ll
index 01c9173c2e98..128d6e5a1dac 100644
--- a/llvm/test/CodeGen/LoongArch/frameaddr-returnaddr.ll
+++ b/llvm/test/CodeGen/LoongArch/frameaddr-returnaddr.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 declare ptr @llvm.frameaddress(i32)
 declare ptr @llvm.returnaddress(i32)
diff --git a/llvm/test/CodeGen/LoongArch/gep-imm.ll b/llvm/test/CodeGen/LoongArch/gep-imm.ll
index c88d0b5a4543..567d3ea43ac9 100644
--- a/llvm/test/CodeGen/LoongArch/gep-imm.ll
+++ b/llvm/test/CodeGen/LoongArch/gep-imm.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 define void @test(ptr %sp, ptr %t, i32 %n) {
 ; CHECK-LABEL: test:
diff --git a/llvm/test/CodeGen/LoongArch/get-reg-error-la32.ll b/llvm/test/CodeGen/LoongArch/get-reg-error-la32.ll
index 7440bfe5c85a..58b533a8b4e2 100644
--- a/llvm/test/CodeGen/LoongArch/get-reg-error-la32.ll
+++ b/llvm/test/CodeGen/LoongArch/get-reg-error-la32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: not llc < %s --mtriple=loongarch32 2>&1 | FileCheck %s
+; RUN: not llc < %s --mtriple=loongarch32 -mattr=+d 2>&1 | FileCheck %s
 
 define i64 @read_sp() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/LoongArch/get-reg-error-la64.ll b/llvm/test/CodeGen/LoongArch/get-reg-error-la64.ll
index 9312aa9020ba..ba2e83420327 100644
--- a/llvm/test/CodeGen/LoongArch/get-reg-error-la64.ll
+++ b/llvm/test/CodeGen/LoongArch/get-reg-error-la64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: not llc < %s --mtriple=loongarch64 2>&1 | FileCheck %s
+; RUN: not llc < %s --mtriple=loongarch64 -mattr=+d 2>&1 | FileCheck %s
 
 define i32 @read_sp() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/LoongArch/get-reg.ll b/llvm/test/CodeGen/LoongArch/get-reg.ll
index 323030da9e7f..e6d1de3baeb7 100644
--- a/llvm/test/CodeGen/LoongArch/get-reg.ll
+++ b/llvm/test/CodeGen/LoongArch/get-reg.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s --mtriple=loongarch64 | FileCheck %s
+; RUN: llc < %s --mtriple=loongarch64 -mattr=+d | FileCheck %s
 
 define i64 @get_stack() nounwind {
 ; CHECK-LABEL: get_stack:
diff --git a/llvm/test/CodeGen/LoongArch/get-setcc-result-type.ll b/llvm/test/CodeGen/LoongArch/get-setcc-result-type.ll
index 5e4c8418b222..6cf9d7d75b99 100644
--- a/llvm/test/CodeGen/LoongArch/get-setcc-result-type.ll
+++ b/llvm/test/CodeGen/LoongArch/get-setcc-result-type.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s
 
 define void @getSetCCResultType(ptr %p) {
diff --git a/llvm/test/CodeGen/LoongArch/global-address.ll b/llvm/test/CodeGen/LoongArch/global-address.ll
index d32a17f488b1..0c8958b6ab33 100644
--- a/llvm/test/CodeGen/LoongArch/global-address.ll
+++ b/llvm/test/CodeGen/LoongArch/global-address.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --relocation-model=static < %s | FileCheck %s --check-prefix=LA32NOPIC
-; RUN: llc --mtriple=loongarch32 --relocation-model=pic < %s | FileCheck %s --check-prefix=LA32PIC
-; RUN: llc --mtriple=loongarch64 --relocation-model=static < %s | FileCheck %s --check-prefix=LA64NOPIC
-; RUN: llc --mtriple=loongarch64 --relocation-model=pic < %s | FileCheck %s --check-prefix=LA64PIC
-; RUN: llc --mtriple=loongarch64 --code-model=large --relocation-model=static < %s | FileCheck %s --check-prefix=LA64LARGENOPIC
-; RUN: llc --mtriple=loongarch64 --code-model=large --relocation-model=pic < %s | FileCheck %s --check-prefix=LA64LARGEPIC
+; RUN: llc --mtriple=loongarch32 -mattr=+d --relocation-model=static < %s | FileCheck %s --check-prefix=LA32NOPIC
+; RUN: llc --mtriple=loongarch32 -mattr=+d --relocation-model=pic < %s | FileCheck %s --check-prefix=LA32PIC
+; RUN: llc --mtriple=loongarch64 -mattr=+d --relocation-model=static < %s | FileCheck %s --check-prefix=LA64NOPIC
+; RUN: llc --mtriple=loongarch64 -mattr=+d --relocation-model=pic < %s | FileCheck %s --check-prefix=LA64PIC
+; RUN: llc --mtriple=loongarch64 -mattr=+d --code-model=large --relocation-model=static < %s | FileCheck %s --check-prefix=LA64LARGENOPIC
+; RUN: llc --mtriple=loongarch64 -mattr=+d --code-model=large --relocation-model=pic < %s | FileCheck %s --check-prefix=LA64LARGEPIC
 
 @g = dso_local global i32 zeroinitializer, align 4
 @G = global i32 zeroinitializer, align 4
diff --git a/llvm/test/CodeGen/LoongArch/global-variable-code-model.ll b/llvm/test/CodeGen/LoongArch/global-variable-code-model.ll
index aa4780834ac3..277b0b906139 100644
--- a/llvm/test/CodeGen/LoongArch/global-variable-code-model.ll
+++ b/llvm/test/CodeGen/LoongArch/global-variable-code-model.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 @a= external dso_local global i32, code_model "small", align 4
 
diff --git a/llvm/test/CodeGen/LoongArch/imm.ll b/llvm/test/CodeGen/LoongArch/imm.ll
index f8b7a61d6097..f84fddaec66b 100644
--- a/llvm/test/CodeGen/LoongArch/imm.ll
+++ b/llvm/test/CodeGen/LoongArch/imm.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 define i64 @imm0() {
 ; CHECK-LABEL: imm0:
diff --git a/llvm/test/CodeGen/LoongArch/inline-asm-constraint-ZB.ll b/llvm/test/CodeGen/LoongArch/inline-asm-constraint-ZB.ll
index 1a8f50abb658..526fd95da83f 100644
--- a/llvm/test/CodeGen/LoongArch/inline-asm-constraint-ZB.ll
+++ b/llvm/test/CodeGen/LoongArch/inline-asm-constraint-ZB.ll
@@ -1,6 +1,6 @@
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=ASM
-; RUN: llc --mtriple=loongarch64 --print-after-isel -o /dev/null 2>&1 < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --print-after-isel -o /dev/null 2>&1 < %s \
 ; RUN:   | FileCheck %s --check-prefix=MACHINE-INSTR
 
 ;; Note amswap.w is not available on loongarch32.
diff --git a/llvm/test/CodeGen/LoongArch/inline-asm-constraint-ZC.ll b/llvm/test/CodeGen/LoongArch/inline-asm-constraint-ZC.ll
index 9c053c4d2485..435235abed1a 100644
--- a/llvm/test/CodeGen/LoongArch/inline-asm-constraint-ZC.ll
+++ b/llvm/test/CodeGen/LoongArch/inline-asm-constraint-ZC.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64
 
 define i32 @ZC_offset_neg_32769(ptr %p) nounwind {
 ; LA32-LABEL: ZC_offset_neg_32769:
diff --git a/llvm/test/CodeGen/LoongArch/inline-asm-constraint-k.ll b/llvm/test/CodeGen/LoongArch/inline-asm-constraint-k.ll
index 5ffe4b48c3f5..dccf2957981c 100644
--- a/llvm/test/CodeGen/LoongArch/inline-asm-constraint-k.ll
+++ b/llvm/test/CodeGen/LoongArch/inline-asm-constraint-k.ll
@@ -1,6 +1,6 @@
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=ASM
-; RUN: llc --mtriple=loongarch64 --print-after-isel -o /dev/null 2>&1 < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --print-after-isel -o /dev/null 2>&1 < %s \
 ; RUN:   | FileCheck %s --check-prefix=MACHINE-INSTR
 
 define i64 @k_variable_offset(ptr %p, i64 %idx) nounwind {
diff --git a/llvm/test/CodeGen/LoongArch/inline-asm-constraint-m.ll b/llvm/test/CodeGen/LoongArch/inline-asm-constraint-m.ll
index b6d8893d8ac9..281d52c47b68 100644
--- a/llvm/test/CodeGen/LoongArch/inline-asm-constraint-m.ll
+++ b/llvm/test/CodeGen/LoongArch/inline-asm-constraint-m.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s | FileCheck %s --check-prefix=LA64
 
 define i32 @m_offset_neg_2049(ptr %p) nounwind {
 ; LA32-LABEL: m_offset_neg_2049:
diff --git a/llvm/test/CodeGen/LoongArch/inline-asm-constraint.ll b/llvm/test/CodeGen/LoongArch/inline-asm-constraint.ll
index 6ad52756b666..4bcc88be9739 100644
--- a/llvm/test/CodeGen/LoongArch/inline-asm-constraint.ll
+++ b/llvm/test/CodeGen/LoongArch/inline-asm-constraint.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs --no-integrated-as < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs --no-integrated-as < %s \
 ; RUN:   | FileCheck %s
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs --no-integrated-as < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs --no-integrated-as < %s \
 ; RUN:   | FileCheck %s
 
 @gi = external dso_local global i32, align 4
diff --git a/llvm/test/CodeGen/LoongArch/inline-asm-operand-modifiers.ll b/llvm/test/CodeGen/LoongArch/inline-asm-operand-modifiers.ll
index d3cf288bfd01..33651446c69f 100644
--- a/llvm/test/CodeGen/LoongArch/inline-asm-operand-modifiers.ll
+++ b/llvm/test/CodeGen/LoongArch/inline-asm-operand-modifiers.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s | FileCheck %s
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s | FileCheck %s
 
 define i32 @modifier_z_zero(i32 %a) nounwind {
 ; CHECK-LABEL: modifier_z_zero:
diff --git a/llvm/test/CodeGen/LoongArch/inline-asm-reg-names-error.ll b/llvm/test/CodeGen/LoongArch/inline-asm-reg-names-error.ll
index 56c335ffb3a6..c6c0ac61607a 100644
--- a/llvm/test/CodeGen/LoongArch/inline-asm-reg-names-error.ll
+++ b/llvm/test/CodeGen/LoongArch/inline-asm-reg-names-error.ll
@@ -1,5 +1,5 @@
-; RUN: not llc --mtriple=loongarch32 2>&1 < %s | FileCheck %s
-; RUN: not llc --mtriple=loongarch64 2>&1 < %s | FileCheck %s
+; RUN: not llc --mtriple=loongarch32 -mattr=+d 2>&1 < %s | FileCheck %s
+; RUN: not llc --mtriple=loongarch64 -mattr=+d 2>&1 < %s | FileCheck %s
 
 define i32 @non_exit_r32(i32 %a) nounwind {
 ; CHECK: error: couldn't allocate input reg for constraint '{$r32}'
diff --git a/llvm/test/CodeGen/LoongArch/inline-asm-reg-names.ll b/llvm/test/CodeGen/LoongArch/inline-asm-reg-names.ll
index 4bc16e6cc5fb..36ccc1bb0163 100644
--- a/llvm/test/CodeGen/LoongArch/inline-asm-reg-names.ll
+++ b/llvm/test/CodeGen/LoongArch/inline-asm-reg-names.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck --check-prefix=LA32 %s
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck --check-prefix=LA64 %s
 
 ;; These test that we can use architectural names ($r*) refer to registers in
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-csr-side-effects.ll b/llvm/test/CodeGen/LoongArch/intrinsic-csr-side-effects.ll
index e3e23e46b04b..d14483939fbd 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-csr-side-effects.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-csr-side-effects.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 declare i32 @llvm.loongarch.csrrd.w(i32 immarg) nounwind
 declare i32 @llvm.loongarch.csrwr.w(i32, i32 immarg) nounwind
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-iocsr-side-effects.ll b/llvm/test/CodeGen/LoongArch/intrinsic-iocsr-side-effects.ll
index ad78f7f53be1..e2a1f8a7ccd0 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-iocsr-side-effects.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-iocsr-side-effects.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 declare i32 @llvm.loongarch.iocsrrd.b(i32) nounwind
 declare void @llvm.loongarch.iocsrwr.b(i32, i32) nounwind
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-la32-error.ll b/llvm/test/CodeGen/LoongArch/intrinsic-la32-error.ll
index 5302ba558940..bdbdaec8a1a0 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-la32-error.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-la32-error.ll
@@ -1,4 +1,4 @@
-; RUN: not llc --mtriple=loongarch32 < %s 2>&1 | FileCheck %s
+; RUN: not llc --mtriple=loongarch32 -mattr=+d < %s 2>&1 | FileCheck %s
 
 declare void @llvm.loongarch.cacop.w(i32, i32, i32)
 declare i32 @llvm.loongarch.crc.w.b.w(i32, i32)
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-la32.ll b/llvm/test/CodeGen/LoongArch/intrinsic-la32.ll
index 37e0902625a2..56f5146bb985 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-la32.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-la32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s
 
 declare void @llvm.loongarch.cacop.w(i32, i32, i32)
 
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-la64-error.ll b/llvm/test/CodeGen/LoongArch/intrinsic-la64-error.ll
index 4716d401d9fd..0a24e03838fd 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-la64-error.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-la64-error.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: not llc --mtriple=loongarch64 < %s 2>&1 | FileCheck %s
+; RUN: not llc --mtriple=loongarch64 -mattr=+d < %s 2>&1 | FileCheck %s
 
 declare void @llvm.loongarch.cacop.w(i32, i32, i32)
 declare void @llvm.loongarch.cacop.d(i64, i64, i64)
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll b/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll
index f0ebd8508ad1..4a59e2af533e 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 declare void @llvm.loongarch.cacop.d(i64, i64, i64)
 declare i32 @llvm.loongarch.crc.w.b.w(i32, i32)
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
index 06ef4d2f6c15..622001db3295 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 %Box = type [6 x i64]
 
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-not-constant-error.ll b/llvm/test/CodeGen/LoongArch/intrinsic-not-constant-error.ll
index 9cb89670c293..9c6eb678588a 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-not-constant-error.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-not-constant-error.ll
@@ -1,5 +1,5 @@
-; RUN: not llc --mtriple=loongarch32 < %s 2>&1 | FileCheck %s
-; RUN: not llc --mtriple=loongarch64 < %s 2>&1 | FileCheck %s
+; RUN: not llc --mtriple=loongarch32 -mattr=+d < %s 2>&1 | FileCheck %s
+; RUN: not llc --mtriple=loongarch64 -mattr=+d < %s 2>&1 | FileCheck %s
 
 declare void @llvm.loongarch.dbar(i32)
 declare void @llvm.loongarch.ibar(i32)
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/add.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/add.ll
index 709e0faeff90..c10d4949438f 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/add.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/add.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Exercise the 'add' LLVM IR: https://llvm.org/docs/LangRef.html#add-instruction
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/and.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/and.ll
index b3e32cc5c00c..730d2609e64d 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/and.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/and.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Exercise the 'and' LLVM IR: https://llvm.org/docs/LangRef.html#and-instruction
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/ashr.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/ashr.ll
index d4f766d460d1..6d04372d9222 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/ashr.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/ashr.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Exercise the 'ashr' LLVM IR: https://llvm.org/docs/LangRef.html#ashr-instruction
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll
index 495974a59ba6..ad98397dfe8f 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define void @cmpxchg_i8_acquire_acquire(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; LA64-LABEL: cmpxchg_i8_acquire_acquire:
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll
index 794242f45fdb..2bd29c2670a6 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s | \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s | \
 ; RUN:   FileCheck %s --check-prefix=LA64
 
 ;; TODO: Testing for LA32 architecture will be added later
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll
index 9b83b4c9535e..f2f459ecaa2e 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define i8 @atomicrmw_xchg_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA32-LABEL: atomicrmw_xchg_i8_acquire:
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll
index 36e39cc6d848..02375a925723 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/br.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefixes=ALL,LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefixes=ALL,LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefixes=ALL,LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefixes=ALL,LA64
 
 define void @foo() noreturn nounwind {
 ; ALL-LABEL: foo:
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/call.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/call.ll
index 90ee9490de74..697f7f79aa00 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/call.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/call.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck --check-prefix=LA32 %s
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck --check-prefix=LA64 %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck --check-prefix=LA32 %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck --check-prefix=LA64 %s
 
 declare i32 @external_function(i32)
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/fence-singlethread.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/fence-singlethread.ll
index a8b164a4cd3c..cd62f4b1be68 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/fence-singlethread.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/fence-singlethread.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define void @fence_singlethread() {
 ; LA32-LABEL: fence_singlethread:
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/fence.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/fence.ll
index c5b2232f9b80..717c5d0dc41b 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/fence.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/fence.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define void @fence_acquire() nounwind {
 ; LA32-LABEL: fence_acquire:
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/icmp.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/icmp.ll
index 605b3ab29378..fabfbc3beeda 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/icmp.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/icmp.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Exercise the 'icmp' LLVM IR: https://llvm.org/docs/LangRef.html#icmp-instruction
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/indirectbr.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/indirectbr.ll
index cd60183a0933..8a8fb5056609 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/indirectbr.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/indirectbr.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 define i32 @indirectbr(ptr %target) nounwind {
 ; CHECK-LABEL: indirectbr:
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
index 8b170c479eed..c51fded410e8 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define i8 @load_acquire_i8(ptr %ptr) {
 ; LA32-LABEL: load_acquire_i8:
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/lshr.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/lshr.ll
index 7b28872780e8..ce7d1ec93d9d 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/lshr.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/lshr.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Exercise the 'lshr' LLVM IR: https://llvm.org/docs/LangRef.html#lshr-instruction
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll
index cfa6ceae78f9..58cc0e7d6484 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Exercise the 'mul' LLVM IR: https://llvm.org/docs/LangRef.html#mul-instruction
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/or.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/or.ll
index ead72507d751..7dacd3f6105d 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/or.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/or.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Exercise the 'or' LLVM IR: https://llvm.org/docs/LangRef.html#or-instruction
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem.ll
index 9c94bfeeadc0..381f69bb46f8 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
-; RUN: llc --mtriple=loongarch32 -loongarch-check-zero-division < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d -loongarch-check-zero-division < %s \
 ; RUN:     | FileCheck %s --check-prefix=LA32-TRAP
-; RUN: llc --mtriple=loongarch64 -loongarch-check-zero-division < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d -loongarch-check-zero-division < %s \
 ; RUN:     | FileCheck %s --check-prefix=LA64-TRAP
 
 ;; Test the sdiv/udiv/srem/urem LLVM IR.
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/select-bare-int.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/select-bare-int.ll
index ad0a241f5fd3..7239e27d9944 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/select-bare-int.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/select-bare-int.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Exercise the bare integers 'select' LLVM IR: https://llvm.org/docs/LangRef.html#select-instruction
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/select-icc-int.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/select-icc-int.ll
index 0acf31f8bb1a..6a2e1f6972ad 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/select-icc-int.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/select-icc-int.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Test integers selection after integers comparison
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/sext-zext-trunc.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/sext-zext-trunc.ll
index 7053d5340896..255bb79ad158 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/sext-zext-trunc.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/sext-zext-trunc.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Test sext/zext/trunc
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/shl.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/shl.ll
index 3f35b76b1603..2be777cc8db0 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/shl.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/shl.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Exercise the 'shl' LLVM IR: https://llvm.org/docs/LangRef.html#shl-instruction
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll
index bb236c11bb81..a593a66441b7 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Exercise the 'sub' LLVM IR: https://llvm.org/docs/LangRef.html#sub-instruction
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/xor.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/xor.ll
index 373c9cf4b64e..703b812e8bcd 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/xor.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/xor.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Exercise the 'xor' LLVM IR: https://llvm.org/docs/LangRef.html#xor-instruction
 
diff --git a/llvm/test/CodeGen/LoongArch/jump-table.ll b/llvm/test/CodeGen/LoongArch/jump-table.ll
index 0cd6ef02d8da..bb3d49157ac6 100644
--- a/llvm/test/CodeGen/LoongArch/jump-table.ll
+++ b/llvm/test/CodeGen/LoongArch/jump-table.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --min-jump-table-entries=5 < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --min-jump-table-entries=5 < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --min-jump-table-entries=5 < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --min-jump-table-entries=5 < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
-; RUN: llc --mtriple=loongarch32 --min-jump-table-entries=4 < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --min-jump-table-entries=4 < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32-JT
-; RUN: llc --mtriple=loongarch64 --min-jump-table-entries=4 < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --min-jump-table-entries=4 < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64-JT
 
 ;; The default mininum number of entries to use a jump table is 4.
diff --git a/llvm/test/CodeGen/LoongArch/ldptr.ll b/llvm/test/CodeGen/LoongArch/ldptr.ll
index c7c2374d5fd5..c3656a6bdafb 100644
--- a/llvm/test/CodeGen/LoongArch/ldptr.ll
+++ b/llvm/test/CodeGen/LoongArch/ldptr.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Check that ldptr.w is not emitted for small offsets.
 define signext i32 @ldptr_w_too_small_offset(ptr %p) nounwind {
diff --git a/llvm/test/CodeGen/LoongArch/ldx-stx-sp-2.ll b/llvm/test/CodeGen/LoongArch/ldx-stx-sp-2.ll
index be125f25ab2b..b5c9d1e7ff0f 100644
--- a/llvm/test/CodeGen/LoongArch/ldx-stx-sp-2.ll
+++ b/llvm/test/CodeGen/LoongArch/ldx-stx-sp-2.ll
@@ -1,5 +1,5 @@
-; RUN: llc --mtriple=loongarch32 < %s
-; RUN: llc --mtriple=loongarch64 < %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s
 
 ;; This should not crash the code generator.
 
diff --git a/llvm/test/CodeGen/LoongArch/ldx-stx-sp-3.ll b/llvm/test/CodeGen/LoongArch/ldx-stx-sp-3.ll
index 45d2450bd64c..2bf9961b0af5 100644
--- a/llvm/test/CodeGen/LoongArch/ldx-stx-sp-3.ll
+++ b/llvm/test/CodeGen/LoongArch/ldx-stx-sp-3.ll
@@ -1,5 +1,5 @@
-; RUN: llc --mtriple=loongarch32 < %s
-; RUN: llc --mtriple=loongarch64 < %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s
 
 ;; This should not crash the code generator.
 
diff --git a/llvm/test/CodeGen/LoongArch/legalicmpimm.ll b/llvm/test/CodeGen/LoongArch/legalicmpimm.ll
index 3dc8785631dc..71faf232640d 100644
--- a/llvm/test/CodeGen/LoongArch/legalicmpimm.ll
+++ b/llvm/test/CodeGen/LoongArch/legalicmpimm.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define i32 @icmpimm(i32 %x) {
 ; LA32-LABEL: icmpimm:
diff --git a/llvm/test/CodeGen/LoongArch/load-store-offset.ll b/llvm/test/CodeGen/LoongArch/load-store-offset.ll
index 68777dfe0c2a..b83d5dc67dd4 100644
--- a/llvm/test/CodeGen/LoongArch/load-store-offset.ll
+++ b/llvm/test/CodeGen/LoongArch/load-store-offset.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define i8 @load_i8() nounwind {
 ; LA32-LABEL: load_i8:
diff --git a/llvm/test/CodeGen/LoongArch/memcmp.ll b/llvm/test/CodeGen/LoongArch/memcmp.ll
index 4d4f376cd538..d8e322b3afe4 100644
--- a/llvm/test/CodeGen/LoongArch/memcmp.ll
+++ b/llvm/test/CodeGen/LoongArch/memcmp.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 ;; Before getSelectionDAGInfo() interface hooks were defined DAGBuilder
 ;; would crash.
diff --git a/llvm/test/CodeGen/LoongArch/mir-target-flags.ll b/llvm/test/CodeGen/LoongArch/mir-target-flags.ll
index 9f3a061fe724..f530e3ef237c 100644
--- a/llvm/test/CodeGen/LoongArch/mir-target-flags.ll
+++ b/llvm/test/CodeGen/LoongArch/mir-target-flags.ll
@@ -1,6 +1,6 @@
-; RUN: llc --mtriple=loongarch64 --stop-after loongarch-prera-expand-pseudo \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --stop-after loongarch-prera-expand-pseudo \
 ; RUN:     --relocation-model=pic %s -o %t.mir
-; RUN: llc --mtriple=loongarch64 --run-pass loongarch-prera-expand-pseudo \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --run-pass loongarch-prera-expand-pseudo \
 ; RUN:     %t.mir -o - | FileCheck %s
 
 ;; This tests the LoongArch-specific serialization and deserialization of
diff --git a/llvm/test/CodeGen/LoongArch/nomerge.ll b/llvm/test/CodeGen/LoongArch/nomerge.ll
index a8d5116f6b67..d35d3186b031 100644
--- a/llvm/test/CodeGen/LoongArch/nomerge.ll
+++ b/llvm/test/CodeGen/LoongArch/nomerge.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 define void @foo(i32 %i) nounwind {
 ; CHECK-LABEL: foo:
diff --git a/llvm/test/CodeGen/LoongArch/not.ll b/llvm/test/CodeGen/LoongArch/not.ll
index b9e02bdf111d..05ece715e1c4 100644
--- a/llvm/test/CodeGen/LoongArch/not.ll
+++ b/llvm/test/CodeGen/LoongArch/not.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define i8 @nor_i8(i8 %a, i8 %b) nounwind {
 ; LA32-LABEL: nor_i8:
diff --git a/llvm/test/CodeGen/LoongArch/numeric-reg-names.ll b/llvm/test/CodeGen/LoongArch/numeric-reg-names.ll
index 153a697a55b9..10a97a1778df 100644
--- a/llvm/test/CodeGen/LoongArch/numeric-reg-names.ll
+++ b/llvm/test/CodeGen/LoongArch/numeric-reg-names.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --loongarch-numeric-reg < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --loongarch-numeric-reg < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --loongarch-numeric-reg < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-numeric-reg < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
 
 @.str_1 = internal constant [7 x i8] c"hello\0A\00"
diff --git a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
index 803985fde215..f976dd8f9868 100644
--- a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
+++ b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
@@ -1,16 +1,16 @@
 ;; When EXPENSIVE_CHECKS are enabled, the machine verifier appears between each
 ;; pass. Ignore it with 'grep -v'.
-; RUN: llc --mtriple=loongarch32 -O1 --debug-pass=Structure %s -o /dev/null 2>&1 | \
+; RUN: llc --mtriple=loongarch32 -mattr=+d -O1 --debug-pass=Structure %s -o /dev/null 2>&1 | \
 ; RUN:   grep -v "Verify generated machine code" | FileCheck %s --check-prefix=LAXX
-; RUN: llc --mtriple=loongarch32 -O2 --debug-pass=Structure %s -o /dev/null 2>&1 | \
+; RUN: llc --mtriple=loongarch32 -mattr=+d -O2 --debug-pass=Structure %s -o /dev/null 2>&1 | \
 ; RUN:   grep -v "Verify generated machine code" | FileCheck %s --check-prefix=LAXX
-; RUN: llc --mtriple=loongarch32 -O3 --debug-pass=Structure %s -o /dev/null 2>&1 | \
+; RUN: llc --mtriple=loongarch32 -mattr=+d -O3 --debug-pass=Structure %s -o /dev/null 2>&1 | \
 ; RUN:   grep -v "Verify generated machine code" | FileCheck %s --check-prefix=LAXX
-; RUN: llc --mtriple=loongarch64 -O1 --debug-pass=Structure %s -o /dev/null 2>&1 | \
+; RUN: llc --mtriple=loongarch64 -mattr=+d -O1 --debug-pass=Structure %s -o /dev/null 2>&1 | \
 ; RUN:   grep -v "Verify generated machine code" | FileCheck %s --check-prefixes=LAXX,LA64
-; RUN: llc --mtriple=loongarch64 -O2 --debug-pass=Structure %s -o /dev/null 2>&1 | \
+; RUN: llc --mtriple=loongarch64 -mattr=+d -O2 --debug-pass=Structure %s -o /dev/null 2>&1 | \
 ; RUN:   grep -v "Verify generated machine code" | FileCheck %s --check-prefixes=LAXX,LA64
-; RUN: llc --mtriple=loongarch64 -O3 --debug-pass=Structure %s -o /dev/null 2>&1 | \
+; RUN: llc --mtriple=loongarch64 -mattr=+d -O3 --debug-pass=Structure %s -o /dev/null 2>&1 | \
 ; RUN:   grep -v "Verify generated machine code" | FileCheck %s --check-prefixes=LAXX,LA64
 
 ; REQUIRES: asserts
diff --git a/llvm/test/CodeGen/LoongArch/patchable-function-entry.ll b/llvm/test/CodeGen/LoongArch/patchable-function-entry.ll
index 2e390d1e2c33..fdb9288c2bd7 100644
--- a/llvm/test/CodeGen/LoongArch/patchable-function-entry.ll
+++ b/llvm/test/CodeGen/LoongArch/patchable-function-entry.ll
@@ -1,7 +1,7 @@
 ;; Test the function attribute "patchable-function-entry".
 ;; Adapted from the RISCV test case.
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefixes=CHECK,LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefixes=CHECK,LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefixes=CHECK,LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefixes=CHECK,LA64
 
 define void @f0() "patchable-function-entry"="0" {
 ; CHECK-LABEL: f0:
diff --git a/llvm/test/CodeGen/LoongArch/prefer-w-inst.ll b/llvm/test/CodeGen/LoongArch/prefer-w-inst.ll
index 385f27f04d5f..8a7c7183a75c 100644
--- a/llvm/test/CodeGen/LoongArch/prefer-w-inst.ll
+++ b/llvm/test/CodeGen/LoongArch/prefer-w-inst.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck --check-prefixes=NO-PREFER-W-INST %s
-; RUN: llc --mtriple=loongarch64 --loongarch-disable-cvt-to-d-suffix --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --loongarch-disable-cvt-to-d-suffix --verify-machineinstrs < %s \
 ; RUN:   | FileCheck --check-prefixes=NO-CVT-D-INST %s
 ; RUN: llc --mtriple=loongarch64 --mattr=+prefer-w-inst --verify-machineinstrs < %s \
 ; RUN:   | FileCheck --check-prefixes=PREFER-W-INST %s
diff --git a/llvm/test/CodeGen/LoongArch/preferred-alignments.ll b/llvm/test/CodeGen/LoongArch/preferred-alignments.ll
index 30305127b94f..c3618db64601 100644
--- a/llvm/test/CodeGen/LoongArch/preferred-alignments.ll
+++ b/llvm/test/CodeGen/LoongArch/preferred-alignments.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck --check-prefix=LA464 %s
-; RUN: llc --mtriple=loongarch64 --mcpu=la464 < %s | FileCheck --check-prefix=LA464 %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck --check-prefix=LA464 %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d --mcpu=la464 < %s | FileCheck --check-prefix=LA464 %s
 
 define signext i32 @sum(ptr noalias nocapture noundef readonly %0, i32 noundef signext %1) {
 ; LA464-LABEL: sum:
diff --git a/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll b/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll
index d6f3e3469f75..0effd469e3fb 100644
--- a/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll
+++ b/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc --mtriple=loongarch64 --code-model=medium --post-RA-scheduler=0 < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --code-model=medium --post-RA-scheduler=0 < %s \
 ; RUN:     | FileCheck %s --check-prefix=MEDIUM_NO_SCH
-; RUN: llc --mtriple=loongarch64 --code-model=medium --post-RA-scheduler=1 < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --code-model=medium --post-RA-scheduler=1 < %s \
 ; RUN:     | FileCheck %s --check-prefix=MEDIUM_SCH
-; RUN: llc --mtriple=loongarch64 --code-model=large --post-RA-scheduler=0 < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --code-model=large --post-RA-scheduler=0 < %s \
 ; RUN:     | FileCheck %s --check-prefix=LARGE_NO_SCH
-; RUN: llc --mtriple=loongarch64 --code-model=large --post-RA-scheduler=1 < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --code-model=large --post-RA-scheduler=1 < %s \
 ; RUN:     | FileCheck %s --check-prefix=LARGE_SCH
 
 ;; FIXME: According to the description of the psABI v2.30, the code sequences
diff --git a/llvm/test/CodeGen/LoongArch/register-coalescer-crash-pr79718.mir b/llvm/test/CodeGen/LoongArch/register-coalescer-crash-pr79718.mir
index b3c44affb785..9c62983c6d45 100644
--- a/llvm/test/CodeGen/LoongArch/register-coalescer-crash-pr79718.mir
+++ b/llvm/test/CodeGen/LoongArch/register-coalescer-crash-pr79718.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
-# RUN: llc -o - %s -mtriple=loongarch64 \
+# RUN: llc -o - %s -mtriple=loongarch64 -mattr=+d \
 # RUN:  -run-pass=register-coalescer -join-liveintervals=1 -join-splitedges=0 | FileCheck %s
 
 ---
diff --git a/llvm/test/CodeGen/LoongArch/returnaddr-error.ll b/llvm/test/CodeGen/LoongArch/returnaddr-error.ll
index 6ac1e0afcd5c..efb23e0c2fa4 100644
--- a/llvm/test/CodeGen/LoongArch/returnaddr-error.ll
+++ b/llvm/test/CodeGen/LoongArch/returnaddr-error.ll
@@ -1,4 +1,4 @@
-; RUN: not llc --mtriple=loongarch64 < %s 2>&1 | FileCheck %s
+; RUN: not llc --mtriple=loongarch64 -mattr=+d < %s 2>&1 | FileCheck %s
 
 declare ptr @llvm.returnaddress(i32 immarg)
 
diff --git a/llvm/test/CodeGen/LoongArch/rotl-rotr.ll b/llvm/test/CodeGen/LoongArch/rotl-rotr.ll
index b9fbd962e6bb..b2d46f5c088b 100644
--- a/llvm/test/CodeGen/LoongArch/rotl-rotr.ll
+++ b/llvm/test/CodeGen/LoongArch/rotl-rotr.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define signext i32 @rotl_32(i32 signext %x, i32 signext %y) nounwind {
 ; LA32-LABEL: rotl_32:
diff --git a/llvm/test/CodeGen/LoongArch/select-const.ll b/llvm/test/CodeGen/LoongArch/select-const.ll
index 6a61cb66ef99..e9506b3a8359 100644
--- a/llvm/test/CodeGen/LoongArch/select-const.ll
+++ b/llvm/test/CodeGen/LoongArch/select-const.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define signext i32 @select_const_int_one_away(i1 zeroext %a) nounwind {
 ; LA32-LABEL: select_const_int_one_away:
diff --git a/llvm/test/CodeGen/LoongArch/select-to-shiftand.ll b/llvm/test/CodeGen/LoongArch/select-to-shiftand.ll
index fa8879ea69dd..a40e31c5d303 100644
--- a/llvm/test/CodeGen/LoongArch/select-to-shiftand.ll
+++ b/llvm/test/CodeGen/LoongArch/select-to-shiftand.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Compare if positive and select variable or zero.
 define i8 @pos_sel_variable_and_zero_i8(i8 signext %a, i8 signext %b) {
diff --git a/llvm/test/CodeGen/LoongArch/sext-cheaper-than-zext.ll b/llvm/test/CodeGen/LoongArch/sext-cheaper-than-zext.ll
index c363948a1494..fafd9ef092da 100644
--- a/llvm/test/CodeGen/LoongArch/sext-cheaper-than-zext.ll
+++ b/llvm/test/CodeGen/LoongArch/sext-cheaper-than-zext.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 define signext i32 @sext_icmp(i32 signext %x, i32 signext %y) {
 ; CHECK-LABEL: sext_icmp:
diff --git a/llvm/test/CodeGen/LoongArch/shift-masked-shamt.ll b/llvm/test/CodeGen/LoongArch/shift-masked-shamt.ll
index 3494329e3e7c..4909a5f7098d 100644
--- a/llvm/test/CodeGen/LoongArch/shift-masked-shamt.ll
+++ b/llvm/test/CodeGen/LoongArch/shift-masked-shamt.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; This test checks that unnecessary masking of shift amount operands is
 ;; eliminated during instruction selection. The test needs to ensure that the
diff --git a/llvm/test/CodeGen/LoongArch/shrinkwrap.ll b/llvm/test/CodeGen/LoongArch/shrinkwrap.ll
index 0323b56080f8..8e5ec17d6124 100644
--- a/llvm/test/CodeGen/LoongArch/shrinkwrap.ll
+++ b/llvm/test/CodeGen/LoongArch/shrinkwrap.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 -O0 < %s | FileCheck %s --check-prefix=NOSHRINKW
-; RUN: llc --mtriple=loongarch64 -O2 < %s | FileCheck %s --check-prefix=SHRINKW
+; RUN: llc --mtriple=loongarch64 -mattr=+d -O0 < %s | FileCheck %s --check-prefix=NOSHRINKW
+; RUN: llc --mtriple=loongarch64 -mattr=+d -O2 < %s | FileCheck %s --check-prefix=SHRINKW
 
 declare void @abort()
 
diff --git a/llvm/test/CodeGen/LoongArch/smul-with-overflow.ll b/llvm/test/CodeGen/LoongArch/smul-with-overflow.ll
index 0efb5fd4e640..739680e6141d 100644
--- a/llvm/test/CodeGen/LoongArch/smul-with-overflow.ll
+++ b/llvm/test/CodeGen/LoongArch/smul-with-overflow.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 define zeroext i1 @smuloi64(i64 %v1, i64 %v2, ptr %res) {
 ; LA32-LABEL: smuloi64:
diff --git a/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll b/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll
index 7a52697d1529..08534e307e4e 100644
--- a/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll
+++ b/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -O0 --mtriple=loongarch64 --verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -O0 --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s | FileCheck %s
 
 ;; This test case is reduced from pr17377.c of the GCC C Torture Suite using
 ;; bugpoint.
diff --git a/llvm/test/CodeGen/LoongArch/split-sp-adjust.ll b/llvm/test/CodeGen/LoongArch/split-sp-adjust.ll
index 8217336637da..0605ceedf3e2 100644
--- a/llvm/test/CodeGen/LoongArch/split-sp-adjust.ll
+++ b/llvm/test/CodeGen/LoongArch/split-sp-adjust.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s
 
 ;; The stack size is 2048 and the SP adjustment will be split.
diff --git a/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll b/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
index 497ac065a8c3..1246b8bfa110 100644
--- a/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
+++ b/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
 
 declare void @callee(ptr, ptr)
diff --git a/llvm/test/CodeGen/LoongArch/stack-realignment.ll b/llvm/test/CodeGen/LoongArch/stack-realignment.ll
index ac1397a9370d..43e61adb2bdc 100644
--- a/llvm/test/CodeGen/LoongArch/stack-realignment.ll
+++ b/llvm/test/CodeGen/LoongArch/stack-realignment.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
 
 declare void @callee(ptr)
diff --git a/llvm/test/CodeGen/LoongArch/stptr.ll b/llvm/test/CodeGen/LoongArch/stptr.ll
index cc198f9c2f8c..d70f9f4ba160 100644
--- a/llvm/test/CodeGen/LoongArch/stptr.ll
+++ b/llvm/test/CodeGen/LoongArch/stptr.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64
 
 ;; Check that stptr.w is not emitted for small offsets.
 define void @stptr_w_too_small_offset(ptr %p, i32 signext %val) nounwind {
diff --git a/llvm/test/CodeGen/LoongArch/tail-calls.ll b/llvm/test/CodeGen/LoongArch/tail-calls.ll
index c22a65c77e70..8298d76d8e3a 100644
--- a/llvm/test/CodeGen/LoongArch/tail-calls.ll
+++ b/llvm/test/CodeGen/LoongArch/tail-calls.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
 
 ;; Perform tail call optimization for global address.
 declare i32 @callee_tail(i32 %i)
diff --git a/llvm/test/CodeGen/LoongArch/test_bl_fixupkind.mir b/llvm/test/CodeGen/LoongArch/test_bl_fixupkind.mir
index 70cd5fb8d7eb..7511193f1359 100644
--- a/llvm/test/CodeGen/LoongArch/test_bl_fixupkind.mir
+++ b/llvm/test/CodeGen/LoongArch/test_bl_fixupkind.mir
@@ -1,4 +1,4 @@
-# RUN: llc --mtriple=loongarch64 --filetype=obj %s -o - | \
+# RUN: llc --mtriple=loongarch64 -mattr=+d --filetype=obj %s -o - | \
 # RUN: llvm-objdump -d - | FileCheck %s
 
 # REQUIRES: asserts
diff --git a/llvm/test/CodeGen/LoongArch/thread-pointer.ll b/llvm/test/CodeGen/LoongArch/thread-pointer.ll
index 805709e61c54..438f07e14665 100644
--- a/llvm/test/CodeGen/LoongArch/thread-pointer.ll
+++ b/llvm/test/CodeGen/LoongArch/thread-pointer.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s --mtriple=loongarch32 | FileCheck %s
-; RUN: llc < %s --mtriple=loongarch64 | FileCheck %s
+; RUN: llc < %s --mtriple=loongarch32 -mattr=+d | FileCheck %s
+; RUN: llc < %s --mtriple=loongarch64 -mattr=+d | FileCheck %s
 
 declare ptr @llvm.thread.pointer()
 
diff --git a/llvm/test/CodeGen/LoongArch/tls-models.ll b/llvm/test/CodeGen/LoongArch/tls-models.ll
index 6b250ec02162..bb89794d1c84 100644
--- a/llvm/test/CodeGen/LoongArch/tls-models.ll
+++ b/llvm/test/CodeGen/LoongArch/tls-models.ll
@@ -1,15 +1,15 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --relocation-model=pic < %s | FileCheck %s --check-prefix=LA32PIC
-; RUN: llc --mtriple=loongarch64 --relocation-model=pic < %s | FileCheck %s --check-prefix=LA64PIC
-; RUN: llc --mtriple=loongarch64 --code-model=large --relocation-model=pic < %s | FileCheck %s --check-prefix=LA64LARGEPIC
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32NOPIC
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64NOPIC
-; RUN: llc --mtriple=loongarch64 --code-model=large < %s | FileCheck %s --check-prefix=LA64LARGENOPIC
-; RUN: llc --mtriple=loongarch32 --relocation-model=pic --enable-tlsdesc < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d --relocation-model=pic < %s | FileCheck %s --check-prefix=LA32PIC
+; RUN: llc --mtriple=loongarch64 -mattr=+d --relocation-model=pic < %s | FileCheck %s --check-prefix=LA64PIC
+; RUN: llc --mtriple=loongarch64 -mattr=+d --code-model=large --relocation-model=pic < %s | FileCheck %s --check-prefix=LA64LARGEPIC
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32NOPIC
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64NOPIC
+; RUN: llc --mtriple=loongarch64 -mattr=+d --code-model=large < %s | FileCheck %s --check-prefix=LA64LARGENOPIC
+; RUN: llc --mtriple=loongarch32 -mattr=+d --relocation-model=pic --enable-tlsdesc < %s \
 ; RUN:     | FileCheck %s --check-prefix=LA32DESC
-; RUN: llc --mtriple=loongarch64 --relocation-model=pic --enable-tlsdesc < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --relocation-model=pic --enable-tlsdesc < %s \
 ; RUN:     | FileCheck %s --check-prefix=LA64DESC
-; RUN: llc --mtriple=loongarch64 --relocation-model=pic --enable-tlsdesc \
+; RUN: llc --mtriple=loongarch64 -mattr=+d --relocation-model=pic --enable-tlsdesc \
 ; RUN:     --code-model=large < %s | FileCheck %s --check-prefix=DESC64
 
 ;; Check that TLS symbols are lowered correctly based on the specified
diff --git a/llvm/test/CodeGen/LoongArch/trap.ll b/llvm/test/CodeGen/LoongArch/trap.ll
index 718b99160b20..15a7ad82bd7a 100644
--- a/llvm/test/CodeGen/LoongArch/trap.ll
+++ b/llvm/test/CodeGen/LoongArch/trap.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --verify-machineinstrs < %s | FileCheck %s
-; RUN: llc --mtriple=loongarch64 --verify-machineinstrs < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch32 -mattr=+d --verify-machineinstrs < %s | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d --verify-machineinstrs < %s | FileCheck %s
 
 ;; Verify that we lower @llvm.trap() and @llvm.debugtrap() correctly.
 
diff --git a/llvm/test/CodeGen/LoongArch/unaligned-access.ll b/llvm/test/CodeGen/LoongArch/unaligned-access.ll
index dd5b585fcca5..1eadbad65aa7 100644
--- a/llvm/test/CodeGen/LoongArch/unaligned-access.ll
+++ b/llvm/test/CodeGen/LoongArch/unaligned-access.ll
@@ -2,11 +2,11 @@
 
 ;; Test the ual feature which is similar to AArch64/arm64-strict-align.ll.
 
-; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32-ALIGNED
+; RUN: llc --mtriple=loongarch32 -mattr=+d < %s | FileCheck %s --check-prefix=LA32-ALIGNED
 ; RUN: llc --mtriple=loongarch32 --mattr=+ual < %s | FileCheck %s --check-prefix=LA32-UNALIGNED
 ; RUN: llc --mtriple=loongarch32 --mattr=-ual < %s | FileCheck %s --check-prefix=LA32-ALIGNED
 
-; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64-UNALIGNED
+; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s --check-prefix=LA64-UNALIGNED
 ; RUN: llc --mtriple=loongarch64 --mattr=+ual < %s | FileCheck %s --check-prefix=LA64-UNALIGNED
 ; RUN: llc --mtriple=loongarch64 --mattr=-ual < %s | FileCheck %s --check-prefix=LA64-ALIGNED
 
diff --git a/llvm/test/CodeGen/LoongArch/xray-attribute-instrumentation.ll b/llvm/test/CodeGen/LoongArch/xray-attribute-instrumentation.ll
index 09442216c469..8999c2038700 100644
--- a/llvm/test/CodeGen/LoongArch/xray-attribute-instrumentation.ll
+++ b/llvm/test/CodeGen/LoongArch/xray-attribute-instrumentation.ll
@@ -1,5 +1,5 @@
-; RUN: llc --mtriple=loongarch64 %s -o - | FileCheck %s
-; RUN: llc --mtriple=loongarch64 -filetype=obj %s -o %t
+; RUN: llc --mtriple=loongarch64 -mattr=+d %s -o - | FileCheck %s
+; RUN: llc --mtriple=loongarch64 -mattr=+d -filetype=obj %s -o %t
 ; RUN: llvm-readobj -r %t | FileCheck %s --check-prefix=RELOC
 
 define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always" {
diff --git a/llvm/test/CodeGen/LoongArch/zext-with-load-is-free.ll b/llvm/test/CodeGen/LoongArch/zext-with-load-is-free.ll
index d5c505f7160e..d745cd637803 100644
--- a/llvm/test/CodeGen/LoongArch/zext-with-load-is-free.ll
+++ b/llvm/test/CodeGen/LoongArch/zext-with-load-is-free.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc --mtriple=loongarch32 -verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA32
-; RUN: llc --mtriple=loongarch64 -verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefix=LA64
 
 define zeroext i8 @test_zext_i8(ptr %p) nounwind {
diff --git a/llvm/test/Transforms/AtomicExpand/LoongArch/load-store-atomic.ll b/llvm/test/Transforms/AtomicExpand/LoongArch/load-store-atomic.ll
index 77c237a38a48..69448dc2bc8e 100644
--- a/llvm/test/Transforms/AtomicExpand/LoongArch/load-store-atomic.ll
+++ b/llvm/test/Transforms/AtomicExpand/LoongArch/load-store-atomic.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S --mtriple=loongarch32 --passes=atomic-expand %s | FileCheck %s --check-prefix=LA32
-; RUN: opt -S --mtriple=loongarch64 --passes=atomic-expand %s | FileCheck %s --check-prefix=LA64
+; RUN: opt -S --mtriple=loongarch32 -mattr=+d --passes=atomic-expand %s | FileCheck %s --check-prefix=LA32
+; RUN: opt -S --mtriple=loongarch64 -mattr=+d --passes=atomic-expand %s | FileCheck %s --check-prefix=LA64
 
 define i8 @load_acquire_i8(ptr %ptr) {
 ; LA32-LABEL: @load_acquire_i8(
diff --git a/llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll b/llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll
index 20cc25e95adf..304e703c09a1 100644
--- a/llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll
@@ -1,4 +1,4 @@
-; RUN: opt --mtriple=loongarch64 -S --passes='require<profile-summary>,function(codegenprepare)' %s | FileCheck %s
+; RUN: opt --mtriple=loongarch64 -mattr=+d -S --passes='require<profile-summary>,function(codegenprepare)' %s | FileCheck %s
 
 ; Check that we have deterministic output
 define void @test(ptr %sp, ptr %t, i32 %n) {
diff --git a/llvm/test/Transforms/LoopDataPrefetch/LoongArch/basic.ll b/llvm/test/Transforms/LoopDataPrefetch/LoongArch/basic.ll
index 55a2a2970d2d..8553171ac68a 100644
--- a/llvm/test/Transforms/LoopDataPrefetch/LoongArch/basic.ll
+++ b/llvm/test/Transforms/LoopDataPrefetch/LoongArch/basic.ll
@@ -1,6 +1,6 @@
 ;; Tag this 'XFAIL' because we need a few more TTIs and ISels.
 ; XFAIL: *
-; RUN: opt --mtriple=loongarch64 --passes=loop-data-prefetch -loongarch-enable-loop-data-prefetch -S < %s | FileCheck %s
+; RUN: opt --mtriple=loongarch64 -mattr=+d --passes=loop-data-prefetch -loongarch-enable-loop-data-prefetch -S < %s | FileCheck %s
 
 define void @foo(ptr %a, ptr %b) {
 entry:
-- 
GitLab


From 415616daa0bdf6c0065c4c1967f1c4050e6ea836 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Tue, 14 May 2024 07:33:07 -0500
Subject: [PATCH 0725/1206] [flang][OpenMP] Lower standalone ops via OMP
 dispatch, NFC (#92045)

This moves lowering of standalone OpenMP ops into the dispatch function.
Follow-up to PR90098.
---
 flang/lib/Lower/OpenMP/OpenMP.cpp | 68 ++++++++++++-------------------
 1 file changed, 27 insertions(+), 41 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index f9ba2fcbbca7..f21acdd64d7c 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -1959,6 +1959,9 @@ static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
   const List<Clause> &clauses = item->clauses;
 
   switch (llvm::omp::Directive dir = item->id) {
+  case llvm::omp::Directive::OMPD_barrier:
+    genBarrierOp(converter, symTable, semaCtx, eval, loc, queue, item);
+    break;
   case llvm::omp::Directive::OMPD_distribute:
     genDistributeOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
                     item);
@@ -1968,8 +1971,6 @@ static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
     break;
   case llvm::omp::Directive::OMPD_loop:
   case llvm::omp::Directive::OMPD_masked:
-  case llvm::omp::Directive::OMPD_tile:
-  case llvm::omp::Directive::OMPD_unroll:
     TODO(loc, "Unhandled loop directive (" +
                   llvm::omp::getOpenMPDirectiveName(dir) + ")");
     break;
@@ -1977,6 +1978,7 @@ static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
     genMasterOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
     break;
   case llvm::omp::Directive::OMPD_ordered:
+    // Block-associated "ordered" construct.
     genOrderedRegionOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
                        item);
     break;
@@ -1984,6 +1986,10 @@ static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
     genParallelOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item,
                   /*outerCombined=*/false);
     break;
+  case llvm::omp::Directive::OMPD_section:
+    genSectionOp(converter, symTable, semaCtx, eval, loc, /*clauses=*/{}, queue,
+                 item);
+    break;
   case llvm::omp::Directive::OMPD_sections:
     genSectionsOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
                   item);
@@ -2025,9 +2031,20 @@ static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
     genTaskloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
                   item);
     break;
+  case llvm::omp::Directive::OMPD_taskwait:
+    genTaskwaitOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
+                  item);
+    break;
+  case llvm::omp::Directive::OMPD_taskyield:
+    genTaskyieldOp(converter, symTable, semaCtx, eval, loc, queue, item);
+    break;
   case llvm::omp::Directive::OMPD_teams:
     genTeamsOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
     break;
+  case llvm::omp::Directive::OMPD_tile:
+  case llvm::omp::Directive::OMPD_unroll:
+    TODO(loc, "Unhandled loop directive (" +
+                  llvm::omp::getOpenMPDirectiveName(dir) + ")");
   // case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_workshare:
     // FIXME: Workshare is not a commonly used OpenMP construct, an
@@ -2035,6 +2052,7 @@ static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
     // that use this construct, add a single construct for now.
     genSingleOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
     break;
+
   // Composite constructs
   case llvm::omp::Directive::OMPD_distribute_parallel_do:
     genCompositeDistributeParallelDo(converter, symTable, semaCtx, eval, loc,
@@ -2174,45 +2192,14 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
   ConstructQueue queue{
       buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
                           eval, directive.source, directive.v, clauses)};
-
-  switch (directive.v) {
-  default:
-    break;
-  case llvm::omp::Directive::OMPD_barrier:
-    genBarrierOp(converter, symTable, semaCtx, eval, currentLocation, queue,
-                 queue.begin());
-    break;
-  case llvm::omp::Directive::OMPD_taskwait:
-    genTaskwaitOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
-                  queue, queue.begin());
-    break;
-  case llvm::omp::Directive::OMPD_taskyield:
-    genTaskyieldOp(converter, symTable, semaCtx, eval, currentLocation, queue,
-                   queue.begin());
-    break;
-  case llvm::omp::Directive::OMPD_target_data:
-    genTargetDataOp(converter, symTable, semaCtx, eval, currentLocation,
-                    clauses, queue, queue.begin());
-    break;
-  case llvm::omp::Directive::OMPD_target_enter_data:
-    genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(
-        converter, symTable, semaCtx, currentLocation, clauses, queue,
-        queue.begin());
-    break;
-  case llvm::omp::Directive::OMPD_target_exit_data:
-    genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(
-        converter, symTable, semaCtx, currentLocation, clauses, queue,
-        queue.begin());
-    break;
-  case llvm::omp::Directive::OMPD_target_update:
-    genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(
-        converter, symTable, semaCtx, currentLocation, clauses, queue,
-        queue.begin());
-    break;
-  case llvm::omp::Directive::OMPD_ordered:
+  if (directive.v == llvm::omp::Directive::OMPD_ordered) {
+    // Standalone "ordered" directive.
     genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
                  queue, queue.begin());
-    break;
+  } else {
+    // Dispatch handles the "block-associated" variant of "ordered".
+    genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
+                   queue.begin());
   }
 }
 
@@ -2466,8 +2453,7 @@ genOMP(Fortran::lower::AbstractConverter &converter,
   ConstructQueue queue{buildConstructQueue(
       converter.getFirOpBuilder().getModule(), semaCtx, eval,
       sectionConstruct.source, llvm::omp::Directive::OMPD_section, {})};
-  genSectionOp(converter, symTable, semaCtx, eval, loc,
-               /*clauses=*/{}, queue, queue.begin());
+  genOMPDispatch(converter, symTable, semaCtx, eval, loc, queue, queue.begin());
 }
 
 static void
-- 
GitLab


From cc54129b983799e1aaea77aa0ff3040dc30cbc8c Mon Sep 17 00:00:00 2001
From: Justin Cady <desk@justincady.com>
Date: Tue, 14 May 2024 08:48:04 -0400
Subject: [PATCH 0726/1206] Add option to exclude headers from clang-tidy
 analysis (#91400)

This is a renewed attempt to land @toddlipcon's D34654. The comments on
that patch indicate a broad desire for some ability to ignore headers.

After considering various options, including migrating to std::regex, I
believe this is the best path forward. It's intuitive to have separate
regexes for including headers versus excluding them, and this approach
has the added benefit of being completely opt-in. No existing configs
will break, regardless of existing HeaderFilterRegex values.

This functionality is useful for improving performance when analyzing a
targeted subset of code, as well as in cases where some collection of
headers cannot be modified (third party source, for example).
---
 .../ClangTidyDiagnosticConsumer.cpp           |  28 ++-
 .../clang-tidy/ClangTidyDiagnosticConsumer.h  |   1 +
 .../clang-tidy/ClangTidyOptions.cpp           |   6 +-
 .../clang-tidy/ClangTidyOptions.h             |   4 +
 .../clang-tidy/tool/ClangTidyMain.cpp         |  18 ++
 .../clang-tidy/tool/run-clang-tidy.py         |  13 +
 clang-tools-extra/docs/ReleaseNotes.rst       |   3 +
 clang-tools-extra/docs/clang-tidy/index.rst   | 235 +++++++++---------
 .../Inputs/config-files/.clang-tidy           |   1 +
 .../Inputs/config-files/1/.clang-tidy         |   1 +
 .../Inputs/config-files/3/.clang-tidy         |   1 +
 .../infrastructure/config-files.cpp           |  15 +-
 .../clang-tidy/infrastructure/file-filter.cpp |   7 +
 13 files changed, 207 insertions(+), 126 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
index de2a3b51422a..200bb87a5ac3 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
@@ -311,7 +311,18 @@ ClangTidyDiagnosticConsumer::ClangTidyDiagnosticConsumer(
     : Context(Ctx), ExternalDiagEngine(ExternalDiagEngine),
       RemoveIncompatibleErrors(RemoveIncompatibleErrors),
       GetFixesFromNotes(GetFixesFromNotes),
-      EnableNolintBlocks(EnableNolintBlocks) {}
+      EnableNolintBlocks(EnableNolintBlocks) {
+
+  if (Context.getOptions().HeaderFilterRegex &&
+      !Context.getOptions().HeaderFilterRegex->empty())
+    HeaderFilter =
+        std::make_unique<llvm::Regex>(*Context.getOptions().HeaderFilterRegex);
+
+  if (Context.getOptions().ExcludeHeaderFilterRegex &&
+      !Context.getOptions().ExcludeHeaderFilterRegex->empty())
+    ExcludeHeaderFilter = std::make_unique<llvm::Regex>(
+        *Context.getOptions().ExcludeHeaderFilterRegex);
+}
 
 void ClangTidyDiagnosticConsumer::finalizeLastError() {
   if (!Errors.empty()) {
@@ -562,22 +573,17 @@ void ClangTidyDiagnosticConsumer::checkFilters(SourceLocation Location,
   }
 
   StringRef FileName(File->getName());
-  LastErrorRelatesToUserCode = LastErrorRelatesToUserCode ||
-                               Sources.isInMainFile(Location) ||
-                               getHeaderFilter()->match(FileName);
+  LastErrorRelatesToUserCode =
+      LastErrorRelatesToUserCode || Sources.isInMainFile(Location) ||
+      (HeaderFilter &&
+       (HeaderFilter->match(FileName) &&
+        !(ExcludeHeaderFilter && ExcludeHeaderFilter->match(FileName))));
 
   unsigned LineNumber = Sources.getExpansionLineNumber(Location);
   LastErrorPassesLineFilter =
       LastErrorPassesLineFilter || passesLineFilter(FileName, LineNumber);
 }
 
-llvm::Regex *ClangTidyDiagnosticConsumer::getHeaderFilter() {
-  if (!HeaderFilter)
-    HeaderFilter =
-        std::make_unique<llvm::Regex>(*Context.getOptions().HeaderFilterRegex);
-  return HeaderFilter.get();
-}
-
 void ClangTidyDiagnosticConsumer::removeIncompatibleErrors() {
   // Each error is modelled as the set of intervals in which it applies
   // replacements. To detect overlapping replacements, we use a sweep line
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
index 9280eb1e1f21..97e16a12febd 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
@@ -313,6 +313,7 @@ private:
   bool EnableNolintBlocks;
   std::vector<ClangTidyError> Errors;
   std::unique_ptr<llvm::Regex> HeaderFilter;
+  std::unique_ptr<llvm::Regex> ExcludeHeaderFilter;
   bool LastErrorRelatesToUserCode = false;
   bool LastErrorPassesLineFilter = false;
   bool LastErrorWasIgnored = false;
diff --git a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
index cbf21a0e2ae3..445c7f85c900 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
@@ -170,6 +170,8 @@ template <> struct MappingTraits<ClangTidyOptions> {
     IO.mapOptional("ImplementationFileExtensions",
                    Options.ImplementationFileExtensions);
     IO.mapOptional("HeaderFilterRegex", Options.HeaderFilterRegex);
+    IO.mapOptional("ExcludeHeaderFilterRegex",
+                   Options.ExcludeHeaderFilterRegex);
     IO.mapOptional("FormatStyle", Options.FormatStyle);
     IO.mapOptional("User", Options.User);
     IO.mapOptional("CheckOptions", Options.CheckOptions);
@@ -191,7 +193,8 @@ ClangTidyOptions ClangTidyOptions::getDefaults() {
   Options.WarningsAsErrors = "";
   Options.HeaderFileExtensions = {"", "h", "hh", "hpp", "hxx"};
   Options.ImplementationFileExtensions = {"c", "cc", "cpp", "cxx"};
-  Options.HeaderFilterRegex = "";
+  Options.HeaderFilterRegex = std::nullopt;
+  Options.ExcludeHeaderFilterRegex = std::nullopt;
   Options.SystemHeaders = false;
   Options.FormatStyle = "none";
   Options.User = std::nullopt;
@@ -231,6 +234,7 @@ ClangTidyOptions &ClangTidyOptions::mergeWith(const ClangTidyOptions &Other,
   overrideValue(ImplementationFileExtensions,
                 Other.ImplementationFileExtensions);
   overrideValue(HeaderFilterRegex, Other.HeaderFilterRegex);
+  overrideValue(ExcludeHeaderFilterRegex, Other.ExcludeHeaderFilterRegex);
   overrideValue(SystemHeaders, Other.SystemHeaders);
   overrideValue(FormatStyle, Other.FormatStyle);
   overrideValue(User, Other.User);
diff --git a/clang-tools-extra/clang-tidy/ClangTidyOptions.h b/clang-tools-extra/clang-tidy/ClangTidyOptions.h
index e7636cb5d9b0..85d5a02ebbc1 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyOptions.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyOptions.h
@@ -83,6 +83,10 @@ struct ClangTidyOptions {
   /// main files will always be displayed.
   std::optional<std::string> HeaderFilterRegex;
 
+  /// \brief Exclude warnings from headers matching this filter, even if they
+  /// match \c HeaderFilterRegex.
+  std::optional<std::string> ExcludeHeaderFilterRegex;
+
   /// Output warnings from system headers matching \c HeaderFilterRegex.
   std::optional<bool> SystemHeaders;
 
diff --git a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
index f82f4417141d..7388f20ef288 100644
--- a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
+++ b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
@@ -53,6 +53,7 @@ Configuration files:
   Checks                       - Same as '--checks'. Additionally, the list of
                                  globs can be specified as a list instead of a
                                  string.
+  ExcludeHeaderFilterRegex     - Same as '--exclude-header-filter'.
   ExtraArgs                    - Same as '--extra-args'.
   ExtraArgsBefore              - Same as '--extra-args-before'.
   FormatStyle                  - Same as '--format-style'.
@@ -132,6 +133,20 @@ option in .clang-tidy file, if any.
                                          cl::init(""),
                                          cl::cat(ClangTidyCategory));
 
+static cl::opt<std::string> ExcludeHeaderFilter("exclude-header-filter",
+                                                desc(R"(
+Regular expression matching the names of the
+headers to exclude diagnostics from. Diagnostics
+from the main file of each translation unit are
+always displayed.
+Must be used together with --header-filter.
+Can be used together with -line-filter.
+This option overrides the 'ExcludeHeaderFilterRegex'
+option in .clang-tidy file, if any.
+)"),
+                                                cl::init(""),
+                                                cl::cat(ClangTidyCategory));
+
 static cl::opt<bool> SystemHeaders("system-headers", desc(R"(
 Display the errors from system headers.
 This option overrides the 'SystemHeaders' option
@@ -353,6 +368,7 @@ static std::unique_ptr<ClangTidyOptionsProvider> createOptionsProvider(
   DefaultOptions.Checks = DefaultChecks;
   DefaultOptions.WarningsAsErrors = "";
   DefaultOptions.HeaderFilterRegex = HeaderFilter;
+  DefaultOptions.ExcludeHeaderFilterRegex = ExcludeHeaderFilter;
   DefaultOptions.SystemHeaders = SystemHeaders;
   DefaultOptions.FormatStyle = FormatStyle;
   DefaultOptions.User = llvm::sys::Process::GetEnv("USER");
@@ -367,6 +383,8 @@ static std::unique_ptr<ClangTidyOptionsProvider> createOptionsProvider(
     OverrideOptions.WarningsAsErrors = WarningsAsErrors;
   if (HeaderFilter.getNumOccurrences() > 0)
     OverrideOptions.HeaderFilterRegex = HeaderFilter;
+  if (ExcludeHeaderFilter.getNumOccurrences() > 0)
+    OverrideOptions.ExcludeHeaderFilterRegex = ExcludeHeaderFilter;
   if (SystemHeaders.getNumOccurrences() > 0)
     OverrideOptions.SystemHeaders = SystemHeaders;
   if (FormatStyle.getNumOccurrences() > 0)
diff --git a/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py b/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
index 1bd4a5b28309..4dd20bec81d3 100755
--- a/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
+++ b/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
@@ -106,11 +106,14 @@ def get_tidy_invocation(
     use_color,
     plugins,
     warnings_as_errors,
+    exclude_header_filter,
 ):
     """Gets a command line for clang-tidy."""
     start = [clang_tidy_binary]
     if allow_enabling_alpha_checkers:
         start.append("-allow-enabling-analyzer-alpha-checkers")
+    if exclude_header_filter is not None:
+        start.append("--exclude-header-filter=" + exclude_header_filter)
     if header_filter is not None:
         start.append("-header-filter=" + header_filter)
     if line_filter is not None:
@@ -228,6 +231,7 @@ def run_tidy(args, clang_tidy_binary, tmpdir, build_path, queue, lock, failed_fi
             args.use_color,
             args.plugins,
             args.warnings_as_errors,
+            args.exclude_header_filter,
         )
 
         proc = subprocess.Popen(
@@ -292,6 +296,14 @@ def main():
         "-config option after reading specified config file. "
         "Use either -config-file or -config, not both.",
     )
+    parser.add_argument(
+        "-exclude-header-filter",
+        default=None,
+        help="Regular expression matching the names of the "
+        "headers to exclude diagnostics from. Diagnostics from "
+        "the main file of each translation unit are always "
+        "displayed.",
+    )
     parser.add_argument(
         "-header-filter",
         default=None,
@@ -450,6 +462,7 @@ def main():
             args.use_color,
             args.plugins,
             args.warnings_as_errors,
+            args.exclude_header_filter,
         )
         invocation.append("-list-checks")
         invocation.append("-")
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 898c7acc1310..19f830741295 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -115,6 +115,9 @@ Improvements to clang-tidy
 - Fixed `--verify-config` option not properly parsing checks when using the
   literal operator in the `.clang-tidy` config.
 
+- Added argument `--exclude-header-filter` and config option `ExcludeHeaderFilterRegex`
+  to exclude headers from analysis via a RegEx.
+
 New checks
 ^^^^^^^^^^
 
diff --git a/clang-tools-extra/docs/clang-tidy/index.rst b/clang-tools-extra/docs/clang-tidy/index.rst
index 852566f26672..9ccacefa3c2c 100644
--- a/clang-tools-extra/docs/clang-tidy/index.rst
+++ b/clang-tools-extra/docs/clang-tidy/index.rst
@@ -116,122 +116,130 @@ An overview of all the command-line options:
 
   Generic Options:
 
-    --help                          - Display available options (--help-hidden for more)
-    --help-list                     - Display list of available options (--help-list-hidden for more)
-    --version                       - Display the version of this program
+    --help                           - Display available options (--help-hidden for more)
+    --help-list                      - Display list of available options (--help-list-hidden for more)
+    --version                        - Display the version of this program
 
   clang-tidy options:
 
-    --checks=<string>               - Comma-separated list of globs with optional '-'
-                                      prefix. Globs are processed in order of
-                                      appearance in the list. Globs without '-'
-                                      prefix add checks with matching names to the
-                                      set, globs with the '-' prefix remove checks
-                                      with matching names from the set of enabled
-                                      checks. This option's value is appended to the
-                                      value of the 'Checks' option in .clang-tidy
-                                      file, if any.
-    --config=<string>               - Specifies a configuration in YAML/JSON format:
-                                        -config="{Checks: '*',
-                                                  CheckOptions: {x: y}}"
-                                      When the value is empty, clang-tidy will
-                                      attempt to find a file named .clang-tidy for
-                                      each source file in its parent directories.
-    --config-file=<string>          - Specify the path of .clang-tidy or custom config file:
-                                       e.g. --config-file=/some/path/myTidyConfigFile
-                                      This option internally works exactly the same way as
-                                       --config option after reading specified config file.
-                                      Use either --config-file or --config, not both.
-    --dump-config                   - Dumps configuration in the YAML format to
-                                      stdout. This option can be used along with a
-                                      file name (and '--' if the file is outside of a
-                                      project with configured compilation database).
-                                      The configuration used for this file will be
-                                      printed.
-                                      Use along with -checks=* to include
-                                      configuration of all checks.
-    --enable-check-profile          - Enable per-check timing profiles, and print a
-                                      report to stderr.
-    --enable-module-headers-parsing - Enables preprocessor-level module header parsing
-                                      for C++20 and above, empowering specific checks
-                                      to detect macro definitions within modules. This
-                                      feature may cause performance and parsing issues
-                                      and is therefore considered experimental.
-    --explain-config                - For each enabled check explains, where it is
-                                      enabled, i.e. in clang-tidy binary, command
-                                      line or a specific configuration file.
-    --export-fixes=<filename>       - YAML file to store suggested fixes in. The
-                                      stored fixes can be applied to the input source
-                                      code with clang-apply-replacements.
-    --extra-arg=<string>            - Additional argument to append to the compiler command line
-    --extra-arg-before=<string>     - Additional argument to prepend to the compiler command line
-    --fix                           - Apply suggested fixes. Without -fix-errors
-                                      clang-tidy will bail out if any compilation
-                                      errors were found.
-    --fix-errors                    - Apply suggested fixes even if compilation
-                                      errors were found. If compiler errors have
-                                      attached fix-its, clang-tidy will apply them as
-                                      well.
-    --fix-notes                     - If a warning has no fix, but a single fix can
-                                      be found through an associated diagnostic note,
-                                      apply the fix.
-                                      Specifying this flag will implicitly enable the
-                                      '--fix' flag.
-    --format-style=<string>         - Style for formatting code around applied fixes:
-                                        - 'none' (default) turns off formatting
-                                        - 'file' (literally 'file', not a placeholder)
-                                          uses .clang-format file in the closest parent
-                                          directory
-                                        - '{ <json> }' specifies options inline, e.g.
-                                          -format-style='{BasedOnStyle: llvm, IndentWidth: 8}'
-                                        - 'llvm', 'google', 'webkit', 'mozilla'
-                                      See clang-format documentation for the up-to-date
-                                      information about formatting styles and options.
-                                      This option overrides the 'FormatStyle` option in
-                                      .clang-tidy file, if any.
-    --header-filter=<string>        - Regular expression matching the names of the
-                                      headers to output diagnostics from. Diagnostics
-                                      from the main file of each translation unit are
-                                      always displayed.
-                                      Can be used together with -line-filter.
-                                      This option overrides the 'HeaderFilterRegex'
-                                      option in .clang-tidy file, if any.
-    --line-filter=<string>          - List of files with line ranges to filter the
-                                      warnings. Can be used together with
-                                      -header-filter. The format of the list is a
-                                      JSON array of objects:
-                                        [
-                                          {"name":"file1.cpp","lines":[[1,3],[5,7]]},
-                                          {"name":"file2.h"}
-                                        ]
-    --list-checks                   - List all enabled checks and exit. Use with
-                                      -checks=* to list all available checks.
-    --load=<pluginfilename>         - Load the specified plugin
-    -p <string>                     - Build path
-    --quiet                         - Run clang-tidy in quiet mode. This suppresses
-                                      printing statistics about ignored warnings and
-                                      warnings treated as errors if the respective
-                                      options are specified.
-    --store-check-profile=<prefix>  - By default reports are printed in tabulated
-                                      format to stderr. When this option is passed,
-                                      these per-TU profiles are instead stored as JSON.
-    --system-headers                - Display the errors from system headers.
-                                      This option overrides the 'SystemHeaders' option
-                                      in .clang-tidy file, if any.
-    --use-color                     - Use colors in diagnostics. If not set, colors
-                                      will be used if the terminal connected to
-                                      standard output supports colors.
-                                      This option overrides the 'UseColor' option in
-                                      .clang-tidy file, if any.
-    --verify-config                 - Check the config files to ensure each check and
-                                      option is recognized.
-    --vfsoverlay=<filename>         - Overlay the virtual filesystem described by file
-                                      over the real file system.
-    --warnings-as-errors=<string>   - Upgrades warnings to errors. Same format as
-                                      '-checks'.
-                                      This option's value is appended to the value of
-                                      the 'WarningsAsErrors' option in .clang-tidy
-                                      file, if any.
+    --checks=<string>                - Comma-separated list of globs with optional '-'
+                                       prefix. Globs are processed in order of
+                                       appearance in the list. Globs without '-'
+                                       prefix add checks with matching names to the
+                                       set, globs with the '-' prefix remove checks
+                                       with matching names from the set of enabled
+                                       checks. This option's value is appended to the
+                                       value of the 'Checks' option in .clang-tidy
+                                       file, if any.
+    --config=<string>                - Specifies a configuration in YAML/JSON format:
+                                         -config="{Checks: '*',
+                                                   CheckOptions: {x: y}}"
+                                       When the value is empty, clang-tidy will
+                                       attempt to find a file named .clang-tidy for
+                                       each source file in its parent directories.
+    --config-file=<string>           - Specify the path of .clang-tidy or custom config file:
+                                        e.g. --config-file=/some/path/myTidyConfigFile
+                                       This option internally works exactly the same way as
+                                        --config option after reading specified config file.
+                                       Use either --config-file or --config, not both.
+    --dump-config                    - Dumps configuration in the YAML format to
+                                       stdout. This option can be used along with a
+                                       file name (and '--' if the file is outside of a
+                                       project with configured compilation database).
+                                       The configuration used for this file will be
+                                       printed.
+                                       Use along with -checks=* to include
+                                       configuration of all checks.
+    --enable-check-profile           - Enable per-check timing profiles, and print a
+                                       report to stderr.
+    --enable-module-headers-parsing  - Enables preprocessor-level module header parsing
+                                       for C++20 and above, empowering specific checks
+                                       to detect macro definitions within modules. This
+                                       feature may cause performance and parsing issues
+                                       and is therefore considered experimental.
+    --exclude-header-filter=<string> - Regular expression matching the names of the
+                                       headers to exclude diagnostics from. Diagnostics
+                                       from the main file of each translation unit are
+                                       always displayed.
+                                       Must be used together with --header-filter.
+                                       Can be used together with -line-filter.
+                                       This option overrides the 'ExcludeHeaderFilterRegex'
+                                       option in .clang-tidy file, if any.
+    --explain-config                 - For each enabled check explains, where it is
+                                       enabled, i.e. in clang-tidy binary, command
+                                       line or a specific configuration file.
+    --export-fixes=<filename>        - YAML file to store suggested fixes in. The
+                                       stored fixes can be applied to the input source
+                                       code with clang-apply-replacements.
+    --extra-arg=<string>             - Additional argument to append to the compiler command line
+    --extra-arg-before=<string>      - Additional argument to prepend to the compiler command line
+    --fix                            - Apply suggested fixes. Without -fix-errors
+                                       clang-tidy will bail out if any compilation
+                                       errors were found.
+    --fix-errors                     - Apply suggested fixes even if compilation
+                                       errors were found. If compiler errors have
+                                       attached fix-its, clang-tidy will apply them as
+                                       well.
+    --fix-notes                      - If a warning has no fix, but a single fix can
+                                       be found through an associated diagnostic note,
+                                       apply the fix.
+                                       Specifying this flag will implicitly enable the
+                                       '--fix' flag.
+    --format-style=<string>          - Style for formatting code around applied fixes:
+                                         - 'none' (default) turns off formatting
+                                         - 'file' (literally 'file', not a placeholder)
+                                           uses .clang-format file in the closest parent
+                                           directory
+                                         - '{ <json> }' specifies options inline, e.g.
+                                           -format-style='{BasedOnStyle: llvm, IndentWidth: 8}'
+                                         - 'llvm', 'google', 'webkit', 'mozilla'
+                                       See clang-format documentation for the up-to-date
+                                       information about formatting styles and options.
+                                       This option overrides the 'FormatStyle` option in
+                                       .clang-tidy file, if any.
+    --header-filter=<string>         - Regular expression matching the names of the
+                                       headers to output diagnostics from. Diagnostics
+                                       from the main file of each translation unit are
+                                       always displayed.
+                                       Can be used together with -line-filter.
+                                       This option overrides the 'HeaderFilterRegex'
+                                       option in .clang-tidy file, if any.
+    --line-filter=<string>           - List of files with line ranges to filter the
+                                       warnings. Can be used together with
+                                       -header-filter. The format of the list is a
+                                       JSON array of objects:
+                                         [
+                                           {"name":"file1.cpp","lines":[[1,3],[5,7]]},
+                                           {"name":"file2.h"}
+                                         ]
+    --list-checks                    - List all enabled checks and exit. Use with
+                                       -checks=* to list all available checks.
+    --load=<pluginfilename>          - Load the specified plugin
+    -p <string>                      - Build path
+    --quiet                          - Run clang-tidy in quiet mode. This suppresses
+                                       printing statistics about ignored warnings and
+                                       warnings treated as errors if the respective
+                                       options are specified.
+    --store-check-profile=<prefix>   - By default reports are printed in tabulated
+                                       format to stderr. When this option is passed,
+                                       these per-TU profiles are instead stored as JSON.
+    --system-headers                 - Display the errors from system headers.
+                                       This option overrides the 'SystemHeaders' option
+                                       in .clang-tidy file, if any.
+    --use-color                      - Use colors in diagnostics. If not set, colors
+                                       will be used if the terminal connected to
+                                       standard output supports colors.
+                                       This option overrides the 'UseColor' option in
+                                       .clang-tidy file, if any.
+    --verify-config                  - Check the config files to ensure each check and
+                                       option is recognized.
+    --vfsoverlay=<filename>          - Overlay the virtual filesystem described by file
+                                       over the real file system.
+    --warnings-as-errors=<string>    - Upgrades warnings to errors. Same format as
+                                       '-checks'.
+                                       This option's value is appended to the value of
+                                       the 'WarningsAsErrors' option in .clang-tidy
+                                       file, if any.
 
   -p <build-path> is used to read a compile command database.
 
@@ -269,6 +277,7 @@ An overview of all the command-line options:
     Checks                       - Same as '--checks'. Additionally, the list of
                                    globs can be specified as a list instead of a
                                    string.
+    ExcludeHeaderFilterRegex     - Same as '--exclude-header-filter'.
     ExtraArgs                    - Same as '--extra-args'.
     ExtraArgsBefore              - Same as '--extra-args-before'.
     FormatStyle                  - Same as '--format-style'.
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/.clang-tidy b/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/.clang-tidy
index 942169f2ec42..83605c85dd92 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/.clang-tidy
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/.clang-tidy
@@ -1,2 +1,3 @@
 Checks: 'from-parent'
 HeaderFilterRegex: 'parent'
+ExcludeHeaderFilterRegex: 'exc-parent'
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/1/.clang-tidy b/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/1/.clang-tidy
index 800fd4e8eb2a..c37f16bc2d7d 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/1/.clang-tidy
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/1/.clang-tidy
@@ -1,2 +1,3 @@
 Checks: 'from-child1'
 HeaderFilterRegex: 'child1'
+ExcludeHeaderFilterRegex: 'exc-child1'
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/3/.clang-tidy b/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/3/.clang-tidy
index 28dc8517ac9f..9365108255bd 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/3/.clang-tidy
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/Inputs/config-files/3/.clang-tidy
@@ -1,3 +1,4 @@
 InheritParentConfig: true
 Checks: 'from-child3'
 HeaderFilterRegex: 'child3'
+ExcludeHeaderFilterRegex: 'exc-child3'
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/config-files.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/config-files.cpp
index d287412454ca..44d43ebbf8d2 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/config-files.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/config-files.cpp
@@ -1,18 +1,23 @@
 // RUN: clang-tidy -dump-config %S/Inputs/config-files/- -- | FileCheck %s -check-prefix=CHECK-BASE
 // CHECK-BASE: Checks: {{.*}}from-parent
 // CHECK-BASE: HeaderFilterRegex: parent
+// CHECK-BASE: ExcludeHeaderFilterRegex: exc-parent
 // RUN: clang-tidy -dump-config %S/Inputs/config-files/1/- -- | FileCheck %s -check-prefix=CHECK-CHILD1
 // CHECK-CHILD1: Checks: {{.*}}from-child1
 // CHECK-CHILD1: HeaderFilterRegex: child1
+// CHECK-CHILD1: ExcludeHeaderFilterRegex: exc-child1
 // RUN: clang-tidy -dump-config %S/Inputs/config-files/2/- -- | FileCheck %s -check-prefix=CHECK-CHILD2
 // CHECK-CHILD2: Checks: {{.*}}from-parent
 // CHECK-CHILD2: HeaderFilterRegex: parent
+// CHECK-CHILD2: ExcludeHeaderFilterRegex: exc-parent
 // RUN: clang-tidy -dump-config %S/Inputs/config-files/3/- -- | FileCheck %s -check-prefix=CHECK-CHILD3
 // CHECK-CHILD3: Checks: {{.*}}from-parent,from-child3
 // CHECK-CHILD3: HeaderFilterRegex: child3
-// RUN: clang-tidy -dump-config -checks='from-command-line' -header-filter='from command line' %S/Inputs/config-files/- -- | FileCheck %s -check-prefix=CHECK-COMMAND-LINE
+// CHECK-CHILD3: ExcludeHeaderFilterRegex: exc-child3
+// RUN: clang-tidy -dump-config -checks='from-command-line' -header-filter='from command line' -exclude-header-filter='from_command_line' %S/Inputs/config-files/- -- | FileCheck %s -check-prefix=CHECK-COMMAND-LINE
 // CHECK-COMMAND-LINE: Checks: {{.*}}from-parent,from-command-line
 // CHECK-COMMAND-LINE: HeaderFilterRegex: from command line
+// CHECK-COMMAND-LINE: ExcludeHeaderFilterRegex: from_command_line
 
 // For this test we have to use names of the real checks because otherwise values are ignored.
 // Running with the old key: <Key>, value: <value> CheckOptions
@@ -68,3 +73,11 @@
 // Dumped config does not overflow for unsigned options
 // RUN: clang-tidy --dump-config %S/Inputs/config-files/5/- -- | FileCheck %s -check-prefix=CHECK-OVERFLOW
 // CHECK-OVERFLOW: misc-throw-by-value-catch-by-reference.MaxSize: '1152921504606846976'
+
+// RUN: clang-tidy -dump-config -checks='readability-function-size' -header-filter='foo/*' -exclude-header-filter='bar*' %S/Inputs/config-files/- -- | FileCheck %s -check-prefix=CHECK-EXCLUDE-HEADERS
+// CHECK-EXCLUDE-HEADERS: HeaderFilterRegex: 'foo/*'
+// CHECK-EXCLUDE-HEADERS: ExcludeHeaderFilterRegex: 'bar*'
+
+// RUN: clang-tidy -dump-config -checks='readability-function-size' -header-filter='' -exclude-header-filter='' %S/Inputs/config-files/- -- | FileCheck %s -check-prefix=EMPTY-CHECK-EXCLUDE-HEADERS
+// EMPTY-CHECK-EXCLUDE-HEADERS: HeaderFilterRegex: ''
+// EMPTY-CHECK-EXCLUDE-HEADERS: ExcludeHeaderFilterRegex: ''
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp
index a7498723de2b..448ef9ddf166 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp
@@ -11,6 +11,7 @@
 // RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -system-headers -quiet %s -- -I %S/Inputs/file-filter/system/.. -isystem %S/Inputs/file-filter/system 2>&1 | FileCheck --check-prefix=CHECK4-QUIET %s
 // RUN: clang-tidy -checks='-*,cppcoreguidelines-pro-type-cstyle-cast' -header-filter='.*' -system-headers %s -- -I %S/Inputs/file-filter/system/.. -isystem %S/Inputs/file-filter/system 2>&1 | FileCheck --check-prefix=CHECK5 %s
 // RUN: clang-tidy -checks='-*,cppcoreguidelines-pro-type-cstyle-cast' -header-filter='.*' %s -- -I %S/Inputs/file-filter/system/.. -isystem %S/Inputs/file-filter/system 2>&1 | FileCheck --check-prefix=CHECK5-NO-SYSTEM-HEADERS %s
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -exclude-header-filter='header1\.h' %s -- -I %S/Inputs/file-filter/ -isystem %S/Inputs/file-filter/system 2>&1 | FileCheck --check-prefix=CHECK6 %s
 
 #include "header1.h"
 // CHECK-NOT: warning:
@@ -21,6 +22,7 @@
 // CHECK3-QUIET-NOT: warning:
 // CHECK4: header1.h:1:12: warning: single-argument constructors
 // CHECK4-QUIET: header1.h:1:12: warning: single-argument constructors
+// CHECK6-NOT: warning:
 
 #include "header2.h"
 // CHECK-NOT: warning:
@@ -31,6 +33,7 @@
 // CHECK3-QUIET: header2.h:1:12: warning: single-argument constructors
 // CHECK4: header2.h:1:12: warning: single-argument constructors
 // CHECK4-QUIET: header2.h:1:12: warning: single-argument constructors
+// CHECK6: header2.h:1:12: warning: single-argument constructors
 
 #include <system-header.h>
 // CHECK-NOT: warning:
@@ -41,6 +44,7 @@
 // CHECK3-QUIET-NOT: warning:
 // CHECK4: system-header.h:1:12: warning: single-argument constructors
 // CHECK4-QUIET: system-header.h:1:12: warning: single-argument constructors
+// CHECK6-NOT: warning:
 
 class A { A(int); };
 // CHECK: :[[@LINE-1]]:11: warning: single-argument constructors
@@ -51,6 +55,7 @@ class A { A(int); };
 // CHECK3-QUIET: :[[@LINE-6]]:11: warning: single-argument constructors
 // CHECK4: :[[@LINE-7]]:11: warning: single-argument constructors
 // CHECK4-QUIET: :[[@LINE-8]]:11: warning: single-argument constructors
+// CHECK6: :[[@LINE-9]]:11: warning: single-argument constructors
 
 // CHECK-NOT: warning:
 // CHECK-QUIET-NOT: warning:
@@ -73,6 +78,8 @@ class A { A(int); };
 // CHECK4-NOT: Suppressed {{.*}} warnings
 // CHECK4-NOT: Use -header-filter=.* {{.*}}
 // CHECK4-QUIET-NOT: Suppressed
+// CHECK6: Suppressed 2 warnings (2 in non-user code)
+// CHECK6: Use -header-filter=.* {{.*}}
 
 int x = 123;
 auto x_ptr = TO_FLOAT_PTR(&x);
-- 
GitLab


From bf7a0f9958b93d9979e0adf93b80ad056615706d Mon Sep 17 00:00:00 2001
From: AdityaK <hiraditya@msn.com>
Date: Tue, 14 May 2024 06:13:11 -0700
Subject: [PATCH 0727/1206] Fix incorrect codegen with respect to GEPs #85333
 (#92047)

As mentioned in #68882 and
https://discourse.llvm.org/t/rfc-replacing-getelementptr-with-ptradd/68699

Gep arithmetic isn't consistent with different types. GVNSink didn't
realize this and sank all geps as long as their operands can be wired
via PHIs
in a post-dominator.

Fixes: #85333
Reapply: #88440 after fixing the non-determinism issues in #90995
---
 llvm/lib/Transforms/Scalar/GVNSink.cpp        |   9 +-
 .../Transforms/GVNSink/different-gep-types.ll | 101 ++++++++++++++++++
 2 files changed, 105 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/Transforms/GVNSink/different-gep-types.ll

diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp
index 95a4c644a91a..7a183e4d3aa8 100644
--- a/llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -763,12 +763,11 @@ GVNSink::analyzeInstructionForSinking(LockstepReverseIterator &LRI,
   // try and continue making progress.
   Instruction *I0 = NewInsts[0];
 
-  // If all instructions that are going to participate don't have the same
-  // number of operands, we can't do any useful PHI analysis for all operands.
-  auto hasDifferentNumOperands = [&I0](Instruction *I) {
-    return I->getNumOperands() != I0->getNumOperands();
+  auto isNotSameOperation = [&I0](Instruction *I) {
+    return !I0->isSameOperationAs(I);
   };
-  if (any_of(NewInsts, hasDifferentNumOperands))
+
+  if (any_of(NewInsts, isNotSameOperation))
     return std::nullopt;
 
   for (unsigned OpNum = 0, E = I0->getNumOperands(); OpNum != E; ++OpNum) {
diff --git a/llvm/test/Transforms/GVNSink/different-gep-types.ll b/llvm/test/Transforms/GVNSink/different-gep-types.ll
new file mode 100644
index 000000000000..659be51cc56e
--- /dev/null
+++ b/llvm/test/Transforms/GVNSink/different-gep-types.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=gvn-sink -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+
+%"struct.std::pair" = type <{ i32, %struct.substruct, [2 x i8] }>
+%struct.substruct = type { i8, i8 }
+%"struct.std::random_access_iterator_tag" = type { i8 }
+
+; Check that gep is not sunk as they are of different types.
+define void @bar(ptr noundef nonnull align 4 dereferenceable(4) %__i, i32 noundef %__n) {
+; CHECK-LABEL: define void @bar(
+; CHECK-SAME: ptr noundef nonnull align 4 dereferenceable(4) [[__I:%.*]], i32 noundef [[__N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[__N]], 1
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__I]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 -8
+; CHECK-NEXT:    br label [[IF_END6:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__I]], align 4
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds %"struct.std::pair", ptr [[TMP1]], i32 [[__N]]
+; CHECK-NEXT:    br label [[IF_END6]]
+; CHECK:       if.end6:
+; CHECK-NEXT:    [[INCDEC_PTR_SINK:%.*]] = phi ptr [ [[INCDEC_PTR4]], [[IF_THEN]] ], [ [[ADD_PTR]], [[IF_ELSE]] ]
+; CHECK-NEXT:    store ptr [[INCDEC_PTR_SINK]], ptr [[__I]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp eq i32 %__n, 1
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %3 = load ptr, ptr %__i, align 4
+  %incdec.ptr4 = getelementptr inbounds i8, ptr %3, i32 -8
+  br label %if.end6
+
+if.else:
+  %4 = load ptr, ptr %__i, align 4
+  %add.ptr = getelementptr inbounds %"struct.std::pair", ptr %4, i32 %__n
+  br label %if.end6
+
+if.end6:
+  %incdec.ptr.sink = phi ptr [ %incdec.ptr4, %if.then ], [ %add.ptr, %if.else ]
+  store ptr %incdec.ptr.sink, ptr %__i, align 4
+  ret void
+}
+
+; Check that load,gep, and store are all sunk as they are safe to do.
+define void @foo(ptr noundef nonnull align 4 dereferenceable(4) %__i, i32 noundef %__n) {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: ptr noundef nonnull align 4 dereferenceable(4) [[__I:%.*]], i32 noundef [[__N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[__N]], 1
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[IF_END6:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[__N]], -1
+; CHECK-NEXT:    br i1 [[CMP2]], label [[IF_THEN3:%.*]], label [[IF_ELSE5:%.*]]
+; CHECK:       if.then3:
+; CHECK-NEXT:    br label [[IF_END6]]
+; CHECK:       if.else5:
+; CHECK-NEXT:    br label [[IF_END6]]
+; CHECK:       if.end6:
+; CHECK-NEXT:    [[DOTSINK1:%.*]] = phi i32 [ -4, [[IF_ELSE5]] ], [ -8, [[IF_THEN3]] ], [ 8, [[IF_THEN]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__I]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 [[DOTSINK1]]
+; CHECK-NEXT:    store ptr [[INCDEC_PTR]], ptr [[__I]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp eq i32 %__n, 1
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %1 = load ptr, ptr %__i, align 4
+  %incdec.ptr = getelementptr inbounds i8, ptr %1, i32 8
+  store ptr %incdec.ptr, ptr %__i, align 4
+  br label %if.end6
+
+if.else:
+  %cmp2 = icmp eq i32 %__n, -1
+  br i1 %cmp2, label %if.then3, label %if.else5
+
+if.then3:
+  %3 = load ptr, ptr %__i, align 4
+  %incdec.ptr4 = getelementptr inbounds i8, ptr %3, i32 -8
+  store ptr %incdec.ptr4, ptr %__i, align 4
+  br label %if.end6
+
+if.else5:
+  %4 = load ptr, ptr %__i, align 4
+  %add.ptr = getelementptr inbounds i8, ptr %4, i32 -4
+  store ptr %add.ptr, ptr %__i, align 4
+  br label %if.end6
+
+if.end6:
+  ret void
+}
-- 
GitLab


From e1685eb8d7de66ce6420cdd3340a2e3f892c09bd Mon Sep 17 00:00:00 2001
From: Mubashar Ahmad <mubashar.ahmad@arm.com>
Date: Tue, 14 May 2024 14:32:50 +0100
Subject: [PATCH 0728/1206] [mlir][llvm] Add llvm.vector.deinterleave2
 intrinsic (#91986)

Adds the LLVM vector.deinterleave2 intrinsic to the MLIR LLVM dialect. The
deinterleave2 intrinsic takes a vector and returns two vectors with the first
having even elements and the second with odd elements from the input
vector. The inverse of vector.interleave2.
---
 .../include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td |  6 ++++++
 mlir/test/Dialect/LLVMIR/roundtrip.mlir             |  7 +++++++
 mlir/test/Target/LLVMIR/Import/intrinsic.ll         |  9 +++++++++
 mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir      | 13 +++++++++++++
 4 files changed, 35 insertions(+)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 759cbe6c1564..bd347d0cf630 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1074,6 +1074,12 @@ def LLVM_vector_interleave2
         ]>,
         Arguments<(ins LLVM_AnyVector:$vec1, LLVM_AnyVector:$vec2)>;
 
+def LLVM_vector_deinterleave2
+    : LLVM_OneResultIntrOp<"vector.deinterleave2",
+        /*overloadedResults=*/[], /*overloadedOperands=*/[0],
+        /*traits=*/[Pure]>,
+        Arguments<(ins LLVM_AnyVector:$vec)>;
+
 //
 // LLVM Vector Predication operations.
 //
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index 3b94db389f54..410122df1c14 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -349,6 +349,13 @@ func.func @vector_interleave2(%vec1: vector<[4]xf16>, %vec2 : vector<[4]xf16>) {
   return
 }
 
+// CHECK-LABEL: @vector_deinterleave2
+func.func @vector_deinterleave2(%vec: vector<[8]xf16>) {
+  // CHECK: = "llvm.intr.vector.deinterleave2"({{.*}}) : (vector<[8]xf16>) -> !llvm.struct<(vector<[4]xf16>, vector<[4]xf16>)>
+  %0 = "llvm.intr.vector.deinterleave2"(%vec) : (vector<[8]xf16>) -> !llvm.struct<(vector<[4]xf16>, vector<[4]xf16>)>
+  return
+}
+
 // CHECK-LABEL: @alloca
 func.func @alloca(%size : i64) {
   // CHECK: llvm.alloca %{{.*}} x i32 : (i64) -> !llvm.ptr
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index bf6847a32ff4..e43024ff868e 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -786,6 +786,15 @@ define void @vector_extract(<vscale x 4 x float> %0) {
   ret void
 }
 
+; CHECK-LABEL: llvm.func @vector_deinterleave2
+define void @vector_deinterleave2(<4 x double> %0, <vscale x 8 x i32> %1) {
+  ; CHECK: "llvm.intr.vector.deinterleave2"(%{{.*}}) : (vector<4xf64>) -> !llvm.struct<(vector<2xf64>, vector<2xf64>)>
+  %3 = call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %0);
+  ; CHECK: "llvm.intr.vector.deinterleave2"(%{{.*}}) : (!llvm.vec<? x 8 x i32>) -> !llvm.struct<(vec<? x 4 x i32>, vec<? x 4 x i32>)>
+  %4 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %1);
+  ret void
+}
+
 ; CHECK-LABEL:  llvm.func @vector_predication_intrinsics
 define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x float> %2, <8 x float> %3, <8 x i64> %4, <8 x double> %5, <8 x ptr> %6, i32 %7, float %8, ptr %9, ptr %10, <8 x i1> %11, i32 %12) {
   ; CHECK: "llvm.intr.vp.add"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi1>, i32) -> vector<8xi32>
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index db5184a63d98..238c3e4263cb 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -942,6 +942,17 @@ llvm.func @vector_insert_extract(%f256: vector<8xi32>, %f128: vector<4xi32>,
   llvm.return
 }
 
+// CHECK-LABEL: @vector_deinterleave2
+llvm.func @vector_deinterleave2(%vec1: vector<4xf64>, %vec2: vector<[8]xi32>) {
+  // CHECK: call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %{{.*}})
+  %0 = "llvm.intr.vector.deinterleave2" (%vec1) :
+              (vector<4xf64>) -> !llvm.struct<(vector<2xf64>, vector<2xf64>)>
+  // CHECK: call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %{{.*}})
+  %1 = "llvm.intr.vector.deinterleave2" (%vec2) :
+              (vector<[8]xi32>) -> !llvm.struct<(vector<[4]xi32>, vector<[4]xi32>)>
+  llvm.return
+}
+
 // CHECK-LABEL: @lifetime
 llvm.func @lifetime(%p: !llvm.ptr) {
   // CHECK: call void @llvm.lifetime.start
@@ -1148,6 +1159,8 @@ llvm.func @experimental_constrained_fptrunc(%s: f64, %v: vector<4xf32>) {
 // CHECK-DAG: declare <8 x i32> @llvm.vector.extract.v8i32.nxv4i32(<vscale x 4 x i32>, i64 immarg)
 // CHECK-DAG: declare <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32>, i64 immarg)
 // CHECK-DAG: declare <2 x i32> @llvm.vector.extract.v2i32.v8i32(<8 x i32>, i64 immarg)
+// CHECK-DAG: declare { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double>)
+// CHECK-DAG: declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32>)
 // CHECK-DAG: declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
 // CHECK-DAG: declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
 // CHECK-DAG: declare ptr @llvm.invariant.start.p0(i64 immarg, ptr nocapture)
-- 
GitLab


From cfa09473a6f904d214a1b514f41b9d4d9276c927 Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Tue, 14 May 2024 09:35:30 -0400
Subject: [PATCH 0729/1206] Cope with MCOperand null Insts (#91794)

MCOperand has a constructor that permits a nullptr MCInst, and BOLT makes use of that. Adjust MCOperand's dumper to permit such use.
---
 llvm/lib/MC/MCInst.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/MC/MCInst.cpp b/llvm/lib/MC/MCInst.cpp
index 3cc50ff43513..639619fe4e99 100644
--- a/llvm/lib/MC/MCInst.cpp
+++ b/llvm/lib/MC/MCInst.cpp
@@ -38,7 +38,10 @@ void MCOperand::print(raw_ostream &OS, const MCRegisterInfo *RegInfo) const {
     OS << "Expr:(" << *getExpr() << ")";
   } else if (isInst()) {
     OS << "Inst:(";
-    getInst()->print(OS, RegInfo);
+    if (const auto *Inst = getInst())
+      Inst->print(OS, RegInfo);
+    else
+      OS << "NULL";
     OS << ")";
   } else
     OS << "UNDEFINED";
-- 
GitLab


From 725014d866e2a75bfe293ee2d168d4c8f302fc74 Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Tue, 14 May 2024 09:36:34 -0400
Subject: [PATCH 0730/1206] [BOLT][NFC] Simplify CFG validation (#91977)

Remove 'Valid' local boolean that has a single use, and return directly instead.
---
 bolt/lib/Core/BinaryFunction.cpp | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index de34421ebeb0..4f44ba0d970c 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -3252,12 +3252,9 @@ bool BinaryFunction::validateCFG() const {
   if (CurrentState == State::CFG_Finalized)
     return true;
 
-  bool Valid = true;
   for (BinaryBasicBlock *BB : BasicBlocks)
-    Valid &= BB->validateSuccessorInvariants();
-
-  if (!Valid)
-    return Valid;
+    if (!BB->validateSuccessorInvariants())
+      return false;
 
   // Make sure all blocks in CFG are valid.
   auto validateBlock = [this](const BinaryBasicBlock *BB, StringRef Desc) {
@@ -3326,7 +3323,7 @@ bool BinaryFunction::validateCFG() const {
     }
   }
 
-  return Valid;
+  return true;
 }
 
 void BinaryFunction::fixBranches() {
-- 
GitLab


From 312f83f0e0672118a6d82d4b4d3568e9c812086d Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Tue, 14 May 2024 15:37:06 +0200
Subject: [PATCH 0731/1206] [Clang] Fix dependency computation for pack
 indexing expression (#91933)

Given `foo...[idx]` if idx is value dependent, the expression is type
dependent.

Fixes #91885
Fixes #91884
---
 clang/lib/AST/ComputeDependence.cpp        | 13 +++++++++---
 clang/lib/Sema/SemaType.cpp                | 13 ++++++++----
 clang/test/SemaCXX/cxx2c-pack-indexing.cpp | 23 ++++++++++++++++++++++
 3 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/clang/lib/AST/ComputeDependence.cpp b/clang/lib/AST/ComputeDependence.cpp
index bad8e75b2f87..62ca15ea398f 100644
--- a/clang/lib/AST/ComputeDependence.cpp
+++ b/clang/lib/AST/ComputeDependence.cpp
@@ -375,12 +375,19 @@ ExprDependence clang::computeDependence(PackExpansionExpr *E) {
 }
 
 ExprDependence clang::computeDependence(PackIndexingExpr *E) {
+
+  ExprDependence PatternDep = E->getPackIdExpression()->getDependence() &
+                              ~ExprDependence::UnexpandedPack;
+
   ExprDependence D = E->getIndexExpr()->getDependence();
+  if (D & ExprDependence::TypeValueInstantiation)
+    D |= E->getIndexExpr()->getDependence() | PatternDep |
+         ExprDependence::Instantiation;
+
   ArrayRef<Expr *> Exprs = E->getExpressions();
   if (Exprs.empty())
-    D |= (E->getPackIdExpression()->getDependence() |
-          ExprDependence::TypeValueInstantiation) &
-         ~ExprDependence::UnexpandedPack;
+    D |= PatternDep | ExprDependence::Instantiation;
+
   else if (!E->getIndexExpr()->isInstantiationDependent()) {
     std::optional<unsigned> Index = E->getSelectedIndex();
     assert(Index && *Index < Exprs.size() && "pack index out of bound");
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index bfa3799bda06..d65fafc8cf4f 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -9351,15 +9351,20 @@ QualType Sema::BuildCountAttributedArrayType(QualType WrappedTy,
 /// that expression, according to the rules in C++11
 /// [dcl.type.simple]p4 and C++11 [expr.lambda.prim]p18.
 QualType Sema::getDecltypeForExpr(Expr *E) {
-  if (E->isTypeDependent())
-    return Context.DependentTy;
 
   Expr *IDExpr = E;
   if (auto *ImplCastExpr = dyn_cast<ImplicitCastExpr>(E))
     IDExpr = ImplCastExpr->getSubExpr();
 
-  if (auto *PackExpr = dyn_cast<PackIndexingExpr>(E))
-    IDExpr = PackExpr->getSelectedExpr();
+  if (auto *PackExpr = dyn_cast<PackIndexingExpr>(E)) {
+    if (E->isInstantiationDependent())
+      IDExpr = PackExpr->getPackIdExpression();
+    else
+      IDExpr = PackExpr->getSelectedExpr();
+  }
+
+  if (E->isTypeDependent())
+    return Context.DependentTy;
 
   // C++11 [dcl.type.simple]p4:
   //   The type denoted by decltype(e) is defined as follows:
diff --git a/clang/test/SemaCXX/cxx2c-pack-indexing.cpp b/clang/test/SemaCXX/cxx2c-pack-indexing.cpp
index a3e5a0931491..0ac85b5bcc14 100644
--- a/clang/test/SemaCXX/cxx2c-pack-indexing.cpp
+++ b/clang/test/SemaCXX/cxx2c-pack-indexing.cpp
@@ -194,3 +194,26 @@ void h() {
   // expected-note-re@-2 {{function template specialization '{{.*}}' requested here}}
 }
 }
+
+namespace GH91885 {
+
+void test(auto...args){
+    [&]<int idx>(){
+        using R = decltype( args...[idx] ) ;
+    }.template operator()<0>();
+}
+
+template<int... args>
+void test2(){
+  [&]<int idx>(){
+    using R = decltype( args...[idx] ) ;
+  }.template operator()<0>();
+}
+
+void f( ) {
+  test(1);
+  test2<1>();
+}
+
+
+}
-- 
GitLab


From 1aff294f6ef9c0a1a264c55d55e441e37a353f17 Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Tue, 14 May 2024 09:38:32 -0400
Subject: [PATCH 0732/1206] [BOLT][NFC] Simplify successor check (#91980)

Remove excess parentheses and use `boolean ? true-case : false-case` idiom.
---
 bolt/lib/Core/BinaryBasicBlock.cpp | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/bolt/lib/Core/BinaryBasicBlock.cpp b/bolt/lib/Core/BinaryBasicBlock.cpp
index 4a83fece0e43..a4b9a7f558cd 100644
--- a/bolt/lib/Core/BinaryBasicBlock.cpp
+++ b/bolt/lib/Core/BinaryBasicBlock.cpp
@@ -131,11 +131,10 @@ bool BinaryBasicBlock::validateSuccessorInvariants() {
         break;
       }
       case 2:
-        Valid = (CondBranch &&
-                 (TBB == getConditionalSuccessor(true)->getLabel() &&
-                  ((!UncondBranch && !FBB) ||
-                   (UncondBranch &&
-                    FBB == getConditionalSuccessor(false)->getLabel()))));
+        Valid =
+            CondBranch && TBB == getConditionalSuccessor(true)->getLabel() &&
+            (UncondBranch ? FBB == getConditionalSuccessor(false)->getLabel()
+                          : !FBB);
         break;
       }
     }
-- 
GitLab


From 03eba209852c769ab6993be3bc01cdcc57d787b0 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Tue, 14 May 2024 06:47:02 -0700
Subject: [PATCH 0733/1206] [OpenACC] Fix ast-print of device_type clause

When writing the test for this I seemingly forgot to put 'CHECK' on the
lines, so I didn't notice that I was printing the identifiers as
pointers rather than their names.  This patch corrects the tests and the
print behavior.
---
 clang/lib/AST/OpenACCClause.cpp                        |  2 +-
 clang/test/AST/ast-print-openacc-compute-construct.cpp | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index f80ecc90d396..8ff6dabcbc48 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -473,7 +473,7 @@ void OpenACCClausePrinter::VisitDeviceTypeClause(
                           if (Arch.first == nullptr)
                             OS << "*";
                           else
-                            OS << Arch.first;
+                            OS << Arch.first->getName();
                         });
   OS << ")";
 }
diff --git a/clang/test/AST/ast-print-openacc-compute-construct.cpp b/clang/test/AST/ast-print-openacc-compute-construct.cpp
index cdd9ab3377d0..19965e749141 100644
--- a/clang/test/AST/ast-print-openacc-compute-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-compute-construct.cpp
@@ -111,23 +111,23 @@ void foo() {
   bool SomeB;
   struct SomeStruct{} SomeStructImpl;
 
-//#pragma acc parallel dtype(SomeB)
+//CHECK: #pragma acc parallel dtype(SomeB)
 #pragma acc parallel dtype(SomeB)
   while(true);
 
-//#pragma acc parallel device_type(SomeStruct)
+//CHECK: #pragma acc parallel device_type(SomeStruct)
 #pragma acc parallel device_type(SomeStruct)
   while(true);
 
-//#pragma acc parallel device_type(int)
+//CHECK: #pragma acc parallel device_type(int)
 #pragma acc parallel device_type(int)
   while(true);
 
-//#pragma acc parallel dtype(bool)
+//CHECK: #pragma acc parallel dtype(bool)
 #pragma acc parallel dtype(bool)
   while(true);
 
-//#pragma acc parallel device_type (SomeStructImpl)
+//CHECK: #pragma acc parallel device_type(SomeStructImpl)
 #pragma acc parallel device_type (SomeStructImpl)
   while(true);
 }
-- 
GitLab


From e60b83a645685f22375af9bca5af6624b3a805d0 Mon Sep 17 00:00:00 2001
From: Youngsuk Kim <youngsuk.kim@hpe.com>
Date: Tue, 14 May 2024 08:49:24 -0500
Subject: [PATCH 0734/1206] [libclc] Clarify condition expression (NFC)

Closes #91188
---
 libclc/generic/lib/math/log_base.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libclc/generic/lib/math/log_base.h b/libclc/generic/lib/math/log_base.h
index 2558f016f60b..4e20329f641b 100644
--- a/libclc/generic/lib/math/log_base.h
+++ b/libclc/generic/lib/math/log_base.h
@@ -209,7 +209,7 @@ log(double x)
     const double log_thresh1 = 0x1.e0faap-1;
     const double log_thresh2 = 0x1.1082cp+0;
 
-    int is_near = x >= log_thresh1 & x <= log_thresh2;
+    bool is_near = x >= log_thresh1 && x <= log_thresh2;
 
     // Near 1 code
     double r = x - 1.0;
-- 
GitLab


From d422e90fcbdddd68749918ddd86c94188807efce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Tue, 14 May 2024 15:54:05 +0200
Subject: [PATCH 0735/1206] [GlobalIsel][AArch64] fix out of range access in
 regbankselect (#92072)

Fixes https://github.com/llvm/llvm-project/issues/92062
---
 .../AArch64/GISel/AArch64RegisterBankInfo.cpp |  5 ++++-
 llvm/test/CodeGen/AArch64/pr92062.ll          | 21 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AArch64/pr92062.ll

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
index 44ba9f0429e6..7785e020eaaf 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
@@ -600,8 +600,11 @@ bool AArch64RegisterBankInfo::isLoadFromFPType(const MachineInstr &MI) const {
     EltTy = GV->getValueType();
     // Look at the first element of the struct to determine the type we are
     // loading
-    while (StructType *StructEltTy = dyn_cast<StructType>(EltTy))
+    while (StructType *StructEltTy = dyn_cast<StructType>(EltTy)) {
+      if (StructEltTy->getNumElements() == 0)
+        break;
       EltTy = StructEltTy->getTypeAtIndex(0U);
+    }
     // Look at the first element of the array to determine its type
     if (isa<ArrayType>(EltTy))
       EltTy = EltTy->getArrayElementType();
diff --git a/llvm/test/CodeGen/AArch64/pr92062.ll b/llvm/test/CodeGen/AArch64/pr92062.ll
new file mode 100644
index 000000000000..6111ee0fbe18
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/pr92062.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64 -O0 -global-isel %s -o - 2>&1 | FileCheck %s
+
+target triple = "arm64"
+
+@p = external global { {}, { ptr } }
+
+define void @foo() {
+; CHECK-LABEL: foo:
+; CHECK:       // %bb.0: // %bb
+; CHECK-NEXT:    adrp x8, :got:p
+; CHECK-NEXT:    ldr x8, [x8, :got_lo12:p]
+; CHECK-NEXT:    ldr x8, [x8]
+; CHECK-NEXT:    mov x9, xzr
+; CHECK-NEXT:    str x8, [x9]
+; CHECK-NEXT:    ret
+bb:
+  %i1 = load ptr, ptr @p, align 8
+  store ptr %i1, ptr null, align 8
+  ret void
+}
-- 
GitLab


From 041baf2f60ac3e107399641aea04c77019e7eab8 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Tue, 14 May 2024 14:59:01 +0100
Subject: [PATCH 0736/1206] [mlir][ArmSME] Use liveness information in the tile
 allocator (#90448)

This patch rewrites the ArmSME tile allocator to use liveness
information to make better tile allocation decisions and improve the
correctness of the ArmSME dialect. This algorithm used here is a linear
scan over live ranges, where live ranges are assigned to tiles as they
appear in the program (chronologically). Live ranges release their
assigned tile ID when the current program point is passed their end.
This is a greedy algorithm (which is mainly to keep the implementation
relatively straightforward), and because it seems to be sufficient for
most kernels (e.g. matmuls) that use ArmSME. The general steps of this
are roughly from
https://link.springer.com/content/pdf/10.1007/3-540-45937-5_17.pdf,
though there have been a few simplifications and assumptions made for
our use case.

Hopefully, the only changes needed for a user of the ArmSME dialect is
that:

- `-allocate-arm-sme-tiles` will no longer be a standalone pass
  - `-test-arm-sme-tile-allocation` is only for unit tests
- `-convert-arm-sme-to-llvm` must happen after `-convert-scf-to-cf`
   - SME tile allocation is now part of the LLVM conversion

By integrating this into the `ArmSME -> LLVM` conversion we can allow
high-level (value-based) ArmSME operations to be side-effect-free, as we
can guarantee nothing will rearrange ArmSME operations before we emit
intrinsics (which could invalidate the tile allocation).

The hope is for ArmSME operations to have no hidden state/side effects
and allow easily lowering dialects such as `vector` and `arith` to SME,
without making assumptions about how the input IR looks, as the
semantics of the operations will be the same. That is no (new) side
effects and the IR follows the rules of SSA (a value will never change).

The aim is correctness, so we have a base for working on optimizations.
---
 .../Conversion/ArmSMEToLLVM/ArmSMEToLLVM.h    |   4 +-
 mlir/include/mlir/Conversion/Passes.td        |   7 +-
 mlir/include/mlir/Dialect/ArmSME/IR/ArmSME.h  |   6 +-
 .../Dialect/ArmSME/IR/ArmSMEOpInterfaces.h    |  27 +
 .../mlir/Dialect/ArmSME/IR/ArmSMEOps.td       | 141 ++--
 .../mlir/Dialect/ArmSME/Transforms/Passes.h   |   3 -
 .../mlir/Dialect/ArmSME/Transforms/Passes.td  |  22 +-
 .../Dialect/ArmSME/Transforms/Transforms.h    |   9 +
 .../include/mlir/Dialect/ArmSME/Utils/Utils.h |  32 +
 .../Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp  | 125 +--
 .../Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp    |  29 +-
 mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp         |   6 +
 mlir/lib/Dialect/ArmSME/IR/Utils.cpp          |  53 ++
 .../ArmSME/Transforms/TileAllocation.cpp      | 791 ++++++++++++++----
 .../ArmSMEToLLVM/arm-sme-to-llvm.mlir         |   3 +-
 .../ArmSMEToLLVM/tile-spills-and-fills.mlir   |  17 +-
 .../Conversion/ArmSMEToLLVM/unsupported.mlir  |   2 +-
 .../Dialect/ArmSME/basic-tile-allocation.mlir |   2 +-
 mlir/test/Dialect/ArmSME/canonicalize.mlir    |  14 +-
 mlir/test/Dialect/ArmSME/cse.mlir             |  30 -
 mlir/test/Dialect/ArmSME/roundtrip.mlir       |   9 +
 .../ArmSME/tile-allocation-copies.mlir        | 159 ++++
 .../ArmSME/tile-allocation-invalid.mlir       |  14 +-
 .../ArmSME/tile-allocation-liveness.mlir      | 381 +++++++--
 ...location-spills-with-mixed-tile-types.mlir |  38 +
 mlir/test/Dialect/ArmSME/tile-zero-masks.mlir |   2 +-
 .../Linalg/CPU/ArmSME/use-too-many-tiles.mlir |   7 +-
 .../ArmSME/Emulated/test-setArmSVLBits.mlir   |   3 +-
 mlir/test/lib/Dialect/ArmSME/CMakeLists.txt   |   1 +
 .../lib/Dialect/ArmSME/TestLowerToArmSME.cpp  |  19 +-
 30 files changed, 1470 insertions(+), 486 deletions(-)
 create mode 100644 mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOpInterfaces.h
 delete mode 100644 mlir/test/Dialect/ArmSME/cse.mlir
 create mode 100644 mlir/test/Dialect/ArmSME/tile-allocation-copies.mlir
 create mode 100644 mlir/test/Dialect/ArmSME/tile-allocation-spills-with-mixed-tile-types.mlir

diff --git a/mlir/include/mlir/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.h b/mlir/include/mlir/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.h
index eab871ab4999..403f811a2569 100644
--- a/mlir/include/mlir/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.h
+++ b/mlir/include/mlir/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.h
@@ -12,6 +12,7 @@
 #include <memory>
 
 #include "mlir/Dialect/ArmSME/Transforms/Passes.h"
+#include "mlir/Interfaces/FunctionInterfaces.h"
 
 namespace mlir {
 class Pass;
@@ -21,7 +22,8 @@ class RewritePatternSet;
 #include "mlir/Conversion/Passes.h.inc"
 
 /// Create a pass to convert from the ArmSME dialect to LLVM intrinsics.
-std::unique_ptr<Pass> createConvertArmSMEToLLVMPass();
+std::unique_ptr<Pass>
+createConvertArmSMEToLLVMPass(bool dumpTileLiveRanges = false);
 
 /// Configure target to convert from the ArmSME dialect to LLVM intrinsics.
 void configureArmSMEToLLVMConversionLegality(ConversionTarget &target);
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index d094ee3b36ab..e6d678dc1b12 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -1285,7 +1285,7 @@ def ConvertArmSMEToSCF : Pass<"convert-arm-sme-to-scf"> {
 // ArmSMEToLLVM
 //===----------------------------------------------------------------------===//
 
-def ConvertArmSMEToLLVM : Pass<"convert-arm-sme-to-llvm"> {
+def ConvertArmSMEToLLVM : InterfacePass<"convert-arm-sme-to-llvm", "FunctionOpInterface"> {
   let summary = "Lower the operations from the ArmSME dialect into the LLVM "
                 "dialect";
   let constructor = "mlir::createConvertArmSMEToLLVMPass()";
@@ -1293,6 +1293,11 @@ def ConvertArmSMEToLLVM : Pass<"convert-arm-sme-to-llvm"> {
     "arm_sme::ArmSMEDialect",
     "LLVM::LLVMDialect"
   ];
+  let options = [
+    Option<"dumpTileLiveRanges", "dump-tile-live-ranges",
+           "bool", /*default=*/"false",
+           "Dump the live ranges of SME tiles (for debugging)">
+  ];
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSME.h b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSME.h
index c507cea5357a..dac54712c7f4 100644
--- a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSME.h
+++ b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSME.h
@@ -15,6 +15,7 @@
 
 #include "mlir/Bytecode/BytecodeOpInterface.h"
 #include "mlir/Dialect/ArmSME/IR/ArmSMEEnums.h"
+#include "mlir/Dialect/ArmSME/IR/ArmSMEOpInterfaces.h"
 #include "mlir/Dialect/ArmSME/Utils/Utils.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
@@ -24,11 +25,6 @@
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
-namespace mlir::arm_sme {
-static constexpr unsigned kInMemoryTileIdBase = 16;
-#include "mlir/Dialect/ArmSME/IR/ArmSMEOpInterfaces.h.inc"
-} // namespace mlir::arm_sme
-
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/ArmSME/IR/ArmSMEAttrDefs.h.inc"
 
diff --git a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOpInterfaces.h b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOpInterfaces.h
new file mode 100644
index 000000000000..9153fbb57ea8
--- /dev/null
+++ b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOpInterfaces.h
@@ -0,0 +1,27 @@
+//===- ArmSMEOpInterfaces.h - Arm SME Dialect OpInterfaces ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_ARMSME_OPINTERFACES_H
+#define MLIR_DIALECT_ARMSME_OPINTERFACES_H
+
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+
+namespace mlir::arm_sme {
+
+namespace detail {
+LogicalResult verifyArmSMETileOpInterface(Operation *);
+}
+
+// The first in-memory SME tile ID. This is set to 16 as that is the first tile
+// ID larger than any virtual tile ID supported by the SME ISA.
+static constexpr unsigned kInMemoryTileIdBase = 16;
+
+#include "mlir/Dialect/ArmSME/IR/ArmSMEOpInterfaces.h.inc"
+} // namespace mlir::arm_sme
+
+#endif // MLIR_DIALECT_ARMSME_OPINTERFACES_H
diff --git a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOps.td b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOps.td
index 239c4beab10d..9178655f010c 100644
--- a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOps.td
+++ b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOps.td
@@ -39,10 +39,10 @@ def ArmSMETileType : I32EnumAttr<"ArmSMETileType", "Arm SME tile type",
 
 def ArmSMETileOpInterface : OpInterface<"ArmSMETileOpInterface"> {
   let description = [{
-    An interface for operations that use or allocate Arm SME tiles. These
-    operations need to be assigned a tile ID, an i32 attribute, which specifies
-    which virtual tile within the ZA storage to use. The number of tiles
-    available depends on the type of the tile. This is summarized below:
+    An interface for operations that use Arm SME tiles. These operations need to
+    be assigned a tile ID, an i32 attribute, which specifies which virtual tile
+    within the ZA storage to use. The number of tiles available depends on the
+    type of the tile. This is summarized below:
 
     | Tile Vector Types                                                       | Possible Tile IDs   |
     |-------------------------------------------------------------------------|---------------------|
@@ -51,10 +51,6 @@ def ArmSMETileOpInterface : OpInterface<"ArmSMETileOpInterface"> {
     | `vector<[4]x[4]xi32>` or `vector<[4]x[4]xf32>`                          | 0 to 3 (inclusive)  |
     | `vector<[2]x[2]xi64>` or `vector<[2]x[2]xf64>`                          | 0 to 7 (inclusive)  |
     | `vector<[1]x[1]xi128>`                                                  | 0 to 15 (inclusive) |
-
-    Operations that allocate a new tile (such as arm_sme.get_tile), are used as
-    the roots for tile allocation, with all operations that (transitively)
-    depend on a root being assigned the same tile ID.
   }];
   let methods = [
     InterfaceMethod<
@@ -84,20 +80,6 @@ def ArmSMETileOpInterface : OpInterface<"ArmSMETileOpInterface"> {
         return op->getAttrOfType<mlir::IntegerAttr>("tile_id");
       }]
     >,
-    InterfaceMethod<
-      [{
-        The type of tile this operation allocates. Returns none (std::nullopt)
-        if this operation does not allocate a tile.
-      }],
-      /*returnType=*/"std::optional<::mlir::arm_sme::ArmSMETileType>",
-      /*methodName=*/"getAllocatedTileType",
-      /*arguments=*/(ins),
-      /*methodBody=*/[{}],
-      /*defaultImpl=*/ [{
-        // This operation does not allocate a tile.
-        return std::nullopt;
-      }]
-    >,
     InterfaceMethod<
       "Returns the VectorType of the tile used by this operation.",
       /*returnType=*/"VectorType",
@@ -106,30 +88,13 @@ def ArmSMETileOpInterface : OpInterface<"ArmSMETileOpInterface"> {
   ];
 
   let extraSharedClassDeclaration = [{
-    // A helper to create a new operation and propagate this operations tile ID.
-    template<typename T, typename... Args>
-    T createOpAndForwardTileId(::mlir::RewriterBase& rewriter, ::mlir::Location loc, Args &&...args) {
-      auto op = rewriter.create<T>(loc, std::forward<Args>(args)...);
-      if (auto tileOp = ::llvm::dyn_cast<ArmSMETileOpInterface>(op.getOperation()))
-        tileOp.setTileId($_op.getTileId());
-      return op;
-    }
-
-    // A helper to replace this operation and forward its tile ID (if present).
-    template<typename T, typename... Args>
-    T replaceWithAndForwardTileId(::mlir::RewriterBase& rewriter, Args &&...args) {
-      auto newOp = createOpAndForwardTileId<T>(rewriter, $_op.getLoc(), std::forward<Args>(args)...);
-      rewriter.replaceOp($_op, newOp);
-      return newOp;
-    }
-
     bool isInMemoryTile() {
       auto tileId = getTileId();
       return tileId && tileId.getInt() >= kInMemoryTileIdBase;
     }
   }];
 
-  let verify = [{ return ::mlir::arm_sme::verifyOperationHasValidTileId($_op); }];
+  let verify = [{ return detail::verifyArmSMETileOpInterface($_op); }];
 }
 
 //===----------------------------------------------------------------------===//
@@ -255,30 +220,30 @@ def ArmSME_TypeSizeAttr : EnumAttr<ArmSME_Dialect, TypeSize,
 class ArmSME_Op<string mnemonic, list<Trait> traits = []> :
   Op<ArmSME_Dialect, mnemonic, traits> {}
 
-def GetTileOp : ArmSME_Op<"get_tile", [ArmSMETileOpInterface]> {
-  let summary = "Returns a SME virtual tile";
+def GetTileOp : ArmSME_Op<"get_tile", [ArmSMETileOpInterface, Pure]> {
+  let summary = "Creates an undefined value of SME virtual tile type";
   let description = [{
-    Allocates a new SME "virtual tile" within a function. The contents of the
-    tile returned from this operation are undefined.
+    Creates a new SME "virtual tile" value within a function. The contents of
+    the tile returned from this operation are undefined.
 
     Example 1:
 
     ```mlir
-    // Allocate an 8-bit element "virtual tile"
+    // Create an 8-bit element "virtual tile" value:
     %za0_b = arm_sme.get_tile: vector<[16]x[16]xi8>
     ```
 
     Example 2:
 
     ```mlir
-    // Allocate two 16-bit element "virtual tiles"
+    // Create two 16-bit element "virtual tiles" values:
     %za0_h = arm_sme.get_tile : vector<[8]x[8]xi16>
     %za1_h = arm_sme.get_tile : vector<[8]x[8]xi16>
     ```
 
     Example 3:
     ```mlir
-    // Allocate an 128-bit element "virtual tile"
+    // Create an 128-bit element "virtual tile" value:
     %za0_q = arm_sme.get_tile : vector<[1]x[1]xi128>
     ```
   }];
@@ -290,37 +255,15 @@ def GetTileOp : ArmSME_Op<"get_tile", [ArmSMETileOpInterface]> {
     VectorType getTileType() {
       return ::llvm::cast<VectorType>(getTile().getType());
     }
-
-    std::optional<arm_sme::ArmSMETileType> getAllocatedTileType() {
-      return arm_sme::getSMETileType(getTileType());
-    }
-  }];
-}
-
-def MaterializeSSATileOp : ArmSME_Op<"materialize_ssa_tile", [Pure]> {
-  let summary = "SME tile placeholder";
-  let description = [{
-    A placeholder to preserve dataflow while lowering to SME intrinsics (which
-    do not take or return SME virtual tile values). This operation is intended
-    to be DCE'd once all ArmSME operations have been lowered.
-
-    This operation is not intended to be used outside of the ArmSME -> LLVM
-    conversion.
   }];
-  let results = (outs SMETile:$tile);
-  let assemblyFormat = "attr-dict `:` type($tile)";
 }
 
-//
-// Tile reset.
-//
-
-def ZeroOp : ArmSME_Op<"zero", [ArmSMETileOpInterface]> {
-  let summary = "Initialize the two-dimensional ZA array with 0s";
+def ZeroOp : ArmSME_Op<"zero", [ArmSMETileOpInterface, Pure]> {
+  let summary = "Creates a zero-initialized value of SME virtual tile type";
   let results = (outs SMETile:$res);
   let description = [{
-    Initialise ZA with 0. This operation is convenient wrapper for the SME
-    `zero` intrinsic and instruction.
+    Creates a new SME "virtual tile" value within a function. The contents of
+    the tile returned from this operation are zero-initialized.
 
     Example 1: Zero an 8-bit element ZA tile.
 
@@ -338,9 +281,6 @@ def ZeroOp : ArmSME_Op<"zero", [ArmSMETileOpInterface]> {
     VectorType getVectorType() {
       return ::llvm::cast<VectorType>(getRes().getType());
     }
-    std::optional<arm_sme::ArmSMETileType> getAllocatedTileType() {
-      return arm_sme::getSMETileType(getVectorType());
-    }
     VectorType getTileType() {
       return getVectorType();
     }
@@ -348,6 +288,32 @@ def ZeroOp : ArmSME_Op<"zero", [ArmSMETileOpInterface]> {
   let assemblyFormat = "attr-dict `:` type($res)";
 }
 
+def CopyTileOp : ArmSME_Op<"copy_tile", [
+  Pure,
+  ArmSMETileOpInterface,
+  AllTypesMatch<["tile", "result"]>
+]> {
+  let summary = "Copies an SME tile value";
+  let arguments = (ins SMETile:$tile);
+  let results = (outs SMETile:$result);
+  let description = [{
+    Copies an SME "virtual tile" value to a new SSA value. This operation is
+    primarily intended to be used to normalize the IR prior to tile allocation.
+
+    Example:
+
+    ```mlir
+    %copy = arm_sme.copy_tile %tile : vector<[4]x[4]xf32>
+    ```
+  }];
+  let extraClassDeclaration = [{
+    VectorType getTileType() {
+      return ::llvm::cast<VectorType>(getResult().getType());
+    }
+  }];
+  let assemblyFormat = "$tile attr-dict `:` type($result)";
+}
+
 def TileLoadOp : ArmSME_Op<"tile_load", [
   ArmSMETileOpInterface,
   AttrSizedOperandSegments,
@@ -417,9 +383,6 @@ def TileLoadOp : ArmSME_Op<"tile_load", [
     VectorType getVectorType() {
       return ::llvm::cast<VectorType>(getResult().getType());
     }
-    std::optional<arm_sme::ArmSMETileType> getAllocatedTileType() {
-      return arm_sme::getSMETileType(getVectorType());
-    }
     VectorType getTileType() {
       return getVectorType();
     }
@@ -545,7 +508,7 @@ def LoadTileSliceOp : ArmSME_Op<"load_tile_slice", [
     ```
   }];
   let arguments = (ins
-    Arg<AnyMemRef, "the reference to load from">:$base, SVEPredicate:$mask,
+    Arg<AnyMemRef, "the reference to load from", [MemRead]>:$base, SVEPredicate:$mask,
     SMETile:$tile, Variadic<Index>:$indices, Index:$tile_slice_index,
     ArmSME_TileSliceLayoutAttr:$layout
   );
@@ -630,7 +593,7 @@ def StoreTileSliceOp : ArmSME_Op<"store_tile_slice", [
 }
 
 def MoveVectorToTileSliceOp : ArmSME_Op<"move_vector_to_tile_slice", [
-    ArmSMETileOpInterface,
+    ArmSMETileOpInterface, Pure,
     AllTypesMatch<["tile", "result"]>,
     TypesMatchWith<
       "type of 'vector' matches type of 'tile' slice",
@@ -679,7 +642,7 @@ def MoveVectorToTileSliceOp : ArmSME_Op<"move_vector_to_tile_slice", [
 }
 
 def MoveTileSliceToVectorOp : ArmSME_Op<"move_tile_slice_to_vector", [
-    ArmSMETileOpInterface,
+    ArmSMETileOpInterface, Pure,
     TypesMatchWith<
       "type of 'result' matches type of 'tile' slice",
       "tile", "result",
@@ -736,6 +699,7 @@ class OuterProductResultTileTypeConstraint<string operand> :
 
 def OuterProductOp :
   ArmSME_Op<"outerproduct", [
+    Pure,
     ArmSMETileOpInterface,
     AttrSizedOperandSegments,
     AllTypesMatch<["lhs", "rhs"]>,
@@ -802,12 +766,6 @@ let arguments = (ins
     VectorType getLhsType() { return llvm::cast<VectorType>(getLhs().getType()); }
     VectorType getRhsType() { return llvm::cast<VectorType>(getRhs().getType()); }
     VectorType getResultType() { return llvm::cast<VectorType>(getResult().getType()); }
-    std::optional<arm_sme::ArmSMETileType> getAllocatedTileType() {
-      // The outerproduct op allocates a new tile if no accumulator is passed.
-      if (!getAcc())
-        return arm_sme::getSMETileType(getResultType());
-      return std::nullopt;
-    }
     VectorType getTileType() {
       return getResultType();
     }
@@ -819,6 +777,7 @@ class OuterProductWideningBase<string mnemonic,
                                list<Type> allowedResultVectorTypes,
                                int numOuterProducts> :
   ArmSME_Op<mnemonic, [
+    Pure,
     ArmSMETileOpInterface,
     AttrSizedOperandSegments,
     AllTypesMatch<["lhs", "rhs"]>,
@@ -857,12 +816,6 @@ class OuterProductWideningBase<string mnemonic,
     VectorType getLhsType() { return llvm::cast<VectorType>(getLhs().getType()); }
     VectorType getRhsType() { return llvm::cast<VectorType>(getRhs().getType()); }
     VectorType getResultType() { return llvm::cast<VectorType>(getResult().getType()); }
-    std::optional<arm_sme::ArmSMETileType> getAllocatedTileType() {
-      // The outerproduct op allocates a new tile if no accumulator is passed.
-      if (!getAcc())
-        return arm_sme::getSMETileType(getResultType());
-      return std::nullopt;
-    }
     VectorType getTileType() {
       return getResultType();
     }
diff --git a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.h b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.h
index c2f1b1f1b874..156744ba57e7 100644
--- a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.h
@@ -29,9 +29,6 @@ std::unique_ptr<Pass> createEnableArmStreamingPass(
     const ArmStreamingMode = ArmStreamingMode::Streaming,
     const ArmZaMode = ArmZaMode::Disabled, bool onlyIfRequiredByOps = false);
 
-/// Pass that allocates tile IDs to ArmSME operations.
-std::unique_ptr<Pass> createTileAllocationPass();
-
 /// Pass that fuses 'arm_sme.outerproduct' ops into 2-way or 4-way widening
 /// variants.
 std::unique_ptr<Pass> createOuterProductFusionPass();
diff --git a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td
index 7959d291e892..869a031d6cae 100644
--- a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td
@@ -124,17 +124,25 @@ def EnableArmStreaming
   let dependentDialects = ["func::FuncDialect"];
 }
 
-def TileAllocation
-    : Pass<"allocate-arm-sme-tiles", "mlir::func::FuncOp"> {
-  let summary = "Allocate SME tiles";
+def TestTileAllocation
+    : Pass<"test-arm-sme-tile-allocation", "mlir::func::FuncOp"> {
+  let summary = "Tests SME 'virtual tile' allocation";
   let description = [{
     This pass does tile allocation for SME "virtual tiles". It is run at the
     'func.func' op level, and assigns tile IDs (via an attribute) to all ops
-    that implement the `ArmSMETileOpInterface`. An error will be emitted when
-    there's no tiles left.
+    that implement the `ArmSMETileOpInterface`. Note: This pass is only intended
+    to be used for testing, tile allocation is done as part of the ArmSME to
+    LLVM conversion (`convert-arm-sme-to-llvm`).
   }];
-  let constructor = "mlir::arm_sme::createTileAllocationPass()";
-  let dependentDialects = ["func::FuncDialect"];
+  let options = [
+    Option<"dumpTileLiveRanges", "dump-tile-live-ranges",
+           "bool", /*default=*/"false",
+           "Dump the live ranges of SME tiles (for debugging)">,
+    Option<"preprocessOnly", "preprocess-only", "bool", /*default=*/"false",
+           "Only preprocess IR so it is ready for tile allocation "
+           "(but do not allocate any tiles)">
+  ];
+  let dependentDialects = ["func::FuncDialect", "arm_sme::ArmSMEDialect"];
 }
 
 def OuterProductFusion
diff --git a/mlir/include/mlir/Dialect/ArmSME/Transforms/Transforms.h b/mlir/include/mlir/Dialect/ArmSME/Transforms/Transforms.h
index e00c7503e699..a25b844f01ea 100644
--- a/mlir/include/mlir/Dialect/ArmSME/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/ArmSME/Transforms/Transforms.h
@@ -9,6 +9,8 @@
 #ifndef MLIR_DIALECT_ARMSME_TRANSFORMS_H
 #define MLIR_DIALECT_ARMSME_TRANSFORMS_H
 
+#include "mlir/Interfaces/FunctionInterfaces.h"
+
 namespace mlir {
 
 class LLVMConversionTarget;
@@ -16,7 +18,14 @@ class LLVMTypeConverter;
 class RewritePatternSet;
 
 namespace arm_sme {
+
 void populateOuterProductFusionPatterns(RewritePatternSet &patterns);
+
+/// Allocate tile IDs to all ArmSME operations in a function. Requires the
+/// function to be lowered to control flow (cf dialect).
+LogicalResult allocateSMETiles(FunctionOpInterface function,
+                               bool dumpRanges = false);
+
 } // namespace arm_sme
 
 } // namespace mlir
diff --git a/mlir/include/mlir/Dialect/ArmSME/Utils/Utils.h b/mlir/include/mlir/Dialect/ArmSME/Utils/Utils.h
index 027ad8954f92..1f40eb6fc693 100644
--- a/mlir/include/mlir/Dialect/ArmSME/Utils/Utils.h
+++ b/mlir/include/mlir/Dialect/ArmSME/Utils/Utils.h
@@ -16,8 +16,10 @@
 #define MLIR_DIALECT_ARMSME_UTILS_UTILS_H_
 
 #include "mlir/Dialect/ArmSME/IR/ArmSMEEnums.h"
+#include "mlir/Dialect/ArmSME/IR/ArmSMEOpInterfaces.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/Interfaces/FunctionInterfaces.h"
 #include <optional>
 
 namespace mlir {
@@ -42,6 +44,11 @@ bool isValidSMETileElementType(Type type);
 /// otherwise.
 bool isValidSMETileVectorType(VectorType vType);
 
+inline bool isValidSMETileVectorType(Type type) {
+  auto vType = dyn_cast<VectorType>(type);
+  return vType && isValidSMETileVectorType(vType);
+}
+
 /// Returns the type of SME tile this vector type corresponds to, or none if the
 /// vector type does not fit within an SME tile.
 std::optional<ArmSMETileType> getSMETileType(VectorType);
@@ -63,6 +70,31 @@ bool isMultipleOfSMETileVectorType(VectorType vType);
 /// Creates a vector type for the SME tile of `elementType`.
 VectorType getSMETileTypeForElement(Type elementType);
 
+/// Erase trivially dead tile ops from a function.
+void eraseTriviallyDeadTileOps(IRRewriter &rewriter,
+                               FunctionOpInterface function);
+
+/// Returns true if `tileOp` is trivially cloneable. A tile operation is
+/// trivially cloneable if:
+///
+///  1. It has no operands (and only a single tile result)
+///  2. It is 'Pure'
+///
+/// This ensures that the cloned operation will not share any dependencies with
+/// the original operation (which could also need to be considered), and that
+/// inserting the cloned operation at a different point in the program won't
+/// change the semantics of the program (as it has no side effects).
+bool isTriviallyCloneableTileOp(arm_sme::ArmSMETileOpInterface tileOp);
+
+/// Returns true if `tileOp` produces a tile result.
+bool hasTileResult(arm_sme::ArmSMETileOpInterface tileOp);
+
+/// Returns the tile `OpOperand` for this `tileOp` (or null).
+OpOperand *getTileOpOperand(arm_sme::ArmSMETileOpInterface tileOp);
+
+/// Returns true `typeA` is >= (in terms of bytes) than `typeB`.
+bool isTileTypeGreaterOrEqual(ArmSMETileType typeA, ArmSMETileType typeB);
+
 } // namespace mlir::arm_sme
 
 #endif // MLIR_DIALECT_ARMSME_UTILS_UTILS_H_
diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index 1ba1b88fc123..3dbc8e9916df 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/ArmSME/IR/ArmSME.h"
+#include "mlir/Dialect/ArmSME/Transforms/Transforms.h"
 #include "mlir/Dialect/ArmSME/Utils/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
@@ -245,6 +246,10 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
     if (!tileOp.isInMemoryTile())
       return failure();
 
+    tileOp->emitWarning(
+        "failed to allocate SME virtual tile to operation, all tile "
+        "operations will go through memory, expect degraded performance");
+
     // Step 1. Create an alloca for the tile at the top of the function (if one
     // does not already exist).
     auto loc = tileOp.getLoc();
@@ -391,20 +396,6 @@ addArmSMEConversionPatterns(RewritePatternSet &patterns,
   (addArmSMEConversionPattern<Patterns>(patterns, typeConverter), ...);
 }
 
-struct GetTileConversion
-    : public ConvertArmSMEOpToLLVMPattern<arm_sme::GetTileOp,
-                                          RequiresSpillsAndFills::No> {
-  using ConvertArmSMEOpToLLVMPattern::ConvertArmSMEOpToLLVMPattern;
-
-  LogicalResult
-  matchAndRewrite(arm_sme::GetTileOp getTile, OpAdaptor,
-                  ConversionPatternRewriter &rewriter) const override {
-    rewriter.replaceOpWithNewOp<arm_sme::MaterializeSSATileOp>(
-        getTile, getTile.getTileType());
-    return success();
-  }
-};
-
 /// Lower 'arm_sme.zero' to SME intrinsics.
 ///
 ///  BEFORE:
@@ -415,11 +406,11 @@ struct GetTileConversion
 ///  AFTER:
 ///  ```mlir
 ///     "arm_sme.intr.zero"() <{tile_mask = 17 : i32}> : () -> ()
-///     %v = arm_sme.materialize_ssa_tile : vector<[4]x[4]xi32>
+///     %v = arm_sme.get_tile : vector<[4]x[4]xi32>
 ///  ```
 ///
-///  The 'arm_sme.materialize_ssa_tile' (which models the return) will fold away
-///  once all ArmSME ops have been converted to LLVM intrinsics.
+///  The 'arm_sme.get_tile' (which models the return) will fold away once all
+///  ArmSME ops have been converted to LLVM intrinsics.
 struct ZeroOpConversion : public ConvertArmSMEOpToLLVMPattern<arm_sme::ZeroOp> {
   using ConvertArmSMEOpToLLVMPattern::ConvertArmSMEOpToLLVMPattern;
 
@@ -436,7 +427,8 @@ struct ZeroOpConversion : public ConvertArmSMEOpToLLVMPattern<arm_sme::ZeroOp> {
     // The base mask is just the mask to zero the first tile (of a size).
     // These masks are derived from:
     // https://developer.arm.com/documentation/ddi0602/2022-06/SME-Instructions/ZERO--Zero-a-list-of-64-bit-element-ZA-tiles-
-    arm_sme::ArmSMETileType tileType = *zero.getAllocatedTileType();
+    arm_sme::ArmSMETileType tileType =
+        *arm_sme::getSMETileType(zero.getTileType());
     auto baseMaskForSize = [&] {
       switch (tileType) {
       case arm_sme::ArmSMETileType::ZAB:
@@ -488,8 +480,7 @@ struct ZeroOpConversion : public ConvertArmSMEOpToLLVMPattern<arm_sme::ZeroOp> {
         loc, rewriter.getI32IntegerAttr(zeroMask));
 
     // Create a placeholder op to preserve dataflow.
-    rewriter.replaceOpWithNewOp<arm_sme::MaterializeSSATileOp>(
-        zero, zero.getVectorType());
+    rewriter.replaceOpWithNewOp<arm_sme::GetTileOp>(zero, zero.getVectorType());
 
     return success();
   }
@@ -746,10 +737,12 @@ struct OuterProductOpConversion
     auto loc = outerProductOp.getLoc();
 
     Value acc = outerProductOp.getAcc();
-    if (!acc)
+    if (!acc) {
       // Initalize accumulator with zero.
-      acc = outerProductOp.createOpAndForwardTileId<arm_sme::ZeroOp>(
-          rewriter, loc, resultVectorType);
+      auto zero = rewriter.create<arm_sme::ZeroOp>(loc, resultVectorType);
+      zero.setTileId(tileId);
+      acc = zero;
+    }
 
     Value lhsMask = outerProductOp.getLhsMask();
     Value rhsMask = outerProductOp.getRhsMask();
@@ -791,25 +784,27 @@ struct OuterProductWideningOpConversion
     if (!tileId)
       return failure();
 
+    auto loc = op.getLoc();
     Value acc = op.getAcc();
-    if (!acc)
+    if (!acc) {
       // Initalize accumulator with zero.
-      acc = op.template createOpAndForwardTileId<arm_sme::ZeroOp>(
-          rewriter, op.getLoc(), op.getResultType());
+      auto zero = rewriter.create<arm_sme::ZeroOp>(loc, op.getResultType());
+      zero.setTileId(tileId);
+      acc = zero;
+    }
 
     Value lhsMask = op.getLhsMask();
     Value rhsMask = op.getRhsMask();
     if (!lhsMask || !rhsMask) {
       auto predTy = op.getLhsType().cloneWith({}, rewriter.getI1Type());
       Value allActiveMask = rewriter.create<arith::ConstantOp>(
-          op.getLoc(), DenseElementsAttr::get(predTy, true));
+          loc, DenseElementsAttr::get(predTy, true));
       lhsMask = allActiveMask;
       rhsMask = allActiveMask;
     }
 
-    rewriter.create<OuterProductWideningIntrOp>(op.getLoc(), tileId, lhsMask,
-                                                rhsMask, adaptor.getLhs(),
-                                                adaptor.getRhs());
+    rewriter.create<OuterProductWideningIntrOp>(
+        loc, tileId, lhsMask, rhsMask, adaptor.getLhs(), adaptor.getRhs());
 
     // The outerproduct intrinsics have no result, replace
     // 'arm_sme.outerproduct' with the input tile to preserve dataflow.
@@ -865,15 +860,22 @@ namespace {
 
 struct ConvertArmSMEToLLVMPass
     : public impl::ConvertArmSMEToLLVMBase<ConvertArmSMEToLLVMPass> {
+  ConvertArmSMEToLLVMPass(bool dumpTileLiveRanges) {
+    this->dumpTileLiveRanges = dumpTileLiveRanges;
+  }
   void runOnOperation() override {
+    auto function = getOperation();
+
+    if (failed(arm_sme::allocateSMETiles(function, dumpTileLiveRanges)))
+      return signalPassFailure();
+
     LLVMConversionTarget target(getContext());
     RewritePatternSet patterns(&getContext());
     LLVMTypeConverter converter(&getContext());
     configureArmSMEToLLVMConversionLegality(target);
     populateArmSMEToLLVMConversionPatterns(converter, patterns);
 
-    if (failed(applyPartialConversion(getOperation(), target,
-                                      std::move(patterns))))
+    if (failed(applyPartialConversion(function, target, std::move(patterns))))
       signalPassFailure();
   }
 };
@@ -883,34 +885,38 @@ struct ConvertArmSMEToLLVMPass
 void mlir::configureArmSMEToLLVMConversionLegality(ConversionTarget &target) {
   target.addIllegalDialect<arm_sme::ArmSMEDialect>();
   target.addLegalOp<
-      arm_sme::MaterializeSSATileOp, arm_sme::aarch64_sme_zero,
-      arm_sme::aarch64_sme_str, arm_sme::aarch64_sme_ld1b_horiz,
-      arm_sme::aarch64_sme_ld1h_horiz, arm_sme::aarch64_sme_ld1w_horiz,
-      arm_sme::aarch64_sme_ld1d_horiz, arm_sme::aarch64_sme_ld1q_horiz,
-      arm_sme::aarch64_sme_st1b_horiz, arm_sme::aarch64_sme_st1h_horiz,
-      arm_sme::aarch64_sme_st1w_horiz, arm_sme::aarch64_sme_st1d_horiz,
-      arm_sme::aarch64_sme_st1q_horiz, arm_sme::aarch64_sme_ld1b_vert,
-      arm_sme::aarch64_sme_ld1h_vert, arm_sme::aarch64_sme_ld1w_vert,
-      arm_sme::aarch64_sme_ld1d_vert, arm_sme::aarch64_sme_ld1q_vert,
-      arm_sme::aarch64_sme_st1b_vert, arm_sme::aarch64_sme_st1h_vert,
-      arm_sme::aarch64_sme_st1w_vert, arm_sme::aarch64_sme_st1d_vert,
-      arm_sme::aarch64_sme_st1q_vert, arm_sme::aarch64_sme_read_horiz,
-      arm_sme::aarch64_sme_read_vert, arm_sme::aarch64_sme_write_horiz,
-      arm_sme::aarch64_sme_write_vert, arm_sme::aarch64_sme_mopa,
-      arm_sme::aarch64_sme_mopa_wide, arm_sme::aarch64_sme_mops_wide,
-      arm_sme::aarch64_sme_smopa_wide, arm_sme::aarch64_sme_smops_wide,
-      arm_sme::aarch64_sme_umopa_wide, arm_sme::aarch64_sme_umops_wide,
-      arm_sme::aarch64_sme_smopa_za32, arm_sme::aarch64_sme_smops_za32,
-      arm_sme::aarch64_sme_umopa_za32, arm_sme::aarch64_sme_umops_za32,
-      arm_sme::aarch64_sme_sumopa_wide, arm_sme::aarch64_sme_sumops_wide,
-      arm_sme::aarch64_sme_usmopa_wide, arm_sme::aarch64_sme_usmops_wide,
-      arm_sme::aarch64_sme_cntsb, arm_sme::aarch64_sme_cntsh,
-      arm_sme::aarch64_sme_cntsw, arm_sme::aarch64_sme_cntsd>();
+      arm_sme::aarch64_sme_zero, arm_sme::aarch64_sme_str,
+      arm_sme::aarch64_sme_ld1b_horiz, arm_sme::aarch64_sme_ld1h_horiz,
+      arm_sme::aarch64_sme_ld1w_horiz, arm_sme::aarch64_sme_ld1d_horiz,
+      arm_sme::aarch64_sme_ld1q_horiz, arm_sme::aarch64_sme_st1b_horiz,
+      arm_sme::aarch64_sme_st1h_horiz, arm_sme::aarch64_sme_st1w_horiz,
+      arm_sme::aarch64_sme_st1d_horiz, arm_sme::aarch64_sme_st1q_horiz,
+      arm_sme::aarch64_sme_ld1b_vert, arm_sme::aarch64_sme_ld1h_vert,
+      arm_sme::aarch64_sme_ld1w_vert, arm_sme::aarch64_sme_ld1d_vert,
+      arm_sme::aarch64_sme_ld1q_vert, arm_sme::aarch64_sme_st1b_vert,
+      arm_sme::aarch64_sme_st1h_vert, arm_sme::aarch64_sme_st1w_vert,
+      arm_sme::aarch64_sme_st1d_vert, arm_sme::aarch64_sme_st1q_vert,
+      arm_sme::aarch64_sme_read_horiz, arm_sme::aarch64_sme_read_vert,
+      arm_sme::aarch64_sme_write_horiz, arm_sme::aarch64_sme_write_vert,
+      arm_sme::aarch64_sme_mopa, arm_sme::aarch64_sme_mopa_wide,
+      arm_sme::aarch64_sme_mops_wide, arm_sme::aarch64_sme_smopa_wide,
+      arm_sme::aarch64_sme_smops_wide, arm_sme::aarch64_sme_umopa_wide,
+      arm_sme::aarch64_sme_umops_wide, arm_sme::aarch64_sme_smopa_za32,
+      arm_sme::aarch64_sme_smops_za32, arm_sme::aarch64_sme_umopa_za32,
+      arm_sme::aarch64_sme_umops_za32, arm_sme::aarch64_sme_sumopa_wide,
+      arm_sme::aarch64_sme_sumops_wide, arm_sme::aarch64_sme_usmopa_wide,
+      arm_sme::aarch64_sme_usmops_wide, arm_sme::aarch64_sme_cntsb,
+      arm_sme::aarch64_sme_cntsh, arm_sme::aarch64_sme_cntsw,
+      arm_sme::aarch64_sme_cntsd>();
   target.addLegalDialect<arith::ArithDialect,
                          /* The following are used to lower tile spills/fills */
                          vector::VectorDialect, scf::SCFDialect,
                          memref::MemRefDialect>();
-  target.addLegalOp<UnrealizedConversionCastOp>();
+  // Pseudo operations. These cannot be code-generated but may exist in the
+  // input IR, or be generated during the conversion. They need to be eliminated
+  // before the final conversion to LLVM IR (and likely will be due to DCE).
+  target.addLegalOp<arm_sme::GetTileOp, arm_sme::CopyTileOp,
+                    UnrealizedConversionCastOp>();
 }
 
 void mlir::populateArmSMEToLLVMConversionPatterns(LLVMTypeConverter &converter,
@@ -955,9 +961,10 @@ void mlir::populateArmSMEToLLVMConversionPatterns(LLVMTypeConverter &converter,
                                        arm_sme::aarch64_sme_usmopa_wide>,
       OuterProductWideningOpConversion<arm_sme::UsMops4WayOp,
                                        arm_sme::aarch64_sme_usmops_wide>,
-      ZeroOpConversion, GetTileConversion>(patterns, converter);
+      ZeroOpConversion>(patterns, converter);
 }
 
-std::unique_ptr<Pass> mlir::createConvertArmSMEToLLVMPass() {
-  return std::make_unique<ConvertArmSMEToLLVMPass>();
+std::unique_ptr<Pass>
+mlir::createConvertArmSMEToLLVMPass(bool dumpTileLiveRanges) {
+  return std::make_unique<ConvertArmSMEToLLVMPass>(dumpTileLiveRanges);
 }
diff --git a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
index 16b61c282749..9f55932c33af 100644
--- a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
+++ b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
@@ -196,12 +196,9 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
       // Initialize tile with zero to satisfy padding. Inactive cols will be
       // zeroed anyway since the loads use zeroing predication. For inactive
       // rows however, no load will occur so these need to be zeroed.
-      initTile = tileLoadOp.createOpAndForwardTileId<arm_sme::ZeroOp>(
-          rewriter, loc, tileType);
+      initTile = rewriter.create<arm_sme::ZeroOp>(loc, tileType);
     } else {
-      // Allocate a new SME tile.
-      initTile = tileLoadOp.createOpAndForwardTileId<arm_sme::GetTileOp>(
-          rewriter, loc, tileType);
+      initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
     }
 
     // Create a loop to load the active tile slices from memory.
@@ -212,10 +209,9 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
             Value currentTile) -> Value {
           // Create 'arm_sme.load_tile_slice' to load tile slice from memory
           // into tile.
-          return tileLoadOp.createOpAndForwardTileId<arm_sme::LoadTileSliceOp>(
-              rewriter, loc, tileType, tileLoadOp.getBase(), predicate,
-              currentTile, memrefIndices, tileSliceIndex,
-              tileLoadOp.getLayout());
+          return rewriter.create<arm_sme::LoadTileSliceOp>(
+              loc, tileType, tileLoadOp.getBase(), predicate, currentTile,
+              memrefIndices, tileSliceIndex, tileLoadOp.getLayout());
         });
 
     if (failed(forOp))
@@ -292,9 +288,7 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
     auto numColsI32 = rewriter.create<arith::IndexCastUIOp>(
         loc, rewriter.getI32Type(), numCols);
 
-    // Allocate a new SME tile.
-    auto initTile = tileLoadOp.createOpAndForwardTileId<arm_sme::GetTileOp>(
-        rewriter, loc, tileType);
+    auto initTile = rewriter.create<arm_sme::GetTileOp>(loc, tileType);
 
     // Create a loop that loads each ZA tile slice from memory.
     auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
@@ -339,10 +333,9 @@ struct TileLoadOpWithMaskAndPadNonZeroConversion
         /*passthru=*/pad1DOp);
 
     // Create 'arm_sme.move_vector_to_tile_slice' to move slice into tile.
-    auto moveSlice =
-        tileLoadOp.createOpAndForwardTileId<arm_sme::MoveVectorToTileSliceOp>(
-            rewriter, loc, tileType, loadSlice->getResult(0), currentTile,
-            tileSliceIndex, tileLoadOp.getLayout());
+    auto moveSlice = rewriter.create<arm_sme::MoveVectorToTileSliceOp>(
+        loc, tileType, loadSlice->getResult(0), currentTile, tileSliceIndex,
+        tileLoadOp.getLayout());
     rewriter.create<scf::YieldOp>(loc, moveSlice.getResult());
 
     rewriter.setInsertionPointAfter(forOp);
@@ -386,8 +379,8 @@ struct TileStoreOpConversion : public OpRewritePattern<arm_sme::TileStoreOp> {
         tileStoreOp.getIndices(), tileStoreOp.getMemRefType().getRank(),
         tileStoreOp.getMask(),
         [&](Value tileSliceIndex, ValueRange memrefIndices, Value predicate) {
-          tileStoreOp.replaceWithAndForwardTileId<arm_sme::StoreTileSliceOp>(
-              rewriter, tileStoreOp.getValueToStore(), tileSliceIndex,
+          rewriter.replaceOpWithNewOp<arm_sme::StoreTileSliceOp>(
+              tileStoreOp, tileStoreOp.getValueToStore(), tileSliceIndex,
               predicate, tileStoreOp.getBase(), memrefIndices,
               tileStoreOp.getLayout());
         });
diff --git a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
index 29fa9085a0a9..cb3a66584487 100644
--- a/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/ArmSME.cpp
@@ -20,6 +20,12 @@
 using namespace mlir;
 using namespace mlir::arm_sme;
 
+namespace mlir::arm_sme::detail {
+LogicalResult verifyArmSMETileOpInterface(Operation *op) {
+  return verifyOperationHasValidTileId(op);
+}
+} // namespace mlir::arm_sme::detail
+
 //===----------------------------------------------------------------------===//
 // Tablegen Definitions
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
index 6a9e02218222..1f7305a5f814 100644
--- a/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
+++ b/mlir/lib/Dialect/ArmSME/IR/Utils.cpp
@@ -116,4 +116,57 @@ VectorType getSMETileTypeForElement(Type elementType) {
   return VectorType::get({minNumElts, minNumElts}, elementType, {true, true});
 }
 
+void eraseTriviallyDeadTileOps(IRRewriter &rewriter,
+                               FunctionOpInterface function) {
+  SmallVector<Operation *> worklist;
+  function->walk([&](Operation *op) {
+    auto armSMEOp = dyn_cast<arm_sme::ArmSMETileOpInterface>(op);
+    if (armSMEOp && isOpTriviallyDead(armSMEOp))
+      worklist.push_back(armSMEOp);
+  });
+  while (!worklist.empty()) {
+    Operation *op = worklist.pop_back_val();
+    if (!isOpTriviallyDead(op))
+      continue;
+    for (Value value : op->getOperands()) {
+      if (auto armSMEOp = value.getDefiningOp<arm_sme::ArmSMETileOpInterface>())
+        worklist.push_back(armSMEOp);
+    }
+    rewriter.eraseOp(op);
+  }
+}
+
+bool isTriviallyCloneableTileOp(arm_sme::ArmSMETileOpInterface tileOp) {
+  return tileOp && tileOp->getNumResults() == 1 &&
+         tileOp->getNumOperands() == 0 && isPure(tileOp);
+}
+
+bool hasTileResult(arm_sme::ArmSMETileOpInterface tileOp) {
+  for (Value result : tileOp->getResults()) {
+    if (arm_sme::isValidSMETileVectorType(result.getType()))
+      return true;
+  }
+  return false;
+}
+
+OpOperand *getTileOpOperand(arm_sme::ArmSMETileOpInterface tileOp) {
+  if (!tileOp)
+    return nullptr;
+  auto isTileOperandType = [](OpOperand &operand) {
+    return arm_sme::isValidSMETileVectorType(operand.get().getType());
+  };
+  assert(llvm::count_if(tileOp->getOpOperands(), isTileOperandType) <= 1 &&
+         "expected at most one tile operand");
+  OpOperand *tileOperand =
+      llvm::find_if(tileOp->getOpOperands(), isTileOperandType);
+  if (tileOperand == tileOp->getOpOperands().end())
+    return nullptr;
+  return tileOperand;
+}
+
+bool isTileTypeGreaterOrEqual(ArmSMETileType typeA, ArmSMETileType typeB) {
+  // Note: This is <= due to how tile types are numbered in ArmSMEOps.td.
+  return static_cast<unsigned>(typeA) <= static_cast<unsigned>(typeB);
+}
+
 } // namespace mlir::arm_sme
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
index 4acb2a8fb7b5..1e1e0e569124 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
@@ -6,12 +6,18 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This pass allocates SME tiles at the 'func.func' op level for ArmSME
-// operations. It does this using a 16-bit tile mask that has a bit for each
-// 128-bit element tile (ZA0.Q-ZA15.Q), the smallest ZA tile granule.
+// This transform allocates SME tiles at the 'func.func' op level for ArmSME
+// operations. It roughly implements a linear scan register allocator, similar
+// to the one outlined in [1], but with simplifications and assumptions made for
+// our use case. Note that this is a greedy allocator (so it may not always find
+// the most optimal allocation of tiles).
+//
+// The allocator operates at the CF dialect level. It is the responsibility of
+// users to ensure the IR has been lowered to CF before invoking the tile
+// allocator.
 //
 // The 128-bit tiles overlap with other element tiles as follows (see section
-// B2.3.2 of SME spec [1]):
+// B2.3.2 of SME spec [2]):
 //
 //   Tile    Overlaps
 //   ---------------------------------------------------------------------------
@@ -32,39 +38,34 @@
 //   ZA6.D   ZA6.Q, ZA14.Q
 //   ZA7.D   ZA7.Q, ZA15.Q
 //
-// The tiles in use are tracked via a function attribute 'arm_sme.tiles_in_use'
-// that is initalized during the first tile allocation within a function and
-// updated on each subsequent allocation.
-//
-// [1] https://developer.arm.com/documentation/ddi0616/aa
+// [1] "Linear Scan Register Allocation in the Context of SSA Form and Register
+//      Constraints" (Hanspeter Mössenböck and Michael Pfeiffer)
+//     https://link.springer.com/content/pdf/10.1007/3-540-45937-5_17.pdf
+// [2] https://developer.arm.com/documentation/ddi0616/aa
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Analysis/Liveness.h"
 #include "mlir/Dialect/ArmSME/IR/ArmSME.h"
 #include "mlir/Dialect/ArmSME/Transforms/Passes.h"
+#include "mlir/Dialect/ArmSME/Transforms/Transforms.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "mlir/Transforms/RegionUtils.h"
+#include "llvm/ADT/IntervalMap.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include <algorithm>
 
-#define DEBUG_TYPE "allocate-arm-sme-tiles"
-
-namespace mlir {
-namespace arm_sme {
-#define GEN_PASS_DEF_TILEALLOCATION
+namespace mlir::arm_sme {
+#define GEN_PASS_DEF_TESTTILEALLOCATION
 #include "mlir/Dialect/ArmSME/Transforms/Passes.h.inc"
-} // namespace arm_sme
-} // namespace mlir
+} // namespace mlir::arm_sme
 
 using namespace mlir;
 using namespace mlir::arm_sme;
 
 namespace {
 
-static constexpr StringLiteral kTilesInUseAttr("arm_sme.tiles_in_use");
-static constexpr StringLiteral
-    kNextInMemoryTileIdAttr("arm_sme.next_in_memory_tile_id");
-
 enum class TileMask : unsigned {
   // clang-format off
   kZA0B  = 0xffff, // 1111 1111 1111 1111
@@ -137,172 +138,640 @@ static ArrayRef<TileMask> getMasks(ArmSMETileType type) {
   }
 }
 
-/// Allocates and returns a tile ID. Returns an error if there are no tiles
-/// left.
-static FailureOr<unsigned> allocateTileId(ArmSMETileType tileType,
-                                          TileMask &tilesInUse) {
-  auto masks = getMasks(tileType);
-  for (auto [tileId, tileMask] : llvm::enumerate(masks)) {
-    if ((tilesInUse & tileMask) == TileMask::kNone) {
-      tilesInUse |= tileMask;
-      return tileId;
+class TileAllocator {
+public:
+  /// Allocates and returns a tile ID. Fails if there are no tiles left.
+  FailureOr<unsigned> allocateTileId(ArmSMETileType tileType) {
+    auto masks = getMasks(tileType);
+    for (auto [tileId, tileMask] : llvm::enumerate(masks)) {
+      if ((tilesInUse & tileMask) == TileMask::kNone) {
+        tilesInUse |= tileMask;
+        return tileId;
+      }
     }
+    return failure();
+  }
+
+  /// Releases a previously allocated tile ID.
+  void releaseTileId(ArmSMETileType tileType, unsigned tileId) {
+    TileMask tileMask = getMasks(tileType)[tileId];
+    assert((tilesInUse & tileMask) != TileMask::kNone &&
+           "cannot release unallocated tile!");
+    tilesInUse ^= tileMask;
   }
-  return failure();
-}
 
-/// Collects transitive uses of a root value through control flow. This can
-/// handle basic SCF constructs, along with control flow (br and cond_br).
-/// Simple loops work at the SCF level, while more complex control flow can be
-/// dealt with after lowering to CF. This is used to implement basic tile
-/// allocation.
-static void findDependantOps(Value rootValue,
-                             SetVector<Operation *> &dependantOps) {
-  auto traverseCorrespondingValues = [&](auto inputValues, auto exitValues) {
-    for (auto [idx, value] : llvm::enumerate(inputValues)) {
-      if (value == rootValue)
-        findDependantOps(exitValues[idx], dependantOps);
+  /// Allocates an in-memory tile ID.
+  unsigned allocateInMemoryTileId() {
+    // Note: We never release in-memory tile IDs. We could, which may allow
+    // reusing an allocation, but as we _never_ want to spill an SME tile this
+    // is not optimized.
+    return nextInMemoryTileId++;
+  }
+
+private:
+  TileMask tilesInUse = TileMask::kNone;
+  unsigned nextInMemoryTileId = kInMemoryTileIdBase;
+};
+
+/// Add new intermediate blocks for the true and false destinations of
+/// `cf.cond_br`s that contain tile operands. This prevents spurious liveness
+/// overlaps due to copies at branches.
+///
+///  BEFORE:
+///  ```mlir
+///  cf.cond_br %cond, ^bb1(%tile: vector<[4]x[4]xf32>), ^bb2
+///  ```
+///
+///  AFTER:
+///  ```mlir
+///    cf.cond_br %cond, ^bb1_copy, ^bb2_copy
+///  ^bb1_copy:
+///    cf.br ^bb1(%tile: vector<[4]x[4]xf32>)
+///  ^bb2_copy:
+///    cf.br ^bb2
+///  ```
+void splitCondBranches(IRRewriter &rewriter, FunctionOpInterface function) {
+  SmallVector<cf::CondBranchOp> worklist;
+  function.walk([&](cf::CondBranchOp condBranch) {
+    if (llvm::any_of(condBranch->getOperands(), [&](Value value) {
+          return isValidSMETileVectorType(value.getType());
+        })) {
+      worklist.push_back(condBranch);
     }
+  });
+
+  auto insertJump = [&](Location loc, Block *source, Block *dest, auto args) {
+    rewriter.setInsertionPointToEnd(source);
+    rewriter.create<cf::BranchOp>(loc, dest, args);
   };
-  for (Operation *user : rootValue.getUsers()) {
-    if (dependantOps.contains(user))
+
+  for (auto condBranch : worklist) {
+    auto loc = condBranch.getLoc();
+    Block *block = condBranch->getBlock();
+    auto newTrueBranch = rewriter.splitBlock(block, block->end());
+    auto newFalseBranch = rewriter.splitBlock(block, block->end());
+    insertJump(loc, newTrueBranch, condBranch.getTrueDest(),
+               condBranch.getTrueDestOperands());
+    insertJump(loc, newFalseBranch, condBranch.getFalseDest(),
+               condBranch.getFalseDestOperands());
+    rewriter.modifyOpInPlace(condBranch, [&] {
+      condBranch.getFalseDestOperandsMutable().clear();
+      condBranch.getTrueDestOperandsMutable().clear();
+      condBranch.setSuccessor(newTrueBranch, 0);
+      condBranch.setSuccessor(newFalseBranch, 1);
+    });
+  }
+}
+
+/// Inserts tile copies at `cf.br` operations.
+///
+///  BEFORE:
+///  ```mlir
+///  cf.br ^bb1(%tile: vector<[4]x[4]xf32>)
+///  ```
+///
+///  AFTER:
+///  ```mlir
+///  %copy = arm_sme.copy_tile %tile : vector<[4]x[4]xf32>
+///  cf.br ^bb1(%copy: vector<[4]x[4]xf32>)
+///  ```
+void insertCopiesAtBranches(IRRewriter &rewriter,
+                            FunctionOpInterface function) {
+  for (Block &block : function.getBlocks()) {
+    Operation *terminator = block.getTerminator();
+    if (!isa<cf::BranchOp>(terminator))
       continue;
-    dependantOps.insert(user);
-    TypeSwitch<Operation *>(user)
-        .Case<cf::BranchOp>([&](auto branchOp) {
-          // (CF) Follow branch.
-          traverseCorrespondingValues(branchOp.getDestOperands(),
-                                      branchOp.getDest()->getArguments());
-        })
-        .Case<cf::CondBranchOp>([&](auto condBranchOp) {
-          // (CF) Follow true branch.
-          traverseCorrespondingValues(
-              condBranchOp.getTrueOperands(),
-              condBranchOp.getTrueDest()->getArguments());
-          // (CF) Follow false branch.
-          traverseCorrespondingValues(
-              condBranchOp.getFalseOperands(),
-              condBranchOp.getFalseDest()->getArguments());
-        })
-        .Case<LoopLikeOpInterface>([&](auto loopOp) {
-          // (SCF) Follow iter_args of (basic) loops (e.g. for loops).
-          traverseCorrespondingValues(loopOp.getInits(),
-                                      loopOp.getRegionIterArgs());
-        })
-        .Case<scf::YieldOp>([&](auto yieldOp) {
-          // (SCF) Follow yields of (basic) control flow (e.g. for loops).
-          auto parent = user->getParentOp();
-          traverseCorrespondingValues(user->getOperands(),
-                                      parent->getResults());
+    rewriter.setInsertionPoint(terminator);
+    for (OpOperand &operand : terminator->getOpOperands()) {
+      if (isValidSMETileVectorType(operand.get().getType())) {
+        auto copy =
+            rewriter.create<CopyTileOp>(terminator->getLoc(), operand.get());
+        rewriter.modifyOpInPlace(terminator, [&] { operand.assign(copy); });
+      }
+    }
+  }
+}
+
+/// Prepares the IR for tile allocation. It does this by first 'splitting'
+/// conditional branches (see `splitCondBranches`), then inserting tile copies
+/// at branch operations. The conditional branches are split to prevent the
+/// copies needed for them overlapping between the true and false paths of the
+/// branch (see `tile-allocation-copies.mlir` and
+/// `tile-allocation-liveness.mlir` for examples). The copies break up live
+/// ranges and ensure when moving out of SSA the semantics of the program are
+/// preserved.
+void preprocessForTileAllocation(IRRewriter &rewriter,
+                                 FunctionOpInterface function) {
+  splitCondBranches(rewriter, function);
+  insertCopiesAtBranches(rewriter, function);
+}
+
+/// A live range for a (collection of) tile values. A live range is built up of
+/// non-overlapping intervals [start, end) which represent parts of the program
+/// where a value in the range needs to be live (i.e. in an SME virtual tile).
+/// Note that as the intervals are non-overlapping all values within a live
+/// range can be allocated to the same SME virtual tile.
+struct LiveRange {
+  using RangeSet = llvm::IntervalMap<uint64_t, uint8_t, 16,
+                                     llvm::IntervalMapHalfOpenInfo<unsigned>>;
+  using Allocator = RangeSet::Allocator;
+  // Dummy value for the IntervalMap. Only the keys matter (the intervals).
+  static constexpr uint8_t kValidLiveRange = 0xff;
+
+  LiveRange(Allocator &allocator)
+      : ranges(std::make_unique<RangeSet>(allocator)) {}
+
+  /// Returns true if this range overlaps with `otherRange`.
+  bool overlaps(LiveRange const &otherRange) const {
+    return llvm::IntervalMapOverlaps<RangeSet, RangeSet>(*ranges,
+                                                         *otherRange.ranges)
+        .valid();
+  }
+
+  /// Unions this live range with `otherRange`, aborts if the ranges overlap.
+  void unionWith(LiveRange const &otherRange) {
+    for (auto it = otherRange.ranges->begin(); it != otherRange.ranges->end();
+         ++it)
+      ranges->insert(it.start(), it.stop(), kValidLiveRange);
+    values.set_union(otherRange.values);
+  }
+
+  /// Inserts an interval [start, end) for `value` into this range.
+  void insert(Value value, unsigned start, unsigned end) {
+    values.insert(value);
+    if (start != end)
+      ranges->insert(start, end, kValidLiveRange);
+  }
+
+  bool empty() const { return ranges->empty(); }
+  unsigned start() const { return ranges->start(); }
+  unsigned end() const { return ranges->stop(); }
+  bool operator<(LiveRange const &other) const {
+    return start() < other.start();
+  }
+
+  ArmSMETileType getTileType() const {
+    return *getSMETileType(cast<VectorType>(values[0].getType()));
+  }
+
+  /// The values contained in this live range.
+  SetVector<Value> values;
+
+  /// A set of (non-overlapping) intervals that mark where any value in `values`
+  /// is live.
+  std::unique_ptr<RangeSet> ranges;
+
+  /// The tile ID (or none) assigned to this live range.
+  std::optional<unsigned> tileId;
+};
+
+/// Number operations within a function to allow computing live ranges.
+/// Operations are numbered consecutively wihin blocks, and the blocks are
+/// topologically sorted (using forward edges). This function is only correct if
+/// all ArmSME have been converted to CF (which is asserted).
+DenseMap<Operation *, unsigned>
+generateOperationNumbering(FunctionOpInterface function) {
+  unsigned index = 0;
+  SetVector<Block *> blocks =
+      getTopologicallySortedBlocks(function.getFunctionBody());
+  DenseMap<Operation *, unsigned> operationToIndexMap;
+  for (Block *block : blocks) {
+    index++; // We want block args to have their own number.
+    for (Operation &op : block->getOperations()) {
+#ifndef NDEBUG
+      op.walk([&](ArmSMETileOpInterface nestedOp) {
+        assert(&op == nestedOp.getOperation() &&
+               "ArmSME tile allocation does not support nested regions");
+      });
+#endif
+      operationToIndexMap.try_emplace(&op, index++);
+    }
+  }
+  return operationToIndexMap;
+}
+
+/// Gather live ranges for SME tiles from the MLIR liveness analysis.
+DenseMap<Value, LiveRange>
+gatherTileLiveRanges(DenseMap<Operation *, unsigned> const &operationToIndexMap,
+                     LiveRange::Allocator &liveRangeAllocator,
+                     Liveness &liveness, FunctionOpInterface function) {
+  assert(!operationToIndexMap.empty() && "expected operation numbering");
+  DenseMap<Value, LiveRange> liveRanges;
+  /// Defines or updates a live range for an SME tile value. Live-ins may update
+  /// an existing live range (rather than define a new one). Note: If
+  /// `liveAtBlockEntry` is true then `firstUseOrDef` is the first operation in
+  /// the block.
+  auto defineOrUpdateValueLiveRange = [&](Value value, Operation *firstUseOrDef,
+                                          LivenessBlockInfo const &livenessInfo,
+                                          bool liveAtBlockEntry = false) {
+    if (!isValidSMETileVectorType(value.getType()))
+      return;
+    // Find or create a live range for `value`.
+    auto [it, _] = liveRanges.try_emplace(value, liveRangeAllocator);
+    LiveRange &valueLiveRange = it->second;
+    auto lastUseInBlock = livenessInfo.getEndOperation(value, firstUseOrDef);
+    // Add the interval [firstUseOrDef, lastUseInBlock) to the live range.
+    unsigned startOpIdx =
+        operationToIndexMap.at(firstUseOrDef) + (liveAtBlockEntry ? -1 : 0);
+    unsigned endOpIdx = operationToIndexMap.at(lastUseInBlock);
+    valueLiveRange.insert(value, startOpIdx, endOpIdx);
+  };
+
+  for (Block &block : function.getBlocks()) {
+    LivenessBlockInfo const *livenessInfo = liveness.getLiveness(&block);
+    // Handle block arguments:
+    for (Value argument : block.getArguments())
+      defineOrUpdateValueLiveRange(argument, &block.front(), *livenessInfo,
+                                   /*liveAtBlockEntry=*/true);
+    // Handle live-ins:
+    for (Value liveIn : livenessInfo->in())
+      defineOrUpdateValueLiveRange(liveIn, &block.front(), *livenessInfo,
+                                   /*liveAtBlockEntry=*/true);
+    // Handle new definitions:
+    for (Operation &op : block) {
+      for (Value result : op.getResults())
+        defineOrUpdateValueLiveRange(result, &op, *livenessInfo);
+    }
+  }
+
+  return liveRanges;
+}
+
+/// Iterate over all predecessor tile values to a (tile) block argument.
+static void forEachPredecessorTileValue(BlockArgument blockArg,
+                                        function_ref<void(Value)> callback) {
+  Block *block = blockArg.getOwner();
+  unsigned argNumber = blockArg.getArgNumber();
+  for (Block *pred : block->getPredecessors()) {
+    TypeSwitch<Operation *>(pred->getTerminator())
+        .Case<cf::BranchOp>([&](auto branch) {
+          Value predecessorOperand = branch.getDestOperands()[argNumber];
+          callback(predecessorOperand);
         })
-        .Default([&](auto) {
-          // Otherwise, assume users of _any_ result are dependant.
-          for (Value result : user->getResults())
-            findDependantOps(result, dependantOps);
+        .Case<cf::CondBranchOp>([&](auto condBranch) {
+          if (condBranch.getFalseDest() == block) {
+            Value predecessorOperand =
+                condBranch.getFalseDestOperands()[argNumber];
+            callback(predecessorOperand);
+          }
+          if (condBranch.getTrueDest() == block) {
+            Value predecessorOperand =
+                condBranch.getTrueDestOperands()[argNumber];
+            callback(predecessorOperand);
+          }
         });
   }
 }
-struct AssignTileIDsPattern
-    : public OpInterfaceRewritePattern<ArmSMETileOpInterface> {
-  using OpInterfaceRewritePattern::OpInterfaceRewritePattern;
-  LogicalResult matchAndRewrite(ArmSMETileOpInterface tileOp,
-                                PatternRewriter &rewriter) const override {
-    if (tileOp.getTileId())
-      return failure();
-
-    auto func = tileOp->getParentOfType<FunctionOpInterface>();
-    auto getDiscardableIntAttr = [&](StringRef name, unsigned defaultVal = 0) {
-      if (auto attr = llvm::dyn_cast_or_null<IntegerAttr>(
-              func->getDiscardableAttr(name)))
-        return unsigned(attr.getInt());
-      return defaultVal;
-    };
-    auto setDiscardableIntAttr = [&](StringRef name, auto value) {
-      rewriter.modifyOpInPlace(tileOp, [&] {
-        func->setDiscardableAttr(name,
-                                 rewriter.getI32IntegerAttr((unsigned)value));
-      });
-    };
 
-    std::optional<ArmSMETileType> tileType = tileOp.getAllocatedTileType();
-    if (!tileType)
-      return rewriter.notifyMatchFailure(tileOp, "op does not allocate a tile");
-
-    TileMask tilesInUse =
-        static_cast<TileMask>(getDiscardableIntAttr(kTilesInUseAttr));
-    auto tileId = allocateTileId(*tileType, tilesInUse);
-    bool tileIsInMemory = failed(tileId);
-    if (tileIsInMemory) {
-      // If we could not find a real tile ID, use an in-memory tile ID (ID >=
-      // 16). A later pass will insert the necessary spills and reloads.
-      tileId =
-          getDiscardableIntAttr(kNextInMemoryTileIdAttr, kInMemoryTileIdBase);
-      tileOp->emitWarning(
-          "failed to allocate SME virtual tile to operation, all tile "
-          "operations will go through memory, expect degraded performance");
+/// Coalesce live ranges where it would prevent unnecessary tile moves.
+SmallVector<LiveRange *>
+coalesceTileLiveRanges(DenseMap<Value, LiveRange> &initialLiveRanges) {
+  DenseMap<Value, LiveRange *> liveRanges;
+  for (auto &[value, liveRange] : initialLiveRanges) {
+    liveRanges.insert({value, &liveRange});
+  }
+
+  // Merge the live ranges of values `a` and `b` into one (if they do not
+  // overlap). After this, the values `a` and `b` will both point to the same
+  // live range (which will contain multiple values).
+  auto mergeValuesIfNonOverlapping = [&](Value a, Value b) {
+    LiveRange *aLiveRange = liveRanges.at(a);
+    LiveRange *bLiveRange = liveRanges.at(b);
+    if (aLiveRange != bLiveRange && !aLiveRange->overlaps(*bLiveRange)) {
+      aLiveRange->unionWith(*bLiveRange);
+      for (Value value : bLiveRange->values)
+        liveRanges[value] = aLiveRange;
+    }
+  };
+
+  // Merge the live ranges of new definitions with their tile operands.
+  auto unifyDefinitionsWithOperands = [&](Value value) {
+    auto armSMEOp = value.getDefiningOp<ArmSMETileOpInterface>();
+    if (!armSMEOp)
+      return;
+    for (auto operand : armSMEOp->getOperands()) {
+      if (isValidSMETileVectorType(operand.getType()))
+        mergeValuesIfNonOverlapping(value, operand);
     }
+  };
+
+  // Merge the live ranges of block arguments with their predecessors.
+  auto unifyBlockArgumentsWithPredecessors = [&](Value value) {
+    auto blockArg = dyn_cast<BlockArgument>(value);
+    if (!blockArg)
+      return;
+    forEachPredecessorTileValue(blockArg, [&](Value predecessorTile) {
+      mergeValuesIfNonOverlapping(blockArg, predecessorTile);
+    });
+  };
+
+  auto applyRule = [&](auto rule) {
+    llvm::for_each(llvm::make_first_range(initialLiveRanges), rule);
+  };
+
+  // Unify as many live ranges as we can. This prevents unnecessary moves.
+  applyRule(unifyBlockArgumentsWithPredecessors);
+  applyRule(unifyDefinitionsWithOperands);
+
+  // Remove duplicate live range entries.
+  SetVector<LiveRange *> uniqueLiveRanges;
+  for (auto [_, liveRange] : liveRanges) {
+    if (!liveRange->empty())
+      uniqueLiveRanges.insert(liveRange);
+  }
+
+  // Sort the new live ranges by starting point (ready for tile allocation).
+  auto coalescedLiveRanges = uniqueLiveRanges.takeVector();
+  std::sort(coalescedLiveRanges.begin(), coalescedLiveRanges.end(),
+            [](LiveRange *a, LiveRange *b) { return *a < *b; });
+  return std::move(coalescedLiveRanges);
+}
+
+/// Choose a live range to spill (via some heuristics). This picks either an
+/// active live range from `activeRanges` or the new live range `newRange`.
+LiveRange *chooseSpillUsingHeuristics(ArrayRef<LiveRange *> activeRanges,
+                                      LiveRange *newRange) {
+  // Heuristic: Spill trivially copyable operations (usually free).
+  auto isTrivialSpill = [&](LiveRange *allocatedRange) {
+    return isTileTypeGreaterOrEqual(allocatedRange->getTileType(),
+                                    newRange->getTileType()) &&
+           allocatedRange->values.size() == 1 &&
+           isTriviallyCloneableTileOp(
+               allocatedRange->values[0]
+                   .getDefiningOp<ArmSMETileOpInterface>());
+  };
+  if (isTrivialSpill(newRange))
+    return newRange;
+  auto trivialSpill = llvm::find_if(activeRanges, isTrivialSpill);
+  if (trivialSpill != activeRanges.end())
+    return *trivialSpill;
+
+  // Heuristic: Spill the range that ends last (with a compatible tile type).
+  auto isSmallerTileTypeOrEndsEarlier = [](LiveRange *a, LiveRange *b) {
+    return !isTileTypeGreaterOrEqual(a->getTileType(), b->getTileType()) ||
+           a->end() < b->end();
+  };
+  LiveRange *lastActiveLiveRange = *std::max_element(
+      activeRanges.begin(), activeRanges.end(), isSmallerTileTypeOrEndsEarlier);
+  if (!isSmallerTileTypeOrEndsEarlier(lastActiveLiveRange, newRange))
+    return lastActiveLiveRange;
+  return newRange;
+}
+
+/// Greedily allocate tile IDs to live ranges. Spill using simple heuristics.
+/// Note: This does not attempt to fill holes in active live ranges.
+void allocateTilesToLiveRanges(
+    ArrayRef<LiveRange *> liveRangesSortedByStartPoint) {
+  TileAllocator tileAllocator;
+  SetVector<LiveRange *> activeRanges;
+  for (LiveRange *nextRange : liveRangesSortedByStartPoint) {
+    // Release tile IDs from live ranges that have ended.
+    activeRanges.remove_if([&](LiveRange *activeRange) {
+      if (activeRange->end() <= nextRange->start()) {
+        tileAllocator.releaseTileId(activeRange->getTileType(),
+                                    *activeRange->tileId);
+        return true;
+      }
+      return false;
+    });
 
-    // Set all operations dependent on `tileOp` to use the same tile ID.
-    // This is a naive tile allocation scheme, but works for common cases. For
-    // example, as this only allocates tile IDs to existing ops, it can't solve
-    // cases like this (%tileA and %tileB come from different root operations):
-    //
-    // %tile = scf.if %some_cond -> vector<[4]x[4]xi32> {
-    //   scf.yield %tileA {tile_id = 0} : vector<[4]x[4]xi32>
-    // } else {
-    //   scf.yield %tileB {tile_id = 1} : vector<[4]x[4]xi32>
-    // }
-    //
-    // This case would require allocating a new tile for the result of the
-    // scf.if, and moving the contents of %tileA or %tileB to result tile (based
-    // on the %some_cond).
-    // Find all the ops that (transitively) depend on this tile.
-    SetVector<Operation *> dependantOps;
-    findDependantOps(tileOp->getResult(0), dependantOps);
-    auto tileIDAttr = rewriter.getI32IntegerAttr(*tileId);
-    for (auto *op : dependantOps) {
-      if (auto dependantTileOp = llvm::dyn_cast<ArmSMETileOpInterface>(op)) {
-        auto currentTileId = dependantTileOp.getTileId();
-        if (currentTileId && unsigned(currentTileId.getInt()) != tileId)
-          return dependantTileOp.emitOpError(
-              "already assigned different SME virtual tile!");
+    // Allocate a tile ID to `nextRange`.
+    auto rangeTileType = nextRange->getTileType();
+    auto tileId = tileAllocator.allocateTileId(rangeTileType);
+    if (succeeded(tileId)) {
+      nextRange->tileId = *tileId;
+    } else {
+      LiveRange *rangeToSpill =
+          chooseSpillUsingHeuristics(activeRanges.getArrayRef(), nextRange);
+      if (rangeToSpill != nextRange) {
+        // Spill an active live range (so release its tile ID first).
+        tileAllocator.releaseTileId(rangeToSpill->getTileType(),
+                                    *rangeToSpill->tileId);
+        activeRanges.remove(rangeToSpill);
+        // This will always succeed after a spill (of an active live range).
+        nextRange->tileId = *tileAllocator.allocateTileId(rangeTileType);
       }
+      rangeToSpill->tileId = tileAllocator.allocateInMemoryTileId();
+    }
+
+    // Insert the live range into the active ranges.
+    if (nextRange->tileId < kInMemoryTileIdBase)
+      activeRanges.insert(nextRange);
+  }
+}
+
+/// Assigns a tile ID to an MLIR value.
+void assignTileIdToValue(IRRewriter &rewriter, Value value,
+                         IntegerAttr tileIdAttr) {
+  if (auto tileOp = value.getDefiningOp<ArmSMETileOpInterface>())
+    rewriter.modifyOpInPlace(tileOp, [&] { tileOp.setTileId(tileIdAttr); });
+  for (Operation *user : value.getUsers()) {
+    if (auto tileOp = dyn_cast<ArmSMETileOpInterface>(user)) {
+      // Ensure ArmSME ops that don't produce a value still get a tile ID.
+      if (!hasTileResult(tileOp))
+        rewriter.modifyOpInPlace(tileOp, [&] { tileOp.setTileId(tileIdAttr); });
     }
+  }
+}
+
+/// Assign tile IDs back to IR and attempt to resolve trivial tile ID conflicts.
+LogicalResult assignTileIdsAndResolveTrivialConflicts(
+    IRRewriter &rewriter, FunctionOpInterface function,
+    ArrayRef<LiveRange *> allocatedLiveRanges) {
+  for (LiveRange const *liveRange : allocatedLiveRanges) {
+    auto tileIdAttr = rewriter.getI32IntegerAttr(*liveRange->tileId);
+    auto isAllocatedToSameTile = [&](Value value) {
+      if (auto tileOp = value.getDefiningOp<ArmSMETileOpInterface>();
+          tileOp && tileOp.getTileId() == tileIdAttr)
+        return true;
+      return liveRange->values.contains(value);
+    };
+
+    /// Eliminates copies where the operand has the same tile ID.
+    auto foldRedundantCopies = [&](Value value) -> LogicalResult {
+      auto copyOp = value.getDefiningOp<CopyTileOp>();
+      if (!copyOp || !isAllocatedToSameTile(copyOp.getTile()))
+        return failure();
+      rewriter.replaceAllUsesWith(copyOp, copyOp.getTile());
+      return success();
+    };
+
+    /// Validates each predecessor to a tile block argument has been assigned
+    /// the same tile ID.
+    auto validateBlockArguments = [&](Value value) {
+      auto blockArg = dyn_cast<BlockArgument>(value);
+      if (!blockArg) {
+        // Not a block argument (nothing to validate).
+        return success();
+      }
+      bool tileMismatch = false;
+      forEachPredecessorTileValue(blockArg, [&](Value predecessorTile) {
+        if (tileMismatch)
+          return;
+        if (!isAllocatedToSameTile(predecessorTile)) {
+          blockArg.getOwner()->getParentOp()->emitOpError(
+              "block argument not allocated to the same SME virtial tile as "
+              "predecessors");
+          tileMismatch = true;
+        }
+      });
+      return success(/*isSuccess=*/!tileMismatch);
+    };
 
-    // Rewrite IR.
-    if (!tileIsInMemory)
-      setDiscardableIntAttr(kTilesInUseAttr, tilesInUse);
-    else
-      setDiscardableIntAttr(kNextInMemoryTileIdAttr, *tileId + 1);
-    rewriter.modifyOpInPlace(tileOp, [&] { tileOp.setTileId(tileIDAttr); });
-    for (auto *op : dependantOps) {
-      if (auto dependantTileOp = llvm::dyn_cast<ArmSMETileOpInterface>(op)) {
+    /// Attempts to resolve (trivial) tile ID conflicts.
+    auto resolveTrivialTileConflicts = [&](Value value) -> LogicalResult {
+      auto tileOp = value.getDefiningOp<ArmSMETileOpInterface>();
+      OpOperand *tileOperand = getTileOpOperand(tileOp);
+      if (!tileOperand || isAllocatedToSameTile(tileOperand->get())) {
+        // Operand already allocated to the correct tile.
+        // No conflict to resolve.
+        return success();
+      }
+      auto operandTileOp =
+          tileOperand->get().getDefiningOp<ArmSMETileOpInterface>();
+      if (!isTriviallyCloneableTileOp(operandTileOp)) {
+        auto error =
+            tileOp.emitOpError("tile operand allocated to different SME "
+                               "virtial tile (move required)");
+        error.attachNote(tileOperand->get().getLoc())
+            << "tile operand is: " << tileOperand->get();
+        return error;
+      }
+      // Cloning prevents a move/spill (though may require recomputation).
+      rewriter.setInsertionPoint(tileOp);
+      auto clonedOp = operandTileOp.clone();
+      rewriter.modifyOpInPlace(clonedOp,
+                               [&] { clonedOp.setTileId(tileOp.getTileId()); });
+      rewriter.insert(clonedOp);
+      if (isa<CopyTileOp>(tileOp)) {
+        rewriter.replaceAllUsesWith(tileOp->getResult(0),
+                                    clonedOp->getResult(0));
+      } else {
         rewriter.modifyOpInPlace(
-            dependantTileOp, [&] { dependantTileOp.setTileId(tileIDAttr); });
+            tileOp, [&] { tileOperand->assign(clonedOp->getResult(0)); });
       }
+      return success();
+    };
+
+    for (Value value : liveRange->values) {
+      // 1. Assign the tile ID to the value.
+      assignTileIdToValue(rewriter, value, tileIdAttr);
+
+      // 2. Attempt to eliminate redundant tile copies.
+      if (succeeded(foldRedundantCopies(value)))
+        continue;
+
+      // 3. Validate tile block arguments.
+      if (failed(validateBlockArguments(value)))
+        return failure();
+
+      // 4. Attempt to resolve (trivial) tile ID conflicts.
+      if (failed(resolveTrivialTileConflicts(value)))
+        return failure();
     }
+  }
+  return success();
+}
 
-    return success();
+/// Prints live ranges alongside operation names for debugging.
+void dumpLiveRanges(DenseMap<Operation *, unsigned> const &operationToIndexMap,
+                    ArrayRef<LiveRange const *> liveRanges,
+                    FunctionOpInterface function) {
+  llvm::errs() << "SME Tile Liveness: @" << function.getName()
+               << "\nKey:\nS - Start\nE - End\n| - Live\n";
+  for (auto [blockIdx, block] : llvm::enumerate(function.getBlocks())) {
+    llvm::errs() << "^bb" << blockIdx << ":\n";
+    for (Operation &op : block.getOperations()) {
+      unsigned operationIndex = operationToIndexMap.at(&op);
+      for (LiveRange const *range : liveRanges) {
+        char liveness = ' ';
+        for (auto it = range->ranges->begin(); it != range->ranges->end();
+             ++it) {
+          if (it.start() == operationIndex)
+            liveness = (liveness == 'E' ? '|' : 'S');
+          else if (it.stop() == operationIndex)
+            liveness = (liveness == 'S' ? '|' : 'E');
+          else if (operationIndex >= it.start() && operationIndex < it.stop())
+            liveness = '|';
+        }
+        llvm::errs() << liveness;
+      }
+      llvm::errs() << ' ' << op.getName() << '\n';
+    }
   }
-};
+  llvm::errs() << "==========\n";
+}
 
-struct TileAllocationPass
-    : public arm_sme::impl::TileAllocationBase<TileAllocationPass> {
+struct TestTileAllocationPass
+    : public arm_sme::impl::TestTileAllocationBase<TestTileAllocationPass> {
+  using TestTileAllocationBase::TestTileAllocationBase;
   void runOnOperation() override {
-    RewritePatternSet patterns(&getContext());
-    patterns.add<AssignTileIDsPattern>(patterns.getContext());
-    GreedyRewriteConfig config;
-    // Setting useTopDownTraversal ensures tiles are allocated in program
-    // order.
-    config.useTopDownTraversal = true;
-    if (mlir::failed(mlir::applyPatternsAndFoldGreedily(
-            getOperation(), std::move(patterns), config))) {
-      signalPassFailure();
+    FunctionOpInterface function = getOperation();
+    if (preprocessOnly) {
+      IRRewriter rewriter(function);
+      return preprocessForTileAllocation(rewriter, function);
     }
+    if (failed(arm_sme::allocateSMETiles(function, dumpTileLiveRanges)))
+      signalPassFailure();
   }
 };
 } // namespace
 
-std::unique_ptr<Pass> mlir::arm_sme::createTileAllocationPass() {
-  return std::make_unique<TileAllocationPass>();
+LogicalResult mlir::arm_sme::allocateSMETiles(FunctionOpInterface function,
+                                              bool dumpRanges) {
+  if (function.empty()) {
+    // TODO: Also return early if the function contains no ArmSME ops?
+    return success();
+  }
+
+  LiveRange::Allocator liveRangeAllocator;
+  IRRewriter rewriter(function.getContext());
+
+  // 1. Preprocess the IR for tile allocation.
+  preprocessForTileAllocation(rewriter, function);
+
+  // 2. Gather live ranges for each ArmSME tile within the function.
+  Liveness liveness(function);
+  auto operationToIndexMap = generateOperationNumbering(function);
+  auto initialLiveRanges = gatherTileLiveRanges(
+      operationToIndexMap, liveRangeAllocator, liveness, function);
+  if (initialLiveRanges.empty())
+    return success();
+
+  if (dumpRanges) {
+    // Wrangle initial live ranges into a form suitable for printing.
+    auto nonEmpty = llvm::make_filter_range(
+        llvm::make_second_range(initialLiveRanges),
+        [&](LiveRange const &liveRange) { return !liveRange.empty(); });
+    auto initialRanges = llvm::to_vector(llvm::map_range(
+        nonEmpty, [](LiveRange const &liveRange) { return &liveRange; }));
+    std::sort(initialRanges.begin(), initialRanges.end(),
+              [](LiveRange const *a, LiveRange const *b) { return *a < *b; });
+    llvm::errs() << "\n========== Initial Live Ranges:\n";
+    dumpLiveRanges(operationToIndexMap, initialRanges, function);
+  }
+
+  // 3. Coalesce (non-overlapping) live ranges where it would be beneficial
+  // for tile allocation. E.g. Unify the result of an operation with its
+  // operands.
+  auto coalescedLiveRanges = coalesceTileLiveRanges(initialLiveRanges);
+
+  if (dumpRanges) {
+    llvm::errs() << "\n========== Coalesced Live Ranges:\n";
+    dumpLiveRanges(operationToIndexMap, coalescedLiveRanges, function);
+  }
+
+  // 4. Allocate tile IDs to live ranges.
+  allocateTilesToLiveRanges(coalescedLiveRanges);
+
+  // 5. Assign the tile IDs back to the ArmSME operations.
+  if (failed(assignTileIdsAndResolveTrivialConflicts(rewriter, function,
+                                                     coalescedLiveRanges))) {
+    return failure();
+  }
+
+  // 6. Erase trivially dead tile operations (e.g. a ZeroOp with no
+  // users). This prevents the LLVM conversion needlessly inserting spills.
+  eraseTriviallyDeadTileOps(rewriter, function);
+  return success();
 }
diff --git a/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir b/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir
index f48046a8d799..14b1f323da3a 100644
--- a/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir
+++ b/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir
@@ -1,5 +1,4 @@
-// RUN: mlir-opt %s -allocate-arm-sme-tiles -convert-arm-sme-to-llvm -cse -canonicalize -split-input-file -verify-diagnostics | FileCheck %s
-
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(func.func(convert-arm-sme-to-llvm,cse,canonicalize))" -split-input-file | FileCheck %s
 // Test conversion of ArmSME ops to LLVM intrinsics.
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir b/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
index a9c1a65a296f..2c3868d7f25c 100644
--- a/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
+++ b/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt %s -allocate-arm-sme-tiles -split-input-file -verify-diagnostics | \
+// RUN: mlir-opt %s -test-arm-sme-tile-allocation -split-input-file | \
 // RUN: FileCheck %s  --check-prefix=AFTER-TILE-ALLOC
-// RUN: mlir-opt %s -allocate-arm-sme-tiles -convert-arm-sme-to-llvm -canonicalize -cse \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(func.func(convert-arm-sme-to-llvm,cse,canonicalize))" \
 // RUN:   -split-input-file -verify-diagnostics | \
 // RUN: FileCheck %s  --check-prefix=AFTER-LLVM-LOWERING
 
@@ -56,6 +56,9 @@ func.func @use_too_many_tiles() {
   %1 = arm_sme.zero : vector<[4]x[4]xi32>
   // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %2 = arm_sme.zero : vector<[8]x[8]xi16>
+  "test.some_use"(%0) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%1) : (vector<[4]x[4]xi32>) -> ()
+  "test.some_use"(%2) : (vector<[8]x[8]xi16>) -> ()
   return
 }
 // AFTER-TILE-ALLOC-LABEL: @use_too_many_tiles
@@ -131,18 +134,16 @@ func.func @use_too_many_tiles() {
 /// Note: In this example an entire tile swap is inserted before/after the
 /// `arm_sme.load_tile_slice` operation. Really, this only needs to spill a
 /// single tile slice (and can omit the initial load, like in the previous example).
-func.func @very_excessive_spills(%memref : memref<?x?xf32>) -> vector<[4]x[4]xf32> {
-  %useAllTiles = arm_sme.get_tile : vector<[16]x[16]xi8>
+func.func @very_excessive_spills(%useAllTiles : vector<[16]x[16]xi8>, %memref: memref<?x?xf32>) -> vector<[4]x[4]xf32> {
   %c0 = arith.constant 0 : index
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %tile = arm_sme.get_tile : vector<[4]x[4]xf32>
   %mask = vector.constant_mask [4] : vector<[4]xi1>
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %loadSlice = arm_sme.load_tile_slice %memref[%c0, %c0], %mask, %tile, %c0 : memref<?x?xf32>, vector<[4]xi1>, vector<[4]x[4]xf32>
-  "test.some_use"(%loadSlice) : (vector<[4]x[4]xf32>) -> ()
+  "test.some_use"(%useAllTiles) : (vector<[16]x[16]xi8>) -> ()
+  return %loadSlice : vector<[4]x[4]xf32>
 }
 // AFTER-TILE-ALLOC-LABEL: @very_excessive_spills
-//      AFTER-TILE-ALLOC: arm_sme.get_tile
-// AFTER-TILE-ALLOC-SAME:   tile_id = 0
 //      AFTER-TILE-ALLOC: arm_sme.load_tile_slice
 // AFTER-TILE-ALLOC-SAME:   tile_id = 16
 
diff --git a/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir b/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir
index 15767ff1dec3..a62ca080ab8d 100644
--- a/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir
+++ b/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -allocate-arm-sme-tiles -convert-arm-sme-to-llvm -split-input-file -allow-unregistered-dialect -verify-diagnostics
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(func.func(convert-arm-sme-to-llvm))" -verify-diagnostics
 
 //===----------------------------------------------------------------------===//
 // arm_sme.outerproduct
diff --git a/mlir/test/Dialect/ArmSME/basic-tile-allocation.mlir b/mlir/test/Dialect/ArmSME/basic-tile-allocation.mlir
index e144bac970a7..8b46998d56b0 100644
--- a/mlir/test/Dialect/ArmSME/basic-tile-allocation.mlir
+++ b/mlir/test/Dialect/ArmSME/basic-tile-allocation.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -allocate-arm-sme-tiles -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -test-arm-sme-tile-allocation -split-input-file | FileCheck %s
 
 // -----
 
diff --git a/mlir/test/Dialect/ArmSME/canonicalize.mlir b/mlir/test/Dialect/ArmSME/canonicalize.mlir
index b7ba3f728c70..643dfd4a7cbd 100644
--- a/mlir/test/Dialect/ArmSME/canonicalize.mlir
+++ b/mlir/test/Dialect/ArmSME/canonicalize.mlir
@@ -1,18 +1,14 @@
-// RUN: mlir-opt -canonicalize -split-input-file -verify-diagnostics %s | mlir-opt | FileCheck %s
+// RUN: mlir-opt %s -canonicalize | mlir-opt | FileCheck %s
 
-// This tests that the `arm_sme.materialize_ssa_tile` placeholder is removed
-// once it becomes unused, after lowering to control flow.
+// This tests that dead tile values are removed from control flow.
 
-// -----
-
-// CHECK-LABEL: @unused_materialize_ssa_tile_is_removed_from_blocks
-// CHECK-NOT: arm_sme.materialize_ssa_tile
+// CHECK-LABEL: @unused_ssa_tile_is_removed_from_blocks
 // CHECK-NOT: vector<[4]x[4]xf32>
-func.func @unused_materialize_ssa_tile_is_removed_from_blocks(%arg0: memref<?x?xi32>) {
+func.func @unused_ssa_tile_is_removed_from_blocks(%arg0: memref<?x?xi32>) {
   %c10 = arith.constant 10 : index
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
-  %tile = arm_sme.materialize_ssa_tile : vector<[4]x[4]xf32>
+  %tile = arm_sme.get_tile : vector<[4]x[4]xf32>
   cf.br ^bb1(%c0, %tile : index, vector<[4]x[4]xf32>)
 ^bb1(%1: index, %2: vector<[4]x[4]xf32>):  // 2 preds: ^bb0, ^bb2
   %3 = arith.cmpi slt, %1, %c10 : index
diff --git a/mlir/test/Dialect/ArmSME/cse.mlir b/mlir/test/Dialect/ArmSME/cse.mlir
deleted file mode 100644
index 74e7293eaeca..000000000000
--- a/mlir/test/Dialect/ArmSME/cse.mlir
+++ /dev/null
@@ -1,30 +0,0 @@
-// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='builtin.module(func.func(cse))' | FileCheck %s
-
-// This test is checking that CSE does not remove 'arm_sme.zero/get_tile' ops as
-// duplicates.
-
-// CHECK-LABEL: @zero_tile
-// CHECK: %[[TILE_0:.*]] = arm_sme.zero : vector<[4]x[4]xi32>
-// CHECK: %[[TILE_1:.*]] = arm_sme.zero : vector<[4]x[4]xi32>
-// CHECK: "prevent.dce"(%[[TILE_0]]) : (vector<[4]x[4]xi32>) -> ()
-// CHECK: "prevent.dce"(%[[TILE_1]]) : (vector<[4]x[4]xi32>) -> ()
-func.func @zero_tile() {
-  %tile_1 = arm_sme.zero : vector<[4]x[4]xi32>
-  %tile_2 = arm_sme.zero : vector<[4]x[4]xi32>
-  "prevent.dce"(%tile_1) : (vector<[4]x[4]xi32>) -> ()
-  "prevent.dce"(%tile_2) : (vector<[4]x[4]xi32>) -> ()
-  return
-}
-
-// CHECK-LABEL: @get_tile
-// CHECK: %[[TILE_0:.*]] = arm_sme.get_tile : vector<[4]x[4]xi32>
-// CHECK: %[[TILE_1:.*]] = arm_sme.get_tile : vector<[4]x[4]xi32>
-// CHECK: "prevent.dce"(%[[TILE_0]]) : (vector<[4]x[4]xi32>) -> ()
-// CHECK: "prevent.dce"(%[[TILE_1]]) : (vector<[4]x[4]xi32>) -> ()
-func.func @get_tile() {
-  %tile_1 = arm_sme.get_tile : vector<[4]x[4]xi32>
-  %tile_2 = arm_sme.get_tile : vector<[4]x[4]xi32>
-  "prevent.dce"(%tile_1) : (vector<[4]x[4]xi32>) -> ()
-  "prevent.dce"(%tile_2) : (vector<[4]x[4]xi32>) -> ()
-  return
-}
diff --git a/mlir/test/Dialect/ArmSME/roundtrip.mlir b/mlir/test/Dialect/ArmSME/roundtrip.mlir
index ab46c7adca59..6095fdc11ead 100644
--- a/mlir/test/Dialect/ArmSME/roundtrip.mlir
+++ b/mlir/test/Dialect/ArmSME/roundtrip.mlir
@@ -1403,3 +1403,12 @@ func.func @arm_sme_usmops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vect
   %reuslt = arm_sme.usmops_4way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[2]x[2]xi64>
   return %reuslt : vector<[2]x[2]xi64>
 }
+
+//===----------------------------------------------------------------------===//
+// arm_sme.copy_tile
+//===----------------------------------------------------------------------===//
+
+func.func @arm_sme_copy_tile(%vec: vector<[4]x[4]xf32>) -> vector<[4]x[4]xf32> {
+  %result = arm_sme.copy_tile %vec : vector<[4]x[4]xf32>
+  return %result : vector<[4]x[4]xf32>
+}
diff --git a/mlir/test/Dialect/ArmSME/tile-allocation-copies.mlir b/mlir/test/Dialect/ArmSME/tile-allocation-copies.mlir
new file mode 100644
index 000000000000..6d9cbf36a162
--- /dev/null
+++ b/mlir/test/Dialect/ArmSME/tile-allocation-copies.mlir
@@ -0,0 +1,159 @@
+// RUN: mlir-opt %s -test-arm-sme-tile-allocation=preprocess-only -split-input-file | FileCheck %s
+
+// This file tests the inserting copies for the SME tile allocation. Copies are
+// inserted at `cf.br` ops (the predecessors to block arguments). Conditional
+// branches are split to prevent conflicts (see cond_br_with_backedge).
+
+// CHECK-LABEL: func.func @simple_branch(
+//  CHECK-SAME:   %[[TILE:.*]]: vector<[4]x[4]xf32>)
+//   %[[COPY:.*]] = arm_sme.copy_tile %[[TILE]] : vector<[4]x[4]xf32>
+//   cf.br ^bb1(%[[COPY]] : vector<[4]x[4]xf32>)
+// ^bb1(%[[BLOCK_ARG:.*]]: vector<[4]x[4]xf32>):
+
+func.func @simple_branch(%tile : vector<[4]x[4]xf32>) {
+  cf.br ^bb1(%tile: vector<[4]x[4]xf32>)
+^bb1(%blockArg: vector<[4]x[4]xf32>):
+  return
+}
+
+// -----
+
+// Note: The ^POINTLESS_SHIM_FOR_BB2 block is added as the cond_br splitting does
+// not check if it needs to insert a copy or not (there is no harm in the empty
+// block though -- it will fold away later).
+
+// CHECK-LABEL: func.func @cond_branch(
+//  CHECK-SAME:   %[[COND:.*]]: i1, %[[TILE:.*]]: vector<[4]x[4]xf32>
+//       CHECK:   cf.cond_br %[[COND]], ^[[BB1_COPIES:[[:alnum:]]+]], ^[[POINTLESS_SHIM_FOR_BB2:[[:alnum:]]+]]
+//       CHECK: ^[[POINTLESS_SHIM_FOR_BB2]]:
+//       CHECK:   cf.br ^[[BB2:.*]]
+//       CHECK: ^[[BB1_COPIES]]:
+//       CHECK:   arm_sme.copy_tile %[[TILE]] : vector<[4]x[4]xf32>
+//       CHECK:   cf.br ^[[BB1:.*]]
+func.func @cond_branch(%cond: i1, %tile: vector<[4]x[4]xf32>) {
+  cf.cond_br %cond, ^bb1(%tile: vector<[4]x[4]xf32>), ^bb2
+^bb1(%blockArg: vector<[4]x[4]xf32>):
+  return
+^bb2:
+  return
+}
+
+// -----
+
+// Reduction of a real world example that shows why we must split conditional branches.
+
+// CHECK-LABEL: @cond_branch_with_backedge(
+//  CHECK-SAME:    %[[TILEA:[[:alnum:]]+]]: vector<[4]x[4]xf32>, %[[TILEB:[[:alnum:]]+]]: vector<[4]x[4]xf32>,
+//  CHECK-SAME:    %[[TILEC:[[:alnum:]]+]]: vector<[4]x[4]xf32>, %[[TILED:[[:alnum:]]+]]: vector<[4]x[4]xf32>,
+//       CHECK:   %[[BB1_COPY_0:.*]] = arm_sme.copy_tile %[[TILEA]] : vector<[4]x[4]xf32>
+//       CHECK:   cf.br ^bb1(%{{[[:alnum:]]+}}, %[[BB1_COPY_0]]
+//       CHECK: ^bb1(%[[CURRENT_INDEX:.*]]: index, %[[ITER_TILE:.*]]: vector<[4]x[4]xf32>):
+//       CHECK:   %[[CONTINUE_LOOP:.*]] = arith.cmpi
+//       CHECK:   cf.cond_br %[[CONTINUE_LOOP]], ^[[BB2_COPIES:[[:alnum:]]+]], ^[[BB3_COPIES:[[:alnum:]]+]]
+//       CHECK: ^[[BB3_COPIES]]:
+//  CHECK-NEXT:   %[[BB3_COPY_0:.*]] = arm_sme.copy_tile %[[ITER_TILE]] : vector<[4]x[4]xf32>
+//  CHECK-NEXT:   %[[BB3_COPY_1:.*]] = arm_sme.copy_tile %[[TILEB]] : vector<[4]x[4]xf32>
+//  CHECK-NEXT:   %[[BB3_COPY_2:.*]] = arm_sme.copy_tile %[[TILEC]] : vector<[4]x[4]xf32>
+//  CHECK-NEXT:   %[[BB3_COPY_3:.*]] = arm_sme.copy_tile %[[TILED]] : vector<[4]x[4]xf32>
+//  CHECK-NEXT:   cf.br ^[[BB3:[[:alnum:]]+]](%[[BB3_COPY_0]], %[[BB3_COPY_1]], %[[BB3_COPY_2]], %[[BB3_COPY_3]]
+//       CHECK: ^[[BB2_COPIES]]:
+//  CHECK-NEXT:   cf.br ^[[BB2:[[:alnum:]]+]]
+//       CHECK: ^[[BB2]]:
+//  CHECK-NEXT:   %[[NEXT_TILE:.*]] = arm_sme.move_vector_to_tile_slice %{{.*}}, %[[ITER_TILE]]
+//       CHECK:   %[[BB1_COPY_1:.*]] = arm_sme.copy_tile %[[NEXT_TILE]] : vector<[4]x[4]xf32>
+//       CHECK:   cf.br ^bb1(%{{[[:alnum:]]+}}, %[[BB1_COPY_1]]
+//       CHECK: ^[[BB3]](%{{.*}}: vector<[4]x[4]xf32>):
+//  CHECK-NEXT:   return
+func.func @cond_branch_with_backedge(%tileA: vector<[4]x[4]xf32>, %tileB: vector<[4]x[4]xf32>, %tileC: vector<[4]x[4]xf32>, %tileD: vector<[4]x[4]xf32>, %slice: vector<[4]xf32>) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c10 = arith.constant 10 : index
+  // Live here: %tileA, %tileB, %tileC, %tileD
+  cf.br ^bb1(%c0, %tileA : index, vector<[4]x[4]xf32>)
+^bb1(%currentIndex: index, %iterTile: vector<[4]x[4]xf32>):
+  %continueLoop = arith.cmpi slt, %currentIndex, %c10 : index
+  // Live here: %iterTile, %tileB, %tileC, %tileD
+  // %iterTile, %tileB, %tileC, %tileD are live out (in the ^bb2 case). If we
+  // inserted the (four) `arm_sme.copy_tile` operations here we would run out of tiles.
+  // However, note that the copies are only needed if we take the ^bb3 path. So, if we add
+  // a new block along that path we can insert the copies without any conflicts.
+  cf.cond_br %continueLoop, ^bb2, ^bb3(%iterTile, %tileB, %tileC, %tileD : vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>)
+^bb2:
+  // Live here: %iterTile, %tileB, %tileC, %tileD
+  %nextTile = arm_sme.move_vector_to_tile_slice %slice, %iterTile, %currentIndex : vector<[4]xf32> into vector<[4]x[4]xf32>
+  %nextIndex = arith.addi %currentIndex, %c1 : index
+  cf.br ^bb1(%nextIndex, %nextTile : index, vector<[4]x[4]xf32>)
+^bb3(%finalTileA: vector<[4]x[4]xf32>, %finalTileB: vector<[4]x[4]xf32>, %finalTileC: vector<[4]x[4]xf32>, %finalTileD: vector<[4]x[4]xf32>):
+  // Live here: %finalTileA, %finalTileB, %finalTileC, %finalTileD
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @tile_dominance
+// CHECK-NOT: arm_sme.copy_tile
+func.func @tile_dominance(%arg0: vector<[4]x[4]xf32>) {
+  cf.br ^bb1
+^bb1:  // 2 preds: ^bb0, ^bb4
+  "test.some_use"(%arg0) : (vector<[4]x[4]xf32>) -> ()
+  return
+^bb2:  // no predecessors
+  %0 = arm_sme.get_tile : vector<[4]x[4]xf32>
+  cf.br ^bb3
+^bb3:  // pred: ^bb2
+  "test.some_use"(%0) : (vector<[4]x[4]xf32>) -> ()
+  return
+^bb4:  // no predecessors
+  cf.br ^bb1
+^bb5:  // no predecessors
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @cond_branch_true_and_false_tile_args(
+//  CHECK-SAME:   %[[COND:.*]]: i1, %[[TILE:.*]]: vector<[4]x[4]xf32>
+//  CHECK-NEXT:   cf.cond_br %[[COND]], ^[[BB1_COPIES:[[:alnum:]]+]], ^[[BB2_COPIES:[[:alnum:]]+]]
+//       CHECK: ^[[BB2_COPIES]]:
+//  CHECK-NEXT:   %[[COPY_0:.*]] = arm_sme.copy_tile %[[TILE]] :  vector<[4]x[4]xf32>
+//  CHECK-NEXT:   cf.br ^[[BB2:[[:alnum:]]+]](%[[COPY_0]]
+//       CHECK: ^[[BB1_COPIES]]:
+//  CHECK-NEXT:   %[[COPY_1:.*]] = arm_sme.copy_tile %[[TILE]] :  vector<[4]x[4]xf32>
+//  CHECK-NEXT:   cf.br ^[[BB1:[[:alnum:]]+]](%[[COPY_1]]
+//       CHECK: ^[[BB1]]{{.*}}:
+//  CHECK-NEXT:   return
+//       CHECK: ^[[BB2]]{{.*}}:
+//  CHECK-NEXT:   return
+func.func @cond_branch_true_and_false_tile_args(%cond: i1, %tile: vector<[4]x[4]xf32>) {
+  cf.cond_br %cond, ^bb1(%tile: vector<[4]x[4]xf32>), ^bb2(%tile: vector<[4]x[4]xf32>)
+^bb1(%blockArg0: vector<[4]x[4]xf32>):
+  return
+^bb2(%blockArg1: vector<[4]x[4]xf32>):
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @multiple_predecessors
+//      CHECK: ^bb1:
+// CHECK-NEXT:   %[[TILE:.*]] = arm_sme.get_tile : vector<[4]x[4]xf32>
+// CHECK-NEXT:   %[[COPY_0:.*]] = arm_sme.copy_tile %[[TILE]] : vector<[4]x[4]xf32>
+// CHECK-NEXT:    cf.br ^bb3(%[[COPY_0]] : vector<[4]x[4]xf32>)
+//      CHECK: ^bb2:
+// CHECK-NEXT:   %[[ZERO:.*]] = arm_sme.zero : vector<[4]x[4]xf32>
+// CHECK-NEXT:   %[[COPY_1:.*]] = arm_sme.copy_tile %[[ZERO]] : vector<[4]x[4]xf32>
+// CHECK-NEXT:    cf.br ^bb3(%[[COPY_1]] : vector<[4]x[4]xf32>)
+//      CHECK: ^bb3({{.*}}):
+// CHECK-NEXT:  return
+func.func @multiple_predecessors(%cond: i1)
+{
+  cf.cond_br %cond, ^bb1, ^bb2
+^bb1:
+  %tile = arm_sme.get_tile : vector<[4]x[4]xf32>
+  cf.br ^bb3(%tile : vector<[4]x[4]xf32>)
+^bb2:
+  %zero = arm_sme.zero : vector<[4]x[4]xf32>
+  cf.br ^bb3(%zero : vector<[4]x[4]xf32>)
+^bb3(%blockArg: vector<[4]x[4]xf32>): // pred: ^bb1, ^bb2
+  return
+}
diff --git a/mlir/test/Dialect/ArmSME/tile-allocation-invalid.mlir b/mlir/test/Dialect/ArmSME/tile-allocation-invalid.mlir
index 39d9ab6491e3..6b5e44365bf5 100644
--- a/mlir/test/Dialect/ArmSME/tile-allocation-invalid.mlir
+++ b/mlir/test/Dialect/ArmSME/tile-allocation-invalid.mlir
@@ -1,19 +1,17 @@
-// RUN: mlir-opt %s -allocate-arm-sme-tiles -split-input-file -verify-diagnostics
+// RUN: mlir-opt %s -convert-scf-to-cf -test-arm-sme-tile-allocation -verify-diagnostics
 
-// -----
+// Select between tileA and tileB. This is currently unsupported as it would
+// require inserting (runtime) tile moves.
 
-func.func @selecting_between_different_tiles_is_unsupported(%dest : memref<?x?xi32>, %cond: i1) {
+// expected-note@below {{tile operand is: <block argument> of type 'vector<[4]x[4]xi32>'}}
+func.func @selecting_between_different_tiles_is_unsupported(%dest : memref<?x?xi32>, %tileA : vector<[4]x[4]xi32>, %tileB : vector<[4]x[4]xi32>, %cond: i1) {
   %c0 = arith.constant 0 : index
-  %tileA = arm_sme.get_tile : vector<[4]x[4]xi32>
-  %tileB = arm_sme.get_tile : vector<[4]x[4]xi32>
-  // Select between tileA and tileB. This is currently unsupported as it would
-  // require inserting tile move operations during tile allocation.
+  // expected-error@below {{op tile operand allocated to different SME virtial tile (move required)}}
   %tile = scf.if %cond -> vector<[4]x[4]xi32> {
     scf.yield %tileA : vector<[4]x[4]xi32>
   } else {
     scf.yield %tileB : vector<[4]x[4]xi32>
   }
-  // expected-error@+1 {{op already assigned different SME virtual tile!}}
   arm_sme.tile_store %tile, %dest[%c0, %c0] : memref<?x?xi32>, vector<[4]x[4]xi32>
   return
 }
diff --git a/mlir/test/Dialect/ArmSME/tile-allocation-liveness.mlir b/mlir/test/Dialect/ArmSME/tile-allocation-liveness.mlir
index 2dedcb2fbc24..88fc8a8923d3 100644
--- a/mlir/test/Dialect/ArmSME/tile-allocation-liveness.mlir
+++ b/mlir/test/Dialect/ArmSME/tile-allocation-liveness.mlir
@@ -1,18 +1,26 @@
-// RUN: mlir-opt %s -allocate-arm-sme-tiles -split-input-file -verify-diagnostics | FileCheck %s --check-prefix=CHECK-BAD
+// RUN: mlir-opt %s -convert-scf-to-cf -test-arm-sme-tile-allocation -split-input-file -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt %s -convert-scf-to-cf -test-arm-sme-tile-allocation=dump-tile-live-ranges -mlir-disable-threading -split-input-file -verify-diagnostics 2>&1 >/dev/null | FileCheck %s --check-prefix=CHECK-LIVE-RANGE
 
-// This file tests some aspects of liveness issues in the SME tile allocator.
-// These tests were designed with a new liveness-based tile allocator in mind
-// (where the names of test cases make more sense), with the current tile
-// allocator these tests all give incorrect results (which is documented by
-// `CHECK-BAD`).
+// This file tests some simple aspects of using liveness in the SME tile allocator.
+// Note: We use -convert-scf-to-cf first as the tile allocator expects CF, but
+// some of these tests are written in SCF (to make things easier to follow).
 
-// Incorrect result! The second `move_vector_to_tile_slice` overwrites the first (which is still live).
-//
-// CHECK-BAD-LABEL: @constant_with_multiple_users
-// CHECK-BAD: %[[ZERO_TILE:.*]] = arm_sme.zero {tile_id = 0 : i32} : vector<[4]x[4]xf32>
-// CHECK-BAD: %[[INSERT_TILE_1:.*]] = arm_sme.move_vector_to_tile_slice %{{.*}} {tile_id = 0 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
-// CHECK-BAD: %[[INSERT_TILE_0:.*]] = arm_sme.move_vector_to_tile_slice %{{.*}} {tile_id = 0 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
+//  CHECK-LIVE-RANGE-LABEL: @constant_with_multiple_users
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb0:
+//        CHECK-LIVE-RANGE: S  arm_sme.zero
+//   CHECK-LIVE-RANGE-NEXT: |S arm_sme.move_vector_to_tile_slice
+//   CHECK-LIVE-RANGE-NEXT: || arm_sme.move_vector_to_tile_slice
+//   CHECK-LIVE-RANGE-NEXT: |E test.some_use
+//   CHECK-LIVE-RANGE-NEXT: E  test.some_use
+
+// CHECK-LABEL: @constant_with_multiple_users(
+// CHECK-SAME:                                %[[VECTOR_A:.*]]: vector<[4]xf32>, %[[VECTOR_B:.*]]: vector<[4]xf32>
 func.func @constant_with_multiple_users(%a: vector<[4]xf32>, %b: vector<[4]xf32>, %index: index) {
+  // CHECK-NEXT: %[[ZERO_TILE_0:.*]] = arm_sme.zero {tile_id = 0 : i32} : vector<[4]x[4]xf32>
+  // CHECK-NEXT: %[[ZERO_TILE_1:.*]] = arm_sme.zero {tile_id = 1 : i32} : vector<[4]x[4]xf32>
+  // CHECK-NEXT: %[[INSERT_TILE_1:.*]] = arm_sme.move_vector_to_tile_slice %[[VECTOR_A]], %[[ZERO_TILE_1]], %{{.*}} {tile_id = 1 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
+  // CHECK-NEXT: %[[INSERT_TILE_0:.*]] = arm_sme.move_vector_to_tile_slice %[[VECTOR_B]], %[[ZERO_TILE_0]], %{{.*}} {tile_id = 0 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
   %zero = arm_sme.zero : vector<[4]x[4]xf32>
   %tile_a = arm_sme.move_vector_to_tile_slice %a, %zero, %index : vector<[4]xf32> into vector<[4]x[4]xf32>
   %tile_b = arm_sme.move_vector_to_tile_slice %b, %zero, %index : vector<[4]xf32> into vector<[4]x[4]xf32>
@@ -23,12 +31,17 @@ func.func @constant_with_multiple_users(%a: vector<[4]xf32>, %b: vector<[4]xf32>
 
 // -----
 
-// (No tile IDs -- the current tile allocator ignores this case)
+//  CHECK-LIVE-RANGE-LABEL: @value_with_multiple_users
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb0:
+//   CHECK-LIVE-RANGE-NEXT: |S arm_sme.move_vector_to_tile_slice
+//   CHECK-LIVE-RANGE-NEXT: || arm_sme.move_vector_to_tile_slice
+//   CHECK-LIVE-RANGE-NEXT: |E test.some_use
+//   CHECK-LIVE-RANGE-NEXT: E  test.some_use
 
-// CHECK-BAD-LABEL: @value_with_multiple_users
-// CHECK-BAD-NOT: tile_id
+// expected-note@below {{tile operand is: <block argument> of type 'vector<[4]x[4]xf32>'}}
 func.func @value_with_multiple_users(%tile: vector<[4]x[4]xf32>, %a: vector<[4]xf32>, %b: vector<[4]xf32>, %index: index) {
-  // A future allocator should error here (as `%tile` would need to be copied).
+  // expected-error@below {{op tile operand allocated to different SME virtial tile (move required)}}
   %tile_a = arm_sme.move_vector_to_tile_slice %a, %tile, %index : vector<[4]xf32> into vector<[4]x[4]xf32>
   %tile_b = arm_sme.move_vector_to_tile_slice %b, %tile, %index : vector<[4]xf32> into vector<[4]x[4]xf32>
   "test.some_use"(%tile_a) : (vector<[4]x[4]xf32>) -> ()
@@ -38,12 +51,38 @@ func.func @value_with_multiple_users(%tile: vector<[4]x[4]xf32>, %a: vector<[4]x
 
 // -----
 
-// CHECK-BAD-LABEL: @reuse_tiles_after_initial_use
+//  CHECK-LIVE-RANGE-LABEL: @reuse_tiles_after_initial_use
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb0:
+//   CHECK-LIVE-RANGE-NEXT: S        arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: |S       arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: ||S      arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: |||S     arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: ||||     test.dummy
+//   CHECK-LIVE-RANGE-NEXT: ||||     test.dummy
+//   CHECK-LIVE-RANGE-NEXT: ||||     test.dummy
+//   CHECK-LIVE-RANGE-NEXT: E|||     test.some_use
+//   CHECK-LIVE-RANGE-NEXT:  E||     test.some_use
+//   CHECK-LIVE-RANGE-NEXT:   E|     test.some_use
+//   CHECK-LIVE-RANGE-NEXT:    E     test.some_use
+//   CHECK-LIVE-RANGE-NEXT:     S    arm_sme.zero
+//   CHECK-LIVE-RANGE-NEXT:     |S   arm_sme.zero
+//   CHECK-LIVE-RANGE-NEXT:     ||S  arm_sme.zero
+//   CHECK-LIVE-RANGE-NEXT:     |||S arm_sme.zero
+//   CHECK-LIVE-RANGE-NEXT:     |||| test.dummy
+//   CHECK-LIVE-RANGE-NEXT:     |||| test.dummy
+//   CHECK-LIVE-RANGE-NEXT:     |||| test.dummy
+//   CHECK-LIVE-RANGE-NEXT:     E||| test.some_use
+//   CHECK-LIVE-RANGE-NEXT:      E|| test.some_use
+//   CHECK-LIVE-RANGE-NEXT:       E| test.some_use
+//   CHECK-LIVE-RANGE-NEXT:        E test.some_use
+
+// CHECK-LABEL: @reuse_tiles_after_initial_use
 func.func @reuse_tiles_after_initial_use() {
-  // CHECK-BAD: arm_sme.get_tile {tile_id = 0 : i32}
-  // CHECK-BAD: arm_sme.get_tile {tile_id = 1 : i32}
-  // CHECK-BAD: arm_sme.get_tile {tile_id = 2 : i32}
-  // CHECK-BAD: arm_sme.get_tile {tile_id = 3 : i32}
+  // CHECK: arm_sme.get_tile {tile_id = 0 : i32}
+  // CHECK: arm_sme.get_tile {tile_id = 1 : i32}
+  // CHECK: arm_sme.get_tile {tile_id = 2 : i32}
+  // CHECK: arm_sme.get_tile {tile_id = 3 : i32}
   %tile_a = arm_sme.get_tile : vector<[4]x[4]xf32>
   %tile_b = arm_sme.get_tile : vector<[4]x[4]xf32>
   %tile_c = arm_sme.get_tile : vector<[4]x[4]xf32>
@@ -55,19 +94,13 @@ func.func @reuse_tiles_after_initial_use() {
   "test.some_use"(%tile_b) : (vector<[4]x[4]xf32>) -> ()
   "test.some_use"(%tile_c) : (vector<[4]x[4]xf32>) -> ()
   "test.some_use"(%tile_d) : (vector<[4]x[4]xf32>) -> ()
-  // -> Spills after the fourth tile (unnecessary):
-  // CHECK-BAD: arm_sme.zero {tile_id = 16 : i32}
-  // CHECK-BAD: arm_sme.zero {tile_id = 17 : i32}
-  // CHECK-BAD: arm_sme.zero {tile_id = 18 : i32}
-  // CHECK-BAD: arm_sme.zero {tile_id = 19 : i32}
-  // Unnecessary spills:
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // CHECK: arm_sme.zero {tile_id = 0 : i32}
+  // CHECK: arm_sme.zero {tile_id = 1 : i32}
+  // CHECK: arm_sme.zero {tile_id = 2 : i32}
+  // CHECK: arm_sme.zero {tile_id = 3 : i32}
   %tile_1 = arm_sme.zero : vector<[4]x[4]xf32>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %tile_2 = arm_sme.zero : vector<[4]x[4]xf32>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %tile_3 = arm_sme.zero : vector<[4]x[4]xf32>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %tile_4 = arm_sme.zero : vector<[4]x[4]xf32>
   "test.dummy"(): () -> ()
   "test.dummy"(): () -> ()
@@ -81,16 +114,123 @@ func.func @reuse_tiles_after_initial_use() {
 
 // -----
 
-// Incorrect result! Both branches should yield the result via the same tile.
+//  CHECK-LIVE-RANGE-LABEL: @tile_live_ins
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb0:
+//   CHECK-LIVE-RANGE-NEXT: S  arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: |S arm_sme.zero
+//   CHECK-LIVE-RANGE-NEXT: EE cf.br
+//   CHECK-LIVE-RANGE-NEXT: ^bb1:
+//   CHECK-LIVE-RANGE-NEXT: || test.dummy
+//   CHECK-LIVE-RANGE-NEXT: || test.dummy
+//   CHECK-LIVE-RANGE-NEXT: EE cf.br
+//   CHECK-LIVE-RANGE-NEXT: ^bb2:
+//   CHECK-LIVE-RANGE-NEXT: || test.dummy
+//   CHECK-LIVE-RANGE-NEXT: || test.dummy
+//   CHECK-LIVE-RANGE-NEXT: EE cf.br
+//   CHECK-LIVE-RANGE-NEXT: ^bb3:
+//   CHECK-LIVE-RANGE-NEXT: E| test.some_use
+//   CHECK-LIVE-RANGE-NEXT:  E test.some_use
+
+// CHECK-LABEL: @tile_live_ins
+func.func @tile_live_ins()
+{
+  // CHECK: arm_sme.get_tile {tile_id = 0 : i32} : vector<[4]x[4]xf32>
+  // CHECK: arm_sme.zero {tile_id = 1 : i32} : vector<[4]x[4]xf32>
+  %tile_1 = arm_sme.get_tile : vector<[4]x[4]xf32>
+  %tile_2 = arm_sme.zero : vector<[4]x[4]xf32>
+  cf.br ^bb1
+^bb1:
+  "test.dummy"(): () -> ()
+  "test.dummy"(): () -> ()
+  cf.br ^bb2
+^bb2:
+  "test.dummy"(): () -> ()
+  "test.dummy"(): () -> ()
+  cf.br ^bb3
+^bb3:
+  "test.some_use"(%tile_1) : (vector<[4]x[4]xf32>) -> ()
+  "test.some_use"(%tile_2) : (vector<[4]x[4]xf32>) -> ()
+  return
+}
+
+// -----
+
+// This is basically the same test as tile_live_ins but shows that the order of
+// the blocks within the source does not relate to the liveness, which is based
+// on successors and predecessors (not textual order).
+//
+// So %tile_1 is live on the path bb0 -> bb2 -> bb1 (and dies in bb1). The
+// 'hole' when looking at the live range dump comes from the textual order
+// (and would disappear if bb1 was moved before bb2 in the source).
 //
-// CHECK-BAD-LABEL: @non_overlapping_branches
-// CHECK-BAD: arm_sme.zero {tile_id = 0 : i32} : vector<[4]x[4]xf32>
-// CHECK-BAD: arm_sme.get_tile {tile_id = 1 : i32} : vector<[4]x[4]xf32>
+// When looking at the live range dump (outside of straight-line code) it
+// normally makes more sense to consider blocks in isolation (and how they
+// relate to the CFG).
+
+//  CHECK-LIVE-RANGE-LABEL: @non_sequential_live_ins
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb0:
+//   CHECK-LIVE-RANGE-NEXT: S  arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: |  test.dummy
+//   CHECK-LIVE-RANGE-NEXT: E  cf.br
+//   CHECK-LIVE-RANGE-NEXT: ^bb1:
+//   CHECK-LIVE-RANGE-NEXT: E| test.some_use
+//   CHECK-LIVE-RANGE-NEXT:  | test.dummy
+//   CHECK-LIVE-RANGE-NEXT:  E cf.br
+//   CHECK-LIVE-RANGE-NEXT: ^bb2:
+//   CHECK-LIVE-RANGE-NEXT: |S arm_sme.zero
+//   CHECK-LIVE-RANGE-NEXT: || test.dummy
+//   CHECK-LIVE-RANGE-NEXT: EE cf.cond_br
+//   CHECK-LIVE-RANGE-NEXT: ^bb3:
+//   CHECK-LIVE-RANGE-NEXT:  | test.dummy
+//   CHECK-LIVE-RANGE-NEXT:  E test.some_use
+//   CHECK-LIVE-RANGE-NEXT:    func.return
+
+// CHECK-LABEL: @non_sequential_live_ins
+func.func @non_sequential_live_ins(%cond: i1) {
+  // CHECK: arm_sme.get_tile {tile_id = 0 : i32} : vector<[4]x[4]xf32>
+  // CHECK: arm_sme.zero {tile_id = 1 : i32} : vector<[4]x[4]xf32>
+  %tile_1 = arm_sme.get_tile : vector<[4]x[4]xf32>
+  "test.dummy"(): () -> ()
+  cf.br ^bb2
+^bb1:
+  "test.some_use"(%tile_1) : (vector<[4]x[4]xf32>) -> ()
+  "test.dummy"(): () -> ()
+  cf.br ^bb3
+^bb2:
+  %tile_2 = arm_sme.zero : vector<[4]x[4]xf32>
+  "test.dummy"(): () -> ()
+  cf.cond_br %cond, ^bb1, ^bb3
+^bb3:
+  "test.dummy"(): () -> ()
+  "test.some_use"(%tile_2) : (vector<[4]x[4]xf32>) -> ()
+  return
+}
+
+// -----
+
+//  CHECK-LIVE-RANGE-LABEL: @non_overlapping_branches
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb1:
+//   CHECK-LIVE-RANGE-NEXT: S arm_sme.zero
+//   CHECK-LIVE-RANGE-NEXT: | arm_sme.copy_tile
+//   CHECK-LIVE-RANGE-NEXT: E cf.br
+//   CHECK-LIVE-RANGE-NEXT: ^bb2:
+//   CHECK-LIVE-RANGE-NEXT: S arm_sme.get_tile
+//   CHECK-LIVE-RANGE-NEXT: | arm_sme.copy_tile
+//   CHECK-LIVE-RANGE-NEXT: E cf.br
+
+// CHECK-LABEL: @non_overlapping_branches
 func.func @non_overlapping_branches(%cond: i1) {
+  // CHECK: arm_sme.zero {tile_id = 0 : i32} : vector<[4]x[4]xf32>
+  // CHECK: arm_sme.get_tile {tile_id = 0 : i32} : vector<[4]x[4]xf32>
   %tile = scf.if %cond -> vector<[4]x[4]xf32> {
+    // ^bb1:
     %zero = arm_sme.zero : vector<[4]x[4]xf32>
     scf.yield %zero : vector<[4]x[4]xf32>
   } else {
+    // ^bb2:
     %undef = arm_sme.get_tile : vector<[4]x[4]xf32>
     scf.yield %undef : vector<[4]x[4]xf32>
   }
@@ -100,52 +240,65 @@ func.func @non_overlapping_branches(%cond: i1) {
 
 // -----
 
-// Incorrect result! Everything assigned to tile 0 (which means values that are still live are overwritten).
-//
-// CHECK-BAD-LABEL: @constant_loop_init_with_multiple_users
-// CHECK-BAD: arm_sme.zero {tile_id = 0 : i32} : vector<[4]x[4]xf32>
-// CHECK-BAD: arm_sme.move_vector_to_tile_slice {{.*}} {tile_id = 0 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
-// CHECK-BAD: arm_sme.move_vector_to_tile_slice {{.*}} {tile_id = 0 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
-func.func @constant_loop_init_with_multiple_users(%a: vector<[4]xf32>, %b: vector<[4]xf32>) {
-  %c0 = arith.constant 0 : index
-  %c1 = arith.constant 1 : index
-  %c10 = arith.constant 10 : index
-  %init = arm_sme.zero : vector<[4]x[4]xf32>
-  %tile_a = scf.for %i = %c0 to %c10 step %c1 iter_args(%iter = %init) -> vector<[4]x[4]xf32> {
-    %new_tile = arm_sme.move_vector_to_tile_slice %a, %iter, %i : vector<[4]xf32> into vector<[4]x[4]xf32>
-    scf.yield %new_tile : vector<[4]x[4]xf32>
-  }
-  %tile_b = scf.for %i = %c0 to %c10 step %c1 iter_args(%iter = %init) -> vector<[4]x[4]xf32> {
-    %new_tile = arm_sme.move_vector_to_tile_slice %a, %iter, %i : vector<[4]xf32> into vector<[4]x[4]xf32>
-    scf.yield %new_tile : vector<[4]x[4]xf32>
+// Here %vecA and %vecB are not merged into the same live range (as they are unknown values).
+// This means that %vecA and %vecB are both allocated to different tiles (which is not legal).
+
+// expected-note@below {{tile operand is: <block argument> of type 'vector<[4]x[4]xf32>'}}
+func.func @overlapping_branches(%cond: i1, %vecA: vector<[4]x[4]xf32>, %vecB: vector<[4]x[4]xf32>) {
+  // expected-error@below {{op tile operand allocated to different SME virtial tile (move required)}}
+  %tile = scf.if %cond -> vector<[4]x[4]xf32> {
+    scf.yield %vecA : vector<[4]x[4]xf32>
+  } else {
+    scf.yield %vecB : vector<[4]x[4]xf32>
   }
-  "test.some_use"(%tile_a) : (vector<[4]x[4]xf32>) -> ()
-  "test.some_use"(%tile_b) : (vector<[4]x[4]xf32>) -> ()
+  "test.some_use"(%tile) : (vector<[4]x[4]xf32>) -> ()
   return
 }
 
 // -----
 
-// Incorrect result! Everything assigned to tile 0 (which means values that are still live are overwritten).
-//
-// CHECK-BAD-LABEL: @run_out_of_tiles_but_avoid_spill
-// CHECK-BAD: arm_sme.zero {tile_id = 0 : i32}
-// CHECK-BAD-COUNT-4: arm_sme.move_vector_to_tile_slice {{.*}} {tile_id = 0 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
+//  CHECK-LIVE-RANGE-LABEL: @run_out_of_tiles_but_avoid_spill
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb2:
+//   CHECK-LIVE-RANGE-NEXT: |S    arm_sme.copy_tile
+//   CHECK-LIVE-RANGE-NEXT: ||S   arm_sme.copy_tile
+//   CHECK-LIVE-RANGE-NEXT: |||S  arm_sme.copy_tile
+//   CHECK-LIVE-RANGE-NEXT: ||||S arm_sme.copy_tile
+//   CHECK-LIVE-RANGE-NEXT: EEEEE cf.br
+
+// Note in the live ranges (above) there is five tile values, but we only have four tiles.
+// There is no 'real' spill as we spill the `arm_sme.zero` but are then able to clone it
+// at each of its uses.
+
+// CHECK-LABEL: @run_out_of_tiles_but_avoid_spill
 func.func @run_out_of_tiles_but_avoid_spill(%a: vector<[4]xf32>, %b: vector<[4]xf32>, %c: vector<[4]xf32>, %d: vector<[4]xf32>) {
   %init = arm_sme.zero : vector<[4]x[4]xf32>
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
   %c10 = arith.constant 10 : index
+  // Live = %init
   scf.for %i = %c0 to %c10 step %c1 {
+    // CHECK: arm_sme.zero {tile_id = 1 : i32}
+    // CHECK: arm_sme.zero {tile_id = 2 : i32}
+    // CHECK: arm_sme.zero {tile_id = 3 : i32}
+    // CHECK: arm_sme.zero {tile_id = 0 : i32}
     %tile_a, %tile_b, %tile_c, %tile_d = scf.for %j = %c0 to %c10 step %c1
       iter_args(%iter_a = %init, %iter_b = %init, %iter_c = %init, %iter_d = %init)
         -> (vector<[4]x[4]xf32>, vector<[4]x[4]xf32> , vector<[4]x[4]xf32> , vector<[4]x[4]xf32>) {
+        // ^bb2:
+        // CHECK: arm_sme.move_vector_to_tile_slice {{.*}} {tile_id = 1 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
+        // CHECK: arm_sme.move_vector_to_tile_slice {{.*}} {tile_id = 2 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
+        // CHECK: arm_sme.move_vector_to_tile_slice {{.*}} {tile_id = 3 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
+        // CHECK: arm_sme.move_vector_to_tile_slice {{.*}} {tile_id = 0 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
         %new_a = arm_sme.move_vector_to_tile_slice %a, %iter_a, %i : vector<[4]xf32> into vector<[4]x[4]xf32>
         %new_b = arm_sme.move_vector_to_tile_slice %b, %iter_b, %i : vector<[4]xf32> into vector<[4]x[4]xf32>
         %new_c = arm_sme.move_vector_to_tile_slice %c, %iter_c, %i : vector<[4]xf32> into vector<[4]x[4]xf32>
         %new_d = arm_sme.move_vector_to_tile_slice %d, %iter_d, %i : vector<[4]xf32> into vector<[4]x[4]xf32>
         scf.yield %new_a, %new_b, %new_c, %new_d : vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>
     }
+    // Live = %init, %tile_a, %tile_b, %tile_c, %tile_d (out of tiles!)
+    // This should be resolved by duplicating the arm_sme.zero (from folding
+    // arm_sme.copy_tile operations inserted by the tile allocator).
     "test.some_use"(%tile_a) : (vector<[4]x[4]xf32>) -> ()
     "test.some_use"(%tile_b) : (vector<[4]x[4]xf32>) -> ()
     "test.some_use"(%tile_c) : (vector<[4]x[4]xf32>) -> ()
@@ -156,24 +309,48 @@ func.func @run_out_of_tiles_but_avoid_spill(%a: vector<[4]xf32>, %b: vector<[4]x
 
 // -----
 
-// Incorrect result! Everything other than zero assigned to tile 1 (which means values that are still live are overwritten).
-//
-// CHECK-BAD-LABEL: @avoidable_spill
-// CHECK-BAD: arm_sme.zero {tile_id = 0 : i32}
-// CHECK-BAD: arm_sme.get_tile {tile_id = 1 : i32}
-// CHECK-BAD-COUNT-4: arm_sme.move_vector_to_tile_slice {{.*}} {tile_id = 1 : i32}
+// We should be able to avoid spills like this, but logic handling this case is
+// not implemented yet. Note tile ID >= 16 means a spill/in-memory tile.
+
+//  CHECK-LIVE-RANGE-LABEL: @avoidable_spill
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb2:
+//   CHECK-LIVE-RANGE-NEXT: ||     test.some_use
+//   CHECK-LIVE-RANGE-NEXT: ||S    arm_sme.move_vector_to_tile_slice
+//   CHECK-LIVE-RANGE-NEXT: |||S   arm_sme.move_vector_to_tile_slice
+//   CHECK-LIVE-RANGE-NEXT: ||||S  arm_sme.move_vector_to_tile_slice
+//   CHECK-LIVE-RANGE-NEXT: |||||S arm_sme.move_vector_to_tile_slice
+//   CHECK-LIVE-RANGE-NEXT: ||E||| test.some_use
+//   CHECK-LIVE-RANGE-NEXT: || E|| test.some_use
+//   CHECK-LIVE-RANGE-NEXT: ||  E| test.some_use
+//   CHECK-LIVE-RANGE-NEXT: ||   E test.some_use
+//   CHECK-LIVE-RANGE-NEXT: ||     arith.addi
+//   CHECK-LIVE-RANGE-NEXT: EE     cf.br
+
+// Note in the live ranges (above) there is two constant live-ins (first two ranges),
+// which gives six overlapping live ranges (at the point where %tile_d is defined).
+// The allocator currently will spill the first constant (which results in a real
+// spill at it's use), however, this could be avoided by using the knowledge that
+// at the first "test.some_use" there's actually only two live ranges (so we can
+// fix this be duplicating the constant).
+
+// CHECK-LABEL: @avoidable_spill
 func.func @avoidable_spill(%a: vector<[4]xf32>, %b: vector<[4]xf32>, %c: vector<[4]xf32>, %d: vector<[4]xf32>) {
+  // CHECK: arm_sme.zero {tile_id = 16 : i32} : vector<[4]x[4]xf32>
   %zero = arm_sme.zero : vector<[4]x[4]xf32>
   %tile = arm_sme.get_tile : vector<[4]x[4]xf32>
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
   %c10 = arith.constant 10 : index
   scf.for %i = %c0 to %c10 step %c1 {
+    // So spilled here (unnecessarily).
+    // The arm_sme.zero op could be moved into the loop to avoid this.
     "test.some_use"(%zero) : (vector<[4]x[4]xf32>) -> ()
     %tile_a = arm_sme.move_vector_to_tile_slice %a, %tile, %c0 : vector<[4]xf32> into vector<[4]x[4]xf32>
     %tile_b = arm_sme.move_vector_to_tile_slice %b, %tile, %c0 : vector<[4]xf32> into vector<[4]x[4]xf32>
     %tile_c = arm_sme.move_vector_to_tile_slice %c, %tile, %c0 : vector<[4]xf32> into vector<[4]x[4]xf32>
     %tile_d = arm_sme.move_vector_to_tile_slice %d, %tile, %c0 : vector<[4]xf32> into vector<[4]x[4]xf32>
+    // %zero is still live here (due the the backedge)
     "test.some_use"(%tile_a) : (vector<[4]x[4]xf32>) -> ()
     "test.some_use"(%tile_b) : (vector<[4]x[4]xf32>) -> ()
     "test.some_use"(%tile_c) : (vector<[4]x[4]xf32>) -> ()
@@ -181,3 +358,75 @@ func.func @avoidable_spill(%a: vector<[4]xf32>, %b: vector<[4]xf32>, %c: vector<
   }
   return
 }
+
+// -----
+
+// This test is a follow up to the test of the same name in `tile-allocation-copies.mlir`.
+// This shows the live ranges (which are why we need to split the conditional branch).
+
+//  CHECK-LIVE-RANGE-LABEL: @cond_branch_with_backedge
+//        CHECK-LIVE-RANGE: ^bb1:
+//  CHECK-LIVE-RANGE--NEXT:  ||| |           arith.cmpi
+//  CHECK-LIVE-RANGE--NEXT:  EEE E           cf.cond_br
+//
+//  CHECK-LIVE-RANGE--NEXT: ^[[BB3_COPIES:[[:alnum:]]+]]:
+//  CHECK-LIVE-RANGE--NEXT:  ||| ES          arm_sme.copy_tile
+//  CHECK-LIVE-RANGE--NEXT:  E||  |S         arm_sme.copy_tile
+//  CHECK-LIVE-RANGE--NEXT:   E|  ||S        arm_sme.copy_tile
+//  CHECK-LIVE-RANGE--NEXT:    E  |||S       arm_sme.copy_tile
+//  CHECK-LIVE-RANGE--NEXT:       EEEE       cf.br
+//
+// It is important to note that the first three live ranges in ^bb1 do not end
+// at the `cf.cond_br` they are live-out via the backedge bb1 -> bb2 -> bb1.
+// This means that if we placed the `arm_sme.tile_copies` before the `cf.cond_br`
+// then those live ranges would not end at the copies, resulting in unwanted
+// overlapping live ranges (and hence tile spills).
+//
+// With the conditional branch split and the copies placed in the BB3_COPIES
+// block the first three live ranges end at the copy operations (as the
+// BB3_COPIES block is on the path out of the loop and has no backedge). This
+// means there is no overlaps and the live ranges all merge, as shown below.
+//
+//        CHECK-LIVE-RANGE: ========== Coalesced Live Ranges:
+//        CHECK-LIVE-RANGE: ^bb1:
+//  CHECK-LIVE-RANGE--NEXT: |||| arith.cmpi
+//  CHECK-LIVE-RANGE--NEXT: EEEE cf.cond_br
+//
+//  CHECK-LIVE-RANGE--NEXT: ^[[BB3_COPIES]]:
+//  CHECK-LIVE-RANGE--NEXT: |||| arm_sme.copy_tile
+//  CHECK-LIVE-RANGE--NEXT: |||| arm_sme.copy_tile
+//  CHECK-LIVE-RANGE--NEXT: |||| arm_sme.copy_tile
+//  CHECK-LIVE-RANGE--NEXT: |||| arm_sme.copy_tile
+//  CHECK-LIVE-RANGE--NEXT: EEEE cf.br
+
+// CHECK-LABEL: @cond_branch_with_backedge
+// CHECK-NOT: tile_id = 16
+// CHECK: arm_sme.get_tile {tile_id = 0 : i32} : vector<[4]x[4]xf32>
+// CHECK: arm_sme.get_tile {tile_id = 1 : i32} : vector<[4]x[4]xf32>
+// CHECK: arm_sme.get_tile {tile_id = 2 : i32} : vector<[4]x[4]xf32>
+// CHECK: arm_sme.get_tile {tile_id = 3 : i32} : vector<[4]x[4]xf32>
+// CHECK: arm_sme.move_vector_to_tile_slice {{.*}} {tile_id = 0 : i32} : vector<[4]xf32> into vector<[4]x[4]xf32>
+// CHECK-NOT tile_id = 16
+func.func @cond_branch_with_backedge(%slice: vector<[4]xf32>) {
+  %tileA = arm_sme.get_tile : vector<[4]x[4]xf32>
+  %tileB = arm_sme.get_tile : vector<[4]x[4]xf32>
+  %tileC = arm_sme.get_tile : vector<[4]x[4]xf32>
+  %tileD = arm_sme.get_tile : vector<[4]x[4]xf32>
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c10 = arith.constant 10 : index
+  // Live here: %tileA, %tileB, %tileC, %tileD
+  cf.br ^bb1(%c0, %tileA : index, vector<[4]x[4]xf32>)
+^bb1(%currentIndex: index, %iterTile: vector<[4]x[4]xf32>):
+  %continueLoop = arith.cmpi slt, %currentIndex, %c10 : index
+  // Live here: %iterTile, %tileB, %tileC, %tileD
+  cf.cond_br %continueLoop, ^bb2, ^bb3(%iterTile, %tileB, %tileC, %tileD : vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>)
+^bb2:
+  // Live here: %iterTile, %tileB, %tileC, %tileD
+  %nextTile = arm_sme.move_vector_to_tile_slice %slice, %iterTile, %currentIndex : vector<[4]xf32> into vector<[4]x[4]xf32>
+  %nextIndex = arith.addi %currentIndex, %c1 : index
+  cf.br ^bb1(%nextIndex, %nextTile : index, vector<[4]x[4]xf32>)
+^bb3(%finalTileA: vector<[4]x[4]xf32>, %finalTileB: vector<[4]x[4]xf32>, %finalTileC: vector<[4]x[4]xf32>, %finalTileD: vector<[4]x[4]xf32>):
+  // Live here: %finalTileA, %finalTileB, %finalTileC, %finalTileD
+  return
+}
diff --git a/mlir/test/Dialect/ArmSME/tile-allocation-spills-with-mixed-tile-types.mlir b/mlir/test/Dialect/ArmSME/tile-allocation-spills-with-mixed-tile-types.mlir
new file mode 100644
index 000000000000..27757e29c1e2
--- /dev/null
+++ b/mlir/test/Dialect/ArmSME/tile-allocation-spills-with-mixed-tile-types.mlir
@@ -0,0 +1,38 @@
+
+// RUN: mlir-opt %s -test-arm-sme-tile-allocation -split-input-file | FileCheck %s
+
+// CHECK-LABEL: @always_spill_larger_or_equal_tile_type
+// CHECK: arm_sme.zero {tile_id = 0 : i32} : vector<[4]x[4]xf32>
+// CHECK: arm_sme.zero {tile_id = 1 : i32} : vector<[4]x[4]xf32>
+// CHECK: arm_sme.zero {tile_id = 2 : i32} : vector<[4]x[4]xf32>
+// CHECK: arm_sme.zero {tile_id = 3 : i32} : vector<[4]x[4]xf32>
+// CHECK: arm_sme.tile_load {{.*}} {tile_id = 16 : i32} : memref<?x?xf16>, vector<[8]x[8]xf16>
+func.func @always_spill_larger_or_equal_tile_type(%memref: memref<?x?xf16>) -> (vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[8]x[8]xf16>) {
+  %c0 = arith.constant 0 : index
+  %0 = arm_sme.zero : vector<[4]x[4]xf32>
+  %1 = arm_sme.zero : vector<[4]x[4]xf32>
+  %2 = arm_sme.zero : vector<[4]x[4]xf32>
+  %3 = arm_sme.zero : vector<[4]x[4]xf32>
+  // The load will be spilled (even though the zero's are 'trivial' spills) as a single `f32` tile would not fit the load.
+  %load = arm_sme.tile_load %memref[%c0, %c0] : memref<?x?xf16>, vector<[8]x[8]xf16>
+  return %0, %1, %2, %3, %load : vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[8]x[8]xf16>
+}
+
+// -----
+
+// CHECK-LABEL: @spill_larger_tile_type
+// CHECK: arm_sme.zero {tile_id = 16 : i32} : vector<[16]x[16]xi8>
+// CHECK: arm_sme.tile_load {{.*}} {tile_id = 0 : i32} : memref<?x?xf32>, vector<[4]x[4]xf32>
+// CHECK: arm_sme.tile_load {{.*}} {tile_id = 1 : i32} : memref<?x?xf32>, vector<[4]x[4]xf32>
+// CHECK: arm_sme.tile_load {{.*}} {tile_id = 2 : i32} : memref<?x?xf32>, vector<[4]x[4]xf32>
+// CHECK: arm_sme.tile_load {{.*}} {tile_id = 3 : i32} : memref<?x?xf32>, vector<[4]x[4]xf32>
+func.func @spill_larger_tile_type(%memref: memref<?x?xf32>) -> (vector<[16]x[16]xi8>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>) {
+  %c0 = arith.constant 0 : index
+  // Spilling the `arm_sme.zero` should free up space for all four f32 tiles.
+  %0 = arm_sme.zero : vector<[16]x[16]xi8>
+  %1 = arm_sme.tile_load %memref[%c0, %c0] : memref<?x?xf32>, vector<[4]x[4]xf32>
+  %2 = arm_sme.tile_load %memref[%c0, %c0] : memref<?x?xf32>, vector<[4]x[4]xf32>
+  %3 = arm_sme.tile_load %memref[%c0, %c0] : memref<?x?xf32>, vector<[4]x[4]xf32>
+  %4 = arm_sme.tile_load %memref[%c0, %c0] : memref<?x?xf32>, vector<[4]x[4]xf32>
+  return %0, %1, %2, %3, %4 : vector<[16]x[16]xi8>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>, vector<[4]x[4]xf32>
+}
diff --git a/mlir/test/Dialect/ArmSME/tile-zero-masks.mlir b/mlir/test/Dialect/ArmSME/tile-zero-masks.mlir
index cac2dcc24d10..ca339be5fb56 100644
--- a/mlir/test/Dialect/ArmSME/tile-zero-masks.mlir
+++ b/mlir/test/Dialect/ArmSME/tile-zero-masks.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -allocate-arm-sme-tiles -convert-arm-sme-to-llvm -canonicalize | FileCheck %s
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(func.func(convert-arm-sme-to-llvm,canonicalize))" | FileCheck %s
 
 // This test verifies the tile mask operand of the zero intrinsic zeroes
 // the correct tiles. Both integer and floating-point datatypes are checked.
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
index 588b44a36c29..14d9712e971a 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
@@ -13,26 +13,29 @@
 /// performance (hence the warning).
 func.func @use_too_many_tiles(%a: memref<?x?xi16>, %b:  memref<?x?xi16>, %c: memref<?x?xi16>) {
   %c0 = arith.constant 0 : index
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %tile_a = arith.constant dense<0> : vector<[8]x[8]xi16>
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %tile_b = arith.constant dense<1> : vector<[8]x[8]xi16>
   // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %tile_c = arm_sme.tile_load %a[%c0, %c0] : memref<?x?xi16>, vector<[8]x[8]xi16>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %tile_d = arm_sme.tile_load %b[%c0, %c0] : memref<?x?xi16>, vector<[8]x[8]xi16>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %tile_e = arm_sme.tile_load %c[%c0, %c0] : memref<?x?xi16>, vector<[8]x[8]xi16>
 
   // CHECK-LABEL: tile_a:
   // CHECK-COUNT-8: ( 0, 0, 0, 0, 0, 0, 0, 0
   vector.print str "tile_a:\n"
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   vector.print %tile_a : vector<[8]x[8]xi16>
   // CHECK-LABEL: tile_b:
   // CHECK-COUNT-8: ( 1, 1, 1, 1, 1, 1, 1, 1
   vector.print str "tile_b:\n"
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   vector.print %tile_b : vector<[8]x[8]xi16>
   // CHECK-LABEL: tile_c:
   // CHECK-COUNT-8: ( 2, 2, 2, 2, 2, 2, 2, 2
   vector.print str "tile_c:\n"
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   vector.print %tile_c : vector<[8]x[8]xi16>
   // CHECK-LABEL: tile_d:
   // CHECK-COUNT-8: ( 3, 3, 3, 3, 3, 3, 3, 3
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/test-setArmSVLBits.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/test-setArmSVLBits.mlir
index 1794564a6a72..0648e771b889 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/test-setArmSVLBits.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/Emulated/test-setArmSVLBits.mlir
@@ -1,5 +1,6 @@
 // DEFINE: %{entry_point} = main
-// DEFINE: %{compile} = mlir-opt %s -convert-arm-sme-to-llvm -test-lower-to-llvm
+// DEFINE: %{compile} = mlir-opt %s \
+// DEFINE: --pass-pipeline="builtin.module(func.func(convert-arm-sme-to-llvm),test-lower-to-llvm)"
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
diff --git a/mlir/test/lib/Dialect/ArmSME/CMakeLists.txt b/mlir/test/lib/Dialect/ArmSME/CMakeLists.txt
index e942c7b8ac05..cdd8afe14142 100644
--- a/mlir/test/lib/Dialect/ArmSME/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/ArmSME/CMakeLists.txt
@@ -15,4 +15,5 @@ add_mlir_library(MLIRArmSMETestPasses
   MLIRTransforms
   MLIRVectorToArmSME
   MLIRVectorToSCF
+  MLIRSCFToControlFlow
   )
diff --git a/mlir/test/lib/Dialect/ArmSME/TestLowerToArmSME.cpp b/mlir/test/lib/Dialect/ArmSME/TestLowerToArmSME.cpp
index 48d4a5859f8a..d3dabaf200fd 100644
--- a/mlir/test/lib/Dialect/ArmSME/TestLowerToArmSME.cpp
+++ b/mlir/test/lib/Dialect/ArmSME/TestLowerToArmSME.cpp
@@ -14,10 +14,12 @@
 #include "mlir/Conversion/ArithToArmSME/ArithToArmSME.h"
 #include "mlir/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.h"
 #include "mlir/Conversion/ArmSMEToSCF/ArmSMEToSCF.h"
+#include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h"
 #include "mlir/Conversion/VectorToArmSME/VectorToArmSME.h"
 #include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
 #include "mlir/Dialect/ArmSME/Transforms/Passes.h"
 #include "mlir/Dialect/ArmSVE/Transforms/Passes.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/DialectRegistry.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
@@ -34,6 +36,10 @@ struct TestLowerToArmSMEOptions
       llvm::cl::desc("Fuse outer product operations via "
                      "'-arm-sme-outer-product-fusion' pass"),
       llvm::cl::init(true)};
+  PassOptions::Option<bool> dumpTileLiveRanges{
+      *this, "dump-tile-live-ranges",
+      llvm::cl::desc("Dump the live ranges of SME tiles (for debugging)"),
+      llvm::cl::init(false)};
 };
 
 void buildTestLowerToArmSME(OpPassManager &pm,
@@ -65,20 +71,17 @@ void buildTestLowerToArmSME(OpPassManager &pm,
   pm.addPass(createConvertVectorToSCFPass(
       VectorTransferToSCFOptions().enableFullUnroll()));
 
-  // Allocate tiles for ArmSME operations.
-  //
-  // Later passes may create further ArmSME ops that implement the
-  // ArmSMETileOpInterface, but tiles are allocated for root operations,
-  // all of which should now exist.
-  pm.addPass(arm_sme::createTileAllocationPass());
-
   // Enable streaming-mode and ZA.
   pm.addPass(arm_sme::createEnableArmStreamingPass(
       arm_sme::ArmStreamingMode::StreamingLocally, arm_sme::ArmZaMode::NewZA,
       /*onlyIfRequiredByOps=*/true));
 
+  // Convert SCF to CF (required for ArmSME tile allocation).
+  pm.addPass(createConvertSCFToCFPass());
+
   // Convert ArmSME to LLVM.
-  pm.addPass(createConvertArmSMEToLLVMPass());
+  pm.addNestedPass<func::FuncOp>(
+      createConvertArmSMEToLLVMPass(options.dumpTileLiveRanges));
 
   // Sprinkle some cleanups.
   pm.addPass(createCanonicalizerPass());
-- 
GitLab


From 2b15c4a62be6ceab124cb2505ae8dc6a98ba6e7d Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter@arm.com>
Date: Tue, 14 May 2024 15:16:42 +0100
Subject: [PATCH 0737/1206] [AArch64] Postcommit fixes for histogram intrinsic
 (#92095)

A buildbot with expensive checks enabled flagged some problems with my patch. There was also a post-commit nit on the langref changes.
---
 llvm/docs/LangRef.rst                              |  4 ++--
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp    | 14 ++++++++------
 .../Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp |  2 +-
 .../CodeGen/AArch64/neon-scalarize-histogram.ll    |  2 +-
 llvm/test/CodeGen/AArch64/sve2-histcnt.ll          |  2 +-
 5 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 06809f8bf445..e2f4d8bfcaee 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -19143,8 +19143,8 @@ will be on any later loop iteration.
 This intrinsic will only return 0 if the input count is also 0. A non-zero input
 count will produce a non-zero result.
 
-'``llvm.experimental.vector.histogram.*``' Intrinsics
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+'``llvm.experimental.vector.histogram.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 These intrinsics are overloaded.
 
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 33cc8ffaf85d..f6d80f78910c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27395,9 +27395,11 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
   SDValue IncSplat = DAG.getSplatVector(MemVT, DL, Inc);
   SDValue Ops[] = {Chain, PassThru, Mask, Ptr, Index, Scale};
 
-  // Set the MMO to load only, rather than load|store.
-  MachineMemOperand *GMMO = HG->getMemOperand();
-  GMMO->setFlags(MachineMemOperand::MOLoad);
+  MachineMemOperand *MMO = HG->getMemOperand();
+  // Create an MMO for the gather, without load|store flags.
+  MachineMemOperand *GMMO = DAG.getMachineFunction().getMachineMemOperand(
+      MMO->getPointerInfo(), MachineMemOperand::MOLoad, MMO->getSize(),
+      MMO->getAlign(), MMO->getAAInfo());
   ISD::MemIndexType IndexType = HG->getIndexType();
   SDValue Gather =
       DAG.getMaskedGather(DAG.getVTList(MemVT, MVT::Other), MemVT, DL, Ops,
@@ -27412,10 +27414,10 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
   SDValue Mul = DAG.getNode(ISD::MUL, DL, MemVT, HistCnt, IncSplat);
   SDValue Add = DAG.getNode(ISD::ADD, DL, MemVT, Gather, Mul);
 
-  // Create a new MMO for the scatter.
+  // Create an MMO for the scatter, without load|store flags.
   MachineMemOperand *SMMO = DAG.getMachineFunction().getMachineMemOperand(
-      GMMO->getPointerInfo(), MachineMemOperand::MOStore, GMMO->getSize(),
-      GMMO->getAlign(), GMMO->getAAInfo());
+      MMO->getPointerInfo(), MachineMemOperand::MOStore, MMO->getSize(),
+      MMO->getAlign(), MMO->getAAInfo());
 
   SDValue ScatterOps[] = {GChain, Add, Mask, Ptr, Index, Scale};
   SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MemVT, DL,
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index de80fa2c0502..8f820a3bba2b 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -1006,7 +1006,7 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
                                            CI->getArgOperand(1)->getType()))
         return false;
       scalarizeMaskedVectorHistogram(DL, CI, DTU, ModifiedDT);
-      break;
+      return true;
     case Intrinsic::masked_load:
       // Scalarize unsupported vector masked load
       if (TTI.isLegalMaskedLoad(
diff --git a/llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll b/llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll
index 45f1429a810a..e59d9098a30d 100644
--- a/llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll
+++ b/llvm/test/CodeGen/AArch64/neon-scalarize-histogram.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=aarch64 < %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s -o - | FileCheck %s
 
 ;; This test exercises the default lowering of the histogram to scalarized code.
 
diff --git a/llvm/test/CodeGen/AArch64/sve2-histcnt.ll b/llvm/test/CodeGen/AArch64/sve2-histcnt.ll
index 557a42116cdb..db164e288abd 100644
--- a/llvm/test/CodeGen/AArch64/sve2-histcnt.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-histcnt.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=aarch64 < %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s -o - | FileCheck %s
 
 define void @histogram_i64(<vscale x 2 x ptr> %buckets, i64 %inc, <vscale x 2 x i1> %mask) #0 {
 ; CHECK-LABEL: histogram_i64:
-- 
GitLab


From d9db2664994ff672f50d7fd0117477935dac04f1 Mon Sep 17 00:00:00 2001
From: Jake Egan <Jake.egan@ibm.com>
Date: Tue, 14 May 2024 10:32:30 -0400
Subject: [PATCH 0738/1206] [PowerPC][test] Catch any exception when retrieving
 git revision (#92004)

This makes the `vc-rev-enabled` feature unsupported if we fail to
retrieve the git revision for any reason, such as if git is not
installed.
---
 llvm/test/CodeGen/PowerPC/lit.local.cfg | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/test/CodeGen/PowerPC/lit.local.cfg b/llvm/test/CodeGen/PowerPC/lit.local.cfg
index 4cc802afef4a..e56ce48eff40 100644
--- a/llvm/test/CodeGen/PowerPC/lit.local.cfg
+++ b/llvm/test/CodeGen/PowerPC/lit.local.cfg
@@ -9,8 +9,8 @@ def get_revision(repo_path):
     cmd = ['git', '-C', repo_path, 'rev-parse', 'HEAD']
     try:
         return subprocess.run(cmd, stdout=subprocess.PIPE, check=True).stdout.decode()
-    except subprocess.CalledProcessError:
-        print("An error occurred retrieving the git revision.")
+    except Exception as e:
+        print("An error occurred retrieving the git revision:", e)
         return None
 
 if config.have_vc_rev:
-- 
GitLab


From 8070b2defa6df1f1a3f3d4ed4989047b0e1bb639 Mon Sep 17 00:00:00 2001
From: Younan Zhang <zyn7109@gmail.com>
Date: Tue, 14 May 2024 22:44:01 +0800
Subject: [PATCH 0739/1206] [Clang] Retain the angle loci for invented template
 parameters of constraints (#92104)

Clangd uses it to determine whether the argument is within the selection
range.

Fixes https://github.com/clangd/clangd/issues/2033
---
 .../clangd/unittests/SelectionTests.cpp             |  6 ++++++
 clang/lib/Sema/SemaType.cpp                         |  3 ++-
 clang/test/AST/ast-dump-concepts.cpp                | 13 +++++++++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/clang-tools-extra/clangd/unittests/SelectionTests.cpp b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
index db516a1f62a3..aaaf758e7223 100644
--- a/clang-tools-extra/clangd/unittests/SelectionTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
@@ -589,6 +589,12 @@ TEST(SelectionTest, CommonAncestor) {
         auto x = [[ns::^C<int>]];
       )cpp",
        "ConceptReference"},
+      {R"cpp(
+        template <typename T, typename K>
+        concept D = true;
+        template <typename T> void g(D<[[^T]]> auto abc) {}
+      )cpp",
+       "TemplateTypeParmTypeLoc"},
   };
 
   for (const Case &C : Cases) {
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index d65fafc8cf4f..eb67546d048a 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -3099,7 +3099,8 @@ InventTemplateParameter(TypeProcessingState &state, QualType T,
       // The 'auto' appears in the decl-specifiers; we've not finished forming
       // TypeSourceInfo for it yet.
       TemplateIdAnnotation *TemplateId = D.getDeclSpec().getRepAsTemplateId();
-      TemplateArgumentListInfo TemplateArgsInfo;
+      TemplateArgumentListInfo TemplateArgsInfo(TemplateId->LAngleLoc,
+                                                TemplateId->RAngleLoc);
       bool Invalid = false;
       if (TemplateId->LAngleLoc.isValid()) {
         ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
diff --git a/clang/test/AST/ast-dump-concepts.cpp b/clang/test/AST/ast-dump-concepts.cpp
index 5bb174e3548e..a5e0673c241e 100644
--- a/clang/test/AST/ast-dump-concepts.cpp
+++ b/clang/test/AST/ast-dump-concepts.cpp
@@ -107,3 +107,16 @@ auto FooFunc(C auto V) -> C decltype(auto) {
 }
 
 }
+
+namespace constraint_auto_params {
+
+template <class T, class K>
+concept C = true;
+
+template<class T>
+void g(C<T> auto Foo) {}
+
+// CHECK: TemplateTypeParmDecl {{.*}} depth 0 index 1 Foo:auto
+// CHECK-NEXT: `-ConceptSpecializationExpr {{.*}} <col:8, col:11>
+
+}
-- 
GitLab


From a4accdfe0c9415ad1bd3dac7dda8cb8bbcd1be2f Mon Sep 17 00:00:00 2001
From: Artem Chikin <achikin@apple.com>
Date: Tue, 14 May 2024 10:53:53 -0400
Subject: [PATCH 0740/1206] [Support] Add option to print SMDiagnostic into a
 buffer without the filename and location info (#92050)

---
 llvm/include/llvm/Support/SourceMgr.h    |  2 +-
 llvm/lib/Support/SourceMgr.cpp           |  4 ++--
 llvm/unittests/Support/SourceMgrTest.cpp | 11 +++++++++++
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h
index 6f5bee7f8cc2..7a4b6de1162d 100644
--- a/llvm/include/llvm/Support/SourceMgr.h
+++ b/llvm/include/llvm/Support/SourceMgr.h
@@ -317,7 +317,7 @@ public:
   ArrayRef<SMFixIt> getFixIts() const { return FixIts; }
 
   void print(const char *ProgName, raw_ostream &S, bool ShowColors = true,
-             bool ShowKindLabel = true) const;
+             bool ShowKindLabel = true, bool ShowLocation = true) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Support/SourceMgr.cpp b/llvm/lib/Support/SourceMgr.cpp
index ebeff87c3954..6eaa8783a531 100644
--- a/llvm/lib/Support/SourceMgr.cpp
+++ b/llvm/lib/Support/SourceMgr.cpp
@@ -482,7 +482,7 @@ static void printSourceLine(raw_ostream &S, StringRef LineContents) {
 static bool isNonASCII(char c) { return c & 0x80; }
 
 void SMDiagnostic::print(const char *ProgName, raw_ostream &OS, bool ShowColors,
-                         bool ShowKindLabel) const {
+                         bool ShowKindLabel, bool ShowLocation) const {
   ColorMode Mode = ShowColors ? ColorMode::Auto : ColorMode::Disable;
 
   {
@@ -491,7 +491,7 @@ void SMDiagnostic::print(const char *ProgName, raw_ostream &OS, bool ShowColors,
     if (ProgName && ProgName[0])
       S << ProgName << ": ";
 
-    if (!Filename.empty()) {
+    if (ShowLocation && !Filename.empty()) {
       if (Filename == "-")
         S << "<stdin>";
       else
diff --git a/llvm/unittests/Support/SourceMgrTest.cpp b/llvm/unittests/Support/SourceMgrTest.cpp
index cbd17ff47689..75a0cfe35aaf 100644
--- a/llvm/unittests/Support/SourceMgrTest.cpp
+++ b/llvm/unittests/Support/SourceMgrTest.cpp
@@ -532,3 +532,14 @@ TEST_F(SourceMgrTest, FixitForTab) {
             Output);
 }
 
+TEST_F(SourceMgrTest, PrintWithoutLoc) {
+  raw_string_ostream OS(Output);
+  auto Diag =
+      llvm::SMDiagnostic("file.in", llvm::SourceMgr::DK_Error, "message");
+  Diag.print(nullptr, OS);
+  OS.flush();
+  EXPECT_EQ("file.in: error: message\n", Output);
+  Output.clear();
+  Diag.print(nullptr, OS, false, false, false);
+  EXPECT_EQ("message\n", Output);
+}
-- 
GitLab


From e08f1fda7508138d408cd61608bcbf30f8c3bb4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche@google.com>
Date: Tue, 14 May 2024 17:00:40 +0200
Subject: [PATCH 0741/1206] [clang][SPIR-V] Always add convergence intrinsics
 (#88918)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

PR #80680 added bits in the codegen to lazily add convergence intrinsics
when required. This logic relied on the LoopStack. The issue is when
parsing the condition, the loopstack doesn't yet reflect the correct
values, as expected since we are not yet in the loop.

However, convergence tokens should sometimes already be available. The
solution which seemed the simplest is to greedily generate the tokens
when we generate SPIR-V.

Fixes #88144

---------

Signed-off-by: Nathan Gauër <brioche@google.com>
---
 clang/lib/CodeGen/CGBuiltin.cpp               |  88 +------------
 clang/lib/CodeGen/CGCall.cpp                  |   5 +-
 clang/lib/CodeGen/CGStmt.cpp                  |  93 ++++++++++++++
 clang/lib/CodeGen/CodeGenFunction.cpp         |   9 ++
 clang/lib/CodeGen/CodeGenFunction.h           |   9 +-
 clang/lib/CodeGen/CodeGenModule.h             |   8 ++
 .../builtins/RWBuffer-constructor.hlsl        |   1 -
 clang/test/CodeGenHLSL/builtins/lerp.hlsl     |  82 ++++++------
 clang/test/CodeGenHLSL/builtins/mad.hlsl      | 104 +++++++++++----
 .../CodeGenHLSL/convergence/do.while.hlsl     |  90 +++++++++++++
 clang/test/CodeGenHLSL/convergence/for.hlsl   | 121 ++++++++++++++++++
 clang/test/CodeGenHLSL/convergence/while.hlsl | 119 +++++++++++++++++
 12 files changed, 580 insertions(+), 149 deletions(-)
 create mode 100644 clang/test/CodeGenHLSL/convergence/do.while.hlsl
 create mode 100644 clang/test/CodeGenHLSL/convergence/for.hlsl
 create mode 100644 clang/test/CodeGenHLSL/convergence/while.hlsl

diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index f9ee93049b12..e251091c6ce3 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -1141,91 +1141,8 @@ struct BitTest {
   static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
 };
 
-// Returns the first convergence entry/loop/anchor instruction found in |BB|.
-// std::nullptr otherwise.
-llvm::IntrinsicInst *getConvergenceToken(llvm::BasicBlock *BB) {
-  for (auto &I : *BB) {
-    auto *II = dyn_cast<llvm::IntrinsicInst>(&I);
-    if (II && isConvergenceControlIntrinsic(II->getIntrinsicID()))
-      return II;
-  }
-  return nullptr;
-}
-
 } // namespace
 
-llvm::CallBase *
-CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input,
-                                            llvm::Value *ParentToken) {
-  llvm::Value *bundleArgs[] = {ParentToken};
-  llvm::OperandBundleDef OB("convergencectrl", bundleArgs);
-  auto Output = llvm::CallBase::addOperandBundle(
-      Input, llvm::LLVMContext::OB_convergencectrl, OB, Input);
-  Input->replaceAllUsesWith(Output);
-  Input->eraseFromParent();
-  return Output;
-}
-
-llvm::IntrinsicInst *
-CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB,
-                                          llvm::Value *ParentToken) {
-  CGBuilderTy::InsertPoint IP = Builder.saveIP();
-  Builder.SetInsertPoint(&BB->front());
-  auto CB = Builder.CreateIntrinsic(
-      llvm::Intrinsic::experimental_convergence_loop, {}, {});
-  Builder.restoreIP(IP);
-
-  auto I = addConvergenceControlToken(CB, ParentToken);
-  return cast<llvm::IntrinsicInst>(I);
-}
-
-llvm::IntrinsicInst *
-CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
-  auto *BB = &F->getEntryBlock();
-  auto *token = getConvergenceToken(BB);
-  if (token)
-    return token;
-
-  // Adding a convergence token requires the function to be marked as
-  // convergent.
-  F->setConvergent();
-
-  CGBuilderTy::InsertPoint IP = Builder.saveIP();
-  Builder.SetInsertPoint(&BB->front());
-  auto I = Builder.CreateIntrinsic(
-      llvm::Intrinsic::experimental_convergence_entry, {}, {});
-  assert(isa<llvm::IntrinsicInst>(I));
-  Builder.restoreIP(IP);
-
-  return cast<llvm::IntrinsicInst>(I);
-}
-
-llvm::IntrinsicInst *
-CodeGenFunction::getOrEmitConvergenceLoopToken(const LoopInfo *LI) {
-  assert(LI != nullptr);
-
-  auto *token = getConvergenceToken(LI->getHeader());
-  if (token)
-    return token;
-
-  llvm::IntrinsicInst *PII =
-      LI->getParent()
-          ? emitConvergenceLoopToken(
-                LI->getHeader(), getOrEmitConvergenceLoopToken(LI->getParent()))
-          : getOrEmitConvergenceEntryToken(LI->getHeader()->getParent());
-
-  return emitConvergenceLoopToken(LI->getHeader(), PII);
-}
-
-llvm::CallBase *
-CodeGenFunction::addControlledConvergenceToken(llvm::CallBase *Input) {
-  llvm::Value *ParentToken =
-      LoopStack.hasInfo()
-          ? getOrEmitConvergenceLoopToken(&LoopStack.getInfo())
-          : getOrEmitConvergenceEntryToken(Input->getFunction());
-  return addConvergenceControlToken(Input, ParentToken);
-}
-
 BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
   switch (BuiltinID) {
     // Main portable variants.
@@ -18402,12 +18319,9 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
         ArrayRef<Value *>{Op0}, nullptr, "dx.rsqrt");
   }
   case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
-    auto *CI = EmitRuntimeCall(CGM.CreateRuntimeFunction(
+    return EmitRuntimeCall(CGM.CreateRuntimeFunction(
         llvm::FunctionType::get(IntTy, {}, false), "__hlsl_wave_get_lane_index",
         {}, false, true));
-    if (getTarget().getTriple().isSPIRVLogical())
-      CI = dyn_cast<CallInst>(addControlledConvergenceToken(CI));
-    return CI;
   }
   }
   return nullptr;
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 0c7eef59db53..1b4ca2a8b2fe 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -4830,6 +4830,9 @@ llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
   llvm::CallInst *call = Builder.CreateCall(
       callee, args, getBundlesForFunclet(callee.getCallee()), name);
   call->setCallingConv(getRuntimeCC());
+
+  if (CGM.shouldEmitConvergenceTokens() && call->isConvergent())
+    return addControlledConvergenceToken(call);
   return call;
 }
 
@@ -5730,7 +5733,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   if (!CI->getType()->isVoidTy())
     CI->setName("call");
 
-  if (getTarget().getTriple().isSPIRVLogical() && CI->isConvergent())
+  if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())
     CI = addControlledConvergenceToken(CI);
 
   // Update largest vector width from the return type.
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 479945e3b4cb..36776846cd44 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -978,6 +978,10 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
   JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond");
   EmitBlock(LoopHeader.getBlock());
 
+  if (CGM.shouldEmitConvergenceTokens())
+    ConvergenceTokenStack.push_back(emitConvergenceLoopToken(
+        LoopHeader.getBlock(), ConvergenceTokenStack.back()));
+
   // Create an exit block for when the condition fails, which will
   // also become the break target.
   JumpDest LoopExit = getJumpDestInCurrentScope("while.end");
@@ -1079,6 +1083,9 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
   // block.
   if (llvm::EnableSingleByteCoverage)
     incrementProfileCounter(&S);
+
+  if (CGM.shouldEmitConvergenceTokens())
+    ConvergenceTokenStack.pop_back();
 }
 
 void CodeGenFunction::EmitDoStmt(const DoStmt &S,
@@ -1098,6 +1105,11 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S,
     EmitBlockWithFallThrough(LoopBody, S.getBody());
   else
     EmitBlockWithFallThrough(LoopBody, &S);
+
+  if (CGM.shouldEmitConvergenceTokens())
+    ConvergenceTokenStack.push_back(
+        emitConvergenceLoopToken(LoopBody, ConvergenceTokenStack.back()));
+
   {
     RunCleanupsScope BodyScope(*this);
     EmitStmt(S.getBody());
@@ -1151,6 +1163,9 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S,
   // block.
   if (llvm::EnableSingleByteCoverage)
     incrementProfileCounter(&S);
+
+  if (CGM.shouldEmitConvergenceTokens())
+    ConvergenceTokenStack.pop_back();
 }
 
 void CodeGenFunction::EmitForStmt(const ForStmt &S,
@@ -1170,6 +1185,10 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
   llvm::BasicBlock *CondBlock = CondDest.getBlock();
   EmitBlock(CondBlock);
 
+  if (CGM.shouldEmitConvergenceTokens())
+    ConvergenceTokenStack.push_back(
+        emitConvergenceLoopToken(CondBlock, ConvergenceTokenStack.back()));
+
   const SourceRange &R = S.getSourceRange();
   LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs,
                  SourceLocToDebugLoc(R.getBegin()),
@@ -1279,6 +1298,9 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
   // block.
   if (llvm::EnableSingleByteCoverage)
     incrementProfileCounter(&S);
+
+  if (CGM.shouldEmitConvergenceTokens())
+    ConvergenceTokenStack.pop_back();
 }
 
 void
@@ -1301,6 +1323,10 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
   llvm::BasicBlock *CondBlock = createBasicBlock("for.cond");
   EmitBlock(CondBlock);
 
+  if (CGM.shouldEmitConvergenceTokens())
+    ConvergenceTokenStack.push_back(
+        emitConvergenceLoopToken(CondBlock, ConvergenceTokenStack.back()));
+
   const SourceRange &R = S.getSourceRange();
   LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs,
                  SourceLocToDebugLoc(R.getBegin()),
@@ -1369,6 +1395,9 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
   // block.
   if (llvm::EnableSingleByteCoverage)
     incrementProfileCounter(&S);
+
+  if (CGM.shouldEmitConvergenceTokens())
+    ConvergenceTokenStack.pop_back();
 }
 
 void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
@@ -3158,3 +3187,67 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
 
   return F;
 }
+
+namespace {
+// Returns the first convergence entry/loop/anchor instruction found in |BB|.
+// std::nullptr otherwise.
+llvm::IntrinsicInst *getConvergenceToken(llvm::BasicBlock *BB) {
+  for (auto &I : *BB) {
+    auto *II = dyn_cast<llvm::IntrinsicInst>(&I);
+    if (II && llvm::isConvergenceControlIntrinsic(II->getIntrinsicID()))
+      return II;
+  }
+  return nullptr;
+}
+
+} // namespace
+
+llvm::CallBase *
+CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input,
+                                            llvm::Value *ParentToken) {
+  llvm::Value *bundleArgs[] = {ParentToken};
+  llvm::OperandBundleDef OB("convergencectrl", bundleArgs);
+  auto Output = llvm::CallBase::addOperandBundle(
+      Input, llvm::LLVMContext::OB_convergencectrl, OB, Input);
+  Input->replaceAllUsesWith(Output);
+  Input->eraseFromParent();
+  return Output;
+}
+
+llvm::IntrinsicInst *
+CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB,
+                                          llvm::Value *ParentToken) {
+  CGBuilderTy::InsertPoint IP = Builder.saveIP();
+  if (BB->empty())
+    Builder.SetInsertPoint(BB);
+  else
+    Builder.SetInsertPoint(BB->getFirstInsertionPt());
+
+  llvm::CallBase *CB = Builder.CreateIntrinsic(
+      llvm::Intrinsic::experimental_convergence_loop, {}, {});
+  Builder.restoreIP(IP);
+
+  llvm::CallBase *I = addConvergenceControlToken(CB, ParentToken);
+  return cast<llvm::IntrinsicInst>(I);
+}
+
+llvm::IntrinsicInst *
+CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
+  llvm::BasicBlock *BB = &F->getEntryBlock();
+  llvm::IntrinsicInst *Token = getConvergenceToken(BB);
+  if (Token)
+    return Token;
+
+  // Adding a convergence token requires the function to be marked as
+  // convergent.
+  F->setConvergent();
+
+  CGBuilderTy::InsertPoint IP = Builder.saveIP();
+  Builder.SetInsertPoint(&BB->front());
+  llvm::CallBase *I = Builder.CreateIntrinsic(
+      llvm::Intrinsic::experimental_convergence_entry, {}, {});
+  assert(isa<llvm::IntrinsicInst>(I));
+  Builder.restoreIP(IP);
+
+  return cast<llvm::IntrinsicInst>(I);
+}
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 9f16fcb43855..34dc0bd5c15b 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -353,6 +353,12 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
   assert(DeferredDeactivationCleanupStack.empty() &&
          "mismatched activate/deactivate of cleanups!");
 
+  if (CGM.shouldEmitConvergenceTokens()) {
+    ConvergenceTokenStack.pop_back();
+    assert(ConvergenceTokenStack.empty() &&
+           "mismatched push/pop in convergence stack!");
+  }
+
   bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
     && NumSimpleReturnExprs == NumReturnExprs
     && ReturnBlock.getBlock()->use_empty();
@@ -1277,6 +1283,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
   if (CurFuncDecl)
     if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
       LargestVectorWidth = VecWidth->getVectorWidth();
+
+  if (CGM.shouldEmitConvergenceTokens())
+    ConvergenceTokenStack.push_back(getOrEmitConvergenceEntryToken(CurFn));
 }
 
 void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index e1e687af6a78..362f4a5fe72a 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -315,6 +315,9 @@ public:
   /// Stack to track the Logical Operator recursion nest for MC/DC.
   SmallVector<const BinaryOperator *, 16> MCDCLogOpStack;
 
+  /// Stack to track the controlled convergence tokens.
+  SmallVector<llvm::IntrinsicInst *, 4> ConvergenceTokenStack;
+
   /// Number of nested loop to be consumed by the last surrounding
   /// loop-associated directive.
   int ExpectedOMPLoopDepth = 0;
@@ -5076,7 +5079,11 @@ public:
                                      const llvm::Twine &Name = "");
   // Adds a convergence_ctrl token to |Input| and emits the required parent
   // convergence instructions.
-  llvm::CallBase *addControlledConvergenceToken(llvm::CallBase *Input);
+  template <typename CallType>
+  CallType *addControlledConvergenceToken(CallType *Input) {
+    return cast<CallType>(
+        addConvergenceControlToken(Input, ConvergenceTokenStack.back()));
+  }
 
 private:
   // Emits a convergence_loop instruction for the given |BB|, with |ParentToken|
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index be43a18fc608..0f68418130ea 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -1586,6 +1586,14 @@ public:
   void AddGlobalDtor(llvm::Function *Dtor, int Priority = 65535,
                      bool IsDtorAttrFunc = false);
 
+  // Return whether structured convergence intrinsics should be generated for
+  // this target.
+  bool shouldEmitConvergenceTokens() const {
+    // TODO: this should probably become unconditional once the controlled
+    // convergence becomes the norm.
+    return getTriple().isSPIRVLogical();
+  }
+
 private:
   llvm::Constant *GetOrCreateLLVMFunction(
       StringRef MangledName, llvm::Type *Ty, GlobalDecl D, bool ForVTable,
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
index 74b3f59bf760..e51eac7f57c2 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
@@ -1,4 +1,3 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple spirv-vulkan-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --check-prefix=CHECK-SPIRV
 
 RWBuffer<float> Buf;
diff --git a/clang/test/CodeGenHLSL/builtins/lerp.hlsl b/clang/test/CodeGenHLSL/builtins/lerp.hlsl
index 87b2e3af5765..bbb419acaf3b 100644
--- a/clang/test/CodeGenHLSL/builtins/lerp.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/lerp.hlsl
@@ -14,88 +14,98 @@
 // RUN:   -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF,SPIR_NO_HALF,SPIR_CHECK
 
 
-// DXIL_NATIVE_HALF: %hlsl.lerp = call half @llvm.dx.lerp.f16(half %0, half %1, half %2)
-// SPIR_NATIVE_HALF: %hlsl.lerp = call half @llvm.spv.lerp.f16(half %0, half %1, half %2)
+// DXIL_NATIVE_HALF: %hlsl.lerp = call half @llvm.dx.lerp.f16(half %{{.*}}, half %{{.*}}, half %{{.*}})
+// SPIR_NATIVE_HALF: %hlsl.lerp = call half @llvm.spv.lerp.f16(half %{{.*}}, half %{{.*}}, half %{{.*}})
 // NATIVE_HALF: ret half %hlsl.lerp
-// DXIL_NO_HALF: %hlsl.lerp = call float @llvm.dx.lerp.f32(float %0, float %1, float %2)
-// SPIR_NO_HALF: %hlsl.lerp = call float @llvm.spv.lerp.f32(float %0, float %1, float %2)
+// DXIL_NO_HALF: %hlsl.lerp = call float @llvm.dx.lerp.f32(float %{{.*}}, float %{{.*}}, float %{{.*}})
+// SPIR_NO_HALF: %hlsl.lerp = call float @llvm.spv.lerp.f32(float %{{.*}}, float %{{.*}}, float %{{.*}})
 // NO_HALF: ret float %hlsl.lerp
 half test_lerp_half(half p0) { return lerp(p0, p0, p0); }
 
-// DXIL_NATIVE_HALF: %hlsl.lerp = call <2 x half> @llvm.dx.lerp.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2)
-// SPIR_NATIVE_HALF: %hlsl.lerp = call <2 x half> @llvm.spv.lerp.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2)
+// DXIL_NATIVE_HALF: %hlsl.lerp = call <2 x half> @llvm.dx.lerp.v2f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+// SPIR_NATIVE_HALF: %hlsl.lerp = call <2 x half> @llvm.spv.lerp.v2f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
 // NATIVE_HALF: ret <2 x half> %hlsl.lerp
-// DXIL_NO_HALF: %hlsl.lerp = call <2 x float> @llvm.dx.lerp.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2)
-// SPIR_NO_HALF: %hlsl.lerp = call <2 x float> @llvm.spv.lerp.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2)
+// DXIL_NO_HALF: %hlsl.lerp = call <2 x float> @llvm.dx.lerp.v2f32(<2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}})
+// SPIR_NO_HALF: %hlsl.lerp = call <2 x float> @llvm.spv.lerp.v2f32(<2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}})
 // NO_HALF: ret <2 x float> %hlsl.lerp
 half2 test_lerp_half2(half2 p0) { return lerp(p0, p0, p0); }
 
-// DXIL_NATIVE_HALF: %hlsl.lerp = call <3 x half> @llvm.dx.lerp.v3f16(<3 x half> %0, <3 x half> %1, <3 x half> %2)
-// SPIR_NATIVE_HALF: %hlsl.lerp = call <3 x half> @llvm.spv.lerp.v3f16(<3 x half> %0, <3 x half> %1, <3 x half> %2)
+// DXIL_NATIVE_HALF: %hlsl.lerp = call <3 x half> @llvm.dx.lerp.v3f16(<3 x half> %{{.*}}, <3 x half> %{{.*}}, <3 x half> %{{.*}})
+// SPIR_NATIVE_HALF: %hlsl.lerp = call <3 x half> @llvm.spv.lerp.v3f16(<3 x half> %{{.*}}, <3 x half> %{{.*}}, <3 x half> %{{.*}})
 // NATIVE_HALF: ret <3 x half> %hlsl.lerp
-// DXIL_NO_HALF: %hlsl.lerp = call <3 x float> @llvm.dx.lerp.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %2)
-// SPIR_NO_HALF: %hlsl.lerp = call <3 x float> @llvm.spv.lerp.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %2)
+// DXIL_NO_HALF: %hlsl.lerp = call <3 x float> @llvm.dx.lerp.v3f32(<3 x float> %{{.*}}, <3 x float> %{{.*}}, <3 x float> %{{.*}})
+// SPIR_NO_HALF: %hlsl.lerp = call <3 x float> @llvm.spv.lerp.v3f32(<3 x float> %{{.*}}, <3 x float> %{{.*}}, <3 x float> %{{.*}})
 // NO_HALF: ret <3 x float> %hlsl.lerp
 half3 test_lerp_half3(half3 p0) { return lerp(p0, p0, p0); }
 
-// DXIL_NATIVE_HALF: %hlsl.lerp = call <4 x half> @llvm.dx.lerp.v4f16(<4 x half> %0, <4 x half> %1, <4 x half> %2)
-// SPIR_NATIVE_HALF: %hlsl.lerp = call <4 x half> @llvm.spv.lerp.v4f16(<4 x half> %0, <4 x half> %1, <4 x half> %2)
+// DXIL_NATIVE_HALF: %hlsl.lerp = call <4 x half> @llvm.dx.lerp.v4f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <4 x half> %{{.*}})
+// SPIR_NATIVE_HALF: %hlsl.lerp = call <4 x half> @llvm.spv.lerp.v4f16(<4 x half> %{{.*}}, <4 x half> %{{.*}}, <4 x half> %{{.*}})
 // NATIVE_HALF: ret <4 x half> %hlsl.lerp
-// DXIL_NO_HALF: %hlsl.lerp = call <4 x float> @llvm.dx.lerp.v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2)
-// SPIR_NO_HALF: %hlsl.lerp = call <4 x float> @llvm.spv.lerp.v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2)
+// DXIL_NO_HALF: %hlsl.lerp = call <4 x float> @llvm.dx.lerp.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
+// SPIR_NO_HALF: %hlsl.lerp = call <4 x float> @llvm.spv.lerp.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
 // NO_HALF: ret <4 x float> %hlsl.lerp
 half4 test_lerp_half4(half4 p0) { return lerp(p0, p0, p0); }
 
-// DXIL_CHECK: %hlsl.lerp = call float @llvm.dx.lerp.f32(float %0, float %1, float %2)
-// SPIR_CHECK: %hlsl.lerp = call float @llvm.spv.lerp.f32(float %0, float %1, float %2)
+// DXIL_CHECK: %hlsl.lerp = call float @llvm.dx.lerp.f32(float %{{.*}}, float %{{.*}}, float %{{.*}})
+// SPIR_CHECK: %hlsl.lerp = call float @llvm.spv.lerp.f32(float %{{.*}}, float %{{.*}}, float %{{.*}})
 // CHECK: ret float %hlsl.lerp
 float test_lerp_float(float p0) { return lerp(p0, p0, p0); }
 
-// DXIL_CHECK: %hlsl.lerp = call <2 x float> @llvm.dx.lerp.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2)
-// SPIR_CHECK: %hlsl.lerp = call <2 x float> @llvm.spv.lerp.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2)
+// DXIL_CHECK: %hlsl.lerp = call <2 x float> @llvm.dx.lerp.v2f32(<2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}})
+// SPIR_CHECK: %hlsl.lerp = call <2 x float> @llvm.spv.lerp.v2f32(<2 x float> %{{.*}}, <2 x float> %{{.*}}, <2 x float> %{{.*}})
 // CHECK: ret <2 x float> %hlsl.lerp
 float2 test_lerp_float2(float2 p0) { return lerp(p0, p0, p0); }
 
-// DXIL_CHECK: %hlsl.lerp = call <3 x float> @llvm.dx.lerp.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %2)
-// SPIR_CHECK: %hlsl.lerp = call <3 x float> @llvm.spv.lerp.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %2)
+// DXIL_CHECK: %hlsl.lerp = call <3 x float> @llvm.dx.lerp.v3f32(<3 x float> %{{.*}}, <3 x float> %{{.*}}, <3 x float> %{{.*}})
+// SPIR_CHECK: %hlsl.lerp = call <3 x float> @llvm.spv.lerp.v3f32(<3 x float> %{{.*}}, <3 x float> %{{.*}}, <3 x float> %{{.*}})
 // CHECK: ret <3 x float> %hlsl.lerp
 float3 test_lerp_float3(float3 p0) { return lerp(p0, p0, p0); }
 
-// DXIL_CHECK: %hlsl.lerp = call <4 x float> @llvm.dx.lerp.v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2)
-// SPIR_CHECK: %hlsl.lerp = call <4 x float> @llvm.spv.lerp.v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2)
+// DXIL_CHECK: %hlsl.lerp = call <4 x float> @llvm.dx.lerp.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
+// SPIR_CHECK: %hlsl.lerp = call <4 x float> @llvm.spv.lerp.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
 // CHECK: ret <4 x float> %hlsl.lerp
 float4 test_lerp_float4(float4 p0) { return lerp(p0, p0, p0); }
 
-// DXIL_CHECK: %hlsl.lerp = call <2 x float> @llvm.dx.lerp.v2f32(<2 x float> %splat.splat, <2 x float> %1, <2 x float> %2)
-// SPIR_CHECK: %hlsl.lerp = call <2 x float> @llvm.spv.lerp.v2f32(<2 x float> %splat.splat, <2 x float> %1, <2 x float> %2)
+// CHECK: %[[b:.*]] = load <2 x float>, ptr %p1.addr, align 8
+// CHECK: %[[c:.*]] = load <2 x float>, ptr %p1.addr, align 8
+// DXIL_CHECK: %hlsl.lerp = call <2 x float> @llvm.dx.lerp.v2f32(<2 x float> %splat.splat, <2 x float> %[[b]], <2 x float> %[[c]])
+// SPIR_CHECK: %hlsl.lerp = call <2 x float> @llvm.spv.lerp.v2f32(<2 x float> %splat.splat, <2 x float> %[[b]], <2 x float> %[[c]])
 // CHECK: ret <2 x float> %hlsl.lerp
 float2 test_lerp_float2_splat(float p0, float2 p1) { return lerp(p0, p1, p1); }
 
-// DXIL_CHECK: %hlsl.lerp = call <3 x float> @llvm.dx.lerp.v3f32(<3 x float> %splat.splat, <3 x float> %1, <3 x float> %2)
-// SPIR_CHECK: %hlsl.lerp = call <3 x float> @llvm.spv.lerp.v3f32(<3 x float> %splat.splat, <3 x float> %1, <3 x float> %2)
+// CHECK: %[[b:.*]] = load <3 x float>, ptr %p1.addr, align 16
+// CHECK: %[[c:.*]] = load <3 x float>, ptr %p1.addr, align 16
+// DXIL_CHECK: %hlsl.lerp = call <3 x float> @llvm.dx.lerp.v3f32(<3 x float> %splat.splat, <3 x float> %[[b]], <3 x float> %[[c]])
+// SPIR_CHECK: %hlsl.lerp = call <3 x float> @llvm.spv.lerp.v3f32(<3 x float> %splat.splat, <3 x float> %[[b]], <3 x float> %[[c]])
 // CHECK: ret <3 x float> %hlsl.lerp
 float3 test_lerp_float3_splat(float p0, float3 p1) { return lerp(p0, p1, p1); }
 
-// DXIL_CHECK:  %hlsl.lerp = call <4 x float> @llvm.dx.lerp.v4f32(<4 x float> %splat.splat, <4 x float> %1, <4 x float> %2)
-// SPIR_CHECK:  %hlsl.lerp = call <4 x float> @llvm.spv.lerp.v4f32(<4 x float> %splat.splat, <4 x float> %1, <4 x float> %2)
+// CHECK: %[[b:.*]] = load <4 x float>, ptr %p1.addr, align 16
+// CHECK: %[[c:.*]] = load <4 x float>, ptr %p1.addr, align 16
+// DXIL_CHECK: %hlsl.lerp = call <4 x float> @llvm.dx.lerp.v4f32(<4 x float> %splat.splat, <4 x float> %[[b]], <4 x float> %[[c]])
+// SPIR_CHECK: %hlsl.lerp = call <4 x float> @llvm.spv.lerp.v4f32(<4 x float> %splat.splat, <4 x float> %[[b]], <4 x float> %[[c]])
 // CHECK:  ret <4 x float> %hlsl.lerp
 float4 test_lerp_float4_splat(float p0, float4 p1) { return lerp(p0, p1, p1); }
 
-// CHECK: %conv = sitofp i32 %2 to float
+// CHECK: %[[a:.*]] = load <2 x float>, ptr %p0.addr, align 8
+// CHECK: %[[b:.*]] = load <2 x float>, ptr %p0.addr, align 8
+// CHECK: %conv = sitofp i32 {{.*}} to float
 // CHECK: %splat.splatinsert = insertelement <2 x float> poison, float %conv, i64 0
 // CHECK: %splat.splat = shufflevector <2 x float> %splat.splatinsert, <2 x float> poison, <2 x i32> zeroinitializer
-// DXIL_CHECK: %hlsl.lerp = call <2 x float> @llvm.dx.lerp.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %splat.splat)
-// SPIR_CHECK: %hlsl.lerp = call <2 x float> @llvm.spv.lerp.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %splat.splat)
+// DXIL_CHECK: %hlsl.lerp = call <2 x float> @llvm.dx.lerp.v2f32(<2 x float> %[[a]], <2 x float> %[[b]], <2 x float> %splat.splat)
+// SPIR_CHECK: %hlsl.lerp = call <2 x float> @llvm.spv.lerp.v2f32(<2 x float> %[[a]], <2 x float> %[[b]], <2 x float> %splat.splat)
 // CHECK: ret <2 x float> %hlsl.lerp
 float2 test_lerp_float2_int_splat(float2 p0, int p1) {
   return lerp(p0, p0, p1);
 }
 
-// CHECK: %conv = sitofp i32 %2 to float
+// CHECK: %[[a:.*]] = load <3 x float>, ptr %p0.addr, align 16
+// CHECK: %[[b:.*]] = load <3 x float>, ptr %p0.addr, align 16
+// CHECK: %conv = sitofp i32 {{.*}} to float
 // CHECK: %splat.splatinsert = insertelement <3 x float> poison, float %conv, i64 0
 // CHECK: %splat.splat = shufflevector <3 x float> %splat.splatinsert, <3 x float> poison, <3 x i32> zeroinitializer
-// DXIL_CHECK:  %hlsl.lerp = call <3 x float> @llvm.dx.lerp.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %splat.splat)
-// SPIR_CHECK:  %hlsl.lerp = call <3 x float> @llvm.spv.lerp.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %splat.splat)
+// DXIL_CHECK:  %hlsl.lerp = call <3 x float> @llvm.dx.lerp.v3f32(<3 x float> %[[a]], <3 x float> %[[b]], <3 x float> %splat.splat)
+// SPIR_CHECK:  %hlsl.lerp = call <3 x float> @llvm.spv.lerp.v3f32(<3 x float> %[[a]], <3 x float> %[[b]], <3 x float> %splat.splat)
 // CHECK: ret <3 x float> %hlsl.lerp
 float3 test_lerp_float3_int_splat(float3 p0, int p1) {
   return lerp(p0, p0, p1);
diff --git a/clang/test/CodeGenHLSL/builtins/mad.hlsl b/clang/test/CodeGenHLSL/builtins/mad.hlsl
index b4dc636b00b7..559e1d1dd390 100644
--- a/clang/test/CodeGenHLSL/builtins/mad.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/mad.hlsl
@@ -64,59 +64,107 @@ int16_t3 test_mad_int16_t3(int16_t3 p0, int16_t3 p1, int16_t3 p2) { return mad(p
 int16_t4 test_mad_int16_t4(int16_t4 p0, int16_t4 p1, int16_t4 p2) { return mad(p0, p1, p2); }
 #endif // __HLSL_ENABLE_16_BIT
 
-// NATIVE_HALF: %hlsl.fmad = call half @llvm.fmuladd.f16(half %0, half %1, half %2)
+// NATIVE_HALF: %[[p0:.*]] = load half, ptr %p0.addr, align 2
+// NATIVE_HALF: %[[p1:.*]] = load half, ptr %p1.addr, align 2
+// NATIVE_HALF: %[[p2:.*]] = load half, ptr %p2.addr, align 2
+// NATIVE_HALF: %hlsl.fmad = call half @llvm.fmuladd.f16(half %[[p0]], half %[[p1]], half %[[p2]])
 // NATIVE_HALF: ret half %hlsl.fmad
-// NO_HALF: %hlsl.fmad = call float @llvm.fmuladd.f32(float %0, float %1, float %2)
+// NO_HALF: %[[p0:.*]] = load float, ptr %p0.addr, align 4
+// NO_HALF: %[[p1:.*]] = load float, ptr %p1.addr, align 4
+// NO_HALF: %[[p2:.*]] = load float, ptr %p2.addr, align 4
+// NO_HALF: %hlsl.fmad = call float @llvm.fmuladd.f32(float %[[p0]], float %[[p1]], float %[[p2]])
 // NO_HALF: ret float %hlsl.fmad
 half test_mad_half(half p0, half p1, half p2) { return mad(p0, p1, p2); }
 
-// NATIVE_HALF: %hlsl.fmad = call <2 x half>  @llvm.fmuladd.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2)
+// NATIVE_HALF: %[[p0:.*]] = load <2 x half>, ptr %p0.addr, align 4
+// NATIVE_HALF: %[[p1:.*]] = load <2 x half>, ptr %p1.addr, align 4
+// NATIVE_HALF: %[[p2:.*]] = load <2 x half>, ptr %p2.addr, align 4
+// NATIVE_HALF: %hlsl.fmad = call <2 x half>  @llvm.fmuladd.v2f16(<2 x half> %[[p0]], <2 x half> %[[p1]], <2 x half> %[[p2]])
 // NATIVE_HALF: ret <2 x half> %hlsl.fmad
-// NO_HALF: %hlsl.fmad = call <2 x float>  @llvm.fmuladd.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2)
+// NO_HALF: %[[p0:.*]] = load <2 x float>, ptr %p0.addr, align 8
+// NO_HALF: %[[p1:.*]] = load <2 x float>, ptr %p1.addr, align 8
+// NO_HALF: %[[p2:.*]] = load <2 x float>, ptr %p2.addr, align 8
+// NO_HALF: %hlsl.fmad = call <2 x float>  @llvm.fmuladd.v2f32(<2 x float> %[[p0]], <2 x float> %[[p1]], <2 x float> %[[p2]])
 // NO_HALF: ret <2 x float> %hlsl.fmad
 half2 test_mad_half2(half2 p0, half2 p1, half2 p2) { return mad(p0, p1, p2); }
 
-// NATIVE_HALF: %hlsl.fmad = call <3 x half>  @llvm.fmuladd.v3f16(<3 x half> %0, <3 x half> %1, <3 x half> %2)
+// NATIVE_HALF: %[[p0:.*]] = load <3 x half>, ptr %p0.addr, align 8
+// NATIVE_HALF: %[[p1:.*]] = load <3 x half>, ptr %p1.addr, align 8
+// NATIVE_HALF: %[[p2:.*]] = load <3 x half>, ptr %p2.addr, align 8
+// NATIVE_HALF: %hlsl.fmad = call <3 x half>  @llvm.fmuladd.v3f16(<3 x half> %[[p0]], <3 x half> %[[p1]], <3 x half> %[[p2]])
 // NATIVE_HALF: ret <3 x half> %hlsl.fmad
-// NO_HALF: %hlsl.fmad = call <3 x float>  @llvm.fmuladd.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %2)
+// NO_HALF: %[[p0:.*]] = load <3 x float>, ptr %p0.addr, align 16
+// NO_HALF: %[[p1:.*]] = load <3 x float>, ptr %p1.addr, align 16
+// NO_HALF: %[[p2:.*]] = load <3 x float>, ptr %p2.addr, align 16
+// NO_HALF: %hlsl.fmad = call <3 x float>  @llvm.fmuladd.v3f32(<3 x float> %[[p0]], <3 x float> %[[p1]], <3 x float> %[[p2]])
 // NO_HALF: ret <3 x float> %hlsl.fmad
 half3 test_mad_half3(half3 p0, half3 p1, half3 p2) { return mad(p0, p1, p2); }
 
-// NATIVE_HALF: %hlsl.fmad = call <4 x half>  @llvm.fmuladd.v4f16(<4 x half> %0, <4 x half> %1, <4 x half> %2)
+// NATIVE_HALF: %[[p0:.*]] = load <4 x half>, ptr %p0.addr, align 8
+// NATIVE_HALF: %[[p1:.*]] = load <4 x half>, ptr %p1.addr, align 8
+// NATIVE_HALF: %[[p2:.*]] = load <4 x half>, ptr %p2.addr, align 8
+// NATIVE_HALF: %hlsl.fmad = call <4 x half>  @llvm.fmuladd.v4f16(<4 x half> %[[p0]], <4 x half> %[[p1]], <4 x half> %[[p2]])
 // NATIVE_HALF: ret <4 x half> %hlsl.fmad
-// NO_HALF: %hlsl.fmad = call <4 x float>  @llvm.fmuladd.v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2)
+// NO_HALF: %[[p0:.*]] = load <4 x float>, ptr %p0.addr, align 16
+// NO_HALF: %[[p1:.*]] = load <4 x float>, ptr %p1.addr, align 16
+// NO_HALF: %[[p2:.*]] = load <4 x float>, ptr %p2.addr, align 16
+// NO_HALF: %hlsl.fmad = call <4 x float>  @llvm.fmuladd.v4f32(<4 x float> %[[p0]], <4 x float> %[[p1]], <4 x float> %[[p2]])
 // NO_HALF: ret <4 x float> %hlsl.fmad
 half4 test_mad_half4(half4 p0, half4 p1, half4 p2) { return mad(p0, p1, p2); }
 
-// CHECK: %hlsl.fmad = call float @llvm.fmuladd.f32(float %0, float %1, float %2)
+// CHECK: %[[p0:.*]] = load float, ptr %p0.addr, align 4
+// CHECK: %[[p1:.*]] = load float, ptr %p1.addr, align 4
+// CHECK: %[[p2:.*]] = load float, ptr %p2.addr, align 4
+// CHECK: %hlsl.fmad = call float @llvm.fmuladd.f32(float %[[p0]], float %[[p1]], float %[[p2]])
 // CHECK: ret float %hlsl.fmad
 float test_mad_float(float p0, float p1, float p2) { return mad(p0, p1, p2); }
 
-// CHECK: %hlsl.fmad = call <2 x float>  @llvm.fmuladd.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2)
+// CHECK: %[[p0:.*]] = load <2 x float>, ptr %p0.addr, align 8
+// CHECK: %[[p1:.*]] = load <2 x float>, ptr %p1.addr, align 8
+// CHECK: %[[p2:.*]] = load <2 x float>, ptr %p2.addr, align 8
+// CHECK: %hlsl.fmad = call <2 x float>  @llvm.fmuladd.v2f32(<2 x float> %[[p0]], <2 x float> %[[p1]], <2 x float> %[[p2]])
 // CHECK: ret <2 x float> %hlsl.fmad
 float2 test_mad_float2(float2 p0, float2 p1, float2 p2) { return mad(p0, p1, p2); }
 
-// CHECK: %hlsl.fmad = call <3 x float>  @llvm.fmuladd.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %2)
+// CHECK: %[[p0:.*]] = load <3 x float>, ptr %p0.addr, align 16
+// CHECK: %[[p1:.*]] = load <3 x float>, ptr %p1.addr, align 16
+// CHECK: %[[p2:.*]] = load <3 x float>, ptr %p2.addr, align 16
+// CHECK: %hlsl.fmad = call <3 x float>  @llvm.fmuladd.v3f32(<3 x float> %[[p0]], <3 x float> %[[p1]], <3 x float> %[[p2]])
 // CHECK: ret <3 x float> %hlsl.fmad
 float3 test_mad_float3(float3 p0, float3 p1, float3 p2) { return mad(p0, p1, p2); }
 
-// CHECK: %hlsl.fmad = call <4 x float>  @llvm.fmuladd.v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2)
+// CHECK: %[[p0:.*]] = load <4 x float>, ptr %p0.addr, align 16
+// CHECK: %[[p1:.*]] = load <4 x float>, ptr %p1.addr, align 16
+// CHECK: %[[p2:.*]] = load <4 x float>, ptr %p2.addr, align 16
+// CHECK: %hlsl.fmad = call <4 x float>  @llvm.fmuladd.v4f32(<4 x float> %[[p0]], <4 x float> %[[p1]], <4 x float> %[[p2]])
 // CHECK: ret <4 x float> %hlsl.fmad
 float4 test_mad_float4(float4 p0, float4 p1, float4 p2) { return mad(p0, p1, p2); }
 
-// CHECK: %hlsl.fmad = call double @llvm.fmuladd.f64(double %0, double %1, double %2)
+// CHECK: %[[p0:.*]] = load double, ptr %p0.addr, align 8
+// CHECK: %[[p1:.*]] = load double, ptr %p1.addr, align 8
+// CHECK: %[[p2:.*]] = load double, ptr %p2.addr, align 8
+// CHECK: %hlsl.fmad = call double @llvm.fmuladd.f64(double %[[p0]], double %[[p1]], double %[[p2]])
 // CHECK: ret double %hlsl.fmad
 double test_mad_double(double p0, double p1, double p2) { return mad(p0, p1, p2); }
 
-// CHECK: %hlsl.fmad = call <2 x double>  @llvm.fmuladd.v2f64(<2 x double> %0, <2 x double> %1, <2 x double> %2)
+// CHECK: %[[p0:.*]] = load <2 x double>, ptr %p0.addr, align 16
+// CHECK: %[[p1:.*]] = load <2 x double>, ptr %p1.addr, align 16
+// CHECK: %[[p2:.*]] = load <2 x double>, ptr %p2.addr, align 16
+// CHECK: %hlsl.fmad = call <2 x double>  @llvm.fmuladd.v2f64(<2 x double> %[[p0]], <2 x double> %[[p1]], <2 x double> %[[p2]])
 // CHECK: ret <2 x double> %hlsl.fmad
 double2 test_mad_double2(double2 p0, double2 p1, double2 p2) { return mad(p0, p1, p2); }
 
-// CHECK: %hlsl.fmad = call <3 x double>  @llvm.fmuladd.v3f64(<3 x double> %0, <3 x double> %1, <3 x double> %2)
+// CHECK: %[[p0:.*]] = load <3 x double>, ptr %p0.addr, align 32
+// CHECK: %[[p1:.*]] = load <3 x double>, ptr %p1.addr, align 32
+// CHECK: %[[p2:.*]] = load <3 x double>, ptr %p2.addr, align 32
+// CHECK: %hlsl.fmad = call <3 x double>  @llvm.fmuladd.v3f64(<3 x double> %[[p0]], <3 x double> %[[p1]], <3 x double> %[[p2]])
 // CHECK: ret <3 x double> %hlsl.fmad
 double3 test_mad_double3(double3 p0, double3 p1, double3 p2) { return mad(p0, p1, p2); }
 
-// CHECK: %hlsl.fmad = call <4 x double>  @llvm.fmuladd.v4f64(<4 x double> %0, <4 x double> %1, <4 x double> %2)
+// CHECK: %[[p0:.*]] = load <4 x double>, ptr %p0.addr, align 32
+// CHECK: %[[p1:.*]] = load <4 x double>, ptr %p1.addr, align 32
+// CHECK: %[[p2:.*]] = load <4 x double>, ptr %p2.addr, align 32
+// CHECK: %hlsl.fmad = call <4 x double>  @llvm.fmuladd.v4f64(<4 x double> %[[p0]], <4 x double> %[[p1]], <4 x double> %[[p2]])
 // CHECK: ret <4 x double> %hlsl.fmad
 double4 test_mad_double4(double4 p0, double4 p1, double4 p2) { return mad(p0, p1, p2); }
 
@@ -216,31 +264,41 @@ uint64_t3 test_mad_uint64_t3(uint64_t3 p0, uint64_t3 p1, uint64_t3 p2) { return
 // SPIR_CHECK: add nuw <4 x i64>  %{{.*}}, %{{.*}}
 uint64_t4 test_mad_uint64_t4(uint64_t4 p0, uint64_t4 p1, uint64_t4 p2) { return mad(p0, p1, p2); }
 
-// CHECK: %hlsl.fmad = call <2 x float>  @llvm.fmuladd.v2f32(<2 x float> %splat.splat, <2 x float> %1, <2 x float> %2)
+// CHECK: %[[p1:.*]] = load <2 x float>, ptr %p1.addr, align 8
+// CHECK: %[[p2:.*]] = load <2 x float>, ptr %p2.addr, align 8
+// CHECK: %hlsl.fmad = call <2 x float>  @llvm.fmuladd.v2f32(<2 x float> %splat.splat, <2 x float> %[[p1]], <2 x float> %[[p2]])
 // CHECK: ret <2 x float> %hlsl.fmad
 float2 test_mad_float2_splat(float p0, float2 p1, float2 p2) { return mad(p0, p1, p2); }
 
-// CHECK: %hlsl.fmad = call <3 x float>  @llvm.fmuladd.v3f32(<3 x float> %splat.splat, <3 x float> %1, <3 x float> %2)
+// CHECK: %[[p1:.*]] = load <3 x float>, ptr %p1.addr, align 16
+// CHECK: %[[p2:.*]] = load <3 x float>, ptr %p2.addr, align 16
+// CHECK: %hlsl.fmad = call <3 x float>  @llvm.fmuladd.v3f32(<3 x float> %splat.splat, <3 x float> %[[p1]], <3 x float> %[[p2]])
 // CHECK: ret <3 x float> %hlsl.fmad
 float3 test_mad_float3_splat(float p0, float3 p1, float3 p2) { return mad(p0, p1, p2); }
 
-// CHECK:  %hlsl.fmad = call <4 x float>  @llvm.fmuladd.v4f32(<4 x float> %splat.splat, <4 x float> %1, <4 x float> %2)
+// CHECK: %[[p1:.*]] = load <4 x float>, ptr %p1.addr, align 16
+// CHECK: %[[p2:.*]] = load <4 x float>, ptr %p2.addr, align 16
+// CHECK:  %hlsl.fmad = call <4 x float>  @llvm.fmuladd.v4f32(<4 x float> %splat.splat, <4 x float> %[[p1]], <4 x float> %[[p2]])
 // CHECK:  ret <4 x float> %hlsl.fmad
 float4 test_mad_float4_splat(float p0, float4 p1, float4 p2) { return mad(p0, p1, p2); }
 
-// CHECK: %conv = sitofp i32 %2 to float
+// CHECK: %[[p0:.*]] = load <2 x float>, ptr %p0.addr, align 8
+// CHECK: %[[p1:.*]] = load <2 x float>, ptr %p1.addr, align 8
+// CHECK: %conv = sitofp i32 %{{.*}} to float
 // CHECK: %splat.splatinsert = insertelement <2 x float> poison, float %conv, i64 0
 // CHECK: %splat.splat = shufflevector <2 x float> %splat.splatinsert, <2 x float> poison, <2 x i32> zeroinitializer
-// CHECK: %hlsl.fmad = call <2 x float>  @llvm.fmuladd.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %splat.splat)
+// CHECK: %hlsl.fmad = call <2 x float>  @llvm.fmuladd.v2f32(<2 x float> %[[p0]], <2 x float> %[[p1]], <2 x float> %splat.splat)
 // CHECK: ret <2 x float> %hlsl.fmad
 float2 test_mad_float2_int_splat(float2 p0, float2 p1, int p2) {
   return mad(p0, p1, p2);
 }
 
-// CHECK: %conv = sitofp i32 %2 to float
+// CHECK: %[[p0:.*]] = load <3 x float>, ptr %p0.addr, align 16
+// CHECK: %[[p1:.*]] = load <3 x float>, ptr %p1.addr, align 16
+// CHECK: %conv = sitofp i32 %{{.*}} to float
 // CHECK: %splat.splatinsert = insertelement <3 x float> poison, float %conv, i64 0
 // CHECK: %splat.splat = shufflevector <3 x float> %splat.splatinsert, <3 x float> poison, <3 x i32> zeroinitializer
-// CHECK:  %hlsl.fmad = call <3 x float>  @llvm.fmuladd.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %splat.splat)
+// CHECK:  %hlsl.fmad = call <3 x float>  @llvm.fmuladd.v3f32(<3 x float> %[[p0]], <3 x float> %[[p1]], <3 x float> %splat.splat)
 // CHECK: ret <3 x float> %hlsl.fmad
 float3 test_mad_float3_int_splat(float3 p0, float3 p1, int p2) {
   return mad(p0, p1, p2);
diff --git a/clang/test/CodeGenHLSL/convergence/do.while.hlsl b/clang/test/CodeGenHLSL/convergence/do.while.hlsl
new file mode 100644
index 000000000000..ea5a45ba8fd7
--- /dev/null
+++ b/clang/test/CodeGenHLSL/convergence/do.while.hlsl
@@ -0,0 +1,90 @@
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-pc-vulkan-library %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+
+bool cond();
+void foo();
+
+void test1() {
+  do {
+  } while (cond());
+}
+// CHECK: define spir_func void @_Z5test1v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: do.body:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK: do.cond:
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+void test2() {
+  do {
+    foo();
+  } while (cond());
+}
+// CHECK: define spir_func void @_Z5test2v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: do.body:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func void @_Z3foov() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: do.cond:
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+void test3() {
+  do {
+    if (cond())
+      foo();
+  } while (cond());
+}
+// CHECK: define spir_func void @_Z5test3v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: do.body:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK: if.then:
+// CHECK:                    call spir_func void @_Z3foov() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: do.cond:
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+void test4() {
+  do {
+    if (cond()) {
+      foo();
+      break;
+    }
+  } while (cond());
+}
+// CHECK: define spir_func void @_Z5test4v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: do.body:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK: if.then:
+// CHECK:                    call spir_func void @_Z3foov() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: do.cond:
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+void test5() {
+  do {
+    while (cond()) {
+      if (cond()) {
+        foo();
+        break;
+      }
+    }
+  } while (cond());
+}
+// CHECK: define spir_func void @_Z5test5v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: do.body:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK: while.cond:
+// CHECK:   [[T2:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T1]]) ]
+// CHECK: if.then:
+// CHECK:                    call spir_func void @_Z3foov() [[A3]] [ "convergencectrl"(token [[T2]]) ]
+// CHECK: do.cond:
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+// CHECK-DAG: attributes [[A0]] = { {{.*}}convergent{{.*}} }
+// CHECK-DAG: attributes [[A3]] = { {{.*}}convergent{{.*}} }
diff --git a/clang/test/CodeGenHLSL/convergence/for.hlsl b/clang/test/CodeGenHLSL/convergence/for.hlsl
new file mode 100644
index 000000000000..180fae74ba75
--- /dev/null
+++ b/clang/test/CodeGenHLSL/convergence/for.hlsl
@@ -0,0 +1,121 @@
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-pc-vulkan-library %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+
+bool cond();
+bool cond2();
+void foo();
+
+void test1() {
+  for (;;) {
+    foo();
+  }
+}
+// CHECK: define spir_func void @_Z5test1v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: for.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func void @_Z3foov() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+void test2() {
+  for (;cond();) {
+    foo();
+  }
+}
+// CHECK: define spir_func void @_Z5test2v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: for.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: for.body:
+// CHECK:                    call spir_func void @_Z3foov() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+void test3() {
+  for (cond();;) {
+    foo();
+  }
+}
+// CHECK: define spir_func void @_Z5test3v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T0]]) ]
+// CHECK: for.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func void @_Z3foov() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+void test4() {
+  for (cond();cond2();) {
+    foo();
+  }
+}
+// CHECK: define spir_func void @_Z5test4v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T0]]) ]
+// CHECK: for.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z5cond2v() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: for.body:
+// CHECK:                    call spir_func void @_Z3foov() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+void test5() {
+  for (cond();cond2();foo()) {
+  }
+}
+// CHECK: define spir_func void @_Z5test5v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T0]]) ]
+// CHECK: for.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z5cond2v() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: for.inc:
+// CHECK:                    call spir_func void @_Z3foov() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+void test6() {
+  for (cond();cond2();foo()) {
+    if (cond()) {
+      foo();
+      break;
+    }
+  }
+}
+// CHECK: define spir_func void @_Z5test6v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T0]]) ]
+// CHECK: for.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z5cond2v() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: for.body:
+// CHECK:   [[C1:%[a-zA-Z0-9]+]] = call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK:   br i1 [[C1]], label %if.then, label %if.end
+// CHECK: if.then:
+// CHECK    call spir_func void @_Z3foov() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK:   br label %for.end
+// CHECK: if.end:
+// CHECK:   br label %for.inc
+// CHECK: for.inc:
+// CHECK:                    call spir_func void @_Z3foov() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+void test7() {
+  for (cond();;) {
+    for (cond();;) {
+      foo();
+    }
+  }
+}
+// CHECK: define spir_func void @_Z5test7v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T0]]) ]
+// CHECK: for.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: for.cond3:
+// CHECK:   [[T2:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T1]]) ]
+// CHECK:                    call spir_func void @_Z3foov() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T2]]) ]
+
+// CHECK-DAG: attributes [[A0]] = { {{.*}}convergent{{.*}} }
+// CHECK-DAG: attributes [[A3]] = { {{.*}}convergent{{.*}} }
diff --git a/clang/test/CodeGenHLSL/convergence/while.hlsl b/clang/test/CodeGenHLSL/convergence/while.hlsl
new file mode 100644
index 000000000000..92777000190d
--- /dev/null
+++ b/clang/test/CodeGenHLSL/convergence/while.hlsl
@@ -0,0 +1,119 @@
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-pc-vulkan-library %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+
+bool cond();
+void foo();
+
+void test1() {
+  while (cond()) {
+  }
+}
+// CHECK: define spir_func void @_Z5test1v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: while.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3:#[0-9]+]] [ "convergencectrl"(token [[T1]]) ]
+
+void test2() {
+  while (cond()) {
+    foo();
+  }
+}
+// CHECK: define spir_func void @_Z5test2v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: while.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: while.body:
+// CHECK:   call spir_func void @_Z3foov() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+
+void test3() {
+  while (cond()) {
+    if (cond())
+      break;
+    foo();
+  }
+}
+// CHECK: define spir_func void @_Z5test3v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: while.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: if.then:
+// CHECK:   br label %while.end
+// CHECK: if.end:
+// CHECK:   call spir_func void @_Z3foov() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK:   br label %while.cond
+
+void test4() {
+  while (cond()) {
+    if (cond()) {
+      foo();
+      break;
+    }
+  }
+}
+// CHECK: define spir_func void @_Z5test4v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: while.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: if.then:
+// CHECK:   call spir_func void @_Z3foov() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK:   br label %while.end
+// CHECK: if.end:
+// CHECK:   br label %while.cond
+
+void test5() {
+  while (cond()) {
+    while (cond()) {
+      if (cond()) {
+        foo();
+        break;
+      }
+    }
+  }
+}
+// CHECK: define spir_func void @_Z5test5v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: while.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: while.cond2:
+// CHECK:   [[T2:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T1]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T2]]) ]
+// CHECK: if.then:
+// CHECK:   call spir_func void @_Z3foov() [[A3]] [ "convergencectrl"(token [[T2]]) ]
+// CHECK:   br label %while.end
+
+void test6() {
+  while (cond()) {
+    while (cond()) {
+    }
+
+    if (cond()) {
+      foo();
+      break;
+    }
+  }
+}
+// CHECK: define spir_func void @_Z5test6v() [[A0:#[0-9]+]] {
+// CHECK: entry:
+// CHECK:   [[T0:%[0-9]+]] = call token @llvm.experimental.convergence.entry()
+// CHECK: while.cond:
+// CHECK:   [[T1:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T0]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK: while.cond2:
+// CHECK:   [[T2:%[0-9]+]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[T1]]) ]
+// CHECK:                    call spir_func noundef i1 @_Z4condv() [[A3]] [ "convergencectrl"(token [[T2]]) ]
+// CHECK: if.then:
+// CHECK:   call spir_func void @_Z3foov() [[A3]] [ "convergencectrl"(token [[T1]]) ]
+// CHECK:   br label %while.end
+
+// CHECK-DAG: attributes [[A0]] = { {{.*}}convergent{{.*}} }
+// CHECK-DAG: attributes [[A3]] = { {{.*}}convergent{{.*}} }
-- 
GitLab


From ac0d415552922436c3136e3dd1446294858c2d7d Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak@amd.com>
Date: Tue, 14 May 2024 10:03:48 -0500
Subject: [PATCH 0742/1206] Update documentation for buffer fat pointers
 (#92034)

Now that we've got (minus some issues around datatypes and invariant
loads) working lowerings for address space 7, update the table in the
AMDGPU usage guide to properly indicate the nature of these address
spaces.
---
 llvm/docs/AMDGPUUsage.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 51969be85648..75536bc5bea6 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -824,8 +824,8 @@ supported for the ``amdgcn`` target.
      Constant                              4               constant    *same as global* 64      0x0000000000000000
      Private                               5               private     scratch          32      0xFFFFFFFF
      Constant 32-bit                       6               *TODO*                               0x00000000
-     Buffer Fat Pointer (experimental)     7               *TODO*
-     Buffer Resource (experimental)        8               *TODO*
+     Buffer Fat Pointer                    7               N/A         N/A              160     0
+     Buffer Resource                       8               N/A         V#               128     0x00000000000000000000000000000000
      Buffer Strided Pointer (experimental) 9               *TODO*
      Streamout Registers                   128             N/A         GS_REGS
      ===================================== =============== =========== ================ ======= ============================
-- 
GitLab


From 736ffdc38347f3f83cf7b3c034b8e837f46f7eab Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 14 May 2024 08:06:10 -0700
Subject: [PATCH 0743/1206] [RISCV] Add X27 to SavedRegs when X26 is in
 SavedRegs for cm.push/pop (#92067)

cm.push can't save X26 without also saving X27. This removes two other
checks for this case.

This causes CFI to be emitted since X27 is now explicitly a callee saved
register.

The affected tests use inline assembly to clobber X26 rather than the
whole range of s0-s10.
---
 llvm/lib/Target/RISCV/RISCVFrameLowering.cpp | 10 ++++++----
 llvm/test/CodeGen/RISCV/push-pop-popret.ll   |  4 ++++
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 316f6a90893a..436bd4a38a31 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -265,7 +265,6 @@ getPushPopEncodingAndNum(const Register MaxReg) {
   default:
     llvm_unreachable("Unexpected Reg for Push/Pop Inst");
   case RISCV::X27: /*s11*/
-  case RISCV::X26: /*s10*/
     return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S11, 13);
   case RISCV::X25: /*s9*/
     return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S9, 11);
@@ -302,9 +301,7 @@ static Register getMaxPushPopReg(const MachineFunction &MF,
         }) != std::end(FixedCSRFIMap))
       MaxPushPopReg = std::max(MaxPushPopReg.id(), CS.getReg().id());
   }
-  // if rlist is {rs, s0-s10}, then s11 will also be included
-  if (MaxPushPopReg == RISCV::X26)
-    MaxPushPopReg = RISCV::X27;
+  assert(MaxPushPopReg != RISCV::X26 && "x26 requires x27 to also be pushed");
   return MaxPushPopReg;
 }
 
@@ -1047,6 +1044,11 @@ void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
   // Mark BP as used if function has dedicated base pointer.
   if (hasBP(MF))
     SavedRegs.set(RISCVABI::getBPReg());
+
+  // When using cm.push/pop we must save X27 if we save X26.
+  auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
+  if (RVFI->isPushable(MF) && SavedRegs.test(RISCV::X26))
+    SavedRegs.set(RISCV::X27);
 }
 
 std::pair<int64_t, Align>
diff --git a/llvm/test/CodeGen/RISCV/push-pop-popret.ll b/llvm/test/CodeGen/RISCV/push-pop-popret.ll
index e007dcc764e9..7548faaae61f 100644
--- a/llvm/test/CodeGen/RISCV/push-pop-popret.ll
+++ b/llvm/test/CodeGen/RISCV/push-pop-popret.ll
@@ -3225,6 +3225,7 @@ define void @spill_x10() {
 ; RV32IZCMP-NEXT:    cm.push {ra, s0-s11}, -64
 ; RV32IZCMP-NEXT:    .cfi_def_cfa_offset 64
 ; RV32IZCMP-NEXT:    .cfi_offset s10, -8
+; RV32IZCMP-NEXT:    .cfi_offset s11, -4
 ; RV32IZCMP-NEXT:    #APP
 ; RV32IZCMP-NEXT:    li s10, 0
 ; RV32IZCMP-NEXT:    #NO_APP
@@ -3235,6 +3236,7 @@ define void @spill_x10() {
 ; RV64IZCMP-NEXT:    cm.push {ra, s0-s11}, -112
 ; RV64IZCMP-NEXT:    .cfi_def_cfa_offset 112
 ; RV64IZCMP-NEXT:    .cfi_offset s10, -16
+; RV64IZCMP-NEXT:    .cfi_offset s11, -8
 ; RV64IZCMP-NEXT:    #APP
 ; RV64IZCMP-NEXT:    li s10, 0
 ; RV64IZCMP-NEXT:    #NO_APP
@@ -3245,6 +3247,7 @@ define void @spill_x10() {
 ; RV32IZCMP-SR-NEXT:    cm.push {ra, s0-s11}, -64
 ; RV32IZCMP-SR-NEXT:    .cfi_def_cfa_offset 64
 ; RV32IZCMP-SR-NEXT:    .cfi_offset s10, -8
+; RV32IZCMP-SR-NEXT:    .cfi_offset s11, -4
 ; RV32IZCMP-SR-NEXT:    #APP
 ; RV32IZCMP-SR-NEXT:    li s10, 0
 ; RV32IZCMP-SR-NEXT:    #NO_APP
@@ -3255,6 +3258,7 @@ define void @spill_x10() {
 ; RV64IZCMP-SR-NEXT:    cm.push {ra, s0-s11}, -112
 ; RV64IZCMP-SR-NEXT:    .cfi_def_cfa_offset 112
 ; RV64IZCMP-SR-NEXT:    .cfi_offset s10, -16
+; RV64IZCMP-SR-NEXT:    .cfi_offset s11, -8
 ; RV64IZCMP-SR-NEXT:    #APP
 ; RV64IZCMP-SR-NEXT:    li s10, 0
 ; RV64IZCMP-SR-NEXT:    #NO_APP
-- 
GitLab


From 2766f2174e428842a9ab1a9ba5b320be5878f87d Mon Sep 17 00:00:00 2001
From: Ben Langmuir <blangmuir@apple.com>
Date: Tue, 14 May 2024 08:08:18 -0700
Subject: [PATCH 0744/1206] [ORC] Loosen __objc_imageinfo flag merging to match
 ld (#91767)

Allow mixing objects with/without signed class ro data and category
class properties as long as it happens before we register the metadata.
These combinations are a warning in ld, not a hard error. The only case
that is ABI-breaking is if we already registered with the feature
enabled but later try to load an object that doesn't support it.

rdar://127336061
---
 .../TestCases/Darwin/arm64/objc-imageinfo.S   | 22 ++++++++++++++++---
 .../TestCases/Darwin/x86-64/objc-imageinfo.S  | 22 ++++++++++++++++---
 .../lib/ExecutionEngine/Orc/MachOPlatform.cpp | 14 ++++++++++--
 3 files changed, 50 insertions(+), 8 deletions(-)

diff --git a/compiler-rt/test/orc/TestCases/Darwin/arm64/objc-imageinfo.S b/compiler-rt/test/orc/TestCases/Darwin/arm64/objc-imageinfo.S
index d58943f9681d..2ee7d3f5eac1 100644
--- a/compiler-rt/test/orc/TestCases/Darwin/arm64/objc-imageinfo.S
+++ b/compiler-rt/test/orc/TestCases/Darwin/arm64/objc-imageinfo.S
@@ -27,9 +27,6 @@
 
 // Check error conditions.
 
-// RUN: not %llvm_jitlink %t/main.o %t/objc_old.o %t/objc_new.o 2>&1 | FileCheck %s -check-prefix=CATEGORY
-// CATEGORY: ObjC category class property support in {{.*}} does not match first registered flags
-
 // RUN: not %llvm_jitlink %t/main.o %t/swift_4.o %t/swift_5.o 2>&1 | FileCheck %s -check-prefix=SWIFT_ABI
 // SWIFT_ABI: Swift ABI version in {{.*}} does not match first registered flags
 
@@ -47,6 +44,14 @@
 // RUN: %llvm_jitlink -debug-only=orc %t/main.o %t/swift_59.o %t/swift_5.o %t/objc_new.o 2>&1 | FileCheck %s -check-prefix=SWIFT_MIX3
 // SWIFT_MIX3: MachOPlatform: Merging __objc_imageinfo flags for main {{.*}} -> 0x5000740
 
+// Disable categories.
+// RUN: %llvm_jitlink -debug-only=orc %t/main.o %t/objc_old.o %t/objc_new.o 2>&1 | FileCheck %s -check-prefix=SWIFT_MIX4
+// SWIFT_MIX4: MachOPlatform: Merging __objc_imageinfo flags for main {{.*}} -> 0x0000
+
+// Disable signed class_ro.
+// RUN: %llvm_jitlink -debug-only=orc %t/main.o %t/objc_new.o %t/objc_new_signed_ro.o 2>&1 | FileCheck %s -check-prefix=SWIFT_MIX5
+// SWIFT_MIX5: MachOPlatform: Merging __objc_imageinfo flags for main {{.*}} -> 0x0040
+
 //--- main.S
 .section  __TEXT,__text,regular,pure_instructions
 .globl _main
@@ -76,6 +81,17 @@ L_OBJC_IMAGE_INFO:
   .long 0
   .long 64
 
+//--- objc_new_signed_ro.S
+.section  __TEXT,__text,regular,pure_instructions
+.globl _objc3
+_objc3:
+  ret
+
+  .section  __DATA,__objc_imageinfo,regular,no_dead_strip
+L_OBJC_IMAGE_INFO:
+  .long 0
+  .long 80
+
 //--- swift_4.S
 .section  __TEXT,__text,regular,pure_instructions
 .globl _swift4
diff --git a/compiler-rt/test/orc/TestCases/Darwin/x86-64/objc-imageinfo.S b/compiler-rt/test/orc/TestCases/Darwin/x86-64/objc-imageinfo.S
index 90b5c3a38eeb..d4e9b4b05fb8 100644
--- a/compiler-rt/test/orc/TestCases/Darwin/x86-64/objc-imageinfo.S
+++ b/compiler-rt/test/orc/TestCases/Darwin/x86-64/objc-imageinfo.S
@@ -27,9 +27,6 @@
 
 // Check error conditions.
 
-// RUN: not %llvm_jitlink %t/main.o %t/objc_old.o %t/objc_new.o 2>&1 | FileCheck %s -check-prefix=CATEGORY
-// CATEGORY: ObjC category class property support in {{.*}} does not match first registered flags
-
 // RUN: not %llvm_jitlink %t/main.o %t/swift_4.o %t/swift_5.o 2>&1 | FileCheck %s -check-prefix=SWIFT_ABI
 // SWIFT_ABI: Swift ABI version in {{.*}} does not match first registered flags
 
@@ -47,6 +44,14 @@
 // RUN: %llvm_jitlink -debug-only=orc %t/main.o %t/swift_59.o %t/swift_5.o %t/objc_new.o 2>&1 | FileCheck %s -check-prefix=SWIFT_MIX3
 // SWIFT_MIX3: MachOPlatform: Merging __objc_imageinfo flags for main {{.*}} -> 0x5000740
 
+// Disable categories.
+// RUN: %llvm_jitlink -debug-only=orc %t/main.o %t/objc_old.o %t/objc_new.o 2>&1 | FileCheck %s -check-prefix=SWIFT_MIX4
+// SWIFT_MIX4: MachOPlatform: Merging __objc_imageinfo flags for main {{.*}} -> 0x0000
+
+// Disable signed class_ro.
+// RUN: %llvm_jitlink -debug-only=orc %t/main.o %t/objc_new.o %t/objc_new_signed_ro.o 2>&1 | FileCheck %s -check-prefix=SWIFT_MIX5
+// SWIFT_MIX5: MachOPlatform: Merging __objc_imageinfo flags for main {{.*}} -> 0x0040
+
 //--- main.S
 .section  __TEXT,__text,regular,pure_instructions
 .globl _main
@@ -76,6 +81,17 @@ L_OBJC_IMAGE_INFO:
   .long 0
   .long 64
 
+//--- objc_new_signed_ro.S
+.section  __TEXT,__text,regular,pure_instructions
+.globl _objc3
+_objc3:
+  ret
+
+  .section  __DATA,__objc_imageinfo,regular,no_dead_strip
+L_OBJC_IMAGE_INFO:
+  .long 0
+  .long 80
+
 //--- swift_4.S
 .section  __TEXT,__text,regular,pure_instructions
 .globl _swift4
diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index 1fa8a1274911..2b397b2d48e7 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -1134,12 +1134,16 @@ Error MachOPlatform::MachOPlatformPlugin::mergeImageInfoFlags(
                                        " does not match first registered flags",
                                    inconvertibleErrorCode());
 
-  if (Old.HasCategoryClassProperties != New.HasCategoryClassProperties)
+  // HasCategoryClassProperties and HasSignedObjCClassROs can be disabled before
+  // they are registered, if necessary, but once they are in use must be
+  // supported by subsequent objects.
+  if (Info.Finalized && Old.HasCategoryClassProperties &&
+      !New.HasCategoryClassProperties)
     return make_error<StringError>("ObjC category class property support in " +
                                        G.getName() +
                                        " does not match first registered flags",
                                    inconvertibleErrorCode());
-  if (Old.HasSignedObjCClassROs != New.HasSignedObjCClassROs)
+  if (Info.Finalized && Old.HasSignedObjCClassROs && !New.HasSignedObjCClassROs)
     return make_error<StringError>("ObjC class_ro_t pointer signing in " +
                                        G.getName() +
                                        " does not match first registered flags",
@@ -1158,6 +1162,12 @@ Error MachOPlatform::MachOPlatformPlugin::mergeImageInfoFlags(
   // Add a Swift ABI version if it was pure objc before.
   if (!New.SwiftABIVersion)
     New.SwiftABIVersion = Old.SwiftABIVersion;
+  // Disable class properties if any object does not support it.
+  if (Old.HasCategoryClassProperties != New.HasCategoryClassProperties)
+    New.HasCategoryClassProperties = false;
+  // Disable signed class ro data if any object does not support it.
+  if (Old.HasSignedObjCClassROs != New.HasSignedObjCClassROs)
+    New.HasSignedObjCClassROs = false;
 
   LLVM_DEBUG({
     dbgs() << "MachOPlatform: Merging __objc_imageinfo flags for "
-- 
GitLab


From 82434c70b792c4a3773515f8d3172df11e4e615f Mon Sep 17 00:00:00 2001
From: WANG Rui <wangrui@loongson.cn>
Date: Tue, 14 May 2024 22:45:09 +0800
Subject: [PATCH 0745/1206] [LoongArch] Add test cases for div/mod to cover
 various extended combinations of 32-bit integers. NFC

---
 .../ir-instruction/sdiv-udiv-srem-urem.ll     | 528 ++++++++++++++++--
 1 file changed, 484 insertions(+), 44 deletions(-)

diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem.ll
index 381f69bb46f8..2064c398948f 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/sdiv-udiv-srem-urem.ll
@@ -148,6 +148,113 @@ entry:
   ret i32 %r
 }
 
+define i32 @sdiv_ui32_si32_si32(i32 signext %a, i32 signext %b) {
+; LA32-LABEL: sdiv_ui32_si32_si32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    div.w $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: sdiv_ui32_si32_si32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    div.d $a0, $a0, $a1
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: sdiv_ui32_si32_si32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    div.w $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB4_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB4_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: sdiv_ui32_si32_si32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    div.d $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB4_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB4_2: # %entry
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = sdiv i32 %a, %b
+  ret i32 %r
+}
+
+define signext i32 @sdiv_si32_ui32_ui32(i32 %a, i32 %b) {
+; LA32-LABEL: sdiv_si32_ui32_ui32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    div.w $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: sdiv_si32_ui32_ui32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    addi.w $a0, $a0, 0
+; LA64-NEXT:    div.d $a0, $a0, $a1
+; LA64-NEXT:    addi.w $a0, $a0, 0
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: sdiv_si32_ui32_ui32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    div.w $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB5_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB5_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: sdiv_si32_ui32_ui32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    addi.w $a1, $a1, 0
+; LA64-TRAP-NEXT:    addi.w $a0, $a0, 0
+; LA64-TRAP-NEXT:    div.d $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB5_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB5_2: # %entry
+; LA64-TRAP-NEXT:    addi.w $a0, $a0, 0
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = sdiv i32 %a, %b
+  ret i32 %r
+}
+
+define signext i32 @sdiv_si32_si32_si32(i32 signext %a, i32 signext %b) {
+; LA32-LABEL: sdiv_si32_si32_si32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    div.w $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: sdiv_si32_si32_si32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    div.d $a0, $a0, $a1
+; LA64-NEXT:    addi.w $a0, $a0, 0
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: sdiv_si32_si32_si32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    div.w $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB6_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB6_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: sdiv_si32_si32_si32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    div.d $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB6_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB6_2: # %entry
+; LA64-TRAP-NEXT:    addi.w $a0, $a0, 0
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = sdiv i32 %a, %b
+  ret i32 %r
+}
+
 define i64 @sdiv_i64(i64 %a, i64 %b) {
 ; LA32-LABEL: sdiv_i64:
 ; LA32:       # %bb.0: # %entry
@@ -179,10 +286,10 @@ define i64 @sdiv_i64(i64 %a, i64 %b) {
 ; LA64-TRAP-LABEL: sdiv_i64:
 ; LA64-TRAP:       # %bb.0: # %entry
 ; LA64-TRAP-NEXT:    div.d $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB4_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB7_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB4_2: # %entry
+; LA64-TRAP-NEXT:  .LBB7_2: # %entry
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = sdiv i64 %a, %b
@@ -230,10 +337,10 @@ define i8 @udiv_i8(i8 %a, i8 %b) {
 ; LA32-TRAP-NEXT:    andi $a1, $a1, 255
 ; LA32-TRAP-NEXT:    andi $a0, $a0, 255
 ; LA32-TRAP-NEXT:    div.wu $a0, $a0, $a1
-; LA32-TRAP-NEXT:    bnez $a1, .LBB6_2
+; LA32-TRAP-NEXT:    bnez $a1, .LBB9_2
 ; LA32-TRAP-NEXT:  # %bb.1: # %entry
 ; LA32-TRAP-NEXT:    break 7
-; LA32-TRAP-NEXT:  .LBB6_2: # %entry
+; LA32-TRAP-NEXT:  .LBB9_2: # %entry
 ; LA32-TRAP-NEXT:    ret
 ;
 ; LA64-TRAP-LABEL: udiv_i8:
@@ -241,10 +348,10 @@ define i8 @udiv_i8(i8 %a, i8 %b) {
 ; LA64-TRAP-NEXT:    andi $a1, $a1, 255
 ; LA64-TRAP-NEXT:    andi $a0, $a0, 255
 ; LA64-TRAP-NEXT:    div.du $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB6_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB9_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB6_2: # %entry
+; LA64-TRAP-NEXT:  .LBB9_2: # %entry
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = udiv i8 %a, %b
@@ -271,10 +378,10 @@ define i16 @udiv_i16(i16 %a, i16 %b) {
 ; LA32-TRAP-NEXT:    bstrpick.w $a1, $a1, 15, 0
 ; LA32-TRAP-NEXT:    bstrpick.w $a0, $a0, 15, 0
 ; LA32-TRAP-NEXT:    div.wu $a0, $a0, $a1
-; LA32-TRAP-NEXT:    bnez $a1, .LBB7_2
+; LA32-TRAP-NEXT:    bnez $a1, .LBB10_2
 ; LA32-TRAP-NEXT:  # %bb.1: # %entry
 ; LA32-TRAP-NEXT:    break 7
-; LA32-TRAP-NEXT:  .LBB7_2: # %entry
+; LA32-TRAP-NEXT:  .LBB10_2: # %entry
 ; LA32-TRAP-NEXT:    ret
 ;
 ; LA64-TRAP-LABEL: udiv_i16:
@@ -282,10 +389,10 @@ define i16 @udiv_i16(i16 %a, i16 %b) {
 ; LA64-TRAP-NEXT:    bstrpick.d $a1, $a1, 15, 0
 ; LA64-TRAP-NEXT:    bstrpick.d $a0, $a0, 15, 0
 ; LA64-TRAP-NEXT:    div.du $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB7_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB10_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB7_2: # %entry
+; LA64-TRAP-NEXT:  .LBB10_2: # %entry
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = udiv i16 %a, %b
@@ -308,10 +415,10 @@ define i32 @udiv_i32(i32 %a, i32 %b) {
 ; LA32-TRAP-LABEL: udiv_i32:
 ; LA32-TRAP:       # %bb.0: # %entry
 ; LA32-TRAP-NEXT:    div.wu $a0, $a0, $a1
-; LA32-TRAP-NEXT:    bnez $a1, .LBB8_2
+; LA32-TRAP-NEXT:    bnez $a1, .LBB11_2
 ; LA32-TRAP-NEXT:  # %bb.1: # %entry
 ; LA32-TRAP-NEXT:    break 7
-; LA32-TRAP-NEXT:  .LBB8_2: # %entry
+; LA32-TRAP-NEXT:  .LBB11_2: # %entry
 ; LA32-TRAP-NEXT:    ret
 ;
 ; LA64-TRAP-LABEL: udiv_i32:
@@ -319,10 +426,125 @@ define i32 @udiv_i32(i32 %a, i32 %b) {
 ; LA64-TRAP-NEXT:    bstrpick.d $a1, $a1, 31, 0
 ; LA64-TRAP-NEXT:    bstrpick.d $a0, $a0, 31, 0
 ; LA64-TRAP-NEXT:    div.du $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB8_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB11_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB11_2: # %entry
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = udiv i32 %a, %b
+  ret i32 %r
+}
+
+define i32 @udiv_ui32_si32_si32(i32 signext %a, i32 signext %b) {
+; LA32-LABEL: udiv_ui32_si32_si32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    div.wu $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: udiv_ui32_si32_si32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-NEXT:    div.du $a0, $a0, $a1
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: udiv_ui32_si32_si32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    div.wu $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB12_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB12_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: udiv_ui32_si32_si32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-TRAP-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-TRAP-NEXT:    div.du $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB12_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB12_2: # %entry
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = udiv i32 %a, %b
+  ret i32 %r
+}
+
+define signext i32 @udiv_si32_ui32_ui32(i32 %a, i32 %b) {
+; LA32-LABEL: udiv_si32_ui32_ui32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    div.wu $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: udiv_si32_ui32_ui32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-NEXT:    div.du $a0, $a0, $a1
+; LA64-NEXT:    addi.w $a0, $a0, 0
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: udiv_si32_ui32_ui32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    div.wu $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB13_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB13_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: udiv_si32_ui32_ui32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-TRAP-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-TRAP-NEXT:    div.du $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB13_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB13_2: # %entry
+; LA64-TRAP-NEXT:    addi.w $a0, $a0, 0
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = udiv i32 %a, %b
+  ret i32 %r
+}
+
+define signext i32 @udiv_si32_si32_si32(i32 signext %a, i32 signext %b) {
+; LA32-LABEL: udiv_si32_si32_si32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    div.wu $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: udiv_si32_si32_si32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-NEXT:    div.du $a0, $a0, $a1
+; LA64-NEXT:    addi.w $a0, $a0, 0
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: udiv_si32_si32_si32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    div.wu $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB14_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB14_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: udiv_si32_si32_si32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-TRAP-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-TRAP-NEXT:    div.du $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB14_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB8_2: # %entry
+; LA64-TRAP-NEXT:  .LBB14_2: # %entry
+; LA64-TRAP-NEXT:    addi.w $a0, $a0, 0
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = udiv i32 %a, %b
@@ -360,10 +582,10 @@ define i64 @udiv_i64(i64 %a, i64 %b) {
 ; LA64-TRAP-LABEL: udiv_i64:
 ; LA64-TRAP:       # %bb.0: # %entry
 ; LA64-TRAP-NEXT:    div.du $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB9_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB15_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB9_2: # %entry
+; LA64-TRAP-NEXT:  .LBB15_2: # %entry
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = udiv i64 %a, %b
@@ -415,10 +637,10 @@ define i8 @srem_i8(i8 %a, i8 %b) {
 ; LA32-TRAP-NEXT:    ext.w.b $a1, $a1
 ; LA32-TRAP-NEXT:    ext.w.b $a0, $a0
 ; LA32-TRAP-NEXT:    mod.w $a0, $a0, $a1
-; LA32-TRAP-NEXT:    bnez $a1, .LBB11_2
+; LA32-TRAP-NEXT:    bnez $a1, .LBB17_2
 ; LA32-TRAP-NEXT:  # %bb.1: # %entry
 ; LA32-TRAP-NEXT:    break 7
-; LA32-TRAP-NEXT:  .LBB11_2: # %entry
+; LA32-TRAP-NEXT:  .LBB17_2: # %entry
 ; LA32-TRAP-NEXT:    ret
 ;
 ; LA64-TRAP-LABEL: srem_i8:
@@ -426,10 +648,10 @@ define i8 @srem_i8(i8 %a, i8 %b) {
 ; LA64-TRAP-NEXT:    ext.w.b $a1, $a1
 ; LA64-TRAP-NEXT:    ext.w.b $a0, $a0
 ; LA64-TRAP-NEXT:    mod.d $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB11_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB17_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB11_2: # %entry
+; LA64-TRAP-NEXT:  .LBB17_2: # %entry
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = srem i8 %a, %b
@@ -456,10 +678,10 @@ define i16 @srem_i16(i16 %a, i16 %b) {
 ; LA32-TRAP-NEXT:    ext.w.h $a1, $a1
 ; LA32-TRAP-NEXT:    ext.w.h $a0, $a0
 ; LA32-TRAP-NEXT:    mod.w $a0, $a0, $a1
-; LA32-TRAP-NEXT:    bnez $a1, .LBB12_2
+; LA32-TRAP-NEXT:    bnez $a1, .LBB18_2
 ; LA32-TRAP-NEXT:  # %bb.1: # %entry
 ; LA32-TRAP-NEXT:    break 7
-; LA32-TRAP-NEXT:  .LBB12_2: # %entry
+; LA32-TRAP-NEXT:  .LBB18_2: # %entry
 ; LA32-TRAP-NEXT:    ret
 ;
 ; LA64-TRAP-LABEL: srem_i16:
@@ -467,10 +689,10 @@ define i16 @srem_i16(i16 %a, i16 %b) {
 ; LA64-TRAP-NEXT:    ext.w.h $a1, $a1
 ; LA64-TRAP-NEXT:    ext.w.h $a0, $a0
 ; LA64-TRAP-NEXT:    mod.d $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB12_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB18_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB12_2: # %entry
+; LA64-TRAP-NEXT:  .LBB18_2: # %entry
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = srem i16 %a, %b
@@ -493,10 +715,10 @@ define i32 @srem_i32(i32 %a, i32 %b) {
 ; LA32-TRAP-LABEL: srem_i32:
 ; LA32-TRAP:       # %bb.0: # %entry
 ; LA32-TRAP-NEXT:    mod.w $a0, $a0, $a1
-; LA32-TRAP-NEXT:    bnez $a1, .LBB13_2
+; LA32-TRAP-NEXT:    bnez $a1, .LBB19_2
 ; LA32-TRAP-NEXT:  # %bb.1: # %entry
 ; LA32-TRAP-NEXT:    break 7
-; LA32-TRAP-NEXT:  .LBB13_2: # %entry
+; LA32-TRAP-NEXT:  .LBB19_2: # %entry
 ; LA32-TRAP-NEXT:    ret
 ;
 ; LA64-TRAP-LABEL: srem_i32:
@@ -504,10 +726,113 @@ define i32 @srem_i32(i32 %a, i32 %b) {
 ; LA64-TRAP-NEXT:    addi.w $a1, $a1, 0
 ; LA64-TRAP-NEXT:    addi.w $a0, $a0, 0
 ; LA64-TRAP-NEXT:    mod.d $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB13_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB19_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB13_2: # %entry
+; LA64-TRAP-NEXT:  .LBB19_2: # %entry
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = srem i32 %a, %b
+  ret i32 %r
+}
+
+define i32 @srem_ui32_si32_si32(i32 signext %a, i32 signext %b) {
+; LA32-LABEL: srem_ui32_si32_si32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    mod.w $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: srem_ui32_si32_si32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    mod.d $a0, $a0, $a1
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: srem_ui32_si32_si32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    mod.w $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB20_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB20_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: srem_ui32_si32_si32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    mod.d $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB20_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB20_2: # %entry
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = srem i32 %a, %b
+  ret i32 %r
+}
+
+define signext i32 @srem_si32_ui32_ui32(i32 %a, i32 %b) {
+; LA32-LABEL: srem_si32_ui32_ui32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    mod.w $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: srem_si32_ui32_ui32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    addi.w $a0, $a0, 0
+; LA64-NEXT:    mod.d $a0, $a0, $a1
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: srem_si32_ui32_ui32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    mod.w $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB21_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB21_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: srem_si32_ui32_ui32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    addi.w $a1, $a1, 0
+; LA64-TRAP-NEXT:    addi.w $a0, $a0, 0
+; LA64-TRAP-NEXT:    mod.d $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB21_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB21_2: # %entry
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = srem i32 %a, %b
+  ret i32 %r
+}
+
+define signext i32 @srem_si32_si32_si32(i32 signext %a, i32 signext %b) {
+; LA32-LABEL: srem_si32_si32_si32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    mod.w $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: srem_si32_si32_si32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    mod.d $a0, $a0, $a1
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: srem_si32_si32_si32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    mod.w $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB22_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB22_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: srem_si32_si32_si32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    mod.d $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB22_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB22_2: # %entry
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = srem i32 %a, %b
@@ -545,10 +870,10 @@ define i64 @srem_i64(i64 %a, i64 %b) {
 ; LA64-TRAP-LABEL: srem_i64:
 ; LA64-TRAP:       # %bb.0: # %entry
 ; LA64-TRAP-NEXT:    mod.d $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB14_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB23_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB14_2: # %entry
+; LA64-TRAP-NEXT:  .LBB23_2: # %entry
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = srem i64 %a, %b
@@ -600,10 +925,10 @@ define i8 @urem_i8(i8 %a, i8 %b) {
 ; LA32-TRAP-NEXT:    andi $a1, $a1, 255
 ; LA32-TRAP-NEXT:    andi $a0, $a0, 255
 ; LA32-TRAP-NEXT:    mod.wu $a0, $a0, $a1
-; LA32-TRAP-NEXT:    bnez $a1, .LBB16_2
+; LA32-TRAP-NEXT:    bnez $a1, .LBB25_2
 ; LA32-TRAP-NEXT:  # %bb.1: # %entry
 ; LA32-TRAP-NEXT:    break 7
-; LA32-TRAP-NEXT:  .LBB16_2: # %entry
+; LA32-TRAP-NEXT:  .LBB25_2: # %entry
 ; LA32-TRAP-NEXT:    ret
 ;
 ; LA64-TRAP-LABEL: urem_i8:
@@ -611,10 +936,10 @@ define i8 @urem_i8(i8 %a, i8 %b) {
 ; LA64-TRAP-NEXT:    andi $a1, $a1, 255
 ; LA64-TRAP-NEXT:    andi $a0, $a0, 255
 ; LA64-TRAP-NEXT:    mod.du $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB16_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB25_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB16_2: # %entry
+; LA64-TRAP-NEXT:  .LBB25_2: # %entry
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = urem i8 %a, %b
@@ -641,10 +966,10 @@ define i16 @urem_i16(i16 %a, i16 %b) {
 ; LA32-TRAP-NEXT:    bstrpick.w $a1, $a1, 15, 0
 ; LA32-TRAP-NEXT:    bstrpick.w $a0, $a0, 15, 0
 ; LA32-TRAP-NEXT:    mod.wu $a0, $a0, $a1
-; LA32-TRAP-NEXT:    bnez $a1, .LBB17_2
+; LA32-TRAP-NEXT:    bnez $a1, .LBB26_2
 ; LA32-TRAP-NEXT:  # %bb.1: # %entry
 ; LA32-TRAP-NEXT:    break 7
-; LA32-TRAP-NEXT:  .LBB17_2: # %entry
+; LA32-TRAP-NEXT:  .LBB26_2: # %entry
 ; LA32-TRAP-NEXT:    ret
 ;
 ; LA64-TRAP-LABEL: urem_i16:
@@ -652,10 +977,10 @@ define i16 @urem_i16(i16 %a, i16 %b) {
 ; LA64-TRAP-NEXT:    bstrpick.d $a1, $a1, 15, 0
 ; LA64-TRAP-NEXT:    bstrpick.d $a0, $a0, 15, 0
 ; LA64-TRAP-NEXT:    mod.du $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB17_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB26_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB17_2: # %entry
+; LA64-TRAP-NEXT:  .LBB26_2: # %entry
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = urem i16 %a, %b
@@ -678,10 +1003,10 @@ define i32 @urem_i32(i32 %a, i32 %b) {
 ; LA32-TRAP-LABEL: urem_i32:
 ; LA32-TRAP:       # %bb.0: # %entry
 ; LA32-TRAP-NEXT:    mod.wu $a0, $a0, $a1
-; LA32-TRAP-NEXT:    bnez $a1, .LBB18_2
+; LA32-TRAP-NEXT:    bnez $a1, .LBB27_2
 ; LA32-TRAP-NEXT:  # %bb.1: # %entry
 ; LA32-TRAP-NEXT:    break 7
-; LA32-TRAP-NEXT:  .LBB18_2: # %entry
+; LA32-TRAP-NEXT:  .LBB27_2: # %entry
 ; LA32-TRAP-NEXT:    ret
 ;
 ; LA64-TRAP-LABEL: urem_i32:
@@ -689,10 +1014,125 @@ define i32 @urem_i32(i32 %a, i32 %b) {
 ; LA64-TRAP-NEXT:    bstrpick.d $a1, $a1, 31, 0
 ; LA64-TRAP-NEXT:    bstrpick.d $a0, $a0, 31, 0
 ; LA64-TRAP-NEXT:    mod.du $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB18_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB27_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB18_2: # %entry
+; LA64-TRAP-NEXT:  .LBB27_2: # %entry
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = urem i32 %a, %b
+  ret i32 %r
+}
+
+define i32 @urem_ui32_si32_si32(i32 signext %a, i32 signext %b) {
+; LA32-LABEL: urem_ui32_si32_si32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    mod.wu $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: urem_ui32_si32_si32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-NEXT:    mod.du $a0, $a0, $a1
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: urem_ui32_si32_si32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    mod.wu $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB28_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB28_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: urem_ui32_si32_si32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-TRAP-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-TRAP-NEXT:    mod.du $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB28_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB28_2: # %entry
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = urem i32 %a, %b
+  ret i32 %r
+}
+
+define signext i32 @urem_si32_ui32_ui32(i32 %a, i32 %b) {
+; LA32-LABEL: urem_si32_ui32_ui32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    mod.wu $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: urem_si32_ui32_ui32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-NEXT:    mod.du $a0, $a0, $a1
+; LA64-NEXT:    addi.w $a0, $a0, 0
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: urem_si32_ui32_ui32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    mod.wu $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB29_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB29_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: urem_si32_ui32_ui32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-TRAP-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-TRAP-NEXT:    mod.du $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB29_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB29_2: # %entry
+; LA64-TRAP-NEXT:    addi.w $a0, $a0, 0
+; LA64-TRAP-NEXT:    ret
+entry:
+  %r = urem i32 %a, %b
+  ret i32 %r
+}
+
+define signext i32 @urem_si32_si32_si32(i32 signext %a, i32 signext %b) {
+; LA32-LABEL: urem_si32_si32_si32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    mod.wu $a0, $a0, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: urem_si32_si32_si32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-NEXT:    mod.du $a0, $a0, $a1
+; LA64-NEXT:    addi.w $a0, $a0, 0
+; LA64-NEXT:    ret
+;
+; LA32-TRAP-LABEL: urem_si32_si32_si32:
+; LA32-TRAP:       # %bb.0: # %entry
+; LA32-TRAP-NEXT:    mod.wu $a0, $a0, $a1
+; LA32-TRAP-NEXT:    bnez $a1, .LBB30_2
+; LA32-TRAP-NEXT:  # %bb.1: # %entry
+; LA32-TRAP-NEXT:    break 7
+; LA32-TRAP-NEXT:  .LBB30_2: # %entry
+; LA32-TRAP-NEXT:    ret
+;
+; LA64-TRAP-LABEL: urem_si32_si32_si32:
+; LA64-TRAP:       # %bb.0: # %entry
+; LA64-TRAP-NEXT:    bstrpick.d $a1, $a1, 31, 0
+; LA64-TRAP-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-TRAP-NEXT:    mod.du $a0, $a0, $a1
+; LA64-TRAP-NEXT:    bnez $a1, .LBB30_2
+; LA64-TRAP-NEXT:  # %bb.1: # %entry
+; LA64-TRAP-NEXT:    break 7
+; LA64-TRAP-NEXT:  .LBB30_2: # %entry
+; LA64-TRAP-NEXT:    addi.w $a0, $a0, 0
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = urem i32 %a, %b
@@ -730,10 +1170,10 @@ define i64 @urem_i64(i64 %a, i64 %b) {
 ; LA64-TRAP-LABEL: urem_i64:
 ; LA64-TRAP:       # %bb.0: # %entry
 ; LA64-TRAP-NEXT:    mod.du $a0, $a0, $a1
-; LA64-TRAP-NEXT:    bnez $a1, .LBB19_2
+; LA64-TRAP-NEXT:    bnez $a1, .LBB31_2
 ; LA64-TRAP-NEXT:  # %bb.1: # %entry
 ; LA64-TRAP-NEXT:    break 7
-; LA64-TRAP-NEXT:  .LBB19_2: # %entry
+; LA64-TRAP-NEXT:  .LBB31_2: # %entry
 ; LA64-TRAP-NEXT:    ret
 entry:
   %r = urem i64 %a, %b
-- 
GitLab


From 08536b0f9ccc208ea170b9451026eb1fe1fbb780 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <r@artagnon.com>
Date: Tue, 14 May 2024 16:19:55 +0100
Subject: [PATCH 0746/1206] [LAA] refactor tryToCreateDiffCheck (NFC) (#92110)

tryToCreateDiffCheck has one caller, and exits early if CanUseDiffCheck
is false. Hence, we can get/set CanUseDiffCheck in the caller to avoid
wastefully calling tryToCreateDiffCheck. This patch is an NFC
simplification of program logic.
---
 .../llvm/Analysis/LoopAccessAnalysis.h        |  2 +-
 llvm/lib/Analysis/LoopAccessAnalysis.cpp      | 55 +++++++------------
 2 files changed, 21 insertions(+), 36 deletions(-)

diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 6ebd0fb8477a..c22e1d470f38 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -540,7 +540,7 @@ private:
   /// Try to create add a new (pointer-difference, access size) pair to
   /// DiffCheck for checking groups \p CGI and \p CGJ. If pointer-difference
   /// checks cannot be used for the groups, set CanUseDiffCheck to false.
-  void tryToCreateDiffCheck(const RuntimeCheckingPtrGroup &CGI,
+  bool tryToCreateDiffCheck(const RuntimeCheckingPtrGroup &CGI,
                             const RuntimeCheckingPtrGroup &CGJ);
 
   MemoryDepChecker &DC;
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index d071e5332440..e92aa0265a1f 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -250,18 +250,13 @@ void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr,
                         NeedsFreeze);
 }
 
-void RuntimePointerChecking::tryToCreateDiffCheck(
+bool RuntimePointerChecking::tryToCreateDiffCheck(
     const RuntimeCheckingPtrGroup &CGI, const RuntimeCheckingPtrGroup &CGJ) {
-  if (!CanUseDiffCheck)
-    return;
-
   // If either group contains multiple different pointers, bail out.
   // TODO: Support multiple pointers by using the minimum or maximum pointer,
   // depending on src & sink.
-  if (CGI.Members.size() != 1 || CGJ.Members.size() != 1) {
-    CanUseDiffCheck = false;
-    return;
-  }
+  if (CGI.Members.size() != 1 || CGJ.Members.size() != 1)
+    return false;
 
   PointerInfo *Src = &Pointers[CGI.Members[0]];
   PointerInfo *Sink = &Pointers[CGJ.Members[0]];
@@ -269,10 +264,8 @@ void RuntimePointerChecking::tryToCreateDiffCheck(
   // If either pointer is read and written, multiple checks may be needed. Bail
   // out.
   if (!DC.getOrderForAccess(Src->PointerValue, !Src->IsWritePtr).empty() ||
-      !DC.getOrderForAccess(Sink->PointerValue, !Sink->IsWritePtr).empty()) {
-    CanUseDiffCheck = false;
-    return;
-  }
+      !DC.getOrderForAccess(Sink->PointerValue, !Sink->IsWritePtr).empty())
+    return false;
 
   ArrayRef<unsigned> AccSrc =
       DC.getOrderForAccess(Src->PointerValue, Src->IsWritePtr);
@@ -280,10 +273,9 @@ void RuntimePointerChecking::tryToCreateDiffCheck(
       DC.getOrderForAccess(Sink->PointerValue, Sink->IsWritePtr);
   // If either pointer is accessed multiple times, there may not be a clear
   // src/sink relation. Bail out for now.
-  if (AccSrc.size() != 1 || AccSink.size() != 1) {
-    CanUseDiffCheck = false;
-    return;
-  }
+  if (AccSrc.size() != 1 || AccSink.size() != 1)
+    return false;
+
   // If the sink is accessed before src, swap src/sink.
   if (AccSink[0] < AccSrc[0])
     std::swap(Src, Sink);
@@ -291,10 +283,8 @@ void RuntimePointerChecking::tryToCreateDiffCheck(
   auto *SrcAR = dyn_cast<SCEVAddRecExpr>(Src->Expr);
   auto *SinkAR = dyn_cast<SCEVAddRecExpr>(Sink->Expr);
   if (!SrcAR || !SinkAR || SrcAR->getLoop() != DC.getInnermostLoop() ||
-      SinkAR->getLoop() != DC.getInnermostLoop()) {
-    CanUseDiffCheck = false;
-    return;
-  }
+      SinkAR->getLoop() != DC.getInnermostLoop())
+    return false;
 
   SmallVector<Instruction *, 4> SrcInsts =
       DC.getInstructionsForAccess(Src->PointerValue, Src->IsWritePtr);
@@ -302,10 +292,9 @@ void RuntimePointerChecking::tryToCreateDiffCheck(
       DC.getInstructionsForAccess(Sink->PointerValue, Sink->IsWritePtr);
   Type *SrcTy = getLoadStoreType(SrcInsts[0]);
   Type *DstTy = getLoadStoreType(SinkInsts[0]);
-  if (isa<ScalableVectorType>(SrcTy) || isa<ScalableVectorType>(DstTy)) {
-    CanUseDiffCheck = false;
-    return;
-  }
+  if (isa<ScalableVectorType>(SrcTy) || isa<ScalableVectorType>(DstTy))
+    return false;
+
   const DataLayout &DL =
       SinkAR->getLoop()->getHeader()->getModule()->getDataLayout();
   unsigned AllocSize =
@@ -316,10 +305,8 @@ void RuntimePointerChecking::tryToCreateDiffCheck(
   // future.
   auto *Step = dyn_cast<SCEVConstant>(SinkAR->getStepRecurrence(*SE));
   if (!Step || Step != SrcAR->getStepRecurrence(*SE) ||
-      Step->getAPInt().abs() != AllocSize) {
-    CanUseDiffCheck = false;
-    return;
-  }
+      Step->getAPInt().abs() != AllocSize)
+    return false;
 
   IntegerType *IntTy =
       IntegerType::get(Src->PointerValue->getContext(),
@@ -332,10 +319,8 @@ void RuntimePointerChecking::tryToCreateDiffCheck(
   const SCEV *SinkStartInt = SE->getPtrToIntExpr(SinkAR->getStart(), IntTy);
   const SCEV *SrcStartInt = SE->getPtrToIntExpr(SrcAR->getStart(), IntTy);
   if (isa<SCEVCouldNotCompute>(SinkStartInt) ||
-      isa<SCEVCouldNotCompute>(SrcStartInt)) {
-    CanUseDiffCheck = false;
-    return;
-  }
+      isa<SCEVCouldNotCompute>(SrcStartInt))
+    return false;
 
   const Loop *InnerLoop = SrcAR->getLoop();
   // If the start values for both Src and Sink also vary according to an outer
@@ -356,8 +341,7 @@ void RuntimePointerChecking::tryToCreateDiffCheck(
             SinkStartAR->getStepRecurrence(*SE)) {
       LLVM_DEBUG(dbgs() << "LAA: Not creating diff runtime check, since these "
                            "cannot be hoisted out of the outer loop\n");
-      CanUseDiffCheck = false;
-      return;
+      return false;
     }
   }
 
@@ -366,6 +350,7 @@ void RuntimePointerChecking::tryToCreateDiffCheck(
                     << "SinkStartInt: " << *SinkStartInt << '\n');
   DiffChecks.emplace_back(SrcStartInt, SinkStartInt, AllocSize,
                           Src->NeedsFreeze || Sink->NeedsFreeze);
+  return true;
 }
 
 SmallVector<RuntimePointerCheck, 4> RuntimePointerChecking::generateChecks() {
@@ -377,7 +362,7 @@ SmallVector<RuntimePointerCheck, 4> RuntimePointerChecking::generateChecks() {
       const RuntimeCheckingPtrGroup &CGJ = CheckingGroups[J];
 
       if (needsChecking(CGI, CGJ)) {
-        tryToCreateDiffCheck(CGI, CGJ);
+        CanUseDiffCheck = CanUseDiffCheck && tryToCreateDiffCheck(CGI, CGJ);
         Checks.push_back(std::make_pair(&CGI, &CGJ));
       }
     }
-- 
GitLab


From 3d6f18db7b5bbf85bdd40c7c7d627baff2802b7c Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Tue, 14 May 2024 16:28:28 +0100
Subject: [PATCH 0747/1206] [AArch64] Remove redundant FDIV Combine. (#91924)

The target combine is no longer required because InstCombine will
transform the DIV by a power of 2 into a multiply, so in practice
this case will never trigger.

Additionally, the generated code would have been incorrect for
streaming(-compatible) functions, because it assumed NEON was available.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  73 +---------
 llvm/test/CodeGen/AArch64/fdiv_combine.ll     | 126 ------------------
 .../CodeGen/AArch64/sitofp-fixed-legal.ll     |  42 ------
 3 files changed, 1 insertion(+), 240 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AArch64/fdiv_combine.ll
 delete mode 100644 llvm/test/CodeGen/AArch64/sitofp-fixed-legal.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index f6d80f78910c..2ec9f66214b6 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1040,7 +1040,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
                        ISD::UINT_TO_FP});
 
   setTargetDAGCombine({ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
-                       ISD::FP_TO_UINT_SAT, ISD::FADD, ISD::FDIV});
+                       ISD::FP_TO_UINT_SAT, ISD::FADD});
 
   // Try and combine setcc with csel
   setTargetDAGCombine(ISD::SETCC);
@@ -17963,75 +17963,6 @@ static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG,
   return FixConv;
 }
 
-/// Fold a floating-point divide by power of two into fixed-point to
-/// floating-point conversion.
-static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG,
-                                  TargetLowering::DAGCombinerInfo &DCI,
-                                  const AArch64Subtarget *Subtarget) {
-  if (!Subtarget->hasNEON())
-    return SDValue();
-
-  SDValue Op = N->getOperand(0);
-  unsigned Opc = Op->getOpcode();
-  if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
-      !Op.getOperand(0).getValueType().isSimple() ||
-      (Opc != ISD::SINT_TO_FP && Opc != ISD::UINT_TO_FP))
-    return SDValue();
-
-  SDValue ConstVec = N->getOperand(1);
-  if (!isa<BuildVectorSDNode>(ConstVec))
-    return SDValue();
-
-  MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
-  int32_t IntBits = IntTy.getSizeInBits();
-  if (IntBits != 16 && IntBits != 32 && IntBits != 64)
-    return SDValue();
-
-  MVT FloatTy = N->getSimpleValueType(0).getVectorElementType();
-  int32_t FloatBits = FloatTy.getSizeInBits();
-  if (FloatBits != 32 && FloatBits != 64)
-    return SDValue();
-
-  // Avoid conversions where iN is larger than the float (e.g., i64 -> float).
-  if (IntBits > FloatBits)
-    return SDValue();
-
-  BitVector UndefElements;
-  BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
-  int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, FloatBits + 1);
-  if (C == -1 || C == 0 || C > FloatBits)
-    return SDValue();
-
-  MVT ResTy;
-  unsigned NumLanes = Op.getValueType().getVectorNumElements();
-  switch (NumLanes) {
-  default:
-    return SDValue();
-  case 2:
-    ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64;
-    break;
-  case 4:
-    ResTy = FloatBits == 32 ? MVT::v4i32 : MVT::v4i64;
-    break;
-  }
-
-  if (ResTy == MVT::v4i64 && DCI.isBeforeLegalizeOps())
-    return SDValue();
-
-  SDLoc DL(N);
-  SDValue ConvInput = Op.getOperand(0);
-  bool IsSigned = Opc == ISD::SINT_TO_FP;
-  if (IntBits < FloatBits)
-    ConvInput = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
-                            ResTy, ConvInput);
-
-  unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfxs2fp
-                                      : Intrinsic::aarch64_neon_vcvtfxu2fp;
-  return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
-                     DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), ConvInput,
-                     DAG.getConstant(C, DL, MVT::i32));
-}
-
 static SDValue tryCombineToBSL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                                const AArch64TargetLowering &TLI) {
   EVT VT = N->getValueType(0);
@@ -24720,8 +24651,6 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::FP_TO_SINT_SAT:
   case ISD::FP_TO_UINT_SAT:
     return performFpToIntCombine(N, DAG, DCI, Subtarget);
-  case ISD::FDIV:
-    return performFDivCombine(N, DAG, DCI, Subtarget);
   case ISD::OR:
     return performORCombine(N, DCI, Subtarget, *this);
   case ISD::AND:
diff --git a/llvm/test/CodeGen/AArch64/fdiv_combine.ll b/llvm/test/CodeGen/AArch64/fdiv_combine.ll
deleted file mode 100644
index 10b5f4386dd5..000000000000
--- a/llvm/test/CodeGen/AArch64/fdiv_combine.ll
+++ /dev/null
@@ -1,126 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-neon-syntax=apple -verify-machineinstrs -o - %s | FileCheck %s
-
-; Test signed conversion.
-define <2 x float> @test1(<2 x i32> %in) {
-; CHECK-LABEL: test1:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    scvtf.2s v0, v0, #4
-; CHECK-NEXT:    ret
-entry:
-  %vcvt.i = sitofp <2 x i32> %in to <2 x float>
-  %div.i = fdiv <2 x float> %vcvt.i, <float 16.0, float 16.0>
-  ret <2 x float> %div.i
-}
-
-; Test unsigned conversion.
-define <2 x float> @test2(<2 x i32> %in) {
-; CHECK-LABEL: test2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ucvtf.2s v0, v0, #3
-; CHECK-NEXT:    ret
-entry:
-  %vcvt.i = uitofp <2 x i32> %in to <2 x float>
-  %div.i = fdiv <2 x float> %vcvt.i, <float 8.0, float 8.0>
-  ret <2 x float> %div.i
-}
-
-; Test which should not fold due to non-power of 2.
-define <2 x float> @test3(<2 x i32> %in) {
-; CHECK-LABEL: test3:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov.2s v1, #9.00000000
-; CHECK-NEXT:    scvtf.2s v0, v0
-; CHECK-NEXT:    fdiv.2s v0, v0, v1
-; CHECK-NEXT:    ret
-entry:
-  %vcvt.i = sitofp <2 x i32> %in to <2 x float>
-  %div.i = fdiv <2 x float> %vcvt.i, <float 9.0, float 9.0>
-  ret <2 x float> %div.i
-}
-
-; Test which should not fold due to power of 2 out of range.
-define <2 x float> @test4(<2 x i32> %in) {
-; CHECK-LABEL: test4:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi.2s v1, #80, lsl #24
-; CHECK-NEXT:    scvtf.2s v0, v0
-; CHECK-NEXT:    fdiv.2s v0, v0, v1
-; CHECK-NEXT:    ret
-entry:
-  %vcvt.i = sitofp <2 x i32> %in to <2 x float>
-  %div.i = fdiv <2 x float> %vcvt.i, <float 0x4200000000000000, float 0x4200000000000000>
-  ret <2 x float> %div.i
-}
-
-; Test case where const is max power of 2 (i.e., 2^32).
-define <2 x float> @test5(<2 x i32> %in) {
-; CHECK-LABEL: test5:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    scvtf.2s v0, v0, #32
-; CHECK-NEXT:    ret
-entry:
-  %vcvt.i = sitofp <2 x i32> %in to <2 x float>
-  %div.i = fdiv <2 x float> %vcvt.i, <float 0x41F0000000000000, float 0x41F0000000000000>
-  ret <2 x float> %div.i
-}
-
-; Test quadword.
-define <4 x float> @test6(<4 x i32> %in) {
-; CHECK-LABEL: test6:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    scvtf.4s v0, v0, #2
-; CHECK-NEXT:    ret
-entry:
-  %vcvt.i = sitofp <4 x i32> %in to <4 x float>
-  %div.i = fdiv <4 x float> %vcvt.i, <float 4.0, float 4.0, float 4.0, float 4.0>
-  ret <4 x float> %div.i
-}
-
-; Test unsigned i16 to float
-define <4 x float> @test7(<4 x i16> %in) {
-; CHECK-LABEL: test7:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ucvtf.4s v0, v0, #1
-; CHECK-NEXT:    ret
-  %conv = uitofp <4 x i16> %in to <4 x float>
-  %shift = fdiv <4 x float> %conv, <float 2.0, float 2.0, float 2.0, float 2.0>
-  ret <4 x float> %shift
-}
-
-; Test signed i16 to float
-define <4 x float> @test8(<4 x i16> %in) {
-; CHECK-LABEL: test8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    scvtf.4s v0, v0, #2
-; CHECK-NEXT:    ret
-  %conv = sitofp <4 x i16> %in to <4 x float>
-  %shift = fdiv <4 x float> %conv, <float 4.0, float 4.0, float 4.0, float 4.0>
-  ret <4 x float> %shift
-}
-
-; Can't convert i64 to float.
-define <2 x float> @test9(<2 x i64> %in) {
-; CHECK-LABEL: test9:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ucvtf.2d v0, v0
-; CHECK-NEXT:    movi.2s v1, #64, lsl #24
-; CHECK-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-NEXT:    fdiv.2s v0, v0, v1
-; CHECK-NEXT:    ret
-  %conv = uitofp <2 x i64> %in to <2 x float>
-  %shift = fdiv <2 x float> %conv, <float 2.0, float 2.0>
-  ret <2 x float> %shift
-}
-
-define <2 x double> @test10(<2 x i64> %in) {
-; CHECK-LABEL: test10:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ucvtf.2d v0, v0, #1
-; CHECK-NEXT:    ret
-  %conv = uitofp <2 x i64> %in to <2 x double>
-  %shift = fdiv <2 x double> %conv, <double 2.0, double 2.0>
-  ret <2 x double> %shift
-}
diff --git a/llvm/test/CodeGen/AArch64/sitofp-fixed-legal.ll b/llvm/test/CodeGen/AArch64/sitofp-fixed-legal.ll
deleted file mode 100644
index 5a5a669e92ee..000000000000
--- a/llvm/test/CodeGen/AArch64/sitofp-fixed-legal.ll
+++ /dev/null
@@ -1,42 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-apple-ios %s -o - | FileCheck %s
-
-define <16 x double> @test_sitofp_fixed(<16 x i32> %in) {
-; CHECK-LABEL: test_sitofp_fixed:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    sshll2.2d v4, v0, #0
-; CHECK-NEXT:    sshll.2d v0, v0, #0
-; CHECK-NEXT:    sshll2.2d v5, v1, #0
-; CHECK-NEXT:    sshll.2d v6, v1, #0
-; CHECK-NEXT:    sshll.2d v7, v2, #0
-; CHECK-NEXT:    sshll2.2d v16, v2, #0
-; CHECK-NEXT:    sshll2.2d v17, v3, #0
-; CHECK-NEXT:    sshll.2d v18, v3, #0
-; CHECK-NEXT:    scvtf.2d v1, v4, #6
-; CHECK-NEXT:    scvtf.2d v0, v0, #6
-; CHECK-NEXT:    scvtf.2d v3, v5, #6
-; CHECK-NEXT:    scvtf.2d v2, v6, #6
-; CHECK-NEXT:    scvtf.2d v4, v7, #6
-; CHECK-NEXT:    scvtf.2d v5, v16, #6
-; CHECK-NEXT:    scvtf.2d v7, v17, #6
-; CHECK-NEXT:    scvtf.2d v6, v18, #6
-; CHECK-NEXT:    ret
-
-  %flt = sitofp <16 x i32> %in to <16 x double>
-  %res = fdiv <16 x double> %flt, <double 64.0, double 64.0, double 64.0, double 64.0, double 64.0, double 64.0, double 64.0, double 64.0, double 64.0, double 64.0, double 64.0, double 64.0, double 64.0, double 64.0, double 64.0, double 64.0>
-  ret <16 x double> %res
-}
-
-; This one is small enough to satisfy isSimple, but still illegally large.
-define <4 x double> @test_sitofp_fixed_shortish(<4 x i64> %in) {
-; CHECK-LABEL: test_sitofp_fixed_shortish:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    scvtf.2d v0, v0, #6
-; CHECK-NEXT:    scvtf.2d v1, v1, #6
-; CHECK-NEXT:    ret
-
-
-  %flt = sitofp <4 x i64> %in to <4 x double>
-  %res = fdiv <4 x double> %flt, <double 64.0, double 64.0, double 64.0, double 64.0>
-  ret <4 x double> %res
-}
-- 
GitLab


From b2c5e9b9bf2a1cb4a8d4fc67f3201db55ae2cae1 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 14 May 2024 16:36:25 +0100
Subject: [PATCH 0748/1206] [ARM] iabs.ll - regenerate test checks

---
 llvm/test/CodeGen/ARM/iabs.ll | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/llvm/test/CodeGen/ARM/iabs.ll b/llvm/test/CodeGen/ARM/iabs.ll
index bcedcc8fe63b..fffa9555b296 100644
--- a/llvm/test/CodeGen/ARM/iabs.ll
+++ b/llvm/test/CodeGen/ARM/iabs.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=arm-eabi -mattr=+v4t %s -o - | FileCheck %s
 
 ;; Integer absolute value, should produce something as good as: ARM:
@@ -6,13 +7,15 @@
 ;;   bx lr
 
 define i32 @test(i32 %a) {
-        %tmp1neg = sub i32 0, %a
-        %b = icmp sgt i32 %a, -1
-        %abs = select i1 %b, i32 %a, i32 %tmp1neg
-        ret i32 %abs
-; CHECK:  cmp
-; CHECK:  rsbmi r0, r0, #0
-; CHECK:  bx lr
+; CHECK-LABEL: test:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    rsbmi r0, r0, #0
+; CHECK-NEXT:    bx lr
+  %tmp1neg = sub i32 0, %a
+  %b = icmp sgt i32 %a, -1
+  %abs = select i1 %b, i32 %a, i32 %tmp1neg
+  ret i32 %abs
 }
 
 ; rdar://11633193
@@ -21,11 +24,12 @@ define i32 @test(i32 %a) {
 ;;   rsbmi
 ;;   bx
 define i32 @test2(i32 %a, i32 %b) nounwind readnone ssp {
+; CHECK-LABEL: test2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, r0, r1
+; CHECK-NEXT:    rsbmi r0, r0, #0
+; CHECK-NEXT:    bx lr
 entry:
-; CHECK: test2
-; CHECK: subs
-; CHECK-NEXT: rsbmi
-; CHECK-NEXT: bx
   %sub = sub nsw i32 %a, %b
   %cmp = icmp sgt i32 %sub, -1
   %sub1 = sub nsw i32 0, %sub
-- 
GitLab


From 0f17d9a28c40eebd42c83956e2a7b5186c1814d7 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Tue, 14 May 2024 20:05:22 +0400
Subject: [PATCH 0749/1206] [lldb] Fixed the test TestThreadStates when run
 with a remote target (#92086)

self.wait_for_running_event(process) is always called after
self.runCmd("continue"). It is strange to expect eStateConnected here.
This test failed in case of a remote target. The correct state is
eStateRunning. Removed incorrect checking.
---
 .../test/API/functionalities/thread/state/TestThreadStates.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lldb/test/API/functionalities/thread/state/TestThreadStates.py b/lldb/test/API/functionalities/thread/state/TestThreadStates.py
index f4c17df52338..4dbe230c0ce8 100644
--- a/lldb/test/API/functionalities/thread/state/TestThreadStates.py
+++ b/lldb/test/API/functionalities/thread/state/TestThreadStates.py
@@ -102,10 +102,6 @@ class ThreadStateTestCase(TestBase):
 
     def wait_for_running_event(self, process):
         listener = self.dbg.GetListener()
-        if lldb.remote_platform:
-            lldbutil.expect_state_changes(
-                self, listener, process, [lldb.eStateConnected]
-            )
         lldbutil.expect_state_changes(self, listener, process, [lldb.eStateRunning])
 
     def thread_state_after_continue_test(self):
-- 
GitLab


From 4c68de5a0027fca9ebff5f8ffec3a35a43d14e74 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu@sifive.com>
Date: Tue, 14 May 2024 09:18:08 -0700
Subject: [PATCH 0750/1206] [RISCV][CostModel] Add cost model for
 experimental.cttz.elts (#91778)

The cost of `experimental.cttz.elts` in RISC-V equals to the cost of
vfirst when the zero_is_poison argument is true. Otherwise, we add
additional costs of cmp + select to convert the -1 result from vfirst to
EVL.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.h     |  4 +-
 .../Target/RISCV/RISCVTargetTransformInfo.cpp | 21 ++++++++
 .../Analysis/CostModel/RISCV/cttz_elts.ll     | 48 +++++++++----------
 3 files changed, 47 insertions(+), 26 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index afc317f94dae..1efc54566b4b 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -567,6 +567,8 @@ public:
   shouldExpandBuildVectorWithShuffles(EVT VT,
                                       unsigned DefinedValues) const override;
 
+  bool shouldExpandCttzElements(EVT VT) const override;
+
   /// Return the cost of LMUL for linear operations.
   InstructionCost getLMULCost(MVT VT) const;
 
@@ -1001,8 +1003,6 @@ private:
   bool shouldExpandGetVectorLength(EVT TripCountVT, unsigned VF,
                                    bool IsScalable) const override;
 
-  bool shouldExpandCttzElements(EVT VT) const override;
-
   /// RVV code generation for fixed length vectors does not lower all
   /// BUILD_VECTORs. This makes BUILD_VECTOR legalisation a source of stores to
   /// merge. However, merging them creates a BUILD_VECTOR that is just as
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index d94dff5f2b1f..4d2479fc233f 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -97,6 +97,7 @@ RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
     case RISCV::VMANDN_MM:
     case RISCV::VMNAND_MM:
     case RISCV::VCPOP_M:
+    case RISCV::VFIRST_M:
       Cost += 1;
       break;
     default:
@@ -901,6 +902,26 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                  getRISCVInstructionCost(RISCV::VADD_VX, LT.second, CostKind);
     return 1 + (LT.first - 1);
   }
+  case Intrinsic::experimental_cttz_elts: {
+    Type *ArgTy = ICA.getArgTypes()[0];
+    EVT ArgType = TLI->getValueType(DL, ArgTy, true);
+    if (getTLI()->shouldExpandCttzElements(ArgType))
+      break;
+    InstructionCost Cost = getRISCVInstructionCost(
+        RISCV::VFIRST_M, getTypeLegalizationCost(ArgTy).second, CostKind);
+
+    // If zero_is_poison is false, then we will generate additional
+    // cmp + select instructions to convert -1 to EVL.
+    Type *BoolTy = Type::getInt1Ty(RetTy->getContext());
+    if (ICA.getArgs().size() > 1 &&
+        cast<ConstantInt>(ICA.getArgs()[1])->isZero())
+      Cost += getCmpSelInstrCost(Instruction::ICmp, BoolTy, RetTy,
+                                 CmpInst::ICMP_SLT, CostKind) +
+              getCmpSelInstrCost(Instruction::Select, RetTy, BoolTy,
+                                 CmpInst::BAD_ICMP_PREDICATE, CostKind);
+
+    return Cost;
+  }
   case Intrinsic::vp_rint: {
     // RISC-V target uses at least 5 instructions to lower rounding intrinsics.
     unsigned Cost = 5;
diff --git a/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll b/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
index ca09d027b547..118d92b80d25 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
@@ -17,19 +17,19 @@ define void @foo_no_vscale_range() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 390 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 781 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 390 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -83,19 +83,19 @@ define void @foo_vscale_range_2_16() vscale_range(2,16) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-- 
GitLab


From 8019cbbbbc94658d133583f7be6cd0023d30b0f3 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Tue, 14 May 2024 12:28:58 -0400
Subject: [PATCH 0751/1206] [Clang][Sema] Earlier type checking for builtin
 unary operators (#90500)

Currently, clang postpones all semantic analysis of unary operators with
operands of pointer/pointer to member/array/function type until
instantiation whenever that type is dependent (e.g. `T*` where `T` is a
type template parameter). Consequently, the uninstantiated AST nodes all
have the type `ASTContext::DependentTy` (which, for the purposes of
#90152, is undesirable as that type may be the current instantiation!
(e.g. `*this`))

This patch moves the point at which we perform semantic analysis for
such expression to be prior to instantiation.
---
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/AST/Type.h                |   5 +-
 clang/lib/Sema/SemaExpr.cpp                   | 354 +++++++++---------
 clang/test/AST/ast-dump-expr-json.cpp         |   4 +-
 clang/test/AST/ast-dump-expr.cpp              |   2 +-
 clang/test/AST/ast-dump-lambda.cpp            |   2 +-
 .../expr/expr.unary/expr.unary.general/p1.cpp |  65 ++++
 clang/test/CXX/over/over.built/ast.cpp        | 158 ++++++--
 clang/test/CXX/over/over.built/p10.cpp        |   2 +-
 clang/test/CXX/over/over.built/p11.cpp        |   2 +-
 .../temp.res/temp.dep/temp.dep.type/p4.cpp    |  25 +-
 clang/test/Frontend/noderef_templates.cpp     |   4 +-
 clang/test/SemaCXX/cxx2b-deducing-this.cpp    |   6 +-
 .../test/SemaTemplate/class-template-spec.cpp |  12 +-
 .../ASTMatchers/ASTMatchersNarrowingTest.cpp  |   6 +-
 15 files changed, 404 insertions(+), 246 deletions(-)
 create mode 100644 clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 49ab222bec40..a2e44efe4134 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -55,6 +55,9 @@ C++ Specific Potentially Breaking Changes
 
 - Clang now rejects pointer to member from parenthesized expression in unevaluated context such as ``decltype(&(foo::bar))``. (#GH40906).
 
+- Clang now performs semantic analysis for unary operators with dependent operands
+  that are known to be of non-class non-enumeration type prior to instantiation.
+
 ABI Changes in This Version
 ---------------------------
 - Fixed Microsoft name mangling of implicitly defined variables used for thread
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index e6643469e0b3..da3834f19ca0 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -8044,7 +8044,10 @@ inline bool Type::isUndeducedType() const {
 /// Determines whether this is a type for which one can define
 /// an overloaded operator.
 inline bool Type::isOverloadableType() const {
-  return isDependentType() || isRecordType() || isEnumeralType();
+  if (!CanonicalType->isDependentType())
+    return isRecordType() || isEnumeralType();
+  return !isArrayType() && !isFunctionType() && !isAnyPointerType() &&
+         !isMemberPointerType();
 }
 
 /// Determines whether this type is written as a typedef-name.
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index e6c3fa51d54d..18fd5ba700ad 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -672,12 +672,12 @@ ExprResult Sema::DefaultLvalueConversion(Expr *E) {
 
   // We don't want to throw lvalue-to-rvalue casts on top of
   // expressions of certain types in C++.
-  if (getLangOpts().CPlusPlus &&
-      (E->getType() == Context.OverloadTy ||
-       // FIXME: This is a hack! We want the lvalue-to-rvalue conversion applied
-       // to pointer types even if the pointee type is dependent.
-       (T->isDependentType() && !T->isPointerType()) || T->isRecordType()))
-    return E;
+  if (getLangOpts().CPlusPlus) {
+    if (T == Context.OverloadTy || T->isRecordType() ||
+        (T->isDependentType() && !T->isAnyPointerType() &&
+         !T->isMemberPointerType()))
+      return E;
+  }
 
   // The C standard is actually really unclear on this point, and
   // DR106 tells us what the result should be but not why.  It's
@@ -10827,7 +10827,7 @@ static bool checkArithmeticIncompletePointerType(Sema &S, SourceLocation Loc,
   if (const AtomicType *ResAtomicType = ResType->getAs<AtomicType>())
     ResType = ResAtomicType->getValueType();
 
-  assert(ResType->isAnyPointerType() && !ResType->isDependentType());
+  assert(ResType->isAnyPointerType());
   QualType PointeeTy = ResType->getPointeeType();
   return S.RequireCompleteSizedType(
       Loc, PointeeTy,
@@ -13955,11 +13955,8 @@ static QualType CheckCommaOperands(Sema &S, ExprResult &LHS, ExprResult &RHS,
 static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
                                                ExprValueKind &VK,
                                                ExprObjectKind &OK,
-                                               SourceLocation OpLoc,
-                                               bool IsInc, bool IsPrefix) {
-  if (Op->isTypeDependent())
-    return S.Context.DependentTy;
-
+                                               SourceLocation OpLoc, bool IsInc,
+                                               bool IsPrefix) {
   QualType ResType = Op->getType();
   // Atomic types can be used for increment / decrement where the non-atomic
   // versions can, so ignore the _Atomic() specifier for the purpose of
@@ -14041,7 +14038,6 @@ static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
   }
 }
 
-
 /// getPrimaryDecl - Helper function for CheckAddressOfOperand().
 /// This routine allows us to typecheck complex/recursive expressions
 /// where the declaration is needed for type checking. We only need to
@@ -14411,9 +14407,6 @@ static void RecordModifiableNonNullParam(Sema &S, const Expr *Exp) {
 static QualType CheckIndirectionOperand(Sema &S, Expr *Op, ExprValueKind &VK,
                                         SourceLocation OpLoc,
                                         bool IsAfterAmp = false) {
-  if (Op->isTypeDependent())
-    return S.Context.DependentTy;
-
   ExprResult ConvResult = S.UsualUnaryConversions(Op);
   if (ConvResult.isInvalid())
     return QualType();
@@ -15467,188 +15460,191 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
       return ExprError(Diag(OpLoc, diag::err_hlsl_operator_unsupported) << 1);
   }
 
-  switch (Opc) {
-  case UO_PreInc:
-  case UO_PreDec:
-  case UO_PostInc:
-  case UO_PostDec:
-    resultType = CheckIncrementDecrementOperand(*this, Input.get(), VK, OK,
-                                                OpLoc,
-                                                Opc == UO_PreInc ||
-                                                Opc == UO_PostInc,
-                                                Opc == UO_PreInc ||
-                                                Opc == UO_PreDec);
-    CanOverflow = isOverflowingIntegerType(Context, resultType);
-    break;
-  case UO_AddrOf:
-    resultType = CheckAddressOfOperand(Input, OpLoc);
-    CheckAddressOfNoDeref(InputExpr);
-    RecordModifiableNonNullParam(*this, InputExpr);
-    break;
-  case UO_Deref: {
-    Input = DefaultFunctionArrayLvalueConversion(Input.get());
-    if (Input.isInvalid()) return ExprError();
-    resultType =
-        CheckIndirectionOperand(*this, Input.get(), VK, OpLoc, IsAfterAmp);
-    break;
-  }
-  case UO_Plus:
-  case UO_Minus:
-    CanOverflow = Opc == UO_Minus &&
-                  isOverflowingIntegerType(Context, Input.get()->getType());
-    Input = UsualUnaryConversions(Input.get());
-    if (Input.isInvalid()) return ExprError();
-    // Unary plus and minus require promoting an operand of half vector to a
-    // float vector and truncating the result back to a half vector. For now, we
-    // do this only when HalfArgsAndReturns is set (that is, when the target is
-    // arm or arm64).
-    ConvertHalfVec = needsConversionOfHalfVec(true, Context, Input.get());
-
-    // If the operand is a half vector, promote it to a float vector.
-    if (ConvertHalfVec)
-      Input = convertVector(Input.get(), Context.FloatTy, *this);
-    resultType = Input.get()->getType();
-    if (resultType->isDependentType())
-      break;
-    if (resultType->isArithmeticType()) // C99 6.5.3.3p1
-      break;
-    else if (resultType->isVectorType() &&
-             // The z vector extensions don't allow + or - with bool vectors.
-             (!Context.getLangOpts().ZVector ||
-              resultType->castAs<VectorType>()->getVectorKind() !=
-                  VectorKind::AltiVecBool))
-      break;
-    else if (resultType->isSveVLSBuiltinType()) // SVE vectors allow + and -
-      break;
-    else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
-             Opc == UO_Plus &&
-             resultType->isPointerType())
+  if (InputExpr->isTypeDependent() &&
+      InputExpr->getType()->isSpecificBuiltinType(BuiltinType::Dependent)) {
+    resultType = Context.DependentTy;
+  } else {
+    switch (Opc) {
+    case UO_PreInc:
+    case UO_PreDec:
+    case UO_PostInc:
+    case UO_PostDec:
+      resultType =
+          CheckIncrementDecrementOperand(*this, Input.get(), VK, OK, OpLoc,
+                                         Opc == UO_PreInc || Opc == UO_PostInc,
+                                         Opc == UO_PreInc || Opc == UO_PreDec);
+      CanOverflow = isOverflowingIntegerType(Context, resultType);
       break;
-
-    return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-      << resultType << Input.get()->getSourceRange());
-
-  case UO_Not: // bitwise complement
-    Input = UsualUnaryConversions(Input.get());
-    if (Input.isInvalid())
-      return ExprError();
-    resultType = Input.get()->getType();
-    if (resultType->isDependentType())
+    case UO_AddrOf:
+      resultType = CheckAddressOfOperand(Input, OpLoc);
+      CheckAddressOfNoDeref(InputExpr);
+      RecordModifiableNonNullParam(*this, InputExpr);
       break;
-    // C99 6.5.3.3p1. We allow complex int and float as a GCC extension.
-    if (resultType->isComplexType() || resultType->isComplexIntegerType())
-      // C99 does not support '~' for complex conjugation.
-      Diag(OpLoc, diag::ext_integer_complement_complex)
-          << resultType << Input.get()->getSourceRange();
-    else if (resultType->hasIntegerRepresentation())
+    case UO_Deref: {
+      Input = DefaultFunctionArrayLvalueConversion(Input.get());
+      if (Input.isInvalid())
+        return ExprError();
+      resultType =
+          CheckIndirectionOperand(*this, Input.get(), VK, OpLoc, IsAfterAmp);
       break;
-    else if (resultType->isExtVectorType() && Context.getLangOpts().OpenCL) {
-      // OpenCL v1.1 s6.3.f: The bitwise operator not (~) does not operate
-      // on vector float types.
-      QualType T = resultType->castAs<ExtVectorType>()->getElementType();
-      if (!T->isIntegerType())
-        return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                          << resultType << Input.get()->getSourceRange());
-    } else {
-      return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                       << resultType << Input.get()->getSourceRange());
-    }
-    break;
-
-  case UO_LNot: // logical negation
-    // Unlike +/-/~, integer promotions aren't done here (C99 6.5.3.3p5).
-    Input = DefaultFunctionArrayLvalueConversion(Input.get());
-    if (Input.isInvalid()) return ExprError();
-    resultType = Input.get()->getType();
-
-    // Though we still have to promote half FP to float...
-    if (resultType->isHalfType() && !Context.getLangOpts().NativeHalfType) {
-      Input = ImpCastExprToType(Input.get(), Context.FloatTy, CK_FloatingCast).get();
-      resultType = Context.FloatTy;
     }
+    case UO_Plus:
+    case UO_Minus:
+      CanOverflow = Opc == UO_Minus &&
+                    isOverflowingIntegerType(Context, Input.get()->getType());
+      Input = UsualUnaryConversions(Input.get());
+      if (Input.isInvalid())
+        return ExprError();
+      // Unary plus and minus require promoting an operand of half vector to a
+      // float vector and truncating the result back to a half vector. For now,
+      // we do this only when HalfArgsAndReturns is set (that is, when the
+      // target is arm or arm64).
+      ConvertHalfVec = needsConversionOfHalfVec(true, Context, Input.get());
+
+      // If the operand is a half vector, promote it to a float vector.
+      if (ConvertHalfVec)
+        Input = convertVector(Input.get(), Context.FloatTy, *this);
+      resultType = Input.get()->getType();
+      if (resultType->isArithmeticType()) // C99 6.5.3.3p1
+        break;
+      else if (resultType->isVectorType() &&
+               // The z vector extensions don't allow + or - with bool vectors.
+               (!Context.getLangOpts().ZVector ||
+                resultType->castAs<VectorType>()->getVectorKind() !=
+                    VectorKind::AltiVecBool))
+        break;
+      else if (resultType->isSveVLSBuiltinType()) // SVE vectors allow + and -
+        break;
+      else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
+               Opc == UO_Plus && resultType->isPointerType())
+        break;
 
-    // WebAsembly tables can't be used in unary expressions.
-    if (resultType->isPointerType() &&
-        resultType->getPointeeType().isWebAssemblyReferenceType()) {
       return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
                        << resultType << Input.get()->getSourceRange());
-    }
 
-    if (resultType->isDependentType())
-      break;
-    if (resultType->isScalarType() && !isScopedEnumerationType(resultType)) {
-      // C99 6.5.3.3p1: ok, fallthrough;
-      if (Context.getLangOpts().CPlusPlus) {
-        // C++03 [expr.unary.op]p8, C++0x [expr.unary.op]p9:
-        // operand contextually converted to bool.
-        Input = ImpCastExprToType(Input.get(), Context.BoolTy,
-                                  ScalarTypeToBooleanCastKind(resultType));
-      } else if (Context.getLangOpts().OpenCL &&
-                 Context.getLangOpts().OpenCLVersion < 120) {
-        // OpenCL v1.1 6.3.h: The logical operator not (!) does not
-        // operate on scalar float types.
-        if (!resultType->isIntegerType() && !resultType->isPointerType())
-          return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                           << resultType << Input.get()->getSourceRange());
-      }
-    } else if (resultType->isExtVectorType()) {
-      if (Context.getLangOpts().OpenCL &&
-          Context.getLangOpts().getOpenCLCompatibleVersion() < 120) {
-        // OpenCL v1.1 6.3.h: The logical operator not (!) does not
-        // operate on vector float types.
+    case UO_Not: // bitwise complement
+      Input = UsualUnaryConversions(Input.get());
+      if (Input.isInvalid())
+        return ExprError();
+      resultType = Input.get()->getType();
+      // C99 6.5.3.3p1. We allow complex int and float as a GCC extension.
+      if (resultType->isComplexType() || resultType->isComplexIntegerType())
+        // C99 does not support '~' for complex conjugation.
+        Diag(OpLoc, diag::ext_integer_complement_complex)
+            << resultType << Input.get()->getSourceRange();
+      else if (resultType->hasIntegerRepresentation())
+        break;
+      else if (resultType->isExtVectorType() && Context.getLangOpts().OpenCL) {
+        // OpenCL v1.1 s6.3.f: The bitwise operator not (~) does not operate
+        // on vector float types.
         QualType T = resultType->castAs<ExtVectorType>()->getElementType();
         if (!T->isIntegerType())
           return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
                            << resultType << Input.get()->getSourceRange());
+      } else {
+        return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                         << resultType << Input.get()->getSourceRange());
       }
-      // Vector logical not returns the signed variant of the operand type.
-      resultType = GetSignedVectorType(resultType);
       break;
-    } else if (Context.getLangOpts().CPlusPlus && resultType->isVectorType()) {
-      const VectorType *VTy = resultType->castAs<VectorType>();
-      if (VTy->getVectorKind() != VectorKind::Generic)
+
+    case UO_LNot: // logical negation
+      // Unlike +/-/~, integer promotions aren't done here (C99 6.5.3.3p5).
+      Input = DefaultFunctionArrayLvalueConversion(Input.get());
+      if (Input.isInvalid())
+        return ExprError();
+      resultType = Input.get()->getType();
+
+      // Though we still have to promote half FP to float...
+      if (resultType->isHalfType() && !Context.getLangOpts().NativeHalfType) {
+        Input = ImpCastExprToType(Input.get(), Context.FloatTy, CK_FloatingCast)
+                    .get();
+        resultType = Context.FloatTy;
+      }
+
+      // WebAsembly tables can't be used in unary expressions.
+      if (resultType->isPointerType() &&
+          resultType->getPointeeType().isWebAssemblyReferenceType()) {
         return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
                          << resultType << Input.get()->getSourceRange());
+      }
 
-      // Vector logical not returns the signed variant of the operand type.
-      resultType = GetSignedVectorType(resultType);
-      break;
-    } else {
-      return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-        << resultType << Input.get()->getSourceRange());
-    }
+      if (resultType->isScalarType() && !isScopedEnumerationType(resultType)) {
+        // C99 6.5.3.3p1: ok, fallthrough;
+        if (Context.getLangOpts().CPlusPlus) {
+          // C++03 [expr.unary.op]p8, C++0x [expr.unary.op]p9:
+          // operand contextually converted to bool.
+          Input = ImpCastExprToType(Input.get(), Context.BoolTy,
+                                    ScalarTypeToBooleanCastKind(resultType));
+        } else if (Context.getLangOpts().OpenCL &&
+                   Context.getLangOpts().OpenCLVersion < 120) {
+          // OpenCL v1.1 6.3.h: The logical operator not (!) does not
+          // operate on scalar float types.
+          if (!resultType->isIntegerType() && !resultType->isPointerType())
+            return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                             << resultType << Input.get()->getSourceRange());
+        }
+      } else if (resultType->isExtVectorType()) {
+        if (Context.getLangOpts().OpenCL &&
+            Context.getLangOpts().getOpenCLCompatibleVersion() < 120) {
+          // OpenCL v1.1 6.3.h: The logical operator not (!) does not
+          // operate on vector float types.
+          QualType T = resultType->castAs<ExtVectorType>()->getElementType();
+          if (!T->isIntegerType())
+            return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                             << resultType << Input.get()->getSourceRange());
+        }
+        // Vector logical not returns the signed variant of the operand type.
+        resultType = GetSignedVectorType(resultType);
+        break;
+      } else if (Context.getLangOpts().CPlusPlus &&
+                 resultType->isVectorType()) {
+        const VectorType *VTy = resultType->castAs<VectorType>();
+        if (VTy->getVectorKind() != VectorKind::Generic)
+          return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                           << resultType << Input.get()->getSourceRange());
 
-    // LNot always has type int. C99 6.5.3.3p5.
-    // In C++, it's bool. C++ 5.3.1p8
-    resultType = Context.getLogicalOperationType();
-    break;
-  case UO_Real:
-  case UO_Imag:
-    resultType = CheckRealImagOperand(*this, Input, OpLoc, Opc == UO_Real);
-    // _Real maps ordinary l-values into ordinary l-values. _Imag maps ordinary
-    // complex l-values to ordinary l-values and all other values to r-values.
-    if (Input.isInvalid()) return ExprError();
-    if (Opc == UO_Real || Input.get()->getType()->isAnyComplexType()) {
-      if (Input.get()->isGLValue() &&
-          Input.get()->getObjectKind() == OK_Ordinary)
-        VK = Input.get()->getValueKind();
-    } else if (!getLangOpts().CPlusPlus) {
-      // In C, a volatile scalar is read by __imag. In C++, it is not.
-      Input = DefaultLvalueConversion(Input.get());
+        // Vector logical not returns the signed variant of the operand type.
+        resultType = GetSignedVectorType(resultType);
+        break;
+      } else {
+        return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                         << resultType << Input.get()->getSourceRange());
+      }
+
+      // LNot always has type int. C99 6.5.3.3p5.
+      // In C++, it's bool. C++ 5.3.1p8
+      resultType = Context.getLogicalOperationType();
+      break;
+    case UO_Real:
+    case UO_Imag:
+      resultType = CheckRealImagOperand(*this, Input, OpLoc, Opc == UO_Real);
+      // _Real maps ordinary l-values into ordinary l-values. _Imag maps
+      // ordinary complex l-values to ordinary l-values and all other values to
+      // r-values.
+      if (Input.isInvalid())
+        return ExprError();
+      if (Opc == UO_Real || Input.get()->getType()->isAnyComplexType()) {
+        if (Input.get()->isGLValue() &&
+            Input.get()->getObjectKind() == OK_Ordinary)
+          VK = Input.get()->getValueKind();
+      } else if (!getLangOpts().CPlusPlus) {
+        // In C, a volatile scalar is read by __imag. In C++, it is not.
+        Input = DefaultLvalueConversion(Input.get());
+      }
+      break;
+    case UO_Extension:
+      resultType = Input.get()->getType();
+      VK = Input.get()->getValueKind();
+      OK = Input.get()->getObjectKind();
+      break;
+    case UO_Coawait:
+      // It's unnecessary to represent the pass-through operator co_await in the
+      // AST; just return the input expression instead.
+      assert(!Input.get()->getType()->isDependentType() &&
+             "the co_await expression must be non-dependant before "
+             "building operator co_await");
+      return Input;
     }
-    break;
-  case UO_Extension:
-    resultType = Input.get()->getType();
-    VK = Input.get()->getValueKind();
-    OK = Input.get()->getObjectKind();
-    break;
-  case UO_Coawait:
-    // It's unnecessary to represent the pass-through operator co_await in the
-    // AST; just return the input expression instead.
-    assert(!Input.get()->getType()->isDependentType() &&
-                   "the co_await expression must be non-dependant before "
-                   "building operator co_await");
-    return Input;
   }
   if (resultType.isNull() || Input.isInvalid())
     return ExprError();
diff --git a/clang/test/AST/ast-dump-expr-json.cpp b/clang/test/AST/ast-dump-expr-json.cpp
index 0fb07b0b434c..4b7365e554cb 100644
--- a/clang/test/AST/ast-dump-expr-json.cpp
+++ b/clang/test/AST/ast-dump-expr-json.cpp
@@ -4261,9 +4261,9 @@ void TestNonADLCall3() {
 // CHECK-NEXT:                     }
 // CHECK-NEXT:                    },
 // CHECK-NEXT:                    "type": {
-// CHECK-NEXT:                     "qualType": "<dependent type>"
+// CHECK-NEXT:                     "qualType": "V"
 // CHECK-NEXT:                    },
-// CHECK-NEXT:                    "valueCategory": "prvalue",
+// CHECK-NEXT:                    "valueCategory": "lvalue",
 // CHECK-NEXT:                    "isPostfix": false,
 // CHECK-NEXT:                    "opcode": "*",
 // CHECK-NEXT:                    "canOverflow": false,
diff --git a/clang/test/AST/ast-dump-expr.cpp b/clang/test/AST/ast-dump-expr.cpp
index 69e65e22d61d..4df5ba4276ab 100644
--- a/clang/test/AST/ast-dump-expr.cpp
+++ b/clang/test/AST/ast-dump-expr.cpp
@@ -282,7 +282,7 @@ void PrimaryExpressions(Ts... a) {
       // CHECK-NEXT: CompoundStmt
       // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <col:8> col:8 implicit 'V'
       // CHECK-NEXT: ParenListExpr 0x{{[^ ]*}} <col:8> 'NULL TYPE'
-      // CHECK-NEXT: UnaryOperator 0x{{[^ ]*}} <col:8> '<dependent type>' prefix '*' cannot overflow
+      // CHECK-NEXT: UnaryOperator 0x{{[^ ]*}} <col:8> 'V' lvalue prefix '*' cannot overflow
       // CHECK-NEXT: CXXThisExpr 0x{{[^ ]*}} <col:8> 'V *' this
     }
   };
diff --git a/clang/test/AST/ast-dump-lambda.cpp b/clang/test/AST/ast-dump-lambda.cpp
index ef8789cd97d3..a4d3fe4fbda5 100644
--- a/clang/test/AST/ast-dump-lambda.cpp
+++ b/clang/test/AST/ast-dump-lambda.cpp
@@ -81,7 +81,7 @@ template <typename... Ts> void test(Ts... a) {
 // CHECK-NEXT:    |         | | `-CompoundStmt {{.*}} <col:15, col:16>
 // CHECK-NEXT:    |         | `-FieldDecl {{.*}} <col:8> col:8{{( imported)?}} implicit 'V'
 // CHECK-NEXT:    |         |-ParenListExpr {{.*}} <col:8> 'NULL TYPE'
-// CHECK-NEXT:    |         | `-UnaryOperator {{.*}} <col:8> '<dependent type>' prefix '*' cannot overflow
+// CHECK-NEXT:    |         | `-UnaryOperator {{.*}} <col:8> 'V' lvalue prefix '*' cannot overflow
 // CHECK-NEXT:    |         |   `-CXXThisExpr {{.*}} <col:8> 'V *' this
 // CHECK-NEXT:    |         `-CompoundStmt {{.*}} <col:15, col:16>
 // CHECK-NEXT:    |-DeclStmt {{.*}} <line:22:3, col:11>
diff --git a/clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp b/clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp
new file mode 100644
index 000000000000..6744ce1cad17
--- /dev/null
+++ b/clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp
@@ -0,0 +1,65 @@
+// RUN: %clang_cc1 -Wno-unused -fsyntax-only %s -verify
+
+struct A {
+  void operator*();
+  void operator+();
+  void operator-();
+  void operator!();
+  void operator~();
+  void operator&();
+  void operator++();
+  void operator--();
+};
+
+struct B { };
+
+template<typename T, typename U>
+void dependent(T t, T* pt, T U::* mpt, T(&ft)(), T(&at)[4]) {
+  *t;
+  +t;
+  -t;
+  !t;
+  ~t;
+  &t;
+  ++t;
+  --t;
+
+  *pt;
+  +pt;
+  -pt; // expected-error {{invalid argument type 'T *' to unary expression}}
+  !pt;
+  ~pt; // expected-error {{invalid argument type 'T *' to unary expression}}
+  &pt;
+  ++pt;
+  --pt;
+
+  *mpt; // expected-error {{indirection requires pointer operand ('T U::*' invalid)}}
+  +mpt; // expected-error {{invalid argument type 'T U::*' to unary expression}}
+  -mpt; // expected-error {{invalid argument type 'T U::*' to unary expression}}
+  !mpt;
+  ~mpt; // expected-error {{invalid argument type 'T U::*' to unary expression}}
+  &mpt;
+  ++mpt; // expected-error {{cannot increment value of type 'T U::*'}}
+  --mpt; // expected-error {{cannot decrement value of type 'T U::*'}}
+
+  *ft;
+  +ft;
+  -ft; // expected-error {{invalid argument type 'T (*)()' to unary expression}}
+  !ft;
+  ~ft; // expected-error {{invalid argument type 'T (*)()' to unary expression}}
+  &ft;
+  ++ft; // expected-error {{cannot increment value of type 'T ()'}}
+  --ft; // expected-error {{cannot decrement value of type 'T ()'}}
+
+  *at;
+  +at;
+  -at; // expected-error {{invalid argument type 'T *' to unary expression}}
+  !at;
+  ~at; // expected-error {{invalid argument type 'T *' to unary expression}}
+  &at;
+  ++at; // expected-error {{cannot increment value of type 'T[4]'}}
+  --at; // expected-error {{cannot decrement value of type 'T[4]'}}
+}
+
+// Make sure we only emit diagnostics once.
+template void dependent(A t, A* pt, A B::* mpt, A(&ft)(), A(&at)[4]);
diff --git a/clang/test/CXX/over/over.built/ast.cpp b/clang/test/CXX/over/over.built/ast.cpp
index 56a63431269f..78f86edb1e96 100644
--- a/clang/test/CXX/over/over.built/ast.cpp
+++ b/clang/test/CXX/over/over.built/ast.cpp
@@ -1,41 +1,139 @@
-// RUN: %clang_cc1 -std=c++17 -ast-dump %s -ast-dump-filter Test | FileCheck %s
+// RUN: %clang_cc1 -std=c++17 -Wno-unused -ast-dump %s -ast-dump-filter Test | FileCheck %s
 
-struct A{};
+namespace Test {
+  template<typename T, typename U>
+  void Unary(T t, T* pt, T U::* mpt, T(&ft)(), T(&at)[4]) {
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '*' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    *t;
 
-template <typename T, typename U>
-auto Test(T* pt, U* pu) {
-  // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '*'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  (void)*pt;
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '+' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    +t;
 
-  // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '++'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  (void)(++pt);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '-' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    -t;
 
-  // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '+'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  (void)(+pt);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '!' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    !t;
 
-  // CHECK: BinaryOperator {{.*}} '<dependent type>' '+'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 3
-  (void)(pt + 3);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '~' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    ~t;
 
-  // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  (void)(pt - pt);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    &t;
 
-  // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
-  (void)(pt - pu);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '++' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    ++t;
 
-  // CHECK: BinaryOperator {{.*}} '<dependent type>' '=='
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
-  (void)(pt == pu);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '--' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    --t;
 
-}
+    // CHECK: UnaryOperator {{.*}} 'T' lvalue prefix '*' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <LValueToRValue>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    *pt;
 
+    // CHECK: UnaryOperator {{.*}} 'T *' prefix '+' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <LValueToRValue>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    +pt;
 
+    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <PointerToBoolean>
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <LValueToRValue>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    !pt;
+
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    &pt;
+
+    // CHECK: UnaryOperator {{.*}} 'T *' lvalue prefix '++' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    ++pt;
+
+    // CHECK: UnaryOperator {{.*}} 'T *' lvalue prefix '--' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    --pt;
+
+    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <MemberPointerToBoolean>
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T U::*' <LValueToRValue>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T U::*' lvalue ParmVar {{.*}} 'mpt' 'T U::*'
+    !mpt;
+
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T U::*' lvalue ParmVar {{.*}} 'mpt' 'T U::*'
+    &mpt;
+
+    // CHECK: UnaryOperator {{.*}} 'T ()' lvalue prefix '*' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T (*)()' <FunctionToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
+    *ft;
+
+    // CHECK: UnaryOperator {{.*}} 'T (*)()' prefix '+' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T (*)()' <FunctionToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
+    +ft;
+
+    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <PointerToBoolean>
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T (*)()' <FunctionToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
+    !ft;
+
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
+    &ft;
+
+    // CHECK: UnaryOperator {{.*}} 'T' lvalue prefix '*' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <ArrayToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
+    *at;
+
+    // CHECK: UnaryOperator {{.*}} 'T *' prefix '+' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <ArrayToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
+    +at;
+
+    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <PointerToBoolean>
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <ArrayToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
+    !at;
+
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
+    &at;
+  }
+
+  template<typename T, typename U>
+  void Binary(T* pt, U* pu) {
+    // CHECK: BinaryOperator {{.*}} '<dependent type>' '+'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 3
+    pt + 3;
+
+    // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    pt - pt;
+
+    // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
+    pt - pu;
+
+    // CHECK: BinaryOperator {{.*}} '<dependent type>' '=='
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
+    pt == pu;
+  }
+} // namespace Test
diff --git a/clang/test/CXX/over/over.built/p10.cpp b/clang/test/CXX/over/over.built/p10.cpp
index 678056da5820..8ff2396d0b6f 100644
--- a/clang/test/CXX/over/over.built/p10.cpp
+++ b/clang/test/CXX/over/over.built/p10.cpp
@@ -15,6 +15,6 @@ void f(int i, float f, bool b, char c, int* pi, A* pa, T* pt) {
 
   (void)-pi; // expected-error {{invalid argument type}}
   (void)-pa; // expected-error {{invalid argument type}}
-  (void)-pt; // FIXME: we should be able to give an error here.
+  (void)-pt; // expected-error {{invalid argument type}}
 }
 
diff --git a/clang/test/CXX/over/over.built/p11.cpp b/clang/test/CXX/over/over.built/p11.cpp
index 7ebf16b95439..f7a741db726d 100644
--- a/clang/test/CXX/over/over.built/p11.cpp
+++ b/clang/test/CXX/over/over.built/p11.cpp
@@ -7,6 +7,6 @@ void f(int i, float f, bool b, char c, int* pi, T* pt) {
   (void)~b;
   (void)~c;
   (void)~pi; // expected-error {{invalid argument type}}
-  (void)~pt; // FIXME: we should be able to give an error here.
+  (void)~pt; // expected-error {{invalid argument type}}
 }
 
diff --git a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
index 3ca7c6c7eb8e..982e5372f5b0 100644
--- a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
@@ -357,17 +357,14 @@ namespace N0 {
       a->A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
       a->B::A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
 
-      // FIXME: An overloaded unary 'operator*' is built for these
-      // even though the operand is a pointer (to a dependent type).
-      // Type::isOverloadableType should return false for such cases.
-      (*this).x4;
-      (*this).B::x4;
-      (*this).A::x4;
-      (*this).B::A::x4;
-      (*this).f4();
-      (*this).B::f4();
-      (*this).A::f4();
-      (*this).B::A::f4();
+      (*this).x4; // expected-error{{no member named 'x4' in 'B<T>'}}
+      (*this).B::x4; // expected-error{{no member named 'x4' in 'B<T>'}}
+      (*this).A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
+      (*this).B::A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
+      (*this).f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
+      (*this).B::f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
+      (*this).A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
+      (*this).B::A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
 
       b.x4; // expected-error{{no member named 'x4' in 'B<T>'}}
       b.B::x4; // expected-error{{no member named 'x4' in 'B<T>'}}
@@ -399,15 +396,13 @@ namespace N1 {
       f<0>();
       this->f<0>();
       a->f<0>();
-      // FIXME: This should not require 'template'!
-      (*this).f<0>(); // expected-error{{missing 'template' keyword prior to dependent template name 'f'}}
+      (*this).f<0>();
       b.f<0>();
 
       x.f<0>();
       this->x.f<0>();
       a->x.f<0>();
-      // FIXME: This should not require 'template'!
-      (*this).x.f<0>(); // expected-error{{missing 'template' keyword prior to dependent template name 'f'}}
+      (*this).x.f<0>();
       b.x.f<0>();
 
       // FIXME: None of these should require 'template'!
diff --git a/clang/test/Frontend/noderef_templates.cpp b/clang/test/Frontend/noderef_templates.cpp
index 5fde6efd87c7..9e54cd5d7889 100644
--- a/clang/test/Frontend/noderef_templates.cpp
+++ b/clang/test/Frontend/noderef_templates.cpp
@@ -3,8 +3,8 @@
 #define NODEREF __attribute__((noderef))
 
 template <typename T>
-int func(T NODEREF *a) { // expected-note 2 {{a declared here}}
-  return *a + 1;         // expected-warning 2 {{dereferencing a; was declared with a 'noderef' type}}
+int func(T NODEREF *a) { // expected-note 3 {{a declared here}}
+  return *a + 1;         // expected-warning 3 {{dereferencing a; was declared with a 'noderef' type}}
 }
 
 void func() {
diff --git a/clang/test/SemaCXX/cxx2b-deducing-this.cpp b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
index 5f29a955e053..aa64530bd5be 100644
--- a/clang/test/SemaCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
@@ -19,7 +19,7 @@ struct S {
     // new and delete are implicitly static
     void *operator new(this unsigned long); // expected-error{{an explicit object parameter cannot appear in a static function}}
     void operator delete(this void*); // expected-error{{an explicit object parameter cannot appear in a static function}}
-    
+
     void g(this auto) const; // expected-error{{explicit object member function cannot have 'const' qualifier}}
     void h(this auto) &; // expected-error{{explicit object member function cannot have '&' qualifier}}
     void i(this auto) &&; // expected-error{{explicit object member function cannot have '&&' qualifier}}
@@ -198,9 +198,7 @@ void func(int i) {
 void TestMutationInLambda() {
     [i = 0](this auto &&){ i++; }();
     [i = 0](this auto){ i++; }();
-    [i = 0](this const auto&){ i++; }();
-    // expected-error@-1 {{cannot assign to a variable captured by copy in a non-mutable lambda}}
-    // expected-note@-2 {{in instantiation of}}
+    [i = 0](this const auto&){ i++; }(); // expected-error {{cannot assign to a variable captured by copy in a non-mutable lambda}}
 
     int x;
     const auto l1 = [x](this auto&) { x = 42; }; // expected-error {{cannot assign to a variable captured by copy in a non-mutable lambda}}
diff --git a/clang/test/SemaTemplate/class-template-spec.cpp b/clang/test/SemaTemplate/class-template-spec.cpp
index 56b8207bd9a4..faa54c367538 100644
--- a/clang/test/SemaTemplate/class-template-spec.cpp
+++ b/clang/test/SemaTemplate/class-template-spec.cpp
@@ -18,7 +18,7 @@ int test_specs(A<float, float> *a1, A<float, int> *a2) {
   return a1->x + a2->y;
 }
 
-int test_incomplete_specs(A<double, double> *a1, 
+int test_incomplete_specs(A<double, double> *a1,
                           A<double> *a2)
 {
   (void)a1->x; // expected-error{{member access into incomplete type}}
@@ -39,7 +39,7 @@ template <> struct X<int, int> { int foo(); }; // #1
 template <> struct X<float> { int bar(); };  // #2
 
 typedef int int_type;
-void testme(X<int_type> *x1, X<float, int> *x2) { 
+void testme(X<int_type> *x1, X<float, int> *x2) {
   (void)x1->foo(); // okay: refers to #1
   (void)x2->bar(); // okay: refers to #2
 }
@@ -53,7 +53,7 @@ struct A<char> {
 A<char>::A() { }
 
 // Make sure we can see specializations defined before the primary template.
-namespace N{ 
+namespace N{
   template<typename T> struct A0;
 }
 
@@ -97,7 +97,7 @@ namespace M {
   template<> struct ::A<long double>; // expected-error{{must occur at global scope}}
 }
 
-template<> struct N::B<char> { 
+template<> struct N::B<char> {
   int testf(int x) { return f(x); }
 };
 
@@ -138,9 +138,9 @@ namespace PR18009 {
 
   template <typename T> struct C {
     template <int N, int M> struct S;
-    template <int N> struct S<N, N ? **(T(*)[N])0 : 0> {}; // expected-error {{depends on a template parameter of the partial specialization}}
+    template <int N> struct S<N, N ? **(T(*)[N])0 : 0> {}; // ok
   };
-  C<int> c; // expected-note {{in instantiation of}}
+  C<int> c;
 
   template<int A> struct outer {
     template<int B, int C> struct inner {};
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index c08deb903f12..f26140675fd4 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -1572,9 +1572,9 @@ TEST_P(ASTMatchersTest, IsArrow_MatchesMemberVariablesViaArrow) {
   EXPECT_TRUE(
       matches("template <class T> class Y { void x() { this->m; } int m; };",
               memberExpr(isArrow())));
-  EXPECT_TRUE(
-      notMatches("template <class T> class Y { void x() { (*this).m; } };",
-                 cxxDependentScopeMemberExpr(isArrow())));
+  EXPECT_TRUE(notMatches(
+      "template <class T> class Y { void x() { (*this).m; } int m; };",
+      memberExpr(isArrow())));
 }
 
 TEST_P(ASTMatchersTest, IsArrow_MatchesStaticMemberVariablesViaArrow) {
-- 
GitLab


From 2ff43ce87e66d9324370e35ea6743ef57400c76e Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Tue, 14 May 2024 09:45:39 -0700
Subject: [PATCH 0752/1206] Restore #91137 (#92003)

#91137 reverted in #92001

A build error fix added in 28d5ece8ca93ef04fee9b0258b70b750b66c05ca

---------

Co-authored-by: Jeremy Kun <j2kun@users.noreply.github.com>
---
 .../mlir/Dialect/Polynomial/IR/Polynomial.h   | 193 ++++++++++++++----
 .../mlir/Dialect/Polynomial/IR/Polynomial.td  | 139 ++++++++-----
 mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp |  75 +++----
 .../Polynomial/IR/PolynomialAttributes.cpp    | 172 +++++++---------
 mlir/test/Dialect/Polynomial/attributes.mlir  |  22 +-
 mlir/test/Dialect/Polynomial/ops.mlir         |  64 +++---
 mlir/test/Dialect/Polynomial/ops_errors.mlir  |  66 +++---
 mlir/test/Dialect/Polynomial/types.mlir       |  65 +++---
 8 files changed, 448 insertions(+), 348 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
index 3325a6fa3f9f..7f44c29a9870 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
@@ -11,10 +11,13 @@
 
 #include "mlir/Support/LLVM.h"
 #include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Hashing.h"
-#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/raw_ostream.h"
 
 namespace mlir {
 
@@ -27,98 +30,202 @@ namespace polynomial {
 /// would want to specify 128-bit polynomials statically in the source code.
 constexpr unsigned apintBitWidth = 64;
 
-/// A class representing a monomial of a single-variable polynomial with integer
-/// coefficients.
-class Monomial {
+template <typename CoefficientType>
+class MonomialBase {
 public:
-  Monomial(int64_t coeff, uint64_t expo)
-      : coefficient(apintBitWidth, coeff), exponent(apintBitWidth, expo) {}
-
-  Monomial(const APInt &coeff, const APInt &expo)
+  MonomialBase(const CoefficientType &coeff, const APInt &expo)
       : coefficient(coeff), exponent(expo) {}
+  virtual ~MonomialBase() = default;
 
-  Monomial() : coefficient(apintBitWidth, 0), exponent(apintBitWidth, 0) {}
+  const CoefficientType &getCoefficient() const { return coefficient; }
+  CoefficientType &getMutableCoefficient() { return coefficient; }
+  const APInt &getExponent() const { return exponent; }
+  void setCoefficient(const CoefficientType &coeff) { coefficient = coeff; }
+  void setExponent(const APInt &exp) { exponent = exp; }
 
-  bool operator==(const Monomial &other) const {
+  bool operator==(const MonomialBase &other) const {
     return other.coefficient == coefficient && other.exponent == exponent;
   }
-  bool operator!=(const Monomial &other) const {
+  bool operator!=(const MonomialBase &other) const {
     return other.coefficient != coefficient || other.exponent != exponent;
   }
 
   /// Monomials are ordered by exponent.
-  bool operator<(const Monomial &other) const {
+  bool operator<(const MonomialBase &other) const {
     return (exponent.ult(other.exponent));
   }
 
-  friend ::llvm::hash_code hash_value(const Monomial &arg);
+  virtual bool isMonic() const = 0;
+  virtual void
+  coefficientToString(llvm::SmallString<16> &coeffString) const = 0;
 
-public:
-  APInt coefficient;
+  template <typename T>
+  friend ::llvm::hash_code hash_value(const MonomialBase<T> &arg);
 
-  // Always unsigned
+protected:
+  CoefficientType coefficient;
   APInt exponent;
 };
 
-/// A single-variable polynomial with integer coefficients.
-///
-/// Eg: x^1024 + x + 1
-///
-/// The symbols used as the polynomial's indeterminate don't matter, so long as
-/// it is used consistently throughout the polynomial.
-class Polynomial {
+/// A class representing a monomial of a single-variable polynomial with integer
+/// coefficients.
+class IntMonomial : public MonomialBase<APInt> {
 public:
-  Polynomial() = delete;
+  IntMonomial(int64_t coeff, uint64_t expo)
+      : MonomialBase(APInt(apintBitWidth, coeff), APInt(apintBitWidth, expo)) {}
 
-  explicit Polynomial(ArrayRef<Monomial> terms) : terms(terms){};
+  IntMonomial()
+      : MonomialBase(APInt(apintBitWidth, 0), APInt(apintBitWidth, 0)) {}
 
-  // Returns a Polynomial from a list of monomials.
-  // Fails if two monomials have the same exponent.
-  static FailureOr<Polynomial> fromMonomials(ArrayRef<Monomial> monomials);
+  ~IntMonomial() = default;
 
-  /// Returns a polynomial with coefficients given by `coeffs`. The value
-  /// coeffs[i] is converted to a monomial with exponent i.
-  static Polynomial fromCoefficients(ArrayRef<int64_t> coeffs);
+  bool isMonic() const override { return coefficient == 1; }
+
+  void coefficientToString(llvm::SmallString<16> &coeffString) const override {
+    coefficient.toStringSigned(coeffString);
+  }
+};
+
+/// A class representing a monomial of a single-variable polynomial with integer
+/// coefficients.
+class FloatMonomial : public MonomialBase<APFloat> {
+public:
+  FloatMonomial(double coeff, uint64_t expo)
+      : MonomialBase(APFloat(coeff), APInt(apintBitWidth, expo)) {}
+
+  FloatMonomial() : MonomialBase(APFloat((double)0), APInt(apintBitWidth, 0)) {}
+
+  ~FloatMonomial() = default;
+
+  bool isMonic() const override { return coefficient == APFloat(1.0); }
+
+  void coefficientToString(llvm::SmallString<16> &coeffString) const override {
+    coefficient.toString(coeffString);
+  }
+};
+
+template <typename Monomial>
+class PolynomialBase {
+public:
+  PolynomialBase() = delete;
+
+  explicit PolynomialBase(ArrayRef<Monomial> terms) : terms(terms) {};
 
   explicit operator bool() const { return !terms.empty(); }
-  bool operator==(const Polynomial &other) const {
+  bool operator==(const PolynomialBase &other) const {
     return other.terms == terms;
   }
-  bool operator!=(const Polynomial &other) const {
+  bool operator!=(const PolynomialBase &other) const {
     return !(other.terms == terms);
   }
 
-  // Prints polynomial to 'os'.
-  void print(raw_ostream &os) const;
   void print(raw_ostream &os, ::llvm::StringRef separator,
-             ::llvm::StringRef exponentiation) const;
+             ::llvm::StringRef exponentiation) const {
+    bool first = true;
+    for (const Monomial &term : getTerms()) {
+      if (first) {
+        first = false;
+      } else {
+        os << separator;
+      }
+      std::string coeffToPrint;
+      if (term.isMonic() && term.getExponent().uge(1)) {
+        coeffToPrint = "";
+      } else {
+        llvm::SmallString<16> coeffString;
+        term.coefficientToString(coeffString);
+        coeffToPrint = coeffString.str();
+      }
+
+      if (term.getExponent() == 0) {
+        os << coeffToPrint;
+      } else if (term.getExponent() == 1) {
+        os << coeffToPrint << "x";
+      } else {
+        llvm::SmallString<16> expString;
+        term.getExponent().toStringSigned(expString);
+        os << coeffToPrint << "x" << exponentiation << expString;
+      }
+    }
+  }
+
+  // Prints polynomial to 'os'.
+  void print(raw_ostream &os) const { print(os, " + ", "**"); }
+
   void dump() const;
 
   // Prints polynomial so that it can be used as a valid identifier
-  std::string toIdentifier() const;
+  std::string toIdentifier() const {
+    std::string result;
+    llvm::raw_string_ostream os(result);
+    print(os, "_", "");
+    return os.str();
+  }
 
-  unsigned getDegree() const;
+  unsigned getDegree() const {
+    return terms.back().getExponent().getZExtValue();
+  }
 
   ArrayRef<Monomial> getTerms() const { return terms; }
 
-  friend ::llvm::hash_code hash_value(const Polynomial &arg);
+  template <typename T>
+  friend ::llvm::hash_code hash_value(const PolynomialBase<T> &arg);
 
 private:
   // The monomial terms for this polynomial.
   SmallVector<Monomial> terms;
 };
 
-// Make Polynomial hashable.
-inline ::llvm::hash_code hash_value(const Polynomial &arg) {
+/// A single-variable polynomial with integer coefficients.
+///
+/// Eg: x^1024 + x + 1
+class IntPolynomial : public PolynomialBase<IntMonomial> {
+public:
+  explicit IntPolynomial(ArrayRef<IntMonomial> terms) : PolynomialBase(terms) {}
+
+  // Returns a Polynomial from a list of monomials.
+  // Fails if two monomials have the same exponent.
+  static FailureOr<IntPolynomial>
+  fromMonomials(ArrayRef<IntMonomial> monomials);
+
+  /// Returns a polynomial with coefficients given by `coeffs`. The value
+  /// coeffs[i] is converted to a monomial with exponent i.
+  static IntPolynomial fromCoefficients(ArrayRef<int64_t> coeffs);
+};
+
+/// A single-variable polynomial with double coefficients.
+///
+/// Eg: 1.0 x^1024 + 3.5 x + 1e-05
+class FloatPolynomial : public PolynomialBase<FloatMonomial> {
+public:
+  explicit FloatPolynomial(ArrayRef<FloatMonomial> terms)
+      : PolynomialBase(terms) {}
+
+  // Returns a Polynomial from a list of monomials.
+  // Fails if two monomials have the same exponent.
+  static FailureOr<FloatPolynomial>
+  fromMonomials(ArrayRef<FloatMonomial> monomials);
+
+  /// Returns a polynomial with coefficients given by `coeffs`. The value
+  /// coeffs[i] is converted to a monomial with exponent i.
+  static FloatPolynomial fromCoefficients(ArrayRef<double> coeffs);
+};
+
+// Make Polynomials hashable.
+template <typename T>
+inline ::llvm::hash_code hash_value(const PolynomialBase<T> &arg) {
   return ::llvm::hash_combine_range(arg.terms.begin(), arg.terms.end());
 }
 
-inline ::llvm::hash_code hash_value(const Monomial &arg) {
+template <typename T>
+inline ::llvm::hash_code hash_value(const MonomialBase<T> &arg) {
   return llvm::hash_combine(::llvm::hash_value(arg.coefficient),
                             ::llvm::hash_value(arg.exponent));
 }
 
-inline raw_ostream &operator<<(raw_ostream &os, const Polynomial &polynomial) {
+template <typename T>
+inline raw_ostream &operator<<(raw_ostream &os,
+                               const PolynomialBase<T> &polynomial) {
   polynomial.print(os);
   return os;
 }
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
index ed1f4ce8b7e5..ae8484501a50 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
@@ -39,14 +39,14 @@ def Polynomial_Dialect : Dialect {
     %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
 
     // A constant polynomial in a ring with i32 coefficients, modulo (x^1024 + 1)
-    #modulus = #polynomial.polynomial<1 + x**1024>
+    #modulus = #polynomial.int_polynomial<1 + x**1024>
     #ring = #polynomial.ring<coefficientType=i32, polynomialModulus=#modulus>
     %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
 
     // A constant polynomial in a ring with i32 coefficients, with a polynomial
     // modulus of (x^1024 + 1) and a coefficient modulus of 17.
-    #modulus = #polynomial.polynomial<1 + x**1024>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=17, polynomialModulus=#modulus>
+    #modulus = #polynomial.int_polynomial<1 + x**1024>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=17:i32, polynomialModulus=#modulus>
     %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
     ```
   }];
@@ -60,12 +60,12 @@ class Polynomial_Attr<string name, string attrMnemonic, list<Trait> traits = []>
   let mnemonic = attrMnemonic;
 }
 
-def Polynomial_PolynomialAttr : Polynomial_Attr<"Polynomial", "polynomial"> {
-  let summary = "An attribute containing a single-variable polynomial.";
+def Polynomial_IntPolynomialAttr : Polynomial_Attr<"IntPolynomial", "int_polynomial"> {
+  let summary = "An attribute containing a single-variable polynomial with integer coefficients.";
   let description = [{
-    A polynomial attribute represents a single-variable polynomial, which
-    is used to define the modulus of a `RingAttr`, as well as to define constants
-    and perform constant folding for `polynomial` ops.
+    A polynomial attribute represents a single-variable polynomial with integer
+    coefficients, which is used to define the modulus of a `RingAttr`, as well
+    as to define constants and perform constant folding for `polynomial` ops.
 
     The polynomial must be expressed as a list of monomial terms, with addition
     or subtraction between them. The choice of variable name is arbitrary, but
@@ -76,10 +76,32 @@ def Polynomial_PolynomialAttr : Polynomial_Attr<"Polynomial", "polynomial"> {
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 + 1>
+    #poly = #polynomial.int_polynomial<x**1024 + 1>
     ```
   }];
-  let parameters = (ins "::mlir::polynomial::Polynomial":$polynomial);
+  let parameters = (ins "::mlir::polynomial::IntPolynomial":$polynomial);
+  let hasCustomAssemblyFormat = 1;
+}
+
+def Polynomial_FloatPolynomialAttr : Polynomial_Attr<"FloatPolynomial", "float_polynomial"> {
+  let summary = "An attribute containing a single-variable polynomial with double precision floating point coefficients.";
+  let description = [{
+    A polynomial attribute represents a single-variable polynomial with double
+    precision floating point coefficients.
+
+    The polynomial must be expressed as a list of monomial terms, with addition
+    or subtraction between them. The choice of variable name is arbitrary, but
+    must be consistent across all the monomials used to define a single
+    attribute. The order of monomial terms is arbitrary, each monomial degree
+    must occur at most once.
+
+    Example:
+
+    ```mlir
+    #poly = #polynomial.float_polynomial<0.5 x**7 + 1.5>
+    ```
+  }];
+  let parameters = (ins "FloatPolynomial":$polynomial);
   let hasCustomAssemblyFormat = 1;
 }
 
@@ -104,9 +126,9 @@ def Polynomial_RingAttr : Polynomial_Attr<"Ring", "ring"> {
     `x**1024 - 1`.
 
     ```mlir
-    #poly_mod = #polynomial.polynomial<-1 + x**1024>
+    #poly_mod = #polynomial.int_polynomial<-1 + x**1024>
     #ring = #polynomial.ring<coefficientType=i32,
-                             coefficientModulus=4294967291,
+                             coefficientModulus=4294967291:i32,
                              polynomialModulus=#poly_mod>
 
     %0 = ... : polynomial.polynomial<#ring>
@@ -123,19 +145,24 @@ def Polynomial_RingAttr : Polynomial_Attr<"Ring", "ring"> {
   let parameters = (ins
     "Type": $coefficientType,
     OptionalParameter<"::mlir::IntegerAttr">: $coefficientModulus,
-    OptionalParameter<"::mlir::polynomial::PolynomialAttr">: $polynomialModulus,
+    OptionalParameter<"::mlir::polynomial::IntPolynomialAttr">: $polynomialModulus,
     OptionalParameter<"::mlir::IntegerAttr">: $primitiveRoot
   );
-
+  let assemblyFormat = "`<` struct(params) `>`";
   let builders = [
-    AttrBuilder<
+    AttrBuilderWithInferredContext<
         (ins "::mlir::Type":$coefficientTy,
-             "::mlir::IntegerAttr":$coefficientModulusAttr,
-             "::mlir::polynomial::PolynomialAttr":$polynomialModulusAttr), [{
-      return $_get($_ctxt, coefficientTy, coefficientModulusAttr, polynomialModulusAttr, nullptr);
-    }]>
+              CArg<"::mlir::IntegerAttr", "nullptr"> :$coefficientModulusAttr,
+              CArg<"::mlir::polynomial::IntPolynomialAttr", "nullptr"> :$polynomialModulusAttr,
+              CArg<"::mlir::IntegerAttr", "nullptr"> :$primitiveRootAttr), [{
+      return $_get(
+        coefficientTy.getContext(),
+        coefficientTy,
+        coefficientModulusAttr,
+        polynomialModulusAttr,
+        primitiveRootAttr);
+    }]>,
   ];
-  let hasCustomAssemblyFormat = 1;
 }
 
 class Polynomial_Type<string name, string typeMnemonic>
@@ -149,7 +176,7 @@ def Polynomial_PolynomialType : Polynomial_Type<"Polynomial", "polynomial"> {
     A type for polynomials in a polynomial quotient ring.
   }];
   let parameters = (ins Polynomial_RingAttr:$ring);
-  let assemblyFormat = "`<` $ring `>`";
+  let assemblyFormat = "`<` struct(params) `>`";
 }
 
 def PolynomialLike: TypeOrContainer<Polynomial_PolynomialType, "polynomial-like">;
@@ -187,10 +214,10 @@ def Polynomial_AddOp : Polynomial_BinaryOp<"add", [Commutative]> {
 
     ```mlir
     // add two polynomials modulo x^1024 - 1
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
-    %1 = polynomial.constant #polynomial.polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    %1 = polynomial.constant #polynomial.int_polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
     %2 = polynomial.add %0, %1 : !polynomial.polynomial<#ring>
     ```
   }];
@@ -211,10 +238,10 @@ def Polynomial_SubOp : Polynomial_BinaryOp<"sub"> {
 
     ```mlir
     // subtract two polynomials modulo x^1024 - 1
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
-    %1 = polynomial.constant #polynomial.polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    %1 = polynomial.constant #polynomial.int_polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
     %2 = polynomial.sub %0, %1 : !polynomial.polynomial<#ring>
     ```
   }];
@@ -235,10 +262,10 @@ def Polynomial_MulOp : Polynomial_BinaryOp<"mul", [Commutative]> {
 
     ```mlir
     // multiply two polynomials modulo x^1024 - 1
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
-    %1 = polynomial.constant #polynomial.polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    %1 = polynomial.constant #polynomial.int_polynomial<x**5 - x + 1> : !polynomial.polynomial<#ring>
     %2 = polynomial.mul %0, %1 : !polynomial.polynomial<#ring>
     ```
   }];
@@ -260,9 +287,9 @@ def Polynomial_MulScalarOp : Polynomial_Op<"mul_scalar", [
 
     ```mlir
     // multiply two polynomials modulo x^1024 - 1
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
     %1 = arith.constant 3 : i32
     %2 = polynomial.mul_scalar %0, %1 : !polynomial.polynomial<#ring>, i32
     ```
@@ -291,9 +318,9 @@ def Polynomial_LeadingTermOp: Polynomial_Op<"leading_term"> {
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
     %1, %2 = polynomial.leading_term %0 : !polynomial.polynomial<#ring> -> (index, i32)
     ```
   }];
@@ -314,8 +341,8 @@ def Polynomial_MonomialOp: Polynomial_Op<"monomial"> {
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
     %deg = arith.constant 1023 : index
     %five = arith.constant 5 : i32
     %0 = polynomial.monomial %five, %deg : (i32, index) -> !polynomial.polynomial<#ring>
@@ -354,8 +381,8 @@ def Polynomial_FromTensorOp : Polynomial_Op<"from_tensor", [Pure]> {
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
     %two = arith.constant 2 : i32
     %five = arith.constant 5 : i32
     %coeffs = tensor.from_elements %two, %two, %five : tensor<3xi32>
@@ -393,8 +420,8 @@ def Polynomial_ToTensorOp : Polynomial_Op<"to_tensor", [Pure]> {
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
     %two = arith.constant 2 : i32
     %five = arith.constant 5 : i32
     %coeffs = tensor.from_elements %two, %two, %five : tensor<3xi32>
@@ -405,24 +432,32 @@ def Polynomial_ToTensorOp : Polynomial_Op<"to_tensor", [Pure]> {
   let arguments = (ins Polynomial_PolynomialType:$input);
   let results = (outs RankedTensorOf<[AnyInteger]>:$output);
   let assemblyFormat = "$input attr-dict `:` type($input) `->` type($output)";
-
   let hasVerifier = 1;
 }
 
-def Polynomial_ConstantOp : Polynomial_Op<"constant", [Pure]> {
+def Polynomial_AnyPolynomialAttr : AnyAttrOf<[
+  Polynomial_FloatPolynomialAttr,
+  Polynomial_IntPolynomialAttr
+]>;
+
+// Not deriving from Polynomial_Op due to need for custom assembly format
+def Polynomial_ConstantOp : Op<Polynomial_Dialect, "constant", [Pure]> {
   let summary = "Define a constant polynomial via an attribute.";
   let description = [{
     Example:
 
     ```mlir
-    #poly = #polynomial.polynomial<x**1024 - 1>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536, polynomialModulus=#poly>
-    %0 = polynomial.constant #polynomial.polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+    #poly = #polynomial.int_polynomial<x**1024 - 1>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=65536:i32, polynomialModulus=#poly>
+    %0 = polynomial.constant #polynomial.int_polynomial<1 + x**2> : !polynomial.polynomial<#ring>
+
+    #float_ring = #polynomial.ring<coefficientType=f32>
+    %0 = polynomial.constant #polynomial.float_polynomial<0.5 + 1.3e06 x**2> : !polynomial.polynomial<#float_ring>
     ```
   }];
-  let arguments = (ins Polynomial_PolynomialAttr:$input);
+  let arguments = (ins Polynomial_AnyPolynomialAttr:$value);
   let results = (outs Polynomial_PolynomialType:$output);
-  let assemblyFormat = "$input attr-dict `:` type($output)";
+  let assemblyFormat = "attr-dict `:` type($output)";
 }
 
 def Polynomial_NTTOp : Polynomial_Op<"ntt", [Pure]> {
diff --git a/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp b/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
index 5916ffba78e2..e85bced3cca7 100644
--- a/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
+++ b/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
@@ -9,87 +9,60 @@
 #include "mlir/Dialect/Polynomial/IR/Polynomial.h"
 
 #include "mlir/Support/LogicalResult.h"
-#include "llvm/ADT/APInt.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Support/raw_ostream.h"
 
 namespace mlir {
 namespace polynomial {
 
-FailureOr<Polynomial> Polynomial::fromMonomials(ArrayRef<Monomial> monomials) {
+template <typename PolyT, typename MonomialT>
+FailureOr<PolyT> fromMonomialsImpl(ArrayRef<MonomialT> monomials) {
   // A polynomial's terms are canonically stored in order of increasing degree.
-  auto monomialsCopy = llvm::SmallVector<Monomial>(monomials);
+  auto monomialsCopy = llvm::SmallVector<MonomialT>(monomials);
   std::sort(monomialsCopy.begin(), monomialsCopy.end());
 
   // Ensure non-unique exponents are not present. Since we sorted the list by
   // exponent, a linear scan of adjancent monomials suffices.
   if (std::adjacent_find(monomialsCopy.begin(), monomialsCopy.end(),
-                         [](const Monomial &lhs, const Monomial &rhs) {
-                           return lhs.exponent == rhs.exponent;
+                         [](const MonomialT &lhs, const MonomialT &rhs) {
+                           return lhs.getExponent() == rhs.getExponent();
                          }) != monomialsCopy.end()) {
     return failure();
   }
 
-  return Polynomial(monomialsCopy);
+  return PolyT(monomialsCopy);
 }
 
-Polynomial Polynomial::fromCoefficients(ArrayRef<int64_t> coeffs) {
-  llvm::SmallVector<Monomial> monomials;
+FailureOr<IntPolynomial>
+IntPolynomial::fromMonomials(ArrayRef<IntMonomial> monomials) {
+  return fromMonomialsImpl<IntPolynomial, IntMonomial>(monomials);
+}
+
+FailureOr<FloatPolynomial>
+FloatPolynomial::fromMonomials(ArrayRef<FloatMonomial> monomials) {
+  return fromMonomialsImpl<FloatPolynomial, FloatMonomial>(monomials);
+}
+
+template <typename PolyT, typename MonomialT, typename CoeffT>
+PolyT fromCoefficientsImpl(ArrayRef<CoeffT> coeffs) {
+  llvm::SmallVector<MonomialT> monomials;
   auto size = coeffs.size();
   monomials.reserve(size);
   for (size_t i = 0; i < size; i++) {
     monomials.emplace_back(coeffs[i], i);
   }
-  auto result = Polynomial::fromMonomials(monomials);
+  auto result = PolyT::fromMonomials(monomials);
   // Construction guarantees unique exponents, so the failure mode of
   // fromMonomials can be bypassed.
   assert(succeeded(result));
   return result.value();
 }
 
-void Polynomial::print(raw_ostream &os, ::llvm::StringRef separator,
-                       ::llvm::StringRef exponentiation) const {
-  bool first = true;
-  for (const Monomial &term : terms) {
-    if (first) {
-      first = false;
-    } else {
-      os << separator;
-    }
-    std::string coeffToPrint;
-    if (term.coefficient == 1 && term.exponent.uge(1)) {
-      coeffToPrint = "";
-    } else {
-      llvm::SmallString<16> coeffString;
-      term.coefficient.toStringSigned(coeffString);
-      coeffToPrint = coeffString.str();
-    }
-
-    if (term.exponent == 0) {
-      os << coeffToPrint;
-    } else if (term.exponent == 1) {
-      os << coeffToPrint << "x";
-    } else {
-      llvm::SmallString<16> expString;
-      term.exponent.toStringSigned(expString);
-      os << coeffToPrint << "x" << exponentiation << expString;
-    }
-  }
-}
-
-void Polynomial::print(raw_ostream &os) const { print(os, " + ", "**"); }
-
-std::string Polynomial::toIdentifier() const {
-  std::string result;
-  llvm::raw_string_ostream os(result);
-  print(os, "_", "");
-  return os.str();
+IntPolynomial IntPolynomial::fromCoefficients(ArrayRef<int64_t> coeffs) {
+  return fromCoefficientsImpl<IntPolynomial, IntMonomial, int64_t>(coeffs);
 }
 
-unsigned Polynomial::getDegree() const {
-  return terms.back().exponent.getZExtValue();
+FloatPolynomial FloatPolynomial::fromCoefficients(ArrayRef<double> coeffs) {
+  return fromCoefficientsImpl<FloatPolynomial, FloatMonomial, double>(coeffs);
 }
 
 } // namespace polynomial
diff --git a/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp b/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
index 236bb7896635..890ce5226c30 100644
--- a/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
+++ b/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
@@ -10,6 +10,7 @@
 #include "mlir/Dialect/Polynomial/IR/Polynomial.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
@@ -17,22 +18,31 @@
 namespace mlir {
 namespace polynomial {
 
-void PolynomialAttr::print(AsmPrinter &p) const {
-  p << '<';
-  p << getPolynomial();
-  p << '>';
+void IntPolynomialAttr::print(AsmPrinter &p) const {
+  p << '<' << getPolynomial() << '>';
 }
 
+void FloatPolynomialAttr::print(AsmPrinter &p) const {
+  p << '<' << getPolynomial() << '>';
+}
+
+/// A callable that parses the coefficient using the appropriate method for the
+/// given monomial type, and stores the parsed coefficient value on the
+/// monomial.
+template <typename MonomialType>
+using ParseCoefficientFn = std::function<OptionalParseResult(MonomialType &)>;
+
 /// Try to parse a monomial. If successful, populate the fields of the outparam
 /// `monomial` with the results, and the `variable` outparam with the parsed
 /// variable name. Sets shouldParseMore to true if the monomial is followed by
 /// a '+'.
-ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
-                          llvm::StringRef &variable, bool &isConstantTerm,
-                          bool &shouldParseMore) {
-  APInt parsedCoeff(apintBitWidth, 1);
-  auto parsedCoeffResult = parser.parseOptionalInteger(parsedCoeff);
-  monomial.coefficient = parsedCoeff;
+///
+template <typename Monomial>
+ParseResult
+parseMonomial(AsmParser &parser, Monomial &monomial, llvm::StringRef &variable,
+              bool &isConstantTerm, bool &shouldParseMore,
+              ParseCoefficientFn<Monomial> parseAndStoreCoefficient) {
+  OptionalParseResult parsedCoeffResult = parseAndStoreCoefficient(monomial);
 
   isConstantTerm = false;
   shouldParseMore = false;
@@ -44,7 +54,7 @@ ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
     if (!parsedCoeffResult.has_value()) {
       return failure();
     }
-    monomial.exponent = APInt(apintBitWidth, 0);
+    monomial.setExponent(APInt(apintBitWidth, 0));
     isConstantTerm = true;
     shouldParseMore = true;
     return success();
@@ -58,7 +68,7 @@ ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
       return failure();
     }
 
-    monomial.exponent = APInt(apintBitWidth, 0);
+    monomial.setExponent(APInt(apintBitWidth, 0));
     isConstantTerm = true;
     return success();
   }
@@ -80,9 +90,9 @@ ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
       return failure();
     }
 
-    monomial.exponent = parsedExponent;
+    monomial.setExponent(parsedExponent);
   } else {
-    monomial.exponent = APInt(apintBitWidth, 1);
+    monomial.setExponent(APInt(apintBitWidth, 1));
   }
 
   if (succeeded(parser.parseOptionalPlus())) {
@@ -91,22 +101,21 @@ ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
   return success();
 }
 
-Attribute PolynomialAttr::parse(AsmParser &parser, Type type) {
-  if (failed(parser.parseLess()))
-    return {};
-
-  llvm::SmallVector<Monomial> monomials;
-  llvm::StringSet<> variables;
-
+template <typename PolynoimalAttrTy, typename Monomial>
+LogicalResult
+parsePolynomialAttr(AsmParser &parser, llvm::SmallVector<Monomial> &monomials,
+                    llvm::StringSet<> &variables,
+                    ParseCoefficientFn<Monomial> parseAndStoreCoefficient) {
   while (true) {
     Monomial parsedMonomial;
     llvm::StringRef parsedVariableRef;
     bool isConstantTerm;
     bool shouldParseMore;
-    if (failed(parseMonomial(parser, parsedMonomial, parsedVariableRef,
-                             isConstantTerm, shouldParseMore))) {
+    if (failed(parseMonomial<Monomial>(
+            parser, parsedMonomial, parsedVariableRef, isConstantTerm,
+            shouldParseMore, parseAndStoreCoefficient))) {
       parser.emitError(parser.getCurrentLocation(), "expected a monomial");
-      return {};
+      return failure();
     }
 
     if (!isConstantTerm) {
@@ -124,7 +133,7 @@ Attribute PolynomialAttr::parse(AsmParser &parser, Type type) {
     parser.emitError(
         parser.getCurrentLocation(),
         "expected + and more monomials, or > to end polynomial attribute");
-    return {};
+    return failure();
   }
 
   if (variables.size() > 1) {
@@ -133,96 +142,67 @@ Attribute PolynomialAttr::parse(AsmParser &parser, Type type) {
         parser.getCurrentLocation(),
         "polynomials must have one indeterminate, but there were multiple: " +
             vars);
+    return failure();
   }
 
-  auto result = Polynomial::fromMonomials(monomials);
-  if (failed(result)) {
-    parser.emitError(parser.getCurrentLocation())
-        << "parsed polynomial must have unique exponents among monomials";
-    return {};
-  }
-  return PolynomialAttr::get(parser.getContext(), result.value());
-}
-
-void RingAttr::print(AsmPrinter &p) const {
-  p << "#polynomial.ring<coefficientType=" << getCoefficientType()
-    << ", coefficientModulus=" << getCoefficientModulus()
-    << ", polynomialModulus=" << getPolynomialModulus() << '>';
+  return success();
 }
 
-Attribute RingAttr::parse(AsmParser &parser, Type type) {
+Attribute IntPolynomialAttr::parse(AsmParser &parser, Type type) {
   if (failed(parser.parseLess()))
     return {};
 
-  if (failed(parser.parseKeyword("coefficientType")))
-    return {};
+  llvm::SmallVector<IntMonomial> monomials;
+  llvm::StringSet<> variables;
 
-  if (failed(parser.parseEqual()))
+  if (failed(parsePolynomialAttr<IntPolynomialAttr, IntMonomial>(
+          parser, monomials, variables,
+          [&](IntMonomial &monomial) -> OptionalParseResult {
+            APInt parsedCoeff(apintBitWidth, 1);
+            OptionalParseResult result =
+                parser.parseOptionalInteger(parsedCoeff);
+            monomial.setCoefficient(parsedCoeff);
+            return result;
+          }))) {
     return {};
+  }
 
-  Type ty;
-  if (failed(parser.parseType(ty)))
+  auto result = IntPolynomial::fromMonomials(monomials);
+  if (failed(result)) {
+    parser.emitError(parser.getCurrentLocation())
+        << "parsed polynomial must have unique exponents among monomials";
     return {};
+  }
+  return IntPolynomialAttr::get(parser.getContext(), result.value());
+}
 
-  if (failed(parser.parseComma()))
+Attribute FloatPolynomialAttr::parse(AsmParser &parser, Type type) {
+  if (failed(parser.parseLess()))
     return {};
 
-  IntegerAttr coefficientModulusAttr = nullptr;
-  if (succeeded(parser.parseKeyword("coefficientModulus"))) {
-    if (failed(parser.parseEqual()))
-      return {};
-
-    IntegerType iType = mlir::dyn_cast<IntegerType>(ty);
-    if (!iType) {
-      parser.emitError(parser.getCurrentLocation(),
-                       "coefficientType must specify an integer type");
-      return {};
-    }
-    APInt coefficientModulus(iType.getWidth(), 0);
-    auto result = parser.parseInteger(coefficientModulus);
-    if (failed(result)) {
-      parser.emitError(parser.getCurrentLocation(),
-                       "invalid coefficient modulus");
-      return {};
-    }
-    coefficientModulusAttr = IntegerAttr::get(iType, coefficientModulus);
-
-    if (failed(parser.parseComma()))
-      return {};
-  }
-
-  PolynomialAttr polyAttr = nullptr;
-  if (succeeded(parser.parseKeyword("polynomialModulus"))) {
-    if (failed(parser.parseEqual()))
-      return {};
+  llvm::SmallVector<FloatMonomial> monomials;
+  llvm::StringSet<> variables;
 
-    PolynomialAttr attr;
-    if (failed(parser.parseAttribute<PolynomialAttr>(attr)))
-      return {};
-    polyAttr = attr;
-  }
+  ParseCoefficientFn<FloatMonomial> parseAndStoreCoefficient =
+      [&](FloatMonomial &monomial) -> OptionalParseResult {
+    double coeffValue = 1.0;
+    ParseResult result = parser.parseFloat(coeffValue);
+    monomial.setCoefficient(APFloat(coeffValue));
+    return OptionalParseResult(result);
+  };
 
-  Polynomial poly = polyAttr.getPolynomial();
-  APInt root(coefficientModulusAttr.getValue().getBitWidth(), 0);
-  IntegerAttr rootAttr = nullptr;
-  if (succeeded(parser.parseOptionalComma())) {
-    if (failed(parser.parseKeyword("primitiveRoot")) ||
-        failed(parser.parseEqual()))
-      return {};
-
-    ParseResult result = parser.parseInteger(root);
-    if (failed(result)) {
-      parser.emitError(parser.getCurrentLocation(), "invalid primitiveRoot");
-      return {};
-    }
-    rootAttr = IntegerAttr::get(coefficientModulusAttr.getType(), root);
+  if (failed(parsePolynomialAttr<FloatPolynomialAttr, FloatMonomial>(
+          parser, monomials, variables, parseAndStoreCoefficient))) {
+    return {};
   }
 
-  if (failed(parser.parseGreater()))
+  auto result = FloatPolynomial::fromMonomials(monomials);
+  if (failed(result)) {
+    parser.emitError(parser.getCurrentLocation())
+        << "parsed polynomial must have unique exponents among monomials";
     return {};
-
-  return RingAttr::get(parser.getContext(), ty, coefficientModulusAttr,
-                       polyAttr, rootAttr);
+  }
+  return FloatPolynomialAttr::get(parser.getContext(), result.value());
 }
 
 } // namespace polynomial
diff --git a/mlir/test/Dialect/Polynomial/attributes.mlir b/mlir/test/Dialect/Polynomial/attributes.mlir
index 3973ae394433..4bdfd44fd4d1 100644
--- a/mlir/test/Dialect/Polynomial/attributes.mlir
+++ b/mlir/test/Dialect/Polynomial/attributes.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s --split-input-file --verify-diagnostics
 
-#my_poly = #polynomial.polynomial<y + x**1024>
+#my_poly = #polynomial.int_polynomial<y + x**1024>
 // expected-error@below {{polynomials must have one indeterminate, but there were multiple: x, y}}
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
@@ -9,37 +9,31 @@
 // expected-error@below {{expected integer value}}
 // expected-error@below {{expected a monomial}}
 // expected-error@below {{found invalid integer exponent}}
-#my_poly = #polynomial.polynomial<5 + x**f>
+#my_poly = #polynomial.int_polynomial<5 + x**f>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 // -----
 
-#my_poly = #polynomial.polynomial<5 + x**2 + 3x**2>
+#my_poly = #polynomial.int_polynomial<5 + x**2 + 3x**2>
 // expected-error@below {{parsed polynomial must have unique exponents among monomials}}
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 // -----
 
 // expected-error@below {{expected + and more monomials, or > to end polynomial attribute}}
-#my_poly = #polynomial.polynomial<5 + x**2 7>
+#my_poly = #polynomial.int_polynomial<5 + x**2 7>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 // -----
 
 // expected-error@below {{expected a monomial}}
-#my_poly = #polynomial.polynomial<5 + x**2 +>
+#my_poly = #polynomial.int_polynomial<5 + x**2 +>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
 
 
 // -----
 
-#my_poly = #polynomial.polynomial<5 + x**2>
-// expected-error@below {{coefficientType must specify an integer type}}
-#ring1 = #polynomial.ring<coefficientType=f64, coefficientModulus=2837465, polynomialModulus=#my_poly>
-
-// -----
-
-#my_poly = #polynomial.polynomial<5 + x**2>
-// expected-error@below {{expected integer value}}
-// expected-error@below {{invalid coefficient modulus}}
+#my_poly = #polynomial.int_polynomial<5 + x**2>
+// expected-error@below {{failed to parse Polynomial_RingAttr parameter 'coefficientModulus' which is to be a `::mlir::IntegerAttr`}}
+// expected-error@below {{expected attribute value}}
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=x, polynomialModulus=#my_poly>
diff --git a/mlir/test/Dialect/Polynomial/ops.mlir b/mlir/test/Dialect/Polynomial/ops.mlir
index a29cfc2e9cc5..ff709960c50e 100644
--- a/mlir/test/Dialect/Polynomial/ops.mlir
+++ b/mlir/test/Dialect/Polynomial/ops.mlir
@@ -2,85 +2,87 @@
 
 // This simply tests for syntax.
 
-#my_poly = #polynomial.polynomial<1 + x**1024>
-#my_poly_2 = #polynomial.polynomial<2>
-#my_poly_3 = #polynomial.polynomial<3x>
-#my_poly_4 = #polynomial.polynomial<t**3 + 4t + 2>
+#my_poly = #polynomial.int_polynomial<1 + x**1024>
+#my_poly_2 = #polynomial.int_polynomial<2>
+#my_poly_3 = #polynomial.int_polynomial<3x>
+#my_poly_4 = #polynomial.int_polynomial<t**3 + 4t + 2>
 #ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
-#one_plus_x_squared = #polynomial.polynomial<1 + x**2>
+#ring2 = #polynomial.ring<coefficientType=f32>
+#one_plus_x_squared = #polynomial.int_polynomial<1 + x**2>
 
-#ideal = #polynomial.polynomial<-1 + x**1024>
+#ideal = #polynomial.int_polynomial<-1 + x**1024>
 #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#ideal, primitiveRoot=193>
-!poly_ty = !polynomial.polynomial<#ring>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
-#ntt_poly = #polynomial.polynomial<-1 + x**8>
+#ntt_poly = #polynomial.int_polynomial<-1 + x**8>
 #ntt_ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#ntt_poly, primitiveRoot=31>
-!ntt_poly_ty = !polynomial.polynomial<#ntt_ring>
+!ntt_poly_ty = !polynomial.polynomial<ring=#ntt_ring>
 
 module {
-  func.func @test_multiply() -> !polynomial.polynomial<#ring1> {
+  func.func @test_multiply() -> !polynomial.polynomial<ring=#ring1> {
     %c0 = arith.constant 0 : index
     %two = arith.constant 2 : i16
     %five = arith.constant 5 : i16
     %coeffs1 = tensor.from_elements %two, %two, %five : tensor<3xi16>
     %coeffs2 = tensor.from_elements %five, %five, %two : tensor<3xi16>
 
-    %poly1 = polynomial.from_tensor %coeffs1 : tensor<3xi16> -> !polynomial.polynomial<#ring1>
-    %poly2 = polynomial.from_tensor %coeffs2 : tensor<3xi16> -> !polynomial.polynomial<#ring1>
+    %poly1 = polynomial.from_tensor %coeffs1 : tensor<3xi16> -> !polynomial.polynomial<ring=#ring1>
+    %poly2 = polynomial.from_tensor %coeffs2 : tensor<3xi16> -> !polynomial.polynomial<ring=#ring1>
 
-    %3 = polynomial.mul %poly1, %poly2 : !polynomial.polynomial<#ring1>
+    %3 = polynomial.mul %poly1, %poly2 : !polynomial.polynomial<ring=#ring1>
 
-    return %3 : !polynomial.polynomial<#ring1>
+    return %3 : !polynomial.polynomial<ring=#ring1>
   }
 
-  func.func @test_elementwise(%p0 : !polynomial.polynomial<#ring1>, %p1: !polynomial.polynomial<#ring1>) {
-    %tp0 = tensor.from_elements %p0, %p1 : tensor<2x!polynomial.polynomial<#ring1>>
-    %tp1 = tensor.from_elements %p1, %p0 : tensor<2x!polynomial.polynomial<#ring1>>
+  func.func @test_elementwise(%p0 : !polynomial.polynomial<ring=#ring1>, %p1: !polynomial.polynomial<ring=#ring1>) {
+    %tp0 = tensor.from_elements %p0, %p1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
+    %tp1 = tensor.from_elements %p1, %p0 : tensor<2x!polynomial.polynomial<ring=#ring1>>
 
     %c = arith.constant 2 : i32
-    %mul_const_sclr = polynomial.mul_scalar %tp0, %c : tensor<2x!polynomial.polynomial<#ring1>>, i32
+    %mul_const_sclr = polynomial.mul_scalar %tp0, %c : tensor<2x!polynomial.polynomial<ring=#ring1>>, i32
 
-    %add = polynomial.add %tp0, %tp1 : tensor<2x!polynomial.polynomial<#ring1>>
-    %sub = polynomial.sub %tp0, %tp1 : tensor<2x!polynomial.polynomial<#ring1>>
-    %mul = polynomial.mul %tp0, %tp1 : tensor<2x!polynomial.polynomial<#ring1>>
+    %add = polynomial.add %tp0, %tp1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
+    %sub = polynomial.sub %tp0, %tp1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
+    %mul = polynomial.mul %tp0, %tp1 : tensor<2x!polynomial.polynomial<ring=#ring1>>
 
     return
   }
 
-  func.func @test_to_from_tensor(%p0 : !polynomial.polynomial<#ring1>) {
+  func.func @test_to_from_tensor(%p0 : !polynomial.polynomial<ring=#ring1>) {
     %c0 = arith.constant 0 : index
     %two = arith.constant 2 : i16
     %coeffs1 = tensor.from_elements %two, %two : tensor<2xi16>
     // CHECK: from_tensor
-    %poly = polynomial.from_tensor %coeffs1 : tensor<2xi16> -> !polynomial.polynomial<#ring1>
+    %poly = polynomial.from_tensor %coeffs1 : tensor<2xi16> -> !polynomial.polynomial<ring=#ring1>
     // CHECK: to_tensor
-    %tensor = polynomial.to_tensor %poly : !polynomial.polynomial<#ring1> -> tensor<1024xi16>
+    %tensor = polynomial.to_tensor %poly : !polynomial.polynomial<ring=#ring1> -> tensor<1024xi16>
 
     return
   }
 
-  func.func @test_degree(%p0 : !polynomial.polynomial<#ring1>) {
-    %0, %1 = polynomial.leading_term %p0 : !polynomial.polynomial<#ring1> -> (index, i32)
+  func.func @test_degree(%p0 : !polynomial.polynomial<ring=#ring1>) {
+    %0, %1 = polynomial.leading_term %p0 : !polynomial.polynomial<ring=#ring1> -> (index, i32)
     return
   }
 
   func.func @test_monomial() {
     %deg = arith.constant 1023 : index
     %five = arith.constant 5 : i16
-    %0 = polynomial.monomial %five, %deg : (i16, index) -> !polynomial.polynomial<#ring1>
+    %0 = polynomial.monomial %five, %deg : (i16, index) -> !polynomial.polynomial<ring=#ring1>
     return
   }
 
   func.func @test_monic_monomial_mul() {
     %five = arith.constant 5 : index
-    %0 = polynomial.constant #one_plus_x_squared : !polynomial.polynomial<#ring1>
-    %1 = polynomial.monic_monomial_mul %0, %five : (!polynomial.polynomial<#ring1>, index) -> !polynomial.polynomial<#ring1>
+    %0 = polynomial.constant {value=#one_plus_x_squared} : !polynomial.polynomial<ring=#ring1>
+    %1 = polynomial.monic_monomial_mul %0, %five : (!polynomial.polynomial<ring=#ring1>, index) -> !polynomial.polynomial<ring=#ring1>
     return
   }
 
   func.func @test_constant() {
-    %0 = polynomial.constant #one_plus_x_squared : !polynomial.polynomial<#ring1>
-    %1 = polynomial.constant <1 + x**2> : !polynomial.polynomial<#ring1>
+    %0 = polynomial.constant {value=#one_plus_x_squared} : !polynomial.polynomial<ring=#ring1>
+    %1 = polynomial.constant {value=#polynomial.int_polynomial<1 + x**2>} : !polynomial.polynomial<ring=#ring1>
+    %2 = polynomial.constant {value=#polynomial.float_polynomial<1.5 + 0.5 x**2>} : !polynomial.polynomial<ring=#ring2>
     return
   }
 
diff --git a/mlir/test/Dialect/Polynomial/ops_errors.mlir b/mlir/test/Dialect/Polynomial/ops_errors.mlir
index 2c20e7bcbf1d..af8e4aa5da86 100644
--- a/mlir/test/Dialect/Polynomial/ops_errors.mlir
+++ b/mlir/test/Dialect/Polynomial/ops_errors.mlir
@@ -1,8 +1,8 @@
 // RUN: mlir-opt --split-input-file --verify-diagnostics %s
 
-#my_poly = #polynomial.polynomial<1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i32, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<ring=#ring>
 
 func.func @test_from_tensor_too_large_coeffs() {
   %two = arith.constant 2 : i32
@@ -15,13 +15,13 @@ func.func @test_from_tensor_too_large_coeffs() {
 
 // -----
 
-#my_poly = #polynomial.polynomial<1 + x**4>
-#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<1 + x**4>
+#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256:i32, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<ring=#ring>
 func.func @test_from_tensor_wrong_tensor_type() {
   %two = arith.constant 2 : i32
   %coeffs1 = tensor.from_elements %two, %two, %two, %two, %two : tensor<5xi32>
-  // expected-error@below {{input type 'tensor<5xi32>' does not match output type '!polynomial.polynomial<#polynomial.ring<coefficientType=i32, coefficientModulus=256 : i32, polynomialModulus=#polynomial.polynomial<1 + x**4>>>'}}
+  // expected-error@below {{input type 'tensor<5xi32>' does not match output type '!polynomial.polynomial<ring = <coefficientType = i32, coefficientModulus = 256 : i32, polynomialModulus = <1 + x**4>>>'}}
   // expected-note@below {{at most the degree of the polynomialModulus of the output type's ring attribute}}
   %poly = polynomial.from_tensor %coeffs1 : tensor<5xi32> -> !ty
   return
@@ -29,11 +29,11 @@ func.func @test_from_tensor_wrong_tensor_type() {
 
 // -----
 
-#my_poly = #polynomial.polynomial<1 + x**4>
-#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<1 + x**4>
+#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256:i32, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<ring=#ring>
 func.func @test_to_tensor_wrong_output_tensor_type(%arg0 : !ty) {
-  // expected-error@below {{input type '!polynomial.polynomial<#polynomial.ring<coefficientType=i32, coefficientModulus=256 : i32, polynomialModulus=#polynomial.polynomial<1 + x**4>>>' does not match output type 'tensor<5xi32>'}}
+  // expected-error@below {{input type '!polynomial.polynomial<ring = <coefficientType = i32, coefficientModulus = 256 : i32, polynomialModulus = <1 + x**4>>>' does not match output type 'tensor<5xi32>'}}
   // expected-note@below {{at most the degree of the polynomialModulus of the input type's ring attribute}}
   %tensor = polynomial.to_tensor %arg0 : !ty -> tensor<5xi32>
   return
@@ -41,9 +41,9 @@ func.func @test_to_tensor_wrong_output_tensor_type(%arg0 : !ty) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i32, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<ring=#ring>
 
 func.func @test_mul_scalar_wrong_type(%arg0: !ty) -> !ty {
   %scalar = arith.constant 2 : i32  // should be i16
@@ -54,9 +54,9 @@ func.func @test_mul_scalar_wrong_type(%arg0: !ty) -> !ty {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=31>
-!poly_ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_ntt
 // CHECK-NOT: polynomial.ntt
@@ -68,9 +68,9 @@ func.func @test_invalid_ntt(%0 : !poly_ty) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=31>
-!poly_ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_ntt
 // CHECK-NOT: polynomial.ntt
@@ -82,10 +82,10 @@ func.func @test_invalid_ntt(%0 : !poly_ty) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
-#ring1 = #polynomial.ring<coefficientType=i16, coefficientModulus=257, polynomialModulus=#my_poly, primitiveRoot=31>
-!poly_ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly>
+#ring1 = #polynomial.ring<coefficientType=i16, coefficientModulus=257:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_intt
 // CHECK-NOT: polynomial.intt
@@ -97,9 +97,9 @@ func.func @test_invalid_intt(%0 : tensor<1024xi32, #ring1>) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=31>
-!poly_ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=31:i16>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_intt
 // CHECK-NOT: polynomial.intt
@@ -112,9 +112,9 @@ func.func @test_invalid_intt(%0 : tensor<1025xi32, #ring>) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**1024>
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly>
-!poly_ty = !polynomial.polynomial<#ring>
+#my_poly = #polynomial.int_polynomial<-1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_ntt
 // CHECK-NOT: polynomial.ntt
@@ -126,10 +126,10 @@ func.func @test_invalid_ntt(%0 : !poly_ty) {
 
 // -----
 
-#my_poly = #polynomial.polynomial<-1 + x**8>
+#my_poly = #polynomial.int_polynomial<-1 + x**8>
 // A valid root is 31
-#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256, polynomialModulus=#my_poly, primitiveRoot=32>
-!poly_ty = !polynomial.polynomial<#ring>
+#ring = #polynomial.ring<coefficientType=i16, coefficientModulus=256:i16, polynomialModulus=#my_poly, primitiveRoot=32:i16>
+!poly_ty = !polynomial.polynomial<ring=#ring>
 
 // CHECK-NOT: @test_invalid_intt
 // CHECK-NOT: polynomial.intt
diff --git a/mlir/test/Dialect/Polynomial/types.mlir b/mlir/test/Dialect/Polynomial/types.mlir
index 00296a36e890..dcc5663ceb84 100644
--- a/mlir/test/Dialect/Polynomial/types.mlir
+++ b/mlir/test/Dialect/Polynomial/types.mlir
@@ -2,13 +2,13 @@
 
 // CHECK-LABEL: func @test_types
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    #polynomial.ring<
-// CHECK-SAME:       coefficientType=i32,
-// CHECK-SAME:       coefficientModulus=2837465 : i32,
-// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<1 + x**1024>>>
-#my_poly = #polynomial.polynomial<1 + x**1024>
-#ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
-!ty = !polynomial.polynomial<#ring1>
+// CHECK-SAME:    ring = <
+// CHECK-SAME:       coefficientType = i32,
+// CHECK-SAME:       coefficientModulus = 2837465 : i32,
+// CHECK-SAME:       polynomialModulus = <1 + x**1024>>>
+#my_poly = #polynomial.int_polynomial<1 + x**1024>
+#ring1 = #polynomial.ring<coefficientType = i32, coefficientModulus = 2837465 : i32, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<ring=#ring1>
 func.func @test_types(%0: !ty) -> !ty {
   return %0 : !ty
 }
@@ -16,13 +16,13 @@ func.func @test_types(%0: !ty) -> !ty {
 
 // CHECK-LABEL: func @test_non_x_variable_64_bit
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    #polynomial.ring<
-// CHECK-SAME:       coefficientType=i64,
-// CHECK-SAME:       coefficientModulus=2837465 : i64,
-// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<2 + 4x + x**3>>>
-#my_poly_2 = #polynomial.polynomial<t**3 + 4t + 2>
-#ring2 = #polynomial.ring<coefficientType=i64, coefficientModulus=2837465, polynomialModulus=#my_poly_2>
-!ty2 = !polynomial.polynomial<#ring2>
+// CHECK-SAME:    ring = <
+// CHECK-SAME:       coefficientType = i64,
+// CHECK-SAME:       coefficientModulus = 2837465 : i64,
+// CHECK-SAME:       polynomialModulus = <2 + 4x + x**3>>>
+#my_poly_2 = #polynomial.int_polynomial<t**3 + 4t + 2>
+#ring2 = #polynomial.ring<coefficientType = i64, coefficientModulus = 2837465 : i64, polynomialModulus=#my_poly_2>
+!ty2 = !polynomial.polynomial<ring=#ring2>
 func.func @test_non_x_variable_64_bit(%0: !ty2) -> !ty2 {
   return %0 : !ty2
 }
@@ -30,27 +30,36 @@ func.func @test_non_x_variable_64_bit(%0: !ty2) -> !ty2 {
 
 // CHECK-LABEL: func @test_linear_poly
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    #polynomial.ring<
-// CHECK-SAME:       coefficientType=i32,
-// CHECK-SAME:       coefficientModulus=12 : i32,
-// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<4x>>
-#my_poly_3 = #polynomial.polynomial<4x>
-#ring3 = #polynomial.ring<coefficientType=i32, coefficientModulus=12, polynomialModulus=#my_poly_3>
-!ty3 = !polynomial.polynomial<#ring3>
+// CHECK-SAME:    ring = <
+// CHECK-SAME:       coefficientType = i32,
+// CHECK-SAME:       coefficientModulus = 12 : i32,
+// CHECK-SAME:       polynomialModulus = <4x>>
+#my_poly_3 = #polynomial.int_polynomial<4x>
+#ring3 = #polynomial.ring<coefficientType = i32, coefficientModulus=12 : i32, polynomialModulus=#my_poly_3>
+!ty3 = !polynomial.polynomial<ring=#ring3>
 func.func @test_linear_poly(%0: !ty3) -> !ty3 {
   return %0 : !ty3
 }
 
 // CHECK-LABEL: func @test_negative_leading_1
 // CHECK-SAME:  !polynomial.polynomial<
-// CHECK-SAME:    #polynomial.ring<
-// CHECK-SAME:       coefficientType=i32,
-// CHECK-SAME:       coefficientModulus=2837465 : i32,
-// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<-1 + x**1024>>>
-#my_poly_4 = #polynomial.polynomial<-1 + x**1024>
-#ring4 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly_4>
-!ty4 = !polynomial.polynomial<#ring4>
+// CHECK-SAME:    ring = <
+// CHECK-SAME:       coefficientType = i32,
+// CHECK-SAME:       coefficientModulus = 2837465 : i32,
+// CHECK-SAME:       polynomialModulus = <-1 + x**1024>>>
+#my_poly_4 = #polynomial.int_polynomial<-1 + x**1024>
+#ring4 = #polynomial.ring<coefficientType = i32, coefficientModulus = 2837465 : i32, polynomialModulus=#my_poly_4>
+!ty4 = !polynomial.polynomial<ring=#ring4>
 func.func @test_negative_leading_1(%0: !ty4) -> !ty4 {
   return %0 : !ty4
 }
 
+// CHECK-LABEL: func @test_float_coefficients
+// CHECK-SAME:  !polynomial.polynomial<ring = <coefficientType = f32>>
+#my_poly_5 = #polynomial.float_polynomial<0.5 + 1.6e03 x**1024>
+#ring5 = #polynomial.ring<coefficientType=f32>
+!ty5 = !polynomial.polynomial<ring=#ring5>
+func.func @test_float_coefficients(%0: !ty5) -> !ty5 {
+  return %0 : !ty5
+}
+
-- 
GitLab


From 344c73ee831995d78d6eca1bed101878b5bae1bc Mon Sep 17 00:00:00 2001
From: Dmitri Gribenko <gribozavr@gmail.com>
Date: Tue, 14 May 2024 19:04:40 +0200
Subject: [PATCH 0753/1206] [libc][bazel] Updates for
 292b300c5131e54b9977305bb4aca9a03e1b4fed

---
 .../libc/test/src/string/BUILD.bazel          | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
index d96f390c0c38..fb0046e9f89d 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
@@ -126,13 +126,26 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "protected_pages",
+    hdrs = ["memory_utils/protected_pages.h"],
+    deps = [
+        "//libc:__support_macros_attributes",
+        "//libc:__support_macros_properties_os",
+    ],
+)
+
 libc_test(
     name = "memcpy_test",
     srcs = ["memcpy_test.cpp"],
     libc_function_deps = [
         "//libc:memcpy",
     ],
-    deps = [":memory_check_utils"],
+    deps = [
+        ":memory_check_utils",
+        ":protected_pages",
+        "//libc:__support_macros_properties_os",
+    ],
 )
 
 libc_test(
@@ -149,7 +162,11 @@ libc_test(
     libc_function_deps = [
         "//libc:memset",
     ],
-    deps = [":memory_check_utils"],
+    deps = [
+        ":memory_check_utils",
+        ":protected_pages",
+        "//libc:__support_macros_properties_os",
+    ],
 )
 
 libc_test(
-- 
GitLab


From 5f7477a72b826d0d6e7369ebe93cefcd55682d95 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 14 May 2024 10:07:13 -0700
Subject: [PATCH 0754/1206] RISCVAsmParser: Make diagnostics more conventional

Most diagnostics obey
https://llvm.org/docs/CodingStandards.html#error-and-warning-messages
but some diverge. Fix them.

While here, adjust some diagnostics.

Pull Request: https://github.com/llvm/llvm-project/pull/92024
---
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp |  47 ++--
 llvm/test/MC/RISCV/option-invalid.s           |   6 +-
 .../test/MC/RISCV/rv32xtheadmempair-invalid.s |   8 +-
 llvm/test/MC/RISCV/rv32zcmp-invalid.s         |   2 +-
 .../test/MC/RISCV/rv64xtheadmempair-invalid.s |  12 +-
 llvm/test/MC/RISCV/rv64zcmp-invalid.s         |   2 +-
 llvm/test/MC/RISCV/rvv/invalid.s              | 260 +++++++++---------
 llvm/test/MC/RISCV/rvv/xsfvcp-invalid.s       |   8 +-
 llvm/test/MC/RISCV/rvv/zvbb-invalid.s         |   6 +-
 llvm/test/MC/RISCV/rvv/zvkned-invalid.s       |  10 +-
 llvm/test/MC/RISCV/rvv/zvknh-invalid.s        |  12 +-
 llvm/test/MC/RISCV/rvv/zvksed-invalid.s       |   2 +-
 llvm/test/MC/RISCV/rvv/zvksh-invalid.s        |   4 +-
 13 files changed, 189 insertions(+), 190 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 6af1d5010d3a..7da1b7e360c7 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -2829,12 +2829,12 @@ bool RISCVAsmParser::parseDirectiveOption() {
 
       if (isDigit(Arch.back()))
         return Error(
-            Loc, "Extension version number parsing not currently implemented");
+            Loc, "extension version number parsing not currently implemented");
 
       std::string Feature = RISCVISAInfo::getTargetFeatureForExtension(Arch);
       if (!enableExperimentalExtension() &&
           StringRef(Feature).starts_with("experimental-"))
-        return Error(Loc, "Unexpected experimental extensions.");
+        return Error(Loc, "unexpected experimental extensions");
       auto Ext = llvm::lower_bound(RISCVFeatureKV, Feature);
       if (Ext == std::end(RISCVFeatureKV) || StringRef(Ext->Key) != Feature)
         return Error(Loc, "unknown extension feature");
@@ -2866,10 +2866,10 @@ bool RISCVAsmParser::parseDirectiveOption() {
         for (auto &Feature : RISCVFeatureKV) {
           if (getSTI().hasFeature(Feature.Value) &&
               Feature.Implies.test(Ext->Value))
-            return Error(Loc,
-                         Twine("Can't disable ") + Ext->Key + " extension, " +
-                             Feature.Key + " extension requires " + Ext->Key +
-                             " extension be enabled");
+            return Error(Loc, Twine("can't disable ") + Ext->Key +
+                                  " extension; " + Feature.Key +
+                                  " extension requires " + Ext->Key +
+                                  " extension");
         }
 
         clearFeatureBits(Ext->Value, Ext->Key);
@@ -3382,8 +3382,8 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
     unsigned TempReg = Inst.getOperand(1).getReg();
     if (DestReg == TempReg) {
       SMLoc Loc = Operands.back()->getStartLoc();
-      return Error(Loc, "The temporary vector register cannot be the same as "
-                        "the destination register.");
+      return Error(Loc, "the temporary vector register cannot be the same as "
+                        "the destination register");
     }
   }
 
@@ -3395,8 +3395,7 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
     // The encoding with rd1 == rd2 == rs1 is reserved for XTHead load pair.
     if (Rs1 == Rd1 && Rs1 == Rd2) {
       SMLoc Loc = Operands[1]->getStartLoc();
-      return Error(Loc, "The source register and destination registers "
-                        "cannot be equal.");
+      return Error(Loc, "rs1, rd1, and rd2 cannot all be the same");
     }
   }
 
@@ -3405,7 +3404,7 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
     unsigned Rd2 = Inst.getOperand(1).getReg();
     if (Rd1 == Rd2) {
       SMLoc Loc = Operands[1]->getStartLoc();
-      return Error(Loc, "'rs1' and 'rs2' must be different.");
+      return Error(Loc, "rs1 and rs2 must be different");
     }
   }
 
@@ -3416,10 +3415,10 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
   // depending on the data width.
   if (IsTHeadMemPair32 && Inst.getOperand(4).getImm() != 3) {
     SMLoc Loc = Operands.back()->getStartLoc();
-    return Error(Loc, "Operand must be constant 3.");
+    return Error(Loc, "operand must be constant 3");
   } else if (IsTHeadMemPair64 && Inst.getOperand(4).getImm() != 4) {
     SMLoc Loc = Operands.back()->getStartLoc();
-    return Error(Loc, "Operand must be constant 4.");
+    return Error(Loc, "operand must be constant 4");
   }
 
   const MCInstrDesc &MCID = MII.get(Opcode);
@@ -3434,14 +3433,14 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
     if (MCID.TSFlags & RISCVII::VS1Constraint) {
       unsigned VCIXRs1 = Inst.getOperand(Inst.getNumOperands() - 1).getReg();
       if (VCIXDst == VCIXRs1)
-        return Error(VCIXDstLoc, "The destination vector register group cannot"
-                                 " overlap the source vector register group.");
+        return Error(VCIXDstLoc, "the destination vector register group cannot"
+                                 " overlap the source vector register group");
     }
     if (MCID.TSFlags & RISCVII::VS2Constraint) {
       unsigned VCIXRs2 = Inst.getOperand(Inst.getNumOperands() - 2).getReg();
       if (VCIXDst == VCIXRs2)
-        return Error(VCIXDstLoc, "The destination vector register group cannot"
-                                 " overlap the source vector register group.");
+        return Error(VCIXDstLoc, "the destination vector register group cannot"
+                                 " overlap the source vector register group");
     }
     return false;
   }
@@ -3457,14 +3456,14 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
   if (MCID.TSFlags & RISCVII::VS2Constraint) {
     unsigned CheckReg = Inst.getOperand(Offset + 1).getReg();
     if (DestReg == CheckReg)
-      return Error(Loc, "The destination vector register group cannot overlap"
-                        " the source vector register group.");
+      return Error(Loc, "the destination vector register group cannot overlap"
+                        " the source vector register group");
   }
   if ((MCID.TSFlags & RISCVII::VS1Constraint) && Inst.getOperand(Offset + 2).isReg()) {
     unsigned CheckReg = Inst.getOperand(Offset + 2).getReg();
     if (DestReg == CheckReg)
-      return Error(Loc, "The destination vector register group cannot overlap"
-                        " the source vector register group.");
+      return Error(Loc, "the destination vector register group cannot overlap"
+                        " the source vector register group");
   }
   if ((MCID.TSFlags & RISCVII::VMConstraint) && (DestReg == RISCV::V0)) {
     // vadc, vsbc are special cases. These instructions have no mask register.
@@ -3474,7 +3473,7 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
         Opcode == RISCV::VSBC_VXM || Opcode == RISCV::VFMERGE_VFM ||
         Opcode == RISCV::VMERGE_VIM || Opcode == RISCV::VMERGE_VVM ||
         Opcode == RISCV::VMERGE_VXM)
-      return Error(Loc, "The destination vector register group cannot be V0.");
+      return Error(Loc, "the destination vector register group cannot be V0");
 
     // Regardless masked or unmasked version, the number of operands is the
     // same. For example, "viota.m v0, v2" is "viota.m v0, v2, NoRegister"
@@ -3485,8 +3484,8 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
            "Unexpected register for mask operand");
 
     if (DestReg == CheckReg)
-      return Error(Loc, "The destination vector register group cannot overlap"
-                        " the mask register.");
+      return Error(Loc, "the destination vector register group cannot overlap"
+                        " the mask register");
   }
   return false;
 }
diff --git a/llvm/test/MC/RISCV/option-invalid.s b/llvm/test/MC/RISCV/option-invalid.s
index ee520e08746a..7f6de5f38c52 100644
--- a/llvm/test/MC/RISCV/option-invalid.s
+++ b/llvm/test/MC/RISCV/option-invalid.s
@@ -34,14 +34,14 @@
 # CHECK: :[[#@LINE+1]]:18: error: expected comma
 .option arch, +c foo
 
-# CHECK: :[[#@LINE+1]]:16: error: Extension version number parsing not currently implemented
+# CHECK: :[[#@LINE+1]]:16: error: extension version number parsing not currently implemented
 .option arch, +c2p0
 
 .option arch, +d
-# CHECK: :[[#@LINE+1]]:16: error: Can't disable f extension, d extension requires f extension be enabled
+# CHECK: :[[#@LINE+1]]:16: error: can't disable f extension; d extension requires f extension
 .option arch, -f
 
-# CHECK: :[[#@LINE+1]]:16: error: Can't disable zicsr extension, f extension requires zicsr extension be enabled
+# CHECK: :[[#@LINE+1]]:16: error: can't disable zicsr extension; f extension requires zicsr extension
 .option arch, -zicsr
 
 # CHECK: :[[#@LINE+1]]:20: error: 'f' and 'zfinx' extensions are incompatible
diff --git a/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s b/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
index 94319dea1c17..9124218c1f8f 100644
--- a/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
+++ b/llvm/test/MC/RISCV/rv32xtheadmempair-invalid.s
@@ -8,13 +8,13 @@ th.sdd a0, a1, (a2)         # CHECK: [[@LINE]]:1: error: too few operands for in
 th.sdd a0, a1, (a2), 3, 5   # CHECK: [[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set{{$}}
 th.lwud t0, t1, (t2), 5, 4  # CHECK: [[@LINE]]:23: error: immediate must be an integer in the range [0, 3]
 th.lwud t0, t1, (t2)        # CHECK: [[@LINE]]:1: error: too few operands for instruction
-th.lwud t0, t1, (t2), 3, 5  # CHECK: [[@LINE]]:26: error: Operand must be constant 3.
+th.lwud t0, t1, (t2), 3, 5  # CHECK: [[@LINE]]:26: error: operand must be constant 3
 th.lwd a3, a4, (a5), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.lwd a3, a4, (a5)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
-th.lwd a3, a4, (a5), 3, 5   # CHECK: [[@LINE]]:25: error: Operand must be constant 3.
+th.lwd a3, a4, (a5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 3
 th.swd t3, t4, (t5), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.swd t3, t4, (t5)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
-th.swd t3, t4, (t5), 3, 5   # CHECK: [[@LINE]]:25: error: Operand must be constant 3.
-th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: The source register and destination registers cannot be equal.
+th.swd t3, t4, (t5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 3
+th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot all be the same
 th.ldd t0, t1, (t2), 2, 4   # CHECK: [[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set{{$}}
 th.sdd t0, t1, (t2), 2, 4   # CHECK: [[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set{{$}}
diff --git a/llvm/test/MC/RISCV/rv32zcmp-invalid.s b/llvm/test/MC/RISCV/rv32zcmp-invalid.s
index 1acea187585f..2ed82bc55be3 100644
--- a/llvm/test/MC/RISCV/rv32zcmp-invalid.s
+++ b/llvm/test/MC/RISCV/rv32zcmp-invalid.s
@@ -4,7 +4,7 @@
 # CHECK-ERROR: error: invalid operand for instruction
 cm.mvsa01 a1, a2
 
-# CHECK-ERROR: error: 'rs1' and 'rs2' must be different
+# CHECK-ERROR: error: rs1 and rs2 must be different
 cm.mvsa01 s0, s0
 
 # CHECK-ERROR: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s b/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
index cf56a1fcccaf..342db21bbca6 100644
--- a/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
+++ b/llvm/test/MC/RISCV/rv64xtheadmempair-invalid.s
@@ -2,17 +2,17 @@
 
 th.ldd t0, t1, (t2), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.ldd t0, t1, (t2)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
-th.ldd t0, t1, (t2), 3, 5   # CHECK: [[@LINE]]:25: error: Operand must be constant 4.
+th.ldd t0, t1, (t2), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 4
 th.sdd a0, a1, (a2), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.sdd a0, a1, (a2)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
-th.sdd a0, a1, (a2), 3, 5   # CHECK: [[@LINE]]:25: error: Operand must be constant 4.
+th.sdd a0, a1, (a2), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 4
 th.lwud t0, t1, (t2), 5, 4  # CHECK: [[@LINE]]:23: error: immediate must be an integer in the range [0, 3]
 th.lwud t0, t1, (t2)        # CHECK: [[@LINE]]:1: error: too few operands for instruction
-th.lwud t0, t1, (t2), 3, 5  # CHECK: [[@LINE]]:26: error: Operand must be constant 3.
+th.lwud t0, t1, (t2), 3, 5  # CHECK: [[@LINE]]:26: error: operand must be constant 3
 th.lwd a3, a4, (a5), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.lwd a3, a4, (a5)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
-th.lwd a3, a4, (a5), 3, 5   # CHECK: [[@LINE]]:25: error: Operand must be constant 3.
+th.lwd a3, a4, (a5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 3
 th.swd t3, t4, (t5), 5, 4   # CHECK: [[@LINE]]:22: error: immediate must be an integer in the range [0, 3]
 th.swd t3, t4, (t5)         # CHECK: [[@LINE]]:1: error: too few operands for instruction
-th.swd t3, t4, (t5), 3, 5   # CHECK: [[@LINE]]:25: error: Operand must be constant 3.
-th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: The source register and destination registers cannot be equal.
+th.swd t3, t4, (t5), 3, 5   # CHECK: [[@LINE]]:25: error: operand must be constant 3
+th.lwud x6, x6, (x6), 2, 3  # CHECK: [[@LINE]]:9: error: rs1, rd1, and rd2 cannot all be the same
diff --git a/llvm/test/MC/RISCV/rv64zcmp-invalid.s b/llvm/test/MC/RISCV/rv64zcmp-invalid.s
index bf34554095ea..8f353e8a7bb4 100644
--- a/llvm/test/MC/RISCV/rv64zcmp-invalid.s
+++ b/llvm/test/MC/RISCV/rv64zcmp-invalid.s
@@ -4,7 +4,7 @@
 # CHECK-ERROR: error: invalid operand for instruction
 cm.mvsa01 a1, a2
 
-# CHECK-ERROR: error: 'rs1' and 'rs2' must be different
+# CHECK-ERROR: error: rs1 and rs2 must be different
 cm.mvsa01 s0, s0
 
 # CHECK-ERROR: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/rvv/invalid.s b/llvm/test/MC/RISCV/rvv/invalid.s
index 09fd2c3bebf0..8c50f7ed048c 100644
--- a/llvm/test/MC/RISCV/rvv/invalid.s
+++ b/llvm/test/MC/RISCV/rvv/invalid.s
@@ -96,471 +96,471 @@ vmslt.vi v1, v2, 17
 # CHECK-ERROR: immediate must be in the range [-15, 16]
 
 viota.m v0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: viota.m v0, v2, v0.t
 
 viota.m v2, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: viota.m v2, v2
 
 vfwcvt.xu.f.v v0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwcvt.xu.f.v v0, v2, v0.t
 
 vfwcvt.xu.f.v v2, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwcvt.xu.f.v v2, v2
 
 vfwcvt.x.f.v v0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwcvt.x.f.v v0, v2, v0.t
 
 vfwcvt.x.f.v v2, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwcvt.x.f.v v2, v2
 
 vfwcvt.f.xu.v v0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwcvt.f.xu.v v0, v2, v0.t
 
 vfwcvt.f.xu.v v2, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwcvt.f.xu.v v2, v2
 
 vfwcvt.f.x.v v0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwcvt.f.x.v v0, v2, v0.t
 
 vfwcvt.f.x.v v2, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwcvt.f.x.v v2, v2
 
 vfwcvt.f.f.v v0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwcvt.f.f.v v0, v2, v0.t
 
 vfwcvt.f.f.v v2, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwcvt.f.f.v v2, v2
 
 vslideup.vx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vslideup.vx v0, v2, a0, v0.t
 
 vslideup.vx v2, v2, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vslideup.vx v2, v2, a0
 
 vslideup.vi v0, v2, 31, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vslideup.vi v0, v2, 31, v0.t
 
 vslideup.vi v2, v2, 31
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vslideup.vi v2, v2, 31
 
 vslide1up.vx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vslide1up.vx v0, v2, a0, v0.t
 
 vslide1up.vx v2, v2, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vslide1up.vx v2, v2, a0
 
 vrgather.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vrgather.vv v0, v2, v4, v0.t
 
 vrgather.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vrgather.vv v2, v2, v4
 
 vrgather.vx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vrgather.vx v0, v2, a0, v0.t
 
 vrgather.vx v2, v2, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vrgather.vx v2, v2, a0
 
 vrgather.vi v0, v2, 31, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vrgather.vi v0, v2, 31, v0.t
 
 vrgather.vi v2, v2, 31
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vrgather.vi v2, v2, 31
 
 vwaddu.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwaddu.vv v0, v2, v4, v0.t
 
 vwaddu.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwaddu.vv v2, v2, v4
 
 vwsubu.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwsubu.vv v0, v2, v4, v0.t
 
 vwsubu.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwsubu.vv v2, v2, v4
 
 vwadd.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwadd.vv v0, v2, v4, v0.t
 
 vwadd.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwadd.vv v2, v2, v4
 
 vwsub.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwsub.vv v0, v2, v4, v0.t
 
 vwsub.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwsub.vv v2, v2, v4
 
 vwmul.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmul.vv v0, v2, v4, v0.t
 
 vwmul.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmul.vv v2, v2, v4
 
 vwmulu.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmulu.vv v0, v2, v4, v0.t
 
 vwmulu.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmulu.vv v2, v2, v4
 
 vwmulsu.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmulsu.vv v0, v2, v4, v0.t
 
 vwmulsu.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmulsu.vv v2, v2, v4
 
 vwmaccu.vv v0, v4, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmaccu.vv v0, v4, v2, v0.t
 
 vwmaccu.vv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmaccu.vv v2, v4, v2
 
 vwmacc.vv v0, v4, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmacc.vv v0, v4, v2, v0.t
 
 vwmacc.vv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmacc.vv v2, v4, v2
 
 vwmaccsu.vv v0, v4, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmaccsu.vv v0, v4, v2, v0.t
 
 vwmaccsu.vv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmaccsu.vv v2, v4, v2
 
 vfwadd.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwadd.vv v0, v2, v4, v0.t
 
 vfwadd.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwadd.vv v2, v2, v4
 
 vfwsub.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwsub.vv v0, v2, v4, v0.t
 
 vfwsub.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwsub.vv v2, v2, v4
 
 vfwmul.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwmul.vv v0, v2, v4, v0.t
 
 vfwmul.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwmul.vv v2, v2, v4
 
 vfwmacc.vv v0, v4, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwmacc.vv v0, v4, v2, v0.t
 
 vfwmacc.vv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwmacc.vv v2, v4, v2
 
 vfwnmacc.vv v0, v4, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwnmacc.vv v0, v4, v2, v0.t
 
 vfwnmacc.vv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwnmacc.vv v2, v4, v2
 
 vfwmsac.vv v0, v4, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwmsac.vv v0, v4, v2, v0.t
 
 vfwmsac.vv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwmsac.vv v2, v4, v2
 
 vfwnmsac.vv v0, v4, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwnmsac.vv v0, v4, v2, v0.t
 
 vfwnmsac.vv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwnmsac.vv v2, v4, v2
 
 vwaddu.vx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwaddu.vx v0, v2, a0, v0.t
 
 vwaddu.vx v2, v2, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwaddu.vx v2, v2, a0
 
 vwsubu.vx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwsubu.vx v0, v2, a0, v0.t
 
 vwsubu.vx v2, v2, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwsubu.vx v2, v2, a0
 
 vwadd.vx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwadd.vx v0, v2, a0, v0.t
 
 vwadd.vx v2, v2, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwadd.vx v2, v2, a0
 
 vwsub.vx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwsub.vx v0, v2, a0, v0.t
 
 vwsub.vx v2, v2, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwsub.vx v2, v2, a0
 
 vwmul.vx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmul.vx v0, v2, a0, v0.t
 
 vwmul.vx v2, v2, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmul.vx v2, v2, a0
 
 vwmulu.vx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmulu.vx v0, v2, a0, v0.t
 
 vwmulu.vx v2, v2, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmulu.vx v2, v2, a0
 
 vwmulsu.vx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmulsu.vx v0, v2, a0, v0.t
 
 vwmulsu.vx v2, v2, a0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmulsu.vx v2, v2, a0
 
 vwmaccu.vx v0, a0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmaccu.vx v0, a0, v2, v0.t
 
 vwmaccu.vx v2, a0, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmaccu.vx v2, a0, v2
 
 vwmacc.vx v0, a0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmacc.vx v0, a0, v2, v0.t
 
 vwmacc.vx v2, a0, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmacc.vx v2, a0, v2
 
 vwmaccsu.vx v0, a0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmaccsu.vx v0, a0, v2, v0.t
 
 vwmaccsu.vx v2, a0, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmaccsu.vx v2, a0, v2
 
 vwmaccus.vx v0, a0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwmaccus.vx v0, a0, v2, v0.t
 
 vwmaccus.vx v2, a0, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwmaccus.vx v2, a0, v2
 
 vfwadd.vf v0, v2, fa0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwadd.vf v0, v2, fa0, v0.t
 
 vfwadd.vf v2, v2, fa0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwadd.vf v2, v2, fa0
 
 vfwsub.vf v0, v2, fa0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwsub.vf v0, v2, fa0, v0.t
 
 vfwsub.vf v2, v2, fa0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwsub.vf v2, v2, fa0
 
 vfwmul.vf v0, v2, fa0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwmul.vf v0, v2, fa0, v0.t
 
 vfwmul.vf v2, v2, fa0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwmul.vf v2, v2, fa0
 
 vfwmacc.vf v0, fa0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwmacc.vf v0, fa0, v2, v0.t
 
 vfwmacc.vf v2, fa0, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwmacc.vf v2, fa0, v2
 
 vfwnmacc.vf v0, fa0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwnmacc.vf v0, fa0, v2, v0.t
 
 vfwnmacc.vf v2, fa0, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwnmacc.vf v2, fa0, v2
 
 vfwmsac.vf v0, fa0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwmsac.vf v0, fa0, v2, v0.t
 
 vfwmsac.vf v2, fa0, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwmsac.vf v2, fa0, v2
 
 vfwnmsac.vf v0, fa0, v2, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwnmsac.vf v0, fa0, v2, v0.t
 
 vfwnmsac.vf v2, fa0, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwnmsac.vf v2, fa0, v2
 
 vcompress.vm v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vcompress.vm v2, v2, v4
 
 vwaddu.wv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwaddu.wv v0, v2, v4, v0.t
 
 vwaddu.wv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwaddu.wv v2, v4, v2
 
 vwsubu.wv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwsubu.wv v0, v2, v4, v0.t
 
 vwsubu.wv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwsubu.wv v2, v4, v2
 
 vwadd.wv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwadd.wv v0, v2, v4, v0.t
 
 vwadd.wv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwadd.wv v2, v4, v2
 
 vwsub.wv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwsub.wv v0, v2, v4, v0.t
 
 vwsub.wv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwsub.wv v2, v4, v2
 
 vfwadd.wv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwadd.wv v0, v2, v4, v0.t
 
 vfwadd.wv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwadd.wv v2, v4, v2
 
 vfwsub.wv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwsub.wv v0, v2, v4, v0.t
 
 vfwsub.wv v2, v4, v2
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vfwsub.wv v2, v4, v2
 
 vwaddu.wx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwaddu.wx v0, v2, a0, v0.t
 
 vwsubu.wx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwsubu.wx v0, v2, a0, v0.t
 
 vwadd.wx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwadd.wx v0, v2, a0, v0.t
 
 vwsub.wx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vwsub.wx v0, v2, a0, v0.t
 
 vfwadd.wf v0, v2, fa0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwadd.wf v0, v2, fa0, v0.t
 
 vfwsub.wf v0, v2, fa0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfwsub.wf v0, v2, fa0, v0.t
 
 vadc.vvm v0, v2, v4, v0
-# CHECK-ERROR: The destination vector register group cannot be V0.
+# CHECK-ERROR: the destination vector register group cannot be V0
 # CHECK-ERROR-LABEL: vadc.vvm v0, v2, v4, v0
 
 vadd.vv v0, v2, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vadd.vv v0, v2, v4, v0.t
 
 vadd.vx v0, v2, a0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vadd.vx v0, v2, a0, v0.t
 
 vadd.vi v0, v2, 1, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vadd.vi v0, v2, 1, v0.t
 
 vmsge.vx v0, v4, a0, v0.t
@@ -568,47 +568,47 @@ vmsge.vx v0, v4, a0, v0.t
 # CHECK-ERROR-LABEL: vmsge.vx v0, v4, a0, v0.t
 
 vmerge.vim v0, v1, 1, v0
-# CHECK-ERROR: The destination vector register group cannot be V0.
+# CHECK-ERROR: the destination vector register group cannot be V0
 # CHECK-ERROR-LABEL: vmerge.vim v0, v1, 1, v0
 
 vmerge.vvm v0, v1, v2, v0
-# CHECK-ERROR: The destination vector register group cannot be V0.
+# CHECK-ERROR: the destination vector register group cannot be V0
 # CHECK-ERROR-LABEL: vmerge.vvm v0, v1, v2, v0
 
 vmerge.vxm v0, v1, x1, v0
-# CHECK-ERROR: The destination vector register group cannot be V0.
+# CHECK-ERROR: the destination vector register group cannot be V0
 # CHECK-ERROR-LABEL: vmerge.vxm v0, v1, x1, v0
 
 vfmerge.vfm v0, v1, f1, v0
-# CHECK-ERROR: The destination vector register group cannot be V0.
+# CHECK-ERROR: the destination vector register group cannot be V0
 # CHECK-ERROR-LABEL: vfmerge.vfm v0, v1, f1, v0
 
 vle8.v v0, (a0), v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vle8.v v0, (a0), v0.t
 
 vfclass.v v0, v1, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfclass.v v0, v1, v0.t
 
 vfsqrt.v v0, v1, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfsqrt.v v0, v1, v0.t
 
 vzext.vf2 v0, v1, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vzext.vf2 v0, v1, v0.t
 
 vid.v v0, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vid.v v0, v0.t
 
 vnsrl.wv v0, v4, v20, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vnsrl.wv v0, v4, v20, v0.t
 
 vfncvt.xu.f.w v0, v4, v0.t
-# CHECK-ERROR: The destination vector register group cannot overlap the mask register.
+# CHECK-ERROR: the destination vector register group cannot overlap the mask register
 # CHECK-ERROR-LABEL: vfncvt.xu.f.w v0, v4, v0.t
 
 vl2re8.v v1, (a0)
@@ -750,7 +750,7 @@ vmsgeu.vx v2, v4, a0, v0.t, v0
 # CHECK-ERROR: invalid operand for instruction
 
 vmsge.vx v2, v4, a0, v0.t, v2
-# CHECK-ERROR: The temporary vector register cannot be the same as the destination register.
+# CHECK-ERROR: the temporary vector register cannot be the same as the destination register
 
 vmsgeu.vx v2, v4, a0, v0.t, v2
-# CHECK-ERROR: The temporary vector register cannot be the same as the destination register.
+# CHECK-ERROR: the temporary vector register cannot be the same as the destination register
diff --git a/llvm/test/MC/RISCV/rvv/xsfvcp-invalid.s b/llvm/test/MC/RISCV/rvv/xsfvcp-invalid.s
index 52dc32b8c6a8..9ba2a8d2f729 100644
--- a/llvm/test/MC/RISCV/rvv/xsfvcp-invalid.s
+++ b/llvm/test/MC/RISCV/rvv/xsfvcp-invalid.s
@@ -4,17 +4,17 @@
 # RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 
 sf.vc.v.vvw 0x3, v0, v2, v0
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.{{$}}
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group{{$}}
 # CHECK-ERROR-LABEL: sf.vc.v.vvw 0x3, v0, v2, v0{{$}}
 
 sf.vc.v.xvw 0x3, v0, v0, a1
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.{{$}}
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group{{$}}
 # CHECK-ERROR-LABEL: sf.vc.v.xvw 0x3, v0, v0, a1{{$}}
 
 sf.vc.v.ivw 0x3, v0, v0, 15
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.{{$}}
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group{{$}}
 # CHECK-ERROR-LABEL: sf.vc.v.ivw 0x3, v0, v0, 15{{$}}
 
 sf.vc.v.fvw 0x1, v0, v0, fa1
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.{{$}}
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group{{$}}
 # CHECK-ERROR-LABEL: sf.vc.v.fvw 0x1, v0, v0, fa1{{$}}
diff --git a/llvm/test/MC/RISCV/rvv/zvbb-invalid.s b/llvm/test/MC/RISCV/rvv/zvbb-invalid.s
index ca581de02fd6..42113c3c0ffe 100644
--- a/llvm/test/MC/RISCV/rvv/zvbb-invalid.s
+++ b/llvm/test/MC/RISCV/rvv/zvbb-invalid.s
@@ -2,13 +2,13 @@
 # RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 
 vwsll.vv v2, v2, v4
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwsll.vv v2, v2, v4
 
 vwsll.vx v2, v2, x10
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwsll.vx v2, v2, x10
 
 vwsll.vi v2, v2, 1
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vwsll.vi v2, v2, 1
diff --git a/llvm/test/MC/RISCV/rvv/zvkned-invalid.s b/llvm/test/MC/RISCV/rvv/zvkned-invalid.s
index 9230bc08e3fa..2d57a2816980 100644
--- a/llvm/test/MC/RISCV/rvv/zvkned-invalid.s
+++ b/llvm/test/MC/RISCV/rvv/zvkned-invalid.s
@@ -2,22 +2,22 @@
 # RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 
 vaesdf.vs v10, v10
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vaesdf.vs v10, v10
 
 vaesef.vs v11, v11
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vaesef.vs v11, v11
 
 vaesdm.vs v12, v12
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vaesdm.vs v12, v12
 
 vaesem.vs v13, v13
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vaesem.vs v13, v13
 
 vaesz.vs v14, v14
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vaesz.vs v14, v14
 
diff --git a/llvm/test/MC/RISCV/rvv/zvknh-invalid.s b/llvm/test/MC/RISCV/rvv/zvknh-invalid.s
index d9902511c0e1..f5c9ee57ff78 100644
--- a/llvm/test/MC/RISCV/rvv/zvknh-invalid.s
+++ b/llvm/test/MC/RISCV/rvv/zvknh-invalid.s
@@ -2,25 +2,25 @@
 # RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 
 vsha2ms.vv v10, v10, v11
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vsha2ms.vv v10, v10, v11
 
 vsha2ms.vv v11, v10, v11
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vsha2ms.vv v11, v10, v11
 
 vsha2ch.vv v12, v12, v11
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vsha2ch.vv v12, v12, v11
 
 vsha2ch.vv v11, v12, v11
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vsha2ch.vv v11, v12, v11
 
 vsha2cl.vv v13, v13, v15
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vsha2cl.vv v13, v13, v15
 
 vsha2cl.vv v15, v13, v15
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vsha2cl.vv v15, v13, v15
diff --git a/llvm/test/MC/RISCV/rvv/zvksed-invalid.s b/llvm/test/MC/RISCV/rvv/zvksed-invalid.s
index 41df8d3bc296..0f348a2ca5ad 100644
--- a/llvm/test/MC/RISCV/rvv/zvksed-invalid.s
+++ b/llvm/test/MC/RISCV/rvv/zvksed-invalid.s
@@ -2,5 +2,5 @@
 # RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 
 vsm4r.vs v10, v10
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vsm4r.vs v10, v10
diff --git a/llvm/test/MC/RISCV/rvv/zvksh-invalid.s b/llvm/test/MC/RISCV/rvv/zvksh-invalid.s
index cccec44b8191..27a11ed9b969 100644
--- a/llvm/test/MC/RISCV/rvv/zvksh-invalid.s
+++ b/llvm/test/MC/RISCV/rvv/zvksh-invalid.s
@@ -2,9 +2,9 @@
 # RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 
 vsm3me.vv v10, v10, v8
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vsm3me.vv v10, v10, v8
 
 vsm3c.vi v9, v9, 7
-# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR: the destination vector register group cannot overlap the source vector register group
 # CHECK-ERROR-LABEL: vsm3c.vi v9, v9, 7
-- 
GitLab


From 1355dcbb6e8b40780f1fbaa89cde50aa763dab89 Mon Sep 17 00:00:00 2001
From: quanwanandy <150498259+quanwanandy@users.noreply.github.com>
Date: Tue, 14 May 2024 10:07:26 -0700
Subject: [PATCH 0755/1206] Fix Bazel Build (#92139)

---
 .../llvm-project-overlay/mlir/BUILD.bazel     | 19 +++++++++++++++++++
 .../mlir/test/BUILD.bazel                     |  2 ++
 2 files changed, 21 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index debd8daf5549..a3171287a84b 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -2092,6 +2092,20 @@ gentbl_cc_library(
     deps = [":ArmSMETdFiles"],
 )
 
+cc_library(
+    name = "ArmSMEOpInterfaces",
+    hdrs = [
+        "include/mlir/Dialect/ArmSME/IR/ArmSMEOpInterfaces.h",
+    ],
+    includes = ["include"],
+    deps = [
+        ":ArmSMEOpInterfacesIncGen",
+        ":IR",
+        ":Support",
+        "//llvm:Support",
+    ],
+)
+
 gentbl_cc_library(
     name = "ArmSMEIntrinsicOpsIncGen",
     tbl_outs = [
@@ -2137,9 +2151,11 @@ cc_library(
         ":ArmSMEEnums",
         ":ArmSMEIncGen",
         ":ArmSMEIntrinsicOpsIncGen",
+        ":ArmSMEOpInterfaces",
         ":ArmSMEOpInterfacesIncGen",
         ":ArmSMEOpsIncGen",
         ":BytecodeOpInterface",
+	":FunctionInterfaces",
         ":IR",
         ":LLVMDialect",
         ":MemRefDialect",
@@ -2156,6 +2172,7 @@ cc_library(
     hdrs = glob(["include/mlir/Dialect/ArmSME/Transforms/*.h"]),
     includes = ["include"],
     deps = [
+	":Analysis",
         ":ArithUtils",
         ":ArmSMEDialect",
         ":ArmSMETransformsPassIncGen",
@@ -2163,6 +2180,7 @@ cc_library(
         ":DialectUtils",
         ":FuncDialect",
         ":FuncTransforms",
+	":FunctionInterfaces",
         ":IR",
         ":LLVMCommonConversion",
         ":LLVMDialect",
@@ -2200,6 +2218,7 @@ cc_library(
         ":ArmSMETransforms",
         ":ConversionPassIncGen",
         ":FuncDialect",
+	":FunctionInterfaces",
         ":LLVMCommonConversion",
         ":LLVMDialect",
         ":MemRefDialect",
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 65b31dc97e2d..258cc88ebbf3 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -951,8 +951,10 @@ cc_library(
         "//mlir:ArmSMEToSCF",
         "//mlir:ArmSMETransforms",
         "//mlir:ArmSVETransforms",
+	"//mlir:FuncDialect",
         "//mlir:IR",
         "//mlir:Pass",
+	"//mlir:SCFToControlFlow",
         "//mlir:Transforms",
         "//mlir:VectorToArmSME",
         "//mlir:VectorToSCF",
-- 
GitLab


From 77b80bd8186b392ed9bc87cd526a64093dd03522 Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer@amd.com>
Date: Tue, 14 May 2024 18:08:10 +0100
Subject: [PATCH 0756/1206] [mlir][flang] Improve handling of fortran module
 variables. (#91604)

Currently, only those global variables which are at compile unit scope
are added to the 'globals' list of the DICompileUnit. This does not work
for languages which support modules (e.g. Fortran) where hierarchy
can be
variable -> module -> compile unit.

To fix this, if a variable scope points to a module, we walk one level
up and see if module is in the compile unit scope.

This was initially part of #91582 which adds debug information for
Fortran module variables. @kiranchandramohan pointed out that MLIR
changes should go in separate PRs.
---
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 15 +++++++++++++--
 mlir/test/Target/LLVMIR/llvmir-debug.mlir    | 19 +++++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 669b95a9c6a5..2e4b0feb1973 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -1030,10 +1030,21 @@ LogicalResult ModuleTranslation::convertGlobals() {
       llvm::DIGlobalVariable *diGlobalVar = diGlobalExpr->getVariable();
       var->addDebugInfo(diGlobalExpr);
 
+      // There is no `globals` field in DICompileUnitAttr which can be directly
+      // assigned to DICompileUnit. We have to build the list by looking at the
+      // dbgExpr of all the GlobalOps. The scope of the variable is used to get
+      // the DICompileUnit in which to add it. But for the languages that
+      // support modules, the scope hierarchy can be
+      // variable -> module -> compile unit
+      // If a variable scope points to the module then we use the scope of the
+      // module to get the compile unit.
+      llvm::DIScope *scope = diGlobalVar->getScope();
+      if (llvm::DIModule *mod = dyn_cast_if_present<llvm::DIModule>(scope))
+        scope = mod->getScope();
+
       // Get the compile unit (scope) of the the global variable.
       if (llvm::DICompileUnit *compileUnit =
-              dyn_cast_if_present<llvm::DICompileUnit>(
-                  diGlobalVar->getScope())) {
+              dyn_cast_if_present<llvm::DICompileUnit>(scope)) {
         // Update the compile unit with this incoming global variable expression
         // during the finalizing step later.
         allGVars[compileUnit].push_back(diGlobalExpr);
diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
index 1f0fc969364a..15d4b8ccd88b 100644
--- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
@@ -311,6 +311,25 @@ llvm.mlir.global external @global_with_expr_2() {addr_space = 0 : i32, dbg_expr
 
 // -----
 
+// CHECK: @module_global = external global i64, !dbg {{.*}}
+// CHECK: !llvm.module.flags = !{{{.*}}}
+// CHECK: !llvm.dbg.cu = !{{{.*}}}
+// CHECK-DAG: ![[FILE:.*]] = !DIFile(filename: "test.f90", directory: "existence")
+// CHECK-DAG: ![[TYPE:.*]] = !DIBasicType(name: "integer", size: 64, encoding: DW_ATE_signed)
+// CHECK-DAG: ![[SCOPE:.*]] = distinct !DICompileUnit(language: DW_LANG_Fortran95, file: ![[FILE]], producer: "MLIR", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: ![[GVALS:.*]])
+// CHECK-DAG: ![[SCOPE1:.*]] = !DIModule(scope: ![[SCOPE]], name: "module2", file: ![[FILE]], line: 120)
+// CHECK-DAG: ![[GVAR:.*]] = distinct !DIGlobalVariable(name: "module_global", linkageName: "module_global", scope: ![[SCOPE1]], file: ![[FILE]], line: 121, type: ![[TYPE]], isLocal: false, isDefinition: true)
+// CHECK-DAG: ![[GEXPR:.*]] = !DIGlobalVariableExpression(var: ![[GVAR]], expr: !DIExpression())
+// CHECK-DAG: ![[GVALS]] = !{![[GEXPR]]}
+
+#di_file = #llvm.di_file<"test.f90" in "existence">
+#di_compile_unit = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "MLIR", isOptimized = true, emissionKind = Full>
+#di_basic_type = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 64, encoding = DW_ATE_signed>
+#di_module = #llvm.di_module<file = #di_file, scope = #di_compile_unit, name = "module2", configMacros = "", includePath = "", apinotes = "", line = 120, isDecl = false >
+llvm.mlir.global external @module_global() {dbg_expr = #llvm.di_global_variable_expression<var = <scope = #di_module, name = "module_global", linkageName = "module_global", file = #di_file, line = 121, type = #di_basic_type, isLocalToUnit = false, isDefined = true>, expr = <>>} : i64
+
+// -----
+
 // Nameless and scopeless global constant.
 
 // CHECK-LABEL: @.str.1 = external constant [10 x i8]
-- 
GitLab


From b04c07bf271be097a5e5726730c538454fd30992 Mon Sep 17 00:00:00 2001
From: Palmer Dabbelt <palmer@dabbelt.com>
Date: Tue, 14 May 2024 10:11:20 -0700
Subject: [PATCH 0757/1206] [RISCV] Only allow up to e64 in vsetvli (#92010)

These larger SEWs aren't in the ratified V spec. Thanks to dzaima and
sorear on IRC for pointing this one out.

Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
---
 .../llvm/TargetParser/RISCVTargetParser.h     |  2 +-
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp |  2 +-
 llvm/test/MC/RISCV/rvv/invalid.s              | 64 +++++++++++--------
 3 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/llvm/include/llvm/TargetParser/RISCVTargetParser.h b/llvm/include/llvm/TargetParser/RISCVTargetParser.h
index cdd19189f8dc..5b1494efe7bd 100644
--- a/llvm/include/llvm/TargetParser/RISCVTargetParser.h
+++ b/llvm/include/llvm/TargetParser/RISCVTargetParser.h
@@ -61,7 +61,7 @@ enum {
 namespace RISCVVType {
 // Is this a SEW value that can be encoded into the VTYPE format.
 inline static bool isValidSEW(unsigned SEW) {
-  return isPowerOf2_32(SEW) && SEW >= 8 && SEW <= 1024;
+  return isPowerOf2_32(SEW) && SEW >= 8 && SEW <= 64;
 }
 
 // Is this a LMUL value that can be encoded into the VTYPE format.
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 7da1b7e360c7..d92998ced91e 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -2225,7 +2225,7 @@ bool RISCVAsmParser::generateVTypeError(SMLoc ErrorLoc) {
   return Error(
       ErrorLoc,
       "operand must be "
-      "e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]");
+      "e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]");
 }
 
 ParseStatus RISCVAsmParser::parseMaskReg(OperandVector &Operands) {
diff --git a/llvm/test/MC/RISCV/rvv/invalid.s b/llvm/test/MC/RISCV/rvv/invalid.s
index 8c50f7ed048c..07e7b9db6606 100644
--- a/llvm/test/MC/RISCV/rvv/invalid.s
+++ b/llvm/test/MC/RISCV/rvv/invalid.s
@@ -2,83 +2,95 @@
 # RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 
 vsetivli a2, 32, e8,m1
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetivli a2, zero, e8,m1
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetivli a2, 5, (1 << 10)
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetivli a2, 5, 0x400
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetivli a2, 5, e31
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, (1 << 11)
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, 0x800
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 
 vsetvli a2, a0, e31
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e32,m3
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, m1,e32
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e32,m16
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e128,m8
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e256,m8
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e512,m8
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e1024,m8
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e2048,m8
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e1,m8
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8,m1,tx
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8,m1,ta,mx
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8,m1,ma
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8,m1,mu
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8x,m1,tu,mu
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8,m1z,tu,mu
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8,mf1,tu,mu
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8,m1,tu,mut
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8,m1,tut,mu
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8,m1
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8,m1,ta
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vsetvli a2, a0, e8,1,ta,ma
-# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+# CHECK-ERROR: operand must be e[8|16|32|64],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
 vadd.vv v1, v3, v2, v4.t
 # CHECK-ERROR: operand must be v0.t
-- 
GitLab


From 6c8ebc053533c691099ab60c41261b3cb4ba2fa3 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Tue, 14 May 2024 10:13:57 -0700
Subject: [PATCH 0758/1206] [NFC][CallPromotionUtils]Extract a helper function
 versionCallSiteWithCond from versionCallSite (#81181)

* This is to be used by https://github.com/llvm/llvm-project/pull/81378
to implement a variant of versionCallSite that compares vtables.
* The parent patch is https://github.com/llvm/llvm-project/pull/81051
---
 .../Transforms/Utils/CallPromotionUtils.cpp   | 39 ++++++++++++-------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
index 48c33d6c0c8e..9ca9aaf9ee9d 100644
--- a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
+++ b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
@@ -188,10 +188,9 @@ static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) {
 /// Predicate and clone the given call site.
 ///
 /// This function creates an if-then-else structure at the location of the call
-/// site. The "if" condition compares the call site's called value to the given
-/// callee. The original call site is moved into the "else" block, and a clone
-/// of the call site is placed in the "then" block. The cloned instruction is
-/// returned.
+/// site. The "if" condition is specified by `Cond`. The original call site is
+/// moved into the "else" block, and a clone of the call site is placed in the
+/// "then" block. The cloned instruction is returned.
 ///
 /// For example, the call instruction below:
 ///
@@ -202,7 +201,7 @@ static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) {
 /// Is replace by the following:
 ///
 ///   orig_bb:
-///     %cond = icmp eq i32 ()* %ptr, @func
+///     %cond = Cond
 ///     br i1 %cond, %then_bb, %else_bb
 ///
 ///   then_bb:
@@ -232,7 +231,7 @@ static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) {
 /// Is replace by the following:
 ///
 ///   orig_bb:
-///     %cond = icmp eq i32 ()* %ptr, @func
+///     %cond = Cond
 ///     br i1 %cond, %then_bb, %else_bb
 ///
 ///   then_bb:
@@ -267,7 +266,7 @@ static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) {
 /// Is replaced by the following:
 ///
 ///   cond_bb:
-///     %cond = icmp eq i32 ()* %ptr, @func
+///     %cond = Cond
 ///     br i1 %cond, %then_bb, %orig_bb
 ///
 ///   then_bb:
@@ -280,19 +279,13 @@ static void createRetBitCast(CallBase &CB, Type *RetTy, CastInst **RetBitCast) {
 ///     ; The original call instruction stays in its original block.
 ///     %t0 = musttail call i32 %ptr()
 ///     ret %t0
-CallBase &llvm::versionCallSite(CallBase &CB, Value *Callee,
-                                MDNode *BranchWeights) {
+static CallBase &versionCallSiteWithCond(CallBase &CB, Value *Cond,
+                                         MDNode *BranchWeights) {
 
   IRBuilder<> Builder(&CB);
   CallBase *OrigInst = &CB;
   BasicBlock *OrigBlock = OrigInst->getParent();
 
-  // Create the compare. The called value and callee must have the same type to
-  // be compared.
-  if (CB.getCalledOperand()->getType() != Callee->getType())
-    Callee = Builder.CreateBitCast(Callee, CB.getCalledOperand()->getType());
-  auto *Cond = Builder.CreateICmpEQ(CB.getCalledOperand(), Callee);
-
   if (OrigInst->isMustTailCall()) {
     // Create an if-then structure. The original instruction stays in its block,
     // and a clone of the original instruction is placed in the "then" block.
@@ -380,6 +373,22 @@ CallBase &llvm::versionCallSite(CallBase &CB, Value *Callee,
   return *NewInst;
 }
 
+// Predicate and clone the given call site using condition `CB.callee ==
+// Callee`. See the comment `versionCallSiteWithCond` for the transformation.
+CallBase &llvm::versionCallSite(CallBase &CB, Value *Callee,
+                                MDNode *BranchWeights) {
+
+  IRBuilder<> Builder(&CB);
+
+  // Create the compare. The called value and callee must have the same type to
+  // be compared.
+  if (CB.getCalledOperand()->getType() != Callee->getType())
+    Callee = Builder.CreateBitCast(Callee, CB.getCalledOperand()->getType());
+  auto *Cond = Builder.CreateICmpEQ(CB.getCalledOperand(), Callee);
+
+  return versionCallSiteWithCond(CB, Cond, BranchWeights);
+}
+
 bool llvm::isLegalToPromote(const CallBase &CB, Function *Callee,
                             const char **FailureReason) {
   assert(!CB.getCalledFunction() && "Only indirect call sites can be promoted");
-- 
GitLab


From 54435b5df32d80c68c94acf96a7565ffd3d86542 Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Tue, 14 May 2024 19:17:47 +0200
Subject: [PATCH 0759/1206] [clang-tidy] Ignore implicit casts with errors in
 bugprone-implicit-widening-of-multiplication-result (#92025)

When expression got errors (missing typedef) and clang-tidy is compiled
with asserts enabled, then we crash in this check on assert because type
with errors is visible as an dependent one. This is issue caused by
invalid input.

But as there is not point to crash in such case and generate additional
confusion, such expressions with errors will be now ignored.

Fixes #89515, #55293
---
 .../ImplicitWideningOfMultiplicationResultCheck.cpp   | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/bugprone/ImplicitWideningOfMultiplicationResultCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ImplicitWideningOfMultiplicationResultCheck.cpp
index 6f22f02f3018..f99beac668ce 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ImplicitWideningOfMultiplicationResultCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ImplicitWideningOfMultiplicationResultCheck.cpp
@@ -9,20 +9,20 @@
 #include "ImplicitWideningOfMultiplicationResultCheck.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchersMacros.h"
 #include "clang/Lex/Lexer.h"
 #include <optional>
 
 using namespace clang::ast_matchers;
 
-namespace clang {
+namespace clang::tidy::bugprone {
+
 namespace {
 AST_MATCHER(ImplicitCastExpr, isPartOfExplicitCast) {
   return Node.isPartOfExplicitCast();
 }
+AST_MATCHER(Expr, containsErrors) { return Node.containsErrors(); }
 } // namespace
-} // namespace clang
-
-namespace clang::tidy::bugprone {
 
 static const Expr *getLHSOfMulBinOp(const Expr *E) {
   assert(E == E->IgnoreParens() && "Already skipped all parens!");
@@ -250,7 +250,8 @@ void ImplicitWideningOfMultiplicationResultCheck::handlePointerOffsetting(
 
 void ImplicitWideningOfMultiplicationResultCheck::registerMatchers(
     MatchFinder *Finder) {
-  Finder->addMatcher(implicitCastExpr(unless(anyOf(isInTemplateInstantiation(),
+  Finder->addMatcher(implicitCastExpr(unless(anyOf(containsErrors(),
+                                                   isInTemplateInstantiation(),
                                                    isPartOfExplicitCast())),
                                       hasCastKind(CK_IntegralCast))
                          .bind("x"),
-- 
GitLab


From 97e35e0098e863bff959f726f1492654a6cfe441 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Tue, 14 May 2024 13:22:01 -0400
Subject: [PATCH 0760/1206] Revert "[Clang][Sema] Earlier type checking for
 builtin unary operators (#90500)" (#92149)

This reverts commit 8019cbbbbc94658d133583f7be6cd0023d30b0f3.
---
 clang/docs/ReleaseNotes.rst                   |   3 -
 clang/include/clang/AST/Type.h                |   5 +-
 clang/lib/Sema/SemaExpr.cpp                   | 351 +++++++++---------
 clang/test/AST/ast-dump-expr-json.cpp         |   4 +-
 clang/test/AST/ast-dump-expr.cpp              |   2 +-
 clang/test/AST/ast-dump-lambda.cpp            |   2 +-
 .../expr/expr.unary/expr.unary.general/p1.cpp |  65 ----
 clang/test/CXX/over/over.built/ast.cpp        | 158 ++------
 clang/test/CXX/over/over.built/p10.cpp        |   2 +-
 clang/test/CXX/over/over.built/p11.cpp        |   2 +-
 .../temp.res/temp.dep/temp.dep.type/p4.cpp    |  25 +-
 clang/test/Frontend/noderef_templates.cpp     |   4 +-
 clang/test/SemaCXX/cxx2b-deducing-this.cpp    |   6 +-
 .../test/SemaTemplate/class-template-spec.cpp |  12 +-
 .../ASTMatchers/ASTMatchersNarrowingTest.cpp  |   6 +-
 15 files changed, 245 insertions(+), 402 deletions(-)
 delete mode 100644 clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index a2e44efe4134..49ab222bec40 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -55,9 +55,6 @@ C++ Specific Potentially Breaking Changes
 
 - Clang now rejects pointer to member from parenthesized expression in unevaluated context such as ``decltype(&(foo::bar))``. (#GH40906).
 
-- Clang now performs semantic analysis for unary operators with dependent operands
-  that are known to be of non-class non-enumeration type prior to instantiation.
-
 ABI Changes in This Version
 ---------------------------
 - Fixed Microsoft name mangling of implicitly defined variables used for thread
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index da3834f19ca0..e6643469e0b3 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -8044,10 +8044,7 @@ inline bool Type::isUndeducedType() const {
 /// Determines whether this is a type for which one can define
 /// an overloaded operator.
 inline bool Type::isOverloadableType() const {
-  if (!CanonicalType->isDependentType())
-    return isRecordType() || isEnumeralType();
-  return !isArrayType() && !isFunctionType() && !isAnyPointerType() &&
-         !isMemberPointerType();
+  return isDependentType() || isRecordType() || isEnumeralType();
 }
 
 /// Determines whether this type is written as a typedef-name.
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 18fd5ba700ad..ec84798e4ce6 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -672,12 +672,12 @@ ExprResult Sema::DefaultLvalueConversion(Expr *E) {
 
   // We don't want to throw lvalue-to-rvalue casts on top of
   // expressions of certain types in C++.
-  if (getLangOpts().CPlusPlus) {
-    if (T == Context.OverloadTy || T->isRecordType() ||
-        (T->isDependentType() && !T->isAnyPointerType() &&
-         !T->isMemberPointerType()))
-      return E;
-  }
+  if (getLangOpts().CPlusPlus &&
+      (E->getType() == Context.OverloadTy ||
+       // FIXME: This is a hack! We want the lvalue-to-rvalue conversion applied
+       // to pointer types even if the pointee type is dependent.
+       (T->isDependentType() && !T->isPointerType()) || T->isRecordType()))
+    return E;
 
   // The C standard is actually really unclear on this point, and
   // DR106 tells us what the result should be but not why.  It's
@@ -10827,7 +10827,7 @@ static bool checkArithmeticIncompletePointerType(Sema &S, SourceLocation Loc,
   if (const AtomicType *ResAtomicType = ResType->getAs<AtomicType>())
     ResType = ResAtomicType->getValueType();
 
-  assert(ResType->isAnyPointerType());
+  assert(ResType->isAnyPointerType() && !ResType->isDependentType());
   QualType PointeeTy = ResType->getPointeeType();
   return S.RequireCompleteSizedType(
       Loc, PointeeTy,
@@ -13957,6 +13957,9 @@ static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
                                                ExprObjectKind &OK,
                                                SourceLocation OpLoc, bool IsInc,
                                                bool IsPrefix) {
+  if (Op->isTypeDependent())
+    return S.Context.DependentTy;
+
   QualType ResType = Op->getType();
   // Atomic types can be used for increment / decrement where the non-atomic
   // versions can, so ignore the _Atomic() specifier for the purpose of
@@ -14407,6 +14410,9 @@ static void RecordModifiableNonNullParam(Sema &S, const Expr *Exp) {
 static QualType CheckIndirectionOperand(Sema &S, Expr *Op, ExprValueKind &VK,
                                         SourceLocation OpLoc,
                                         bool IsAfterAmp = false) {
+  if (Op->isTypeDependent())
+    return S.Context.DependentTy;
+
   ExprResult ConvResult = S.UsualUnaryConversions(Op);
   if (ConvResult.isInvalid())
     return QualType();
@@ -15460,191 +15466,190 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
       return ExprError(Diag(OpLoc, diag::err_hlsl_operator_unsupported) << 1);
   }
 
-  if (InputExpr->isTypeDependent() &&
-      InputExpr->getType()->isSpecificBuiltinType(BuiltinType::Dependent)) {
-    resultType = Context.DependentTy;
-  } else {
-    switch (Opc) {
-    case UO_PreInc:
-    case UO_PreDec:
-    case UO_PostInc:
-    case UO_PostDec:
-      resultType =
-          CheckIncrementDecrementOperand(*this, Input.get(), VK, OK, OpLoc,
-                                         Opc == UO_PreInc || Opc == UO_PostInc,
-                                         Opc == UO_PreInc || Opc == UO_PreDec);
-      CanOverflow = isOverflowingIntegerType(Context, resultType);
+  switch (Opc) {
+  case UO_PreInc:
+  case UO_PreDec:
+  case UO_PostInc:
+  case UO_PostDec:
+    resultType =
+        CheckIncrementDecrementOperand(*this, Input.get(), VK, OK, OpLoc,
+                                       Opc == UO_PreInc || Opc == UO_PostInc,
+                                       Opc == UO_PreInc || Opc == UO_PreDec);
+    CanOverflow = isOverflowingIntegerType(Context, resultType);
+    break;
+  case UO_AddrOf:
+    resultType = CheckAddressOfOperand(Input, OpLoc);
+    CheckAddressOfNoDeref(InputExpr);
+    RecordModifiableNonNullParam(*this, InputExpr);
+    break;
+  case UO_Deref: {
+    Input = DefaultFunctionArrayLvalueConversion(Input.get());
+    if (Input.isInvalid())
+      return ExprError();
+    resultType =
+        CheckIndirectionOperand(*this, Input.get(), VK, OpLoc, IsAfterAmp);
+    break;
+  }
+  case UO_Plus:
+  case UO_Minus:
+    CanOverflow = Opc == UO_Minus &&
+                  isOverflowingIntegerType(Context, Input.get()->getType());
+    Input = UsualUnaryConversions(Input.get());
+    if (Input.isInvalid())
+      return ExprError();
+    // Unary plus and minus require promoting an operand of half vector to a
+    // float vector and truncating the result back to a half vector. For now, we
+    // do this only when HalfArgsAndReturns is set (that is, when the target is
+    // arm or arm64).
+    ConvertHalfVec = needsConversionOfHalfVec(true, Context, Input.get());
+
+    // If the operand is a half vector, promote it to a float vector.
+    if (ConvertHalfVec)
+      Input = convertVector(Input.get(), Context.FloatTy, *this);
+    resultType = Input.get()->getType();
+    if (resultType->isDependentType())
       break;
-    case UO_AddrOf:
-      resultType = CheckAddressOfOperand(Input, OpLoc);
-      CheckAddressOfNoDeref(InputExpr);
-      RecordModifiableNonNullParam(*this, InputExpr);
+    if (resultType->isArithmeticType()) // C99 6.5.3.3p1
       break;
-    case UO_Deref: {
-      Input = DefaultFunctionArrayLvalueConversion(Input.get());
-      if (Input.isInvalid())
-        return ExprError();
-      resultType =
-          CheckIndirectionOperand(*this, Input.get(), VK, OpLoc, IsAfterAmp);
+    else if (resultType->isVectorType() &&
+             // The z vector extensions don't allow + or - with bool vectors.
+             (!Context.getLangOpts().ZVector ||
+              resultType->castAs<VectorType>()->getVectorKind() !=
+                  VectorKind::AltiVecBool))
+      break;
+    else if (resultType->isSveVLSBuiltinType()) // SVE vectors allow + and -
+      break;
+    else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
+             Opc == UO_Plus && resultType->isPointerType())
       break;
+
+    return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                     << resultType << Input.get()->getSourceRange());
+
+  case UO_Not: // bitwise complement
+    Input = UsualUnaryConversions(Input.get());
+    if (Input.isInvalid())
+      return ExprError();
+    resultType = Input.get()->getType();
+    if (resultType->isDependentType())
+      break;
+    // C99 6.5.3.3p1. We allow complex int and float as a GCC extension.
+    if (resultType->isComplexType() || resultType->isComplexIntegerType())
+      // C99 does not support '~' for complex conjugation.
+      Diag(OpLoc, diag::ext_integer_complement_complex)
+          << resultType << Input.get()->getSourceRange();
+    else if (resultType->hasIntegerRepresentation())
+      break;
+    else if (resultType->isExtVectorType() && Context.getLangOpts().OpenCL) {
+      // OpenCL v1.1 s6.3.f: The bitwise operator not (~) does not operate
+      // on vector float types.
+      QualType T = resultType->castAs<ExtVectorType>()->getElementType();
+      if (!T->isIntegerType())
+        return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                         << resultType << Input.get()->getSourceRange());
+    } else {
+      return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                       << resultType << Input.get()->getSourceRange());
     }
-    case UO_Plus:
-    case UO_Minus:
-      CanOverflow = Opc == UO_Minus &&
-                    isOverflowingIntegerType(Context, Input.get()->getType());
-      Input = UsualUnaryConversions(Input.get());
-      if (Input.isInvalid())
-        return ExprError();
-      // Unary plus and minus require promoting an operand of half vector to a
-      // float vector and truncating the result back to a half vector. For now,
-      // we do this only when HalfArgsAndReturns is set (that is, when the
-      // target is arm or arm64).
-      ConvertHalfVec = needsConversionOfHalfVec(true, Context, Input.get());
-
-      // If the operand is a half vector, promote it to a float vector.
-      if (ConvertHalfVec)
-        Input = convertVector(Input.get(), Context.FloatTy, *this);
-      resultType = Input.get()->getType();
-      if (resultType->isArithmeticType()) // C99 6.5.3.3p1
-        break;
-      else if (resultType->isVectorType() &&
-               // The z vector extensions don't allow + or - with bool vectors.
-               (!Context.getLangOpts().ZVector ||
-                resultType->castAs<VectorType>()->getVectorKind() !=
-                    VectorKind::AltiVecBool))
-        break;
-      else if (resultType->isSveVLSBuiltinType()) // SVE vectors allow + and -
-        break;
-      else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
-               Opc == UO_Plus && resultType->isPointerType())
-        break;
+    break;
 
+  case UO_LNot: // logical negation
+    // Unlike +/-/~, integer promotions aren't done here (C99 6.5.3.3p5).
+    Input = DefaultFunctionArrayLvalueConversion(Input.get());
+    if (Input.isInvalid())
+      return ExprError();
+    resultType = Input.get()->getType();
+
+    // Though we still have to promote half FP to float...
+    if (resultType->isHalfType() && !Context.getLangOpts().NativeHalfType) {
+      Input = ImpCastExprToType(Input.get(), Context.FloatTy, CK_FloatingCast)
+                  .get();
+      resultType = Context.FloatTy;
+    }
+
+    // WebAsembly tables can't be used in unary expressions.
+    if (resultType->isPointerType() &&
+        resultType->getPointeeType().isWebAssemblyReferenceType()) {
       return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
                        << resultType << Input.get()->getSourceRange());
+    }
 
-    case UO_Not: // bitwise complement
-      Input = UsualUnaryConversions(Input.get());
-      if (Input.isInvalid())
-        return ExprError();
-      resultType = Input.get()->getType();
-      // C99 6.5.3.3p1. We allow complex int and float as a GCC extension.
-      if (resultType->isComplexType() || resultType->isComplexIntegerType())
-        // C99 does not support '~' for complex conjugation.
-        Diag(OpLoc, diag::ext_integer_complement_complex)
-            << resultType << Input.get()->getSourceRange();
-      else if (resultType->hasIntegerRepresentation())
-        break;
-      else if (resultType->isExtVectorType() && Context.getLangOpts().OpenCL) {
-        // OpenCL v1.1 s6.3.f: The bitwise operator not (~) does not operate
-        // on vector float types.
+    if (resultType->isDependentType())
+      break;
+    if (resultType->isScalarType() && !isScopedEnumerationType(resultType)) {
+      // C99 6.5.3.3p1: ok, fallthrough;
+      if (Context.getLangOpts().CPlusPlus) {
+        // C++03 [expr.unary.op]p8, C++0x [expr.unary.op]p9:
+        // operand contextually converted to bool.
+        Input = ImpCastExprToType(Input.get(), Context.BoolTy,
+                                  ScalarTypeToBooleanCastKind(resultType));
+      } else if (Context.getLangOpts().OpenCL &&
+                 Context.getLangOpts().OpenCLVersion < 120) {
+        // OpenCL v1.1 6.3.h: The logical operator not (!) does not
+        // operate on scalar float types.
+        if (!resultType->isIntegerType() && !resultType->isPointerType())
+          return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                           << resultType << Input.get()->getSourceRange());
+      }
+    } else if (resultType->isExtVectorType()) {
+      if (Context.getLangOpts().OpenCL &&
+          Context.getLangOpts().getOpenCLCompatibleVersion() < 120) {
+        // OpenCL v1.1 6.3.h: The logical operator not (!) does not
+        // operate on vector float types.
         QualType T = resultType->castAs<ExtVectorType>()->getElementType();
         if (!T->isIntegerType())
           return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
                            << resultType << Input.get()->getSourceRange());
-      } else {
-        return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                         << resultType << Input.get()->getSourceRange());
       }
+      // Vector logical not returns the signed variant of the operand type.
+      resultType = GetSignedVectorType(resultType);
       break;
-
-    case UO_LNot: // logical negation
-      // Unlike +/-/~, integer promotions aren't done here (C99 6.5.3.3p5).
-      Input = DefaultFunctionArrayLvalueConversion(Input.get());
-      if (Input.isInvalid())
-        return ExprError();
-      resultType = Input.get()->getType();
-
-      // Though we still have to promote half FP to float...
-      if (resultType->isHalfType() && !Context.getLangOpts().NativeHalfType) {
-        Input = ImpCastExprToType(Input.get(), Context.FloatTy, CK_FloatingCast)
-                    .get();
-        resultType = Context.FloatTy;
-      }
-
-      // WebAsembly tables can't be used in unary expressions.
-      if (resultType->isPointerType() &&
-          resultType->getPointeeType().isWebAssemblyReferenceType()) {
+    } else if (Context.getLangOpts().CPlusPlus && resultType->isVectorType()) {
+      const VectorType *VTy = resultType->castAs<VectorType>();
+      if (VTy->getVectorKind() != VectorKind::Generic)
         return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
                          << resultType << Input.get()->getSourceRange());
-      }
 
-      if (resultType->isScalarType() && !isScopedEnumerationType(resultType)) {
-        // C99 6.5.3.3p1: ok, fallthrough;
-        if (Context.getLangOpts().CPlusPlus) {
-          // C++03 [expr.unary.op]p8, C++0x [expr.unary.op]p9:
-          // operand contextually converted to bool.
-          Input = ImpCastExprToType(Input.get(), Context.BoolTy,
-                                    ScalarTypeToBooleanCastKind(resultType));
-        } else if (Context.getLangOpts().OpenCL &&
-                   Context.getLangOpts().OpenCLVersion < 120) {
-          // OpenCL v1.1 6.3.h: The logical operator not (!) does not
-          // operate on scalar float types.
-          if (!resultType->isIntegerType() && !resultType->isPointerType())
-            return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                             << resultType << Input.get()->getSourceRange());
-        }
-      } else if (resultType->isExtVectorType()) {
-        if (Context.getLangOpts().OpenCL &&
-            Context.getLangOpts().getOpenCLCompatibleVersion() < 120) {
-          // OpenCL v1.1 6.3.h: The logical operator not (!) does not
-          // operate on vector float types.
-          QualType T = resultType->castAs<ExtVectorType>()->getElementType();
-          if (!T->isIntegerType())
-            return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                             << resultType << Input.get()->getSourceRange());
-        }
-        // Vector logical not returns the signed variant of the operand type.
-        resultType = GetSignedVectorType(resultType);
-        break;
-      } else if (Context.getLangOpts().CPlusPlus &&
-                 resultType->isVectorType()) {
-        const VectorType *VTy = resultType->castAs<VectorType>();
-        if (VTy->getVectorKind() != VectorKind::Generic)
-          return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                           << resultType << Input.get()->getSourceRange());
-
-        // Vector logical not returns the signed variant of the operand type.
-        resultType = GetSignedVectorType(resultType);
-        break;
-      } else {
-        return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                         << resultType << Input.get()->getSourceRange());
-      }
-
-      // LNot always has type int. C99 6.5.3.3p5.
-      // In C++, it's bool. C++ 5.3.1p8
-      resultType = Context.getLogicalOperationType();
+      // Vector logical not returns the signed variant of the operand type.
+      resultType = GetSignedVectorType(resultType);
       break;
-    case UO_Real:
-    case UO_Imag:
-      resultType = CheckRealImagOperand(*this, Input, OpLoc, Opc == UO_Real);
-      // _Real maps ordinary l-values into ordinary l-values. _Imag maps
-      // ordinary complex l-values to ordinary l-values and all other values to
-      // r-values.
-      if (Input.isInvalid())
-        return ExprError();
-      if (Opc == UO_Real || Input.get()->getType()->isAnyComplexType()) {
-        if (Input.get()->isGLValue() &&
-            Input.get()->getObjectKind() == OK_Ordinary)
-          VK = Input.get()->getValueKind();
-      } else if (!getLangOpts().CPlusPlus) {
-        // In C, a volatile scalar is read by __imag. In C++, it is not.
-        Input = DefaultLvalueConversion(Input.get());
-      }
-      break;
-    case UO_Extension:
-      resultType = Input.get()->getType();
-      VK = Input.get()->getValueKind();
-      OK = Input.get()->getObjectKind();
-      break;
-    case UO_Coawait:
-      // It's unnecessary to represent the pass-through operator co_await in the
-      // AST; just return the input expression instead.
-      assert(!Input.get()->getType()->isDependentType() &&
-             "the co_await expression must be non-dependant before "
-             "building operator co_await");
-      return Input;
+    } else {
+      return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                       << resultType << Input.get()->getSourceRange());
+    }
+
+    // LNot always has type int. C99 6.5.3.3p5.
+    // In C++, it's bool. C++ 5.3.1p8
+    resultType = Context.getLogicalOperationType();
+    break;
+  case UO_Real:
+  case UO_Imag:
+    resultType = CheckRealImagOperand(*this, Input, OpLoc, Opc == UO_Real);
+    // _Real maps ordinary l-values into ordinary l-values. _Imag maps ordinary
+    // complex l-values to ordinary l-values and all other values to r-values.
+    if (Input.isInvalid())
+      return ExprError();
+    if (Opc == UO_Real || Input.get()->getType()->isAnyComplexType()) {
+      if (Input.get()->isGLValue() &&
+          Input.get()->getObjectKind() == OK_Ordinary)
+        VK = Input.get()->getValueKind();
+    } else if (!getLangOpts().CPlusPlus) {
+      // In C, a volatile scalar is read by __imag. In C++, it is not.
+      Input = DefaultLvalueConversion(Input.get());
     }
+    break;
+  case UO_Extension:
+    resultType = Input.get()->getType();
+    VK = Input.get()->getValueKind();
+    OK = Input.get()->getObjectKind();
+    break;
+  case UO_Coawait:
+    // It's unnecessary to represent the pass-through operator co_await in the
+    // AST; just return the input expression instead.
+    assert(!Input.get()->getType()->isDependentType() &&
+           "the co_await expression must be non-dependant before "
+           "building operator co_await");
+    return Input;
   }
   if (resultType.isNull() || Input.isInvalid())
     return ExprError();
diff --git a/clang/test/AST/ast-dump-expr-json.cpp b/clang/test/AST/ast-dump-expr-json.cpp
index 4b7365e554cb..0fb07b0b434c 100644
--- a/clang/test/AST/ast-dump-expr-json.cpp
+++ b/clang/test/AST/ast-dump-expr-json.cpp
@@ -4261,9 +4261,9 @@ void TestNonADLCall3() {
 // CHECK-NEXT:                     }
 // CHECK-NEXT:                    },
 // CHECK-NEXT:                    "type": {
-// CHECK-NEXT:                     "qualType": "V"
+// CHECK-NEXT:                     "qualType": "<dependent type>"
 // CHECK-NEXT:                    },
-// CHECK-NEXT:                    "valueCategory": "lvalue",
+// CHECK-NEXT:                    "valueCategory": "prvalue",
 // CHECK-NEXT:                    "isPostfix": false,
 // CHECK-NEXT:                    "opcode": "*",
 // CHECK-NEXT:                    "canOverflow": false,
diff --git a/clang/test/AST/ast-dump-expr.cpp b/clang/test/AST/ast-dump-expr.cpp
index 4df5ba4276ab..69e65e22d61d 100644
--- a/clang/test/AST/ast-dump-expr.cpp
+++ b/clang/test/AST/ast-dump-expr.cpp
@@ -282,7 +282,7 @@ void PrimaryExpressions(Ts... a) {
       // CHECK-NEXT: CompoundStmt
       // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <col:8> col:8 implicit 'V'
       // CHECK-NEXT: ParenListExpr 0x{{[^ ]*}} <col:8> 'NULL TYPE'
-      // CHECK-NEXT: UnaryOperator 0x{{[^ ]*}} <col:8> 'V' lvalue prefix '*' cannot overflow
+      // CHECK-NEXT: UnaryOperator 0x{{[^ ]*}} <col:8> '<dependent type>' prefix '*' cannot overflow
       // CHECK-NEXT: CXXThisExpr 0x{{[^ ]*}} <col:8> 'V *' this
     }
   };
diff --git a/clang/test/AST/ast-dump-lambda.cpp b/clang/test/AST/ast-dump-lambda.cpp
index a4d3fe4fbda5..ef8789cd97d3 100644
--- a/clang/test/AST/ast-dump-lambda.cpp
+++ b/clang/test/AST/ast-dump-lambda.cpp
@@ -81,7 +81,7 @@ template <typename... Ts> void test(Ts... a) {
 // CHECK-NEXT:    |         | | `-CompoundStmt {{.*}} <col:15, col:16>
 // CHECK-NEXT:    |         | `-FieldDecl {{.*}} <col:8> col:8{{( imported)?}} implicit 'V'
 // CHECK-NEXT:    |         |-ParenListExpr {{.*}} <col:8> 'NULL TYPE'
-// CHECK-NEXT:    |         | `-UnaryOperator {{.*}} <col:8> 'V' lvalue prefix '*' cannot overflow
+// CHECK-NEXT:    |         | `-UnaryOperator {{.*}} <col:8> '<dependent type>' prefix '*' cannot overflow
 // CHECK-NEXT:    |         |   `-CXXThisExpr {{.*}} <col:8> 'V *' this
 // CHECK-NEXT:    |         `-CompoundStmt {{.*}} <col:15, col:16>
 // CHECK-NEXT:    |-DeclStmt {{.*}} <line:22:3, col:11>
diff --git a/clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp b/clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp
deleted file mode 100644
index 6744ce1cad17..000000000000
--- a/clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-// RUN: %clang_cc1 -Wno-unused -fsyntax-only %s -verify
-
-struct A {
-  void operator*();
-  void operator+();
-  void operator-();
-  void operator!();
-  void operator~();
-  void operator&();
-  void operator++();
-  void operator--();
-};
-
-struct B { };
-
-template<typename T, typename U>
-void dependent(T t, T* pt, T U::* mpt, T(&ft)(), T(&at)[4]) {
-  *t;
-  +t;
-  -t;
-  !t;
-  ~t;
-  &t;
-  ++t;
-  --t;
-
-  *pt;
-  +pt;
-  -pt; // expected-error {{invalid argument type 'T *' to unary expression}}
-  !pt;
-  ~pt; // expected-error {{invalid argument type 'T *' to unary expression}}
-  &pt;
-  ++pt;
-  --pt;
-
-  *mpt; // expected-error {{indirection requires pointer operand ('T U::*' invalid)}}
-  +mpt; // expected-error {{invalid argument type 'T U::*' to unary expression}}
-  -mpt; // expected-error {{invalid argument type 'T U::*' to unary expression}}
-  !mpt;
-  ~mpt; // expected-error {{invalid argument type 'T U::*' to unary expression}}
-  &mpt;
-  ++mpt; // expected-error {{cannot increment value of type 'T U::*'}}
-  --mpt; // expected-error {{cannot decrement value of type 'T U::*'}}
-
-  *ft;
-  +ft;
-  -ft; // expected-error {{invalid argument type 'T (*)()' to unary expression}}
-  !ft;
-  ~ft; // expected-error {{invalid argument type 'T (*)()' to unary expression}}
-  &ft;
-  ++ft; // expected-error {{cannot increment value of type 'T ()'}}
-  --ft; // expected-error {{cannot decrement value of type 'T ()'}}
-
-  *at;
-  +at;
-  -at; // expected-error {{invalid argument type 'T *' to unary expression}}
-  !at;
-  ~at; // expected-error {{invalid argument type 'T *' to unary expression}}
-  &at;
-  ++at; // expected-error {{cannot increment value of type 'T[4]'}}
-  --at; // expected-error {{cannot decrement value of type 'T[4]'}}
-}
-
-// Make sure we only emit diagnostics once.
-template void dependent(A t, A* pt, A B::* mpt, A(&ft)(), A(&at)[4]);
diff --git a/clang/test/CXX/over/over.built/ast.cpp b/clang/test/CXX/over/over.built/ast.cpp
index 78f86edb1e96..56a63431269f 100644
--- a/clang/test/CXX/over/over.built/ast.cpp
+++ b/clang/test/CXX/over/over.built/ast.cpp
@@ -1,139 +1,41 @@
-// RUN: %clang_cc1 -std=c++17 -Wno-unused -ast-dump %s -ast-dump-filter Test | FileCheck %s
+// RUN: %clang_cc1 -std=c++17 -ast-dump %s -ast-dump-filter Test | FileCheck %s
 
-namespace Test {
-  template<typename T, typename U>
-  void Unary(T t, T* pt, T U::* mpt, T(&ft)(), T(&at)[4]) {
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '*' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
-    *t;
+struct A{};
 
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '+' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
-    +t;
+template <typename T, typename U>
+auto Test(T* pt, U* pu) {
+  // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '*'
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+  (void)*pt;
 
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '-' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
-    -t;
+  // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '++'
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+  (void)(++pt);
 
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '!' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
-    !t;
+  // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '+'
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+  (void)(+pt);
 
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '~' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
-    ~t;
+  // CHECK: BinaryOperator {{.*}} '<dependent type>' '+'
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+  // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 3
+  (void)(pt + 3);
 
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
-    &t;
+  // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+  (void)(pt - pt);
 
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '++' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
-    ++t;
+  // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
+  (void)(pt - pu);
 
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '--' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
-    --t;
+  // CHECK: BinaryOperator {{.*}} '<dependent type>' '=='
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
+  (void)(pt == pu);
 
-    // CHECK: UnaryOperator {{.*}} 'T' lvalue prefix '*' cannot overflow
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <LValueToRValue>
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-    *pt;
+}
 
-    // CHECK: UnaryOperator {{.*}} 'T *' prefix '+' cannot overflow
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <LValueToRValue>
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-    +pt;
 
-    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <PointerToBoolean>
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <LValueToRValue>
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-    !pt;
-
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-    &pt;
-
-    // CHECK: UnaryOperator {{.*}} 'T *' lvalue prefix '++' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-    ++pt;
-
-    // CHECK: UnaryOperator {{.*}} 'T *' lvalue prefix '--' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-    --pt;
-
-    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <MemberPointerToBoolean>
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T U::*' <LValueToRValue>
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T U::*' lvalue ParmVar {{.*}} 'mpt' 'T U::*'
-    !mpt;
-
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T U::*' lvalue ParmVar {{.*}} 'mpt' 'T U::*'
-    &mpt;
-
-    // CHECK: UnaryOperator {{.*}} 'T ()' lvalue prefix '*' cannot overflow
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T (*)()' <FunctionToPointerDecay>
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
-    *ft;
-
-    // CHECK: UnaryOperator {{.*}} 'T (*)()' prefix '+' cannot overflow
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T (*)()' <FunctionToPointerDecay>
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
-    +ft;
-
-    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <PointerToBoolean>
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T (*)()' <FunctionToPointerDecay>
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
-    !ft;
-
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
-    &ft;
-
-    // CHECK: UnaryOperator {{.*}} 'T' lvalue prefix '*' cannot overflow
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <ArrayToPointerDecay>
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
-    *at;
-
-    // CHECK: UnaryOperator {{.*}} 'T *' prefix '+' cannot overflow
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <ArrayToPointerDecay>
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
-    +at;
-
-    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <PointerToBoolean>
-    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <ArrayToPointerDecay>
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
-    !at;
-
-    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
-    &at;
-  }
-
-  template<typename T, typename U>
-  void Binary(T* pt, U* pu) {
-    // CHECK: BinaryOperator {{.*}} '<dependent type>' '+'
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-    // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 3
-    pt + 3;
-
-    // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-    pt - pt;
-
-    // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
-    pt - pu;
-
-    // CHECK: BinaryOperator {{.*}} '<dependent type>' '=='
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-    // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
-    pt == pu;
-  }
-} // namespace Test
diff --git a/clang/test/CXX/over/over.built/p10.cpp b/clang/test/CXX/over/over.built/p10.cpp
index 8ff2396d0b6f..678056da5820 100644
--- a/clang/test/CXX/over/over.built/p10.cpp
+++ b/clang/test/CXX/over/over.built/p10.cpp
@@ -15,6 +15,6 @@ void f(int i, float f, bool b, char c, int* pi, A* pa, T* pt) {
 
   (void)-pi; // expected-error {{invalid argument type}}
   (void)-pa; // expected-error {{invalid argument type}}
-  (void)-pt; // expected-error {{invalid argument type}}
+  (void)-pt; // FIXME: we should be able to give an error here.
 }
 
diff --git a/clang/test/CXX/over/over.built/p11.cpp b/clang/test/CXX/over/over.built/p11.cpp
index f7a741db726d..7ebf16b95439 100644
--- a/clang/test/CXX/over/over.built/p11.cpp
+++ b/clang/test/CXX/over/over.built/p11.cpp
@@ -7,6 +7,6 @@ void f(int i, float f, bool b, char c, int* pi, T* pt) {
   (void)~b;
   (void)~c;
   (void)~pi; // expected-error {{invalid argument type}}
-  (void)~pt; // expected-error {{invalid argument type}}
+  (void)~pt; // FIXME: we should be able to give an error here.
 }
 
diff --git a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
index 982e5372f5b0..3ca7c6c7eb8e 100644
--- a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
@@ -357,14 +357,17 @@ namespace N0 {
       a->A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
       a->B::A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
 
-      (*this).x4; // expected-error{{no member named 'x4' in 'B<T>'}}
-      (*this).B::x4; // expected-error{{no member named 'x4' in 'B<T>'}}
-      (*this).A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
-      (*this).B::A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
-      (*this).f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
-      (*this).B::f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
-      (*this).A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
-      (*this).B::A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
+      // FIXME: An overloaded unary 'operator*' is built for these
+      // even though the operand is a pointer (to a dependent type).
+      // Type::isOverloadableType should return false for such cases.
+      (*this).x4;
+      (*this).B::x4;
+      (*this).A::x4;
+      (*this).B::A::x4;
+      (*this).f4();
+      (*this).B::f4();
+      (*this).A::f4();
+      (*this).B::A::f4();
 
       b.x4; // expected-error{{no member named 'x4' in 'B<T>'}}
       b.B::x4; // expected-error{{no member named 'x4' in 'B<T>'}}
@@ -396,13 +399,15 @@ namespace N1 {
       f<0>();
       this->f<0>();
       a->f<0>();
-      (*this).f<0>();
+      // FIXME: This should not require 'template'!
+      (*this).f<0>(); // expected-error{{missing 'template' keyword prior to dependent template name 'f'}}
       b.f<0>();
 
       x.f<0>();
       this->x.f<0>();
       a->x.f<0>();
-      (*this).x.f<0>();
+      // FIXME: This should not require 'template'!
+      (*this).x.f<0>(); // expected-error{{missing 'template' keyword prior to dependent template name 'f'}}
       b.x.f<0>();
 
       // FIXME: None of these should require 'template'!
diff --git a/clang/test/Frontend/noderef_templates.cpp b/clang/test/Frontend/noderef_templates.cpp
index 9e54cd5d7889..5fde6efd87c7 100644
--- a/clang/test/Frontend/noderef_templates.cpp
+++ b/clang/test/Frontend/noderef_templates.cpp
@@ -3,8 +3,8 @@
 #define NODEREF __attribute__((noderef))
 
 template <typename T>
-int func(T NODEREF *a) { // expected-note 3 {{a declared here}}
-  return *a + 1;         // expected-warning 3 {{dereferencing a; was declared with a 'noderef' type}}
+int func(T NODEREF *a) { // expected-note 2 {{a declared here}}
+  return *a + 1;         // expected-warning 2 {{dereferencing a; was declared with a 'noderef' type}}
 }
 
 void func() {
diff --git a/clang/test/SemaCXX/cxx2b-deducing-this.cpp b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
index aa64530bd5be..5f29a955e053 100644
--- a/clang/test/SemaCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
@@ -19,7 +19,7 @@ struct S {
     // new and delete are implicitly static
     void *operator new(this unsigned long); // expected-error{{an explicit object parameter cannot appear in a static function}}
     void operator delete(this void*); // expected-error{{an explicit object parameter cannot appear in a static function}}
-
+    
     void g(this auto) const; // expected-error{{explicit object member function cannot have 'const' qualifier}}
     void h(this auto) &; // expected-error{{explicit object member function cannot have '&' qualifier}}
     void i(this auto) &&; // expected-error{{explicit object member function cannot have '&&' qualifier}}
@@ -198,7 +198,9 @@ void func(int i) {
 void TestMutationInLambda() {
     [i = 0](this auto &&){ i++; }();
     [i = 0](this auto){ i++; }();
-    [i = 0](this const auto&){ i++; }(); // expected-error {{cannot assign to a variable captured by copy in a non-mutable lambda}}
+    [i = 0](this const auto&){ i++; }();
+    // expected-error@-1 {{cannot assign to a variable captured by copy in a non-mutable lambda}}
+    // expected-note@-2 {{in instantiation of}}
 
     int x;
     const auto l1 = [x](this auto&) { x = 42; }; // expected-error {{cannot assign to a variable captured by copy in a non-mutable lambda}}
diff --git a/clang/test/SemaTemplate/class-template-spec.cpp b/clang/test/SemaTemplate/class-template-spec.cpp
index faa54c367538..56b8207bd9a4 100644
--- a/clang/test/SemaTemplate/class-template-spec.cpp
+++ b/clang/test/SemaTemplate/class-template-spec.cpp
@@ -18,7 +18,7 @@ int test_specs(A<float, float> *a1, A<float, int> *a2) {
   return a1->x + a2->y;
 }
 
-int test_incomplete_specs(A<double, double> *a1,
+int test_incomplete_specs(A<double, double> *a1, 
                           A<double> *a2)
 {
   (void)a1->x; // expected-error{{member access into incomplete type}}
@@ -39,7 +39,7 @@ template <> struct X<int, int> { int foo(); }; // #1
 template <> struct X<float> { int bar(); };  // #2
 
 typedef int int_type;
-void testme(X<int_type> *x1, X<float, int> *x2) {
+void testme(X<int_type> *x1, X<float, int> *x2) { 
   (void)x1->foo(); // okay: refers to #1
   (void)x2->bar(); // okay: refers to #2
 }
@@ -53,7 +53,7 @@ struct A<char> {
 A<char>::A() { }
 
 // Make sure we can see specializations defined before the primary template.
-namespace N{
+namespace N{ 
   template<typename T> struct A0;
 }
 
@@ -97,7 +97,7 @@ namespace M {
   template<> struct ::A<long double>; // expected-error{{must occur at global scope}}
 }
 
-template<> struct N::B<char> {
+template<> struct N::B<char> { 
   int testf(int x) { return f(x); }
 };
 
@@ -138,9 +138,9 @@ namespace PR18009 {
 
   template <typename T> struct C {
     template <int N, int M> struct S;
-    template <int N> struct S<N, N ? **(T(*)[N])0 : 0> {}; // ok
+    template <int N> struct S<N, N ? **(T(*)[N])0 : 0> {}; // expected-error {{depends on a template parameter of the partial specialization}}
   };
-  C<int> c;
+  C<int> c; // expected-note {{in instantiation of}}
 
   template<int A> struct outer {
     template<int B, int C> struct inner {};
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index f26140675fd4..c08deb903f12 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -1572,9 +1572,9 @@ TEST_P(ASTMatchersTest, IsArrow_MatchesMemberVariablesViaArrow) {
   EXPECT_TRUE(
       matches("template <class T> class Y { void x() { this->m; } int m; };",
               memberExpr(isArrow())));
-  EXPECT_TRUE(notMatches(
-      "template <class T> class Y { void x() { (*this).m; } int m; };",
-      memberExpr(isArrow())));
+  EXPECT_TRUE(
+      notMatches("template <class T> class Y { void x() { (*this).m; } };",
+                 cxxDependentScopeMemberExpr(isArrow())));
 }
 
 TEST_P(ASTMatchersTest, IsArrow_MatchesStaticMemberVariablesViaArrow) {
-- 
GitLab


From 86f655cb4e2f4134d48219a2959a10c90e3396cb Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 14 May 2024 18:27:37 +0100
Subject: [PATCH 0761/1206] [LAA] Add tests showing unnecessary RT check due to
 applying loop guards

Test courtesy to @bjope showing a regression due to
ecae3ed958481cba7d60868cf3504292f7f4fdf5.
---
 .../is-safe-dep-distance-with-loop-guards.ll  | 140 ++++++++++++++++++
 1 file changed, 140 insertions(+)
 create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/is-safe-dep-distance-with-loop-guards.ll

diff --git a/llvm/test/Analysis/LoopAccessAnalysis/is-safe-dep-distance-with-loop-guards.ll b/llvm/test/Analysis/LoopAccessAnalysis/is-safe-dep-distance-with-loop-guards.ll
new file mode 100644
index 000000000000..bfa735df064d
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/is-safe-dep-distance-with-loop-guards.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck %s
+
+target datalayout = "S16-p:16:16-i1:16-i8:8-i32:16-i64:16-i128:16"
+
+define void @safe_deps_1_due_to_dependence_distance(i16  %n, ptr %p) {
+; CHECK-LABEL: 'safe_deps_1_due_to_dependence_distance'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP1:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.iv = getelementptr inbounds i32, ptr %p, i16 %iv
+; CHECK-NEXT:        Against group ([[GRP2:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %gep.off.iv = getelementptr i32, ptr %gep.off, i16 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP1]]:
+; CHECK-NEXT:          (Low: %p High: ((4 * %n) + %p))
+; CHECK-NEXT:            Member: {%p,+,4}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP2]]:
+; CHECK-NEXT:          (Low: ((4 * %n) + %p) High: ((8 * %n) + %p))
+; CHECK-NEXT:            Member: {((4 * %n) + %p),+,4}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %cmp42 = icmp sgt i16 %n, 0
+  br i1 %cmp42, label %ph, label %exit
+
+ph:
+  %gep.off = getelementptr i32, ptr %p, i16 %n
+  br label %loop
+
+loop:
+  %iv = phi i16 [ 0, %ph ], [ %iv.next, %loop ]
+  %gep.iv = getelementptr inbounds i32, ptr %p, i16 %iv
+  store i32 0, ptr %gep.iv, align 1
+  %gep.off.iv = getelementptr i32, ptr %gep.off, i16 %iv
+  store i32 1, ptr %gep.off.iv, align 1
+  %iv.next = add i16 %iv, 1
+  %exitcond.not = icmp eq i16 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @safe_deps_2_due_to_dependence_distance(i16 %n, ptr %p3, i16 noundef %q, ptr %p1, ptr %p2) {
+; CHECK-LABEL: 'safe_deps_2_due_to_dependence_distance'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group ([[GRP3:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %arrayidx22 = getelementptr inbounds [2 x i32], ptr %alloca, i16 %iv
+; CHECK-NEXT:        Against group ([[GRP4:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %arrayidx33 = getelementptr inbounds i8, ptr %arrayidx22, i16 4
+; CHECK-NEXT:      Check 1:
+; CHECK-NEXT:        Comparing group ([[GRP3]]):
+; CHECK-NEXT:          %arrayidx22 = getelementptr inbounds [2 x i32], ptr %alloca, i16 %iv
+; CHECK-NEXT:        Against group ([[GRP5:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %arrayidx42 = getelementptr inbounds [2 x i32], ptr %arrayidx40, i16 %iv
+; CHECK-NEXT:      Check 2:
+; CHECK-NEXT:        Comparing group ([[GRP3]]):
+; CHECK-NEXT:          %arrayidx22 = getelementptr inbounds [2 x i32], ptr %alloca, i16 %iv
+; CHECK-NEXT:        Against group ([[GRP6:0x[0-9a-f]+]]):
+; CHECK-NEXT:          %arrayidx53 = getelementptr inbounds i8, ptr %arrayidx42, i16 4
+; CHECK-NEXT:      Check 3:
+; CHECK-NEXT:        Comparing group ([[GRP4]]):
+; CHECK-NEXT:          %arrayidx33 = getelementptr inbounds i8, ptr %arrayidx22, i16 4
+; CHECK-NEXT:        Against group ([[GRP5]]):
+; CHECK-NEXT:          %arrayidx42 = getelementptr inbounds [2 x i32], ptr %arrayidx40, i16 %iv
+; CHECK-NEXT:      Check 4:
+; CHECK-NEXT:        Comparing group ([[GRP4]]):
+; CHECK-NEXT:          %arrayidx33 = getelementptr inbounds i8, ptr %arrayidx22, i16 4
+; CHECK-NEXT:        Against group ([[GRP6]]):
+; CHECK-NEXT:          %arrayidx53 = getelementptr inbounds i8, ptr %arrayidx42, i16 4
+; CHECK-NEXT:      Check 5:
+; CHECK-NEXT:        Comparing group ([[GRP5]]):
+; CHECK-NEXT:          %arrayidx42 = getelementptr inbounds [2 x i32], ptr %arrayidx40, i16 %iv
+; CHECK-NEXT:        Against group ([[GRP6]]):
+; CHECK-NEXT:          %arrayidx53 = getelementptr inbounds i8, ptr %arrayidx42, i16 4
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group [[GRP3]]:
+; CHECK-NEXT:          (Low: %alloca High: (-4 + (8 * %n) + %alloca))
+; CHECK-NEXT:            Member: {%alloca,+,8}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP4]]:
+; CHECK-NEXT:          (Low: (4 + %alloca) High: ((8 * %n) + %alloca))
+; CHECK-NEXT:            Member: {(4 + %alloca),+,8}<nuw><%loop>
+; CHECK-NEXT:        Group [[GRP5]]:
+; CHECK-NEXT:          (Low: ((8 * %n) + %alloca) High: (-4 + (16 * %n) + %alloca))
+; CHECK-NEXT:            Member: {((8 * %n) + %alloca),+,8}<nw><%loop>
+; CHECK-NEXT:        Group [[GRP6]]:
+; CHECK-NEXT:          (Low: (4 + (8 * %n) + %alloca) High: ((16 * %n) + %alloca))
+; CHECK-NEXT:            Member: {(4 + (8 * %n) + %alloca),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-NEXT:      {((8 * %n) + %alloca),+,8}<nw><%loop> Added Flags: <nusw>
+; CHECK-NEXT:      {(4 + (8 * %n) + %alloca),+,8}<nw><%loop> Added Flags: <nusw>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %0 = shl i16 %n, 1
+  %alloca = alloca [2 x i32], i16 %0
+  %arrayidx1 = getelementptr inbounds i32, ptr %p1, i16 %q
+  %arrayidx2 = getelementptr inbounds i8, ptr %p3, i16 2
+  %arrayidx4 = getelementptr inbounds i32, ptr %p2, i16 %q
+  %cmp42 = icmp sgt i16 %n, 0
+  br i1 %cmp42, label %ph, label %exit
+
+ph:
+  %arrayidx40 = getelementptr inbounds [2 x i32], ptr %alloca, i16 %n
+  br label %loop
+
+loop:
+  %iv = phi i16 [ 0, %ph ], [ %iv.next, %loop ]
+  %arrayidx6 = getelementptr inbounds i32, ptr %arrayidx1, i16 %iv
+  %arrayidx11 = getelementptr inbounds i32, ptr %arrayidx4, i16 %iv
+  %arrayidx22 = getelementptr inbounds [2 x i32], ptr %alloca, i16 %iv
+  store i32 10, ptr %arrayidx22
+  %arrayidx33 = getelementptr inbounds i8, ptr %arrayidx22, i16 4
+  store i32 16, ptr %arrayidx33
+  %arrayidx42 = getelementptr inbounds [2 x i32], ptr %arrayidx40, i16 %iv
+  store i32 19, ptr %arrayidx42
+  %arrayidx53 = getelementptr inbounds i8, ptr %arrayidx42, i16 4
+  store i32 23, ptr %arrayidx53
+  %iv.next = add nuw nsw i16 %iv, 1
+  %exitcond.not = icmp eq i16 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
-- 
GitLab


From 39d123f58a0e3c5f1a928940244b8dfd827fd4e5 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Tue, 14 May 2024 10:39:05 -0700
Subject: [PATCH 0762/1206] [nfc] const-ify `IntOrString::equalsLower` (#92152)

---
 llvm/tools/llvm-rc/ResourceScriptStmt.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/tools/llvm-rc/ResourceScriptStmt.h b/llvm/tools/llvm-rc/ResourceScriptStmt.h
index 05865e582859..0d8ec3e5b787 100644
--- a/llvm/tools/llvm-rc/ResourceScriptStmt.h
+++ b/llvm/tools/llvm-rc/ResourceScriptStmt.h
@@ -145,7 +145,7 @@ public:
   IntOrString(const RCToken &Token)
       : Data(Token), IsInt(Token.kind() == RCToken::Kind::Int) {}
 
-  bool equalsLower(const char *Str) {
+  bool equalsLower(const char *Str) const {
     return !IsInt && Data.String.equals_insensitive(Str);
   }
 
-- 
GitLab


From de14b749fee41d4ded711e771e43043ae3100cb3 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Tue, 14 May 2024 13:44:25 +0100
Subject: [PATCH 0763/1206] [RISCV][test] Precommit tests for byte store of -1

Although we can't reduce the number of instructions, if we selected `li
rd, -1` instead then this could be encoded in a 16-bit instruction.
---
 llvm/test/CodeGen/RISCV/imm.ll | 74 +++++++++++++++++++++++++++++-----
 1 file changed, 64 insertions(+), 10 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/imm.ll b/llvm/test/CodeGen/RISCV/imm.ll
index c5c1657b526a..344ed0d2b083 100644
--- a/llvm/test/CodeGen/RISCV/imm.ll
+++ b/llvm/test/CodeGen/RISCV/imm.ll
@@ -1558,6 +1558,60 @@ define i64 @imm_2reg_1() nounwind {
   ret i64 -1152921504301427080 ; 0xF000_0000_1234_5678
 }
 
+; TODO: Selecting -1 would be better in this case as it can be loaded with a
+; 16 bit instruction when the compressed extension is enabled.
+define void @imm_store_i8_neg1(ptr %p) nounwind {
+; RV32I-LABEL: imm_store_i8_neg1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 255
+; RV32I-NEXT:    sb a1, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: imm_store_i8_neg1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 255
+; RV64I-NEXT:    sb a1, 0(a0)
+; RV64I-NEXT:    ret
+;
+; RV64IZBA-LABEL: imm_store_i8_neg1:
+; RV64IZBA:       # %bb.0:
+; RV64IZBA-NEXT:    li a1, 255
+; RV64IZBA-NEXT:    sb a1, 0(a0)
+; RV64IZBA-NEXT:    ret
+;
+; RV64IZBB-LABEL: imm_store_i8_neg1:
+; RV64IZBB:       # %bb.0:
+; RV64IZBB-NEXT:    li a1, 255
+; RV64IZBB-NEXT:    sb a1, 0(a0)
+; RV64IZBB-NEXT:    ret
+;
+; RV64IZBS-LABEL: imm_store_i8_neg1:
+; RV64IZBS:       # %bb.0:
+; RV64IZBS-NEXT:    li a1, 255
+; RV64IZBS-NEXT:    sb a1, 0(a0)
+; RV64IZBS-NEXT:    ret
+;
+; RV64IXTHEADBB-LABEL: imm_store_i8_neg1:
+; RV64IXTHEADBB:       # %bb.0:
+; RV64IXTHEADBB-NEXT:    li a1, 255
+; RV64IXTHEADBB-NEXT:    sb a1, 0(a0)
+; RV64IXTHEADBB-NEXT:    ret
+;
+; RV32-REMAT-LABEL: imm_store_i8_neg1:
+; RV32-REMAT:       # %bb.0:
+; RV32-REMAT-NEXT:    li a1, 255
+; RV32-REMAT-NEXT:    sb a1, 0(a0)
+; RV32-REMAT-NEXT:    ret
+;
+; RV64-REMAT-LABEL: imm_store_i8_neg1:
+; RV64-REMAT:       # %bb.0:
+; RV64-REMAT-NEXT:    li a1, 255
+; RV64-REMAT-NEXT:    sb a1, 0(a0)
+; RV64-REMAT-NEXT:    ret
+  store i8 -1, ptr %p
+  ret void
+}
+
 define void @imm_store_i16_neg1(ptr %p) nounwind {
 ; RV32I-LABEL: imm_store_i16_neg1:
 ; RV32I:       # %bb.0:
@@ -2121,8 +2175,8 @@ define i64 @imm_70370820078523() {
 ;
 ; RV64I-POOL-LABEL: imm_70370820078523:
 ; RV64I-POOL:       # %bb.0:
-; RV64I-POOL-NEXT:    lui a0, %hi(.LCPI37_0)
-; RV64I-POOL-NEXT:    ld a0, %lo(.LCPI37_0)(a0)
+; RV64I-POOL-NEXT:    lui a0, %hi(.LCPI38_0)
+; RV64I-POOL-NEXT:    ld a0, %lo(.LCPI38_0)(a0)
 ; RV64I-POOL-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: imm_70370820078523:
@@ -2266,8 +2320,8 @@ define i64 @imm_neg_9223301666034697285() {
 ;
 ; RV64I-POOL-LABEL: imm_neg_9223301666034697285:
 ; RV64I-POOL:       # %bb.0:
-; RV64I-POOL-NEXT:    lui a0, %hi(.LCPI39_0)
-; RV64I-POOL-NEXT:    ld a0, %lo(.LCPI39_0)(a0)
+; RV64I-POOL-NEXT:    lui a0, %hi(.LCPI40_0)
+; RV64I-POOL-NEXT:    ld a0, %lo(.LCPI40_0)(a0)
 ; RV64I-POOL-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: imm_neg_9223301666034697285:
@@ -2544,8 +2598,8 @@ define i64 @imm_neg_9223354442718100411() {
 ;
 ; RV64I-POOL-LABEL: imm_neg_9223354442718100411:
 ; RV64I-POOL:       # %bb.0:
-; RV64I-POOL-NEXT:    lui a0, %hi(.LCPI43_0)
-; RV64I-POOL-NEXT:    ld a0, %lo(.LCPI43_0)(a0)
+; RV64I-POOL-NEXT:    lui a0, %hi(.LCPI44_0)
+; RV64I-POOL-NEXT:    ld a0, %lo(.LCPI44_0)(a0)
 ; RV64I-POOL-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: imm_neg_9223354442718100411:
@@ -3855,8 +3909,8 @@ define i64 @imm64_same_lo_hi_optsize() nounwind optsize {
 ;
 ; RV64I-POOL-LABEL: imm64_same_lo_hi_optsize:
 ; RV64I-POOL:       # %bb.0:
-; RV64I-POOL-NEXT:    lui a0, %hi(.LCPI64_0)
-; RV64I-POOL-NEXT:    ld a0, %lo(.LCPI64_0)(a0)
+; RV64I-POOL-NEXT:    lui a0, %hi(.LCPI65_0)
+; RV64I-POOL-NEXT:    ld a0, %lo(.LCPI65_0)(a0)
 ; RV64I-POOL-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: imm64_same_lo_hi_optsize:
@@ -3930,8 +3984,8 @@ define i64 @imm64_same_lo_hi_negative() nounwind {
 ;
 ; RV64I-POOL-LABEL: imm64_same_lo_hi_negative:
 ; RV64I-POOL:       # %bb.0:
-; RV64I-POOL-NEXT:    lui a0, %hi(.LCPI65_0)
-; RV64I-POOL-NEXT:    ld a0, %lo(.LCPI65_0)(a0)
+; RV64I-POOL-NEXT:    lui a0, %hi(.LCPI66_0)
+; RV64I-POOL-NEXT:    ld a0, %lo(.LCPI66_0)(a0)
 ; RV64I-POOL-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: imm64_same_lo_hi_negative:
-- 
GitLab


From 90109d444839683b09f0aafdc50b749cb4b3203b Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Tue, 14 May 2024 19:08:04 +0100
Subject: [PATCH 0764/1206] [RISCV] Improve constant materialisation for stores
 of i8 negative constants (#92131)

This follows the same pattern as 20e62658735a1b03ecadc. Although we
can't reduce the number of instructions used, if we are able to use a
sign-extended 6-bit immediate then the 16-bit c.li instruction can be
selected (thus saving code size). Although this _could_ be gated so it
only happens if C is enabled, I've opted not to because at worst it's
neutral and it doesn't seem helpful to add unnecessary divergence
between the RVC and non-RVC paths.
---
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp    |  5 +++++
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h      |  1 +
 llvm/test/CodeGen/RISCV/imm.ll                 | 18 ++++++++----------
 .../RISCV/rvv/fixed-vectors-fp-shuffles.ll     |  2 +-
 .../RISCV/rvv/fixed-vectors-int-buildvec.ll    |  2 +-
 .../RISCV/rvv/fixed-vectors-int-shuffles.ll    |  6 +++---
 .../rvv/fixed-vectors-reduction-formation.ll   |  8 ++++----
 .../test/CodeGen/RISCV/unaligned-load-store.ll |  2 +-
 8 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 3c4646b95715..d965dd4fc9a9 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -902,6 +902,11 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
       return;
     }
     int64_t Imm = ConstNode->getSExtValue();
+    // If only the lower 8 bits are used, try to convert this to a simm6 by
+    // sign-extending bit 7. This is neutral without the C extension, and
+    // allows C.LI to be used if C is present.
+    if (isUInt<8>(Imm) && isInt<6>(SignExtend64<8>(Imm)) && hasAllBUsers(Node))
+      Imm = SignExtend64<8>(Imm);
     // If the upper XLen-16 bits are not used, try to convert this to a simm12
     // by sign extending bit 15.
     if (isUInt<16>(Imm) && isInt<12>(SignExtend64<16>(Imm)) &&
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index 7d4aec2dfdc9..ece04dd7f4b7 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -121,6 +121,7 @@ public:
 
   bool hasAllNBitUsers(SDNode *Node, unsigned Bits,
                        const unsigned Depth = 0) const;
+  bool hasAllBUsers(SDNode *Node) const { return hasAllNBitUsers(Node, 8); }
   bool hasAllHUsers(SDNode *Node) const { return hasAllNBitUsers(Node, 16); }
   bool hasAllWUsers(SDNode *Node) const { return hasAllNBitUsers(Node, 32); }
 
diff --git a/llvm/test/CodeGen/RISCV/imm.ll b/llvm/test/CodeGen/RISCV/imm.ll
index 344ed0d2b083..5fd25ab60db0 100644
--- a/llvm/test/CodeGen/RISCV/imm.ll
+++ b/llvm/test/CodeGen/RISCV/imm.ll
@@ -1558,54 +1558,52 @@ define i64 @imm_2reg_1() nounwind {
   ret i64 -1152921504301427080 ; 0xF000_0000_1234_5678
 }
 
-; TODO: Selecting -1 would be better in this case as it can be loaded with a
-; 16 bit instruction when the compressed extension is enabled.
 define void @imm_store_i8_neg1(ptr %p) nounwind {
 ; RV32I-LABEL: imm_store_i8_neg1:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    li a1, 255
+; RV32I-NEXT:    li a1, -1
 ; RV32I-NEXT:    sb a1, 0(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm_store_i8_neg1:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a1, 255
+; RV64I-NEXT:    li a1, -1
 ; RV64I-NEXT:    sb a1, 0(a0)
 ; RV64I-NEXT:    ret
 ;
 ; RV64IZBA-LABEL: imm_store_i8_neg1:
 ; RV64IZBA:       # %bb.0:
-; RV64IZBA-NEXT:    li a1, 255
+; RV64IZBA-NEXT:    li a1, -1
 ; RV64IZBA-NEXT:    sb a1, 0(a0)
 ; RV64IZBA-NEXT:    ret
 ;
 ; RV64IZBB-LABEL: imm_store_i8_neg1:
 ; RV64IZBB:       # %bb.0:
-; RV64IZBB-NEXT:    li a1, 255
+; RV64IZBB-NEXT:    li a1, -1
 ; RV64IZBB-NEXT:    sb a1, 0(a0)
 ; RV64IZBB-NEXT:    ret
 ;
 ; RV64IZBS-LABEL: imm_store_i8_neg1:
 ; RV64IZBS:       # %bb.0:
-; RV64IZBS-NEXT:    li a1, 255
+; RV64IZBS-NEXT:    li a1, -1
 ; RV64IZBS-NEXT:    sb a1, 0(a0)
 ; RV64IZBS-NEXT:    ret
 ;
 ; RV64IXTHEADBB-LABEL: imm_store_i8_neg1:
 ; RV64IXTHEADBB:       # %bb.0:
-; RV64IXTHEADBB-NEXT:    li a1, 255
+; RV64IXTHEADBB-NEXT:    li a1, -1
 ; RV64IXTHEADBB-NEXT:    sb a1, 0(a0)
 ; RV64IXTHEADBB-NEXT:    ret
 ;
 ; RV32-REMAT-LABEL: imm_store_i8_neg1:
 ; RV32-REMAT:       # %bb.0:
-; RV32-REMAT-NEXT:    li a1, 255
+; RV32-REMAT-NEXT:    li a1, -1
 ; RV32-REMAT-NEXT:    sb a1, 0(a0)
 ; RV32-REMAT-NEXT:    ret
 ;
 ; RV64-REMAT-LABEL: imm_store_i8_neg1:
 ; RV64-REMAT:       # %bb.0:
-; RV64-REMAT-NEXT:    li a1, 255
+; RV64-REMAT-NEXT:    li a1, -1
 ; RV64-REMAT-NEXT:    sb a1, 0(a0)
 ; RV64-REMAT-NEXT:    ret
   store i8 -1, ptr %p
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
index b0f6bebea038..8dc32d13e4a3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
@@ -17,7 +17,7 @@ define <4 x half> @shuffle_v4f16(<4 x half> %x, <4 x half> %y) {
 define <8 x float> @shuffle_v8f32(<8 x float> %x, <8 x float> %y) {
 ; CHECK-LABEL: shuffle_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 236
+; CHECK-NEXT:    li a0, -20
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a0
 ; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
index ed6c01aaf7fe..592ce6fc5be0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
@@ -260,7 +260,7 @@ define <4 x i8> @buildvec_vid_stepn3_add3_v4i8() {
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v9, 3
 ; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    li a0, 253
+; CHECK-NEXT:    li a0, -3
 ; CHECK-NEXT:    vmadd.vx v8, a0, v9
 ; CHECK-NEXT:    ret
   ret <4 x i8> <i8 3, i8 0, i8 -3, i8 -6>
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
index 58af6ac246d1..aba69dc84620 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
@@ -611,7 +611,7 @@ define <8 x i8> @concat_4xi8_start_undef(<8 x i8> %v, <8 x i8> %w) {
 define <8 x i8> @concat_4xi8_start_undef_at_start(<8 x i8> %v, <8 x i8> %w) {
 ; CHECK-LABEL: concat_4xi8_start_undef_at_start:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 224
+; CHECK-NEXT:    li a0, -32
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv.s.x v0, a0
 ; CHECK-NEXT:    vslideup.vi v8, v9, 4, v0.t
@@ -682,7 +682,7 @@ define <8 x i8> @merge_non_contiguous_slideup_slidedown(<8 x i8> %v, <8 x i8> %w
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 2
-; CHECK-NEXT:    li a0, 234
+; CHECK-NEXT:    li a0, -22
 ; CHECK-NEXT:    vmv.s.x v0, a0
 ; CHECK-NEXT:    vslideup.vi v8, v9, 1, v0.t
 ; CHECK-NEXT:    ret
@@ -699,7 +699,7 @@ define <8 x i8> @unmergable(<8 x i8> %v, <8 x i8> %w) {
 ; CHECK-NEXT:    lui a0, %hi(.LCPI46_0)
 ; CHECK-NEXT:    addi a0, a0, %lo(.LCPI46_0)
 ; CHECK-NEXT:    vle8.v v10, (a0)
-; CHECK-NEXT:    li a0, 234
+; CHECK-NEXT:    li a0, -22
 ; CHECK-NEXT:    vmv.s.x v0, a0
 ; CHECK-NEXT:    vrgather.vv v8, v9, v10, v0.t
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-formation.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-formation.ll
index 5f456c782431..03624113a826 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-formation.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-formation.ll
@@ -160,7 +160,7 @@ define i32 @reduce_sum_16xi32_prefix4(ptr %p) {
 define i32 @reduce_sum_16xi32_prefix5(ptr %p) {
 ; CHECK-LABEL: reduce_sum_16xi32_prefix5:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 224
+; CHECK-NEXT:    li a1, -32
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a1
 ; CHECK-NEXT:    vmv.v.i v8, -1
@@ -532,7 +532,7 @@ define i32 @reduce_xor_16xi32_prefix2(ptr %p) {
 define i32 @reduce_xor_16xi32_prefix5(ptr %p) {
 ; CHECK-LABEL: reduce_xor_16xi32_prefix5:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 224
+; CHECK-NEXT:    li a1, -32
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a1
 ; CHECK-NEXT:    vmv.v.i v8, -1
@@ -620,7 +620,7 @@ define i32 @reduce_or_16xi32_prefix2(ptr %p) {
 define i32 @reduce_or_16xi32_prefix5(ptr %p) {
 ; CHECK-LABEL: reduce_or_16xi32_prefix5:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 224
+; CHECK-NEXT:    li a1, -32
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a1
 ; CHECK-NEXT:    vmv.v.i v8, -1
@@ -757,7 +757,7 @@ define i32 @reduce_umax_16xi32_prefix2(ptr %p) {
 define i32 @reduce_umax_16xi32_prefix5(ptr %p) {
 ; CHECK-LABEL: reduce_umax_16xi32_prefix5:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 224
+; CHECK-NEXT:    li a1, -32
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a1
 ; CHECK-NEXT:    vmv.v.i v8, -1
diff --git a/llvm/test/CodeGen/RISCV/unaligned-load-store.ll b/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
index ce0d8fedbfb8..10497db6edc4 100644
--- a/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
@@ -419,7 +419,7 @@ define void @merge_stores_i32_i64(ptr %p) {
 define void @store_large_constant(ptr %x) {
 ; SLOW-LABEL: store_large_constant:
 ; SLOW:       # %bb.0:
-; SLOW-NEXT:    li a1, 254
+; SLOW-NEXT:    li a1, -2
 ; SLOW-NEXT:    sb a1, 7(a0)
 ; SLOW-NEXT:    li a1, 220
 ; SLOW-NEXT:    sb a1, 6(a0)
-- 
GitLab


From 67d840b60fbd75ca1b52d77bd3353771ec853735 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 14 May 2024 19:10:48 +0100
Subject: [PATCH 0765/1206] [VPlan] Relax over-aggressive assertion in
 VPTransformState::get().

There are cases where a vector value has some users that demand the
the single scalar value only (NeedsScalar), while other users demand the
vector value (see attached test cases). In those cases, the NeedsScalar
users should only demand the first lane.

Fixes https://github.com/llvm/llvm-project/issues/91883.
---
 llvm/lib/Transforms/Vectorize/VPlan.cpp       |   2 +-
 ...ned-value-used-as-scalar-and-first-lane.ll | 226 ++++++++++++++++++
 2 files changed, 227 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 999236ae8489..27f8e239b1c0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -246,7 +246,7 @@ Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) {
 
 Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
   if (NeedsScalar) {
-    assert((VF.isScalar() || Def->isLiveIn() ||
+    assert((VF.isScalar() || Def->isLiveIn() || hasVectorValue(Def, Part) ||
             (hasScalarValue(Def, VPIteration(Part, 0)) &&
              Data.PerPartScalars[Def][Part].size() == 1)) &&
            "Trying to access a single scalar per part but has multiple scalars "
diff --git a/llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll b/llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll
new file mode 100644
index 000000000000..b22801ba9208
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll
@@ -0,0 +1,226 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S -passes=loop-vectorize -mcpu=skylake-avx512 -mtriple=x86_64-apple-macosx -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+
+; Test cases based on https://github.com/llvm/llvm-project/issues/91883.
+define void @iv.4_used_as_vector_and_first_lane(ptr %src, ptr noalias %dst) {
+; CHECK-LABEL: define void @iv.4_used_as_vector_and_first_lane(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[STEP_ADD1:%.*]] = add <4 x i64> [[STEP_ADD]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[STEP_ADD2:%.*]] = add <4 x i64> [[STEP_ADD1]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 4
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 8
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 12
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i64>, ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i64>, ptr [[TMP11]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[TMP13:%.*]] = add <4 x i64> [[STEP_ADD]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[TMP14:%.*]] = add <4 x i64> [[STEP_ADD1]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[TMP15:%.*]] = add <4 x i64> [[STEP_ADD2]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD]], <i64 128, i64 128, i64 128, i64 128>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD4]], <i64 128, i64 128, i64 128, i64 128>
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD5]], <i64 128, i64 128, i64 128, i64 128>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD6]], <i64 128, i64 128, i64 128, i64 128>
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP20]], 1
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i64> [[TMP13]], i32 0
+; CHECK-NEXT:    [[TMP23:%.*]] = add i64 [[TMP22]], 1
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP24]], 1
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x i64> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP27:%.*]] = add i64 [[TMP26]], 1
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP23]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP25]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP27]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i64, ptr [[TMP28]], i32 0
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i64, ptr [[TMP28]], i32 4
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr i64, ptr [[TMP28]], i32 8
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr i64, ptr [[TMP28]], i32 12
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP12]], ptr [[TMP32]], i32 4, <4 x i1> [[TMP16]])
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP13]], ptr [[TMP33]], i32 4, <4 x i1> [[TMP17]])
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP14]], ptr [[TMP34]], i32 4, <4 x i1> [[TMP18]])
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP15]], ptr [[TMP35]], i32 4, <4 x i1> [[TMP19]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD2]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
+; CHECK-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 32, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[G_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[G_SRC]], align 8
+; CHECK-NEXT:    [[IV_4:%.*]] = add nuw nsw i64 [[IV]], 4
+; CHECK-NEXT:    [[C:%.*]] = icmp ule i64 [[L]], 128
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_THEN:%.*]], label [[LOOP_LATCH]]
+; CHECK:       loop.then:
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i64 [[IV_4]], 1
+; CHECK-NEXT:    [[G_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[OR]]
+; CHECK-NEXT:    store i64 [[IV_4]], ptr [[G_DST]], align 4
+; CHECK-NEXT:    br label [[LOOP_LATCH]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 32
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %g.src = getelementptr inbounds i64, ptr %src, i64 %iv
+  %l = load i64, ptr %g.src
+  %iv.4 = add nuw nsw i64 %iv, 4
+  %c = icmp ule i64 %l, 128
+  br i1 %c, label %loop.then, label %loop.latch
+
+loop.then:
+  %or = or disjoint i64 %iv.4, 1
+  %g.dst = getelementptr inbounds i64, ptr %dst, i64 %or
+  store i64 %iv.4, ptr %g.dst, align 4
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 32
+  br i1 %exitcond, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @iv.4_used_as_first_lane(ptr %src, ptr noalias %dst) {
+; CHECK-LABEL: define void @iv.4_used_as_first_lane(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 4
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 8
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 12
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i64>, ptr [[TMP11]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD]], <i64 128, i64 128, i64 128, i64 128>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD1]], <i64 128, i64 128, i64 128, i64 128>
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD2]], <i64 128, i64 128, i64 128, i64 128>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD3]], <i64 128, i64 128, i64 128, i64 128>
+; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[TMP12]], 1
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP13]], 1
+; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[TMP14]], 1
+; CHECK-NEXT:    [[TMP23:%.*]] = add i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP23]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[TMP24]], i32 0
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i64, ptr [[TMP24]], i32 4
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i64, ptr [[TMP24]], i32 8
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i64, ptr [[TMP24]], i32 12
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_LOAD]], ptr [[TMP28]], i32 4, <4 x i1> [[TMP16]])
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_LOAD1]], ptr [[TMP29]], i32 4, <4 x i1> [[TMP17]])
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_LOAD2]], ptr [[TMP30]], i32 4, <4 x i1> [[TMP18]])
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_LOAD3]], ptr [[TMP31]], i32 4, <4 x i1> [[TMP19]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
+; CHECK-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 32, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[G_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[G_SRC]], align 8
+; CHECK-NEXT:    [[IV_4:%.*]] = add nuw nsw i64 [[IV]], 4
+; CHECK-NEXT:    [[C:%.*]] = icmp ule i64 [[L]], 128
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_THEN:%.*]], label [[LOOP_LATCH]]
+; CHECK:       loop.then:
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint i64 [[IV_4]], 1
+; CHECK-NEXT:    [[G_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[OR]]
+; CHECK-NEXT:    store i64 [[L]], ptr [[G_DST]], align 4
+; CHECK-NEXT:    br label [[LOOP_LATCH]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 32
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %g.src = getelementptr inbounds i64, ptr %src, i64 %iv
+  %l = load i64, ptr %g.src
+  %iv.4 = add nuw nsw i64 %iv, 4
+  %c = icmp ule i64 %l, 128
+  br i1 %c, label %loop.then, label %loop.latch
+
+loop.then:
+  %or = or disjoint i64 %iv.4, 1
+  %g.dst = getelementptr inbounds i64, ptr %dst, i64 %or
+  store i64 %l, ptr %g.dst, align 4
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 32
+  br i1 %exitcond, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+;.
-- 
GitLab


From f89b1b8a68065c4b880417abb0563bce21399b52 Mon Sep 17 00:00:00 2001
From: Oleg Shyshkov <shyshkov@google.com>
Date: Tue, 14 May 2024 20:14:13 +0200
Subject: [PATCH 0766/1206] [mlir][bazel] Fix bazel build.

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index a3171287a84b..751cd94d5ff1 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -2102,6 +2102,7 @@ cc_library(
         ":ArmSMEOpInterfacesIncGen",
         ":IR",
         ":Support",
+        ":VectorDialect",
         "//llvm:Support",
     ],
 )
-- 
GitLab


From 19008d32182ebbe421aaa222ee8af5c3e134e550 Mon Sep 17 00:00:00 2001
From: PiJoules <6019989+PiJoules@users.noreply.github.com>
Date: Tue, 14 May 2024 11:23:45 -0700
Subject: [PATCH 0767/1206] [llvm] Support fixed point multiplication on
 AArch64 (#84237)

Prior to this, fixed point multiplication would lead to this assertion
error on AArhc64, armv8, and armv7.

```
 _Accum f(_Accum x, _Accum y) { return x * y; }

// ./bin/clang++ -ffixed-point /tmp/test2.cc -c -S -o - -target aarch64 -O3
clang++: llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp:10245: void llvm::TargetLowering::forceExpandWideMUL(SelectionDAG &, const SDLoc &, bool, EVT, const SDValue, const SDValue, const SDValue, const SDValue, SDValue &, SDValue &) const: Assertion `Ret.getOpcode() == ISD::MERGE_VALUES && "Ret value is a collection of constituent nodes holding result."' failed.
```

This path into forceExpandWideMUL should only be taken if we don't
support [US]MUL_LOHI or MULH[US] for the operand size (32 in this case).
But we should also check if we can just leverage regular wide
multiplication. That is, extend the operands from 32 to 64, do a regular
64-bit mul, then trunc and shift. These ops are certainly available on
aarch64 but for wider types.
---
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |  12 +
 llvm/test/CodeGen/AArch64/smul_fix.ll         | 139 +++++++++
 llvm/test/CodeGen/AArch64/smul_fix_sat.ll     | 271 ++++++++++++++++++
 llvm/test/CodeGen/AArch64/umul_fix.ll         | 147 ++++++++++
 llvm/test/CodeGen/AArch64/umul_fix_sat.ll     | 206 +++++++++++++
 5 files changed, 775 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/smul_fix.ll
 create mode 100644 llvm/test/CodeGen/AArch64/smul_fix_sat.ll
 create mode 100644 llvm/test/CodeGen/AArch64/umul_fix.ll
 create mode 100644 llvm/test/CodeGen/AArch64/umul_fix_sat.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 7beaeb9b7a17..9ddb14e11dab 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -10500,6 +10500,7 @@ TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
   SDValue Lo, Hi;
   unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
   unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
+  EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
   if (isOperationLegalOrCustom(LoHiOp, VT)) {
     SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);
     Lo = Result.getValue(0);
@@ -10507,6 +10508,17 @@ TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
   } else if (isOperationLegalOrCustom(HiOp, VT)) {
     Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
     Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS);
+  } else if (isOperationLegalOrCustom(ISD::MUL, WideVT)) {
+    // Try for a multiplication using a wider type.
+    unsigned Ext = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
+    SDValue LHSExt = DAG.getNode(Ext, dl, WideVT, LHS);
+    SDValue RHSExt = DAG.getNode(Ext, dl, WideVT, RHS);
+    SDValue Res = DAG.getNode(ISD::MUL, dl, WideVT, LHSExt, RHSExt);
+    Lo = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
+    SDValue Shifted =
+        DAG.getNode(ISD::SRA, dl, WideVT, Res,
+                    DAG.getShiftAmountConstant(VTSize, WideVT, dl));
+    Hi = DAG.getNode(ISD::TRUNCATE, dl, VT, Shifted);
   } else if (VT.isVector()) {
     return SDValue();
   } else {
diff --git a/llvm/test/CodeGen/AArch64/smul_fix.ll b/llvm/test/CodeGen/AArch64/smul_fix.ll
new file mode 100644
index 000000000000..7bd80a00ec81
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/smul_fix.ll
@@ -0,0 +1,139 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s
+
+define i32 @func(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull x8, w0, w1
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    extr w0, w9, w8, #2
+; CHECK-NEXT:    ret
+  %tmp = call i32 @llvm.smul.fix.i32(i32 %x, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i64 @func2(i64 %x, i64 %y) {
+; CHECK-LABEL: func2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x8, x0, x1
+; CHECK-NEXT:    smulh x9, x0, x1
+; CHECK-NEXT:    extr x0, x9, x8, #2
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 2)
+  ret i64 %tmp
+}
+
+define i4 @func3(i4 %x, i4 %y) nounwind {
+; CHECK-LABEL: func3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sbfx w8, w1, #0, #4
+; CHECK-NEXT:    sbfx w9, w0, #0, #4
+; CHECK-NEXT:    smull x8, w9, w8
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    extr w0, w9, w8, #2
+; CHECK-NEXT:    ret
+  %tmp = call i4 @llvm.smul.fix.i4(i4 %x, i4 %y, i32 2)
+  ret i4 %tmp
+}
+
+;; These result in regular integer multiplication
+define i32 @func4(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w0, w0, w1
+; CHECK-NEXT:    ret
+  %tmp = call i32 @llvm.smul.fix.i32(i32 %x, i32 %y, i32 0)
+  ret i32 %tmp
+}
+
+define i64 @func5(i64 %x, i64 %y) {
+; CHECK-LABEL: func5:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x0, x0, x1
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 0)
+  ret i64 %tmp
+}
+
+define i4 @func6(i4 %x, i4 %y) nounwind {
+; CHECK-LABEL: func6:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sbfx w8, w1, #0, #4
+; CHECK-NEXT:    sbfx w9, w0, #0, #4
+; CHECK-NEXT:    mul w0, w9, w8
+; CHECK-NEXT:    ret
+  %tmp = call i4 @llvm.smul.fix.i4(i4 %x, i4 %y, i32 0)
+  ret i4 %tmp
+}
+
+define i64 @func7(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func7:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x8, x0, x1
+; CHECK-NEXT:    smulh x9, x0, x1
+; CHECK-NEXT:    extr x0, x9, x8, #32
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 32)
+  ret i64 %tmp
+}
+
+define i64 @func8(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x8, x0, x1
+; CHECK-NEXT:    smulh x9, x0, x1
+; CHECK-NEXT:    extr x0, x9, x8, #63
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 63)
+  ret i64 %tmp
+}
+
+define <2 x i32> @vec(<2 x i32> %x, <2 x i32> %y) nounwind {
+; CHECK-LABEL: vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    ret
+  %tmp = call <2 x i32> @llvm.smul.fix.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
+  ret <2 x i32> %tmp
+}
+
+define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
+; CHECK-LABEL: vec2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %tmp = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0)
+  ret <4 x i32> %tmp
+}
+
+define <4 x i64> @vec3(<4 x i64> %x, <4 x i64> %y) nounwind {
+; CHECK-LABEL: vec3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v2.d[1]
+; CHECK-NEXT:    mov x9, v0.d[1]
+; CHECK-NEXT:    fmov x10, d2
+; CHECK-NEXT:    fmov x11, d0
+; CHECK-NEXT:    mov x14, v3.d[1]
+; CHECK-NEXT:    mov x15, v1.d[1]
+; CHECK-NEXT:    mul x12, x11, x10
+; CHECK-NEXT:    mul x13, x9, x8
+; CHECK-NEXT:    smulh x8, x9, x8
+; CHECK-NEXT:    smulh x9, x11, x10
+; CHECK-NEXT:    fmov x10, d3
+; CHECK-NEXT:    fmov x11, d1
+; CHECK-NEXT:    mul x16, x11, x10
+; CHECK-NEXT:    extr x8, x8, x13, #32
+; CHECK-NEXT:    smulh x10, x11, x10
+; CHECK-NEXT:    extr x9, x9, x12, #32
+; CHECK-NEXT:    mul x11, x15, x14
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    smulh x14, x15, x14
+; CHECK-NEXT:    extr x10, x10, x16, #32
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    fmov d1, x10
+; CHECK-NEXT:    extr x11, x14, x11, #32
+; CHECK-NEXT:    mov v1.d[1], x11
+; CHECK-NEXT:    ret
+  %tmp = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> %x, <4 x i64> %y, i32 32)
+  ret <4 x i64> %tmp
+}
diff --git a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
new file mode 100644
index 000000000000..c2d8d34b9305
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
@@ -0,0 +1,271 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s
+
+define i32 @func(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull x9, w0, w1
+; CHECK-NEXT:    mov w8, #2147483647 // =0x7fffffff
+; CHECK-NEXT:    lsr x10, x9, #32
+; CHECK-NEXT:    extr w9, w10, w9, #2
+; CHECK-NEXT:    cmp w10, #1
+; CHECK-NEXT:    csel w8, w8, w9, gt
+; CHECK-NEXT:    cmn w10, #2
+; CHECK-NEXT:    mov w9, #-2147483648 // =0x80000000
+; CHECK-NEXT:    csel w0, w9, w8, lt
+; CHECK-NEXT:    ret
+  %tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x9, x0, x1
+; CHECK-NEXT:    mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-NEXT:    smulh x10, x0, x1
+; CHECK-NEXT:    extr x9, x10, x9, #2
+; CHECK-NEXT:    cmp x10, #1
+; CHECK-NEXT:    csel x8, x8, x9, gt
+; CHECK-NEXT:    cmn x10, #2
+; CHECK-NEXT:    mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-NEXT:    csel x0, x9, x8, lt
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 2)
+  ret i64 %tmp
+}
+
+define i4 @func3(i4 %x, i4 %y) nounwind {
+; CHECK-LABEL: func3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sbfx w9, w1, #0, #4
+; CHECK-NEXT:    lsl w10, w0, #28
+; CHECK-NEXT:    mov w8, #2147483647 // =0x7fffffff
+; CHECK-NEXT:    smull x9, w10, w9
+; CHECK-NEXT:    lsr x10, x9, #32
+; CHECK-NEXT:    extr w9, w10, w9, #2
+; CHECK-NEXT:    cmp w10, #1
+; CHECK-NEXT:    csel w8, w8, w9, gt
+; CHECK-NEXT:    cmn w10, #2
+; CHECK-NEXT:    mov w9, #-2147483648 // =0x80000000
+; CHECK-NEXT:    csel w8, w9, w8, lt
+; CHECK-NEXT:    asr w0, w8, #28
+; CHECK-NEXT:    ret
+  %tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 2)
+  ret i4 %tmp
+}
+
+;; These result in regular integer multiplication with a saturation check.
+define i32 @func4(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull x9, w0, w1
+; CHECK-NEXT:    eor w10, w0, w1
+; CHECK-NEXT:    mov w8, #-2147483648 // =0x80000000
+; CHECK-NEXT:    cmp w10, #0
+; CHECK-NEXT:    cinv w8, w8, ge
+; CHECK-NEXT:    cmp x9, w9, sxtw
+; CHECK-NEXT:    csel w0, w8, w9, ne
+; CHECK-NEXT:    ret
+  %tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 0)
+  ret i32 %tmp
+}
+
+define i64 @func5(i64 %x, i64 %y) {
+; CHECK-LABEL: func5:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x9, x0, x1
+; CHECK-NEXT:    eor x11, x0, x1
+; CHECK-NEXT:    mov x8, #-9223372036854775808 // =0x8000000000000000
+; CHECK-NEXT:    cmp x11, #0
+; CHECK-NEXT:    smulh x10, x0, x1
+; CHECK-NEXT:    cinv x8, x8, ge
+; CHECK-NEXT:    cmp x10, x9, asr #63
+; CHECK-NEXT:    csel x0, x8, x9, ne
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 0)
+  ret i64 %tmp
+}
+
+define i4 @func6(i4 %x, i4 %y) nounwind {
+; CHECK-LABEL: func6:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sbfx w9, w1, #0, #4
+; CHECK-NEXT:    lsl w10, w0, #28
+; CHECK-NEXT:    mov w8, #-2147483648 // =0x80000000
+; CHECK-NEXT:    smull x11, w10, w9
+; CHECK-NEXT:    eor w9, w10, w9
+; CHECK-NEXT:    cmp w9, #0
+; CHECK-NEXT:    cinv w8, w8, ge
+; CHECK-NEXT:    cmp x11, w11, sxtw
+; CHECK-NEXT:    csel w8, w8, w11, ne
+; CHECK-NEXT:    asr w0, w8, #28
+; CHECK-NEXT:    ret
+  %tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 0)
+  ret i4 %tmp
+}
+
+define i64 @func7(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func7:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x9, x0, x1
+; CHECK-NEXT:    mov w8, #2147483647 // =0x7fffffff
+; CHECK-NEXT:    mov x11, #-2147483648 // =0xffffffff80000000
+; CHECK-NEXT:    smulh x10, x0, x1
+; CHECK-NEXT:    extr x9, x10, x9, #32
+; CHECK-NEXT:    cmp x10, x8
+; CHECK-NEXT:    mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-NEXT:    csel x8, x8, x9, gt
+; CHECK-NEXT:    cmp x10, x11
+; CHECK-NEXT:    mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-NEXT:    csel x0, x9, x8, lt
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 32)
+  ret i64 %tmp
+}
+
+define i64 @func8(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x9, x0, x1
+; CHECK-NEXT:    mov x8, #4611686018427387903 // =0x3fffffffffffffff
+; CHECK-NEXT:    mov x11, #-4611686018427387904 // =0xc000000000000000
+; CHECK-NEXT:    smulh x10, x0, x1
+; CHECK-NEXT:    extr x9, x10, x9, #63
+; CHECK-NEXT:    cmp x10, x8
+; CHECK-NEXT:    mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-NEXT:    csel x8, x8, x9, gt
+; CHECK-NEXT:    cmp x10, x11
+; CHECK-NEXT:    mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-NEXT:    csel x0, x9, x8, lt
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 63)
+  ret i64 %tmp
+}
+
+define <2 x i32> @vec(<2 x i32> %x, <2 x i32> %y) nounwind {
+; CHECK-LABEL: vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov w9, v1.s[1]
+; CHECK-NEXT:    mov w10, v0.s[1]
+; CHECK-NEXT:    mov w8, #-2147483648 // =0x80000000
+; CHECK-NEXT:    fmov w12, s0
+; CHECK-NEXT:    smull x11, w10, w9
+; CHECK-NEXT:    eor w9, w10, w9
+; CHECK-NEXT:    fmov w10, s1
+; CHECK-NEXT:    cmp w9, #0
+; CHECK-NEXT:    smull x9, w12, w10
+; CHECK-NEXT:    eor w10, w12, w10
+; CHECK-NEXT:    cinv w12, w8, ge
+; CHECK-NEXT:    cmp x11, w11, sxtw
+; CHECK-NEXT:    csel w11, w12, w11, ne
+; CHECK-NEXT:    cmp w10, #0
+; CHECK-NEXT:    cinv w8, w8, ge
+; CHECK-NEXT:    cmp x9, w9, sxtw
+; CHECK-NEXT:    csel w8, w8, w9, ne
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    mov v0.s[1], w11
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+  %tmp = call <2 x i32> @llvm.smul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
+  ret <2 x i32> %tmp
+}
+
+define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
+; CHECK-LABEL: vec2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, v1.s[1]
+; CHECK-NEXT:    mov w10, v0.s[1]
+; CHECK-NEXT:    mov w8, #-2147483648 // =0x80000000
+; CHECK-NEXT:    fmov w12, s1
+; CHECK-NEXT:    fmov w13, s0
+; CHECK-NEXT:    mov w14, v0.s[2]
+; CHECK-NEXT:    eor w11, w10, w9
+; CHECK-NEXT:    smull x9, w10, w9
+; CHECK-NEXT:    mov w10, v1.s[2]
+; CHECK-NEXT:    cmp w11, #0
+; CHECK-NEXT:    smull x11, w13, w12
+; CHECK-NEXT:    eor w12, w13, w12
+; CHECK-NEXT:    cinv w13, w8, ge
+; CHECK-NEXT:    cmp x9, w9, sxtw
+; CHECK-NEXT:    csel w9, w13, w9, ne
+; CHECK-NEXT:    cmp w12, #0
+; CHECK-NEXT:    mov w13, v1.s[3]
+; CHECK-NEXT:    cinv w12, w8, ge
+; CHECK-NEXT:    cmp x11, w11, sxtw
+; CHECK-NEXT:    csel w11, w12, w11, ne
+; CHECK-NEXT:    mov w12, v0.s[3]
+; CHECK-NEXT:    fmov s0, w11
+; CHECK-NEXT:    smull x11, w14, w10
+; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    eor w9, w14, w10
+; CHECK-NEXT:    smull x10, w12, w13
+; CHECK-NEXT:    cmp w9, #0
+; CHECK-NEXT:    cinv w9, w8, ge
+; CHECK-NEXT:    cmp x11, w11, sxtw
+; CHECK-NEXT:    csel w9, w9, w11, ne
+; CHECK-NEXT:    mov v0.s[2], w9
+; CHECK-NEXT:    eor w9, w12, w13
+; CHECK-NEXT:    cmp w9, #0
+; CHECK-NEXT:    cinv w8, w8, ge
+; CHECK-NEXT:    cmp x10, w10, sxtw
+; CHECK-NEXT:    csel w8, w8, w10, ne
+; CHECK-NEXT:    mov v0.s[3], w8
+; CHECK-NEXT:    ret
+  %tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0)
+  ret <4 x i32> %tmp
+}
+
+define <4 x i64> @vec3(<4 x i64> %x, <4 x i64> %y) nounwind {
+; CHECK-LABEL: vec3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v2.d[1]
+; CHECK-NEXT:    mov x9, v0.d[1]
+; CHECK-NEXT:    mov w16, #2147483647 // =0x7fffffff
+; CHECK-NEXT:    fmov x10, d2
+; CHECK-NEXT:    fmov x11, d0
+; CHECK-NEXT:    mov x18, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-NEXT:    mov x14, v3.d[1]
+; CHECK-NEXT:    mov x15, v1.d[1]
+; CHECK-NEXT:    mul x13, x9, x8
+; CHECK-NEXT:    smulh x8, x9, x8
+; CHECK-NEXT:    mul x12, x11, x10
+; CHECK-NEXT:    smulh x9, x11, x10
+; CHECK-NEXT:    extr x13, x8, x13, #32
+; CHECK-NEXT:    cmp x8, x16
+; CHECK-NEXT:    mul x10, x15, x14
+; CHECK-NEXT:    csel x13, x18, x13, gt
+; CHECK-NEXT:    smulh x11, x15, x14
+; CHECK-NEXT:    fmov x14, d3
+; CHECK-NEXT:    fmov x15, d1
+; CHECK-NEXT:    extr x12, x9, x12, #32
+; CHECK-NEXT:    mul x17, x15, x14
+; CHECK-NEXT:    smulh x14, x15, x14
+; CHECK-NEXT:    mov x15, #-2147483648 // =0xffffffff80000000
+; CHECK-NEXT:    cmp x8, x15
+; CHECK-NEXT:    mov x8, #-9223372036854775808 // =0x8000000000000000
+; CHECK-NEXT:    csel x13, x8, x13, lt
+; CHECK-NEXT:    cmp x9, x16
+; CHECK-NEXT:    csel x12, x18, x12, gt
+; CHECK-NEXT:    cmp x9, x15
+; CHECK-NEXT:    extr x9, x11, x10, #32
+; CHECK-NEXT:    csel x10, x8, x12, lt
+; CHECK-NEXT:    cmp x11, x16
+; CHECK-NEXT:    csel x9, x18, x9, gt
+; CHECK-NEXT:    cmp x11, x15
+; CHECK-NEXT:    extr x11, x14, x17, #32
+; CHECK-NEXT:    csel x9, x8, x9, lt
+; CHECK-NEXT:    cmp x14, x16
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    csel x11, x18, x11, gt
+; CHECK-NEXT:    cmp x14, x15
+; CHECK-NEXT:    csel x8, x8, x11, lt
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    mov v0.d[1], x13
+; CHECK-NEXT:    mov v1.d[1], x9
+; CHECK-NEXT:    ret
+  %tmp = call <4 x i64> @llvm.smul.fix.sat.v4i64(<4 x i64> %x, <4 x i64> %y, i32 32)
+  ret <4 x i64> %tmp
+}
diff --git a/llvm/test/CodeGen/AArch64/umul_fix.ll b/llvm/test/CodeGen/AArch64/umul_fix.ll
new file mode 100644
index 000000000000..6ec683299384
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/umul_fix.ll
@@ -0,0 +1,147 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s
+
+define i32 @func(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull x8, w0, w1
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    extr w0, w9, w8, #2
+; CHECK-NEXT:    ret
+  %tmp = call i32 @llvm.umul.fix.i32(i32 %x, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x8, x0, x1
+; CHECK-NEXT:    umulh x9, x0, x1
+; CHECK-NEXT:    extr x0, x9, x8, #2
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 2)
+  ret i64 %tmp
+}
+
+define i4 @func3(i4 %x, i4 %y) nounwind {
+; CHECK-LABEL: func3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w1, #0xf
+; CHECK-NEXT:    and w9, w0, #0xf
+; CHECK-NEXT:    mul w8, w9, w8
+; CHECK-NEXT:    lsr w0, w8, #2
+; CHECK-NEXT:    ret
+  %tmp = call i4 @llvm.umul.fix.i4(i4 %x, i4 %y, i32 2)
+  ret i4 %tmp
+}
+
+;; These result in regular integer multiplication
+define i32 @func4(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul w0, w0, w1
+; CHECK-NEXT:    ret
+  %tmp = call i32 @llvm.umul.fix.i32(i32 %x, i32 %y, i32 0)
+  ret i32 %tmp
+}
+
+define i64 @func5(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func5:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x0, x0, x1
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 0)
+  ret i64 %tmp
+}
+
+define i4 @func6(i4 %x, i4 %y) nounwind {
+; CHECK-LABEL: func6:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w1, #0xf
+; CHECK-NEXT:    and w9, w0, #0xf
+; CHECK-NEXT:    mul w0, w9, w8
+; CHECK-NEXT:    ret
+  %tmp = call i4 @llvm.umul.fix.i4(i4 %x, i4 %y, i32 0)
+  ret i4 %tmp
+}
+
+define i64 @func7(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func7:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x8, x0, x1
+; CHECK-NEXT:    umulh x9, x0, x1
+; CHECK-NEXT:    extr x0, x9, x8, #32
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 32)
+  ret i64 %tmp
+}
+
+define i64 @func8(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x8, x0, x1
+; CHECK-NEXT:    umulh x9, x0, x1
+; CHECK-NEXT:    extr x0, x9, x8, #63
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 63)
+  ret i64 %tmp
+}
+
+define i64 @func9(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func9:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umulh x0, x0, x1
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 64)
+  ret i64 %tmp
+}
+
+define <2 x i32> @vec(<2 x i32> %x, <2 x i32> %y) nounwind {
+; CHECK-LABEL: vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    ret
+  %tmp = call <2 x i32> @llvm.umul.fix.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
+  ret <2 x i32> %tmp
+}
+
+define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
+; CHECK-LABEL: vec2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %tmp = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0)
+  ret <4 x i32> %tmp
+}
+
+define <4 x i64> @vec3(<4 x i64> %x, <4 x i64> %y) nounwind {
+; CHECK-LABEL: vec3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v2.d[1]
+; CHECK-NEXT:    mov x9, v0.d[1]
+; CHECK-NEXT:    fmov x10, d2
+; CHECK-NEXT:    fmov x11, d0
+; CHECK-NEXT:    mov x14, v3.d[1]
+; CHECK-NEXT:    mov x15, v1.d[1]
+; CHECK-NEXT:    mul x12, x11, x10
+; CHECK-NEXT:    mul x13, x9, x8
+; CHECK-NEXT:    umulh x8, x9, x8
+; CHECK-NEXT:    umulh x9, x11, x10
+; CHECK-NEXT:    fmov x10, d3
+; CHECK-NEXT:    fmov x11, d1
+; CHECK-NEXT:    mul x16, x11, x10
+; CHECK-NEXT:    extr x8, x8, x13, #32
+; CHECK-NEXT:    umulh x10, x11, x10
+; CHECK-NEXT:    extr x9, x9, x12, #32
+; CHECK-NEXT:    mul x11, x15, x14
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    umulh x14, x15, x14
+; CHECK-NEXT:    extr x10, x10, x16, #32
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    fmov d1, x10
+; CHECK-NEXT:    extr x11, x14, x11, #32
+; CHECK-NEXT:    mov v1.d[1], x11
+; CHECK-NEXT:    ret
+  %tmp = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> %x, <4 x i64> %y, i32 32)
+  ret <4 x i64> %tmp
+}
diff --git a/llvm/test/CodeGen/AArch64/umul_fix_sat.ll b/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
new file mode 100644
index 000000000000..e9965fd5319b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
@@ -0,0 +1,206 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s
+
+define i32 @func(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull x8, w0, w1
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    extr w8, w9, w8, #2
+; CHECK-NEXT:    cmp w9, #3
+; CHECK-NEXT:    csinv w0, w8, wzr, ls
+; CHECK-NEXT:    ret
+  %tmp = call i32 @llvm.umul.fix.sat.i32(i32 %x, i32 %y, i32 2)
+  ret i32 %tmp
+}
+
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x8, x0, x1
+; CHECK-NEXT:    umulh x9, x0, x1
+; CHECK-NEXT:    extr x8, x9, x8, #2
+; CHECK-NEXT:    cmp x9, #3
+; CHECK-NEXT:    csinv x0, x8, xzr, ls
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 2)
+  ret i64 %tmp
+}
+
+define i4 @func3(i4 %x, i4 %y) nounwind {
+; CHECK-LABEL: func3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl w8, w0, #28
+; CHECK-NEXT:    and w9, w1, #0xf
+; CHECK-NEXT:    umull x8, w8, w9
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    extr w8, w9, w8, #2
+; CHECK-NEXT:    cmp w9, #3
+; CHECK-NEXT:    csinv w8, w8, wzr, ls
+; CHECK-NEXT:    lsr w0, w8, #28
+; CHECK-NEXT:    ret
+  %tmp = call i4 @llvm.umul.fix.sat.i4(i4 %x, i4 %y, i32 2)
+  ret i4 %tmp
+}
+
+;; These result in regular integer multiplication with a saturation check.
+define i32 @func4(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: func4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull x8, w0, w1
+; CHECK-NEXT:    tst x8, #0xffffffff00000000
+; CHECK-NEXT:    csinv w0, w8, wzr, eq
+; CHECK-NEXT:    ret
+  %tmp = call i32 @llvm.umul.fix.sat.i32(i32 %x, i32 %y, i32 0)
+  ret i32 %tmp
+}
+
+define i64 @func5(i64 %x, i64 %y) {
+; CHECK-LABEL: func5:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umulh x8, x0, x1
+; CHECK-NEXT:    mul x9, x0, x1
+; CHECK-NEXT:    cmp xzr, x8
+; CHECK-NEXT:    csinv x0, x9, xzr, eq
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 0)
+  ret i64 %tmp
+}
+
+define i4 @func6(i4 %x, i4 %y) nounwind {
+; CHECK-LABEL: func6:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl w8, w0, #28
+; CHECK-NEXT:    and w9, w1, #0xf
+; CHECK-NEXT:    umull x8, w8, w9
+; CHECK-NEXT:    tst x8, #0xffffffff00000000
+; CHECK-NEXT:    csinv w8, w8, wzr, eq
+; CHECK-NEXT:    lsr w0, w8, #28
+; CHECK-NEXT:    ret
+  %tmp = call i4 @llvm.umul.fix.sat.i4(i4 %x, i4 %y, i32 0)
+  ret i4 %tmp
+}
+
+define <2 x i32> @vec(<2 x i32> %x, <2 x i32> %y) nounwind {
+; CHECK-LABEL: vec:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    mov w9, v0.s[1]
+; CHECK-NEXT:    fmov w10, s0
+; CHECK-NEXT:    umull x8, w9, w8
+; CHECK-NEXT:    fmov w9, s1
+; CHECK-NEXT:    umull x9, w10, w9
+; CHECK-NEXT:    tst x8, #0xffffffff00000000
+; CHECK-NEXT:    csinv w8, w8, wzr, eq
+; CHECK-NEXT:    tst x9, #0xffffffff00000000
+; CHECK-NEXT:    csinv w9, w9, wzr, eq
+; CHECK-NEXT:    fmov s0, w9
+; CHECK-NEXT:    mov v0.s[1], w8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+  %tmp = call <2 x i32> @llvm.umul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
+  ret <2 x i32> %tmp
+}
+
+define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
+; CHECK-LABEL: vec2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    mov w9, v0.s[1]
+; CHECK-NEXT:    fmov w10, s0
+; CHECK-NEXT:    mov w11, v0.s[2]
+; CHECK-NEXT:    mov w13, v0.s[3]
+; CHECK-NEXT:    mov w12, v1.s[3]
+; CHECK-NEXT:    umull x8, w9, w8
+; CHECK-NEXT:    fmov w9, s1
+; CHECK-NEXT:    umull x9, w10, w9
+; CHECK-NEXT:    tst x8, #0xffffffff00000000
+; CHECK-NEXT:    mov w10, v1.s[2]
+; CHECK-NEXT:    csinv w8, w8, wzr, eq
+; CHECK-NEXT:    tst x9, #0xffffffff00000000
+; CHECK-NEXT:    csinv w9, w9, wzr, eq
+; CHECK-NEXT:    fmov s0, w9
+; CHECK-NEXT:    umull x9, w11, w10
+; CHECK-NEXT:    mov v0.s[1], w8
+; CHECK-NEXT:    tst x9, #0xffffffff00000000
+; CHECK-NEXT:    csinv w8, w9, wzr, eq
+; CHECK-NEXT:    umull x9, w13, w12
+; CHECK-NEXT:    mov v0.s[2], w8
+; CHECK-NEXT:    tst x9, #0xffffffff00000000
+; CHECK-NEXT:    csinv w8, w9, wzr, eq
+; CHECK-NEXT:    mov v0.s[3], w8
+; CHECK-NEXT:    ret
+  %tmp = call <4 x i32> @llvm.umul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0)
+  ret <4 x i32> %tmp
+}
+
+define <4 x i64> @vec3(<4 x i64> %x, <4 x i64> %y) nounwind {
+; CHECK-LABEL: vec3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v2.d[1]
+; CHECK-NEXT:    mov x9, v0.d[1]
+; CHECK-NEXT:    mov x14, v3.d[1]
+; CHECK-NEXT:    mov x15, v1.d[1]
+; CHECK-NEXT:    fmov x10, d2
+; CHECK-NEXT:    fmov x11, d0
+; CHECK-NEXT:    mul x12, x11, x10
+; CHECK-NEXT:    mul x13, x9, x8
+; CHECK-NEXT:    umulh x8, x9, x8
+; CHECK-NEXT:    umulh x9, x11, x10
+; CHECK-NEXT:    mul x10, x15, x14
+; CHECK-NEXT:    extr x13, x8, x13, #32
+; CHECK-NEXT:    umulh x11, x15, x14
+; CHECK-NEXT:    fmov x14, d3
+; CHECK-NEXT:    fmov x15, d1
+; CHECK-NEXT:    mul x16, x15, x14
+; CHECK-NEXT:    umulh x14, x15, x14
+; CHECK-NEXT:    mov w15, #-1 // =0xffffffff
+; CHECK-NEXT:    cmp x8, x15
+; CHECK-NEXT:    extr x8, x9, x12, #32
+; CHECK-NEXT:    csinv x12, x13, xzr, ls
+; CHECK-NEXT:    cmp x9, x15
+; CHECK-NEXT:    extr x9, x11, x10, #32
+; CHECK-NEXT:    csinv x8, x8, xzr, ls
+; CHECK-NEXT:    cmp x11, x15
+; CHECK-NEXT:    csinv x9, x9, xzr, ls
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    extr x10, x14, x16, #32
+; CHECK-NEXT:    cmp x14, x15
+; CHECK-NEXT:    csinv x10, x10, xzr, ls
+; CHECK-NEXT:    mov v0.d[1], x12
+; CHECK-NEXT:    fmov d1, x10
+; CHECK-NEXT:    mov v1.d[1], x9
+; CHECK-NEXT:    ret
+  %tmp = call <4 x i64> @llvm.umul.fix.sat.v4i64(<4 x i64> %x, <4 x i64> %y, i32 32)
+  ret <4 x i64> %tmp
+}
+
+define i64 @func7(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func7:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x9, x0, x1
+; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-NEXT:    umulh x10, x0, x1
+; CHECK-NEXT:    extr x9, x10, x9, #32
+; CHECK-NEXT:    cmp x10, x8
+; CHECK-NEXT:    csinv x0, x9, xzr, ls
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 32)
+  ret i64 %tmp
+}
+
+define i64 @func8(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: func8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mul x9, x0, x1
+; CHECK-NEXT:    mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-NEXT:    umulh x10, x0, x1
+; CHECK-NEXT:    extr x9, x10, x9, #63
+; CHECK-NEXT:    cmp x10, x8
+; CHECK-NEXT:    csinv x0, x9, xzr, ls
+; CHECK-NEXT:    ret
+  %tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 63)
+  ret i64 %tmp
+}
-- 
GitLab


From acd100747fff85e7cfb67caa6c0f1053e820c1ac Mon Sep 17 00:00:00 2001
From: Felix Schneider <fx.schn@gmail.com>
Date: Tue, 14 May 2024 20:33:16 +0200
Subject: [PATCH 0768/1206] [mlir][test] Extend `InferIntRangeInterface` test
 Ops to arbitrary ints (#91850)

This PR is in preparation to some extensions to the
`InferIntRangeInterface` around the `nsw` and `nuw` flags supported in
the `arith` dialect and LLVM.

We provide some common inference logic for `index` and `arith` in
`InferIntRangeCommon.h` but our Test Ops are currently fixed to `Index`
Types. As we test the range inference for arith Ops, especially around
the overflow behaviour, it's handy to have native support for the
typical integer types in the test Ops.

This patch
1. Changes the Attributes of `test.with_bounds` ops from `Index` to
`APInt` which matches the internal representation in
`ConstantIntRanges`.
2. Allows the use of `AnyInteger` in addition to `Index` for the
operands and results of the test Ops. This now requires explicit
specification of the type in the IR, where before `Index` was implicit.
3. Requires bounds Attrs to be specified in the precision of the SSA
value, eliminating any implicit truncation or extension. (*Could this
lead to problems?*)
---
 .../Dialect/Arith/int-range-interface.mlir    |  10 ++
 mlir/test/Dialect/Arith/int-range-opts.mlir   |  36 +++---
 .../test/Dialect/GPU/int-range-interface.mlir | 106 +++++++++---------
 .../infer-int-range-test-ops.mlir             |  56 ++++-----
 mlir/test/lib/Dialect/Test/TestOpDefs.cpp     |  14 +--
 mlir/test/lib/Dialect/Test/TestOps.td         |  50 +++++----
 6 files changed, 140 insertions(+), 132 deletions(-)

diff --git a/mlir/test/Dialect/Arith/int-range-interface.mlir b/mlir/test/Dialect/Arith/int-range-interface.mlir
index 02a9827d19d8..16524b363472 100644
--- a/mlir/test/Dialect/Arith/int-range-interface.mlir
+++ b/mlir/test/Dialect/Arith/int-range-interface.mlir
@@ -756,3 +756,13 @@ func.func private @callee(%arg0: memref<?xindex, 4>) {
   }
   return
 }
+
+// CHECK-LABEL: func @test_i8_bounds
+// CHECK: test.reflect_bounds {smax = 127 : i8, smin = -128 : i8, umax = -1 : i8, umin = 0 : i8}
+func.func @test_i8_bounds() -> i8 {
+  %cst1 = arith.constant 1 : i8
+  %0 = test.with_bounds { umin = 0 : i8, umax = 255 : i8, smin = -128 : i8, smax = 127 : i8 } : i8
+  %1 = arith.addi %0, %cst1 : i8
+  %2 = test.reflect_bounds %1 : i8
+  return %2: i8
+}
diff --git a/mlir/test/Dialect/Arith/int-range-opts.mlir b/mlir/test/Dialect/Arith/int-range-opts.mlir
index 4c3c0854ed02..6179003ab4e7 100644
--- a/mlir/test/Dialect/Arith/int-range-opts.mlir
+++ b/mlir/test/Dialect/Arith/int-range-opts.mlir
@@ -5,7 +5,7 @@
 //       CHECK:   return %[[C]]
 func.func @test() -> i1 {
   %cst1 = arith.constant -1 : index
-  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index }
+  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index } : index
   %1 = arith.cmpi eq, %0, %cst1 : index
   return %1: i1
 }
@@ -17,7 +17,7 @@ func.func @test() -> i1 {
 //       CHECK:   return %[[C]]
 func.func @test() -> i1 {
   %cst1 = arith.constant -1 : index
-  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index }
+  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index } : index
   %1 = arith.cmpi ne, %0, %cst1 : index
   return %1: i1
 }
@@ -30,7 +30,7 @@ func.func @test() -> i1 {
 //       CHECK:   return %[[C]]
 func.func @test() -> i1 {
   %cst = arith.constant 0 : index
-  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index }
+  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index } : index
   %1 = arith.cmpi sge, %0, %cst : index
   return %1: i1
 }
@@ -42,7 +42,7 @@ func.func @test() -> i1 {
 //       CHECK:   return %[[C]]
 func.func @test() -> i1 {
   %cst = arith.constant 0 : index
-  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index }
+  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index } : index
   %1 = arith.cmpi slt, %0, %cst : index
   return %1: i1
 }
@@ -55,7 +55,7 @@ func.func @test() -> i1 {
 //       CHECK:   return %[[C]]
 func.func @test() -> i1 {
   %cst1 = arith.constant -1 : index
-  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index }
+  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index } : index
   %1 = arith.cmpi sgt, %0, %cst1 : index
   return %1: i1
 }
@@ -67,7 +67,7 @@ func.func @test() -> i1 {
 //       CHECK:   return %[[C]]
 func.func @test() -> i1 {
   %cst1 = arith.constant -1 : index
-  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index }
+  %0 = test.with_bounds { umin = 0 : index, umax = 0x7fffffffffffffff : index, smin = 0 : index, smax = 0x7fffffffffffffff : index } : index
   %1 = arith.cmpi sle, %0, %cst1 : index
   return %1: i1
 }
@@ -75,28 +75,24 @@ func.func @test() -> i1 {
 // -----
 
 // CHECK-LABEL: func @test
-// CHECK: test.reflect_bounds {smax = 24 : index, smin = 0 : index, umax = 24 : index, umin = 0 : index}
-func.func @test() -> index {
+// CHECK: test.reflect_bounds {smax = 24 : i8, smin = 0 : i8, umax = 24 : i8, umin = 0 : i8}
+func.func @test() -> i8 {
   %cst1 = arith.constant 1 : i8
-  %0 = test.with_bounds { umin = 0 : index, umax = 12 : index, smin = 0 : index, smax = 12 : index }
-  %i8val = arith.index_cast %0 : index to i8
+  %i8val = test.with_bounds { umin = 0 : i8, umax = 12 : i8, smin = 0 : i8, smax = 12 : i8 } : i8
   %shifted = arith.shli %i8val, %cst1 : i8
-  %si = arith.index_cast %shifted : i8 to index
-  %1 = test.reflect_bounds %si
-  return %1: index
+  %1 = test.reflect_bounds %shifted : i8
+  return %1: i8
 }
 
 // -----
 
 // CHECK-LABEL: func @test
-// CHECK: test.reflect_bounds {smax = 127 : index, smin = -128 : index, umax = -1 : index, umin = 0 : index}
-func.func @test() -> index {
+// CHECK: test.reflect_bounds {smax = 127 : i8, smin = -128 : i8, umax = -1 : i8, umin = 0 : i8}
+func.func @test() -> i8 {
   %cst1 = arith.constant 1 : i8
-  %0 = test.with_bounds { umin = 0 : index, umax = 127 : index, smin = 0 : index, smax = 127 : index }
-  %i8val = arith.index_cast %0 : index to i8
+  %i8val = test.with_bounds { umin = 0 : i8, umax = 127 : i8, smin = 0 : i8, smax = 127 : i8 } : i8
   %shifted = arith.shli %i8val, %cst1 : i8
-  %si = arith.index_cast %shifted : i8 to index
-  %1 = test.reflect_bounds %si
-  return %1: index
+  %1 = test.reflect_bounds %shifted : i8
+  return %1: i8
 }
 
diff --git a/mlir/test/Dialect/GPU/int-range-interface.mlir b/mlir/test/Dialect/GPU/int-range-interface.mlir
index 02aec9dc0476..980f7e5873e0 100644
--- a/mlir/test/Dialect/GPU/int-range-interface.mlir
+++ b/mlir/test/Dialect/GPU/int-range-interface.mlir
@@ -5,46 +5,46 @@ func.func @launch_func(%arg0 : index) {
   %0 = test.with_bounds {
     umin = 3 : index, umax = 5 : index,
     smin = 3 : index, smax = 5 : index
-  }
+  } : index
   %1 = test.with_bounds {
     umin = 7 : index, umax = 11 : index,
     smin = 7 : index, smax = 11 : index
-  }
+  } : index
   gpu.launch blocks(%block_id_x, %block_id_y, %block_id_z) in (%grid_dim_x = %0, %grid_dim_y = %1, %grid_dim_z = %arg0)
       threads(%thread_id_x, %thread_id_y, %thread_id_z) in (%block_dim_x = %arg0, %block_dim_y = %0, %block_dim_z = %1) {
 
     // CHECK: test.reflect_bounds {smax = 5 : index, smin = 3 : index, umax = 5 : index, umin = 3 : index}
     // CHECK: test.reflect_bounds {smax = 11 : index, smin = 7 : index, umax = 11 : index, umin = 7 : index}
     // CHECK: test.reflect_bounds {smax = 4294967295 : index, smin = 1 : index, umax = 4294967295 : index, umin = 1 : index}
-    %grid_dim_x0 = test.reflect_bounds %grid_dim_x
-    %grid_dim_y0 = test.reflect_bounds %grid_dim_y
-    %grid_dim_z0 = test.reflect_bounds %grid_dim_z
+    %grid_dim_x0 = test.reflect_bounds %grid_dim_x : index
+    %grid_dim_y0 = test.reflect_bounds %grid_dim_y : index
+    %grid_dim_z0 = test.reflect_bounds %grid_dim_z : index
 
     // CHECK: test.reflect_bounds {smax = 4 : index, smin = 0 : index, umax = 4 : index, umin = 0 : index}
     // CHECK: test.reflect_bounds {smax = 10 : index, smin = 0 : index, umax = 10 : index, umin = 0 : index}
     // CHECK: test.reflect_bounds {smax = 4294967294 : index, smin = 0 : index, umax = 4294967294 : index, umin = 0 : index}
-    %block_id_x0 = test.reflect_bounds %block_id_x
-    %block_id_y0 = test.reflect_bounds %block_id_y
-    %block_id_z0 = test.reflect_bounds %block_id_z
+    %block_id_x0 = test.reflect_bounds %block_id_x : index
+    %block_id_y0 = test.reflect_bounds %block_id_y : index
+    %block_id_z0 = test.reflect_bounds %block_id_z : index
 
     // CHECK: test.reflect_bounds {smax = 4294967295 : index, smin = 1 : index, umax = 4294967295 : index, umin = 1 : index}
     // CHECK: test.reflect_bounds {smax = 5 : index, smin = 3 : index, umax = 5 : index, umin = 3 : index}
     // CHECK: test.reflect_bounds {smax = 11 : index, smin = 7 : index, umax = 11 : index, umin = 7 : index}
-    %block_dim_x0 = test.reflect_bounds %block_dim_x
-    %block_dim_y0 = test.reflect_bounds %block_dim_y
-    %block_dim_z0 = test.reflect_bounds %block_dim_z
+    %block_dim_x0 = test.reflect_bounds %block_dim_x : index
+    %block_dim_y0 = test.reflect_bounds %block_dim_y : index
+    %block_dim_z0 = test.reflect_bounds %block_dim_z : index
 
     // CHECK: test.reflect_bounds {smax = 4294967294 : index, smin = 0 : index, umax = 4294967294 : index, umin = 0 : index}
     // CHECK: test.reflect_bounds {smax = 4 : index, smin = 0 : index, umax = 4 : index, umin = 0 : index}
     // CHECK: test.reflect_bounds {smax = 10 : index, smin = 0 : index, umax = 10 : index, umin = 0 : index}
-    %thread_id_x0 = test.reflect_bounds %thread_id_x
-    %thread_id_y0 = test.reflect_bounds %thread_id_y
-    %thread_id_z0 = test.reflect_bounds %thread_id_z
+    %thread_id_x0 = test.reflect_bounds %thread_id_x : index
+    %thread_id_y0 = test.reflect_bounds %thread_id_y : index
+    %thread_id_z0 = test.reflect_bounds %thread_id_z : index
 
     // The launch bounds are not constant, and so this can't infer anything
     // CHECK: test.reflect_bounds {smax = 4294967294 : index, smin = 0 : index, umax = 4294967294 : index, umin = 0 : index}
     %thread_id_op = gpu.thread_id y
-    %thread_id_op0 = test.reflect_bounds %thread_id_op
+    %thread_id_op0 = test.reflect_bounds %thread_id_op : index
     gpu.terminator
   }
 
@@ -65,9 +65,9 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 4294967295 : index, smin = 1 : index, umax = 4294967295 : index, umin = 1 : index}
       // CHECK: test.reflect_bounds {smax = 4294967295 : index, smin = 1 : index, umax = 4294967295 : index, umin = 1 : index}
       // CHECK: test.reflect_bounds {smax = 4294967295 : index, smin = 1 : index, umax = 4294967295 : index, umin = 1 : index}
-      %grid_dim_x0 = test.reflect_bounds %grid_dim_x
-      %grid_dim_y0 = test.reflect_bounds %grid_dim_y
-      %grid_dim_z0 = test.reflect_bounds %grid_dim_z
+      %grid_dim_x0 = test.reflect_bounds %grid_dim_x : index
+      %grid_dim_y0 = test.reflect_bounds %grid_dim_y : index
+      %grid_dim_z0 = test.reflect_bounds %grid_dim_z : index
 
       %block_id_x = gpu.block_id x
       %block_id_y = gpu.block_id y
@@ -76,9 +76,9 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 4294967294 : index, smin = 0 : index, umax = 4294967294 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 4294967294 : index, smin = 0 : index, umax = 4294967294 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 4294967294 : index, smin = 0 : index, umax = 4294967294 : index, umin = 0 : index}
-      %block_id_x0 = test.reflect_bounds %block_id_x
-      %block_id_y0 = test.reflect_bounds %block_id_y
-      %block_id_z0 = test.reflect_bounds %block_id_z
+      %block_id_x0 = test.reflect_bounds %block_id_x : index
+      %block_id_y0 = test.reflect_bounds %block_id_y : index
+      %block_id_z0 = test.reflect_bounds %block_id_z : index
 
       %block_dim_x = gpu.block_dim x
       %block_dim_y = gpu.block_dim y
@@ -87,9 +87,9 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 4294967295 : index, smin = 1 : index, umax = 4294967295 : index, umin = 1 : index}
       // CHECK: test.reflect_bounds {smax = 4294967295 : index, smin = 1 : index, umax = 4294967295 : index, umin = 1 : index}
       // CHECK: test.reflect_bounds {smax = 4294967295 : index, smin = 1 : index, umax = 4294967295 : index, umin = 1 : index}
-      %block_dim_x0 = test.reflect_bounds %block_dim_x
-      %block_dim_y0 = test.reflect_bounds %block_dim_y
-      %block_dim_z0 = test.reflect_bounds %block_dim_z
+      %block_dim_x0 = test.reflect_bounds %block_dim_x : index
+      %block_dim_y0 = test.reflect_bounds %block_dim_y : index
+      %block_dim_z0 = test.reflect_bounds %block_dim_z : index
 
       %thread_id_x = gpu.thread_id x
       %thread_id_y = gpu.thread_id y
@@ -98,9 +98,9 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 4294967294 : index, smin = 0 : index, umax = 4294967294 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 4294967294 : index, smin = 0 : index, umax = 4294967294 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 4294967294 : index, smin = 0 : index, umax = 4294967294 : index, umin = 0 : index}
-      %thread_id_x0 = test.reflect_bounds %thread_id_x
-      %thread_id_y0 = test.reflect_bounds %thread_id_y
-      %thread_id_z0 = test.reflect_bounds %thread_id_z
+      %thread_id_x0 = test.reflect_bounds %thread_id_x : index
+      %thread_id_y0 = test.reflect_bounds %thread_id_y : index
+      %thread_id_z0 = test.reflect_bounds %thread_id_z : index
 
       %global_id_x = gpu.global_id x
       %global_id_y = gpu.global_id y
@@ -109,9 +109,9 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 9223372036854775807 : index, smin = -9223372036854775808 : index, umax = -8589934592 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 9223372036854775807 : index, smin = -9223372036854775808 : index, umax = -8589934592 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 9223372036854775807 : index, smin = -9223372036854775808 : index, umax = -8589934592 : index, umin = 0 : index}
-      %global_id_x0 = test.reflect_bounds %global_id_x
-      %global_id_y0 = test.reflect_bounds %global_id_y
-      %global_id_z0 = test.reflect_bounds %global_id_z
+      %global_id_x0 = test.reflect_bounds %global_id_x : index
+      %global_id_y0 = test.reflect_bounds %global_id_y : index
+      %global_id_z0 = test.reflect_bounds %global_id_z : index
 
       %subgroup_size = gpu.subgroup_size : index
       %lane_id = gpu.lane_id
@@ -122,10 +122,10 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 127 : index, smin = 0 : index, umax = 127 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 4294967295 : index, smin = 1 : index, umax = 4294967295 : index, umin = 1 : index}
       // CHECK: test.reflect_bounds {smax = 4294967294 : index, smin = 0 : index, umax = 4294967294 : index, umin = 0 : index}
-      %subgroup_size0 = test.reflect_bounds %subgroup_size
-      %lane_id0 = test.reflect_bounds %lane_id
-      %num_subgroups0 = test.reflect_bounds %num_subgroups
-      %subgroup_id0 = test.reflect_bounds %subgroup_id
+      %subgroup_size0 = test.reflect_bounds %subgroup_size : index
+      %lane_id0 = test.reflect_bounds %lane_id : index
+      %num_subgroups0 = test.reflect_bounds %num_subgroups : index
+      %subgroup_id0 = test.reflect_bounds %subgroup_id : index
 
       llvm.return
     }
@@ -148,9 +148,9 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 20 : index, smin = 20 : index, umax = 20 : index, umin = 20 : index}
       // CHECK: test.reflect_bounds {smax = 24 : index, smin = 24 : index, umax = 24 : index, umin = 24 : index}
       // CHECK: test.reflect_bounds {smax = 28 : index, smin = 28 : index, umax = 28 : index, umin = 28 : index}
-      %grid_dim_x0 = test.reflect_bounds %grid_dim_x
-      %grid_dim_y0 = test.reflect_bounds %grid_dim_y
-      %grid_dim_z0 = test.reflect_bounds %grid_dim_z
+      %grid_dim_x0 = test.reflect_bounds %grid_dim_x : index
+      %grid_dim_y0 = test.reflect_bounds %grid_dim_y : index
+      %grid_dim_z0 = test.reflect_bounds %grid_dim_z : index
 
       %block_id_x = gpu.block_id x
       %block_id_y = gpu.block_id y
@@ -159,9 +159,9 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 19 : index, smin = 0 : index, umax = 19 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 23 : index, smin = 0 : index, umax = 23 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 27 : index, smin = 0 : index, umax = 27 : index, umin = 0 : index}
-      %block_id_x0 = test.reflect_bounds %block_id_x
-      %block_id_y0 = test.reflect_bounds %block_id_y
-      %block_id_z0 = test.reflect_bounds %block_id_z
+      %block_id_x0 = test.reflect_bounds %block_id_x : index
+      %block_id_y0 = test.reflect_bounds %block_id_y : index
+      %block_id_z0 = test.reflect_bounds %block_id_z : index
 
       %block_dim_x = gpu.block_dim x
       %block_dim_y = gpu.block_dim y
@@ -170,9 +170,9 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 8 : index, smin = 8 : index, umax = 8 : index, umin = 8 : index}
       // CHECK: test.reflect_bounds {smax = 12 : index, smin = 12 : index, umax = 12 : index, umin = 12 : index}
       // CHECK: test.reflect_bounds {smax = 16 : index, smin = 16 : index, umax = 16 : index, umin = 16 : index}
-      %block_dim_x0 = test.reflect_bounds %block_dim_x
-      %block_dim_y0 = test.reflect_bounds %block_dim_y
-      %block_dim_z0 = test.reflect_bounds %block_dim_z
+      %block_dim_x0 = test.reflect_bounds %block_dim_x : index
+      %block_dim_y0 = test.reflect_bounds %block_dim_y : index
+      %block_dim_z0 = test.reflect_bounds %block_dim_z : index
 
       %thread_id_x = gpu.thread_id x
       %thread_id_y = gpu.thread_id y
@@ -181,9 +181,9 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 7 : index, smin = 0 : index, umax = 7 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 11 : index, smin = 0 : index, umax = 11 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 15 : index, smin = 0 : index, umax = 15 : index, umin = 0 : index}
-      %thread_id_x0 = test.reflect_bounds %thread_id_x
-      %thread_id_y0 = test.reflect_bounds %thread_id_y
-      %thread_id_z0 = test.reflect_bounds %thread_id_z
+      %thread_id_x0 = test.reflect_bounds %thread_id_x : index
+      %thread_id_y0 = test.reflect_bounds %thread_id_y : index
+      %thread_id_z0 = test.reflect_bounds %thread_id_z : index
 
       %global_id_x = gpu.global_id x
       %global_id_y = gpu.global_id y
@@ -192,9 +192,9 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 159 : index, smin = 0 : index, umax = 159 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 287 : index, smin = 0 : index, umax = 287 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 447 : index, smin = 0 : index, umax = 447 : index, umin = 0 : index}
-      %global_id_x0 = test.reflect_bounds %global_id_x
-      %global_id_y0 = test.reflect_bounds %global_id_y
-      %global_id_z0 = test.reflect_bounds %global_id_z
+      %global_id_x0 = test.reflect_bounds %global_id_x : index
+      %global_id_y0 = test.reflect_bounds %global_id_y : index
+      %global_id_z0 = test.reflect_bounds %global_id_z : index
 
       %subgroup_size = gpu.subgroup_size : index
       %lane_id = gpu.lane_id
@@ -205,10 +205,10 @@ module attributes {gpu.container_module} {
       // CHECK: test.reflect_bounds {smax = 127 : index, smin = 0 : index, umax = 127 : index, umin = 0 : index}
       // CHECK: test.reflect_bounds {smax = 4294967295 : index, smin = 1 : index, umax = 4294967295 : index, umin = 1 : index}
       // CHECK: test.reflect_bounds {smax = 4294967294 : index, smin = 0 : index, umax = 4294967294 : index, umin = 0 : index}
-      %subgroup_size0 = test.reflect_bounds %subgroup_size
-      %lane_id0 = test.reflect_bounds %lane_id
-      %num_subgroups0 = test.reflect_bounds %num_subgroups
-      %subgroup_id0 = test.reflect_bounds %subgroup_id
+      %subgroup_size0 = test.reflect_bounds %subgroup_size : index
+      %lane_id0 = test.reflect_bounds %lane_id : index
+      %num_subgroups0 = test.reflect_bounds %num_subgroups : index
+      %subgroup_id0 = test.reflect_bounds %subgroup_id : index
 
       gpu.return
     }
diff --git a/mlir/test/Interfaces/InferIntRangeInterface/infer-int-range-test-ops.mlir b/mlir/test/Interfaces/InferIntRangeInterface/infer-int-range-test-ops.mlir
index c74af447d1b1..2106eeefdca4 100644
--- a/mlir/test/Interfaces/InferIntRangeInterface/infer-int-range-test-ops.mlir
+++ b/mlir/test/Interfaces/InferIntRangeInterface/infer-int-range-test-ops.mlir
@@ -5,7 +5,7 @@
 // CHECK: return %[[cst]]
 func.func @constant() -> index {
   %0 = test.with_bounds { umin = 3 : index, umax = 3 : index,
-                               smin = 3 : index, smax = 3 : index}
+                               smin = 3 : index, smax = 3 : index} : index
   func.return %0 : index
 }
 
@@ -13,8 +13,8 @@ func.func @constant() -> index {
 // CHECK: %[[cst:.*]] = "test.constant"() <{value = 4 : index}
 // CHECK: return %[[cst]]
 func.func @increment() -> index {
-  %0 = test.with_bounds { umin = 3 : index, umax = 3 : index, smin = 0 : index, smax = 0x7fffffffffffffff : index }
-  %1 = test.increment %0
+  %0 = test.with_bounds { umin = 3 : index, umax = 3 : index, smin = 0 : index, smax = 0x7fffffffffffffff : index } : index
+  %1 = test.increment %0 : index
   func.return %1 : index
 }
 
@@ -22,14 +22,14 @@ func.func @increment() -> index {
 // CHECK: test.reflect_bounds {smax = 4 : index, smin = 3 : index, umax = 4 : index, umin = 3 : index}
 func.func @maybe_increment(%arg0 : i1) -> index {
   %0 = test.with_bounds { umin = 3 : index, umax = 3 : index,
-                               smin = 3 : index, smax = 3 : index}
+                               smin = 3 : index, smax = 3 : index} : index
   %1 = scf.if %arg0 -> index {
     scf.yield %0 : index
   } else {
-    %2 = test.increment %0
+    %2 = test.increment %0 : index
     scf.yield %2 : index
   }
-  %3 = test.reflect_bounds %1
+  %3 = test.reflect_bounds %1 : index
   func.return %3 : index
 }
 
@@ -37,15 +37,15 @@ func.func @maybe_increment(%arg0 : i1) -> index {
 // CHECK: test.reflect_bounds {smax = 4 : index, smin = 3 : index, umax = 4 : index, umin = 3 : index}
 func.func @maybe_increment_br(%arg0 : i1) -> index {
   %0 = test.with_bounds { umin = 3 : index, umax = 3 : index,
-                               smin = 3 : index, smax = 3 : index}
+                               smin = 3 : index, smax = 3 : index} : index
   cf.cond_br %arg0, ^bb0, ^bb1
 ^bb0:
-    %1 = test.increment %0
+    %1 = test.increment %0 : index
     cf.br ^bb2(%1 : index)
 ^bb1:
     cf.br ^bb2(%0 : index)
 ^bb2(%2 : index):
-  %3 = test.reflect_bounds %2
+  %3 = test.reflect_bounds %2 : index
   func.return %3 : index
 }
 
@@ -53,16 +53,16 @@ func.func @maybe_increment_br(%arg0 : i1) -> index {
 // CHECK: test.reflect_bounds {smax = 1 : index, smin = 0 : index, umax = 1 : index, umin = 0 : index}
 func.func @for_bounds() -> index {
   %c0 = test.with_bounds { umin = 0 : index, umax = 0 : index,
-                                smin = 0 : index, smax = 0 : index}
+                                smin = 0 : index, smax = 0 : index} : index
   %c1 = test.with_bounds { umin = 1 : index, umax = 1 : index,
-                                smin = 1 : index, smax = 1 : index}
+                                smin = 1 : index, smax = 1 : index} : index
   %c2 = test.with_bounds { umin = 2 : index, umax = 2 : index,
-                                smin = 2 : index, smax = 2 : index}
+                                smin = 2 : index, smax = 2 : index} : index
 
   %0 = scf.for %arg0 = %c0 to %c2 step %c1 iter_args(%arg2 = %c0) -> index {
     scf.yield %arg0 : index
   }
-  %1 = test.reflect_bounds %0
+  %1 = test.reflect_bounds %0 : index
   func.return %1 : index
 }
 
@@ -70,17 +70,17 @@ func.func @for_bounds() -> index {
 // CHECK: test.reflect_bounds {smax = 9223372036854775807 : index, smin = -9223372036854775808 : index, umax = -1 : index, umin = 0 : index}
 func.func @no_analysis_of_loop_variants() -> index {
   %c0 = test.with_bounds { umin = 0 : index, umax = 0 : index,
-                                smin = 0 : index, smax = 0 : index}
+                                smin = 0 : index, smax = 0 : index} : index
   %c1 = test.with_bounds { umin = 1 : index, umax = 1 : index,
-                                smin = 1 : index, smax = 1 : index}
+                                smin = 1 : index, smax = 1 : index} : index
   %c2 = test.with_bounds { umin = 2 : index, umax = 2 : index,
-                                smin = 2 : index, smax = 2 : index}
+                                smin = 2 : index, smax = 2 : index} : index
 
   %0 = scf.for %arg0 = %c0 to %c2 step %c1 iter_args(%arg2 = %c0) -> index {
-    %1 = test.increment %arg2
+    %1 = test.increment %arg2 : index
     scf.yield %1 : index
   }
-  %2 = test.reflect_bounds %0
+  %2 = test.reflect_bounds %0 : index
   func.return %2 : index
 }
 
@@ -88,8 +88,8 @@ func.func @no_analysis_of_loop_variants() -> index {
 // CHECK: test.reflect_bounds {smax = 4 : index, smin = 3 : index, umax = 4 : index, umin = 3 : index}
 func.func @region_args() {
   test.with_bounds_region { umin = 3 : index, umax = 4 : index,
-                            smin = 3 : index, smax = 4 : index } %arg0 {
-    %0 = test.reflect_bounds %arg0
+                            smin = 3 : index, smax = 4 : index } %arg0 : index {
+    %0 = test.reflect_bounds %arg0 : index
   }
   func.return
 }
@@ -97,7 +97,7 @@ func.func @region_args() {
 // CHECK-LABEL: func @func_args_unbound
 // CHECK: test.reflect_bounds {smax = 9223372036854775807 : index, smin = -9223372036854775808 : index, umax = -1 : index, umin = 0 : index}
 func.func @func_args_unbound(%arg0 : index) -> index {
-  %0 = test.reflect_bounds %arg0
+  %0 = test.reflect_bounds %arg0 : index
   func.return %0 : index
 }
 
@@ -106,7 +106,7 @@ func.func @propagate_across_while_loop_false() -> index {
   // CHECK-DAG: %[[C0:.*]] = "test.constant"() <{value = 0
   // CHECK-DAG: %[[C1:.*]] = "test.constant"() <{value = 1
   %0 = test.with_bounds { umin = 0 : index, umax = 0 : index,
-                          smin = 0 : index, smax = 0 : index }
+                          smin = 0 : index, smax = 0 : index } : index
   %1 = scf.while : () -> index {
     %false = arith.constant false
     // CHECK: scf.condition(%{{.*}}) %[[C0]]
@@ -116,7 +116,7 @@ func.func @propagate_across_while_loop_false() -> index {
     scf.yield
   }
   // CHECK: return %[[C1]]
-  %2 = test.increment %1
+  %2 = test.increment %1 : index
   return %2 : index
 }
 
@@ -125,7 +125,7 @@ func.func @propagate_across_while_loop(%arg0 : i1) -> index {
   // CHECK-DAG: %[[C0:.*]] = "test.constant"() <{value = 0
   // CHECK-DAG: %[[C1:.*]] = "test.constant"() <{value = 1
   %0 = test.with_bounds { umin = 0 : index, umax = 0 : index,
-                          smin = 0 : index, smax = 0 : index }
+                          smin = 0 : index, smax = 0 : index } : index
   %1 = scf.while : () -> index {
     // CHECK: scf.condition(%{{.*}}) %[[C0]]
     scf.condition(%arg0) %0 : index
@@ -134,7 +134,7 @@ func.func @propagate_across_while_loop(%arg0 : i1) -> index {
     scf.yield
   }
   // CHECK: return %[[C1]]
-  %2 = test.increment %1
+  %2 = test.increment %1 : index
   return %2 : index
 }
 
@@ -142,7 +142,7 @@ func.func @propagate_across_while_loop(%arg0 : i1) -> index {
 func.func @dont_propagate_across_infinite_loop() -> index {
   // CHECK: %[[C0:.*]] = "test.constant"() <{value = 0
   %0 = test.with_bounds { umin = 0 : index, umax = 0 : index,
-                          smin = 0 : index, smax = 0 : index }
+                          smin = 0 : index, smax = 0 : index } : index
   // CHECK: %[[loopRes:.*]] = scf.while
   %1 = scf.while : () -> index {
     %true = arith.constant true
@@ -152,8 +152,8 @@ func.func @dont_propagate_across_infinite_loop() -> index {
   ^bb0(%i1: index):
     scf.yield
   }
-  // CHECK: %[[ret:.*]] = test.reflect_bounds %[[loopRes]]
-  %2 = test.reflect_bounds %1
+  // CHECK: %[[ret:.*]] = test.reflect_bounds %[[loopRes]] : index
+  %2 = test.reflect_bounds %1 : index
   // CHECK: return %[[ret]]
   return %2 : index
 }
diff --git a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
index 0b676db18af4..bfee0391f670 100644
--- a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
+++ b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
@@ -663,8 +663,7 @@ ParseResult TestWithBoundsRegionOp::parse(OpAsmParser &parser,
 
   // Parse the input argument
   OpAsmParser::Argument argInfo;
-  argInfo.type = parser.getBuilder().getIndexType();
-  if (failed(parser.parseArgument(argInfo)))
+  if (failed(parser.parseArgument(argInfo, true)))
     return failure();
 
   // Parse the body region, and reuse the operand info as the argument info.
@@ -676,7 +675,7 @@ void TestWithBoundsRegionOp::print(OpAsmPrinter &p) {
   p.printOptionalAttrDict((*this)->getAttrs());
   p << ' ';
   p.printRegionArgument(getRegion().getArgument(0), /*argAttrs=*/{},
-                        /*omitType=*/true);
+                        /*omitType=*/false);
   p << ' ';
   p.printRegion(getRegion(), /*printEntryBlockArgs=*/false);
 }
@@ -707,10 +706,11 @@ void TestReflectBoundsOp::inferResultRanges(
   const ConstantIntRanges &range = argRanges[0];
   MLIRContext *ctx = getContext();
   Builder b(ctx);
-  setUminAttr(b.getIndexAttr(range.umin().getZExtValue()));
-  setUmaxAttr(b.getIndexAttr(range.umax().getZExtValue()));
-  setSminAttr(b.getIndexAttr(range.smin().getSExtValue()));
-  setSmaxAttr(b.getIndexAttr(range.smax().getSExtValue()));
+  auto intTy = getType();
+  setUminAttr(b.getIntegerAttr(intTy, range.umin()));
+  setUmaxAttr(b.getIntegerAttr(intTy, range.umax()));
+  setSminAttr(b.getIntegerAttr(intTy, range.smin()));
+  setSmaxAttr(b.getIntegerAttr(intTy, range.smax()));
   setResultRanges(getResult(), range);
 }
 
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 7fc3d22d1895..befe6aa6cede 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -2734,49 +2734,51 @@ def TestGraphLoopOp : TEST_Op<"graph_loop",
 //===----------------------------------------------------------------------===//
 // Test InferIntRangeInterface
 //===----------------------------------------------------------------------===//
+def InferIntRangeType : AnyTypeOf<[AnyInteger, Index]>;
+
 def TestWithBoundsOp : TEST_Op<"with_bounds",
                           [DeclareOpInterfaceMethods<InferIntRangeInterface>,
                            NoMemoryEffect]> {
-  let arguments = (ins IndexAttr:$umin,
-                       IndexAttr:$umax,
-                       IndexAttr:$smin,
-                       IndexAttr:$smax);
-  let results = (outs Index:$fakeVal);
+  let arguments = (ins APIntAttr:$umin,
+                       APIntAttr:$umax,
+                       APIntAttr:$smin,
+                       APIntAttr:$smax);
+  let results = (outs InferIntRangeType:$fakeVal);
 
-  let assemblyFormat = "attr-dict";
+  let assemblyFormat = "attr-dict `:` type($fakeVal)";
 }
 
 def TestWithBoundsRegionOp : TEST_Op<"with_bounds_region",
                           [DeclareOpInterfaceMethods<InferIntRangeInterface>,
                            SingleBlock, NoTerminator]> {
-  let arguments = (ins IndexAttr:$umin,
-                       IndexAttr:$umax,
-                       IndexAttr:$smin,
-                       IndexAttr:$smax);
-  // The region has one argument of index type
+  let arguments = (ins APIntAttr:$umin,
+                       APIntAttr:$umax,
+                       APIntAttr:$smin,
+                       APIntAttr:$smax);
+  // The region has one argument of any integer type
   let regions = (region SizedRegion<1>:$region);
   let hasCustomAssemblyFormat = 1;
 }
 
 def TestIncrementOp : TEST_Op<"increment",
                          [DeclareOpInterfaceMethods<InferIntRangeInterface>,
-                         NoMemoryEffect]> {
-  let arguments = (ins Index:$value);
-  let results = (outs Index:$result);
+                         NoMemoryEffect, AllTypesMatch<["value", "result"]>]> {
+  let arguments = (ins InferIntRangeType:$value);
+  let results = (outs InferIntRangeType:$result);
 
-  let assemblyFormat = "attr-dict $value";
+  let assemblyFormat = "attr-dict $value `:` type($result)";
 }
 
 def TestReflectBoundsOp : TEST_Op<"reflect_bounds",
-                         [DeclareOpInterfaceMethods<InferIntRangeInterface>]> {
-  let arguments = (ins Index:$value,
-                       OptionalAttr<IndexAttr>:$umin,
-                       OptionalAttr<IndexAttr>:$umax,
-                       OptionalAttr<IndexAttr>:$smin,
-                       OptionalAttr<IndexAttr>:$smax);
-  let results = (outs Index:$result);
-
-  let assemblyFormat = "attr-dict $value";
+                         [DeclareOpInterfaceMethods<InferIntRangeInterface>, AllTypesMatch<["value", "result"]>]> {
+  let arguments = (ins InferIntRangeType:$value,
+                       OptionalAttr<APIntAttr>:$umin,
+                       OptionalAttr<APIntAttr>:$umax,
+                       OptionalAttr<APIntAttr>:$smin,
+                       OptionalAttr<APIntAttr>:$smax);
+  let results = (outs InferIntRangeType:$result);
+
+  let assemblyFormat = "attr-dict $value `:` type($result)";
 }
 
 //===----------------------------------------------------------------------===//
-- 
GitLab


From 80f8ae3f8485b62529c32683ca48822d700c7716 Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Tue, 14 May 2024 11:34:27 -0700
Subject: [PATCH 0769/1206] [NFC] add explanation to register flags doc
 (#91803)

---
 llvm/docs/MIRLangRef.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/llvm/docs/MIRLangRef.rst b/llvm/docs/MIRLangRef.rst
index e248a14636a8..ec29870128c1 100644
--- a/llvm/docs/MIRLangRef.rst
+++ b/llvm/docs/MIRLangRef.rst
@@ -540,41 +540,55 @@ Register Flags
 The table below shows all of the possible register flags along with the
 corresponding internal ``llvm::RegState`` representation:
 
+..
+   Keep this in sync with MachineInstrBuilder.h
+
 .. list-table::
    :header-rows: 1
 
    * - Flag
      - Internal Value
+     - Meaning
 
    * - ``implicit``
      - ``RegState::Implicit``
+     - Not emitted register (e.g. carry, or temporary result).
 
    * - ``implicit-def``
      - ``RegState::ImplicitDefine``
+     - ``implicit`` and ``def``
 
    * - ``def``
      - ``RegState::Define``
+     - Register definition.
 
    * - ``dead``
      - ``RegState::Dead``
+     - Unused definition.
 
    * - ``killed``
      - ``RegState::Kill``
+     - The last use of a register.
 
    * - ``undef``
      - ``RegState::Undef``
+     - Value of the register doesn't matter.
 
    * - ``internal``
      - ``RegState::InternalRead``
+     - Register reads a value that is defined inside the same instruction or bundle.
 
    * - ``early-clobber``
      - ``RegState::EarlyClobber``
+     - Register definition happens before uses.
 
    * - ``debug-use``
      - ``RegState::Debug``
+     - Register 'use' is for debugging purpose.
 
    * - ``renamable``
      - ``RegState::Renamable``
+     - Register that may be renamed.
 
 .. _subregister-indices:
 
-- 
GitLab


From 5adfcb07501f1d128e6517e60d30f2e3a0dc8eaa Mon Sep 17 00:00:00 2001
From: Zequan Wu <zequanwu@google.com>
Date: Tue, 14 May 2024 14:40:33 -0400
Subject: [PATCH 0770/1206] Allow passing creduce options through
 creduce-clang-crash.py (#92141)

This change allows us to pass creduce options to creduce-clang-crash.py
script. With this, `--n` is no longer needed to specify the number of
cores, so removed the flag.

The motivation is
https://github.com/llvm/llvm-project/pull/87933#issuecomment-2109463497
suggests that disabling creduce renaming passes helps people to further
reduce crash manually.
---
 clang/utils/creduce-clang-crash.py | 32 ++++++++++++------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/clang/utils/creduce-clang-crash.py b/clang/utils/creduce-clang-crash.py
index 4d0c8224d8b4..db4a3435a3ae 100755
--- a/clang/utils/creduce-clang-crash.py
+++ b/clang/utils/creduce-clang-crash.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """Calls C-Reduce to create a minimal reproducer for clang crashes.
+Unknown arguments are treated at creduce options.
 
 Output files:
   *.reduced.sh -- crash reproducer with minimal arguments
@@ -70,7 +71,7 @@ def write_to_script(text, filename):
 
 
 class Reduce(object):
-    def __init__(self, crash_script, file_to_reduce, core_number):
+    def __init__(self, crash_script, file_to_reduce, creduce_flags):
         crash_script_name, crash_script_ext = os.path.splitext(crash_script)
         file_reduce_name, file_reduce_ext = os.path.splitext(file_to_reduce)
 
@@ -83,8 +84,7 @@ class Reduce(object):
         self.clang_args = []
         self.expected_output = []
         self.needs_stack_trace = False
-        self.creduce_flags = ["--tidy"]
-        self.creduce_flags = ["--n", str(core_number)]
+        self.creduce_flags = ["--tidy"] + creduce_flags
 
         self.read_clang_args(crash_script, file_to_reduce)
         self.read_expected_output()
@@ -412,13 +412,13 @@ fi
         print("Reduced command:", reduced_cmd)
 
     def run_creduce(self):
+        full_creduce_cmd = (
+            [creduce_cmd] + self.creduce_flags + [self.testfile, self.file_to_reduce]
+        )
         print("\nRunning C-Reduce...")
+        verbose_print(quote_cmd(full_creduce_cmd))
         try:
-            p = subprocess.Popen(
-                [creduce_cmd]
-                + self.creduce_flags
-                + [self.testfile, self.file_to_reduce]
-            )
+            p = subprocess.Popen(full_creduce_cmd)
             p.communicate()
         except KeyboardInterrupt:
             # Hack to kill C-Reduce because it jumps into its own pgid
@@ -458,26 +458,20 @@ def main():
         help="The path to the `creduce` executable. "
         "Required if `creduce` is not in PATH environment.",
     )
-    parser.add_argument(
-        "--n",
-        dest="core_number",
-        type=int,
-        default=max(4, multiprocessing.cpu_count() // 2),
-        help="Number of cores to use.",
-    )
     parser.add_argument("-v", "--verbose", action="store_true")
-    args = parser.parse_args()
-
+    args, creduce_flags = parser.parse_known_args()
     verbose = args.verbose
     llvm_bin = os.path.abspath(args.llvm_bin) if args.llvm_bin else None
     creduce_cmd = check_cmd("creduce", None, args.creduce)
     clang_cmd = check_cmd("clang", llvm_bin, args.clang)
-    core_number = args.core_number
 
     crash_script = check_file(args.crash_script[0])
     file_to_reduce = check_file(args.file_to_reduce[0])
 
-    r = Reduce(crash_script, file_to_reduce, core_number)
+    if "--n" not in creduce_flags:
+        creduce_flags += ["--n", str(max(4, multiprocessing.cpu_count() // 2))]
+
+    r = Reduce(crash_script, file_to_reduce, creduce_flags)
 
     r.simplify_clang_args()
     r.write_interestingness_test()
-- 
GitLab


From 179efe5abc745b5646efeb33fef86c55aa4fd7dc Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 14 May 2024 19:47:24 +0100
Subject: [PATCH 0771/1206] [LAA] Delay applying loop guards until after
 isSafeDependenceDistance.

Applying the loop guards to the distance may prevent
isSafeDependenceDistance from determining NoDep, unless loop guards are
also applied to the backedge-taken-count.

Instead of applying the guards to both Dist and the
backedge-taken-count, just apply them after handling
isSafeDependenceDistance and constant distances; there is no benefit to
applying the guards before then.

This fixes a regression flagged by @bjope due to
ecae3ed958481cba7d60868cf3504292f7f4fdf5.
---
 llvm/lib/Analysis/LoopAccessAnalysis.cpp      |  8 +--
 .../is-safe-dep-distance-with-loop-guards.ll  | 59 +------------------
 2 files changed, 5 insertions(+), 62 deletions(-)

diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index e92aa0265a1f..4ba2e1522210 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1991,9 +1991,6 @@ getDependenceDistanceStrideAndSize(
     return MemoryDepChecker::Dependence::Unknown;
   }
 
-  if (!isa<SCEVConstant, SCEVCouldNotCompute>(Dist))
-    Dist = SE.applyLoopGuards(Dist, InnermostLoop);
-
   uint64_t TypeByteSize = DL.getTypeAllocSize(ATy);
   bool HasSameSize =
       DL.getTypeStoreSizeInBits(ATy) == DL.getTypeStoreSizeInBits(BTy);
@@ -2019,7 +2016,7 @@ MemoryDepChecker::Dependence::DepType MemoryDepChecker::isDependent(
   if (std::holds_alternative<Dependence::DepType>(Res))
     return std::get<Dependence::DepType>(Res);
 
-  const auto &[Dist, StrideA, StrideB, TypeByteSize, AIsWrite, BIsWrite] =
+  auto &[Dist, StrideA, StrideB, TypeByteSize, AIsWrite, BIsWrite] =
       std::get<DepDistanceStrideAndSizeInfo>(Res);
   bool HasSameSize = TypeByteSize > 0;
 
@@ -2062,7 +2059,8 @@ MemoryDepChecker::Dependence::DepType MemoryDepChecker::isDependent(
       LLVM_DEBUG(dbgs() << "LAA: Strided accesses are independent\n");
       return Dependence::NoDep;
     }
-  }
+  } else
+    Dist = SE.applyLoopGuards(Dist, InnermostLoop);
 
   // Negative distances are not plausible dependencies.
   if (SE.isKnownNonPositive(Dist)) {
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/is-safe-dep-distance-with-loop-guards.ll b/llvm/test/Analysis/LoopAccessAnalysis/is-safe-dep-distance-with-loop-guards.ll
index bfa735df064d..9cc0a976c900 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/is-safe-dep-distance-with-loop-guards.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/is-safe-dep-distance-with-loop-guards.ll
@@ -6,21 +6,10 @@ target datalayout = "S16-p:16:16-i1:16-i8:8-i32:16-i64:16-i128:16"
 define void @safe_deps_1_due_to_dependence_distance(i16  %n, ptr %p) {
 ; CHECK-LABEL: 'safe_deps_1_due_to_dependence_distance'
 ; CHECK-NEXT:    loop:
-; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Memory dependences are safe
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Check 0:
-; CHECK-NEXT:        Comparing group ([[GRP1:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep.iv = getelementptr inbounds i32, ptr %p, i16 %iv
-; CHECK-NEXT:        Against group ([[GRP2:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %gep.off.iv = getelementptr i32, ptr %gep.off, i16 %iv
 ; CHECK-NEXT:      Grouped accesses:
-; CHECK-NEXT:        Group [[GRP1]]:
-; CHECK-NEXT:          (Low: %p High: ((4 * %n) + %p))
-; CHECK-NEXT:            Member: {%p,+,4}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP2]]:
-; CHECK-NEXT:          (Low: ((4 * %n) + %p) High: ((8 * %n) + %p))
-; CHECK-NEXT:            Member: {((4 * %n) + %p),+,4}<nw><%loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
@@ -52,57 +41,13 @@ exit:
 define void @safe_deps_2_due_to_dependence_distance(i16 %n, ptr %p3, i16 noundef %q, ptr %p1, ptr %p2) {
 ; CHECK-LABEL: 'safe_deps_2_due_to_dependence_distance'
 ; CHECK-NEXT:    loop:
-; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Memory dependences are safe
 ; CHECK-NEXT:      Dependences:
 ; CHECK-NEXT:      Run-time memory checks:
-; CHECK-NEXT:      Check 0:
-; CHECK-NEXT:        Comparing group ([[GRP3:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %arrayidx22 = getelementptr inbounds [2 x i32], ptr %alloca, i16 %iv
-; CHECK-NEXT:        Against group ([[GRP4:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %arrayidx33 = getelementptr inbounds i8, ptr %arrayidx22, i16 4
-; CHECK-NEXT:      Check 1:
-; CHECK-NEXT:        Comparing group ([[GRP3]]):
-; CHECK-NEXT:          %arrayidx22 = getelementptr inbounds [2 x i32], ptr %alloca, i16 %iv
-; CHECK-NEXT:        Against group ([[GRP5:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %arrayidx42 = getelementptr inbounds [2 x i32], ptr %arrayidx40, i16 %iv
-; CHECK-NEXT:      Check 2:
-; CHECK-NEXT:        Comparing group ([[GRP3]]):
-; CHECK-NEXT:          %arrayidx22 = getelementptr inbounds [2 x i32], ptr %alloca, i16 %iv
-; CHECK-NEXT:        Against group ([[GRP6:0x[0-9a-f]+]]):
-; CHECK-NEXT:          %arrayidx53 = getelementptr inbounds i8, ptr %arrayidx42, i16 4
-; CHECK-NEXT:      Check 3:
-; CHECK-NEXT:        Comparing group ([[GRP4]]):
-; CHECK-NEXT:          %arrayidx33 = getelementptr inbounds i8, ptr %arrayidx22, i16 4
-; CHECK-NEXT:        Against group ([[GRP5]]):
-; CHECK-NEXT:          %arrayidx42 = getelementptr inbounds [2 x i32], ptr %arrayidx40, i16 %iv
-; CHECK-NEXT:      Check 4:
-; CHECK-NEXT:        Comparing group ([[GRP4]]):
-; CHECK-NEXT:          %arrayidx33 = getelementptr inbounds i8, ptr %arrayidx22, i16 4
-; CHECK-NEXT:        Against group ([[GRP6]]):
-; CHECK-NEXT:          %arrayidx53 = getelementptr inbounds i8, ptr %arrayidx42, i16 4
-; CHECK-NEXT:      Check 5:
-; CHECK-NEXT:        Comparing group ([[GRP5]]):
-; CHECK-NEXT:          %arrayidx42 = getelementptr inbounds [2 x i32], ptr %arrayidx40, i16 %iv
-; CHECK-NEXT:        Against group ([[GRP6]]):
-; CHECK-NEXT:          %arrayidx53 = getelementptr inbounds i8, ptr %arrayidx42, i16 4
 ; CHECK-NEXT:      Grouped accesses:
-; CHECK-NEXT:        Group [[GRP3]]:
-; CHECK-NEXT:          (Low: %alloca High: (-4 + (8 * %n) + %alloca))
-; CHECK-NEXT:            Member: {%alloca,+,8}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP4]]:
-; CHECK-NEXT:          (Low: (4 + %alloca) High: ((8 * %n) + %alloca))
-; CHECK-NEXT:            Member: {(4 + %alloca),+,8}<nuw><%loop>
-; CHECK-NEXT:        Group [[GRP5]]:
-; CHECK-NEXT:          (Low: ((8 * %n) + %alloca) High: (-4 + (16 * %n) + %alloca))
-; CHECK-NEXT:            Member: {((8 * %n) + %alloca),+,8}<nw><%loop>
-; CHECK-NEXT:        Group [[GRP6]]:
-; CHECK-NEXT:          (Low: (4 + (8 * %n) + %alloca) High: ((16 * %n) + %alloca))
-; CHECK-NEXT:            Member: {(4 + (8 * %n) + %alloca),+,8}<nw><%loop>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
-; CHECK-NEXT:      {((8 * %n) + %alloca),+,8}<nw><%loop> Added Flags: <nusw>
-; CHECK-NEXT:      {(4 + (8 * %n) + %alloca),+,8}<nw><%loop> Added Flags: <nusw>
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Expressions re-written:
 ;
-- 
GitLab


From 302db1ab5a054103e411997fd75b2bf6ef7f448c Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 14 May 2024 13:57:34 -0500
Subject: [PATCH 0772/1206] [Offload] Do not link every target for JIT (#92013)

Summary:
The offload library supports basic JIT functionality, however we
currently link against every single target even though only AMDGPU and
NVPTX are supported. This somewhat bloats the dynamic library list, so
we should constrain it to what's actually used.
---
 offload/plugins-nextgen/CMakeLists.txt        |  1 -
 offload/plugins-nextgen/common/CMakeLists.txt | 10 +++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/offload/plugins-nextgen/CMakeLists.txt b/offload/plugins-nextgen/CMakeLists.txt
index d1079f8a3e9c..79f1c6dc66e8 100644
--- a/offload/plugins-nextgen/CMakeLists.txt
+++ b/offload/plugins-nextgen/CMakeLists.txt
@@ -16,7 +16,6 @@ add_subdirectory(common)
 function(add_target_library target_name lib_name)
   add_llvm_library(${target_name} STATIC
     LINK_COMPONENTS
-      ${LLVM_TARGETS_TO_BUILD}
       AggressiveInstCombine
       Analysis
       BinaryFormat
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index a470dcee6d85..6064468ea5b5 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -22,9 +22,13 @@ add_library(PluginCommon OBJECT
 add_dependencies(PluginCommon intrinsics_gen)
 
 # Only enable JIT for those targets that LLVM can support.
-string(TOUPPER "${LLVM_TARGETS_TO_BUILD}" TargetsSupported)
-foreach(Target ${TargetsSupported})
-	target_compile_definitions(PluginCommon PRIVATE "LIBOMPTARGET_JIT_${Target}")
+set(supported_jit_targets AMDGPU NVPTX)
+foreach(target IN LISTS supported_jit_targets)
+  if("${target}" IN_LIST LLVM_TARGETS_TO_BUILD)
+	  target_compile_definitions(PluginCommon PRIVATE "LIBOMPTARGET_JIT_${target}")
+    llvm_map_components_to_libnames(llvm_libs ${target})
+    target_link_libraries(PluginCommon PRIVATE ${llvm_libs})
+  endif()
 endforeach()
 
 # Include the RPC server from the `libc` project if availible.
-- 
GitLab


From 332f5e7113c409982e6429b135bb1a7055c11e77 Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Tue, 14 May 2024 12:24:12 -0700
Subject: [PATCH 0773/1206] [compiler-rt][ORC] Remove unused cmake variables
 (NFC) (#87742)

---
 compiler-rt/lib/orc/CMakeLists.txt | 36 ------------------------------
 1 file changed, 36 deletions(-)

diff --git a/compiler-rt/lib/orc/CMakeLists.txt b/compiler-rt/lib/orc/CMakeLists.txt
index 6bcbf05f0d8b..36f4349a240e 100644
--- a/compiler-rt/lib/orc/CMakeLists.txt
+++ b/compiler-rt/lib/orc/CMakeLists.txt
@@ -9,24 +9,6 @@ set(ORC_COMMON_SOURCES
   dlfcn_wrapper.cpp
   )
 
-# ORC runtime library implementation files for all ORC architectures.s
-set(ALL_ORC_SOURCES
-  ${ORC_COMMON_SOURCES}
-  coff_platform.cpp
-  coff_platform.per_jd.cpp
-  elfnix_platform.cpp
-  macho_platform.cpp
-  )
-
-# Implementation files for all ORC architectures.
-set(ALL_ORC_ASM_SOURCES
-  macho_tlv.x86-64.S
-  macho_tlv.arm64.S
-  elfnix_tls.x86-64.S
-  elfnix_tls.aarch64.S
-  elfnix_tls.ppc64.S
-  )
-
 # Common implementation headers will go here.
 set(ORC_COMMON_IMPL_HEADERS
   adt.h
@@ -41,24 +23,6 @@ set(ORC_COMMON_IMPL_HEADERS
   wrapper_function_utils.h
   )
 
-# Implementation headers for all ORC architectures.
-set(ALL_ORC_IMPL_HEADERS
-  ${ORC_COMMON_IMPL_HEADERS}
-  macho_platform.h
-  coff_platform.h
-  elfnix_platform.h
-  )
-
-# Create list of all source files for
-# consumption by tests.
-set(ORC_ALL_SOURCE_FILES
-  ${ALL_ORC_SOURCES}
-  ${ALL_ORC_ASM_SOURCES}
-  ${ALL_ORC_IMPL_HEADERS}
-  )
-
-list(REMOVE_DUPLICATES ORC_ALL_SOURCE_FILES)
-
 # Now put it all together...
 include_directories(..)
 include_directories(../../include)
-- 
GitLab


From 18ba0cc26e079c7150feb5eaf631d1834e49ca1a Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Tue, 14 May 2024 12:24:32 -0700
Subject: [PATCH 0774/1206] [github] Add keith to bazel owners (NFC) (#92164)

I'm interested in being CC'd on these changes
---
 .github/CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index e25b2f50b1b4..6adae58832e1 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -132,7 +132,7 @@ clang/test/AST/Interp/ @tbaederr
 /bolt/ @aaupov @maksfb @rafaelauler @ayermolo @dcci
 
 # Bazel build system.
-/utils/bazel/ @rupprecht
+/utils/bazel/ @rupprecht @keith
 
 # InstallAPI and TextAPI
 /llvm/**/TextAPI/ @cyndyishida
-- 
GitLab


From cf5db39907f7ec66c084a07c6eb9a83eef506b3c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 14 May 2024 20:26:14 +0100
Subject: [PATCH 0775/1206] [LV] Add tests with trip counts containing UDIV
 expressions.

Add test cases for
https://github.com/llvm/llvm-project/issues/89958.
---
 .../trip-count-expansion-may-introduce-ub.ll  | 368 ++++++++++++++++++
 1 file changed, 368 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll

diff --git a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
new file mode 100644
index 000000000000..85fad6fb3632
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
@@ -0,0 +1,368 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+; Test cases with trip counts containing UDIV expressions for
+; https://github.com/llvm/llvm-project/issues/89958.
+
+define i64 @multi_exit_1_exit_count_with_udiv_in_header(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_1_exit_count_with_udiv_in_header(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[UMIN]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[D:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %d = udiv i64 42, %N
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
+define i64 @multi_exit_2_exit_count_with_udiv_in_block_executed_unconditionally(ptr %A, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_2_exit_count_with_udiv_in_block_executed_unconditionally(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[UMIN]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], <i32 10, i32 10, i32 10, i32 10>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP7]], i32 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP9]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; CHECK:       pred.store.continue:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP7]], i32 1
+; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
+; CHECK:       pred.store.if1:
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP12]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; CHECK:       pred.store.continue2:
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP7]], i32 2
+; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
+; CHECK:       pred.store.if3:
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP15]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
+; CHECK:       pred.store.continue4:
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP7]], i32 3
+; CHECK-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
+; CHECK:       pred.store.if5:
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP17]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP18]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
+; CHECK:       pred.store.continue6:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[C_2:%.*]] = icmp eq i32 [[L]], 10
+; CHECK-NEXT:    br i1 [[C_2]], label [[THEN:%.*]], label [[CONTINUE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    br label [[CONTINUE]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[D:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[CONTINUE]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+  %l = load i32, ptr %gep
+  %c.2 = icmp eq i32 %l, 10
+  br i1 %c.2, label %then, label %continue
+
+then:
+  store i32 1, ptr %gep
+  br label %continue
+
+continue:
+  %d = udiv i64 42, %N
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %continue ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
+define i64 @multi_exit_3_exit_count_with_udiv_in_block_executed_conditionally(ptr %A, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_3_exit_count_with_udiv_in_block_executed_conditionally(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[C_2:%.*]] = icmp eq i32 [[L]], 10
+; CHECK-NEXT:    br i1 [[C_2]], label [[THEN:%.*]], label [[LOOP_LATCH]]
+; CHECK:       then:
+; CHECK-NEXT:    [[D:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_HEADER]], label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[THEN]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+  %l = load i32, ptr %gep
+  %c.2 = icmp eq i32 %l, 10
+  br i1 %c.2, label %then, label %loop.latch
+
+then:
+  %d = udiv i64 42, %N
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.latch, label %exit
+
+loop.latch:
+  store i32 1, ptr %gep
+  %iv.next = add i64 %iv, 1
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %then ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
+; FIXME: Currently miscompiled as we unconditionally execute udiv after
+; vectorization.
+define i64 @multi_exit_4_exit_count_with_udiv_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_in_latch(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[UMIN]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[D:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %d = udiv i64 42, %N
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
+define void @single_exit_tc_with_udiv(ptr %dst, i64 %N) {
+; CHECK-LABEL: define void @single_exit_tc_with_udiv(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[D:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %iv.next = add i64 %iv, 1
+  %d = udiv i64 42, %N
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+;.
-- 
GitLab


From 4cfe347c107485aab6bd003f99ab06aac242b0fd Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 14 May 2024 12:58:49 -0700
Subject: [PATCH 0776/1206] [clangd] Fix -Wunused-but-set-variable after #82396

---
 clang-tools-extra/clangd/refactor/Rename.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang-tools-extra/clangd/refactor/Rename.cpp b/clang-tools-extra/clangd/refactor/Rename.cpp
index 75b30e66d637..c0fc4453a3fc 100644
--- a/clang-tools-extra/clangd/refactor/Rename.cpp
+++ b/clang-tools-extra/clangd/refactor/Rename.cpp
@@ -1090,11 +1090,10 @@ llvm::Expected<RenameResult> rename(const RenameInputs &RInputs) {
     return MainFileRenameEdit.takeError();
 
   llvm::DenseSet<Range> RenamedRanges;
-  if (const auto *MD = dyn_cast<ObjCMethodDecl>(&RenameDecl)) {
+  if (!isa<ObjCMethodDecl>(RenameDecl)) {
     // TODO: Insert the ranges from the ObjCMethodDecl/ObjCMessageExpr selector
     // pieces which are being renamed. This will require us to make changes to
     // locateDeclAt to preserve this AST node.
-  } else {
     RenamedRanges.insert(CurrentIdentifier);
   }
 
-- 
GitLab


From d4c86e7f3ea298b259e673142470a7b838f5f302 Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara@apple.com>
Date: Tue, 14 May 2024 13:00:19 -0700
Subject: [PATCH 0777/1206] [AArch64][SME] Fix frame lowering not using a base
 pointer for SME functions. (#91643)

The existing code is checking for the presence of the +sve subtarget
feature
when deciding to use a base pointer for the function, but this check
doesn't
work when only +sme is used.

rdar://126878490
---
 .../Target/AArch64/AArch64RegisterInfo.cpp    |    3 +-
 .../AArch64/framelayout-sve-basepointer.mir   |   31 +-
 .../CodeGen/AArch64/sme-framelower-use-bp.ll  | 1049 +++++++++++++++++
 3 files changed, 1081 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/sme-framelower-use-bp.ll

diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index ad29003f1e81..5a5a18edb12e 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -552,7 +552,8 @@ bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
     if (hasStackRealignment(MF))
       return true;
 
-    if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) {
+    auto &ST = MF.getSubtarget<AArch64Subtarget>();
+    if (ST.hasSVE() || ST.isStreaming()) {
       const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
       // Frames that have variable sized objects and scalable SVE objects,
       // should always use a basepointer.
diff --git a/llvm/test/CodeGen/AArch64/framelayout-sve-basepointer.mir b/llvm/test/CodeGen/AArch64/framelayout-sve-basepointer.mir
index 265c474fbc5d..26b7ca3bf8c0 100644
--- a/llvm/test/CodeGen/AArch64/framelayout-sve-basepointer.mir
+++ b/llvm/test/CodeGen/AArch64/framelayout-sve-basepointer.mir
@@ -1,4 +1,11 @@
-# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass=prologepilog -mattr=+sve %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass=prologepilog %s -o - | FileCheck %s
+--- |
+    define void @hasBasepointer() #0 { ret void }
+    define void @hasBasepointer_sme_streaming() #1 { ret void }
+
+    attributes #0 = { "target-features"="+sve" }
+    attributes #1 = { "target-features"="+sme" "aarch64_pstate_sm_enabled" }
+...
 ---
 # This test verifies that the basepointer is available in presence of SVE stack objects.
 name: hasBasepointer
@@ -21,3 +28,25 @@ body: |
     STRXui $x0, %stack.1, 0
     RET_ReallyLR
 ...
+---
+# Likewise with only SME with a streaming function.
+name: hasBasepointer_sme_streaming
+# CHECK-LABEL: name: hasBasepointer_sme_streaming
+# CHECK: bb.0:
+# CHECK:      $sp = frame-setup SUBXri $sp, 16, 0
+# CHECK-NEXT: $sp = frame-setup ADDVL_XXI $sp, -1
+# CHECK-NEXT: $x19 = ADDXri $sp, 0, 0
+# CHECK:      STRXui $x0, $x19, 0
+tracksRegLiveness: true
+frameInfo:
+  isFrameAddressTaken: true
+stack:
+  - { id: 0, type: variable-sized,  alignment: 1 }
+  - { id: 1, name: '', size: 16, alignment: 8 }
+  - { id: 2, stack-id: scalable-vector, size: 16, alignment: 16 }
+body: |
+  bb.0:
+    liveins: $x0
+    STRXui $x0, %stack.1, 0
+    RET_ReallyLR
+...
diff --git a/llvm/test/CodeGen/AArch64/sme-framelower-use-bp.ll b/llvm/test/CodeGen/AArch64/sme-framelower-use-bp.ll
new file mode 100644
index 000000000000..7db0cf7f18c5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-framelower-use-bp.ll
@@ -0,0 +1,1049 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64 -O0 -mattr=+sme < %s | FileCheck %s
+
+target triple = "aarch64-linux-gnu"
+
+declare void @llvm.trap() #0
+
+; This test checks that we don't assert/crash due to not being able to reach the
+; emergency spill slot by ensuring that we use a BP for streaming functions.
+
+define void @quux() #1 {
+; CHECK-LABEL: quux:
+; CHECK:       // %bb.0: // %prelude
+; CHECK-NEXT:    stp x29, x30, [sp, #-96]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x28, x27, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x26, x25, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x24, x23, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x22, x21, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #400
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    mov x19, sp
+; CHECK-NEXT:    .cfi_def_cfa w29, 96
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w21, -24
+; CHECK-NEXT:    .cfi_offset w22, -32
+; CHECK-NEXT:    .cfi_offset w23, -40
+; CHECK-NEXT:    .cfi_offset w24, -48
+; CHECK-NEXT:    .cfi_offset w25, -56
+; CHECK-NEXT:    .cfi_offset w26, -64
+; CHECK-NEXT:    .cfi_offset w27, -72
+; CHECK-NEXT:    .cfi_offset w28, -80
+; CHECK-NEXT:    .cfi_offset w30, -88
+; CHECK-NEXT:    .cfi_offset w29, -96
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mul x9, x8, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    subs x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    str x8, [x19, #384]
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    strh w8, [x19, #394]
+; CHECK-NEXT:    str w8, [x19, #396]
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    cbz x8, .LBB0_2
+; CHECK-NEXT:    b .LBB0_1
+; CHECK-NEXT:  .LBB0_1: // %save.za
+; CHECK-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    b .LBB0_2
+; CHECK-NEXT:  .LBB0_2: // %bb
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    zero {za}
+; CHECK-NEXT:    mov w9, #15 // =0xf
+; CHECK-NEXT:    // implicit-def: $x8
+; CHECK-NEXT:    mov w8, w9
+; CHECK-NEXT:    mov x9, x8
+; CHECK-NEXT:    incd x9
+; CHECK-NEXT:    mov w0, w9
+; CHECK-NEXT:    // implicit-def: $x9
+; CHECK-NEXT:    mov w9, w0
+; CHECK-NEXT:    and x14, x9, #0x70
+; CHECK-NEXT:    str x14, [x19, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x14
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #24] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x14
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    addvl x9, x8, #1
+; CHECK-NEXT:    mov w0, w9
+; CHECK-NEXT:    // implicit-def: $x9
+; CHECK-NEXT:    mov w9, w0
+; CHECK-NEXT:    and x10, x9, #0x3f0
+; CHECK-NEXT:    str x10, [x19, #40] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x10
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x10
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #56] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x14
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x14
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #72] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x10
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x10
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #88] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x14
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x14
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #104] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x10
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #112] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x10
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #120] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x14
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #128] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x14
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #136] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x10
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #144] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x10
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    str x9, [x19, #152] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, #16
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, #16
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, #16
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, #16
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, #16
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, x14
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x9, x9, #16
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    addvl x9, x8, #2
+; CHECK-NEXT:    mov w0, w9
+; CHECK-NEXT:    // implicit-def: $x9
+; CHECK-NEXT:    mov w9, w0
+; CHECK-NEXT:    and x9, x9, #0x7f0
+; CHECK-NEXT:    mov x10, sp
+; CHECK-NEXT:    subs x10, x10, x9
+; CHECK-NEXT:    and x10, x10, #0xffffffffffffffe0
+; CHECK-NEXT:    mov sp, x10
+; CHECK-NEXT:    mov x2, sp
+; CHECK-NEXT:    subs x10, x2, #16
+; CHECK-NEXT:    mov sp, x10
+; CHECK-NEXT:    str x10, [x19, #160] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x10, sp
+; CHECK-NEXT:    subs x11, x10, x14
+; CHECK-NEXT:    mov sp, x11
+; CHECK-NEXT:    mov x10, x11
+; CHECK-NEXT:    str x10, [x19, #168] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x0, sp
+; CHECK-NEXT:    subs x10, x0, #16
+; CHECK-NEXT:    mov sp, x10
+; CHECK-NEXT:    str x10, [x19, #176] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x17, sp
+; CHECK-NEXT:    subs x10, x17, #16
+; CHECK-NEXT:    mov sp, x10
+; CHECK-NEXT:    str x10, [x19, #184] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x10, sp
+; CHECK-NEXT:    subs x10, x10, x14
+; CHECK-NEXT:    str x10, [x19, #360] // 8-byte Folded Spill
+; CHECK-NEXT:    mov sp, x10
+; CHECK-NEXT:    str x10, [x19, #192] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x15, sp
+; CHECK-NEXT:    subs x10, x15, #16
+; CHECK-NEXT:    mov sp, x10
+; CHECK-NEXT:    str x10, [x19, #200] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x13, sp
+; CHECK-NEXT:    subs x10, x13, #16
+; CHECK-NEXT:    mov sp, x10
+; CHECK-NEXT:    str x10, [x19, #208] // 8-byte Folded Spill
+; CHECK-NEXT:    incw x8
+; CHECK-NEXT:    mov w1, w8
+; CHECK-NEXT:    // implicit-def: $x8
+; CHECK-NEXT:    mov w8, w1
+; CHECK-NEXT:    and x12, x8, #0xf0
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    subs x10, x8, x12
+; CHECK-NEXT:    mov sp, x10
+; CHECK-NEXT:    mov x8, x10
+; CHECK-NEXT:    str x8, [x19, #216] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    subs x8, x8, x12
+; CHECK-NEXT:    str x8, [x19, #368] // 8-byte Folded Spill
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    str x8, [x19, #224] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    subs x8, x8, x9
+; CHECK-NEXT:    and x8, x8, #0xffffffffffffffe0
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    str x8, [x19, #232] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    subs x8, x8, x9
+; CHECK-NEXT:    and x8, x8, #0xffffffffffffffe0
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    str x8, [x19, #240] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #336] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #344] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x24, sp
+; CHECK-NEXT:    subs x8, x24, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x7, sp
+; CHECK-NEXT:    subs x8, x7, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x27, sp
+; CHECK-NEXT:    subs x8, x27, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x26, sp
+; CHECK-NEXT:    subs x8, x26, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x1, sp
+; CHECK-NEXT:    subs x8, x1, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    subs x8, x9, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x20, sp
+; CHECK-NEXT:    subs x8, x20, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x16, sp
+; CHECK-NEXT:    subs x8, x16, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #248] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x5, sp
+; CHECK-NEXT:    subs x8, x5, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x12, sp
+; CHECK-NEXT:    subs x8, x12, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x22, sp
+; CHECK-NEXT:    subs x8, x22, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x25, sp
+; CHECK-NEXT:    subs x8, x25, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x30, sp
+; CHECK-NEXT:    subs x8, x30, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #296] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #328] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #264] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #256] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #272] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #312] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #280] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #304] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x6, sp
+; CHECK-NEXT:    subs x8, x6, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x21, sp
+; CHECK-NEXT:    subs x8, x21, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    str x8, [x19, #352] // 8-byte Folded Spill
+; CHECK-NEXT:    subs x8, x8, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x28, sp
+; CHECK-NEXT:    subs x8, x28, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    subs x4, x8, x14
+; CHECK-NEXT:    mov sp, x4
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    subs x3, x8, x14
+; CHECK-NEXT:    mov sp, x3
+; CHECK-NEXT:    mov x23, sp
+; CHECK-NEXT:    subs x8, x23, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x18, sp
+; CHECK-NEXT:    subs x8, x18, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov x14, sp
+; CHECK-NEXT:    subs x8, x14, #16
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    sturb w8, [x9, #-16]
+; CHECK-NEXT:    ldr x9, [x19, #248] // 8-byte Folded Reload
+; CHECK-NEXT:    sturb w8, [x9, #-16]
+; CHECK-NEXT:    ldr x9, [x19, #296] // 8-byte Folded Reload
+; CHECK-NEXT:    sturb w8, [x30, #-16]
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    str x8, [x19, #376] // 8-byte Folded Spill
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldur x8, [x20, #-16]
+; CHECK-NEXT:    ldur x9, [x27, #-16]
+; CHECK-NEXT:    add x30, x8, x9, lsl #2
+; CHECK-NEXT:    ldur x8, [x1, #-16]
+; CHECK-NEXT:    subs x8, x8, #1
+; CHECK-NEXT:    ldur x9, [x16, #-16]
+; CHECK-NEXT:    mul x8, x8, x9
+; CHECK-NEXT:    ldr x9, [x19, #328] // 8-byte Folded Reload
+; CHECK-NEXT:    add x30, x30, x8, lsl #2
+; CHECK-NEXT:    ldr x8, [x19, #296] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x30, [x8, #-16]
+; CHECK-NEXT:    ldr x8, [x19, #376] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldur x8, [x5, #-16]
+; CHECK-NEXT:    ldur x9, [x26, #-16]
+; CHECK-NEXT:    add x30, x8, x9, lsl #2
+; CHECK-NEXT:    ldur x8, [x1, #-16]
+; CHECK-NEXT:    subs x8, x8, #1
+; CHECK-NEXT:    ldur x9, [x12, #-16]
+; CHECK-NEXT:    mul x8, x8, x9
+; CHECK-NEXT:    ldr x9, [x19, #264] // 8-byte Folded Reload
+; CHECK-NEXT:    add x30, x30, x8, lsl #2
+; CHECK-NEXT:    ldr x8, [x19, #328] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x30, [x8, #-16]
+; CHECK-NEXT:    ldr x8, [x19, #376] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldur x8, [x22, #-16]
+; CHECK-NEXT:    ldur x9, [x27, #-16]
+; CHECK-NEXT:    add x30, x8, x9, lsl #2
+; CHECK-NEXT:    ldur x8, [x26, #-16]
+; CHECK-NEXT:    subs x8, x8, #1
+; CHECK-NEXT:    ldur x9, [x25, #-16]
+; CHECK-NEXT:    mul x8, x8, x9
+; CHECK-NEXT:    ldr x9, [x19, #256] // 8-byte Folded Reload
+; CHECK-NEXT:    add x30, x30, x8, lsl #2
+; CHECK-NEXT:    ldr x8, [x19, #264] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x30, [x8, #-16]
+; CHECK-NEXT:    ldr x8, [x19, #272] // 8-byte Folded Reload
+; CHECK-NEXT:    mov w30, #32 // =0x20
+; CHECK-NEXT:    // kill: def $lr killed $w30
+; CHECK-NEXT:    stur x30, [x9, #-16]
+; CHECK-NEXT:    ldr x9, [x19, #312] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x30, [x8, #-16]
+; CHECK-NEXT:    ldr x8, [x19, #376] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldur x8, [x1, #-16]
+; CHECK-NEXT:    lsl x8, x8, #5
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldr x9, [x19, #280] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x8, [x19, #376] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x30, [x16, #-16]
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldur x8, [x27, #-16]
+; CHECK-NEXT:    subs x8, x8, #1
+; CHECK-NEXT:    lsr x8, x8, #5
+; CHECK-NEXT:    add x8, x8, #1
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldur x8, [x20, #-16]
+; CHECK-NEXT:    str x8, [x19, #288] // 8-byte Folded Spill
+; CHECK-NEXT:    ldr x8, [x19, #312] // 8-byte Folded Reload
+; CHECK-NEXT:    ldur x9, [x9, #-16]
+; CHECK-NEXT:    ldur x8, [x8, #-16]
+; CHECK-NEXT:    mul x9, x9, x8
+; CHECK-NEXT:    ldr x8, [x19, #288] // 8-byte Folded Reload
+; CHECK-NEXT:    add x8, x8, x9, lsl #2
+; CHECK-NEXT:    ldr x9, [x19, #296] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldr x9, [x19, #304] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x8, [x19, #376] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x30, [x12, #-16]
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldur x8, [x26, #-16]
+; CHECK-NEXT:    subs x8, x8, #1
+; CHECK-NEXT:    lsr x8, x8, #5
+; CHECK-NEXT:    add x8, x8, #1
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldur x8, [x5, #-16]
+; CHECK-NEXT:    str x8, [x19, #320] // 8-byte Folded Spill
+; CHECK-NEXT:    ldr x8, [x19, #312] // 8-byte Folded Reload
+; CHECK-NEXT:    ldur x9, [x9, #-16]
+; CHECK-NEXT:    ldur x8, [x8, #-16]
+; CHECK-NEXT:    mul x9, x9, x8
+; CHECK-NEXT:    ldr x8, [x19, #320] // 8-byte Folded Reload
+; CHECK-NEXT:    add x8, x8, x9, lsl #2
+; CHECK-NEXT:    ldr x9, [x19, #328] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldr x9, [x19, #352] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x8, [x19, #376] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x8, [x6, #-16]
+; CHECK-NEXT:    stur x8, [x6, #-16]
+; CHECK-NEXT:    stur x8, [x21, #-16]
+; CHECK-NEXT:    stur x8, [x21, #-16]
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldur x8, [x27, #-16]
+; CHECK-NEXT:    ldur x9, [x21, #-16]
+; CHECK-NEXT:    subs x8, x8, x9
+; CHECK-NEXT:    ldr x9, [x19, #336] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x8, [x9, #-16]
+; CHECK-NEXT:    ldr x8, [x19, #344] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x30, [x8, #-16]
+; CHECK-NEXT:    ldr x8, [x19, #352] // 8-byte Folded Reload
+; CHECK-NEXT:    ldur x9, [x9, #-16]
+; CHECK-NEXT:    stur x9, [x8, #-16]
+; CHECK-NEXT:    ldr x8, [x19, #376] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x8, [x28, #-16]
+; CHECK-NEXT:    ldur x8, [x26, #-16]
+; CHECK-NEXT:    ldur x9, [x6, #-16]
+; CHECK-NEXT:    subs x8, x8, x9
+; CHECK-NEXT:    ldr x9, [x19, #360] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x8, [x24, #-16]
+; CHECK-NEXT:    ldr x8, [x19, #368] // 8-byte Folded Reload
+; CHECK-NEXT:    stur x30, [x7, #-16]
+; CHECK-NEXT:    ldr x7, [x19, #376] // 8-byte Folded Reload
+; CHECK-NEXT:    ldur x24, [x24, #-16]
+; CHECK-NEXT:    stur x24, [x28, #-16]
+; CHECK-NEXT:    ldur x24, [x21, #-16]
+; CHECK-NEXT:    ldur x27, [x27, #-16]
+; CHECK-NEXT:    whilelt pn8.s, x24, x27, vlx2
+; CHECK-NEXT:    str pn8, [x4]
+; CHECK-NEXT:    ldur x24, [x6, #-16]
+; CHECK-NEXT:    ldur x26, [x26, #-16]
+; CHECK-NEXT:    whilelt pn8.s, x24, x26, vlx2
+; CHECK-NEXT:    str pn8, [x3]
+; CHECK-NEXT:    stur x7, [x23, #-16]
+; CHECK-NEXT:    ldur x22, [x22, #-16]
+; CHECK-NEXT:    ldur x24, [x21, #-16]
+; CHECK-NEXT:    add x22, x22, x24, lsl #2
+; CHECK-NEXT:    ldur x24, [x6, #-16]
+; CHECK-NEXT:    ldur x25, [x25, #-16]
+; CHECK-NEXT:    mul x24, x24, x25
+; CHECK-NEXT:    add x22, x22, x24, lsl #2
+; CHECK-NEXT:    stur x22, [x23, #-16]
+; CHECK-NEXT:    zero {za}
+; CHECK-NEXT:    stur x7, [x18, #-16]
+; CHECK-NEXT:    ldur x20, [x20, #-16]
+; CHECK-NEXT:    ldur x21, [x21, #-16]
+; CHECK-NEXT:    ldur x22, [x1, #-16]
+; CHECK-NEXT:    mul x21, x21, x22
+; CHECK-NEXT:    add x20, x20, x21, lsl #2
+; CHECK-NEXT:    stur x20, [x18, #-16]
+; CHECK-NEXT:    stur x7, [x14, #-16]
+; CHECK-NEXT:    ldur x5, [x5, #-16]
+; CHECK-NEXT:    ldur x6, [x6, #-16]
+; CHECK-NEXT:    ldur x7, [x1, #-16]
+; CHECK-NEXT:    mul x6, x6, x7
+; CHECK-NEXT:    add x5, x5, x6, lsl #2
+; CHECK-NEXT:    stur x5, [x14, #-16]
+; CHECK-NEXT:    ldur x1, [x1, #-16]
+; CHECK-NEXT:    ldr p1, [x4]
+; CHECK-NEXT:    ldur x18, [x18, #-16]
+; CHECK-NEXT:    ldur x16, [x16, #-16]
+; CHECK-NEXT:    lsr x16, x16, #2
+; CHECK-NEXT:    ldr p0, [x3]
+; CHECK-NEXT:    ldur x14, [x14, #-16]
+; CHECK-NEXT:    ldur x12, [x12, #-16]
+; CHECK-NEXT:    lsr x12, x12, #2
+; CHECK-NEXT:    stur x1, [x2, #-16]
+; CHECK-NEXT:    str p1, [x11]
+; CHECK-NEXT:    stur x18, [x0, #-16]
+; CHECK-NEXT:    stur x16, [x17, #-16]
+; CHECK-NEXT:    str p0, [x9]
+; CHECK-NEXT:    stur x14, [x15, #-16]
+; CHECK-NEXT:    stur x12, [x13, #-16]
+; CHECK-NEXT:    ldr p0, [x11]
+; CHECK-NEXT:    mov p8.b, p0.b
+; CHECK-NEXT:    pext { p3.s, p4.s }, pn8[0]
+; CHECK-NEXT:    mov p0.b, p3.b
+; CHECK-NEXT:    ptrue p2.s
+; CHECK-NEXT:    and p0.b, p0/z, p0.b, p2.b
+; CHECK-NEXT:    mov p1.b, p4.b
+; CHECK-NEXT:    and p1.b, p1/z, p1.b, p2.b
+; CHECK-NEXT:    mov x11, x10
+; CHECK-NEXT:    incd x11
+; CHECK-NEXT:    str p1, [x11]
+; CHECK-NEXT:    str p0, [x10]
+; CHECK-NEXT:    ldr p0, [x9]
+; CHECK-NEXT:    mov p8.b, p0.b
+; CHECK-NEXT:    pext { p3.s, p4.s }, pn8[0]
+; CHECK-NEXT:    mov p0.b, p3.b
+; CHECK-NEXT:    and p0.b, p0/z, p0.b, p2.b
+; CHECK-NEXT:    mov p1.b, p4.b
+; CHECK-NEXT:    and p1.b, p1/z, p1.b, p2.b
+; CHECK-NEXT:    mov x9, x8
+; CHECK-NEXT:    incd x9
+; CHECK-NEXT:    str p1, [x9]
+; CHECK-NEXT:    str p0, [x8]
+; CHECK-NEXT:    b .LBB0_3
+; CHECK-NEXT:  .LBB0_3: // %bb178
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr x9, [x19, #160] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x8, [x19, #56] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x10, [x19, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x11, [x19, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x12, [x19, #24] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x13, [x19, #240] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x14, [x19, #232] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x17, [x19, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x18, [x19, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x0, [x19, #72] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x1, [x19, #64] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x2, [x19, #216] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x3, [x19, #120] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x4, [x19, #112] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x5, [x19, #104] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x6, [x19, #96] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x7, [x19, #224] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x20, [x19, #152] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x21, [x19, #144] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x22, [x19, #136] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x23, [x19, #128] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x16, [x19, #200] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x15, [x19, #208] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x24, [x19, #192] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x26, [x19, #176] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x25, [x19, #184] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr x27, [x19, #168] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr p0, [x27]
+; CHECK-NEXT:    ldr x27, [x26]
+; CHECK-NEXT:    mov p8.b, p0.b
+; CHECK-NEXT:    ld1w { z16.s, z24.s }, pn8/z, [x27]
+; CHECK-NEXT:    mov z0.d, z16.d
+; CHECK-NEXT:    mov z1.d, z24.d
+; CHECK-NEXT:    ptrue p2.s
+; CHECK-NEXT:    str p2, [x29, #-1, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    st1w { z1.s }, p2, [x14, #1, mul vl]
+; CHECK-NEXT:    st1w { z0.s }, p2, [x14]
+; CHECK-NEXT:    ldr x27, [x25]
+; CHECK-NEXT:    ldr x25, [x26]
+; CHECK-NEXT:    add x25, x25, x27, lsl #2
+; CHECK-NEXT:    str x25, [x26]
+; CHECK-NEXT:    ldr p0, [x24]
+; CHECK-NEXT:    ldr x24, [x16]
+; CHECK-NEXT:    mov p8.b, p0.b
+; CHECK-NEXT:    ld1w { z16.s, z24.s }, pn8/z, [x24]
+; CHECK-NEXT:    mov z0.d, z16.d
+; CHECK-NEXT:    mov z1.d, z24.d
+; CHECK-NEXT:    st1w { z1.s }, p2, [x13, #1, mul vl]
+; CHECK-NEXT:    st1w { z0.s }, p2, [x13]
+; CHECK-NEXT:    ldr x24, [x15]
+; CHECK-NEXT:    ldr x15, [x16]
+; CHECK-NEXT:    add x15, x15, x24, lsl #2
+; CHECK-NEXT:    str x15, [x16]
+; CHECK-NEXT:    mov x16, x2
+; CHECK-NEXT:    incd x16
+; CHECK-NEXT:    ldr p1, [x2]
+; CHECK-NEXT:    mov x15, x7
+; CHECK-NEXT:    incd x15
+; CHECK-NEXT:    ldr p0, [x7]
+; CHECK-NEXT:    ld1w { z1.s }, p2/z, [x14]
+; CHECK-NEXT:    ld1w { z0.s }, p2/z, [x13]
+; CHECK-NEXT:    str p1, [x23]
+; CHECK-NEXT:    str p0, [x22]
+; CHECK-NEXT:    st1w { z1.s }, p2, [x21]
+; CHECK-NEXT:    st1w { z0.s }, p2, [x20]
+; CHECK-NEXT:    ldr p0, [x23]
+; CHECK-NEXT:    ldr p1, [x22]
+; CHECK-NEXT:    ld1w { z0.s }, p2/z, [x21]
+; CHECK-NEXT:    ld1w { z1.s }, p2/z, [x20]
+; CHECK-NEXT:    fmopa za0.s, p0/m, p1/m, z0.s, z1.s
+; CHECK-NEXT:    ldr p1, [x16]
+; CHECK-NEXT:    ldr p0, [x7]
+; CHECK-NEXT:    ld1w { z1.s }, p2/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1w { z0.s }, p2/z, [x13]
+; CHECK-NEXT:    str p1, [x6]
+; CHECK-NEXT:    str p0, [x5]
+; CHECK-NEXT:    st1w { z1.s }, p2, [x4]
+; CHECK-NEXT:    st1w { z0.s }, p2, [x3]
+; CHECK-NEXT:    ldr p0, [x6]
+; CHECK-NEXT:    ldr p1, [x5]
+; CHECK-NEXT:    ld1w { z0.s }, p2/z, [x4]
+; CHECK-NEXT:    ld1w { z1.s }, p2/z, [x3]
+; CHECK-NEXT:    fmopa za1.s, p0/m, p1/m, z0.s, z1.s
+; CHECK-NEXT:    ldr p1, [x2]
+; CHECK-NEXT:    ldr p0, [x15]
+; CHECK-NEXT:    ld1w { z1.s }, p2/z, [x14]
+; CHECK-NEXT:    ld1w { z0.s }, p2/z, [x13, #1, mul vl]
+; CHECK-NEXT:    str p1, [x1]
+; CHECK-NEXT:    str p0, [x0]
+; CHECK-NEXT:    st1w { z1.s }, p2, [x18]
+; CHECK-NEXT:    st1w { z0.s }, p2, [x17]
+; CHECK-NEXT:    ldr p0, [x1]
+; CHECK-NEXT:    ldr p1, [x0]
+; CHECK-NEXT:    ld1w { z0.s }, p2/z, [x18]
+; CHECK-NEXT:    ld1w { z1.s }, p2/z, [x17]
+; CHECK-NEXT:    fmopa za2.s, p0/m, p1/m, z0.s, z1.s
+; CHECK-NEXT:    ldr p1, [x16]
+; CHECK-NEXT:    ldr p0, [x15]
+; CHECK-NEXT:    ld1w { z1.s }, p2/z, [x14, #1, mul vl]
+; CHECK-NEXT:    ld1w { z0.s }, p2/z, [x13, #1, mul vl]
+; CHECK-NEXT:    str p1, [x12]
+; CHECK-NEXT:    str p0, [x11]
+; CHECK-NEXT:    st1w { z1.s }, p2, [x10]
+; CHECK-NEXT:    st1w { z0.s }, p2, [x8]
+; CHECK-NEXT:    ldr p0, [x12]
+; CHECK-NEXT:    ldr p1, [x11]
+; CHECK-NEXT:    ld1w { z0.s }, p2/z, [x10]
+; CHECK-NEXT:    ld1w { z1.s }, p2/z, [x8]
+; CHECK-NEXT:    fmopa za3.s, p0/m, p1/m, z0.s, z1.s
+; CHECK-NEXT:    ldr x8, [x9]
+; CHECK-NEXT:    subs x8, x8, #1
+; CHECK-NEXT:    str x8, [x9]
+; CHECK-NEXT:    b .LBB0_3
+bb:
+  %alloca = alloca <vscale x 16 x i1>, align 2
+  %alloca1 = alloca <vscale x 16 x i1>, align 2
+  %alloca2 = alloca <vscale x 4 x float>, align 16
+  %alloca3 = alloca <vscale x 4 x float>, align 16
+  %alloca4 = alloca <vscale x 16 x i1>, align 2
+  %alloca5 = alloca <vscale x 16 x i1>, align 2
+  %alloca6 = alloca <vscale x 4 x float>, align 16
+  %alloca7 = alloca <vscale x 4 x float>, align 16
+  %alloca8 = alloca <vscale x 16 x i1>, align 2
+  %alloca9 = alloca <vscale x 16 x i1>, align 2
+  %alloca10 = alloca <vscale x 4 x float>, align 16
+  %alloca11 = alloca <vscale x 4 x float>, align 16
+  %alloca12 = alloca <vscale x 16 x i1>, align 2
+  %alloca13 = alloca <vscale x 16 x i1>, align 2
+  %alloca14 = alloca <vscale x 4 x float>, align 16
+  %alloca15 = alloca <vscale x 4 x float>, align 16
+  %alloca16 = alloca i64, align 8
+  %alloca17 = alloca i64, align 8
+  %alloca18 = alloca ptr, align 8
+  %alloca19 = alloca i64, align 8
+  %alloca20 = alloca i64, align 8
+  %alloca21 = alloca target("aarch64.svcount"), align 2
+  %alloca22 = alloca i32, align 4
+  %alloca23 = alloca <vscale x 32 x i8>, align 16
+  %alloca24 = alloca i64, align 8
+  %alloca25 = alloca target("aarch64.svcount"), align 2
+  %alloca26 = alloca ptr, align 8
+  %alloca27 = alloca i64, align 8
+  %alloca28 = alloca target("aarch64.svcount"), align 2
+  %alloca29 = alloca ptr, align 8
+  %alloca30 = alloca i64, align 8
+  %alloca31 = alloca <vscale x 32 x i1>, align 2
+  %alloca32 = alloca <vscale x 32 x i1>, align 2
+  %alloca33 = alloca <vscale x 8 x float>, align 16
+  %alloca34 = alloca <vscale x 8 x float>, align 16
+  %alloca35 = alloca i64, align 8
+  %alloca36 = alloca i64, align 8
+  %alloca37 = alloca i64, align 8
+  %alloca38 = alloca i64, align 8
+  %alloca39 = alloca i64, align 8
+  %alloca40 = alloca i64, align 8
+  %alloca41 = alloca i64, align 8
+  %alloca42 = alloca i8, align 1
+  %alloca43 = alloca ptr, align 8
+  %alloca44 = alloca i64, align 8
+  %alloca45 = alloca i8, align 1
+  %alloca46 = alloca ptr, align 8
+  %alloca47 = alloca i64, align 8
+  %alloca48 = alloca ptr, align 8
+  %alloca49 = alloca i64, align 8
+  %alloca50 = alloca i8, align 1
+  %alloca51 = alloca ptr, align 8
+  %alloca52 = alloca ptr, align 8
+  %alloca53 = alloca ptr, align 8
+  %alloca54 = alloca i64, align 8
+  %alloca55 = alloca i64, align 8
+  %alloca56 = alloca i64, align 8
+  %alloca57 = alloca i64, align 8
+  %alloca58 = alloca i64, align 8
+  %alloca59 = alloca i64, align 8
+  %alloca60 = alloca i64, align 8
+  %alloca61 = alloca i64, align 8
+  %alloca62 = alloca i64, align 8
+  %alloca63 = alloca target("aarch64.svcount"), align 2
+  %alloca64 = alloca target("aarch64.svcount"), align 2
+  %alloca65 = alloca ptr, align 8
+  %alloca66 = alloca ptr, align 8
+  %alloca67 = alloca ptr, align 8
+  store i8 0, ptr %alloca42, align 1
+  store i8 0, ptr %alloca45, align 1
+  store i8 0, ptr %alloca50, align 1
+  store ptr null, ptr %alloca51, align 8
+  %load = load ptr, ptr %alloca43, align 8
+  %load68 = load i64, ptr %alloca39, align 8
+  %getelementptr = getelementptr inbounds float, ptr %load, i64 %load68
+  %load69 = load i64, ptr %alloca41, align 8
+  %sub = sub i64 %load69, 1
+  %load70 = load i64, ptr %alloca44, align 8
+  %mul = mul i64 %sub, %load70
+  %getelementptr71 = getelementptr inbounds float, ptr %getelementptr, i64 %mul
+  store ptr %getelementptr71, ptr %alloca51, align 8
+  store ptr null, ptr %alloca52, align 8
+  %load72 = load ptr, ptr %alloca46, align 8
+  %load73 = load i64, ptr %alloca40, align 8
+  %getelementptr74 = getelementptr inbounds float, ptr %load72, i64 %load73
+  %load75 = load i64, ptr %alloca41, align 8
+  %sub76 = sub i64 %load75, 1
+  %load77 = load i64, ptr %alloca47, align 8
+  %mul78 = mul i64 %sub76, %load77
+  %getelementptr79 = getelementptr inbounds float, ptr %getelementptr74, i64 %mul78
+  store ptr %getelementptr79, ptr %alloca52, align 8
+  store ptr null, ptr %alloca53, align 8
+  %load80 = load ptr, ptr %alloca48, align 8
+  %load81 = load i64, ptr %alloca39, align 8
+  %getelementptr82 = getelementptr inbounds float, ptr %load80, i64 %load81
+  %load83 = load i64, ptr %alloca40, align 8
+  %sub84 = sub i64 %load83, 1
+  %load85 = load i64, ptr %alloca49, align 8
+  %mul86 = mul i64 %sub84, %load85
+  %getelementptr87 = getelementptr inbounds float, ptr %getelementptr82, i64 %mul86
+  store ptr %getelementptr87, ptr %alloca53, align 8
+  store i64 32, ptr %alloca54, align 8
+  store i64 32, ptr %alloca55, align 8
+  store i64 0, ptr %alloca56, align 8
+  %load88 = load i64, ptr %alloca41, align 8
+  %mul89 = mul i64 32, %load88
+  store i64 %mul89, ptr %alloca56, align 8
+  %load90 = load i8, ptr %alloca42, align 1
+  %trunc = trunc i8 %load90 to i1
+  store i64 32, ptr %alloca44, align 8
+  store i64 0, ptr %alloca57, align 8
+  %load91 = load i64, ptr %alloca39, align 8
+  %sub92 = sub i64 %load91, 1
+  %udiv = udiv i64 %sub92, 32
+  %add = add i64 %udiv, 1
+  store i64 %add, ptr %alloca57, align 8
+  %load93 = load ptr, ptr %alloca43, align 8
+  %load94 = load i64, ptr %alloca57, align 8
+  %load95 = load i64, ptr %alloca56, align 8
+  %mul96 = mul i64 %load94, %load95
+  %getelementptr97 = getelementptr inbounds float, ptr %load93, i64 %mul96
+  store ptr %getelementptr97, ptr %alloca51, align 8
+  %load98 = load i8, ptr %alloca45, align 1
+  %trunc99 = trunc i8 %load98 to i1
+  store i64 32, ptr %alloca47, align 8
+  store i64 0, ptr %alloca58, align 8
+  %load100 = load i64, ptr %alloca40, align 8
+  %sub101 = sub i64 %load100, 1
+  %udiv102 = udiv i64 %sub101, 32
+  %add103 = add i64 %udiv102, 1
+  store i64 %add103, ptr %alloca58, align 8
+  %load104 = load ptr, ptr %alloca46, align 8
+  %load105 = load i64, ptr %alloca58, align 8
+  %load106 = load i64, ptr %alloca56, align 8
+  %mul107 = mul i64 %load105, %load106
+  %getelementptr108 = getelementptr inbounds float, ptr %load104, i64 %mul107
+  store ptr %getelementptr108, ptr %alloca52, align 8
+  store i64 0, ptr %alloca59, align 8
+  store i64 0, ptr %alloca59, align 8
+  %load109 = load i64, ptr %alloca59, align 8
+  %load110 = load i64, ptr %alloca40, align 8
+  %icmp = icmp ult i64 %load109, %load110
+  store i64 0, ptr %alloca60, align 8
+  store i64 0, ptr %alloca60, align 8
+  %load111 = load i64, ptr %alloca60, align 8
+  %load112 = load i64, ptr %alloca39, align 8
+  %icmp113 = icmp ult i64 %load111, %load112
+  store i64 0, ptr %alloca61, align 8
+  %load114 = load i64, ptr %alloca39, align 8
+  %load115 = load i64, ptr %alloca60, align 8
+  %sub116 = sub i64 %load114, %load115
+  store i64 %sub116, ptr %alloca35, align 8
+  store i64 32, ptr %alloca36, align 8
+  %load117 = load i64, ptr %alloca35, align 8
+  %load118 = load i64, ptr %alloca36, align 8
+  %icmp119 = icmp ult i64 %load117, %load118
+  %load120 = load i64, ptr %alloca35, align 8
+  store i64 %load120, ptr %alloca61, align 8
+  store i64 0, ptr %alloca62, align 8
+  %load121 = load i64, ptr %alloca40, align 8
+  %load122 = load i64, ptr %alloca59, align 8
+  %sub123 = sub i64 %load121, %load122
+  store i64 %sub123, ptr %alloca37, align 8
+  store i64 32, ptr %alloca38, align 8
+  %load124 = load i64, ptr %alloca37, align 8
+  %load125 = load i64, ptr %alloca38, align 8
+  %icmp126 = icmp ult i64 %load124, %load125
+  %load127 = load i64, ptr %alloca37, align 8
+  store i64 %load127, ptr %alloca62, align 8
+  %load128 = load i64, ptr %alloca60, align 8
+  %load129 = load i64, ptr %alloca39, align 8
+  %call = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 %load128, i64 %load129, i32 2)
+  store target("aarch64.svcount") %call, ptr %alloca63, align 2
+  %load130 = load i64, ptr %alloca59, align 8
+  %load131 = load i64, ptr %alloca40, align 8
+  %call132 = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 %load130, i64 %load131, i32 2)
+  store target("aarch64.svcount") %call132, ptr %alloca64, align 2
+  store ptr null, ptr %alloca65, align 8
+  %load133 = load ptr, ptr %alloca48, align 8
+  %load134 = load i64, ptr %alloca60, align 8
+  %getelementptr135 = getelementptr inbounds float, ptr %load133, i64 %load134
+  %load136 = load i64, ptr %alloca59, align 8
+  %load137 = load i64, ptr %alloca49, align 8
+  %mul138 = mul i64 %load136, %load137
+  %getelementptr139 = getelementptr inbounds float, ptr %getelementptr135, i64 %mul138
+  store ptr %getelementptr139, ptr %alloca65, align 8
+  call void @llvm.aarch64.sme.zero(i32 255)
+  store ptr null, ptr %alloca66, align 8
+  %load140 = load i8, ptr %alloca42, align 1
+  %trunc141 = trunc i8 %load140 to i1
+  %load142 = load ptr, ptr %alloca43, align 8
+  %load143 = load i64, ptr %alloca60, align 8
+  %load144 = load i64, ptr %alloca41, align 8
+  %mul145 = mul i64 %load143, %load144
+  %getelementptr146 = getelementptr inbounds float, ptr %load142, i64 %mul145
+  store ptr %getelementptr146, ptr %alloca66, align 8
+  store ptr null, ptr %alloca67, align 8
+  %load147 = load i8, ptr %alloca45, align 1
+  %trunc148 = trunc i8 %load147 to i1
+  %load149 = load ptr, ptr %alloca46, align 8
+  %load150 = load i64, ptr %alloca59, align 8
+  %load151 = load i64, ptr %alloca41, align 8
+  %mul152 = mul i64 %load150, %load151
+  %getelementptr153 = getelementptr inbounds float, ptr %load149, i64 %mul152
+  store ptr %getelementptr153, ptr %alloca67, align 8
+  %load154 = load i64, ptr %alloca41, align 8
+  %load155 = load target("aarch64.svcount"), ptr %alloca63, align 2
+  %load156 = load ptr, ptr %alloca66, align 8
+  %load157 = load i64, ptr %alloca44, align 8
+  %udiv158 = udiv i64 %load157, 4
+  %load159 = load target("aarch64.svcount"), ptr %alloca64, align 2
+  %load160 = load ptr, ptr %alloca67, align 8
+  %load161 = load i64, ptr %alloca47, align 8
+  %udiv162 = udiv i64 %load161, 4
+  store i64 %load154, ptr %alloca24, align 8
+  store target("aarch64.svcount") %load155, ptr %alloca25, align 2
+  store ptr %load156, ptr %alloca26, align 8
+  store i64 %udiv158, ptr %alloca27, align 8
+  store target("aarch64.svcount") %load159, ptr %alloca28, align 2
+  store ptr %load160, ptr %alloca29, align 8
+  store i64 %udiv162, ptr %alloca30, align 8
+  %load163 = load target("aarch64.svcount"), ptr %alloca25, align 2
+  %call164 = call { <vscale x 4 x i1>, <vscale x 4 x i1> } @llvm.aarch64.sve.pext.x2.nxv4i1(target("aarch64.svcount") %load163, i32 0)
+  %extractvalue = extractvalue { <vscale x 4 x i1>, <vscale x 4 x i1> } %call164, 0
+  %call165 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %extractvalue)
+  %call166 = call <vscale x 32 x i1> @llvm.vector.insert.nxv32i1.nxv16i1(<vscale x 32 x i1> poison, <vscale x 16 x i1> %call165, i64 0)
+  %extractvalue167 = extractvalue { <vscale x 4 x i1>, <vscale x 4 x i1> } %call164, 1
+  %call168 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %extractvalue167)
+  %call169 = call <vscale x 32 x i1> @llvm.vector.insert.nxv32i1.nxv16i1(<vscale x 32 x i1> %call166, <vscale x 16 x i1> %call168, i64 16)
+  store <vscale x 32 x i1> %call169, ptr %alloca31, align 2
+  %load170 = load target("aarch64.svcount"), ptr %alloca28, align 2
+  %call171 = call { <vscale x 4 x i1>, <vscale x 4 x i1> } @llvm.aarch64.sve.pext.x2.nxv4i1(target("aarch64.svcount") %load170, i32 0)
+  %extractvalue172 = extractvalue { <vscale x 4 x i1>, <vscale x 4 x i1> } %call171, 0
+  %call173 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %extractvalue172)
+  %call174 = call <vscale x 32 x i1> @llvm.vector.insert.nxv32i1.nxv16i1(<vscale x 32 x i1> poison, <vscale x 16 x i1> %call173, i64 0)
+  %extractvalue175 = extractvalue { <vscale x 4 x i1>, <vscale x 4 x i1> } %call171, 1
+  %call176 = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %extractvalue175)
+  %call177 = call <vscale x 32 x i1> @llvm.vector.insert.nxv32i1.nxv16i1(<vscale x 32 x i1> %call174, <vscale x 16 x i1> %call176, i64 16)
+  store <vscale x 32 x i1> %call177, ptr %alloca32, align 2
+  br label %bb178
+
+bb178:                                            ; preds = %bb178, %bb
+  %load179 = load i64, ptr %alloca24, align 8
+  %icmp180 = icmp ugt i64 %load179, 0
+  %load181 = load target("aarch64.svcount"), ptr %alloca25, align 2
+  %load182 = load ptr, ptr %alloca26, align 8
+  %call183 = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %load181, ptr %load182)
+  %extractvalue184 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %call183, 0
+  %call185 = call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> %extractvalue184, i64 0)
+  %extractvalue186 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %call183, 1
+  %call187 = call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> %call185, <vscale x 4 x float> %extractvalue186, i64 4)
+  store <vscale x 8 x float> %call187, ptr %alloca33, align 16
+  %load188 = load i64, ptr %alloca27, align 8
+  %load189 = load ptr, ptr %alloca26, align 8
+  %getelementptr190 = getelementptr inbounds float, ptr %load189, i64 %load188
+  store ptr %getelementptr190, ptr %alloca26, align 8
+  %load191 = load target("aarch64.svcount"), ptr %alloca28, align 2
+  %load192 = load ptr, ptr %alloca29, align 8
+  %call193 = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %load191, ptr %load192)
+  %extractvalue194 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %call193, 0
+  %call195 = call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> %extractvalue194, i64 0)
+  %extractvalue196 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %call193, 1
+  %call197 = call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> %call195, <vscale x 4 x float> %extractvalue196, i64 4)
+  store <vscale x 8 x float> %call197, ptr %alloca34, align 16
+  %load198 = load i64, ptr %alloca30, align 8
+  %load199 = load ptr, ptr %alloca29, align 8
+  %getelementptr200 = getelementptr inbounds float, ptr %load199, i64 %load198
+  store ptr %getelementptr200, ptr %alloca29, align 8
+  %load201 = load <vscale x 32 x i1>, ptr %alloca31, align 2
+  %call202 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> %load201, i64 0)
+  %load203 = load <vscale x 32 x i1>, ptr %alloca32, align 2
+  %call204 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> %load203, i64 0)
+  %load205 = load <vscale x 8 x float>, ptr %alloca33, align 16
+  %call206 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> %load205, i64 0)
+  %load207 = load <vscale x 8 x float>, ptr %alloca34, align 16
+  %call208 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> %load207, i64 0)
+  store <vscale x 16 x i1> %call202, ptr %alloca12, align 2
+  store <vscale x 16 x i1> %call204, ptr %alloca13, align 2
+  store <vscale x 4 x float> %call206, ptr %alloca14, align 16
+  store <vscale x 4 x float> %call208, ptr %alloca15, align 16
+  %load209 = load <vscale x 16 x i1>, ptr %alloca12, align 2
+  %load210 = load <vscale x 16 x i1>, ptr %alloca13, align 2
+  %load211 = load <vscale x 4 x float>, ptr %alloca14, align 16
+  %load212 = load <vscale x 4 x float>, ptr %alloca15, align 16
+  %call213 = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %load209)
+  %call214 = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %load210)
+  call void @llvm.aarch64.sme.mopa.nxv4f32(i32 0, <vscale x 4 x i1> %call213, <vscale x 4 x i1> %call214, <vscale x 4 x float> %load211, <vscale x 4 x float> %load212)
+  %load215 = load <vscale x 32 x i1>, ptr %alloca31, align 2
+  %call216 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> %load215, i64 16)
+  %load217 = load <vscale x 32 x i1>, ptr %alloca32, align 2
+  %call218 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> %load217, i64 0)
+  %load219 = load <vscale x 8 x float>, ptr %alloca33, align 16
+  %call220 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> %load219, i64 4)
+  %load221 = load <vscale x 8 x float>, ptr %alloca34, align 16
+  %call222 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> %load221, i64 0)
+  store <vscale x 16 x i1> %call216, ptr %alloca8, align 2
+  store <vscale x 16 x i1> %call218, ptr %alloca9, align 2
+  store <vscale x 4 x float> %call220, ptr %alloca10, align 16
+  store <vscale x 4 x float> %call222, ptr %alloca11, align 16
+  %load223 = load <vscale x 16 x i1>, ptr %alloca8, align 2
+  %load224 = load <vscale x 16 x i1>, ptr %alloca9, align 2
+  %load225 = load <vscale x 4 x float>, ptr %alloca10, align 16
+  %load226 = load <vscale x 4 x float>, ptr %alloca11, align 16
+  %call227 = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %load223)
+  %call228 = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %load224)
+  call void @llvm.aarch64.sme.mopa.nxv4f32(i32 1, <vscale x 4 x i1> %call227, <vscale x 4 x i1> %call228, <vscale x 4 x float> %load225, <vscale x 4 x float> %load226)
+  %load229 = load <vscale x 32 x i1>, ptr %alloca31, align 2
+  %call230 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> %load229, i64 0)
+  %load231 = load <vscale x 32 x i1>, ptr %alloca32, align 2
+  %call232 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> %load231, i64 16)
+  %load233 = load <vscale x 8 x float>, ptr %alloca33, align 16
+  %call234 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> %load233, i64 0)
+  %load235 = load <vscale x 8 x float>, ptr %alloca34, align 16
+  %call236 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> %load235, i64 4)
+  store <vscale x 16 x i1> %call230, ptr %alloca4, align 2
+  store <vscale x 16 x i1> %call232, ptr %alloca5, align 2
+  store <vscale x 4 x float> %call234, ptr %alloca6, align 16
+  store <vscale x 4 x float> %call236, ptr %alloca7, align 16
+  %load237 = load <vscale x 16 x i1>, ptr %alloca4, align 2
+  %load238 = load <vscale x 16 x i1>, ptr %alloca5, align 2
+  %load239 = load <vscale x 4 x float>, ptr %alloca6, align 16
+  %load240 = load <vscale x 4 x float>, ptr %alloca7, align 16
+  %call241 = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %load237)
+  %call242 = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %load238)
+  call void @llvm.aarch64.sme.mopa.nxv4f32(i32 2, <vscale x 4 x i1> %call241, <vscale x 4 x i1> %call242, <vscale x 4 x float> %load239, <vscale x 4 x float> %load240)
+  %load243 = load <vscale x 32 x i1>, ptr %alloca31, align 2
+  %call244 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> %load243, i64 16)
+  %load245 = load <vscale x 32 x i1>, ptr %alloca32, align 2
+  %call246 = call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1> %load245, i64 16)
+  %load247 = load <vscale x 8 x float>, ptr %alloca33, align 16
+  %call248 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> %load247, i64 4)
+  %load249 = load <vscale x 8 x float>, ptr %alloca34, align 16
+  %call250 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> %load249, i64 4)
+  store <vscale x 16 x i1> %call244, ptr %alloca, align 2
+  store <vscale x 16 x i1> %call246, ptr %alloca1, align 2
+  store <vscale x 4 x float> %call248, ptr %alloca2, align 16
+  store <vscale x 4 x float> %call250, ptr %alloca3, align 16
+  %load251 = load <vscale x 16 x i1>, ptr %alloca, align 2
+  %load252 = load <vscale x 16 x i1>, ptr %alloca1, align 2
+  %load253 = load <vscale x 4 x float>, ptr %alloca2, align 16
+  %load254 = load <vscale x 4 x float>, ptr %alloca3, align 16
+  %call255 = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %load251)
+  %call256 = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %load252)
+  call void @llvm.aarch64.sme.mopa.nxv4f32(i32 3, <vscale x 4 x i1> %call255, <vscale x 4 x i1> %call256, <vscale x 4 x float> %load253, <vscale x 4 x float> %load254)
+  %load257 = load i64, ptr %alloca24, align 8
+  %add258 = add i64 %load257, -1
+  store i64 %add258, ptr %alloca24, align 8
+  br label %bb178
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64, i64, i32 immarg) #2
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn
+declare void @llvm.aarch64.sme.zero(i32 immarg) #3
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none)
+declare { <vscale x 4 x i1>, <vscale x 4 x i1> } @llvm.aarch64.sve.pext.x2.nxv4i1(target("aarch64.svcount"), i32 immarg) #2
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none)
+declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1>) #2
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <vscale x 32 x i1> @llvm.vector.insert.nxv32i1.nxv16i1(<vscale x 32 x i1>, <vscale x 16 x i1>, i64 immarg) #4
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: read)
+declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount"), ptr) #5
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float>, <vscale x 4 x float>, i64 immarg) #4
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv32i1(<vscale x 32 x i1>, i64 immarg) #4
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float>, i64 immarg) #4
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1>) #2
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn
+declare void @llvm.aarch64.sme.mopa.nxv4f32(i32 immarg, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>) #3
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64, i64, i32 immarg) #2
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn
+declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg2.nxv16i8(i32, i32) #3
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8>, <vscale x 16 x i8>, i64 immarg) #4
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8>, i64 immarg) #4
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: write)
+declare void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, target("aarch64.svcount"), ptr) #6
+
+attributes #0 = { cold noreturn nounwind }
+attributes #1 = { mustprogress noinline optnone ssp uwtable(sync) vscale_range(1,16) "aarch64_new_za" "aarch64_pstate_sm_enabled" "frame-pointer"="non-leaf" "target-features"="+fp-armv8,+fullfp16,+sme,+sme-f64f64,+sme2" }
+attributes #2 = { nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #3 = { nocallback nofree nosync nounwind willreturn }
+attributes #4 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
+attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
-- 
GitLab


From 12028373020739b388eb2b8141742509f1764e3c Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Tue, 14 May 2024 16:09:57 -0400
Subject: [PATCH 0778/1206] Reapply "[Clang] Unify interface for accessing
 template arguments as written for class/variable template specializations
 (#81642)" (#91393)

Reapplies #81642, fixing the crash which occurs when running the lldb test suite.
---
 clang-tools-extra/clangd/AST.cpp              |  37 +-
 .../clangd/SemanticHighlighting.cpp           |  13 +-
 .../include-cleaner/lib/WalkAST.cpp           |  13 +-
 clang/docs/LibASTMatchersReference.html       | 364 +++++++++++++-----
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/AST/DeclTemplate.h        | 226 +++++------
 clang/include/clang/AST/RecursiveASTVisitor.h |  27 +-
 clang/include/clang/ASTMatchers/ASTMatchers.h |  74 ++--
 .../clang/ASTMatchers/ASTMatchersInternal.h   |  50 ++-
 .../include/clang/Serialization/ASTBitCodes.h |   2 +-
 clang/lib/AST/ASTImporter.cpp                 |  68 ++--
 clang/lib/AST/DeclPrinter.cpp                 |  18 +-
 clang/lib/AST/DeclTemplate.cpp                | 266 ++++++++-----
 clang/lib/AST/TypePrinter.cpp                 |  25 +-
 clang/lib/Index/IndexDecl.cpp                 |   9 +-
 clang/lib/Sema/Sema.cpp                       |   2 +-
 clang/lib/Sema/SemaTemplate.cpp               |  58 ++-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  | 163 +++-----
 clang/lib/Serialization/ASTReaderDecl.cpp     |  28 +-
 clang/lib/Serialization/ASTWriterDecl.cpp     |  36 +-
 clang/lib/Tooling/Syntax/BuildTree.cpp        |   3 +-
 clang/test/AST/ast-dump-decl.cpp              |  12 +-
 clang/test/AST/ast-dump-template-decls.cpp    |  18 +-
 clang/test/Index/Core/index-source.cpp        |  24 +-
 clang/test/Index/index-refs.cpp               |   1 -
 clang/tools/libclang/CIndex.cpp               |  29 +-
 .../ASTMatchers/ASTMatchersNodeTest.cpp       |  12 -
 .../ASTMatchers/ASTMatchersTraversalTest.cpp  |  92 ++---
 28 files changed, 914 insertions(+), 759 deletions(-)

diff --git a/clang-tools-extra/clangd/AST.cpp b/clang-tools-extra/clangd/AST.cpp
index 1b86ea19cf28..fda1e5fdf8d8 100644
--- a/clang-tools-extra/clangd/AST.cpp
+++ b/clang-tools-extra/clangd/AST.cpp
@@ -50,16 +50,11 @@ getTemplateSpecializationArgLocs(const NamedDecl &ND) {
     if (const ASTTemplateArgumentListInfo *Args =
             Func->getTemplateSpecializationArgsAsWritten())
       return Args->arguments();
-  } else if (auto *Cls =
-                 llvm::dyn_cast<ClassTemplatePartialSpecializationDecl>(&ND)) {
+  } else if (auto *Cls = llvm::dyn_cast<ClassTemplateSpecializationDecl>(&ND)) {
     if (auto *Args = Cls->getTemplateArgsAsWritten())
       return Args->arguments();
-  } else if (auto *Var =
-                 llvm::dyn_cast<VarTemplatePartialSpecializationDecl>(&ND)) {
-    if (auto *Args = Var->getTemplateArgsAsWritten())
-      return Args->arguments();
   } else if (auto *Var = llvm::dyn_cast<VarTemplateSpecializationDecl>(&ND)) {
-    if (auto *Args = Var->getTemplateArgsInfo())
+    if (auto *Args = Var->getTemplateArgsAsWritten())
       return Args->arguments();
   }
   // We return std::nullopt for ClassTemplateSpecializationDecls because it does
@@ -270,22 +265,10 @@ std::string printTemplateSpecializationArgs(const NamedDecl &ND) {
           getTemplateSpecializationArgLocs(ND)) {
     printTemplateArgumentList(OS, *Args, Policy);
   } else if (auto *Cls = llvm::dyn_cast<ClassTemplateSpecializationDecl>(&ND)) {
-    if (const TypeSourceInfo *TSI = Cls->getTypeAsWritten()) {
-      // ClassTemplateSpecializationDecls do not contain
-      // TemplateArgumentTypeLocs, they only have TemplateArgumentTypes. So we
-      // create a new argument location list from TypeSourceInfo.
-      auto STL = TSI->getTypeLoc().getAs<TemplateSpecializationTypeLoc>();
-      llvm::SmallVector<TemplateArgumentLoc> ArgLocs;
-      ArgLocs.reserve(STL.getNumArgs());
-      for (unsigned I = 0; I < STL.getNumArgs(); ++I)
-        ArgLocs.push_back(STL.getArgLoc(I));
-      printTemplateArgumentList(OS, ArgLocs, Policy);
-    } else {
-      // FIXME: Fix cases when getTypeAsWritten returns null inside clang AST,
-      // e.g. friend decls. Currently we fallback to Template Arguments without
-      // location information.
-      printTemplateArgumentList(OS, Cls->getTemplateArgs().asArray(), Policy);
-    }
+    // FIXME: Fix cases when getTypeAsWritten returns null inside clang AST,
+    // e.g. friend decls. Currently we fallback to Template Arguments without
+    // location information.
+    printTemplateArgumentList(OS, Cls->getTemplateArgs().asArray(), Policy);
   }
   OS.flush();
   return TemplateArgs;
@@ -453,10 +436,12 @@ bool hasReservedScope(const DeclContext &DC) {
 }
 
 QualType declaredType(const TypeDecl *D) {
+  ASTContext &Context = D->getASTContext();
   if (const auto *CTSD = llvm::dyn_cast<ClassTemplateSpecializationDecl>(D))
-    if (const auto *TSI = CTSD->getTypeAsWritten())
-      return TSI->getType();
-  return D->getASTContext().getTypeDeclType(D);
+    if (const auto *Args = CTSD->getTemplateArgsAsWritten())
+      return Context.getTemplateSpecializationType(
+          TemplateName(CTSD->getSpecializedTemplate()), Args->arguments());
+  return Context.getTypeDeclType(D);
 }
 
 namespace {
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index 08f99e11ac9b..eb025f21f361 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -693,17 +693,22 @@ public:
     return true;
   }
 
+  bool
+  VisitClassTemplateSpecializationDecl(ClassTemplateSpecializationDecl *D) {
+    if (auto *Args = D->getTemplateArgsAsWritten())
+      H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
+    return true;
+  }
+
   bool VisitClassTemplatePartialSpecializationDecl(
       ClassTemplatePartialSpecializationDecl *D) {
     if (auto *TPL = D->getTemplateParameters())
       H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
-    if (auto *Args = D->getTemplateArgsAsWritten())
-      H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
     return true;
   }
 
   bool VisitVarTemplateSpecializationDecl(VarTemplateSpecializationDecl *D) {
-    if (auto *Args = D->getTemplateArgsInfo())
+    if (auto *Args = D->getTemplateArgsAsWritten())
       H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
     return true;
   }
@@ -712,8 +717,6 @@ public:
       VarTemplatePartialSpecializationDecl *D) {
     if (auto *TPL = D->getTemplateParameters())
       H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
-    if (auto *Args = D->getTemplateArgsAsWritten())
-      H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
     return true;
   }
 
diff --git a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
index 878067aca017..f7cc9d191236 100644
--- a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
+++ b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
@@ -267,18 +267,21 @@ public:
     return true;
   }
 
-  // Report a reference from explicit specializations to the specialized
-  // template. Implicit ones are filtered out by RAV and explicit instantiations
-  // are already traversed through typelocs.
+  // Report a reference from explicit specializations/instantiations to the
+  // specialized template. Implicit ones are filtered out by RAV.
   bool
   VisitClassTemplateSpecializationDecl(ClassTemplateSpecializationDecl *CTSD) {
-    if (CTSD->isExplicitSpecialization())
+    // if (CTSD->isExplicitSpecialization())
+    if (clang::isTemplateExplicitInstantiationOrSpecialization(
+            CTSD->getTemplateSpecializationKind()))
       report(CTSD->getLocation(),
              CTSD->getSpecializedTemplate()->getTemplatedDecl());
     return true;
   }
   bool VisitVarTemplateSpecializationDecl(VarTemplateSpecializationDecl *VTSD) {
-    if (VTSD->isExplicitSpecialization())
+    // if (VTSD->isExplicitSpecialization())
+    if (clang::isTemplateExplicitInstantiationOrSpecialization(
+            VTSD->getTemplateSpecializationKind()))
       report(VTSD->getLocation(),
              VTSD->getSpecializedTemplate()->getTemplatedDecl());
     return true;
diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html
index bb1b68f6671b..a16b9c44ef0e 100644
--- a/clang/docs/LibASTMatchersReference.html
+++ b/clang/docs/LibASTMatchersReference.html
@@ -3546,33 +3546,35 @@ cxxMethodDecl(isConst()) matches A::foo() but not A::bar()
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isExplicitObjectMemberFunction0')"><a name="isExplicitObjectMemberFunction0Anchor">isExplicitObjectMemberFunction</a></td><td></td></tr>
-<tr><td colspan="4" class="doc" id="isExplicitObjectMemberFunction0"><pre>Matches if the given method declaration declares a member function with an explicit object parameter.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isCopyAssignmentOperator0')"><a name="isCopyAssignmentOperator0Anchor">isCopyAssignmentOperator</a></td><td></td></tr>
+<tr><td colspan="4" class="doc" id="isCopyAssignmentOperator0"><pre>Matches if the given method declaration declares a copy assignment
+operator.
 
 Given
 struct A {
-  int operator-(this A, int);
-  void fun(this A &&self);
-  static int operator()(int);
-  int operator+(int);
+  A &amp;operator=(const A &amp;);
+  A &amp;operator=(A &amp;&amp;);
 };
 
-cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two methods but not the last two.
+cxxMethodDecl(isCopyAssignmentOperator()) matches the first method but not
+the second one.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isCopyAssignmentOperator0')"><a name="isCopyAssignmentOperator0Anchor">isCopyAssignmentOperator</a></td><td></td></tr>
-<tr><td colspan="4" class="doc" id="isCopyAssignmentOperator0"><pre>Matches if the given method declaration declares a copy assignment
-operator.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isExplicitObjectMemberFunction0')"><a name="isExplicitObjectMemberFunction0Anchor">isExplicitObjectMemberFunction</a></td><td></td></tr>
+<tr><td colspan="4" class="doc" id="isExplicitObjectMemberFunction0"><pre>Matches if the given method declaration declares a member function with an
+explicit object parameter.
 
 Given
 struct A {
-  A &amp;operator=(const A &amp;);
-  A &amp;operator=(A &amp;&amp;);
+ int operator-(this A, int);
+ void fun(this A &amp;&amp;self);
+ static int operator()(int);
+ int operator+(int);
 };
 
-cxxMethodDecl(isCopyAssignmentOperator()) matches the first method but not
-the second one.
+cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two
+methods but not the last two.
 </pre></td></tr>
 
 
@@ -6713,7 +6715,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -6757,7 +6759,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -6985,7 +6987,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7219,7 +7221,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7416,7 +7418,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7620,7 +7622,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7677,7 +7679,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -7875,9 +7877,10 @@ int a = b ?: 1;
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument0')"><a name="forEachTemplateArgument0Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="forEachTemplateArgument0"><pre>Matches classTemplateSpecialization, templateSpecializationType and
-functionDecl nodes where the template argument matches the inner matcher.
-This matcher may produce multiple matches.
+<tr><td colspan="4" class="doc" id="forEachTemplateArgument0"><pre>Matches templateSpecializationType, class template specialization,
+variable template specialization, and function template specialization
+nodes where the template argument matches the inner matcher. This matcher
+may produce multiple matches.
 
 Given
   template &lt;typename T, unsigned N, unsigned M&gt;
@@ -7899,10 +7902,25 @@ functionDecl(forEachTemplateArgument(refersToType(builtinType())))
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc0')"><a name="hasAnyTemplateArgumentLoc0Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+that have at least one `TemplateArgumentLoc` matching the given
+`InnerMatcher`.
+
+Given
+  template&lt;typename T&gt; class A {};
+  A&lt;int&gt; a;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+  hasTypeLoc(loc(asString("int")))))))
+  matches `A&lt;int&gt; a`.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument0')"><a name="hasAnyTemplateArgument0Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument0"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl that have at least one TemplateArgument matching the given
-InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument0"><pre>Matches templateSpecializationTypes, class template specializations,
+variable template specializations, and function template specializations
+that have at least one TemplateArgument matching the given InnerMatcher.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -7933,9 +7951,25 @@ classTemplateSpecializationDecl(hasSpecializedTemplate(classTemplateDecl()))
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc0')"><a name="hasTemplateArgumentLoc0Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
+
+Given
+  template&lt;typename T, typename U&gt; class A {};
+  A&lt;double, int&gt; b;
+  A&lt;int, double&gt; c;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+  hasTypeLoc(loc(asString("double")))))))
+  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument0')"><a name="hasTemplateArgument0Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgument0"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasTemplateArgument0"><pre>Matches templateSpecializationType, class template specializations,
+variable template specializations, and function template specializations
+where the n'th TemplateArgument matches the given InnerMatcher.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -7953,34 +7987,6 @@ functionDecl(hasTemplateArgument(0, refersToType(asString("int"))))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc7')"><a name="hasTypeLoc7Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc7"><pre>Matches if the type location of a node matches the inner matcher.
-
-Examples:
-  int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
-
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
-
-struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
-
-Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
-</pre></td></tr>
-
-
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ComplexType.html">ComplexType</a>&gt;</td><td class="name" onclick="toggle('hasElementType1')"><a name="hasElementType1Anchor">hasElementType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td></tr>
 <tr><td colspan="4" class="doc" id="hasElementType1"><pre>Matches arrays and C99 complex types that have a specific element
 type.
@@ -7996,8 +8002,8 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Arra
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc8')"><a name="hasTypeLoc8Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc8"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc7')"><a name="hasTypeLoc7Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc7"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -8017,7 +8023,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -8066,6 +8072,21 @@ with compoundStmt()
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc3')"><a name="hasAnyTemplateArgumentLoc3Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc3"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+that have at least one `TemplateArgumentLoc` matching the given
+`InnerMatcher`.
+
+Given
+  template&lt;typename T&gt; class A {};
+  A&lt;int&gt; a;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+  hasTypeLoc(loc(asString("int")))))))
+  matches `A&lt;int&gt; a`.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration11')"><a name="hasDeclaration11Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration11"><pre>Matches a node if the declaration associated with that node
 matches the given matcher.
@@ -8100,9 +8121,10 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Addr
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc0')"><a name="hasTemplateArgumentLoc0Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s where the n'th
-`TemplateArgumentLoc` matches the given `InnerMatcher`.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc3')"><a name="hasTemplateArgumentLoc3Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc3"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -8176,8 +8198,8 @@ declStmt(hasSingleDecl(anything()))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc9')"><a name="hasTypeLoc9Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc9"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc8')"><a name="hasTypeLoc8Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc8"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -8197,7 +8219,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -8393,8 +8415,8 @@ actual casts "explicit" casts.)
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc10')"><a name="hasTypeLoc10Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc10"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc9')"><a name="hasTypeLoc9Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc9"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -8414,7 +8436,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -8707,9 +8729,10 @@ Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument2')"><a name="forEachTemplateArgument2Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="forEachTemplateArgument2"><pre>Matches classTemplateSpecialization, templateSpecializationType and
-functionDecl nodes where the template argument matches the inner matcher.
-This matcher may produce multiple matches.
+<tr><td colspan="4" class="doc" id="forEachTemplateArgument2"><pre>Matches templateSpecializationType, class template specialization,
+variable template specialization, and function template specialization
+nodes where the template argument matches the inner matcher. This matcher
+may produce multiple matches.
 
 Given
   template &lt;typename T, unsigned N, unsigned M&gt;
@@ -8778,10 +8801,25 @@ matching y.
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc2')"><a name="hasAnyTemplateArgumentLoc2Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc2"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+that have at least one `TemplateArgumentLoc` matching the given
+`InnerMatcher`.
+
+Given
+  template&lt;typename T&gt; class A {};
+  A&lt;int&gt; a;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+  hasTypeLoc(loc(asString("int")))))))
+  matches `A&lt;int&gt; a`.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument2')"><a name="hasAnyTemplateArgument2Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument2"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl that have at least one TemplateArgument matching the given
-InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument2"><pre>Matches templateSpecializationTypes, class template specializations,
+variable template specializations, and function template specializations
+that have at least one TemplateArgument matching the given InnerMatcher.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -8878,9 +8916,25 @@ functionDecl(hasReturnTypeLoc(loc(asString("int"))))
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc2')"><a name="hasTemplateArgumentLoc2Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc2"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
+
+Given
+  template&lt;typename T, typename U&gt; class A {};
+  A&lt;double, int&gt; b;
+  A&lt;int, double&gt; c;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+  hasTypeLoc(loc(asString("double")))))))
+  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument2')"><a name="hasTemplateArgument2Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgument2"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
+<tr><td colspan="4" class="doc" id="hasTemplateArgument2"><pre>Matches templateSpecializationType, class template specializations,
+variable template specializations, and function template specializations
+where the n'th TemplateArgument matches the given InnerMatcher.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -9473,8 +9527,8 @@ matching y.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc11')"><a name="hasTypeLoc11Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc11"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc10')"><a name="hasTypeLoc10Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc10"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -9494,7 +9548,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -9919,8 +9973,8 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Addr
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc12')"><a name="hasTypeLoc12Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc12"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc11')"><a name="hasTypeLoc11Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc11"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -9940,7 +9994,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -10014,9 +10068,11 @@ matches the specialization of struct A generated by A&lt;X&gt;.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc0')"><a name="hasAnyTemplateArgumentLoc0Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc0"><pre>Matches template specialization `TypeLoc`s that have at least one
-`TemplateArgumentLoc` matching the given `InnerMatcher`.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc4')"><a name="hasAnyTemplateArgumentLoc4Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc4"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+that have at least one `TemplateArgumentLoc` matching the given
+`InnerMatcher`.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -10027,9 +10083,10 @@ varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc1')"><a name="hasTemplateArgumentLoc1Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc1"><pre>Matches template specialization `TypeLoc`s where the n'th
-`TemplateArgumentLoc` matches the given `InnerMatcher`.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationTypeLoc.html">TemplateSpecializationTypeLoc</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc4')"><a name="hasTemplateArgumentLoc4Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc4"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -10041,10 +10098,11 @@ varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument1')"><a name="forEachTemplateArgument1Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="forEachTemplateArgument1"><pre>Matches classTemplateSpecialization, templateSpecializationType and
-functionDecl nodes where the template argument matches the inner matcher.
-This matcher may produce multiple matches.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument3')"><a name="forEachTemplateArgument3Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="forEachTemplateArgument3"><pre>Matches templateSpecializationType, class template specialization,
+variable template specialization, and function template specialization
+nodes where the template argument matches the inner matcher. This matcher
+may produce multiple matches.
 
 Given
   template &lt;typename T, unsigned N, unsigned M&gt;
@@ -10066,10 +10124,10 @@ functionDecl(forEachTemplateArgument(refersToType(builtinType())))
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument1')"><a name="hasAnyTemplateArgument1Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument1"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl that have at least one TemplateArgument matching the given
-InnerMatcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument3')"><a name="hasAnyTemplateArgument3Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument3"><pre>Matches templateSpecializationTypes, class template specializations,
+variable template specializations, and function template specializations
+that have at least one TemplateArgument matching the given InnerMatcher.
 
 Given
   template&lt;typename T&gt; class A {};
@@ -10122,9 +10180,10 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Addr
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument1')"><a name="hasTemplateArgument1Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="hasTemplateArgument1"><pre>Matches classTemplateSpecializations, templateSpecializationType and
-functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument3')"><a name="hasTemplateArgument3Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgument3"><pre>Matches templateSpecializationType, class template specializations,
+variable template specializations, and function template specializations
+where the n'th TemplateArgument matches the given InnerMatcher.
 
 Given
   template&lt;typename T, typename U&gt; class A {};
@@ -10182,8 +10241,8 @@ QualType-matcher matches.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc13')"><a name="hasTypeLoc13Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
-<tr><td colspan="4" class="doc" id="hasTypeLoc13"><pre>Matches if the type location of a node matches the inner matcher.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc12')"><a name="hasTypeLoc12Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
+<tr><td colspan="4" class="doc" id="hasTypeLoc12"><pre>Matches if the type location of a node matches the inner matcher.
 
 Examples:
   int x;
@@ -10203,7 +10262,7 @@ Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Bloc
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateSpecializationDecl.html">ClassTemplateSpecializationDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;
@@ -10449,6 +10508,105 @@ Example matches x (matcher = varDecl(hasInitializer(callExpr())))
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('forEachTemplateArgument1')"><a name="forEachTemplateArgument1Anchor">forEachTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="forEachTemplateArgument1"><pre>Matches templateSpecializationType, class template specialization,
+variable template specialization, and function template specialization
+nodes where the template argument matches the inner matcher. This matcher
+may produce multiple matches.
+
+Given
+  template &lt;typename T, unsigned N, unsigned M&gt;
+  struct Matrix {};
+
+  constexpr unsigned R = 2;
+  Matrix&lt;int, R * 2, R * 4&gt; M;
+
+  template &lt;typename T, typename U&gt;
+  void f(T&amp;&amp; t, U&amp;&amp; u) {}
+
+  bool B = false;
+  f(R, B);
+templateSpecializationType(forEachTemplateArgument(isExpr(expr())))
+  matches twice, with expr() matching 'R * 2' and 'R * 4'
+functionDecl(forEachTemplateArgument(refersToType(builtinType())))
+  matches the specialization f&lt;unsigned, bool&gt; twice, for 'unsigned'
+  and 'bool'
+</pre></td></tr>
+
+
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgumentLoc1')"><a name="hasAnyTemplateArgumentLoc1Anchor">hasAnyTemplateArgumentLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgumentLoc1"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+that have at least one `TemplateArgumentLoc` matching the given
+`InnerMatcher`.
+
+Given
+  template&lt;typename T&gt; class A {};
+  A&lt;int&gt; a;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+  hasTypeLoc(loc(asString("int")))))))
+  matches `A&lt;int&gt; a`.
+</pre></td></tr>
+
+
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyTemplateArgument1')"><a name="hasAnyTemplateArgument1Anchor">hasAnyTemplateArgument</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasAnyTemplateArgument1"><pre>Matches templateSpecializationTypes, class template specializations,
+variable template specializations, and function template specializations
+that have at least one TemplateArgument matching the given InnerMatcher.
+
+Given
+  template&lt;typename T&gt; class A {};
+  template&lt;&gt; class A&lt;double&gt; {};
+  A&lt;int&gt; a;
+
+  template&lt;typename T&gt; f() {};
+  void func() { f&lt;int&gt;(); };
+
+classTemplateSpecializationDecl(hasAnyTemplateArgument(
+    refersToType(asString("int"))))
+  matches the specialization A&lt;int&gt;
+
+functionDecl(hasAnyTemplateArgument(refersToType(asString("int"))))
+  matches the specialization f&lt;int&gt;
+</pre></td></tr>
+
+
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc1')"><a name="hasTemplateArgumentLoc1Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgumentLoc1"><pre>Matches template specialization `TypeLoc`s, class template specializations,
+variable template specializations, and function template specializations
+where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
+
+Given
+  template&lt;typename T, typename U&gt; class A {};
+  A&lt;double, int&gt; b;
+  A&lt;int, double&gt; c;
+varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+  hasTypeLoc(loc(asString("double")))))))
+  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
+</pre></td></tr>
+
+
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgument1')"><a name="hasTemplateArgument1Anchor">hasTemplateArgument</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgument.html">TemplateArgument</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasTemplateArgument1"><pre>Matches templateSpecializationType, class template specializations,
+variable template specializations, and function template specializations
+where the n'th TemplateArgument matches the given InnerMatcher.
+
+Given
+  template&lt;typename T, typename U&gt; class A {};
+  A&lt;bool, int&gt; b;
+  A&lt;int, bool&gt; c;
+
+  template&lt;typename T&gt; void f() {}
+  void func() { f&lt;int&gt;(); };
+classTemplateSpecializationDecl(hasTemplateArgument(
+    1, refersToType(asString("int"))))
+  matches the specialization A&lt;bool, int&gt;
+
+functionDecl(hasTemplateArgument(0, refersToType(asString("int"))))
+  matches the specialization f&lt;int&gt;
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VariableArrayType.html">VariableArrayType</a>&gt;</td><td class="name" onclick="toggle('hasSizeExpr0')"><a name="hasSizeExpr0Anchor">hasSizeExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasSizeExpr0"><pre>Matches VariableArrayType nodes that have a specific size
 expression.
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 49ab222bec40..ae699ebfc603 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -114,6 +114,9 @@ Clang Frontend Potentially Breaking Changes
     $ clang --target=<your target triple> -print-target-triple
     <the normalized target triple>
 
+- The ``hasTypeLoc`` AST matcher will no longer match a ``classTemplateSpecializationDecl``;
+  existing uses should switch to ``templateArgumentLoc`` or ``hasAnyTemplateArgumentLoc`` instead.
+
 What's New in Clang |release|?
 ==============================
 Some of the major new features and improvements to Clang are listed
diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index 3ee03eebdb8c..268aeacf2f20 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -1776,6 +1776,25 @@ public:
   BuiltinTemplateKind getBuiltinTemplateKind() const { return BTK; }
 };
 
+/// Provides information about an explicit instantiation of a variable or class
+/// template.
+struct ExplicitInstantiationInfo {
+  /// The template arguments as written..
+  const ASTTemplateArgumentListInfo *TemplateArgsAsWritten = nullptr;
+
+  /// The location of the extern keyword.
+  SourceLocation ExternKeywordLoc;
+
+  /// The location of the template keyword.
+  SourceLocation TemplateKeywordLoc;
+
+  ExplicitInstantiationInfo() = default;
+};
+
+using SpecializationOrInstantiationInfo =
+    llvm::PointerUnion<const ASTTemplateArgumentListInfo *,
+                       ExplicitInstantiationInfo *>;
+
 /// Represents a class template specialization, which refers to
 /// a class template with a given set of template arguments.
 ///
@@ -1789,8 +1808,8 @@ public:
 /// template<>
 /// class array<bool> { }; // class template specialization array<bool>
 /// \endcode
-class ClassTemplateSpecializationDecl
-  : public CXXRecordDecl, public llvm::FoldingSetNode {
+class ClassTemplateSpecializationDecl : public CXXRecordDecl,
+                                        public llvm::FoldingSetNode {
   /// Structure that stores information about a class template
   /// specialization that was instantiated from a class template partial
   /// specialization.
@@ -1808,23 +1827,9 @@ class ClassTemplateSpecializationDecl
   llvm::PointerUnion<ClassTemplateDecl *, SpecializedPartialSpecialization *>
     SpecializedTemplate;
 
-  /// Further info for explicit template specialization/instantiation.
-  struct ExplicitSpecializationInfo {
-    /// The type-as-written.
-    TypeSourceInfo *TypeAsWritten = nullptr;
-
-    /// The location of the extern keyword.
-    SourceLocation ExternLoc;
-
-    /// The location of the template keyword.
-    SourceLocation TemplateKeywordLoc;
-
-    ExplicitSpecializationInfo() = default;
-  };
-
   /// Further info for explicit template specialization/instantiation.
   /// Does not apply to implicit specializations.
-  ExplicitSpecializationInfo *ExplicitInfo = nullptr;
+  SpecializationOrInstantiationInfo ExplicitInfo = nullptr;
 
   /// The template arguments used to describe this specialization.
   const TemplateArgumentList *TemplateArgs;
@@ -2001,44 +2006,49 @@ public:
     SpecializedTemplate = TemplDecl;
   }
 
-  /// Sets the type of this specialization as it was written by
-  /// the user. This will be a class template specialization type.
-  void setTypeAsWritten(TypeSourceInfo *T) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->TypeAsWritten = T;
+  /// Retrieve the template argument list as written in the sources,
+  /// if any.
+  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->TemplateArgsAsWritten;
+    return ExplicitInfo.get<const ASTTemplateArgumentListInfo *>();
   }
 
-  /// Gets the type of this specialization as it was written by
-  /// the user, if it was so written.
-  TypeSourceInfo *getTypeAsWritten() const {
-    return ExplicitInfo ? ExplicitInfo->TypeAsWritten : nullptr;
+  /// Set the template argument list as written in the sources.
+  void
+  setTemplateArgsAsWritten(const ASTTemplateArgumentListInfo *ArgsWritten) {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      Info->TemplateArgsAsWritten = ArgsWritten;
+    else
+      ExplicitInfo = ArgsWritten;
   }
 
-  /// Gets the location of the extern keyword, if present.
-  SourceLocation getExternLoc() const {
-    return ExplicitInfo ? ExplicitInfo->ExternLoc : SourceLocation();
+  /// Set the template argument list as written in the sources.
+  void setTemplateArgsAsWritten(const TemplateArgumentListInfo &ArgsInfo) {
+    setTemplateArgsAsWritten(
+        ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo));
   }
 
-  /// Sets the location of the extern keyword.
-  void setExternLoc(SourceLocation Loc) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->ExternLoc = Loc;
+  /// Gets the location of the extern keyword, if present.
+  SourceLocation getExternKeywordLoc() const {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->ExternKeywordLoc;
+    return SourceLocation();
   }
 
-  /// Sets the location of the template keyword.
-  void setTemplateKeywordLoc(SourceLocation Loc) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->TemplateKeywordLoc = Loc;
-  }
+  /// Sets the location of the extern keyword.
+  void setExternKeywordLoc(SourceLocation Loc);
 
   /// Gets the location of the template keyword, if present.
   SourceLocation getTemplateKeywordLoc() const {
-    return ExplicitInfo ? ExplicitInfo->TemplateKeywordLoc : SourceLocation();
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->TemplateKeywordLoc;
+    return SourceLocation();
   }
 
+  /// Sets the location of the template keyword.
+  void setTemplateKeywordLoc(SourceLocation Loc);
+
   SourceRange getSourceRange() const override LLVM_READONLY;
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
@@ -2066,10 +2076,6 @@ class ClassTemplatePartialSpecializationDecl
   /// The list of template parameters
   TemplateParameterList* TemplateParams = nullptr;
 
-  /// The source info for the template arguments as written.
-  /// FIXME: redundant with TypeAsWritten?
-  const ASTTemplateArgumentListInfo *ArgsAsWritten = nullptr;
-
   /// The class template partial specialization from which this
   /// class template partial specialization was instantiated.
   ///
@@ -2078,15 +2084,11 @@ class ClassTemplatePartialSpecializationDecl
   llvm::PointerIntPair<ClassTemplatePartialSpecializationDecl *, 1, bool>
       InstantiatedFromMember;
 
-  ClassTemplatePartialSpecializationDecl(ASTContext &Context, TagKind TK,
-                                         DeclContext *DC,
-                                         SourceLocation StartLoc,
-                                         SourceLocation IdLoc,
-                                         TemplateParameterList *Params,
-                                         ClassTemplateDecl *SpecializedTemplate,
-                                         ArrayRef<TemplateArgument> Args,
-                               const ASTTemplateArgumentListInfo *ArgsAsWritten,
-                               ClassTemplatePartialSpecializationDecl *PrevDecl);
+  ClassTemplatePartialSpecializationDecl(
+      ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
+      SourceLocation IdLoc, TemplateParameterList *Params,
+      ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
+      ClassTemplatePartialSpecializationDecl *PrevDecl);
 
   ClassTemplatePartialSpecializationDecl(ASTContext &C)
     : ClassTemplateSpecializationDecl(C, ClassTemplatePartialSpecialization),
@@ -2101,11 +2103,8 @@ public:
   static ClassTemplatePartialSpecializationDecl *
   Create(ASTContext &Context, TagKind TK, DeclContext *DC,
          SourceLocation StartLoc, SourceLocation IdLoc,
-         TemplateParameterList *Params,
-         ClassTemplateDecl *SpecializedTemplate,
-         ArrayRef<TemplateArgument> Args,
-         const TemplateArgumentListInfo &ArgInfos,
-         QualType CanonInjectedType,
+         TemplateParameterList *Params, ClassTemplateDecl *SpecializedTemplate,
+         ArrayRef<TemplateArgument> Args, QualType CanonInjectedType,
          ClassTemplatePartialSpecializationDecl *PrevDecl);
 
   static ClassTemplatePartialSpecializationDecl *
@@ -2136,11 +2135,6 @@ public:
     return TemplateParams->hasAssociatedConstraints();
   }
 
-  /// Get the template arguments as written.
-  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
-    return ArgsAsWritten;
-  }
-
   /// Retrieve the member class template partial specialization from
   /// which this particular class template partial specialization was
   /// instantiated.
@@ -2193,7 +2187,7 @@ public:
   /// template<> template<typename T>
   /// struct X<int>::Inner<T*> { /* ... */ };
   /// \endcode
-  bool isMemberSpecialization() {
+  bool isMemberSpecialization() const {
     const auto *First =
         cast<ClassTemplatePartialSpecializationDecl>(getFirstDecl());
     return First->InstantiatedFromMember.getInt();
@@ -2216,6 +2210,8 @@ public:
              ->getInjectedSpecializationType();
   }
 
+  SourceRange getSourceRange() const override LLVM_READONLY;
+
   void Profile(llvm::FoldingSetNodeID &ID) const {
     Profile(ID, getTemplateArgs().asArray(), getTemplateParameters(),
             getASTContext());
@@ -2613,27 +2609,12 @@ class VarTemplateSpecializationDecl : public VarDecl,
   llvm::PointerUnion<VarTemplateDecl *, SpecializedPartialSpecialization *>
   SpecializedTemplate;
 
-  /// Further info for explicit template specialization/instantiation.
-  struct ExplicitSpecializationInfo {
-    /// The type-as-written.
-    TypeSourceInfo *TypeAsWritten = nullptr;
-
-    /// The location of the extern keyword.
-    SourceLocation ExternLoc;
-
-    /// The location of the template keyword.
-    SourceLocation TemplateKeywordLoc;
-
-    ExplicitSpecializationInfo() = default;
-  };
-
   /// Further info for explicit template specialization/instantiation.
   /// Does not apply to implicit specializations.
-  ExplicitSpecializationInfo *ExplicitInfo = nullptr;
+  SpecializationOrInstantiationInfo ExplicitInfo = nullptr;
 
   /// The template arguments used to describe this specialization.
   const TemplateArgumentList *TemplateArgs;
-  const ASTTemplateArgumentListInfo *TemplateArgsInfo = nullptr;
 
   /// The point where this template was instantiated (if any).
   SourceLocation PointOfInstantiation;
@@ -2687,14 +2668,6 @@ public:
   /// specialization.
   const TemplateArgumentList &getTemplateArgs() const { return *TemplateArgs; }
 
-  // TODO: Always set this when creating the new specialization?
-  void setTemplateArgsInfo(const TemplateArgumentListInfo &ArgsInfo);
-  void setTemplateArgsInfo(const ASTTemplateArgumentListInfo *ArgsInfo);
-
-  const ASTTemplateArgumentListInfo *getTemplateArgsInfo() const {
-    return TemplateArgsInfo;
-  }
-
   /// Determine the kind of specialization that this
   /// declaration represents.
   TemplateSpecializationKind getSpecializationKind() const {
@@ -2798,44 +2771,49 @@ public:
     SpecializedTemplate = TemplDecl;
   }
 
-  /// Sets the type of this specialization as it was written by
-  /// the user.
-  void setTypeAsWritten(TypeSourceInfo *T) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->TypeAsWritten = T;
+  /// Retrieve the template argument list as written in the sources,
+  /// if any.
+  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->TemplateArgsAsWritten;
+    return ExplicitInfo.get<const ASTTemplateArgumentListInfo *>();
+  }
+
+  /// Set the template argument list as written in the sources.
+  void
+  setTemplateArgsAsWritten(const ASTTemplateArgumentListInfo *ArgsWritten) {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      Info->TemplateArgsAsWritten = ArgsWritten;
+    else
+      ExplicitInfo = ArgsWritten;
   }
 
-  /// Gets the type of this specialization as it was written by
-  /// the user, if it was so written.
-  TypeSourceInfo *getTypeAsWritten() const {
-    return ExplicitInfo ? ExplicitInfo->TypeAsWritten : nullptr;
+  /// Set the template argument list as written in the sources.
+  void setTemplateArgsAsWritten(const TemplateArgumentListInfo &ArgsInfo) {
+    setTemplateArgsAsWritten(
+        ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo));
   }
 
   /// Gets the location of the extern keyword, if present.
-  SourceLocation getExternLoc() const {
-    return ExplicitInfo ? ExplicitInfo->ExternLoc : SourceLocation();
+  SourceLocation getExternKeywordLoc() const {
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->ExternKeywordLoc;
+    return SourceLocation();
   }
 
   /// Sets the location of the extern keyword.
-  void setExternLoc(SourceLocation Loc) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->ExternLoc = Loc;
-  }
-
-  /// Sets the location of the template keyword.
-  void setTemplateKeywordLoc(SourceLocation Loc) {
-    if (!ExplicitInfo)
-      ExplicitInfo = new (getASTContext()) ExplicitSpecializationInfo;
-    ExplicitInfo->TemplateKeywordLoc = Loc;
-  }
+  void setExternKeywordLoc(SourceLocation Loc);
 
   /// Gets the location of the template keyword, if present.
   SourceLocation getTemplateKeywordLoc() const {
-    return ExplicitInfo ? ExplicitInfo->TemplateKeywordLoc : SourceLocation();
+    if (auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>())
+      return Info->TemplateKeywordLoc;
+    return SourceLocation();
   }
 
+  /// Sets the location of the template keyword.
+  void setTemplateKeywordLoc(SourceLocation Loc);
+
   SourceRange getSourceRange() const override LLVM_READONLY;
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
@@ -2863,10 +2841,6 @@ class VarTemplatePartialSpecializationDecl
   /// The list of template parameters
   TemplateParameterList *TemplateParams = nullptr;
 
-  /// The source info for the template arguments as written.
-  /// FIXME: redundant with TypeAsWritten?
-  const ASTTemplateArgumentListInfo *ArgsAsWritten = nullptr;
-
   /// The variable template partial specialization from which this
   /// variable template partial specialization was instantiated.
   ///
@@ -2879,8 +2853,7 @@ class VarTemplatePartialSpecializationDecl
       ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
       SourceLocation IdLoc, TemplateParameterList *Params,
       VarTemplateDecl *SpecializedTemplate, QualType T, TypeSourceInfo *TInfo,
-      StorageClass S, ArrayRef<TemplateArgument> Args,
-      const ASTTemplateArgumentListInfo *ArgInfos);
+      StorageClass S, ArrayRef<TemplateArgument> Args);
 
   VarTemplatePartialSpecializationDecl(ASTContext &Context)
       : VarTemplateSpecializationDecl(VarTemplatePartialSpecialization,
@@ -2897,8 +2870,8 @@ public:
   Create(ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
          SourceLocation IdLoc, TemplateParameterList *Params,
          VarTemplateDecl *SpecializedTemplate, QualType T,
-         TypeSourceInfo *TInfo, StorageClass S, ArrayRef<TemplateArgument> Args,
-         const TemplateArgumentListInfo &ArgInfos);
+         TypeSourceInfo *TInfo, StorageClass S,
+         ArrayRef<TemplateArgument> Args);
 
   static VarTemplatePartialSpecializationDecl *
   CreateDeserialized(ASTContext &C, GlobalDeclID ID);
@@ -2914,11 +2887,6 @@ public:
     return TemplateParams;
   }
 
-  /// Get the template arguments as written.
-  const ASTTemplateArgumentListInfo *getTemplateArgsAsWritten() const {
-    return ArgsAsWritten;
-  }
-
   /// \brief All associated constraints of this partial specialization,
   /// including the requires clause and any constraints derived from
   /// constrained-parameters.
@@ -2981,7 +2949,7 @@ public:
   /// template<> template<typename T>
   /// U* X<int>::Inner<T*> = (T*)(0) + 1;
   /// \endcode
-  bool isMemberSpecialization() {
+  bool isMemberSpecialization() const {
     const auto *First =
         cast<VarTemplatePartialSpecializationDecl>(getFirstDecl());
     return First->InstantiatedFromMember.getInt();
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index f9b145b4e86a..782f60844506 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -2030,6 +2030,15 @@ DEF_TRAVERSE_DECL(RecordDecl, { TRY_TO(TraverseRecordHelper(D)); })
 
 DEF_TRAVERSE_DECL(CXXRecordDecl, { TRY_TO(TraverseCXXRecordHelper(D)); })
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::TraverseTemplateArgumentLocsHelper(
+    const TemplateArgumentLoc *TAL, unsigned Count) {
+  for (unsigned I = 0; I < Count; ++I) {
+    TRY_TO(TraverseTemplateArgumentLoc(TAL[I]));
+  }
+  return true;
+}
+
 #define DEF_TRAVERSE_TMPL_SPEC_DECL(TMPLDECLKIND, DECLKIND)                    \
   DEF_TRAVERSE_DECL(TMPLDECLKIND##TemplateSpecializationDecl, {                \
     /* For implicit instantiations ("set<int> x;"), we don't want to           \
@@ -2039,9 +2048,12 @@ DEF_TRAVERSE_DECL(CXXRecordDecl, { TRY_TO(TraverseCXXRecordHelper(D)); })
        TemplateSpecializationType).  For explicit instantiations               \
        ("template set<int>;"), we do need a callback, since this               \
        is the only callback that's made for this instantiation.                \
-       We use getTypeAsWritten() to distinguish. */                            \
-    if (TypeSourceInfo *TSI = D->getTypeAsWritten())                           \
-      TRY_TO(TraverseTypeLoc(TSI->getTypeLoc()));                              \
+       We use getTemplateArgsAsWritten() to distinguish. */                    \
+    if (const auto *ArgsWritten = D->getTemplateArgsAsWritten()) {             \
+      /* The args that remains unspecialized. */                               \
+      TRY_TO(TraverseTemplateArgumentLocsHelper(                               \
+          ArgsWritten->getTemplateArgs(), ArgsWritten->NumTemplateArgs));      \
+    }                                                                          \
                                                                                \
     if (getDerived().shouldVisitTemplateInstantiations() ||                    \
         D->getTemplateSpecializationKind() == TSK_ExplicitSpecialization) {    \
@@ -2061,15 +2073,6 @@ DEF_TRAVERSE_DECL(CXXRecordDecl, { TRY_TO(TraverseCXXRecordHelper(D)); })
 DEF_TRAVERSE_TMPL_SPEC_DECL(Class, CXXRecord)
 DEF_TRAVERSE_TMPL_SPEC_DECL(Var, Var)
 
-template <typename Derived>
-bool RecursiveASTVisitor<Derived>::TraverseTemplateArgumentLocsHelper(
-    const TemplateArgumentLoc *TAL, unsigned Count) {
-  for (unsigned I = 0; I < Count; ++I) {
-    TRY_TO(TraverseTemplateArgumentLoc(TAL[I]));
-  }
-  return true;
-}
-
 #define DEF_TRAVERSE_TMPL_PART_SPEC_DECL(TMPLDECLKIND, DECLKIND)               \
   DEF_TRAVERSE_DECL(TMPLDECLKIND##TemplatePartialSpecializationDecl, {         \
     /* The partial specialization. */                                          \
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 8a2bbfff9e9e..0f3257db6f41 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -764,9 +764,9 @@ AST_POLYMORPHIC_MATCHER(isImplicit,
   return Node.isImplicit();
 }
 
-/// Matches classTemplateSpecializations, templateSpecializationType and
-/// functionDecl that have at least one TemplateArgument matching the given
-/// InnerMatcher.
+/// Matches templateSpecializationTypes, class template specializations,
+/// variable template specializations, and function template specializations
+/// that have at least one TemplateArgument matching the given InnerMatcher.
 ///
 /// Given
 /// \code
@@ -788,8 +788,8 @@ AST_POLYMORPHIC_MATCHER(isImplicit,
 AST_POLYMORPHIC_MATCHER_P(
     hasAnyTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
-                                    TemplateSpecializationType,
-                                    FunctionDecl),
+                                    VarTemplateSpecializationDecl, FunctionDecl,
+                                    TemplateSpecializationType),
     internal::Matcher<TemplateArgument>, InnerMatcher) {
   ArrayRef<TemplateArgument> List =
       internal::getTemplateSpecializationArgs(Node);
@@ -1047,8 +1047,9 @@ AST_MATCHER(Expr, isTypeDependent) { return Node.isTypeDependent(); }
 /// expr(isValueDependent()) matches return Size
 AST_MATCHER(Expr, isValueDependent) { return Node.isValueDependent(); }
 
-/// Matches classTemplateSpecializations, templateSpecializationType and
-/// functionDecl where the n'th TemplateArgument matches the given InnerMatcher.
+/// Matches templateSpecializationType, class template specializations,
+/// variable template specializations, and function template specializations
+/// where the n'th TemplateArgument matches the given InnerMatcher.
 ///
 /// Given
 /// \code
@@ -1068,8 +1069,8 @@ AST_MATCHER(Expr, isValueDependent) { return Node.isValueDependent(); }
 AST_POLYMORPHIC_MATCHER_P2(
     hasTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
-                                    TemplateSpecializationType,
-                                    FunctionDecl),
+                                    VarTemplateSpecializationDecl, FunctionDecl,
+                                    TemplateSpecializationType),
     unsigned, N, internal::Matcher<TemplateArgument>, InnerMatcher) {
   ArrayRef<TemplateArgument> List =
       internal::getTemplateSpecializationArgs(Node);
@@ -4066,7 +4067,7 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 ///   Matcher<CXXCtorInitializer>, Matcher<CXXFunctionalCastExpr>,
 ///   Matcher<CXXNewExpr>, Matcher<CXXTemporaryObjectExpr>,
 ///   Matcher<CXXUnresolvedConstructExpr>,
-///   Matcher<ClassTemplateSpecializationDecl>, Matcher<CompoundLiteralExpr>,
+///   Matcher<CompoundLiteralExpr>,
 ///   Matcher<DeclaratorDecl>, Matcher<ExplicitCastExpr>,
 ///   Matcher<ObjCPropertyDecl>, Matcher<TemplateArgumentLoc>,
 ///   Matcher<TypedefNameDecl>
@@ -4075,9 +4076,8 @@ AST_POLYMORPHIC_MATCHER_P(
     AST_POLYMORPHIC_SUPPORTED_TYPES(
         BlockDecl, CXXBaseSpecifier, CXXCtorInitializer, CXXFunctionalCastExpr,
         CXXNewExpr, CXXTemporaryObjectExpr, CXXUnresolvedConstructExpr,
-        ClassTemplateSpecializationDecl, CompoundLiteralExpr, DeclaratorDecl,
-        ExplicitCastExpr, ObjCPropertyDecl, TemplateArgumentLoc,
-        TypedefNameDecl),
+        CompoundLiteralExpr, DeclaratorDecl, ExplicitCastExpr, ObjCPropertyDecl,
+        TemplateArgumentLoc, TypedefNameDecl),
     internal::Matcher<TypeLoc>, Inner) {
   TypeSourceInfo *source = internal::GetTypeSourceInfo(Node);
   if (source == nullptr) {
@@ -5304,9 +5304,10 @@ AST_POLYMORPHIC_MATCHER_P(parameterCountIs,
   return Node.getNumParams() == N;
 }
 
-/// Matches classTemplateSpecialization, templateSpecializationType and
-/// functionDecl nodes where the template argument matches the inner matcher.
-/// This matcher may produce multiple matches.
+/// Matches templateSpecializationType, class template specialization,
+/// variable template specialization, and function template specialization
+/// nodes where the template argument matches the inner matcher. This matcher
+/// may produce multiple matches.
 ///
 /// Given
 /// \code
@@ -5330,7 +5331,8 @@ AST_POLYMORPHIC_MATCHER_P(parameterCountIs,
 AST_POLYMORPHIC_MATCHER_P(
     forEachTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
-                                    TemplateSpecializationType, FunctionDecl),
+                                    VarTemplateSpecializationDecl, FunctionDecl,
+                                    TemplateSpecializationType),
     internal::Matcher<TemplateArgument>, InnerMatcher) {
   ArrayRef<TemplateArgument> TemplateArgs =
       clang::ast_matchers::internal::getTemplateSpecializationArgs(Node);
@@ -6905,8 +6907,10 @@ extern const internal::VariadicDynCastAllOfMatcher<
     TypeLoc, TemplateSpecializationTypeLoc>
     templateSpecializationTypeLoc;
 
-/// Matches template specialization `TypeLoc`s that have at least one
-/// `TemplateArgumentLoc` matching the given `InnerMatcher`.
+/// Matches template specialization `TypeLoc`s, class template specializations,
+/// variable template specializations, and function template specializations
+/// that have at least one `TemplateArgumentLoc` matching the given
+/// `InnerMatcher`.
 ///
 /// Given
 /// \code
@@ -6916,20 +6920,21 @@ extern const internal::VariadicDynCastAllOfMatcher<
 /// varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
 ///   hasTypeLoc(loc(asString("int")))))))
 ///   matches `A<int> a`.
-AST_MATCHER_P(TemplateSpecializationTypeLoc, hasAnyTemplateArgumentLoc,
-              internal::Matcher<TemplateArgumentLoc>, InnerMatcher) {
-  for (unsigned Index = 0, N = Node.getNumArgs(); Index < N; ++Index) {
-    clang::ast_matchers::internal::BoundNodesTreeBuilder Result(*Builder);
-    if (InnerMatcher.matches(Node.getArgLoc(Index), Finder, &Result)) {
-      *Builder = std::move(Result);
-      return true;
-    }
-  }
+AST_POLYMORPHIC_MATCHER_P(
+    hasAnyTemplateArgumentLoc,
+    AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
+                                    VarTemplateSpecializationDecl, FunctionDecl,
+                                    DeclRefExpr, TemplateSpecializationTypeLoc),
+    internal::Matcher<TemplateArgumentLoc>, InnerMatcher) {
+  auto Args = internal::getTemplateArgsWritten(Node);
+  return matchesFirstInRange(InnerMatcher, Args.begin(), Args.end(), Finder,
+                             Builder) != Args.end();
   return false;
 }
 
-/// Matches template specialization `TypeLoc`s where the n'th
-/// `TemplateArgumentLoc` matches the given `InnerMatcher`.
+/// Matches template specialization `TypeLoc`s, class template specializations,
+/// variable template specializations, and function template specializations
+/// where the n'th `TemplateArgumentLoc` matches the given `InnerMatcher`.
 ///
 /// Given
 /// \code
@@ -6942,10 +6947,13 @@ AST_MATCHER_P(TemplateSpecializationTypeLoc, hasAnyTemplateArgumentLoc,
 ///   matches `A<double, int> b`, but not `A<int, double> c`.
 AST_POLYMORPHIC_MATCHER_P2(
     hasTemplateArgumentLoc,
-    AST_POLYMORPHIC_SUPPORTED_TYPES(DeclRefExpr, TemplateSpecializationTypeLoc),
+    AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
+                                    VarTemplateSpecializationDecl, FunctionDecl,
+                                    DeclRefExpr, TemplateSpecializationTypeLoc),
     unsigned, Index, internal::Matcher<TemplateArgumentLoc>, InnerMatcher) {
-  return internal::MatchTemplateArgLocAt(Node, Index, InnerMatcher, Finder,
-                                         Builder);
+  auto Args = internal::getTemplateArgsWritten(Node);
+  return Index < Args.size() &&
+         InnerMatcher.matches(Args[Index], Finder, Builder);
 }
 
 /// Matches C or C++ elaborated `TypeLoc`s.
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
index 47d912c73dd7..c1cc63fdb743 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
@@ -186,10 +186,6 @@ inline TypeSourceInfo *GetTypeSourceInfo(const BlockDecl &Node) {
 inline TypeSourceInfo *GetTypeSourceInfo(const CXXNewExpr &Node) {
   return Node.getAllocatedTypeSourceInfo();
 }
-inline TypeSourceInfo *
-GetTypeSourceInfo(const ClassTemplateSpecializationDecl &Node) {
-  return Node.getTypeAsWritten();
-}
 
 /// Unifies obtaining the FunctionProtoType pointer from both
 /// FunctionProtoType and FunctionDecl nodes..
@@ -1939,6 +1935,11 @@ getTemplateSpecializationArgs(const ClassTemplateSpecializationDecl &D) {
   return D.getTemplateArgs().asArray();
 }
 
+inline ArrayRef<TemplateArgument>
+getTemplateSpecializationArgs(const VarTemplateSpecializationDecl &D) {
+  return D.getTemplateArgs().asArray();
+}
+
 inline ArrayRef<TemplateArgument>
 getTemplateSpecializationArgs(const TemplateSpecializationType &T) {
   return T.template_arguments();
@@ -1948,7 +1949,46 @@ inline ArrayRef<TemplateArgument>
 getTemplateSpecializationArgs(const FunctionDecl &FD) {
   if (const auto* TemplateArgs = FD.getTemplateSpecializationArgs())
     return TemplateArgs->asArray();
-  return ArrayRef<TemplateArgument>();
+  return std::nullopt;
+}
+
+inline ArrayRef<TemplateArgumentLoc>
+getTemplateArgsWritten(const ClassTemplateSpecializationDecl &D) {
+  if (const ASTTemplateArgumentListInfo *Args = D.getTemplateArgsAsWritten())
+    return Args->arguments();
+  return std::nullopt;
+}
+
+inline ArrayRef<TemplateArgumentLoc>
+getTemplateArgsWritten(const VarTemplateSpecializationDecl &D) {
+  if (const ASTTemplateArgumentListInfo *Args = D.getTemplateArgsAsWritten())
+    return Args->arguments();
+  return std::nullopt;
+}
+
+inline ArrayRef<TemplateArgumentLoc>
+getTemplateArgsWritten(const FunctionDecl &FD) {
+  if (const auto *Args = FD.getTemplateSpecializationArgsAsWritten())
+    return Args->arguments();
+  return std::nullopt;
+}
+
+inline ArrayRef<TemplateArgumentLoc>
+getTemplateArgsWritten(const DeclRefExpr &DRE) {
+  if (const auto *Args = DRE.getTemplateArgs())
+    return {Args, DRE.getNumTemplateArgs()};
+  return std::nullopt;
+}
+
+inline SmallVector<TemplateArgumentLoc>
+getTemplateArgsWritten(const TemplateSpecializationTypeLoc &T) {
+  SmallVector<TemplateArgumentLoc> Args;
+  if (!T.isNull()) {
+    Args.reserve(T.getNumArgs());
+    for (unsigned I = 0; I < T.getNumArgs(); ++I)
+      Args.emplace_back(T.getArgLoc(I));
+  }
+  return Args;
 }
 
 struct NotEqualsBoundNodePredicate {
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index d3538e43d3d7..fe1bd47348be 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -43,7 +43,7 @@ namespace serialization {
 /// Version 4 of AST files also requires that the version control branch and
 /// revision match exactly, since there is no backward compatibility of
 /// AST files at this time.
-const unsigned VERSION_MAJOR = 30;
+const unsigned VERSION_MAJOR = 31;
 
 /// AST file minor version number supported by this version of
 /// Clang.
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 60f213322b34..9ff8e1ea78d8 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -443,8 +443,9 @@ namespace clang {
     Expected<FunctionTemplateAndArgsTy>
     ImportFunctionTemplateWithTemplateArgsFromSpecialization(
         FunctionDecl *FromFD);
-    Error ImportTemplateParameterLists(const DeclaratorDecl *FromD,
-                                       DeclaratorDecl *ToD);
+
+    template <typename DeclTy>
+    Error ImportTemplateParameterLists(const DeclTy *FromD, DeclTy *ToD);
 
     Error ImportTemplateInformation(FunctionDecl *FromFD, FunctionDecl *ToFD);
 
@@ -3322,8 +3323,9 @@ ExpectedDecl ASTNodeImporter::VisitEnumConstantDecl(EnumConstantDecl *D) {
   return ToEnumerator;
 }
 
-Error ASTNodeImporter::ImportTemplateParameterLists(const DeclaratorDecl *FromD,
-                                                    DeclaratorDecl *ToD) {
+template <typename DeclTy>
+Error ASTNodeImporter::ImportTemplateParameterLists(const DeclTy *FromD,
+                                                    DeclTy *ToD) {
   unsigned int Num = FromD->getNumTemplateParameterLists();
   if (Num == 0)
     return Error::success();
@@ -6210,15 +6212,16 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
   if (!IdLocOrErr)
     return IdLocOrErr.takeError();
 
+  // Import TemplateArgumentListInfo.
+  TemplateArgumentListInfo ToTAInfo;
+  if (const auto *ASTTemplateArgs = D->getTemplateArgsAsWritten()) {
+    if (Error Err = ImportTemplateArgumentListInfo(*ASTTemplateArgs, ToTAInfo))
+      return std::move(Err);
+  }
+
   // Create the specialization.
   ClassTemplateSpecializationDecl *D2 = nullptr;
   if (PartialSpec) {
-    // Import TemplateArgumentListInfo.
-    TemplateArgumentListInfo ToTAInfo;
-    const auto &ASTTemplateArgs = *PartialSpec->getTemplateArgsAsWritten();
-    if (Error Err = ImportTemplateArgumentListInfo(ASTTemplateArgs, ToTAInfo))
-      return std::move(Err);
-
     QualType CanonInjType;
     if (Error Err = importInto(
         CanonInjType, PartialSpec->getInjectedSpecializationType()))
@@ -6228,7 +6231,7 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
     if (GetImportedOrCreateDecl<ClassTemplatePartialSpecializationDecl>(
             D2, D, Importer.getToContext(), D->getTagKind(), DC, *BeginLocOrErr,
             *IdLocOrErr, ToTPList, ClassTemplate,
-            llvm::ArrayRef(TemplateArgs.data(), TemplateArgs.size()), ToTAInfo,
+            llvm::ArrayRef(TemplateArgs.data(), TemplateArgs.size()),
             CanonInjType,
             cast_or_null<ClassTemplatePartialSpecializationDecl>(PrevDecl)))
       return D2;
@@ -6276,28 +6279,27 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
   else
     return BraceRangeOrErr.takeError();
 
+  if (Error Err = ImportTemplateParameterLists(D, D2))
+    return std::move(Err);
+
   // Import the qualifier, if any.
   if (auto LocOrErr = import(D->getQualifierLoc()))
     D2->setQualifierInfo(*LocOrErr);
   else
     return LocOrErr.takeError();
 
-  if (auto *TSI = D->getTypeAsWritten()) {
-    if (auto TInfoOrErr = import(TSI))
-      D2->setTypeAsWritten(*TInfoOrErr);
-    else
-      return TInfoOrErr.takeError();
+  if (D->getTemplateArgsAsWritten())
+    D2->setTemplateArgsAsWritten(ToTAInfo);
 
-    if (auto LocOrErr = import(D->getTemplateKeywordLoc()))
-      D2->setTemplateKeywordLoc(*LocOrErr);
-    else
-      return LocOrErr.takeError();
+  if (auto LocOrErr = import(D->getTemplateKeywordLoc()))
+    D2->setTemplateKeywordLoc(*LocOrErr);
+  else
+    return LocOrErr.takeError();
 
-    if (auto LocOrErr = import(D->getExternLoc()))
-      D2->setExternLoc(*LocOrErr);
-    else
-      return LocOrErr.takeError();
-  }
+  if (auto LocOrErr = import(D->getExternKeywordLoc()))
+    D2->setExternKeywordLoc(*LocOrErr);
+  else
+    return LocOrErr.takeError();
 
   if (D->getPointOfInstantiation().isValid()) {
     if (auto POIOrErr = import(D->getPointOfInstantiation()))
@@ -6517,7 +6519,7 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
   VarTemplateSpecializationDecl *D2 = nullptr;
 
   TemplateArgumentListInfo ToTAInfo;
-  if (const ASTTemplateArgumentListInfo *Args = D->getTemplateArgsInfo()) {
+  if (const auto *Args = D->getTemplateArgsAsWritten()) {
     if (Error Err = ImportTemplateArgumentListInfo(*Args, ToTAInfo))
       return std::move(Err);
   }
@@ -6525,14 +6527,6 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
   using PartVarSpecDecl = VarTemplatePartialSpecializationDecl;
   // Create a new specialization.
   if (auto *FromPartial = dyn_cast<PartVarSpecDecl>(D)) {
-    // Import TemplateArgumentListInfo
-    TemplateArgumentListInfo ArgInfos;
-    const auto *FromTAArgsAsWritten = FromPartial->getTemplateArgsAsWritten();
-    // NOTE: FromTAArgsAsWritten and template parameter list are non-null.
-    if (Error Err =
-            ImportTemplateArgumentListInfo(*FromTAArgsAsWritten, ArgInfos))
-      return std::move(Err);
-
     auto ToTPListOrErr = import(FromPartial->getTemplateParameters());
     if (!ToTPListOrErr)
       return ToTPListOrErr.takeError();
@@ -6541,7 +6535,7 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
     if (GetImportedOrCreateDecl(ToPartial, D, Importer.getToContext(), DC,
                                 *BeginLocOrErr, *IdLocOrErr, *ToTPListOrErr,
                                 VarTemplate, QualType(), nullptr,
-                                D->getStorageClass(), TemplateArgs, ArgInfos))
+                                D->getStorageClass(), TemplateArgs))
       return ToPartial;
 
     if (Expected<PartVarSpecDecl *> ToInstOrErr =
@@ -6584,7 +6578,9 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateSpecializationDecl(
   }
 
   D2->setSpecializationKind(D->getSpecializationKind());
-  D2->setTemplateArgsInfo(ToTAInfo);
+
+  if (D->getTemplateArgsAsWritten())
+    D2->setTemplateArgsAsWritten(ToTAInfo);
 
   if (auto LocOrErr = import(D->getQualifierLoc()))
     D2->setQualifierInfo(*LocOrErr);
diff --git a/clang/lib/AST/DeclPrinter.cpp b/clang/lib/AST/DeclPrinter.cpp
index 599d379340ab..c5868256b440 100644
--- a/clang/lib/AST/DeclPrinter.cpp
+++ b/clang/lib/AST/DeclPrinter.cpp
@@ -1083,15 +1083,15 @@ void DeclPrinter::VisitCXXRecordDecl(CXXRecordDecl *D) {
       NNS->print(Out, Policy);
     Out << *D;
 
-    if (auto S = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
-      ArrayRef<TemplateArgument> Args = S->getTemplateArgs().asArray();
-      if (!Policy.PrintCanonicalTypes)
-        if (const auto* TSI = S->getTypeAsWritten())
-          if (const auto *TST =
-                  dyn_cast<TemplateSpecializationType>(TSI->getType()))
-            Args = TST->template_arguments();
-      printTemplateArguments(
-          Args, S->getSpecializedTemplate()->getTemplateParameters());
+    if (auto *S = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
+      const TemplateParameterList *TParams =
+          S->getSpecializedTemplate()->getTemplateParameters();
+      const ASTTemplateArgumentListInfo *TArgAsWritten =
+          S->getTemplateArgsAsWritten();
+      if (TArgAsWritten && !Policy.PrintCanonicalTypes)
+        printTemplateArguments(TArgAsWritten->arguments(), TParams);
+      else
+        printTemplateArguments(S->getTemplateArgs().asArray(), TParams);
     }
   }
 
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index d27a30e0c5fc..d22ecc3c032e 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -985,41 +985,67 @@ ClassTemplateSpecializationDecl::getSpecializedTemplate() const {
 
 SourceRange
 ClassTemplateSpecializationDecl::getSourceRange() const {
-  if (ExplicitInfo) {
-    SourceLocation Begin = getTemplateKeywordLoc();
-    if (Begin.isValid()) {
-      // Here we have an explicit (partial) specialization or instantiation.
-      assert(getSpecializationKind() == TSK_ExplicitSpecialization ||
-             getSpecializationKind() == TSK_ExplicitInstantiationDeclaration ||
-             getSpecializationKind() == TSK_ExplicitInstantiationDefinition);
-      if (getExternLoc().isValid())
-        Begin = getExternLoc();
-      SourceLocation End = getBraceRange().getEnd();
-      if (End.isInvalid())
-        End = getTypeAsWritten()->getTypeLoc().getEndLoc();
-      return SourceRange(Begin, End);
-    }
-    // An implicit instantiation of a class template partial specialization
-    // uses ExplicitInfo to record the TypeAsWritten, but the source
-    // locations should be retrieved from the instantiation pattern.
-    using CTPSDecl = ClassTemplatePartialSpecializationDecl;
-    auto *ctpsd = const_cast<CTPSDecl *>(cast<CTPSDecl>(this));
-    CTPSDecl *inst_from = ctpsd->getInstantiatedFromMember();
-    assert(inst_from != nullptr);
-    return inst_from->getSourceRange();
-  }
-  else {
-    // No explicit info available.
+  switch (getSpecializationKind()) {
+  case TSK_Undeclared:
+  case TSK_ImplicitInstantiation: {
     llvm::PointerUnion<ClassTemplateDecl *,
                        ClassTemplatePartialSpecializationDecl *>
-      inst_from = getInstantiatedFrom();
-    if (inst_from.isNull())
-      return getSpecializedTemplate()->getSourceRange();
-    if (const auto *ctd = inst_from.dyn_cast<ClassTemplateDecl *>())
-      return ctd->getSourceRange();
-    return inst_from.get<ClassTemplatePartialSpecializationDecl *>()
-      ->getSourceRange();
+        Pattern = getSpecializedTemplateOrPartial();
+    assert(!Pattern.isNull() &&
+           "Class template specialization without pattern?");
+    if (const auto *CTPSD =
+            Pattern.dyn_cast<ClassTemplatePartialSpecializationDecl *>())
+      return CTPSD->getSourceRange();
+    return Pattern.get<ClassTemplateDecl *>()->getSourceRange();
+  }
+  case TSK_ExplicitSpecialization: {
+    SourceRange Range = CXXRecordDecl::getSourceRange();
+    if (const ASTTemplateArgumentListInfo *Args = getTemplateArgsAsWritten();
+        !isThisDeclarationADefinition() && Args)
+      Range.setEnd(Args->getRAngleLoc());
+    return Range;
+  }
+  case TSK_ExplicitInstantiationDeclaration:
+  case TSK_ExplicitInstantiationDefinition: {
+    SourceRange Range = CXXRecordDecl::getSourceRange();
+    if (SourceLocation ExternKW = getExternKeywordLoc(); ExternKW.isValid())
+      Range.setBegin(ExternKW);
+    else if (SourceLocation TemplateKW = getTemplateKeywordLoc();
+             TemplateKW.isValid())
+      Range.setBegin(TemplateKW);
+    if (const ASTTemplateArgumentListInfo *Args = getTemplateArgsAsWritten())
+      Range.setEnd(Args->getRAngleLoc());
+    return Range;
+  }
   }
+  llvm_unreachable("unhandled template specialization kind");
+}
+
+void ClassTemplateSpecializationDecl::setExternKeywordLoc(SourceLocation Loc) {
+  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  if (!Info) {
+    // Don't allocate if the location is invalid.
+    if (Loc.isInvalid())
+      return;
+    Info = new (getASTContext()) ExplicitInstantiationInfo;
+    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
+    ExplicitInfo = Info;
+  }
+  Info->ExternKeywordLoc = Loc;
+}
+
+void ClassTemplateSpecializationDecl::setTemplateKeywordLoc(
+    SourceLocation Loc) {
+  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  if (!Info) {
+    // Don't allocate if the location is invalid.
+    if (Loc.isInvalid())
+      return;
+    Info = new (getASTContext()) ExplicitInstantiationInfo;
+    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
+    ExplicitInfo = Info;
+  }
+  Info->TemplateKeywordLoc = Loc;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1087,43 +1113,29 @@ void ImplicitConceptSpecializationDecl::setTemplateArguments(
 //===----------------------------------------------------------------------===//
 void ClassTemplatePartialSpecializationDecl::anchor() {}
 
-ClassTemplatePartialSpecializationDecl::
-ClassTemplatePartialSpecializationDecl(ASTContext &Context, TagKind TK,
-                                       DeclContext *DC,
-                                       SourceLocation StartLoc,
-                                       SourceLocation IdLoc,
-                                       TemplateParameterList *Params,
-                                       ClassTemplateDecl *SpecializedTemplate,
-                                       ArrayRef<TemplateArgument> Args,
-                               const ASTTemplateArgumentListInfo *ArgInfos,
-                               ClassTemplatePartialSpecializationDecl *PrevDecl)
-    : ClassTemplateSpecializationDecl(Context,
-                                      ClassTemplatePartialSpecialization,
-                                      TK, DC, StartLoc, IdLoc,
-                                      SpecializedTemplate, Args, PrevDecl),
-      TemplateParams(Params), ArgsAsWritten(ArgInfos),
-      InstantiatedFromMember(nullptr, false) {
+ClassTemplatePartialSpecializationDecl::ClassTemplatePartialSpecializationDecl(
+    ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
+    SourceLocation IdLoc, TemplateParameterList *Params,
+    ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
+    ClassTemplatePartialSpecializationDecl *PrevDecl)
+    : ClassTemplateSpecializationDecl(
+          Context, ClassTemplatePartialSpecialization, TK, DC, StartLoc, IdLoc,
+          SpecializedTemplate, Args, PrevDecl),
+      TemplateParams(Params), InstantiatedFromMember(nullptr, false) {
   if (AdoptTemplateParameterList(Params, this))
     setInvalidDecl();
 }
 
 ClassTemplatePartialSpecializationDecl *
-ClassTemplatePartialSpecializationDecl::
-Create(ASTContext &Context, TagKind TK,DeclContext *DC,
-       SourceLocation StartLoc, SourceLocation IdLoc,
-       TemplateParameterList *Params,
-       ClassTemplateDecl *SpecializedTemplate,
-       ArrayRef<TemplateArgument> Args,
-       const TemplateArgumentListInfo &ArgInfos,
-       QualType CanonInjectedType,
-       ClassTemplatePartialSpecializationDecl *PrevDecl) {
-  const ASTTemplateArgumentListInfo *ASTArgInfos =
-    ASTTemplateArgumentListInfo::Create(Context, ArgInfos);
-
-  auto *Result = new (Context, DC)
-      ClassTemplatePartialSpecializationDecl(Context, TK, DC, StartLoc, IdLoc,
-                                             Params, SpecializedTemplate, Args,
-                                             ASTArgInfos, PrevDecl);
+ClassTemplatePartialSpecializationDecl::Create(
+    ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
+    SourceLocation IdLoc, TemplateParameterList *Params,
+    ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
+    QualType CanonInjectedType,
+    ClassTemplatePartialSpecializationDecl *PrevDecl) {
+  auto *Result = new (Context, DC) ClassTemplatePartialSpecializationDecl(
+      Context, TK, DC, StartLoc, IdLoc, Params, SpecializedTemplate, Args,
+      PrevDecl);
   Result->setSpecializationKind(TSK_ExplicitSpecialization);
   Result->setMayHaveOutOfDateDef(false);
 
@@ -1139,6 +1151,18 @@ ClassTemplatePartialSpecializationDecl::CreateDeserialized(ASTContext &C,
   return Result;
 }
 
+SourceRange ClassTemplatePartialSpecializationDecl::getSourceRange() const {
+  if (const ClassTemplatePartialSpecializationDecl *MT =
+          getInstantiatedFromMember();
+      MT && !isMemberSpecialization())
+    return MT->getSourceRange();
+  SourceRange Range = ClassTemplateSpecializationDecl::getSourceRange();
+  if (const TemplateParameterList *TPL = getTemplateParameters();
+      TPL && !getNumTemplateParameterLists())
+    Range.setBegin(TPL->getTemplateLoc());
+  return Range;
+}
+
 //===----------------------------------------------------------------------===//
 // FriendTemplateDecl Implementation
 //===----------------------------------------------------------------------===//
@@ -1371,27 +1395,74 @@ VarTemplateDecl *VarTemplateSpecializationDecl::getSpecializedTemplate() const {
   return SpecializedTemplate.get<VarTemplateDecl *>();
 }
 
-void VarTemplateSpecializationDecl::setTemplateArgsInfo(
-    const TemplateArgumentListInfo &ArgsInfo) {
-  TemplateArgsInfo =
-      ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo);
-}
-
-void VarTemplateSpecializationDecl::setTemplateArgsInfo(
-    const ASTTemplateArgumentListInfo *ArgsInfo) {
-  TemplateArgsInfo =
-      ASTTemplateArgumentListInfo::Create(getASTContext(), ArgsInfo);
-}
-
 SourceRange VarTemplateSpecializationDecl::getSourceRange() const {
-  if (isExplicitSpecialization() && !hasInit()) {
-    if (const ASTTemplateArgumentListInfo *Info = getTemplateArgsInfo())
-      return SourceRange(getOuterLocStart(), Info->getRAngleLoc());
+  switch (getSpecializationKind()) {
+  case TSK_Undeclared:
+  case TSK_ImplicitInstantiation: {
+    llvm::PointerUnion<VarTemplateDecl *,
+                       VarTemplatePartialSpecializationDecl *>
+        Pattern = getSpecializedTemplateOrPartial();
+    assert(!Pattern.isNull() &&
+           "Variable template specialization without pattern?");
+    if (const auto *VTPSD =
+            Pattern.dyn_cast<VarTemplatePartialSpecializationDecl *>())
+      return VTPSD->getSourceRange();
+    VarTemplateDecl *VTD = Pattern.get<VarTemplateDecl *>();
+    if (hasInit()) {
+      if (VarTemplateDecl *Definition = VTD->getDefinition())
+        return Definition->getSourceRange();
+    }
+    return VTD->getCanonicalDecl()->getSourceRange();
+  }
+  case TSK_ExplicitSpecialization: {
+    SourceRange Range = VarDecl::getSourceRange();
+    if (const ASTTemplateArgumentListInfo *Args = getTemplateArgsAsWritten();
+        !hasInit() && Args)
+      Range.setEnd(Args->getRAngleLoc());
+    return Range;
+  }
+  case TSK_ExplicitInstantiationDeclaration:
+  case TSK_ExplicitInstantiationDefinition: {
+    SourceRange Range = VarDecl::getSourceRange();
+    if (SourceLocation ExternKW = getExternKeywordLoc(); ExternKW.isValid())
+      Range.setBegin(ExternKW);
+    else if (SourceLocation TemplateKW = getTemplateKeywordLoc();
+             TemplateKW.isValid())
+      Range.setBegin(TemplateKW);
+    if (const ASTTemplateArgumentListInfo *Args = getTemplateArgsAsWritten())
+      Range.setEnd(Args->getRAngleLoc());
+    return Range;
   }
-  return VarDecl::getSourceRange();
+  }
+  llvm_unreachable("unhandled template specialization kind");
+}
+
+void VarTemplateSpecializationDecl::setExternKeywordLoc(SourceLocation Loc) {
+  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  if (!Info) {
+    // Don't allocate if the location is invalid.
+    if (Loc.isInvalid())
+      return;
+    Info = new (getASTContext()) ExplicitInstantiationInfo;
+    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
+    ExplicitInfo = Info;
+  }
+  Info->ExternKeywordLoc = Loc;
+}
+
+void VarTemplateSpecializationDecl::setTemplateKeywordLoc(SourceLocation Loc) {
+  auto *Info = ExplicitInfo.dyn_cast<ExplicitInstantiationInfo *>();
+  if (!Info) {
+    // Don't allocate if the location is invalid.
+    if (Loc.isInvalid())
+      return;
+    Info = new (getASTContext()) ExplicitInstantiationInfo;
+    Info->TemplateArgsAsWritten = getTemplateArgsAsWritten();
+    ExplicitInfo = Info;
+  }
+  Info->TemplateKeywordLoc = Loc;
 }
 
-
 //===----------------------------------------------------------------------===//
 // VarTemplatePartialSpecializationDecl Implementation
 //===----------------------------------------------------------------------===//
@@ -1402,13 +1473,11 @@ VarTemplatePartialSpecializationDecl::VarTemplatePartialSpecializationDecl(
     ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
     SourceLocation IdLoc, TemplateParameterList *Params,
     VarTemplateDecl *SpecializedTemplate, QualType T, TypeSourceInfo *TInfo,
-    StorageClass S, ArrayRef<TemplateArgument> Args,
-    const ASTTemplateArgumentListInfo *ArgInfos)
+    StorageClass S, ArrayRef<TemplateArgument> Args)
     : VarTemplateSpecializationDecl(VarTemplatePartialSpecialization, Context,
                                     DC, StartLoc, IdLoc, SpecializedTemplate, T,
                                     TInfo, S, Args),
-      TemplateParams(Params), ArgsAsWritten(ArgInfos),
-      InstantiatedFromMember(nullptr, false) {
+      TemplateParams(Params), InstantiatedFromMember(nullptr, false) {
   if (AdoptTemplateParameterList(Params, DC))
     setInvalidDecl();
 }
@@ -1418,15 +1487,10 @@ VarTemplatePartialSpecializationDecl::Create(
     ASTContext &Context, DeclContext *DC, SourceLocation StartLoc,
     SourceLocation IdLoc, TemplateParameterList *Params,
     VarTemplateDecl *SpecializedTemplate, QualType T, TypeSourceInfo *TInfo,
-    StorageClass S, ArrayRef<TemplateArgument> Args,
-    const TemplateArgumentListInfo &ArgInfos) {
-  const ASTTemplateArgumentListInfo *ASTArgInfos
-    = ASTTemplateArgumentListInfo::Create(Context, ArgInfos);
-
-  auto *Result =
-      new (Context, DC) VarTemplatePartialSpecializationDecl(
-          Context, DC, StartLoc, IdLoc, Params, SpecializedTemplate, T, TInfo,
-          S, Args, ASTArgInfos);
+    StorageClass S, ArrayRef<TemplateArgument> Args) {
+  auto *Result = new (Context, DC) VarTemplatePartialSpecializationDecl(
+      Context, DC, StartLoc, IdLoc, Params, SpecializedTemplate, T, TInfo, S,
+      Args);
   Result->setSpecializationKind(TSK_ExplicitSpecialization);
   return Result;
 }
@@ -1438,11 +1502,15 @@ VarTemplatePartialSpecializationDecl::CreateDeserialized(ASTContext &C,
 }
 
 SourceRange VarTemplatePartialSpecializationDecl::getSourceRange() const {
-  if (isExplicitSpecialization() && !hasInit()) {
-    if (const ASTTemplateArgumentListInfo *Info = getTemplateArgsAsWritten())
-      return SourceRange(getOuterLocStart(), Info->getRAngleLoc());
-  }
-  return VarDecl::getSourceRange();
+  if (const VarTemplatePartialSpecializationDecl *MT =
+          getInstantiatedFromMember();
+      MT && !isMemberSpecialization())
+    return MT->getSourceRange();
+  SourceRange Range = VarTemplateSpecializationDecl::getSourceRange();
+  if (const TemplateParameterList *TPL = getTemplateParameters();
+      TPL && !getNumTemplateParameterLists())
+    Range.setBegin(TPL->getTemplateLoc());
+  return Range;
 }
 
 static TemplateParameterList *
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 9602f448e942..87f0a8728d85 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -1472,21 +1472,18 @@ void TypePrinter::printTag(TagDecl *D, raw_ostream &OS) {
 
   // If this is a class template specialization, print the template
   // arguments.
-  if (const auto *Spec = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
-    ArrayRef<TemplateArgument> Args;
-    TypeSourceInfo *TAW = Spec->getTypeAsWritten();
-    if (!Policy.PrintCanonicalTypes && TAW) {
-      const TemplateSpecializationType *TST =
-        cast<TemplateSpecializationType>(TAW->getType());
-      Args = TST->template_arguments();
-    } else {
-      const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
-      Args = TemplateArgs.asArray();
-    }
+  if (auto *S = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
+    const TemplateParameterList *TParams =
+        S->getSpecializedTemplate()->getTemplateParameters();
+    const ASTTemplateArgumentListInfo *TArgAsWritten =
+        S->getTemplateArgsAsWritten();
     IncludeStrongLifetimeRAII Strong(Policy);
-    printTemplateArgumentList(
-        OS, Args, Policy,
-        Spec->getSpecializedTemplate()->getTemplateParameters());
+    if (TArgAsWritten && !Policy.PrintCanonicalTypes)
+      printTemplateArgumentList(OS, TArgAsWritten->arguments(), Policy,
+                                TParams);
+    else
+      printTemplateArgumentList(OS, S->getTemplateArgs().asArray(), Policy,
+                                TParams);
   }
 
   spaceBeforePlaceHolder(OS);
diff --git a/clang/lib/Index/IndexDecl.cpp b/clang/lib/Index/IndexDecl.cpp
index 1c04aa17d53f..8eb88f5a1e94 100644
--- a/clang/lib/Index/IndexDecl.cpp
+++ b/clang/lib/Index/IndexDecl.cpp
@@ -673,9 +673,12 @@ public:
     IndexCtx.indexTagDecl(
         D, SymbolRelation(SymbolRoleSet(SymbolRole::RelationSpecializationOf),
                           SpecializationOf));
-    if (TypeSourceInfo *TSI = D->getTypeAsWritten())
-      IndexCtx.indexTypeSourceInfo(TSI, /*Parent=*/nullptr,
-                                   D->getLexicalDeclContext());
+    // Template specialization arguments.
+    if (const ASTTemplateArgumentListInfo *TemplateArgInfo =
+            D->getTemplateArgsAsWritten()) {
+      for (const auto &Arg : TemplateArgInfo->arguments())
+        handleTemplateArgumentLoc(Arg, D, D->getLexicalDeclContext());
+    }
     return true;
   }
 
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 7585f1c367be..5d6ce233c775 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -1406,7 +1406,7 @@ void Sema::ActOnEndOfTranslationUnit() {
         SourceRange DiagRange = DiagD->getLocation();
         if (const auto *VTSD = dyn_cast<VarTemplateSpecializationDecl>(DiagD)) {
           if (const ASTTemplateArgumentListInfo *ASTTAL =
-                  VTSD->getTemplateArgsInfo())
+                  VTSD->getTemplateArgsAsWritten())
             DiagRange.setEnd(ASTTAL->RAngleLoc);
         }
         if (DiagD->isReferenced()) {
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 8219d5eed8db..c7aac068e264 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -5300,7 +5300,8 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
         VarTemplatePartialSpecializationDecl::Create(
             Context, VarTemplate->getDeclContext(), TemplateKWLoc,
             TemplateNameLoc, TemplateParams, VarTemplate, DI->getType(), DI, SC,
-            CanonicalConverted, TemplateArgs);
+            CanonicalConverted);
+    Partial->setTemplateArgsAsWritten(TemplateArgs);
 
     if (!PrevPartial)
       VarTemplate->AddPartialSpecialization(Partial, InsertPos);
@@ -5318,7 +5319,7 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
     Specialization = VarTemplateSpecializationDecl::Create(
         Context, VarTemplate->getDeclContext(), TemplateKWLoc, TemplateNameLoc,
         VarTemplate, DI->getType(), DI, SC, CanonicalConverted);
-    Specialization->setTemplateArgsInfo(TemplateArgs);
+    Specialization->setTemplateArgsAsWritten(TemplateArgs);
 
     if (!PrevDecl)
       VarTemplate->AddSpecialization(Specialization, InsertPos);
@@ -5353,7 +5354,6 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
     }
   }
 
-  Specialization->setTemplateKeywordLoc(TemplateKWLoc);
   Specialization->setLexicalDeclContext(CurContext);
 
   // Add the specialization into its lexical context, so that it can
@@ -9414,10 +9414,6 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     MultiTemplateParamsArg TemplateParameterLists, SkipBodyInfo *SkipBody) {
   assert(TUK != TUK_Reference && "References are not specializations");
 
-  // NOTE: KWLoc is the location of the tag keyword. This will instead
-  // store the location of the outermost template keyword in the declaration.
-  SourceLocation TemplateKWLoc = TemplateParameterLists.size() > 0
-    ? TemplateParameterLists[0]->getTemplateLoc() : KWLoc;
   SourceLocation TemplateNameLoc = TemplateId.TemplateNameLoc;
   SourceLocation LAngleLoc = TemplateId.LAngleLoc;
   SourceLocation RAngleLoc = TemplateId.RAngleLoc;
@@ -9629,7 +9625,8 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
         ClassTemplatePartialSpecializationDecl::Create(
             Context, Kind, ClassTemplate->getDeclContext(), KWLoc,
             TemplateNameLoc, TemplateParams, ClassTemplate, CanonicalConverted,
-            TemplateArgs, CanonType, PrevPartial);
+            CanonType, PrevPartial);
+    Partial->setTemplateArgsAsWritten(TemplateArgs);
     SetNestedNameSpecifier(*this, Partial, SS);
     if (TemplateParameterLists.size() > 1 && SS.isSet()) {
       Partial->setTemplateParameterListsInfo(
@@ -9652,6 +9649,7 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     Specialization = ClassTemplateSpecializationDecl::Create(
         Context, Kind, ClassTemplate->getDeclContext(), KWLoc, TemplateNameLoc,
         ClassTemplate, CanonicalConverted, PrevDecl);
+    Specialization->setTemplateArgsAsWritten(TemplateArgs);
     SetNestedNameSpecifier(*this, Specialization, SS);
     if (TemplateParameterLists.size() > 0) {
       Specialization->setTemplateParameterListsInfo(Context,
@@ -9735,21 +9733,6 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
       << (isPartialSpecialization? 1 : 0)
       << FixItHint::CreateRemoval(ModulePrivateLoc);
 
-  // Build the fully-sugared type for this class template
-  // specialization as the user wrote in the specialization
-  // itself. This means that we'll pretty-print the type retrieved
-  // from the specialization's declaration the way that the user
-  // actually wrote the specialization, rather than formatting the
-  // name based on the "canonical" representation used to store the
-  // template arguments in the specialization.
-  TypeSourceInfo *WrittenTy
-    = Context.getTemplateSpecializationTypeInfo(Name, TemplateNameLoc,
-                                                TemplateArgs, CanonType);
-  if (TUK != TUK_Friend) {
-    Specialization->setTypeAsWritten(WrittenTy);
-    Specialization->setTemplateKeywordLoc(TemplateKWLoc);
-  }
-
   // C++ [temp.expl.spec]p9:
   //   A template explicit specialization is in the scope of the
   //   namespace in which the template was defined.
@@ -9765,6 +9748,15 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     Specialization->startDefinition();
 
   if (TUK == TUK_Friend) {
+    // Build the fully-sugared type for this class template
+    // specialization as the user wrote in the specialization
+    // itself. This means that we'll pretty-print the type retrieved
+    // from the specialization's declaration the way that the user
+    // actually wrote the specialization, rather than formatting the
+    // name based on the "canonical" representation used to store the
+    // template arguments in the specialization.
+    TypeSourceInfo *WrittenTy = Context.getTemplateSpecializationTypeInfo(
+        Name, TemplateNameLoc, TemplateArgs, CanonType);
     FriendDecl *Friend = FriendDecl::Create(Context, CurContext,
                                             TemplateNameLoc,
                                             WrittenTy,
@@ -10999,21 +10991,10 @@ DeclResult Sema::ActOnExplicitInstantiation(
     }
   }
 
-  // Build the fully-sugared type for this explicit instantiation as
-  // the user wrote in the explicit instantiation itself. This means
-  // that we'll pretty-print the type retrieved from the
-  // specialization's declaration the way that the user actually wrote
-  // the explicit instantiation, rather than formatting the name based
-  // on the "canonical" representation used to store the template
-  // arguments in the specialization.
-  TypeSourceInfo *WrittenTy
-    = Context.getTemplateSpecializationTypeInfo(Name, TemplateNameLoc,
-                                                TemplateArgs,
-                                  Context.getTypeDeclType(Specialization));
-  Specialization->setTypeAsWritten(WrittenTy);
+  Specialization->setTemplateArgsAsWritten(TemplateArgs);
 
   // Set source locations for keywords.
-  Specialization->setExternLoc(ExternLoc);
+  Specialization->setExternKeywordLoc(ExternLoc);
   Specialization->setTemplateKeywordLoc(TemplateLoc);
   Specialization->setBraceRange(SourceRange());
 
@@ -11426,6 +11407,11 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     if (!HasNoEffect) {
       // Instantiate static data member or variable template.
       Prev->setTemplateSpecializationKind(TSK, D.getIdentifierLoc());
+      if (auto *VTSD = dyn_cast<VarTemplatePartialSpecializationDecl>(Prev)) {
+        VTSD->setExternKeywordLoc(ExternLoc);
+        VTSD->setTemplateKeywordLoc(TemplateLoc);
+      }
+
       // Merge attributes.
       ProcessDeclAttributeList(S, Prev, D.getDeclSpec().getAttributes());
       if (PrevTemplate)
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index e0c1f814f852..381d79b2fcd4 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -3861,15 +3861,16 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
 
   // Substitute into the template arguments of the class template explicit
   // specialization.
-  TemplateSpecializationTypeLoc Loc = D->getTypeAsWritten()->getTypeLoc().
-                                        castAs<TemplateSpecializationTypeLoc>();
-  TemplateArgumentListInfo InstTemplateArgs(Loc.getLAngleLoc(),
-                                            Loc.getRAngleLoc());
-  SmallVector<TemplateArgumentLoc, 4> ArgLocs;
-  for (unsigned I = 0; I != Loc.getNumArgs(); ++I)
-    ArgLocs.push_back(Loc.getArgLoc(I));
-  if (SemaRef.SubstTemplateArguments(ArgLocs, TemplateArgs, InstTemplateArgs))
-    return nullptr;
+  TemplateArgumentListInfo InstTemplateArgs;
+  if (const ASTTemplateArgumentListInfo *TemplateArgsInfo =
+          D->getTemplateArgsAsWritten()) {
+    InstTemplateArgs.setLAngleLoc(TemplateArgsInfo->getLAngleLoc());
+    InstTemplateArgs.setRAngleLoc(TemplateArgsInfo->getRAngleLoc());
+
+    if (SemaRef.SubstTemplateArguments(TemplateArgsInfo->arguments(),
+                                       TemplateArgs, InstTemplateArgs))
+      return nullptr;
+  }
 
   // Check that the template argument list is well-formed for this
   // class template.
@@ -3923,6 +3924,7 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
       ClassTemplateSpecializationDecl::Create(
           SemaRef.Context, D->getTagKind(), Owner, D->getBeginLoc(),
           D->getLocation(), InstClassTemplate, CanonicalConverted, PrevDecl);
+  InstD->setTemplateArgsAsWritten(InstTemplateArgs);
 
   // Add this partial specialization to the set of class template partial
   // specializations.
@@ -3933,28 +3935,10 @@ TemplateDeclInstantiator::VisitClassTemplateSpecializationDecl(
   if (SubstQualifier(D, InstD))
     return nullptr;
 
-  // Build the canonical type that describes the converted template
-  // arguments of the class template explicit specialization.
-  QualType CanonType = SemaRef.Context.getTemplateSpecializationType(
-      TemplateName(InstClassTemplate), CanonicalConverted,
-      SemaRef.Context.getRecordType(InstD));
-
-  // Build the fully-sugared type for this class template
-  // specialization as the user wrote in the specialization
-  // itself. This means that we'll pretty-print the type retrieved
-  // from the specialization's declaration the way that the user
-  // actually wrote the specialization, rather than formatting the
-  // name based on the "canonical" representation used to store the
-  // template arguments in the specialization.
-  TypeSourceInfo *WrittenTy = SemaRef.Context.getTemplateSpecializationTypeInfo(
-      TemplateName(InstClassTemplate), D->getLocation(), InstTemplateArgs,
-      CanonType);
-
   InstD->setAccess(D->getAccess());
   InstD->setInstantiationOfMemberClass(D, TSK_ImplicitInstantiation);
   InstD->setSpecializationKind(D->getSpecializationKind());
-  InstD->setTypeAsWritten(WrittenTy);
-  InstD->setExternLoc(D->getExternLoc());
+  InstD->setExternKeywordLoc(D->getExternKeywordLoc());
   InstD->setTemplateKeywordLoc(D->getTemplateKeywordLoc());
 
   Owner->addDecl(InstD);
@@ -3988,7 +3972,7 @@ Decl *TemplateDeclInstantiator::VisitVarTemplateSpecializationDecl(
 
   // Substitute the current template arguments.
   if (const ASTTemplateArgumentListInfo *TemplateArgsInfo =
-          D->getTemplateArgsInfo()) {
+          D->getTemplateArgsAsWritten()) {
     VarTemplateArgsInfo.setLAngleLoc(TemplateArgsInfo->getLAngleLoc());
     VarTemplateArgsInfo.setRAngleLoc(TemplateArgsInfo->getRAngleLoc());
 
@@ -4046,7 +4030,7 @@ Decl *TemplateDeclInstantiator::VisitVarTemplateSpecializationDecl(
   VarTemplateSpecializationDecl *Var = VarTemplateSpecializationDecl::Create(
       SemaRef.Context, Owner, D->getInnerLocStart(), D->getLocation(),
       VarTemplate, DI->getType(), DI, D->getStorageClass(), Converted);
-  Var->setTemplateArgsInfo(TemplateArgsInfo);
+  Var->setTemplateArgsAsWritten(TemplateArgsInfo);
   if (!PrevDecl) {
     void *InsertPos = nullptr;
     VarTemplate->findSpecialization(Converted, InsertPos);
@@ -4288,19 +4272,21 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
   QualType CanonType = SemaRef.Context.getTemplateSpecializationType(
       TemplateName(ClassTemplate), CanonicalConverted);
 
-  // Build the fully-sugared type for this class template
-  // specialization as the user wrote in the specialization
-  // itself. This means that we'll pretty-print the type retrieved
-  // from the specialization's declaration the way that the user
-  // actually wrote the specialization, rather than formatting the
-  // name based on the "canonical" representation used to store the
-  // template arguments in the specialization.
-  TypeSourceInfo *WrittenTy
-    = SemaRef.Context.getTemplateSpecializationTypeInfo(
-                                                    TemplateName(ClassTemplate),
-                                                    PartialSpec->getLocation(),
-                                                    InstTemplateArgs,
-                                                    CanonType);
+  // Create the class template partial specialization declaration.
+  ClassTemplatePartialSpecializationDecl *InstPartialSpec =
+      ClassTemplatePartialSpecializationDecl::Create(
+          SemaRef.Context, PartialSpec->getTagKind(), Owner,
+          PartialSpec->getBeginLoc(), PartialSpec->getLocation(), InstParams,
+          ClassTemplate, CanonicalConverted, CanonType,
+          /*PrevDecl=*/nullptr);
+
+  InstPartialSpec->setTemplateArgsAsWritten(InstTemplateArgs);
+
+  // Substitute the nested name specifier, if any.
+  if (SubstQualifier(PartialSpec, InstPartialSpec))
+    return nullptr;
+
+  InstPartialSpec->setInstantiatedFromMember(PartialSpec);
 
   if (PrevDecl) {
     // We've already seen a partial specialization with the same template
@@ -4318,28 +4304,14 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
     //
     //   Outer<int, int> outer; // error: the partial specializations of Inner
     //                          // have the same signature.
-    SemaRef.Diag(PartialSpec->getLocation(), diag::err_partial_spec_redeclared)
-      << WrittenTy->getType();
+    SemaRef.Diag(InstPartialSpec->getLocation(),
+                 diag::err_partial_spec_redeclared)
+        << InstPartialSpec;
     SemaRef.Diag(PrevDecl->getLocation(), diag::note_prev_partial_spec_here)
       << SemaRef.Context.getTypeDeclType(PrevDecl);
     return nullptr;
   }
 
-
-  // Create the class template partial specialization declaration.
-  ClassTemplatePartialSpecializationDecl *InstPartialSpec =
-      ClassTemplatePartialSpecializationDecl::Create(
-          SemaRef.Context, PartialSpec->getTagKind(), Owner,
-          PartialSpec->getBeginLoc(), PartialSpec->getLocation(), InstParams,
-          ClassTemplate, CanonicalConverted, InstTemplateArgs, CanonType,
-          nullptr);
-  // Substitute the nested name specifier, if any.
-  if (SubstQualifier(PartialSpec, InstPartialSpec))
-    return nullptr;
-
-  InstPartialSpec->setInstantiatedFromMember(PartialSpec);
-  InstPartialSpec->setTypeAsWritten(WrittenTy);
-
   // Check the completed partial specialization.
   SemaRef.CheckTemplatePartialSpecialization(InstPartialSpec);
 
@@ -4408,46 +4380,6 @@ TemplateDeclInstantiator::InstantiateVarTemplatePartialSpecialization(
       VarTemplate->findPartialSpecialization(CanonicalConverted, InstParams,
                                              InsertPos);
 
-  // Build the canonical type that describes the converted template
-  // arguments of the variable template partial specialization.
-  QualType CanonType = SemaRef.Context.getTemplateSpecializationType(
-      TemplateName(VarTemplate), CanonicalConverted);
-
-  // Build the fully-sugared type for this variable template
-  // specialization as the user wrote in the specialization
-  // itself. This means that we'll pretty-print the type retrieved
-  // from the specialization's declaration the way that the user
-  // actually wrote the specialization, rather than formatting the
-  // name based on the "canonical" representation used to store the
-  // template arguments in the specialization.
-  TypeSourceInfo *WrittenTy = SemaRef.Context.getTemplateSpecializationTypeInfo(
-      TemplateName(VarTemplate), PartialSpec->getLocation(), InstTemplateArgs,
-      CanonType);
-
-  if (PrevDecl) {
-    // We've already seen a partial specialization with the same template
-    // parameters and template arguments. This can happen, for example, when
-    // substituting the outer template arguments ends up causing two
-    // variable template partial specializations of a member variable template
-    // to have identical forms, e.g.,
-    //
-    //   template<typename T, typename U>
-    //   struct Outer {
-    //     template<typename X, typename Y> pair<X,Y> p;
-    //     template<typename Y> pair<T, Y> p;
-    //     template<typename Y> pair<U, Y> p;
-    //   };
-    //
-    //   Outer<int, int> outer; // error: the partial specializations of Inner
-    //                          // have the same signature.
-    SemaRef.Diag(PartialSpec->getLocation(),
-                 diag::err_var_partial_spec_redeclared)
-        << WrittenTy->getType();
-    SemaRef.Diag(PrevDecl->getLocation(),
-                 diag::note_var_prev_partial_spec_here);
-    return nullptr;
-  }
-
   // Do substitution on the type of the declaration
   TypeSourceInfo *DI = SemaRef.SubstType(
       PartialSpec->getTypeSourceInfo(), TemplateArgs,
@@ -4467,16 +4399,39 @@ TemplateDeclInstantiator::InstantiateVarTemplatePartialSpecialization(
       VarTemplatePartialSpecializationDecl::Create(
           SemaRef.Context, Owner, PartialSpec->getInnerLocStart(),
           PartialSpec->getLocation(), InstParams, VarTemplate, DI->getType(),
-          DI, PartialSpec->getStorageClass(), CanonicalConverted,
-          InstTemplateArgs);
+          DI, PartialSpec->getStorageClass(), CanonicalConverted);
+
+  InstPartialSpec->setTemplateArgsAsWritten(InstTemplateArgs);
 
   // Substitute the nested name specifier, if any.
   if (SubstQualifier(PartialSpec, InstPartialSpec))
     return nullptr;
 
   InstPartialSpec->setInstantiatedFromMember(PartialSpec);
-  InstPartialSpec->setTypeAsWritten(WrittenTy);
 
+  if (PrevDecl) {
+    // We've already seen a partial specialization with the same template
+    // parameters and template arguments. This can happen, for example, when
+    // substituting the outer template arguments ends up causing two
+    // variable template partial specializations of a member variable template
+    // to have identical forms, e.g.,
+    //
+    //   template<typename T, typename U>
+    //   struct Outer {
+    //     template<typename X, typename Y> pair<X,Y> p;
+    //     template<typename Y> pair<T, Y> p;
+    //     template<typename Y> pair<U, Y> p;
+    //   };
+    //
+    //   Outer<int, int> outer; // error: the partial specializations of Inner
+    //                          // have the same signature.
+    SemaRef.Diag(PartialSpec->getLocation(),
+                 diag::err_var_partial_spec_redeclared)
+        << InstPartialSpec;
+    SemaRef.Diag(PrevDecl->getLocation(),
+                 diag::note_var_prev_partial_spec_here);
+    return nullptr;
+  }
   // Check the completed partial specialization.
   SemaRef.CheckTemplatePartialSpecialization(InstPartialSpec);
 
@@ -5738,7 +5693,7 @@ void Sema::InstantiateVariableDefinition(SourceLocation PointOfInstantiation,
 
     TemplateArgumentListInfo TemplateArgInfo;
     if (const ASTTemplateArgumentListInfo *ArgInfo =
-            VarSpec->getTemplateArgsInfo()) {
+            VarSpec->getTemplateArgsAsWritten()) {
       TemplateArgInfo.setLAngleLoc(ArgInfo->getLAngleLoc());
       TemplateArgInfo.setRAngleLoc(ArgInfo->getRAngleLoc());
       for (const TemplateArgumentLoc &Arg : ArgInfo->arguments())
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 089ede4f4926..0c647086e304 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -2548,16 +2548,17 @@ ASTDeclReader::VisitClassTemplateSpecializationDeclImpl(
     }
   }
 
-  // Explicit info.
-  if (TypeSourceInfo *TyInfo = readTypeSourceInfo()) {
-    auto *ExplicitInfo =
-        new (C) ClassTemplateSpecializationDecl::ExplicitSpecializationInfo;
-    ExplicitInfo->TypeAsWritten = TyInfo;
-    ExplicitInfo->ExternLoc = readSourceLocation();
+  // extern/template keyword locations for explicit instantiations
+  if (Record.readBool()) {
+    auto *ExplicitInfo = new (C) ExplicitInstantiationInfo;
+    ExplicitInfo->ExternKeywordLoc = readSourceLocation();
     ExplicitInfo->TemplateKeywordLoc = readSourceLocation();
     D->ExplicitInfo = ExplicitInfo;
   }
 
+  if (Record.readBool())
+    D->setTemplateArgsAsWritten(Record.readASTTemplateArgumentListInfo());
+
   return Redecl;
 }
 
@@ -2567,7 +2568,6 @@ void ASTDeclReader::VisitClassTemplatePartialSpecializationDecl(
   // need them for profiling
   TemplateParameterList *Params = Record.readTemplateParameterList();
   D->TemplateParams = Params;
-  D->ArgsAsWritten = Record.readASTTemplateArgumentListInfo();
 
   RedeclarableResult Redecl = VisitClassTemplateSpecializationDeclImpl(D);
 
@@ -2617,16 +2617,17 @@ ASTDeclReader::VisitVarTemplateSpecializationDeclImpl(
     }
   }
 
-  // Explicit info.
-  if (TypeSourceInfo *TyInfo = readTypeSourceInfo()) {
-    auto *ExplicitInfo =
-        new (C) VarTemplateSpecializationDecl::ExplicitSpecializationInfo;
-    ExplicitInfo->TypeAsWritten = TyInfo;
-    ExplicitInfo->ExternLoc = readSourceLocation();
+  // extern/template keyword locations for explicit instantiations
+  if (Record.readBool()) {
+    auto *ExplicitInfo = new (C) ExplicitInstantiationInfo;
+    ExplicitInfo->ExternKeywordLoc = readSourceLocation();
     ExplicitInfo->TemplateKeywordLoc = readSourceLocation();
     D->ExplicitInfo = ExplicitInfo;
   }
 
+  if (Record.readBool())
+    D->setTemplateArgsAsWritten(Record.readASTTemplateArgumentListInfo());
+
   SmallVector<TemplateArgument, 8> TemplArgs;
   Record.readTemplateArgumentList(TemplArgs, /*Canonicalize*/ true);
   D->TemplateArgs = TemplateArgumentList::CreateCopy(C, TemplArgs);
@@ -2666,7 +2667,6 @@ void ASTDeclReader::VisitVarTemplatePartialSpecializationDecl(
     VarTemplatePartialSpecializationDecl *D) {
   TemplateParameterList *Params = Record.readTemplateParameterList();
   D->TemplateParams = Params;
-  D->ArgsAsWritten = Record.readASTTemplateArgumentListInfo();
 
   RedeclarableResult Redecl = VisitVarTemplateSpecializationDeclImpl(D);
 
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 6201d284f0e0..c2f1d1b44241 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -1765,20 +1765,28 @@ void ASTDeclWriter::VisitClassTemplateSpecializationDecl(
     Record.AddDeclRef(D->getSpecializedTemplate()->getCanonicalDecl());
   }
 
-  // Explicit info.
-  Record.AddTypeSourceInfo(D->getTypeAsWritten());
-  if (D->getTypeAsWritten()) {
-    Record.AddSourceLocation(D->getExternLoc());
+  bool ExplicitInstantiation =
+      D->getTemplateSpecializationKind() ==
+          TSK_ExplicitInstantiationDeclaration ||
+      D->getTemplateSpecializationKind() == TSK_ExplicitInstantiationDefinition;
+  Record.push_back(ExplicitInstantiation);
+  if (ExplicitInstantiation) {
+    Record.AddSourceLocation(D->getExternKeywordLoc());
     Record.AddSourceLocation(D->getTemplateKeywordLoc());
   }
 
+  const ASTTemplateArgumentListInfo *ArgsWritten =
+      D->getTemplateArgsAsWritten();
+  Record.push_back(!!ArgsWritten);
+  if (ArgsWritten)
+    Record.AddASTTemplateArgumentListInfo(ArgsWritten);
+
   Code = serialization::DECL_CLASS_TEMPLATE_SPECIALIZATION;
 }
 
 void ASTDeclWriter::VisitClassTemplatePartialSpecializationDecl(
                                     ClassTemplatePartialSpecializationDecl *D) {
   Record.AddTemplateParameterList(D->getTemplateParameters());
-  Record.AddASTTemplateArgumentListInfo(D->getTemplateArgsAsWritten());
 
   VisitClassTemplateSpecializationDecl(D);
 
@@ -1812,13 +1820,22 @@ void ASTDeclWriter::VisitVarTemplateSpecializationDecl(
     Record.AddTemplateArgumentList(&D->getTemplateInstantiationArgs());
   }
 
-  // Explicit info.
-  Record.AddTypeSourceInfo(D->getTypeAsWritten());
-  if (D->getTypeAsWritten()) {
-    Record.AddSourceLocation(D->getExternLoc());
+  bool ExplicitInstantiation =
+      D->getTemplateSpecializationKind() ==
+          TSK_ExplicitInstantiationDeclaration ||
+      D->getTemplateSpecializationKind() == TSK_ExplicitInstantiationDefinition;
+  Record.push_back(ExplicitInstantiation);
+  if (ExplicitInstantiation) {
+    Record.AddSourceLocation(D->getExternKeywordLoc());
     Record.AddSourceLocation(D->getTemplateKeywordLoc());
   }
 
+  const ASTTemplateArgumentListInfo *ArgsWritten =
+      D->getTemplateArgsAsWritten();
+  Record.push_back(!!ArgsWritten);
+  if (ArgsWritten)
+    Record.AddASTTemplateArgumentListInfo(ArgsWritten);
+
   Record.AddTemplateArgumentList(&D->getTemplateArgs());
   Record.AddSourceLocation(D->getPointOfInstantiation());
   Record.push_back(D->getSpecializationKind());
@@ -1839,7 +1856,6 @@ void ASTDeclWriter::VisitVarTemplateSpecializationDecl(
 void ASTDeclWriter::VisitVarTemplatePartialSpecializationDecl(
     VarTemplatePartialSpecializationDecl *D) {
   Record.AddTemplateParameterList(D->getTemplateParameters());
-  Record.AddASTTemplateArgumentListInfo(D->getTemplateArgsAsWritten());
 
   VisitVarTemplateSpecializationDecl(D);
 
diff --git a/clang/lib/Tooling/Syntax/BuildTree.cpp b/clang/lib/Tooling/Syntax/BuildTree.cpp
index cd0261989495..3e50d67f4d6e 100644
--- a/clang/lib/Tooling/Syntax/BuildTree.cpp
+++ b/clang/lib/Tooling/Syntax/BuildTree.cpp
@@ -735,7 +735,8 @@ public:
     auto *Declaration =
         cast<syntax::SimpleDeclaration>(handleFreeStandingTagDecl(C));
     foldExplicitTemplateInstantiation(
-        Builder.getTemplateRange(C), Builder.findToken(C->getExternLoc()),
+        Builder.getTemplateRange(C),
+        Builder.findToken(C->getExternKeywordLoc()),
         Builder.findToken(C->getTemplateKeywordLoc()), Declaration, C);
     return true;
   }
diff --git a/clang/test/AST/ast-dump-decl.cpp b/clang/test/AST/ast-dump-decl.cpp
index d74aa9045532..554cdcf83fcd 100644
--- a/clang/test/AST/ast-dump-decl.cpp
+++ b/clang/test/AST/ast-dump-decl.cpp
@@ -613,15 +613,15 @@ namespace testCanonicalTemplate {
   // CHECK:      VarTemplateDecl 0x{{.+}} <{{.+}}:[[@LINE-11]]:7, col:43> col:43 TestVarTemplate{{$}}
   // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{.+}} <col:16, col:25> col:25 referenced typename depth 0 index 0 T{{$}}
   // CHECK-NEXT: |-VarDecl 0x{{.+}} <col:28, col:43> col:43 TestVarTemplate 'const T' static{{$}}
-  // CHECK-NEXT: |-VarTemplateSpecializationDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <line:[[@LINE-11]]:3, col:34> col:14 referenced TestVarTemplate 'const int' implicit_instantiation cinit{{$}}
+  // CHECK-NEXT: |-VarTemplateSpecializationDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <line:[[@LINE-12]]:3, line:[[@LINE-11]]:34> col:14 referenced TestVarTemplate 'const int' implicit_instantiation cinit{{$}}
   // CHECK-NEXT: | |-NestedNameSpecifier TypeSpec 'testCanonicalTemplate::S'{{$}}
   // CHECK-NEXT: | |-TemplateArgument type 'int'{{$}}
   // CHECK-NEXT: | | `-BuiltinType 0x{{.+}} 'int'{{$}}
   // CHECK-NEXT: | `-InitListExpr 0x{{.+}} <col:32, col:34> 'int'{{$}}
-  // CHECK-NEXT: `-VarTemplateSpecializationDecl 0x{{.+}} <line:[[@LINE-19]]:28, col:43> col:43 referenced TestVarTemplate 'const int' implicit_instantiation static{{$}}
+  // CHECK-NEXT: `-VarTemplateSpecializationDecl 0x{{.+}} <line:[[@LINE-19]]:7, col:43> col:43 referenced TestVarTemplate 'const int' implicit_instantiation static{{$}}
   // CHECK-NEXT:   `-TemplateArgument type 'int'{{$}}
 
-  // CHECK:     VarTemplateSpecializationDecl 0x{{.+}} <{{.+}}:[[@LINE-22]]:28, col:43> col:43 referenced TestVarTemplate 'const int' implicit_instantiation static{{$}}
+  // CHECK:     VarTemplateSpecializationDecl 0x{{.+}} <{{.+}}:[[@LINE-22]]:7, col:43> col:43 referenced TestVarTemplate 'const int' implicit_instantiation static{{$}}
   // CHECK-NEXT:`-TemplateArgument type 'int'{{$}}
   // CHECK-NEXT:  `-BuiltinType 0x{{.+}} 'int'{{$}}
 
@@ -632,13 +632,13 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT: | `-InitListExpr 0x{{.+}} <col:32, col:34> 'void'{{$}}
   // CHECK-NEXT: |-VarTemplateSpecialization 0x{{.+}} 'TestVarTemplate' 'const int'{{$}}
   // CHECK-NEXT: `-VarTemplateSpecialization 0x{{.+}} 'TestVarTemplate' 'const int'{{$}}
-    
-  // CHECK:      VarTemplateSpecializationDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-31]]:3, col:34> col:14 referenced TestVarTemplate 'const int' implicit_instantiation cinit{{$}}
+
+  // CHECK:      VarTemplateSpecializationDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-32]]:3, line:[[@LINE-31]]:34> col:14 referenced TestVarTemplate 'const int' implicit_instantiation cinit{{$}}
   // CHECK-NEXT: |-NestedNameSpecifier TypeSpec 'testCanonicalTemplate::S'{{$}}
   // CHECK-NEXT: |-TemplateArgument type 'int'{{$}}
   // CHECK-NEXT: | `-BuiltinType 0x{{.+}} 'int'{{$}}
   // CHECK-NEXT: `-InitListExpr 0x{{.+}} <col:32, col:34> 'int'{{$}}
-} 
+}
 
 template <class T>
 class TestClassScopeFunctionSpecialization {
diff --git a/clang/test/AST/ast-dump-template-decls.cpp b/clang/test/AST/ast-dump-template-decls.cpp
index 142bc9e6ad9a..37f6d8a0472d 100644
--- a/clang/test/AST/ast-dump-template-decls.cpp
+++ b/clang/test/AST/ast-dump-template-decls.cpp
@@ -1,12 +1,12 @@
 // Test without serialization:
 // RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown -ast-dump %s \
-// RUN: | FileCheck -strict-whitespace %s --check-prefix=DIRECT
+// RUN: | FileCheck -strict-whitespace %s
 //
 // Test with serialization:
 // RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -x c++ -std=c++17 -triple x86_64-unknown-unknown -include-pch %t -ast-dump-all /dev/null \
 // RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
-// RUN: | FileCheck --strict-whitespace %s --check-prefix=SERIALIZED
+// RUN: | FileCheck --strict-whitespace %s
 
 template <typename Ty>
 // CHECK: FunctionTemplateDecl 0x{{[^ ]*}} <{{.*}}:1, line:[[@LINE+2]]:10> col:6 a
@@ -189,15 +189,13 @@ T unTempl = 1;
 
 template<>
 int unTempl<int>;
-// FIXME (#61680) - serializing and loading AST should not affect reported source range
-// DIRECT:     VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:16> col:5 unTempl 'int'
-// SERIALIZED: VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:5> col:5 unTempl 'int'
+// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:16> col:5 unTempl 'int'
 // CHECK-NEXT: `-TemplateArgument type 'int'
 // CHECK-NEXT: `-BuiltinType 0x{{[^ ]*}} 'int'
 
 template<>
 float unTempl<float> = 1;
-// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:24> col:7 unTempl 'float' cinit
+// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:24> col:7 unTempl 'float'
 // CHECK-NEXT: |-TemplateArgument type 'float'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'float'
 // CHECK-NEXT: `-ImplicitCastExpr 0x{{[^ ]*}} <col:24> 'float' <IntegralToFloating>
@@ -222,7 +220,7 @@ int binTempl<int, U>;
 
 template<class U>
 float binTempl<float, U> = 1;
-// CHECK:      VarTemplatePartialSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:24> col:7 binTempl 'float' cinit
+// CHECK:      VarTemplatePartialSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:28> col:7 binTempl 'float'
 // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:10, col:16> col:16 referenced class depth 0 index 0 U
 // CHECK-NEXT: |-TemplateArgument type 'float'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'float'
@@ -233,9 +231,7 @@ float binTempl<float, U> = 1;
 
 template<>
 int binTempl<int, int>;
-// FIXME (#61680) - serializing and loading AST should not affect reported source range
-// DIRECT:     VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:22> col:5 binTempl 'int'
-// SERIALIZED: VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:5> col:5 binTempl 'int'
+// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:22> col:5 binTempl 'int'
 // CHECK-NEXT: |-TemplateArgument type 'int'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'int'
 // CHECK-NEXT: `-TemplateArgument type 'int'
@@ -243,7 +239,7 @@ int binTempl<int, int>;
 
 template<>
 float binTempl<float, float> = 1;
-// CHECK:     VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:32> col:7 binTempl 'float' cinit
+// CHECK:      VarTemplateSpecializationDecl 0x{{[^ ]*}} <line:{{[0-9]+}}:1, line:{{[0-9]+}}:32> col:7 binTempl 'float'
 // CHECK-NEXT: |-TemplateArgument type 'float'
 // CHECK-NEXT: | `-BuiltinType 0x{{[^ ]*}} 'float'
 // CHECK-NEXT: |-TemplateArgument type 'float'
diff --git a/clang/test/Index/Core/index-source.cpp b/clang/test/Index/Core/index-source.cpp
index 8f9fbc4c8d29..043e616a1d36 100644
--- a/clang/test/Index/Core/index-source.cpp
+++ b/clang/test/Index/Core/index-source.cpp
@@ -285,20 +285,17 @@ template<>
 class SpecializationDecl<int>;
 // CHECK: [[@LINE-1]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Decl,RelSpecialization | rel: 1
 // CHECK-NEXT: RelSpecialization | SpecializationDecl | c:@ST>1#T@SpecializationDecl
-// CHECK: [[@LINE-3]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Ref | rel: 0
 
 template<>
 class SpecializationDecl<int> { };
 // CHECK: [[@LINE-1]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Def,RelSpecialization | rel: 1
 // CHECK-NEXT: RelSpecialization | SpecializationDecl | c:@ST>1#T@SpecializationDecl
-// CHECK-NEXT: [[@LINE-3]]:7 | class(Gen,TS)/C++ | SpecializationDecl | c:@S@SpecializationDecl>#I | <no-cgname> | Ref | rel: 0
 
 template<typename T>
 class PartialSpecilizationClass<Cls, T>;
 // CHECK: [[@LINE-1]]:7 | class(Gen,TPS)/C++ | PartialSpecilizationClass | c:@SP>1#T@PartialSpecilizationClass>#$@S@Cls#t0.0 | <no-cgname> | Decl,RelSpecialization | rel: 1
 // CHECK-NEXT: RelSpecialization | PartialSpecilizationClass | c:@ST>2#T#T@PartialSpecilizationClass
-// CHECK: [[@LINE-3]]:7 | class(Gen)/C++ | PartialSpecilizationClass | c:@ST>2#T#T@PartialSpecilizationClass | <no-cgname> | Ref | rel: 0
-// CHECK-NEXT: [[@LINE-4]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
+// CHECK-NEXT: [[@LINE-3]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
 
 template<>
 class PartialSpecilizationClass<Cls, Cls> : Cls { };
@@ -306,9 +303,10 @@ class PartialSpecilizationClass<Cls, Cls> : Cls { };
 // CHECK-NEXT: RelSpecialization | PartialSpecilizationClass | c:@ST>2#T#T@PartialSpecilizationClass
 // CHECK-NEXT: [[@LINE-3]]:45 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelBase,RelCont | rel: 1
 // CHECK-NEXT: RelBase,RelCont | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_
-// CHECK-NEXT: [[@LINE-5]]:7 | class(Gen,TS)/C++ | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_ | <no-cgname> | Ref | rel: 0
-// CHECK-NEXT: [[@LINE-6]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
-// CHECK-NEXT: [[@LINE-7]]:38 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
+// CHECK-NEXT: [[@LINE-5]]:33 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK-NEXT: RelCont | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_
+// CHECK-NEXT: [[@LINE-7]]:38 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK-NEXT: RelCont | PartialSpecilizationClass | c:@S@PartialSpecilizationClass>#$@S@Cls#S0_
 
 template<typename T, int x>
 void functionSp() { }
@@ -332,10 +330,14 @@ class ClassWithCorrectSpecialization { };
 
 template<>
 class ClassWithCorrectSpecialization<SpecializationDecl<Cls>, Record::C> { };
-// CHECK: [[@LINE-1]]:38 | class(Gen)/C++ | SpecializationDecl | c:@ST>1#T@SpecializationDecl | <no-cgname> | Ref | rel: 0
-// CHECK: [[@LINE-2]]:57 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref | rel: 0
-// CHECK: [[@LINE-3]]:71 | static-property/C++ | C | c:@S@Record@C | __ZN6Record1CE | Ref,Read | rel: 0
-// CHECK: [[@LINE-4]]:63 | struct/C++ | Record | c:@S@Record | <no-cgname> | Ref | rel: 0
+// CHECK: [[@LINE-1]]:38 | class(Gen)/C++ | SpecializationDecl | c:@ST>1#T@SpecializationDecl | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
+// CHECK-NEXT: [[@LINE-3]]:57 | class/C++ | Cls | c:@S@Cls | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
+// CHECK-NEXT: [[@LINE-5]]:71 | static-property/C++ | C | c:@S@Record@C | __ZN6Record1CE | Ref,Read,RelCont | rel: 1
+// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
+// CHECK-NEXT: [[@LINE-7]]:63 | struct/C++ | Record | c:@S@Record | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK-NEXT: RelCont | ClassWithCorrectSpecialization | c:@S@ClassWithCorrectSpecialization>#$@S@SpecializationDecl>#$@S@Cls#VI2
 
 namespace ns {
 // CHECK: [[@LINE-1]]:11 | namespace/C++ | ns | c:@N@ns | <no-cgname> | Decl | rel: 0
diff --git a/clang/test/Index/index-refs.cpp b/clang/test/Index/index-refs.cpp
index 0e613e48522b..14946849777d 100644
--- a/clang/test/Index/index-refs.cpp
+++ b/clang/test/Index/index-refs.cpp
@@ -108,7 +108,6 @@ int ginitlist[] = {EnumVal};
 // CHECK:      [indexDeclaration]: kind: c++-class-template | name: TS | {{.*}} | loc: 47:8
 // CHECK-NEXT: [indexDeclaration]: kind: struct-template-partial-spec | name: TS | USR: c:@SP>1#T@TS>#t0.0#I | {{.*}} | loc: 50:8
 // CHECK-NEXT: [indexDeclaration]: kind: typedef | name: MyInt | USR: c:index-refs.cpp@SP>1#T@TS>#t0.0#I@T@MyInt | {{.*}} | loc: 51:15 | semantic-container: [TS:50:8] | lexical-container: [TS:50:8]
-// CHECK-NEXT: [indexEntityReference]: kind: c++-class-template | name: TS | USR: c:@ST>2#T#T@TS | lang: C++ | cursor: TemplateRef=TS:47:8 | loc: 50:8 | <parent>:: <<NULL>> | container: [TU] | refkind: direct | role: ref
 /* when indexing implicit instantiations
   [indexDeclaration]: kind: struct-template-spec | name: TS | USR: c:@S@TS>#I | {{.*}} | loc: 50:8
   [indexDeclaration]: kind: typedef | name: MyInt | USR: c:index-refs.cpp@593@S@TS>#I@T@MyInt | {{.*}} | loc: 51:15 | semantic-container: [TS:50:8] | lexical-container: [TS:50:8]
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 8b9417f985b5..bfbdb5be9ff2 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -743,14 +743,10 @@ bool CursorVisitor::VisitClassTemplateSpecializationDecl(
   }
 
   // Visit the template arguments used in the specialization.
-  if (TypeSourceInfo *SpecType = D->getTypeAsWritten()) {
-    TypeLoc TL = SpecType->getTypeLoc();
-    if (TemplateSpecializationTypeLoc TSTLoc =
-            TL.getAs<TemplateSpecializationTypeLoc>()) {
-      for (unsigned I = 0, N = TSTLoc.getNumArgs(); I != N; ++I)
-        if (VisitTemplateArgumentLoc(TSTLoc.getArgLoc(I)))
-          return true;
-    }
+  if (const auto *ArgsWritten = D->getTemplateArgsAsWritten()) {
+    for (const TemplateArgumentLoc &Arg : ArgsWritten->arguments())
+      if (VisitTemplateArgumentLoc(Arg))
+        return true;
   }
 
   return ShouldVisitBody && VisitCXXRecordDecl(D);
@@ -5667,16 +5663,19 @@ CXString clang_getCursorDisplayName(CXCursor C) {
 
   if (const ClassTemplateSpecializationDecl *ClassSpec =
           dyn_cast<ClassTemplateSpecializationDecl>(D)) {
-    // If the type was explicitly written, use that.
-    if (TypeSourceInfo *TSInfo = ClassSpec->getTypeAsWritten())
-      return cxstring::createDup(TSInfo->getType().getAsString(Policy));
-
     SmallString<128> Str;
     llvm::raw_svector_ostream OS(Str);
     OS << *ClassSpec;
-    printTemplateArgumentList(
-        OS, ClassSpec->getTemplateArgs().asArray(), Policy,
-        ClassSpec->getSpecializedTemplate()->getTemplateParameters());
+    // If the template arguments were written explicitly, use them..
+    if (const auto *ArgsWritten = ClassSpec->getTemplateArgsAsWritten()) {
+      printTemplateArgumentList(
+          OS, ArgsWritten->arguments(), Policy,
+          ClassSpec->getSpecializedTemplate()->getTemplateParameters());
+    } else {
+      printTemplateArgumentList(
+          OS, ClassSpec->getTemplateArgs().asArray(), Policy,
+          ClassSpec->getSpecializedTemplate()->getTemplateParameters());
+    }
     return cxstring::createDup(OS.str());
   }
 
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index b76627cb9be6..65df513d2713 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -2213,18 +2213,6 @@ TEST_P(ASTMatchersTest, ReferenceTypeLocTest_BindsToAnyRvalueReferenceTypeLoc) {
   EXPECT_TRUE(matches("float&& r = 3.0;", matcher));
 }
 
-TEST_P(
-    ASTMatchersTest,
-    TemplateSpecializationTypeLocTest_BindsToTemplateSpecializationExplicitInstantiation) {
-  if (!GetParam().isCXX()) {
-    return;
-  }
-  EXPECT_TRUE(
-      matches("template <typename T> class C {}; template class C<int>;",
-              classTemplateSpecializationDecl(
-                  hasName("C"), hasTypeLoc(templateSpecializationTypeLoc()))));
-}
-
 TEST_P(ASTMatchersTest,
        TemplateSpecializationTypeLocTest_BindsToVarDeclTemplateSpecialization) {
   if (!GetParam().isCXX()) {
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
index f198dc71eb83..af99c73f1945 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
@@ -430,12 +430,6 @@ TEST(HasTypeLoc, MatchesCXXUnresolvedConstructExpr) {
               cxxUnresolvedConstructExpr(hasTypeLoc(loc(asString("T"))))));
 }
 
-TEST(HasTypeLoc, MatchesClassTemplateSpecializationDecl) {
-  EXPECT_TRUE(matches(
-      "template <typename T> class Foo; template <> class Foo<int> {};",
-      classTemplateSpecializationDecl(hasTypeLoc(loc(asString("Foo<int>"))))));
-}
-
 TEST(HasTypeLoc, MatchesCompoundLiteralExpr) {
   EXPECT_TRUE(
       matches("int* x = (int[2]) { 0, 1 };",
@@ -6384,8 +6378,7 @@ TEST(HasAnyTemplateArgumentLoc, BindsToExplicitSpecializationWithIntArgument) {
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(
-              hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))))));
+          hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc,
@@ -6394,8 +6387,7 @@ TEST(HasAnyTemplateArgumentLoc,
       "template<typename T> class A {}; template<> class A<double> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-              hasTypeLoc(loc(asString("double")))))))));
+          hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("double")))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
@@ -6405,24 +6397,20 @@ TEST(HasAnyTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
   )";
   EXPECT_TRUE(
       matches(code, classTemplateSpecializationDecl(
-                        hasName("A"), hasTypeLoc(templateSpecializationTypeLoc(
-                                          hasAnyTemplateArgumentLoc(hasTypeLoc(
-                                              loc(asString("double")))))))));
+                        hasName("A"), hasAnyTemplateArgumentLoc(hasTypeLoc(
+                                          loc(asString("double")))))));
+
   EXPECT_TRUE(matches(
-      code,
-      classTemplateSpecializationDecl(
-          hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(
-              hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))))));
+      code, classTemplateSpecializationDecl(
+                hasName("A"),
+                hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc, DoesNotBindToSpecializationWithIntArgument) {
-  EXPECT_TRUE(notMatches(
-      "template<typename T> class A {}; A<int> a;",
-      classTemplateSpecializationDecl(
-          hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-              hasTypeLoc(loc(asString("double")))))))));
+  EXPECT_TRUE(notMatches("template<typename T> class A {}; A<int> a;",
+                         classTemplateSpecializationDecl(
+                             hasName("A"), hasAnyTemplateArgumentLoc(hasTypeLoc(
+                                               loc(asString("double")))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc,
@@ -6431,8 +6419,7 @@ TEST(HasAnyTemplateArgumentLoc,
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-              hasTypeLoc(loc(asString("double")))))))));
+          hasAnyTemplateArgumentLoc(hasTypeLoc(loc(asString("double")))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToSpecializationWithIntArgument) {
@@ -6453,13 +6440,21 @@ TEST(HasTemplateArgumentLoc, BindsToSpecializationWithDoubleArgument) {
                               0, hasTypeLoc(loc(asString("double")))))))))));
 }
 
+TEST(HasTemplateArgumentLoc, DoesNotBindToSpecializationWithIntArgument) {
+  EXPECT_TRUE(notMatches(
+      "template<typename T> class A {}; A<int> a;",
+      varDecl(hasName("a"),
+              hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+                  templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+                      0, hasTypeLoc(loc(asString("double")))))))))));
+}
+
 TEST(HasTemplateArgumentLoc, BindsToExplicitSpecializationWithIntArgument) {
   EXPECT_TRUE(matches(
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(
-              hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("int")))))))));
+          hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToExplicitSpecializationWithDoubleArgument) {
@@ -6467,8 +6462,7 @@ TEST(HasTemplateArgumentLoc, BindsToExplicitSpecializationWithDoubleArgument) {
       "template<typename T> class A {}; template<> class A<double> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-              0, hasTypeLoc(loc(asString("double")))))))));
+          hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("double")))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
@@ -6478,23 +6472,12 @@ TEST(HasTemplateArgumentLoc, BindsToSpecializationWithMultipleArguments) {
   )";
   EXPECT_TRUE(matches(
       code, classTemplateSpecializationDecl(
-                hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    0, hasTypeLoc(loc(asString("double")))))))));
+                hasName("A"), hasTemplateArgumentLoc(
+                                  0, hasTypeLoc(loc(asString("double")))))));
   EXPECT_TRUE(matches(
       code, classTemplateSpecializationDecl(
                 hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    1, hasTypeLoc(loc(asString("int")))))))));
-}
-
-TEST(HasTemplateArgumentLoc, DoesNotBindToSpecializationWithIntArgument) {
-  EXPECT_TRUE(notMatches(
-      "template<typename T> class A {}; A<int> a;",
-      classTemplateSpecializationDecl(
-          hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-              0, hasTypeLoc(loc(asString("double")))))))));
+                hasTemplateArgumentLoc(1, hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasTemplateArgumentLoc,
@@ -6503,8 +6486,7 @@ TEST(HasTemplateArgumentLoc,
       "template<typename T> class A {}; template<> class A<int> {};",
       classTemplateSpecializationDecl(
           hasName("A"),
-          hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-              0, hasTypeLoc(loc(asString("double")))))))));
+          hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("double")))))));
 }
 
 TEST(HasTemplateArgumentLoc,
@@ -6515,14 +6497,12 @@ TEST(HasTemplateArgumentLoc,
   )";
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
-                hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    1, hasTypeLoc(loc(asString("double")))))))));
+                hasName("A"), hasTemplateArgumentLoc(
+                                  1, hasTypeLoc(loc(asString("double")))))));
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
                 hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    0, hasTypeLoc(loc(asString("int")))))))));
+                hasTemplateArgumentLoc(0, hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasTemplateArgumentLoc, DoesNotBindWithBadIndex) {
@@ -6532,14 +6512,12 @@ TEST(HasTemplateArgumentLoc, DoesNotBindWithBadIndex) {
   )";
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
-                hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    -1, hasTypeLoc(loc(asString("double")))))))));
+                hasName("A"), hasTemplateArgumentLoc(
+                                  -1, hasTypeLoc(loc(asString("double")))))));
   EXPECT_TRUE(notMatches(
       code, classTemplateSpecializationDecl(
-                hasName("A"),
-                hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                    100, hasTypeLoc(loc(asString("int")))))))));
+                hasName("A"), hasTemplateArgumentLoc(
+                                  100, hasTypeLoc(loc(asString("int")))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToDeclRefExprWithIntArgument) {
-- 
GitLab


From fb8f492a1ccb2236a82701c76f82960fd6cdb725 Mon Sep 17 00:00:00 2001
From: Peiming Liu <peiming@google.com>
Date: Tue, 14 May 2024 13:26:49 -0700
Subject: [PATCH 0779/1206] =?UTF-8?q?[mlir][sparse]=20clone=20a=20empty=20?=
 =?UTF-8?q?sparse=20tensor=20when=20fuse=20convert=20into=20pro=E2=80=A6?=
 =?UTF-8?q?=20(#92158)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ducer.
---
 .../Transforms/SparseTensorRewriting.cpp      | 14 +++---
 .../fuse_sparse_convert_into_producer.mlir    | 44 +++++++++++++++++++
 2 files changed, 51 insertions(+), 7 deletions(-)

diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index da635c257888..5fb009e3eebe 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -302,17 +302,17 @@ public:
         !producer.getResult(0).hasOneUse()) {
       return failure();
     }
+    // Clone the materialization operation, but update the result to sparse.
+    rewriter.setInsertionPoint(producer);
+    Operation *init = producer.getDpsInitOperand(0)->get().getDefiningOp();
+    Operation *cloned = rewriter.clone(*init);
+    cloned->getResult(0).setType(op.getResult().getType());
+
     rewriter.modifyOpInPlace(producer, [&]() {
+      producer.getDpsInitsMutable().assign(cloned->getResults());
       producer.getResult(0).setType(op.getResult().getType());
     });
 
-    Operation *materializeOp =
-        producer.getDpsInitOperand(0)->get().getDefiningOp();
-
-    rewriter.modifyOpInPlace(materializeOp, [&]() {
-      materializeOp->getResult(0).setType(op.getResult().getType());
-    });
-
     rewriter.replaceAllOpUsesWith(op, producer);
     op->erase();
 
diff --git a/mlir/test/Dialect/SparseTensor/fuse_sparse_convert_into_producer.mlir b/mlir/test/Dialect/SparseTensor/fuse_sparse_convert_into_producer.mlir
index efa92e565ba5..4e4d2c27b096 100644
--- a/mlir/test/Dialect/SparseTensor/fuse_sparse_convert_into_producer.mlir
+++ b/mlir/test/Dialect/SparseTensor/fuse_sparse_convert_into_producer.mlir
@@ -54,6 +54,50 @@ func.func @fold_convert(%arg0: tensor<128x32x32x1xf32>, %arg1: tensor<128x32x32x
   return %2 : tensor<128x32x32x1xf32, #CCCD>
 }
 
+#trait_bin = {
+  indexing_maps = [
+      affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>,
+      affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>,
+      affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+  ],
+  iterator_types = ["parallel", "parallel", "parallel", "parallel"]
+}
+
+// CHECK-FOLD-LABEL:   func.func @fold_convert_multi_use(
+// CHECK-FOLD:           tensor.empty() : tensor<128x32x32x1xf32>
+// CHECK-FOLD:           linalg.generic
+// CHECK-FOLD:           tensor.empty() : tensor<128x32x32x1xf32, #sparse>
+// CHECK-FOLD:           linalg.generic
+// CHECK-FOLD-NOT:       sparse_tensor.convert
+func.func @fold_convert_multi_use(%arg0: tensor<128x32x32x1xf32>, %arg1: tensor<128x32x32x1xf32>,
+                        %arg2: tensor<128x32x32x1xf32>, %arg3: tensor<128x32x32x1xf32>) -> (tensor<128x32x32x1xf32>, tensor<128x32x32x1xf32, #CCCD>) {
+  %cst = arith.constant 0.000000e+00 : f32
+  %cst_0 = arith.constant 1.000000e+00 : f32
+  %cst_1 = arith.constant 1.000000e+00 : f32
+
+  %0 = tensor.empty() : tensor<128x32x32x1xf32>
+  %1 = linalg.generic #trait_bin
+  ins(%arg0, %arg1 : tensor<128x32x32x1xf32>, tensor<128x32x32x1xf32>)
+  outs(%0 : tensor<128x32x32x1xf32>) {
+    ^bb0(%in: f32, %in_1: f32, %out: f32):
+      %3 = arith.mulf %in, %in_1 : f32
+      linalg.yield %3 : f32
+    } -> tensor<128x32x32x1xf32>
+
+  // A second kernel that uses %0 as the init operand.
+  %3 = linalg.generic #trait_bin
+  ins(%arg2, %arg3 : tensor<128x32x32x1xf32>, tensor<128x32x32x1xf32>)
+  outs(%0 : tensor<128x32x32x1xf32>) {
+    ^bb0(%in: f32, %in_1: f32, %out: f32):
+      %3 = arith.mulf %in, %in_1 : f32
+      linalg.yield %3 : f32
+    } -> tensor<128x32x32x1xf32>
+  %4 = sparse_tensor.convert %3 : tensor<128x32x32x1xf32> to tensor<128x32x32x1xf32, #CCCD>
+
+  return %1, %4 : tensor<128x32x32x1xf32>, tensor<128x32x32x1xf32, #CCCD>
+}
+
+
 
 // FIXME: The following kernel is not sparsifiable because `arith.select`
 // operations is not handled by the sparse compiler at the moment.
-- 
GitLab


From baca93fc83ee3b9ef32cd328dc4275a06177c8c7 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Tue, 14 May 2024 13:33:31 -0700
Subject: [PATCH 0780/1206] [LSR] Tweak debug output to always print initial
 cost

---
 llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp             | 4 ++--
 .../LoopStrengthReduce/RISCV/lsr-drop-solution-dbg-msg.ll     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index eb1904ccaff3..35a17d6060c9 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -5251,8 +5251,6 @@ void LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const {
   assert(Solution.size() == Uses.size() && "Malformed solution!");
 
   if (BaselineCost.isLess(SolutionCost)) {
-    LLVM_DEBUG(dbgs() << "The baseline solution requires ";
-               BaselineCost.print(dbgs()); dbgs() << "\n");
     if (!AllowDropSolutionIfLessProfitable)
       LLVM_DEBUG(
           dbgs() << "Baseline is more profitable than chosen solution, "
@@ -5931,6 +5929,8 @@ LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
 
   LLVM_DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n";
              print_uses(dbgs()));
+  LLVM_DEBUG(dbgs() << "The baseline solution requires ";
+             BaselineCost.print(dbgs()); dbgs() << "\n");
 
   // Now use the reuse data to generate a bunch of interesting ways
   // to formulate the values needed for the uses.
diff --git a/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-drop-solution-dbg-msg.ll b/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-drop-solution-dbg-msg.ll
index 37876a907124..8d9d43202f0d 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-drop-solution-dbg-msg.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-drop-solution-dbg-msg.ll
@@ -6,8 +6,8 @@ target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
 target triple = "riscv64-unknown-linux-gnu"
 
 define ptr @foo(ptr %a0, ptr %a1, i64 %a2) {
-;DEBUG: The chosen solution requires 3 instructions 6 regs, with addrec cost 1, plus 2 base adds, plus 5 setup cost
 ;DEBUG: The baseline solution requires 2 instructions 4 regs, with addrec cost 2, plus 3 setup cost
+;DEBUG: The chosen solution requires 3 instructions 6 regs, with addrec cost 1, plus 2 base adds, plus 5 setup cost
 ;DEBUG: Baseline is more profitable than chosen solution, dropping LSR solution.
 
 ;DEBUG2: Baseline is more profitable than chosen solution, add option 'lsr-drop-solution' to drop LSR solution.
-- 
GitLab


From f83df080a817c99e90ed1a0bd5693c5f07ebf567 Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Tue, 14 May 2024 13:35:04 -0700
Subject: [PATCH 0781/1206] [NFC] add comment to keep RegState in sync with doc
 (#92170)

---
 llvm/include/llvm/CodeGen/MachineInstrBuilder.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h
index a5b8d3af3cc9..0a73b0f6b367 100644
--- a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h
+++ b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h
@@ -40,6 +40,7 @@ class MDNode;
 
 namespace RegState {
 
+// Keep this in sync with the table in MIRLangRef.rst.
 enum {
   /// Register definition.
   Define = 0x2,
-- 
GitLab


From f918c056f06968763870bc3e6b9f9d7074e1f867 Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Tue, 14 May 2024 13:43:04 -0700
Subject: [PATCH 0782/1206] [lldb] Allow env override for LLDB_ARGDUMPER_PATH
 (#91688)

This mirrors the LLDB_DEBUGSERVER_PATH environment variable and allows
you to have lldb-argdumper in a non-standard location and still use it
at runtime.
---
 lldb/source/Host/macosx/objcxx/Host.mm | 35 ++++++++++++++++++--------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/lldb/source/Host/macosx/objcxx/Host.mm b/lldb/source/Host/macosx/objcxx/Host.mm
index 4fba5550ba10..e6f1c0ea3d29 100644
--- a/lldb/source/Host/macosx/objcxx/Host.mm
+++ b/lldb/source/Host/macosx/objcxx/Host.mm
@@ -1387,18 +1387,31 @@ Status Host::LaunchProcess(ProcessLaunchInfo &launch_info) {
 Status Host::ShellExpandArguments(ProcessLaunchInfo &launch_info) {
   Status error;
   if (launch_info.GetFlags().Test(eLaunchFlagShellExpandArguments)) {
-    FileSpec expand_tool_spec = HostInfo::GetSupportExeDir();
-    if (!expand_tool_spec) {
-      error.SetErrorString(
-          "could not get support executable directory for lldb-argdumper tool");
-      return error;
+    FileSpec expand_tool_spec;
+    Environment host_env = Host::GetEnvironment();
+    std::string env_argdumper_path = host_env.lookup("LLDB_ARGDUMPER_PATH");
+    if (!env_argdumper_path.empty()) {
+      expand_tool_spec.SetFile(env_argdumper_path, FileSpec::Style::native);
+      Log *log(GetLog(LLDBLog::Host | LLDBLog::Process));
+      LLDB_LOGF(log,
+                "lldb-argdumper exe path set from environment variable: %s",
+                env_argdumper_path.c_str());
     }
-    expand_tool_spec.AppendPathComponent("lldb-argdumper");
-    if (!FileSystem::Instance().Exists(expand_tool_spec)) {
-      error.SetErrorStringWithFormat(
-          "could not find the lldb-argdumper tool: %s",
-          expand_tool_spec.GetPath().c_str());
-      return error;
+    bool argdumper_exists = FileSystem::Instance().Exists(env_argdumper_path);
+    if (!argdumper_exists) {
+      expand_tool_spec = HostInfo::GetSupportExeDir();
+      if (!expand_tool_spec) {
+        error.SetErrorString("could not get support executable directory for "
+                             "lldb-argdumper tool");
+        return error;
+      }
+      expand_tool_spec.AppendPathComponent("lldb-argdumper");
+      if (!FileSystem::Instance().Exists(expand_tool_spec)) {
+        error.SetErrorStringWithFormat(
+            "could not find the lldb-argdumper tool: %s",
+            expand_tool_spec.GetPath().c_str());
+        return error;
+      }
     }
 
     StreamString expand_tool_spec_stream;
-- 
GitLab


From 67beebfcb9a267cc1e443aa4d3788adbfcf02639 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Tue, 14 May 2024 17:15:19 -0400
Subject: [PATCH 0783/1206] [TableGen][SubtargetEmitter] Refactor
 hasReadOfWrite to CodeGenProcModel (#92032)

SubtargetEmitter::GenSchedClassTables takes a CodeGenProcModel, but
calls hasReadOfWrite which loops over all ProcModels. We move
hasReadOfWrite to CodeGenProcModel and remove the loop over all
ProcModels. This leads to a 144% speedup on the RISC-V backend of our
downstream.
---
 .../utils/TableGen/Common/CodeGenSchedule.cpp | 21 ++++++++-----------
 llvm/utils/TableGen/Common/CodeGenSchedule.h  |  6 +++---
 llvm/utils/TableGen/SubtargetEmitter.cpp      |  4 +---
 3 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/llvm/utils/TableGen/Common/CodeGenSchedule.cpp b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
index 0e81623a6aa3..2ec0812320d1 100644
--- a/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
@@ -746,18 +746,6 @@ unsigned CodeGenSchedModels::getSchedRWIdx(const Record *Def,
   return I == RWVec.end() ? 0 : std::distance(RWVec.begin(), I);
 }
 
-bool CodeGenSchedModels::hasReadOfWrite(Record *WriteDef) const {
-  for (auto &ProcModel : ProcModels) {
-    const RecVec &RADefs = ProcModel.ReadAdvanceDefs;
-    for (auto &RADef : RADefs) {
-      RecVec ValidWrites = RADef->getValueAsListOfDefs("ValidWrites");
-      if (is_contained(ValidWrites, WriteDef))
-        return true;
-    }
-  }
-  return false;
-}
-
 static void splitSchedReadWrites(const RecVec &RWDefs, RecVec &WriteDefs,
                                  RecVec &ReadDefs) {
   for (Record *RWDef : RWDefs) {
@@ -2226,6 +2214,15 @@ bool CodeGenProcModel::isUnsupported(const CodeGenInstruction &Inst) const {
   return false;
 }
 
+bool CodeGenProcModel::hasReadOfWrite(Record *WriteDef) const {
+  for (auto &RADef : ReadAdvanceDefs) {
+    RecVec ValidWrites = RADef->getValueAsListOfDefs("ValidWrites");
+    if (is_contained(ValidWrites, WriteDef))
+      return true;
+  }
+  return false;
+}
+
 #ifndef NDEBUG
 void CodeGenProcModel::dump() const {
   dbgs() << Index << ": " << ModelName << " "
diff --git a/llvm/utils/TableGen/Common/CodeGenSchedule.h b/llvm/utils/TableGen/Common/CodeGenSchedule.h
index 61980e7e196e..10ec7f41f56f 100644
--- a/llvm/utils/TableGen/Common/CodeGenSchedule.h
+++ b/llvm/utils/TableGen/Common/CodeGenSchedule.h
@@ -277,6 +277,9 @@ struct CodeGenProcModel {
 
   bool isUnsupported(const CodeGenInstruction &Inst) const;
 
+  // Return true if the given write record is referenced by a ReadAdvance.
+  bool hasReadOfWrite(Record *WriteDef) const;
+
 #ifndef NDEBUG
   void dump() const;
 #endif
@@ -536,9 +539,6 @@ public:
 
   unsigned getSchedRWIdx(const Record *Def, bool IsRead) const;
 
-  // Return true if the given write record is referenced by a ReadAdvance.
-  bool hasReadOfWrite(Record *WriteDef) const;
-
   // Get a SchedClass from its index.
   CodeGenSchedClass &getSchedClass(unsigned Idx) {
     assert(Idx < SchedClasses.size() && "bad SchedClass index");
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index b6b7641cfb92..9e32d2de19b2 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -1122,10 +1122,8 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
       WriterNames.push_back(SchedModels.getSchedWrite(WriteID).Name);
       // If this Write is not referenced by a ReadAdvance, don't distinguish it
       // from other WriteLatency entries.
-      if (!SchedModels.hasReadOfWrite(
-              SchedModels.getSchedWrite(WriteID).TheDef)) {
+      if (!ProcModel.hasReadOfWrite(SchedModels.getSchedWrite(WriteID).TheDef))
         WriteID = 0;
-      }
       WLEntry.WriteResourceID = WriteID;
 
       for (unsigned WS : WriteSeq) {
-- 
GitLab


From 536abf827b481f78a0879b02202fb9a3ffe3a908 Mon Sep 17 00:00:00 2001
From: GeorgeHuyubo <113479859+GeorgeHuyubo@users.noreply.github.com>
Date: Tue, 14 May 2024 14:35:35 -0700
Subject: [PATCH 0784/1206] Read and store gnu build id from loaded core file
 (#92078)

As we have debuginfod as symbol locator available in lldb now, we want
to make full use of it.
In case of post mortem debugging, we don't always have the main
executable available.
However, the .note.gnu.build-id of the main executable(some other
modules too), should be available in the core file, as those binaries
are loaded in memory and dumped in the core file.

We try to iterate through the NT_FILE entries, read and store the gnu
build id if possible. This will be very useful as this id is the unique
key which is needed for querying the debuginfod server.

Test:
Build and run lldb. Breakpoint set to
https://github.com/llvm/llvm-project/blob/main/lldb/source/Plugins/SymbolLocator/Debuginfod/SymbolLocatorDebuginfod.cpp#L147
Verified after this commit, module_uuid is the correct gnu build id of
the main executable which caused the crash(first in the NT_FILE entry)
---
 lldb/include/lldb/Target/Process.h            | 50 +++++++++++++++
 lldb/source/Commands/CommandObjectMemory.cpp  | 61 +------------------
 .../Process/elf-core/ProcessElfCore.cpp       | 50 +++++++++++++++
 .../Plugins/Process/elf-core/ProcessElfCore.h | 11 ++++
 lldb/source/Target/Process.cpp                | 27 ++++++++
 5 files changed, 140 insertions(+), 59 deletions(-)

diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h
index aac0cf51680a..c8a49edc5c78 100644
--- a/lldb/include/lldb/Target/Process.h
+++ b/lldb/include/lldb/Target/Process.h
@@ -406,6 +406,36 @@ public:
                                   lldb::StateType state);
   } Notifications;
 
+  class ProcessMemoryIterator {
+  public:
+    ProcessMemoryIterator(lldb::ProcessSP process_sp, lldb::addr_t base)
+        : m_process_sp(process_sp), m_base_addr(base) {
+      lldbassert(process_sp.get() != nullptr);
+    }
+
+    bool IsValid() { return m_is_valid; }
+
+    uint8_t operator[](lldb::addr_t offset) {
+      if (!IsValid())
+        return 0;
+
+      uint8_t retval = 0;
+      Status error;
+      if (0 ==
+          m_process_sp->ReadMemory(m_base_addr + offset, &retval, 1, error)) {
+        m_is_valid = false;
+        return 0;
+      }
+
+      return retval;
+    }
+
+  private:
+    lldb::ProcessSP m_process_sp;
+    lldb::addr_t m_base_addr;
+    bool m_is_valid = true;
+  };
+
   class ProcessEventData : public EventData {
     friend class Process;
 
@@ -1649,6 +1679,26 @@ public:
 
   lldb::addr_t ReadPointerFromMemory(lldb::addr_t vm_addr, Status &error);
 
+  /// Find a string within a memory region.
+  ///
+  /// This function searches for the string represented by the provided buffer
+  /// within the memory range specified by the low and high addresses. It uses
+  /// a bad character heuristic to optimize the search process.
+  ///
+  /// \param[in] low The starting address of the memory region to be searched.
+  ///
+  /// \param[in] high The ending address of the memory region to be searched.
+  ///
+  /// \param[in] buffer A pointer to the buffer containing the string to be
+  /// searched.
+  ///
+  /// \param[in] buffer_size The size of the buffer in bytes.
+  ///
+  /// \return The address where the string was found or LLDB_INVALID_ADDRESS if
+  /// not found.
+  lldb::addr_t FindInMemory(lldb::addr_t low, lldb::addr_t high,
+                            uint8_t *buffer, size_t buffer_size);
+
   bool WritePointerToMemory(lldb::addr_t vm_addr, lldb::addr_t ptr_value,
                             Status &error);
 
diff --git a/lldb/source/Commands/CommandObjectMemory.cpp b/lldb/source/Commands/CommandObjectMemory.cpp
index b78a0492cca5..1c13484dede6 100644
--- a/lldb/source/Commands/CommandObjectMemory.cpp
+++ b/lldb/source/Commands/CommandObjectMemory.cpp
@@ -977,35 +977,6 @@ public:
   Options *GetOptions() override { return &m_option_group; }
 
 protected:
-  class ProcessMemoryIterator {
-  public:
-    ProcessMemoryIterator(ProcessSP process_sp, lldb::addr_t base)
-        : m_process_sp(process_sp), m_base_addr(base) {
-      lldbassert(process_sp.get() != nullptr);
-    }
-
-    bool IsValid() { return m_is_valid; }
-
-    uint8_t operator[](lldb::addr_t offset) {
-      if (!IsValid())
-        return 0;
-
-      uint8_t retval = 0;
-      Status error;
-      if (0 ==
-          m_process_sp->ReadMemory(m_base_addr + offset, &retval, 1, error)) {
-        m_is_valid = false;
-        return 0;
-      }
-
-      return retval;
-    }
-
-  private:
-    ProcessSP m_process_sp;
-    lldb::addr_t m_base_addr;
-    bool m_is_valid = true;
-  };
   void DoExecute(Args &command, CommandReturnObject &result) override {
     // No need to check "process" for validity as eCommandRequiresProcess
     // ensures it is valid
@@ -1106,8 +1077,8 @@ protected:
     found_location = low_addr;
     bool ever_found = false;
     while (count) {
-      found_location = FastSearch(found_location, high_addr, buffer.GetBytes(),
-                                  buffer.GetByteSize());
+      found_location = process->FindInMemory(
+          found_location, high_addr, buffer.GetBytes(), buffer.GetByteSize());
       if (found_location == LLDB_INVALID_ADDRESS) {
         if (!ever_found) {
           result.AppendMessage("data not found within the range.\n");
@@ -1144,34 +1115,6 @@ protected:
     result.SetStatus(lldb::eReturnStatusSuccessFinishResult);
   }
 
-  lldb::addr_t FastSearch(lldb::addr_t low, lldb::addr_t high, uint8_t *buffer,
-                          size_t buffer_size) {
-    const size_t region_size = high - low;
-
-    if (region_size < buffer_size)
-      return LLDB_INVALID_ADDRESS;
-
-    std::vector<size_t> bad_char_heuristic(256, buffer_size);
-    ProcessSP process_sp = m_exe_ctx.GetProcessSP();
-    ProcessMemoryIterator iterator(process_sp, low);
-
-    for (size_t idx = 0; idx < buffer_size - 1; idx++) {
-      decltype(bad_char_heuristic)::size_type bcu_idx = buffer[idx];
-      bad_char_heuristic[bcu_idx] = buffer_size - idx - 1;
-    }
-    for (size_t s = 0; s <= (region_size - buffer_size);) {
-      int64_t j = buffer_size - 1;
-      while (j >= 0 && buffer[j] == iterator[s + j])
-        j--;
-      if (j < 0)
-        return low + s;
-      else
-        s += bad_char_heuristic[iterator[s + buffer_size - 1]];
-    }
-
-    return LLDB_INVALID_ADDRESS;
-  }
-
   OptionGroupOptions m_option_group;
   OptionGroupFindMemory m_memory_options;
   OptionGroupMemoryTag m_memory_tag_options;
diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp
index 36812c27a5b6..4ff03eb8ab48 100644
--- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp
+++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp
@@ -6,10 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <cstddef>
 #include <cstdlib>
 
 #include <memory>
 #include <mutex>
+#include <tuple>
 
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleSpec.h"
@@ -210,6 +212,9 @@ Status ProcessElfCore::DoLoadCore() {
     }
   }
 
+  // We need to update uuid after address range is populated.
+  UpdateBuildIdForNTFileEntries();
+
   if (!ranges_are_sorted) {
     m_core_aranges.Sort();
     m_core_range_infos.Sort();
@@ -258,6 +263,7 @@ Status ProcessElfCore::DoLoadCore() {
     if (!m_nt_file_entries.empty()) {
       ModuleSpec exe_module_spec;
       exe_module_spec.GetArchitecture() = arch;
+      exe_module_spec.GetUUID() = m_nt_file_entries[0].uuid;
       exe_module_spec.GetFileSpec().SetFile(m_nt_file_entries[0].path,
                                             FileSpec::Style::native);
       if (exe_module_spec.GetFileSpec()) {
@@ -271,6 +277,16 @@ Status ProcessElfCore::DoLoadCore() {
   return error;
 }
 
+void ProcessElfCore::UpdateBuildIdForNTFileEntries() {
+  if (!m_nt_file_entries.empty()) {
+    for (NT_FILE_Entry &entry : m_nt_file_entries) {
+      std::optional<UUID> uuid = FindBuildId(entry);
+      if (uuid)
+        entry.uuid = uuid.value();
+    }
+  }
+}
+
 lldb_private::DynamicLoader *ProcessElfCore::GetDynamicLoader() {
   if (m_dyld_up.get() == nullptr)
     m_dyld_up.reset(DynamicLoader::FindPlugin(
@@ -983,6 +999,40 @@ llvm::Error ProcessElfCore::ParseThreadContextsFromNoteSegment(
   }
 }
 
+bool ProcessElfCore::IsElf(const NT_FILE_Entry entry) {
+  size_t size = strlen(llvm::ELF::ElfMagic);
+  uint8_t buf[size];
+  Status error;
+  size_t byte_read = ReadMemory(entry.start, buf, size, error);
+  if (byte_read == size)
+    return memcmp(llvm::ELF::ElfMagic, buf, size) == 0;
+  else
+    return false;
+}
+
+std::optional<UUID> ProcessElfCore::FindBuildId(const NT_FILE_Entry entry) {
+  if (!IsElf(entry))
+    return std::nullopt;
+  // Build ID is stored in the ELF file as a section named ".note.gnu.build-id"
+  uint8_t gnu_build_id_bytes[8] = {0x03, 0x00, 0x00, 0x00,
+                                   0x47, 0x4e, 0x55, 0x00};
+  lldb::addr_t gnu_build_id_addr =
+      FindInMemory(entry.start, entry.end, gnu_build_id_bytes, 8);
+  if (gnu_build_id_addr == LLDB_INVALID_ADDRESS)
+    return std::nullopt;
+  uint8_t buf[36];
+  Status error;
+  size_t byte_read = ReadMemory(gnu_build_id_addr - 8, buf, 36, error);
+  // .note.gnu.build-id starts with 04 00 00 00 {id_byte_size} 00 00 00 03 00 00
+  // 00 47 4e 55 00
+  if (byte_read == 36) {
+    if (buf[0] == 0x04) {
+      return UUID(llvm::ArrayRef<uint8_t>(buf + 16, buf[4] /*byte size*/));
+    }
+  }
+  return std::nullopt;
+}
+
 uint32_t ProcessElfCore::GetNumThreadContexts() {
   if (!m_thread_data_valid)
     DoLoadCore();
diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h
index 2cec635bbacf..ae827f3df002 100644
--- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h
+++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h
@@ -117,6 +117,8 @@ private:
     lldb::addr_t end;
     lldb::addr_t file_ofs;
     std::string path;
+    lldb_private::UUID
+        uuid; // extracted from .note.gnu.build-id section from core file
   };
 
   // For ProcessElfCore only
@@ -158,6 +160,15 @@ private:
   // Returns number of thread contexts stored in the core file
   uint32_t GetNumThreadContexts();
 
+  // Populate gnu uuid for each NT_FILE entry
+  void UpdateBuildIdForNTFileEntries();
+
+  // Returns the UUID of a given NT_FILE entry
+  std::optional<lldb_private::UUID> FindBuildId(const NT_FILE_Entry entry);
+
+  // Returns true if the given NT_FILE entry is an ELF file
+  bool IsElf(const NT_FILE_Entry entry);
+
   // Parse a contiguous address range of the process from LOAD segment
   lldb::addr_t
   AddAddressRangeFromLoadSegment(const elf::ELFProgramHeader &header);
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index 25afade9a827..6f5c43bc4108 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -3191,6 +3191,33 @@ Status Process::Halt(bool clear_thread_plans, bool use_run_lock) {
   return Status();
 }
 
+lldb::addr_t Process::FindInMemory(lldb::addr_t low, lldb::addr_t high,
+                                   uint8_t *buffer, size_t buffer_size) {
+  const size_t region_size = high - low;
+
+  if (region_size < buffer_size)
+    return LLDB_INVALID_ADDRESS;
+
+  std::vector<size_t> bad_char_heuristic(256, buffer_size);
+  ProcessMemoryIterator iterator(shared_from_this(), low);
+
+  for (size_t idx = 0; idx < buffer_size - 1; idx++) {
+    decltype(bad_char_heuristic)::size_type bcu_idx = buffer[idx];
+    bad_char_heuristic[bcu_idx] = buffer_size - idx - 1;
+  }
+  for (size_t s = 0; s <= (region_size - buffer_size);) {
+    int64_t j = buffer_size - 1;
+    while (j >= 0 && buffer[j] == iterator[s + j])
+      j--;
+    if (j < 0)
+      return low + s;
+    else
+      s += bad_char_heuristic[iterator[s + buffer_size - 1]];
+  }
+
+  return LLDB_INVALID_ADDRESS;
+}
+
 Status Process::StopForDestroyOrDetach(lldb::EventSP &exit_event_sp) {
   Status error;
 
-- 
GitLab


From 5bf653ca42dceb8266a0ff70634292ccd2ad4c43 Mon Sep 17 00:00:00 2001
From: GeorgeHuyubo <113479859+GeorgeHuyubo@users.noreply.github.com>
Date: Tue, 14 May 2024 14:36:17 -0700
Subject: [PATCH 0785/1206] Revert "Read and store gnu build id from loaded
 core file" (#92181)

Reverts llvm/llvm-project#92078
---
 lldb/include/lldb/Target/Process.h            | 50 ---------------
 lldb/source/Commands/CommandObjectMemory.cpp  | 61 ++++++++++++++++++-
 .../Process/elf-core/ProcessElfCore.cpp       | 50 ---------------
 .../Plugins/Process/elf-core/ProcessElfCore.h | 11 ----
 lldb/source/Target/Process.cpp                | 27 --------
 5 files changed, 59 insertions(+), 140 deletions(-)

diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h
index c8a49edc5c78..aac0cf51680a 100644
--- a/lldb/include/lldb/Target/Process.h
+++ b/lldb/include/lldb/Target/Process.h
@@ -406,36 +406,6 @@ public:
                                   lldb::StateType state);
   } Notifications;
 
-  class ProcessMemoryIterator {
-  public:
-    ProcessMemoryIterator(lldb::ProcessSP process_sp, lldb::addr_t base)
-        : m_process_sp(process_sp), m_base_addr(base) {
-      lldbassert(process_sp.get() != nullptr);
-    }
-
-    bool IsValid() { return m_is_valid; }
-
-    uint8_t operator[](lldb::addr_t offset) {
-      if (!IsValid())
-        return 0;
-
-      uint8_t retval = 0;
-      Status error;
-      if (0 ==
-          m_process_sp->ReadMemory(m_base_addr + offset, &retval, 1, error)) {
-        m_is_valid = false;
-        return 0;
-      }
-
-      return retval;
-    }
-
-  private:
-    lldb::ProcessSP m_process_sp;
-    lldb::addr_t m_base_addr;
-    bool m_is_valid = true;
-  };
-
   class ProcessEventData : public EventData {
     friend class Process;
 
@@ -1679,26 +1649,6 @@ public:
 
   lldb::addr_t ReadPointerFromMemory(lldb::addr_t vm_addr, Status &error);
 
-  /// Find a string within a memory region.
-  ///
-  /// This function searches for the string represented by the provided buffer
-  /// within the memory range specified by the low and high addresses. It uses
-  /// a bad character heuristic to optimize the search process.
-  ///
-  /// \param[in] low The starting address of the memory region to be searched.
-  ///
-  /// \param[in] high The ending address of the memory region to be searched.
-  ///
-  /// \param[in] buffer A pointer to the buffer containing the string to be
-  /// searched.
-  ///
-  /// \param[in] buffer_size The size of the buffer in bytes.
-  ///
-  /// \return The address where the string was found or LLDB_INVALID_ADDRESS if
-  /// not found.
-  lldb::addr_t FindInMemory(lldb::addr_t low, lldb::addr_t high,
-                            uint8_t *buffer, size_t buffer_size);
-
   bool WritePointerToMemory(lldb::addr_t vm_addr, lldb::addr_t ptr_value,
                             Status &error);
 
diff --git a/lldb/source/Commands/CommandObjectMemory.cpp b/lldb/source/Commands/CommandObjectMemory.cpp
index 1c13484dede6..b78a0492cca5 100644
--- a/lldb/source/Commands/CommandObjectMemory.cpp
+++ b/lldb/source/Commands/CommandObjectMemory.cpp
@@ -977,6 +977,35 @@ public:
   Options *GetOptions() override { return &m_option_group; }
 
 protected:
+  class ProcessMemoryIterator {
+  public:
+    ProcessMemoryIterator(ProcessSP process_sp, lldb::addr_t base)
+        : m_process_sp(process_sp), m_base_addr(base) {
+      lldbassert(process_sp.get() != nullptr);
+    }
+
+    bool IsValid() { return m_is_valid; }
+
+    uint8_t operator[](lldb::addr_t offset) {
+      if (!IsValid())
+        return 0;
+
+      uint8_t retval = 0;
+      Status error;
+      if (0 ==
+          m_process_sp->ReadMemory(m_base_addr + offset, &retval, 1, error)) {
+        m_is_valid = false;
+        return 0;
+      }
+
+      return retval;
+    }
+
+  private:
+    ProcessSP m_process_sp;
+    lldb::addr_t m_base_addr;
+    bool m_is_valid = true;
+  };
   void DoExecute(Args &command, CommandReturnObject &result) override {
     // No need to check "process" for validity as eCommandRequiresProcess
     // ensures it is valid
@@ -1077,8 +1106,8 @@ protected:
     found_location = low_addr;
     bool ever_found = false;
     while (count) {
-      found_location = process->FindInMemory(
-          found_location, high_addr, buffer.GetBytes(), buffer.GetByteSize());
+      found_location = FastSearch(found_location, high_addr, buffer.GetBytes(),
+                                  buffer.GetByteSize());
       if (found_location == LLDB_INVALID_ADDRESS) {
         if (!ever_found) {
           result.AppendMessage("data not found within the range.\n");
@@ -1115,6 +1144,34 @@ protected:
     result.SetStatus(lldb::eReturnStatusSuccessFinishResult);
   }
 
+  lldb::addr_t FastSearch(lldb::addr_t low, lldb::addr_t high, uint8_t *buffer,
+                          size_t buffer_size) {
+    const size_t region_size = high - low;
+
+    if (region_size < buffer_size)
+      return LLDB_INVALID_ADDRESS;
+
+    std::vector<size_t> bad_char_heuristic(256, buffer_size);
+    ProcessSP process_sp = m_exe_ctx.GetProcessSP();
+    ProcessMemoryIterator iterator(process_sp, low);
+
+    for (size_t idx = 0; idx < buffer_size - 1; idx++) {
+      decltype(bad_char_heuristic)::size_type bcu_idx = buffer[idx];
+      bad_char_heuristic[bcu_idx] = buffer_size - idx - 1;
+    }
+    for (size_t s = 0; s <= (region_size - buffer_size);) {
+      int64_t j = buffer_size - 1;
+      while (j >= 0 && buffer[j] == iterator[s + j])
+        j--;
+      if (j < 0)
+        return low + s;
+      else
+        s += bad_char_heuristic[iterator[s + buffer_size - 1]];
+    }
+
+    return LLDB_INVALID_ADDRESS;
+  }
+
   OptionGroupOptions m_option_group;
   OptionGroupFindMemory m_memory_options;
   OptionGroupMemoryTag m_memory_tag_options;
diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp
index 4ff03eb8ab48..36812c27a5b6 100644
--- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp
+++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp
@@ -6,12 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <cstddef>
 #include <cstdlib>
 
 #include <memory>
 #include <mutex>
-#include <tuple>
 
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleSpec.h"
@@ -212,9 +210,6 @@ Status ProcessElfCore::DoLoadCore() {
     }
   }
 
-  // We need to update uuid after address range is populated.
-  UpdateBuildIdForNTFileEntries();
-
   if (!ranges_are_sorted) {
     m_core_aranges.Sort();
     m_core_range_infos.Sort();
@@ -263,7 +258,6 @@ Status ProcessElfCore::DoLoadCore() {
     if (!m_nt_file_entries.empty()) {
       ModuleSpec exe_module_spec;
       exe_module_spec.GetArchitecture() = arch;
-      exe_module_spec.GetUUID() = m_nt_file_entries[0].uuid;
       exe_module_spec.GetFileSpec().SetFile(m_nt_file_entries[0].path,
                                             FileSpec::Style::native);
       if (exe_module_spec.GetFileSpec()) {
@@ -277,16 +271,6 @@ Status ProcessElfCore::DoLoadCore() {
   return error;
 }
 
-void ProcessElfCore::UpdateBuildIdForNTFileEntries() {
-  if (!m_nt_file_entries.empty()) {
-    for (NT_FILE_Entry &entry : m_nt_file_entries) {
-      std::optional<UUID> uuid = FindBuildId(entry);
-      if (uuid)
-        entry.uuid = uuid.value();
-    }
-  }
-}
-
 lldb_private::DynamicLoader *ProcessElfCore::GetDynamicLoader() {
   if (m_dyld_up.get() == nullptr)
     m_dyld_up.reset(DynamicLoader::FindPlugin(
@@ -999,40 +983,6 @@ llvm::Error ProcessElfCore::ParseThreadContextsFromNoteSegment(
   }
 }
 
-bool ProcessElfCore::IsElf(const NT_FILE_Entry entry) {
-  size_t size = strlen(llvm::ELF::ElfMagic);
-  uint8_t buf[size];
-  Status error;
-  size_t byte_read = ReadMemory(entry.start, buf, size, error);
-  if (byte_read == size)
-    return memcmp(llvm::ELF::ElfMagic, buf, size) == 0;
-  else
-    return false;
-}
-
-std::optional<UUID> ProcessElfCore::FindBuildId(const NT_FILE_Entry entry) {
-  if (!IsElf(entry))
-    return std::nullopt;
-  // Build ID is stored in the ELF file as a section named ".note.gnu.build-id"
-  uint8_t gnu_build_id_bytes[8] = {0x03, 0x00, 0x00, 0x00,
-                                   0x47, 0x4e, 0x55, 0x00};
-  lldb::addr_t gnu_build_id_addr =
-      FindInMemory(entry.start, entry.end, gnu_build_id_bytes, 8);
-  if (gnu_build_id_addr == LLDB_INVALID_ADDRESS)
-    return std::nullopt;
-  uint8_t buf[36];
-  Status error;
-  size_t byte_read = ReadMemory(gnu_build_id_addr - 8, buf, 36, error);
-  // .note.gnu.build-id starts with 04 00 00 00 {id_byte_size} 00 00 00 03 00 00
-  // 00 47 4e 55 00
-  if (byte_read == 36) {
-    if (buf[0] == 0x04) {
-      return UUID(llvm::ArrayRef<uint8_t>(buf + 16, buf[4] /*byte size*/));
-    }
-  }
-  return std::nullopt;
-}
-
 uint32_t ProcessElfCore::GetNumThreadContexts() {
   if (!m_thread_data_valid)
     DoLoadCore();
diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h
index ae827f3df002..2cec635bbacf 100644
--- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h
+++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h
@@ -117,8 +117,6 @@ private:
     lldb::addr_t end;
     lldb::addr_t file_ofs;
     std::string path;
-    lldb_private::UUID
-        uuid; // extracted from .note.gnu.build-id section from core file
   };
 
   // For ProcessElfCore only
@@ -160,15 +158,6 @@ private:
   // Returns number of thread contexts stored in the core file
   uint32_t GetNumThreadContexts();
 
-  // Populate gnu uuid for each NT_FILE entry
-  void UpdateBuildIdForNTFileEntries();
-
-  // Returns the UUID of a given NT_FILE entry
-  std::optional<lldb_private::UUID> FindBuildId(const NT_FILE_Entry entry);
-
-  // Returns true if the given NT_FILE entry is an ELF file
-  bool IsElf(const NT_FILE_Entry entry);
-
   // Parse a contiguous address range of the process from LOAD segment
   lldb::addr_t
   AddAddressRangeFromLoadSegment(const elf::ELFProgramHeader &header);
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index 6f5c43bc4108..25afade9a827 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -3191,33 +3191,6 @@ Status Process::Halt(bool clear_thread_plans, bool use_run_lock) {
   return Status();
 }
 
-lldb::addr_t Process::FindInMemory(lldb::addr_t low, lldb::addr_t high,
-                                   uint8_t *buffer, size_t buffer_size) {
-  const size_t region_size = high - low;
-
-  if (region_size < buffer_size)
-    return LLDB_INVALID_ADDRESS;
-
-  std::vector<size_t> bad_char_heuristic(256, buffer_size);
-  ProcessMemoryIterator iterator(shared_from_this(), low);
-
-  for (size_t idx = 0; idx < buffer_size - 1; idx++) {
-    decltype(bad_char_heuristic)::size_type bcu_idx = buffer[idx];
-    bad_char_heuristic[bcu_idx] = buffer_size - idx - 1;
-  }
-  for (size_t s = 0; s <= (region_size - buffer_size);) {
-    int64_t j = buffer_size - 1;
-    while (j >= 0 && buffer[j] == iterator[s + j])
-      j--;
-    if (j < 0)
-      return low + s;
-    else
-      s += bad_char_heuristic[iterator[s + buffer_size - 1]];
-  }
-
-  return LLDB_INVALID_ADDRESS;
-}
-
 Status Process::StopForDestroyOrDetach(lldb::EventSP &exit_event_sp) {
   Status error;
 
-- 
GitLab


From 99fad7ebd85e37d3e25500b3411f6b332f68f108 Mon Sep 17 00:00:00 2001
From: Alexander Yermolovich <43973793+ayermolo@users.noreply.github.com>
Date: Tue, 14 May 2024 15:08:45 -0700
Subject: [PATCH 0786/1206] [BOLT][DWARF] Update DW_AT_comp_dir/DW_AT_dwo_name
 for DWO TUs (#91486)

Type unit DIE generated by clang contains DW_AT_comp_dir/DW_AT_dwo_name.
This was added to clang to help LLDB to figure out where type unit come
from when accessing an entry in a .debug_names accelerator table and
type units in .dwp file.

When BOLT writes out .dwo files it changes the name of them. User can
also specify directory of where they can be written out. Added support
to BOLT to update those attributes.
---
 bolt/include/bolt/Core/DIEBuilder.h           |  14 ++
 bolt/include/bolt/Core/DebugData.h            |  14 +-
 bolt/include/bolt/Rewrite/DWARFRewriter.h     |   8 +-
 bolt/lib/Core/DIEBuilder.cpp                  |  85 ++++++++
 bolt/lib/Core/DebugData.cpp                   |  20 +-
 bolt/lib/Rewrite/DWARFRewriter.cpp            | 159 ++++++--------
 .../Inputs/dwarf5-df-types-debug-names-main.s |  22 +-
 ...dwarf5-df-types-modify-dwo-name-mixed.test | 198 ++++++++++++++++++
 .../X86/dwarf5-df-types-modify-dwo-name.test  | 175 ++++++++++++++++
 9 files changed, 575 insertions(+), 120 deletions(-)
 create mode 100644 bolt/test/X86/dwarf5-df-types-modify-dwo-name-mixed.test
 create mode 100644 bolt/test/X86/dwarf5-df-types-modify-dwo-name.test

diff --git a/bolt/include/bolt/Core/DIEBuilder.h b/bolt/include/bolt/Core/DIEBuilder.h
index 06084819ec0b..c5ad0ac18339 100644
--- a/bolt/include/bolt/Core/DIEBuilder.h
+++ b/bolt/include/bolt/Core/DIEBuilder.h
@@ -129,6 +129,9 @@ private:
   uint64_t UnitSize{0};
   llvm::DenseSet<uint64_t> AllProcessed;
   DWARF5AcceleratorTable &DebugNamesTable;
+  // Unordered map to handle name collision if output DWO directory is
+  // specified.
+  std::unordered_map<std::string, uint32_t> NameToIndexMap;
 
   /// Returns current state of the DIEBuilder
   State &getState() { return *BuilderState.get(); }
@@ -384,6 +387,17 @@ public:
   bool deleteValue(DIEValueList *Die, dwarf::Attribute Attribute) {
     return Die->deleteValue(Attribute);
   }
+  /// Updates DWO Name and Compilation directory for Skeleton CU \p Unit.
+  std::string updateDWONameCompDir(DebugStrOffsetsWriter &StrOffstsWriter,
+                                   DebugStrWriter &StrWriter,
+                                   DWARFUnit &SkeletonCU,
+                                   std::optional<StringRef> DwarfOutputPath,
+                                   std::optional<StringRef> DWONameToUse);
+  /// Updates DWO Name and Compilation directory for Type Units.
+  void updateDWONameCompDirForTypes(DebugStrOffsetsWriter &StrOffstsWriter,
+                                    DebugStrWriter &StrWriter, DWARFUnit &Unit,
+                                    std::optional<StringRef> DwarfOutputPath,
+                                    const StringRef DWOName);
 };
 } // namespace bolt
 } // namespace llvm
diff --git a/bolt/include/bolt/Core/DebugData.h b/bolt/include/bolt/Core/DebugData.h
index 166bb3617e57..585bafa08884 100644
--- a/bolt/include/bolt/Core/DebugData.h
+++ b/bolt/include/bolt/Core/DebugData.h
@@ -430,7 +430,7 @@ protected:
 using DebugStrOffsetsBufferVector = SmallVector<char, 16>;
 class DebugStrOffsetsWriter {
 public:
-  DebugStrOffsetsWriter() {
+  DebugStrOffsetsWriter(BinaryContext &BC) : BC(BC) {
     StrOffsetsBuffer = std::make_unique<DebugStrOffsetsBufferVector>();
     StrOffsetsStream = std::make_unique<raw_svector_ostream>(*StrOffsetsBuffer);
   }
@@ -460,6 +460,10 @@ public:
     StrOffsets.clear();
   }
 
+  bool isStrOffsetsSectionModified() const {
+    return StrOffsetSectionWasModified;
+  }
+
 private:
   std::unique_ptr<DebugStrOffsetsBufferVector> StrOffsetsBuffer;
   std::unique_ptr<raw_svector_ostream> StrOffsetsStream;
@@ -467,13 +471,16 @@ private:
   SmallVector<uint32_t, 5> StrOffsets;
   std::unordered_map<uint64_t, uint64_t> ProcessedBaseOffsets;
   bool StrOffsetSectionWasModified = false;
+  BinaryContext &BC;
 };
 
 using DebugStrBufferVector = SmallVector<char, 16>;
 class DebugStrWriter {
 public:
   DebugStrWriter() = delete;
-  DebugStrWriter(BinaryContext &BC) : BC(BC) { create(); }
+  DebugStrWriter(DWARFContext &DwCtx, bool IsDWO) : DwCtx(DwCtx), IsDWO(IsDWO) {
+    create();
+  }
   std::unique_ptr<DebugStrBufferVector> releaseBuffer() {
     return std::move(StrBuffer);
   }
@@ -495,7 +502,8 @@ private:
   void create();
   std::unique_ptr<DebugStrBufferVector> StrBuffer;
   std::unique_ptr<raw_svector_ostream> StrStream;
-  BinaryContext &BC;
+  DWARFContext &DwCtx;
+  bool IsDWO;
 };
 
 enum class LocWriterKind { DebugLocWriter, DebugLoclistWriter };
diff --git a/bolt/include/bolt/Rewrite/DWARFRewriter.h b/bolt/include/bolt/Rewrite/DWARFRewriter.h
index 12e0813d089d..8dec32de9008 100644
--- a/bolt/include/bolt/Rewrite/DWARFRewriter.h
+++ b/bolt/include/bolt/Rewrite/DWARFRewriter.h
@@ -203,13 +203,16 @@ public:
   using OverriddenSectionsMap = std::unordered_map<DWARFSectionKind, StringRef>;
   /// Output .dwo files.
   void writeDWOFiles(DWARFUnit &, const OverriddenSectionsMap &,
-                     const std::string &, DebugLocWriter &);
+                     const std::string &, DebugLocWriter &,
+                     DebugStrOffsetsWriter &, DebugStrWriter &);
   using KnownSectionsEntry = std::pair<MCSection *, DWARFSectionKind>;
   struct DWPState {
     std::unique_ptr<ToolOutputFile> Out;
     std::unique_ptr<BinaryContext> TmpBC;
     std::unique_ptr<MCStreamer> Streamer;
     std::unique_ptr<DWPStringPool> Strings;
+    /// Used to store String sections for .dwo files if they are being modified.
+    std::vector<std::unique_ptr<DebugBufferVector>> StrSections;
     const MCObjectFileInfo *MCOFI = nullptr;
     const DWARFUnitIndex *CUIndex = nullptr;
     std::deque<SmallString<32>> UncompressedSections;
@@ -230,7 +233,8 @@ public:
 
   /// add content of dwo to .dwp file.
   void updateDWP(DWARFUnit &, const OverriddenSectionsMap &, const UnitMeta &,
-                 UnitMetaVectorType &, DWPState &, DebugLocWriter &);
+                 UnitMetaVectorType &, DWPState &, DebugLocWriter &,
+                 DebugStrOffsetsWriter &, DebugStrWriter &);
 };
 
 } // namespace bolt
diff --git a/bolt/lib/Core/DIEBuilder.cpp b/bolt/lib/Core/DIEBuilder.cpp
index c4b0b251c120..34c455a36cce 100644
--- a/bolt/lib/Core/DIEBuilder.cpp
+++ b/bolt/lib/Core/DIEBuilder.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/LEB128.h"
 
 #include <algorithm>
@@ -41,6 +42,90 @@ extern cl::opt<unsigned> Verbosity;
 namespace llvm {
 namespace bolt {
 
+/// Returns DWO Name to be used to update DW_AT_dwo_name/DW_AT_GNU_dwo_name
+/// either in CU or TU unit die. Handles case where user specifies output DWO
+/// directory, and there are duplicate names. Assumes DWO ID is unique.
+static std::string
+getDWOName(llvm::DWARFUnit &CU,
+           std::unordered_map<std::string, uint32_t> &NameToIndexMap,
+           std::optional<StringRef> &DwarfOutputPath) {
+  assert(CU.getDWOId() && "DWO ID not found.");
+  std::string DWOName = dwarf::toString(
+      CU.getUnitDIE().find({dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}),
+      "");
+  assert(!DWOName.empty() &&
+         "DW_AT_dwo_name/DW_AT_GNU_dwo_name does not exist.");
+  if (DwarfOutputPath) {
+    DWOName = std::string(sys::path::filename(DWOName));
+    auto Iter = NameToIndexMap.find(DWOName);
+    if (Iter == NameToIndexMap.end())
+      Iter = NameToIndexMap.insert({DWOName, 0}).first;
+    DWOName.append(std::to_string(Iter->second));
+    ++Iter->second;
+  }
+  DWOName.append(".dwo");
+  return DWOName;
+}
+
+/// Adds a \p Str to .debug_str section.
+/// Uses \p AttrInfoVal to either update entry in a DIE for legacy DWARF using
+/// \p DebugInfoPatcher, or for DWARF5 update an index in .debug_str_offsets
+/// for this contribution of \p Unit.
+static void addStringHelper(DebugStrOffsetsWriter &StrOffstsWriter,
+                            DebugStrWriter &StrWriter, DIEBuilder &DIEBldr,
+                            DIE &Die, const DWARFUnit &Unit,
+                            DIEValue &DIEAttrInfo, StringRef Str) {
+  uint32_t NewOffset = StrWriter.addString(Str);
+  if (Unit.getVersion() >= 5) {
+    StrOffstsWriter.updateAddressMap(DIEAttrInfo.getDIEInteger().getValue(),
+                                     NewOffset);
+    return;
+  }
+  DIEBldr.replaceValue(&Die, DIEAttrInfo.getAttribute(), DIEAttrInfo.getForm(),
+                       DIEInteger(NewOffset));
+}
+
+std::string DIEBuilder::updateDWONameCompDir(
+    DebugStrOffsetsWriter &StrOffstsWriter, DebugStrWriter &StrWriter,
+    DWARFUnit &SkeletonCU, std::optional<StringRef> DwarfOutputPath,
+    std::optional<StringRef> DWONameToUse) {
+  DIE &UnitDIE = *getUnitDIEbyUnit(SkeletonCU);
+  DIEValue DWONameAttrInfo = UnitDIE.findAttribute(dwarf::DW_AT_dwo_name);
+  if (!DWONameAttrInfo)
+    DWONameAttrInfo = UnitDIE.findAttribute(dwarf::DW_AT_GNU_dwo_name);
+  if (!DWONameAttrInfo)
+    return "";
+  std::string ObjectName;
+  if (DWONameToUse)
+    ObjectName = *DWONameToUse;
+  else
+    ObjectName = getDWOName(SkeletonCU, NameToIndexMap, DwarfOutputPath);
+  addStringHelper(StrOffstsWriter, StrWriter, *this, UnitDIE, SkeletonCU,
+                  DWONameAttrInfo, ObjectName);
+
+  DIEValue CompDirAttrInfo = UnitDIE.findAttribute(dwarf::DW_AT_comp_dir);
+  assert(CompDirAttrInfo && "DW_AT_comp_dir is not in Skeleton CU.");
+
+  if (DwarfOutputPath) {
+    if (!sys::fs::exists(*DwarfOutputPath))
+      sys::fs::create_directory(*DwarfOutputPath);
+    addStringHelper(StrOffstsWriter, StrWriter, *this, UnitDIE, SkeletonCU,
+                    CompDirAttrInfo, *DwarfOutputPath);
+  }
+  return ObjectName;
+}
+
+void DIEBuilder::updateDWONameCompDirForTypes(
+    DebugStrOffsetsWriter &StrOffstsWriter, DebugStrWriter &StrWriter,
+    DWARFUnit &Unit, std::optional<StringRef> DwarfOutputPath,
+    const StringRef DWOName) {
+  for (DWARFUnit *DU : getState().DWARF5TUVector)
+    updateDWONameCompDir(StrOffstsWriter, StrWriter, *DU, DwarfOutputPath,
+                         DWOName);
+  if (StrOffstsWriter.isStrOffsetsSectionModified())
+    StrOffstsWriter.finalizeSection(Unit, *this);
+}
+
 void DIEBuilder::updateReferences() {
   for (auto &[SrcDIEInfo, ReferenceInfo] : getState().AddrReferences) {
     DIEInfo *DstDIEInfo = ReferenceInfo.Dst;
diff --git a/bolt/lib/Core/DebugData.cpp b/bolt/lib/Core/DebugData.cpp
index a987a103a08b..f502a5031247 100644
--- a/bolt/lib/Core/DebugData.cpp
+++ b/bolt/lib/Core/DebugData.cpp
@@ -31,6 +31,7 @@
 #include <cstdint>
 #include <functional>
 #include <memory>
+#include <optional>
 #include <unordered_map>
 #include <vector>
 
@@ -867,10 +868,17 @@ void DebugStrOffsetsWriter::finalizeSection(DWARFUnit &Unit,
                                             DIEBuilder &DIEBldr) {
   std::optional<AttrInfo> AttrVal =
       findAttributeInfo(Unit.getUnitDIE(), dwarf::DW_AT_str_offsets_base);
-  if (!AttrVal)
+  if (!AttrVal && !Unit.isDWOUnit())
     return;
-  std::optional<uint64_t> Val = AttrVal->V.getAsSectionOffset();
-  assert(Val && "DW_AT_str_offsets_base Value not present.");
+  std::optional<uint64_t> Val = std::nullopt;
+  if (AttrVal) {
+    Val = AttrVal->V.getAsSectionOffset();
+  } else {
+    if (!Unit.isDWOUnit())
+      BC.errs() << "BOLT-WARNING: [internal-dwarf-error]: "
+                   "DW_AT_str_offsets_base Value not present\n";
+    Val = 0;
+  }
   DIE &Die = *DIEBldr.getUnitDIEbyUnit(Unit);
   DIEValue StrListBaseAttrInfo =
       Die.findAttribute(dwarf::DW_AT_str_offsets_base);
@@ -915,7 +923,11 @@ void DebugStrWriter::create() {
 }
 
 void DebugStrWriter::initialize() {
-  auto StrSection = BC.DwCtx->getDWARFObj().getStrSection();
+  StringRef StrSection;
+  if (IsDWO)
+    StrSection = DwCtx.getDWARFObj().getStrDWOSection();
+  else
+    StrSection = DwCtx.getDWARFObj().getStrSection();
   (*StrStream) << StrSection;
 }
 
diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp
index 9d4297f913f3..d582ce7b33a2 100644
--- a/bolt/lib/Rewrite/DWARFRewriter.cpp
+++ b/bolt/lib/Rewrite/DWARFRewriter.cpp
@@ -458,32 +458,6 @@ static std::optional<uint64_t> getAsAddress(const DWARFUnit &DU,
   return std::nullopt;
 }
 
-/// Returns DWO Name to be used. Handles case where user specifies output DWO
-/// directory, and there are duplicate names. Assumes DWO ID is unique.
-static std::string
-getDWOName(llvm::DWARFUnit &CU,
-           std::unordered_map<std::string, uint32_t> &NameToIndexMap) {
-  std::optional<uint64_t> DWOId = CU.getDWOId();
-  assert(DWOId && "DWO ID not found.");
-  (void)DWOId;
-
-  std::string DWOName = dwarf::toString(
-      CU.getUnitDIE().find({dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}),
-      "");
-  assert(!DWOName.empty() &&
-         "DW_AT_dwo_name/DW_AT_GNU_dwo_name does not exists.");
-  if (!opts::DwarfOutputPath.empty()) {
-    DWOName = std::string(sys::path::filename(DWOName));
-    auto Iter = NameToIndexMap.find(DWOName);
-    if (Iter == NameToIndexMap.end())
-      Iter = NameToIndexMap.insert({DWOName, 0}).first;
-    DWOName.append(std::to_string(Iter->second));
-    ++Iter->second;
-  }
-  DWOName.append(".dwo");
-  return DWOName;
-}
-
 static std::unique_ptr<DIEStreamer>
 createDIEStreamer(const Triple &TheTriple, raw_pwrite_stream &OutFile,
                   StringRef Swift5ReflectionSegmentName, DIEBuilder &DIEBldr,
@@ -515,7 +489,9 @@ static void emitDWOBuilder(const std::string &DWOName,
                            DIEBuilder &DWODIEBuilder, DWARFRewriter &Rewriter,
                            DWARFUnit &SplitCU, DWARFUnit &CU,
                            DWARFRewriter::DWPState &State,
-                           DebugLocWriter &LocWriter) {
+                           DebugLocWriter &LocWriter,
+                           DebugStrOffsetsWriter &StrOffstsWriter,
+                           DebugStrWriter &StrWriter) {
   // Populate debug_info and debug_abbrev for current dwo into StringRef.
   DWODIEBuilder.generateAbbrevs();
   DWODIEBuilder.finish();
@@ -577,54 +553,10 @@ static void emitDWOBuilder(const std::string &DWOName,
   }
   if (opts::WriteDWP)
     Rewriter.updateDWP(CU, OverriddenSections, CUMI, TUMetaVector, State,
-                       LocWriter);
+                       LocWriter, StrOffstsWriter, StrWriter);
   else
-    Rewriter.writeDWOFiles(CU, OverriddenSections, DWOName, LocWriter);
-}
-
-/// Adds a \p Str to .debug_str section.
-/// Uses \p AttrInfoVal to either update entry in a DIE for legacy DWARF using
-/// \p DebugInfoPatcher, or for DWARF5 update an index in .debug_str_offsets
-/// for this contribution of \p Unit.
-static void addStringHelper(DebugStrOffsetsWriter &StrOffstsWriter,
-                            DebugStrWriter &StrWriter, DIEBuilder &DIEBldr,
-                            DIE &Die, const DWARFUnit &Unit,
-                            DIEValue &DIEAttrInfo, StringRef Str) {
-  uint32_t NewOffset = StrWriter.addString(Str);
-  if (Unit.getVersion() >= 5) {
-    StrOffstsWriter.updateAddressMap(DIEAttrInfo.getDIEInteger().getValue(),
-                                     NewOffset);
-    return;
-  }
-  DIEBldr.replaceValue(&Die, DIEAttrInfo.getAttribute(), DIEAttrInfo.getForm(),
-                       DIEInteger(NewOffset));
-}
-
-static std::string
-updateDWONameCompDir(DebugStrOffsetsWriter &StrOffstsWriter,
-                     DebugStrWriter &StrWriter,
-                     std::unordered_map<std::string, uint32_t> &NameToIndexMap,
-                     DWARFUnit &Unit, DIEBuilder &DIEBldr, DIE &UnitDIE) {
-  DIEValue DWONameAttrInfo = UnitDIE.findAttribute(dwarf::DW_AT_dwo_name);
-  if (!DWONameAttrInfo)
-    DWONameAttrInfo = UnitDIE.findAttribute(dwarf::DW_AT_GNU_dwo_name);
-  assert(DWONameAttrInfo && "DW_AT_dwo_name is not in Skeleton CU.");
-  std::string ObjectName;
-
-  ObjectName = getDWOName(Unit, NameToIndexMap);
-  addStringHelper(StrOffstsWriter, StrWriter, DIEBldr, UnitDIE, Unit,
-                  DWONameAttrInfo, ObjectName.c_str());
-
-  DIEValue CompDirAttrInfo = UnitDIE.findAttribute(dwarf::DW_AT_comp_dir);
-  assert(CompDirAttrInfo && "DW_AT_comp_dir is not in Skeleton CU.");
-
-  if (!opts::DwarfOutputPath.empty()) {
-    if (!sys::fs::exists(opts::DwarfOutputPath))
-      sys::fs::create_directory(opts::DwarfOutputPath);
-    addStringHelper(StrOffstsWriter, StrWriter, DIEBldr, UnitDIE, Unit,
-                    CompDirAttrInfo, opts::DwarfOutputPath.c_str());
-  }
-  return ObjectName;
+    Rewriter.writeDWOFiles(CU, OverriddenSections, DWOName, LocWriter,
+                           StrOffstsWriter, StrWriter);
 }
 
 using DWARFUnitVec = std::vector<DWARFUnit *>;
@@ -673,9 +605,8 @@ void DWARFRewriter::updateDebugInfo() {
     return;
 
   ARangesSectionWriter = std::make_unique<DebugARangesSectionWriter>();
-  StrWriter = std::make_unique<DebugStrWriter>(BC);
-
-  StrOffstsWriter = std::make_unique<DebugStrOffsetsWriter>();
+  StrWriter = std::make_unique<DebugStrWriter>(*BC.DwCtx, false);
+  StrOffstsWriter = std::make_unique<DebugStrOffsetsWriter>(BC);
 
   if (!opts::DeterministicDebugInfo) {
     opts::DeterministicDebugInfo = true;
@@ -720,10 +651,6 @@ void DWARFRewriter::updateDebugInfo() {
     return LocListWritersByCU[CUIndex++].get();
   };
 
-  // Unordered maps to handle name collision if output DWO directory is
-  // specified.
-  std::unordered_map<std::string, uint32_t> NameToIndexMap;
-
   DWARF5AcceleratorTable DebugNamesTable(opts::CreateDebugNames, BC,
                                          *StrWriter);
   DWPState State;
@@ -747,13 +674,20 @@ void DWARFRewriter::updateDebugInfo() {
                                Unit);
       DWODIEBuilder.buildDWOUnit(**SplitCU);
       std::string DWOName = "";
+      std::optional<std::string> DwarfOutputPath =
+          opts::DwarfOutputPath.empty()
+              ? std::nullopt
+              : std::optional<std::string>(opts::DwarfOutputPath.c_str());
       {
         std::lock_guard<std::mutex> Lock(AccessMutex);
-        DWOName = updateDWONameCompDir(*StrOffstsWriter, *StrWriter,
-                                       NameToIndexMap, *Unit, *DIEBlder,
-                                       *DIEBlder->getUnitDIEbyUnit(*Unit));
+        DWOName = DIEBlder->updateDWONameCompDir(
+            *StrOffstsWriter, *StrWriter, *Unit, DwarfOutputPath, std::nullopt);
       }
-
+      DebugStrOffsetsWriter DWOStrOffstsWriter(BC);
+      DebugStrWriter DWOStrWriter((*SplitCU)->getContext(), true);
+      DWODIEBuilder.updateDWONameCompDirForTypes(DWOStrOffstsWriter,
+                                                 DWOStrWriter, **SplitCU,
+                                                 DwarfOutputPath, DWOName);
       DebugLoclistWriter DebugLocDWoWriter(*Unit, Unit->getVersion(), true);
       DebugRangesSectionWriter *TempRangesSectionWriter = RangesSectionWriter;
       if (Unit->getVersion() >= 5) {
@@ -771,7 +705,7 @@ void DWARFRewriter::updateDebugInfo() {
         TempRangesSectionWriter->finalizeSection();
 
       emitDWOBuilder(DWOName, DWODIEBuilder, *this, **SplitCU, *Unit, State,
-                     DebugLocDWoWriter);
+                     DebugLocDWoWriter, DWOStrOffstsWriter, DWOStrWriter);
     }
 
     if (Unit->getVersion() >= 5) {
@@ -1736,6 +1670,7 @@ std::optional<StringRef> updateDebugData(
     const DWARFUnitIndex::Entry *CUDWOEntry, uint64_t DWOId,
     std::unique_ptr<DebugBufferVector> &OutputBuffer,
     DebugRangeListsSectionWriter *RangeListsWriter, DebugLocWriter &LocWriter,
+    DebugStrOffsetsWriter &StrOffstsWriter, DebugStrWriter &StrWriter,
     const llvm::bolt::DWARFRewriter::OverriddenSectionsMap &OverridenSections) {
 
   using DWOSectionContribution =
@@ -1774,6 +1709,11 @@ std::optional<StringRef> updateDebugData(
     if (SectionName != "debug_str.dwo")
       errs() << "BOLT-WARNING: unsupported debug section: " << SectionName
              << "\n";
+    if (StrWriter.isInitialized()) {
+      OutputBuffer = StrWriter.releaseBuffer();
+      return StringRef(reinterpret_cast<const char *>(OutputBuffer->data()),
+                       OutputBuffer->size());
+    }
     return SectionContents;
   }
   case DWARFSectionKind::DW_SECT_INFO: {
@@ -1783,6 +1723,11 @@ std::optional<StringRef> updateDebugData(
     return getOverridenSection(DWARFSectionKind::DW_SECT_EXT_TYPES);
   }
   case DWARFSectionKind::DW_SECT_STR_OFFSETS: {
+    if (StrOffstsWriter.isFinalized()) {
+      OutputBuffer = StrOffstsWriter.releaseBuffer();
+      return StringRef(reinterpret_cast<const char *>(OutputBuffer->data()),
+                       OutputBuffer->size());
+    }
     return getSliceData(CUDWOEntry, SectionContents,
                         DWARFSectionKind::DW_SECT_STR_OFFSETS, DWPOffset);
   }
@@ -1884,7 +1829,9 @@ void DWARFRewriter::updateDWP(DWARFUnit &CU,
                               const OverriddenSectionsMap &OverridenSections,
                               const DWARFRewriter::UnitMeta &CUMI,
                               DWARFRewriter::UnitMetaVectorType &TUMetaVector,
-                              DWPState &State, DebugLocWriter &LocWriter) {
+                              DWPState &State, DebugLocWriter &LocWriter,
+                              DebugStrOffsetsWriter &StrOffstsWriter,
+                              DebugStrWriter &StrWriter) {
   const uint64_t DWOId = *CU.getDWOId();
   MCSection *const StrOffsetSection = State.MCOFI->getDwarfStrOffDWOSection();
   assert(StrOffsetSection && "StrOffsetSection does not exist.");
@@ -1941,15 +1888,18 @@ void DWARFRewriter::updateDWP(DWARFUnit &CU,
         TUEntry.Contributions[Index].getLength32();
     State.TypeIndexEntries.insert(std::make_pair(Hash, TUEntry));
   };
+  std::unique_ptr<DebugBufferVector> StrOffsetsOutputData;
+  std::unique_ptr<DebugBufferVector> StrOutputData;
   for (const SectionRef &Section : DWOFile->sections()) {
-    std::unique_ptr<DebugBufferVector> OutputData;
+    std::unique_ptr<DebugBufferVector> OutputData = nullptr;
     StringRef SectionName = getSectionName(Section);
     Expected<StringRef> ContentsExp = Section.getContents();
     assert(ContentsExp && "Invalid contents.");
-    std::optional<StringRef> TOutData = updateDebugData(
-        (*DWOCU)->getContext(), SectionName, *ContentsExp, State.KnownSections,
-        *State.Streamer, *this, CUDWOEntry, DWOId, OutputData,
-        RangeListssWriter, LocWriter, OverridenSections);
+    std::optional<StringRef> TOutData =
+        updateDebugData((*DWOCU)->getContext(), SectionName, *ContentsExp,
+                        State.KnownSections, *State.Streamer, *this, CUDWOEntry,
+                        DWOId, OutputData, RangeListssWriter, LocWriter,
+                        StrOffstsWriter, StrWriter, OverridenSections);
     if (!TOutData)
       continue;
 
@@ -1961,14 +1911,17 @@ void DWARFRewriter::updateDWP(DWARFUnit &CU,
 
     if (SectionName == "debug_str.dwo") {
       CurStrSection = OutData;
+      StrOutputData = std::move(OutputData);
     } else {
       // Since handleDebugDataPatching returned true, we already know this is
       // a known section.
       auto SectionIter = State.KnownSections.find(SectionName);
-      if (SectionIter->second.second == DWARFSectionKind::DW_SECT_STR_OFFSETS)
+      if (SectionIter->second.second == DWARFSectionKind::DW_SECT_STR_OFFSETS) {
         CurStrOffsetSection = OutData;
-      else
+        StrOffsetsOutputData = std::move(OutputData);
+      } else {
         State.Streamer->emitBytes(OutData);
+      }
       unsigned int Index =
           getContributionIndex(SectionIter->second.second, State.IndexVersion);
       uint64_t Offset = State.ContributionOffsets[Index];
@@ -1992,6 +1945,10 @@ void DWARFRewriter::updateDWP(DWARFUnit &CU,
     // based on hash.
     if (!StrSectionWrittenOut && !CurStrOffsetSection.empty() &&
         !CurStrSection.empty()) {
+      // If debug_str.dwo section was modified storing it until dwp is written
+      // out. DWPStringPool stores raw pointers to strings.
+      if (StrOutputData)
+        State.StrSections.push_back(std::move(StrOutputData));
       writeStringsAndOffsets(*State.Streamer.get(), *State.Strings.get(),
                              StrOffsetSection, CurStrSection,
                              CurStrOffsetSection, CU.getVersion());
@@ -2017,7 +1974,8 @@ void DWARFRewriter::updateDWP(DWARFUnit &CU,
 
 void DWARFRewriter::writeDWOFiles(
     DWARFUnit &CU, const OverriddenSectionsMap &OverridenSections,
-    const std::string &DWOName, DebugLocWriter &LocWriter) {
+    const std::string &DWOName, DebugLocWriter &LocWriter,
+    DebugStrOffsetsWriter &StrOffstsWriter, DebugStrWriter &StrWriter) {
   // Setup DWP code once.
   DWARFContext *DWOCtx = BC.getDWOContext();
   const uint64_t DWOId = *CU.getDWOId();
@@ -2072,10 +2030,11 @@ void DWARFRewriter::writeDWOFiles(
     // have .debug_rnglists so won't be part of the loop below.
     if (!RangeListssWriter->empty()) {
       std::unique_ptr<DebugBufferVector> OutputData;
-      if (std::optional<StringRef> OutData = updateDebugData(
-              (*DWOCU)->getContext(), "debug_rnglists.dwo", "", KnownSections,
-              *Streamer, *this, CUDWOEntry, DWOId, OutputData,
-              RangeListssWriter, LocWriter, OverridenSections))
+      if (std::optional<StringRef> OutData =
+              updateDebugData((*DWOCU)->getContext(), "debug_rnglists.dwo", "",
+                              KnownSections, *Streamer, *this, CUDWOEntry,
+                              DWOId, OutputData, RangeListssWriter, LocWriter,
+                              StrOffstsWriter, StrWriter, OverridenSections))
         Streamer->emitBytes(*OutData);
     }
   }
@@ -2090,7 +2049,7 @@ void DWARFRewriter::writeDWOFiles(
     if (std::optional<StringRef> OutData = updateDebugData(
             (*DWOCU)->getContext(), SectionName, *ContentsExp, KnownSections,
             *Streamer, *this, CUDWOEntry, DWOId, OutputData, RangeListssWriter,
-            LocWriter, OverridenSections))
+            LocWriter, StrOffstsWriter, StrWriter, OverridenSections))
       Streamer->emitBytes(*OutData);
   }
   Streamer->finish();
diff --git a/bolt/test/X86/Inputs/dwarf5-df-types-debug-names-main.s b/bolt/test/X86/Inputs/dwarf5-df-types-debug-names-main.s
index f89f28ec13f4..34ba21f69517 100644
--- a/bolt/test/X86/Inputs/dwarf5-df-types-debug-names-main.s
+++ b/bolt/test/X86/Inputs/dwarf5-df-types-debug-names-main.s
@@ -207,7 +207,7 @@ main:                                   # @main
 .Linfo_string5:
 	.asciz	"f2"                            # string offset=24
 .Linfo_string6:
-	.asciz	"/home/ayermolo/local/tasks/T138552329/typeDedupSplit" # string offset=27
+	.asciz	"." # string offset=27
 .Linfo_string7:
 	.asciz	"main.dwo"                      # string offset=80
 .Linfo_string8:
@@ -234,15 +234,15 @@ main:                                   # @main
 	.long	19
 	.long	24
 	.long	27
-	.long	80
-	.long	89
-	.long	92
-	.long	97
-	.long	100
-	.long	103
-	.long	106
-	.long	112
-	.long	220
+	.long	29
+	.long	38
+	.long	41
+	.long	46
+	.long	49
+	.long	52
+	.long	55
+	.long	61
+	.long	169
 	.section	.debug_info.dwo,"e",@progbits
 	.long	.Ldebug_info_dwo_end2-.Ldebug_info_dwo_start2 # Length of Unit
 .Ldebug_info_dwo_start2:
@@ -474,7 +474,7 @@ main:                                   # @main
 	.byte	1
 	.byte	8
 	.byte	2
-	.ascii	"/home/ayermolo/local/tasks/T138552329/typeDedupSplit"
+	.ascii	"."
 	.byte	0
 	.byte	46
 	.byte	0
diff --git a/bolt/test/X86/dwarf5-df-types-modify-dwo-name-mixed.test b/bolt/test/X86/dwarf5-df-types-modify-dwo-name-mixed.test
new file mode 100644
index 000000000000..a4f5ee77ab56
--- /dev/null
+++ b/bolt/test/X86/dwarf5-df-types-modify-dwo-name-mixed.test
@@ -0,0 +1,198 @@
+; RUN: rm -rf %t
+; RUN: mkdir %t
+; RUN: cd %t
+; RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %p/Inputs/dwarf5-df-types-debug-names-main.s \
+; RUN: -split-dwarf-file=main.dwo -o main.o
+; RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %p/Inputs/dwarf5-df-types-dup-helper.s \
+; RUN: -split-dwarf-file=helper.dwo -o helper.o
+; RUN: %clang %cflags -gdwarf-5 -gsplit-dwarf=split main.o helper.o -o main.exe
+; RUN: llvm-bolt main.exe -o main.exe.bolt --update-debug-sections
+; RUN: llvm-dwarfdump --debug-info -r 0 main.exe.bolt > log.txt
+; RUN: llvm-dwarfdump --debug-info -r 0 main.dwo.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-info -r 0 helper.dwo.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-str-offsets main.dwo.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-str-offsets helper.dwo.dwo >> log.txt
+; RUN: cat log.txt | FileCheck -check-prefix=BOLT %s
+
+;; Test is a mix of DWARF5 TUs where one has DW_AT_comp_dir/DW_AT_dwo_name, and another one doesn't.
+;; Tests that BOLT correctly updates DW_AT_dwo_name for TUs.
+
+; BOLT: DW_TAG_skeleton_unit
+; BOLT: DW_AT_comp_dir  (".")
+; BOLT: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT: DW_TAG_skeleton_unit
+; BOLT: DW_AT_comp_dir  (".")
+; BOLT: DW_AT_dwo_name  ("helper.dwo.dwo")
+; BOLT: DW_TAG_type_unit
+; BOLT: DW_AT_comp_dir  (".")
+; BOLT: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT: DW_TAG_type_unit
+; BOLT: DW_AT_comp_dir  (".")
+; BOLT: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT: DW_TAG_type_unit
+; BOLT-NOT: DW_AT_dwo_name
+; BOLT: DW_TAG_type_unit
+; BOLT-NOT: DW_AT_dwo_name
+; BOLT: DW_TAG_compile_unit
+; BOLT:      .debug_str_offsets.dwo contents:
+; BOLT-NEXT: 0x00000000: Contribution size = 68, Format = DWARF32, Version = 5
+; BOLT-NEXT: "main"
+; BOLT-NEXT: "int"
+; BOLT-NEXT: "argc"
+; BOLT-NEXT: "argv"
+; BOLT-NEXT: "char"
+; BOLT-NEXT: "f2"
+; BOLT-NEXT: "."
+; BOLT-NEXT: "main.dwo.dwo"
+; BOLT-NEXT: "c1"
+; BOLT-NEXT: "Foo2"
+; BOLT-NEXT: "f3"
+; BOLT-NEXT: "c2"
+; BOLT-NEXT: "c3"
+; BOLT-NEXT: "Foo2a"
+; BOLT-NEXT: "clang version 18.0.0git (git@github.com:ayermolo/llvm-project.git db35fa8fc524127079662802c4735dbf397f86d0)"
+; BOLT-NEXT: "main.cpp"
+; BOLT-NEXT: helper.dwo.dwo: file format elf64-x86-64
+
+; BOLT:      .debug_str_offsets.dwo contents:
+; BOLT-NEXT: 0x00000000: Contribution size = 64, Format = DWARF32, Version = 5
+; BOLT-NEXT: "fooint"
+; BOLT-NEXT: "int"
+; BOLT-NEXT: "_Z3foov"
+; BOLT-NEXT: "foo"
+; BOLT-NEXT: "fint"
+; BOLT-NEXT: "c1"
+; BOLT-NEXT: "c2"
+; BOLT-NEXT: "Foo2Int"
+; BOLT-NEXT: "f"
+; BOLT-NEXT: "char"
+; BOLT-NEXT: "c3"
+; BOLT-NEXT: "Foo2a"
+; BOLT-NEXT: "clang version 18.0.0"
+; BOLT-NEXT: "helper.cpp"
+; BOLT-NEXT: "helper.dwo"
+
+
+;; Tests that BOLT correctly handles updating DW_AT_dwo_name when it outputs a DWP file.
+;; Currently skipping one of Type units because it is not being de-dupped.
+;; In the tu-index this TU is not present.
+; RUN: rm main.exe.bolt
+; RUN: llvm-bolt main.exe -o main.exe.bolt --update-debug-sections --write-dwp
+; RUN: llvm-dwarfdump --debug-info -r 0 main.exe.bolt.dwp > logDWP.txt
+; RUN: llvm-dwarfdump --debug-str-offsets main.exe.bolt.dwp >> logDWP.txt
+; RUN: cat logDWP.txt | FileCheck -check-prefix=BOLT-DWP %s
+; BOLT-DWP: DW_TAG_type_unit
+; BOLT-DWP: DW_AT_comp_dir  (".")
+; BOLT-DWP: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT-DWP: DW_TAG_type_unit
+; BOLT-DWP: DW_AT_comp_dir  (".")
+; BOLT-DWP: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT-DWP: DW_TAG_compile_unit
+; BOLT-DWP: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT-DWP: DW_TAG_type_unit
+; BOLT-DW-NOT: DW_AT_dwo_name
+; BOLT-DWP:       Contribution size = 68, Format = DWARF32, Version = 5
+; BOLT-DWP-NEXT: "main"
+; BOLT-DWP-NEXT: "int"
+; BOLT-DWP-NEXT: "argc"
+; BOLT-DWP-NEXT: "argv"
+; BOLT-DWP-NEXT: "char"
+; BOLT-DWP-NEXT: "f2"
+; BOLT-DWP-NEXT: "."
+; BOLT-DWP-NEXT: "main.dwo.dwo"
+; BOLT-DWP-NEXT: "c1"
+; BOLT-DWP-NEXT: "Foo2"
+; BOLT-DWP-NEXT: "f3"
+; BOLT-DWP-NEXT: "c2"
+; BOLT-DWP-NEXT: "c3"
+; BOLT-DWP-NEXT: "Foo2a"
+; BOLT-DWP-NEXT: "clang version 18.0.0git (git@github.com:ayermolo/llvm-project.git db35fa8fc524127079662802c4735dbf397f86d0)"
+; BOLT-DWP-NEXT: "main.cpp"
+; BOLT-DWP-NEXT: Contribution size = 64, Format = DWARF32, Version = 5
+; BOLT-DWP-NEXT: "fooint"
+; BOLT-DWP-NEXT: "int"
+; BOLT-DWP-NEXT: "_Z3foov"
+; BOLT-DWP-NEXT: "foo"
+; BOLT-DWP-NEXT: "fint"
+; BOLT-DWP-NEXT: "c1"
+; BOLT-DWP-NEXT: "c2"
+; BOLT-DWP-NEXT: "Foo2Int"
+; BOLT-DWP-NEXT: "f"
+; BOLT-DWP-NEXT: "char"
+; BOLT-DWP-NEXT: "c3"
+; BOLT-DWP-NEXT: "Foo2a"
+; BOLT-DWP-NEXT: "clang version 18.0.0"
+; BOLT-DWP-NEXT: "helper.cpp"
+; BOLT-DWP-NEXT: "helper.dwo
+
+;; Tests that BOLT correctly handles updating DW_AT_comp_dir/DW_AT_dwo_name when outptut directory is specified.
+
+; RUN: mkdir DWOOut
+; RUN: rm main.exe.bolt
+; RUN: llvm-bolt main.exe -o main.exe.bolt --update-debug-sections --dwarf-output-path=%t/DWOOut
+; RUN: cd DWOOut
+; RUN: llvm-dwarfdump --debug-info -r 0 ../main.exe.bolt > log.txt
+; RUN: llvm-dwarfdump --debug-info -r 0 main.dwo0.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-info -r 0 helper.dwo0.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-str-offsets main.dwo0.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-str-offsets helper.dwo0.dwo >> log.txt
+; RUN: cat log.txt | FileCheck -check-prefix=BOLT-PATH %s
+
+; BOLT-PATH: DW_TAG_skeleton_unit
+; BOLT-PATH: DW_AT_comp_dir  ("
+; BOLT-PATH-SAME: dwarf5-df-types-modify-dwo-name-mixed.test.tmp/DWOOut
+; BOLT-PATH: DW_AT_dwo_name  ("main.dwo0.dwo")
+; BOLT-PATH: DW_TAG_skeleton_unit
+; BOLT-PATH: DW_AT_comp_dir  ("
+; BOLT-PATH-SAME: dwarf5-df-types-modify-dwo-name-mixed.test.tmp/DWOOut
+; BOLT-PATH: DW_AT_dwo_name  ("helper.dwo0.dwo")
+; BOLT-PATH: DW_TAG_type_unit
+; BOLT-PATH: DW_AT_comp_dir  ("
+; BOLT-PATH-SAME: dwarf5-df-types-modify-dwo-name-mixed.test.tmp/DWOOut
+; BOLT-PATH: DW_AT_dwo_name  ("main.dwo0.dwo")
+; BOLT-PATH: DW_TAG_type_unit
+; BOLT-PATH: DW_AT_comp_dir  ("
+; BOLT-PATH-SAME: dwarf5-df-types-modify-dwo-name-mixed.test.tmp/DWOOut
+; BOLT-PATH: DW_AT_dwo_name  ("main.dwo0.dwo")
+; BOLT-PATH: DW_TAG_type_unit
+; BOLT-PATH-NOT: DW_AT_comp_dir
+; BOLT-PATH: DW_TAG_type_unit
+; BOLT-PATH-NOT: DW_AT_comp_dir
+; BOLT-PATH: DW_TAG_compile_unit
+; BOLT-PATH:      .debug_str_offsets.dwo contents:
+; BOLT-PATH-NEXT: 0x00000000: Contribution size = 68, Format = DWARF32, Version = 5
+; BOLT-PATH-NEXT: "main"
+; BOLT-PATH-NEXT: "int"
+; BOLT-PATH-NEXT: "argc"
+; BOLT-PATH-NEXT: "argv"
+; BOLT-PATH-NEXT: "char"
+; BOLT-PATH-NEXT: "f2"
+; BOLT-PATH-NEXT: dwarf5-df-types-modify-dwo-name-mixed.test.tmp/DWOOut"
+; BOLT-PATH-NEXT: "main.dwo0.dwo"
+; BOLT-PATH-NEXT: "c1"
+; BOLT-PATH-NEXT: "Foo2"
+; BOLT-PATH-NEXT: "f3"
+; BOLT-PATH-NEXT: "c2"
+; BOLT-PATH-NEXT: "c3"
+; BOLT-PATH-NEXT: "Foo2a"
+; BOLT-PATH-NEXT: "clang version 18.0.0git (git@github.com:ayermolo/llvm-project.git db35fa8fc524127079662802c4735dbf397f86d0)"
+; BOLT-PATH-NEXT: "main.cpp"
+; BOLT-PATH-NEXT: helper.dwo0.dwo: file format elf64-x86-64
+
+; BOLT-PATH:      .debug_str_offsets.dwo contents:
+; BOLT-PATH-NEXT: Contribution size = 64, Format = DWARF32, Version = 5
+; BOLT-PATH-NEXT: "fooint"
+; BOLT-PATH-NEXT: "int"
+; BOLT-PATH-NEXT: "_Z3foov"
+; BOLT-PATH-NEXT: "foo"
+; BOLT-PATH-NEXT: "fint"
+; BOLT-PATH-NEXT: "c1"
+; BOLT-PATH-NEXT: "c2"
+; BOLT-PATH-NEXT: "Foo2Int"
+; BOLT-PATH-NEXT: "f"
+; BOLT-PATH-NEXT: "char"
+; BOLT-PATH-NEXT: "c3"
+; BOLT-PATH-NEXT: "Foo2a"
+; BOLT-PATH-NEXT: "clang version 18.0.0"
+; BOLT-PATH-NEXT: "helper.cpp"
+; BOLT-PATH-NEXT: "helper.dwo"
diff --git a/bolt/test/X86/dwarf5-df-types-modify-dwo-name.test b/bolt/test/X86/dwarf5-df-types-modify-dwo-name.test
new file mode 100644
index 000000000000..086f8f813962
--- /dev/null
+++ b/bolt/test/X86/dwarf5-df-types-modify-dwo-name.test
@@ -0,0 +1,175 @@
+; RUN: rm -rf %t
+; RUN: mkdir %t
+; RUN: cd %t
+; RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %p/Inputs/dwarf5-df-types-debug-names-main.s \
+; RUN: -split-dwarf-file=main.dwo -o main.o
+; RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %p/Inputs/dwarf5-df-types-debug-names-helper.s \
+; RUN: -split-dwarf-file=helper.dwo -o helper.o
+; RUN: %clang %cflags -gdwarf-5 -gsplit-dwarf=split main.o helper.o -o main.exe
+; RUN: llvm-bolt main.exe -o main.exe.bolt --update-debug-sections
+; RUN: llvm-dwarfdump --debug-info -r 0 main.exe.bolt > log.txt
+; RUN: llvm-dwarfdump --debug-info -r 0 main.dwo.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-info -r 0 helper.dwo.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-str-offsets main.dwo.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-str-offsets helper.dwo.dwo >> log.txt
+; RUN: cat log.txt | FileCheck -check-prefix=BOLT %s
+
+;; Tests that BOLT correctly updates DW_AT_dwo_name for TU Untis.
+
+; BOLT: DW_TAG_skeleton_unit
+; BOLT: DW_AT_comp_dir  (".")
+; BOLT: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT: DW_TAG_skeleton_unit
+; BOLT: DW_AT_comp_dir  (".")
+; BOLT: DW_AT_dwo_name  ("helper.dwo.dwo")
+; BOLT: DW_TAG_type_unit
+; BOLT: DW_AT_comp_dir  (".")
+; BOLT: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT: DW_TAG_type_unit
+; BOLT: DW_AT_comp_dir  (".")
+; BOLT: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT: DW_TAG_type_unit
+; BOLT: DW_AT_comp_dir  (".")
+; BOLT: DW_AT_dwo_name  ("helper.dwo.dwo")
+; BOLT: DW_TAG_type_unit
+; BOLT: DW_AT_comp_dir  (".")
+; BOLT: DW_AT_dwo_name  ("helper.dwo.dwo")
+; BOLT:      .debug_str_offsets.dwo contents:
+; BOLT-NEXT: 0x00000000: Contribution size = 68, Format = DWARF32, Version = 5
+; BOLT-NEXT: "main"
+; BOLT-NEXT: "int"
+; BOLT-NEXT: "argc"
+; BOLT-NEXT: "argv"
+; BOLT-NEXT: "char"
+; BOLT-NEXT: "f2"
+; BOLT-NEXT: "."
+; BOLT-NEXT: "main.dwo.dwo"
+; BOLT-NEXT: "c1"
+; BOLT-NEXT: "Foo2"
+; BOLT-NEXT: "f3"
+; BOLT-NEXT: "c2"
+; BOLT-NEXT: "c3"
+; BOLT-NEXT: "Foo2a"
+; BOLT-NEXT: "clang version 18.0.0git (git@github.com:ayermolo/llvm-project.git db35fa8fc524127079662802c4735dbf397f86d0)"
+; BOLT-NEXT: "main.cpp"
+; BOLT-NEXT: helper.dwo.dwo: file format elf64-x86-64
+
+; BOLT:      .debug_str_offsets.dwo contents:
+; BOLT-NEXT: 0x00000000: Contribution size = 68, Format = DWARF32, Version = 5
+; BOLT-NEXT: "fooint"
+; BOLT-NEXT: "int"
+; BOLT-NEXT: "_Z3foov"
+; BOLT-NEXT: "foo"
+; BOLT-NEXT: "fint"
+; BOLT-NEXT: "."
+; BOLT-NEXT: "helper.dwo.dwo"
+; BOLT-NEXT: "c1"
+; BOLT-NEXT: "c2"
+; BOLT-NEXT: "Foo2Int"
+; BOLT-NEXT: "f"
+; BOLT-NEXT: "char"
+; BOLT-NEXT: "c3"
+; BOLT-NEXT: "Foo2a"
+; BOLT-NEXT: "clang version 18.0.0git (git@github.com:ayermolo/llvm-project.git db35fa8fc524127079662802c4735dbf397f86d0)"
+; BOLT-NEXT: "helper.cpp"
+
+
+;; Tests that BOLT correctly handles updating DW_AT_dwo_name when it outputs a DWP file.
+;; Currently skipping one of Type units because it is not being de-dupped.
+;; In the tu-index this TU is not present.
+; RUN: rm main.exe.bolt
+; RUN: llvm-bolt main.exe -o main.exe.bolt --update-debug-sections --write-dwp
+; RUN: llvm-dwarfdump --debug-info -r 0 main.exe.bolt.dwp > logDWP.txt
+; RUN: llvm-dwarfdump --debug-str-offsets main.exe.bolt.dwp >> logDWP.txt
+; RUN: cat logDWP.txt | FileCheck -check-prefix=BOLT-DWP %s
+; BOLT-DWP: DW_TAG_type_unit
+; BOLT-DWP: DW_AT_comp_dir  (".")
+; BOLT-DWP: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT-DWP: DW_TAG_type_unit
+; BOLT-DWP: DW_AT_comp_dir  (".")
+; BOLT-DWP: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT-DWP: DW_TAG_compile_unit
+; BOLT-DWP: DW_AT_dwo_name  ("main.dwo.dwo")
+; BOLT-DWP: DW_TAG_type_unit
+; BOLT-DWP: DW_AT_comp_dir  (".")
+; BOLT-DWP: DW_AT_dwo_name  ("helper.dwo.dwo")
+; BOLT-DWP: DW_TAG_type_unit
+; BOLT-DWP: DW_TAG_compile_unit
+; BOLT-DWP: DW_AT_name  ("helper.cpp")
+; BOLT-DWP: DW_AT_dwo_name  ("helper.dwo.dwo")
+
+;; Tests that BOLT correctly handles updating DW_AT_comp_dir/DW_AT_dwo_name when outptut directory is specified.
+
+; RUN: mkdir DWOOut
+; RUN: rm main.exe.bolt
+; RUN: llvm-bolt main.exe -o main.exe.bolt --update-debug-sections --dwarf-output-path=%t/DWOOut
+; RUN: cd DWOOut
+; RUN: llvm-dwarfdump --debug-info -r 0 ../main.exe.bolt > log.txt
+; RUN: llvm-dwarfdump --debug-info -r 0 main.dwo0.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-info -r 0 helper.dwo0.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-str-offsets main.dwo0.dwo >> log.txt
+; RUN: llvm-dwarfdump --debug-str-offsets helper.dwo0.dwo >> log.txt
+; RUN: cat log.txt | FileCheck -check-prefix=BOLT-PATH %s
+
+; BOLT-PATH: DW_TAG_skeleton_unit
+; BOLT-PATH: DW_AT_comp_dir  ("
+; BOLT-PATH-SAME: dwarf5-df-types-modify-dwo-name.test.tmp/DWOOut
+; BOLT-PATH: DW_AT_dwo_name  ("main.dwo0.dwo")
+; BOLT-PATH: DW_TAG_skeleton_unit
+; BOLT-PATH: DW_AT_comp_dir  ("
+; BOLT-PATH-SAME: dwarf5-df-types-modify-dwo-name.test.tmp/DWOOut
+; BOLT-PATH: DW_AT_dwo_name  ("helper.dwo0.dwo")
+; BOLT-PATH: DW_TAG_type_unit
+; BOLT-PATH: DW_AT_comp_dir  ("
+; BOLT-PATH-SAME: dwarf5-df-types-modify-dwo-name.test.tmp/DWOOut
+; BOLT-PATH: DW_AT_dwo_name  ("main.dwo0.dwo")
+; BOLT-PATH: DW_TAG_type_unit
+; BOLT-PATH: DW_AT_comp_dir  ("
+; BOLT-PATH-SAME: dwarf5-df-types-modify-dwo-name.test.tmp/DWOOut
+; BOLT-PATH: DW_AT_dwo_name  ("main.dwo0.dwo")
+; BOLT-PATH: DW_TAG_type_unit
+; BOLT-PATH: DW_AT_comp_dir  ("
+; BOLT-PATH-SAME: dwarf5-df-types-modify-dwo-name.test.tmp/DWOOut
+; BOLT-PATH: DW_AT_dwo_name  ("helper.dwo0.dwo")
+; BOLT-PATH: DW_TAG_type_unit
+; BOLT-PATH: DW_AT_comp_dir  ("
+; BOLT-PATH-SAME: dwarf5-df-types-modify-dwo-name.test.tmp/DWOOut
+; BOLT-PATH: DW_AT_dwo_name  ("helper.dwo0.dwo")
+; BOLT-PATH:      .debug_str_offsets.dwo contents:
+; BOLT-PATH-NEXT: 0x00000000: Contribution size = 68, Format = DWARF32, Version = 5
+; BOLT-PATH-NEXT: "main"
+; BOLT-PATH-NEXT: "int"
+; BOLT-PATH-NEXT: "argc"
+; BOLT-PATH-NEXT: "argv"
+; BOLT-PATH-NEXT: "char"
+; BOLT-PATH-NEXT: "f2"
+; BOLT-PATH-NEXT: dwarf5-df-types-modify-dwo-name.test.tmp/DWOOut"
+; BOLT-PATH-NEXT: "main.dwo0.dwo"
+; BOLT-PATH-NEXT: "c1"
+; BOLT-PATH-NEXT: "Foo2"
+; BOLT-PATH-NEXT: "f3"
+; BOLT-PATH-NEXT: "c2"
+; BOLT-PATH-NEXT: "c3"
+; BOLT-PATH-NEXT: "Foo2a"
+; BOLT-PATH-NEXT: "clang version 18.0.0git (git@github.com:ayermolo/llvm-project.git db35fa8fc524127079662802c4735dbf397f86d0)"
+; BOLT-PATH-NEXT: "main.cpp"
+; BOLT-PATH-NEXT: helper.dwo0.dwo: file format elf64-x86-64
+
+; BOLT-PATH:      .debug_str_offsets.dwo contents:
+; BOLT-PATH-NEXT: 0x00000000: Contribution size = 68, Format = DWARF32, Version = 5
+; BOLT-PATH-NEXT: "fooint"
+; BOLT-PATH-NEXT: "int"
+; BOLT-PATH-NEXT: "_Z3foov"
+; BOLT-PATH-NEXT: "foo"
+; BOLT-PATH-NEXT: "fint"
+; BOLT-PATH-NEXT: dwarf5-df-types-modify-dwo-name.test.tmp/DWOOut"
+; BOLT-PATH-NEXT: "helper.dwo0.dwo"
+; BOLT-PATH-NEXT: "c1"
+; BOLT-PATH-NEXT: "c2"
+; BOLT-PATH-NEXT: "Foo2Int"
+; BOLT-PATH-NEXT: "f"
+; BOLT-PATH-NEXT: "char"
+; BOLT-PATH-NEXT: "c3"
+; BOLT-PATH-NEXT: "Foo2a"
+; BOLT-PATH-NEXT: "clang version 18.0.0git (git@github.com:ayermolo/llvm-project.git db35fa8fc524127079662802c4735dbf397f86d0)"
+; BOLT-PATH-NEXT: "helper.cpp"
-- 
GitLab


From 844355a8cb4b4fa4a6fa39ac47e1169233bb7130 Mon Sep 17 00:00:00 2001
From: VincentWu <43398706+Xinlong-Wu@users.noreply.github.com>
Date: Wed, 15 May 2024 08:26:45 +1000
Subject: [PATCH 0787/1206] [RISC-V] remove I ext when E ext has been enabled
 (#92070)

After patch https://github.com/llvm/llvm-project/pull/88805

`I` Ext will be added automatically when we running the command like
`./build/bin/llc -mtriple=riscv32 -mattr=+e -target-abi ilp32e
-verify-machineinstrs llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll`

it will generate
```
	.text
	.attribute	4, 16
	.attribute	5, "rv32i2p1_e2pe"
	.file	"zcmp-additional-stack.ll"
	.globl	func                            # -- Begin function func
	.p2align	1
	.type	func,@function
```

This patch reset the I ext in FeatureBit when `+e` has been specify
---
 llvm/lib/TargetParser/RISCVISAInfo.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index e22dd6032cb0..575c9dbad515 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -758,6 +758,8 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
 }
 
 Error RISCVISAInfo::checkDependency() {
+  bool HasE = Exts.count("e") != 0;
+  bool HasI = Exts.count("i") != 0;
   bool HasC = Exts.count("c") != 0;
   bool HasF = Exts.count("f") != 0;
   bool HasZfinx = Exts.count("zfinx") != 0;
@@ -765,6 +767,10 @@ Error RISCVISAInfo::checkDependency() {
   bool HasZvl = MinVLen != 0;
   bool HasZcmt = Exts.count("zcmt") != 0;
 
+  if (HasI && HasE)
+    return createStringError(errc::invalid_argument,
+                             "'I' and 'E' extensions are incompatible");
+
   if (HasF && HasZfinx)
     return createStringError(errc::invalid_argument,
                              "'f' and 'zfinx' extensions are incompatible");
@@ -852,6 +858,9 @@ void RISCVISAInfo::updateImplication() {
     addExtension("i", Version.value());
   }
 
+  if (HasE && HasI)
+    Exts.erase("i");
+
   assert(llvm::is_sorted(ImpliedExts) && "Table not sorted by Name");
 
   // This loop may execute over 1 iteration since implication can be layered
-- 
GitLab


From 3ca428c090624d3cfc530144da6dcd3abfd9ea63 Mon Sep 17 00:00:00 2001
From: Joe Nash <joseph.nash@amd.com>
Date: Tue, 14 May 2024 18:29:16 -0400
Subject: [PATCH 0788/1206] [AMDGPU][True16] Add VOP1Inst_t16_with_profiles
 class (#92184)

NFC. Makes the VOP1Inst_t16 interface more generic to support future
instructions cleanly.
---
 llvm/lib/Target/AMDGPU/VOP1Instructions.td | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 4a56fad0cd60..b875ddc62a7a 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -153,20 +153,26 @@ multiclass VOP1Inst <string opName, VOPProfile P,
     def : LetDummies, AMDGPUMnemonicAlias<opName#"_dpp", opName, AMDGPUAsmVariants.DPP>;
 }
 
-multiclass VOP1Inst_t16<string opName,
+multiclass VOP1Inst_t16_with_profiles<string opName,
                         VOPProfile P,
+                        VOPProfile P_t16,
+                        VOPProfile P_fake16,
                         SDPatternOperator node = null_frag> {
   let OtherPredicates = [NotHasTrue16BitInsts, Has16BitInsts]  in {
     defm NAME : VOP1Inst<opName, P, node>;
   }
   let OtherPredicates = [UseRealTrue16Insts] in {
-    defm _t16 : VOP1Inst<opName#"_t16", VOPProfile_True16<P>, node>;
+    defm _t16 : VOP1Inst<opName#"_t16", P_t16, node>;
   }
   let OtherPredicates = [UseFakeTrue16Insts] in {
-    defm _fake16 : VOP1Inst<opName#"_fake16", VOPProfile_Fake16<P>, node>;
+    defm _fake16 : VOP1Inst<opName#"_fake16", P_fake16, node>;
   }
 }
 
+multiclass VOP1Inst_t16<string opName, VOPProfile P,
+                        SDPatternOperator node = null_frag> :
+  VOP1Inst_t16_with_profiles<opName, P, VOPProfile_True16<P>, VOPProfile_Fake16<P>, node>;
+
 // Special profile for instructions which have clamp
 // and output modifiers (but have no input modifiers)
 class VOPProfileI2F<ValueType dstVt, ValueType srcVt> :
-- 
GitLab


From e417e61532ac373e7b0708262dedefcdaf6ced9c Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 14 May 2024 12:52:48 -0700
Subject: [PATCH 0789/1206] [RISCV][LegalizeTypes] Add additional test coverage
 for type promotion of VP_FSHL/FSHR. NFC

There's a special path when the promoted type has an element size
more than twice the size of the original type.
---
 llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll | 53 +++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll
index f9f085dcc161..277cd7dcdabc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll
@@ -1370,3 +1370,56 @@ define <vscale x 1 x i9> @fshl_v1i9(<vscale x 1 x i9> %a, <vscale x 1 x i9> %b,
   %res = call <vscale x 1 x i9> @llvm.vp.fshl.nxv1i9(<vscale x 1 x i9> %a, <vscale x 1 x i9> %b, <vscale x 1 x i9> %c, <vscale x 1 x i1> %m, i32 %evl)
   ret <vscale x 1 x i9> %res
 }
+
+declare <vscale x 1 x i4> @llvm.vp.trunc.nxv1i4.nxv1i8(<vscale x 1 x i8>, <vscale x 1 x i1>, i32)
+declare <vscale x 1 x i8> @llvm.vp.zext.nxv1i8.nxv1i4(<vscale x 1 x i4>, <vscale x 1 x i1>, i32)
+declare <vscale x 1 x i4> @llvm.vp.fshr.nxv1i4(<vscale x 1 x i4>, <vscale x 1 x i4>, <vscale x 1 x i4>, <vscale x 1 x i1>, i32)
+define <vscale x 1 x i8> @fshr_v1i4(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b, <vscale x 1 x i8> %c, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: fshr_v1i4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vand.vi v10, v10, 15
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vremu.vx v10, v10, a1, v0.t
+; CHECK-NEXT:    vsll.vi v8, v8, 4, v0.t
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vand.vi v9, v9, 15
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vor.vv v8, v8, v9, v0.t
+; CHECK-NEXT:    vsrl.vv v8, v8, v10, v0.t
+; CHECK-NEXT:    vand.vi v8, v8, 15, v0.t
+; CHECK-NEXT:    ret
+  %trunca = call <vscale x 1 x i4> @llvm.vp.trunc.nxv1i4.nxv1i8(<vscale x 1 x i8> %a, <vscale x 1 x i1> %m, i32 zeroext %evl)
+  %truncb = call <vscale x 1 x i4> @llvm.vp.trunc.nxv1i4.nxv1i8(<vscale x 1 x i8> %b, <vscale x 1 x i1> %m, i32 zeroext %evl)
+  %truncc = call <vscale x 1 x i4> @llvm.vp.trunc.nxv1i4.nxv1i8(<vscale x 1 x i8> %c, <vscale x 1 x i1> %m, i32 zeroext %evl)
+  %fshr = call <vscale x 1 x i4> @llvm.vp.fshr.nxv1i4(<vscale x 1 x i4> %trunca, <vscale x 1 x i4> %truncb, <vscale x 1 x i4> %truncc, <vscale x 1 x i1> %m, i32 %evl)
+  %res = call <vscale x 1 x i8> @llvm.vp.zext.nxv1i8.nxv1i4(<vscale x 1 x i4> %fshr, <vscale x 1 x i1> %m, i32 zeroext %evl)
+  ret <vscale x 1 x i8> %res
+}
+
+declare <vscale x 1 x i4> @llvm.vp.fshl.nxv1i4(<vscale x 1 x i4>, <vscale x 1 x i4>, <vscale x 1 x i4>, <vscale x 1 x i1>, i32)
+define <vscale x 1 x i8> @fshl_v1i4(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b, <vscale x 1 x i8> %c, <vscale x 1 x i1> %m, i32 zeroext %evl) {
+; CHECK-LABEL: fshl_v1i4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vand.vi v10, v10, 15
+; CHECK-NEXT:    li a1, 4
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vremu.vx v10, v10, a1, v0.t
+; CHECK-NEXT:    vsll.vi v8, v8, 4, v0.t
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vand.vi v9, v9, 15
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; CHECK-NEXT:    vor.vv v8, v8, v9, v0.t
+; CHECK-NEXT:    vsll.vv v8, v8, v10, v0.t
+; CHECK-NEXT:    vsrl.vi v8, v8, 4, v0.t
+; CHECK-NEXT:    vand.vi v8, v8, 15, v0.t
+; CHECK-NEXT:    ret
+  %trunca = call <vscale x 1 x i4> @llvm.vp.trunc.nxv1i4.nxv1i8(<vscale x 1 x i8> %a, <vscale x 1 x i1> %m, i32 zeroext %evl)
+  %truncb = call <vscale x 1 x i4> @llvm.vp.trunc.nxv1i4.nxv1i8(<vscale x 1 x i8> %b, <vscale x 1 x i1> %m, i32 zeroext %evl)
+  %truncc = call <vscale x 1 x i4> @llvm.vp.trunc.nxv1i4.nxv1i8(<vscale x 1 x i8> %c, <vscale x 1 x i1> %m, i32 zeroext %evl)
+  %fshl = call <vscale x 1 x i4> @llvm.vp.fshl.nxv1i4(<vscale x 1 x i4> %trunca, <vscale x 1 x i4> %truncb, <vscale x 1 x i4> %truncc, <vscale x 1 x i1> %m, i32 %evl)
+  %res = call <vscale x 1 x i8> @llvm.vp.zext.nxv1i8.nxv1i4(<vscale x 1 x i4> %fshl, <vscale x 1 x i1> %m, i32 zeroext %evl)
+  ret <vscale x 1 x i8> %res
+}
-- 
GitLab


From 4a17e86f27a0a527ef5316f14fa0e5a60546d9ec Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 14 May 2024 18:39:48 -0500
Subject: [PATCH 0790/1206] [LinkerWrapper] Add an overriding option for
 debugging (#91984)

Summary:
One of the downsides of the linker wrapper is that it made debugging
more difficult. It is very powerful in that it can resolve a lot of
input matching and library handling that could not be done before.
However, the old method allowed users to simply copy-paste the script
files to modify the output and test it.

This patch attempts to make it easier to debug changes by letting the
user override all the linker inputs. That is, we provide a user-created
binary that is treated like the final output of the device link step.
The intended use-case is for using `-save-temps` to get some IR, then
modifying the IR and sticking it back in to see if it exhibits the old
failures.
---
 clang/docs/ClangLinkerWrapper.rst             | 38 ++++++++++++++++
 clang/test/Driver/linker-wrapper.c            |  7 +++
 .../ClangLinkerWrapper.cpp                    | 43 +++++++++++++++++++
 .../clang-linker-wrapper/LinkerWrapperOpts.td |  4 ++
 4 files changed, 92 insertions(+)

diff --git a/clang/docs/ClangLinkerWrapper.rst b/clang/docs/ClangLinkerWrapper.rst
index 3bef55847573..99352863b477 100644
--- a/clang/docs/ClangLinkerWrapper.rst
+++ b/clang/docs/ClangLinkerWrapper.rst
@@ -46,6 +46,8 @@ only for the linker wrapper will be forwarded to the wrapped linker job.
     -l <libname>           Search for library <libname>
     --opt-level=<O0, O1, O2, or O3>
                            Optimization level for LTO
+    --override-image=<kind=file>
+                            Uses the provided file as if it were the output of the device link step
     -o <path>              Path to file to write output
     --pass-remarks-analysis=<value>
                            Pass remarks for LTO
@@ -87,6 +89,42 @@ other. Generally, this requires that the target triple and architecture match.
 An exception is made when the architecture is listed as ``generic``, which will
 cause it be linked with any other device code with the same target triple.
 
+Debugging
+=========
+
+The linker wrapper performs a lot of steps internally, such as input matching,
+symbol resolution, and image registration. This makes it difficult to debug in
+some scenarios. The behavior of the linker-wrapper is controlled mostly through
+metadata, described in `clang documentation
+<https://clang.llvm.org/docs/OffloadingDesign.html>`_. Intermediate output can
+be obtained from the linker-wrapper using the ``--save-temps`` flag. These files
+can then be modified.
+
+.. code-block:: sh
+
+  $> clang openmp.c -fopenmp --offload-arch=gfx90a -c
+  $> clang openmp.o -fopenmp --offload-arch=gfx90a -Wl,--save-temps
+  $> ; Modify temp files.
+  $> llvm-objcopy --update-section=.llvm.offloading=out.bc openmp.o
+
+Doing this will allow you to override one of the input files by replacing its
+embedded offloading metadata with a user-modified version. However, this will be
+more difficult when there are multiple input files. For a very large hammer, the
+``--override-image=<kind>=<file>`` flag can be used.
+
+In the following example, we use the ``--save-temps`` to obtain the LLVM-IR just
+before running the backend. We then modify it to test altered behavior, and then
+compile it to a binary. This can then be passed to the linker-wrapper which will
+then ignore all embedded metadata and use the provided image as if it were the
+result of the device linking phase.
+
+.. code-block:: sh
+
+  $> clang openmp.c -fopenmp --offload-arch=gfx90a -Wl,--save-temps
+  $> ; Modify temp files.
+  $> clang --target=amdgcn-amd-amdhsa -mcpu=gfx90a -nogpulib out.bc -o a.out
+  $> clang openmp.c -fopenmp --offload-arch=gfx90a -Wl,--override-image=openmp=a.out
+
 Example
 =======
 
diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c
index 51bf98b2ed39..0d05f913aad6 100644
--- a/clang/test/Driver/linker-wrapper.c
+++ b/clang/test/Driver/linker-wrapper.c
@@ -226,3 +226,10 @@ __attribute__((visibility("protected"), used)) int x;
 // RELOCATABLE-LINK-CUDA: fatbinary{{.*}} -64 --create {{.*}}.fatbin --image=profile=sm_89,file={{.*}}.img
 // RELOCATABLE-LINK-CUDA: /usr/bin/ld.lld{{.*}}-r
 // RELOCATABLE-LINK-CUDA: llvm-objcopy{{.*}}a.out --remove-section .llvm.offloading
+
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o
+// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
+// RUN:   --linker-path=/usr/bin/ld --override=image=openmp=%t.o %t.o -o a.out 2>&1 \
+// RUN: | FileCheck %s --check-prefix=OVERRIDE
+// OVERRIDE-NOT: clang
+// OVERRIDE: /usr/bin/ld
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 69d8cb446fad..aee98c5a524a 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -1149,6 +1149,39 @@ DerivedArgList getLinkerArgs(ArrayRef<OffloadFile> Input,
   return DAL;
 }
 
+Error handleOverrideImages(
+    const InputArgList &Args,
+    DenseMap<OffloadKind, SmallVector<OffloadingImage>> &Images) {
+  for (StringRef Arg : Args.getAllArgValues(OPT_override_image)) {
+    OffloadKind Kind = getOffloadKind(Arg.split("=").first);
+    StringRef Filename = Arg.split("=").second;
+
+    ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
+        MemoryBuffer::getFileOrSTDIN(Filename);
+    if (std::error_code EC = BufferOrErr.getError())
+      return createFileError(Filename, EC);
+
+    Expected<std::unique_ptr<ObjectFile>> ElfOrErr =
+        ObjectFile::createELFObjectFile(**BufferOrErr,
+                                        /*InitContent=*/false);
+    if (!ElfOrErr)
+      return ElfOrErr.takeError();
+    ObjectFile &Elf = **ElfOrErr;
+
+    OffloadingImage TheImage{};
+    TheImage.TheImageKind = IMG_Object;
+    TheImage.TheOffloadKind = Kind;
+    TheImage.StringData["triple"] =
+        Args.MakeArgString(Elf.makeTriple().getTriple());
+    if (std::optional<StringRef> CPU = Elf.tryGetCPUName())
+      TheImage.StringData["arch"] = Args.MakeArgString(*CPU);
+    TheImage.Image = std::move(*BufferOrErr);
+
+    Images[Kind].emplace_back(std::move(TheImage));
+  }
+  return Error::success();
+}
+
 /// Transforms all the extracted offloading input files into an image that can
 /// be registered by the runtime.
 Expected<SmallVector<StringRef>> linkAndWrapDeviceFiles(
@@ -1158,6 +1191,12 @@ Expected<SmallVector<StringRef>> linkAndWrapDeviceFiles(
 
   std::mutex ImageMtx;
   DenseMap<OffloadKind, SmallVector<OffloadingImage>> Images;
+
+  // Initialize the images with any overriding inputs.
+  if (Args.hasArg(OPT_override_image))
+    if (Error Err = handleOverrideImages(Args, Images))
+      return Err;
+
   auto Err = parallelForEachError(LinkerInputFiles, [&](auto &Input) -> Error {
     llvm::TimeTraceScope TimeScope("Link device input");
 
@@ -1439,6 +1478,10 @@ Expected<SmallVector<SmallVector<OffloadFile>>>
 getDeviceInput(const ArgList &Args) {
   llvm::TimeTraceScope TimeScope("ExtractDeviceCode");
 
+  // Skip all the input if the user is overriding the output.
+  if (Args.hasArg(OPT_override_image))
+    return SmallVector<SmallVector<OffloadFile>>();
+
   StringRef Root = Args.getLastArgValue(OPT_sysroot_EQ);
   SmallVector<StringRef> LibraryPaths;
   for (const opt::Arg *Arg : Args.filtered(OPT_library_path, OPT_libpath))
diff --git a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
index 0a8bd541c452..eb31b98a3f54 100644
--- a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
+++ b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
@@ -74,6 +74,10 @@ def wrapper_jobs : Joined<["--"], "wrapper-jobs=">,
   Flags<[WrapperOnlyOption]>, MetaVarName<"<number>">,
   HelpText<"Sets the number of parallel jobs to use for device linking">;
 
+def override_image : Joined<["--"], "override-image=">,
+  Flags<[WrapperOnlyOption]>, MetaVarName<"<kind=file>">,
+  HelpText<"Uses the provided file as if it were the output of the device link step">;
+
 // Flags passed to the device linker.
 def arch_EQ : Joined<["--"], "arch=">,
   Flags<[DeviceOnlyOption, HelpHidden]>, MetaVarName<"<arch>">,
-- 
GitLab


From c5cd049566a795ba5de88dfbb2eb563cad4a9d8a Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 14 May 2024 18:43:42 -0500
Subject: [PATCH 0791/1206] [Clang][Fixup] Fix deleted constructor on older
 compilers

---
 clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index aee98c5a524a..07a8d53c04b1 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -1195,7 +1195,7 @@ Expected<SmallVector<StringRef>> linkAndWrapDeviceFiles(
   // Initialize the images with any overriding inputs.
   if (Args.hasArg(OPT_override_image))
     if (Error Err = handleOverrideImages(Args, Images))
-      return Err;
+      return std::move(Err);
 
   auto Err = parallelForEachError(LinkerInputFiles, [&](auto &Input) -> Error {
     llvm::TimeTraceScope TimeScope("Link device input");
-- 
GitLab


From dfdc3dcbe7f38bde64bc83a74b9c3451c50e1ad4 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Tue, 14 May 2024 18:01:23 -0700
Subject: [PATCH 0792/1206] [ctx_profile] Profile reader and writer (#91859)

Utility converting a profile coming from `compiler_rt` to bitstream, and
a reader.

`PGOCtxProfileWriter::write` would be used as the `Writer` parameter for
`__llvm_ctx_profile_fetch` API. This is expected to happen in user code,
for example in the RPC hanler tasked with collecting a profile, and
would look like this:

```
// set up an output stream "Out", which could contain other stuff
{
  // constructing the Writer will start the section, in Out, containing
  // the collected contextual profiles.
  PGOCtxProfWriter Writer(Out);
  __llvm_ctx_profile_fetch(&Writer, +[](void* W, const ContextNode &N) {
    reinterpret_cast<PGOCtxProfWriter*>(W)->write(N);
  });
  // Writer going out of scope will finish up the section.
}
```

The reader produces a data structure suitable for maintenance during IPO
transformations.
---
 .../llvm/ProfileData/PGOCtxProfReader.h       |  92 +++++++
 .../llvm/ProfileData/PGOCtxProfWriter.h       |  91 +++++++
 llvm/lib/ProfileData/CMakeLists.txt           |   2 +
 llvm/lib/ProfileData/PGOCtxProfReader.cpp     | 173 ++++++++++++
 llvm/lib/ProfileData/PGOCtxProfWriter.cpp     |  49 ++++
 llvm/unittests/ProfileData/CMakeLists.txt     |   1 +
 .../PGOCtxProfReaderWriterTest.cpp            | 255 ++++++++++++++++++
 7 files changed, 663 insertions(+)
 create mode 100644 llvm/include/llvm/ProfileData/PGOCtxProfReader.h
 create mode 100644 llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
 create mode 100644 llvm/lib/ProfileData/PGOCtxProfReader.cpp
 create mode 100644 llvm/lib/ProfileData/PGOCtxProfWriter.cpp
 create mode 100644 llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp

diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
new file mode 100644
index 000000000000..a19b3f51d642
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
@@ -0,0 +1,92 @@
+//===--- PGOCtxProfReader.h - Contextual profile reader ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// Reader for contextual iFDO profile, which comes in bitstream format.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
+#define LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
+
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/Bitstream/BitstreamReader.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Support/Error.h"
+#include <map>
+#include <vector>
+
+namespace llvm {
+/// The loaded contextual profile, suitable for mutation during IPO passes. We
+/// generally expect a fraction of counters and of callsites to be populated.
+/// We continue to model counters as vectors, but callsites are modeled as a map
+/// of a map. The expectation is that, typically, there is a small number of
+/// indirect targets (usually, 1 for direct calls); but potentially a large
+/// number of callsites, and, as inlining progresses, the callsite count of a
+/// caller will grow.
+class PGOContextualProfile final {
+public:
+  using CallTargetMapTy = std::map<GlobalValue::GUID, PGOContextualProfile>;
+  using CallsiteMapTy = DenseMap<uint32_t, CallTargetMapTy>;
+
+private:
+  friend class PGOCtxProfileReader;
+  GlobalValue::GUID GUID = 0;
+  SmallVector<uint64_t, 16> Counters;
+  CallsiteMapTy Callsites;
+
+  PGOContextualProfile(GlobalValue::GUID G,
+                       SmallVectorImpl<uint64_t> &&Counters)
+      : GUID(G), Counters(std::move(Counters)) {}
+
+  Expected<PGOContextualProfile &>
+  getOrEmplace(uint32_t Index, GlobalValue::GUID G,
+               SmallVectorImpl<uint64_t> &&Counters);
+
+public:
+  PGOContextualProfile(const PGOContextualProfile &) = delete;
+  PGOContextualProfile &operator=(const PGOContextualProfile &) = delete;
+  PGOContextualProfile(PGOContextualProfile &&) = default;
+  PGOContextualProfile &operator=(PGOContextualProfile &&) = default;
+
+  GlobalValue::GUID guid() const { return GUID; }
+  const SmallVectorImpl<uint64_t> &counters() const { return Counters; }
+  const CallsiteMapTy &callsites() const { return Callsites; }
+  CallsiteMapTy &callsites() { return Callsites; }
+
+  bool hasCallsite(uint32_t I) const {
+    return Callsites.find(I) != Callsites.end();
+  }
+
+  const CallTargetMapTy &callsite(uint32_t I) const {
+    assert(hasCallsite(I) && "Callsite not found");
+    return Callsites.find(I)->second;
+  }
+  void getContainedGuids(DenseSet<GlobalValue::GUID> &Guids) const;
+};
+
+class PGOCtxProfileReader final {
+  BitstreamCursor &Cursor;
+  Expected<BitstreamEntry> advance();
+  Error readMetadata();
+  Error wrongValue(const Twine &);
+  Error unsupported(const Twine &);
+
+  Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
+  readContext(bool ExpectIndex);
+  bool canReadContext();
+
+public:
+  PGOCtxProfileReader(BitstreamCursor &Cursor) : Cursor(Cursor) {}
+
+  Expected<std::map<GlobalValue::GUID, PGOContextualProfile>> loadContexts();
+};
+} // namespace llvm
+#endif
diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
new file mode 100644
index 000000000000..15578c51a495
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
@@ -0,0 +1,91 @@
+//===- PGOCtxProfWriter.h - Contextual Profile Writer -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares a utility for writing a contextual profile to bitstream.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
+#define LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
+
+#include "llvm/Bitstream/BitstreamWriter.h"
+#include "llvm/ProfileData/CtxInstrContextNode.h"
+
+namespace llvm {
+enum PGOCtxProfileRecords { Invalid = 0, Version, Guid, CalleeIndex, Counters };
+
+enum PGOCtxProfileBlockIDs {
+  ProfileMetadataBlockID = 100,
+  ContextNodeBlockID = ProfileMetadataBlockID + 1
+};
+
+/// Write one or more ContextNodes to the provided raw_fd_stream.
+/// The caller must destroy the PGOCtxProfileWriter object before closing the
+/// stream.
+/// The design allows serializing a bunch of contexts embedded in some other
+/// file. The overall format is:
+///
+///  [... other data written to the stream...]
+///  SubBlock(ProfileMetadataBlockID)
+///   Version
+///   SubBlock(ContextNodeBlockID)
+///     [RECORDS]
+///     SubBlock(ContextNodeBlockID)
+///       [RECORDS]
+///       [... more SubBlocks]
+///     EndBlock
+///   EndBlock
+///
+/// The "RECORDS" are bitsream records. The IDs are in CtxProfileCodes (except)
+/// for Version, which is just for metadata). All contexts will have Guid and
+/// Counters, and all but the roots have CalleeIndex. The order in which the
+/// records appear does not matter, but they must precede any subcontexts,
+/// because that helps keep the reader code simpler.
+///
+/// Subblock containment captures the context->subcontext relationship. The
+/// "next()" relationship in the raw profile, between call targets of indirect
+/// calls, are just modeled as peer subblocks where the callee index is the
+/// same.
+///
+/// Versioning: the writer may produce additional records not known by the
+/// reader. The version number indicates a more structural change.
+/// The current version, in particular, is set up to expect optional extensions
+/// like value profiling - which would appear as additional records. For
+/// example, value profiling would produce a new record with a new record ID,
+/// containing the profiled values (much like the counters)
+class PGOCtxProfileWriter final {
+  SmallVector<char, 1 << 20> Buff;
+  BitstreamWriter Writer;
+
+  void writeCounters(const ctx_profile::ContextNode &Node);
+  void writeImpl(std::optional<uint32_t> CallerIndex,
+                 const ctx_profile::ContextNode &Node);
+
+public:
+  PGOCtxProfileWriter(raw_fd_stream &Out,
+                      std::optional<unsigned> VersionOverride = std::nullopt)
+      : Writer(Buff, &Out, 0) {
+    Writer.EnterSubblock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID,
+                         CodeLen);
+    const auto Version = VersionOverride ? *VersionOverride : CurrentVersion;
+    Writer.EmitRecord(PGOCtxProfileRecords::Version,
+                      SmallVector<unsigned, 1>({Version}));
+  }
+
+  ~PGOCtxProfileWriter() { Writer.ExitBlock(); }
+
+  void write(const ctx_profile::ContextNode &);
+
+  // constants used in writing which a reader may find useful.
+  static constexpr unsigned CodeLen = 2;
+  static constexpr uint32_t CurrentVersion = 1;
+  static constexpr unsigned VBREncodingBits = 6;
+};
+
+} // namespace llvm
+#endif
diff --git a/llvm/lib/ProfileData/CMakeLists.txt b/llvm/lib/ProfileData/CMakeLists.txt
index 408f9ff01ec8..2397eebaf7b1 100644
--- a/llvm/lib/ProfileData/CMakeLists.txt
+++ b/llvm/lib/ProfileData/CMakeLists.txt
@@ -7,6 +7,8 @@ add_llvm_component_library(LLVMProfileData
   ItaniumManglingCanonicalizer.cpp
   MemProf.cpp
   MemProfReader.cpp
+  PGOCtxProfReader.cpp
+  PGOCtxProfWriter.cpp
   ProfileSummaryBuilder.cpp
   SampleProf.cpp
   SampleProfReader.cpp
diff --git a/llvm/lib/ProfileData/PGOCtxProfReader.cpp b/llvm/lib/ProfileData/PGOCtxProfReader.cpp
new file mode 100644
index 000000000000..3710f2e4b818
--- /dev/null
+++ b/llvm/lib/ProfileData/PGOCtxProfReader.cpp
@@ -0,0 +1,173 @@
+//===- PGOCtxProfReader.cpp - Contextual Instrumentation profile reader ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Read a contextual profile into a datastructure suitable for maintenance
+// throughout IPO
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ProfileData/PGOCtxProfReader.h"
+#include "llvm/Bitstream/BitCodeEnums.h"
+#include "llvm/Bitstream/BitstreamReader.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+
+using namespace llvm;
+
+// FIXME(#92054) - these Error handling macros are (re-)invented in a few
+// places.
+#define EXPECT_OR_RET(LHS, RHS)                                                \
+  auto LHS = RHS;                                                              \
+  if (!LHS)                                                                    \
+    return LHS.takeError();
+
+#define RET_ON_ERR(EXPR)                                                       \
+  if (auto Err = (EXPR))                                                       \
+    return Err;
+
+Expected<PGOContextualProfile &>
+PGOContextualProfile::getOrEmplace(uint32_t Index, GlobalValue::GUID G,
+                                   SmallVectorImpl<uint64_t> &&Counters) {
+  auto [Iter, Inserted] = Callsites[Index].insert(
+      {G, PGOContextualProfile(G, std::move(Counters))});
+  if (!Inserted)
+    return make_error<InstrProfError>(instrprof_error::invalid_prof,
+                                      "Duplicate GUID for same callsite.");
+  return Iter->second;
+}
+
+void PGOContextualProfile::getContainedGuids(
+    DenseSet<GlobalValue::GUID> &Guids) const {
+  Guids.insert(GUID);
+  for (const auto &[_, Callsite] : Callsites)
+    for (const auto &[_, Callee] : Callsite)
+      Callee.getContainedGuids(Guids);
+}
+
+Expected<BitstreamEntry> PGOCtxProfileReader::advance() {
+  return Cursor.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs);
+}
+
+Error PGOCtxProfileReader::wrongValue(const Twine &Msg) {
+  return make_error<InstrProfError>(instrprof_error::invalid_prof, Msg);
+}
+
+Error PGOCtxProfileReader::unsupported(const Twine &Msg) {
+  return make_error<InstrProfError>(instrprof_error::unsupported_version, Msg);
+}
+
+bool PGOCtxProfileReader::canReadContext() {
+  auto Blk = advance();
+  if (!Blk) {
+    consumeError(Blk.takeError());
+    return false;
+  }
+  return Blk->Kind == BitstreamEntry::SubBlock &&
+         Blk->ID == PGOCtxProfileBlockIDs::ContextNodeBlockID;
+}
+
+Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
+PGOCtxProfileReader::readContext(bool ExpectIndex) {
+  RET_ON_ERR(Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID));
+
+  std::optional<ctx_profile::GUID> Guid;
+  std::optional<SmallVector<uint64_t, 16>> Counters;
+  std::optional<uint32_t> CallsiteIndex;
+
+  SmallVector<uint64_t, 1> RecordValues;
+
+  // We don't prescribe the order in which the records come in, and we are ok
+  // if other unsupported records appear. We seek in the current subblock until
+  // we get all we know.
+  auto GotAllWeNeed = [&]() {
+    return Guid.has_value() && Counters.has_value() &&
+           (!ExpectIndex || CallsiteIndex.has_value());
+  };
+  while (!GotAllWeNeed()) {
+    RecordValues.clear();
+    EXPECT_OR_RET(Entry, advance());
+    if (Entry->Kind != BitstreamEntry::Record)
+      return wrongValue(
+          "Expected records before encountering more subcontexts");
+    EXPECT_OR_RET(ReadRecord,
+                  Cursor.readRecord(bitc::UNABBREV_RECORD, RecordValues));
+    switch (*ReadRecord) {
+    case PGOCtxProfileRecords::Guid:
+      if (RecordValues.size() != 1)
+        return wrongValue("The GUID record should have exactly one value");
+      Guid = RecordValues[0];
+      break;
+    case PGOCtxProfileRecords::Counters:
+      Counters = std::move(RecordValues);
+      if (Counters->empty())
+        return wrongValue("Empty counters. At least the entry counter (one "
+                          "value) was expected");
+      break;
+    case PGOCtxProfileRecords::CalleeIndex:
+      if (!ExpectIndex)
+        return wrongValue("The root context should not have a callee index");
+      if (RecordValues.size() != 1)
+        return wrongValue("The callee index should have exactly one value");
+      CallsiteIndex = RecordValues[0];
+      break;
+    default:
+      // OK if we see records we do not understand, like records (profile
+      // components) introduced later.
+      break;
+    }
+  }
+
+  PGOContextualProfile Ret(*Guid, std::move(*Counters));
+
+  while (canReadContext()) {
+    EXPECT_OR_RET(SC, readContext(true));
+    auto &Targets = Ret.callsites()[*SC->first];
+    auto [_, Inserted] =
+        Targets.insert({SC->second.guid(), std::move(SC->second)});
+    if (!Inserted)
+      return wrongValue(
+          "Unexpected duplicate target (callee) at the same callsite.");
+  }
+  return std::make_pair(CallsiteIndex, std::move(Ret));
+}
+
+Error PGOCtxProfileReader::readMetadata() {
+  EXPECT_OR_RET(Blk, advance());
+  if (Blk->Kind != BitstreamEntry::SubBlock)
+    return unsupported("Expected Version record");
+  RET_ON_ERR(
+      Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID));
+  EXPECT_OR_RET(MData, advance());
+  if (MData->Kind != BitstreamEntry::Record)
+    return unsupported("Expected Version record");
+
+  SmallVector<uint64_t, 1> Ver;
+  EXPECT_OR_RET(Code, Cursor.readRecord(bitc::UNABBREV_RECORD, Ver));
+  if (*Code != PGOCtxProfileRecords::Version)
+    return unsupported("Expected Version record");
+  if (Ver.size() != 1 || Ver[0] > PGOCtxProfileWriter::CurrentVersion)
+    return unsupported("Version " + Twine(*Code) +
+                       " is higher than supported version " +
+                       Twine(PGOCtxProfileWriter::CurrentVersion));
+  return Error::success();
+}
+
+Expected<std::map<GlobalValue::GUID, PGOContextualProfile>>
+PGOCtxProfileReader::loadContexts() {
+  std::map<GlobalValue::GUID, PGOContextualProfile> Ret;
+  RET_ON_ERR(readMetadata());
+  while (canReadContext()) {
+    EXPECT_OR_RET(E, readContext(false));
+    auto Key = E->second.guid();
+    if (!Ret.insert({Key, std::move(E->second)}).second)
+      return wrongValue("Duplicate roots");
+  }
+  return Ret;
+}
diff --git a/llvm/lib/ProfileData/PGOCtxProfWriter.cpp b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
new file mode 100644
index 000000000000..508179756446
--- /dev/null
+++ b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
@@ -0,0 +1,49 @@
+//===- PGOCtxProfWriter.cpp - Contextual Instrumentation profile writer ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Write a contextual profile to bitstream.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Bitstream/BitCodeEnums.h"
+
+using namespace llvm;
+using namespace llvm::ctx_profile;
+
+void PGOCtxProfileWriter::writeCounters(const ContextNode &Node) {
+  Writer.EmitCode(bitc::UNABBREV_RECORD);
+  Writer.EmitVBR(PGOCtxProfileRecords::Counters, VBREncodingBits);
+  Writer.EmitVBR(Node.counters_size(), VBREncodingBits);
+  for (uint32_t I = 0U; I < Node.counters_size(); ++I)
+    Writer.EmitVBR64(Node.counters()[I], VBREncodingBits);
+}
+
+// recursively write all the subcontexts. We do need to traverse depth first to
+// model the context->subcontext implicitly, and since this captures call
+// stacks, we don't really need to be worried about stack overflow and we can
+// keep the implementation simple.
+void PGOCtxProfileWriter::writeImpl(std::optional<uint32_t> CallerIndex,
+                                    const ContextNode &Node) {
+  Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextNodeBlockID, CodeLen);
+  Writer.EmitRecord(PGOCtxProfileRecords::Guid,
+                    SmallVector<uint64_t, 1>{Node.guid()});
+  if (CallerIndex)
+    Writer.EmitRecord(PGOCtxProfileRecords::CalleeIndex,
+                      SmallVector<uint64_t, 1>{*CallerIndex});
+  writeCounters(Node);
+  for (uint32_t I = 0U; I < Node.callsites_size(); ++I)
+    for (const auto *Subcontext = Node.subContexts()[I]; Subcontext;
+         Subcontext = Subcontext->next())
+      writeImpl(I, *Subcontext);
+  Writer.ExitBlock();
+}
+
+void PGOCtxProfileWriter::write(const ContextNode &RootNode) {
+  writeImpl(std::nullopt, RootNode);
+}
diff --git a/llvm/unittests/ProfileData/CMakeLists.txt b/llvm/unittests/ProfileData/CMakeLists.txt
index ce3a0a45ccf1..c92642ded828 100644
--- a/llvm/unittests/ProfileData/CMakeLists.txt
+++ b/llvm/unittests/ProfileData/CMakeLists.txt
@@ -13,6 +13,7 @@ add_llvm_unittest(ProfileDataTests
   InstrProfTest.cpp
   ItaniumManglingCanonicalizerTest.cpp
   MemProfTest.cpp
+  PGOCtxProfReaderWriterTest.cpp
   SampleProfTest.cpp
   SymbolRemappingReaderTest.cpp
   )
diff --git a/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp b/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp
new file mode 100644
index 000000000000..d2cdbb28e2fc
--- /dev/null
+++ b/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp
@@ -0,0 +1,255 @@
+//===-------------- PGOCtxProfReadWriteTest.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Bitstream/BitstreamReader.h"
+#include "llvm/ProfileData/CtxInstrContextNode.h"
+#include "llvm/ProfileData/PGOCtxProfReader.h"
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Testing/Support/SupportHelpers.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::ctx_profile;
+
+class PGOCtxProfRWTest : public ::testing::Test {
+  std::vector<std::unique_ptr<char[]>> Nodes;
+  std::map<GUID, const ContextNode *> Roots;
+
+public:
+  ContextNode *createNode(GUID Guid, uint32_t NrCounters, uint32_t NrCallsites,
+                          ContextNode *Next = nullptr) {
+    auto AllocSize = ContextNode::getAllocSize(NrCounters, NrCallsites);
+    auto *Mem = Nodes.emplace_back(std::make_unique<char[]>(AllocSize)).get();
+    std::memset(Mem, 0, AllocSize);
+    auto *Ret = new (Mem) ContextNode(Guid, NrCounters, NrCallsites, Next);
+    return Ret;
+  }
+
+  void SetUp() override {
+    // Root (guid 1) has 2 callsites, one used for an indirect call to either
+    // guid 2 or 4.
+    // guid 2 calls guid 5
+    // guid 5 calls guid 2
+    // there's also a second root, guid3.
+    auto *Root1 = createNode(1, 2, 2);
+    Root1->counters()[0] = 10;
+    Root1->counters()[1] = 11;
+    Roots.insert({1, Root1});
+    auto *L1 = createNode(2, 1, 1);
+    L1->counters()[0] = 12;
+    Root1->subContexts()[1] = createNode(4, 3, 1, L1);
+    Root1->subContexts()[1]->counters()[0] = 13;
+    Root1->subContexts()[1]->counters()[1] = 14;
+    Root1->subContexts()[1]->counters()[2] = 15;
+
+    auto *L3 = createNode(5, 6, 3);
+    for (auto I = 0; I < 6; ++I)
+      L3->counters()[I] = 16 + I;
+    L1->subContexts()[0] = L3;
+    L3->subContexts()[2] = createNode(2, 1, 1);
+    L3->subContexts()[2]->counters()[0] = 30;
+    auto *Root2 = createNode(3, 1, 0);
+    Root2->counters()[0] = 40;
+    Roots.insert({3, Root2});
+  }
+
+  const std::map<GUID, const ContextNode *> &roots() const { return Roots; }
+};
+
+void checkSame(const ContextNode &Raw, const PGOContextualProfile &Profile) {
+  EXPECT_EQ(Raw.guid(), Profile.guid());
+  ASSERT_EQ(Raw.counters_size(), Profile.counters().size());
+  for (auto I = 0U; I < Raw.counters_size(); ++I)
+    EXPECT_EQ(Raw.counters()[I], Profile.counters()[I]);
+
+  for (auto I = 0U; I < Raw.callsites_size(); ++I) {
+    if (Raw.subContexts()[I] == nullptr)
+      continue;
+    EXPECT_TRUE(Profile.hasCallsite(I));
+    const auto &ProfileTargets = Profile.callsite(I);
+
+    std::map<GUID, const ContextNode *> Targets;
+    for (const auto *N = Raw.subContexts()[I]; N; N = N->next())
+      EXPECT_TRUE(Targets.insert({N->guid(), N}).second);
+
+    EXPECT_EQ(Targets.size(), ProfileTargets.size());
+    for (auto It : Targets) {
+      auto PIt = ProfileTargets.find(It.second->guid());
+      EXPECT_NE(PIt, ProfileTargets.end());
+      checkSame(*It.second, PIt->second);
+    }
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, RoundTrip) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      for (auto &[_, R] : roots())
+        Writer.write(*R);
+    }
+  }
+  {
+    ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
+        MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    ASSERT_TRUE(!!Expected);
+    auto &Ctxes = *Expected;
+    EXPECT_EQ(Ctxes.size(), roots().size());
+    EXPECT_EQ(Ctxes.size(), 2U);
+    for (auto &[G, R] : roots())
+      checkSame(*R, Ctxes.find(G)->second);
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, InvalidCounters) {
+  auto *R = createNode(1, 0, 1);
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      Writer.write(*R);
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, Empty) {
+  BitstreamCursor Cursor("");
+  PGOCtxProfileReader Reader(Cursor);
+  auto Expected = Reader.loadContexts();
+  EXPECT_FALSE(Expected);
+  consumeError(Expected.takeError());
+}
+
+TEST_F(PGOCtxProfRWTest, Invalid) {
+  BitstreamCursor Cursor("Surely this is not valid");
+  PGOCtxProfileReader Reader(Cursor);
+  auto Expected = Reader.loadContexts();
+  EXPECT_FALSE(Expected);
+  consumeError(Expected.takeError());
+}
+
+TEST_F(PGOCtxProfRWTest, ValidButEmpty) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      // don't write anything - this will just produce the metadata subblock.
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_TRUE(!!Expected);
+    EXPECT_TRUE(Expected->empty());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, WrongVersion) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out, PGOCtxProfileWriter::CurrentVersion + 1);
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, DuplicateRoots) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      Writer.write(*createNode(1, 1, 1));
+      Writer.write(*createNode(1, 1, 1));
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, DuplicateTargets) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      auto *R = createNode(1, 1, 1);
+      auto *L1 = createNode(2, 1, 0);
+      auto *L2 = createNode(2, 1, 0, L1);
+      R->subContexts()[0] = L2;
+      PGOCtxProfileWriter Writer(Out);
+      Writer.write(*R);
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
-- 
GitLab


From 03c7458a3603396d2d0e1dee43399d3d1664a264 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Tue, 14 May 2024 18:07:58 -0700
Subject: [PATCH 0793/1206] Revert "[ctx_profile] Profile reader and writer"
 (#92199)

Reverts llvm/llvm-project#91859

Buildbot failures.
---
 .../llvm/ProfileData/PGOCtxProfReader.h       |  92 -------
 .../llvm/ProfileData/PGOCtxProfWriter.h       |  91 -------
 llvm/lib/ProfileData/CMakeLists.txt           |   2 -
 llvm/lib/ProfileData/PGOCtxProfReader.cpp     | 173 ------------
 llvm/lib/ProfileData/PGOCtxProfWriter.cpp     |  49 ----
 llvm/unittests/ProfileData/CMakeLists.txt     |   1 -
 .../PGOCtxProfReaderWriterTest.cpp            | 255 ------------------
 7 files changed, 663 deletions(-)
 delete mode 100644 llvm/include/llvm/ProfileData/PGOCtxProfReader.h
 delete mode 100644 llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
 delete mode 100644 llvm/lib/ProfileData/PGOCtxProfReader.cpp
 delete mode 100644 llvm/lib/ProfileData/PGOCtxProfWriter.cpp
 delete mode 100644 llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp

diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
deleted file mode 100644
index a19b3f51d642..000000000000
--- a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
+++ /dev/null
@@ -1,92 +0,0 @@
-//===--- PGOCtxProfReader.h - Contextual profile reader ---------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-///
-/// Reader for contextual iFDO profile, which comes in bitstream format.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
-#define LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
-
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/Bitstream/BitstreamReader.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/ProfileData/PGOCtxProfWriter.h"
-#include "llvm/Support/Error.h"
-#include <map>
-#include <vector>
-
-namespace llvm {
-/// The loaded contextual profile, suitable for mutation during IPO passes. We
-/// generally expect a fraction of counters and of callsites to be populated.
-/// We continue to model counters as vectors, but callsites are modeled as a map
-/// of a map. The expectation is that, typically, there is a small number of
-/// indirect targets (usually, 1 for direct calls); but potentially a large
-/// number of callsites, and, as inlining progresses, the callsite count of a
-/// caller will grow.
-class PGOContextualProfile final {
-public:
-  using CallTargetMapTy = std::map<GlobalValue::GUID, PGOContextualProfile>;
-  using CallsiteMapTy = DenseMap<uint32_t, CallTargetMapTy>;
-
-private:
-  friend class PGOCtxProfileReader;
-  GlobalValue::GUID GUID = 0;
-  SmallVector<uint64_t, 16> Counters;
-  CallsiteMapTy Callsites;
-
-  PGOContextualProfile(GlobalValue::GUID G,
-                       SmallVectorImpl<uint64_t> &&Counters)
-      : GUID(G), Counters(std::move(Counters)) {}
-
-  Expected<PGOContextualProfile &>
-  getOrEmplace(uint32_t Index, GlobalValue::GUID G,
-               SmallVectorImpl<uint64_t> &&Counters);
-
-public:
-  PGOContextualProfile(const PGOContextualProfile &) = delete;
-  PGOContextualProfile &operator=(const PGOContextualProfile &) = delete;
-  PGOContextualProfile(PGOContextualProfile &&) = default;
-  PGOContextualProfile &operator=(PGOContextualProfile &&) = default;
-
-  GlobalValue::GUID guid() const { return GUID; }
-  const SmallVectorImpl<uint64_t> &counters() const { return Counters; }
-  const CallsiteMapTy &callsites() const { return Callsites; }
-  CallsiteMapTy &callsites() { return Callsites; }
-
-  bool hasCallsite(uint32_t I) const {
-    return Callsites.find(I) != Callsites.end();
-  }
-
-  const CallTargetMapTy &callsite(uint32_t I) const {
-    assert(hasCallsite(I) && "Callsite not found");
-    return Callsites.find(I)->second;
-  }
-  void getContainedGuids(DenseSet<GlobalValue::GUID> &Guids) const;
-};
-
-class PGOCtxProfileReader final {
-  BitstreamCursor &Cursor;
-  Expected<BitstreamEntry> advance();
-  Error readMetadata();
-  Error wrongValue(const Twine &);
-  Error unsupported(const Twine &);
-
-  Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
-  readContext(bool ExpectIndex);
-  bool canReadContext();
-
-public:
-  PGOCtxProfileReader(BitstreamCursor &Cursor) : Cursor(Cursor) {}
-
-  Expected<std::map<GlobalValue::GUID, PGOContextualProfile>> loadContexts();
-};
-} // namespace llvm
-#endif
diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
deleted file mode 100644
index 15578c51a495..000000000000
--- a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
+++ /dev/null
@@ -1,91 +0,0 @@
-//===- PGOCtxProfWriter.h - Contextual Profile Writer -----------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares a utility for writing a contextual profile to bitstream.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
-#define LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
-
-#include "llvm/Bitstream/BitstreamWriter.h"
-#include "llvm/ProfileData/CtxInstrContextNode.h"
-
-namespace llvm {
-enum PGOCtxProfileRecords { Invalid = 0, Version, Guid, CalleeIndex, Counters };
-
-enum PGOCtxProfileBlockIDs {
-  ProfileMetadataBlockID = 100,
-  ContextNodeBlockID = ProfileMetadataBlockID + 1
-};
-
-/// Write one or more ContextNodes to the provided raw_fd_stream.
-/// The caller must destroy the PGOCtxProfileWriter object before closing the
-/// stream.
-/// The design allows serializing a bunch of contexts embedded in some other
-/// file. The overall format is:
-///
-///  [... other data written to the stream...]
-///  SubBlock(ProfileMetadataBlockID)
-///   Version
-///   SubBlock(ContextNodeBlockID)
-///     [RECORDS]
-///     SubBlock(ContextNodeBlockID)
-///       [RECORDS]
-///       [... more SubBlocks]
-///     EndBlock
-///   EndBlock
-///
-/// The "RECORDS" are bitsream records. The IDs are in CtxProfileCodes (except)
-/// for Version, which is just for metadata). All contexts will have Guid and
-/// Counters, and all but the roots have CalleeIndex. The order in which the
-/// records appear does not matter, but they must precede any subcontexts,
-/// because that helps keep the reader code simpler.
-///
-/// Subblock containment captures the context->subcontext relationship. The
-/// "next()" relationship in the raw profile, between call targets of indirect
-/// calls, are just modeled as peer subblocks where the callee index is the
-/// same.
-///
-/// Versioning: the writer may produce additional records not known by the
-/// reader. The version number indicates a more structural change.
-/// The current version, in particular, is set up to expect optional extensions
-/// like value profiling - which would appear as additional records. For
-/// example, value profiling would produce a new record with a new record ID,
-/// containing the profiled values (much like the counters)
-class PGOCtxProfileWriter final {
-  SmallVector<char, 1 << 20> Buff;
-  BitstreamWriter Writer;
-
-  void writeCounters(const ctx_profile::ContextNode &Node);
-  void writeImpl(std::optional<uint32_t> CallerIndex,
-                 const ctx_profile::ContextNode &Node);
-
-public:
-  PGOCtxProfileWriter(raw_fd_stream &Out,
-                      std::optional<unsigned> VersionOverride = std::nullopt)
-      : Writer(Buff, &Out, 0) {
-    Writer.EnterSubblock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID,
-                         CodeLen);
-    const auto Version = VersionOverride ? *VersionOverride : CurrentVersion;
-    Writer.EmitRecord(PGOCtxProfileRecords::Version,
-                      SmallVector<unsigned, 1>({Version}));
-  }
-
-  ~PGOCtxProfileWriter() { Writer.ExitBlock(); }
-
-  void write(const ctx_profile::ContextNode &);
-
-  // constants used in writing which a reader may find useful.
-  static constexpr unsigned CodeLen = 2;
-  static constexpr uint32_t CurrentVersion = 1;
-  static constexpr unsigned VBREncodingBits = 6;
-};
-
-} // namespace llvm
-#endif
diff --git a/llvm/lib/ProfileData/CMakeLists.txt b/llvm/lib/ProfileData/CMakeLists.txt
index 2397eebaf7b1..408f9ff01ec8 100644
--- a/llvm/lib/ProfileData/CMakeLists.txt
+++ b/llvm/lib/ProfileData/CMakeLists.txt
@@ -7,8 +7,6 @@ add_llvm_component_library(LLVMProfileData
   ItaniumManglingCanonicalizer.cpp
   MemProf.cpp
   MemProfReader.cpp
-  PGOCtxProfReader.cpp
-  PGOCtxProfWriter.cpp
   ProfileSummaryBuilder.cpp
   SampleProf.cpp
   SampleProfReader.cpp
diff --git a/llvm/lib/ProfileData/PGOCtxProfReader.cpp b/llvm/lib/ProfileData/PGOCtxProfReader.cpp
deleted file mode 100644
index 3710f2e4b818..000000000000
--- a/llvm/lib/ProfileData/PGOCtxProfReader.cpp
+++ /dev/null
@@ -1,173 +0,0 @@
-//===- PGOCtxProfReader.cpp - Contextual Instrumentation profile reader ---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Read a contextual profile into a datastructure suitable for maintenance
-// throughout IPO
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ProfileData/PGOCtxProfReader.h"
-#include "llvm/Bitstream/BitCodeEnums.h"
-#include "llvm/Bitstream/BitstreamReader.h"
-#include "llvm/ProfileData/InstrProf.h"
-#include "llvm/ProfileData/PGOCtxProfWriter.h"
-#include "llvm/Support/Errc.h"
-#include "llvm/Support/Error.h"
-
-using namespace llvm;
-
-// FIXME(#92054) - these Error handling macros are (re-)invented in a few
-// places.
-#define EXPECT_OR_RET(LHS, RHS)                                                \
-  auto LHS = RHS;                                                              \
-  if (!LHS)                                                                    \
-    return LHS.takeError();
-
-#define RET_ON_ERR(EXPR)                                                       \
-  if (auto Err = (EXPR))                                                       \
-    return Err;
-
-Expected<PGOContextualProfile &>
-PGOContextualProfile::getOrEmplace(uint32_t Index, GlobalValue::GUID G,
-                                   SmallVectorImpl<uint64_t> &&Counters) {
-  auto [Iter, Inserted] = Callsites[Index].insert(
-      {G, PGOContextualProfile(G, std::move(Counters))});
-  if (!Inserted)
-    return make_error<InstrProfError>(instrprof_error::invalid_prof,
-                                      "Duplicate GUID for same callsite.");
-  return Iter->second;
-}
-
-void PGOContextualProfile::getContainedGuids(
-    DenseSet<GlobalValue::GUID> &Guids) const {
-  Guids.insert(GUID);
-  for (const auto &[_, Callsite] : Callsites)
-    for (const auto &[_, Callee] : Callsite)
-      Callee.getContainedGuids(Guids);
-}
-
-Expected<BitstreamEntry> PGOCtxProfileReader::advance() {
-  return Cursor.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs);
-}
-
-Error PGOCtxProfileReader::wrongValue(const Twine &Msg) {
-  return make_error<InstrProfError>(instrprof_error::invalid_prof, Msg);
-}
-
-Error PGOCtxProfileReader::unsupported(const Twine &Msg) {
-  return make_error<InstrProfError>(instrprof_error::unsupported_version, Msg);
-}
-
-bool PGOCtxProfileReader::canReadContext() {
-  auto Blk = advance();
-  if (!Blk) {
-    consumeError(Blk.takeError());
-    return false;
-  }
-  return Blk->Kind == BitstreamEntry::SubBlock &&
-         Blk->ID == PGOCtxProfileBlockIDs::ContextNodeBlockID;
-}
-
-Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
-PGOCtxProfileReader::readContext(bool ExpectIndex) {
-  RET_ON_ERR(Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID));
-
-  std::optional<ctx_profile::GUID> Guid;
-  std::optional<SmallVector<uint64_t, 16>> Counters;
-  std::optional<uint32_t> CallsiteIndex;
-
-  SmallVector<uint64_t, 1> RecordValues;
-
-  // We don't prescribe the order in which the records come in, and we are ok
-  // if other unsupported records appear. We seek in the current subblock until
-  // we get all we know.
-  auto GotAllWeNeed = [&]() {
-    return Guid.has_value() && Counters.has_value() &&
-           (!ExpectIndex || CallsiteIndex.has_value());
-  };
-  while (!GotAllWeNeed()) {
-    RecordValues.clear();
-    EXPECT_OR_RET(Entry, advance());
-    if (Entry->Kind != BitstreamEntry::Record)
-      return wrongValue(
-          "Expected records before encountering more subcontexts");
-    EXPECT_OR_RET(ReadRecord,
-                  Cursor.readRecord(bitc::UNABBREV_RECORD, RecordValues));
-    switch (*ReadRecord) {
-    case PGOCtxProfileRecords::Guid:
-      if (RecordValues.size() != 1)
-        return wrongValue("The GUID record should have exactly one value");
-      Guid = RecordValues[0];
-      break;
-    case PGOCtxProfileRecords::Counters:
-      Counters = std::move(RecordValues);
-      if (Counters->empty())
-        return wrongValue("Empty counters. At least the entry counter (one "
-                          "value) was expected");
-      break;
-    case PGOCtxProfileRecords::CalleeIndex:
-      if (!ExpectIndex)
-        return wrongValue("The root context should not have a callee index");
-      if (RecordValues.size() != 1)
-        return wrongValue("The callee index should have exactly one value");
-      CallsiteIndex = RecordValues[0];
-      break;
-    default:
-      // OK if we see records we do not understand, like records (profile
-      // components) introduced later.
-      break;
-    }
-  }
-
-  PGOContextualProfile Ret(*Guid, std::move(*Counters));
-
-  while (canReadContext()) {
-    EXPECT_OR_RET(SC, readContext(true));
-    auto &Targets = Ret.callsites()[*SC->first];
-    auto [_, Inserted] =
-        Targets.insert({SC->second.guid(), std::move(SC->second)});
-    if (!Inserted)
-      return wrongValue(
-          "Unexpected duplicate target (callee) at the same callsite.");
-  }
-  return std::make_pair(CallsiteIndex, std::move(Ret));
-}
-
-Error PGOCtxProfileReader::readMetadata() {
-  EXPECT_OR_RET(Blk, advance());
-  if (Blk->Kind != BitstreamEntry::SubBlock)
-    return unsupported("Expected Version record");
-  RET_ON_ERR(
-      Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID));
-  EXPECT_OR_RET(MData, advance());
-  if (MData->Kind != BitstreamEntry::Record)
-    return unsupported("Expected Version record");
-
-  SmallVector<uint64_t, 1> Ver;
-  EXPECT_OR_RET(Code, Cursor.readRecord(bitc::UNABBREV_RECORD, Ver));
-  if (*Code != PGOCtxProfileRecords::Version)
-    return unsupported("Expected Version record");
-  if (Ver.size() != 1 || Ver[0] > PGOCtxProfileWriter::CurrentVersion)
-    return unsupported("Version " + Twine(*Code) +
-                       " is higher than supported version " +
-                       Twine(PGOCtxProfileWriter::CurrentVersion));
-  return Error::success();
-}
-
-Expected<std::map<GlobalValue::GUID, PGOContextualProfile>>
-PGOCtxProfileReader::loadContexts() {
-  std::map<GlobalValue::GUID, PGOContextualProfile> Ret;
-  RET_ON_ERR(readMetadata());
-  while (canReadContext()) {
-    EXPECT_OR_RET(E, readContext(false));
-    auto Key = E->second.guid();
-    if (!Ret.insert({Key, std::move(E->second)}).second)
-      return wrongValue("Duplicate roots");
-  }
-  return Ret;
-}
diff --git a/llvm/lib/ProfileData/PGOCtxProfWriter.cpp b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
deleted file mode 100644
index 508179756446..000000000000
--- a/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
+++ /dev/null
@@ -1,49 +0,0 @@
-//===- PGOCtxProfWriter.cpp - Contextual Instrumentation profile writer ---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Write a contextual profile to bitstream.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ProfileData/PGOCtxProfWriter.h"
-#include "llvm/Bitstream/BitCodeEnums.h"
-
-using namespace llvm;
-using namespace llvm::ctx_profile;
-
-void PGOCtxProfileWriter::writeCounters(const ContextNode &Node) {
-  Writer.EmitCode(bitc::UNABBREV_RECORD);
-  Writer.EmitVBR(PGOCtxProfileRecords::Counters, VBREncodingBits);
-  Writer.EmitVBR(Node.counters_size(), VBREncodingBits);
-  for (uint32_t I = 0U; I < Node.counters_size(); ++I)
-    Writer.EmitVBR64(Node.counters()[I], VBREncodingBits);
-}
-
-// recursively write all the subcontexts. We do need to traverse depth first to
-// model the context->subcontext implicitly, and since this captures call
-// stacks, we don't really need to be worried about stack overflow and we can
-// keep the implementation simple.
-void PGOCtxProfileWriter::writeImpl(std::optional<uint32_t> CallerIndex,
-                                    const ContextNode &Node) {
-  Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextNodeBlockID, CodeLen);
-  Writer.EmitRecord(PGOCtxProfileRecords::Guid,
-                    SmallVector<uint64_t, 1>{Node.guid()});
-  if (CallerIndex)
-    Writer.EmitRecord(PGOCtxProfileRecords::CalleeIndex,
-                      SmallVector<uint64_t, 1>{*CallerIndex});
-  writeCounters(Node);
-  for (uint32_t I = 0U; I < Node.callsites_size(); ++I)
-    for (const auto *Subcontext = Node.subContexts()[I]; Subcontext;
-         Subcontext = Subcontext->next())
-      writeImpl(I, *Subcontext);
-  Writer.ExitBlock();
-}
-
-void PGOCtxProfileWriter::write(const ContextNode &RootNode) {
-  writeImpl(std::nullopt, RootNode);
-}
diff --git a/llvm/unittests/ProfileData/CMakeLists.txt b/llvm/unittests/ProfileData/CMakeLists.txt
index c92642ded828..ce3a0a45ccf1 100644
--- a/llvm/unittests/ProfileData/CMakeLists.txt
+++ b/llvm/unittests/ProfileData/CMakeLists.txt
@@ -13,7 +13,6 @@ add_llvm_unittest(ProfileDataTests
   InstrProfTest.cpp
   ItaniumManglingCanonicalizerTest.cpp
   MemProfTest.cpp
-  PGOCtxProfReaderWriterTest.cpp
   SampleProfTest.cpp
   SymbolRemappingReaderTest.cpp
   )
diff --git a/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp b/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp
deleted file mode 100644
index d2cdbb28e2fc..000000000000
--- a/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp
+++ /dev/null
@@ -1,255 +0,0 @@
-//===-------------- PGOCtxProfReadWriteTest.cpp ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitstream/BitstreamReader.h"
-#include "llvm/ProfileData/CtxInstrContextNode.h"
-#include "llvm/ProfileData/PGOCtxProfReader.h"
-#include "llvm/ProfileData/PGOCtxProfWriter.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Testing/Support/SupportHelpers.h"
-#include "gtest/gtest.h"
-
-using namespace llvm;
-using namespace llvm::ctx_profile;
-
-class PGOCtxProfRWTest : public ::testing::Test {
-  std::vector<std::unique_ptr<char[]>> Nodes;
-  std::map<GUID, const ContextNode *> Roots;
-
-public:
-  ContextNode *createNode(GUID Guid, uint32_t NrCounters, uint32_t NrCallsites,
-                          ContextNode *Next = nullptr) {
-    auto AllocSize = ContextNode::getAllocSize(NrCounters, NrCallsites);
-    auto *Mem = Nodes.emplace_back(std::make_unique<char[]>(AllocSize)).get();
-    std::memset(Mem, 0, AllocSize);
-    auto *Ret = new (Mem) ContextNode(Guid, NrCounters, NrCallsites, Next);
-    return Ret;
-  }
-
-  void SetUp() override {
-    // Root (guid 1) has 2 callsites, one used for an indirect call to either
-    // guid 2 or 4.
-    // guid 2 calls guid 5
-    // guid 5 calls guid 2
-    // there's also a second root, guid3.
-    auto *Root1 = createNode(1, 2, 2);
-    Root1->counters()[0] = 10;
-    Root1->counters()[1] = 11;
-    Roots.insert({1, Root1});
-    auto *L1 = createNode(2, 1, 1);
-    L1->counters()[0] = 12;
-    Root1->subContexts()[1] = createNode(4, 3, 1, L1);
-    Root1->subContexts()[1]->counters()[0] = 13;
-    Root1->subContexts()[1]->counters()[1] = 14;
-    Root1->subContexts()[1]->counters()[2] = 15;
-
-    auto *L3 = createNode(5, 6, 3);
-    for (auto I = 0; I < 6; ++I)
-      L3->counters()[I] = 16 + I;
-    L1->subContexts()[0] = L3;
-    L3->subContexts()[2] = createNode(2, 1, 1);
-    L3->subContexts()[2]->counters()[0] = 30;
-    auto *Root2 = createNode(3, 1, 0);
-    Root2->counters()[0] = 40;
-    Roots.insert({3, Root2});
-  }
-
-  const std::map<GUID, const ContextNode *> &roots() const { return Roots; }
-};
-
-void checkSame(const ContextNode &Raw, const PGOContextualProfile &Profile) {
-  EXPECT_EQ(Raw.guid(), Profile.guid());
-  ASSERT_EQ(Raw.counters_size(), Profile.counters().size());
-  for (auto I = 0U; I < Raw.counters_size(); ++I)
-    EXPECT_EQ(Raw.counters()[I], Profile.counters()[I]);
-
-  for (auto I = 0U; I < Raw.callsites_size(); ++I) {
-    if (Raw.subContexts()[I] == nullptr)
-      continue;
-    EXPECT_TRUE(Profile.hasCallsite(I));
-    const auto &ProfileTargets = Profile.callsite(I);
-
-    std::map<GUID, const ContextNode *> Targets;
-    for (const auto *N = Raw.subContexts()[I]; N; N = N->next())
-      EXPECT_TRUE(Targets.insert({N->guid(), N}).second);
-
-    EXPECT_EQ(Targets.size(), ProfileTargets.size());
-    for (auto It : Targets) {
-      auto PIt = ProfileTargets.find(It.second->guid());
-      EXPECT_NE(PIt, ProfileTargets.end());
-      checkSame(*It.second, PIt->second);
-    }
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, RoundTrip) {
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      PGOCtxProfileWriter Writer(Out);
-      for (auto &[_, R] : roots())
-        Writer.write(*R);
-    }
-  }
-  {
-    ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
-        MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    ASSERT_TRUE(!!Expected);
-    auto &Ctxes = *Expected;
-    EXPECT_EQ(Ctxes.size(), roots().size());
-    EXPECT_EQ(Ctxes.size(), 2U);
-    for (auto &[G, R] : roots())
-      checkSame(*R, Ctxes.find(G)->second);
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, InvalidCounters) {
-  auto *R = createNode(1, 0, 1);
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      PGOCtxProfileWriter Writer(Out);
-      Writer.write(*R);
-    }
-  }
-  {
-    auto MB = MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    EXPECT_FALSE(Expected);
-    consumeError(Expected.takeError());
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, Empty) {
-  BitstreamCursor Cursor("");
-  PGOCtxProfileReader Reader(Cursor);
-  auto Expected = Reader.loadContexts();
-  EXPECT_FALSE(Expected);
-  consumeError(Expected.takeError());
-}
-
-TEST_F(PGOCtxProfRWTest, Invalid) {
-  BitstreamCursor Cursor("Surely this is not valid");
-  PGOCtxProfileReader Reader(Cursor);
-  auto Expected = Reader.loadContexts();
-  EXPECT_FALSE(Expected);
-  consumeError(Expected.takeError());
-}
-
-TEST_F(PGOCtxProfRWTest, ValidButEmpty) {
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      PGOCtxProfileWriter Writer(Out);
-      // don't write anything - this will just produce the metadata subblock.
-    }
-  }
-  {
-    auto MB = MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    EXPECT_TRUE(!!Expected);
-    EXPECT_TRUE(Expected->empty());
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, WrongVersion) {
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      PGOCtxProfileWriter Writer(Out, PGOCtxProfileWriter::CurrentVersion + 1);
-    }
-  }
-  {
-    auto MB = MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    EXPECT_FALSE(Expected);
-    consumeError(Expected.takeError());
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, DuplicateRoots) {
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      PGOCtxProfileWriter Writer(Out);
-      Writer.write(*createNode(1, 1, 1));
-      Writer.write(*createNode(1, 1, 1));
-    }
-  }
-  {
-    auto MB = MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    EXPECT_FALSE(Expected);
-    consumeError(Expected.takeError());
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, DuplicateTargets) {
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      auto *R = createNode(1, 1, 1);
-      auto *L1 = createNode(2, 1, 0);
-      auto *L2 = createNode(2, 1, 0, L1);
-      R->subContexts()[0] = L2;
-      PGOCtxProfileWriter Writer(Out);
-      Writer.write(*R);
-    }
-  }
-  {
-    auto MB = MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    EXPECT_FALSE(Expected);
-    consumeError(Expected.takeError());
-  }
-}
-- 
GitLab


From 4d1ecf192313b612090d60181937eff03c1a966b Mon Sep 17 00:00:00 2001
From: AtariDreams <gfunni234@gmail.com>
Date: Tue, 14 May 2024 21:26:23 -0400
Subject: [PATCH 0794/1206] [Transforms] Preserve inbounds attribute of
 transformed GEPs when flattening loops (#86961)

When flattening the loop, if the GEP was inbound, it should stay
inbound, because the only thing that changed is how the pointers are
calculated, not the elements being accessed.

Proof: https://alive2.llvm.org/ce/z/dApMpQ
---
 llvm/lib/Transforms/Scalar/LoopFlatten.cpp           | 6 ++++--
 llvm/test/Transforms/LoopFlatten/loop-flatten-gep.ll | 6 +++---
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
index a7f8a22ece27..bb9632ff73a4 100644
--- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
@@ -808,8 +808,10 @@ static bool DoFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
       // we need to insert the new GEP where the old GEP was.
       if (!DT->dominates(Base, &*Builder.GetInsertPoint()))
         Builder.SetInsertPoint(cast<Instruction>(V));
-      OuterValue = Builder.CreateGEP(GEP->getSourceElementType(), Base,
-                                     OuterValue, "flatten." + V->getName());
+      OuterValue =
+          Builder.CreateGEP(GEP->getSourceElementType(), Base, OuterValue,
+                            "flatten." + V->getName(),
+                            GEP->isInBounds() && InnerGEP->isInBounds());
     }
 
     LLVM_DEBUG(dbgs() << "Replacing: "; V->dump(); dbgs() << "with:      ";
diff --git a/llvm/test/Transforms/LoopFlatten/loop-flatten-gep.ll b/llvm/test/Transforms/LoopFlatten/loop-flatten-gep.ll
index f4b8ea97237f..e30001670b1e 100644
--- a/llvm/test/Transforms/LoopFlatten/loop-flatten-gep.ll
+++ b/llvm/test/Transforms/LoopFlatten/loop-flatten-gep.ll
@@ -15,7 +15,7 @@ for.outer.preheader:
   br label %for.inner.preheader
 
 ; CHECK-LABEL: for.inner.preheader:
-; CHECK: %flatten.arrayidx = getelementptr i32, ptr %A, i32 %i
+; CHECK: %flatten.arrayidx = getelementptr inbounds i32, ptr %A, i32 %i
 for.inner.preheader:
   %i = phi i32 [ 0, %for.outer.preheader ], [ %inc2, %for.outer ]
   br label %for.inner
@@ -61,13 +61,13 @@ for.outer.preheader:
   br label %for.inner.preheader
 
 ; CHECK-LABEL: for.inner.preheader:
-; CHECK-NOT: getelementptr i32, ptr %ptr, i32 %i
+; CHECK-NOT: getelementptr inbounds i32, ptr %ptr, i32 %i
 for.inner.preheader:
   %i = phi i32 [ 0, %for.outer.preheader ], [ %inc2, %for.outer ]
   br label %for.inner
 
 ; CHECK-LABEL: for.inner:
-; CHECK: %flatten.arrayidx = getelementptr i32, ptr %ptr, i32 %i
+; CHECK: %flatten.arrayidx = getelementptr inbounds i32, ptr %ptr, i32 %i
 ; CHECK: store i32 0, ptr %flatten.arrayidx, align 4
 ; CHECK: br label %for.outer
 for.inner:
-- 
GitLab


From 71fbbb69d63c461f391cbabf1e32cd9977c4ce68 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Wed, 15 May 2024 10:13:42 +0900
Subject: [PATCH 0795/1206] [IR] Move GlobalValue::getGUID() out of line (NFC)

Avoid including MD5.h in a core IR header.
---
 llvm/include/llvm/IR/GlobalValue.h                         | 3 +--
 llvm/lib/IR/Globals.cpp                                    | 5 +++++
 llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.cpp            | 1 +
 llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp | 1 +
 llvm/lib/Transforms/Utils/ModuleUtils.cpp                  | 1 +
 5 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/IR/GlobalValue.h b/llvm/include/llvm/IR/GlobalValue.h
index c61d502aa332..b1262b27f022 100644
--- a/llvm/include/llvm/IR/GlobalValue.h
+++ b/llvm/include/llvm/IR/GlobalValue.h
@@ -24,7 +24,6 @@
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MD5.h"
 #include <cassert>
 #include <cstdint>
 #include <string>
@@ -588,7 +587,7 @@ public:
 
   /// Return a 64-bit global unique ID constructed from global value name
   /// (i.e. returned by getGlobalIdentifier()).
-  static GUID getGUID(StringRef GlobalName) { return MD5Hash(GlobalName); }
+  static GUID getGUID(StringRef GlobalName);
 
   /// Return a 64-bit global unique ID constructed from global value name
   /// (i.e. returned by getGlobalIdentifier()).
diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp
index 40f854a2c906..6f071847bb58 100644
--- a/llvm/lib/IR/Globals.cpp
+++ b/llvm/lib/IR/Globals.cpp
@@ -21,6 +21,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MD5.h"
 #include "llvm/TargetParser/Triple.h"
 using namespace llvm;
 
@@ -71,6 +72,10 @@ void GlobalValue::copyAttributesFrom(const GlobalValue *Src) {
     removeSanitizerMetadata();
 }
 
+GlobalValue::GUID GlobalValue::getGUID(StringRef GlobalName) {
+  return MD5Hash(GlobalName);
+}
+
 void GlobalValue::removeFromParent() {
   switch (getValueID()) {
 #define HANDLE_GLOBAL_VALUE(NAME)                                              \
diff --git a/llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.cpp
index f77a1f0272c8..f940dc05948b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.cpp
@@ -22,6 +22,7 @@
 #include "llvm/IR/Value.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MD5.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 
 using namespace llvm;
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index fca1824165e7..8d39217992c7 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -53,6 +53,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/MD5.h"
 #include "llvm/Support/RandomNumberGenerator.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
diff --git a/llvm/lib/Transforms/Utils/ModuleUtils.cpp b/llvm/lib/Transforms/Utils/ModuleUtils.cpp
index 209a6a34a3c9..122279160cc7 100644
--- a/llvm/lib/Transforms/Utils/ModuleUtils.cpp
+++ b/llvm/lib/Transforms/Utils/ModuleUtils.cpp
@@ -18,6 +18,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/MD5.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/xxhash.h"
 
-- 
GitLab


From 6642cc60a21f857bddde8ec2c81008a83a54b4d3 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Wed, 15 May 2024 11:07:40 +0900
Subject: [PATCH 0796/1206] [SCEV] Add tests for ule/sle exit counts (NFC)

---
 .../ScalarEvolution/exit-count-non-strict.ll  | 228 ++++++++++++++++++
 1 file changed, 228 insertions(+)
 create mode 100644 llvm/test/Analysis/ScalarEvolution/exit-count-non-strict.ll

diff --git a/llvm/test/Analysis/ScalarEvolution/exit-count-non-strict.ll b/llvm/test/Analysis/ScalarEvolution/exit-count-non-strict.ll
new file mode 100644
index 000000000000..2117c779f4b3
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/exit-count-non-strict.ll
@@ -0,0 +1,228 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -disable-output "-passes=print<scalar-evolution>" -scalar-evolution-classify-expressions=0 < %s 2>&1 | FileCheck %s
+
+define void @ule_from_zero(i32 %M, i32 %N) {
+; CHECK-LABEL: 'ule_from_zero'
+; CHECK-NEXT:  Determining loop execution counts for: @ule_from_zero
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for latch: %N
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is %N
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for latch: %N
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp ule i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add nuw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @ule_from_one(i32 %M, i32 %N) {
+; CHECK-LABEL: 'ule_from_one'
+; CHECK-NEXT:  Determining loop execution counts for: @ule_from_one
+; CHECK-NEXT:  Loop %loop: <multiple exits> backedge-taken count is (%M umin_seq (-1 + %N))
+; CHECK-NEXT:    exit count for loop: %M
+; CHECK-NEXT:    exit count for latch: (-1 + %N)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (%M umin_seq (-1 + %N))
+; CHECK-NEXT:    symbolic max exit count for loop: %M
+; CHECK-NEXT:    symbolic max exit count for latch: (-1 + %N)
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 1, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp ule i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add nuw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @ule_from_unknown(i32 %M, i32 %N, i32 %S) {
+; CHECK-LABEL: 'ule_from_unknown'
+; CHECK-NEXT:  Determining loop execution counts for: @ule_from_unknown
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for latch: ((-1 * %S) + %N)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((-1 * %S) + %N)
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for latch: ((-1 * %S) + %N)
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ %S, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp ule i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add nuw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @ule_from_zero_no_nuw(i32 %M, i32 %N) {
+; CHECK-LABEL: 'ule_from_zero_no_nuw'
+; CHECK-NEXT:  Determining loop execution counts for: @ule_from_zero_no_nuw
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for latch: %N
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is %N
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for latch: %N
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp ule i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @sle_from_int_min(i32 %M, i32 %N) {
+; CHECK-LABEL: 'sle_from_int_min'
+; CHECK-NEXT:  Determining loop execution counts for: @sle_from_int_min
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for latch: (-2147483648 + %N)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (-2147483648 + %N)
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for latch: (-2147483648 + %N)
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ u0x80000000, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp sle i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add nsw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @sle_from_int_min_plus_one(i32 %M, i32 %N) {
+; CHECK-LABEL: 'sle_from_int_min_plus_one'
+; CHECK-NEXT:  Determining loop execution counts for: @sle_from_int_min_plus_one
+; CHECK-NEXT:  Loop %loop: <multiple exits> backedge-taken count is ((-2147483648 + %M) umin_seq (2147483647 + %N))
+; CHECK-NEXT:    exit count for loop: (-2147483648 + %M)
+; CHECK-NEXT:    exit count for latch: (2147483647 + %N)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((-2147483648 + %M) umin_seq (2147483647 + %N))
+; CHECK-NEXT:    symbolic max exit count for loop: (-2147483648 + %M)
+; CHECK-NEXT:    symbolic max exit count for latch: (2147483647 + %N)
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ u0x80000001, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp sle i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add nsw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @sle_from_unknown(i32 %M, i32 %N, i32 %S) {
+; CHECK-LABEL: 'sle_from_unknown'
+; CHECK-NEXT:  Determining loop execution counts for: @sle_from_unknown
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for latch: ((-1 * %S) + %N)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((-1 * %S) + %N)
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for latch: ((-1 * %S) + %N)
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ %S, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp sle i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add nsw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @sle_from_int_min_no_nsw(i32 %M, i32 %N) {
+; CHECK-LABEL: 'sle_from_int_min_no_nsw'
+; CHECK-NEXT:  Determining loop execution counts for: @sle_from_int_min_no_nsw
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for latch: (-2147483648 + %N)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (-2147483648 + %N)
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for latch: (-2147483648 + %N)
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ u0x80000000, %entry ], [ %iv.next, %latch ]
+  %cmp1 = icmp sle i32 %iv, %M
+  br i1 %cmp1, label %latch, label %exit
+
+latch:
+  %iv.next = add i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, %N
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
-- 
GitLab


From 11b059145d177ee287c7ada9864addf8d083c160 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Tue, 14 May 2024 17:53:17 +0800
Subject: [PATCH 0797/1206] [Serialization] Read the initializer for
 interesting static variables before consuming it

Close https://github.com/llvm/llvm-project/issues/91418

Since we load the variable's initializers lazily, it'd be problematic
if the initializers dependent on each other. So here we try to load the
initializers of static variables to make sure they are passed to code
generator by order. If we read any thing interesting, we would consume
that before emitting the current declaration.
---
 clang/lib/Serialization/ASTReaderDecl.cpp | 29 +++++++++-
 clang/test/Modules/pr91418.cppm           | 67 +++++++++++++++++++++++
 2 files changed, 93 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/Modules/pr91418.cppm

diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 0c647086e304..a6254b70560c 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -4186,12 +4186,35 @@ void ASTReader::PassInterestingDeclsToConsumer() {
     GetDecl(ID);
   EagerlyDeserializedDecls.clear();
 
-  while (!PotentiallyInterestingDecls.empty()) {
-    Decl *D = PotentiallyInterestingDecls.front();
-    PotentiallyInterestingDecls.pop_front();
+  auto ConsumingPotentialInterestingDecls = [this]() {
+    while (!PotentiallyInterestingDecls.empty()) {
+      Decl *D = PotentiallyInterestingDecls.front();
+      PotentiallyInterestingDecls.pop_front();
+      if (isConsumerInterestedIn(D))
+        PassInterestingDeclToConsumer(D);
+    }
+  };
+  std::deque<Decl *> MaybeInterestingDecls =
+      std::move(PotentiallyInterestingDecls);
+  assert(PotentiallyInterestingDecls.empty());
+  while (!MaybeInterestingDecls.empty()) {
+    Decl *D = MaybeInterestingDecls.front();
+    MaybeInterestingDecls.pop_front();
+    // Since we load the variable's initializers lazily, it'd be problematic
+    // if the initializers dependent on each other. So here we try to load the
+    // initializers of static variables to make sure they are passed to code
+    // generator by order. If we read anything interesting, we would consume
+    // that before emitting the current declaration.
+    if (auto *VD = dyn_cast<VarDecl>(D);
+        VD && VD->isFileVarDecl() && !VD->isExternallyVisible())
+      VD->getInit();
+    ConsumingPotentialInterestingDecls();
     if (isConsumerInterestedIn(D))
       PassInterestingDeclToConsumer(D);
   }
+
+  // If we add any new potential interesting decl in the last call, consume it.
+  ConsumingPotentialInterestingDecls();
 }
 
 void ASTReader::loadDeclUpdateRecords(PendingUpdateRecord &Record) {
diff --git a/clang/test/Modules/pr91418.cppm b/clang/test/Modules/pr91418.cppm
new file mode 100644
index 000000000000..33fec992439d
--- /dev/null
+++ b/clang/test/Modules/pr91418.cppm
@@ -0,0 +1,67 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 -x c++-header %t/foo.h \
+// RUN:     -emit-pch -o %t/foo.pch
+// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/use.cpp -include-pch \
+// RUN:     %t/foo.pch -emit-llvm -o - | FileCheck %t/use.cpp
+
+//--- foo.h
+#ifndef FOO_H
+#define FOO_H
+typedef float __m128 __attribute__((__vector_size__(16), __aligned__(16)));
+
+static __inline__ __m128 __attribute__((__always_inline__, __min_vector_width__(128)))
+_mm_setr_ps(float __z, float __y, float __x, float __w)
+{
+  return __extension__ (__m128){ __z, __y, __x, __w };
+}
+
+typedef __m128 VR;
+
+inline VR MakeVR( float X, float Y, float Z, float W )
+{
+ return _mm_setr_ps( X, Y, Z, W );
+}
+
+extern "C" float sqrtf(float);
+
+namespace VectorSinConstantsSSE
+{
+  float a = (16 * sqrtf(0.225f));
+  VR A = MakeVR(a, a, a, a);
+  static const float b = (16 * sqrtf(0.225f));
+  static const VR B = MakeVR(b, b, b, b);
+}
+
+#endif // FOO_H
+
+//--- use.cpp
+#include "foo.h"
+float use() {
+    return VectorSinConstantsSSE::A[0] + VectorSinConstantsSSE::A[1] +
+           VectorSinConstantsSSE::A[2] + VectorSinConstantsSSE::A[3] +
+           VectorSinConstantsSSE::B[0] + VectorSinConstantsSSE::B[1] +
+           VectorSinConstantsSSE::B[2] + VectorSinConstantsSSE::B[3];
+}
+
+// CHECK: define{{.*}}@__cxx_global_var_init(
+// CHECK: store{{.*}}[[a_RESULT:%[a-zA-Z0-9]+]], ptr @_ZN21VectorSinConstantsSSE1aE
+
+// CHECK: define{{.*}}@__cxx_global_var_init.1(
+// CHECK: [[A_CALL:%[a-zA-Z0-9]+]] = call{{.*}}@_Z6MakeVRffff(
+// CHECK: store{{.*}}[[A_CALL]], ptr @_ZN21VectorSinConstantsSSE1AE
+
+// CHECK: define{{.*}}@__cxx_global_var_init.2(
+// CHECK: [[B_CALL:%[a-zA-Z0-9]+]] = call{{.*}}@_Z6MakeVRffff(
+// CHECK: store{{.*}}[[B_CALL]], ptr @_ZN21VectorSinConstantsSSEL1BE
+
+// CHECK: define{{.*}}@__cxx_global_var_init.3(
+// CHECK: store{{.*}}[[b_RESULT:%[a-zA-Z0-9]+]], ptr @_ZN21VectorSinConstantsSSEL1bE
+
+// CHECK: @_GLOBAL__sub_I_use.cpp
+// CHECK: call{{.*}}@__cxx_global_var_init(
+// CHECK: call{{.*}}@__cxx_global_var_init.1(
+// CHECK: call{{.*}}@__cxx_global_var_init.3(
+// CHECK: call{{.*}}@__cxx_global_var_init.2(
-- 
GitLab


From 13b265c7b5c6a989427639e33893c158f737480b Mon Sep 17 00:00:00 2001
From: Freddy Ye <freddy.ye@intel.com>
Date: Wed, 15 May 2024 10:40:16 +0800
Subject: [PATCH 0798/1206] [X86][MC] Support Intel FRED and LKGS instructions.
 (#91909)

Spec reference: https://cdrdv2.intel.com/v1/dl/getContent/678938
---
 llvm/lib/Target/X86/X86InstrSystem.td  | 16 +++++++++++
 llvm/test/MC/Disassembler/X86/fred.txt |  8 ++++++
 llvm/test/MC/Disassembler/X86/lkgs.txt | 35 ++++++++++++++++++++++++
 llvm/test/MC/X86/fred-att.s            | 14 ++++++++++
 llvm/test/MC/X86/fred-intel.s          | 10 +++++++
 llvm/test/MC/X86/lkgs-att.s            | 38 ++++++++++++++++++++++++++
 llvm/test/MC/X86/lkgs-intel.s          | 34 +++++++++++++++++++++++
 llvm/test/TableGen/x86-fold-tables.inc |  1 +
 8 files changed, 156 insertions(+)
 create mode 100644 llvm/test/MC/Disassembler/X86/fred.txt
 create mode 100644 llvm/test/MC/Disassembler/X86/lkgs.txt
 create mode 100644 llvm/test/MC/X86/fred-att.s
 create mode 100644 llvm/test/MC/X86/fred-intel.s
 create mode 100644 llvm/test/MC/X86/lkgs-att.s
 create mode 100644 llvm/test/MC/X86/lkgs-intel.s

diff --git a/llvm/lib/Target/X86/X86InstrSystem.td b/llvm/lib/Target/X86/X86InstrSystem.td
index 56293e20567e..e1573b37d4dc 100644
--- a/llvm/lib/Target/X86/X86InstrSystem.td
+++ b/llvm/lib/Target/X86/X86InstrSystem.td
@@ -68,6 +68,14 @@ def SYSENTER : I<0x34, RawFrm, (outs), (ins), "sysenter", []>, TB;
 def SYSEXIT   : I<0x35, RawFrm, (outs), (ins), "sysexit{l}", []>, TB;
 def SYSEXIT64 :RI<0x35, RawFrm, (outs), (ins), "sysexitq", []>, TB,
                   Requires<[In64BitMode]>;
+
+// FRED Instructions
+let hasSideEffects = 1, Defs = [RSP, EFLAGS] in {
+  def ERETS: I<0x01, MRM_CA, (outs), (ins), "erets",
+              []>, TB, XD, Requires<[In64BitMode]>;
+  def ERETU: I<0x01, MRM_CA, (outs), (ins), "eretu",
+              []>, TB, XS, Requires<[In64BitMode]>;
+} // hasSideEffects = 1, Defs = [RSP, EFLAGS]
 } // SchedRW
 
 def : Pat<(debugtrap),
@@ -212,6 +220,14 @@ def MOV16sm : I<0x8E, MRMSrcMem, (outs SEGMENT_REG:$dst), (ins i16mem:$src),
 
 let SchedRW = [WriteSystem] in {
 def SWAPGS : I<0x01, MRM_F8, (outs), (ins), "swapgs", []>, TB;
+// LKGS instructions
+let hasSideEffects = 1 in {
+  let mayLoad = 1 in
+  def LKGS16m : I<0x00, MRM6m, (outs), (ins i16mem:$src), "lkgs\t$src",
+                  []>, TB, XD, Requires<[In64BitMode]>;
+  def LKGS16r : I<0x00, MRM6r, (outs), (ins GR16:$src), "lkgs\t$src",
+                  []>, TB, XD, Requires<[In64BitMode]>;
+} // hasSideEffects
 
 let Defs = [EFLAGS] in {
 let mayLoad = 1 in
diff --git a/llvm/test/MC/Disassembler/X86/fred.txt b/llvm/test/MC/Disassembler/X86/fred.txt
new file mode 100644
index 000000000000..7a0762e6e4a2
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/fred.txt
@@ -0,0 +1,8 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64
+
+# CHECK: erets
+0xf2,0x0f,0x01,0xca
+
+# CHECK: eretu
+0xf3,0x0f,0x01,0xca
+
diff --git a/llvm/test/MC/Disassembler/X86/lkgs.txt b/llvm/test/MC/Disassembler/X86/lkgs.txt
new file mode 100644
index 000000000000..1ad04e5c2ccb
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/lkgs.txt
@@ -0,0 +1,35 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=x86_64 --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+# ATT:   lkgs %ax
+# INTEL: lkgs ax
+0xf2,0x0f,0x00,0xf0
+
+# ATT:   lkgs %r12w
+# INTEL: lkgs r12w
+0xf2,0x41,0x0f,0x00,0xf4
+
+# ATT:   lkgs  268435456(%rbp,%r14,8)
+# INTEL: lkgs word ptr [rbp + 8*r14 + 268435456]
+0xf2,0x42,0x0f,0x00,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   lkgs  291(%r8,%rax,4)
+# INTEL: lkgs word ptr [r8 + 4*rax + 291]
+0xf2,0x41,0x0f,0x00,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   lkgs  (%rip)
+# INTEL: lkgs word ptr [rip]
+0xf2,0x0f,0x00,0x35,0x00,0x00,0x00,0x00
+
+# ATT:   lkgs  -64(,%rbp,2)
+# INTEL: lkgs word ptr [2*rbp - 64]
+0xf2,0x0f,0x00,0x34,0x6d,0xc0,0xff,0xff,0xff
+
+# ATT:   lkgs  254(%rcx)
+# INTEL: lkgs word ptr [rcx + 254]
+0xf2,0x0f,0x00,0xb1,0xfe,0x00,0x00,0x00
+
+# ATT:   lkgs  -256(%rdx)
+# INTEL: lkgs word ptr [rdx - 256]
+0xf2,0x0f,0x00,0xb2,0x00,0xff,0xff,0xff
+
diff --git a/llvm/test/MC/X86/fred-att.s b/llvm/test/MC/X86/fred-att.s
new file mode 100644
index 000000000000..9eb2aa7555f6
--- /dev/null
+++ b/llvm/test/MC/X86/fred-att.s
@@ -0,0 +1,14 @@
+// RUN: llvm-mc -triple x86_64 --show-encoding %s | FileCheck %s
+// RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+// ERROR-COUNT-2: error:
+// ERROR-NOT: error:
+
+// CHECK: erets
+// CHECK: encoding: [0xf2,0x0f,0x01,0xca]
+          erets
+
+// CHECK: eretu
+// CHECK: encoding: [0xf3,0x0f,0x01,0xca]
+          eretu
+
diff --git a/llvm/test/MC/X86/fred-intel.s b/llvm/test/MC/X86/fred-intel.s
new file mode 100644
index 000000000000..f9175e3c3bba
--- /dev/null
+++ b/llvm/test/MC/X86/fred-intel.s
@@ -0,0 +1,10 @@
+// RUN: llvm-mc -triple x86_64 -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK: erets
+// CHECK: encoding: [0xf2,0x0f,0x01,0xca]
+          erets
+
+// CHECK: eretu
+// CHECK: encoding: [0xf3,0x0f,0x01,0xca]
+          eretu
+
diff --git a/llvm/test/MC/X86/lkgs-att.s b/llvm/test/MC/X86/lkgs-att.s
new file mode 100644
index 000000000000..e948e2ce559d
--- /dev/null
+++ b/llvm/test/MC/X86/lkgs-att.s
@@ -0,0 +1,38 @@
+// RUN: llvm-mc -triple x86_64 --show-encoding %s | FileCheck %s
+// RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+// ERROR-COUNT-8: error:
+// ERROR-NOT: error:
+
+// CHECK: lkgs %ax
+// CHECK: encoding: [0xf2,0x0f,0x00,0xf0]
+          lkgs %ax
+
+// CHECK: lkgs %r12w
+// CHECK: encoding: [0xf2,0x41,0x0f,0x00,0xf4]
+          lkgs %r12w
+
+// CHECK: lkgs  268435456(%rbp,%r14,8)
+// CHECK: encoding: [0xf2,0x42,0x0f,0x00,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          lkgs  268435456(%rbp,%r14,8)
+
+// CHECK: lkgs  291(%r8,%rax,4)
+// CHECK: encoding: [0xf2,0x41,0x0f,0x00,0xb4,0x80,0x23,0x01,0x00,0x00]
+          lkgs  291(%r8,%rax,4)
+
+// CHECK: lkgs  (%rip)
+// CHECK: encoding: [0xf2,0x0f,0x00,0x35,0x00,0x00,0x00,0x00]
+          lkgs  (%rip)
+
+// CHECK: lkgs  -64(,%rbp,2)
+// CHECK: encoding: [0xf2,0x0f,0x00,0x34,0x6d,0xc0,0xff,0xff,0xff]
+          lkgs  -64(,%rbp,2)
+
+// CHECK: lkgs  254(%rcx)
+// CHECK: encoding: [0xf2,0x0f,0x00,0xb1,0xfe,0x00,0x00,0x00]
+          lkgs  254(%rcx)
+
+// CHECK: lkgs  -256(%rdx)
+// CHECK: encoding: [0xf2,0x0f,0x00,0xb2,0x00,0xff,0xff,0xff]
+          lkgs  -256(%rdx)
+
diff --git a/llvm/test/MC/X86/lkgs-intel.s b/llvm/test/MC/X86/lkgs-intel.s
new file mode 100644
index 000000000000..bb94eda2b2b9
--- /dev/null
+++ b/llvm/test/MC/X86/lkgs-intel.s
@@ -0,0 +1,34 @@
+// RUN: llvm-mc -triple x86_64 -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK: lkgs ax
+// CHECK: encoding: [0xf2,0x0f,0x00,0xf0]
+          lkgs ax
+
+// CHECK: lkgs r12w
+// CHECK: encoding: [0xf2,0x41,0x0f,0x00,0xf4]
+          lkgs r12w
+
+// CHECK: lkgs word ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0xf2,0x42,0x0f,0x00,0xb4,0xf5,0x00,0x00,0x00,0x10]
+          lkgs word ptr [rbp + 8*r14 + 268435456]
+
+// CHECK: lkgs word ptr [r8 + 4*rax + 291]
+// CHECK: encoding: [0xf2,0x41,0x0f,0x00,0xb4,0x80,0x23,0x01,0x00,0x00]
+          lkgs word ptr [r8 + 4*rax + 291]
+
+// CHECK: lkgs word ptr [rip]
+// CHECK: encoding: [0xf2,0x0f,0x00,0x35,0x00,0x00,0x00,0x00]
+          lkgs word ptr [rip]
+
+// CHECK: lkgs word ptr [2*rbp - 64]
+// CHECK: encoding: [0xf2,0x0f,0x00,0x34,0x6d,0xc0,0xff,0xff,0xff]
+          lkgs word ptr [2*rbp - 64]
+
+// CHECK: lkgs word ptr [rcx + 254]
+// CHECK: encoding: [0xf2,0x0f,0x00,0xb1,0xfe,0x00,0x00,0x00]
+          lkgs word ptr [rcx + 254]
+
+// CHECK: lkgs word ptr [rdx - 256]
+// CHECK: encoding: [0xf2,0x0f,0x00,0xb2,0x00,0xff,0xff,0xff]
+          lkgs word ptr [rdx - 256]
+
diff --git a/llvm/test/TableGen/x86-fold-tables.inc b/llvm/test/TableGen/x86-fold-tables.inc
index c8f382d45bf6..4a52a58f2de1 100644
--- a/llvm/test/TableGen/x86-fold-tables.inc
+++ b/llvm/test/TableGen/x86-fold-tables.inc
@@ -426,6 +426,7 @@ static const X86FoldTableEntry Table0[] = {
   {X86::JMP64r, X86::JMP64m, TB_FOLDED_LOAD},
   {X86::JMP64r_NT, X86::JMP64m_NT, TB_FOLDED_LOAD},
   {X86::JMP64r_REX, X86::JMP64m_REX, TB_FOLDED_LOAD},
+  {X86::LKGS16r, X86::LKGS16m, TB_FOLDED_LOAD},
   {X86::MMX_MOVD64from64rr, X86::MMX_MOVQ64mr, TB_FOLDED_STORE},
   {X86::MMX_MOVD64grr, X86::MMX_MOVD64mr, TB_FOLDED_STORE},
   {X86::MOV16ri, X86::MOV16mi, TB_FOLDED_STORE},
-- 
GitLab


From eb103104ef08ebc2d0de63db0592e76b294cf8bb Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Tue, 14 May 2024 19:53:28 -0700
Subject: [PATCH 0799/1206] Revert "[Serialization] Read the initializer for
 interesting static variables before consuming it"

This reverts commit 11b059145d177ee287c7ada9864addf8d083c160.

The premerge bot is broken.
---
 clang/lib/Serialization/ASTReaderDecl.cpp | 29 +---------
 clang/test/Modules/pr91418.cppm           | 67 -----------------------
 2 files changed, 3 insertions(+), 93 deletions(-)
 delete mode 100644 clang/test/Modules/pr91418.cppm

diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index a6254b70560c..0c647086e304 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -4186,35 +4186,12 @@ void ASTReader::PassInterestingDeclsToConsumer() {
     GetDecl(ID);
   EagerlyDeserializedDecls.clear();
 
-  auto ConsumingPotentialInterestingDecls = [this]() {
-    while (!PotentiallyInterestingDecls.empty()) {
-      Decl *D = PotentiallyInterestingDecls.front();
-      PotentiallyInterestingDecls.pop_front();
-      if (isConsumerInterestedIn(D))
-        PassInterestingDeclToConsumer(D);
-    }
-  };
-  std::deque<Decl *> MaybeInterestingDecls =
-      std::move(PotentiallyInterestingDecls);
-  assert(PotentiallyInterestingDecls.empty());
-  while (!MaybeInterestingDecls.empty()) {
-    Decl *D = MaybeInterestingDecls.front();
-    MaybeInterestingDecls.pop_front();
-    // Since we load the variable's initializers lazily, it'd be problematic
-    // if the initializers dependent on each other. So here we try to load the
-    // initializers of static variables to make sure they are passed to code
-    // generator by order. If we read anything interesting, we would consume
-    // that before emitting the current declaration.
-    if (auto *VD = dyn_cast<VarDecl>(D);
-        VD && VD->isFileVarDecl() && !VD->isExternallyVisible())
-      VD->getInit();
-    ConsumingPotentialInterestingDecls();
+  while (!PotentiallyInterestingDecls.empty()) {
+    Decl *D = PotentiallyInterestingDecls.front();
+    PotentiallyInterestingDecls.pop_front();
     if (isConsumerInterestedIn(D))
       PassInterestingDeclToConsumer(D);
   }
-
-  // If we add any new potential interesting decl in the last call, consume it.
-  ConsumingPotentialInterestingDecls();
 }
 
 void ASTReader::loadDeclUpdateRecords(PendingUpdateRecord &Record) {
diff --git a/clang/test/Modules/pr91418.cppm b/clang/test/Modules/pr91418.cppm
deleted file mode 100644
index 33fec992439d..000000000000
--- a/clang/test/Modules/pr91418.cppm
+++ /dev/null
@@ -1,67 +0,0 @@
-// RUN: rm -rf %t
-// RUN: mkdir -p %t
-// RUN: split-file %s %t
-//
-// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 -x c++-header %t/foo.h \
-// RUN:     -emit-pch -o %t/foo.pch
-// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/use.cpp -include-pch \
-// RUN:     %t/foo.pch -emit-llvm -o - | FileCheck %t/use.cpp
-
-//--- foo.h
-#ifndef FOO_H
-#define FOO_H
-typedef float __m128 __attribute__((__vector_size__(16), __aligned__(16)));
-
-static __inline__ __m128 __attribute__((__always_inline__, __min_vector_width__(128)))
-_mm_setr_ps(float __z, float __y, float __x, float __w)
-{
-  return __extension__ (__m128){ __z, __y, __x, __w };
-}
-
-typedef __m128 VR;
-
-inline VR MakeVR( float X, float Y, float Z, float W )
-{
- return _mm_setr_ps( X, Y, Z, W );
-}
-
-extern "C" float sqrtf(float);
-
-namespace VectorSinConstantsSSE
-{
-  float a = (16 * sqrtf(0.225f));
-  VR A = MakeVR(a, a, a, a);
-  static const float b = (16 * sqrtf(0.225f));
-  static const VR B = MakeVR(b, b, b, b);
-}
-
-#endif // FOO_H
-
-//--- use.cpp
-#include "foo.h"
-float use() {
-    return VectorSinConstantsSSE::A[0] + VectorSinConstantsSSE::A[1] +
-           VectorSinConstantsSSE::A[2] + VectorSinConstantsSSE::A[3] +
-           VectorSinConstantsSSE::B[0] + VectorSinConstantsSSE::B[1] +
-           VectorSinConstantsSSE::B[2] + VectorSinConstantsSSE::B[3];
-}
-
-// CHECK: define{{.*}}@__cxx_global_var_init(
-// CHECK: store{{.*}}[[a_RESULT:%[a-zA-Z0-9]+]], ptr @_ZN21VectorSinConstantsSSE1aE
-
-// CHECK: define{{.*}}@__cxx_global_var_init.1(
-// CHECK: [[A_CALL:%[a-zA-Z0-9]+]] = call{{.*}}@_Z6MakeVRffff(
-// CHECK: store{{.*}}[[A_CALL]], ptr @_ZN21VectorSinConstantsSSE1AE
-
-// CHECK: define{{.*}}@__cxx_global_var_init.2(
-// CHECK: [[B_CALL:%[a-zA-Z0-9]+]] = call{{.*}}@_Z6MakeVRffff(
-// CHECK: store{{.*}}[[B_CALL]], ptr @_ZN21VectorSinConstantsSSEL1BE
-
-// CHECK: define{{.*}}@__cxx_global_var_init.3(
-// CHECK: store{{.*}}[[b_RESULT:%[a-zA-Z0-9]+]], ptr @_ZN21VectorSinConstantsSSEL1bE
-
-// CHECK: @_GLOBAL__sub_I_use.cpp
-// CHECK: call{{.*}}@__cxx_global_var_init(
-// CHECK: call{{.*}}@__cxx_global_var_init.1(
-// CHECK: call{{.*}}@__cxx_global_var_init.3(
-// CHECK: call{{.*}}@__cxx_global_var_init.2(
-- 
GitLab


From 2ece5cc2bb1b4cc787e33e24a6582043d441a572 Mon Sep 17 00:00:00 2001
From: epitavy <32581827+epitavy@users.noreply.github.com>
Date: Wed, 15 May 2024 05:09:11 +0200
Subject: [PATCH 0800/1206] [ExceptionDemo] Correct and update example
 ExceptionDemo (#69485)

The ExceptionDemo example was no longer compiling (since llvm 14 at
least). The PR makes the example work with the current API and also
transition from MCJIT to ORC.
Fixes #63702
---
 llvm/examples/ExceptionDemo/CMakeLists.txt    |   4 +-
 llvm/examples/ExceptionDemo/ExceptionDemo.cpp | 181 ++++++++----------
 2 files changed, 77 insertions(+), 108 deletions(-)

diff --git a/llvm/examples/ExceptionDemo/CMakeLists.txt b/llvm/examples/ExceptionDemo/CMakeLists.txt
index 793cf291ca6f..0a60ad848dd4 100644
--- a/llvm/examples/ExceptionDemo/CMakeLists.txt
+++ b/llvm/examples/ExceptionDemo/CMakeLists.txt
@@ -1,9 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   Core
   ExecutionEngine
-  MC
-  MCJIT
-  RuntimeDyld
+  ORCJIT
   Support
   Target
   nativecodegen
diff --git a/llvm/examples/ExceptionDemo/ExceptionDemo.cpp b/llvm/examples/ExceptionDemo/ExceptionDemo.cpp
index 0afc6b30d140..41fa0cf626bf 100644
--- a/llvm/examples/ExceptionDemo/ExceptionDemo.cpp
+++ b/llvm/examples/ExceptionDemo/ExceptionDemo.cpp
@@ -49,8 +49,9 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/BinaryFormat/Dwarf.h"
-#include "llvm/ExecutionEngine/MCJIT.h"
-#include "llvm/ExecutionEngine/SectionMemoryManager.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
+#include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/IRBuilder.h"
@@ -84,6 +85,8 @@
 #define USE_GLOBAL_STR_CONSTS true
 #endif
 
+llvm::ExitOnError ExitOnErr;
+
 //
 // Example types
 //
@@ -142,6 +145,7 @@ static llvm::ConstantInt *ourExceptionCaughtState;
 
 typedef std::vector<std::string> ArgNames;
 typedef std::vector<llvm::Type*> ArgTypes;
+typedef llvm::ArrayRef<llvm::Type *> TypeArray;
 
 //
 // Code Generation Utilities
@@ -892,13 +896,10 @@ void generateStringPrint(llvm::LLVMContext &context,
 ///        generated, and is used to hold the constant string. A value of
 ///        false indicates that the constant string will be stored on the
 ///        stack.
-void generateIntegerPrint(llvm::LLVMContext &context,
-                          llvm::Module &module,
+void generateIntegerPrint(llvm::LLVMContext &context, llvm::Module &module,
                           llvm::IRBuilder<> &builder,
-                          llvm::Function &printFunct,
-                          llvm::Value &toPrint,
-                          std::string format,
-                          bool useGlobal = true) {
+                          llvm::Function &printFunct, llvm::Value *toPrint,
+                          std::string format, bool useGlobal = true) {
   llvm::Constant *stringConstant =
     llvm::ConstantDataArray::getString(context, format);
   llvm::Value *stringVar;
@@ -920,10 +921,9 @@ void generateIntegerPrint(llvm::LLVMContext &context,
 
   llvm::Value *cast = builder.CreateBitCast(stringVar,
                                             builder.getPtrTy());
-  builder.CreateCall(&printFunct, {&toPrint, cast});
+  builder.CreateCall(&printFunct, {toPrint, cast});
 }
 
-
 /// Generates code to handle finally block type semantics: always runs
 /// regardless of whether a thrown exception is passing through or the
 /// parent function is simply exiting. In addition to printing some state
@@ -997,10 +997,10 @@ static llvm::BasicBlock *createFinallyBlock(llvm::LLVMContext &context,
                       bufferToPrint.str(),
                       USE_GLOBAL_STR_CONSTS);
 
-  llvm::SwitchInst *theSwitch = builder.CreateSwitch(builder.CreateLoad(
-                                                       *exceptionCaughtFlag),
-                                                     &terminatorBlock,
-                                                     2);
+  llvm::SwitchInst *theSwitch = builder.CreateSwitch(
+      builder.CreateLoad(ourExceptionNotThrownState->getType(),
+                         *exceptionCaughtFlag),
+      &terminatorBlock, 2);
   theSwitch->addCase(ourExceptionCaughtState, &terminatorBlock);
   theSwitch->addCase(ourExceptionThrownState, &unwindResumeBlock);
 
@@ -1186,7 +1186,7 @@ static llvm::Function *createCatchWrappedInvokeFunction(
 
   // Note: function handles NULL exceptions
   builder.CreateCall(deleteOurException,
-                     builder.CreateLoad(exceptionStorage));
+                     builder.CreateLoad(builder.getPtrTy(), exceptionStorage));
   builder.CreateRetVoid();
 
   // Normal Block
@@ -1206,7 +1206,8 @@ static llvm::Function *createCatchWrappedInvokeFunction(
 
   builder.SetInsertPoint(unwindResumeBlock);
 
-  builder.CreateResume(builder.CreateLoad(caughtResultStorage));
+  builder.CreateResume(
+      builder.CreateLoad(ourCaughtResultType, caughtResultStorage));
 
   // Exception Block
 
@@ -1241,8 +1242,9 @@ static llvm::Function *createCatchWrappedInvokeFunction(
   // Retrieve exception_class member from thrown exception
   // (_Unwind_Exception instance). This member tells us whether or not
   // the exception is foreign.
-  llvm::Value *unwindExceptionClass =
-      builder.CreateLoad(builder.CreateStructGEP(
+  llvm::Value *unwindExceptionClass = builder.CreateLoad(
+      builder.getInt64Ty(),
+      builder.CreateStructGEP(
           ourUnwindExceptionType,
           builder.CreatePointerCast(unwindException,
                                     ourUnwindExceptionType->getPointerTo()),
@@ -1278,9 +1280,9 @@ static llvm::Function *createCatchWrappedInvokeFunction(
   //
   // Note: ourBaseFromUnwindOffset is usually negative
   llvm::Value *typeInfoThrown = builder.CreatePointerCast(
-                                  builder.CreateConstGEP1_64(unwindException,
-                                                       ourBaseFromUnwindOffset),
-                                  ourExceptionType->getPointerTo());
+      builder.CreateConstGEP1_64(builder.getPtrTy(), unwindException,
+                                 ourBaseFromUnwindOffset),
+      ourExceptionType->getPointerTo());
 
   // Retrieve thrown exception type info type
   //
@@ -1289,17 +1291,15 @@ static llvm::Function *createCatchWrappedInvokeFunction(
   typeInfoThrown = builder.CreateStructGEP(ourExceptionType, typeInfoThrown, 0);
 
   llvm::Value *typeInfoThrownType =
-      builder.CreateStructGEP(builder.getPtrTy(), typeInfoThrown, 0);
+      builder.CreateStructGEP(ourTypeInfoType, typeInfoThrown, 0);
 
-  generateIntegerPrint(context,
-                       module,
-                       builder,
-                       *toPrint32Int,
-                       *(builder.CreateLoad(typeInfoThrownType)),
+  llvm::Value *ti8 =
+      builder.CreateLoad(builder.getInt8Ty(), typeInfoThrownType);
+  generateIntegerPrint(context, module, builder, *toPrint32Int,
+                       builder.CreateZExt(ti8, builder.getInt32Ty()),
                        "Gen: Exception type <%d> received (stack unwound) "
                        " in " +
-                       ourId +
-                       ".\n",
+                           ourId + ".\n",
                        USE_GLOBAL_STR_CONSTS);
 
   // Route to matched type info catch block or run cleanup finally block
@@ -1311,8 +1311,7 @@ static llvm::Function *createCatchWrappedInvokeFunction(
 
   for (unsigned i = 1; i <= numExceptionsToCatch; ++i) {
     nextTypeToCatch = i - 1;
-    switchToCatchBlock->addCase(llvm::ConstantInt::get(
-                                   llvm::Type::getInt32Ty(context), i),
+    switchToCatchBlock->addCase(llvm::ConstantInt::get(builder.getInt32Ty(), i),
                                 catchBlocks[nextTypeToCatch]);
   }
 
@@ -1387,14 +1386,10 @@ createThrowExceptionFunction(llvm::Module &module, llvm::IRBuilder<> &builder,
   builder.SetInsertPoint(entryBlock);
 
   llvm::Function *toPrint32Int = module.getFunction("print32Int");
-  generateIntegerPrint(context,
-                       module,
-                       builder,
-                       *toPrint32Int,
-                       *exceptionType,
-                       "\nGen: About to throw exception type <%d> in " +
-                       ourId +
-                       ".\n",
+  generateIntegerPrint(context, module, builder, *toPrint32Int,
+                       builder.CreateZExt(exceptionType, builder.getInt32Ty()),
+                       "\nGen: About to throw exception type <%d> in " + ourId +
+                           ".\n",
                        USE_GLOBAL_STR_CONSTS);
 
   // Switches on runtime type info type value to determine whether or not
@@ -1546,15 +1541,13 @@ typedef void (*OurExceptionThrowFunctType) (int32_t typeToThrow);
 /// @param function generated test function to run
 /// @param typeToThrow type info type of generated exception to throw, or
 ///        indicator to cause foreign exception to be thrown.
-static
-void runExceptionThrow(llvm::ExecutionEngine *engine,
-                       llvm::Function *function,
-                       int32_t typeToThrow) {
+static void runExceptionThrow(llvm::orc::LLJIT *JIT, std::string function,
+                              int32_t typeToThrow) {
 
   // Find test's function pointer
   OurExceptionThrowFunctType functPtr =
-    reinterpret_cast<OurExceptionThrowFunctType>(
-       reinterpret_cast<intptr_t>(engine->getPointerToFunction(function)));
+      reinterpret_cast<OurExceptionThrowFunctType>(reinterpret_cast<uintptr_t>(
+          ExitOnErr(JIT->lookup(function)).getValue()));
 
   try {
     // Run test
@@ -1583,8 +1576,6 @@ void runExceptionThrow(llvm::ExecutionEngine *engine,
 // End test functions
 //
 
-typedef llvm::ArrayRef<llvm::Type*> TypeArray;
-
 /// This initialization routine creates type info globals and
 /// adds external function declarations to module.
 /// @param numTypeInfos number of linear type info associated type info types
@@ -1894,93 +1885,73 @@ int main(int argc, char *argv[]) {
     return(0);
   }
 
-  // If not set, exception handling will not be turned on
-  llvm::TargetOptions Opts;
-
   llvm::InitializeNativeTarget();
   llvm::InitializeNativeTargetAsmPrinter();
-  llvm::LLVMContext Context;
-  llvm::IRBuilder<> theBuilder(Context);
+  auto Context = std::make_unique<llvm::LLVMContext>();
+  llvm::IRBuilder<> theBuilder(*Context);
 
   // Make the module, which holds all the code.
   std::unique_ptr<llvm::Module> Owner =
-      std::make_unique<llvm::Module>("my cool jit", Context);
+      std::make_unique<llvm::Module>("my cool jit", *Context);
   llvm::Module *module = Owner.get();
 
-  std::unique_ptr<llvm::RTDyldMemoryManager> MemMgr(new llvm::SectionMemoryManager());
+  // Build LLJIT
+  std::unique_ptr<llvm::orc::LLJIT> JIT =
+      ExitOnErr(llvm::orc::LLJITBuilder().create());
 
-  // Build engine with JIT
-  llvm::EngineBuilder factory(std::move(Owner));
-  factory.setEngineKind(llvm::EngineKind::JIT);
-  factory.setTargetOptions(Opts);
-  factory.setMCJITMemoryManager(std::move(MemMgr));
-  llvm::ExecutionEngine *executionEngine = factory.create();
+  // Set up the optimizer pipeline.
+  llvm::legacy::FunctionPassManager fpm(module);
 
-  {
-    llvm::legacy::FunctionPassManager fpm(module);
-
-    // Set up the optimizer pipeline.
-    // Start with registering info about how the
-    // target lays out data structures.
-    module->setDataLayout(executionEngine->getDataLayout());
-
-    // Optimizations turned on
+  // Optimizations turned on
 #ifdef ADD_OPT_PASSES
 
-    // Basic AliasAnslysis support for GVN.
-    fpm.add(llvm::createBasicAliasAnalysisPass());
+  // Basic AliasAnslysis support for GVN.
+  fpm.add(llvm::createBasicAliasAnalysisPass());
 
-    // Promote allocas to registers.
-    fpm.add(llvm::createPromoteMemoryToRegisterPass());
+  // Promote allocas to registers.
+  fpm.add(llvm::createPromoteMemoryToRegisterPass());
 
-    // Do simple "peephole" optimizations and bit-twiddling optzns.
-    fpm.add(llvm::createInstructionCombiningPass());
+  // Do simple "peephole" optimizations and bit-twiddling optzns.
+  fpm.add(llvm::createInstructionCombiningPass());
 
-    // Reassociate expressions.
-    fpm.add(llvm::createReassociatePass());
+  // Reassociate expressions.
+  fpm.add(llvm::createReassociatePass());
 
-    // Eliminate Common SubExpressions.
-    fpm.add(llvm::createGVNPass());
+  // Eliminate Common SubExpressions.
+  fpm.add(llvm::createGVNPass());
 
-    // Simplify the control flow graph (deleting unreachable
-    // blocks, etc).
-    fpm.add(llvm::createCFGSimplificationPass());
+  // Simplify the control flow graph (deleting unreachable
+  // blocks, etc).
+  fpm.add(llvm::createCFGSimplificationPass());
 #endif  // ADD_OPT_PASSES
 
-    fpm.doInitialization();
+  fpm.doInitialization();
 
-    // Generate test code using function throwCppException(...) as
-    // the function which throws foreign exceptions.
-    llvm::Function *toRun =
-    createUnwindExceptionTest(*module,
-                              theBuilder,
-                              fpm,
-                              "throwCppException");
+  // Generate test code using function throwCppException(...) as
+  // the function which throws foreign exceptions.
+  createUnwindExceptionTest(*module, theBuilder, fpm, "throwCppException");
 
-    executionEngine->finalizeObject();
+  ExitOnErr(JIT->addIRModule(
+      llvm::orc::ThreadSafeModule(std::move(Owner), std::move(Context))));
 
 #ifndef NDEBUG
-    fprintf(stderr, "\nBegin module dump:\n\n");
+  fprintf(stderr, "\nBegin module dump:\n\n");
 
-    module->dump();
+  module->print(llvm::errs(), nullptr);
 
-    fprintf(stderr, "\nEnd module dump:\n");
+  fprintf(stderr, "\nEnd module dump:\n");
 #endif
 
-    fprintf(stderr, "\n\nBegin Test:\n");
-
-    for (int i = 1; i < argc; ++i) {
-      // Run test for each argument whose value is the exception
-      // type to throw.
-      runExceptionThrow(executionEngine,
-                        toRun,
-                        (unsigned) strtoul(argv[i], NULL, 10));
-    }
+  fprintf(stderr, "\n\nBegin Test:\n");
+  std::string toRun = "outerCatchFunct";
 
-    fprintf(stderr, "\nEnd Test:\n\n");
+  for (int i = 1; i < argc; ++i) {
+    // Run test for each argument whose value is the exception
+    // type to throw.
+    runExceptionThrow(JIT.get(), toRun, (unsigned)strtoul(argv[i], NULL, 10));
   }
 
-  delete executionEngine;
+  fprintf(stderr, "\nEnd Test:\n\n");
 
   return 0;
 }
-- 
GitLab


From 72b2c37de6a4bbc2b2d2cda49293684b7cc71508 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Tue, 14 May 2024 20:29:36 -0700
Subject: [PATCH 0801/1206] [clang-format][NFC] Clean up
 TokenAnnotator::mustBreakBefore()

---
 clang/lib/Format/TokenAnnotator.cpp | 35 +++++++++++++++--------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 478cae23d3c8..d0aa0838423e 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -5405,6 +5405,9 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
     return true;
   }
 
+  const auto *BeforeLeft = Left.Previous;
+  const auto *AfterRight = Right.Next;
+
   if (Style.isCSharp()) {
     if (Left.is(TT_FatArrow) && Right.is(tok::l_brace) &&
         Style.BraceWrapping.AfterFunction) {
@@ -5416,7 +5419,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
     }
     if (Right.is(TT_CSharpGenericTypeConstraint))
       return true;
-    if (Right.Next && Right.Next->is(TT_FatArrow) &&
+    if (AfterRight && AfterRight->is(TT_FatArrow) &&
         (Right.is(tok::numeric_constant) ||
          (Right.is(tok::identifier) && Right.TokenText == "_"))) {
       return true;
@@ -5433,15 +5436,14 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
         Left.is(tok::r_square) && Right.is(tok::l_square)) {
       return true;
     }
-
   } else if (Style.isJavaScript()) {
     // FIXME: This might apply to other languages and token kinds.
-    if (Right.is(tok::string_literal) && Left.is(tok::plus) && Left.Previous &&
-        Left.Previous->is(tok::string_literal)) {
+    if (Right.is(tok::string_literal) && Left.is(tok::plus) && BeforeLeft &&
+        BeforeLeft->is(tok::string_literal)) {
       return true;
     }
     if (Left.is(TT_DictLiteral) && Left.is(tok::l_brace) && Line.Level == 0 &&
-        Left.Previous && Left.Previous->is(tok::equal) &&
+        BeforeLeft && BeforeLeft->is(tok::equal) &&
         Line.First->isOneOf(tok::identifier, Keywords.kw_import, tok::kw_export,
                             tok::kw_const) &&
         // kw_var/kw_let are pseudo-tokens that are tok::identifier, so match
@@ -5460,8 +5462,8 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
       // instead of bin-packing.
       return true;
     }
-    if (Right.is(tok::r_brace) && Left.is(tok::l_brace) && Left.Previous &&
-        Left.Previous->is(TT_FatArrow)) {
+    if (Right.is(tok::r_brace) && Left.is(tok::l_brace) && BeforeLeft &&
+        BeforeLeft->is(TT_FatArrow)) {
       // JS arrow function (=> {...}).
       switch (Style.AllowShortLambdasOnASingleLine) {
       case FormatStyle::SLS_All:
@@ -5489,8 +5491,8 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
                   FormatStyle::SFS_InlineOnly);
     }
   } else if (Style.Language == FormatStyle::LK_Java) {
-    if (Right.is(tok::plus) && Left.is(tok::string_literal) && Right.Next &&
-        Right.Next->is(tok::string_literal)) {
+    if (Right.is(tok::plus) && Left.is(tok::string_literal) && AfterRight &&
+        AfterRight->is(tok::string_literal)) {
       return true;
     }
   } else if (Style.isVerilog()) {
@@ -5543,8 +5545,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
       }
       return Style.BreakArrays;
     }
-  }
-  if (Style.isTableGen()) {
+  } else if (Style.isTableGen()) {
     // Break the comma in side cond operators.
     // !cond(case1:1,
     //       case2:0);
@@ -5600,8 +5601,8 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
     return true;
   if (Left.IsUnterminatedLiteral)
     return true;
-  if (Right.is(tok::lessless) && Right.Next && Left.is(tok::string_literal) &&
-      Right.Next->is(tok::string_literal)) {
+  if (Right.is(tok::lessless) && AfterRight && Left.is(tok::string_literal) &&
+      AfterRight->is(tok::string_literal)) {
     return true;
   }
   if (Right.is(TT_RequiresClause)) {
@@ -5678,8 +5679,8 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
     // string literal accordingly. Thus, we try keep existing line breaks.
     return Right.IsMultiline && Right.NewlinesBefore > 0;
   }
-  if ((Left.is(tok::l_brace) || (Left.is(tok::less) && Left.Previous &&
-                                 Left.Previous->is(tok::equal))) &&
+  if ((Left.is(tok::l_brace) ||
+       (Left.is(tok::less) && BeforeLeft && BeforeLeft->is(tok::equal))) &&
       Right.NestingLevel == 1 && Style.Language == FormatStyle::LK_Proto) {
     // Don't put enums or option definitions onto single lines in protocol
     // buffers.
@@ -5793,7 +5794,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
   //
   // We ensure elsewhere that extensions are always on their own line.
   if (Style.isProto() && Right.is(TT_SelectorName) &&
-      Right.isNot(tok::r_square) && Right.Next) {
+      Right.isNot(tok::r_square) && AfterRight) {
     // Keep `@submessage` together in:
     // @submessage { key: value }
     if (Left.is(tok::at))
@@ -5802,7 +5803,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
     // selector { ...
     // selector: { ...
     // selector: @base { ...
-    FormatToken *LBrace = Right.Next;
+    const auto *LBrace = AfterRight;
     if (LBrace && LBrace->is(tok::colon)) {
       LBrace = LBrace->Next;
       if (LBrace && LBrace->is(tok::at)) {
-- 
GitLab


From 1a58e88690c1a48d1082b4ee6b759f5dc49a7144 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Wed, 15 May 2024 11:44:32 +0800
Subject: [PATCH 0802/1206] [RISCV] Move RISCVInsertVSETVLI to after phi
 elimination (#91440)

Split off from #70549, this patch moves RISCVInsertVSETVLI to after phi
elimination where we exit SSA and need to move to LiveVariables.

The motivation for splitting this off is to avoid the large scheduling
diffs from moving completely to after regalloc, and instead focus on
converting the pass to work on LiveIntervals.

The two main changes required are updating VSETVLIInfo to store VNInfos
instead of MachineInstrs, which allows us to still check for PHI defs in
needVSETVLIPHI, and fixing up the live intervals of any AVL operands
after inserting new instructions.

On O3 the pass is inserted after the register coalescer, otherwise we
end up with a bunch of COPYs around eliminated PHIs that trip up
needVSETVLIPHI.

Co-authored-by: Piyou Chen <piyou.chen@sifive.com>
---
 llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp  | 262 +++++++++++-------
 llvm/lib/Target/RISCV/RISCVTargetMachine.cpp  |   9 +-
 llvm/test/CodeGen/RISCV/O0-pipeline.ll        |   6 +-
 llvm/test/CodeGen/RISCV/O3-pipeline.ll        |   2 +-
 llvm/test/CodeGen/RISCV/rvv/combine-vmv.ll    |   2 +-
 .../rvv/concat-vectors-constant-stride.ll     |   2 +-
 .../RISCV/rvv/dont-sink-splat-operands.ll     |   8 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-int.ll    |   2 +-
 .../RISCV/rvv/fixed-vectors-masked-gather.ll  |   2 +-
 .../rvv/fixed-vectors-shuffle-vslide1up.ll    |   2 +-
 .../rvv/fixed-vectors-strided-load-combine.ll |   2 +-
 .../fixed-vectors-strided-load-store-asm.ll   |   2 +-
 .../RISCV/rvv/fixed-vectors-trunc-vp.ll       |   2 +-
 .../RISCV/rvv/fold-scalar-load-crash.ll       |   4 +-
 .../CodeGen/RISCV/rvv/fpclamptosat_vec.ll     |  12 +-
 .../RISCV/rvv/rv32-spill-vector-csr.ll        |   4 +-
 .../CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll   |  10 +-
 .../RISCV/rvv/rv64-spill-vector-csr.ll        |   4 +-
 .../CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll   |  10 +-
 .../RISCV/rvv/rvv-peephole-vmerge-vops.ll     |   8 +-
 .../CodeGen/RISCV/rvv/sink-splat-operands.ll  |  44 +--
 .../RISCV/rvv/undef-earlyclobber-chain.ll     |   2 +-
 llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll    |   4 +-
 .../RISCV/rvv/vector-reassociations.ll        |   8 +-
 llvm/test/CodeGen/RISCV/rvv/vfwmacc-vp.ll     |   2 +-
 .../RISCV/rvv/vrgatherei16-subreg-liveness.ll |   8 +-
 .../RISCV/rvv/vsetvli-insert-crossbb.ll       |   2 -
 .../RISCV/rvv/vsetvli-insert-crossbb.mir      | 136 ++++-----
 llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll |  12 +-
 29 files changed, 312 insertions(+), 261 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 7a8ff84995ea..1c815424bdfa 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -47,6 +47,18 @@ static cl::opt<bool> DisableInsertVSETVLPHIOpt(
 
 namespace {
 
+/// Given a virtual register \p Reg, return the corresponding VNInfo for it.
+/// This should never return nullptr.
+static VNInfo *getVNInfoFromReg(Register Reg, const MachineInstr &MI,
+                                const LiveIntervals *LIS) {
+  assert(Reg.isVirtual());
+  auto &LI = LIS->getInterval(Reg);
+  SlotIndex SI = LIS->getSlotIndexes()->getInstructionIndex(MI);
+  VNInfo *VNI = LI.getVNInfoBefore(SI);
+  assert(VNI);
+  return VNI;
+}
+
 static unsigned getVLOpNum(const MachineInstr &MI) {
   return RISCVII::getVLOpNum(MI.getDesc());
 }
@@ -426,7 +438,8 @@ DemandedFields getDemanded(const MachineInstr &MI, const RISCVSubtarget *ST) {
 /// values of the VL and VTYPE registers after insertion.
 class VSETVLIInfo {
   struct AVLDef {
-    const MachineInstr *DefMI;
+    // Every AVLDef should have a VNInfo.
+    const VNInfo *ValNo;
     Register DefReg;
   };
   union {
@@ -465,9 +478,9 @@ public:
   void setUnknown() { State = Unknown; }
   bool isUnknown() const { return State == Unknown; }
 
-  void setAVLRegDef(const MachineInstr *DefMI, Register AVLReg) {
-    assert(DefMI && AVLReg.isVirtual());
-    AVLRegDef.DefMI = DefMI;
+  void setAVLRegDef(const VNInfo *VNInfo, Register AVLReg) {
+    assert(VNInfo && AVLReg.isVirtual());
+    AVLRegDef.ValNo = VNInfo;
     AVLRegDef.DefReg = AVLReg;
     State = AVLIsReg;
   }
@@ -493,9 +506,18 @@ public:
     assert(hasAVLImm());
     return AVLImm;
   }
-  const MachineInstr &getAVLDefMI() const {
-    assert(hasAVLReg() && AVLRegDef.DefMI);
-    return *AVLRegDef.DefMI;
+  const VNInfo *getAVLVNInfo() const {
+    assert(hasAVLReg());
+    return AVLRegDef.ValNo;
+  }
+  // Most AVLIsReg infos will have a single defining MachineInstr, unless it was
+  // a PHI node. In that case getAVLVNInfo()->def will point to the block
+  // boundary slot.
+  const MachineInstr *getAVLDefMI(const LiveIntervals *LIS) const {
+    assert(hasAVLReg());
+    auto *MI = LIS->getInstructionFromIndex(getAVLVNInfo()->def);
+    assert(!(getAVLVNInfo()->isPHIDef() && MI));
+    return MI;
   }
 
   void setAVL(VSETVLIInfo Info) {
@@ -503,7 +525,7 @@ public:
     if (Info.isUnknown())
       setUnknown();
     else if (Info.hasAVLReg())
-      setAVLRegDef(&Info.getAVLDefMI(), Info.getAVLReg());
+      setAVLRegDef(Info.getAVLVNInfo(), Info.getAVLReg());
     else if (Info.hasAVLVLMAX())
       setAVLVLMAX();
     else if (Info.hasAVLIgnored())
@@ -519,11 +541,13 @@ public:
   bool getTailAgnostic() const { return TailAgnostic; }
   bool getMaskAgnostic() const { return MaskAgnostic; }
 
-  bool hasNonZeroAVL() const {
+  bool hasNonZeroAVL(const LiveIntervals *LIS) const {
     if (hasAVLImm())
       return getAVLImm() > 0;
-    if (hasAVLReg())
-      return isNonZeroLoadImmediate(getAVLDefMI());
+    if (hasAVLReg()) {
+      if (auto *DefMI = getAVLDefMI(LIS))
+        return isNonZeroLoadImmediate(*DefMI);
+    }
     if (hasAVLVLMAX())
       return true;
     if (hasAVLIgnored())
@@ -531,16 +555,17 @@ public:
     return false;
   }
 
-  bool hasEquallyZeroAVL(const VSETVLIInfo &Other) const {
+  bool hasEquallyZeroAVL(const VSETVLIInfo &Other,
+                         const LiveIntervals *LIS) const {
     if (hasSameAVL(Other))
       return true;
-    return (hasNonZeroAVL() && Other.hasNonZeroAVL());
+    return (hasNonZeroAVL(LIS) && Other.hasNonZeroAVL(LIS));
   }
 
   bool hasSameAVL(const VSETVLIInfo &Other) const {
     if (hasAVLReg() && Other.hasAVLReg())
-      return AVLRegDef.DefMI == Other.AVLRegDef.DefMI &&
-             AVLRegDef.DefReg == Other.AVLRegDef.DefReg;
+      return getAVLVNInfo()->id == Other.getAVLVNInfo()->id &&
+             getAVLReg() == Other.getAVLReg();
 
     if (hasAVLImm() && Other.hasAVLImm())
       return getAVLImm() == Other.getAVLImm();
@@ -620,7 +645,7 @@ public:
   // Require are compatible with the previous vsetvli instruction represented
   // by this.  MI is the instruction whose requirements we're considering.
   bool isCompatible(const DemandedFields &Used, const VSETVLIInfo &Require,
-                    const MachineRegisterInfo &MRI) const {
+                    const LiveIntervals *LIS) const {
     assert(isValid() && Require.isValid() &&
            "Can't compare invalid VSETVLIInfos");
     assert(!Require.SEWLMULRatioOnly &&
@@ -636,7 +661,7 @@ public:
     if (Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require)))
       return false;
 
-    if (Used.VLZeroness && !hasEquallyZeroAVL(Require))
+    if (Used.VLZeroness && !hasEquallyZeroAVL(Require, LIS))
       return false;
 
     return hasCompatibleVTYPE(Used, Require);
@@ -765,6 +790,7 @@ class RISCVInsertVSETVLI : public MachineFunctionPass {
   const RISCVSubtarget *ST;
   const TargetInstrInfo *TII;
   MachineRegisterInfo *MRI;
+  LiveIntervals *LIS;
 
   std::vector<BlockData> BlockInfo;
   std::queue<const MachineBasicBlock *> WorkList;
@@ -777,6 +803,14 @@ public:
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
+
+    AU.addRequired<LiveIntervals>();
+    AU.addPreserved<LiveIntervals>();
+    AU.addRequired<SlotIndexes>();
+    AU.addPreserved<SlotIndexes>();
+    AU.addPreserved<LiveDebugVariables>();
+    AU.addPreserved<LiveStacks>();
+
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
@@ -848,7 +882,7 @@ INITIALIZE_PASS(RISCVCoalesceVSETVLI, "riscv-coalesce-vsetvli",
 // Return a VSETVLIInfo representing the changes made by this VSETVLI or
 // VSETIVLI instruction.
 static VSETVLIInfo getInfoForVSETVLI(const MachineInstr &MI,
-                                     const MachineRegisterInfo &MRI) {
+                                     const LiveIntervals *LIS) {
   VSETVLIInfo NewInfo;
   if (MI.getOpcode() == RISCV::PseudoVSETIVLI) {
     NewInfo.setAVLImm(MI.getOperand(1).getImm());
@@ -861,7 +895,7 @@ static VSETVLIInfo getInfoForVSETVLI(const MachineInstr &MI,
     if (AVLReg == RISCV::X0)
       NewInfo.setAVLVLMAX();
     else
-      NewInfo.setAVLRegDef(MRI.getUniqueVRegDef(AVLReg), AVLReg);
+      NewInfo.setAVLRegDef(getVNInfoFromReg(AVLReg, MI, LIS), AVLReg);
   }
   NewInfo.setVTYPE(MI.getOperand(2).getImm());
 
@@ -880,7 +914,7 @@ static unsigned computeVLMAX(unsigned VLEN, unsigned SEW,
 
 static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags,
                                        const RISCVSubtarget &ST,
-                                       const MachineRegisterInfo *MRI) {
+                                       const LiveIntervals *LIS) {
   VSETVLIInfo InstrInfo;
 
   bool TailAgnostic = true;
@@ -933,7 +967,7 @@ static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags,
       else
         InstrInfo.setAVLImm(Imm);
     } else {
-      InstrInfo.setAVLRegDef(MRI->getUniqueVRegDef(VLOp.getReg()),
+      InstrInfo.setAVLRegDef(getVNInfoFromReg(VLOp.getReg(), MI, LIS),
                              VLOp.getReg());
     }
   } else {
@@ -955,9 +989,9 @@ static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags,
   // register AVLs to avoid extending live ranges without being sure we can
   // kill the original source reg entirely.
   if (InstrInfo.hasAVLReg()) {
-    const MachineInstr &DefMI = InstrInfo.getAVLDefMI();
-    if (isVectorConfigInstr(DefMI)) {
-      VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(DefMI, *MRI);
+    if (const MachineInstr *DefMI = InstrInfo.getAVLDefMI(LIS);
+        DefMI && isVectorConfigInstr(*DefMI)) {
+      VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(*DefMI, LIS);
       if (DefInstrInfo.hasSameVLMAX(InstrInfo) &&
           (DefInstrInfo.hasAVLImm() || DefInstrInfo.hasAVLVLMAX()))
         InstrInfo.setAVL(DefInstrInfo);
@@ -983,11 +1017,12 @@ void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
     // Use X0, X0 form if the AVL is the same and the SEW+LMUL gives the same
     // VLMAX.
     if (Info.hasSameAVL(PrevInfo) && Info.hasSameVLMAX(PrevInfo)) {
-      BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
-          .addReg(RISCV::X0, RegState::Define | RegState::Dead)
-          .addReg(RISCV::X0, RegState::Kill)
-          .addImm(Info.encodeVTYPE())
-          .addReg(RISCV::VL, RegState::Implicit);
+      auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
+                    .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+                    .addReg(RISCV::X0, RegState::Kill)
+                    .addImm(Info.encodeVTYPE())
+                    .addReg(RISCV::VL, RegState::Implicit);
+      LIS->InsertMachineInstrInMaps(*MI);
       return;
     }
 
@@ -995,15 +1030,16 @@ void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
     // it has the same VLMAX we want and the last VL/VTYPE we observed is the
     // same, we can use the X0, X0 form.
     if (Info.hasSameVLMAX(PrevInfo) && Info.hasAVLReg()) {
-      const MachineInstr &DefMI = Info.getAVLDefMI();
-      if (isVectorConfigInstr(DefMI)) {
-        VSETVLIInfo DefInfo = getInfoForVSETVLI(DefMI, *MRI);
+      if (const MachineInstr *DefMI = Info.getAVLDefMI(LIS);
+          DefMI && isVectorConfigInstr(*DefMI)) {
+        VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI, LIS);
         if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {
-          BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
-              .addReg(RISCV::X0, RegState::Define | RegState::Dead)
-              .addReg(RISCV::X0, RegState::Kill)
-              .addImm(Info.encodeVTYPE())
-              .addReg(RISCV::VL, RegState::Implicit);
+          auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
+                        .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+                        .addReg(RISCV::X0, RegState::Kill)
+                        .addImm(Info.encodeVTYPE())
+                        .addReg(RISCV::VL, RegState::Implicit);
+          LIS->InsertMachineInstrInMaps(*MI);
           return;
         }
       }
@@ -1011,10 +1047,11 @@ void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
   }
 
   if (Info.hasAVLImm()) {
-    BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))
-        .addReg(RISCV::X0, RegState::Define | RegState::Dead)
-        .addImm(Info.getAVLImm())
-        .addImm(Info.encodeVTYPE());
+    auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))
+                  .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+                  .addImm(Info.getAVLImm())
+                  .addImm(Info.encodeVTYPE());
+    LIS->InsertMachineInstrInMaps(*MI);
     return;
   }
 
@@ -1023,36 +1060,46 @@ void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
     // the previous vl to become invalid.
     if (PrevInfo.isValid() && !PrevInfo.isUnknown() &&
         Info.hasSameVLMAX(PrevInfo)) {
-      BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
-          .addReg(RISCV::X0, RegState::Define | RegState::Dead)
-          .addReg(RISCV::X0, RegState::Kill)
-          .addImm(Info.encodeVTYPE())
-          .addReg(RISCV::VL, RegState::Implicit);
+      auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
+                    .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+                    .addReg(RISCV::X0, RegState::Kill)
+                    .addImm(Info.encodeVTYPE())
+                    .addReg(RISCV::VL, RegState::Implicit);
+      LIS->InsertMachineInstrInMaps(*MI);
       return;
     }
     // Otherwise use an AVL of 1 to avoid depending on previous vl.
-    BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))
-        .addReg(RISCV::X0, RegState::Define | RegState::Dead)
-        .addImm(1)
-        .addImm(Info.encodeVTYPE());
+    auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))
+                  .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+                  .addImm(1)
+                  .addImm(Info.encodeVTYPE());
+    LIS->InsertMachineInstrInMaps(*MI);
     return;
   }
 
   if (Info.hasAVLVLMAX()) {
     Register DestReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
-    BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
-        .addReg(DestReg, RegState::Define | RegState::Dead)
-        .addReg(RISCV::X0, RegState::Kill)
-        .addImm(Info.encodeVTYPE());
+    auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
+                  .addReg(DestReg, RegState::Define | RegState::Dead)
+                  .addReg(RISCV::X0, RegState::Kill)
+                  .addImm(Info.encodeVTYPE());
+    LIS->InsertMachineInstrInMaps(*MI);
+    LIS->createAndComputeVirtRegInterval(DestReg);
     return;
   }
 
   Register AVLReg = Info.getAVLReg();
   MRI->constrainRegClass(AVLReg, &RISCV::GPRNoX0RegClass);
-  BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLI))
-      .addReg(RISCV::X0, RegState::Define | RegState::Dead)
-      .addReg(AVLReg)
-      .addImm(Info.encodeVTYPE());
+  auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLI))
+                .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+                .addReg(AVLReg)
+                .addImm(Info.encodeVTYPE());
+  LIS->InsertMachineInstrInMaps(*MI);
+  // Normally the AVL's live range will already extend past the inserted vsetvli
+  // because the pseudos below will already use the AVL. But this isn't always
+  // the case, e.g. PseudoVMV_X_S doesn't have an AVL operand.
+  LIS->getInterval(AVLReg).extendInBlock(
+      LIS->getMBBStartIdx(&MBB), LIS->getInstructionIndex(*MI).getRegSlot());
 }
 
 static bool isLMUL1OrSmaller(RISCVII::VLMUL LMUL) {
@@ -1065,7 +1112,7 @@ static bool isLMUL1OrSmaller(RISCVII::VLMUL LMUL) {
 bool RISCVInsertVSETVLI::needVSETVLI(const MachineInstr &MI,
                                      const VSETVLIInfo &Require,
                                      const VSETVLIInfo &CurInfo) const {
-  assert(Require == computeInfoForInstr(MI, MI.getDesc().TSFlags, *ST, MRI));
+  assert(Require == computeInfoForInstr(MI, MI.getDesc().TSFlags, *ST, LIS));
 
   if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
     return true;
@@ -1106,7 +1153,7 @@ bool RISCVInsertVSETVLI::needVSETVLI(const MachineInstr &MI,
     Used.TailPolicy = false;
   }
 
-  if (CurInfo.isCompatible(Used, Require, *MRI))
+  if (CurInfo.isCompatible(Used, Require, LIS))
     return false;
 
   // We didn't find a compatible value. If our AVL is a virtual register,
@@ -1114,9 +1161,9 @@ bool RISCVInsertVSETVLI::needVSETVLI(const MachineInstr &MI,
   // and the last VL/VTYPE we observed is the same, we don't need a
   // VSETVLI here.
   if (Require.hasAVLReg() && CurInfo.hasCompatibleVTYPE(Used, Require)) {
-    const MachineInstr &DefMI = Require.getAVLDefMI();
-    if (isVectorConfigInstr(DefMI)) {
-      VSETVLIInfo DefInfo = getInfoForVSETVLI(DefMI, *MRI);
+    if (const MachineInstr *DefMI = Require.getAVLDefMI(LIS);
+        DefMI && isVectorConfigInstr(*DefMI)) {
+      VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI, LIS);
       if (DefInfo.hasSameAVL(CurInfo) && DefInfo.hasSameVLMAX(CurInfo))
         return false;
     }
@@ -1152,7 +1199,7 @@ void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
   if (!RISCVII::hasSEWOp(TSFlags))
     return;
 
-  const VSETVLIInfo NewInfo = computeInfoForInstr(MI, TSFlags, *ST, MRI);
+  const VSETVLIInfo NewInfo = computeInfoForInstr(MI, TSFlags, *ST, LIS);
   assert(NewInfo.isValid() && !NewInfo.isUnknown());
   if (Info.isValid() && !needVSETVLI(MI, NewInfo, Info))
     return;
@@ -1171,7 +1218,7 @@ void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
   // variant, so we avoid the transform to prevent extending live range of an
   // avl register operand.
   // TODO: We can probably relax this for immediates.
-  bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(PrevInfo) &&
+  bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(PrevInfo, LIS) &&
                      IncomingInfo.hasSameVLMAX(PrevInfo);
   if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
     Info.setAVL(IncomingInfo);
@@ -1202,14 +1249,17 @@ void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
 void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
                                        const MachineInstr &MI) const {
   if (isVectorConfigInstr(MI)) {
-    Info = getInfoForVSETVLI(MI, *MRI);
+    Info = getInfoForVSETVLI(MI, LIS);
     return;
   }
 
   if (RISCV::isFaultFirstLoad(MI)) {
     // Update AVL to vl-output of the fault first load.
-    Info.setAVLRegDef(MRI->getUniqueVRegDef(MI.getOperand(1).getReg()),
-                      MI.getOperand(1).getReg());
+    assert(MI.getOperand(1).getReg().isVirtual());
+    auto &LI = LIS->getInterval(MI.getOperand(1).getReg());
+    SlotIndex SI = LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot();
+    VNInfo *VNI = LI.getVNInfoAt(SI);
+    Info.setAVLRegDef(VNI, MI.getOperand(1).getReg());
     return;
   }
 
@@ -1293,7 +1343,7 @@ void RISCVInsertVSETVLI::computeIncomingVLVTYPE(const MachineBasicBlock &MBB) {
 }
 
 // If we weren't able to prove a vsetvli was directly unneeded, it might still
-// be unneeded if the AVL is a phi node where all incoming values are VL
+// be unneeded if the AVL was a phi node where all incoming values are VL
 // outputs from the last VSETVLI in their respective basic blocks.
 bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,
                                         const MachineBasicBlock &MBB) const {
@@ -1303,26 +1353,27 @@ bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,
   if (!Require.hasAVLReg())
     return true;
 
-  // We need the AVL to be produce by a PHI node in this basic block.
-  const MachineInstr *PHI = &Require.getAVLDefMI();
-  if (PHI->getOpcode() != RISCV::PHI || PHI->getParent() != &MBB)
+  // We need the AVL to have been produced by a PHI node in this basic block.
+  const VNInfo *Valno = Require.getAVLVNInfo();
+  if (!Valno->isPHIDef() || LIS->getMBBFromIndex(Valno->def) != &MBB)
     return true;
 
-  for (unsigned PHIOp = 1, NumOps = PHI->getNumOperands(); PHIOp != NumOps;
-       PHIOp += 2) {
-    Register InReg = PHI->getOperand(PHIOp).getReg();
-    MachineBasicBlock *PBB = PHI->getOperand(PHIOp + 1).getMBB();
+  const LiveRange &LR = LIS->getInterval(Require.getAVLReg());
+
+  for (auto *PBB : MBB.predecessors()) {
     const VSETVLIInfo &PBBExit = BlockInfo[PBB->getNumber()].Exit;
 
     // We need the PHI input to the be the output of a VSET(I)VLI.
-    MachineInstr *DefMI = MRI->getUniqueVRegDef(InReg);
-    assert(DefMI);
-    if (!isVectorConfigInstr(*DefMI))
+    const VNInfo *Value = LR.getVNInfoBefore(LIS->getMBBEndIdx(PBB));
+    if (!Value)
+      return true;
+    MachineInstr *DefMI = LIS->getInstructionFromIndex(Value->def);
+    if (!DefMI || !isVectorConfigInstr(*DefMI))
       return true;
 
     // We found a VSET(I)VLI make sure it matches the output of the
     // predecessor block.
-    VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI, *MRI);
+    VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI, LIS);
     if (DefInfo != PBBExit)
       return true;
 
@@ -1377,19 +1428,28 @@ void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {
         MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));
         if (VLOp.isReg()) {
           Register Reg = VLOp.getReg();
-          MachineInstr *VLOpDef = MRI->getUniqueVRegDef(Reg);
-          assert(VLOpDef);
+          LiveInterval &LI = LIS->getInterval(Reg);
 
           // Erase the AVL operand from the instruction.
           VLOp.setReg(RISCV::NoRegister);
           VLOp.setIsKill(false);
+          SmallVector<MachineInstr *> DeadMIs;
+          LIS->shrinkToUses(&LI, &DeadMIs);
+          // We might have separate components that need split due to
+          // needVSETVLIPHI causing us to skip inserting a new VL def.
+          SmallVector<LiveInterval *> SplitLIs;
+          LIS->splitSeparateComponents(LI, SplitLIs);
 
           // If the AVL was an immediate > 31, then it would have been emitted
           // as an ADDI. However, the ADDI might not have been used in the
           // vsetvli, or a vsetvli might not have been emitted, so it may be
           // dead now.
-          if (TII->isAddImmediate(*VLOpDef, Reg) && MRI->use_nodbg_empty(Reg))
-            VLOpDef->eraseFromParent();
+          for (MachineInstr *DeadMI : DeadMIs) {
+            if (!TII->isAddImmediate(*DeadMI, Reg))
+              continue;
+            LIS->RemoveMachineInstrFromMaps(*DeadMI);
+            DeadMI->eraseFromParent();
+          }
         }
         MI.addOperand(MachineOperand::CreateReg(RISCV::VL, /*isDef*/ false,
                                                 /*isImp*/ true));
@@ -1458,14 +1518,14 @@ void RISCVInsertVSETVLI::doPRE(MachineBasicBlock &MBB) {
   // we need to prove the value is available at the point we're going
   // to insert the vsetvli at.
   if (AvailableInfo.hasAVLReg()) {
-    const MachineInstr *AVLDefMI = &AvailableInfo.getAVLDefMI();
+    SlotIndex SI = AvailableInfo.getAVLVNInfo()->def;
     // This is an inline dominance check which covers the case of
     // UnavailablePred being the preheader of a loop.
-    if (AVLDefMI->getParent() != UnavailablePred)
+    if (LIS->getMBBFromIndex(SI) != UnavailablePred)
+      return;
+    if (!UnavailablePred->terminators().empty() &&
+        SI >= LIS->getInstructionIndex(*UnavailablePred->getFirstTerminator()))
       return;
-    for (auto &TermMI : UnavailablePred->terminators())
-      if (&TermMI == AVLDefMI)
-        return;
   }
 
   // If the AVL isn't used in its predecessors then bail, since we have no AVL
@@ -1526,7 +1586,8 @@ void RISCVInsertVSETVLI::doPRE(MachineBasicBlock &MBB) {
 static bool canMutatePriorConfig(const MachineInstr &PrevMI,
                                  const MachineInstr &MI,
                                  const DemandedFields &Used,
-                                 const MachineRegisterInfo &MRI) {
+                                 const MachineRegisterInfo &MRI,
+                                 const LiveIntervals *LIS) {
   // If the VL values aren't equal, return false if either a) the former is
   // demanded, or b) we can't rewrite the former to be the later for
   // implementation reasons.
@@ -1537,8 +1598,8 @@ static bool canMutatePriorConfig(const MachineInstr &PrevMI,
     if (Used.VLZeroness) {
       if (isVLPreservingConfig(PrevMI))
         return false;
-      if (!getInfoForVSETVLI(PrevMI, MRI)
-               .hasEquallyZeroAVL(getInfoForVSETVLI(MI, MRI)))
+      if (!getInfoForVSETVLI(PrevMI, LIS)
+               .hasEquallyZeroAVL(getInfoForVSETVLI(MI, LIS), LIS))
         return false;
     }
 
@@ -1588,7 +1649,7 @@ bool RISCVCoalesceVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) {
         continue;
       }
 
-      if (canMutatePriorConfig(MI, *NextMI, Used, *MRI)) {
+      if (canMutatePriorConfig(MI, *NextMI, Used, *MRI, LIS)) {
         if (!isVLPreservingConfig(*NextMI)) {
           Register DefReg = NextMI->getOperand(0).getReg();
 
@@ -1661,9 +1722,17 @@ void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
     if (RISCV::isFaultFirstLoad(MI)) {
       Register VLOutput = MI.getOperand(1).getReg();
       assert(VLOutput.isVirtual());
-      if (!MRI->use_nodbg_empty(VLOutput))
-        BuildMI(MBB, I, MI.getDebugLoc(), TII->get(RISCV::PseudoReadVL),
-                VLOutput);
+      if (!MI.getOperand(1).isDead()) {
+        auto ReadVLMI = BuildMI(MBB, I, MI.getDebugLoc(),
+                                TII->get(RISCV::PseudoReadVL), VLOutput);
+        // Move the LiveInterval's definition down to PseudoReadVL.
+        SlotIndex NewDefSI =
+            LIS->InsertMachineInstrInMaps(*ReadVLMI).getRegSlot();
+        LiveInterval &DefLI = LIS->getInterval(VLOutput);
+        VNInfo *DefVNI = DefLI.getVNInfoAt(DefLI.beginIndex());
+        DefLI.removeSegment(DefLI.beginIndex(), NewDefSI);
+        DefVNI->def = NewDefSI;
+      }
       // We don't use the vl output of the VLEFF/VLSEGFF anymore.
       MI.getOperand(1).setReg(RISCV::X0);
     }
@@ -1680,6 +1749,7 @@ bool RISCVInsertVSETVLI::runOnMachineFunction(MachineFunction &MF) {
 
   TII = ST->getInstrInfo();
   MRI = &MF.getRegInfo();
+  LIS = &getAnalysis<LiveIntervals>();
 
   assert(BlockInfo.empty() && "Expect empty block infos");
   BlockInfo.resize(MF.getNumBlockIDs());
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 7b2dcadc4191..5d598a275a00 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -541,9 +541,16 @@ void RISCVPassConfig::addPreRegAlloc() {
   addPass(createRISCVPreRAExpandPseudoPass());
   if (TM->getOptLevel() != CodeGenOptLevel::None)
     addPass(createRISCVMergeBaseOffsetOptPass());
+
   addPass(createRISCVInsertReadWriteCSRPass());
   addPass(createRISCVInsertWriteVXRMPass());
-  addPass(createRISCVInsertVSETVLIPass());
+
+  // Run RISCVInsertVSETVLI after PHI elimination. On O1 and above do it after
+  // register coalescing so needVSETVLIPHI doesn't need to look through COPYs.
+  if (TM->getOptLevel() == CodeGenOptLevel::None)
+    insertPass(&PHIEliminationID, createRISCVInsertVSETVLIPass());
+  else
+    insertPass(&RegisterCoalescerID, createRISCVInsertVSETVLIPass());
 }
 
 void RISCVPassConfig::addFastRegAlloc() {
diff --git a/llvm/test/CodeGen/RISCV/O0-pipeline.ll b/llvm/test/CodeGen/RISCV/O0-pipeline.ll
index c4a7f9562534..3aaa5dc03a7d 100644
--- a/llvm/test/CodeGen/RISCV/O0-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O0-pipeline.ll
@@ -42,12 +42,14 @@
 ; CHECK-NEXT:       RISC-V Pre-RA pseudo instruction expansion pass
 ; CHECK-NEXT:       RISC-V Insert Read/Write CSR Pass
 ; CHECK-NEXT:       RISC-V Insert Write VXRM Pass
-; CHECK-NEXT:       RISC-V Insert VSETVLI pass
 ; CHECK-NEXT:       Init Undef Pass
 ; CHECK-NEXT:       Eliminate PHI nodes for register allocation
+; CHECK-NEXT:       MachineDominator Tree Construction
+; CHECK-NEXT:       Slot index numbering
+; CHECK-NEXT:       Live Interval Analysis
+; CHECK-NEXT:       RISC-V Insert VSETVLI pass
 ; CHECK-NEXT:       Two-Address instruction pass
 ; CHECK-NEXT:       Fast Register Allocator
-; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Slot index numbering
 ; CHECK-NEXT:       Live Interval Analysis
 ; CHECK-NEXT:       RISC-V Coalesce VSETVLI pass
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index 4a71d3276d26..52634b2a8162 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -117,7 +117,6 @@
 ; CHECK-NEXT:       RISC-V Merge Base Offset
 ; CHECK-NEXT:       RISC-V Insert Read/Write CSR Pass
 ; CHECK-NEXT:       RISC-V Insert Write VXRM Pass
-; CHECK-NEXT:       RISC-V Insert VSETVLI pass
 ; CHECK-NEXT:       Detect Dead Lanes
 ; CHECK-NEXT:       Init Undef Pass
 ; CHECK-NEXT:       Process Implicit Definitions
@@ -129,6 +128,7 @@
 ; CHECK-NEXT:       Slot index numbering
 ; CHECK-NEXT:       Live Interval Analysis
 ; CHECK-NEXT:       Register Coalescer
+; CHECK-NEXT:       RISC-V Insert VSETVLI pass
 ; CHECK-NEXT:       Rename Disconnected Subregister Components
 ; CHECK-NEXT:       Machine Instruction Scheduler
 ; CHECK-NEXT:       Machine Block Frequency Analysis
diff --git a/llvm/test/CodeGen/RISCV/rvv/combine-vmv.ll b/llvm/test/CodeGen/RISCV/rvv/combine-vmv.ll
index 682ad5768672..61acf1afa94d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/combine-vmv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/combine-vmv.ll
@@ -36,8 +36,8 @@ define <vscale x 4 x i32> @vadd_undef(<vscale x 4 x i32> %a, <vscale x 4 x i32>
 define <vscale x 4 x i32> @vadd_same_passthru(<vscale x 4 x i32> %passthru, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl1, iXLen %vl2) {
 ; CHECK-LABEL: vadd_same_passthru:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    vmv2r.v v14, v8
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, tu, ma
 ; CHECK-NEXT:    vadd.vv v14, v10, v12
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; CHECK-NEXT:    vmv.v.v v8, v14
diff --git a/llvm/test/CodeGen/RISCV/rvv/concat-vectors-constant-stride.ll b/llvm/test/CodeGen/RISCV/rvv/concat-vectors-constant-stride.ll
index ff35043dbd7e..c6b84209a875 100644
--- a/llvm/test/CodeGen/RISCV/rvv/concat-vectors-constant-stride.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/concat-vectors-constant-stride.ll
@@ -149,8 +149,8 @@ define void @constant_zero_stride(ptr %s, ptr %d) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v8
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslideup.vi v9, v8, 2
 ; CHECK-NEXT:    vse8.v v9, (a1)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/dont-sink-splat-operands.ll b/llvm/test/CodeGen/RISCV/rvv/dont-sink-splat-operands.ll
index dc4d28819bbb..2b4b8e979f3d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/dont-sink-splat-operands.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/dont-sink-splat-operands.ll
@@ -141,9 +141,9 @@ define void @sink_splat_add_scalable(ptr nocapture %a, i32 signext %x) {
 ; SINK-NEXT:    andi a4, a3, 1024
 ; SINK-NEXT:    xori a3, a4, 1024
 ; SINK-NEXT:    slli a5, a5, 1
-; SINK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; SINK-NEXT:    mv a6, a0
 ; SINK-NEXT:    mv a7, a3
+; SINK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; SINK-NEXT:  .LBB1_3: # %vector.body
 ; SINK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SINK-NEXT:    vl2re32.v v8, (a6)
@@ -183,9 +183,9 @@ define void @sink_splat_add_scalable(ptr nocapture %a, i32 signext %x) {
 ; DEFAULT-NEXT:    andi a4, a3, 1024
 ; DEFAULT-NEXT:    xori a3, a4, 1024
 ; DEFAULT-NEXT:    slli a5, a5, 1
-; DEFAULT-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; DEFAULT-NEXT:    mv a6, a0
 ; DEFAULT-NEXT:    mv a7, a3
+; DEFAULT-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; DEFAULT-NEXT:  .LBB1_3: # %vector.body
 ; DEFAULT-NEXT:    # =>This Inner Loop Header: Depth=1
 ; DEFAULT-NEXT:    vl2re32.v v8, (a6)
@@ -459,9 +459,9 @@ define void @sink_splat_fadd_scalable(ptr nocapture %a, float %x) {
 ; SINK-NEXT:    addi a3, a2, -1
 ; SINK-NEXT:    andi a4, a3, 1024
 ; SINK-NEXT:    xori a3, a4, 1024
-; SINK-NEXT:    vsetvli a5, zero, e32, m1, ta, ma
 ; SINK-NEXT:    mv a5, a0
 ; SINK-NEXT:    mv a6, a3
+; SINK-NEXT:    vsetvli a7, zero, e32, m1, ta, ma
 ; SINK-NEXT:  .LBB4_3: # %vector.body
 ; SINK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SINK-NEXT:    vl1re32.v v8, (a5)
@@ -500,9 +500,9 @@ define void @sink_splat_fadd_scalable(ptr nocapture %a, float %x) {
 ; DEFAULT-NEXT:    addi a3, a2, -1
 ; DEFAULT-NEXT:    andi a4, a3, 1024
 ; DEFAULT-NEXT:    xori a3, a4, 1024
-; DEFAULT-NEXT:    vsetvli a5, zero, e32, m1, ta, ma
 ; DEFAULT-NEXT:    mv a5, a0
 ; DEFAULT-NEXT:    mv a6, a3
+; DEFAULT-NEXT:    vsetvli a7, zero, e32, m1, ta, ma
 ; DEFAULT-NEXT:  .LBB4_3: # %vector.body
 ; DEFAULT-NEXT:    # =>This Inner Loop Header: Depth=1
 ; DEFAULT-NEXT:    vl1re32.v v8, (a5)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
index 03e99baf91c0..635869904832 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
@@ -1155,8 +1155,8 @@ define void @mulhu_v8i16(ptr %x) {
 ; CHECK-NEXT:    vle16.v v8, (a0)
 ; CHECK-NEXT:    vmv.v.i v9, 0
 ; CHECK-NEXT:    lui a1, 1048568
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, tu, ma
 ; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v10, a1
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v11, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index 539a8403c935..f42f32e24658 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -12092,8 +12092,8 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64V:       # %bb.0:
 ; RV64V-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV64V-NEXT:    vsext.vf8 v16, v8
-; RV64V-NEXT:    vsetvli zero, zero, e8, m1, ta, mu
 ; RV64V-NEXT:    vmv1r.v v12, v10
+; RV64V-NEXT:    vsetvli zero, zero, e8, m1, ta, mu
 ; RV64V-NEXT:    vluxei64.v v12, (a0), v16, v0.t
 ; RV64V-NEXT:    vsetivli zero, 16, e8, m2, ta, ma
 ; RV64V-NEXT:    vslidedown.vi v10, v10, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
index 175a3ee43f33..d1fb30c7daa3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-vslide1up.ll
@@ -369,8 +369,8 @@ define <4 x i8> @vslide1up_4xi8_neg_incorrect_insert3(<4 x i8> %v, i8 %b) {
 define <2 x i8> @vslide1up_4xi8_neg_length_changing(<4 x i8> %v, i8 %b) {
 ; CHECK-LABEL: vslide1up_4xi8_neg_length_changing:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
 ; CHECK-NEXT:    vmv1r.v v9, v8
+; CHECK-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v9, a0
 ; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslideup.vi v9, v8, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll
index f0fcc482e220..0e6b03bf1632 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll
@@ -168,8 +168,8 @@ define void @strided_constant_0(ptr %x, ptr %z) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v9, v8
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vi v9, v8, 4
 ; CHECK-NEXT:    vse16.v v9, (a1)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll
index c38406bafa8a..64ad86db0495 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll
@@ -62,8 +62,8 @@ define void @gather_masked(ptr noalias nocapture %A, ptr noalias nocapture reado
 ; CHECK-NEXT:    li a4, 5
 ; CHECK-NEXT:  .LBB1_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
 ; CHECK-NEXT:    vmv1r.v v9, v8
+; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, mu
 ; CHECK-NEXT:    vlse8.v v9, (a1), a4, v0.t
 ; CHECK-NEXT:    vle8.v v10, (a0)
 ; CHECK-NEXT:    vadd.vv v9, v10, v9
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll
index 4f16ce28bbb7..9fa8ab39723f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll
@@ -394,7 +394,6 @@ define <128 x i32> @vtrunc_v128i32_v128i64(<128 x i64> %a, <128 x i1> %m, i32 ze
 ; CHECK-NEXT:  # %bb.11:
 ; CHECK-NEXT:    li a1, 32
 ; CHECK-NEXT:  .LBB16_12:
-; CHECK-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
 ; CHECK-NEXT:    csrr a4, vlenb
 ; CHECK-NEXT:    li a5, 24
 ; CHECK-NEXT:    mul a4, a4, a5
@@ -402,6 +401,7 @@ define <128 x i32> @vtrunc_v128i32_v128i64(<128 x i64> %a, <128 x i1> %m, i32 ze
 ; CHECK-NEXT:    addi a4, a4, 16
 ; CHECK-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vmv4r.v v24, v8
+; CHECK-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
 ; CHECK-NEXT:    csrr a4, vlenb
 ; CHECK-NEXT:    li a5, 56
 ; CHECK-NEXT:    mul a4, a4, a5
diff --git a/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll b/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll
index 79b1e14b774a..c8bed2de754b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fold-scalar-load-crash.ll
@@ -15,8 +15,8 @@ define i32 @test(i32 %size, ptr %add.ptr, i64 %const) {
 ; RV32-NEXT:  .LBB0_1: # %for.body
 ; RV32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32-NEXT:    vmv.s.x v9, zero
-; RV32-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
 ; RV32-NEXT:    vmv1r.v v10, v8
+; RV32-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
 ; RV32-NEXT:    vslideup.vx v10, v9, a2
 ; RV32-NEXT:    vsetivli zero, 8, e8, mf2, tu, ma
 ; RV32-NEXT:    vmv.s.x v10, a0
@@ -40,8 +40,8 @@ define i32 @test(i32 %size, ptr %add.ptr, i64 %const) {
 ; RV64-NEXT:  .LBB0_1: # %for.body
 ; RV64-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV64-NEXT:    vmv.s.x v9, zero
-; RV64-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
 ; RV64-NEXT:    vmv1r.v v10, v8
+; RV64-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
 ; RV64-NEXT:    vslideup.vx v10, v9, a2
 ; RV64-NEXT:    vsetivli zero, 8, e8, mf2, tu, ma
 ; RV64-NEXT:    vmv.s.x v10, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
index a6b2d3141f22..bb28ff5c6dc4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
@@ -479,11 +479,11 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) {
 ; CHECK-V-NEXT:    addi a0, sp, 16
 ; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
 ; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclip.wi v8, v10, 0
@@ -640,11 +640,11 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) {
 ; CHECK-V-NEXT:    addi a0, sp, 16
 ; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
 ; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
@@ -811,11 +811,11 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) {
 ; CHECK-V-NEXT:    addi a0, sp, 16
 ; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
 ; CHECK-V-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    vslideup.vi v8, v10, 2
 ; CHECK-V-NEXT:    li a0, -1
 ; CHECK-V-NEXT:    srli a0, a0, 32
@@ -3850,11 +3850,11 @@ define <4 x i32> @stest_f16i32_mm(<4 x half> %x) {
 ; CHECK-V-NEXT:    addi a0, sp, 16
 ; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
 ; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclip.wi v8, v10, 0
@@ -4009,11 +4009,11 @@ define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) {
 ; CHECK-V-NEXT:    addi a0, sp, 16
 ; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
 ; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
@@ -4179,11 +4179,11 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) {
 ; CHECK-V-NEXT:    addi a0, sp, 16
 ; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
 ; CHECK-V-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-V-NEXT:    vslideup.vi v8, v10, 2
 ; CHECK-V-NEXT:    li a0, -1
 ; CHECK-V-NEXT:    srli a0, a0, 32
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll
index 129fbcfb8832..e73415ac0085 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-vector-csr.ll
@@ -21,8 +21,8 @@ define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double
 ; SPILL-O0-NEXT:    add a1, sp, a1
 ; SPILL-O0-NEXT:    addi a1, a1, 16
 ; SPILL-O0-NEXT:    vs1r.v v9, (a1) # Unknown-size Folded Spill
-; SPILL-O0-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8
+; SPILL-O0-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; SPILL-O0-NEXT:    vfadd.vv v8, v9, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
 ; SPILL-O0-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
@@ -37,8 +37,8 @@ define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double
 ; SPILL-O0-NEXT:    vl1r.v v9, (a1) # Unknown-size Folded Reload
 ; SPILL-O0-NEXT:    # kill: def $x11 killed $x10
 ; SPILL-O0-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
-; SPILL-O0-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8
+; SPILL-O0-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; SPILL-O0-NEXT:    vfadd.vv v8, v9, v10
 ; SPILL-O0-NEXT:    csrr a0, vlenb
 ; SPILL-O0-NEXT:    slli a0, a0, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
index e7913fc53df0..9f4718db1dfc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
@@ -13,8 +13,8 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8_v9
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -90,8 +90,8 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8_v9
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -167,8 +167,8 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -247,8 +247,8 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 2
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8m4_v12m4
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv4r.v v8, v12
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -327,8 +327,8 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2_v12m2
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vlseg3e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-vector-csr.ll b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-vector-csr.ll
index 34eb58ee4d1c..483f689cf633 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-vector-csr.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-vector-csr.ll
@@ -24,8 +24,8 @@ define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double
 ; SPILL-O0-NEXT:    add a1, sp, a1
 ; SPILL-O0-NEXT:    addi a1, a1, 32
 ; SPILL-O0-NEXT:    vs1r.v v9, (a1) # Unknown-size Folded Spill
-; SPILL-O0-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8
+; SPILL-O0-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; SPILL-O0-NEXT:    vfadd.vv v8, v9, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 32
 ; SPILL-O0-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
@@ -40,8 +40,8 @@ define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double
 ; SPILL-O0-NEXT:    vl1r.v v9, (a1) # Unknown-size Folded Reload
 ; SPILL-O0-NEXT:    # kill: def $x11 killed $x10
 ; SPILL-O0-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
-; SPILL-O0-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8
+; SPILL-O0-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; SPILL-O0-NEXT:    vfadd.vv v8, v9, v10
 ; SPILL-O0-NEXT:    csrr a0, vlenb
 ; SPILL-O0-NEXT:    slli a0, a0, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
index dd575b3fceb5..4ea9fab6a5a0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
@@ -13,8 +13,8 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8_v9
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, mf2, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -90,8 +90,8 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8_v9
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv1r.v v8, v9
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -167,8 +167,8 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -247,8 +247,8 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 2
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8m4_v12m4
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
 ; SPILL-O0-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv4r.v v8, v12
 ; SPILL-O0-NEXT:    addi a0, sp, 16
@@ -327,8 +327,8 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O0-NEXT:    csrr a2, vlenb
 ; SPILL-O0-NEXT:    slli a2, a2, 1
 ; SPILL-O0-NEXT:    sub sp, sp, a2
-; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O0-NEXT:    # implicit-def: $v8m2_v10m2_v12m2
+; SPILL-O0-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
 ; SPILL-O0-NEXT:    vlseg3e32.v v8, (a0)
 ; SPILL-O0-NEXT:    vmv2r.v v8, v10
 ; SPILL-O0-NEXT:    addi a0, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
index 1a3a1a6c1ee6..743016a7cbcd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
@@ -893,10 +893,10 @@ define void @test_dag_loop() {
 ; CHECK-LABEL: test_dag_loop:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
-; CHECK-NEXT:    vmclr.m v0
 ; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmclr.m v0
+; CHECK-NEXT:    vmv.v.i v12, 0
 ; CHECK-NEXT:    vsetivli zero, 0, e8, m4, tu, mu
-; CHECK-NEXT:    vmv4r.v v12, v8
 ; CHECK-NEXT:    vssubu.vx v12, v8, zero, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmseq.vv v0, v12, v8
@@ -942,8 +942,8 @@ declare <vscale x 2 x i32> @llvm.riscv.vredsum.nxv2i32.nxv2i32(
 define <vscale x 2 x i32> @vredsum(<vscale x 2 x i32> %passthru, <vscale x 2 x i32> %x, <vscale x 2 x i32> %y, <vscale x 2 x i1> %m, i64 %vl) {
 ; CHECK-LABEL: vredsum:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v11, v8
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vredsum.vs v11, v9, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, tu, ma
 ; CHECK-NEXT:    vmerge.vvm v8, v8, v11, v0
@@ -967,8 +967,8 @@ define <vscale x 2 x float> @vfredusum(<vscale x 2 x float> %passthru, <vscale x
 ; CHECK-LABEL: vfredusum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fsrmi a1, 0
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v11, v8
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
 ; CHECK-NEXT:    vfredusum.vs v11, v9, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, tu, ma
 ; CHECK-NEXT:    vmerge.vvm v8, v8, v11, v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
index 618672344fe7..8a297db7a3b8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
@@ -255,9 +255,9 @@ define void @sink_splat_mul_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB7_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -346,9 +346,9 @@ define void @sink_splat_add_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB8_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -437,9 +437,9 @@ define void @sink_splat_sub_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB9_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -528,9 +528,9 @@ define void @sink_splat_rsub_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB10_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -619,9 +619,9 @@ define void @sink_splat_and_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB11_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -710,9 +710,9 @@ define void @sink_splat_or_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB12_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -801,9 +801,9 @@ define void @sink_splat_xor_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB13_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -994,9 +994,9 @@ define void @sink_splat_shl_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB17_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -1085,9 +1085,9 @@ define void @sink_splat_lshr_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB18_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -1176,9 +1176,9 @@ define void @sink_splat_ashr_scalable(ptr nocapture %a) {
 ; CHECK-NEXT:    andi a3, a1, 1024
 ; CHECK-NEXT:    xori a1, a3, 1024
 ; CHECK-NEXT:    slli a4, a4, 1
-; CHECK-NEXT:    vsetvli a5, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a5, a0
 ; CHECK-NEXT:    mv a6, a1
+; CHECK-NEXT:    vsetvli a7, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB19_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a5)
@@ -1468,9 +1468,9 @@ define void @sink_splat_fmul_scalable(ptr nocapture %a, float %x) {
 ; CHECK-NEXT:    addi a3, a2, -1
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
-; CHECK-NEXT:    vsetvli a5, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    mv a5, a0
 ; CHECK-NEXT:    mv a6, a3
+; CHECK-NEXT:    vsetvli a7, zero, e32, m1, ta, ma
 ; CHECK-NEXT:  .LBB26_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl1re32.v v8, (a5)
@@ -1558,9 +1558,9 @@ define void @sink_splat_fdiv_scalable(ptr nocapture %a, float %x) {
 ; CHECK-NEXT:    addi a3, a2, -1
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
-; CHECK-NEXT:    vsetvli a5, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    mv a5, a0
 ; CHECK-NEXT:    mv a6, a3
+; CHECK-NEXT:    vsetvli a7, zero, e32, m1, ta, ma
 ; CHECK-NEXT:  .LBB27_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl1re32.v v8, (a5)
@@ -1648,9 +1648,9 @@ define void @sink_splat_frdiv_scalable(ptr nocapture %a, float %x) {
 ; CHECK-NEXT:    addi a3, a2, -1
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
-; CHECK-NEXT:    vsetvli a5, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    mv a5, a0
 ; CHECK-NEXT:    mv a6, a3
+; CHECK-NEXT:    vsetvli a7, zero, e32, m1, ta, ma
 ; CHECK-NEXT:  .LBB28_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl1re32.v v8, (a5)
@@ -1738,9 +1738,9 @@ define void @sink_splat_fadd_scalable(ptr nocapture %a, float %x) {
 ; CHECK-NEXT:    addi a3, a2, -1
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
-; CHECK-NEXT:    vsetvli a5, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    mv a5, a0
 ; CHECK-NEXT:    mv a6, a3
+; CHECK-NEXT:    vsetvli a7, zero, e32, m1, ta, ma
 ; CHECK-NEXT:  .LBB29_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl1re32.v v8, (a5)
@@ -1828,9 +1828,9 @@ define void @sink_splat_fsub_scalable(ptr nocapture %a, float %x) {
 ; CHECK-NEXT:    addi a3, a2, -1
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
-; CHECK-NEXT:    vsetvli a5, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    mv a5, a0
 ; CHECK-NEXT:    mv a6, a3
+; CHECK-NEXT:    vsetvli a7, zero, e32, m1, ta, ma
 ; CHECK-NEXT:  .LBB30_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl1re32.v v8, (a5)
@@ -1918,9 +1918,9 @@ define void @sink_splat_frsub_scalable(ptr nocapture %a, float %x) {
 ; CHECK-NEXT:    addi a3, a2, -1
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
-; CHECK-NEXT:    vsetvli a5, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    mv a5, a0
 ; CHECK-NEXT:    mv a6, a3
+; CHECK-NEXT:    vsetvli a7, zero, e32, m1, ta, ma
 ; CHECK-NEXT:  .LBB31_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl1re32.v v8, (a5)
@@ -2084,10 +2084,10 @@ define void @sink_splat_fma_scalable(ptr noalias nocapture %a, ptr noalias nocap
 ; CHECK-NEXT:    addi a4, a3, -1
 ; CHECK-NEXT:    andi a5, a4, 1024
 ; CHECK-NEXT:    xori a4, a5, 1024
-; CHECK-NEXT:    vsetvli a6, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a1
 ; CHECK-NEXT:    mv t0, a4
+; CHECK-NEXT:    vsetvli t1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:  .LBB34_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl1re32.v v8, (a6)
@@ -2184,10 +2184,10 @@ define void @sink_splat_fma_commute_scalable(ptr noalias nocapture %a, ptr noali
 ; CHECK-NEXT:    addi a4, a3, -1
 ; CHECK-NEXT:    andi a5, a4, 1024
 ; CHECK-NEXT:    xori a4, a5, 1024
-; CHECK-NEXT:    vsetvli a6, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a1
 ; CHECK-NEXT:    mv t0, a4
+; CHECK-NEXT:    vsetvli t1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:  .LBB35_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl1re32.v v8, (a6)
@@ -2498,9 +2498,9 @@ define void @sink_splat_udiv_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB42_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -2589,9 +2589,9 @@ define void @sink_splat_sdiv_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB43_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -2680,9 +2680,9 @@ define void @sink_splat_urem_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB44_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
@@ -2771,9 +2771,9 @@ define void @sink_splat_srem_scalable(ptr nocapture %a, i32 signext %x) {
 ; CHECK-NEXT:    andi a4, a3, 1024
 ; CHECK-NEXT:    xori a3, a4, 1024
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    vsetvli a6, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    mv a6, a0
 ; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    vsetvli t0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:  .LBB45_3: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vl2re32.v v8, (a6)
diff --git a/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll b/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll
index f41a3ec72aed..48c30596ad51 100644
--- a/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll
@@ -161,8 +161,8 @@ declare <vscale x 8 x i8> @llvm.riscv.vrgatherei16.vv.nxv8i8.i64(<vscale x 8 x i
 define void @repeat_shuffle(<2 x double> %v, ptr noalias %q) {
 ; CHECK-LABEL: repeat_shuffle:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vmv2r.v v10, v8
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v10, v8, 2
 ; CHECK-NEXT:    vse64.v v10, (a0)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll b/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
index 25e3468dcb62..439301ff4011 100644
--- a/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/unmasked-tu.ll
@@ -711,8 +711,8 @@ define <vscale x 1 x i64> @intrinsic_vslide1down_vx_nxv1i64_nxv1i64_i64(<vscale
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    vsetvli a2, a2, e64, m1, ta, ma
 ; RV32-NEXT:    slli a2, a2, 1
-; RV32-NEXT:    vsetvli zero, a2, e32, m1, tu, ma
 ; RV32-NEXT:    vmv1r.v v10, v8
+; RV32-NEXT:    vsetvli zero, a2, e32, m1, tu, ma
 ; RV32-NEXT:    vslide1down.vx v10, v9, a0
 ; RV32-NEXT:    vslide1down.vx v8, v10, a1
 ; RV32-NEXT:    ret
@@ -743,8 +743,8 @@ define <vscale x 1 x i64> @intrinsic_vslide1up_vx_nxv1i64_nxv1i64_i64(<vscale x
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    vsetvli a2, a2, e64, m1, ta, ma
 ; RV32-NEXT:    slli a2, a2, 1
-; RV32-NEXT:    vsetvli zero, a2, e32, m1, tu, ma
 ; RV32-NEXT:    vmv1r.v v10, v8
+; RV32-NEXT:    vsetvli zero, a2, e32, m1, tu, ma
 ; RV32-NEXT:    vslide1up.vx v10, v9, a1
 ; RV32-NEXT:    vslide1up.vx v8, v10, a0
 ; RV32-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-reassociations.ll b/llvm/test/CodeGen/RISCV/rvv/vector-reassociations.ll
index 6435c1c14e06..79bd60d1702f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-reassociations.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-reassociations.ll
@@ -120,8 +120,8 @@ entry:
 define <vscale x 1 x i8> @vadd_vv_passthru(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2) nounwind {
 ; CHECK-LABEL: vadd_vv_passthru:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vmv1r.v v10, v8
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vadd.vv v10, v8, v9
 ; CHECK-NEXT:    vmv1r.v v9, v8
 ; CHECK-NEXT:    vadd.vv v9, v8, v8
@@ -152,8 +152,8 @@ entry:
 define <vscale x 1 x i8> @vadd_vv_passthru_negative(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2) nounwind {
 ; CHECK-LABEL: vadd_vv_passthru_negative:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vmv1r.v v10, v8
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; CHECK-NEXT:    vadd.vv v10, v8, v9
 ; CHECK-NEXT:    vadd.vv v9, v8, v10
 ; CHECK-NEXT:    vadd.vv v8, v8, v9
@@ -183,8 +183,8 @@ entry:
 define <vscale x 1 x i8> @vadd_vv_mask(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2, <vscale x 1 x i1> %m) nounwind {
 ; CHECK-LABEL: vadd_vv_mask:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v10, v8
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vadd.vv v10, v8, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v9, v8
 ; CHECK-NEXT:    vadd.vv v9, v8, v8, v0.t
@@ -218,8 +218,8 @@ entry:
 define <vscale x 1 x i8> @vadd_vv_mask_negative(<vscale x 1 x i8> %0, <vscale x 1 x i8> %1, i32 %2, <vscale x 1 x i1> %m, <vscale x 1 x i1> %m2) nounwind {
 ; CHECK-LABEL: vadd_vv_mask_negative:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vmv1r.v v11, v8
+; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, mu
 ; CHECK-NEXT:    vadd.vv v11, v8, v9, v0.t
 ; CHECK-NEXT:    vmv1r.v v9, v8
 ; CHECK-NEXT:    vadd.vv v9, v8, v11, v0.t
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwmacc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfwmacc-vp.ll
index fab76ac56458..78f3792dbaf0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfwmacc-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfwmacc-vp.ll
@@ -85,8 +85,8 @@ define <vscale x 1 x float> @vfmacc_vv_nxv1f32_tu(<vscale x 1 x half> %a, <vscal
 define <vscale x 1 x float> @vfmacc_vv_nxv1f32_masked__tu(<vscale x 1 x half> %a, <vscale x 1 x half> %b, <vscale x 1 x float> %c, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: vfmacc_vv_nxv1f32_masked__tu:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vmv1r.v v11, v10
+; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vfwmacc.vv v11, v8, v9, v0.t
 ; ZVFH-NEXT:    vsetvli zero, zero, e32, mf2, tu, ma
 ; ZVFH-NEXT:    vmerge.vvm v10, v10, v11, v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrgatherei16-subreg-liveness.ll b/llvm/test/CodeGen/RISCV/rvv/vrgatherei16-subreg-liveness.ll
index 0c0a3dc9675b..462d49991ae4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrgatherei16-subreg-liveness.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrgatherei16-subreg-liveness.ll
@@ -16,14 +16,14 @@ define internal void @foo(<vscale x 1 x i16> %v15, <vscale x 1 x i16> %0, <vscal
 ; NOSUBREG-NEXT:    vmv.v.i v14, 0
 ; NOSUBREG-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
 ; NOSUBREG-NEXT:    vmv.v.i v9, 0
+; NOSUBREG-NEXT:    vmv.v.i v8, 0
 ; NOSUBREG-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
-; NOSUBREG-NEXT:    vmv1r.v v8, v9
 ; NOSUBREG-NEXT:    vrgatherei16.vv v8, v9, v14
 ; NOSUBREG-NEXT:  .LBB0_1: # %loopIR3.i.i
 ; NOSUBREG-NEXT:    # =>This Inner Loop Header: Depth=1
 ; NOSUBREG-NEXT:    vl1r.v v9, (zero)
-; NOSUBREG-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
 ; NOSUBREG-NEXT:    vmv1r.v v13, v12
+; NOSUBREG-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
 ; NOSUBREG-NEXT:    vrgatherei16.vv v13, v9, v10
 ; NOSUBREG-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; NOSUBREG-NEXT:    vand.vv v9, v8, v13
@@ -36,14 +36,14 @@ define internal void @foo(<vscale x 1 x i16> %v15, <vscale x 1 x i16> %0, <vscal
 ; SUBREG-NEXT:    vmv.v.i v14, 0
 ; SUBREG-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
 ; SUBREG-NEXT:    vmv.v.i v9, 0
+; SUBREG-NEXT:    vmv.v.i v8, 0
 ; SUBREG-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
-; SUBREG-NEXT:    vmv1r.v v8, v9
 ; SUBREG-NEXT:    vrgatherei16.vv v8, v9, v14
 ; SUBREG-NEXT:  .LBB0_1: # %loopIR3.i.i
 ; SUBREG-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SUBREG-NEXT:    vl1r.v v9, (zero)
-; SUBREG-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
 ; SUBREG-NEXT:    vmv1r.v v13, v12
+; SUBREG-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
 ; SUBREG-NEXT:    vrgatherei16.vv v13, v9, v10
 ; SUBREG-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; SUBREG-NEXT:    vand.vv v9, v8, v13
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.ll
index 088d121564bc..25aa3a7081a1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.ll
@@ -91,13 +91,11 @@ define <vscale x 1 x double> @test3(i64 %avl, i8 zeroext %cond, <vscale x 1 x do
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vfadd.vv v9, v8, v9
 ; CHECK-NEXT:    vfmul.vv v8, v9, v8
-; CHECK-NEXT:    # implicit-def: $x10
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB2_2: # %if.else
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    vfsub.vv v9, v8, v9
 ; CHECK-NEXT:    vfmul.vv v8, v9, v8
-; CHECK-NEXT:    # implicit-def: $x10
 ; CHECK-NEXT:    ret
 entry:
   %tobool = icmp eq i8 %cond, 0
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
index ef834403fb4f..295d4c57a1be 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc %s -o - -mtriple=riscv64 -mattr=v \
-# RUN:     -run-pass=riscv-insert-vsetvli | FileCheck %s
+# RUN:     -run-pass=phi-node-elimination,register-coalescer,riscv-insert-vsetvli | FileCheck %s
 
 --- |
   source_filename = "vsetvli-insert.ll"
@@ -191,8 +191,7 @@ body:             |
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLI [[COPY]], 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
   ; CHECK-NEXT:   [[PseudoVLE64_V_M1_:%[0-9]+]]:vr = PseudoVLE64_V_M1 undef $noreg, [[COPY2]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY $x0
-  ; CHECK-NEXT:   BEQ [[COPY3]], [[COPY4]], %bb.2
+  ; CHECK-NEXT:   BEQ [[COPY3]], $x0, %bb.2
   ; CHECK-NEXT:   PseudoBR %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.if.then:
@@ -204,11 +203,10 @@ body:             |
   ; CHECK-NEXT: bb.2.if.else:
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[PseudoVSUB_VV_M1_:%[0-9]+]]:vr = PseudoVSUB_VV_M1 undef $noreg, [[PseudoVLE64_V_M1_]], [[COPY1]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVSUB_VV_M1 undef $noreg, [[PseudoVLE64_V_M1_]], [[COPY1]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3.if.end:
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:vr = PHI [[PseudoVADD_VV_M1_]], %bb.1, [[PseudoVSUB_VV_M1_]], %bb.2
-  ; CHECK-NEXT:   $v8 = COPY [[PHI]]
+  ; CHECK-NEXT:   $v8 = COPY [[PseudoVADD_VV_M1_]]
   ; CHECK-NEXT:   PseudoRET implicit $v8
   bb.0.entry:
     successors: %bb.2(0x30000000), %bb.1(0x50000000)
@@ -270,26 +268,24 @@ body:             |
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLI [[COPY]], 215 /* e32, mf2, ta, ma */, implicit-def $vl, implicit-def $vtype
   ; CHECK-NEXT:   [[PseudoVLE32_V_MF2_:%[0-9]+]]:vr = PseudoVLE32_V_MF2 undef $noreg, [[COPY2]], $noreg, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY $x0
-  ; CHECK-NEXT:   BEQ [[COPY3]], [[COPY4]], %bb.2
+  ; CHECK-NEXT:   BEQ [[COPY3]], $x0, %bb.2
   ; CHECK-NEXT:   PseudoBR %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.if.then:
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype, implicit $vl
-  ; CHECK-NEXT:   early-clobber %1:vr = PseudoVZEXT_VF2_M1 undef $noreg, [[PseudoVLE32_V_MF2_]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   early-clobber %9:vr = PseudoVZEXT_VF2_M1 undef $noreg, [[PseudoVLE32_V_MF2_]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   PseudoBR %bb.3
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.if.else:
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype, implicit $vl
-  ; CHECK-NEXT:   early-clobber %2:vr = PseudoVSEXT_VF2_M1 undef $noreg, [[PseudoVLE32_V_MF2_]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   early-clobber %9:vr = PseudoVSEXT_VF2_M1 undef $noreg, [[PseudoVLE32_V_MF2_]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3.if.end:
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:vr = PHI %1, %bb.1, %2, %bb.2
-  ; CHECK-NEXT:   PseudoVSE64_V_M1 [[PHI]], [[COPY1]], $noreg, 6 /* e64 */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   PseudoVSE64_V_M1 %9, [[COPY1]], $noreg, 6 /* e64 */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   PseudoRET
   bb.0.entry:
     successors: %bb.2(0x30000000), %bb.1(0x50000000)
@@ -349,8 +345,7 @@ body:             |
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr = COPY $v9
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vr = COPY $v8
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr = COPY $x10
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY $x0
-  ; CHECK-NEXT:   BEQ [[COPY3]], [[COPY4]], %bb.2
+  ; CHECK-NEXT:   BEQ [[COPY3]], $x0, %bb.2
   ; CHECK-NEXT:   PseudoBR %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.if.then:
@@ -364,11 +359,10 @@ body:             |
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLI [[COPY]], 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
-  ; CHECK-NEXT:   [[PseudoVSUB_VV_M1_:%[0-9]+]]:vr = PseudoVSUB_VV_M1 undef $noreg, [[COPY1]], [[COPY1]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVSUB_VV_M1 undef $noreg, [[COPY1]], [[COPY1]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3.if.end:
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:vr = PHI [[PseudoVADD_VV_M1_]], %bb.1, [[PseudoVSUB_VV_M1_]], %bb.2
-  ; CHECK-NEXT:   [[PseudoVMV_X_S:%[0-9]+]]:gpr = PseudoVMV_X_S [[PHI]], 6 /* e64 */, implicit $vtype
+  ; CHECK-NEXT:   [[PseudoVMV_X_S:%[0-9]+]]:gpr = PseudoVMV_X_S [[PseudoVADD_VV_M1_]], 6 /* e64 */, implicit $vtype
   ; CHECK-NEXT:   $x10 = COPY [[PseudoVMV_X_S]]
   ; CHECK-NEXT:   PseudoRET implicit $x10
   bb.0.entry:
@@ -429,9 +423,8 @@ body:             |
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr = COPY $v9
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vr = COPY $v8
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr = COPY $x10
-  ; CHECK-NEXT:   [[PseudoVSETVLI:%[0-9]+]]:gprnox0 = PseudoVSETVLI [[COPY]], 88 /* e64, m1, ta, mu */, implicit-def $vl, implicit-def $vtype
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY $x0
-  ; CHECK-NEXT:   BEQ [[COPY3]], [[COPY4]], %bb.2
+  ; CHECK-NEXT:   dead [[PseudoVSETVLI:%[0-9]+]]:gprnox0 = PseudoVSETVLI [[COPY]], 88 /* e64, m1, ta, mu */, implicit-def $vl, implicit-def $vtype
+  ; CHECK-NEXT:   BEQ [[COPY3]], $x0, %bb.2
   ; CHECK-NEXT:   PseudoBR %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.if.then:
@@ -443,11 +436,10 @@ body:             |
   ; CHECK-NEXT: bb.2.if.else:
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[PseudoVSUB_VV_M1_:%[0-9]+]]:vr = PseudoVSUB_VV_M1 undef $noreg, [[COPY2]], [[COPY1]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVSUB_VV_M1 undef $noreg, [[COPY2]], [[COPY1]], $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3.if.end:
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:vr = PHI [[PseudoVADD_VV_M1_]], %bb.1, [[PseudoVSUB_VV_M1_]], %bb.2
-  ; CHECK-NEXT:   $v8 = COPY [[PHI]]
+  ; CHECK-NEXT:   $v8 = COPY [[PseudoVADD_VV_M1_]]
   ; CHECK-NEXT:   PseudoRET implicit $v8
   bb.0.entry:
     successors: %bb.2(0x30000000), %bb.1(0x50000000)
@@ -499,7 +491,6 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
-  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gpr = IMPLICIT_DEF
   ; CHECK-NEXT:   dead [[PseudoVSETVLIX0_:%[0-9]+]]:gpr = PseudoVSETVLIX0 killed $x0, 223 /* e64, mf2, ta, ma */, implicit-def $vl, implicit-def $vtype
   ; CHECK-NEXT:   [[PseudoVID_V_MF2_:%[0-9]+]]:vr = PseudoVID_V_MF2 undef $noreg, -1, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   dead [[PseudoVSETVLIX0_1:%[0-9]+]]:gpr = PseudoVSETVLIX0 killed $x0, 215 /* e32, mf2, ta, ma */, implicit-def $vl, implicit-def $vtype
@@ -508,25 +499,24 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[PseudoVMSEQ_VI_MF2_:%[0-9]+]]:vmv0 = PseudoVMSEQ_VI_MF2 killed [[PseudoVID_V_MF2_]], 0, -1, 5 /* e32 */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[PseudoVMSEQ_VI_MF2_:%[0-9]+]]:vmv0 = PseudoVMSEQ_VI_MF2 [[PseudoVID_V_MF2_]], 0, -1, 5 /* e32 */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   $v0 = COPY [[PseudoVMSEQ_VI_MF2_]]
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 23 /* e32, mf2, tu, mu */, implicit-def $vl, implicit-def $vtype, implicit $vl
-  ; CHECK-NEXT:   [[PseudoVLE32_V_MF2_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_MF2_MASK [[PseudoVMV_V_I_MF2_]], killed [[COPY]], $v0, -1, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[PseudoVLE32_V_MF2_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_MF2_MASK [[PseudoVMV_V_I_MF2_]], [[COPY]], $v0, -1, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 197 /* e8, mf8, ta, ma */, implicit-def $vl, implicit-def $vtype, implicit $vl
   ; CHECK-NEXT:   [[PseudoVCPOP_M_B1_:%[0-9]+]]:gpr = PseudoVCPOP_M_B1 [[PseudoVMSEQ_VI_MF2_]], -1, 0 /* e8 */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x0
-  ; CHECK-NEXT:   BEQ killed [[PseudoVCPOP_M_B1_]], [[COPY2]], %bb.3
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gpr = IMPLICIT_DEF
+  ; CHECK-NEXT:   BEQ [[PseudoVCPOP_M_B1_]], $x0, %bb.3
   ; CHECK-NEXT:   PseudoBR %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[LWU:%[0-9]+]]:gpr = LWU [[COPY1]], 0
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gpr = LWU [[COPY1]], 0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3:
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gpr = PHI [[DEF]], %bb.1, [[LWU]], %bb.2
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 215 /* e32, mf2, ta, ma */, implicit-def $vl, implicit-def $vtype, implicit $vl
-  ; CHECK-NEXT:   [[PseudoVADD_VX_MF2_:%[0-9]+]]:vr = nsw PseudoVADD_VX_MF2 undef $noreg, [[PseudoVLE32_V_MF2_MASK]], [[PHI]], -1, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[PseudoVADD_VX_MF2_:%[0-9]+]]:vr = nsw PseudoVADD_VX_MF2 undef $noreg, [[PseudoVLE32_V_MF2_MASK]], [[DEF]], -1, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   $v0 = COPY [[PseudoVADD_VX_MF2_]]
   ; CHECK-NEXT:   PseudoRET implicit $v0
   bb.0:
@@ -593,13 +583,12 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gpr = PHI [[COPY2]], %bb.0, %10, %bb.1
-  ; CHECK-NEXT:   [[PseudoVADD_VX_M1_:%[0-9]+]]:vr = PseudoVADD_VX_M1 undef $noreg, [[PseudoVID_V_M1_]], [[PHI]], -1, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[MUL:%[0-9]+]]:gpr = MUL [[PHI]], [[SRLI]]
+  ; CHECK-NEXT:   [[PseudoVADD_VX_M1_:%[0-9]+]]:vr = PseudoVADD_VX_M1 undef $noreg, [[PseudoVID_V_M1_]], [[COPY2]], -1, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[MUL:%[0-9]+]]:gpr = MUL [[COPY2]], [[SRLI]]
   ; CHECK-NEXT:   [[ADD:%[0-9]+]]:gpr = ADD [[COPY]], [[MUL]]
-  ; CHECK-NEXT:   PseudoVSE32_V_MF2 killed [[PseudoVADD_VX_M1_]], killed [[ADD]], -1, 5 /* e32 */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[ADDI:%[0-9]+]]:gpr = ADDI [[PHI]], 1
-  ; CHECK-NEXT:   BLTU [[ADDI]], [[COPY1]], %bb.1
+  ; CHECK-NEXT:   PseudoVSE32_V_MF2 [[PseudoVADD_VX_M1_]], [[ADD]], -1, 5 /* e32 */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = ADDI [[COPY2]], 1
+  ; CHECK-NEXT:   BLTU [[COPY2]], [[COPY1]], %bb.1
   ; CHECK-NEXT:   PseudoBR %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
@@ -661,17 +650,16 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gpr = PHI [[COPY2]], %bb.0, %10, %bb.2
-  ; CHECK-NEXT:   [[PseudoVADD_VX_M1_:%[0-9]+]]:vr = PseudoVADD_VX_M1 undef $noreg, [[PseudoVID_V_M1_]], [[PHI]], -1, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[MUL:%[0-9]+]]:gpr = MUL [[PHI]], [[SRLI]]
+  ; CHECK-NEXT:   [[PseudoVADD_VX_M1_:%[0-9]+]]:vr = PseudoVADD_VX_M1 undef $noreg, [[PseudoVID_V_M1_]], [[COPY2]], -1, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[MUL:%[0-9]+]]:gpr = MUL [[COPY2]], [[SRLI]]
   ; CHECK-NEXT:   [[ADD:%[0-9]+]]:gpr = ADD [[COPY]], [[MUL]]
-  ; CHECK-NEXT:   PseudoVSE32_V_MF2 killed [[PseudoVADD_VX_M1_]], killed [[ADD]], -1, 5 /* e32 */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[ADDI:%[0-9]+]]:gpr = ADDI [[PHI]], 1
+  ; CHECK-NEXT:   PseudoVSE32_V_MF2 [[PseudoVADD_VX_M1_]], [[ADD]], -1, 5 /* e32 */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = ADDI [[COPY2]], 1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BLTU [[ADDI]], [[COPY1]], %bb.1
+  ; CHECK-NEXT:   BLTU [[COPY2]], [[COPY1]], %bb.1
   ; CHECK-NEXT:   PseudoBR %bb.3
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3:
@@ -750,31 +738,24 @@ body:             |
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   dead $x0 = PseudoVSETIVLI 4, 208 /* e32, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
   ; CHECK-NEXT:   [[PseudoVMV_V_I_M1_:%[0-9]+]]:vr = PseudoVMV_V_I_M1 undef $noreg, 0, 4, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vr = COPY [[PseudoVMV_V_I_M1_]]
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vr = COPY [[COPY2]]
   ; CHECK-NEXT:   [[LUI:%[0-9]+]]:gpr = LUI 1
-  ; CHECK-NEXT:   [[ADDIW:%[0-9]+]]:gpr = ADDIW killed [[LUI]], -2048
+  ; CHECK-NEXT:   [[ADDIW:%[0-9]+]]:gpr = ADDIW [[LUI]], -2048
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.vector.body:
   ; CHECK-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gpr = PHI [[COPY1]], %bb.0, %5, %bb.1
-  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:gpr = PHI [[ADDIW]], %bb.0, %4, %bb.1
-  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:vr = PHI [[COPY3]], %bb.0, %16, %bb.1
-  ; CHECK-NEXT:   [[PseudoVLE32_V_M1_:%[0-9]+]]:vr = PseudoVLE32_V_M1 undef $noreg, [[PHI]], 4, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype :: (load (s128) from %ir.lsr.iv12, align 4)
-  ; CHECK-NEXT:   [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVADD_VV_M1 undef $noreg, killed [[PseudoVLE32_V_M1_]], [[PHI2]], 4, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[ADDI:%[0-9]+]]:gpr = nsw ADDI [[PHI1]], -4
-  ; CHECK-NEXT:   [[ADDI1:%[0-9]+]]:gpr = ADDI [[PHI]], 16
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY $x0
-  ; CHECK-NEXT:   BNE [[ADDI]], [[COPY4]], %bb.1
+  ; CHECK-NEXT:   [[PseudoVLE32_V_M1_:%[0-9]+]]:vr = PseudoVLE32_V_M1 undef $noreg, [[COPY1]], 4, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype :: (load (s128) from %ir.lsr.iv12, align 4)
+  ; CHECK-NEXT:   [[PseudoVMV_V_I_M1_:%[0-9]+]]:vr = PseudoVADD_VV_M1 undef $noreg, [[PseudoVLE32_V_M1_]], [[PseudoVMV_V_I_M1_]], 4, 5 /* e32 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[ADDIW:%[0-9]+]]:gpr = nsw ADDI [[ADDIW]], -4
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = ADDI [[COPY1]], 16
+  ; CHECK-NEXT:   BNE [[ADDIW]], $x0, %bb.1
   ; CHECK-NEXT:   PseudoBR %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2.middle.block:
-  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:gpr = COPY $x0
-  ; CHECK-NEXT:   [[PseudoVMV_S_X:%[0-9]+]]:vr = PseudoVMV_S_X undef $noreg, [[COPY5]], 1, 5 /* e32 */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[PseudoVREDSUM_VS_M1_E8_:%[0-9]+]]:vr = PseudoVREDSUM_VS_M1_E8 undef $noreg, [[PseudoVADD_VV_M1_]], killed [[PseudoVMV_S_X]], 4, 5 /* e32 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[PseudoVMV_S_X:%[0-9]+]]:vr = PseudoVMV_S_X undef $noreg, $x0, 1, 5 /* e32 */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[PseudoVREDSUM_VS_M1_E8_:%[0-9]+]]:vr = PseudoVREDSUM_VS_M1_E8 undef $noreg, [[PseudoVMV_V_I_M1_]], [[PseudoVMV_S_X]], 4, 5 /* e32 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   dead $x0 = PseudoVSETIVLI 1, 208 /* e32, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
-  ; CHECK-NEXT:   PseudoVSE32_V_M1 killed [[PseudoVREDSUM_VS_M1_E8_]], [[COPY]], 1, 5 /* e32 */, implicit $vl, implicit $vtype :: (store (s32) into %ir.res)
+  ; CHECK-NEXT:   PseudoVSE32_V_M1 [[PseudoVREDSUM_VS_M1_E8_]], [[COPY]], 1, 5 /* e32 */, implicit $vl, implicit $vtype :: (store (s32) into %ir.res)
   ; CHECK-NEXT:   PseudoRET
   bb.0.entry:
     liveins: $x10, $x12
@@ -824,7 +805,7 @@ body:             |
   ; CHECK-NEXT:   %idxs:vr = COPY $v0
   ; CHECK-NEXT:   %t1:vr = COPY $v1
   ; CHECK-NEXT:   %t3:vr = COPY $v2
-  ; CHECK-NEXT:   %t4:vr = COPY $v3
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vrnov0 = COPY $v3
   ; CHECK-NEXT:   %t5:vrnov0 = COPY $v1
   ; CHECK-NEXT:   dead [[PseudoVSETVLIX0_:%[0-9]+]]:gpr = PseudoVSETVLIX0 killed $x0, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
   ; CHECK-NEXT:   %t6:vr = PseudoVMSEQ_VI_M1 %t1, 0, -1, 6 /* e64 */, implicit $vl, implicit $vtype
@@ -834,8 +815,7 @@ body:             |
   ; CHECK-NEXT:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   %mask:vr = PseudoVMANDN_MM_MF8 %t6, %t3, -1, 0 /* e8 */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   %t2:gpr = COPY $x0
-  ; CHECK-NEXT:   BEQ %a, %t2, %bb.3
+  ; CHECK-NEXT:   BEQ %a, $x0, %bb.3
   ; CHECK-NEXT:   PseudoBR %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
@@ -843,15 +823,13 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $v0 = COPY %mask
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 69 /* e8, mf8, ta, mu */, implicit-def $vl, implicit-def $vtype, implicit $vl
-  ; CHECK-NEXT:   early-clobber %t0:vrnov0 = PseudoVLUXEI64_V_M1_MF8_MASK %t5, killed %inaddr, %idxs, $v0, -1, 3 /* e8 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   %ldval:vr = COPY %t0
+  ; CHECK-NEXT:   early-clobber [[COPY]]:vrnov0 = PseudoVLUXEI64_V_M1_MF8_MASK %t5, %inaddr, %idxs, $v0, -1, 3 /* e8 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   PseudoBR %bb.3
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3:
-  ; CHECK-NEXT:   %stval:vr = PHI %t4, %bb.1, %ldval, %bb.2
   ; CHECK-NEXT:   $v0 = COPY %mask
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 197 /* e8, mf8, ta, ma */, implicit-def $vl, implicit-def $vtype, implicit $vl
-  ; CHECK-NEXT:   PseudoVSOXEI64_V_M1_MF8_MASK killed %stval, killed %b, %idxs, $v0, -1, 3 /* e8 */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   PseudoVSOXEI64_V_M1_MF8_MASK [[COPY]], %b, %idxs, $v0, -1, 3 /* e8 */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   PseudoRET
   bb.0:
     successors: %bb.1
@@ -902,12 +880,12 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   %dst:gpr = COPY $x10
   ; CHECK-NEXT:   %src:gpr = COPY $x11
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $x12
+  ; CHECK-NEXT:   dead [[COPY:%[0-9]+]]:gpr = COPY $x12
   ; CHECK-NEXT:   %tc:gpr = COPY $x13
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x14
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x15
   ; CHECK-NEXT:   %vlenb:gpr = PseudoReadVLENB
-  ; CHECK-NEXT:   %inc:gpr = SRLI killed %vlenb, 3
+  ; CHECK-NEXT:   %inc:gpr = SRLI %vlenb, 3
   ; CHECK-NEXT:   dead [[PseudoVSETVLIX0_:%[0-9]+]]:gpr = PseudoVSETVLIX0 killed $x0, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
   ; CHECK-NEXT:   [[PseudoVID_V_M1_:%[0-9]+]]:vr = PseudoVID_V_M1 undef $noreg, -1, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr = COPY $x0
@@ -916,31 +894,29 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gpr = PHI [[COPY3]], %bb.0, %11, %bb.3
-  ; CHECK-NEXT:   [[ADD:%[0-9]+]]:gpr = ADD [[COPY2]], [[PHI]]
+  ; CHECK-NEXT:   [[ADD:%[0-9]+]]:gpr = ADD [[COPY2]], [[COPY3]]
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype, implicit $vl
-  ; CHECK-NEXT:   [[PseudoVADD_VX_M1_:%[0-9]+]]:vr = PseudoVADD_VX_M1 undef $noreg, [[PseudoVID_V_M1_]], killed [[ADD]], -1, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[PseudoVADD_VX_M1_:%[0-9]+]]:vr = PseudoVADD_VX_M1 undef $noreg, [[PseudoVID_V_M1_]], [[ADD]], -1, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   [[PseudoVMSLTU_VX_M1_:%[0-9]+]]:vr = PseudoVMSLTU_VX_M1 [[PseudoVADD_VX_M1_]], [[COPY1]], -1, 6 /* e64 */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   [[PseudoVCPOP_M_B1_:%[0-9]+]]:gpr = PseudoVCPOP_M_B1 [[PseudoVMSLTU_VX_M1_]], -1, 0 /* e8 */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY $x0
-  ; CHECK-NEXT:   BEQ killed [[PseudoVCPOP_M_B1_]], [[COPY4]], %bb.3
+  ; CHECK-NEXT:   BEQ [[PseudoVCPOP_M_B1_]], $x0, %bb.3
   ; CHECK-NEXT:   PseudoBR %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[ADD1:%[0-9]+]]:gpr = ADD %src, [[PHI]]
-  ; CHECK-NEXT:   [[PseudoVLE8_V_MF8_:%[0-9]+]]:vrnov0 = PseudoVLE8_V_MF8 undef $noreg, killed [[ADD1]], -1, 3 /* e8 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[ADD1:%[0-9]+]]:gpr = ADD %src, [[COPY3]]
+  ; CHECK-NEXT:   [[PseudoVLE8_V_MF8_:%[0-9]+]]:vrnov0 = PseudoVLE8_V_MF8 undef $noreg, [[ADD1]], -1, 3 /* e8 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
   ; CHECK-NEXT:   dead $x0 = PseudoVSETVLIX0 killed $x0, 197 /* e8, mf8, ta, ma */, implicit-def $vl, implicit-def $vtype, implicit $vl
   ; CHECK-NEXT:   [[PseudoVADD_VI_MF8_:%[0-9]+]]:vrnov0 = PseudoVADD_VI_MF8 undef $noreg, [[PseudoVLE8_V_MF8_]], 4, -1, 3 /* e8 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
-  ; CHECK-NEXT:   [[ADD2:%[0-9]+]]:gpr = ADD %dst, [[PHI]]
-  ; CHECK-NEXT:   PseudoVSE8_V_MF8 killed [[PseudoVADD_VI_MF8_]], killed [[ADD2]], -1, 3 /* e8 */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   [[ADD2:%[0-9]+]]:gpr = ADD %dst, [[COPY3]]
+  ; CHECK-NEXT:   PseudoVSE8_V_MF8 [[PseudoVADD_VI_MF8_]], [[ADD2]], -1, 3 /* e8 */, implicit $vl, implicit $vtype
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.1(0x7c000000), %bb.4(0x04000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[ADD3:%[0-9]+]]:gpr = ADD [[PHI]], %inc
-  ; CHECK-NEXT:   BLTU [[ADD3]], %tc, %bb.1
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr = ADD [[COPY3]], %inc
+  ; CHECK-NEXT:   BLTU [[COPY3]], %tc, %bb.1
   ; CHECK-NEXT:   PseudoBR %bb.4
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.4:
@@ -1006,7 +982,7 @@ body: |
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   dead $x0 = PseudoVSETIVLI 1, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
-  ; CHECK-NEXT:   %x:gpr = PseudoVMV_X_S undef $noreg, 6 /* e64 */, implicit $vtype
+  ; CHECK-NEXT:   dead %x:gpr = PseudoVMV_X_S undef $noreg, 6 /* e64 */, implicit $vtype
   ; CHECK-NEXT:   PseudoBR %bb.1
   bb.0:
     PseudoBR %bb.1
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
index 29ce7c52e8fd..5b09aaedd975 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
@@ -352,15 +352,13 @@ entry:
 define <vscale x 1 x double> @test18(<vscale x 1 x double> %a, double %b) nounwind {
 ; CHECK-LABEL: test18:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 6, e64, m1, tu, ma
-; CHECK-NEXT:    vmv1r.v v9, v8
-; CHECK-NEXT:    vfmv.s.f v9, fa0
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfadd.vv v8, v8, v8
+; CHECK-NEXT:    vsetivli zero, 6, e64, m1, ta, ma
+; CHECK-NEXT:    vfadd.vv v9, v8, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
 ; CHECK-NEXT:    vfmv.s.f v8, fa0
+; CHECK-NEXT:    vfmv.s.f v9, fa0
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfadd.vv v8, v9, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v9
 ; CHECK-NEXT:    ret
 entry:
   %x = tail call i64 @llvm.riscv.vsetvli(i64 6, i64 3, i64 0)
@@ -380,8 +378,8 @@ entry:
 define <vscale x 1 x double> @test19(<vscale x 1 x double> %a, double %b) nounwind {
 ; CHECK-LABEL: test19:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, tu, ma
 ; CHECK-NEXT:    vmv1r.v v9, v8
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, tu, ma
 ; CHECK-NEXT:    vfmv.s.f v9, fa0
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfadd.vv v8, v9, v8
-- 
GitLab


From a71e2b9d0f287e4927e51d6764f90e492ba136e1 Mon Sep 17 00:00:00 2001
From: Robin Caloudis <robin.caloudis@gmx.de>
Date: Wed, 15 May 2024 06:02:46 +0200
Subject: [PATCH 0803/1206] [libc][errno] Remove non asm generic error number
 (#92172)

The following small thing caught my eye:

1) `EILSEQ` is not part of the generic asm error number macros. See the
[full list of generic asm errno
codes](https://github.com/torvalds/linux/blob/4b95dc87362aa57bdd0dcbad109ca5e5ef3cbb6c/include/uapi/asm-generic/errno-base.h).
AFAIK the generic asm errno numbers are common between different
operating systems and architectures. `EILSEQ` is not part of this common
set of errno's.

2) `EILSEQ`'s value is wrong. During the addition of `EILSEQ` in
https://reviews.llvm.org/D151129, the value `35` was probably chosen as
its the consecutive number. This is not correct. The actual values can
be looked up for example here:
* [For Linux
kernel](https://github.com/search?q=repo%3Atorvalds%2Flinux+EILSEQ&type=code&p=1):
`EILSEQ = 84` (uapi; i.e. x86_64), `EILSEQ = 88` (mips), `EILSEQ = 47`
(parisc)
* [For Darwin
kernel](https://github.com/apple-oss-distributions/xnu/blob/main/bsd/sys/errno.h#L237):
`EILSEQ = 92`
---
 libc/include/llvm-libc-macros/generic-error-number-macros.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libc/include/llvm-libc-macros/generic-error-number-macros.h b/libc/include/llvm-libc-macros/generic-error-number-macros.h
index b5b1b676dacc..cb4411fbac66 100644
--- a/libc/include/llvm-libc-macros/generic-error-number-macros.h
+++ b/libc/include/llvm-libc-macros/generic-error-number-macros.h
@@ -43,7 +43,6 @@
 #define EPIPE 32
 #define EDOM 33
 #define ERANGE 34
-#define EILSEQ 35
 #define ENAMETOOLONG 36
 #define EOVERFLOW 75
 
-- 
GitLab


From d3455f4ddd16811401fa153298fadd2f59f6914e Mon Sep 17 00:00:00 2001
From: Cyuria <55673467+cyuria@users.noreply.github.com>
Date: Wed, 15 May 2024 14:13:05 +1000
Subject: [PATCH 0804/1206] [libc][docs] Fix outdated code review section, as
 per #91934 (#92051)

As in the title, fixes #91934
---
 libc/src/math/docs/add_math_function.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libc/src/math/docs/add_math_function.md b/libc/src/math/docs/add_math_function.md
index d1bca222b428..9c23b8ca789b 100644
--- a/libc/src/math/docs/add_math_function.md
+++ b/libc/src/math/docs/add_math_function.md
@@ -196,7 +196,8 @@ implementation (which is very often glibc).
 
 ## Code reviews
 
-We follow the code review process of LLVM with Phabricator:
+We use GitHub's inbuilt pull request system for code review:
 ```
-  https://llvm.org/docs/Phabricator.html
+  https://docs.github.com/articles/about-collaborative-development-models
+  https://docs.github.com/articles/about-pull-requests
 ```
-- 
GitLab


From 77047e3cd2edd2b870982fc92f505cbb7fd764cd Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Wed, 15 May 2024 12:32:26 +0800
Subject: [PATCH 0805/1206] [RISCV] Make vsetvli in test not loop invariant.
 NFC (#92094)

The middle end will remove the inner vsetvli otherwise, and it's more
typical to set the AVL to the remaining VL.

This also prevents the test from showing up as a regression in #91319
---
 llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
index 5b09aaedd975..12bb4d27b0f9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
@@ -102,23 +102,23 @@ declare <vscale x 1 x i1> @llvm.riscv.vmand.nxv1i1.i64(<vscale x 1 x i1>, <vscal
 define void @test6(ptr nocapture readonly %A, ptr nocapture %B, i64 %n) {
 ; CHECK-LABEL: test6:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a6, a2, e32, m1, ta, ma
-; CHECK-NEXT:    beqz a6, .LBB5_3
+; CHECK-NEXT:    vsetvli a2, a2, e32, m1, ta, ma
+; CHECK-NEXT:    beqz a2, .LBB5_3
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
-; CHECK-NEXT:    li a4, 0
+; CHECK-NEXT:    li a3, 0
 ; CHECK-NEXT:  .LBB5_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    slli a3, a4, 2
-; CHECK-NEXT:    add a5, a0, a3
+; CHECK-NEXT:    slli a4, a3, 2
+; CHECK-NEXT:    add a5, a0, a4
 ; CHECK-NEXT:    vle32.v v8, (a5)
 ; CHECK-NEXT:    vmsle.vi v9, v8, -3
 ; CHECK-NEXT:    vmsgt.vi v10, v8, 2
 ; CHECK-NEXT:    vmor.mm v0, v9, v10
-; CHECK-NEXT:    add a3, a3, a1
-; CHECK-NEXT:    vse32.v v8, (a3), v0.t
-; CHECK-NEXT:    add a4, a4, a6
-; CHECK-NEXT:    vsetvli a6, a2, e32, m1, ta, ma
-; CHECK-NEXT:    bnez a6, .LBB5_2
+; CHECK-NEXT:    add a4, a4, a1
+; CHECK-NEXT:    vse32.v v8, (a4), v0.t
+; CHECK-NEXT:    add a3, a3, a2
+; CHECK-NEXT:    vsetvli a2, a2, e32, m1, ta, ma
+; CHECK-NEXT:    bnez a2, .LBB5_2
 ; CHECK-NEXT:  .LBB5_3: # %for.cond.cleanup
 ; CHECK-NEXT:    ret
 entry:
@@ -142,7 +142,7 @@ for.body:                                         ; preds = %entry, %for.body
   %7 = bitcast ptr %add.ptr1 to ptr
   tail call void @llvm.riscv.vse.mask.nxv2i32.i64(<vscale x 2 x i32> %3, ptr %7, <vscale x 2 x i1> %6, i64 %1)
   %add = add i64 %1, %i.012
-  %8 = tail call i64 @llvm.riscv.vsetvli.i64(i64 %n, i64 2, i64 0)
+  %8 = tail call i64 @llvm.riscv.vsetvli.i64(i64 %1, i64 2, i64 0)
   %cmp.not = icmp eq i64 %8, 0
   br i1 %cmp.not, label %for.cond.cleanup, label %for.body
 }
-- 
GitLab


From 74d91d9acebdbc81306c8a0ee547002e7e8084f9 Mon Sep 17 00:00:00 2001
From: rahulana-quic <quic_rahulana@quicinc.com>
Date: Wed, 15 May 2024 10:17:31 +0530
Subject: [PATCH 0806/1206] [polly] Port polly tests to use NPM (#90632)

Even as the NPM has been in use by Polly for a while now, the
majority of the tests continue using the LPM passes. This patch
ports the tests to use the NPM passes (for example, by replacing
a flag such as -polly-detect with -passes=polly-detect following
the NPM syntax for specifying passes) with some exceptions for
some missing features in the new passes. Additionally, the lit
substitution %loadPolly is replaced by the substitution of what
was %loadNPMPolly and %loadNPMPolly is removed.
---
 polly/test/CodeGen/20100617.ll                |  2 +-
 polly/test/CodeGen/20100622.ll                |  4 +-
 polly/test/CodeGen/20100707.ll                |  2 +-
 polly/test/CodeGen/20100707_2.ll              |  2 +-
 polly/test/CodeGen/20100708.ll                |  2 +-
 polly/test/CodeGen/20100708_2.ll              |  2 +-
 polly/test/CodeGen/20100713.ll                |  2 +-
 polly/test/CodeGen/20100713_2.ll              |  2 +-
 polly/test/CodeGen/20100717.ll                |  2 +-
 polly/test/CodeGen/20100718-DomInfo-2.ll      |  2 +-
 polly/test/CodeGen/20100718-DomInfo.ll        |  2 +-
 .../CodeGen/20100720-MultipleConditions.ll    |  2 +-
 .../test/CodeGen/20100809-IndependentBlock.ll |  2 +-
 ...0100811-ScalarDependencyBetweenBrAndCnd.ll |  2 +-
 polly/test/CodeGen/20101030-Overflow.ll       |  2 +-
 polly/test/CodeGen/20101103-Overflow3.ll      |  2 +-
 polly/test/CodeGen/20101103-signmissmatch.ll  |  2 +-
 .../test/CodeGen/20110226-Ignore-Dead-Code.ll |  2 +-
 .../test/CodeGen/20110226-PHI-Node-removed.ll |  2 +-
 polly/test/CodeGen/20120316-InvalidCast.ll    |  2 +-
 .../CodeGen/20120403-RHS-type-mismatch.ll     |  2 +-
 polly/test/CodeGen/20130221.ll                |  2 +-
 .../20150328-SCEVExpanderIntroducesNewIV.ll   |  2 +-
 polly/test/CodeGen/Intrinsics/llvm-expect.ll  |  2 +-
 .../do_not_mutate_debug_info.ll               |  2 +-
 .../loop_nest_param_parallel.ll               |  2 +-
 .../single_loop_param_parallel.ll             |  4 +-
 polly/test/CodeGen/MemAccess/bad_alignment.ll |  2 +-
 .../MemAccess/codegen_address_space.ll        |  2 +-
 .../MemAccess/codegen_constant_offset.ll      |  2 +-
 .../test/CodeGen/MemAccess/codegen_simple.ll  |  2 +-
 .../CodeGen/MemAccess/codegen_simple_float.ll |  2 +-
 .../CodeGen/MemAccess/codegen_simple_md.ll    |  4 +-
 .../MemAccess/codegen_simple_md_float.ll      |  4 +-
 .../test/CodeGen/MemAccess/different_types.ll |  4 +-
 polly/test/CodeGen/MemAccess/generate-all.ll  |  4 +-
 .../CodeGen/MemAccess/invariant_base_ptr.ll   |  4 +-
 .../test/CodeGen/MemAccess/multiple_types.ll  |  4 +-
 polly/test/CodeGen/MemAccess/simple.ll        |  2 +-
 .../MemAccess/update_access_functions.ll      |  4 +-
 polly/test/CodeGen/OpenMP/alias-metadata.ll   |  2 +-
 .../floord-as-argument-to-subfunction.ll      |  2 +-
 polly/test/CodeGen/OpenMP/inlineasm.ll        |  2 +-
 .../invariant_base_pointer_preloaded.ll       |  2 +-
 ...ant_base_pointer_preloaded_different_bb.ll |  2 +-
 ...base_pointer_preloaded_pass_only_needed.ll |  2 +-
 .../invariant_base_pointers_preloaded.ll      |  2 +-
 .../OpenMP/loop-body-references-outer-iv.ll   |  4 +-
 .../loop-body-references-outer-values-2.ll    |  4 +-
 .../loop-body-references-outer-values-3.ll    |  4 +-
 .../loop-body-references-outer-values.ll      |  4 +-
 .../OpenMP/loop-bounds-reference-outer-ids.ll |  4 +-
 .../test/CodeGen/OpenMP/mapped-phi-access.ll  |  2 +-
 polly/test/CodeGen/OpenMP/matmul-parallel.ll  |  4 +-
 polly/test/CodeGen/OpenMP/recomputed-srem.ll  |  2 +-
 ...ference-argument-from-non-affine-region.ll |  6 +-
 .../test/CodeGen/OpenMP/reference-other-bb.ll |  2 +-
 .../OpenMP/reference-preceeding-loop.ll       |  4 +-
 polly/test/CodeGen/OpenMP/reference_latest.ll |  2 +-
 polly/test/CodeGen/OpenMP/scev-rewriting.ll   |  2 +-
 polly/test/CodeGen/OpenMP/single_loop.ll      | 18 ++---
 ...single_loop_with_loop_invariant_baseptr.ll |  4 +-
 .../CodeGen/OpenMP/single_loop_with_param.ll  |  6 +-
 ...o-parallel-loops-reference-outer-indvar.ll |  4 +-
 polly/test/CodeGen/PHIInExit.ll               |  2 +-
 .../combine_different_values.ll               |  2 +-
 .../RuntimeDebugBuilder/stmt_tracing.ll       |  2 +-
 polly/test/CodeGen/alias-check-multi-dim.ll   |  2 +-
 .../CodeGen/alias_metadata_too_many_arrays.ll |  2 +-
 ...aliasing_different_base_and_access_type.ll |  2 +-
 .../aliasing_different_pointer_types.ll       |  2 +-
 .../aliasing_multidimensional_access.ll       |  2 +-
 .../CodeGen/aliasing_parametric_simple_1.ll   |  2 +-
 .../CodeGen/aliasing_parametric_simple_2.ll   |  2 +-
 polly/test/CodeGen/aliasing_struct_element.ll |  2 +-
 polly/test/CodeGen/alignment.ll               |  2 +-
 polly/test/CodeGen/annotated_alias_scopes.ll  |  2 +-
 polly/test/CodeGen/blas_sscal_simplified.ll   |  2 +-
 ...code-hosting-and-escape-map-computation.ll |  2 +-
 polly/test/CodeGen/constant_condition.ll      |  2 +-
 polly/test/CodeGen/create-conditional-scop.ll |  2 +-
 ...d_instruction_referenced_by_parameter_1.ll |  2 +-
 ...d_instruction_referenced_by_parameter_2.ll |  2 +-
 polly/test/CodeGen/debug-intrinsics.ll        |  4 +-
 ...nce_problem_after_early_codegen_bailout.ll |  2 +-
 polly/test/CodeGen/empty_domain_in_context.ll |  2 +-
 polly/test/CodeGen/entry_with_trivial_phi.ll  |  2 +-
 .../entry_with_trivial_phi_other_bb.ll        |  2 +-
 .../error-stmt-in-non-affine-region.ll        |  2 +-
 ...or_block_contains_invalid_memory_access.ll |  2 +-
 polly/test/CodeGen/exprModDiv.ll              |  8 +-
 .../hoisted_load_escapes_through_phi.ll       |  4 +-
 polly/test/CodeGen/hoisting_1.ll              |  2 +-
 polly/test/CodeGen/hoisting_2.ll              |  2 +-
 polly/test/CodeGen/inner_scev_sdiv_1.ll       |  2 +-
 polly/test/CodeGen/inner_scev_sdiv_2.ll       |  2 +-
 polly/test/CodeGen/inner_scev_sdiv_3.ll       |  2 +-
 polly/test/CodeGen/inner_scev_sdiv_in_lb.ll   |  4 +-
 .../inner_scev_sdiv_in_lb_invariant.ll        |  2 +-
 polly/test/CodeGen/inner_scev_sdiv_in_rtc.ll  |  2 +-
 polly/test/CodeGen/intrinsics_lifetime.ll     |  2 +-
 polly/test/CodeGen/intrinsics_misc.ll         |  2 +-
 .../inv-load-lnt-crash-wrong-order-2.ll       |  2 +-
 .../inv-load-lnt-crash-wrong-order-3.ll       |  2 +-
 .../CodeGen/inv-load-lnt-crash-wrong-order.ll |  2 +-
 .../test/CodeGen/invariant-load-dimension.ll  |  4 +-
 ...-load-preload-base-pointer-origin-first.ll |  2 +-
 .../CodeGen/invariant_cannot_handle_void.ll   |  4 +-
 polly/test/CodeGen/invariant_load.ll          |  2 +-
 .../CodeGen/invariant_load_address_space.ll   |  2 +-
 .../CodeGen/invariant_load_alias_metadata.ll  |  2 +-
 .../CodeGen/invariant_load_base_pointer.ll    |  2 +-
 ...invariant_load_base_pointer_conditional.ll |  2 +-
 ...variant_load_base_pointer_conditional_2.ll |  6 +-
 ...ariant_load_canonicalize_array_baseptrs.ll |  2 +-
 .../test/CodeGen/invariant_load_condition.ll  |  2 +-
 .../invariant_load_different_sized_types.ll   |  2 +-
 polly/test/CodeGen/invariant_load_escaping.ll |  2 +-
 .../invariant_load_escaping_second_scop.ll    |  2 +-
 .../invariant_load_in_non_affine_subregion.ll |  2 +-
 polly/test/CodeGen/invariant_load_loop_ub.ll  |  2 +-
 ...ant_load_not_executed_but_in_parameters.ll |  2 +-
 .../test/CodeGen/invariant_load_outermost.ll  |  2 +-
 ...riant_load_parameters_cyclic_dependence.ll |  4 +-
 .../CodeGen/invariant_load_ptr_ptr_noalias.ll |  2 +-
 .../test/CodeGen/invariant_load_scalar_dep.ll |  2 +-
 ...riant_load_scalar_escape_alloca_sharing.ll |  2 +-
 ...oads_from_struct_with_different_types_1.ll |  2 +-
 ...oads_from_struct_with_different_types_2.ll |  2 +-
 ...invariant_loads_ignore_parameter_bounds.ll |  2 +-
 .../invariant_verify_function_failed.ll       |  2 +-
 .../invariant_verify_function_failed_2.ll     |  4 +-
 polly/test/CodeGen/issue56692.ll              |  2 +-
 .../large-numbers-in-boundary-context.ll      |  2 +-
 .../test/CodeGen/load_subset_with_context.ll  |  2 +-
 .../loop-invariant-load-type-mismatch.ll      |  2 +-
 polly/test/CodeGen/loop_with_condition.ll     |  2 +-
 polly/test/CodeGen/loop_with_condition_2.ll   |  2 +-
 .../test/CodeGen/loop_with_condition_ineq.ll  |  2 +-
 .../CodeGen/loop_with_condition_nested.ll     |  4 +-
 ..._conditional_entry_edge_split_hard_case.ll |  2 +-
 polly/test/CodeGen/memcpy_annotations.ll      |  2 +-
 .../multidim-non-matching-typesize-2.ll       |  2 +-
 .../CodeGen/multidim-non-matching-typesize.ll |  2 +-
 ..._2d_parametric_array_static_loop_bounds.ll |  2 +-
 polly/test/CodeGen/multidim_alias_check.ll    |  2 +-
 polly/test/CodeGen/multiple-codegens.ll       |  5 +-
 polly/test/CodeGen/multiple-scops-in-a-row.ll |  2 +-
 .../multiple-types-invariant-load-2.ll        |  2 +-
 .../CodeGen/multiple-types-invariant-load.ll  |  2 +-
 .../multiple_sai_fro_same_base_address.ll     |  4 +-
 polly/test/CodeGen/no-overflow-tracking.ll    |  4 +-
 polly/test/CodeGen/no_guard_bb.ll             |  2 +-
 ...non-affine-dominance-generated-entering.ll |  2 +-
 .../CodeGen/non-affine-exit-node-dominance.ll |  2 +-
 .../non-affine-phi-node-expansion-2.ll        |  2 +-
 .../non-affine-phi-node-expansion-3.ll        |  2 +-
 .../non-affine-phi-node-expansion-4.ll        |  2 +-
 .../CodeGen/non-affine-phi-node-expansion.ll  |  2 +-
 ...e-region-exit-phi-incoming-synthesize-2.ll |  2 +-
 ...ine-region-exit-phi-incoming-synthesize.ll |  2 +-
 .../non-affine-region-implicit-store.ll       |  2 +-
 ...ine-region-phi-references-in-scop-value.ll |  2 +-
 .../non-affine-subregion-dominance-reuse.ll   |  2 +-
 polly/test/CodeGen/non-affine-switch.ll       |  2 +-
 .../non-affine-synthesized-in-branch.ll       |  2 +-
 polly/test/CodeGen/non-affine-update.ll       |  4 +-
 .../non-hoisted-load-needed-as-base-ptr.ll    |  2 +-
 .../test/CodeGen/non_affine_float_compare.ll  |  2 +-
 .../CodeGen/only_non_affine_error_region.ll   |  2 +-
 polly/test/CodeGen/openmp_limit_threads.ll    | 12 +--
 .../test/CodeGen/out-of-scop-phi-node-use.ll  |  2 +-
 polly/test/CodeGen/param_div_div_div_2.ll     |  4 +-
 polly/test/CodeGen/partial_write_array.ll     |  2 +-
 polly/test/CodeGen/partial_write_emptyset.ll  |  2 +-
 ...l_write_full_write_that_appears_partial.ll |  2 +-
 .../partial_write_impossible_restriction.ll   |  2 +-
 polly/test/CodeGen/partial_write_in_region.ll |  4 +-
 .../partial_write_in_region_with_loop.ll      |  4 +-
 .../CodeGen/partial_write_mapped_scalar.ll    |  2 +-
 .../partial_write_mapped_scalar_subregion.ll  |  2 +-
 polly/test/CodeGen/perf_monitoring.ll         |  2 +-
 .../perf_monitoring_cycles_per_scop.ll        |  2 +-
 .../perf_monitoring_trip_counts_per_scop.ll   |  2 +-
 polly/test/CodeGen/phi-defined-before-scop.ll |  2 +-
 .../phi_after_error_block_outside_of_scop.ll  |  2 +-
 .../test/CodeGen/phi_condition_modeling_1.ll  |  2 +-
 .../test/CodeGen/phi_condition_modeling_2.ll  |  2 +-
 .../test/CodeGen/phi_conditional_simple_1.ll  |  4 +-
 .../phi_in_exit_early_lnt_failure_1.ll        |  2 +-
 .../phi_in_exit_early_lnt_failure_2.ll        |  2 +-
 .../phi_in_exit_early_lnt_failure_3.ll        |  2 +-
 .../phi_in_exit_early_lnt_failure_5.ll        |  2 +-
 polly/test/CodeGen/phi_loop_carried_float.ll  |  2 +-
 .../CodeGen/phi_loop_carried_float_escape.ll  |  4 +-
 polly/test/CodeGen/phi_scalar_simple_1.ll     |  2 +-
 polly/test/CodeGen/phi_scalar_simple_2.ll     |  2 +-
 .../CodeGen/phi_with_multi_exiting_edges_2.ll |  2 +-
 polly/test/CodeGen/phi_with_one_exit_edge.ll  |  2 +-
 .../CodeGen/pointer-type-expressions-2.ll     |  4 +-
 .../test/CodeGen/pointer-type-expressions.ll  |  4 +-
 .../pointer-type-pointer-type-comparison.ll   |  4 +-
 polly/test/CodeGen/pointer_rem.ll             |  4 +-
 polly/test/CodeGen/pr25241.ll                 |  2 +-
 polly/test/CodeGen/ptrtoint_as_parameter.ll   |  2 +-
 polly/test/CodeGen/read-only-scalars.ll       |  4 +-
 polly/test/CodeGen/reduction.ll               |  2 +-
 polly/test/CodeGen/reduction_2.ll             |  2 +-
 polly/test/CodeGen/reduction_simple_binary.ll |  2 +-
 .../test/CodeGen/region-with-instructions.ll  |  2 +-
 polly/test/CodeGen/region_exiting-domtree.ll  |  2 +-
 .../CodeGen/region_multiexit_partialwrite.ll  |  2 +-
 ...run-time-condition-with-scev-parameters.ll |  4 +-
 polly/test/CodeGen/run-time-condition.ll      |  2 +-
 .../scalar-references-used-in-scop-compute.ll |  2 +-
 .../test/CodeGen/scalar-store-from-same-bb.ll |  2 +-
 polly/test/CodeGen/scalar_codegen_crash.ll    |  2 +-
 polly/test/CodeGen/scev-backedgetaken.ll      |  2 +-
 .../CodeGen/scev-division-invariant-load.ll   |  2 +-
 polly/test/CodeGen/scev.ll                    |  2 +-
 .../CodeGen/scev_expansion_in_nonaffine.ll    |  2 +-
 .../CodeGen/scev_looking_through_bitcasts.ll  |  2 +-
 .../CodeGen/scop_expander_insert_point.ll     |  2 +-
 polly/test/CodeGen/scop_expander_segfault.ll  |  2 +-
 ...p_never_executed_runtime_check_location.ll |  2 +-
 polly/test/CodeGen/select-base-pointer.ll     |  2 +-
 polly/test/CodeGen/sequential_loops.ll        |  2 +-
 .../CodeGen/simple_loop_non_single_exit.ll    |  2 +-
 .../CodeGen/simple_loop_non_single_exit_2.ll  |  2 +-
 polly/test/CodeGen/simple_non_single_entry.ll |  2 +-
 polly/test/CodeGen/simple_nonaffine_loop.ll   |  2 +-
 .../single_do_loop_int_max_iterations.ll      |  2 +-
 .../single_do_loop_int_param_iterations.ll    |  2 +-
 .../single_do_loop_ll_max_iterations.ll       |  4 +-
 .../CodeGen/single_do_loop_one_iteration.ll   |  2 +-
 .../CodeGen/single_do_loop_scev_replace.ll    |  2 +-
 polly/test/CodeGen/single_loop.ll             |  2 +-
 .../CodeGen/single_loop_int_max_iterations.ll |  2 +-
 .../CodeGen/single_loop_ll_max_iterations.ll  |  2 +-
 .../test/CodeGen/single_loop_one_iteration.ll |  2 +-
 polly/test/CodeGen/single_loop_param.ll       |  2 +-
 .../CodeGen/single_loop_param_less_equal.ll   |  6 +-
 .../CodeGen/single_loop_param_less_than.ll    |  4 +-
 .../CodeGen/single_loop_zero_iterations.ll    |  2 +-
 polly/test/CodeGen/split_edge_of_exit.ll      |  4 +-
 polly/test/CodeGen/split_edges.ll             |  2 +-
 polly/test/CodeGen/split_edges_2.ll           |  2 +-
 polly/test/CodeGen/srem-in-other-bb.ll        |  2 +-
 .../stack-overflow-in-load-hoisting.ll        |  2 +-
 .../test/CodeGen/stmt_split_no_dependence.ll  |  2 +-
 .../CodeGen/switch-in-non-affine-region.ll    |  2 +-
 .../synthesizable_phi_write_after_loop.ll     |  2 +-
 .../test-invalid-operands-for-select-2.ll     |  2 +-
 .../test-invalid-operands-for-select.ll       |  2 +-
 polly/test/CodeGen/test.ll                    |  2 +-
 .../two-loops-right-after-each-other-2.ll     |  2 +-
 .../two-scops-in-row-invalidate-scevs.ll      |  2 +-
 polly/test/CodeGen/two-scops-in-row.ll        |  4 +-
 polly/test/CodeGen/udiv_expansion_position.ll |  2 +-
 .../CodeGen/uninitialized_scalar_memory.ll    |  2 +-
 .../unpredictable-loop-unsynthesizable.ll     |  6 +-
 .../test/CodeGen/variant_load_empty_domain.ll |  2 +-
 .../whole-scop-non-affine-subregion.ll        |  2 +-
 polly/test/DeLICM/confused_order.ll           |  4 +-
 ...ontradicting_assumed_context_and_domain.ll |  2 +-
 polly/test/DeLICM/load-in-cond-inf-loop.ll    |  2 +-
 polly/test/DeLICM/map_memset_zero.ll          |  4 +-
 polly/test/DeLICM/nomap_alreadymapped.ll      |  2 +-
 polly/test/DeLICM/nomap_escaping.ll           |  2 +-
 polly/test/DeLICM/nomap_occupied.ll           |  2 +-
 polly/test/DeLICM/nomap_readonly.ll           |  2 +-
 polly/test/DeLICM/nomap_spuriouswrite.ll      |  2 +-
 polly/test/DeLICM/nomap_storagesize.ll        |  2 +-
 polly/test/DeLICM/nomap_writewrite.ll         |  2 +-
 polly/test/DeLICM/outofquota-reverseDomain.ll |  2 +-
 polly/test/DeLICM/pass_existence.ll           |  6 +-
 polly/test/DeLICM/pr41656.ll                  |  2 +-
 polly/test/DeLICM/pr48783.ll                  |  2 +-
 polly/test/DeLICM/reduction.ll                |  2 +-
 .../reduction_looprotate_gvnpre_cond1.ll      |  2 +-
 .../reduction_looprotate_gvnpre_cond2.ll      |  2 +-
 ...reduction_looprotate_gvnpre_nopreheader.ll |  2 +-
 .../reduction_looprotate_licm_nopreheader.ll  |  2 +-
 .../reduction_looprotate_loopguard_gvnpre.ll  |  2 +-
 .../reduction_looprotate_loopguard_licm1.ll   |  2 +-
 .../reduction_looprotate_loopguard_licm2.ll   |  2 +-
 .../reduction_looprotate_loopguard_licm3.ll   |  2 +-
 .../test/DeLICM/reduction_unrelatedunusual.ll |  2 +-
 polly/test/DeLICM/reject_loadafterstore.ll    |  2 +-
 polly/test/DeLICM/reject_outofquota.ll        |  4 +-
 polly/test/DeLICM/reject_storeafterstore.ll   |  2 +-
 polly/test/DeLICM/reject_storeinsubregion.ll  |  2 +-
 polly/test/DeLICM/reject_unusualstore.ll      |  4 +-
 polly/test/DeLICM/skip_maywrite.ll            |  2 +-
 polly/test/DeLICM/skip_multiaccess.ll         |  2 +-
 polly/test/DeLICM/skip_notinloop.ll           |  2 +-
 polly/test/DeLICM/skip_scalaraccess.ll        |  2 +-
 .../DeadCodeElimination/chained_iterations.ll |  4 +-
 .../chained_iterations_2.ll                   |  4 +-
 polly/test/DeadCodeElimination/computeout.ll  |  5 +-
 .../dead_iteration_elimination.ll             |  3 +-
 .../non-affine-affine-mix.ll                  |  2 +-
 polly/test/DeadCodeElimination/non-affine.ll  |  2 +-
 .../test/DeadCodeElimination/null_schedule.ll |  2 +-
 polly/test/DependenceInfo/computeout.ll       |  6 +-
 .../different_schedule_dimensions.ll          |  4 +-
 polly/test/DependenceInfo/do_pluto_matmult.ll |  6 +-
 polly/test/DependenceInfo/fine_grain_dep_0.ll |  7 +-
 .../generate_may_write_dependence_info.ll     |  2 +-
 .../test/DependenceInfo/infeasible_context.ll |  5 +-
 ...writes_do_not_block_must_writes_for_war.ll |  2 +-
 .../nonaffine-condition-buildMemoryAccess.ll  |  2 +-
 .../reduction_complex_location.ll             |  6 +-
 ...ndences_equal_non_reduction_dependences.ll |  2 +-
 .../reduction_dependences_not_null.ll         |  2 +-
 ...reduction_and_non_reduction_dependences.ll |  2 +-
 .../reduction_multiple_loops_array_sum.ll     |  6 +-
 .../reduction_multiple_loops_array_sum_2.ll   |  2 +-
 .../reduction_multiple_loops_array_sum_3.ll   |  2 +-
 .../reduction_multiple_reductions.ll          |  2 +-
 .../reduction_multiple_reductions_2.ll        |  2 +-
 .../reduction_only_reduction_like_access.ll   |  2 +-
 ...lly_escaping_intermediate_in_other_stmt.ll |  2 +-
 .../reduction_privatization_deps.ll           |  2 +-
 .../reduction_privatization_deps_2.ll         |  2 +-
 .../reduction_privatization_deps_3.ll         |  2 +-
 .../reduction_privatization_deps_4.ll         |  2 +-
 .../reduction_privatization_deps_5.ll         |  2 +-
 .../test/DependenceInfo/reduction_sequence.ll |  2 +-
 .../DependenceInfo/reduction_simple_iv.ll     |  2 +-
 ...ion_simple_iv_debug_wrapped_dependences.ll |  2 +-
 .../reduction_simple_privatization_deps_2.ll  |  2 +-
 ...n_simple_privatization_deps_w_parameter.ll |  2 +-
 ...duction_two_reductions_different_rloops.ll |  2 +-
 polly/test/DependenceInfo/sequential_loops.ll | 79 ++++++++-----------
 polly/test/ForwardOpTree/atax.ll              |  2 +-
 polly/test/ForwardOpTree/changed-kind.ll      |  2 +-
 .../test/ForwardOpTree/forward_from_region.ll |  2 +-
 polly/test/ForwardOpTree/forward_hoisted.ll   |  2 +-
 .../test/ForwardOpTree/forward_instruction.ll |  2 +-
 .../test/ForwardOpTree/forward_into_region.ll |  2 +-
 .../forward_into_region_redundant_use.ll      |  2 +-
 polly/test/ForwardOpTree/forward_load.ll      |  3 +-
 .../forward_load_differentarray.ll            |  2 +-
 .../forward_load_double_write.ll              |  2 +-
 .../ForwardOpTree/forward_load_fromloop.ll    |  2 +-
 .../ForwardOpTree/forward_load_indirect.ll    |  2 +-
 .../forward_load_memset_after.ll              |  2 +-
 .../forward_load_memset_before.ll             |  2 +-
 .../ForwardOpTree/forward_load_tripleuse.ll   |  2 +-
 .../forward_load_unrelatedunusual.ll          |  2 +-
 polly/test/ForwardOpTree/forward_phi_load.ll  |  2 +-
 polly/test/ForwardOpTree/forward_readonly.ll  |  4 +-
 polly/test/ForwardOpTree/forward_reusue.ll    |  2 +-
 polly/test/ForwardOpTree/forward_store.ll     |  2 +-
 .../forward_synthesizable_definloop.ll        |  2 +-
 .../forward_synthesizable_indvar.ll           |  2 +-
 .../forward_synthesizable_useinloop.ll        |  2 +-
 .../test/ForwardOpTree/forward_transitive.ll  |  2 +-
 polly/test/ForwardOpTree/jacobi-1d.ll         |  2 +-
 .../ForwardOpTree/noforward_from_region.ll    |  2 +-
 .../noforward_load_conditional.ll             |  2 +-
 .../noforward_load_writebetween.ll            |  2 +-
 .../ForwardOpTree/noforward_outofquota.ll     |  4 +-
 polly/test/ForwardOpTree/noforward_partial.ll |  2 +-
 polly/test/ForwardOpTree/noforward_phi.ll     |  2 +-
 .../ForwardOpTree/noforward_selfrefphi.ll     |  2 +-
 .../ForwardOpTree/noforward_sideffects.ll     |  2 +-
 .../noforward_synthesizable_unknownit.ll      |  2 +-
 polly/test/ForwardOpTree/out-of-quota1.ll     |  2 +-
 .../alias_checks_with_empty_context.ll        |  2 +-
 polly/test/IstAstInfo/alias_simple_1.ll       | 10 +--
 polly/test/IstAstInfo/alias_simple_2.ll       | 12 +--
 polly/test/IstAstInfo/alias_simple_3.ll       | 10 +--
 .../aliasing_arrays_with_identical_base.ll    |  2 +-
 .../aliasing_multiple_alias_groups.ll         |  4 +-
 .../aliasing_parametric_simple_1.ll           |  2 +-
 .../aliasing_parametric_simple_2.ll           |  2 +-
 .../IstAstInfo/dependence_distance_minimal.ll |  2 +-
 .../domain_bounded_only_with_context.ll       |  2 +-
 polly/test/IstAstInfo/non_affine_access.ll    |  2 +-
 ...reduction_clauses_onedimensional_access.ll |  2 +-
 ...ndences_equal_non_reduction_dependences.ll |  2 +-
 .../reduction_different_reduction_clauses.ll  |  2 +-
 ...ction_modulo_and_loop_reversal_schedule.ll |  2 +-
 ...ion_modulo_and_loop_reversal_schedule_2.ll |  2 +-
 ...ion_modulo_schedule_multiple_dimensions.ll |  2 +-
 ...n_modulo_schedule_multiple_dimensions_2.ll |  2 +-
 ...n_modulo_schedule_multiple_dimensions_3.ll |  2 +-
 ...n_modulo_schedule_multiple_dimensions_4.ll |  2 +-
 ...n_modulo_schedule_multiple_dimensions_5.ll |  2 +-
 .../reduction_multiple_dimensions.ll          |  2 +-
 .../reduction_multiple_dimensions_2.ll        |  2 +-
 .../reduction_multiple_dimensions_3.ll        |  2 +-
 .../reduction_multiple_dimensions_4.ll        |  2 +-
 polly/test/IstAstInfo/run-time-condition.ll   |  2 +-
 .../runtime_context_with_error_blocks.ll      |  2 +-
 .../IstAstInfo/simple-run-time-condition.ll   |  2 +-
 .../test/IstAstInfo/single_loop_strip_mine.ll |  4 +-
 .../single_loop_uint_max_iterations.ll        |  2 +-
 .../single_loop_ull_max_iterations.ll         |  2 +-
 .../ImportAccesses-Bad-relation.ll            |  2 +-
 .../ImportAccesses-No-accesses-key.ll         |  2 +-
 .../ImportAccesses-Not-enough-MemAcc.ll       |  2 +-
 .../ImportAccesses-Not-enough-statements.ll   |  2 +-
 .../ImportAccesses-Relation-mispelled.ll      |  2 +-
 .../ImportAccesses-Statements-mispelled.ll    |  2 +-
 ...ImportAccesses-Undeclared-ScopArrayInfo.ll |  2 +-
 .../ImportAccesses-Wrong-number-dimensions.ll |  2 +-
 .../ImportArrays-Mispelled-type.ll            |  2 +-
 .../ImportArrays-Negative-size.ll             |  2 +-
 .../ImportArrays/ImportArrays-No-name.ll      |  2 +-
 .../ImportArrays/ImportArrays-No-sizes-key.ll |  2 +-
 .../ImportArrays/ImportArrays-No-type-key.ll  |  2 +-
 .../ImportContext-Context-mispelled.ll        |  2 +-
 .../ImportContext-Not-parameter-set.ll        |  2 +-
 .../ImportContext-Unvalid-Context.ll          |  2 +-
 .../ImportContext-Wrong-dimension.ll          |  2 +-
 .../ImportSchedule-No-schedule-key.ll         |  2 +-
 .../ImportSchedule-Schedule-not-valid.ll      |  2 +-
 .../ImportSchedule-Statements-mispelled.ll    |  2 +-
 .../ImportSchedule-Wrong-number-statements.ll |  2 +-
 .../load_after_store_same_statement.ll        |  6 +-
 .../read_from_original.ll                     |  6 +-
 .../MaximalStaticExpansion/too_many_writes.ll |  6 +-
 .../working_deps_between_inners.ll            |  3 +-
 .../working_deps_between_inners_phi.ll        |  6 +-
 .../working_expansion.ll                      |  3 +-
 ...sion_multiple_dependences_per_statement.ll |  3 +-
 ...sion_multiple_instruction_per_statement.ll |  3 +-
 .../working_phi_expansion.ll                  |  6 +-
 .../working_phi_two_scalars.ll                |  6 +-
 .../working_value_expansion.ll                |  3 +-
 .../prune_only_scalardeps.ll                  |  3 +-
 .../2012-03-16-Empty-Domain.ll                |  2 +-
 .../2013-04-11-Empty-Domain-two.ll            |  2 +-
 .../GreedyFuse/fuse-double.ll                 |  4 +-
 .../GreedyFuse/fuse-except-first.ll           |  4 +-
 .../GreedyFuse/fuse-except-third.ll           |  4 +-
 .../GreedyFuse/fuse-inner-carried.ll          |  4 +-
 .../GreedyFuse/fuse-inner-third.ll            |  4 +-
 .../GreedyFuse/fuse-inner.ll                  |  4 +-
 .../GreedyFuse/fuse-simple.ll                 |  4 +-
 .../GreedyFuse/nofuse-simple.ll               |  4 +-
 .../GreedyFuse/nofuse-with-middle.ll          |  4 +-
 .../ManualOptimization/disable_nonforced.ll   |  2 +-
 .../distribute_heuristic.ll                   |  4 +-
 .../distribute_illegal_looploc.ll             |  2 +-
 .../distribute_illegal_pragmaloc.ll           |  2 +-
 .../ManualOptimization/unroll_disable.ll      |  2 +-
 .../ManualOptimization/unroll_double.ll       |  2 +-
 .../ManualOptimization/unroll_full.ll         |  2 +-
 .../ManualOptimization/unroll_heuristic.ll    |  4 +-
 .../ManualOptimization/unroll_partial.ll      |  4 +-
 .../unroll_partial_followup.ll                |  8 +-
 .../ScheduleOptimizer/SIMDInParallelFor.ll    |  2 +-
 polly/test/ScheduleOptimizer/computeout.ll    |  6 +-
 .../ensure-correct-tile-sizes.ll              |  4 +-
 .../focaltech_test_detail_threshold-7bc17e.ll |  3 +-
 .../full_partial_tile_separation.ll           |  2 +-
 polly/test/ScheduleOptimizer/line-tiling-2.ll |  2 +-
 polly/test/ScheduleOptimizer/line-tiling.ll   |  2 +-
 .../mat_mul_pattern_data_layout.ll            |  2 +-
 .../mat_mul_pattern_data_layout_2.ll          |  4 +-
 .../ScheduleOptimizer/one-dimensional-band.ll |  2 +-
 .../ScheduleOptimizer/outer_coincidence.ll    |  4 +-
 ...attern-matching-based-opts-after-delicm.ll |  4 +-
 ...tern-matching-based-opts-after-delicm_2.ll |  2 +-
 .../pattern-matching-based-opts.ll            |  8 +-
 .../pattern-matching-based-opts_11.ll         |  4 +-
 .../pattern-matching-based-opts_12.ll         |  2 +-
 .../pattern-matching-based-opts_13.ll         |  2 +-
 .../pattern-matching-based-opts_14.ll         |  4 +-
 .../pattern-matching-based-opts_15.ll         |  2 +-
 .../pattern-matching-based-opts_16.ll         |  2 +-
 .../pattern-matching-based-opts_17.ll         |  2 +-
 .../pattern-matching-based-opts_18.ll         |  2 +-
 .../pattern-matching-based-opts_19.ll         |  2 +-
 .../pattern-matching-based-opts_2.ll          |  2 +-
 .../pattern-matching-based-opts_20.ll         |  2 +-
 .../pattern-matching-based-opts_21.ll         |  2 +-
 .../pattern-matching-based-opts_22.ll         |  2 +-
 .../pattern-matching-based-opts_24.ll         |  2 +-
 .../pattern-matching-based-opts_25.ll         |  4 +-
 .../pattern-matching-based-opts_3.ll          |  8 +-
 .../pattern-matching-based-opts_4.ll          |  8 +-
 .../pattern-matching-based-opts_5.ll          |  6 +-
 .../pattern-matching-based-opts_6.ll          |  6 +-
 .../pattern-matching-based-opts_7.ll          |  2 +-
 .../pattern-matching-based-opts_8.ll          |  2 +-
 .../pattern-matching-based-opts_9.ll          |  4 +-
 .../pattern_matching_based_opts_splitmap.ll   |  2 +-
 .../prevectorization-without-tiling.ll        |  2 +-
 .../ScheduleOptimizer/prevectorization.ll     |  4 +-
 .../ScheduleOptimizer/rectangular-tiling.ll   |  8 +-
 .../ScheduleOptimizer/schedule_computeout.ll  |  2 +-
 polly/test/ScheduleOptimizer/statistics.ll    |  2 +-
 .../ScheduleOptimizer/tile_after_fusion.ll    |  4 +-
 ...vivid_vbi_gen_sliced-before-llvmreduced.ll |  2 +-
 .../aliasing_parametric_simple_1.ll           |  2 +-
 .../aliasing_parametric_simple_2.ll           |  2 +-
 polly/test/ScopDetect/aliasing_simple_1.ll    |  2 +-
 polly/test/ScopDetect/aliasing_simple_2.ll    |  2 +-
 .../base_pointer_load_setNewAccessRelation.ll |  2 +-
 .../base_pointer_setNewAccessRelation.ll      |  2 +-
 polly/test/ScopDetect/callbr.ll               |  4 +-
 .../ScopDetect/collective_invariant_loads.ll  |  2 +-
 .../ScopDetect/cross_loop_non_single_exit.ll  |  2 +-
 .../cross_loop_non_single_exit_2.ll           |  2 +-
 ...ependency_to_phi_node_outside_of_region.ll |  2 +-
 polly/test/ScopDetect/dot-scops-npm.ll        |  2 +-
 polly/test/ScopDetect/dot-scops.ll            |  2 +-
 .../ScopDetect/error-block-always-executed.ll |  2 +-
 .../error-block-referenced-from-scop.ll       |  2 +-
 .../ScopDetect/error-block-unreachable.ll     |  2 +-
 .../ScopDetect/expand-region-correctly-2.ll   |  2 +-
 .../ScopDetect/expand-region-correctly.ll     |  2 +-
 .../test/ScopDetect/ignore_func_flag_regex.ll |  2 +-
 .../index_from_unpredictable_loop.ll          |  4 +-
 .../index_from_unpredictable_loop2.ll         |  4 +-
 polly/test/ScopDetect/indvars.ll              |  2 +-
 polly/test/ScopDetect/intrinsics_1.ll         |  2 +-
 polly/test/ScopDetect/intrinsics_2.ll         |  2 +-
 polly/test/ScopDetect/intrinsics_3.ll         |  2 +-
 .../ScopDetect/invalid-latch-conditions.ll    |  6 +-
 .../ScopDetect/invalidate_scalar_evolution.ll |  2 +-
 .../ScopDetect/invariant-load-before-scop.ll  |  2 +-
 polly/test/ScopDetect/keep_going_expansion.ll |  2 +-
 polly/test/ScopDetect/mod_ref_read_pointer.ll |  4 +-
 polly/test/ScopDetect/more-than-one-loop.ll   |  4 +-
 .../ScopDetect/multidim-with-undef-size.ll    |  2 +-
 polly/test/ScopDetect/multidim.ll             |  2 +-
 .../ScopDetect/multidim_indirect_access.ll    |  2 +-
 ..._two_accesses_different_delinearization.ll |  2 +-
 .../ScopDetect/nested_loop_single_exit.ll     |  4 +-
 .../test/ScopDetect/non-affine-conditional.ll |  2 +-
 .../ScopDetect/non-affine-float-compare.ll    |  2 +-
 ...-affine-loop-condition-dependent-access.ll |  8 +-
 ...ffine-loop-condition-dependent-access_2.ll |  6 +-
 ...ffine-loop-condition-dependent-access_3.ll |  6 +-
 polly/test/ScopDetect/non-affine-loop.ll      | 10 +--
 .../non-beneficial-loops-small-trip-count.ll  |  2 +-
 .../non-constant-add-rec-start-expr.ll        |  2 +-
 .../ScopDetect/non-simple-memory-accesses.ll  |  2 +-
 .../ScopDetect/non_affine_loop_condition.ll   |  4 +-
 polly/test/ScopDetect/only-one-affine-loop.ll |  2 +-
 polly/test/ScopDetect/only_func_flag.ll       |  2 +-
 polly/test/ScopDetect/only_func_flag_regex.ll |  2 +-
 .../parametric-multiply-in-scev-2.ll          |  2 +-
 .../ScopDetect/parametric-multiply-in-scev.ll |  2 +-
 .../phi_with_multi_exiting_edges.ll           |  2 +-
 .../profitability-large-basic-blocks.ll       |  6 +-
 .../profitability-two-nested-loops.ll         |  2 +-
 polly/test/ScopDetect/remove_all_children.ll  |  2 +-
 polly/test/ScopDetect/report-scop-location.ll |  2 +-
 .../restrict-undef-size-scopdetect.ll         |  2 +-
 polly/test/ScopDetect/run_time_alias_check.ll |  2 +-
 polly/test/ScopDetect/scev_remove_max.ll      |  2 +-
 polly/test/ScopDetect/sequential_loops.ll     |  6 +-
 polly/test/ScopDetect/simple_loop.ll          |  2 +-
 .../simple_loop_non_single_entry.ll           |  2 +-
 .../ScopDetect/simple_loop_non_single_exit.ll |  2 +-
 .../simple_loop_non_single_exit_2.ll          |  2 +-
 .../ScopDetect/simple_loop_two_phi_nodes.ll   |  2 +-
 .../test/ScopDetect/simple_loop_with_param.ll |  2 +-
 .../ScopDetect/simple_loop_with_param_2.ll    |  2 +-
 .../ScopDetect/simple_non_single_entry.ll     |  2 +-
 .../ScopDetect/skip_function_attribute.ll     |  2 +-
 .../srem_with_parametric_divisor.ll           |  2 +-
 polly/test/ScopDetect/statistics.ll           |  2 +-
 polly/test/ScopDetect/switch-in-loop-patch.ll |  2 +-
 .../ReportAlias-01.ll                         |  2 +-
 .../ScopDetectionDiagnostics/ReportEntry.ll   |  2 +-
 .../ReportFuncCall-01.ll                      |  2 +-
 .../ReportIrreducibleRegion.ll                |  2 +-
 .../ReportIrreducibleRegionWithoutDebugLoc.ll |  2 +-
 .../ReportLoopBound-01.ll                     |  6 +-
 .../ReportLoopHasNoExit.ll                    |  4 +-
 .../ReportMultipleNonAffineAccesses.ll        | 12 +--
 .../ReportNonAffineAccess-01.ll               |  2 +-
 .../ReportUnprofitable.ll                     |  4 +-
 .../ReportUnreachableInExit.ll                |  2 +-
 .../ReportVariantBasePtr-01.ll                |  2 +-
 .../loop_has_multiple_exits.ll                |  2 +-
 .../loop_partially_in_scop-2.ll               |  2 +-
 .../loop_partially_in_scop.ll                 |  2 +-
 .../ScopInfo/20110312-Fail-without-basicaa.ll |  2 +-
 .../20111108-Parameter-not-detected.ll        |  2 +-
 ...03-16-Crash-because-of-unsigned-in-scev.ll |  2 +-
 .../2015-10-04-Crash-in-domain-generation.ll  |  2 +-
 polly/test/ScopInfo/Alias-0.ll                |  4 +-
 polly/test/ScopInfo/Alias-1.ll                |  4 +-
 polly/test/ScopInfo/Alias-2.ll                |  4 +-
 polly/test/ScopInfo/Alias-3.ll                |  4 +-
 polly/test/ScopInfo/Alias-4.ll                |  4 +-
 .../test/ScopInfo/BoundChecks/single-loop.ll  |  4 +-
 polly/test/ScopInfo/BoundChecks/two-loops.ll  |  4 +-
 polly/test/ScopInfo/NonAffine/div_backedge.ll |  2 +-
 polly/test/ScopInfo/NonAffine/div_domain.ll   |  2 +-
 ...nt_loads_dependent_in_non_affine_region.ll |  2 +-
 .../ScopInfo/NonAffine/modulo_backedge.ll     |  2 +-
 .../test/ScopInfo/NonAffine/modulo_domain.ll  |  2 +-
 ...ffine-loop-condition-dependent-access_1.ll |  4 +-
 ...ffine-loop-condition-dependent-access_2.ll |  6 +-
 ...ffine-loop-condition-dependent-access_3.ll |  6 +-
 .../non_affine_access_with_range_2.ll         |  2 +-
 .../ScopInfo/NonAffine/non_affine_but_sdiv.ll |  2 +-
 .../ScopInfo/NonAffine/non_affine_but_srem.ll |  2 +-
 .../non_affine_conditional_nested.ll          |  2 +-
 ...ine_conditional_surrounding_affine_loop.ll |  4 +-
 ...conditional_surrounding_non_affine_loop.ll |  6 +-
 .../NonAffine/non_affine_float_compare.ll     |  2 +-
 .../NonAffine/non_affine_loop_condition.ll    |  6 +-
 .../NonAffine/non_affine_loop_used_later.ll   |  4 +-
 .../NonAffine/non_affine_parametric_loop.ll   |  2 +-
 .../non_affine_region_guaranteed_non-entry.ll |  2 +-
 ...whole-scop-non-affine-subregion-in-loop.ll |  2 +-
 .../aliasing_conditional_alias_groups_1.ll    |  2 +-
 .../aliasing_conditional_alias_groups_2.ll    |  2 +-
 polly/test/ScopInfo/aliasing_dead_access.ll   |  2 +-
 .../aliasing_many_arrays_to_compare.ll        |  8 +-
 .../aliasing_many_read_only_acesses.ll        |  2 +-
 .../aliasing_multiple_alias_groups.ll         |  4 +-
 .../aliasing_with_non_affine_access.ll        |  2 +-
 .../allow-all-parameters-dereferencable.ll    |  6 +-
 polly/test/ScopInfo/assume_gep_bounds.ll      |  4 +-
 polly/test/ScopInfo/assume_gep_bounds_2.ll    |  2 +-
 polly/test/ScopInfo/assume_gep_bounds_many.ll |  4 +-
 .../avoid_new_parameters_from_geps.ll         |  2 +-
 polly/test/ScopInfo/bool-addrec.ll            |  2 +-
 .../test/ScopInfo/bounded_loop_assumptions.ll |  2 +-
 ...ces-loop-scev-with-unknown-iterations-2.ll |  4 +-
 ...ces-loop-scev-with-unknown-iterations-3.ll |  6 +-
 ...ences-loop-scev-with-unknown-iterations.ll |  6 +-
 polly/test/ScopInfo/bug_2010_10_22.ll         |  2 +-
 polly/test/ScopInfo/bug_2011_1_5.ll           |  2 +-
 .../test/ScopInfo/bug_scev_not_fully_eval.ll  |  2 +-
 polly/test/ScopInfo/cfg_consequences.ll       |  2 +-
 .../test/ScopInfo/complex-branch-structure.ll |  2 +-
 polly/test/ScopInfo/complex-condition.ll      |  2 +-
 polly/test/ScopInfo/complex-expression.ll     |  2 +-
 polly/test/ScopInfo/complex-loop-nesting.ll   |  2 +-
 .../ScopInfo/complex-successor-structure-2.ll |  2 +-
 .../ScopInfo/complex-successor-structure-3.ll |  4 +-
 .../ScopInfo/complex-successor-structure.ll   |  2 +-
 .../complex_domain_binary_condition.ll        |  2 +-
 .../ScopInfo/complex_execution_context.ll     |  2 +-
 polly/test/ScopInfo/cond_constant_in_loop.ll  |  2 +-
 polly/test/ScopInfo/cond_in_loop.ll           |  2 +-
 .../ScopInfo/condition-after-error-block-2.ll |  2 +-
 ...condition-after-error-block-before-scop.ll |  2 +-
 .../ScopInfo/condtion-after-error-block.ll    |  2 +-
 polly/test/ScopInfo/const_srem_sdiv.ll        |  4 +-
 .../constant-non-integer-branch-condition.ll  |  2 +-
 .../ScopInfo/constant_factor_in_parameter.ll  |  4 +-
 ...stant_functions_outside_scop_as_unknown.ll |  2 +-
 polly/test/ScopInfo/constant_start_integer.ll |  2 +-
 polly/test/ScopInfo/debug_call.ll             |  2 +-
 .../delinearize-together-all-data-refs.ll     |  2 +-
 polly/test/ScopInfo/div_by_zero.ll            |  2 +-
 .../do-not-model-error-block-accesses.ll      |  2 +-
 .../eager-binary-and-or-conditions.ll         |  4 +-
 .../early_exit_for_complex_domains.ll         |  2 +-
 polly/test/ScopInfo/error-blocks-1.ll         |  2 +-
 polly/test/ScopInfo/error-blocks-2.ll         |  4 +-
 polly/test/ScopInfo/escaping_empty_scop.ll    |  2 +-
 polly/test/ScopInfo/exit-phi-1.ll             |  4 +-
 polly/test/ScopInfo/exit-phi-2.ll             |  2 +-
 polly/test/ScopInfo/exit_phi_accesses-2.ll    |  2 +-
 polly/test/ScopInfo/exit_phi_accesses.ll      |  2 +-
 .../ScopInfo/expensive-boundary-context.ll    |  4 +-
 ...onstant_factor_introduces_new_parameter.ll |  4 +-
 polly/test/ScopInfo/full-function.ll          |  4 +-
 polly/test/ScopInfo/granularity_same_name.ll  |  8 +-
 .../test/ScopInfo/granularity_scalar-indep.ll |  2 +-
 ...ity_scalar-indep_cross-referencing-phi1.ll |  2 +-
 ...ity_scalar-indep_cross-referencing-phi2.ll |  2 +-
 .../granularity_scalar-indep_epilogue.ll      |  2 +-
 .../granularity_scalar-indep_epilogue_last.ll |  2 +-
 .../granularity_scalar-indep_noepilogue.ll    |  2 +-
 .../granularity_scalar-indep_ordered-2.ll     |  2 +-
 .../granularity_scalar-indep_ordered.ll       |  2 +-
 polly/test/ScopInfo/i1_params.ll              |  2 +-
 polly/test/ScopInfo/infeasible-rtc.ll         |  4 +-
 .../ScopInfo/infeasible_invalid_context.ll    |  4 +-
 polly/test/ScopInfo/int2ptr_ptr2int.ll        |  4 +-
 polly/test/ScopInfo/int2ptr_ptr2int_2.ll      |  8 +-
 polly/test/ScopInfo/integers.ll               |  2 +-
 .../ScopInfo/inter-error-bb-dependence.ll     |  2 +-
 polly/test/ScopInfo/inter_bb_scalar_dep.ll    |  4 +-
 .../intra-non-affine-stmt-phi-node.ll         |  4 +-
 .../ScopInfo/intra_and_inter_bb_scalar_dep.ll |  4 +-
 polly/test/ScopInfo/intra_bb_scalar_dep.ll    |  4 +-
 polly/test/ScopInfo/intrinsics.ll             |  2 +-
 ..._add_rec_after_invariant_load_remapping.ll |  2 +-
 .../invalidate_iterator_during_MA_removal.ll  |  2 +-
 .../test/ScopInfo/invariant-load-instlist.ll  |  2 +-
 ...ariant-loads-leave-read-only-statements.ll |  4 +-
 polly/test/ScopInfo/invariant_load.ll         |  2 +-
 ...load_access_classes_different_base_type.ll |  4 +-
 ...ss_classes_different_base_type_escaping.ll |  4 +-
 ...lasses_different_base_type_same_pointer.ll |  4 +-
 ...fferent_base_type_same_pointer_escaping.ll |  4 +-
 .../ScopInfo/invariant_load_addrec_sum.ll     |  2 +-
 .../ScopInfo/invariant_load_base_pointer.ll   |  2 +-
 ...invariant_load_base_pointer_conditional.ll |  2 +-
 ...ariant_load_base_pointer_in_conditional.ll |  2 +-
 .../invariant_load_branch_condition.ll        |  4 +-
 ...ariant_load_canonicalize_array_baseptrs.ll |  2 +-
 ...iant_load_canonicalize_array_baseptrs_2.ll |  2 +-
 ...iant_load_canonicalize_array_baseptrs_3.ll |  2 +-
 ...iant_load_canonicalize_array_baseptrs_4.ll |  2 +-
 ...ant_load_canonicalize_array_baseptrs_4b.ll |  2 +-
 ...ant_load_canonicalize_array_baseptrs_4c.ll |  2 +-
 ...iant_load_canonicalize_array_baseptrs_5.ll |  2 +-
 .../invariant_load_complex_condition.ll       |  4 +-
 .../test/ScopInfo/invariant_load_condition.ll |  2 +-
 .../invariant_load_dereferenceable.ll         |  4 +-
 ...iant_load_distinct_parameter_valuations.ll |  2 +-
 .../ScopInfo/invariant_load_in_non_affine.ll  |  4 +-
 polly/test/ScopInfo/invariant_load_loop_ub.ll |  4 +-
 .../invariant_load_ptr_ptr_noalias.ll         |  4 +-
 .../ScopInfo/invariant_load_scalar_dep.ll     |  2 +-
 .../ScopInfo/invariant_load_stmt_domain.ll    |  2 +-
 .../invariant_load_zext_parameter-2.ll        |  4 +-
 .../ScopInfo/invariant_load_zext_parameter.ll |  4 +-
 ...load_zextended_in_own_execution_context.ll |  4 +-
 ...invariant_loads_complicated_dependences.ll |  2 +-
 .../invariant_loads_cyclic_dependences.ll     |  2 +-
 polly/test/ScopInfo/invariant_loop_bounds.ll  |  2 +-
 ...ariant_same_loop_bound_multiple_times-1.ll |  2 +-
 ...ariant_same_loop_bound_multiple_times-2.ll |  2 +-
 polly/test/ScopInfo/isl_aff_out_of_bounds.ll  |  2 +-
 polly/test/ScopInfo/isl_trip_count_01.ll      |  2 +-
 polly/test/ScopInfo/isl_trip_count_02.ll      |  2 +-
 polly/test/ScopInfo/isl_trip_count_03.ll      |  2 +-
 .../isl_trip_count_multiple_exiting_blocks.ll |  2 +-
 polly/test/ScopInfo/licm_load.ll              |  4 +-
 polly/test/ScopInfo/licm_potential_store.ll   |  4 +-
 polly/test/ScopInfo/licm_reduction_nested.ll  |  4 +-
 .../long-compile-time-alias-analysis.ll       |  2 +-
 .../long-sequence-of-error-blocks-2.ll        |  2 +-
 .../ScopInfo/long-sequence-of-error-blocks.ll |  4 +-
 .../test/ScopInfo/loop-multiexit-succ-cond.ll |  4 +-
 polly/test/ScopInfo/loop_affine_bound_0.ll    |  4 +-
 polly/test/ScopInfo/loop_affine_bound_1.ll    |  4 +-
 polly/test/ScopInfo/loop_affine_bound_2.ll    |  4 +-
 polly/test/ScopInfo/loop_carry.ll             |  2 +-
 .../test/ScopInfo/many-scalar-dependences.ll  |  2 +-
 polly/test/ScopInfo/max-loop-depth.ll         |  2 +-
 polly/test/ScopInfo/memcpy-raw-source.ll      |  2 +-
 polly/test/ScopInfo/memcpy.ll                 |  4 +-
 polly/test/ScopInfo/memmove.ll                |  4 +-
 polly/test/ScopInfo/memset.ll                 |  4 +-
 polly/test/ScopInfo/memset_null.ll            |  4 +-
 .../ScopInfo/mismatching-array-dimensions.ll  |  2 +-
 .../mod_ref_access_pointee_arguments.ll       |  6 +-
 .../mod_ref_read_pointee_arguments.ll         |  6 +-
 polly/test/ScopInfo/mod_ref_read_pointer.ll   |  4 +-
 polly/test/ScopInfo/mod_ref_read_pointers.ll  |  6 +-
 polly/test/ScopInfo/modulo_zext_1.ll          |  2 +-
 polly/test/ScopInfo/modulo_zext_2.ll          |  2 +-
 polly/test/ScopInfo/modulo_zext_3.ll          |  2 +-
 polly/test/ScopInfo/multi-scop.ll             |  2 +-
 .../ScopInfo/multidim_2d-diagonal-matrix.ll   |  4 +-
 .../multidim_2d_outer_parametric_offset.ll    |  2 +-
 ..._2d_parametric_array_static_loop_bounds.ll |  2 +-
 .../ScopInfo/multidim_2d_with_modref_call.ll  |  8 +-
 .../multidim_2d_with_modref_call_2.ll         |  8 +-
 ..._3d_parametric_array_static_loop_bounds.ll |  2 +-
 ...idim_fixedsize_different_dimensionality.ll |  2 +-
 .../multidim_fixedsize_multi_offset.ll        |  2 +-
 .../ScopInfo/multidim_fold_constant_dim.ll    |  2 +-
 .../multidim_fold_constant_dim_zero.ll        |  2 +-
 polly/test/ScopInfo/multidim_fortran_2d.ll    |  4 +-
 .../ScopInfo/multidim_fortran_2d_params.ll    |  4 +-
 .../multidim_fortran_2d_with_modref_call.ll   |  8 +-
 polly/test/ScopInfo/multidim_fortran_srem.ll  |  2 +-
 .../test/ScopInfo/multidim_gep_pointercast.ll |  2 +-
 .../ScopInfo/multidim_gep_pointercast2.ll     |  2 +-
 .../multidim_ivs_and_integer_offsets_3d.ll    |  2 +-
 ...multidim_ivs_and_parameteric_offsets_3d.ll |  2 +-
 .../test/ScopInfo/multidim_many_references.ll |  4 +-
 .../ScopInfo/multidim_nested_start_integer.ll |  4 +-
 .../multidim_nested_start_share_parameter.ll  |  2 +-
 polly/test/ScopInfo/multidim_only_ivs_2d.ll   |  2 +-
 polly/test/ScopInfo/multidim_only_ivs_3d.ll   |  2 +-
 .../ScopInfo/multidim_only_ivs_3d_cast.ll     |  2 +-
 .../ScopInfo/multidim_only_ivs_3d_reverse.ll  |  2 +-
 .../ScopInfo/multidim_param_in_subscript-2.ll |  2 +-
 .../ScopInfo/multidim_param_in_subscript.ll   |  2 +-
 .../multidim_parameter_addrec_product.ll      |  2 +-
 .../multidim_single_and_multidim_array.ll     | 16 ++--
 polly/test/ScopInfo/multidim_srem.ll          |  2 +-
 polly/test/ScopInfo/multidim_with_bitcast.ll  |  2 +-
 .../ScopInfo/multiple-binary-or-conditions.ll |  4 +-
 ...ss-offset-not-dividable-by-element-size.ll |  2 +-
 .../ScopInfo/multiple-types-non-affine-2.ll   |  4 +-
 .../ScopInfo/multiple-types-non-affine.ll     |  4 +-
 .../multiple-types-non-power-of-two-2.ll      |  2 +-
 .../multiple-types-non-power-of-two.ll        |  2 +-
 .../multiple-types-two-dimensional-2.ll       |  2 +-
 .../multiple-types-two-dimensional.ll         |  2 +-
 polly/test/ScopInfo/multiple-types.ll         |  4 +-
 .../test/ScopInfo/multiple_exiting_blocks.ll  |  2 +-
 .../multiple_exiting_blocks_two_loop.ll       |  2 +-
 polly/test/ScopInfo/multiple_latch_blocks.ll  |  2 +-
 polly/test/ScopInfo/nested-loops.ll           |  2 +-
 .../no-scalar-deps-in-non-affine-subregion.ll |  2 +-
 polly/test/ScopInfo/non-affine-region-phi.ll  |  4 +-
 .../ScopInfo/non-affine-region-with-loop-2.ll |  2 +-
 .../ScopInfo/non-affine-region-with-loop.ll   |  4 +-
 polly/test/ScopInfo/non-precise-inv-load-1.ll |  2 +-
 polly/test/ScopInfo/non-precise-inv-load-2.ll |  2 +-
 polly/test/ScopInfo/non-precise-inv-load-3.ll |  2 +-
 polly/test/ScopInfo/non-precise-inv-load-4.ll |  2 +-
 polly/test/ScopInfo/non-precise-inv-load-5.ll |  2 +-
 polly/test/ScopInfo/non-precise-inv-load-6.ll |  2 +-
 polly/test/ScopInfo/non-pure-function-call.ll |  2 +-
 ...-pure-function-calls-causes-dead-blocks.ll |  2 +-
 .../test/ScopInfo/non-pure-function-calls.ll  |  2 +-
 polly/test/ScopInfo/non_affine_access.ll      |  4 +-
 polly/test/ScopInfo/non_affine_region_1.ll    |  2 +-
 polly/test/ScopInfo/non_affine_region_2.ll    |  2 +-
 polly/test/ScopInfo/non_affine_region_3.ll    |  4 +-
 polly/test/ScopInfo/non_affine_region_4.ll    |  2 +-
 .../ScopInfo/nonaffine-buildMemoryAccess.ll   |  2 +-
 polly/test/ScopInfo/not-a-reduction.ll        |  2 +-
 polly/test/ScopInfo/opaque-struct.ll          |  2 +-
 ...gion-entry-phi-node-nonaffine-subregion.ll |  2 +-
 ...ut-of-scop-use-in-region-entry-phi-node.ll |  2 +-
 .../ScopInfo/parameter-constant-division.ll   |  4 +-
 .../ScopInfo/parameter_in_dead_statement.ll   |  8 +-
 polly/test/ScopInfo/parameter_product.ll      |  2 +-
 .../parameter_with_constant_factor_in_add.ll  |  2 +-
 .../ScopInfo/partially_invariant_load_1.ll    |  4 +-
 .../ScopInfo/partially_invariant_load_2.ll    |  2 +-
 .../test/ScopInfo/phi-in-non-affine-region.ll |  2 +-
 polly/test/ScopInfo/phi_after_error_block.ll  |  2 +-
 .../test/ScopInfo/phi_condition_modeling_1.ll |  2 +-
 .../test/ScopInfo/phi_condition_modeling_2.ll |  2 +-
 .../test/ScopInfo/phi_conditional_simple_1.ll |  2 +-
 polly/test/ScopInfo/phi_loop_carried_float.ll |  2 +-
 polly/test/ScopInfo/phi_not_grouped_at_top.ll |  2 +-
 polly/test/ScopInfo/phi_scalar_simple_1.ll    |  2 +-
 polly/test/ScopInfo/phi_scalar_simple_2.ll    |  2 +-
 polly/test/ScopInfo/phi_with_invoke_edge.ll   |  2 +-
 .../ScopInfo/pointer-comparison-no-nsw.ll     |  2 +-
 polly/test/ScopInfo/pointer-comparison.ll     |  2 +-
 .../test/ScopInfo/pointer-type-expressions.ll |  2 +-
 ...er-used-as-base-pointer-and-scalar-read.ll |  2 +-
 .../polly-timeout-parameter-bounds.ll         |  2 +-
 ...eserve-equiv-class-order-in-basic_block.ll |  2 +-
 .../test/ScopInfo/process_added_dimensions.ll |  2 +-
 .../test/ScopInfo/pwaff-complexity-bailout.ll |  2 +-
 polly/test/ScopInfo/ranged_parameter.ll       |  2 +-
 polly/test/ScopInfo/ranged_parameter_2.ll     |  2 +-
 polly/test/ScopInfo/ranged_parameter_wrap.ll  |  2 +-
 .../test/ScopInfo/ranged_parameter_wrap_2.ll  |  2 +-
 .../read-only-scalar-used-in-phi-2.ll         |  2 +-
 .../ScopInfo/read-only-scalar-used-in-phi.ll  |  2 +-
 polly/test/ScopInfo/read-only-scalars.ll      |  4 +-
 polly/test/ScopInfo/read-only-statements.ll   |  2 +-
 .../ScopInfo/reduction_alternating_base.ll    |  2 +-
 ...uction_chain_partially_outside_the_scop.ll |  2 +-
 .../ScopInfo/reduction_different_index.ll     |  2 +-
 .../ScopInfo/reduction_different_index1.ll    |  2 +-
 .../reduction_disabled_multiplicative.ll      |  2 +-
 .../reduction_escaping_intermediate.ll        |  2 +-
 .../reduction_escaping_intermediate_2.ll      |  2 +-
 .../reduction_invalid_different_operators.ll  |  2 +-
 .../reduction_invalid_overlapping_accesses.ll |  2 +-
 .../reduction_multiple_loops_array_sum.ll     |  2 +-
 .../reduction_multiple_loops_array_sum_1.ll   |  2 +-
 .../reduction_multiple_simple_binary.ll       |  2 +-
 .../reduction_non_overlapping_chains.ll       |  2 +-
 .../reduction_only_reduction_like_access.ll   |  2 +-
 polly/test/ScopInfo/reduction_simple_fp.ll    |  2 +-
 .../ScopInfo/reduction_simple_w_constant.ll   |  2 +-
 polly/test/ScopInfo/reduction_simple_w_iv.ll  |  2 +-
 .../ScopInfo/reduction_two_identical_reads.ll |  4 +-
 .../redundant_parameter_constraint.ll         |  2 +-
 .../test/ScopInfo/region-with-instructions.ll |  2 +-
 polly/test/ScopInfo/remarks.ll                |  2 +-
 .../required-invariant-loop-bounds.ll         |  4 +-
 .../ScopInfo/restriction_in_dead_block.ll     |  2 +-
 .../run-time-check-many-array-disjuncts.ll    |  4 +-
 .../run-time-check-many-parameters.ll         |  2 +-
 .../run-time-check-many-piecewise-aliasing.ll |  4 +-
 .../run-time-check-read-only-arrays.ll        |  2 +-
 .../same-base-address-scalar-and-array.ll     |  2 +-
 polly/test/ScopInfo/scalar.ll                 |  2 +-
 .../ScopInfo/scalar_dependence_cond_br.ll     |  2 +-
 polly/test/ScopInfo/scalar_to_array.ll        |  4 +-
 .../scev-div-with-evaluatable-divisor.ll      |  2 +-
 polly/test/ScopInfo/scev-invalidated.ll       |  2 +-
 .../schedule-const-post-dominator-walk-2.ll   |  2 +-
 .../schedule-const-post-dominator-walk.ll     |  2 +-
 .../schedule-constuction-endless-loop1.ll     |  2 +-
 .../schedule-constuction-endless-loop2.ll     |  2 +-
 ...tly-contructed-in-case-of-infinite-loop.ll |  2 +-
 .../scop-affine-parameter-ordering.ll         |  2 +-
 polly/test/ScopInfo/sign_wrapped_set.ll       |  2 +-
 polly/test/ScopInfo/simple_loop_1.ll          |  2 +-
 polly/test/ScopInfo/simple_loop_2.ll          |  2 +-
 polly/test/ScopInfo/simple_loop_unsigned.ll   |  2 +-
 polly/test/ScopInfo/simple_loop_unsigned_2.ll |  2 +-
 polly/test/ScopInfo/simple_loop_unsigned_3.ll |  2 +-
 .../ScopInfo/simple_nonaffine_loop_not.ll     |  2 +-
 polly/test/ScopInfo/smax.ll                   |  2 +-
 polly/test/ScopInfo/statistics.ll             |  2 +-
 .../stmt_split_exit_of_region_stmt.ll         |  2 +-
 .../ScopInfo/stmt_split_no_after_split.ll     |  2 +-
 .../test/ScopInfo/stmt_split_no_dependence.ll |  2 +-
 polly/test/ScopInfo/stmt_split_on_store.ll    |  2 +-
 .../ScopInfo/stmt_split_on_synthesizable.ll   |  2 +-
 .../stmt_split_phi_in_beginning_bb.ll         |  2 +-
 polly/test/ScopInfo/stmt_split_phi_in_stmt.ll |  2 +-
 .../ScopInfo/stmt_split_scalar_dependence.ll  |  2 +-
 polly/test/ScopInfo/stmt_split_within_loop.ll |  2 +-
 .../stmt_with_read_but_without_sideffect.ll   |  2 +-
 polly/test/ScopInfo/switch-1.ll               |  4 +-
 polly/test/ScopInfo/switch-2.ll               |  4 +-
 polly/test/ScopInfo/switch-3.ll               |  4 +-
 polly/test/ScopInfo/switch-4.ll               |  4 +-
 polly/test/ScopInfo/switch-5.ll               |  4 +-
 polly/test/ScopInfo/switch-6.ll               |  4 +-
 polly/test/ScopInfo/switch-7.ll               |  5 +-
 polly/test/ScopInfo/tempscop-printing.ll      |  2 +-
 .../ScopInfo/test-wrapping-in-condition.ll    |  4 +-
 polly/test/ScopInfo/truncate-1.ll             |  2 +-
 polly/test/ScopInfo/truncate-2.ll             |  2 +-
 polly/test/ScopInfo/truncate-3.ll             |  2 +-
 polly/test/ScopInfo/two-loops-one-infinite.ll |  2 +-
 .../two-loops-right-after-each-other.ll       |  2 +-
 polly/test/ScopInfo/undef_in_cond.ll          |  2 +-
 polly/test/ScopInfo/unnamed_nonaffine.ll      |  4 +-
 polly/test/ScopInfo/unnamed_stmts.ll          |  2 +-
 .../ScopInfo/unpredictable_nonscop_loop.ll    |  2 +-
 .../test/ScopInfo/unprofitable_scalar-accs.ll |  4 +-
 polly/test/ScopInfo/unsigned-condition.ll     |  2 +-
 polly/test/ScopInfo/unsigned-division-1.ll    |  2 +-
 polly/test/ScopInfo/unsigned-division-2.ll    |  2 +-
 polly/test/ScopInfo/unsigned-division-3.ll    |  2 +-
 polly/test/ScopInfo/unsigned-division-4.ll    |  2 +-
 polly/test/ScopInfo/unsigned-division-5.ll    |  2 +-
 polly/test/ScopInfo/unsigned_wrap_uge.ll      |  2 +-
 polly/test/ScopInfo/unsigned_wrap_ugt.ll      |  2 +-
 polly/test/ScopInfo/unsigned_wrap_ule.ll      |  2 +-
 polly/test/ScopInfo/unsigned_wrap_ult.ll      |  2 +-
 polly/test/ScopInfo/user_context.ll           |  8 +-
 ...ed_assumptions-in-bb-signed-conditional.ll |  4 +-
 .../user_provided_assumptions-in-bb-signed.ll |  2 +-
 ...ser_provided_assumptions-in-bb-unsigned.ll |  4 +-
 .../ScopInfo/user_provided_assumptions.ll     |  4 +-
 .../ScopInfo/user_provided_assumptions_2.ll   |  4 +-
 .../ScopInfo/user_provided_assumptions_3.ll   |  4 +-
 ...ser_provided_non_dominating_assumptions.ll |  4 +-
 polly/test/ScopInfo/variant_base_pointer.ll   |  4 +-
 .../ScopInfo/variant_load_empty_domain.ll     |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_0.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_1.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_2.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_3.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_4.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_5.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_6.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_7.ll  |  2 +-
 .../ScopInfo/wraping_signed_expr_slow_1.ll    |  2 +-
 .../ScopInfo/wraping_signed_expr_slow_2.ll    |  2 +-
 polly/test/ScopInfo/zero_ext_of_truncate.ll   |  2 +-
 polly/test/ScopInfo/zero_ext_of_truncate_2.ll |  2 +-
 .../test/ScopInfo/zero_ext_space_mismatch.ll  |  2 +-
 polly/test/ScopInliner/invariant-load-func.ll |  2 +-
 polly/test/Simplify/coalesce_3partials.ll     |  2 +-
 .../Simplify/coalesce_disjointelements.ll     |  2 +-
 polly/test/Simplify/coalesce_overlapping.ll   |  2 +-
 polly/test/Simplify/coalesce_partial.ll       |  2 +-
 polly/test/Simplify/dead_access_load.ll       |  3 +-
 polly/test/Simplify/dead_access_phi.ll        |  3 +-
 polly/test/Simplify/dead_access_value.ll      |  3 +-
 polly/test/Simplify/dead_instruction.ll       |  3 +-
 polly/test/Simplify/emptyaccessdomain.ll      |  2 +-
 polly/test/Simplify/exit_phi_accesses-2.ll    |  2 +-
 polly/test/Simplify/func-b320a7.ll            |  2 +-
 polly/test/Simplify/gemm.ll                   |  2 +-
 .../Simplify/nocoalesce_differentvalues.ll    |  2 +-
 .../Simplify/nocoalesce_elementmismatch.ll    |  2 +-
 polly/test/Simplify/nocoalesce_readbetween.ll |  2 +-
 .../test/Simplify/nocoalesce_writebetween.ll  |  2 +-
 polly/test/Simplify/notdead_region_exitphi.ll |  3 +-
 .../test/Simplify/notdead_region_innerphi.ll  |  3 +-
 .../test/Simplify/notredundant_region_loop.ll |  2 +-
 .../Simplify/notredundant_region_middle.ll    |  3 +-
 .../notredundant_synthesizable_unknownit.ll   |  3 +-
 ...ut-of-scop-use-in-region-entry-phi-node.ll |  2 +-
 polly/test/Simplify/overwritten.ll            |  3 +-
 polly/test/Simplify/overwritten_3phi.ll       |  2 +-
 polly/test/Simplify/overwritten_3store.ll     |  3 +-
 .../overwritten_implicit_and_explicit.ll      |  2 +-
 .../test/Simplify/overwritten_loadbetween.ll  |  3 +-
 polly/test/Simplify/overwritten_scalar.ll     |  2 +-
 polly/test/Simplify/pass_existence.ll         |  3 +-
 polly/test/Simplify/phi_in_regionstmt.ll      |  3 +-
 polly/test/Simplify/pr33323.ll                |  2 +-
 polly/test/Simplify/redundant.ll              |  3 +-
 .../test/Simplify/redundant_differentindex.ll |  3 +-
 polly/test/Simplify/redundant_region.ll       |  2 +-
 .../test/Simplify/redundant_region_scalar.ll  |  2 +-
 polly/test/Simplify/redundant_scalarwrite.ll  |  2 +-
 polly/test/Simplify/redundant_storebetween.ll |  3 +-
 polly/test/Simplify/scalability1.ll           |  2 +-
 polly/test/Simplify/scalability2.ll           |  2 +-
 polly/test/Simplify/sweep_mapped_phi.ll       |  2 +-
 polly/test/Simplify/sweep_mapped_value.ll     |  2 +-
 .../Simplify/ununsed_read_in_region_entry.ll  |  4 +-
 polly/test/Support/Plugins.ll                 |  2 +-
 polly/test/Support/defaultpipelines.ll        | 12 +--
 polly/test/Support/dumpfunction.ll            |  4 +-
 polly/test/Support/dumpmodule.ll              |  4 +-
 polly/test/Support/exportjson.ll              |  2 +-
 polly/test/Support/isl-args.ll                |  8 +-
 polly/test/Support/pipelineposition.ll        |  6 +-
 polly/test/Support/pollyDebug.ll              |  2 +-
 polly/test/lit.site.cfg.in                    |  7 +-
 polly/test/polly.ll                           |  2 +-
 1026 files changed, 1404 insertions(+), 1472 deletions(-)

diff --git a/polly/test/CodeGen/20100617.ll b/polly/test/CodeGen/20100617.ll
index 71a889f067b8..320c48192a8a 100644
--- a/polly/test/CodeGen/20100617.ll
+++ b/polly/test/CodeGen/20100617.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @init_array() nounwind {
diff --git a/polly/test/CodeGen/20100622.ll b/polly/test/CodeGen/20100622.ll
index 872d6a0d75cf..584107df8971 100644
--- a/polly/test/CodeGen/20100622.ll
+++ b/polly/test/CodeGen/20100622.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-codegen -disable-output < %s
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | not FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s | not FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
 
diff --git a/polly/test/CodeGen/20100707.ll b/polly/test/CodeGen/20100707.ll
index 338198084fc7..1a4d3556bae8 100644
--- a/polly/test/CodeGen/20100707.ll
+++ b/polly/test/CodeGen/20100707.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @clause_SetSplitField(i32 %Length) nounwind inlinehint {
diff --git a/polly/test/CodeGen/20100707_2.ll b/polly/test/CodeGen/20100707_2.ll
index df784c6d7957..96f329ccbfa9 100644
--- a/polly/test/CodeGen/20100707_2.ll
+++ b/polly/test/CodeGen/20100707_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @win193 = external global [4 x [36 x double]], align 32 ; <ptr> [#uses=3]
diff --git a/polly/test/CodeGen/20100708.ll b/polly/test/CodeGen/20100708.ll
index 50b8e385df53..00fb6fa694c1 100644
--- a/polly/test/CodeGen/20100708.ll
+++ b/polly/test/CodeGen/20100708.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-detect < %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define fastcc void @execute() nounwind {
diff --git a/polly/test/CodeGen/20100708_2.ll b/polly/test/CodeGen/20100708_2.ll
index 2f4807d9e4d7..67f3913d69d4 100644
--- a/polly/test/CodeGen/20100708_2.ll
+++ b/polly/test/CodeGen/20100708_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @init_array() nounwind {
diff --git a/polly/test/CodeGen/20100713.ll b/polly/test/CodeGen/20100713.ll
index edd352a4c4cc..73ba0272e5c9 100644
--- a/polly/test/CodeGen/20100713.ll
+++ b/polly/test/CodeGen/20100713.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @fft_float(i32 %NumSamples) nounwind {
diff --git a/polly/test/CodeGen/20100713_2.ll b/polly/test/CodeGen/20100713_2.ll
index 92f8959d91d6..c146e00d4231 100644
--- a/polly/test/CodeGen/20100713_2.ll
+++ b/polly/test/CodeGen/20100713_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define hidden void @luaD_callhook() nounwind {
diff --git a/polly/test/CodeGen/20100717.ll b/polly/test/CodeGen/20100717.ll
index a400eeaa3370..24114f2c70c6 100644
--- a/polly/test/CodeGen/20100717.ll
+++ b/polly/test/CodeGen/20100717.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly  -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly  -passes=polly-codegen -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @matrixTranspose(ptr %A) nounwind {
diff --git a/polly/test/CodeGen/20100718-DomInfo-2.ll b/polly/test/CodeGen/20100718-DomInfo-2.ll
index 512b4c5c99af..396a1009a415 100644
--- a/polly/test/CodeGen/20100718-DomInfo-2.ll
+++ b/polly/test/CodeGen/20100718-DomInfo-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -verify-dom-info -disable-output < %s
+; RUN: opt %loadPolly -passes=polly-codegen -verify-dom-info -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @getNonAffNeighbour() nounwind {
diff --git a/polly/test/CodeGen/20100718-DomInfo.ll b/polly/test/CodeGen/20100718-DomInfo.ll
index e12334359c33..7a7f4300e107 100644
--- a/polly/test/CodeGen/20100718-DomInfo.ll
+++ b/polly/test/CodeGen/20100718-DomInfo.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -verify-dom-info -disable-output < %s
+; RUN: opt %loadPolly -passes=polly-codegen -verify-dom-info -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @intrapred_luma_16x16(i32 %predmode) nounwind {
diff --git a/polly/test/CodeGen/20100720-MultipleConditions.ll b/polly/test/CodeGen/20100720-MultipleConditions.ll
index 9f2268713853..ca3758d95681 100644
--- a/polly/test/CodeGen/20100720-MultipleConditions.ll
+++ b/polly/test/CodeGen/20100720-MultipleConditions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-ast -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s
 
 ;int bar1();
 ;int bar2();
diff --git a/polly/test/CodeGen/20100809-IndependentBlock.ll b/polly/test/CodeGen/20100809-IndependentBlock.ll
index 8d596689d8ae..849594afa871 100644
--- a/polly/test/CodeGen/20100809-IndependentBlock.ll
+++ b/polly/test/CodeGen/20100809-IndependentBlock.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 define void @cfft2(ptr %x) nounwind {
 entry:
diff --git a/polly/test/CodeGen/20100811-ScalarDependencyBetweenBrAndCnd.ll b/polly/test/CodeGen/20100811-ScalarDependencyBetweenBrAndCnd.ll
index 261a205560b5..313dad6fe3f7 100644
--- a/polly/test/CodeGen/20100811-ScalarDependencyBetweenBrAndCnd.ll
+++ b/polly/test/CodeGen/20100811-ScalarDependencyBetweenBrAndCnd.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
 target datalayout =
 "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/CodeGen/20101030-Overflow.ll b/polly/test/CodeGen/20101030-Overflow.ll
index caaa4851f93e..bf91272399d2 100644
--- a/polly/test/CodeGen/20101030-Overflow.ll
+++ b/polly/test/CodeGen/20101030-Overflow.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @compdecomp() nounwind {
diff --git a/polly/test/CodeGen/20101103-Overflow3.ll b/polly/test/CodeGen/20101103-Overflow3.ll
index b2faf14fba0b..bab00231758f 100644
--- a/polly/test/CodeGen/20101103-Overflow3.ll
+++ b/polly/test/CodeGen/20101103-Overflow3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 define void @Reflection_coefficients(ptr %r) nounwind {
 bb20:
diff --git a/polly/test/CodeGen/20101103-signmissmatch.ll b/polly/test/CodeGen/20101103-signmissmatch.ll
index e157d292dc8a..d12ed4be4cdf 100644
--- a/polly/test/CodeGen/20101103-signmissmatch.ll
+++ b/polly/test/CodeGen/20101103-signmissmatch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @CleanNet() nounwind {
diff --git a/polly/test/CodeGen/20110226-Ignore-Dead-Code.ll b/polly/test/CodeGen/20110226-Ignore-Dead-Code.ll
index c792d8c3d0bf..1c82d2aba887 100644
--- a/polly/test/CodeGen/20110226-Ignore-Dead-Code.ll
+++ b/polly/test/CodeGen/20110226-Ignore-Dead-Code.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @main() nounwind {
diff --git a/polly/test/CodeGen/20110226-PHI-Node-removed.ll b/polly/test/CodeGen/20110226-PHI-Node-removed.ll
index 3458d75c47a0..d7003882a776 100644
--- a/polly/test/CodeGen/20110226-PHI-Node-removed.ll
+++ b/polly/test/CodeGen/20110226-PHI-Node-removed.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/CodeGen/20120316-InvalidCast.ll b/polly/test/CodeGen/20120316-InvalidCast.ll
index 8355cc51c468..14717dd29b23 100644
--- a/polly/test/CodeGen/20120316-InvalidCast.ll
+++ b/polly/test/CodeGen/20120316-InvalidCast.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 
 ; CHECK: polly.start
 
diff --git a/polly/test/CodeGen/20120403-RHS-type-mismatch.ll b/polly/test/CodeGen/20120403-RHS-type-mismatch.ll
index 1d629e388452..2d3e3b02dd38 100644
--- a/polly/test/CodeGen/20120403-RHS-type-mismatch.ll
+++ b/polly/test/CodeGen/20120403-RHS-type-mismatch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 
 ; We just check that this compilation does not crash.
 
diff --git a/polly/test/CodeGen/20130221.ll b/polly/test/CodeGen/20130221.ll
index 45414671081a..e5f63adabc25 100644
--- a/polly/test/CodeGen/20130221.ll
+++ b/polly/test/CodeGen/20130221.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 define void @list_sequence(ptr %A) {
diff --git a/polly/test/CodeGen/20150328-SCEVExpanderIntroducesNewIV.ll b/polly/test/CodeGen/20150328-SCEVExpanderIntroducesNewIV.ll
index d54be5c3f35f..786adc7286e6 100644
--- a/polly/test/CodeGen/20150328-SCEVExpanderIntroducesNewIV.ll
+++ b/polly/test/CodeGen/20150328-SCEVExpanderIntroducesNewIV.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/Intrinsics/llvm-expect.ll b/polly/test/CodeGen/Intrinsics/llvm-expect.ll
index 84057e276521..ac65f6f439ae 100644
--- a/polly/test/CodeGen/Intrinsics/llvm-expect.ll
+++ b/polly/test/CodeGen/Intrinsics/llvm-expect.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ; Check that we generate code without crashing.
 ;
diff --git a/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll b/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll
index b04319550938..fb02d7f55e23 100644
--- a/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll
+++ b/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll
@@ -1,6 +1,6 @@
 ; This test checks that we do not accidently mutate the debug info when
 ; inserting loop parallel metadata.
-; RUN: opt %loadPolly < %s  -S -polly -polly-codegen -polly-ast-detect-parallel | FileCheck %s
+; RUN: opt %loadPolly < %s  -S -polly -passes=polly-codegen -polly-ast-detect-parallel | FileCheck %s
 ; CHECK-NOT: !7 = !{!7}
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/LoopParallelMD/loop_nest_param_parallel.ll b/polly/test/CodeGen/LoopParallelMD/loop_nest_param_parallel.ll
index 7b131c5ebcbd..c6c27b4a75a8 100644
--- a/polly/test/CodeGen/LoopParallelMD/loop_nest_param_parallel.ll
+++ b/polly/test/CodeGen/LoopParallelMD/loop_nest_param_parallel.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-ast-detect-parallel -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-ast-detect-parallel -S < %s | FileCheck %s
 ;
 ; Check that we mark multiple parallel loops correctly including the memory instructions.
 ;
diff --git a/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll b/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll
index ec927acb1ec7..f91bd64a895a 100644
--- a/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll
+++ b/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=SEQUENTIAL
-; RUN: opt %loadPolly -polly-codegen -polly-ast-detect-parallel -S < %s | FileCheck %s -check-prefix=PARALLEL
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=SEQUENTIAL
+; RUN: opt %loadPolly -passes=polly-codegen -polly-ast-detect-parallel -S < %s | FileCheck %s -check-prefix=PARALLEL
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ; This is a trivially parallel loop. We just use it to ensure that we actually
diff --git a/polly/test/CodeGen/MemAccess/bad_alignment.ll b/polly/test/CodeGen/MemAccess/bad_alignment.ll
index 32f3cfe963b7..2f297384ccdb 100644
--- a/polly/test/CodeGen/MemAccess/bad_alignment.ll
+++ b/polly/test/CodeGen/MemAccess/bad_alignment.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -disable-output 2>&1 < %s | FileCheck %s
+; RUN: not --crash opt %loadPolly -passes=polly-import-jscop -disable-output 2>&1 < %s | FileCheck %s
 ;
 ; Check that we do not allow to access elements not accessed before because the
 ; alignment information would become invalid.
diff --git a/polly/test/CodeGen/MemAccess/codegen_address_space.ll b/polly/test/CodeGen/MemAccess/codegen_address_space.ll
index 7c9b12d64f9c..986775ed5717 100644
--- a/polly/test/CodeGen/MemAccess/codegen_address_space.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_address_space.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
+;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed < %s -S | FileCheck %s
 
 ;int A[100];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll b/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll
index e008a789fe7d..b4168b89267b 100644
--- a/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
+;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed < %s -S | FileCheck %s
 
 ;int A[100];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_simple.ll b/polly/test/CodeGen/MemAccess/codegen_simple.ll
index 5ba6f3269fb9..cf2a57ad8a1f 100644
--- a/polly/test/CodeGen/MemAccess/codegen_simple.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_simple.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
+;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed < %s -S | FileCheck %s
 
 ;int A[100];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_simple_float.ll b/polly/test/CodeGen/MemAccess/codegen_simple_float.ll
index cf8913fc5197..bdd0e56b0a29 100644
--- a/polly/test/CodeGen/MemAccess/codegen_simple_float.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_simple_float.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
+;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed < %s -S | FileCheck %s
 ;
 ;float A[100];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_simple_md.ll b/polly/test/CodeGen/MemAccess/codegen_simple_md.ll
index e4afcc8d2243..8f676725146d 100644
--- a/polly/test/CodeGen/MemAccess/codegen_simple_md.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_simple_md.ll
@@ -1,5 +1,5 @@
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed+withconst -polly-codegen < %s -S | FileCheck -check-prefix=WITHCONST %s
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed+withoutconst -polly-codegen < %s -S | FileCheck -check-prefix=WITHOUTCONST %s
+;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed+withconst < %s -S | FileCheck -check-prefix=WITHCONST %s
+;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed+withoutconst < %s -S | FileCheck -check-prefix=WITHOUTCONST %s
 
 ;int A[1040];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_simple_md_float.ll b/polly/test/CodeGen/MemAccess/codegen_simple_md_float.ll
index c9913f3ed873..1ac1efa2f727 100644
--- a/polly/test/CodeGen/MemAccess/codegen_simple_md_float.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_simple_md_float.ll
@@ -1,5 +1,5 @@
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed+withconst -polly-codegen < %s -S | FileCheck -check-prefix=WITHCONST %s
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed+withoutconst -polly-codegen < %s -S | FileCheck -check-prefix=WITHOUTCONST %s
+;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed+withconst < %s -S | FileCheck -check-prefix=WITHCONST %s
+;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed+withoutconst < %s -S | FileCheck -check-prefix=WITHOUTCONST %s
 ;
 ;float A[1040];
 ;
diff --git a/polly/test/CodeGen/MemAccess/different_types.ll b/polly/test/CodeGen/MemAccess/different_types.ll
index 624de62911ff..52fca9d87759 100644
--- a/polly/test/CodeGen/MemAccess/different_types.ll
+++ b/polly/test/CodeGen/MemAccess/different_types.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-import-jscop \
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
 ; RUN: \
-; RUN: -polly-codegen -S < %s | FileCheck %s
+; RUN: -S < %s | FileCheck %s
 ;
 ;    void foo(float A[], float B[]) {
 ;      for (long i = 0; i < 100; i++)
diff --git a/polly/test/CodeGen/MemAccess/generate-all.ll b/polly/test/CodeGen/MemAccess/generate-all.ll
index 6f92ba13587e..a64c6db0978c 100644
--- a/polly/test/CodeGen/MemAccess/generate-all.ll
+++ b/polly/test/CodeGen/MemAccess/generate-all.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-codegen -polly-codegen-generate-expressions=false \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-codegen-generate-expressions=false \
 ; RUN:     -S < %s | FileCheck %s -check-prefix=SCEV
-; RUN: opt %loadPolly -polly-codegen -polly-codegen-generate-expressions=true \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-codegen-generate-expressions=true \
 ; RUN:     -S < %s | FileCheck %s -check-prefix=ASTEXPR
 ;
 ;    void foo(float A[]) {
diff --git a/polly/test/CodeGen/MemAccess/invariant_base_ptr.ll b/polly/test/CodeGen/MemAccess/invariant_base_ptr.ll
index a6d1de0aac63..12c38b7a66c4 100644
--- a/polly/test/CodeGen/MemAccess/invariant_base_ptr.ll
+++ b/polly/test/CodeGen/MemAccess/invariant_base_ptr.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-import-jscop \
-; RUN:   -polly-codegen -polly-invariant-load-hoisting -S \
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
+; RUN:   -polly-invariant-load-hoisting -S \
 ; RUN:   2>&1 < %s | FileCheck %s
 
 ; Setting new access functions where the base pointer of the array that is newly
diff --git a/polly/test/CodeGen/MemAccess/multiple_types.ll b/polly/test/CodeGen/MemAccess/multiple_types.ll
index 1793bd30fc5b..7c0ddffbd6d1 100644
--- a/polly/test/CodeGen/MemAccess/multiple_types.ll
+++ b/polly/test/CodeGen/MemAccess/multiple_types.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,polly-codegen' \
 ; RUN: -polly-allow-differing-element-types \
-; RUN:   -polly-codegen -S    < %s | FileCheck %s
+; RUN:   -S    < %s | FileCheck %s
 ;
 ;    // Check that accessing one array with different types works.
 ;    void multiple_types(char *Short, char *Float, char *Double) {
diff --git a/polly/test/CodeGen/MemAccess/simple.ll b/polly/test/CodeGen/MemAccess/simple.ll
index 39e8a2c91b79..8964e189b8a7 100644
--- a/polly/test/CodeGen/MemAccess/simple.ll
+++ b/polly/test/CodeGen/MemAccess/simple.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -stats < %s 2>&1  | FileCheck %s
+;RUN: opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -stats < %s 2>&1  | FileCheck %s
 ; REQUIRES: asserts
 
 ;int A[100];
diff --git a/polly/test/CodeGen/MemAccess/update_access_functions.ll b/polly/test/CodeGen/MemAccess/update_access_functions.ll
index 05d208708a36..00644c52ecd7 100644
--- a/polly/test/CodeGen/MemAccess/update_access_functions.ll
+++ b/polly/test/CodeGen/MemAccess/update_access_functions.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-import-jscop \
-; RUN:                -polly-import-jscop-postfix=transformed -polly-codegen \
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
+; RUN:                -polly-import-jscop-postfix=transformed \
 ; RUN:                 < %s -S | FileCheck %s
 
 ; CHECK-LABEL: polly.stmt.loop1:
diff --git a/polly/test/CodeGen/OpenMP/alias-metadata.ll b/polly/test/CodeGen/OpenMP/alias-metadata.ll
index 07d79631b2cb..e7ca6abac283 100644
--- a/polly/test/CodeGen/OpenMP/alias-metadata.ll
+++ b/polly/test/CodeGen/OpenMP/alias-metadata.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-parallel -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -S < %s | FileCheck %s
 ;
 ;    void foo(float *A, float *B) {
 ;      for (long i = 0; i < 1000; i++)
diff --git a/polly/test/CodeGen/OpenMP/floord-as-argument-to-subfunction.ll b/polly/test/CodeGen/OpenMP/floord-as-argument-to-subfunction.ll
index eb9dfcd9e920..40ea62088940 100644
--- a/polly/test/CodeGen/OpenMP/floord-as-argument-to-subfunction.ll
+++ b/polly/test/CodeGen/OpenMP/floord-as-argument-to-subfunction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-opt-max-coefficient=-1 -polly-parallel -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-opt-max-coefficient=-1 -polly-parallel -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; Check that we do not crash but generate parallel code
 ;
diff --git a/polly/test/CodeGen/OpenMP/inlineasm.ll b/polly/test/CodeGen/OpenMP/inlineasm.ll
index 69b1b0aa53f3..a2ca2f79d649 100644
--- a/polly/test/CodeGen/OpenMP/inlineasm.ll
+++ b/polly/test/CodeGen/OpenMP/inlineasm.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-parallel -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-opt-isl,polly-codegen' -polly-parallel -S < %s | FileCheck %s
 ; llvm.org/PR51960
 
 ; CHECK-LABEL: define internal void @foo_polly_subfn
diff --git a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded.ll b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded.ll
index 30beef5b0709..9394755254a7 100644
--- a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded.ll
+++ b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
 ; RUN: -polly-parallel-force -S < %s | FileCheck %s
 ;
 ; Test to verify that we hand down the preloaded A[0] to the OpenMP subfunction.
diff --git a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_different_bb.ll b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_different_bb.ll
index fe5d2ab8c96d..48f075c37d82 100644
--- a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_different_bb.ll
+++ b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_different_bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
 ; RUN: -polly-parallel-force -S < %s | FileCheck %s
 ;
 ; Test to verify that we hand down the preloaded A[0] to the OpenMP subfunction.
diff --git a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_pass_only_needed.ll b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_pass_only_needed.ll
index 49b9321c40b8..e4273384dd72 100644
--- a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_pass_only_needed.ll
+++ b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_pass_only_needed.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
 ; RUN: -polly-parallel-force -S < %s | FileCheck %s
 ;
 ; Test to verify that we hand down the preloaded A[0] to the OpenMP subfunction but
diff --git a/polly/test/CodeGen/OpenMP/invariant_base_pointers_preloaded.ll b/polly/test/CodeGen/OpenMP/invariant_base_pointers_preloaded.ll
index 06c4cdab45f1..8400a5594d10 100644
--- a/polly/test/CodeGen/OpenMP/invariant_base_pointers_preloaded.ll
+++ b/polly/test/CodeGen/OpenMP/invariant_base_pointers_preloaded.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
 ; RUN: -polly-parallel-force -S < %s | FileCheck %s
 ;
 ; Test to verify that we hand down the preloaded A[0] to the OpenMP subfunction.
diff --git a/polly/test/CodeGen/OpenMP/loop-body-references-outer-iv.ll b/polly/test/CodeGen/OpenMP/loop-body-references-outer-iv.ll
index db58c3ab7593..b3297b5b32af 100644
--- a/polly/test/CodeGen/OpenMP/loop-body-references-outer-iv.ll
+++ b/polly/test/CodeGen/OpenMP/loop-body-references-outer-iv.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; This code has failed the scev based code generation as the scev in the scop
 ; contains an AddRecExpr of an outer loop. When generating code, we did not
diff --git a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-2.ll b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-2.ll
index c2ddc1e26496..c478e91eb9db 100644
--- a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-2.ll
+++ b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; AST: #pragma simd
 ; AST: #pragma omp parallel for
diff --git a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-3.ll b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-3.ll
index 0f025bb94112..26c2fe6da6ca 100644
--- a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-3.ll
+++ b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -basic-aa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; The interesting part of this test case is the instruction:
 ;   %tmp = bitcast i8* %call to i64**
diff --git a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values.ll b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values.ll
index f9612d77533d..debd89ab151d 100644
--- a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values.ll
+++ b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=IR
 
 ; Make sure we correctly forward the reference to 'A' to the OpenMP subfunction.
 ;
diff --git a/polly/test/CodeGen/OpenMP/loop-bounds-reference-outer-ids.ll b/polly/test/CodeGen/OpenMP/loop-bounds-reference-outer-ids.ll
index da9da18c89b2..db2299e5e73d 100644
--- a/polly/test/CodeGen/OpenMP/loop-bounds-reference-outer-ids.ll
+++ b/polly/test/CodeGen/OpenMP/loop-bounds-reference-outer-ids.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-codegen -S < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=IR
 ;
 ; float A[100];
 ;
diff --git a/polly/test/CodeGen/OpenMP/mapped-phi-access.ll b/polly/test/CodeGen/OpenMP/mapped-phi-access.ll
index 1b8433693abf..4b71760ea224 100644
--- a/polly/test/CodeGen/OpenMP/mapped-phi-access.ll
+++ b/polly/test/CodeGen/OpenMP/mapped-phi-access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-parallel -polly-delicm -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-parallel '-passes=polly-delicm,polly-codegen' -S < %s | FileCheck %s
 ;
 ; Verify that -polly-parallel can handle mapped scalar MemoryAccesses.
 ;
diff --git a/polly/test/CodeGen/OpenMP/matmul-parallel.ll b/polly/test/CodeGen/OpenMP/matmul-parallel.ll
index 5ee9a7c7a824..1f3ad5ca8426 100644
--- a/polly/test/CodeGen/OpenMP/matmul-parallel.ll
+++ b/polly/test/CodeGen/OpenMP/matmul-parallel.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-opt-isl -polly-ast -disable-output -debug-only=polly-ast < %s 2>&1 | FileCheck --check-prefix=AST %s
-; RUN: opt %loadPolly -polly-parallel -polly-opt-isl -polly-codegen -S < %s | FileCheck --check-prefix=CODEGEN %s
+; RUN: opt %loadPolly -polly-parallel '-passes=polly-opt-isl,print<polly-ast>' -disable-output -debug-only=polly-ast < %s 2>&1 | FileCheck --check-prefix=AST %s
+; RUN: opt %loadPolly -polly-parallel '-passes=polly-opt-isl,polly-codegen' -S < %s | FileCheck --check-prefix=CODEGEN %s
 ; REQUIRES: asserts
 
 ; Parallelization of detected matrix-multiplication.
diff --git a/polly/test/CodeGen/OpenMP/recomputed-srem.ll b/polly/test/CodeGen/OpenMP/recomputed-srem.ll
index cfae8e943cf1..3411308a4b11 100644
--- a/polly/test/CodeGen/OpenMP/recomputed-srem.ll
+++ b/polly/test/CodeGen/OpenMP/recomputed-srem.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-codegen -polly-parallel \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -passes=polly-codegen -polly-parallel \
 ; RUN: -polly-parallel-force -S < %s | FileCheck %s
 ;
 ; Test to verify that we pass %rem96 to the parallel subfunction.
diff --git a/polly/test/CodeGen/OpenMP/reference-argument-from-non-affine-region.ll b/polly/test/CodeGen/OpenMP/reference-argument-from-non-affine-region.ll
index f243c3a04949..247e89be0027 100644
--- a/polly/test/CodeGen/OpenMP/reference-argument-from-non-affine-region.ll
+++ b/polly/test/CodeGen/OpenMP/reference-argument-from-non-affine-region.ll
@@ -1,15 +1,15 @@
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -polly-codegen \
+; RUN: -polly-parallel-force -passes=polly-codegen \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=IR
 
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -polly-codegen -polly-scheduling=runtime \
+; RUN: -polly-parallel-force -passes=polly-codegen -polly-scheduling=runtime \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=IR
 
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM \
+; RUN: -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=LIBOMP-IR
 
diff --git a/polly/test/CodeGen/OpenMP/reference-other-bb.ll b/polly/test/CodeGen/OpenMP/reference-other-bb.ll
index b7abdc23d258..2c399f12af24 100644
--- a/polly/test/CodeGen/OpenMP/reference-other-bb.ll
+++ b/polly/test/CodeGen/OpenMP/reference-other-bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; IR: @foo_polly_subfn
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/CodeGen/OpenMP/reference-preceeding-loop.ll b/polly/test/CodeGen/OpenMP/reference-preceeding-loop.ll
index b88589f39a6f..90a95dccbcb1 100644
--- a/polly/test/CodeGen/OpenMP/reference-preceeding-loop.ll
+++ b/polly/test/CodeGen/OpenMP/reference-preceeding-loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 
 ; - Test the case where scalar evolution references a loop that is outside
diff --git a/polly/test/CodeGen/OpenMP/reference_latest.ll b/polly/test/CodeGen/OpenMP/reference_latest.ll
index 54875c2630f0..696c3c74ad0e 100644
--- a/polly/test/CodeGen/OpenMP/reference_latest.ll
+++ b/polly/test/CodeGen/OpenMP/reference_latest.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-delicm -polly-simplify -polly-parallel -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-delicm,polly-simplify,polly-codegen' -polly-parallel -S < %s | FileCheck %s
 ;
 ; Test that parallel codegen handles scalars mapped to other arrays.
 ; After mapping "store double %add10" references the array "MemRef2".
diff --git a/polly/test/CodeGen/OpenMP/scev-rewriting.ll b/polly/test/CodeGen/OpenMP/scev-rewriting.ll
index 1b229fc19d25..551946fc698b 100644
--- a/polly/test/CodeGen/OpenMP/scev-rewriting.ll
+++ b/polly/test/CodeGen/OpenMP/scev-rewriting.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly < %s -polly-vectorizer=stripmine -polly-parallel -polly-parallel-force -polly-process-unprofitable -polly-codegen -S | FileCheck %s
+; RUN: opt %loadPolly < %s -polly-vectorizer=stripmine -polly-parallel -polly-parallel-force -polly-process-unprofitable -passes=polly-codegen -S | FileCheck %s
 ; CHECK: define internal void @DoStringSort_polly_subfn
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnueabi"
diff --git a/polly/test/CodeGen/OpenMP/single_loop.ll b/polly/test/CodeGen/OpenMP/single_loop.ll
index f79653a08d21..7e45ab08080e 100644
--- a/polly/test/CodeGen/OpenMP/single_loop.ll
+++ b/polly/test/CodeGen/OpenMP/single_loop.ll
@@ -1,14 +1,14 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-import-jscop -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST-STRIDE4
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-import-jscop -polly-codegen -S < %s | FileCheck %s -check-prefix=IR-STRIDE4
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=polly-import-jscop,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST-STRIDE4
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=polly-import-jscop,polly-codegen' -S < %s | FileCheck %s -check-prefix=IR-STRIDE4
 
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=static -polly-scheduling-chunksize=43 -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-STATIC-CHUNKED
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=static -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-STATIC
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=dynamic -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-DYNAMIC
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=dynamic -polly-scheduling-chunksize=4 -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-DYNAMIC-FOUR
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-import-jscop -polly-codegen -polly-omp-backend=LLVM -S < %s | FileCheck %s -check-prefix=LIBOMP-IR-STRIDE4
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM -polly-scheduling=static -polly-scheduling-chunksize=43 -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-STATIC-CHUNKED
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM -polly-scheduling=static -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-STATIC
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM -polly-scheduling=dynamic -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-DYNAMIC
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM -polly-scheduling=dynamic -polly-scheduling-chunksize=4 -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-DYNAMIC-FOUR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=polly-import-jscop,polly-codegen' -polly-omp-backend=LLVM -S < %s | FileCheck %s -check-prefix=LIBOMP-IR-STRIDE4
 
 ; This extensive test case tests the creation of the full set of OpenMP calls
 ; as well as the subfunction creation using a trivial loop as example.
diff --git a/polly/test/CodeGen/OpenMP/single_loop_with_loop_invariant_baseptr.ll b/polly/test/CodeGen/OpenMP/single_loop_with_loop_invariant_baseptr.ll
index 50da5dd2b7c0..519cbbc496b1 100644
--- a/polly/test/CodeGen/OpenMP/single_loop_with_loop_invariant_baseptr.ll
+++ b/polly/test/CodeGen/OpenMP/single_loop_with_loop_invariant_baseptr.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -tbaa -polly-parallel -polly-parallel-force -polly-parallel-force -polly-invariant-load-hoisting=true -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -tbaa -polly-parallel -polly-parallel-force -polly-parallel-force -polly-invariant-load-hoisting=true -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -aa-pipeline=tbaa -polly-parallel -polly-parallel-force -polly-parallel-force -polly-invariant-load-hoisting=true '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -aa-pipeline=tbaa -polly-parallel -polly-parallel-force -polly-parallel-force -polly-invariant-load-hoisting=true -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; #define N 1024
 ; float A[N];
diff --git a/polly/test/CodeGen/OpenMP/single_loop_with_param.ll b/polly/test/CodeGen/OpenMP/single_loop_with_param.ll
index d01b7a2fdcad..0288d4c8f5e4 100644
--- a/polly/test/CodeGen/OpenMP/single_loop_with_param.ll
+++ b/polly/test/CodeGen/OpenMP/single_loop_with_param.ll
@@ -1,15 +1,15 @@
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -polly-codegen \
+; RUN: -polly-parallel-force -passes=polly-codegen \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=IR
 
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM \
+; RUN: -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=LIBOMP-IR
 
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM \
+; RUN: -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM \
 ; RUN: -polly-scheduling=static \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=LIBOMP-STATIC-IR
diff --git a/polly/test/CodeGen/OpenMP/two-parallel-loops-reference-outer-indvar.ll b/polly/test/CodeGen/OpenMP/two-parallel-loops-reference-outer-indvar.ll
index 05c6ed177e9c..133a00c3be71 100644
--- a/polly/test/CodeGen/OpenMP/two-parallel-loops-reference-outer-indvar.ll
+++ b/polly/test/CodeGen/OpenMP/two-parallel-loops-reference-outer-indvar.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; This test case verifies that we create correct code even if two OpenMP loops
 ; share common outer variables.
diff --git a/polly/test/CodeGen/PHIInExit.ll b/polly/test/CodeGen/PHIInExit.ll
index eadd6054386b..5617d873e529 100644
--- a/polly/test/CodeGen/PHIInExit.ll
+++ b/polly/test/CodeGen/PHIInExit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 %struct..0__pthread_mutex_s = type { i32, i32, i32, i32, i32, i32, %struct.__pthread_list_t }
diff --git a/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll b/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll
index 84827dd26049..c7f9186c3777 100644
--- a/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll
+++ b/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S \
+; RUN: opt %loadPolly -passes=polly-codegen -S \
 ; RUN: -polly-codegen-add-debug-printing \
 ; RUN: -polly-ignore-aliasing < %s | FileCheck %s
 
diff --git a/polly/test/CodeGen/RuntimeDebugBuilder/stmt_tracing.ll b/polly/test/CodeGen/RuntimeDebugBuilder/stmt_tracing.ll
index 822eccc306ef..80fa3ee8d0f2 100644
--- a/polly/test/CodeGen/RuntimeDebugBuilder/stmt_tracing.ll
+++ b/polly/test/CodeGen/RuntimeDebugBuilder/stmt_tracing.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen-trace-stmts -polly-codegen-trace-scalars -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen-trace-stmts -polly-codegen-trace-scalars -passes=polly-codegen -S < %s | FileCheck %s
 ;
 
 define void @func(i32 %n, ptr %A) {
diff --git a/polly/test/CodeGen/alias-check-multi-dim.ll b/polly/test/CodeGen/alias-check-multi-dim.ll
index d923a4cc14fd..821e19290612 100644
--- a/polly/test/CodeGen/alias-check-multi-dim.ll
+++ b/polly/test/CodeGen/alias-check-multi-dim.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen \
+; RUN: opt %loadPolly -passes=polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/alias_metadata_too_many_arrays.ll b/polly/test/CodeGen/alias_metadata_too_many_arrays.ll
index 7c5ca012a378..9207f3015e3a 100644
--- a/polly/test/CodeGen/alias_metadata_too_many_arrays.ll
+++ b/polly/test/CodeGen/alias_metadata_too_many_arrays.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-ignore-aliasing -S < %s \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-ignore-aliasing -S < %s \
 ; RUN:   | FileCheck %s
 ;
 ;    void manyarrays(float A1[], float A2[], float A3[], float A4[], float A5[],
diff --git a/polly/test/CodeGen/aliasing_different_base_and_access_type.ll b/polly/test/CodeGen/aliasing_different_base_and_access_type.ll
index a087414b8403..d74f51702f36 100644
--- a/polly/test/CodeGen/aliasing_different_base_and_access_type.ll
+++ b/polly/test/CodeGen/aliasing_different_base_and_access_type.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ; We have to cast %B to "short *" before we create RTCs.
 ;
diff --git a/polly/test/CodeGen/aliasing_different_pointer_types.ll b/polly/test/CodeGen/aliasing_different_pointer_types.ll
index 91f5eab6b2a6..5ba1d1b587bf 100644
--- a/polly/test/CodeGen/aliasing_different_pointer_types.ll
+++ b/polly/test/CodeGen/aliasing_different_pointer_types.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; Check that we cast the different pointer types correctly before we compare
 ; them in the RTC's. We use i8* as max pointer type.
diff --git a/polly/test/CodeGen/aliasing_multidimensional_access.ll b/polly/test/CodeGen/aliasing_multidimensional_access.ll
index 48768399e850..338ab05e4ad3 100644
--- a/polly/test/CodeGen/aliasing_multidimensional_access.ll
+++ b/polly/test/CodeGen/aliasing_multidimensional_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ; Check that we calculate the maximal access into array A correctly and track the overflow state.
 ;
diff --git a/polly/test/CodeGen/aliasing_parametric_simple_1.ll b/polly/test/CodeGen/aliasing_parametric_simple_1.ll
index 5422da4426e9..281ecf488fd4 100644
--- a/polly/test/CodeGen/aliasing_parametric_simple_1.ll
+++ b/polly/test/CodeGen/aliasing_parametric_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ;    void jd(int *A, int *B, int c) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/CodeGen/aliasing_parametric_simple_2.ll b/polly/test/CodeGen/aliasing_parametric_simple_2.ll
index de945d403f92..9ac59f93febb 100644
--- a/polly/test/CodeGen/aliasing_parametric_simple_2.ll
+++ b/polly/test/CodeGen/aliasing_parametric_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ;    void jd(int *A, int *B, int c) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/CodeGen/aliasing_struct_element.ll b/polly/test/CodeGen/aliasing_struct_element.ll
index 2219ca9d28bb..32b7c1ac905c 100644
--- a/polly/test/CodeGen/aliasing_struct_element.ll
+++ b/polly/test/CodeGen/aliasing_struct_element.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ; We should only access (or compute the address of) "the first element" of %S
 ; as it is a single struct not a struct array. The maximal access to S, thus
diff --git a/polly/test/CodeGen/alignment.ll b/polly/test/CodeGen/alignment.ll
index a94b1f7e2883..f3c786780f0a 100644
--- a/polly/test/CodeGen/alignment.ll
+++ b/polly/test/CodeGen/alignment.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; Check that the special alignment information is kept
 ;
diff --git a/polly/test/CodeGen/annotated_alias_scopes.ll b/polly/test/CodeGen/annotated_alias_scopes.ll
index f8d14cd34b62..1dd409edc9a5 100644
--- a/polly/test/CodeGen/annotated_alias_scopes.ll
+++ b/polly/test/CodeGen/annotated_alias_scopes.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s --check-prefix=SCOPES
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s --check-prefix=SCOPES
 ;
 ; Check that we create alias scopes that indicate the accesses to A, B and C cannot alias in any way.
 ;
diff --git a/polly/test/CodeGen/blas_sscal_simplified.ll b/polly/test/CodeGen/blas_sscal_simplified.ll
index a370fcff46f8..b2072d3822a2 100644
--- a/polly/test/CodeGen/blas_sscal_simplified.ll
+++ b/polly/test/CodeGen/blas_sscal_simplified.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 ;
 ; Regression test for a bug in the runtime check generation.
 
diff --git a/polly/test/CodeGen/conflict-between-loop-invariant-code-hosting-and-escape-map-computation.ll b/polly/test/CodeGen/conflict-between-loop-invariant-code-hosting-and-escape-map-computation.ll
index e0f8c435879a..e977627ed7be 100644
--- a/polly/test/CodeGen/conflict-between-loop-invariant-code-hosting-and-escape-map-computation.ll
+++ b/polly/test/CodeGen/conflict-between-loop-invariant-code-hosting-and-escape-map-computation.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-process-unprofitable -passes=polly-codegen -disable-output < %s
 ;
 ; CHECK: store i32 %tmp14_p_scalar_, ptr %tmp14.s2a
 ; CHECK: %tmp14.final_reload = load i32, ptr %tmp14.s2a
diff --git a/polly/test/CodeGen/constant_condition.ll b/polly/test/CodeGen/constant_condition.ll
index dad1f6cffd17..e259b5799763 100644
--- a/polly/test/CodeGen/constant_condition.ll
+++ b/polly/test/CodeGen/constant_condition.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly -polly-prepare -polly-print-ast -disable-output < %s | FileCheck %s
+;RUN: opt %loadPolly '-passes=polly-prepare,scop(print<polly-ast>)' -disable-output < %s 2>&1 | FileCheck %s
 
 ;#include <string.h>
 ;int A[1];
diff --git a/polly/test/CodeGen/create-conditional-scop.ll b/polly/test/CodeGen/create-conditional-scop.ll
index f51a2dcc9b3c..235726fdaf15 100644
--- a/polly/test/CodeGen/create-conditional-scop.ll
+++ b/polly/test/CodeGen/create-conditional-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-codegen -verify-loop-info < %s -S | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -verify-loop-info < %s -S | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
 
diff --git a/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_1.ll b/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_1.ll
index 991e3c83eef1..220bdc179158 100644
--- a/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_1.ll
+++ b/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 ;
 ; Check we do not crash even though the dead %tmp8 is referenced by a parameter
 ; and we do not pre-load it (as it is dead).
diff --git a/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_2.ll b/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_2.ll
index 153f6912cea5..4830de888237 100644
--- a/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_2.ll
+++ b/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 ;
 ; Check we do not crash even though there is a dead load that is referenced by
 ; a parameter and we do not pre-load it (as it is dead).
diff --git a/polly/test/CodeGen/debug-intrinsics.ll b/polly/test/CodeGen/debug-intrinsics.ll
index 2feeb7c838b0..c98fae8e2e10 100644
--- a/polly/test/CodeGen/debug-intrinsics.ll
+++ b/polly/test/CodeGen/debug-intrinsics.ll
@@ -1,9 +1,9 @@
 ; RUN: opt %loadPolly \
-; RUN: -polly-analyze-read-only-scalars=false -polly-codegen -S < %s | \
+; RUN: -polly-analyze-read-only-scalars=false -passes=polly-codegen -S < %s | \
 ; RUN: FileCheck %s
 
 ; RUN: opt %loadPolly \
-; RUN: -polly-analyze-read-only-scalars=true -polly-codegen -S < %s | \
+; RUN: -polly-analyze-read-only-scalars=true -passes=polly-codegen -S < %s | \
 ; RUN: FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/CodeGen/dominance_problem_after_early_codegen_bailout.ll b/polly/test/CodeGen/dominance_problem_after_early_codegen_bailout.ll
index c9e006a01204..2a3f3bd8a065 100644
--- a/polly/test/CodeGen/dominance_problem_after_early_codegen_bailout.ll
+++ b/polly/test/CodeGen/dominance_problem_after_early_codegen_bailout.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
 ;
 ; This caused dominance problems at some point as we do bail out during
 ; code generation. Just verify it runs through.
diff --git a/polly/test/CodeGen/empty_domain_in_context.ll b/polly/test/CodeGen/empty_domain_in_context.ll
index c67ace9502e1..b7a6b95171cb 100644
--- a/polly/test/CodeGen/empty_domain_in_context.ll
+++ b/polly/test/CodeGen/empty_domain_in_context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-optree -polly-opt-isl -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-optree,polly-opt-isl,polly-codegen' -S < %s | FileCheck %s
 ;
 ; llvm.org/PR35362
 ; isl codegen does not allow to generate isl_ast_expr from pw_aff which have an
diff --git a/polly/test/CodeGen/entry_with_trivial_phi.ll b/polly/test/CodeGen/entry_with_trivial_phi.ll
index b057690ab29b..99d0776f3aa9 100644
--- a/polly/test/CodeGen/entry_with_trivial_phi.ll
+++ b/polly/test/CodeGen/entry_with_trivial_phi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s
 ;
 ; The entry of this scop's simple region (entry.split => for.end) has an trivial
 ; PHI node. LCSSA may create such PHI nodes. This is a breakdown of this case in
diff --git a/polly/test/CodeGen/entry_with_trivial_phi_other_bb.ll b/polly/test/CodeGen/entry_with_trivial_phi_other_bb.ll
index 5673cc746b5f..ec3967b51049 100644
--- a/polly/test/CodeGen/entry_with_trivial_phi_other_bb.ll
+++ b/polly/test/CodeGen/entry_with_trivial_phi_other_bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; The entry of this scop's simple region (entry.split => for.end) has an trivial
 ; PHI node that is used in a different of the scop region. LCSSA may create such
diff --git a/polly/test/CodeGen/error-stmt-in-non-affine-region.ll b/polly/test/CodeGen/error-stmt-in-non-affine-region.ll
index 9832afe7a5fd..ab85fbabd3f8 100644
--- a/polly/test/CodeGen/error-stmt-in-non-affine-region.ll
+++ b/polly/test/CodeGen/error-stmt-in-non-affine-region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ; XFAIL: *
 ;
 ; CHECK-LABEL: polly.stmt.if.then:
diff --git a/polly/test/CodeGen/error_block_contains_invalid_memory_access.ll b/polly/test/CodeGen/error_block_contains_invalid_memory_access.ll
index 048847f3e322..6d1c3f74cc03 100644
--- a/polly/test/CodeGen/error_block_contains_invalid_memory_access.ll
+++ b/polly/test/CodeGen/error_block_contains_invalid_memory_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/exprModDiv.ll b/polly/test/CodeGen/exprModDiv.ll
index 936b018bc1ad..625e0e6464d7 100644
--- a/polly/test/CodeGen/exprModDiv.ll
+++ b/polly/test/CodeGen/exprModDiv.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-import-jscop \
-; RUN:     -polly-codegen -S < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-import-jscop \
-; RUN:     -polly-codegen -polly-import-jscop-postfix=pow2 \
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
+; RUN:     -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
+; RUN:     -polly-import-jscop-postfix=pow2 \
 ; RUN:     -S < %s | FileCheck %s -check-prefix=POW2
 ;
 ;    void exprModDiv(float *A, float *B, float *C, long N, long p) {
diff --git a/polly/test/CodeGen/hoisted_load_escapes_through_phi.ll b/polly/test/CodeGen/hoisted_load_escapes_through_phi.ll
index d7588b3b8e00..ad7d84648a09 100644
--- a/polly/test/CodeGen/hoisted_load_escapes_through_phi.ll
+++ b/polly/test/CodeGen/hoisted_load_escapes_through_phi.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -S -polly-codegen \
+; RUN: opt %loadPolly -S -passes=polly-codegen \
 ; RUN: -polly-invariant-load-hoisting=false < %s | FileCheck %s
-; RUN: opt %loadPolly -S -polly-codegen \
+; RUN: opt %loadPolly -S -passes=polly-codegen \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; Check that we generate valid code even if the load of cont_STACKPOINTER is
diff --git a/polly/test/CodeGen/hoisting_1.ll b/polly/test/CodeGen/hoisting_1.ll
index 86b56637bc2c..e04ee68cc4c9 100644
--- a/polly/test/CodeGen/hoisting_1.ll
+++ b/polly/test/CodeGen/hoisting_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -tbaa -polly-codegen -polly-allow-differing-element-types -disable-output %s
+; RUN: opt %loadPolly -aa-pipeline=tbaa -passes=polly-codegen -polly-allow-differing-element-types -disable-output %s
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/hoisting_2.ll b/polly/test/CodeGen/hoisting_2.ll
index 1f1be11c2d98..d5c27f58b95b 100644
--- a/polly/test/CodeGen/hoisting_2.ll
+++ b/polly/test/CodeGen/hoisting_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -tbaa -polly-codegen -polly-allow-differing-element-types -disable-output %s
+; RUN: opt %loadPolly -aa-pipeline=tbaa -passes=polly-codegen -polly-allow-differing-element-types -disable-output %s
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/inner_scev_sdiv_1.ll b/polly/test/CodeGen/inner_scev_sdiv_1.ll
index 1a463fc178d1..25f2abd9b63e 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_1.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s
 ;
 ; Excerpt from the test-suite's oggenc reduced using bugpoint.
 ;
diff --git a/polly/test/CodeGen/inner_scev_sdiv_2.ll b/polly/test/CodeGen/inner_scev_sdiv_2.ll
index 76138034603e..4d80c2a170d6 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_2.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ; The SCEV expression in this test case refers to a sequence of sdiv
 ; instructions, which are part of different bbs in the SCoP. When code
diff --git a/polly/test/CodeGen/inner_scev_sdiv_3.ll b/polly/test/CodeGen/inner_scev_sdiv_3.ll
index 874ead14ded2..8d13c8e168bf 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_3.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ; This test case has a inner SCEV sdiv that will escape the SCoP. Just check we
 ; do not crash and generate valid code.
diff --git a/polly/test/CodeGen/inner_scev_sdiv_in_lb.ll b/polly/test/CodeGen/inner_scev_sdiv_in_lb.ll
index 6514e18687e4..b53ee034f960 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_in_lb.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_in_lb.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s --check-prefix=CODEGEN
 ;
 ; CHECK: [N] -> { Stmt_bb11[i0, i1] : i0 < N and i1 >= 0 and 3i1 <= -3 + i0 };
 ; CODEGEN: polly
diff --git a/polly/test/CodeGen/inner_scev_sdiv_in_lb_invariant.ll b/polly/test/CodeGen/inner_scev_sdiv_in_lb_invariant.ll
index 032942923379..3c392a2c4c4c 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_in_lb_invariant.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_in_lb_invariant.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen \
+; RUN: opt %loadPolly -S -passes=polly-codegen \
 ; RUN:     < %s | FileCheck %s
 ;
 ; Check that this will not crash our code generation.
diff --git a/polly/test/CodeGen/inner_scev_sdiv_in_rtc.ll b/polly/test/CodeGen/inner_scev_sdiv_in_rtc.ll
index f7292ca3073a..0748a274bb7d 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_in_rtc.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_in_rtc.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen \
+; RUN: opt %loadPolly -passes=polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 ;
 ; This will just check that we generate valid code here.
diff --git a/polly/test/CodeGen/intrinsics_lifetime.ll b/polly/test/CodeGen/intrinsics_lifetime.ll
index 6141b3abdd8a..5782b4724649 100644
--- a/polly/test/CodeGen/intrinsics_lifetime.ll
+++ b/polly/test/CodeGen/intrinsics_lifetime.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; Verify that we remove the lifetime markers from everywhere.
 ;
diff --git a/polly/test/CodeGen/intrinsics_misc.ll b/polly/test/CodeGen/intrinsics_misc.ll
index c0a52fe97329..9b208b4e600b 100644
--- a/polly/test/CodeGen/intrinsics_misc.ll
+++ b/polly/test/CodeGen/intrinsics_misc.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; Verify that we remove the misc intrinsics  from the optimized SCoP.
 ;
diff --git a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-2.ll b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-2.ll
index 6727247a7f04..fd66cf0a47c9 100644
--- a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-2.ll
+++ b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S \
+; RUN: opt %loadPolly -passes=polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; This crashed our codegen at some point, verify it runs through
diff --git a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-3.ll b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-3.ll
index a573049c8f67..408f0086a260 100644
--- a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-3.ll
+++ b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S \
+; RUN: opt %loadPolly -passes=polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; This crashed our codegen at some point, verify it runs through
diff --git a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order.ll b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order.ll
index e05ca9951434..45bb5d041a49 100644
--- a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order.ll
+++ b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S \
+; RUN: opt %loadPolly -passes=polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; This crashed our codegen at some point, verify it runs through
diff --git a/polly/test/CodeGen/invariant-load-dimension.ll b/polly/test/CodeGen/invariant-load-dimension.ll
index 7793c3b3bee3..07bd6923b54b 100644
--- a/polly/test/CodeGen/invariant-load-dimension.ll
+++ b/polly/test/CodeGen/invariant-load-dimension.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-invariant-load-hoisting -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=SCOPS
-; RUN: opt %loadPolly -S < %s -polly-codegen -polly-process-unprofitable -polly-invariant-load-hoisting | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-invariant-load-hoisting '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=SCOPS
+; RUN: opt %loadPolly -S < %s -passes=polly-codegen -polly-process-unprofitable -polly-invariant-load-hoisting | FileCheck %s -check-prefix=CODEGEN
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n8:16:32-S64"
 
diff --git a/polly/test/CodeGen/invariant-load-preload-base-pointer-origin-first.ll b/polly/test/CodeGen/invariant-load-preload-base-pointer-origin-first.ll
index 474100995fd8..c4166dd4d2a9 100644
--- a/polly/test/CodeGen/invariant-load-preload-base-pointer-origin-first.ll
+++ b/polly/test/CodeGen/invariant-load-preload-base-pointer-origin-first.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen -polly-invariant-load-hoisting=true < %s
+; RUN: opt %loadPolly -S -passes=polly-codegen -polly-invariant-load-hoisting=true < %s
 ;
 ; Check that we generate valid code as we did non preload the base pointer
 ; origin of %tmp4 at some point.
diff --git a/polly/test/CodeGen/invariant_cannot_handle_void.ll b/polly/test/CodeGen/invariant_cannot_handle_void.ll
index de5d13d6a69a..633955b6053a 100644
--- a/polly/test/CodeGen/invariant_cannot_handle_void.ll
+++ b/polly/test/CodeGen/invariant_cannot_handle_void.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=SCOP
-; RUN: opt %loadPolly -S -polly-codegen -polly-invariant-load-hoisting=true %s | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -S -passes=polly-codegen -polly-invariant-load-hoisting=true %s | FileCheck %s
 ;
 ; The offset of the %tmp1 load wrt. to %buff (62 bytes) is not divisible
 ; by the type size (i32 = 4 bytes), thus we will have to represent %buff
diff --git a/polly/test/CodeGen/invariant_load.ll b/polly/test/CodeGen/invariant_load.ll
index be3f7a32f35b..bef3862e9c79 100644
--- a/polly/test/CodeGen/invariant_load.ll
+++ b/polly/test/CodeGen/invariant_load.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK-NEXT:    %polly.access.B = getelementptr i32, ptr %B, i64 0
diff --git a/polly/test/CodeGen/invariant_load_address_space.ll b/polly/test/CodeGen/invariant_load_address_space.ll
index 7c611ad3dd87..9ffef5757cb8 100644
--- a/polly/test/CodeGen/invariant_load_address_space.ll
+++ b/polly/test/CodeGen/invariant_load_address_space.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK-NEXT:    %polly.access.B = getelementptr i32, ptr addrspace(1) %B, i64 0
diff --git a/polly/test/CodeGen/invariant_load_alias_metadata.ll b/polly/test/CodeGen/invariant_load_alias_metadata.ll
index 5a82d82d43f8..a992a926880b 100644
--- a/polly/test/CodeGen/invariant_load_alias_metadata.ll
+++ b/polly/test/CodeGen/invariant_load_alias_metadata.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true \
 ; RUN: -S < %s | FileCheck %s
 ;
 ; This test case checks whether Polly generates alias metadata in case of
diff --git a/polly/test/CodeGen/invariant_load_base_pointer.ll b/polly/test/CodeGen/invariant_load_base_pointer.ll
index eb07f8317b79..c7d9e8df5b59 100644
--- a/polly/test/CodeGen/invariant_load_base_pointer.ll
+++ b/polly/test/CodeGen/invariant_load_base_pointer.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly  -polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
+; RUN: opt %loadPolly  -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK-NEXT:    %polly.access.BPLoc = getelementptr ptr, ptr %BPLoc, i64 0
diff --git a/polly/test/CodeGen/invariant_load_base_pointer_conditional.ll b/polly/test/CodeGen/invariant_load_base_pointer_conditional.ll
index 538077bb09e8..f24d8b7d7525 100644
--- a/polly/test/CodeGen/invariant_load_base_pointer_conditional.ll
+++ b/polly/test/CodeGen/invariant_load_base_pointer_conditional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly  -polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
+; RUN: opt %loadPolly  -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK-NEXT:    %0 = sext i32 %N to i64
diff --git a/polly/test/CodeGen/invariant_load_base_pointer_conditional_2.ll b/polly/test/CodeGen/invariant_load_base_pointer_conditional_2.ll
index 7c2fb3ef97ed..210060c90681 100644
--- a/polly/test/CodeGen/invariant_load_base_pointer_conditional_2.ll
+++ b/polly/test/CodeGen/invariant_load_base_pointer_conditional_2.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -S -polly-codegen -polly-invariant-load-hoisting=true < %s | FileCheck %s --check-prefix=IR
-; RUN: opt %loadPolly -S -polly-codegen -polly-invariant-load-hoisting=true --polly-overflow-tracking=always < %s | FileCheck %s --check-prefix=IRA
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen -polly-invariant-load-hoisting=true < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -S -passes=polly-codegen -polly-invariant-load-hoisting=true --polly-overflow-tracking=always < %s | FileCheck %s --check-prefix=IRA
 ;
 ; As (p + q) can overflow we have to check that we load from
 ; I[p + q] only if it does not.
diff --git a/polly/test/CodeGen/invariant_load_canonicalize_array_baseptrs.ll b/polly/test/CodeGen/invariant_load_canonicalize_array_baseptrs.ll
index dc5a4c890381..f6d4fef13d7a 100644
--- a/polly/test/CodeGen/invariant_load_canonicalize_array_baseptrs.ll
+++ b/polly/test/CodeGen/invariant_load_canonicalize_array_baseptrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s \
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/CodeGen/invariant_load_condition.ll b/polly/test/CodeGen/invariant_load_condition.ll
index edf0814d8983..1aab3b8497fc 100644
--- a/polly/test/CodeGen/invariant_load_condition.ll
+++ b/polly/test/CodeGen/invariant_load_condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK-NEXT:     %polly.access.C = getelementptr i32, ptr %C, i64 0
diff --git a/polly/test/CodeGen/invariant_load_different_sized_types.ll b/polly/test/CodeGen/invariant_load_different_sized_types.ll
index 5b91a1901061..952786d71cc9 100644
--- a/polly/test/CodeGen/invariant_load_different_sized_types.ll
+++ b/polly/test/CodeGen/invariant_load_different_sized_types.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S \
 ; RUN: -polly-allow-differing-element-types < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/CodeGen/invariant_load_escaping.ll b/polly/test/CodeGen/invariant_load_escaping.ll
index efccdf468a18..31d2066745ef 100644
--- a/polly/test/CodeGen/invariant_load_escaping.ll
+++ b/polly/test/CodeGen/invariant_load_escaping.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ;    int f(int *A, int *B) {
 ;      // Possible aliasing between A and B but if not then *B would be
diff --git a/polly/test/CodeGen/invariant_load_escaping_second_scop.ll b/polly/test/CodeGen/invariant_load_escaping_second_scop.ll
index c0ea888acdde..5dbf261d84e5 100644
--- a/polly/test/CodeGen/invariant_load_escaping_second_scop.ll
+++ b/polly/test/CodeGen/invariant_load_escaping_second_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true  -polly-process-unprofitable -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true  -polly-process-unprofitable -S < %s | FileCheck %s
 ;
 ;    void fence(void);
 ;
diff --git a/polly/test/CodeGen/invariant_load_in_non_affine_subregion.ll b/polly/test/CodeGen/invariant_load_in_non_affine_subregion.ll
index 241252b5d549..57b4a655cc24 100644
--- a/polly/test/CodeGen/invariant_load_in_non_affine_subregion.ll
+++ b/polly/test/CodeGen/invariant_load_in_non_affine_subregion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; This crashed at some point as the invariant load is in a non-affine
 ; subregion. Just check it does not anymore.
diff --git a/polly/test/CodeGen/invariant_load_loop_ub.ll b/polly/test/CodeGen/invariant_load_loop_ub.ll
index ab9aa0dc69a7..a4b96fad0559 100644
--- a/polly/test/CodeGen/invariant_load_loop_ub.ll
+++ b/polly/test/CodeGen/invariant_load_loop_ub.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -polly-process-unprofitable -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-process-unprofitable -S < %s | FileCheck %s
 ;
 ; CHECK: polly.start
 ;
diff --git a/polly/test/CodeGen/invariant_load_not_executed_but_in_parameters.ll b/polly/test/CodeGen/invariant_load_not_executed_but_in_parameters.ll
index 08ff0871b610..2970dbb5bed9 100644
--- a/polly/test/CodeGen/invariant_load_not_executed_but_in_parameters.ll
+++ b/polly/test/CodeGen/invariant_load_not_executed_but_in_parameters.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
 ;
 ; Check that this does not crash as the invariant load is not executed (thus
 ; not preloaded) but still referenced by one of the parameters.
diff --git a/polly/test/CodeGen/invariant_load_outermost.ll b/polly/test/CodeGen/invariant_load_outermost.ll
index f42135c09014..eda42f76ed5d 100644
--- a/polly/test/CodeGen/invariant_load_outermost.ll
+++ b/polly/test/CodeGen/invariant_load_outermost.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 
 ; CHECK: polly.start
 
diff --git a/polly/test/CodeGen/invariant_load_parameters_cyclic_dependence.ll b/polly/test/CodeGen/invariant_load_parameters_cyclic_dependence.ll
index d365c99eff66..73fb038baed2 100644
--- a/polly/test/CodeGen/invariant_load_parameters_cyclic_dependence.ll
+++ b/polly/test/CodeGen/invariant_load_parameters_cyclic_dependence.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s --check-prefix=SCOP
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; SCOP:         Assumed Context:
 ; SCOP-NEXT:    [p_0, tmp4] -> {  :  }
diff --git a/polly/test/CodeGen/invariant_load_ptr_ptr_noalias.ll b/polly/test/CodeGen/invariant_load_ptr_ptr_noalias.ll
index b4d4c55f0d9b..82e957b9ef1d 100644
--- a/polly/test/CodeGen/invariant_load_ptr_ptr_noalias.ll
+++ b/polly/test/CodeGen/invariant_load_ptr_ptr_noalias.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -S  < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -S  < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK:   %polly.access.A = getelementptr ptr, ptr %A, i64 42
diff --git a/polly/test/CodeGen/invariant_load_scalar_dep.ll b/polly/test/CodeGen/invariant_load_scalar_dep.ll
index 05a40a4c47cc..3906d5ab890f 100644
--- a/polly/test/CodeGen/invariant_load_scalar_dep.ll
+++ b/polly/test/CodeGen/invariant_load_scalar_dep.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly  -polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
+; RUN: opt %loadPolly  -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK:    %polly.access.B = getelementptr i32, ptr %B, i64 0
diff --git a/polly/test/CodeGen/invariant_load_scalar_escape_alloca_sharing.ll b/polly/test/CodeGen/invariant_load_scalar_escape_alloca_sharing.ll
index 44c035855b76..82e38e86bf77 100644
--- a/polly/test/CodeGen/invariant_load_scalar_escape_alloca_sharing.ll
+++ b/polly/test/CodeGen/invariant_load_scalar_escape_alloca_sharing.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; Verify the preloaded %tmp0 is stored and communicated in the same alloca.
 ; In this case, we do not reload %ncol.load from the scalar stack slot, but
diff --git a/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_1.ll b/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_1.ll
index 0b6929a5fd3f..ef382eb20d0f 100644
--- a/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_1.ll
+++ b/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true < %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true < %s
 ;
 ; Check we do not crash even though we pre-load values with different types
 ; from the same base pointer.
diff --git a/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_2.ll b/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_2.ll
index 2eb913fed447..07e7b97ed9de 100644
--- a/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_2.ll
+++ b/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true < %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true < %s
 ;
 ; Check we do not crash even though we pre-load values with different types
 ; from the same base pointer.
diff --git a/polly/test/CodeGen/invariant_loads_ignore_parameter_bounds.ll b/polly/test/CodeGen/invariant_loads_ignore_parameter_bounds.ll
index a0c1f891bdf6..8be087467e99 100644
--- a/polly/test/CodeGen/invariant_loads_ignore_parameter_bounds.ll
+++ b/polly/test/CodeGen/invariant_loads_ignore_parameter_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting \
 ; RUN:     -polly-ignore-parameter-bounds -S < %s | FileCheck %s
 
 ; CHECK: polly.preload.begin:
diff --git a/polly/test/CodeGen/invariant_verify_function_failed.ll b/polly/test/CodeGen/invariant_verify_function_failed.ll
index 6020caeee85d..86308a7fbbfc 100644
--- a/polly/test/CodeGen/invariant_verify_function_failed.ll
+++ b/polly/test/CodeGen/invariant_verify_function_failed.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,scop(polly-codegen)' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; This crashed at some point as the pointer returned by the call
 ; to @__errno_location is invariant and defined in the SCoP but not
diff --git a/polly/test/CodeGen/invariant_verify_function_failed_2.ll b/polly/test/CodeGen/invariant_verify_function_failed_2.ll
index 81a4bd1dc153..97faa6155f18 100644
--- a/polly/test/CodeGen/invariant_verify_function_failed_2.ll
+++ b/polly/test/CodeGen/invariant_verify_function_failed_2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -S -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s -check-prefix=SCOPS
-; RUN: opt %loadPolly -S -polly-codegen -polly-invariant-load-hoisting=true %s | FileCheck %s
+; RUN: opt %loadPolly -S '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s -check-prefix=SCOPS
+; RUN: opt %loadPolly -S -passes=polly-codegen -polly-invariant-load-hoisting=true %s | FileCheck %s
 ;
 ; Check we generate valid code.
 
diff --git a/polly/test/CodeGen/issue56692.ll b/polly/test/CodeGen/issue56692.ll
index e935e43bfa44..b5ab63c3b72b 100644
--- a/polly/test/CodeGen/issue56692.ll
+++ b/polly/test/CodeGen/issue56692.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-omp-backend=LLVM -polly-codegen-verify -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-omp-backend=LLVM -polly-codegen-verify -passes=polly-codegen -S < %s | FileCheck %s
 ; https://github.com/llvm/llvm-project/issues/56692
 ;
 ; CHECK: call void (ptr, i32, ptr, ...) @__kmpc_fork_call({{.*}}), !dbg ![[OPTLOC:[0-9]+]]
diff --git a/polly/test/CodeGen/large-numbers-in-boundary-context.ll b/polly/test/CodeGen/large-numbers-in-boundary-context.ll
index a0328dfec651..519511f1ddd5 100644
--- a/polly/test/CodeGen/large-numbers-in-boundary-context.ll
+++ b/polly/test/CodeGen/large-numbers-in-boundary-context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ; XFAIL: *
 ;
 ; The boundary context contains a constant that does not fit in 64 bits. Hence,
diff --git a/polly/test/CodeGen/load_subset_with_context.ll b/polly/test/CodeGen/load_subset_with_context.ll
index ef0e051d5635..980a06f23c73 100644
--- a/polly/test/CodeGen/load_subset_with_context.ll
+++ b/polly/test/CodeGen/load_subset_with_context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
 ;
 ; A load must provide a value for every statement instance.
 ; Statement instances not in the SCoP's context are irrelevant.
diff --git a/polly/test/CodeGen/loop-invariant-load-type-mismatch.ll b/polly/test/CodeGen/loop-invariant-load-type-mismatch.ll
index 90c61c591623..ca3463d72af3 100644
--- a/polly/test/CodeGen/loop-invariant-load-type-mismatch.ll
+++ b/polly/test/CodeGen/loop-invariant-load-type-mismatch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/polly/test/CodeGen/loop_with_condition.ll b/polly/test/CodeGen/loop_with_condition.ll
index 618a542c179a..436d37d1261a 100644
--- a/polly/test/CodeGen/loop_with_condition.ll
+++ b/polly/test/CodeGen/loop_with_condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ;#include <string.h>
 ;#define N 1024
diff --git a/polly/test/CodeGen/loop_with_condition_2.ll b/polly/test/CodeGen/loop_with_condition_2.ll
index b1a116785069..47ec693efbd5 100644
--- a/polly/test/CodeGen/loop_with_condition_2.ll
+++ b/polly/test/CodeGen/loop_with_condition_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 
 ; Verify that we actually detect this loop as the innermost loop even though
 ; there is a conditional inside.
diff --git a/polly/test/CodeGen/loop_with_condition_ineq.ll b/polly/test/CodeGen/loop_with_condition_ineq.ll
index c35208c72dfe..98866000b0cb 100644
--- a/polly/test/CodeGen/loop_with_condition_ineq.ll
+++ b/polly/test/CodeGen/loop_with_condition_ineq.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ;#include <string.h>
 ;#define N 1024
diff --git a/polly/test/CodeGen/loop_with_condition_nested.ll b/polly/test/CodeGen/loop_with_condition_nested.ll
index 24a49b47d9e6..d9a9dafd4b7e 100644
--- a/polly/test/CodeGen/loop_with_condition_nested.ll
+++ b/polly/test/CodeGen/loop_with_condition_nested.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -basic-aa -polly-codegen < %s | opt -passes='print<loops>' -disable-output 2>&1 | FileCheck %s -check-prefix=LOOPS
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen < %s | opt -passes='print<loops>' -disable-output 2>&1 | FileCheck %s -check-prefix=LOOPS
 
 
 ;#include <string.h>
diff --git a/polly/test/CodeGen/loop_with_conditional_entry_edge_split_hard_case.ll b/polly/test/CodeGen/loop_with_conditional_entry_edge_split_hard_case.ll
index 4444cf1dc4dd..3687868b0baf 100644
--- a/polly/test/CodeGen/loop_with_conditional_entry_edge_split_hard_case.ll
+++ b/polly/test/CodeGen/loop_with_conditional_entry_edge_split_hard_case.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; Test case to trigger the hard way of creating a unique entering
 ; edge for the SCoP. It is triggered because the entering edge
diff --git a/polly/test/CodeGen/memcpy_annotations.ll b/polly/test/CodeGen/memcpy_annotations.ll
index a0a09b75c82e..42fe5ca92b94 100644
--- a/polly/test/CodeGen/memcpy_annotations.ll
+++ b/polly/test/CodeGen/memcpy_annotations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; Verify that @llvm.memcpy does not get a !alias.scope annotation.
 ; @llvm.memcpy takes two pointers, it is ambiguous to which the
diff --git a/polly/test/CodeGen/multidim-non-matching-typesize-2.ll b/polly/test/CodeGen/multidim-non-matching-typesize-2.ll
index 63afad6e2f41..cfd52a0b7793 100644
--- a/polly/test/CodeGen/multidim-non-matching-typesize-2.ll
+++ b/polly/test/CodeGen/multidim-non-matching-typesize-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -disable-basic-aa -polly-codegen \
+; RUN: opt %loadPolly -disable-basic-aa -passes=polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 ; CHECK: polly
 target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
diff --git a/polly/test/CodeGen/multidim-non-matching-typesize.ll b/polly/test/CodeGen/multidim-non-matching-typesize.ll
index d117cefe3376..b3f70226f743 100644
--- a/polly/test/CodeGen/multidim-non-matching-typesize.ll
+++ b/polly/test/CodeGen/multidim-non-matching-typesize.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -disable-basic-aa -polly-codegen \
+; RUN: opt %loadPolly -disable-basic-aa -passes=polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
diff --git a/polly/test/CodeGen/multidim_2d_parametric_array_static_loop_bounds.ll b/polly/test/CodeGen/multidim_2d_parametric_array_static_loop_bounds.ll
index 464ddb3740f7..11976874ed84 100644
--- a/polly/test/CodeGen/multidim_2d_parametric_array_static_loop_bounds.ll
+++ b/polly/test/CodeGen/multidim_2d_parametric_array_static_loop_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/CodeGen/multidim_alias_check.ll b/polly/test/CodeGen/multidim_alias_check.ll
index 585577da0e6d..15390433a1b1 100644
--- a/polly/test/CodeGen/multidim_alias_check.ll
+++ b/polly/test/CodeGen/multidim_alias_check.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; CHECK: %polly.access.sext.A = sext i32 %n to i64
diff --git a/polly/test/CodeGen/multiple-codegens.ll b/polly/test/CodeGen/multiple-codegens.ll
index f950fa4a3e1d..683ccdfa092b 100644
--- a/polly/test/CodeGen/multiple-codegens.ll
+++ b/polly/test/CodeGen/multiple-codegens.ll
@@ -1,6 +1,5 @@
-; RUN: opt %loadPolly -polly-scops -polly-opt-isl -polly-codegen -polly-scops -polly-codegen -S < %s | FileCheck %s
-; RUN: opt %loadNPMPolly "-passes=scop(polly-opt-isl,polly-codegen,polly-codegen)" -S < %s | FileCheck %s
-; RUN: opt %loadNPMPolly "-passes=scop(polly-opt-isl,polly-codegen),scop(polly-codegen)" -S < %s | FileCheck %s
+; RUN: opt %loadPolly "-passes=scop(polly-opt-isl,polly-codegen,polly-codegen)" -S < %s | FileCheck %s
+; RUN: opt %loadPolly "-passes=scop(polly-opt-isl,polly-codegen),scop(polly-codegen)" -S < %s | FileCheck %s
 ;
 ; llvm.org/PR34441
 ; Properly handle multiple -polly-scops/-polly-codegen in the same
diff --git a/polly/test/CodeGen/multiple-scops-in-a-row.ll b/polly/test/CodeGen/multiple-scops-in-a-row.ll
index a24a2e71ad4e..0ac158c79129 100644
--- a/polly/test/CodeGen/multiple-scops-in-a-row.ll
+++ b/polly/test/CodeGen/multiple-scops-in-a-row.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 
 ; This test case has two scops in a row. When code generating the first scop,
 ; the second scop is invalidated. This test case verifies that we do not crash
diff --git a/polly/test/CodeGen/multiple-types-invariant-load-2.ll b/polly/test/CodeGen/multiple-types-invariant-load-2.ll
index 0fd1df75e2ec..7916d32bd7de 100644
--- a/polly/test/CodeGen/multiple-types-invariant-load-2.ll
+++ b/polly/test/CodeGen/multiple-types-invariant-load-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S \
+; RUN: opt %loadPolly -passes=polly-codegen -S \
 ; RUN: -polly-allow-differing-element-types < %s | FileCheck %s
 
 ; CHECK: polly
diff --git a/polly/test/CodeGen/multiple-types-invariant-load.ll b/polly/test/CodeGen/multiple-types-invariant-load.ll
index b1434679e3d1..5ce698d4fb4d 100644
--- a/polly/test/CodeGen/multiple-types-invariant-load.ll
+++ b/polly/test/CodeGen/multiple-types-invariant-load.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-differing-element-types -polly-codegen -S \
+; RUN: opt %loadPolly -polly-allow-differing-element-types -passes=polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ; CHECK: %polly.access.global.load = getelementptr i32, ptr %global.load, i64 0
diff --git a/polly/test/CodeGen/multiple_sai_fro_same_base_address.ll b/polly/test/CodeGen/multiple_sai_fro_same_base_address.ll
index 0163f248229e..94c208e6692f 100644
--- a/polly/test/CodeGen/multiple_sai_fro_same_base_address.ll
+++ b/polly/test/CodeGen/multiple_sai_fro_same_base_address.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-position=before-vectorizer -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=SCOP
-; RUN: opt %loadPolly -polly-position=before-vectorizer -polly-codegen -S < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -polly-position=before-vectorizer '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -polly-position=before-vectorizer -passes=polly-codegen -S < %s | FileCheck %s --check-prefix=IR
 
 ; The IR has two ScopArrayInfo for the value %next.0. This used to produce two
 ; phi nodes in polly.merge_new_and_old, one illegaly using the result of the
diff --git a/polly/test/CodeGen/no-overflow-tracking.ll b/polly/test/CodeGen/no-overflow-tracking.ll
index f11e8927ddee..ff4a8023dead 100644
--- a/polly/test/CodeGen/no-overflow-tracking.ll
+++ b/polly/test/CodeGen/no-overflow-tracking.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-overflow-tracking=never -polly-codegen -S < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-overflow-tracking=never -passes=polly-codegen -S < %s | FileCheck %s --check-prefix=IR
 ;
 ; As (p + q) can overflow we have to check that we load from
 ; I[p + q] only if it does not.
diff --git a/polly/test/CodeGen/no_guard_bb.ll b/polly/test/CodeGen/no_guard_bb.ll
index 47c87ff7c868..6635048e0f9b 100644
--- a/polly/test/CodeGen/no_guard_bb.ll
+++ b/polly/test/CodeGen/no_guard_bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S -verify-dom-info < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s
 ;
 ; CHECK-NOT: br i1 true, label %polly.{{.*}}, label %polly.{{.*}}
 ;
diff --git a/polly/test/CodeGen/non-affine-dominance-generated-entering.ll b/polly/test/CodeGen/non-affine-dominance-generated-entering.ll
index ebf36acc8d96..d1d2fc644010 100644
--- a/polly/test/CodeGen/non-affine-dominance-generated-entering.ll
+++ b/polly/test/CodeGen/non-affine-dominance-generated-entering.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; llvm.org/PR25439
 ; Scalar reloads in the generated entering block were not recognized as
diff --git a/polly/test/CodeGen/non-affine-exit-node-dominance.ll b/polly/test/CodeGen/non-affine-exit-node-dominance.ll
index af19d2420e3e..8039f3b08543 100644
--- a/polly/test/CodeGen/non-affine-exit-node-dominance.ll
+++ b/polly/test/CodeGen/non-affine-exit-node-dominance.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; llvm.org/PR25439
 ; The dominance of the generated non-affine subregion block was based on the
diff --git a/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll b/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll
index 2aca316d4c88..5b6c2ecc81c8 100644
--- a/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll
+++ b/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen \
+; RUN: opt %loadPolly -passes=polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/non-affine-phi-node-expansion-3.ll b/polly/test/CodeGen/non-affine-phi-node-expansion-3.ll
index 18a4b6e4ed4a..9a4e1fa26a9f 100644
--- a/polly/test/CodeGen/non-affine-phi-node-expansion-3.ll
+++ b/polly/test/CodeGen/non-affine-phi-node-expansion-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen \
+; RUN: opt %loadPolly -passes=polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 
 define void @foo(ptr %A, i1 %cond0, i1 %cond1) {
diff --git a/polly/test/CodeGen/non-affine-phi-node-expansion-4.ll b/polly/test/CodeGen/non-affine-phi-node-expansion-4.ll
index 8a07ee7c7424..b5380b5d225c 100644
--- a/polly/test/CodeGen/non-affine-phi-node-expansion-4.ll
+++ b/polly/test/CodeGen/non-affine-phi-node-expansion-4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen \
+; RUN: opt %loadPolly -passes=polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 
 define void @foo(ptr %A, i1 %cond0, i1 %cond1) {
diff --git a/polly/test/CodeGen/non-affine-phi-node-expansion.ll b/polly/test/CodeGen/non-affine-phi-node-expansion.ll
index 091fc3e323dc..7c29868675f4 100644
--- a/polly/test/CodeGen/non-affine-phi-node-expansion.ll
+++ b/polly/test/CodeGen/non-affine-phi-node-expansion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen \
+; RUN: opt %loadPolly -passes=polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize-2.ll b/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize-2.ll
index 6a1d1f12ba9c..f5c446a22c02 100644
--- a/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize-2.ll
+++ b/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused the code generation to generate invalid code as the same operand
 ; of the PHI node in the non-affine region was synthesized at the wrong place.
diff --git a/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize.ll b/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize.ll
index 036bf34cb7f7..fc894775b755 100644
--- a/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize.ll
+++ b/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused the code generation to generate invalid code as the same BBMap was
 ; used for the whole non-affine region. When %add is synthesized for the
diff --git a/polly/test/CodeGen/non-affine-region-implicit-store.ll b/polly/test/CodeGen/non-affine-region-implicit-store.ll
index e89197e24852..6f2c7005d1bb 100644
--- a/polly/test/CodeGen/non-affine-region-implicit-store.ll
+++ b/polly/test/CodeGen/non-affine-region-implicit-store.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; llvm.org/PR25438
 ; After loop versioning, a dominance check of a non-affine subregion's exit node
diff --git a/polly/test/CodeGen/non-affine-region-phi-references-in-scop-value.ll b/polly/test/CodeGen/non-affine-region-phi-references-in-scop-value.ll
index f6e4eb57319d..6151562555ee 100644
--- a/polly/test/CodeGen/non-affine-region-phi-references-in-scop-value.ll
+++ b/polly/test/CodeGen/non-affine-region-phi-references-in-scop-value.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-allow-nonaffine-loops \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-allow-nonaffine-loops \
 ; RUN: -S < %s | FileCheck %s
 
 ; This test verifies that values defined in another scop statement and used by
diff --git a/polly/test/CodeGen/non-affine-subregion-dominance-reuse.ll b/polly/test/CodeGen/non-affine-subregion-dominance-reuse.ll
index 6c749a404336..c2af74c215f9 100644
--- a/polly/test/CodeGen/non-affine-subregion-dominance-reuse.ll
+++ b/polly/test/CodeGen/non-affine-subregion-dominance-reuse.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S -verify-dom-info \
+; RUN: opt %loadPolly -passes=polly-codegen -S -verify-dom-info \
 ; RUN:     < %s | FileCheck %s
 ;
 ; Check that we do not reuse the B[i-1] GEP created in block S again in
diff --git a/polly/test/CodeGen/non-affine-switch.ll b/polly/test/CodeGen/non-affine-switch.ll
index 9c08b98700ae..c829b682fb24 100644
--- a/polly/test/CodeGen/non-affine-switch.ll
+++ b/polly/test/CodeGen/non-affine-switch.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly \
-; RUN: -S -polly-codegen < %s | FileCheck %s
+; RUN: -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/CodeGen/non-affine-synthesized-in-branch.ll b/polly/test/CodeGen/non-affine-synthesized-in-branch.ll
index cc0e60abcd09..a0febb348434 100644
--- a/polly/test/CodeGen/non-affine-synthesized-in-branch.ll
+++ b/polly/test/CodeGen/non-affine-synthesized-in-branch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; llvm.org/PR25412
 ; %synthgep caused %gep to be synthesized in subregion_if which was reused for
diff --git a/polly/test/CodeGen/non-affine-update.ll b/polly/test/CodeGen/non-affine-update.ll
index d2b7fae75b23..aacbb9e766c4 100644
--- a/polly/test/CodeGen/non-affine-update.ll
+++ b/polly/test/CodeGen/non-affine-update.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-import-jscop \
-; RUN:     -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
+; RUN:     -S < %s | FileCheck %s
 ;
 ;    void non-affine-update(double A[], double C[], double B[]) {
 ;      for (int i = 0; i < 10; i++) {
diff --git a/polly/test/CodeGen/non-hoisted-load-needed-as-base-ptr.ll b/polly/test/CodeGen/non-hoisted-load-needed-as-base-ptr.ll
index 5f6642b0630d..0c7c5f9a1700 100644
--- a/polly/test/CodeGen/non-hoisted-load-needed-as-base-ptr.ll
+++ b/polly/test/CodeGen/non-hoisted-load-needed-as-base-ptr.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -tbaa -polly-codegen -disable-output %s
+; RUN: opt %loadPolly -aa-pipeline=tbaa -passes=polly-codegen -disable-output %s
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/non_affine_float_compare.ll b/polly/test/CodeGen/non_affine_float_compare.ll
index be310b5bf5ca..0b4813ac11ae 100644
--- a/polly/test/CodeGen/non_affine_float_compare.ll
+++ b/polly/test/CodeGen/non_affine_float_compare.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen \
+; RUN: opt %loadPolly -passes=polly-codegen \
 ; RUN:     -polly-allow-nonaffine-branches -S -verify-dom-info \
 ; RUN:     < %s | FileCheck %s
 ;
diff --git a/polly/test/CodeGen/only_non_affine_error_region.ll b/polly/test/CodeGen/only_non_affine_error_region.ll
index b2ad1c1fe3fd..472aec927dcd 100644
--- a/polly/test/CodeGen/only_non_affine_error_region.ll
+++ b/polly/test/CodeGen/only_non_affine_error_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; CHECK-NOT: polly.start
 ;
diff --git a/polly/test/CodeGen/openmp_limit_threads.ll b/polly/test/CodeGen/openmp_limit_threads.ll
index e8eb819f13d9..70f78ebac173 100644
--- a/polly/test/CodeGen/openmp_limit_threads.ll
+++ b/polly/test/CodeGen/openmp_limit_threads.ll
@@ -1,10 +1,10 @@
-; RUN: opt %loadPolly -polly-codegen -polly-parallel -S < %s | FileCheck %s --check-prefix=AUTO
-; RUN: opt %loadPolly -polly-codegen -polly-parallel -polly-num-threads=1 -S < %s | FileCheck %s --check-prefix=ONE
-; RUN: opt %loadPolly -polly-codegen -polly-parallel -polly-num-threads=4 -S < %s | FileCheck %s --check-prefix=FOUR
+; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -S < %s | FileCheck %s --check-prefix=AUTO
+; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -polly-num-threads=1 -S < %s | FileCheck %s --check-prefix=ONE
+; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -polly-num-threads=4 -S < %s | FileCheck %s --check-prefix=FOUR
 
-; RUN: opt %loadPolly -polly-codegen -polly-parallel -polly-omp-backend=LLVM -S < %s | FileCheck %s --check-prefix=LIBOMP-AUTO
-; RUN: opt %loadPolly -polly-codegen -polly-parallel -polly-omp-backend=LLVM -polly-num-threads=1 -S < %s | FileCheck %s --check-prefix=LIBOMP-ONE
-; RUN: opt %loadPolly -polly-codegen -polly-parallel -polly-omp-backend=LLVM -polly-num-threads=4 -S < %s | FileCheck %s --check-prefix=LIBOMP-FOUR
+; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -polly-omp-backend=LLVM -S < %s | FileCheck %s --check-prefix=LIBOMP-AUTO
+; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -polly-omp-backend=LLVM -polly-num-threads=1 -S < %s | FileCheck %s --check-prefix=LIBOMP-ONE
+; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -polly-omp-backend=LLVM -polly-num-threads=4 -S < %s | FileCheck %s --check-prefix=LIBOMP-FOUR
 
 ; Ensure that the provided thread numbers are forwarded to the OpenMP calls.
 ;
diff --git a/polly/test/CodeGen/out-of-scop-phi-node-use.ll b/polly/test/CodeGen/out-of-scop-phi-node-use.ll
index 54e909ecf378..9ef0586d7077 100644
--- a/polly/test/CodeGen/out-of-scop-phi-node-use.ll
+++ b/polly/test/CodeGen/out-of-scop-phi-node-use.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/polly/test/CodeGen/param_div_div_div_2.ll b/polly/test/CodeGen/param_div_div_div_2.ll
index 764ca241f166..027c147b173e 100644
--- a/polly/test/CodeGen/param_div_div_div_2.ll
+++ b/polly/test/CodeGen/param_div_div_div_2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s --check-prefix=IR
 ;
 ; Check that we guard the divisions because we moved them and thereby increased
 ; their domain.
diff --git a/polly/test/CodeGen/partial_write_array.ll b/polly/test/CodeGen/partial_write_array.ll
index 6dc5550d82af..82277d631e4c 100644
--- a/polly/test/CodeGen/partial_write_array.ll
+++ b/polly/test/CodeGen/partial_write_array.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
 ;
 ; Partial write of an array access.
 ;
diff --git a/polly/test/CodeGen/partial_write_emptyset.ll b/polly/test/CodeGen/partial_write_emptyset.ll
index a25195f11ed7..687599025e09 100644
--- a/polly/test/CodeGen/partial_write_emptyset.ll
+++ b/polly/test/CodeGen/partial_write_emptyset.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
 ;
 ; Partial write, where "partial" is the empty set.
 ; The store is never executed in this case and we do generate it in the
diff --git a/polly/test/CodeGen/partial_write_full_write_that_appears_partial.ll b/polly/test/CodeGen/partial_write_full_write_that_appears_partial.ll
index 18a809b30557..261d6fe7a0c5 100644
--- a/polly/test/CodeGen/partial_write_full_write_that_appears_partial.ll
+++ b/polly/test/CodeGen/partial_write_full_write_that_appears_partial.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 
 ; CHECK:      polly.stmt.if.then81:                             ; preds = %polly.stmt.if.end75
 ; CHECK-NEXT:   store float undef, ptr %fX64, align 4, !alias.scope !0, !noalias !3
diff --git a/polly/test/CodeGen/partial_write_impossible_restriction.ll b/polly/test/CodeGen/partial_write_impossible_restriction.ll
index 178227fef8e5..d041edb9262e 100644
--- a/polly/test/CodeGen/partial_write_impossible_restriction.ll
+++ b/polly/test/CodeGen/partial_write_impossible_restriction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
 ;
 ; The isl scheduler isolates %cond.false into two instances.
 ; A partial write access in one of the instances was never executed,
diff --git a/polly/test/CodeGen/partial_write_in_region.ll b/polly/test/CodeGen/partial_write_in_region.ll
index d8f57b35d585..be5025778096 100644
--- a/polly/test/CodeGen/partial_write_in_region.ll
+++ b/polly/test/CodeGen/partial_write_in_region.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-import-jscop \
-; RUN: -polly-import-jscop-postfix=transformed -polly-codegen \
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
+; RUN: -polly-import-jscop-postfix=transformed \
 ; RUN: -verify-dom-info \
 ; RUN: -S < %s | FileCheck %s
 ;
diff --git a/polly/test/CodeGen/partial_write_in_region_with_loop.ll b/polly/test/CodeGen/partial_write_in_region_with_loop.ll
index 48a9dbef21d1..8379a5f279ed 100644
--- a/polly/test/CodeGen/partial_write_in_region_with_loop.ll
+++ b/polly/test/CodeGen/partial_write_in_region_with_loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-import-jscop \
-; RUN: -polly-import-jscop-postfix=transformed -polly-codegen \
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
+; RUN: -polly-import-jscop-postfix=transformed \
 ; RUN: -verify-dom-info -polly-allow-nonaffine-loops \
 ; RUN: -S < %s | FileCheck %s
 
diff --git a/polly/test/CodeGen/partial_write_mapped_scalar.ll b/polly/test/CodeGen/partial_write_mapped_scalar.ll
index 9137ef2123c8..b74705250be5 100644
--- a/polly/test/CodeGen/partial_write_mapped_scalar.ll
+++ b/polly/test/CodeGen/partial_write_mapped_scalar.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
 ;
 ; Partial write of a (mapped) scalar.
 ;
diff --git a/polly/test/CodeGen/partial_write_mapped_scalar_subregion.ll b/polly/test/CodeGen/partial_write_mapped_scalar_subregion.ll
index e054b65eadf3..80a97e1aa657 100644
--- a/polly/test/CodeGen/partial_write_mapped_scalar_subregion.ll
+++ b/polly/test/CodeGen/partial_write_mapped_scalar_subregion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
 ;
 ; Partial write of a (mapped) scalar in a non-affine subregion.
 ;
diff --git a/polly/test/CodeGen/perf_monitoring.ll b/polly/test/CodeGen/perf_monitoring.ll
index 2abbf24f5e78..dde7853b37fd 100644
--- a/polly/test/CodeGen/perf_monitoring.ll
+++ b/polly/test/CodeGen/perf_monitoring.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-codegen-perf-monitoring \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-codegen-perf-monitoring \
 ; RUN:   -S < %s | FileCheck %s
 
 ; void f(long A[], long N) {
diff --git a/polly/test/CodeGen/perf_monitoring_cycles_per_scop.ll b/polly/test/CodeGen/perf_monitoring_cycles_per_scop.ll
index 11d63fc47658..d9efed5f8efa 100644
--- a/polly/test/CodeGen/perf_monitoring_cycles_per_scop.ll
+++ b/polly/test/CodeGen/perf_monitoring_cycles_per_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-codegen-perf-monitoring \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-codegen-perf-monitoring \
 ; RUN:   -S < %s | FileCheck %s
 
 ; void f(long A[], long N) {
diff --git a/polly/test/CodeGen/perf_monitoring_trip_counts_per_scop.ll b/polly/test/CodeGen/perf_monitoring_trip_counts_per_scop.ll
index 9b7f324df8e4..560b44561bf5 100644
--- a/polly/test/CodeGen/perf_monitoring_trip_counts_per_scop.ll
+++ b/polly/test/CodeGen/perf_monitoring_trip_counts_per_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-codegen-perf-monitoring \
+; RUN: opt %loadPolly -passes=polly-codegen -polly-codegen-perf-monitoring \
 ; RUN:   -S < %s | FileCheck %s
 
 ; void f(long A[], long N) {
diff --git a/polly/test/CodeGen/phi-defined-before-scop.ll b/polly/test/CodeGen/phi-defined-before-scop.ll
index a3b1ba264f04..1e6266673a8c 100644
--- a/polly/test/CodeGen/phi-defined-before-scop.ll
+++ b/polly/test/CodeGen/phi-defined-before-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 
 ; CHECK-LABEL: polly.merge_new_and_old:
 ; CHECK-NEXT: %tmp7.ph.merge = phi ptr [ %tmp7.ph.final_reload, %polly.exiting ], [ %tmp7.ph, %bb6.region_exiting ]
diff --git a/polly/test/CodeGen/phi_after_error_block_outside_of_scop.ll b/polly/test/CodeGen/phi_after_error_block_outside_of_scop.ll
index c34ebfc3ca02..0121888f9fa9 100644
--- a/polly/test/CodeGen/phi_after_error_block_outside_of_scop.ll
+++ b/polly/test/CodeGen/phi_after_error_block_outside_of_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 
 ; Make sure code generation does not break in case an 'error block' is detected
 ; outside of the scope. In this situation, we should not affect code generation.
diff --git a/polly/test/CodeGen/phi_condition_modeling_1.ll b/polly/test/CodeGen/phi_condition_modeling_1.ll
index b14d32921cf7..cf464fe234e3 100644
--- a/polly/test/CodeGen/phi_condition_modeling_1.ll
+++ b/polly/test/CodeGen/phi_condition_modeling_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ;    void f(int *A, int c, int N) {
 ;      int tmp;
diff --git a/polly/test/CodeGen/phi_condition_modeling_2.ll b/polly/test/CodeGen/phi_condition_modeling_2.ll
index dab2977bf065..25c67e625542 100644
--- a/polly/test/CodeGen/phi_condition_modeling_2.ll
+++ b/polly/test/CodeGen/phi_condition_modeling_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S  -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S  -passes=polly-codegen < %s | FileCheck %s
 ;
 ;    void f(int *A, int c, int N) {
 ;      int tmp;
diff --git a/polly/test/CodeGen/phi_conditional_simple_1.ll b/polly/test/CodeGen/phi_conditional_simple_1.ll
index f1b93b540f70..3d0e35ccd353 100644
--- a/polly/test/CodeGen/phi_conditional_simple_1.ll
+++ b/polly/test/CodeGen/phi_conditional_simple_1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ;    void jd(int *A, int c) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_1.ll b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_1.ll
index 13688480e315..70af1d96f4e7 100644
--- a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_1.ll
+++ b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused an lnt crash at some point, just verify it will run through.
 ;
diff --git a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_2.ll b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_2.ll
index 01dd450590d9..f17d4a6b9160 100644
--- a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_2.ll
+++ b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused an lnt crash at some point, just verify it will run through and
 ; produce the PHI node in the exit we are looking for.
diff --git a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_3.ll b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_3.ll
index 66b95b0e0317..cf9b4050a39c 100644
--- a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_3.ll
+++ b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused an lnt crash at some point, just verify it will run through and
 ; produce the PHI node in the exit we are looking for.
diff --git a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_5.ll b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_5.ll
index 9a046367e768..dfc3e6757af8 100644
--- a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_5.ll
+++ b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused an lnt crash at some point, just verify it will run through and
 ; produce the PHI node in the exit we are looking for.
diff --git a/polly/test/CodeGen/phi_loop_carried_float.ll b/polly/test/CodeGen/phi_loop_carried_float.ll
index ca1870fb3a09..df51c3599b29 100644
--- a/polly/test/CodeGen/phi_loop_carried_float.ll
+++ b/polly/test/CodeGen/phi_loop_carried_float.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S  -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S  -passes=polly-codegen < %s | FileCheck %s
 ;
 ;    float f(float *A, int N) {
 ;      float tmp = 0;
diff --git a/polly/test/CodeGen/phi_loop_carried_float_escape.ll b/polly/test/CodeGen/phi_loop_carried_float_escape.ll
index 3b2ed01863b1..10bc751150e2 100644
--- a/polly/test/CodeGen/phi_loop_carried_float_escape.ll
+++ b/polly/test/CodeGen/phi_loop_carried_float_escape.ll
@@ -1,8 +1,8 @@
 ; RUN: opt %loadPolly -S \
-; RUN: -polly-analyze-read-only-scalars=false -polly-codegen < %s | FileCheck %s
+; RUN: -polly-analyze-read-only-scalars=false -passes=polly-codegen < %s | FileCheck %s
 
 ; RUN: opt %loadPolly -S \
-; RUN: -polly-analyze-read-only-scalars=true -polly-codegen < %s | FileCheck %s
+; RUN: -polly-analyze-read-only-scalars=true -passes=polly-codegen < %s | FileCheck %s
 ;
 ;    float f(float *A, int N) {
 ;      float tmp = 0;
diff --git a/polly/test/CodeGen/phi_scalar_simple_1.ll b/polly/test/CodeGen/phi_scalar_simple_1.ll
index d62975b6a7b3..07b8021fb743 100644
--- a/polly/test/CodeGen/phi_scalar_simple_1.ll
+++ b/polly/test/CodeGen/phi_scalar_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ;    int jd(int *restrict A, int x, int N) {
 ;      for (int i = 1; i < N; i++)
diff --git a/polly/test/CodeGen/phi_scalar_simple_2.ll b/polly/test/CodeGen/phi_scalar_simple_2.ll
index e58945d39960..ab89b74c35a7 100644
--- a/polly/test/CodeGen/phi_scalar_simple_2.ll
+++ b/polly/test/CodeGen/phi_scalar_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ;    int jd(int *restrict A, int x, int N, int c) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/CodeGen/phi_with_multi_exiting_edges_2.ll b/polly/test/CodeGen/phi_with_multi_exiting_edges_2.ll
index 17e4b7d6b4de..313ae27a4165 100644
--- a/polly/test/CodeGen/phi_with_multi_exiting_edges_2.ll
+++ b/polly/test/CodeGen/phi_with_multi_exiting_edges_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; CHECK: polly.merge_new_and_old:
 ; CHECK:   %result.ph.merge = phi float [ %result.ph.final_reload, %polly.exiting ], [ %result.ph, %next.region_exiting ]
diff --git a/polly/test/CodeGen/phi_with_one_exit_edge.ll b/polly/test/CodeGen/phi_with_one_exit_edge.ll
index 81fd73b51c79..fa692a9cdd38 100644
--- a/polly/test/CodeGen/phi_with_one_exit_edge.ll
+++ b/polly/test/CodeGen/phi_with_one_exit_edge.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ;
 ; CHECK: polly.merge_new_and_old:
diff --git a/polly/test/CodeGen/pointer-type-expressions-2.ll b/polly/test/CodeGen/pointer-type-expressions-2.ll
index b261cfe53321..013a9634844b 100644
--- a/polly/test/CodeGen/pointer-type-expressions-2.ll
+++ b/polly/test/CodeGen/pointer-type-expressions-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @foo(ptr %start, ptr %end) {
diff --git a/polly/test/CodeGen/pointer-type-expressions.ll b/polly/test/CodeGen/pointer-type-expressions.ll
index 6bb3fa242362..ad0c0639a9d9 100644
--- a/polly/test/CodeGen/pointer-type-expressions.ll
+++ b/polly/test/CodeGen/pointer-type-expressions.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
 
 ; void f(int a[], int N, float *P) {
 ;   int i;
diff --git a/polly/test/CodeGen/pointer-type-pointer-type-comparison.ll b/polly/test/CodeGen/pointer-type-pointer-type-comparison.ll
index eaef64017aa7..5627a85c01b8 100644
--- a/polly/test/CodeGen/pointer-type-pointer-type-comparison.ll
+++ b/polly/test/CodeGen/pointer-type-pointer-type-comparison.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
 ;
 
 ;    void f(int a[], int N, float *P, float *Q) {
diff --git a/polly/test/CodeGen/pointer_rem.ll b/polly/test/CodeGen/pointer_rem.ll
index 5c92ee52da2c..a82c5cefa26d 100644
--- a/polly/test/CodeGen/pointer_rem.ll
+++ b/polly/test/CodeGen/pointer_rem.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-scops -polly-print-ast -disable-output -S < %s | FileCheck %s --check-prefix=AST
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-scops -polly-codegen -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>,scop(print<polly-ast>)' -disable-output -S < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>,scop(polly-codegen)' -S < %s | FileCheck %s --check-prefix=CODEGEN
 
 target datalayout = "e-m:e-i64:64-i128:128-n8:16:32:64-S128"
 target triple = "aarch64--linux-gnu"
diff --git a/polly/test/CodeGen/pr25241.ll b/polly/test/CodeGen/pr25241.ll
index 9fa67e083a6c..2f982c5ffbfc 100644
--- a/polly/test/CodeGen/pr25241.ll
+++ b/polly/test/CodeGen/pr25241.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 
 ; PR25241 (https://llvm.org/bugs/show_bug.cgi?id=25241)
 ; Ensure that synthesized values of a PHI node argument are generated in the
diff --git a/polly/test/CodeGen/ptrtoint_as_parameter.ll b/polly/test/CodeGen/ptrtoint_as_parameter.ll
index 4f6c8079729d..ea7cd57bdcc0 100644
--- a/polly/test/CodeGen/ptrtoint_as_parameter.ll
+++ b/polly/test/CodeGen/ptrtoint_as_parameter.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; CHECK:      if.then260:
 ; CHECK-NEXT:   %p.4 = getelementptr inbounds i8, ptr null, i64 1
diff --git a/polly/test/CodeGen/read-only-scalars.ll b/polly/test/CodeGen/read-only-scalars.ll
index a5e1d2719d7d..318362d6a27a 100644
--- a/polly/test/CodeGen/read-only-scalars.ll
+++ b/polly/test/CodeGen/read-only-scalars.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-analyze-read-only-scalars=false -polly-codegen \
+; RUN: opt %loadPolly -polly-analyze-read-only-scalars=false -passes=polly-codegen \
 ; RUN:     \
 ; RUN:     -S < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-analyze-read-only-scalars=true -polly-codegen \
+; RUN: opt %loadPolly -polly-analyze-read-only-scalars=true -passes=polly-codegen \
 ; RUN:     \
 ; RUN:     -S < %s | FileCheck %s -check-prefix=SCALAR
 
diff --git a/polly/test/CodeGen/reduction.ll b/polly/test/CodeGen/reduction.ll
index 6e5a230ad231..1af5a0d80124 100644
--- a/polly/test/CodeGen/reduction.ll
+++ b/polly/test/CodeGen/reduction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s 2>&1 | not FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s 2>&1 | not FileCheck %s
 
 ;#include <string.h>
 ;#include <stdio.h>
diff --git a/polly/test/CodeGen/reduction_2.ll b/polly/test/CodeGen/reduction_2.ll
index 7a50cea31400..b4ed4d95d543 100644
--- a/polly/test/CodeGen/reduction_2.ll
+++ b/polly/test/CodeGen/reduction_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-invariant-load-hoisting=true -polly-print-ast -disable-output < %s | FileCheck %s --allow-empty
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-invariant-load-hoisting=true '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s --allow-empty
 
 ;#include <string.h>
 ;#include <stdio.h>
diff --git a/polly/test/CodeGen/reduction_simple_binary.ll b/polly/test/CodeGen/reduction_simple_binary.ll
index c7c5501bb7ed..25903d6554c5 100644
--- a/polly/test/CodeGen/reduction_simple_binary.ll
+++ b/polly/test/CodeGen/reduction_simple_binary.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK: pragma simd reduction
 ;
diff --git a/polly/test/CodeGen/region-with-instructions.ll b/polly/test/CodeGen/region-with-instructions.ll
index 28cabefbf68b..125b791cbcf2 100644
--- a/polly/test/CodeGen/region-with-instructions.ll
+++ b/polly/test/CodeGen/region-with-instructions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 
 ; CHECK-LABEL:   polly.stmt.bb48:
 ; CHECK-NEXT:   %[[offset:.*]] = shl i64 %polly.indvar, 3
diff --git a/polly/test/CodeGen/region_exiting-domtree.ll b/polly/test/CodeGen/region_exiting-domtree.ll
index 05983da0a3e3..354f631e1002 100644
--- a/polly/test/CodeGen/region_exiting-domtree.ll
+++ b/polly/test/CodeGen/region_exiting-domtree.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -verify-dom-info -disable-output < %s
+; RUN: opt %loadPolly -passes=polly-codegen -verify-dom-info -disable-output < %s
 
 ; Verify that the DominatorTree is preserved correctly for the inserted
 ; %polly.stmt.exit.exit block, which serves as new exit block for the generated
diff --git a/polly/test/CodeGen/region_multiexit_partialwrite.ll b/polly/test/CodeGen/region_multiexit_partialwrite.ll
index b98d7f58732a..49547c6615d5 100644
--- a/polly/test/CodeGen/region_multiexit_partialwrite.ll
+++ b/polly/test/CodeGen/region_multiexit_partialwrite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
 ;
 ; This text case has a partial write of PHI in a region-statement. It
 ; requires that the new PHINode from the region's exiting block is
diff --git a/polly/test/CodeGen/run-time-condition-with-scev-parameters.ll b/polly/test/CodeGen/run-time-condition-with-scev-parameters.ll
index 0f62a8c743df..df6377ace33a 100644
--- a/polly/test/CodeGen/run-time-condition-with-scev-parameters.ll
+++ b/polly/test/CodeGen/run-time-condition-with-scev-parameters.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 
 ; TODO: FIXME: Simplify the context.
 ; AST: if (n >= 1 && 0 == n <= -1)
diff --git a/polly/test/CodeGen/run-time-condition.ll b/polly/test/CodeGen/run-time-condition.ll
index 0faefad8aef4..2a8cc72f8e9e 100644
--- a/polly/test/CodeGen/run-time-condition.ll
+++ b/polly/test/CodeGen/run-time-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/scalar-references-used-in-scop-compute.ll b/polly/test/CodeGen/scalar-references-used-in-scop-compute.ll
index 3f88942c2300..3519ffaef99b 100644
--- a/polly/test/CodeGen/scalar-references-used-in-scop-compute.ll
+++ b/polly/test/CodeGen/scalar-references-used-in-scop-compute.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 
 ; Test the code generation in the presence of a scalar out-of-scop value being
 ; used from within the SCoP.
diff --git a/polly/test/CodeGen/scalar-store-from-same-bb.ll b/polly/test/CodeGen/scalar-store-from-same-bb.ll
index ac8fab4b7a0d..016784a9deea 100644
--- a/polly/test/CodeGen/scalar-store-from-same-bb.ll
+++ b/polly/test/CodeGen/scalar-store-from-same-bb.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly \
-; RUN: -polly-codegen -S < %s | FileCheck %s
+; RUN: -passes=polly-codegen -S < %s | FileCheck %s
 
 ; This test ensures that the expression N + 1 that is stored in the phi-node
 ; alloca, is directly computed and not incorrectly transfered through memory.
diff --git a/polly/test/CodeGen/scalar_codegen_crash.ll b/polly/test/CodeGen/scalar_codegen_crash.ll
index c41a00f59e81..e89c3558a187 100644
--- a/polly/test/CodeGen/scalar_codegen_crash.ll
+++ b/polly/test/CodeGen/scalar_codegen_crash.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly \
-; RUN:     -polly-codegen -S < %s | FileCheck %s
+; RUN:     -passes=polly-codegen -S < %s | FileCheck %s
 
 ; This test cases used to crash the scalar code generation. Check that we
 ; can generate code for it.
diff --git a/polly/test/CodeGen/scev-backedgetaken.ll b/polly/test/CodeGen/scev-backedgetaken.ll
index 15e12ee8b451..00fcf0b03482 100644
--- a/polly/test/CodeGen/scev-backedgetaken.ll
+++ b/polly/test/CodeGen/scev-backedgetaken.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; llvm.org/PR48422
 ; Use of ScalarEvolution in Codegen not possible because DominatorTree is not updated.
diff --git a/polly/test/CodeGen/scev-division-invariant-load.ll b/polly/test/CodeGen/scev-division-invariant-load.ll
index 3156bdc9f5ce..242fb75c3883 100644
--- a/polly/test/CodeGen/scev-division-invariant-load.ll
+++ b/polly/test/CodeGen/scev-division-invariant-load.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s
 ;
 ; Check that we generate valid code as we did not use the preloaded
 ; value of %tmp1 for the access function of the preloaded %tmp4.
diff --git a/polly/test/CodeGen/scev.ll b/polly/test/CodeGen/scev.ll
index 07d726d97caf..74faf062bdc7 100644
--- a/polly/test/CodeGen/scev.ll
+++ b/polly/test/CodeGen/scev.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-detect < %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define fastcc void @f () inlinehint align 2 {
diff --git a/polly/test/CodeGen/scev_expansion_in_nonaffine.ll b/polly/test/CodeGen/scev_expansion_in_nonaffine.ll
index f61f21d4adb8..0575795cde48 100644
--- a/polly/test/CodeGen/scev_expansion_in_nonaffine.ll
+++ b/polly/test/CodeGen/scev_expansion_in_nonaffine.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S \
+; RUN: opt %loadPolly -passes=polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ; bugpoint-reduced testcase of MiBench/consumer-lame/quantize-pvt.c from the
diff --git a/polly/test/CodeGen/scev_looking_through_bitcasts.ll b/polly/test/CodeGen/scev_looking_through_bitcasts.ll
index c87d932479b7..776bb3332085 100644
--- a/polly/test/CodeGen/scev_looking_through_bitcasts.ll
+++ b/polly/test/CodeGen/scev_looking_through_bitcasts.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; Scalar write of bitcasted value. Instead of writing %b of type
 ; %structty, the SCEV expression looks through the bitcast such that
diff --git a/polly/test/CodeGen/scop_expander_insert_point.ll b/polly/test/CodeGen/scop_expander_insert_point.ll
index 8492873b22ed..8434e7ec1aa8 100644
--- a/polly/test/CodeGen/scop_expander_insert_point.ll
+++ b/polly/test/CodeGen/scop_expander_insert_point.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S \
+; RUN: opt %loadPolly -passes=polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; CHECK:      entry:
diff --git a/polly/test/CodeGen/scop_expander_segfault.ll b/polly/test/CodeGen/scop_expander_segfault.ll
index 293c1e527959..73145c4bfa23 100644
--- a/polly/test/CodeGen/scop_expander_segfault.ll
+++ b/polly/test/CodeGen/scop_expander_segfault.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S %s | FileCheck %s
 ;
 ; This test was extracted from gcc in SPEC2006 and it crashed our code
 ; generation, or to be more precise, the ScopExpander due to a endless
diff --git a/polly/test/CodeGen/scop_never_executed_runtime_check_location.ll b/polly/test/CodeGen/scop_never_executed_runtime_check_location.ll
index 91a58159b5f9..cc76ad1771c9 100644
--- a/polly/test/CodeGen/scop_never_executed_runtime_check_location.ll
+++ b/polly/test/CodeGen/scop_never_executed_runtime_check_location.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 
 ; Verify that we generate the runtime check code after the conditional branch
 ; in the SCoP region entering block (here %entry).
diff --git a/polly/test/CodeGen/select-base-pointer.ll b/polly/test/CodeGen/select-base-pointer.ll
index 29bc40074e1f..9748736147ab 100644
--- a/polly/test/CodeGen/select-base-pointer.ll
+++ b/polly/test/CodeGen/select-base-pointer.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -tbaa -polly-codegen -disable-output %s
+; RUN: opt %loadPolly -aa-pipeline=tbaa -passes=polly-codegen -disable-output %s
 ;
 ; Check that we do not crash here.
 ;
diff --git a/polly/test/CodeGen/sequential_loops.ll b/polly/test/CodeGen/sequential_loops.ll
index 97d280de3cd2..a0cecfbd817b 100644
--- a/polly/test/CodeGen/sequential_loops.ll
+++ b/polly/test/CodeGen/sequential_loops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ;#include <string.h>
 ;#define N 1024
diff --git a/polly/test/CodeGen/simple_loop_non_single_exit.ll b/polly/test/CodeGen/simple_loop_non_single_exit.ll
index dc1b09b765a1..a6f115ba1112 100644
--- a/polly/test/CodeGen/simple_loop_non_single_exit.ll
+++ b/polly/test/CodeGen/simple_loop_non_single_exit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/CodeGen/simple_loop_non_single_exit_2.ll b/polly/test/CodeGen/simple_loop_non_single_exit_2.ll
index 178601cac9b8..d58a6ed7b746 100644
--- a/polly/test/CodeGen/simple_loop_non_single_exit_2.ll
+++ b/polly/test/CodeGen/simple_loop_non_single_exit_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/CodeGen/simple_non_single_entry.ll b/polly/test/CodeGen/simple_non_single_entry.ll
index 3b4bf59bdc65..2b472496d364 100644
--- a/polly/test/CodeGen/simple_non_single_entry.ll
+++ b/polly/test/CodeGen/simple_non_single_entry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/CodeGen/simple_nonaffine_loop.ll b/polly/test/CodeGen/simple_nonaffine_loop.ll
index d4e9c6082e6c..4074237d1bfb 100644
--- a/polly/test/CodeGen/simple_nonaffine_loop.ll
+++ b/polly/test/CodeGen/simple_nonaffine_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-allow-nonaffine -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-allow-nonaffine -disable-output < %s | FileCheck %s
 
 ;#include <stdio.h>
 ;#include <stdlib.h>
diff --git a/polly/test/CodeGen/single_do_loop_int_max_iterations.ll b/polly/test/CodeGen/single_do_loop_int_max_iterations.ll
index 9648fbe1cf12..0b1d3e14f68a 100644
--- a/polly/test/CodeGen/single_do_loop_int_max_iterations.ll
+++ b/polly/test/CodeGen/single_do_loop_int_max_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ;#define N 20
 ;#include "limits.h"
diff --git a/polly/test/CodeGen/single_do_loop_int_param_iterations.ll b/polly/test/CodeGen/single_do_loop_int_param_iterations.ll
index f28d828a5da0..459ba18edac4 100644
--- a/polly/test/CodeGen/single_do_loop_int_param_iterations.ll
+++ b/polly/test/CodeGen/single_do_loop_int_param_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 ; XFAIL: *
 
 ;define N 20
diff --git a/polly/test/CodeGen/single_do_loop_ll_max_iterations.ll b/polly/test/CodeGen/single_do_loop_ll_max_iterations.ll
index 68aaab96083a..5fdb5b14df4d 100644
--- a/polly/test/CodeGen/single_do_loop_ll_max_iterations.ll
+++ b/polly/test/CodeGen/single_do_loop_ll_max_iterations.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen < %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen < %s
 
 ;#define N 20
 ;#include "limits.h"
diff --git a/polly/test/CodeGen/single_do_loop_one_iteration.ll b/polly/test/CodeGen/single_do_loop_one_iteration.ll
index 9d97cb854734..18bab682adeb 100644
--- a/polly/test/CodeGen/single_do_loop_one_iteration.ll
+++ b/polly/test/CodeGen/single_do_loop_one_iteration.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 ; XFAIL: *
 
 ;#define N 20
diff --git a/polly/test/CodeGen/single_do_loop_scev_replace.ll b/polly/test/CodeGen/single_do_loop_scev_replace.ll
index 7963d9d29fe8..9bdc3d7cd9b5 100644
--- a/polly/test/CodeGen/single_do_loop_scev_replace.ll
+++ b/polly/test/CodeGen/single_do_loop_scev_replace.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ;#define N 20
 ;#include "limits.h"
diff --git a/polly/test/CodeGen/single_loop.ll b/polly/test/CodeGen/single_loop.ll
index 68cc498b43e0..fe68c81567f8 100644
--- a/polly/test/CodeGen/single_loop.ll
+++ b/polly/test/CodeGen/single_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ;#include <string.h>
 ;#define N 1024
diff --git a/polly/test/CodeGen/single_loop_int_max_iterations.ll b/polly/test/CodeGen/single_loop_int_max_iterations.ll
index bfb5e4ab2698..017f1a3114cc 100644
--- a/polly/test/CodeGen/single_loop_int_max_iterations.ll
+++ b/polly/test/CodeGen/single_loop_int_max_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ;#define N 20
 ;#include "limits.h"
diff --git a/polly/test/CodeGen/single_loop_ll_max_iterations.ll b/polly/test/CodeGen/single_loop_ll_max_iterations.ll
index bdfd7fce4204..89fb9be7f9e1 100644
--- a/polly/test/CodeGen/single_loop_ll_max_iterations.ll
+++ b/polly/test/CodeGen/single_loop_ll_max_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ;#include "limits.h"
 ;#define N 20
diff --git a/polly/test/CodeGen/single_loop_one_iteration.ll b/polly/test/CodeGen/single_loop_one_iteration.ll
index 7d4dd590fab9..fcbe34da2a92 100644
--- a/polly/test/CodeGen/single_loop_one_iteration.ll
+++ b/polly/test/CodeGen/single_loop_one_iteration.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ;#define N 20
 ;
diff --git a/polly/test/CodeGen/single_loop_param.ll b/polly/test/CodeGen/single_loop_param.ll
index 5d72da354fdc..19f3fb42a475 100644
--- a/polly/test/CodeGen/single_loop_param.ll
+++ b/polly/test/CodeGen/single_loop_param.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @A = common global [1024 x i32] zeroinitializer, align 16 ; <ptr> [#uses=3]
diff --git a/polly/test/CodeGen/single_loop_param_less_equal.ll b/polly/test/CodeGen/single_loop_param_less_equal.ll
index e63ee299a37c..07d0ef24f4c0 100644
--- a/polly/test/CodeGen/single_loop_param_less_equal.ll
+++ b/polly/test/CodeGen/single_loop_param_less_equal.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen  -S < %s | FileCheck %s -check-prefix=CODEGEN
-; RUN: opt %loadPolly -polly-codegen < %s | opt -passes='print<loops>' -disable-output 2>&1 | FileCheck %s -check-prefix=LOOPS
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen  -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly -passes=polly-codegen < %s | opt -passes='print<loops>' -disable-output 2>&1 | FileCheck %s -check-prefix=LOOPS
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @A = common global [1024 x i32] zeroinitializer
diff --git a/polly/test/CodeGen/single_loop_param_less_than.ll b/polly/test/CodeGen/single_loop_param_less_than.ll
index 95130f926450..26dddea44b20 100644
--- a/polly/test/CodeGen/single_loop_param_less_than.ll
+++ b/polly/test/CodeGen/single_loop_param_less_than.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen  -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen  -S < %s | FileCheck %s -check-prefix=CODEGEN
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @A = common global [1024 x i32] zeroinitializer
diff --git a/polly/test/CodeGen/single_loop_zero_iterations.ll b/polly/test/CodeGen/single_loop_zero_iterations.ll
index 4f189687d330..e4a5b7670d77 100644
--- a/polly/test/CodeGen/single_loop_zero_iterations.ll
+++ b/polly/test/CodeGen/single_loop_zero_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=SCALAR --allow-empty
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=SCALAR --allow-empty
 
 ;#define N 20
 ;
diff --git a/polly/test/CodeGen/split_edge_of_exit.ll b/polly/test/CodeGen/split_edge_of_exit.ll
index 56ce215a62b2..3f2a1389f886 100644
--- a/polly/test/CodeGen/split_edge_of_exit.ll
+++ b/polly/test/CodeGen/split_edge_of_exit.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -verify-region-info -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -verify-region-info -disable-output < %s
 ;
 ; This is a scop directly precedented by a region, i.e. the scop's entry is the
 ; region's exit block. This test is to ensure that the RegionInfo is correctly
diff --git a/polly/test/CodeGen/split_edges.ll b/polly/test/CodeGen/split_edges.ll
index e01d901e298c..0fc705becb91 100644
--- a/polly/test/CodeGen/split_edges.ll
+++ b/polly/test/CodeGen/split_edges.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -verify-region-info -verify-dom-info -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -verify-region-info -verify-dom-info -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @A = common global [1536 x float] zeroinitializer
diff --git a/polly/test/CodeGen/split_edges_2.ll b/polly/test/CodeGen/split_edges_2.ll
index 4135d6feeb3e..84449327bf4c 100644
--- a/polly/test/CodeGen/split_edges_2.ll
+++ b/polly/test/CodeGen/split_edges_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -verify-region-info -verify-dom-info -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -verify-region-info -verify-dom-info -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/CodeGen/srem-in-other-bb.ll b/polly/test/CodeGen/srem-in-other-bb.ll
index 8bde1a3bbc1d..eaad663159f0 100644
--- a/polly/test/CodeGen/srem-in-other-bb.ll
+++ b/polly/test/CodeGen/srem-in-other-bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S \
+; RUN: opt %loadPolly -passes=polly-codegen -S \
 ; RUN:     < %s | FileCheck %s
 ;
 ;    void pos(float *A, long n) {
diff --git a/polly/test/CodeGen/stack-overflow-in-load-hoisting.ll b/polly/test/CodeGen/stack-overflow-in-load-hoisting.ll
index 02dfe96e3e91..41241de132a3 100644
--- a/polly/test/CodeGen/stack-overflow-in-load-hoisting.ll
+++ b/polly/test/CodeGen/stack-overflow-in-load-hoisting.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -verify-dom-info -polly-codegen -S < %s \
+; RUN: opt %loadPolly -verify-dom-info -passes=polly-codegen -S < %s \
 ; RUN: -polly-invariant-load-hoisting=true | FileCheck %s
 ;
 ; This caused an infinite recursion during invariant load hoisting at some
diff --git a/polly/test/CodeGen/stmt_split_no_dependence.ll b/polly/test/CodeGen/stmt_split_no_dependence.ll
index a395aa14b4c8..94407e8e7c38 100644
--- a/polly/test/CodeGen/stmt_split_no_dependence.ll
+++ b/polly/test/CodeGen/stmt_split_no_dependence.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; CHECK:   store i32 %9, ptr %scevgep, align 4, !alias.scope !1, !noalias !4
 ; CHECK:   store i32 %11, ptr %scevgep4, align 4, !alias.scope !4, !noalias !1
diff --git a/polly/test/CodeGen/switch-in-non-affine-region.ll b/polly/test/CodeGen/switch-in-non-affine-region.ll
index 930755ef5648..2524699157b7 100644
--- a/polly/test/CodeGen/switch-in-non-affine-region.ll
+++ b/polly/test/CodeGen/switch-in-non-affine-region.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly \
-; RUN: -S -polly-codegen < %s | FileCheck %s
+; RUN: -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/CodeGen/synthesizable_phi_write_after_loop.ll b/polly/test/CodeGen/synthesizable_phi_write_after_loop.ll
index 6a8d3b94d1cc..86745d71953e 100644
--- a/polly/test/CodeGen/synthesizable_phi_write_after_loop.ll
+++ b/polly/test/CodeGen/synthesizable_phi_write_after_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; Check for the correct written value of a scalar phi write whose value is
 ; defined within the loop, but its effective value is its last definition when
diff --git a/polly/test/CodeGen/test-invalid-operands-for-select-2.ll b/polly/test/CodeGen/test-invalid-operands-for-select-2.ll
index 5fa4773398fd..9b3608c81f50 100644
--- a/polly/test/CodeGen/test-invalid-operands-for-select-2.ll
+++ b/polly/test/CodeGen/test-invalid-operands-for-select-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen -verify-loop-info < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen -verify-loop-info < %s | FileCheck %s
 ;
 ; Check that we do not crash as described here: http://llvm.org/bugs/show_bug.cgi?id=21167
 ;
diff --git a/polly/test/CodeGen/test-invalid-operands-for-select.ll b/polly/test/CodeGen/test-invalid-operands-for-select.ll
index 40695af3e847..a10603126cf6 100644
--- a/polly/test/CodeGen/test-invalid-operands-for-select.ll
+++ b/polly/test/CodeGen/test-invalid-operands-for-select.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ; Check that we do not crash as described here: http://llvm.org/PR21167
 ;
diff --git a/polly/test/CodeGen/test.ll b/polly/test/CodeGen/test.ll
index ac99688ed9e8..1038e57c358f 100644
--- a/polly/test/CodeGen/test.ll
+++ b/polly/test/CodeGen/test.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 ; XFAIL: *
 
 ;int bar1();
diff --git a/polly/test/CodeGen/two-loops-right-after-each-other-2.ll b/polly/test/CodeGen/two-loops-right-after-each-other-2.ll
index a7cae0a921ca..71bec88e7dde 100644
--- a/polly/test/CodeGen/two-loops-right-after-each-other-2.ll
+++ b/polly/test/CodeGen/two-loops-right-after-each-other-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 
 ; CHECK:       polly.merge_new_and_old:
 ; CHECK-NEXT:    merge = phi
diff --git a/polly/test/CodeGen/two-scops-in-row-invalidate-scevs.ll b/polly/test/CodeGen/two-scops-in-row-invalidate-scevs.ll
index 4470f970fc1e..327430da11bb 100644
--- a/polly/test/CodeGen/two-scops-in-row-invalidate-scevs.ll
+++ b/polly/test/CodeGen/two-scops-in-row-invalidate-scevs.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: for.cond:
 ; CHECK:         %num.0 = phi i32 [ %add, %for.body15 ], [ 0, %for.cond.pre_entry_bb ]
diff --git a/polly/test/CodeGen/two-scops-in-row.ll b/polly/test/CodeGen/two-scops-in-row.ll
index 3e922cba1916..06e7ea096d79 100644
--- a/polly/test/CodeGen/two-scops-in-row.ll
+++ b/polly/test/CodeGen/two-scops-in-row.ll
@@ -1,6 +1,6 @@
 
-; RUN: opt %loadPolly -polly-print-ast -polly-ignore-aliasing -disable-output < %s | FileCheck %s -check-prefix=SCALAR
-; RUN: opt %loadPolly -polly-codegen -polly-ignore-aliasing -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ignore-aliasing -disable-output < %s | FileCheck %s -check-prefix=SCALAR
+; RUN: opt %loadPolly -passes=polly-codegen -polly-ignore-aliasing -disable-output < %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; SCALAR: if (
diff --git a/polly/test/CodeGen/udiv_expansion_position.ll b/polly/test/CodeGen/udiv_expansion_position.ll
index bb37fed4a41e..39df17dc2030 100644
--- a/polly/test/CodeGen/udiv_expansion_position.ll
+++ b/polly/test/CodeGen/udiv_expansion_position.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
 ;
 ; Verify we do not crash when we synthezise code for the udiv in the SCoP.
 ;
diff --git a/polly/test/CodeGen/uninitialized_scalar_memory.ll b/polly/test/CodeGen/uninitialized_scalar_memory.ll
index 935ccc3d6289..89eb32c4cf0f 100644
--- a/polly/test/CodeGen/uninitialized_scalar_memory.ll
+++ b/polly/test/CodeGen/uninitialized_scalar_memory.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
 ;
 ; Verify we initialize the scalar locations reserved for the incoming phi
 ; values.
diff --git a/polly/test/CodeGen/unpredictable-loop-unsynthesizable.ll b/polly/test/CodeGen/unpredictable-loop-unsynthesizable.ll
index 9164bb4532e6..e3d0f2df7351 100644
--- a/polly/test/CodeGen/unpredictable-loop-unsynthesizable.ll
+++ b/polly/test/CodeGen/unpredictable-loop-unsynthesizable.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops \
-; RUN: -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-codegen \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' \
+; RUN: -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -passes=polly-codegen \
 ; RUN: -polly-invariant-load-hoisting=true -disable-output < %s
 
 ; The loop for.body is a scop with invariant load hoisting, but does not
diff --git a/polly/test/CodeGen/variant_load_empty_domain.ll b/polly/test/CodeGen/variant_load_empty_domain.ll
index f5ad0b195818..0ea3b0d1ed1f 100644
--- a/polly/test/CodeGen/variant_load_empty_domain.ll
+++ b/polly/test/CodeGen/variant_load_empty_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
 ;
 ;
 ;    void f(int *A) {
diff --git a/polly/test/CodeGen/whole-scop-non-affine-subregion.ll b/polly/test/CodeGen/whole-scop-non-affine-subregion.ll
index 931e644f6b8f..9c911715904d 100644
--- a/polly/test/CodeGen/whole-scop-non-affine-subregion.ll
+++ b/polly/test/CodeGen/whole-scop-non-affine-subregion.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly \
-; RUN: -polly-codegen -S < %s | FileCheck %s
+; RUN: -passes=polly-codegen -S < %s | FileCheck %s
 
 ; CHECK: polly.start
 ;    int /* pure */ g()
diff --git a/polly/test/DeLICM/confused_order.ll b/polly/test/DeLICM/confused_order.ll
index 2015ebcf58f1..62f59cdef315 100644
--- a/polly/test/DeLICM/confused_order.ll
+++ b/polly/test/DeLICM/confused_order.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-delicm -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-delicm -disable-output -pass-remarks-missed=polly-delicm < %s 2>&1 | FileCheck %s -check-prefix=REMARKS
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-delicm>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-delicm' -polly-import-jscop-postfix=transformed -disable-output -pass-remarks-missed=polly-delicm < %s 2>&1 | FileCheck %s -check-prefix=REMARKS
 ;
 ; ForwardOptree changes the SCoP and may already map some accesses.
 ; DeLICM must be prepared to encounter implicit reads
diff --git a/polly/test/DeLICM/contradicting_assumed_context_and_domain.ll b/polly/test/DeLICM/contradicting_assumed_context_and_domain.ll
index 4e039b22b415..768cb23631e7 100644
--- a/polly/test/DeLICM/contradicting_assumed_context_and_domain.ll
+++ b/polly/test/DeLICM/contradicting_assumed_context_and_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 ;
 ; The domain of bb14 contradicts the SCoP's assumptions. This leads to
 ; 'anything goes' inside the statement since it is never executed,
diff --git a/polly/test/DeLICM/load-in-cond-inf-loop.ll b/polly/test/DeLICM/load-in-cond-inf-loop.ll
index f0aecfd87a15..40e30a52c545 100644
--- a/polly/test/DeLICM/load-in-cond-inf-loop.ll
+++ b/polly/test/DeLICM/load-in-cond-inf-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 
 ; When %b is 0, %for.body13 is an infite loop. In this case the loaded
 ; value %1 is not used anywhere.
diff --git a/polly/test/DeLICM/map_memset_zero.ll b/polly/test/DeLICM/map_memset_zero.ll
index 1a08eee63fe9..6789577cb046 100644
--- a/polly/test/DeLICM/map_memset_zero.ll
+++ b/polly/test/DeLICM/map_memset_zero.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck -match-full-lines %s
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-delicm>)" -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Check that PHI mapping works even in presence of a memset whose'
 ; zero value is used.
diff --git a/polly/test/DeLICM/nomap_alreadymapped.ll b/polly/test/DeLICM/nomap_alreadymapped.ll
index 7adf4ba88385..bf26a809324b 100644
--- a/polly/test/DeLICM/nomap_alreadymapped.ll
+++ b/polly/test/DeLICM/nomap_alreadymapped.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/nomap_escaping.ll b/polly/test/DeLICM/nomap_escaping.ll
index 034c0a96ccf2..17451a2941ea 100644
--- a/polly/test/DeLICM/nomap_escaping.ll
+++ b/polly/test/DeLICM/nomap_escaping.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/nomap_occupied.ll b/polly/test/DeLICM/nomap_occupied.ll
index db33532b1e65..e6ca903dae03 100644
--- a/polly/test/DeLICM/nomap_occupied.ll
+++ b/polly/test/DeLICM/nomap_occupied.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/nomap_readonly.ll b/polly/test/DeLICM/nomap_readonly.ll
index 1f3b5746fe9b..2c19bb9bc495 100644
--- a/polly/test/DeLICM/nomap_readonly.ll
+++ b/polly/test/DeLICM/nomap_readonly.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      fsomeval = 21.0 + 21.0;
diff --git a/polly/test/DeLICM/nomap_spuriouswrite.ll b/polly/test/DeLICM/nomap_spuriouswrite.ll
index ef470f715bbe..f561a4b189ee 100644
--- a/polly/test/DeLICM/nomap_spuriouswrite.ll
+++ b/polly/test/DeLICM/nomap_spuriouswrite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/nomap_storagesize.ll b/polly/test/DeLICM/nomap_storagesize.ll
index fab8d54c2bdf..2c116cc85609 100644
--- a/polly/test/DeLICM/nomap_storagesize.ll
+++ b/polly/test/DeLICM/nomap_storagesize.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 ;
 ;    void func(float *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/nomap_writewrite.ll b/polly/test/DeLICM/nomap_writewrite.ll
index 06192d9ae19e..6e3b06a4c57f 100644
--- a/polly/test/DeLICM/nomap_writewrite.ll
+++ b/polly/test/DeLICM/nomap_writewrite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/outofquota-reverseDomain.ll b/polly/test/DeLICM/outofquota-reverseDomain.ll
index d40ee03cf3bc..d917d294dcdf 100644
--- a/polly/test/DeLICM/outofquota-reverseDomain.ll
+++ b/polly/test/DeLICM/outofquota-reverseDomain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-delicm-max-ops=1000000 -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-delicm-max-ops=1000000 '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 ;
 ; This causes an assertion to fail on out-of-quota after 1000000 operations.
 ; (The error was specific to -polly-delicm-max-ops=1000000 and changes
diff --git a/polly/test/DeLICM/pass_existence.ll b/polly/test/DeLICM/pass_existence.ll
index 7ed2da9c1da1..57adf45b207c 100644
--- a/polly/test/DeLICM/pass_existence.ll
+++ b/polly/test/DeLICM/pass_existence.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-delicm -disable-output < %s
-; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-delicm>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-delicm -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=scop(print<polly-delicm>)' -disable-output < %s | FileCheck %s
 ;
 ; Simple test for the existence of the DeLICM pass.
 ;
diff --git a/polly/test/DeLICM/pr41656.ll b/polly/test/DeLICM/pr41656.ll
index 965ad9f62ac3..ba65b5bf0416 100644
--- a/polly/test/DeLICM/pr41656.ll
+++ b/polly/test/DeLICM/pr41656.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>,scop(print<polly-delicm>)' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; llvm.org/PR41656
 ;
diff --git a/polly/test/DeLICM/pr48783.ll b/polly/test/DeLICM/pr48783.ll
index 3cbd54b93baf..2bba7f731f56 100644
--- a/polly/test/DeLICM/pr48783.ll
+++ b/polly/test/DeLICM/pr48783.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>,scop(print<polly-delicm>)' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; llvm.org/PR48783
 ;
diff --git a/polly/test/DeLICM/reduction.ll b/polly/test/DeLICM/reduction.ll
index 78c1a4ce5288..a6f1e032f6e4 100644
--- a/polly/test/DeLICM/reduction.ll
+++ b/polly/test/DeLICM/reduction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-delicm-partial-writes=true -polly-print-delicm -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-delicm-partial-writes=true '-passes=print<polly-delicm>' -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/reduction_looprotate_gvnpre_cond1.ll b/polly/test/DeLICM/reduction_looprotate_gvnpre_cond1.ll
index b5bc0d589c65..e30fe12c8a74 100644
--- a/polly/test/DeLICM/reduction_looprotate_gvnpre_cond1.ll
+++ b/polly/test/DeLICM/reduction_looprotate_gvnpre_cond1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Load (but not store) of A[j] hoisted, reduction only over some iterations.
 ;
diff --git a/polly/test/DeLICM/reduction_looprotate_gvnpre_cond2.ll b/polly/test/DeLICM/reduction_looprotate_gvnpre_cond2.ll
index e995be1143a6..a830782e9c44 100644
--- a/polly/test/DeLICM/reduction_looprotate_gvnpre_cond2.ll
+++ b/polly/test/DeLICM/reduction_looprotate_gvnpre_cond2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Load (but not store) of A[j] hoisted, reduction not written in all iterations.
 ; FIXME: %join is not mapped because the MemoryKind::Value mapping does not
diff --git a/polly/test/DeLICM/reduction_looprotate_gvnpre_nopreheader.ll b/polly/test/DeLICM/reduction_looprotate_gvnpre_nopreheader.ll
index ca3a1211ca49..903bc80b9d47 100644
--- a/polly/test/DeLICM/reduction_looprotate_gvnpre_nopreheader.ll
+++ b/polly/test/DeLICM/reduction_looprotate_gvnpre_nopreheader.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Hosted reduction load (but not the store) without preheader.
 ;
diff --git a/polly/test/DeLICM/reduction_looprotate_licm_nopreheader.ll b/polly/test/DeLICM/reduction_looprotate_licm_nopreheader.ll
index 41538239fbd8..23497f19402b 100644
--- a/polly/test/DeLICM/reduction_looprotate_licm_nopreheader.ll
+++ b/polly/test/DeLICM/reduction_looprotate_licm_nopreheader.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
 ;
 ; Register-promoted reduction but without preheader.
 ;
diff --git a/polly/test/DeLICM/reduction_looprotate_loopguard_gvnpre.ll b/polly/test/DeLICM/reduction_looprotate_loopguard_gvnpre.ll
index 35c723e864d2..c932a311a7aa 100644
--- a/polly/test/DeLICM/reduction_looprotate_loopguard_gvnpre.ll
+++ b/polly/test/DeLICM/reduction_looprotate_loopguard_gvnpre.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Reduction over parametric number of elements and a loopguard if the
 ; reduction loop is not executed at all. Load hoisted before loop.
diff --git a/polly/test/DeLICM/reduction_looprotate_loopguard_licm1.ll b/polly/test/DeLICM/reduction_looprotate_loopguard_licm1.ll
index 2b5f4d8151a8..52073eb35014 100644
--- a/polly/test/DeLICM/reduction_looprotate_loopguard_licm1.ll
+++ b/polly/test/DeLICM/reduction_looprotate_loopguard_licm1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Reduction over parametric number of elements and a loopguard if the
 ; reduction loop is not executed at all.
diff --git a/polly/test/DeLICM/reduction_looprotate_loopguard_licm2.ll b/polly/test/DeLICM/reduction_looprotate_loopguard_licm2.ll
index 2e92813d5551..88ae8b3b0f20 100644
--- a/polly/test/DeLICM/reduction_looprotate_loopguard_licm2.ll
+++ b/polly/test/DeLICM/reduction_looprotate_loopguard_licm2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Reduction over parametric number of elements and a loopguard if the
 ; reduction loop is not executed at all, such that A[j] is also not written to.
diff --git a/polly/test/DeLICM/reduction_looprotate_loopguard_licm3.ll b/polly/test/DeLICM/reduction_looprotate_loopguard_licm3.ll
index 784c8ef2d321..3a356926afa0 100644
--- a/polly/test/DeLICM/reduction_looprotate_loopguard_licm3.ll
+++ b/polly/test/DeLICM/reduction_looprotate_loopguard_licm3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Reduction over parametric number of elements and a loopguard if the
 ; reduction loop is not executed at all, such that A[j] is also not accessed.
diff --git a/polly/test/DeLICM/reduction_unrelatedunusual.ll b/polly/test/DeLICM/reduction_unrelatedunusual.ll
index 04c437770700..00097dfc92e8 100644
--- a/polly/test/DeLICM/reduction_unrelatedunusual.ll
+++ b/polly/test/DeLICM/reduction_unrelatedunusual.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-delicm-partial-writes=true -polly-print-delicm -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-delicm-partial-writes=true '-passes=print<polly-delicm>' -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Map %add and %phi to A[j].
 ; The non-analyzable store to C[0] is unrelated and can be ignored.
diff --git a/polly/test/DeLICM/reject_loadafterstore.ll b/polly/test/DeLICM/reject_loadafterstore.ll
index 8af6e5e4818c..2a153b5cd710 100644
--- a/polly/test/DeLICM/reject_loadafterstore.ll
+++ b/polly/test/DeLICM/reject_loadafterstore.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output -pass-remarks-missed=polly-delicm < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output -pass-remarks-missed=polly-delicm < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/reject_outofquota.ll b/polly/test/DeLICM/reject_outofquota.ll
index 551431f0823c..35001ec1ab2d 100644
--- a/polly/test/DeLICM/reject_outofquota.ll
+++ b/polly/test/DeLICM/reject_outofquota.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-delicm -pass-remarks-analysis=polly-delicm -polly-delicm-max-ops=1 -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-delicm -polly-print-dependences -polly-delicm-max-ops=1 -polly-dependences-computeout=0 -disable-output < %s | FileCheck %s -check-prefix=DEP
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -pass-remarks-analysis=polly-delicm -polly-delicm-max-ops=1 -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-delicm,print<polly-dependences>' -polly-delicm-max-ops=1 -polly-dependences-computeout=0 -disable-output < %s | FileCheck %s -check-prefix=DEP
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/reject_storeafterstore.ll b/polly/test/DeLICM/reject_storeafterstore.ll
index 1ec5ef67344c..715375fcdea2 100644
--- a/polly/test/DeLICM/reject_storeafterstore.ll
+++ b/polly/test/DeLICM/reject_storeafterstore.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/reject_storeinsubregion.ll b/polly/test/DeLICM/reject_storeinsubregion.ll
index 1d38e8066568..6490dc25d4a5 100644
--- a/polly/test/DeLICM/reject_storeinsubregion.ll
+++ b/polly/test/DeLICM/reject_storeinsubregion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/reject_unusualstore.ll b/polly/test/DeLICM/reject_unusualstore.ll
index a18a0c3ce9c4..22c7d8ca0ea6 100644
--- a/polly/test/DeLICM/reject_unusualstore.ll
+++ b/polly/test/DeLICM/reject_unusualstore.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-delicm -disable-output -stats < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -passes=polly-delicm -disable-output -stats < %s 2>&1 | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 ;
 ;    void func(double *A) {
diff --git a/polly/test/DeLICM/skip_maywrite.ll b/polly/test/DeLICM/skip_maywrite.ll
index 1e5f6b169fe4..4da0ddb9dfd2 100644
--- a/polly/test/DeLICM/skip_maywrite.ll
+++ b/polly/test/DeLICM/skip_maywrite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/skip_multiaccess.ll b/polly/test/DeLICM/skip_multiaccess.ll
index 6a8c8e5325e1..0eea60d0f488 100644
--- a/polly/test/DeLICM/skip_multiaccess.ll
+++ b/polly/test/DeLICM/skip_multiaccess.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; llvm.org/PR34485
 ; llvm.org/PR34989
diff --git a/polly/test/DeLICM/skip_notinloop.ll b/polly/test/DeLICM/skip_notinloop.ll
index 0730a3a9a4f5..caa70179d666 100644
--- a/polly/test/DeLICM/skip_notinloop.ll
+++ b/polly/test/DeLICM/skip_notinloop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      double phi = 0.0;
diff --git a/polly/test/DeLICM/skip_scalaraccess.ll b/polly/test/DeLICM/skip_scalaraccess.ll
index fa95d382409a..8c9728dc6c42 100644
--- a/polly/test/DeLICM/skip_scalaraccess.ll
+++ b/polly/test/DeLICM/skip_scalaraccess.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeadCodeElimination/chained_iterations.ll b/polly/test/DeadCodeElimination/chained_iterations.ll
index b79fdd659aae..10be83559f29 100644
--- a/polly/test/DeadCodeElimination/chained_iterations.ll
+++ b/polly/test/DeadCodeElimination/chained_iterations.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-dependences-analysis-type=value-based '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-dependences-analysis-type=value-based '-passes=polly-dce,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ;
 ; for(i = 0; i < 200; i++ )
diff --git a/polly/test/DeadCodeElimination/chained_iterations_2.ll b/polly/test/DeadCodeElimination/chained_iterations_2.ll
index 1d1af92db5da..42242c40eac9 100644
--- a/polly/test/DeadCodeElimination/chained_iterations_2.ll
+++ b/polly/test/DeadCodeElimination/chained_iterations_2.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-dependences-analysis-type=value-based '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-dependences-analysis-type=value-based '-passes=polly-dce,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ;
 ; for(i = 0; i < 200; i++ )
diff --git a/polly/test/DeadCodeElimination/computeout.ll b/polly/test/DeadCodeElimination/computeout.ll
index 51850d7da349..2ac6b8cbbedf 100644
--- a/polly/test/DeadCodeElimination/computeout.ll
+++ b/polly/test/DeadCodeElimination/computeout.ll
@@ -1,6 +1,5 @@
-; RUN: opt -S %loadPolly -basic-aa -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadNPMPolly "-passes=scop(polly-dce,print<polly-ast>)" < %s | FileCheck %s
-; RUN: opt -S %loadPolly -basic-aa -polly-dce -polly-print-ast -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
+; RUN: opt -S %loadPolly "-passes=scop(polly-dce,print<polly-ast>)" < %s | FileCheck %s
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa "-passes=scop(polly-dce,print<polly-ast>)" -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;     for(i = 0; i < 100; i++ )
diff --git a/polly/test/DeadCodeElimination/dead_iteration_elimination.ll b/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
index f496f7828e3d..4247ccbcd123 100644
--- a/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
+++ b/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
@@ -1,5 +1,4 @@
-; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-dce-precise-steps=2 -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadNPMPolly "-passes=scop(polly-dce,print<polly-ast>)" -polly-dependences-analysis-type=value-based -polly-dce-precise-steps=2 < %s | FileCheck %s
+; RUN: opt -S %loadPolly "-passes=scop(polly-dce,print<polly-ast>)" -polly-dependences-analysis-type=value-based -polly-dce-precise-steps=2 < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ;
 ; for(i = 0; i < 200; i++ )
diff --git a/polly/test/DeadCodeElimination/non-affine-affine-mix.ll b/polly/test/DeadCodeElimination/non-affine-affine-mix.ll
index e6a5dd204ca1..e290d9997d96 100644
--- a/polly/test/DeadCodeElimination/non-affine-affine-mix.ll
+++ b/polly/test/DeadCodeElimination/non-affine-affine-mix.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=polly-dce,print<polly-ast>' -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/DeadCodeElimination/non-affine.ll b/polly/test/DeadCodeElimination/non-affine.ll
index 38a7fcbcf9c9..8f437ef3e32a 100644
--- a/polly/test/DeadCodeElimination/non-affine.ll
+++ b/polly/test/DeadCodeElimination/non-affine.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=polly-dce,print<polly-ast>' -disable-output < %s | FileCheck %s
 ;
 ; CHECK: for (int c0 = 0; c0 <= 1023; c0 += 1)
 ;
diff --git a/polly/test/DeadCodeElimination/null_schedule.ll b/polly/test/DeadCodeElimination/null_schedule.ll
index 633a84b5d92b..13c8a127b173 100644
--- a/polly/test/DeadCodeElimination/null_schedule.ll
+++ b/polly/test/DeadCodeElimination/null_schedule.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-dependences-analysis-type=value-based '-passes=polly-dce,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ; A[0] = 1;
 ;
diff --git a/polly/test/DependenceInfo/computeout.ll b/polly/test/DependenceInfo/computeout.ll
index 048de29864d3..0e64cd1a3725 100644
--- a/polly/test/DependenceInfo/computeout.ll
+++ b/polly/test/DependenceInfo/computeout.ll
@@ -1,7 +1,5 @@
-; RUN: opt -S %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s -check-prefix=VALUE
-; RUN: opt -S %loadPolly -polly-print-function-dependences -disable-output < %s | FileCheck %s -check-prefix=FUNC-VALUE
-; RUN: opt -S %loadPolly -polly-print-dependences -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
-; RUN: opt -S %loadPolly -polly-print-function-dependences -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
+; RUN: opt -S %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt -S %loadPolly '-passes=print<polly-dependences>' -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;     for(i = 0; i < 100; i++ )
diff --git a/polly/test/DependenceInfo/different_schedule_dimensions.ll b/polly/test/DependenceInfo/different_schedule_dimensions.ll
index 3f966168d3b7..edb8371d6e68 100644
--- a/polly/test/DependenceInfo/different_schedule_dimensions.ll
+++ b/polly/test/DependenceInfo/different_schedule_dimensions.ll
@@ -1,7 +1,5 @@
-; RUN: opt -S %loadPolly -polly-print-dependences \
+; RUN: opt -S %loadPolly '-passes=print<polly-dependences>' \
 ; RUN:                   -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadPolly -polly-print-function-dependences \
-; RUN:                   -disable-output < %s | FileCheck %s -check-prefix=FUNC
 
 ; CHECK: RAW dependences:
 ; CHECK:   { Stmt_bb9[0] -> Stmt_bb10[0] }
diff --git a/polly/test/DependenceInfo/do_pluto_matmult.ll b/polly/test/DependenceInfo/do_pluto_matmult.ll
index d71608e80e70..9532f9c7fb12 100644
--- a/polly/test/DependenceInfo/do_pluto_matmult.ll
+++ b/polly/test/DependenceInfo/do_pluto_matmult.ll
@@ -1,7 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-dependences -polly-dependences-analysis-type=value-based -disable-output < %s | FileCheck %s -check-prefix=VALUE
-; RUN: opt %loadPolly -basic-aa -polly-print-dependences -polly-dependences-analysis-type=memory-based -disable-output < %s | FileCheck %s -check-prefix=MEMORY
-; RUN: opt %loadPolly -basic-aa -polly-print-function-dependences -polly-dependences-analysis-type=value-based -disable-output < %s | FileCheck %s -check-prefix=FUNC-VALUE
-; RUN: opt %loadPolly -basic-aa -polly-print-function-dependences -polly-dependences-analysis-type=memory-based -disable-output < %s | FileCheck %s -check-prefix=FUNC-MEMORY
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -polly-dependences-analysis-type=value-based -disable-output < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -polly-dependences-analysis-type=memory-based -disable-output < %s | FileCheck %s -check-prefix=MEMORY
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/DependenceInfo/fine_grain_dep_0.ll b/polly/test/DependenceInfo/fine_grain_dep_0.ll
index 9c79e360690a..e7fc66882465 100644
--- a/polly/test/DependenceInfo/fine_grain_dep_0.ll
+++ b/polly/test/DependenceInfo/fine_grain_dep_0.ll
@@ -1,7 +1,6 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-dependences -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s --check-prefix=REF
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-dependences -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s --check-prefix=ACC
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-function-dependences -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s --check-prefix=ACC
-;
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-dependences>' -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s --check-prefix=REF
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-dependences>' -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s --check-prefix=ACC
+
 ; REF:      RAW dependences:
 ; REF-NEXT:     [N] -> { [Stmt_for_body[i0] -> MemRef_b[]] -> [Stmt_for_body[6 + i0] -> MemRef_b[]] : 0 <= i0 <= -13 + N; Stmt_for_body[i0] -> Stmt_for_body[6 + i0] : 0 <= i0 <= -13 + N; Stmt_for_body[i0] -> Stmt_for_body[4 + i0] : 0 <= i0 <= -11 + N; [Stmt_for_body[i0] -> MemRef_a[]] -> [Stmt_for_body[4 + i0] -> MemRef_a[]] : 0 <= i0 <= -11 + N }
 ; REF-NEXT: WAR dependences:
diff --git a/polly/test/DependenceInfo/generate_may_write_dependence_info.ll b/polly/test/DependenceInfo/generate_may_write_dependence_info.ll
index 0b7f2d48da9f..7f6f5f3e3b94 100644
--- a/polly/test/DependenceInfo/generate_may_write_dependence_info.ll
+++ b/polly/test/DependenceInfo/generate_may_write_dependence_info.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt -S %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s -check-prefix=VALUE
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 ;                     for (int i = 0; i < N; i++) {
diff --git a/polly/test/DependenceInfo/infeasible_context.ll b/polly/test/DependenceInfo/infeasible_context.ll
index d701b821e15c..aab6072e4a45 100644
--- a/polly/test/DependenceInfo/infeasible_context.ll
+++ b/polly/test/DependenceInfo/infeasible_context.ll
@@ -1,10 +1,9 @@
-; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN:  | FileCheck %s -check-prefix=FUNC-SCOP
-; RUN: opt %loadPolly -polly-print-function-dependences -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-detect>,scop(print<polly-dependences>)' -disable-output < %s 2>&1 \
 ; RUN:  | FileCheck %s -check-prefix=FUNC-DEPS
 ;
 ; FUNC-SCOP-NOT: Statement
-; FUNC-DEPS-LABEL: Printing analysis 'Polly - Calculate dependences for all the SCoPs of a function' for function 'readgeo'
 ; FUNC-DEPS-NOT: RAW dependences
 ;
 ; Due to an infeasible run-time check, scop object is empty and we do not compute dependences.
diff --git a/polly/test/DependenceInfo/may_writes_do_not_block_must_writes_for_war.ll b/polly/test/DependenceInfo/may_writes_do_not_block_must_writes_for_war.ll
index 09c516274708..662cb3232e7e 100644
--- a/polly/test/DependenceInfo/may_writes_do_not_block_must_writes_for_war.ll
+++ b/polly/test/DependenceInfo/may_writes_do_not_block_must_writes_for_war.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; Verify that the presence of a may-write (S1) between a read (S0) and a
 ; must-write (S2) does not block the generation of RAW dependences. This makes
diff --git a/polly/test/DependenceInfo/nonaffine-condition-buildMemoryAccess.ll b/polly/test/DependenceInfo/nonaffine-condition-buildMemoryAccess.ll
index 25c7e3d6e442..d8361a2b74a6 100644
--- a/polly/test/DependenceInfo/nonaffine-condition-buildMemoryAccess.ll
+++ b/polly/test/DependenceInfo/nonaffine-condition-buildMemoryAccess.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -polly-allow-nonaffine-loops -polly-allow-nonaffine -debug-only=polly-dependence < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-allow-nonaffine-loops -polly-allow-nonaffine -debug-only=polly-dependence < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; CHECK:        MayWriteAccess :=   [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/DependenceInfo/reduction_complex_location.ll b/polly/test/DependenceInfo/reduction_complex_location.ll
index 7ca839996326..2c14f116c904 100644
--- a/polly/test/DependenceInfo/reduction_complex_location.ll
+++ b/polly/test/DependenceInfo/reduction_complex_location.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-dependences -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-dependences -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     {  }
diff --git a/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll b/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll
index 3632bd202da2..e32217910fc7 100644
--- a/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll
+++ b/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; This loopnest contains a reduction which imposes the same dependences as the
 ; accesses to the array A. We need to ensure we keep the dependences of A.
diff --git a/polly/test/DependenceInfo/reduction_dependences_not_null.ll b/polly/test/DependenceInfo/reduction_dependences_not_null.ll
index 69fd74478ecc..852f03cb6f70 100644
--- a/polly/test/DependenceInfo/reduction_dependences_not_null.ll
+++ b/polly/test/DependenceInfo/reduction_dependences_not_null.ll
@@ -1,7 +1,7 @@
 ; Test that the reduction dependences are always initialised, even in a case
 ; where we have no reduction. If this object is NULL, then isl operations on
 ; it will fail.
-; RUN: opt -S %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt -S %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s -check-prefix=VALUE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;     for(i = 0; i < 100; i++ )
diff --git a/polly/test/DependenceInfo/reduction_mixed_reduction_and_non_reduction_dependences.ll b/polly/test/DependenceInfo/reduction_mixed_reduction_and_non_reduction_dependences.ll
index 71903d9e7111..4c78d80b8ceb 100644
--- a/polly/test/DependenceInfo/reduction_mixed_reduction_and_non_reduction_dependences.ll
+++ b/polly/test/DependenceInfo/reduction_mixed_reduction_and_non_reduction_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_for_body3[i0, i1] -> Stmt_for_body3[i0 + i1, o1] : i0 >= 0 and 0 <= i1 <= 1023 - i0 and i1 <= 1 and 0 < o1 <= 511 }
diff --git a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll
index 234de5c367a0..7e05265f6e6d 100644
--- a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll
@@ -1,6 +1,6 @@
-; RUN: opt -basic-aa %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
-; RUN: opt -basic-aa %loadPolly -polly-print-dependences -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s
-; RUN: opt -basic-aa %loadPolly -polly-print-dependences -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa %loadPolly '-passes=print<polly-dependences>' -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa %loadPolly '-passes=print<polly-dependences>' -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s
 ;
 ; Verify that only the inner reduction like accesses cause reduction dependences
 ;
diff --git a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll
index acd674dc0117..7e8a66a50bfd 100644
--- a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -basic-aa -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     {  }
diff --git a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll
index bdfcfc99c8cb..adb04f305993 100644
--- a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -basic-aa -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      Reduction dependences:
 ; CHECK-NEXT:     { Stmt_for_inc[i0, i1] -> Stmt_for_inc[i0, 1 + i1] : 0 <= i0 <= 99 and 0 <= i1 <= 98 }
diff --git a/polly/test/DependenceInfo/reduction_multiple_reductions.ll b/polly/test/DependenceInfo/reduction_multiple_reductions.ll
index cf705080e03d..2d810915bada 100644
--- a/polly/test/DependenceInfo/reduction_multiple_reductions.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_reductions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; Verify we do not have dependences between the if and the else clause
 ;
diff --git a/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll b/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll
index 8d8557a129ab..326dcaee5f07 100644
--- a/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ;
 ; These are the important RAW dependences, as they need to originate/end in only one iteration:
diff --git a/polly/test/DependenceInfo/reduction_only_reduction_like_access.ll b/polly/test/DependenceInfo/reduction_only_reduction_like_access.ll
index 7b4a68a2a897..1e6455b264d3 100644
--- a/polly/test/DependenceInfo/reduction_only_reduction_like_access.ll
+++ b/polly/test/DependenceInfo/reduction_only_reduction_like_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; FIXME: Change the comment once we allow different pointers
 ; The statement is "almost" reduction like but should not yield any reduction dependences
diff --git a/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll b/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll
index 0d09e5a861a0..158e6a29d8a7 100644
--- a/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll
+++ b/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -basic-aa -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      Reduction dependences:
 ; CHECK-NEXT:     [N] -> { Stmt_for_body3[i0, i1] -> Stmt_for_body3[i0, 1 + i1] : 0 <= i0 <= 1023 and i1 >= 0 and 1024 - N + i0 <= i1 <= 1022 }
diff --git a/polly/test/DependenceInfo/reduction_privatization_deps.ll b/polly/test/DependenceInfo/reduction_privatization_deps.ll
index ce90e21a898d..5d62b8f0a10a 100644
--- a/polly/test/DependenceInfo/reduction_privatization_deps.ll
+++ b/polly/test/DependenceInfo/reduction_privatization_deps.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_S1[i0, i1] -> Stmt_S2[-1 + i0 + i1] : 0 <= i0 <= 1023 and i1 >= 0 and -i0 < i1 <= 1024 - i0 and i1 <= 1023; Stmt_S0[i0] -> Stmt_S1[o0, i0 - o0] : i0 <= 1023 and 0 <= o0 <= i0 }
diff --git a/polly/test/DependenceInfo/reduction_privatization_deps_2.ll b/polly/test/DependenceInfo/reduction_privatization_deps_2.ll
index 4904004d4781..ed936bfb3b7f 100644
--- a/polly/test/DependenceInfo/reduction_privatization_deps_2.ll
+++ b/polly/test/DependenceInfo/reduction_privatization_deps_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; We have privatization dependences from a textually later statement to a
 ; textually earlier one, but the dependences still go forward in time.
diff --git a/polly/test/DependenceInfo/reduction_privatization_deps_3.ll b/polly/test/DependenceInfo/reduction_privatization_deps_3.ll
index a3935ebd6cc4..58ef9a4774b9 100644
--- a/polly/test/DependenceInfo/reduction_privatization_deps_3.ll
+++ b/polly/test/DependenceInfo/reduction_privatization_deps_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_S1[i0] -> Stmt_S3[2 + i0] : 0 <= i0 <= 96; Stmt_S2[i0, i1] -> Stmt_S3[o0] : i1 <= 1 - i0 and -i1 < o0 <= 1 and o0 <= 1 + i0 - i1; Stmt_S3[i0] -> Stmt_S2[o0, 1 - i0] : 0 <= i0 <= 1 and i0 < o0 <= 98 }
diff --git a/polly/test/DependenceInfo/reduction_privatization_deps_4.ll b/polly/test/DependenceInfo/reduction_privatization_deps_4.ll
index 10d726af5145..3dad7b217486 100644
--- a/polly/test/DependenceInfo/reduction_privatization_deps_4.ll
+++ b/polly/test/DependenceInfo/reduction_privatization_deps_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_S1[i0] -> Stmt_S2[i0, i0] : 0 <= i0 <= 98; Stmt_S2[i0, i0] -> Stmt_S3[i0] : 0 <= i0 <= 98; Stmt_S3[i0] -> Stmt_S2[o0, i0] : i0 >= 0 and i0 < o0 <= 98; Stmt_S2[i0, i1] -> Stmt_S1[i1] : i0 >= 0 and i0 < i1 <= 98 }
diff --git a/polly/test/DependenceInfo/reduction_privatization_deps_5.ll b/polly/test/DependenceInfo/reduction_privatization_deps_5.ll
index e8d51181725e..0c445d23a92d 100644
--- a/polly/test/DependenceInfo/reduction_privatization_deps_5.ll
+++ b/polly/test/DependenceInfo/reduction_privatization_deps_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_S1[i0, 0] -> Stmt_S2[i0, 0] : 0 <= i0 <= 98; Stmt_S2[i0, 0] -> Stmt_S1[1 + i0, 0] : 0 <= i0 <= 97 }
diff --git a/polly/test/DependenceInfo/reduction_sequence.ll b/polly/test/DependenceInfo/reduction_sequence.ll
index 4a4688953938..7e1ebd4ab67c 100644
--- a/polly/test/DependenceInfo/reduction_sequence.ll
+++ b/polly/test/DependenceInfo/reduction_sequence.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 
 ;    void manyreductions(long *A) {
 ;      for (long i = 0; i < 1024; i++)
diff --git a/polly/test/DependenceInfo/reduction_simple_iv.ll b/polly/test/DependenceInfo/reduction_simple_iv.ll
index e3307afae08b..64f6de22b078 100644
--- a/polly/test/DependenceInfo/reduction_simple_iv.ll
+++ b/polly/test/DependenceInfo/reduction_simple_iv.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     {  }
diff --git a/polly/test/DependenceInfo/reduction_simple_iv_debug_wrapped_dependences.ll b/polly/test/DependenceInfo/reduction_simple_iv_debug_wrapped_dependences.ll
index c7651c39a563..61ee3dbc5b02 100644
--- a/polly/test/DependenceInfo/reduction_simple_iv_debug_wrapped_dependences.ll
+++ b/polly/test/DependenceInfo/reduction_simple_iv_debug_wrapped_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -debug-only=polly-dependence -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -debug-only=polly-dependence -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/DependenceInfo/reduction_simple_privatization_deps_2.ll b/polly/test/DependenceInfo/reduction_simple_privatization_deps_2.ll
index b61fd8453a8c..a181d2acb651 100644
--- a/polly/test/DependenceInfo/reduction_simple_privatization_deps_2.ll
+++ b/polly/test/DependenceInfo/reduction_simple_privatization_deps_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_S1[i0, i1] -> Stmt_S2[i0] : 0 <= i0 <= 99 and 0 <= i1 <= 99; Stmt_S0[i0] -> Stmt_S1[i0, o1] : 0 <= i0 <= 99 and 0 <= o1 <= 99; Stmt_S2[i0] -> Stmt_S0[1 + i0] : 0 <= i0 <= 98 }
diff --git a/polly/test/DependenceInfo/reduction_simple_privatization_deps_w_parameter.ll b/polly/test/DependenceInfo/reduction_simple_privatization_deps_w_parameter.ll
index a3a87c70d905..ffa26b5eb76c 100644
--- a/polly/test/DependenceInfo/reduction_simple_privatization_deps_w_parameter.ll
+++ b/polly/test/DependenceInfo/reduction_simple_privatization_deps_w_parameter.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     [N] -> { Stmt_S1[i0] -> Stmt_S2[] : N >= 11 and 0 <= i0 <= 1023; Stmt_S0[] -> Stmt_S1[o0] : N >= 11 and 0 <= o0 <= 1023 }
diff --git a/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll b/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll
index c90462962ce0..25d117d4cbf9 100644
--- a/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll
+++ b/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     {  }
diff --git a/polly/test/DependenceInfo/sequential_loops.ll b/polly/test/DependenceInfo/sequential_loops.ll
index 8dfa13cb9db8..14c9a6429c67 100644
--- a/polly/test/DependenceInfo/sequential_loops.ll
+++ b/polly/test/DependenceInfo/sequential_loops.ll
@@ -1,34 +1,43 @@
-; RUN: opt -S %loadPolly -basic-aa -polly-print-dependences -polly-dependences-analysis-type=value-based -disable-output < %s | FileCheck %s -check-prefix=VALUE
-; RUN: opt -S %loadPolly -basic-aa -polly-print-dependences -polly-dependences-analysis-type=memory-based -disable-output < %s | FileCheck %s -check-prefix=MEMORY
-; RUN: opt -S %loadPolly -basic-aa -polly-print-dependences -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s -check-prefix=VALUE_ACCESS
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -polly-dependences-analysis-type=value-based -disable-output < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -polly-dependences-analysis-type=memory-based -disable-output < %s | FileCheck %s -check-prefix=MEMORY
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s -check-prefix=VALUE_ACCESS
 
-; VALUE-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'sequential_writes':
-; VALUE-NEXT:      RAW dependences:
+; VALUE:      RAW dependences:
 ; VALUE-NEXT:          {  }
 ; VALUE-NEXT:      WAR dependences:
 ; VALUE-NEXT:          {  }
 ; VALUE-NEXT:      WAW dependences:
 ; VALUE-NEXT:          { Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99; Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9 }
 ;
-;VALUE_ACCESS-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'sequential_writes':
-;VALUE_ACCESS-NEXT:        RAW dependences:
-;VALUE_ACCESS-NEXT:                {  }
-;VALUE_ACCESS-NEXT:        WAR dependences:
-;VALUE_ACCESS-NEXT:                {  }
-;VALUE_ACCESS-NEXT:        WAW dependences:
-;VALUE_ACCESS-NEXT:                { Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99; Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; [Stmt_S2[i0] -> Stmt_S2_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Write0[]] : 0 <= i0 <= 9; Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S2[i0] -> Stmt_S2_Write0[]] : 0 <= i0 <= 9; [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Write0[]] : 10 <= i0 <= 99 }
-
-;
-; VALUE-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'read_after_writes':
-; VALUE-NEXT:      RAW dependences:
+; VALUE:      RAW dependences:
 ; VALUE-NEXT:          { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99 }
 ; VALUE-NEXT:      WAR dependences:
 ; VALUE-NEXT:          {  }
 ; VALUE-NEXT:      WAW dependences:
 ; VALUE-NEXT:          { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9 }
 ;
-;VALUE_ACCESS-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'read_after_writes':
-;VALUE_ACCESS-NEXT:        RAW dependences:
+; VALUE:      RAW dependences:
+; VALUE-NEXT:          {  }
+; VALUE-NEXT:      WAR dependences:
+; VALUE-NEXT:          { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99 }
+; VALUE-NEXT:      WAW dependences:
+; VALUE-NEXT:          { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9 }
+;
+; VALUE:      RAW dependences:
+; VALUE-NEXT:          [p] -> { Stmt_S1[i0] -> Stmt_S2[-p + i0] : i0 >= p and 0 <= i0 <= 99 and i0 <= 9 + p }
+; VALUE-NEXT:      WAR dependences:
+; VALUE-NEXT:          [p] -> {  }
+; VALUE-NEXT:      WAW dependences:
+; VALUE-NEXT:          [p] -> {  }
+;
+;VALUE_ACCESS:        RAW dependences:
+;VALUE_ACCESS-NEXT:                {  }
+;VALUE_ACCESS-NEXT:        WAR dependences:
+;VALUE_ACCESS-NEXT:                {  }
+;VALUE_ACCESS-NEXT:        WAW dependences:
+;VALUE_ACCESS-NEXT:                { Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99; Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; [Stmt_S2[i0] -> Stmt_S2_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Write0[]] : 0 <= i0 <= 9; Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S2[i0] -> Stmt_S2_Write0[]] : 0 <= i0 <= 9; [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Write0[]] : 10 <= i0 <= 99 }
+;
+;VALUE_ACCESS:        RAW dependences:
 ;VALUE_ACCESS-NEXT:                { Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99; Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; [Stmt_S2[i0] -> Stmt_S2_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Read0[]] : 0 <= i0 <= 9; [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Read0[]] : 10 <= i0 <= 99 }
 
 ;VALUE_ACCESS-NEXT:        WAR dependences:
@@ -36,64 +45,42 @@
 ;VALUE_ACCESS-NEXT:        WAW dependences:
 ;VALUE_ACCESS-NEXT:                { [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S2[i0] -> Stmt_S2_Write0[]] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9 }
 ;
-; VALUE-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'write_after_read':
-; VALUE-NEXT:      RAW dependences:
-; VALUE-NEXT:          {  }
-; VALUE-NEXT:      WAR dependences:
-; VALUE-NEXT:          { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99 }
-; VALUE-NEXT:      WAW dependences:
-; VALUE-NEXT:          { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9 }
-;
-;VALUE_ACCESS-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'write_after_read':
-;VALUE_ACCESS-NEXT:         RAW dependences:
+;VALUE_ACCESS:         RAW dependences:
 ;VALUE_ACCESS-NEXT:                 {  }
 ;VALUE_ACCESS-NEXT:         WAR dependences:
 ;VALUE_ACCESS-NEXT:                { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99; [Stmt_S1[i0] -> Stmt_S1_Read0[]] -> [Stmt_S2[i0] -> Stmt_S2_Write0[]] : 0 <= i0 <= 9; [Stmt_S1[i0] -> Stmt_S1_Read0[]] -> [Stmt_S3[i0] -> Stmt_S3_Write0[]] : 10 <= i0 <= 99 }
 ;VALUE_ACCESS-NEXT:         WAW dependences:
 ;VALUE_ACCESS-NEXT:                { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; [Stmt_S2[i0] -> Stmt_S2_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Write0[]] : 0 <= i0 <= 9 }
 ;
-; VALUE-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.2' in function 'parametric_offset':
-; VALUE-NEXT:      RAW dependences:
-; VALUE-NEXT:          [p] -> { Stmt_S1[i0] -> Stmt_S2[-p + i0] : i0 >= p and 0 <= i0 <= 99 and i0 <= 9 + p }
-; VALUE-NEXT:      WAR dependences:
-; VALUE-NEXT:          [p] -> {  }
-; VALUE-NEXT:      WAW dependences:
-; VALUE-NEXT:          [p] -> {  }
-;
-;VALUE_ACCESS-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.2' in function 'parametric_offset':
-;VALUE_ACCESS-NEXT:        RAW dependences:
+;VALUE_ACCESS:        RAW dependences:
 ;VALUE_ACCESS-NEXT:                [p] -> { Stmt_S1[i0] -> Stmt_S2[-p + i0] : i0 >= p and 0 <= i0 <= 99 and i0 <= 9 + p; [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S2[-p + i0] -> Stmt_S2_Read0[]] : i0 >= p and 0 <= i0 <= 99 and i0 <= 9 + p }
 ;VALUE_ACCESS-NEXT:        WAR dependences:
 ;VALUE_ACCESS-NEXT:                [p] -> {  }
 ;VALUE_ACCESS-NEXT:        WAW dependences:
 ;VALUE_ACCESS-NEXT:                [p] -> {  }
 
-; MEMORY-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'sequential_writes':
-; MEMORY-NEXT:      RAW dependences:
+; MEMORY:      RAW dependences:
 ; MEMORY-NEXT:          {  }
 ; MEMORY-NEXT:      WAR dependences:
 ; MEMORY-NEXT:          {  }
 ; MEMORY-NEXT:      WAW dependences:
 ; MEMORY-NEXT:          { Stmt_S1[i0] -> Stmt_S3[i0] : 0 <= i0 <= 99; Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9 }
 ;
-; MEMORY-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'read_after_writes':
-; MEMORY-NEXT:      RAW dependences:
+; MEMORY:      RAW dependences:
 ; MEMORY-NEXT:          { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 0 <= i0 <= 99 }
 ; MEMORY-NEXT:      WAR dependences:
 ; MEMORY-NEXT:          {  }
 ; MEMORY-NEXT:      WAW dependences:
 ; MEMORY-NEXT:          { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9 }
 ;
-; MEMORY-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'write_after_read':
-; MEMORY-NEXT:      RAW dependences:
+; MEMORY:      RAW dependences:
 ; MEMORY-NEXT:          {  }
 ; MEMORY-NEXT:      WAR dependences:
 ; MEMORY-NEXT:          { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 0 <= i0 <= 99 }
 ; MEMORY-NEXT:      WAW dependences:
 ; MEMORY-NEXT:          { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9 }
 ;
-; MEMORY-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.2' in function 'parametric_offset':
-; MEMORY-NEXT:      RAW dependences:
+; MEMORY:      RAW dependences:
 ; MEMORY-NEXT:          [p] -> { Stmt_S1[i0] -> Stmt_S2[-p + i0] : i0 >= p and 0 <= i0 <= 99 and i0 <= 9 + p }
 ; MEMORY-NEXT:      WAR dependences:
 ; MEMORY-NEXT:          [p] -> {  }
diff --git a/polly/test/ForwardOpTree/atax.ll b/polly/test/ForwardOpTree/atax.ll
index 0690c1b000fa..7cc40fe7e1cb 100644
--- a/polly/test/ForwardOpTree/atax.ll
+++ b/polly/test/ForwardOpTree/atax.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-optree-normalize-phi=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-optree-normalize-phi=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ForwardOpTree/changed-kind.ll b/polly/test/ForwardOpTree/changed-kind.ll
index a1d59825b3b2..3c3d7f738779 100644
--- a/polly/test/ForwardOpTree/changed-kind.ll
+++ b/polly/test/ForwardOpTree/changed-kind.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 
 ; In the code below, %0 is known to be equal to the content of @c (constant 0).
 ; Thus, in order to save a scalar dependency, forward-optree replaces
diff --git a/polly/test/ForwardOpTree/forward_from_region.ll b/polly/test/ForwardOpTree/forward_from_region.ll
index 53d22800081e..90448d800286 100644
--- a/polly/test/ForwardOpTree/forward_from_region.ll
+++ b/polly/test/ForwardOpTree/forward_from_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move instructions from region statements.
 ;
diff --git a/polly/test/ForwardOpTree/forward_hoisted.ll b/polly/test/ForwardOpTree/forward_hoisted.ll
index 32fca00141dd..163d43f56e8a 100644
--- a/polly/test/ForwardOpTree/forward_hoisted.ll
+++ b/polly/test/ForwardOpTree/forward_hoisted.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move %val to %bodyB, so %bodyA can be removed (by -polly-simplify).
 ; This involves making the load-hoisted %val1 to be made available in %bodyB.
diff --git a/polly/test/ForwardOpTree/forward_instruction.ll b/polly/test/ForwardOpTree/forward_instruction.ll
index 1dcd64357324..6269e359a59a 100644
--- a/polly/test/ForwardOpTree/forward_instruction.ll
+++ b/polly/test/ForwardOpTree/forward_instruction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move %val to %bodyB, so %bodyA can be removed (by -polly-simplify)
 ;
diff --git a/polly/test/ForwardOpTree/forward_into_region.ll b/polly/test/ForwardOpTree/forward_into_region.ll
index dd18cfe5e61a..be102a9574dc 100644
--- a/polly/test/ForwardOpTree/forward_into_region.ll
+++ b/polly/test/ForwardOpTree/forward_into_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move instructions to region statements.
 ;
diff --git a/polly/test/ForwardOpTree/forward_into_region_redundant_use.ll b/polly/test/ForwardOpTree/forward_into_region_redundant_use.ll
index e5458c027880..883a784230d9 100644
--- a/polly/test/ForwardOpTree/forward_into_region_redundant_use.ll
+++ b/polly/test/ForwardOpTree/forward_into_region_redundant_use.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 
 define void @foo(ptr %A, i32 %p, ptr %B) {
diff --git a/polly/test/ForwardOpTree/forward_load.ll b/polly/test/ForwardOpTree/forward_load.ll
index 86e3cb0203fa..dec6812aade5 100644
--- a/polly/test/ForwardOpTree/forward_load.ll
+++ b/polly/test/ForwardOpTree/forward_load.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-optree>)" -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly "-passes=scop(print<polly-optree>)" -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load.
 ;
diff --git a/polly/test/ForwardOpTree/forward_load_differentarray.ll b/polly/test/ForwardOpTree/forward_load_differentarray.ll
index 786277bdeb87..a3ca0bad54ab 100644
--- a/polly/test/ForwardOpTree/forward_load_differentarray.ll
+++ b/polly/test/ForwardOpTree/forward_load_differentarray.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; To forward %val, B[j] cannot be reused in bodyC because it is overwritten
 ; between. Verify that instead the alternative C[j] is used.
diff --git a/polly/test/ForwardOpTree/forward_load_double_write.ll b/polly/test/ForwardOpTree/forward_load_double_write.ll
index 1618722381fc..b0fbb69dc7ae 100644
--- a/polly/test/ForwardOpTree/forward_load_double_write.ll
+++ b/polly/test/ForwardOpTree/forward_load_double_write.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load even in case two writes of identical values are in
 ; one scop statement.
diff --git a/polly/test/ForwardOpTree/forward_load_fromloop.ll b/polly/test/ForwardOpTree/forward_load_fromloop.ll
index 8f08a1356c38..62351883a189 100644
--- a/polly/test/ForwardOpTree/forward_load_fromloop.ll
+++ b/polly/test/ForwardOpTree/forward_load_fromloop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Forward a the LoadInst %val into %bodyB. %val is executed multiple times,
 ; we must get the last loaded values.
diff --git a/polly/test/ForwardOpTree/forward_load_indirect.ll b/polly/test/ForwardOpTree/forward_load_indirect.ll
index f83af61e6741..c8144861abc4 100644
--- a/polly/test/ForwardOpTree/forward_load_indirect.ll
+++ b/polly/test/ForwardOpTree/forward_load_indirect.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Forward an operand tree consisting of a speculatable instruction (%add)
 ; and a load (%val).
diff --git a/polly/test/ForwardOpTree/forward_load_memset_after.ll b/polly/test/ForwardOpTree/forward_load_memset_after.ll
index 13797a44c862..22a2ddf94888 100644
--- a/polly/test/ForwardOpTree/forward_load_memset_after.ll
+++ b/polly/test/ForwardOpTree/forward_load_memset_after.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load in the presence of a non-store WRITE access.
 ;
diff --git a/polly/test/ForwardOpTree/forward_load_memset_before.ll b/polly/test/ForwardOpTree/forward_load_memset_before.ll
index 60b1e076b980..3d3c90e941d6 100644
--- a/polly/test/ForwardOpTree/forward_load_memset_before.ll
+++ b/polly/test/ForwardOpTree/forward_load_memset_before.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load in the presence of a non-store WRITE access.
 ;
diff --git a/polly/test/ForwardOpTree/forward_load_tripleuse.ll b/polly/test/ForwardOpTree/forward_load_tripleuse.ll
index 1d0df2a22e87..03a47360c362 100644
--- a/polly/test/ForwardOpTree/forward_load_tripleuse.ll
+++ b/polly/test/ForwardOpTree/forward_load_tripleuse.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-optree -polly-codegen -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-optree>,polly-codegen' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; %val1 is used three times: Twice by its own operand tree of %val2 and once
 ; more by the store in %bodyB.
diff --git a/polly/test/ForwardOpTree/forward_load_unrelatedunusual.ll b/polly/test/ForwardOpTree/forward_load_unrelatedunusual.ll
index b7bae5628986..dbeebbc27eba 100644
--- a/polly/test/ForwardOpTree/forward_load_unrelatedunusual.ll
+++ b/polly/test/ForwardOpTree/forward_load_unrelatedunusual.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load.
 ; The non-analyzable store to C[0] is unrelated and can be ignored.
diff --git a/polly/test/ForwardOpTree/forward_phi_load.ll b/polly/test/ForwardOpTree/forward_phi_load.ll
index 0b0bb209a3ef..029261f269c3 100644
--- a/polly/test/ForwardOpTree/forward_phi_load.ll
+++ b/polly/test/ForwardOpTree/forward_phi_load.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-optree-normalize-phi=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-optree-normalize-phi=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load.
 ;
diff --git a/polly/test/ForwardOpTree/forward_readonly.ll b/polly/test/ForwardOpTree/forward_readonly.ll
index a29c5bff5d70..7ded946a6ff8 100644
--- a/polly/test/ForwardOpTree/forward_readonly.ll
+++ b/polly/test/ForwardOpTree/forward_readonly.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-analyze-read-only-scalars=true  -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines -check-prefixes=STATS,MODEL
-; RUN: opt %loadPolly -polly-analyze-read-only-scalars=false -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines -check-prefixes=STATS,NOMODEL
+; RUN: opt %loadPolly -polly-analyze-read-only-scalars=true  '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines -check-prefixes=STATS,MODEL
+; RUN: opt %loadPolly -polly-analyze-read-only-scalars=false '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines -check-prefixes=STATS,NOMODEL
 ;
 ; Move %val to %bodyB, so %bodyA can be removed (by -polly-simplify)
 ;
diff --git a/polly/test/ForwardOpTree/forward_reusue.ll b/polly/test/ForwardOpTree/forward_reusue.ll
index ead8c7379803..1151aa94e1f9 100644
--- a/polly/test/ForwardOpTree/forward_reusue.ll
+++ b/polly/test/ForwardOpTree/forward_reusue.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move operand tree without duplicating values used multiple times.
 ;
diff --git a/polly/test/ForwardOpTree/forward_store.ll b/polly/test/ForwardOpTree/forward_store.ll
index a6369eb303c1..e02c6891f436 100644
--- a/polly/test/ForwardOpTree/forward_store.ll
+++ b/polly/test/ForwardOpTree/forward_store.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load.
 ;
diff --git a/polly/test/ForwardOpTree/forward_synthesizable_definloop.ll b/polly/test/ForwardOpTree/forward_synthesizable_definloop.ll
index f0da9320c43f..90e489ad6f54 100644
--- a/polly/test/ForwardOpTree/forward_synthesizable_definloop.ll
+++ b/polly/test/ForwardOpTree/forward_synthesizable_definloop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Copy %val to bodyB, assuming the exit value of %i.
 ;
diff --git a/polly/test/ForwardOpTree/forward_synthesizable_indvar.ll b/polly/test/ForwardOpTree/forward_synthesizable_indvar.ll
index a38ab543e255..395e68482657 100644
--- a/polly/test/ForwardOpTree/forward_synthesizable_indvar.ll
+++ b/polly/test/ForwardOpTree/forward_synthesizable_indvar.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Test support for (synthesizable) inducation variables.
 ;
diff --git a/polly/test/ForwardOpTree/forward_synthesizable_useinloop.ll b/polly/test/ForwardOpTree/forward_synthesizable_useinloop.ll
index bb1760ae0ffb..a45d420e5c20 100644
--- a/polly/test/ForwardOpTree/forward_synthesizable_useinloop.ll
+++ b/polly/test/ForwardOpTree/forward_synthesizable_useinloop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Synthesizable values defined outside of a loop can be used
 ; inside the loop.
diff --git a/polly/test/ForwardOpTree/forward_transitive.ll b/polly/test/ForwardOpTree/forward_transitive.ll
index 243889437149..69cfb555f315 100644
--- a/polly/test/ForwardOpTree/forward_transitive.ll
+++ b/polly/test/ForwardOpTree/forward_transitive.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move %v and %val to %bodyB, so %bodyA can be removed (by -polly-simplify)
 ;
diff --git a/polly/test/ForwardOpTree/jacobi-1d.ll b/polly/test/ForwardOpTree/jacobi-1d.ll
index 05ccd998c1a2..dbc051dde425 100644
--- a/polly/test/ForwardOpTree/jacobi-1d.ll
+++ b/polly/test/ForwardOpTree/jacobi-1d.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-optree-normalize-phi=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-optree-normalize-phi=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ForwardOpTree/noforward_from_region.ll b/polly/test/ForwardOpTree/noforward_from_region.ll
index 30150912f32e..11d4312ad3bc 100644
--- a/polly/test/ForwardOpTree/noforward_from_region.ll
+++ b/polly/test/ForwardOpTree/noforward_from_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Ensure we do not move instructions from region statements in case the
 ; instruction to move loads from an array which is also written to from
diff --git a/polly/test/ForwardOpTree/noforward_load_conditional.ll b/polly/test/ForwardOpTree/noforward_load_conditional.ll
index eaa0fc52186b..053134196001 100644
--- a/polly/test/ForwardOpTree/noforward_load_conditional.ll
+++ b/polly/test/ForwardOpTree/noforward_load_conditional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; B[j] is overwritten by at least one statement between the
 ; definition of %val and its use. Hence, it cannot be forwarded.
diff --git a/polly/test/ForwardOpTree/noforward_load_writebetween.ll b/polly/test/ForwardOpTree/noforward_load_writebetween.ll
index e2272c1c1f13..4a281b66c618 100644
--- a/polly/test/ForwardOpTree/noforward_load_writebetween.ll
+++ b/polly/test/ForwardOpTree/noforward_load_writebetween.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Cannot rematerialize %val from B[0] at bodyC because B[0] has been
 ; overwritten in bodyB.
diff --git a/polly/test/ForwardOpTree/noforward_outofquota.ll b/polly/test/ForwardOpTree/noforward_outofquota.ll
index 2ec965d71184..9c17349fb9e2 100644
--- a/polly/test/ForwardOpTree/noforward_outofquota.ll
+++ b/polly/test/ForwardOpTree/noforward_outofquota.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-optree-max-ops=1 -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadPolly -polly-optree-max-ops=1 -polly-optree -disable-output -stats < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=STATS
+; RUN: opt %loadPolly -polly-optree-max-ops=1 '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-optree-max-ops=1 -passes=polly-optree -disable-output -stats < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=STATS
 ; REQUIRES: asserts
 ;
 ; for (int j = 0; j < n; j += 1) {
diff --git a/polly/test/ForwardOpTree/noforward_partial.ll b/polly/test/ForwardOpTree/noforward_partial.ll
index 127ac9ff5f14..67bda40337e0 100644
--- a/polly/test/ForwardOpTree/noforward_partial.ll
+++ b/polly/test/ForwardOpTree/noforward_partial.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Not the entire operand tree can be forwarded,
 ; some scalar dependencies would remain.
diff --git a/polly/test/ForwardOpTree/noforward_phi.ll b/polly/test/ForwardOpTree/noforward_phi.ll
index 58d41a410d3b..455edfd1a831 100644
--- a/polly/test/ForwardOpTree/noforward_phi.ll
+++ b/polly/test/ForwardOpTree/noforward_phi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Do not move PHI nodes.
 ;
diff --git a/polly/test/ForwardOpTree/noforward_selfrefphi.ll b/polly/test/ForwardOpTree/noforward_selfrefphi.ll
index b2d4dc51c978..e7ab21b6ba28 100644
--- a/polly/test/ForwardOpTree/noforward_selfrefphi.ll
+++ b/polly/test/ForwardOpTree/noforward_selfrefphi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-optree-normalize-phi=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-optree-normalize-phi=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Contains a self-referencing PHINode that would require a
 ; transitive closure to handle.
diff --git a/polly/test/ForwardOpTree/noforward_sideffects.ll b/polly/test/ForwardOpTree/noforward_sideffects.ll
index a5633769f670..0298da90e4ac 100644
--- a/polly/test/ForwardOpTree/noforward_sideffects.ll
+++ b/polly/test/ForwardOpTree/noforward_sideffects.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly  -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly  '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Do not forward instructions with side-effects (here: function call).
 ;
diff --git a/polly/test/ForwardOpTree/noforward_synthesizable_unknownit.ll b/polly/test/ForwardOpTree/noforward_synthesizable_unknownit.ll
index f589fde6e415..159972345bdc 100644
--- a/polly/test/ForwardOpTree/noforward_synthesizable_unknownit.ll
+++ b/polly/test/ForwardOpTree/noforward_synthesizable_unknownit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Do not try to forward %i.trunc, it is not synthesizable in %body.
 ;
diff --git a/polly/test/ForwardOpTree/out-of-quota1.ll b/polly/test/ForwardOpTree/out-of-quota1.ll
index 7afdb8e60244..5a69f6c3ad60 100644
--- a/polly/test/ForwardOpTree/out-of-quota1.ll
+++ b/polly/test/ForwardOpTree/out-of-quota1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-optree -disable-output %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output %s | FileCheck %s
 
 ; This used to loop infinitely because of UINT_MAX returned by ISL on out-of-quota.
 
diff --git a/polly/test/IstAstInfo/alias_checks_with_empty_context.ll b/polly/test/IstAstInfo/alias_checks_with_empty_context.ll
index 9b95cd5b4bbd..d64f7529135e 100644
--- a/polly/test/IstAstInfo/alias_checks_with_empty_context.ll
+++ b/polly/test/IstAstInfo/alias_checks_with_empty_context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s \
 ; RUN:     | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/IstAstInfo/alias_simple_1.ll b/polly/test/IstAstInfo/alias_simple_1.ll
index 83d470c2d19b..659c17879ed5 100644
--- a/polly/test/IstAstInfo/alias_simple_1.ll
+++ b/polly/test/IstAstInfo/alias_simple_1.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -disable-output < %s | FileCheck %s --check-prefix=NOAA
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=BASI
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline= -disable-output < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s --check-prefix=BASI
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
 ;
 ;    int A[1024];
 ;
diff --git a/polly/test/IstAstInfo/alias_simple_2.ll b/polly/test/IstAstInfo/alias_simple_2.ll
index bbf528f93b47..569fe45e1e02 100644
--- a/polly/test/IstAstInfo/alias_simple_2.ll
+++ b/polly/test/IstAstInfo/alias_simple_2.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -disable-output < %s | FileCheck %s --check-prefix=NOAA
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=BASI
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -globals-aa -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline= -disable-output < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s --check-prefix=BASI
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=globals-aa -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
 ;
 ;    int A[1024], B[1024];
 ;
diff --git a/polly/test/IstAstInfo/alias_simple_3.ll b/polly/test/IstAstInfo/alias_simple_3.ll
index 9067521323ab..8bad170eda79 100644
--- a/polly/test/IstAstInfo/alias_simple_3.ll
+++ b/polly/test/IstAstInfo/alias_simple_3.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -disable-output < %s | FileCheck %s --check-prefix=NOAA
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=BASI
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
-; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline= -disable-output < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s --check-prefix=BASI
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
+; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
 ;
 ;    int A[1024];
 ;    float B[1024];
diff --git a/polly/test/IstAstInfo/aliasing_arrays_with_identical_base.ll b/polly/test/IstAstInfo/aliasing_arrays_with_identical_base.ll
index 0cabd20168ba..2bf71fb8fd2c 100644
--- a/polly/test/IstAstInfo/aliasing_arrays_with_identical_base.ll
+++ b/polly/test/IstAstInfo/aliasing_arrays_with_identical_base.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s \
 ; RUN:   -polly-invariant-load-hoisting \
 ; RUN:   | FileCheck %s
 
diff --git a/polly/test/IstAstInfo/aliasing_multiple_alias_groups.ll b/polly/test/IstAstInfo/aliasing_multiple_alias_groups.ll
index b824c211fd31..6cdc5b0fdcec 100644
--- a/polly/test/IstAstInfo/aliasing_multiple_alias_groups.ll
+++ b/polly/test/IstAstInfo/aliasing_multiple_alias_groups.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-ast       -disable-output < %s | FileCheck %s --check-prefix=NOAA
-; RUN: opt %loadPolly -polly-print-ast -tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -aa-pipeline=      -disable-output < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -aa-pipeline=tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
 ;
 ;    void jd(int *Int0, int *Int1, float *Float0, float *Float1) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/IstAstInfo/aliasing_parametric_simple_1.ll b/polly/test/IstAstInfo/aliasing_parametric_simple_1.ll
index e0c3255dd766..a63854d94b68 100644
--- a/polly/test/IstAstInfo/aliasing_parametric_simple_1.ll
+++ b/polly/test/IstAstInfo/aliasing_parametric_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output %s | FileCheck %s
 ;
 ;    void jd(int *A, int *B, int c) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/IstAstInfo/aliasing_parametric_simple_2.ll b/polly/test/IstAstInfo/aliasing_parametric_simple_2.ll
index 74bad6c75784..7b2d163d54a2 100644
--- a/polly/test/IstAstInfo/aliasing_parametric_simple_2.ll
+++ b/polly/test/IstAstInfo/aliasing_parametric_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 ;
 ;    void jd(int *A, int *B, int c) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/IstAstInfo/dependence_distance_minimal.ll b/polly/test/IstAstInfo/dependence_distance_minimal.ll
index c6b1d156e55d..4a77123e5031 100644
--- a/polly/test/IstAstInfo/dependence_distance_minimal.ll
+++ b/polly/test/IstAstInfo/dependence_distance_minimal.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; The minimal dependence distance of the innermost loop should be 1 instead of 250.
 ; CHECK:    #pragma minimal dependence distance: 1
diff --git a/polly/test/IstAstInfo/domain_bounded_only_with_context.ll b/polly/test/IstAstInfo/domain_bounded_only_with_context.ll
index 32cebd7a3a8b..bcf6fd394209 100644
--- a/polly/test/IstAstInfo/domain_bounded_only_with_context.ll
+++ b/polly/test/IstAstInfo/domain_bounded_only_with_context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ; CHECK:      {
 ; CHECK-NEXT:    if (p <= -1 || p >= 1)
diff --git a/polly/test/IstAstInfo/non_affine_access.ll b/polly/test/IstAstInfo/non_affine_access.ll
index d8757b2e21cf..b3f669ee4670 100644
--- a/polly/test/IstAstInfo/non_affine_access.ll
+++ b/polly/test/IstAstInfo/non_affine_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-ast-print-accesses -polly-allow-nonaffine -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-print-accesses -polly-allow-nonaffine -disable-output < %s | FileCheck %s
 ;
 ;    void non_affine_access(float A[]) {
 ;      for (long i = 0; i < 1024; i++)
diff --git a/polly/test/IstAstInfo/reduction_clauses_onedimensional_access.ll b/polly/test/IstAstInfo/reduction_clauses_onedimensional_access.ll
index 8d52e345a76d..ea47e63bb7a6 100644
--- a/polly/test/IstAstInfo/reduction_clauses_onedimensional_access.ll
+++ b/polly/test/IstAstInfo/reduction_clauses_onedimensional_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK: #pragma known-parallel reduction (^ : MemRef_sum)
 ;        void f(int N, int M, int *sum) {
diff --git a/polly/test/IstAstInfo/reduction_dependences_equal_non_reduction_dependences.ll b/polly/test/IstAstInfo/reduction_dependences_equal_non_reduction_dependences.ll
index 9c6eea6aaa1e..6650fc034f87 100644
--- a/polly/test/IstAstInfo/reduction_dependences_equal_non_reduction_dependences.ll
+++ b/polly/test/IstAstInfo/reduction_dependences_equal_non_reduction_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; This loopnest contains a reduction which imposes the same dependences as the
 ; accesses to the array A. We need to ensure we do __not__ parallelize anything
diff --git a/polly/test/IstAstInfo/reduction_different_reduction_clauses.ll b/polly/test/IstAstInfo/reduction_different_reduction_clauses.ll
index 5104f716d810..d89953cdd2fb 100644
--- a/polly/test/IstAstInfo/reduction_different_reduction_clauses.ll
+++ b/polly/test/IstAstInfo/reduction_different_reduction_clauses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK: #pragma simd reduction (+ : MemRef_sum{{[1,2]}}, MemRef_sum{{[1,2]}}) reduction (* : MemRef_prod) reduction (| : MemRef_or) reduction (& : MemRef_and)
 ; CHECK: #pragma known-parallel reduction (+ : MemRef_sum{{[1,2]}}, MemRef_sum{{[1,2]}}) reduction (* : MemRef_prod) reduction (| : MemRef_or) reduction (& : MemRef_and)
diff --git a/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule.ll b/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule.ll
index 8a42cf8bd165..a8ce13b4c56d 100644
--- a/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK:          #pragma known-parallel reduction (+ : MemRef_A)
 ; CHECK-NEXT:     for (int c0 = 0; c0 <= 2; c0 += 1) {
diff --git a/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule_2.ll b/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule_2.ll
index 8f5efd165546..535ec397969d 100644
--- a/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule_2.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    #pragma known-parallel reduction
 ; CHECK:    for (int c0 = 0; c0 <= 2; c0 += 1) {
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions.ll
index a711a36a367f..0ea3916fb274 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    #pragma known-parallel
 ; CHECK:    for (int c0 = 0; c0 <= 1; c0 += 1)
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll
index 485d6965b6d3..703b0f3e04ea 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; Verify that the outer dimension doesnt't carry reduction dependences
 ;
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll
index 375fabbf6a8b..3a847e939b5b 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; Verify that the outer dimension doesnt't carry reduction dependences
 ;
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll
index 584c076dcff4..f2691bc6a503 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; Verify that the outer dimension doesnt't carry reduction dependences
 ;
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_5.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_5.ll
index eaa3444a04d7..480af5505a33 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_5.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; Verify that only the outer dimension needs privatization
 ;
diff --git a/polly/test/IstAstInfo/reduction_multiple_dimensions.ll b/polly/test/IstAstInfo/reduction_multiple_dimensions.ll
index 9618ec872c38..96ecb0c078fe 100644
--- a/polly/test/IstAstInfo/reduction_multiple_dimensions.ll
+++ b/polly/test/IstAstInfo/reduction_multiple_dimensions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT:#pragma known-parallel reduction
 ; CHECK:    #pragma known-parallel
diff --git a/polly/test/IstAstInfo/reduction_multiple_dimensions_2.ll b/polly/test/IstAstInfo/reduction_multiple_dimensions_2.ll
index af317570eb37..d2232ca88143 100644
--- a/polly/test/IstAstInfo/reduction_multiple_dimensions_2.ll
+++ b/polly/test/IstAstInfo/reduction_multiple_dimensions_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT:#pragma known-parallel reduction
 ; CHECK:    #pragma known-parallel
diff --git a/polly/test/IstAstInfo/reduction_multiple_dimensions_3.ll b/polly/test/IstAstInfo/reduction_multiple_dimensions_3.ll
index 1f7191433bf8..dfc1682bd95b 100644
--- a/polly/test/IstAstInfo/reduction_multiple_dimensions_3.ll
+++ b/polly/test/IstAstInfo/reduction_multiple_dimensions_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT:#pragma known-parallel reduction
 ; CHECK:    #pragma known-parallel
diff --git a/polly/test/IstAstInfo/reduction_multiple_dimensions_4.ll b/polly/test/IstAstInfo/reduction_multiple_dimensions_4.ll
index 40bae5e9ac6c..d0bad81efb2f 100644
--- a/polly/test/IstAstInfo/reduction_multiple_dimensions_4.ll
+++ b/polly/test/IstAstInfo/reduction_multiple_dimensions_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT:#pragma known-parallel reduction
 ; CHECK:    #pragma known-parallel
diff --git a/polly/test/IstAstInfo/run-time-condition.ll b/polly/test/IstAstInfo/run-time-condition.ll
index ccc9c7cfd321..c3ea8c460b6d 100644
--- a/polly/test/IstAstInfo/run-time-condition.ll
+++ b/polly/test/IstAstInfo/run-time-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ; for (i = 0; i < 1024; i++)
 ;   A[i] = B[i];
diff --git a/polly/test/IstAstInfo/runtime_context_with_error_blocks.ll b/polly/test/IstAstInfo/runtime_context_with_error_blocks.ll
index 2853e0acf9b8..26a242b00ee7 100644
--- a/polly/test/IstAstInfo/runtime_context_with_error_blocks.ll
+++ b/polly/test/IstAstInfo/runtime_context_with_error_blocks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; Verify we do not simplify the runtime check to "true" due to the domain
 ; constraints as the test contains an error block that influenced the domains
diff --git a/polly/test/IstAstInfo/simple-run-time-condition.ll b/polly/test/IstAstInfo/simple-run-time-condition.ll
index 5fb99f0676b7..c6a6f027652f 100644
--- a/polly/test/IstAstInfo/simple-run-time-condition.ll
+++ b/polly/test/IstAstInfo/simple-run-time-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-precise-inbounds -polly-precise-fold-accesses -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-precise-inbounds -polly-precise-fold-accesses -disable-output < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/polly/test/IstAstInfo/single_loop_strip_mine.ll b/polly/test/IstAstInfo/single_loop_strip_mine.ll
index 1c627f817b0b..4405ee1dc138 100644
--- a/polly/test/IstAstInfo/single_loop_strip_mine.ll
+++ b/polly/test/IstAstInfo/single_loop_strip_mine.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -basic-aa -polly-import-jscop -polly-ast-print-accesses -polly-ast-detect-parallel -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=CHECK-VECTOR
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-ast-print-accesses -polly-ast-detect-parallel '-passes=polly-import-jscop,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=CHECK-VECTOR
 
 ; for (i = 0; i < 1024; i++)
 ;   A[i] = B[i];
diff --git a/polly/test/IstAstInfo/single_loop_uint_max_iterations.ll b/polly/test/IstAstInfo/single_loop_uint_max_iterations.ll
index f1cd5dae11ce..3f673584a268 100644
--- a/polly/test/IstAstInfo/single_loop_uint_max_iterations.ll
+++ b/polly/test/IstAstInfo/single_loop_uint_max_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 ; XFAIL: *
 
 ;#include "limits.h"
diff --git a/polly/test/IstAstInfo/single_loop_ull_max_iterations.ll b/polly/test/IstAstInfo/single_loop_ull_max_iterations.ll
index d421e221240a..ff56cfb11780 100644
--- a/polly/test/IstAstInfo/single_loop_ull_max_iterations.ll
+++ b/polly/test/IstAstInfo/single_loop_ull_max_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
 ; XFAIL: *
 
 ;#include "limits.h"
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Bad-relation.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Bad-relation.ll
index d4a1a6222518..545a88909bf6 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Bad-relation.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Bad-relation.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: expecting other token
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-No-accesses-key.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-No-accesses-key.ll
index 43f9d3eda049..f9f13b4cff58 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-No-accesses-key.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-No-accesses-key.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: Statement from JScop file has no key name 'accesses' for index 1.
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-MemAcc.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-MemAcc.ll
index 24ad03741216..6031465cd03f 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-MemAcc.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-MemAcc.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: The number of memory accesses in the JSop file and the number of memory accesses differ for index 0.
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-statements.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-statements.ll
index 1060926e7fac..fc67ec51f218 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-statements.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-statements.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: The number of indices and the number of statements differ.
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Relation-mispelled.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Relation-mispelled.ll
index 07975976c38b..64b7b14c1939 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Relation-mispelled.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Relation-mispelled.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: Memory access number 0 has no key name 'relation' for statement number 1.
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Statements-mispelled.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Statements-mispelled.ll
index 9f7259633811..bd58cb3be5c3 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Statements-mispelled.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Statements-mispelled.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: JScop file has no key name 'statements'.
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Undeclared-ScopArrayInfo.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Undeclared-ScopArrayInfo.ll
index df7eb42da85f..565117584861 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Undeclared-ScopArrayInfo.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Undeclared-ScopArrayInfo.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: JScop file contains access function with undeclared ScopArrayInfo
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Wrong-number-dimensions.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Wrong-number-dimensions.ll
index 61c1173db2e7..4a7d51019133 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Wrong-number-dimensions.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Wrong-number-dimensions.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: JScop file changes the number of parameter dimensions.
 ;
diff --git a/polly/test/JSONExporter/ImportArrays/ImportArrays-Mispelled-type.ll b/polly/test/JSONExporter/ImportArrays/ImportArrays-Mispelled-type.ll
index a14ae5c4d1bc..1dd71ba5816a 100644
--- a/polly/test/JSONExporter/ImportArrays/ImportArrays-Mispelled-type.ll
+++ b/polly/test/JSONExporter/ImportArrays/ImportArrays-Mispelled-type.ll
@@ -1,4 +1,4 @@
-  ; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
+  ; RUN: not --crash opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Array has not a valid type.
 ;
diff --git a/polly/test/JSONExporter/ImportArrays/ImportArrays-Negative-size.ll b/polly/test/JSONExporter/ImportArrays/ImportArrays-Negative-size.ll
index 2a03197f1c1b..5270cc8c680f 100644
--- a/polly/test/JSONExporter/ImportArrays/ImportArrays-Negative-size.ll
+++ b/polly/test/JSONExporter/ImportArrays/ImportArrays-Negative-size.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-stmt-granularity=bb -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; #define Ni 1056
 ; #define Nj 1056
diff --git a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-name.ll b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-name.ll
index 45bb3495de08..21c4a736a25e 100644
--- a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-name.ll
+++ b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-name.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
+; RUN: not --crash opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Array has no key 'name'.
 ;
diff --git a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-sizes-key.ll b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-sizes-key.ll
index 5bbb974346ba..930bfce45df0 100644
--- a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-sizes-key.ll
+++ b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-sizes-key.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
+; RUN: not --crash opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Array has no key 'sizes'.
 ;
diff --git a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-type-key.ll b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-type-key.ll
index af013992fca0..eb13390dd2b9 100644
--- a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-type-key.ll
+++ b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-type-key.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
+; RUN: not --crash opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Array has no key 'type'.
 ;
diff --git a/polly/test/JSONExporter/ImportContext/ImportContext-Context-mispelled.ll b/polly/test/JSONExporter/ImportContext/ImportContext-Context-mispelled.ll
index 2490e44ec347..79e175378af6 100644
--- a/polly/test/JSONExporter/ImportContext/ImportContext-Context-mispelled.ll
+++ b/polly/test/JSONExporter/ImportContext/ImportContext-Context-mispelled.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: JScop file has no key named 'context'.
 ;
diff --git a/polly/test/JSONExporter/ImportContext/ImportContext-Not-parameter-set.ll b/polly/test/JSONExporter/ImportContext/ImportContext-Not-parameter-set.ll
index 66ce6a6ed922..96539188cdff 100644
--- a/polly/test/JSONExporter/ImportContext/ImportContext-Not-parameter-set.ll
+++ b/polly/test/JSONExporter/ImportContext/ImportContext-Not-parameter-set.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: The isl_set is not a parameter set.
 ;
diff --git a/polly/test/JSONExporter/ImportContext/ImportContext-Unvalid-Context.ll b/polly/test/JSONExporter/ImportContext/ImportContext-Unvalid-Context.ll
index 7bcc54dde52e..0e80d3623c10 100644
--- a/polly/test/JSONExporter/ImportContext/ImportContext-Unvalid-Context.ll
+++ b/polly/test/JSONExporter/ImportContext/ImportContext-Unvalid-Context.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: unexpected isl_token
 ;
diff --git a/polly/test/JSONExporter/ImportContext/ImportContext-Wrong-dimension.ll b/polly/test/JSONExporter/ImportContext/ImportContext-Wrong-dimension.ll
index 65cdcbdcdef6..9a04d89b18d1 100644
--- a/polly/test/JSONExporter/ImportContext/ImportContext-Wrong-dimension.ll
+++ b/polly/test/JSONExporter/ImportContext/ImportContext-Wrong-dimension.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: Imported context has the wrong number of parameters : Found 2 Expected 1
 ;
diff --git a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-No-schedule-key.ll b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-No-schedule-key.ll
index b52db0876cc5..6347d17da4cc 100644
--- a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-No-schedule-key.ll
+++ b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-No-schedule-key.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: Statement 0 has no 'schedule' key.
 ;
diff --git a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Schedule-not-valid.ll b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Schedule-not-valid.ll
index 5ce3ad267bb0..b6f4d188fad0 100644
--- a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Schedule-not-valid.ll
+++ b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Schedule-not-valid.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: expecting other token
 ;
diff --git a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Statements-mispelled.ll b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Statements-mispelled.ll
index 4329653899b2..9c325f9bfb77 100644
--- a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Statements-mispelled.ll
+++ b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Statements-mispelled.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: JScop file has no key name 'statements'.
 ;
diff --git a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Wrong-number-statements.ll b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Wrong-number-statements.ll
index f66fc6c1e5d7..a6a3c7d35668 100644
--- a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Wrong-number-statements.ll
+++ b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Wrong-number-statements.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: The number of indices and the number of statements differ.
 ;
diff --git a/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll b/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll
index 791210f7710d..a3611f0b89f3 100644
--- a/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll
+++ b/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll
@@ -1,7 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1| FileCheck %s --check-prefix=MSE
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that the expansion of an array with load after store in a same statement is not done.
 ;
diff --git a/polly/test/MaximalStaticExpansion/read_from_original.ll b/polly/test/MaximalStaticExpansion/read_from_original.ll
index 59f9379516c7..fe9c5850bc5f 100644
--- a/polly/test/MaximalStaticExpansion/read_from_original.ll
+++ b/polly/test/MaximalStaticExpansion/read_from_original.ll
@@ -1,7 +1,5 @@
-; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1| FileCheck %s --check-prefix=MSE
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that Polly detects problems and does not expand the array
 ;
diff --git a/polly/test/MaximalStaticExpansion/too_many_writes.ll b/polly/test/MaximalStaticExpansion/too_many_writes.ll
index 50a66cd11d0a..01cdd2b46682 100644
--- a/polly/test/MaximalStaticExpansion/too_many_writes.ll
+++ b/polly/test/MaximalStaticExpansion/too_many_writes.ll
@@ -1,7 +1,5 @@
-; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that Polly detects problems and does not expand the array
 ;
diff --git a/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll b/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll
index 8e2707cfee64..3b9f951be2b4 100644
--- a/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll
+++ b/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::Array
 ;
diff --git a/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll b/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll
index 2bf49b89db05..32ed2b99bde2 100644
--- a/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll
+++ b/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll
@@ -1,7 +1,5 @@
-; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::Array and MemoryKind::PHI.
 ; tmp_06_phi is not expanded because it need copy in.
diff --git a/polly/test/MaximalStaticExpansion/working_expansion.ll b/polly/test/MaximalStaticExpansion/working_expansion.ll
index bb5b2360143f..29ac90174f88 100644
--- a/polly/test/MaximalStaticExpansion/working_expansion.ll
+++ b/polly/test/MaximalStaticExpansion/working_expansion.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::Array
 ;
diff --git a/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll b/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll
index 89ff7890fc7e..6ef2f298adb9 100644
--- a/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll
+++ b/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
 ;
 ; Verify that the accesses are correctly expanded
 ;
diff --git a/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll b/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll
index 7ffd39f0f534..6c7ea23d2f53 100644
--- a/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll
+++ b/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
 ;
 ; Verify that the accesses are correctly expanded
 ;
diff --git a/polly/test/MaximalStaticExpansion/working_phi_expansion.ll b/polly/test/MaximalStaticExpansion/working_phi_expansion.ll
index 43919c61b045..0d4f18f21ade 100644
--- a/polly/test/MaximalStaticExpansion/working_phi_expansion.ll
+++ b/polly/test/MaximalStaticExpansion/working_phi_expansion.ll
@@ -1,7 +1,5 @@
-; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::PHI
 ; tmp_04 is not expanded because it need copy-in.
diff --git a/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll b/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll
index a581a389e742..93e984b95c95 100644
--- a/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll
+++ b/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll
@@ -1,7 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::PHI
 ; tmp_05 and tmp2_06 are not expanded because they need copy-in.
diff --git a/polly/test/MaximalStaticExpansion/working_value_expansion.ll b/polly/test/MaximalStaticExpansion/working_value_expansion.ll
index d54eff9e03ec..27c4304d2fe5 100644
--- a/polly/test/MaximalStaticExpansion/working_value_expansion.ll
+++ b/polly/test/MaximalStaticExpansion/working_value_expansion.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::Value
 ;
diff --git a/polly/test/PruneUnprofitable/prune_only_scalardeps.ll b/polly/test/PruneUnprofitable/prune_only_scalardeps.ll
index 31db5560c051..c64512fc3d7a 100644
--- a/polly/test/PruneUnprofitable/prune_only_scalardeps.ll
+++ b/polly/test/PruneUnprofitable/prune_only_scalardeps.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=false -polly-prune-unprofitable -disable-output -stats < %s 2>&1 | FileCheck -match-full-lines %s
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=false "-passes=scop(polly-prune-unprofitable)" -disable-output -stats < %s 2>&1 | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=false "-passes=scop(polly-prune-unprofitable)" -disable-output -stats < %s 2>&1 | FileCheck -match-full-lines %s
 ; REQUIRES: asserts
 ;
 ; Skip this SCoP for having scalar dependencies between all statements,
diff --git a/polly/test/ScheduleOptimizer/2012-03-16-Empty-Domain.ll b/polly/test/ScheduleOptimizer/2012-03-16-Empty-Domain.ll
index 5acc35343ac3..b9baa35b0cf8 100644
--- a/polly/test/ScheduleOptimizer/2012-03-16-Empty-Domain.ll
+++ b/polly/test/ScheduleOptimizer/2012-03-16-Empty-Domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -S < %s
+; RUN: opt %loadPolly -passes=polly-opt-isl -S < %s
 target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
 
 define void @sdbout_label() nounwind {
diff --git a/polly/test/ScheduleOptimizer/2013-04-11-Empty-Domain-two.ll b/polly/test/ScheduleOptimizer/2013-04-11-Empty-Domain-two.ll
index 3f4237b330b2..e3e1d61f74c1 100644
--- a/polly/test/ScheduleOptimizer/2013-04-11-Empty-Domain-two.ll
+++ b/polly/test/ScheduleOptimizer/2013-04-11-Empty-Domain-two.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -S < %s
+; RUN: opt %loadPolly -passes=polly-opt-isl -S < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Check that we handle statements with an empty iteration domain correctly.
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-double.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-double.ll
index a61af2d092f3..b1acd130acb6 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-double.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-double.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
 
 define void @func(i32 %n, ptr  noalias nonnull %A,  ptr  noalias nonnull %B) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-first.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-first.ll
index 185d5c5b8c25..4590b01f6112 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-first.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-first.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,OPT
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,OPT
 
 define void @func(i32 %n, ptr noalias nonnull %A, ptr noalias nonnull %B, i32 %k) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-third.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-third.ll
index f1eca0ede061..a9afceac3880 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-third.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-third.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK
 
 define void @func(i32 %n, ptr noalias nonnull %A, ptr noalias nonnull %B, i32 %k) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-carried.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-carried.ll
index 35903ced7741..16dc6d8bc673 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-carried.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-carried.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,OPT
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,OPT
 
 define void @func(i32 %n, ptr noalias nonnull %A) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-third.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-third.ll
index 1fb8c001069f..54611c7a1cb8 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-third.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-third.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK
 
 define void @func(i32 %n, ptr noalias nonnull %A, ptr noalias nonnull %B, i32 %k) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner.ll
index 2db6833fa897..905a7804215f 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
 
 define void @func(i32 %n, ptr noalias nonnull %A) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-simple.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-simple.ll
index 49d008ba2cfa..32be9181c66b 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-simple.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-simple.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
 
 define void @func(i32 %n, ptr noalias nonnull %A) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-simple.ll b/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-simple.ll
index 175b85997ec0..d15d69e69fac 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-simple.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-simple.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
 
 ; This could theoretically be fused by adjusting the offset of the second loop by %k (instead of relying on schedule dimensions).
 
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-with-middle.ll b/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-with-middle.ll
index 48ba20347d55..5dc5330db107 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-with-middle.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-with-middle.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
 
 define void @func(i32 %n, ptr noalias nonnull %A, ptr noalias nonnull %B, i32 %k) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/disable_nonforced.ll b/polly/test/ScheduleOptimizer/ManualOptimization/disable_nonforced.ll
index 537721f8718a..8676b2dd7e18 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/disable_nonforced.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/disable_nonforced.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-opt-isl -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Check that the disable_nonforced metadata is honored; optimization
 ; heuristics/rescheduling must not be applied.
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_heuristic.ll b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_heuristic.ll
index aaf4d27f4c5e..90dc071ae8f6 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_heuristic.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_heuristic.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-pragma-based-opts=1 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines --check-prefix=ON
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-pragma-based-opts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines --check-prefix=OFF
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-pragma-based-opts=1 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines --check-prefix=ON
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-pragma-based-opts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines --check-prefix=OFF
 ;
 define void @func(i32 %n, ptr noalias nonnull %A, ptr noalias nonnull %B) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_looploc.ll b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_looploc.ll
index b1e94227c9a5..f513835c60eb 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_looploc.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_looploc.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-reschedule=0 -polly-pragma-based-opts=1 -disable-output < %s 2>&1 | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-reschedule=0 -polly-pragma-based-opts=1 -disable-output < %s 2>&1 | FileCheck %s --match-full-lines
 ;
 ; CHECK: warning: distribute_illegal.c:2:3: not applying loop fission/distribution: cannot ensure semantic equivalence due to possible dependency violations
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_pragmaloc.ll b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_pragmaloc.ll
index fc0df85b1346..c18b9bb72ad7 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_pragmaloc.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_pragmaloc.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-reschedule=0 -polly-pragma-based-opts=1 -disable-output < %s 2>&1 | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-reschedule=0 -polly-pragma-based-opts=1 -disable-output < %s 2>&1 | FileCheck %s --match-full-lines
 ;
 ; CHECK: warning: distribute_illegal.c:1:42: not applying loop fission/distribution: cannot ensure semantic equivalence due to possible dependency violations
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_disable.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_disable.ll
index 9537f3a9b0a8..20f5a4538b16 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_disable.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_disable.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-pragma-based-opts=1 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-pragma-based-opts=1 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
 ;
 ; Override unroll metadata with llvm.loop.unroll.disable.
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll
index b0310970f8d6..afc39a72da1d 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
 ;
 ; Apply two loop transformations. First partial, then full unrolling.
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_full.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_full.ll
index b9a4c845477c..a166421ca21a 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_full.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_full.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
 ;
 ; Full unroll of a loop with 5 iterations.
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_heuristic.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_heuristic.ll
index 0387aecd683b..68ee147895cf 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_heuristic.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_heuristic.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-pragma-based-opts=1 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
-; RUN: opt %loadPolly -polly-pragma-based-opts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-pragma-based-opts=1 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-pragma-based-opts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
 ;
 ; Unrolling with heuristic factor.
 ; Currently not supported and expected to be handled by LLVM's unroll pass.
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial.ll
index 81e40f0a98bb..042e1b4e088a 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-pragma-based-opts=1 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
-; RUN: opt %loadPolly -polly-pragma-based-opts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefix=OFF --match-full-lines
+; RUN: opt %loadPolly -polly-pragma-based-opts=1 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-pragma-based-opts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefix=OFF --match-full-lines
 ;
 ; Partial unroll by a factor of 4.
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial_followup.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial_followup.ll
index 8665f68b99c1..893d40a41777 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial_followup.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial_followup.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefix=OPT --match-full-lines
-; RUN: opt %loadPolly -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST --match-full-lines
-; RUN: opt %loadPolly -polly-opt-isl -polly-codegen -simplifycfg -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefix=OPT --match-full-lines
+; RUN: opt %loadPolly '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=AST --match-full-lines
+; RUN: opt %loadPolly '-passes=scop(polly-opt-isl,polly-codegen),simplifycfg' -S < %s | FileCheck %s --check-prefix=CODEGEN
 ;
 ; Partial unroll by a factor of 4.
 ;
@@ -49,7 +49,7 @@ return:
 ; OPT-NEXT:        - filter: "[n] -> { Stmt_body[i0] : (1 + i0) mod 4 = 0 }"
 
 
-; AST-LABEL: Printing analysis 'Polly - Generate an AST of the SCoP (isl)'for => return' in function 'func':
+; AST-LABEL: :: isl ast :: func :: %for---%return
 ; AST:       // Loop with Metadata
 ; AST-NEXT:  for (int c0 = 0; c0 < n; c0 += 4) {
 
diff --git a/polly/test/ScheduleOptimizer/SIMDInParallelFor.ll b/polly/test/ScheduleOptimizer/SIMDInParallelFor.ll
index 8585634e10ff..b0385d50c6c6 100644
--- a/polly/test/ScheduleOptimizer/SIMDInParallelFor.ll
+++ b/polly/test/ScheduleOptimizer/SIMDInParallelFor.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-parallel -polly-vectorizer=stripmine -polly-codegen-verify -polly-opt-isl -polly-print-ast -polly-codegen -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-parallel -polly-vectorizer=stripmine -passes=polly-codegen-verify '-passes=polly-opt-isl,print<polly-ast>,polly-codegen' -disable-output < %s | FileCheck %s
 ;
 ; Check that there are no nested #pragma omp parallel for inside a
 ; #pragma omp parallel for loop.
diff --git a/polly/test/ScheduleOptimizer/computeout.ll b/polly/test/ScheduleOptimizer/computeout.ll
index 35e3416f91d1..1cf6513e7a5c 100644
--- a/polly/test/ScheduleOptimizer/computeout.ll
+++ b/polly/test/ScheduleOptimizer/computeout.ll
@@ -1,7 +1,5 @@
-; RUN: opt -S %loadPolly -basic-aa -polly-opt-isl -polly-isl-arg=--no-schedule-serialize-sccs -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadNPMPolly "-passes=scop(polly-opt-isl,print<polly-ast>)" -polly-isl-arg=--no-schedule-serialize-sccs -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadPolly -basic-aa -polly-opt-isl -polly-isl-arg=--schedule-serialize-sccs -polly-dependences-computeout=1 -polly-print-ast -disable-output  < %s | FileCheck %s -check-prefix=TIMEOUT
-; RUN: opt -S %loadNPMPolly "-passes=scop(polly-opt-isl,print<polly-ast>)" -polly-isl-arg=--no-schedule-serialize-sccs -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
+; RUN: opt -S %loadPolly "-passes=scop(polly-opt-isl,print<polly-ast>)" -polly-isl-arg=--no-schedule-serialize-sccs -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly "-passes=scop(polly-opt-isl,print<polly-ast>)" -polly-isl-arg=--no-schedule-serialize-sccs -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;     for(i = 0; i < 100; i++ )
diff --git a/polly/test/ScheduleOptimizer/ensure-correct-tile-sizes.ll b/polly/test/ScheduleOptimizer/ensure-correct-tile-sizes.ll
index 43caca5372ad..87f6c6c4eee6 100644
--- a/polly/test/ScheduleOptimizer/ensure-correct-tile-sizes.ll
+++ b/polly/test/ScheduleOptimizer/ensure-correct-tile-sizes.ll
@@ -1,9 +1,9 @@
 ; RUN: opt %loadPolly -polly-process-unprofitable -polly-remarks-minimal \
-; RUN:     -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN:     '-passes=polly-opt-isl,print<polly-ast>' -polly-pattern-matching-based-opts=true \
 ; RUN:     -polly-target-throughput-vector-fma=1 \
 ; RUN:     -polly-target-latency-vector-fma=1 \
 ; RUN:     -polly-target-vector-register-bitwidth=4096 \
-; RUN:     -polly-target-1st-cache-level-associativity=3 -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN:     -polly-target-1st-cache-level-associativity=3 -disable-output < %s | FileCheck %s
 ;
 ;     /* Test that Polly does not crash due to configurations that can lead to
 ;    incorrect tile size computations.
diff --git a/polly/test/ScheduleOptimizer/focaltech_test_detail_threshold-7bc17e.ll b/polly/test/ScheduleOptimizer/focaltech_test_detail_threshold-7bc17e.ll
index daa1afdd0aa8..483737ee5928 100644
--- a/polly/test/ScheduleOptimizer/focaltech_test_detail_threshold-7bc17e.ll
+++ b/polly/test/ScheduleOptimizer/focaltech_test_detail_threshold-7bc17e.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-vectorizer=stripmine -polly-invariant-load-hoisting -polly-optimized-scops -polly-print-opt-isl -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-opt-isl>)" -polly-vectorizer=stripmine -polly-invariant-load-hoisting -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly "-passes=scop(print<polly-opt-isl>)" -polly-vectorizer=stripmine -polly-invariant-load-hoisting -disable-output < %s | FileCheck %s
 ;
 ; llvm.org/PR46578
 ;
diff --git a/polly/test/ScheduleOptimizer/full_partial_tile_separation.ll b/polly/test/ScheduleOptimizer/full_partial_tile_separation.ll
index 06e86d7da1c6..9c4627717ee8 100644
--- a/polly/test/ScheduleOptimizer/full_partial_tile_separation.ll
+++ b/polly/test/ScheduleOptimizer/full_partial_tile_separation.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S %loadPolly -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 ; CHECK:          // 1st level tiling - Tiles
 ; CHECK-NEXT:    #pragma known-parallel
 ; CHECK-NEXT:    for (int c0 = 0; c0 <= floord(ni - 1, 32); c0 += 1)
diff --git a/polly/test/ScheduleOptimizer/line-tiling-2.ll b/polly/test/ScheduleOptimizer/line-tiling-2.ll
index eb374cb07cf3..d3d11a7990a6 100644
--- a/polly/test/ScheduleOptimizer/line-tiling-2.ll
+++ b/polly/test/ScheduleOptimizer/line-tiling-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-tile-sizes=1,64 -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-tile-sizes=1,64 '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ; CHECK: for (int c0 = 0; c0 <= 1023; c0 += 1)
 ; CHECK:   for (int c1 = 0; c1 <= 7; c1 += 1)
diff --git a/polly/test/ScheduleOptimizer/line-tiling.ll b/polly/test/ScheduleOptimizer/line-tiling.ll
index 2f14ac1d02a5..273a27ba5931 100644
--- a/polly/test/ScheduleOptimizer/line-tiling.ll
+++ b/polly/test/ScheduleOptimizer/line-tiling.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-tile-sizes=64,1 -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-tile-sizes=64,1 '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 
 ; CHECK: for (int c0 = 0; c0 <= 15; c0 += 1)
 ; CHECK:   for (int c1 = 0; c1 <= 511; c1 += 1)
diff --git a/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout.ll b/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout.ll
index faf51e097a70..69dfd383060e 100644
--- a/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout.ll
+++ b/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-target-throughput-vector-fma=1 \
 ; RUN: -polly-target-latency-vector-fma=8 \
 ; RUN: -polly-target-1st-cache-level-associativity=8 \
diff --git a/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll b/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll
index 30b693a2e241..fe1bc0518e2c 100644
--- a/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll
+++ b/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
-; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 ;
 ;    /* C := alpha*A*B + beta*C */
 ;    /* _PB_NK % Kc != 0 */
@@ -18,7 +18,7 @@
 ;	     C[i][j] += alpha * A[i][k] * B[k][j];
 ;        }
 ;
-; CHECK-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'bb8 => bb32' in function 'kernel_gemm':
+; CHECK-LABEL:     :: isl ast :: kernel_gemm :: %bb8---%bb32
 ; CHECK:    {
 ; CHECK-NEXT:      // 1st level tiling - Tiles
 ; CHECK-NEXT:      for (int c0 = 0; c0 <= 32; c0 += 1)
diff --git a/polly/test/ScheduleOptimizer/one-dimensional-band.ll b/polly/test/ScheduleOptimizer/one-dimensional-band.ll
index 4592907a44ad..594386662ef3 100644
--- a/polly/test/ScheduleOptimizer/one-dimensional-band.ll
+++ b/polly/test/ScheduleOptimizer/one-dimensional-band.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 ;
 ;    void jacobi1d(long T, long N, float *A, float *B) {
 ;      long t, i, j;
diff --git a/polly/test/ScheduleOptimizer/outer_coincidence.ll b/polly/test/ScheduleOptimizer/outer_coincidence.ll
index 2ab33edda86b..4a92f416fecc 100644
--- a/polly/test/ScheduleOptimizer/outer_coincidence.ll
+++ b/polly/test/ScheduleOptimizer/outer_coincidence.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-tiling=0 -polly-parallel -polly-opt-outer-coincidence=no  -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-tiling=0 -polly-parallel -polly-opt-outer-coincidence=yes -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=OUTER
+; RUN: opt %loadPolly -polly-tiling=0 -polly-parallel -polly-opt-outer-coincidence=no  '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-tiling=0 -polly-parallel -polly-opt-outer-coincidence=yes '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=OUTER
 
 ; By skewing, the diagonal can be made parallel. ISL does this when the Check
 ; the 'outer_coincidence' option is enabled.
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll
index 66011168fcc1..979df17632b2 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll
@@ -1,7 +1,7 @@
 ; RUN: opt %loadPolly \
 ; RUN: -polly-pattern-matching-based-opts=true \
-; RUN: -polly-optree -polly-delicm -polly-simplify \
-; RUN: -polly-opt-isl -polly-tc-opt=true -debug -disable-output < %s 2>&1 \
+; RUN: '-passes=polly-optree,polly-delicm,polly-simplify,polly-opt-isl' \
+; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 \
 ; RUN: | FileCheck %s
 ; REQUIRES: asserts
 
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll
index 95da89f90755..80cae8554fdb 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-delicm -polly-simplify -polly-opt-isl \
+; RUN: opt %loadPolly '-passes=polly-delicm,polly-simplify,polly-opt-isl' \
 ; RUN: -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts.ll
index 7604257f98e0..5e0bb81c5908 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=false \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=false \
 ; RUN: -debug -polly-tc-opt -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true -debug -polly-tc-opt -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PATTERN-MATCHING-OPTS
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true -polly-ast-detect-parallel -polly-print-ast -disable-output  < %s | FileCheck %s --check-prefix=PARALLEL-AST
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true -stats -disable-output < %s 2>&1 | FileCheck %s --check-prefix=STATS -match-full-lines
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true -debug -polly-tc-opt -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PATTERN-MATCHING-OPTS
+; RUN: opt %loadPolly '-passes=polly-opt-isl,print<polly-ast>' -polly-pattern-matching-based-opts=true -polly-ast-detect-parallel -disable-output  < %s | FileCheck %s --check-prefix=PARALLEL-AST
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true -stats -disable-output < %s 2>&1 | FileCheck %s --check-prefix=STATS -match-full-lines
 ; REQUIRES: asserts
 ;
 ;    /* C := alpha*A*B + beta*C */
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_11.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_11.ll
index ccdb39b60d75..5c4391693b13 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_11.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_11.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop \
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-opt-isl' \
 ; RUN: -polly-import-jscop-postfix=transformed \
 ; RUN: -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-target-throughput-vector-fma=1 \
@@ -8,7 +8,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-opt-isl -debug \
+; RUN: -debug \
 ; RUN: -polly-tc-opt=true -disable-output < %s 2>&1 \
 ; RUN: | FileCheck %s
 ; REQUIRES: asserts
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_12.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_12.ll
index dd39fec5e21f..b21a26b4772d 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_12.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_12.ll
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-opt-isl -disable-output < %s
+; RUN: -passes=polly-opt-isl -disable-output < %s
 ;
 ; Test whether isolation works as expected.
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_13.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_13.ll
index e086dd36c4d9..a16ecf6af6ce 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_13.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_13.ll
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=128 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 ;
 ; Test whether isolation works as expected.
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_14.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_14.ll
index a4c71c2dace5..2c23ebbac43c 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_14.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_14.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-opt-isl  \
+; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-opt-isl,polly-codegen'  \
 ; RUN: -polly-target-throughput-vector-fma=1 \
 ; RUN: -polly-target-latency-vector-fma=8 \
 ; RUN: -polly-target-1st-cache-level-associativity=8 \
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-import-jscop-postfix=transformed -polly-codegen -S < %s \
+; RUN: -polly-import-jscop-postfix=transformed -S < %s \
 ; RUN: | FileCheck %s
 ;
 ; Check that we disable the Loop Vectorizer.
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_15.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_15.ll
index a8da21955b63..c8d8d295e8c8 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_15.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_15.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -debug-only=polly-opt-isl -disable-output \
 ; RUN: -polly-tc-opt=true < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_16.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_16.ll
index c1ad3017a0d4..970b4a0cf932 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_16.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_16.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_17.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_17.ll
index 002816a4ae80..e44e3bfa04c2 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_17.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_17.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_18.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_18.ll
index d5679c7ae2f7..2612321b3d09 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_18.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_18.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_19.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_19.ll
index 4e1620abd252..bd5f0ed40953 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_19.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_19.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_2.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_2.ll
index 01e336ebc60f..573c35256992 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_2.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_20.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_20.ll
index 0be08d8d493c..78cc48d830c8 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_20.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_20.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_21.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_21.ll
index 9b2df49698a1..adfba0584a88 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_21.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_21.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_22.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_22.ll
index 3d3641df5098..54e03b301e69 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_22.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_22.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_24.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_24.ll
index 895961488014..fee5027848a8 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_24.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_24.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-opt-isl \
+; RUN: opt %loadPolly -polly-reschedule=0 -passes=polly-opt-isl \
 ; RUN: -polly-pattern-matching-based-opts=true -polly-tc-opt=true \
 ; RUN: -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_25.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_25.ll
index 8a3957909d9d..029dcc491f02 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_25.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_25.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
@@ -53,4 +53,4 @@ for.body8:                                        ; preds = %for.body8, %for.con
   br i1 %exitcond.not, label %for.cond.cleanup7, label %for.body8
 }
 
-declare double @llvm.fmuladd.f64(double, double, double)
\ No newline at end of file
+declare double @llvm.fmuladd.f64(double, double, double)
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll
index fab3ac5e58dc..46c8c7e35f2a 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll
@@ -3,7 +3,7 @@
 ; RUN: -polly-target-latency-vector-fma=8 \
 ; RUN: -polly-target-1st-cache-level-size=0 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
-; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s 2>&1 | FileCheck %s
+; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; RUN: opt %loadPolly -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-target-throughput-vector-fma=1 \
@@ -13,7 +13,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s 2>&1 | FileCheck %s --check-prefix=EXTRACTION-OF-MACRO-KERNEL
+; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=EXTRACTION-OF-MACRO-KERNEL
 ;
 ;    /* C := alpha*A*B + beta*C */
 ;    for (i = 0; i < _PB_NI; i++)
@@ -24,7 +24,7 @@
 ;	     C[i][j] += alpha * A[i][k] * B[k][j];
 ;        }
 ;
-; CHECK-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'bb8 => bb32' in function 'kernel_gemm':
+; CHECK-LABEL: :: isl ast :: kernel_gemm :: %bb8---%bb32 
 ; CHECK:    {
 ; CHECK-NEXT:      // 1st level tiling - Tiles
 ; CHECK-NEXT:      for (int c0 = 0; c0 <= 32; c0 += 1)
@@ -76,7 +76,7 @@
 ; CHECK-NEXT:          }
 ; CHECK-NEXT:    }
 ;
-; EXTRACTION-OF-MACRO-KERNEL-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'bb8 => bb32' in function 'kernel_gemm':
+; EXTRACTION-OF-MACRO-KERNEL-LABEL: :: isl ast :: kernel_gemm :: %bb8---%bb32 
 ; EXTRACTION-OF-MACRO-KERNEL:    {
 ; EXTRACTION-OF-MACRO-KERNEL-NEXT:      // 1st level tiling - Tiles
 ; EXTRACTION-OF-MACRO-KERNEL-NEXT:      for (int c0 = 0; c0 <= 32; c0 += 1)
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_4.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_4.ll
index dc0edc6c5a3b..ec1926ebb75f 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_4.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_4.ll
@@ -1,12 +1,12 @@
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -debug -polly-tc-opt=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly '-passes=polly-opt-isl,print<polly-ast>' -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-target-throughput-vector-fma=1 \
 ; RUN: -polly-target-latency-vector-fma=8 \
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
-; RUN: -polly-target-2nd-cache-level-size=262144 -polly-print-ast \
-; RUN: -polly-tc-opt=true -disable-output -polly-opt-isl < %s |  \
+; RUN: -polly-target-2nd-cache-level-size=262144 \
+; RUN: -polly-tc-opt=true -disable-output < %s |  \
 ; RUN: FileCheck %s --check-prefix=PATTERN-MATCHING-OPTS
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_5.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_5.ll
index 6581566bf13f..bfb378259210 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_5.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_5.ll
@@ -6,12 +6,12 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 ;
-; opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+; opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; -polly-target-throughput-vector-fma=1 \
 ; -polly-target-latency-vector-fma=8 \
-; -polly-codegen -polly-target-1st-cache-level-associativity=8 \
+; -passes=polly-codegen -polly-target-1st-cache-level-associativity=8 \
 ; -polly-target-2nd-cache-level-associativity=8 \
 ; -polly-target-1st-cache-level-size=32768 \
 ; -polly-target-vector-register-bitwidth=256 \
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_6.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_6.ll
index bcf1fc9fe813..684cd9be1728 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_6.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_6.ll
@@ -6,12 +6,12 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 ;
-;  opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
+;  opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
 ;  -polly-target-throughput-vector-fma=1 \
 ;  -polly-target-latency-vector-fma=8 \
-;  -polly-codegen -polly-target-1st-cache-level-associativity=8 \
+;  -passes=polly-codegen -polly-target-1st-cache-level-associativity=8 \
 ;  -polly-target-2nd-cache-level-associativity=8 \
 ;  -polly-target-1st-cache-level-size=32768 \
 ;  -polly-target-vector-register-bitwidth=256 \
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_7.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_7.ll
index 77a3e02a0063..b34b4f020ccb 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_7.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_7.ll
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 ;
 ;    /* C := A * B + C */
 ;    /* Elements of the matrices A, B, C have the float type. */
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_8.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_8.ll
index d02bc359e79d..a65d92940665 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_8.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_8.ll
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 ;
 ;    /* C := A * B + C */
 ;    /* Elements of the matrices B, C have the double type. */
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_9.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_9.ll
index 144abfd7622f..5a35e2c049f0 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_9.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_9.ll
@@ -6,9 +6,9 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-opt-isl -disable-output < %s
+; RUN: -passes=polly-opt-isl -disable-output < %s
 ;
-; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s --check-prefix=DEPENDENCES
+; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s --check-prefix=DEPENDENCES
 ;
 ;    /* C := A * B + C */
 ;    /* Elements of the matrices A, B, C have the char type. */
diff --git a/polly/test/ScheduleOptimizer/pattern_matching_based_opts_splitmap.ll b/polly/test/ScheduleOptimizer/pattern_matching_based_opts_splitmap.ll
index 5b9783d20bfc..05362c712a4d 100644
--- a/polly/test/ScheduleOptimizer/pattern_matching_based_opts_splitmap.ll
+++ b/polly/test/ScheduleOptimizer/pattern_matching_based_opts_splitmap.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-opt-isl -debug-only=polly-opt-isl -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -passes=polly-opt-isl -debug-only=polly-opt-isl -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
 ; void pattern_matching_based_opts_splitmap(double C[static const restrict 2][2], double A[static const restrict 2][784], double B[static const restrict 784][2]) {
diff --git a/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll b/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll
index fea2155b1e4e..5c44f73c287c 100644
--- a/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll
+++ b/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S %loadPolly -basic-aa -polly-tiling=false -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-tiling=false -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 @C = common global [1536 x [1536 x float]] zeroinitializer, align 16
diff --git a/polly/test/ScheduleOptimizer/prevectorization.ll b/polly/test/ScheduleOptimizer/prevectorization.ll
index 385ebf14712a..6a8ec549c784 100644
--- a/polly/test/ScheduleOptimizer/prevectorization.ll
+++ b/polly/test/ScheduleOptimizer/prevectorization.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S %loadPolly -basic-aa -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine                         -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadPolly -basic-aa -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine -polly-prevect-width=16 -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=VEC16
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine                         '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine -polly-prevect-width=16 '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=VEC16
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScheduleOptimizer/rectangular-tiling.ll b/polly/test/ScheduleOptimizer/rectangular-tiling.ll
index b527255ab5f7..9d34c7c17a79 100644
--- a/polly/test/ScheduleOptimizer/rectangular-tiling.ll
+++ b/polly/test/ScheduleOptimizer/rectangular-tiling.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-tile-sizes=256,16                                                                                                                                        -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-tiling=false                                                                                                                    -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=NOTILING
-; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-2nd-level-tiling -polly-2nd-level-tile-sizes=16,8                                                                               -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=TWOLEVEL
-; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-2nd-level-tiling -polly-2nd-level-tile-sizes=16,8 -polly-register-tiling                                                        -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=TWO-PLUS-REGISTER
+; RUN: opt %loadPolly -polly-tile-sizes=256,16                                                                                                                                        '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-tiling=false                                                                                                                    '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=NOTILING
+; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-2nd-level-tiling -polly-2nd-level-tile-sizes=16,8                                                                               '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=TWOLEVEL
+; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-2nd-level-tiling -polly-2nd-level-tile-sizes=16,8 -polly-register-tiling                                                        '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=TWO-PLUS-REGISTER
 
 ; CHECK: // 1st level tiling - Tiles
 ; CHECK: for (int c0 = 0; c0 <= 3; c0 += 1)
diff --git a/polly/test/ScheduleOptimizer/schedule_computeout.ll b/polly/test/ScheduleOptimizer/schedule_computeout.ll
index acc8601a31a8..6e60fe1cd6f3 100644
--- a/polly/test/ScheduleOptimizer/schedule_computeout.ll
+++ b/polly/test/ScheduleOptimizer/schedule_computeout.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -polly-optree -polly-delicm  -polly-opt-isl -polly-schedule-computeout=10000 -debug-only="polly-opt-isl" < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-optree -passes=polly-delicm  -passes=polly-opt-isl -polly-schedule-computeout=10000 -debug-only="polly-opt-isl" < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; Bailout if the computations of schedule compute exceeds the max scheduling quota.
diff --git a/polly/test/ScheduleOptimizer/statistics.ll b/polly/test/ScheduleOptimizer/statistics.ll
index 472febea173f..56bf06894dd4 100644
--- a/polly/test/ScheduleOptimizer/statistics.ll
+++ b/polly/test/ScheduleOptimizer/statistics.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-opt-isl -stats -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -passes=polly-opt-isl -stats -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 
 ; REQUIRES: asserts
 
diff --git a/polly/test/ScheduleOptimizer/tile_after_fusion.ll b/polly/test/ScheduleOptimizer/tile_after_fusion.ll
index 8e5849234af6..b834b354af4d 100644
--- a/polly/test/ScheduleOptimizer/tile_after_fusion.ll
+++ b/polly/test/ScheduleOptimizer/tile_after_fusion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-isl-arg=--no-schedule-serialize-sccs -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-isl-arg=--no-schedule-serialize-sccs '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
 ;
 ;
 ;    void tf(int C[256][256][256], int A0[256][256][256], int A1[256][256][256]) {
@@ -17,7 +17,7 @@
 ; checks whether they are tiled after being fused when polly-opt-fusion equals
 ; "max".
 ;
-; CHECK-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'for.cond => for.end56' in function 'tf':
+; CHECK-LABEL: :: isl ast :: tf :: %for.cond---%for.end56 
 ; CHECK:       1st level tiling - Tiles
 ; CHECK-NEXT:     for (int c0 = 0; c0 <= 7; c0 += 1)
 ; CHECK-NEXT:       for (int c1 = 0; c1 <= 7; c1 += 1)
diff --git a/polly/test/ScheduleOptimizer/vivid-vbi-gen-vivid_vbi_gen_sliced-before-llvmreduced.ll b/polly/test/ScheduleOptimizer/vivid-vbi-gen-vivid_vbi_gen_sliced-before-llvmreduced.ll
index d08595db8fce..bfa1f017b61a 100644
--- a/polly/test/ScheduleOptimizer/vivid-vbi-gen-vivid_vbi_gen_sliced-before-llvmreduced.ll
+++ b/polly/test/ScheduleOptimizer/vivid-vbi-gen-vivid_vbi_gen_sliced-before-llvmreduced.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-vectorizer=stripmine -polly-isl-arg=--no-schedule-serialize-sccs -polly-tiling=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-vectorizer=stripmine -polly-isl-arg=--no-schedule-serialize-sccs -polly-tiling=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
 
 ; isl_schedule_node_band_sink may sink into multiple children.
 ; https://llvm.org/PR52637
diff --git a/polly/test/ScopDetect/aliasing_parametric_simple_1.ll b/polly/test/ScopDetect/aliasing_parametric_simple_1.ll
index 2eddbd4cb262..8a2317446d64 100644
--- a/polly/test/ScopDetect/aliasing_parametric_simple_1.ll
+++ b/polly/test/ScopDetect/aliasing_parametric_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop:
 ;
diff --git a/polly/test/ScopDetect/aliasing_parametric_simple_2.ll b/polly/test/ScopDetect/aliasing_parametric_simple_2.ll
index c111f686c462..df1d1f8d56bc 100644
--- a/polly/test/ScopDetect/aliasing_parametric_simple_2.ll
+++ b/polly/test/ScopDetect/aliasing_parametric_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop:
 ;
diff --git a/polly/test/ScopDetect/aliasing_simple_1.ll b/polly/test/ScopDetect/aliasing_simple_1.ll
index 524ca19ae398..af5b7cf1dfb7 100644
--- a/polly/test/ScopDetect/aliasing_simple_1.ll
+++ b/polly/test/ScopDetect/aliasing_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop:
 ;
diff --git a/polly/test/ScopDetect/aliasing_simple_2.ll b/polly/test/ScopDetect/aliasing_simple_2.ll
index 457df996c7b8..cd3155ec613a 100644
--- a/polly/test/ScopDetect/aliasing_simple_2.ll
+++ b/polly/test/ScopDetect/aliasing_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop:
 ;
diff --git a/polly/test/ScopDetect/base_pointer_load_setNewAccessRelation.ll b/polly/test/ScopDetect/base_pointer_load_setNewAccessRelation.ll
index 0411aed6ae04..ab56f344f093 100644
--- a/polly/test/ScopDetect/base_pointer_load_setNewAccessRelation.ll
+++ b/polly/test/ScopDetect/base_pointer_load_setNewAccessRelation.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true -polly-scops -polly-print-import-jscop -polly-codegen -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>,scop(polly-import-jscop,polly-codegen)' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; This violated an assertion in setNewAccessRelation that assumed base pointers
 ; to be load-hoisted. Without this assertion, it codegen would generate invalid
diff --git a/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll b/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll
index ff9be6ea16e8..df7f3d706fd3 100644
--- a/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll
+++ b/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -disable-basic-aa -polly-detect -polly-print-import-jscop -polly-codegen -disable-output < %s | FileCheck %s --allow-empty
+; RUN: opt %loadPolly '-passes=print<polly-detect>,scop(polly-import-jscop,polly-codegen)' -disable-output < %s 2>&1 | FileCheck %s --allow-empty
 ;
 ; Polly codegen used to generate invalid code (referring to %ptr from the
 ; original region) when regeneration of the access function is necessary.
diff --git a/polly/test/ScopDetect/callbr.ll b/polly/test/ScopDetect/callbr.ll
index d65ab934bf2e..5183b9c1a085 100644
--- a/polly/test/ScopDetect/callbr.ll
+++ b/polly/test/ScopDetect/callbr.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-detect -polly-detect-track-failures -disable-output -pass-remarks-missed=polly-detect < %s 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt %loadPolly -polly-detect -polly-detect-track-failures -disable-output -stats                            < %s 2>&1 | FileCheck %s --check-prefix=STAT
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -polly-detect-track-failures -disable-output -pass-remarks-missed=polly-detect < %s 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -polly-detect-track-failures -disable-output -stats                            < %s 2>&1 | FileCheck %s --check-prefix=STAT
 ; REQUIRES: asserts
 
 ; REMARK: Branch from indirect terminator.
diff --git a/polly/test/ScopDetect/collective_invariant_loads.ll b/polly/test/ScopDetect/collective_invariant_loads.ll
index f1d2eea520c6..96f154f07d2e 100644
--- a/polly/test/ScopDetect/collective_invariant_loads.ll
+++ b/polly/test/ScopDetect/collective_invariant_loads.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting -disable-output< %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting -disable-output< %s 2>&1 | FileCheck %s
 
 ;CHECK:     Function: test_init_chpl
 ;CHECK-NEXT:     Region: %bb1---%bb16
diff --git a/polly/test/ScopDetect/cross_loop_non_single_exit.ll b/polly/test/ScopDetect/cross_loop_non_single_exit.ll
index ae23930b92a6..e54d30fc2164 100644
--- a/polly/test/ScopDetect/cross_loop_non_single_exit.ll
+++ b/polly/test/ScopDetect/cross_loop_non_single_exit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/cross_loop_non_single_exit_2.ll b/polly/test/ScopDetect/cross_loop_non_single_exit_2.ll
index 5c25da66d7ef..f3bd0d097b71 100644
--- a/polly/test/ScopDetect/cross_loop_non_single_exit_2.ll
+++ b/polly/test/ScopDetect/cross_loop_non_single_exit_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/dependency_to_phi_node_outside_of_region.ll b/polly/test/ScopDetect/dependency_to_phi_node_outside_of_region.ll
index 12983d2321cc..6d262a9a464b 100644
--- a/polly/test/ScopDetect/dependency_to_phi_node_outside_of_region.ll
+++ b/polly/test/ScopDetect/dependency_to_phi_node_outside_of_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-detect -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
 define void @f(ptr %A, i64 %N, i64 %M) nounwind {
diff --git a/polly/test/ScopDetect/dot-scops-npm.ll b/polly/test/ScopDetect/dot-scops-npm.ll
index 7c8be032fd4f..9de6a5e2e1a5 100644
--- a/polly/test/ScopDetect/dot-scops-npm.ll
+++ b/polly/test/ScopDetect/dot-scops-npm.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadNPMPolly "-passes=polly-scop-printer" -disable-output < %s
+; RUN: opt %loadPolly '-passes=polly-scop-printer' -disable-output < %s
 ; RUN: FileCheck %s -input-file=scops.func_npm.dot
 ;
 ; Check that the ScopPrinter does not crash.
diff --git a/polly/test/ScopDetect/dot-scops.ll b/polly/test/ScopDetect/dot-scops.ll
index c31562e4c62d..2297fd3253ca 100644
--- a/polly/test/ScopDetect/dot-scops.ll
+++ b/polly/test/ScopDetect/dot-scops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-scops -dot-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>,polly-scop-printer' -disable-output < %s
 ;
 ; Check that the ScopPrinter does not crash.
 ; ScopPrinter needs the ScopDetection pass, which should depend on
diff --git a/polly/test/ScopDetect/error-block-always-executed.ll b/polly/test/ScopDetect/error-block-always-executed.ll
index 894be2119941..312c48cfee96 100644
--- a/polly/test/ScopDetect/error-block-always-executed.ll
+++ b/polly/test/ScopDetect/error-block-always-executed.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK-NOT: Valid Region for Scop:
 
diff --git a/polly/test/ScopDetect/error-block-referenced-from-scop.ll b/polly/test/ScopDetect/error-block-referenced-from-scop.ll
index 085351482139..d3e56472e497 100644
--- a/polly/test/ScopDetect/error-block-referenced-from-scop.ll
+++ b/polly/test/ScopDetect/error-block-referenced-from-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK-NOT: Valid Region for Scop:
 
diff --git a/polly/test/ScopDetect/error-block-unreachable.ll b/polly/test/ScopDetect/error-block-unreachable.ll
index 48f6fe8e0547..72d43004e322 100644
--- a/polly/test/ScopDetect/error-block-unreachable.ll
+++ b/polly/test/ScopDetect/error-block-unreachable.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-detect -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s
 
 ; Verify that the scop detection does not crash on inputs with unreachable
 ; blocks. Earlier we crashed when detecting error blocks.
diff --git a/polly/test/ScopDetect/expand-region-correctly-2.ll b/polly/test/ScopDetect/expand-region-correctly-2.ll
index fadb503cff35..b6632c643fdf 100644
--- a/polly/test/ScopDetect/expand-region-correctly-2.ll
+++ b/polly/test/ScopDetect/expand-region-correctly-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop: if.end.1631 => for.cond.1647.outer
 ;
diff --git a/polly/test/ScopDetect/expand-region-correctly.ll b/polly/test/ScopDetect/expand-region-correctly.ll
index 72082a32fa79..022dfb68ecd9 100644
--- a/polly/test/ScopDetect/expand-region-correctly.ll
+++ b/polly/test/ScopDetect/expand-region-correctly.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK: Valid Region for Scop: if.end.1631 => for.cond.1647.outer
 
diff --git a/polly/test/ScopDetect/ignore_func_flag_regex.ll b/polly/test/ScopDetect/ignore_func_flag_regex.ll
index 224126ec010e..15b92b418bcb 100644
--- a/polly/test/ScopDetect/ignore_func_flag_regex.ll
+++ b/polly/test/ScopDetect/ignore_func_flag_regex.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-ignore-func=f.*,g.* -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-ignore-func=f.*,g.* '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that the flag `-polly-ignore-func` works with regexes.
 ;
diff --git a/polly/test/ScopDetect/index_from_unpredictable_loop.ll b/polly/test/ScopDetect/index_from_unpredictable_loop.ll
index 27ed64da17e6..e0be6243ebf8 100644
--- a/polly/test/ScopDetect/index_from_unpredictable_loop.ll
+++ b/polly/test/ScopDetect/index_from_unpredictable_loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly                        -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=AFFINE
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN: opt %loadPolly                        '-passes=print<polly-function-scops>' -disable-output < %s | FileCheck %s --check-prefix=AFFINE
+; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
 
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopDetect/index_from_unpredictable_loop2.ll b/polly/test/ScopDetect/index_from_unpredictable_loop2.ll
index 9b5a3a4389d4..4d4b6f988b69 100644
--- a/polly/test/ScopDetect/index_from_unpredictable_loop2.ll
+++ b/polly/test/ScopDetect/index_from_unpredictable_loop2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly                        -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=AFFINE
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN: opt %loadPolly                        '-passes=print<polly-function-scops>' -disable-output < %s | FileCheck %s --check-prefix=AFFINE
+; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
 
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopDetect/indvars.ll b/polly/test/ScopDetect/indvars.ll
index 2ba4d1f5aabf..023f68435988 100644
--- a/polly/test/ScopDetect/indvars.ll
+++ b/polly/test/ScopDetect/indvars.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -polly-codegen -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,scop(polly-codegen)' -disable-output < %s 2>&1 | FileCheck %s
 ;
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/ScopDetect/intrinsics_1.ll b/polly/test/ScopDetect/intrinsics_1.ll
index 65d3968e247c..61e1c1fc3d86 100644
--- a/polly/test/ScopDetect/intrinsics_1.ll
+++ b/polly/test/ScopDetect/intrinsics_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop: for.cond => for.end
 ;
diff --git a/polly/test/ScopDetect/intrinsics_2.ll b/polly/test/ScopDetect/intrinsics_2.ll
index f0575511b2ef..0c2aa49c2d21 100644
--- a/polly/test/ScopDetect/intrinsics_2.ll
+++ b/polly/test/ScopDetect/intrinsics_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify that we allow the lifetime markers for the tmp array.
 ;
diff --git a/polly/test/ScopDetect/intrinsics_3.ll b/polly/test/ScopDetect/intrinsics_3.ll
index bce90d136a41..16d41d0550af 100644
--- a/polly/test/ScopDetect/intrinsics_3.ll
+++ b/polly/test/ScopDetect/intrinsics_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify that we allow the misc intrinsics.
 ;
diff --git a/polly/test/ScopDetect/invalid-latch-conditions.ll b/polly/test/ScopDetect/invalid-latch-conditions.ll
index eb8097470ecf..1264ba0483c0 100644
--- a/polly/test/ScopDetect/invalid-latch-conditions.ll
+++ b/polly/test/ScopDetect/invalid-latch-conditions.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly                              -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops                                   -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=NALOOPS
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly                              -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops                                   '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NALOOPS
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
 
 ; The latch conditions of the outer loop are not affine, thus the loop cannot
 ; handled by the domain generation and needs to be overapproximated.
diff --git a/polly/test/ScopDetect/invalidate_scalar_evolution.ll b/polly/test/ScopDetect/invalidate_scalar_evolution.ll
index 01d34c49e289..c691c7a633d2 100644
--- a/polly/test/ScopDetect/invalidate_scalar_evolution.ll
+++ b/polly/test/ScopDetect/invalidate_scalar_evolution.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=PHI
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=PHI
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/invariant-load-before-scop.ll b/polly/test/ScopDetect/invariant-load-before-scop.ll
index f72085ff88a1..ee2eba5e8ec6 100644
--- a/polly/test/ScopDetect/invariant-load-before-scop.ll
+++ b/polly/test/ScopDetect/invariant-load-before-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 ;
 ; The LoadInst %.b761 is defined outside the SCoP, hence is always constant
 ; within it. It is no "required invariant load".
diff --git a/polly/test/ScopDetect/keep_going_expansion.ll b/polly/test/ScopDetect/keep_going_expansion.ll
index 9bcfb3924f6a..7da7cd41c1f9 100644
--- a/polly/test/ScopDetect/keep_going_expansion.ll
+++ b/polly/test/ScopDetect/keep_going_expansion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-detect-track-failures -polly-detect-keep-going -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-detect-track-failures -polly-detect-keep-going '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopDetect/mod_ref_read_pointer.ll b/polly/test/ScopDetect/mod_ref_read_pointer.ll
index 95a4649f4705..7b185a87c821 100644
--- a/polly/test/ScopDetect/mod_ref_read_pointer.ll
+++ b/polly/test/ScopDetect/mod_ref_read_pointer.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-allow-modref-calls -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=MODREF
-; RUN: opt %loadPolly -basic-aa                           -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-modref-calls '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=MODREF
+; RUN: opt %loadPolly -aa-pipeline=basic-aa                           '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK-NOT: Valid Region for Scop: for.body => for.end
 ; MODREF: Valid Region for Scop: for.body => for.end
diff --git a/polly/test/ScopDetect/more-than-one-loop.ll b/polly/test/ScopDetect/more-than-one-loop.ll
index bfd226c1bcfc..5972fbf50889 100644
--- a/polly/test/ScopDetect/more-than-one-loop.ll
+++ b/polly/test/ScopDetect/more-than-one-loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-process-unprofitable=true -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable=true '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK: Valid Region for Scop:
 
diff --git a/polly/test/ScopDetect/multidim-with-undef-size.ll b/polly/test/ScopDetect/multidim-with-undef-size.ll
index 9973c6c72169..2adf7bb00b42 100644
--- a/polly/test/ScopDetect/multidim-with-undef-size.ll
+++ b/polly/test/ScopDetect/multidim-with-undef-size.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; CHECK: Valid Region for Scop: bb14 => bb17
diff --git a/polly/test/ScopDetect/multidim.ll b/polly/test/ScopDetect/multidim.ll
index f43698819f32..a1a6167a121e 100644
--- a/polly/test/ScopDetect/multidim.ll
+++ b/polly/test/ScopDetect/multidim.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; CHECK: Valid Region for Scop: bb19 => bb20
diff --git a/polly/test/ScopDetect/multidim_indirect_access.ll b/polly/test/ScopDetect/multidim_indirect_access.ll
index 3e06251f5fd1..4a3012d1c93c 100644
--- a/polly/test/ScopDetect/multidim_indirect_access.ll
+++ b/polly/test/ScopDetect/multidim_indirect_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that we will recognize this SCoP.
 ;
diff --git a/polly/test/ScopDetect/multidim_two_accesses_different_delinearization.ll b/polly/test/ScopDetect/multidim_two_accesses_different_delinearization.ll
index ed554a24a6d6..23d520ed58be 100644
--- a/polly/test/ScopDetect/multidim_two_accesses_different_delinearization.ll
+++ b/polly/test/ScopDetect/multidim_two_accesses_different_delinearization.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/ScopDetect/nested_loop_single_exit.ll b/polly/test/ScopDetect/nested_loop_single_exit.ll
index 377e8088eedb..a794e2c48ff9 100644
--- a/polly/test/ScopDetect/nested_loop_single_exit.ll
+++ b/polly/test/ScopDetect/nested_loop_single_exit.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
 
 ; void f(long A[], long N) {
 ;   long i, j;
diff --git a/polly/test/ScopDetect/non-affine-conditional.ll b/polly/test/ScopDetect/non-affine-conditional.ll
index fc2d0c02d2da..f69b6f8cd1ed 100644
--- a/polly/test/ScopDetect/non-affine-conditional.ll
+++ b/polly/test/ScopDetect/non-affine-conditional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/ScopDetect/non-affine-float-compare.ll b/polly/test/ScopDetect/non-affine-float-compare.ll
index 984f14aaff8f..1e4c580fa00d 100644
--- a/polly/test/ScopDetect/non-affine-float-compare.ll
+++ b/polly/test/ScopDetect/non-affine-float-compare.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(float *A) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll
index 068367fa1e3c..443a0e13c6ca 100644
--- a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll
+++ b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                                                          -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                                           -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine                                   -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                                                          '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                                           '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine                                   '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
 ;
 ; Here we have a non-affine loop but also a non-affine access which should
 ; be rejected as long as -polly-allow-nonaffine isn't given.
diff --git a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_2.ll b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_2.ll
index cd2140518b46..77733a8d9b96 100644
--- a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_2.ll
+++ b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_2.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                        -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                         -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                        '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                         '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
 ;
 ; Here we have a non-affine loop (in the context of the loop nest)
 ; and also a non-affine access (A[k]). While we can always detect the
diff --git a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_3.ll b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_3.ll
index fb936216e45c..034ab61fa03e 100644
--- a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_3.ll
+++ b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_3.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                        -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                         -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                        '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                         '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
 ;
 ; Here we have a non-affine loop (in the context of the loop nest)
 ; and also a non-affine access (A[k]). While we can always detect the
diff --git a/polly/test/ScopDetect/non-affine-loop.ll b/polly/test/ScopDetect/non-affine-loop.ll
index d5f7ea128a79..d17fd39da701 100644
--- a/polly/test/ScopDetect/non-affine-loop.ll
+++ b/polly/test/ScopDetect/non-affine-loop.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                                                          -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                                           -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false -polly-allow-nonaffine                                   -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINEREGIONSANDACCESSES
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine                                   -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                                                          '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                                           '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false -polly-allow-nonaffine                                   '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINEREGIONSANDACCESSES
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine                                   '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
 ;
 ; This function/region does contain a loop, however it is non-affine, hence the access
 ; A[i] is also. Furthermore, it is the only loop, thus when we over approximate
diff --git a/polly/test/ScopDetect/non-beneficial-loops-small-trip-count.ll b/polly/test/ScopDetect/non-beneficial-loops-small-trip-count.ll
index 43af1684dccb..d5901b63dd37 100644
--- a/polly/test/ScopDetect/non-beneficial-loops-small-trip-count.ll
+++ b/polly/test/ScopDetect/non-beneficial-loops-small-trip-count.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK-NOT: Valid
 ;
diff --git a/polly/test/ScopDetect/non-constant-add-rec-start-expr.ll b/polly/test/ScopDetect/non-constant-add-rec-start-expr.ll
index 4cddcc916a76..f39e774021a9 100644
--- a/polly/test/ScopDetect/non-constant-add-rec-start-expr.ll
+++ b/polly/test/ScopDetect/non-constant-add-rec-start-expr.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK: Valid Region for Scop: bb11 => bb25
 
diff --git a/polly/test/ScopDetect/non-simple-memory-accesses.ll b/polly/test/ScopDetect/non-simple-memory-accesses.ll
index a82228982885..d1c2ce63059e 100644
--- a/polly/test/ScopDetect/non-simple-memory-accesses.ll
+++ b/polly/test/ScopDetect/non-simple-memory-accesses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify that we do not model atomic memory accesses. We did not reason about
 ; how to handle them correctly and the Alias Set Tracker models some of them
diff --git a/polly/test/ScopDetect/non_affine_loop_condition.ll b/polly/test/ScopDetect/non_affine_loop_condition.ll
index f268442cd8ee..1d67df58d9bb 100644
--- a/polly/test/ScopDetect/non_affine_loop_condition.ll
+++ b/polly/test/ScopDetect/non_affine_loop_condition.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops                                   -polly-print-detect -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops                                   '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
 ;
 ;    void f(int *A) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/ScopDetect/only-one-affine-loop.ll b/polly/test/ScopDetect/only-one-affine-loop.ll
index d6d50bb611d9..3f4305ab83e7 100644
--- a/polly/test/ScopDetect/only-one-affine-loop.ll
+++ b/polly/test/ScopDetect/only-one-affine-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable=false -polly-allow-nonaffine-loops -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable=false -polly-allow-nonaffine-loops '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Even if we allow non-affine loops we can only model the outermost loop, all
 ; other loops are boxed in non-affine regions. However, the inner loops can be
diff --git a/polly/test/ScopDetect/only_func_flag.ll b/polly/test/ScopDetect/only_func_flag.ll
index d465cd0f50f7..35a38e875e12 100644
--- a/polly/test/ScopDetect/only_func_flag.ll
+++ b/polly/test/ScopDetect/only_func_flag.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-only-func=f,g -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-only-func=f,g '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that the flag `-polly-only-func` limits analysis to `f` and `g`.
 ;
diff --git a/polly/test/ScopDetect/only_func_flag_regex.ll b/polly/test/ScopDetect/only_func_flag_regex.ll
index e6675798eeb9..3b577b100db3 100644
--- a/polly/test/ScopDetect/only_func_flag_regex.ll
+++ b/polly/test/ScopDetect/only_func_flag_regex.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-only-func=f.*,g.* -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-only-func=f.*,g.* '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that the flag `-polly-only-func` works with regexes.
 ;
diff --git a/polly/test/ScopDetect/parametric-multiply-in-scev-2.ll b/polly/test/ScopDetect/parametric-multiply-in-scev-2.ll
index fc957a7f912c..6cb8a9ee5125 100644
--- a/polly/test/ScopDetect/parametric-multiply-in-scev-2.ll
+++ b/polly/test/ScopDetect/parametric-multiply-in-scev-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 
 ; CHECK-NOT: Valid Region
diff --git a/polly/test/ScopDetect/parametric-multiply-in-scev.ll b/polly/test/ScopDetect/parametric-multiply-in-scev.ll
index 9c6e5ccc8f52..bb6dd6c73593 100644
--- a/polly/test/ScopDetect/parametric-multiply-in-scev.ll
+++ b/polly/test/ScopDetect/parametric-multiply-in-scev.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ;  foo(float *A, long n, long k) {
 ;    if (true)
diff --git a/polly/test/ScopDetect/phi_with_multi_exiting_edges.ll b/polly/test/ScopDetect/phi_with_multi_exiting_edges.ll
index 054de168d76b..c88dcfd860f4 100644
--- a/polly/test/ScopDetect/phi_with_multi_exiting_edges.ll
+++ b/polly/test/ScopDetect/phi_with_multi_exiting_edges.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Region with an exit node that has a PHI node multiple incoming edges from
 ; inside the region. Motivation for supporting such cases in Polly.
diff --git a/polly/test/ScopDetect/profitability-large-basic-blocks.ll b/polly/test/ScopDetect/profitability-large-basic-blocks.ll
index e1650febf11c..7296812ad85c 100644
--- a/polly/test/ScopDetect/profitability-large-basic-blocks.ll
+++ b/polly/test/ScopDetect/profitability-large-basic-blocks.ll
@@ -1,12 +1,12 @@
 ; RUN: opt %loadPolly -polly-process-unprofitable=false \
 ; RUN:                -polly-detect-profitability-min-per-loop-insts=40 \
-; RUN: -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=PROFITABLE
+; RUN: '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=PROFITABLE
 
 ; RUN: opt %loadPolly -polly-process-unprofitable=true \
-; RUN: -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=PROFITABLE
+; RUN: '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=PROFITABLE
 
 ; RUN: opt %loadPolly -polly-process-unprofitable=false \
-; RUN: -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=UNPROFITABLE
+; RUN: '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=UNPROFITABLE
 
 ; UNPROFITABLE-NOT: Valid Region for Scop:
 ; PROFITABLE: Valid Region for Scop:
diff --git a/polly/test/ScopDetect/profitability-two-nested-loops.ll b/polly/test/ScopDetect/profitability-two-nested-loops.ll
index 525f91cbc2f4..9311fc87d378 100644
--- a/polly/test/ScopDetect/profitability-two-nested-loops.ll
+++ b/polly/test/ScopDetect/profitability-two-nested-loops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK: Valid Region for Scop: next => bb3
 ;
diff --git a/polly/test/ScopDetect/remove_all_children.ll b/polly/test/ScopDetect/remove_all_children.ll
index 6d5097b80607..a6b211ccfe0b 100644
--- a/polly/test/ScopDetect/remove_all_children.ll
+++ b/polly/test/ScopDetect/remove_all_children.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopDetect/report-scop-location.ll b/polly/test/ScopDetect/report-scop-location.ll
index 750699cbe763..03043faedc34 100644
--- a/polly/test/ScopDetect/report-scop-location.ll
+++ b/polly/test/ScopDetect/report-scop-location.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-detect -polly-report -disable-output < %s  2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -polly-report -disable-output < %s  2>&1 | FileCheck %s
 target datalayout = "e-i64:64-f80:128-s:64-n8:16:32:64-S128"
 
 ; Function Attrs: nounwind uwtable
diff --git a/polly/test/ScopDetect/restrict-undef-size-scopdetect.ll b/polly/test/ScopDetect/restrict-undef-size-scopdetect.ll
index e94f1e7728c5..04d6b151ebba 100644
--- a/polly/test/ScopDetect/restrict-undef-size-scopdetect.ll
+++ b/polly/test/ScopDetect/restrict-undef-size-scopdetect.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ; CHECK-NOT: Valid Region for Scop:
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/ScopDetect/run_time_alias_check.ll b/polly/test/ScopDetect/run_time_alias_check.ll
index 672f3dfa6365..aa6ba8698fe9 100644
--- a/polly/test/ScopDetect/run_time_alias_check.ll
+++ b/polly/test/ScopDetect/run_time_alias_check.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
diff --git a/polly/test/ScopDetect/scev_remove_max.ll b/polly/test/ScopDetect/scev_remove_max.ll
index 5353e06bdf2f..5aa121977cfe 100644
--- a/polly/test/ScopDetect/scev_remove_max.ll
+++ b/polly/test/ScopDetect/scev_remove_max.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-detect < %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' < %s
 
 ; This test case helps to determine wether SCEVRemoveMax::remove produces
 ; an infinite loop and a segmentation fault, if it processes, for example,
diff --git a/polly/test/ScopDetect/sequential_loops.ll b/polly/test/ScopDetect/sequential_loops.ll
index e6ac38aa1604..df10da3aac36 100644
--- a/polly/test/ScopDetect/sequential_loops.ll
+++ b/polly/test/ScopDetect/sequential_loops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
@@ -13,7 +13,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 ; }
 
 define void @f1(ptr %A, i64 %N) nounwind {
-; CHECK-LABEL: 'Polly - Detect static control parts (SCoPs)' for function 'f1'
+; CHECK-LABEL: Detected Scops in Function f1 
 entry:
   fence seq_cst
   br label %for.i.1
@@ -60,7 +60,7 @@ return:
 ;     }
 
 define void @f2(ptr %A, i64 %N) nounwind {
-; CHECK-LABEL: 'Polly - Detect static control parts (SCoPs)' for function 'f2'
+; CHECK-LABEL: Detected Scops in Function f2
 entry:
   fence seq_cst
   br label %for.i.1
diff --git a/polly/test/ScopDetect/simple_loop.ll b/polly/test/ScopDetect/simple_loop.ll
index c8ed89a97d00..376a7dfa9f9b 100644
--- a/polly/test/ScopDetect/simple_loop.ll
+++ b/polly/test/ScopDetect/simple_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/simple_loop_non_single_entry.ll b/polly/test/ScopDetect/simple_loop_non_single_entry.ll
index 22adec5d2039..64e2a084188d 100644
--- a/polly/test/ScopDetect/simple_loop_non_single_entry.ll
+++ b/polly/test/ScopDetect/simple_loop_non_single_entry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/simple_loop_non_single_exit.ll b/polly/test/ScopDetect/simple_loop_non_single_exit.ll
index 71ac830cae7d..4c3a1aea1ad8 100644
--- a/polly/test/ScopDetect/simple_loop_non_single_exit.ll
+++ b/polly/test/ScopDetect/simple_loop_non_single_exit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/simple_loop_non_single_exit_2.ll b/polly/test/ScopDetect/simple_loop_non_single_exit_2.ll
index d9915dc130d5..ae4ffa7b4972 100644
--- a/polly/test/ScopDetect/simple_loop_non_single_exit_2.ll
+++ b/polly/test/ScopDetect/simple_loop_non_single_exit_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/simple_loop_two_phi_nodes.ll b/polly/test/ScopDetect/simple_loop_two_phi_nodes.ll
index 867bd50513f0..a6bca0ec9d73 100644
--- a/polly/test/ScopDetect/simple_loop_two_phi_nodes.ll
+++ b/polly/test/ScopDetect/simple_loop_two_phi_nodes.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/simple_loop_with_param.ll b/polly/test/ScopDetect/simple_loop_with_param.ll
index 1ae5c6608739..ab48709d290d 100644
--- a/polly/test/ScopDetect/simple_loop_with_param.ll
+++ b/polly/test/ScopDetect/simple_loop_with_param.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=PHI
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=PHI
 
 ; void f(long A[], long N, long *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopDetect/simple_loop_with_param_2.ll b/polly/test/ScopDetect/simple_loop_with_param_2.ll
index 1a4750621c19..baa647489543 100644
--- a/polly/test/ScopDetect/simple_loop_with_param_2.ll
+++ b/polly/test/ScopDetect/simple_loop_with_param_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopDetect/simple_non_single_entry.ll b/polly/test/ScopDetect/simple_non_single_entry.ll
index a1995a427903..1f1cc95147a5 100644
--- a/polly/test/ScopDetect/simple_non_single_entry.ll
+++ b/polly/test/ScopDetect/simple_non_single_entry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/skip_function_attribute.ll b/polly/test/ScopDetect/skip_function_attribute.ll
index e85dbd4c2b83..d30c042fb74e 100644
--- a/polly/test/ScopDetect/skip_function_attribute.ll
+++ b/polly/test/ScopDetect/skip_function_attribute.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify polly skips this function
 ;
diff --git a/polly/test/ScopDetect/srem_with_parametric_divisor.ll b/polly/test/ScopDetect/srem_with_parametric_divisor.ll
index 4b5c3b04c2ce..9a6352b9afe6 100644
--- a/polly/test/ScopDetect/srem_with_parametric_divisor.ll
+++ b/polly/test/ScopDetect/srem_with_parametric_divisor.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK-NOT: Valid Region for Scop:
 ;
diff --git a/polly/test/ScopDetect/statistics.ll b/polly/test/ScopDetect/statistics.ll
index 64df3d081605..5789677325d8 100644
--- a/polly/test/ScopDetect/statistics.ll
+++ b/polly/test/ScopDetect/statistics.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-detect -stats -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -stats -disable-output < %s 2>&1 | FileCheck %s
 
 ; REQUIRES: asserts
 
diff --git a/polly/test/ScopDetect/switch-in-loop-patch.ll b/polly/test/ScopDetect/switch-in-loop-patch.ll
index ab4729fc09a4..508f59ee398e 100644
--- a/polly/test/ScopDetect/switch-in-loop-patch.ll
+++ b/polly/test/ScopDetect/switch-in-loop-patch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK-NOT: Valid
 
diff --git a/polly/test/ScopDetectionDiagnostics/ReportAlias-01.ll b/polly/test/ScopDetectionDiagnostics/ReportAlias-01.ll
index 97ba7f9634e9..8f575df22f84 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportAlias-01.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportAlias-01.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-use-runtime-alias-checks=false -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -polly-use-runtime-alias-checks=false -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1| FileCheck %s
 
 ;void f(int A[], int B[]) {
 ;  for (int i=0; i<42; i++)
diff --git a/polly/test/ScopDetectionDiagnostics/ReportEntry.ll b/polly/test/ScopDetectionDiagnostics/ReportEntry.ll
index fc21e192f32c..f80b48fc3b22 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportEntry.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportEntry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-detect -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
 
 ; CHECK: remark: <unknown>:0:0: Scop contains function entry (not yet supported).
 
diff --git a/polly/test/ScopDetectionDiagnostics/ReportFuncCall-01.ll b/polly/test/ScopDetectionDiagnostics/ReportFuncCall-01.ll
index abace4ba520d..2d9175607ba4 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportFuncCall-01.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportFuncCall-01.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; #define N 1024
 ; double invalidCall(double A[N]);
diff --git a/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegion.ll b/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegion.ll
index 8368a68b42f0..8f5f08fb27c1 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegion.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ;void foo(int a, int b) {
diff --git a/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegionWithoutDebugLoc.ll b/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegionWithoutDebugLoc.ll
index 82c6c33e287c..f5ca683f0fd5 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegionWithoutDebugLoc.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegionWithoutDebugLoc.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-detect -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
 
 ; CHECK: remark: <unknown>:0:0: Irreducible region encountered in control flow.
 
diff --git a/polly/test/ScopDetectionDiagnostics/ReportLoopBound-01.ll b/polly/test/ScopDetectionDiagnostics/ReportLoopBound-01.ll
index 35986b5e0b35..27d26e665193 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportLoopBound-01.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportLoopBound-01.ll
@@ -1,15 +1,15 @@
 ; RUN: opt %loadPolly \
 ; RUN:     -pass-remarks-missed="polly-detect" -polly-detect-track-failures \
-; RUN:     -polly-allow-nonaffine-loops=false -polly-print-detect -disable-output \
+; RUN:     -polly-allow-nonaffine-loops=false '-passes=print<polly-detect>' -disable-output \
 ; RUN:     < %s 2>&1| FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
 ; RUN: opt %loadPolly \
 ; RUN:     -pass-remarks-missed="polly-detect" -polly-detect-track-failures \
-; RUN:     -polly-allow-nonaffine-loops=true -polly-print-detect -disable-output \
+; RUN:     -polly-allow-nonaffine-loops=true '-passes=print<polly-detect>' -disable-output \
 ; RUN:     < %s 2>&1| FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
 ; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" \
 ; RUN:     -polly-process-unprofitable=false \
 ; RUN:     -polly-detect-track-failures -polly-allow-nonaffine-loops=true \
-; RUN:     -polly-allow-nonaffine -polly-print-detect -disable-output < %s 2>&1 \
+; RUN:     -polly-allow-nonaffine '-passes=print<polly-detect>' -disable-output < %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=ALLOWNONAFFINEALL
 
 ; void f(int A[], int n) {
diff --git a/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll b/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll
index 5dbeaded45c9..a40b423f04fc 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll
@@ -4,8 +4,8 @@
 ; the PostDominatorTree. Infinite loops are postdominated ony by the virtual
 ; root, which causes them not to appear in regions in ScopDetection anymore.
 
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-allow-nonaffine-loops -polly-print-detect -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-allow-nonaffine-loops=false -polly-print-detect -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-allow-nonaffine-loops '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-allow-nonaffine-loops=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void func (int param0, int N, int *A)
 ; {
diff --git a/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll b/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll
index 634b63e6d44d..b5eaaea1327e 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1| FileCheck %s
-; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -polly-delinearize=false -polly-detect-keep-going -disable-output < %s 2>&1| FileCheck %s -check-prefix=ALL
-; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1| FileCheck %s -check-prefix=DELIN
-; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -polly-detect-keep-going -disable-output < %s 2>&1| FileCheck %s -check-prefix=DELIN-ALL
-; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -polly-allow-nonaffine -disable-output < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
-; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -polly-allow-nonaffine -disable-output < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -polly-delinearize=false -polly-detect-keep-going -disable-output < %s 2>&1| FileCheck %s -check-prefix=ALL
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1| FileCheck %s -check-prefix=DELIN
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -polly-detect-keep-going -disable-output < %s 2>&1| FileCheck %s -check-prefix=DELIN-ALL
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -polly-allow-nonaffine -disable-output < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -polly-allow-nonaffine -disable-output < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
 
 ;  1 void manyaccesses(float A[restrict], long n, float B[restrict][n])
 ;  2 {
diff --git a/polly/test/ScopDetectionDiagnostics/ReportNonAffineAccess-01.ll b/polly/test/ScopDetectionDiagnostics/ReportNonAffineAccess-01.ll
index 23d8c9c061c9..369a464a0f77 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportNonAffineAccess-01.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportNonAffineAccess-01.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1| FileCheck %s
 
 ; void f(int A[]) {
 ;   for(int i=0; i<42; ++i)
diff --git a/polly/test/ScopDetectionDiagnostics/ReportUnprofitable.ll b/polly/test/ScopDetectionDiagnostics/ReportUnprofitable.ll
index d35b7a28ba89..c606fc2b6921 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportUnprofitable.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportUnprofitable.ll
@@ -1,9 +1,9 @@
 ; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" \
-; RUN:     -polly-detect-track-failures -polly-print-detect -disable-output \
+; RUN:     -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output \
 ; RUN:     -polly-process-unprofitable=false < %s 2>&1| FileCheck %s
 
 ; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" \
-; RUN:     -polly-detect-track-failures -polly-print-detect -disable-output \
+; RUN:     -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output \
 ; RUN:     -polly-process-unprofitable=false < %s 2>&1 -pass-remarks-output=%t.yaml
 ; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/ScopDetectionDiagnostics/ReportUnreachableInExit.ll b/polly/test/ScopDetectionDiagnostics/ReportUnreachableInExit.ll
index 6c868db78ce7..5b20d6bea3dc 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportUnreachableInExit.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportUnreachableInExit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s \
 ; RUN:     -pass-remarks-missed="polly-detect" 2>&1 | FileCheck %s
 
 ; void f(long A[], long N) {
diff --git a/polly/test/ScopDetectionDiagnostics/ReportVariantBasePtr-01.ll b/polly/test/ScopDetectionDiagnostics/ReportVariantBasePtr-01.ll
index a82f56b7a5fa..ca10b0ac0256 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportVariantBasePtr-01.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportVariantBasePtr-01.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1| FileCheck %s
 
 ; struct b {
 ;   double **b;
diff --git a/polly/test/ScopDetectionDiagnostics/loop_has_multiple_exits.ll b/polly/test/ScopDetectionDiagnostics/loop_has_multiple_exits.ll
index a0f2704b1372..5d0a2bf8f1d5 100644
--- a/polly/test/ScopDetectionDiagnostics/loop_has_multiple_exits.ll
+++ b/polly/test/ScopDetectionDiagnostics/loop_has_multiple_exits.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -disable-output 2>&1 < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output 2>&1 < %s | FileCheck %s -match-full-lines
 ;
 ; Derived from test-suite/MultiSource/Benchmarks/BitBench/uuencode/uuencode.c
 ;
diff --git a/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop-2.ll b/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop-2.ll
index 667ed7d18ab5..4370380cb71e 100644
--- a/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop-2.ll
+++ b/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
 
 ; CHECK: remark: <unknown>:0:0: Loop cannot be handled because not all latches are part of loop region.
 
diff --git a/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop.ll b/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop.ll
index 9dce56a3a3c4..05bd165d38cf 100644
--- a/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop.ll
+++ b/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
 
 ; CHECK: remark: <unknown>:0:0: Loop cannot be handled because not all latches are part of loop region.
 ; CHECK: remark: <unknown>:0:0: Loop cannot be handled because not all latches are part of loop region.
diff --git a/polly/test/ScopInfo/20110312-Fail-without-basicaa.ll b/polly/test/ScopInfo/20110312-Fail-without-basicaa.ll
index 94dd5824777c..6f08e433c4ee 100644
--- a/polly/test/ScopInfo/20110312-Fail-without-basicaa.ll
+++ b/polly/test/ScopInfo/20110312-Fail-without-basicaa.ll
@@ -1,5 +1,5 @@
 ; This should be run without alias analysis enabled.
-;RUN: opt %loadPolly -polly-scops  -disable-output < %s
+;RUN: opt %loadPolly '-passes=print<polly-function-scops>'  -disable-output < %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
 
 define i32 @main() nounwind {
diff --git a/polly/test/ScopInfo/20111108-Parameter-not-detected.ll b/polly/test/ScopInfo/20111108-Parameter-not-detected.ll
index f80177cb90e7..531e4149cffa 100644
--- a/polly/test/ScopInfo/20111108-Parameter-not-detected.ll
+++ b/polly/test/ScopInfo/20111108-Parameter-not-detected.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 declare void @foo()
diff --git a/polly/test/ScopInfo/2012-03-16-Crash-because-of-unsigned-in-scev.ll b/polly/test/ScopInfo/2012-03-16-Crash-because-of-unsigned-in-scev.ll
index b55d635947e5..d92b35de2de7 100644
--- a/polly/test/ScopInfo/2012-03-16-Crash-because-of-unsigned-in-scev.ll
+++ b/polly/test/ScopInfo/2012-03-16-Crash-because-of-unsigned-in-scev.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
 
diff --git a/polly/test/ScopInfo/2015-10-04-Crash-in-domain-generation.ll b/polly/test/ScopInfo/2015-10-04-Crash-in-domain-generation.ll
index d4d931fd2e0c..05165ed02a90 100644
--- a/polly/test/ScopInfo/2015-10-04-Crash-in-domain-generation.ll
+++ b/polly/test/ScopInfo/2015-10-04-Crash-in-domain-generation.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-scops -disable-output < %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops '-passes=print<polly-function-scops>' -disable-output < %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/Alias-0.ll b/polly/test/ScopInfo/Alias-0.ll
index 0fc4ad91b7db..4e7e8fa11a08 100644
--- a/polly/test/ScopInfo/Alias-0.ll
+++ b/polly/test/ScopInfo/Alias-0.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
-; RUN: opt %loadPolly -polly-print-scops -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/Alias-1.ll b/polly/test/ScopInfo/Alias-1.ll
index eab8c062f4ba..7a734f810976 100644
--- a/polly/test/ScopInfo/Alias-1.ll
+++ b/polly/test/ScopInfo/Alias-1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
-; RUN: opt %loadPolly -polly-print-scops -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/Alias-2.ll b/polly/test/ScopInfo/Alias-2.ll
index 64f1e0bc919d..9d3d44826237 100644
--- a/polly/test/ScopInfo/Alias-2.ll
+++ b/polly/test/ScopInfo/Alias-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
-; RUN: opt %loadPolly -polly-print-scops -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/Alias-3.ll b/polly/test/ScopInfo/Alias-3.ll
index 5e9b94e692bc..83d68ddf371b 100644
--- a/polly/test/ScopInfo/Alias-3.ll
+++ b/polly/test/ScopInfo/Alias-3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
-; RUN: opt %loadPolly -polly-print-scops -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/Alias-4.ll b/polly/test/ScopInfo/Alias-4.ll
index 4d5a91abb96f..bdcf729f4061 100644
--- a/polly/test/ScopInfo/Alias-4.ll
+++ b/polly/test/ScopInfo/Alias-4.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -disable-basic-aa -polly-print-scops -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
-; RUN: opt %loadPolly -disable-basic-aa -polly-print-scops -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
+; RUN: opt %loadPolly -aa-pipeline= '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
+; RUN: opt %loadPolly -aa-pipeline= '-passes=print<polly-detect>,print<polly-function-scops>' -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/BoundChecks/single-loop.ll b/polly/test/ScopInfo/BoundChecks/single-loop.ll
index bc96c907afc9..0ada318baabf 100644
--- a/polly/test/ScopInfo/BoundChecks/single-loop.ll
+++ b/polly/test/ScopInfo/BoundChecks/single-loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
 ;
 ; This only works after the post-dominator tree has been fixed.
 ;
diff --git a/polly/test/ScopInfo/BoundChecks/two-loops.ll b/polly/test/ScopInfo/BoundChecks/two-loops.ll
index 14e07f42a3ae..38ea23b80b9d 100644
--- a/polly/test/ScopInfo/BoundChecks/two-loops.ll
+++ b/polly/test/ScopInfo/BoundChecks/two-loops.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output< %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output< %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
 ;
 ; This only works after the post-dominator tree has fixed.
 ; XFAIL: *
diff --git a/polly/test/ScopInfo/NonAffine/div_backedge.ll b/polly/test/ScopInfo/NonAffine/div_backedge.ll
index a6aca032ef62..69af32d92325 100644
--- a/polly/test/ScopInfo/NonAffine/div_backedge.ll
+++ b/polly/test/ScopInfo/NonAffine/div_backedge.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void foo(float *A) {
 ;      for (long i = 1;; i++) {
diff --git a/polly/test/ScopInfo/NonAffine/div_domain.ll b/polly/test/ScopInfo/NonAffine/div_domain.ll
index f61c4eb459ed..27cc284f53c4 100644
--- a/polly/test/ScopInfo/NonAffine/div_domain.ll
+++ b/polly/test/ScopInfo/NonAffine/div_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void foo(float *A) {
 ;      for (long i = 0; i < 16; i++) {
diff --git a/polly/test/ScopInfo/NonAffine/invariant_loads_dependent_in_non_affine_region.ll b/polly/test/ScopInfo/NonAffine/invariant_loads_dependent_in_non_affine_region.ll
index f5d63dfb9d2c..4cf60324f99b 100644
--- a/polly/test/ScopInfo/NonAffine/invariant_loads_dependent_in_non_affine_region.ll
+++ b/polly/test/ScopInfo/NonAffine/invariant_loads_dependent_in_non_affine_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, int *B, int *C) {
 ;      for (int i = 0; i < 1000; i++)
diff --git a/polly/test/ScopInfo/NonAffine/modulo_backedge.ll b/polly/test/ScopInfo/NonAffine/modulo_backedge.ll
index dec63ca6813d..322720ae0633 100644
--- a/polly/test/ScopInfo/NonAffine/modulo_backedge.ll
+++ b/polly/test/ScopInfo/NonAffine/modulo_backedge.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Domain :=
 ; CHECK:   { Stmt_for_body[i0] : 0 <= i0 <= 6 };
diff --git a/polly/test/ScopInfo/NonAffine/modulo_domain.ll b/polly/test/ScopInfo/NonAffine/modulo_domain.ll
index f5ebec2b0346..cbd9d8901ce3 100644
--- a/polly/test/ScopInfo/NonAffine/modulo_domain.ll
+++ b/polly/test/ScopInfo/NonAffine/modulo_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; TODO: The new domain generation cannot handle modulo domain constraints,
 ;       hence modulo handling has been disabled completely. Once this is
diff --git a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll
index 837d9b21b16e..e38de4ab6aba 100644
--- a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll
+++ b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                   -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=SCALAR
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-process-unprofitable=false -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=PROFIT
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                   '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=SCALAR
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-process-unprofitable=false '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=PROFIT
 ;
 ; SCALAR:      Function: f
 ; SCALAR-NEXT: Region: %bb1---%bb13
diff --git a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll
index e39569abc52d..4e792e5e4e50 100644
--- a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll
+++ b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                       -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                        -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=ALL
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                       '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                        '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALL
 ;
 ; Here we have a non-affine loop (in the context of the loop nest)
 ; and also a non-affine access (A[k]). While we can always model the
diff --git a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll
index 75dd7ac26bb3..1845505a5a2d 100644
--- a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll
+++ b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                       -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                        -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=ALL
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                       '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                        '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALL
 ;
 ; Here we have a non-affine loop (in the context of the loop nest)
 ; and also a non-affine access (A[k]). While we can always model the
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_access_with_range_2.ll b/polly/test/ScopInfo/NonAffine/non_affine_access_with_range_2.ll
index 34b04933af86..2ba4065770bd 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_access_with_range_2.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_access_with_range_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A) {
 ;      for (int i = 0; i < 128; i++)
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_but_sdiv.ll b/polly/test/ScopInfo/NonAffine/non_affine_but_sdiv.ll
index 9955c88b2cfd..8ba1f013fda4 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_but_sdiv.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_but_sdiv.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:      Statements {
 ; CHECK-NEXT:     Stmt_for_body
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_but_srem.ll b/polly/test/ScopInfo/NonAffine/non_affine_but_srem.ll
index b194ee762e9f..059129168bad 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_but_srem.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_but_srem.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void pos(float *A, long n) {
 ;      for (long i = 0; i < 100; i++)
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_conditional_nested.ll b/polly/test/ScopInfo/NonAffine/non_affine_conditional_nested.ll
index 1f55530b137d..a25b27297b0f 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_conditional_nested.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_conditional_nested.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_affine_loop.ll b/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_affine_loop.ll
index 3511362304b4..1f66b8a096c9 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_affine_loop.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_affine_loop.ll
@@ -1,11 +1,11 @@
 ; RUN: opt %loadPolly -polly-allow-nonaffine-branches \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -polly-allow-nonaffine-loops=true \
-; RUN:     -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
+; RUN:     '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
 ; RUN: opt %loadPolly -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true \
-; RUN:     -polly-print-scops -disable-output < %s | FileCheck %s \
+; RUN:     '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s \
 ; RUN:     --check-prefix=ALL
 ;
 ; Negative test for INNERMOST.
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_non_affine_loop.ll b/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_non_affine_loop.ll
index c2e1e46f6f18..5127481dde29 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_non_affine_loop.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_non_affine_loop.ll
@@ -1,16 +1,16 @@
 ; RUN: opt %loadPolly -polly-allow-nonaffine-branches \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -polly-allow-nonaffine-loops=true \
-; RUN:     -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
+; RUN:     '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
 ; RUN: opt %loadPolly -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true \
-; RUN:     -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=ALL
+; RUN:     '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALL
 ; RUN: opt %loadPolly -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -polly-process-unprofitable=false \
 ; RUN:     -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true \
-; RUN:     -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=PROFIT
+; RUN:     '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
 ;
 ; Negative test for INNERMOST.
 ; At the moment we will optimistically assume A[i] in the conditional before the inner
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_float_compare.ll b/polly/test/ScopInfo/NonAffine/non_affine_float_compare.ll
index c62447b6c15c..ced23ab8f8df 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_float_compare.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_float_compare.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(float *A) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_loop_condition.ll b/polly/test/ScopInfo/NonAffine/non_affine_loop_condition.ll
index 873b44b9c8cf..ce00233fd7af 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_loop_condition.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_loop_condition.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops                                   -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-process-unprofitable=false -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=PROFIT
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-detect-reductions=false    -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=NO-REDUCTION
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops                                   '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-process-unprofitable=false '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-detect-reductions=false    '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=NO-REDUCTION
 ;
 ;    void f(int *A, int *C) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll b/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll
index 127bf80b9451..daf06a3b5f89 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops                                                                        -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-unprofitable-scalar-accs=true -polly-process-unprofitable=false -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops                                                                        '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-unprofitable-scalar-accs=true -polly-process-unprofitable=false '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
 ;
 ; Verify that we over approximate the read acces of A[j] in the last statement as j is
 ; computed in a non-affine loop we do not model.
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll b/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll
index de011e29aeea..be94e0e3307e 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, double A[], int INDEX[]) {
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_region_guaranteed_non-entry.ll b/polly/test/ScopInfo/NonAffine/non_affine_region_guaranteed_non-entry.ll
index 7303b4ea47fd..a4daca4d9f36 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_region_guaranteed_non-entry.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_region_guaranteed_non-entry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-detect -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-detect '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/NonAffine/whole-scop-non-affine-subregion-in-loop.ll b/polly/test/ScopInfo/NonAffine/whole-scop-non-affine-subregion-in-loop.ll
index 4f54d03d43fb..ba9146a8eca7 100644
--- a/polly/test/ScopInfo/NonAffine/whole-scop-non-affine-subregion-in-loop.ll
+++ b/polly/test/ScopInfo/NonAffine/whole-scop-non-affine-subregion-in-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
 ;
 ; Regression test that triggered a memory leak at some point (24947).
 ;
diff --git a/polly/test/ScopInfo/aliasing_conditional_alias_groups_1.ll b/polly/test/ScopInfo/aliasing_conditional_alias_groups_1.ll
index dc59fbfc66a8..a2836c199514 100644
--- a/polly/test/ScopInfo/aliasing_conditional_alias_groups_1.ll
+++ b/polly/test/ScopInfo/aliasing_conditional_alias_groups_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that there is no alias group because we either access A or B never both.
 ;
diff --git a/polly/test/ScopInfo/aliasing_conditional_alias_groups_2.ll b/polly/test/ScopInfo/aliasing_conditional_alias_groups_2.ll
index a19d60dd9147..15cea1c4f8cb 100644
--- a/polly/test/ScopInfo/aliasing_conditional_alias_groups_2.ll
+++ b/polly/test/ScopInfo/aliasing_conditional_alias_groups_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that we create two alias groups since the minimal/maximal accesses
 ; depend on %b.
diff --git a/polly/test/ScopInfo/aliasing_dead_access.ll b/polly/test/ScopInfo/aliasing_dead_access.ll
index 2a725cf3c855..400fea0573a1 100644
--- a/polly/test/ScopInfo/aliasing_dead_access.ll
+++ b/polly/test/ScopInfo/aliasing_dead_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that we do not create a SCoP if there is no statement executed.
 ;
diff --git a/polly/test/ScopInfo/aliasing_many_arrays_to_compare.ll b/polly/test/ScopInfo/aliasing_many_arrays_to_compare.ll
index 937d4ada3ec9..a1b954cb63ee 100644
--- a/polly/test/ScopInfo/aliasing_many_arrays_to_compare.ll
+++ b/polly/test/ScopInfo/aliasing_many_arrays_to_compare.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output \
-; RUN:                < %s | FileCheck %s --check-prefix=FOUND
-; RUN: opt %loadPolly -polly-print-scops -disable-output \
-; RUN:                -polly-rtc-max-arrays-per-group=3 < %s | FileCheck %s \
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output \
+; RUN:                < %s 2>&1 | FileCheck %s --check-prefix=FOUND
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output \
+; RUN:                -polly-rtc-max-arrays-per-group=3 < %s 2>&1 | FileCheck %s \
 ; RUN:                --check-prefix=IGNORED
 ;
 ; FOUND: Function: foo
diff --git a/polly/test/ScopInfo/aliasing_many_read_only_acesses.ll b/polly/test/ScopInfo/aliasing_many_read_only_acesses.ll
index c22cfe55e118..9447744a12a8 100644
--- a/polly/test/ScopInfo/aliasing_many_read_only_acesses.ll
+++ b/polly/test/ScopInfo/aliasing_many_read_only_acesses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Assumed Context:
 ; CHECK-NEXT: { : }
diff --git a/polly/test/ScopInfo/aliasing_multiple_alias_groups.ll b/polly/test/ScopInfo/aliasing_multiple_alias_groups.ll
index 16cb3dc0f5ac..5c4a864ec777 100644
--- a/polly/test/ScopInfo/aliasing_multiple_alias_groups.ll
+++ b/polly/test/ScopInfo/aliasing_multiple_alias_groups.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output          < %s | FileCheck %s --check-prefix=NOAA
-; RUN: opt %loadPolly -polly-print-scops -disable-output -tbaa    < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -aa-pipeline=         < %s 2>&1 | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -aa-pipeline=tbaa    < %s 2>&1 | FileCheck %s --check-prefix=TBAA
 ;
 ;    void jd(int *Int0, int *Int1, float *Float0, float *Float1) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/ScopInfo/aliasing_with_non_affine_access.ll b/polly/test/ScopInfo/aliasing_with_non_affine_access.ll
index 056b644cd5ed..76bc18e8ac53 100644
--- a/polly/test/ScopInfo/aliasing_with_non_affine_access.ll
+++ b/polly/test/ScopInfo/aliasing_with_non_affine_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-ast -polly-process-unprofitable -polly-allow-nonaffine -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-process-unprofitable -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; @test1
 ; Make sure we generate the correct aliasing check for a fixed-size memset operation.
diff --git a/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll b/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll
index d170a50e26fc..b37b560599ad 100644
--- a/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll
+++ b/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll
@@ -1,14 +1,14 @@
 ; RUN: opt %loadPolly -disable-output -polly-invariant-load-hoisting \
 ; RUN: -polly-allow-dereference-of-all-function-parameters \
-; RUN: -polly-print-scops < %s | FileCheck %s --check-prefix=SCOP
+; RUN: '-passes=print<polly-function-scops>' < %s 2>&1 | FileCheck %s --check-prefix=SCOP
 
 ; RUN: opt %loadPolly -S -polly-invariant-load-hoisting \
-; RUN: -polly-codegen < %s | FileCheck %s --check-prefix=CODE-RTC
+; RUN: -passes=polly-codegen < %s 2>&1 | FileCheck %s --check-prefix=CODE-RTC
 
 
 ; RUN: opt %loadPolly -S -polly-invariant-load-hoisting \
 ; RUN: -polly-allow-dereference-of-all-function-parameters \
-; RUN: -polly-codegen < %s | FileCheck %s --check-prefix=CODE
+; RUN: -passes=polly-codegen < %s 2>&1 | FileCheck %s --check-prefix=CODE
 
 ; SCOP:      Function: hoge
 ; SCOP-NEXT: Region: %bb15---%bb37
diff --git a/polly/test/ScopInfo/assume_gep_bounds.ll b/polly/test/ScopInfo/assume_gep_bounds.ll
index d0ce47148071..7b7fd15e3346 100644
--- a/polly/test/ScopInfo/assume_gep_bounds.ll
+++ b/polly/test/ScopInfo/assume_gep_bounds.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ;    void foo(float A[][20][30], long n, long m, long p) {
 ;      for (long i = 0; i < n; i++)
diff --git a/polly/test/ScopInfo/assume_gep_bounds_2.ll b/polly/test/ScopInfo/assume_gep_bounds_2.ll
index e327195da94c..3ada0f817300 100644
--- a/polly/test/ScopInfo/assume_gep_bounds_2.ll
+++ b/polly/test/ScopInfo/assume_gep_bounds_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output < %s \
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN:  -polly-precise-inbounds | FileCheck %s
 ;
 ;    void foo(float A[restrict][20], float B[restrict][20], long n, long m,
diff --git a/polly/test/ScopInfo/assume_gep_bounds_many.ll b/polly/test/ScopInfo/assume_gep_bounds_many.ll
index 261491564fc2..2106b96ecd56 100644
--- a/polly/test/ScopInfo/assume_gep_bounds_many.ll
+++ b/polly/test/ScopInfo/assume_gep_bounds_many.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -disable-output -polly-print-scops -polly-ignore-aliasing \
-; RUN:    < %s | FileCheck %s
+; RUN: opt %loadPolly -disable-output '-passes=print<polly-function-scops>' -polly-ignore-aliasing \
+; RUN:    < %s 2>&1 | FileCheck %s
 
 ; CHECK: Assumed Context:
 ; CHECK-NEXT: [n1_a, n1_b, n1_c, n1_d, n2_a, n2_b, n2_c, n2_d, n3_a, n3_b, n3_c, n3_d, n4_a, n4_b, n4_c, n4_d, n5_a, n5_b, n5_c, n5_d, n6_a, n6_b, n6_c, n6_d, n7_a, n7_b, n7_c, n7_d, n8_a, n8_b, n8_c, n8_d, n9_a, n9_b, n9_c, n9_d, p1_b, p1_c, p1_d, p2_b, p2_c, p2_d, p3_b, p3_c, p3_d, p4_b, p4_c, p4_d, p5_b, p5_c, p5_d, p6_b, p6_c, p6_d, p7_b, p7_c, p7_d, p8_b, p8_c, p8_d, p9_b, p9_c, p9_d] -> {  : p1_b >= n1_b and p1_c >= n1_c and p1_d >= n1_d and p2_b >= n2_b and p2_c >= n2_c and p2_d >= n2_d and p3_b >= n3_b and p3_c >= n3_c and p3_d >= n3_d and p4_b >= n4_b and p4_c >= n4_c and p4_d >= n4_d and p5_b >= n5_b and p5_c >= n5_c and p5_d >= n5_d and p6_b >= n6_b and p6_c >= n6_c and p6_d >= n6_d and p7_b >= n7_b and p7_c >= n7_c and p7_d >= n7_d and p8_b >= n8_b and p8_c >= n8_c and p8_d >= n8_d and p9_b >= n9_b and p9_c >= n9_c and p9_d >= n9_d }
diff --git a/polly/test/ScopInfo/avoid_new_parameters_from_geps.ll b/polly/test/ScopInfo/avoid_new_parameters_from_geps.ll
index 0e17eb1d3668..e81b791f9312 100644
--- a/polly/test/ScopInfo/avoid_new_parameters_from_geps.ll
+++ b/polly/test/ScopInfo/avoid_new_parameters_from_geps.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that we do no introduce a parameter here that is actually not needed.
 ;
diff --git a/polly/test/ScopInfo/bool-addrec.ll b/polly/test/ScopInfo/bool-addrec.ll
index 1924a4b5266b..51687cd9caa5 100644
--- a/polly/test/ScopInfo/bool-addrec.ll
+++ b/polly/test/ScopInfo/bool-addrec.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -disable-output -polly-print-ast -polly-process-unprofitable < %s | FileCheck %s
+; RUN: opt %loadPolly -disable-output '-passes=print<polly-ast>' -polly-process-unprofitable < %s 2>&1 | FileCheck %s
 
 ; CHECK:      for (int c0 = 0; c0 <= 19999; c0 += 1) {
 ; CHECK-NEXT:   if (c0 % 2 == 0)
diff --git a/polly/test/ScopInfo/bounded_loop_assumptions.ll b/polly/test/ScopInfo/bounded_loop_assumptions.ll
index d472c7586c53..6b8acfc97c60 100644
--- a/polly/test/ScopInfo/bounded_loop_assumptions.ll
+++ b/polly/test/ScopInfo/bounded_loop_assumptions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; The assumed context is tricky here as the equality test for the inner loop
 ; allows an "unbounded" loop trip count. We assume that does not happen, thus
diff --git a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-2.ll b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-2.ll
index 5c5f264aab60..bbf151f0206a 100644
--- a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-2.ll
+++ b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-2.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | \
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | \
 ; RUN:     FileCheck %s -check-prefix=DETECT
 
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | \
 ; RUN:     FileCheck %s -check-prefix=SCOP
 
 ; DETECT: Valid Region for Scop: loop => barrier
diff --git a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-3.ll b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-3.ll
index d69d3a16c0d7..9b72a74e8628 100644
--- a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-3.ll
+++ b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-3.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | \
 ; RUN:     FileCheck %s -check-prefix=NONAFFINE
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output \
-; RUN:     -polly-allow-nonaffine-branches=false < %s | \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output \
+; RUN:     -polly-allow-nonaffine-branches=false < %s 2>&1 | \
 ; RUN:     FileCheck %s -check-prefix=NO-NONEAFFINE
 
 ; NONAFFINE:      Statements {
diff --git a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations.ll b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations.ll
index 57918fa5c92d..c6b1cddeaf85 100644
--- a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations.ll
+++ b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | \
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | \
 ; RUN:     FileCheck %s -check-prefix=NONAFFINE
-; RUN: opt %loadPolly -polly-print-scops -disable-output \
-; RUN:     -polly-allow-nonaffine-branches=false < %s | \
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output \
+; RUN:     -polly-allow-nonaffine-branches=false < %s 2>&1 | \
 ; RUN:     FileCheck %s -check-prefix=NO-NONEAFFINE
 
 ; NONAFFINE-NOT: Statements
diff --git a/polly/test/ScopInfo/bug_2010_10_22.ll b/polly/test/ScopInfo/bug_2010_10_22.ll
index 7ba996b6d0f1..2e492e7633a0 100644
--- a/polly/test/ScopInfo/bug_2010_10_22.ll
+++ b/polly/test/ScopInfo/bug_2010_10_22.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/ScopInfo/bug_2011_1_5.ll b/polly/test/ScopInfo/bug_2011_1_5.ll
index 95c25f9d9cdb..ce815e57e627 100644
--- a/polly/test/ScopInfo/bug_2011_1_5.ll
+++ b/polly/test/ScopInfo/bug_2011_1_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
 
 ; Bug description: Alias Analysis thinks IntToPtrInst aliases with alloca instructions created by IndependentBlocks Pass.
 ;                  This will trigger the assertion when we are verifying the SCoP after IndependentBlocks.
diff --git a/polly/test/ScopInfo/bug_scev_not_fully_eval.ll b/polly/test/ScopInfo/bug_scev_not_fully_eval.ll
index 89d5f318829e..8711f8e58a21 100644
--- a/polly/test/ScopInfo/bug_scev_not_fully_eval.ll
+++ b/polly/test/ScopInfo/bug_scev_not_fully_eval.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | not FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | not FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @edge.8265 = external global [72 x i32], align 32 ; <ptr> [#uses=1]
diff --git a/polly/test/ScopInfo/cfg_consequences.ll b/polly/test/ScopInfo/cfg_consequences.ll
index 84f94b135735..bd23ec7fc3bf 100644
--- a/polly/test/ScopInfo/cfg_consequences.ll
+++ b/polly/test/ScopInfo/cfg_consequences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; void consequences(int *A, int bool_cond, int lhs, int rhs) {
 ;
diff --git a/polly/test/ScopInfo/complex-branch-structure.ll b/polly/test/ScopInfo/complex-branch-structure.ll
index 24ebdcf213f8..69eb716e6877 100644
--- a/polly/test/ScopInfo/complex-branch-structure.ll
+++ b/polly/test/ScopInfo/complex-branch-structure.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 
 ; We build a scop of the following form to check that the domain construction
diff --git a/polly/test/ScopInfo/complex-condition.ll b/polly/test/ScopInfo/complex-condition.ll
index 31d34b033725..348446a40007 100644
--- a/polly/test/ScopInfo/complex-condition.ll
+++ b/polly/test/ScopInfo/complex-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/complex-expression.ll b/polly/test/ScopInfo/complex-expression.ll
index 1822c9de852a..1340b0e0c981 100644
--- a/polly/test/ScopInfo/complex-expression.ll
+++ b/polly/test/ScopInfo/complex-expression.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/complex-loop-nesting.ll b/polly/test/ScopInfo/complex-loop-nesting.ll
index 97a9bfd939d5..a0e4a1e92050 100644
--- a/polly/test/ScopInfo/complex-loop-nesting.ll
+++ b/polly/test/ScopInfo/complex-loop-nesting.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/complex-successor-structure-2.ll b/polly/test/ScopInfo/complex-successor-structure-2.ll
index 6bb7bb14a8cc..eceadc818b48 100644
--- a/polly/test/ScopInfo/complex-successor-structure-2.ll
+++ b/polly/test/ScopInfo/complex-successor-structure-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 
diff --git a/polly/test/ScopInfo/complex-successor-structure-3.ll b/polly/test/ScopInfo/complex-successor-structure-3.ll
index 14c3fc1babeb..b5ba6958c57b 100644
--- a/polly/test/ScopInfo/complex-successor-structure-3.ll
+++ b/polly/test/ScopInfo/complex-successor-structure-3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -disable-output -polly-print-scops \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
+; RUN: opt %loadPolly -disable-output '-passes=print<polly-function-scops>' \
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
 ;
 ; Check that propagation of domains from A(X) to A(X+1) will keep the
 ; domains small and concise.
diff --git a/polly/test/ScopInfo/complex-successor-structure.ll b/polly/test/ScopInfo/complex-successor-structure.ll
index 364344045a6a..f39ab9bfa29b 100644
--- a/polly/test/ScopInfo/complex-successor-structure.ll
+++ b/polly/test/ScopInfo/complex-successor-structure.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 
diff --git a/polly/test/ScopInfo/complex_domain_binary_condition.ll b/polly/test/ScopInfo/complex_domain_binary_condition.ll
index cec26855debb..4dd39733eae2 100644
--- a/polly/test/ScopInfo/complex_domain_binary_condition.ll
+++ b/polly/test/ScopInfo/complex_domain_binary_condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Low complexity assumption: {  : false }
diff --git a/polly/test/ScopInfo/complex_execution_context.ll b/polly/test/ScopInfo/complex_execution_context.ll
index 164254308fa9..c58d6fe9a50a 100644
--- a/polly/test/ScopInfo/complex_execution_context.ll
+++ b/polly/test/ScopInfo/complex_execution_context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/cond_constant_in_loop.ll b/polly/test/ScopInfo/cond_constant_in_loop.ll
index ef7d857e1084..45cae34e1af2 100644
--- a/polly/test/ScopInfo/cond_constant_in_loop.ll
+++ b/polly/test/ScopInfo/cond_constant_in_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ;void f(long a[], long N, long M) {
 ;  long i, j, k;
diff --git a/polly/test/ScopInfo/cond_in_loop.ll b/polly/test/ScopInfo/cond_in_loop.ll
index 2d435f6a6a93..2101b25e4799 100644
--- a/polly/test/ScopInfo/cond_in_loop.ll
+++ b/polly/test/ScopInfo/cond_in_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ;void f(long a[], long N, long M) {
 ;  long i, j, k;
diff --git a/polly/test/ScopInfo/condition-after-error-block-2.ll b/polly/test/ScopInfo/condition-after-error-block-2.ll
index 695d864e483c..e3025d0ca259 100644
--- a/polly/test/ScopInfo/condition-after-error-block-2.ll
+++ b/polly/test/ScopInfo/condition-after-error-block-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; Verify that we do not allow PHI nodes such as %phi, if they reference an error
 ; block and are used by anything else than a terminator instruction.
diff --git a/polly/test/ScopInfo/condition-after-error-block-before-scop.ll b/polly/test/ScopInfo/condition-after-error-block-before-scop.ll
index 184be3642f0c..7a4d1467de46 100644
--- a/polly/test/ScopInfo/condition-after-error-block-before-scop.ll
+++ b/polly/test/ScopInfo/condition-after-error-block-before-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/polly/test/ScopInfo/condtion-after-error-block.ll b/polly/test/ScopInfo/condtion-after-error-block.ll
index 92e743e2d879..1a8681f829e4 100644
--- a/polly/test/ScopInfo/condtion-after-error-block.ll
+++ b/polly/test/ScopInfo/condtion-after-error-block.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; Verify that we allow scops containing uniform branch conditions, where all
 ; but one incoming block comes from an error condition.
diff --git a/polly/test/ScopInfo/const_srem_sdiv.ll b/polly/test/ScopInfo/const_srem_sdiv.ll
index 3acca980da70..cf243fc74a9a 100644
--- a/polly/test/ScopInfo/const_srem_sdiv.ll
+++ b/polly/test/ScopInfo/const_srem_sdiv.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
 ;
 ; See http://research.microsoft.com/pubs/151917/divmodnote-letter.pdf
 ;
diff --git a/polly/test/ScopInfo/constant-non-integer-branch-condition.ll b/polly/test/ScopInfo/constant-non-integer-branch-condition.ll
index fc95a4cc7891..8c8beac8f304 100644
--- a/polly/test/ScopInfo/constant-non-integer-branch-condition.ll
+++ b/polly/test/ScopInfo/constant-non-integer-branch-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; At some point this caused a problem in the domain generation as we
 ; assumed any constant branch condition to be valid. However, only constant
diff --git a/polly/test/ScopInfo/constant_factor_in_parameter.ll b/polly/test/ScopInfo/constant_factor_in_parameter.ll
index 1f0173c0edf9..ca7d094be300 100644
--- a/polly/test/ScopInfo/constant_factor_in_parameter.ll
+++ b/polly/test/ScopInfo/constant_factor_in_parameter.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -disable-output -polly-print-scops < %s | FileCheck %s
-; RUN: opt %loadPolly -disable-output -polly-print-function-scops < %s | FileCheck %s
+; RUN: opt %loadPolly -disable-output '-passes=print<polly-function-scops>' < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -disable-output '-passes=print<polly-function-scops>' < %s 2>&1 | FileCheck %s
 ;
 ; Check that the constant part of the N * M * 4 expression is not part of the
 ; parameter but explicit in the access function. This can avoid existentially
diff --git a/polly/test/ScopInfo/constant_functions_outside_scop_as_unknown.ll b/polly/test/ScopInfo/constant_functions_outside_scop_as_unknown.ll
index 38b2b8958e2f..afb8ec3eabed 100644
--- a/polly/test/ScopInfo/constant_functions_outside_scop_as_unknown.ll
+++ b/polly/test/ScopInfo/constant_functions_outside_scop_as_unknown.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/polly/test/ScopInfo/constant_start_integer.ll b/polly/test/ScopInfo/constant_start_integer.ll
index aa6640c98f73..94e43b324f98 100644
--- a/polly/test/ScopInfo/constant_start_integer.ll
+++ b/polly/test/ScopInfo/constant_start_integer.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(float *input) {
diff --git a/polly/test/ScopInfo/debug_call.ll b/polly/test/ScopInfo/debug_call.ll
index 93b5bc520a00..ba74e68c9f36 100644
--- a/polly/test/ScopInfo/debug_call.ll
+++ b/polly/test/ScopInfo/debug_call.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-debug-func=dbg_printf -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-debug-func=dbg_printf '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 ;
 ; Check that the call to dbg_printf is accepted as a debug-function.
 ;
diff --git a/polly/test/ScopInfo/delinearize-together-all-data-refs.ll b/polly/test/ScopInfo/delinearize-together-all-data-refs.ll
index 108392b27f07..ac17ba005bd0 100644
--- a/polly/test/ScopInfo/delinearize-together-all-data-refs.ll
+++ b/polly/test/ScopInfo/delinearize-together-all-data-refs.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
 ;   for (long i = 0; i < n-3; i++)
diff --git a/polly/test/ScopInfo/div_by_zero.ll b/polly/test/ScopInfo/div_by_zero.ll
index 2205b85a9ebc..74380f7a4c71 100644
--- a/polly/test/ScopInfo/div_by_zero.ll
+++ b/polly/test/ScopInfo/div_by_zero.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/do-not-model-error-block-accesses.ll b/polly/test/ScopInfo/do-not-model-error-block-accesses.ll
index 997e0d4b37cf..563e5d11474b 100644
--- a/polly/test/ScopInfo/do-not-model-error-block-accesses.ll
+++ b/polly/test/ScopInfo/do-not-model-error-block-accesses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
 
 ; Check that we do not crash on this input. Earlier this indeed crashed as
 ; we tried to model the access functions in an error block.
diff --git a/polly/test/ScopInfo/eager-binary-and-or-conditions.ll b/polly/test/ScopInfo/eager-binary-and-or-conditions.ll
index e9ad63c51b85..ee846d3ca98c 100644
--- a/polly/test/ScopInfo/eager-binary-and-or-conditions.ll
+++ b/polly/test/ScopInfo/eager-binary-and-or-conditions.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output< %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output< %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
 ;
 ; void or(float *A, long n, long m) {
 ;   for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/early_exit_for_complex_domains.ll b/polly/test/ScopInfo/early_exit_for_complex_domains.ll
index a72ea031c236..2a8e6d15fea4 100644
--- a/polly/test/ScopInfo/early_exit_for_complex_domains.ll
+++ b/polly/test/ScopInfo/early_exit_for_complex_domains.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
 ;
 ; Check we do not crash.
 ;
diff --git a/polly/test/ScopInfo/error-blocks-1.ll b/polly/test/ScopInfo/error-blocks-1.ll
index 03353edf297a..e0b59e01d13d 100644
--- a/polly/test/ScopInfo/error-blocks-1.ll
+++ b/polly/test/ScopInfo/error-blocks-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:         Context:
 ; CHECK-NEXT:    [N] -> {  : -2147483648 <= N <= 2147483647 }
diff --git a/polly/test/ScopInfo/error-blocks-2.ll b/polly/test/ScopInfo/error-blocks-2.ll
index 29095dacacfb..59096e3315f5 100644
--- a/polly/test/ScopInfo/error-blocks-2.ll
+++ b/polly/test/ScopInfo/error-blocks-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:         Invariant Accesses: {
 ; CHECK-NEXT:            ReadAccess :=	[Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/escaping_empty_scop.ll b/polly/test/ScopInfo/escaping_empty_scop.ll
index 8837e19eefe4..e27130af952a 100644
--- a/polly/test/ScopInfo/escaping_empty_scop.ll
+++ b/polly/test/ScopInfo/escaping_empty_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void g();
 ;    int f(int *A) {
diff --git a/polly/test/ScopInfo/exit-phi-1.ll b/polly/test/ScopInfo/exit-phi-1.ll
index 8e6c5fb9e211..41b56dde043a 100644
--- a/polly/test/ScopInfo/exit-phi-1.ll
+++ b/polly/test/ScopInfo/exit-phi-1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-codegen -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -passes=polly-codegen -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
 ;
 ; Check for correct code generation of exit PHIs, even if the same PHI value
 ; is used again inside the the SCoP.
diff --git a/polly/test/ScopInfo/exit-phi-2.ll b/polly/test/ScopInfo/exit-phi-2.ll
index d218d5fa039b..c2b463f657ea 100644
--- a/polly/test/ScopInfo/exit-phi-2.ll
+++ b/polly/test/ScopInfo/exit-phi-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that there is no MK_ExitPHI READ access.
 ;
diff --git a/polly/test/ScopInfo/exit_phi_accesses-2.ll b/polly/test/ScopInfo/exit_phi_accesses-2.ll
index e376f0df9d54..cfc385dc6aba 100644
--- a/polly/test/ScopInfo/exit_phi_accesses-2.ll
+++ b/polly/test/ScopInfo/exit_phi_accesses-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: Function: foo
 ;
diff --git a/polly/test/ScopInfo/exit_phi_accesses.ll b/polly/test/ScopInfo/exit_phi_accesses.ll
index f4fbe31f6b24..c598e411739c 100644
--- a/polly/test/ScopInfo/exit_phi_accesses.ll
+++ b/polly/test/ScopInfo/exit_phi_accesses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; Check that PHI nodes only create PHI access and nothing else (e.g. unnecessary
 ; SCALAR accesses). In this case, for a PHI in the exit node, hence there is no
diff --git a/polly/test/ScopInfo/expensive-boundary-context.ll b/polly/test/ScopInfo/expensive-boundary-context.ll
index 7001b96acd21..dd660c543f2c 100644
--- a/polly/test/ScopInfo/expensive-boundary-context.ll
+++ b/polly/test/ScopInfo/expensive-boundary-context.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output \
-; RUN:                < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output \
+; RUN:                < %s 2>&1 | FileCheck %s
 
 ; CHECK-NOT:   Assumed Context:
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/ScopInfo/extract_constant_factor_introduces_new_parameter.ll b/polly/test/ScopInfo/extract_constant_factor_introduces_new_parameter.ll
index 89ca344fdf54..9475a870e4b5 100644
--- a/polly/test/ScopInfo/extract_constant_factor_introduces_new_parameter.ll
+++ b/polly/test/ScopInfo/extract_constant_factor_introduces_new_parameter.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
 
 ; CHECK: Valid Region for Scop: bb10 => bb16
 
diff --git a/polly/test/ScopInfo/full-function.ll b/polly/test/ScopInfo/full-function.ll
index 670472576fe7..bb2d12fb0e3f 100644
--- a/polly/test/ScopInfo/full-function.ll
+++ b/polly/test/ScopInfo/full-function.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output -polly-detect-full-functions < %s \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-detect-full-functions < %s 2>&1 \
 ; RUN: | FileCheck %s -check-prefix=FULL
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN: | FileCheck %s -check-prefix=WITHOUT-FULL
 
 ; FULL:      Region: %bb---FunctionExit
diff --git a/polly/test/ScopInfo/granularity_same_name.ll b/polly/test/ScopInfo/granularity_same_name.ll
index 1ebf5c6f71a2..4e8dd1840890 100644
--- a/polly/test/ScopInfo/granularity_same_name.ll
+++ b/polly/test/ScopInfo/granularity_same_name.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb           -polly-use-llvm-names=0 -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines -check-prefix=IDX
-; RUN: opt %loadPolly -polly-stmt-granularity=bb           -polly-use-llvm-names=1 -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines -check-prefix=BB
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-use-llvm-names=0 -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines -check-prefix=IDX
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-use-llvm-names=1 -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines -check-prefix=BB
+; RUN: opt %loadPolly -polly-stmt-granularity=bb           -polly-use-llvm-names=0 '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=IDX
+; RUN: opt %loadPolly -polly-stmt-granularity=bb           -polly-use-llvm-names=1 '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=BB
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-use-llvm-names=0 '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=IDX
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-use-llvm-names=1 '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=BB
 ;
 ; Check that the statement has the same name, regardless of how the
 ; basic block is split into multiple statements.
diff --git a/polly/test/ScopInfo/granularity_scalar-indep.ll b/polly/test/ScopInfo/granularity_scalar-indep.ll
index fe509b468272..b28060d87180 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 ;
 ; Split a block into two independent statements that share no scalar.
 ; This case has the instructions of the two statements interleaved, such that
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi1.ll b/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi1.ll
index 56bc11aed28d..fb2bfa663ef2 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi1.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 ;
 ; Two PHIs, cross-referencing each other. The PHI READs must be carried-out
 ; before the PHI WRITEs to ensure that the value when entering the block is
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi2.ll b/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi2.ll
index f46cf4e6a0a2..aa066b5cd46c 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi2.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 ;
 ; Two PHIs, cross-referencing each other. The PHI READs must be carried-out
 ; before the PHI WRITEs to ensure that the value when entering the block is
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_epilogue.ll b/polly/test/ScopInfo/granularity_scalar-indep_epilogue.ll
index e202e38f0844..da326191762a 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_epilogue.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_epilogue.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 ;
 ; Split a block into two independent statements that share no scalar.
 ; This case has an independent statement just for PHI writes.
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_epilogue_last.ll b/polly/test/ScopInfo/granularity_scalar-indep_epilogue_last.ll
index 40af34bfb067..19484319ee35 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_epilogue_last.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_epilogue_last.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 ;
 ; Check that the PHI Write of value that is defined in the same basic
 ; block is in the statement where it is defined.
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_noepilogue.ll b/polly/test/ScopInfo/granularity_scalar-indep_noepilogue.ll
index 9a0d207c0c2a..484156abdbcd 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_noepilogue.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_noepilogue.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 ;
 ; This case has no explicit epilogue for PHI writes because it would
 ; have a scalar dependency to the previous statement.
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_ordered-2.ll b/polly/test/ScopInfo/granularity_scalar-indep_ordered-2.ll
index d093806bc9cc..ec69b8000bf5 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_ordered-2.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_ordered-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 ;
 ; This case should be split into two statements because {X[0], Y[0]}
 ; and {A[0], B[0]} do not intersect.
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_ordered.ll b/polly/test/ScopInfo/granularity_scalar-indep_ordered.ll
index b1d2936882aa..a7f8e2697cac 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_ordered.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_ordered.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 ;
 ; This case cannot be split into two statements because the order of
 ; loads and store would be violated.
diff --git a/polly/test/ScopInfo/i1_params.ll b/polly/test/ScopInfo/i1_params.ll
index 1cb1329b08f9..28eb838b56b0 100644
--- a/polly/test/ScopInfo/i1_params.ll
+++ b/polly/test/ScopInfo/i1_params.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that both a signed as well as an unsigned extended i1 parameter
 ; is represented correctly.
diff --git a/polly/test/ScopInfo/infeasible-rtc.ll b/polly/test/ScopInfo/infeasible-rtc.ll
index ef96627e640e..5540b2365ce3 100644
--- a/polly/test/ScopInfo/infeasible-rtc.ll
+++ b/polly/test/ScopInfo/infeasible-rtc.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 \
 ; RUN:  | FileCheck %s -check-prefix=DETECT
 
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN:  | FileCheck %s -check-prefix=SCOPS
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/polly/test/ScopInfo/infeasible_invalid_context.ll b/polly/test/ScopInfo/infeasible_invalid_context.ll
index 2c299f06c12e..86ecb4053d5a 100644
--- a/polly/test/ScopInfo/infeasible_invalid_context.ll
+++ b/polly/test/ScopInfo/infeasible_invalid_context.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 \
 ; RUN:  | FileCheck %s -check-prefix=DETECT
 
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN:  | FileCheck %s -check-prefix=SCOPS
 
 ; DETECT: Valid Region for Scop: if.end116 => for.inc216
diff --git a/polly/test/ScopInfo/int2ptr_ptr2int.ll b/polly/test/ScopInfo/int2ptr_ptr2int.ll
index 9fadc5a8eb28..f375e79e807d 100644
--- a/polly/test/ScopInfo/int2ptr_ptr2int.ll
+++ b/polly/test/ScopInfo/int2ptr_ptr2int.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen < %s 2>&1 | FileCheck %s --check-prefix=IR
 ;
 ;    void f(long *A, long *ptr, long val) {
 ;      for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/int2ptr_ptr2int_2.ll b/polly/test/ScopInfo/int2ptr_ptr2int_2.ll
index 97878f7091b1..1c8251361b51 100644
--- a/polly/test/ScopInfo/int2ptr_ptr2int_2.ll
+++ b/polly/test/ScopInfo/int2ptr_ptr2int_2.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-print-scops \
-; RUN: -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -S -polly-codegen \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' \
+; RUN: -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -S -passes=polly-codegen \
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s --check-prefix=IR
 ;
 ;    void f(long *A, long *B, long *ptr, long val) {
 ;      for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/integers.ll b/polly/test/ScopInfo/integers.ll
index b608bf84cffa..87bc31e214ff 100644
--- a/polly/test/ScopInfo/integers.ll
+++ b/polly/test/ScopInfo/integers.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; Check that we correctly convert integers to isl values.
 
diff --git a/polly/test/ScopInfo/inter-error-bb-dependence.ll b/polly/test/ScopInfo/inter-error-bb-dependence.ll
index 4e23de7e6a99..00f482267a0b 100644
--- a/polly/test/ScopInfo/inter-error-bb-dependence.ll
+++ b/polly/test/ScopInfo/inter-error-bb-dependence.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops -disable-output < %s 2>&1 > /dev/null | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 > /dev/null | FileCheck %s
 ;
 ; Error statements (%bb33) do not require their uses to be verified.
 ; In this case it uses %tmp32 from %bb31 which is not available because
diff --git a/polly/test/ScopInfo/inter_bb_scalar_dep.ll b/polly/test/ScopInfo/inter_bb_scalar_dep.ll
index 456f7a773f04..0af814516038 100644
--- a/polly/test/ScopInfo/inter_bb_scalar_dep.ll
+++ b/polly/test/ScopInfo/inter_bb_scalar_dep.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-scops \
-; RUN: -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' \
+; RUN: -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/intra-non-affine-stmt-phi-node.ll b/polly/test/ScopInfo/intra-non-affine-stmt-phi-node.ll
index 859972b27402..0fc635abae45 100644
--- a/polly/test/ScopInfo/intra-non-affine-stmt-phi-node.ll
+++ b/polly/test/ScopInfo/intra-non-affine-stmt-phi-node.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output \
-; RUN:     < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
+; RUN:     < %s 2>&1 | FileCheck %s
 
 ; CHECK:      Statements {
 ; CHECK-NEXT:     Stmt_loop__TO__backedge
diff --git a/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll b/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll
index 37f4e0513ed3..209c475b5bbb 100644
--- a/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll
+++ b/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/intra_bb_scalar_dep.ll b/polly/test/ScopInfo/intra_bb_scalar_dep.ll
index 0252273d3107..8ad5ac175802 100644
--- a/polly/test/ScopInfo/intra_bb_scalar_dep.ll
+++ b/polly/test/ScopInfo/intra_bb_scalar_dep.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/intrinsics.ll b/polly/test/ScopInfo/intrinsics.ll
index 853429341381..8b484cc21b52 100644
--- a/polly/test/ScopInfo/intrinsics.ll
+++ b/polly/test/ScopInfo/intrinsics.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-print-instructions -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-print-instructions -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify that we remove the ignored intrinsics from the instruction list.
 ;
diff --git a/polly/test/ScopInfo/invalid_add_rec_after_invariant_load_remapping.ll b/polly/test/ScopInfo/invalid_add_rec_after_invariant_load_remapping.ll
index 8d0de03e9866..d9f64d5eba9c 100644
--- a/polly/test/ScopInfo/invalid_add_rec_after_invariant_load_remapping.ll
+++ b/polly/test/ScopInfo/invalid_add_rec_after_invariant_load_remapping.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
 ;
 ; This crashed at some point as we place %1 and %4 in the same equivalence class
 ; for invariant loads and when we remap SCEVs to use %4 instead of %1 AddRec SCEVs
diff --git a/polly/test/ScopInfo/invalidate_iterator_during_MA_removal.ll b/polly/test/ScopInfo/invalidate_iterator_during_MA_removal.ll
index dcb0ad301ba3..0f48052a40e1 100644
--- a/polly/test/ScopInfo/invalidate_iterator_during_MA_removal.ll
+++ b/polly/test/ScopInfo/invalidate_iterator_during_MA_removal.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
 ;
 ; Check that no invalidated iterator is accessed while elements from
 ; the list of MemoryAccesses are removed.
diff --git a/polly/test/ScopInfo/invariant-load-instlist.ll b/polly/test/ScopInfo/invariant-load-instlist.ll
index 7f4cf050f064..b65d843f0ab6 100644
--- a/polly/test/ScopInfo/invariant-load-instlist.ll
+++ b/polly/test/ScopInfo/invariant-load-instlist.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
 
 ; The load is a required invariant load and at the same time used in a store.
 ; Polly used to add two MemoryAccesses for it which caused an assertion to fail.
diff --git a/polly/test/ScopInfo/invariant-loads-leave-read-only-statements.ll b/polly/test/ScopInfo/invariant-loads-leave-read-only-statements.ll
index b97fe22e076e..12a72bd47fdf 100644
--- a/polly/test/ScopInfo/invariant-loads-leave-read-only-statements.ll
+++ b/polly/test/ScopInfo/invariant-loads-leave-read-only-statements.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
 
 ; CHECK:      Statements {
 ; CHECK-NEXT: 	Stmt_L_4
diff --git a/polly/test/ScopInfo/invariant_load.ll b/polly/test/ScopInfo/invariant_load.ll
index fcea77e19b85..47501022247e 100644
--- a/polly/test/ScopInfo/invariant_load.ll
+++ b/polly/test/ScopInfo/invariant_load.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type.ll b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type.ll
index 100a8db2a9d1..c402f90cbdd1 100644
--- a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type.ll
+++ b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
 ;
 ;    struct {
 ;      int a;
diff --git a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_escaping.ll b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_escaping.ll
index e31deb6fd472..435c9a575d73 100644
--- a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_escaping.ll
+++ b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_escaping.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
 ;
 ;    struct {
 ;      int a;
diff --git a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer.ll b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer.ll
index bbf6d69a5fbb..a2f43f2348b8 100644
--- a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer.ll
+++ b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
 ;
 ;    int U;
 ;    void f(int *A) {
diff --git a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer_escaping.ll b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer_escaping.ll
index 011c2fe3d549..412e62f26511 100644
--- a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer_escaping.ll
+++ b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer_escaping.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
 ;
 ;    int U;
 ;    int f(int *A) {
diff --git a/polly/test/ScopInfo/invariant_load_addrec_sum.ll b/polly/test/ScopInfo/invariant_load_addrec_sum.ll
index 09b158d342ed..8026d351fdf7 100644
--- a/polly/test/ScopInfo/invariant_load_addrec_sum.ll
+++ b/polly/test/ScopInfo/invariant_load_addrec_sum.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Region: %entry.split---%if.end
 ; CHECK:     Invariant Accesses: {
diff --git a/polly/test/ScopInfo/invariant_load_base_pointer.ll b/polly/test/ScopInfo/invariant_load_base_pointer.ll
index ddf11d892adb..7bf9c5a44788 100644
--- a/polly/test/ScopInfo/invariant_load_base_pointer.ll
+++ b/polly/test/ScopInfo/invariant_load_base_pointer.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_base_pointer_conditional.ll b/polly/test/ScopInfo/invariant_load_base_pointer_conditional.ll
index 07f2c3768b0a..42dfc7b3885a 100644
--- a/polly/test/ScopInfo/invariant_load_base_pointer_conditional.ll
+++ b/polly/test/ScopInfo/invariant_load_base_pointer_conditional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_base_pointer_in_conditional.ll b/polly/test/ScopInfo/invariant_load_base_pointer_in_conditional.ll
index d66d718d492a..68c60edce09c 100644
--- a/polly/test/ScopInfo/invariant_load_base_pointer_in_conditional.ll
+++ b/polly/test/ScopInfo/invariant_load_base_pointer_in_conditional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -polly-ignore-aliasing -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-ignore-aliasing -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_branch_condition.ll b/polly/test/ScopInfo/invariant_load_branch_condition.ll
index 4f49d2969d86..e8c18adee1f3 100644
--- a/polly/test/ScopInfo/invariant_load_branch_condition.ll
+++ b/polly/test/ScopInfo/invariant_load_branch_condition.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output \
-; RUN:     -polly-invariant-load-hoisting < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
+; RUN:     -polly-invariant-load-hoisting < %s 2>&1 | FileCheck %s
 
 ; CHECK:      Invariant Accesses: {
 ; CHECK-NEXT:         ReadAccess :=	[Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs.ll
index c6a7faf2e355..d19018db7967 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_2.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_2.ll
index 921dd4fbde5c..556b50284f34 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_2.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_3.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_3.ll
index c15d11ca865d..58c553023191 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_3.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4.ll
index 0495a330792c..881003b15839 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4b.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4b.ll
index 9144fcf186c3..6f4d40ec98f7 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4b.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4b.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4c.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4c.ll
index aefacff6b46f..d1859cb54241 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4c.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4c.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_5.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_5.ll
index ecc0c0a23014..74d6f1021124 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_5.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_complex_condition.ll b/polly/test/ScopInfo/invariant_load_complex_condition.ll
index e721c222db5f..34f5c45ac2b4 100644
--- a/polly/test/ScopInfo/invariant_load_complex_condition.ll
+++ b/polly/test/ScopInfo/invariant_load_complex_condition.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -S -polly-print-scops -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -S '-passes=print<polly-function-scops>' -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/invariant_load_condition.ll b/polly/test/ScopInfo/invariant_load_condition.ll
index 84546984709e..7d14a398d874 100644
--- a/polly/test/ScopInfo/invariant_load_condition.ll
+++ b/polly/test/ScopInfo/invariant_load_condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_dereferenceable.ll b/polly/test/ScopInfo/invariant_load_dereferenceable.ll
index adba32d8d463..c590e4043f64 100644
--- a/polly/test/ScopInfo/invariant_load_dereferenceable.ll
+++ b/polly/test/ScopInfo/invariant_load_dereferenceable.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-print-detect -polly-print-scops \
+; RUN: opt %loadPolly '-passes=print<polly-detect>' '-passes=print<polly-function-scops>' \
 ; RUN: -polly-invariant-load-hoisting=true \
-; RUN: -disable-output < %s | FileCheck %s
+; RUN: -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK-NOT: Function: foo_undereferanceable
 
diff --git a/polly/test/ScopInfo/invariant_load_distinct_parameter_valuations.ll b/polly/test/ScopInfo/invariant_load_distinct_parameter_valuations.ll
index 60b4a1daa824..b9b55cc4ecb1 100644
--- a/polly/test/ScopInfo/invariant_load_distinct_parameter_valuations.ll
+++ b/polly/test/ScopInfo/invariant_load_distinct_parameter_valuations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that we do not consolidate the invariant loads to smp[order - 1] and
 ; smp[order - 2] in the blocks %0 and %16. While they have the same pointer
diff --git a/polly/test/ScopInfo/invariant_load_in_non_affine.ll b/polly/test/ScopInfo/invariant_load_in_non_affine.ll
index d00bc2d642e0..08c8bd28caa7 100644
--- a/polly/test/ScopInfo/invariant_load_in_non_affine.ll
+++ b/polly/test/ScopInfo/invariant_load_in_non_affine.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output \
-; RUN:   -polly-invariant-load-hoisting=true < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output \
+; RUN:   -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
 ;
 ; CHECK-NOT: Valid Region for Scop
 ;
diff --git a/polly/test/ScopInfo/invariant_load_loop_ub.ll b/polly/test/ScopInfo/invariant_load_loop_ub.ll
index 856b6e4dd508..009f036bcb62 100644
--- a/polly/test/ScopInfo/invariant_load_loop_ub.ll
+++ b/polly/test/ScopInfo/invariant_load_loop_ub.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -polly-process-unprofitable -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -polly-invariant-load-hoisting=true -polly-process-unprofitable -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-process-unprofitable -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-process-unprofitable -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_ptr_ptr_noalias.ll b/polly/test/ScopInfo/invariant_load_ptr_ptr_noalias.ll
index 69463d420aca..ba381dea72e9 100644
--- a/polly/test/ScopInfo/invariant_load_ptr_ptr_noalias.ll
+++ b/polly/test/ScopInfo/invariant_load_ptr_ptr_noalias.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -tbaa -polly-print-scops -polly-invariant-load-hoisting=true -polly-ignore-aliasing \
-; RUN:                -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=tbaa '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-ignore-aliasing \
+; RUN:                -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Note: The order of the invariant accesses is important because A is the
 ;       base pointer of tmp3 and we will generate code in the same order as
diff --git a/polly/test/ScopInfo/invariant_load_scalar_dep.ll b/polly/test/ScopInfo/invariant_load_scalar_dep.ll
index 79a10426862a..ea0227d4a6e9 100644
--- a/polly/test/ScopInfo/invariant_load_scalar_dep.ll
+++ b/polly/test/ScopInfo/invariant_load_scalar_dep.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_stmt_domain.ll b/polly/test/ScopInfo/invariant_load_stmt_domain.ll
index 6cd71c85ea2f..31da46f40f8c 100644
--- a/polly/test/ScopInfo/invariant_load_stmt_domain.ll
+++ b/polly/test/ScopInfo/invariant_load_stmt_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 
 ; This test case verifies that the statement domain of the invariant access
 ; is the universe. In earlier versions of Polly, we accidentally computed an
diff --git a/polly/test/ScopInfo/invariant_load_zext_parameter-2.ll b/polly/test/ScopInfo/invariant_load_zext_parameter-2.ll
index e77515280241..2bf6d1d4d8b1 100644
--- a/polly/test/ScopInfo/invariant_load_zext_parameter-2.ll
+++ b/polly/test/ScopInfo/invariant_load_zext_parameter-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -scalar-evolution-max-value-compare-depth=3 -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -scalar-evolution-max-value-compare-depth=3 -polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
+; RUN: opt %loadPolly -scalar-evolution-max-value-compare-depth=3 '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -scalar-evolution-max-value-compare-depth=3 -passes=polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
 ;
 ; Stress test for the code generation of invariant accesses.
 ;
diff --git a/polly/test/ScopInfo/invariant_load_zext_parameter.ll b/polly/test/ScopInfo/invariant_load_zext_parameter.ll
index 1bde70282d44..41559ff14efa 100644
--- a/polly/test/ScopInfo/invariant_load_zext_parameter.ll
+++ b/polly/test/ScopInfo/invariant_load_zext_parameter.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
 ;
 ;    void f(int *I0, int *I1, int *V) {
 ;      for (int i = 0; i < 1000; i++) {
diff --git a/polly/test/ScopInfo/invariant_load_zextended_in_own_execution_context.ll b/polly/test/ScopInfo/invariant_load_zextended_in_own_execution_context.ll
index 775369e55c92..2ae99acd47ec 100644
--- a/polly/test/ScopInfo/invariant_load_zextended_in_own_execution_context.ll
+++ b/polly/test/ScopInfo/invariant_load_zextended_in_own_execution_context.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
 ;
 ; CHECK: Execution Context: [p_0_loaded_from_currpc] -> {  :  }
 ;
diff --git a/polly/test/ScopInfo/invariant_loads_complicated_dependences.ll b/polly/test/ScopInfo/invariant_loads_complicated_dependences.ll
index 1d54ccc69023..9cee220d1f3c 100644
--- a/polly/test/ScopInfo/invariant_loads_complicated_dependences.ll
+++ b/polly/test/ScopInfo/invariant_loads_complicated_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:      Invariant Accesses: {
 ; CHECK-NEXT:         ReadAccess :=    [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_loads_cyclic_dependences.ll b/polly/test/ScopInfo/invariant_loads_cyclic_dependences.ll
index e97de0c936bc..0434bbd34e4c 100644
--- a/polly/test/ScopInfo/invariant_loads_cyclic_dependences.ll
+++ b/polly/test/ScopInfo/invariant_loads_cyclic_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Negative test. If we assume UB[*V] to be invariant we get a cyclic
 ; dependence in the invariant loads that needs to be resolved by
diff --git a/polly/test/ScopInfo/invariant_loop_bounds.ll b/polly/test/ScopInfo/invariant_loop_bounds.ll
index 4e1fd88fac30..279199b805f1 100644
--- a/polly/test/ScopInfo/invariant_loop_bounds.ll
+++ b/polly/test/ScopInfo/invariant_loop_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:      Invariant Accesses: {
 ; CHECK-NEXT:         ReadAccess :=    [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll
index 3d5737bbe168..d8e293f7e6f3 100644
--- a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll
+++ b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify that we only have one parameter and one invariant load for all
 ; three loads that occure in the region but actually access the same
diff --git a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll
index e2de503eb83f..e68f3be7fb30 100644
--- a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll
+++ b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify that we only have one parameter and one invariant load for all
 ; three loads that occure in the region but actually access the same
diff --git a/polly/test/ScopInfo/isl_aff_out_of_bounds.ll b/polly/test/ScopInfo/isl_aff_out_of_bounds.ll
index ca1b235be358..4f7a604272bb 100644
--- a/polly/test/ScopInfo/isl_aff_out_of_bounds.ll
+++ b/polly/test/ScopInfo/isl_aff_out_of_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-detect < %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' < %s 2>&1
 
 ; Used to fail with:
 ; ../../isl/isl_aff.c:591: position out of bounds
diff --git a/polly/test/ScopInfo/isl_trip_count_01.ll b/polly/test/ScopInfo/isl_trip_count_01.ll
index fc6b79c5a68a..6ad4929888e3 100644
--- a/polly/test/ScopInfo/isl_trip_count_01.ll
+++ b/polly/test/ScopInfo/isl_trip_count_01.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: [M, N] -> { Stmt_while_body[i0] : i0 > 0 and 4i0 <= -M + N; Stmt_while_body[0] };
 ;
diff --git a/polly/test/ScopInfo/isl_trip_count_02.ll b/polly/test/ScopInfo/isl_trip_count_02.ll
index 9376cb415cec..b356fa9fdf22 100644
--- a/polly/test/ScopInfo/isl_trip_count_02.ll
+++ b/polly/test/ScopInfo/isl_trip_count_02.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; TODO: We do not allow unbounded loops at the moment.
 ;
diff --git a/polly/test/ScopInfo/isl_trip_count_03.ll b/polly/test/ScopInfo/isl_trip_count_03.ll
index f5b0048a0e0e..886143114df6 100644
--- a/polly/test/ScopInfo/isl_trip_count_03.ll
+++ b/polly/test/ScopInfo/isl_trip_count_03.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Test comes from a bug (15771) or better a feature request. It was not allowed
 ; in Polly in the old domain generation as ScalarEvolution cannot figure out the
diff --git a/polly/test/ScopInfo/isl_trip_count_multiple_exiting_blocks.ll b/polly/test/ScopInfo/isl_trip_count_multiple_exiting_blocks.ll
index 91bc19e2de44..8ddb26a6ac88 100644
--- a/polly/test/ScopInfo/isl_trip_count_multiple_exiting_blocks.ll
+++ b/polly/test/ScopInfo/isl_trip_count_multiple_exiting_blocks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/licm_load.ll b/polly/test/ScopInfo/licm_load.ll
index ade640976d00..c4695ecddaf7 100644
--- a/polly/test/ScopInfo/licm_load.ll
+++ b/polly/test/ScopInfo/licm_load.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadNPMPolly -passes='loop(loop-rotate,indvars),polly-prepare,print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -passes='loop(loop-rotate,indvars),polly-prepare,print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt %loadNPMPolly -passes='loop-mssa(loop-rotate,indvars,licm),polly-prepare,print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -passes='loop-mssa(loop-rotate,indvars,licm),polly-prepare,print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 \
 ; RUN: | FileCheck %s
 ;
 ;    void foo(int n, float A[static const restrict n],
diff --git a/polly/test/ScopInfo/licm_potential_store.ll b/polly/test/ScopInfo/licm_potential_store.ll
index 8a36ee84313a..fd19df793306 100644
--- a/polly/test/ScopInfo/licm_potential_store.ll
+++ b/polly/test/ScopInfo/licm_potential_store.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadNPMPolly -passes='sroa,instcombine,simplifycfg,reassociate,loop(loop-rotate),instcombine,indvars,polly-prepare,print<polly-function-scops>' \
+; RUN: opt %loadPolly -passes='sroa,instcombine,simplifycfg,reassociate,loop(loop-rotate),instcombine,indvars,polly-prepare,print<polly-function-scops>' \
 ; RUN:    -tailcallopt -disable-output < %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=NOLICM
 
-; RUN: opt %loadNPMPolly -passes='sroa,instcombine,simplifycfg,reassociate,loop(loop-rotate),instcombine,indvars,loop-mssa(licm),polly-prepare,print<polly-function-scops>' \
+; RUN: opt %loadPolly -passes='sroa,instcombine,simplifycfg,reassociate,loop(loop-rotate),instcombine,indvars,loop-mssa(licm),polly-prepare,print<polly-function-scops>' \
 ; RUN:    -tailcallopt -disable-output < %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=LICM
 
diff --git a/polly/test/ScopInfo/licm_reduction_nested.ll b/polly/test/ScopInfo/licm_reduction_nested.ll
index a3ba478cd9ff..98d6dfcfa074 100644
--- a/polly/test/ScopInfo/licm_reduction_nested.ll
+++ b/polly/test/ScopInfo/licm_reduction_nested.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars       -polly-prepare -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars -licm -polly-prepare -polly-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -loop-rotate -indvars       -passes=polly-prepare '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -loop-rotate -indvars -licm -passes=polly-prepare '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; XFAIL: *
 ;
diff --git a/polly/test/ScopInfo/long-compile-time-alias-analysis.ll b/polly/test/ScopInfo/long-compile-time-alias-analysis.ll
index 1cbecf086968..5fa9a74d0fe3 100644
--- a/polly/test/ScopInfo/long-compile-time-alias-analysis.ll
+++ b/polly/test/ScopInfo/long-compile-time-alias-analysis.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
 
 ; Verify that the compilation of this test case does not take infinite time.
 ; At some point Polly tried to model this test case and got stuck in
diff --git a/polly/test/ScopInfo/long-sequence-of-error-blocks-2.ll b/polly/test/ScopInfo/long-sequence-of-error-blocks-2.ll
index c88ea1327389..283a0bea7c49 100644
--- a/polly/test/ScopInfo/long-sequence-of-error-blocks-2.ll
+++ b/polly/test/ScopInfo/long-sequence-of-error-blocks-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/polly/test/ScopInfo/long-sequence-of-error-blocks.ll b/polly/test/ScopInfo/long-sequence-of-error-blocks.ll
index 5b6ea9cc212d..812de273a7df 100644
--- a/polly/test/ScopInfo/long-sequence-of-error-blocks.ll
+++ b/polly/test/ScopInfo/long-sequence-of-error-blocks.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/polly/test/ScopInfo/loop-multiexit-succ-cond.ll b/polly/test/ScopInfo/loop-multiexit-succ-cond.ll
index 350db05c6dc0..f8f47a332a13 100644
--- a/polly/test/ScopInfo/loop-multiexit-succ-cond.ll
+++ b/polly/test/ScopInfo/loop-multiexit-succ-cond.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s 2>&1 | FileCheck %s --check-prefix=IR
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/loop_affine_bound_0.ll b/polly/test/ScopInfo/loop_affine_bound_0.ll
index 33f49df7780f..77b0ebe8e494 100644
--- a/polly/test/ScopInfo/loop_affine_bound_0.ll
+++ b/polly/test/ScopInfo/loop_affine_bound_0.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long a[][128], long N, long M) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/loop_affine_bound_1.ll b/polly/test/ScopInfo/loop_affine_bound_1.ll
index 38e47b74465b..7c1eaa5dff8c 100644
--- a/polly/test/ScopInfo/loop_affine_bound_1.ll
+++ b/polly/test/ScopInfo/loop_affine_bound_1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output< %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output< %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ;void f(long a[][128], long N, long M) {
 ;  long i, j;
diff --git a/polly/test/ScopInfo/loop_affine_bound_2.ll b/polly/test/ScopInfo/loop_affine_bound_2.ll
index e34662f4e6ab..12d81c78b794 100644
--- a/polly/test/ScopInfo/loop_affine_bound_2.ll
+++ b/polly/test/ScopInfo/loop_affine_bound_2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long a[][128], long N, long M) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/loop_carry.ll b/polly/test/ScopInfo/loop_carry.ll
index f7c1dca0919c..856efb730056 100644
--- a/polly/test/ScopInfo/loop_carry.ll
+++ b/polly/test/ScopInfo/loop_carry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/ScopInfo/many-scalar-dependences.ll b/polly/test/ScopInfo/many-scalar-dependences.ll
index aaa02f581a1c..56e02d56254d 100644
--- a/polly/test/ScopInfo/many-scalar-dependences.ll
+++ b/polly/test/ScopInfo/many-scalar-dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(float a[100][100]) {
 ;      float x;
diff --git a/polly/test/ScopInfo/max-loop-depth.ll b/polly/test/ScopInfo/max-loop-depth.ll
index 3c7db4458604..4da0f35121d0 100644
--- a/polly/test/ScopInfo/max-loop-depth.ll
+++ b/polly/test/ScopInfo/max-loop-depth.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void bar();
 ;    void foo(int *A, int *B, long int N, long int M) {
diff --git a/polly/test/ScopInfo/memcpy-raw-source.ll b/polly/test/ScopInfo/memcpy-raw-source.ll
index 137ab8229220..c3ecce23cf5c 100644
--- a/polly/test/ScopInfo/memcpy-raw-source.ll
+++ b/polly/test/ScopInfo/memcpy-raw-source.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -scoped-noalias-aa -tbaa -polly-print-scops -disable-output < %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa,scoped-noalias-aa,tbaa '-passes=print<polly-function-scops>' -disable-output < %s
 ;
 ; Ensure that ScopInfo's alias analysis llvm.memcpy for,
 ; like the AliasSetTracker, preserves bitcasts.
diff --git a/polly/test/ScopInfo/memcpy.ll b/polly/test/ScopInfo/memcpy.ll
index 705dea769e42..2e34b0d87a5f 100644
--- a/polly/test/ScopInfo/memcpy.ll
+++ b/polly/test/ScopInfo/memcpy.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-allow-differing-element-types -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -S -basic-aa -polly-allow-differing-element-types -polly-codegen < %s | FileCheck --check-prefix=IR %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-differing-element-types '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -S -aa-pipeline=basic-aa -polly-allow-differing-element-types -passes=polly-codegen < %s 2>&1 | FileCheck --check-prefix=IR %s
 ;
 ; CHECK:         Arrays {
 ; CHECK-NEXT:        i8 MemRef_A[*]; // Element size 1
diff --git a/polly/test/ScopInfo/memmove.ll b/polly/test/ScopInfo/memmove.ll
index 15123422f419..28a4ebeee7e2 100644
--- a/polly/test/ScopInfo/memmove.ll
+++ b/polly/test/ScopInfo/memmove.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-allow-differing-element-types -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -S -basic-aa -polly-allow-differing-element-types -polly-codegen < %s | FileCheck --check-prefix=IR %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-differing-element-types '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -S -aa-pipeline=basic-aa -polly-allow-differing-element-types -passes=polly-codegen < %s 2>&1 | FileCheck --check-prefix=IR %s
 ;
 ; CHECK:         Arrays {
 ; CHECK-NEXT:        i8 MemRef_A[*]; // Element size 1
diff --git a/polly/test/ScopInfo/memset.ll b/polly/test/ScopInfo/memset.ll
index ef86b4c275e5..163b58dc54dc 100644
--- a/polly/test/ScopInfo/memset.ll
+++ b/polly/test/ScopInfo/memset.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-differing-element-types -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -S -polly-allow-differing-element-types -polly-codegen < %s | FileCheck --check-prefix=IR %s
+; RUN: opt %loadPolly -polly-allow-differing-element-types '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -S -polly-allow-differing-element-types -passes=polly-codegen < %s 2>&1 | FileCheck --check-prefix=IR %s
 ;
 ; CHECK:         Arrays {
 ; CHECK-NEXT:        i8 MemRef_A[*]; // Element size 1
diff --git a/polly/test/ScopInfo/memset_null.ll b/polly/test/ScopInfo/memset_null.ll
index 1608ff6ebef4..3d38fa1e6851 100644
--- a/polly/test/ScopInfo/memset_null.ll
+++ b/polly/test/ScopInfo/memset_null.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-modref-calls -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-modref-calls -S -polly-codegen < %s
+; RUN: opt %loadPolly -polly-allow-modref-calls '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-modref-calls -S -passes=polly-codegen < %s
 ;
 ; Verify we can handle a memset to "null" and that we do not model it.
 ; TODO: FIXME: We could use the undefined memset to optimize the code further,
diff --git a/polly/test/ScopInfo/mismatching-array-dimensions.ll b/polly/test/ScopInfo/mismatching-array-dimensions.ll
index a1c6d4e82127..a2deef16eafd 100644
--- a/polly/test/ScopInfo/mismatching-array-dimensions.ll
+++ b/polly/test/ScopInfo/mismatching-array-dimensions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK-NOT: AssumedContext
 
diff --git a/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll b/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll
index 72889324e37e..202295abf57c 100644
--- a/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll
+++ b/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-scops -polly-allow-modref-calls \
-; RUN:     -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-codegen -polly-allow-modref-calls \
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
+; RUN:     -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb -passes=polly-codegen -polly-allow-modref-calls \
 ; RUN:     -disable-output < %s
 ;
 ; Verify that we model the may-write access of the prefetch intrinsic
diff --git a/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll b/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll
index 2f6c6792fd9d..f1124f0a977b 100644
--- a/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll
+++ b/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-scops -polly-allow-modref-calls \
-; RUN:     -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -basic-aa -polly-codegen -disable-output \
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
+; RUN:     -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -disable-output \
 ; RUN:     -polly-allow-modref-calls < %s
 ;
 ; Verify that we model the read access of the gcread intrinsic
diff --git a/polly/test/ScopInfo/mod_ref_read_pointer.ll b/polly/test/ScopInfo/mod_ref_read_pointer.ll
index 657e37c68a7b..24f276621532 100644
--- a/polly/test/ScopInfo/mod_ref_read_pointer.ll
+++ b/polly/test/ScopInfo/mod_ref_read_pointer.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-allow-modref-calls -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -basic-aa -polly-allow-modref-calls -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-modref-calls '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-modref-calls -passes=polly-codegen -disable-output < %s
 ;
 ; Check that we assume the call to func has a read on the whole A array.
 ;
diff --git a/polly/test/ScopInfo/mod_ref_read_pointers.ll b/polly/test/ScopInfo/mod_ref_read_pointers.ll
index 7ed3423a2aeb..260f759c1449 100644
--- a/polly/test/ScopInfo/mod_ref_read_pointers.ll
+++ b/polly/test/ScopInfo/mod_ref_read_pointers.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-scops -polly-allow-modref-calls \
-; RUN:     -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -basic-aa -polly-codegen -disable-output \
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
+; RUN:     -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -disable-output \
 ; RUN:     -polly-allow-modref-calls < %s
 ;
 ; Check that the call to func will "read" not only the A array but also the
diff --git a/polly/test/ScopInfo/modulo_zext_1.ll b/polly/test/ScopInfo/modulo_zext_1.ll
index d611ec4807b5..60083098eb08 100644
--- a/polly/test/ScopInfo/modulo_zext_1.ll
+++ b/polly/test/ScopInfo/modulo_zext_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:         Assumed Context:
 ; CHECK-NEXT:    [N] -> {  :  }
diff --git a/polly/test/ScopInfo/modulo_zext_2.ll b/polly/test/ScopInfo/modulo_zext_2.ll
index 8d2321849174..7ed70d085546 100644
--- a/polly/test/ScopInfo/modulo_zext_2.ll
+++ b/polly/test/ScopInfo/modulo_zext_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:         Assumed Context:
 ; CHECK-NEXT:    [N] -> {  :  }
diff --git a/polly/test/ScopInfo/modulo_zext_3.ll b/polly/test/ScopInfo/modulo_zext_3.ll
index acb26dc1c77f..67b26d813918 100644
--- a/polly/test/ScopInfo/modulo_zext_3.ll
+++ b/polly/test/ScopInfo/modulo_zext_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:         Assumed Context:
 ; CHECK-NEXT:    [N] -> {  :  }
diff --git a/polly/test/ScopInfo/multi-scop.ll b/polly/test/ScopInfo/multi-scop.ll
index e26c8c7bae10..747f76bbf275 100644
--- a/polly/test/ScopInfo/multi-scop.ll
+++ b/polly/test/ScopInfo/multi-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-detect -polly-scops  -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>'  -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; This test case contains two scops.
diff --git a/polly/test/ScopInfo/multidim_2d-diagonal-matrix.ll b/polly/test/ScopInfo/multidim_2d-diagonal-matrix.ll
index 278c06a2fdba..8d023a671d86 100644
--- a/polly/test/ScopInfo/multidim_2d-diagonal-matrix.ll
+++ b/polly/test/ScopInfo/multidim_2d-diagonal-matrix.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/ScopInfo/multidim_2d_outer_parametric_offset.ll b/polly/test/ScopInfo/multidim_2d_outer_parametric_offset.ll
index 06a76466c25e..6cd612d155cb 100644
--- a/polly/test/ScopInfo/multidim_2d_outer_parametric_offset.ll
+++ b/polly/test/ScopInfo/multidim_2d_outer_parametric_offset.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/ScopInfo/multidim_2d_parametric_array_static_loop_bounds.ll b/polly/test/ScopInfo/multidim_2d_parametric_array_static_loop_bounds.ll
index bfbe5682d44a..aed318b51b15 100644
--- a/polly/test/ScopInfo/multidim_2d_parametric_array_static_loop_bounds.ll
+++ b/polly/test/ScopInfo/multidim_2d_parametric_array_static_loop_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/ScopInfo/multidim_2d_with_modref_call.ll b/polly/test/ScopInfo/multidim_2d_with_modref_call.ll
index ba934adb675a..872544d872cd 100644
--- a/polly/test/ScopInfo/multidim_2d_with_modref_call.ll
+++ b/polly/test/ScopInfo/multidim_2d_with_modref_call.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-modref-calls \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-nonaffine \
+; RUN:     -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -polly-allow-modref-calls -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN:     -polly-allow-modref-calls -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NONAFFINE
 
 ;  TODO: We should delinearize the accesses despite the use in a call to a
 ;        readonly function. For now we verify we do not delinearize them though.
diff --git a/polly/test/ScopInfo/multidim_2d_with_modref_call_2.ll b/polly/test/ScopInfo/multidim_2d_with_modref_call_2.ll
index 3da123fd1f60..add130e2087f 100644
--- a/polly/test/ScopInfo/multidim_2d_with_modref_call_2.ll
+++ b/polly/test/ScopInfo/multidim_2d_with_modref_call_2.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-modref-calls \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-nonaffine \
+; RUN:     -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -polly-allow-modref-calls -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN:     -polly-allow-modref-calls -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NONAFFINE
 
 ;  TODO: We should delinearize the accesses despite the use in a call to a
 ;        readonly function. For now we verify we do not delinearize them though.
diff --git a/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll b/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll
index 988475575fec..5e0f58cb276b 100644
--- a/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll
+++ b/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
diff --git a/polly/test/ScopInfo/multidim_fixedsize_different_dimensionality.ll b/polly/test/ScopInfo/multidim_fixedsize_different_dimensionality.ll
index ddc35a46a633..765077142df6 100644
--- a/polly/test/ScopInfo/multidim_fixedsize_different_dimensionality.ll
+++ b/polly/test/ScopInfo/multidim_fixedsize_different_dimensionality.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    #define N 400
 ;
diff --git a/polly/test/ScopInfo/multidim_fixedsize_multi_offset.ll b/polly/test/ScopInfo/multidim_fixedsize_multi_offset.ll
index 9c749f0c48c8..8e26a2ad6a21 100644
--- a/polly/test/ScopInfo/multidim_fixedsize_multi_offset.ll
+++ b/polly/test/ScopInfo/multidim_fixedsize_multi_offset.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:      Context:
 ; CHECK-NEXT: {  :  }
diff --git a/polly/test/ScopInfo/multidim_fold_constant_dim.ll b/polly/test/ScopInfo/multidim_fold_constant_dim.ll
index e95d400a860c..b142a2ab442f 100644
--- a/polly/test/ScopInfo/multidim_fold_constant_dim.ll
+++ b/polly/test/ScopInfo/multidim_fold_constant_dim.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    struct com {
 ;      double Real;
diff --git a/polly/test/ScopInfo/multidim_fold_constant_dim_zero.ll b/polly/test/ScopInfo/multidim_fold_constant_dim_zero.ll
index 57275e4024ab..dac9dac1e7ad 100644
--- a/polly/test/ScopInfo/multidim_fold_constant_dim_zero.ll
+++ b/polly/test/ScopInfo/multidim_fold_constant_dim_zero.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -debug -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -debug -disable-output < %s 2>&1 | FileCheck %s
 
 ; REQUIRES: asserts
 
diff --git a/polly/test/ScopInfo/multidim_fortran_2d.ll b/polly/test/ScopInfo/multidim_fortran_2d.ll
index 29279a4e886b..ee13da875d4e 100644
--- a/polly/test/ScopInfo/multidim_fortran_2d.ll
+++ b/polly/test/ScopInfo/multidim_fortran_2d.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops \
-; RUN: -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' \
+; RUN: -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 
 ;   subroutine init_array(ni, nj, pi, pj, a)
 ;   implicit none
diff --git a/polly/test/ScopInfo/multidim_fortran_2d_params.ll b/polly/test/ScopInfo/multidim_fortran_2d_params.ll
index 93145b399ca5..f4978ecb35f7 100644
--- a/polly/test/ScopInfo/multidim_fortran_2d_params.ll
+++ b/polly/test/ScopInfo/multidim_fortran_2d_params.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
 ; RUN: -polly-precise-fold-accesses \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
 
 ;   subroutine init_array(ni, nj, pi, pj, a)
 ;   implicit none
diff --git a/polly/test/ScopInfo/multidim_fortran_2d_with_modref_call.ll b/polly/test/ScopInfo/multidim_fortran_2d_with_modref_call.ll
index dff6a8be85cf..f3a5e0ba9305 100644
--- a/polly/test/ScopInfo/multidim_fortran_2d_with_modref_call.ll
+++ b/polly/test/ScopInfo/multidim_fortran_2d_with_modref_call.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-modref-calls \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-nonaffine \
+; RUN:     -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -polly-allow-modref-calls -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN:     -polly-allow-modref-calls -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NONAFFINE
 
 ;  TODO: We should delinearize the accesses despite the use in a call to a
 ;        readonly function. For now we verify we do not delinearize them though.
diff --git a/polly/test/ScopInfo/multidim_fortran_srem.ll b/polly/test/ScopInfo/multidim_fortran_srem.ll
index 8c24c5b8ee71..fc65ff2954cf 100644
--- a/polly/test/ScopInfo/multidim_fortran_srem.ll
+++ b/polly/test/ScopInfo/multidim_fortran_srem.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-S128-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f16:16:16-f32:32:32-f64:64:64-f128:128:128-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ; CHECK:      Statements {
diff --git a/polly/test/ScopInfo/multidim_gep_pointercast.ll b/polly/test/ScopInfo/multidim_gep_pointercast.ll
index 20d59fa91eaf..6b69dd7bb571 100644
--- a/polly/test/ScopInfo/multidim_gep_pointercast.ll
+++ b/polly/test/ScopInfo/multidim_gep_pointercast.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; The load access to A has a pointer-bitcast to another elements size before the
 ; GetElementPtr. Verify that we do not the GEP delinearization because it
diff --git a/polly/test/ScopInfo/multidim_gep_pointercast2.ll b/polly/test/ScopInfo/multidim_gep_pointercast2.ll
index deed9c7c3f57..0c8139f7f8e1 100644
--- a/polly/test/ScopInfo/multidim_gep_pointercast2.ll
+++ b/polly/test/ScopInfo/multidim_gep_pointercast2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verfy that we do not use the GetElementPtr information to delinearize A
 ; because of the cast in-between. Use the single-dimensional modeling instead.
diff --git a/polly/test/ScopInfo/multidim_ivs_and_integer_offsets_3d.ll b/polly/test/ScopInfo/multidim_ivs_and_integer_offsets_3d.ll
index 9f7e6bc4a2a2..4610f46ea082 100644
--- a/polly/test/ScopInfo/multidim_ivs_and_integer_offsets_3d.ll
+++ b/polly/test/ScopInfo/multidim_ivs_and_integer_offsets_3d.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
diff --git a/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll b/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll
index 131bb7b3ebed..b64e0a99c73b 100644
--- a/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll
+++ b/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-precise-fold-accesses -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-precise-fold-accesses '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o], long p, long q, long r) {
diff --git a/polly/test/ScopInfo/multidim_many_references.ll b/polly/test/ScopInfo/multidim_many_references.ll
index b0483b267260..0736fd947a31 100644
--- a/polly/test/ScopInfo/multidim_many_references.ll
+++ b/polly/test/ScopInfo/multidim_many_references.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-ignore-aliasing -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -polly-ignore-aliasing -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-ignore-aliasing -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-ignore-aliasing -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/multidim_nested_start_integer.ll b/polly/test/ScopInfo/multidim_nested_start_integer.ll
index 741a0ef45c27..db99ab176a3a 100644
--- a/polly/test/ScopInfo/multidim_nested_start_integer.ll
+++ b/polly/test/ScopInfo/multidim_nested_start_integer.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
diff --git a/polly/test/ScopInfo/multidim_nested_start_share_parameter.ll b/polly/test/ScopInfo/multidim_nested_start_share_parameter.ll
index 692746bad3d7..0ecac3cdaaab 100644
--- a/polly/test/ScopInfo/multidim_nested_start_share_parameter.ll
+++ b/polly/test/ScopInfo/multidim_nested_start_share_parameter.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
diff --git a/polly/test/ScopInfo/multidim_only_ivs_2d.ll b/polly/test/ScopInfo/multidim_only_ivs_2d.ll
index 71245642e751..ccfa7a7de8e2 100644
--- a/polly/test/ScopInfo/multidim_only_ivs_2d.ll
+++ b/polly/test/ScopInfo/multidim_only_ivs_2d.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/ScopInfo/multidim_only_ivs_3d.ll b/polly/test/ScopInfo/multidim_only_ivs_3d.ll
index a019d58b241d..7a18aec573e8 100644
--- a/polly/test/ScopInfo/multidim_only_ivs_3d.ll
+++ b/polly/test/ScopInfo/multidim_only_ivs_3d.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
diff --git a/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll b/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll
index 41577ef1a0be..dd9c4d374f28 100644
--- a/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll
+++ b/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void foo(int n, int m, int o, double A[n][m][o]) {
 ;
diff --git a/polly/test/ScopInfo/multidim_only_ivs_3d_reverse.ll b/polly/test/ScopInfo/multidim_only_ivs_3d_reverse.ll
index 25907f2ee79c..be1b1eb8fc8d 100644
--- a/polly/test/ScopInfo/multidim_only_ivs_3d_reverse.ll
+++ b/polly/test/ScopInfo/multidim_only_ivs_3d_reverse.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; This test case checks for array access functions where the order in which the
diff --git a/polly/test/ScopInfo/multidim_param_in_subscript-2.ll b/polly/test/ScopInfo/multidim_param_in_subscript-2.ll
index 0790664f7129..f0cd1589b987 100644
--- a/polly/test/ScopInfo/multidim_param_in_subscript-2.ll
+++ b/polly/test/ScopInfo/multidim_param_in_subscript-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-precise-fold-accesses -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-precise-fold-accesses '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void foo(long n, long m, float A[][n][m]) {
 ;      for (long i = 0; i < 100; i++)
diff --git a/polly/test/ScopInfo/multidim_param_in_subscript.ll b/polly/test/ScopInfo/multidim_param_in_subscript.ll
index b8ec80b321fe..ca423919e488 100644
--- a/polly/test/ScopInfo/multidim_param_in_subscript.ll
+++ b/polly/test/ScopInfo/multidim_param_in_subscript.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;
 ;    void foo(long n, float A[][n]) {
diff --git a/polly/test/ScopInfo/multidim_parameter_addrec_product.ll b/polly/test/ScopInfo/multidim_parameter_addrec_product.ll
index 7db3e9dc3b5f..ec311d42386a 100644
--- a/polly/test/ScopInfo/multidim_parameter_addrec_product.ll
+++ b/polly/test/ScopInfo/multidim_parameter_addrec_product.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void foo(float *A, long *p) {
 ;      for (long i = 0; i < 100; i++)
diff --git a/polly/test/ScopInfo/multidim_single_and_multidim_array.ll b/polly/test/ScopInfo/multidim_single_and_multidim_array.ll
index 1e302dec4861..0d51aa115559 100644
--- a/polly/test/ScopInfo/multidim_single_and_multidim_array.ll
+++ b/polly/test/ScopInfo/multidim_single_and_multidim_array.ll
@@ -1,11 +1,11 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-delinearize=false -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-scops -polly-delinearize=false -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=DELIN
-; RUN: opt %loadPolly -polly-print-scops -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=DELIN
-; RUN: opt %loadPolly -polly-print-function-scops -polly-delinearize=false -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -polly-delinearize=false -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
-; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s --check-prefix=DELIN
-; RUN: opt %loadPolly -polly-print-function-scops -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=DELIN
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-delinearize=false -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-delinearize=false -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NONAFFINE
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=DELIN
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s --check-prefix=DELIN
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-delinearize=false -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-delinearize=false -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NONAFFINE
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=DELIN
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s --check-prefix=DELIN
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/multidim_srem.ll b/polly/test/ScopInfo/multidim_srem.ll
index f89843f0a5bc..b4eee668207c 100644
--- a/polly/test/ScopInfo/multidim_srem.ll
+++ b/polly/test/ScopInfo/multidim_srem.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void foo(long n, float A[][n][n]) {
 ;      for (long i = 0; i < 200; i++)
diff --git a/polly/test/ScopInfo/multidim_with_bitcast.ll b/polly/test/ScopInfo/multidim_with_bitcast.ll
index b77ff689b953..8af2e18265b0 100644
--- a/polly/test/ScopInfo/multidim_with_bitcast.ll
+++ b/polly/test/ScopInfo/multidim_with_bitcast.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/multiple-binary-or-conditions.ll b/polly/test/ScopInfo/multiple-binary-or-conditions.ll
index b905a11f577c..481f799ed0fe 100644
--- a/polly/test/ScopInfo/multiple-binary-or-conditions.ll
+++ b/polly/test/ScopInfo/multiple-binary-or-conditions.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
 ;
 ; void or(float *A, long n, long m) {
 ;   for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/multiple-types-access-offset-not-dividable-by-element-size.ll b/polly/test/ScopInfo/multiple-types-access-offset-not-dividable-by-element-size.ll
index 2d03ad941c05..69b00402ba6b 100644
--- a/polly/test/ScopInfo/multiple-types-access-offset-not-dividable-by-element-size.ll
+++ b/polly/test/ScopInfo/multiple-types-access-offset-not-dividable-by-element-size.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -pass-remarks-analysis="polly-scops" \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -pass-remarks-analysis="polly-scops" \
 ; RUN:     -polly-allow-differing-element-types \
 ; RUN:     -disable-output < %s  2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/multiple-types-non-affine-2.ll b/polly/test/ScopInfo/multiple-types-non-affine-2.ll
index 5b0aa5de1e71..a7e9a31744b4 100644
--- a/polly/test/ScopInfo/multiple-types-non-affine-2.ll
+++ b/polly/test/ScopInfo/multiple-types-non-affine-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -polly-print-scops -polly-allow-nonaffine -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -polly-codegen -polly-allow-nonaffine -disable-output
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types '-passes=print<polly-function-scops>' -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -passes=polly-codegen -polly-allow-nonaffine -disable-output
 ;
 ;    // Check that accessing one array with different types works,
 ;    // even though some accesses are non-affine.
diff --git a/polly/test/ScopInfo/multiple-types-non-affine.ll b/polly/test/ScopInfo/multiple-types-non-affine.ll
index 8e4be4c86d5a..e49612aa2791 100644
--- a/polly/test/ScopInfo/multiple-types-non-affine.ll
+++ b/polly/test/ScopInfo/multiple-types-non-affine.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -polly-print-scops -polly-allow-nonaffine -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -polly-codegen -polly-allow-nonaffine -disable-output
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types '-passes=print<polly-function-scops>' -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -passes=polly-codegen -polly-allow-nonaffine -disable-output
 ;
 ;    // Check that accessing one array with different types works,
 ;    // even though some accesses are non-affine.
diff --git a/polly/test/ScopInfo/multiple-types-non-power-of-two-2.ll b/polly/test/ScopInfo/multiple-types-non-power-of-two-2.ll
index 01f5923457b4..8347de936eff 100644
--- a/polly/test/ScopInfo/multiple-types-non-power-of-two-2.ll
+++ b/polly/test/ScopInfo/multiple-types-non-power-of-two-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-allow-differing-element-types -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-allow-differing-element-types -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;  void multiple_types(i8 *A) {
 ;    for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/multiple-types-non-power-of-two.ll b/polly/test/ScopInfo/multiple-types-non-power-of-two.ll
index 142a5ac395b3..077ff1a741ad 100644
--- a/polly/test/ScopInfo/multiple-types-non-power-of-two.ll
+++ b/polly/test/ScopInfo/multiple-types-non-power-of-two.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-allow-differing-element-types -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-allow-differing-element-types -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;  void multiple_types(i8 *A) {
 ;    for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/multiple-types-two-dimensional-2.ll b/polly/test/ScopInfo/multiple-types-two-dimensional-2.ll
index 1e2e53e85c25..431a0c1e966f 100644
--- a/polly/test/ScopInfo/multiple-types-two-dimensional-2.ll
+++ b/polly/test/ScopInfo/multiple-types-two-dimensional-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -pass-remarks-analysis="polly-scops" \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -pass-remarks-analysis="polly-scops" \
 ; RUN:                -polly-allow-differing-element-types \
 ; RUN:                -disable-output < %s 2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/multiple-types-two-dimensional.ll b/polly/test/ScopInfo/multiple-types-two-dimensional.ll
index 21dc96e6f95d..2cc2454ca6e4 100644
--- a/polly/test/ScopInfo/multiple-types-two-dimensional.ll
+++ b/polly/test/ScopInfo/multiple-types-two-dimensional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -pass-remarks-analysis="polly-scops" \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -pass-remarks-analysis="polly-scops" \
 ; RUN:     -polly-allow-differing-element-types \
 ; RUN:     -disable-output < %s  2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/multiple-types.ll b/polly/test/ScopInfo/multiple-types.ll
index 16db191c522f..b7006c104426 100644
--- a/polly/test/ScopInfo/multiple-types.ll
+++ b/polly/test/ScopInfo/multiple-types.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops \
-; RUN: -polly-allow-differing-element-types -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' \
+; RUN: -polly-allow-differing-element-types -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    // Check that accessing one array with different types works.
 ;    void multiple_types(char *Short, char *Float, char *Double) {
diff --git a/polly/test/ScopInfo/multiple_exiting_blocks.ll b/polly/test/ScopInfo/multiple_exiting_blocks.ll
index f8e5d4106a16..a21a7252c2ff 100644
--- a/polly/test/ScopInfo/multiple_exiting_blocks.ll
+++ b/polly/test/ScopInfo/multiple_exiting_blocks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/multiple_exiting_blocks_two_loop.ll b/polly/test/ScopInfo/multiple_exiting_blocks_two_loop.ll
index c695f3c913db..e56c8b7094ea 100644
--- a/polly/test/ScopInfo/multiple_exiting_blocks_two_loop.ll
+++ b/polly/test/ScopInfo/multiple_exiting_blocks_two_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/multiple_latch_blocks.ll b/polly/test/ScopInfo/multiple_latch_blocks.ll
index d3949e7e2c3c..c1a64ee1c3cb 100644
--- a/polly/test/ScopInfo/multiple_latch_blocks.ll
+++ b/polly/test/ScopInfo/multiple_latch_blocks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Domain :=
 ; CHECK:   [N, P] -> { Stmt_if_end[i0] : 0 <= i0 < N and (i0 > P or i0 < P) };
diff --git a/polly/test/ScopInfo/nested-loops.ll b/polly/test/ScopInfo/nested-loops.ll
index ed814f826829..2819ae461fce 100644
--- a/polly/test/ScopInfo/nested-loops.ll
+++ b/polly/test/ScopInfo/nested-loops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
diff --git a/polly/test/ScopInfo/no-scalar-deps-in-non-affine-subregion.ll b/polly/test/ScopInfo/no-scalar-deps-in-non-affine-subregion.ll
index 7c55e242641c..6fc364076750 100644
--- a/polly/test/ScopInfo/no-scalar-deps-in-non-affine-subregion.ll
+++ b/polly/test/ScopInfo/no-scalar-deps-in-non-affine-subregion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that we do not generate any scalar dependences regarding x. It is
 ; defined and used on the non-affine subregion only, thus we do not need
diff --git a/polly/test/ScopInfo/non-affine-region-phi.ll b/polly/test/ScopInfo/non-affine-region-phi.ll
index f99782b9a0ff..4de76c4adfb1 100644
--- a/polly/test/ScopInfo/non-affine-region-phi.ll
+++ b/polly/test/ScopInfo/non-affine-region-phi.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -S < %s | FileCheck %s --check-prefix=CODE
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -S < %s 2>&1 | FileCheck %s --check-prefix=CODE
+; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify there is a phi in the non-affine region but it is not represented in
 ; the SCoP as all operands as well as the uses are inside the region too.
diff --git a/polly/test/ScopInfo/non-affine-region-with-loop-2.ll b/polly/test/ScopInfo/non-affine-region-with-loop-2.ll
index b673fda5ec3c..9870b813d287 100644
--- a/polly/test/ScopInfo/non-affine-region-with-loop-2.ll
+++ b/polly/test/ScopInfo/non-affine-region-with-loop-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-nonaffine-loops -polly-print-scops -polly-codegen -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-nonaffine-loops '-passes=print<polly-detect>,print<polly-function-scops>,scop(polly-codegen)' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:    Stmt_loop3
 ; CHECK:            Domain :=
diff --git a/polly/test/ScopInfo/non-affine-region-with-loop.ll b/polly/test/ScopInfo/non-affine-region-with-loop.ll
index 32dde8b4a682..e1342e1b5257 100644
--- a/polly/test/ScopInfo/non-affine-region-with-loop.ll
+++ b/polly/test/ScopInfo/non-affine-region-with-loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-codegen -disable-output
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops -passes=polly-codegen -disable-output
 ;
 ; CHECK:      Domain :=
 ; CHECK-NEXT:   { Stmt_loop2__TO__loop[] };
diff --git a/polly/test/ScopInfo/non-precise-inv-load-1.ll b/polly/test/ScopInfo/non-precise-inv-load-1.ll
index 5394206dd547..f35235d2de8f 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-1.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify we do hoist the invariant access to I with a execution context
 ; as the address computation might wrap in the original but not in our
diff --git a/polly/test/ScopInfo/non-precise-inv-load-2.ll b/polly/test/ScopInfo/non-precise-inv-load-2.ll
index 5c0c56513a08..c538c0e7bda0 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-2.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;
 ; CHECK:       Invariant Accesses: {
diff --git a/polly/test/ScopInfo/non-precise-inv-load-3.ll b/polly/test/ScopInfo/non-precise-inv-load-3.ll
index 09d09319656b..c16879a18562 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-3.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:        Invariant Accesses: {
 ; CHECK-NEXT:     ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/non-precise-inv-load-4.ll b/polly/test/ScopInfo/non-precise-inv-load-4.ll
index da5f656576d1..24f9d45d28e7 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-4.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify we hoist I[0] without execution context even though it
 ; is executed in a statement with an invalid domain.
diff --git a/polly/test/ScopInfo/non-precise-inv-load-5.ll b/polly/test/ScopInfo/non-precise-inv-load-5.ll
index bff5f59a3302..17046685562d 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-5.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify we do not hoist I[c] without execution context because it
 ; is executed in a statement with an invalid domain and it depends
diff --git a/polly/test/ScopInfo/non-precise-inv-load-6.ll b/polly/test/ScopInfo/non-precise-inv-load-6.ll
index 03540a8ead96..eeada91299f6 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-6.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-6.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that we model the execution context correctly.
 ;
diff --git a/polly/test/ScopInfo/non-pure-function-call.ll b/polly/test/ScopInfo/non-pure-function-call.ll
index 4ffb8d28865d..974c9ba0527d 100644
--- a/polly/test/ScopInfo/non-pure-function-call.ll
+++ b/polly/test/ScopInfo/non-pure-function-call.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:      Assumed Context:
 ; CHECK-NEXT:   [N] -> {  :  }
diff --git a/polly/test/ScopInfo/non-pure-function-calls-causes-dead-blocks.ll b/polly/test/ScopInfo/non-pure-function-calls-causes-dead-blocks.ll
index 27998b50b74f..983c45bc7536 100644
--- a/polly/test/ScopInfo/non-pure-function-calls-causes-dead-blocks.ll
+++ b/polly/test/ScopInfo/non-pure-function-calls-causes-dead-blocks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Error blocks are skipped during SCoP detection. We skip them during
 ; SCoP formation too as they might contain instructions we can not handle.
diff --git a/polly/test/ScopInfo/non-pure-function-calls.ll b/polly/test/ScopInfo/non-pure-function-calls.ll
index 3ecf75853773..fd6a6dc3f8f1 100644
--- a/polly/test/ScopInfo/non-pure-function-calls.ll
+++ b/polly/test/ScopInfo/non-pure-function-calls.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Allow the user to define function names that are treated as
 ; error functions and assumed not to be executed.
diff --git a/polly/test/ScopInfo/non_affine_access.ll b/polly/test/ScopInfo/non_affine_access.ll
index a83c9484ad52..e20be38598db 100644
--- a/polly/test/ScopInfo/non_affine_access.ll
+++ b/polly/test/ScopInfo/non_affine_access.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-scops -polly-allow-nonaffine -disable-output < %s | FileCheck %s -check-prefix=NONAFFINE
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s -check-prefix=NONAFFINE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long *A) {
diff --git a/polly/test/ScopInfo/non_affine_region_1.ll b/polly/test/ScopInfo/non_affine_region_1.ll
index 7c4312599cf0..623d322c508c 100644
--- a/polly/test/ScopInfo/non_affine_region_1.ll
+++ b/polly/test/ScopInfo/non_affine_region_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify only the incoming scalar x is modeled as a read in the non-affine
 ; region.
diff --git a/polly/test/ScopInfo/non_affine_region_2.ll b/polly/test/ScopInfo/non_affine_region_2.ll
index 0bc467c92bcb..ba20d9e6cf6e 100644
--- a/polly/test/ScopInfo/non_affine_region_2.ll
+++ b/polly/test/ScopInfo/non_affine_region_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify the scalar x defined in a non-affine subregion is written as it
 ; escapes the region. In this test the two conditionals inside the region
diff --git a/polly/test/ScopInfo/non_affine_region_3.ll b/polly/test/ScopInfo/non_affine_region_3.ll
index 6d5f94df6110..ff619b579355 100644
--- a/polly/test/ScopInfo/non_affine_region_3.ll
+++ b/polly/test/ScopInfo/non_affine_region_3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify the scalar x defined in a non-affine subregion is written as it
 ; escapes the region. In this test the two conditionals inside the region
diff --git a/polly/test/ScopInfo/non_affine_region_4.ll b/polly/test/ScopInfo/non_affine_region_4.ll
index f37e0ecb89d1..70f40727849a 100644
--- a/polly/test/ScopInfo/non_affine_region_4.ll
+++ b/polly/test/ScopInfo/non_affine_region_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify that both scalars (x and y) are properly written in the non-affine
 ; region and read afterwards.
diff --git a/polly/test/ScopInfo/nonaffine-buildMemoryAccess.ll b/polly/test/ScopInfo/nonaffine-buildMemoryAccess.ll
index 445dd164898b..246610ee5194 100644
--- a/polly/test/ScopInfo/nonaffine-buildMemoryAccess.ll
+++ b/polly/test/ScopInfo/nonaffine-buildMemoryAccess.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:      Domain :=
 ; CHECK-NEXT:   { Stmt_while_cond_i__TO__while_end_i[] };
diff --git a/polly/test/ScopInfo/not-a-reduction.ll b/polly/test/ScopInfo/not-a-reduction.ll
index 87909290fd71..7fe41332c67d 100644
--- a/polly/test/ScopInfo/not-a-reduction.ll
+++ b/polly/test/ScopInfo/not-a-reduction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s 2>&1 | not FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | not FileCheck %s
 
 ;#define TYPE float
 ;#define NUM 4
diff --git a/polly/test/ScopInfo/opaque-struct.ll b/polly/test/ScopInfo/opaque-struct.ll
index 19fdd9bf9179..1a0859f71f36 100644
--- a/polly/test/ScopInfo/opaque-struct.ll
+++ b/polly/test/ScopInfo/opaque-struct.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-scops -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
 ;
 ; Check that we do not crash with unsized (opaque) types.
 ;
diff --git a/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node-nonaffine-subregion.ll b/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node-nonaffine-subregion.ll
index 394173bdc986..58c2116ab9dc 100644
--- a/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node-nonaffine-subregion.ll
+++ b/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node-nonaffine-subregion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S < %s 2>&1 | FileCheck %s
 ;
 ; Check whether %newval is identified as escaping value, even though it is used
 ; in a phi that is in the region. Non-affine subregion case.
diff --git a/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node.ll b/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node.ll
index e17164e89372..7986399a6c7e 100644
--- a/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node.ll
+++ b/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK: MustWriteAccess :=  [Reduction Type: NONE] [Scalar: 1]
 ; CHECK-NEXT: [p_0] -> { Stmt_bb3[] -> MemRef_tmp5[] };
diff --git a/polly/test/ScopInfo/parameter-constant-division.ll b/polly/test/ScopInfo/parameter-constant-division.ll
index cd6b9e3526aa..f1e006e29454 100644
--- a/polly/test/ScopInfo/parameter-constant-division.ll
+++ b/polly/test/ScopInfo/parameter-constant-division.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -disable-output < %s | FileCheck %s
+; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:          Invariant Accesses: {
 ; CHECK-NEXT:            ReadAccess :=	[Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/parameter_in_dead_statement.ll b/polly/test/ScopInfo/parameter_in_dead_statement.ll
index 4b4a87f098d7..13602515d46f 100644
--- a/polly/test/ScopInfo/parameter_in_dead_statement.ll
+++ b/polly/test/ScopInfo/parameter_in_dead_statement.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -S \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -S \
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s --check-prefix=IR
 ;
 ; Verify we do not create assumptions based on the parameter p_1 which is the
 ; load %0 and due to error-assumptions not "part of the SCoP".
diff --git a/polly/test/ScopInfo/parameter_product.ll b/polly/test/ScopInfo/parameter_product.ll
index 1ba7280f97c9..9783268f2a10 100644
--- a/polly/test/ScopInfo/parameter_product.ll
+++ b/polly/test/ScopInfo/parameter_product.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; int n, m;
 ; void foo(char* __restrict a)
diff --git a/polly/test/ScopInfo/parameter_with_constant_factor_in_add.ll b/polly/test/ScopInfo/parameter_with_constant_factor_in_add.ll
index 72d580801573..f750cf7f3320 100644
--- a/polly/test/ScopInfo/parameter_with_constant_factor_in_add.ll
+++ b/polly/test/ScopInfo/parameter_with_constant_factor_in_add.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that the access function of the store is simple and concise
 ;
diff --git a/polly/test/ScopInfo/partially_invariant_load_1.ll b/polly/test/ScopInfo/partially_invariant_load_1.ll
index 274a7873c782..5757e4d7095c 100644
--- a/polly/test/ScopInfo/partially_invariant_load_1.ll
+++ b/polly/test/ScopInfo/partially_invariant_load_1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=IR
 ;
 ; CHECK:          Invariant Accesses: {
 ; CHECK-NEXT:             ReadAccess :=	[Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/partially_invariant_load_2.ll b/polly/test/ScopInfo/partially_invariant_load_2.ll
index ee1092883f72..e07d6d39132b 100644
--- a/polly/test/ScopInfo/partially_invariant_load_2.ll
+++ b/polly/test/ScopInfo/partially_invariant_load_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that we do not try to preload *I and assume p != 42.
 ;
diff --git a/polly/test/ScopInfo/phi-in-non-affine-region.ll b/polly/test/ScopInfo/phi-in-non-affine-region.ll
index 6ef24e3f1456..c8e81fd1ba98 100644
--- a/polly/test/ScopInfo/phi-in-non-affine-region.ll
+++ b/polly/test/ScopInfo/phi-in-non-affine-region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; Verify that 'tmp' is stored in bb1 and read by bb3, as it is needed as
 ; incoming value for the tmp11 PHI node.
diff --git a/polly/test/ScopInfo/phi_after_error_block.ll b/polly/test/ScopInfo/phi_after_error_block.ll
index 039fb86bec5b..21d532b06d60 100644
--- a/polly/test/ScopInfo/phi_after_error_block.ll
+++ b/polly/test/ScopInfo/phi_after_error_block.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 declare void @bar()
 
diff --git a/polly/test/ScopInfo/phi_condition_modeling_1.ll b/polly/test/ScopInfo/phi_condition_modeling_1.ll
index a879c2005ad8..ca87055630c6 100644
--- a/polly/test/ScopInfo/phi_condition_modeling_1.ll
+++ b/polly/test/ScopInfo/phi_condition_modeling_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, int c, int N) {
 ;      int tmp;
diff --git a/polly/test/ScopInfo/phi_condition_modeling_2.ll b/polly/test/ScopInfo/phi_condition_modeling_2.ll
index cedc140f8438..10511ef0009f 100644
--- a/polly/test/ScopInfo/phi_condition_modeling_2.ll
+++ b/polly/test/ScopInfo/phi_condition_modeling_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, int c, int N) {
 ;      int tmp;
diff --git a/polly/test/ScopInfo/phi_conditional_simple_1.ll b/polly/test/ScopInfo/phi_conditional_simple_1.ll
index 90213a953767..2a009dc26d24 100644
--- a/polly/test/ScopInfo/phi_conditional_simple_1.ll
+++ b/polly/test/ScopInfo/phi_conditional_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void jd(int *A, int c) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/ScopInfo/phi_loop_carried_float.ll b/polly/test/ScopInfo/phi_loop_carried_float.ll
index d8d2608329bc..5bb740154877 100644
--- a/polly/test/ScopInfo/phi_loop_carried_float.ll
+++ b/polly/test/ScopInfo/phi_loop_carried_float.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    float f(float *A, int N) {
 ;      float tmp = 0;
diff --git a/polly/test/ScopInfo/phi_not_grouped_at_top.ll b/polly/test/ScopInfo/phi_not_grouped_at_top.ll
index be082165b635..1ed22b3fec42 100644
--- a/polly/test/ScopInfo/phi_not_grouped_at_top.ll
+++ b/polly/test/ScopInfo/phi_not_grouped_at_top.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-prepare -disable-output < %s
+; RUN: opt %loadPolly -passes=polly-prepare -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 declare i32 @funa() align 2
diff --git a/polly/test/ScopInfo/phi_scalar_simple_1.ll b/polly/test/ScopInfo/phi_scalar_simple_1.ll
index d042613c023f..eab261b0d153 100644
--- a/polly/test/ScopInfo/phi_scalar_simple_1.ll
+++ b/polly/test/ScopInfo/phi_scalar_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; The assumed context should be empty since the <nsw> flags on the IV
 ; increments already guarantee that there is no wrap in the loop trip
diff --git a/polly/test/ScopInfo/phi_scalar_simple_2.ll b/polly/test/ScopInfo/phi_scalar_simple_2.ll
index fb4292e05ca6..73bef9601f33 100644
--- a/polly/test/ScopInfo/phi_scalar_simple_2.ll
+++ b/polly/test/ScopInfo/phi_scalar_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    int jd(int *restrict A, int x, int N, int c) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/phi_with_invoke_edge.ll b/polly/test/ScopInfo/phi_with_invoke_edge.ll
index dbcf04c0561a..3d7b7d3d38d5 100644
--- a/polly/test/ScopInfo/phi_with_invoke_edge.ll
+++ b/polly/test/ScopInfo/phi_with_invoke_edge.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-detect -disable-output < %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 declare i32 @generic_personality_v0(i32, i64, ptr, ptr)
diff --git a/polly/test/ScopInfo/pointer-comparison-no-nsw.ll b/polly/test/ScopInfo/pointer-comparison-no-nsw.ll
index 094c5ccab54d..40d2138f81d5 100644
--- a/polly/test/ScopInfo/pointer-comparison-no-nsw.ll
+++ b/polly/test/ScopInfo/pointer-comparison-no-nsw.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, int *B) {
 ;      while (A != B) {
diff --git a/polly/test/ScopInfo/pointer-comparison.ll b/polly/test/ScopInfo/pointer-comparison.ll
index 15ce0491209a..960b9c5f3132 100644
--- a/polly/test/ScopInfo/pointer-comparison.ll
+++ b/polly/test/ScopInfo/pointer-comparison.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; TODO: FIXME: Investigate why we need a InvalidContext here.
 ;
diff --git a/polly/test/ScopInfo/pointer-type-expressions.ll b/polly/test/ScopInfo/pointer-type-expressions.ll
index ebbb644340f6..919a9fd4000b 100644
--- a/polly/test/ScopInfo/pointer-type-expressions.ll
+++ b/polly/test/ScopInfo/pointer-type-expressions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(int a[], int N, float *P) {
 ;   int i;
diff --git a/polly/test/ScopInfo/pointer-used-as-base-pointer-and-scalar-read.ll b/polly/test/ScopInfo/pointer-used-as-base-pointer-and-scalar-read.ll
index 3ac86a3443af..1eb053c83132 100644
--- a/polly/test/ScopInfo/pointer-used-as-base-pointer-and-scalar-read.ll
+++ b/polly/test/ScopInfo/pointer-used-as-base-pointer-and-scalar-read.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; In this test case we pass a pointer %A into a PHI node and also use this
 ; pointer as base pointer of an array store. As a result, we get both scalar
diff --git a/polly/test/ScopInfo/polly-timeout-parameter-bounds.ll b/polly/test/ScopInfo/polly-timeout-parameter-bounds.ll
index 8152010c2c99..7723c185e41c 100644
--- a/polly/test/ScopInfo/polly-timeout-parameter-bounds.ll
+++ b/polly/test/ScopInfo/polly-timeout-parameter-bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK:      Statements {
 ; CHECK-NEXT:  	Stmt_bb9
diff --git a/polly/test/ScopInfo/preserve-equiv-class-order-in-basic_block.ll b/polly/test/ScopInfo/preserve-equiv-class-order-in-basic_block.ll
index 4a68acd3d509..3cc3e51ef013 100644
--- a/polly/test/ScopInfo/preserve-equiv-class-order-in-basic_block.ll
+++ b/polly/test/ScopInfo/preserve-equiv-class-order-in-basic_block.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/process_added_dimensions.ll b/polly/test/ScopInfo/process_added_dimensions.ll
index 6cb270a071f4..66c9ded40f7c 100644
--- a/polly/test/ScopInfo/process_added_dimensions.ll
+++ b/polly/test/ScopInfo/process_added_dimensions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK:      Context:
 ; CHECK-NEXT: {  :  }
diff --git a/polly/test/ScopInfo/pwaff-complexity-bailout.ll b/polly/test/ScopInfo/pwaff-complexity-bailout.ll
index 19dd156d27db..5119334745bc 100644
--- a/polly/test/ScopInfo/pwaff-complexity-bailout.ll
+++ b/polly/test/ScopInfo/pwaff-complexity-bailout.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-scops -pass-remarks-analysis=.* -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -pass-remarks-analysis=.* -disable-output < %s 2>&1 | FileCheck %s
 
 ; Make sure we hit the complexity bailout, and don't crash.
 ; CHECK: Low complexity assumption:       {  : false }
diff --git a/polly/test/ScopInfo/ranged_parameter.ll b/polly/test/ScopInfo/ranged_parameter.ll
index 4b04960ee845..b5cb77593352 100644
--- a/polly/test/ScopInfo/ranged_parameter.ll
+++ b/polly/test/ScopInfo/ranged_parameter.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that the constraints on the parameter derived from the
 ; range metadata (see bottom of the file) are present:
diff --git a/polly/test/ScopInfo/ranged_parameter_2.ll b/polly/test/ScopInfo/ranged_parameter_2.ll
index cd7d2bfb84d0..52933398f796 100644
--- a/polly/test/ScopInfo/ranged_parameter_2.ll
+++ b/polly/test/ScopInfo/ranged_parameter_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output -polly-allow-nonaffine -polly-invariant-load-hoisting=true < %s \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-allow-nonaffine -polly-invariant-load-hoisting=true < %s \
 ; RUN:  -debug 2>&1 | FileCheck %s
 
 ; REQUIRES: asserts
diff --git a/polly/test/ScopInfo/ranged_parameter_wrap.ll b/polly/test/ScopInfo/ranged_parameter_wrap.ll
index 173746352cf0..724427fabfd1 100644
--- a/polly/test/ScopInfo/ranged_parameter_wrap.ll
+++ b/polly/test/ScopInfo/ranged_parameter_wrap.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that the constraints on the parameter derived from the
 ; __wrapping__ range metadata (see bottom of the file) are present:
diff --git a/polly/test/ScopInfo/ranged_parameter_wrap_2.ll b/polly/test/ScopInfo/ranged_parameter_wrap_2.ll
index 33f57f37a1e8..234c3edde14e 100644
--- a/polly/test/ScopInfo/ranged_parameter_wrap_2.ll
+++ b/polly/test/ScopInfo/ranged_parameter_wrap_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that the context is built fast and does not explode due to us
 ; combining a large number of non-convex ranges. Instead, after a certain
diff --git a/polly/test/ScopInfo/read-only-scalar-used-in-phi-2.ll b/polly/test/ScopInfo/read-only-scalar-used-in-phi-2.ll
index 23c7aa261ac0..1ab8fe897308 100644
--- a/polly/test/ScopInfo/read-only-scalar-used-in-phi-2.ll
+++ b/polly/test/ScopInfo/read-only-scalar-used-in-phi-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    float foo(float sum, float A[]) {
 ;
diff --git a/polly/test/ScopInfo/read-only-scalar-used-in-phi.ll b/polly/test/ScopInfo/read-only-scalar-used-in-phi.ll
index 20f44c94251c..358b51904c72 100644
--- a/polly/test/ScopInfo/read-only-scalar-used-in-phi.ll
+++ b/polly/test/ScopInfo/read-only-scalar-used-in-phi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    float foo(float sum, float A[]) {
 ;
diff --git a/polly/test/ScopInfo/read-only-scalars.ll b/polly/test/ScopInfo/read-only-scalars.ll
index 71c2d21e357a..43a456ea9977 100644
--- a/polly/test/ScopInfo/read-only-scalars.ll
+++ b/polly/test/ScopInfo/read-only-scalars.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-analyze-read-only-scalars=false -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-analyze-read-only-scalars=true  -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=SCALARS
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-analyze-read-only-scalars=false '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-analyze-read-only-scalars=true  '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=SCALARS
 
 ; CHECK-NOT: Memref_scalar
 
diff --git a/polly/test/ScopInfo/read-only-statements.ll b/polly/test/ScopInfo/read-only-statements.ll
index a93063ea3ad6..3fa72789f4e1 100644
--- a/polly/test/ScopInfo/read-only-statements.ll
+++ b/polly/test/ScopInfo/read-only-statements.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check we remove read only statements.
 ;
diff --git a/polly/test/ScopInfo/reduction_alternating_base.ll b/polly/test/ScopInfo/reduction_alternating_base.ll
index 854e28023a3e..f44367f295ff 100644
--- a/polly/test/ScopInfo/reduction_alternating_base.ll
+++ b/polly/test/ScopInfo/reduction_alternating_base.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;
 ;    void f(int *A) {
diff --git a/polly/test/ScopInfo/reduction_chain_partially_outside_the_scop.ll b/polly/test/ScopInfo/reduction_chain_partially_outside_the_scop.ll
index fb0274972082..5636ee7ed828 100644
--- a/polly/test/ScopInfo/reduction_chain_partially_outside_the_scop.ll
+++ b/polly/test/ScopInfo/reduction_chain_partially_outside_the_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Reduction Type: NONE
 ;
diff --git a/polly/test/ScopInfo/reduction_different_index.ll b/polly/test/ScopInfo/reduction_different_index.ll
index 575e5a16d7b2..7ed9e662a15c 100644
--- a/polly/test/ScopInfo/reduction_different_index.ll
+++ b/polly/test/ScopInfo/reduction_different_index.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ; Verify if the following case is not detected as reduction.
 ;
 ; void f(int *A,int *sum) {
diff --git a/polly/test/ScopInfo/reduction_different_index1.ll b/polly/test/ScopInfo/reduction_different_index1.ll
index 39bd3c4b9abe..f868bd657f3b 100644
--- a/polly/test/ScopInfo/reduction_different_index1.ll
+++ b/polly/test/ScopInfo/reduction_different_index1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ; Verify if the following case is not detected as reduction.
 ;
 ; void f(int *A, int *sum, int i1, int i2) {
diff --git a/polly/test/ScopInfo/reduction_disabled_multiplicative.ll b/polly/test/ScopInfo/reduction_disabled_multiplicative.ll
index 7120740fbf34..b031fd352323 100644
--- a/polly/test/ScopInfo/reduction_disabled_multiplicative.ll
+++ b/polly/test/ScopInfo/reduction_disabled_multiplicative.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basic-aa %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-disable-multiplicative-reductions -disable-output < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-disable-multiplicative-reductions -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: ReadAccess :=       [Reduction Type: +
 ; CHECK:     { Stmt_for_body[i0] -> MemRef_sum[0] };
diff --git a/polly/test/ScopInfo/reduction_escaping_intermediate.ll b/polly/test/ScopInfo/reduction_escaping_intermediate.ll
index dde09108ecc4..dbfa1f1b1d59 100644
--- a/polly/test/ScopInfo/reduction_escaping_intermediate.ll
+++ b/polly/test/ScopInfo/reduction_escaping_intermediate.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; void f(int N, int * restrict sums, int * restrict escape) {
 ;   int i, j;
diff --git a/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll b/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll
index 702fc56025d9..1fa8bbcc53e0 100644
--- a/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll
+++ b/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; void f(int N, int * restrict sums, int * restrict escape) {
 ;   int i, j;
diff --git a/polly/test/ScopInfo/reduction_invalid_different_operators.ll b/polly/test/ScopInfo/reduction_invalid_different_operators.ll
index f47919dcad99..d1bfb71fbbd4 100644
--- a/polly/test/ScopInfo/reduction_invalid_different_operators.ll
+++ b/polly/test/ScopInfo/reduction_invalid_different_operators.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; int f() {
 ;   int i, sum = 0, sth = 0;
diff --git a/polly/test/ScopInfo/reduction_invalid_overlapping_accesses.ll b/polly/test/ScopInfo/reduction_invalid_overlapping_accesses.ll
index be1d7b5bbbd9..654475253799 100644
--- a/polly/test/ScopInfo/reduction_invalid_overlapping_accesses.ll
+++ b/polly/test/ScopInfo/reduction_invalid_overlapping_accesses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; void f(int *sums) {
 ;   int i, j;
diff --git a/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll b/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll
index 8d20fa13ffe5..a8167f1c38de 100644
--- a/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll
+++ b/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basic-aa %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Stmt_for_body
 ; CHECK: Reduction Type: *
diff --git a/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll b/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll
index 782332b56aad..a0c54572b599 100644
--- a/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll
+++ b/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basic-aa %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Stmt_for_body
 ; CHECK: Reduction Type: NONE
diff --git a/polly/test/ScopInfo/reduction_multiple_simple_binary.ll b/polly/test/ScopInfo/reduction_multiple_simple_binary.ll
index 0f1a3ad90dac..3ed664050a0b 100644
--- a/polly/test/ScopInfo/reduction_multiple_simple_binary.ll
+++ b/polly/test/ScopInfo/reduction_multiple_simple_binary.ll
@@ -1,4 +1,4 @@
-; RUN: opt -basic-aa %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: ReadAccess :=       [Reduction Type: NONE
 ; CHECK:     { Stmt_for_body[i0] -> MemRef_A[1 + i0] };
diff --git a/polly/test/ScopInfo/reduction_non_overlapping_chains.ll b/polly/test/ScopInfo/reduction_non_overlapping_chains.ll
index 4e3f841cd8e1..7c8c8616a1cd 100644
--- a/polly/test/ScopInfo/reduction_non_overlapping_chains.ll
+++ b/polly/test/ScopInfo/reduction_non_overlapping_chains.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Reduction Type: +
 ; CHECK: Reduction Type: +
diff --git a/polly/test/ScopInfo/reduction_only_reduction_like_access.ll b/polly/test/ScopInfo/reduction_only_reduction_like_access.ll
index 0c61d63a2d45..95cda973a9b0 100644
--- a/polly/test/ScopInfo/reduction_only_reduction_like_access.ll
+++ b/polly/test/ScopInfo/reduction_only_reduction_like_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Reduction Type: +
 ;
diff --git a/polly/test/ScopInfo/reduction_simple_fp.ll b/polly/test/ScopInfo/reduction_simple_fp.ll
index ba0a034a17e3..37693353376b 100644
--- a/polly/test/ScopInfo/reduction_simple_fp.ll
+++ b/polly/test/ScopInfo/reduction_simple_fp.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Function: f_no_fast_math
 ; CHECK: Reduction Type: NONE
diff --git a/polly/test/ScopInfo/reduction_simple_w_constant.ll b/polly/test/ScopInfo/reduction_simple_w_constant.ll
index dc1f8550602d..550882300116 100644
--- a/polly/test/ScopInfo/reduction_simple_w_constant.ll
+++ b/polly/test/ScopInfo/reduction_simple_w_constant.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Reduction Type: +
 ;
diff --git a/polly/test/ScopInfo/reduction_simple_w_iv.ll b/polly/test/ScopInfo/reduction_simple_w_iv.ll
index b6c3229d08d5..480c2ebf8d47 100644
--- a/polly/test/ScopInfo/reduction_simple_w_iv.ll
+++ b/polly/test/ScopInfo/reduction_simple_w_iv.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Reduction Type: +
 ;
diff --git a/polly/test/ScopInfo/reduction_two_identical_reads.ll b/polly/test/ScopInfo/reduction_two_identical_reads.ll
index 19d45a5f4ea9..7fce22d15c77 100644
--- a/polly/test/ScopInfo/reduction_two_identical_reads.ll
+++ b/polly/test/ScopInfo/reduction_two_identical_reads.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Reduction Type: NONE
 ;
diff --git a/polly/test/ScopInfo/redundant_parameter_constraint.ll b/polly/test/ScopInfo/redundant_parameter_constraint.ll
index c9d912191eed..231cab0fda1f 100644
--- a/polly/test/ScopInfo/redundant_parameter_constraint.ll
+++ b/polly/test/ScopInfo/redundant_parameter_constraint.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; The constraint that r2 has to be bigger than r1 is implicitly contained in
 ; the domain, hence we do not want to see it explicitly.
diff --git a/polly/test/ScopInfo/region-with-instructions.ll b/polly/test/ScopInfo/region-with-instructions.ll
index 39d4a72a7814..a3040636836f 100644
--- a/polly/test/ScopInfo/region-with-instructions.ll
+++ b/polly/test/ScopInfo/region-with-instructions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -polly-print-instructions -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-print-instructions -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK: Statements {
 ; CHECK: 	Stmt_bb46
diff --git a/polly/test/ScopInfo/remarks.ll b/polly/test/ScopInfo/remarks.ll
index dcdeb58c7694..0a6ef2f1e5f7 100644
--- a/polly/test/ScopInfo/remarks.ll
+++ b/polly/test/ScopInfo/remarks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
 ; RUN: -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: remark: test/ScopInfo/remarks.c:4:7: SCoP begins here.
diff --git a/polly/test/ScopInfo/required-invariant-loop-bounds.ll b/polly/test/ScopInfo/required-invariant-loop-bounds.ll
index 248acbea6e68..19ed625a85db 100644
--- a/polly/test/ScopInfo/required-invariant-loop-bounds.ll
+++ b/polly/test/ScopInfo/required-invariant-loop-bounds.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:      Invariant Accesses: {
 ; CHECK-NEXT:       ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/restriction_in_dead_block.ll b/polly/test/ScopInfo/restriction_in_dead_block.ll
index 81d9b96be419..27df53f03b03 100644
--- a/polly/test/ScopInfo/restriction_in_dead_block.ll
+++ b/polly/test/ScopInfo/restriction_in_dead_block.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify we do not generate an empty invalid context only because the wrap
 ; in the second conditional will always happen if the block is executed.
diff --git a/polly/test/ScopInfo/run-time-check-many-array-disjuncts.ll b/polly/test/ScopInfo/run-time-check-many-array-disjuncts.ll
index d36da2b2becf..e84a1b3e5bc6 100644
--- a/polly/test/ScopInfo/run-time-check-many-array-disjuncts.ll
+++ b/polly/test/ScopInfo/run-time-check-many-array-disjuncts.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 \
 ; RUN: | FileCheck %s -check-prefix=DETECT
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; DETECT: Valid Region for Scop: bb124 => bb176
 ;
diff --git a/polly/test/ScopInfo/run-time-check-many-parameters.ll b/polly/test/ScopInfo/run-time-check-many-parameters.ll
index 30f8d5fff34c..540ea57fad0c 100644
--- a/polly/test/ScopInfo/run-time-check-many-parameters.ll
+++ b/polly/test/ScopInfo/run-time-check-many-parameters.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; A valid Scop would print the list of it's statements, we check that we do not
 ; see that list.
diff --git a/polly/test/ScopInfo/run-time-check-many-piecewise-aliasing.ll b/polly/test/ScopInfo/run-time-check-many-piecewise-aliasing.ll
index 487c803bba98..cefda1eed0c6 100644
--- a/polly/test/ScopInfo/run-time-check-many-piecewise-aliasing.ll
+++ b/polly/test/ScopInfo/run-time-check-many-piecewise-aliasing.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-print-detect -disable-output < %s \
+; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 \
 ; RUN: | FileCheck %s -check-prefix=DETECT
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; DETECT: Valid Region for Scop: for => return
 ;
diff --git a/polly/test/ScopInfo/run-time-check-read-only-arrays.ll b/polly/test/ScopInfo/run-time-check-read-only-arrays.ll
index d590aaf00ddb..395622b12616 100644
--- a/polly/test/ScopInfo/run-time-check-read-only-arrays.ll
+++ b/polly/test/ScopInfo/run-time-check-read-only-arrays.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; void foo(float *A, float *B, float *C, long N) {
 ; 	for (long i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/same-base-address-scalar-and-array.ll b/polly/test/ScopInfo/same-base-address-scalar-and-array.ll
index a5f353e7ad2a..22cf77636d10 100644
--- a/polly/test/ScopInfo/same-base-address-scalar-and-array.ll
+++ b/polly/test/ScopInfo/same-base-address-scalar-and-array.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify we introduce two ScopArrayInfo objects (or virtual arrays) for the %out variable
 ; as it is used as a memory base pointer (%0) but also as a scalar (%out.addr.0.lcssa).
diff --git a/polly/test/ScopInfo/scalar.ll b/polly/test/ScopInfo/scalar.ll
index c38eaa853b9b..80493723bbfc 100644
--- a/polly/test/ScopInfo/scalar.ll
+++ b/polly/test/ScopInfo/scalar.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
diff --git a/polly/test/ScopInfo/scalar_dependence_cond_br.ll b/polly/test/ScopInfo/scalar_dependence_cond_br.ll
index 3303bfb7c6c5..940dabbc4cfc 100644
--- a/polly/test/ScopInfo/scalar_dependence_cond_br.ll
+++ b/polly/test/ScopInfo/scalar_dependence_cond_br.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output< %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output< %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, int c, int d) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/ScopInfo/scalar_to_array.ll b/polly/test/ScopInfo/scalar_to_array.ll
index 5c275108602a..692a0dbd67c8 100644
--- a/polly/test/ScopInfo/scalar_to_array.ll
+++ b/polly/test/ScopInfo/scalar_to_array.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -basic-aa -polly-print-function-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; ModuleID = 'scalar_to_array.ll'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/scev-div-with-evaluatable-divisor.ll b/polly/test/ScopInfo/scev-div-with-evaluatable-divisor.ll
index fc7a1bfc3d5e..f969176cee16 100644
--- a/polly/test/ScopInfo/scev-div-with-evaluatable-divisor.ll
+++ b/polly/test/ScopInfo/scev-div-with-evaluatable-divisor.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; Derived from test-suite/SingleSource/UnitTests/Vector/SSE/sse.stepfft.c
 
diff --git a/polly/test/ScopInfo/scev-invalidated.ll b/polly/test/ScopInfo/scev-invalidated.ll
index 97fc5ec3d4ca..921cb06a0cd5 100644
--- a/polly/test/ScopInfo/scev-invalidated.ll
+++ b/polly/test/ScopInfo/scev-invalidated.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Region: %if.then6---%return
 ;
diff --git a/polly/test/ScopInfo/schedule-const-post-dominator-walk-2.ll b/polly/test/ScopInfo/schedule-const-post-dominator-walk-2.ll
index 2fdf7d66c3ad..e956b0ed7cb9 100644
--- a/polly/test/ScopInfo/schedule-const-post-dominator-walk-2.ll
+++ b/polly/test/ScopInfo/schedule-const-post-dominator-walk-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/schedule-const-post-dominator-walk.ll b/polly/test/ScopInfo/schedule-const-post-dominator-walk.ll
index 92685858610c..325d19dd210d 100644
--- a/polly/test/ScopInfo/schedule-const-post-dominator-walk.ll
+++ b/polly/test/ScopInfo/schedule-const-post-dominator-walk.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/schedule-constuction-endless-loop1.ll b/polly/test/ScopInfo/schedule-constuction-endless-loop1.ll
index 413d1d8ec556..0225c53faa9a 100644
--- a/polly/test/ScopInfo/schedule-constuction-endless-loop1.ll
+++ b/polly/test/ScopInfo/schedule-constuction-endless-loop1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that we do not build a SCoP and do not crash.
 ;
diff --git a/polly/test/ScopInfo/schedule-constuction-endless-loop2.ll b/polly/test/ScopInfo/schedule-constuction-endless-loop2.ll
index be254477286f..bc34070dcf46 100644
--- a/polly/test/ScopInfo/schedule-constuction-endless-loop2.ll
+++ b/polly/test/ScopInfo/schedule-constuction-endless-loop2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Check that we do not build a SCoP and do not crash.
 ;
diff --git a/polly/test/ScopInfo/schedule-incorrectly-contructed-in-case-of-infinite-loop.ll b/polly/test/ScopInfo/schedule-incorrectly-contructed-in-case-of-infinite-loop.ll
index ff339e03fb5a..f8b3dfea844f 100644
--- a/polly/test/ScopInfo/schedule-incorrectly-contructed-in-case-of-infinite-loop.ll
+++ b/polly/test/ScopInfo/schedule-incorrectly-contructed-in-case-of-infinite-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-scops -disable-output < %s
+; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>' -disable-output < %s
 ;
 ; This test contains a infinite loop (bb13) and crashed the domain generation
 ; at some point. Just verify it does not anymore.
diff --git a/polly/test/ScopInfo/scop-affine-parameter-ordering.ll b/polly/test/ScopInfo/scop-affine-parameter-ordering.ll
index 24c028a6764a..5a8019eabe9d 100644
--- a/polly/test/ScopInfo/scop-affine-parameter-ordering.ll
+++ b/polly/test/ScopInfo/scop-affine-parameter-ordering.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-m:e-i64:64-i128:128-n8:16:32:64-S128"
 target triple = "aarch64--linux-android"
 
diff --git a/polly/test/ScopInfo/sign_wrapped_set.ll b/polly/test/ScopInfo/sign_wrapped_set.ll
index 23c9c8a3b84d..7b24f29563ea 100644
--- a/polly/test/ScopInfo/sign_wrapped_set.ll
+++ b/polly/test/ScopInfo/sign_wrapped_set.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-process-unprofitable -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-process-unprofitable '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:        Domain :=
 ; CHECK-NEXT:       [srcHeight] -> { Stmt_for_cond6_preheader_us[i0] : 0 <= i0 <= -3 + srcHeight };
diff --git a/polly/test/ScopInfo/simple_loop_1.ll b/polly/test/ScopInfo/simple_loop_1.ll
index 2c3481facc02..4872b8e59ba9 100644
--- a/polly/test/ScopInfo/simple_loop_1.ll
+++ b/polly/test/ScopInfo/simple_loop_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(int a[], int N) {
 ;   int i;
diff --git a/polly/test/ScopInfo/simple_loop_2.ll b/polly/test/ScopInfo/simple_loop_2.ll
index 2f580094a147..120b5e790077 100644
--- a/polly/test/ScopInfo/simple_loop_2.ll
+++ b/polly/test/ScopInfo/simple_loop_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(int a[], int N) {
 ;   int i;
diff --git a/polly/test/ScopInfo/simple_loop_unsigned.ll b/polly/test/ScopInfo/simple_loop_unsigned.ll
index 12903d9c1580..6c0e8798a6cd 100644
--- a/polly/test/ScopInfo/simple_loop_unsigned.ll
+++ b/polly/test/ScopInfo/simple_loop_unsigned.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(int a[], unsigned N) {
 ;   unsigned i;
diff --git a/polly/test/ScopInfo/simple_loop_unsigned_2.ll b/polly/test/ScopInfo/simple_loop_unsigned_2.ll
index 1379180a6dd9..4b19a8c52c6b 100644
--- a/polly/test/ScopInfo/simple_loop_unsigned_2.ll
+++ b/polly/test/ScopInfo/simple_loop_unsigned_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK:      Assumed Context:
 ; CHECK-NEXT: [N] -> {  :  }
diff --git a/polly/test/ScopInfo/simple_loop_unsigned_3.ll b/polly/test/ScopInfo/simple_loop_unsigned_3.ll
index 7783c4681e1f..fd974f219bec 100644
--- a/polly/test/ScopInfo/simple_loop_unsigned_3.ll
+++ b/polly/test/ScopInfo/simple_loop_unsigned_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK:      Assumed Context:
 ; CHECK-NEXT: [N] -> { : }
diff --git a/polly/test/ScopInfo/simple_nonaffine_loop_not.ll b/polly/test/ScopInfo/simple_nonaffine_loop_not.ll
index 42eff85d8c9b..d2aa22f8cca7 100644
--- a/polly/test/ScopInfo/simple_nonaffine_loop_not.ll
+++ b/polly/test/ScopInfo/simple_nonaffine_loop_not.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | not FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | not FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 @.str = private unnamed_addr constant [17 x i8] c"Random Value: %d\00", align 1
diff --git a/polly/test/ScopInfo/smax.ll b/polly/test/ScopInfo/smax.ll
index b938e4e412da..502d52baaaef 100644
--- a/polly/test/ScopInfo/smax.ll
+++ b/polly/test/ScopInfo/smax.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:64:128-a0:0:32-n32-S64"
 
 define void @foo(ptr noalias %data, ptr noalias %ptr, i32 %x_pos, i32 %w) {
diff --git a/polly/test/ScopInfo/statistics.ll b/polly/test/ScopInfo/statistics.ll
index 3797b7d71df9..c69852e21875 100644
--- a/polly/test/ScopInfo/statistics.ll
+++ b/polly/test/ScopInfo/statistics.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-scops -stats -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -stats -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; CHECK-DAG:  4 polly-scops      - Maximal number of loops in scops
diff --git a/polly/test/ScopInfo/stmt_split_exit_of_region_stmt.ll b/polly/test/ScopInfo/stmt_split_exit_of_region_stmt.ll
index d86d2418cf9b..1d4d3f19b571 100644
--- a/polly/test/ScopInfo/stmt_split_exit_of_region_stmt.ll
+++ b/polly/test/ScopInfo/stmt_split_exit_of_region_stmt.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:   Stmt_Region__TO__Stmt
diff --git a/polly/test/ScopInfo/stmt_split_no_after_split.ll b/polly/test/ScopInfo/stmt_split_no_after_split.ll
index f8339bd8ae94..e3e440584f25 100644
--- a/polly/test/ScopInfo/stmt_split_no_after_split.ll
+++ b/polly/test/ScopInfo/stmt_split_no_after_split.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT: 	Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_split_no_dependence.ll b/polly/test/ScopInfo/stmt_split_no_dependence.ll
index 7ad48f499792..0bf98c9b70be 100644
--- a/polly/test/ScopInfo/stmt_split_no_dependence.ll
+++ b/polly/test/ScopInfo/stmt_split_no_dependence.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ;      void func(int *A, int *B){
 ;        for (int i = 0; i < 1024; i+=1) {
diff --git a/polly/test/ScopInfo/stmt_split_on_store.ll b/polly/test/ScopInfo/stmt_split_on_store.ll
index 6af3dc8633dd..82b1f5bbc3cd 100644
--- a/polly/test/ScopInfo/stmt_split_on_store.ll
+++ b/polly/test/ScopInfo/stmt_split_on_store.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=store -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=store -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ;      void func(int *A, int *B){
 ;        for (int i = 0; i < 1024; i+=1) {
diff --git a/polly/test/ScopInfo/stmt_split_on_synthesizable.ll b/polly/test/ScopInfo/stmt_split_on_synthesizable.ll
index 92855cfd0124..323c83bc570e 100644
--- a/polly/test/ScopInfo/stmt_split_on_synthesizable.ll
+++ b/polly/test/ScopInfo/stmt_split_on_synthesizable.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:   Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_split_phi_in_beginning_bb.ll b/polly/test/ScopInfo/stmt_split_phi_in_beginning_bb.ll
index ee6afa4638d2..7f72e672a1a6 100644
--- a/polly/test/ScopInfo/stmt_split_phi_in_beginning_bb.ll
+++ b/polly/test/ScopInfo/stmt_split_phi_in_beginning_bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:  	Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_split_phi_in_stmt.ll b/polly/test/ScopInfo/stmt_split_phi_in_stmt.ll
index 0a5f41d637e7..9306cdc7615a 100644
--- a/polly/test/ScopInfo/stmt_split_phi_in_stmt.ll
+++ b/polly/test/ScopInfo/stmt_split_phi_in_stmt.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:  	Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_split_scalar_dependence.ll b/polly/test/ScopInfo/stmt_split_scalar_dependence.ll
index 5b02d1b5d08a..efd5cf14def0 100644
--- a/polly/test/ScopInfo/stmt_split_scalar_dependence.ll
+++ b/polly/test/ScopInfo/stmt_split_scalar_dependence.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:  	Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_split_within_loop.ll b/polly/test/ScopInfo/stmt_split_within_loop.ll
index 3ed9bbbeaccb..f24904df307c 100644
--- a/polly/test/ScopInfo/stmt_split_within_loop.ll
+++ b/polly/test/ScopInfo/stmt_split_within_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:  	Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_with_read_but_without_sideffect.ll b/polly/test/ScopInfo/stmt_with_read_but_without_sideffect.ll
index 73fc543a66e8..41f58844569f 100644
--- a/polly/test/ScopInfo/stmt_with_read_but_without_sideffect.ll
+++ b/polly/test/ScopInfo/stmt_with_read_but_without_sideffect.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; The statement Stmt_for_if_else_1 should be removed because it has no
 ; sideeffects.  But it has a use of MemRef_tmp21 that must also be
diff --git a/polly/test/ScopInfo/switch-1.ll b/polly/test/ScopInfo/switch-1.ll
index 0ea40a7ed251..6bc630834e93 100644
--- a/polly/test/ScopInfo/switch-1.ll
+++ b/polly/test/ScopInfo/switch-1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/switch-2.ll b/polly/test/ScopInfo/switch-2.ll
index 7956058c9de6..a64d133baae7 100644
--- a/polly/test/ScopInfo/switch-2.ll
+++ b/polly/test/ScopInfo/switch-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/switch-3.ll b/polly/test/ScopInfo/switch-3.ll
index aa7ada4edbb8..3aa2d7811c77 100644
--- a/polly/test/ScopInfo/switch-3.ll
+++ b/polly/test/ScopInfo/switch-3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/switch-4.ll b/polly/test/ScopInfo/switch-4.ll
index 6aeb7197e382..567c3de030ea 100644
--- a/polly/test/ScopInfo/switch-4.ll
+++ b/polly/test/ScopInfo/switch-4.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/switch-5.ll b/polly/test/ScopInfo/switch-5.ll
index 24cc92a0933d..b6a42d9da749 100644
--- a/polly/test/ScopInfo/switch-5.ll
+++ b/polly/test/ScopInfo/switch-5.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/switch-6.ll b/polly/test/ScopInfo/switch-6.ll
index efb3df504d23..24538328c581 100644
--- a/polly/test/ScopInfo/switch-6.ll
+++ b/polly/test/ScopInfo/switch-6.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++) {
diff --git a/polly/test/ScopInfo/switch-7.ll b/polly/test/ScopInfo/switch-7.ll
index 2f0d034e84fe..99c1bed81874 100644
--- a/polly/test/ScopInfo/switch-7.ll
+++ b/polly/test/ScopInfo/switch-7.ll
@@ -1,6 +1,5 @@
-
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int c, int N) {
 ;      switch (c) {
diff --git a/polly/test/ScopInfo/tempscop-printing.ll b/polly/test/ScopInfo/tempscop-printing.ll
index 80c675d4c3d3..e99a6f2582ee 100644
--- a/polly/test/ScopInfo/tempscop-printing.ll
+++ b/polly/test/ScopInfo/tempscop-printing.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -basic-aa -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/test-wrapping-in-condition.ll b/polly/test/ScopInfo/test-wrapping-in-condition.ll
index 3ff978f7265e..7c1301748c39 100644
--- a/polly/test/ScopInfo/test-wrapping-in-condition.ll
+++ b/polly/test/ScopInfo/test-wrapping-in-condition.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:    Invalid Context:
 ; CHECK:        [N] -> {  : N >= 129 }
diff --git a/polly/test/ScopInfo/truncate-1.ll b/polly/test/ScopInfo/truncate-1.ll
index 5c5fac150b4b..b21755c67ac4 100644
--- a/polly/test/ScopInfo/truncate-1.ll
+++ b/polly/test/ScopInfo/truncate-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(char *A, short N) {
 ;      for (char i = 0; i < (char)N; i++)
diff --git a/polly/test/ScopInfo/truncate-2.ll b/polly/test/ScopInfo/truncate-2.ll
index e6c5f2cb32d0..0d4abb343993 100644
--- a/polly/test/ScopInfo/truncate-2.ll
+++ b/polly/test/ScopInfo/truncate-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(char *A, short N) {
 ;      for (short i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/truncate-3.ll b/polly/test/ScopInfo/truncate-3.ll
index dd0fe489e990..f9cdd0274f22 100644
--- a/polly/test/ScopInfo/truncate-3.ll
+++ b/polly/test/ScopInfo/truncate-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-scops -pass-remarks-analysis="polly-scops" \
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -pass-remarks-analysis="polly-scops" \
 ; RUN:                -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK: Signed-unsigned restriction: [p] -> {  : p <= -129 or p >= 128 }
diff --git a/polly/test/ScopInfo/two-loops-one-infinite.ll b/polly/test/ScopInfo/two-loops-one-infinite.ll
index 71f72383b048..02ad18e3d567 100644
--- a/polly/test/ScopInfo/two-loops-one-infinite.ll
+++ b/polly/test/ScopInfo/two-loops-one-infinite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Verify we do not create a SCoP in the presence of infinite loops.
 ;
diff --git a/polly/test/ScopInfo/two-loops-right-after-each-other.ll b/polly/test/ScopInfo/two-loops-right-after-each-other.ll
index dd457c31afdd..36ab13a68c1e 100644
--- a/polly/test/ScopInfo/two-loops-right-after-each-other.ll
+++ b/polly/test/ScopInfo/two-loops-right-after-each-other.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK:      Statements {
 ; CHECK-NEXT:     Stmt_loop_1
diff --git a/polly/test/ScopInfo/undef_in_cond.ll b/polly/test/ScopInfo/undef_in_cond.ll
index 5282a853c17a..4bdcc1697068 100644
--- a/polly/test/ScopInfo/undef_in_cond.ll
+++ b/polly/test/ScopInfo/undef_in_cond.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define fastcc void @fix_operands() nounwind {
diff --git a/polly/test/ScopInfo/unnamed_nonaffine.ll b/polly/test/ScopInfo/unnamed_nonaffine.ll
index bf32cc7806f4..d9415eabab94 100644
--- a/polly/test/ScopInfo/unnamed_nonaffine.ll
+++ b/polly/test/ScopInfo/unnamed_nonaffine.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-use-llvm-names=true  -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-use-llvm-names=false -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=UNNAMED
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-use-llvm-names=true  '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-use-llvm-names=false '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=UNNAMED
 ;
 ;    void f(int *A, int b) {
 ;      int x;
diff --git a/polly/test/ScopInfo/unnamed_stmts.ll b/polly/test/ScopInfo/unnamed_stmts.ll
index 686c0f87d9cf..0bd53d8c8425 100644
--- a/polly/test/ScopInfo/unnamed_stmts.ll
+++ b/polly/test/ScopInfo/unnamed_stmts.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; This test case verifies that we generate numbered statement names in case
 ; no LLVM-IR names are used in the test case. We also verify, that we
diff --git a/polly/test/ScopInfo/unpredictable_nonscop_loop.ll b/polly/test/ScopInfo/unpredictable_nonscop_loop.ll
index 0656b77e3409..c0e768216eb0 100644
--- a/polly/test/ScopInfo/unpredictable_nonscop_loop.ll
+++ b/polly/test/ScopInfo/unpredictable_nonscop_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 ; Derived from test-suite/MultiSource/Applications/sgefa/blas.c
 ;
 ; The exit value of %i.0320 in land.rhs is not computable.
diff --git a/polly/test/ScopInfo/unprofitable_scalar-accs.ll b/polly/test/ScopInfo/unprofitable_scalar-accs.ll
index 9703587091a7..e7c8a57093b8 100644
--- a/polly/test/ScopInfo/unprofitable_scalar-accs.ll
+++ b/polly/test/ScopInfo/unprofitable_scalar-accs.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=false -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=true  -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=HEURISTIC
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=false '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=true  '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=HEURISTIC
 
 ; Check the effect of -polly-unprofitable-scalar-accs
 
diff --git a/polly/test/ScopInfo/unsigned-condition.ll b/polly/test/ScopInfo/unsigned-condition.ll
index 35673d1b6a36..1dca9bab41ec 100644
--- a/polly/test/ScopInfo/unsigned-condition.ll
+++ b/polly/test/ScopInfo/unsigned-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(int a[], int N, unsigned P) {
 ;   int i;
diff --git a/polly/test/ScopInfo/unsigned-division-1.ll b/polly/test/ScopInfo/unsigned-division-1.ll
index 8c65062bd941..da080b3a306b 100644
--- a/polly/test/ScopInfo/unsigned-division-1.ll
+++ b/polly/test/ScopInfo/unsigned-division-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, unsigned N) {
 ;      for (unsigned i = 0; i < N / 2; i++)
diff --git a/polly/test/ScopInfo/unsigned-division-2.ll b/polly/test/ScopInfo/unsigned-division-2.ll
index bf4ebce9099a..2fe4207d1bd1 100644
--- a/polly/test/ScopInfo/unsigned-division-2.ll
+++ b/polly/test/ScopInfo/unsigned-division-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, unsigned N) {
 ;      for (unsigned i = 0; i < N / 2 + 3; i++)
diff --git a/polly/test/ScopInfo/unsigned-division-3.ll b/polly/test/ScopInfo/unsigned-division-3.ll
index 47ba1f2ef09d..aefb590b28df 100644
--- a/polly/test/ScopInfo/unsigned-division-3.ll
+++ b/polly/test/ScopInfo/unsigned-division-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, unsigned char N) {
 ;      for (unsigned i = 0; i <= N / -128; i++)
diff --git a/polly/test/ScopInfo/unsigned-division-4.ll b/polly/test/ScopInfo/unsigned-division-4.ll
index edcd8a18a854..9fe10d7440ef 100644
--- a/polly/test/ScopInfo/unsigned-division-4.ll
+++ b/polly/test/ScopInfo/unsigned-division-4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, unsigned char N) {
 ;      for (unsigned i = 0; i < (N / -128) + 3; i++)
diff --git a/polly/test/ScopInfo/unsigned-division-5.ll b/polly/test/ScopInfo/unsigned-division-5.ll
index f9a3d39288a9..fb90345f477e 100644
--- a/polly/test/ScopInfo/unsigned-division-5.ll
+++ b/polly/test/ScopInfo/unsigned-division-5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, unsigned N) {
 ;      for (unsigned i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/unsigned_wrap_uge.ll b/polly/test/ScopInfo/unsigned_wrap_uge.ll
index 89c50ee3764b..3d54cad70285 100644
--- a/polly/test/ScopInfo/unsigned_wrap_uge.ll
+++ b/polly/test/ScopInfo/unsigned_wrap_uge.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Unsigned wrap-around check.
 ;
diff --git a/polly/test/ScopInfo/unsigned_wrap_ugt.ll b/polly/test/ScopInfo/unsigned_wrap_ugt.ll
index 3249123c9918..8c98f13cfb72 100644
--- a/polly/test/ScopInfo/unsigned_wrap_ugt.ll
+++ b/polly/test/ScopInfo/unsigned_wrap_ugt.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Unsigned wrap-around check.
 ;
diff --git a/polly/test/ScopInfo/unsigned_wrap_ule.ll b/polly/test/ScopInfo/unsigned_wrap_ule.ll
index 3c6ea18b439c..e0b0339475fc 100644
--- a/polly/test/ScopInfo/unsigned_wrap_ule.ll
+++ b/polly/test/ScopInfo/unsigned_wrap_ule.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Unsigned wrap-around check.
 ;
diff --git a/polly/test/ScopInfo/unsigned_wrap_ult.ll b/polly/test/ScopInfo/unsigned_wrap_ult.ll
index 5d859f85d52b..cb15bc04669e 100644
--- a/polly/test/ScopInfo/unsigned_wrap_ult.ll
+++ b/polly/test/ScopInfo/unsigned_wrap_ult.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; Unsigned wrap-around check.
 ;
diff --git a/polly/test/ScopInfo/user_context.ll b/polly/test/ScopInfo/user_context.ll
index 46232cd59c03..d67244e1ad95 100644
--- a/polly/test/ScopInfo/user_context.ll
+++ b/polly/test/ScopInfo/user_context.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly                                      -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-context='[N] -> {: N = 1024}' -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=CTX
-; RUN: opt %loadPolly -polly-context='[N,M] -> {: 1 = 0}'  -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-context='[] -> {: 1 = 0}'     -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly                                      '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-context='[N] -> {: N = 1024}' '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CTX
+; RUN: opt %loadPolly -polly-context='[N,M] -> {: 1 = 0}'  '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-context='[] -> {: 1 = 0}'     '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 
 ; void f(int a[], int N) {
 ;   int i;
diff --git a/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed-conditional.ll b/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed-conditional.ll
index 4bd02c96a3d2..829d1ef10664 100644
--- a/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed-conditional.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed-conditional.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; REMARK: remark: <unknown>:0:0: Use user assumption: [n, b] -> {  : n <= 100 or (b = 0 and n >= 101) }
 ;
diff --git a/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed.ll b/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed.ll
index 262bd1349a69..8518a0ece23c 100644
--- a/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Context:
 ; CHECK-NEXT: [n] -> {  : -9223372036854775808 <= n <= 100 }
diff --git a/polly/test/ScopInfo/user_provided_assumptions-in-bb-unsigned.ll b/polly/test/ScopInfo/user_provided_assumptions-in-bb-unsigned.ll
index 4a10fcff929a..678be1c06d0a 100644
--- a/polly/test/ScopInfo/user_provided_assumptions-in-bb-unsigned.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions-in-bb-unsigned.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; REMARK:      remark: <unknown>:0:0: SCoP begins here.
 ; REMARK-NEXT: remark: <unknown>:0:0: Use user assumption: [n] -> {  : n <= 100 }
diff --git a/polly/test/ScopInfo/user_provided_assumptions.ll b/polly/test/ScopInfo/user_provided_assumptions.ll
index 6640e4a65e36..e4556eb3a386 100644
--- a/polly/test/ScopInfo/user_provided_assumptions.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
 ;
 ; CHECK:      remark: <unknown>:0:0: SCoP begins here.
 ; CHECK-NEXT: remark: <unknown>:0:0: Use user assumption: [M, N] -> {  : N <= 2147483647 - M }
diff --git a/polly/test/ScopInfo/user_provided_assumptions_2.ll b/polly/test/ScopInfo/user_provided_assumptions_2.ll
index 994cd6f15103..98057740eab3 100644
--- a/polly/test/ScopInfo/user_provided_assumptions_2.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions_2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
 ;
 ; CHECK:      remark: <unknown>:0:0: SCoP begins here.
 ; CHECK-NEXT: remark: <unknown>:0:0: Use user assumption: { : }
diff --git a/polly/test/ScopInfo/user_provided_assumptions_3.ll b/polly/test/ScopInfo/user_provided_assumptions_3.ll
index 2fcde8bd1826..de3fbba46e0a 100644
--- a/polly/test/ScopInfo/user_provided_assumptions_3.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions_3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
 ;
 ; CHECK:      remark: <unknown>:0:0: SCoP begins here.
 ; CHECK-NEXT: remark: <unknown>:0:0: Use user assumption: [N] -> { : N >= 2 }
diff --git a/polly/test/ScopInfo/user_provided_non_dominating_assumptions.ll b/polly/test/ScopInfo/user_provided_non_dominating_assumptions.ll
index 1eb3c15810e4..4f3be408c9ef 100644
--- a/polly/test/ScopInfo/user_provided_non_dominating_assumptions.ll
+++ b/polly/test/ScopInfo/user_provided_non_dominating_assumptions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
 ; RUN:    -polly-precise-inbounds -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:      remark: <unknown>:0:0: SCoP begins here.
@@ -18,7 +18,7 @@
 ;
 
 
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
 ; RUN:    -polly-precise-inbounds -disable-output < %s 2>&1 -pass-remarks-output=%t.yaml
 ; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
 ; YAML: --- !Analysis
diff --git a/polly/test/ScopInfo/variant_base_pointer.ll b/polly/test/ScopInfo/variant_base_pointer.ll
index 321657c87e79..3a6ea88d3473 100644
--- a/polly/test/ScopInfo/variant_base_pointer.ll
+++ b/polly/test/ScopInfo/variant_base_pointer.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true -passes=polly-codegen -disable-output < %s
 ;
 ; %tmp is added to the list of required hoists by -polly-scops and just
 ; assumed to be hoisted. Only -polly-scops recognizes it to be unhoistable
diff --git a/polly/test/ScopInfo/variant_load_empty_domain.ll b/polly/test/ScopInfo/variant_load_empty_domain.ll
index 0e685c3c7e73..4b91778a225b 100644
--- a/polly/test/ScopInfo/variant_load_empty_domain.ll
+++ b/polly/test/ScopInfo/variant_load_empty_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:      Invariant Accesses: {
 ; CHECK-NEXT: }
diff --git a/polly/test/ScopInfo/wraping_signed_expr_0.ll b/polly/test/ScopInfo/wraping_signed_expr_0.ll
index 7ad0f64028b6..bbb49bffe925 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_0.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_0.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, char N, char p) {
 ;      for (char i = 0; i < N; i++) {
diff --git a/polly/test/ScopInfo/wraping_signed_expr_1.ll b/polly/test/ScopInfo/wraping_signed_expr_1.ll
index 0a62b9cf542c..e43a691312a0 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_1.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(long *A, long N, long p) {
 ;      for (long i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/wraping_signed_expr_2.ll b/polly/test/ScopInfo/wraping_signed_expr_2.ll
index f3b4665f7f37..eef357acc582 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_2.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, int N, int p) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/wraping_signed_expr_3.ll b/polly/test/ScopInfo/wraping_signed_expr_3.ll
index 7a5cbba9436b..a0500eb48941 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_3.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(int *A, int N, int p) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/wraping_signed_expr_4.ll b/polly/test/ScopInfo/wraping_signed_expr_4.ll
index ec65f70a092f..d21f321e4ac9 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_4.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(char *A, char N, char p) {
 ;      for (char i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/wraping_signed_expr_5.ll b/polly/test/ScopInfo/wraping_signed_expr_5.ll
index 5f3b09ba33c1..395342d2f55a 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_5.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; We should not generate runtime check for ((int)r1 + (int)r2) as it is known not
 ; to overflow. However (p + q) can, thus checks are needed.
diff --git a/polly/test/ScopInfo/wraping_signed_expr_6.ll b/polly/test/ScopInfo/wraping_signed_expr_6.ll
index 23258bb513bf..4147f7fa20cf 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_6.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_6.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:    Invalid Context:
 ; CHECK:        [N] -> {  : N >= 129 }
diff --git a/polly/test/ScopInfo/wraping_signed_expr_7.ll b/polly/test/ScopInfo/wraping_signed_expr_7.ll
index 0663d4e0bc10..f41e89c07db0 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_7.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_7.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:    Invalid Context:
 ; CHECK:        [N] -> {  : N >= 129 }
diff --git a/polly/test/ScopInfo/wraping_signed_expr_slow_1.ll b/polly/test/ScopInfo/wraping_signed_expr_slow_1.ll
index ec36d2c5fcde..ddaeed06874a 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_slow_1.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_slow_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; This checks that the no-wraps checks will be computed fast as some example
 ; already showed huge slowdowns even though the inbounds and nsw flags were
diff --git a/polly/test/ScopInfo/wraping_signed_expr_slow_2.ll b/polly/test/ScopInfo/wraping_signed_expr_slow_2.ll
index 6db33ab166d5..798a5d0855b3 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_slow_2.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_slow_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; This checks that the no-wraps checks will be computed fast as some example
 ; already showed huge slowdowns even though the inbounds and nsw flags were
diff --git a/polly/test/ScopInfo/zero_ext_of_truncate.ll b/polly/test/ScopInfo/zero_ext_of_truncate.ll
index fc55df5e053c..bf5b6354a6d6 100644
--- a/polly/test/ScopInfo/zero_ext_of_truncate.ll
+++ b/polly/test/ScopInfo/zero_ext_of_truncate.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(unsigned *restrict I, unsigned *restrict A, unsigned N, unsigned M) {
 ;      for (unsigned i = 0; i < N; i++) {
diff --git a/polly/test/ScopInfo/zero_ext_of_truncate_2.ll b/polly/test/ScopInfo/zero_ext_of_truncate_2.ll
index 13e9c03ecd2d..595b21c71869 100644
--- a/polly/test/ScopInfo/zero_ext_of_truncate_2.ll
+++ b/polly/test/ScopInfo/zero_ext_of_truncate_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void f(unsigned long *restrict I, unsigned *restrict A, unsigned N) {
 ;      for (unsigned i = 0; i < N; i++) {
diff --git a/polly/test/ScopInfo/zero_ext_space_mismatch.ll b/polly/test/ScopInfo/zero_ext_space_mismatch.ll
index 835a8664b75e..0a329fdef8c2 100644
--- a/polly/test/ScopInfo/zero_ext_space_mismatch.ll
+++ b/polly/test/ScopInfo/zero_ext_space_mismatch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:         Assumed Context:
 ; CHECK-NEXT:    [dim] -> {  : dim > 0 }
diff --git a/polly/test/ScopInliner/invariant-load-func.ll b/polly/test/ScopInliner/invariant-load-func.ll
index 38e4a15aab94..8da50f90beba 100644
--- a/polly/test/ScopInliner/invariant-load-func.ll
+++ b/polly/test/ScopInliner/invariant-load-func.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly -polly-detect-full-functions -polly-scop-inliner \
-; RUN: -polly-invariant-load-hoisting -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: -polly-invariant-load-hoisting '-passes=print<polly-function-scops>' -disable-output < %s | FileCheck %s
 
 ; Check that we inline a function that requires invariant load hoisting
 ; correctly.
diff --git a/polly/test/Simplify/coalesce_3partials.ll b/polly/test/Simplify/coalesce_3partials.ll
index 0c1556ff263a..937f655c344c 100644
--- a/polly/test/Simplify/coalesce_3partials.ll
+++ b/polly/test/Simplify/coalesce_3partials.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Combine 3 partial accesses into one.
 ;
diff --git a/polly/test/Simplify/coalesce_disjointelements.ll b/polly/test/Simplify/coalesce_disjointelements.ll
index 2f4cf4e3f920..6080ee4dde81 100644
--- a/polly/test/Simplify/coalesce_disjointelements.ll
+++ b/polly/test/Simplify/coalesce_disjointelements.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Combine four partial stores into two.
 ; The stores write to the same array, but never the same element.
diff --git a/polly/test/Simplify/coalesce_overlapping.ll b/polly/test/Simplify/coalesce_overlapping.ll
index 78ed21e9855b..3c52d44e8003 100644
--- a/polly/test/Simplify/coalesce_overlapping.ll
+++ b/polly/test/Simplify/coalesce_overlapping.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Combine two partial stores (with overlapping domains) into one.
 ;
diff --git a/polly/test/Simplify/coalesce_partial.ll b/polly/test/Simplify/coalesce_partial.ll
index c42aaa113035..cec58a9121b2 100644
--- a/polly/test/Simplify/coalesce_partial.ll
+++ b/polly/test/Simplify/coalesce_partial.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Combine two partial stores (with disjoint domains) into one.
 ;
diff --git a/polly/test/Simplify/dead_access_load.ll b/polly/test/Simplify/dead_access_load.ll
index 1804613c0a79..5e0a9b574516 100644
--- a/polly/test/Simplify/dead_access_load.ll
+++ b/polly/test/Simplify/dead_access_load.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Remove a dead load-instruction
 ; (an load whose result is not used anywhere)
diff --git a/polly/test/Simplify/dead_access_phi.ll b/polly/test/Simplify/dead_access_phi.ll
index d263b89aff58..6044f7f50be7 100644
--- a/polly/test/Simplify/dead_access_phi.ll
+++ b/polly/test/Simplify/dead_access_phi.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Remove a dead PHI write/read pair
 ; (accesses that are effectively not used)
diff --git a/polly/test/Simplify/dead_access_value.ll b/polly/test/Simplify/dead_access_value.ll
index 6e3c211577f6..a3b9d5ebe76a 100644
--- a/polly/test/Simplify/dead_access_value.ll
+++ b/polly/test/Simplify/dead_access_value.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Remove a dead value write/read pair
 ; (accesses that are effectively not used)
diff --git a/polly/test/Simplify/dead_instruction.ll b/polly/test/Simplify/dead_instruction.ll
index 4e693b0ccb44..2bf7f8571a46 100644
--- a/polly/test/Simplify/dead_instruction.ll
+++ b/polly/test/Simplify/dead_instruction.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Remove a dead instruction
 ; (an instruction whose result is not used anywhere)
diff --git a/polly/test/Simplify/emptyaccessdomain.ll b/polly/test/Simplify/emptyaccessdomain.ll
index 54ac14ab398c..bf6d4d9dc8bd 100644
--- a/polly/test/Simplify/emptyaccessdomain.ll
+++ b/polly/test/Simplify/emptyaccessdomain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; for (int j = 0; j < n; j += 1) {
 ;   A[0] = 42.0;
diff --git a/polly/test/Simplify/exit_phi_accesses-2.ll b/polly/test/Simplify/exit_phi_accesses-2.ll
index 01748aa59bd3..2116d8008aec 100644
--- a/polly/test/Simplify/exit_phi_accesses-2.ll
+++ b/polly/test/Simplify/exit_phi_accesses-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-scops -polly-print-simplify -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>,scop(print<polly-simplify>)' -disable-output < %s | FileCheck %s
 ;
 ; The use of %sum.next by %phi counts as an escaping use.
 ; Don't remove the scalar write of %sum.next.
diff --git a/polly/test/Simplify/func-b320a7.ll b/polly/test/Simplify/func-b320a7.ll
index c8a823a468d7..c5afc37eb7de 100644
--- a/polly/test/Simplify/func-b320a7.ll
+++ b/polly/test/Simplify/func-b320a7.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-simplify -polly-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=print<polly-simplify>,polly-optree' -disable-output < %s | FileCheck %s -match-full-lines
 
 ; llvm.org/PR47098
 ; Use-after-free by reference to Stmt remaining in InstStmtMap after removing it has been removed by Scop::simplifyScop.
diff --git a/polly/test/Simplify/gemm.ll b/polly/test/Simplify/gemm.ll
index 23f8de5573cd..4074078742fc 100644
--- a/polly/test/Simplify/gemm.ll
+++ b/polly/test/Simplify/gemm.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s
 ;
 ;    void gemm(float A[][1024], float B[][1024], float C[][1024]) {
 ;      for (long i = 0; i < 1024; i++)
diff --git a/polly/test/Simplify/nocoalesce_differentvalues.ll b/polly/test/Simplify/nocoalesce_differentvalues.ll
index 68991d2eecf5..d08c80ee0c06 100644
--- a/polly/test/Simplify/nocoalesce_differentvalues.ll
+++ b/polly/test/Simplify/nocoalesce_differentvalues.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Do not combine stores that write different values.
 ;
diff --git a/polly/test/Simplify/nocoalesce_elementmismatch.ll b/polly/test/Simplify/nocoalesce_elementmismatch.ll
index 2bab360e6858..af12e611fdbc 100644
--- a/polly/test/Simplify/nocoalesce_elementmismatch.ll
+++ b/polly/test/Simplify/nocoalesce_elementmismatch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Do not combine stores that do not write to different elements in the
 ; same instance.
diff --git a/polly/test/Simplify/nocoalesce_readbetween.ll b/polly/test/Simplify/nocoalesce_readbetween.ll
index ada79dc18b87..1a71d2da4c1f 100644
--- a/polly/test/Simplify/nocoalesce_readbetween.ll
+++ b/polly/test/Simplify/nocoalesce_readbetween.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Do not combine stores if there is a read between them.
 ; Note: The read between is unused, so will be removed by markAndSweep.
diff --git a/polly/test/Simplify/nocoalesce_writebetween.ll b/polly/test/Simplify/nocoalesce_writebetween.ll
index 48e785ec2c26..bc2c47a4c9ec 100644
--- a/polly/test/Simplify/nocoalesce_writebetween.ll
+++ b/polly/test/Simplify/nocoalesce_writebetween.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Do not combine stores if there is a write between them.
 ;
diff --git a/polly/test/Simplify/notdead_region_exitphi.ll b/polly/test/Simplify/notdead_region_exitphi.ll
index bd29fd578b97..a796f2a419ad 100644
--- a/polly/test/Simplify/notdead_region_exitphi.ll
+++ b/polly/test/Simplify/notdead_region_exitphi.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Do not remove dependencies of a phi node in a region's exit block.
 ;
diff --git a/polly/test/Simplify/notdead_region_innerphi.ll b/polly/test/Simplify/notdead_region_innerphi.ll
index a176a28af233..c76485cb5019 100644
--- a/polly/test/Simplify/notdead_region_innerphi.ll
+++ b/polly/test/Simplify/notdead_region_innerphi.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Do not remove dependencies of a phi node within a region statement (%phi).
 ;
diff --git a/polly/test/Simplify/notredundant_region_loop.ll b/polly/test/Simplify/notredundant_region_loop.ll
index 0ea9be7e9d2d..0bf0dd531524 100644
--- a/polly/test/Simplify/notredundant_region_loop.ll
+++ b/polly/test/Simplify/notredundant_region_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-allow-nonaffine-loops -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -polly-allow-nonaffine-loops -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Do not remove the store in region_entry. It can be executed multiple times
 ; due to being part of a non-affine loop.
diff --git a/polly/test/Simplify/notredundant_region_middle.ll b/polly/test/Simplify/notredundant_region_middle.ll
index 84598746e0bb..392dd48ae985 100644
--- a/polly/test/Simplify/notredundant_region_middle.ll
+++ b/polly/test/Simplify/notredundant_region_middle.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Do not remove redundant stores in the middle of region statements.
 ; The store in region_true could be removed, but in practice we do try to
diff --git a/polly/test/Simplify/notredundant_synthesizable_unknownit.ll b/polly/test/Simplify/notredundant_synthesizable_unknownit.ll
index 2affdbb2f1de..d522d5dd68ae 100644
--- a/polly/test/Simplify/notredundant_synthesizable_unknownit.ll
+++ b/polly/test/Simplify/notredundant_synthesizable_unknownit.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Do not remove the scalar value write of %i.trunc in inner.for.
 ; It is used by body.
diff --git a/polly/test/Simplify/out-of-scop-use-in-region-entry-phi-node.ll b/polly/test/Simplify/out-of-scop-use-in-region-entry-phi-node.ll
index 511f35a9388e..fe57a0ef6c9d 100644
--- a/polly/test/Simplify/out-of-scop-use-in-region-entry-phi-node.ll
+++ b/polly/test/Simplify/out-of-scop-use-in-region-entry-phi-node.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-print-simplify -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>,scop(print<polly-simplify>)' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; %tmp5 must keep the Value WRITE MemoryAccess, because as an incoming value of
 ; %tmp4, it is an "external use".
diff --git a/polly/test/Simplify/overwritten.ll b/polly/test/Simplify/overwritten.ll
index a32d6a8daeb0..b693e9c0db27 100644
--- a/polly/test/Simplify/overwritten.ll
+++ b/polly/test/Simplify/overwritten.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
 ;
 ; Remove a store that is overwritten by another store in the same statement.
 ;
diff --git a/polly/test/Simplify/overwritten_3phi.ll b/polly/test/Simplify/overwritten_3phi.ll
index 24758b9b7cf9..84cf67fa7cdc 100644
--- a/polly/test/Simplify/overwritten_3phi.ll
+++ b/polly/test/Simplify/overwritten_3phi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Remove identical writes
 ; (two stores in the same statement that write the same value to the same
diff --git a/polly/test/Simplify/overwritten_3store.ll b/polly/test/Simplify/overwritten_3store.ll
index 63eb5b54f931..72e9917b36a9 100644
--- a/polly/test/Simplify/overwritten_3store.ll
+++ b/polly/test/Simplify/overwritten_3store.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
-; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
 ;
 ; Remove a store that is overwritten by another store in the same statement.
 ; Check that even multiple stores are removed.
diff --git a/polly/test/Simplify/overwritten_implicit_and_explicit.ll b/polly/test/Simplify/overwritten_implicit_and_explicit.ll
index 56c63b48f761..ba7c6d0f20b1 100644
--- a/polly/test/Simplify/overwritten_implicit_and_explicit.ll
+++ b/polly/test/Simplify/overwritten_implicit_and_explicit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Remove a store that is overwritten by another store in the same statement.
 ; Check that this works even if one of the writes is a scalar MemoryKind.
diff --git a/polly/test/Simplify/overwritten_loadbetween.ll b/polly/test/Simplify/overwritten_loadbetween.ll
index b31f45d5db62..f271b4559dc8 100644
--- a/polly/test/Simplify/overwritten_loadbetween.ll
+++ b/polly/test/Simplify/overwritten_loadbetween.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
 ;
 ; Do not remove overwrites when the value is read before.
 ;
diff --git a/polly/test/Simplify/overwritten_scalar.ll b/polly/test/Simplify/overwritten_scalar.ll
index d55ea7712c36..41c5c6fa2470 100644
--- a/polly/test/Simplify/overwritten_scalar.ll
+++ b/polly/test/Simplify/overwritten_scalar.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Remove identical writes
 ; (two stores in the same statement that write the same value to the same
diff --git a/polly/test/Simplify/pass_existence.ll b/polly/test/Simplify/pass_existence.ll
index fc5287ed2ee2..a8fc184b1616 100644
--- a/polly/test/Simplify/pass_existence.ll
+++ b/polly/test/Simplify/pass_existence.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s
-; RUN: opt %loadNPMPolly -disable-output "-passes=scop(print<polly-simplify>)" < %s -aa-pipeline=basic-aa < %s | FileCheck %s
+; RUN: opt %loadPolly -disable-output "-passes=scop(print<polly-simplify>)" < %s -aa-pipeline=basic-aa < %s | FileCheck %s
 ;
 ; Simple test for the existence of the Simplify pass.
 ;
diff --git a/polly/test/Simplify/phi_in_regionstmt.ll b/polly/test/Simplify/phi_in_regionstmt.ll
index 32bb75427589..4c6a8744e200 100644
--- a/polly/test/Simplify/phi_in_regionstmt.ll
+++ b/polly/test/Simplify/phi_in_regionstmt.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; The PHINode %cond91.sink.sink.us.sink.6 is in the middle of a region
 ; statement.
diff --git a/polly/test/Simplify/pr33323.ll b/polly/test/Simplify/pr33323.ll
index 751f0bff5961..de2e00e8e2e9 100644
--- a/polly/test/Simplify/pr33323.ll
+++ b/polly/test/Simplify/pr33323.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s
 ;
 ; llvm.org/PR33323
 ;
diff --git a/polly/test/Simplify/redundant.ll b/polly/test/Simplify/redundant.ll
index e85352bc889f..720f2e3d0ef2 100644
--- a/polly/test/Simplify/redundant.ll
+++ b/polly/test/Simplify/redundant.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Remove redundant store (a store that writes the same value already
 ; at the destination)
diff --git a/polly/test/Simplify/redundant_differentindex.ll b/polly/test/Simplify/redundant_differentindex.ll
index 23531c24344f..c79364608e58 100644
--- a/polly/test/Simplify/redundant_differentindex.ll
+++ b/polly/test/Simplify/redundant_differentindex.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; A store that has a different index than the load it is storing is
 ; not redundant.
diff --git a/polly/test/Simplify/redundant_region.ll b/polly/test/Simplify/redundant_region.ll
index dbcb420ac2f3..d5c9586283de 100644
--- a/polly/test/Simplify/redundant_region.ll
+++ b/polly/test/Simplify/redundant_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Remove redundant store (a store that writes the same value already
 ; at the destination) in a region.
diff --git a/polly/test/Simplify/redundant_region_scalar.ll b/polly/test/Simplify/redundant_region_scalar.ll
index 95a581ad6f57..ab07126fe268 100644
--- a/polly/test/Simplify/redundant_region_scalar.ll
+++ b/polly/test/Simplify/redundant_region_scalar.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Remove redundant store (a store that writes the same value already
 ; at the destination) in a region.
diff --git a/polly/test/Simplify/redundant_scalarwrite.ll b/polly/test/Simplify/redundant_scalarwrite.ll
index e2f7bbedc023..c09be5f61837 100644
--- a/polly/test/Simplify/redundant_scalarwrite.ll
+++ b/polly/test/Simplify/redundant_scalarwrite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Remove redundant scalar stores.
 ;
diff --git a/polly/test/Simplify/redundant_storebetween.ll b/polly/test/Simplify/redundant_storebetween.ll
index f624b6e5b995..f87c1126cd26 100644
--- a/polly/test/Simplify/redundant_storebetween.ll
+++ b/polly/test/Simplify/redundant_storebetween.ll
@@ -1,5 +1,4 @@
-; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Don't remove store where there is another store to the same target
 ; in-between them.
diff --git a/polly/test/Simplify/scalability1.ll b/polly/test/Simplify/scalability1.ll
index 0ef99ce1ad8e..a91574e2b274 100644
--- a/polly/test/Simplify/scalability1.ll
+++ b/polly/test/Simplify/scalability1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-ignore-inbounds -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-ignore-inbounds '-passes=print<polly-simplify>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Test scalability.
 ;
diff --git a/polly/test/Simplify/scalability2.ll b/polly/test/Simplify/scalability2.ll
index bac0810b0afa..4e4874df219e 100644
--- a/polly/test/Simplify/scalability2.ll
+++ b/polly/test/Simplify/scalability2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-ignore-inbounds -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-ignore-inbounds '-passes=print<polly-simplify>' -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Test scalability.
 ;
diff --git a/polly/test/Simplify/sweep_mapped_phi.ll b/polly/test/Simplify/sweep_mapped_phi.ll
index add1681cdf36..3b9e61c72b4d 100644
--- a/polly/test/Simplify/sweep_mapped_phi.ll
+++ b/polly/test/Simplify/sweep_mapped_phi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Map %phi to A[j], so the scalar write in Stmt_for_bodyA can be removed.
 ;
diff --git a/polly/test/Simplify/sweep_mapped_value.ll b/polly/test/Simplify/sweep_mapped_value.ll
index 2e2f9c37febe..5992e2401ca5 100644
--- a/polly/test/Simplify/sweep_mapped_value.ll
+++ b/polly/test/Simplify/sweep_mapped_value.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Map %val to A[j], so the scalar write on Stmt_for_bodyB can be removed.
 ;
diff --git a/polly/test/Simplify/ununsed_read_in_region_entry.ll b/polly/test/Simplify/ununsed_read_in_region_entry.ll
index 9b2d4521e2d6..111c19f706b9 100644
--- a/polly/test/Simplify/ununsed_read_in_region_entry.ll
+++ b/polly/test/Simplify/ununsed_read_in_region_entry.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-print-simplify -disable-output< %s | FileCheck %s -match-full-lines
-; RUN: opt %loadPolly -polly-simplify -polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly '-passes=print<polly-simplify>' -disable-output< %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly '-passes=polly-simplify,polly-codegen' -S < %s | FileCheck %s -check-prefix=CODEGEN
 ;
 ; for (int i = 0; i < n; i+=1) {
 ;    (void)A[0];
diff --git a/polly/test/Support/Plugins.ll b/polly/test/Support/Plugins.ll
index cee878f1c6ac..c4579470192b 100644
--- a/polly/test/Support/Plugins.ll
+++ b/polly/test/Support/Plugins.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadNPMPolly -passes='polly-prepare,scop(print<polly-ast>)' -S < %s \
+; RUN: opt %loadPolly '-passes=polly-prepare,scop(print<polly-ast>)' -S < %s \
 ; RUN: | FileCheck %s
 
 ; This testcase tests plugin registration. Check-lines below serve to verify
diff --git a/polly/test/Support/defaultpipelines.ll b/polly/test/Support/defaultpipelines.ll
index ab0329a70327..6681042727c6 100644
--- a/polly/test/Support/defaultpipelines.ll
+++ b/polly/test/Support/defaultpipelines.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadNPMPolly -polly -O0 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
-; RUN: opt %loadNPMPolly -polly -O1 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
-; RUN: opt %loadNPMPolly -polly -O2 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
-; RUN: opt %loadNPMPolly -polly -O3 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
-; RUN: opt %loadNPMPolly -polly -Os -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
-; RUN: opt %loadNPMPolly -polly -Oz -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
+; RUN: opt %loadPolly -polly -O0 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
+; RUN: opt %loadPolly -polly -O1 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
+; RUN: opt %loadPolly -polly -O2 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
+; RUN: opt %loadPolly -polly -O3 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
+; RUN: opt %loadPolly -polly -Os -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
+; RUN: opt %loadPolly -polly -Oz -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
 ;
 ; Check that Polly's default pipeline works from detection to code generation
 ; with either pass manager.
diff --git a/polly/test/Support/dumpfunction.ll b/polly/test/Support/dumpfunction.ll
index 863212b2ef7d..e99261508f1a 100644
--- a/polly/test/Support/dumpfunction.ll
+++ b/polly/test/Support/dumpfunction.ll
@@ -1,9 +1,9 @@
 ; New pass manager
-; RUN: opt %loadNPMPolly -O3 -polly -polly-position=before-vectorizer -polly-dump-before --disable-output %s
+; RUN: opt %loadPolly -O3 -polly -polly-position=before-vectorizer -polly-dump-before --disable-output %s
 ; RUN: FileCheck --input-file=dumpfunction-callee-before.ll --check-prefix=CHECK --check-prefix=CALLEE %s
 ; RUN: FileCheck --input-file=dumpfunction-caller-before.ll --check-prefix=CHECK --check-prefix=CALLER %s
 ;
-; RUN: opt %loadNPMPolly -O3 -polly -polly-position=before-vectorizer -polly-dump-after --disable-output %s
+; RUN: opt %loadPolly -O3 -polly -polly-position=before-vectorizer -polly-dump-after --disable-output %s
 ; RUN: FileCheck --input-file=dumpfunction-callee-after.ll --check-prefix=CHECK --check-prefix=CALLEE %s
 ; RUN: FileCheck --input-file=dumpfunction-caller-after.ll --check-prefix=CHECK --check-prefix=CALLER %s
 
diff --git a/polly/test/Support/dumpmodule.ll b/polly/test/Support/dumpmodule.ll
index 693fe4bc6cde..d7aa88439f64 100644
--- a/polly/test/Support/dumpmodule.ll
+++ b/polly/test/Support/dumpmodule.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadNPMPolly -O3 -polly -polly-position=early             -polly-dump-before-file=%t-npm-before-early.ll    --disable-output < %s && FileCheck --input-file=%t-npm-before-early.ll    --check-prefix=EARLY %s
-; RUN: opt %loadNPMPolly -O3 -polly -polly-position=early             -polly-dump-after-file=%t-npm-after-early.ll      --disable-output < %s && FileCheck --input-file=%t-npm-after-early.ll     --check-prefix=EARLY --check-prefix=AFTEREARLY %s
+; RUN: opt %loadPolly -O3 -polly -polly-position=early             -polly-dump-before-file=%t-npm-before-early.ll    --disable-output < %s && FileCheck --input-file=%t-npm-before-early.ll    --check-prefix=EARLY %s
+; RUN: opt %loadPolly -O3 -polly -polly-position=early             -polly-dump-after-file=%t-npm-after-early.ll      --disable-output < %s && FileCheck --input-file=%t-npm-after-early.ll     --check-prefix=EARLY --check-prefix=AFTEREARLY %s
 ;
 ; Check the module dumping before Polly at specific positions in the
 ; pass pipeline.
diff --git a/polly/test/Support/exportjson.ll b/polly/test/Support/exportjson.ll
index 22cfea23534c..22ba845bafc2 100644
--- a/polly/test/Support/exportjson.ll
+++ b/polly/test/Support/exportjson.ll
@@ -1,6 +1,6 @@
 ; RUN: rm -rf %t
 ; RUN: mkdir -p %t
-; RUN: opt %loadNPMPolly -polly-import-jscop-dir=%t -polly -O2 -polly-export -S < %s
+; RUN: opt %loadPolly -polly-import-jscop-dir=%t -polly -O2 -polly-export -S < %s
 ; RUN: FileCheck %s -input-file %t/exportjson___%entry.split---%return.jscop
 ;
 ; for (int j = 0; j < n; j += 1) {
diff --git a/polly/test/Support/isl-args.ll b/polly/test/Support/isl-args.ll
index efa94194bc3f..442742d55f81 100644
--- a/polly/test/Support/isl-args.ll
+++ b/polly/test/Support/isl-args.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-scops -disable-output -polly-isl-arg=-V < %s | FileCheck %s -match-full-lines --check-prefix=VERSION
-; RUN: opt %loadPolly -polly-scops -disable-output -polly-isl-arg=-h < %s | FileCheck %s -match-full-lines --check-prefix=HELP
-; RUN: not opt %loadPolly -polly-scops -disable-output -polly-isl-arg=-asdf < %s 2>&1| FileCheck %s -match-full-lines --check-prefix=UNKNOWN
-; RUN: opt %loadPolly -polly-scops -disable-output -polly-isl-arg=--schedule-algorithm=feautrier < %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-isl-arg=-V < %s | FileCheck %s -match-full-lines --check-prefix=VERSION
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-isl-arg=-h < %s | FileCheck %s -match-full-lines --check-prefix=HELP
+; RUN: not opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-isl-arg=-asdf < %s 2>&1| FileCheck %s -match-full-lines --check-prefix=UNKNOWN
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-isl-arg=--schedule-algorithm=feautrier < %s
 
 ; VERSION: isl-{{.*}}-IMath-32
 ; HELP: Usage: -polly-isl-arg [OPTION...]
diff --git a/polly/test/Support/pipelineposition.ll b/polly/test/Support/pipelineposition.ll
index a4506ba1d64e..757af91011fb 100644
--- a/polly/test/Support/pipelineposition.ll
+++ b/polly/test/Support/pipelineposition.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadNPMPolly -O3 -polly -polly-position=early                    -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=NOINLINE
-; RUN: opt %loadNPMPolly -O3 -polly -polly-position=early -polly-run-inliner -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED1
-; RUN: opt %loadNPMPolly -O3 -polly -polly-position=before-vectorizer        -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED3
+; RUN: opt %loadPolly -O3 -polly -polly-position=early                    -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=NOINLINE
+; RUN: opt %loadPolly -O3 -polly -polly-position=early -polly-run-inliner -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED1
+; RUN: opt %loadPolly -O3 -polly -polly-position=before-vectorizer        -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED3
 ;
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/Support/pollyDebug.ll b/polly/test/Support/pollyDebug.ll
index ada079023b6c..e5e327c3976e 100644
--- a/polly/test/Support/pollyDebug.ll
+++ b/polly/test/Support/pollyDebug.ll
@@ -1,5 +1,5 @@
 ; Test if "polly-debug" flag enables debug prints from different parts of polly
-; RUN: opt %loadNPMPolly -O3 -polly -polly-debug --disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -O3 -polly -polly-debug --disable-output < %s 2>&1 | FileCheck %s
 ;
 ; REQUIRES: asserts
 
diff --git a/polly/test/lit.site.cfg.in b/polly/test/lit.site.cfg.in
index b44061260834..703752896239 100644
--- a/polly/test/lit.site.cfg.in
+++ b/polly/test/lit.site.cfg.in
@@ -38,16 +38,11 @@ if config.llvm_polly_link_into_tools == '' or \
    config.llvm_polly_link_into_tools.lower() == 'false' or \
    config.llvm_polly_link_into_tools.lower() == 'notfound' or \
    config.llvm_polly_link_into_tools.lower() == 'llvm_polly_link_into_tools-notfound':
-    config.substitutions.append(('%loadPolly', '-load '
-                                 + config.polly_lib_dir + '/LLVMPolly@LLVM_SHLIBEXT@'
-                                 + commonOpts ))
-    config.substitutions.append(('%loadNPMPolly', '-load-pass-plugin '
+    config.substitutions.append(('%loadPolly', '-load-pass-plugin '
                                  + config.polly_lib_dir + '/LLVMPolly@LLVM_SHLIBEXT@'
                                  + commonOpts ))
 else:
     config.substitutions.append(('%loadPolly', commonOpts ))
-    config.substitutions.append(('%loadNPMPolly', commonOpts ))
-
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)
diff --git a/polly/test/polly.ll b/polly/test/polly.ll
index f78cceacfb12..6654468470a6 100644
--- a/polly/test/polly.ll
+++ b/polly/test/polly.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-scops -S < %s | FileCheck %s
+; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -S < %s 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 define void @foo() nounwind {
 start:
-- 
GitLab


From 847c83f7cceeaec6676f33291081912d6b8fda5e Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 15 May 2024 07:02:31 +0200
Subject: [PATCH 0807/1206] InstCombine: Process addrspacecast uses in
 PointerReplacer (#91953)

This was looking through an addrspacecast, and not finding a later
unfoldable cast to another address space. Fixes improperly deleting
a required alloca + memcpy and introducing an illegal addrspacecast.

This also required fixing some worklist management issues with
addrspacecast, and assuming that only memcpy sources could need
replacement.

Regresses one test function, but this looks like it optimized
before by accident. It never saw the pointer use by the call
to readonly_callee, which should require insertion of a new cast.

Fixes #68120
---
 .../InstCombineLoadStoreAlloca.cpp            | 33 ++++----
 .../InstCombine/AMDGPU/issue68120.ll          | 81 +++++++++++++++++++
 .../InstCombine/ptr-replace-alloca.ll         |  6 +-
 3 files changed, 104 insertions(+), 16 deletions(-)
 create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/issue68120.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index 537890d9025f..4351a55ea1d3 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -342,9 +342,13 @@ bool PointerReplacer::collectUsersRecursive(Instruction &I) {
       Worklist.insert(Inst);
     } else if (isEqualOrValidAddrSpaceCast(Inst, FromAS)) {
       Worklist.insert(Inst);
+      if (!collectUsersRecursive(*Inst))
+        return false;
     } else if (Inst->isLifetimeStartOrEnd()) {
       continue;
     } else {
+      // TODO: For arbitrary uses with address space mismatches, should we check
+      // if we can introduce a valid addrspacecast?
       LLVM_DEBUG(dbgs() << "Cannot handle pointer user: " << *U << '\n');
       return false;
     }
@@ -406,20 +410,18 @@ void PointerReplacer::replace(Instruction *I) {
     NewSI->takeName(SI);
     WorkMap[SI] = NewSI;
   } else if (auto *MemCpy = dyn_cast<MemTransferInst>(I)) {
-    auto *SrcV = getReplacement(MemCpy->getRawSource());
-    // The pointer may appear in the destination of a copy, but we don't want to
-    // replace it.
-    if (!SrcV) {
-      assert(getReplacement(MemCpy->getRawDest()) &&
-             "destination not in replace list");
-      return;
-    }
+    auto *DestV = MemCpy->getRawDest();
+    auto *SrcV = MemCpy->getRawSource();
+
+    if (auto *DestReplace = getReplacement(DestV))
+      DestV = DestReplace;
+    if (auto *SrcReplace = getReplacement(SrcV))
+      SrcV = SrcReplace;
 
     IC.Builder.SetInsertPoint(MemCpy);
     auto *NewI = IC.Builder.CreateMemTransferInst(
-        MemCpy->getIntrinsicID(), MemCpy->getRawDest(), MemCpy->getDestAlign(),
-        SrcV, MemCpy->getSourceAlign(), MemCpy->getLength(),
-        MemCpy->isVolatile());
+        MemCpy->getIntrinsicID(), DestV, MemCpy->getDestAlign(), SrcV,
+        MemCpy->getSourceAlign(), MemCpy->getLength(), MemCpy->isVolatile());
     AAMDNodes AAMD = MemCpy->getAAMetadata();
     if (AAMD)
       NewI->setAAMetadata(AAMD);
@@ -432,16 +434,17 @@ void PointerReplacer::replace(Instruction *I) {
     assert(isEqualOrValidAddrSpaceCast(
                ASC, V->getType()->getPointerAddressSpace()) &&
            "Invalid address space cast!");
-    auto *NewV = V;
+
     if (V->getType()->getPointerAddressSpace() !=
         ASC->getType()->getPointerAddressSpace()) {
       auto *NewI = new AddrSpaceCastInst(V, ASC->getType(), "");
       NewI->takeName(ASC);
       IC.InsertNewInstWith(NewI, ASC->getIterator());
-      NewV = NewI;
+      WorkMap[ASC] = NewI;
+    } else {
+      WorkMap[ASC] = V;
     }
-    IC.replaceInstUsesWith(*ASC, NewV);
-    IC.eraseInstFromFunction(*ASC);
+
   } else {
     llvm_unreachable("should never reach here");
   }
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/issue68120.ll b/llvm/test/Transforms/InstCombine/AMDGPU/issue68120.ll
new file mode 100644
index 000000000000..346c64f096ba
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/issue68120.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=instcombine %s | FileCheck %s
+
+; It is illegal to pass ptr addrspace(4) in %arg by addrspacecasting
+; to ptr addrspace(5) to pass off to the stack argument. A temporary
+; alloca and memcpy is necessary.
+define void @issue68120_invalid_addrspacecast_introduced_0(ptr addrspace(4) byref([56 x i8]) %arg) {
+; CHECK-LABEL: define void @issue68120_invalid_addrspacecast_introduced_0(
+; CHECK-SAME: ptr addrspace(4) byref([56 x i8]) [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ADDRSPACECAST_0_TO_5:%.*]] = alloca [56 x i8], align 1, addrspace(5)
+; CHECK-NEXT:    call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef align 1 dereferenceable(56) [[ADDRSPACECAST_0_TO_5]], ptr addrspace(4) noundef align 1 dereferenceable(56) [[ARG]], i64 56, i1 false)
+; CHECK-NEXT:    call void @byval_func(ptr addrspace(5) [[ADDRSPACECAST_0_TO_5]])
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca [56 x i8], addrspace(5)
+  %alloca1 = alloca [56 x i8], addrspace(5)
+  call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) %alloca, ptr addrspace(4) %arg, i64 56, i1 false)
+  call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) %alloca1, ptr addrspace(5) %alloca, i64 56, i1 false)
+  %addrspacecast.alloca1 = addrspacecast ptr addrspace(5) %alloca1 to ptr
+  %addrspacecast.0.to.5 = addrspacecast ptr %addrspacecast.alloca1 to ptr addrspace(5)
+  call void @byval_func(ptr addrspace(5) %addrspacecast.0.to.5)
+  ret void
+}
+
+; Further reduced variant that already eliminated one of the copies
+define void @issue68120_invalid_addrspacecast_introduced_1(ptr addrspace(4) byref([56 x i8]) %arg) {
+; CHECK-LABEL: define void @issue68120_invalid_addrspacecast_introduced_1(
+; CHECK-SAME: ptr addrspace(4) byref([56 x i8]) [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ADDRSPACECAST_0_TO_5:%.*]] = alloca [56 x i8], align 1, addrspace(5)
+; CHECK-NEXT:    call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef align 1 dereferenceable(56) [[ADDRSPACECAST_0_TO_5]], ptr addrspace(4) noundef align 1 dereferenceable(56) [[ARG]], i64 56, i1 false)
+; CHECK-NEXT:    call void @byval_func(ptr addrspace(5) [[ADDRSPACECAST_0_TO_5]])
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca [56 x i8], addrspace(5)
+  call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) %alloca, ptr addrspace(4) %arg, i64 56, i1 false)
+  %addrspacecast.alloca = addrspacecast ptr addrspace(5) %alloca to ptr
+  %addrspacecast.0.to.5 = addrspacecast ptr %addrspacecast.alloca to ptr addrspace(5)
+  call void @byval_func(ptr addrspace(5) %addrspacecast.0.to.5)
+  ret void
+}
+
+define void @issue68120_use_cast_to_as0(ptr addrspace(4) byref([56 x i8]) %arg) {
+; CHECK-LABEL: define void @issue68120_use_cast_to_as0(
+; CHECK-SAME: ptr addrspace(4) byref([56 x i8]) [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [56 x i8], align 1, addrspace(5)
+; CHECK-NEXT:    call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef align 1 dereferenceable(56) [[ALLOCA]], ptr addrspace(4) noundef align 1 dereferenceable(56) [[ARG]], i64 56, i1 false)
+; CHECK-NEXT:    [[ADDRSPACECAST_ALLOCA:%.*]] = addrspacecast ptr addrspace(5) [[ALLOCA]] to ptr
+; CHECK-NEXT:    call void @other_func(ptr [[ADDRSPACECAST_ALLOCA]])
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca [56 x i8], addrspace(5)
+  call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) %alloca, ptr addrspace(4) %arg, i64 56, i1 false)
+  %addrspacecast.alloca = addrspacecast ptr addrspace(5) %alloca to ptr
+  %addrspacecast.0.to.5 = addrspacecast ptr %addrspacecast.alloca to ptr addrspace(5)
+  call void @other_func(ptr %addrspacecast.alloca)
+  ret void
+}
+
+define void @issue68120_uses_invalid_addrspacecast(ptr addrspace(4) byref([56 x i8]) %arg) {
+; CHECK-LABEL: define void @issue68120_uses_invalid_addrspacecast(
+; CHECK-SAME: ptr addrspace(4) byref([56 x i8]) [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ADDRSPACECAST_0_TO_5:%.*]] = alloca [56 x i8], align 1, addrspace(5)
+; CHECK-NEXT:    call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef align 1 dereferenceable(56) [[ADDRSPACECAST_0_TO_5]], ptr addrspace(4) noundef align 1 dereferenceable(56) [[ARG]], i64 56, i1 false)
+; CHECK-NEXT:    call void @byval_func(ptr addrspace(5) [[ADDRSPACECAST_0_TO_5]])
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca [56 x i8], addrspace(5)
+  %alloca1 = alloca [56 x i8], addrspace(5)
+  %illegal.cast = addrspacecast ptr addrspace(4) %arg to ptr addrspace(5)
+  call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) %alloca, ptr addrspace(5) %illegal.cast, i64 56, i1 false)
+  call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) %alloca1, ptr addrspace(5) %alloca, i64 56, i1 false)
+  %addrspacecast.alloca1 = addrspacecast ptr addrspace(5) %alloca1 to ptr
+  %addrspacecast.0.to.5 = addrspacecast ptr %addrspacecast.alloca1 to ptr addrspace(5)
+  call void @byval_func(ptr addrspace(5) %addrspacecast.0.to.5)
+  ret void
+}
+
+
+declare void @byval_func(ptr addrspace(5) byval([56 x i8]))
+declare void @other_func(ptr)
+
diff --git a/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll b/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
index a7aa3a8ef1be..c783b101251d 100644
--- a/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
+++ b/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
@@ -429,9 +429,13 @@ entry:
 
 declare i8 @readonly_callee(ptr readonly nocapture)
 
+; FIXME: This should be able to fold to call i8 @readonly_callee(ptr nonnull @g1)
 define i8 @call_readonly_remove_alloca() {
 ; CHECK-LABEL: @call_readonly_remove_alloca(
-; CHECK-NEXT:    [[V:%.*]] = call i8 @readonly_callee(ptr nonnull @g1)
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [32 x i8], align 1, addrspace(1)
+; CHECK-NEXT:    call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef align 1 dereferenceable(32) [[ALLOCA]], ptr noundef nonnull align 16 dereferenceable(32) @g1, i64 32, i1 false)
+; CHECK-NEXT:    [[P:%.*]] = addrspacecast ptr addrspace(1) [[ALLOCA]] to ptr
+; CHECK-NEXT:    [[V:%.*]] = call i8 @readonly_callee(ptr [[P]])
 ; CHECK-NEXT:    ret i8 [[V]]
 ;
   %alloca = alloca [32 x i8], addrspace(1)
-- 
GitLab


From 15397583e3d85eb1f1a051de26eb409aaedd3b54 Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma@quicinc.com>
Date: Tue, 14 May 2024 22:04:50 -0700
Subject: [PATCH 0808/1206] Revert "[polly] Port polly tests to use NPM"
 (#92215)

Reverts llvm/llvm-project#90632.

Causing failures on buildbots that dynamically load polly. Reverting
while we sort it out.
---
 polly/test/CodeGen/20100617.ll                |  2 +-
 polly/test/CodeGen/20100622.ll                |  4 +-
 polly/test/CodeGen/20100707.ll                |  2 +-
 polly/test/CodeGen/20100707_2.ll              |  2 +-
 polly/test/CodeGen/20100708.ll                |  2 +-
 polly/test/CodeGen/20100708_2.ll              |  2 +-
 polly/test/CodeGen/20100713.ll                |  2 +-
 polly/test/CodeGen/20100713_2.ll              |  2 +-
 polly/test/CodeGen/20100717.ll                |  2 +-
 polly/test/CodeGen/20100718-DomInfo-2.ll      |  2 +-
 polly/test/CodeGen/20100718-DomInfo.ll        |  2 +-
 .../CodeGen/20100720-MultipleConditions.ll    |  2 +-
 .../test/CodeGen/20100809-IndependentBlock.ll |  2 +-
 ...0100811-ScalarDependencyBetweenBrAndCnd.ll |  2 +-
 polly/test/CodeGen/20101030-Overflow.ll       |  2 +-
 polly/test/CodeGen/20101103-Overflow3.ll      |  2 +-
 polly/test/CodeGen/20101103-signmissmatch.ll  |  2 +-
 .../test/CodeGen/20110226-Ignore-Dead-Code.ll |  2 +-
 .../test/CodeGen/20110226-PHI-Node-removed.ll |  2 +-
 polly/test/CodeGen/20120316-InvalidCast.ll    |  2 +-
 .../CodeGen/20120403-RHS-type-mismatch.ll     |  2 +-
 polly/test/CodeGen/20130221.ll                |  2 +-
 .../20150328-SCEVExpanderIntroducesNewIV.ll   |  2 +-
 polly/test/CodeGen/Intrinsics/llvm-expect.ll  |  2 +-
 .../do_not_mutate_debug_info.ll               |  2 +-
 .../loop_nest_param_parallel.ll               |  2 +-
 .../single_loop_param_parallel.ll             |  4 +-
 polly/test/CodeGen/MemAccess/bad_alignment.ll |  2 +-
 .../MemAccess/codegen_address_space.ll        |  2 +-
 .../MemAccess/codegen_constant_offset.ll      |  2 +-
 .../test/CodeGen/MemAccess/codegen_simple.ll  |  2 +-
 .../CodeGen/MemAccess/codegen_simple_float.ll |  2 +-
 .../CodeGen/MemAccess/codegen_simple_md.ll    |  4 +-
 .../MemAccess/codegen_simple_md_float.ll      |  4 +-
 .../test/CodeGen/MemAccess/different_types.ll |  4 +-
 polly/test/CodeGen/MemAccess/generate-all.ll  |  4 +-
 .../CodeGen/MemAccess/invariant_base_ptr.ll   |  4 +-
 .../test/CodeGen/MemAccess/multiple_types.ll  |  4 +-
 polly/test/CodeGen/MemAccess/simple.ll        |  2 +-
 .../MemAccess/update_access_functions.ll      |  4 +-
 polly/test/CodeGen/OpenMP/alias-metadata.ll   |  2 +-
 .../floord-as-argument-to-subfunction.ll      |  2 +-
 polly/test/CodeGen/OpenMP/inlineasm.ll        |  2 +-
 .../invariant_base_pointer_preloaded.ll       |  2 +-
 ...ant_base_pointer_preloaded_different_bb.ll |  2 +-
 ...base_pointer_preloaded_pass_only_needed.ll |  2 +-
 .../invariant_base_pointers_preloaded.ll      |  2 +-
 .../OpenMP/loop-body-references-outer-iv.ll   |  4 +-
 .../loop-body-references-outer-values-2.ll    |  4 +-
 .../loop-body-references-outer-values-3.ll    |  4 +-
 .../loop-body-references-outer-values.ll      |  4 +-
 .../OpenMP/loop-bounds-reference-outer-ids.ll |  4 +-
 .../test/CodeGen/OpenMP/mapped-phi-access.ll  |  2 +-
 polly/test/CodeGen/OpenMP/matmul-parallel.ll  |  4 +-
 polly/test/CodeGen/OpenMP/recomputed-srem.ll  |  2 +-
 ...ference-argument-from-non-affine-region.ll |  6 +-
 .../test/CodeGen/OpenMP/reference-other-bb.ll |  2 +-
 .../OpenMP/reference-preceeding-loop.ll       |  4 +-
 polly/test/CodeGen/OpenMP/reference_latest.ll |  2 +-
 polly/test/CodeGen/OpenMP/scev-rewriting.ll   |  2 +-
 polly/test/CodeGen/OpenMP/single_loop.ll      | 18 ++---
 ...single_loop_with_loop_invariant_baseptr.ll |  4 +-
 .../CodeGen/OpenMP/single_loop_with_param.ll  |  6 +-
 ...o-parallel-loops-reference-outer-indvar.ll |  4 +-
 polly/test/CodeGen/PHIInExit.ll               |  2 +-
 .../combine_different_values.ll               |  2 +-
 .../RuntimeDebugBuilder/stmt_tracing.ll       |  2 +-
 polly/test/CodeGen/alias-check-multi-dim.ll   |  2 +-
 .../CodeGen/alias_metadata_too_many_arrays.ll |  2 +-
 ...aliasing_different_base_and_access_type.ll |  2 +-
 .../aliasing_different_pointer_types.ll       |  2 +-
 .../aliasing_multidimensional_access.ll       |  2 +-
 .../CodeGen/aliasing_parametric_simple_1.ll   |  2 +-
 .../CodeGen/aliasing_parametric_simple_2.ll   |  2 +-
 polly/test/CodeGen/aliasing_struct_element.ll |  2 +-
 polly/test/CodeGen/alignment.ll               |  2 +-
 polly/test/CodeGen/annotated_alias_scopes.ll  |  2 +-
 polly/test/CodeGen/blas_sscal_simplified.ll   |  2 +-
 ...code-hosting-and-escape-map-computation.ll |  2 +-
 polly/test/CodeGen/constant_condition.ll      |  2 +-
 polly/test/CodeGen/create-conditional-scop.ll |  2 +-
 ...d_instruction_referenced_by_parameter_1.ll |  2 +-
 ...d_instruction_referenced_by_parameter_2.ll |  2 +-
 polly/test/CodeGen/debug-intrinsics.ll        |  4 +-
 ...nce_problem_after_early_codegen_bailout.ll |  2 +-
 polly/test/CodeGen/empty_domain_in_context.ll |  2 +-
 polly/test/CodeGen/entry_with_trivial_phi.ll  |  2 +-
 .../entry_with_trivial_phi_other_bb.ll        |  2 +-
 .../error-stmt-in-non-affine-region.ll        |  2 +-
 ...or_block_contains_invalid_memory_access.ll |  2 +-
 polly/test/CodeGen/exprModDiv.ll              |  8 +-
 .../hoisted_load_escapes_through_phi.ll       |  4 +-
 polly/test/CodeGen/hoisting_1.ll              |  2 +-
 polly/test/CodeGen/hoisting_2.ll              |  2 +-
 polly/test/CodeGen/inner_scev_sdiv_1.ll       |  2 +-
 polly/test/CodeGen/inner_scev_sdiv_2.ll       |  2 +-
 polly/test/CodeGen/inner_scev_sdiv_3.ll       |  2 +-
 polly/test/CodeGen/inner_scev_sdiv_in_lb.ll   |  4 +-
 .../inner_scev_sdiv_in_lb_invariant.ll        |  2 +-
 polly/test/CodeGen/inner_scev_sdiv_in_rtc.ll  |  2 +-
 polly/test/CodeGen/intrinsics_lifetime.ll     |  2 +-
 polly/test/CodeGen/intrinsics_misc.ll         |  2 +-
 .../inv-load-lnt-crash-wrong-order-2.ll       |  2 +-
 .../inv-load-lnt-crash-wrong-order-3.ll       |  2 +-
 .../CodeGen/inv-load-lnt-crash-wrong-order.ll |  2 +-
 .../test/CodeGen/invariant-load-dimension.ll  |  4 +-
 ...-load-preload-base-pointer-origin-first.ll |  2 +-
 .../CodeGen/invariant_cannot_handle_void.ll   |  4 +-
 polly/test/CodeGen/invariant_load.ll          |  2 +-
 .../CodeGen/invariant_load_address_space.ll   |  2 +-
 .../CodeGen/invariant_load_alias_metadata.ll  |  2 +-
 .../CodeGen/invariant_load_base_pointer.ll    |  2 +-
 ...invariant_load_base_pointer_conditional.ll |  2 +-
 ...variant_load_base_pointer_conditional_2.ll |  6 +-
 ...ariant_load_canonicalize_array_baseptrs.ll |  2 +-
 .../test/CodeGen/invariant_load_condition.ll  |  2 +-
 .../invariant_load_different_sized_types.ll   |  2 +-
 polly/test/CodeGen/invariant_load_escaping.ll |  2 +-
 .../invariant_load_escaping_second_scop.ll    |  2 +-
 .../invariant_load_in_non_affine_subregion.ll |  2 +-
 polly/test/CodeGen/invariant_load_loop_ub.ll  |  2 +-
 ...ant_load_not_executed_but_in_parameters.ll |  2 +-
 .../test/CodeGen/invariant_load_outermost.ll  |  2 +-
 ...riant_load_parameters_cyclic_dependence.ll |  4 +-
 .../CodeGen/invariant_load_ptr_ptr_noalias.ll |  2 +-
 .../test/CodeGen/invariant_load_scalar_dep.ll |  2 +-
 ...riant_load_scalar_escape_alloca_sharing.ll |  2 +-
 ...oads_from_struct_with_different_types_1.ll |  2 +-
 ...oads_from_struct_with_different_types_2.ll |  2 +-
 ...invariant_loads_ignore_parameter_bounds.ll |  2 +-
 .../invariant_verify_function_failed.ll       |  2 +-
 .../invariant_verify_function_failed_2.ll     |  4 +-
 polly/test/CodeGen/issue56692.ll              |  2 +-
 .../large-numbers-in-boundary-context.ll      |  2 +-
 .../test/CodeGen/load_subset_with_context.ll  |  2 +-
 .../loop-invariant-load-type-mismatch.ll      |  2 +-
 polly/test/CodeGen/loop_with_condition.ll     |  2 +-
 polly/test/CodeGen/loop_with_condition_2.ll   |  2 +-
 .../test/CodeGen/loop_with_condition_ineq.ll  |  2 +-
 .../CodeGen/loop_with_condition_nested.ll     |  4 +-
 ..._conditional_entry_edge_split_hard_case.ll |  2 +-
 polly/test/CodeGen/memcpy_annotations.ll      |  2 +-
 .../multidim-non-matching-typesize-2.ll       |  2 +-
 .../CodeGen/multidim-non-matching-typesize.ll |  2 +-
 ..._2d_parametric_array_static_loop_bounds.ll |  2 +-
 polly/test/CodeGen/multidim_alias_check.ll    |  2 +-
 polly/test/CodeGen/multiple-codegens.ll       |  5 +-
 polly/test/CodeGen/multiple-scops-in-a-row.ll |  2 +-
 .../multiple-types-invariant-load-2.ll        |  2 +-
 .../CodeGen/multiple-types-invariant-load.ll  |  2 +-
 .../multiple_sai_fro_same_base_address.ll     |  4 +-
 polly/test/CodeGen/no-overflow-tracking.ll    |  4 +-
 polly/test/CodeGen/no_guard_bb.ll             |  2 +-
 ...non-affine-dominance-generated-entering.ll |  2 +-
 .../CodeGen/non-affine-exit-node-dominance.ll |  2 +-
 .../non-affine-phi-node-expansion-2.ll        |  2 +-
 .../non-affine-phi-node-expansion-3.ll        |  2 +-
 .../non-affine-phi-node-expansion-4.ll        |  2 +-
 .../CodeGen/non-affine-phi-node-expansion.ll  |  2 +-
 ...e-region-exit-phi-incoming-synthesize-2.ll |  2 +-
 ...ine-region-exit-phi-incoming-synthesize.ll |  2 +-
 .../non-affine-region-implicit-store.ll       |  2 +-
 ...ine-region-phi-references-in-scop-value.ll |  2 +-
 .../non-affine-subregion-dominance-reuse.ll   |  2 +-
 polly/test/CodeGen/non-affine-switch.ll       |  2 +-
 .../non-affine-synthesized-in-branch.ll       |  2 +-
 polly/test/CodeGen/non-affine-update.ll       |  4 +-
 .../non-hoisted-load-needed-as-base-ptr.ll    |  2 +-
 .../test/CodeGen/non_affine_float_compare.ll  |  2 +-
 .../CodeGen/only_non_affine_error_region.ll   |  2 +-
 polly/test/CodeGen/openmp_limit_threads.ll    | 12 +--
 .../test/CodeGen/out-of-scop-phi-node-use.ll  |  2 +-
 polly/test/CodeGen/param_div_div_div_2.ll     |  4 +-
 polly/test/CodeGen/partial_write_array.ll     |  2 +-
 polly/test/CodeGen/partial_write_emptyset.ll  |  2 +-
 ...l_write_full_write_that_appears_partial.ll |  2 +-
 .../partial_write_impossible_restriction.ll   |  2 +-
 polly/test/CodeGen/partial_write_in_region.ll |  4 +-
 .../partial_write_in_region_with_loop.ll      |  4 +-
 .../CodeGen/partial_write_mapped_scalar.ll    |  2 +-
 .../partial_write_mapped_scalar_subregion.ll  |  2 +-
 polly/test/CodeGen/perf_monitoring.ll         |  2 +-
 .../perf_monitoring_cycles_per_scop.ll        |  2 +-
 .../perf_monitoring_trip_counts_per_scop.ll   |  2 +-
 polly/test/CodeGen/phi-defined-before-scop.ll |  2 +-
 .../phi_after_error_block_outside_of_scop.ll  |  2 +-
 .../test/CodeGen/phi_condition_modeling_1.ll  |  2 +-
 .../test/CodeGen/phi_condition_modeling_2.ll  |  2 +-
 .../test/CodeGen/phi_conditional_simple_1.ll  |  4 +-
 .../phi_in_exit_early_lnt_failure_1.ll        |  2 +-
 .../phi_in_exit_early_lnt_failure_2.ll        |  2 +-
 .../phi_in_exit_early_lnt_failure_3.ll        |  2 +-
 .../phi_in_exit_early_lnt_failure_5.ll        |  2 +-
 polly/test/CodeGen/phi_loop_carried_float.ll  |  2 +-
 .../CodeGen/phi_loop_carried_float_escape.ll  |  4 +-
 polly/test/CodeGen/phi_scalar_simple_1.ll     |  2 +-
 polly/test/CodeGen/phi_scalar_simple_2.ll     |  2 +-
 .../CodeGen/phi_with_multi_exiting_edges_2.ll |  2 +-
 polly/test/CodeGen/phi_with_one_exit_edge.ll  |  2 +-
 .../CodeGen/pointer-type-expressions-2.ll     |  4 +-
 .../test/CodeGen/pointer-type-expressions.ll  |  4 +-
 .../pointer-type-pointer-type-comparison.ll   |  4 +-
 polly/test/CodeGen/pointer_rem.ll             |  4 +-
 polly/test/CodeGen/pr25241.ll                 |  2 +-
 polly/test/CodeGen/ptrtoint_as_parameter.ll   |  2 +-
 polly/test/CodeGen/read-only-scalars.ll       |  4 +-
 polly/test/CodeGen/reduction.ll               |  2 +-
 polly/test/CodeGen/reduction_2.ll             |  2 +-
 polly/test/CodeGen/reduction_simple_binary.ll |  2 +-
 .../test/CodeGen/region-with-instructions.ll  |  2 +-
 polly/test/CodeGen/region_exiting-domtree.ll  |  2 +-
 .../CodeGen/region_multiexit_partialwrite.ll  |  2 +-
 ...run-time-condition-with-scev-parameters.ll |  4 +-
 polly/test/CodeGen/run-time-condition.ll      |  2 +-
 .../scalar-references-used-in-scop-compute.ll |  2 +-
 .../test/CodeGen/scalar-store-from-same-bb.ll |  2 +-
 polly/test/CodeGen/scalar_codegen_crash.ll    |  2 +-
 polly/test/CodeGen/scev-backedgetaken.ll      |  2 +-
 .../CodeGen/scev-division-invariant-load.ll   |  2 +-
 polly/test/CodeGen/scev.ll                    |  2 +-
 .../CodeGen/scev_expansion_in_nonaffine.ll    |  2 +-
 .../CodeGen/scev_looking_through_bitcasts.ll  |  2 +-
 .../CodeGen/scop_expander_insert_point.ll     |  2 +-
 polly/test/CodeGen/scop_expander_segfault.ll  |  2 +-
 ...p_never_executed_runtime_check_location.ll |  2 +-
 polly/test/CodeGen/select-base-pointer.ll     |  2 +-
 polly/test/CodeGen/sequential_loops.ll        |  2 +-
 .../CodeGen/simple_loop_non_single_exit.ll    |  2 +-
 .../CodeGen/simple_loop_non_single_exit_2.ll  |  2 +-
 polly/test/CodeGen/simple_non_single_entry.ll |  2 +-
 polly/test/CodeGen/simple_nonaffine_loop.ll   |  2 +-
 .../single_do_loop_int_max_iterations.ll      |  2 +-
 .../single_do_loop_int_param_iterations.ll    |  2 +-
 .../single_do_loop_ll_max_iterations.ll       |  4 +-
 .../CodeGen/single_do_loop_one_iteration.ll   |  2 +-
 .../CodeGen/single_do_loop_scev_replace.ll    |  2 +-
 polly/test/CodeGen/single_loop.ll             |  2 +-
 .../CodeGen/single_loop_int_max_iterations.ll |  2 +-
 .../CodeGen/single_loop_ll_max_iterations.ll  |  2 +-
 .../test/CodeGen/single_loop_one_iteration.ll |  2 +-
 polly/test/CodeGen/single_loop_param.ll       |  2 +-
 .../CodeGen/single_loop_param_less_equal.ll   |  6 +-
 .../CodeGen/single_loop_param_less_than.ll    |  4 +-
 .../CodeGen/single_loop_zero_iterations.ll    |  2 +-
 polly/test/CodeGen/split_edge_of_exit.ll      |  4 +-
 polly/test/CodeGen/split_edges.ll             |  2 +-
 polly/test/CodeGen/split_edges_2.ll           |  2 +-
 polly/test/CodeGen/srem-in-other-bb.ll        |  2 +-
 .../stack-overflow-in-load-hoisting.ll        |  2 +-
 .../test/CodeGen/stmt_split_no_dependence.ll  |  2 +-
 .../CodeGen/switch-in-non-affine-region.ll    |  2 +-
 .../synthesizable_phi_write_after_loop.ll     |  2 +-
 .../test-invalid-operands-for-select-2.ll     |  2 +-
 .../test-invalid-operands-for-select.ll       |  2 +-
 polly/test/CodeGen/test.ll                    |  2 +-
 .../two-loops-right-after-each-other-2.ll     |  2 +-
 .../two-scops-in-row-invalidate-scevs.ll      |  2 +-
 polly/test/CodeGen/two-scops-in-row.ll        |  4 +-
 polly/test/CodeGen/udiv_expansion_position.ll |  2 +-
 .../CodeGen/uninitialized_scalar_memory.ll    |  2 +-
 .../unpredictable-loop-unsynthesizable.ll     |  6 +-
 .../test/CodeGen/variant_load_empty_domain.ll |  2 +-
 .../whole-scop-non-affine-subregion.ll        |  2 +-
 polly/test/DeLICM/confused_order.ll           |  4 +-
 ...ontradicting_assumed_context_and_domain.ll |  2 +-
 polly/test/DeLICM/load-in-cond-inf-loop.ll    |  2 +-
 polly/test/DeLICM/map_memset_zero.ll          |  4 +-
 polly/test/DeLICM/nomap_alreadymapped.ll      |  2 +-
 polly/test/DeLICM/nomap_escaping.ll           |  2 +-
 polly/test/DeLICM/nomap_occupied.ll           |  2 +-
 polly/test/DeLICM/nomap_readonly.ll           |  2 +-
 polly/test/DeLICM/nomap_spuriouswrite.ll      |  2 +-
 polly/test/DeLICM/nomap_storagesize.ll        |  2 +-
 polly/test/DeLICM/nomap_writewrite.ll         |  2 +-
 polly/test/DeLICM/outofquota-reverseDomain.ll |  2 +-
 polly/test/DeLICM/pass_existence.ll           |  6 +-
 polly/test/DeLICM/pr41656.ll                  |  2 +-
 polly/test/DeLICM/pr48783.ll                  |  2 +-
 polly/test/DeLICM/reduction.ll                |  2 +-
 .../reduction_looprotate_gvnpre_cond1.ll      |  2 +-
 .../reduction_looprotate_gvnpre_cond2.ll      |  2 +-
 ...reduction_looprotate_gvnpre_nopreheader.ll |  2 +-
 .../reduction_looprotate_licm_nopreheader.ll  |  2 +-
 .../reduction_looprotate_loopguard_gvnpre.ll  |  2 +-
 .../reduction_looprotate_loopguard_licm1.ll   |  2 +-
 .../reduction_looprotate_loopguard_licm2.ll   |  2 +-
 .../reduction_looprotate_loopguard_licm3.ll   |  2 +-
 .../test/DeLICM/reduction_unrelatedunusual.ll |  2 +-
 polly/test/DeLICM/reject_loadafterstore.ll    |  2 +-
 polly/test/DeLICM/reject_outofquota.ll        |  4 +-
 polly/test/DeLICM/reject_storeafterstore.ll   |  2 +-
 polly/test/DeLICM/reject_storeinsubregion.ll  |  2 +-
 polly/test/DeLICM/reject_unusualstore.ll      |  4 +-
 polly/test/DeLICM/skip_maywrite.ll            |  2 +-
 polly/test/DeLICM/skip_multiaccess.ll         |  2 +-
 polly/test/DeLICM/skip_notinloop.ll           |  2 +-
 polly/test/DeLICM/skip_scalaraccess.ll        |  2 +-
 .../DeadCodeElimination/chained_iterations.ll |  4 +-
 .../chained_iterations_2.ll                   |  4 +-
 polly/test/DeadCodeElimination/computeout.ll  |  5 +-
 .../dead_iteration_elimination.ll             |  3 +-
 .../non-affine-affine-mix.ll                  |  2 +-
 polly/test/DeadCodeElimination/non-affine.ll  |  2 +-
 .../test/DeadCodeElimination/null_schedule.ll |  2 +-
 polly/test/DependenceInfo/computeout.ll       |  6 +-
 .../different_schedule_dimensions.ll          |  4 +-
 polly/test/DependenceInfo/do_pluto_matmult.ll |  6 +-
 polly/test/DependenceInfo/fine_grain_dep_0.ll |  7 +-
 .../generate_may_write_dependence_info.ll     |  2 +-
 .../test/DependenceInfo/infeasible_context.ll |  5 +-
 ...writes_do_not_block_must_writes_for_war.ll |  2 +-
 .../nonaffine-condition-buildMemoryAccess.ll  |  2 +-
 .../reduction_complex_location.ll             |  6 +-
 ...ndences_equal_non_reduction_dependences.ll |  2 +-
 .../reduction_dependences_not_null.ll         |  2 +-
 ...reduction_and_non_reduction_dependences.ll |  2 +-
 .../reduction_multiple_loops_array_sum.ll     |  6 +-
 .../reduction_multiple_loops_array_sum_2.ll   |  2 +-
 .../reduction_multiple_loops_array_sum_3.ll   |  2 +-
 .../reduction_multiple_reductions.ll          |  2 +-
 .../reduction_multiple_reductions_2.ll        |  2 +-
 .../reduction_only_reduction_like_access.ll   |  2 +-
 ...lly_escaping_intermediate_in_other_stmt.ll |  2 +-
 .../reduction_privatization_deps.ll           |  2 +-
 .../reduction_privatization_deps_2.ll         |  2 +-
 .../reduction_privatization_deps_3.ll         |  2 +-
 .../reduction_privatization_deps_4.ll         |  2 +-
 .../reduction_privatization_deps_5.ll         |  2 +-
 .../test/DependenceInfo/reduction_sequence.ll |  2 +-
 .../DependenceInfo/reduction_simple_iv.ll     |  2 +-
 ...ion_simple_iv_debug_wrapped_dependences.ll |  2 +-
 .../reduction_simple_privatization_deps_2.ll  |  2 +-
 ...n_simple_privatization_deps_w_parameter.ll |  2 +-
 ...duction_two_reductions_different_rloops.ll |  2 +-
 polly/test/DependenceInfo/sequential_loops.ll | 79 +++++++++++--------
 polly/test/ForwardOpTree/atax.ll              |  2 +-
 polly/test/ForwardOpTree/changed-kind.ll      |  2 +-
 .../test/ForwardOpTree/forward_from_region.ll |  2 +-
 polly/test/ForwardOpTree/forward_hoisted.ll   |  2 +-
 .../test/ForwardOpTree/forward_instruction.ll |  2 +-
 .../test/ForwardOpTree/forward_into_region.ll |  2 +-
 .../forward_into_region_redundant_use.ll      |  2 +-
 polly/test/ForwardOpTree/forward_load.ll      |  3 +-
 .../forward_load_differentarray.ll            |  2 +-
 .../forward_load_double_write.ll              |  2 +-
 .../ForwardOpTree/forward_load_fromloop.ll    |  2 +-
 .../ForwardOpTree/forward_load_indirect.ll    |  2 +-
 .../forward_load_memset_after.ll              |  2 +-
 .../forward_load_memset_before.ll             |  2 +-
 .../ForwardOpTree/forward_load_tripleuse.ll   |  2 +-
 .../forward_load_unrelatedunusual.ll          |  2 +-
 polly/test/ForwardOpTree/forward_phi_load.ll  |  2 +-
 polly/test/ForwardOpTree/forward_readonly.ll  |  4 +-
 polly/test/ForwardOpTree/forward_reusue.ll    |  2 +-
 polly/test/ForwardOpTree/forward_store.ll     |  2 +-
 .../forward_synthesizable_definloop.ll        |  2 +-
 .../forward_synthesizable_indvar.ll           |  2 +-
 .../forward_synthesizable_useinloop.ll        |  2 +-
 .../test/ForwardOpTree/forward_transitive.ll  |  2 +-
 polly/test/ForwardOpTree/jacobi-1d.ll         |  2 +-
 .../ForwardOpTree/noforward_from_region.ll    |  2 +-
 .../noforward_load_conditional.ll             |  2 +-
 .../noforward_load_writebetween.ll            |  2 +-
 .../ForwardOpTree/noforward_outofquota.ll     |  4 +-
 polly/test/ForwardOpTree/noforward_partial.ll |  2 +-
 polly/test/ForwardOpTree/noforward_phi.ll     |  2 +-
 .../ForwardOpTree/noforward_selfrefphi.ll     |  2 +-
 .../ForwardOpTree/noforward_sideffects.ll     |  2 +-
 .../noforward_synthesizable_unknownit.ll      |  2 +-
 polly/test/ForwardOpTree/out-of-quota1.ll     |  2 +-
 .../alias_checks_with_empty_context.ll        |  2 +-
 polly/test/IstAstInfo/alias_simple_1.ll       | 10 +--
 polly/test/IstAstInfo/alias_simple_2.ll       | 12 +--
 polly/test/IstAstInfo/alias_simple_3.ll       | 10 +--
 .../aliasing_arrays_with_identical_base.ll    |  2 +-
 .../aliasing_multiple_alias_groups.ll         |  4 +-
 .../aliasing_parametric_simple_1.ll           |  2 +-
 .../aliasing_parametric_simple_2.ll           |  2 +-
 .../IstAstInfo/dependence_distance_minimal.ll |  2 +-
 .../domain_bounded_only_with_context.ll       |  2 +-
 polly/test/IstAstInfo/non_affine_access.ll    |  2 +-
 ...reduction_clauses_onedimensional_access.ll |  2 +-
 ...ndences_equal_non_reduction_dependences.ll |  2 +-
 .../reduction_different_reduction_clauses.ll  |  2 +-
 ...ction_modulo_and_loop_reversal_schedule.ll |  2 +-
 ...ion_modulo_and_loop_reversal_schedule_2.ll |  2 +-
 ...ion_modulo_schedule_multiple_dimensions.ll |  2 +-
 ...n_modulo_schedule_multiple_dimensions_2.ll |  2 +-
 ...n_modulo_schedule_multiple_dimensions_3.ll |  2 +-
 ...n_modulo_schedule_multiple_dimensions_4.ll |  2 +-
 ...n_modulo_schedule_multiple_dimensions_5.ll |  2 +-
 .../reduction_multiple_dimensions.ll          |  2 +-
 .../reduction_multiple_dimensions_2.ll        |  2 +-
 .../reduction_multiple_dimensions_3.ll        |  2 +-
 .../reduction_multiple_dimensions_4.ll        |  2 +-
 polly/test/IstAstInfo/run-time-condition.ll   |  2 +-
 .../runtime_context_with_error_blocks.ll      |  2 +-
 .../IstAstInfo/simple-run-time-condition.ll   |  2 +-
 .../test/IstAstInfo/single_loop_strip_mine.ll |  4 +-
 .../single_loop_uint_max_iterations.ll        |  2 +-
 .../single_loop_ull_max_iterations.ll         |  2 +-
 .../ImportAccesses-Bad-relation.ll            |  2 +-
 .../ImportAccesses-No-accesses-key.ll         |  2 +-
 .../ImportAccesses-Not-enough-MemAcc.ll       |  2 +-
 .../ImportAccesses-Not-enough-statements.ll   |  2 +-
 .../ImportAccesses-Relation-mispelled.ll      |  2 +-
 .../ImportAccesses-Statements-mispelled.ll    |  2 +-
 ...ImportAccesses-Undeclared-ScopArrayInfo.ll |  2 +-
 .../ImportAccesses-Wrong-number-dimensions.ll |  2 +-
 .../ImportArrays-Mispelled-type.ll            |  2 +-
 .../ImportArrays-Negative-size.ll             |  2 +-
 .../ImportArrays/ImportArrays-No-name.ll      |  2 +-
 .../ImportArrays/ImportArrays-No-sizes-key.ll |  2 +-
 .../ImportArrays/ImportArrays-No-type-key.ll  |  2 +-
 .../ImportContext-Context-mispelled.ll        |  2 +-
 .../ImportContext-Not-parameter-set.ll        |  2 +-
 .../ImportContext-Unvalid-Context.ll          |  2 +-
 .../ImportContext-Wrong-dimension.ll          |  2 +-
 .../ImportSchedule-No-schedule-key.ll         |  2 +-
 .../ImportSchedule-Schedule-not-valid.ll      |  2 +-
 .../ImportSchedule-Statements-mispelled.ll    |  2 +-
 .../ImportSchedule-Wrong-number-statements.ll |  2 +-
 .../load_after_store_same_statement.ll        |  6 +-
 .../read_from_original.ll                     |  6 +-
 .../MaximalStaticExpansion/too_many_writes.ll |  6 +-
 .../working_deps_between_inners.ll            |  3 +-
 .../working_deps_between_inners_phi.ll        |  6 +-
 .../working_expansion.ll                      |  3 +-
 ...sion_multiple_dependences_per_statement.ll |  3 +-
 ...sion_multiple_instruction_per_statement.ll |  3 +-
 .../working_phi_expansion.ll                  |  6 +-
 .../working_phi_two_scalars.ll                |  6 +-
 .../working_value_expansion.ll                |  3 +-
 .../prune_only_scalardeps.ll                  |  3 +-
 .../2012-03-16-Empty-Domain.ll                |  2 +-
 .../2013-04-11-Empty-Domain-two.ll            |  2 +-
 .../GreedyFuse/fuse-double.ll                 |  4 +-
 .../GreedyFuse/fuse-except-first.ll           |  4 +-
 .../GreedyFuse/fuse-except-third.ll           |  4 +-
 .../GreedyFuse/fuse-inner-carried.ll          |  4 +-
 .../GreedyFuse/fuse-inner-third.ll            |  4 +-
 .../GreedyFuse/fuse-inner.ll                  |  4 +-
 .../GreedyFuse/fuse-simple.ll                 |  4 +-
 .../GreedyFuse/nofuse-simple.ll               |  4 +-
 .../GreedyFuse/nofuse-with-middle.ll          |  4 +-
 .../ManualOptimization/disable_nonforced.ll   |  2 +-
 .../distribute_heuristic.ll                   |  4 +-
 .../distribute_illegal_looploc.ll             |  2 +-
 .../distribute_illegal_pragmaloc.ll           |  2 +-
 .../ManualOptimization/unroll_disable.ll      |  2 +-
 .../ManualOptimization/unroll_double.ll       |  2 +-
 .../ManualOptimization/unroll_full.ll         |  2 +-
 .../ManualOptimization/unroll_heuristic.ll    |  4 +-
 .../ManualOptimization/unroll_partial.ll      |  4 +-
 .../unroll_partial_followup.ll                |  8 +-
 .../ScheduleOptimizer/SIMDInParallelFor.ll    |  2 +-
 polly/test/ScheduleOptimizer/computeout.ll    |  6 +-
 .../ensure-correct-tile-sizes.ll              |  4 +-
 .../focaltech_test_detail_threshold-7bc17e.ll |  3 +-
 .../full_partial_tile_separation.ll           |  2 +-
 polly/test/ScheduleOptimizer/line-tiling-2.ll |  2 +-
 polly/test/ScheduleOptimizer/line-tiling.ll   |  2 +-
 .../mat_mul_pattern_data_layout.ll            |  2 +-
 .../mat_mul_pattern_data_layout_2.ll          |  4 +-
 .../ScheduleOptimizer/one-dimensional-band.ll |  2 +-
 .../ScheduleOptimizer/outer_coincidence.ll    |  4 +-
 ...attern-matching-based-opts-after-delicm.ll |  4 +-
 ...tern-matching-based-opts-after-delicm_2.ll |  2 +-
 .../pattern-matching-based-opts.ll            |  8 +-
 .../pattern-matching-based-opts_11.ll         |  4 +-
 .../pattern-matching-based-opts_12.ll         |  2 +-
 .../pattern-matching-based-opts_13.ll         |  2 +-
 .../pattern-matching-based-opts_14.ll         |  4 +-
 .../pattern-matching-based-opts_15.ll         |  2 +-
 .../pattern-matching-based-opts_16.ll         |  2 +-
 .../pattern-matching-based-opts_17.ll         |  2 +-
 .../pattern-matching-based-opts_18.ll         |  2 +-
 .../pattern-matching-based-opts_19.ll         |  2 +-
 .../pattern-matching-based-opts_2.ll          |  2 +-
 .../pattern-matching-based-opts_20.ll         |  2 +-
 .../pattern-matching-based-opts_21.ll         |  2 +-
 .../pattern-matching-based-opts_22.ll         |  2 +-
 .../pattern-matching-based-opts_24.ll         |  2 +-
 .../pattern-matching-based-opts_25.ll         |  4 +-
 .../pattern-matching-based-opts_3.ll          |  8 +-
 .../pattern-matching-based-opts_4.ll          |  8 +-
 .../pattern-matching-based-opts_5.ll          |  6 +-
 .../pattern-matching-based-opts_6.ll          |  6 +-
 .../pattern-matching-based-opts_7.ll          |  2 +-
 .../pattern-matching-based-opts_8.ll          |  2 +-
 .../pattern-matching-based-opts_9.ll          |  4 +-
 .../pattern_matching_based_opts_splitmap.ll   |  2 +-
 .../prevectorization-without-tiling.ll        |  2 +-
 .../ScheduleOptimizer/prevectorization.ll     |  4 +-
 .../ScheduleOptimizer/rectangular-tiling.ll   |  8 +-
 .../ScheduleOptimizer/schedule_computeout.ll  |  2 +-
 polly/test/ScheduleOptimizer/statistics.ll    |  2 +-
 .../ScheduleOptimizer/tile_after_fusion.ll    |  4 +-
 ...vivid_vbi_gen_sliced-before-llvmreduced.ll |  2 +-
 .../aliasing_parametric_simple_1.ll           |  2 +-
 .../aliasing_parametric_simple_2.ll           |  2 +-
 polly/test/ScopDetect/aliasing_simple_1.ll    |  2 +-
 polly/test/ScopDetect/aliasing_simple_2.ll    |  2 +-
 .../base_pointer_load_setNewAccessRelation.ll |  2 +-
 .../base_pointer_setNewAccessRelation.ll      |  2 +-
 polly/test/ScopDetect/callbr.ll               |  4 +-
 .../ScopDetect/collective_invariant_loads.ll  |  2 +-
 .../ScopDetect/cross_loop_non_single_exit.ll  |  2 +-
 .../cross_loop_non_single_exit_2.ll           |  2 +-
 ...ependency_to_phi_node_outside_of_region.ll |  2 +-
 polly/test/ScopDetect/dot-scops-npm.ll        |  2 +-
 polly/test/ScopDetect/dot-scops.ll            |  2 +-
 .../ScopDetect/error-block-always-executed.ll |  2 +-
 .../error-block-referenced-from-scop.ll       |  2 +-
 .../ScopDetect/error-block-unreachable.ll     |  2 +-
 .../ScopDetect/expand-region-correctly-2.ll   |  2 +-
 .../ScopDetect/expand-region-correctly.ll     |  2 +-
 .../test/ScopDetect/ignore_func_flag_regex.ll |  2 +-
 .../index_from_unpredictable_loop.ll          |  4 +-
 .../index_from_unpredictable_loop2.ll         |  4 +-
 polly/test/ScopDetect/indvars.ll              |  2 +-
 polly/test/ScopDetect/intrinsics_1.ll         |  2 +-
 polly/test/ScopDetect/intrinsics_2.ll         |  2 +-
 polly/test/ScopDetect/intrinsics_3.ll         |  2 +-
 .../ScopDetect/invalid-latch-conditions.ll    |  6 +-
 .../ScopDetect/invalidate_scalar_evolution.ll |  2 +-
 .../ScopDetect/invariant-load-before-scop.ll  |  2 +-
 polly/test/ScopDetect/keep_going_expansion.ll |  2 +-
 polly/test/ScopDetect/mod_ref_read_pointer.ll |  4 +-
 polly/test/ScopDetect/more-than-one-loop.ll   |  4 +-
 .../ScopDetect/multidim-with-undef-size.ll    |  2 +-
 polly/test/ScopDetect/multidim.ll             |  2 +-
 .../ScopDetect/multidim_indirect_access.ll    |  2 +-
 ..._two_accesses_different_delinearization.ll |  2 +-
 .../ScopDetect/nested_loop_single_exit.ll     |  4 +-
 .../test/ScopDetect/non-affine-conditional.ll |  2 +-
 .../ScopDetect/non-affine-float-compare.ll    |  2 +-
 ...-affine-loop-condition-dependent-access.ll |  8 +-
 ...ffine-loop-condition-dependent-access_2.ll |  6 +-
 ...ffine-loop-condition-dependent-access_3.ll |  6 +-
 polly/test/ScopDetect/non-affine-loop.ll      | 10 +--
 .../non-beneficial-loops-small-trip-count.ll  |  2 +-
 .../non-constant-add-rec-start-expr.ll        |  2 +-
 .../ScopDetect/non-simple-memory-accesses.ll  |  2 +-
 .../ScopDetect/non_affine_loop_condition.ll   |  4 +-
 polly/test/ScopDetect/only-one-affine-loop.ll |  2 +-
 polly/test/ScopDetect/only_func_flag.ll       |  2 +-
 polly/test/ScopDetect/only_func_flag_regex.ll |  2 +-
 .../parametric-multiply-in-scev-2.ll          |  2 +-
 .../ScopDetect/parametric-multiply-in-scev.ll |  2 +-
 .../phi_with_multi_exiting_edges.ll           |  2 +-
 .../profitability-large-basic-blocks.ll       |  6 +-
 .../profitability-two-nested-loops.ll         |  2 +-
 polly/test/ScopDetect/remove_all_children.ll  |  2 +-
 polly/test/ScopDetect/report-scop-location.ll |  2 +-
 .../restrict-undef-size-scopdetect.ll         |  2 +-
 polly/test/ScopDetect/run_time_alias_check.ll |  2 +-
 polly/test/ScopDetect/scev_remove_max.ll      |  2 +-
 polly/test/ScopDetect/sequential_loops.ll     |  6 +-
 polly/test/ScopDetect/simple_loop.ll          |  2 +-
 .../simple_loop_non_single_entry.ll           |  2 +-
 .../ScopDetect/simple_loop_non_single_exit.ll |  2 +-
 .../simple_loop_non_single_exit_2.ll          |  2 +-
 .../ScopDetect/simple_loop_two_phi_nodes.ll   |  2 +-
 .../test/ScopDetect/simple_loop_with_param.ll |  2 +-
 .../ScopDetect/simple_loop_with_param_2.ll    |  2 +-
 .../ScopDetect/simple_non_single_entry.ll     |  2 +-
 .../ScopDetect/skip_function_attribute.ll     |  2 +-
 .../srem_with_parametric_divisor.ll           |  2 +-
 polly/test/ScopDetect/statistics.ll           |  2 +-
 polly/test/ScopDetect/switch-in-loop-patch.ll |  2 +-
 .../ReportAlias-01.ll                         |  2 +-
 .../ScopDetectionDiagnostics/ReportEntry.ll   |  2 +-
 .../ReportFuncCall-01.ll                      |  2 +-
 .../ReportIrreducibleRegion.ll                |  2 +-
 .../ReportIrreducibleRegionWithoutDebugLoc.ll |  2 +-
 .../ReportLoopBound-01.ll                     |  6 +-
 .../ReportLoopHasNoExit.ll                    |  4 +-
 .../ReportMultipleNonAffineAccesses.ll        | 12 +--
 .../ReportNonAffineAccess-01.ll               |  2 +-
 .../ReportUnprofitable.ll                     |  4 +-
 .../ReportUnreachableInExit.ll                |  2 +-
 .../ReportVariantBasePtr-01.ll                |  2 +-
 .../loop_has_multiple_exits.ll                |  2 +-
 .../loop_partially_in_scop-2.ll               |  2 +-
 .../loop_partially_in_scop.ll                 |  2 +-
 .../ScopInfo/20110312-Fail-without-basicaa.ll |  2 +-
 .../20111108-Parameter-not-detected.ll        |  2 +-
 ...03-16-Crash-because-of-unsigned-in-scev.ll |  2 +-
 .../2015-10-04-Crash-in-domain-generation.ll  |  2 +-
 polly/test/ScopInfo/Alias-0.ll                |  4 +-
 polly/test/ScopInfo/Alias-1.ll                |  4 +-
 polly/test/ScopInfo/Alias-2.ll                |  4 +-
 polly/test/ScopInfo/Alias-3.ll                |  4 +-
 polly/test/ScopInfo/Alias-4.ll                |  4 +-
 .../test/ScopInfo/BoundChecks/single-loop.ll  |  4 +-
 polly/test/ScopInfo/BoundChecks/two-loops.ll  |  4 +-
 polly/test/ScopInfo/NonAffine/div_backedge.ll |  2 +-
 polly/test/ScopInfo/NonAffine/div_domain.ll   |  2 +-
 ...nt_loads_dependent_in_non_affine_region.ll |  2 +-
 .../ScopInfo/NonAffine/modulo_backedge.ll     |  2 +-
 .../test/ScopInfo/NonAffine/modulo_domain.ll  |  2 +-
 ...ffine-loop-condition-dependent-access_1.ll |  4 +-
 ...ffine-loop-condition-dependent-access_2.ll |  6 +-
 ...ffine-loop-condition-dependent-access_3.ll |  6 +-
 .../non_affine_access_with_range_2.ll         |  2 +-
 .../ScopInfo/NonAffine/non_affine_but_sdiv.ll |  2 +-
 .../ScopInfo/NonAffine/non_affine_but_srem.ll |  2 +-
 .../non_affine_conditional_nested.ll          |  2 +-
 ...ine_conditional_surrounding_affine_loop.ll |  4 +-
 ...conditional_surrounding_non_affine_loop.ll |  6 +-
 .../NonAffine/non_affine_float_compare.ll     |  2 +-
 .../NonAffine/non_affine_loop_condition.ll    |  6 +-
 .../NonAffine/non_affine_loop_used_later.ll   |  4 +-
 .../NonAffine/non_affine_parametric_loop.ll   |  2 +-
 .../non_affine_region_guaranteed_non-entry.ll |  2 +-
 ...whole-scop-non-affine-subregion-in-loop.ll |  2 +-
 .../aliasing_conditional_alias_groups_1.ll    |  2 +-
 .../aliasing_conditional_alias_groups_2.ll    |  2 +-
 polly/test/ScopInfo/aliasing_dead_access.ll   |  2 +-
 .../aliasing_many_arrays_to_compare.ll        |  8 +-
 .../aliasing_many_read_only_acesses.ll        |  2 +-
 .../aliasing_multiple_alias_groups.ll         |  4 +-
 .../aliasing_with_non_affine_access.ll        |  2 +-
 .../allow-all-parameters-dereferencable.ll    |  6 +-
 polly/test/ScopInfo/assume_gep_bounds.ll      |  4 +-
 polly/test/ScopInfo/assume_gep_bounds_2.ll    |  2 +-
 polly/test/ScopInfo/assume_gep_bounds_many.ll |  4 +-
 .../avoid_new_parameters_from_geps.ll         |  2 +-
 polly/test/ScopInfo/bool-addrec.ll            |  2 +-
 .../test/ScopInfo/bounded_loop_assumptions.ll |  2 +-
 ...ces-loop-scev-with-unknown-iterations-2.ll |  4 +-
 ...ces-loop-scev-with-unknown-iterations-3.ll |  6 +-
 ...ences-loop-scev-with-unknown-iterations.ll |  6 +-
 polly/test/ScopInfo/bug_2010_10_22.ll         |  2 +-
 polly/test/ScopInfo/bug_2011_1_5.ll           |  2 +-
 .../test/ScopInfo/bug_scev_not_fully_eval.ll  |  2 +-
 polly/test/ScopInfo/cfg_consequences.ll       |  2 +-
 .../test/ScopInfo/complex-branch-structure.ll |  2 +-
 polly/test/ScopInfo/complex-condition.ll      |  2 +-
 polly/test/ScopInfo/complex-expression.ll     |  2 +-
 polly/test/ScopInfo/complex-loop-nesting.ll   |  2 +-
 .../ScopInfo/complex-successor-structure-2.ll |  2 +-
 .../ScopInfo/complex-successor-structure-3.ll |  4 +-
 .../ScopInfo/complex-successor-structure.ll   |  2 +-
 .../complex_domain_binary_condition.ll        |  2 +-
 .../ScopInfo/complex_execution_context.ll     |  2 +-
 polly/test/ScopInfo/cond_constant_in_loop.ll  |  2 +-
 polly/test/ScopInfo/cond_in_loop.ll           |  2 +-
 .../ScopInfo/condition-after-error-block-2.ll |  2 +-
 ...condition-after-error-block-before-scop.ll |  2 +-
 .../ScopInfo/condtion-after-error-block.ll    |  2 +-
 polly/test/ScopInfo/const_srem_sdiv.ll        |  4 +-
 .../constant-non-integer-branch-condition.ll  |  2 +-
 .../ScopInfo/constant_factor_in_parameter.ll  |  4 +-
 ...stant_functions_outside_scop_as_unknown.ll |  2 +-
 polly/test/ScopInfo/constant_start_integer.ll |  2 +-
 polly/test/ScopInfo/debug_call.ll             |  2 +-
 .../delinearize-together-all-data-refs.ll     |  2 +-
 polly/test/ScopInfo/div_by_zero.ll            |  2 +-
 .../do-not-model-error-block-accesses.ll      |  2 +-
 .../eager-binary-and-or-conditions.ll         |  4 +-
 .../early_exit_for_complex_domains.ll         |  2 +-
 polly/test/ScopInfo/error-blocks-1.ll         |  2 +-
 polly/test/ScopInfo/error-blocks-2.ll         |  4 +-
 polly/test/ScopInfo/escaping_empty_scop.ll    |  2 +-
 polly/test/ScopInfo/exit-phi-1.ll             |  4 +-
 polly/test/ScopInfo/exit-phi-2.ll             |  2 +-
 polly/test/ScopInfo/exit_phi_accesses-2.ll    |  2 +-
 polly/test/ScopInfo/exit_phi_accesses.ll      |  2 +-
 .../ScopInfo/expensive-boundary-context.ll    |  4 +-
 ...onstant_factor_introduces_new_parameter.ll |  4 +-
 polly/test/ScopInfo/full-function.ll          |  4 +-
 polly/test/ScopInfo/granularity_same_name.ll  |  8 +-
 .../test/ScopInfo/granularity_scalar-indep.ll |  2 +-
 ...ity_scalar-indep_cross-referencing-phi1.ll |  2 +-
 ...ity_scalar-indep_cross-referencing-phi2.ll |  2 +-
 .../granularity_scalar-indep_epilogue.ll      |  2 +-
 .../granularity_scalar-indep_epilogue_last.ll |  2 +-
 .../granularity_scalar-indep_noepilogue.ll    |  2 +-
 .../granularity_scalar-indep_ordered-2.ll     |  2 +-
 .../granularity_scalar-indep_ordered.ll       |  2 +-
 polly/test/ScopInfo/i1_params.ll              |  2 +-
 polly/test/ScopInfo/infeasible-rtc.ll         |  4 +-
 .../ScopInfo/infeasible_invalid_context.ll    |  4 +-
 polly/test/ScopInfo/int2ptr_ptr2int.ll        |  4 +-
 polly/test/ScopInfo/int2ptr_ptr2int_2.ll      |  8 +-
 polly/test/ScopInfo/integers.ll               |  2 +-
 .../ScopInfo/inter-error-bb-dependence.ll     |  2 +-
 polly/test/ScopInfo/inter_bb_scalar_dep.ll    |  4 +-
 .../intra-non-affine-stmt-phi-node.ll         |  4 +-
 .../ScopInfo/intra_and_inter_bb_scalar_dep.ll |  4 +-
 polly/test/ScopInfo/intra_bb_scalar_dep.ll    |  4 +-
 polly/test/ScopInfo/intrinsics.ll             |  2 +-
 ..._add_rec_after_invariant_load_remapping.ll |  2 +-
 .../invalidate_iterator_during_MA_removal.ll  |  2 +-
 .../test/ScopInfo/invariant-load-instlist.ll  |  2 +-
 ...ariant-loads-leave-read-only-statements.ll |  4 +-
 polly/test/ScopInfo/invariant_load.ll         |  2 +-
 ...load_access_classes_different_base_type.ll |  4 +-
 ...ss_classes_different_base_type_escaping.ll |  4 +-
 ...lasses_different_base_type_same_pointer.ll |  4 +-
 ...fferent_base_type_same_pointer_escaping.ll |  4 +-
 .../ScopInfo/invariant_load_addrec_sum.ll     |  2 +-
 .../ScopInfo/invariant_load_base_pointer.ll   |  2 +-
 ...invariant_load_base_pointer_conditional.ll |  2 +-
 ...ariant_load_base_pointer_in_conditional.ll |  2 +-
 .../invariant_load_branch_condition.ll        |  4 +-
 ...ariant_load_canonicalize_array_baseptrs.ll |  2 +-
 ...iant_load_canonicalize_array_baseptrs_2.ll |  2 +-
 ...iant_load_canonicalize_array_baseptrs_3.ll |  2 +-
 ...iant_load_canonicalize_array_baseptrs_4.ll |  2 +-
 ...ant_load_canonicalize_array_baseptrs_4b.ll |  2 +-
 ...ant_load_canonicalize_array_baseptrs_4c.ll |  2 +-
 ...iant_load_canonicalize_array_baseptrs_5.ll |  2 +-
 .../invariant_load_complex_condition.ll       |  4 +-
 .../test/ScopInfo/invariant_load_condition.ll |  2 +-
 .../invariant_load_dereferenceable.ll         |  4 +-
 ...iant_load_distinct_parameter_valuations.ll |  2 +-
 .../ScopInfo/invariant_load_in_non_affine.ll  |  4 +-
 polly/test/ScopInfo/invariant_load_loop_ub.ll |  4 +-
 .../invariant_load_ptr_ptr_noalias.ll         |  4 +-
 .../ScopInfo/invariant_load_scalar_dep.ll     |  2 +-
 .../ScopInfo/invariant_load_stmt_domain.ll    |  2 +-
 .../invariant_load_zext_parameter-2.ll        |  4 +-
 .../ScopInfo/invariant_load_zext_parameter.ll |  4 +-
 ...load_zextended_in_own_execution_context.ll |  4 +-
 ...invariant_loads_complicated_dependences.ll |  2 +-
 .../invariant_loads_cyclic_dependences.ll     |  2 +-
 polly/test/ScopInfo/invariant_loop_bounds.ll  |  2 +-
 ...ariant_same_loop_bound_multiple_times-1.ll |  2 +-
 ...ariant_same_loop_bound_multiple_times-2.ll |  2 +-
 polly/test/ScopInfo/isl_aff_out_of_bounds.ll  |  2 +-
 polly/test/ScopInfo/isl_trip_count_01.ll      |  2 +-
 polly/test/ScopInfo/isl_trip_count_02.ll      |  2 +-
 polly/test/ScopInfo/isl_trip_count_03.ll      |  2 +-
 .../isl_trip_count_multiple_exiting_blocks.ll |  2 +-
 polly/test/ScopInfo/licm_load.ll              |  4 +-
 polly/test/ScopInfo/licm_potential_store.ll   |  4 +-
 polly/test/ScopInfo/licm_reduction_nested.ll  |  4 +-
 .../long-compile-time-alias-analysis.ll       |  2 +-
 .../long-sequence-of-error-blocks-2.ll        |  2 +-
 .../ScopInfo/long-sequence-of-error-blocks.ll |  4 +-
 .../test/ScopInfo/loop-multiexit-succ-cond.ll |  4 +-
 polly/test/ScopInfo/loop_affine_bound_0.ll    |  4 +-
 polly/test/ScopInfo/loop_affine_bound_1.ll    |  4 +-
 polly/test/ScopInfo/loop_affine_bound_2.ll    |  4 +-
 polly/test/ScopInfo/loop_carry.ll             |  2 +-
 .../test/ScopInfo/many-scalar-dependences.ll  |  2 +-
 polly/test/ScopInfo/max-loop-depth.ll         |  2 +-
 polly/test/ScopInfo/memcpy-raw-source.ll      |  2 +-
 polly/test/ScopInfo/memcpy.ll                 |  4 +-
 polly/test/ScopInfo/memmove.ll                |  4 +-
 polly/test/ScopInfo/memset.ll                 |  4 +-
 polly/test/ScopInfo/memset_null.ll            |  4 +-
 .../ScopInfo/mismatching-array-dimensions.ll  |  2 +-
 .../mod_ref_access_pointee_arguments.ll       |  6 +-
 .../mod_ref_read_pointee_arguments.ll         |  6 +-
 polly/test/ScopInfo/mod_ref_read_pointer.ll   |  4 +-
 polly/test/ScopInfo/mod_ref_read_pointers.ll  |  6 +-
 polly/test/ScopInfo/modulo_zext_1.ll          |  2 +-
 polly/test/ScopInfo/modulo_zext_2.ll          |  2 +-
 polly/test/ScopInfo/modulo_zext_3.ll          |  2 +-
 polly/test/ScopInfo/multi-scop.ll             |  2 +-
 .../ScopInfo/multidim_2d-diagonal-matrix.ll   |  4 +-
 .../multidim_2d_outer_parametric_offset.ll    |  2 +-
 ..._2d_parametric_array_static_loop_bounds.ll |  2 +-
 .../ScopInfo/multidim_2d_with_modref_call.ll  |  8 +-
 .../multidim_2d_with_modref_call_2.ll         |  8 +-
 ..._3d_parametric_array_static_loop_bounds.ll |  2 +-
 ...idim_fixedsize_different_dimensionality.ll |  2 +-
 .../multidim_fixedsize_multi_offset.ll        |  2 +-
 .../ScopInfo/multidim_fold_constant_dim.ll    |  2 +-
 .../multidim_fold_constant_dim_zero.ll        |  2 +-
 polly/test/ScopInfo/multidim_fortran_2d.ll    |  4 +-
 .../ScopInfo/multidim_fortran_2d_params.ll    |  4 +-
 .../multidim_fortran_2d_with_modref_call.ll   |  8 +-
 polly/test/ScopInfo/multidim_fortran_srem.ll  |  2 +-
 .../test/ScopInfo/multidim_gep_pointercast.ll |  2 +-
 .../ScopInfo/multidim_gep_pointercast2.ll     |  2 +-
 .../multidim_ivs_and_integer_offsets_3d.ll    |  2 +-
 ...multidim_ivs_and_parameteric_offsets_3d.ll |  2 +-
 .../test/ScopInfo/multidim_many_references.ll |  4 +-
 .../ScopInfo/multidim_nested_start_integer.ll |  4 +-
 .../multidim_nested_start_share_parameter.ll  |  2 +-
 polly/test/ScopInfo/multidim_only_ivs_2d.ll   |  2 +-
 polly/test/ScopInfo/multidim_only_ivs_3d.ll   |  2 +-
 .../ScopInfo/multidim_only_ivs_3d_cast.ll     |  2 +-
 .../ScopInfo/multidim_only_ivs_3d_reverse.ll  |  2 +-
 .../ScopInfo/multidim_param_in_subscript-2.ll |  2 +-
 .../ScopInfo/multidim_param_in_subscript.ll   |  2 +-
 .../multidim_parameter_addrec_product.ll      |  2 +-
 .../multidim_single_and_multidim_array.ll     | 16 ++--
 polly/test/ScopInfo/multidim_srem.ll          |  2 +-
 polly/test/ScopInfo/multidim_with_bitcast.ll  |  2 +-
 .../ScopInfo/multiple-binary-or-conditions.ll |  4 +-
 ...ss-offset-not-dividable-by-element-size.ll |  2 +-
 .../ScopInfo/multiple-types-non-affine-2.ll   |  4 +-
 .../ScopInfo/multiple-types-non-affine.ll     |  4 +-
 .../multiple-types-non-power-of-two-2.ll      |  2 +-
 .../multiple-types-non-power-of-two.ll        |  2 +-
 .../multiple-types-two-dimensional-2.ll       |  2 +-
 .../multiple-types-two-dimensional.ll         |  2 +-
 polly/test/ScopInfo/multiple-types.ll         |  4 +-
 .../test/ScopInfo/multiple_exiting_blocks.ll  |  2 +-
 .../multiple_exiting_blocks_two_loop.ll       |  2 +-
 polly/test/ScopInfo/multiple_latch_blocks.ll  |  2 +-
 polly/test/ScopInfo/nested-loops.ll           |  2 +-
 .../no-scalar-deps-in-non-affine-subregion.ll |  2 +-
 polly/test/ScopInfo/non-affine-region-phi.ll  |  4 +-
 .../ScopInfo/non-affine-region-with-loop-2.ll |  2 +-
 .../ScopInfo/non-affine-region-with-loop.ll   |  4 +-
 polly/test/ScopInfo/non-precise-inv-load-1.ll |  2 +-
 polly/test/ScopInfo/non-precise-inv-load-2.ll |  2 +-
 polly/test/ScopInfo/non-precise-inv-load-3.ll |  2 +-
 polly/test/ScopInfo/non-precise-inv-load-4.ll |  2 +-
 polly/test/ScopInfo/non-precise-inv-load-5.ll |  2 +-
 polly/test/ScopInfo/non-precise-inv-load-6.ll |  2 +-
 polly/test/ScopInfo/non-pure-function-call.ll |  2 +-
 ...-pure-function-calls-causes-dead-blocks.ll |  2 +-
 .../test/ScopInfo/non-pure-function-calls.ll  |  2 +-
 polly/test/ScopInfo/non_affine_access.ll      |  4 +-
 polly/test/ScopInfo/non_affine_region_1.ll    |  2 +-
 polly/test/ScopInfo/non_affine_region_2.ll    |  2 +-
 polly/test/ScopInfo/non_affine_region_3.ll    |  4 +-
 polly/test/ScopInfo/non_affine_region_4.ll    |  2 +-
 .../ScopInfo/nonaffine-buildMemoryAccess.ll   |  2 +-
 polly/test/ScopInfo/not-a-reduction.ll        |  2 +-
 polly/test/ScopInfo/opaque-struct.ll          |  2 +-
 ...gion-entry-phi-node-nonaffine-subregion.ll |  2 +-
 ...ut-of-scop-use-in-region-entry-phi-node.ll |  2 +-
 .../ScopInfo/parameter-constant-division.ll   |  4 +-
 .../ScopInfo/parameter_in_dead_statement.ll   |  8 +-
 polly/test/ScopInfo/parameter_product.ll      |  2 +-
 .../parameter_with_constant_factor_in_add.ll  |  2 +-
 .../ScopInfo/partially_invariant_load_1.ll    |  4 +-
 .../ScopInfo/partially_invariant_load_2.ll    |  2 +-
 .../test/ScopInfo/phi-in-non-affine-region.ll |  2 +-
 polly/test/ScopInfo/phi_after_error_block.ll  |  2 +-
 .../test/ScopInfo/phi_condition_modeling_1.ll |  2 +-
 .../test/ScopInfo/phi_condition_modeling_2.ll |  2 +-
 .../test/ScopInfo/phi_conditional_simple_1.ll |  2 +-
 polly/test/ScopInfo/phi_loop_carried_float.ll |  2 +-
 polly/test/ScopInfo/phi_not_grouped_at_top.ll |  2 +-
 polly/test/ScopInfo/phi_scalar_simple_1.ll    |  2 +-
 polly/test/ScopInfo/phi_scalar_simple_2.ll    |  2 +-
 polly/test/ScopInfo/phi_with_invoke_edge.ll   |  2 +-
 .../ScopInfo/pointer-comparison-no-nsw.ll     |  2 +-
 polly/test/ScopInfo/pointer-comparison.ll     |  2 +-
 .../test/ScopInfo/pointer-type-expressions.ll |  2 +-
 ...er-used-as-base-pointer-and-scalar-read.ll |  2 +-
 .../polly-timeout-parameter-bounds.ll         |  2 +-
 ...eserve-equiv-class-order-in-basic_block.ll |  2 +-
 .../test/ScopInfo/process_added_dimensions.ll |  2 +-
 .../test/ScopInfo/pwaff-complexity-bailout.ll |  2 +-
 polly/test/ScopInfo/ranged_parameter.ll       |  2 +-
 polly/test/ScopInfo/ranged_parameter_2.ll     |  2 +-
 polly/test/ScopInfo/ranged_parameter_wrap.ll  |  2 +-
 .../test/ScopInfo/ranged_parameter_wrap_2.ll  |  2 +-
 .../read-only-scalar-used-in-phi-2.ll         |  2 +-
 .../ScopInfo/read-only-scalar-used-in-phi.ll  |  2 +-
 polly/test/ScopInfo/read-only-scalars.ll      |  4 +-
 polly/test/ScopInfo/read-only-statements.ll   |  2 +-
 .../ScopInfo/reduction_alternating_base.ll    |  2 +-
 ...uction_chain_partially_outside_the_scop.ll |  2 +-
 .../ScopInfo/reduction_different_index.ll     |  2 +-
 .../ScopInfo/reduction_different_index1.ll    |  2 +-
 .../reduction_disabled_multiplicative.ll      |  2 +-
 .../reduction_escaping_intermediate.ll        |  2 +-
 .../reduction_escaping_intermediate_2.ll      |  2 +-
 .../reduction_invalid_different_operators.ll  |  2 +-
 .../reduction_invalid_overlapping_accesses.ll |  2 +-
 .../reduction_multiple_loops_array_sum.ll     |  2 +-
 .../reduction_multiple_loops_array_sum_1.ll   |  2 +-
 .../reduction_multiple_simple_binary.ll       |  2 +-
 .../reduction_non_overlapping_chains.ll       |  2 +-
 .../reduction_only_reduction_like_access.ll   |  2 +-
 polly/test/ScopInfo/reduction_simple_fp.ll    |  2 +-
 .../ScopInfo/reduction_simple_w_constant.ll   |  2 +-
 polly/test/ScopInfo/reduction_simple_w_iv.ll  |  2 +-
 .../ScopInfo/reduction_two_identical_reads.ll |  4 +-
 .../redundant_parameter_constraint.ll         |  2 +-
 .../test/ScopInfo/region-with-instructions.ll |  2 +-
 polly/test/ScopInfo/remarks.ll                |  2 +-
 .../required-invariant-loop-bounds.ll         |  4 +-
 .../ScopInfo/restriction_in_dead_block.ll     |  2 +-
 .../run-time-check-many-array-disjuncts.ll    |  4 +-
 .../run-time-check-many-parameters.ll         |  2 +-
 .../run-time-check-many-piecewise-aliasing.ll |  4 +-
 .../run-time-check-read-only-arrays.ll        |  2 +-
 .../same-base-address-scalar-and-array.ll     |  2 +-
 polly/test/ScopInfo/scalar.ll                 |  2 +-
 .../ScopInfo/scalar_dependence_cond_br.ll     |  2 +-
 polly/test/ScopInfo/scalar_to_array.ll        |  4 +-
 .../scev-div-with-evaluatable-divisor.ll      |  2 +-
 polly/test/ScopInfo/scev-invalidated.ll       |  2 +-
 .../schedule-const-post-dominator-walk-2.ll   |  2 +-
 .../schedule-const-post-dominator-walk.ll     |  2 +-
 .../schedule-constuction-endless-loop1.ll     |  2 +-
 .../schedule-constuction-endless-loop2.ll     |  2 +-
 ...tly-contructed-in-case-of-infinite-loop.ll |  2 +-
 .../scop-affine-parameter-ordering.ll         |  2 +-
 polly/test/ScopInfo/sign_wrapped_set.ll       |  2 +-
 polly/test/ScopInfo/simple_loop_1.ll          |  2 +-
 polly/test/ScopInfo/simple_loop_2.ll          |  2 +-
 polly/test/ScopInfo/simple_loop_unsigned.ll   |  2 +-
 polly/test/ScopInfo/simple_loop_unsigned_2.ll |  2 +-
 polly/test/ScopInfo/simple_loop_unsigned_3.ll |  2 +-
 .../ScopInfo/simple_nonaffine_loop_not.ll     |  2 +-
 polly/test/ScopInfo/smax.ll                   |  2 +-
 polly/test/ScopInfo/statistics.ll             |  2 +-
 .../stmt_split_exit_of_region_stmt.ll         |  2 +-
 .../ScopInfo/stmt_split_no_after_split.ll     |  2 +-
 .../test/ScopInfo/stmt_split_no_dependence.ll |  2 +-
 polly/test/ScopInfo/stmt_split_on_store.ll    |  2 +-
 .../ScopInfo/stmt_split_on_synthesizable.ll   |  2 +-
 .../stmt_split_phi_in_beginning_bb.ll         |  2 +-
 polly/test/ScopInfo/stmt_split_phi_in_stmt.ll |  2 +-
 .../ScopInfo/stmt_split_scalar_dependence.ll  |  2 +-
 polly/test/ScopInfo/stmt_split_within_loop.ll |  2 +-
 .../stmt_with_read_but_without_sideffect.ll   |  2 +-
 polly/test/ScopInfo/switch-1.ll               |  4 +-
 polly/test/ScopInfo/switch-2.ll               |  4 +-
 polly/test/ScopInfo/switch-3.ll               |  4 +-
 polly/test/ScopInfo/switch-4.ll               |  4 +-
 polly/test/ScopInfo/switch-5.ll               |  4 +-
 polly/test/ScopInfo/switch-6.ll               |  4 +-
 polly/test/ScopInfo/switch-7.ll               |  5 +-
 polly/test/ScopInfo/tempscop-printing.ll      |  2 +-
 .../ScopInfo/test-wrapping-in-condition.ll    |  4 +-
 polly/test/ScopInfo/truncate-1.ll             |  2 +-
 polly/test/ScopInfo/truncate-2.ll             |  2 +-
 polly/test/ScopInfo/truncate-3.ll             |  2 +-
 polly/test/ScopInfo/two-loops-one-infinite.ll |  2 +-
 .../two-loops-right-after-each-other.ll       |  2 +-
 polly/test/ScopInfo/undef_in_cond.ll          |  2 +-
 polly/test/ScopInfo/unnamed_nonaffine.ll      |  4 +-
 polly/test/ScopInfo/unnamed_stmts.ll          |  2 +-
 .../ScopInfo/unpredictable_nonscop_loop.ll    |  2 +-
 .../test/ScopInfo/unprofitable_scalar-accs.ll |  4 +-
 polly/test/ScopInfo/unsigned-condition.ll     |  2 +-
 polly/test/ScopInfo/unsigned-division-1.ll    |  2 +-
 polly/test/ScopInfo/unsigned-division-2.ll    |  2 +-
 polly/test/ScopInfo/unsigned-division-3.ll    |  2 +-
 polly/test/ScopInfo/unsigned-division-4.ll    |  2 +-
 polly/test/ScopInfo/unsigned-division-5.ll    |  2 +-
 polly/test/ScopInfo/unsigned_wrap_uge.ll      |  2 +-
 polly/test/ScopInfo/unsigned_wrap_ugt.ll      |  2 +-
 polly/test/ScopInfo/unsigned_wrap_ule.ll      |  2 +-
 polly/test/ScopInfo/unsigned_wrap_ult.ll      |  2 +-
 polly/test/ScopInfo/user_context.ll           |  8 +-
 ...ed_assumptions-in-bb-signed-conditional.ll |  4 +-
 .../user_provided_assumptions-in-bb-signed.ll |  2 +-
 ...ser_provided_assumptions-in-bb-unsigned.ll |  4 +-
 .../ScopInfo/user_provided_assumptions.ll     |  4 +-
 .../ScopInfo/user_provided_assumptions_2.ll   |  4 +-
 .../ScopInfo/user_provided_assumptions_3.ll   |  4 +-
 ...ser_provided_non_dominating_assumptions.ll |  4 +-
 polly/test/ScopInfo/variant_base_pointer.ll   |  4 +-
 .../ScopInfo/variant_load_empty_domain.ll     |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_0.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_1.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_2.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_3.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_4.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_5.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_6.ll  |  2 +-
 polly/test/ScopInfo/wraping_signed_expr_7.ll  |  2 +-
 .../ScopInfo/wraping_signed_expr_slow_1.ll    |  2 +-
 .../ScopInfo/wraping_signed_expr_slow_2.ll    |  2 +-
 polly/test/ScopInfo/zero_ext_of_truncate.ll   |  2 +-
 polly/test/ScopInfo/zero_ext_of_truncate_2.ll |  2 +-
 .../test/ScopInfo/zero_ext_space_mismatch.ll  |  2 +-
 polly/test/ScopInliner/invariant-load-func.ll |  2 +-
 polly/test/Simplify/coalesce_3partials.ll     |  2 +-
 .../Simplify/coalesce_disjointelements.ll     |  2 +-
 polly/test/Simplify/coalesce_overlapping.ll   |  2 +-
 polly/test/Simplify/coalesce_partial.ll       |  2 +-
 polly/test/Simplify/dead_access_load.ll       |  3 +-
 polly/test/Simplify/dead_access_phi.ll        |  3 +-
 polly/test/Simplify/dead_access_value.ll      |  3 +-
 polly/test/Simplify/dead_instruction.ll       |  3 +-
 polly/test/Simplify/emptyaccessdomain.ll      |  2 +-
 polly/test/Simplify/exit_phi_accesses-2.ll    |  2 +-
 polly/test/Simplify/func-b320a7.ll            |  2 +-
 polly/test/Simplify/gemm.ll                   |  2 +-
 .../Simplify/nocoalesce_differentvalues.ll    |  2 +-
 .../Simplify/nocoalesce_elementmismatch.ll    |  2 +-
 polly/test/Simplify/nocoalesce_readbetween.ll |  2 +-
 .../test/Simplify/nocoalesce_writebetween.ll  |  2 +-
 polly/test/Simplify/notdead_region_exitphi.ll |  3 +-
 .../test/Simplify/notdead_region_innerphi.ll  |  3 +-
 .../test/Simplify/notredundant_region_loop.ll |  2 +-
 .../Simplify/notredundant_region_middle.ll    |  3 +-
 .../notredundant_synthesizable_unknownit.ll   |  3 +-
 ...ut-of-scop-use-in-region-entry-phi-node.ll |  2 +-
 polly/test/Simplify/overwritten.ll            |  3 +-
 polly/test/Simplify/overwritten_3phi.ll       |  2 +-
 polly/test/Simplify/overwritten_3store.ll     |  3 +-
 .../overwritten_implicit_and_explicit.ll      |  2 +-
 .../test/Simplify/overwritten_loadbetween.ll  |  3 +-
 polly/test/Simplify/overwritten_scalar.ll     |  2 +-
 polly/test/Simplify/pass_existence.ll         |  3 +-
 polly/test/Simplify/phi_in_regionstmt.ll      |  3 +-
 polly/test/Simplify/pr33323.ll                |  2 +-
 polly/test/Simplify/redundant.ll              |  3 +-
 .../test/Simplify/redundant_differentindex.ll |  3 +-
 polly/test/Simplify/redundant_region.ll       |  2 +-
 .../test/Simplify/redundant_region_scalar.ll  |  2 +-
 polly/test/Simplify/redundant_scalarwrite.ll  |  2 +-
 polly/test/Simplify/redundant_storebetween.ll |  3 +-
 polly/test/Simplify/scalability1.ll           |  2 +-
 polly/test/Simplify/scalability2.ll           |  2 +-
 polly/test/Simplify/sweep_mapped_phi.ll       |  2 +-
 polly/test/Simplify/sweep_mapped_value.ll     |  2 +-
 .../Simplify/ununsed_read_in_region_entry.ll  |  4 +-
 polly/test/Support/Plugins.ll                 |  2 +-
 polly/test/Support/defaultpipelines.ll        | 12 +--
 polly/test/Support/dumpfunction.ll            |  4 +-
 polly/test/Support/dumpmodule.ll              |  4 +-
 polly/test/Support/exportjson.ll              |  2 +-
 polly/test/Support/isl-args.ll                |  8 +-
 polly/test/Support/pipelineposition.ll        |  6 +-
 polly/test/Support/pollyDebug.ll              |  2 +-
 polly/test/lit.site.cfg.in                    |  7 +-
 polly/test/polly.ll                           |  2 +-
 1026 files changed, 1472 insertions(+), 1404 deletions(-)

diff --git a/polly/test/CodeGen/20100617.ll b/polly/test/CodeGen/20100617.ll
index 320c48192a8a..71a889f067b8 100644
--- a/polly/test/CodeGen/20100617.ll
+++ b/polly/test/CodeGen/20100617.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @init_array() nounwind {
diff --git a/polly/test/CodeGen/20100622.ll b/polly/test/CodeGen/20100622.ll
index 584107df8971..872d6a0d75cf 100644
--- a/polly/test/CodeGen/20100622.ll
+++ b/polly/test/CodeGen/20100622.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s | not FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | not FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
 
diff --git a/polly/test/CodeGen/20100707.ll b/polly/test/CodeGen/20100707.ll
index 1a4d3556bae8..338198084fc7 100644
--- a/polly/test/CodeGen/20100707.ll
+++ b/polly/test/CodeGen/20100707.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @clause_SetSplitField(i32 %Length) nounwind inlinehint {
diff --git a/polly/test/CodeGen/20100707_2.ll b/polly/test/CodeGen/20100707_2.ll
index 96f329ccbfa9..df784c6d7957 100644
--- a/polly/test/CodeGen/20100707_2.ll
+++ b/polly/test/CodeGen/20100707_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @win193 = external global [4 x [36 x double]], align 32 ; <ptr> [#uses=3]
diff --git a/polly/test/CodeGen/20100708.ll b/polly/test/CodeGen/20100708.ll
index 00fb6fa694c1..50b8e385df53 100644
--- a/polly/test/CodeGen/20100708.ll
+++ b/polly/test/CodeGen/20100708.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' < %s
+; RUN: opt %loadPolly -polly-detect < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define fastcc void @execute() nounwind {
diff --git a/polly/test/CodeGen/20100708_2.ll b/polly/test/CodeGen/20100708_2.ll
index 67f3913d69d4..2f4807d9e4d7 100644
--- a/polly/test/CodeGen/20100708_2.ll
+++ b/polly/test/CodeGen/20100708_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @init_array() nounwind {
diff --git a/polly/test/CodeGen/20100713.ll b/polly/test/CodeGen/20100713.ll
index 73ba0272e5c9..edd352a4c4cc 100644
--- a/polly/test/CodeGen/20100713.ll
+++ b/polly/test/CodeGen/20100713.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @fft_float(i32 %NumSamples) nounwind {
diff --git a/polly/test/CodeGen/20100713_2.ll b/polly/test/CodeGen/20100713_2.ll
index c146e00d4231..92f8959d91d6 100644
--- a/polly/test/CodeGen/20100713_2.ll
+++ b/polly/test/CodeGen/20100713_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define hidden void @luaD_callhook() nounwind {
diff --git a/polly/test/CodeGen/20100717.ll b/polly/test/CodeGen/20100717.ll
index 24114f2c70c6..a400eeaa3370 100644
--- a/polly/test/CodeGen/20100717.ll
+++ b/polly/test/CodeGen/20100717.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly  -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly  -polly-codegen -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @matrixTranspose(ptr %A) nounwind {
diff --git a/polly/test/CodeGen/20100718-DomInfo-2.ll b/polly/test/CodeGen/20100718-DomInfo-2.ll
index 396a1009a415..512b4c5c99af 100644
--- a/polly/test/CodeGen/20100718-DomInfo-2.ll
+++ b/polly/test/CodeGen/20100718-DomInfo-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -verify-dom-info -disable-output < %s
+; RUN: opt %loadPolly -polly-codegen -verify-dom-info -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @getNonAffNeighbour() nounwind {
diff --git a/polly/test/CodeGen/20100718-DomInfo.ll b/polly/test/CodeGen/20100718-DomInfo.ll
index 7a7f4300e107..e12334359c33 100644
--- a/polly/test/CodeGen/20100718-DomInfo.ll
+++ b/polly/test/CodeGen/20100718-DomInfo.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -verify-dom-info -disable-output < %s
+; RUN: opt %loadPolly -polly-codegen -verify-dom-info -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @intrapred_luma_16x16(i32 %predmode) nounwind {
diff --git a/polly/test/CodeGen/20100720-MultipleConditions.ll b/polly/test/CodeGen/20100720-MultipleConditions.ll
index ca3758d95681..9f2268713853 100644
--- a/polly/test/CodeGen/20100720-MultipleConditions.ll
+++ b/polly/test/CodeGen/20100720-MultipleConditions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s
+; RUN: opt %loadPolly -polly-ast -disable-output < %s
 
 ;int bar1();
 ;int bar2();
diff --git a/polly/test/CodeGen/20100809-IndependentBlock.ll b/polly/test/CodeGen/20100809-IndependentBlock.ll
index 849594afa871..8d596689d8ae 100644
--- a/polly/test/CodeGen/20100809-IndependentBlock.ll
+++ b/polly/test/CodeGen/20100809-IndependentBlock.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-codegen -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 define void @cfft2(ptr %x) nounwind {
 entry:
diff --git a/polly/test/CodeGen/20100811-ScalarDependencyBetweenBrAndCnd.ll b/polly/test/CodeGen/20100811-ScalarDependencyBetweenBrAndCnd.ll
index 313dad6fe3f7..261a205560b5 100644
--- a/polly/test/CodeGen/20100811-ScalarDependencyBetweenBrAndCnd.ll
+++ b/polly/test/CodeGen/20100811-ScalarDependencyBetweenBrAndCnd.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-codegen -disable-output < %s
 target datalayout =
 "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/CodeGen/20101030-Overflow.ll b/polly/test/CodeGen/20101030-Overflow.ll
index bf91272399d2..caaa4851f93e 100644
--- a/polly/test/CodeGen/20101030-Overflow.ll
+++ b/polly/test/CodeGen/20101030-Overflow.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @compdecomp() nounwind {
diff --git a/polly/test/CodeGen/20101103-Overflow3.ll b/polly/test/CodeGen/20101103-Overflow3.ll
index bab00231758f..b2faf14fba0b 100644
--- a/polly/test/CodeGen/20101103-Overflow3.ll
+++ b/polly/test/CodeGen/20101103-Overflow3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 define void @Reflection_coefficients(ptr %r) nounwind {
 bb20:
diff --git a/polly/test/CodeGen/20101103-signmissmatch.ll b/polly/test/CodeGen/20101103-signmissmatch.ll
index d12ed4be4cdf..e157d292dc8a 100644
--- a/polly/test/CodeGen/20101103-signmissmatch.ll
+++ b/polly/test/CodeGen/20101103-signmissmatch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @CleanNet() nounwind {
diff --git a/polly/test/CodeGen/20110226-Ignore-Dead-Code.ll b/polly/test/CodeGen/20110226-Ignore-Dead-Code.ll
index 1c82d2aba887..c792d8c3d0bf 100644
--- a/polly/test/CodeGen/20110226-Ignore-Dead-Code.ll
+++ b/polly/test/CodeGen/20110226-Ignore-Dead-Code.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define void @main() nounwind {
diff --git a/polly/test/CodeGen/20110226-PHI-Node-removed.ll b/polly/test/CodeGen/20110226-PHI-Node-removed.ll
index d7003882a776..3458d75c47a0 100644
--- a/polly/test/CodeGen/20110226-PHI-Node-removed.ll
+++ b/polly/test/CodeGen/20110226-PHI-Node-removed.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/CodeGen/20120316-InvalidCast.ll b/polly/test/CodeGen/20120316-InvalidCast.ll
index 14717dd29b23..8355cc51c468 100644
--- a/polly/test/CodeGen/20120316-InvalidCast.ll
+++ b/polly/test/CodeGen/20120316-InvalidCast.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 
 ; CHECK: polly.start
 
diff --git a/polly/test/CodeGen/20120403-RHS-type-mismatch.ll b/polly/test/CodeGen/20120403-RHS-type-mismatch.ll
index 2d3e3b02dd38..1d629e388452 100644
--- a/polly/test/CodeGen/20120403-RHS-type-mismatch.ll
+++ b/polly/test/CodeGen/20120403-RHS-type-mismatch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 
 ; We just check that this compilation does not crash.
 
diff --git a/polly/test/CodeGen/20130221.ll b/polly/test/CodeGen/20130221.ll
index e5f63adabc25..45414671081a 100644
--- a/polly/test/CodeGen/20130221.ll
+++ b/polly/test/CodeGen/20130221.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s
+; RUN: opt %loadPolly -polly-codegen -S < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 define void @list_sequence(ptr %A) {
diff --git a/polly/test/CodeGen/20150328-SCEVExpanderIntroducesNewIV.ll b/polly/test/CodeGen/20150328-SCEVExpanderIntroducesNewIV.ll
index 786adc7286e6..d54be5c3f35f 100644
--- a/polly/test/CodeGen/20150328-SCEVExpanderIntroducesNewIV.ll
+++ b/polly/test/CodeGen/20150328-SCEVExpanderIntroducesNewIV.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/Intrinsics/llvm-expect.ll b/polly/test/CodeGen/Intrinsics/llvm-expect.ll
index ac65f6f439ae..84057e276521 100644
--- a/polly/test/CodeGen/Intrinsics/llvm-expect.ll
+++ b/polly/test/CodeGen/Intrinsics/llvm-expect.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ; Check that we generate code without crashing.
 ;
diff --git a/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll b/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll
index fb02d7f55e23..b04319550938 100644
--- a/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll
+++ b/polly/test/CodeGen/LoopParallelMD/do_not_mutate_debug_info.ll
@@ -1,6 +1,6 @@
 ; This test checks that we do not accidently mutate the debug info when
 ; inserting loop parallel metadata.
-; RUN: opt %loadPolly < %s  -S -polly -passes=polly-codegen -polly-ast-detect-parallel | FileCheck %s
+; RUN: opt %loadPolly < %s  -S -polly -polly-codegen -polly-ast-detect-parallel | FileCheck %s
 ; CHECK-NOT: !7 = !{!7}
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/LoopParallelMD/loop_nest_param_parallel.ll b/polly/test/CodeGen/LoopParallelMD/loop_nest_param_parallel.ll
index c6c27b4a75a8..7b131c5ebcbd 100644
--- a/polly/test/CodeGen/LoopParallelMD/loop_nest_param_parallel.ll
+++ b/polly/test/CodeGen/LoopParallelMD/loop_nest_param_parallel.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-ast-detect-parallel -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-ast-detect-parallel -S < %s | FileCheck %s
 ;
 ; Check that we mark multiple parallel loops correctly including the memory instructions.
 ;
diff --git a/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll b/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll
index f91bd64a895a..ec927acb1ec7 100644
--- a/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll
+++ b/polly/test/CodeGen/LoopParallelMD/single_loop_param_parallel.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=SEQUENTIAL
-; RUN: opt %loadPolly -passes=polly-codegen -polly-ast-detect-parallel -S < %s | FileCheck %s -check-prefix=PARALLEL
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=SEQUENTIAL
+; RUN: opt %loadPolly -polly-codegen -polly-ast-detect-parallel -S < %s | FileCheck %s -check-prefix=PARALLEL
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ; This is a trivially parallel loop. We just use it to ensure that we actually
diff --git a/polly/test/CodeGen/MemAccess/bad_alignment.ll b/polly/test/CodeGen/MemAccess/bad_alignment.ll
index 2f297384ccdb..32f3cfe963b7 100644
--- a/polly/test/CodeGen/MemAccess/bad_alignment.ll
+++ b/polly/test/CodeGen/MemAccess/bad_alignment.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -passes=polly-import-jscop -disable-output 2>&1 < %s | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -disable-output 2>&1 < %s | FileCheck %s
 ;
 ; Check that we do not allow to access elements not accessed before because the
 ; alignment information would become invalid.
diff --git a/polly/test/CodeGen/MemAccess/codegen_address_space.ll b/polly/test/CodeGen/MemAccess/codegen_address_space.ll
index 986775ed5717..7c9b12d64f9c 100644
--- a/polly/test/CodeGen/MemAccess/codegen_address_space.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_address_space.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed < %s -S | FileCheck %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
 
 ;int A[100];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll b/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll
index b4168b89267b..e008a789fe7d 100644
--- a/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed < %s -S | FileCheck %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
 
 ;int A[100];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_simple.ll b/polly/test/CodeGen/MemAccess/codegen_simple.ll
index cf2a57ad8a1f..5ba6f3269fb9 100644
--- a/polly/test/CodeGen/MemAccess/codegen_simple.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_simple.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed < %s -S | FileCheck %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
 
 ;int A[100];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_simple_float.ll b/polly/test/CodeGen/MemAccess/codegen_simple_float.ll
index bdd0e56b0a29..cf8913fc5197 100644
--- a/polly/test/CodeGen/MemAccess/codegen_simple_float.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_simple_float.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed < %s -S | FileCheck %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
 ;
 ;float A[100];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_simple_md.ll b/polly/test/CodeGen/MemAccess/codegen_simple_md.ll
index 8f676725146d..e4afcc8d2243 100644
--- a/polly/test/CodeGen/MemAccess/codegen_simple_md.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_simple_md.ll
@@ -1,5 +1,5 @@
-;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed+withconst < %s -S | FileCheck -check-prefix=WITHCONST %s
-;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed+withoutconst < %s -S | FileCheck -check-prefix=WITHOUTCONST %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed+withconst -polly-codegen < %s -S | FileCheck -check-prefix=WITHCONST %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed+withoutconst -polly-codegen < %s -S | FileCheck -check-prefix=WITHOUTCONST %s
 
 ;int A[1040];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_simple_md_float.ll b/polly/test/CodeGen/MemAccess/codegen_simple_md_float.ll
index 1ac1efa2f727..c9913f3ed873 100644
--- a/polly/test/CodeGen/MemAccess/codegen_simple_md_float.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_simple_md_float.ll
@@ -1,5 +1,5 @@
-;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed+withconst < %s -S | FileCheck -check-prefix=WITHCONST %s
-;RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed+withoutconst < %s -S | FileCheck -check-prefix=WITHOUTCONST %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed+withconst -polly-codegen < %s -S | FileCheck -check-prefix=WITHCONST %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed+withoutconst -polly-codegen < %s -S | FileCheck -check-prefix=WITHOUTCONST %s
 ;
 ;float A[1040];
 ;
diff --git a/polly/test/CodeGen/MemAccess/different_types.ll b/polly/test/CodeGen/MemAccess/different_types.ll
index 52fca9d87759..624de62911ff 100644
--- a/polly/test/CodeGen/MemAccess/different_types.ll
+++ b/polly/test/CodeGen/MemAccess/different_types.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
+; RUN: opt %loadPolly -polly-import-jscop \
 ; RUN: \
-; RUN: -S < %s | FileCheck %s
+; RUN: -polly-codegen -S < %s | FileCheck %s
 ;
 ;    void foo(float A[], float B[]) {
 ;      for (long i = 0; i < 100; i++)
diff --git a/polly/test/CodeGen/MemAccess/generate-all.ll b/polly/test/CodeGen/MemAccess/generate-all.ll
index a64c6db0978c..6f92ba13587e 100644
--- a/polly/test/CodeGen/MemAccess/generate-all.ll
+++ b/polly/test/CodeGen/MemAccess/generate-all.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-codegen-generate-expressions=false \
+; RUN: opt %loadPolly -polly-codegen -polly-codegen-generate-expressions=false \
 ; RUN:     -S < %s | FileCheck %s -check-prefix=SCEV
-; RUN: opt %loadPolly -passes=polly-codegen -polly-codegen-generate-expressions=true \
+; RUN: opt %loadPolly -polly-codegen -polly-codegen-generate-expressions=true \
 ; RUN:     -S < %s | FileCheck %s -check-prefix=ASTEXPR
 ;
 ;    void foo(float A[]) {
diff --git a/polly/test/CodeGen/MemAccess/invariant_base_ptr.ll b/polly/test/CodeGen/MemAccess/invariant_base_ptr.ll
index 12c38b7a66c4..a6d1de0aac63 100644
--- a/polly/test/CodeGen/MemAccess/invariant_base_ptr.ll
+++ b/polly/test/CodeGen/MemAccess/invariant_base_ptr.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
-; RUN:   -polly-invariant-load-hoisting -S \
+; RUN: opt %loadPolly -polly-import-jscop \
+; RUN:   -polly-codegen -polly-invariant-load-hoisting -S \
 ; RUN:   2>&1 < %s | FileCheck %s
 
 ; Setting new access functions where the base pointer of the array that is newly
diff --git a/polly/test/CodeGen/MemAccess/multiple_types.ll b/polly/test/CodeGen/MemAccess/multiple_types.ll
index 7c0ddffbd6d1..1793bd30fc5b 100644
--- a/polly/test/CodeGen/MemAccess/multiple_types.ll
+++ b/polly/test/CodeGen/MemAccess/multiple_types.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,polly-codegen' \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop \
 ; RUN: -polly-allow-differing-element-types \
-; RUN:   -S    < %s | FileCheck %s
+; RUN:   -polly-codegen -S    < %s | FileCheck %s
 ;
 ;    // Check that accessing one array with different types works.
 ;    void multiple_types(char *Short, char *Float, char *Double) {
diff --git a/polly/test/CodeGen/MemAccess/simple.ll b/polly/test/CodeGen/MemAccess/simple.ll
index 8964e189b8a7..39e8a2c91b79 100644
--- a/polly/test/CodeGen/MemAccess/simple.ll
+++ b/polly/test/CodeGen/MemAccess/simple.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -stats < %s 2>&1  | FileCheck %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -stats < %s 2>&1  | FileCheck %s
 ; REQUIRES: asserts
 
 ;int A[100];
diff --git a/polly/test/CodeGen/MemAccess/update_access_functions.ll b/polly/test/CodeGen/MemAccess/update_access_functions.ll
index 00644c52ecd7..05d208708a36 100644
--- a/polly/test/CodeGen/MemAccess/update_access_functions.ll
+++ b/polly/test/CodeGen/MemAccess/update_access_functions.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
-; RUN:                -polly-import-jscop-postfix=transformed \
+; RUN: opt %loadPolly -polly-import-jscop \
+; RUN:                -polly-import-jscop-postfix=transformed -polly-codegen \
 ; RUN:                 < %s -S | FileCheck %s
 
 ; CHECK-LABEL: polly.stmt.loop1:
diff --git a/polly/test/CodeGen/OpenMP/alias-metadata.ll b/polly/test/CodeGen/OpenMP/alias-metadata.ll
index e7ca6abac283..07d79631b2cb 100644
--- a/polly/test/CodeGen/OpenMP/alias-metadata.ll
+++ b/polly/test/CodeGen/OpenMP/alias-metadata.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-parallel -S < %s | FileCheck %s
 ;
 ;    void foo(float *A, float *B) {
 ;      for (long i = 0; i < 1000; i++)
diff --git a/polly/test/CodeGen/OpenMP/floord-as-argument-to-subfunction.ll b/polly/test/CodeGen/OpenMP/floord-as-argument-to-subfunction.ll
index 40ea62088940..eb9dfcd9e920 100644
--- a/polly/test/CodeGen/OpenMP/floord-as-argument-to-subfunction.ll
+++ b/polly/test/CodeGen/OpenMP/floord-as-argument-to-subfunction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-opt-max-coefficient=-1 -polly-parallel -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-opt-isl -polly-opt-max-coefficient=-1 -polly-parallel -polly-codegen -S < %s | FileCheck %s
 ;
 ; Check that we do not crash but generate parallel code
 ;
diff --git a/polly/test/CodeGen/OpenMP/inlineasm.ll b/polly/test/CodeGen/OpenMP/inlineasm.ll
index a2ca2f79d649..69b1b0aa53f3 100644
--- a/polly/test/CodeGen/OpenMP/inlineasm.ll
+++ b/polly/test/CodeGen/OpenMP/inlineasm.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-opt-isl,polly-codegen' -polly-parallel -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-opt-isl -polly-parallel -polly-codegen -S < %s | FileCheck %s
 ; llvm.org/PR51960
 
 ; CHECK-LABEL: define internal void @foo_polly_subfn
diff --git a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded.ll b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded.ll
index 9394755254a7..30beef5b0709 100644
--- a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded.ll
+++ b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
 ; RUN: -polly-parallel-force -S < %s | FileCheck %s
 ;
 ; Test to verify that we hand down the preloaded A[0] to the OpenMP subfunction.
diff --git a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_different_bb.ll b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_different_bb.ll
index 48f075c37d82..fe5d2ab8c96d 100644
--- a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_different_bb.ll
+++ b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_different_bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
 ; RUN: -polly-parallel-force -S < %s | FileCheck %s
 ;
 ; Test to verify that we hand down the preloaded A[0] to the OpenMP subfunction.
diff --git a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_pass_only_needed.ll b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_pass_only_needed.ll
index e4273384dd72..49b9321c40b8 100644
--- a/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_pass_only_needed.ll
+++ b/polly/test/CodeGen/OpenMP/invariant_base_pointer_preloaded_pass_only_needed.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
 ; RUN: -polly-parallel-force -S < %s | FileCheck %s
 ;
 ; Test to verify that we hand down the preloaded A[0] to the OpenMP subfunction but
diff --git a/polly/test/CodeGen/OpenMP/invariant_base_pointers_preloaded.ll b/polly/test/CodeGen/OpenMP/invariant_base_pointers_preloaded.ll
index 8400a5594d10..06c4cdab45f1 100644
--- a/polly/test/CodeGen/OpenMP/invariant_base_pointers_preloaded.ll
+++ b/polly/test/CodeGen/OpenMP/invariant_base_pointers_preloaded.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -polly-parallel \
 ; RUN: -polly-parallel-force -S < %s | FileCheck %s
 ;
 ; Test to verify that we hand down the preloaded A[0] to the OpenMP subfunction.
diff --git a/polly/test/CodeGen/OpenMP/loop-body-references-outer-iv.ll b/polly/test/CodeGen/OpenMP/loop-body-references-outer-iv.ll
index b3297b5b32af..db58c3ab7593 100644
--- a/polly/test/CodeGen/OpenMP/loop-body-references-outer-iv.ll
+++ b/polly/test/CodeGen/OpenMP/loop-body-references-outer-iv.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; This code has failed the scev based code generation as the scev in the scop
 ; contains an AddRecExpr of an outer loop. When generating code, we did not
diff --git a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-2.ll b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-2.ll
index c478e91eb9db..c2ddc1e26496 100644
--- a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-2.ll
+++ b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; AST: #pragma simd
 ; AST: #pragma omp parallel for
diff --git a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-3.ll b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-3.ll
index 26c2fe6da6ca..0f025bb94112 100644
--- a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-3.ll
+++ b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values-3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -basic-aa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -basic-aa -polly-parallel -polly-parallel-force -polly-invariant-load-hoisting=true -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; The interesting part of this test case is the instruction:
 ;   %tmp = bitcast i8* %call to i64**
diff --git a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values.ll b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values.ll
index debd89ab151d..f9612d77533d 100644
--- a/polly/test/CodeGen/OpenMP/loop-body-references-outer-values.ll
+++ b/polly/test/CodeGen/OpenMP/loop-body-references-outer-values.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S < %s | FileCheck %s -check-prefix=IR
 
 ; Make sure we correctly forward the reference to 'A' to the OpenMP subfunction.
 ;
diff --git a/polly/test/CodeGen/OpenMP/loop-bounds-reference-outer-ids.ll b/polly/test/CodeGen/OpenMP/loop-bounds-reference-outer-ids.ll
index db2299e5e73d..da9da18c89b2 100644
--- a/polly/test/CodeGen/OpenMP/loop-bounds-reference-outer-ids.ll
+++ b/polly/test/CodeGen/OpenMP/loop-bounds-reference-outer-ids.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-codegen -S < %s | FileCheck %s -check-prefix=IR
 ;
 ; float A[100];
 ;
diff --git a/polly/test/CodeGen/OpenMP/mapped-phi-access.ll b/polly/test/CodeGen/OpenMP/mapped-phi-access.ll
index 4b71760ea224..1b8433693abf 100644
--- a/polly/test/CodeGen/OpenMP/mapped-phi-access.ll
+++ b/polly/test/CodeGen/OpenMP/mapped-phi-access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-parallel '-passes=polly-delicm,polly-codegen' -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-parallel -polly-delicm -polly-codegen -S < %s | FileCheck %s
 ;
 ; Verify that -polly-parallel can handle mapped scalar MemoryAccesses.
 ;
diff --git a/polly/test/CodeGen/OpenMP/matmul-parallel.ll b/polly/test/CodeGen/OpenMP/matmul-parallel.ll
index 1f3ad5ca8426..5ee9a7c7a824 100644
--- a/polly/test/CodeGen/OpenMP/matmul-parallel.ll
+++ b/polly/test/CodeGen/OpenMP/matmul-parallel.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel '-passes=polly-opt-isl,print<polly-ast>' -disable-output -debug-only=polly-ast < %s 2>&1 | FileCheck --check-prefix=AST %s
-; RUN: opt %loadPolly -polly-parallel '-passes=polly-opt-isl,polly-codegen' -S < %s | FileCheck --check-prefix=CODEGEN %s
+; RUN: opt %loadPolly -polly-parallel -polly-opt-isl -polly-ast -disable-output -debug-only=polly-ast < %s 2>&1 | FileCheck --check-prefix=AST %s
+; RUN: opt %loadPolly -polly-parallel -polly-opt-isl -polly-codegen -S < %s | FileCheck --check-prefix=CODEGEN %s
 ; REQUIRES: asserts
 
 ; Parallelization of detected matrix-multiplication.
diff --git a/polly/test/CodeGen/OpenMP/recomputed-srem.ll b/polly/test/CodeGen/OpenMP/recomputed-srem.ll
index 3411308a4b11..cfae8e943cf1 100644
--- a/polly/test/CodeGen/OpenMP/recomputed-srem.ll
+++ b/polly/test/CodeGen/OpenMP/recomputed-srem.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -passes=polly-codegen -polly-parallel \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-codegen -polly-parallel \
 ; RUN: -polly-parallel-force -S < %s | FileCheck %s
 ;
 ; Test to verify that we pass %rem96 to the parallel subfunction.
diff --git a/polly/test/CodeGen/OpenMP/reference-argument-from-non-affine-region.ll b/polly/test/CodeGen/OpenMP/reference-argument-from-non-affine-region.ll
index 247e89be0027..f243c3a04949 100644
--- a/polly/test/CodeGen/OpenMP/reference-argument-from-non-affine-region.ll
+++ b/polly/test/CodeGen/OpenMP/reference-argument-from-non-affine-region.ll
@@ -1,15 +1,15 @@
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -passes=polly-codegen \
+; RUN: -polly-parallel-force -polly-codegen \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=IR
 
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -passes=polly-codegen -polly-scheduling=runtime \
+; RUN: -polly-parallel-force -polly-codegen -polly-scheduling=runtime \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=IR
 
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM \
+; RUN: -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=LIBOMP-IR
 
diff --git a/polly/test/CodeGen/OpenMP/reference-other-bb.ll b/polly/test/CodeGen/OpenMP/reference-other-bb.ll
index 2c399f12af24..b7abdc23d258 100644
--- a/polly/test/CodeGen/OpenMP/reference-other-bb.ll
+++ b/polly/test/CodeGen/OpenMP/reference-other-bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; IR: @foo_polly_subfn
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/CodeGen/OpenMP/reference-preceeding-loop.ll b/polly/test/CodeGen/OpenMP/reference-preceeding-loop.ll
index 90a95dccbcb1..b88589f39a6f 100644
--- a/polly/test/CodeGen/OpenMP/reference-preceeding-loop.ll
+++ b/polly/test/CodeGen/OpenMP/reference-preceeding-loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 
 ; - Test the case where scalar evolution references a loop that is outside
diff --git a/polly/test/CodeGen/OpenMP/reference_latest.ll b/polly/test/CodeGen/OpenMP/reference_latest.ll
index 696c3c74ad0e..54875c2630f0 100644
--- a/polly/test/CodeGen/OpenMP/reference_latest.ll
+++ b/polly/test/CodeGen/OpenMP/reference_latest.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-delicm,polly-simplify,polly-codegen' -polly-parallel -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-delicm -polly-simplify -polly-parallel -polly-codegen -S < %s | FileCheck %s
 ;
 ; Test that parallel codegen handles scalars mapped to other arrays.
 ; After mapping "store double %add10" references the array "MemRef2".
diff --git a/polly/test/CodeGen/OpenMP/scev-rewriting.ll b/polly/test/CodeGen/OpenMP/scev-rewriting.ll
index 551946fc698b..1b229fc19d25 100644
--- a/polly/test/CodeGen/OpenMP/scev-rewriting.ll
+++ b/polly/test/CodeGen/OpenMP/scev-rewriting.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly < %s -polly-vectorizer=stripmine -polly-parallel -polly-parallel-force -polly-process-unprofitable -passes=polly-codegen -S | FileCheck %s
+; RUN: opt %loadPolly < %s -polly-vectorizer=stripmine -polly-parallel -polly-parallel-force -polly-process-unprofitable -polly-codegen -S | FileCheck %s
 ; CHECK: define internal void @DoStringSort_polly_subfn
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnueabi"
diff --git a/polly/test/CodeGen/OpenMP/single_loop.ll b/polly/test/CodeGen/OpenMP/single_loop.ll
index 7e45ab08080e..f79653a08d21 100644
--- a/polly/test/CodeGen/OpenMP/single_loop.ll
+++ b/polly/test/CodeGen/OpenMP/single_loop.ll
@@ -1,14 +1,14 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=polly-import-jscop,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST-STRIDE4
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=polly-import-jscop,polly-codegen' -S < %s | FileCheck %s -check-prefix=IR-STRIDE4
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-import-jscop -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST-STRIDE4
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-import-jscop -polly-codegen -S < %s | FileCheck %s -check-prefix=IR-STRIDE4
 
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM -polly-scheduling=static -polly-scheduling-chunksize=43 -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-STATIC-CHUNKED
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM -polly-scheduling=static -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-STATIC
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM -polly-scheduling=dynamic -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-DYNAMIC
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM -polly-scheduling=dynamic -polly-scheduling-chunksize=4 -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-DYNAMIC-FOUR
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=polly-import-jscop,polly-codegen' -polly-omp-backend=LLVM -S < %s | FileCheck %s -check-prefix=LIBOMP-IR-STRIDE4
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=static -polly-scheduling-chunksize=43 -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-STATIC-CHUNKED
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=static -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-STATIC
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=dynamic -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-DYNAMIC
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=dynamic -polly-scheduling-chunksize=4 -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-DYNAMIC-FOUR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-import-jscop -polly-codegen -polly-omp-backend=LLVM -S < %s | FileCheck %s -check-prefix=LIBOMP-IR-STRIDE4
 
 ; This extensive test case tests the creation of the full set of OpenMP calls
 ; as well as the subfunction creation using a trivial loop as example.
diff --git a/polly/test/CodeGen/OpenMP/single_loop_with_loop_invariant_baseptr.ll b/polly/test/CodeGen/OpenMP/single_loop_with_loop_invariant_baseptr.ll
index 519cbbc496b1..50da5dd2b7c0 100644
--- a/polly/test/CodeGen/OpenMP/single_loop_with_loop_invariant_baseptr.ll
+++ b/polly/test/CodeGen/OpenMP/single_loop_with_loop_invariant_baseptr.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=tbaa -polly-parallel -polly-parallel-force -polly-parallel-force -polly-invariant-load-hoisting=true '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -aa-pipeline=tbaa -polly-parallel -polly-parallel-force -polly-parallel-force -polly-invariant-load-hoisting=true -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -tbaa -polly-parallel -polly-parallel-force -polly-parallel-force -polly-invariant-load-hoisting=true -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -tbaa -polly-parallel -polly-parallel-force -polly-parallel-force -polly-invariant-load-hoisting=true -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; #define N 1024
 ; float A[N];
diff --git a/polly/test/CodeGen/OpenMP/single_loop_with_param.ll b/polly/test/CodeGen/OpenMP/single_loop_with_param.ll
index 0288d4c8f5e4..d01b7a2fdcad 100644
--- a/polly/test/CodeGen/OpenMP/single_loop_with_param.ll
+++ b/polly/test/CodeGen/OpenMP/single_loop_with_param.ll
@@ -1,15 +1,15 @@
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -passes=polly-codegen \
+; RUN: -polly-parallel-force -polly-codegen \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=IR
 
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM \
+; RUN: -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=LIBOMP-IR
 
 ; RUN: opt %loadPolly -polly-parallel \
-; RUN: -polly-parallel-force -passes=polly-codegen -polly-omp-backend=LLVM \
+; RUN: -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM \
 ; RUN: -polly-scheduling=static \
 ; RUN: -S -verify-dom-info < %s \
 ; RUN: | FileCheck %s -check-prefix=LIBOMP-STATIC-IR
diff --git a/polly/test/CodeGen/OpenMP/two-parallel-loops-reference-outer-indvar.ll b/polly/test/CodeGen/OpenMP/two-parallel-loops-reference-outer-indvar.ll
index 133a00c3be71..05c6ed177e9c 100644
--- a/polly/test/CodeGen/OpenMP/two-parallel-loops-reference-outer-indvar.ll
+++ b/polly/test/CodeGen/OpenMP/two-parallel-loops-reference-outer-indvar.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=AST
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=AST
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -S -verify-dom-info < %s | FileCheck %s -check-prefix=IR
 
 ; This test case verifies that we create correct code even if two OpenMP loops
 ; share common outer variables.
diff --git a/polly/test/CodeGen/PHIInExit.ll b/polly/test/CodeGen/PHIInExit.ll
index 5617d873e529..eadd6054386b 100644
--- a/polly/test/CodeGen/PHIInExit.ll
+++ b/polly/test/CodeGen/PHIInExit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 %struct..0__pthread_mutex_s = type { i32, i32, i32, i32, i32, i32, %struct.__pthread_list_t }
diff --git a/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll b/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll
index c7f9186c3777..84827dd26049 100644
--- a/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll
+++ b/polly/test/CodeGen/RuntimeDebugBuilder/combine_different_values.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S \
+; RUN: opt %loadPolly -polly-codegen -S \
 ; RUN: -polly-codegen-add-debug-printing \
 ; RUN: -polly-ignore-aliasing < %s | FileCheck %s
 
diff --git a/polly/test/CodeGen/RuntimeDebugBuilder/stmt_tracing.ll b/polly/test/CodeGen/RuntimeDebugBuilder/stmt_tracing.ll
index 80fa3ee8d0f2..822eccc306ef 100644
--- a/polly/test/CodeGen/RuntimeDebugBuilder/stmt_tracing.ll
+++ b/polly/test/CodeGen/RuntimeDebugBuilder/stmt_tracing.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-codegen-trace-stmts -polly-codegen-trace-scalars -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen-trace-stmts -polly-codegen-trace-scalars -polly-codegen -S < %s | FileCheck %s
 ;
 
 define void @func(i32 %n, ptr %A) {
diff --git a/polly/test/CodeGen/alias-check-multi-dim.ll b/polly/test/CodeGen/alias-check-multi-dim.ll
index 821e19290612..d923a4cc14fd 100644
--- a/polly/test/CodeGen/alias-check-multi-dim.ll
+++ b/polly/test/CodeGen/alias-check-multi-dim.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen \
+; RUN: opt %loadPolly -polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/alias_metadata_too_many_arrays.ll b/polly/test/CodeGen/alias_metadata_too_many_arrays.ll
index 9207f3015e3a..7c5ca012a378 100644
--- a/polly/test/CodeGen/alias_metadata_too_many_arrays.ll
+++ b/polly/test/CodeGen/alias_metadata_too_many_arrays.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-ignore-aliasing -S < %s \
+; RUN: opt %loadPolly -polly-codegen -polly-ignore-aliasing -S < %s \
 ; RUN:   | FileCheck %s
 ;
 ;    void manyarrays(float A1[], float A2[], float A3[], float A4[], float A5[],
diff --git a/polly/test/CodeGen/aliasing_different_base_and_access_type.ll b/polly/test/CodeGen/aliasing_different_base_and_access_type.ll
index d74f51702f36..a087414b8403 100644
--- a/polly/test/CodeGen/aliasing_different_base_and_access_type.ll
+++ b/polly/test/CodeGen/aliasing_different_base_and_access_type.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ; We have to cast %B to "short *" before we create RTCs.
 ;
diff --git a/polly/test/CodeGen/aliasing_different_pointer_types.ll b/polly/test/CodeGen/aliasing_different_pointer_types.ll
index 5ba1d1b587bf..91f5eab6b2a6 100644
--- a/polly/test/CodeGen/aliasing_different_pointer_types.ll
+++ b/polly/test/CodeGen/aliasing_different_pointer_types.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; Check that we cast the different pointer types correctly before we compare
 ; them in the RTC's. We use i8* as max pointer type.
diff --git a/polly/test/CodeGen/aliasing_multidimensional_access.ll b/polly/test/CodeGen/aliasing_multidimensional_access.ll
index 338ab05e4ad3..48768399e850 100644
--- a/polly/test/CodeGen/aliasing_multidimensional_access.ll
+++ b/polly/test/CodeGen/aliasing_multidimensional_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ; Check that we calculate the maximal access into array A correctly and track the overflow state.
 ;
diff --git a/polly/test/CodeGen/aliasing_parametric_simple_1.ll b/polly/test/CodeGen/aliasing_parametric_simple_1.ll
index 281ecf488fd4..5422da4426e9 100644
--- a/polly/test/CodeGen/aliasing_parametric_simple_1.ll
+++ b/polly/test/CodeGen/aliasing_parametric_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ;    void jd(int *A, int *B, int c) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/CodeGen/aliasing_parametric_simple_2.ll b/polly/test/CodeGen/aliasing_parametric_simple_2.ll
index 9ac59f93febb..de945d403f92 100644
--- a/polly/test/CodeGen/aliasing_parametric_simple_2.ll
+++ b/polly/test/CodeGen/aliasing_parametric_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ;    void jd(int *A, int *B, int c) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/CodeGen/aliasing_struct_element.ll b/polly/test/CodeGen/aliasing_struct_element.ll
index 32b7c1ac905c..2219ca9d28bb 100644
--- a/polly/test/CodeGen/aliasing_struct_element.ll
+++ b/polly/test/CodeGen/aliasing_struct_element.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ; We should only access (or compute the address of) "the first element" of %S
 ; as it is a single struct not a struct array. The maximal access to S, thus
diff --git a/polly/test/CodeGen/alignment.ll b/polly/test/CodeGen/alignment.ll
index f3c786780f0a..a94b1f7e2883 100644
--- a/polly/test/CodeGen/alignment.ll
+++ b/polly/test/CodeGen/alignment.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; Check that the special alignment information is kept
 ;
diff --git a/polly/test/CodeGen/annotated_alias_scopes.ll b/polly/test/CodeGen/annotated_alias_scopes.ll
index 1dd409edc9a5..f8d14cd34b62 100644
--- a/polly/test/CodeGen/annotated_alias_scopes.ll
+++ b/polly/test/CodeGen/annotated_alias_scopes.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s --check-prefix=SCOPES
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s --check-prefix=SCOPES
 ;
 ; Check that we create alias scopes that indicate the accesses to A, B and C cannot alias in any way.
 ;
diff --git a/polly/test/CodeGen/blas_sscal_simplified.ll b/polly/test/CodeGen/blas_sscal_simplified.ll
index b2072d3822a2..a370fcff46f8 100644
--- a/polly/test/CodeGen/blas_sscal_simplified.ll
+++ b/polly/test/CodeGen/blas_sscal_simplified.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 ;
 ; Regression test for a bug in the runtime check generation.
 
diff --git a/polly/test/CodeGen/conflict-between-loop-invariant-code-hosting-and-escape-map-computation.ll b/polly/test/CodeGen/conflict-between-loop-invariant-code-hosting-and-escape-map-computation.ll
index e977627ed7be..e0f8c435879a 100644
--- a/polly/test/CodeGen/conflict-between-loop-invariant-code-hosting-and-escape-map-computation.ll
+++ b/polly/test/CodeGen/conflict-between-loop-invariant-code-hosting-and-escape-map-computation.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-codegen -disable-output < %s
 ;
 ; CHECK: store i32 %tmp14_p_scalar_, ptr %tmp14.s2a
 ; CHECK: %tmp14.final_reload = load i32, ptr %tmp14.s2a
diff --git a/polly/test/CodeGen/constant_condition.ll b/polly/test/CodeGen/constant_condition.ll
index e259b5799763..dad1f6cffd17 100644
--- a/polly/test/CodeGen/constant_condition.ll
+++ b/polly/test/CodeGen/constant_condition.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly '-passes=polly-prepare,scop(print<polly-ast>)' -disable-output < %s 2>&1 | FileCheck %s
+;RUN: opt %loadPolly -polly-prepare -polly-print-ast -disable-output < %s | FileCheck %s
 
 ;#include <string.h>
 ;int A[1];
diff --git a/polly/test/CodeGen/create-conditional-scop.ll b/polly/test/CodeGen/create-conditional-scop.ll
index 235726fdaf15..f51a2dcc9b3c 100644
--- a/polly/test/CodeGen/create-conditional-scop.ll
+++ b/polly/test/CodeGen/create-conditional-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -verify-loop-info < %s -S | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -verify-loop-info < %s -S | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
 
diff --git a/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_1.ll b/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_1.ll
index 220bdc179158..991e3c83eef1 100644
--- a/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_1.ll
+++ b/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 ;
 ; Check we do not crash even though the dead %tmp8 is referenced by a parameter
 ; and we do not pre-load it (as it is dead).
diff --git a/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_2.ll b/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_2.ll
index 4830de888237..153f6912cea5 100644
--- a/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_2.ll
+++ b/polly/test/CodeGen/dead_invariant_load_instruction_referenced_by_parameter_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 ;
 ; Check we do not crash even though there is a dead load that is referenced by
 ; a parameter and we do not pre-load it (as it is dead).
diff --git a/polly/test/CodeGen/debug-intrinsics.ll b/polly/test/CodeGen/debug-intrinsics.ll
index c98fae8e2e10..2feeb7c838b0 100644
--- a/polly/test/CodeGen/debug-intrinsics.ll
+++ b/polly/test/CodeGen/debug-intrinsics.ll
@@ -1,9 +1,9 @@
 ; RUN: opt %loadPolly \
-; RUN: -polly-analyze-read-only-scalars=false -passes=polly-codegen -S < %s | \
+; RUN: -polly-analyze-read-only-scalars=false -polly-codegen -S < %s | \
 ; RUN: FileCheck %s
 
 ; RUN: opt %loadPolly \
-; RUN: -polly-analyze-read-only-scalars=true -passes=polly-codegen -S < %s | \
+; RUN: -polly-analyze-read-only-scalars=true -polly-codegen -S < %s | \
 ; RUN: FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/CodeGen/dominance_problem_after_early_codegen_bailout.ll b/polly/test/CodeGen/dominance_problem_after_early_codegen_bailout.ll
index 2a3f3bd8a065..c9e006a01204 100644
--- a/polly/test/CodeGen/dominance_problem_after_early_codegen_bailout.ll
+++ b/polly/test/CodeGen/dominance_problem_after_early_codegen_bailout.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-codegen -disable-output < %s
 ;
 ; This caused dominance problems at some point as we do bail out during
 ; code generation. Just verify it runs through.
diff --git a/polly/test/CodeGen/empty_domain_in_context.ll b/polly/test/CodeGen/empty_domain_in_context.ll
index b7a6b95171cb..c67ace9502e1 100644
--- a/polly/test/CodeGen/empty_domain_in_context.ll
+++ b/polly/test/CodeGen/empty_domain_in_context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-optree,polly-opt-isl,polly-codegen' -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-optree -polly-opt-isl -polly-codegen -S < %s | FileCheck %s
 ;
 ; llvm.org/PR35362
 ; isl codegen does not allow to generate isl_ast_expr from pw_aff which have an
diff --git a/polly/test/CodeGen/entry_with_trivial_phi.ll b/polly/test/CodeGen/entry_with_trivial_phi.ll
index 99d0776f3aa9..b057690ab29b 100644
--- a/polly/test/CodeGen/entry_with_trivial_phi.ll
+++ b/polly/test/CodeGen/entry_with_trivial_phi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s
+; RUN: opt %loadPolly -polly-codegen -S < %s
 ;
 ; The entry of this scop's simple region (entry.split => for.end) has an trivial
 ; PHI node. LCSSA may create such PHI nodes. This is a breakdown of this case in
diff --git a/polly/test/CodeGen/entry_with_trivial_phi_other_bb.ll b/polly/test/CodeGen/entry_with_trivial_phi_other_bb.ll
index ec3967b51049..5673cc746b5f 100644
--- a/polly/test/CodeGen/entry_with_trivial_phi_other_bb.ll
+++ b/polly/test/CodeGen/entry_with_trivial_phi_other_bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; The entry of this scop's simple region (entry.split => for.end) has an trivial
 ; PHI node that is used in a different of the scop region. LCSSA may create such
diff --git a/polly/test/CodeGen/error-stmt-in-non-affine-region.ll b/polly/test/CodeGen/error-stmt-in-non-affine-region.ll
index ab85fbabd3f8..9832afe7a5fd 100644
--- a/polly/test/CodeGen/error-stmt-in-non-affine-region.ll
+++ b/polly/test/CodeGen/error-stmt-in-non-affine-region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ; XFAIL: *
 ;
 ; CHECK-LABEL: polly.stmt.if.then:
diff --git a/polly/test/CodeGen/error_block_contains_invalid_memory_access.ll b/polly/test/CodeGen/error_block_contains_invalid_memory_access.ll
index 6d1c3f74cc03..048847f3e322 100644
--- a/polly/test/CodeGen/error_block_contains_invalid_memory_access.ll
+++ b/polly/test/CodeGen/error_block_contains_invalid_memory_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/exprModDiv.ll b/polly/test/CodeGen/exprModDiv.ll
index 625e0e6464d7..936b018bc1ad 100644
--- a/polly/test/CodeGen/exprModDiv.ll
+++ b/polly/test/CodeGen/exprModDiv.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
-; RUN:     -S < %s | FileCheck %s
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
-; RUN:     -polly-import-jscop-postfix=pow2 \
+; RUN: opt %loadPolly -polly-import-jscop \
+; RUN:     -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop \
+; RUN:     -polly-codegen -polly-import-jscop-postfix=pow2 \
 ; RUN:     -S < %s | FileCheck %s -check-prefix=POW2
 ;
 ;    void exprModDiv(float *A, float *B, float *C, long N, long p) {
diff --git a/polly/test/CodeGen/hoisted_load_escapes_through_phi.ll b/polly/test/CodeGen/hoisted_load_escapes_through_phi.ll
index ad7d84648a09..d7588b3b8e00 100644
--- a/polly/test/CodeGen/hoisted_load_escapes_through_phi.ll
+++ b/polly/test/CodeGen/hoisted_load_escapes_through_phi.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen \
+; RUN: opt %loadPolly -S -polly-codegen \
 ; RUN: -polly-invariant-load-hoisting=false < %s | FileCheck %s
-; RUN: opt %loadPolly -S -passes=polly-codegen \
+; RUN: opt %loadPolly -S -polly-codegen \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; Check that we generate valid code even if the load of cont_STACKPOINTER is
diff --git a/polly/test/CodeGen/hoisting_1.ll b/polly/test/CodeGen/hoisting_1.ll
index e04ee68cc4c9..86b56637bc2c 100644
--- a/polly/test/CodeGen/hoisting_1.ll
+++ b/polly/test/CodeGen/hoisting_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=tbaa -passes=polly-codegen -polly-allow-differing-element-types -disable-output %s
+; RUN: opt %loadPolly -tbaa -polly-codegen -polly-allow-differing-element-types -disable-output %s
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/hoisting_2.ll b/polly/test/CodeGen/hoisting_2.ll
index d5c27f58b95b..1f1be11c2d98 100644
--- a/polly/test/CodeGen/hoisting_2.ll
+++ b/polly/test/CodeGen/hoisting_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=tbaa -passes=polly-codegen -polly-allow-differing-element-types -disable-output %s
+; RUN: opt %loadPolly -tbaa -polly-codegen -polly-allow-differing-element-types -disable-output %s
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/inner_scev_sdiv_1.ll b/polly/test/CodeGen/inner_scev_sdiv_1.ll
index 25f2abd9b63e..1a463fc178d1 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_1.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s
+; RUN: opt %loadPolly -S -polly-codegen < %s
 ;
 ; Excerpt from the test-suite's oggenc reduced using bugpoint.
 ;
diff --git a/polly/test/CodeGen/inner_scev_sdiv_2.ll b/polly/test/CodeGen/inner_scev_sdiv_2.ll
index 4d80c2a170d6..76138034603e 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_2.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ; The SCEV expression in this test case refers to a sequence of sdiv
 ; instructions, which are part of different bbs in the SCoP. When code
diff --git a/polly/test/CodeGen/inner_scev_sdiv_3.ll b/polly/test/CodeGen/inner_scev_sdiv_3.ll
index 8d13c8e168bf..874ead14ded2 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_3.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ; This test case has a inner SCEV sdiv that will escape the SCoP. Just check we
 ; do not crash and generate valid code.
diff --git a/polly/test/CodeGen/inner_scev_sdiv_in_lb.ll b/polly/test/CodeGen/inner_scev_sdiv_in_lb.ll
index b53ee034f960..6514e18687e4 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_in_lb.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_in_lb.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s --check-prefix=CODEGEN
 ;
 ; CHECK: [N] -> { Stmt_bb11[i0, i1] : i0 < N and i1 >= 0 and 3i1 <= -3 + i0 };
 ; CODEGEN: polly
diff --git a/polly/test/CodeGen/inner_scev_sdiv_in_lb_invariant.ll b/polly/test/CodeGen/inner_scev_sdiv_in_lb_invariant.ll
index 3c392a2c4c4c..032942923379 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_in_lb_invariant.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_in_lb_invariant.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen \
+; RUN: opt %loadPolly -S -polly-codegen \
 ; RUN:     < %s | FileCheck %s
 ;
 ; Check that this will not crash our code generation.
diff --git a/polly/test/CodeGen/inner_scev_sdiv_in_rtc.ll b/polly/test/CodeGen/inner_scev_sdiv_in_rtc.ll
index 0748a274bb7d..f7292ca3073a 100644
--- a/polly/test/CodeGen/inner_scev_sdiv_in_rtc.ll
+++ b/polly/test/CodeGen/inner_scev_sdiv_in_rtc.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen \
+; RUN: opt %loadPolly -polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 ;
 ; This will just check that we generate valid code here.
diff --git a/polly/test/CodeGen/intrinsics_lifetime.ll b/polly/test/CodeGen/intrinsics_lifetime.ll
index 5782b4724649..6141b3abdd8a 100644
--- a/polly/test/CodeGen/intrinsics_lifetime.ll
+++ b/polly/test/CodeGen/intrinsics_lifetime.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -S < %s | FileCheck %s
 ;
 ; Verify that we remove the lifetime markers from everywhere.
 ;
diff --git a/polly/test/CodeGen/intrinsics_misc.ll b/polly/test/CodeGen/intrinsics_misc.ll
index 9b208b4e600b..c0a52fe97329 100644
--- a/polly/test/CodeGen/intrinsics_misc.ll
+++ b/polly/test/CodeGen/intrinsics_misc.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -S < %s | FileCheck %s
 ;
 ; Verify that we remove the misc intrinsics  from the optimized SCoP.
 ;
diff --git a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-2.ll b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-2.ll
index fd66cf0a47c9..6727247a7f04 100644
--- a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-2.ll
+++ b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S \
+; RUN: opt %loadPolly -polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; This crashed our codegen at some point, verify it runs through
diff --git a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-3.ll b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-3.ll
index 408f0086a260..a573049c8f67 100644
--- a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-3.ll
+++ b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S \
+; RUN: opt %loadPolly -polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; This crashed our codegen at some point, verify it runs through
diff --git a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order.ll b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order.ll
index 45bb5d041a49..e05ca9951434 100644
--- a/polly/test/CodeGen/inv-load-lnt-crash-wrong-order.ll
+++ b/polly/test/CodeGen/inv-load-lnt-crash-wrong-order.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S \
+; RUN: opt %loadPolly -polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; This crashed our codegen at some point, verify it runs through
diff --git a/polly/test/CodeGen/invariant-load-dimension.ll b/polly/test/CodeGen/invariant-load-dimension.ll
index 07bd6923b54b..7793c3b3bee3 100644
--- a/polly/test/CodeGen/invariant-load-dimension.ll
+++ b/polly/test/CodeGen/invariant-load-dimension.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -polly-invariant-load-hoisting '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=SCOPS
-; RUN: opt %loadPolly -S < %s -passes=polly-codegen -polly-process-unprofitable -polly-invariant-load-hoisting | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-invariant-load-hoisting -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=SCOPS
+; RUN: opt %loadPolly -S < %s -polly-codegen -polly-process-unprofitable -polly-invariant-load-hoisting | FileCheck %s -check-prefix=CODEGEN
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n8:16:32-S64"
 
diff --git a/polly/test/CodeGen/invariant-load-preload-base-pointer-origin-first.ll b/polly/test/CodeGen/invariant-load-preload-base-pointer-origin-first.ll
index c4166dd4d2a9..474100995fd8 100644
--- a/polly/test/CodeGen/invariant-load-preload-base-pointer-origin-first.ll
+++ b/polly/test/CodeGen/invariant-load-preload-base-pointer-origin-first.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen -polly-invariant-load-hoisting=true < %s
+; RUN: opt %loadPolly -S -polly-codegen -polly-invariant-load-hoisting=true < %s
 ;
 ; Check that we generate valid code as we did non preload the base pointer
 ; origin of %tmp4 at some point.
diff --git a/polly/test/CodeGen/invariant_cannot_handle_void.ll b/polly/test/CodeGen/invariant_cannot_handle_void.ll
index 633955b6053a..de5d13d6a69a 100644
--- a/polly/test/CodeGen/invariant_cannot_handle_void.ll
+++ b/polly/test/CodeGen/invariant_cannot_handle_void.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
-; RUN: opt %loadPolly -S -passes=polly-codegen -polly-invariant-load-hoisting=true %s | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -S -polly-codegen -polly-invariant-load-hoisting=true %s | FileCheck %s
 ;
 ; The offset of the %tmp1 load wrt. to %buff (62 bytes) is not divisible
 ; by the type size (i32 = 4 bytes), thus we will have to represent %buff
diff --git a/polly/test/CodeGen/invariant_load.ll b/polly/test/CodeGen/invariant_load.ll
index bef3862e9c79..be3f7a32f35b 100644
--- a/polly/test/CodeGen/invariant_load.ll
+++ b/polly/test/CodeGen/invariant_load.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK-NEXT:    %polly.access.B = getelementptr i32, ptr %B, i64 0
diff --git a/polly/test/CodeGen/invariant_load_address_space.ll b/polly/test/CodeGen/invariant_load_address_space.ll
index 9ffef5757cb8..7c611ad3dd87 100644
--- a/polly/test/CodeGen/invariant_load_address_space.ll
+++ b/polly/test/CodeGen/invariant_load_address_space.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK-NEXT:    %polly.access.B = getelementptr i32, ptr addrspace(1) %B, i64 0
diff --git a/polly/test/CodeGen/invariant_load_alias_metadata.ll b/polly/test/CodeGen/invariant_load_alias_metadata.ll
index a992a926880b..5a82d82d43f8 100644
--- a/polly/test/CodeGen/invariant_load_alias_metadata.ll
+++ b/polly/test/CodeGen/invariant_load_alias_metadata.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true \
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true \
 ; RUN: -S < %s | FileCheck %s
 ;
 ; This test case checks whether Polly generates alias metadata in case of
diff --git a/polly/test/CodeGen/invariant_load_base_pointer.ll b/polly/test/CodeGen/invariant_load_base_pointer.ll
index c7d9e8df5b59..eb07f8317b79 100644
--- a/polly/test/CodeGen/invariant_load_base_pointer.ll
+++ b/polly/test/CodeGen/invariant_load_base_pointer.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly  -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
+; RUN: opt %loadPolly  -polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK-NEXT:    %polly.access.BPLoc = getelementptr ptr, ptr %BPLoc, i64 0
diff --git a/polly/test/CodeGen/invariant_load_base_pointer_conditional.ll b/polly/test/CodeGen/invariant_load_base_pointer_conditional.ll
index f24d8b7d7525..538077bb09e8 100644
--- a/polly/test/CodeGen/invariant_load_base_pointer_conditional.ll
+++ b/polly/test/CodeGen/invariant_load_base_pointer_conditional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly  -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
+; RUN: opt %loadPolly  -polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK-NEXT:    %0 = sext i32 %N to i64
diff --git a/polly/test/CodeGen/invariant_load_base_pointer_conditional_2.ll b/polly/test/CodeGen/invariant_load_base_pointer_conditional_2.ll
index 210060c90681..7c2fb3ef97ed 100644
--- a/polly/test/CodeGen/invariant_load_base_pointer_conditional_2.ll
+++ b/polly/test/CodeGen/invariant_load_base_pointer_conditional_2.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -S -passes=polly-codegen -polly-invariant-load-hoisting=true < %s | FileCheck %s --check-prefix=IR
-; RUN: opt %loadPolly -S -passes=polly-codegen -polly-invariant-load-hoisting=true --polly-overflow-tracking=always < %s | FileCheck %s --check-prefix=IRA
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen -polly-invariant-load-hoisting=true < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -S -polly-codegen -polly-invariant-load-hoisting=true --polly-overflow-tracking=always < %s | FileCheck %s --check-prefix=IRA
 ;
 ; As (p + q) can overflow we have to check that we load from
 ; I[p + q] only if it does not.
diff --git a/polly/test/CodeGen/invariant_load_canonicalize_array_baseptrs.ll b/polly/test/CodeGen/invariant_load_canonicalize_array_baseptrs.ll
index f6d4fef13d7a..dc5a4c890381 100644
--- a/polly/test/CodeGen/invariant_load_canonicalize_array_baseptrs.ll
+++ b/polly/test/CodeGen/invariant_load_canonicalize_array_baseptrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s \
+; RUN: opt %loadPolly -polly-codegen -S < %s \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/CodeGen/invariant_load_condition.ll b/polly/test/CodeGen/invariant_load_condition.ll
index 1aab3b8497fc..edf0814d8983 100644
--- a/polly/test/CodeGen/invariant_load_condition.ll
+++ b/polly/test/CodeGen/invariant_load_condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK-NEXT:     %polly.access.C = getelementptr i32, ptr %C, i64 0
diff --git a/polly/test/CodeGen/invariant_load_different_sized_types.ll b/polly/test/CodeGen/invariant_load_different_sized_types.ll
index 952786d71cc9..5b91a1901061 100644
--- a/polly/test/CodeGen/invariant_load_different_sized_types.ll
+++ b/polly/test/CodeGen/invariant_load_different_sized_types.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S \
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S \
 ; RUN: -polly-allow-differing-element-types < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/CodeGen/invariant_load_escaping.ll b/polly/test/CodeGen/invariant_load_escaping.ll
index 31d2066745ef..efccdf468a18 100644
--- a/polly/test/CodeGen/invariant_load_escaping.ll
+++ b/polly/test/CodeGen/invariant_load_escaping.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ;    int f(int *A, int *B) {
 ;      // Possible aliasing between A and B but if not then *B would be
diff --git a/polly/test/CodeGen/invariant_load_escaping_second_scop.ll b/polly/test/CodeGen/invariant_load_escaping_second_scop.ll
index 5dbf261d84e5..c0ea888acdde 100644
--- a/polly/test/CodeGen/invariant_load_escaping_second_scop.ll
+++ b/polly/test/CodeGen/invariant_load_escaping_second_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true  -polly-process-unprofitable -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true  -polly-process-unprofitable -S < %s | FileCheck %s
 ;
 ;    void fence(void);
 ;
diff --git a/polly/test/CodeGen/invariant_load_in_non_affine_subregion.ll b/polly/test/CodeGen/invariant_load_in_non_affine_subregion.ll
index 57b4a655cc24..241252b5d549 100644
--- a/polly/test/CodeGen/invariant_load_in_non_affine_subregion.ll
+++ b/polly/test/CodeGen/invariant_load_in_non_affine_subregion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; This crashed at some point as the invariant load is in a non-affine
 ; subregion. Just check it does not anymore.
diff --git a/polly/test/CodeGen/invariant_load_loop_ub.ll b/polly/test/CodeGen/invariant_load_loop_ub.ll
index a4b96fad0559..ab9aa0dc69a7 100644
--- a/polly/test/CodeGen/invariant_load_loop_ub.ll
+++ b/polly/test/CodeGen/invariant_load_loop_ub.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-process-unprofitable -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -polly-process-unprofitable -S < %s | FileCheck %s
 ;
 ; CHECK: polly.start
 ;
diff --git a/polly/test/CodeGen/invariant_load_not_executed_but_in_parameters.ll b/polly/test/CodeGen/invariant_load_not_executed_but_in_parameters.ll
index 2970dbb5bed9..08ff0871b610 100644
--- a/polly/test/CodeGen/invariant_load_not_executed_but_in_parameters.ll
+++ b/polly/test/CodeGen/invariant_load_not_executed_but_in_parameters.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
 ;
 ; Check that this does not crash as the invariant load is not executed (thus
 ; not preloaded) but still referenced by one of the parameters.
diff --git a/polly/test/CodeGen/invariant_load_outermost.ll b/polly/test/CodeGen/invariant_load_outermost.ll
index eda42f76ed5d..f42135c09014 100644
--- a/polly/test/CodeGen/invariant_load_outermost.ll
+++ b/polly/test/CodeGen/invariant_load_outermost.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 
 ; CHECK: polly.start
 
diff --git a/polly/test/CodeGen/invariant_load_parameters_cyclic_dependence.ll b/polly/test/CodeGen/invariant_load_parameters_cyclic_dependence.ll
index 73fb038baed2..d365c99eff66 100644
--- a/polly/test/CodeGen/invariant_load_parameters_cyclic_dependence.ll
+++ b/polly/test/CodeGen/invariant_load_parameters_cyclic_dependence.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; SCOP:         Assumed Context:
 ; SCOP-NEXT:    [p_0, tmp4] -> {  :  }
diff --git a/polly/test/CodeGen/invariant_load_ptr_ptr_noalias.ll b/polly/test/CodeGen/invariant_load_ptr_ptr_noalias.ll
index 82e957b9ef1d..b4d4c55f0d9b 100644
--- a/polly/test/CodeGen/invariant_load_ptr_ptr_noalias.ll
+++ b/polly/test/CodeGen/invariant_load_ptr_ptr_noalias.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -S  < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -S  < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK:   %polly.access.A = getelementptr ptr, ptr %A, i64 42
diff --git a/polly/test/CodeGen/invariant_load_scalar_dep.ll b/polly/test/CodeGen/invariant_load_scalar_dep.ll
index 3906d5ab890f..05a40a4c47cc 100644
--- a/polly/test/CodeGen/invariant_load_scalar_dep.ll
+++ b/polly/test/CodeGen/invariant_load_scalar_dep.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly  -passes=polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
+; RUN: opt %loadPolly  -polly-codegen -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: polly.preload.begin:
 ; CHECK:    %polly.access.B = getelementptr i32, ptr %B, i64 0
diff --git a/polly/test/CodeGen/invariant_load_scalar_escape_alloca_sharing.ll b/polly/test/CodeGen/invariant_load_scalar_escape_alloca_sharing.ll
index 82e38e86bf77..44c035855b76 100644
--- a/polly/test/CodeGen/invariant_load_scalar_escape_alloca_sharing.ll
+++ b/polly/test/CodeGen/invariant_load_scalar_escape_alloca_sharing.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s
 ;
 ; Verify the preloaded %tmp0 is stored and communicated in the same alloca.
 ; In this case, we do not reload %ncol.load from the scalar stack slot, but
diff --git a/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_1.ll b/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_1.ll
index ef382eb20d0f..0b6929a5fd3f 100644
--- a/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_1.ll
+++ b/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true < %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true < %s
 ;
 ; Check we do not crash even though we pre-load values with different types
 ; from the same base pointer.
diff --git a/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_2.ll b/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_2.ll
index 07e7b97ed9de..2eb913fed447 100644
--- a/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_2.ll
+++ b/polly/test/CodeGen/invariant_loads_from_struct_with_different_types_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true < %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true < %s
 ;
 ; Check we do not crash even though we pre-load values with different types
 ; from the same base pointer.
diff --git a/polly/test/CodeGen/invariant_loads_ignore_parameter_bounds.ll b/polly/test/CodeGen/invariant_loads_ignore_parameter_bounds.ll
index 8be087467e99..a0c1f891bdf6 100644
--- a/polly/test/CodeGen/invariant_loads_ignore_parameter_bounds.ll
+++ b/polly/test/CodeGen/invariant_loads_ignore_parameter_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting \
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting \
 ; RUN:     -polly-ignore-parameter-bounds -S < %s | FileCheck %s
 
 ; CHECK: polly.preload.begin:
diff --git a/polly/test/CodeGen/invariant_verify_function_failed.ll b/polly/test/CodeGen/invariant_verify_function_failed.ll
index 86308a7fbbfc..6020caeee85d 100644
--- a/polly/test/CodeGen/invariant_verify_function_failed.ll
+++ b/polly/test/CodeGen/invariant_verify_function_failed.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,scop(polly-codegen)' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; This crashed at some point as the pointer returned by the call
 ; to @__errno_location is invariant and defined in the SCoP but not
diff --git a/polly/test/CodeGen/invariant_verify_function_failed_2.ll b/polly/test/CodeGen/invariant_verify_function_failed_2.ll
index 97faa6155f18..81a4bd1dc153 100644
--- a/polly/test/CodeGen/invariant_verify_function_failed_2.ll
+++ b/polly/test/CodeGen/invariant_verify_function_failed_2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -S '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s -check-prefix=SCOPS
-; RUN: opt %loadPolly -S -passes=polly-codegen -polly-invariant-load-hoisting=true %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s -check-prefix=SCOPS
+; RUN: opt %loadPolly -S -polly-codegen -polly-invariant-load-hoisting=true %s | FileCheck %s
 ;
 ; Check we generate valid code.
 
diff --git a/polly/test/CodeGen/issue56692.ll b/polly/test/CodeGen/issue56692.ll
index b5ab63c3b72b..e935e43bfa44 100644
--- a/polly/test/CodeGen/issue56692.ll
+++ b/polly/test/CodeGen/issue56692.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-omp-backend=LLVM -polly-codegen-verify -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-omp-backend=LLVM -polly-codegen-verify -polly-codegen -S < %s | FileCheck %s
 ; https://github.com/llvm/llvm-project/issues/56692
 ;
 ; CHECK: call void (ptr, i32, ptr, ...) @__kmpc_fork_call({{.*}}), !dbg ![[OPTLOC:[0-9]+]]
diff --git a/polly/test/CodeGen/large-numbers-in-boundary-context.ll b/polly/test/CodeGen/large-numbers-in-boundary-context.ll
index 519511f1ddd5..a0328dfec651 100644
--- a/polly/test/CodeGen/large-numbers-in-boundary-context.ll
+++ b/polly/test/CodeGen/large-numbers-in-boundary-context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ; XFAIL: *
 ;
 ; The boundary context contains a constant that does not fit in 64 bits. Hence,
diff --git a/polly/test/CodeGen/load_subset_with_context.ll b/polly/test/CodeGen/load_subset_with_context.ll
index 980a06f23c73..ef0e051d5635 100644
--- a/polly/test/CodeGen/load_subset_with_context.ll
+++ b/polly/test/CodeGen/load_subset_with_context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
 ;
 ; A load must provide a value for every statement instance.
 ; Statement instances not in the SCoP's context are irrelevant.
diff --git a/polly/test/CodeGen/loop-invariant-load-type-mismatch.ll b/polly/test/CodeGen/loop-invariant-load-type-mismatch.ll
index ca3463d72af3..90c61c591623 100644
--- a/polly/test/CodeGen/loop-invariant-load-type-mismatch.ll
+++ b/polly/test/CodeGen/loop-invariant-load-type-mismatch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/polly/test/CodeGen/loop_with_condition.ll b/polly/test/CodeGen/loop_with_condition.ll
index 436d37d1261a..618a542c179a 100644
--- a/polly/test/CodeGen/loop_with_condition.ll
+++ b/polly/test/CodeGen/loop_with_condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-ast -disable-output < %s | FileCheck %s
 
 ;#include <string.h>
 ;#define N 1024
diff --git a/polly/test/CodeGen/loop_with_condition_2.ll b/polly/test/CodeGen/loop_with_condition_2.ll
index 47ec693efbd5..b1a116785069 100644
--- a/polly/test/CodeGen/loop_with_condition_2.ll
+++ b/polly/test/CodeGen/loop_with_condition_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 
 ; Verify that we actually detect this loop as the innermost loop even though
 ; there is a conditional inside.
diff --git a/polly/test/CodeGen/loop_with_condition_ineq.ll b/polly/test/CodeGen/loop_with_condition_ineq.ll
index 98866000b0cb..c35208c72dfe 100644
--- a/polly/test/CodeGen/loop_with_condition_ineq.ll
+++ b/polly/test/CodeGen/loop_with_condition_ineq.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-ast -disable-output < %s | FileCheck %s
 
 ;#include <string.h>
 ;#define N 1024
diff --git a/polly/test/CodeGen/loop_with_condition_nested.ll b/polly/test/CodeGen/loop_with_condition_nested.ll
index d9a9dafd4b7e..24a49b47d9e6 100644
--- a/polly/test/CodeGen/loop_with_condition_nested.ll
+++ b/polly/test/CodeGen/loop_with_condition_nested.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen < %s | opt -passes='print<loops>' -disable-output 2>&1 | FileCheck %s -check-prefix=LOOPS
+; RUN: opt %loadPolly -basic-aa -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen < %s | opt -passes='print<loops>' -disable-output 2>&1 | FileCheck %s -check-prefix=LOOPS
 
 
 ;#include <string.h>
diff --git a/polly/test/CodeGen/loop_with_conditional_entry_edge_split_hard_case.ll b/polly/test/CodeGen/loop_with_conditional_entry_edge_split_hard_case.ll
index 3687868b0baf..4444cf1dc4dd 100644
--- a/polly/test/CodeGen/loop_with_conditional_entry_edge_split_hard_case.ll
+++ b/polly/test/CodeGen/loop_with_conditional_entry_edge_split_hard_case.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; Test case to trigger the hard way of creating a unique entering
 ; edge for the SCoP. It is triggered because the entering edge
diff --git a/polly/test/CodeGen/memcpy_annotations.ll b/polly/test/CodeGen/memcpy_annotations.ll
index 42fe5ca92b94..a0a09b75c82e 100644
--- a/polly/test/CodeGen/memcpy_annotations.ll
+++ b/polly/test/CodeGen/memcpy_annotations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; Verify that @llvm.memcpy does not get a !alias.scope annotation.
 ; @llvm.memcpy takes two pointers, it is ambiguous to which the
diff --git a/polly/test/CodeGen/multidim-non-matching-typesize-2.ll b/polly/test/CodeGen/multidim-non-matching-typesize-2.ll
index cfd52a0b7793..63afad6e2f41 100644
--- a/polly/test/CodeGen/multidim-non-matching-typesize-2.ll
+++ b/polly/test/CodeGen/multidim-non-matching-typesize-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -disable-basic-aa -passes=polly-codegen \
+; RUN: opt %loadPolly -disable-basic-aa -polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 ; CHECK: polly
 target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
diff --git a/polly/test/CodeGen/multidim-non-matching-typesize.ll b/polly/test/CodeGen/multidim-non-matching-typesize.ll
index b3f70226f743..d117cefe3376 100644
--- a/polly/test/CodeGen/multidim-non-matching-typesize.ll
+++ b/polly/test/CodeGen/multidim-non-matching-typesize.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -disable-basic-aa -passes=polly-codegen \
+; RUN: opt %loadPolly -disable-basic-aa -polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
diff --git a/polly/test/CodeGen/multidim_2d_parametric_array_static_loop_bounds.ll b/polly/test/CodeGen/multidim_2d_parametric_array_static_loop_bounds.ll
index 11976874ed84..464ddb3740f7 100644
--- a/polly/test/CodeGen/multidim_2d_parametric_array_static_loop_bounds.ll
+++ b/polly/test/CodeGen/multidim_2d_parametric_array_static_loop_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/CodeGen/multidim_alias_check.ll b/polly/test/CodeGen/multidim_alias_check.ll
index 15390433a1b1..585577da0e6d 100644
--- a/polly/test/CodeGen/multidim_alias_check.ll
+++ b/polly/test/CodeGen/multidim_alias_check.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-codegen < %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; CHECK: %polly.access.sext.A = sext i32 %n to i64
diff --git a/polly/test/CodeGen/multiple-codegens.ll b/polly/test/CodeGen/multiple-codegens.ll
index 683ccdfa092b..f950fa4a3e1d 100644
--- a/polly/test/CodeGen/multiple-codegens.ll
+++ b/polly/test/CodeGen/multiple-codegens.ll
@@ -1,5 +1,6 @@
-; RUN: opt %loadPolly "-passes=scop(polly-opt-isl,polly-codegen,polly-codegen)" -S < %s | FileCheck %s
-; RUN: opt %loadPolly "-passes=scop(polly-opt-isl,polly-codegen),scop(polly-codegen)" -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-scops -polly-opt-isl -polly-codegen -polly-scops -polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadNPMPolly "-passes=scop(polly-opt-isl,polly-codegen,polly-codegen)" -S < %s | FileCheck %s
+; RUN: opt %loadNPMPolly "-passes=scop(polly-opt-isl,polly-codegen),scop(polly-codegen)" -S < %s | FileCheck %s
 ;
 ; llvm.org/PR34441
 ; Properly handle multiple -polly-scops/-polly-codegen in the same
diff --git a/polly/test/CodeGen/multiple-scops-in-a-row.ll b/polly/test/CodeGen/multiple-scops-in-a-row.ll
index 0ac158c79129..a24a2e71ad4e 100644
--- a/polly/test/CodeGen/multiple-scops-in-a-row.ll
+++ b/polly/test/CodeGen/multiple-scops-in-a-row.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 
 ; This test case has two scops in a row. When code generating the first scop,
 ; the second scop is invalidated. This test case verifies that we do not crash
diff --git a/polly/test/CodeGen/multiple-types-invariant-load-2.ll b/polly/test/CodeGen/multiple-types-invariant-load-2.ll
index 7916d32bd7de..0fd1df75e2ec 100644
--- a/polly/test/CodeGen/multiple-types-invariant-load-2.ll
+++ b/polly/test/CodeGen/multiple-types-invariant-load-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S \
+; RUN: opt %loadPolly -polly-codegen -S \
 ; RUN: -polly-allow-differing-element-types < %s | FileCheck %s
 
 ; CHECK: polly
diff --git a/polly/test/CodeGen/multiple-types-invariant-load.ll b/polly/test/CodeGen/multiple-types-invariant-load.ll
index 5ce698d4fb4d..b1434679e3d1 100644
--- a/polly/test/CodeGen/multiple-types-invariant-load.ll
+++ b/polly/test/CodeGen/multiple-types-invariant-load.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-differing-element-types -passes=polly-codegen -S \
+; RUN: opt %loadPolly -polly-allow-differing-element-types -polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ; CHECK: %polly.access.global.load = getelementptr i32, ptr %global.load, i64 0
diff --git a/polly/test/CodeGen/multiple_sai_fro_same_base_address.ll b/polly/test/CodeGen/multiple_sai_fro_same_base_address.ll
index 94c208e6692f..0163f248229e 100644
--- a/polly/test/CodeGen/multiple_sai_fro_same_base_address.ll
+++ b/polly/test/CodeGen/multiple_sai_fro_same_base_address.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-position=before-vectorizer '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
-; RUN: opt %loadPolly -polly-position=before-vectorizer -passes=polly-codegen -S < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -polly-position=before-vectorizer -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -polly-position=before-vectorizer -polly-codegen -S < %s | FileCheck %s --check-prefix=IR
 
 ; The IR has two ScopArrayInfo for the value %next.0. This used to produce two
 ; phi nodes in polly.merge_new_and_old, one illegaly using the result of the
diff --git a/polly/test/CodeGen/no-overflow-tracking.ll b/polly/test/CodeGen/no-overflow-tracking.ll
index ff4a8023dead..f11e8927ddee 100644
--- a/polly/test/CodeGen/no-overflow-tracking.ll
+++ b/polly/test/CodeGen/no-overflow-tracking.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-overflow-tracking=never -passes=polly-codegen -S < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-overflow-tracking=never -polly-codegen -S < %s | FileCheck %s --check-prefix=IR
 ;
 ; As (p + q) can overflow we have to check that we load from
 ; I[p + q] only if it does not.
diff --git a/polly/test/CodeGen/no_guard_bb.ll b/polly/test/CodeGen/no_guard_bb.ll
index 6635048e0f9b..47c87ff7c868 100644
--- a/polly/test/CodeGen/no_guard_bb.ll
+++ b/polly/test/CodeGen/no_guard_bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S -verify-dom-info < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S -verify-dom-info < %s | FileCheck %s
 ;
 ; CHECK-NOT: br i1 true, label %polly.{{.*}}, label %polly.{{.*}}
 ;
diff --git a/polly/test/CodeGen/non-affine-dominance-generated-entering.ll b/polly/test/CodeGen/non-affine-dominance-generated-entering.ll
index d1d2fc644010..ebf36acc8d96 100644
--- a/polly/test/CodeGen/non-affine-dominance-generated-entering.ll
+++ b/polly/test/CodeGen/non-affine-dominance-generated-entering.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; llvm.org/PR25439
 ; Scalar reloads in the generated entering block were not recognized as
diff --git a/polly/test/CodeGen/non-affine-exit-node-dominance.ll b/polly/test/CodeGen/non-affine-exit-node-dominance.ll
index 8039f3b08543..af19d2420e3e 100644
--- a/polly/test/CodeGen/non-affine-exit-node-dominance.ll
+++ b/polly/test/CodeGen/non-affine-exit-node-dominance.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; llvm.org/PR25439
 ; The dominance of the generated non-affine subregion block was based on the
diff --git a/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll b/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll
index 5b6c2ecc81c8..2aca316d4c88 100644
--- a/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll
+++ b/polly/test/CodeGen/non-affine-phi-node-expansion-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen \
+; RUN: opt %loadPolly -polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/non-affine-phi-node-expansion-3.ll b/polly/test/CodeGen/non-affine-phi-node-expansion-3.ll
index 9a4e1fa26a9f..18a4b6e4ed4a 100644
--- a/polly/test/CodeGen/non-affine-phi-node-expansion-3.ll
+++ b/polly/test/CodeGen/non-affine-phi-node-expansion-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen \
+; RUN: opt %loadPolly -polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 
 define void @foo(ptr %A, i1 %cond0, i1 %cond1) {
diff --git a/polly/test/CodeGen/non-affine-phi-node-expansion-4.ll b/polly/test/CodeGen/non-affine-phi-node-expansion-4.ll
index b5380b5d225c..8a07ee7c7424 100644
--- a/polly/test/CodeGen/non-affine-phi-node-expansion-4.ll
+++ b/polly/test/CodeGen/non-affine-phi-node-expansion-4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen \
+; RUN: opt %loadPolly -polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 
 define void @foo(ptr %A, i1 %cond0, i1 %cond1) {
diff --git a/polly/test/CodeGen/non-affine-phi-node-expansion.ll b/polly/test/CodeGen/non-affine-phi-node-expansion.ll
index 7c29868675f4..091fc3e323dc 100644
--- a/polly/test/CodeGen/non-affine-phi-node-expansion.ll
+++ b/polly/test/CodeGen/non-affine-phi-node-expansion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen \
+; RUN: opt %loadPolly -polly-codegen \
 ; RUN:     -S < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize-2.ll b/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize-2.ll
index f5c446a22c02..6a1d1f12ba9c 100644
--- a/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize-2.ll
+++ b/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused the code generation to generate invalid code as the same operand
 ; of the PHI node in the non-affine region was synthesized at the wrong place.
diff --git a/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize.ll b/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize.ll
index fc894775b755..036bf34cb7f7 100644
--- a/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize.ll
+++ b/polly/test/CodeGen/non-affine-region-exit-phi-incoming-synthesize.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused the code generation to generate invalid code as the same BBMap was
 ; used for the whole non-affine region. When %add is synthesized for the
diff --git a/polly/test/CodeGen/non-affine-region-implicit-store.ll b/polly/test/CodeGen/non-affine-region-implicit-store.ll
index 6f2c7005d1bb..e89197e24852 100644
--- a/polly/test/CodeGen/non-affine-region-implicit-store.ll
+++ b/polly/test/CodeGen/non-affine-region-implicit-store.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; llvm.org/PR25438
 ; After loop versioning, a dominance check of a non-affine subregion's exit node
diff --git a/polly/test/CodeGen/non-affine-region-phi-references-in-scop-value.ll b/polly/test/CodeGen/non-affine-region-phi-references-in-scop-value.ll
index 6151562555ee..f6e4eb57319d 100644
--- a/polly/test/CodeGen/non-affine-region-phi-references-in-scop-value.ll
+++ b/polly/test/CodeGen/non-affine-region-phi-references-in-scop-value.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-allow-nonaffine-loops \
+; RUN: opt %loadPolly -polly-codegen -polly-allow-nonaffine-loops \
 ; RUN: -S < %s | FileCheck %s
 
 ; This test verifies that values defined in another scop statement and used by
diff --git a/polly/test/CodeGen/non-affine-subregion-dominance-reuse.ll b/polly/test/CodeGen/non-affine-subregion-dominance-reuse.ll
index c2af74c215f9..6c749a404336 100644
--- a/polly/test/CodeGen/non-affine-subregion-dominance-reuse.ll
+++ b/polly/test/CodeGen/non-affine-subregion-dominance-reuse.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S -verify-dom-info \
+; RUN: opt %loadPolly -polly-codegen -S -verify-dom-info \
 ; RUN:     < %s | FileCheck %s
 ;
 ; Check that we do not reuse the B[i-1] GEP created in block S again in
diff --git a/polly/test/CodeGen/non-affine-switch.ll b/polly/test/CodeGen/non-affine-switch.ll
index c829b682fb24..9c08b98700ae 100644
--- a/polly/test/CodeGen/non-affine-switch.ll
+++ b/polly/test/CodeGen/non-affine-switch.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly \
-; RUN: -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: -S -polly-codegen < %s | FileCheck %s
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/CodeGen/non-affine-synthesized-in-branch.ll b/polly/test/CodeGen/non-affine-synthesized-in-branch.ll
index a0febb348434..cc0e60abcd09 100644
--- a/polly/test/CodeGen/non-affine-synthesized-in-branch.ll
+++ b/polly/test/CodeGen/non-affine-synthesized-in-branch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-codegen -S < %s | FileCheck %s
 ;
 ; llvm.org/PR25412
 ; %synthgep caused %gep to be synthesized in subregion_if which was reused for
diff --git a/polly/test/CodeGen/non-affine-update.ll b/polly/test/CodeGen/non-affine-update.ll
index aacbb9e766c4..d2b7fae75b23 100644
--- a/polly/test/CodeGen/non-affine-update.ll
+++ b/polly/test/CodeGen/non-affine-update.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
-; RUN:     -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop \
+; RUN:     -polly-codegen -S < %s | FileCheck %s
 ;
 ;    void non-affine-update(double A[], double C[], double B[]) {
 ;      for (int i = 0; i < 10; i++) {
diff --git a/polly/test/CodeGen/non-hoisted-load-needed-as-base-ptr.ll b/polly/test/CodeGen/non-hoisted-load-needed-as-base-ptr.ll
index 0c7c5f9a1700..5f6642b0630d 100644
--- a/polly/test/CodeGen/non-hoisted-load-needed-as-base-ptr.ll
+++ b/polly/test/CodeGen/non-hoisted-load-needed-as-base-ptr.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=tbaa -passes=polly-codegen -disable-output %s
+; RUN: opt %loadPolly -tbaa -polly-codegen -disable-output %s
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/non_affine_float_compare.ll b/polly/test/CodeGen/non_affine_float_compare.ll
index 0b4813ac11ae..be310b5bf5ca 100644
--- a/polly/test/CodeGen/non_affine_float_compare.ll
+++ b/polly/test/CodeGen/non_affine_float_compare.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen \
+; RUN: opt %loadPolly -polly-codegen \
 ; RUN:     -polly-allow-nonaffine-branches -S -verify-dom-info \
 ; RUN:     < %s | FileCheck %s
 ;
diff --git a/polly/test/CodeGen/only_non_affine_error_region.ll b/polly/test/CodeGen/only_non_affine_error_region.ll
index 472aec927dcd..b2ad1c1fe3fd 100644
--- a/polly/test/CodeGen/only_non_affine_error_region.ll
+++ b/polly/test/CodeGen/only_non_affine_error_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; CHECK-NOT: polly.start
 ;
diff --git a/polly/test/CodeGen/openmp_limit_threads.ll b/polly/test/CodeGen/openmp_limit_threads.ll
index 70f78ebac173..e8eb819f13d9 100644
--- a/polly/test/CodeGen/openmp_limit_threads.ll
+++ b/polly/test/CodeGen/openmp_limit_threads.ll
@@ -1,10 +1,10 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -S < %s | FileCheck %s --check-prefix=AUTO
-; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -polly-num-threads=1 -S < %s | FileCheck %s --check-prefix=ONE
-; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -polly-num-threads=4 -S < %s | FileCheck %s --check-prefix=FOUR
+; RUN: opt %loadPolly -polly-codegen -polly-parallel -S < %s | FileCheck %s --check-prefix=AUTO
+; RUN: opt %loadPolly -polly-codegen -polly-parallel -polly-num-threads=1 -S < %s | FileCheck %s --check-prefix=ONE
+; RUN: opt %loadPolly -polly-codegen -polly-parallel -polly-num-threads=4 -S < %s | FileCheck %s --check-prefix=FOUR
 
-; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -polly-omp-backend=LLVM -S < %s | FileCheck %s --check-prefix=LIBOMP-AUTO
-; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -polly-omp-backend=LLVM -polly-num-threads=1 -S < %s | FileCheck %s --check-prefix=LIBOMP-ONE
-; RUN: opt %loadPolly -passes=polly-codegen -polly-parallel -polly-omp-backend=LLVM -polly-num-threads=4 -S < %s | FileCheck %s --check-prefix=LIBOMP-FOUR
+; RUN: opt %loadPolly -polly-codegen -polly-parallel -polly-omp-backend=LLVM -S < %s | FileCheck %s --check-prefix=LIBOMP-AUTO
+; RUN: opt %loadPolly -polly-codegen -polly-parallel -polly-omp-backend=LLVM -polly-num-threads=1 -S < %s | FileCheck %s --check-prefix=LIBOMP-ONE
+; RUN: opt %loadPolly -polly-codegen -polly-parallel -polly-omp-backend=LLVM -polly-num-threads=4 -S < %s | FileCheck %s --check-prefix=LIBOMP-FOUR
 
 ; Ensure that the provided thread numbers are forwarded to the OpenMP calls.
 ;
diff --git a/polly/test/CodeGen/out-of-scop-phi-node-use.ll b/polly/test/CodeGen/out-of-scop-phi-node-use.ll
index 9ef0586d7077..54e909ecf378 100644
--- a/polly/test/CodeGen/out-of-scop-phi-node-use.ll
+++ b/polly/test/CodeGen/out-of-scop-phi-node-use.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/polly/test/CodeGen/param_div_div_div_2.ll b/polly/test/CodeGen/param_div_div_div_2.ll
index 027c147b173e..764ca241f166 100644
--- a/polly/test/CodeGen/param_div_div_div_2.ll
+++ b/polly/test/CodeGen/param_div_div_div_2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s --check-prefix=IR
 ;
 ; Check that we guard the divisions because we moved them and thereby increased
 ; their domain.
diff --git a/polly/test/CodeGen/partial_write_array.ll b/polly/test/CodeGen/partial_write_array.ll
index 82277d631e4c..6dc5550d82af 100644
--- a/polly/test/CodeGen/partial_write_array.ll
+++ b/polly/test/CodeGen/partial_write_array.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
 ;
 ; Partial write of an array access.
 ;
diff --git a/polly/test/CodeGen/partial_write_emptyset.ll b/polly/test/CodeGen/partial_write_emptyset.ll
index 687599025e09..a25195f11ed7 100644
--- a/polly/test/CodeGen/partial_write_emptyset.ll
+++ b/polly/test/CodeGen/partial_write_emptyset.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
 ;
 ; Partial write, where "partial" is the empty set.
 ; The store is never executed in this case and we do generate it in the
diff --git a/polly/test/CodeGen/partial_write_full_write_that_appears_partial.ll b/polly/test/CodeGen/partial_write_full_write_that_appears_partial.ll
index 261d6fe7a0c5..18a809b30557 100644
--- a/polly/test/CodeGen/partial_write_full_write_that_appears_partial.ll
+++ b/polly/test/CodeGen/partial_write_full_write_that_appears_partial.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 ; CHECK:      polly.stmt.if.then81:                             ; preds = %polly.stmt.if.end75
 ; CHECK-NEXT:   store float undef, ptr %fX64, align 4, !alias.scope !0, !noalias !3
diff --git a/polly/test/CodeGen/partial_write_impossible_restriction.ll b/polly/test/CodeGen/partial_write_impossible_restriction.ll
index d041edb9262e..178227fef8e5 100644
--- a/polly/test/CodeGen/partial_write_impossible_restriction.ll
+++ b/polly/test/CodeGen/partial_write_impossible_restriction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
 ;
 ; The isl scheduler isolates %cond.false into two instances.
 ; A partial write access in one of the instances was never executed,
diff --git a/polly/test/CodeGen/partial_write_in_region.ll b/polly/test/CodeGen/partial_write_in_region.ll
index be5025778096..d8f57b35d585 100644
--- a/polly/test/CodeGen/partial_write_in_region.ll
+++ b/polly/test/CodeGen/partial_write_in_region.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
-; RUN: -polly-import-jscop-postfix=transformed \
+; RUN: opt %loadPolly -polly-import-jscop \
+; RUN: -polly-import-jscop-postfix=transformed -polly-codegen \
 ; RUN: -verify-dom-info \
 ; RUN: -S < %s | FileCheck %s
 ;
diff --git a/polly/test/CodeGen/partial_write_in_region_with_loop.ll b/polly/test/CodeGen/partial_write_in_region_with_loop.ll
index 8379a5f279ed..48a9dbef21d1 100644
--- a/polly/test/CodeGen/partial_write_in_region_with_loop.ll
+++ b/polly/test/CodeGen/partial_write_in_region_with_loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' \
-; RUN: -polly-import-jscop-postfix=transformed \
+; RUN: opt %loadPolly -polly-import-jscop \
+; RUN: -polly-import-jscop-postfix=transformed -polly-codegen \
 ; RUN: -verify-dom-info -polly-allow-nonaffine-loops \
 ; RUN: -S < %s | FileCheck %s
 
diff --git a/polly/test/CodeGen/partial_write_mapped_scalar.ll b/polly/test/CodeGen/partial_write_mapped_scalar.ll
index b74705250be5..9137ef2123c8 100644
--- a/polly/test/CodeGen/partial_write_mapped_scalar.ll
+++ b/polly/test/CodeGen/partial_write_mapped_scalar.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
 ;
 ; Partial write of a (mapped) scalar.
 ;
diff --git a/polly/test/CodeGen/partial_write_mapped_scalar_subregion.ll b/polly/test/CodeGen/partial_write_mapped_scalar_subregion.ll
index 80a97e1aa657..e054b65eadf3 100644
--- a/polly/test/CodeGen/partial_write_mapped_scalar_subregion.ll
+++ b/polly/test/CodeGen/partial_write_mapped_scalar_subregion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
 ;
 ; Partial write of a (mapped) scalar in a non-affine subregion.
 ;
diff --git a/polly/test/CodeGen/perf_monitoring.ll b/polly/test/CodeGen/perf_monitoring.ll
index dde7853b37fd..2abbf24f5e78 100644
--- a/polly/test/CodeGen/perf_monitoring.ll
+++ b/polly/test/CodeGen/perf_monitoring.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-codegen-perf-monitoring \
+; RUN: opt %loadPolly -polly-codegen -polly-codegen-perf-monitoring \
 ; RUN:   -S < %s | FileCheck %s
 
 ; void f(long A[], long N) {
diff --git a/polly/test/CodeGen/perf_monitoring_cycles_per_scop.ll b/polly/test/CodeGen/perf_monitoring_cycles_per_scop.ll
index d9efed5f8efa..11d63fc47658 100644
--- a/polly/test/CodeGen/perf_monitoring_cycles_per_scop.ll
+++ b/polly/test/CodeGen/perf_monitoring_cycles_per_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-codegen-perf-monitoring \
+; RUN: opt %loadPolly -polly-codegen -polly-codegen-perf-monitoring \
 ; RUN:   -S < %s | FileCheck %s
 
 ; void f(long A[], long N) {
diff --git a/polly/test/CodeGen/perf_monitoring_trip_counts_per_scop.ll b/polly/test/CodeGen/perf_monitoring_trip_counts_per_scop.ll
index 560b44561bf5..9b7f324df8e4 100644
--- a/polly/test/CodeGen/perf_monitoring_trip_counts_per_scop.ll
+++ b/polly/test/CodeGen/perf_monitoring_trip_counts_per_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-codegen-perf-monitoring \
+; RUN: opt %loadPolly -polly-codegen -polly-codegen-perf-monitoring \
 ; RUN:   -S < %s | FileCheck %s
 
 ; void f(long A[], long N) {
diff --git a/polly/test/CodeGen/phi-defined-before-scop.ll b/polly/test/CodeGen/phi-defined-before-scop.ll
index 1e6266673a8c..a3b1ba264f04 100644
--- a/polly/test/CodeGen/phi-defined-before-scop.ll
+++ b/polly/test/CodeGen/phi-defined-before-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 ; CHECK-LABEL: polly.merge_new_and_old:
 ; CHECK-NEXT: %tmp7.ph.merge = phi ptr [ %tmp7.ph.final_reload, %polly.exiting ], [ %tmp7.ph, %bb6.region_exiting ]
diff --git a/polly/test/CodeGen/phi_after_error_block_outside_of_scop.ll b/polly/test/CodeGen/phi_after_error_block_outside_of_scop.ll
index 0121888f9fa9..c34ebfc3ca02 100644
--- a/polly/test/CodeGen/phi_after_error_block_outside_of_scop.ll
+++ b/polly/test/CodeGen/phi_after_error_block_outside_of_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 ; Make sure code generation does not break in case an 'error block' is detected
 ; outside of the scope. In this situation, we should not affect code generation.
diff --git a/polly/test/CodeGen/phi_condition_modeling_1.ll b/polly/test/CodeGen/phi_condition_modeling_1.ll
index cf464fe234e3..b14d32921cf7 100644
--- a/polly/test/CodeGen/phi_condition_modeling_1.ll
+++ b/polly/test/CodeGen/phi_condition_modeling_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ;    void f(int *A, int c, int N) {
 ;      int tmp;
diff --git a/polly/test/CodeGen/phi_condition_modeling_2.ll b/polly/test/CodeGen/phi_condition_modeling_2.ll
index 25c67e625542..dab2977bf065 100644
--- a/polly/test/CodeGen/phi_condition_modeling_2.ll
+++ b/polly/test/CodeGen/phi_condition_modeling_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S  -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S  -polly-codegen < %s | FileCheck %s
 ;
 ;    void f(int *A, int c, int N) {
 ;      int tmp;
diff --git a/polly/test/CodeGen/phi_conditional_simple_1.ll b/polly/test/CodeGen/phi_conditional_simple_1.ll
index 3d0e35ccd353..f1b93b540f70 100644
--- a/polly/test/CodeGen/phi_conditional_simple_1.ll
+++ b/polly/test/CodeGen/phi_conditional_simple_1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=AST
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ;    void jd(int *A, int c) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_1.ll b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_1.ll
index 70af1d96f4e7..13688480e315 100644
--- a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_1.ll
+++ b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused an lnt crash at some point, just verify it will run through.
 ;
diff --git a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_2.ll b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_2.ll
index f17d4a6b9160..01dd450590d9 100644
--- a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_2.ll
+++ b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused an lnt crash at some point, just verify it will run through and
 ; produce the PHI node in the exit we are looking for.
diff --git a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_3.ll b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_3.ll
index cf9b4050a39c..66b95b0e0317 100644
--- a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_3.ll
+++ b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused an lnt crash at some point, just verify it will run through and
 ; produce the PHI node in the exit we are looking for.
diff --git a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_5.ll b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_5.ll
index dfc3e6757af8..9a046367e768 100644
--- a/polly/test/CodeGen/phi_in_exit_early_lnt_failure_5.ll
+++ b/polly/test/CodeGen/phi_in_exit_early_lnt_failure_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; This caused an lnt crash at some point, just verify it will run through and
 ; produce the PHI node in the exit we are looking for.
diff --git a/polly/test/CodeGen/phi_loop_carried_float.ll b/polly/test/CodeGen/phi_loop_carried_float.ll
index df51c3599b29..ca1870fb3a09 100644
--- a/polly/test/CodeGen/phi_loop_carried_float.ll
+++ b/polly/test/CodeGen/phi_loop_carried_float.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S  -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S  -polly-codegen < %s | FileCheck %s
 ;
 ;    float f(float *A, int N) {
 ;      float tmp = 0;
diff --git a/polly/test/CodeGen/phi_loop_carried_float_escape.ll b/polly/test/CodeGen/phi_loop_carried_float_escape.ll
index 10bc751150e2..3b2ed01863b1 100644
--- a/polly/test/CodeGen/phi_loop_carried_float_escape.ll
+++ b/polly/test/CodeGen/phi_loop_carried_float_escape.ll
@@ -1,8 +1,8 @@
 ; RUN: opt %loadPolly -S \
-; RUN: -polly-analyze-read-only-scalars=false -passes=polly-codegen < %s | FileCheck %s
+; RUN: -polly-analyze-read-only-scalars=false -polly-codegen < %s | FileCheck %s
 
 ; RUN: opt %loadPolly -S \
-; RUN: -polly-analyze-read-only-scalars=true -passes=polly-codegen < %s | FileCheck %s
+; RUN: -polly-analyze-read-only-scalars=true -polly-codegen < %s | FileCheck %s
 ;
 ;    float f(float *A, int N) {
 ;      float tmp = 0;
diff --git a/polly/test/CodeGen/phi_scalar_simple_1.ll b/polly/test/CodeGen/phi_scalar_simple_1.ll
index 07b8021fb743..d62975b6a7b3 100644
--- a/polly/test/CodeGen/phi_scalar_simple_1.ll
+++ b/polly/test/CodeGen/phi_scalar_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ;    int jd(int *restrict A, int x, int N) {
 ;      for (int i = 1; i < N; i++)
diff --git a/polly/test/CodeGen/phi_scalar_simple_2.ll b/polly/test/CodeGen/phi_scalar_simple_2.ll
index ab89b74c35a7..e58945d39960 100644
--- a/polly/test/CodeGen/phi_scalar_simple_2.ll
+++ b/polly/test/CodeGen/phi_scalar_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ;    int jd(int *restrict A, int x, int N, int c) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/CodeGen/phi_with_multi_exiting_edges_2.ll b/polly/test/CodeGen/phi_with_multi_exiting_edges_2.ll
index 313ae27a4165..17e4b7d6b4de 100644
--- a/polly/test/CodeGen/phi_with_multi_exiting_edges_2.ll
+++ b/polly/test/CodeGen/phi_with_multi_exiting_edges_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; CHECK: polly.merge_new_and_old:
 ; CHECK:   %result.ph.merge = phi float [ %result.ph.final_reload, %polly.exiting ], [ %result.ph, %next.region_exiting ]
diff --git a/polly/test/CodeGen/phi_with_one_exit_edge.ll b/polly/test/CodeGen/phi_with_one_exit_edge.ll
index fa692a9cdd38..81fd73b51c79 100644
--- a/polly/test/CodeGen/phi_with_one_exit_edge.ll
+++ b/polly/test/CodeGen/phi_with_one_exit_edge.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ;
 ; CHECK: polly.merge_new_and_old:
diff --git a/polly/test/CodeGen/pointer-type-expressions-2.ll b/polly/test/CodeGen/pointer-type-expressions-2.ll
index 013a9634844b..b261cfe53321 100644
--- a/polly/test/CodeGen/pointer-type-expressions-2.ll
+++ b/polly/test/CodeGen/pointer-type-expressions-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @foo(ptr %start, ptr %end) {
diff --git a/polly/test/CodeGen/pointer-type-expressions.ll b/polly/test/CodeGen/pointer-type-expressions.ll
index ad0c0639a9d9..6bb3fa242362 100644
--- a/polly/test/CodeGen/pointer-type-expressions.ll
+++ b/polly/test/CodeGen/pointer-type-expressions.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
 
 ; void f(int a[], int N, float *P) {
 ;   int i;
diff --git a/polly/test/CodeGen/pointer-type-pointer-type-comparison.ll b/polly/test/CodeGen/pointer-type-pointer-type-comparison.ll
index 5627a85c01b8..eaef64017aa7 100644
--- a/polly/test/CodeGen/pointer-type-pointer-type-comparison.ll
+++ b/polly/test/CodeGen/pointer-type-pointer-type-comparison.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
 ;
 
 ;    void f(int a[], int N, float *P, float *Q) {
diff --git a/polly/test/CodeGen/pointer_rem.ll b/polly/test/CodeGen/pointer_rem.ll
index a82c5cefa26d..5c92ee52da2c 100644
--- a/polly/test/CodeGen/pointer_rem.ll
+++ b/polly/test/CodeGen/pointer_rem.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>,scop(print<polly-ast>)' -disable-output -S < %s | FileCheck %s --check-prefix=AST
-; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>,scop(polly-codegen)' -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-scops -polly-print-ast -disable-output -S < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-scops -polly-codegen -S < %s | FileCheck %s --check-prefix=CODEGEN
 
 target datalayout = "e-m:e-i64:64-i128:128-n8:16:32:64-S128"
 target triple = "aarch64--linux-gnu"
diff --git a/polly/test/CodeGen/pr25241.ll b/polly/test/CodeGen/pr25241.ll
index 2f982c5ffbfc..9fa67e083a6c 100644
--- a/polly/test/CodeGen/pr25241.ll
+++ b/polly/test/CodeGen/pr25241.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 ; PR25241 (https://llvm.org/bugs/show_bug.cgi?id=25241)
 ; Ensure that synthesized values of a PHI node argument are generated in the
diff --git a/polly/test/CodeGen/ptrtoint_as_parameter.ll b/polly/test/CodeGen/ptrtoint_as_parameter.ll
index ea7cd57bdcc0..4f6c8079729d 100644
--- a/polly/test/CodeGen/ptrtoint_as_parameter.ll
+++ b/polly/test/CodeGen/ptrtoint_as_parameter.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; CHECK:      if.then260:
 ; CHECK-NEXT:   %p.4 = getelementptr inbounds i8, ptr null, i64 1
diff --git a/polly/test/CodeGen/read-only-scalars.ll b/polly/test/CodeGen/read-only-scalars.ll
index 318362d6a27a..a5e1d2719d7d 100644
--- a/polly/test/CodeGen/read-only-scalars.ll
+++ b/polly/test/CodeGen/read-only-scalars.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-analyze-read-only-scalars=false -passes=polly-codegen \
+; RUN: opt %loadPolly -polly-analyze-read-only-scalars=false -polly-codegen \
 ; RUN:     \
 ; RUN:     -S < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-analyze-read-only-scalars=true -passes=polly-codegen \
+; RUN: opt %loadPolly -polly-analyze-read-only-scalars=true -polly-codegen \
 ; RUN:     \
 ; RUN:     -S < %s | FileCheck %s -check-prefix=SCALAR
 
diff --git a/polly/test/CodeGen/reduction.ll b/polly/test/CodeGen/reduction.ll
index 1af5a0d80124..6e5a230ad231 100644
--- a/polly/test/CodeGen/reduction.ll
+++ b/polly/test/CodeGen/reduction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s 2>&1 | not FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s 2>&1 | not FileCheck %s
 
 ;#include <string.h>
 ;#include <stdio.h>
diff --git a/polly/test/CodeGen/reduction_2.ll b/polly/test/CodeGen/reduction_2.ll
index b4ed4d95d543..7a50cea31400 100644
--- a/polly/test/CodeGen/reduction_2.ll
+++ b/polly/test/CodeGen/reduction_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-invariant-load-hoisting=true '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s --allow-empty
+; RUN: opt %loadPolly -basic-aa -polly-invariant-load-hoisting=true -polly-print-ast -disable-output < %s | FileCheck %s --allow-empty
 
 ;#include <string.h>
 ;#include <stdio.h>
diff --git a/polly/test/CodeGen/reduction_simple_binary.ll b/polly/test/CodeGen/reduction_simple_binary.ll
index 25903d6554c5..c7c5501bb7ed 100644
--- a/polly/test/CodeGen/reduction_simple_binary.ll
+++ b/polly/test/CodeGen/reduction_simple_binary.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK: pragma simd reduction
 ;
diff --git a/polly/test/CodeGen/region-with-instructions.ll b/polly/test/CodeGen/region-with-instructions.ll
index 125b791cbcf2..28cabefbf68b 100644
--- a/polly/test/CodeGen/region-with-instructions.ll
+++ b/polly/test/CodeGen/region-with-instructions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 ; CHECK-LABEL:   polly.stmt.bb48:
 ; CHECK-NEXT:   %[[offset:.*]] = shl i64 %polly.indvar, 3
diff --git a/polly/test/CodeGen/region_exiting-domtree.ll b/polly/test/CodeGen/region_exiting-domtree.ll
index 354f631e1002..05983da0a3e3 100644
--- a/polly/test/CodeGen/region_exiting-domtree.ll
+++ b/polly/test/CodeGen/region_exiting-domtree.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -verify-dom-info -disable-output < %s
+; RUN: opt %loadPolly -polly-codegen -verify-dom-info -disable-output < %s
 
 ; Verify that the DominatorTree is preserved correctly for the inserted
 ; %polly.stmt.exit.exit block, which serves as new exit block for the generated
diff --git a/polly/test/CodeGen/region_multiexit_partialwrite.ll b/polly/test/CodeGen/region_multiexit_partialwrite.ll
index 49547c6615d5..b98d7f58732a 100644
--- a/polly/test/CodeGen/region_multiexit_partialwrite.ll
+++ b/polly/test/CodeGen/region_multiexit_partialwrite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-codegen' -polly-import-jscop-postfix=transformed -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -S < %s | FileCheck %s
 ;
 ; This text case has a partial write of PHI in a region-statement. It
 ; requires that the new PHINode from the region's exiting block is
diff --git a/polly/test/CodeGen/run-time-condition-with-scev-parameters.ll b/polly/test/CodeGen/run-time-condition-with-scev-parameters.ll
index df6377ace33a..0f62a8c743df 100644
--- a/polly/test/CodeGen/run-time-condition-with-scev-parameters.ll
+++ b/polly/test/CodeGen/run-time-condition-with-scev-parameters.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=AST
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 ; TODO: FIXME: Simplify the context.
 ; AST: if (n >= 1 && 0 == n <= -1)
diff --git a/polly/test/CodeGen/run-time-condition.ll b/polly/test/CodeGen/run-time-condition.ll
index 2a8cc72f8e9e..0faefad8aef4 100644
--- a/polly/test/CodeGen/run-time-condition.ll
+++ b/polly/test/CodeGen/run-time-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/polly/test/CodeGen/scalar-references-used-in-scop-compute.ll b/polly/test/CodeGen/scalar-references-used-in-scop-compute.ll
index 3519ffaef99b..3f88942c2300 100644
--- a/polly/test/CodeGen/scalar-references-used-in-scop-compute.ll
+++ b/polly/test/CodeGen/scalar-references-used-in-scop-compute.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 ; Test the code generation in the presence of a scalar out-of-scop value being
 ; used from within the SCoP.
diff --git a/polly/test/CodeGen/scalar-store-from-same-bb.ll b/polly/test/CodeGen/scalar-store-from-same-bb.ll
index 016784a9deea..ac8fab4b7a0d 100644
--- a/polly/test/CodeGen/scalar-store-from-same-bb.ll
+++ b/polly/test/CodeGen/scalar-store-from-same-bb.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly \
-; RUN: -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: -polly-codegen -S < %s | FileCheck %s
 
 ; This test ensures that the expression N + 1 that is stored in the phi-node
 ; alloca, is directly computed and not incorrectly transfered through memory.
diff --git a/polly/test/CodeGen/scalar_codegen_crash.ll b/polly/test/CodeGen/scalar_codegen_crash.ll
index e89c3558a187..c41a00f59e81 100644
--- a/polly/test/CodeGen/scalar_codegen_crash.ll
+++ b/polly/test/CodeGen/scalar_codegen_crash.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly \
-; RUN:     -passes=polly-codegen -S < %s | FileCheck %s
+; RUN:     -polly-codegen -S < %s | FileCheck %s
 
 ; This test cases used to crash the scalar code generation. Check that we
 ; can generate code for it.
diff --git a/polly/test/CodeGen/scev-backedgetaken.ll b/polly/test/CodeGen/scev-backedgetaken.ll
index 00fcf0b03482..15e12ee8b451 100644
--- a/polly/test/CodeGen/scev-backedgetaken.ll
+++ b/polly/test/CodeGen/scev-backedgetaken.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; llvm.org/PR48422
 ; Use of ScalarEvolution in Codegen not possible because DominatorTree is not updated.
diff --git a/polly/test/CodeGen/scev-division-invariant-load.ll b/polly/test/CodeGen/scev-division-invariant-load.ll
index 242fb75c3883..3156bdc9f5ce 100644
--- a/polly/test/CodeGen/scev-division-invariant-load.ll
+++ b/polly/test/CodeGen/scev-division-invariant-load.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s
+; RUN: opt %loadPolly -S -polly-codegen < %s
 ;
 ; Check that we generate valid code as we did not use the preloaded
 ; value of %tmp1 for the access function of the preloaded %tmp4.
diff --git a/polly/test/CodeGen/scev.ll b/polly/test/CodeGen/scev.ll
index 74faf062bdc7..07d726d97caf 100644
--- a/polly/test/CodeGen/scev.ll
+++ b/polly/test/CodeGen/scev.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' < %s
+; RUN: opt %loadPolly -polly-detect < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define fastcc void @f () inlinehint align 2 {
diff --git a/polly/test/CodeGen/scev_expansion_in_nonaffine.ll b/polly/test/CodeGen/scev_expansion_in_nonaffine.ll
index 0575795cde48..f61f21d4adb8 100644
--- a/polly/test/CodeGen/scev_expansion_in_nonaffine.ll
+++ b/polly/test/CodeGen/scev_expansion_in_nonaffine.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S \
+; RUN: opt %loadPolly -polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ; bugpoint-reduced testcase of MiBench/consumer-lame/quantize-pvt.c from the
diff --git a/polly/test/CodeGen/scev_looking_through_bitcasts.ll b/polly/test/CodeGen/scev_looking_through_bitcasts.ll
index 776bb3332085..c87d932479b7 100644
--- a/polly/test/CodeGen/scev_looking_through_bitcasts.ll
+++ b/polly/test/CodeGen/scev_looking_through_bitcasts.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; Scalar write of bitcasted value. Instead of writing %b of type
 ; %structty, the SCEV expression looks through the bitcast such that
diff --git a/polly/test/CodeGen/scop_expander_insert_point.ll b/polly/test/CodeGen/scop_expander_insert_point.ll
index 8434e7ec1aa8..8492873b22ed 100644
--- a/polly/test/CodeGen/scop_expander_insert_point.ll
+++ b/polly/test/CodeGen/scop_expander_insert_point.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S \
+; RUN: opt %loadPolly -polly-codegen -S \
 ; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; CHECK:      entry:
diff --git a/polly/test/CodeGen/scop_expander_segfault.ll b/polly/test/CodeGen/scop_expander_segfault.ll
index 73145c4bfa23..293c1e527959 100644
--- a/polly/test/CodeGen/scop_expander_segfault.ll
+++ b/polly/test/CodeGen/scop_expander_segfault.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S %s | FileCheck %s
 ;
 ; This test was extracted from gcc in SPEC2006 and it crashed our code
 ; generation, or to be more precise, the ScopExpander due to a endless
diff --git a/polly/test/CodeGen/scop_never_executed_runtime_check_location.ll b/polly/test/CodeGen/scop_never_executed_runtime_check_location.ll
index cc76ad1771c9..91a58159b5f9 100644
--- a/polly/test/CodeGen/scop_never_executed_runtime_check_location.ll
+++ b/polly/test/CodeGen/scop_never_executed_runtime_check_location.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 ; Verify that we generate the runtime check code after the conditional branch
 ; in the SCoP region entering block (here %entry).
diff --git a/polly/test/CodeGen/select-base-pointer.ll b/polly/test/CodeGen/select-base-pointer.ll
index 9748736147ab..29bc40074e1f 100644
--- a/polly/test/CodeGen/select-base-pointer.ll
+++ b/polly/test/CodeGen/select-base-pointer.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=tbaa -passes=polly-codegen -disable-output %s
+; RUN: opt %loadPolly -tbaa -polly-codegen -disable-output %s
 ;
 ; Check that we do not crash here.
 ;
diff --git a/polly/test/CodeGen/sequential_loops.ll b/polly/test/CodeGen/sequential_loops.ll
index a0cecfbd817b..97d280de3cd2 100644
--- a/polly/test/CodeGen/sequential_loops.ll
+++ b/polly/test/CodeGen/sequential_loops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 
 ;#include <string.h>
 ;#define N 1024
diff --git a/polly/test/CodeGen/simple_loop_non_single_exit.ll b/polly/test/CodeGen/simple_loop_non_single_exit.ll
index a6f115ba1112..dc1b09b765a1 100644
--- a/polly/test/CodeGen/simple_loop_non_single_exit.ll
+++ b/polly/test/CodeGen/simple_loop_non_single_exit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/CodeGen/simple_loop_non_single_exit_2.ll b/polly/test/CodeGen/simple_loop_non_single_exit_2.ll
index d58a6ed7b746..178601cac9b8 100644
--- a/polly/test/CodeGen/simple_loop_non_single_exit_2.ll
+++ b/polly/test/CodeGen/simple_loop_non_single_exit_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/CodeGen/simple_non_single_entry.ll b/polly/test/CodeGen/simple_non_single_entry.ll
index 2b472496d364..3b4bf59bdc65 100644
--- a/polly/test/CodeGen/simple_non_single_entry.ll
+++ b/polly/test/CodeGen/simple_non_single_entry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s -check-prefix=CHECK-CODE
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/CodeGen/simple_nonaffine_loop.ll b/polly/test/CodeGen/simple_nonaffine_loop.ll
index 4074237d1bfb..d4e9c6082e6c 100644
--- a/polly/test/CodeGen/simple_nonaffine_loop.ll
+++ b/polly/test/CodeGen/simple_nonaffine_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-allow-nonaffine -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-allow-nonaffine -disable-output < %s | FileCheck %s
 
 ;#include <stdio.h>
 ;#include <stdlib.h>
diff --git a/polly/test/CodeGen/single_do_loop_int_max_iterations.ll b/polly/test/CodeGen/single_do_loop_int_max_iterations.ll
index 0b1d3e14f68a..9648fbe1cf12 100644
--- a/polly/test/CodeGen/single_do_loop_int_max_iterations.ll
+++ b/polly/test/CodeGen/single_do_loop_int_max_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 
 ;#define N 20
 ;#include "limits.h"
diff --git a/polly/test/CodeGen/single_do_loop_int_param_iterations.ll b/polly/test/CodeGen/single_do_loop_int_param_iterations.ll
index 459ba18edac4..f28d828a5da0 100644
--- a/polly/test/CodeGen/single_do_loop_int_param_iterations.ll
+++ b/polly/test/CodeGen/single_do_loop_int_param_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 ; XFAIL: *
 
 ;define N 20
diff --git a/polly/test/CodeGen/single_do_loop_ll_max_iterations.ll b/polly/test/CodeGen/single_do_loop_ll_max_iterations.ll
index 5fdb5b14df4d..68aaab96083a 100644
--- a/polly/test/CodeGen/single_do_loop_ll_max_iterations.ll
+++ b/polly/test/CodeGen/single_do_loop_ll_max_iterations.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen < %s
 
 ;#define N 20
 ;#include "limits.h"
diff --git a/polly/test/CodeGen/single_do_loop_one_iteration.ll b/polly/test/CodeGen/single_do_loop_one_iteration.ll
index 18bab682adeb..9d97cb854734 100644
--- a/polly/test/CodeGen/single_do_loop_one_iteration.ll
+++ b/polly/test/CodeGen/single_do_loop_one_iteration.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 ; XFAIL: *
 
 ;#define N 20
diff --git a/polly/test/CodeGen/single_do_loop_scev_replace.ll b/polly/test/CodeGen/single_do_loop_scev_replace.ll
index 9bdc3d7cd9b5..7963d9d29fe8 100644
--- a/polly/test/CodeGen/single_do_loop_scev_replace.ll
+++ b/polly/test/CodeGen/single_do_loop_scev_replace.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 
 ;#define N 20
 ;#include "limits.h"
diff --git a/polly/test/CodeGen/single_loop.ll b/polly/test/CodeGen/single_loop.ll
index fe68c81567f8..68cc498b43e0 100644
--- a/polly/test/CodeGen/single_loop.ll
+++ b/polly/test/CodeGen/single_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 
 ;#include <string.h>
 ;#define N 1024
diff --git a/polly/test/CodeGen/single_loop_int_max_iterations.ll b/polly/test/CodeGen/single_loop_int_max_iterations.ll
index 017f1a3114cc..bfb5e4ab2698 100644
--- a/polly/test/CodeGen/single_loop_int_max_iterations.ll
+++ b/polly/test/CodeGen/single_loop_int_max_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 
 ;#define N 20
 ;#include "limits.h"
diff --git a/polly/test/CodeGen/single_loop_ll_max_iterations.ll b/polly/test/CodeGen/single_loop_ll_max_iterations.ll
index 89fb9be7f9e1..bdfd7fce4204 100644
--- a/polly/test/CodeGen/single_loop_ll_max_iterations.ll
+++ b/polly/test/CodeGen/single_loop_ll_max_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 
 ;#include "limits.h"
 ;#define N 20
diff --git a/polly/test/CodeGen/single_loop_one_iteration.ll b/polly/test/CodeGen/single_loop_one_iteration.ll
index fcbe34da2a92..7d4dd590fab9 100644
--- a/polly/test/CodeGen/single_loop_one_iteration.ll
+++ b/polly/test/CodeGen/single_loop_one_iteration.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 
 ;#define N 20
 ;
diff --git a/polly/test/CodeGen/single_loop_param.ll b/polly/test/CodeGen/single_loop_param.ll
index 19f3fb42a475..5d72da354fdc 100644
--- a/polly/test/CodeGen/single_loop_param.ll
+++ b/polly/test/CodeGen/single_loop_param.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @A = common global [1024 x i32] zeroinitializer, align 16 ; <ptr> [#uses=3]
diff --git a/polly/test/CodeGen/single_loop_param_less_equal.ll b/polly/test/CodeGen/single_loop_param_less_equal.ll
index 07d0ef24f4c0..e63ee299a37c 100644
--- a/polly/test/CodeGen/single_loop_param_less_equal.ll
+++ b/polly/test/CodeGen/single_loop_param_less_equal.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen  -S < %s | FileCheck %s -check-prefix=CODEGEN
-; RUN: opt %loadPolly -passes=polly-codegen < %s | opt -passes='print<loops>' -disable-output 2>&1 | FileCheck %s -check-prefix=LOOPS
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen  -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-codegen < %s | opt -passes='print<loops>' -disable-output 2>&1 | FileCheck %s -check-prefix=LOOPS
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @A = common global [1024 x i32] zeroinitializer
diff --git a/polly/test/CodeGen/single_loop_param_less_than.ll b/polly/test/CodeGen/single_loop_param_less_than.ll
index 26dddea44b20..95130f926450 100644
--- a/polly/test/CodeGen/single_loop_param_less_than.ll
+++ b/polly/test/CodeGen/single_loop_param_less_than.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen  -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen  -S < %s | FileCheck %s -check-prefix=CODEGEN
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @A = common global [1024 x i32] zeroinitializer
diff --git a/polly/test/CodeGen/single_loop_zero_iterations.ll b/polly/test/CodeGen/single_loop_zero_iterations.ll
index e4a5b7670d77..4f189687d330 100644
--- a/polly/test/CodeGen/single_loop_zero_iterations.ll
+++ b/polly/test/CodeGen/single_loop_zero_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=SCALAR --allow-empty
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=SCALAR --allow-empty
 
 ;#define N 20
 ;
diff --git a/polly/test/CodeGen/split_edge_of_exit.ll b/polly/test/CodeGen/split_edge_of_exit.ll
index 3f2a1389f886..56ce215a62b2 100644
--- a/polly/test/CodeGen/split_edge_of_exit.ll
+++ b/polly/test/CodeGen/split_edge_of_exit.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -verify-region-info -disable-output < %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -verify-region-info -disable-output < %s
 ;
 ; This is a scop directly precedented by a region, i.e. the scop's entry is the
 ; region's exit block. This test is to ensure that the RegionInfo is correctly
diff --git a/polly/test/CodeGen/split_edges.ll b/polly/test/CodeGen/split_edges.ll
index 0fc705becb91..e01d901e298c 100644
--- a/polly/test/CodeGen/split_edges.ll
+++ b/polly/test/CodeGen/split_edges.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -verify-region-info -verify-dom-info -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -verify-region-info -verify-dom-info -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @A = common global [1536 x float] zeroinitializer
diff --git a/polly/test/CodeGen/split_edges_2.ll b/polly/test/CodeGen/split_edges_2.ll
index 84449327bf4c..4135d6feeb3e 100644
--- a/polly/test/CodeGen/split_edges_2.ll
+++ b/polly/test/CodeGen/split_edges_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -verify-region-info -verify-dom-info -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -verify-region-info -verify-dom-info -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/CodeGen/srem-in-other-bb.ll b/polly/test/CodeGen/srem-in-other-bb.ll
index eaad663159f0..8bde1a3bbc1d 100644
--- a/polly/test/CodeGen/srem-in-other-bb.ll
+++ b/polly/test/CodeGen/srem-in-other-bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S \
+; RUN: opt %loadPolly -polly-codegen -S \
 ; RUN:     < %s | FileCheck %s
 ;
 ;    void pos(float *A, long n) {
diff --git a/polly/test/CodeGen/stack-overflow-in-load-hoisting.ll b/polly/test/CodeGen/stack-overflow-in-load-hoisting.ll
index 41241de132a3..02dfe96e3e91 100644
--- a/polly/test/CodeGen/stack-overflow-in-load-hoisting.ll
+++ b/polly/test/CodeGen/stack-overflow-in-load-hoisting.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -verify-dom-info -passes=polly-codegen -S < %s \
+; RUN: opt %loadPolly -verify-dom-info -polly-codegen -S < %s \
 ; RUN: -polly-invariant-load-hoisting=true | FileCheck %s
 ;
 ; This caused an infinite recursion during invariant load hoisting at some
diff --git a/polly/test/CodeGen/stmt_split_no_dependence.ll b/polly/test/CodeGen/stmt_split_no_dependence.ll
index 94407e8e7c38..a395aa14b4c8 100644
--- a/polly/test/CodeGen/stmt_split_no_dependence.ll
+++ b/polly/test/CodeGen/stmt_split_no_dependence.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; CHECK:   store i32 %9, ptr %scevgep, align 4, !alias.scope !1, !noalias !4
 ; CHECK:   store i32 %11, ptr %scevgep4, align 4, !alias.scope !4, !noalias !1
diff --git a/polly/test/CodeGen/switch-in-non-affine-region.ll b/polly/test/CodeGen/switch-in-non-affine-region.ll
index 2524699157b7..930755ef5648 100644
--- a/polly/test/CodeGen/switch-in-non-affine-region.ll
+++ b/polly/test/CodeGen/switch-in-non-affine-region.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly \
-; RUN: -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: -S -polly-codegen < %s | FileCheck %s
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/CodeGen/synthesizable_phi_write_after_loop.ll b/polly/test/CodeGen/synthesizable_phi_write_after_loop.ll
index 86745d71953e..6a8d3b94d1cc 100644
--- a/polly/test/CodeGen/synthesizable_phi_write_after_loop.ll
+++ b/polly/test/CodeGen/synthesizable_phi_write_after_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; Check for the correct written value of a scalar phi write whose value is
 ; defined within the loop, but its effective value is its last definition when
diff --git a/polly/test/CodeGen/test-invalid-operands-for-select-2.ll b/polly/test/CodeGen/test-invalid-operands-for-select-2.ll
index 9b3608c81f50..5fa4773398fd 100644
--- a/polly/test/CodeGen/test-invalid-operands-for-select-2.ll
+++ b/polly/test/CodeGen/test-invalid-operands-for-select-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen -verify-loop-info < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen -verify-loop-info < %s | FileCheck %s
 ;
 ; Check that we do not crash as described here: http://llvm.org/bugs/show_bug.cgi?id=21167
 ;
diff --git a/polly/test/CodeGen/test-invalid-operands-for-select.ll b/polly/test/CodeGen/test-invalid-operands-for-select.ll
index a10603126cf6..40695af3e847 100644
--- a/polly/test/CodeGen/test-invalid-operands-for-select.ll
+++ b/polly/test/CodeGen/test-invalid-operands-for-select.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ; Check that we do not crash as described here: http://llvm.org/PR21167
 ;
diff --git a/polly/test/CodeGen/test.ll b/polly/test/CodeGen/test.ll
index 1038e57c358f..ac99688ed9e8 100644
--- a/polly/test/CodeGen/test.ll
+++ b/polly/test/CodeGen/test.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 ; XFAIL: *
 
 ;int bar1();
diff --git a/polly/test/CodeGen/two-loops-right-after-each-other-2.ll b/polly/test/CodeGen/two-loops-right-after-each-other-2.ll
index 71bec88e7dde..a7cae0a921ca 100644
--- a/polly/test/CodeGen/two-loops-right-after-each-other-2.ll
+++ b/polly/test/CodeGen/two-loops-right-after-each-other-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 ; CHECK:       polly.merge_new_and_old:
 ; CHECK-NEXT:    merge = phi
diff --git a/polly/test/CodeGen/two-scops-in-row-invalidate-scevs.ll b/polly/test/CodeGen/two-scops-in-row-invalidate-scevs.ll
index 327430da11bb..4470f970fc1e 100644
--- a/polly/test/CodeGen/two-scops-in-row-invalidate-scevs.ll
+++ b/polly/test/CodeGen/two-scops-in-row-invalidate-scevs.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; CHECK-LABEL: for.cond:
 ; CHECK:         %num.0 = phi i32 [ %add, %for.body15 ], [ 0, %for.cond.pre_entry_bb ]
diff --git a/polly/test/CodeGen/two-scops-in-row.ll b/polly/test/CodeGen/two-scops-in-row.ll
index 06e7ea096d79..3e922cba1916 100644
--- a/polly/test/CodeGen/two-scops-in-row.ll
+++ b/polly/test/CodeGen/two-scops-in-row.ll
@@ -1,6 +1,6 @@
 
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ignore-aliasing -disable-output < %s | FileCheck %s -check-prefix=SCALAR
-; RUN: opt %loadPolly -passes=polly-codegen -polly-ignore-aliasing -disable-output < %s
+; RUN: opt %loadPolly -polly-print-ast -polly-ignore-aliasing -disable-output < %s | FileCheck %s -check-prefix=SCALAR
+; RUN: opt %loadPolly -polly-codegen -polly-ignore-aliasing -disable-output < %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; SCALAR: if (
diff --git a/polly/test/CodeGen/udiv_expansion_position.ll b/polly/test/CodeGen/udiv_expansion_position.ll
index 39df17dc2030..bb37fed4a41e 100644
--- a/polly/test/CodeGen/udiv_expansion_position.ll
+++ b/polly/test/CodeGen/udiv_expansion_position.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; Verify we do not crash when we synthezise code for the udiv in the SCoP.
 ;
diff --git a/polly/test/CodeGen/uninitialized_scalar_memory.ll b/polly/test/CodeGen/uninitialized_scalar_memory.ll
index 89eb32c4cf0f..935ccc3d6289 100644
--- a/polly/test/CodeGen/uninitialized_scalar_memory.ll
+++ b/polly/test/CodeGen/uninitialized_scalar_memory.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s
 ;
 ; Verify we initialize the scalar locations reserved for the incoming phi
 ; values.
diff --git a/polly/test/CodeGen/unpredictable-loop-unsynthesizable.ll b/polly/test/CodeGen/unpredictable-loop-unsynthesizable.ll
index e3d0f2df7351..9164bb4532e6 100644
--- a/polly/test/CodeGen/unpredictable-loop-unsynthesizable.ll
+++ b/polly/test/CodeGen/unpredictable-loop-unsynthesizable.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' \
-; RUN: -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -passes=polly-codegen \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops \
+; RUN: -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-codegen \
 ; RUN: -polly-invariant-load-hoisting=true -disable-output < %s
 
 ; The loop for.body is a scop with invariant load hoisting, but does not
diff --git a/polly/test/CodeGen/variant_load_empty_domain.ll b/polly/test/CodeGen/variant_load_empty_domain.ll
index 0ea3b0d1ed1f..f5ad0b195818 100644
--- a/polly/test/CodeGen/variant_load_empty_domain.ll
+++ b/polly/test/CodeGen/variant_load_empty_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-codegen -disable-output < %s
 ;
 ;
 ;    void f(int *A) {
diff --git a/polly/test/CodeGen/whole-scop-non-affine-subregion.ll b/polly/test/CodeGen/whole-scop-non-affine-subregion.ll
index 9c911715904d..931e644f6b8f 100644
--- a/polly/test/CodeGen/whole-scop-non-affine-subregion.ll
+++ b/polly/test/CodeGen/whole-scop-non-affine-subregion.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly \
-; RUN: -passes=polly-codegen -S < %s | FileCheck %s
+; RUN: -polly-codegen -S < %s | FileCheck %s
 
 ; CHECK: polly.start
 ;    int /* pure */ g()
diff --git a/polly/test/DeLICM/confused_order.ll b/polly/test/DeLICM/confused_order.ll
index 62f59cdef315..2015ebcf58f1 100644
--- a/polly/test/DeLICM/confused_order.ll
+++ b/polly/test/DeLICM/confused_order.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-delicm>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-delicm' -polly-import-jscop-postfix=transformed -disable-output -pass-remarks-missed=polly-delicm < %s 2>&1 | FileCheck %s -check-prefix=REMARKS
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-delicm -disable-output -pass-remarks-missed=polly-delicm < %s 2>&1 | FileCheck %s -check-prefix=REMARKS
 ;
 ; ForwardOptree changes the SCoP and may already map some accesses.
 ; DeLICM must be prepared to encounter implicit reads
diff --git a/polly/test/DeLICM/contradicting_assumed_context_and_domain.ll b/polly/test/DeLICM/contradicting_assumed_context_and_domain.ll
index 768cb23631e7..4e039b22b415 100644
--- a/polly/test/DeLICM/contradicting_assumed_context_and_domain.ll
+++ b/polly/test/DeLICM/contradicting_assumed_context_and_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ; The domain of bb14 contradicts the SCoP's assumptions. This leads to
 ; 'anything goes' inside the statement since it is never executed,
diff --git a/polly/test/DeLICM/load-in-cond-inf-loop.ll b/polly/test/DeLICM/load-in-cond-inf-loop.ll
index 40e30a52c545..f0aecfd87a15 100644
--- a/polly/test/DeLICM/load-in-cond-inf-loop.ll
+++ b/polly/test/DeLICM/load-in-cond-inf-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
 
 ; When %b is 0, %for.body13 is an infite loop. In this case the loaded
 ; value %1 is not used anywhere.
diff --git a/polly/test/DeLICM/map_memset_zero.ll b/polly/test/DeLICM/map_memset_zero.ll
index 6789577cb046..1a08eee63fe9 100644
--- a/polly/test/DeLICM/map_memset_zero.ll
+++ b/polly/test/DeLICM/map_memset_zero.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck -match-full-lines %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-delicm>)" -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Check that PHI mapping works even in presence of a memset whose'
 ; zero value is used.
diff --git a/polly/test/DeLICM/nomap_alreadymapped.ll b/polly/test/DeLICM/nomap_alreadymapped.ll
index bf26a809324b..7adf4ba88385 100644
--- a/polly/test/DeLICM/nomap_alreadymapped.ll
+++ b/polly/test/DeLICM/nomap_alreadymapped.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/nomap_escaping.ll b/polly/test/DeLICM/nomap_escaping.ll
index 17451a2941ea..034c0a96ccf2 100644
--- a/polly/test/DeLICM/nomap_escaping.ll
+++ b/polly/test/DeLICM/nomap_escaping.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/nomap_occupied.ll b/polly/test/DeLICM/nomap_occupied.ll
index e6ca903dae03..db33532b1e65 100644
--- a/polly/test/DeLICM/nomap_occupied.ll
+++ b/polly/test/DeLICM/nomap_occupied.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/nomap_readonly.ll b/polly/test/DeLICM/nomap_readonly.ll
index 2c19bb9bc495..1f3b5746fe9b 100644
--- a/polly/test/DeLICM/nomap_readonly.ll
+++ b/polly/test/DeLICM/nomap_readonly.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      fsomeval = 21.0 + 21.0;
diff --git a/polly/test/DeLICM/nomap_spuriouswrite.ll b/polly/test/DeLICM/nomap_spuriouswrite.ll
index f561a4b189ee..ef470f715bbe 100644
--- a/polly/test/DeLICM/nomap_spuriouswrite.ll
+++ b/polly/test/DeLICM/nomap_spuriouswrite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/nomap_storagesize.ll b/polly/test/DeLICM/nomap_storagesize.ll
index 2c116cc85609..fab8d54c2bdf 100644
--- a/polly/test/DeLICM/nomap_storagesize.ll
+++ b/polly/test/DeLICM/nomap_storagesize.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ;    void func(float *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/nomap_writewrite.ll b/polly/test/DeLICM/nomap_writewrite.ll
index 6e3b06a4c57f..06192d9ae19e 100644
--- a/polly/test/DeLICM/nomap_writewrite.ll
+++ b/polly/test/DeLICM/nomap_writewrite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/outofquota-reverseDomain.ll b/polly/test/DeLICM/outofquota-reverseDomain.ll
index d917d294dcdf..d40ee03cf3bc 100644
--- a/polly/test/DeLICM/outofquota-reverseDomain.ll
+++ b/polly/test/DeLICM/outofquota-reverseDomain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-delicm-max-ops=1000000 '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-delicm-max-ops=1000000 -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ; This causes an assertion to fail on out-of-quota after 1000000 operations.
 ; (The error was specific to -polly-delicm-max-ops=1000000 and changes
diff --git a/polly/test/DeLICM/pass_existence.ll b/polly/test/DeLICM/pass_existence.ll
index 57adf45b207c..7ed2da9c1da1 100644
--- a/polly/test/DeLICM/pass_existence.ll
+++ b/polly/test/DeLICM/pass_existence.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -passes=polly-delicm -disable-output < %s
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly '-passes=scop(print<polly-delicm>)' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-delicm -disable-output < %s
+; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-delicm>)" -disable-output < %s | FileCheck %s
 ;
 ; Simple test for the existence of the DeLICM pass.
 ;
diff --git a/polly/test/DeLICM/pr41656.ll b/polly/test/DeLICM/pr41656.ll
index ba65b5bf0416..965ad9f62ac3 100644
--- a/polly/test/DeLICM/pr41656.ll
+++ b/polly/test/DeLICM/pr41656.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>,scop(print<polly-delicm>)' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ; llvm.org/PR41656
 ;
diff --git a/polly/test/DeLICM/pr48783.ll b/polly/test/DeLICM/pr48783.ll
index 2bba7f731f56..3cbd54b93baf 100644
--- a/polly/test/DeLICM/pr48783.ll
+++ b/polly/test/DeLICM/pr48783.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>,scop(print<polly-delicm>)' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ; llvm.org/PR48783
 ;
diff --git a/polly/test/DeLICM/reduction.ll b/polly/test/DeLICM/reduction.ll
index a6f1e032f6e4..78c1a4ce5288 100644
--- a/polly/test/DeLICM/reduction.ll
+++ b/polly/test/DeLICM/reduction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-delicm-partial-writes=true '-passes=print<polly-delicm>' -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-delicm-partial-writes=true -polly-print-delicm -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/reduction_looprotate_gvnpre_cond1.ll b/polly/test/DeLICM/reduction_looprotate_gvnpre_cond1.ll
index e30fe12c8a74..b5bc0d589c65 100644
--- a/polly/test/DeLICM/reduction_looprotate_gvnpre_cond1.ll
+++ b/polly/test/DeLICM/reduction_looprotate_gvnpre_cond1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Load (but not store) of A[j] hoisted, reduction only over some iterations.
 ;
diff --git a/polly/test/DeLICM/reduction_looprotate_gvnpre_cond2.ll b/polly/test/DeLICM/reduction_looprotate_gvnpre_cond2.ll
index a830782e9c44..e995be1143a6 100644
--- a/polly/test/DeLICM/reduction_looprotate_gvnpre_cond2.ll
+++ b/polly/test/DeLICM/reduction_looprotate_gvnpre_cond2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Load (but not store) of A[j] hoisted, reduction not written in all iterations.
 ; FIXME: %join is not mapped because the MemoryKind::Value mapping does not
diff --git a/polly/test/DeLICM/reduction_looprotate_gvnpre_nopreheader.ll b/polly/test/DeLICM/reduction_looprotate_gvnpre_nopreheader.ll
index 903bc80b9d47..ca3a1211ca49 100644
--- a/polly/test/DeLICM/reduction_looprotate_gvnpre_nopreheader.ll
+++ b/polly/test/DeLICM/reduction_looprotate_gvnpre_nopreheader.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Hosted reduction load (but not the store) without preheader.
 ;
diff --git a/polly/test/DeLICM/reduction_looprotate_licm_nopreheader.ll b/polly/test/DeLICM/reduction_looprotate_licm_nopreheader.ll
index 23497f19402b..41538239fbd8 100644
--- a/polly/test/DeLICM/reduction_looprotate_licm_nopreheader.ll
+++ b/polly/test/DeLICM/reduction_looprotate_licm_nopreheader.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ; Register-promoted reduction but without preheader.
 ;
diff --git a/polly/test/DeLICM/reduction_looprotate_loopguard_gvnpre.ll b/polly/test/DeLICM/reduction_looprotate_loopguard_gvnpre.ll
index c932a311a7aa..35c723e864d2 100644
--- a/polly/test/DeLICM/reduction_looprotate_loopguard_gvnpre.ll
+++ b/polly/test/DeLICM/reduction_looprotate_loopguard_gvnpre.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Reduction over parametric number of elements and a loopguard if the
 ; reduction loop is not executed at all. Load hoisted before loop.
diff --git a/polly/test/DeLICM/reduction_looprotate_loopguard_licm1.ll b/polly/test/DeLICM/reduction_looprotate_loopguard_licm1.ll
index 52073eb35014..2b5f4d8151a8 100644
--- a/polly/test/DeLICM/reduction_looprotate_loopguard_licm1.ll
+++ b/polly/test/DeLICM/reduction_looprotate_loopguard_licm1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Reduction over parametric number of elements and a loopguard if the
 ; reduction loop is not executed at all.
diff --git a/polly/test/DeLICM/reduction_looprotate_loopguard_licm2.ll b/polly/test/DeLICM/reduction_looprotate_loopguard_licm2.ll
index 88ae8b3b0f20..2e92813d5551 100644
--- a/polly/test/DeLICM/reduction_looprotate_loopguard_licm2.ll
+++ b/polly/test/DeLICM/reduction_looprotate_loopguard_licm2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Reduction over parametric number of elements and a loopguard if the
 ; reduction loop is not executed at all, such that A[j] is also not written to.
diff --git a/polly/test/DeLICM/reduction_looprotate_loopguard_licm3.ll b/polly/test/DeLICM/reduction_looprotate_loopguard_licm3.ll
index 3a356926afa0..784c8ef2d321 100644
--- a/polly/test/DeLICM/reduction_looprotate_loopguard_licm3.ll
+++ b/polly/test/DeLICM/reduction_looprotate_loopguard_licm3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Reduction over parametric number of elements and a loopguard if the
 ; reduction loop is not executed at all, such that A[j] is also not accessed.
diff --git a/polly/test/DeLICM/reduction_unrelatedunusual.ll b/polly/test/DeLICM/reduction_unrelatedunusual.ll
index 00097dfc92e8..04c437770700 100644
--- a/polly/test/DeLICM/reduction_unrelatedunusual.ll
+++ b/polly/test/DeLICM/reduction_unrelatedunusual.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-delicm-partial-writes=true '-passes=print<polly-delicm>' -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-delicm-partial-writes=true -polly-print-delicm -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Map %add and %phi to A[j].
 ; The non-analyzable store to C[0] is unrelated and can be ignored.
diff --git a/polly/test/DeLICM/reject_loadafterstore.ll b/polly/test/DeLICM/reject_loadafterstore.ll
index 2a153b5cd710..8af6e5e4818c 100644
--- a/polly/test/DeLICM/reject_loadafterstore.ll
+++ b/polly/test/DeLICM/reject_loadafterstore.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output -pass-remarks-missed=polly-delicm < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output -pass-remarks-missed=polly-delicm < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/reject_outofquota.ll b/polly/test/DeLICM/reject_outofquota.ll
index 35001ec1ab2d..551431f0823c 100644
--- a/polly/test/DeLICM/reject_outofquota.ll
+++ b/polly/test/DeLICM/reject_outofquota.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -pass-remarks-analysis=polly-delicm -polly-delicm-max-ops=1 -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=polly-delicm,print<polly-dependences>' -polly-delicm-max-ops=1 -polly-dependences-computeout=0 -disable-output < %s | FileCheck %s -check-prefix=DEP
+; RUN: opt %loadPolly -polly-print-delicm -pass-remarks-analysis=polly-delicm -polly-delicm-max-ops=1 -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-delicm -polly-print-dependences -polly-delicm-max-ops=1 -polly-dependences-computeout=0 -disable-output < %s | FileCheck %s -check-prefix=DEP
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/reject_storeafterstore.ll b/polly/test/DeLICM/reject_storeafterstore.ll
index 715375fcdea2..1ec5ef67344c 100644
--- a/polly/test/DeLICM/reject_storeafterstore.ll
+++ b/polly/test/DeLICM/reject_storeafterstore.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/reject_storeinsubregion.ll b/polly/test/DeLICM/reject_storeinsubregion.ll
index 6490dc25d4a5..1d38e8066568 100644
--- a/polly/test/DeLICM/reject_storeinsubregion.ll
+++ b/polly/test/DeLICM/reject_storeinsubregion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/reject_unusualstore.ll b/polly/test/DeLICM/reject_unusualstore.ll
index 22c7d8ca0ea6..a18a0c3ce9c4 100644
--- a/polly/test/DeLICM/reject_unusualstore.ll
+++ b/polly/test/DeLICM/reject_unusualstore.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-delicm>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -passes=polly-delicm -disable-output -stats < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-delicm -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-delicm -disable-output -stats < %s 2>&1 | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 ;
 ;    void func(double *A) {
diff --git a/polly/test/DeLICM/skip_maywrite.ll b/polly/test/DeLICM/skip_maywrite.ll
index 4da0ddb9dfd2..1e5f6b169fe4 100644
--- a/polly/test/DeLICM/skip_maywrite.ll
+++ b/polly/test/DeLICM/skip_maywrite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeLICM/skip_multiaccess.ll b/polly/test/DeLICM/skip_multiaccess.ll
index 0eea60d0f488..6a8c8e5325e1 100644
--- a/polly/test/DeLICM/skip_multiaccess.ll
+++ b/polly/test/DeLICM/skip_multiaccess.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; llvm.org/PR34485
 ; llvm.org/PR34989
diff --git a/polly/test/DeLICM/skip_notinloop.ll b/polly/test/DeLICM/skip_notinloop.ll
index caa70179d666..0730a3a9a4f5 100644
--- a/polly/test/DeLICM/skip_notinloop.ll
+++ b/polly/test/DeLICM/skip_notinloop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      double phi = 0.0;
diff --git a/polly/test/DeLICM/skip_scalaraccess.ll b/polly/test/DeLICM/skip_scalaraccess.ll
index 8c9728dc6c42..fa95d382409a 100644
--- a/polly/test/DeLICM/skip_scalaraccess.ll
+++ b/polly/test/DeLICM/skip_scalaraccess.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -pass-remarks-missed=polly-delicm -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void func(double *A) {
 ;      for (int j = 0; j < 2; j += 1) { /* outer */
diff --git a/polly/test/DeadCodeElimination/chained_iterations.ll b/polly/test/DeadCodeElimination/chained_iterations.ll
index 10be83559f29..b79fdd659aae 100644
--- a/polly/test/DeadCodeElimination/chained_iterations.ll
+++ b/polly/test/DeadCodeElimination/chained_iterations.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-dependences-analysis-type=value-based '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-dependences-analysis-type=value-based '-passes=polly-dce,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ;
 ; for(i = 0; i < 200; i++ )
diff --git a/polly/test/DeadCodeElimination/chained_iterations_2.ll b/polly/test/DeadCodeElimination/chained_iterations_2.ll
index 42242c40eac9..1d1af92db5da 100644
--- a/polly/test/DeadCodeElimination/chained_iterations_2.ll
+++ b/polly/test/DeadCodeElimination/chained_iterations_2.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-dependences-analysis-type=value-based '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-dependences-analysis-type=value-based '-passes=polly-dce,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ;
 ; for(i = 0; i < 200; i++ )
diff --git a/polly/test/DeadCodeElimination/computeout.ll b/polly/test/DeadCodeElimination/computeout.ll
index 2ac6b8cbbedf..51850d7da349 100644
--- a/polly/test/DeadCodeElimination/computeout.ll
+++ b/polly/test/DeadCodeElimination/computeout.ll
@@ -1,5 +1,6 @@
-; RUN: opt -S %loadPolly "-passes=scop(polly-dce,print<polly-ast>)" < %s | FileCheck %s
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa "-passes=scop(polly-dce,print<polly-ast>)" -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
+; RUN: opt -S %loadPolly -basic-aa -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadNPMPolly "-passes=scop(polly-dce,print<polly-ast>)" < %s | FileCheck %s
+; RUN: opt -S %loadPolly -basic-aa -polly-dce -polly-print-ast -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;     for(i = 0; i < 100; i++ )
diff --git a/polly/test/DeadCodeElimination/dead_iteration_elimination.ll b/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
index 4247ccbcd123..f496f7828e3d 100644
--- a/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
+++ b/polly/test/DeadCodeElimination/dead_iteration_elimination.ll
@@ -1,4 +1,5 @@
-; RUN: opt -S %loadPolly "-passes=scop(polly-dce,print<polly-ast>)" -polly-dependences-analysis-type=value-based -polly-dce-precise-steps=2 < %s | FileCheck %s
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-dce-precise-steps=2 -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadNPMPolly "-passes=scop(polly-dce,print<polly-ast>)" -polly-dependences-analysis-type=value-based -polly-dce-precise-steps=2 < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ;
 ; for(i = 0; i < 200; i++ )
diff --git a/polly/test/DeadCodeElimination/non-affine-affine-mix.ll b/polly/test/DeadCodeElimination/non-affine-affine-mix.ll
index e290d9997d96..e6a5dd204ca1 100644
--- a/polly/test/DeadCodeElimination/non-affine-affine-mix.ll
+++ b/polly/test/DeadCodeElimination/non-affine-affine-mix.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=polly-dce,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/DeadCodeElimination/non-affine.ll b/polly/test/DeadCodeElimination/non-affine.ll
index 8f437ef3e32a..38a7fcbcf9c9 100644
--- a/polly/test/DeadCodeElimination/non-affine.ll
+++ b/polly/test/DeadCodeElimination/non-affine.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=polly-dce,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s
 ;
 ; CHECK: for (int c0 = 0; c0 <= 1023; c0 += 1)
 ;
diff --git a/polly/test/DeadCodeElimination/null_schedule.ll b/polly/test/DeadCodeElimination/null_schedule.ll
index 13c8a127b173..633a84b5d92b 100644
--- a/polly/test/DeadCodeElimination/null_schedule.ll
+++ b/polly/test/DeadCodeElimination/null_schedule.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-dependences-analysis-type=value-based '-passes=polly-dce,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
+; RUN: opt -S %loadPolly -basic-aa -polly-dependences-analysis-type=value-based -polly-dce -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=CHECK-DCE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 ; A[0] = 1;
 ;
diff --git a/polly/test/DependenceInfo/computeout.ll b/polly/test/DependenceInfo/computeout.ll
index 0e64cd1a3725..048de29864d3 100644
--- a/polly/test/DependenceInfo/computeout.ll
+++ b/polly/test/DependenceInfo/computeout.ll
@@ -1,5 +1,7 @@
-; RUN: opt -S %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s -check-prefix=VALUE
-; RUN: opt -S %loadPolly '-passes=print<polly-dependences>' -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
+; RUN: opt -S %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt -S %loadPolly -polly-print-function-dependences -disable-output < %s | FileCheck %s -check-prefix=FUNC-VALUE
+; RUN: opt -S %loadPolly -polly-print-dependences -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
+; RUN: opt -S %loadPolly -polly-print-function-dependences -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;     for(i = 0; i < 100; i++ )
diff --git a/polly/test/DependenceInfo/different_schedule_dimensions.ll b/polly/test/DependenceInfo/different_schedule_dimensions.ll
index edb8371d6e68..3f966168d3b7 100644
--- a/polly/test/DependenceInfo/different_schedule_dimensions.ll
+++ b/polly/test/DependenceInfo/different_schedule_dimensions.ll
@@ -1,5 +1,7 @@
-; RUN: opt -S %loadPolly '-passes=print<polly-dependences>' \
+; RUN: opt -S %loadPolly -polly-print-dependences \
 ; RUN:                   -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -polly-print-function-dependences \
+; RUN:                   -disable-output < %s | FileCheck %s -check-prefix=FUNC
 
 ; CHECK: RAW dependences:
 ; CHECK:   { Stmt_bb9[0] -> Stmt_bb10[0] }
diff --git a/polly/test/DependenceInfo/do_pluto_matmult.ll b/polly/test/DependenceInfo/do_pluto_matmult.ll
index 9532f9c7fb12..d71608e80e70 100644
--- a/polly/test/DependenceInfo/do_pluto_matmult.ll
+++ b/polly/test/DependenceInfo/do_pluto_matmult.ll
@@ -1,5 +1,7 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -polly-dependences-analysis-type=value-based -disable-output < %s | FileCheck %s -check-prefix=VALUE
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -polly-dependences-analysis-type=memory-based -disable-output < %s | FileCheck %s -check-prefix=MEMORY
+; RUN: opt %loadPolly -basic-aa -polly-print-dependences -polly-dependences-analysis-type=value-based -disable-output < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt %loadPolly -basic-aa -polly-print-dependences -polly-dependences-analysis-type=memory-based -disable-output < %s | FileCheck %s -check-prefix=MEMORY
+; RUN: opt %loadPolly -basic-aa -polly-print-function-dependences -polly-dependences-analysis-type=value-based -disable-output < %s | FileCheck %s -check-prefix=FUNC-VALUE
+; RUN: opt %loadPolly -basic-aa -polly-print-function-dependences -polly-dependences-analysis-type=memory-based -disable-output < %s | FileCheck %s -check-prefix=FUNC-MEMORY
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/DependenceInfo/fine_grain_dep_0.ll b/polly/test/DependenceInfo/fine_grain_dep_0.ll
index e7fc66882465..9c79e360690a 100644
--- a/polly/test/DependenceInfo/fine_grain_dep_0.ll
+++ b/polly/test/DependenceInfo/fine_grain_dep_0.ll
@@ -1,6 +1,7 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-dependences>' -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s --check-prefix=REF
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-dependences>' -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s --check-prefix=ACC
-
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-dependences -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s --check-prefix=REF
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-dependences -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s --check-prefix=ACC
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-function-dependences -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s --check-prefix=ACC
+;
 ; REF:      RAW dependences:
 ; REF-NEXT:     [N] -> { [Stmt_for_body[i0] -> MemRef_b[]] -> [Stmt_for_body[6 + i0] -> MemRef_b[]] : 0 <= i0 <= -13 + N; Stmt_for_body[i0] -> Stmt_for_body[6 + i0] : 0 <= i0 <= -13 + N; Stmt_for_body[i0] -> Stmt_for_body[4 + i0] : 0 <= i0 <= -11 + N; [Stmt_for_body[i0] -> MemRef_a[]] -> [Stmt_for_body[4 + i0] -> MemRef_a[]] : 0 <= i0 <= -11 + N }
 ; REF-NEXT: WAR dependences:
diff --git a/polly/test/DependenceInfo/generate_may_write_dependence_info.ll b/polly/test/DependenceInfo/generate_may_write_dependence_info.ll
index 7f6f5f3e3b94..0b7f2d48da9f 100644
--- a/polly/test/DependenceInfo/generate_may_write_dependence_info.ll
+++ b/polly/test/DependenceInfo/generate_may_write_dependence_info.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt -S %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s -check-prefix=VALUE
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 ;                     for (int i = 0; i < N; i++) {
diff --git a/polly/test/DependenceInfo/infeasible_context.ll b/polly/test/DependenceInfo/infeasible_context.ll
index aab6072e4a45..d701b821e15c 100644
--- a/polly/test/DependenceInfo/infeasible_context.ll
+++ b/polly/test/DependenceInfo/infeasible_context.ll
@@ -1,9 +1,10 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s \
 ; RUN:  | FileCheck %s -check-prefix=FUNC-SCOP
-; RUN: opt %loadPolly '-passes=print<polly-detect>,scop(print<polly-dependences>)' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-function-dependences -disable-output < %s \
 ; RUN:  | FileCheck %s -check-prefix=FUNC-DEPS
 ;
 ; FUNC-SCOP-NOT: Statement
+; FUNC-DEPS-LABEL: Printing analysis 'Polly - Calculate dependences for all the SCoPs of a function' for function 'readgeo'
 ; FUNC-DEPS-NOT: RAW dependences
 ;
 ; Due to an infeasible run-time check, scop object is empty and we do not compute dependences.
diff --git a/polly/test/DependenceInfo/may_writes_do_not_block_must_writes_for_war.ll b/polly/test/DependenceInfo/may_writes_do_not_block_must_writes_for_war.ll
index 662cb3232e7e..09c516274708 100644
--- a/polly/test/DependenceInfo/may_writes_do_not_block_must_writes_for_war.ll
+++ b/polly/test/DependenceInfo/may_writes_do_not_block_must_writes_for_war.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; Verify that the presence of a may-write (S1) between a read (S0) and a
 ; must-write (S2) does not block the generation of RAW dependences. This makes
diff --git a/polly/test/DependenceInfo/nonaffine-condition-buildMemoryAccess.ll b/polly/test/DependenceInfo/nonaffine-condition-buildMemoryAccess.ll
index d8361a2b74a6..25c7e3d6e442 100644
--- a/polly/test/DependenceInfo/nonaffine-condition-buildMemoryAccess.ll
+++ b/polly/test/DependenceInfo/nonaffine-condition-buildMemoryAccess.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -polly-allow-nonaffine-loops -polly-allow-nonaffine -debug-only=polly-dependence < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-allow-nonaffine-loops -polly-allow-nonaffine -debug-only=polly-dependence < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; CHECK:        MayWriteAccess :=   [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/DependenceInfo/reduction_complex_location.ll b/polly/test/DependenceInfo/reduction_complex_location.ll
index 2c14f116c904..7ca839996326 100644
--- a/polly/test/DependenceInfo/reduction_complex_location.ll
+++ b/polly/test/DependenceInfo/reduction_complex_location.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     {  }
diff --git a/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll b/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll
index e32217910fc7..3632bd202da2 100644
--- a/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll
+++ b/polly/test/DependenceInfo/reduction_dependences_equal_non_reduction_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; This loopnest contains a reduction which imposes the same dependences as the
 ; accesses to the array A. We need to ensure we keep the dependences of A.
diff --git a/polly/test/DependenceInfo/reduction_dependences_not_null.ll b/polly/test/DependenceInfo/reduction_dependences_not_null.ll
index 852f03cb6f70..69fd74478ecc 100644
--- a/polly/test/DependenceInfo/reduction_dependences_not_null.ll
+++ b/polly/test/DependenceInfo/reduction_dependences_not_null.ll
@@ -1,7 +1,7 @@
 ; Test that the reduction dependences are always initialised, even in a case
 ; where we have no reduction. If this object is NULL, then isl operations on
 ; it will fail.
-; RUN: opt -S %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt -S %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s -check-prefix=VALUE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;     for(i = 0; i < 100; i++ )
diff --git a/polly/test/DependenceInfo/reduction_mixed_reduction_and_non_reduction_dependences.ll b/polly/test/DependenceInfo/reduction_mixed_reduction_and_non_reduction_dependences.ll
index 4c78d80b8ceb..71903d9e7111 100644
--- a/polly/test/DependenceInfo/reduction_mixed_reduction_and_non_reduction_dependences.ll
+++ b/polly/test/DependenceInfo/reduction_mixed_reduction_and_non_reduction_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_for_body3[i0, i1] -> Stmt_for_body3[i0 + i1, o1] : i0 >= 0 and 0 <= i1 <= 1023 - i0 and i1 <= 1 and 0 < o1 <= 511 }
diff --git a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll
index 7e05265f6e6d..234de5c367a0 100644
--- a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum.ll
@@ -1,6 +1,6 @@
-; RUN: opt -aa-pipeline=basic-aa %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
-; RUN: opt -aa-pipeline=basic-aa %loadPolly '-passes=print<polly-dependences>' -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s
-; RUN: opt -aa-pipeline=basic-aa %loadPolly '-passes=print<polly-dependences>' -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-print-dependences -polly-dependences-analysis-level=reference-wise -disable-output < %s | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-print-dependences -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s
 ;
 ; Verify that only the inner reduction like accesses cause reduction dependences
 ;
diff --git a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll
index 7e8a66a50bfd..acd674dc0117 100644
--- a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -basic-aa -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     {  }
diff --git a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll
index adb04f305993..bdfcfc99c8cb 100644
--- a/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_loops_array_sum_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -basic-aa -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      Reduction dependences:
 ; CHECK-NEXT:     { Stmt_for_inc[i0, i1] -> Stmt_for_inc[i0, 1 + i1] : 0 <= i0 <= 99 and 0 <= i1 <= 98 }
diff --git a/polly/test/DependenceInfo/reduction_multiple_reductions.ll b/polly/test/DependenceInfo/reduction_multiple_reductions.ll
index 2d810915bada..cf705080e03d 100644
--- a/polly/test/DependenceInfo/reduction_multiple_reductions.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_reductions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; Verify we do not have dependences between the if and the else clause
 ;
diff --git a/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll b/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll
index 326dcaee5f07..8d8557a129ab 100644
--- a/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll
+++ b/polly/test/DependenceInfo/reduction_multiple_reductions_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ;
 ; These are the important RAW dependences, as they need to originate/end in only one iteration:
diff --git a/polly/test/DependenceInfo/reduction_only_reduction_like_access.ll b/polly/test/DependenceInfo/reduction_only_reduction_like_access.ll
index 1e6455b264d3..7b4a68a2a897 100644
--- a/polly/test/DependenceInfo/reduction_only_reduction_like_access.ll
+++ b/polly/test/DependenceInfo/reduction_only_reduction_like_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; FIXME: Change the comment once we allow different pointers
 ; The statement is "almost" reduction like but should not yield any reduction dependences
diff --git a/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll b/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll
index 158e6a29d8a7..0d09e5a861a0 100644
--- a/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll
+++ b/polly/test/DependenceInfo/reduction_partially_escaping_intermediate_in_other_stmt.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -basic-aa -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      Reduction dependences:
 ; CHECK-NEXT:     [N] -> { Stmt_for_body3[i0, i1] -> Stmt_for_body3[i0, 1 + i1] : 0 <= i0 <= 1023 and i1 >= 0 and 1024 - N + i0 <= i1 <= 1022 }
diff --git a/polly/test/DependenceInfo/reduction_privatization_deps.ll b/polly/test/DependenceInfo/reduction_privatization_deps.ll
index 5d62b8f0a10a..ce90e21a898d 100644
--- a/polly/test/DependenceInfo/reduction_privatization_deps.ll
+++ b/polly/test/DependenceInfo/reduction_privatization_deps.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_S1[i0, i1] -> Stmt_S2[-1 + i0 + i1] : 0 <= i0 <= 1023 and i1 >= 0 and -i0 < i1 <= 1024 - i0 and i1 <= 1023; Stmt_S0[i0] -> Stmt_S1[o0, i0 - o0] : i0 <= 1023 and 0 <= o0 <= i0 }
diff --git a/polly/test/DependenceInfo/reduction_privatization_deps_2.ll b/polly/test/DependenceInfo/reduction_privatization_deps_2.ll
index ed936bfb3b7f..4904004d4781 100644
--- a/polly/test/DependenceInfo/reduction_privatization_deps_2.ll
+++ b/polly/test/DependenceInfo/reduction_privatization_deps_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; We have privatization dependences from a textually later statement to a
 ; textually earlier one, but the dependences still go forward in time.
diff --git a/polly/test/DependenceInfo/reduction_privatization_deps_3.ll b/polly/test/DependenceInfo/reduction_privatization_deps_3.ll
index 58ef9a4774b9..a3935ebd6cc4 100644
--- a/polly/test/DependenceInfo/reduction_privatization_deps_3.ll
+++ b/polly/test/DependenceInfo/reduction_privatization_deps_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_S1[i0] -> Stmt_S3[2 + i0] : 0 <= i0 <= 96; Stmt_S2[i0, i1] -> Stmt_S3[o0] : i1 <= 1 - i0 and -i1 < o0 <= 1 and o0 <= 1 + i0 - i1; Stmt_S3[i0] -> Stmt_S2[o0, 1 - i0] : 0 <= i0 <= 1 and i0 < o0 <= 98 }
diff --git a/polly/test/DependenceInfo/reduction_privatization_deps_4.ll b/polly/test/DependenceInfo/reduction_privatization_deps_4.ll
index 3dad7b217486..10d726af5145 100644
--- a/polly/test/DependenceInfo/reduction_privatization_deps_4.ll
+++ b/polly/test/DependenceInfo/reduction_privatization_deps_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_S1[i0] -> Stmt_S2[i0, i0] : 0 <= i0 <= 98; Stmt_S2[i0, i0] -> Stmt_S3[i0] : 0 <= i0 <= 98; Stmt_S3[i0] -> Stmt_S2[o0, i0] : i0 >= 0 and i0 < o0 <= 98; Stmt_S2[i0, i1] -> Stmt_S1[i1] : i0 >= 0 and i0 < i1 <= 98 }
diff --git a/polly/test/DependenceInfo/reduction_privatization_deps_5.ll b/polly/test/DependenceInfo/reduction_privatization_deps_5.ll
index 0c445d23a92d..e8d51181725e 100644
--- a/polly/test/DependenceInfo/reduction_privatization_deps_5.ll
+++ b/polly/test/DependenceInfo/reduction_privatization_deps_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_S1[i0, 0] -> Stmt_S2[i0, 0] : 0 <= i0 <= 98; Stmt_S2[i0, 0] -> Stmt_S1[1 + i0, 0] : 0 <= i0 <= 97 }
diff --git a/polly/test/DependenceInfo/reduction_sequence.ll b/polly/test/DependenceInfo/reduction_sequence.ll
index 7e1ebd4ab67c..4a4688953938 100644
--- a/polly/test/DependenceInfo/reduction_sequence.ll
+++ b/polly/test/DependenceInfo/reduction_sequence.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 
 ;    void manyreductions(long *A) {
 ;      for (long i = 0; i < 1024; i++)
diff --git a/polly/test/DependenceInfo/reduction_simple_iv.ll b/polly/test/DependenceInfo/reduction_simple_iv.ll
index 64f6de22b078..e3307afae08b 100644
--- a/polly/test/DependenceInfo/reduction_simple_iv.ll
+++ b/polly/test/DependenceInfo/reduction_simple_iv.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     {  }
diff --git a/polly/test/DependenceInfo/reduction_simple_iv_debug_wrapped_dependences.ll b/polly/test/DependenceInfo/reduction_simple_iv_debug_wrapped_dependences.ll
index 61ee3dbc5b02..c7651c39a563 100644
--- a/polly/test/DependenceInfo/reduction_simple_iv_debug_wrapped_dependences.ll
+++ b/polly/test/DependenceInfo/reduction_simple_iv_debug_wrapped_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -debug-only=polly-dependence -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -debug-only=polly-dependence -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/DependenceInfo/reduction_simple_privatization_deps_2.ll b/polly/test/DependenceInfo/reduction_simple_privatization_deps_2.ll
index a181d2acb651..b61fd8453a8c 100644
--- a/polly/test/DependenceInfo/reduction_simple_privatization_deps_2.ll
+++ b/polly/test/DependenceInfo/reduction_simple_privatization_deps_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     { Stmt_S1[i0, i1] -> Stmt_S2[i0] : 0 <= i0 <= 99 and 0 <= i1 <= 99; Stmt_S0[i0] -> Stmt_S1[i0, o1] : 0 <= i0 <= 99 and 0 <= o1 <= 99; Stmt_S2[i0] -> Stmt_S0[1 + i0] : 0 <= i0 <= 98 }
diff --git a/polly/test/DependenceInfo/reduction_simple_privatization_deps_w_parameter.ll b/polly/test/DependenceInfo/reduction_simple_privatization_deps_w_parameter.ll
index ffa26b5eb76c..a3a87c70d905 100644
--- a/polly/test/DependenceInfo/reduction_simple_privatization_deps_w_parameter.ll
+++ b/polly/test/DependenceInfo/reduction_simple_privatization_deps_w_parameter.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     [N] -> { Stmt_S1[i0] -> Stmt_S2[] : N >= 11 and 0 <= i0 <= 1023; Stmt_S0[] -> Stmt_S1[o0] : N >= 11 and 0 <= o0 <= 1023 }
diff --git a/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll b/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll
index 25d117d4cbf9..c90462962ce0 100644
--- a/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll
+++ b/polly/test/DependenceInfo/reduction_two_reductions_different_rloops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-dependences -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      RAW dependences:
 ; CHECK-NEXT:     {  }
diff --git a/polly/test/DependenceInfo/sequential_loops.ll b/polly/test/DependenceInfo/sequential_loops.ll
index 14c9a6429c67..8dfa13cb9db8 100644
--- a/polly/test/DependenceInfo/sequential_loops.ll
+++ b/polly/test/DependenceInfo/sequential_loops.ll
@@ -1,43 +1,34 @@
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -polly-dependences-analysis-type=value-based -disable-output < %s | FileCheck %s -check-prefix=VALUE
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -polly-dependences-analysis-type=memory-based -disable-output < %s | FileCheck %s -check-prefix=MEMORY
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-dependences>' -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s -check-prefix=VALUE_ACCESS
+; RUN: opt -S %loadPolly -basic-aa -polly-print-dependences -polly-dependences-analysis-type=value-based -disable-output < %s | FileCheck %s -check-prefix=VALUE
+; RUN: opt -S %loadPolly -basic-aa -polly-print-dependences -polly-dependences-analysis-type=memory-based -disable-output < %s | FileCheck %s -check-prefix=MEMORY
+; RUN: opt -S %loadPolly -basic-aa -polly-print-dependences -polly-dependences-analysis-type=value-based -polly-dependences-analysis-level=access-wise -disable-output < %s | FileCheck %s -check-prefix=VALUE_ACCESS
 
-; VALUE:      RAW dependences:
+; VALUE-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'sequential_writes':
+; VALUE-NEXT:      RAW dependences:
 ; VALUE-NEXT:          {  }
 ; VALUE-NEXT:      WAR dependences:
 ; VALUE-NEXT:          {  }
 ; VALUE-NEXT:      WAW dependences:
 ; VALUE-NEXT:          { Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99; Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9 }
 ;
-; VALUE:      RAW dependences:
-; VALUE-NEXT:          { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99 }
-; VALUE-NEXT:      WAR dependences:
-; VALUE-NEXT:          {  }
-; VALUE-NEXT:      WAW dependences:
-; VALUE-NEXT:          { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9 }
-;
-; VALUE:      RAW dependences:
-; VALUE-NEXT:          {  }
-; VALUE-NEXT:      WAR dependences:
-; VALUE-NEXT:          { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99 }
-; VALUE-NEXT:      WAW dependences:
-; VALUE-NEXT:          { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9 }
-;
-; VALUE:      RAW dependences:
-; VALUE-NEXT:          [p] -> { Stmt_S1[i0] -> Stmt_S2[-p + i0] : i0 >= p and 0 <= i0 <= 99 and i0 <= 9 + p }
-; VALUE-NEXT:      WAR dependences:
-; VALUE-NEXT:          [p] -> {  }
-; VALUE-NEXT:      WAW dependences:
-; VALUE-NEXT:          [p] -> {  }
-;
-;VALUE_ACCESS:        RAW dependences:
+;VALUE_ACCESS-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'sequential_writes':
+;VALUE_ACCESS-NEXT:        RAW dependences:
 ;VALUE_ACCESS-NEXT:                {  }
 ;VALUE_ACCESS-NEXT:        WAR dependences:
 ;VALUE_ACCESS-NEXT:                {  }
 ;VALUE_ACCESS-NEXT:        WAW dependences:
 ;VALUE_ACCESS-NEXT:                { Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99; Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; [Stmt_S2[i0] -> Stmt_S2_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Write0[]] : 0 <= i0 <= 9; Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S2[i0] -> Stmt_S2_Write0[]] : 0 <= i0 <= 9; [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Write0[]] : 10 <= i0 <= 99 }
+
 ;
-;VALUE_ACCESS:        RAW dependences:
+; VALUE-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'read_after_writes':
+; VALUE-NEXT:      RAW dependences:
+; VALUE-NEXT:          { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99 }
+; VALUE-NEXT:      WAR dependences:
+; VALUE-NEXT:          {  }
+; VALUE-NEXT:      WAW dependences:
+; VALUE-NEXT:          { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9 }
+;
+;VALUE_ACCESS-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'read_after_writes':
+;VALUE_ACCESS-NEXT:        RAW dependences:
 ;VALUE_ACCESS-NEXT:                { Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99; Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; [Stmt_S2[i0] -> Stmt_S2_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Read0[]] : 0 <= i0 <= 9; [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Read0[]] : 10 <= i0 <= 99 }
 
 ;VALUE_ACCESS-NEXT:        WAR dependences:
@@ -45,42 +36,64 @@
 ;VALUE_ACCESS-NEXT:        WAW dependences:
 ;VALUE_ACCESS-NEXT:                { [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S2[i0] -> Stmt_S2_Write0[]] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9 }
 ;
-;VALUE_ACCESS:         RAW dependences:
+; VALUE-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'write_after_read':
+; VALUE-NEXT:      RAW dependences:
+; VALUE-NEXT:          {  }
+; VALUE-NEXT:      WAR dependences:
+; VALUE-NEXT:          { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99 }
+; VALUE-NEXT:      WAW dependences:
+; VALUE-NEXT:          { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9 }
+;
+;VALUE_ACCESS-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'write_after_read':
+;VALUE_ACCESS-NEXT:         RAW dependences:
 ;VALUE_ACCESS-NEXT:                 {  }
 ;VALUE_ACCESS-NEXT:         WAR dependences:
 ;VALUE_ACCESS-NEXT:                { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 10 <= i0 <= 99; [Stmt_S1[i0] -> Stmt_S1_Read0[]] -> [Stmt_S2[i0] -> Stmt_S2_Write0[]] : 0 <= i0 <= 9; [Stmt_S1[i0] -> Stmt_S1_Read0[]] -> [Stmt_S3[i0] -> Stmt_S3_Write0[]] : 10 <= i0 <= 99 }
 ;VALUE_ACCESS-NEXT:         WAW dependences:
 ;VALUE_ACCESS-NEXT:                { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; [Stmt_S2[i0] -> Stmt_S2_Write0[]] -> [Stmt_S3[i0] -> Stmt_S3_Write0[]] : 0 <= i0 <= 9 }
 ;
-;VALUE_ACCESS:        RAW dependences:
+; VALUE-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.2' in function 'parametric_offset':
+; VALUE-NEXT:      RAW dependences:
+; VALUE-NEXT:          [p] -> { Stmt_S1[i0] -> Stmt_S2[-p + i0] : i0 >= p and 0 <= i0 <= 99 and i0 <= 9 + p }
+; VALUE-NEXT:      WAR dependences:
+; VALUE-NEXT:          [p] -> {  }
+; VALUE-NEXT:      WAW dependences:
+; VALUE-NEXT:          [p] -> {  }
+;
+;VALUE_ACCESS-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.2' in function 'parametric_offset':
+;VALUE_ACCESS-NEXT:        RAW dependences:
 ;VALUE_ACCESS-NEXT:                [p] -> { Stmt_S1[i0] -> Stmt_S2[-p + i0] : i0 >= p and 0 <= i0 <= 99 and i0 <= 9 + p; [Stmt_S1[i0] -> Stmt_S1_Write0[]] -> [Stmt_S2[-p + i0] -> Stmt_S2_Read0[]] : i0 >= p and 0 <= i0 <= 99 and i0 <= 9 + p }
 ;VALUE_ACCESS-NEXT:        WAR dependences:
 ;VALUE_ACCESS-NEXT:                [p] -> {  }
 ;VALUE_ACCESS-NEXT:        WAW dependences:
 ;VALUE_ACCESS-NEXT:                [p] -> {  }
 
-; MEMORY:      RAW dependences:
+; MEMORY-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'sequential_writes':
+; MEMORY-NEXT:      RAW dependences:
 ; MEMORY-NEXT:          {  }
 ; MEMORY-NEXT:      WAR dependences:
 ; MEMORY-NEXT:          {  }
 ; MEMORY-NEXT:      WAW dependences:
 ; MEMORY-NEXT:          { Stmt_S1[i0] -> Stmt_S3[i0] : 0 <= i0 <= 99; Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9 }
 ;
-; MEMORY:      RAW dependences:
+; MEMORY-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'read_after_writes':
+; MEMORY-NEXT:      RAW dependences:
 ; MEMORY-NEXT:          { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 0 <= i0 <= 99 }
 ; MEMORY-NEXT:      WAR dependences:
 ; MEMORY-NEXT:          {  }
 ; MEMORY-NEXT:      WAW dependences:
 ; MEMORY-NEXT:          { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9 }
 ;
-; MEMORY:      RAW dependences:
+; MEMORY-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.3' in function 'write_after_read':
+; MEMORY-NEXT:      RAW dependences:
 ; MEMORY-NEXT:          {  }
 ; MEMORY-NEXT:      WAR dependences:
 ; MEMORY-NEXT:          { Stmt_S1[i0] -> Stmt_S2[i0] : 0 <= i0 <= 9; Stmt_S1[i0] -> Stmt_S3[i0] : 0 <= i0 <= 99 }
 ; MEMORY-NEXT:      WAW dependences:
 ; MEMORY-NEXT:          { Stmt_S2[i0] -> Stmt_S3[i0] : 0 <= i0 <= 9 }
 ;
-; MEMORY:      RAW dependences:
+; MEMORY-LABEL: Printing analysis 'Polly - Calculate dependences' for region: 'S1 => exit.2' in function 'parametric_offset':
+; MEMORY-NEXT:      RAW dependences:
 ; MEMORY-NEXT:          [p] -> { Stmt_S1[i0] -> Stmt_S2[-p + i0] : i0 >= p and 0 <= i0 <= 99 and i0 <= 9 + p }
 ; MEMORY-NEXT:      WAR dependences:
 ; MEMORY-NEXT:          [p] -> {  }
diff --git a/polly/test/ForwardOpTree/atax.ll b/polly/test/ForwardOpTree/atax.ll
index 7cc40fe7e1cb..0690c1b000fa 100644
--- a/polly/test/ForwardOpTree/atax.ll
+++ b/polly/test/ForwardOpTree/atax.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-optree-normalize-phi=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-optree-normalize-phi=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ForwardOpTree/changed-kind.ll b/polly/test/ForwardOpTree/changed-kind.ll
index 3c3d7f738779..a1d59825b3b2 100644
--- a/polly/test/ForwardOpTree/changed-kind.ll
+++ b/polly/test/ForwardOpTree/changed-kind.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 
 ; In the code below, %0 is known to be equal to the content of @c (constant 0).
 ; Thus, in order to save a scalar dependency, forward-optree replaces
diff --git a/polly/test/ForwardOpTree/forward_from_region.ll b/polly/test/ForwardOpTree/forward_from_region.ll
index 90448d800286..53d22800081e 100644
--- a/polly/test/ForwardOpTree/forward_from_region.ll
+++ b/polly/test/ForwardOpTree/forward_from_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move instructions from region statements.
 ;
diff --git a/polly/test/ForwardOpTree/forward_hoisted.ll b/polly/test/ForwardOpTree/forward_hoisted.ll
index 163d43f56e8a..32fca00141dd 100644
--- a/polly/test/ForwardOpTree/forward_hoisted.ll
+++ b/polly/test/ForwardOpTree/forward_hoisted.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move %val to %bodyB, so %bodyA can be removed (by -polly-simplify).
 ; This involves making the load-hoisted %val1 to be made available in %bodyB.
diff --git a/polly/test/ForwardOpTree/forward_instruction.ll b/polly/test/ForwardOpTree/forward_instruction.ll
index 6269e359a59a..1dcd64357324 100644
--- a/polly/test/ForwardOpTree/forward_instruction.ll
+++ b/polly/test/ForwardOpTree/forward_instruction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move %val to %bodyB, so %bodyA can be removed (by -polly-simplify)
 ;
diff --git a/polly/test/ForwardOpTree/forward_into_region.ll b/polly/test/ForwardOpTree/forward_into_region.ll
index be102a9574dc..dd18cfe5e61a 100644
--- a/polly/test/ForwardOpTree/forward_into_region.ll
+++ b/polly/test/ForwardOpTree/forward_into_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move instructions to region statements.
 ;
diff --git a/polly/test/ForwardOpTree/forward_into_region_redundant_use.ll b/polly/test/ForwardOpTree/forward_into_region_redundant_use.ll
index 883a784230d9..e5458c027880 100644
--- a/polly/test/ForwardOpTree/forward_into_region_redundant_use.ll
+++ b/polly/test/ForwardOpTree/forward_into_region_redundant_use.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 
 define void @foo(ptr %A, i32 %p, ptr %B) {
diff --git a/polly/test/ForwardOpTree/forward_load.ll b/polly/test/ForwardOpTree/forward_load.ll
index dec6812aade5..86e3cb0203fa 100644
--- a/polly/test/ForwardOpTree/forward_load.ll
+++ b/polly/test/ForwardOpTree/forward_load.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-optree>)" -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-optree>)" -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load.
 ;
diff --git a/polly/test/ForwardOpTree/forward_load_differentarray.ll b/polly/test/ForwardOpTree/forward_load_differentarray.ll
index a3ca0bad54ab..786277bdeb87 100644
--- a/polly/test/ForwardOpTree/forward_load_differentarray.ll
+++ b/polly/test/ForwardOpTree/forward_load_differentarray.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; To forward %val, B[j] cannot be reused in bodyC because it is overwritten
 ; between. Verify that instead the alternative C[j] is used.
diff --git a/polly/test/ForwardOpTree/forward_load_double_write.ll b/polly/test/ForwardOpTree/forward_load_double_write.ll
index b0fbb69dc7ae..1618722381fc 100644
--- a/polly/test/ForwardOpTree/forward_load_double_write.ll
+++ b/polly/test/ForwardOpTree/forward_load_double_write.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load even in case two writes of identical values are in
 ; one scop statement.
diff --git a/polly/test/ForwardOpTree/forward_load_fromloop.ll b/polly/test/ForwardOpTree/forward_load_fromloop.ll
index 62351883a189..8f08a1356c38 100644
--- a/polly/test/ForwardOpTree/forward_load_fromloop.ll
+++ b/polly/test/ForwardOpTree/forward_load_fromloop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Forward a the LoadInst %val into %bodyB. %val is executed multiple times,
 ; we must get the last loaded values.
diff --git a/polly/test/ForwardOpTree/forward_load_indirect.ll b/polly/test/ForwardOpTree/forward_load_indirect.ll
index c8144861abc4..f83af61e6741 100644
--- a/polly/test/ForwardOpTree/forward_load_indirect.ll
+++ b/polly/test/ForwardOpTree/forward_load_indirect.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Forward an operand tree consisting of a speculatable instruction (%add)
 ; and a load (%val).
diff --git a/polly/test/ForwardOpTree/forward_load_memset_after.ll b/polly/test/ForwardOpTree/forward_load_memset_after.ll
index 22a2ddf94888..13797a44c862 100644
--- a/polly/test/ForwardOpTree/forward_load_memset_after.ll
+++ b/polly/test/ForwardOpTree/forward_load_memset_after.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load in the presence of a non-store WRITE access.
 ;
diff --git a/polly/test/ForwardOpTree/forward_load_memset_before.ll b/polly/test/ForwardOpTree/forward_load_memset_before.ll
index 3d3c90e941d6..60b1e076b980 100644
--- a/polly/test/ForwardOpTree/forward_load_memset_before.ll
+++ b/polly/test/ForwardOpTree/forward_load_memset_before.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load in the presence of a non-store WRITE access.
 ;
diff --git a/polly/test/ForwardOpTree/forward_load_tripleuse.ll b/polly/test/ForwardOpTree/forward_load_tripleuse.ll
index 03a47360c362..1d0df2a22e87 100644
--- a/polly/test/ForwardOpTree/forward_load_tripleuse.ll
+++ b/polly/test/ForwardOpTree/forward_load_tripleuse.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-optree>,polly-codegen' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-optree -polly-codegen -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; %val1 is used three times: Twice by its own operand tree of %val2 and once
 ; more by the store in %bodyB.
diff --git a/polly/test/ForwardOpTree/forward_load_unrelatedunusual.ll b/polly/test/ForwardOpTree/forward_load_unrelatedunusual.ll
index dbeebbc27eba..b7bae5628986 100644
--- a/polly/test/ForwardOpTree/forward_load_unrelatedunusual.ll
+++ b/polly/test/ForwardOpTree/forward_load_unrelatedunusual.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load.
 ; The non-analyzable store to C[0] is unrelated and can be ignored.
diff --git a/polly/test/ForwardOpTree/forward_phi_load.ll b/polly/test/ForwardOpTree/forward_phi_load.ll
index 029261f269c3..0b0bb209a3ef 100644
--- a/polly/test/ForwardOpTree/forward_phi_load.ll
+++ b/polly/test/ForwardOpTree/forward_phi_load.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-optree-normalize-phi=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-optree-normalize-phi=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load.
 ;
diff --git a/polly/test/ForwardOpTree/forward_readonly.ll b/polly/test/ForwardOpTree/forward_readonly.ll
index 7ded946a6ff8..a29c5bff5d70 100644
--- a/polly/test/ForwardOpTree/forward_readonly.ll
+++ b/polly/test/ForwardOpTree/forward_readonly.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-analyze-read-only-scalars=true  '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines -check-prefixes=STATS,MODEL
-; RUN: opt %loadPolly -polly-analyze-read-only-scalars=false '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines -check-prefixes=STATS,NOMODEL
+; RUN: opt %loadPolly -polly-analyze-read-only-scalars=true  -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines -check-prefixes=STATS,MODEL
+; RUN: opt %loadPolly -polly-analyze-read-only-scalars=false -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines -check-prefixes=STATS,NOMODEL
 ;
 ; Move %val to %bodyB, so %bodyA can be removed (by -polly-simplify)
 ;
diff --git a/polly/test/ForwardOpTree/forward_reusue.ll b/polly/test/ForwardOpTree/forward_reusue.ll
index 1151aa94e1f9..ead8c7379803 100644
--- a/polly/test/ForwardOpTree/forward_reusue.ll
+++ b/polly/test/ForwardOpTree/forward_reusue.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move operand tree without duplicating values used multiple times.
 ;
diff --git a/polly/test/ForwardOpTree/forward_store.ll b/polly/test/ForwardOpTree/forward_store.ll
index e02c6891f436..a6369eb303c1 100644
--- a/polly/test/ForwardOpTree/forward_store.ll
+++ b/polly/test/ForwardOpTree/forward_store.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Rematerialize a load.
 ;
diff --git a/polly/test/ForwardOpTree/forward_synthesizable_definloop.ll b/polly/test/ForwardOpTree/forward_synthesizable_definloop.ll
index 90e489ad6f54..f0da9320c43f 100644
--- a/polly/test/ForwardOpTree/forward_synthesizable_definloop.ll
+++ b/polly/test/ForwardOpTree/forward_synthesizable_definloop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Copy %val to bodyB, assuming the exit value of %i.
 ;
diff --git a/polly/test/ForwardOpTree/forward_synthesizable_indvar.ll b/polly/test/ForwardOpTree/forward_synthesizable_indvar.ll
index 395e68482657..a38ab543e255 100644
--- a/polly/test/ForwardOpTree/forward_synthesizable_indvar.ll
+++ b/polly/test/ForwardOpTree/forward_synthesizable_indvar.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Test support for (synthesizable) inducation variables.
 ;
diff --git a/polly/test/ForwardOpTree/forward_synthesizable_useinloop.ll b/polly/test/ForwardOpTree/forward_synthesizable_useinloop.ll
index a45d420e5c20..bb1760ae0ffb 100644
--- a/polly/test/ForwardOpTree/forward_synthesizable_useinloop.ll
+++ b/polly/test/ForwardOpTree/forward_synthesizable_useinloop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Synthesizable values defined outside of a loop can be used
 ; inside the loop.
diff --git a/polly/test/ForwardOpTree/forward_transitive.ll b/polly/test/ForwardOpTree/forward_transitive.ll
index 69cfb555f315..243889437149 100644
--- a/polly/test/ForwardOpTree/forward_transitive.ll
+++ b/polly/test/ForwardOpTree/forward_transitive.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Move %v and %val to %bodyB, so %bodyA can be removed (by -polly-simplify)
 ;
diff --git a/polly/test/ForwardOpTree/jacobi-1d.ll b/polly/test/ForwardOpTree/jacobi-1d.ll
index dbc051dde425..05ccd998c1a2 100644
--- a/polly/test/ForwardOpTree/jacobi-1d.ll
+++ b/polly/test/ForwardOpTree/jacobi-1d.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-optree-normalize-phi=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-optree-normalize-phi=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ForwardOpTree/noforward_from_region.ll b/polly/test/ForwardOpTree/noforward_from_region.ll
index 11d4312ad3bc..30150912f32e 100644
--- a/polly/test/ForwardOpTree/noforward_from_region.ll
+++ b/polly/test/ForwardOpTree/noforward_from_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Ensure we do not move instructions from region statements in case the
 ; instruction to move loads from an array which is also written to from
diff --git a/polly/test/ForwardOpTree/noforward_load_conditional.ll b/polly/test/ForwardOpTree/noforward_load_conditional.ll
index 053134196001..eaa0fc52186b 100644
--- a/polly/test/ForwardOpTree/noforward_load_conditional.ll
+++ b/polly/test/ForwardOpTree/noforward_load_conditional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; B[j] is overwritten by at least one statement between the
 ; definition of %val and its use. Hence, it cannot be forwarded.
diff --git a/polly/test/ForwardOpTree/noforward_load_writebetween.ll b/polly/test/ForwardOpTree/noforward_load_writebetween.ll
index 4a281b66c618..e2272c1c1f13 100644
--- a/polly/test/ForwardOpTree/noforward_load_writebetween.ll
+++ b/polly/test/ForwardOpTree/noforward_load_writebetween.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Cannot rematerialize %val from B[0] at bodyC because B[0] has been
 ; overwritten in bodyB.
diff --git a/polly/test/ForwardOpTree/noforward_outofquota.ll b/polly/test/ForwardOpTree/noforward_outofquota.ll
index 9c17349fb9e2..2ec965d71184 100644
--- a/polly/test/ForwardOpTree/noforward_outofquota.ll
+++ b/polly/test/ForwardOpTree/noforward_outofquota.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-optree-max-ops=1 '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
-; RUN: opt %loadPolly -polly-optree-max-ops=1 -passes=polly-optree -disable-output -stats < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=STATS
+; RUN: opt %loadPolly -polly-optree-max-ops=1 -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-optree-max-ops=1 -polly-optree -disable-output -stats < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=STATS
 ; REQUIRES: asserts
 ;
 ; for (int j = 0; j < n; j += 1) {
diff --git a/polly/test/ForwardOpTree/noforward_partial.ll b/polly/test/ForwardOpTree/noforward_partial.ll
index 67bda40337e0..127ac9ff5f14 100644
--- a/polly/test/ForwardOpTree/noforward_partial.ll
+++ b/polly/test/ForwardOpTree/noforward_partial.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Not the entire operand tree can be forwarded,
 ; some scalar dependencies would remain.
diff --git a/polly/test/ForwardOpTree/noforward_phi.ll b/polly/test/ForwardOpTree/noforward_phi.ll
index 455edfd1a831..58d41a410d3b 100644
--- a/polly/test/ForwardOpTree/noforward_phi.ll
+++ b/polly/test/ForwardOpTree/noforward_phi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Do not move PHI nodes.
 ;
diff --git a/polly/test/ForwardOpTree/noforward_selfrefphi.ll b/polly/test/ForwardOpTree/noforward_selfrefphi.ll
index e7ab21b6ba28..b2d4dc51c978 100644
--- a/polly/test/ForwardOpTree/noforward_selfrefphi.ll
+++ b/polly/test/ForwardOpTree/noforward_selfrefphi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-optree-normalize-phi=true '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-optree-normalize-phi=true -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Contains a self-referencing PHINode that would require a
 ; transitive closure to handle.
diff --git a/polly/test/ForwardOpTree/noforward_sideffects.ll b/polly/test/ForwardOpTree/noforward_sideffects.ll
index 0298da90e4ac..a5633769f670 100644
--- a/polly/test/ForwardOpTree/noforward_sideffects.ll
+++ b/polly/test/ForwardOpTree/noforward_sideffects.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly  '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly  -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Do not forward instructions with side-effects (here: function call).
 ;
diff --git a/polly/test/ForwardOpTree/noforward_synthesizable_unknownit.ll b/polly/test/ForwardOpTree/noforward_synthesizable_unknownit.ll
index 159972345bdc..f589fde6e415 100644
--- a/polly/test/ForwardOpTree/noforward_synthesizable_unknownit.ll
+++ b/polly/test/ForwardOpTree/noforward_synthesizable_unknownit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-optree -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Do not try to forward %i.trunc, it is not synthesizable in %body.
 ;
diff --git a/polly/test/ForwardOpTree/out-of-quota1.ll b/polly/test/ForwardOpTree/out-of-quota1.ll
index 5a69f6c3ad60..7afdb8e60244 100644
--- a/polly/test/ForwardOpTree/out-of-quota1.ll
+++ b/polly/test/ForwardOpTree/out-of-quota1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-optree>' -disable-output %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-optree -disable-output %s | FileCheck %s
 
 ; This used to loop infinitely because of UINT_MAX returned by ISL on out-of-quota.
 
diff --git a/polly/test/IstAstInfo/alias_checks_with_empty_context.ll b/polly/test/IstAstInfo/alias_checks_with_empty_context.ll
index d64f7529135e..9b95cd5b4bbd 100644
--- a/polly/test/IstAstInfo/alias_checks_with_empty_context.ll
+++ b/polly/test/IstAstInfo/alias_checks_with_empty_context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s \
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s \
 ; RUN:     | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/IstAstInfo/alias_simple_1.ll b/polly/test/IstAstInfo/alias_simple_1.ll
index 659c17879ed5..83d470c2d19b 100644
--- a/polly/test/IstAstInfo/alias_simple_1.ll
+++ b/polly/test/IstAstInfo/alias_simple_1.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline= -disable-output < %s | FileCheck %s --check-prefix=NOAA
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s --check-prefix=BASI
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -disable-output < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=BASI
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
 ;
 ;    int A[1024];
 ;
diff --git a/polly/test/IstAstInfo/alias_simple_2.ll b/polly/test/IstAstInfo/alias_simple_2.ll
index 569fe45e1e02..bbf528f93b47 100644
--- a/polly/test/IstAstInfo/alias_simple_2.ll
+++ b/polly/test/IstAstInfo/alias_simple_2.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline= -disable-output < %s | FileCheck %s --check-prefix=NOAA
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s --check-prefix=BASI
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=globals-aa -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -disable-output < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=BASI
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -globals-aa -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
 ;
 ;    int A[1024], B[1024];
 ;
diff --git a/polly/test/IstAstInfo/alias_simple_3.ll b/polly/test/IstAstInfo/alias_simple_3.ll
index 8bad170eda79..9067521323ab 100644
--- a/polly/test/IstAstInfo/alias_simple_3.ll
+++ b/polly/test/IstAstInfo/alias_simple_3.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline= -disable-output < %s | FileCheck %s --check-prefix=NOAA
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=basic-aa -disable-output < %s | FileCheck %s --check-prefix=BASI
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
-; RUN: opt %loadPolly '-passes=print<polly-ast>'  -aa-pipeline=globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -disable-output < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=BASI
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -scev-aa -disable-output < %s | FileCheck %s --check-prefix=SCEV
+; RUN: opt %loadPolly -polly-print-ast -disable-basic-aa -globals-aa -disable-output < %s | FileCheck %s --check-prefix=GLOB
 ;
 ;    int A[1024];
 ;    float B[1024];
diff --git a/polly/test/IstAstInfo/aliasing_arrays_with_identical_base.ll b/polly/test/IstAstInfo/aliasing_arrays_with_identical_base.ll
index 2bf71fb8fd2c..0cabd20168ba 100644
--- a/polly/test/IstAstInfo/aliasing_arrays_with_identical_base.ll
+++ b/polly/test/IstAstInfo/aliasing_arrays_with_identical_base.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s \
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s \
 ; RUN:   -polly-invariant-load-hoisting \
 ; RUN:   | FileCheck %s
 
diff --git a/polly/test/IstAstInfo/aliasing_multiple_alias_groups.ll b/polly/test/IstAstInfo/aliasing_multiple_alias_groups.ll
index 6cdc5b0fdcec..b824c211fd31 100644
--- a/polly/test/IstAstInfo/aliasing_multiple_alias_groups.ll
+++ b/polly/test/IstAstInfo/aliasing_multiple_alias_groups.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -aa-pipeline=      -disable-output < %s | FileCheck %s --check-prefix=NOAA
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -aa-pipeline=tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly -polly-print-ast       -disable-output < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly -polly-print-ast -tbaa -disable-output < %s | FileCheck %s --check-prefix=TBAA
 ;
 ;    void jd(int *Int0, int *Int1, float *Float0, float *Float1) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/IstAstInfo/aliasing_parametric_simple_1.ll b/polly/test/IstAstInfo/aliasing_parametric_simple_1.ll
index a63854d94b68..e0c3255dd766 100644
--- a/polly/test/IstAstInfo/aliasing_parametric_simple_1.ll
+++ b/polly/test/IstAstInfo/aliasing_parametric_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output %s | FileCheck %s
 ;
 ;    void jd(int *A, int *B, int c) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/IstAstInfo/aliasing_parametric_simple_2.ll b/polly/test/IstAstInfo/aliasing_parametric_simple_2.ll
index 7b2d163d54a2..74bad6c75784 100644
--- a/polly/test/IstAstInfo/aliasing_parametric_simple_2.ll
+++ b/polly/test/IstAstInfo/aliasing_parametric_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 ;
 ;    void jd(int *A, int *B, int c) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/IstAstInfo/dependence_distance_minimal.ll b/polly/test/IstAstInfo/dependence_distance_minimal.ll
index 4a77123e5031..c6b1d156e55d 100644
--- a/polly/test/IstAstInfo/dependence_distance_minimal.ll
+++ b/polly/test/IstAstInfo/dependence_distance_minimal.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; The minimal dependence distance of the innermost loop should be 1 instead of 250.
 ; CHECK:    #pragma minimal dependence distance: 1
diff --git a/polly/test/IstAstInfo/domain_bounded_only_with_context.ll b/polly/test/IstAstInfo/domain_bounded_only_with_context.ll
index bcf6fd394209..32cebd7a3a8b 100644
--- a/polly/test/IstAstInfo/domain_bounded_only_with_context.ll
+++ b/polly/test/IstAstInfo/domain_bounded_only_with_context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 
 ; CHECK:      {
 ; CHECK-NEXT:    if (p <= -1 || p >= 1)
diff --git a/polly/test/IstAstInfo/non_affine_access.ll b/polly/test/IstAstInfo/non_affine_access.ll
index b3f669ee4670..d8757b2e21cf 100644
--- a/polly/test/IstAstInfo/non_affine_access.ll
+++ b/polly/test/IstAstInfo/non_affine_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-print-accesses -polly-allow-nonaffine -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-ast-print-accesses -polly-allow-nonaffine -disable-output < %s | FileCheck %s
 ;
 ;    void non_affine_access(float A[]) {
 ;      for (long i = 0; i < 1024; i++)
diff --git a/polly/test/IstAstInfo/reduction_clauses_onedimensional_access.ll b/polly/test/IstAstInfo/reduction_clauses_onedimensional_access.ll
index ea47e63bb7a6..8d52e345a76d 100644
--- a/polly/test/IstAstInfo/reduction_clauses_onedimensional_access.ll
+++ b/polly/test/IstAstInfo/reduction_clauses_onedimensional_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK: #pragma known-parallel reduction (^ : MemRef_sum)
 ;        void f(int N, int M, int *sum) {
diff --git a/polly/test/IstAstInfo/reduction_dependences_equal_non_reduction_dependences.ll b/polly/test/IstAstInfo/reduction_dependences_equal_non_reduction_dependences.ll
index 6650fc034f87..9c6eea6aaa1e 100644
--- a/polly/test/IstAstInfo/reduction_dependences_equal_non_reduction_dependences.ll
+++ b/polly/test/IstAstInfo/reduction_dependences_equal_non_reduction_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; This loopnest contains a reduction which imposes the same dependences as the
 ; accesses to the array A. We need to ensure we do __not__ parallelize anything
diff --git a/polly/test/IstAstInfo/reduction_different_reduction_clauses.ll b/polly/test/IstAstInfo/reduction_different_reduction_clauses.ll
index d89953cdd2fb..5104f716d810 100644
--- a/polly/test/IstAstInfo/reduction_different_reduction_clauses.ll
+++ b/polly/test/IstAstInfo/reduction_different_reduction_clauses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK: #pragma simd reduction (+ : MemRef_sum{{[1,2]}}, MemRef_sum{{[1,2]}}) reduction (* : MemRef_prod) reduction (| : MemRef_or) reduction (& : MemRef_and)
 ; CHECK: #pragma known-parallel reduction (+ : MemRef_sum{{[1,2]}}, MemRef_sum{{[1,2]}}) reduction (* : MemRef_prod) reduction (| : MemRef_or) reduction (& : MemRef_and)
diff --git a/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule.ll b/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule.ll
index a8ce13b4c56d..8a42cf8bd165 100644
--- a/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK:          #pragma known-parallel reduction (+ : MemRef_A)
 ; CHECK-NEXT:     for (int c0 = 0; c0 <= 2; c0 += 1) {
diff --git a/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule_2.ll b/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule_2.ll
index 535ec397969d..8f5efd165546 100644
--- a/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule_2.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_and_loop_reversal_schedule_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    #pragma known-parallel reduction
 ; CHECK:    for (int c0 = 0; c0 <= 2; c0 += 1) {
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions.ll
index 0ea3916fb274..a711a36a367f 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    #pragma known-parallel
 ; CHECK:    for (int c0 = 0; c0 <= 1; c0 += 1)
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll
index 703b0f3e04ea..485d6965b6d3 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; Verify that the outer dimension doesnt't carry reduction dependences
 ;
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll
index 3a847e939b5b..375fabbf6a8b 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; Verify that the outer dimension doesnt't carry reduction dependences
 ;
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll
index f2691bc6a503..584c076dcff4 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; Verify that the outer dimension doesnt't carry reduction dependences
 ;
diff --git a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_5.ll b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_5.ll
index 480af5505a33..eaa3444a04d7 100644
--- a/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_5.ll
+++ b/polly/test/IstAstInfo/reduction_modulo_schedule_multiple_dimensions_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; Verify that only the outer dimension needs privatization
 ;
diff --git a/polly/test/IstAstInfo/reduction_multiple_dimensions.ll b/polly/test/IstAstInfo/reduction_multiple_dimensions.ll
index 96ecb0c078fe..9618ec872c38 100644
--- a/polly/test/IstAstInfo/reduction_multiple_dimensions.ll
+++ b/polly/test/IstAstInfo/reduction_multiple_dimensions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT:#pragma known-parallel reduction
 ; CHECK:    #pragma known-parallel
diff --git a/polly/test/IstAstInfo/reduction_multiple_dimensions_2.ll b/polly/test/IstAstInfo/reduction_multiple_dimensions_2.ll
index d2232ca88143..af317570eb37 100644
--- a/polly/test/IstAstInfo/reduction_multiple_dimensions_2.ll
+++ b/polly/test/IstAstInfo/reduction_multiple_dimensions_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT:#pragma known-parallel reduction
 ; CHECK:    #pragma known-parallel
diff --git a/polly/test/IstAstInfo/reduction_multiple_dimensions_3.ll b/polly/test/IstAstInfo/reduction_multiple_dimensions_3.ll
index dfc1682bd95b..1f7191433bf8 100644
--- a/polly/test/IstAstInfo/reduction_multiple_dimensions_3.ll
+++ b/polly/test/IstAstInfo/reduction_multiple_dimensions_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT:#pragma known-parallel reduction
 ; CHECK:    #pragma known-parallel
diff --git a/polly/test/IstAstInfo/reduction_multiple_dimensions_4.ll b/polly/test/IstAstInfo/reduction_multiple_dimensions_4.ll
index d0bad81efb2f..40bae5e9ac6c 100644
--- a/polly/test/IstAstInfo/reduction_multiple_dimensions_4.ll
+++ b/polly/test/IstAstInfo/reduction_multiple_dimensions_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-ast-detect-parallel -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT:#pragma known-parallel reduction
 ; CHECK:    #pragma known-parallel
diff --git a/polly/test/IstAstInfo/run-time-condition.ll b/polly/test/IstAstInfo/run-time-condition.ll
index c3ea8c460b6d..ccc9c7cfd321 100644
--- a/polly/test/IstAstInfo/run-time-condition.ll
+++ b/polly/test/IstAstInfo/run-time-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-ast -disable-output < %s | FileCheck %s
 
 ; for (i = 0; i < 1024; i++)
 ;   A[i] = B[i];
diff --git a/polly/test/IstAstInfo/runtime_context_with_error_blocks.ll b/polly/test/IstAstInfo/runtime_context_with_error_blocks.ll
index 26a242b00ee7..2853e0acf9b8 100644
--- a/polly/test/IstAstInfo/runtime_context_with_error_blocks.ll
+++ b/polly/test/IstAstInfo/runtime_context_with_error_blocks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; Verify we do not simplify the runtime check to "true" due to the domain
 ; constraints as the test contains an error block that influenced the domains
diff --git a/polly/test/IstAstInfo/simple-run-time-condition.ll b/polly/test/IstAstInfo/simple-run-time-condition.ll
index c6a6f027652f..5fb99f0676b7 100644
--- a/polly/test/IstAstInfo/simple-run-time-condition.ll
+++ b/polly/test/IstAstInfo/simple-run-time-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-precise-inbounds -polly-precise-fold-accesses -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-precise-inbounds -polly-precise-fold-accesses -disable-output < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/polly/test/IstAstInfo/single_loop_strip_mine.ll b/polly/test/IstAstInfo/single_loop_strip_mine.ll
index 4405ee1dc138..1c627f817b0b 100644
--- a/polly/test/IstAstInfo/single_loop_strip_mine.ll
+++ b/polly/test/IstAstInfo/single_loop_strip_mine.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-ast-print-accesses -polly-ast-detect-parallel '-passes=polly-import-jscop,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=CHECK-VECTOR
+; RUN: opt %loadPolly -basic-aa -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-import-jscop -polly-ast-print-accesses -polly-ast-detect-parallel -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=CHECK-VECTOR
 
 ; for (i = 0; i < 1024; i++)
 ;   A[i] = B[i];
diff --git a/polly/test/IstAstInfo/single_loop_uint_max_iterations.ll b/polly/test/IstAstInfo/single_loop_uint_max_iterations.ll
index 3f673584a268..f1cd5dae11ce 100644
--- a/polly/test/IstAstInfo/single_loop_uint_max_iterations.ll
+++ b/polly/test/IstAstInfo/single_loop_uint_max_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 ; XFAIL: *
 
 ;#include "limits.h"
diff --git a/polly/test/IstAstInfo/single_loop_ull_max_iterations.ll b/polly/test/IstAstInfo/single_loop_ull_max_iterations.ll
index ff56cfb11780..d421e221240a 100644
--- a/polly/test/IstAstInfo/single_loop_ull_max_iterations.ll
+++ b/polly/test/IstAstInfo/single_loop_ull_max_iterations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s
 ; XFAIL: *
 
 ;#include "limits.h"
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Bad-relation.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Bad-relation.ll
index 545a88909bf6..d4a1a6222518 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Bad-relation.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Bad-relation.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: expecting other token
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-No-accesses-key.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-No-accesses-key.ll
index f9f13b4cff58..43f9d3eda049 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-No-accesses-key.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-No-accesses-key.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: Statement from JScop file has no key name 'accesses' for index 1.
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-MemAcc.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-MemAcc.ll
index 6031465cd03f..24ad03741216 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-MemAcc.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-MemAcc.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: The number of memory accesses in the JSop file and the number of memory accesses differ for index 0.
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-statements.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-statements.ll
index fc67ec51f218..1060926e7fac 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-statements.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Not-enough-statements.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: The number of indices and the number of statements differ.
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Relation-mispelled.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Relation-mispelled.ll
index 64b7b14c1939..07975976c38b 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Relation-mispelled.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Relation-mispelled.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: Memory access number 0 has no key name 'relation' for statement number 1.
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Statements-mispelled.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Statements-mispelled.ll
index bd58cb3be5c3..9f7259633811 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Statements-mispelled.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Statements-mispelled.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: JScop file has no key name 'statements'.
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Undeclared-ScopArrayInfo.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Undeclared-ScopArrayInfo.ll
index 565117584861..df7eb42da85f 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Undeclared-ScopArrayInfo.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Undeclared-ScopArrayInfo.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: JScop file contains access function with undeclared ScopArrayInfo
 ;
diff --git a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Wrong-number-dimensions.ll b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Wrong-number-dimensions.ll
index 4a7d51019133..61c1173db2e7 100644
--- a/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Wrong-number-dimensions.ll
+++ b/polly/test/JSONExporter/ImportAccesses/ImportAccesses-Wrong-number-dimensions.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: JScop file changes the number of parameter dimensions.
 ;
diff --git a/polly/test/JSONExporter/ImportArrays/ImportArrays-Mispelled-type.ll b/polly/test/JSONExporter/ImportArrays/ImportArrays-Mispelled-type.ll
index 1dd71ba5816a..a14ae5c4d1bc 100644
--- a/polly/test/JSONExporter/ImportArrays/ImportArrays-Mispelled-type.ll
+++ b/polly/test/JSONExporter/ImportArrays/ImportArrays-Mispelled-type.ll
@@ -1,4 +1,4 @@
-  ; RUN: not --crash opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
+  ; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Array has not a valid type.
 ;
diff --git a/polly/test/JSONExporter/ImportArrays/ImportArrays-Negative-size.ll b/polly/test/JSONExporter/ImportArrays/ImportArrays-Negative-size.ll
index 5270cc8c680f..2a03197f1c1b 100644
--- a/polly/test/JSONExporter/ImportArrays/ImportArrays-Negative-size.ll
+++ b/polly/test/JSONExporter/ImportArrays/ImportArrays-Negative-size.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -polly-stmt-granularity=bb -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; #define Ni 1056
 ; #define Nj 1056
diff --git a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-name.ll b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-name.ll
index 21c4a736a25e..45bb3495de08 100644
--- a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-name.ll
+++ b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-name.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Array has no key 'name'.
 ;
diff --git a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-sizes-key.ll b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-sizes-key.ll
index 930bfce45df0..5bbb974346ba 100644
--- a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-sizes-key.ll
+++ b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-sizes-key.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Array has no key 'sizes'.
 ;
diff --git a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-type-key.ll b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-type-key.ll
index eb13390dd2b9..af013992fca0 100644
--- a/polly/test/JSONExporter/ImportArrays/ImportArrays-No-type-key.ll
+++ b/polly/test/JSONExporter/ImportArrays/ImportArrays-No-type-key.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Array has no key 'type'.
 ;
diff --git a/polly/test/JSONExporter/ImportContext/ImportContext-Context-mispelled.ll b/polly/test/JSONExporter/ImportContext/ImportContext-Context-mispelled.ll
index 79e175378af6..2490e44ec347 100644
--- a/polly/test/JSONExporter/ImportContext/ImportContext-Context-mispelled.ll
+++ b/polly/test/JSONExporter/ImportContext/ImportContext-Context-mispelled.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: JScop file has no key named 'context'.
 ;
diff --git a/polly/test/JSONExporter/ImportContext/ImportContext-Not-parameter-set.ll b/polly/test/JSONExporter/ImportContext/ImportContext-Not-parameter-set.ll
index 96539188cdff..66ce6a6ed922 100644
--- a/polly/test/JSONExporter/ImportContext/ImportContext-Not-parameter-set.ll
+++ b/polly/test/JSONExporter/ImportContext/ImportContext-Not-parameter-set.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: The isl_set is not a parameter set.
 ;
diff --git a/polly/test/JSONExporter/ImportContext/ImportContext-Unvalid-Context.ll b/polly/test/JSONExporter/ImportContext/ImportContext-Unvalid-Context.ll
index 0e80d3623c10..7bcc54dde52e 100644
--- a/polly/test/JSONExporter/ImportContext/ImportContext-Unvalid-Context.ll
+++ b/polly/test/JSONExporter/ImportContext/ImportContext-Unvalid-Context.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: unexpected isl_token
 ;
diff --git a/polly/test/JSONExporter/ImportContext/ImportContext-Wrong-dimension.ll b/polly/test/JSONExporter/ImportContext/ImportContext-Wrong-dimension.ll
index 9a04d89b18d1..65cdcbdcdef6 100644
--- a/polly/test/JSONExporter/ImportContext/ImportContext-Wrong-dimension.ll
+++ b/polly/test/JSONExporter/ImportContext/ImportContext-Wrong-dimension.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: Imported context has the wrong number of parameters : Found 2 Expected 1
 ;
diff --git a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-No-schedule-key.ll b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-No-schedule-key.ll
index 6347d17da4cc..b52db0876cc5 100644
--- a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-No-schedule-key.ll
+++ b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-No-schedule-key.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: Statement 0 has no 'schedule' key.
 ;
diff --git a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Schedule-not-valid.ll b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Schedule-not-valid.ll
index b6f4d188fad0..5ce3ad267bb0 100644
--- a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Schedule-not-valid.ll
+++ b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Schedule-not-valid.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: expecting other token
 ;
diff --git a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Statements-mispelled.ll b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Statements-mispelled.ll
index 9c325f9bfb77..4329653899b2 100644
--- a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Statements-mispelled.ll
+++ b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Statements-mispelled.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: JScop file has no key name 'statements'.
 ;
diff --git a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Wrong-number-statements.ll b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Wrong-number-statements.ll
index a6a3c7d35668..f66fc6c1e5d7 100644
--- a/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Wrong-number-statements.ll
+++ b/polly/test/JSONExporter/ImportSchedule/ImportSchedule-Wrong-number-statements.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash opt %loadPolly '-passes=polly-import-jscop,print<polly-ast>' -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
+; RUN: not --crash opt %loadPolly -polly-import-jscop -polly-ast -polly-ast-detect-parallel -disable-output < %s 2>&1 >/dev/null | FileCheck %s
 ;
 ; CHECK: The number of indices and the number of statements differ.
 ;
diff --git a/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll b/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll
index a3611f0b89f3..791210f7710d 100644
--- a/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll
+++ b/polly/test/MaximalStaticExpansion/load_after_store_same_statement.ll
@@ -1,5 +1,7 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1| FileCheck %s --check-prefix=MSE
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that the expansion of an array with load after store in a same statement is not done.
 ;
diff --git a/polly/test/MaximalStaticExpansion/read_from_original.ll b/polly/test/MaximalStaticExpansion/read_from_original.ll
index fe9c5850bc5f..59f9379516c7 100644
--- a/polly/test/MaximalStaticExpansion/read_from_original.ll
+++ b/polly/test/MaximalStaticExpansion/read_from_original.ll
@@ -1,5 +1,7 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1| FileCheck %s --check-prefix=MSE
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that Polly detects problems and does not expand the array
 ;
diff --git a/polly/test/MaximalStaticExpansion/too_many_writes.ll b/polly/test/MaximalStaticExpansion/too_many_writes.ll
index 01cdd2b46682..50a66cd11d0a 100644
--- a/polly/test/MaximalStaticExpansion/too_many_writes.ll
+++ b/polly/test/MaximalStaticExpansion/too_many_writes.ll
@@ -1,5 +1,7 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that Polly detects problems and does not expand the array
 ;
diff --git a/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll b/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll
index 3b9f951be2b4..8e2707cfee64 100644
--- a/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll
+++ b/polly/test/MaximalStaticExpansion/working_deps_between_inners.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::Array
 ;
diff --git a/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll b/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll
index 32ed2b99bde2..2bf49b89db05 100644
--- a/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll
+++ b/polly/test/MaximalStaticExpansion/working_deps_between_inners_phi.ll
@@ -1,5 +1,7 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::Array and MemoryKind::PHI.
 ; tmp_06_phi is not expanded because it need copy in.
diff --git a/polly/test/MaximalStaticExpansion/working_expansion.ll b/polly/test/MaximalStaticExpansion/working_expansion.ll
index 29ac90174f88..bb5b2360143f 100644
--- a/polly/test/MaximalStaticExpansion/working_expansion.ll
+++ b/polly/test/MaximalStaticExpansion/working_expansion.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::Array
 ;
diff --git a/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll b/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll
index 6ef2f298adb9..89ff7890fc7e 100644
--- a/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll
+++ b/polly/test/MaximalStaticExpansion/working_expansion_multiple_dependences_per_statement.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
 ;
 ; Verify that the accesses are correctly expanded
 ;
diff --git a/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll b/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll
index 6c7ea23d2f53..7ffd39f0f534 100644
--- a/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll
+++ b/polly/test/MaximalStaticExpansion/working_expansion_multiple_instruction_per_statement.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
 ;
 ; Verify that the accesses are correctly expanded
 ;
diff --git a/polly/test/MaximalStaticExpansion/working_phi_expansion.ll b/polly/test/MaximalStaticExpansion/working_phi_expansion.ll
index 0d4f18f21ade..43919c61b045 100644
--- a/polly/test/MaximalStaticExpansion/working_phi_expansion.ll
+++ b/polly/test/MaximalStaticExpansion/working_phi_expansion.ll
@@ -1,5 +1,7 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::PHI
 ; tmp_04 is not expanded because it need copy-in.
diff --git a/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll b/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll
index 93e984b95c95..a581a389e742 100644
--- a/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll
+++ b/polly/test/MaximalStaticExpansion/working_phi_two_scalars.ll
@@ -1,5 +1,7 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-mse -polly-print-scops -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-mse>)" -pass-remarks-analysis="polly-mse" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=MSE
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::PHI
 ; tmp_05 and tmp2_06 are not expanded because they need copy-in.
diff --git a/polly/test/MaximalStaticExpansion/working_value_expansion.ll b/polly/test/MaximalStaticExpansion/working_value_expansion.ll
index 27c4304d2fe5..d54eff9e03ec 100644
--- a/polly/test/MaximalStaticExpansion/working_value_expansion.ll
+++ b/polly/test/MaximalStaticExpansion/working_value_expansion.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-mse -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-mse>)" -disable-output < %s | FileCheck %s
 ;
 ; Verify that the accesses are correctly expanded for MemoryKind::Value
 ;
diff --git a/polly/test/PruneUnprofitable/prune_only_scalardeps.ll b/polly/test/PruneUnprofitable/prune_only_scalardeps.ll
index c64512fc3d7a..31db5560c051 100644
--- a/polly/test/PruneUnprofitable/prune_only_scalardeps.ll
+++ b/polly/test/PruneUnprofitable/prune_only_scalardeps.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=false "-passes=scop(polly-prune-unprofitable)" -disable-output -stats < %s 2>&1 | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=false -polly-prune-unprofitable -disable-output -stats < %s 2>&1 | FileCheck -match-full-lines %s
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=false "-passes=scop(polly-prune-unprofitable)" -disable-output -stats < %s 2>&1 | FileCheck -match-full-lines %s
 ; REQUIRES: asserts
 ;
 ; Skip this SCoP for having scalar dependencies between all statements,
diff --git a/polly/test/ScheduleOptimizer/2012-03-16-Empty-Domain.ll b/polly/test/ScheduleOptimizer/2012-03-16-Empty-Domain.ll
index b9baa35b0cf8..5acc35343ac3 100644
--- a/polly/test/ScheduleOptimizer/2012-03-16-Empty-Domain.ll
+++ b/polly/test/ScheduleOptimizer/2012-03-16-Empty-Domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -S < %s
+; RUN: opt %loadPolly -polly-opt-isl -S < %s
 target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
 
 define void @sdbout_label() nounwind {
diff --git a/polly/test/ScheduleOptimizer/2013-04-11-Empty-Domain-two.ll b/polly/test/ScheduleOptimizer/2013-04-11-Empty-Domain-two.ll
index e3e1d61f74c1..3f4237b330b2 100644
--- a/polly/test/ScheduleOptimizer/2013-04-11-Empty-Domain-two.ll
+++ b/polly/test/ScheduleOptimizer/2013-04-11-Empty-Domain-two.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -S < %s
+; RUN: opt %loadPolly -polly-opt-isl -S < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Check that we handle statements with an empty iteration domain correctly.
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-double.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-double.ll
index b1acd130acb6..a61af2d092f3 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-double.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-double.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
 
 define void @func(i32 %n, ptr  noalias nonnull %A,  ptr  noalias nonnull %B) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-first.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-first.ll
index 4590b01f6112..185d5c5b8c25 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-first.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-first.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,OPT
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,OPT
 
 define void @func(i32 %n, ptr noalias nonnull %A, ptr noalias nonnull %B, i32 %k) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-third.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-third.ll
index a9afceac3880..f1eca0ede061 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-third.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-except-third.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK
 
 define void @func(i32 %n, ptr noalias nonnull %A, ptr noalias nonnull %B, i32 %k) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-carried.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-carried.ll
index 16dc6d8bc673..35903ced7741 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-carried.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-carried.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,OPT
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,OPT
 
 define void @func(i32 %n, ptr noalias nonnull %A) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-third.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-third.ll
index 54611c7a1cb8..1fb8c001069f 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-third.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner-third.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK,RAW
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefixes=CHECK
 
 define void @func(i32 %n, ptr noalias nonnull %A, ptr noalias nonnull %B, i32 %k) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner.ll
index 905a7804215f..2db6833fa897 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-inner.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
 
 define void @func(i32 %n, ptr noalias nonnull %A) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-simple.ll b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-simple.ll
index 32be9181c66b..49d008ba2cfa 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/fuse-simple.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/fuse-simple.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
 
 define void @func(i32 %n, ptr noalias nonnull %A) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-simple.ll b/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-simple.ll
index d15d69e69fac..175b85997ec0 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-simple.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-simple.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
 
 ; This could theoretically be fused by adjusting the offset of the second loop by %k (instead of relying on schedule dimensions).
 
diff --git a/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-with-middle.ll b/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-with-middle.ll
index 5dc5330db107..48ba20347d55 100644
--- a/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-with-middle.ll
+++ b/polly/test/ScheduleOptimizer/GreedyFuse/nofuse-with-middle.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-reschedule=1 -polly-loopfusion-greedy=1 -polly-postopts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
 
 define void @func(i32 %n, ptr noalias nonnull %A, ptr noalias nonnull %B, i32 %k) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/disable_nonforced.ll b/polly/test/ScheduleOptimizer/ManualOptimization/disable_nonforced.ll
index 8676b2dd7e18..537721f8718a 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/disable_nonforced.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/disable_nonforced.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-opt-isl -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Check that the disable_nonforced metadata is honored; optimization
 ; heuristics/rescheduling must not be applied.
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_heuristic.ll b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_heuristic.ll
index 90dc071ae8f6..aaf4d27f4c5e 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_heuristic.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_heuristic.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-pragma-based-opts=1 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines --check-prefix=ON
-; RUN: opt %loadPolly -polly-reschedule=0 -polly-pragma-based-opts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines --check-prefix=OFF
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-pragma-based-opts=1 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines --check-prefix=ON
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-pragma-based-opts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines --check-prefix=OFF
 ;
 define void @func(i32 %n, ptr noalias nonnull %A, ptr noalias nonnull %B) {
 entry:
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_looploc.ll b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_looploc.ll
index f513835c60eb..b1e94227c9a5 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_looploc.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_looploc.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-reschedule=0 -polly-pragma-based-opts=1 -disable-output < %s 2>&1 | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-opt-isl -polly-reschedule=0 -polly-pragma-based-opts=1 -disable-output < %s 2>&1 | FileCheck %s --match-full-lines
 ;
 ; CHECK: warning: distribute_illegal.c:2:3: not applying loop fission/distribution: cannot ensure semantic equivalence due to possible dependency violations
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_pragmaloc.ll b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_pragmaloc.ll
index c18b9bb72ad7..fc0df85b1346 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_pragmaloc.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/distribute_illegal_pragmaloc.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-reschedule=0 -polly-pragma-based-opts=1 -disable-output < %s 2>&1 | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-opt-isl -polly-reschedule=0 -polly-pragma-based-opts=1 -disable-output < %s 2>&1 | FileCheck %s --match-full-lines
 ;
 ; CHECK: warning: distribute_illegal.c:1:42: not applying loop fission/distribution: cannot ensure semantic equivalence due to possible dependency violations
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_disable.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_disable.ll
index 20f5a4538b16..9537f3a9b0a8 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_disable.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_disable.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-pragma-based-opts=1 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-pragma-based-opts=1 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
 ;
 ; Override unroll metadata with llvm.loop.unroll.disable.
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll
index afc39a72da1d..b0310970f8d6 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
 ;
 ; Apply two loop transformations. First partial, then full unrolling.
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_full.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_full.ll
index a166421ca21a..b9a4c845477c 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_full.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_full.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
 ;
 ; Full unroll of a loop with 5 iterations.
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_heuristic.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_heuristic.ll
index 68ee147895cf..0387aecd683b 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_heuristic.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_heuristic.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-pragma-based-opts=1 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
-; RUN: opt %loadPolly -polly-pragma-based-opts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-pragma-based-opts=1 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-pragma-based-opts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
 ;
 ; Unrolling with heuristic factor.
 ; Currently not supported and expected to be handled by LLVM's unroll pass.
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial.ll
index 042e1b4e088a..81e40f0a98bb 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-pragma-based-opts=1 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --match-full-lines
-; RUN: opt %loadPolly -polly-pragma-based-opts=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefix=OFF --match-full-lines
+; RUN: opt %loadPolly -polly-pragma-based-opts=1 -polly-print-opt-isl -disable-output < %s | FileCheck %s --match-full-lines
+; RUN: opt %loadPolly -polly-pragma-based-opts=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefix=OFF --match-full-lines
 ;
 ; Partial unroll by a factor of 4.
 ;
diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial_followup.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial_followup.ll
index 893d40a41777..8665f68b99c1 100644
--- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial_followup.ll
+++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_partial_followup.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s --check-prefix=OPT --match-full-lines
-; RUN: opt %loadPolly '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=AST --match-full-lines
-; RUN: opt %loadPolly '-passes=scop(polly-opt-isl,polly-codegen),simplifycfg' -S < %s | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-opt-isl -disable-output < %s | FileCheck %s --check-prefix=OPT --match-full-lines
+; RUN: opt %loadPolly -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST --match-full-lines
+; RUN: opt %loadPolly -polly-opt-isl -polly-codegen -simplifycfg -S < %s | FileCheck %s --check-prefix=CODEGEN
 ;
 ; Partial unroll by a factor of 4.
 ;
@@ -49,7 +49,7 @@ return:
 ; OPT-NEXT:        - filter: "[n] -> { Stmt_body[i0] : (1 + i0) mod 4 = 0 }"
 
 
-; AST-LABEL: :: isl ast :: func :: %for---%return
+; AST-LABEL: Printing analysis 'Polly - Generate an AST of the SCoP (isl)'for => return' in function 'func':
 ; AST:       // Loop with Metadata
 ; AST-NEXT:  for (int c0 = 0; c0 < n; c0 += 4) {
 
diff --git a/polly/test/ScheduleOptimizer/SIMDInParallelFor.ll b/polly/test/ScheduleOptimizer/SIMDInParallelFor.ll
index b0385d50c6c6..8585634e10ff 100644
--- a/polly/test/ScheduleOptimizer/SIMDInParallelFor.ll
+++ b/polly/test/ScheduleOptimizer/SIMDInParallelFor.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-parallel -polly-vectorizer=stripmine -passes=polly-codegen-verify '-passes=polly-opt-isl,print<polly-ast>,polly-codegen' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-parallel -polly-vectorizer=stripmine -polly-codegen-verify -polly-opt-isl -polly-print-ast -polly-codegen -disable-output < %s | FileCheck %s
 ;
 ; Check that there are no nested #pragma omp parallel for inside a
 ; #pragma omp parallel for loop.
diff --git a/polly/test/ScheduleOptimizer/computeout.ll b/polly/test/ScheduleOptimizer/computeout.ll
index 1cf6513e7a5c..35e3416f91d1 100644
--- a/polly/test/ScheduleOptimizer/computeout.ll
+++ b/polly/test/ScheduleOptimizer/computeout.ll
@@ -1,5 +1,7 @@
-; RUN: opt -S %loadPolly "-passes=scop(polly-opt-isl,print<polly-ast>)" -polly-isl-arg=--no-schedule-serialize-sccs -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadPolly "-passes=scop(polly-opt-isl,print<polly-ast>)" -polly-isl-arg=--no-schedule-serialize-sccs -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
+; RUN: opt -S %loadPolly -basic-aa -polly-opt-isl -polly-isl-arg=--no-schedule-serialize-sccs -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadNPMPolly "-passes=scop(polly-opt-isl,print<polly-ast>)" -polly-isl-arg=--no-schedule-serialize-sccs -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -basic-aa -polly-opt-isl -polly-isl-arg=--schedule-serialize-sccs -polly-dependences-computeout=1 -polly-print-ast -disable-output  < %s | FileCheck %s -check-prefix=TIMEOUT
+; RUN: opt -S %loadNPMPolly "-passes=scop(polly-opt-isl,print<polly-ast>)" -polly-isl-arg=--no-schedule-serialize-sccs -polly-dependences-computeout=1 -disable-output < %s | FileCheck %s -check-prefix=TIMEOUT
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ;     for(i = 0; i < 100; i++ )
diff --git a/polly/test/ScheduleOptimizer/ensure-correct-tile-sizes.ll b/polly/test/ScheduleOptimizer/ensure-correct-tile-sizes.ll
index 87f6c6c4eee6..43caca5372ad 100644
--- a/polly/test/ScheduleOptimizer/ensure-correct-tile-sizes.ll
+++ b/polly/test/ScheduleOptimizer/ensure-correct-tile-sizes.ll
@@ -1,9 +1,9 @@
 ; RUN: opt %loadPolly -polly-process-unprofitable -polly-remarks-minimal \
-; RUN:     '-passes=polly-opt-isl,print<polly-ast>' -polly-pattern-matching-based-opts=true \
+; RUN:     -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN:     -polly-target-throughput-vector-fma=1 \
 ; RUN:     -polly-target-latency-vector-fma=1 \
 ; RUN:     -polly-target-vector-register-bitwidth=4096 \
-; RUN:     -polly-target-1st-cache-level-associativity=3 -disable-output < %s | FileCheck %s
+; RUN:     -polly-target-1st-cache-level-associativity=3 -polly-print-ast -disable-output < %s | FileCheck %s
 ;
 ;     /* Test that Polly does not crash due to configurations that can lead to
 ;    incorrect tile size computations.
diff --git a/polly/test/ScheduleOptimizer/focaltech_test_detail_threshold-7bc17e.ll b/polly/test/ScheduleOptimizer/focaltech_test_detail_threshold-7bc17e.ll
index 483737ee5928..daa1afdd0aa8 100644
--- a/polly/test/ScheduleOptimizer/focaltech_test_detail_threshold-7bc17e.ll
+++ b/polly/test/ScheduleOptimizer/focaltech_test_detail_threshold-7bc17e.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-opt-isl>)" -polly-vectorizer=stripmine -polly-invariant-load-hoisting -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-vectorizer=stripmine -polly-invariant-load-hoisting -polly-optimized-scops -polly-print-opt-isl -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-opt-isl>)" -polly-vectorizer=stripmine -polly-invariant-load-hoisting -disable-output < %s | FileCheck %s
 ;
 ; llvm.org/PR46578
 ;
diff --git a/polly/test/ScheduleOptimizer/full_partial_tile_separation.ll b/polly/test/ScheduleOptimizer/full_partial_tile_separation.ll
index 9c4627717ee8..06e86d7da1c6 100644
--- a/polly/test/ScheduleOptimizer/full_partial_tile_separation.ll
+++ b/polly/test/ScheduleOptimizer/full_partial_tile_separation.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S %loadPolly -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 ; CHECK:          // 1st level tiling - Tiles
 ; CHECK-NEXT:    #pragma known-parallel
 ; CHECK-NEXT:    for (int c0 = 0; c0 <= floord(ni - 1, 32); c0 += 1)
diff --git a/polly/test/ScheduleOptimizer/line-tiling-2.ll b/polly/test/ScheduleOptimizer/line-tiling-2.ll
index d3d11a7990a6..eb374cb07cf3 100644
--- a/polly/test/ScheduleOptimizer/line-tiling-2.ll
+++ b/polly/test/ScheduleOptimizer/line-tiling-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-tile-sizes=1,64 '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-tile-sizes=1,64 -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 
 ; CHECK: for (int c0 = 0; c0 <= 1023; c0 += 1)
 ; CHECK:   for (int c1 = 0; c1 <= 7; c1 += 1)
diff --git a/polly/test/ScheduleOptimizer/line-tiling.ll b/polly/test/ScheduleOptimizer/line-tiling.ll
index 273a27ba5931..2f14ac1d02a5 100644
--- a/polly/test/ScheduleOptimizer/line-tiling.ll
+++ b/polly/test/ScheduleOptimizer/line-tiling.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-tile-sizes=64,1 '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-tile-sizes=64,1 -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 
 ; CHECK: for (int c0 = 0; c0 <= 15; c0 += 1)
 ; CHECK:   for (int c1 = 0; c1 <= 511; c1 += 1)
diff --git a/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout.ll b/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout.ll
index 69dfd383060e..faf51e097a70 100644
--- a/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout.ll
+++ b/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-target-throughput-vector-fma=1 \
 ; RUN: -polly-target-latency-vector-fma=8 \
 ; RUN: -polly-target-1st-cache-level-associativity=8 \
diff --git a/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll b/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll
index fe1bc0518e2c..30b693a2e241 100644
--- a/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll
+++ b/polly/test/ScheduleOptimizer/mat_mul_pattern_data_layout_2.ll
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
-; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 ;
 ;    /* C := alpha*A*B + beta*C */
 ;    /* _PB_NK % Kc != 0 */
@@ -18,7 +18,7 @@
 ;	     C[i][j] += alpha * A[i][k] * B[k][j];
 ;        }
 ;
-; CHECK-LABEL:     :: isl ast :: kernel_gemm :: %bb8---%bb32
+; CHECK-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'bb8 => bb32' in function 'kernel_gemm':
 ; CHECK:    {
 ; CHECK-NEXT:      // 1st level tiling - Tiles
 ; CHECK-NEXT:      for (int c0 = 0; c0 <= 32; c0 += 1)
diff --git a/polly/test/ScheduleOptimizer/one-dimensional-band.ll b/polly/test/ScheduleOptimizer/one-dimensional-band.ll
index 594386662ef3..4592907a44ad 100644
--- a/polly/test/ScheduleOptimizer/one-dimensional-band.ll
+++ b/polly/test/ScheduleOptimizer/one-dimensional-band.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 ;
 ;    void jacobi1d(long T, long N, float *A, float *B) {
 ;      long t, i, j;
diff --git a/polly/test/ScheduleOptimizer/outer_coincidence.ll b/polly/test/ScheduleOptimizer/outer_coincidence.ll
index 4a92f416fecc..2ab33edda86b 100644
--- a/polly/test/ScheduleOptimizer/outer_coincidence.ll
+++ b/polly/test/ScheduleOptimizer/outer_coincidence.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-tiling=0 -polly-parallel -polly-opt-outer-coincidence=no  '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-tiling=0 -polly-parallel -polly-opt-outer-coincidence=yes '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=OUTER
+; RUN: opt %loadPolly -polly-tiling=0 -polly-parallel -polly-opt-outer-coincidence=no  -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-tiling=0 -polly-parallel -polly-opt-outer-coincidence=yes -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=OUTER
 
 ; By skewing, the diagonal can be made parallel. ISL does this when the Check
 ; the 'outer_coincidence' option is enabled.
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll
index 979df17632b2..66011168fcc1 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm.ll
@@ -1,7 +1,7 @@
 ; RUN: opt %loadPolly \
 ; RUN: -polly-pattern-matching-based-opts=true \
-; RUN: '-passes=polly-optree,polly-delicm,polly-simplify,polly-opt-isl' \
-; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 \
+; RUN: -polly-optree -polly-delicm -polly-simplify \
+; RUN: -polly-opt-isl -polly-tc-opt=true -debug -disable-output < %s 2>&1 \
 ; RUN: | FileCheck %s
 ; REQUIRES: asserts
 
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll
index 80cae8554fdb..95da89f90755 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts-after-delicm_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-delicm,polly-simplify,polly-opt-isl' \
+; RUN: opt %loadPolly -polly-delicm -polly-simplify -polly-opt-isl \
 ; RUN: -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts.ll
index 5e0bb81c5908..7604257f98e0 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=false \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=false \
 ; RUN: -debug -polly-tc-opt -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true -debug -polly-tc-opt -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PATTERN-MATCHING-OPTS
-; RUN: opt %loadPolly '-passes=polly-opt-isl,print<polly-ast>' -polly-pattern-matching-based-opts=true -polly-ast-detect-parallel -disable-output  < %s | FileCheck %s --check-prefix=PARALLEL-AST
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true -stats -disable-output < %s 2>&1 | FileCheck %s --check-prefix=STATS -match-full-lines
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true -debug -polly-tc-opt -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PATTERN-MATCHING-OPTS
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true -polly-ast-detect-parallel -polly-print-ast -disable-output  < %s | FileCheck %s --check-prefix=PARALLEL-AST
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true -stats -disable-output < %s 2>&1 | FileCheck %s --check-prefix=STATS -match-full-lines
 ; REQUIRES: asserts
 ;
 ;    /* C := alpha*A*B + beta*C */
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_11.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_11.ll
index 5c4391693b13..ccdb39b60d75 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_11.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_11.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-opt-isl' \
+; RUN: opt %loadPolly -polly-import-jscop \
 ; RUN: -polly-import-jscop-postfix=transformed \
 ; RUN: -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-target-throughput-vector-fma=1 \
@@ -8,7 +8,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -debug \
+; RUN: -polly-opt-isl -debug \
 ; RUN: -polly-tc-opt=true -disable-output < %s 2>&1 \
 ; RUN: | FileCheck %s
 ; REQUIRES: asserts
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_12.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_12.ll
index b21a26b4772d..dd39fec5e21f 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_12.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_12.ll
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -passes=polly-opt-isl -disable-output < %s
+; RUN: -polly-opt-isl -disable-output < %s
 ;
 ; Test whether isolation works as expected.
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_13.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_13.ll
index a16ecf6af6ce..e086dd36c4d9 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_13.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_13.ll
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=128 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 ;
 ; Test whether isolation works as expected.
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_14.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_14.ll
index 2c23ebbac43c..a4c71c2dace5 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_14.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_14.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,polly-opt-isl,polly-codegen'  \
+; RUN: opt %loadPolly -polly-import-jscop -polly-opt-isl  \
 ; RUN: -polly-target-throughput-vector-fma=1 \
 ; RUN: -polly-target-latency-vector-fma=8 \
 ; RUN: -polly-target-1st-cache-level-associativity=8 \
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-import-jscop-postfix=transformed -S < %s \
+; RUN: -polly-import-jscop-postfix=transformed -polly-codegen -S < %s \
 ; RUN: | FileCheck %s
 ;
 ; Check that we disable the Loop Vectorizer.
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_15.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_15.ll
index c8d8d295e8c8..a8da21955b63 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_15.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_15.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -debug-only=polly-opt-isl -disable-output \
 ; RUN: -polly-tc-opt=true < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_16.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_16.ll
index 970b4a0cf932..c1ad3017a0d4 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_16.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_16.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_17.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_17.ll
index e44e3bfa04c2..002816a4ae80 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_17.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_17.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_18.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_18.ll
index 2612321b3d09..d5679c7ae2f7 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_18.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_18.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_19.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_19.ll
index bd5f0ed40953..4e1620abd252 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_19.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_19.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_2.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_2.ll
index 573c35256992..01e336ebc60f 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_2.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_20.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_20.ll
index 78cc48d830c8..0be08d8d493c 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_20.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_20.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_21.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_21.ll
index adfba0584a88..9b2df49698a1 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_21.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_21.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_22.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_22.ll
index 54e03b301e69..3d3641df5098 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_22.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_22.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_24.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_24.ll
index fee5027848a8..895961488014 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_24.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_24.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-reschedule=0 -passes=polly-opt-isl \
+; RUN: opt %loadPolly -polly-reschedule=0 -polly-opt-isl \
 ; RUN: -polly-pattern-matching-based-opts=true -polly-tc-opt=true \
 ; RUN: -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_25.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_25.ll
index 029dcc491f02..8a3957909d9d 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_25.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_25.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-tc-opt=true -debug -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
@@ -53,4 +53,4 @@ for.body8:                                        ; preds = %for.body8, %for.con
   br i1 %exitcond.not, label %for.cond.cleanup7, label %for.body8
 }
 
-declare double @llvm.fmuladd.f64(double, double, double)
+declare double @llvm.fmuladd.f64(double, double, double)
\ No newline at end of file
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll
index 46c8c7e35f2a..fab3ac5e58dc 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_3.ll
@@ -3,7 +3,7 @@
 ; RUN: -polly-target-latency-vector-fma=8 \
 ; RUN: -polly-target-1st-cache-level-size=0 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
-; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s 2>&1 | FileCheck %s
 
 ; RUN: opt %loadPolly -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-target-throughput-vector-fma=1 \
@@ -13,7 +13,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=EXTRACTION-OF-MACRO-KERNEL
+; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s 2>&1 | FileCheck %s --check-prefix=EXTRACTION-OF-MACRO-KERNEL
 ;
 ;    /* C := alpha*A*B + beta*C */
 ;    for (i = 0; i < _PB_NI; i++)
@@ -24,7 +24,7 @@
 ;	     C[i][j] += alpha * A[i][k] * B[k][j];
 ;        }
 ;
-; CHECK-LABEL: :: isl ast :: kernel_gemm :: %bb8---%bb32 
+; CHECK-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'bb8 => bb32' in function 'kernel_gemm':
 ; CHECK:    {
 ; CHECK-NEXT:      // 1st level tiling - Tiles
 ; CHECK-NEXT:      for (int c0 = 0; c0 <= 32; c0 += 1)
@@ -76,7 +76,7 @@
 ; CHECK-NEXT:          }
 ; CHECK-NEXT:    }
 ;
-; EXTRACTION-OF-MACRO-KERNEL-LABEL: :: isl ast :: kernel_gemm :: %bb8---%bb32 
+; EXTRACTION-OF-MACRO-KERNEL-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'bb8 => bb32' in function 'kernel_gemm':
 ; EXTRACTION-OF-MACRO-KERNEL:    {
 ; EXTRACTION-OF-MACRO-KERNEL-NEXT:      // 1st level tiling - Tiles
 ; EXTRACTION-OF-MACRO-KERNEL-NEXT:      for (int c0 = 0; c0 <= 32; c0 += 1)
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_4.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_4.ll
index ec1926ebb75f..dc0edc6c5a3b 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_4.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_4.ll
@@ -1,12 +1,12 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -debug -polly-tc-opt=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=polly-opt-isl,print<polly-ast>' -polly-pattern-matching-based-opts=true \
+; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; RUN: -polly-target-throughput-vector-fma=1 \
 ; RUN: -polly-target-latency-vector-fma=8 \
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
-; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -polly-tc-opt=true -disable-output < %s |  \
+; RUN: -polly-target-2nd-cache-level-size=262144 -polly-print-ast \
+; RUN: -polly-tc-opt=true -disable-output -polly-opt-isl < %s |  \
 ; RUN: FileCheck %s --check-prefix=PATTERN-MATCHING-OPTS
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_5.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_5.ll
index bfb378259210..6581566bf13f 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_5.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_5.ll
@@ -6,12 +6,12 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 ;
-; opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+; opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ; -polly-target-throughput-vector-fma=1 \
 ; -polly-target-latency-vector-fma=8 \
-; -passes=polly-codegen -polly-target-1st-cache-level-associativity=8 \
+; -polly-codegen -polly-target-1st-cache-level-associativity=8 \
 ; -polly-target-2nd-cache-level-associativity=8 \
 ; -polly-target-1st-cache-level-size=32768 \
 ; -polly-target-vector-register-bitwidth=256 \
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_6.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_6.ll
index 684cd9be1728..bcf1fc9fe813 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_6.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_6.ll
@@ -6,12 +6,12 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 ;
-;  opt %loadPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
+;  opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
 ;  -polly-target-throughput-vector-fma=1 \
 ;  -polly-target-latency-vector-fma=8 \
-;  -passes=polly-codegen -polly-target-1st-cache-level-associativity=8 \
+;  -polly-codegen -polly-target-1st-cache-level-associativity=8 \
 ;  -polly-target-2nd-cache-level-associativity=8 \
 ;  -polly-target-1st-cache-level-size=32768 \
 ;  -polly-target-vector-register-bitwidth=256 \
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_7.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_7.ll
index b34b4f020ccb..77a3e02a0063 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_7.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_7.ll
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 ;
 ;    /* C := A * B + C */
 ;    /* Elements of the matrices A, B, C have the float type. */
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_8.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_8.ll
index a65d92940665..d02bc359e79d 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_8.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_8.ll
@@ -6,7 +6,7 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 ;
 ;    /* C := A * B + C */
 ;    /* Elements of the matrices B, C have the double type. */
diff --git a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_9.ll b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_9.ll
index 5a35e2c049f0..144abfd7622f 100644
--- a/polly/test/ScheduleOptimizer/pattern-matching-based-opts_9.ll
+++ b/polly/test/ScheduleOptimizer/pattern-matching-based-opts_9.ll
@@ -6,9 +6,9 @@
 ; RUN: -polly-target-1st-cache-level-size=32768 \
 ; RUN: -polly-target-vector-register-bitwidth=256 \
 ; RUN: -polly-target-2nd-cache-level-size=262144 \
-; RUN: -passes=polly-opt-isl -disable-output < %s
+; RUN: -polly-opt-isl -disable-output < %s
 ;
-; RUN: opt %loadPolly '-passes=print<polly-dependences>' -disable-output < %s | FileCheck %s --check-prefix=DEPENDENCES
+; RUN: opt %loadPolly -polly-print-dependences -disable-output < %s | FileCheck %s --check-prefix=DEPENDENCES
 ;
 ;    /* C := A * B + C */
 ;    /* Elements of the matrices A, B, C have the char type. */
diff --git a/polly/test/ScheduleOptimizer/pattern_matching_based_opts_splitmap.ll b/polly/test/ScheduleOptimizer/pattern_matching_based_opts_splitmap.ll
index 05362c712a4d..5b9783d20bfc 100644
--- a/polly/test/ScheduleOptimizer/pattern_matching_based_opts_splitmap.ll
+++ b/polly/test/ScheduleOptimizer/pattern_matching_based_opts_splitmap.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-import-jscop -polly-import-jscop-postfix=transformed -passes=polly-opt-isl -debug-only=polly-opt-isl -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-opt-isl -debug-only=polly-opt-isl -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
 ; void pattern_matching_based_opts_splitmap(double C[static const restrict 2][2], double A[static const restrict 2][784], double B[static const restrict 784][2]) {
diff --git a/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll b/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll
index 5c44f73c287c..fea2155b1e4e 100644
--- a/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll
+++ b/polly/test/ScheduleOptimizer/prevectorization-without-tiling.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-tiling=false -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -basic-aa -polly-tiling=false -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 @C = common global [1536 x [1536 x float]] zeroinitializer, align 16
diff --git a/polly/test/ScheduleOptimizer/prevectorization.ll b/polly/test/ScheduleOptimizer/prevectorization.ll
index 6a8ec549c784..385ebf14712a 100644
--- a/polly/test/ScheduleOptimizer/prevectorization.ll
+++ b/polly/test/ScheduleOptimizer/prevectorization.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine                         '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt -S %loadPolly -aa-pipeline=basic-aa -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine -polly-prevect-width=16 '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s -check-prefix=VEC16
+; RUN: opt -S %loadPolly -basic-aa -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine                         -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt -S %loadPolly -basic-aa -polly-pattern-matching-based-opts=false -polly-vectorizer=stripmine -polly-prevect-width=16 -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s -check-prefix=VEC16
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScheduleOptimizer/rectangular-tiling.ll b/polly/test/ScheduleOptimizer/rectangular-tiling.ll
index 9d34c7c17a79..b527255ab5f7 100644
--- a/polly/test/ScheduleOptimizer/rectangular-tiling.ll
+++ b/polly/test/ScheduleOptimizer/rectangular-tiling.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-tile-sizes=256,16                                                                                                                                        '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
-; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-tiling=false                                                                                                                    '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=NOTILING
-; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-2nd-level-tiling -polly-2nd-level-tile-sizes=16,8                                                                               '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=TWOLEVEL
-; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-2nd-level-tiling -polly-2nd-level-tile-sizes=16,8 -polly-register-tiling                                                        '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s --check-prefix=TWO-PLUS-REGISTER
+; RUN: opt %loadPolly -polly-tile-sizes=256,16                                                                                                                                        -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-tiling=false                                                                                                                    -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=NOTILING
+; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-2nd-level-tiling -polly-2nd-level-tile-sizes=16,8                                                                               -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=TWOLEVEL
+; RUN: opt %loadPolly -polly-tile-sizes=256,16 -polly-2nd-level-tiling -polly-2nd-level-tile-sizes=16,8 -polly-register-tiling                                                        -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=TWO-PLUS-REGISTER
 
 ; CHECK: // 1st level tiling - Tiles
 ; CHECK: for (int c0 = 0; c0 <= 3; c0 += 1)
diff --git a/polly/test/ScheduleOptimizer/schedule_computeout.ll b/polly/test/ScheduleOptimizer/schedule_computeout.ll
index 6e60fe1cd6f3..acc8601a31a8 100644
--- a/polly/test/ScheduleOptimizer/schedule_computeout.ll
+++ b/polly/test/ScheduleOptimizer/schedule_computeout.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -S -passes=polly-optree -passes=polly-delicm  -passes=polly-opt-isl -polly-schedule-computeout=10000 -debug-only="polly-opt-isl" < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -S -polly-optree -polly-delicm  -polly-opt-isl -polly-schedule-computeout=10000 -debug-only="polly-opt-isl" < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; Bailout if the computations of schedule compute exceeds the max scheduling quota.
diff --git a/polly/test/ScheduleOptimizer/statistics.ll b/polly/test/ScheduleOptimizer/statistics.ll
index 56bf06894dd4..472febea173f 100644
--- a/polly/test/ScheduleOptimizer/statistics.ll
+++ b/polly/test/ScheduleOptimizer/statistics.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-opt-isl -stats -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-opt-isl -stats -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
 
 ; REQUIRES: asserts
 
diff --git a/polly/test/ScheduleOptimizer/tile_after_fusion.ll b/polly/test/ScheduleOptimizer/tile_after_fusion.ll
index b834b354af4d..8e5849234af6 100644
--- a/polly/test/ScheduleOptimizer/tile_after_fusion.ll
+++ b/polly/test/ScheduleOptimizer/tile_after_fusion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-isl-arg=--no-schedule-serialize-sccs '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-isl-arg=--no-schedule-serialize-sccs -polly-opt-isl -polly-print-ast -disable-output < %s | FileCheck %s
 ;
 ;
 ;    void tf(int C[256][256][256], int A0[256][256][256], int A1[256][256][256]) {
@@ -17,7 +17,7 @@
 ; checks whether they are tiled after being fused when polly-opt-fusion equals
 ; "max".
 ;
-; CHECK-LABEL: :: isl ast :: tf :: %for.cond---%for.end56 
+; CHECK-LABEL: Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: 'for.cond => for.end56' in function 'tf':
 ; CHECK:       1st level tiling - Tiles
 ; CHECK-NEXT:     for (int c0 = 0; c0 <= 7; c0 += 1)
 ; CHECK-NEXT:       for (int c1 = 0; c1 <= 7; c1 += 1)
diff --git a/polly/test/ScheduleOptimizer/vivid-vbi-gen-vivid_vbi_gen_sliced-before-llvmreduced.ll b/polly/test/ScheduleOptimizer/vivid-vbi-gen-vivid_vbi_gen_sliced-before-llvmreduced.ll
index bfa1f017b61a..d08595db8fce 100644
--- a/polly/test/ScheduleOptimizer/vivid-vbi-gen-vivid_vbi_gen_sliced-before-llvmreduced.ll
+++ b/polly/test/ScheduleOptimizer/vivid-vbi-gen-vivid_vbi_gen_sliced-before-llvmreduced.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-vectorizer=stripmine -polly-isl-arg=--no-schedule-serialize-sccs -polly-tiling=0 '-passes=print<polly-opt-isl>' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-vectorizer=stripmine -polly-isl-arg=--no-schedule-serialize-sccs -polly-tiling=0 -polly-print-opt-isl -disable-output < %s | FileCheck %s
 
 ; isl_schedule_node_band_sink may sink into multiple children.
 ; https://llvm.org/PR52637
diff --git a/polly/test/ScopDetect/aliasing_parametric_simple_1.ll b/polly/test/ScopDetect/aliasing_parametric_simple_1.ll
index 8a2317446d64..2eddbd4cb262 100644
--- a/polly/test/ScopDetect/aliasing_parametric_simple_1.ll
+++ b/polly/test/ScopDetect/aliasing_parametric_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop:
 ;
diff --git a/polly/test/ScopDetect/aliasing_parametric_simple_2.ll b/polly/test/ScopDetect/aliasing_parametric_simple_2.ll
index df1d1f8d56bc..c111f686c462 100644
--- a/polly/test/ScopDetect/aliasing_parametric_simple_2.ll
+++ b/polly/test/ScopDetect/aliasing_parametric_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop:
 ;
diff --git a/polly/test/ScopDetect/aliasing_simple_1.ll b/polly/test/ScopDetect/aliasing_simple_1.ll
index af5b7cf1dfb7..524ca19ae398 100644
--- a/polly/test/ScopDetect/aliasing_simple_1.ll
+++ b/polly/test/ScopDetect/aliasing_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop:
 ;
diff --git a/polly/test/ScopDetect/aliasing_simple_2.ll b/polly/test/ScopDetect/aliasing_simple_2.ll
index cd3155ec613a..457df996c7b8 100644
--- a/polly/test/ScopDetect/aliasing_simple_2.ll
+++ b/polly/test/ScopDetect/aliasing_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop:
 ;
diff --git a/polly/test/ScopDetect/base_pointer_load_setNewAccessRelation.ll b/polly/test/ScopDetect/base_pointer_load_setNewAccessRelation.ll
index ab56f344f093..0411aed6ae04 100644
--- a/polly/test/ScopDetect/base_pointer_load_setNewAccessRelation.ll
+++ b/polly/test/ScopDetect/base_pointer_load_setNewAccessRelation.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>,scop(polly-import-jscop,polly-codegen)' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true -polly-scops -polly-print-import-jscop -polly-codegen -disable-output < %s | FileCheck %s
 ;
 ; This violated an assertion in setNewAccessRelation that assumed base pointers
 ; to be load-hoisted. Without this assertion, it codegen would generate invalid
diff --git a/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll b/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll
index df7f3d706fd3..ff9be6ea16e8 100644
--- a/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll
+++ b/polly/test/ScopDetect/base_pointer_setNewAccessRelation.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,scop(polly-import-jscop,polly-codegen)' -disable-output < %s 2>&1 | FileCheck %s --allow-empty
+; RUN: opt %loadPolly -disable-basic-aa -polly-detect -polly-print-import-jscop -polly-codegen -disable-output < %s | FileCheck %s --allow-empty
 ;
 ; Polly codegen used to generate invalid code (referring to %ptr from the
 ; original region) when regeneration of the access function is necessary.
diff --git a/polly/test/ScopDetect/callbr.ll b/polly/test/ScopDetect/callbr.ll
index 5183b9c1a085..d65ab934bf2e 100644
--- a/polly/test/ScopDetect/callbr.ll
+++ b/polly/test/ScopDetect/callbr.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -polly-detect-track-failures -disable-output -pass-remarks-missed=polly-detect < %s 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -polly-detect-track-failures -disable-output -stats                            < %s 2>&1 | FileCheck %s --check-prefix=STAT
+; RUN: opt %loadPolly -polly-detect -polly-detect-track-failures -disable-output -pass-remarks-missed=polly-detect < %s 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt %loadPolly -polly-detect -polly-detect-track-failures -disable-output -stats                            < %s 2>&1 | FileCheck %s --check-prefix=STAT
 ; REQUIRES: asserts
 
 ; REMARK: Branch from indirect terminator.
diff --git a/polly/test/ScopDetect/collective_invariant_loads.ll b/polly/test/ScopDetect/collective_invariant_loads.ll
index 96f154f07d2e..f1d2eea520c6 100644
--- a/polly/test/ScopDetect/collective_invariant_loads.ll
+++ b/polly/test/ScopDetect/collective_invariant_loads.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting -disable-output< %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting -disable-output< %s | FileCheck %s
 
 ;CHECK:     Function: test_init_chpl
 ;CHECK-NEXT:     Region: %bb1---%bb16
diff --git a/polly/test/ScopDetect/cross_loop_non_single_exit.ll b/polly/test/ScopDetect/cross_loop_non_single_exit.ll
index e54d30fc2164..ae23930b92a6 100644
--- a/polly/test/ScopDetect/cross_loop_non_single_exit.ll
+++ b/polly/test/ScopDetect/cross_loop_non_single_exit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/cross_loop_non_single_exit_2.ll b/polly/test/ScopDetect/cross_loop_non_single_exit_2.ll
index f3bd0d097b71..5c25da66d7ef 100644
--- a/polly/test/ScopDetect/cross_loop_non_single_exit_2.ll
+++ b/polly/test/ScopDetect/cross_loop_non_single_exit_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/dependency_to_phi_node_outside_of_region.ll b/polly/test/ScopDetect/dependency_to_phi_node_outside_of_region.ll
index 6d262a9a464b..12983d2321cc 100644
--- a/polly/test/ScopDetect/dependency_to_phi_node_outside_of_region.ll
+++ b/polly/test/ScopDetect/dependency_to_phi_node_outside_of_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s
+; RUN: opt %loadPolly -polly-detect -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
 define void @f(ptr %A, i64 %N, i64 %M) nounwind {
diff --git a/polly/test/ScopDetect/dot-scops-npm.ll b/polly/test/ScopDetect/dot-scops-npm.ll
index 9de6a5e2e1a5..7c8be032fd4f 100644
--- a/polly/test/ScopDetect/dot-scops-npm.ll
+++ b/polly/test/ScopDetect/dot-scops-npm.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-scop-printer' -disable-output < %s
+; RUN: opt %loadNPMPolly "-passes=polly-scop-printer" -disable-output < %s
 ; RUN: FileCheck %s -input-file=scops.func_npm.dot
 ;
 ; Check that the ScopPrinter does not crash.
diff --git a/polly/test/ScopDetect/dot-scops.ll b/polly/test/ScopDetect/dot-scops.ll
index 2297fd3253ca..c31562e4c62d 100644
--- a/polly/test/ScopDetect/dot-scops.ll
+++ b/polly/test/ScopDetect/dot-scops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>,polly-scop-printer' -disable-output < %s
+; RUN: opt %loadPolly -polly-scops -dot-scops -disable-output < %s
 ;
 ; Check that the ScopPrinter does not crash.
 ; ScopPrinter needs the ScopDetection pass, which should depend on
diff --git a/polly/test/ScopDetect/error-block-always-executed.ll b/polly/test/ScopDetect/error-block-always-executed.ll
index 312c48cfee96..894be2119941 100644
--- a/polly/test/ScopDetect/error-block-always-executed.ll
+++ b/polly/test/ScopDetect/error-block-always-executed.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT: Valid Region for Scop:
 
diff --git a/polly/test/ScopDetect/error-block-referenced-from-scop.ll b/polly/test/ScopDetect/error-block-referenced-from-scop.ll
index d3e56472e497..085351482139 100644
--- a/polly/test/ScopDetect/error-block-referenced-from-scop.ll
+++ b/polly/test/ScopDetect/error-block-referenced-from-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT: Valid Region for Scop:
 
diff --git a/polly/test/ScopDetect/error-block-unreachable.ll b/polly/test/ScopDetect/error-block-unreachable.ll
index 72d43004e322..48f6fe8e0547 100644
--- a/polly/test/ScopDetect/error-block-unreachable.ll
+++ b/polly/test/ScopDetect/error-block-unreachable.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s
+; RUN: opt %loadPolly -polly-detect -disable-output < %s
 
 ; Verify that the scop detection does not crash on inputs with unreachable
 ; blocks. Earlier we crashed when detecting error blocks.
diff --git a/polly/test/ScopDetect/expand-region-correctly-2.ll b/polly/test/ScopDetect/expand-region-correctly-2.ll
index b6632c643fdf..fadb503cff35 100644
--- a/polly/test/ScopDetect/expand-region-correctly-2.ll
+++ b/polly/test/ScopDetect/expand-region-correctly-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop: if.end.1631 => for.cond.1647.outer
 ;
diff --git a/polly/test/ScopDetect/expand-region-correctly.ll b/polly/test/ScopDetect/expand-region-correctly.ll
index 022dfb68ecd9..72082a32fa79 100644
--- a/polly/test/ScopDetect/expand-region-correctly.ll
+++ b/polly/test/ScopDetect/expand-region-correctly.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; CHECK: Valid Region for Scop: if.end.1631 => for.cond.1647.outer
 
diff --git a/polly/test/ScopDetect/ignore_func_flag_regex.ll b/polly/test/ScopDetect/ignore_func_flag_regex.ll
index 15b92b418bcb..224126ec010e 100644
--- a/polly/test/ScopDetect/ignore_func_flag_regex.ll
+++ b/polly/test/ScopDetect/ignore_func_flag_regex.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-ignore-func=f.*,g.* '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-ignore-func=f.*,g.* -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that the flag `-polly-ignore-func` works with regexes.
 ;
diff --git a/polly/test/ScopDetect/index_from_unpredictable_loop.ll b/polly/test/ScopDetect/index_from_unpredictable_loop.ll
index e0be6243ebf8..27ed64da17e6 100644
--- a/polly/test/ScopDetect/index_from_unpredictable_loop.ll
+++ b/polly/test/ScopDetect/index_from_unpredictable_loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly                        '-passes=print<polly-function-scops>' -disable-output < %s | FileCheck %s --check-prefix=AFFINE
-; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN: opt %loadPolly                        -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=AFFINE
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
 
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopDetect/index_from_unpredictable_loop2.ll b/polly/test/ScopDetect/index_from_unpredictable_loop2.ll
index 4d4b6f988b69..9b5a3a4389d4 100644
--- a/polly/test/ScopDetect/index_from_unpredictable_loop2.ll
+++ b/polly/test/ScopDetect/index_from_unpredictable_loop2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly                        '-passes=print<polly-function-scops>' -disable-output < %s | FileCheck %s --check-prefix=AFFINE
-; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN: opt %loadPolly                        -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=AFFINE
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
 
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopDetect/indvars.ll b/polly/test/ScopDetect/indvars.ll
index 023f68435988..2ba4d1f5aabf 100644
--- a/polly/test/ScopDetect/indvars.ll
+++ b/polly/test/ScopDetect/indvars.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,scop(polly-codegen)' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -polly-codegen -disable-output < %s | FileCheck %s
 ;
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/ScopDetect/intrinsics_1.ll b/polly/test/ScopDetect/intrinsics_1.ll
index 61e1c1fc3d86..65d3968e247c 100644
--- a/polly/test/ScopDetect/intrinsics_1.ll
+++ b/polly/test/ScopDetect/intrinsics_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Valid Region for Scop: for.cond => for.end
 ;
diff --git a/polly/test/ScopDetect/intrinsics_2.ll b/polly/test/ScopDetect/intrinsics_2.ll
index 0c2aa49c2d21..f0575511b2ef 100644
--- a/polly/test/ScopDetect/intrinsics_2.ll
+++ b/polly/test/ScopDetect/intrinsics_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; Verify that we allow the lifetime markers for the tmp array.
 ;
diff --git a/polly/test/ScopDetect/intrinsics_3.ll b/polly/test/ScopDetect/intrinsics_3.ll
index 16d41d0550af..bce90d136a41 100644
--- a/polly/test/ScopDetect/intrinsics_3.ll
+++ b/polly/test/ScopDetect/intrinsics_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; Verify that we allow the misc intrinsics.
 ;
diff --git a/polly/test/ScopDetect/invalid-latch-conditions.ll b/polly/test/ScopDetect/invalid-latch-conditions.ll
index 1264ba0483c0..eb8097470ecf 100644
--- a/polly/test/ScopDetect/invalid-latch-conditions.ll
+++ b/polly/test/ScopDetect/invalid-latch-conditions.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly                              -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops                                   '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NALOOPS
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly                              -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops                                   -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=NALOOPS
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=PROFIT
 
 ; The latch conditions of the outer loop are not affine, thus the loop cannot
 ; handled by the domain generation and needs to be overapproximated.
diff --git a/polly/test/ScopDetect/invalidate_scalar_evolution.ll b/polly/test/ScopDetect/invalidate_scalar_evolution.ll
index c691c7a633d2..01d34c49e289 100644
--- a/polly/test/ScopDetect/invalidate_scalar_evolution.ll
+++ b/polly/test/ScopDetect/invalidate_scalar_evolution.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=PHI
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=PHI
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/invariant-load-before-scop.ll b/polly/test/ScopDetect/invariant-load-before-scop.ll
index ee2eba5e8ec6..f72085ff88a1 100644
--- a/polly/test/ScopDetect/invariant-load-before-scop.ll
+++ b/polly/test/ScopDetect/invariant-load-before-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; The LoadInst %.b761 is defined outside the SCoP, hence is always constant
 ; within it. It is no "required invariant load".
diff --git a/polly/test/ScopDetect/keep_going_expansion.ll b/polly/test/ScopDetect/keep_going_expansion.ll
index 7da7cd41c1f9..9bcfb3924f6a 100644
--- a/polly/test/ScopDetect/keep_going_expansion.ll
+++ b/polly/test/ScopDetect/keep_going_expansion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-detect-track-failures -polly-detect-keep-going '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-detect-track-failures -polly-detect-keep-going -polly-print-detect -disable-output < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopDetect/mod_ref_read_pointer.ll b/polly/test/ScopDetect/mod_ref_read_pointer.ll
index 7b185a87c821..95a4649f4705 100644
--- a/polly/test/ScopDetect/mod_ref_read_pointer.ll
+++ b/polly/test/ScopDetect/mod_ref_read_pointer.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-modref-calls '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=MODREF
-; RUN: opt %loadPolly -aa-pipeline=basic-aa                           '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-allow-modref-calls -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=MODREF
+; RUN: opt %loadPolly -basic-aa                           -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT: Valid Region for Scop: for.body => for.end
 ; MODREF: Valid Region for Scop: for.body => for.end
diff --git a/polly/test/ScopDetect/more-than-one-loop.ll b/polly/test/ScopDetect/more-than-one-loop.ll
index 5972fbf50889..bfd226c1bcfc 100644
--- a/polly/test/ScopDetect/more-than-one-loop.ll
+++ b/polly/test/ScopDetect/more-than-one-loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-process-unprofitable=true '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable=true -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; CHECK: Valid Region for Scop:
 
diff --git a/polly/test/ScopDetect/multidim-with-undef-size.ll b/polly/test/ScopDetect/multidim-with-undef-size.ll
index 2adf7bb00b42..9973c6c72169 100644
--- a/polly/test/ScopDetect/multidim-with-undef-size.ll
+++ b/polly/test/ScopDetect/multidim-with-undef-size.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; CHECK: Valid Region for Scop: bb14 => bb17
diff --git a/polly/test/ScopDetect/multidim.ll b/polly/test/ScopDetect/multidim.ll
index a1a6167a121e..f43698819f32 100644
--- a/polly/test/ScopDetect/multidim.ll
+++ b/polly/test/ScopDetect/multidim.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; CHECK: Valid Region for Scop: bb19 => bb20
diff --git a/polly/test/ScopDetect/multidim_indirect_access.ll b/polly/test/ScopDetect/multidim_indirect_access.ll
index 4a3012d1c93c..3e06251f5fd1 100644
--- a/polly/test/ScopDetect/multidim_indirect_access.ll
+++ b/polly/test/ScopDetect/multidim_indirect_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; Check that we will recognize this SCoP.
 ;
diff --git a/polly/test/ScopDetect/multidim_two_accesses_different_delinearization.ll b/polly/test/ScopDetect/multidim_two_accesses_different_delinearization.ll
index 23d520ed58be..ed554a24a6d6 100644
--- a/polly/test/ScopDetect/multidim_two_accesses_different_delinearization.ll
+++ b/polly/test/ScopDetect/multidim_two_accesses_different_delinearization.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/ScopDetect/nested_loop_single_exit.ll b/polly/test/ScopDetect/nested_loop_single_exit.ll
index a794e2c48ff9..377e8088eedb 100644
--- a/polly/test/ScopDetect/nested_loop_single_exit.ll
+++ b/polly/test/ScopDetect/nested_loop_single_exit.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -disable-output < %s
 
 ; void f(long A[], long N) {
 ;   long i, j;
diff --git a/polly/test/ScopDetect/non-affine-conditional.ll b/polly/test/ScopDetect/non-affine-conditional.ll
index f69b6f8cd1ed..fc2d0c02d2da 100644
--- a/polly/test/ScopDetect/non-affine-conditional.ll
+++ b/polly/test/ScopDetect/non-affine-conditional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/ScopDetect/non-affine-float-compare.ll b/polly/test/ScopDetect/non-affine-float-compare.ll
index 1e4c580fa00d..984f14aaff8f 100644
--- a/polly/test/ScopDetect/non-affine-float-compare.ll
+++ b/polly/test/ScopDetect/non-affine-float-compare.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ;    void f(float *A) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll
index 443a0e13c6ca..068367fa1e3c 100644
--- a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll
+++ b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                                                          '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                                           '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine                                   '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                                                          -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                                           -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine                                   -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=PROFIT
 ;
 ; Here we have a non-affine loop but also a non-affine access which should
 ; be rejected as long as -polly-allow-nonaffine isn't given.
diff --git a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_2.ll b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_2.ll
index 77733a8d9b96..cd2140518b46 100644
--- a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_2.ll
+++ b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_2.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                        '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                         '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                        -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                         -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
 ;
 ; Here we have a non-affine loop (in the context of the loop nest)
 ; and also a non-affine access (A[k]). While we can always detect the
diff --git a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_3.ll b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_3.ll
index 034ab61fa03e..fb936216e45c 100644
--- a/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_3.ll
+++ b/polly/test/ScopDetect/non-affine-loop-condition-dependent-access_3.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                        '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                         '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                        -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                         -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
 ;
 ; Here we have a non-affine loop (in the context of the loop nest)
 ; and also a non-affine access (A[k]). While we can always detect the
diff --git a/polly/test/ScopDetect/non-affine-loop.ll b/polly/test/ScopDetect/non-affine-loop.ll
index d17fd39da701..d5f7ea128a79 100644
--- a/polly/test/ScopDetect/non-affine-loop.ll
+++ b/polly/test/ScopDetect/non-affine-loop.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                                                          '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                                           '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false -polly-allow-nonaffine                                   '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINEREGIONSANDACCESSES
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine                                   '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                                                          -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                                           -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false -polly-allow-nonaffine                                   -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINEREGIONSANDACCESSES
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine                                   -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=ALLOWNONAFFINELOOPSANDACCESSES
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true  -polly-allow-nonaffine -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=PROFIT
 ;
 ; This function/region does contain a loop, however it is non-affine, hence the access
 ; A[i] is also. Furthermore, it is the only loop, thus when we over approximate
diff --git a/polly/test/ScopDetect/non-beneficial-loops-small-trip-count.ll b/polly/test/ScopDetect/non-beneficial-loops-small-trip-count.ll
index d5901b63dd37..43af1684dccb 100644
--- a/polly/test/ScopDetect/non-beneficial-loops-small-trip-count.ll
+++ b/polly/test/ScopDetect/non-beneficial-loops-small-trip-count.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT: Valid
 ;
diff --git a/polly/test/ScopDetect/non-constant-add-rec-start-expr.ll b/polly/test/ScopDetect/non-constant-add-rec-start-expr.ll
index f39e774021a9..4cddcc916a76 100644
--- a/polly/test/ScopDetect/non-constant-add-rec-start-expr.ll
+++ b/polly/test/ScopDetect/non-constant-add-rec-start-expr.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; CHECK: Valid Region for Scop: bb11 => bb25
 
diff --git a/polly/test/ScopDetect/non-simple-memory-accesses.ll b/polly/test/ScopDetect/non-simple-memory-accesses.ll
index d1c2ce63059e..a82228982885 100644
--- a/polly/test/ScopDetect/non-simple-memory-accesses.ll
+++ b/polly/test/ScopDetect/non-simple-memory-accesses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; Verify that we do not model atomic memory accesses. We did not reason about
 ; how to handle them correctly and the Alias Set Tracker models some of them
diff --git a/polly/test/ScopDetect/non_affine_loop_condition.ll b/polly/test/ScopDetect/non_affine_loop_condition.ll
index 1d67df58d9bb..f268442cd8ee 100644
--- a/polly/test/ScopDetect/non_affine_loop_condition.ll
+++ b/polly/test/ScopDetect/non_affine_loop_condition.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops                                   '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-process-unprofitable=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops                                   -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-process-unprofitable=false -polly-print-detect -disable-output < %s | FileCheck %s --check-prefix=PROFIT
 ;
 ;    void f(int *A) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/ScopDetect/only-one-affine-loop.ll b/polly/test/ScopDetect/only-one-affine-loop.ll
index 3f4305ab83e7..d6d50bb611d9 100644
--- a/polly/test/ScopDetect/only-one-affine-loop.ll
+++ b/polly/test/ScopDetect/only-one-affine-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable=false -polly-allow-nonaffine-loops '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable=false -polly-allow-nonaffine-loops -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; Even if we allow non-affine loops we can only model the outermost loop, all
 ; other loops are boxed in non-affine regions. However, the inner loops can be
diff --git a/polly/test/ScopDetect/only_func_flag.ll b/polly/test/ScopDetect/only_func_flag.ll
index 35a38e875e12..d465cd0f50f7 100644
--- a/polly/test/ScopDetect/only_func_flag.ll
+++ b/polly/test/ScopDetect/only_func_flag.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-only-func=f,g '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-only-func=f,g -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that the flag `-polly-only-func` limits analysis to `f` and `g`.
 ;
diff --git a/polly/test/ScopDetect/only_func_flag_regex.ll b/polly/test/ScopDetect/only_func_flag_regex.ll
index 3b577b100db3..e6675798eeb9 100644
--- a/polly/test/ScopDetect/only_func_flag_regex.ll
+++ b/polly/test/ScopDetect/only_func_flag_regex.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-only-func=f.*,g.* '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-only-func=f.*,g.* -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that the flag `-polly-only-func` works with regexes.
 ;
diff --git a/polly/test/ScopDetect/parametric-multiply-in-scev-2.ll b/polly/test/ScopDetect/parametric-multiply-in-scev-2.ll
index 6cb8a9ee5125..fc957a7f912c 100644
--- a/polly/test/ScopDetect/parametric-multiply-in-scev-2.ll
+++ b/polly/test/ScopDetect/parametric-multiply-in-scev-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 
 ; CHECK-NOT: Valid Region
diff --git a/polly/test/ScopDetect/parametric-multiply-in-scev.ll b/polly/test/ScopDetect/parametric-multiply-in-scev.ll
index bb6dd6c73593..9c6e5ccc8f52 100644
--- a/polly/test/ScopDetect/parametric-multiply-in-scev.ll
+++ b/polly/test/ScopDetect/parametric-multiply-in-scev.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ;  foo(float *A, long n, long k) {
 ;    if (true)
diff --git a/polly/test/ScopDetect/phi_with_multi_exiting_edges.ll b/polly/test/ScopDetect/phi_with_multi_exiting_edges.ll
index c88dcfd860f4..054de168d76b 100644
--- a/polly/test/ScopDetect/phi_with_multi_exiting_edges.ll
+++ b/polly/test/ScopDetect/phi_with_multi_exiting_edges.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; Region with an exit node that has a PHI node multiple incoming edges from
 ; inside the region. Motivation for supporting such cases in Polly.
diff --git a/polly/test/ScopDetect/profitability-large-basic-blocks.ll b/polly/test/ScopDetect/profitability-large-basic-blocks.ll
index 7296812ad85c..e1650febf11c 100644
--- a/polly/test/ScopDetect/profitability-large-basic-blocks.ll
+++ b/polly/test/ScopDetect/profitability-large-basic-blocks.ll
@@ -1,12 +1,12 @@
 ; RUN: opt %loadPolly -polly-process-unprofitable=false \
 ; RUN:                -polly-detect-profitability-min-per-loop-insts=40 \
-; RUN: '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=PROFITABLE
+; RUN: -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=PROFITABLE
 
 ; RUN: opt %loadPolly -polly-process-unprofitable=true \
-; RUN: '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=PROFITABLE
+; RUN: -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=PROFITABLE
 
 ; RUN: opt %loadPolly -polly-process-unprofitable=false \
-; RUN: '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=UNPROFITABLE
+; RUN: -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=UNPROFITABLE
 
 ; UNPROFITABLE-NOT: Valid Region for Scop:
 ; PROFITABLE: Valid Region for Scop:
diff --git a/polly/test/ScopDetect/profitability-two-nested-loops.ll b/polly/test/ScopDetect/profitability-two-nested-loops.ll
index 9311fc87d378..525f91cbc2f4 100644
--- a/polly/test/ScopDetect/profitability-two-nested-loops.ll
+++ b/polly/test/ScopDetect/profitability-two-nested-loops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; CHECK: Valid Region for Scop: next => bb3
 ;
diff --git a/polly/test/ScopDetect/remove_all_children.ll b/polly/test/ScopDetect/remove_all_children.ll
index a6b211ccfe0b..6d5097b80607 100644
--- a/polly/test/ScopDetect/remove_all_children.ll
+++ b/polly/test/ScopDetect/remove_all_children.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopDetect/report-scop-location.ll b/polly/test/ScopDetect/report-scop-location.ll
index 03043faedc34..750699cbe763 100644
--- a/polly/test/ScopDetect/report-scop-location.ll
+++ b/polly/test/ScopDetect/report-scop-location.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -polly-report -disable-output < %s  2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-detect -polly-report -disable-output < %s  2>&1 | FileCheck %s
 target datalayout = "e-i64:64-f80:128-s:64-n8:16:32:64-S128"
 
 ; Function Attrs: nounwind uwtable
diff --git a/polly/test/ScopDetect/restrict-undef-size-scopdetect.ll b/polly/test/ScopDetect/restrict-undef-size-scopdetect.ll
index 04d6b151ebba..e94f1e7728c5 100644
--- a/polly/test/ScopDetect/restrict-undef-size-scopdetect.ll
+++ b/polly/test/ScopDetect/restrict-undef-size-scopdetect.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ; CHECK-NOT: Valid Region for Scop:
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/ScopDetect/run_time_alias_check.ll b/polly/test/ScopDetect/run_time_alias_check.ll
index aa6ba8698fe9..672f3dfa6365 100644
--- a/polly/test/ScopDetect/run_time_alias_check.ll
+++ b/polly/test/ScopDetect/run_time_alias_check.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
diff --git a/polly/test/ScopDetect/scev_remove_max.ll b/polly/test/ScopDetect/scev_remove_max.ll
index 5aa121977cfe..5353e06bdf2f 100644
--- a/polly/test/ScopDetect/scev_remove_max.ll
+++ b/polly/test/ScopDetect/scev_remove_max.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' < %s
+; RUN: opt %loadPolly -polly-detect < %s
 
 ; This test case helps to determine wether SCEVRemoveMax::remove produces
 ; an infinite loop and a segmentation fault, if it processes, for example,
diff --git a/polly/test/ScopDetect/sequential_loops.ll b/polly/test/ScopDetect/sequential_loops.ll
index df10da3aac36..e6ac38aa1604 100644
--- a/polly/test/ScopDetect/sequential_loops.ll
+++ b/polly/test/ScopDetect/sequential_loops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
@@ -13,7 +13,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 ; }
 
 define void @f1(ptr %A, i64 %N) nounwind {
-; CHECK-LABEL: Detected Scops in Function f1 
+; CHECK-LABEL: 'Polly - Detect static control parts (SCoPs)' for function 'f1'
 entry:
   fence seq_cst
   br label %for.i.1
@@ -60,7 +60,7 @@ return:
 ;     }
 
 define void @f2(ptr %A, i64 %N) nounwind {
-; CHECK-LABEL: Detected Scops in Function f2
+; CHECK-LABEL: 'Polly - Detect static control parts (SCoPs)' for function 'f2'
 entry:
   fence seq_cst
   br label %for.i.1
diff --git a/polly/test/ScopDetect/simple_loop.ll b/polly/test/ScopDetect/simple_loop.ll
index 376a7dfa9f9b..c8ed89a97d00 100644
--- a/polly/test/ScopDetect/simple_loop.ll
+++ b/polly/test/ScopDetect/simple_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/simple_loop_non_single_entry.ll b/polly/test/ScopDetect/simple_loop_non_single_entry.ll
index 64e2a084188d..22adec5d2039 100644
--- a/polly/test/ScopDetect/simple_loop_non_single_entry.ll
+++ b/polly/test/ScopDetect/simple_loop_non_single_entry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/simple_loop_non_single_exit.ll b/polly/test/ScopDetect/simple_loop_non_single_exit.ll
index 4c3a1aea1ad8..71ac830cae7d 100644
--- a/polly/test/ScopDetect/simple_loop_non_single_exit.ll
+++ b/polly/test/ScopDetect/simple_loop_non_single_exit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/simple_loop_non_single_exit_2.ll b/polly/test/ScopDetect/simple_loop_non_single_exit_2.ll
index ae4ffa7b4972..d9915dc130d5 100644
--- a/polly/test/ScopDetect/simple_loop_non_single_exit_2.ll
+++ b/polly/test/ScopDetect/simple_loop_non_single_exit_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/simple_loop_two_phi_nodes.ll b/polly/test/ScopDetect/simple_loop_two_phi_nodes.ll
index a6bca0ec9d73..867bd50513f0 100644
--- a/polly/test/ScopDetect/simple_loop_two_phi_nodes.ll
+++ b/polly/test/ScopDetect/simple_loop_two_phi_nodes.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/simple_loop_with_param.ll b/polly/test/ScopDetect/simple_loop_with_param.ll
index ab48709d290d..1ae5c6608739 100644
--- a/polly/test/ScopDetect/simple_loop_with_param.ll
+++ b/polly/test/ScopDetect/simple_loop_with_param.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=PHI
+; RUN: opt %loadPolly -basic-aa -polly-print-detect -disable-output < %s | FileCheck %s -check-prefix=PHI
 
 ; void f(long A[], long N, long *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopDetect/simple_loop_with_param_2.ll b/polly/test/ScopDetect/simple_loop_with_param_2.ll
index baa647489543..1a4750621c19 100644
--- a/polly/test/ScopDetect/simple_loop_with_param_2.ll
+++ b/polly/test/ScopDetect/simple_loop_with_param_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopDetect/simple_non_single_entry.ll b/polly/test/ScopDetect/simple_non_single_entry.ll
index 1f1cc95147a5..a1995a427903 100644
--- a/polly/test/ScopDetect/simple_non_single_entry.ll
+++ b/polly/test/ScopDetect/simple_non_single_entry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; void f(long A[], long N) {
 ;   long i;
diff --git a/polly/test/ScopDetect/skip_function_attribute.ll b/polly/test/ScopDetect/skip_function_attribute.ll
index d30c042fb74e..e85dbd4c2b83 100644
--- a/polly/test/ScopDetect/skip_function_attribute.ll
+++ b/polly/test/ScopDetect/skip_function_attribute.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; Verify polly skips this function
 ;
diff --git a/polly/test/ScopDetect/srem_with_parametric_divisor.ll b/polly/test/ScopDetect/srem_with_parametric_divisor.ll
index 9a6352b9afe6..4b5c3b04c2ce 100644
--- a/polly/test/ScopDetect/srem_with_parametric_divisor.ll
+++ b/polly/test/ScopDetect/srem_with_parametric_divisor.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 ;
 ; CHECK-NOT: Valid Region for Scop:
 ;
diff --git a/polly/test/ScopDetect/statistics.ll b/polly/test/ScopDetect/statistics.ll
index 5789677325d8..64df3d081605 100644
--- a/polly/test/ScopDetect/statistics.ll
+++ b/polly/test/ScopDetect/statistics.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -stats -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-detect -stats -disable-output < %s 2>&1 | FileCheck %s
 
 ; REQUIRES: asserts
 
diff --git a/polly/test/ScopDetect/switch-in-loop-patch.ll b/polly/test/ScopDetect/switch-in-loop-patch.ll
index 508f59ee398e..ab4729fc09a4 100644
--- a/polly/test/ScopDetect/switch-in-loop-patch.ll
+++ b/polly/test/ScopDetect/switch-in-loop-patch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
 
 ; CHECK-NOT: Valid
 
diff --git a/polly/test/ScopDetectionDiagnostics/ReportAlias-01.ll b/polly/test/ScopDetectionDiagnostics/ReportAlias-01.ll
index 8f575df22f84..97ba7f9634e9 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportAlias-01.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportAlias-01.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-use-runtime-alias-checks=false -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -polly-use-runtime-alias-checks=false -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1| FileCheck %s
 
 ;void f(int A[], int B[]) {
 ;  for (int i=0; i<42; i++)
diff --git a/polly/test/ScopDetectionDiagnostics/ReportEntry.ll b/polly/test/ScopDetectionDiagnostics/ReportEntry.ll
index f80b48fc3b22..fc21e192f32c 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportEntry.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportEntry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -polly-detect -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
 
 ; CHECK: remark: <unknown>:0:0: Scop contains function entry (not yet supported).
 
diff --git a/polly/test/ScopDetectionDiagnostics/ReportFuncCall-01.ll b/polly/test/ScopDetectionDiagnostics/ReportFuncCall-01.ll
index 2d9175607ba4..abace4ba520d 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportFuncCall-01.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportFuncCall-01.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1 | FileCheck %s
 
 ; #define N 1024
 ; double invalidCall(double A[N]);
diff --git a/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegion.ll b/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegion.ll
index 8f5f08fb27c1..8368a68b42f0 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegion.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ;void foo(int a, int b) {
diff --git a/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegionWithoutDebugLoc.ll b/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegionWithoutDebugLoc.ll
index f5ca683f0fd5..82c6c33e287c 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegionWithoutDebugLoc.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportIrreducibleRegionWithoutDebugLoc.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -polly-detect -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
 
 ; CHECK: remark: <unknown>:0:0: Irreducible region encountered in control flow.
 
diff --git a/polly/test/ScopDetectionDiagnostics/ReportLoopBound-01.ll b/polly/test/ScopDetectionDiagnostics/ReportLoopBound-01.ll
index 27d26e665193..35986b5e0b35 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportLoopBound-01.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportLoopBound-01.ll
@@ -1,15 +1,15 @@
 ; RUN: opt %loadPolly \
 ; RUN:     -pass-remarks-missed="polly-detect" -polly-detect-track-failures \
-; RUN:     -polly-allow-nonaffine-loops=false '-passes=print<polly-detect>' -disable-output \
+; RUN:     -polly-allow-nonaffine-loops=false -polly-print-detect -disable-output \
 ; RUN:     < %s 2>&1| FileCheck %s --check-prefix=REJECTNONAFFINELOOPS
 ; RUN: opt %loadPolly \
 ; RUN:     -pass-remarks-missed="polly-detect" -polly-detect-track-failures \
-; RUN:     -polly-allow-nonaffine-loops=true '-passes=print<polly-detect>' -disable-output \
+; RUN:     -polly-allow-nonaffine-loops=true -polly-print-detect -disable-output \
 ; RUN:     < %s 2>&1| FileCheck %s --check-prefix=ALLOWNONAFFINELOOPS
 ; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" \
 ; RUN:     -polly-process-unprofitable=false \
 ; RUN:     -polly-detect-track-failures -polly-allow-nonaffine-loops=true \
-; RUN:     -polly-allow-nonaffine '-passes=print<polly-detect>' -disable-output < %s 2>&1 \
+; RUN:     -polly-allow-nonaffine -polly-print-detect -disable-output < %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=ALLOWNONAFFINEALL
 
 ; void f(int A[], int n) {
diff --git a/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll b/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll
index a40b423f04fc..5dbeaded45c9 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportLoopHasNoExit.ll
@@ -4,8 +4,8 @@
 ; the PostDominatorTree. Infinite loops are postdominated ony by the virtual
 ; root, which causes them not to appear in regions in ScopDetection anymore.
 
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-allow-nonaffine-loops '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-allow-nonaffine-loops=false '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-allow-nonaffine-loops -polly-print-detect -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-allow-nonaffine-loops=false -polly-print-detect -disable-output < %s 2>&1 | FileCheck %s
 
 ; void func (int param0, int N, int *A)
 ; {
diff --git a/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll b/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll
index b5eaaea1327e..634b63e6d44d 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportMultipleNonAffineAccesses.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1| FileCheck %s
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -polly-delinearize=false -polly-detect-keep-going -disable-output < %s 2>&1| FileCheck %s -check-prefix=ALL
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1| FileCheck %s -check-prefix=DELIN
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -polly-detect-keep-going -disable-output < %s 2>&1| FileCheck %s -check-prefix=DELIN-ALL
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -polly-allow-nonaffine -disable-output < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -polly-allow-nonaffine -disable-output < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -polly-delinearize=false -polly-detect-keep-going -disable-output < %s 2>&1| FileCheck %s -check-prefix=ALL
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1| FileCheck %s -check-prefix=DELIN
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -polly-detect-keep-going -disable-output < %s 2>&1| FileCheck %s -check-prefix=DELIN-ALL
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -polly-allow-nonaffine -disable-output < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
+; RUN: opt %loadPolly -basic-aa -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -polly-allow-nonaffine -disable-output < %s 2>&1| FileCheck %s -check-prefix=NONAFFINE
 
 ;  1 void manyaccesses(float A[restrict], long n, float B[restrict][n])
 ;  2 {
diff --git a/polly/test/ScopDetectionDiagnostics/ReportNonAffineAccess-01.ll b/polly/test/ScopDetectionDiagnostics/ReportNonAffineAccess-01.ll
index 369a464a0f77..23d8c9c061c9 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportNonAffineAccess-01.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportNonAffineAccess-01.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1| FileCheck %s
 
 ; void f(int A[]) {
 ;   for(int i=0; i<42; ++i)
diff --git a/polly/test/ScopDetectionDiagnostics/ReportUnprofitable.ll b/polly/test/ScopDetectionDiagnostics/ReportUnprofitable.ll
index c606fc2b6921..d35b7a28ba89 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportUnprofitable.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportUnprofitable.ll
@@ -1,9 +1,9 @@
 ; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" \
-; RUN:     -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output \
+; RUN:     -polly-detect-track-failures -polly-print-detect -disable-output \
 ; RUN:     -polly-process-unprofitable=false < %s 2>&1| FileCheck %s
 
 ; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" \
-; RUN:     -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output \
+; RUN:     -polly-detect-track-failures -polly-print-detect -disable-output \
 ; RUN:     -polly-process-unprofitable=false < %s 2>&1 -pass-remarks-output=%t.yaml
 ; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/ScopDetectionDiagnostics/ReportUnreachableInExit.ll b/polly/test/ScopDetectionDiagnostics/ReportUnreachableInExit.ll
index 5b20d6bea3dc..6c868db78ce7 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportUnreachableInExit.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportUnreachableInExit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s \
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s \
 ; RUN:     -pass-remarks-missed="polly-detect" 2>&1 | FileCheck %s
 
 ; void f(long A[], long N) {
diff --git a/polly/test/ScopDetectionDiagnostics/ReportVariantBasePtr-01.ll b/polly/test/ScopDetectionDiagnostics/ReportVariantBasePtr-01.ll
index ca10b0ac0256..a82f56b7a5fa 100644
--- a/polly/test/ScopDetectionDiagnostics/ReportVariantBasePtr-01.ll
+++ b/polly/test/ScopDetectionDiagnostics/ReportVariantBasePtr-01.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-print-detect -disable-output < %s 2>&1| FileCheck %s
 
 ; struct b {
 ;   double **b;
diff --git a/polly/test/ScopDetectionDiagnostics/loop_has_multiple_exits.ll b/polly/test/ScopDetectionDiagnostics/loop_has_multiple_exits.ll
index 5d0a2bf8f1d5..a0f2704b1372 100644
--- a/polly/test/ScopDetectionDiagnostics/loop_has_multiple_exits.ll
+++ b/polly/test/ScopDetectionDiagnostics/loop_has_multiple_exits.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures '-passes=print<polly-detect>' -disable-output 2>&1 < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -pass-remarks-missed="polly-detect" -polly-detect-track-failures -polly-detect -disable-output 2>&1 < %s | FileCheck %s -match-full-lines
 ;
 ; Derived from test-suite/MultiSource/Benchmarks/BitBench/uuencode/uuencode.c
 ;
diff --git a/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop-2.ll b/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop-2.ll
index 4370380cb71e..667ed7d18ab5 100644
--- a/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop-2.ll
+++ b/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
 
 ; CHECK: remark: <unknown>:0:0: Loop cannot be handled because not all latches are part of loop region.
 
diff --git a/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop.ll b/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop.ll
index 05bd165d38cf..9dce56a3a3c4 100644
--- a/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop.ll
+++ b/polly/test/ScopDetectionDiagnostics/loop_partially_in_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -pass-remarks-missed="polly-detect" -disable-output < %s 2>&1| FileCheck %s
 
 ; CHECK: remark: <unknown>:0:0: Loop cannot be handled because not all latches are part of loop region.
 ; CHECK: remark: <unknown>:0:0: Loop cannot be handled because not all latches are part of loop region.
diff --git a/polly/test/ScopInfo/20110312-Fail-without-basicaa.ll b/polly/test/ScopInfo/20110312-Fail-without-basicaa.ll
index 6f08e433c4ee..94dd5824777c 100644
--- a/polly/test/ScopInfo/20110312-Fail-without-basicaa.ll
+++ b/polly/test/ScopInfo/20110312-Fail-without-basicaa.ll
@@ -1,5 +1,5 @@
 ; This should be run without alias analysis enabled.
-;RUN: opt %loadPolly '-passes=print<polly-function-scops>'  -disable-output < %s
+;RUN: opt %loadPolly -polly-scops  -disable-output < %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
 
 define i32 @main() nounwind {
diff --git a/polly/test/ScopInfo/20111108-Parameter-not-detected.ll b/polly/test/ScopInfo/20111108-Parameter-not-detected.ll
index 531e4149cffa..f80177cb90e7 100644
--- a/polly/test/ScopInfo/20111108-Parameter-not-detected.ll
+++ b/polly/test/ScopInfo/20111108-Parameter-not-detected.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 declare void @foo()
diff --git a/polly/test/ScopInfo/2012-03-16-Crash-because-of-unsigned-in-scev.ll b/polly/test/ScopInfo/2012-03-16-Crash-because-of-unsigned-in-scev.ll
index d92b35de2de7..b55d635947e5 100644
--- a/polly/test/ScopInfo/2012-03-16-Crash-because-of-unsigned-in-scev.ll
+++ b/polly/test/ScopInfo/2012-03-16-Crash-because-of-unsigned-in-scev.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-a0:0-n32"
 
diff --git a/polly/test/ScopInfo/2015-10-04-Crash-in-domain-generation.ll b/polly/test/ScopInfo/2015-10-04-Crash-in-domain-generation.ll
index 05165ed02a90..d4d931fd2e0c 100644
--- a/polly/test/ScopInfo/2015-10-04-Crash-in-domain-generation.ll
+++ b/polly/test/ScopInfo/2015-10-04-Crash-in-domain-generation.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-scops -disable-output < %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/Alias-0.ll b/polly/test/ScopInfo/Alias-0.ll
index 4e7e8fa11a08..0fc4ad91b7db 100644
--- a/polly/test/ScopInfo/Alias-0.ll
+++ b/polly/test/ScopInfo/Alias-0.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
+; RUN: opt %loadPolly -polly-print-scops -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/Alias-1.ll b/polly/test/ScopInfo/Alias-1.ll
index 7a734f810976..eab8c062f4ba 100644
--- a/polly/test/ScopInfo/Alias-1.ll
+++ b/polly/test/ScopInfo/Alias-1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
+; RUN: opt %loadPolly -polly-print-scops -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/Alias-2.ll b/polly/test/ScopInfo/Alias-2.ll
index 9d3d44826237..64f1e0bc919d 100644
--- a/polly/test/ScopInfo/Alias-2.ll
+++ b/polly/test/ScopInfo/Alias-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
+; RUN: opt %loadPolly -polly-print-scops -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/Alias-3.ll b/polly/test/ScopInfo/Alias-3.ll
index 83d68ddf371b..5e9b94e692bc 100644
--- a/polly/test/ScopInfo/Alias-3.ll
+++ b/polly/test/ScopInfo/Alias-3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
+; RUN: opt %loadPolly -polly-print-scops -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/Alias-4.ll b/polly/test/ScopInfo/Alias-4.ll
index bdcf729f4061..4d5a91abb96f 100644
--- a/polly/test/ScopInfo/Alias-4.ll
+++ b/polly/test/ScopInfo/Alias-4.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline= '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
-; RUN: opt %loadPolly -aa-pipeline= '-passes=print<polly-detect>,print<polly-function-scops>' -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
+; RUN: opt %loadPolly -disable-basic-aa -polly-print-scops -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=RTA
+; RUN: opt %loadPolly -disable-basic-aa -polly-print-scops -polly-use-runtime-alias-checks=false -disable-output < %s -stats 2>&1 | FileCheck %s --check-prefix=NORTA
 ; REQUIRES: asserts
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/BoundChecks/single-loop.ll b/polly/test/ScopInfo/BoundChecks/single-loop.ll
index 0ada318baabf..bc96c907afc9 100644
--- a/polly/test/ScopInfo/BoundChecks/single-loop.ll
+++ b/polly/test/ScopInfo/BoundChecks/single-loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
 ;
 ; This only works after the post-dominator tree has been fixed.
 ;
diff --git a/polly/test/ScopInfo/BoundChecks/two-loops.ll b/polly/test/ScopInfo/BoundChecks/two-loops.ll
index 38ea23b80b9d..14e07f42a3ae 100644
--- a/polly/test/ScopInfo/BoundChecks/two-loops.ll
+++ b/polly/test/ScopInfo/BoundChecks/two-loops.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output< %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -polly-print-scops -disable-output< %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
 ;
 ; This only works after the post-dominator tree has fixed.
 ; XFAIL: *
diff --git a/polly/test/ScopInfo/NonAffine/div_backedge.ll b/polly/test/ScopInfo/NonAffine/div_backedge.ll
index 69af32d92325..a6aca032ef62 100644
--- a/polly/test/ScopInfo/NonAffine/div_backedge.ll
+++ b/polly/test/ScopInfo/NonAffine/div_backedge.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void foo(float *A) {
 ;      for (long i = 1;; i++) {
diff --git a/polly/test/ScopInfo/NonAffine/div_domain.ll b/polly/test/ScopInfo/NonAffine/div_domain.ll
index 27cc284f53c4..f61c4eb459ed 100644
--- a/polly/test/ScopInfo/NonAffine/div_domain.ll
+++ b/polly/test/ScopInfo/NonAffine/div_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void foo(float *A) {
 ;      for (long i = 0; i < 16; i++) {
diff --git a/polly/test/ScopInfo/NonAffine/invariant_loads_dependent_in_non_affine_region.ll b/polly/test/ScopInfo/NonAffine/invariant_loads_dependent_in_non_affine_region.ll
index 4cf60324f99b..f5d63dfb9d2c 100644
--- a/polly/test/ScopInfo/NonAffine/invariant_loads_dependent_in_non_affine_region.ll
+++ b/polly/test/ScopInfo/NonAffine/invariant_loads_dependent_in_non_affine_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, int *B, int *C) {
 ;      for (int i = 0; i < 1000; i++)
diff --git a/polly/test/ScopInfo/NonAffine/modulo_backedge.ll b/polly/test/ScopInfo/NonAffine/modulo_backedge.ll
index 322720ae0633..dec63ca6813d 100644
--- a/polly/test/ScopInfo/NonAffine/modulo_backedge.ll
+++ b/polly/test/ScopInfo/NonAffine/modulo_backedge.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Domain :=
 ; CHECK:   { Stmt_for_body[i0] : 0 <= i0 <= 6 };
diff --git a/polly/test/ScopInfo/NonAffine/modulo_domain.ll b/polly/test/ScopInfo/NonAffine/modulo_domain.ll
index cbd9d8901ce3..f5ebec2b0346 100644
--- a/polly/test/ScopInfo/NonAffine/modulo_domain.ll
+++ b/polly/test/ScopInfo/NonAffine/modulo_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; TODO: The new domain generation cannot handle modulo domain constraints,
 ;       hence modulo handling has been disabled completely. Once this is
diff --git a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll
index e38de4ab6aba..837d9b21b16e 100644
--- a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll
+++ b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                   '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=SCALAR
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-process-unprofitable=false '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=PROFIT
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                                   -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=SCALAR
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-process-unprofitable=false -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=PROFIT
 ;
 ; SCALAR:      Function: f
 ; SCALAR-NEXT: Region: %bb1---%bb13
diff --git a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll
index 4e792e5e4e50..e39569abc52d 100644
--- a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll
+++ b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_2.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                       '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                        '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALL
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                       -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                        -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=ALL
 ;
 ; Here we have a non-affine loop (in the context of the loop nest)
 ; and also a non-affine access (A[k]). While we can always model the
diff --git a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll
index 1845505a5a2d..75dd7ac26bb3 100644
--- a/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll
+++ b/polly/test/ScopInfo/NonAffine/non-affine-loop-condition-dependent-access_3.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                       '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                        '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALL
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=false                       -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true                        -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=ALL
 ;
 ; Here we have a non-affine loop (in the context of the loop nest)
 ; and also a non-affine access (A[k]). While we can always model the
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_access_with_range_2.ll b/polly/test/ScopInfo/NonAffine/non_affine_access_with_range_2.ll
index 2ba4065770bd..34b04933af86 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_access_with_range_2.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_access_with_range_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A) {
 ;      for (int i = 0; i < 128; i++)
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_but_sdiv.ll b/polly/test/ScopInfo/NonAffine/non_affine_but_sdiv.ll
index 8ba1f013fda4..9955c88b2cfd 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_but_sdiv.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_but_sdiv.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      Statements {
 ; CHECK-NEXT:     Stmt_for_body
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_but_srem.ll b/polly/test/ScopInfo/NonAffine/non_affine_but_srem.ll
index 059129168bad..b194ee762e9f 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_but_srem.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_but_srem.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void pos(float *A, long n) {
 ;      for (long i = 0; i < 100; i++)
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_conditional_nested.ll b/polly/test/ScopInfo/NonAffine/non_affine_conditional_nested.ll
index a25b27297b0f..1f55530b137d 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_conditional_nested.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_conditional_nested.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_affine_loop.ll b/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_affine_loop.ll
index 1f66b8a096c9..3511362304b4 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_affine_loop.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_affine_loop.ll
@@ -1,11 +1,11 @@
 ; RUN: opt %loadPolly -polly-allow-nonaffine-branches \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -polly-allow-nonaffine-loops=true \
-; RUN:     '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
+; RUN:     -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
 ; RUN: opt %loadPolly -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true \
-; RUN:     '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s \
+; RUN:     -polly-print-scops -disable-output < %s | FileCheck %s \
 ; RUN:     --check-prefix=ALL
 ;
 ; Negative test for INNERMOST.
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_non_affine_loop.ll b/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_non_affine_loop.ll
index 5127481dde29..c2e1e46f6f18 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_non_affine_loop.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_conditional_surrounding_non_affine_loop.ll
@@ -1,16 +1,16 @@
 ; RUN: opt %loadPolly -polly-allow-nonaffine-branches \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -polly-allow-nonaffine-loops=true \
-; RUN:     '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=INNERMOST
+; RUN:     -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=INNERMOST
 ; RUN: opt %loadPolly -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true \
-; RUN:     '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ALL
+; RUN:     -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=ALL
 ; RUN: opt %loadPolly -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -polly-process-unprofitable=false \
 ; RUN:     -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops=true \
-; RUN:     '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
+; RUN:     -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=PROFIT
 ;
 ; Negative test for INNERMOST.
 ; At the moment we will optimistically assume A[i] in the conditional before the inner
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_float_compare.ll b/polly/test/ScopInfo/NonAffine/non_affine_float_compare.ll
index ced23ab8f8df..c62447b6c15c 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_float_compare.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_float_compare.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(float *A) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_loop_condition.ll b/polly/test/ScopInfo/NonAffine/non_affine_loop_condition.ll
index ce00233fd7af..873b44b9c8cf 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_loop_condition.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_loop_condition.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops                                   '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-process-unprofitable=false '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
-; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-detect-reductions=false    '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=NO-REDUCTION
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops                                   -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-process-unprofitable=false -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-detect-reductions=false    -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=NO-REDUCTION
 ;
 ;    void f(int *A, int *C) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll b/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll
index daf06a3b5f89..127bf80b9451 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_loop_used_later.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops                                                                        '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-unprofitable-scalar-accs=true -polly-process-unprofitable=false '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=PROFIT
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops                                                                        -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-allow-nonaffine-branches -polly-allow-nonaffine-loops -polly-unprofitable-scalar-accs=true -polly-process-unprofitable=false -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=PROFIT
 ;
 ; Verify that we over approximate the read acces of A[j] in the last statement as j is
 ; computed in a non-affine loop we do not model.
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll b/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll
index be94e0e3307e..de011e29aeea 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_parametric_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, double A[], int INDEX[]) {
diff --git a/polly/test/ScopInfo/NonAffine/non_affine_region_guaranteed_non-entry.ll b/polly/test/ScopInfo/NonAffine/non_affine_region_guaranteed_non-entry.ll
index a4daca4d9f36..7303b4ea47fd 100644
--- a/polly/test/ScopInfo/NonAffine/non_affine_region_guaranteed_non-entry.ll
+++ b/polly/test/ScopInfo/NonAffine/non_affine_region_guaranteed_non-entry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-detect '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-detect -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/NonAffine/whole-scop-non-affine-subregion-in-loop.ll b/polly/test/ScopInfo/NonAffine/whole-scop-non-affine-subregion-in-loop.ll
index ba9146a8eca7..4f54d03d43fb 100644
--- a/polly/test/ScopInfo/NonAffine/whole-scop-non-affine-subregion-in-loop.ll
+++ b/polly/test/ScopInfo/NonAffine/whole-scop-non-affine-subregion-in-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
 ;
 ; Regression test that triggered a memory leak at some point (24947).
 ;
diff --git a/polly/test/ScopInfo/aliasing_conditional_alias_groups_1.ll b/polly/test/ScopInfo/aliasing_conditional_alias_groups_1.ll
index a2836c199514..dc59fbfc66a8 100644
--- a/polly/test/ScopInfo/aliasing_conditional_alias_groups_1.ll
+++ b/polly/test/ScopInfo/aliasing_conditional_alias_groups_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that there is no alias group because we either access A or B never both.
 ;
diff --git a/polly/test/ScopInfo/aliasing_conditional_alias_groups_2.ll b/polly/test/ScopInfo/aliasing_conditional_alias_groups_2.ll
index 15cea1c4f8cb..a19d60dd9147 100644
--- a/polly/test/ScopInfo/aliasing_conditional_alias_groups_2.ll
+++ b/polly/test/ScopInfo/aliasing_conditional_alias_groups_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that we create two alias groups since the minimal/maximal accesses
 ; depend on %b.
diff --git a/polly/test/ScopInfo/aliasing_dead_access.ll b/polly/test/ScopInfo/aliasing_dead_access.ll
index 400fea0573a1..2a725cf3c855 100644
--- a/polly/test/ScopInfo/aliasing_dead_access.ll
+++ b/polly/test/ScopInfo/aliasing_dead_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that we do not create a SCoP if there is no statement executed.
 ;
diff --git a/polly/test/ScopInfo/aliasing_many_arrays_to_compare.ll b/polly/test/ScopInfo/aliasing_many_arrays_to_compare.ll
index a1b954cb63ee..937d4ada3ec9 100644
--- a/polly/test/ScopInfo/aliasing_many_arrays_to_compare.ll
+++ b/polly/test/ScopInfo/aliasing_many_arrays_to_compare.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output \
-; RUN:                < %s 2>&1 | FileCheck %s --check-prefix=FOUND
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output \
-; RUN:                -polly-rtc-max-arrays-per-group=3 < %s 2>&1 | FileCheck %s \
+; RUN: opt %loadPolly -polly-print-scops -disable-output \
+; RUN:                < %s | FileCheck %s --check-prefix=FOUND
+; RUN: opt %loadPolly -polly-print-scops -disable-output \
+; RUN:                -polly-rtc-max-arrays-per-group=3 < %s | FileCheck %s \
 ; RUN:                --check-prefix=IGNORED
 ;
 ; FOUND: Function: foo
diff --git a/polly/test/ScopInfo/aliasing_many_read_only_acesses.ll b/polly/test/ScopInfo/aliasing_many_read_only_acesses.ll
index 9447744a12a8..c22cfe55e118 100644
--- a/polly/test/ScopInfo/aliasing_many_read_only_acesses.ll
+++ b/polly/test/ScopInfo/aliasing_many_read_only_acesses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Assumed Context:
 ; CHECK-NEXT: { : }
diff --git a/polly/test/ScopInfo/aliasing_multiple_alias_groups.ll b/polly/test/ScopInfo/aliasing_multiple_alias_groups.ll
index 5c4a864ec777..16cb3dc0f5ac 100644
--- a/polly/test/ScopInfo/aliasing_multiple_alias_groups.ll
+++ b/polly/test/ScopInfo/aliasing_multiple_alias_groups.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -aa-pipeline=         < %s 2>&1 | FileCheck %s --check-prefix=NOAA
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -aa-pipeline=tbaa    < %s 2>&1 | FileCheck %s --check-prefix=TBAA
+; RUN: opt %loadPolly -polly-print-scops -disable-output          < %s | FileCheck %s --check-prefix=NOAA
+; RUN: opt %loadPolly -polly-print-scops -disable-output -tbaa    < %s | FileCheck %s --check-prefix=TBAA
 ;
 ;    void jd(int *Int0, int *Int1, float *Float0, float *Float1) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/ScopInfo/aliasing_with_non_affine_access.ll b/polly/test/ScopInfo/aliasing_with_non_affine_access.ll
index 76bc18e8ac53..056b644cd5ed 100644
--- a/polly/test/ScopInfo/aliasing_with_non_affine_access.ll
+++ b/polly/test/ScopInfo/aliasing_with_non_affine_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -polly-process-unprofitable -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -polly-process-unprofitable -polly-allow-nonaffine -disable-output < %s | FileCheck %s
 ;
 ; @test1
 ; Make sure we generate the correct aliasing check for a fixed-size memset operation.
diff --git a/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll b/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll
index b37b560599ad..d170a50e26fc 100644
--- a/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll
+++ b/polly/test/ScopInfo/allow-all-parameters-dereferencable.ll
@@ -1,14 +1,14 @@
 ; RUN: opt %loadPolly -disable-output -polly-invariant-load-hoisting \
 ; RUN: -polly-allow-dereference-of-all-function-parameters \
-; RUN: '-passes=print<polly-function-scops>' < %s 2>&1 | FileCheck %s --check-prefix=SCOP
+; RUN: -polly-print-scops < %s | FileCheck %s --check-prefix=SCOP
 
 ; RUN: opt %loadPolly -S -polly-invariant-load-hoisting \
-; RUN: -passes=polly-codegen < %s 2>&1 | FileCheck %s --check-prefix=CODE-RTC
+; RUN: -polly-codegen < %s | FileCheck %s --check-prefix=CODE-RTC
 
 
 ; RUN: opt %loadPolly -S -polly-invariant-load-hoisting \
 ; RUN: -polly-allow-dereference-of-all-function-parameters \
-; RUN: -passes=polly-codegen < %s 2>&1 | FileCheck %s --check-prefix=CODE
+; RUN: -polly-codegen < %s | FileCheck %s --check-prefix=CODE
 
 ; SCOP:      Function: hoge
 ; SCOP-NEXT: Region: %bb15---%bb37
diff --git a/polly/test/ScopInfo/assume_gep_bounds.ll b/polly/test/ScopInfo/assume_gep_bounds.ll
index 7b7fd15e3346..d0ce47148071 100644
--- a/polly/test/ScopInfo/assume_gep_bounds.ll
+++ b/polly/test/ScopInfo/assume_gep_bounds.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
 
 ;    void foo(float A[][20][30], long n, long m, long p) {
 ;      for (long i = 0; i < n; i++)
diff --git a/polly/test/ScopInfo/assume_gep_bounds_2.ll b/polly/test/ScopInfo/assume_gep_bounds_2.ll
index 3ada0f817300..e327195da94c 100644
--- a/polly/test/ScopInfo/assume_gep_bounds_2.ll
+++ b/polly/test/ScopInfo/assume_gep_bounds_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output < %s \
 ; RUN:  -polly-precise-inbounds | FileCheck %s
 ;
 ;    void foo(float A[restrict][20], float B[restrict][20], long n, long m,
diff --git a/polly/test/ScopInfo/assume_gep_bounds_many.ll b/polly/test/ScopInfo/assume_gep_bounds_many.ll
index 2106b96ecd56..261491564fc2 100644
--- a/polly/test/ScopInfo/assume_gep_bounds_many.ll
+++ b/polly/test/ScopInfo/assume_gep_bounds_many.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -disable-output '-passes=print<polly-function-scops>' -polly-ignore-aliasing \
-; RUN:    < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -disable-output -polly-print-scops -polly-ignore-aliasing \
+; RUN:    < %s | FileCheck %s
 
 ; CHECK: Assumed Context:
 ; CHECK-NEXT: [n1_a, n1_b, n1_c, n1_d, n2_a, n2_b, n2_c, n2_d, n3_a, n3_b, n3_c, n3_d, n4_a, n4_b, n4_c, n4_d, n5_a, n5_b, n5_c, n5_d, n6_a, n6_b, n6_c, n6_d, n7_a, n7_b, n7_c, n7_d, n8_a, n8_b, n8_c, n8_d, n9_a, n9_b, n9_c, n9_d, p1_b, p1_c, p1_d, p2_b, p2_c, p2_d, p3_b, p3_c, p3_d, p4_b, p4_c, p4_d, p5_b, p5_c, p5_d, p6_b, p6_c, p6_d, p7_b, p7_c, p7_d, p8_b, p8_c, p8_d, p9_b, p9_c, p9_d] -> {  : p1_b >= n1_b and p1_c >= n1_c and p1_d >= n1_d and p2_b >= n2_b and p2_c >= n2_c and p2_d >= n2_d and p3_b >= n3_b and p3_c >= n3_c and p3_d >= n3_d and p4_b >= n4_b and p4_c >= n4_c and p4_d >= n4_d and p5_b >= n5_b and p5_c >= n5_c and p5_d >= n5_d and p6_b >= n6_b and p6_c >= n6_c and p6_d >= n6_d and p7_b >= n7_b and p7_c >= n7_c and p7_d >= n7_d and p8_b >= n8_b and p8_c >= n8_c and p8_d >= n8_d and p9_b >= n9_b and p9_c >= n9_c and p9_d >= n9_d }
diff --git a/polly/test/ScopInfo/avoid_new_parameters_from_geps.ll b/polly/test/ScopInfo/avoid_new_parameters_from_geps.ll
index e81b791f9312..0e17eb1d3668 100644
--- a/polly/test/ScopInfo/avoid_new_parameters_from_geps.ll
+++ b/polly/test/ScopInfo/avoid_new_parameters_from_geps.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that we do no introduce a parameter here that is actually not needed.
 ;
diff --git a/polly/test/ScopInfo/bool-addrec.ll b/polly/test/ScopInfo/bool-addrec.ll
index 51687cd9caa5..1924a4b5266b 100644
--- a/polly/test/ScopInfo/bool-addrec.ll
+++ b/polly/test/ScopInfo/bool-addrec.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -disable-output '-passes=print<polly-ast>' -polly-process-unprofitable < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -disable-output -polly-print-ast -polly-process-unprofitable < %s | FileCheck %s
 
 ; CHECK:      for (int c0 = 0; c0 <= 19999; c0 += 1) {
 ; CHECK-NEXT:   if (c0 % 2 == 0)
diff --git a/polly/test/ScopInfo/bounded_loop_assumptions.ll b/polly/test/ScopInfo/bounded_loop_assumptions.ll
index 6b8acfc97c60..d472c7586c53 100644
--- a/polly/test/ScopInfo/bounded_loop_assumptions.ll
+++ b/polly/test/ScopInfo/bounded_loop_assumptions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; The assumed context is tricky here as the equality test for the inner loop
 ; allows an "unbounded" loop trip count. We assume that does not happen, thus
diff --git a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-2.ll b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-2.ll
index bbf151f0206a..5c5f264aab60 100644
--- a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-2.ll
+++ b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-2.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | \
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | \
 ; RUN:     FileCheck %s -check-prefix=DETECT
 
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | \
 ; RUN:     FileCheck %s -check-prefix=SCOP
 
 ; DETECT: Valid Region for Scop: loop => barrier
diff --git a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-3.ll b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-3.ll
index 9b72a74e8628..d69d3a16c0d7 100644
--- a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-3.ll
+++ b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations-3.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | \
 ; RUN:     FileCheck %s -check-prefix=NONAFFINE
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output \
-; RUN:     -polly-allow-nonaffine-branches=false < %s 2>&1 | \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output \
+; RUN:     -polly-allow-nonaffine-branches=false < %s | \
 ; RUN:     FileCheck %s -check-prefix=NO-NONEAFFINE
 
 ; NONAFFINE:      Statements {
diff --git a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations.ll b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations.ll
index c6b1cddeaf85..57918fa5c92d 100644
--- a/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations.ll
+++ b/polly/test/ScopInfo/branch-references-loop-scev-with-unknown-iterations.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | \
 ; RUN:     FileCheck %s -check-prefix=NONAFFINE
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output \
-; RUN:     -polly-allow-nonaffine-branches=false < %s 2>&1 | \
+; RUN: opt %loadPolly -polly-print-scops -disable-output \
+; RUN:     -polly-allow-nonaffine-branches=false < %s | \
 ; RUN:     FileCheck %s -check-prefix=NO-NONEAFFINE
 
 ; NONAFFINE-NOT: Statements
diff --git a/polly/test/ScopInfo/bug_2010_10_22.ll b/polly/test/ScopInfo/bug_2010_10_22.ll
index 2e492e7633a0..7ba996b6d0f1 100644
--- a/polly/test/ScopInfo/bug_2010_10_22.ll
+++ b/polly/test/ScopInfo/bug_2010_10_22.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-scops -disable-output < %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/ScopInfo/bug_2011_1_5.ll b/polly/test/ScopInfo/bug_2011_1_5.ll
index ce815e57e627..95c25f9d9cdb 100644
--- a/polly/test/ScopInfo/bug_2011_1_5.ll
+++ b/polly/test/ScopInfo/bug_2011_1_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-scops -disable-output < %s
 
 ; Bug description: Alias Analysis thinks IntToPtrInst aliases with alloca instructions created by IndependentBlocks Pass.
 ;                  This will trigger the assertion when we are verifying the SCoP after IndependentBlocks.
diff --git a/polly/test/ScopInfo/bug_scev_not_fully_eval.ll b/polly/test/ScopInfo/bug_scev_not_fully_eval.ll
index 8711f8e58a21..89d5f318829e 100644
--- a/polly/test/ScopInfo/bug_scev_not_fully_eval.ll
+++ b/polly/test/ScopInfo/bug_scev_not_fully_eval.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | not FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | not FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 @edge.8265 = external global [72 x i32], align 32 ; <ptr> [#uses=1]
diff --git a/polly/test/ScopInfo/cfg_consequences.ll b/polly/test/ScopInfo/cfg_consequences.ll
index bd23ec7fc3bf..84f94b135735 100644
--- a/polly/test/ScopInfo/cfg_consequences.ll
+++ b/polly/test/ScopInfo/cfg_consequences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; void consequences(int *A, int bool_cond, int lhs, int rhs) {
 ;
diff --git a/polly/test/ScopInfo/complex-branch-structure.ll b/polly/test/ScopInfo/complex-branch-structure.ll
index 69eb716e6877..24ebdcf213f8 100644
--- a/polly/test/ScopInfo/complex-branch-structure.ll
+++ b/polly/test/ScopInfo/complex-branch-structure.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 
 ; We build a scop of the following form to check that the domain construction
diff --git a/polly/test/ScopInfo/complex-condition.ll b/polly/test/ScopInfo/complex-condition.ll
index 348446a40007..31d34b033725 100644
--- a/polly/test/ScopInfo/complex-condition.ll
+++ b/polly/test/ScopInfo/complex-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/complex-expression.ll b/polly/test/ScopInfo/complex-expression.ll
index 1340b0e0c981..1822c9de852a 100644
--- a/polly/test/ScopInfo/complex-expression.ll
+++ b/polly/test/ScopInfo/complex-expression.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/complex-loop-nesting.ll b/polly/test/ScopInfo/complex-loop-nesting.ll
index a0e4a1e92050..97a9bfd939d5 100644
--- a/polly/test/ScopInfo/complex-loop-nesting.ll
+++ b/polly/test/ScopInfo/complex-loop-nesting.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/complex-successor-structure-2.ll b/polly/test/ScopInfo/complex-successor-structure-2.ll
index eceadc818b48..6bb7bb14a8cc 100644
--- a/polly/test/ScopInfo/complex-successor-structure-2.ll
+++ b/polly/test/ScopInfo/complex-successor-structure-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 
diff --git a/polly/test/ScopInfo/complex-successor-structure-3.ll b/polly/test/ScopInfo/complex-successor-structure-3.ll
index b5ba6958c57b..14c3fc1babeb 100644
--- a/polly/test/ScopInfo/complex-successor-structure-3.ll
+++ b/polly/test/ScopInfo/complex-successor-structure-3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -disable-output '-passes=print<polly-function-scops>' \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -disable-output -polly-print-scops \
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; Check that propagation of domains from A(X) to A(X+1) will keep the
 ; domains small and concise.
diff --git a/polly/test/ScopInfo/complex-successor-structure.ll b/polly/test/ScopInfo/complex-successor-structure.ll
index f39ab9bfa29b..364344045a6a 100644
--- a/polly/test/ScopInfo/complex-successor-structure.ll
+++ b/polly/test/ScopInfo/complex-successor-structure.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 
diff --git a/polly/test/ScopInfo/complex_domain_binary_condition.ll b/polly/test/ScopInfo/complex_domain_binary_condition.ll
index 4dd39733eae2..cec26855debb 100644
--- a/polly/test/ScopInfo/complex_domain_binary_condition.ll
+++ b/polly/test/ScopInfo/complex_domain_binary_condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: Low complexity assumption: {  : false }
diff --git a/polly/test/ScopInfo/complex_execution_context.ll b/polly/test/ScopInfo/complex_execution_context.ll
index c58d6fe9a50a..164254308fa9 100644
--- a/polly/test/ScopInfo/complex_execution_context.ll
+++ b/polly/test/ScopInfo/complex_execution_context.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops \
 ; RUN:     -polly-invariant-load-hoisting=true \
 ; RUN:     -disable-output < %s 2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/cond_constant_in_loop.ll b/polly/test/ScopInfo/cond_constant_in_loop.ll
index 45cae34e1af2..ef7d857e1084 100644
--- a/polly/test/ScopInfo/cond_constant_in_loop.ll
+++ b/polly/test/ScopInfo/cond_constant_in_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ;void f(long a[], long N, long M) {
 ;  long i, j, k;
diff --git a/polly/test/ScopInfo/cond_in_loop.ll b/polly/test/ScopInfo/cond_in_loop.ll
index 2101b25e4799..2d435f6a6a93 100644
--- a/polly/test/ScopInfo/cond_in_loop.ll
+++ b/polly/test/ScopInfo/cond_in_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ;void f(long a[], long N, long M) {
 ;  long i, j, k;
diff --git a/polly/test/ScopInfo/condition-after-error-block-2.ll b/polly/test/ScopInfo/condition-after-error-block-2.ll
index e3025d0ca259..695d864e483c 100644
--- a/polly/test/ScopInfo/condition-after-error-block-2.ll
+++ b/polly/test/ScopInfo/condition-after-error-block-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; Verify that we do not allow PHI nodes such as %phi, if they reference an error
 ; block and are used by anything else than a terminator instruction.
diff --git a/polly/test/ScopInfo/condition-after-error-block-before-scop.ll b/polly/test/ScopInfo/condition-after-error-block-before-scop.ll
index 7a4d1467de46..184be3642f0c 100644
--- a/polly/test/ScopInfo/condition-after-error-block-before-scop.ll
+++ b/polly/test/ScopInfo/condition-after-error-block-before-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/polly/test/ScopInfo/condtion-after-error-block.ll b/polly/test/ScopInfo/condtion-after-error-block.ll
index 1a8681f829e4..92e743e2d879 100644
--- a/polly/test/ScopInfo/condtion-after-error-block.ll
+++ b/polly/test/ScopInfo/condtion-after-error-block.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; Verify that we allow scops containing uniform branch conditions, where all
 ; but one incoming block comes from an error condition.
diff --git a/polly/test/ScopInfo/const_srem_sdiv.ll b/polly/test/ScopInfo/const_srem_sdiv.ll
index cf243fc74a9a..3acca980da70 100644
--- a/polly/test/ScopInfo/const_srem_sdiv.ll
+++ b/polly/test/ScopInfo/const_srem_sdiv.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; See http://research.microsoft.com/pubs/151917/divmodnote-letter.pdf
 ;
diff --git a/polly/test/ScopInfo/constant-non-integer-branch-condition.ll b/polly/test/ScopInfo/constant-non-integer-branch-condition.ll
index 8c8beac8f304..fc95a4cc7891 100644
--- a/polly/test/ScopInfo/constant-non-integer-branch-condition.ll
+++ b/polly/test/ScopInfo/constant-non-integer-branch-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; At some point this caused a problem in the domain generation as we
 ; assumed any constant branch condition to be valid. However, only constant
diff --git a/polly/test/ScopInfo/constant_factor_in_parameter.ll b/polly/test/ScopInfo/constant_factor_in_parameter.ll
index ca7d094be300..1f0173c0edf9 100644
--- a/polly/test/ScopInfo/constant_factor_in_parameter.ll
+++ b/polly/test/ScopInfo/constant_factor_in_parameter.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -disable-output '-passes=print<polly-function-scops>' < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -disable-output '-passes=print<polly-function-scops>' < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -disable-output -polly-print-scops < %s | FileCheck %s
+; RUN: opt %loadPolly -disable-output -polly-print-function-scops < %s | FileCheck %s
 ;
 ; Check that the constant part of the N * M * 4 expression is not part of the
 ; parameter but explicit in the access function. This can avoid existentially
diff --git a/polly/test/ScopInfo/constant_functions_outside_scop_as_unknown.ll b/polly/test/ScopInfo/constant_functions_outside_scop_as_unknown.ll
index afb8ec3eabed..38b2b8958e2f 100644
--- a/polly/test/ScopInfo/constant_functions_outside_scop_as_unknown.ll
+++ b/polly/test/ScopInfo/constant_functions_outside_scop_as_unknown.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-print-scops -disable-output < %s | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/polly/test/ScopInfo/constant_start_integer.ll b/polly/test/ScopInfo/constant_start_integer.ll
index 94e43b324f98..aa6640c98f73 100644
--- a/polly/test/ScopInfo/constant_start_integer.ll
+++ b/polly/test/ScopInfo/constant_start_integer.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(float *input) {
diff --git a/polly/test/ScopInfo/debug_call.ll b/polly/test/ScopInfo/debug_call.ll
index ba74e68c9f36..93b5bc520a00 100644
--- a/polly/test/ScopInfo/debug_call.ll
+++ b/polly/test/ScopInfo/debug_call.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-debug-func=dbg_printf '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-debug-func=dbg_printf -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Check that the call to dbg_printf is accepted as a debug-function.
 ;
diff --git a/polly/test/ScopInfo/delinearize-together-all-data-refs.ll b/polly/test/ScopInfo/delinearize-together-all-data-refs.ll
index ac17ba005bd0..108392b27f07 100644
--- a/polly/test/ScopInfo/delinearize-together-all-data-refs.ll
+++ b/polly/test/ScopInfo/delinearize-together-all-data-refs.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
 ;   for (long i = 0; i < n-3; i++)
diff --git a/polly/test/ScopInfo/div_by_zero.ll b/polly/test/ScopInfo/div_by_zero.ll
index 74380f7a4c71..2205b85a9ebc 100644
--- a/polly/test/ScopInfo/div_by_zero.ll
+++ b/polly/test/ScopInfo/div_by_zero.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/do-not-model-error-block-accesses.ll b/polly/test/ScopInfo/do-not-model-error-block-accesses.ll
index 563e5d11474b..997e0d4b37cf 100644
--- a/polly/test/ScopInfo/do-not-model-error-block-accesses.ll
+++ b/polly/test/ScopInfo/do-not-model-error-block-accesses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
 
 ; Check that we do not crash on this input. Earlier this indeed crashed as
 ; we tried to model the access functions in an error block.
diff --git a/polly/test/ScopInfo/eager-binary-and-or-conditions.ll b/polly/test/ScopInfo/eager-binary-and-or-conditions.ll
index ee846d3ca98c..e9ad63c51b85 100644
--- a/polly/test/ScopInfo/eager-binary-and-or-conditions.ll
+++ b/polly/test/ScopInfo/eager-binary-and-or-conditions.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output< %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output< %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -disable-output < %s
 ;
 ; void or(float *A, long n, long m) {
 ;   for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/early_exit_for_complex_domains.ll b/polly/test/ScopInfo/early_exit_for_complex_domains.ll
index 2a8e6d15fea4..a72ea031c236 100644
--- a/polly/test/ScopInfo/early_exit_for_complex_domains.ll
+++ b/polly/test/ScopInfo/early_exit_for_complex_domains.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-scops -disable-output < %s
 ;
 ; Check we do not crash.
 ;
diff --git a/polly/test/ScopInfo/error-blocks-1.ll b/polly/test/ScopInfo/error-blocks-1.ll
index e0b59e01d13d..03353edf297a 100644
--- a/polly/test/ScopInfo/error-blocks-1.ll
+++ b/polly/test/ScopInfo/error-blocks-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:         Context:
 ; CHECK-NEXT:    [N] -> {  : -2147483648 <= N <= 2147483647 }
diff --git a/polly/test/ScopInfo/error-blocks-2.ll b/polly/test/ScopInfo/error-blocks-2.ll
index 59096e3315f5..29095dacacfb 100644
--- a/polly/test/ScopInfo/error-blocks-2.ll
+++ b/polly/test/ScopInfo/error-blocks-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; CHECK:         Invariant Accesses: {
 ; CHECK-NEXT:            ReadAccess :=	[Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/escaping_empty_scop.ll b/polly/test/ScopInfo/escaping_empty_scop.ll
index e27130af952a..8837e19eefe4 100644
--- a/polly/test/ScopInfo/escaping_empty_scop.ll
+++ b/polly/test/ScopInfo/escaping_empty_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void g();
 ;    int f(int *A) {
diff --git a/polly/test/ScopInfo/exit-phi-1.ll b/polly/test/ScopInfo/exit-phi-1.ll
index 41b56dde043a..8e6c5fb9e211 100644
--- a/polly/test/ScopInfo/exit-phi-1.ll
+++ b/polly/test/ScopInfo/exit-phi-1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -passes=polly-codegen -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-codegen -S < %s | FileCheck %s --check-prefix=CODEGEN
 ;
 ; Check for correct code generation of exit PHIs, even if the same PHI value
 ; is used again inside the the SCoP.
diff --git a/polly/test/ScopInfo/exit-phi-2.ll b/polly/test/ScopInfo/exit-phi-2.ll
index c2b463f657ea..d218d5fa039b 100644
--- a/polly/test/ScopInfo/exit-phi-2.ll
+++ b/polly/test/ScopInfo/exit-phi-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that there is no MK_ExitPHI READ access.
 ;
diff --git a/polly/test/ScopInfo/exit_phi_accesses-2.ll b/polly/test/ScopInfo/exit_phi_accesses-2.ll
index cfc385dc6aba..e376f0df9d54 100644
--- a/polly/test/ScopInfo/exit_phi_accesses-2.ll
+++ b/polly/test/ScopInfo/exit_phi_accesses-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; CHECK-LABEL: Function: foo
 ;
diff --git a/polly/test/ScopInfo/exit_phi_accesses.ll b/polly/test/ScopInfo/exit_phi_accesses.ll
index c598e411739c..f4fbe31f6b24 100644
--- a/polly/test/ScopInfo/exit_phi_accesses.ll
+++ b/polly/test/ScopInfo/exit_phi_accesses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; Check that PHI nodes only create PHI access and nothing else (e.g. unnecessary
 ; SCALAR accesses). In this case, for a PHI in the exit node, hence there is no
diff --git a/polly/test/ScopInfo/expensive-boundary-context.ll b/polly/test/ScopInfo/expensive-boundary-context.ll
index dd660c543f2c..7001b96acd21 100644
--- a/polly/test/ScopInfo/expensive-boundary-context.ll
+++ b/polly/test/ScopInfo/expensive-boundary-context.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output \
-; RUN:                < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output \
+; RUN:                < %s | FileCheck %s
 
 ; CHECK-NOT:   Assumed Context:
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/polly/test/ScopInfo/extract_constant_factor_introduces_new_parameter.ll b/polly/test/ScopInfo/extract_constant_factor_introduces_new_parameter.ll
index 9475a870e4b5..89ca344fdf54 100644
--- a/polly/test/ScopInfo/extract_constant_factor_introduces_new_parameter.ll
+++ b/polly/test/ScopInfo/extract_constant_factor_introduces_new_parameter.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-scops -disable-output < %s
 
 ; CHECK: Valid Region for Scop: bb10 => bb16
 
diff --git a/polly/test/ScopInfo/full-function.ll b/polly/test/ScopInfo/full-function.ll
index bb2d12fb0e3f..670472576fe7 100644
--- a/polly/test/ScopInfo/full-function.ll
+++ b/polly/test/ScopInfo/full-function.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-detect-full-functions < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-scops -disable-output -polly-detect-full-functions < %s \
 ; RUN: | FileCheck %s -check-prefix=FULL
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
 ; RUN: | FileCheck %s -check-prefix=WITHOUT-FULL
 
 ; FULL:      Region: %bb---FunctionExit
diff --git a/polly/test/ScopInfo/granularity_same_name.ll b/polly/test/ScopInfo/granularity_same_name.ll
index 4e8dd1840890..1ebf5c6f71a2 100644
--- a/polly/test/ScopInfo/granularity_same_name.ll
+++ b/polly/test/ScopInfo/granularity_same_name.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb           -polly-use-llvm-names=0 '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=IDX
-; RUN: opt %loadPolly -polly-stmt-granularity=bb           -polly-use-llvm-names=1 '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=BB
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-use-llvm-names=0 '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=IDX
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-use-llvm-names=1 '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines -check-prefix=BB
+; RUN: opt %loadPolly -polly-stmt-granularity=bb           -polly-use-llvm-names=0 -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines -check-prefix=IDX
+; RUN: opt %loadPolly -polly-stmt-granularity=bb           -polly-use-llvm-names=1 -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines -check-prefix=BB
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-use-llvm-names=0 -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines -check-prefix=IDX
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-use-llvm-names=1 -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines -check-prefix=BB
 ;
 ; Check that the statement has the same name, regardless of how the
 ; basic block is split into multiple statements.
diff --git a/polly/test/ScopInfo/granularity_scalar-indep.ll b/polly/test/ScopInfo/granularity_scalar-indep.ll
index b28060d87180..fe509b468272 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Split a block into two independent statements that share no scalar.
 ; This case has the instructions of the two statements interleaved, such that
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi1.ll b/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi1.ll
index fb2bfa663ef2..56bc11aed28d 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi1.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Two PHIs, cross-referencing each other. The PHI READs must be carried-out
 ; before the PHI WRITEs to ensure that the value when entering the block is
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi2.ll b/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi2.ll
index aa066b5cd46c..f46cf4e6a0a2 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi2.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_cross-referencing-phi2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Two PHIs, cross-referencing each other. The PHI READs must be carried-out
 ; before the PHI WRITEs to ensure that the value when entering the block is
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_epilogue.ll b/polly/test/ScopInfo/granularity_scalar-indep_epilogue.ll
index da326191762a..e202e38f0844 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_epilogue.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_epilogue.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Split a block into two independent statements that share no scalar.
 ; This case has an independent statement just for PHI writes.
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_epilogue_last.ll b/polly/test/ScopInfo/granularity_scalar-indep_epilogue_last.ll
index 19484319ee35..40af34bfb067 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_epilogue_last.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_epilogue_last.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Check that the PHI Write of value that is defined in the same basic
 ; block is in the statement where it is defined.
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_noepilogue.ll b/polly/test/ScopInfo/granularity_scalar-indep_noepilogue.ll
index 484156abdbcd..9a0d207c0c2a 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_noepilogue.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_noepilogue.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; This case has no explicit epilogue for PHI writes because it would
 ; have a scalar dependency to the previous statement.
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_ordered-2.ll b/polly/test/ScopInfo/granularity_scalar-indep_ordered-2.ll
index ec69b8000bf5..d093806bc9cc 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_ordered-2.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_ordered-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; This case should be split into two statements because {X[0], Y[0]}
 ; and {A[0], B[0]} do not intersect.
diff --git a/polly/test/ScopInfo/granularity_scalar-indep_ordered.ll b/polly/test/ScopInfo/granularity_scalar-indep_ordered.ll
index a7f8e2697cac..b1d2936882aa 100644
--- a/polly/test/ScopInfo/granularity_scalar-indep_ordered.ll
+++ b/polly/test/ScopInfo/granularity_scalar-indep_ordered.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; This case cannot be split into two statements because the order of
 ; loads and store would be violated.
diff --git a/polly/test/ScopInfo/i1_params.ll b/polly/test/ScopInfo/i1_params.ll
index 28eb838b56b0..1cb1329b08f9 100644
--- a/polly/test/ScopInfo/i1_params.ll
+++ b/polly/test/ScopInfo/i1_params.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that both a signed as well as an unsigned extended i1 parameter
 ; is represented correctly.
diff --git a/polly/test/ScopInfo/infeasible-rtc.ll b/polly/test/ScopInfo/infeasible-rtc.ll
index 5540b2365ce3..ef96627e640e 100644
--- a/polly/test/ScopInfo/infeasible-rtc.ll
+++ b/polly/test/ScopInfo/infeasible-rtc.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s \
 ; RUN:  | FileCheck %s -check-prefix=DETECT
 
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
 ; RUN:  | FileCheck %s -check-prefix=SCOPS
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/polly/test/ScopInfo/infeasible_invalid_context.ll b/polly/test/ScopInfo/infeasible_invalid_context.ll
index 86ecb4053d5a..2c299f06c12e 100644
--- a/polly/test/ScopInfo/infeasible_invalid_context.ll
+++ b/polly/test/ScopInfo/infeasible_invalid_context.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s \
 ; RUN:  | FileCheck %s -check-prefix=DETECT
 
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
 ; RUN:  | FileCheck %s -check-prefix=SCOPS
 
 ; DETECT: Valid Region for Scop: if.end116 => for.inc216
diff --git a/polly/test/ScopInfo/int2ptr_ptr2int.ll b/polly/test/ScopInfo/int2ptr_ptr2int.ll
index f375e79e807d..9fadc5a8eb28 100644
--- a/polly/test/ScopInfo/int2ptr_ptr2int.ll
+++ b/polly/test/ScopInfo/int2ptr_ptr2int.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -S -passes=polly-codegen < %s 2>&1 | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen < %s | FileCheck %s --check-prefix=IR
 ;
 ;    void f(long *A, long *ptr, long val) {
 ;      for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/int2ptr_ptr2int_2.ll b/polly/test/ScopInfo/int2ptr_ptr2int_2.ll
index 1c8251361b51..97878f7091b1 100644
--- a/polly/test/ScopInfo/int2ptr_ptr2int_2.ll
+++ b/polly/test/ScopInfo/int2ptr_ptr2int_2.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' \
-; RUN: -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -S -passes=polly-codegen \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -polly-print-scops \
+; RUN: -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-codegen \
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s --check-prefix=IR
 ;
 ;    void f(long *A, long *B, long *ptr, long val) {
 ;      for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/integers.ll b/polly/test/ScopInfo/integers.ll
index 87bc31e214ff..b608bf84cffa 100644
--- a/polly/test/ScopInfo/integers.ll
+++ b/polly/test/ScopInfo/integers.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; Check that we correctly convert integers to isl values.
 
diff --git a/polly/test/ScopInfo/inter-error-bb-dependence.ll b/polly/test/ScopInfo/inter-error-bb-dependence.ll
index 00f482267a0b..4e23de7e6a99 100644
--- a/polly/test/ScopInfo/inter-error-bb-dependence.ll
+++ b/polly/test/ScopInfo/inter-error-bb-dependence.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 > /dev/null | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-print-scops -disable-output < %s 2>&1 > /dev/null | FileCheck %s
 ;
 ; Error statements (%bb33) do not require their uses to be verified.
 ; In this case it uses %tmp32 from %bb31 which is not available because
diff --git a/polly/test/ScopInfo/inter_bb_scalar_dep.ll b/polly/test/ScopInfo/inter_bb_scalar_dep.ll
index 0af814516038..456f7a773f04 100644
--- a/polly/test/ScopInfo/inter_bb_scalar_dep.ll
+++ b/polly/test/ScopInfo/inter_bb_scalar_dep.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' \
-; RUN: -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-scops \
+; RUN: -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/intra-non-affine-stmt-phi-node.ll b/polly/test/ScopInfo/intra-non-affine-stmt-phi-node.ll
index 0fc635abae45..859972b27402 100644
--- a/polly/test/ScopInfo/intra-non-affine-stmt-phi-node.ll
+++ b/polly/test/ScopInfo/intra-non-affine-stmt-phi-node.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
-; RUN:     < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output \
+; RUN:     < %s | FileCheck %s
 
 ; CHECK:      Statements {
 ; CHECK-NEXT:     Stmt_loop__TO__backedge
diff --git a/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll b/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll
index 209c475b5bbb..37f4e0513ed3 100644
--- a/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll
+++ b/polly/test/ScopInfo/intra_and_inter_bb_scalar_dep.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/intra_bb_scalar_dep.ll b/polly/test/ScopInfo/intra_bb_scalar_dep.ll
index 8ad5ac175802..0252273d3107 100644
--- a/polly/test/ScopInfo/intra_bb_scalar_dep.ll
+++ b/polly/test/ScopInfo/intra_bb_scalar_dep.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/intrinsics.ll b/polly/test/ScopInfo/intrinsics.ll
index 8b484cc21b52..853429341381 100644
--- a/polly/test/ScopInfo/intrinsics.ll
+++ b/polly/test/ScopInfo/intrinsics.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-print-instructions -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-print-instructions -disable-output < %s | FileCheck %s
 ;
 ; Verify that we remove the ignored intrinsics from the instruction list.
 ;
diff --git a/polly/test/ScopInfo/invalid_add_rec_after_invariant_load_remapping.ll b/polly/test/ScopInfo/invalid_add_rec_after_invariant_load_remapping.ll
index d9f64d5eba9c..8d0de03e9866 100644
--- a/polly/test/ScopInfo/invalid_add_rec_after_invariant_load_remapping.ll
+++ b/polly/test/ScopInfo/invalid_add_rec_after_invariant_load_remapping.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
 ;
 ; This crashed at some point as we place %1 and %4 in the same equivalence class
 ; for invariant loads and when we remap SCEVs to use %4 instead of %1 AddRec SCEVs
diff --git a/polly/test/ScopInfo/invalidate_iterator_during_MA_removal.ll b/polly/test/ScopInfo/invalidate_iterator_during_MA_removal.ll
index 0f48052a40e1..dcb0ad301ba3 100644
--- a/polly/test/ScopInfo/invalidate_iterator_during_MA_removal.ll
+++ b/polly/test/ScopInfo/invalidate_iterator_during_MA_removal.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
 ;
 ; Check that no invalidated iterator is accessed while elements from
 ; the list of MemoryAccesses are removed.
diff --git a/polly/test/ScopInfo/invariant-load-instlist.ll b/polly/test/ScopInfo/invariant-load-instlist.ll
index b65d843f0ab6..7f4cf050f064 100644
--- a/polly/test/ScopInfo/invariant-load-instlist.ll
+++ b/polly/test/ScopInfo/invariant-load-instlist.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
 
 ; The load is a required invariant load and at the same time used in a store.
 ; Polly used to add two MemoryAccesses for it which caused an assertion to fail.
diff --git a/polly/test/ScopInfo/invariant-loads-leave-read-only-statements.ll b/polly/test/ScopInfo/invariant-loads-leave-read-only-statements.ll
index 12a72bd47fdf..b97fe22e076e 100644
--- a/polly/test/ScopInfo/invariant-loads-leave-read-only-statements.ll
+++ b/polly/test/ScopInfo/invariant-loads-leave-read-only-statements.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
 
 ; CHECK:      Statements {
 ; CHECK-NEXT: 	Stmt_L_4
diff --git a/polly/test/ScopInfo/invariant_load.ll b/polly/test/ScopInfo/invariant_load.ll
index 47501022247e..fcea77e19b85 100644
--- a/polly/test/ScopInfo/invariant_load.ll
+++ b/polly/test/ScopInfo/invariant_load.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type.ll b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type.ll
index c402f90cbdd1..100a8db2a9d1 100644
--- a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type.ll
+++ b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=CODEGEN
 ;
 ;    struct {
 ;      int a;
diff --git a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_escaping.ll b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_escaping.ll
index 435c9a575d73..e31deb6fd472 100644
--- a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_escaping.ll
+++ b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_escaping.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=CODEGEN
 ;
 ;    struct {
 ;      int a;
diff --git a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer.ll b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer.ll
index a2f43f2348b8..bbf6d69a5fbb 100644
--- a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer.ll
+++ b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=CODEGEN
 ;
 ;    int U;
 ;    void f(int *A) {
diff --git a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer_escaping.ll b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer_escaping.ll
index 412e62f26511..011c2fe3d549 100644
--- a/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer_escaping.ll
+++ b/polly/test/ScopInfo/invariant_load_access_classes_different_base_type_same_pointer_escaping.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=CODEGEN
 ;
 ;    int U;
 ;    int f(int *A) {
diff --git a/polly/test/ScopInfo/invariant_load_addrec_sum.ll b/polly/test/ScopInfo/invariant_load_addrec_sum.ll
index 8026d351fdf7..09b158d342ed 100644
--- a/polly/test/ScopInfo/invariant_load_addrec_sum.ll
+++ b/polly/test/ScopInfo/invariant_load_addrec_sum.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Region: %entry.split---%if.end
 ; CHECK:     Invariant Accesses: {
diff --git a/polly/test/ScopInfo/invariant_load_base_pointer.ll b/polly/test/ScopInfo/invariant_load_base_pointer.ll
index 7bf9c5a44788..ddf11d892adb 100644
--- a/polly/test/ScopInfo/invariant_load_base_pointer.ll
+++ b/polly/test/ScopInfo/invariant_load_base_pointer.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_base_pointer_conditional.ll b/polly/test/ScopInfo/invariant_load_base_pointer_conditional.ll
index 42dfc7b3885a..07f2c3768b0a 100644
--- a/polly/test/ScopInfo/invariant_load_base_pointer_conditional.ll
+++ b/polly/test/ScopInfo/invariant_load_base_pointer_conditional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -polly-ignore-aliasing -polly-process-unprofitable -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_base_pointer_in_conditional.ll b/polly/test/ScopInfo/invariant_load_base_pointer_in_conditional.ll
index 68c60edce09c..d66d718d492a 100644
--- a/polly/test/ScopInfo/invariant_load_base_pointer_in_conditional.ll
+++ b/polly/test/ScopInfo/invariant_load_base_pointer_in_conditional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-ignore-aliasing -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -polly-ignore-aliasing -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_branch_condition.ll b/polly/test/ScopInfo/invariant_load_branch_condition.ll
index e8c18adee1f3..4f49d2969d86 100644
--- a/polly/test/ScopInfo/invariant_load_branch_condition.ll
+++ b/polly/test/ScopInfo/invariant_load_branch_condition.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
-; RUN:     -polly-invariant-load-hoisting < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output \
+; RUN:     -polly-invariant-load-hoisting < %s | FileCheck %s
 
 ; CHECK:      Invariant Accesses: {
 ; CHECK-NEXT:         ReadAccess :=	[Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs.ll
index d19018db7967..c6a7faf2e355 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_2.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_2.ll
index 556b50284f34..921dd4fbde5c 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_2.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_3.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_3.ll
index 58c553023191..c15d11ca865d 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_3.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4.ll
index 881003b15839..0495a330792c 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4b.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4b.ll
index 6f4d40ec98f7..9144fcf186c3 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4b.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4b.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4c.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4c.ll
index d1859cb54241..aefacff6b46f 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4c.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_4c.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_5.ll b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_5.ll
index 74d6f1021124..ecc0c0a23014 100644
--- a/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_5.ll
+++ b/polly/test/ScopInfo/invariant_load_canonicalize_array_baseptrs_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s \
 ; RUN:  -polly-invariant-load-hoisting \
 ; RUN:  | FileCheck %s
 
diff --git a/polly/test/ScopInfo/invariant_load_complex_condition.ll b/polly/test/ScopInfo/invariant_load_complex_condition.ll
index 34f5c45ac2b4..e721c222db5f 100644
--- a/polly/test/ScopInfo/invariant_load_complex_condition.ll
+++ b/polly/test/ScopInfo/invariant_load_complex_condition.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -S '-passes=print<polly-function-scops>' -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -S -polly-print-scops -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/invariant_load_condition.ll b/polly/test/ScopInfo/invariant_load_condition.ll
index 7d14a398d874..84546984709e 100644
--- a/polly/test/ScopInfo/invariant_load_condition.ll
+++ b/polly/test/ScopInfo/invariant_load_condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_dereferenceable.ll b/polly/test/ScopInfo/invariant_load_dereferenceable.ll
index c590e4043f64..adba32d8d463 100644
--- a/polly/test/ScopInfo/invariant_load_dereferenceable.ll
+++ b/polly/test/ScopInfo/invariant_load_dereferenceable.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -polly-print-detect -polly-print-scops \
 ; RUN: -polly-invariant-load-hoisting=true \
-; RUN: -disable-output < %s 2>&1 | FileCheck %s
+; RUN: -disable-output < %s | FileCheck %s
 
 ; CHECK-NOT: Function: foo_undereferanceable
 
diff --git a/polly/test/ScopInfo/invariant_load_distinct_parameter_valuations.ll b/polly/test/ScopInfo/invariant_load_distinct_parameter_valuations.ll
index b9b55cc4ecb1..60b4a1daa824 100644
--- a/polly/test/ScopInfo/invariant_load_distinct_parameter_valuations.ll
+++ b/polly/test/ScopInfo/invariant_load_distinct_parameter_valuations.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; Check that we do not consolidate the invariant loads to smp[order - 1] and
 ; smp[order - 2] in the blocks %0 and %16. While they have the same pointer
diff --git a/polly/test/ScopInfo/invariant_load_in_non_affine.ll b/polly/test/ScopInfo/invariant_load_in_non_affine.ll
index 08c8bd28caa7..d00bc2d642e0 100644
--- a/polly/test/ScopInfo/invariant_load_in_non_affine.ll
+++ b/polly/test/ScopInfo/invariant_load_in_non_affine.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output \
-; RUN:   -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -disable-output \
+; RUN:   -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; CHECK-NOT: Valid Region for Scop
 ;
diff --git a/polly/test/ScopInfo/invariant_load_loop_ub.ll b/polly/test/ScopInfo/invariant_load_loop_ub.ll
index 009f036bcb62..856b6e4dd508 100644
--- a/polly/test/ScopInfo/invariant_load_loop_ub.ll
+++ b/polly/test/ScopInfo/invariant_load_loop_ub.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-process-unprofitable -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-process-unprofitable -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -polly-process-unprofitable -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -polly-invariant-load-hoisting=true -polly-process-unprofitable -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_ptr_ptr_noalias.ll b/polly/test/ScopInfo/invariant_load_ptr_ptr_noalias.ll
index ba381dea72e9..69463d420aca 100644
--- a/polly/test/ScopInfo/invariant_load_ptr_ptr_noalias.ll
+++ b/polly/test/ScopInfo/invariant_load_ptr_ptr_noalias.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=tbaa '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -polly-ignore-aliasing \
-; RUN:                -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -tbaa -polly-print-scops -polly-invariant-load-hoisting=true -polly-ignore-aliasing \
+; RUN:                -disable-output < %s | FileCheck %s
 ;
 ; Note: The order of the invariant accesses is important because A is the
 ;       base pointer of tmp3 and we will generate code in the same order as
diff --git a/polly/test/ScopInfo/invariant_load_scalar_dep.ll b/polly/test/ScopInfo/invariant_load_scalar_dep.ll
index ea0227d4a6e9..79a10426862a 100644
--- a/polly/test/ScopInfo/invariant_load_scalar_dep.ll
+++ b/polly/test/ScopInfo/invariant_load_scalar_dep.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Invariant Accesses:
 ; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_load_stmt_domain.ll b/polly/test/ScopInfo/invariant_load_stmt_domain.ll
index 31da46f40f8c..6cd71c85ea2f 100644
--- a/polly/test/ScopInfo/invariant_load_stmt_domain.ll
+++ b/polly/test/ScopInfo/invariant_load_stmt_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 
 ; This test case verifies that the statement domain of the invariant access
 ; is the universe. In earlier versions of Polly, we accidentally computed an
diff --git a/polly/test/ScopInfo/invariant_load_zext_parameter-2.ll b/polly/test/ScopInfo/invariant_load_zext_parameter-2.ll
index 2bf6d1d4d8b1..e77515280241 100644
--- a/polly/test/ScopInfo/invariant_load_zext_parameter-2.ll
+++ b/polly/test/ScopInfo/invariant_load_zext_parameter-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -scalar-evolution-max-value-compare-depth=3 '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -scalar-evolution-max-value-compare-depth=3 -passes=polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
+; RUN: opt %loadPolly -scalar-evolution-max-value-compare-depth=3 -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -scalar-evolution-max-value-compare-depth=3 -polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
 ;
 ; Stress test for the code generation of invariant accesses.
 ;
diff --git a/polly/test/ScopInfo/invariant_load_zext_parameter.ll b/polly/test/ScopInfo/invariant_load_zext_parameter.ll
index 41559ff14efa..1bde70282d44 100644
--- a/polly/test/ScopInfo/invariant_load_zext_parameter.ll
+++ b/polly/test/ScopInfo/invariant_load_zext_parameter.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=CODEGEN
 ;
 ;    void f(int *I0, int *I1, int *V) {
 ;      for (int i = 0; i < 1000; i++) {
diff --git a/polly/test/ScopInfo/invariant_load_zextended_in_own_execution_context.ll b/polly/test/ScopInfo/invariant_load_zextended_in_own_execution_context.ll
index 2ae99acd47ec..775369e55c92 100644
--- a/polly/test/ScopInfo/invariant_load_zextended_in_own_execution_context.ll
+++ b/polly/test/ScopInfo/invariant_load_zextended_in_own_execution_context.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -disable-output < %s
 ;
 ; CHECK: Execution Context: [p_0_loaded_from_currpc] -> {  :  }
 ;
diff --git a/polly/test/ScopInfo/invariant_loads_complicated_dependences.ll b/polly/test/ScopInfo/invariant_loads_complicated_dependences.ll
index 9cee220d1f3c..1d54ccc69023 100644
--- a/polly/test/ScopInfo/invariant_loads_complicated_dependences.ll
+++ b/polly/test/ScopInfo/invariant_loads_complicated_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      Invariant Accesses: {
 ; CHECK-NEXT:         ReadAccess :=    [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_loads_cyclic_dependences.ll b/polly/test/ScopInfo/invariant_loads_cyclic_dependences.ll
index 0434bbd34e4c..e97de0c936bc 100644
--- a/polly/test/ScopInfo/invariant_loads_cyclic_dependences.ll
+++ b/polly/test/ScopInfo/invariant_loads_cyclic_dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; Negative test. If we assume UB[*V] to be invariant we get a cyclic
 ; dependence in the invariant loads that needs to be resolved by
diff --git a/polly/test/ScopInfo/invariant_loop_bounds.ll b/polly/test/ScopInfo/invariant_loop_bounds.ll
index 279199b805f1..4e1fd88fac30 100644
--- a/polly/test/ScopInfo/invariant_loop_bounds.ll
+++ b/polly/test/ScopInfo/invariant_loop_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      Invariant Accesses: {
 ; CHECK-NEXT:         ReadAccess :=    [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll
index d8e293f7e6f3..3d5737bbe168 100644
--- a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll
+++ b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; Verify that we only have one parameter and one invariant load for all
 ; three loads that occure in the region but actually access the same
diff --git a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll
index e68f3be7fb30..e2de503eb83f 100644
--- a/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll
+++ b/polly/test/ScopInfo/invariant_same_loop_bound_multiple_times-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; Verify that we only have one parameter and one invariant load for all
 ; three loads that occure in the region but actually access the same
diff --git a/polly/test/ScopInfo/isl_aff_out_of_bounds.ll b/polly/test/ScopInfo/isl_aff_out_of_bounds.ll
index 4f7a604272bb..ca1b235be358 100644
--- a/polly/test/ScopInfo/isl_aff_out_of_bounds.ll
+++ b/polly/test/ScopInfo/isl_aff_out_of_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' < %s 2>&1
+; RUN: opt %loadPolly -basic-aa -polly-detect < %s
 
 ; Used to fail with:
 ; ../../isl/isl_aff.c:591: position out of bounds
diff --git a/polly/test/ScopInfo/isl_trip_count_01.ll b/polly/test/ScopInfo/isl_trip_count_01.ll
index 6ad4929888e3..fc6b79c5a68a 100644
--- a/polly/test/ScopInfo/isl_trip_count_01.ll
+++ b/polly/test/ScopInfo/isl_trip_count_01.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: [M, N] -> { Stmt_while_body[i0] : i0 > 0 and 4i0 <= -M + N; Stmt_while_body[0] };
 ;
diff --git a/polly/test/ScopInfo/isl_trip_count_02.ll b/polly/test/ScopInfo/isl_trip_count_02.ll
index b356fa9fdf22..9376cb415cec 100644
--- a/polly/test/ScopInfo/isl_trip_count_02.ll
+++ b/polly/test/ScopInfo/isl_trip_count_02.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; TODO: We do not allow unbounded loops at the moment.
 ;
diff --git a/polly/test/ScopInfo/isl_trip_count_03.ll b/polly/test/ScopInfo/isl_trip_count_03.ll
index 886143114df6..f5b0048a0e0e 100644
--- a/polly/test/ScopInfo/isl_trip_count_03.ll
+++ b/polly/test/ScopInfo/isl_trip_count_03.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Test comes from a bug (15771) or better a feature request. It was not allowed
 ; in Polly in the old domain generation as ScalarEvolution cannot figure out the
diff --git a/polly/test/ScopInfo/isl_trip_count_multiple_exiting_blocks.ll b/polly/test/ScopInfo/isl_trip_count_multiple_exiting_blocks.ll
index 8ddb26a6ac88..91bc19e2de44 100644
--- a/polly/test/ScopInfo/isl_trip_count_multiple_exiting_blocks.ll
+++ b/polly/test/ScopInfo/isl_trip_count_multiple_exiting_blocks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/licm_load.ll b/polly/test/ScopInfo/licm_load.ll
index c4695ecddaf7..ade640976d00 100644
--- a/polly/test/ScopInfo/licm_load.ll
+++ b/polly/test/ScopInfo/licm_load.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -passes='loop(loop-rotate,indvars),polly-prepare,print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 \
+; RUN: opt %loadNPMPolly -passes='loop(loop-rotate,indvars),polly-prepare,print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 \
 ; RUN: | FileCheck %s
-; RUN: opt %loadPolly -passes='loop-mssa(loop-rotate,indvars,licm),polly-prepare,print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 \
+; RUN: opt %loadNPMPolly -passes='loop-mssa(loop-rotate,indvars,licm),polly-prepare,print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 \
 ; RUN: | FileCheck %s
 ;
 ;    void foo(int n, float A[static const restrict n],
diff --git a/polly/test/ScopInfo/licm_potential_store.ll b/polly/test/ScopInfo/licm_potential_store.ll
index fd19df793306..8a36ee84313a 100644
--- a/polly/test/ScopInfo/licm_potential_store.ll
+++ b/polly/test/ScopInfo/licm_potential_store.ll
@@ -1,8 +1,8 @@
-; RUN: opt %loadPolly -passes='sroa,instcombine,simplifycfg,reassociate,loop(loop-rotate),instcombine,indvars,polly-prepare,print<polly-function-scops>' \
+; RUN: opt %loadNPMPolly -passes='sroa,instcombine,simplifycfg,reassociate,loop(loop-rotate),instcombine,indvars,polly-prepare,print<polly-function-scops>' \
 ; RUN:    -tailcallopt -disable-output < %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=NOLICM
 
-; RUN: opt %loadPolly -passes='sroa,instcombine,simplifycfg,reassociate,loop(loop-rotate),instcombine,indvars,loop-mssa(licm),polly-prepare,print<polly-function-scops>' \
+; RUN: opt %loadNPMPolly -passes='sroa,instcombine,simplifycfg,reassociate,loop(loop-rotate),instcombine,indvars,loop-mssa(licm),polly-prepare,print<polly-function-scops>' \
 ; RUN:    -tailcallopt -disable-output < %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=LICM
 
diff --git a/polly/test/ScopInfo/licm_reduction_nested.ll b/polly/test/ScopInfo/licm_reduction_nested.ll
index 98d6dfcfa074..a3ba478cd9ff 100644
--- a/polly/test/ScopInfo/licm_reduction_nested.ll
+++ b/polly/test/ScopInfo/licm_reduction_nested.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -loop-rotate -indvars       -passes=polly-prepare '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -loop-rotate -indvars -licm -passes=polly-prepare '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars       -polly-prepare -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -loop-rotate -indvars -licm -polly-prepare -polly-scops -disable-output < %s | FileCheck %s
 ;
 ; XFAIL: *
 ;
diff --git a/polly/test/ScopInfo/long-compile-time-alias-analysis.ll b/polly/test/ScopInfo/long-compile-time-alias-analysis.ll
index 5fa9a74d0fe3..1cbecf086968 100644
--- a/polly/test/ScopInfo/long-compile-time-alias-analysis.ll
+++ b/polly/test/ScopInfo/long-compile-time-alias-analysis.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s
 
 ; Verify that the compilation of this test case does not take infinite time.
 ; At some point Polly tried to model this test case and got stuck in
diff --git a/polly/test/ScopInfo/long-sequence-of-error-blocks-2.ll b/polly/test/ScopInfo/long-sequence-of-error-blocks-2.ll
index 283a0bea7c49..c88ea1327389 100644
--- a/polly/test/ScopInfo/long-sequence-of-error-blocks-2.ll
+++ b/polly/test/ScopInfo/long-sequence-of-error-blocks-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/polly/test/ScopInfo/long-sequence-of-error-blocks.ll b/polly/test/ScopInfo/long-sequence-of-error-blocks.ll
index 812de273a7df..5b6ea9cc212d 100644
--- a/polly/test/ScopInfo/long-sequence-of-error-blocks.ll
+++ b/polly/test/ScopInfo/long-sequence-of-error-blocks.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/polly/test/ScopInfo/loop-multiexit-succ-cond.ll b/polly/test/ScopInfo/loop-multiexit-succ-cond.ll
index f8f47a332a13..350db05c6dc0 100644
--- a/polly/test/ScopInfo/loop-multiexit-succ-cond.ll
+++ b/polly/test/ScopInfo/loop-multiexit-succ-cond.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s 2>&1 | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s --check-prefix=IR
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/loop_affine_bound_0.ll b/polly/test/ScopInfo/loop_affine_bound_0.ll
index 77b0ebe8e494..33f49df7780f 100644
--- a/polly/test/ScopInfo/loop_affine_bound_0.ll
+++ b/polly/test/ScopInfo/loop_affine_bound_0.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
 
 ; void f(long a[][128], long N, long M) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/loop_affine_bound_1.ll b/polly/test/ScopInfo/loop_affine_bound_1.ll
index 7c1eaa5dff8c..38e47b74465b 100644
--- a/polly/test/ScopInfo/loop_affine_bound_1.ll
+++ b/polly/test/ScopInfo/loop_affine_bound_1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output< %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output< %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
 
 ;void f(long a[][128], long N, long M) {
 ;  long i, j;
diff --git a/polly/test/ScopInfo/loop_affine_bound_2.ll b/polly/test/ScopInfo/loop_affine_bound_2.ll
index 12d81c78b794..e34662f4e6ab 100644
--- a/polly/test/ScopInfo/loop_affine_bound_2.ll
+++ b/polly/test/ScopInfo/loop_affine_bound_2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
 
 ; void f(long a[][128], long N, long M) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/loop_carry.ll b/polly/test/ScopInfo/loop_carry.ll
index 856efb730056..f7c1dca0919c 100644
--- a/polly/test/ScopInfo/loop_carry.ll
+++ b/polly/test/ScopInfo/loop_carry.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
diff --git a/polly/test/ScopInfo/many-scalar-dependences.ll b/polly/test/ScopInfo/many-scalar-dependences.ll
index 56e02d56254d..aaa02f581a1c 100644
--- a/polly/test/ScopInfo/many-scalar-dependences.ll
+++ b/polly/test/ScopInfo/many-scalar-dependences.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(float a[100][100]) {
 ;      float x;
diff --git a/polly/test/ScopInfo/max-loop-depth.ll b/polly/test/ScopInfo/max-loop-depth.ll
index 4da0f35121d0..3c7db4458604 100644
--- a/polly/test/ScopInfo/max-loop-depth.ll
+++ b/polly/test/ScopInfo/max-loop-depth.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void bar();
 ;    void foo(int *A, int *B, long int N, long int M) {
diff --git a/polly/test/ScopInfo/memcpy-raw-source.ll b/polly/test/ScopInfo/memcpy-raw-source.ll
index c3ecce23cf5c..137ab8229220 100644
--- a/polly/test/ScopInfo/memcpy-raw-source.ll
+++ b/polly/test/ScopInfo/memcpy-raw-source.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa,scoped-noalias-aa,tbaa '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -basic-aa -scoped-noalias-aa -tbaa -polly-print-scops -disable-output < %s
 ;
 ; Ensure that ScopInfo's alias analysis llvm.memcpy for,
 ; like the AliasSetTracker, preserves bitcasts.
diff --git a/polly/test/ScopInfo/memcpy.ll b/polly/test/ScopInfo/memcpy.ll
index 2e34b0d87a5f..705dea769e42 100644
--- a/polly/test/ScopInfo/memcpy.ll
+++ b/polly/test/ScopInfo/memcpy.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-differing-element-types '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -S -aa-pipeline=basic-aa -polly-allow-differing-element-types -passes=polly-codegen < %s 2>&1 | FileCheck --check-prefix=IR %s
+; RUN: opt %loadPolly -basic-aa -polly-allow-differing-element-types -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -S -basic-aa -polly-allow-differing-element-types -polly-codegen < %s | FileCheck --check-prefix=IR %s
 ;
 ; CHECK:         Arrays {
 ; CHECK-NEXT:        i8 MemRef_A[*]; // Element size 1
diff --git a/polly/test/ScopInfo/memmove.ll b/polly/test/ScopInfo/memmove.ll
index 28a4ebeee7e2..15123422f419 100644
--- a/polly/test/ScopInfo/memmove.ll
+++ b/polly/test/ScopInfo/memmove.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-differing-element-types '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -S -aa-pipeline=basic-aa -polly-allow-differing-element-types -passes=polly-codegen < %s 2>&1 | FileCheck --check-prefix=IR %s
+; RUN: opt %loadPolly -basic-aa -polly-allow-differing-element-types -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -S -basic-aa -polly-allow-differing-element-types -polly-codegen < %s | FileCheck --check-prefix=IR %s
 ;
 ; CHECK:         Arrays {
 ; CHECK-NEXT:        i8 MemRef_A[*]; // Element size 1
diff --git a/polly/test/ScopInfo/memset.ll b/polly/test/ScopInfo/memset.ll
index 163b58dc54dc..ef86b4c275e5 100644
--- a/polly/test/ScopInfo/memset.ll
+++ b/polly/test/ScopInfo/memset.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-differing-element-types '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -S -polly-allow-differing-element-types -passes=polly-codegen < %s 2>&1 | FileCheck --check-prefix=IR %s
+; RUN: opt %loadPolly -polly-allow-differing-element-types -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -S -polly-allow-differing-element-types -polly-codegen < %s | FileCheck --check-prefix=IR %s
 ;
 ; CHECK:         Arrays {
 ; CHECK-NEXT:        i8 MemRef_A[*]; // Element size 1
diff --git a/polly/test/ScopInfo/memset_null.ll b/polly/test/ScopInfo/memset_null.ll
index 3d38fa1e6851..1608ff6ebef4 100644
--- a/polly/test/ScopInfo/memset_null.ll
+++ b/polly/test/ScopInfo/memset_null.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-modref-calls '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-modref-calls -S -passes=polly-codegen < %s
+; RUN: opt %loadPolly -polly-allow-modref-calls -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-modref-calls -S -polly-codegen < %s
 ;
 ; Verify we can handle a memset to "null" and that we do not model it.
 ; TODO: FIXME: We could use the undefined memset to optimize the code further,
diff --git a/polly/test/ScopInfo/mismatching-array-dimensions.ll b/polly/test/ScopInfo/mismatching-array-dimensions.ll
index a2deef16eafd..a1c6d4e82127 100644
--- a/polly/test/ScopInfo/mismatching-array-dimensions.ll
+++ b/polly/test/ScopInfo/mismatching-array-dimensions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; CHECK-NOT: AssumedContext
 
diff --git a/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll b/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll
index 202295abf57c..72889324e37e 100644
--- a/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll
+++ b/polly/test/ScopInfo/mod_ref_access_pointee_arguments.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
-; RUN:     -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb -passes=polly-codegen -polly-allow-modref-calls \
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-scops -polly-allow-modref-calls \
+; RUN:     -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-codegen -polly-allow-modref-calls \
 ; RUN:     -disable-output < %s
 ;
 ; Verify that we model the may-write access of the prefetch intrinsic
diff --git a/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll b/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll
index f1124f0a977b..2f6c6792fd9d 100644
--- a/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll
+++ b/polly/test/ScopInfo/mod_ref_read_pointee_arguments.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
-; RUN:     -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -disable-output \
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-scops -polly-allow-modref-calls \
+; RUN:     -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -disable-output \
 ; RUN:     -polly-allow-modref-calls < %s
 ;
 ; Verify that we model the read access of the gcread intrinsic
diff --git a/polly/test/ScopInfo/mod_ref_read_pointer.ll b/polly/test/ScopInfo/mod_ref_read_pointer.ll
index 24f276621532..657e37c68a7b 100644
--- a/polly/test/ScopInfo/mod_ref_read_pointer.ll
+++ b/polly/test/ScopInfo/mod_ref_read_pointer.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-modref-calls '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-allow-modref-calls -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -basic-aa -polly-allow-modref-calls -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-allow-modref-calls -polly-codegen -disable-output < %s
 ;
 ; Check that we assume the call to func has a read on the whole A array.
 ;
diff --git a/polly/test/ScopInfo/mod_ref_read_pointers.ll b/polly/test/ScopInfo/mod_ref_read_pointers.ll
index 260f759c1449..7ed3423a2aeb 100644
--- a/polly/test/ScopInfo/mod_ref_read_pointers.ll
+++ b/polly/test/ScopInfo/mod_ref_read_pointers.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
-; RUN:     -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -passes=polly-codegen -disable-output \
+; RUN: opt %loadPolly -basic-aa -polly-print-scops -polly-allow-modref-calls \
+; RUN:     -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-codegen -disable-output \
 ; RUN:     -polly-allow-modref-calls < %s
 ;
 ; Check that the call to func will "read" not only the A array but also the
diff --git a/polly/test/ScopInfo/modulo_zext_1.ll b/polly/test/ScopInfo/modulo_zext_1.ll
index 60083098eb08..d611ec4807b5 100644
--- a/polly/test/ScopInfo/modulo_zext_1.ll
+++ b/polly/test/ScopInfo/modulo_zext_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:         Assumed Context:
 ; CHECK-NEXT:    [N] -> {  :  }
diff --git a/polly/test/ScopInfo/modulo_zext_2.ll b/polly/test/ScopInfo/modulo_zext_2.ll
index 7ed70d085546..8d2321849174 100644
--- a/polly/test/ScopInfo/modulo_zext_2.ll
+++ b/polly/test/ScopInfo/modulo_zext_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:         Assumed Context:
 ; CHECK-NEXT:    [N] -> {  :  }
diff --git a/polly/test/ScopInfo/modulo_zext_3.ll b/polly/test/ScopInfo/modulo_zext_3.ll
index 67b26d813918..acb26dc1c77f 100644
--- a/polly/test/ScopInfo/modulo_zext_3.ll
+++ b/polly/test/ScopInfo/modulo_zext_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:         Assumed Context:
 ; CHECK-NEXT:    [N] -> {  :  }
diff --git a/polly/test/ScopInfo/multi-scop.ll b/polly/test/ScopInfo/multi-scop.ll
index 747f76bbf275..e26c8c7bae10 100644
--- a/polly/test/ScopInfo/multi-scop.ll
+++ b/polly/test/ScopInfo/multi-scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>'  -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-detect -polly-scops  -disable-output < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; This test case contains two scops.
diff --git a/polly/test/ScopInfo/multidim_2d-diagonal-matrix.ll b/polly/test/ScopInfo/multidim_2d-diagonal-matrix.ll
index 8d023a671d86..278c06a2fdba 100644
--- a/polly/test/ScopInfo/multidim_2d-diagonal-matrix.ll
+++ b/polly/test/ScopInfo/multidim_2d-diagonal-matrix.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/ScopInfo/multidim_2d_outer_parametric_offset.ll b/polly/test/ScopInfo/multidim_2d_outer_parametric_offset.ll
index 6cd612d155cb..06a76466c25e 100644
--- a/polly/test/ScopInfo/multidim_2d_outer_parametric_offset.ll
+++ b/polly/test/ScopInfo/multidim_2d_outer_parametric_offset.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/ScopInfo/multidim_2d_parametric_array_static_loop_bounds.ll b/polly/test/ScopInfo/multidim_2d_parametric_array_static_loop_bounds.ll
index aed318b51b15..bfbe5682d44a 100644
--- a/polly/test/ScopInfo/multidim_2d_parametric_array_static_loop_bounds.ll
+++ b/polly/test/ScopInfo/multidim_2d_parametric_array_static_loop_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/ScopInfo/multidim_2d_with_modref_call.ll b/polly/test/ScopInfo/multidim_2d_with_modref_call.ll
index 872544d872cd..ba934adb675a 100644
--- a/polly/test/ScopInfo/multidim_2d_with_modref_call.ll
+++ b/polly/test/ScopInfo/multidim_2d_with_modref_call.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-modref-calls \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-nonaffine \
+; RUN:     -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -polly-allow-modref-calls -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NONAFFINE
+; RUN:     -polly-allow-modref-calls -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
 
 ;  TODO: We should delinearize the accesses despite the use in a call to a
 ;        readonly function. For now we verify we do not delinearize them though.
diff --git a/polly/test/ScopInfo/multidim_2d_with_modref_call_2.ll b/polly/test/ScopInfo/multidim_2d_with_modref_call_2.ll
index add130e2087f..3da123fd1f60 100644
--- a/polly/test/ScopInfo/multidim_2d_with_modref_call_2.ll
+++ b/polly/test/ScopInfo/multidim_2d_with_modref_call_2.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-modref-calls \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-nonaffine \
+; RUN:     -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -polly-allow-modref-calls -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NONAFFINE
+; RUN:     -polly-allow-modref-calls -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
 
 ;  TODO: We should delinearize the accesses despite the use in a call to a
 ;        readonly function. For now we verify we do not delinearize them though.
diff --git a/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll b/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll
index 5e0f58cb276b..988475575fec 100644
--- a/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll
+++ b/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
diff --git a/polly/test/ScopInfo/multidim_fixedsize_different_dimensionality.ll b/polly/test/ScopInfo/multidim_fixedsize_different_dimensionality.ll
index 765077142df6..ddc35a46a633 100644
--- a/polly/test/ScopInfo/multidim_fixedsize_different_dimensionality.ll
+++ b/polly/test/ScopInfo/multidim_fixedsize_different_dimensionality.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    #define N 400
 ;
diff --git a/polly/test/ScopInfo/multidim_fixedsize_multi_offset.ll b/polly/test/ScopInfo/multidim_fixedsize_multi_offset.ll
index 8e26a2ad6a21..9c749f0c48c8 100644
--- a/polly/test/ScopInfo/multidim_fixedsize_multi_offset.ll
+++ b/polly/test/ScopInfo/multidim_fixedsize_multi_offset.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      Context:
 ; CHECK-NEXT: {  :  }
diff --git a/polly/test/ScopInfo/multidim_fold_constant_dim.ll b/polly/test/ScopInfo/multidim_fold_constant_dim.ll
index b142a2ab442f..e95d400a860c 100644
--- a/polly/test/ScopInfo/multidim_fold_constant_dim.ll
+++ b/polly/test/ScopInfo/multidim_fold_constant_dim.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    struct com {
 ;      double Real;
diff --git a/polly/test/ScopInfo/multidim_fold_constant_dim_zero.ll b/polly/test/ScopInfo/multidim_fold_constant_dim_zero.ll
index dac9dac1e7ad..57275e4024ab 100644
--- a/polly/test/ScopInfo/multidim_fold_constant_dim_zero.ll
+++ b/polly/test/ScopInfo/multidim_fold_constant_dim_zero.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -debug -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -debug -disable-output < %s 2>&1 | FileCheck %s
 
 ; REQUIRES: asserts
 
diff --git a/polly/test/ScopInfo/multidim_fortran_2d.ll b/polly/test/ScopInfo/multidim_fortran_2d.ll
index ee13da875d4e..29279a4e886b 100644
--- a/polly/test/ScopInfo/multidim_fortran_2d.ll
+++ b/polly/test/ScopInfo/multidim_fortran_2d.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' \
-; RUN: -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops \
+; RUN: -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 
 ;   subroutine init_array(ni, nj, pi, pj, a)
 ;   implicit none
diff --git a/polly/test/ScopInfo/multidim_fortran_2d_params.ll b/polly/test/ScopInfo/multidim_fortran_2d_params.ll
index f4978ecb35f7..93145b399ca5 100644
--- a/polly/test/ScopInfo/multidim_fortran_2d_params.ll
+++ b/polly/test/ScopInfo/multidim_fortran_2d_params.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
+; RUN: opt %loadPolly -polly-print-scops -disable-output \
 ; RUN: -polly-precise-fold-accesses \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 
 ;   subroutine init_array(ni, nj, pi, pj, a)
 ;   implicit none
diff --git a/polly/test/ScopInfo/multidim_fortran_2d_with_modref_call.ll b/polly/test/ScopInfo/multidim_fortran_2d_with_modref_call.ll
index f3a5e0ba9305..dff6a8be85cf 100644
--- a/polly/test/ScopInfo/multidim_fortran_2d_with_modref_call.ll
+++ b/polly/test/ScopInfo/multidim_fortran_2d_with_modref_call.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-modref-calls \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-modref-calls \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-allow-nonaffine \
+; RUN:     -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-allow-nonaffine \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -polly-allow-modref-calls -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NONAFFINE
+; RUN:     -polly-allow-modref-calls -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
 
 ;  TODO: We should delinearize the accesses despite the use in a call to a
 ;        readonly function. For now we verify we do not delinearize them though.
diff --git a/polly/test/ScopInfo/multidim_fortran_srem.ll b/polly/test/ScopInfo/multidim_fortran_srem.ll
index fc65ff2954cf..8c24c5b8ee71 100644
--- a/polly/test/ScopInfo/multidim_fortran_srem.ll
+++ b/polly/test/ScopInfo/multidim_fortran_srem.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-S128-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f16:16:16-f32:32:32-f64:64:64-f128:128:128-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 ; CHECK:      Statements {
diff --git a/polly/test/ScopInfo/multidim_gep_pointercast.ll b/polly/test/ScopInfo/multidim_gep_pointercast.ll
index 6b69dd7bb571..20d59fa91eaf 100644
--- a/polly/test/ScopInfo/multidim_gep_pointercast.ll
+++ b/polly/test/ScopInfo/multidim_gep_pointercast.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; The load access to A has a pointer-bitcast to another elements size before the
 ; GetElementPtr. Verify that we do not the GEP delinearization because it
diff --git a/polly/test/ScopInfo/multidim_gep_pointercast2.ll b/polly/test/ScopInfo/multidim_gep_pointercast2.ll
index 0c8139f7f8e1..deed9c7c3f57 100644
--- a/polly/test/ScopInfo/multidim_gep_pointercast2.ll
+++ b/polly/test/ScopInfo/multidim_gep_pointercast2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Verfy that we do not use the GetElementPtr information to delinearize A
 ; because of the cast in-between. Use the single-dimensional modeling instead.
diff --git a/polly/test/ScopInfo/multidim_ivs_and_integer_offsets_3d.ll b/polly/test/ScopInfo/multidim_ivs_and_integer_offsets_3d.ll
index 4610f46ea082..9f7e6bc4a2a2 100644
--- a/polly/test/ScopInfo/multidim_ivs_and_integer_offsets_3d.ll
+++ b/polly/test/ScopInfo/multidim_ivs_and_integer_offsets_3d.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
diff --git a/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll b/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll
index b64e0a99c73b..131bb7b3ebed 100644
--- a/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll
+++ b/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-precise-fold-accesses '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-precise-fold-accesses -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o], long p, long q, long r) {
diff --git a/polly/test/ScopInfo/multidim_many_references.ll b/polly/test/ScopInfo/multidim_many_references.ll
index 0736fd947a31..b0483b267260 100644
--- a/polly/test/ScopInfo/multidim_many_references.ll
+++ b/polly/test/ScopInfo/multidim_many_references.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-ignore-aliasing -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-ignore-aliasing -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-ignore-aliasing -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -polly-ignore-aliasing -disable-output < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/multidim_nested_start_integer.ll b/polly/test/ScopInfo/multidim_nested_start_integer.ll
index db99ab176a3a..741a0ef45c27 100644
--- a/polly/test/ScopInfo/multidim_nested_start_integer.ll
+++ b/polly/test/ScopInfo/multidim_nested_start_integer.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
diff --git a/polly/test/ScopInfo/multidim_nested_start_share_parameter.ll b/polly/test/ScopInfo/multidim_nested_start_share_parameter.ll
index 0ecac3cdaaab..692746bad3d7 100644
--- a/polly/test/ScopInfo/multidim_nested_start_share_parameter.ll
+++ b/polly/test/ScopInfo/multidim_nested_start_share_parameter.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
diff --git a/polly/test/ScopInfo/multidim_only_ivs_2d.ll b/polly/test/ScopInfo/multidim_only_ivs_2d.ll
index ccfa7a7de8e2..71245642e751 100644
--- a/polly/test/ScopInfo/multidim_only_ivs_2d.ll
+++ b/polly/test/ScopInfo/multidim_only_ivs_2d.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; Derived from the following code:
diff --git a/polly/test/ScopInfo/multidim_only_ivs_3d.ll b/polly/test/ScopInfo/multidim_only_ivs_3d.ll
index 7a18aec573e8..a019d58b241d 100644
--- a/polly/test/ScopInfo/multidim_only_ivs_3d.ll
+++ b/polly/test/ScopInfo/multidim_only_ivs_3d.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
diff --git a/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll b/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll
index dd9c4d374f28..41577ef1a0be 100644
--- a/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll
+++ b/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; void foo(int n, int m, int o, double A[n][m][o]) {
 ;
diff --git a/polly/test/ScopInfo/multidim_only_ivs_3d_reverse.ll b/polly/test/ScopInfo/multidim_only_ivs_3d_reverse.ll
index be1b1eb8fc8d..25907f2ee79c 100644
--- a/polly/test/ScopInfo/multidim_only_ivs_3d_reverse.ll
+++ b/polly/test/ScopInfo/multidim_only_ivs_3d_reverse.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; This test case checks for array access functions where the order in which the
diff --git a/polly/test/ScopInfo/multidim_param_in_subscript-2.ll b/polly/test/ScopInfo/multidim_param_in_subscript-2.ll
index f0cd1589b987..0790664f7129 100644
--- a/polly/test/ScopInfo/multidim_param_in_subscript-2.ll
+++ b/polly/test/ScopInfo/multidim_param_in_subscript-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-precise-fold-accesses '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-precise-fold-accesses -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void foo(long n, long m, float A[][n][m]) {
 ;      for (long i = 0; i < 100; i++)
diff --git a/polly/test/ScopInfo/multidim_param_in_subscript.ll b/polly/test/ScopInfo/multidim_param_in_subscript.ll
index ca423919e488..b8ec80b321fe 100644
--- a/polly/test/ScopInfo/multidim_param_in_subscript.ll
+++ b/polly/test/ScopInfo/multidim_param_in_subscript.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;
 ;    void foo(long n, float A[][n]) {
diff --git a/polly/test/ScopInfo/multidim_parameter_addrec_product.ll b/polly/test/ScopInfo/multidim_parameter_addrec_product.ll
index ec311d42386a..7db3e9dc3b5f 100644
--- a/polly/test/ScopInfo/multidim_parameter_addrec_product.ll
+++ b/polly/test/ScopInfo/multidim_parameter_addrec_product.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void foo(float *A, long *p) {
 ;      for (long i = 0; i < 100; i++)
diff --git a/polly/test/ScopInfo/multidim_single_and_multidim_array.ll b/polly/test/ScopInfo/multidim_single_and_multidim_array.ll
index 0d51aa115559..1e302dec4861 100644
--- a/polly/test/ScopInfo/multidim_single_and_multidim_array.ll
+++ b/polly/test/ScopInfo/multidim_single_and_multidim_array.ll
@@ -1,11 +1,11 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-delinearize=false -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-delinearize=false -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NONAFFINE
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=DELIN
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s --check-prefix=DELIN
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-delinearize=false -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-delinearize=false -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NONAFFINE
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=DELIN
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s --check-prefix=DELIN
+; RUN: opt %loadPolly -polly-print-scops -polly-delinearize=false -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-delinearize=false -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=DELIN
+; RUN: opt %loadPolly -polly-print-scops -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=DELIN
+; RUN: opt %loadPolly -polly-print-function-scops -polly-delinearize=false -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -polly-delinearize=false -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=NONAFFINE
+; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s --check-prefix=DELIN
+; RUN: opt %loadPolly -polly-print-function-scops -polly-allow-nonaffine -disable-output < %s | FileCheck %s --check-prefix=DELIN
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/multidim_srem.ll b/polly/test/ScopInfo/multidim_srem.ll
index b4eee668207c..f89843f0a5bc 100644
--- a/polly/test/ScopInfo/multidim_srem.ll
+++ b/polly/test/ScopInfo/multidim_srem.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void foo(long n, float A[][n][n]) {
 ;      for (long i = 0; i < 200; i++)
diff --git a/polly/test/ScopInfo/multidim_with_bitcast.ll b/polly/test/ScopInfo/multidim_with_bitcast.ll
index 8af2e18265b0..b77ff689b953 100644
--- a/polly/test/ScopInfo/multidim_with_bitcast.ll
+++ b/polly/test/ScopInfo/multidim_with_bitcast.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/multiple-binary-or-conditions.ll b/polly/test/ScopInfo/multiple-binary-or-conditions.ll
index 481f799ed0fe..b905a11f577c 100644
--- a/polly/test/ScopInfo/multiple-binary-or-conditions.ll
+++ b/polly/test/ScopInfo/multiple-binary-or-conditions.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -disable-output < %s
 ;
 ; void or(float *A, long n, long m) {
 ;   for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/multiple-types-access-offset-not-dividable-by-element-size.ll b/polly/test/ScopInfo/multiple-types-access-offset-not-dividable-by-element-size.ll
index 69b00402ba6b..2d03ad941c05 100644
--- a/polly/test/ScopInfo/multiple-types-access-offset-not-dividable-by-element-size.ll
+++ b/polly/test/ScopInfo/multiple-types-access-offset-not-dividable-by-element-size.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -pass-remarks-analysis="polly-scops" \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -pass-remarks-analysis="polly-scops" \
 ; RUN:     -polly-allow-differing-element-types \
 ; RUN:     -disable-output < %s  2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/multiple-types-non-affine-2.ll b/polly/test/ScopInfo/multiple-types-non-affine-2.ll
index a7e9a31744b4..5b0aa5de1e71 100644
--- a/polly/test/ScopInfo/multiple-types-non-affine-2.ll
+++ b/polly/test/ScopInfo/multiple-types-non-affine-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types '-passes=print<polly-function-scops>' -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -passes=polly-codegen -polly-allow-nonaffine -disable-output
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -polly-print-scops -polly-allow-nonaffine -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -polly-codegen -polly-allow-nonaffine -disable-output
 ;
 ;    // Check that accessing one array with different types works,
 ;    // even though some accesses are non-affine.
diff --git a/polly/test/ScopInfo/multiple-types-non-affine.ll b/polly/test/ScopInfo/multiple-types-non-affine.ll
index e49612aa2791..8e4be4c86d5a 100644
--- a/polly/test/ScopInfo/multiple-types-non-affine.ll
+++ b/polly/test/ScopInfo/multiple-types-non-affine.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types '-passes=print<polly-function-scops>' -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -passes=polly-codegen -polly-allow-nonaffine -disable-output
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -polly-print-scops -polly-allow-nonaffine -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-differing-element-types -polly-codegen -polly-allow-nonaffine -disable-output
 ;
 ;    // Check that accessing one array with different types works,
 ;    // even though some accesses are non-affine.
diff --git a/polly/test/ScopInfo/multiple-types-non-power-of-two-2.ll b/polly/test/ScopInfo/multiple-types-non-power-of-two-2.ll
index 8347de936eff..01f5923457b4 100644
--- a/polly/test/ScopInfo/multiple-types-non-power-of-two-2.ll
+++ b/polly/test/ScopInfo/multiple-types-non-power-of-two-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-allow-differing-element-types -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-allow-differing-element-types -disable-output < %s | FileCheck %s
 ;
 ;  void multiple_types(i8 *A) {
 ;    for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/multiple-types-non-power-of-two.ll b/polly/test/ScopInfo/multiple-types-non-power-of-two.ll
index 077ff1a741ad..142a5ac395b3 100644
--- a/polly/test/ScopInfo/multiple-types-non-power-of-two.ll
+++ b/polly/test/ScopInfo/multiple-types-non-power-of-two.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-allow-differing-element-types -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-allow-differing-element-types -disable-output < %s | FileCheck %s
 ;
 ;  void multiple_types(i8 *A) {
 ;    for (long i = 0; i < 100; i++) {
diff --git a/polly/test/ScopInfo/multiple-types-two-dimensional-2.ll b/polly/test/ScopInfo/multiple-types-two-dimensional-2.ll
index 431a0c1e966f..1e2e53e85c25 100644
--- a/polly/test/ScopInfo/multiple-types-two-dimensional-2.ll
+++ b/polly/test/ScopInfo/multiple-types-two-dimensional-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -pass-remarks-analysis="polly-scops" \
+; RUN: opt %loadPolly -polly-print-scops -pass-remarks-analysis="polly-scops" \
 ; RUN:                -polly-allow-differing-element-types \
 ; RUN:                -disable-output < %s 2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/multiple-types-two-dimensional.ll b/polly/test/ScopInfo/multiple-types-two-dimensional.ll
index 2cc2454ca6e4..21dc96e6f95d 100644
--- a/polly/test/ScopInfo/multiple-types-two-dimensional.ll
+++ b/polly/test/ScopInfo/multiple-types-two-dimensional.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -pass-remarks-analysis="polly-scops" \
+; RUN: opt %loadPolly -polly-print-scops -pass-remarks-analysis="polly-scops" \
 ; RUN:     -polly-allow-differing-element-types \
 ; RUN:     -disable-output < %s  2>&1 | FileCheck %s
 ;
diff --git a/polly/test/ScopInfo/multiple-types.ll b/polly/test/ScopInfo/multiple-types.ll
index b7006c104426..16db191c522f 100644
--- a/polly/test/ScopInfo/multiple-types.ll
+++ b/polly/test/ScopInfo/multiple-types.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' \
-; RUN: -polly-allow-differing-element-types -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops \
+; RUN: -polly-allow-differing-element-types -disable-output < %s | FileCheck %s
 ;
 ;    // Check that accessing one array with different types works.
 ;    void multiple_types(char *Short, char *Float, char *Double) {
diff --git a/polly/test/ScopInfo/multiple_exiting_blocks.ll b/polly/test/ScopInfo/multiple_exiting_blocks.ll
index a21a7252c2ff..f8e5d4106a16 100644
--- a/polly/test/ScopInfo/multiple_exiting_blocks.ll
+++ b/polly/test/ScopInfo/multiple_exiting_blocks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/multiple_exiting_blocks_two_loop.ll b/polly/test/ScopInfo/multiple_exiting_blocks_two_loop.ll
index e56c8b7094ea..c695f3c913db 100644
--- a/polly/test/ScopInfo/multiple_exiting_blocks_two_loop.ll
+++ b/polly/test/ScopInfo/multiple_exiting_blocks_two_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/multiple_latch_blocks.ll b/polly/test/ScopInfo/multiple_latch_blocks.ll
index c1a64ee1c3cb..d3949e7e2c3c 100644
--- a/polly/test/ScopInfo/multiple_latch_blocks.ll
+++ b/polly/test/ScopInfo/multiple_latch_blocks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Domain :=
 ; CHECK:   [N, P] -> { Stmt_if_end[i0] : 0 <= i0 < N and (i0 > P or i0 < P) };
diff --git a/polly/test/ScopInfo/nested-loops.ll b/polly/test/ScopInfo/nested-loops.ll
index 2819ae461fce..ed814f826829 100644
--- a/polly/test/ScopInfo/nested-loops.ll
+++ b/polly/test/ScopInfo/nested-loops.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
diff --git a/polly/test/ScopInfo/no-scalar-deps-in-non-affine-subregion.ll b/polly/test/ScopInfo/no-scalar-deps-in-non-affine-subregion.ll
index 6fc364076750..7c55e242641c 100644
--- a/polly/test/ScopInfo/no-scalar-deps-in-non-affine-subregion.ll
+++ b/polly/test/ScopInfo/no-scalar-deps-in-non-affine-subregion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that we do not generate any scalar dependences regarding x. It is
 ; defined and used on the non-affine subregion only, thus we do not need
diff --git a/polly/test/ScopInfo/non-affine-region-phi.ll b/polly/test/ScopInfo/non-affine-region-phi.ll
index 4de76c4adfb1..f99782b9a0ff 100644
--- a/polly/test/ScopInfo/non-affine-region-phi.ll
+++ b/polly/test/ScopInfo/non-affine-region-phi.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -S < %s 2>&1 | FileCheck %s --check-prefix=CODE
-; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -S < %s | FileCheck %s --check-prefix=CODE
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Verify there is a phi in the non-affine region but it is not represented in
 ; the SCoP as all operands as well as the uses are inside the region too.
diff --git a/polly/test/ScopInfo/non-affine-region-with-loop-2.ll b/polly/test/ScopInfo/non-affine-region-with-loop-2.ll
index 9870b813d287..b673fda5ec3c 100644
--- a/polly/test/ScopInfo/non-affine-region-with-loop-2.ll
+++ b/polly/test/ScopInfo/non-affine-region-with-loop-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-nonaffine-loops '-passes=print<polly-detect>,print<polly-function-scops>,scop(polly-codegen)' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-allow-nonaffine-loops -polly-print-scops -polly-codegen -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    Stmt_loop3
 ; CHECK:            Domain :=
diff --git a/polly/test/ScopInfo/non-affine-region-with-loop.ll b/polly/test/ScopInfo/non-affine-region-with-loop.ll
index e1342e1b5257..32dde8b4a682 100644
--- a/polly/test/ScopInfo/non-affine-region-with-loop.ll
+++ b/polly/test/ScopInfo/non-affine-region-with-loop.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops -passes=polly-codegen -disable-output
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-codegen -disable-output
 ;
 ; CHECK:      Domain :=
 ; CHECK-NEXT:   { Stmt_loop2__TO__loop[] };
diff --git a/polly/test/ScopInfo/non-precise-inv-load-1.ll b/polly/test/ScopInfo/non-precise-inv-load-1.ll
index f35235d2de8f..5394206dd547 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-1.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; Verify we do hoist the invariant access to I with a execution context
 ; as the address computation might wrap in the original but not in our
diff --git a/polly/test/ScopInfo/non-precise-inv-load-2.ll b/polly/test/ScopInfo/non-precise-inv-load-2.ll
index c538c0e7bda0..5c0c56513a08 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-2.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ;
 ; CHECK:       Invariant Accesses: {
diff --git a/polly/test/ScopInfo/non-precise-inv-load-3.ll b/polly/test/ScopInfo/non-precise-inv-load-3.ll
index c16879a18562..09d09319656b 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-3.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; CHECK:        Invariant Accesses: {
 ; CHECK-NEXT:     ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/non-precise-inv-load-4.ll b/polly/test/ScopInfo/non-precise-inv-load-4.ll
index 24f9d45d28e7..da5f656576d1 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-4.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; Verify we hoist I[0] without execution context even though it
 ; is executed in a statement with an invalid domain.
diff --git a/polly/test/ScopInfo/non-precise-inv-load-5.ll b/polly/test/ScopInfo/non-precise-inv-load-5.ll
index 17046685562d..bff5f59a3302 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-5.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; Verify we do not hoist I[c] without execution context because it
 ; is executed in a statement with an invalid domain and it depends
diff --git a/polly/test/ScopInfo/non-precise-inv-load-6.ll b/polly/test/ScopInfo/non-precise-inv-load-6.ll
index eeada91299f6..03540a8ead96 100644
--- a/polly/test/ScopInfo/non-precise-inv-load-6.ll
+++ b/polly/test/ScopInfo/non-precise-inv-load-6.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; Check that we model the execution context correctly.
 ;
diff --git a/polly/test/ScopInfo/non-pure-function-call.ll b/polly/test/ScopInfo/non-pure-function-call.ll
index 974c9ba0527d..4ffb8d28865d 100644
--- a/polly/test/ScopInfo/non-pure-function-call.ll
+++ b/polly/test/ScopInfo/non-pure-function-call.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      Assumed Context:
 ; CHECK-NEXT:   [N] -> {  :  }
diff --git a/polly/test/ScopInfo/non-pure-function-calls-causes-dead-blocks.ll b/polly/test/ScopInfo/non-pure-function-calls-causes-dead-blocks.ll
index 983c45bc7536..27998b50b74f 100644
--- a/polly/test/ScopInfo/non-pure-function-calls-causes-dead-blocks.ll
+++ b/polly/test/ScopInfo/non-pure-function-calls-causes-dead-blocks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Error blocks are skipped during SCoP detection. We skip them during
 ; SCoP formation too as they might contain instructions we can not handle.
diff --git a/polly/test/ScopInfo/non-pure-function-calls.ll b/polly/test/ScopInfo/non-pure-function-calls.ll
index fd6a6dc3f8f1..3ecf75853773 100644
--- a/polly/test/ScopInfo/non-pure-function-calls.ll
+++ b/polly/test/ScopInfo/non-pure-function-calls.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Allow the user to define function names that are treated as
 ; error functions and assumed not to be executed.
diff --git a/polly/test/ScopInfo/non_affine_access.ll b/polly/test/ScopInfo/non_affine_access.ll
index e20be38598db..a83c9484ad52 100644
--- a/polly/test/ScopInfo/non_affine_access.ll
+++ b/polly/test/ScopInfo/non_affine_access.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -polly-allow-nonaffine -disable-output < %s 2>&1 | FileCheck %s -check-prefix=NONAFFINE
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-allow-nonaffine -disable-output < %s | FileCheck %s -check-prefix=NONAFFINE
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 ; void foo(long *A) {
diff --git a/polly/test/ScopInfo/non_affine_region_1.ll b/polly/test/ScopInfo/non_affine_region_1.ll
index 623d322c508c..7c4312599cf0 100644
--- a/polly/test/ScopInfo/non_affine_region_1.ll
+++ b/polly/test/ScopInfo/non_affine_region_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Verify only the incoming scalar x is modeled as a read in the non-affine
 ; region.
diff --git a/polly/test/ScopInfo/non_affine_region_2.ll b/polly/test/ScopInfo/non_affine_region_2.ll
index ba20d9e6cf6e..0bc467c92bcb 100644
--- a/polly/test/ScopInfo/non_affine_region_2.ll
+++ b/polly/test/ScopInfo/non_affine_region_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Verify the scalar x defined in a non-affine subregion is written as it
 ; escapes the region. In this test the two conditionals inside the region
diff --git a/polly/test/ScopInfo/non_affine_region_3.ll b/polly/test/ScopInfo/non_affine_region_3.ll
index ff619b579355..6d5f94df6110 100644
--- a/polly/test/ScopInfo/non_affine_region_3.ll
+++ b/polly/test/ScopInfo/non_affine_region_3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
 ;
 ; Verify the scalar x defined in a non-affine subregion is written as it
 ; escapes the region. In this test the two conditionals inside the region
diff --git a/polly/test/ScopInfo/non_affine_region_4.ll b/polly/test/ScopInfo/non_affine_region_4.ll
index 70f40727849a..f37e0ecb89d1 100644
--- a/polly/test/ScopInfo/non_affine_region_4.ll
+++ b/polly/test/ScopInfo/non_affine_region_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Verify that both scalars (x and y) are properly written in the non-affine
 ; region and read afterwards.
diff --git a/polly/test/ScopInfo/nonaffine-buildMemoryAccess.ll b/polly/test/ScopInfo/nonaffine-buildMemoryAccess.ll
index 246610ee5194..445dd164898b 100644
--- a/polly/test/ScopInfo/nonaffine-buildMemoryAccess.ll
+++ b/polly/test/ScopInfo/nonaffine-buildMemoryAccess.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine-loops '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine-loops -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      Domain :=
 ; CHECK-NEXT:   { Stmt_while_cond_i__TO__while_end_i[] };
diff --git a/polly/test/ScopInfo/not-a-reduction.ll b/polly/test/ScopInfo/not-a-reduction.ll
index 7fe41332c67d..87909290fd71 100644
--- a/polly/test/ScopInfo/not-a-reduction.ll
+++ b/polly/test/ScopInfo/not-a-reduction.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | not FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s 2>&1 | not FileCheck %s
 
 ;#define TYPE float
 ;#define NUM 4
diff --git a/polly/test/ScopInfo/opaque-struct.ll b/polly/test/ScopInfo/opaque-struct.ll
index 1a0859f71f36..19fdd9bf9179 100644
--- a/polly/test/ScopInfo/opaque-struct.ll
+++ b/polly/test/ScopInfo/opaque-struct.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-scops -disable-output < %s
 ;
 ; Check that we do not crash with unsized (opaque) types.
 ;
diff --git a/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node-nonaffine-subregion.ll b/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node-nonaffine-subregion.ll
index 58c2116ab9dc..394173bdc986 100644
--- a/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node-nonaffine-subregion.ll
+++ b/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node-nonaffine-subregion.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-codegen -S < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
 ;
 ; Check whether %newval is identified as escaping value, even though it is used
 ; in a phi that is in the region. Non-affine subregion case.
diff --git a/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node.ll b/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node.ll
index 7986399a6c7e..e17164e89372 100644
--- a/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node.ll
+++ b/polly/test/ScopInfo/out-of-scop-use-in-region-entry-phi-node.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; CHECK: MustWriteAccess :=  [Reduction Type: NONE] [Scalar: 1]
 ; CHECK-NEXT: [p_0] -> { Stmt_bb3[] -> MemRef_tmp5[] };
diff --git a/polly/test/ScopInfo/parameter-constant-division.ll b/polly/test/ScopInfo/parameter-constant-division.ll
index f1e006e29454..cd6b9e3526aa 100644
--- a/polly/test/ScopInfo/parameter-constant-division.ll
+++ b/polly/test/ScopInfo/parameter-constant-division.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops \
 ; RUN:     -polly-invariant-load-hoisting=true \
-; RUN:     -disable-output < %s 2>&1 | FileCheck %s
+; RUN:     -disable-output < %s | FileCheck %s
 ;
 ; CHECK:          Invariant Accesses: {
 ; CHECK-NEXT:            ReadAccess :=	[Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/parameter_in_dead_statement.ll b/polly/test/ScopInfo/parameter_in_dead_statement.ll
index 13602515d46f..4b4a87f098d7 100644
--- a/polly/test/ScopInfo/parameter_in_dead_statement.ll
+++ b/polly/test/ScopInfo/parameter_in_dead_statement.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -S \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -polly-print-scops -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -S \
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s --check-prefix=IR
 ;
 ; Verify we do not create assumptions based on the parameter p_1 which is the
 ; load %0 and due to error-assumptions not "part of the SCoP".
diff --git a/polly/test/ScopInfo/parameter_product.ll b/polly/test/ScopInfo/parameter_product.ll
index 9783268f2a10..1ba7280f97c9 100644
--- a/polly/test/ScopInfo/parameter_product.ll
+++ b/polly/test/ScopInfo/parameter_product.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; int n, m;
 ; void foo(char* __restrict a)
diff --git a/polly/test/ScopInfo/parameter_with_constant_factor_in_add.ll b/polly/test/ScopInfo/parameter_with_constant_factor_in_add.ll
index f750cf7f3320..72d580801573 100644
--- a/polly/test/ScopInfo/parameter_with_constant_factor_in_add.ll
+++ b/polly/test/ScopInfo/parameter_with_constant_factor_in_add.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that the access function of the store is simple and concise
 ;
diff --git a/polly/test/ScopInfo/partially_invariant_load_1.ll b/polly/test/ScopInfo/partially_invariant_load_1.ll
index 5757e4d7095c..274a7873c782 100644
--- a/polly/test/ScopInfo/partially_invariant_load_1.ll
+++ b/polly/test/ScopInfo/partially_invariant_load_1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -passes=polly-codegen -polly-invariant-load-hoisting=true -S < %s 2>&1 | FileCheck %s --check-prefix=IR
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-codegen -polly-invariant-load-hoisting=true -S < %s | FileCheck %s --check-prefix=IR
 ;
 ; CHECK:          Invariant Accesses: {
 ; CHECK-NEXT:             ReadAccess :=	[Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/partially_invariant_load_2.ll b/polly/test/ScopInfo/partially_invariant_load_2.ll
index e07d6d39132b..ee1092883f72 100644
--- a/polly/test/ScopInfo/partially_invariant_load_2.ll
+++ b/polly/test/ScopInfo/partially_invariant_load_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-invariant-load-hoisting=true -disable-output < %s | FileCheck %s
 ;
 ; Check that we do not try to preload *I and assume p != 42.
 ;
diff --git a/polly/test/ScopInfo/phi-in-non-affine-region.ll b/polly/test/ScopInfo/phi-in-non-affine-region.ll
index c8e81fd1ba98..6ef24e3f1456 100644
--- a/polly/test/ScopInfo/phi-in-non-affine-region.ll
+++ b/polly/test/ScopInfo/phi-in-non-affine-region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; Verify that 'tmp' is stored in bb1 and read by bb3, as it is needed as
 ; incoming value for the tmp11 PHI node.
diff --git a/polly/test/ScopInfo/phi_after_error_block.ll b/polly/test/ScopInfo/phi_after_error_block.ll
index 21d532b06d60..039fb86bec5b 100644
--- a/polly/test/ScopInfo/phi_after_error_block.ll
+++ b/polly/test/ScopInfo/phi_after_error_block.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 
 declare void @bar()
 
diff --git a/polly/test/ScopInfo/phi_condition_modeling_1.ll b/polly/test/ScopInfo/phi_condition_modeling_1.ll
index ca87055630c6..a879c2005ad8 100644
--- a/polly/test/ScopInfo/phi_condition_modeling_1.ll
+++ b/polly/test/ScopInfo/phi_condition_modeling_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, int c, int N) {
 ;      int tmp;
diff --git a/polly/test/ScopInfo/phi_condition_modeling_2.ll b/polly/test/ScopInfo/phi_condition_modeling_2.ll
index 10511ef0009f..cedc140f8438 100644
--- a/polly/test/ScopInfo/phi_condition_modeling_2.ll
+++ b/polly/test/ScopInfo/phi_condition_modeling_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, int c, int N) {
 ;      int tmp;
diff --git a/polly/test/ScopInfo/phi_conditional_simple_1.ll b/polly/test/ScopInfo/phi_conditional_simple_1.ll
index 2a009dc26d24..90213a953767 100644
--- a/polly/test/ScopInfo/phi_conditional_simple_1.ll
+++ b/polly/test/ScopInfo/phi_conditional_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void jd(int *A, int c) {
 ;      for (int i = 0; i < 1024; i++) {
diff --git a/polly/test/ScopInfo/phi_loop_carried_float.ll b/polly/test/ScopInfo/phi_loop_carried_float.ll
index 5bb740154877..d8d2608329bc 100644
--- a/polly/test/ScopInfo/phi_loop_carried_float.ll
+++ b/polly/test/ScopInfo/phi_loop_carried_float.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    float f(float *A, int N) {
 ;      float tmp = 0;
diff --git a/polly/test/ScopInfo/phi_not_grouped_at_top.ll b/polly/test/ScopInfo/phi_not_grouped_at_top.ll
index 1ed22b3fec42..be082165b635 100644
--- a/polly/test/ScopInfo/phi_not_grouped_at_top.ll
+++ b/polly/test/ScopInfo/phi_not_grouped_at_top.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -passes=polly-prepare -disable-output < %s
+; RUN: opt %loadPolly -polly-prepare -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 declare i32 @funa() align 2
diff --git a/polly/test/ScopInfo/phi_scalar_simple_1.ll b/polly/test/ScopInfo/phi_scalar_simple_1.ll
index eab261b0d153..d042613c023f 100644
--- a/polly/test/ScopInfo/phi_scalar_simple_1.ll
+++ b/polly/test/ScopInfo/phi_scalar_simple_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; The assumed context should be empty since the <nsw> flags on the IV
 ; increments already guarantee that there is no wrap in the loop trip
diff --git a/polly/test/ScopInfo/phi_scalar_simple_2.ll b/polly/test/ScopInfo/phi_scalar_simple_2.ll
index 73bef9601f33..fb4292e05ca6 100644
--- a/polly/test/ScopInfo/phi_scalar_simple_2.ll
+++ b/polly/test/ScopInfo/phi_scalar_simple_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    int jd(int *restrict A, int x, int N, int c) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/phi_with_invoke_edge.ll b/polly/test/ScopInfo/phi_with_invoke_edge.ll
index 3d7b7d3d38d5..dbcf04c0561a 100644
--- a/polly/test/ScopInfo/phi_with_invoke_edge.ll
+++ b/polly/test/ScopInfo/phi_with_invoke_edge.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s
+; RUN: opt %loadPolly -polly-detect -disable-output < %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 declare i32 @generic_personality_v0(i32, i64, ptr, ptr)
diff --git a/polly/test/ScopInfo/pointer-comparison-no-nsw.ll b/polly/test/ScopInfo/pointer-comparison-no-nsw.ll
index 40d2138f81d5..094c5ccab54d 100644
--- a/polly/test/ScopInfo/pointer-comparison-no-nsw.ll
+++ b/polly/test/ScopInfo/pointer-comparison-no-nsw.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, int *B) {
 ;      while (A != B) {
diff --git a/polly/test/ScopInfo/pointer-comparison.ll b/polly/test/ScopInfo/pointer-comparison.ll
index 960b9c5f3132..15ce0491209a 100644
--- a/polly/test/ScopInfo/pointer-comparison.ll
+++ b/polly/test/ScopInfo/pointer-comparison.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; TODO: FIXME: Investigate why we need a InvalidContext here.
 ;
diff --git a/polly/test/ScopInfo/pointer-type-expressions.ll b/polly/test/ScopInfo/pointer-type-expressions.ll
index 919a9fd4000b..ebbb644340f6 100644
--- a/polly/test/ScopInfo/pointer-type-expressions.ll
+++ b/polly/test/ScopInfo/pointer-type-expressions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; void f(int a[], int N, float *P) {
 ;   int i;
diff --git a/polly/test/ScopInfo/pointer-used-as-base-pointer-and-scalar-read.ll b/polly/test/ScopInfo/pointer-used-as-base-pointer-and-scalar-read.ll
index 1eb053c83132..3ac86a3443af 100644
--- a/polly/test/ScopInfo/pointer-used-as-base-pointer-and-scalar-read.ll
+++ b/polly/test/ScopInfo/pointer-used-as-base-pointer-and-scalar-read.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; In this test case we pass a pointer %A into a PHI node and also use this
 ; pointer as base pointer of an array store. As a result, we get both scalar
diff --git a/polly/test/ScopInfo/polly-timeout-parameter-bounds.ll b/polly/test/ScopInfo/polly-timeout-parameter-bounds.ll
index 7723c185e41c..8152010c2c99 100644
--- a/polly/test/ScopInfo/polly-timeout-parameter-bounds.ll
+++ b/polly/test/ScopInfo/polly-timeout-parameter-bounds.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; CHECK:      Statements {
 ; CHECK-NEXT:  	Stmt_bb9
diff --git a/polly/test/ScopInfo/preserve-equiv-class-order-in-basic_block.ll b/polly/test/ScopInfo/preserve-equiv-class-order-in-basic_block.ll
index 3cc3e51ef013..4a68acd3d509 100644
--- a/polly/test/ScopInfo/preserve-equiv-class-order-in-basic_block.ll
+++ b/polly/test/ScopInfo/preserve-equiv-class-order-in-basic_block.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=scalar-indep -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
 
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/polly/test/ScopInfo/process_added_dimensions.ll b/polly/test/ScopInfo/process_added_dimensions.ll
index 66c9ded40f7c..6cb270a071f4 100644
--- a/polly/test/ScopInfo/process_added_dimensions.ll
+++ b/polly/test/ScopInfo/process_added_dimensions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; CHECK:      Context:
 ; CHECK-NEXT: {  :  }
diff --git a/polly/test/ScopInfo/pwaff-complexity-bailout.ll b/polly/test/ScopInfo/pwaff-complexity-bailout.ll
index 5119334745bc..19dd156d27db 100644
--- a/polly/test/ScopInfo/pwaff-complexity-bailout.ll
+++ b/polly/test/ScopInfo/pwaff-complexity-bailout.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -pass-remarks-analysis=.* -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-scops -pass-remarks-analysis=.* -disable-output < %s 2>&1 | FileCheck %s
 
 ; Make sure we hit the complexity bailout, and don't crash.
 ; CHECK: Low complexity assumption:       {  : false }
diff --git a/polly/test/ScopInfo/ranged_parameter.ll b/polly/test/ScopInfo/ranged_parameter.ll
index b5cb77593352..4b04960ee845 100644
--- a/polly/test/ScopInfo/ranged_parameter.ll
+++ b/polly/test/ScopInfo/ranged_parameter.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that the constraints on the parameter derived from the
 ; range metadata (see bottom of the file) are present:
diff --git a/polly/test/ScopInfo/ranged_parameter_2.ll b/polly/test/ScopInfo/ranged_parameter_2.ll
index 52933398f796..cd7d2bfb84d0 100644
--- a/polly/test/ScopInfo/ranged_parameter_2.ll
+++ b/polly/test/ScopInfo/ranged_parameter_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-allow-nonaffine -polly-invariant-load-hoisting=true < %s \
+; RUN: opt %loadPolly -polly-print-scops -disable-output -polly-allow-nonaffine -polly-invariant-load-hoisting=true < %s \
 ; RUN:  -debug 2>&1 | FileCheck %s
 
 ; REQUIRES: asserts
diff --git a/polly/test/ScopInfo/ranged_parameter_wrap.ll b/polly/test/ScopInfo/ranged_parameter_wrap.ll
index 724427fabfd1..173746352cf0 100644
--- a/polly/test/ScopInfo/ranged_parameter_wrap.ll
+++ b/polly/test/ScopInfo/ranged_parameter_wrap.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that the constraints on the parameter derived from the
 ; __wrapping__ range metadata (see bottom of the file) are present:
diff --git a/polly/test/ScopInfo/ranged_parameter_wrap_2.ll b/polly/test/ScopInfo/ranged_parameter_wrap_2.ll
index 234c3edde14e..33f57f37a1e8 100644
--- a/polly/test/ScopInfo/ranged_parameter_wrap_2.ll
+++ b/polly/test/ScopInfo/ranged_parameter_wrap_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that the context is built fast and does not explode due to us
 ; combining a large number of non-convex ranges. Instead, after a certain
diff --git a/polly/test/ScopInfo/read-only-scalar-used-in-phi-2.ll b/polly/test/ScopInfo/read-only-scalar-used-in-phi-2.ll
index 1ab8fe897308..23c7aa261ac0 100644
--- a/polly/test/ScopInfo/read-only-scalar-used-in-phi-2.ll
+++ b/polly/test/ScopInfo/read-only-scalar-used-in-phi-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    float foo(float sum, float A[]) {
 ;
diff --git a/polly/test/ScopInfo/read-only-scalar-used-in-phi.ll b/polly/test/ScopInfo/read-only-scalar-used-in-phi.ll
index 358b51904c72..20f44c94251c 100644
--- a/polly/test/ScopInfo/read-only-scalar-used-in-phi.ll
+++ b/polly/test/ScopInfo/read-only-scalar-used-in-phi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    float foo(float sum, float A[]) {
 ;
diff --git a/polly/test/ScopInfo/read-only-scalars.ll b/polly/test/ScopInfo/read-only-scalars.ll
index 43a456ea9977..71c2d21e357a 100644
--- a/polly/test/ScopInfo/read-only-scalars.ll
+++ b/polly/test/ScopInfo/read-only-scalars.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-analyze-read-only-scalars=false '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-analyze-read-only-scalars=true  '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=SCALARS
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-analyze-read-only-scalars=false -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-analyze-read-only-scalars=true  -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=SCALARS
 
 ; CHECK-NOT: Memref_scalar
 
diff --git a/polly/test/ScopInfo/read-only-statements.ll b/polly/test/ScopInfo/read-only-statements.ll
index 3fa72789f4e1..a93063ea3ad6 100644
--- a/polly/test/ScopInfo/read-only-statements.ll
+++ b/polly/test/ScopInfo/read-only-statements.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check we remove read only statements.
 ;
diff --git a/polly/test/ScopInfo/reduction_alternating_base.ll b/polly/test/ScopInfo/reduction_alternating_base.ll
index f44367f295ff..854e28023a3e 100644
--- a/polly/test/ScopInfo/reduction_alternating_base.ll
+++ b/polly/test/ScopInfo/reduction_alternating_base.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;
 ;    void f(int *A) {
diff --git a/polly/test/ScopInfo/reduction_chain_partially_outside_the_scop.ll b/polly/test/ScopInfo/reduction_chain_partially_outside_the_scop.ll
index 5636ee7ed828..fb0274972082 100644
--- a/polly/test/ScopInfo/reduction_chain_partially_outside_the_scop.ll
+++ b/polly/test/ScopInfo/reduction_chain_partially_outside_the_scop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Reduction Type: NONE
 ;
diff --git a/polly/test/ScopInfo/reduction_different_index.ll b/polly/test/ScopInfo/reduction_different_index.ll
index 7ed9e662a15c..575e5a16d7b2 100644
--- a/polly/test/ScopInfo/reduction_different_index.ll
+++ b/polly/test/ScopInfo/reduction_different_index.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ; Verify if the following case is not detected as reduction.
 ;
 ; void f(int *A,int *sum) {
diff --git a/polly/test/ScopInfo/reduction_different_index1.ll b/polly/test/ScopInfo/reduction_different_index1.ll
index f868bd657f3b..39bd3c4b9abe 100644
--- a/polly/test/ScopInfo/reduction_different_index1.ll
+++ b/polly/test/ScopInfo/reduction_different_index1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ; Verify if the following case is not detected as reduction.
 ;
 ; void f(int *A, int *sum, int i1, int i2) {
diff --git a/polly/test/ScopInfo/reduction_disabled_multiplicative.ll b/polly/test/ScopInfo/reduction_disabled_multiplicative.ll
index b031fd352323..7120740fbf34 100644
--- a/polly/test/ScopInfo/reduction_disabled_multiplicative.ll
+++ b/polly/test/ScopInfo/reduction_disabled_multiplicative.ll
@@ -1,4 +1,4 @@
-; RUN: opt -aa-pipeline=basic-aa %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -polly-disable-multiplicative-reductions -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-disable-multiplicative-reductions -disable-output < %s | FileCheck %s
 ;
 ; CHECK: ReadAccess :=       [Reduction Type: +
 ; CHECK:     { Stmt_for_body[i0] -> MemRef_sum[0] };
diff --git a/polly/test/ScopInfo/reduction_escaping_intermediate.ll b/polly/test/ScopInfo/reduction_escaping_intermediate.ll
index dbfa1f1b1d59..dde09108ecc4 100644
--- a/polly/test/ScopInfo/reduction_escaping_intermediate.ll
+++ b/polly/test/ScopInfo/reduction_escaping_intermediate.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; void f(int N, int * restrict sums, int * restrict escape) {
 ;   int i, j;
diff --git a/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll b/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll
index 1fa8bbcc53e0..702fc56025d9 100644
--- a/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll
+++ b/polly/test/ScopInfo/reduction_escaping_intermediate_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; void f(int N, int * restrict sums, int * restrict escape) {
 ;   int i, j;
diff --git a/polly/test/ScopInfo/reduction_invalid_different_operators.ll b/polly/test/ScopInfo/reduction_invalid_different_operators.ll
index d1bfb71fbbd4..f47919dcad99 100644
--- a/polly/test/ScopInfo/reduction_invalid_different_operators.ll
+++ b/polly/test/ScopInfo/reduction_invalid_different_operators.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; int f() {
 ;   int i, sum = 0, sth = 0;
diff --git a/polly/test/ScopInfo/reduction_invalid_overlapping_accesses.ll b/polly/test/ScopInfo/reduction_invalid_overlapping_accesses.ll
index 654475253799..be1d7b5bbbd9 100644
--- a/polly/test/ScopInfo/reduction_invalid_overlapping_accesses.ll
+++ b/polly/test/ScopInfo/reduction_invalid_overlapping_accesses.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; void f(int *sums) {
 ;   int i, j;
diff --git a/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll b/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll
index a8167f1c38de..8d20fa13ffe5 100644
--- a/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll
+++ b/polly/test/ScopInfo/reduction_multiple_loops_array_sum.ll
@@ -1,4 +1,4 @@
-; RUN: opt -aa-pipeline=basic-aa %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Stmt_for_body
 ; CHECK: Reduction Type: *
diff --git a/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll b/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll
index a0c54572b599..782332b56aad 100644
--- a/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll
+++ b/polly/test/ScopInfo/reduction_multiple_loops_array_sum_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt -aa-pipeline=basic-aa %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Stmt_for_body
 ; CHECK: Reduction Type: NONE
diff --git a/polly/test/ScopInfo/reduction_multiple_simple_binary.ll b/polly/test/ScopInfo/reduction_multiple_simple_binary.ll
index 3ed664050a0b..0f1a3ad90dac 100644
--- a/polly/test/ScopInfo/reduction_multiple_simple_binary.ll
+++ b/polly/test/ScopInfo/reduction_multiple_simple_binary.ll
@@ -1,4 +1,4 @@
-; RUN: opt -aa-pipeline=basic-aa %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -basic-aa %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: ReadAccess :=       [Reduction Type: NONE
 ; CHECK:     { Stmt_for_body[i0] -> MemRef_A[1 + i0] };
diff --git a/polly/test/ScopInfo/reduction_non_overlapping_chains.ll b/polly/test/ScopInfo/reduction_non_overlapping_chains.ll
index 7c8c8616a1cd..4e3f841cd8e1 100644
--- a/polly/test/ScopInfo/reduction_non_overlapping_chains.ll
+++ b/polly/test/ScopInfo/reduction_non_overlapping_chains.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Reduction Type: +
 ; CHECK: Reduction Type: +
diff --git a/polly/test/ScopInfo/reduction_only_reduction_like_access.ll b/polly/test/ScopInfo/reduction_only_reduction_like_access.ll
index 95cda973a9b0..0c61d63a2d45 100644
--- a/polly/test/ScopInfo/reduction_only_reduction_like_access.ll
+++ b/polly/test/ScopInfo/reduction_only_reduction_like_access.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Reduction Type: +
 ;
diff --git a/polly/test/ScopInfo/reduction_simple_fp.ll b/polly/test/ScopInfo/reduction_simple_fp.ll
index 37693353376b..ba0a034a17e3 100644
--- a/polly/test/ScopInfo/reduction_simple_fp.ll
+++ b/polly/test/ScopInfo/reduction_simple_fp.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Function: f_no_fast_math
 ; CHECK: Reduction Type: NONE
diff --git a/polly/test/ScopInfo/reduction_simple_w_constant.ll b/polly/test/ScopInfo/reduction_simple_w_constant.ll
index 550882300116..dc1f8550602d 100644
--- a/polly/test/ScopInfo/reduction_simple_w_constant.ll
+++ b/polly/test/ScopInfo/reduction_simple_w_constant.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Reduction Type: +
 ;
diff --git a/polly/test/ScopInfo/reduction_simple_w_iv.ll b/polly/test/ScopInfo/reduction_simple_w_iv.ll
index 480c2ebf8d47..b6c3229d08d5 100644
--- a/polly/test/ScopInfo/reduction_simple_w_iv.ll
+++ b/polly/test/ScopInfo/reduction_simple_w_iv.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Reduction Type: +
 ;
diff --git a/polly/test/ScopInfo/reduction_two_identical_reads.ll b/polly/test/ScopInfo/reduction_two_identical_reads.ll
index 7fce22d15c77..19d45a5f4ea9 100644
--- a/polly/test/ScopInfo/reduction_two_identical_reads.ll
+++ b/polly/test/ScopInfo/reduction_two_identical_reads.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Reduction Type: NONE
 ;
diff --git a/polly/test/ScopInfo/redundant_parameter_constraint.ll b/polly/test/ScopInfo/redundant_parameter_constraint.ll
index 231cab0fda1f..c9d912191eed 100644
--- a/polly/test/ScopInfo/redundant_parameter_constraint.ll
+++ b/polly/test/ScopInfo/redundant_parameter_constraint.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; The constraint that r2 has to be bigger than r1 is implicitly contained in
 ; the domain, hence we do not want to see it explicitly.
diff --git a/polly/test/ScopInfo/region-with-instructions.ll b/polly/test/ScopInfo/region-with-instructions.ll
index a3040636836f..39d4a72a7814 100644
--- a/polly/test/ScopInfo/region-with-instructions.ll
+++ b/polly/test/ScopInfo/region-with-instructions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -polly-print-instructions -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -polly-print-instructions -disable-output < %s | FileCheck %s
 
 ; CHECK: Statements {
 ; CHECK: 	Stmt_bb46
diff --git a/polly/test/ScopInfo/remarks.ll b/polly/test/ScopInfo/remarks.ll
index 0a6ef2f1e5f7..dcdeb58c7694 100644
--- a/polly/test/ScopInfo/remarks.ll
+++ b/polly/test/ScopInfo/remarks.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops \
 ; RUN: -polly-invariant-load-hoisting=true -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK: remark: test/ScopInfo/remarks.c:4:7: SCoP begins here.
diff --git a/polly/test/ScopInfo/required-invariant-loop-bounds.ll b/polly/test/ScopInfo/required-invariant-loop-bounds.ll
index 19ed625a85db..248acbea6e68 100644
--- a/polly/test/ScopInfo/required-invariant-loop-bounds.ll
+++ b/polly/test/ScopInfo/required-invariant-loop-bounds.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output \
-; RUN: -polly-invariant-load-hoisting=true < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output \
+; RUN: -polly-invariant-load-hoisting=true < %s | FileCheck %s
 ;
 ; CHECK:      Invariant Accesses: {
 ; CHECK-NEXT:       ReadAccess := [Reduction Type: NONE] [Scalar: 0]
diff --git a/polly/test/ScopInfo/restriction_in_dead_block.ll b/polly/test/ScopInfo/restriction_in_dead_block.ll
index 27df53f03b03..81d9b96be419 100644
--- a/polly/test/ScopInfo/restriction_in_dead_block.ll
+++ b/polly/test/ScopInfo/restriction_in_dead_block.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Verify we do not generate an empty invalid context only because the wrap
 ; in the second conditional will always happen if the block is executed.
diff --git a/polly/test/ScopInfo/run-time-check-many-array-disjuncts.ll b/polly/test/ScopInfo/run-time-check-many-array-disjuncts.ll
index e84a1b3e5bc6..d36da2b2becf 100644
--- a/polly/test/ScopInfo/run-time-check-many-array-disjuncts.ll
+++ b/polly/test/ScopInfo/run-time-check-many-array-disjuncts.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s \
 ; RUN: | FileCheck %s -check-prefix=DETECT
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; DETECT: Valid Region for Scop: bb124 => bb176
 ;
diff --git a/polly/test/ScopInfo/run-time-check-many-parameters.ll b/polly/test/ScopInfo/run-time-check-many-parameters.ll
index 540ea57fad0c..30f8d5fff34c 100644
--- a/polly/test/ScopInfo/run-time-check-many-parameters.ll
+++ b/polly/test/ScopInfo/run-time-check-many-parameters.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; A valid Scop would print the list of it's statements, we check that we do not
 ; see that list.
diff --git a/polly/test/ScopInfo/run-time-check-many-piecewise-aliasing.ll b/polly/test/ScopInfo/run-time-check-many-piecewise-aliasing.ll
index cefda1eed0c6..487c803bba98 100644
--- a/polly/test/ScopInfo/run-time-check-many-piecewise-aliasing.ll
+++ b/polly/test/ScopInfo/run-time-check-many-piecewise-aliasing.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>' -disable-output < %s 2>&1 \
+; RUN: opt %loadPolly -polly-print-detect -disable-output < %s \
 ; RUN: | FileCheck %s -check-prefix=DETECT
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; DETECT: Valid Region for Scop: for => return
 ;
diff --git a/polly/test/ScopInfo/run-time-check-read-only-arrays.ll b/polly/test/ScopInfo/run-time-check-read-only-arrays.ll
index 395622b12616..d590aaf00ddb 100644
--- a/polly/test/ScopInfo/run-time-check-read-only-arrays.ll
+++ b/polly/test/ScopInfo/run-time-check-read-only-arrays.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; void foo(float *A, float *B, float *C, long N) {
 ; 	for (long i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/same-base-address-scalar-and-array.ll b/polly/test/ScopInfo/same-base-address-scalar-and-array.ll
index 22cf77636d10..a5f353e7ad2a 100644
--- a/polly/test/ScopInfo/same-base-address-scalar-and-array.ll
+++ b/polly/test/ScopInfo/same-base-address-scalar-and-array.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Verify we introduce two ScopArrayInfo objects (or virtual arrays) for the %out variable
 ; as it is used as a memory base pointer (%0) but also as a scalar (%out.addr.0.lcssa).
diff --git a/polly/test/ScopInfo/scalar.ll b/polly/test/ScopInfo/scalar.ll
index 80493723bbfc..c38eaa853b9b 100644
--- a/polly/test/ScopInfo/scalar.ll
+++ b/polly/test/ScopInfo/scalar.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 
diff --git a/polly/test/ScopInfo/scalar_dependence_cond_br.ll b/polly/test/ScopInfo/scalar_dependence_cond_br.ll
index 940dabbc4cfc..3303bfb7c6c5 100644
--- a/polly/test/ScopInfo/scalar_dependence_cond_br.ll
+++ b/polly/test/ScopInfo/scalar_dependence_cond_br.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output< %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output< %s | FileCheck %s
 ;
 ;    void f(int *A, int c, int d) {
 ;      for (int i = 0; i < 1024; i++)
diff --git a/polly/test/ScopInfo/scalar_to_array.ll b/polly/test/ScopInfo/scalar_to_array.ll
index 692a0dbd67c8..5c275108602a 100644
--- a/polly/test/ScopInfo/scalar_to_array.ll
+++ b/polly/test/ScopInfo/scalar_to_array.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -aa-pipeline=basic-aa '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-print-function-scops -disable-output < %s | FileCheck %s
 
 ; ModuleID = 'scalar_to_array.ll'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/polly/test/ScopInfo/scev-div-with-evaluatable-divisor.ll b/polly/test/ScopInfo/scev-div-with-evaluatable-divisor.ll
index f969176cee16..fc7a1bfc3d5e 100644
--- a/polly/test/ScopInfo/scev-div-with-evaluatable-divisor.ll
+++ b/polly/test/ScopInfo/scev-div-with-evaluatable-divisor.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; Derived from test-suite/SingleSource/UnitTests/Vector/SSE/sse.stepfft.c
 
diff --git a/polly/test/ScopInfo/scev-invalidated.ll b/polly/test/ScopInfo/scev-invalidated.ll
index 921cb06a0cd5..97fc5ec3d4ca 100644
--- a/polly/test/ScopInfo/scev-invalidated.ll
+++ b/polly/test/ScopInfo/scev-invalidated.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Region: %if.then6---%return
 ;
diff --git a/polly/test/ScopInfo/schedule-const-post-dominator-walk-2.ll b/polly/test/ScopInfo/schedule-const-post-dominator-walk-2.ll
index e956b0ed7cb9..2fdf7d66c3ad 100644
--- a/polly/test/ScopInfo/schedule-const-post-dominator-walk-2.ll
+++ b/polly/test/ScopInfo/schedule-const-post-dominator-walk-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/schedule-const-post-dominator-walk.ll b/polly/test/ScopInfo/schedule-const-post-dominator-walk.ll
index 325d19dd210d..92685858610c 100644
--- a/polly/test/ScopInfo/schedule-const-post-dominator-walk.ll
+++ b/polly/test/ScopInfo/schedule-const-post-dominator-walk.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/schedule-constuction-endless-loop1.ll b/polly/test/ScopInfo/schedule-constuction-endless-loop1.ll
index 0225c53faa9a..413d1d8ec556 100644
--- a/polly/test/ScopInfo/schedule-constuction-endless-loop1.ll
+++ b/polly/test/ScopInfo/schedule-constuction-endless-loop1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that we do not build a SCoP and do not crash.
 ;
diff --git a/polly/test/ScopInfo/schedule-constuction-endless-loop2.ll b/polly/test/ScopInfo/schedule-constuction-endless-loop2.ll
index bc34070dcf46..be254477286f 100644
--- a/polly/test/ScopInfo/schedule-constuction-endless-loop2.ll
+++ b/polly/test/ScopInfo/schedule-constuction-endless-loop2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Check that we do not build a SCoP and do not crash.
 ;
diff --git a/polly/test/ScopInfo/schedule-incorrectly-contructed-in-case-of-infinite-loop.ll b/polly/test/ScopInfo/schedule-incorrectly-contructed-in-case-of-infinite-loop.ll
index f8b3dfea844f..ff339e03fb5a 100644
--- a/polly/test/ScopInfo/schedule-incorrectly-contructed-in-case-of-infinite-loop.ll
+++ b/polly/test/ScopInfo/schedule-incorrectly-contructed-in-case-of-infinite-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-process-unprofitable '-passes=print<polly-function-scops>' -disable-output < %s
+; RUN: opt %loadPolly -polly-process-unprofitable -polly-scops -disable-output < %s
 ;
 ; This test contains a infinite loop (bb13) and crashed the domain generation
 ; at some point. Just verify it does not anymore.
diff --git a/polly/test/ScopInfo/scop-affine-parameter-ordering.ll b/polly/test/ScopInfo/scop-affine-parameter-ordering.ll
index 5a8019eabe9d..24c028a6764a 100644
--- a/polly/test/ScopInfo/scop-affine-parameter-ordering.ll
+++ b/polly/test/ScopInfo/scop-affine-parameter-ordering.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-i128:128-n8:16:32:64-S128"
 target triple = "aarch64--linux-android"
 
diff --git a/polly/test/ScopInfo/sign_wrapped_set.ll b/polly/test/ScopInfo/sign_wrapped_set.ll
index 7b24f29563ea..23c9c8a3b84d 100644
--- a/polly/test/ScopInfo/sign_wrapped_set.ll
+++ b/polly/test/ScopInfo/sign_wrapped_set.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-process-unprofitable '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-process-unprofitable -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:        Domain :=
 ; CHECK-NEXT:       [srcHeight] -> { Stmt_for_cond6_preheader_us[i0] : 0 <= i0 <= -3 + srcHeight };
diff --git a/polly/test/ScopInfo/simple_loop_1.ll b/polly/test/ScopInfo/simple_loop_1.ll
index 4872b8e59ba9..2c3481facc02 100644
--- a/polly/test/ScopInfo/simple_loop_1.ll
+++ b/polly/test/ScopInfo/simple_loop_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; void f(int a[], int N) {
 ;   int i;
diff --git a/polly/test/ScopInfo/simple_loop_2.ll b/polly/test/ScopInfo/simple_loop_2.ll
index 120b5e790077..2f580094a147 100644
--- a/polly/test/ScopInfo/simple_loop_2.ll
+++ b/polly/test/ScopInfo/simple_loop_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; void f(int a[], int N) {
 ;   int i;
diff --git a/polly/test/ScopInfo/simple_loop_unsigned.ll b/polly/test/ScopInfo/simple_loop_unsigned.ll
index 6c0e8798a6cd..12903d9c1580 100644
--- a/polly/test/ScopInfo/simple_loop_unsigned.ll
+++ b/polly/test/ScopInfo/simple_loop_unsigned.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; void f(int a[], unsigned N) {
 ;   unsigned i;
diff --git a/polly/test/ScopInfo/simple_loop_unsigned_2.ll b/polly/test/ScopInfo/simple_loop_unsigned_2.ll
index 4b19a8c52c6b..1379180a6dd9 100644
--- a/polly/test/ScopInfo/simple_loop_unsigned_2.ll
+++ b/polly/test/ScopInfo/simple_loop_unsigned_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; CHECK:      Assumed Context:
 ; CHECK-NEXT: [N] -> {  :  }
diff --git a/polly/test/ScopInfo/simple_loop_unsigned_3.ll b/polly/test/ScopInfo/simple_loop_unsigned_3.ll
index fd974f219bec..7783c4681e1f 100644
--- a/polly/test/ScopInfo/simple_loop_unsigned_3.ll
+++ b/polly/test/ScopInfo/simple_loop_unsigned_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; CHECK:      Assumed Context:
 ; CHECK-NEXT: [N] -> { : }
diff --git a/polly/test/ScopInfo/simple_nonaffine_loop_not.ll b/polly/test/ScopInfo/simple_nonaffine_loop_not.ll
index d2aa22f8cca7..42eff85d8c9b 100644
--- a/polly/test/ScopInfo/simple_nonaffine_loop_not.ll
+++ b/polly/test/ScopInfo/simple_nonaffine_loop_not.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | not FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | not FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 @.str = private unnamed_addr constant [17 x i8] c"Random Value: %d\00", align 1
diff --git a/polly/test/ScopInfo/smax.ll b/polly/test/ScopInfo/smax.ll
index 502d52baaaef..b938e4e412da 100644
--- a/polly/test/ScopInfo/smax.ll
+++ b/polly/test/ScopInfo/smax.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:64:128-a0:0:32-n32-S64"
 
 define void @foo(ptr noalias %data, ptr noalias %ptr, i32 %x_pos, i32 %w) {
diff --git a/polly/test/ScopInfo/statistics.ll b/polly/test/ScopInfo/statistics.ll
index c69852e21875..3797b7d71df9 100644
--- a/polly/test/ScopInfo/statistics.ll
+++ b/polly/test/ScopInfo/statistics.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -stats -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-scops -stats -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; CHECK-DAG:  4 polly-scops      - Maximal number of loops in scops
diff --git a/polly/test/ScopInfo/stmt_split_exit_of_region_stmt.ll b/polly/test/ScopInfo/stmt_split_exit_of_region_stmt.ll
index 1d4d3f19b571..d86d2418cf9b 100644
--- a/polly/test/ScopInfo/stmt_split_exit_of_region_stmt.ll
+++ b/polly/test/ScopInfo/stmt_split_exit_of_region_stmt.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:   Stmt_Region__TO__Stmt
diff --git a/polly/test/ScopInfo/stmt_split_no_after_split.ll b/polly/test/ScopInfo/stmt_split_no_after_split.ll
index e3e440584f25..f8339bd8ae94 100644
--- a/polly/test/ScopInfo/stmt_split_no_after_split.ll
+++ b/polly/test/ScopInfo/stmt_split_no_after_split.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT: 	Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_split_no_dependence.ll b/polly/test/ScopInfo/stmt_split_no_dependence.ll
index 0bf98c9b70be..7ad48f499792 100644
--- a/polly/test/ScopInfo/stmt_split_no_dependence.ll
+++ b/polly/test/ScopInfo/stmt_split_no_dependence.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
 
 ;      void func(int *A, int *B){
 ;        for (int i = 0; i < 1024; i+=1) {
diff --git a/polly/test/ScopInfo/stmt_split_on_store.ll b/polly/test/ScopInfo/stmt_split_on_store.ll
index 82b1f5bbc3cd..6af3dc8633dd 100644
--- a/polly/test/ScopInfo/stmt_split_on_store.ll
+++ b/polly/test/ScopInfo/stmt_split_on_store.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=store -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=store -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
 
 ;      void func(int *A, int *B){
 ;        for (int i = 0; i < 1024; i+=1) {
diff --git a/polly/test/ScopInfo/stmt_split_on_synthesizable.ll b/polly/test/ScopInfo/stmt_split_on_synthesizable.ll
index 323c83bc570e..92855cfd0124 100644
--- a/polly/test/ScopInfo/stmt_split_on_synthesizable.ll
+++ b/polly/test/ScopInfo/stmt_split_on_synthesizable.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:   Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_split_phi_in_beginning_bb.ll b/polly/test/ScopInfo/stmt_split_phi_in_beginning_bb.ll
index 7f72e672a1a6..ee6afa4638d2 100644
--- a/polly/test/ScopInfo/stmt_split_phi_in_beginning_bb.ll
+++ b/polly/test/ScopInfo/stmt_split_phi_in_beginning_bb.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:  	Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_split_phi_in_stmt.ll b/polly/test/ScopInfo/stmt_split_phi_in_stmt.ll
index 9306cdc7615a..0a5f41d637e7 100644
--- a/polly/test/ScopInfo/stmt_split_phi_in_stmt.ll
+++ b/polly/test/ScopInfo/stmt_split_phi_in_stmt.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:  	Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_split_scalar_dependence.ll b/polly/test/ScopInfo/stmt_split_scalar_dependence.ll
index efd5cf14def0..5b02d1b5d08a 100644
--- a/polly/test/ScopInfo/stmt_split_scalar_dependence.ll
+++ b/polly/test/ScopInfo/stmt_split_scalar_dependence.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:  	Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_split_within_loop.ll b/polly/test/ScopInfo/stmt_split_within_loop.ll
index f24904df307c..3ed9bbbeaccb 100644
--- a/polly/test/ScopInfo/stmt_split_within_loop.ll
+++ b/polly/test/ScopInfo/stmt_split_within_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-instructions -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    Statements {
 ; CHECK-NEXT:  	Stmt_Stmt
diff --git a/polly/test/ScopInfo/stmt_with_read_but_without_sideffect.ll b/polly/test/ScopInfo/stmt_with_read_but_without_sideffect.ll
index 41f58844569f..73fc543a66e8 100644
--- a/polly/test/ScopInfo/stmt_with_read_but_without_sideffect.ll
+++ b/polly/test/ScopInfo/stmt_with_read_but_without_sideffect.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-delicm>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-delicm -disable-output < %s | FileCheck %s
 ;
 ; The statement Stmt_for_if_else_1 should be removed because it has no
 ; sideeffects.  But it has a use of MemRef_tmp21 that must also be
diff --git a/polly/test/ScopInfo/switch-1.ll b/polly/test/ScopInfo/switch-1.ll
index 6bc630834e93..0ea40a7ed251 100644
--- a/polly/test/ScopInfo/switch-1.ll
+++ b/polly/test/ScopInfo/switch-1.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/switch-2.ll b/polly/test/ScopInfo/switch-2.ll
index a64d133baae7..7956058c9de6 100644
--- a/polly/test/ScopInfo/switch-2.ll
+++ b/polly/test/ScopInfo/switch-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/switch-3.ll b/polly/test/ScopInfo/switch-3.ll
index 3aa2d7811c77..aa7ada4edbb8 100644
--- a/polly/test/ScopInfo/switch-3.ll
+++ b/polly/test/ScopInfo/switch-3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/switch-4.ll b/polly/test/ScopInfo/switch-4.ll
index 567c3de030ea..6aeb7197e382 100644
--- a/polly/test/ScopInfo/switch-4.ll
+++ b/polly/test/ScopInfo/switch-4.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/switch-5.ll b/polly/test/ScopInfo/switch-5.ll
index b6a42d9da749..24cc92a0933d 100644
--- a/polly/test/ScopInfo/switch-5.ll
+++ b/polly/test/ScopInfo/switch-5.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
 ;
 ; The SCoP contains a loop with multiple exit blocks (BBs after leaving
 ; the loop). The current implementation of deriving their domain derives
diff --git a/polly/test/ScopInfo/switch-6.ll b/polly/test/ScopInfo/switch-6.ll
index 24538328c581..efb3df504d23 100644
--- a/polly/test/ScopInfo/switch-6.ll
+++ b/polly/test/ScopInfo/switch-6.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int N) {
 ;      for (int i = 0; i < N; i++) {
diff --git a/polly/test/ScopInfo/switch-7.ll b/polly/test/ScopInfo/switch-7.ll
index 99c1bed81874..2f0d034e84fe 100644
--- a/polly/test/ScopInfo/switch-7.ll
+++ b/polly/test/ScopInfo/switch-7.ll
@@ -1,5 +1,6 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=AST
+
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-ast -disable-output < %s | FileCheck %s --check-prefix=AST
 ;
 ;    void f(int *A, int c, int N) {
 ;      switch (c) {
diff --git a/polly/test/ScopInfo/tempscop-printing.ll b/polly/test/ScopInfo/tempscop-printing.ll
index e99a6f2582ee..80c675d4c3d3 100644
--- a/polly/test/ScopInfo/tempscop-printing.ll
+++ b/polly/test/ScopInfo/tempscop-printing.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -aa-pipeline=basic-aa -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -basic-aa -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; void f(long A[], int N, int *init_ptr) {
 ;   long i, j;
diff --git a/polly/test/ScopInfo/test-wrapping-in-condition.ll b/polly/test/ScopInfo/test-wrapping-in-condition.ll
index 7c1301748c39..3ff978f7265e 100644
--- a/polly/test/ScopInfo/test-wrapping-in-condition.ll
+++ b/polly/test/ScopInfo/test-wrapping-in-condition.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-function-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    Invalid Context:
 ; CHECK:        [N] -> {  : N >= 129 }
diff --git a/polly/test/ScopInfo/truncate-1.ll b/polly/test/ScopInfo/truncate-1.ll
index b21755c67ac4..5c5fac150b4b 100644
--- a/polly/test/ScopInfo/truncate-1.ll
+++ b/polly/test/ScopInfo/truncate-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(char *A, short N) {
 ;      for (char i = 0; i < (char)N; i++)
diff --git a/polly/test/ScopInfo/truncate-2.ll b/polly/test/ScopInfo/truncate-2.ll
index 0d4abb343993..e6c5f2cb32d0 100644
--- a/polly/test/ScopInfo/truncate-2.ll
+++ b/polly/test/ScopInfo/truncate-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(char *A, short N) {
 ;      for (short i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/truncate-3.ll b/polly/test/ScopInfo/truncate-3.ll
index f9cdd0274f22..dd0fe489e990 100644
--- a/polly/test/ScopInfo/truncate-3.ll
+++ b/polly/test/ScopInfo/truncate-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -pass-remarks-analysis="polly-scops" \
+; RUN: opt %loadPolly -polly-scops -pass-remarks-analysis="polly-scops" \
 ; RUN:                -disable-output < %s 2>&1 | FileCheck %s
 
 ; CHECK: Signed-unsigned restriction: [p] -> {  : p <= -129 or p >= 128 }
diff --git a/polly/test/ScopInfo/two-loops-one-infinite.ll b/polly/test/ScopInfo/two-loops-one-infinite.ll
index 02ad18e3d567..71f72383b048 100644
--- a/polly/test/ScopInfo/two-loops-one-infinite.ll
+++ b/polly/test/ScopInfo/two-loops-one-infinite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Verify we do not create a SCoP in the presence of infinite loops.
 ;
diff --git a/polly/test/ScopInfo/two-loops-right-after-each-other.ll b/polly/test/ScopInfo/two-loops-right-after-each-other.ll
index 36ab13a68c1e..dd457c31afdd 100644
--- a/polly/test/ScopInfo/two-loops-right-after-each-other.ll
+++ b/polly/test/ScopInfo/two-loops-right-after-each-other.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; CHECK:      Statements {
 ; CHECK-NEXT:     Stmt_loop_1
diff --git a/polly/test/ScopInfo/undef_in_cond.ll b/polly/test/ScopInfo/undef_in_cond.ll
index 4bdcc1697068..5282a853c17a 100644
--- a/polly/test/ScopInfo/undef_in_cond.ll
+++ b/polly/test/ScopInfo/undef_in_cond.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define fastcc void @fix_operands() nounwind {
diff --git a/polly/test/ScopInfo/unnamed_nonaffine.ll b/polly/test/ScopInfo/unnamed_nonaffine.ll
index d9415eabab94..bf32cc7806f4 100644
--- a/polly/test/ScopInfo/unnamed_nonaffine.ll
+++ b/polly/test/ScopInfo/unnamed_nonaffine.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-use-llvm-names=true  '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-allow-nonaffine -polly-use-llvm-names=false '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -check-prefix=UNNAMED
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-use-llvm-names=true  -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-allow-nonaffine -polly-use-llvm-names=false -polly-print-scops -disable-output < %s | FileCheck %s -check-prefix=UNNAMED
 ;
 ;    void f(int *A, int b) {
 ;      int x;
diff --git a/polly/test/ScopInfo/unnamed_stmts.ll b/polly/test/ScopInfo/unnamed_stmts.ll
index 0bd53d8c8425..686c0f87d9cf 100644
--- a/polly/test/ScopInfo/unnamed_stmts.ll
+++ b/polly/test/ScopInfo/unnamed_stmts.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; This test case verifies that we generate numbered statement names in case
 ; no LLVM-IR names are used in the test case. We also verify, that we
diff --git a/polly/test/ScopInfo/unpredictable_nonscop_loop.ll b/polly/test/ScopInfo/unpredictable_nonscop_loop.ll
index c0e768216eb0..0656b77e3409 100644
--- a/polly/test/ScopInfo/unpredictable_nonscop_loop.ll
+++ b/polly/test/ScopInfo/unpredictable_nonscop_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -disable-output < %s | FileCheck %s -match-full-lines
 ; Derived from test-suite/MultiSource/Applications/sgefa/blas.c
 ;
 ; The exit value of %i.0320 in land.rhs is not computable.
diff --git a/polly/test/ScopInfo/unprofitable_scalar-accs.ll b/polly/test/ScopInfo/unprofitable_scalar-accs.ll
index e7c8a57093b8..9703587091a7 100644
--- a/polly/test/ScopInfo/unprofitable_scalar-accs.ll
+++ b/polly/test/ScopInfo/unprofitable_scalar-accs.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=false '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=true  '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=HEURISTIC
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=false -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-process-unprofitable=false -polly-unprofitable-scalar-accs=true  -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=HEURISTIC
 
 ; Check the effect of -polly-unprofitable-scalar-accs
 
diff --git a/polly/test/ScopInfo/unsigned-condition.ll b/polly/test/ScopInfo/unsigned-condition.ll
index 1dca9bab41ec..35673d1b6a36 100644
--- a/polly/test/ScopInfo/unsigned-condition.ll
+++ b/polly/test/ScopInfo/unsigned-condition.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; void f(int a[], int N, unsigned P) {
 ;   int i;
diff --git a/polly/test/ScopInfo/unsigned-division-1.ll b/polly/test/ScopInfo/unsigned-division-1.ll
index da080b3a306b..8c65062bd941 100644
--- a/polly/test/ScopInfo/unsigned-division-1.ll
+++ b/polly/test/ScopInfo/unsigned-division-1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, unsigned N) {
 ;      for (unsigned i = 0; i < N / 2; i++)
diff --git a/polly/test/ScopInfo/unsigned-division-2.ll b/polly/test/ScopInfo/unsigned-division-2.ll
index 2fe4207d1bd1..bf4ebce9099a 100644
--- a/polly/test/ScopInfo/unsigned-division-2.ll
+++ b/polly/test/ScopInfo/unsigned-division-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, unsigned N) {
 ;      for (unsigned i = 0; i < N / 2 + 3; i++)
diff --git a/polly/test/ScopInfo/unsigned-division-3.ll b/polly/test/ScopInfo/unsigned-division-3.ll
index aefb590b28df..47ba1f2ef09d 100644
--- a/polly/test/ScopInfo/unsigned-division-3.ll
+++ b/polly/test/ScopInfo/unsigned-division-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, unsigned char N) {
 ;      for (unsigned i = 0; i <= N / -128; i++)
diff --git a/polly/test/ScopInfo/unsigned-division-4.ll b/polly/test/ScopInfo/unsigned-division-4.ll
index 9fe10d7440ef..edcd8a18a854 100644
--- a/polly/test/ScopInfo/unsigned-division-4.ll
+++ b/polly/test/ScopInfo/unsigned-division-4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, unsigned char N) {
 ;      for (unsigned i = 0; i < (N / -128) + 3; i++)
diff --git a/polly/test/ScopInfo/unsigned-division-5.ll b/polly/test/ScopInfo/unsigned-division-5.ll
index fb90345f477e..f9a3d39288a9 100644
--- a/polly/test/ScopInfo/unsigned-division-5.ll
+++ b/polly/test/ScopInfo/unsigned-division-5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, unsigned N) {
 ;      for (unsigned i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/unsigned_wrap_uge.ll b/polly/test/ScopInfo/unsigned_wrap_uge.ll
index 3d54cad70285..89c50ee3764b 100644
--- a/polly/test/ScopInfo/unsigned_wrap_uge.ll
+++ b/polly/test/ScopInfo/unsigned_wrap_uge.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Unsigned wrap-around check.
 ;
diff --git a/polly/test/ScopInfo/unsigned_wrap_ugt.ll b/polly/test/ScopInfo/unsigned_wrap_ugt.ll
index 8c98f13cfb72..3249123c9918 100644
--- a/polly/test/ScopInfo/unsigned_wrap_ugt.ll
+++ b/polly/test/ScopInfo/unsigned_wrap_ugt.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Unsigned wrap-around check.
 ;
diff --git a/polly/test/ScopInfo/unsigned_wrap_ule.ll b/polly/test/ScopInfo/unsigned_wrap_ule.ll
index e0b0339475fc..3c6ea18b439c 100644
--- a/polly/test/ScopInfo/unsigned_wrap_ule.ll
+++ b/polly/test/ScopInfo/unsigned_wrap_ule.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Unsigned wrap-around check.
 ;
diff --git a/polly/test/ScopInfo/unsigned_wrap_ult.ll b/polly/test/ScopInfo/unsigned_wrap_ult.ll
index cb15bc04669e..5d859f85d52b 100644
--- a/polly/test/ScopInfo/unsigned_wrap_ult.ll
+++ b/polly/test/ScopInfo/unsigned_wrap_ult.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; Unsigned wrap-around check.
 ;
diff --git a/polly/test/ScopInfo/user_context.ll b/polly/test/ScopInfo/user_context.ll
index d67244e1ad95..46232cd59c03 100644
--- a/polly/test/ScopInfo/user_context.ll
+++ b/polly/test/ScopInfo/user_context.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly                                      '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-context='[N] -> {: N = 1024}' '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CTX
-; RUN: opt %loadPolly -polly-context='[N,M] -> {: 1 = 0}'  '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-context='[] -> {: 1 = 0}'     '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly                                      -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-context='[N] -> {: N = 1024}' -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=CTX
+; RUN: opt %loadPolly -polly-context='[N,M] -> {: 1 = 0}'  -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-context='[] -> {: 1 = 0}'     -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; void f(int a[], int N) {
 ;   int i;
diff --git a/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed-conditional.ll b/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed-conditional.ll
index 829d1ef10664..4bd02c96a3d2 100644
--- a/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed-conditional.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed-conditional.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; REMARK: remark: <unknown>:0:0: Use user assumption: [n, b] -> {  : n <= 100 or (b = 0 and n >= 101) }
 ;
diff --git a/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed.ll b/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed.ll
index 8518a0ece23c..262bd1349a69 100644
--- a/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions-in-bb-signed.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK: Context:
 ; CHECK-NEXT: [n] -> {  : -9223372036854775808 <= n <= 100 }
diff --git a/polly/test/ScopInfo/user_provided_assumptions-in-bb-unsigned.ll b/polly/test/ScopInfo/user_provided_assumptions-in-bb-unsigned.ll
index 678be1c06d0a..4a10fcff929a 100644
--- a/polly/test/ScopInfo/user_provided_assumptions-in-bb-unsigned.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions-in-bb-unsigned.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops -disable-output < %s 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; REMARK:      remark: <unknown>:0:0: SCoP begins here.
 ; REMARK-NEXT: remark: <unknown>:0:0: Use user assumption: [n] -> {  : n <= 100 }
diff --git a/polly/test/ScopInfo/user_provided_assumptions.ll b/polly/test/ScopInfo/user_provided_assumptions.ll
index e4556eb3a386..6640e4a65e36 100644
--- a/polly/test/ScopInfo/user_provided_assumptions.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=SCOP
 ;
 ; CHECK:      remark: <unknown>:0:0: SCoP begins here.
 ; CHECK-NEXT: remark: <unknown>:0:0: Use user assumption: [M, N] -> {  : N <= 2147483647 - M }
diff --git a/polly/test/ScopInfo/user_provided_assumptions_2.ll b/polly/test/ScopInfo/user_provided_assumptions_2.ll
index 98057740eab3..994cd6f15103 100644
--- a/polly/test/ScopInfo/user_provided_assumptions_2.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions_2.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=SCOP
 ;
 ; CHECK:      remark: <unknown>:0:0: SCoP begins here.
 ; CHECK-NEXT: remark: <unknown>:0:0: Use user assumption: { : }
diff --git a/polly/test/ScopInfo/user_provided_assumptions_3.ll b/polly/test/ScopInfo/user_provided_assumptions_3.ll
index de3fbba46e0a..2fcde8bd1826 100644
--- a/polly/test/ScopInfo/user_provided_assumptions_3.ll
+++ b/polly/test/ScopInfo/user_provided_assumptions_3.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SCOP
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s --check-prefix=SCOP
 ;
 ; CHECK:      remark: <unknown>:0:0: SCoP begins here.
 ; CHECK-NEXT: remark: <unknown>:0:0: Use user assumption: [N] -> { : N >= 2 }
diff --git a/polly/test/ScopInfo/user_provided_non_dominating_assumptions.ll b/polly/test/ScopInfo/user_provided_non_dominating_assumptions.ll
index 4f3be408c9ef..1eb3c15810e4 100644
--- a/polly/test/ScopInfo/user_provided_non_dominating_assumptions.ll
+++ b/polly/test/ScopInfo/user_provided_non_dominating_assumptions.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops \
 ; RUN:    -polly-precise-inbounds -disable-output < %s 2>&1 | FileCheck %s
 ;
 ; CHECK:      remark: <unknown>:0:0: SCoP begins here.
@@ -18,7 +18,7 @@
 ;
 
 
-; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" '-passes=print<polly-function-scops>' \
+; RUN: opt %loadPolly -pass-remarks-analysis="polly-scops" -polly-scops \
 ; RUN:    -polly-precise-inbounds -disable-output < %s 2>&1 -pass-remarks-output=%t.yaml
 ; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
 ; YAML: --- !Analysis
diff --git a/polly/test/ScopInfo/variant_base_pointer.ll b/polly/test/ScopInfo/variant_base_pointer.ll
index 3a6ea88d3473..321657c87e79 100644
--- a/polly/test/ScopInfo/variant_base_pointer.ll
+++ b/polly/test/ScopInfo/variant_base_pointer.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true '-passes=print<polly-detect>,print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
-; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true -passes=polly-codegen -disable-output < %s
+; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-ignore-aliasing -polly-invariant-load-hoisting=true -polly-codegen -disable-output < %s
 ;
 ; %tmp is added to the list of required hoists by -polly-scops and just
 ; assumed to be hoisted. Only -polly-scops recognizes it to be unhoistable
diff --git a/polly/test/ScopInfo/variant_load_empty_domain.ll b/polly/test/ScopInfo/variant_load_empty_domain.ll
index 4b91778a225b..0e685c3c7e73 100644
--- a/polly/test/ScopInfo/variant_load_empty_domain.ll
+++ b/polly/test/ScopInfo/variant_load_empty_domain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:      Invariant Accesses: {
 ; CHECK-NEXT: }
diff --git a/polly/test/ScopInfo/wraping_signed_expr_0.ll b/polly/test/ScopInfo/wraping_signed_expr_0.ll
index bbb49bffe925..7ad0f64028b6 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_0.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_0.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, char N, char p) {
 ;      for (char i = 0; i < N; i++) {
diff --git a/polly/test/ScopInfo/wraping_signed_expr_1.ll b/polly/test/ScopInfo/wraping_signed_expr_1.ll
index e43a691312a0..0a62b9cf542c 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_1.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(long *A, long N, long p) {
 ;      for (long i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/wraping_signed_expr_2.ll b/polly/test/ScopInfo/wraping_signed_expr_2.ll
index eef357acc582..f3b4665f7f37 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_2.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, int N, int p) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/wraping_signed_expr_3.ll b/polly/test/ScopInfo/wraping_signed_expr_3.ll
index a0500eb48941..7a5cbba9436b 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_3.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_3.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(int *A, int N, int p) {
 ;      for (int i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/wraping_signed_expr_4.ll b/polly/test/ScopInfo/wraping_signed_expr_4.ll
index d21f321e4ac9..ec65f70a092f 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_4.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_4.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(char *A, char N, char p) {
 ;      for (char i = 0; i < N; i++)
diff --git a/polly/test/ScopInfo/wraping_signed_expr_5.ll b/polly/test/ScopInfo/wraping_signed_expr_5.ll
index 395342d2f55a..5f3b09ba33c1 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_5.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_5.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; We should not generate runtime check for ((int)r1 + (int)r2) as it is known not
 ; to overflow. However (p + q) can, thus checks are needed.
diff --git a/polly/test/ScopInfo/wraping_signed_expr_6.ll b/polly/test/ScopInfo/wraping_signed_expr_6.ll
index 4147f7fa20cf..23258bb513bf 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_6.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_6.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    Invalid Context:
 ; CHECK:        [N] -> {  : N >= 129 }
diff --git a/polly/test/ScopInfo/wraping_signed_expr_7.ll b/polly/test/ScopInfo/wraping_signed_expr_7.ll
index f41e89c07db0..0663d4e0bc10 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_7.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_7.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:    Invalid Context:
 ; CHECK:        [N] -> {  : N >= 129 }
diff --git a/polly/test/ScopInfo/wraping_signed_expr_slow_1.ll b/polly/test/ScopInfo/wraping_signed_expr_slow_1.ll
index ddaeed06874a..ec36d2c5fcde 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_slow_1.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_slow_1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; This checks that the no-wraps checks will be computed fast as some example
 ; already showed huge slowdowns even though the inbounds and nsw flags were
diff --git a/polly/test/ScopInfo/wraping_signed_expr_slow_2.ll b/polly/test/ScopInfo/wraping_signed_expr_slow_2.ll
index 798a5d0855b3..6db33ab166d5 100644
--- a/polly/test/ScopInfo/wraping_signed_expr_slow_2.ll
+++ b/polly/test/ScopInfo/wraping_signed_expr_slow_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; This checks that the no-wraps checks will be computed fast as some example
 ; already showed huge slowdowns even though the inbounds and nsw flags were
diff --git a/polly/test/ScopInfo/zero_ext_of_truncate.ll b/polly/test/ScopInfo/zero_ext_of_truncate.ll
index bf5b6354a6d6..fc55df5e053c 100644
--- a/polly/test/ScopInfo/zero_ext_of_truncate.ll
+++ b/polly/test/ScopInfo/zero_ext_of_truncate.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(unsigned *restrict I, unsigned *restrict A, unsigned N, unsigned M) {
 ;      for (unsigned i = 0; i < N; i++) {
diff --git a/polly/test/ScopInfo/zero_ext_of_truncate_2.ll b/polly/test/ScopInfo/zero_ext_of_truncate_2.ll
index 595b21c71869..13e9c03ecd2d 100644
--- a/polly/test/ScopInfo/zero_ext_of_truncate_2.ll
+++ b/polly/test/ScopInfo/zero_ext_of_truncate_2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-invariant-load-hoisting=true '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-invariant-load-hoisting=true -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ;    void f(unsigned long *restrict I, unsigned *restrict A, unsigned N) {
 ;      for (unsigned i = 0; i < N; i++) {
diff --git a/polly/test/ScopInfo/zero_ext_space_mismatch.ll b/polly/test/ScopInfo/zero_ext_space_mismatch.ll
index 0a329fdef8c2..835a8664b75e 100644
--- a/polly/test/ScopInfo/zero_ext_space_mismatch.ll
+++ b/polly/test/ScopInfo/zero_ext_space_mismatch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-print-scops -disable-output < %s | FileCheck %s
 ;
 ; CHECK:         Assumed Context:
 ; CHECK-NEXT:    [dim] -> {  : dim > 0 }
diff --git a/polly/test/ScopInliner/invariant-load-func.ll b/polly/test/ScopInliner/invariant-load-func.ll
index 8da50f90beba..38e4a15aab94 100644
--- a/polly/test/ScopInliner/invariant-load-func.ll
+++ b/polly/test/ScopInliner/invariant-load-func.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %loadPolly -polly-detect-full-functions -polly-scop-inliner \
-; RUN: -polly-invariant-load-hoisting '-passes=print<polly-function-scops>' -disable-output < %s | FileCheck %s
+; RUN: -polly-invariant-load-hoisting -polly-print-scops -disable-output < %s | FileCheck %s
 
 ; Check that we inline a function that requires invariant load hoisting
 ; correctly.
diff --git a/polly/test/Simplify/coalesce_3partials.ll b/polly/test/Simplify/coalesce_3partials.ll
index 937f655c344c..0c1556ff263a 100644
--- a/polly/test/Simplify/coalesce_3partials.ll
+++ b/polly/test/Simplify/coalesce_3partials.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Combine 3 partial accesses into one.
 ;
diff --git a/polly/test/Simplify/coalesce_disjointelements.ll b/polly/test/Simplify/coalesce_disjointelements.ll
index 6080ee4dde81..2f4cf4e3f920 100644
--- a/polly/test/Simplify/coalesce_disjointelements.ll
+++ b/polly/test/Simplify/coalesce_disjointelements.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Combine four partial stores into two.
 ; The stores write to the same array, but never the same element.
diff --git a/polly/test/Simplify/coalesce_overlapping.ll b/polly/test/Simplify/coalesce_overlapping.ll
index 3c52d44e8003..78ed21e9855b 100644
--- a/polly/test/Simplify/coalesce_overlapping.ll
+++ b/polly/test/Simplify/coalesce_overlapping.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Combine two partial stores (with overlapping domains) into one.
 ;
diff --git a/polly/test/Simplify/coalesce_partial.ll b/polly/test/Simplify/coalesce_partial.ll
index cec58a9121b2..c42aaa113035 100644
--- a/polly/test/Simplify/coalesce_partial.ll
+++ b/polly/test/Simplify/coalesce_partial.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Combine two partial stores (with disjoint domains) into one.
 ;
diff --git a/polly/test/Simplify/dead_access_load.ll b/polly/test/Simplify/dead_access_load.ll
index 5e0a9b574516..1804613c0a79 100644
--- a/polly/test/Simplify/dead_access_load.ll
+++ b/polly/test/Simplify/dead_access_load.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Remove a dead load-instruction
 ; (an load whose result is not used anywhere)
diff --git a/polly/test/Simplify/dead_access_phi.ll b/polly/test/Simplify/dead_access_phi.ll
index 6044f7f50be7..d263b89aff58 100644
--- a/polly/test/Simplify/dead_access_phi.ll
+++ b/polly/test/Simplify/dead_access_phi.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Remove a dead PHI write/read pair
 ; (accesses that are effectively not used)
diff --git a/polly/test/Simplify/dead_access_value.ll b/polly/test/Simplify/dead_access_value.ll
index a3b9d5ebe76a..6e3c211577f6 100644
--- a/polly/test/Simplify/dead_access_value.ll
+++ b/polly/test/Simplify/dead_access_value.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Remove a dead value write/read pair
 ; (accesses that are effectively not used)
diff --git a/polly/test/Simplify/dead_instruction.ll b/polly/test/Simplify/dead_instruction.ll
index 2bf7f8571a46..4e693b0ccb44 100644
--- a/polly/test/Simplify/dead_instruction.ll
+++ b/polly/test/Simplify/dead_instruction.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Remove a dead instruction
 ; (an instruction whose result is not used anywhere)
diff --git a/polly/test/Simplify/emptyaccessdomain.ll b/polly/test/Simplify/emptyaccessdomain.ll
index bf6d4d9dc8bd..54ac14ab398c 100644
--- a/polly/test/Simplify/emptyaccessdomain.ll
+++ b/polly/test/Simplify/emptyaccessdomain.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; for (int j = 0; j < n; j += 1) {
 ;   A[0] = 42.0;
diff --git a/polly/test/Simplify/exit_phi_accesses-2.ll b/polly/test/Simplify/exit_phi_accesses-2.ll
index 2116d8008aec..01748aa59bd3 100644
--- a/polly/test/Simplify/exit_phi_accesses-2.ll
+++ b/polly/test/Simplify/exit_phi_accesses-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>,scop(print<polly-simplify>)' -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-scops -polly-print-simplify -disable-output < %s | FileCheck %s
 ;
 ; The use of %sum.next by %phi counts as an escaping use.
 ; Don't remove the scalar write of %sum.next.
diff --git a/polly/test/Simplify/func-b320a7.ll b/polly/test/Simplify/func-b320a7.ll
index c5afc37eb7de..c8a823a468d7 100644
--- a/polly/test/Simplify/func-b320a7.ll
+++ b/polly/test/Simplify/func-b320a7.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-simplify>,polly-optree' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-simplify -polly-optree -disable-output < %s | FileCheck %s -match-full-lines
 
 ; llvm.org/PR47098
 ; Use-after-free by reference to Stmt remaining in InstStmtMap after removing it has been removed by Scop::simplifyScop.
diff --git a/polly/test/Simplify/gemm.ll b/polly/test/Simplify/gemm.ll
index 4074078742fc..23f8de5573cd 100644
--- a/polly/test/Simplify/gemm.ll
+++ b/polly/test/Simplify/gemm.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s
 ;
 ;    void gemm(float A[][1024], float B[][1024], float C[][1024]) {
 ;      for (long i = 0; i < 1024; i++)
diff --git a/polly/test/Simplify/nocoalesce_differentvalues.ll b/polly/test/Simplify/nocoalesce_differentvalues.ll
index d08c80ee0c06..68991d2eecf5 100644
--- a/polly/test/Simplify/nocoalesce_differentvalues.ll
+++ b/polly/test/Simplify/nocoalesce_differentvalues.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Do not combine stores that write different values.
 ;
diff --git a/polly/test/Simplify/nocoalesce_elementmismatch.ll b/polly/test/Simplify/nocoalesce_elementmismatch.ll
index af12e611fdbc..2bab360e6858 100644
--- a/polly/test/Simplify/nocoalesce_elementmismatch.ll
+++ b/polly/test/Simplify/nocoalesce_elementmismatch.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Do not combine stores that do not write to different elements in the
 ; same instance.
diff --git a/polly/test/Simplify/nocoalesce_readbetween.ll b/polly/test/Simplify/nocoalesce_readbetween.ll
index 1a71d2da4c1f..ada79dc18b87 100644
--- a/polly/test/Simplify/nocoalesce_readbetween.ll
+++ b/polly/test/Simplify/nocoalesce_readbetween.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Do not combine stores if there is a read between them.
 ; Note: The read between is unused, so will be removed by markAndSweep.
diff --git a/polly/test/Simplify/nocoalesce_writebetween.ll b/polly/test/Simplify/nocoalesce_writebetween.ll
index bc2c47a4c9ec..48e785ec2c26 100644
--- a/polly/test/Simplify/nocoalesce_writebetween.ll
+++ b/polly/test/Simplify/nocoalesce_writebetween.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Do not combine stores if there is a write between them.
 ;
diff --git a/polly/test/Simplify/notdead_region_exitphi.ll b/polly/test/Simplify/notdead_region_exitphi.ll
index a796f2a419ad..bd29fd578b97 100644
--- a/polly/test/Simplify/notdead_region_exitphi.ll
+++ b/polly/test/Simplify/notdead_region_exitphi.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Do not remove dependencies of a phi node in a region's exit block.
 ;
diff --git a/polly/test/Simplify/notdead_region_innerphi.ll b/polly/test/Simplify/notdead_region_innerphi.ll
index c76485cb5019..a176a28af233 100644
--- a/polly/test/Simplify/notdead_region_innerphi.ll
+++ b/polly/test/Simplify/notdead_region_innerphi.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Do not remove dependencies of a phi node within a region statement (%phi).
 ;
diff --git a/polly/test/Simplify/notredundant_region_loop.ll b/polly/test/Simplify/notredundant_region_loop.ll
index 0bf0dd531524..0ea9be7e9d2d 100644
--- a/polly/test/Simplify/notredundant_region_loop.ll
+++ b/polly/test/Simplify/notredundant_region_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -polly-allow-nonaffine-loops -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-allow-nonaffine-loops -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Do not remove the store in region_entry. It can be executed multiple times
 ; due to being part of a non-affine loop.
diff --git a/polly/test/Simplify/notredundant_region_middle.ll b/polly/test/Simplify/notredundant_region_middle.ll
index 392dd48ae985..84598746e0bb 100644
--- a/polly/test/Simplify/notredundant_region_middle.ll
+++ b/polly/test/Simplify/notredundant_region_middle.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Do not remove redundant stores in the middle of region statements.
 ; The store in region_true could be removed, but in practice we do try to
diff --git a/polly/test/Simplify/notredundant_synthesizable_unknownit.ll b/polly/test/Simplify/notredundant_synthesizable_unknownit.ll
index d522d5dd68ae..2affdbb2f1de 100644
--- a/polly/test/Simplify/notredundant_synthesizable_unknownit.ll
+++ b/polly/test/Simplify/notredundant_synthesizable_unknownit.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Do not remove the scalar value write of %i.trunc in inner.for.
 ; It is used by body.
diff --git a/polly/test/Simplify/out-of-scop-use-in-region-entry-phi-node.ll b/polly/test/Simplify/out-of-scop-use-in-region-entry-phi-node.ll
index fe57a0ef6c9d..511f35a9388e 100644
--- a/polly/test/Simplify/out-of-scop-use-in-region-entry-phi-node.ll
+++ b/polly/test/Simplify/out-of-scop-use-in-region-entry-phi-node.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb '-passes=print<polly-function-scops>,scop(print<polly-simplify>)' -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-scops -polly-print-simplify -disable-output < %s | FileCheck %s
 ;
 ; %tmp5 must keep the Value WRITE MemoryAccess, because as an incoming value of
 ; %tmp4, it is an "external use".
diff --git a/polly/test/Simplify/overwritten.ll b/polly/test/Simplify/overwritten.ll
index b693e9c0db27..a32d6a8daeb0 100644
--- a/polly/test/Simplify/overwritten.ll
+++ b/polly/test/Simplify/overwritten.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
 ;
 ; Remove a store that is overwritten by another store in the same statement.
 ;
diff --git a/polly/test/Simplify/overwritten_3phi.ll b/polly/test/Simplify/overwritten_3phi.ll
index 84cf67fa7cdc..24758b9b7cf9 100644
--- a/polly/test/Simplify/overwritten_3phi.ll
+++ b/polly/test/Simplify/overwritten_3phi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Remove identical writes
 ; (two stores in the same statement that write the same value to the same
diff --git a/polly/test/Simplify/overwritten_3store.ll b/polly/test/Simplify/overwritten_3store.ll
index 72e9917b36a9..63eb5b54f931 100644
--- a/polly/test/Simplify/overwritten_3store.ll
+++ b/polly/test/Simplify/overwritten_3store.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-stmt-granularity=bb -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadNPMPolly -polly-stmt-granularity=bb "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
 ;
 ; Remove a store that is overwritten by another store in the same statement.
 ; Check that even multiple stores are removed.
diff --git a/polly/test/Simplify/overwritten_implicit_and_explicit.ll b/polly/test/Simplify/overwritten_implicit_and_explicit.ll
index ba7c6d0f20b1..56c63b48f761 100644
--- a/polly/test/Simplify/overwritten_implicit_and_explicit.ll
+++ b/polly/test/Simplify/overwritten_implicit_and_explicit.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Remove a store that is overwritten by another store in the same statement.
 ; Check that this works even if one of the writes is a scalar MemoryKind.
diff --git a/polly/test/Simplify/overwritten_loadbetween.ll b/polly/test/Simplify/overwritten_loadbetween.ll
index f271b4559dc8..b31f45d5db62 100644
--- a/polly/test/Simplify/overwritten_loadbetween.ll
+++ b/polly/test/Simplify/overwritten_loadbetween.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck -match-full-lines %s
 ;
 ; Do not remove overwrites when the value is read before.
 ;
diff --git a/polly/test/Simplify/overwritten_scalar.ll b/polly/test/Simplify/overwritten_scalar.ll
index 41c5c6fa2470..d55ea7712c36 100644
--- a/polly/test/Simplify/overwritten_scalar.ll
+++ b/polly/test/Simplify/overwritten_scalar.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck -match-full-lines %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck -match-full-lines %s
 ;
 ; Remove identical writes
 ; (two stores in the same statement that write the same value to the same
diff --git a/polly/test/Simplify/pass_existence.ll b/polly/test/Simplify/pass_existence.ll
index a8fc184b1616..fc5287ed2ee2 100644
--- a/polly/test/Simplify/pass_existence.ll
+++ b/polly/test/Simplify/pass_existence.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly -disable-output "-passes=scop(print<polly-simplify>)" < %s -aa-pipeline=basic-aa < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s
+; RUN: opt %loadNPMPolly -disable-output "-passes=scop(print<polly-simplify>)" < %s -aa-pipeline=basic-aa < %s | FileCheck %s
 ;
 ; Simple test for the existence of the Simplify pass.
 ;
diff --git a/polly/test/Simplify/phi_in_regionstmt.ll b/polly/test/Simplify/phi_in_regionstmt.ll
index 4c6a8744e200..32bb75427589 100644
--- a/polly/test/Simplify/phi_in_regionstmt.ll
+++ b/polly/test/Simplify/phi_in_regionstmt.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; The PHINode %cond91.sink.sink.us.sink.6 is in the middle of a region
 ; statement.
diff --git a/polly/test/Simplify/pr33323.ll b/polly/test/Simplify/pr33323.ll
index de2e00e8e2e9..751f0bff5961 100644
--- a/polly/test/Simplify/pr33323.ll
+++ b/polly/test/Simplify/pr33323.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s
 ;
 ; llvm.org/PR33323
 ;
diff --git a/polly/test/Simplify/redundant.ll b/polly/test/Simplify/redundant.ll
index 720f2e3d0ef2..e85352bc889f 100644
--- a/polly/test/Simplify/redundant.ll
+++ b/polly/test/Simplify/redundant.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Remove redundant store (a store that writes the same value already
 ; at the destination)
diff --git a/polly/test/Simplify/redundant_differentindex.ll b/polly/test/Simplify/redundant_differentindex.ll
index c79364608e58..23531c24344f 100644
--- a/polly/test/Simplify/redundant_differentindex.ll
+++ b/polly/test/Simplify/redundant_differentindex.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; A store that has a different index than the load it is storing is
 ; not redundant.
diff --git a/polly/test/Simplify/redundant_region.ll b/polly/test/Simplify/redundant_region.ll
index d5c9586283de..dbcb420ac2f3 100644
--- a/polly/test/Simplify/redundant_region.ll
+++ b/polly/test/Simplify/redundant_region.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Remove redundant store (a store that writes the same value already
 ; at the destination) in a region.
diff --git a/polly/test/Simplify/redundant_region_scalar.ll b/polly/test/Simplify/redundant_region_scalar.ll
index ab07126fe268..95a581ad6f57 100644
--- a/polly/test/Simplify/redundant_region_scalar.ll
+++ b/polly/test/Simplify/redundant_region_scalar.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Remove redundant store (a store that writes the same value already
 ; at the destination) in a region.
diff --git a/polly/test/Simplify/redundant_scalarwrite.ll b/polly/test/Simplify/redundant_scalarwrite.ll
index c09be5f61837..e2f7bbedc023 100644
--- a/polly/test/Simplify/redundant_scalarwrite.ll
+++ b/polly/test/Simplify/redundant_scalarwrite.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Remove redundant scalar stores.
 ;
diff --git a/polly/test/Simplify/redundant_storebetween.ll b/polly/test/Simplify/redundant_storebetween.ll
index f87c1126cd26..f624b6e5b995 100644
--- a/polly/test/Simplify/redundant_storebetween.ll
+++ b/polly/test/Simplify/redundant_storebetween.ll
@@ -1,4 +1,5 @@
-; RUN: opt %loadPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadNPMPolly "-passes=scop(print<polly-simplify>)" -disable-output -aa-pipeline=basic-aa < %s | FileCheck %s -match-full-lines
 ;
 ; Don't remove store where there is another store to the same target
 ; in-between them.
diff --git a/polly/test/Simplify/scalability1.ll b/polly/test/Simplify/scalability1.ll
index a91574e2b274..0ef99ce1ad8e 100644
--- a/polly/test/Simplify/scalability1.ll
+++ b/polly/test/Simplify/scalability1.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-ignore-inbounds '-passes=print<polly-simplify>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-ignore-inbounds -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Test scalability.
 ;
diff --git a/polly/test/Simplify/scalability2.ll b/polly/test/Simplify/scalability2.ll
index 4e4874df219e..bac0810b0afa 100644
--- a/polly/test/Simplify/scalability2.ll
+++ b/polly/test/Simplify/scalability2.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly -polly-ignore-inbounds '-passes=print<polly-simplify>' -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-ignore-inbounds -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Test scalability.
 ;
diff --git a/polly/test/Simplify/sweep_mapped_phi.ll b/polly/test/Simplify/sweep_mapped_phi.ll
index 3b9e61c72b4d..add1681cdf36 100644
--- a/polly/test/Simplify/sweep_mapped_phi.ll
+++ b/polly/test/Simplify/sweep_mapped_phi.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Map %phi to A[j], so the scalar write in Stmt_for_bodyA can be removed.
 ;
diff --git a/polly/test/Simplify/sweep_mapped_value.ll b/polly/test/Simplify/sweep_mapped_value.ll
index 5992e2401ca5..2e2f9c37febe 100644
--- a/polly/test/Simplify/sweep_mapped_value.ll
+++ b/polly/test/Simplify/sweep_mapped_value.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-import-jscop,print<polly-simplify>' -polly-import-jscop-postfix=transformed -disable-output < %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-print-simplify -disable-output < %s | FileCheck %s -match-full-lines
 ;
 ; Map %val to A[j], so the scalar write on Stmt_for_bodyB can be removed.
 ;
diff --git a/polly/test/Simplify/ununsed_read_in_region_entry.ll b/polly/test/Simplify/ununsed_read_in_region_entry.ll
index 111c19f706b9..9b2d4521e2d6 100644
--- a/polly/test/Simplify/ununsed_read_in_region_entry.ll
+++ b/polly/test/Simplify/ununsed_read_in_region_entry.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly '-passes=print<polly-simplify>' -disable-output< %s | FileCheck %s -match-full-lines
-; RUN: opt %loadPolly '-passes=polly-simplify,polly-codegen' -S < %s | FileCheck %s -check-prefix=CODEGEN
+; RUN: opt %loadPolly -polly-print-simplify -disable-output< %s | FileCheck %s -match-full-lines
+; RUN: opt %loadPolly -polly-simplify -polly-codegen -S < %s | FileCheck %s -check-prefix=CODEGEN
 ;
 ; for (int i = 0; i < n; i+=1) {
 ;    (void)A[0];
diff --git a/polly/test/Support/Plugins.ll b/polly/test/Support/Plugins.ll
index c4579470192b..cee878f1c6ac 100644
--- a/polly/test/Support/Plugins.ll
+++ b/polly/test/Support/Plugins.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=polly-prepare,scop(print<polly-ast>)' -S < %s \
+; RUN: opt %loadNPMPolly -passes='polly-prepare,scop(print<polly-ast>)' -S < %s \
 ; RUN: | FileCheck %s
 
 ; This testcase tests plugin registration. Check-lines below serve to verify
diff --git a/polly/test/Support/defaultpipelines.ll b/polly/test/Support/defaultpipelines.ll
index 6681042727c6..ab0329a70327 100644
--- a/polly/test/Support/defaultpipelines.ll
+++ b/polly/test/Support/defaultpipelines.ll
@@ -1,9 +1,9 @@
-; RUN: opt %loadPolly -polly -O0 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
-; RUN: opt %loadPolly -polly -O1 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
-; RUN: opt %loadPolly -polly -O2 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
-; RUN: opt %loadPolly -polly -O3 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
-; RUN: opt %loadPolly -polly -Os -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
-; RUN: opt %loadPolly -polly -Oz -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
+; RUN: opt %loadNPMPolly -polly -O0 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
+; RUN: opt %loadNPMPolly -polly -O1 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
+; RUN: opt %loadNPMPolly -polly -O2 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
+; RUN: opt %loadNPMPolly -polly -O3 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ON
+; RUN: opt %loadNPMPolly -polly -Os -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
+; RUN: opt %loadNPMPolly -polly -Oz -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=OFF
 ;
 ; Check that Polly's default pipeline works from detection to code generation
 ; with either pass manager.
diff --git a/polly/test/Support/dumpfunction.ll b/polly/test/Support/dumpfunction.ll
index e99261508f1a..863212b2ef7d 100644
--- a/polly/test/Support/dumpfunction.ll
+++ b/polly/test/Support/dumpfunction.ll
@@ -1,9 +1,9 @@
 ; New pass manager
-; RUN: opt %loadPolly -O3 -polly -polly-position=before-vectorizer -polly-dump-before --disable-output %s
+; RUN: opt %loadNPMPolly -O3 -polly -polly-position=before-vectorizer -polly-dump-before --disable-output %s
 ; RUN: FileCheck --input-file=dumpfunction-callee-before.ll --check-prefix=CHECK --check-prefix=CALLEE %s
 ; RUN: FileCheck --input-file=dumpfunction-caller-before.ll --check-prefix=CHECK --check-prefix=CALLER %s
 ;
-; RUN: opt %loadPolly -O3 -polly -polly-position=before-vectorizer -polly-dump-after --disable-output %s
+; RUN: opt %loadNPMPolly -O3 -polly -polly-position=before-vectorizer -polly-dump-after --disable-output %s
 ; RUN: FileCheck --input-file=dumpfunction-callee-after.ll --check-prefix=CHECK --check-prefix=CALLEE %s
 ; RUN: FileCheck --input-file=dumpfunction-caller-after.ll --check-prefix=CHECK --check-prefix=CALLER %s
 
diff --git a/polly/test/Support/dumpmodule.ll b/polly/test/Support/dumpmodule.ll
index d7aa88439f64..693fe4bc6cde 100644
--- a/polly/test/Support/dumpmodule.ll
+++ b/polly/test/Support/dumpmodule.ll
@@ -1,5 +1,5 @@
-; RUN: opt %loadPolly -O3 -polly -polly-position=early             -polly-dump-before-file=%t-npm-before-early.ll    --disable-output < %s && FileCheck --input-file=%t-npm-before-early.ll    --check-prefix=EARLY %s
-; RUN: opt %loadPolly -O3 -polly -polly-position=early             -polly-dump-after-file=%t-npm-after-early.ll      --disable-output < %s && FileCheck --input-file=%t-npm-after-early.ll     --check-prefix=EARLY --check-prefix=AFTEREARLY %s
+; RUN: opt %loadNPMPolly -O3 -polly -polly-position=early             -polly-dump-before-file=%t-npm-before-early.ll    --disable-output < %s && FileCheck --input-file=%t-npm-before-early.ll    --check-prefix=EARLY %s
+; RUN: opt %loadNPMPolly -O3 -polly -polly-position=early             -polly-dump-after-file=%t-npm-after-early.ll      --disable-output < %s && FileCheck --input-file=%t-npm-after-early.ll     --check-prefix=EARLY --check-prefix=AFTEREARLY %s
 ;
 ; Check the module dumping before Polly at specific positions in the
 ; pass pipeline.
diff --git a/polly/test/Support/exportjson.ll b/polly/test/Support/exportjson.ll
index 22ba845bafc2..22cfea23534c 100644
--- a/polly/test/Support/exportjson.ll
+++ b/polly/test/Support/exportjson.ll
@@ -1,6 +1,6 @@
 ; RUN: rm -rf %t
 ; RUN: mkdir -p %t
-; RUN: opt %loadPolly -polly-import-jscop-dir=%t -polly -O2 -polly-export -S < %s
+; RUN: opt %loadNPMPolly -polly-import-jscop-dir=%t -polly -O2 -polly-export -S < %s
 ; RUN: FileCheck %s -input-file %t/exportjson___%entry.split---%return.jscop
 ;
 ; for (int j = 0; j < n; j += 1) {
diff --git a/polly/test/Support/isl-args.ll b/polly/test/Support/isl-args.ll
index 442742d55f81..efa94194bc3f 100644
--- a/polly/test/Support/isl-args.ll
+++ b/polly/test/Support/isl-args.ll
@@ -1,7 +1,7 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-isl-arg=-V < %s | FileCheck %s -match-full-lines --check-prefix=VERSION
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-isl-arg=-h < %s | FileCheck %s -match-full-lines --check-prefix=HELP
-; RUN: not opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-isl-arg=-asdf < %s 2>&1| FileCheck %s -match-full-lines --check-prefix=UNKNOWN
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -disable-output -polly-isl-arg=--schedule-algorithm=feautrier < %s
+; RUN: opt %loadPolly -polly-scops -disable-output -polly-isl-arg=-V < %s | FileCheck %s -match-full-lines --check-prefix=VERSION
+; RUN: opt %loadPolly -polly-scops -disable-output -polly-isl-arg=-h < %s | FileCheck %s -match-full-lines --check-prefix=HELP
+; RUN: not opt %loadPolly -polly-scops -disable-output -polly-isl-arg=-asdf < %s 2>&1| FileCheck %s -match-full-lines --check-prefix=UNKNOWN
+; RUN: opt %loadPolly -polly-scops -disable-output -polly-isl-arg=--schedule-algorithm=feautrier < %s
 
 ; VERSION: isl-{{.*}}-IMath-32
 ; HELP: Usage: -polly-isl-arg [OPTION...]
diff --git a/polly/test/Support/pipelineposition.ll b/polly/test/Support/pipelineposition.ll
index 757af91011fb..a4506ba1d64e 100644
--- a/polly/test/Support/pipelineposition.ll
+++ b/polly/test/Support/pipelineposition.ll
@@ -1,6 +1,6 @@
-; RUN: opt %loadPolly -O3 -polly -polly-position=early                    -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=NOINLINE
-; RUN: opt %loadPolly -O3 -polly -polly-position=early -polly-run-inliner -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED1
-; RUN: opt %loadPolly -O3 -polly -polly-position=before-vectorizer        -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED3
+; RUN: opt %loadNPMPolly -O3 -polly -polly-position=early                    -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=NOINLINE
+; RUN: opt %loadNPMPolly -O3 -polly -polly-position=early -polly-run-inliner -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED1
+; RUN: opt %loadNPMPolly -O3 -polly -polly-position=before-vectorizer        -disable-output -debug-only=polly-scops < %s 2>&1 | FileCheck %s --check-prefix=INLINED3
 ;
 ; REQUIRES: asserts
 ;
diff --git a/polly/test/Support/pollyDebug.ll b/polly/test/Support/pollyDebug.ll
index e5e327c3976e..ada079023b6c 100644
--- a/polly/test/Support/pollyDebug.ll
+++ b/polly/test/Support/pollyDebug.ll
@@ -1,5 +1,5 @@
 ; Test if "polly-debug" flag enables debug prints from different parts of polly
-; RUN: opt %loadPolly -O3 -polly -polly-debug --disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt %loadNPMPolly -O3 -polly -polly-debug --disable-output < %s 2>&1 | FileCheck %s
 ;
 ; REQUIRES: asserts
 
diff --git a/polly/test/lit.site.cfg.in b/polly/test/lit.site.cfg.in
index 703752896239..b44061260834 100644
--- a/polly/test/lit.site.cfg.in
+++ b/polly/test/lit.site.cfg.in
@@ -38,11 +38,16 @@ if config.llvm_polly_link_into_tools == '' or \
    config.llvm_polly_link_into_tools.lower() == 'false' or \
    config.llvm_polly_link_into_tools.lower() == 'notfound' or \
    config.llvm_polly_link_into_tools.lower() == 'llvm_polly_link_into_tools-notfound':
-    config.substitutions.append(('%loadPolly', '-load-pass-plugin '
+    config.substitutions.append(('%loadPolly', '-load '
+                                 + config.polly_lib_dir + '/LLVMPolly@LLVM_SHLIBEXT@'
+                                 + commonOpts ))
+    config.substitutions.append(('%loadNPMPolly', '-load-pass-plugin '
                                  + config.polly_lib_dir + '/LLVMPolly@LLVM_SHLIBEXT@'
                                  + commonOpts ))
 else:
     config.substitutions.append(('%loadPolly', commonOpts ))
+    config.substitutions.append(('%loadNPMPolly', commonOpts ))
+
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)
diff --git a/polly/test/polly.ll b/polly/test/polly.ll
index 6654468470a6..f78cceacfb12 100644
--- a/polly/test/polly.ll
+++ b/polly/test/polly.ll
@@ -1,4 +1,4 @@
-; RUN: opt %loadPolly '-passes=print<polly-function-scops>' -S < %s 2>&1 | FileCheck %s
+; RUN: opt %loadPolly -polly-scops -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 define void @foo() nounwind {
 start:
-- 
GitLab


From 24180ea0c295856a696992f072c36259a266226b Mon Sep 17 00:00:00 2001
From: Ryosuke Niwa <rniwa@webkit.org>
Date: Tue, 14 May 2024 22:16:06 -0700
Subject: [PATCH 0809/1206] [analyzer] Treat break, continue, goto, and label
 statements as trivial in WebKit checkers. (#91873)

Also allow CXXBindTemporaryExpr, which creates a temporary object with a
non-trivial destructor, and add a few more std and WTF functions to the
explicitly allowed list.
---
 .../Checkers/WebKit/PtrTypesSemantics.cpp     | 23 +++++-
 .../Checkers/WebKit/uncounted-obj-arg.cpp     | 78 +++++++++++++++++--
 2 files changed, 94 insertions(+), 7 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index ad493587affa..950d35a090a3 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -308,6 +308,12 @@ public:
   bool VisitCaseStmt(const CaseStmt *CS) { return VisitChildren(CS); }
   bool VisitDefaultStmt(const DefaultStmt *DS) { return VisitChildren(DS); }
 
+  // break, continue, goto, and label statements are always trivial.
+  bool VisitBreakStmt(const BreakStmt *) { return true; }
+  bool VisitContinueStmt(const ContinueStmt *) { return true; }
+  bool VisitGotoStmt(const GotoStmt *) { return true; }
+  bool VisitLabelStmt(const LabelStmt *) { return true; }
+
   bool VisitUnaryOperator(const UnaryOperator *UO) {
     // Unary operators are trivial if its operand is trivial except co_await.
     return UO->getOpcode() != UO_Coawait && Visit(UO->getSubExpr());
@@ -349,12 +355,17 @@ public:
       return false;
     const auto &Name = safeGetName(Callee);
 
+    if (Callee->isInStdNamespace() &&
+        (Name == "addressof" || Name == "forward" || Name == "move"))
+      return true;
+
     if (Name == "WTFCrashWithInfo" || Name == "WTFBreakpointTrap" ||
+        Name == "WTFCrashWithSecurityImplication" || Name == "WTFCrash" ||
         Name == "WTFReportAssertionFailure" || Name == "isMainThread" ||
         Name == "isMainThreadOrGCThread" || Name == "isMainRunLoop" ||
         Name == "isWebThread" || Name == "isUIThread" ||
-        Name == "compilerFenceForCrash" || Name == "bitwise_cast" ||
-        Name == "addressof" || Name.find("__builtin") == 0)
+        Name == "mayBeGCThread" || Name == "compilerFenceForCrash" ||
+        Name == "bitwise_cast" || Name.find("__builtin") == 0)
       return true;
 
     return TrivialFunctionAnalysis::isTrivialImpl(Callee, Cache);
@@ -445,6 +456,14 @@ public:
     return Visit(VMT->getSubExpr());
   }
 
+  bool VisitCXXBindTemporaryExpr(const CXXBindTemporaryExpr *BTE) {
+    if (auto *Temp = BTE->getTemporary()) {
+      if (!TrivialFunctionAnalysis::isTrivialImpl(Temp->getDestructor(), Cache))
+        return false;
+    }
+    return Visit(BTE->getSubExpr());
+  }
+
   bool VisitExprWithCleanups(const ExprWithCleanups *EWC) {
     return Visit(EWC->getSubExpr());
   }
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
index 073f3252160e..ed37671df3d3 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
@@ -7,6 +7,9 @@ void WTFBreakpointTrap();
 void WTFCrashWithInfo(int, const char*, const char*, int);
 void WTFReportAssertionFailure(const char* file, int line, const char* function, const char* assertion);
 
+void WTFCrash(void);
+void WTFCrashWithSecurityImplication(void);
+
 inline void compilerFenceForCrash()
 {
     asm volatile("" ::: "memory");
@@ -62,14 +65,25 @@ void WTFCrashWithInfo(int line, const char* file, const char* function, int coun
 template<typename ToType, typename FromType>
 ToType bitwise_cast(FromType from);
 
+namespace std {
+
 template<typename T>
 T* addressof(T& arg);
 
+template<typename T>
+T&& forward(T& arg);
+
+template<typename T>
+T&& move( T&& t );
+
+} // namespace std
+
 bool isMainThread();
 bool isMainThreadOrGCThread();
 bool isMainRunLoop();
 bool isWebThread();
 bool isUIThread();
+bool mayBeGCThread();
 
 enum class Flags : unsigned short {
   Flag1 = 1 << 0,
@@ -161,16 +175,42 @@ private:
 
 class ComplexNumber {
 public:
-  ComplexNumber() : real(0), complex(0) { }
+  ComplexNumber() : realPart(0), complexPart(0) { }
   ComplexNumber(const ComplexNumber&);
-  ComplexNumber& operator++() { real.someMethod(); return *this; }
+  ComplexNumber& operator++() { realPart.someMethod(); return *this; }
   ComplexNumber operator++(int);
   ComplexNumber& operator<<(int);
   ComplexNumber& operator+();
 
+  const Number& real() const { return realPart; }
+
 private:
-  Number real;
-  Number complex;
+  Number realPart;
+  Number complexPart;
+};
+
+class ObjectWithNonTrivialDestructor {
+public:
+  ObjectWithNonTrivialDestructor() { }
+  ObjectWithNonTrivialDestructor(unsigned v) : v(v) { }
+  ~ObjectWithNonTrivialDestructor() { }
+
+  unsigned value() const { return v; }
+
+private:
+  unsigned v { 0 };
+};
+
+class ObjectWithMutatingDestructor {
+public:
+  ObjectWithMutatingDestructor() : n(0) { }
+  ObjectWithMutatingDestructor(int n) : n(n) { }
+  ~ObjectWithMutatingDestructor() { n.someMethod(); }
+
+  unsigned value() const { return n.value(); }
+
+private:
+  Number n;
 };
 
 class RefCounted {
@@ -248,7 +288,7 @@ public:
   int trivial40() { return v << 2; }
   unsigned trivial41() { v = ++s_v; return v; }
   unsigned trivial42() { return bitwise_cast<unsigned long>(nullptr); }
-  Number* trivial43() { return addressof(*number); }
+  Number* trivial43() { return std::addressof(*number); }
   Number* trivial44() { return new Number(1); }
   ComplexNumber* trivial45() { return new ComplexNumber(); }
   void trivial46() { ASSERT(isMainThread()); }
@@ -256,6 +296,21 @@ public:
   void trivial48() { ASSERT(isMainRunLoop()); }
   void trivial49() { ASSERT(isWebThread()); }
   void trivial50() { ASSERT(isUIThread()); }
+  void trivial51() { ASSERT(mayBeGCThread()); }
+  void trivial52() { WTFCrash(); }
+  void trivial53() { WTFCrashWithSecurityImplication(); }
+  unsigned trivial54() { return ComplexNumber().real().value(); }
+  Number&& trivial55() { return std::forward(*number); }
+  unsigned trivial56() { Number n { 5 }; return std::move(n).value(); }
+  void trivial57() { do { break; } while (1); }
+  void trivial58() { do { continue; } while (0); }
+  void trivial59() {
+    do { goto label; }
+    while (0);
+  label:
+    return;
+  }
+  unsigned trivial60() { return ObjectWithNonTrivialDestructor { 5 }.value(); }
 
   static RefCounted& singleton() {
     static RefCounted s_RefCounted;
@@ -335,6 +390,7 @@ public:
   ComplexNumber nonTrivial17() { return complex << 2; }
   ComplexNumber nonTrivial18() { return +complex; }
   ComplexNumber* nonTrivial19() { return new ComplexNumber(complex); }
+  unsigned nonTrivial20() { return ObjectWithMutatingDestructor { 7 }.value(); }
 
   static unsigned s_v;
   unsigned v { 0 };
@@ -413,6 +469,16 @@ public:
     getFieldTrivial().trivial48(); // no-warning
     getFieldTrivial().trivial49(); // no-warning
     getFieldTrivial().trivial50(); // no-warning
+    getFieldTrivial().trivial51(); // no-warning
+    getFieldTrivial().trivial52(); // no-warning
+    getFieldTrivial().trivial53(); // no-warning
+    getFieldTrivial().trivial54(); // no-warning
+    getFieldTrivial().trivial55(); // no-warning
+    getFieldTrivial().trivial56(); // no-warning
+    getFieldTrivial().trivial57(); // no-warning
+    getFieldTrivial().trivial58(); // no-warning
+    getFieldTrivial().trivial59(); // no-warning
+    getFieldTrivial().trivial60(); // no-warning
 
     RefCounted::singleton().trivial18(); // no-warning
     RefCounted::singleton().someFunction(); // no-warning
@@ -457,6 +523,8 @@ public:
     // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
     getFieldTrivial().nonTrivial19();
     // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    getFieldTrivial().nonTrivial20();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
   }
 };
 
-- 
GitLab


From 0980f715cf7c3d78be6ba64e902bd2dfad3ebc75 Mon Sep 17 00:00:00 2001
From: Robin Caloudis <robin.caloudis@gmx.de>
Date: Wed, 15 May 2024 07:26:23 +0200
Subject: [PATCH 0810/1206] [libc][errno] Remove previously added errno numbers
 (#92163)

Introduced in https://github.com/llvm/llvm-project/pull/91150. Not
needed anymore as https://github.com/llvm/llvm-project/pull/92041 fixed
the root cause. `ENAMETOOLONG` and `EOVERFLOW` are well defined in
`<linux/errno.h>`.

Post mortem: Due to the previously missing inclusion of
`<linux/errno.h>` (fixed with
https://github.com/llvm/llvm-project/pull/92041), I misinterpreted an
undefined macro issue during the development of
https://github.com/llvm/llvm-project/pull/91150 as being caused by a
missing definition rather than by the missing inclusion of the linux
header. I realized too late that `ENAMETOOLONG` and `EOVERFLOW` were
correctly defined in `<linux/errno.h>` and that it was my missing
inclusion that caused the problem.
---
 libc/include/llvm-libc-macros/generic-error-number-macros.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/libc/include/llvm-libc-macros/generic-error-number-macros.h b/libc/include/llvm-libc-macros/generic-error-number-macros.h
index cb4411fbac66..59b121ef1481 100644
--- a/libc/include/llvm-libc-macros/generic-error-number-macros.h
+++ b/libc/include/llvm-libc-macros/generic-error-number-macros.h
@@ -43,7 +43,5 @@
 #define EPIPE 32
 #define EDOM 33
 #define ERANGE 34
-#define ENAMETOOLONG 36
-#define EOVERFLOW 75
 
 #endif // LLVM_LIBC_MACROS_GENERIC_ERROR_NUMBER_MACROS_H
-- 
GitLab


From d7bb0723fe79d2b75d41789d2ffadda3567dd94e Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 15 May 2024 07:41:28 +0200
Subject: [PATCH 0811/1206] InstCombine: Emit ldexp intrinsic in exp2->ldexp
 combine (#92039)

Prefer to emit the intrinsic over a libcall in the
intrinsic or no-math-errno case.
---
 .../lib/Transforms/Utils/SimplifyLibCalls.cpp |  17 +-
 llvm/test/Transforms/InstCombine/exp2-1.ll    | 123 ++++++++++++--
 .../Transforms/InstCombine/exp2-to-ldexp.ll   | 150 ++++++++++++++++++
 3 files changed, 272 insertions(+), 18 deletions(-)
 create mode 100644 llvm/test/Transforms/InstCombine/exp2-to-ldexp.ll

diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 174cc7a3c778..9cb8e20b4806 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -2389,12 +2389,21 @@ Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
   if ((isa<SIToFPInst>(Op) || isa<UIToFPInst>(Op)) &&
       hasFloatFn(M, TLI, Ty, LibFunc_ldexp, LibFunc_ldexpf, LibFunc_ldexpl)) {
     if (Value *Exp = getIntToFPVal(Op, B, TLI->getIntSize())) {
+      Constant *One = ConstantFP::get(Ty, 1.0);
+
+      // TODO: Emitting the intrinsic should not depend on whether the libcall
+      // is available.
+      if (CI->doesNotAccessMemory()) {
+        return copyFlags(*CI, B.CreateIntrinsic(Intrinsic::ldexp,
+                                                {Ty, Exp->getType()},
+                                                {One, Exp}, CI));
+      }
+
       IRBuilderBase::FastMathFlagGuard Guard(B);
       B.setFastMathFlags(CI->getFastMathFlags());
-      return copyFlags(
-          *CI, emitBinaryFloatFnCall(ConstantFP::get(Ty, 1.0), Exp, TLI,
-                                     LibFunc_ldexp, LibFunc_ldexpf,
-                                     LibFunc_ldexpl, B, AttributeList()));
+      return copyFlags(*CI, emitBinaryFloatFnCall(
+                                One, Exp, TLI, LibFunc_ldexp, LibFunc_ldexpf,
+                                LibFunc_ldexpl, B, AttributeList()));
     }
   }
 
diff --git a/llvm/test/Transforms/InstCombine/exp2-1.ll b/llvm/test/Transforms/InstCombine/exp2-1.ll
index 8419854d3ec6..5bf70320d9ec 100644
--- a/llvm/test/Transforms/InstCombine/exp2-1.ll
+++ b/llvm/test/Transforms/InstCombine/exp2-1.ll
@@ -14,6 +14,7 @@ declare float @exp2f(float)
 declare double @llvm.exp2.f64(double)
 declare float @llvm.exp2.f32(float)
 declare <2 x float> @llvm.exp2.v2f32(<2 x float>)
+declare fp128 @exp2l(fp128)
 
 
 ; Check exp2(sitofp(x)) -> ldexp(1.0, sext(x)).
@@ -227,18 +228,18 @@ define float @test_simplify8(i8 zeroext %x) {
 define double @test_simplify9(i8 zeroext %x) {
 ; LDEXP32-LABEL: @test_simplify9(
 ; LDEXP32-NEXT:    [[TMP1:%.*]] = zext i8 [[X:%.*]] to i32
-; LDEXP32-NEXT:    [[LDEXP:%.*]] = call double @ldexp(double 1.000000e+00, i32 [[TMP1]])
-; LDEXP32-NEXT:    ret double [[LDEXP]]
+; LDEXP32-NEXT:    [[RET:%.*]] = call double @llvm.ldexp.f64.i32(double 1.000000e+00, i32 [[TMP1]])
+; LDEXP32-NEXT:    ret double [[RET]]
 ;
 ; LDEXP16-LABEL: @test_simplify9(
 ; LDEXP16-NEXT:    [[TMP1:%.*]] = zext i8 [[X:%.*]] to i16
-; LDEXP16-NEXT:    [[LDEXP:%.*]] = call double @ldexp(double 1.000000e+00, i16 [[TMP1]])
-; LDEXP16-NEXT:    ret double [[LDEXP]]
+; LDEXP16-NEXT:    [[RET:%.*]] = call double @llvm.ldexp.f64.i16(double 1.000000e+00, i16 [[TMP1]])
+; LDEXP16-NEXT:    ret double [[RET]]
 ;
 ; NOLDEXPF-LABEL: @test_simplify9(
 ; NOLDEXPF-NEXT:    [[TMP1:%.*]] = zext i8 [[X:%.*]] to i32
-; NOLDEXPF-NEXT:    [[LDEXP:%.*]] = call double @ldexp(double 1.000000e+00, i32 [[TMP1]])
-; NOLDEXPF-NEXT:    ret double [[LDEXP]]
+; NOLDEXPF-NEXT:    [[RET:%.*]] = call double @llvm.ldexp.f64.i32(double 1.000000e+00, i32 [[TMP1]])
+; NOLDEXPF-NEXT:    ret double [[RET]]
 ;
 ; NOLDEXP-LABEL: @test_simplify9(
 ; NOLDEXP-NEXT:    [[CONV:%.*]] = uitofp i8 [[X:%.*]] to double
@@ -253,13 +254,13 @@ define double @test_simplify9(i8 zeroext %x) {
 define float @test_simplify10(i8 zeroext %x) {
 ; LDEXP32-LABEL: @test_simplify10(
 ; LDEXP32-NEXT:    [[TMP1:%.*]] = zext i8 [[X:%.*]] to i32
-; LDEXP32-NEXT:    [[LDEXPF:%.*]] = call float @ldexpf(float 1.000000e+00, i32 [[TMP1]])
-; LDEXP32-NEXT:    ret float [[LDEXPF]]
+; LDEXP32-NEXT:    [[RET:%.*]] = call float @llvm.ldexp.f32.i32(float 1.000000e+00, i32 [[TMP1]])
+; LDEXP32-NEXT:    ret float [[RET]]
 ;
 ; LDEXP16-LABEL: @test_simplify10(
 ; LDEXP16-NEXT:    [[TMP1:%.*]] = zext i8 [[X:%.*]] to i16
-; LDEXP16-NEXT:    [[LDEXPF:%.*]] = call float @ldexpf(float 1.000000e+00, i16 [[TMP1]])
-; LDEXP16-NEXT:    ret float [[LDEXPF]]
+; LDEXP16-NEXT:    [[RET:%.*]] = call float @llvm.ldexp.f32.i16(float 1.000000e+00, i16 [[TMP1]])
+; LDEXP16-NEXT:    ret float [[RET]]
 ;
 ; NOLDEXPF-LABEL: @test_simplify10(
 ; NOLDEXPF-NEXT:    [[CONV:%.*]] = uitofp i8 [[X:%.*]] to float
@@ -279,13 +280,13 @@ define float @test_simplify10(i8 zeroext %x) {
 define float @sitofp_scalar_intrinsic_with_FMF(i8 %x) {
 ; LDEXP32-LABEL: @sitofp_scalar_intrinsic_with_FMF(
 ; LDEXP32-NEXT:    [[TMP1:%.*]] = sext i8 [[X:%.*]] to i32
-; LDEXP32-NEXT:    [[LDEXPF:%.*]] = tail call nnan float @ldexpf(float 1.000000e+00, i32 [[TMP1]])
-; LDEXP32-NEXT:    ret float [[LDEXPF]]
+; LDEXP32-NEXT:    [[R:%.*]] = tail call nnan float @llvm.ldexp.f32.i32(float 1.000000e+00, i32 [[TMP1]])
+; LDEXP32-NEXT:    ret float [[R]]
 ;
 ; LDEXP16-LABEL: @sitofp_scalar_intrinsic_with_FMF(
 ; LDEXP16-NEXT:    [[TMP1:%.*]] = sext i8 [[X:%.*]] to i16
-; LDEXP16-NEXT:    [[LDEXPF:%.*]] = tail call nnan float @ldexpf(float 1.000000e+00, i16 [[TMP1]])
-; LDEXP16-NEXT:    ret float [[LDEXPF]]
+; LDEXP16-NEXT:    [[R:%.*]] = tail call nnan float @llvm.ldexp.f32.i16(float 1.000000e+00, i16 [[TMP1]])
+; LDEXP16-NEXT:    ret float [[R]]
 ;
 ; NOLDEXPF-LABEL: @sitofp_scalar_intrinsic_with_FMF(
 ; NOLDEXPF-NEXT:    [[S:%.*]] = sitofp i8 [[X:%.*]] to float
@@ -330,3 +331,97 @@ define <2 x float> @sitofp_vector_intrinsic_with_FMF(<2 x i8> %x) {
   %r = call nnan <2 x float> @llvm.exp2.v2f32(<2 x float> %s)
   ret <2 x float> %r
 }
+
+define double @test_readonly_exp2_f64_of_sitofp(i32 %x) {
+; LDEXP32-LABEL: @test_readonly_exp2_f64_of_sitofp(
+; LDEXP32-NEXT:    [[LDEXP:%.*]] = call double @ldexp(double 1.000000e+00, i32 [[X:%.*]])
+; LDEXP32-NEXT:    ret double [[LDEXP]]
+;
+; LDEXP16-LABEL: @test_readonly_exp2_f64_of_sitofp(
+; LDEXP16-NEXT:    [[CONV:%.*]] = sitofp i32 [[X:%.*]] to double
+; LDEXP16-NEXT:    [[RET:%.*]] = call double @exp2(double [[CONV]]) #[[ATTR2:[0-9]+]]
+; LDEXP16-NEXT:    ret double [[RET]]
+;
+; NOLDEXPF-LABEL: @test_readonly_exp2_f64_of_sitofp(
+; NOLDEXPF-NEXT:    [[LDEXP:%.*]] = call double @ldexp(double 1.000000e+00, i32 [[X:%.*]])
+; NOLDEXPF-NEXT:    ret double [[LDEXP]]
+;
+; NOLDEXP-LABEL: @test_readonly_exp2_f64_of_sitofp(
+; NOLDEXP-NEXT:    [[CONV:%.*]] = sitofp i32 [[X:%.*]] to double
+; NOLDEXP-NEXT:    [[RET:%.*]] = call double @exp2(double [[CONV]]) #[[ATTR1:[0-9]+]]
+; NOLDEXP-NEXT:    ret double [[RET]]
+;
+  %conv = sitofp i32 %x to double
+  %ret = call double @exp2(double %conv) readonly
+  ret double %ret
+}
+
+define float @test_readonly_exp2f_f32_of_sitofp(i32 %x) {
+; LDEXP32-LABEL: @test_readonly_exp2f_f32_of_sitofp(
+; LDEXP32-NEXT:    [[LDEXPF:%.*]] = call float @ldexpf(float 1.000000e+00, i32 [[X:%.*]])
+; LDEXP32-NEXT:    ret float [[LDEXPF]]
+;
+; LDEXP16-LABEL: @test_readonly_exp2f_f32_of_sitofp(
+; LDEXP16-NEXT:    [[CONV:%.*]] = sitofp i32 [[X:%.*]] to float
+; LDEXP16-NEXT:    [[RET:%.*]] = call float @exp2f(float [[CONV]]) #[[ATTR2]]
+; LDEXP16-NEXT:    ret float [[RET]]
+;
+; NOLDEXPF-LABEL: @test_readonly_exp2f_f32_of_sitofp(
+; NOLDEXPF-NEXT:    [[CONV:%.*]] = sitofp i32 [[X:%.*]] to float
+; NOLDEXPF-NEXT:    [[RET:%.*]] = call float @exp2f(float [[CONV]]) #[[ATTR2:[0-9]+]]
+; NOLDEXPF-NEXT:    ret float [[RET]]
+;
+; NOLDEXP-LABEL: @test_readonly_exp2f_f32_of_sitofp(
+; NOLDEXP-NEXT:    [[CONV:%.*]] = sitofp i32 [[X:%.*]] to float
+; NOLDEXP-NEXT:    [[RET:%.*]] = call float @exp2f(float [[CONV]]) #[[ATTR1]]
+; NOLDEXP-NEXT:    ret float [[RET]]
+;
+  %conv = sitofp i32 %x to float
+  %ret = call float @exp2f(float %conv) readonly
+  ret float %ret
+}
+
+define fp128 @test_readonly_exp2l_fp128_of_sitofp(i32 %x) {
+; LDEXP32-LABEL: @test_readonly_exp2l_fp128_of_sitofp(
+; LDEXP32-NEXT:    [[LDEXPL:%.*]] = call fp128 @ldexpl(fp128 0xL00000000000000003FFF000000000000, i32 [[X:%.*]])
+; LDEXP32-NEXT:    ret fp128 [[LDEXPL]]
+;
+; LDEXP16-LABEL: @test_readonly_exp2l_fp128_of_sitofp(
+; LDEXP16-NEXT:    [[CONV:%.*]] = sitofp i32 [[X:%.*]] to fp128
+; LDEXP16-NEXT:    [[RET:%.*]] = call fp128 @exp2l(fp128 [[CONV]]) #[[ATTR2]]
+; LDEXP16-NEXT:    ret fp128 [[RET]]
+;
+; NOLDEXP-LABEL: @test_readonly_exp2l_fp128_of_sitofp(
+; NOLDEXP-NEXT:    [[CONV:%.*]] = sitofp i32 [[X:%.*]] to fp128
+; NOLDEXP-NEXT:    [[RET:%.*]] = call fp128 @exp2l(fp128 [[CONV]]) #[[ATTR1]]
+; NOLDEXP-NEXT:    ret fp128 [[RET]]
+;
+  %conv = sitofp i32 %x to fp128
+  %ret = call fp128 @exp2l(fp128 %conv) readonly
+  ret fp128 %ret
+}
+
+define float @test_readonly_exp2f_f32_of_sitofp_flags(i32 %x) {
+; LDEXP32-LABEL: @test_readonly_exp2f_f32_of_sitofp_flags(
+; LDEXP32-NEXT:    [[LDEXPF:%.*]] = call nnan ninf float @ldexpf(float 1.000000e+00, i32 [[X:%.*]])
+; LDEXP32-NEXT:    ret float [[LDEXPF]]
+;
+; LDEXP16-LABEL: @test_readonly_exp2f_f32_of_sitofp_flags(
+; LDEXP16-NEXT:    [[CONV:%.*]] = sitofp i32 [[X:%.*]] to float
+; LDEXP16-NEXT:    [[RET:%.*]] = call nnan ninf float @exp2f(float [[CONV]]) #[[ATTR2]]
+; LDEXP16-NEXT:    ret float [[RET]]
+;
+; NOLDEXPF-LABEL: @test_readonly_exp2f_f32_of_sitofp_flags(
+; NOLDEXPF-NEXT:    [[CONV:%.*]] = sitofp i32 [[X:%.*]] to float
+; NOLDEXPF-NEXT:    [[RET:%.*]] = call nnan ninf float @exp2f(float [[CONV]]) #[[ATTR2]]
+; NOLDEXPF-NEXT:    ret float [[RET]]
+;
+; NOLDEXP-LABEL: @test_readonly_exp2f_f32_of_sitofp_flags(
+; NOLDEXP-NEXT:    [[CONV:%.*]] = sitofp i32 [[X:%.*]] to float
+; NOLDEXP-NEXT:    [[RET:%.*]] = call nnan ninf float @exp2f(float [[CONV]]) #[[ATTR1]]
+; NOLDEXP-NEXT:    ret float [[RET]]
+;
+  %conv = sitofp i32 %x to float
+  %ret = call nnan ninf float @exp2f(float %conv) readonly
+  ret float %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/exp2-to-ldexp.ll b/llvm/test/Transforms/InstCombine/exp2-to-ldexp.ll
new file mode 100644
index 000000000000..3069ee65e238
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/exp2-to-ldexp.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S -passes=instcombine %s | FileCheck -check-prefixes=CHECK,LDEXP %s
+; RUN: opt -S -passes=instcombine -disable-builtin=ldexpf -disable-builtin=ldexp -disable-builtin=ldexpl %s | FileCheck -check-prefixes=CHECK,NOLDEXP %s
+
+define float @exp2_f32_sitofp_i8(i8 %x) {
+; LDEXP-LABEL: define float @exp2_f32_sitofp_i8(
+; LDEXP-SAME: i8 [[X:%.*]]) {
+; LDEXP-NEXT:    [[TMP1:%.*]] = sext i8 [[X]] to i32
+; LDEXP-NEXT:    [[LDEXPF:%.*]] = call float @llvm.ldexp.f32.i32(float 1.000000e+00, i32 [[TMP1]])
+; LDEXP-NEXT:    ret float [[LDEXPF]]
+;
+; NOLDEXP-LABEL: define float @exp2_f32_sitofp_i8(
+; NOLDEXP-SAME: i8 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i8 [[X]] to float
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = call float @llvm.exp2.f32(float [[ITOFP]])
+; NOLDEXP-NEXT:    ret float [[EXP2]]
+;
+  %itofp = sitofp i8 %x to float
+  %exp2 = call float @llvm.exp2.f32(float %itofp)
+  ret float %exp2
+}
+
+define float @exp2_f32_sitofp_i8_flags(i8 %x) {
+; LDEXP-LABEL: define float @exp2_f32_sitofp_i8_flags(
+; LDEXP-SAME: i8 [[X:%.*]]) {
+; LDEXP-NEXT:    [[TMP1:%.*]] = sext i8 [[X]] to i32
+; LDEXP-NEXT:    [[LDEXPF:%.*]] = call nnan ninf float @llvm.ldexp.f32.i32(float 1.000000e+00, i32 [[TMP1]])
+; LDEXP-NEXT:    ret float [[LDEXPF]]
+;
+; NOLDEXP-LABEL: define float @exp2_f32_sitofp_i8_flags(
+; NOLDEXP-SAME: i8 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i8 [[X]] to float
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = call nnan ninf float @llvm.exp2.f32(float [[ITOFP]])
+; NOLDEXP-NEXT:    ret float [[EXP2]]
+;
+  %itofp = sitofp i8 %x to float
+  %exp2 = call nnan ninf float @llvm.exp2.f32(float %itofp)
+  ret float %exp2
+}
+
+define <2 x float> @exp2_v2f32_sitofp_v2i8(<2 x i8> %x) {
+; CHECK-LABEL: define <2 x float> @exp2_v2f32_sitofp_v2i8(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i8> [[X]] to <2 x float>
+; CHECK-NEXT:    [[EXP2:%.*]] = call <2 x float> @llvm.exp2.v2f32(<2 x float> [[ITOFP]])
+; CHECK-NEXT:    ret <2 x float> [[EXP2]]
+;
+  %itofp = sitofp <2 x i8> %x to <2 x float>
+  %exp2 = call <2 x float> @llvm.exp2.v2f32(<2 x float> %itofp)
+  ret <2 x float> %exp2
+}
+
+define float @exp2_f32_uitofp_i8(i8 %x) {
+; LDEXP-LABEL: define float @exp2_f32_uitofp_i8(
+; LDEXP-SAME: i8 [[X:%.*]]) {
+; LDEXP-NEXT:    [[TMP1:%.*]] = zext i8 [[X]] to i32
+; LDEXP-NEXT:    [[LDEXPF:%.*]] = call float @llvm.ldexp.f32.i32(float 1.000000e+00, i32 [[TMP1]])
+; LDEXP-NEXT:    ret float [[LDEXPF]]
+;
+; NOLDEXP-LABEL: define float @exp2_f32_uitofp_i8(
+; NOLDEXP-SAME: i8 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = uitofp i8 [[X]] to float
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = call float @llvm.exp2.f32(float [[ITOFP]])
+; NOLDEXP-NEXT:    ret float [[EXP2]]
+;
+  %itofp = uitofp i8 %x to float
+  %exp2 = call float @llvm.exp2.f32(float %itofp)
+  ret float %exp2
+}
+
+define half @exp2_f16_sitofp_i8(i8 %x) {
+; CHECK-LABEL: define half @exp2_f16_sitofp_i8(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ITOFP:%.*]] = sitofp i8 [[X]] to half
+; CHECK-NEXT:    [[EXP2:%.*]] = call half @llvm.exp2.f16(half [[ITOFP]])
+; CHECK-NEXT:    ret half [[EXP2]]
+;
+  %itofp = sitofp i8 %x to half
+  %exp2 = call half @llvm.exp2.f16(half %itofp)
+  ret half %exp2
+}
+
+define double @exp2_f64_sitofp_i8(i8 %x) {
+; LDEXP-LABEL: define double @exp2_f64_sitofp_i8(
+; LDEXP-SAME: i8 [[X:%.*]]) {
+; LDEXP-NEXT:    [[TMP1:%.*]] = sext i8 [[X]] to i32
+; LDEXP-NEXT:    [[LDEXP:%.*]] = call double @llvm.ldexp.f64.i32(double 1.000000e+00, i32 [[TMP1]])
+; LDEXP-NEXT:    ret double [[LDEXP]]
+;
+; NOLDEXP-LABEL: define double @exp2_f64_sitofp_i8(
+; NOLDEXP-SAME: i8 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i8 [[X]] to double
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = call double @llvm.exp2.f64(double [[ITOFP]])
+; NOLDEXP-NEXT:    ret double [[EXP2]]
+;
+  %itofp = sitofp i8 %x to double
+  %exp2 = call double @llvm.exp2.f64(double %itofp)
+  ret double %exp2
+}
+
+define fp128 @exp2_fp128_sitofp_i8(i8 %x) {
+; LDEXP-LABEL: define fp128 @exp2_fp128_sitofp_i8(
+; LDEXP-SAME: i8 [[X:%.*]]) {
+; LDEXP-NEXT:    [[TMP1:%.*]] = sext i8 [[X]] to i32
+; LDEXP-NEXT:    [[LDEXPL:%.*]] = call fp128 @llvm.ldexp.f128.i32(fp128 0xL00000000000000003FFF000000000000, i32 [[TMP1]])
+; LDEXP-NEXT:    ret fp128 [[LDEXPL]]
+;
+; NOLDEXP-LABEL: define fp128 @exp2_fp128_sitofp_i8(
+; NOLDEXP-SAME: i8 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i8 [[X]] to fp128
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = call fp128 @llvm.exp2.f128(fp128 [[ITOFP]])
+; NOLDEXP-NEXT:    ret fp128 [[EXP2]]
+;
+  %itofp = sitofp i8 %x to fp128
+  %exp2 = call fp128 @llvm.exp2.fp128(fp128 %itofp)
+  ret fp128 %exp2
+}
+
+define <vscale x 4 x float> @exp2_nxv4f32_sitofp_i8(<vscale x 4 x i8> %x) {
+; CHECK-LABEL: define <vscale x 4 x float> @exp2_nxv4f32_sitofp_i8(
+; CHECK-SAME: <vscale x 4 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[ITOFP:%.*]] = sitofp <vscale x 4 x i8> [[X]] to <vscale x 4 x float>
+; CHECK-NEXT:    [[EXP2:%.*]] = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> [[ITOFP]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[EXP2]]
+;
+  %itofp = sitofp <vscale x 4 x i8> %x to <vscale x 4 x float>
+  %exp2 = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %itofp)
+  ret <vscale x 4 x float> %exp2
+}
+
+; FIXME: This asserts
+; define bfloat @exp2_bf16_sitofp_i8(i8 %x) {
+;   %itofp = sitofp i8 %x to bfloat
+;   %exp2 = call bfloat @llvm.exp2.bf16(bfloat %itofp)
+;   ret bfloat %exp2
+; }
+
+; FIXME: This asserts
+; define ppc_fp128 @exp2_ppc_fp128_sitofp_i8(i8 %x) {
+;   %itofp = sitofp i8 %x to ppc_fp128
+;   %exp2 = call ppc_fp128 @llvm.exp2.ppcf128(ppc_fp128 %itofp)
+;   ret ppc_fp128 %exp2
+; }
+
+; FIXME: This asserts
+; define x86_fp80 @exp2_x86_fp80_sitofp_i8(i8 %x) {
+;   %itofp = sitofp i8 %x to x86_fp80
+;   %exp2 = call x86_fp80 @llvm.exp2.f80(x86_fp80 %itofp)
+;   ret x86_fp80 %exp2
+; }
-- 
GitLab


From e6216906f528b948018b883068cef0fd4157bfd1 Mon Sep 17 00:00:00 2001
From: Christian Ulmann <christianulmann@gmail.com>
Date: Wed, 15 May 2024 08:31:09 +0200
Subject: [PATCH 0812/1206] [MLIR][LLVM] Improve atomic verifier to properly
 support larger types (#92120)

This commit extends the verifier for atomics to properly verify larger
types. Beforehand, the verifier strictly rejected larger integer types,
while it now consults the data layout to determine if their bitsize is a
power of two. This behavior reflects what LLVM's verifier is checking
for.
---
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 37 ++++++++++------------
 mlir/test/Dialect/LLVMIR/invalid.mlir      | 14 ++++++++
 mlir/test/Dialect/LLVMIR/roundtrip.mlir    |  6 +++-
 3 files changed, 35 insertions(+), 22 deletions(-)

diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 7d33d05feb65..dcf3f3b52a60 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -839,25 +839,19 @@ void LoadOp::getEffects(
 }
 
 /// Returns true if the given type is supported by atomic operations. All
-/// integer and float types with limited bit width are supported. Additionally,
-/// depending on the operation pointers may be supported as well.
-static bool isTypeCompatibleWithAtomicOp(Type type, bool isPointerTypeAllowed) {
-  if (llvm::isa<LLVMPointerType>(type))
-    return isPointerTypeAllowed;
-
-  std::optional<unsigned> bitWidth;
-  if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
+/// integer, float, and pointer types with a power-of-two bitsize and a minimal
+/// size of 8 bits are supported.
+static bool isTypeCompatibleWithAtomicOp(Type type,
+                                         const DataLayout &dataLayout) {
+  if (!isa<IntegerType, LLVMPointerType>(type))
     if (!isCompatibleFloatingPointType(type))
       return false;
-    bitWidth = floatType.getWidth();
-  }
-  if (auto integerType = llvm::dyn_cast<IntegerType>(type))
-    bitWidth = integerType.getWidth();
-  // The type is neither an integer, float, or pointer type.
-  if (!bitWidth)
+
+  llvm::TypeSize bitWidth = dataLayout.getTypeSizeInBits(type);
+  if (bitWidth.isScalable())
     return false;
-  return *bitWidth == 8 || *bitWidth == 16 || *bitWidth == 32 ||
-         *bitWidth == 64;
+  // Needs to be at least 8 bits and a power of two.
+  return bitWidth >= 8 && (bitWidth & (bitWidth - 1)) == 0;
 }
 
 /// Verifies the attributes and the type of atomic memory access operations.
@@ -865,8 +859,8 @@ template <typename OpTy>
 LogicalResult verifyAtomicMemOp(OpTy memOp, Type valueType,
                                 ArrayRef<AtomicOrdering> unsupportedOrderings) {
   if (memOp.getOrdering() != AtomicOrdering::not_atomic) {
-    if (!isTypeCompatibleWithAtomicOp(valueType,
-                                      /*isPointerTypeAllowed=*/true))
+    DataLayout dataLayout = DataLayout::closest(memOp);
+    if (!isTypeCompatibleWithAtomicOp(valueType, dataLayout))
       return memOp.emitOpError("unsupported type ")
              << valueType << " for atomic access";
     if (llvm::is_contained(unsupportedOrderings, memOp.getOrdering()))
@@ -2694,7 +2688,8 @@ LogicalResult AtomicRMWOp::verify() {
     if (!mlir::LLVM::isCompatibleFloatingPointType(valType))
       return emitOpError("expected LLVM IR floating point type");
   } else if (getBinOp() == AtomicBinOp::xchg) {
-    if (!isTypeCompatibleWithAtomicOp(valType, /*isPointerTypeAllowed=*/true))
+    DataLayout dataLayout = DataLayout::closest(*this);
+    if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
       return emitOpError("unexpected LLVM IR type for 'xchg' bin_op");
   } else {
     auto intType = llvm::dyn_cast<IntegerType>(valType);
@@ -2741,8 +2736,8 @@ LogicalResult AtomicCmpXchgOp::verify() {
   if (!ptrType)
     return emitOpError("expected LLVM IR pointer type for operand #0");
   auto valType = getVal().getType();
-  if (!isTypeCompatibleWithAtomicOp(valType,
-                                    /*isPointerTypeAllowed=*/true))
+  DataLayout dataLayout = DataLayout::closest(*this);
+  if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
     return emitOpError("unexpected LLVM IR type");
   if (getSuccessOrdering() < AtomicOrdering::monotonic ||
       getFailureOrdering() < AtomicOrdering::monotonic)
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 0914f0023210..a1d340910948 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -160,6 +160,13 @@ func.func @load_unsupported_type(%ptr : !llvm.ptr) {
 
 // -----
 
+func.func @load_unsupported_type(%ptr : !llvm.ptr) {
+  // expected-error@below {{unsupported type 'i33' for atomic access}}
+  %1 = llvm.load %ptr atomic monotonic {alignment = 16 : i64} : !llvm.ptr -> i33
+}
+
+// -----
+
 func.func @load_unaligned_atomic(%ptr : !llvm.ptr) {
   // expected-error@below {{expected alignment for atomic access}}
   %1 = llvm.load %ptr atomic monotonic : !llvm.ptr -> f32
@@ -195,6 +202,13 @@ func.func @store_unsupported_type(%val : i1, %ptr : !llvm.ptr) {
 
 // -----
 
+func.func @store_unsupported_type(%val : i48, %ptr : !llvm.ptr) {
+  // expected-error@below {{unsupported type 'i48' for atomic access}}
+  llvm.store %val, %ptr atomic monotonic {alignment = 16 : i64} : i48, !llvm.ptr
+}
+
+// -----
+
 func.func @store_unaligned_atomic(%val : f32, %ptr : !llvm.ptr) {
   // expected-error@below {{expected alignment for atomic access}}
   llvm.store %val, %ptr atomic monotonic : f32, !llvm.ptr
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index 410122df1c14..2386dde19301 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -385,15 +385,19 @@ func.func @atomic_load(%ptr : !llvm.ptr) {
   %0 = llvm.load %ptr atomic monotonic {alignment = 4 : i64} : !llvm.ptr -> f32
   // CHECK: llvm.load volatile %{{.*}} atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr -> f32
   %1 = llvm.load volatile %ptr atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr -> f32
+  // CHECK: llvm.load %{{.*}} atomic monotonic {alignment = 4 : i64} : !llvm.ptr -> i128
+  %2 = llvm.load %ptr atomic monotonic {alignment = 4 : i64} : !llvm.ptr -> i128
   llvm.return
 }
 
 // CHECK-LABEL: @atomic_store
-func.func @atomic_store(%val : f32, %ptr : !llvm.ptr) {
+func.func @atomic_store(%val : f32, %large_val : i256, %ptr : !llvm.ptr) {
   // CHECK: llvm.store %{{.*}}, %{{.*}} atomic monotonic {alignment = 4 : i64} : f32, !llvm.ptr
   llvm.store %val, %ptr atomic monotonic {alignment = 4 : i64} : f32, !llvm.ptr
   // CHECK: llvm.store volatile %{{.*}}, %{{.*}} atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : f32, !llvm.ptr
   llvm.store volatile %val, %ptr atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : f32, !llvm.ptr
+  // CHECK: llvm.store %{{.*}}, %{{.*}} atomic monotonic {alignment = 4 : i64} : i256, !llvm.ptr
+  llvm.store %large_val, %ptr atomic monotonic {alignment = 4 : i64} : i256, !llvm.ptr
   llvm.return
 }
 
-- 
GitLab


From d6ee7e8481fbaee30f37d82778ef12e135db5e67 Mon Sep 17 00:00:00 2001
From: Jonas Paulsson <paulson1@linux.ibm.com>
Date: Wed, 15 May 2024 08:36:26 +0200
Subject: [PATCH 0813/1206] [SystemZ] Handle address clobbering in splitMove().
 (#92105)

When expanding an L128 (which is used to reload i128) it is
possible that the quadword destination register clobbers an
address register. This patch adds an assertion against the case
where both of the expanded parts clobber the address, and in the
case where one of the expanded parts do so puts it last.

Fixes #91437
---
 llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp  | 65 +++++++++++--------
 .../CodeGen/SystemZ/splitMove_addressReg.mir  | 26 ++++++++
 2 files changed, 65 insertions(+), 26 deletions(-)
 create mode 100644 llvm/test/CodeGen/SystemZ/splitMove_addressReg.mir

diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index 0a29b4f79c7d..16bbfd44ef8a 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -70,49 +70,62 @@ void SystemZInstrInfo::splitMove(MachineBasicBlock::iterator MI,
   MachineBasicBlock *MBB = MI->getParent();
   MachineFunction &MF = *MBB->getParent();
 
-  // Get two load or store instructions.  Use the original instruction for one
-  // of them (arbitrarily the second here) and create a clone for the other.
-  MachineInstr *EarlierMI = MF.CloneMachineInstr(&*MI);
-  MBB->insert(MI, EarlierMI);
+  // Get two load or store instructions.  Use the original instruction for
+  // one of them and create a clone for the other.
+  MachineInstr *HighPartMI = MF.CloneMachineInstr(&*MI);
+  MachineInstr *LowPartMI = &*MI;
+  MBB->insert(LowPartMI, HighPartMI);
 
   // Set up the two 64-bit registers and remember super reg and its flags.
-  MachineOperand &HighRegOp = EarlierMI->getOperand(0);
-  MachineOperand &LowRegOp = MI->getOperand(0);
+  MachineOperand &HighRegOp = HighPartMI->getOperand(0);
+  MachineOperand &LowRegOp = LowPartMI->getOperand(0);
   Register Reg128 = LowRegOp.getReg();
   unsigned Reg128Killed = getKillRegState(LowRegOp.isKill());
   unsigned Reg128Undef  = getUndefRegState(LowRegOp.isUndef());
   HighRegOp.setReg(RI.getSubReg(HighRegOp.getReg(), SystemZ::subreg_h64));
   LowRegOp.setReg(RI.getSubReg(LowRegOp.getReg(), SystemZ::subreg_l64));
 
-  if (MI->mayStore()) {
-    // Add implicit uses of the super register in case one of the subregs is
-    // undefined. We could track liveness and skip storing an undefined
-    // subreg, but this is hopefully rare (discovered with llvm-stress).
-    // If Reg128 was killed, set kill flag on MI.
-    unsigned Reg128UndefImpl = (Reg128Undef | RegState::Implicit);
-    MachineInstrBuilder(MF, EarlierMI).addReg(Reg128, Reg128UndefImpl);
-    MachineInstrBuilder(MF, MI).addReg(Reg128, (Reg128UndefImpl | Reg128Killed));
-  }
-
   // The address in the first (high) instruction is already correct.
   // Adjust the offset in the second (low) instruction.
-  MachineOperand &HighOffsetOp = EarlierMI->getOperand(2);
-  MachineOperand &LowOffsetOp = MI->getOperand(2);
+  MachineOperand &HighOffsetOp = HighPartMI->getOperand(2);
+  MachineOperand &LowOffsetOp = LowPartMI->getOperand(2);
   LowOffsetOp.setImm(LowOffsetOp.getImm() + 8);
 
-  // Clear the kill flags on the registers in the first instruction.
-  if (EarlierMI->getOperand(0).isReg() && EarlierMI->getOperand(0).isUse())
-    EarlierMI->getOperand(0).setIsKill(false);
-  EarlierMI->getOperand(1).setIsKill(false);
-  EarlierMI->getOperand(3).setIsKill(false);
-
   // Set the opcodes.
   unsigned HighOpcode = getOpcodeForOffset(NewOpcode, HighOffsetOp.getImm());
   unsigned LowOpcode = getOpcodeForOffset(NewOpcode, LowOffsetOp.getImm());
   assert(HighOpcode && LowOpcode && "Both offsets should be in range");
+  HighPartMI->setDesc(get(HighOpcode));
+  LowPartMI->setDesc(get(LowOpcode));
+
+  MachineInstr *FirstMI = HighPartMI;
+  if (MI->mayStore()) {
+    FirstMI->getOperand(0).setIsKill(false);
+    // Add implicit uses of the super register in case one of the subregs is
+    // undefined. We could track liveness and skip storing an undefined
+    // subreg, but this is hopefully rare (discovered with llvm-stress).
+    // If Reg128 was killed, set kill flag on MI.
+    unsigned Reg128UndefImpl = (Reg128Undef | RegState::Implicit);
+    MachineInstrBuilder(MF, HighPartMI).addReg(Reg128, Reg128UndefImpl);
+    MachineInstrBuilder(MF, LowPartMI).addReg(Reg128, (Reg128UndefImpl | Reg128Killed));
+  } else {
+    // If HighPartMI clobbers any of the address registers, it needs to come
+    // after LowPartMI.
+    auto overlapsAddressReg = [&](Register Reg) -> bool {
+      return RI.regsOverlap(Reg, MI->getOperand(1).getReg()) ||
+             RI.regsOverlap(Reg, MI->getOperand(3).getReg());
+    };
+    if (overlapsAddressReg(HighRegOp.getReg())) {
+      assert(!overlapsAddressReg(LowRegOp.getReg()) &&
+             "Both loads clobber address!");
+      MBB->splice(HighPartMI, MBB, LowPartMI);
+      FirstMI = LowPartMI;
+    }
+  }
 
-  EarlierMI->setDesc(get(HighOpcode));
-  MI->setDesc(get(LowOpcode));
+  // Clear the kill flags on the address registers in the first instruction.
+  FirstMI->getOperand(1).setIsKill(false);
+  FirstMI->getOperand(3).setIsKill(false);
 }
 
 // Split ADJDYNALLOC instruction MI.
diff --git a/llvm/test/CodeGen/SystemZ/splitMove_addressReg.mir b/llvm/test/CodeGen/SystemZ/splitMove_addressReg.mir
new file mode 100644
index 000000000000..64ed2d8f2c00
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/splitMove_addressReg.mir
@@ -0,0 +1,26 @@
+# RUN: llc -mtriple=s390x-linux-gnu -run-pass=postrapseudos \
+# RUN:   %s -o - -verify-machineinstrs | FileCheck %s
+#
+# Test that a L128 reload do not overwrite an address register prematurely
+# after being split into two LGs.
+
+--- |
+  target triple = "s390x-unknown-unknown"
+
+  define void @fun() {
+    ret void
+  }
+
+...
+
+# CHECK: name: fun
+
+---
+name:            'fun'
+body:             |
+  bb.0:
+    liveins: $r4d, $r15d
+    $r4q = L128 $r15d, 14920, killed $r4d
+    Return
+
+...
-- 
GitLab


From 45726c1a3a3d89ff9f6ebe657c3cb7bcd59b88db Mon Sep 17 00:00:00 2001
From: Daniel Kiss <daniel.kiss@arm.com>
Date: Wed, 15 May 2024 08:40:16 +0200
Subject: [PATCH 0814/1206] [LLVM] Make sanitizers respect the
 disable_santizer_instrumentation attribute. (#91732)

`disable_sanitizer_instrumetation` is attached to functions that shall
not be instrumented e.g. ifunc resolver because those run before
everything is initialised.
Some sanitizer already handles this attribute, this patch adds it to
DataFLow and Coverage too.
---
 .../Instrumentation/DataFlowSanitizer.cpp     |  3 +-
 .../Instrumentation/SanitizerCoverage.cpp     |  2 +
 ...aflow-disable-sanitizer-instrumentation.ll | 47 +++++++++++++++++++
 ...erage-disable-sanitizer-instrumentation.ll | 46 ++++++++++++++++++
 4 files changed, 97 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Instrumentation/DataFlowSanitizer/dataflow-disable-sanitizer-instrumentation.ll
 create mode 100644 llvm/test/Instrumentation/SanitizerCoverage/coverage-disable-sanitizer-instrumentation.ll

diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 851edb4ce829..20d11e0ab55f 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -1546,7 +1546,8 @@ bool DataFlowSanitizer::runImpl(
   SmallPtrSet<Constant *, 1> PersonalityFns;
   for (Function &F : M)
     if (!F.isIntrinsic() && !DFSanRuntimeFunctions.contains(&F) &&
-        !LibAtomicFunction(F)) {
+        !LibAtomicFunction(F) &&
+        !F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation)) {
       FnsToInstrument.push_back(&F);
       if (F.hasPersonalityFn())
         PersonalityFns.insert(F.getPersonalityFn()->stripPointerCasts());
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index 56d4907ae47a..6a89cee9aaf6 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -631,6 +631,8 @@ void ModuleSanitizerCoverage::instrumentFunction(Function &F) {
     return;
   if (F.hasFnAttribute(Attribute::NoSanitizeCoverage))
     return;
+  if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
+    return;
   if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) {
     SplitAllCriticalEdges(
         F, CriticalEdgeSplittingOptions().setIgnoreUnreachableDests());
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/dataflow-disable-sanitizer-instrumentation.ll b/llvm/test/Instrumentation/DataFlowSanitizer/dataflow-disable-sanitizer-instrumentation.ll
new file mode 100644
index 000000000000..3fb922736d02
--- /dev/null
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/dataflow-disable-sanitizer-instrumentation.ll
@@ -0,0 +1,47 @@
+
+; This test checks that we are not instrumenting sanitizer code.
+; RUN: opt < %s -passes='module(msan)' -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function with sanitize_memory is instrumented.
+; Function Attrs: nounwind uwtable
+define void @instr_sa(ptr %a) sanitize_memory {
+entry:
+  %tmp1 = load i32, ptr %a, align 4
+  %tmp2 = add i32 %tmp1,  1
+  store i32 %tmp2, ptr %a, align 4
+  ret void
+}
+
+; CHECK-LABEL: @instr_sa
+; CHECK: %0 = load i64, ptr @__msan_param_tls
+
+
+; Function with disable_sanitizer_instrumentation is not instrumented.
+; Function Attrs: nounwind uwtable
+define void @noinstr_dsi(ptr %a) disable_sanitizer_instrumentation {
+entry:
+  %tmp1 = load i32, ptr %a, align 4
+  %tmp2 = add i32 %tmp1,  1
+  store i32 %tmp2, ptr %a, align 4
+  ret void
+}
+
+; CHECK-LABEL: @noinstr_dsi
+; CHECK-NOT: %0 = load i64, ptr @__msan_param_tls
+
+
+; disable_sanitizer_instrumentation takes precedence over sanitize_memory.
+; Function Attrs: nounwind uwtable
+define void @noinstr_dsi_sa(ptr %a) disable_sanitizer_instrumentation sanitize_memory {
+entry:
+  %tmp1 = load i32, ptr %a, align 4
+  %tmp2 = add i32 %tmp1,  1
+  store i32 %tmp2, ptr %a, align 4
+  ret void
+}
+
+; CHECK-LABEL: @noinstr_dsi_sa
+; CHECK-NOT: %0 = load i64, ptr @__msan_param_tls
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage-disable-sanitizer-instrumentation.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage-disable-sanitizer-instrumentation.ll
new file mode 100644
index 000000000000..dc3d48622015
--- /dev/null
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage-disable-sanitizer-instrumentation.ll
@@ -0,0 +1,46 @@
+; This test checks that we are not instrumenting sanitizer code.
+; RUN: opt < %s -passes='module(sancov-module)' -sanitizer-coverage-level=3 -sanitizer-coverage-control-flow -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function with sanitize_address is instrumented.
+; Function Attrs: nounwind uwtable
+define void @instr_sa(ptr %a) sanitize_address {
+entry:
+  %tmp1 = load i32, ptr %a, align 4
+  %tmp2 = add i32 %tmp1,  1
+  store i32 %tmp2, ptr %a, align 4
+  ret void
+}
+
+; CHECK-LABEL: @instr_sa
+; CHECK: call void @__sanitizer_cov_trace_pc_guard(
+
+
+; Function with disable_sanitizer_instrumentation is not instrumented.
+; Function Attrs: nounwind uwtable
+define void @noinstr_dsi(ptr %a) disable_sanitizer_instrumentation {
+entry:
+  %tmp1 = load i32, ptr %a, align 4
+  %tmp2 = add i32 %tmp1,  1
+  store i32 %tmp2, ptr %a, align 4
+  ret void
+}
+
+; CHECK-LABEL: @noinstr_dsi
+; CHECK-NOT: call void @__sanitizer_cov_trace_pc_guard(
+
+
+; disable_sanitizer_instrumentation takes precedence over sanitize_address.
+; Function Attrs: nounwind uwtable
+define void @noinstr_dsi_sa(ptr %a) disable_sanitizer_instrumentation sanitize_address {
+entry:
+  %tmp1 = load i32, ptr %a, align 4
+  %tmp2 = add i32 %tmp1,  1
+  store i32 %tmp2, ptr %a, align 4
+  ret void
+}
+
+; CHECK-LABEL: @noinstr_dsi_sa
+; CHECK-NOT: call void @__sanitizer_cov_trace_pc_guard(
-- 
GitLab


From 4688df68f9d022dd8bc102675a9e86ad274355d6 Mon Sep 17 00:00:00 2001
From: Stephan Bergmann <sbergman@redhat.com>
Date: Wed, 15 May 2024 08:58:14 +0200
Subject: [PATCH 0815/1206] Avoid partial munmap (#92109)

...which caused issues like

> ==42==ERROR: AddressSanitizer failed to deallocate 0x32 (50) bytes at
address 0x117e0000 (error code: 28)
> ==42==Cannot dump memory map on emscriptenAddressSanitizer: CHECK
failed: sanitizer_common.cpp:81 "((0 && "unable to unmmap")) != (0)"
(0x0, 0x0) (tid=288045824)
> #0 0x14f73b0c in __asan::CheckUnwind()+0x14f73b0c
(this.program+0x14f73b0c)
> #1 0x14f8a3c2 in __sanitizer::CheckFailed(char const*, int, char
const*, unsigned long long, unsigned long long)+0x14f8a3c2
(this.program+0x14f8a3c2)
> #2 0x14f7d6e1 in __sanitizer::ReportMunmapFailureAndDie(void*,
unsigned long, int, bool)+0x14f7d6e1 (this.program+0x14f7d6e1)
> #3 0x14f81fbd in __sanitizer::UnmapOrDie(void*, unsigned
long)+0x14f81fbd (this.program+0x14f81fbd)
> #4 0x14f875df in __sanitizer::SuppressionContext::ParseFromFile(char
const*)+0x14f875df (this.program+0x14f875df)
> #5 0x14f74eab in __asan::InitializeSuppressions()+0x14f74eab
(this.program+0x14f74eab)
> #6 0x14f73a1a in __asan::AsanInitInternal()+0x14f73a1a
(this.program+0x14f73a1a)

when trying to use an ASan suppressions file under Emscripten: Even
though it would be considered OK by SUSv4, the Emscripten runtime states
"We don't support partial munmapping" (see

<https://github.com/emscripten-core/emscripten/commit/f4115eb2c36bac741685300273cb9fa186d719e1>
"Implement MAP_ANONYMOUS on top of malloc in STANDALONE_WASM mode
(#16289)").

Co-authored-by: Stephan Bergmann <stephan.bergmann@allotropia.de>
---
 compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp
index 9c8c4bf9d1a4..62ebbb38feae 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp
@@ -86,7 +86,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {
   }
 
   Parse(file_contents);
-  UnmapOrDie(file_contents, contents_size);
+  UnmapOrDie(file_contents, buffer_size);
 }
 
 bool SuppressionContext::Match(const char *str, const char *type,
-- 
GitLab


From 73324cbc9c5892541aa82d466799748b435ece29 Mon Sep 17 00:00:00 2001
From: Enna1 <xumingjie.enna1@bytedance.com>
Date: Wed, 15 May 2024 15:04:34 +0800
Subject: [PATCH 0816/1206] [ASan] Remove
 COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION. (#91832)

Since the set of COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION is removed in
commit 8421fa5d536aadf42c0e54c566bc439a40ebdb8e,
cleanup the use of COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION.
---
 compiler-rt/lib/asan/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt
index 601750f72175..463ea233b37a 100644
--- a/compiler-rt/lib/asan/CMakeLists.txt
+++ b/compiler-rt/lib/asan/CMakeLists.txt
@@ -88,7 +88,7 @@ set(ASAN_CFLAGS ${SANITIZER_COMMON_CFLAGS})
 
 append_list_if(MSVC /Zl ASAN_CFLAGS)
 
-set(ASAN_COMMON_DEFINITIONS ${COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION})
+set(ASAN_COMMON_DEFINITIONS "")
 
 append_rtti_flag(OFF ASAN_CFLAGS)
 
-- 
GitLab


From 7f3ac51b946bf6d6fa8c8443457ebee219879302 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Tue, 14 May 2024 16:41:28 +0200
Subject: [PATCH 0817/1206] [clang][Interp] Only accept constant variables in
 c++98

---
 clang/lib/AST/Interp/Interp.cpp | 14 +++++++-------
 clang/test/AST/Interp/cxx98.cpp |  5 +++++
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index 2607e0743251..3e4da487e43c 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -302,7 +302,9 @@ bool CheckConstant(InterpState &S, CodePtr OpPC, const Descriptor *Desc) {
 
     QualType T = VD->getType();
     if (S.getLangOpts().CPlusPlus && !S.getLangOpts().CPlusPlus11)
-      return T->isSignedIntegerOrEnumerationType() || T->isUnsignedIntegerOrEnumerationType();
+      return (T->isSignedIntegerOrEnumerationType() ||
+              T->isUnsignedIntegerOrEnumerationType()) &&
+             T.isConstQualified();
 
     if (T.isConstQualified())
       return true;
@@ -316,12 +318,10 @@ bool CheckConstant(InterpState &S, CodePtr OpPC, const Descriptor *Desc) {
     return false;
   };
 
-  if (const auto *D = Desc->asValueDecl()) {
-    if (const auto *VD = dyn_cast<VarDecl>(D);
-        VD && VD->hasGlobalStorage() && !IsConstType(VD)) {
-      diagnoseNonConstVariable(S, OpPC, VD);
-      return S.inConstantContext();
-    }
+  if (const auto *D = Desc->asVarDecl();
+      D && D->hasGlobalStorage() && !IsConstType(D)) {
+    diagnoseNonConstVariable(S, OpPC, D);
+    return S.inConstantContext();
   }
 
   return true;
diff --git a/clang/test/AST/Interp/cxx98.cpp b/clang/test/AST/Interp/cxx98.cpp
index ba6bcd97d920..be81735329db 100644
--- a/clang/test/AST/Interp/cxx98.cpp
+++ b/clang/test/AST/Interp/cxx98.cpp
@@ -45,3 +45,8 @@ struct C0 {
 };
 const int c0_test = C0::Data<int*>;
 _Static_assert(c0_test == 0, "");
+
+
+int a = 0; // both-note {{declared here}}
+_Static_assert(a == 0, ""); // both-error {{static assertion expression is not an integral constant expression}} \
+                            // both-note {{read of non-const variable 'a' is not allowed in a constant expression}}
-- 
GitLab


From d12c48cad52798f4846dd8ef882af0f854118d16 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Wed, 15 May 2024 10:02:24 +0200
Subject: [PATCH 0818/1206] [lldb/aarch64] Allow unaligned PC addresses below a
 trap handler (#92093)

The stack validation heuristic is counter-productive in this case, as
the unaligned address is most likely the thing that caused the signal in
the first place.
---
 lldb/source/Target/UnwindLLDB.cpp             |  7 ++++-
 .../Shell/Unwind/Inputs/unaligned-pc-sigbus.c | 21 +++++++++++++
 .../Shell/Unwind/unaligned-pc-sigbus.test     | 31 +++++++++++++++++++
 3 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100644 lldb/test/Shell/Unwind/Inputs/unaligned-pc-sigbus.c
 create mode 100644 lldb/test/Shell/Unwind/unaligned-pc-sigbus.test

diff --git a/lldb/source/Target/UnwindLLDB.cpp b/lldb/source/Target/UnwindLLDB.cpp
index 1d8bf2f88ae6..f43e940492b0 100644
--- a/lldb/source/Target/UnwindLLDB.cpp
+++ b/lldb/source/Target/UnwindLLDB.cpp
@@ -261,7 +261,12 @@ UnwindLLDB::CursorSP UnwindLLDB::GetOneMoreFrame(ABI *abi) {
               cur_idx < 100 ? cur_idx : 100, "", cur_idx);
     return nullptr;
   }
-  if (abi && !abi->CodeAddressIsValid(cursor_sp->start_pc)) {
+
+  // Invalid code addresses should not appear on the stack *unless* we're
+  // directly below a trap handler frame (in this case, the invalid address is
+  // likely the cause of the trap).
+  if (abi && !abi->CodeAddressIsValid(cursor_sp->start_pc) &&
+      !prev_frame->reg_ctx_lldb_sp->IsTrapHandlerFrame()) {
     // If the RegisterContextUnwind has a fallback UnwindPlan, it will switch to
     // that and return true.  Subsequent calls to TryFallbackUnwindPlan() will
     // return false.
diff --git a/lldb/test/Shell/Unwind/Inputs/unaligned-pc-sigbus.c b/lldb/test/Shell/Unwind/Inputs/unaligned-pc-sigbus.c
new file mode 100644
index 000000000000..b4818de3b7fb
--- /dev/null
+++ b/lldb/test/Shell/Unwind/Inputs/unaligned-pc-sigbus.c
@@ -0,0 +1,21 @@
+#include <signal.h>
+#include <stdint.h>
+#include <unistd.h>
+
+void sigbus_handler(int signo) { _exit(47); }
+
+int target_function() { return 47; }
+
+int main() {
+  signal(SIGBUS, sigbus_handler);
+
+  // Generate a SIGBUS by deliverately calling through an unaligned function
+  // pointer.
+  union {
+    int (*t)();
+    uintptr_t p;
+  } u;
+  u.t = target_function;
+  u.p |= 1;
+  return u.t();
+}
diff --git a/lldb/test/Shell/Unwind/unaligned-pc-sigbus.test b/lldb/test/Shell/Unwind/unaligned-pc-sigbus.test
new file mode 100644
index 000000000000..5ebfba54301e
--- /dev/null
+++ b/lldb/test/Shell/Unwind/unaligned-pc-sigbus.test
@@ -0,0 +1,31 @@
+# REQUIRES: (target-aarch64 || target-arm) && native
+# UNSUPPORTED: system-windows
+# llvm.org/pr91610, rdar://128031075
+# XFAIL: system-darwin
+
+# RUN: %clang_host %S/Inputs/unaligned-pc-sigbus.c -o %t
+# RUN: %lldb -s %s -o exit %t | FileCheck %s
+
+# Convert EXC_BAD_ACCESS into SIGBUS on darwin.
+settings set platform.plugin.darwin.ignored-exceptions EXC_BAD_ACCESS
+
+breakpoint set -n sigbus_handler
+# CHECK: Breakpoint 1: where = {{.*}}`sigbus_handler
+
+run
+# CHECK: thread #1, {{.*}} stop reason = signal SIGBUS
+
+thread backtrace
+# CHECK: (lldb) thread backtrace
+# CHECK: frame #0: [[TARGET:0x[0-9a-fA-F]*]] {{.*}}`target_function
+
+continue
+# CHECK: thread #1, {{.*}} stop reason = breakpoint 1
+
+
+thread backtrace
+# CHECK: (lldb) thread backtrace
+# CHECK: frame #0: {{.*}}`sigbus_handler
+# Unknown number of signal trampoline frames
+# CHECK: frame #{{[0-9]+}}: [[TARGET]] {{.*}}`target_function
+
-- 
GitLab


From 6479e3cb66895754089dc017a33478e9eb4b8d65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Wed, 15 May 2024 08:05:24 +0200
Subject: [PATCH 0819/1206] [clang][Interp] Use proper type for non-primitive
 reference dummies

---
 clang/lib/AST/Interp/Program.cpp    |  3 +--
 clang/test/AST/Interp/bitfields.cpp | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/Interp/Program.cpp b/clang/lib/AST/Interp/Program.cpp
index 31a64e13d2b1..e3d48d5a8ddb 100644
--- a/clang/lib/AST/Interp/Program.cpp
+++ b/clang/lib/AST/Interp/Program.cpp
@@ -152,8 +152,7 @@ std::optional<unsigned> Program::getOrCreateDummy(const ValueDecl *VD) {
   if (std::optional<PrimType> T = Ctx.classify(QT))
     Desc = createDescriptor(VD, *T, std::nullopt, true, false);
   else
-    Desc = createDescriptor(VD, VD->getType().getTypePtr(), std::nullopt, true,
-                            false);
+    Desc = createDescriptor(VD, QT.getTypePtr(), std::nullopt, true, false);
   if (!Desc)
     Desc = allocateDescriptor(VD);
 
diff --git a/clang/test/AST/Interp/bitfields.cpp b/clang/test/AST/Interp/bitfields.cpp
index d3a8a083063a..5fc34bb1229d 100644
--- a/clang/test/AST/Interp/bitfields.cpp
+++ b/clang/test/AST/Interp/bitfields.cpp
@@ -102,3 +102,24 @@ namespace Compound {
   }
   static_assert(div() == 1, "");
 }
+
+namespace test0 {
+  extern int int_source();
+  struct A {
+    int aField;
+    int bField;
+  };
+
+  struct B {
+    int onebit : 2;
+    int twobit : 6;
+    int intField;
+  };
+
+  struct C : A, B {
+  };
+
+  void b(C &c) {
+    c.onebit = int_source();
+  }
+}
-- 
GitLab


From 1d43ec8191e55d6efd552a1510ce63dbdea00cc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Wed, 15 May 2024 08:21:17 +0200
Subject: [PATCH 0820/1206] [clang][Interp][NFC] Remove unnecessary if
 condition

This is already in a if(isBlockPointer()) block.
---
 clang/lib/AST/Interp/Pointer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp
index d2e34f2c7f09..ee8cedccb8d4 100644
--- a/clang/lib/AST/Interp/Pointer.cpp
+++ b/clang/lib/AST/Interp/Pointer.cpp
@@ -191,7 +191,7 @@ void Pointer::print(llvm::raw_ostream &OS) const {
     else
       OS << Offset << ", ";
 
-    if (isBlockPointer() && PointeeStorage.BS.Pointee)
+    if (PointeeStorage.BS.Pointee)
       OS << PointeeStorage.BS.Pointee->getSize();
     else
       OS << "nullptr";
-- 
GitLab


From afba3daf822c839db1be40464041307679c803a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Wed, 15 May 2024 10:16:23 +0200
Subject: [PATCH 0821/1206] [clang][Interp] Add basic support for
 AddrLabelExprs

Just create a local variable for them.
---
 clang/lib/AST/Interp/ByteCodeExprGen.cpp | 10 ++++++++++
 clang/lib/AST/Interp/ByteCodeExprGen.h   |  1 +
 clang/test/AST/Interp/c.c                |  5 +++++
 3 files changed, 16 insertions(+)

diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 7b10482dff23..1da74ac7c8bd 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -2488,6 +2488,16 @@ bool ByteCodeExprGen<Emitter>::VisitRecoveryExpr(const RecoveryExpr *E) {
   return this->emitError(E);
 }
 
+template <class Emitter>
+bool ByteCodeExprGen<Emitter>::VisitAddrLabelExpr(const AddrLabelExpr *E) {
+  assert(E->getType()->isVoidPointerType());
+
+  unsigned Offset = allocateLocalPrimitive(
+      E->getLabel(), PT_Ptr, /*IsConst=*/true, /*IsExtended=*/false);
+
+  return this->emitGetLocal(PT_Ptr, Offset, E);
+}
+
 template <class Emitter> bool ByteCodeExprGen<Emitter>::discard(const Expr *E) {
   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/true,
                              /*NewInitializing=*/false);
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.h b/clang/lib/AST/Interp/ByteCodeExprGen.h
index 9f83d173bbae..6039a54d32a5 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.h
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.h
@@ -122,6 +122,7 @@ public:
   bool VisitPseudoObjectExpr(const PseudoObjectExpr *E);
   bool VisitPackIndexingExpr(const PackIndexingExpr *E);
   bool VisitRecoveryExpr(const RecoveryExpr *E);
+  bool VisitAddrLabelExpr(const AddrLabelExpr *E);
 
 protected:
   bool visitExpr(const Expr *E) override;
diff --git a/clang/test/AST/Interp/c.c b/clang/test/AST/Interp/c.c
index 2c675f4418ef..2a75457a4693 100644
--- a/clang/test/AST/Interp/c.c
+++ b/clang/test/AST/Interp/c.c
@@ -273,3 +273,8 @@ int test3(void) {
 /// This tests that we have full type info, even for values we cannot read.
 int dummyarray[5];
 _Static_assert(&dummyarray[0] < &dummyarray[1], ""); // pedantic-warning {{GNU extension}}
+
+void addrlabelexpr(void) {
+ a0: ;
+  static void *ps[] = { &&a0 }; // pedantic-warning {{use of GNU address-of-label extension}}
+}
-- 
GitLab


From ca4a405232cf170f20a2f111bf72beab82095935 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Don=C3=A1t=20Nagy?= <donat.nagy@ericsson.com>
Date: Wed, 15 May 2024 10:53:54 +0200
Subject: [PATCH 0822/1206] [analyzer] Refactor recognition of the errno getter
 functions (#91531)

There are many environments where `errno` is a macro that expands to
something like `(*__errno())` (different standard library
implementations use different names instead of "__errno").

In these environments the ErrnoModeling checker creates a symbolic
region which will be used to represent the return value of this "get the
location of errno" function.

Previously this symbol was only created when the checker was able to
find the declaration of the "get the location of errno" function; but
this commit eliminates the complex logic that was responsible for this
and always creates the symbolic region when `errno` is not available as
a "regular" global variable.

This significantly simplifies a code and only introduces a minimal
performance reduction (one extra symbol) in the case when `errno` is not
declared (neither as a variable nor as a function).


In addition to this simplification, this commit specifies that the
`CallDescription`s for the "get the location of errno" functions are
matched in `CDM::CLibrary` mode. (This was my original goal, but I was
sidetracked by resolving a FIXME above the `CallDescriptionSet` in
`ErrnoModeling.cpp`.)

This change is very close to being NFC, but it fixes weird corner cases
like the handling of a C++ method that happens to be named "__errno()"
(previously it could've been recognized as an errno location getter
function).
---
 .../StaticAnalyzer/Checkers/ErrnoChecker.cpp  |   2 +-
 .../StaticAnalyzer/Checkers/ErrnoModeling.cpp | 131 ++++++------------
 .../StaticAnalyzer/Checkers/ErrnoModeling.h   |   9 +-
 clang/test/Analysis/memory-model.cpp          |  18 +--
 4 files changed, 56 insertions(+), 104 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/ErrnoChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ErrnoChecker.cpp
index 18e718e08553..72fd6781a756 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ErrnoChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ErrnoChecker.cpp
@@ -205,7 +205,7 @@ void ErrnoChecker::checkPreCall(const CallEvent &Call,
   // Probably 'strerror'?
   if (CallF->isExternC() && CallF->isGlobal() &&
       C.getSourceManager().isInSystemHeader(CallF->getLocation()) &&
-      !isErrno(CallF)) {
+      !isErrnoLocationCall(Call)) {
     if (getErrnoState(C.getState()) == MustBeChecked) {
       std::optional<ento::Loc> ErrnoLoc = getErrnoLoc(C.getState());
       assert(ErrnoLoc && "ErrnoLoc should exist if an errno state is set.");
diff --git a/clang/lib/StaticAnalyzer/Checkers/ErrnoModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/ErrnoModeling.cpp
index 1b34ea0e056e..6ffc05f06742 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ErrnoModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ErrnoModeling.cpp
@@ -39,10 +39,15 @@ namespace {
 // Name of the "errno" variable.
 // FIXME: Is there a system where it is not called "errno" but is a variable?
 const char *ErrnoVarName = "errno";
+
 // Names of functions that return a location of the "errno" value.
 // FIXME: Are there other similar function names?
-const char *ErrnoLocationFuncNames[] = {"__errno_location", "___errno",
-                                        "__errno", "_errno", "__error"};
+CallDescriptionSet ErrnoLocationCalls{
+    {CDM::CLibrary, {"__errno_location"}, 0, 0},
+    {CDM::CLibrary, {"___errno"}, 0, 0},
+    {CDM::CLibrary, {"__errno"}, 0, 0},
+    {CDM::CLibrary, {"_errno"}, 0, 0},
+    {CDM::CLibrary, {"__error"}, 0, 0}};
 
 class ErrnoModeling
     : public Checker<check::ASTDecl<TranslationUnitDecl>, check::BeginFunction,
@@ -54,16 +59,10 @@ public:
   void checkLiveSymbols(ProgramStateRef State, SymbolReaper &SR) const;
   bool evalCall(const CallEvent &Call, CheckerContext &C) const;
 
-  // The declaration of an "errno" variable or "errno location" function.
-  mutable const Decl *ErrnoDecl = nullptr;
-
 private:
-  // FIXME: Names from `ErrnoLocationFuncNames` are used to build this set.
-  CallDescriptionSet ErrnoLocationCalls{{{"__errno_location"}, 0, 0},
-                                        {{"___errno"}, 0, 0},
-                                        {{"__errno"}, 0, 0},
-                                        {{"_errno"}, 0, 0},
-                                        {{"__error"}, 0, 0}};
+  // The declaration of an "errno" variable on systems where errno is
+  // represented by a variable (and not a function that queries its location).
+  mutable const VarDecl *ErrnoDecl = nullptr;
 };
 
 } // namespace
@@ -74,9 +73,13 @@ REGISTER_TRAIT_WITH_PROGRAMSTATE(ErrnoRegion, const MemRegion *)
 
 REGISTER_TRAIT_WITH_PROGRAMSTATE(ErrnoState, errno_modeling::ErrnoCheckState)
 
-/// Search for a variable called "errno" in the AST.
-/// Return nullptr if not found.
-static const VarDecl *getErrnoVar(ASTContext &ACtx) {
+void ErrnoModeling::checkASTDecl(const TranslationUnitDecl *D,
+                                 AnalysisManager &Mgr, BugReporter &BR) const {
+  // Try to find the declaration of the external variable `int errno;`.
+  // There are also C library implementations, where the `errno` location is
+  // accessed via a function that returns its address; in those environments
+  // this callback has no effect.
+  ASTContext &ACtx = Mgr.getASTContext();
   IdentifierInfo &II = ACtx.Idents.get(ErrnoVarName);
   auto LookupRes = ACtx.getTranslationUnitDecl()->lookup(&II);
   auto Found = llvm::find_if(LookupRes, [&ACtx](const Decl *D) {
@@ -86,47 +89,8 @@ static const VarDecl *getErrnoVar(ASTContext &ACtx) {
              VD->getType().getCanonicalType() == ACtx.IntTy;
     return false;
   });
-  if (Found == LookupRes.end())
-    return nullptr;
-
-  return cast<VarDecl>(*Found);
-}
-
-/// Search for a function with a specific name that is used to return a pointer
-/// to "errno".
-/// Return nullptr if no such function was found.
-static const FunctionDecl *getErrnoFunc(ASTContext &ACtx) {
-  SmallVector<const Decl *> LookupRes;
-  for (StringRef ErrnoName : ErrnoLocationFuncNames) {
-    IdentifierInfo &II = ACtx.Idents.get(ErrnoName);
-    llvm::append_range(LookupRes, ACtx.getTranslationUnitDecl()->lookup(&II));
-  }
-
-  auto Found = llvm::find_if(LookupRes, [&ACtx](const Decl *D) {
-    if (auto *FD = dyn_cast<FunctionDecl>(D))
-      return ACtx.getSourceManager().isInSystemHeader(FD->getLocation()) &&
-             FD->isExternC() && FD->getNumParams() == 0 &&
-             FD->getReturnType().getCanonicalType() ==
-                 ACtx.getPointerType(ACtx.IntTy);
-    return false;
-  });
-  if (Found == LookupRes.end())
-    return nullptr;
-
-  return cast<FunctionDecl>(*Found);
-}
-
-void ErrnoModeling::checkASTDecl(const TranslationUnitDecl *D,
-                                 AnalysisManager &Mgr, BugReporter &BR) const {
-  // Try to find an usable `errno` value.
-  // It can be an external variable called "errno" or a function that returns a
-  // pointer to the "errno" value. This function can have different names.
-  // The actual case is dependent on the C library implementation, we
-  // can only search for a match in one of these variations.
-  // We assume that exactly one of these cases might be true.
-  ErrnoDecl = getErrnoVar(Mgr.getASTContext());
-  if (!ErrnoDecl)
-    ErrnoDecl = getErrnoFunc(Mgr.getASTContext());
+  if (Found != LookupRes.end())
+    ErrnoDecl = cast<VarDecl>(*Found);
 }
 
 void ErrnoModeling::checkBeginFunction(CheckerContext &C) const {
@@ -136,25 +100,18 @@ void ErrnoModeling::checkBeginFunction(CheckerContext &C) const {
   ASTContext &ACtx = C.getASTContext();
   ProgramStateRef State = C.getState();
 
-  if (const auto *ErrnoVar = dyn_cast_or_null<VarDecl>(ErrnoDecl)) {
-    // There is an external 'errno' variable.
-    // Use its memory region.
-    // The memory region for an 'errno'-like variable is allocated in system
-    // space by MemRegionManager.
-    const MemRegion *ErrnoR =
-        State->getRegion(ErrnoVar, C.getLocationContext());
+  const MemRegion *ErrnoR = nullptr;
+
+  if (ErrnoDecl) {
+    // There is an external 'errno' variable, so we can simply use the memory
+    // region that's associated with it.
+    ErrnoR = State->getRegion(ErrnoDecl, C.getLocationContext());
     assert(ErrnoR && "Memory region should exist for the 'errno' variable.");
-    State = State->set<ErrnoRegion>(ErrnoR);
-    State =
-        errno_modeling::setErrnoValue(State, C, 0, errno_modeling::Irrelevant);
-    C.addTransition(State);
-  } else if (ErrnoDecl) {
-    assert(isa<FunctionDecl>(ErrnoDecl) && "Invalid errno location function.");
-    // There is a function that returns the location of 'errno'.
-    // We must create a memory region for it in system space.
-    // Currently a symbolic region is used with an artifical symbol.
-    // FIXME: It is better to have a custom (new) kind of MemRegion for such
-    // cases.
+  } else {
+    // There is no 'errno' variable, so create a new symbolic memory region
+    // that can be used to model the return value of the "get the location of
+    // errno" internal functions.
+    // NOTE: this `SVal` is created even if errno is not defined or used.
     SValBuilder &SVB = C.getSValBuilder();
     MemRegionManager &RMgr = C.getStateManager().getRegionManager();
 
@@ -162,27 +119,31 @@ void ErrnoModeling::checkBeginFunction(CheckerContext &C) const {
         RMgr.getGlobalsRegion(MemRegion::GlobalSystemSpaceRegionKind);
 
     // Create an artifical symbol for the region.
-    // It is not possible to associate a statement or expression in this case.
+    // Note that it is not possible to associate a statement or expression in
+    // this case and the `symbolTag` (opaque pointer tag) is just the address
+    // of the data member `ErrnoDecl` of the singleton `ErrnoModeling` checker
+    // object.
     const SymbolConjured *Sym = SVB.conjureSymbol(
         nullptr, C.getLocationContext(),
         ACtx.getLValueReferenceType(ACtx.IntTy), C.blockCount(), &ErrnoDecl);
 
     // The symbolic region is untyped, create a typed sub-region in it.
     // The ElementRegion is used to make the errno region a typed region.
-    const MemRegion *ErrnoR = RMgr.getElementRegion(
+    ErrnoR = RMgr.getElementRegion(
         ACtx.IntTy, SVB.makeZeroArrayIndex(),
         RMgr.getSymbolicRegion(Sym, GlobalSystemSpace), C.getASTContext());
-    State = State->set<ErrnoRegion>(ErrnoR);
-    State =
-        errno_modeling::setErrnoValue(State, C, 0, errno_modeling::Irrelevant);
-    C.addTransition(State);
   }
+  assert(ErrnoR);
+  State = State->set<ErrnoRegion>(ErrnoR);
+  State =
+      errno_modeling::setErrnoValue(State, C, 0, errno_modeling::Irrelevant);
+  C.addTransition(State);
 }
 
 bool ErrnoModeling::evalCall(const CallEvent &Call, CheckerContext &C) const {
   // Return location of "errno" at a call to an "errno address returning"
   // function.
-  if (ErrnoLocationCalls.contains(Call)) {
+  if (errno_modeling::isErrnoLocationCall(Call)) {
     ProgramStateRef State = C.getState();
 
     const MemRegion *ErrnoR = State->get<ErrnoRegion>();
@@ -260,14 +221,8 @@ ProgramStateRef clearErrnoState(ProgramStateRef State) {
   return setErrnoState(State, Irrelevant);
 }
 
-bool isErrno(const Decl *D) {
-  if (const auto *VD = dyn_cast_or_null<VarDecl>(D))
-    if (const IdentifierInfo *II = VD->getIdentifier())
-      return II->getName() == ErrnoVarName;
-  if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D))
-    if (const IdentifierInfo *II = FD->getIdentifier())
-      return llvm::is_contained(ErrnoLocationFuncNames, II->getName());
-  return false;
+bool isErrnoLocationCall(const CallEvent &CE) {
+  return ErrnoLocationCalls.contains(CE);
 }
 
 const NoteTag *getErrnoNoteTag(CheckerContext &C, const std::string &Message) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/ErrnoModeling.h b/clang/lib/StaticAnalyzer/Checkers/ErrnoModeling.h
index 6b53572fe5e2..95da8a28d325 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ErrnoModeling.h
+++ b/clang/lib/StaticAnalyzer/Checkers/ErrnoModeling.h
@@ -71,12 +71,9 @@ ProgramStateRef setErrnoState(ProgramStateRef State, ErrnoCheckState EState);
 /// Clear state of errno (make it irrelevant).
 ProgramStateRef clearErrnoState(ProgramStateRef State);
 
-/// Determine if a `Decl` node related to 'errno'.
-/// This is true if the declaration is the errno variable or a function
-/// that returns a pointer to the 'errno' value (usually the 'errno' macro is
-/// defined with this function). \p D is not required to be a canonical
-/// declaration.
-bool isErrno(const Decl *D);
+/// Determine if `Call` is a call to an internal function that returns the
+/// location of `errno` (in environments where errno is accessed this way).
+bool isErrnoLocationCall(const CallEvent &Call);
 
 /// Create a NoteTag that displays the message if the 'errno' memory region is
 /// marked as interesting, and resets the interestingness.
diff --git a/clang/test/Analysis/memory-model.cpp b/clang/test/Analysis/memory-model.cpp
index fd5a286acb60..cd42e8c72b8b 100644
--- a/clang/test/Analysis/memory-model.cpp
+++ b/clang/test/Analysis/memory-model.cpp
@@ -34,9 +34,9 @@ void var_simple_ref() {
 }
 
 void var_simple_ptr(int *a) {
-  clang_analyzer_dump(a);             // expected-warning {{SymRegion{reg_$0<int * a>}}}
-  clang_analyzer_dumpExtent(a);       // expected-warning {{extent_$1{SymRegion{reg_$0<int * a>}}}}
-  clang_analyzer_dumpElementCount(a); // expected-warning {{(extent_$1{SymRegion{reg_$0<int * a>}}) / 4}}
+  clang_analyzer_dump(a);             // expected-warning {{SymRegion{reg_$1<int * a>}}}
+  clang_analyzer_dumpExtent(a);       // expected-warning {{extent_$2{SymRegion{reg_$1<int * a>}}}}
+  clang_analyzer_dumpElementCount(a); // expected-warning {{(extent_$2{SymRegion{reg_$1<int * a>}}) / 4}}
 }
 
 void var_array() {
@@ -53,9 +53,9 @@ void string() {
 }
 
 void struct_simple_ptr(S *a) {
-  clang_analyzer_dump(a);             // expected-warning {{SymRegion{reg_$0<S * a>}}}
-  clang_analyzer_dumpExtent(a);       // expected-warning {{extent_$1{SymRegion{reg_$0<S * a>}}}}
-  clang_analyzer_dumpElementCount(a); // expected-warning {{(extent_$1{SymRegion{reg_$0<S * a>}}) / 4}}
+  clang_analyzer_dump(a);             // expected-warning {{SymRegion{reg_$1<S * a>}}}
+  clang_analyzer_dumpExtent(a);       // expected-warning {{extent_$2{SymRegion{reg_$1<S * a>}}}}
+  clang_analyzer_dumpElementCount(a); // expected-warning {{(extent_$2{SymRegion{reg_$1<S * a>}}) / 4}}
 }
 
 void field_ref(S a) {
@@ -65,9 +65,9 @@ void field_ref(S a) {
 }
 
 void field_ptr(S *a) {
-  clang_analyzer_dump(&a->f);             // expected-warning {{Element{SymRegion{reg_$0<S * a>},0 S64b,struct S}.f}}
-  clang_analyzer_dumpExtent(&a->f);       // expected-warning {{extent_$1{SymRegion{reg_$0<S * a>}}}}
-  clang_analyzer_dumpElementCount(&a->f); // expected-warning {{(extent_$1{SymRegion{reg_$0<S * a>}}) / 4U}}
+  clang_analyzer_dump(&a->f);             // expected-warning {{Element{SymRegion{reg_$1<S * a>},0 S64b,struct S}.f}}
+  clang_analyzer_dumpExtent(&a->f);       // expected-warning {{extent_$2{SymRegion{reg_$1<S * a>}}}}
+  clang_analyzer_dumpElementCount(&a->f); // expected-warning {{(extent_$2{SymRegion{reg_$1<S * a>}}) / 4U}}
 }
 
 void symbolic_array() {
-- 
GitLab


From f090801a9651cf4f0d05cc361a2a1b14805b62bf Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Wed, 15 May 2024 09:18:42 +0000
Subject: [PATCH 0823/1206] [lldb] Disable unaligned-pc-sigbus.test on arm(32)

I though the test could work there as well, but (of course) it does not,
because the lowest bit just means "run the code as thumb".
---
 lldb/test/Shell/Unwind/unaligned-pc-sigbus.test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/test/Shell/Unwind/unaligned-pc-sigbus.test b/lldb/test/Shell/Unwind/unaligned-pc-sigbus.test
index 5ebfba54301e..49f771cae95b 100644
--- a/lldb/test/Shell/Unwind/unaligned-pc-sigbus.test
+++ b/lldb/test/Shell/Unwind/unaligned-pc-sigbus.test
@@ -1,4 +1,4 @@
-# REQUIRES: (target-aarch64 || target-arm) && native
+# REQUIRES: target-aarch64 && native
 # UNSUPPORTED: system-windows
 # llvm.org/pr91610, rdar://128031075
 # XFAIL: system-darwin
-- 
GitLab


From 2f6c0e6e180c81087c26f4afac2155ea70472ec6 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Wed, 15 May 2024 10:25:51 +0100
Subject: [PATCH 0824/1206] [flang][Alias Analysis] not all block arguments are
 dummy arguments (#92156)

Arguments to openmp regions should not be tagged as dummy arguments.
This is particularly unsafe because these openmp blocks will eventually
be inlined into the calling function, where they will trivially alias
with other values inside of the calling function.

This is probably a theoretical issue because the calls to openmp runtime
function calls would act as barriers, preventing optimizations that are
too aggressive. But a lot more thought would need to go into a bet like
that.

This came out of discussion on
https://github.com/llvm/llvm-project/pull/92036
---
 .../lib/Optimizer/Analysis/AliasAnalysis.cpp  |  6 ++-
 .../lib/Optimizer/Transforms/AddAliasTags.cpp |  1 +
 flang/test/Transforms/tbaa.fir                | 37 +++++++++++++++++++
 3 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
index f723e8f66e3e..ed1101dc5e8d 100644
--- a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
+++ b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
@@ -33,7 +33,11 @@ static bool isDummyArgument(mlir::Value v) {
   if (!blockArg)
     return false;
 
-  return blockArg.getOwner()->isEntryBlock();
+  mlir::Block *owner = blockArg.getOwner();
+  if (!owner->isEntryBlock() ||
+      !mlir::isa<mlir::FunctionOpInterface>(owner->getParentOp()))
+    return false;
+  return true;
 }
 
 /// Temporary function to skip through all the no op operations
diff --git a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
index 684aa4462915..3642a812096d 100644
--- a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
+++ b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
@@ -105,6 +105,7 @@ static std::string getFuncArgName(mlir::Value arg) {
          "arg is a function argument");
   mlir::FunctionOpInterface func = mlir::dyn_cast<mlir::FunctionOpInterface>(
       blockArg.getOwner()->getParentOp());
+  assert(func && "This is not a function argument");
   mlir::StringAttr attr = func.getArgAttrOfType<mlir::StringAttr>(
       blockArg.getArgNumber(), "fir.bindc_name");
   if (!attr)
diff --git a/flang/test/Transforms/tbaa.fir b/flang/test/Transforms/tbaa.fir
index 7825ae60c71e..f94bbe4bf948 100644
--- a/flang/test/Transforms/tbaa.fir
+++ b/flang/test/Transforms/tbaa.fir
@@ -173,3 +173,40 @@
 // CHECK:           fir.store %[[VAL_8]] to %[[VAL_12]] : !fir.ref<i32>
 // CHECK:           return
 // CHECK:         }
+
+// -----
+
+// Make sure we don't mistake other block arguments as dummy arguments:
+
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %c0_i32 = arith.constant 0 : i32
+  omp.yield(%c0_i32 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = arith.addi %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+
+func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
+  %c10_i32 = arith.constant 10 : i32
+  %c6_i32 = arith.constant 6 : i32
+  %c-1_i32 = arith.constant -1 : i32
+  %0 = fir.address_of(@_QFEi) : !fir.ref<i32>
+  %1 = fir.declare %0 {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+  omp.parallel reduction(@add_reduction_i32 %1 -> %arg0 : !fir.ref<i32>) {
+// CHECK: omp.parallel reduction({{.*}}) {
+    %8 = fir.declare %arg0 {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK-NEXT: %[[DECL:.*]] = fir.declare
+    fir.store %c-1_i32 to %8 : !fir.ref<i32>
+// CHECK-NOT: fir.store %{{.*}} to %[[DECL]] {tbaa = %{{.*}}} : !fir.ref<i32>
+// CHECK:     fir.store %{{.*}} to %[[DECL]] : !fir.ref<i32>
+    omp.terminator
+  }
+  return
+}
+
+fir.global internal @_QFEi : i32 {
+  %c0_i32 = arith.constant 0 : i32
+  fir.has_value %c0_i32 : i32
+}
-- 
GitLab


From f39e75b45160ae69222d6ae197ee20c365146717 Mon Sep 17 00:00:00 2001
From: Jacek Caban <jacek@codeweavers.com>
Date: Wed, 15 May 2024 11:35:02 +0200
Subject: [PATCH 0825/1206] [CodeGen][ARM64EC][NFC] Factor out
 emitFunctionAlias and getSymbolFromMetadata in emitFunctionEntryLabel.
 (#92098)

---
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 71 +++++++++----------
 1 file changed, 32 insertions(+), 39 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index ee39c6355c29..3ce766fc173c 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -1161,52 +1161,45 @@ void AArch64AsmPrinter::emitFunctionEntryLabel() {
     TS->emitDirectiveVariantPCS(CurrentFnSym);
   }
 
+  AsmPrinter::emitFunctionEntryLabel();
+
   if (TM.getTargetTriple().isWindowsArm64EC() &&
       !MF->getFunction().hasLocalLinkage()) {
     // For ARM64EC targets, a function definition's name is mangled differently
-    // from the normal symbol. We emit the alias from the unmangled symbol to
-    // mangled symbol name here.
-    if (MDNode *Unmangled =
-            MF->getFunction().getMetadata("arm64ec_unmangled_name")) {
-      AsmPrinter::emitFunctionEntryLabel();
-
-      if (MDNode *ECMangled =
-              MF->getFunction().getMetadata("arm64ec_ecmangled_name")) {
-        StringRef UnmangledStr =
-            cast<MDString>(Unmangled->getOperand(0))->getString();
-        MCSymbol *UnmangledSym =
-            MMI->getContext().getOrCreateSymbol(UnmangledStr);
-        StringRef ECMangledStr =
-            cast<MDString>(ECMangled->getOperand(0))->getString();
-        MCSymbol *ECMangledSym =
-            MMI->getContext().getOrCreateSymbol(ECMangledStr);
-        OutStreamer->emitSymbolAttribute(UnmangledSym, MCSA_WeakAntiDep);
-        OutStreamer->emitAssignment(
-            UnmangledSym,
-            MCSymbolRefExpr::create(ECMangledSym, MCSymbolRefExpr::VK_WEAKREF,
-                                    MMI->getContext()));
-        OutStreamer->emitSymbolAttribute(ECMangledSym, MCSA_WeakAntiDep);
-        OutStreamer->emitAssignment(
-            ECMangledSym,
-            MCSymbolRefExpr::create(CurrentFnSym, MCSymbolRefExpr::VK_WEAKREF,
-                                    MMI->getContext()));
-        return;
+    // from the normal symbol, emit required aliases here.
+    auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
+      OutStreamer->emitSymbolAttribute(Src, MCSA_WeakAntiDep);
+      OutStreamer->emitAssignment(
+          Src, MCSymbolRefExpr::create(Dst, MCSymbolRefExpr::VK_WEAKREF,
+                                       MMI->getContext()));
+    };
+
+    auto getSymbolFromMetadata = [&](StringRef Name) {
+      MCSymbol *Sym = nullptr;
+      if (MDNode *Node = MF->getFunction().getMetadata(Name)) {
+        StringRef NameStr = cast<MDString>(Node->getOperand(0))->getString();
+        Sym = MMI->getContext().getOrCreateSymbol(NameStr);
+      }
+      return Sym;
+    };
+
+    if (MCSymbol *UnmangledSym =
+            getSymbolFromMetadata("arm64ec_unmangled_name")) {
+      MCSymbol *ECMangledSym = getSymbolFromMetadata("arm64ec_ecmangled_name");
+
+      if (ECMangledSym) {
+        // An external function, emit the alias from the unmangled symbol to
+        // mangled symbol name and the alias from the mangled symbol to guest
+        // exit thunk.
+        emitFunctionAlias(UnmangledSym, ECMangledSym);
+        emitFunctionAlias(ECMangledSym, CurrentFnSym);
       } else {
-        StringRef UnmangledStr =
-            cast<MDString>(Unmangled->getOperand(0))->getString();
-        MCSymbol *UnmangledSym =
-            MMI->getContext().getOrCreateSymbol(UnmangledStr);
-        OutStreamer->emitSymbolAttribute(UnmangledSym, MCSA_WeakAntiDep);
-        OutStreamer->emitAssignment(
-            UnmangledSym,
-            MCSymbolRefExpr::create(CurrentFnSym, MCSymbolRefExpr::VK_WEAKREF,
-                                    MMI->getContext()));
-        return;
+        // A function implementation, emit the alias from the unmangled symbol
+        // to mangled symbol name.
+        emitFunctionAlias(UnmangledSym, CurrentFnSym);
       }
     }
   }
-
-  return AsmPrinter::emitFunctionEntryLabel();
 }
 
 /// Small jump tables contain an unsigned byte or half, representing the offset
-- 
GitLab


From 421862f8e4ffddf57e210a205984a0ee39c57d96 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Wed, 15 May 2024 10:51:32 +0100
Subject: [PATCH 0826/1206] [Clang] Fix incorrect passing of _BitInt args
 (#90741)

This patch removes incorrect `byval` attribute from pointer argument
passed with >128 bit long _BitInt types.
---
 clang/lib/CodeGen/Targets/AArch64.cpp | 2 +-
 clang/test/CodeGen/ext-int-cc.c       | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp b/clang/lib/CodeGen/Targets/AArch64.cpp
index e32b060ebeb9..0a4711fb2170 100644
--- a/clang/lib/CodeGen/Targets/AArch64.cpp
+++ b/clang/lib/CodeGen/Targets/AArch64.cpp
@@ -317,7 +317,7 @@ AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic,
 
     if (const auto *EIT = Ty->getAs<BitIntType>())
       if (EIT->getNumBits() > 128)
-        return getNaturalAlignIndirect(Ty);
+        return getNaturalAlignIndirect(Ty, false);
 
     return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
                 ? ABIArgInfo::getExtend(Ty)
diff --git a/clang/test/CodeGen/ext-int-cc.c b/clang/test/CodeGen/ext-int-cc.c
index 001e866d34b4..508728172ab4 100644
--- a/clang/test/CodeGen/ext-int-cc.c
+++ b/clang/test/CodeGen/ext-int-cc.c
@@ -22,9 +22,9 @@
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple systemz -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=SYSTEMZ
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple ppc64 -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=PPC64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple ppc -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=PPC32
-// RUN: %clang_cc1 -no-enable-noundef-analysis -triple aarch64 -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=AARCH64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -triple aarch64 -O3 -disable-llvm-passes -fexperimental-max-bitint-width=1024 -emit-llvm -o - %s | FileCheck %s --check-prefixes=AARCH64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple aarch64 -target-abi darwinpcs -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=AARCH64DARWIN
-// RUN: %clang_cc1 -no-enable-noundef-analysis -triple arm64_32-apple-ios -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=AARCH64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -triple arm64_32-apple-ios -O3 -disable-llvm-passes -fexperimental-max-bitint-width=1024 -emit-llvm -o - %s | FileCheck %s --check-prefixes=AARCH64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple arm64_32-apple-ios -target-abi darwinpcs -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=AARCH64DARWIN
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple arm -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=ARM
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple loongarch64 -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=LA64
@@ -135,6 +135,7 @@ void ParamPassing4(_BitInt(129) a) {}
 // WIN64: define dso_local void @ParamPassing4(ptr %{{.+}})
 // LIN32: define{{.*}} void @ParamPassing4(ptr %{{.+}})
 // WIN32: define dso_local void @ParamPassing4(ptr %{{.+}})
+// AARCH64: define{{.*}} void @ParamPassing4(ptr %{{.+}})
 // NACL-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // NVPTX64-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // NVPTX-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
@@ -155,7 +156,6 @@ void ParamPassing4(_BitInt(129) a) {}
 // SYSTEMZ-NOT: define{{.*}} void @ParamPassing4(ptr %{{.+}})
 // PPC64-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // PPC32-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
-// AARCH64-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // AARCH64DARWIN-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // ARM-NOT: define{{.*}} arm_aapcscc void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // LA64-NOT: define{{.*}} void @ParamPassing4(ptr %{{.+}})
@@ -294,6 +294,7 @@ _BitInt(129) ReturnPassing5(void){}
 // WIN64: define dso_local void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // LIN32: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // WIN32: define dso_local void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
+// AARCH64: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // NACL-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // NVPTX64-NOT: define{{.*}} i129 @ReturnPassing5(
 // NVPTX-NOT: define{{.*}} i129 @ReturnPassing5(
@@ -314,7 +315,6 @@ _BitInt(129) ReturnPassing5(void){}
 // SYSTEMZ-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // PPC64-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // PPC32-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
-// AARCH64-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // AARCH64DARWIN-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // ARM-NOT: define{{.*}} arm_aapcscc void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // LA64-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
-- 
GitLab


From d187005cad8c2cb7d44ba3dd6b01c5f0e4c14ae7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 15 May 2024 11:00:14 +0100
Subject: [PATCH 0827/1206] [VPlan] Update VPBlendRecipe codegen for for
 first-lane only.

Update VPBlendRecipe::execute to support generating code for first-lane
only. This fixes a crash in the newly added test
@test_not_first_lane_only_wide_compare_incoming_order_swapped.
---
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |  35 +++---
 ...ruction-or-drop-poison-generating-flags.ll |   8 +-
 .../Transforms/LoopVectorize/uniform-blend.ll |  20 +---
 .../unused-blend-mask-for-first-operand.ll    | 105 +++++++++++++++++-
 4 files changed, 125 insertions(+), 43 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index fa634e774b5c..5eb99ffd1e10 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1506,24 +1506,25 @@ void VPBlendRecipe::execute(VPTransformState &State) {
   // Note that Mask0 is never used: lanes for which no path reaches this phi and
   // are essentially undef are taken from In0.
  VectorParts Entry(State.UF);
-  for (unsigned In = 0; In < NumIncoming; ++In) {
-    for (unsigned Part = 0; Part < State.UF; ++Part) {
-      // We might have single edge PHIs (blocks) - use an identity
-      // 'select' for the first PHI operand.
-      Value *In0 = State.get(getIncomingValue(In), Part);
-      if (In == 0)
-        Entry[Part] = In0; // Initialize with the first incoming value.
-      else {
-        // Select between the current value and the previous incoming edge
-        // based on the incoming mask.
-        Value *Cond = State.get(getMask(In), Part);
-        Entry[Part] =
-            State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi");
-      }
-    }
-  }
+ bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
+ for (unsigned In = 0; In < NumIncoming; ++In) {
+   for (unsigned Part = 0; Part < State.UF; ++Part) {
+     // We might have single edge PHIs (blocks) - use an identity
+     // 'select' for the first PHI operand.
+     Value *In0 = State.get(getIncomingValue(In), Part, OnlyFirstLaneUsed);
+     if (In == 0)
+       Entry[Part] = In0; // Initialize with the first incoming value.
+     else {
+       // Select between the current value and the previous incoming edge
+       // based on the incoming mask.
+       Value *Cond = State.get(getMask(In), Part, OnlyFirstLaneUsed);
+       Entry[Part] =
+           State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi");
+     }
+   }
+ }
   for (unsigned Part = 0; Part < State.UF; ++Part)
-    State.set(this, Entry[Part], Part);
+    State.set(this, Entry[Part], Part, OnlyFirstLaneUsed);
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
index 4e38630209b2..5f8141600371 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
@@ -40,8 +40,6 @@ define void @pr87378_vpinstruction_or_drop_poison_generating_flags(ptr %arg, i64
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP12]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ule <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp ule <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = select <vscale x 8 x i1> [[TMP13]], <vscale x 8 x i1> [[TMP14]], <vscale x 8 x i1> zeroinitializer
@@ -52,9 +50,9 @@ define void @pr87378_vpinstruction_or_drop_poison_generating_flags(ptr %arg, i64
 ; CHECK-NEXT:    [[TMP20:%.*]] = xor <vscale x 8 x i1> [[TMP14]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP21:%.*]] = select <vscale x 8 x i1> [[TMP13]], <vscale x 8 x i1> [[TMP20]], <vscale x 8 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP22:%.*]] = or <vscale x 8 x i1> [[TMP19]], [[TMP21]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP19]], <vscale x 8 x i64> [[BROADCAST_SPLAT6]], <vscale x 8 x i64> shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 poison, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <vscale x 8 x i64> [[PREDPHI]], i32 0
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i16, ptr [[ARG]], i64 [[TMP23]]
+; CHECK-NEXT:    [[EXT:%.+]] = extractelement <vscale x 8 x i1> [[TMP19]], i32 0
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select i1 [[EXT]], i64 [[TMP12]], i64 poison
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i16, ptr [[ARG]], i64 [[PREDPHI]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i16, ptr [[TMP24]], i32 0
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> zeroinitializer, ptr [[TMP25]], i32 2, <vscale x 8 x i1> [[TMP22]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index 71eed3b2985d..19cbcac6090c 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -4,19 +4,12 @@
 
 define void @blend_uniform_iv_trunc(i1 %c) {
 ; CHECK-LABEL: @blend_uniform_iv_trunc(
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[MASK0:%.*]] = insertelement <4 x i1> poison, i1 %c, i64 0
-; CHECK-NEXT:    [[MASK1:%.*]] = shufflevector <4 x i1> [[MASK0]], <4 x i1> poison, <4 x i32> zeroinitializer
-
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDEX]] to i16
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i16 [[TMP1]], 0
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i16> poison, i16 [[TMP2]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT1]], <4 x i16> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[MASK1]], <4 x i16> [[BROADCAST_SPLAT2]], <4 x i16> undef
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i16> [[PREDPHI]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i16 [[TMP4]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select i1 %c, i16 [[TMP2]], i16 undef
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i16 [[PREDPHI]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[TMP5]], i32 0
 ; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr [[TMP6]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -49,17 +42,12 @@ exit:                                             ; preds = %loop.latch
 define void @blend_uniform_iv(i1 %c) {
 ; CHECK-LABEL: @blend_uniform_iv(
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[MASK0:%.*]] = insertelement <4 x i1> poison, i1 %c, i64 0
-; CHECK-NEXT:    [[MASK1:%.*]] = shufflevector <4 x i1> [[MASK0]], <4 x i1> poison, <4 x i32> zeroinitializer
 
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[MASK1]], <4 x i64> [[BROADCAST_SPLAT2]], <4 x i64> undef
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i64> [[PREDPHI]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP2]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select i1 %c, i64 [[TMP0]], i64 undef
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[PREDPHI]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[TMP3]], i32 0
 ; CHECK-NEXT:    store <4 x i16> zeroinitializer, ptr [[TMP4]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll b/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
index d0c74897f264..0f7bd3d71feb 100644
--- a/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
+++ b/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
@@ -16,7 +16,7 @@ define void @test_not_first_lane_only_constant(ptr %A, ptr noalias %B)  {
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[TMP0]]
-; CHECK-NEXT:    [[TMP13:%.*]] = load i16, ptr %B, align 2
+; CHECK-NEXT:    [[TMP13:%.*]] = load i16, ptr [[B]], align 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i16> poison, i16 [[TMP13]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT5]], <4 x i16> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
@@ -86,8 +86,6 @@ define void @test_not_first_lane_only_wide_compare(ptr %A, ptr noalias %B, i16 %
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x ptr> poison, ptr [[B]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT3]], <4 x ptr> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -97,8 +95,8 @@ define void @test_not_first_lane_only_wide_compare(ptr %A, ptr noalias %B, i16 %
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult <4 x i16> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP5]], <4 x ptr> poison, <4 x ptr> [[BROADCAST_SPLAT4]]
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x ptr> [[PREDPHI]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP4]], ptr poison, ptr [[B]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i16, ptr [[TMP12]], align 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i16> poison, i16 [[TMP13]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT5]], <4 x i16> poison, <4 x i32> zeroinitializer
@@ -162,6 +160,101 @@ loop.latch:
   %c.2 = icmp  eq i16 %iv.next, 1000
   br i1 %c.2, label %exit, label %loop.header
 
+exit:
+  ret void
+}
+
+define void @test_not_first_lane_only_wide_compare_incoming_order_swapped(ptr %A, ptr noalias %B, i16 %x, i16 %y)  {
+; CHECK-LABEL: define void @test_not_first_lane_only_wide_compare_incoming_order_swapped(
+; CHECK-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], i16 [[X:%.*]], i16 [[Y:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[X]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i16> poison, i16 [[Y]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT1]], <4 x i16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult <4 x i16> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult <4 x i16> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i1> [[TMP5]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select i1 [[TMP11]], ptr [[B]], ptr poison
+; CHECK-NEXT:    [[TMP12:%.*]] = load i16, ptr [[PREDPHI]], align 2
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i16> poison, i16 [[TMP12]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT3]], <4 x i16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i16> [[BROADCAST_SPLAT4]], ptr [[TMP2]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[IV]]
+; CHECK-NEXT:    [[L_0:%.*]] = load i16, ptr [[GEP_A]], align 2
+; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i16 [[L_0]], [[X]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_LATCH]], label [[ELSE_1:%.*]]
+; CHECK:       else.1:
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ult i16 [[L_0]], [[Y]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[THEN_2:%.*]], label [[ELSE_2:%.*]]
+; CHECK:       then.2:
+; CHECK-NEXT:    br label [[ELSE_2]]
+; CHECK:       else.2:
+; CHECK-NEXT:    br label [[LOOP_LATCH]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi ptr [ poison, [[LOOP_HEADER]] ], [ [[B]], [[ELSE_2]] ]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[MERGE]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add i16 [[IV]], 1
+; CHECK-NEXT:    store i16 [[L]], ptr [[GEP_A]], align 2
+; CHECK-NEXT:    [[C_2:%.*]] = icmp eq i16 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[C_2]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i16 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep.A = getelementptr inbounds i16, ptr %A, i16 %iv
+  %l.0 = load i16, ptr %gep.A
+  %c.0 = icmp ult i16 %l.0, %x
+  br i1 %c.0, label %loop.latch, label %else.1
+
+else.1:
+  %c.1 = icmp ult i16 %l.0, %y
+  br i1 %c.1, label %then.2, label %else.2
+
+then.2:
+  br label %else.2
+
+else.2:
+  br label %loop.latch
+
+loop.latch:
+  %merge = phi ptr [ poison, %loop.header ], [ %B, %else.2 ]
+  %l = load i16, ptr %merge, align 2
+  %iv.next = add i16 %iv, 1
+  store i16 %l, ptr %gep.A
+  %c.2 = icmp  eq i16 %iv.next, 1000
+  br i1 %c.2, label %exit, label %loop.header
+
 exit:
   ret void
 }
@@ -172,4 +265,6 @@ exit:
 ; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
 ; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
 ; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
 ;.
-- 
GitLab


From b0a1ae2cca4a438753e093df2f949e73a313dbe2 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 15 May 2024 11:17:23 +0100
Subject: [PATCH 0828/1206] [LV] Add additional variants of tests with
 udiv/urem/sdiv/srem in TC.

Add additional tests with udiv/urem/sdiv/srem in trip counts, where the
divisor is constant.

For https://github.com/llvm/llvm-project/pull/92177.
---
 .../trip-count-expansion-may-introduce-ub.ll  | 553 +++++++++++++++++-
 1 file changed, 539 insertions(+), 14 deletions(-)

diff --git a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
index 85fad6fb3632..ce5d4427fa01 100644
--- a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+++ b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
@@ -4,8 +4,8 @@
 ; Test cases with trip counts containing UDIV expressions for
 ; https://github.com/llvm/llvm-project/issues/89958.
 
-define i64 @multi_exit_1_exit_count_with_udiv_in_header(ptr %dst, i64 %N) {
-; CHECK-LABEL: define i64 @multi_exit_1_exit_count_with_udiv_in_header(
+define i64 @multi_exit_1_exit_count_with_udiv_by_value_in_header(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_1_exit_count_with_udiv_by_value_in_header(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
@@ -70,8 +70,74 @@ exit:
   ret i64 %p
 }
 
-define i64 @multi_exit_2_exit_count_with_udiv_in_block_executed_unconditionally(ptr %A, i64 %N) {
-; CHECK-LABEL: define i64 @multi_exit_2_exit_count_with_udiv_in_block_executed_unconditionally(
+define i64 @multi_exit_1_exit_count_with_udiv_by_constant_in_header(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_1_exit_count_with_udiv_by_constant_in_header(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 [[N]], 42
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[UMIN]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[D:%.*]] = udiv i64 [[N]], 42
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %d = udiv i64 %N, 42
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
+define i64 @multi_exit_2_exit_count_with_udiv_by_value_in_block_executed_unconditionally(ptr %A, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_2_exit_count_with_udiv_by_value_in_block_executed_unconditionally(
 ; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
@@ -126,7 +192,7 @@ define i64 @multi_exit_2_exit_count_with_udiv_in_block_executed_unconditionally(
 ; CHECK:       pred.store.continue6:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -148,7 +214,7 @@ define i64 @multi_exit_2_exit_count_with_udiv_in_block_executed_unconditionally(
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
-; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[CONTINUE]] ], [ 0, [[LOOP_LATCH]] ]
 ; CHECK-NEXT:    ret i64 [[P]]
@@ -182,8 +248,120 @@ exit:
   ret i64 %p
 }
 
-define i64 @multi_exit_3_exit_count_with_udiv_in_block_executed_conditionally(ptr %A, i64 %N) {
-; CHECK-LABEL: define i64 @multi_exit_3_exit_count_with_udiv_in_block_executed_conditionally(
+define i64 @multi_exit_2_exit_count_with_udiv_by_constant_in_block_executed_unconditionally(ptr %A, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_2_exit_count_with_udiv_by_constant_in_block_executed_unconditionally(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 [[N]], 42
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[UMIN]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], <i32 10, i32 10, i32 10, i32 10>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP7]], i32 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP9]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; CHECK:       pred.store.continue:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP7]], i32 1
+; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
+; CHECK:       pred.store.if1:
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP12]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE2]]
+; CHECK:       pred.store.continue2:
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP7]], i32 2
+; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
+; CHECK:       pred.store.if3:
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP15]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
+; CHECK:       pred.store.continue4:
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP7]], i32 3
+; CHECK-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
+; CHECK:       pred.store.if5:
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP17]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP18]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
+; CHECK:       pred.store.continue6:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[C_2:%.*]] = icmp eq i32 [[L]], 10
+; CHECK-NEXT:    br i1 [[C_2]], label [[THEN:%.*]], label [[CONTINUE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    br label [[CONTINUE]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[D:%.*]] = udiv i64 [[N]], 42
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[CONTINUE]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+  %l = load i32, ptr %gep
+  %c.2 = icmp eq i32 %l, 10
+  br i1 %c.2, label %then, label %continue
+
+then:
+  store i32 1, ptr %gep
+  br label %continue
+
+continue:
+  %d = udiv i64 %N, 42
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %continue ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
+define i64 @multi_exit_3_exit_count_with_udiv_by_value_in_block_executed_conditionally(ptr %A, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_3_exit_count_with_udiv_by_value_in_block_executed_conditionally(
 ; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
@@ -232,10 +410,59 @@ exit:
   ret i64 %p
 }
 
+define i64 @multi_exit_3_exit_count_with_udiv_by_constant_in_block_executed_conditionally(ptr %A, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_3_exit_count_with_udiv_by_constant_in_block_executed_conditionally(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[D:%.*]] = udiv i64 [[N]], 42
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[C_2:%.*]] = icmp ne i32 [[L]], 10
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C_2]], i1 true, i1 [[C_1]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_HEADER]], label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 0, [[LOOP_LATCH]] ], [ 1, [[LOOP_HEADER]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+  %l = load i32, ptr %gep
+  %c.2 = icmp eq i32 %l, 10
+  br i1 %c.2, label %then, label %loop.latch
+
+then:
+  %d = udiv i64 %N, 42
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.latch, label %exit
+
+loop.latch:
+  store i32 1, ptr %gep
+  %iv.next = add i64 %iv, 1
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %then ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
 ; FIXME: Currently miscompiled as we unconditionally execute udiv after
 ; vectorization.
-define i64 @multi_exit_4_exit_count_with_udiv_in_latch(ptr %dst, i64 %N) {
-; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_in_latch(
+define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
@@ -258,7 +485,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_in_latch(ptr %dst, i64 %N) {
 ; CHECK-NEXT:    store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -274,7 +501,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_in_latch(ptr %dst, i64 %N) {
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[D:%.*]] = udiv i64 42, [[N]]
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
-; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
 ; CHECK-NEXT:    ret i64 [[P]]
@@ -300,6 +527,72 @@ exit:
   ret i64 %p
 }
 
+define i64 @multi_exit_4_exit_count_with_udiv_by_constant_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_constant_in_latch(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 [[N]], 42
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[UMIN]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[D:%.*]] = udiv i64 [[N]], 42
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %d = udiv i64 %N, 42
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
 define void @single_exit_tc_with_udiv(ptr %dst, i64 %N) {
 ; CHECK-LABEL: define void @single_exit_tc_with_udiv(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
@@ -320,7 +613,7 @@ define void @single_exit_tc_with_udiv(ptr %dst, i64 %N) {
 ; CHECK-NEXT:    store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP4]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -334,7 +627,7 @@ define void @single_exit_tc_with_udiv(ptr %dst, i64 %N) {
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[D:%.*]] = udiv i64 42, [[N]]
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
-; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -354,6 +647,228 @@ exit:
   ret void
 }
 
+; FIXME: Currently miscompiled as we unconditionally execute udiv after
+; vectorization.
+define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 42, [[TMP1]]
+; CHECK-NEXT:    [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP2]], i64 0)
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[SMAX1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw i64 [[UMIN]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0
+; CHECK-NEXT:    store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[D:%.*]] = urem i64 42, [[N]]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %d = urem i64 42, %N
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
+define i64 @multi_exit_4_exit_count_with_urem_by_constant_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_urem_by_constant_in_latch(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 [[N]], 42
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 42
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP2]], i64 0)
+; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[SMAX1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw i64 [[UMIN]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0
+; CHECK-NEXT:    store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[D:%.*]] = urem i64 [[N]], 42
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %d = urem i64 %N, 42
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
+define i64 @multi_exit_4_exit_count_with_srem_by_value_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_srem_by_value_in_latch(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[D:%.*]] = srem i64 42, [[N]]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %d = srem i64 42, %N
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
+define i64 @multi_exit_4_exit_count_with_sdiv_by_value_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_sdiv_by_value_in_latch(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[D:%.*]] = sdiv i64 42, [[N]]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+  store i32 1, ptr %gep
+  %c.0 = icmp slt i64 %iv, %N
+  br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+  %iv.next = add i64 %iv, 1
+  %d = sdiv i64 42, %N
+  %c.1 = icmp slt i64 %iv, %d
+  br i1 %c.1, label %loop.header, label %exit
+
+exit:
+  %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+  ret i64 %p
+}
+
 ;.
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -365,4 +880,14 @@ exit:
 ; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
 ; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
 ; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META2]]}
+; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META2]], [[META1]]}
 ;.
-- 
GitLab


From b6f050fa129b08b6bc35168f0b8010742cd1ed9d Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Wed, 15 May 2024 11:20:58 +0100
Subject: [PATCH 0829/1206] [lldb] Document some more packets (#92124)

Comparing a bit of the mock GDB server code to what was in the document
I found these:
* QLaunchArch
* qSpeedTest
* qSymbol

qSymbol is the most mysterious but it did have some examples in a
comment so I've adapted that.
---
 lldb/docs/resources/lldbgdbremote.md | 87 ++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)

diff --git a/lldb/docs/resources/lldbgdbremote.md b/lldb/docs/resources/lldbgdbremote.md
index 1467723fb79d..7076a75032da 100644
--- a/lldb/docs/resources/lldbgdbremote.md
+++ b/lldb/docs/resources/lldbgdbremote.md
@@ -867,6 +867,22 @@ error replies.
 **Priority To Implement:** Low. Only needed if the remote target wants to
 provide strings that are human readable along with an error code.
 
+## QLaunchArch
+
+Set the architecture to use when launching a process for hosts that can run
+multiple architecture slices that are contained in a single universal program
+file.
+
+```
+send packet: $QLaunchArch:<architecture>
+```
+
+The response is `OK` if the value in `<architecture>` was recognised as valid
+and will be used for the next launch request. `E63` if not.
+
+**Priority To Implement:** Only required for hosts that support program files
+that contain code for multiple architectures.
+
 ## QListThreadsInStopReply
 
 Enable the `threads:` and `thread-pcs:` data in the question-mark packet
@@ -1883,6 +1899,77 @@ some platforms know, or can find out where this information is.
 Low if you have a debug target where all object and symbol files
 contain static load addresses.
 
+## qSpeedTest
+
+Test the maximum speed at which packets can be sent and received.
+
+```
+send packet: qSpeedTest:response_size:<response size>;
+read packet: data:<response data>
+```
+
+`<response size>` is a hex encoded unsigned number up to 64 bits in size.
+The remote will respond with `data:` followed by a block of `a` characters
+whose size should match `<response size>`, if the connection is stable.
+
+If there is an error parsing the packet, the response is `E79`.
+
+This packet is used by LLDB to discover how reliable the connection is by
+varying the amount of data requested by `<response size>` and checking whether
+the expected amount and values were received.
+
+**Priority to Implemment:** Not required for debugging on the same host, otherwise
+low unless you know your connection quality is variable.
+
+## qSymbol
+
+Notify the remote that LLDB is ready to do symbol lookups on behalf of the
+debug server. The response is the symbol name the debug server wants to know the
+value of, or `OK` if the debug server does not need to know any more symbol values.
+
+The exchange always begins with:
+```
+send packet: qSymbol::
+```
+
+The `::` are delimiters for fields that may be filled in future responses. These
+delimiters must be included even in the first packet sent.
+
+The debug server can reply one of two ways. If it doesn't need any symbol values:
+```
+read packet: OK
+```
+
+If it does need a symbol value, it includes the ASCII hex encoded name of the
+symbol:
+```
+read packet: qSymbol:6578616D706C65
+```
+
+This should be looked up by LLDB then sent back to the server. Include the name
+again, with the vaue as a hex number:
+```
+read packet: qSymbol:6578616D706C65:CAFEF00D
+```
+
+If LLDB cannot find the value, it should respond with only the name. Note that
+the second `:` is not included here, whereas it is in the initial packet.
+```
+read packet: qSymbol:6578616D706C65
+```
+
+If LLDB is asked for any symbols that it cannot find, it should send the
+initial `qSymbol::` again at any point where new libraries are loaded. In case
+the symbol can now be resolved.
+
+If the debug server has requested all the symbols it wants, the final response
+will be `OK` (whether they were all found or not).
+
+If LLDB did find all the symbols and recieves an `OK` it does not need to send
+`qSymbol::` again during the debug session.
+
+**Priority To Implement:** Low, this is rarely used.
+
 ## qThreadStopInfo\<tid\>
 
 Get information about why a thread, whose ID is `<tid>`, is stopped.
-- 
GitLab


From 03bdfb65617e3cf714a106fdf7a6ae7551d17bce Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Wed, 15 May 2024 11:25:15 +0100
Subject: [PATCH 0830/1206] [lldb][test][FreeBSD] Fix some concurrent event
 tests (#84155)

A lot of `TestConcurrent*.py` expect one of the threads to crash, but we
weren't checking for it properly.

Possibly because signal reporting got better on FreeBSD at some point,
and it now shows the same info as Linux does.

```
  lldb-api :: functionalities/inferior-changed/TestInferiorChanged.py
  lldb-api :: functionalities/inferior-crashing/TestInferiorCrashing.py
  lldb-api :: functionalities/inferior-crashing/TestInferiorCrashingStep.py
  lldb-api :: functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferior.py
  lldb-api :: functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferiorStep.py
  lldb-api :: functionalities/thread/concurrent_events/TestConcurrentCrashWithBreak.py
  lldb-api :: functionalities/thread/concurrent_events/TestConcurrentCrashWithSignal.py
  lldb-api :: functionalities/thread/concurrent_events/TestConcurrentCrashWithWatchpoint.py
  lldb-api :: functionalities/thread/concurrent_events/TestConcurrentCrashWithWatchpointBreakpointSignal.py
```

Fixes #48777

`TestConcurrentTwoBreakpointsOneSignal.py` no longer fails, at least on
an AWS instance, so I've removed the xfail there.
---
 lldb/packages/Python/lldbsuite/test/lldbutil.py                | 2 +-
 .../concurrent_events/TestConcurrentTwoBreakpointsOneSignal.py | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/lldbutil.py b/lldb/packages/Python/lldbsuite/test/lldbutil.py
index 1ec036f885e7..02ec65170f20 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbutil.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbutil.py
@@ -809,7 +809,7 @@ def is_thread_crashed(test, thread):
             thread.GetStopReason() == lldb.eStopReasonException
             and "EXC_BAD_ACCESS" in thread.GetStopDescription(100)
         )
-    elif test.getPlatform() == "linux":
+    elif test.getPlatform() in ["linux", "freebsd"]:
         return (
             thread.GetStopReason() == lldb.eStopReasonSignal
             and thread.GetStopReasonDataAtIndex(0)
diff --git a/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneSignal.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneSignal.py
index c66905af9e92..4960c4b241fb 100644
--- a/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneSignal.py
+++ b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneSignal.py
@@ -8,9 +8,6 @@ class ConcurrentTwoBreakpointsOneSignal(ConcurrentEventsBase):
     # Atomic sequences are not supported yet for MIPS in LLDB.
     @skipIf(triple="^mips")
     @expectedFlakeyNetBSD
-    @expectedFailureAll(
-        archs=["aarch64"], oslist=["freebsd"], bugnumber="llvm.org/pr49433"
-    )
     def test(self):
         """Test two threads that trigger a breakpoint and one signal thread."""
         self.build()
-- 
GitLab


From e67080df999c035d764c42aaa6d85417331ac52c Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Wed, 15 May 2024 03:25:51 -0700
Subject: [PATCH 0831/1206] [mlir][ods] Populate properties in generated
 builder (#90430)

Previously this was only populated in the create method later. This
resolves some of invalid builder paths. This may also be sufficient that
type inference functions no longer have to consider whether property
conversion has happened (but haven't verified that yet).

This also makes Attributes corresponding to Properties as optional
inside the set from attributes method. Today that is in effect what
happens with Property value initialization and folks use it to define
custom C++ types whose default initialization is what they want. This is
the behavior users get if they use properties directly. Propagating
Attributes without allowing partial setting would require iterating over
the dictionary attribute considering the properties of the op type that
will be created. This could also have been an additional method
generated or optional behavior on the set method. But doing it
consistently seems better. In terms of whats lost, it doesn't seem like
anything compared to the pure Property path where Property is default
value initialized and then partially overwritten (this doesn't seem to
buy anything else verification wise).

Default valued Properties (as specified ODS side rather than C++ side)
triggered error as the containing class was not yet complete but
referenced nested class, so that we couldn't have default initializer
for them in the parent class. Added an additional forwarding builder to
avoid needing to update call sites. This could be split out to separate
change.

Inlined templated function in unit test that was only used once. Moved
initialization earlier where seen.
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td   |   4 +-
 .../mlir/Dialect/MemRef/IR/MemRefOps.td       |   2 +-
 .../mlir/Dialect/Tensor/IR/TensorOps.td       |   2 +-
 mlir/include/mlir/IR/Operation.h              |   5 +-
 .../Func/Transforms/OneToNFuncConversions.cpp |   6 +-
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp      |   8 +-
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      |  12 +-
 mlir/test/Dialect/OpenMP/invalid.mlir         |  14 +--
 mlir/test/Dialect/OpenMP/ops.mlir             |   4 +-
 mlir/test/lib/Dialect/Test/TestOps.td         |   7 ++
 mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp   | 104 ++++++++++++----
 mlir/unittests/TableGen/OpBuildGen.cpp        | 114 ++++++++++++++----
 12 files changed, 207 insertions(+), 75 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 4b91708ea1aa..84e67d2c11db 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -65,13 +65,13 @@ class LLVM_IntArithmeticOpWithOverflowFlag<string mnemonic, string instName,
   let builders = [
     OpBuilder<(ins "Type":$type, "Value":$lhs, "Value":$rhs,
                    "IntegerOverflowFlags":$overflowFlags), [{
-      build($_builder, $_state, type, lhs, rhs);
       $_state.getOrAddProperties<Properties>().overflowFlags = overflowFlags;
+      build($_builder, $_state, type, lhs, rhs);
     }]>,
     OpBuilder<(ins "Value":$lhs, "Value":$rhs,
                    "IntegerOverflowFlags":$overflowFlags), [{
-      build($_builder, $_state, lhs, rhs);
       $_state.getOrAddProperties<Properties>().overflowFlags = overflowFlags;
+      build($_builder, $_state, lhs, rhs);
     }]>
   ];
 
diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
index 5738b6ca51c1..63e6ed059deb 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
@@ -1764,9 +1764,9 @@ def MemRef_CollapseShapeOp : MemRef_ReassociativeReshapeOp<"collapse_shape", [
       "ArrayRef<ReassociationIndices>":$reassociation,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs),
     [{
-      build($_builder, $_state, resultType, src, attrs);
       $_state.addAttribute("reassociation",
                           getReassociationIndicesAttribute($_builder, reassociation));
+      build($_builder, $_state, resultType, src, attrs);
     }]>,
     OpBuilder<(ins "Type":$resultType, "Value":$src,
       "ArrayRef<ReassociationExprs>":$reassociation,
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
index a403e89a39f9..cafc3d91fd1e 100644
--- a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
+++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
@@ -1216,9 +1216,9 @@ def Tensor_CollapseShapeOp : Tensor_ReassociativeReshapeOp<"collapse_shape"> {
       "ArrayRef<ReassociationIndices>":$reassociation,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs),
     [{
-      build($_builder, $_state, resultType, src, attrs);
       $_state.addAttribute("reassociation",
           getReassociationIndicesAttribute($_builder, reassociation));
+      build($_builder, $_state, resultType, src, attrs);
     }]>,
     OpBuilder<(ins "Type":$resultType, "Value":$src,
       "ArrayRef<ReassociationExprs>":$reassociation,
diff --git a/mlir/include/mlir/IR/Operation.h b/mlir/include/mlir/IR/Operation.h
index c52a6fcac10c..f0dd7c517805 100644
--- a/mlir/include/mlir/IR/Operation.h
+++ b/mlir/include/mlir/IR/Operation.h
@@ -916,11 +916,12 @@ public:
   /// operation. Returns an empty attribute if no properties are present.
   Attribute getPropertiesAsAttribute();
 
-  /// Set the properties from the provided  attribute.
+  /// Set the properties from the provided attribute.
   /// This is an expensive operation that can fail if the attribute is not
   /// matching the expectations of the properties for this operation. This is
   /// mostly useful for unregistered operations or used when parsing the
-  /// generic format. An optional diagnostic can be passed in for richer errors.
+  /// generic format. An optional diagnostic emitter can be passed in for richer
+  /// errors, if none is passed then behavior is undefined in error case.
   LogicalResult
   setPropertiesFromAttribute(Attribute attr,
                              function_ref<InFlightDiagnostic()> emitError);
diff --git a/mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp b/mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp
index c04986cad84f..a5b88338e638 100644
--- a/mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp
+++ b/mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp
@@ -40,9 +40,9 @@ public:
       return failure();
 
     // Create new CallOp.
-    auto newOp = rewriter.create<CallOp>(loc, resultMapping.getConvertedTypes(),
-                                         adaptor.getFlatOperands());
-    newOp->setAttrs(op->getAttrs());
+    auto newOp =
+        rewriter.create<CallOp>(loc, resultMapping.getConvertedTypes(),
+                                adaptor.getFlatOperands(), op->getAttrs());
 
     rewriter.replaceOp(op, newOp->getResults(), resultMapping);
     return success();
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index 199e7330a233..45f39c80041c 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -1806,11 +1806,11 @@ void ReinterpretCastOp::build(OpBuilder &b, OperationState &result,
   dispatchIndexOpFoldResults(offset, dynamicOffsets, staticOffsets);
   dispatchIndexOpFoldResults(sizes, dynamicSizes, staticSizes);
   dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides);
+  result.addAttributes(attrs);
   build(b, result, resultType, source, dynamicOffsets, dynamicSizes,
         dynamicStrides, b.getDenseI64ArrayAttr(staticOffsets),
         b.getDenseI64ArrayAttr(staticSizes),
         b.getDenseI64ArrayAttr(staticStrides));
-  result.addAttributes(attrs);
 }
 
 void ReinterpretCastOp::build(OpBuilder &b, OperationState &result,
@@ -2483,9 +2483,9 @@ void CollapseShapeOp::build(OpBuilder &b, OperationState &result, Value src,
   auto srcType = llvm::cast<MemRefType>(src.getType());
   MemRefType resultType =
       CollapseShapeOp::computeCollapsedType(srcType, reassociation);
-  build(b, result, resultType, src, attrs);
   result.addAttribute(::mlir::getReassociationAttrName(),
                       getReassociationIndicesAttribute(b, reassociation));
+  build(b, result, resultType, src, attrs);
 }
 
 LogicalResult CollapseShapeOp::verify() {
@@ -2781,11 +2781,11 @@ void SubViewOp::build(OpBuilder &b, OperationState &result,
     resultType = llvm::cast<MemRefType>(SubViewOp::inferResultType(
         sourceMemRefType, staticOffsets, staticSizes, staticStrides));
   }
+  result.addAttributes(attrs);
   build(b, result, resultType, source, dynamicOffsets, dynamicSizes,
         dynamicStrides, b.getDenseI64ArrayAttr(staticOffsets),
         b.getDenseI64ArrayAttr(staticSizes),
         b.getDenseI64ArrayAttr(staticStrides));
-  result.addAttributes(attrs);
 }
 
 // Build a SubViewOp with mixed static and dynamic entries and inferred result
@@ -3320,8 +3320,8 @@ void TransposeOp::build(OpBuilder &b, OperationState &result, Value in,
   // Compute result type.
   MemRefType resultType = inferTransposeResultType(memRefType, permutationMap);
 
-  build(b, result, resultType, in, attrs);
   result.addAttribute(TransposeOp::getPermutationAttrStrName(), permutation);
+  build(b, result, resultType, in, attrs);
 }
 
 // transpose $in $permutation attr-dict : type($in) `to` type(results)
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 414bd7459af8..d3b1754cbe1c 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -1743,9 +1743,9 @@ void CollapseShapeOp::build(OpBuilder &b, OperationState &result, Value src,
       llvm::cast<RankedTensorType>(src.getType()),
       getSymbolLessAffineMaps(
           convertReassociationIndicesToExprs(b.getContext(), reassociation)));
-  build(b, result, resultType, src, attrs);
   result.addAttribute(getReassociationAttrStrName(),
                       getReassociationIndicesAttribute(b, reassociation));
+  build(b, result, resultType, src, attrs);
 }
 
 template <typename TensorReshapeOp, bool isExpansion = std::is_same<
@@ -2100,11 +2100,11 @@ void ExtractSliceOp::build(OpBuilder &b, OperationState &result,
     resultType = llvm::cast<RankedTensorType>(ExtractSliceOp::inferResultType(
         sourceRankedTensorType, staticOffsets, staticSizes, staticStrides));
   }
+  result.addAttributes(attrs);
   build(b, result, resultType, source, dynamicOffsets, dynamicSizes,
         dynamicStrides, b.getDenseI64ArrayAttr(staticOffsets),
         b.getDenseI64ArrayAttr(staticSizes),
         b.getDenseI64ArrayAttr(staticStrides));
-  result.addAttributes(attrs);
 }
 
 /// Build an ExtractSliceOp with mixed static and dynamic entries and inferred
@@ -2499,11 +2499,11 @@ void InsertSliceOp::build(OpBuilder &b, OperationState &result, Value source,
   dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets);
   dispatchIndexOpFoldResults(sizes, dynamicSizes, staticSizes);
   dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides);
+  result.addAttributes(attrs);
   build(b, result, dest.getType(), source, dest, dynamicOffsets, dynamicSizes,
         dynamicStrides, b.getDenseI64ArrayAttr(staticOffsets),
         b.getDenseI64ArrayAttr(staticSizes),
         b.getDenseI64ArrayAttr(staticStrides));
-  result.addAttributes(attrs);
 }
 
 /// Build an InsertSliceOp with mixed static and dynamic entries packed into a
@@ -2967,10 +2967,10 @@ void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
   auto sourceType = llvm::cast<RankedTensorType>(source.getType());
   if (!resultType)
     resultType = inferResultType(sourceType, staticLow, staticHigh);
+  result.addAttributes(attrs);
   build(b, result, resultType, source, low, high,
         b.getDenseI64ArrayAttr(staticLow), b.getDenseI64ArrayAttr(staticHigh),
         nofold ? b.getUnitAttr() : UnitAttr());
-  result.addAttributes(attrs);
 }
 
 void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
@@ -3000,10 +3000,10 @@ void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
     resultType = PadOp::inferResultType(sourceType, staticLow, staticHigh);
   }
   assert(llvm::isa<RankedTensorType>(resultType));
+  result.addAttributes(attrs);
   build(b, result, resultType, source, dynamicLow, dynamicHigh,
         b.getDenseI64ArrayAttr(staticLow), b.getDenseI64ArrayAttr(staticHigh),
         nofold ? b.getUnitAttr() : UnitAttr());
-  result.addAttributes(attrs);
 }
 
 void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
@@ -3447,11 +3447,11 @@ void ParallelInsertSliceOp::build(OpBuilder &b, OperationState &result,
   dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets);
   dispatchIndexOpFoldResults(sizes, dynamicSizes, staticSizes);
   dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides);
+  result.addAttributes(attrs);
   build(b, result, {}, source, dest, dynamicOffsets, dynamicSizes,
         dynamicStrides, b.getDenseI64ArrayAttr(staticOffsets),
         b.getDenseI64ArrayAttr(staticSizes),
         b.getDenseI64ArrayAttr(staticStrides));
-  result.addAttributes(attrs);
 }
 
 /// Build an ParallelInsertSliceOp with mixed static and dynamic entries
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 138c2c9d418d..aee79264313e 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -2113,23 +2113,23 @@ func.func @omp_distribute_allocate(%data_var : memref<i32>) -> () {
 
 func.func @omp_distribute_wrapper() -> () {
   // expected-error @below {{op must be a loop wrapper}}
-  "omp.distribute"() ({
+  omp.distribute {
       %0 = arith.constant 0 : i32
       "omp.terminator"() : () -> ()
-    }) : () -> ()
+  }
 }
 
 // -----
 
 func.func @omp_distribute_nested_wrapper(%data_var : memref<i32>) -> () {
   // expected-error @below {{only supported nested wrappers are 'omp.parallel' and 'omp.simd'}}
-  "omp.distribute"() ({
-      "omp.wsloop"() ({
-        %0 = arith.constant 0 : i32
-        "omp.terminator"() : () -> ()
-      }) : () -> ()
+  omp.distribute {
+    "omp.wsloop"() ({
+      %0 = arith.constant 0 : i32
       "omp.terminator"() : () -> ()
     }) : () -> ()
+    "omp.terminator"() : () -> ()
+  }
 }
 
 // -----
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 420cb226d593..fc6d1c316531 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -521,13 +521,13 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
 // CHECK-LABEL: omp_simd
 func.func @omp_simd(%lb : index, %ub : index, %step : index) -> () {
   // CHECK: omp.simd
-  "omp.simd" () ({
+  omp.simd {
     "omp.loop_nest" (%lb, %ub, %step) ({
     ^bb1(%iv2: index):
       "omp.yield"() : () -> ()
     }) : (index, index, index) -> ()
     "omp.terminator"() : () -> ()
-  }) : () -> ()
+  }
 
   return
 }
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index befe6aa6cede..c5d0341b7de7 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -2402,6 +2402,13 @@ def TableGenBuildOp5 : TableGenBuildInferReturnTypeBaseOp<
   let regions = (region AnyRegion:$body);
 }
 
+// Two variadic args, non variadic results, with AttrSizedOperandSegments
+// Test build method generation for property conversion & type inference.
+def TableGenBuildOp6 : TEST_Op<"tblgen_build_6", [AttrSizedOperandSegments]> {
+  let arguments = (ins Variadic<AnyType>:$a, Variadic<AnyType>:$b);
+  let results = (outs F32:$result);
+}
+
 //===----------------------------------------------------------------------===//
 // Test BufferPlacement
 //===----------------------------------------------------------------------===//
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 63fe5a809907..e013ccac5dd0 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -1311,22 +1311,24 @@ void OpEmitter::genPropertiesSupport() {
     return ::mlir::failure();
   }
     )decl";
-  // TODO: properties might be optional as well.
-  const char *propFromAttrFmt = R"decl(;
-    {{
+  const char *propFromAttrFmt = R"decl(
       auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr,
                ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {{
-        {0};
+        {0}
       };
       {2};
-      if (!attr) {{
-        emitError() << "expected key entry for {1} in DictionaryAttr to set "
-                   "Properties.";
+)decl";
+  const char *attrGetNoDefaultFmt = R"decl(;
+      if (attr && ::mlir::failed(setFromAttr(prop.{0}, attr, emitError)))
         return ::mlir::failure();
+)decl";
+  const char *attrGetDefaultFmt = R"decl(;
+      if (attr) {{
+        if (::mlir::failed(setFromAttr(prop.{0}, attr, emitError)))
+          return ::mlir::failure();
+      } else {{
+        prop.{0} = {1};
       }
-      if (::mlir::failed(setFromAttr(prop.{1}, attr, emitError)))
-        return ::mlir::failure();
-    }
 )decl";
 
   for (const auto &attrOrProp : attrOrProperties) {
@@ -1349,13 +1351,20 @@ void OpEmitter::genPropertiesSupport() {
       }
       os.flush();
 
-      setPropMethod << formatv(propFromAttrFmt,
+      setPropMethod << "{\n"
+                    << formatv(propFromAttrFmt,
                                tgfmt(prop.getConvertFromAttributeCall(),
                                      &fctx.addSubst("_attr", propertyAttr)
                                           .addSubst("_storage", propertyStorage)
                                           .addSubst("_diag", propertyDiag)),
                                name, getAttr);
-
+      if (prop.hasDefaultValue()) {
+        setPropMethod << formatv(attrGetDefaultFmt, name,
+                                 prop.getDefaultValue());
+      } else {
+        setPropMethod << formatv(attrGetNoDefaultFmt, name);
+      }
+      setPropMethod << "  }\n";
     } else {
       const auto *namedAttr =
           llvm::dyn_cast_if_present<const AttributeMetadata *>(attrOrProp);
@@ -1376,13 +1385,8 @@ void OpEmitter::genPropertiesSupport() {
       setPropMethod << formatv(R"decl(
   {{
     auto &propStorage = prop.{0};
-    {2}
-    if (attr || /*isRequired=*/{1}) {{
-      if (!attr) {{
-        emitError() << "expected key entry for {0} in DictionaryAttr to set "
-                   "Properties.";
-        return ::mlir::failure();
-      }
+    {1}
+    if (attr) {{
       auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
       if (convertedAttr) {{
         propStorage = convertedAttr;
@@ -1393,7 +1397,7 @@ void OpEmitter::genPropertiesSupport() {
     }
   }
 )decl",
-                               name, namedAttr->isRequired, getAttr);
+                               name, getAttr);
     }
   }
   setPropMethod << "  return ::mlir::success();\n";
@@ -2650,6 +2654,21 @@ void OpEmitter::genInferredTypeCollectiveParamBuilder() {
   }
 
   // Result types
+  if (emitHelper.hasProperties()) {
+    // Initialize the properties from Attributes before invoking the infer
+    // function.
+    body << formatv(R"(
+  if (!attributes.empty()) {
+    ::mlir::OpaqueProperties properties =
+      &{1}.getOrAddProperties<{0}::Properties>();
+    std::optional<::mlir::RegisteredOperationName> info =
+      {1}.name.getRegisteredInfo();
+    if (failed(info->setOpPropertiesFromAttribute({1}.name, properties,
+        {1}.attributes.getDictionary({1}.getContext()), nullptr)))
+      ::llvm::report_fatal_error("Property conversion failed.");
+  })",
+                    opClass.getClassName(), builderOpState);
+  }
   body << formatv(R"(
   ::llvm::SmallVector<::mlir::Type, 2> inferredReturnTypes;
   if (::mlir::succeeded({0}::inferReturnTypes(odsBuilder.getContext(),
@@ -2879,6 +2898,22 @@ void OpEmitter::genCollectiveParamBuilder() {
          << "u && \"mismatched number of return types\");\n";
   body << "  " << builderOpState << ".addTypes(resultTypes);\n";
 
+  if (emitHelper.hasProperties()) {
+    // Initialize the properties from Attributes before invoking the infer
+    // function.
+    body << formatv(R"(
+  if (!attributes.empty()) {
+    ::mlir::OpaqueProperties properties =
+      &{1}.getOrAddProperties<{0}::Properties>();
+    std::optional<::mlir::RegisteredOperationName> info =
+      {1}.name.getRegisteredInfo();
+    if (failed(info->setOpPropertiesFromAttribute({1}.name, properties,
+        {1}.attributes.getDictionary({1}.getContext()), nullptr)))
+      ::llvm::report_fatal_error("Property conversion failed.");
+  })",
+                    opClass.getClassName(), builderOpState);
+  }
+
   // Generate builder that infers type too.
   // TODO: Expand to handle successors.
   if (canInferType(op) && op.getNumSuccessors() == 0)
@@ -4054,13 +4089,17 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
       op.getTrait("::mlir::OpTrait::AttrSizedOperandSegments");
   {
     SmallVector<MethodParameter> paramList;
-    paramList.emplace_back("::mlir::DictionaryAttr", "attrs",
-                           attrSizedOperands ? "" : "nullptr");
-    if (useProperties)
-      paramList.emplace_back("const Properties &", "properties", "{}");
-    else
+    if (useProperties) {
+      // Properties can't be given a default constructor here due to Properties
+      // struct being defined in the enclosing class which isn't complete by
+      // here.
+      paramList.emplace_back("::mlir::DictionaryAttr", "attrs");
+      paramList.emplace_back("const Properties &", "properties");
+    } else {
+      paramList.emplace_back("::mlir::DictionaryAttr", "attrs", "{}");
       paramList.emplace_back("const ::mlir::EmptyProperties &", "properties",
                              "{}");
+    }
     paramList.emplace_back("::mlir::RegionRange", "regions", "{}");
     auto *baseConstructor = genericAdaptorBase.addConstructor(paramList);
     baseConstructor->addMemberInitializer("odsAttrs", "attrs");
@@ -4102,6 +4141,21 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
           "::mlir::EmptyProperties{}), "
           "regions");
     }
+
+    // Add forwarding constructor that constructs Properties.
+    if (useProperties) {
+      SmallVector<MethodParameter> paramList;
+      paramList.emplace_back("RangeT", "values");
+      paramList.emplace_back("::mlir::DictionaryAttr", "attrs",
+                             attrSizedOperands ? "" : "nullptr");
+      auto *noPropertiesConstructor =
+          genericAdaptor.addConstructor(std::move(paramList));
+      noPropertiesConstructor->addMemberInitializer(
+          genericAdaptor.getClassName(), "values, "
+                                         "attrs, "
+                                         "Properties{}, "
+                                         "{}");
+    }
   }
 
   // Create constructors constructing the adaptor from an instance of the op.
diff --git a/mlir/unittests/TableGen/OpBuildGen.cpp b/mlir/unittests/TableGen/OpBuildGen.cpp
index c83ac9088114..94fbfa28803c 100644
--- a/mlir/unittests/TableGen/OpBuildGen.cpp
+++ b/mlir/unittests/TableGen/OpBuildGen.cpp
@@ -66,29 +66,44 @@ protected:
       EXPECT_EQ(op->getAttr(attrs[idx].getName().strref()),
                 attrs[idx].getValue());
 
+    EXPECT_TRUE(mlir::succeeded(concreteOp.verify()));
     concreteOp.erase();
   }
 
-  // Helper method to test ops with inferred result types and single variadic
-  // input.
   template <typename OpTy>
-  void testSingleVariadicInputInferredType() {
-    // Test separate arg, separate param build method.
-    auto op = builder.create<OpTy>(loc, i32Ty, ValueRange{*cstI32, *cstI32});
-    verifyOp(std::move(op), {i32Ty}, {*cstI32, *cstI32}, noAttrs);
-
-    // Test collective params build method.
-    op = builder.create<OpTy>(loc, TypeRange{i32Ty},
-                              ValueRange{*cstI32, *cstI32});
-    verifyOp(std::move(op), {i32Ty}, {*cstI32, *cstI32}, noAttrs);
-
-    // Test build method with no result types, default value of attributes.
-    op = builder.create<OpTy>(loc, ValueRange{*cstI32, *cstI32});
-    verifyOp(std::move(op), {i32Ty}, {*cstI32, *cstI32}, noAttrs);
-
-    // Test build method with no result types and supplied attributes.
-    op = builder.create<OpTy>(loc, ValueRange{*cstI32, *cstI32}, attrs);
-    verifyOp(std::move(op), {i32Ty}, {*cstI32, *cstI32}, attrs);
+  void verifyOp(OpTy &&concreteOp, std::vector<Type> resultTypes,
+                std::vector<Value> operands1, std::vector<Value> operands2,
+                std::vector<NamedAttribute> attrs) {
+    ASSERT_NE(concreteOp, nullptr);
+    Operation *op = concreteOp.getOperation();
+
+    EXPECT_EQ(op->getNumResults(), resultTypes.size());
+    for (unsigned idx : llvm::seq(0U, op->getNumResults()))
+      EXPECT_EQ(op->getResult(idx).getType(), resultTypes[idx]);
+
+    auto operands = llvm::to_vector(llvm::concat<Value>(operands1, operands2));
+    EXPECT_EQ(op->getNumOperands(), operands.size());
+    for (unsigned idx : llvm::seq(0U, op->getNumOperands()))
+      EXPECT_EQ(op->getOperand(idx), operands[idx]);
+
+    EXPECT_EQ(op->getAttrs().size(), attrs.size());
+    if (op->getAttrs().size() != attrs.size()) {
+      // Simple export where there is mismatch count.
+      llvm::errs() << "Op attrs:\n";
+      for (auto it : op->getAttrs())
+        llvm::errs() << "\t" << it.getName() << " = " << it.getValue() << "\n";
+
+      llvm::errs() << "Expected attrs:\n";
+      for (auto it : attrs)
+        llvm::errs() << "\t" << it.getName() << " = " << it.getValue() << "\n";
+    } else {
+      for (unsigned idx : llvm::seq<unsigned>(0U, attrs.size()))
+        EXPECT_EQ(op->getAttr(attrs[idx].getName().strref()),
+                  attrs[idx].getValue());
+    }
+
+    EXPECT_TRUE(mlir::succeeded(concreteOp.verify()));
+    concreteOp.erase();
   }
 
 protected:
@@ -205,13 +220,31 @@ TEST_F(OpBuildGenTest,
   verifyOp(op, {i32Ty, f32Ty}, {*cstI32}, attrs);
 }
 
-// The next test checks supression of ambiguous build methods for ops that
+// The next test checks suppression of ambiguous build methods for ops that
 // have a single variadic input, and single non-variadic result, and which
-// support the SameOperandsAndResultType trait and and optionally the
+// support the SameOperandsAndResultType trait and optionally the
 // InferOpTypeInterface interface. For such ops, the ODS framework generates
 // build methods with no result types as they are inferred from the input types.
 TEST_F(OpBuildGenTest, BuildMethodsSameOperandsAndResultTypeSuppression) {
-  testSingleVariadicInputInferredType<test::TableGenBuildOp4>();
+  // Test separate arg, separate param build method.
+  auto op = builder.create<test::TableGenBuildOp4>(
+      loc, i32Ty, ValueRange{*cstI32, *cstI32});
+  verifyOp(std::move(op), {i32Ty}, {*cstI32, *cstI32}, noAttrs);
+
+  // Test collective params build method.
+  op = builder.create<test::TableGenBuildOp4>(loc, TypeRange{i32Ty},
+                                              ValueRange{*cstI32, *cstI32});
+  verifyOp(std::move(op), {i32Ty}, {*cstI32, *cstI32}, noAttrs);
+
+  // Test build method with no result types, default value of attributes.
+  op =
+      builder.create<test::TableGenBuildOp4>(loc, ValueRange{*cstI32, *cstI32});
+  verifyOp(std::move(op), {i32Ty}, {*cstI32, *cstI32}, noAttrs);
+
+  // Test build method with no result types and supplied attributes.
+  op = builder.create<test::TableGenBuildOp4>(loc, ValueRange{*cstI32, *cstI32},
+                                              attrs);
+  verifyOp(std::move(op), {i32Ty}, {*cstI32, *cstI32}, attrs);
 }
 
 TEST_F(OpBuildGenTest, BuildMethodsRegionsAndInferredType) {
@@ -221,4 +254,41 @@ TEST_F(OpBuildGenTest, BuildMethodsRegionsAndInferredType) {
   verifyOp(op, {i32Ty}, {*cstI32, *cstF32}, noAttrs);
 }
 
+TEST_F(OpBuildGenTest, BuildMethodsVariadicProperties) {
+  // Account for conversion as part of getAttrs().
+  std::vector<NamedAttribute> noAttrsStorage;
+  auto segmentSize = builder.getNamedAttr("operandSegmentSizes",
+                                          builder.getDenseI32ArrayAttr({1, 1}));
+  noAttrsStorage.push_back(segmentSize);
+  ArrayRef<NamedAttribute> noAttrs(noAttrsStorage);
+  std::vector<NamedAttribute> attrsStorage = this->attrStorage;
+  attrsStorage.push_back(segmentSize);
+  ArrayRef<NamedAttribute> attrs(attrsStorage);
+
+  // Test separate arg, separate param build method.
+  auto op = builder.create<test::TableGenBuildOp6>(
+      loc, f32Ty, ValueRange{*cstI32}, ValueRange{*cstI32});
+  verifyOp(std::move(op), {f32Ty}, {*cstI32}, {*cstI32}, noAttrs);
+
+  // Test build method with no result types, default value of attributes.
+  op = builder.create<test::TableGenBuildOp6>(loc, ValueRange{*cstI32},
+                                              ValueRange{*cstI32});
+  verifyOp(std::move(op), {f32Ty}, {*cstI32}, {*cstI32}, noAttrs);
+
+  // Test collective params build method.
+  op = builder.create<test::TableGenBuildOp6>(
+      loc, TypeRange{f32Ty}, ValueRange{*cstI32}, ValueRange{*cstI32});
+  verifyOp(std::move(op), {f32Ty}, {*cstI32}, {*cstI32}, noAttrs);
+
+  // Test build method with result types, supplied attributes.
+  op = builder.create<test::TableGenBuildOp6>(
+      loc, TypeRange{f32Ty}, ValueRange{*cstI32, *cstI32}, attrs);
+  verifyOp(std::move(op), {f32Ty}, {*cstI32}, {*cstI32}, attrs);
+
+  // Test build method with no result types and supplied attributes.
+  op = builder.create<test::TableGenBuildOp6>(loc, ValueRange{*cstI32, *cstI32},
+                                              attrs);
+  verifyOp(std::move(op), {f32Ty}, {*cstI32}, {*cstI32}, attrs);
+}
+
 } // namespace mlir
-- 
GitLab


From 7621a0d36465cf870769cd54035d254d409c2ce4 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Wed, 15 May 2024 11:27:06 +0100
Subject: [PATCH 0832/1206] [LLVM][CodeGen][SVE] Improve custom lowering for
 EXTRACT_SUBVECTOR. (#90963)

We can extract any legal fixed length vector from a scalable vector by
using VECTOR_SPLICE.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  59 ++++----
 .../sve-extract-fixed-from-scalable-vector.ll |  48 ++----
 .../AArch64/sve-extract-fixed-vector.ll       | 142 +++---------------
 ...e-streaming-mode-fixed-length-int-to-fp.ll |  30 ++--
 4 files changed, 77 insertions(+), 202 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2ec9f66214b6..afa023220d35 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -13912,45 +13912,52 @@ AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
 
 SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
                                                       SelectionDAG &DAG) const {
-  assert(Op.getValueType().isFixedLengthVector() &&
+  EVT VT = Op.getValueType();
+  assert(VT.isFixedLengthVector() &&
          "Only cases that extract a fixed length vector are supported!");
-
   EVT InVT = Op.getOperand(0).getValueType();
-  unsigned Idx = Op.getConstantOperandVal(1);
-  unsigned Size = Op.getValueSizeInBits();
 
   // If we don't have legal types yet, do nothing
-  if (!DAG.getTargetLoweringInfo().isTypeLegal(InVT))
+  if (!isTypeLegal(InVT))
     return SDValue();
 
-  if (InVT.isScalableVector()) {
-    // This will be matched by custom code during ISelDAGToDAG.
-    if (Idx == 0 && isPackedVectorType(InVT, DAG))
+  if (InVT.is128BitVector()) {
+    assert(VT.is64BitVector() && "Extracting unexpected vector type!");
+    unsigned Idx = Op.getConstantOperandVal(1);
+
+    // This will get lowered to an appropriate EXTRACT_SUBREG in ISel.
+    if (Idx == 0)
       return Op;
 
-    return SDValue();
+    // If this is extracting the upper 64-bits of a 128-bit vector, we match
+    // that directly.
+    if (Idx * InVT.getScalarSizeInBits() == 64 && Subtarget->isNeonAvailable())
+      return Op;
   }
 
-  // This will get lowered to an appropriate EXTRACT_SUBREG in ISel.
-  if (Idx == 0 && InVT.getSizeInBits() <= 128)
-    return Op;
-
-  // If this is extracting the upper 64-bits of a 128-bit vector, we match
-  // that directly.
-  if (Size == 64 && Idx * InVT.getScalarSizeInBits() == 64 &&
-      InVT.getSizeInBits() == 128 && Subtarget->isNeonAvailable())
-    return Op;
-
-  if (useSVEForFixedLengthVectorVT(InVT, !Subtarget->isNeonAvailable())) {
+  if (InVT.isScalableVector() ||
+      useSVEForFixedLengthVectorVT(InVT, !Subtarget->isNeonAvailable())) {
     SDLoc DL(Op);
+    SDValue Vec = Op.getOperand(0);
+    SDValue Idx = Op.getOperand(1);
 
-    EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
-    SDValue NewInVec =
-        convertToScalableVector(DAG, ContainerVT, Op.getOperand(0));
+    EVT PackedVT = getPackedSVEVectorVT(InVT.getVectorElementType());
+    if (PackedVT != InVT) {
+      // Pack input into the bottom part of an SVE register and try again.
+      SDValue Container = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PackedVT,
+                                      DAG.getUNDEF(PackedVT), Vec,
+                                      DAG.getVectorIdxConstant(0, DL));
+      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Container, Idx);
+    }
+
+    // This will get matched by custom code during ISelDAGToDAG.
+    if (isNullConstant(Idx))
+      return Op;
 
-    SDValue Splice = DAG.getNode(ISD::VECTOR_SPLICE, DL, ContainerVT, NewInVec,
-                                 NewInVec, DAG.getConstant(Idx, DL, MVT::i64));
-    return convertFromScalableVector(DAG, Op.getValueType(), Splice);
+    assert(InVT.isScalableVector() && "Unexpected vector type!");
+    // Move requested subvector to the start of the vector and try again.
+    SDValue Splice = DAG.getNode(ISD::VECTOR_SPLICE, DL, InVT, Vec, Vec, Idx);
+    return convertFromScalableVector(DAG, VT, Splice);
   }
 
   return SDValue();
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
index e91aac430110..641050ae69d9 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll
@@ -143,15 +143,8 @@ define <4 x float> @extract_v4f32_nxv16f32_12(<vscale x 16 x float> %arg) {
 define <2 x float> @extract_v2f32_nxv16f32_2(<vscale x 16 x float> %arg) {
 ; CHECK-LABEL: extract_v2f32_nxv16f32_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
-; CHECK-NEXT:    ldr d0, [sp, #8]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %ext = call <2 x float> @llvm.vector.extract.v2f32.nxv16f32(<vscale x 16 x float> %arg, i64 2)
   ret <2 x float> %ext
@@ -274,15 +267,8 @@ define <4 x i3> @extract_v4i3_nxv32i3_16(<vscale x 32 x i3> %arg) {
 define <2 x i32> @extract_v2i32_nxv16i32_2(<vscale x 16 x i32> %arg) {
 ; CHECK-LABEL: extract_v2i32_nxv16i32_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
-; CHECK-NEXT:    ldr d0, [sp, #8]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %ext = call <2 x i32> @llvm.vector.extract.v2i32.nxv16i32(<vscale x 16 x i32> %arg, i64 2)
   ret <2 x i32> %ext
@@ -314,16 +300,9 @@ define <4 x half> @extract_v4f16_nxv2f16_0(<vscale x 2 x half> %arg) {
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    cntd x8
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    addpl x9, sp, #6
-; CHECK-NEXT:    subs x8, x8, #4
-; CHECK-NEXT:    csel x8, xzr, x8, lo
-; CHECK-NEXT:    st1h { z0.d }, p0, [sp, #3, mul vl]
-; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    csel x8, x8, xzr, lo
-; CHECK-NEXT:    lsl x8, x8, #1
-; CHECK-NEXT:    ldr d0, [x9, x8]
+; CHECK-NEXT:    st1h { z0.d }, p0, [sp]
+; CHECK-NEXT:    ldr d0, [sp]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -338,17 +317,12 @@ define <4 x half> @extract_v4f16_nxv2f16_4(<vscale x 2 x half> %arg) {
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    cntd x8
-; CHECK-NEXT:    mov w9, #4 // =0x4
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    subs x8, x8, #4
-; CHECK-NEXT:    csel x8, xzr, x8, lo
-; CHECK-NEXT:    st1h { z0.d }, p0, [sp, #3, mul vl]
-; CHECK-NEXT:    cmp x8, #4
-; CHECK-NEXT:    csel x8, x8, x9, lo
-; CHECK-NEXT:    addpl x9, sp, #6
-; CHECK-NEXT:    lsl x8, x8, #1
-; CHECK-NEXT:    ldr d0, [x9, x8]
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    st1h { z0.d }, p0, [sp]
+; CHECK-NEXT:    ld1h { z0.h }, p1/z, [sp]
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
index 88268104889f..b05b46a75b69 100644
--- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll
@@ -15,20 +15,8 @@ define <2 x i64> @extract_v2i64_nxv2i64(<vscale x 2 x i64> %vec) nounwind {
 define <2 x i64> @extract_v2i64_nxv2i64_idx2(<vscale x 2 x i64> %vec) nounwind {
 ; CHECK-LABEL: extract_v2i64_nxv2i64_idx2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    cntd x8
-; CHECK-NEXT:    mov w9, #2 // =0x2
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    sub x8, x8, #2
-; CHECK-NEXT:    cmp x8, #2
-; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
-; CHECK-NEXT:    csel x8, x8, x9, lo
-; CHECK-NEXT:    mov x9, sp
-; CHECK-NEXT:    lsl x8, x8, #3
-; CHECK-NEXT:    ldr q0, [x9, x8]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #16
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %retval = call <2 x i64> @llvm.vector.extract.v2i64.nxv2i64(<vscale x 2 x i64> %vec, i64 2)
   ret <2 x i64> %retval
@@ -48,20 +36,8 @@ define <4 x i32> @extract_v4i32_nxv4i32(<vscale x 4 x i32> %vec) nounwind {
 define <4 x i32> @extract_v4i32_nxv4i32_idx4(<vscale x 4 x i32> %vec) nounwind {
 ; CHECK-LABEL: extract_v4i32_nxv4i32_idx4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    cntw x8
-; CHECK-NEXT:    mov w9, #4 // =0x4
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    sub x8, x8, #4
-; CHECK-NEXT:    cmp x8, #4
-; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
-; CHECK-NEXT:    csel x8, x8, x9, lo
-; CHECK-NEXT:    mov x9, sp
-; CHECK-NEXT:    lsl x8, x8, #2
-; CHECK-NEXT:    ldr q0, [x9, x8]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #16
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %retval = call <4 x i32> @llvm.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> %vec, i64 4)
   ret <4 x i32> %retval
@@ -82,18 +58,9 @@ define <4 x i32> @extract_v4i32_nxv2i32(<vscale x 2 x i32> %vec) nounwind #1 {
 define <4 x i32> @extract_v4i32_nxv2i32_idx4(<vscale x 2 x i32> %vec) nounwind #1 {
 ; CHECK-LABEL: extract_v4i32_nxv2i32_idx4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov x8, #4 // =0x4
-; CHECK-NEXT:    mov x9, sp
-; CHECK-NEXT:    ptrue p1.d, vl4
-; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
-; CHECK-NEXT:    ld1d { z0.d }, p1/z, [x9, x8, lsl #3]
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #32
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %retval = call <4 x i32> @llvm.vector.extract.v4i32.nxv2i32(<vscale x 2 x i32> %vec, i64 4)
   ret <4 x i32> %retval
@@ -113,20 +80,8 @@ define <8 x i16> @extract_v8i16_nxv8i16(<vscale x 8 x i16> %vec) nounwind {
 define <8 x i16> @extract_v8i16_nxv8i16_idx8(<vscale x 8 x i16> %vec) nounwind {
 ; CHECK-LABEL: extract_v8i16_nxv8i16_idx8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    cnth x8
-; CHECK-NEXT:    mov w9, #8 // =0x8
-; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    sub x8, x8, #8
-; CHECK-NEXT:    cmp x8, #8
-; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
-; CHECK-NEXT:    csel x8, x8, x9, lo
-; CHECK-NEXT:    mov x9, sp
-; CHECK-NEXT:    lsl x8, x8, #1
-; CHECK-NEXT:    ldr q0, [x9, x8]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #16
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %retval = call <8 x i16> @llvm.vector.extract.v8i16.nxv8i16(<vscale x 8 x i16> %vec, i64 8)
   ret <8 x i16> %retval
@@ -147,18 +102,9 @@ define <8 x i16> @extract_v8i16_nxv4i16(<vscale x 4 x i16> %vec) nounwind #1 {
 define <8 x i16> @extract_v8i16_nxv4i16_idx8(<vscale x 4 x i16> %vec) nounwind #1 {
 ; CHECK-LABEL: extract_v8i16_nxv4i16_idx8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    mov x8, #8 // =0x8
-; CHECK-NEXT:    mov x9, sp
-; CHECK-NEXT:    ptrue p1.s, vl8
-; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
-; CHECK-NEXT:    ld1w { z0.s }, p1/z, [x9, x8, lsl #2]
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #32
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %retval = call <8 x i16> @llvm.vector.extract.v8i16.nxv4i16(<vscale x 4 x i16> %vec, i64 8)
   ret <8 x i16> %retval
@@ -180,19 +126,10 @@ define <8 x i16> @extract_v8i16_nxv2i16(<vscale x 2 x i16> %vec) nounwind #1 {
 define <8 x i16> @extract_v8i16_nxv2i16_idx8(<vscale x 2 x i16> %vec) nounwind #1 {
 ; CHECK-LABEL: extract_v8i16_nxv2i16_idx8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov x8, #8 // =0x8
-; CHECK-NEXT:    mov x9, sp
-; CHECK-NEXT:    ptrue p1.d, vl8
-; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
-; CHECK-NEXT:    ld1d { z0.d }, p1/z, [x9, x8, lsl #3]
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #64
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %retval = call <8 x i16> @llvm.vector.extract.v8i16.nxv2i16(<vscale x 2 x i16> %vec, i64 8)
   ret <8 x i16> %retval
@@ -212,19 +149,8 @@ define <16 x i8> @extract_v16i8_nxv16i8(<vscale x 16 x i8> %vec) nounwind {
 define <16 x i8> @extract_v16i8_nxv16i8_idx16(<vscale x 16 x i8> %vec) nounwind {
 ; CHECK-LABEL: extract_v16i8_nxv16i8_idx16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    rdvl x8, #1
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    mov w9, #16 // =0x10
-; CHECK-NEXT:    sub x8, x8, #16
-; CHECK-NEXT:    cmp x8, #16
-; CHECK-NEXT:    st1b { z0.b }, p0, [sp]
-; CHECK-NEXT:    csel x8, x8, x9, lo
-; CHECK-NEXT:    mov x9, sp
-; CHECK-NEXT:    ldr q0, [x9, x8]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #16
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %retval = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> %vec, i64 16)
   ret <16 x i8> %retval
@@ -245,18 +171,9 @@ define <16 x i8> @extract_v16i8_nxv8i8(<vscale x 8 x i8> %vec) nounwind #1 {
 define <16 x i8> @extract_v16i8_nxv8i8_idx16(<vscale x 8 x i8> %vec) nounwind #1 {
 ; CHECK-LABEL: extract_v16i8_nxv8i8_idx16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    mov x8, #16 // =0x10
-; CHECK-NEXT:    mov x9, sp
-; CHECK-NEXT:    ptrue p1.h, vl16
-; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
-; CHECK-NEXT:    ld1h { z0.h }, p1/z, [x9, x8, lsl #1]
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #32
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %retval = call <16 x i8> @llvm.vector.extract.v16i8.nxv8i8(<vscale x 8 x i8> %vec, i64 16)
   ret <16 x i8> %retval
@@ -278,19 +195,10 @@ define <16 x i8> @extract_v16i8_nxv4i8(<vscale x 4 x i8> %vec) nounwind #1 {
 define <16 x i8> @extract_v16i8_nxv4i8_idx16(<vscale x 4 x i8> %vec) nounwind #1 {
 ; CHECK-LABEL: extract_v16i8_nxv4i8_idx16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    mov x8, #16 // =0x10
-; CHECK-NEXT:    mov x9, sp
-; CHECK-NEXT:    ptrue p1.s, vl16
-; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
-; CHECK-NEXT:    ld1w { z0.s }, p1/z, [x9, x8, lsl #2]
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #64
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %retval = call <16 x i8> @llvm.vector.extract.v16i8.nxv4i8(<vscale x 4 x i8> %vec, i64 16)
   ret <16 x i8> %retval
@@ -313,17 +221,11 @@ define <16 x i8> @extract_v16i8_nxv2i8(<vscale x 2 x i8> %vec) nounwind #1 {
 define <16 x i8> @extract_v16i8_nxv2i8_idx16(<vscale x 2 x i8> %vec) nounwind #1 {
 ; CHECK-LABEL: extract_v16i8_nxv2i8_idx16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #128
 ; CHECK-NEXT:    uzp1 z0.s, z0.s, z0.s
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %retval = call <16 x i8> @llvm.vector.extract.v16i8.nxv2i8(<vscale x 2 x i8> %vec, i64 16)
   ret <16 x i8> %retval
@@ -434,13 +336,8 @@ define <16 x i1> @extract_v16i1_nxv16i1(<vscale x 16 x i1> %inmask) {
 define <2 x i64> @extract_fixed_v2i64_nxv2i64(<vscale x 2 x i64> %vec) nounwind #0 {
 ; CHECK-LABEL: extract_fixed_v2i64_nxv2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
-; CHECK-NEXT:    ldr q0, [sp, #16]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #16
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %retval = call <2 x i64> @llvm.vector.extract.v2i64.nxv2i64(<vscale x 2 x i64> %vec, i64 2)
   ret <2 x i64> %retval
@@ -449,14 +346,9 @@ define <2 x i64> @extract_fixed_v2i64_nxv2i64(<vscale x 2 x i64> %vec) nounwind
 define void @extract_fixed_v4i64_nxv2i64(<vscale x 2 x i64> %vec, ptr %p) nounwind #0 {
 ; CHECK-LABEL: extract_fixed_v4i64_nxv2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #32
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %retval = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> %vec, i64 4)
   store <4 x i64> %retval, ptr %p
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
index b285659258f3..a9b52c93006d 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
@@ -177,18 +177,19 @@ define void @ucvtf_v8i16_v8f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    uunpklo z1.s, z0.h
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z2.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z3.d, z0.s
+; CHECK-NEXT:    mov z3.d, z1.d
+; CHECK-NEXT:    uunpklo z2.d, z0.s
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    ucvtf z2.d, p0/m, z2.d
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    ucvtf z3.d, p0/m, z3.d
+; CHECK-NEXT:    uunpklo z3.d, z3.s
+; CHECK-NEXT:    ucvtf z2.d, p0/m, z2.d
 ; CHECK-NEXT:    ucvtf z1.d, p0/m, z1.d
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    stp q2, q1, [x1]
-; CHECK-NEXT:    stp q3, q0, [x1, #32]
+; CHECK-NEXT:    ucvtf z3.d, p0/m, z3.d
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %res = uitofp <8 x i16> %op1 to <8 x double>
@@ -750,18 +751,19 @@ define void @scvtf_v8i16_v8f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sunpklo z1.s, z0.h
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    sunpklo z2.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z3.d, z0.s
+; CHECK-NEXT:    mov z3.d, z1.d
+; CHECK-NEXT:    sunpklo z2.d, z0.s
 ; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z1.d, z1.s
-; CHECK-NEXT:    scvtf z2.d, p0/m, z2.d
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
-; CHECK-NEXT:    scvtf z3.d, p0/m, z3.d
+; CHECK-NEXT:    sunpklo z3.d, z3.s
+; CHECK-NEXT:    scvtf z2.d, p0/m, z2.d
 ; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    stp q2, q1, [x1]
-; CHECK-NEXT:    stp q3, q0, [x1, #32]
+; CHECK-NEXT:    scvtf z3.d, p0/m, z3.d
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q2, q0, [x1, #32]
 ; CHECK-NEXT:    ret
   %op1 = load <8 x i16>, ptr %a
   %res = sitofp <8 x i16> %op1 to <8 x double>
-- 
GitLab


From eacefba9aa3d1a5181d3d49823df24aca0d2b344 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Wed, 15 May 2024 14:44:24 +0400
Subject: [PATCH 0833/1206] [lldb][Windows] Fixed tests TestPty and
 TestPtyServer (#92090)

The tests TestPty and TestPtyServer use the Unix specific python builtin
module termios. They are failed in case of Windows host and Linux
target. Disable them for Windows host too.
---
 lldb/test/API/functionalities/gdb_remote_client/TestPty.py | 2 +-
 lldb/test/API/tools/lldb-server/TestPtyServer.py           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/test/API/functionalities/gdb_remote_client/TestPty.py b/lldb/test/API/functionalities/gdb_remote_client/TestPty.py
index 4d4dd489b294..94eeb6e3ba11 100644
--- a/lldb/test/API/functionalities/gdb_remote_client/TestPty.py
+++ b/lldb/test/API/functionalities/gdb_remote_client/TestPty.py
@@ -5,7 +5,7 @@ from lldbsuite.test.gdbclientutils import *
 from lldbsuite.test.lldbgdbclient import GDBRemoteTestBase
 
 
-@skipIfWindows
+@skipIf(hostoslist=["windows"])
 class TestPty(GDBRemoteTestBase):
     server_socket_class = PtyServerSocket
 
diff --git a/lldb/test/API/tools/lldb-server/TestPtyServer.py b/lldb/test/API/tools/lldb-server/TestPtyServer.py
index aa5bd635650a..4bfcf70bfa01 100644
--- a/lldb/test/API/tools/lldb-server/TestPtyServer.py
+++ b/lldb/test/API/tools/lldb-server/TestPtyServer.py
@@ -7,7 +7,7 @@ from lldbgdbserverutils import *
 import xml.etree.ElementTree as ET
 
 
-@skipIfWindows
+@skipIf(hostoslist=["windows"])
 class PtyServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase):
     def setUp(self):
         super().setUp()
-- 
GitLab


From 3c3f6d877623d0d821f59f4ec6038b27f27ee01d Mon Sep 17 00:00:00 2001
From: Ivan Kosarev <ivan.kosarev@amd.com>
Date: Wed, 15 May 2024 13:53:38 +0300
Subject: [PATCH 0834/1206] [AMDGPU][AsmParser][NFC] Eliminate Match_PreferE32.
 (#92159)

Was added in 88e0b251815563016ad50241dd592e304bc03ee5 and is unused
since fcef407aa21ad5a79d66a088e6f2a66a5745725d.
---
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      | 20 +++++--------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index d47a5f8ebb81..c08c35c45984 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -1388,9 +1388,6 @@ private:
                     bool IsAtomic);
 
 public:
-  enum AMDGPUMatchResultTy {
-    Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
-  };
   enum OperandMode {
     OperandMode_Default,
     OperandMode_NSA,
@@ -5262,15 +5259,11 @@ bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                   Variant);
     // We order match statuses from least to most specific. We use most specific
     // status as resulting
-    // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
-    if ((R == Match_Success) ||
-        (R == Match_PreferE32) ||
-        (R == Match_MissingFeature && Result != Match_PreferE32) ||
-        (R == Match_InvalidOperand && Result != Match_MissingFeature
-                                   && Result != Match_PreferE32) ||
-        (R == Match_MnemonicFail   && Result != Match_InvalidOperand
-                                   && Result != Match_MissingFeature
-                                   && Result != Match_PreferE32)) {
+    // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature
+    if (R == Match_Success || R == Match_MissingFeature ||
+        (R == Match_InvalidOperand && Result != Match_MissingFeature) ||
+        (R == Match_MnemonicFail && Result != Match_InvalidOperand &&
+         Result != Match_MissingFeature)) {
       Result = R;
       ErrorInfo = EI;
     }
@@ -5316,9 +5309,6 @@ bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     return Error(ErrorLoc, "invalid operand for instruction");
   }
 
-  case Match_PreferE32:
-    return Error(IDLoc, "internal error: instruction without _e64 suffix "
-                        "should be encoded as e32");
   case Match_MnemonicFail:
     llvm_unreachable("Invalid instructions should have been handled already");
   }
-- 
GitLab


From de18f5ecf80ef7183625c80b04445c614a17c483 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 15 May 2024 06:23:34 -0500
Subject: [PATCH 0835/1206] [flang][OpenMP] Remove `allocate` from `taskgroup`
 in test (#92173)

Remove the `allocate`, because it needs to be used together with a
privatizing clause. The only such clause for `taskgroup` is
`task_reduction`, but it's not yet supported.
---
 flang/test/Lower/OpenMP/taskgroup.f90 | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/flang/test/Lower/OpenMP/taskgroup.f90 b/flang/test/Lower/OpenMP/taskgroup.f90
index 76458f1f1127..d9d262bdd2c0 100644
--- a/flang/test/Lower/OpenMP/taskgroup.f90
+++ b/flang/test/Lower/OpenMP/taskgroup.f90
@@ -1,17 +1,14 @@
-! REQUIRES: openmp_runtime
-
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
+! The "allocate" clause has been removed, because it needs to be used
+! together with a privatizing clause. The only such clause for "taskgroup"
+! is "task_reduction", but it's not yet supported.
+
 !CHECK-LABEL: @_QPomp_taskgroup
 subroutine omp_taskgroup
-use omp_lib
-integer :: allocated_x
-!CHECK: %[[ALLOC_X_REF:.*]] = fir.alloca i32 {bindc_name = "allocated_x", uniq_name = "_QFomp_taskgroupEallocated_x"}
-!CHECK-NEXT: %[[ALLOC_X_DECL:.*]]:2 = hlfir.declare %[[ALLOC_X_REF]] {uniq_name = "_QFomp_taskgroupEallocated_x"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[C4:.*]] = arith.constant 4 : i64
-
-!CHECK: omp.taskgroup  allocate(%[[C4]] : i64 -> %[[ALLOC_X_DECL]]#1 : !fir.ref<i32>)
-!$omp taskgroup allocate(omp_high_bw_mem_alloc: allocated_x)
+!CHECK: omp.taskgroup
+!$omp taskgroup
+!CHECK: omp.task
 !$omp task
 !CHECK: fir.call @_QPwork() {{.*}}: () -> ()
    call work()
-- 
GitLab


From e6ef836f23aa44520e0823c38e44b2f58eb5a52f Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 15 May 2024 06:45:57 -0500
Subject: [PATCH 0836/1206] [flang][OpenMP] Add -fopenmp-version=52 to
 teams.f90 (#92180)

One of the functions in the test has `teams if(...)`. The `if` clause
was only allowed on the `teams` directive in OpenMP 5.2.
---
 flang/test/Lower/OpenMP/teams.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Lower/OpenMP/teams.f90 b/flang/test/Lower/OpenMP/teams.f90
index f122a578a6e1..b1b2e7080676 100644
--- a/flang/test/Lower/OpenMP/teams.f90
+++ b/flang/test/Lower/OpenMP/teams.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPteams_simple
 subroutine teams_simple()
-- 
GitLab


From ccbf908b0836d8e3945f9331fd3679cbc6be0be1 Mon Sep 17 00:00:00 2001
From: Jan Patrick Lehr <JanPatrick.Lehr@amd.com>
Date: Wed, 15 May 2024 14:06:40 +0200
Subject: [PATCH 0837/1206] [libc] Fix GPU test build error (#92235)

This fixes a build error on the AMDGPU buildbot introduced in PR
https://github.com/llvm/llvm-project/pull/92172
---
 libc/src/__support/StringUtil/tables/stdc_errors.h | 3 +--
 libc/test/src/string/strerror_test.cpp             | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/libc/src/__support/StringUtil/tables/stdc_errors.h b/libc/src/__support/StringUtil/tables/stdc_errors.h
index a9c152783455..6873d6bd5107 100644
--- a/libc/src/__support/StringUtil/tables/stdc_errors.h
+++ b/libc/src/__support/StringUtil/tables/stdc_errors.h
@@ -15,11 +15,10 @@
 
 namespace LIBC_NAMESPACE {
 
-LIBC_INLINE_VAR constexpr const MsgTable<4> STDC_ERRORS = {
+LIBC_INLINE_VAR constexpr const MsgTable<3> STDC_ERRORS = {
     MsgMapping(0, "Success"),
     MsgMapping(EDOM, "Numerical argument out of domain"),
     MsgMapping(ERANGE, "Numerical result out of range"),
-    MsgMapping(EILSEQ, "Invalid or incomplete multibyte or wide character"),
 };
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/test/src/string/strerror_test.cpp b/libc/test/src/string/strerror_test.cpp
index ec9827b75cfc..2d6c230573a4 100644
--- a/libc/test/src/string/strerror_test.cpp
+++ b/libc/test/src/string/strerror_test.cpp
@@ -97,7 +97,7 @@ TEST(LlvmLibcStrErrorTest, KnownErrors) {
       ".lib section in a.out corrupted",
       "Attempting to link in too many shared libraries",
       "Cannot exec a shared library directly",
-      "Invalid or incomplete multibyte or wide character",
+      "Unknown Error 84", // Unknown
       "Interrupted system call should be restarted",
       "Streams pipe error",
       "Too many users",
-- 
GitLab


From 1650f1b3d7f97ca95eb930984e74bdfd91b02b4e Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Wed, 15 May 2024 13:10:16 +0100
Subject: [PATCH 0838/1206] Fix typo "indicies" (#92232)

---
 clang/include/clang/AST/VTTBuilder.h          |  6 +-
 clang/lib/AST/VTTBuilder.cpp                  |  2 +-
 clang/lib/CodeGen/CGVTT.cpp                   | 17 ++---
 clang/lib/CodeGen/CGVTables.h                 |  6 +-
 .../command/commands/DexExpectStepOrder.py    |  2 +-
 flang/docs/HighLevelFIR.md                    |  2 +-
 flang/test/Lower/HLFIR/forall.f90             |  2 +-
 libc/src/stdio/printf_core/parser.h           |  2 +-
 .../views/mdspan/CustomTestLayouts.h          |  2 +-
 llvm/docs/GlobalISel/GenericOpcode.rst        |  4 +-
 llvm/include/llvm/Target/Target.td            |  4 +-
 llvm/lib/Analysis/DependenceAnalysis.cpp      | 10 +--
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp     | 12 ++--
 llvm/lib/Bitcode/Writer/ValueEnumerator.cpp   |  2 +-
 llvm/lib/Bitcode/Writer/ValueEnumerator.h     |  2 +-
 .../LiveDebugValues/VarLocBasedImpl.cpp       |  2 +-
 llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp   |  2 +-
 llvm/lib/CodeGen/PrologEpilogInserter.cpp     |  2 +-
 llvm/lib/Support/ELFAttributeParser.cpp       | 10 +--
 .../Target/AArch64/AArch64ISelLowering.cpp    |  2 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  2 +-
 .../DirectX/DXILWriter/DXILBitcodeWriter.cpp  | 10 +--
 .../DXILWriter/DXILValueEnumerator.cpp        |  2 +-
 .../DirectX/DXILWriter/DXILValueEnumerator.h  |  2 +-
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |  2 +-
 .../Transforms/InstCombine/InstCombinePHI.cpp |  4 +-
 .../Scalar/SeparateConstOffsetFromGEP.cpp     |  2 +-
 .../Utils/SampleProfileInference.cpp          |  2 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 64 +++++++++----------
 llvm/test/CodeGen/X86/avx-vperm2x128.ll       |  2 +-
 .../test/DebugInfo/PDB/Inputs/every-type.yaml |  4 +-
 ...h-directive-personalityindex-diagnostics.s |  6 +-
 .../InstCombine/phi-extractvalue.ll           |  8 +--
 .../InstCombine/phi-of-insertvalues.ll        |  6 +-
 .../VectorCombine/X86/scalarize-vector-gep.ll | 12 ++--
 .../Linalg/Transforms/Vectorization.cpp       |  6 +-
 36 files changed, 114 insertions(+), 113 deletions(-)

diff --git a/clang/include/clang/AST/VTTBuilder.h b/clang/include/clang/AST/VTTBuilder.h
index 4acbc1f9e96b..3c19e61a8701 100644
--- a/clang/include/clang/AST/VTTBuilder.h
+++ b/clang/include/clang/AST/VTTBuilder.h
@@ -92,7 +92,7 @@ class VTTBuilder {
   using AddressPointsMapTy = llvm::DenseMap<BaseSubobject, uint64_t>;
 
   /// The sub-VTT indices for the bases of the most derived class.
-  llvm::DenseMap<BaseSubobject, uint64_t> SubVTTIndicies;
+  llvm::DenseMap<BaseSubobject, uint64_t> SubVTTIndices;
 
   /// The secondary virtual pointer indices of all subobjects of
   /// the most derived class.
@@ -148,8 +148,8 @@ public:
   }
 
   /// Returns a reference to the sub-VTT indices.
-  const llvm::DenseMap<BaseSubobject, uint64_t> &getSubVTTIndicies() const {
-    return SubVTTIndicies;
+  const llvm::DenseMap<BaseSubobject, uint64_t> &getSubVTTIndices() const {
+    return SubVTTIndices;
   }
 
   /// Returns a reference to the secondary virtual pointer indices.
diff --git a/clang/lib/AST/VTTBuilder.cpp b/clang/lib/AST/VTTBuilder.cpp
index d58e87517785..464a2014c430 100644
--- a/clang/lib/AST/VTTBuilder.cpp
+++ b/clang/lib/AST/VTTBuilder.cpp
@@ -189,7 +189,7 @@ void VTTBuilder::LayoutVTT(BaseSubobject Base, bool BaseIsVirtual) {
 
   if (!IsPrimaryVTT) {
     // Remember the sub-VTT index.
-    SubVTTIndicies[Base] = VTTComponents.size();
+    SubVTTIndices[Base] = VTTComponents.size();
   }
 
   uint64_t VTableIndex = VTTVTables.size();
diff --git a/clang/lib/CodeGen/CGVTT.cpp b/clang/lib/CodeGen/CGVTT.cpp
index d2376b14dd58..4cebb750c89e 100644
--- a/clang/lib/CodeGen/CGVTT.cpp
+++ b/clang/lib/CodeGen/CGVTT.cpp
@@ -138,23 +138,24 @@ uint64_t CodeGenVTables::getSubVTTIndex(const CXXRecordDecl *RD,
                                         BaseSubobject Base) {
   BaseSubobjectPairTy ClassSubobjectPair(RD, Base);
 
-  SubVTTIndiciesMapTy::iterator I = SubVTTIndicies.find(ClassSubobjectPair);
-  if (I != SubVTTIndicies.end())
+  SubVTTIndicesMapTy::iterator I = SubVTTIndices.find(ClassSubobjectPair);
+  if (I != SubVTTIndices.end())
     return I->second;
 
   VTTBuilder Builder(CGM.getContext(), RD, /*GenerateDefinition=*/false);
 
-  for (llvm::DenseMap<BaseSubobject, uint64_t>::const_iterator I =
-       Builder.getSubVTTIndicies().begin(),
-       E = Builder.getSubVTTIndicies().end(); I != E; ++I) {
+  for (llvm::DenseMap<BaseSubobject, uint64_t>::const_iterator
+           I = Builder.getSubVTTIndices().begin(),
+           E = Builder.getSubVTTIndices().end();
+       I != E; ++I) {
     // Insert all indices.
     BaseSubobjectPairTy ClassSubobjectPair(RD, I->first);
 
-    SubVTTIndicies.insert(std::make_pair(ClassSubobjectPair, I->second));
+    SubVTTIndices.insert(std::make_pair(ClassSubobjectPair, I->second));
   }
 
-  I = SubVTTIndicies.find(ClassSubobjectPair);
-  assert(I != SubVTTIndicies.end() && "Did not find index!");
+  I = SubVTTIndices.find(ClassSubobjectPair);
+  assert(I != SubVTTIndices.end() && "Did not find index!");
 
   return I->second;
 }
diff --git a/clang/lib/CodeGen/CGVTables.h b/clang/lib/CodeGen/CGVTables.h
index 9d4223547050..c06bf7a525d9 100644
--- a/clang/lib/CodeGen/CGVTables.h
+++ b/clang/lib/CodeGen/CGVTables.h
@@ -38,10 +38,10 @@ class CodeGenVTables {
   typedef VTableLayout::AddressPointsMapTy VTableAddressPointsMapTy;
 
   typedef std::pair<const CXXRecordDecl *, BaseSubobject> BaseSubobjectPairTy;
-  typedef llvm::DenseMap<BaseSubobjectPairTy, uint64_t> SubVTTIndiciesMapTy;
+  typedef llvm::DenseMap<BaseSubobjectPairTy, uint64_t> SubVTTIndicesMapTy;
 
-  /// SubVTTIndicies - Contains indices into the various sub-VTTs.
-  SubVTTIndiciesMapTy SubVTTIndicies;
+  /// SubVTTIndices - Contains indices into the various sub-VTTs.
+  SubVTTIndicesMapTy SubVTTIndices;
 
   typedef llvm::DenseMap<BaseSubobjectPairTy, uint64_t>
     SecondaryVirtualPointerIndicesMapTy;
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/command/commands/DexExpectStepOrder.py b/cross-project-tests/debuginfo-tests/dexter/dex/command/commands/DexExpectStepOrder.py
index cb5579b523dc..d6954a440f1a 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/command/commands/DexExpectStepOrder.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/command/commands/DexExpectStepOrder.py
@@ -12,7 +12,7 @@ from dex.dextIR import ValueIR
 
 class DexExpectStepOrder(CommandBase):
     """Expect the line every `DexExpectStepOrder` is found on to be stepped on
-    in `order`. Each instance must have a set of unique ascending indicies.
+    in `order`. Each instance must have a set of unique ascending indices.
 
     DexExpectStepOrder(*order)
 
diff --git a/flang/docs/HighLevelFIR.md b/flang/docs/HighLevelFIR.md
index de8dc5a1959b..2399efcdeacd 100644
--- a/flang/docs/HighLevelFIR.md
+++ b/flang/docs/HighLevelFIR.md
@@ -590,7 +590,7 @@ Syntax:
 
 Note that %indices are not operands, they are the elemental region block
 arguments, representing the array iteration space in a one based fashion.
-The choice of using one based indicies is to match Fortran default for
+The choice of using one based indices is to match Fortran default for
 array variables, so that there is no need to generate bound adjustments
 when working with one based array variables in an expression.
 
diff --git a/flang/test/Lower/HLFIR/forall.f90 b/flang/test/Lower/HLFIR/forall.f90
index 9941ed194010..c12f0c6a826b 100644
--- a/flang/test/Lower/HLFIR/forall.f90
+++ b/flang/test/Lower/HLFIR/forall.f90
@@ -144,7 +144,7 @@ subroutine test_nested_foralls()
     ! ifoo and ibar could depend on x since it is a module
     ! variable use associated. The calls in the control value
     ! computation cannot be hoisted from the outer forall
-    ! even when they do not depend on outer forall indicies.
+    ! even when they do not depend on outer forall indices.
     forall (integer(8)::j=jfoo():jbar())
       x(i, j) = x(j, i)
     end forall
diff --git a/libc/src/stdio/printf_core/parser.h b/libc/src/stdio/printf_core/parser.h
index eda978a83ea8..b9a8f303dd67 100644
--- a/libc/src/stdio/printf_core/parser.h
+++ b/libc/src/stdio/printf_core/parser.h
@@ -496,7 +496,7 @@ private:
   // the type of index, and returns a TypeDesc describing that type. It does not
   // modify cur_pos.
   LIBC_INLINE TypeDesc get_type_desc(size_t index) {
-    // index mode is assumed, and the indicies start at 1, so an index
+    // index mode is assumed, and the indices start at 1, so an index
     // of 0 is invalid.
     size_t local_pos = 0;
 
diff --git a/libcxx/test/std/containers/views/mdspan/CustomTestLayouts.h b/libcxx/test/std/containers/views/mdspan/CustomTestLayouts.h
index 3ac142cce3a3..588a5e9774a5 100644
--- a/libcxx/test/std/containers/views/mdspan/CustomTestLayouts.h
+++ b/libcxx/test/std/containers/views/mdspan/CustomTestLayouts.h
@@ -29,7 +29,7 @@
 #include <utility>
 
 // Layout that wraps indices to test some idiosyncratic behavior
-// - basically it is a layout_left where indicies are first wrapped i.e. i%Wrap
+// - basically it is a layout_left where indices are first wrapped i.e. i%Wrap
 // - only accepts integers as indices
 // - is_always_strided and is_always_unique are false
 // - is_strided and is_unique are true if all extents are smaller than Wrap
diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index 52dc039df777..5c28c6fcd30f 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -644,7 +644,7 @@ source vector should be inserted into.
 The index must be a constant multiple of the second source vector's minimum
 vector length. If the vectors are scalable, then the index is first scaled by
 the runtime scaling factor. The indices inserted in the source vector must be
-valid indicies of that vector. If this condition cannot be determined statically
+valid indices of that vector. If this condition cannot be determined statically
 but is false at runtime, then the result vector is undefined.
 
 .. code-block:: none
@@ -661,7 +661,7 @@ the source vector.
 The index must be a constant multiple of the source vector's minimum vector
 length. If the source vector is a scalable vector, then the index is first
 scaled by the runtime scaling factor. The indices extracted from the source
-vector must be valid indicies of that vector. If this condition cannot be
+vector must be valid indices of that vector. If this condition cannot be
 determined statically but is false at runtime, then the result vector is
 undefined.
 
diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td
index 1f7dc6922f13..343323860858 100644
--- a/llvm/include/llvm/Target/Target.td
+++ b/llvm/include/llvm/Target/Target.td
@@ -765,8 +765,8 @@ class Instruction : InstructionEncoding {
 
   /// Should generate helper functions that help you to map a logical operand's
   /// index to the underlying MIOperand's index.
-  /// In most architectures logical operand indicies are equal to
-  /// MIOperand indicies, but for some CISC architectures, a logical operand
+  /// In most architectures logical operand indices are equal to
+  /// MIOperand indices, but for some CISC architectures, a logical operand
   /// might be consist of multiple MIOperand (e.g. a logical operand that
   /// uses complex address mode).
   bit UseLogicalOperandMappings = false;
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index 1bce9aae09bb..e0e7dd18cd8d 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -3444,9 +3444,9 @@ bool DependenceInfo::tryDelinearizeFixedSize(
   // iff the subscripts are positive and are less than the range of the
   // dimension.
   if (!DisableDelinearizationChecks) {
-    auto AllIndiciesInRange = [&](SmallVector<int, 4> &DimensionSizes,
-                                  SmallVectorImpl<const SCEV *> &Subscripts,
-                                  Value *Ptr) {
+    auto AllIndicesInRange = [&](SmallVector<int, 4> &DimensionSizes,
+                                 SmallVectorImpl<const SCEV *> &Subscripts,
+                                 Value *Ptr) {
       size_t SSize = Subscripts.size();
       for (size_t I = 1; I < SSize; ++I) {
         const SCEV *S = Subscripts[I];
@@ -3462,8 +3462,8 @@ bool DependenceInfo::tryDelinearizeFixedSize(
       return true;
     };
 
-    if (!AllIndiciesInRange(SrcSizes, SrcSubscripts, SrcPtr) ||
-        !AllIndiciesInRange(DstSizes, DstSubscripts, DstPtr)) {
+    if (!AllIndicesInRange(SrcSizes, SrcSubscripts, SrcPtr) ||
+        !AllIndicesInRange(DstSizes, DstSubscripts, DstPtr)) {
       SrcSubscripts.clear();
       DstSubscripts.clear();
       return false;
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 6d01e3b4d821..c4cea3d6eef2 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -986,7 +986,7 @@ void ModuleBitcodeWriter::writeTypeTable() {
   Stream.EnterSubblock(bitc::TYPE_BLOCK_ID_NEW, 4 /*count from # abbrevs */);
   SmallVector<uint64_t, 64> TypeVals;
 
-  uint64_t NumBits = VE.computeBitsRequiredForTypeIndicies();
+  uint64_t NumBits = VE.computeBitsRequiredForTypeIndices();
 
   // Abbrev for TYPE_CODE_OPAQUE_POINTER.
   auto Abbv = std::make_shared<BitCodeAbbrev>();
@@ -3721,7 +3721,7 @@ void ModuleBitcodeWriter::writeBlockInfo() {
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
-                              VE.computeBitsRequiredForTypeIndicies()));
+                              VE.computeBitsRequiredForTypeIndices()));
     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
         CONSTANTS_SETTYPE_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering!");
@@ -3741,7 +3741,7 @@ void ModuleBitcodeWriter::writeBlockInfo() {
     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4));  // cast opc
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,       // typeid
-                              VE.computeBitsRequiredForTypeIndicies()));
+                              VE.computeBitsRequiredForTypeIndices()));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));    // value id
 
     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
@@ -3763,7 +3763,7 @@ void ModuleBitcodeWriter::writeBlockInfo() {
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Ptr
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,    // dest ty
-                              VE.computeBitsRequiredForTypeIndicies()));
+                              VE.computeBitsRequiredForTypeIndices()));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
@@ -3815,7 +3815,7 @@ void ModuleBitcodeWriter::writeBlockInfo() {
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));    // OpVal
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,       // dest ty
-                              VE.computeBitsRequiredForTypeIndicies()));
+                              VE.computeBitsRequiredForTypeIndices()));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4));  // opc
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_CAST_ABBREV)
@@ -3826,7 +3826,7 @@ void ModuleBitcodeWriter::writeBlockInfo() {
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpVal
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,    // dest ty
-                              VE.computeBitsRequiredForTypeIndicies()));
+                              VE.computeBitsRequiredForTypeIndices()));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
diff --git a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
index e6787e245a49..631f31cba976 100644
--- a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
+++ b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
@@ -1191,6 +1191,6 @@ unsigned ValueEnumerator::getGlobalBasicBlockID(const BasicBlock *BB) const {
   return getGlobalBasicBlockID(BB);
 }
 
-uint64_t ValueEnumerator::computeBitsRequiredForTypeIndicies() const {
+uint64_t ValueEnumerator::computeBitsRequiredForTypeIndices() const {
   return Log2_32_Ceil(getTypes().size() + 1);
 }
diff --git a/llvm/lib/Bitcode/Writer/ValueEnumerator.h b/llvm/lib/Bitcode/Writer/ValueEnumerator.h
index 4b45503595f6..8348d6728a5c 100644
--- a/llvm/lib/Bitcode/Writer/ValueEnumerator.h
+++ b/llvm/lib/Bitcode/Writer/ValueEnumerator.h
@@ -234,7 +234,7 @@ public:
   void incorporateFunction(const Function &F);
 
   void purgeFunction();
-  uint64_t computeBitsRequiredForTypeIndicies() const;
+  uint64_t computeBitsRequiredForTypeIndices() const;
 
 private:
   void OptimizeConstants(unsigned CstStart, unsigned CstEnd);
diff --git a/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
index bf730be00a9a..e146fb7e5768 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
@@ -86,7 +86,7 @@
 /// lookup the VarLoc in the VarLocMap. Rather than operate directly on machine
 /// locations, the dataflow analysis in this pass identifies locations by their
 /// indices in the VarLocMap, meaning all the variable locations in a block can
-/// be described by a sparse vector of VarLocMap indicies.
+/// be described by a sparse vector of VarLocMap indices.
 ///
 /// All the storage for the dataflow analysis is local to the ExtendRanges
 /// method and passed down to helper methods. "OutLocs" and "InLocs" record the
diff --git a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
index 114e7910dc27..f3a961f88351 100644
--- a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
@@ -212,7 +212,7 @@ static const std::vector<int64_t> PerLiveRangeShape{1, NumberOfInterferences};
   M(float, mbb_frequencies, MBBFrequencyShape,                                 \
     "A vector of machine basic block frequencies")                             \
   M(int64_t, mbb_mapping, InstructionsShape,                                   \
-    "A vector of indicies mapping instructions to MBBs")
+    "A vector of indices mapping instructions to MBBs")
 #else
 #define RA_EVICT_FIRST_DEVELOPMENT_FEATURE(M)
 #define RA_EVICT_REST_DEVELOPMENT_FEATURES(M)
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index eaf96ec5cbde..6a72797de493 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -1444,7 +1444,7 @@ bool PEI::replaceFrameIndexDebugInstr(MachineFunction &MF, MachineInstr &MI,
   // pointer as the base register.
   if (MI.getOpcode() == TargetOpcode::STATEPOINT) {
     assert((!MI.isDebugValue() || OpIdx == 0) &&
-           "Frame indicies can only appear as the first operand of a "
+           "Frame indices can only appear as the first operand of a "
            "DBG_VALUE machine instruction");
     Register Reg;
     MachineOperand &Offset = MI.getOperand(OpIdx + 1);
diff --git a/llvm/lib/Support/ELFAttributeParser.cpp b/llvm/lib/Support/ELFAttributeParser.cpp
index d3100c9ebb21..26c3d54e17ad 100644
--- a/llvm/lib/Support/ELFAttributeParser.cpp
+++ b/llvm/lib/Support/ELFAttributeParser.cpp
@@ -154,7 +154,7 @@ Error ELFAttributeParser::parseSubsection(uint32_t length) {
                                    Twine::utohexstr(cursor.tell() - 5));
 
     StringRef scopeName, indexName;
-    SmallVector<uint8_t, 8> indicies;
+    SmallVector<uint8_t, 8> indices;
     switch (tag) {
     case ELFAttrs::File:
       scopeName = "FileAttributes";
@@ -162,12 +162,12 @@ Error ELFAttributeParser::parseSubsection(uint32_t length) {
     case ELFAttrs::Section:
       scopeName = "SectionAttributes";
       indexName = "Sections";
-      parseIndexList(indicies);
+      parseIndexList(indices);
       break;
     case ELFAttrs::Symbol:
       scopeName = "SymbolAttributes";
       indexName = "Symbols";
-      parseIndexList(indicies);
+      parseIndexList(indices);
       break;
     default:
       return createStringError(errc::invalid_argument,
@@ -178,8 +178,8 @@ Error ELFAttributeParser::parseSubsection(uint32_t length) {
 
     if (sw) {
       DictScope scope(*sw, scopeName);
-      if (!indicies.empty())
-        sw->printList(indexName, indicies);
+      if (!indices.empty())
+        sw->printList(indexName, indices);
       if (Error e = parseAttributeList(size - 5))
         return e;
     } else if (Error e = parseAttributeList(size - 5))
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index afa023220d35..6223c211b33b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -23881,7 +23881,7 @@ static SDValue performScatterStoreCombine(SDNode *N, SelectionDAG &DAG,
 
   // For "scalar + vector of indices", just scale the indices. This only
   // applies to non-temporal scatters because there's no instruction that takes
-  // indicies.
+  // indices.
   if (Opcode == AArch64ISD::SSTNT1_INDEX_PRED) {
     Offset =
         getScaledOffsetForBitWidth(DAG, Offset, DL, SrcElVT.getSizeInBits());
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8f741ffc58a8..89e83babcfef 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7053,7 +7053,7 @@ SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
   SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT,
                                DAG.getSplatBuildVector(VecVT, SL, InsVal));
 
-  // 2. Mask off all other indicies except the required index within (1).
+  // 2. Mask off all other indices except the required index within (1).
   SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal);
 
   // 3. Mask off the required index within the target vector.
diff --git a/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp b/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp
index ebb269c6e6e0..87297ac86b9d 100644
--- a/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp
+++ b/llvm/lib/Target/DirectX/DXILWriter/DXILBitcodeWriter.cpp
@@ -958,7 +958,7 @@ void DXILBitcodeWriter::writeTypeTable() {
   Stream.EnterSubblock(bitc::TYPE_BLOCK_ID_NEW, 4 /*count from # abbrevs */);
   SmallVector<uint64_t, 64> TypeVals;
 
-  uint64_t NumBits = VE.computeBitsRequiredForTypeIndicies();
+  uint64_t NumBits = VE.computeBitsRequiredForTypeIndices();
 
   // Abbrev for TYPE_CODE_POINTER.
   auto Abbv = std::make_shared<BitCodeAbbrev>();
@@ -2747,7 +2747,7 @@ void DXILBitcodeWriter::writeBlockInfo() {
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
-                              VE.computeBitsRequiredForTypeIndicies()));
+                              VE.computeBitsRequiredForTypeIndices()));
     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, std::move(Abbv)) !=
         CONSTANTS_SETTYPE_ABBREV)
       assert(false && "Unexpected abbrev ordering!");
@@ -2767,7 +2767,7 @@ void DXILBitcodeWriter::writeBlockInfo() {
     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,      // typeid
-                              VE.computeBitsRequiredForTypeIndicies()));
+                              VE.computeBitsRequiredForTypeIndices()));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
 
     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, std::move(Abbv)) !=
@@ -2789,7 +2789,7 @@ void DXILBitcodeWriter::writeBlockInfo() {
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Ptr
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,    // dest ty
-                              VE.computeBitsRequiredForTypeIndicies()));
+                              VE.computeBitsRequiredForTypeIndices()));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // Align
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, std::move(Abbv)) !=
@@ -2822,7 +2822,7 @@ void DXILBitcodeWriter::writeBlockInfo() {
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpVal
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,    // dest ty
-                              VE.computeBitsRequiredForTypeIndicies()));
+                              VE.computeBitsRequiredForTypeIndices()));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, std::move(Abbv)) !=
         (unsigned)FUNCTION_INST_CAST_ABBREV)
diff --git a/llvm/lib/Target/DirectX/DXILWriter/DXILValueEnumerator.cpp b/llvm/lib/Target/DirectX/DXILWriter/DXILValueEnumerator.cpp
index d90ab968c5d3..9a8d0afa6292 100644
--- a/llvm/lib/Target/DirectX/DXILWriter/DXILValueEnumerator.cpp
+++ b/llvm/lib/Target/DirectX/DXILWriter/DXILValueEnumerator.cpp
@@ -1140,6 +1140,6 @@ unsigned ValueEnumerator::getGlobalBasicBlockID(const BasicBlock *BB) const {
   return getGlobalBasicBlockID(BB);
 }
 
-uint64_t ValueEnumerator::computeBitsRequiredForTypeIndicies() const {
+uint64_t ValueEnumerator::computeBitsRequiredForTypeIndices() const {
   return Log2_32_Ceil(getTypes().size() + 1);
 }
diff --git a/llvm/lib/Target/DirectX/DXILWriter/DXILValueEnumerator.h b/llvm/lib/Target/DirectX/DXILWriter/DXILValueEnumerator.h
index 66a5d96080bc..f0f91c6182e3 100644
--- a/llvm/lib/Target/DirectX/DXILWriter/DXILValueEnumerator.h
+++ b/llvm/lib/Target/DirectX/DXILWriter/DXILValueEnumerator.h
@@ -236,7 +236,7 @@ public:
   void incorporateFunction(const Function &F);
 
   void purgeFunction();
-  uint64_t computeBitsRequiredForTypeIndicies() const;
+  uint64_t computeBitsRequiredForTypeIndices() const;
 
   void EnumerateType(Type *T);
 
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 0a7483fc45b2..ad86c393ba79 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -14945,7 +14945,7 @@ static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) {
     }
   }
 
-  // If the vector extract indicies are not correct, add the appropriate
+  // If the vector extract indices are not correct, add the appropriate
   // vector_shuffle.
   int TgtElemArrayIdx;
   int InputSize = Input.getValueType().getScalarSizeInBits();
diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
index 52803e9bea45..dd8eb4688d3c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
@@ -331,7 +331,7 @@ Instruction *
 InstCombinerImpl::foldPHIArgInsertValueInstructionIntoPHI(PHINode &PN) {
   auto *FirstIVI = cast<InsertValueInst>(PN.getIncomingValue(0));
 
-  // Scan to see if all operands are `insertvalue`'s with the same indicies,
+  // Scan to see if all operands are `insertvalue`'s with the same indices,
   // and all have a single use.
   for (Value *V : drop_begin(PN.incoming_values())) {
     auto *I = dyn_cast<InsertValueInst>(V);
@@ -371,7 +371,7 @@ Instruction *
 InstCombinerImpl::foldPHIArgExtractValueInstructionIntoPHI(PHINode &PN) {
   auto *FirstEVI = cast<ExtractValueInst>(PN.getIncomingValue(0));
 
-  // Scan to see if all operands are `extractvalue`'s with the same indicies,
+  // Scan to see if all operands are `extractvalue`'s with the same indices,
   // and all have a single use.
   for (Value *V : drop_begin(PN.incoming_values())) {
     auto *I = dyn_cast<ExtractValueInst>(V);
diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 9f85396cde25..1a9eaf28f6e4 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -1008,7 +1008,7 @@ bool SeparateConstOffsetFromGEP::reorderGEP(GetElementPtrInst *GEP,
   }
 
   IRBuilder<> Builder(GEP);
-  // For trivial GEP chains, we can swap the indicies.
+  // For trivial GEP chains, we can swap the indices.
   Value *NewSrc = Builder.CreateGEP(
       GEP->getSourceElementType(), PtrGEP->getPointerOperand(),
       SmallVector<Value *, 4>(GEP->indices()), "", IsChainInBounds);
diff --git a/llvm/lib/Transforms/Utils/SampleProfileInference.cpp b/llvm/lib/Transforms/Utils/SampleProfileInference.cpp
index 101b70d8def4..54d46117729c 100644
--- a/llvm/lib/Transforms/Utils/SampleProfileInference.cpp
+++ b/llvm/lib/Transforms/Utils/SampleProfileInference.cpp
@@ -1061,7 +1061,7 @@ void initializeNetwork(const ProfiParams &Params, MinCostMaxFlow &Network,
   assert(NumJumps > 0 && "Too few jumps in a function");
 
   // Introducing dummy source/sink pairs to allow flow circulation.
-  // The nodes corresponding to blocks of the function have indicies in
+  // The nodes corresponding to blocks of the function have indices in
   // the range [0 .. 2 * NumBlocks); the dummy sources/sinks are indexed by the
   // next four values.
   uint64_t S = 2 * NumBlocks;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 2e0a39c4b4fd..d21b5e1cc041 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6485,7 +6485,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
                             const EdgeInfo &UserTreeIdx) {
   assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
 
-  SmallVector<int> ReuseShuffleIndicies;
+  SmallVector<int> ReuseShuffleIndices;
   SmallVector<Value *> UniqueValues;
   SmallVector<Value *> NonUniqueValueVL;
   auto TryToFindDuplicates = [&](const InstructionsState &S,
@@ -6494,19 +6494,19 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     DenseMap<Value *, unsigned> UniquePositions(VL.size());
     for (Value *V : VL) {
       if (isConstant(V)) {
-        ReuseShuffleIndicies.emplace_back(
+        ReuseShuffleIndices.emplace_back(
             isa<UndefValue>(V) ? PoisonMaskElem : UniqueValues.size());
         UniqueValues.emplace_back(V);
         continue;
       }
       auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
-      ReuseShuffleIndicies.emplace_back(Res.first->second);
+      ReuseShuffleIndices.emplace_back(Res.first->second);
       if (Res.second)
         UniqueValues.emplace_back(V);
     }
     size_t NumUniqueScalarValues = UniqueValues.size();
     if (NumUniqueScalarValues == VL.size()) {
-      ReuseShuffleIndicies.clear();
+      ReuseShuffleIndices.clear();
     } else {
       // FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops.
       if (UserTreeIdx.UserTE && UserTreeIdx.UserTE->isNonPowOf2Vec()) {
@@ -6532,7 +6532,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
             })) {
           unsigned PWSz = PowerOf2Ceil(UniqueValues.size());
           if (PWSz == VL.size()) {
-            ReuseShuffleIndicies.clear();
+            ReuseShuffleIndices.clear();
           } else {
             NonUniqueValueVL.assign(UniqueValues.begin(), UniqueValues.end());
             NonUniqueValueVL.append(PWSz - UniqueValues.size(),
@@ -6579,7 +6579,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
     if (TryToFindDuplicates(S))
       newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
-                   ReuseShuffleIndicies);
+                   ReuseShuffleIndices);
     return;
   }
 
@@ -6590,7 +6590,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
     if (TryToFindDuplicates(S))
       newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
-                   ReuseShuffleIndicies);
+                   ReuseShuffleIndices);
     return;
   }
 
@@ -6694,7 +6694,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O, small shuffle. \n");
     if (TryToFindDuplicates(S))
       newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
-                   ReuseShuffleIndicies);
+                   ReuseShuffleIndices);
     return;
   }
 
@@ -6722,7 +6722,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
         LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
         if (TryToFindDuplicates(S))
           newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
-                       ReuseShuffleIndicies);
+                       ReuseShuffleIndices);
         return;
       }
     } else {
@@ -6745,7 +6745,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
                         << ") is already in tree.\n");
       if (TryToFindDuplicates(S))
         newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
-                     ReuseShuffleIndicies);
+                     ReuseShuffleIndices);
       return;
     }
   }
@@ -6757,7 +6757,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
         LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
         if (TryToFindDuplicates(S))
           newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
-                       ReuseShuffleIndicies);
+                       ReuseShuffleIndices);
         return;
       }
     }
@@ -6810,7 +6810,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps);
   if (State == TreeEntry::NeedToGather) {
     newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
-                 ReuseShuffleIndicies);
+                 ReuseShuffleIndices);
     return;
   }
 
@@ -6832,7 +6832,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
             !BS.getScheduleData(VL0)->isPartOfBundle()) &&
            "tryScheduleBundle should cancelScheduling on failure");
     newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
-                 ReuseShuffleIndicies);
+                 ReuseShuffleIndices);
     NonScheduledFirst.insert(VL.front());
     return;
   }
@@ -6845,7 +6845,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       auto *PH = cast<PHINode>(VL0);
 
       TreeEntry *TE =
-          newTreeEntry(VL, Bundle, S, UserTreeIdx, ReuseShuffleIndicies);
+          newTreeEntry(VL, Bundle, S, UserTreeIdx, ReuseShuffleIndices);
       LLVM_DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n");
 
       // Keeps the reordered operands to avoid code duplication.
@@ -6862,7 +6862,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       if (CurrentOrder.empty()) {
         LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n");
         newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                     ReuseShuffleIndicies);
+                     ReuseShuffleIndices);
         // This is a special case, as it does not gather, but at the same time
         // we are not extending buildTree_rec() towards the operands.
         ValueList Op0;
@@ -6881,7 +6881,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       // Insert new order with initial value 0, if it does not exist,
       // otherwise return the iterator to the existing one.
       newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                   ReuseShuffleIndicies, CurrentOrder);
+                   ReuseShuffleIndices, CurrentOrder);
       // This is a special case, as it does not gather, but at the same time
       // we are not extending buildTree_rec() towards the operands.
       ValueList Op0;
@@ -6890,7 +6890,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       return;
     }
     case Instruction::InsertElement: {
-      assert(ReuseShuffleIndicies.empty() && "All inserts should be unique");
+      assert(ReuseShuffleIndices.empty() && "All inserts should be unique");
 
       auto OrdCompare = [](const std::pair<int, int> &P1,
                            const std::pair<int, int> &P2) {
@@ -6941,12 +6941,12 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
         if (CurrentOrder.empty()) {
           // Original loads are consecutive and does not require reordering.
           TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                            ReuseShuffleIndicies);
+                            ReuseShuffleIndices);
           LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n");
         } else {
           // Need to reorder.
           TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                            ReuseShuffleIndicies, CurrentOrder);
+                            ReuseShuffleIndices, CurrentOrder);
           LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n");
         }
         TE->setOperandsInOrder();
@@ -6955,10 +6955,10 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
         // Vectorizing non-consecutive loads with `llvm.masked.gather`.
         if (CurrentOrder.empty()) {
           TE = newTreeEntry(VL, TreeEntry::StridedVectorize, Bundle, S,
-                            UserTreeIdx, ReuseShuffleIndicies);
+                            UserTreeIdx, ReuseShuffleIndices);
         } else {
           TE = newTreeEntry(VL, TreeEntry::StridedVectorize, Bundle, S,
-                            UserTreeIdx, ReuseShuffleIndicies, CurrentOrder);
+                            UserTreeIdx, ReuseShuffleIndices, CurrentOrder);
         }
         TE->setOperandsInOrder();
         LLVM_DEBUG(dbgs() << "SLP: added a vector of strided loads.\n");
@@ -6966,7 +6966,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       case TreeEntry::ScatterVectorize:
         // Vectorizing non-consecutive loads with `llvm.masked.gather`.
         TE = newTreeEntry(VL, TreeEntry::ScatterVectorize, Bundle, S,
-                          UserTreeIdx, ReuseShuffleIndicies);
+                          UserTreeIdx, ReuseShuffleIndices);
         TE->setOperandsInOrder();
         buildTree_rec(PointerOps, Depth + 1, {TE, 0});
         LLVM_DEBUG(dbgs() << "SLP: added a vector of non-consecutive loads.\n");
@@ -7020,7 +7020,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
           ExtraBitWidthNodes.insert(VectorizableTree.size() + 1);
       }
       TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                                   ReuseShuffleIndicies);
+                                   ReuseShuffleIndices);
       LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n");
 
       TE->setOperandsInOrder();
@@ -7039,7 +7039,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       // Check that all of the compares have the same predicate.
       CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
       TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                                   ReuseShuffleIndicies);
+                                   ReuseShuffleIndices);
       LLVM_DEBUG(dbgs() << "SLP: added a vector of compares.\n");
 
       ValueList Left, Right;
@@ -7100,7 +7100,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     case Instruction::Or:
     case Instruction::Xor: {
       TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                                   ReuseShuffleIndicies);
+                                   ReuseShuffleIndices);
       LLVM_DEBUG(dbgs() << "SLP: added a vector of un/bin op.\n");
 
       // Sort operands of the instructions so that each side is more likely to
@@ -7128,7 +7128,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     }
     case Instruction::GetElementPtr: {
       TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                                   ReuseShuffleIndicies);
+                                   ReuseShuffleIndices);
       LLVM_DEBUG(dbgs() << "SLP: added a vector of GEPs.\n");
       SmallVector<ValueList, 2> Operands(2);
       // Prepare the operand vector for pointer operands.
@@ -7194,14 +7194,14 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       if (CurrentOrder.empty()) {
         // Original stores are consecutive and does not require reordering.
         TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                                     ReuseShuffleIndicies);
+                                     ReuseShuffleIndices);
         TE->setOperandsInOrder();
         buildTree_rec(Operands, Depth + 1, {TE, 0});
         LLVM_DEBUG(dbgs() << "SLP: added a vector of stores.\n");
       } else {
         fixupOrderingIndices(CurrentOrder);
         TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                                     ReuseShuffleIndicies, CurrentOrder);
+                                     ReuseShuffleIndices, CurrentOrder);
         TE->setOperandsInOrder();
         buildTree_rec(Operands, Depth + 1, {TE, 0});
         LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled stores.\n");
@@ -7215,7 +7215,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
 
       TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                                   ReuseShuffleIndicies);
+                                   ReuseShuffleIndices);
       // Sort operands of the instructions so that each side is more likely to
       // have the same opcode.
       if (isCommutative(VL0)) {
@@ -7261,7 +7261,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     }
     case Instruction::ShuffleVector: {
       TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
-                                   ReuseShuffleIndicies);
+                                   ReuseShuffleIndices);
       LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
 
       // Reorder operands if reordering would enable vectorization.
@@ -12107,8 +12107,8 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
   unsigned VF = E->getVectorFactor();
 
   bool NeedFreeze = false;
-  SmallVector<int> ReuseShuffleIndicies(E->ReuseShuffleIndices.begin(),
-                                        E->ReuseShuffleIndices.end());
+  SmallVector<int> ReuseShuffleIndices(E->ReuseShuffleIndices.begin(),
+                                       E->ReuseShuffleIndices.end());
   SmallVector<Value *> GatheredScalars(E->Scalars.begin(), E->Scalars.end());
   // Build a mask out of the reorder indices and reorder scalars per this
   // mask.
diff --git a/llvm/test/CodeGen/X86/avx-vperm2x128.ll b/llvm/test/CodeGen/X86/avx-vperm2x128.ll
index a11b92a663c4..60fab8bc6737 100644
--- a/llvm/test/CodeGen/X86/avx-vperm2x128.ll
+++ b/llvm/test/CodeGen/X86/avx-vperm2x128.ll
@@ -234,7 +234,7 @@ entry:
   ret <16 x i16> %shuffle
 }
 
-;;;; Cases with undef indicies mixed in the mask
+;;;; Cases with undef indices mixed in the mask
 
 define <8 x float> @shuffle_v8f32_uu67u9ub(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
 ; ALL-LABEL: shuffle_v8f32_uu67u9ub:
diff --git a/llvm/test/DebugInfo/PDB/Inputs/every-type.yaml b/llvm/test/DebugInfo/PDB/Inputs/every-type.yaml
index 191c49042be6..4cd3034fe40a 100644
--- a/llvm/test/DebugInfo/PDB/Inputs/every-type.yaml
+++ b/llvm/test/DebugInfo/PDB/Inputs/every-type.yaml
@@ -19,11 +19,11 @@ TpiStream:
   # (int, char **)                                  [Index: 0x1003]
   - Kind:            LF_ARGLIST
     ArgList:
-      ArgIndicies:  [ 116, 0x1002 ]
+      ArgIndices:  [ 116, 0x1002 ]
   # (int, double)                                   [Index: 0x1004]
   - Kind:            LF_ARGLIST
     ArgList:
-      ArgIndicies:  [ 116, 65 ]       # (int, double)
+      ArgIndices:  [ 116, 65 ]       # (int, double)
   # int main(int argc, char **argv)                 [Index: 0x1005]
   - Kind:            LF_PROCEDURE
     Procedure:
diff --git a/llvm/test/MC/ARM/eh-directive-personalityindex-diagnostics.s b/llvm/test/MC/ARM/eh-directive-personalityindex-diagnostics.s
index 2dc2c8045a65..0158035c682c 100644
--- a/llvm/test/MC/ARM/eh-directive-personalityindex-diagnostics.s
+++ b/llvm/test/MC/ARM/eh-directive-personalityindex-diagnostics.s
@@ -65,10 +65,10 @@ multiple_personality:
 @ CHECK: 	.personalityindex 0
 @ CHECK:       ^
 
-	.global multiple_personality_indicies
-	.type multiple_personality_indicies,%function
+	.global multiple_personality_indices
+	.type multiple_personality_indices,%function
 	.thumb_func
-multiple_personality_indicies:
+multiple_personality_indices:
 	.fnstart
 	.personalityindex 0
 	.personalityindex 1
diff --git a/llvm/test/Transforms/InstCombine/phi-extractvalue.ll b/llvm/test/Transforms/InstCombine/phi-extractvalue.ll
index 75fd4718721c..26893c178f59 100644
--- a/llvm/test/Transforms/InstCombine/phi-extractvalue.ll
+++ b/llvm/test/Transforms/InstCombine/phi-extractvalue.ll
@@ -131,7 +131,7 @@ end:
   ret i32 %r
 }
 
-; But the indicies must match
+; But the indices must match
 define i32 @test4({ i32, i32 } %agg_left, { i32, i32 } %agg_right, i1 %c) {
 ; CHECK-LABEL: @test4(
 ; CHECK-NEXT:  entry:
@@ -162,7 +162,7 @@ end:
   ret i32 %r
 }
 
-; More complex aggregates are fine, too, as long as indicies match.
+; More complex aggregates are fine, too, as long as indices match.
 define i32 @test5({{ i32, i32 }, { i32, i32 }} %agg_left, {{ i32, i32 }, { i32, i32 }} %agg_right, i1 %c) {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:  entry:
@@ -192,7 +192,7 @@ end:
   ret i32 %r
 }
 
-; The indicies must fully match, on all levels.
+; The indices must fully match, on all levels.
 define i32 @test6({{ i32, i32 }, { i32, i32 }} %agg_left, {{ i32, i32 }, { i32, i32 }} %agg_right, i1 %c) {
 ; CHECK-LABEL: @test6(
 ; CHECK-NEXT:  entry:
@@ -282,7 +282,7 @@ end:
 }
 
 ; Also, unlike PHI-of-insertvalues, here the base aggregates of extractvalue
-; can have different types, and just checking the indicies is not enough.
+; can have different types, and just checking the indices is not enough.
 define i32 @test9({ i32, i32 } %agg_left, { i32, { i32, i32 } } %agg_right, i1 %c) {
 ; CHECK-LABEL: @test9(
 ; CHECK-NEXT:  entry:
diff --git a/llvm/test/Transforms/InstCombine/phi-of-insertvalues.ll b/llvm/test/Transforms/InstCombine/phi-of-insertvalues.ll
index 548fb3bc9ddb..3596ef198303 100644
--- a/llvm/test/Transforms/InstCombine/phi-of-insertvalues.ll
+++ b/llvm/test/Transforms/InstCombine/phi-of-insertvalues.ll
@@ -192,7 +192,7 @@ end:
   ret { i32, i32 } %r
 }
 
-; But the indicies must match
+; But the indices must match
 define { i32, i32 } @test6({ i32, i32 } %agg, i32 %val_left, i32 %val_right, i1 %c) {
 ; CHECK-LABEL: @test6(
 ; CHECK-NEXT:  entry:
@@ -223,7 +223,7 @@ end:
   ret { i32, i32 } %r
 }
 
-; More complex aggregates are fine, too, as long as indicies match.
+; More complex aggregates are fine, too, as long as indices match.
 define {{ i32, i32 }, { i32, i32 }} @test7({{ i32, i32 }, { i32, i32 }} %agg, i32 %val_left, i32 %val_right, i1 %c) {
 ; CHECK-LABEL: @test7(
 ; CHECK-NEXT:  entry:
@@ -253,7 +253,7 @@ end:
   ret {{ i32, i32 }, { i32, i32 }} %r
 }
 
-; The indicies must fully match, on all levels.
+; The indices must fully match, on all levels.
 define {{ i32, i32 }, { i32, i32 }} @test8({{ i32, i32 }, { i32, i32 }} %agg, i32 %val_left, i32 %val_right, i1 %c) {
 ; CHECK-LABEL: @test8(
 ; CHECK-NEXT:  entry:
diff --git a/llvm/test/Transforms/VectorCombine/X86/scalarize-vector-gep.ll b/llvm/test/Transforms/VectorCombine/X86/scalarize-vector-gep.ll
index e227e9911bc3..ccdc007f674f 100644
--- a/llvm/test/Transforms/VectorCombine/X86/scalarize-vector-gep.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/scalarize-vector-gep.ll
@@ -85,8 +85,8 @@ define void @both_operands_need_extraction.4elts(<4 x ptr> %baseptrs, <4 x i64>
 
 ;-------------------------------------------------------------------------------
 
-define void @indicies_need_extraction.2elts(ptr %baseptr, <2 x i64> %indices) {
-; CHECK-LABEL: @indicies_need_extraction.2elts(
+define void @indices_need_extraction.2elts(ptr %baseptr, <2 x i64> %indices) {
+; CHECK-LABEL: @indices_need_extraction.2elts(
 ; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr inbounds i64, ptr [[BASEPTR:%.*]], <2 x i64> [[INDICES:%.*]]
 ; CHECK-NEXT:    [[PTR_0:%.*]] = extractelement <2 x ptr> [[PTRS]], i64 0
 ; CHECK-NEXT:    call void @use(ptr [[PTR_0]])
@@ -105,8 +105,8 @@ define void @indicies_need_extraction.2elts(ptr %baseptr, <2 x i64> %indices) {
   ret void
 }
 
-define void @indicies_need_extraction.3elts(ptr %baseptr, <3 x i64> %indices) {
-; CHECK-LABEL: @indicies_need_extraction.3elts(
+define void @indices_need_extraction.3elts(ptr %baseptr, <3 x i64> %indices) {
+; CHECK-LABEL: @indices_need_extraction.3elts(
 ; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr inbounds i64, ptr [[BASEPTR:%.*]], <3 x i64> [[INDICES:%.*]]
 ; CHECK-NEXT:    [[PTR_0:%.*]] = extractelement <3 x ptr> [[PTRS]], i64 0
 ; CHECK-NEXT:    call void @use(ptr [[PTR_0]])
@@ -130,8 +130,8 @@ define void @indicies_need_extraction.3elts(ptr %baseptr, <3 x i64> %indices) {
   ret void
 }
 
-define void @indicies_need_extraction.4elts(ptr %baseptr, <4 x i64> %indices) {
-; CHECK-LABEL: @indicies_need_extraction.4elts(
+define void @indices_need_extraction.4elts(ptr %baseptr, <4 x i64> %indices) {
+; CHECK-LABEL: @indices_need_extraction.4elts(
 ; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr inbounds i64, ptr [[BASEPTR:%.*]], <4 x i64> [[INDICES:%.*]]
 ; CHECK-NEXT:    [[PTR_0:%.*]] = extractelement <4 x ptr> [[PTRS]], i64 0
 ; CHECK-NEXT:    call void @use(ptr [[PTR_0]])
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 7b4507c52e02..511835a226e7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -3385,13 +3385,13 @@ struct Conv1DGenerator
       auto rhsSize = cast<VectorType>(rhs.getType()).getShape()[0];
       auto resSize = cast<VectorType>(res.getType()).getShape()[1];
 
-      SmallVector<int64_t, 16> indicies;
+      SmallVector<int64_t, 16> indices;
       for (int i = 0; i < resSize / rhsSize; ++i) {
         for (int j = 0; j < rhsSize; ++j)
-          indicies.push_back(j);
+          indices.push_back(j);
       }
 
-      rhs = rewriter.create<vector::ShuffleOp>(loc, rhs, rhs, indicies);
+      rhs = rewriter.create<vector::ShuffleOp>(loc, rhs, rhs, indices);
     }
     // Broadcast the filter to match the output vector
     rhs = rewriter.create<vector::BroadcastOp>(
-- 
GitLab


From 398162ddbcf741c49e86bef2ef4aaa3fd0213916 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Wed, 15 May 2024 07:12:25 -0500
Subject: [PATCH 0839/1206] [libc] Fix typo in test message

---
 libc/test/src/string/strerror_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/test/src/string/strerror_test.cpp b/libc/test/src/string/strerror_test.cpp
index 2d6c230573a4..cfc79481699b 100644
--- a/libc/test/src/string/strerror_test.cpp
+++ b/libc/test/src/string/strerror_test.cpp
@@ -97,7 +97,7 @@ TEST(LlvmLibcStrErrorTest, KnownErrors) {
       ".lib section in a.out corrupted",
       "Attempting to link in too many shared libraries",
       "Cannot exec a shared library directly",
-      "Unknown Error 84", // Unknown
+      "Unknown error 84", // Unknown
       "Interrupted system call should be restarted",
       "Streams pipe error",
       "Too many users",
-- 
GitLab


From 3b5a121a2478e586f59e3277d04d17fb63be5d76 Mon Sep 17 00:00:00 2001
From: Julian Schmidt <git.julian.schmidt@gmail.com>
Date: Wed, 15 May 2024 14:35:07 +0200
Subject: [PATCH 0840/1206] [clang-tidy][NFC] replace comparison of begin and
 end iterators with range empty (#91994)

Improves readability by changing comparisons of `*_begin` and `*_end`
iterators into `.empty()` on their range.
---
 .../clang-tidy/bugprone/SuspiciousEnumUsageCheck.cpp          | 3 +--
 .../MisleadingCaptureDefaultByValueCheck.cpp                  | 3 +--
 clang-tools-extra/clang-tidy/misc/UnusedParametersCheck.cpp   | 4 +---
 .../clang-tidy/modernize/UseConstraintsCheck.cpp              | 2 +-
 4 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/bugprone/SuspiciousEnumUsageCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SuspiciousEnumUsageCheck.cpp
index ca1ae551cc63..2fca7ae2e7ee 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SuspiciousEnumUsageCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SuspiciousEnumUsageCheck.cpp
@@ -171,8 +171,7 @@ void SuspiciousEnumUsageCheck::check(const MatchFinder::MatchResult &Result) {
     // Skip when one of the parameters is an empty enum. The
     // hasDisjointValueRange function could not decide the values properly in
     // case of an empty enum.
-    if (EnumDec->enumerator_begin() == EnumDec->enumerator_end() ||
-        OtherEnumDec->enumerator_begin() == OtherEnumDec->enumerator_end())
+    if (EnumDec->enumerators().empty() || OtherEnumDec->enumerators().empty())
       return;
 
     if (!hasDisjointValueRange(EnumDec, OtherEnumDec))
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/MisleadingCaptureDefaultByValueCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/MisleadingCaptureDefaultByValueCheck.cpp
index 00dfa17a1ccf..5dee7f91a934 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/MisleadingCaptureDefaultByValueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/MisleadingCaptureDefaultByValueCheck.cpp
@@ -67,8 +67,7 @@ static std::string createReplacementText(const LambdaExpr *Lambda) {
       AppendName("this");
     }
   }
-  if (!Replacement.empty() &&
-      Lambda->explicit_capture_begin() != Lambda->explicit_capture_end()) {
+  if (!Replacement.empty() && !Lambda->explicit_captures().empty()) {
     // Add back separator if we are adding explicit capture variables.
     Stream << ", ";
   }
diff --git a/clang-tools-extra/clang-tidy/misc/UnusedParametersCheck.cpp b/clang-tools-extra/clang-tidy/misc/UnusedParametersCheck.cpp
index 3f1d2f9f5809..c2d9286312dc 100644
--- a/clang-tools-extra/clang-tidy/misc/UnusedParametersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/UnusedParametersCheck.cpp
@@ -192,9 +192,7 @@ void UnusedParametersCheck::check(const MatchFinder::MatchResult &Result) {
 
     // In non-strict mode ignore function definitions with empty bodies
     // (constructor initializer counts for non-empty body).
-    if (StrictMode ||
-        (Function->getBody()->child_begin() !=
-         Function->getBody()->child_end()) ||
+    if (StrictMode || !Function->getBody()->children().empty() ||
         (isa<CXXConstructorDecl>(Function) &&
          cast<CXXConstructorDecl>(Function)->getNumCtorInitializers() > 0))
       warnOnUnusedParameter(Result, Function, I);
diff --git a/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
index 6d7d1d6b87c6..1585925ee996 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
@@ -254,7 +254,7 @@ findInsertionForConstraint(const FunctionDecl *Function, ASTContext &Context) {
         return utils::lexer::findPreviousTokenKind(Init->getSourceLocation(),
                                                    SM, LangOpts, tok::colon);
     }
-    if (Constructor->init_begin() != Constructor->init_end())
+    if (!Constructor->inits().empty())
       return std::nullopt;
   }
   if (Function->isDeleted()) {
-- 
GitLab


From 8a71284cb9463a90fab0d9e8edbeb5d879531e32 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 15 May 2024 13:43:27 +0100
Subject: [PATCH 0841/1206] [MC][X86] Cleanup check prefixes identified in
 #92248

Avoid using numbers as check prefix - replace with actual triple config names where possible
---
 llvm/test/MC/X86/abs8.s                       |  12 +-
 .../test/MC/X86/align-branch-variant-symbol.s |  33 ++--
 llvm/test/MC/X86/data-prefix-fail.s           |  34 ++--
 llvm/test/MC/X86/displacement-overflow.s      |  12 +-
 llvm/test/MC/X86/dwarf-segment-register.s     |  28 +--
 llvm/test/MC/X86/index-operations.s           | 162 ++++++++--------
 llvm/test/MC/X86/ret.s                        | 182 +++++++++---------
 llvm/test/MC/X86/x86_errors.s                 | 158 +++++++--------
 8 files changed, 310 insertions(+), 311 deletions(-)

diff --git a/llvm/test/MC/X86/abs8.s b/llvm/test/MC/X86/abs8.s
index b933c9ddff90..71936acb3740 100644
--- a/llvm/test/MC/X86/abs8.s
+++ b/llvm/test/MC/X86/abs8.s
@@ -1,8 +1,8 @@
-// RUN: llvm-mc -filetype=obj %s -o - -triple i686-pc-linux | llvm-objdump --no-print-imm-hex -d -r - | FileCheck --check-prefix=32 %s
-// RUN: llvm-mc -filetype=obj %s -o - -triple x86_64-pc-linux | llvm-objdump --no-print-imm-hex -d -r - | FileCheck --check-prefix=64 %s
+// RUN: llvm-mc -filetype=obj %s -o - -triple i686-pc-linux | llvm-objdump --no-print-imm-hex -d -r - | FileCheck --check-prefix=X86 %s
+// RUN: llvm-mc -filetype=obj %s -o - -triple x86_64-pc-linux | llvm-objdump --no-print-imm-hex -d -r - | FileCheck --check-prefix=X64 %s
 
-// 32: 0: 83 ff 00  cmpl $0, %edi
-// 32:   00000002:  R_386_8 foo
-// 64: 0: 83 ff 00  cmpl $0, %edi
-// 64:  0000000000000002:  R_X86_64_8 foo
+// X86: 0: 83 ff 00  cmpl $0, %edi
+// X86:   00000002:  R_386_8 foo
+// X64: 0: 83 ff 00  cmpl $0, %edi
+// X64:  0000000000000002:  R_X86_64_8 foo
 cmp $foo@ABS8, %edi
diff --git a/llvm/test/MC/X86/align-branch-variant-symbol.s b/llvm/test/MC/X86/align-branch-variant-symbol.s
index 53afdf58bff3..a1b7b895a354 100644
--- a/llvm/test/MC/X86/align-branch-variant-symbol.s
+++ b/llvm/test/MC/X86/align-branch-variant-symbol.s
@@ -1,6 +1,5 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64 --x86-align-branch-boundary=32 --x86-align-branch=call+indirect %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s --check-prefixes=64BIT,CHECK
-
-# RUN: llvm-mc -filetype=obj -triple i386 --x86-align-branch-boundary=32 --x86-align-branch=call+indirect %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s --check-prefixes=32BIT,CHECK
+# RUN: llvm-mc -filetype=obj -triple x86_64 --x86-align-branch-boundary=32 --x86-align-branch=call+indirect %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s --check-prefixes=CHECK,X64
+# RUN: llvm-mc -filetype=obj -triple i386 --x86-align-branch-boundary=32 --x86-align-branch=call+indirect %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s --check-prefixes=CHECK,X86
 
   # Exercise cases where the instruction to be aligned has a variant symbol
   # operand, and we can't add before it since linker may rewrite it.
@@ -14,8 +13,8 @@ foo:
   int3
   .endr
   # CHECK:    1d:          int3
-  # 64BIT:    1e:          callq
-  # 32BIT:    1e:          calll
+  # X64:    1e:          callq
+  # X86:    1e:          calll
   # CHECK:    23:          int3
   call    ___tls_get_addr@PLT
   int3
@@ -25,10 +24,10 @@ foo:
   int3
   .endr
   # CHECK:    5d:          int3
-  # 64BIT:    5e:          callq    *(%ecx)
-  # 64BIT:    65:          int3
-  # 32BIT:    5e:          calll    *(%ecx)
-  # 32BIT:    64:          int3
+  # X64:    5e:          callq    *(%ecx)
+  # X64:    65:          int3
+  # X86:    5e:          calll    *(%ecx)
+  # X86:    64:          int3
   call *___tls_get_addr@GOT(%ecx)
   int3
 
@@ -37,10 +36,10 @@ foo:
   int3
   .endr
   # CHECK:    9d:          int3
-  # 64BIT:    9e:          callq    *(%eax)
-  # 64BIT:    a1:          int3
-  # 32BIT:    9e:          calll    *(%eax)
-  # 32BIT:    a0:          int3
+  # X64:    9e:          callq    *(%eax)
+  # X64:    a1:          int3
+  # X86:    9e:          calll    *(%eax)
+  # X86:    a0:          int3
   call *foo@tlscall(%eax)
   int3
 
@@ -49,9 +48,9 @@ foo:
   int3
   .endr
   # CHECK:    dd:          int3
-  # 64BIT:    de:          jmpq    *(%eax)
-  # 64BIT:    e1:          int3
-  # 32BIT:    de:          jmpl    *(%eax)
-  # 32BIT:    e0:          int3
+  # X64:    de:          jmpq    *(%eax)
+  # X64:    e1:          int3
+  # X86:    de:          jmpl    *(%eax)
+  # X86:    e0:          int3
   jmp  *foo@tlscall(%eax)
   int3
diff --git a/llvm/test/MC/X86/data-prefix-fail.s b/llvm/test/MC/X86/data-prefix-fail.s
index bd5b62ddc9be..3088864fc909 100644
--- a/llvm/test/MC/X86/data-prefix-fail.s
+++ b/llvm/test/MC/X86/data-prefix-fail.s
@@ -1,30 +1,30 @@
-// RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=64 %s
+// RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=X86-64 %s
 // RUN: FileCheck --check-prefix=ERR64 < %t.err %s
-// RUN: not llvm-mc -triple i386-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=32 %s
+// RUN: not llvm-mc -triple i386-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=X86-32 %s
 // RUN: FileCheck --check-prefix=ERR32 < %t.err %s
-// RUN: not llvm-mc -triple i386-unknown-unknown-code16 --show-encoding %s 2> %t.err | FileCheck --check-prefix=16 %s
+// RUN: not llvm-mc -triple i386-unknown-unknown-code16 --show-encoding %s 2> %t.err | FileCheck --check-prefix=X86-16 %s
 // RUN: FileCheck --check-prefix=ERR16 < %t.err %s
 
 // ERR64: error: 'data32' is not supported in 64-bit mode
 // ERR32: error: redundant data32 prefix
-// 16: lgdtl 0
-// 16-SAME: encoding: [0x66,0x0f,0x01,0x16,0x00,0x00]
+// X86-16: lgdtl 0
+// X86-16-SAME: encoding: [0x66,0x0f,0x01,0x16,0x00,0x00]
 data32 lgdt 0
 
-// 64: data16
-// 64: encoding: [0x66]
-// 64: lgdtq 0
-// 64: encoding: [0x0f,0x01,0x14,0x25,0x00,0x00,0x00,0x00]
-// 32: data16
-// 32: encoding: [0x66]
-// 32: lgdtl 0
-// 32: encoding: [0x0f,0x01,0x15,0x00,0x00,0x00,0x00]
+// X86-64: data16
+// X86-64: encoding: [0x66]
+// X86-64: lgdtq 0
+// X86-64: encoding: [0x0f,0x01,0x14,0x25,0x00,0x00,0x00,0x00]
+// X86-32: data16
+// X86-32: encoding: [0x66]
+// X86-32: lgdtl 0
+// X86-32: encoding: [0x0f,0x01,0x15,0x00,0x00,0x00,0x00]
 // ERR16: error: redundant data16 prefix
 data16 lgdt 0
 
-// 64:      data16    # encoding: [0x66]
-// 64-NEXT: callq  0  # encoding: [0xe8,A,A,A,A]
-// 32:      data16    # encoding: [0x66]
-// 32-NEXT: calll  0  # encoding: [0xe8,A,A,A,A]
+// X86-64:      data16    # encoding: [0x66]
+// X86-64-NEXT: callq  0  # encoding: [0xe8,A,A,A,A]
+// X86-32:      data16    # encoding: [0x66]
+// X86-32-NEXT: calll  0  # encoding: [0xe8,A,A,A,A]
 // ERR16: {{.*}}.s:[[#@LINE+1]]:1: error: redundant data16 prefix
 data16 call 0
diff --git a/llvm/test/MC/X86/displacement-overflow.s b/llvm/test/MC/X86/displacement-overflow.s
index 2882147af482..933f98cb21fc 100644
--- a/llvm/test/MC/X86/displacement-overflow.s
+++ b/llvm/test/MC/X86/displacement-overflow.s
@@ -1,14 +1,14 @@
-# RUN: not llvm-mc -triple=x86_64 %s 2>&1 | FileCheck %s --check-prefixes=CHECK,64 --implicit-check-not=error: --implicit-check-not=warning:
-# RUN: llvm-mc -triple=i686 --defsym A16=1 %s 2>&1 | FileCheck %s --check-prefixes=CHECK,32 --implicit-check-not=error: --implicit-check-not=warning:
+# RUN: not llvm-mc -triple=x86_64 %s 2>&1 | FileCheck %s --check-prefixes=CHECK,X64 --implicit-check-not=error: --implicit-check-not=warning:
+# RUN: llvm-mc -triple=i686 --defsym A16=1 %s 2>&1 | FileCheck %s --check-prefixes=CHECK,X86 --implicit-check-not=error: --implicit-check-not=warning:
 
 .ifndef A16
 movq 0x80000000-1(%rip), %rax
 leaq -0x80000000(%rip), %rax
 
-# 64: [[#@LINE+1]]:17: error: displacement 2147483648 is not within [-2147483648, 2147483647]
+# X64: [[#@LINE+1]]:17: error: displacement 2147483648 is not within [-2147483648, 2147483647]
 movq 0x80000000(%rip), %rax
 
-# 64: [[#@LINE+1]]:18: error: displacement -2147483649 is not within [-2147483648, 2147483647]
+# X64: [[#@LINE+1]]:18: error: displacement -2147483649 is not within [-2147483648, 2147483647]
 leaq -0x80000001(%rip), %rax
 .endif
 
@@ -31,8 +31,8 @@ leal -0xffffffff-2(%eax), %eax
 movw $0, 0xffff(%bp)
 movw $0, -0xffff(%si)
 
-# 32: [[#@LINE+1]]:19: warning: displacement 65536 shortened to 16-bit signed 0
+# X86: [[#@LINE+1]]:19: warning: displacement 65536 shortened to 16-bit signed 0
 movw $0, 0xffff+1(%bp)
-# 32: [[#@LINE+1]]:20: warning: displacement -65536 shortened to 16-bit signed 0
+# X86: [[#@LINE+1]]:20: warning: displacement -65536 shortened to 16-bit signed 0
 movw $0, -0xffff-1(%si)
 .endif
diff --git a/llvm/test/MC/X86/dwarf-segment-register.s b/llvm/test/MC/X86/dwarf-segment-register.s
index a68576807470..5482588df821 100644
--- a/llvm/test/MC/X86/dwarf-segment-register.s
+++ b/llvm/test/MC/X86/dwarf-segment-register.s
@@ -1,7 +1,7 @@
 // RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o %t.64
-// RUN: llvm-objdump --dwarf=frames %t.64 | FileCheck %s --check-prefixes=64,CHECK
+// RUN: llvm-objdump --dwarf=frames %t.64 | FileCheck %s --check-prefixes=X64,CHECK
 // RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -o %t.32
-// RUN: llvm-objdump --dwarf=frames %t.32 | FileCheck %s --check-prefixes=32,CHECK
+// RUN: llvm-objdump --dwarf=frames %t.32 | FileCheck %s --check-prefixes=X86,CHECK
 
 .cfi_startproc
 .cfi_offset %cs, -40
@@ -12,26 +12,26 @@
 .cfi_offset %gs, 0
 .cfi_endproc
 
-// 64: reg51
-// 32: reg41
+// X64: reg51
+// X86: reg41
 // CHECK-SAME: -40
 
-// 64: reg53
-// 32: reg43
+// X64: reg53
+// X86: reg43
 // CHECK-SAME: -32
 
-// 64: reg52
-// 32: reg42
+// X64: reg52
+// X86: reg42
 // CHECK-SAME: -24
 
-// 64: reg50
-// 32: reg40
+// X64: reg50
+// X86: reg40
 // CHECK-SAME: -16
 
-// 64: reg54
-// 32: reg44
+// X64: reg54
+// X86: reg44
 // CHECK-SAME: -8
 
-// 64: reg55
-// 32: reg45
+// X64: reg55
+// X86: reg45
 // CHECK-SAME: 0
diff --git a/llvm/test/MC/X86/index-operations.s b/llvm/test/MC/X86/index-operations.s
index 59498b0ced12..c425ba8057c5 100644
--- a/llvm/test/MC/X86/index-operations.s
+++ b/llvm/test/MC/X86/index-operations.s
@@ -1,34 +1,34 @@
-// RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=64 %s
+// RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=X86-64 %s
 // RUN: FileCheck --input-file=%t.err %s --check-prefix=ERR64 --implicit-check-not=error:
-// RUN: not llvm-mc -triple i386-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=32 %s
+// RUN: not llvm-mc -triple i386-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=X86-32 %s
 // RUN: FileCheck --check-prefix=ERR32 < %t.err %s
-// RUN: not llvm-mc -triple i386-unknown-unknown-code16 --show-encoding %s 2> %t.err | FileCheck --check-prefix=16 %s
+// RUN: not llvm-mc -triple i386-unknown-unknown-code16 --show-encoding %s 2> %t.err | FileCheck --check-prefix=X86-16 %s
 // RUN: FileCheck --check-prefix=ERR16 < %t.err %s
 
 lodsb
-// 64: lodsb (%rsi), %al # encoding: [0xac]
-// 32: lodsb (%esi), %al # encoding: [0xac]
-// 16: lodsb (%si), %al # encoding: [0xac]
+// X86-64: lodsb (%rsi), %al # encoding: [0xac]
+// X86-32: lodsb (%esi), %al # encoding: [0xac]
+// X86-16: lodsb (%si), %al # encoding: [0xac]
 
 lodsb (%rsi), %al
-// 64: lodsb (%rsi), %al # encoding: [0xac]
+// X86-64: lodsb (%rsi), %al # encoding: [0xac]
 // ERR32: 64-bit
 // ERR16: 64-bit
 
 lodsb (%esi), %al
-// 64: lodsb (%esi), %al # encoding: [0x67,0xac]
-// 32: lodsb (%esi), %al # encoding: [0xac]
-// 16: lodsb (%esi), %al # encoding: [0x67,0xac]
+// X86-64: lodsb (%esi), %al # encoding: [0x67,0xac]
+// X86-32: lodsb (%esi), %al # encoding: [0xac]
+// X86-16: lodsb (%esi), %al # encoding: [0x67,0xac]
 
 lodsb (%si), %al
 // ERR64: [[#@LINE-1]]:[[#]]: error: invalid 16-bit base register
-// 32: lodsb (%si), %al # encoding: [0x67,0xac]
-// 16: lodsb (%si), %al # encoding: [0xac]
+// X86-32: lodsb (%si), %al # encoding: [0x67,0xac]
+// X86-16: lodsb (%si), %al # encoding: [0xac]
 
 lodsl %gs:(%esi)
-// 64: lodsl %gs:(%esi), %eax # encoding: [0x67,0x65,0xad]
-// 32: lodsl %gs:(%esi), %eax # encoding: [0x65,0xad]
-// 16: lodsl %gs:(%esi), %eax # encoding: [0x67,0x65,0x66,0xad]
+// X86-64: lodsl %gs:(%esi), %eax # encoding: [0x67,0x65,0xad]
+// X86-32: lodsl %gs:(%esi), %eax # encoding: [0x65,0xad]
+// X86-16: lodsl %gs:(%esi), %eax # encoding: [0x67,0x65,0x66,0xad]
 
 lodsl (%edi), %eax
 // ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand
@@ -41,19 +41,19 @@ lodsl 44(%edi), %eax
 // ERR16: invalid operand
 
 lods (%esi), %ax
-// 64: lodsw (%esi), %ax # encoding: [0x67,0x66,0xad]
-// 32: lodsw (%esi), %ax # encoding: [0x66,0xad]
-// 16: lodsw (%esi), %ax # encoding: [0x67,0xad]
+// X86-64: lodsw (%esi), %ax # encoding: [0x67,0x66,0xad]
+// X86-32: lodsw (%esi), %ax # encoding: [0x66,0xad]
+// X86-16: lodsw (%esi), %ax # encoding: [0x67,0xad]
 
 stosw
-// 64: stosw %ax, %es:(%rdi) # encoding: [0x66,0xab]
-// 32: stosw %ax, %es:(%edi) # encoding: [0x66,0xab]
-// 16: stosw %ax, %es:(%di) # encoding: [0xab]
+// X86-64: stosw %ax, %es:(%rdi) # encoding: [0x66,0xab]
+// X86-32: stosw %ax, %es:(%edi) # encoding: [0x66,0xab]
+// X86-16: stosw %ax, %es:(%di) # encoding: [0xab]
 
 stos %eax, (%edi)
-// 64: stosl %eax, %es:(%edi) # encoding: [0x67,0xab]
-// 32: stosl %eax, %es:(%edi) # encoding: [0xab]
-// 16: stosl %eax, %es:(%edi) # encoding: [0x67,0x66,0xab]
+// X86-64: stosl %eax, %es:(%edi) # encoding: [0x67,0xab]
+// X86-32: stosl %eax, %es:(%edi) # encoding: [0xab]
+// X86-16: stosl %eax, %es:(%edi) # encoding: [0x67,0x66,0xab]
 
 stosb %al, %fs:(%edi)
 // ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand for instruction
@@ -61,27 +61,27 @@ stosb %al, %fs:(%edi)
 // ERR16: invalid operand for instruction
 
 stosb %al, %es:(%edi)
-// 64: stosb %al, %es:(%edi) # encoding: [0x67,0xaa]
-// 32: stosb %al, %es:(%edi) # encoding: [0xaa]
-// 16: stosb %al, %es:(%edi) # encoding: [0x67,0xaa]
+// X86-64: stosb %al, %es:(%edi) # encoding: [0x67,0xaa]
+// X86-32: stosb %al, %es:(%edi) # encoding: [0xaa]
+// X86-16: stosb %al, %es:(%edi) # encoding: [0x67,0xaa]
 
 stosq
-// 64: stosq %rax, %es:(%rdi) # encoding: [0x48,0xab]
+// X86-64: stosq %rax, %es:(%rdi) # encoding: [0x48,0xab]
 // ERR32: 64-bit
 // ERR16: 64-bit
 
 stos %rax, (%edi)
-// 64: 	stosq %rax, %es:(%edi) # encoding: [0x67,0x48,0xab]
+// X86-64: 	stosq %rax, %es:(%edi) # encoding: [0x67,0x48,0xab]
 // ERR32: only available in 64-bit mode
 // ERR16: only available in 64-bit mode
 
 scas %es:(%edi), %al
-// 64: scasb %es:(%edi), %al # encoding: [0x67,0xae]
-// 32: scasb %es:(%edi), %al # encoding: [0xae]
-// 16: scasb %es:(%edi), %al # encoding: [0x67,0xae]
+// X86-64: scasb %es:(%edi), %al # encoding: [0x67,0xae]
+// X86-32: scasb %es:(%edi), %al # encoding: [0xae]
+// X86-16: scasb %es:(%edi), %al # encoding: [0x67,0xae]
 
 scasq %es:(%edi)
-// 64: scasq %es:(%edi), %rax # encoding: [0x67,0x48,0xaf]
+// X86-64: scasq %es:(%edi), %rax # encoding: [0x67,0x48,0xaf]
 // ERR32: 64-bit
 // ERR16: 64-bit
 
@@ -92,18 +92,18 @@ scasl %es:(%edi), %al
 
 scas %es:(%di), %ax
 // ERR64: [[#@LINE-1]]:[[#]]: error: invalid 16-bit base register
-// 16: scasw %es:(%di), %ax # encoding: [0xaf]
-// 32: scasw %es:(%di), %ax # encoding: [0x67,0x66,0xaf]
+// X86-16: scasw %es:(%di), %ax # encoding: [0xaf]
+// X86-32: scasw %es:(%di), %ax # encoding: [0x67,0x66,0xaf]
 
 cmpsb
-// 64: cmpsb %es:(%rdi), (%rsi) # encoding: [0xa6]
-// 32: cmpsb %es:(%edi), (%esi) # encoding: [0xa6]
-// 16: cmpsb %es:(%di), (%si) # encoding: [0xa6]
+// X86-64: cmpsb %es:(%rdi), (%rsi) # encoding: [0xa6]
+// X86-32: cmpsb %es:(%edi), (%esi) # encoding: [0xa6]
+// X86-16: cmpsb %es:(%di), (%si) # encoding: [0xa6]
 
 cmpsw (%edi), (%esi)
-// 64: cmpsw %es:(%edi), (%esi) # encoding: [0x67,0x66,0xa7]
-// 32: cmpsw %es:(%edi), (%esi) # encoding: [0x66,0xa7]
-// 16: cmpsw %es:(%edi), (%esi) # encoding: [0x67,0xa7]
+// X86-64: cmpsw %es:(%edi), (%esi) # encoding: [0x67,0x66,0xa7]
+// X86-32: cmpsw %es:(%edi), (%esi) # encoding: [0x66,0xa7]
+// X86-16: cmpsw %es:(%edi), (%esi) # encoding: [0x67,0xa7]
 
 cmpsb (%di), (%esi)
 // ERR64: [[#@LINE-1]]:[[#]]: error: invalid 16-bit base register
@@ -111,52 +111,52 @@ cmpsb (%di), (%esi)
 // ERR16: mismatching source and destination
 
 cmpsl %es:(%edi), %ss:(%esi)
-// 64: cmpsl %es:(%edi), %ss:(%esi) # encoding: [0x67,0x36,0xa7]
-// 32: cmpsl %es:(%edi), %ss:(%esi) # encoding: [0x36,0xa7]
-// 16: cmpsl %es:(%edi), %ss:(%esi) # encoding: [0x67,0x36,0x66,0xa7]
+// X86-64: cmpsl %es:(%edi), %ss:(%esi) # encoding: [0x67,0x36,0xa7]
+// X86-32: cmpsl %es:(%edi), %ss:(%esi) # encoding: [0x36,0xa7]
+// X86-16: cmpsl %es:(%edi), %ss:(%esi) # encoding: [0x67,0x36,0x66,0xa7]
 
 cmpsq (%rdi), (%rsi)
-// 64: cmpsq %es:(%rdi), (%rsi) # encoding: [0x48,0xa7]
+// X86-64: cmpsq %es:(%rdi), (%rsi) # encoding: [0x48,0xa7]
 // ERR32: 64-bit
 // ERR16: 64-bit
 
 movsb (%esi), (%edi)
-// 64: movsb (%esi), %es:(%edi) # encoding: [0x67,0xa4]
-// 32: movsb (%esi), %es:(%edi) # encoding: [0xa4]
-// 16: movsb (%esi), %es:(%edi) # encoding: [0x67,0xa4]
+// X86-64: movsb (%esi), %es:(%edi) # encoding: [0x67,0xa4]
+// X86-32: movsb (%esi), %es:(%edi) # encoding: [0xa4]
+// X86-16: movsb (%esi), %es:(%edi) # encoding: [0x67,0xa4]
 
 movsl %gs:(%esi), (%edi)
-// 64: movsl %gs:(%esi), %es:(%edi) # encoding: [0x67,0x65,0xa5]
-// 32: movsl %gs:(%esi), %es:(%edi) # encoding: [0x65,0xa5]
-// 16: movsl %gs:(%esi), %es:(%edi) # encoding: [0x67,0x65,0x66,0xa5]
+// X86-64: movsl %gs:(%esi), %es:(%edi) # encoding: [0x67,0x65,0xa5]
+// X86-32: movsl %gs:(%esi), %es:(%edi) # encoding: [0x65,0xa5]
+// X86-16: movsl %gs:(%esi), %es:(%edi) # encoding: [0x67,0x65,0x66,0xa5]
 
 outsb
-// 64: outsb (%rsi), %dx # encoding: [0x6e]
-// 32: outsb (%esi), %dx # encoding: [0x6e]
-// 16: outsb (%si), %dx # encoding: [0x6e]
+// X86-64: outsb (%rsi), %dx # encoding: [0x6e]
+// X86-32: outsb (%esi), %dx # encoding: [0x6e]
+// X86-16: outsb (%si), %dx # encoding: [0x6e]
 
 outsw %fs:(%esi), %dx
-// 64: outsw %fs:(%esi), %dx # encoding: [0x67,0x64,0x66,0x6f]
-// 32: outsw %fs:(%esi), %dx # encoding: [0x64,0x66,0x6f]
-// 16: outsw %fs:(%esi), %dx # encoding: [0x67,0x64,0x6f]
+// X86-64: outsw %fs:(%esi), %dx # encoding: [0x67,0x64,0x66,0x6f]
+// X86-32: outsw %fs:(%esi), %dx # encoding: [0x64,0x66,0x6f]
+// X86-16: outsw %fs:(%esi), %dx # encoding: [0x67,0x64,0x6f]
 
 insw %dx, (%edi)
-// 64: insw %dx, %es:(%edi) # encoding: [0x67,0x66,0x6d]
-// 32: insw %dx, %es:(%edi) # encoding: [0x66,0x6d]
-// 16: insw %dx, %es:(%edi) # encoding: [0x67,0x6d]
+// X86-64: insw %dx, %es:(%edi) # encoding: [0x67,0x66,0x6d]
+// X86-32: insw %dx, %es:(%edi) # encoding: [0x66,0x6d]
+// X86-16: insw %dx, %es:(%edi) # encoding: [0x67,0x6d]
 
 insw %dx, (%bx)
 // ERR64: [[#@LINE-1]]:[[#]]: error: invalid 16-bit base register
-// 32: insw %dx, %es:(%di) # encoding: [0x67,0x66,0x6d]
-// 16: insw %dx, %es:(%di) # encoding: [0x6d]
+// X86-32: insw %dx, %es:(%di) # encoding: [0x67,0x66,0x6d]
+// X86-16: insw %dx, %es:(%di) # encoding: [0x6d]
 
 insw %dx, (%ebx)
-// 64: insw %dx, %es:(%edi) # encoding: [0x67,0x66,0x6d]
-// 32: insw %dx, %es:(%edi) # encoding: [0x66,0x6d]
-// 16: insw %dx, %es:(%edi) # encoding: [0x67,0x6d]
+// X86-64: insw %dx, %es:(%edi) # encoding: [0x67,0x66,0x6d]
+// X86-32: insw %dx, %es:(%edi) # encoding: [0x66,0x6d]
+// X86-16: insw %dx, %es:(%edi) # encoding: [0x67,0x6d]
 
 insw %dx, (%rbx)
-// 64: insw %dx, %es:(%rdi) # encoding: [0x66,0x6d]
+// X86-64: insw %dx, %es:(%rdi) # encoding: [0x66,0x6d]
 // ERR32: 64-bit
 // ERR16: 64-bit
 
@@ -177,17 +177,17 @@ movdir64b (%edx), %r15
 // ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand
 
 movdir64b (%eip), %ebx
-// 64: movdir64b (%eip), %ebx # encoding: [0x67,0x66,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
+// X86-64: movdir64b (%eip), %ebx # encoding: [0x67,0x66,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
 
 movdir64b (%rip), %rbx
-// 64: movdir64b (%rip), %rbx # encoding: [0x66,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
+// X86-64: movdir64b (%rip), %rbx # encoding: [0x66,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
 
 movdir64b 291(%esi, %eiz, 4), %ebx
-// 64: movdir64b 291(%esi,%eiz,4), %ebx # encoding: [0x67,0x66,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
-// 32: movdir64b 291(%esi,%eiz,4), %ebx # encoding: [0x66,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+// X86-64: movdir64b 291(%esi,%eiz,4), %ebx # encoding: [0x67,0x66,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+// X86-32: movdir64b 291(%esi,%eiz,4), %ebx # encoding: [0x66,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
 
 movdir64b 291(%rsi, %riz, 4), %rbx
-// 64: movdir64b 291(%rsi,%riz,4), %rbx # encoding: [0x66,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+// X86-64: movdir64b 291(%rsi,%riz,4), %rbx # encoding: [0x66,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
 
 enqcmd	291(%si), %ecx
 // ERR64: error: invalid 16-bit base register
@@ -206,17 +206,17 @@ enqcmd (%edx), %r15
 // ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand
 
 enqcmd (%eip), %ebx
-// 64: enqcmd (%eip), %ebx # encoding: [0x67,0xf2,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
+// X86-64: enqcmd (%eip), %ebx # encoding: [0x67,0xf2,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
 
 enqcmd (%rip), %rbx
-// 64: enqcmd (%rip), %rbx # encoding: [0xf2,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
+// X86-64: enqcmd (%rip), %rbx # encoding: [0xf2,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
 
 enqcmd 291(%esi, %eiz, 4), %ebx
-// 64: enqcmd 291(%esi,%eiz,4), %ebx # encoding: [0x67,0xf2,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
-// 32: enqcmd 291(%esi,%eiz,4), %ebx # encoding: [0xf2,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+// X86-64: enqcmd 291(%esi,%eiz,4), %ebx # encoding: [0x67,0xf2,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+// X86-32: enqcmd 291(%esi,%eiz,4), %ebx # encoding: [0xf2,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
 
 enqcmd 291(%rsi, %riz, 4), %rbx
-// 64: enqcmd 291(%rsi,%riz,4), %rbx # encoding: [0xf2,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+// X86-64: enqcmd 291(%rsi,%riz,4), %rbx # encoding: [0xf2,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
 
 enqcmds	291(%si), %ecx
 // ERR64: error: invalid 16-bit base register
@@ -235,14 +235,14 @@ enqcmds (%edx), %r15
 // ERR64: [[#@LINE-1]]:[[#]]: error: invalid operand
 
 enqcmds (%eip), %ebx
-// 64: enqcmds (%eip), %ebx # encoding: [0x67,0xf3,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
+// X86-64: enqcmds (%eip), %ebx # encoding: [0x67,0xf3,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
 
 enqcmds (%rip), %rbx
-// 64: enqcmds (%rip), %rbx # encoding: [0xf3,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
+// X86-64: enqcmds (%rip), %rbx # encoding: [0xf3,0x0f,0x38,0xf8,0x1d,0x00,0x00,0x00,0x00]
 
 enqcmds 291(%esi, %eiz, 4), %ebx
-// 64: enqcmds 291(%esi,%eiz,4), %ebx # encoding: [0x67,0xf3,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
-// 32: enqcmds 291(%esi,%eiz,4), %ebx # encoding: [0xf3,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+// X86-64: enqcmds 291(%esi,%eiz,4), %ebx # encoding: [0x67,0xf3,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+// X86-32: enqcmds 291(%esi,%eiz,4), %ebx # encoding: [0xf3,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
 
 enqcmds 291(%rsi, %riz, 4), %rbx
-// 64: enqcmds 291(%rsi,%riz,4), %rbx # encoding: [0xf3,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
+// X86-64: enqcmds 291(%rsi,%riz,4), %rbx # encoding: [0xf3,0x0f,0x38,0xf8,0x9c,0xa6,0x23,0x01,0x00,0x00]
diff --git a/llvm/test/MC/X86/ret.s b/llvm/test/MC/X86/ret.s
index 142a4614ba4f..7b5bcd4ad990 100644
--- a/llvm/test/MC/X86/ret.s
+++ b/llvm/test/MC/X86/ret.s
@@ -1,129 +1,129 @@
-// RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=64 %s
+// RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=X86-64 %s
 // RUN: FileCheck --check-prefix=ERR64 < %t.err %s
-// RUN: not llvm-mc -triple i386-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=32 %s
+// RUN: not llvm-mc -triple i386-unknown-unknown --show-encoding %s 2> %t.err | FileCheck --check-prefix=X86-32 %s
 // RUN: FileCheck --check-prefix=ERR32 < %t.err %s
-// RUN: not llvm-mc -triple i386-unknown-unknown-code16 --show-encoding %s 2> %t.err | FileCheck --check-prefix=16 %s
+// RUN: not llvm-mc -triple i386-unknown-unknown-code16 --show-encoding %s 2> %t.err | FileCheck --check-prefix=X86-16 %s
 // RUN: FileCheck --check-prefix=ERR16 < %t.err %s
 
 	ret
-// 64: retq
-// 64: encoding: [0xc3]
-// 32: retl
-// 32: encoding: [0xc3]
-// 16: retw
-// 16: encoding: [0xc3]
+// X86-64: retq
+// X86-64: encoding: [0xc3]
+// X86-32: retl
+// X86-32: encoding: [0xc3]
+// X86-16: retw
+// X86-16: encoding: [0xc3]
 	retw
-// 64: retw
-// 64: encoding: [0x66,0xc3]
-// 32: retw
-// 32: encoding: [0x66,0xc3]
-// 16: retw
-// 16: encoding: [0xc3]
+// X86-64: retw
+// X86-64: encoding: [0x66,0xc3]
+// X86-32: retw
+// X86-32: encoding: [0x66,0xc3]
+// X86-16: retw
+// X86-16: encoding: [0xc3]
 	retl
 // ERR64: error: instruction requires: Not 64-bit mode
-// 32: retl
-// 32: encoding: [0xc3]
-// 16: retl
-// 16: encoding: [0x66,0xc3]
+// X86-32: retl
+// X86-32: encoding: [0xc3]
+// X86-16: retl
+// X86-16: encoding: [0x66,0xc3]
 	retq
-// 64: retq
-// 64: encoding: [0xc3]
+// X86-64: retq
+// X86-64: encoding: [0xc3]
 // ERR32: error: instruction requires: 64-bit mode
 // ERR16: error: instruction requires: 64-bit mode
 
 	ret $0
-// 64: retq $0
-// 64: encoding: [0xc2,0x00,0x00]
-// 32: retl $0
-// 32: encoding: [0xc2,0x00,0x00]
-// 16: retw $0
-// 16: encoding: [0xc2,0x00,0x00]
+// X86-64: retq $0
+// X86-64: encoding: [0xc2,0x00,0x00]
+// X86-32: retl $0
+// X86-32: encoding: [0xc2,0x00,0x00]
+// X86-16: retw $0
+// X86-16: encoding: [0xc2,0x00,0x00]
 	retw $0
-// 64: retw $0
-// 64: encoding: [0x66,0xc2,0x00,0x00]
-// 32: retw $0
-// 32: encoding: [0x66,0xc2,0x00,0x00]
-// 16: retw $0
-// 16: encoding: [0xc2,0x00,0x00]
+// X86-64: retw $0
+// X86-64: encoding: [0x66,0xc2,0x00,0x00]
+// X86-32: retw $0
+// X86-32: encoding: [0x66,0xc2,0x00,0x00]
+// X86-16: retw $0
+// X86-16: encoding: [0xc2,0x00,0x00]
 	retl $0
 // ERR64: error: instruction requires: Not 64-bit mode
-// 32: retl $0
-// 32: encoding: [0xc2,0x00,0x00]
-// 16: retl $0
-// 16: encoding: [0x66,0xc2,0x00,0x00]
+// X86-32: retl $0
+// X86-32: encoding: [0xc2,0x00,0x00]
+// X86-16: retl $0
+// X86-16: encoding: [0x66,0xc2,0x00,0x00]
 	retq $0
-// 64: retq $0
-// 64: encoding: [0xc2,0x00,0x00]
+// X86-64: retq $0
+// X86-64: encoding: [0xc2,0x00,0x00]
 // ERR32: error: instruction requires: 64-bit mode
 // ERR16: error: instruction requires: 64-bit mode
 
 	retn
-// 64: retq
-// 64: encoding: [0xc3]
-// 32: retl
-// 32: encoding: [0xc3]
-// 16: retw
-// 16: encoding: [0xc3]
+// X86-64: retq
+// X86-64: encoding: [0xc3]
+// X86-32: retl
+// X86-32: encoding: [0xc3]
+// X86-16: retw
+// X86-16: encoding: [0xc3]
 
   retn $0
-// 64: retq $0
-// 64: encoding: [0xc2,0x00,0x00]
-// 32: retl $0
-// 32: encoding: [0xc2,0x00,0x00]
-// 16: retw $0
-// 16: encoding: [0xc2,0x00,0x00]
+// X86-64: retq $0
+// X86-64: encoding: [0xc2,0x00,0x00]
+// X86-32: retl $0
+// X86-32: encoding: [0xc2,0x00,0x00]
+// X86-16: retw $0
+// X86-16: encoding: [0xc2,0x00,0x00]
 
 	lret
-// 64: lretl
-// 64: encoding: [0xcb]
-// 32: lretl
-// 32: encoding: [0xcb]
-// 16: lretw
-// 16: encoding: [0xcb]
+// X86-64: lretl
+// X86-64: encoding: [0xcb]
+// X86-32: lretl
+// X86-32: encoding: [0xcb]
+// X86-16: lretw
+// X86-16: encoding: [0xcb]
 	lretw
-// 64: lretw
-// 64: encoding: [0x66,0xcb]
-// 32: lretw
-// 32: encoding: [0x66,0xcb]
-// 16: lretw
-// 16: encoding: [0xcb]
+// X86-64: lretw
+// X86-64: encoding: [0x66,0xcb]
+// X86-32: lretw
+// X86-32: encoding: [0x66,0xcb]
+// X86-16: lretw
+// X86-16: encoding: [0xcb]
 	lretl
-// 64: lretl
-// 64: encoding: [0xcb]
-// 32: lretl
-// 32: encoding: [0xcb]
-// 16: lretl
-// 16: encoding: [0x66,0xcb]
+// X86-64: lretl
+// X86-64: encoding: [0xcb]
+// X86-32: lretl
+// X86-32: encoding: [0xcb]
+// X86-16: lretl
+// X86-16: encoding: [0x66,0xcb]
 	lretq
-// 64: lretq
-// 64: encoding: [0x48,0xcb]
+// X86-64: lretq
+// X86-64: encoding: [0x48,0xcb]
 // ERR32: error: instruction requires: 64-bit mode
 // ERR16: error: instruction requires: 64-bit mode
 
 	lret $0
-// 64: lretl $0
-// 64: encoding: [0xca,0x00,0x00]
-// 32: lretl $0
-// 32: encoding: [0xca,0x00,0x00]
-// 16: lretw $0
-// 16: encoding: [0xca,0x00,0x00]
+// X86-64: lretl $0
+// X86-64: encoding: [0xca,0x00,0x00]
+// X86-32: lretl $0
+// X86-32: encoding: [0xca,0x00,0x00]
+// X86-16: lretw $0
+// X86-16: encoding: [0xca,0x00,0x00]
 	lretw $0
-// 64: lretw $0
-// 64: encoding: [0x66,0xca,0x00,0x00]
-// 32: lretw $0
-// 32: encoding: [0x66,0xca,0x00,0x00]
-// 16: lretw $0
-// 16: encoding: [0xca,0x00,0x00]
+// X86-64: lretw $0
+// X86-64: encoding: [0x66,0xca,0x00,0x00]
+// X86-32: lretw $0
+// X86-32: encoding: [0x66,0xca,0x00,0x00]
+// X86-16: lretw $0
+// X86-16: encoding: [0xca,0x00,0x00]
 	lretl $0
-// 64: lretl $0
-// 64: encoding: [0xca,0x00,0x00]
-// 32: lretl $0
-// 32: encoding: [0xca,0x00,0x00]
-// 16: lretl $0
-// 16: encoding: [0x66,0xca,0x00,0x00]
+// X86-64: lretl $0
+// X86-64: encoding: [0xca,0x00,0x00]
+// X86-32: lretl $0
+// X86-32: encoding: [0xca,0x00,0x00]
+// X86-16: lretl $0
+// X86-16: encoding: [0x66,0xca,0x00,0x00]
 	lretq $0
-// 64: lretq $0
-// 64: encoding: [0x48,0xca,0x00,0x00]
+// X86-64: lretq $0
+// X86-64: encoding: [0x48,0xca,0x00,0x00]
 // ERR32: error: instruction requires: 64-bit mode
 // ERR16: error: instruction requires: 64-bit mode
 
diff --git a/llvm/test/MC/X86/x86_errors.s b/llvm/test/MC/X86/x86_errors.s
index f1c7110fde1f..da8659f3621e 100644
--- a/llvm/test/MC/X86/x86_errors.s
+++ b/llvm/test/MC/X86/x86_errors.s
@@ -1,122 +1,122 @@
 // RUN: not llvm-mc -triple x86_64-unknown-unknown %s 2> %t.err
-// RUN: FileCheck --check-prefix=64 < %t.err %s
+// RUN: FileCheck --check-prefix=X64 < %t.err %s
 
 // RUN: not llvm-mc -triple i386-unknown-unknown %s 2> %t.err
-// RUN: FileCheck --check-prefix=32 < %t.err %s
+// RUN: FileCheck --check-prefix=X86 < %t.err %s
 // rdar://8204588
 
-// 64: error: ambiguous instructions require an explicit suffix (could be 'cmpb', 'cmpw', 'cmpl', or 'cmpq')
+// X64: error: ambiguous instructions require an explicit suffix (could be 'cmpb', 'cmpw', 'cmpl', or 'cmpq')
 cmp $0, 0(%eax)
 
-// 32: error: register %rax is only available in 64-bit mode
+// X86: error: register %rax is only available in 64-bit mode
 addl $0, 0(%rax)
 
-// 32: test.s:8:2: error: invalid instruction mnemonic 'movi'
+// X86: test.s:8:2: error: invalid instruction mnemonic 'movi'
 
 # 8 "test.s"
  movi $8,%eax
 
 movl 0(%rax), 0(%edx)  // error: invalid operand for instruction
 
-// 32: error: instruction requires: 64-bit mode
+// X86: error: instruction requires: 64-bit mode
 sysexitq
 
 // rdar://10710167
-// 64: error: expected scale expression
+// X64: error: expected scale expression
 lea (%rsp, %rbp, $4), %rax
 
 // rdar://10423777
-// 64: error: base register is 64-bit, but index register is not
+// X64: error: base register is 64-bit, but index register is not
 movq (%rsi,%ecx),%xmm0
 
-// 64: error: invalid 16-bit base register
+// X64: error: invalid 16-bit base register
 movl %eax,(%bp,%si)
 
-// 32: error: scale factor in 16-bit address must be 1
+// X86: error: scale factor in 16-bit address must be 1
 movl %eax,(%bp,%si,2)
 
-// 32: error: invalid 16-bit base register
+// X86: error: invalid 16-bit base register
 movl %eax,(%cx)
 
-// 32: error: invalid 16-bit base/index register combination
+// X86: error: invalid 16-bit base/index register combination
 movl %eax,(%bp,%bx)
 
-// 32: error: 16-bit memory operand may not include only index register
+// X86: error: 16-bit memory operand may not include only index register
 movl %eax,(,%bx)
 
-// 32: error: invalid operand for instruction
+// X86: error: invalid operand for instruction
 outb al, 4
 
-// 32: error: invalid segment register
-// 64: error: invalid segment register
+// X86: error: invalid segment register
+// X64: error: invalid segment register
 movl %eax:0x00, %ebx
 
-// 32: error: invalid operand for instruction
-// 64: error: invalid operand for instruction
+// X86: error: invalid operand for instruction
+// X64: error: invalid operand for instruction
 cmpps $-129, %xmm0, %xmm0
 
-// 32: error: invalid operand for instruction
-// 64: error: invalid operand for instruction
+// X86: error: invalid operand for instruction
+// X64: error: invalid operand for instruction
 cmppd $256, %xmm0, %xmm0
 
-// 32: error: instruction requires: 64-bit mode
+// X86: error: instruction requires: 64-bit mode
 jrcxz 1
 
-// 64: error: instruction requires: Not 64-bit mode
+// X64: error: instruction requires: Not 64-bit mode
 jcxz 1
 
-// 32: error: register %cr8 is only available in 64-bit mode
+// X86: error: register %cr8 is only available in 64-bit mode
 movl %edx, %cr8
 
-// 32: error: register %dr8 is only available in 64-bit mode
+// X86: error: register %dr8 is only available in 64-bit mode
 movl %edx, %dr8
 
-// 32: error: register %rip is only available in 64-bit mode
-// 64: error: %rip can only be used as a base register
+// X86: error: register %rip is only available in 64-bit mode
+// X64: error: %rip can only be used as a base register
 mov %rip, %rax
 
-// 32: error: register %rax is only available in 64-bit mode
-// 64: error: %rip is not allowed as an index register
+// X86: error: register %rax is only available in 64-bit mode
+// X64: error: %rip is not allowed as an index register
 mov (%rax,%rip), %rbx
 
-// 32: error: instruction requires: 64-bit mode
+// X86: error: instruction requires: 64-bit mode
 ljmpq *(%eax)
 
-// 32: error: register %rax is only available in 64-bit mode
-// 64: error: invalid base+index expression
+// X86: error: register %rax is only available in 64-bit mode
+// X64: error: invalid base+index expression
 leaq (%rax,%rsp), %rax
 
-// 32: error: invalid base+index expression
-// 64: error: invalid base+index expression
+// X86: error: invalid base+index expression
+// X64: error: invalid base+index expression
 leaq (%eax,%esp), %eax
 
-// 32: error: invalid 16-bit base/index register combination
-// 64: error: invalid 16-bit base register
+// X86: error: invalid 16-bit base/index register combination
+// X64: error: invalid 16-bit base register
 lea (%si,%bp), %ax
-// 32: error: invalid 16-bit base/index register combination
-// 64: error: invalid 16-bit base register
+// X86: error: invalid 16-bit base/index register combination
+// X64: error: invalid 16-bit base register
 lea (%di,%bp), %ax
-// 32: error: invalid 16-bit base/index register combination
-// 64: error: invalid 16-bit base register
+// X86: error: invalid 16-bit base/index register combination
+// X64: error: invalid 16-bit base register
 lea (%si,%bx), %ax
-// 32: error: invalid 16-bit base/index register combination
-// 64: error: invalid 16-bit base register
+// X86: error: invalid 16-bit base/index register combination
+// X64: error: invalid 16-bit base register
 lea (%di,%bx), %ax
 
-// 32: error: invalid base+index expression
-// 64: error: invalid base+index expression
+// X86: error: invalid base+index expression
+// X64: error: invalid base+index expression
 mov (,%eip), %rbx
 
-// 32: error: invalid base+index expression
-// 64: error: invalid base+index expression
+// X86: error: invalid base+index expression
+// X64: error: invalid base+index expression
 mov (%eip,%eax), %rbx
 
-// 32: error: register %rax is only available in 64-bit mode
-// 64: error: base register is 64-bit, but index register is not
+// X86: error: register %rax is only available in 64-bit mode
+// X64: error: base register is 64-bit, but index register is not
 mov (%rax,%eiz), %ebx
 
-// 32: error: register %riz is only available in 64-bit mode
-// 64: error: base register is 32-bit, but index register is not
+// X86: error: register %riz is only available in 64-bit mode
+// X64: error: base register is 32-bit, but index register is not
 mov (%eax,%riz), %ebx
 
 
@@ -128,68 +128,68 @@ v_gs  = %gs
 v_imm = 4
 $test = %ebx
 
-// 32: 7: error: expected register here
-// 64: 7: error: expected register here
+// X86: 7: error: expected register here
+// X64: 7: error: expected register here
 mov 4(4), %eax	
 
-// 32: 7: error: expected register here
-// 64: 7: error: expected register here
+// X86: 7: error: expected register here
+// X64: 7: error: expected register here
 mov 5(v_imm), %eax		
 	
-// 32: 7: error: invalid register name
-// 64: 7: error: invalid register name
+// X86: 7: error: invalid register name
+// X64: 7: error: invalid register name
 mov 6(%v_imm), %eax		
 	
-// 32: 8: warning: scale factor without index register is ignored
-// 64: 8: warning: scale factor without index register is ignored
+// X86: 8: warning: scale factor without index register is ignored
+// X64: 8: warning: scale factor without index register is ignored
 mov 7(,v_imm), %eax		
 
-// 64: 6: error: expected immediate expression
+// X64: 6: error: expected immediate expression
 mov $%eax, %ecx
 
-// 32: 6: error: expected immediate expression
-// 64: 6: error: expected immediate expression
+// X86: 6: error: expected immediate expression
+// X64: 6: error: expected immediate expression
 mov $v_eax, %ecx
 
-// 32: error: unexpected token in argument list
-// 64: error: unexpected token in argument list
+// X86: error: unexpected token in argument list
+// X64: error: unexpected token in argument list
 mov v_ecx(%eax), %ecx	
 
-// 32: 7: error: invalid operand for instruction
-// 64: 7: error: invalid operand for instruction
+// X86: 7: error: invalid operand for instruction
+// X64: 7: error: invalid operand for instruction
 addb (%dx), %al
 
-// 32: error: instruction requires: 64-bit mode
+// X86: error: instruction requires: 64-bit mode
 cqto
 
-// 32: error: instruction requires: 64-bit mode
+// X86: error: instruction requires: 64-bit mode
 cltq
 
-// 32: error: instruction requires: 64-bit mode
+// X86: error: instruction requires: 64-bit mode
 cmpxchg16b (%eax)
 
-// 32: error: unsupported instruction
-// 64: error: unsupported instruction
+// X86: error: unsupported instruction
+// X64: error: unsupported instruction
 {vex} vmovdqu32 %xmm0, %xmm0
 
-// 32: error: unsupported instruction
-// 64: error: unsupported instruction
+// X86: error: unsupported instruction
+// X64: error: unsupported instruction
 {vex2} vmovdqu32 %xmm0, %xmm0
 
-// 32: error: unsupported instruction
-// 64: error: unsupported instruction
+// X86: error: unsupported instruction
+// X64: error: unsupported instruction
 {vex3} vmovdqu32 %xmm0, %xmm0
 
-// 32: error: unsupported instruction
-// 64: error: unsupported instruction
+// X86: error: unsupported instruction
+// X64: error: unsupported instruction
 {evex} vmovdqu %xmm0, %xmm0
 
-// 32: 12: error: immediate must be an integer in range [0, 15]
-// 64: 12: error: immediate must be an integer in range [0, 15]
+// X86: 12: error: immediate must be an integer in range [0, 15]
+// X64: 12: error: immediate must be an integer in range [0, 15]
 vpermil2pd $16, %xmm3, %xmm5, %xmm1, %xmm2
 
-// 32: error: instruction requires: 64-bit mode
+// X86: error: instruction requires: 64-bit mode
 pbndkb
 
-// 32: error: register %r16d is only available in 64-bit mode
+// X86: error: register %r16d is only available in 64-bit mode
 movl %eax, %r16d
-- 
GitLab


From 89873694654a635cabdd861ddebd61a041d8342f Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 15 May 2024 13:49:39 +0100
Subject: [PATCH 0842/1206] [X86] sibcall - cleanup check prefixes identified
 in #92248

Avoid using numbers as check prefix - replace with actual triple config names
---
 llvm/test/CodeGen/X86/sibcall-2.ll     | 36 +++++++++++++-------------
 llvm/test/CodeGen/X86/sibcall-byval.ll | 20 +++++++-------
 2 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/llvm/test/CodeGen/X86/sibcall-2.ll b/llvm/test/CodeGen/X86/sibcall-2.ll
index a4345cf28335..ca42fbafde14 100644
--- a/llvm/test/CodeGen/X86/sibcall-2.ll
+++ b/llvm/test/CodeGen/X86/sibcall-2.ll
@@ -1,48 +1,48 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=i386-apple-darwin   -frame-pointer=all | FileCheck %s -check-prefix=32
-; RUN: llc -verify-machineinstrs < %s -mtriple=x86_64-apple-darwin -frame-pointer=all | FileCheck %s -check-prefix=64
+; RUN: llc -verify-machineinstrs < %s -mtriple=i386-apple-darwin   -frame-pointer=all | FileCheck %s -check-prefix=X86
+; RUN: llc -verify-machineinstrs < %s -mtriple=x86_64-apple-darwin -frame-pointer=all | FileCheck %s -check-prefix=X64
 
 ; Tail call should not use ebp / rbp after it's popped. Use esp / rsp.
 
 define void @t1(ptr nocapture %value) nounwind {
 entry:
-; 32-LABEL: t1:
-; 32: jmpl *4(%esp)
+; X86-LABEL: t1:
+; X86: jmpl *4(%esp)
 
-; 64-LABEL: t1:
-; 64: jmpq *%rdi
+; X64-LABEL: t1:
+; X64: jmpq *%rdi
   tail call void %value() nounwind
   ret void
 }
 
 define void @t2(i32 %a, ptr nocapture %value) nounwind {
 entry:
-; 32-LABEL: t2:
-; 32: jmpl *8(%esp)
+; X86-LABEL: t2:
+; X86: jmpl *8(%esp)
 
-; 64-LABEL: t2:
-; 64: jmpq *%rsi
+; X64-LABEL: t2:
+; X64: jmpq *%rsi
   tail call void %value() nounwind
   ret void
 }
 
 define void @t3(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, ptr nocapture %value) nounwind {
 entry:
-; 32-LABEL: t3:
-; 32: jmpl *28(%esp)
+; X86-LABEL: t3:
+; X86: jmpl *28(%esp)
 
-; 64-LABEL: t3:
-; 64: jmpq *8(%rsp)
+; X64-LABEL: t3:
+; X64: jmpq *8(%rsp)
   tail call void %value() nounwind
   ret void
 }
 
 define void @t4(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, ptr nocapture %value) nounwind {
 entry:
-; 32-LABEL: t4:
-; 32: jmpl *32(%esp)
+; X86-LABEL: t4:
+; X86: jmpl *32(%esp)
 
-; 64-LABEL: t4:
-; 64: jmpq *16(%rsp)
+; X64-LABEL: t4:
+; X64: jmpq *16(%rsp)
   tail call void %value() nounwind
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/sibcall-byval.ll b/llvm/test/CodeGen/X86/sibcall-byval.ll
index 12dbac1389a0..0e06833ad70b 100644
--- a/llvm/test/CodeGen/X86/sibcall-byval.ll
+++ b/llvm/test/CodeGen/X86/sibcall-byval.ll
@@ -1,15 +1,15 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin9   | FileCheck %s -check-prefix=32
-; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s -check-prefix=64
+; RUN: llc < %s -mtriple=i386-apple-darwin9   | FileCheck %s -check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s -check-prefix=X64
 
 %struct.p = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
 
 define i32 @f(ptr byval(%struct.p) align 4 %q) nounwind ssp {
 entry:
-; 32: _f:
-; 32: jmp _g
+; X86: _f:
+; X86: jmp _g
 
-; 64: _f:
-; 64: jmp _g
+; X64: _f:
+; X64: jmp _g
   %call = tail call i32 @g(ptr byval(%struct.p) align 4 %q) nounwind
   ret i32 %call
 }
@@ -18,11 +18,11 @@ declare i32 @g(ptr byval(%struct.p) align 4)
 
 define i32 @h(ptr byval(%struct.p) align 4 %q, i32 %r) nounwind ssp {
 entry:
-; 32: _h:
-; 32: jmp _i
+; X86: _h:
+; X86: jmp _i
 
-; 64: _h:
-; 64: jmp _i
+; X64: _h:
+; X64: jmp _i
 
   %call = tail call i32 @i(ptr byval(%struct.p) align 4 %q, i32 %r) nounwind
   ret i32 %call
-- 
GitLab


From 932f0de43a9e334e161a69a50bd6b01cd51e238e Mon Sep 17 00:00:00 2001
From: Julian Schmidt <git.julian.schmidt@gmail.com>
Date: Wed, 15 May 2024 14:52:32 +0200
Subject: [PATCH 0843/1206] [clang-tidy] fix crash due to assumed callee in
 min-max-use-initializer-list (#91992)

Previously, the call to `findArgs` for a `CallExpr` inside of a `min` or
`max` call would call `findArgs` before checking if the argument is a
call to `min` or `max`, which is what `findArgs` is expecting.
The fix moves the name checking before the call to `findArgs`, such that
only a `min` or `max` function call is used as an argument.

Fixes #91982
Fixes #92249
---
 .../MinMaxUseInitializerListCheck.cpp         | 10 ++++-----
 .../min-max-use-initializer-list.cpp          | 21 +++++++++++++++++++
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/modernize/MinMaxUseInitializerListCheck.cpp b/clang-tools-extra/clang-tidy/modernize/MinMaxUseInitializerListCheck.cpp
index 45f7700463d5..418699ffbc4d 100644
--- a/clang-tools-extra/clang-tidy/modernize/MinMaxUseInitializerListCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/MinMaxUseInitializerListCheck.cpp
@@ -129,17 +129,17 @@ generateReplacements(const MatchFinder::MatchResult &Match,
       continue;
     }
 
+    // if the nested call is not the same as the top call
+    if (InnerCall->getDirectCallee()->getQualifiedNameAsString() !=
+        TopCall->getDirectCallee()->getQualifiedNameAsString())
+      continue;
+
     const FindArgsResult InnerResult = findArgs(InnerCall);
 
     // if the nested call doesn't have arguments skip it
     if (!InnerResult.First || !InnerResult.Last)
       continue;
 
-    // if the nested call is not the same as the top call
-    if (InnerCall->getDirectCallee()->getQualifiedNameAsString() !=
-        TopCall->getDirectCallee()->getQualifiedNameAsString())
-      continue;
-
     // if the nested call doesn't have the same compare function
     if ((Result.Compare || InnerResult.Compare) &&
         !utils::areStatementsIdentical(Result.Compare, InnerResult.Compare,
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/min-max-use-initializer-list.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/min-max-use-initializer-list.cpp
index 51ab9bda975f..1f2dad2b933c 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/min-max-use-initializer-list.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/min-max-use-initializer-list.cpp
@@ -300,6 +300,27 @@ B maxTT2 = std::max(B(), std::max(B(), B()));
 B maxTT3 = std::max(B(), std::max(B(), B()), [](const B &lhs, const B &rhs) { return lhs.a[0] < rhs.a[0]; });
 // CHECK-FIXES: B maxTT3 = std::max(B(), std::max(B(), B()), [](const B &lhs, const B &rhs) { return lhs.a[0] < rhs.a[0]; });
 
+struct GH91982 {
+  int fun0Args();
+  int fun1Arg(int a);
+  int fun2Args(int a, int b);
+  int fun3Args(int a, int b, int c);
+  int fun4Args(int a, int b, int c, int d);
+
+  int foo() {
+    return std::max(
+        fun0Args(),
+        std::max(fun1Arg(0),
+                 std::max(fun2Args(0, 1),
+                          std::max(fun3Args(0, 1, 2), fun4Args(0, 1, 2, 3)))));
+// CHECK-MESSAGES: :[[@LINE-5]]:12: warning: do not use nested 'std::max' calls, use an initializer list instead [modernize-min-max-use-initializer-list]
+// CHECK-FIXES: return std::max(
+// CHECK-FIXES-NEXT: {fun0Args(),
+// CHECK-FIXES-NEXT: fun1Arg(0),
+// CHECK-FIXES-NEXT: fun2Args(0, 1),
+// CHECK-FIXES-NEXT: fun3Args(0, 1, 2), fun4Args(0, 1, 2, 3)});
+  }
+};
 
 } // namespace
 
-- 
GitLab


From 83d9aa27680b6a7f3556fcf13ada70b4be95bab2 Mon Sep 17 00:00:00 2001
From: Pietro Ghiglio <pietro.ghiglio@codeplay.com>
Date: Wed, 15 May 2024 15:03:21 +0200
Subject: [PATCH 0844/1206] [VPlan] Add scalar inferencing support for
 addrspace cast (#92107)

Fixes https://github.com/llvm/llvm-project/issues/91434

PR: https://github.com/llvm/llvm-project/pull/92107
---
 .../Transforms/Vectorize/VPlanAnalysis.cpp    |  1 +
 llvm/test/Transforms/LoopVectorize/as_cast.ll | 40 +++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/as_cast.ll

diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 5f93339083f0..efe8c21874a3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -171,6 +171,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPReplicateRecipe *R) {
   case Instruction::ICmp:
   case Instruction::FCmp:
     return IntegerType::get(Ctx, 1);
+  case Instruction::AddrSpaceCast:
   case Instruction::Alloca:
   case Instruction::BitCast:
   case Instruction::Trunc:
diff --git a/llvm/test/Transforms/LoopVectorize/as_cast.ll b/llvm/test/Transforms/LoopVectorize/as_cast.ll
new file mode 100644
index 000000000000..58a8c3d078f0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/as_cast.ll
@@ -0,0 +1,40 @@
+; RUN: opt -passes=loop-vectorize %s -force-vector-width=1 -force-vector-interleave=2 -S -o - | FileCheck %s
+
+define void @foo(ptr addrspace(1) %in) {
+entry:
+  br label %loop
+
+loop:
+  %iter = phi i64 [ %next, %loop ], [ 0, %entry ]
+  %ascast = addrspacecast ptr addrspace(1) %in to ptr
+  %next = add i64 %iter, 1
+  %arrayidx = getelementptr inbounds i64, ptr %ascast, i64 %next
+  store i64 %next, ptr %arrayidx, align 4
+
+; check that we find the two interleaved blocks with ascast, gep and store:
+; CHECK: pred.store.if:
+; CHECK: [[ID1:%.*]] = add i64 %{{.*}}, 1
+; CHECK: [[AS1:%.*]] = addrspacecast ptr addrspace(1) %{{.*}} to ptr
+; CHECK: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[AS1]], i64 [[ID1]]
+; CHECK: store i64 [[ID1]], ptr [[GEP1]]
+
+; CHECK: pred.store.if1:
+; CHECK: [[ID2:%.*]] = add i64 %{{.*}}, 1
+; CHECK: [[AS2:%.*]] = addrspacecast ptr addrspace(1) %in to ptr
+; CHECK: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[AS2]], i64 [[ID2]]
+; CHECK: store i64 [[ID2]], ptr %9, align 4
+
+  %cmp = icmp eq i64 %next, 7
+  br i1 %cmp, label %exit, label %loop
+
+; check that we branch to the exit block
+; CHECK: middle.block:
+; CHECK: br i1 true, label %exit, label %scalar.ph
+
+exit:
+  ret void
+; CHECK: exit:
+; CHECK:   ret void
+}
+
+; CHECK: !{{[0-9]*}} = !{!"llvm.loop.isvectorized", i32 1}
-- 
GitLab


From d06270ee00e37b247eb99268fb2f106dbeee08ff Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar@redhat.com>
Date: Wed, 15 May 2024 06:08:29 -0700
Subject: [PATCH 0845/1206] [workflows] Fix libclang-abi-tests to work with new
 version scheme (#91865)

---
 .github/workflows/libclang-abi-tests.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/libclang-abi-tests.yml b/.github/workflows/libclang-abi-tests.yml
index ccfc1e5fb8a7..972d21c3bced 100644
--- a/.github/workflows/libclang-abi-tests.yml
+++ b/.github/workflows/libclang-abi-tests.yml
@@ -33,7 +33,6 @@ jobs:
       ABI_HEADERS: ${{ steps.vars.outputs.ABI_HEADERS }}
       ABI_LIBS: ${{ steps.vars.outputs.ABI_LIBS }}
       BASELINE_VERSION_MAJOR: ${{ steps.vars.outputs.BASELINE_VERSION_MAJOR }}
-      BASELINE_VERSION_MINOR: ${{ steps.vars.outputs.BASELINE_VERSION_MINOR }}
       LLVM_VERSION_MAJOR: ${{ steps.version.outputs.LLVM_VERSION_MAJOR }}
       LLVM_VERSION_MINOR: ${{ steps.version.outputs.LLVM_VERSION_MINOR }}
       LLVM_VERSION_PATCH: ${{ steps.version.outputs.LLVM_VERSION_PATCH }}
@@ -51,9 +50,9 @@ jobs:
         id: vars
         run: |
           remote_repo='https://github.com/llvm/llvm-project'
-          if [ ${{ steps.version.outputs.LLVM_VERSION_MINOR }} -ne 0 ] || [ ${{ steps.version.outputs.LLVM_VERSION_PATCH }} -eq 0 ]; then
+          if [ ${{ steps.version.outputs.LLVM_VERSION_PATCH }} -eq 0 ]; then
             major_version=$(( ${{ steps.version.outputs.LLVM_VERSION_MAJOR }} - 1))
-            baseline_ref="llvmorg-$major_version.0.0"
+            baseline_ref="llvmorg-$major_version.1.0"
 
             # If there is a minor release, we want to use that as the base line.
             minor_ref=$(git ls-remote --refs -t "$remote_repo" llvmorg-"$major_version".[1-9].[0-9] | tail -n1 | grep -o 'llvmorg-.\+' || true)
@@ -75,7 +74,7 @@ jobs:
           else
             {
               echo "BASELINE_VERSION_MAJOR=${{ steps.version.outputs.LLVM_VERSION_MAJOR }}"
-              echo "BASELINE_REF=llvmorg-${{ steps.version.outputs.LLVM_VERSION_MAJOR }}.0.0"
+              echo "BASELINE_REF=llvmorg-${{ steps.version.outputs.LLVM_VERSION_MAJOR }}.1.0"
               echo "ABI_HEADERS=."
               echo "ABI_LIBS=libclang.so libclang-cpp.so"
             } >> "$GITHUB_OUTPUT"
-- 
GitLab


From 97418bb519d90542aad3c1f82c80264381a5758e Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 15 May 2024 14:03:13 +0100
Subject: [PATCH 0846/1206] [X86] patchable functions - cleanup check prefixes
 identified in #92248

Avoid using numbers as check prefix - replace with actual triple config names
---
 .../X86/patchable-function-entry-ibt.ll       |  44 +++---
 .../CodeGen/X86/patchable-function-entry.ll   |  62 ++++----
 llvm/test/CodeGen/X86/patchable-prologue.ll   | 134 +++++++++---------
 3 files changed, 120 insertions(+), 120 deletions(-)

diff --git a/llvm/test/CodeGen/X86/patchable-function-entry-ibt.ll b/llvm/test/CodeGen/X86/patchable-function-entry-ibt.ll
index d0a9bee7878c..bcb1106de749 100644
--- a/llvm/test/CodeGen/X86/patchable-function-entry-ibt.ll
+++ b/llvm/test/CodeGen/X86/patchable-function-entry-ibt.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=i686 %s -o - | FileCheck --check-prefixes=CHECK,32 %s
-; RUN: llc -mtriple=x86_64 %s -o - | FileCheck --check-prefixes=CHECK,64 %s
+; RUN: llc -mtriple=i686 %s -o - | FileCheck --check-prefixes=CHECK,X86 %s
+; RUN: llc -mtriple=x86_64 %s -o - | FileCheck --check-prefixes=CHECK,X64 %s
 
 ;; -fpatchable-function-entry=0 -fcf-protection=branch
 define void @f0() "patchable-function-entry"="0" {
@@ -7,8 +7,8 @@ define void @f0() "patchable-function-entry"="0" {
 ; CHECK-NEXT: .Lfunc_begin0:
 ; CHECK-NEXT: .cfi_startproc
 ; CHECK-NEXT: # %bb.0:
-; 32-NEXT:     endbr32
-; 64-NEXT:     endbr64
+; X86-NEXT:     endbr32
+; X64-NEXT:     endbr64
 ; CHECK-NEXT:  ret
 ; CHECK-NOT:  .section __patchable_function_entries
   ret void
@@ -22,16 +22,16 @@ define void @f1() "patchable-function-entry"="1" {
 ; CHECK-NEXT: .Lfunc_begin1:
 ; CHECK-NEXT: .cfi_startproc
 ; CHECK-NEXT: # %bb.0:
-; 32-NEXT:     endbr32
-; 64-NEXT:     endbr64
+; X86-NEXT:     endbr32
+; X64-NEXT:     endbr64
 ; CHECK-NEXT: .Lpatch0:
 ; CHECK-NEXT:  nop
 ; CHECK-NEXT:  ret
 ; CHECK:      .section __patchable_function_entries,"awo",@progbits,f1{{$}}
-; 32-NEXT:    .p2align 2
-; 32-NEXT:    .long .Lpatch0
-; 64-NEXT:    .p2align 3
-; 64-NEXT:    .quad .Lpatch0
+; X86-NEXT:    .p2align 2
+; X86-NEXT:    .long .Lpatch0
+; X64-NEXT:    .p2align 3
+; X64-NEXT:    .quad .Lpatch0
   ret void
 }
 
@@ -44,17 +44,17 @@ define void @f2_1() "patchable-function-entry"="1" "patchable-function-prefix"="
 ; CHECK-NEXT: .Lfunc_begin2:
 ; CHECK-NEXT: .cfi_startproc
 ; CHECK-NEXT: # %bb.0:
-; 32-NEXT:     endbr32
-; 64-NEXT:     endbr64
+; X86-NEXT:     endbr32
+; X64-NEXT:     endbr64
 ; CHECK-NEXT:  nop
 ; CHECK-NEXT:  ret
 ; CHECK:      .Lfunc_end2:
 ; CHECK-NEXT: .size f2_1, .Lfunc_end2-f2_1
 ; CHECK:      .section __patchable_function_entries,"awo",@progbits,f2_1{{$}}
-; 32-NEXT:    .p2align 2
-; 32-NEXT:    .long .Ltmp0
-; 64-NEXT:    .p2align 3
-; 64-NEXT:    .quad .Ltmp0
+; X86-NEXT:    .p2align 2
+; X86-NEXT:    .long .Ltmp0
+; X64-NEXT:    .p2align 3
+; X64-NEXT:    .quad .Ltmp0
   ret void
 }
 
@@ -74,10 +74,10 @@ define internal void @f1i() "patchable-function-entry"="1" {
 ;; Another basic block has ENDBR, but it doesn't affect our decision to not create .Lpatch0
 ; CHECK:       endbr
 ; CHECK:      .section __patchable_function_entries,"awo",@progbits,f1i{{$}}
-; 32-NEXT:    .p2align 2
-; 32-NEXT:    .long .Lfunc_begin3
-; 64-NEXT:    .p2align 3
-; 64-NEXT:    .quad .Lfunc_begin3
+; X86-NEXT:    .p2align 2
+; X86-NEXT:    .long .Lfunc_begin3
+; X64-NEXT:    .p2align 3
+; X64-NEXT:    .quad .Lfunc_begin3
 entry:
   tail call i32 @llvm.eh.sjlj.setjmp(ptr @buf)
   ret void
@@ -93,8 +93,8 @@ entry:
 ; CHECK-NEXT: .Lfunc_begin{{.*}}:
 ; CHECK-NEXT:   .cfi_startproc
 ; CHECK-NEXT:   # %bb.0:
-; 32-NEXT:      endbr32
-; 64-NEXT:      endbr64
+; X86-NEXT:      endbr32
+; X64-NEXT:      endbr64
 ; CHECK-NEXT:   nop
 ; CHECK-NEXT:   ret
 define void @sanitize_function(ptr noundef %x) "patchable-function-prefix"="1" "patchable-function-entry"="1" !func_sanitize !1 {
diff --git a/llvm/test/CodeGen/X86/patchable-function-entry.ll b/llvm/test/CodeGen/X86/patchable-function-entry.ll
index 8c37f5451080..54ecd8b1e5da 100644
--- a/llvm/test/CodeGen/X86/patchable-function-entry.ll
+++ b/llvm/test/CodeGen/X86/patchable-function-entry.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=i386 %s -o - | FileCheck --check-prefixes=CHECK,32 %s
-; RUN: llc -mtriple=x86_64 %s -o - | FileCheck --check-prefixes=CHECK,64 %s
-; RUN: llc -mtriple=x86_64 -function-sections %s -o - | FileCheck --check-prefixes=CHECK,64 %s
+; RUN: llc -mtriple=i386 %s -o - | FileCheck --check-prefixes=CHECK,X86 %s
+; RUN: llc -mtriple=x86_64 %s -o - | FileCheck --check-prefixes=CHECK,X64 %s
+; RUN: llc -mtriple=x86_64 -function-sections %s -o - | FileCheck --check-prefixes=CHECK,X64 %s
 
 define void @f0() "patchable-function-entry"="0" {
 ; CHECK-LABEL: f0:
@@ -17,10 +17,10 @@ define void @f1() "patchable-function-entry"="1" {
 ; CHECK:       nop
 ; CHECK-NEXT:  ret
 ; CHECK:       .section __patchable_function_entries,"awo",@progbits,f1{{$}}
-; 32:          .p2align 2
-; 32-NEXT:     .long .Lfunc_begin1
-; 64:          .p2align 3
-; 64-NEXT:     .quad .Lfunc_begin1
+; X86:          .p2align 2
+; X86-NEXT:     .long .Lfunc_begin1
+; X64:          .p2align 3
+; X64-NEXT:     .quad .Lfunc_begin1
   ret void
 }
 
@@ -31,14 +31,14 @@ define void @f1() "patchable-function-entry"="1" {
 define void @f2() "patchable-function-entry"="2" {
 ; CHECK-LABEL: f2:
 ; CHECK-NEXT: .Lfunc_begin2:
-; 32:          xchgw %ax, %ax
-; 64:          xchgw %ax, %ax
+; X86:          xchgw %ax, %ax
+; X64:          xchgw %ax, %ax
 ; CHECK-NEXT:  ret
 ; CHECK:       .section __patchable_function_entries,"awo",@progbits,f2{{$}}
-; 32:          .p2align 2
-; 32-NEXT:     .long .Lfunc_begin2
-; 64:          .p2align 3
-; 64-NEXT:     .quad .Lfunc_begin2
+; X86:          .p2align 2
+; X86-NEXT:     .long .Lfunc_begin2
+; X64:          .p2align 3
+; X64-NEXT:     .quad .Lfunc_begin2
   ret void
 }
 
@@ -46,15 +46,15 @@ $f3 = comdat any
 define void @f3() "patchable-function-entry"="3" comdat {
 ; CHECK-LABEL: f3:
 ; CHECK-NEXT: .Lfunc_begin3:
-; 32:          xchgw %ax, %ax
-; 32-NEXT:     nop
-; 64:          nopl (%rax)
+; X86:          xchgw %ax, %ax
+; X86-NEXT:     nop
+; X64:          nopl (%rax)
 ; CHECK:       ret
 ; CHECK:       .section __patchable_function_entries,"awoG",@progbits,f3,f3,comdat{{$}}
-; 32:          .p2align 2
-; 32-NEXT:     .long .Lfunc_begin3
-; 64:          .p2align 3
-; 64-NEXT:     .quad .Lfunc_begin3
+; X86:          .p2align 2
+; X86-NEXT:     .long .Lfunc_begin3
+; X64:          .p2align 3
+; X64-NEXT:     .quad .Lfunc_begin3
   ret void
 }
 
@@ -62,15 +62,15 @@ $f5 = comdat any
 define void @f5() "patchable-function-entry"="5" comdat {
 ; CHECK-LABEL: f5:
 ; CHECK-NEXT: .Lfunc_begin4:
-; 32-COUNT-2:  xchgw %ax, %ax
-; 32-NEXT:     nop
-; 64:          nopl 8(%rax,%rax)
+; X86-COUNT-2:  xchgw %ax, %ax
+; X86-NEXT:     nop
+; X64:          nopl 8(%rax,%rax)
 ; CHECK-NEXT:  ret
 ; CHECK:       .section __patchable_function_entries,"awoG",@progbits,f5,f5,comdat{{$}}
-; 32:          .p2align 2
-; 32-NEXT:     .long .Lfunc_begin4
-; 64:          .p2align 3
-; 64-NEXT:     .quad .Lfunc_begin4
+; X86:          .p2align 2
+; X86-NEXT:     .long .Lfunc_begin4
+; X64:          .p2align 3
+; X64-NEXT:     .quad .Lfunc_begin4
   ret void
 }
 
@@ -91,10 +91,10 @@ define void @f3_2() "patchable-function-entry"="1" "patchable-function-prefix"="
 ; CHECK:      .Lfunc_end5:
 ; CHECK-NEXT: .size f3_2, .Lfunc_end5-f3_2
 ; CHECK:      .section __patchable_function_entries,"awo",@progbits,f3_2{{$}}
-; 32:         .p2align 2
-; 32-NEXT:    .long .Ltmp0
-; 64:         .p2align 3
-; 64-NEXT:    .quad .Ltmp0
+; X86:         .p2align 2
+; X86-NEXT:    .long .Ltmp0
+; X64:         .p2align 3
+; X64-NEXT:    .quad .Ltmp0
   %frame = alloca i8, i32 16
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/patchable-prologue.ll b/llvm/test/CodeGen/X86/patchable-prologue.ll
index 43761e3d1e1e..aec76a359d26 100644
--- a/llvm/test/CodeGen/X86/patchable-prologue.ll
+++ b/llvm/test/CodeGen/X86/patchable-prologue.ll
@@ -1,10 +1,10 @@
 ; RUN: llc -verify-machineinstrs -filetype=obj -o - -mtriple=x86_64-apple-macosx < %s | llvm-objdump --no-print-imm-hex --triple=x86_64-apple-macosx -d - | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mtriple=x86_64-apple-macosx < %s | FileCheck %s --check-prefix=CHECK-ALIGN
-; RUN: llc -verify-machineinstrs -show-mc-encoding -mtriple=i386 < %s | FileCheck %s --check-prefixes=32,32CFI,XCHG
-; RUN: llc -verify-machineinstrs -show-mc-encoding -mtriple=i386-windows-msvc < %s | FileCheck %s --check-prefixes=32,MOV
-; RUN: llc -verify-machineinstrs -show-mc-encoding -mtriple=i386-windows-msvc -mcpu=pentium3 < %s | FileCheck %s --check-prefixes=32,MOV
-; RUN: llc -verify-machineinstrs -show-mc-encoding -mtriple=i386-windows-msvc -mcpu=pentium4 < %s | FileCheck %s --check-prefixes=32,XCHG
-; RUN: llc -verify-machineinstrs -show-mc-encoding -mtriple=x86_64-windows-msvc < %s | FileCheck %s --check-prefix=64
+; RUN: llc -verify-machineinstrs -show-mc-encoding -mtriple=i386 < %s | FileCheck %s --check-prefixes=X86,X86CFI,XCHG
+; RUN: llc -verify-machineinstrs -show-mc-encoding -mtriple=i386-windows-msvc < %s | FileCheck %s --check-prefixes=X86,MOV
+; RUN: llc -verify-machineinstrs -show-mc-encoding -mtriple=i386-windows-msvc -mcpu=pentium3 < %s | FileCheck %s --check-prefixes=X86,MOV
+; RUN: llc -verify-machineinstrs -show-mc-encoding -mtriple=i386-windows-msvc -mcpu=pentium4 < %s | FileCheck %s --check-prefixes=X86,XCHG
+; RUN: llc -verify-machineinstrs -show-mc-encoding -mtriple=x86_64-windows-msvc < %s | FileCheck %s --check-prefix=X64
 
 declare void @callee(ptr)
 
@@ -15,18 +15,18 @@ define void @f0() "patchable-function"="prologue-short-redirect" {
 ; CHECK-ALIGN: 	.p2align	4, 0x90
 ; CHECK-ALIGN: _f0:
 
-; 32: f0:
-; 32CFI-NEXT: .cfi_startproc
-; 32-NEXT: # %bb.0:
+; X86: f0:
+; X86CFI-NEXT: .cfi_startproc
+; X86-NEXT: # %bb.0:
 ; XCHG-NEXT: xchgw   %ax, %ax                # encoding: [0x66,0x90]
 ; MOV-NEXT: movl    %edi, %edi              # encoding: [0x8b,0xff]
-; 32-NEXT: retl
+; X86-NEXT: retl
+
+; X64: f0:
+; X64-NEXT: # %bb.0:
+; X64-NEXT: xchgw   %ax, %ax                # encoding: [0x66,0x90]
+; X64-NEXT: retq
 
-; 64: f0:
-; 64-NEXT: # %bb.0:
-; 64-NEXT: xchgw   %ax, %ax                # encoding: [0x66,0x90]
-; 64-NEXT: retq
-		
   ret void
 }
 
@@ -38,19 +38,19 @@ define void @f1() "patchable-function"="prologue-short-redirect" "frame-pointer"
 ; CHECK-ALIGN: 	.p2align	4, 0x90
 ; CHECK-ALIGN: _f1:
 
-; 32: f1:
-; 32CFI-NEXT: .cfi_startproc
-; 32-NEXT: # %bb.0:
+; X86: f1:
+; X86CFI-NEXT: .cfi_startproc
+; X86-NEXT: # %bb.0:
 ; XCHG-NEXT: xchgw   %ax, %ax                # encoding: [0x66,0x90]
 ; MOV-NEXT: movl    %edi, %edi              # encoding: [0x8b,0xff]
-; 32-NEXT: pushl   %ebp
-
-; 64: f1:
-; 64-NEXT: .seh_proc f1
-; 64-NEXT: # %bb.0:
-; 64-NEXT: xchgw %ax, %ax
-; 64-NEXT: pushq   %rbp
-		
+; X86-NEXT: pushl   %ebp
+
+; X64: f1:
+; X64-NEXT: .seh_proc f1
+; X64-NEXT: # %bb.0:
+; X64-NEXT: xchgw %ax, %ax
+; X64-NEXT: pushq   %rbp
+
   ret void
 }
 
@@ -61,18 +61,18 @@ define void @f2() "patchable-function"="prologue-short-redirect" {
 ; CHECK-ALIGN: 	.p2align	4, 0x90
 ; CHECK-ALIGN: _f2:
 
-; 32: f2:
-; 32CFI-NEXT: .cfi_startproc
-; 32-NEXT: # %bb.0:
+; X86: f2:
+; X86CFI-NEXT: .cfi_startproc
+; X86-NEXT: # %bb.0:
 ; XCHG-NEXT: xchgw   %ax, %ax                # encoding: [0x66,0x90]
 ; MOV-NEXT: movl    %edi, %edi              # encoding: [0x8b,0xff]
-; 32-NEXT: pushl   %ebp
+; X86-NEXT: pushl   %ebp
+
+; X64: f2:
+; X64-NEXT: .seh_proc f2
+; X64-NEXT: # %bb.0:
+; X64-NEXT: subq    $200, %rsp
 
-; 64: f2:
-; 64-NEXT: .seh_proc f2
-; 64-NEXT: # %bb.0:
-; 64-NEXT: subq    $200, %rsp
-		
   %ptr = alloca i64, i32 20
   call void @callee(ptr %ptr)
   ret void
@@ -85,17 +85,17 @@ define void @f3() "patchable-function"="prologue-short-redirect" optsize {
 ; CHECK-ALIGN: 	.p2align	4, 0x90
 ; CHECK-ALIGN: _f3:
 
-; 32: f3:
-; 32CFI-NEXT: .cfi_startproc
-; 32-NEXT: # %bb.0:
+; X86: f3:
+; X86CFI-NEXT: .cfi_startproc
+; X86-NEXT: # %bb.0:
 ; XCHG-NEXT: xchgw   %ax, %ax
 ; MOV-NEXT: movl   %edi, %edi
-; 32-NEXT: retl
+; X86-NEXT: retl
 
-; 64: f3:
-; 64-NEXT: # %bb.0:
-; 64-NEXT: xchgw   %ax, %ax
-; 64-NEXT: retq
+; X64: f3:
+; X64-NEXT: # %bb.0:
+; X64-NEXT: xchgw   %ax, %ax
+; X64-NEXT: retq
 
   ret void
 }
@@ -105,16 +105,16 @@ define void @f3() "patchable-function"="prologue-short-redirect" optsize {
 ; patchable one.
 ; CHECK-LABEL: f4{{>?}}:
 ; CHECK-NEXT: 8b 0c 37  movl  (%rdi,%rsi), %ecx
-; 32: f4:
-; 32CFI-NEXT: .cfi_startproc
-; 32-NEXT: # %bb.0:
+; X86: f4:
+; X86CFI-NEXT: .cfi_startproc
+; X86-NEXT: # %bb.0:
 ; XCHG-NEXT: xchgw   %ax, %ax
 ; MOV-NEXT: movl   %edi, %edi
-; 32-NEXT: pushl   %ebx
+; X86-NEXT: pushl   %ebx
 
-; 64: f4:
-; 64-NEXT: # %bb.0:
-; 64-NOT: xchgw   %ax, %ax
+; X64: f4:
+; X64-NEXT: # %bb.0:
+; X64-NOT: xchgw   %ax, %ax
 
 define i32 @f4(ptr %arg1, i64 %arg2, i32 %arg3) "patchable-function"="prologue-short-redirect" {
 bb:
@@ -143,15 +143,15 @@ bb21:
 ; CHECK-ALIGN: 	.p2align	4, 0x90
 ; CHECK-ALIGN: _emptyfunc:
 
-; 32: emptyfunc:
-; 32CFI-NEXT: .cfi_startproc
-; 32-NEXT: # %bb.0:
+; X86: emptyfunc:
+; X86CFI-NEXT: .cfi_startproc
+; X86-NEXT: # %bb.0:
 ; XCHG-NEXT: xchgw   %ax, %ax
 ; MOV-NEXT: movl   %edi, %edi
 
-; 64: emptyfunc:
-; 64-NEXT: # %bb.0:
-; 64-NEXT: xchgw   %ax, %ax
+; X64: emptyfunc:
+; X64-NEXT: # %bb.0:
+; X64-NEXT: xchgw   %ax, %ax
 
 ; From code: int emptyfunc() {}
 define i32 @emptyfunc() "patchable-function"="prologue-short-redirect" {
@@ -169,15 +169,15 @@ define i32 @emptyfunc() "patchable-function"="prologue-short-redirect" {
 ; CHECK-ALIGN: 	.p2align	4, 0x90
 ; CHECK-ALIGN: _jmp_to_start:
 
-; 32: jmp_to_start:
-; 32CFI-NEXT: .cfi_startproc
-; 32-NEXT: # %bb.0:
+; X86: jmp_to_start:
+; X86CFI-NEXT: .cfi_startproc
+; X86-NEXT: # %bb.0:
 ; XCHG-NEXT: xchgw   %ax, %ax
 ; MOV-NEXT: movl   %edi, %edi
 
-; 64: jmp_to_start:
-; 64-NEXT: # %bb.0:
-; 64-NEXT: xchgw   %ax, %ax
+; X64: jmp_to_start:
+; X64-NEXT: # %bb.0:
+; X64-NEXT: xchgw   %ax, %ax
 
 define dso_local void @jmp_to_start(ptr inreg nocapture noundef %b) "patchable-function"="prologue-short-redirect" {
 entry:
@@ -198,12 +198,12 @@ do.end:                                           ; preds = %do.body
 ; Test that inline asm is properly hotpatched. We currently don't examine the
 ; asm instruction when printing it, thus we always emit patching NOPs.
 
-; 64: inline_asm:
-; 64-NEXT: # %bb.0:
-; 64-NEXT: xchgw   %ax, %ax                        # encoding: [0x66,0x90]
-; 64-NEXT: #APP
-; 64-NEXT: int3                                    # encoding: [0xcc]
-; 64-NEXT: #NO_APP
+; X64: inline_asm:
+; X64-NEXT: # %bb.0:
+; X64-NEXT: xchgw   %ax, %ax                        # encoding: [0x66,0x90]
+; X64-NEXT: #APP
+; X64-NEXT: int3                                    # encoding: [0xcc]
+; X64-NEXT: #NO_APP
 
 define dso_local void @inline_asm() "patchable-function"="prologue-short-redirect" {
 entry:
-- 
GitLab


From 96ac2e3af78a45c4fdf4ecc3f9a76cc00663cac7 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 15 May 2024 14:04:42 +0100
Subject: [PATCH 0847/1206] [X86] cmpxchg-clobber-flags.ll - cleanup check
 prefixes identified in #92248

Avoid using numbers as check prefix - replace with actual triple config names
---
 .../test/CodeGen/X86/cmpxchg-clobber-flags.ll | 394 +++++++++---------
 1 file changed, 197 insertions(+), 197 deletions(-)

diff --git a/llvm/test/CodeGen/X86/cmpxchg-clobber-flags.ll b/llvm/test/CodeGen/X86/cmpxchg-clobber-flags.ll
index 7738ce49a763..29751dcfca5d 100644
--- a/llvm/test/CodeGen/X86/cmpxchg-clobber-flags.ll
+++ b/llvm/test/CodeGen/X86/cmpxchg-clobber-flags.ll
@@ -1,12 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=i386-linux-gnu -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=32-ALL,32-GOOD-RA
-; RUN: llc -mtriple=i386-linux-gnu -verify-machineinstrs -pre-RA-sched=fast %s -o - | FileCheck %s --check-prefixes=32-ALL,32-FAST-RA
+; RUN: llc -mtriple=i386-linux-gnu -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=X86-ALL,X86-GOOD-RA
+; RUN: llc -mtriple=i386-linux-gnu -verify-machineinstrs -pre-RA-sched=fast %s -o - | FileCheck %s --check-prefixes=X86-ALL,X86-FAST-RA
 
-; RUN: llc -mtriple=x86_64-linux-gnu -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=64-ALL
-; RUN: llc -mtriple=x86_64-linux-gnu -verify-machineinstrs -pre-RA-sched=fast %s -o - | FileCheck %s --check-prefix=64-ALL
-; RUN: llc -mtriple=x86_64-linux-gnu -verify-machineinstrs -mattr=+sahf %s -o - | FileCheck %s --check-prefix=64-ALL
-; RUN: llc -mtriple=x86_64-linux-gnu -verify-machineinstrs -mattr=+sahf -pre-RA-sched=fast %s -o - | FileCheck %s --check-prefix=64-ALL
-; RUN: llc -mtriple=x86_64-linux-gnu -verify-machineinstrs -mcpu=corei7 %s -o - | FileCheck %s --check-prefix=64-ALL
+; RUN: llc -mtriple=x86_64-linux-gnu -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=X64-ALL
+; RUN: llc -mtriple=x86_64-linux-gnu -verify-machineinstrs -pre-RA-sched=fast %s -o - | FileCheck %s --check-prefix=X64-ALL
+; RUN: llc -mtriple=x86_64-linux-gnu -verify-machineinstrs -mattr=+sahf %s -o - | FileCheck %s --check-prefix=X64-ALL
+; RUN: llc -mtriple=x86_64-linux-gnu -verify-machineinstrs -mattr=+sahf -pre-RA-sched=fast %s -o - | FileCheck %s --check-prefix=X64-ALL
+; RUN: llc -mtriple=x86_64-linux-gnu -verify-machineinstrs -mcpu=corei7 %s -o - | FileCheck %s --check-prefix=X64-ALL
 
 declare i32 @foo()
 declare i32 @bar(i64)
@@ -24,86 +24,86 @@ declare i32 @bar(i64)
 ; repeated saving and restoring logic and can be trivially managed by the
 ; register allocator.
 define i64 @test_intervening_call(ptr %foo, i64 %bar, i64 %baz) nounwind {
-; 32-GOOD-RA-LABEL: test_intervening_call:
-; 32-GOOD-RA:       # %bb.0: # %entry
-; 32-GOOD-RA-NEXT:    pushl %ebx
-; 32-GOOD-RA-NEXT:    pushl %esi
-; 32-GOOD-RA-NEXT:    pushl %eax
-; 32-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; 32-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; 32-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; 32-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; 32-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; 32-GOOD-RA-NEXT:    lock cmpxchg8b (%esi)
-; 32-GOOD-RA-NEXT:    setne %bl
-; 32-GOOD-RA-NEXT:    subl $8, %esp
-; 32-GOOD-RA-NEXT:    pushl %edx
-; 32-GOOD-RA-NEXT:    pushl %eax
-; 32-GOOD-RA-NEXT:    calll bar@PLT
-; 32-GOOD-RA-NEXT:    addl $16, %esp
-; 32-GOOD-RA-NEXT:    testb %bl, %bl
-; 32-GOOD-RA-NEXT:    jne .LBB0_3
-; 32-GOOD-RA-NEXT:  # %bb.1: # %t
-; 32-GOOD-RA-NEXT:    movl $42, %eax
-; 32-GOOD-RA-NEXT:    jmp .LBB0_2
-; 32-GOOD-RA-NEXT:  .LBB0_3: # %f
-; 32-GOOD-RA-NEXT:    xorl %eax, %eax
-; 32-GOOD-RA-NEXT:  .LBB0_2: # %t
-; 32-GOOD-RA-NEXT:    xorl %edx, %edx
-; 32-GOOD-RA-NEXT:    addl $4, %esp
-; 32-GOOD-RA-NEXT:    popl %esi
-; 32-GOOD-RA-NEXT:    popl %ebx
-; 32-GOOD-RA-NEXT:    retl
+; X86-GOOD-RA-LABEL: test_intervening_call:
+; X86-GOOD-RA:       # %bb.0: # %entry
+; X86-GOOD-RA-NEXT:    pushl %ebx
+; X86-GOOD-RA-NEXT:    pushl %esi
+; X86-GOOD-RA-NEXT:    pushl %eax
+; X86-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-GOOD-RA-NEXT:    lock cmpxchg8b (%esi)
+; X86-GOOD-RA-NEXT:    setne %bl
+; X86-GOOD-RA-NEXT:    subl $8, %esp
+; X86-GOOD-RA-NEXT:    pushl %edx
+; X86-GOOD-RA-NEXT:    pushl %eax
+; X86-GOOD-RA-NEXT:    calll bar@PLT
+; X86-GOOD-RA-NEXT:    addl $16, %esp
+; X86-GOOD-RA-NEXT:    testb %bl, %bl
+; X86-GOOD-RA-NEXT:    jne .LBB0_3
+; X86-GOOD-RA-NEXT:  # %bb.1: # %t
+; X86-GOOD-RA-NEXT:    movl $42, %eax
+; X86-GOOD-RA-NEXT:    jmp .LBB0_2
+; X86-GOOD-RA-NEXT:  .LBB0_3: # %f
+; X86-GOOD-RA-NEXT:    xorl %eax, %eax
+; X86-GOOD-RA-NEXT:  .LBB0_2: # %t
+; X86-GOOD-RA-NEXT:    xorl %edx, %edx
+; X86-GOOD-RA-NEXT:    addl $4, %esp
+; X86-GOOD-RA-NEXT:    popl %esi
+; X86-GOOD-RA-NEXT:    popl %ebx
+; X86-GOOD-RA-NEXT:    retl
 ;
-; 32-FAST-RA-LABEL: test_intervening_call:
-; 32-FAST-RA:       # %bb.0: # %entry
-; 32-FAST-RA-NEXT:    pushl %ebx
-; 32-FAST-RA-NEXT:    pushl %esi
-; 32-FAST-RA-NEXT:    pushl %eax
-; 32-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; 32-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; 32-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; 32-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; 32-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; 32-FAST-RA-NEXT:    lock cmpxchg8b (%esi)
-; 32-FAST-RA-NEXT:    setne %bl
-; 32-FAST-RA-NEXT:    subl $8, %esp
-; 32-FAST-RA-NEXT:    pushl %edx
-; 32-FAST-RA-NEXT:    pushl %eax
-; 32-FAST-RA-NEXT:    calll bar@PLT
-; 32-FAST-RA-NEXT:    addl $16, %esp
-; 32-FAST-RA-NEXT:    testb %bl, %bl
-; 32-FAST-RA-NEXT:    jne .LBB0_3
-; 32-FAST-RA-NEXT:  # %bb.1: # %t
-; 32-FAST-RA-NEXT:    movl $42, %eax
-; 32-FAST-RA-NEXT:    jmp .LBB0_2
-; 32-FAST-RA-NEXT:  .LBB0_3: # %f
-; 32-FAST-RA-NEXT:    xorl %eax, %eax
-; 32-FAST-RA-NEXT:  .LBB0_2: # %t
-; 32-FAST-RA-NEXT:    xorl %edx, %edx
-; 32-FAST-RA-NEXT:    addl $4, %esp
-; 32-FAST-RA-NEXT:    popl %esi
-; 32-FAST-RA-NEXT:    popl %ebx
-; 32-FAST-RA-NEXT:    retl
+; X86-FAST-RA-LABEL: test_intervening_call:
+; X86-FAST-RA:       # %bb.0: # %entry
+; X86-FAST-RA-NEXT:    pushl %ebx
+; X86-FAST-RA-NEXT:    pushl %esi
+; X86-FAST-RA-NEXT:    pushl %eax
+; X86-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-FAST-RA-NEXT:    lock cmpxchg8b (%esi)
+; X86-FAST-RA-NEXT:    setne %bl
+; X86-FAST-RA-NEXT:    subl $8, %esp
+; X86-FAST-RA-NEXT:    pushl %edx
+; X86-FAST-RA-NEXT:    pushl %eax
+; X86-FAST-RA-NEXT:    calll bar@PLT
+; X86-FAST-RA-NEXT:    addl $16, %esp
+; X86-FAST-RA-NEXT:    testb %bl, %bl
+; X86-FAST-RA-NEXT:    jne .LBB0_3
+; X86-FAST-RA-NEXT:  # %bb.1: # %t
+; X86-FAST-RA-NEXT:    movl $42, %eax
+; X86-FAST-RA-NEXT:    jmp .LBB0_2
+; X86-FAST-RA-NEXT:  .LBB0_3: # %f
+; X86-FAST-RA-NEXT:    xorl %eax, %eax
+; X86-FAST-RA-NEXT:  .LBB0_2: # %t
+; X86-FAST-RA-NEXT:    xorl %edx, %edx
+; X86-FAST-RA-NEXT:    addl $4, %esp
+; X86-FAST-RA-NEXT:    popl %esi
+; X86-FAST-RA-NEXT:    popl %ebx
+; X86-FAST-RA-NEXT:    retl
 ;
-; 64-ALL-LABEL: test_intervening_call:
-; 64-ALL:       # %bb.0: # %entry
-; 64-ALL-NEXT:    pushq %rbx
-; 64-ALL-NEXT:    movq %rsi, %rax
-; 64-ALL-NEXT:    lock cmpxchgq %rdx, (%rdi)
-; 64-ALL-NEXT:    setne %bl
-; 64-ALL-NEXT:    movq %rax, %rdi
-; 64-ALL-NEXT:    callq bar@PLT
-; 64-ALL-NEXT:    testb %bl, %bl
-; 64-ALL-NEXT:    jne .LBB0_2
-; 64-ALL-NEXT:  # %bb.1: # %t
-; 64-ALL-NEXT:    movl $42, %eax
-; 64-ALL-NEXT:    popq %rbx
-; 64-ALL-NEXT:    retq
-; 64-ALL-NEXT:  .LBB0_2: # %f
-; 64-ALL-NEXT:    xorl %eax, %eax
-; 64-ALL-NEXT:    popq %rbx
-; 64-ALL-NEXT:    retq
+; X64-ALL-LABEL: test_intervening_call:
+; X64-ALL:       # %bb.0: # %entry
+; X64-ALL-NEXT:    pushq %rbx
+; X64-ALL-NEXT:    movq %rsi, %rax
+; X64-ALL-NEXT:    lock cmpxchgq %rdx, (%rdi)
+; X64-ALL-NEXT:    setne %bl
+; X64-ALL-NEXT:    movq %rax, %rdi
+; X64-ALL-NEXT:    callq bar@PLT
+; X64-ALL-NEXT:    testb %bl, %bl
+; X64-ALL-NEXT:    jne .LBB0_2
+; X64-ALL-NEXT:  # %bb.1: # %t
+; X64-ALL-NEXT:    movl $42, %eax
+; X64-ALL-NEXT:    popq %rbx
+; X64-ALL-NEXT:    retq
+; X64-ALL-NEXT:  .LBB0_2: # %f
+; X64-ALL-NEXT:    xorl %eax, %eax
+; X64-ALL-NEXT:    popq %rbx
+; X64-ALL-NEXT:    retq
 entry:
   %cx = cmpxchg ptr %foo, i64 %bar, i64 %baz seq_cst seq_cst
   %v = extractvalue { i64, i1 } %cx, 0
@@ -120,61 +120,61 @@ f:
 
 ; Interesting in producing a clobber without any function calls.
 define i32 @test_control_flow(ptr %p, i32 %i, i32 %j) nounwind {
-; 32-ALL-LABEL: test_control_flow:
-; 32-ALL:       # %bb.0: # %entry
-; 32-ALL-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; 32-ALL-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
-; 32-ALL-NEXT:    jle .LBB1_6
-; 32-ALL-NEXT:  # %bb.1: # %loop_start
-; 32-ALL-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; 32-ALL-NEXT:    .p2align 4, 0x90
-; 32-ALL-NEXT:  .LBB1_2: # %while.condthread-pre-split.i
-; 32-ALL-NEXT:    # =>This Loop Header: Depth=1
-; 32-ALL-NEXT:    # Child Loop BB1_3 Depth 2
-; 32-ALL-NEXT:    movl (%ecx), %edx
-; 32-ALL-NEXT:    .p2align 4, 0x90
-; 32-ALL-NEXT:  .LBB1_3: # %while.cond.i
-; 32-ALL-NEXT:    # Parent Loop BB1_2 Depth=1
-; 32-ALL-NEXT:    # => This Inner Loop Header: Depth=2
-; 32-ALL-NEXT:    movl %edx, %eax
-; 32-ALL-NEXT:    xorl %edx, %edx
-; 32-ALL-NEXT:    testl %eax, %eax
-; 32-ALL-NEXT:    je .LBB1_3
-; 32-ALL-NEXT:  # %bb.4: # %while.body.i
-; 32-ALL-NEXT:    # in Loop: Header=BB1_2 Depth=1
-; 32-ALL-NEXT:    lock cmpxchgl %eax, (%ecx)
-; 32-ALL-NEXT:    jne .LBB1_2
-; 32-ALL-NEXT:  # %bb.5:
-; 32-ALL-NEXT:    xorl %eax, %eax
-; 32-ALL-NEXT:  .LBB1_6: # %cond.end
-; 32-ALL-NEXT:    retl
+; X86-ALL-LABEL: test_control_flow:
+; X86-ALL:       # %bb.0: # %entry
+; X86-ALL-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ALL-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
+; X86-ALL-NEXT:    jle .LBB1_6
+; X86-ALL-NEXT:  # %bb.1: # %loop_start
+; X86-ALL-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-ALL-NEXT:    .p2align 4, 0x90
+; X86-ALL-NEXT:  .LBB1_2: # %while.condthread-pre-split.i
+; X86-ALL-NEXT:    # =>This Loop Header: Depth=1
+; X86-ALL-NEXT:    # Child Loop BB1_3 Depth 2
+; X86-ALL-NEXT:    movl (%ecx), %edx
+; X86-ALL-NEXT:    .p2align 4, 0x90
+; X86-ALL-NEXT:  .LBB1_3: # %while.cond.i
+; X86-ALL-NEXT:    # Parent Loop BB1_2 Depth=1
+; X86-ALL-NEXT:    # => This Inner Loop Header: Depth=2
+; X86-ALL-NEXT:    movl %edx, %eax
+; X86-ALL-NEXT:    xorl %edx, %edx
+; X86-ALL-NEXT:    testl %eax, %eax
+; X86-ALL-NEXT:    je .LBB1_3
+; X86-ALL-NEXT:  # %bb.4: # %while.body.i
+; X86-ALL-NEXT:    # in Loop: Header=BB1_2 Depth=1
+; X86-ALL-NEXT:    lock cmpxchgl %eax, (%ecx)
+; X86-ALL-NEXT:    jne .LBB1_2
+; X86-ALL-NEXT:  # %bb.5:
+; X86-ALL-NEXT:    xorl %eax, %eax
+; X86-ALL-NEXT:  .LBB1_6: # %cond.end
+; X86-ALL-NEXT:    retl
 ;
-; 64-ALL-LABEL: test_control_flow:
-; 64-ALL:       # %bb.0: # %entry
-; 64-ALL-NEXT:    movl %esi, %eax
-; 64-ALL-NEXT:    cmpl %edx, %esi
-; 64-ALL-NEXT:    jle .LBB1_5
-; 64-ALL-NEXT:    .p2align 4, 0x90
-; 64-ALL-NEXT:  .LBB1_1: # %while.condthread-pre-split.i
-; 64-ALL-NEXT:    # =>This Loop Header: Depth=1
-; 64-ALL-NEXT:    # Child Loop BB1_2 Depth 2
-; 64-ALL-NEXT:    movl (%rdi), %ecx
-; 64-ALL-NEXT:    .p2align 4, 0x90
-; 64-ALL-NEXT:  .LBB1_2: # %while.cond.i
-; 64-ALL-NEXT:    # Parent Loop BB1_1 Depth=1
-; 64-ALL-NEXT:    # => This Inner Loop Header: Depth=2
-; 64-ALL-NEXT:    movl %ecx, %eax
-; 64-ALL-NEXT:    xorl %ecx, %ecx
-; 64-ALL-NEXT:    testl %eax, %eax
-; 64-ALL-NEXT:    je .LBB1_2
-; 64-ALL-NEXT:  # %bb.3: # %while.body.i
-; 64-ALL-NEXT:    # in Loop: Header=BB1_1 Depth=1
-; 64-ALL-NEXT:    lock cmpxchgl %eax, (%rdi)
-; 64-ALL-NEXT:    jne .LBB1_1
-; 64-ALL-NEXT:  # %bb.4:
-; 64-ALL-NEXT:    xorl %eax, %eax
-; 64-ALL-NEXT:  .LBB1_5: # %cond.end
-; 64-ALL-NEXT:    retq
+; X64-ALL-LABEL: test_control_flow:
+; X64-ALL:       # %bb.0: # %entry
+; X64-ALL-NEXT:    movl %esi, %eax
+; X64-ALL-NEXT:    cmpl %edx, %esi
+; X64-ALL-NEXT:    jle .LBB1_5
+; X64-ALL-NEXT:    .p2align 4, 0x90
+; X64-ALL-NEXT:  .LBB1_1: # %while.condthread-pre-split.i
+; X64-ALL-NEXT:    # =>This Loop Header: Depth=1
+; X64-ALL-NEXT:    # Child Loop BB1_2 Depth 2
+; X64-ALL-NEXT:    movl (%rdi), %ecx
+; X64-ALL-NEXT:    .p2align 4, 0x90
+; X64-ALL-NEXT:  .LBB1_2: # %while.cond.i
+; X64-ALL-NEXT:    # Parent Loop BB1_1 Depth=1
+; X64-ALL-NEXT:    # => This Inner Loop Header: Depth=2
+; X64-ALL-NEXT:    movl %ecx, %eax
+; X64-ALL-NEXT:    xorl %ecx, %ecx
+; X64-ALL-NEXT:    testl %eax, %eax
+; X64-ALL-NEXT:    je .LBB1_2
+; X64-ALL-NEXT:  # %bb.3: # %while.body.i
+; X64-ALL-NEXT:    # in Loop: Header=BB1_1 Depth=1
+; X64-ALL-NEXT:    lock cmpxchgl %eax, (%rdi)
+; X64-ALL-NEXT:    jne .LBB1_1
+; X64-ALL-NEXT:  # %bb.4:
+; X64-ALL-NEXT:    xorl %eax, %eax
+; X64-ALL-NEXT:  .LBB1_5: # %cond.end
+; X64-ALL-NEXT:    retq
 entry:
   %cmp = icmp sgt i32 %i, %j
   br i1 %cmp, label %loop_start, label %cond.end
@@ -208,66 +208,66 @@ cond.end:
 ; This one is an interesting case because CMOV doesn't have a chain
 ; operand. Naive attempts to limit cmpxchg EFLAGS use are likely to fail here.
 define i32 @test_feed_cmov(ptr %addr, i32 %desired, i32 %new) nounwind {
-; 32-GOOD-RA-LABEL: test_feed_cmov:
-; 32-GOOD-RA:       # %bb.0: # %entry
-; 32-GOOD-RA-NEXT:    pushl %ebx
-; 32-GOOD-RA-NEXT:    pushl %esi
-; 32-GOOD-RA-NEXT:    pushl %eax
-; 32-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; 32-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; 32-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; 32-GOOD-RA-NEXT:    lock cmpxchgl %esi, (%ecx)
-; 32-GOOD-RA-NEXT:    sete %bl
-; 32-GOOD-RA-NEXT:    calll foo@PLT
-; 32-GOOD-RA-NEXT:    testb %bl, %bl
-; 32-GOOD-RA-NEXT:    jne .LBB2_2
-; 32-GOOD-RA-NEXT:  # %bb.1: # %entry
-; 32-GOOD-RA-NEXT:    movl %eax, %esi
-; 32-GOOD-RA-NEXT:  .LBB2_2: # %entry
-; 32-GOOD-RA-NEXT:    movl %esi, %eax
-; 32-GOOD-RA-NEXT:    addl $4, %esp
-; 32-GOOD-RA-NEXT:    popl %esi
-; 32-GOOD-RA-NEXT:    popl %ebx
-; 32-GOOD-RA-NEXT:    retl
+; X86-GOOD-RA-LABEL: test_feed_cmov:
+; X86-GOOD-RA:       # %bb.0: # %entry
+; X86-GOOD-RA-NEXT:    pushl %ebx
+; X86-GOOD-RA-NEXT:    pushl %esi
+; X86-GOOD-RA-NEXT:    pushl %eax
+; X86-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-GOOD-RA-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-GOOD-RA-NEXT:    lock cmpxchgl %esi, (%ecx)
+; X86-GOOD-RA-NEXT:    sete %bl
+; X86-GOOD-RA-NEXT:    calll foo@PLT
+; X86-GOOD-RA-NEXT:    testb %bl, %bl
+; X86-GOOD-RA-NEXT:    jne .LBB2_2
+; X86-GOOD-RA-NEXT:  # %bb.1: # %entry
+; X86-GOOD-RA-NEXT:    movl %eax, %esi
+; X86-GOOD-RA-NEXT:  .LBB2_2: # %entry
+; X86-GOOD-RA-NEXT:    movl %esi, %eax
+; X86-GOOD-RA-NEXT:    addl $4, %esp
+; X86-GOOD-RA-NEXT:    popl %esi
+; X86-GOOD-RA-NEXT:    popl %ebx
+; X86-GOOD-RA-NEXT:    retl
 ;
-; 32-FAST-RA-LABEL: test_feed_cmov:
-; 32-FAST-RA:       # %bb.0: # %entry
-; 32-FAST-RA-NEXT:    pushl %ebx
-; 32-FAST-RA-NEXT:    pushl %esi
-; 32-FAST-RA-NEXT:    pushl %eax
-; 32-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; 32-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; 32-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; 32-FAST-RA-NEXT:    lock cmpxchgl %esi, (%ecx)
-; 32-FAST-RA-NEXT:    sete %bl
-; 32-FAST-RA-NEXT:    calll foo@PLT
-; 32-FAST-RA-NEXT:    testb %bl, %bl
-; 32-FAST-RA-NEXT:    jne .LBB2_2
-; 32-FAST-RA-NEXT:  # %bb.1: # %entry
-; 32-FAST-RA-NEXT:    movl %eax, %esi
-; 32-FAST-RA-NEXT:  .LBB2_2: # %entry
-; 32-FAST-RA-NEXT:    movl %esi, %eax
-; 32-FAST-RA-NEXT:    addl $4, %esp
-; 32-FAST-RA-NEXT:    popl %esi
-; 32-FAST-RA-NEXT:    popl %ebx
-; 32-FAST-RA-NEXT:    retl
+; X86-FAST-RA-LABEL: test_feed_cmov:
+; X86-FAST-RA:       # %bb.0: # %entry
+; X86-FAST-RA-NEXT:    pushl %ebx
+; X86-FAST-RA-NEXT:    pushl %esi
+; X86-FAST-RA-NEXT:    pushl %eax
+; X86-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-FAST-RA-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-RA-NEXT:    lock cmpxchgl %esi, (%ecx)
+; X86-FAST-RA-NEXT:    sete %bl
+; X86-FAST-RA-NEXT:    calll foo@PLT
+; X86-FAST-RA-NEXT:    testb %bl, %bl
+; X86-FAST-RA-NEXT:    jne .LBB2_2
+; X86-FAST-RA-NEXT:  # %bb.1: # %entry
+; X86-FAST-RA-NEXT:    movl %eax, %esi
+; X86-FAST-RA-NEXT:  .LBB2_2: # %entry
+; X86-FAST-RA-NEXT:    movl %esi, %eax
+; X86-FAST-RA-NEXT:    addl $4, %esp
+; X86-FAST-RA-NEXT:    popl %esi
+; X86-FAST-RA-NEXT:    popl %ebx
+; X86-FAST-RA-NEXT:    retl
 ;
-; 64-ALL-LABEL: test_feed_cmov:
-; 64-ALL:       # %bb.0: # %entry
-; 64-ALL-NEXT:    pushq %rbp
-; 64-ALL-NEXT:    pushq %rbx
-; 64-ALL-NEXT:    pushq %rax
-; 64-ALL-NEXT:    movl %edx, %ebx
-; 64-ALL-NEXT:    movl %esi, %eax
-; 64-ALL-NEXT:    lock cmpxchgl %edx, (%rdi)
-; 64-ALL-NEXT:    sete %bpl
-; 64-ALL-NEXT:    callq foo@PLT
-; 64-ALL-NEXT:    testb %bpl, %bpl
-; 64-ALL-NEXT:    cmovnel %ebx, %eax
-; 64-ALL-NEXT:    addq $8, %rsp
-; 64-ALL-NEXT:    popq %rbx
-; 64-ALL-NEXT:    popq %rbp
-; 64-ALL-NEXT:    retq
+; X64-ALL-LABEL: test_feed_cmov:
+; X64-ALL:       # %bb.0: # %entry
+; X64-ALL-NEXT:    pushq %rbp
+; X64-ALL-NEXT:    pushq %rbx
+; X64-ALL-NEXT:    pushq %rax
+; X64-ALL-NEXT:    movl %edx, %ebx
+; X64-ALL-NEXT:    movl %esi, %eax
+; X64-ALL-NEXT:    lock cmpxchgl %edx, (%rdi)
+; X64-ALL-NEXT:    sete %bpl
+; X64-ALL-NEXT:    callq foo@PLT
+; X64-ALL-NEXT:    testb %bpl, %bpl
+; X64-ALL-NEXT:    cmovnel %ebx, %eax
+; X64-ALL-NEXT:    addq $8, %rsp
+; X64-ALL-NEXT:    popq %rbx
+; X64-ALL-NEXT:    popq %rbp
+; X64-ALL-NEXT:    retq
 entry:
   %res = cmpxchg ptr %addr, i32 %desired, i32 %new seq_cst seq_cst
   %success = extractvalue { i32, i1 } %res, 1
-- 
GitLab


From e26eacf771fed3226058a84d5d83f94994f583b2 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 15 May 2024 14:10:56 +0100
Subject: [PATCH 0848/1206] [X86] prefetch.ll - cleanup check prefixes
 identified in #92248

Avoid using leading numbers in check prefixes - replace with actual triple config names (and makes it easier to add X64 test coverage in a future commit).
---
 llvm/test/CodeGen/X86/prefetch.ll | 138 +++++++++++++++---------------
 1 file changed, 69 insertions(+), 69 deletions(-)

diff --git a/llvm/test/CodeGen/X86/prefetch.ll b/llvm/test/CodeGen/X86/prefetch.ll
index 3cfa0e3efcb1..404d49b63f25 100644
--- a/llvm/test/CodeGen/X86/prefetch.ll
+++ b/llvm/test/CodeGen/X86/prefetch.ll
@@ -1,16 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse | FileCheck %s --check-prefix=SSE
-; RUN: llc < %s -mtriple=i686-- -mattr=+avx | FileCheck %s --check-prefix=SSE
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse,+prfchw | FileCheck %s -check-prefix=PRFCHWSSE
-; RUN: llc < %s -mtriple=i686-- -mattr=+prfchw | FileCheck %s -check-prefix=PRFCHWSSE
-; RUN: llc < %s -mtriple=i686-- -mcpu=slm | FileCheck %s -check-prefix=PRFCHWSSE
-; RUN: llc < %s -mtriple=i686-- -mcpu=btver2 | FileCheck %s -check-prefix=PRFCHWSSE
-; RUN: llc < %s -mtriple=i686-- -mcpu=btver2 -mattr=-prfchw | FileCheck %s -check-prefix=SSE
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse,+prefetchwt1 | FileCheck %s -check-prefix=PREFETCHWT1
-; RUN: llc < %s -mtriple=i686-- -mattr=-sse,+prefetchwt1 | FileCheck %s -check-prefix=PREFETCHWT1
-; RUN: llc < %s -mtriple=i686-- -mattr=-sse,+3dnow,+prefetchwt1 | FileCheck %s -check-prefix=PREFETCHWT1
-; RUN: llc < %s -mtriple=i686-- -mattr=+3dnow | FileCheck %s -check-prefix=3DNOW
-; RUN: llc < %s -mtriple=i686-- -mattr=+3dnow,+prfchw | FileCheck %s -check-prefix=3DNOW
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse | FileCheck %s --check-prefix=X86-SSE
+; RUN: llc < %s -mtriple=i686-- -mattr=+avx | FileCheck %s --check-prefix=X86-SSE
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse,+prfchw | FileCheck %s -check-prefix=X86-PRFCHWSSE
+; RUN: llc < %s -mtriple=i686-- -mattr=+prfchw | FileCheck %s -check-prefix=X86-PRFCHWSSE
+; RUN: llc < %s -mtriple=i686-- -mcpu=slm | FileCheck %s -check-prefix=X86-PRFCHWSSE
+; RUN: llc < %s -mtriple=i686-- -mcpu=btver2 | FileCheck %s -check-prefix=X86-PRFCHWSSE
+; RUN: llc < %s -mtriple=i686-- -mcpu=btver2 -mattr=-prfchw | FileCheck %s -check-prefix=X86-SSE
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse,+prefetchwt1 | FileCheck %s -check-prefix=X86-PREFETCHWT1
+; RUN: llc < %s -mtriple=i686-- -mattr=-sse,+prefetchwt1 | FileCheck %s -check-prefix=X86-PREFETCHWT1
+; RUN: llc < %s -mtriple=i686-- -mattr=-sse,+3dnow,+prefetchwt1 | FileCheck %s -check-prefix=X86-PREFETCHWT1
+; RUN: llc < %s -mtriple=i686-- -mattr=+3dnow | FileCheck %s -check-prefix=X86-3DNOW
+; RUN: llc < %s -mtriple=i686-- -mattr=+3dnow,+prfchw | FileCheck %s -check-prefix=X86-3DNOW
 
 ; Rules:
 ; 3dnow by itself get you just the single prefetch instruction with no hints
@@ -22,67 +22,67 @@
 ; rdar://10538297
 
 define void @t(ptr %ptr) nounwind  {
-; SSE-LABEL: t:
-; SSE:       # %bb.0: # %entry
-; SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SSE-NEXT:    prefetcht2 (%eax)
-; SSE-NEXT:    prefetcht1 (%eax)
-; SSE-NEXT:    prefetcht0 (%eax)
-; SSE-NEXT:    prefetchnta (%eax)
-; SSE-NEXT:    prefetcht2 (%eax)
-; SSE-NEXT:    prefetcht1 (%eax)
-; SSE-NEXT:    prefetcht0 (%eax)
-; SSE-NEXT:    prefetchnta (%eax)
-; SSE-NEXT:    retl
+; X86-SSE-LABEL: t:
+; X86-SSE:       # %bb.0: # %entry
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    prefetcht2 (%eax)
+; X86-SSE-NEXT:    prefetcht1 (%eax)
+; X86-SSE-NEXT:    prefetcht0 (%eax)
+; X86-SSE-NEXT:    prefetchnta (%eax)
+; X86-SSE-NEXT:    prefetcht2 (%eax)
+; X86-SSE-NEXT:    prefetcht1 (%eax)
+; X86-SSE-NEXT:    prefetcht0 (%eax)
+; X86-SSE-NEXT:    prefetchnta (%eax)
+; X86-SSE-NEXT:    retl
 ;
-; PRFCHWSSE-LABEL: t:
-; PRFCHWSSE:       # %bb.0: # %entry
-; PRFCHWSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; PRFCHWSSE-NEXT:    prefetcht2 (%eax)
-; PRFCHWSSE-NEXT:    prefetcht1 (%eax)
-; PRFCHWSSE-NEXT:    prefetcht0 (%eax)
-; PRFCHWSSE-NEXT:    prefetchnta (%eax)
-; PRFCHWSSE-NEXT:    prefetchw (%eax)
-; PRFCHWSSE-NEXT:    prefetchw (%eax)
-; PRFCHWSSE-NEXT:    prefetchw (%eax)
-; PRFCHWSSE-NEXT:    prefetchw (%eax)
-; PRFCHWSSE-NEXT:    retl
+; X86-PRFCHWSSE-LABEL: t:
+; X86-PRFCHWSSE:       # %bb.0: # %entry
+; X86-PRFCHWSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-PRFCHWSSE-NEXT:    prefetcht2 (%eax)
+; X86-PRFCHWSSE-NEXT:    prefetcht1 (%eax)
+; X86-PRFCHWSSE-NEXT:    prefetcht0 (%eax)
+; X86-PRFCHWSSE-NEXT:    prefetchnta (%eax)
+; X86-PRFCHWSSE-NEXT:    prefetchw (%eax)
+; X86-PRFCHWSSE-NEXT:    prefetchw (%eax)
+; X86-PRFCHWSSE-NEXT:    prefetchw (%eax)
+; X86-PRFCHWSSE-NEXT:    prefetchw (%eax)
+; X86-PRFCHWSSE-NEXT:    retl
 ;
-; PREFETCHWT1-LABEL: t:
-; PREFETCHWT1:       # %bb.0: # %entry
-; PREFETCHWT1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; PREFETCHWT1-NEXT:    prefetcht2 (%eax)
-; PREFETCHWT1-NEXT:    prefetcht1 (%eax)
-; PREFETCHWT1-NEXT:    prefetcht0 (%eax)
-; PREFETCHWT1-NEXT:    prefetchnta (%eax)
-; PREFETCHWT1-NEXT:    prefetchwt1 (%eax)
-; PREFETCHWT1-NEXT:    prefetchwt1 (%eax)
-; PREFETCHWT1-NEXT:    prefetchw (%eax)
-; PREFETCHWT1-NEXT:    prefetchwt1 (%eax)
-; PREFETCHWT1-NEXT:    retl
+; X86-PREFETCHWT1-LABEL: t:
+; X86-PREFETCHWT1:       # %bb.0: # %entry
+; X86-PREFETCHWT1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-PREFETCHWT1-NEXT:    prefetcht2 (%eax)
+; X86-PREFETCHWT1-NEXT:    prefetcht1 (%eax)
+; X86-PREFETCHWT1-NEXT:    prefetcht0 (%eax)
+; X86-PREFETCHWT1-NEXT:    prefetchnta (%eax)
+; X86-PREFETCHWT1-NEXT:    prefetchwt1 (%eax)
+; X86-PREFETCHWT1-NEXT:    prefetchwt1 (%eax)
+; X86-PREFETCHWT1-NEXT:    prefetchw (%eax)
+; X86-PREFETCHWT1-NEXT:    prefetchwt1 (%eax)
+; X86-PREFETCHWT1-NEXT:    retl
 ;
-; 3DNOW-LABEL: t:
-; 3DNOW:       # %bb.0: # %entry
-; 3DNOW-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; 3DNOW-NEXT:    prefetch (%eax)
-; 3DNOW-NEXT:    prefetch (%eax)
-; 3DNOW-NEXT:    prefetch (%eax)
-; 3DNOW-NEXT:    prefetch (%eax)
-; 3DNOW-NEXT:    prefetchw (%eax)
-; 3DNOW-NEXT:    prefetchw (%eax)
-; 3DNOW-NEXT:    prefetchw (%eax)
-; 3DNOW-NEXT:    prefetchw (%eax)
-; 3DNOW-NEXT:    retl
+; X86-3DNOW-LABEL: t:
+; X86-3DNOW:       # %bb.0: # %entry
+; X86-3DNOW-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-3DNOW-NEXT:    prefetch (%eax)
+; X86-3DNOW-NEXT:    prefetch (%eax)
+; X86-3DNOW-NEXT:    prefetch (%eax)
+; X86-3DNOW-NEXT:    prefetch (%eax)
+; X86-3DNOW-NEXT:    prefetchw (%eax)
+; X86-3DNOW-NEXT:    prefetchw (%eax)
+; X86-3DNOW-NEXT:    prefetchw (%eax)
+; X86-3DNOW-NEXT:    prefetchw (%eax)
+; X86-3DNOW-NEXT:    retl
 entry:
-	tail call void @llvm.prefetch( ptr %ptr, i32 0, i32 1, i32 1 )
-	tail call void @llvm.prefetch( ptr %ptr, i32 0, i32 2, i32 1 )
-	tail call void @llvm.prefetch( ptr %ptr, i32 0, i32 3, i32 1 )
-	tail call void @llvm.prefetch( ptr %ptr, i32 0, i32 0, i32 1 )
-	tail call void @llvm.prefetch( ptr %ptr, i32 1, i32 1, i32 1 )
-	tail call void @llvm.prefetch( ptr %ptr, i32 1, i32 2, i32 1 )
-	tail call void @llvm.prefetch( ptr %ptr, i32 1, i32 3, i32 1 )
-	tail call void @llvm.prefetch( ptr %ptr, i32 1, i32 0, i32 1 )
-	ret void
+  tail call void @llvm.prefetch( ptr %ptr, i32 0, i32 1, i32 1 )
+  tail call void @llvm.prefetch( ptr %ptr, i32 0, i32 2, i32 1 )
+  tail call void @llvm.prefetch( ptr %ptr, i32 0, i32 3, i32 1 )
+  tail call void @llvm.prefetch( ptr %ptr, i32 0, i32 0, i32 1 )
+  tail call void @llvm.prefetch( ptr %ptr, i32 1, i32 1, i32 1 )
+  tail call void @llvm.prefetch( ptr %ptr, i32 1, i32 2, i32 1 )
+  tail call void @llvm.prefetch( ptr %ptr, i32 1, i32 3, i32 1 )
+  tail call void @llvm.prefetch( ptr %ptr, i32 1, i32 0, i32 1 )
+  ret void
 }
 
 declare void @llvm.prefetch(ptr, i32, i32, i32) nounwind
-- 
GitLab


From 3f07430c383dffad77a120c91df79cbc7d99313c Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 15 May 2024 14:23:46 +0100
Subject: [PATCH 0849/1206] [X86] avoid-sfb-g-no-change.mir - cleanup check
 prefixes identified in #92248

Don't include "-LABEL" (or any other FileCheck modifier) in the core check prefix name
---
 llvm/test/CodeGen/X86/avoid-sfb-g-no-change.mir | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/test/CodeGen/X86/avoid-sfb-g-no-change.mir b/llvm/test/CodeGen/X86/avoid-sfb-g-no-change.mir
index 679a908893f6..93292990dee0 100644
--- a/llvm/test/CodeGen/X86/avoid-sfb-g-no-change.mir
+++ b/llvm/test/CodeGen/X86/avoid-sfb-g-no-change.mir
@@ -1,5 +1,5 @@
-# RUN: llc %s -run-pass x86-avoid-SFB -mtriple=x86_64-unknown-linux-gnu -o - | FileCheck %s -check-prefixes DEBUG-LABEL,CHECK
-# RUN: llc %s -run-pass x86-avoid-SFB -mtriple=x86_64-unknown-linux-gnu -o - | FileCheck %s -check-prefixes NODEBUG-LABEL,CHECK
+# RUN: llc %s -run-pass x86-avoid-SFB -mtriple=x86_64-unknown-linux-gnu -o - | FileCheck %s -check-prefixes=CHECK,DEBUG
+# RUN: llc %s -run-pass x86-avoid-SFB -mtriple=x86_64-unknown-linux-gnu -o - | FileCheck %s -check-prefixes=CHECK,NODEBUG
 #
 # This was generated from:
 #
@@ -202,8 +202,8 @@ body:             |
     MOVAPSmr %1, 1, $noreg, 0, $noreg, killed %2 :: (store (s128) into %ir.p2)
     RET 0
 
-    ; DEBUG-LABEL: name: debug
-    ; NODEBUG-LABEL: name: nodebug
+    ; DEBUG: name: debug
+    ; NODEBUG: name: nodebug
     ; CHECK: %1:gr64 = COPY
     ; CHECK: %0:gr64 = COPY
     ; CHECK: MOV8mi
@@ -218,5 +218,5 @@ body:             |
     ; CHECK: %7:gr8 = MOV8rm
     ; CHECK: MOV8mr
     ; CHECK: RET 0
-    ; DEBUG-LABEL: name: nodebug
+    ; DEBUG: name: nodebug
 ...
-- 
GitLab


From f8395f8420cee8fc0854f43c9e88819c0ed54696 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 15 May 2024 14:25:29 +0100
Subject: [PATCH 0850/1206] [X86] Cleanup check prefixes identified in #92248

Avoid using leading numbers in check prefixes - replace with actual triple config names.
---
 .../align-branch-boundary-suppressions-tls.ll |  22 +--
 llvm/test/CodeGen/X86/asm-modifier.ll         |  30 +--
 llvm/test/CodeGen/X86/pr32345.ll              | 180 +++++++++---------
 llvm/test/CodeGen/X86/x32-va_start.ll         |  46 ++---
 4 files changed, 139 insertions(+), 139 deletions(-)

diff --git a/llvm/test/CodeGen/X86/align-branch-boundary-suppressions-tls.ll b/llvm/test/CodeGen/X86/align-branch-boundary-suppressions-tls.ll
index fd58cfeb65fd..0e2e6f3ef81d 100644
--- a/llvm/test/CodeGen/X86/align-branch-boundary-suppressions-tls.ll
+++ b/llvm/test/CodeGen/X86/align-branch-boundary-suppressions-tls.ll
@@ -2,8 +2,8 @@
 ;; sequence. It uses prefixes to allow linker relaxation. We need to disable
 ;; prefix or nop padding for it. For simplicity and consistency, disable for
 ;; Local Dynamic and 32-bit as well.
-; RUN: llc -mtriple=i386 -relocation-model=pic -x86-branches-within-32B-boundaries < %s | FileCheck --check-prefixes=CHECK,32 %s
-; RUN: llc -mtriple=x86_64 -relocation-model=pic -x86-branches-within-32B-boundaries < %s | FileCheck --check-prefixes=CHECK,64 %s
+; RUN: llc -mtriple=i386 -relocation-model=pic -x86-branches-within-32B-boundaries < %s | FileCheck --check-prefixes=CHECK,X86 %s
+; RUN: llc -mtriple=x86_64 -relocation-model=pic -x86-branches-within-32B-boundaries < %s | FileCheck --check-prefixes=CHECK,X64 %s
 
 @gd = external thread_local global i32
 @ld = internal thread_local global i32 0
@@ -11,17 +11,17 @@
 define i32 @tls_get_addr() {
 ; CHECK-LABEL: tls_get_addr:
 ; CHECK: #noautopadding
-; 32: leal gd@TLSGD(,%ebx), %eax
-; 32: calll ___tls_get_addr@PLT
-; 64: data16
-; 64: leaq gd@TLSGD(%rip), %rdi
-; 64: callq __tls_get_addr@PLT
+; X86: leal gd@TLSGD(,%ebx), %eax
+; X86: calll ___tls_get_addr@PLT
+; X64: data16
+; X64: leaq gd@TLSGD(%rip), %rdi
+; X64: callq __tls_get_addr@PLT
 ; CHECK: #autopadding
 ; CHECK: #noautopadding
-; 32: leal ld@TLSLDM(%ebx), %eax
-; 32: calll ___tls_get_addr@PLT
-; 64: leaq ld@TLSLD(%rip), %rdi
-; 64: callq __tls_get_addr@PLT
+; X86: leal ld@TLSLDM(%ebx), %eax
+; X86: calll ___tls_get_addr@PLT
+; X64: leaq ld@TLSLD(%rip), %rdi
+; X64: callq __tls_get_addr@PLT
 ; CHECK: #autopadding
   %1 = load i32, ptr @gd
   %2 = load i32, ptr @ld
diff --git a/llvm/test/CodeGen/X86/asm-modifier.ll b/llvm/test/CodeGen/X86/asm-modifier.ll
index c121b46f8450..9a69402d2216 100644
--- a/llvm/test/CodeGen/X86/asm-modifier.ll
+++ b/llvm/test/CodeGen/X86/asm-modifier.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=i686 < %s | FileCheck %s --check-prefixes=CHECK,32
-; RUN: llc -mtriple=x86_64 < %s | FileCheck %s --check-prefixes=CHECK,64
+; RUN: llc -mtriple=i686 < %s | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc -mtriple=x86_64 < %s | FileCheck %s --check-prefixes=CHECK,X64
 
 @var = internal global i32 0, align 4
 
@@ -43,20 +43,20 @@ entry:
 }
 
 define void @test_V(ptr %p) {
-; 32-LABEL: test_V:
-; 32:       # %bb.0: # %entry
-; 32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; 32-NEXT:    #APP
-; 32-NEXT:    calll __x86_indirect_thunk_eax
-; 32-NEXT:    #NO_APP
-; 32-NEXT:    retl
+; X86-LABEL: test_V:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    #APP
+; X86-NEXT:    calll __x86_indirect_thunk_eax
+; X86-NEXT:    #NO_APP
+; X86-NEXT:    retl
 ;
-; 64-LABEL: test_V:
-; 64:       # %bb.0: # %entry
-; 64-NEXT:    #APP
-; 64-NEXT:    callq __x86_indirect_thunk_rdi
-; 64-NEXT:    #NO_APP
-; 64-NEXT:    retq
+; X64-LABEL: test_V:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    #APP
+; X64-NEXT:    callq __x86_indirect_thunk_rdi
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    retq
 entry:
   tail call void asm sideeffect "call __x86_indirect_thunk_${0:V}", "r,~{dirflag},~{fpsr},~{flags}"(ptr %p)
   ret void
diff --git a/llvm/test/CodeGen/X86/pr32345.ll b/llvm/test/CodeGen/X86/pr32345.ll
index 2745cb8bb908..c7405e982660 100644
--- a/llvm/test/CodeGen/X86/pr32345.ll
+++ b/llvm/test/CodeGen/X86/pr32345.ll
@@ -1,74 +1,74 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -O0 -mtriple=x86_64-unknown-linux-gnu -o - %s | FileCheck %s -check-prefix=X640
-; RUN: llc -O0 -mtriple=i686-unknown             -o - %s | FileCheck %s -check-prefix=6860
-; RUN: llc     -mtriple=x86_64-unknown-linux-gnu -o - %s | FileCheck %s -check-prefix=X64
-; RUN: llc     -mtriple=i686-unknown             -o - %s | FileCheck %s -check-prefix=686
+; RUN: llc -O0 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s -check-prefix=X64-O0
+; RUN: llc -O0 -mtriple=i686-unknown             < %s | FileCheck %s -check-prefix=X86-O0
+; RUN: llc     -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s -check-prefix=X64
+; RUN: llc     -mtriple=i686-unknown             < %s | FileCheck %s -check-prefix=X86
 
 @var_22 = external dso_local global i16, align 2
 @var_27 = external dso_local global i16, align 2
 
 define void @foo() {
-; X640-LABEL: foo:
-; X640:       # %bb.0: # %bb
-; X640-NEXT:    movzwl var_22, %eax
-; X640-NEXT:    movzwl var_27, %ecx
-; X640-NEXT:    xorl %ecx, %eax
-; X640-NEXT:    movzwl var_27, %ecx
-; X640-NEXT:    xorl %ecx, %eax
-; X640-NEXT:    cltq
-; X640-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; X640-NEXT:    movzwl var_22, %eax
-; X640-NEXT:    movzwl var_27, %ecx
-; X640-NEXT:    xorl %ecx, %eax
-; X640-NEXT:    movzwl var_27, %ecx
-; X640-NEXT:    xorl %ecx, %eax
-; X640-NEXT:    cltq
-; X640-NEXT:    movzwl var_27, %ecx
-; X640-NEXT:    subl $16610, %ecx # imm = 0x40E2
-; X640-NEXT:    movl %ecx, %ecx
-; X640-NEXT:    # kill: def $rcx killed $ecx
-; X640-NEXT:    # kill: def $cl killed $rcx
-; X640-NEXT:    sarq %cl, %rax
-; X640-NEXT:    movb %al, %cl
-; X640-NEXT:    # implicit-def: $rax
-; X640-NEXT:    movb %cl, (%rax)
-; X640-NEXT:    retq
+; X64-O0-LABEL: foo:
+; X64-O0:       # %bb.0: # %bb
+; X64-O0-NEXT:    movzwl var_22, %eax
+; X64-O0-NEXT:    movzwl var_27, %ecx
+; X64-O0-NEXT:    xorl %ecx, %eax
+; X64-O0-NEXT:    movzwl var_27, %ecx
+; X64-O0-NEXT:    xorl %ecx, %eax
+; X64-O0-NEXT:    cltq
+; X64-O0-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-O0-NEXT:    movzwl var_22, %eax
+; X64-O0-NEXT:    movzwl var_27, %ecx
+; X64-O0-NEXT:    xorl %ecx, %eax
+; X64-O0-NEXT:    movzwl var_27, %ecx
+; X64-O0-NEXT:    xorl %ecx, %eax
+; X64-O0-NEXT:    cltq
+; X64-O0-NEXT:    movzwl var_27, %ecx
+; X64-O0-NEXT:    subl $16610, %ecx # imm = 0x40E2
+; X64-O0-NEXT:    movl %ecx, %ecx
+; X64-O0-NEXT:    # kill: def $rcx killed $ecx
+; X64-O0-NEXT:    # kill: def $cl killed $rcx
+; X64-O0-NEXT:    sarq %cl, %rax
+; X64-O0-NEXT:    movb %al, %cl
+; X64-O0-NEXT:    # implicit-def: $rax
+; X64-O0-NEXT:    movb %cl, (%rax)
+; X64-O0-NEXT:    retq
 ;
-; 6860-LABEL: foo:
-; 6860:       # %bb.0: # %bb
-; 6860-NEXT:    pushl %ebp
-; 6860-NEXT:    .cfi_def_cfa_offset 8
-; 6860-NEXT:    .cfi_offset %ebp, -8
-; 6860-NEXT:    movl %esp, %ebp
-; 6860-NEXT:    .cfi_def_cfa_register %ebp
-; 6860-NEXT:    andl $-8, %esp
-; 6860-NEXT:    subl $24, %esp
-; 6860-NEXT:    movzwl var_22, %eax
-; 6860-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; 6860-NEXT:    movl $0, {{[0-9]+}}(%esp)
-; 6860-NEXT:    movzwl var_22, %edx
-; 6860-NEXT:    movb var_27, %cl
-; 6860-NEXT:    addb $30, %cl
-; 6860-NEXT:    movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; 6860-NEXT:    xorl %eax, %eax
-; 6860-NEXT:    shrdl %cl, %eax, %edx
-; 6860-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
-; 6860-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; 6860-NEXT:    testb $32, %cl
-; 6860-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; 6860-NEXT:    jne .LBB0_2
-; 6860-NEXT:  # %bb.1: # %bb
-; 6860-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; 6860-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; 6860-NEXT:  .LBB0_2: # %bb
-; 6860-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; 6860-NEXT:    movb %al, %cl
-; 6860-NEXT:    # implicit-def: $eax
-; 6860-NEXT:    movb %cl, (%eax)
-; 6860-NEXT:    movl %ebp, %esp
-; 6860-NEXT:    popl %ebp
-; 6860-NEXT:    .cfi_def_cfa %esp, 4
-; 6860-NEXT:    retl
+; X86-O0-LABEL: foo:
+; X86-O0:       # %bb.0: # %bb
+; X86-O0-NEXT:    pushl %ebp
+; X86-O0-NEXT:    .cfi_def_cfa_offset 8
+; X86-O0-NEXT:    .cfi_offset %ebp, -8
+; X86-O0-NEXT:    movl %esp, %ebp
+; X86-O0-NEXT:    .cfi_def_cfa_register %ebp
+; X86-O0-NEXT:    andl $-8, %esp
+; X86-O0-NEXT:    subl $24, %esp
+; X86-O0-NEXT:    movzwl var_22, %eax
+; X86-O0-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-O0-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; X86-O0-NEXT:    movzwl var_22, %edx
+; X86-O0-NEXT:    movb var_27, %cl
+; X86-O0-NEXT:    addb $30, %cl
+; X86-O0-NEXT:    movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-O0-NEXT:    xorl %eax, %eax
+; X86-O0-NEXT:    shrdl %cl, %eax, %edx
+; X86-O0-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; X86-O0-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-O0-NEXT:    testb $32, %cl
+; X86-O0-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-O0-NEXT:    jne .LBB0_2
+; X86-O0-NEXT:  # %bb.1: # %bb
+; X86-O0-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-O0-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-O0-NEXT:  .LBB0_2: # %bb
+; X86-O0-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-O0-NEXT:    movb %al, %cl
+; X86-O0-NEXT:    # implicit-def: $eax
+; X86-O0-NEXT:    movb %cl, (%eax)
+; X86-O0-NEXT:    movl %ebp, %esp
+; X86-O0-NEXT:    popl %ebp
+; X86-O0-NEXT:    .cfi_def_cfa %esp, 4
+; X86-O0-NEXT:    retl
 ;
 ; X64-LABEL: foo:
 ; X64:       # %bb.0: # %bb
@@ -80,32 +80,32 @@ define void @foo() {
 ; X64-NEXT:    movb %al, (%rax)
 ; X64-NEXT:    retq
 ;
-; 686-LABEL: foo:
-; 686:       # %bb.0: # %bb
-; 686-NEXT:    pushl %ebp
-; 686-NEXT:    .cfi_def_cfa_offset 8
-; 686-NEXT:    .cfi_offset %ebp, -8
-; 686-NEXT:    movl %esp, %ebp
-; 686-NEXT:    .cfi_def_cfa_register %ebp
-; 686-NEXT:    andl $-8, %esp
-; 686-NEXT:    subl $8, %esp
-; 686-NEXT:    movzbl var_27, %ecx
-; 686-NEXT:    movzwl var_22, %eax
-; 686-NEXT:    movl %eax, (%esp)
-; 686-NEXT:    movl $0, {{[0-9]+}}(%esp)
-; 686-NEXT:    addb $30, %cl
-; 686-NEXT:    xorl %edx, %edx
-; 686-NEXT:    shrdl %cl, %edx, %eax
-; 686-NEXT:    testb $32, %cl
-; 686-NEXT:    jne .LBB0_2
-; 686-NEXT:  # %bb.1: # %bb
-; 686-NEXT:    movl %eax, %edx
-; 686-NEXT:  .LBB0_2: # %bb
-; 686-NEXT:    movb %dl, (%eax)
-; 686-NEXT:    movl %ebp, %esp
-; 686-NEXT:    popl %ebp
-; 686-NEXT:    .cfi_def_cfa %esp, 4
-; 686-NEXT:    retl
+; X86-LABEL: foo:
+; X86:       # %bb.0: # %bb
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    .cfi_def_cfa_register %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movzbl var_27, %ecx
+; X86-NEXT:    movzwl var_22, %eax
+; X86-NEXT:    movl %eax, (%esp)
+; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    addb $30, %cl
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    shrdl %cl, %edx, %eax
+; X86-NEXT:    testb $32, %cl
+; X86-NEXT:    jne .LBB0_2
+; X86-NEXT:  # %bb.1: # %bb
+; X86-NEXT:    movl %eax, %edx
+; X86-NEXT:  .LBB0_2: # %bb
+; X86-NEXT:    movb %dl, (%eax)
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    .cfi_def_cfa %esp, 4
+; X86-NEXT:    retl
 bb:
   %tmp = alloca i64, align 8
   %tmp1 = load i16, ptr @var_22, align 2
diff --git a/llvm/test/CodeGen/X86/x32-va_start.ll b/llvm/test/CodeGen/X86/x32-va_start.ll
index e61e5765f124..31c8aee3fdde 100644
--- a/llvm/test/CodeGen/X86/x32-va_start.ll
+++ b/llvm/test/CodeGen/X86/x32-va_start.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-linux-gnux32 | FileCheck %s -check-prefix=SSE
 ; RUN: llc < %s -mtriple=x86_64-linux-gnux32 -mattr=-sse | FileCheck %s -check-prefix=NOSSE
-; RUN: llc < %s -mtriple=i386-linux-gnux32 | FileCheck %s -check-prefix=32BITABI
-; RUN: llc < %s -mtriple=i686-linux-gnux32 | FileCheck %s -check-prefix=32BITABI
+; RUN: llc < %s -mtriple=i386-linux-gnux32 | FileCheck %s -check-prefix=X32BITABI
+; RUN: llc < %s -mtriple=i686-linux-gnux32 | FileCheck %s -check-prefix=X32BITABI
 ;
 ; Verifies that x32 va_start lowering is sane. To regenerate this test, use
 ; cat <<EOF |
@@ -97,27 +97,27 @@ define i32 @foo(float %a, ptr nocapture readnone %fmt, ...) nounwind {
 ; NOSSE-NEXT:    movl (%eax), %eax
 ; NOSSE-NEXT:    retq
 ;
-; 32BITABI-LABEL: foo:
-; 32BITABI:       # %bb.0: # %entry
-; 32BITABI-NEXT:    subl $28, %esp
-; 32BITABI-NEXT:    leal {{[0-9]+}}(%esp), %ecx
-; 32BITABI-NEXT:    movl %ecx, (%esp)
-; 32BITABI-NEXT:    cmpl $40, %ecx
-; 32BITABI-NEXT:    ja .LBB0_2
-; 32BITABI-NEXT:  # %bb.1: # %vaarg.in_reg
-; 32BITABI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; 32BITABI-NEXT:    addl %ecx, %eax
-; 32BITABI-NEXT:    addl $8, %ecx
-; 32BITABI-NEXT:    movl %ecx, (%esp)
-; 32BITABI-NEXT:    jmp .LBB0_3
-; 32BITABI-NEXT:  .LBB0_2: # %vaarg.in_mem
-; 32BITABI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; 32BITABI-NEXT:    leal 8(%eax), %ecx
-; 32BITABI-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; 32BITABI-NEXT:  .LBB0_3: # %vaarg.end
-; 32BITABI-NEXT:    movl (%eax), %eax
-; 32BITABI-NEXT:    addl $28, %esp
-; 32BITABI-NEXT:    retl
+; X32BITABI-LABEL: foo:
+; X32BITABI:       # %bb.0: # %entry
+; X32BITABI-NEXT:    subl $28, %esp
+; X32BITABI-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; X32BITABI-NEXT:    movl %ecx, (%esp)
+; X32BITABI-NEXT:    cmpl $40, %ecx
+; X32BITABI-NEXT:    ja .LBB0_2
+; X32BITABI-NEXT:  # %bb.1: # %vaarg.in_reg
+; X32BITABI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32BITABI-NEXT:    addl %ecx, %eax
+; X32BITABI-NEXT:    addl $8, %ecx
+; X32BITABI-NEXT:    movl %ecx, (%esp)
+; X32BITABI-NEXT:    jmp .LBB0_3
+; X32BITABI-NEXT:  .LBB0_2: # %vaarg.in_mem
+; X32BITABI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32BITABI-NEXT:    leal 8(%eax), %ecx
+; X32BITABI-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X32BITABI-NEXT:  .LBB0_3: # %vaarg.end
+; X32BITABI-NEXT:    movl (%eax), %eax
+; X32BITABI-NEXT:    addl $28, %esp
+; X32BITABI-NEXT:    retl
 entry:
   %ap = alloca [1 x %struct.__va_list_tag], align 16
   call void @llvm.lifetime.start.p0(i64 16, ptr %ap) #2
-- 
GitLab


From b59760d83d10c27f22d77b7df24871500167c3e8 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamleshbhalui@gmail.com>
Date: Wed, 15 May 2024 18:57:49 +0530
Subject: [PATCH 0851/1206] Revert "[ExceptionDemo] Correct and update example
 ExceptionDemo" (#92257)

Reverts llvm/llvm-project#69485
---
 llvm/examples/ExceptionDemo/CMakeLists.txt    |   4 +-
 llvm/examples/ExceptionDemo/ExceptionDemo.cpp | 181 ++++++++++--------
 2 files changed, 108 insertions(+), 77 deletions(-)

diff --git a/llvm/examples/ExceptionDemo/CMakeLists.txt b/llvm/examples/ExceptionDemo/CMakeLists.txt
index 0a60ad848dd4..793cf291ca6f 100644
--- a/llvm/examples/ExceptionDemo/CMakeLists.txt
+++ b/llvm/examples/ExceptionDemo/CMakeLists.txt
@@ -1,7 +1,9 @@
 set(LLVM_LINK_COMPONENTS
   Core
   ExecutionEngine
-  ORCJIT
+  MC
+  MCJIT
+  RuntimeDyld
   Support
   Target
   nativecodegen
diff --git a/llvm/examples/ExceptionDemo/ExceptionDemo.cpp b/llvm/examples/ExceptionDemo/ExceptionDemo.cpp
index 41fa0cf626bf..0afc6b30d140 100644
--- a/llvm/examples/ExceptionDemo/ExceptionDemo.cpp
+++ b/llvm/examples/ExceptionDemo/ExceptionDemo.cpp
@@ -49,9 +49,8 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/BinaryFormat/Dwarf.h"
-#include "llvm/ExecutionEngine/Orc/Core.h"
-#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
-#include "llvm/ExecutionEngine/Orc/LLJIT.h"
+#include "llvm/ExecutionEngine/MCJIT.h"
+#include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/IRBuilder.h"
@@ -85,8 +84,6 @@
 #define USE_GLOBAL_STR_CONSTS true
 #endif
 
-llvm::ExitOnError ExitOnErr;
-
 //
 // Example types
 //
@@ -145,7 +142,6 @@ static llvm::ConstantInt *ourExceptionCaughtState;
 
 typedef std::vector<std::string> ArgNames;
 typedef std::vector<llvm::Type*> ArgTypes;
-typedef llvm::ArrayRef<llvm::Type *> TypeArray;
 
 //
 // Code Generation Utilities
@@ -896,10 +892,13 @@ void generateStringPrint(llvm::LLVMContext &context,
 ///        generated, and is used to hold the constant string. A value of
 ///        false indicates that the constant string will be stored on the
 ///        stack.
-void generateIntegerPrint(llvm::LLVMContext &context, llvm::Module &module,
+void generateIntegerPrint(llvm::LLVMContext &context,
+                          llvm::Module &module,
                           llvm::IRBuilder<> &builder,
-                          llvm::Function &printFunct, llvm::Value *toPrint,
-                          std::string format, bool useGlobal = true) {
+                          llvm::Function &printFunct,
+                          llvm::Value &toPrint,
+                          std::string format,
+                          bool useGlobal = true) {
   llvm::Constant *stringConstant =
     llvm::ConstantDataArray::getString(context, format);
   llvm::Value *stringVar;
@@ -921,9 +920,10 @@ void generateIntegerPrint(llvm::LLVMContext &context, llvm::Module &module,
 
   llvm::Value *cast = builder.CreateBitCast(stringVar,
                                             builder.getPtrTy());
-  builder.CreateCall(&printFunct, {toPrint, cast});
+  builder.CreateCall(&printFunct, {&toPrint, cast});
 }
 
+
 /// Generates code to handle finally block type semantics: always runs
 /// regardless of whether a thrown exception is passing through or the
 /// parent function is simply exiting. In addition to printing some state
@@ -997,10 +997,10 @@ static llvm::BasicBlock *createFinallyBlock(llvm::LLVMContext &context,
                       bufferToPrint.str(),
                       USE_GLOBAL_STR_CONSTS);
 
-  llvm::SwitchInst *theSwitch = builder.CreateSwitch(
-      builder.CreateLoad(ourExceptionNotThrownState->getType(),
-                         *exceptionCaughtFlag),
-      &terminatorBlock, 2);
+  llvm::SwitchInst *theSwitch = builder.CreateSwitch(builder.CreateLoad(
+                                                       *exceptionCaughtFlag),
+                                                     &terminatorBlock,
+                                                     2);
   theSwitch->addCase(ourExceptionCaughtState, &terminatorBlock);
   theSwitch->addCase(ourExceptionThrownState, &unwindResumeBlock);
 
@@ -1186,7 +1186,7 @@ static llvm::Function *createCatchWrappedInvokeFunction(
 
   // Note: function handles NULL exceptions
   builder.CreateCall(deleteOurException,
-                     builder.CreateLoad(builder.getPtrTy(), exceptionStorage));
+                     builder.CreateLoad(exceptionStorage));
   builder.CreateRetVoid();
 
   // Normal Block
@@ -1206,8 +1206,7 @@ static llvm::Function *createCatchWrappedInvokeFunction(
 
   builder.SetInsertPoint(unwindResumeBlock);
 
-  builder.CreateResume(
-      builder.CreateLoad(ourCaughtResultType, caughtResultStorage));
+  builder.CreateResume(builder.CreateLoad(caughtResultStorage));
 
   // Exception Block
 
@@ -1242,9 +1241,8 @@ static llvm::Function *createCatchWrappedInvokeFunction(
   // Retrieve exception_class member from thrown exception
   // (_Unwind_Exception instance). This member tells us whether or not
   // the exception is foreign.
-  llvm::Value *unwindExceptionClass = builder.CreateLoad(
-      builder.getInt64Ty(),
-      builder.CreateStructGEP(
+  llvm::Value *unwindExceptionClass =
+      builder.CreateLoad(builder.CreateStructGEP(
           ourUnwindExceptionType,
           builder.CreatePointerCast(unwindException,
                                     ourUnwindExceptionType->getPointerTo()),
@@ -1280,9 +1278,9 @@ static llvm::Function *createCatchWrappedInvokeFunction(
   //
   // Note: ourBaseFromUnwindOffset is usually negative
   llvm::Value *typeInfoThrown = builder.CreatePointerCast(
-      builder.CreateConstGEP1_64(builder.getPtrTy(), unwindException,
-                                 ourBaseFromUnwindOffset),
-      ourExceptionType->getPointerTo());
+                                  builder.CreateConstGEP1_64(unwindException,
+                                                       ourBaseFromUnwindOffset),
+                                  ourExceptionType->getPointerTo());
 
   // Retrieve thrown exception type info type
   //
@@ -1291,15 +1289,17 @@ static llvm::Function *createCatchWrappedInvokeFunction(
   typeInfoThrown = builder.CreateStructGEP(ourExceptionType, typeInfoThrown, 0);
 
   llvm::Value *typeInfoThrownType =
-      builder.CreateStructGEP(ourTypeInfoType, typeInfoThrown, 0);
+      builder.CreateStructGEP(builder.getPtrTy(), typeInfoThrown, 0);
 
-  llvm::Value *ti8 =
-      builder.CreateLoad(builder.getInt8Ty(), typeInfoThrownType);
-  generateIntegerPrint(context, module, builder, *toPrint32Int,
-                       builder.CreateZExt(ti8, builder.getInt32Ty()),
+  generateIntegerPrint(context,
+                       module,
+                       builder,
+                       *toPrint32Int,
+                       *(builder.CreateLoad(typeInfoThrownType)),
                        "Gen: Exception type <%d> received (stack unwound) "
                        " in " +
-                           ourId + ".\n",
+                       ourId +
+                       ".\n",
                        USE_GLOBAL_STR_CONSTS);
 
   // Route to matched type info catch block or run cleanup finally block
@@ -1311,7 +1311,8 @@ static llvm::Function *createCatchWrappedInvokeFunction(
 
   for (unsigned i = 1; i <= numExceptionsToCatch; ++i) {
     nextTypeToCatch = i - 1;
-    switchToCatchBlock->addCase(llvm::ConstantInt::get(builder.getInt32Ty(), i),
+    switchToCatchBlock->addCase(llvm::ConstantInt::get(
+                                   llvm::Type::getInt32Ty(context), i),
                                 catchBlocks[nextTypeToCatch]);
   }
 
@@ -1386,10 +1387,14 @@ createThrowExceptionFunction(llvm::Module &module, llvm::IRBuilder<> &builder,
   builder.SetInsertPoint(entryBlock);
 
   llvm::Function *toPrint32Int = module.getFunction("print32Int");
-  generateIntegerPrint(context, module, builder, *toPrint32Int,
-                       builder.CreateZExt(exceptionType, builder.getInt32Ty()),
-                       "\nGen: About to throw exception type <%d> in " + ourId +
-                           ".\n",
+  generateIntegerPrint(context,
+                       module,
+                       builder,
+                       *toPrint32Int,
+                       *exceptionType,
+                       "\nGen: About to throw exception type <%d> in " +
+                       ourId +
+                       ".\n",
                        USE_GLOBAL_STR_CONSTS);
 
   // Switches on runtime type info type value to determine whether or not
@@ -1541,13 +1546,15 @@ typedef void (*OurExceptionThrowFunctType) (int32_t typeToThrow);
 /// @param function generated test function to run
 /// @param typeToThrow type info type of generated exception to throw, or
 ///        indicator to cause foreign exception to be thrown.
-static void runExceptionThrow(llvm::orc::LLJIT *JIT, std::string function,
-                              int32_t typeToThrow) {
+static
+void runExceptionThrow(llvm::ExecutionEngine *engine,
+                       llvm::Function *function,
+                       int32_t typeToThrow) {
 
   // Find test's function pointer
   OurExceptionThrowFunctType functPtr =
-      reinterpret_cast<OurExceptionThrowFunctType>(reinterpret_cast<uintptr_t>(
-          ExitOnErr(JIT->lookup(function)).getValue()));
+    reinterpret_cast<OurExceptionThrowFunctType>(
+       reinterpret_cast<intptr_t>(engine->getPointerToFunction(function)));
 
   try {
     // Run test
@@ -1576,6 +1583,8 @@ static void runExceptionThrow(llvm::orc::LLJIT *JIT, std::string function,
 // End test functions
 //
 
+typedef llvm::ArrayRef<llvm::Type*> TypeArray;
+
 /// This initialization routine creates type info globals and
 /// adds external function declarations to module.
 /// @param numTypeInfos number of linear type info associated type info types
@@ -1885,73 +1894,93 @@ int main(int argc, char *argv[]) {
     return(0);
   }
 
+  // If not set, exception handling will not be turned on
+  llvm::TargetOptions Opts;
+
   llvm::InitializeNativeTarget();
   llvm::InitializeNativeTargetAsmPrinter();
-  auto Context = std::make_unique<llvm::LLVMContext>();
-  llvm::IRBuilder<> theBuilder(*Context);
+  llvm::LLVMContext Context;
+  llvm::IRBuilder<> theBuilder(Context);
 
   // Make the module, which holds all the code.
   std::unique_ptr<llvm::Module> Owner =
-      std::make_unique<llvm::Module>("my cool jit", *Context);
+      std::make_unique<llvm::Module>("my cool jit", Context);
   llvm::Module *module = Owner.get();
 
-  // Build LLJIT
-  std::unique_ptr<llvm::orc::LLJIT> JIT =
-      ExitOnErr(llvm::orc::LLJITBuilder().create());
+  std::unique_ptr<llvm::RTDyldMemoryManager> MemMgr(new llvm::SectionMemoryManager());
 
-  // Set up the optimizer pipeline.
-  llvm::legacy::FunctionPassManager fpm(module);
+  // Build engine with JIT
+  llvm::EngineBuilder factory(std::move(Owner));
+  factory.setEngineKind(llvm::EngineKind::JIT);
+  factory.setTargetOptions(Opts);
+  factory.setMCJITMemoryManager(std::move(MemMgr));
+  llvm::ExecutionEngine *executionEngine = factory.create();
 
-  // Optimizations turned on
+  {
+    llvm::legacy::FunctionPassManager fpm(module);
+
+    // Set up the optimizer pipeline.
+    // Start with registering info about how the
+    // target lays out data structures.
+    module->setDataLayout(executionEngine->getDataLayout());
+
+    // Optimizations turned on
 #ifdef ADD_OPT_PASSES
 
-  // Basic AliasAnslysis support for GVN.
-  fpm.add(llvm::createBasicAliasAnalysisPass());
+    // Basic AliasAnslysis support for GVN.
+    fpm.add(llvm::createBasicAliasAnalysisPass());
 
-  // Promote allocas to registers.
-  fpm.add(llvm::createPromoteMemoryToRegisterPass());
+    // Promote allocas to registers.
+    fpm.add(llvm::createPromoteMemoryToRegisterPass());
 
-  // Do simple "peephole" optimizations and bit-twiddling optzns.
-  fpm.add(llvm::createInstructionCombiningPass());
+    // Do simple "peephole" optimizations and bit-twiddling optzns.
+    fpm.add(llvm::createInstructionCombiningPass());
 
-  // Reassociate expressions.
-  fpm.add(llvm::createReassociatePass());
+    // Reassociate expressions.
+    fpm.add(llvm::createReassociatePass());
 
-  // Eliminate Common SubExpressions.
-  fpm.add(llvm::createGVNPass());
+    // Eliminate Common SubExpressions.
+    fpm.add(llvm::createGVNPass());
 
-  // Simplify the control flow graph (deleting unreachable
-  // blocks, etc).
-  fpm.add(llvm::createCFGSimplificationPass());
+    // Simplify the control flow graph (deleting unreachable
+    // blocks, etc).
+    fpm.add(llvm::createCFGSimplificationPass());
 #endif  // ADD_OPT_PASSES
 
-  fpm.doInitialization();
+    fpm.doInitialization();
 
-  // Generate test code using function throwCppException(...) as
-  // the function which throws foreign exceptions.
-  createUnwindExceptionTest(*module, theBuilder, fpm, "throwCppException");
+    // Generate test code using function throwCppException(...) as
+    // the function which throws foreign exceptions.
+    llvm::Function *toRun =
+    createUnwindExceptionTest(*module,
+                              theBuilder,
+                              fpm,
+                              "throwCppException");
 
-  ExitOnErr(JIT->addIRModule(
-      llvm::orc::ThreadSafeModule(std::move(Owner), std::move(Context))));
+    executionEngine->finalizeObject();
 
 #ifndef NDEBUG
-  fprintf(stderr, "\nBegin module dump:\n\n");
+    fprintf(stderr, "\nBegin module dump:\n\n");
 
-  module->print(llvm::errs(), nullptr);
+    module->dump();
 
-  fprintf(stderr, "\nEnd module dump:\n");
+    fprintf(stderr, "\nEnd module dump:\n");
 #endif
 
-  fprintf(stderr, "\n\nBegin Test:\n");
-  std::string toRun = "outerCatchFunct";
+    fprintf(stderr, "\n\nBegin Test:\n");
+
+    for (int i = 1; i < argc; ++i) {
+      // Run test for each argument whose value is the exception
+      // type to throw.
+      runExceptionThrow(executionEngine,
+                        toRun,
+                        (unsigned) strtoul(argv[i], NULL, 10));
+    }
 
-  for (int i = 1; i < argc; ++i) {
-    // Run test for each argument whose value is the exception
-    // type to throw.
-    runExceptionThrow(JIT.get(), toRun, (unsigned)strtoul(argv[i], NULL, 10));
+    fprintf(stderr, "\nEnd Test:\n\n");
   }
 
-  fprintf(stderr, "\nEnd Test:\n\n");
+  delete executionEngine;
 
   return 0;
 }
-- 
GitLab


From 3bb39690d729d85cd93c9dd6e750d82d6f367541 Mon Sep 17 00:00:00 2001
From: Hans <hans@hanshq.net>
Date: Wed, 15 May 2024 15:29:08 +0200
Subject: [PATCH 0852/1206] [coro] Lower `llvm.coro.await.suspend.handle` to
 resume with tail call (#89751)

The C++ standard requires that symmetric transfer from one coroutine to
another is performed via a tail call. Failure to do so is a miscompile
and often breaks programs by quickly overflowing the stack.

Until now, the coro split pass tried to ensure this in the
`addMustTailToCoroResumes()` function by searching for
`llvm.coro.resume` calls to lower as tail calls if the conditions were
right: the right function arguments, attributes, calling convention
etc., and if a `ret void` was sure to be reached after traversal with
some ad-hoc constant folding following the call.

This was brittle, as the kind of implicit variants required for a tail
call to happen could easily be broken by other passes (e.g. if some
instruction got in between the `resume` and `ret`), see for example
9d1cb18d19862fc0627e4a56e1e491a498e84c71 and
284da049f5feb62b40f5abc41dda7895e3d81d72.

Also the logic seemed backwards: instead of searching for possible tail
call candidates and doing them if the circumstances are right, it seems
better to start with the intention of making the tail calls we need, and
forcing the circumstances to be right.

Now that we have the `llvm.coro.await.suspend.handle` intrinsic (since
f78688134026686288a8d310b493d9327753a022) which corresponds exactly to
symmetric transfer, change the lowering of that to also include the
`resume` part, always lowered as a tail call.
---
 clang/lib/CodeGen/CGCoroutine.cpp             |  11 +-
 clang/test/CodeGenCoroutines/coro-await.cpp   |   4 +-
 .../coro-symmetric-transfer-01.cpp            |  54 ----
 .../coro-symmetric-transfer-02.cpp            |   6 +-
 llvm/docs/Coroutines.rst                      |  19 +-
 llvm/include/llvm/IR/Intrinsics.td            |   2 +-
 llvm/lib/Transforms/Coroutines/CoroInternal.h |   3 +-
 llvm/lib/Transforms/Coroutines/CoroSplit.cpp  | 240 +++++-------------
 llvm/lib/Transforms/Coroutines/Coroutines.cpp |   4 +-
 .../coro-await-suspend-lower-invoke.ll        |   5 +-
 .../Coroutines/coro-await-suspend-lower.ll    |   5 +-
 .../Coroutines/coro-preserve-final.ll         | 131 ----------
 ...-split-musttail-chain-pgo-counter-promo.ll |   9 +-
 .../Coroutines/coro-split-musttail.ll         |  17 +-
 .../Coroutines/coro-split-musttail1.ll        |  35 ++-
 .../Coroutines/coro-split-musttail10.ll       |   6 +-
 .../Coroutines/coro-split-musttail2.ll        |  15 +-
 .../Coroutines/coro-split-musttail3.ll        |  36 ++-
 .../Coroutines/coro-split-musttail4.ll        |   8 +-
 .../Coroutines/coro-split-musttail5.ll        |   8 +-
 .../Coroutines/coro-split-musttail6.ll        |  11 +-
 .../Coroutines/coro-split-musttail7.ll        |  17 +-
 22 files changed, 168 insertions(+), 478 deletions(-)
 delete mode 100644 clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp
 delete mode 100644 llvm/test/Transforms/Coroutines/coro-preserve-final.ll

diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp
index 567e85a02dc6..b4c724422c14 100644
--- a/clang/lib/CodeGen/CGCoroutine.cpp
+++ b/clang/lib/CodeGen/CGCoroutine.cpp
@@ -278,7 +278,11 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co
 
   llvm::Function *AwaitSuspendIntrinsic = CGF.CGM.getIntrinsic(AwaitSuspendIID);
 
-  const auto AwaitSuspendCanThrow = StmtCanThrow(S.getSuspendExpr());
+  // SuspendHandle might throw since it also resumes the returned handle.
+  const bool AwaitSuspendCanThrow =
+      SuspendReturnType ==
+          CoroutineSuspendExpr::SuspendReturnType::SuspendHandle ||
+      StmtCanThrow(S.getSuspendExpr());
 
   llvm::CallBase *SuspendRet = nullptr;
   // FIXME: add call attributes?
@@ -307,10 +311,7 @@ static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Co
     break;
   }
   case CoroutineSuspendExpr::SuspendReturnType::SuspendHandle: {
-    assert(SuspendRet->getType()->isPointerTy());
-
-    auto ResumeIntrinsic = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_resume);
-    Builder.CreateCall(ResumeIntrinsic, SuspendRet);
+    assert(SuspendRet->getType()->isVoidTy());
     break;
   }
   }
diff --git a/clang/test/CodeGenCoroutines/coro-await.cpp b/clang/test/CodeGenCoroutines/coro-await.cpp
index 65bfb0994688..c7a09e8b8bc7 100644
--- a/clang/test/CodeGenCoroutines/coro-await.cpp
+++ b/clang/test/CodeGenCoroutines/coro-await.cpp
@@ -370,8 +370,8 @@ extern "C" void TestTailcall() {
   // ---------------------------
   // Call coro.await.suspend
   // ---------------------------
-  // CHECK-NEXT: %[[RESUMED:.+]] = call ptr @llvm.coro.await.suspend.handle(ptr %[[AWAITABLE]], ptr %[[FRAME]], ptr @TestTailcall.__await_suspend_wrapper__await)
-  // CHECK-NEXT: call void @llvm.coro.resume(ptr %[[RESUMED]])
+  // Note: The call must not be nounwind since the resumed function could throw.
+  // CHECK-NEXT: call void @llvm.coro.await.suspend.handle(ptr %[[AWAITABLE]], ptr %[[FRAME]], ptr @TestTailcall.__await_suspend_wrapper__await){{$}}
   // CHECK-NEXT: %[[OUTCOME:.+]] = call i8 @llvm.coro.suspend(token %[[SUSPEND_ID]], i1 false)
   // CHECK-NEXT: switch i8 %[[OUTCOME]], label %[[RET_BB:.+]] [
   // CHECK-NEXT:   i8 0, label %[[READY_BB]]
diff --git a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp
deleted file mode 100644
index da30e12c63cf..000000000000
--- a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-01.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 -O0 -emit-llvm %s -o - -disable-llvm-passes | FileCheck %s
-// RUN: %clang -std=c++20 -O0 -emit-llvm -c  %s -o %t -Xclang -disable-llvm-passes && %clang -c %t
-
-#include "Inputs/coroutine.h"
-
-struct detached_task {
-  struct promise_type {
-    detached_task get_return_object() noexcept {
-      return detached_task{std::coroutine_handle<promise_type>::from_promise(*this)};
-    }
-
-    void return_void() noexcept {}
-
-    struct final_awaiter {
-      bool await_ready() noexcept { return false; }
-      std::coroutine_handle<> await_suspend(std::coroutine_handle<promise_type> h) noexcept {
-        h.destroy();
-        return {};
-      }
-      void await_resume() noexcept {}
-    };
-
-    void unhandled_exception() noexcept {}
-
-    final_awaiter final_suspend() noexcept { return {}; }
-
-    std::suspend_always initial_suspend() noexcept { return {}; }
-  };
-
-  ~detached_task() {
-    if (coro_) {
-      coro_.destroy();
-      coro_ = {};
-    }
-  }
-
-  void start() && {
-    auto tmp = coro_;
-    coro_ = {};
-    tmp.resume();
-  }
-
-  std::coroutine_handle<promise_type> coro_;
-};
-
-detached_task foo() {
-  co_return;
-}
-
-// check that the lifetime of the coroutine handle used to obtain the address is contained within single basic block, and hence does not live across suspension points.
-// CHECK-LABEL: final.suspend:
-// CHECK:         %{{.+}} = call token @llvm.coro.save(ptr null)
-// CHECK:         %[[HDL_TRANSFER:.+]] = call ptr @llvm.coro.await.suspend.handle
-// CHECK:         call void @llvm.coro.resume(ptr %[[HDL_TRANSFER]])
diff --git a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-02.cpp b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-02.cpp
index ca6cf74115a3..f36f89926505 100644
--- a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-02.cpp
+++ b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-02.cpp
@@ -89,8 +89,7 @@ Task bar() {
 // CHECK:         br i1 %{{.+}}, label %[[CASE1_AWAIT_READY:.+]], label %[[CASE1_AWAIT_SUSPEND:.+]]
 // CHECK:       [[CASE1_AWAIT_SUSPEND]]:
 // CHECK-NEXT:    %{{.+}} = call token @llvm.coro.save(ptr null)
-// CHECK-NEXT:    %[[HANDLE1_PTR:.+]] = call ptr @llvm.coro.await.suspend.handle
-// CHECK-NEXT:    call void @llvm.coro.resume(ptr %[[HANDLE1_PTR]])
+// CHECK-NEXT:    call void @llvm.coro.await.suspend.handle
 // CHECK-NEXT:    %{{.+}} = call i8 @llvm.coro.suspend
 // CHECK-NEXT:    switch i8 %{{.+}}, label %coro.ret [
 // CHECK-NEXT:      i8 0, label %[[CASE1_AWAIT_READY]]
@@ -104,8 +103,7 @@ Task bar() {
 // CHECK:         br i1 %{{.+}}, label %[[CASE2_AWAIT_READY:.+]], label %[[CASE2_AWAIT_SUSPEND:.+]]
 // CHECK:       [[CASE2_AWAIT_SUSPEND]]:
 // CHECK-NEXT:    %{{.+}} = call token @llvm.coro.save(ptr null)
-// CHECK-NEXT:    %[[HANDLE2_PTR:.+]] = call ptr @llvm.coro.await.suspend.handle
-// CHECK-NEXT:    call void @llvm.coro.resume(ptr %[[HANDLE2_PTR]])
+// CHECK-NEXT:    call void @llvm.coro.await.suspend.handle
 // CHECK-NEXT:    %{{.+}} = call i8 @llvm.coro.suspend
 // CHECK-NEXT:    switch i8 %{{.+}}, label %coro.ret [
 // CHECK-NEXT:      i8 0, label %[[CASE2_AWAIT_READY]]
diff --git a/llvm/docs/Coroutines.rst b/llvm/docs/Coroutines.rst
index 83369d93c309..36092325e536 100644
--- a/llvm/docs/Coroutines.rst
+++ b/llvm/docs/Coroutines.rst
@@ -1922,7 +1922,7 @@ Example:
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 ::
 
-  declare ptr @llvm.coro.await.suspend.handle(
+  declare void @llvm.coro.await.suspend.handle(
                 ptr <awaiter>,
                 ptr <handle>,
                 ptr <await_suspend_function>)
@@ -1967,7 +1967,9 @@ The intrinsic must be used between corresponding `coro.save`_ and
 `await_suspend_function` call during `CoroSplit`_ pass.
 
 `await_suspend_function` must return a pointer to a valid
-coroutine frame, which is immediately resumed
+coroutine frame. The intrinsic will be lowered to a tail call resuming the
+returned coroutine frame. It will be marked `musttail` on targets that support
+that. Instructions following the intrinsic will become unreachable.
 
 Example:
 """"""""
@@ -1977,11 +1979,10 @@ Example:
   ; before lowering
   await.suspend:
     %save = call token @llvm.coro.save(ptr %hdl)
-    %next = call ptr @llvm.coro.await.suspend.handle(
-                ptr %awaiter,
-                ptr %hdl,
-                ptr @await_suspend_function)
-    call void @llvm.coro.resume(%next)
+    call void @llvm.coro.await.suspend.handle(
+        ptr %awaiter,
+        ptr %hdl,
+        ptr @await_suspend_function)
     %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
     ...
 
@@ -1992,8 +1993,8 @@ Example:
     %next = call ptr @await_suspend_function(
                 ptr %awaiter,
                 ptr %hdl)
-    call void @llvm.coro.resume(%next)
-    %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
+    musttail call void @llvm.coro.resume(%next)
+    ret void
     ...
 
   ; wrapper function example
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index f1c7d950f927..78f0dbec863e 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1717,7 +1717,7 @@ def int_coro_await_suspend_bool : Intrinsic<[llvm_i1_ty],
                                             [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
                                             [Throws]>;
 
-def int_coro_await_suspend_handle : Intrinsic<[llvm_ptr_ty],
+def int_coro_await_suspend_handle : Intrinsic<[],
                                               [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
                                               [Throws]>;
 
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index 84fd88806154..5716fd0ea4ab 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -47,7 +47,7 @@ struct LowererBase {
   ConstantPointerNull *const NullPtr;
 
   LowererBase(Module &M);
-  Value *makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt);
+  CallInst *makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt);
 };
 
 enum class ABI {
@@ -85,6 +85,7 @@ struct LLVM_LIBRARY_VISIBILITY Shape {
   SmallVector<AnyCoroSuspendInst *, 4> CoroSuspends;
   SmallVector<CallInst*, 2> SwiftErrorOps;
   SmallVector<CoroAwaitSuspendInst *, 4> CoroAwaitSuspends;
+  SmallVector<CallInst *, 2> SymmetricTransfers;
 
   // Field indexes for special fields in the switch lowering.
   struct SwitchFieldIndex {
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 4eb6e75d09fa..450ea8234371 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -113,21 +113,24 @@ private:
   /// ABIs.
   AnyCoroSuspendInst *ActiveSuspend = nullptr;
 
+  TargetTransformInfo &TTI;
+
 public:
   /// Create a cloner for a switch lowering.
   CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
-             Kind FKind)
+             Kind FKind, TargetTransformInfo &TTI)
       : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape), FKind(FKind),
-        Builder(OrigF.getContext()) {
+        Builder(OrigF.getContext()), TTI(TTI) {
     assert(Shape.ABI == coro::ABI::Switch);
   }
 
   /// Create a cloner for a continuation lowering.
   CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
-             Function *NewF, AnyCoroSuspendInst *ActiveSuspend)
+             Function *NewF, AnyCoroSuspendInst *ActiveSuspend,
+             TargetTransformInfo &TTI)
       : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape),
         FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation),
-        Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend) {
+        Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend), TTI(TTI) {
     assert(Shape.ABI == coro::ABI::Retcon ||
            Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async);
     assert(NewF && "need existing function for continuation");
@@ -171,7 +174,8 @@ private:
 // Lower the intrinisc in CoroEarly phase if coroutine frame doesn't escape
 // and it is known that other transformations, for example, sanitizers
 // won't lead to incorrect code.
-static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB) {
+static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB,
+                              coro::Shape &Shape) {
   auto Wrapper = CB->getWrapperFunction();
   auto Awaiter = CB->getAwaiter();
   auto FramePtr = CB->getFrame();
@@ -206,6 +210,31 @@ static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB) {
     llvm_unreachable("Unexpected coro_await_suspend invocation method");
   }
 
+  if (CB->getCalledFunction()->getIntrinsicID() ==
+      Intrinsic::coro_await_suspend_handle) {
+    // Follow the lowered await_suspend call above with a lowered resume call
+    // to the returned coroutine.
+    if (auto *Invoke = dyn_cast<InvokeInst>(CB)) {
+      // If the await_suspend call is an invoke, we continue in the next block.
+      Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstInsertionPt());
+    }
+
+    coro::LowererBase LB(*Wrapper->getParent());
+    auto *ResumeAddr = LB.makeSubFnCall(NewCall, CoroSubFnInst::ResumeIndex,
+                                        &*Builder.GetInsertPoint());
+
+    LLVMContext &Ctx = Builder.getContext();
+    FunctionType *ResumeTy = FunctionType::get(
+        Type::getVoidTy(Ctx), PointerType::getUnqual(Ctx), false);
+    auto *ResumeCall = Builder.CreateCall(ResumeTy, ResumeAddr, {NewCall});
+
+    // We can't insert the 'ret' instruction and adjust the cc until the
+    // function has been split, so remember this for later.
+    Shape.SymmetricTransfers.push_back(ResumeCall);
+
+    NewCall = ResumeCall;
+  }
+
   CB->replaceAllUsesWith(NewCall);
   CB->eraseFromParent();
 }
@@ -213,7 +242,7 @@ static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB) {
 static void lowerAwaitSuspends(Function &F, coro::Shape &Shape) {
   IRBuilder<> Builder(F.getContext());
   for (auto *AWS : Shape.CoroAwaitSuspends)
-    lowerAwaitSuspend(Builder, AWS);
+    lowerAwaitSuspend(Builder, AWS, Shape);
 }
 
 static void maybeFreeRetconStorage(IRBuilder<> &Builder,
@@ -1056,6 +1085,25 @@ void CoroCloner::create() {
   // Set up the new entry block.
   replaceEntryBlock();
 
+  // Turn symmetric transfers into musttail calls.
+  for (CallInst *ResumeCall : Shape.SymmetricTransfers) {
+    ResumeCall = cast<CallInst>(VMap[ResumeCall]);
+    ResumeCall->setCallingConv(NewF->getCallingConv());
+    if (TTI.supportsTailCallFor(ResumeCall)) {
+      // FIXME: Could we support symmetric transfer effectively without
+      // musttail?
+      ResumeCall->setTailCallKind(CallInst::TCK_MustTail);
+    }
+
+    // Put a 'ret void' after the call, and split any remaining instructions to
+    // an unreachable block.
+    BasicBlock *BB = ResumeCall->getParent();
+    BB->splitBasicBlock(ResumeCall->getNextNode());
+    Builder.SetInsertPoint(BB->getTerminator());
+    Builder.CreateRetVoid();
+    BB->getTerminator()->eraseFromParent();
+  }
+
   Builder.SetInsertPoint(&NewF->getEntryBlock().front());
   NewFramePtr = deriveNewFramePointer();
 
@@ -1186,130 +1234,6 @@ scanPHIsAndUpdateValueMap(Instruction *Prev, BasicBlock *NewBlock,
   }
 }
 
-// Replace a sequence of branches leading to a ret, with a clone of a ret
-// instruction. Suspend instruction represented by a switch, track the PHI
-// values and select the correct case successor when possible.
-static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) {
-  // There is nothing to simplify.
-  if (isa<ReturnInst>(InitialInst))
-    return false;
-
-  DenseMap<Value *, Value *> ResolvedValues;
-  assert(InitialInst->getModule());
-  const DataLayout &DL = InitialInst->getModule()->getDataLayout();
-
-  auto TryResolveConstant = [&ResolvedValues](Value *V) {
-    auto It = ResolvedValues.find(V);
-    if (It != ResolvedValues.end())
-      V = It->second;
-    return dyn_cast<ConstantInt>(V);
-  };
-
-  Instruction *I = InitialInst;
-  while (true) {
-    if (isa<ReturnInst>(I)) {
-      assert(!cast<ReturnInst>(I)->getReturnValue());
-      ReplaceInstWithInst(InitialInst, I->clone());
-      return true;
-    }
-
-    if (auto *BR = dyn_cast<BranchInst>(I)) {
-      unsigned SuccIndex = 0;
-      if (BR->isConditional()) {
-        // Handle the case the condition of the conditional branch is constant.
-        // e.g.,
-        //
-        //     br i1 false, label %cleanup, label %CoroEnd
-        //
-        // It is possible during the transformation. We could continue the
-        // simplifying in this case.
-        ConstantInt *Cond = TryResolveConstant(BR->getCondition());
-        if (!Cond)
-          return false;
-
-        SuccIndex = Cond->isOne() ? 0 : 1;
-      }
-
-      BasicBlock *Succ = BR->getSuccessor(SuccIndex);
-      scanPHIsAndUpdateValueMap(I, Succ, ResolvedValues);
-      I = Succ->getFirstNonPHIOrDbgOrLifetime();
-      continue;
-    }
-
-    if (auto *Cmp = dyn_cast<CmpInst>(I)) {
-      // If the case number of suspended switch instruction is reduced to
-      // 1, then it is simplified to CmpInst in llvm::ConstantFoldTerminator.
-      // Try to constant fold it.
-      ConstantInt *Cond0 = TryResolveConstant(Cmp->getOperand(0));
-      ConstantInt *Cond1 = TryResolveConstant(Cmp->getOperand(1));
-      if (Cond0 && Cond1) {
-        ConstantInt *Result =
-            dyn_cast_or_null<ConstantInt>(ConstantFoldCompareInstOperands(
-                Cmp->getPredicate(), Cond0, Cond1, DL));
-        if (Result) {
-          ResolvedValues[Cmp] = Result;
-          I = I->getNextNode();
-          continue;
-        }
-      }
-    }
-
-    if (auto *SI = dyn_cast<SwitchInst>(I)) {
-      ConstantInt *Cond = TryResolveConstant(SI->getCondition());
-      if (!Cond)
-        return false;
-
-      BasicBlock *Succ = SI->findCaseValue(Cond)->getCaseSuccessor();
-      scanPHIsAndUpdateValueMap(I, Succ, ResolvedValues);
-      I = Succ->getFirstNonPHIOrDbgOrLifetime();
-      continue;
-    }
-
-    if (I->isDebugOrPseudoInst() || I->isLifetimeStartOrEnd() ||
-        wouldInstructionBeTriviallyDead(I)) {
-      // We can skip instructions without side effects. If their values are
-      // needed, we'll notice later, e.g. when hitting a conditional branch.
-      I = I->getNextNode();
-      continue;
-    }
-
-    break;
-  }
-
-  return false;
-}
-
-// Check whether CI obeys the rules of musttail attribute.
-static bool shouldBeMustTail(const CallInst &CI, const Function &F) {
-  if (CI.isInlineAsm())
-    return false;
-
-  // Match prototypes and calling conventions of resume function.
-  FunctionType *CalleeTy = CI.getFunctionType();
-  if (!CalleeTy->getReturnType()->isVoidTy() || (CalleeTy->getNumParams() != 1))
-    return false;
-
-  Type *CalleeParmTy = CalleeTy->getParamType(0);
-  if (!CalleeParmTy->isPointerTy() ||
-      (CalleeParmTy->getPointerAddressSpace() != 0))
-    return false;
-
-  if (CI.getCallingConv() != F.getCallingConv())
-    return false;
-
-  // CI should not has any ABI-impacting function attributes.
-  static const Attribute::AttrKind ABIAttrs[] = {
-      Attribute::StructRet,    Attribute::ByVal,     Attribute::InAlloca,
-      Attribute::Preallocated, Attribute::InReg,     Attribute::Returned,
-      Attribute::SwiftSelf,    Attribute::SwiftError};
-  AttributeList Attrs = CI.getAttributes();
-  for (auto AK : ABIAttrs)
-    if (Attrs.hasParamAttr(0, AK))
-      return false;
-
-  return true;
-}
-
 // Coroutine has no suspend points. Remove heap allocation for the coroutine
 // frame if possible.
 static void handleNoSuspendCoroutine(coro::Shape &Shape) {
@@ -1523,24 +1447,16 @@ struct SwitchCoroutineSplitter {
 
     createResumeEntryBlock(F, Shape);
     auto *ResumeClone =
-        createClone(F, ".resume", Shape, CoroCloner::Kind::SwitchResume);
+        createClone(F, ".resume", Shape, CoroCloner::Kind::SwitchResume, TTI);
     auto *DestroyClone =
-        createClone(F, ".destroy", Shape, CoroCloner::Kind::SwitchUnwind);
+        createClone(F, ".destroy", Shape, CoroCloner::Kind::SwitchUnwind, TTI);
     auto *CleanupClone =
-        createClone(F, ".cleanup", Shape, CoroCloner::Kind::SwitchCleanup);
+        createClone(F, ".cleanup", Shape, CoroCloner::Kind::SwitchCleanup, TTI);
 
     postSplitCleanup(*ResumeClone);
     postSplitCleanup(*DestroyClone);
     postSplitCleanup(*CleanupClone);
 
-    // Adding musttail call to support symmetric transfer.
-    // Skip targets which don't support tail call.
-    //
-    // FIXME: Could we support symmetric transfer effectively without musttail
-    // call?
-    if (TTI.supportsTailCalls())
-      addMustTailToCoroResumes(*ResumeClone, TTI);
-
     // Store addresses resume/destroy/cleanup functions in the coroutine frame.
     updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);
 
@@ -1560,8 +1476,9 @@ private:
   // new entry block and replacing coro.suspend an appropriate value to force
   // resume or cleanup pass for every suspend point.
   static Function *createClone(Function &F, const Twine &Suffix,
-                               coro::Shape &Shape, CoroCloner::Kind FKind) {
-    CoroCloner Cloner(F, Suffix, Shape, FKind);
+                               coro::Shape &Shape, CoroCloner::Kind FKind,
+                               TargetTransformInfo &TTI) {
+    CoroCloner Cloner(F, Suffix, Shape, FKind, TTI);
     Cloner.create();
     return Cloner.getFunction();
   }
@@ -1662,34 +1579,6 @@ private:
     Shape.SwitchLowering.ResumeEntryBlock = NewEntry;
   }
 
-  // Add musttail to any resume instructions that is immediately followed by a
-  // suspend (i.e. ret). We do this even in -O0 to support guaranteed tail call
-  // for symmetrical coroutine control transfer (C++ Coroutines TS extension).
-  // This transformation is done only in the resume part of the coroutine that
-  // has identical signature and calling convention as the coro.resume call.
-  static void addMustTailToCoroResumes(Function &F, TargetTransformInfo &TTI) {
-    bool Changed = false;
-
-    // Collect potential resume instructions.
-    SmallVector<CallInst *, 4> Resumes;
-    for (auto &I : instructions(F))
-      if (auto *Call = dyn_cast<CallInst>(&I))
-        if (shouldBeMustTail(*Call, F))
-          Resumes.push_back(Call);
-
-    // Set musttail on those that are followed by a ret instruction.
-    for (CallInst *Call : Resumes)
-      // Skip targets which don't support tail call on the specific case.
-      if (TTI.supportsTailCallFor(Call) &&
-          simplifyTerminatorLeadingToRet(Call->getNextNode())) {
-        Call->setTailCallKind(CallInst::TCK_MustTail);
-        Changed = true;
-      }
-
-    if (Changed)
-      removeUnreachableBlocks(F);
-  }
-
   // Store addresses of Resume/Destroy/Cleanup functions in the coroutine frame.
   static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
                               Function *DestroyFn, Function *CleanupFn) {
@@ -1893,12 +1782,13 @@ static void splitAsyncCoroutine(Function &F, coro::Shape &Shape,
     auto *Suspend = Shape.CoroSuspends[Idx];
     auto *Clone = Clones[Idx];
 
-    CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend).create();
+    CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend, TTI).create();
   }
 }
 
 static void splitRetconCoroutine(Function &F, coro::Shape &Shape,
-                                 SmallVectorImpl<Function *> &Clones) {
+                                 SmallVectorImpl<Function *> &Clones,
+                                 TargetTransformInfo &TTI) {
   assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce);
   assert(Clones.empty());
 
@@ -2021,7 +1911,7 @@ static void splitRetconCoroutine(Function &F, coro::Shape &Shape,
     auto Suspend = Shape.CoroSuspends[i];
     auto Clone = Clones[i];
 
-    CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend).create();
+    CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend, TTI).create();
   }
 }
 
@@ -2073,7 +1963,7 @@ splitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
       break;
     case coro::ABI::Retcon:
     case coro::ABI::RetconOnce:
-      splitRetconCoroutine(F, Shape, Clones);
+      splitRetconCoroutine(F, Shape, Clones, TTI);
       break;
     }
   }
diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
index a1c78d6a44ef..1a92bc163625 100644
--- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp
+++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
@@ -47,8 +47,8 @@ coro::LowererBase::LowererBase(Module &M)
 //
 //    call ptr @llvm.coro.subfn.addr(ptr %Arg, i8 %index)
 
-Value *coro::LowererBase::makeSubFnCall(Value *Arg, int Index,
-                                        Instruction *InsertPt) {
+CallInst *coro::LowererBase::makeSubFnCall(Value *Arg, int Index,
+                                           Instruction *InsertPt) {
   auto *IndexVal = ConstantInt::get(Type::getInt8Ty(Context), Index);
   auto *Fn = Intrinsic::getDeclaration(&TheModule, Intrinsic::coro_subfn_addr);
 
diff --git a/llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll b/llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll
index fbc4a2c006f8..fd3b7bd81530 100644
--- a/llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll
+++ b/llvm/test/Transforms/Coroutines/coro-await-suspend-lower-invoke.ll
@@ -58,14 +58,13 @@ suspend.cond:
 ; CHECK-NEXT:       to label %[[STEP2_CONT:[^ ]+]] unwind label %[[PAD]]
 step2:
   %save2 = call token @llvm.coro.save(ptr null)
-  %resume.handle = invoke ptr @llvm.coro.await.suspend.handle(ptr %awaiter, ptr %hdl, ptr @await_suspend_wrapper_handle)
+  invoke void @llvm.coro.await.suspend.handle(ptr %awaiter, ptr %hdl, ptr @await_suspend_wrapper_handle)
     to label %step2.continue unwind label %pad
 
 ; CHECK:      [[STEP2_CONT]]:
 ; CHECK-NEXT:   %[[NEXT_RESUME:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[NEXT_HDL]], i8 0)
 ; CHECK-NEXT:   musttail call {{.*}} void %[[NEXT_RESUME]](ptr %[[NEXT_HDL]])
 step2.continue:
-  call void @llvm.coro.resume(ptr %resume.handle)
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
   switch i8 %suspend2, label %ret [
     i8 0, label %step3
@@ -112,7 +111,7 @@ declare i1 @llvm.coro.alloc(token)
 declare ptr @llvm.coro.begin(token, ptr)
 declare void @llvm.coro.await.suspend.void(ptr, ptr, ptr)
 declare i1 @llvm.coro.await.suspend.bool(ptr, ptr, ptr)
-declare ptr @llvm.coro.await.suspend.handle(ptr, ptr, ptr)
+declare void @llvm.coro.await.suspend.handle(ptr, ptr, ptr)
 declare i1 @llvm.coro.end(ptr, i1, token)
 
 declare ptr @__cxa_begin_catch(ptr)
diff --git a/llvm/test/Transforms/Coroutines/coro-await-suspend-lower.ll b/llvm/test/Transforms/Coroutines/coro-await-suspend-lower.ll
index 0f574c4acc26..8d019e695462 100644
--- a/llvm/test/Transforms/Coroutines/coro-await-suspend-lower.ll
+++ b/llvm/test/Transforms/Coroutines/coro-await-suspend-lower.ll
@@ -49,8 +49,7 @@ suspend.cond:
 ; CHECK-NEXT:   musttail call {{.*}} void %[[CONT]](ptr %[[NEXT_HDL]])
 step2:
   %save2 = call token @llvm.coro.save(ptr null)
-  %resume.handle = call ptr @llvm.coro.await.suspend.handle(ptr %awaiter, ptr %hdl, ptr @await_suspend_wrapper_handle)
-  call void @llvm.coro.resume(ptr %resume.handle)
+  call void @llvm.coro.await.suspend.handle(ptr %awaiter, ptr %hdl, ptr @await_suspend_wrapper_handle)
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
   switch i8 %suspend2, label %ret [
     i8 0, label %step3
@@ -89,7 +88,7 @@ declare i1 @llvm.coro.alloc(token)
 declare ptr @llvm.coro.begin(token, ptr)
 declare void @llvm.coro.await.suspend.void(ptr, ptr, ptr)
 declare i1 @llvm.coro.await.suspend.bool(ptr, ptr, ptr)
-declare ptr @llvm.coro.await.suspend.handle(ptr, ptr, ptr)
+declare void @llvm.coro.await.suspend.handle(ptr, ptr, ptr)
 declare i1 @llvm.coro.end(ptr, i1, token)
 
 declare noalias ptr @malloc(i32)
diff --git a/llvm/test/Transforms/Coroutines/coro-preserve-final.ll b/llvm/test/Transforms/Coroutines/coro-preserve-final.ll
deleted file mode 100644
index 16eeb84e7915..000000000000
--- a/llvm/test/Transforms/Coroutines/coro-preserve-final.ll
+++ /dev/null
@@ -1,131 +0,0 @@
-; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
-
-%"struct.std::__n4861::noop_coroutine_promise" = type { i8 }
-%struct.Promise = type { %"struct.std::__n4861::coroutine_handle" }
-%"struct.std::__n4861::coroutine_handle" = type { ptr }
-
-define dso_local ptr @_Z5Outerv() #1 {
-entry:
-  %__promise = alloca %struct.Promise, align 8
-  %0 = call token @llvm.coro.id(i32 16, ptr nonnull %__promise, ptr nonnull @_Z5Outerv, ptr null)
-  %1 = call i1 @llvm.coro.alloc(token %0)
-  br i1 %1, label %coro.alloc, label %init.suspend
-
-coro.alloc:                                       ; preds = %entry
-  %2 = tail call i64 @llvm.coro.size.i64()
-  %call = call noalias noundef nonnull ptr @_Znwm(i64 noundef %2) #12
-  br label %init.suspend
-
-init.suspend:                                     ; preds = %entry, %coro.alloc
-  %3 = phi ptr [ null, %entry ], [ %call, %coro.alloc ]
-  %4 = call ptr @llvm.coro.begin(token %0, ptr %3) #13
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %__promise) #3
-  store ptr null, ptr %__promise, align 8
-  %5 = call token @llvm.coro.save(ptr null)
-  %6 = call i8 @llvm.coro.suspend(token %5, i1 false)
-  switch i8 %6, label %coro.ret [
-  i8 0, label %await.suspend
-  i8 1, label %cleanup62
-  ]
-
-await.suspend:                                    ; preds = %init.suspend
-  %7 = call token @llvm.coro.save(ptr null)
-  %8 = call ptr @llvm.coro.subfn.addr(ptr %4, i8 0)
-  call fastcc void %8(ptr %4) #3
-  %9 = call i8 @llvm.coro.suspend(token %7, i1 false)
-  switch i8 %9, label %coro.ret [
-  i8 0, label %await2.suspend
-  i8 1, label %cleanup62
-  ]
-
-await2.suspend:                                   ; preds = %await.suspend
-  %call27 = call ptr @_Z5Innerv() #3
-  %10 = call token @llvm.coro.save(ptr null)
-  %11 = getelementptr inbounds i8, ptr %__promise, i64 -16
-  store ptr %11, ptr %call27, align 8
-  %12 = getelementptr inbounds i8, ptr %call27, i64 -16
-  %13 = call ptr @llvm.coro.subfn.addr(ptr nonnull %12, i8 0)
-  call fastcc void %13(ptr nonnull %12) #3
-  %14 = call i8 @llvm.coro.suspend(token %10, i1 false)
-  switch i8 %14, label %coro.ret [
-  i8 0, label %final.suspend
-  i8 1, label %cleanup62
-  ]
-
-final.suspend:                                    ; preds = %await2.suspend
-  %15 = call ptr @llvm.coro.subfn.addr(ptr nonnull %12, i8 1)
-  call fastcc void %15(ptr nonnull %12) #3
-  %16 = call token @llvm.coro.save(ptr null)
-  %retval.sroa.0.0.copyload.i = load ptr, ptr %__promise, align 8
-  %17 = call ptr @llvm.coro.subfn.addr(ptr %retval.sroa.0.0.copyload.i, i8 0)
-  call fastcc void %17(ptr %retval.sroa.0.0.copyload.i) #3
-  %18 = call i8 @llvm.coro.suspend(token %16, i1 true) #13
-  switch i8 %18, label %coro.ret [
-  i8 0, label %final.ready
-  i8 1, label %cleanup62
-  ]
-
-final.ready:                                      ; preds = %final.suspend
-  call void @_Z5_exiti(i32 noundef 1) #14
-  unreachable
-
-cleanup62:                                        ; preds = %await2.suspend, %await.suspend, %init.suspend, %final.suspend
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %__promise) #3
-  %19 = call ptr @llvm.coro.free(token %0, ptr %4)
-  %.not = icmp eq ptr %19, null
-  br i1 %.not, label %coro.ret, label %coro.free
-
-coro.free:                                        ; preds = %cleanup62
-  call void @_ZdlPv(ptr noundef nonnull %19) #3
-  br label %coro.ret
-
-coro.ret:                                         ; preds = %coro.free, %cleanup62, %final.suspend, %await2.suspend, %await.suspend, %init.suspend
-  %20 = call i1 @llvm.coro.end(ptr null, i1 false, token none) #13
-  ret ptr %__promise
-}
-
-declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #2
-declare i1 @llvm.coro.alloc(token) #3
-declare dso_local noundef nonnull ptr @_Znwm(i64 noundef) local_unnamed_addr #4
-declare i64 @llvm.coro.size.i64() #5
-declare ptr @llvm.coro.begin(token, ptr writeonly) #3
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #6
-declare token @llvm.coro.save(ptr) #7
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #6
-declare i8 @llvm.coro.suspend(token, i1) #3
-declare dso_local ptr @_Z5Innerv() local_unnamed_addr #8
-declare dso_local void @_ZdlPv(ptr noundef) local_unnamed_addr #9
-declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
-declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare dso_local void @_Z5_exiti(i32 noundef) local_unnamed_addr #10
-declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #11
-
-attributes #0 = { mustprogress nounwind uwtable "frame-pointer"="none" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
-attributes #1 = { nounwind presplitcoroutine uwtable "frame-pointer"="none" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
-attributes #2 = { argmemonly nofree nounwind readonly }
-attributes #3 = { nounwind }
-attributes #4 = { nobuiltin allocsize(0) "frame-pointer"="none" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
-attributes #5 = { nofree nosync nounwind readnone }
-attributes #6 = { argmemonly mustprogress nocallback nofree nosync nounwind willreturn }
-attributes #7 = { nomerge nounwind }
-attributes #8 = { "frame-pointer"="none" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
-attributes #9 = { nobuiltin nounwind "frame-pointer"="none" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
-attributes #10 = { noreturn "frame-pointer"="none" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
-attributes #11 = { argmemonly nounwind readonly }
-attributes #12 = { nounwind allocsize(0) }
-attributes #13 = { noduplicate }
-attributes #14 = { noreturn nounwind }
-
-; CHECK: define{{.*}}@_Z5Outerv.resume(
-; CHECK: entry.resume:
-; CHECK: switch i2 %index
-; CHECK-NEXT:    i2 0, label %await2.suspend
-; CHECK-NEXT:    i2 1, label %final.suspend
-;
-; CHECK: await2.suspend:
-; CHECK: musttail call
-; CHECK-NEXT: ret void
-;
-; CHECK: final.suspend:
-; CHECK: musttail call
-; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail-chain-pgo-counter-promo.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail-chain-pgo-counter-promo.ll
index ddd293eed240..e2ed205f2c2f 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail-chain-pgo-counter-promo.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail-chain-pgo-counter-promo.ll
@@ -90,10 +90,7 @@ define ptr @f(i32 %0) presplitcoroutine align 32 {
   %25 = getelementptr inbounds { ptr, ptr }, ptr %5, i64 0, i32 1
   store ptr %24, ptr %25, align 8
   %26 = call token @llvm.coro.save(ptr null)
-  %27 = call ptr @await_transform_await_suspend(ptr noundef nonnull align 8 dereferenceable(16) %5, ptr %14)
-  %28 = call ptr @llvm.coro.subfn.addr(ptr %27, i8 0)
-  %29 = ptrtoint ptr %28 to i64
-  call fastcc void %28(ptr %27) #9
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_transform_await_suspend)
   %30 = call i8 @llvm.coro.suspend(token %26, i1 false)
   switch i8 %30, label %60 [
     i8 0, label %31
@@ -123,9 +120,7 @@ define ptr @f(i32 %0) presplitcoroutine align 32 {
   br i1 %42, label %43, label %46
 
 43:                                               ; preds = %36
-  %44 = call ptr @llvm.coro.subfn.addr(ptr nonnull %14, i8 1)
-  %45 = ptrtoint ptr %44 to i64
-  call fastcc void %44(ptr nonnull %14) #9
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_transform_await_suspend)
   br label %47
 
 46:                                               ; preds = %36
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail.ll
index 825e44471db2..70f29f4a9a4d 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail.ll
@@ -1,7 +1,6 @@
-; Tests that coro-split will convert coro.resume followed by a suspend to a
-; musttail call.
-; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,NOPGO %s
-; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,PGO %s
+; Tests that coro-split will convert coro.await.suspend.handle to a musttail call.
+; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK %s
+; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK %s
 
 define void @f() #0 {
 entry:
@@ -20,8 +19,7 @@ entry:
   ]
 await.ready:
   %save2 = call token @llvm.coro.save(ptr null)
-  %addr2 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0)
-  call fastcc void %addr2(ptr null)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_suspend_function)
 
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
   switch i8 %suspend2, label %exit [
@@ -40,10 +38,8 @@ exit:
 
 ; Verify that in the resume part resume call is marked with musttail.
 ; CHECK-LABEL: @f.resume(
-; CHECK: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0)
-; NOPGO-NEXT: musttail call fastcc void %[[addr2]](ptr null)
-; PGO: call void @llvm.instrprof
-; PGO-NEXT: musttail call fastcc void %[[addr2]](ptr null)
+; CHECK: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr
+; CHECK-NEXT: musttail call fastcc void %[[addr2]]
 ; CHECK-NEXT: ret void
 
 declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1
@@ -57,6 +53,7 @@ declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1
 declare i1 @llvm.coro.end(ptr, i1, token) #2
 declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1
 declare ptr @malloc(i64)
+declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine }
 attributes #1 = { argmemonly nounwind readonly }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail1.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail1.ll
index d0d11fc4495e..3edb8728d855 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail1.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail1.ll
@@ -1,7 +1,6 @@
-; Tests that coro-split will convert coro.resume followed by a suspend to a
-; musttail call.
-; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,NOPGO %s
-; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,PGO %s
+; Tests that coro-split will convert coro.await.suspend.handle to a musttail call.
+; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK %s
+; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK %s
 
 define void @f() #0 {
 entry:
@@ -28,17 +27,14 @@ await.suspend:
   ]
 await.resume1:
   %hdl = call ptr @g()
-  %addr2 = call ptr @llvm.coro.subfn.addr(ptr %hdl, i8 0)
-  call fastcc void %addr2(ptr %hdl)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr %hdl, ptr @await_suspend_function)
   br label %final.suspend
 await.resume2:
   %hdl2 = call ptr @h()
-  %addr3 = call ptr @llvm.coro.subfn.addr(ptr %hdl2, i8 0)
-  call fastcc void %addr3(ptr %hdl2)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr %hdl2, ptr @await_suspend_function)
   br label %final.suspend
 await.resume3:
-  %addr4 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0)
-  call fastcc void %addr4(ptr null)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_suspend_function)
   br label %final.suspend
 final.suspend:
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
@@ -63,18 +59,18 @@ unreach:
 ; Verify that in the resume part resume call is marked with musttail.
 ; CHECK-LABEL: @f.resume(
 ; CHECK: %[[hdl:.+]] = call ptr @g()
-; CHECK-NEXT: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl]], i8 0)
-; NOPGO-NEXT: musttail call fastcc void %[[addr2]](ptr %[[hdl]])
-; PGO: musttail call fastcc void %[[addr2]](ptr %[[hdl]])
+; CHECK-NEXT: call ptr @await_suspend_function
+; CHECK-NEXT: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr
+; CHECK-NEXT: musttail call fastcc void %[[addr2]]
 ; CHECK-NEXT: ret void
 ; CHECK: %[[hdl2:.+]] = call ptr @h()
-; CHECK-NEXT: %[[addr3:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl2]], i8 0)
-; NOPGO-NEXT: musttail call fastcc void %[[addr3]](ptr %[[hdl2]])
-; PGO: musttail call fastcc void %[[addr3]](ptr %[[hdl2]])
+; CHECK-NEXT: call ptr @await_suspend_function
+; CHECK-NEXT: %[[addr3:.+]] = call ptr @llvm.coro.subfn.addr
+; CHECK-NEXT: musttail call fastcc void %[[addr3]]
 ; CHECK-NEXT: ret void
-; CHECK: %[[addr4:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0)
-; NOPGO-NEXT: musttail call fastcc void %[[addr4]](ptr null)
-; PGO: musttail call fastcc void %[[addr4]](ptr null)
+; CHECK: call ptr @await_suspend_function
+; CHECK: %[[addr4:.+]] = call ptr @llvm.coro.subfn.addr
+; CHECK-NEXT: musttail call fastcc void %[[addr4]]
 ; CHECK-NEXT: ret void
 
 
@@ -93,6 +89,7 @@ declare ptr @malloc(i64)
 declare i8 @switch_result()
 declare ptr @g()
 declare ptr @h()
+declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine }
 attributes #1 = { argmemonly nounwind readonly }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail10.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail10.ll
index 3e91b79c10f7..a55b3d16e2de 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail10.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail10.ll
@@ -1,4 +1,4 @@
-; Tests that we would convert coro.resume to a musttail call if the target is
+; Tests that coro-split will convert coro.await.suspend.handle to a musttail call if the target is
 ; Wasm64 or Wasm32 with tail-call support.
 ; REQUIRES: webassembly-registered-target
 
@@ -25,8 +25,7 @@ entry:
   ]
 await.ready:
   %save2 = call token @llvm.coro.save(ptr null)
-  %addr2 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0)
-  call fastcc void %addr2(ptr null)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_suspend_function)
 
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
   switch i8 %suspend2, label %exit [
@@ -51,6 +50,7 @@ declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1
 declare i1 @llvm.coro.end(ptr, i1, token) #2
 declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1
 declare ptr @malloc(i64)
+declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine "target-features"="+tail-call" }
 attributes #1 = { argmemonly nounwind readonly }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail2.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail2.ll
index 2f27f79480ab..ca1611e19b9f 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail2.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail2.ll
@@ -1,5 +1,4 @@
-; Tests that coro-split will convert coro.resume followed by a suspend to a
-; musttail call.
+; Tests that coro-split will convert coro.await.suspend.handle to a musttail call.
 ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
 ; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
 
@@ -8,11 +7,6 @@ entry:
   ret void;
 }
 
-define void @fakeresume2(ptr align 8)  {
-entry:
-  ret void;
-}
-
 define void @g() #0 {
 entry:
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
@@ -29,7 +23,7 @@ entry:
   ]
 await.ready:
   %save2 = call token @llvm.coro.save(ptr null)
-  call fastcc void @fakeresume2(ptr align 8 null)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_suspend_function)
 
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
   switch i8 %suspend2, label %exit [
@@ -47,7 +41,9 @@ exit:
 
 ; Verify that in the resume part resume call is marked with musttail.
 ; CHECK-LABEL: @g.resume(
-; CHECK: musttail call fastcc void @fakeresume2(ptr align 8 null)
+; CHECK: call ptr @await_suspend_function
+; CHECK-NEXT: call ptr @llvm.coro.subfn.addr
+; CHECK-NEXT: musttail call fastcc void
 ; CHECK-NEXT: ret void
 
 declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1
@@ -61,6 +57,7 @@ declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1
 declare i1 @llvm.coro.end(ptr, i1, token) #2
 declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1
 declare ptr @malloc(i64)
+declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine }
 attributes #1 = { argmemonly nounwind readonly }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail3.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail3.ll
index 4778e3dcaf99..84cdac17beeb 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail3.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail3.ll
@@ -1,7 +1,6 @@
-; Tests that coro-split will convert coro.resume followed by a suspend to a
-; musttail call.
-; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,NOPGO %s
-; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK,PGO %s
+; Tests that coro-split will convert coro.await.suspend.handle to a musttail call.
+; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK %s
+; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck --check-prefixes=CHECK %s
 
 define void @f() #0 {
 entry:
@@ -26,17 +25,14 @@ await.suspend:
   ]
 await.resume1:
   %hdl = call ptr @g()
-  %addr2 = call ptr @llvm.coro.subfn.addr(ptr %hdl, i8 0)
-  call fastcc void %addr2(ptr %hdl)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr %hdl, ptr @await_suspend_function)
   br label %final.suspend
 await.resume2:
   %hdl2 = call ptr @h()
-  %addr3 = call ptr @llvm.coro.subfn.addr(ptr %hdl2, i8 0)
-  call fastcc void %addr3(ptr %hdl2)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr %hdl2, ptr @await_suspend_function)
   br label %final.suspend
 await.resume3:
-  %addr4 = call ptr @llvm.coro.subfn.addr(ptr null, i8 0)
-  call fastcc void %addr4(ptr null)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_suspend_function)
   br label %final.suspend
 final.suspend:
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
@@ -59,22 +55,21 @@ unreach:
 ; Verify that in the resume part resume call is marked with musttail.
 ; CHECK-LABEL: @f.resume(
 ; CHECK: %[[hdl:.+]] = call ptr @g()
-; CHECK-NEXT: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl]], i8 0)
-; NOPGO-NEXT: musttail call fastcc void %[[addr2]](ptr %[[hdl]])
-; PGO: musttail call fastcc void %[[addr2]](ptr %[[hdl]])
+; CHECK-NEXT: call ptr @await_suspend_function
+; CHECK-NEXT: %[[addr2:.+]] = call ptr @llvm.coro.subfn.addr
+; CHECK-NEXT: musttail call fastcc void %[[addr2]]
 ; CHECK-NEXT: ret void
 ; CHECK: %[[hdl2:.+]] = call ptr @h()
-; CHECK-NEXT: %[[addr3:.+]] = call ptr @llvm.coro.subfn.addr(ptr %[[hdl2]], i8 0)
-; NOPGO-NEXT: musttail call fastcc void %[[addr3]](ptr %[[hdl2]])
-; PGO: musttail call fastcc void %[[addr3]](ptr %[[hdl2]])
+; CHECK-NEXT: call ptr @await_suspend_function
+; CHECK-NEXT: %[[addr3:.+]] = call ptr @llvm.coro.subfn.addr
+; CHECK-NEXT: musttail call fastcc void %[[addr3]]
 ; CHECK-NEXT: ret void
-; CHECK: %[[addr4:.+]] = call ptr @llvm.coro.subfn.addr(ptr null, i8 0)
-; NOPGO-NEXT: musttail call fastcc void %[[addr4]](ptr null)
-; PGO: musttail call fastcc void %[[addr4]](ptr null)
+; CHECK: call ptr @await_suspend_function
+; CHECK: %[[addr4:.+]] = call ptr @llvm.coro.subfn.addr
+; CHECK-NEXT: musttail call fastcc void %[[addr4]]
 ; CHECK-NEXT: ret void
 
 
-
 declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1
 declare i1 @llvm.coro.alloc(token) #2
 declare i64 @llvm.coro.size.i64() #3
@@ -89,6 +84,7 @@ declare ptr @malloc(i64)
 declare i8 @switch_result()
 declare ptr @g()
 declare ptr @h()
+declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine }
 attributes #1 = { argmemonly nounwind readonly }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail4.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail4.ll
index 00ee422ce586..b647bd2e4a20 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail4.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail4.ll
@@ -1,5 +1,4 @@
-; Tests that coro-split will convert a call before coro.suspend to a musttail call
-; while the user of the coro.suspend is a icmpinst.
+; Tests that coro-split will convert coro.await.suspend.handle to a musttail call.
 ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
 ; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
 
@@ -24,7 +23,7 @@ entry:
 await.ready:
   %save2 = call token @llvm.coro.save(ptr null)
 
-  call fastcc void @fakeresume1(ptr align 8 null)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_suspend_function)
   %suspend = call i8 @llvm.coro.suspend(token %save2, i1 true)
   %switch = icmp ult i8 %suspend, 2
   br i1 %switch, label %cleanup, label %coro.end
@@ -44,7 +43,7 @@ coro.end:
 }
 
 ; CHECK-LABEL: @f.resume(
-; CHECK:          musttail call fastcc void @fakeresume1(
+; CHECK:          musttail call fastcc void
 ; CHECK-NEXT:     ret void
 
 declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1
@@ -59,6 +58,7 @@ declare i1 @llvm.coro.end(ptr, i1, token) #2
 declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1
 declare ptr @malloc(i64)
 declare void @delete(ptr nonnull) #2
+declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine }
 attributes #1 = { argmemonly nounwind readonly }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll
index 9afc79abbe88..7c1a13fd83ce 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll
@@ -1,5 +1,4 @@
-; Tests that sinked lifetime markers wouldn't provent optimization
-; to convert a resuming call to a musttail call.
+; Tests that coro-split will convert coro.await.suspend.handle to a musttail call.
 ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
 ; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
 
@@ -22,7 +21,7 @@ entry:
   ]
 await.suspend:
   %save2 = call token @llvm.coro.save(ptr null)
-  call fastcc void @fakeresume1(ptr align 8 null)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_suspend_function)
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
   switch i8 %suspend2, label %exit [
     i8 0, label %await.ready
@@ -39,7 +38,7 @@ exit:
 
 ; Verify that in the resume part resume call is marked with musttail.
 ; CHECK-LABEL: @g.resume(
-; CHECK:          musttail call fastcc void @fakeresume1(ptr align 8 null)
+; CHECK:          musttail call fastcc void
 ; CHECK-NEXT:     ret void
 
 declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1
@@ -56,6 +55,7 @@ declare ptr @malloc(i64)
 declare void @consume(ptr)
 declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
 declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine }
 attributes #1 = { argmemonly nounwind readonly }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll
index d9dba92ec4eb..e05169a72916 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll
@@ -1,5 +1,5 @@
 ; Tests that sinked lifetime markers wouldn't provent optimization
-; to convert a resuming call to a musttail call.
+; to convert a coro.await.suspend.handle call to a musttail call.
 ; The difference between this and coro-split-musttail5.ll is that there is
 ; an extra bitcast instruction in the path, which makes it harder to
 ; optimize.
@@ -25,7 +25,7 @@ entry:
   ]
 await.suspend:
   %save2 = call token @llvm.coro.save(ptr null)
-  call fastcc void @fakeresume1(ptr align 8 null)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_suspend_function)
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
   switch i8 %suspend2, label %exit [
     i8 0, label %await.ready
@@ -42,7 +42,7 @@ exit:
 
 ; Verify that in the resume part resume call is marked with musttail.
 ; CHECK-LABEL: @g.resume(
-; CHECK:      musttail call fastcc void @fakeresume1(ptr align 8 null)
+; CHECK:      musttail call fastcc void
 ; CHECK-NEXT: ret void
 
 ; It has a cleanup bb.
@@ -63,7 +63,7 @@ entry:
   ]
 await.suspend:
   %save2 = call token @llvm.coro.save(ptr null)
-  call fastcc void @fakeresume1(ptr align 8 null)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_suspend_function)
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
   switch i8 %suspend2, label %exit [
     i8 0, label %await.ready
@@ -90,7 +90,7 @@ exit:
 
 ; Verify that in the resume part resume call is marked with musttail.
 ; CHECK-LABEL: @f.resume(
-; CHECK:      musttail call fastcc void @fakeresume1(ptr align 8 null)
+; CHECK:      musttail call fastcc void
 ; CHECK-NEXT: ret void
 
 declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1
@@ -108,6 +108,7 @@ declare void @delete(ptr nonnull) #2
 declare void @consume(ptr)
 declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
 declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine }
 attributes #1 = { argmemonly nounwind readonly }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll
index d0d5005587bd..8ceb0dda94f6 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll
@@ -1,13 +1,11 @@
 ; Tests that sinked lifetime markers wouldn't provent optimization
-; to convert a resuming call to a musttail call.
+; to convert a coro.await.suspend.handle call to a musttail call.
 ; The difference between this and coro-split-musttail5.ll and coro-split-musttail6.ll
 ; is that this contains dead instruction generated during the transformation,
 ; which makes the optimization harder.
 ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
 ; RUN: opt < %s -passes='pgo-instr-gen,cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
 
-declare void @fakeresume1(ptr align 8)
-
 define i64 @g() #0 {
 entry:
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
@@ -25,7 +23,7 @@ entry:
   ]
 await.suspend:
   %save2 = call token @llvm.coro.save(ptr null)
-  call fastcc void @fakeresume1(ptr align 8 null)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_suspend_function)
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
 
   ; These (non-trivially) dead instructions are in the way.
@@ -48,7 +46,9 @@ exit:
 
 ; Verify that in the resume part resume call is marked with musttail.
 ; CHECK-LABEL: @g.resume(
-; CHECK:         musttail call fastcc void @fakeresume1(ptr align 8 null)
+; CHECK:         %[[FRAME:[0-9]+]] = call ptr @await_suspend_function(ptr null, ptr null)
+; CHECK:         %[[RESUMEADDR:[0-9]+]] = call ptr @llvm.coro.subfn.addr(ptr %[[FRAME]], i8 0)
+; CHECK:         musttail call fastcc void %[[RESUMEADDR]](ptr %[[FRAME]])
 ; CHECK-NEXT:    ret void
 
 ; It has a cleanup bb.
@@ -69,7 +69,7 @@ entry:
   ]
 await.suspend:
   %save2 = call token @llvm.coro.save(ptr null)
-  call fastcc void @fakeresume1(ptr align 8 null)
+  call void @llvm.coro.await.suspend.handle(ptr null, ptr null, ptr @await_suspend_function)
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
   switch i8 %suspend2, label %exit [
     i8 0, label %await.ready
@@ -96,7 +96,9 @@ exit:
 
 ; Verify that in the resume part resume call is marked with musttail.
 ; CHECK-LABEL: @f.resume(
-; CHECK:         musttail call fastcc void @fakeresume1(ptr align 8 null)
+; CHECK:         %[[FRAME:[0-9]+]] = call ptr @await_suspend_function(ptr null, ptr null)
+; CHECK:         %[[RESUMEADDR:[0-9]+]] = call ptr @llvm.coro.subfn.addr(ptr %[[FRAME]], i8 0)
+; CHECK:         musttail call fastcc void %[[RESUMEADDR]](ptr %[[FRAME]])
 ; CHECK-NEXT:    ret void
 
 declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1
@@ -114,6 +116,7 @@ declare void @delete(ptr nonnull) #2
 declare void @consume(ptr)
 declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
 declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine }
 attributes #1 = { argmemonly nounwind readonly }
-- 
GitLab


From 95e307caeb17c080724921564d96e1b8457264bc Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 15 May 2024 08:07:17 -0500
Subject: [PATCH 0853/1206] Fix bolt build with -DBUILD_SHARED_LIBS=ON after
 71fbbb69d

Commit 71fbbb69d63c461f391cbabf1e32cd9977c4ce68 moved getGUID out of
line in llvm/IR/GlobalValue, now users have to link LLVMCore to have
the definition of it.

/usr/bin/ld: CMakeFiles/LLVMBOLTRewrite.dir/PseudoProbeRewriter.cpp.o: in function `(anonymous namespace)::PseudoProbeRewriter::parsePseudoProbe()':
PseudoProbeRewriter.cpp:(.text._ZN12_GLOBAL__N_119PseudoProbeRewriter16parsePseudoProbeEv+0x3d0): undefined reference to `llvm::GlobalValue::getGUID(llvm::StringRef)'
/usr/bin/ld: CMakeFiles/LLVMBOLTRewrite.dir/PseudoProbeRewriter.cpp.o: in function `(anonymous namespace)::PseudoProbeRewriter::encodePseudoProbes()':
PseudoProbeRewriter.cpp:(.text._ZN12_GLOBAL__N_119PseudoProbeRewriter18encodePseudoProbesEv+0x11a1): undefined reference to `llvm::GlobalValue::getGUID(llvm::StringRef)'
collect2: error: ld returned 1 exit status
make[2]: *** [tools/bolt/lib/Rewrite/CMakeFiles/LLVMBOLTRewrite.dir/build.make:275: lib/libLLVMBOLTRewrite.so.19.0git] Error 1
---
 bolt/lib/Rewrite/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bolt/lib/Rewrite/CMakeLists.txt b/bolt/lib/Rewrite/CMakeLists.txt
index 6890f52e2b28..578f1763bfe4 100644
--- a/bolt/lib/Rewrite/CMakeLists.txt
+++ b/bolt/lib/Rewrite/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
+  Core
   DebugInfoDWARF
   DWP
   JITLink
-- 
GitLab


From c2fba6df944e11e2c9a7073405c6a817fdba14e3 Mon Sep 17 00:00:00 2001
From: Koakuma <koachan@protonmail.com>
Date: Wed, 15 May 2024 20:49:28 +0700
Subject: [PATCH 0854/1206] [clang][SPARC] Treat empty structs as if it's a
 one-bit type in the CC (#90338)

Make sure that empty structs are treated as if it has a size of one bit
in function parameters and return types so that it occupies a full
argument and/or return register slot.

This fixes crashes and miscompilations when passing and/or returning
empty structs.

Reviewed by: @s-barannikov
---
 clang/lib/CodeGen/Targets/Sparc.cpp         |  5 ++-
 clang/test/CodeGen/sparcv9-abi.c            | 41 +++++++++++++++++++++
 clang/test/CodeGen/sparcv9-class-return.cpp | 24 ++++++++++++
 3 files changed, 69 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/CodeGen/sparcv9-class-return.cpp

diff --git a/clang/lib/CodeGen/Targets/Sparc.cpp b/clang/lib/CodeGen/Targets/Sparc.cpp
index 9025a633f328..b82e9a69e196 100644
--- a/clang/lib/CodeGen/Targets/Sparc.cpp
+++ b/clang/lib/CodeGen/Targets/Sparc.cpp
@@ -263,7 +263,10 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const {
 
   CoerceBuilder CB(getVMContext(), getDataLayout());
   CB.addStruct(0, StrTy);
-  CB.pad(llvm::alignTo(CB.DL.getTypeSizeInBits(StrTy), 64));
+  // All structs, even empty ones, should take up a register argument slot,
+  // so pin the minimum struct size to one bit.
+  CB.pad(llvm::alignTo(
+      std::max(CB.DL.getTypeSizeInBits(StrTy).getKnownMinValue(), 1UL), 64));
 
   // Try to use the original type for coercion.
   llvm::Type *CoerceTy = CB.isUsableType(StrTy) ? StrTy : CB.getType();
diff --git a/clang/test/CodeGen/sparcv9-abi.c b/clang/test/CodeGen/sparcv9-abi.c
index 5e74a9a883ce..616e24e7c519 100644
--- a/clang/test/CodeGen/sparcv9-abi.c
+++ b/clang/test/CodeGen/sparcv9-abi.c
@@ -21,6 +21,47 @@ char f_int_4(char x) { return x; }
 // CHECK-LABEL: define{{.*}} fp128 @f_ld(fp128 noundef %x)
 long double f_ld(long double x) { return x; }
 
+// Zero-sized structs reserves an argument register slot if passed directly.
+struct empty {};
+struct emptyarr { struct empty a[10]; };
+
+// CHECK-LABEL: define{{.*}} i64 @f_empty(i64 %x.coerce)
+struct empty f_empty(struct empty x) { return x; }
+
+// CHECK-LABEL: define{{.*}} i64 @f_emptyarr(i64 %x.coerce)
+struct empty f_emptyarr(struct emptyarr x) { return x.a[0]; }
+
+// CHECK-LABEL: define{{.*}} i64 @f_emptyvar(i32 noundef zeroext %count, ...)
+long f_emptyvar(unsigned count, ...) {
+    long ret;
+    va_list args;
+    va_start(args, count);
+
+// CHECK: %[[CUR:[^ ]+]] = load ptr, ptr %args
+// CHECK-DAG: %[[NXT:[^ ]+]] = getelementptr inbounds i8, ptr %[[CUR]], i64 8
+// CHECK-DAG: store ptr %[[NXT]], ptr %args
+    va_arg(args, struct empty);
+
+// CHECK: %[[CUR:[^ ]+]] = load ptr, ptr %args
+// CHECK-DAG: %[[NXT:[^ ]+]] = getelementptr inbounds i8, ptr %[[CUR]], i64 8
+// CHECK-DAG: store ptr %[[NXT]], ptr %args
+// CHECK-DAG: load i64, ptr %[[CUR]]
+    ret = va_arg(args, long);
+    va_end(args);
+    return ret;
+}
+
+// If the zero-sized struct is contained in a non-zero-sized struct,
+// though, it doesn't reserve any registers.
+struct emptymixed { struct empty a; long b; };
+struct emptyflex { unsigned count; struct empty data[10]; };
+
+// CHECK-LABEL: define{{.*}} i64 @f_emptymixed(i64 %x.coerce)
+long f_emptymixed(struct emptymixed x) { return x.b; }
+
+// CHECK-LABEL: define{{.*}} i64 @f_emptyflex(i64 %x.coerce, i64 noundef %y)
+long f_emptyflex(struct emptyflex x, long y) { return y; }
+
 // Small structs are passed in registers.
 struct small {
   int *a, *b;
diff --git a/clang/test/CodeGen/sparcv9-class-return.cpp b/clang/test/CodeGen/sparcv9-class-return.cpp
new file mode 100644
index 000000000000..2428219422d8
--- /dev/null
+++ b/clang/test/CodeGen/sparcv9-class-return.cpp
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple sparcv9-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+
+class Empty {
+};
+
+class Long : public Empty {
+public:
+  long l;
+};
+
+// CHECK: define{{.*}} i64 @_Z4foo15Empty(i64 %e.coerce)
+Empty foo1(Empty e) {
+  return e;
+}
+
+// CHECK: define{{.*}} %class.Long @_Z4foo24Long(i64 %l.coerce)
+Long foo2(Long l) {
+  return l;
+}
+
+// CHECK: define{{.*}} i64 @_Z4foo34Long(i64 %l.coerce)
+long foo3(Long l) {
+  return l.l;
+}
-- 
GitLab


From 97a30448f9477e0196f9340303aa20d544f1629a Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 15 May 2024 08:57:45 -0500
Subject: [PATCH 0855/1206] [flang][OpenMP] Add `private` to `allocate` in
 parallel-sections.f90 (#92185)

Add a privatizing clause to the construct that uses `allocate` clause.
Amend the CHECK lines to reflect the expected output.
---
 flang/test/Lower/OpenMP/parallel-sections.f90 | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/flang/test/Lower/OpenMP/parallel-sections.f90 b/flang/test/Lower/OpenMP/parallel-sections.f90
index 2f78dd4562b0..285102e06cad 100644
--- a/flang/test/Lower/OpenMP/parallel-sections.f90
+++ b/flang/test/Lower/OpenMP/parallel-sections.f90
@@ -39,13 +39,10 @@ end subroutine omp_parallel_sections
 subroutine omp_parallel_sections_allocate(x, y)
   use omp_lib
   integer, intent(inout) :: x, y
+  !CHECK: omp.parallel
   !CHECK: %[[allocator_1:.*]] = arith.constant 4 : i64
-  !CHECK: %[[allocator_2:.*]] = arith.constant 4 : i64
-  !CHECK: omp.parallel allocate(
-  !CHECK: %[[allocator_2]] : i64 -> %{{.*}} : !fir.ref<i32>) {
-  !CHECK: omp.sections allocate(
-  !CHECK: %[[allocator_1]] : i64 -> %{{.*}} : !fir.ref<i32>) {
-  !$omp parallel sections allocate(omp_high_bw_mem_alloc: x)
+  !CHECK: omp.sections allocate(%[[allocator_1]] : i64 -> %{{.*}} : !fir.ref<i32>) {
+  !$omp parallel sections allocate(omp_high_bw_mem_alloc: x) private(x, y)
     !CHECK: omp.section {
     !$omp section
       x = x + 12
-- 
GitLab


From 4ec4a8e7fe463852e197d4ff396f4911ccce7449 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 15 May 2024 09:04:07 -0500
Subject: [PATCH 0856/1206] [Frontend][OpenMP] Privatizing clauses in construct
 decomposition (#92176)

Add remaining clauses with the "privatizing" property to construct
decomposition, specifically to the part handling the `allocate` clause.

---------

Co-authored-by: Tom Eccles <t@freedommail.info>
---
 .../Frontend/OpenMP/ConstructCompositionT.h   |  20 ++-
 .../Frontend/OpenMP/ConstructDecompositionT.h |   5 +
 .../Frontend/OpenMPDecompositionTest.cpp      | 123 ++++++++++++++++--
 3 files changed, 139 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
index 9dcb115a0c51..f6ee963bd885 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructCompositionT.h
@@ -343,13 +343,31 @@ template <typename C> void ConstructCompositionT<C>::mergeDSA() {
     }
   }
 
-  // Check reductions as well, clear "shared" if set.
+  // Check other privatizing clauses as well, clear "shared" if set.
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_in_reduction]) {
+    using InReductionTy = tomp::clause::InReductionT<TypeTy, IdTy, ExprTy>;
+    using ListTy = typename InReductionTy::List;
+    for (auto &object : std::get<ListTy>(std::get<InReductionTy>(clause.u).t))
+      getDsa(object).second &= ~DSA::Shared;
+  }
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_linear]) {
+    using LinearTy = tomp::clause::LinearT<TypeTy, IdTy, ExprTy>;
+    using ListTy = typename LinearTy::List;
+    for (auto &object : std::get<ListTy>(std::get<LinearTy>(clause.u).t))
+      getDsa(object).second &= ~DSA::Shared;
+  }
   for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_reduction]) {
     using ReductionTy = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
     using ListTy = typename ReductionTy::List;
     for (auto &object : std::get<ListTy>(std::get<ReductionTy>(clause.u).t))
       getDsa(object).second &= ~DSA::Shared;
   }
+  for (auto &clause : clauseSets[llvm::omp::Clause::OMPC_task_reduction]) {
+    using TaskReductionTy = tomp::clause::TaskReductionT<TypeTy, IdTy, ExprTy>;
+    using ListTy = typename TaskReductionTy::List;
+    for (auto &object : std::get<ListTy>(std::get<TaskReductionTy>(clause.u).t))
+      getDsa(object).second &= ~DSA::Shared;
+  }
 
   tomp::ListT<ObjectTy> privateObj, sharedObj, firstpObj, lastpObj, lastpcObj;
   for (auto &[object, dsa] : objectDsa) {
diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
index 5f12c62b832f..02c88a58e099 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
@@ -793,9 +793,14 @@ bool ConstructDecompositionT<C, H>::applyClause(
   // [5.2:340:33]
   auto canMakePrivateCopy = [](llvm::omp::Clause id) {
     switch (id) {
+    // Clauses with "privatization" property:
     case llvm::omp::Clause::OMPC_firstprivate:
+    case llvm::omp::Clause::OMPC_in_reduction:
     case llvm::omp::Clause::OMPC_lastprivate:
+    case llvm::omp::Clause::OMPC_linear:
     case llvm::omp::Clause::OMPC_private:
+    case llvm::omp::Clause::OMPC_reduction:
+    case llvm::omp::Clause::OMPC_task_reduction:
       return true;
     default:
       return false;
diff --git a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
index df48e9cc0ff4..8157e41e833a 100644
--- a/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPDecompositionTest.cpp
@@ -288,6 +288,14 @@ std::string stringify(const omp::DirectiveWithClauses &DWC) {
 
 // --- Tests ----------------------------------------------------------
 
+namespace red {
+// Make it easier to construct reduction operators from built-in intrinsics.
+omp::clause::ReductionOperator
+makeOp(omp::clause::DefinedOperator::IntrinsicOperator Op) {
+  return omp::clause::ReductionOperator{omp::clause::DefinedOperator{Op}};
+}
+} // namespace red
+
 namespace {
 using namespace llvm::omp;
 
@@ -699,6 +707,92 @@ TEST_F(OpenMPDecompositionTest, Order1) {
 TEST_F(OpenMPDecompositionTest, Allocate1) {
   omp::Object x{"x"};
 
+  // Allocate + firstprivate
+  omp::List<omp::Clause> Clauses{
+      {OMPC_allocate,
+       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_firstprivate, omp::clause::Firstprivate{{x}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel shared(x)");                           // (33)
+  ASSERT_EQ(Dir1, "sections firstprivate(x) allocate(, , , (x))"); // (33)
+}
+
+TEST_F(OpenMPDecompositionTest, Allocate2) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  // Allocate + in_reduction
+  omp::List<omp::Clause> Clauses{
+      {OMPC_allocate,
+       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_in_reduction, omp::clause::InReduction{{{Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_target_parallel,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "target in_reduction((3), (x)) allocate(, , , (x))"); // (33)
+  ASSERT_EQ(Dir1, "parallel");                                          // (33)
+}
+
+TEST_F(OpenMPDecompositionTest, Allocate3) {
+  omp::Object x{"x"};
+
+  // Allocate + linear
+  omp::List<omp::Clause> Clauses{
+      {OMPC_allocate,
+       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_linear,
+       omp::clause::Linear{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_for,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  // The "shared" clause is duplicated---this isn't harmful, but it
+  // should be fixed eventually.
+  ASSERT_EQ(Dir0, "parallel shared(x) shared(x)"); // (33)
+  ASSERT_EQ(Dir1, "for linear(, , , (x)) firstprivate(x) lastprivate(, (x)) "
+                  "allocate(, , , (x))"); // (33)
+}
+
+TEST_F(OpenMPDecompositionTest, Allocate4) {
+  omp::Object x{"x"};
+
+  // Allocate + lastprivate
+  omp::List<omp::Clause> Clauses{
+      {OMPC_allocate,
+       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_lastprivate, omp::clause::Lastprivate{{std::nullopt, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel shared(x)");                              // (33)
+  ASSERT_EQ(Dir1, "sections lastprivate(, (x)) allocate(, , , (x))"); // (33)
+}
+
+TEST_F(OpenMPDecompositionTest, Allocate5) {
+  omp::Object x{"x"};
+
+  // Allocate + private
   omp::List<omp::Clause> Clauses{
       {OMPC_allocate,
        omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
@@ -715,6 +809,27 @@ TEST_F(OpenMPDecompositionTest, Allocate1) {
   ASSERT_EQ(Dir1, "sections private(x) allocate(, , , (x))"); // (33)
 }
 
+TEST_F(OpenMPDecompositionTest, Allocate6) {
+  omp::Object x{"x"};
+  auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
+
+  // Allocate + reduction
+  omp::List<omp::Clause> Clauses{
+      {OMPC_allocate,
+       omp::clause::Allocate{{std::nullopt, std::nullopt, std::nullopt, {x}}}},
+      {OMPC_reduction, omp::clause::Reduction{{std::nullopt, {Add}, {x}}}},
+  };
+
+  omp::ConstructDecomposition Dec(AnyVersion, Helper, OMPD_parallel_sections,
+                                  Clauses);
+  ASSERT_EQ(Dec.output.size(), 2u);
+
+  std::string Dir0 = stringify(Dec.output[0]);
+  std::string Dir1 = stringify(Dec.output[1]);
+  ASSERT_EQ(Dir0, "parallel shared(x)");                                 // (33)
+  ASSERT_EQ(Dir1, "sections reduction(, (3), (x)) allocate(, , , (x))"); // (33)
+}
+
 // REDUCTION
 // [5.2:134:17-18]
 // Directives: do, for, loop, parallel, scope, sections, simd, taskloop, teams
@@ -741,14 +856,6 @@ TEST_F(OpenMPDecompositionTest, Allocate1) {
 // clause on the construct, then the effect is as if the list item in the
 // reduction clause appears as a list item in a map clause with a map-type of
 // tofrom.
-namespace red {
-// Make is easier to construct reduction operators from built-in intrinsics.
-omp::clause::ReductionOperator
-makeOp(omp::clause::DefinedOperator::IntrinsicOperator Op) {
-  return omp::clause::ReductionOperator{omp::clause::DefinedOperator{Op}};
-}
-} // namespace red
-
 TEST_F(OpenMPDecompositionTest, Reduction1) {
   omp::Object x{"x"};
   auto Add = red::makeOp(omp::clause::DefinedOperator::IntrinsicOperator::Add);
-- 
GitLab


From 8d386c63a8d38bc50acba8dba2cd5f0daca57012 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 15 May 2024 09:05:02 -0500
Subject: [PATCH 0857/1206] [Frontend][OpenMP] Reduction modifier must be
 applied somewhere (#92160)

Detect the case when a reduction modifier ends up not being applied
after construct decomposition, treat it as an error.

This fixes a regression in the gfortran test suite after PR90098.
---
 .../Lower/OpenMP/invalid-reduction-modifier.f90    | 14 ++++++++++++++
 .../llvm/Frontend/OpenMP/ConstructDecompositionT.h |  3 ++-
 2 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Lower/OpenMP/invalid-reduction-modifier.f90

diff --git a/flang/test/Lower/OpenMP/invalid-reduction-modifier.f90 b/flang/test/Lower/OpenMP/invalid-reduction-modifier.f90
new file mode 100644
index 000000000000..817c5b731c62
--- /dev/null
+++ b/flang/test/Lower/OpenMP/invalid-reduction-modifier.f90
@@ -0,0 +1,14 @@
+!Remove the --crash below once we can diagnose the issue more gracefully.
+!RUN: not --crash %flang_fc1 -fopenmp -emit-hlfir -o - %s
+
+! Check that we reject the "task" reduction modifier on the "simd" directive.
+
+subroutine fred(x)
+  integer, intent(inout) :: x
+
+  !$omp simd reduction(task, +:x)
+  do i = 1, 100
+    x = foo(i)
+  enddo
+  !$omp end simd
+end
diff --git a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
index 02c88a58e099..3fa27608ead9 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ConstructDecompositionT.h
@@ -935,7 +935,8 @@ bool ConstructDecompositionT<C, H>::applyClause(
       // Apply clause without modifier.
       leaf.clauses.push_back(unmodified);
     }
-    applied = true;
+    // The modifier must be applied to some construct.
+    applied = effectiveApplied;
   }
 
   if (!applied)
-- 
GitLab


From 7c8176ebd39c357fc4fa488861318409cd87d8f2 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Wed, 15 May 2024 22:08:17 +0800
Subject: [PATCH 0858/1206] [Coroutines] Remove unused function (NFC)

llvm-project/llvm/lib/Transforms/Coroutines/CoroSplit.cpp:1223:1:
error: unused function 'scanPHIsAndUpdateValueMap' [-Werror,-Wunused-function]
scanPHIsAndUpdateValueMap(Instruction *Prev, BasicBlock *NewBlock,
^
1 error generated.
---
 llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 450ea8234371..1d9cf185b75a 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -1217,23 +1217,6 @@ static void postSplitCleanup(Function &F) {
 #endif
 }
 
-// Assuming we arrived at the block NewBlock from Prev instruction, store
-// PHI's incoming values in the ResolvedValues map.
-static void
-scanPHIsAndUpdateValueMap(Instruction *Prev, BasicBlock *NewBlock,
-                          DenseMap<Value *, Value *> &ResolvedValues) {
-  auto *PrevBB = Prev->getParent();
-  for (PHINode &PN : NewBlock->phis()) {
-    auto V = PN.getIncomingValueForBlock(PrevBB);
-    // See if we already resolved it.
-    auto VI = ResolvedValues.find(V);
-    if (VI != ResolvedValues.end())
-      V = VI->second;
-    // Remember the value.
-    ResolvedValues[&PN] = V;
-  }
-}
-
 // Coroutine has no suspend points. Remove heap allocation for the coroutine
 // frame if possible.
 static void handleNoSuspendCoroutine(coro::Shape &Shape) {
-- 
GitLab


From 466d266945196ebbdefd8d72f654551d54d68600 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Wed, 15 May 2024 15:13:53 +0100
Subject: [PATCH 0859/1206] [AMDGPU] Fix GFX90x check prefixes in tests
 (#92254)

---
 llvm/test/CodeGen/AMDGPU/agpr-register-count.ll | 6 ++----
 llvm/test/CodeGen/AMDGPU/spill-vgpr.ll          | 4 ++--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll b/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll
index f84d476fc122..bd5dc6e20709 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll
@@ -157,10 +157,8 @@ declare void @undef_func()
 ; GFX908: .amdhsa_next_free_vgpr 32
 ; GFX90A: .amdhsa_next_free_vgpr 64
 ; GFX90A: .amdhsa_accum_offset 32
-; GCN908: NumVgprs: 128
-; GCN908: NumAgprs: 128
-; GCN90A: NumVgprs: 256
-; GCN90A: NumAgprs: 256
+; GCN:    NumVgprs: 32
+; GCN:    NumAgprs: 32
 ; GFX908: TotalNumVgprs: 32
 ; GFX90A: TotalNumVgprs: 64
 ; GFX908: VGPRBlocks: 7
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
index 9eacb88066c0..1cc5b7f7d14e 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
@@ -88,8 +88,8 @@ define amdgpu_kernel void @max_10_vgprs_spill_v32(ptr addrspace(1) %p) #0 {
 ; GFX900: ScratchSize: 132
 ; GFX908: NumVgprs: 252
 ; GFX908: ScratchSize: 0
-; GCN900:    VGPRBlocks: 63
-; GCN908:    VGPRBlocks: 62
+; GFX900:    VGPRBlocks: 63
+; GFX908:    VGPRBlocks: 62
 ; GFX900:    NumVGPRsForWavesPerEU: 256
 ; GFX908:    NumVGPRsForWavesPerEU: 252
 define amdgpu_kernel void @max_256_vgprs_spill_9x32(ptr addrspace(1) %p) #1 {
-- 
GitLab


From 61da6366d043792d7db280ce9edd2db62516e0e8 Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer@amd.com>
Date: Wed, 15 May 2024 15:20:27 +0100
Subject: [PATCH 0860/1206] [flang] Initial debug info support for local
 variables. (#90905)

We need the information in the `DeclareOp` to generate debug information
for variables.  Currently, cg-rewrite removes the `DeclareOp`. As
`AddDebugInfo` runs after that, it cannot process the `DeclareOp`. My
initial plan was to make the `AddDebugInfo` pass run before the cg-rewrite
but that has few issues.

1. Initially I was thinking to use the memref op to carry the variable
attr. But as @tblah suggested in the #86939, it makes more sense to
carry that information on `DeclareOp`. It also makes it easy to handle
it in codegen and there is no special handling needed for arguments. For
this reason, we need to preserve the `DeclareOp` till the codegen.

2. Running earlier, we will miss the changes in passes that run between
cg-rewrite and codegen.

But not removing the DeclareOp in cg-rewrite has the issue that ShapeOp
remains and it causes errors during codegen. To solve this problem, I
convert DeclareOp to XDeclareOp in cg-rewrite instead of removing
it. This was mentioned as possible solution by @jeanPerier in
https://reviews.llvm.org/D136254

The conversion follows similar logic as used for other operators in that
file. The FortranAttr and CudaAttr are currently not converted but left
as TODO when the need arise.

Now `AddDebugInfo` pass can extracts information about local variables
from `XDeclareOp` and creates `DILocalVariableAttr`. These are attached
to `XDeclareOp` using `FusedLoc` approach. Codegen can use them to
create `DbgDeclareOp`.  I have added tests that checks the debug
information in mlir from and also in llvm ir.

Currently we only handle very limited types. Rest are given a place
holder type. The previous placeholder type was basic type with
`DW_ATE_address` encoding. When variables are added, it started
causing assertions in the llvm debug info generation logic for some
types. It has been changed to an interger type to prevent these issues
until we handle those types properly.
---
 .../flang}/Optimizer/CodeGen/CGOps.h          |  1 +
 .../include/flang/Optimizer/CodeGen/CGOps.td  | 34 +++++++
 .../flang/Optimizer/CodeGen/CGPasses.td       |  4 +
 .../include/flang/Optimizer/CodeGen/CodeGen.h |  6 +-
 flang/include/flang/Tools/CLOptions.inc       | 11 ++-
 flang/lib/Optimizer/CodeGen/CGOps.cpp         |  2 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 50 +++++++---
 flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp  | 49 ++++++++--
 .../lib/Optimizer/Transforms/AddDebugInfo.cpp | 56 +++++++++++-
 .../Transforms/DebugTypeGenerator.cpp         | 10 +-
 flang/test/Fir/declare-codegen.fir            | 22 +++--
 flang/test/Fir/dummy-scope-codegen.fir        | 11 ++-
 flang/test/Transforms/debug-local-var-2.f90   | 91 +++++++++++++++++++
 flang/test/Transforms/debug-local-var.f90     | 54 +++++++++++
 14 files changed, 354 insertions(+), 47 deletions(-)
 rename flang/{lib => include/flang}/Optimizer/CodeGen/CGOps.h (94%)
 create mode 100644 flang/test/Transforms/debug-local-var-2.f90
 create mode 100644 flang/test/Transforms/debug-local-var.f90

diff --git a/flang/lib/Optimizer/CodeGen/CGOps.h b/flang/include/flang/Optimizer/CodeGen/CGOps.h
similarity index 94%
rename from flang/lib/Optimizer/CodeGen/CGOps.h
rename to flang/include/flang/Optimizer/CodeGen/CGOps.h
index b5a6d5bb9a9e..df909d9ee81c 100644
--- a/flang/lib/Optimizer/CodeGen/CGOps.h
+++ b/flang/include/flang/Optimizer/CodeGen/CGOps.h
@@ -13,6 +13,7 @@
 #ifndef OPTIMIZER_CODEGEN_CGOPS_H
 #define OPTIMIZER_CODEGEN_CGOPS_H
 
+#include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 
diff --git a/flang/include/flang/Optimizer/CodeGen/CGOps.td b/flang/include/flang/Optimizer/CodeGen/CGOps.td
index 35e70fa2ffa3..c375edee1fa7 100644
--- a/flang/include/flang/Optimizer/CodeGen/CGOps.td
+++ b/flang/include/flang/Optimizer/CodeGen/CGOps.td
@@ -16,6 +16,8 @@
 
 include "mlir/IR/SymbolInterfaces.td"
 include "flang/Optimizer/Dialect/FIRTypes.td"
+include "flang/Optimizer/Dialect/FIRAttr.td"
+include "mlir/IR/BuiltinAttributes.td"
 
 def fircg_Dialect : Dialect {
   let name = "fircg";
@@ -202,4 +204,36 @@ def fircg_XArrayCoorOp : fircg_Op<"ext_array_coor", [AttrSizedOperandSegments]>
   }];
 }
 
+// Extended Declare operation.
+def fircg_XDeclareOp : fircg_Op<"ext_declare", [AttrSizedOperandSegments]> {
+  let summary = "for internal conversion only";
+
+  let description = [{
+    Prior to lowering to LLVM IR dialect, a DeclareOp will
+    be converted to an extended DeclareOp.
+  }];
+
+  let arguments = (ins
+    AnyRefOrBox:$memref,
+    Variadic<AnyIntegerType>:$shape,
+    Variadic<AnyIntegerType>:$shift,
+    Variadic<AnyIntegerType>:$typeparams,
+    Optional<fir_DummyScopeType>:$dummy_scope,
+    Builtin_StringAttr:$uniq_name
+  );
+  let results = (outs AnyRefOrBox);
+
+  let assemblyFormat = [{
+    $memref (`(` $shape^ `)`)? (`origin` $shift^)? (`typeparams` $typeparams^)?
+    (`dummy_scope` $dummy_scope^)?
+    attr-dict `:` functional-type(operands, results)
+  }];
+
+  let extraClassDeclaration = [{
+    // Shape is optional, but if it exists, it will be at offset 1.
+    unsigned shapeOffset() { return 1; }
+    unsigned shiftOffset() { return shapeOffset() + getShape().size(); }
+  }];
+}
+
 #endif
diff --git a/flang/include/flang/Optimizer/CodeGen/CGPasses.td b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
index f524fb423734..565920e55e6a 100644
--- a/flang/include/flang/Optimizer/CodeGen/CGPasses.td
+++ b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
@@ -47,6 +47,10 @@ def CodeGenRewrite : Pass<"cg-rewrite", "mlir::ModuleOp"> {
   let dependentDialects = [
     "fir::FIROpsDialect", "fir::FIRCodeGenDialect"
   ];
+  let options = [
+    Option<"preserveDeclare", "preserve-declare", "bool", /*default=*/"false",
+           "Preserve DeclareOp during pre codegen re-write.">
+  ];
   let statistics = [
     Statistic<"numDCE", "num-dce'd", "Number of operations eliminated">
   ];
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 26097dabf56c..4d2b191b46d0 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -30,7 +30,8 @@ struct NameUniquer;
 
 /// Prerequiste pass for code gen. Perform intermediate rewrites to perform
 /// the code gen (to LLVM-IR dialect) conversion.
-std::unique_ptr<mlir::Pass> createFirCodeGenRewritePass();
+std::unique_ptr<mlir::Pass> createFirCodeGenRewritePass(
+    CodeGenRewriteOptions Options = CodeGenRewriteOptions{});
 
 /// FirTargetRewritePass options.
 struct TargetRewriteOptions {
@@ -88,7 +89,8 @@ void populateFIRToLLVMConversionPatterns(fir::LLVMTypeConverter &converter,
                                          fir::FIRToLLVMPassOptions &options);
 
 /// Populate the pattern set with the PreCGRewrite patterns.
-void populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns);
+void populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns,
+                                  bool preserveDeclare);
 
 // declarative passes
 #define GEN_PASS_REGISTRATION
diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc
index cc3431d5b71d..761315e0abc8 100644
--- a/flang/include/flang/Tools/CLOptions.inc
+++ b/flang/include/flang/Tools/CLOptions.inc
@@ -169,9 +169,11 @@ inline void addMemoryAllocationOpt(mlir::PassManager &pm) {
 }
 
 #if !defined(FLANG_EXCLUDE_CODEGEN)
-inline void addCodeGenRewritePass(mlir::PassManager &pm) {
-  addPassConditionally(
-      pm, disableCodeGenRewrite, fir::createFirCodeGenRewritePass);
+inline void addCodeGenRewritePass(mlir::PassManager &pm, bool preserveDeclare) {
+  fir::CodeGenRewriteOptions options;
+  options.preserveDeclare = preserveDeclare;
+  addPassConditionally(pm, disableCodeGenRewrite,
+      [&]() { return fir::createFirCodeGenRewritePass(options); });
 }
 
 inline void addTargetRewritePass(mlir::PassManager &pm) {
@@ -353,7 +355,8 @@ inline void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
     MLIRToLLVMPassPipelineConfig config, llvm::StringRef inputFilename = {}) {
   fir::addBoxedProcedurePass(pm);
   addNestedPassToAllTopLevelOperations(pm, fir::createAbstractResultOpt);
-  fir::addCodeGenRewritePass(pm);
+  fir::addCodeGenRewritePass(
+      pm, (config.DebugInfo != llvm::codegenoptions::NoDebugInfo));
   fir::addTargetRewritePass(pm);
   fir::addExternalNameConversionPass(pm, config.Underscoring);
   fir::createDebugPasses(pm, config.DebugInfo, config.OptLevel, inputFilename);
diff --git a/flang/lib/Optimizer/CodeGen/CGOps.cpp b/flang/lib/Optimizer/CodeGen/CGOps.cpp
index 44d07d26dd2b..6b8ba7452555 100644
--- a/flang/lib/Optimizer/CodeGen/CGOps.cpp
+++ b/flang/lib/Optimizer/CodeGen/CGOps.cpp
@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CGOps.h"
+#include "flang/Optimizer/CodeGen/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 21154902d23f..72172f63888e 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -12,7 +12,7 @@
 
 #include "flang/Optimizer/CodeGen/CodeGen.h"
 
-#include "CGOps.h"
+#include "flang/Optimizer/CodeGen/CGOps.h"
 #include "flang/Optimizer/CodeGen/CodeGenOpenMP.h"
 #include "flang/Optimizer/CodeGen/FIROpPatterns.h"
 #include "flang/Optimizer/CodeGen/TypeConverter.h"
@@ -170,6 +170,28 @@ genAllocationScaleSize(OP op, mlir::Type ity,
   return nullptr;
 }
 
+namespace {
+struct DeclareOpConversion : public fir::FIROpConversion<fir::cg::XDeclareOp> {
+public:
+  using FIROpConversion::FIROpConversion;
+  mlir::LogicalResult
+  matchAndRewrite(fir::cg::XDeclareOp declareOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    auto memRef = adaptor.getOperands()[0];
+    if (auto fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(declareOp.getLoc())) {
+      if (auto varAttr =
+              mlir::dyn_cast_or_null<mlir::LLVM::DILocalVariableAttr>(
+                  fusedLoc.getMetadata())) {
+        rewriter.create<mlir::LLVM::DbgDeclareOp>(memRef.getLoc(), memRef,
+                                                  varAttr, nullptr);
+      }
+    }
+    rewriter.replaceOp(declareOp, memRef);
+    return mlir::success();
+  }
+};
+} // namespace
+
 namespace {
 /// convert to LLVM IR dialect `alloca`
 struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
@@ -3714,19 +3736,19 @@ void fir::populateFIRToLLVMConversionPatterns(
       BoxOffsetOpConversion, BoxProcHostOpConversion, BoxRankOpConversion,
       BoxTypeCodeOpConversion, BoxTypeDescOpConversion, CallOpConversion,
       CmpcOpConversion, ConstcOpConversion, ConvertOpConversion,
-      CoordinateOpConversion, DTEntryOpConversion, DivcOpConversion,
-      EmboxOpConversion, EmboxCharOpConversion, EmboxProcOpConversion,
-      ExtractValueOpConversion, FieldIndexOpConversion, FirEndOpConversion,
-      FreeMemOpConversion, GlobalLenOpConversion, GlobalOpConversion,
-      HasValueOpConversion, InsertOnRangeOpConversion, InsertValueOpConversion,
-      IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
-      MulcOpConversion, NegcOpConversion, NoReassocOpConversion,
-      SelectCaseOpConversion, SelectOpConversion, SelectRankOpConversion,
-      SelectTypeOpConversion, ShapeOpConversion, ShapeShiftOpConversion,
-      ShiftOpConversion, SliceOpConversion, StoreOpConversion,
-      StringLitOpConversion, SubcOpConversion, TypeDescOpConversion,
-      TypeInfoOpConversion, UnboxCharOpConversion, UnboxProcOpConversion,
-      UndefOpConversion, UnreachableOpConversion,
+      CoordinateOpConversion, DTEntryOpConversion, DeclareOpConversion,
+      DivcOpConversion, EmboxOpConversion, EmboxCharOpConversion,
+      EmboxProcOpConversion, ExtractValueOpConversion, FieldIndexOpConversion,
+      FirEndOpConversion, FreeMemOpConversion, GlobalLenOpConversion,
+      GlobalOpConversion, HasValueOpConversion, InsertOnRangeOpConversion,
+      InsertValueOpConversion, IsPresentOpConversion, LenParamIndexOpConversion,
+      LoadOpConversion, MulcOpConversion, NegcOpConversion,
+      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
+      ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
+      StoreOpConversion, StringLitOpConversion, SubcOpConversion,
+      TypeDescOpConversion, TypeInfoOpConversion, UnboxCharOpConversion,
+      UnboxProcOpConversion, UndefOpConversion, UnreachableOpConversion,
       UnrealizedConversionCastOpConversion, XArrayCoorOpConversion,
       XEmboxOpConversion, XReboxOpConversion, ZeroOpConversion>(converter,
                                                                 options);
diff --git a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
index 5bd3ec8d1845..c54a7457db76 100644
--- a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
@@ -12,8 +12,8 @@
 
 #include "flang/Optimizer/CodeGen/CodeGen.h"
 
-#include "CGOps.h"
 #include "flang/Optimizer/Builder/Todo.h" // remove when TODO's are done
+#include "flang/Optimizer/CodeGen/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
@@ -270,13 +270,43 @@ public:
 };
 
 class DeclareOpConversion : public mlir::OpRewritePattern<fir::DeclareOp> {
+  bool preserveDeclare;
+
 public:
   using OpRewritePattern::OpRewritePattern;
+  DeclareOpConversion(mlir::MLIRContext *ctx, bool preserveDecl)
+      : OpRewritePattern(ctx), preserveDeclare(preserveDecl) {}
 
   mlir::LogicalResult
   matchAndRewrite(fir::DeclareOp declareOp,
                   mlir::PatternRewriter &rewriter) const override {
-    rewriter.replaceOp(declareOp, declareOp.getMemref());
+    if (!preserveDeclare) {
+      rewriter.replaceOp(declareOp, declareOp.getMemref());
+      return mlir::success();
+    }
+    auto loc = declareOp.getLoc();
+    llvm::SmallVector<mlir::Value> shapeOpers;
+    llvm::SmallVector<mlir::Value> shiftOpers;
+    if (auto shapeVal = declareOp.getShape()) {
+      if (auto shapeOp = mlir::dyn_cast<fir::ShapeOp>(shapeVal.getDefiningOp()))
+        populateShape(shapeOpers, shapeOp);
+      else if (auto shiftOp =
+                   mlir::dyn_cast<fir::ShapeShiftOp>(shapeVal.getDefiningOp()))
+        populateShapeAndShift(shapeOpers, shiftOpers, shiftOp);
+      else if (auto shiftOp =
+                   mlir::dyn_cast<fir::ShiftOp>(shapeVal.getDefiningOp()))
+        populateShift(shiftOpers, shiftOp);
+      else
+        return mlir::failure();
+    }
+    // FIXME: Add FortranAttrs and CudaAttrs
+    auto xDeclOp = rewriter.create<fir::cg::XDeclareOp>(
+        loc, declareOp.getType(), declareOp.getMemref(), shapeOpers, shiftOpers,
+        declareOp.getTypeparams(), declareOp.getDummyScope(),
+        declareOp.getUniqName());
+    LLVM_DEBUG(llvm::dbgs()
+               << "rewriting " << declareOp << " to " << xDeclOp << '\n');
+    rewriter.replaceOp(declareOp, xDeclOp.getOperation()->getResults());
     return mlir::success();
   }
 };
@@ -297,6 +327,7 @@ public:
 
 class CodeGenRewrite : public fir::impl::CodeGenRewriteBase<CodeGenRewrite> {
 public:
+  CodeGenRewrite(fir::CodeGenRewriteOptions opts) : Base(opts) {}
   void runOnOperation() override final {
     mlir::ModuleOp mod = getOperation();
 
@@ -314,7 +345,7 @@ public:
                    mlir::cast<fir::BaseBoxType>(embox.getType()).getEleTy()));
     });
     mlir::RewritePatternSet patterns(&context);
-    fir::populatePreCGRewritePatterns(patterns);
+    fir::populatePreCGRewritePatterns(patterns, preserveDeclare);
     if (mlir::failed(
             mlir::applyPartialConversion(mod, target, std::move(patterns)))) {
       mlir::emitError(mlir::UnknownLoc::get(&context),
@@ -330,12 +361,14 @@ public:
 
 } // namespace
 
-std::unique_ptr<mlir::Pass> fir::createFirCodeGenRewritePass() {
-  return std::make_unique<CodeGenRewrite>();
+std::unique_ptr<mlir::Pass>
+fir::createFirCodeGenRewritePass(fir::CodeGenRewriteOptions Options) {
+  return std::make_unique<CodeGenRewrite>(Options);
 }
 
-void fir::populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns) {
+void fir::populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns,
+                                       bool preserveDeclare) {
   patterns.insert<EmboxConversion, ArrayCoorConversion, ReboxConversion,
-                  DeclareOpConversion, DummyScopeOpConversion>(
-      patterns.getContext());
+                  DummyScopeOpConversion>(patterns.getContext());
+  patterns.add<DeclareOpConversion>(patterns.getContext(), preserveDeclare);
 }
diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 908c8fc96f63..cfad366cb5cb 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -15,6 +15,7 @@
 #include "flang/Common/Version.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/CodeGen/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
@@ -45,13 +46,59 @@ namespace fir {
 namespace {
 
 class AddDebugInfoPass : public fir::impl::AddDebugInfoBase<AddDebugInfoPass> {
+  void handleDeclareOp(fir::cg::XDeclareOp declOp,
+                       mlir::LLVM::DIFileAttr fileAttr,
+                       mlir::LLVM::DIScopeAttr scopeAttr,
+                       fir::DebugTypeGenerator &typeGen);
+
 public:
   AddDebugInfoPass(fir::AddDebugInfoOptions options) : Base(options) {}
   void runOnOperation() override;
 };
 
+static uint32_t getLineFromLoc(mlir::Location loc) {
+  uint32_t line = 1;
+  if (auto fileLoc = mlir::dyn_cast<mlir::FileLineColLoc>(loc))
+    line = fileLoc.getLine();
+  return line;
+}
+
 } // namespace
 
+void AddDebugInfoPass::handleDeclareOp(fir::cg::XDeclareOp declOp,
+                                       mlir::LLVM::DIFileAttr fileAttr,
+                                       mlir::LLVM::DIScopeAttr scopeAttr,
+                                       fir::DebugTypeGenerator &typeGen) {
+  mlir::MLIRContext *context = &getContext();
+  mlir::OpBuilder builder(context);
+  auto result = fir::NameUniquer::deconstruct(declOp.getUniqName());
+
+  if (result.first != fir::NameUniquer::NameKind::VARIABLE)
+    return;
+
+  // Only accept local variables.
+  if (result.second.procs.empty())
+    return;
+
+  // FIXME: There may be cases where an argument is processed a bit before
+  // DeclareOp is generated. In that case, DeclareOp may point to an
+  // intermediate op and not to BlockArgument. We need to find those cases and
+  // walk the chain to get to the actual argument.
+
+  unsigned argNo = 0;
+  if (auto Arg = llvm::dyn_cast<mlir::BlockArgument>(declOp.getMemref()))
+    argNo = Arg.getArgNumber() + 1;
+
+  auto tyAttr = typeGen.convertType(fir::unwrapRefType(declOp.getType()),
+                                    fileAttr, scopeAttr, declOp.getLoc());
+
+  auto localVarAttr = mlir::LLVM::DILocalVariableAttr::get(
+      context, scopeAttr, mlir::StringAttr::get(context, result.second.name),
+      fileAttr, getLineFromLoc(declOp.getLoc()), argNo, /* alignInBits*/ 0,
+      tyAttr);
+  declOp->setLoc(builder.getFusedLoc({declOp->getLoc()}, localVarAttr));
+}
+
 void AddDebugInfoPass::runOnOperation() {
   mlir::ModuleOp module = getOperation();
   mlir::MLIRContext *context = &getContext();
@@ -144,14 +191,15 @@ void AddDebugInfoPass::runOnOperation() {
       subprogramFlags =
           subprogramFlags | mlir::LLVM::DISubprogramFlags::Definition;
     }
-    unsigned line = 1;
-    if (auto funcLoc = mlir::dyn_cast<mlir::FileLineColLoc>(l))
-      line = funcLoc.getLine();
-
+    unsigned line = getLineFromLoc(l);
     auto spAttr = mlir::LLVM::DISubprogramAttr::get(
         context, id, compilationUnit, fileAttr, funcName, fullName,
         funcFileAttr, line, line, subprogramFlags, subTypeAttr);
     funcOp->setLoc(builder.getFusedLoc({funcOp->getLoc()}, spAttr));
+
+    funcOp.walk([&](fir::cg::XDeclareOp declOp) {
+      handleDeclareOp(declOp, fileAttr, spAttr, typeGen);
+    });
   });
 }
 
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index e5b4050dfb24..64c6547e06e0 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -24,11 +24,6 @@ DebugTypeGenerator::DebugTypeGenerator(mlir::ModuleOp m)
   LLVM_DEBUG(llvm::dbgs() << "DITypeAttr generator\n");
 }
 
-static mlir::LLVM::DITypeAttr genPlaceholderType(mlir::MLIRContext *context) {
-  return mlir::LLVM::DIBasicTypeAttr::get(
-      context, llvm::dwarf::DW_TAG_base_type, "void", 32, 1);
-}
-
 static mlir::LLVM::DITypeAttr genBasicType(mlir::MLIRContext *context,
                                            mlir::StringAttr name,
                                            unsigned bitSize,
@@ -37,6 +32,11 @@ static mlir::LLVM::DITypeAttr genBasicType(mlir::MLIRContext *context,
       context, llvm::dwarf::DW_TAG_base_type, name, bitSize, decoding);
 }
 
+static mlir::LLVM::DITypeAttr genPlaceholderType(mlir::MLIRContext *context) {
+  return genBasicType(context, mlir::StringAttr::get(context, "integer"), 32,
+                      llvm::dwarf::DW_ATE_signed);
+}
+
 mlir::LLVM::DITypeAttr
 DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
                                 mlir::LLVM::DIScopeAttr scope,
diff --git a/flang/test/Fir/declare-codegen.fir b/flang/test/Fir/declare-codegen.fir
index 9d68d3b2f9d4..c5879facb157 100644
--- a/flang/test/Fir/declare-codegen.fir
+++ b/flang/test/Fir/declare-codegen.fir
@@ -1,5 +1,7 @@
 // Test rewrite of fir.declare. The result is replaced by the memref operand.
-// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s
+// RUN: fir-opt --cg-rewrite="preserve-declare=true" %s -o - | FileCheck %s --check-prefixes DECL
+// RUN: fir-opt --cg-rewrite="preserve-declare=false" %s -o - | FileCheck %s --check-prefixes NODECL
+// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s --check-prefixes NODECL
 
 
 func.func @test(%arg0: !fir.ref<!fir.array<12x23xi32>>) {
@@ -15,9 +17,14 @@ func.func @test(%arg0: !fir.ref<!fir.array<12x23xi32>>) {
 func.func private @bar(%arg0: !fir.ref<!fir.array<12x23xi32>>)
 
 
-// CHECK-LABEL: func.func @test(
-// CHECK-SAME: %[[arg0:.*]]: !fir.ref<!fir.array<12x23xi32>>) {
-// CHECK-NEXT: fir.call @bar(%[[arg0]]) : (!fir.ref<!fir.array<12x23xi32>>) -> ()
+// NODECL-LABEL: func.func @test(
+// NODECL-SAME: %[[arg0:.*]]: !fir.ref<!fir.array<12x23xi32>>) {
+// NODECL-NEXT: fir.call @bar(%[[arg0]]) : (!fir.ref<!fir.array<12x23xi32>>) -> ()
+
+// DECL-LABEL: func.func @test(
+// DECL-SAME: %[[arg0:.*]]: !fir.ref<!fir.array<12x23xi32>>) {
+// DECL: fircg.ext_declare
+
 
 func.func @useless_shape_with_duplicate_extent_operand(%arg0: !fir.ref<!fir.array<3x3xf32>>) {
   %c3 = arith.constant 3 : index
@@ -26,5 +33,8 @@ func.func @useless_shape_with_duplicate_extent_operand(%arg0: !fir.ref<!fir.arra
   return
 }
 
-// CHECK-LABEL: func.func @useless_shape_with_duplicate_extent_operand(
-// CHECK-NEXT: return
+// NODECL-LABEL: func.func @useless_shape_with_duplicate_extent_operand(
+// NODECL-NEXT: return
+
+// DECL-LABEL: func.func @useless_shape_with_duplicate_extent_operand(
+// DECL: fircg.ext_declare
diff --git a/flang/test/Fir/dummy-scope-codegen.fir b/flang/test/Fir/dummy-scope-codegen.fir
index caef3c1b2578..2e5c091552a7 100644
--- a/flang/test/Fir/dummy-scope-codegen.fir
+++ b/flang/test/Fir/dummy-scope-codegen.fir
@@ -1,9 +1,14 @@
-// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s
+// RUN: fir-opt --cg-rewrite="preserve-declare=true" %s -o - | FileCheck %s --check-prefixes DECL
+// RUN: fir-opt --cg-rewrite="preserve-declare=false" %s -o - | FileCheck %s --check-prefixes NODECL
+// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s  --check-prefixes NODECL
 
 func.func @dummy_scope(%arg0: !fir.ref<f32>) {
   %scope = fir.dummy_scope : !fir.dscope
   %0 = fir.declare %arg0 dummy_scope %scope {uniq_name = "x"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
   return
 }
-// CHECK-LABEL: func.func @dummy_scope(
-// CHECK-NEXT: return
+// DECL-LABEL: func.func @dummy_scope(
+// DECL: fircg.ext_declare
+
+// NODECL-LABEL: func.func @dummy_scope(
+// NODECL-NEXT: return
\ No newline at end of file
diff --git a/flang/test/Transforms/debug-local-var-2.f90 b/flang/test/Transforms/debug-local-var-2.f90
new file mode 100644
index 000000000000..15b9b148492e
--- /dev/null
+++ b/flang/test/Transforms/debug-local-var-2.f90
@@ -0,0 +1,91 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck %s
+
+! This tests checks the debug information for local variables in llvm IR.
+
+! CHECK-LABEL: define void @_QQmain
+! CHECK-DAG: %[[AL11:.*]] = alloca i32
+! CHECK-DAG: %[[AL12:.*]] = alloca i64
+! CHECK-DAG: %[[AL13:.*]] = alloca i8
+! CHECK-DAG: %[[AL14:.*]] = alloca i32
+! CHECK-DAG: %[[AL15:.*]] = alloca float
+! CHECK-DAG: %[[AL16:.*]] = alloca double
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL11]], metadata ![[I4:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL12]], metadata ![[I8:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL13]], metadata ![[L1:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL14]], metadata ![[L4:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL15]], metadata ![[R4:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL16]], metadata ![[R8:.*]], metadata !DIExpression())
+! CHECK-LABEL: }
+
+! CHECK-LABEL: define {{.*}}i64 @_QFPfn1
+! CHECK-SAME: (ptr %[[ARG1:.*]], ptr %[[ARG2:.*]], ptr %[[ARG3:.*]])
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG1]], metadata ![[A1:.*]], metadata !DIExpression())
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG2]], metadata ![[B1:.*]], metadata !DIExpression())
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG3]], metadata ![[C1:.*]], metadata !DIExpression())
+! CHECK-DAG: %[[AL2:.*]] = alloca i64
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[AL2]], metadata ![[RES1:.*]], metadata !DIExpression())
+! CHECK-LABEL: }
+
+! CHECK-LABEL: define {{.*}}i32 @_QFPfn2
+! CHECK-SAME: (ptr %[[FN2ARG1:.*]], ptr %[[FN2ARG2:.*]], ptr %[[FN2ARG3:.*]])
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG1]], metadata ![[A2:.*]], metadata !DIExpression())
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG2]], metadata ![[B2:.*]], metadata !DIExpression())
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG3]], metadata ![[C2:.*]], metadata !DIExpression())
+! CHECK-DAG: %[[AL3:.*]] = alloca i32
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[AL3]], metadata ![[RES2:.*]], metadata !DIExpression())
+! CHECK-LABEL: }
+
+program mn
+! CHECK-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "_QQmain", {{.*}})
+
+! CHECK-DAG: ![[TYI32:.*]] = !DIBasicType(name: "integer", size: 32, encoding: DW_ATE_signed)
+! CHECK-DAG: ![[TYI64:.*]] = !DIBasicType(name: "integer", size: 64, encoding: DW_ATE_signed)
+! CHECK-DAG: ![[TYL8:.*]]  = !DIBasicType(name: "logical", size: 8, encoding: DW_ATE_boolean)
+! CHECK-DAG: ![[TYL32:.*]] = !DIBasicType(name: "logical", size: 32, encoding: DW_ATE_boolean)
+! CHECK-DAG: ![[TYR32:.*]] = !DIBasicType(name: "real", size: 32, encoding: DW_ATE_float)
+! CHECK-DAG: ![[TYR64:.*]] = !DIBasicType(name: "real", size: 64, encoding: DW_ATE_float)
+
+! CHECK-DAG: ![[I4]] = !DILocalVariable(name: "i4", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYI32]])
+! CHECK-DAG: ![[I8]] = !DILocalVariable(name: "i8", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYI64]])
+! CHECK-DAG: ![[R4]] = !DILocalVariable(name: "r4", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYR32]])
+! CHECK-DAG: ![[R8]] = !DILocalVariable(name: "r8", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYR64]])
+! CHECK-DAG: ![[L1]] = !DILocalVariable(name: "l1", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYL8]])
+! CHECK-DAG: ![[L4]] = !DILocalVariable(name: "l4", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYL32]])
+  integer(kind=4) :: i4
+  integer(kind=8) :: i8
+  real(kind=4) :: r4
+  real(kind=8) :: r8
+  logical(kind=1) :: l1
+  logical(kind=4) :: l4
+
+  i8 = fn1(i4, r8, l1)
+  i4 = fn2(i8, r4, l4)
+contains
+! CHECK-DAG: ![[FN1:.*]] = distinct !DISubprogram(name: "fn1", {{.*}})
+! CHECK-DAG: ![[A1]] = !DILocalVariable(name: "a1", arg: 1, scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI32]])
+! CHECK-DAG: ![[B1]] = !DILocalVariable(name: "b1", arg: 2, scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYR64]])
+! CHECK-DAG: ![[C1]] = !DILocalVariable(name: "c1", arg: 3, scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYL8]])
+! CHECK-DAG: ![[RES1]] = !DILocalVariable(name: "res1", scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI64]])
+  function fn1(a1, b1, c1) result (res1)
+    integer(kind=4), intent(in) :: a1
+    real(kind=8), intent(in) :: b1
+    logical(kind=1), intent(in) :: c1
+    integer(kind=8) :: res1
+
+    res1 = a1 + b1
+  end function
+
+! CHECK-DAG: ![[FN2:.*]] = distinct !DISubprogram(name: "fn2", {{.*}})
+! CHECK-DAG: ![[A2]] = !DILocalVariable(name: "a2", arg: 1, scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI64]])
+! CHECK-DAG: ![[B2]] = !DILocalVariable(name: "b2", arg: 2, scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYR32]])
+! CHECK-DAG: ![[C2]] = !DILocalVariable(name: "c2", arg: 3, scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYL32]])
+! CHECK-DAG: ![[RES2]] = !DILocalVariable(name: "res2", scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI32]])
+  function fn2(a2, b2, c2) result (res2)
+    integer(kind=8), intent(in) :: a2
+    real(kind=4), intent(in) :: b2
+    logical(kind=4), intent(in) :: c2
+    integer(kind=4) :: res2
+
+    res2 = a2 + b2
+  end function
+end program
diff --git a/flang/test/Transforms/debug-local-var.f90 b/flang/test/Transforms/debug-local-var.f90
new file mode 100644
index 000000000000..96dc111ad308
--- /dev/null
+++ b/flang/test/Transforms/debug-local-var.f90
@@ -0,0 +1,54 @@
+! RUN: %flang_fc1 -emit-fir -debug-info-kind=standalone -mmlir --mlir-print-debuginfo %s -o - | \
+! RUN: fir-opt --cg-rewrite="preserve-declare=true" --mlir-print-debuginfo | fir-opt --add-debug-info --mlir-print-debuginfo | FileCheck %s
+
+! CHECK-DAG: #[[INT8:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 64, encoding = DW_ATE_signed>
+! CHECK-DAG: #[[INT4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 32, encoding = DW_ATE_signed>
+! CHECK-DAG: #[[REAL8:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 64, encoding = DW_ATE_float>
+! CHECK-DAG: #[[LOG1:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "logical", sizeInBits = 8, encoding = DW_ATE_boolean>
+! CHECK-DAG: #[[REAL4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 32, encoding = DW_ATE_float>
+! CHECK-DAG: #[[LOG4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "logical", sizeInBits = 32, encoding = DW_ATE_boolean>
+! CHECK-DAG: #[[MAIN:.*]] = #llvm.di_subprogram<{{.*}}name = "_QQmain"{{.*}}>
+! CHECK-DAG: #[[FN1:.*]] = #llvm.di_subprogram<{{.*}}name = "fn1"{{.*}}>
+! CHECK-DAG: #[[FN2:.*]] = #llvm.di_subprogram<{{.*}}name = "fn2"{{.*}}>
+
+program mn
+! CHECK-DAG: #[[I4:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "i4", file = #{{.*}}, line = [[@LINE+6]], type = #[[INT4]]>
+! CHECK-DAG: #[[I8:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "i8", file = #{{.*}}, line = [[@LINE+6]], type = #[[INT8]]>
+! CHECK-DAG: #[[R4:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "r4", file = #{{.*}}, line = [[@LINE+6]], type = #[[REAL4]]>
+! CHECK-DAG: #[[R8:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "r8", file = #{{.*}}, line = [[@LINE+6]], type = #[[REAL8]]>
+! CHECK-DAG: #[[L1:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "l1", file = #{{.*}}, line = [[@LINE+6]], type = #[[LOG1]]>
+! CHECK-DAG: #[[L4:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "l4", file = #{{.*}}, line = [[@LINE+6]], type = #[[LOG4]]>
+  integer(kind=4) :: i4
+  integer(kind=8) :: i8
+  real(kind=4) :: r4
+  real(kind=8) :: r8
+  logical(kind=1) :: l1
+  logical(kind=4) :: l4
+  i8 = fn1(i4, r8, l1)
+  i4 = fn2(i8, r4, l4)
+contains
+  function fn1(a1, b1, c1) result (res1)
+! CHECK-DAG: #[[A1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "a1", file = #{{.*}}, line = [[@LINE+4]], arg = 1, type = #[[INT4]]>
+! CHECK-DAG: #[[B1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "b1", file = #{{.*}}, line = [[@LINE+4]], arg = 2, type = #[[REAL8]]>
+! CHECK-DAG: #[[C1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "c1", file = #{{.*}}, line = [[@LINE+4]], arg = 3, type = #[[LOG1]]>
+! CHECK-DAG: #[[RES1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "res1", file = #{{.*}}, line = [[@LINE+4]], type = #[[INT8]]>
+    integer(kind=4), intent(in) :: a1
+    real(kind=8), intent(in) :: b1
+    logical(kind=1), intent(in) :: c1
+    integer(kind=8) :: res1
+    res1 = a1 + b1
+  end function
+
+  function fn2(a2, b2, c2) result (res2)
+    implicit none
+! CHECK-DAG: #[[A2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "a2", file = #{{.*}}, line = [[@LINE+4]], arg = 1, type = #[[INT8]]>
+! CHECK-DAG: #[[B2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "b2", file = #{{.*}}, line = [[@LINE+4]], arg = 2, type = #[[REAL4]]>
+! CHECK-DAG: #[[C2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "c2", file = #{{.*}}, line = [[@LINE+4]], arg = 3, type = #[[LOG4]]>
+! CHECK-DAG: #[[RES2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "res2", file = #{{.*}}, line = [[@LINE+4]], type = #[[INT4]]>
+    integer(kind=8), intent(in) :: a2
+    real(kind=4), intent(in) :: b2
+    logical(kind=4), intent(in) :: c2
+    integer(kind=4) :: res2
+    res2 = a2 + b2
+  end function
+end program
-- 
GitLab


From eda098aadea3e542f95b5f0d4173f00eae42dc72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Endre=20F=C3=BCl=C3=B6p?= <endre.fulop@sigmatechnology.com>
Date: Wed, 15 May 2024 16:26:17 +0200
Subject: [PATCH 0861/1206] [clang][analyzer] Fix a crash in
 alpha.unix.BlockInCriticalSection (#90030)

When analyzing C code with function pointers the checker crashes because
of how the implementation extracts `IdentifierInfo`. Without the fix, this
test crashes.
---
 .../Checkers/BlockInCriticalSectionChecker.cpp              | 5 ++---
 clang/test/Analysis/block-in-critical-section.c             | 6 ++++++
 2 files changed, 8 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/Analysis/block-in-critical-section.c

diff --git a/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp
index e138debd1361..92347f8fafc0 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp
@@ -103,9 +103,8 @@ class RAIIMutexDescriptor {
       // this function is called instead of early returning it. To avoid this, a
       // bool variable (IdentifierInfoInitialized) is used and the function will
       // be run only once.
-      Guard = &Call.getCalleeAnalysisDeclContext()->getASTContext().Idents.get(
-          GuardName);
-      IdentifierInfoInitialized = true;
+      const auto &ASTCtx = Call.getState()->getStateManager().getContext();
+      Guard = &ASTCtx.Idents.get(GuardName);
     }
   }
 
diff --git a/clang/test/Analysis/block-in-critical-section.c b/clang/test/Analysis/block-in-critical-section.c
new file mode 100644
index 000000000000..1e174af541b1
--- /dev/null
+++ b/clang/test/Analysis/block-in-critical-section.c
@@ -0,0 +1,6 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.BlockInCriticalSection -verify %s
+// expected-no-diagnostics
+
+// This should not crash
+int (*a)(void);
+void b(void) { a(); }
-- 
GitLab


From da116bd82c0a78d2022c34b56e45cf6e4f91eaed Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 15 May 2024 15:37:51 +0100
Subject: [PATCH 0862/1206] [Clang] Use ULL for std::max constant argument to
 fix build failure.

getKnownMinValue returns uint64_t, use ULL to make sure the second arg
is also 64 bit.
---
 clang/lib/CodeGen/Targets/Sparc.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/Targets/Sparc.cpp b/clang/lib/CodeGen/Targets/Sparc.cpp
index b82e9a69e196..13e9550781d1 100644
--- a/clang/lib/CodeGen/Targets/Sparc.cpp
+++ b/clang/lib/CodeGen/Targets/Sparc.cpp
@@ -266,7 +266,7 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const {
   // All structs, even empty ones, should take up a register argument slot,
   // so pin the minimum struct size to one bit.
   CB.pad(llvm::alignTo(
-      std::max(CB.DL.getTypeSizeInBits(StrTy).getKnownMinValue(), 1UL), 64));
+      std::max(CB.DL.getTypeSizeInBits(StrTy).getKnownMinValue(), 1ULL), 64));
 
   // Try to use the original type for coercion.
   llvm::Type *CoerceTy = CB.isUsableType(StrTy) ? StrTy : CB.getType();
-- 
GitLab


From b42d245b77a83f8f6ca88c2dc441a96a5e8d5b52 Mon Sep 17 00:00:00 2001
From: AdityaK <hiraditya@msn.com>
Date: Wed, 15 May 2024 07:44:34 -0700
Subject: [PATCH 0863/1206] [GVNHoist] Replace combineKnownMetadata with
 combineMetadataForCSE (#92197)

There is no reason to call combineMetadata directly with a list of MD_
nodes. The combineMetadataForCSE function handles all the metadata
correctly

Partially fixes: #30866
---
 llvm/lib/Transforms/Scalar/GVNHoist.cpp | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/GVNHoist.cpp b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
index 261c1259c9c9..b5333c532280 100644
--- a/llvm/lib/Transforms/Scalar/GVNHoist.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
@@ -238,18 +238,6 @@ public:
   const VNtoInsns &getStoreVNTable() const { return VNtoCallsStores; }
 };
 
-static void combineKnownMetadata(Instruction *ReplInst, Instruction *I) {
-  static const unsigned KnownIDs[] = {LLVMContext::MD_tbaa,
-                                      LLVMContext::MD_alias_scope,
-                                      LLVMContext::MD_noalias,
-                                      LLVMContext::MD_range,
-                                      LLVMContext::MD_fpmath,
-                                      LLVMContext::MD_invariant_load,
-                                      LLVMContext::MD_invariant_group,
-                                      LLVMContext::MD_access_group};
-  combineMetadata(ReplInst, I, KnownIDs, true);
-}
-
 // This pass hoists common computations across branches sharing common
 // dominator. The primary goal is to reduce the code size, and in some
 // cases reduce critical path (by exposing more ILP).
@@ -996,8 +984,8 @@ unsigned GVNHoist::rauw(const SmallVecInsn &Candidates, Instruction *Repl,
         MSSAUpdater->removeMemoryAccess(OldMA);
       }
 
+      combineMetadataForCSE(Repl, I, true);
       Repl->andIRFlags(I);
-      combineKnownMetadata(Repl, I);
       I->replaceAllUsesWith(Repl);
       // Also invalidate the Alias Analysis cache.
       MD->removeInstruction(I);
-- 
GitLab


From 8a4cbeada930bf11fe740a2038bd5a3230712284 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 15 May 2024 15:48:22 +0100
Subject: [PATCH 0864/1206] [Clang] Unbreak build take 2 using uint64_t()
 explicitly.

---
 clang/lib/CodeGen/Targets/Sparc.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/Targets/Sparc.cpp b/clang/lib/CodeGen/Targets/Sparc.cpp
index 13e9550781d1..561f0b514d90 100644
--- a/clang/lib/CodeGen/Targets/Sparc.cpp
+++ b/clang/lib/CodeGen/Targets/Sparc.cpp
@@ -266,7 +266,8 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const {
   // All structs, even empty ones, should take up a register argument slot,
   // so pin the minimum struct size to one bit.
   CB.pad(llvm::alignTo(
-      std::max(CB.DL.getTypeSizeInBits(StrTy).getKnownMinValue(), 1ULL), 64));
+      std::max(CB.DL.getTypeSizeInBits(StrTy).getKnownMinValue(), uint64_t(1)),
+      64));
 
   // Try to use the original type for coercion.
   llvm::Type *CoerceTy = CB.isUsableType(StrTy) ? StrTy : CB.getType();
-- 
GitLab


From dceaa0f4491ebe30c0b0f1bc7fa5ec365b60ced6 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Wed, 15 May 2024 10:55:34 -0400
Subject: [PATCH 0865/1206] [Support] Use malloc instead of non-throwing new
 (#92157)

When allocating a memory buffer, we use a non-throwing new so that we
can explicitly handle memory buffers that are too large to fit into
memory. However, when exceptions are disabled, LLVM installs a custom
new handler

(https://github.com/llvm/llvm-project/blob/90109d444839683b09f0aafdc50b749cb4b3203b/llvm/lib/Support/InitLLVM.cpp#L61)
that explicitly crashes when we run out of memory

(https://github.com/llvm/llvm-project/blob/de14b749fee41d4ded711e771e43043ae3100cb3/llvm/lib/Support/ErrorHandling.cpp#L188)
and that means this particular out-of-memory situation cannot be
gracefully handled.

This was discovered while working on #embed
(https://github.com/llvm/llvm-project/pull/68620) on Windows and
resulted in a crash rather than the preprocessor issuing a diagnostic as
expected.

This patch switches away from the non-throwing new to a call to malloc
(and free), which will return a null pointer without calling a custom
new handler. It is the only instance in Clang or LLVM that I could find
which used a non-throwing new, so I did not think we would need anything
more involved than this change.

Testing this would be highly platform dependent and so it does not come
with test coverage. And because it doesn't change behavior that users
are likely to be able to observe, it does not come with a release note.
---
 llvm/lib/Support/MemoryBuffer.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Support/MemoryBuffer.cpp b/llvm/lib/Support/MemoryBuffer.cpp
index 4cc4fe019b75..50308bd2bf4a 100644
--- a/llvm/lib/Support/MemoryBuffer.cpp
+++ b/llvm/lib/Support/MemoryBuffer.cpp
@@ -98,7 +98,7 @@ public:
 
   /// Disable sized deallocation for MemoryBufferMem, because it has
   /// tail-allocated data.
-  void operator delete(void *p) { ::operator delete(p); }
+  void operator delete(void *p) { std::free(p); }
 
   StringRef getBufferIdentifier() const override {
     // The name is stored after the class itself.
@@ -315,7 +315,14 @@ WritableMemoryBuffer::getNewUninitMemBuffer(size_t Size,
   size_t RealLen = StringLen + Size + 1 + BufAlign.value();
   if (RealLen <= Size) // Check for rollover.
     return nullptr;
-  char *Mem = static_cast<char*>(operator new(RealLen, std::nothrow));
+  // We use a call to malloc() rather than a call to a non-throwing operator
+  // new() because LLVM unconditionally installs an out of memory new handler
+  // when exceptions are disabled. This new handler intentionally crashes to
+  // aid with debugging, but that makes non-throwing new calls unhelpful.
+  // See MemoryBufferMem::operator delete() for the paired call to free(), and
+  // llvm::install_out_of_memory_new_handler() for the installation of the
+  // custom new handler.
+  char *Mem = static_cast<char *>(std::malloc(RealLen));
   if (!Mem)
     return nullptr;
 
-- 
GitLab


From 8ab753c121447c1388c4cb1af08ab27b2cd62a82 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 15 May 2024 10:10:49 -0500
Subject: [PATCH 0866/1206] [flang][OpenMP] Add `REQUIRES: asserts` to test
 that relies on it

This should fix failures in release builds.
---
 flang/test/Lower/OpenMP/invalid-reduction-modifier.f90 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang/test/Lower/OpenMP/invalid-reduction-modifier.f90 b/flang/test/Lower/OpenMP/invalid-reduction-modifier.f90
index 817c5b731c62..53871276761f 100644
--- a/flang/test/Lower/OpenMP/invalid-reduction-modifier.f90
+++ b/flang/test/Lower/OpenMP/invalid-reduction-modifier.f90
@@ -1,4 +1,5 @@
 !Remove the --crash below once we can diagnose the issue more gracefully.
+!REQUIRES: asserts
 !RUN: not --crash %flang_fc1 -fopenmp -emit-hlfir -o - %s
 
 ! Check that we reject the "task" reduction modifier on the "simd" directive.
-- 
GitLab


From 413aaf11cd74f422f05b990613f822dc10db4391 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Wed, 15 May 2024 13:03:35 +0200
Subject: [PATCH 0867/1206] [clang][Interp][NFC] Support IntAP(S) in
 emitPrimCast

---
 clang/lib/AST/Interp/ByteCodeExprGen.cpp | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 1da74ac7c8bd..7cdc1c6d1947 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -3685,12 +3685,22 @@ bool ByteCodeExprGen<Emitter>::emitPrimCast(PrimType FromT, PrimType ToT,
       return this->emitCastFP(ToSem, getRoundingMode(E), E);
     }
 
+    if (ToT == PT_IntAP)
+      return this->emitCastFloatingIntegralAP(Ctx.getBitWidth(ToQT), E);
+    if (ToT == PT_IntAPS)
+      return this->emitCastFloatingIntegralAPS(Ctx.getBitWidth(ToQT), E);
+
     // Float to integral.
     if (isIntegralType(ToT) || ToT == PT_Bool)
       return this->emitCastFloatingIntegral(ToT, E);
   }
 
   if (isIntegralType(FromT) || FromT == PT_Bool) {
+    if (ToT == PT_IntAP)
+      return this->emitCastAP(FromT, Ctx.getBitWidth(ToQT), E);
+    if (ToT == PT_IntAPS)
+      return this->emitCastAPS(FromT, Ctx.getBitWidth(ToQT), E);
+
     // Integral to integral.
     if (isIntegralType(ToT) || ToT == PT_Bool)
       return FromT != ToT ? this->emitCast(FromT, ToT, E) : true;
-- 
GitLab


From 28d5f7907e8c3adb6f0e2e16c9673a99f5e07522 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Wed, 15 May 2024 15:15:07 +0200
Subject: [PATCH 0868/1206] [clang][Interp][NFC] Use a smaller default size for
 IntegralAP

Since we later possibly initialize the value by using operator-new,
we need the default value to _not_ allocate memory.
---
 clang/lib/AST/Interp/IntegralAP.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/AST/Interp/IntegralAP.h b/clang/lib/AST/Interp/IntegralAP.h
index fb7ee1451571..7464f15cdb03 100644
--- a/clang/lib/AST/Interp/IntegralAP.h
+++ b/clang/lib/AST/Interp/IntegralAP.h
@@ -61,7 +61,7 @@ public:
 
   IntegralAP(APInt V) : V(V) {}
   /// Arbitrary value for uninitialized variables.
-  IntegralAP() : IntegralAP(-1, 1024) {}
+  IntegralAP() : IntegralAP(-1, 3) {}
 
   IntegralAP operator-() const { return IntegralAP(-V); }
   IntegralAP operator-(const IntegralAP &Other) const {
-- 
GitLab


From 4527adc500ea0dc4b942a51dc7209da4ea26d9a2 Mon Sep 17 00:00:00 2001
From: Daniel Kuts <kutz@ispras.ru>
Date: Wed, 15 May 2024 18:15:13 +0300
Subject: [PATCH 0869/1206] Fix null pointer dereference in logging in mlir
 TransformOps (#92237)

A variable `typeConverterOp` may be nullptr after dynamic cast. There is
a security guard for this, but during logging error message the variable
getting dereferenced.
Found with static analysis.
---
 mlir/lib/Dialect/Transform/IR/TransformOps.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index eb09f007fbca..247759e21efb 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -648,13 +648,14 @@ LogicalResult transform::ApplyConversionPatternsOp::verify() {
     if (!llvm::hasSingleElement(typeConverterRegion.front()))
       return emitOpError()
              << "expected exactly one op in default type converter region";
+    Operation *maybeTypeConverter = &typeConverterRegion.front().front();
     auto typeConverterOp = dyn_cast<transform::TypeConverterBuilderOpInterface>(
-        &typeConverterRegion.front().front());
+        maybeTypeConverter);
     if (!typeConverterOp) {
       InFlightDiagnostic diag = emitOpError()
                                 << "expected default converter child op to "
                                    "implement TypeConverterBuilderOpInterface";
-      diag.attachNote(typeConverterOp->getLoc()) << "op without interface";
+      diag.attachNote(maybeTypeConverter->getLoc()) << "op without interface";
       return diag;
     }
     // Check default type converter type.
-- 
GitLab


From b576a6b0452b9bfb634feaa215506d8a1afe857d Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Wed, 15 May 2024 23:15:48 +0800
Subject: [PATCH 0870/1206] [X86][AMX] Fix a bug after #83628 (#91207)

We need to check if `GR64Cand` a valid register before using it.

Test is not needed since it's covered in llvm-test-suite.

Fixes #90954
---
 llvm/lib/Target/X86/X86LowerTileCopy.cpp    |   4 +-
 llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll | 153 ++++++++++++++++++++
 2 files changed, 155 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/X86/X86LowerTileCopy.cpp b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
index fd05e16ac1ce..60c024556ff1 100644
--- a/llvm/lib/Target/X86/X86LowerTileCopy.cpp
+++ b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
@@ -146,7 +146,7 @@ bool X86LowerTileCopy::runOnMachineFunction(MachineFunction &MF) {
           addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc)), TileSS)
               .addReg(SrcReg, getKillRegState(SrcMO.isKill()));
       MachineOperand &MO = NewMI->getOperand(2);
-      MO.setReg(GR64Cand);
+      MO.setReg(GR64Cand ? GR64Cand : X86::RAX);
       MO.setIsKill(true);
       // tileloadd (%sp, %idx), %tmm
       Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
@@ -157,7 +157,7 @@ bool X86LowerTileCopy::runOnMachineFunction(MachineFunction &MF) {
         // restore %rax
         // mov (%sp) %rax
         addFrameReference(
-            BuildMI(MBB, MI, DL, TII->get(X86::MOV64rm), GR64Cand), StrideSS);
+            BuildMI(MBB, MI, DL, TII->get(X86::MOV64rm), X86::RAX), StrideSS);
       }
       MI.eraseFromParent();
       Changed = true;
diff --git a/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll b/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll
index 4a9f9d3bf77a..7511e5953dac 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-tile-basic.ll
@@ -51,3 +51,156 @@ declare x86_amx @llvm.x86.tdpbusd.internal(i16, i16, i16, x86_amx, x86_amx, x86_
 declare x86_amx @llvm.x86.tdpbuud.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
 declare x86_amx @llvm.x86.tdpbf16ps.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
 declare void @llvm.x86.tilestored64.internal(i16, i16, ptr, i64, x86_amx)
+
+define void @PR90954(ptr %0, ptr %1, i32 %2) {
+; CHECK-LABEL: PR90954:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    andq $-1024, %rsp # imm = 0xFC00
+; CHECK-NEXT:    subq $5120, %rsp # imm = 0x1400
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vmovups %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $16, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw $64, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $16, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw $64, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $16, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw $64, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    shll $4, %edx
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    movw $64, %cx
+; CHECK-NEXT:    movw $16, %di
+; CHECK-NEXT:    movb $1, %r8b
+; CHECK-NEXT:    movl $64, %r9d
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %r11
+; CHECK-NEXT:    xorl %ebx, %ebx
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    jmp .LBB1_1
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_5: # in Loop: Header=BB1_1 Depth=1
+; CHECK-NEXT:    incq %r14
+; CHECK-NEXT:    addl %edx, %ebx
+; CHECK-NEXT:  .LBB1_1: # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB1_2 Depth 2
+; CHECK-NEXT:    movslq %ebx, %r15
+; CHECK-NEXT:    leaq (%rsi,%r15,4), %r15
+; CHECK-NEXT:    xorl %r12d, %r12d
+; CHECK-NEXT:    xorl %r13d, %r13d
+; CHECK-NEXT:    jmp .LBB1_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_4: # in Loop: Header=BB1_2 Depth=2
+; CHECK-NEXT:    tilestored %tmm1, (%r15,%rax)
+; CHECK-NEXT:    incq %r13
+; CHECK-NEXT:    addq $64, %r15
+; CHECK-NEXT:    decq %r12
+; CHECK-NEXT:    je .LBB1_5
+; CHECK-NEXT:  .LBB1_2: # Parent Loop BB1_1 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    tilezero %tmm0
+; CHECK-NEXT:    tilezero %tmm1
+; CHECK-NEXT:    testb %r8b, %r8b
+; CHECK-NEXT:    jne .LBB1_4
+; CHECK-NEXT:  # %bb.3: # in Loop: Header=BB1_2 Depth=2
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    tileloadd (%r10,%r9), %tmm1
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    vmovaps %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    tileloadd (%r11,%r9), %tmm2
+; CHECK-NEXT:    tdpbf16ps %tmm2, %tmm1, %tmm0
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movabsq $64, %rax
+; CHECK-NEXT:    tilestored %tmm0, 3072(%rsp,%rax) # 1024-byte Folded Spill
+; CHECK-NEXT:    tileloadd {{[-0-9]+}}(%r{{[sb]}}p), %tmm1 # 1024-byte Folded Reload
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    jmp .LBB1_4
+  %4 = shl i32 %2, 4
+  %5 = icmp eq i64 0, 0
+  br label %6
+
+6:                                                ; preds = %31, %3
+  %7 = phi i64 [ 0, %3 ], [ %32, %31 ]
+  %8 = trunc nuw nsw i64 %7 to i32
+  %9 = mul i32 %4, %8
+  %10 = mul i32 0, %8
+  %11 = sext i32 %9 to i64
+  %12 = getelementptr inbounds i32, ptr %1, i64 %11
+  br label %13
+
+13:                                               ; preds = %25, %6
+  %14 = phi i64 [ %29, %25 ], [ 0, %6 ]
+  %15 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+  %16 = tail call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %15)
+  %17 = shl nsw i64 %14, 4
+  %18 = getelementptr i32, ptr %0, i64 %17
+  br i1 %5, label %25, label %19
+
+19:                                               ; preds = %13
+  %20 = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %16)
+  %21 = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> zeroinitializer)
+  %22 = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> zeroinitializer)
+  %23 = tail call x86_amx @llvm.x86.tdpbf16ps.internal(i16 16, i16 64, i16 64, x86_amx %20, x86_amx %21, x86_amx %22)
+  %24 = tail call noundef <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %23)
+  br label %25
+
+25:                                               ; preds = %19, %13
+  %26 = phi <256 x i32> [ undef, %13 ], [ %24, %19 ]
+  %27 = getelementptr inbounds i32, ptr %12, i64 %17
+  %28 = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %26)
+  tail call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr %27, i64 0, x86_amx %28)
+  %29 = add nuw nsw i64 %14, 1
+  %30 = icmp eq i64 %29, 0
+  br i1 %30, label %31, label %13
+
+31:                                               ; preds = %25
+  %32 = add nuw nsw i64 %7, 1
+  br label %6
+}
+
+declare x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32>)
+declare <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx)
-- 
GitLab


From bed5546bb53bdb231b62f569b67f449019426ce8 Mon Sep 17 00:00:00 2001
From: Rajveer Singh Bharadwaj <rajveer.developer@icloud.com>
Date: Wed, 15 May 2024 20:46:15 +0530
Subject: [PATCH 0871/1206] [DebugInfo] Get rid of redundant conditional checks
 in `/DebugInfo` (#92111)

Resolves #90326
---
 llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h
index 3f7f8c7838fd..efc8db12a697 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h
@@ -246,20 +246,17 @@ public:
   virtual void setName(StringRef ObjectName) {}
 
   LVElement *getParent() const {
-    assert((!Parent.Element ||
-            (Parent.Element && static_cast<LVElement *>(Parent.Element))) &&
+    assert((!Parent.Element || static_cast<LVElement *>(Parent.Element)) &&
            "Invalid element");
     return Parent.Element;
   }
   LVScope *getParentScope() const {
-    assert((!Parent.Scope ||
-            (Parent.Scope && static_cast<LVScope *>(Parent.Scope))) &&
+    assert((!Parent.Scope || static_cast<LVScope *>(Parent.Scope)) &&
            "Invalid scope");
     return Parent.Scope;
   }
   LVSymbol *getParentSymbol() const {
-    assert((!Parent.Symbol ||
-            (Parent.Symbol && static_cast<LVSymbol *>(Parent.Symbol))) &&
+    assert((!Parent.Symbol || static_cast<LVSymbol *>(Parent.Symbol)) &&
            "Invalid symbol");
     return Parent.Symbol;
   }
-- 
GitLab


From dcf3102be8458fe7588f9d11315beddfca4323b0 Mon Sep 17 00:00:00 2001
From: Elvina Yakubova <eyakubova@nvidia.com>
Date: Wed, 15 May 2024 16:16:39 +0100
Subject: [PATCH 0872/1206] [BOLT][NFC] Add documentation on BOLT options
 (#92117)

Add .md file documentation with all BOLT options to display it more
conveniently.
---
 bolt/docs/CommandLineArgumentReference.md | 1213 +++++++++++++++++++++
 1 file changed, 1213 insertions(+)
 create mode 100644 bolt/docs/CommandLineArgumentReference.md

diff --git a/bolt/docs/CommandLineArgumentReference.md b/bolt/docs/CommandLineArgumentReference.md
new file mode 100644
index 000000000000..1951ad5a2dc5
--- /dev/null
+++ b/bolt/docs/CommandLineArgumentReference.md
@@ -0,0 +1,1213 @@
+# BOLT - a post-link optimizer developed to speed up large applications
+
+## SYNOPSIS
+
+`llvm-bolt <executable> [-o outputfile] <executable>.bolt [-data=perf.fdata] [options]`
+
+## OPTIONS
+
+### Generic options
+
+- `-h`
+
+  Alias for `--help`
+
+- `--help`
+
+  Display available options (`--help-hidden` for more).
+
+- `--help-hidden`
+
+  Display all available options.
+
+- `--help-list`
+
+  Display list of available options (`--help-list-hidden` for more).
+
+- `--help-list-hidden`
+
+  Display list of all available options.
+
+- `--print-all-options`
+
+  Print all option values after command line parsing.
+
+- `--print-options`
+
+  Print non-default options after command line parsing.
+
+- `--version`
+
+  Display the version of this program.
+
+### Output options
+
+- `-o <string>`
+
+  output file
+
+- `-w <string>`
+
+  Save recorded profile to a file
+
+### BOLT generic options
+
+- `--align-text=<uint>`
+
+  Alignment of .text section
+
+- `--allow-stripped`
+
+  Allow processing of stripped binaries
+
+- `--asm-dump[=<dump folder>]`
+
+  Dump function into assembly
+
+- `-b`
+
+  Alias for -data
+
+- `--bolt-id=<string>`
+
+  Add any string to tag this execution in the output binary via bolt info section
+
+- `--break-funcs=<func1,func2,func3,...>`
+
+  List of functions to core dump on (debugging)
+
+- `--check-encoding`
+
+  Perform verification of LLVM instruction encoding/decoding. Every instruction
+  in the input is decoded and re-encoded. If the resulting bytes do not match
+  the input, a warning message is printed.
+
+- `--cu-processing-batch-size=<uint>`
+
+  Specifies the size of batches for processing CUs. Higher number has better
+  performance, but more memory usage. Default value is 1.
+
+- `--data=<string>`
+
+  <data file>
+
+- `--debug-skeleton-cu`
+
+  Prints out offsets for abbrev and debug_info of Skeleton CUs that get patched.
+
+- `--deterministic-debuginfo`
+
+  Disables parallel execution of tasks that may produce nondeterministic debug info
+
+- `--dot-tooltip-code`
+
+  Add basic block instructions as tool tips on nodes
+
+- `--dump-cg=<string>`
+
+  Dump callgraph to the given file
+
+- `--dump-data`
+
+  Dump parsed bolt data for debugging
+
+- `--dump-dot-all`
+
+  Dump function CFGs to graphviz format after each stage; enable '-print-loops'
+  for color-coded blocks
+
+- `--dump-orc`
+
+  Dump raw ORC unwind information (sorted)
+
+- `--dwarf-output-path=<string>`
+
+  Path to where .dwo files or dwp file will be written out to.
+
+- `--dwp=<string>`
+
+  Path and name to DWP file.
+
+- `--dyno-stats`
+
+  Print execution info based on profile
+
+- `--dyno-stats-all`
+
+  Print dyno stats after each stage
+
+- `--dyno-stats-scale=<uint>`
+
+  Scale to be applied while reporting dyno stats
+
+- `--enable-bat`
+
+  Write BOLT Address Translation tables
+
+- `--force-data-relocations`
+
+  Force relocations to data sections to always be processed
+
+- `--force-patch`
+
+  Force patching of original entry points
+
+- `--funcs=<func1,func2,func3,...>`
+
+  Limit optimizations to functions from the list
+
+- `--funcs-file=<string>`
+
+  File with list of functions to optimize
+
+- `--funcs-file-no-regex=<string>`
+
+  File with list of functions to optimize (non-regex)
+
+- `--funcs-no-regex=<func1,func2,func3,...>`
+
+  Limit optimizations to functions from the list (non-regex)
+
+- `--hot-data`
+
+  Hot data symbols support (relocation mode)
+
+- `--hot-functions-at-end`
+
+  If reorder-functions is used, order functions putting hottest last
+
+- `--hot-text`
+
+  Generate hot text symbols. Apply this option to a precompiled binary that
+  manually calls into hugify, such that at runtime hugify call will put hot
+  code into 2M pages. This requires relocation.
+
+- `--hot-text-move-sections=<sec1,sec2,sec3,...>`
+
+  List of sections containing functions used for hugifying hot text. BOLT makes
+  sure these functions are not placed on the same page as the hot text.
+  (default='.stub,.mover').
+
+- `--insert-retpolines`
+
+  Run retpoline insertion pass
+
+- `--keep-aranges`
+
+  Keep or generate .debug_aranges section if .gdb_index is written
+
+- `--keep-tmp`
+
+  Preserve intermediate .o file
+
+- `--lite`
+
+  Skip processing of cold functions
+
+- `--max-data-relocations=<uint>`
+
+  Maximum number of data relocations to process
+
+- `--max-funcs=<uint>`
+
+  Maximum number of functions to process
+
+- `--no-huge-pages`
+
+  Use regular size pages for code alignment
+
+- `--no-threads`
+
+  Disable multithreading
+
+- `--pad-funcs=<func1:pad1,func2:pad2,func3:pad3,...>`
+
+  List of functions to pad with amount of bytes
+
+- `--profile-format=<value>`
+
+  Format to dump profile output in aggregation mode, default is fdata
+  - `=fdata`: offset-based plaintext format
+  - `=yaml`: dense YAML representation
+
+- `--r11-availability=<value>`
+
+  Determine the availability of r11 before indirect branches
+  - `=never`: r11 not available
+  - `=always`: r11 available before calls and jumps
+  - `=abi`r11 available before calls but not before jumps
+
+- `--relocs`
+
+  Use relocations in the binary (default=autodetect)
+
+- `--remove-symtab`
+
+  Remove .symtab section
+
+- `--reorder-skip-symbols=<symbol1,symbol2,symbol3,...>`
+
+  List of symbol names that cannot be reordered
+
+- `--reorder-symbols=<symbol1,symbol2,symbol3,...>`
+
+  List of symbol names that can be reordered
+
+- `--retpoline-lfence`
+
+  Determine if lfence instruction should exist in the retpoline
+
+- `--skip-funcs=<func1,func2,func3,...>`
+
+  List of functions to skip
+
+- `--skip-funcs-file=<string>`
+
+  File with list of functions to skip
+
+- `--strict`
+
+  Trust the input to be from a well-formed source
+
+- `--tasks-per-thread=<uint>`
+
+  Number of tasks to be created per thread
+
+- `--thread-count=<uint>`
+
+  Number of threads
+
+- `--top-called-limit=<uint>`
+
+  Maximum number of functions to print in top called functions section
+
+- `--trap-avx512`
+
+  In relocation mode trap upon entry to any function that uses AVX-512 instructions
+
+- `--trap-old-code`
+
+  Insert traps in old function bodies (relocation mode)
+
+- `--update-debug-sections`
+
+  Update DWARF debug sections of the executable
+
+- `--use-gnu-stack`
+
+  Use GNU_STACK program header for new segment (workaround for issues with
+  strip/objcopy)
+
+- `--use-old-text`
+
+  Re-use space in old .text if possible (relocation mode)
+
+- `-v <uint>`
+
+  Set verbosity level for diagnostic output
+
+- `--write-dwp`
+
+  Output a single dwarf package file (dwp) instead of multiple non-relocatable
+  dwarf object files (dwo).
+
+### BOLT optimization options
+
+- `--align-blocks`
+
+  Align basic blocks
+
+- `--align-blocks-min-size=<uint>`
+
+  Minimal size of the basic block that should be aligned
+
+- `--align-blocks-threshold=<uint>`
+
+  Align only blocks with frequency larger than containing function execution
+  frequency specified in percent. E.g. 1000 means aligning blocks that are 10
+  times more frequently executed than the containing function.
+
+- `--align-functions=<uint>`
+
+  Align functions at a given value (relocation mode)
+
+- `--align-functions-max-bytes=<uint>`
+
+  Maximum number of bytes to use to align functions
+
+- `--assume-abi`
+
+  Assume the ABI is never violated
+
+- `--block-alignment=<uint>`
+
+  Boundary to use for alignment of basic blocks
+
+- `--bolt-seed=<uint>`
+
+  Seed for randomization
+
+- `--cg-from-perf-data`
+
+  Use perf data directly when constructing the call graph for stale functions
+
+- `--cg-ignore-recursive-calls`
+
+  Ignore recursive calls when constructing the call graph
+
+- `--cg-use-split-hot-size`
+
+  Use hot/cold data on basic blocks to determine hot sizes for call graph functions
+
+- `--cold-threshold=<uint>`
+
+  Tenths of percents of main entry frequency to use as a threshold when
+  evaluating whether a basic block is cold (0 means it is only considered
+  cold if the block has zero samples). Default: 0
+
+- `--elim-link-veneers`
+
+  Run veneer elimination pass
+
+- `--eliminate-unreachable`
+
+  Eliminate unreachable code
+
+- `--equalize-bb-counts`
+
+  Use same count for BBs that should have equivalent count (used in non-LBR
+  and shrink wrapping)
+
+- `--execution-count-threshold=<uint>`
+
+  Perform profiling accuracy-sensitive optimizations only if function execution
+  count >= the threshold (default: 0)
+
+- `--fix-block-counts`
+
+  Adjust block counts based on outgoing branch counts
+
+- `--fix-func-counts`
+
+  Adjust function counts based on basic blocks execution count
+
+- `--force-inline=<func1,func2,func3,...>`
+
+  List of functions to always consider for inlining
+
+- `--frame-opt=<value>`
+
+  Optimize stack frame accesses
+  - `none`: do not perform frame optimization
+  - `hot`: perform FOP on hot functions
+  - `all`: perform FOP on all functions
+
+- `--frame-opt-rm-stores`
+
+  Apply additional analysis to remove stores (experimental)
+
+- `--function-order=<string>`
+
+  File containing an ordered list of functions to use for function reordering
+
+- `--generate-function-order=<string>`
+
+  File to dump the ordered list of functions to use for function reordering
+
+- `--generate-link-sections=<string>`
+
+  Generate a list of function sections in a format suitable for inclusion in a
+  linker script
+
+- `--group-stubs`
+
+  Share stubs across functions
+
+- `--hugify`
+
+  Automatically put hot code on 2MB page(s) (hugify) at runtime. No manual call
+  to hugify is needed in the binary (which is what --hot-text relies on).
+
+- `--icf`
+
+  Fold functions with identical code
+
+- `--icp`
+
+  Alias for --indirect-call-promotion
+
+- `--icp-calls-remaining-percent-threshold=<uint>`
+
+  The percentage threshold against remaining unpromoted indirect call count
+  for the promotion for calls
+
+- `--icp-calls-topn`
+
+  Alias for --indirect-call-promotion-calls-topn
+
+- `--icp-calls-total-percent-threshold=<uint>`
+
+  The percentage threshold against total count for the promotion for calls
+
+- `--icp-eliminate-loads`
+
+  Enable load elimination using memory profiling data when performing ICP
+
+- `--icp-funcs=<func1,func2,func3,...>`
+
+  List of functions to enable ICP for
+
+- `--icp-inline`
+
+  Only promote call targets eligible for inlining
+
+- `--icp-jt-remaining-percent-threshold=<uint>`
+
+  The percentage threshold against remaining unpromoted indirect call count for
+  the promotion for jump tables
+
+- `--icp-jt-targets`
+
+  Alias for --icp-jump-tables-targets
+
+- `--icp-jt-topn`
+
+  Alias for --indirect-call-promotion-jump-tables-topn
+
+- `--icp-jt-total-percent-threshold=<uint>`
+
+  The percentage threshold against total count for the promotion for jump tables
+
+- `--icp-jump-tables-targets`
+
+  For jump tables, optimize indirect jmp targets instead of indices
+
+- `--icp-mp-threshold`
+
+  Alias for --indirect-call-promotion-mispredict-threshold
+
+- `--icp-old-code-sequence`
+
+  Use old code sequence for promoted calls
+
+- `--icp-top-callsites=<uint>`
+
+  Optimize hottest calls until at least this percentage of all indirect calls
+  frequency is covered. 0 = all callsites
+
+- `--icp-topn`
+
+  Alias for --indirect-call-promotion-topn
+
+- `--icp-use-mp`
+
+  Alias for --indirect-call-promotion-use-mispredicts
+
+- `--indirect-call-promotion=<value>`
+
+  Indirect call promotion
+  - `none`: do not perform indirect call promotion
+  - `calls`: perform ICP on indirect calls
+  - `jump-tables`: perform ICP on jump tables
+  - `all`: perform ICP on calls and jump tables
+
+- `--indirect-call-promotion-calls-topn=<uint>`
+
+  Limit number of targets to consider when doing indirect call promotion on
+  calls. 0 = no limit
+
+- `--indirect-call-promotion-jump-tables-topn=<uint>`
+
+  Limit number of targets to consider when doing indirect call promotion on
+  jump tables. 0 = no limit
+
+- `--indirect-call-promotion-mispredict-threshold=<uint>`
+
+  Misprediction threshold for skipping ICP on an indirect call
+
+- `--indirect-call-promotion-topn=<uint>`
+
+  Limit number of targets to consider when doing indirect call promotion.
+  0 = no limit
+
+- `--indirect-call-promotion-use-mispredicts`
+
+  Use misprediction frequency for determining whether or not ICP should be
+  applied at a callsite. The `-indirect-call-promotion-mispredict-threshold`
+  value will be used by this heuristic
+
+- `--infer-fall-throughs`
+
+  Infer execution count for fall-through blocks
+
+- `--infer-stale-profile`
+
+  Infer counts from stale profile data.
+
+- `--inline-all`
+
+  Inline all functions
+
+- `--inline-ap`
+
+  Adjust function profile after inlining
+
+- `--inline-limit=<uint>`
+
+  Maximum number of call sites to inline
+
+- `--inline-max-iters=<uint>`
+
+  Maximum number of inline iterations
+
+- `--inline-memcpy`
+
+  Inline memcpy using 'rep movsb' instruction (X86-only)
+
+- `--inline-small-functions`
+
+  Inline functions if increase in size is less than defined by `-inline-small-functions-bytes`
+
+- `--inline-small-functions-bytes=<uint>`
+
+  Max number of bytes for the function to be considered small for inlining purposes
+
+- `--instrument`
+
+  Instrument code to generate accurate profile data
+
+- `--iterative-guess`
+
+  In non-LBR mode, guess edge counts using iterative technique
+
+- `--jt-footprint-optimize-for-icache`
+
+  With jt-footprint-reduction, only process PIC jumptables and turn off other
+  transformations that increase code size
+
+- `--jt-footprint-reduction`
+
+  Make jump tables size smaller at the cost of using more instructions at jump
+  sites
+
+- `-jump-tables=<value>`
+
+  Jump tables support (default=basic)
+  - `none`: do not optimize functions with jump tables
+  - `basic`: optimize functions with jump tables
+  - `move`: move jump tables to a separate section
+  - `split`: split jump tables section into hot and cold based on function
+  execution frequency
+  - `aggressive`: aggressively split jump tables section based on usage of the
+  tables
+
+- `--keep-nops`
+
+  Keep no-op instructions. By default they are removed.
+
+- `--lite-threshold-count=<uint>`
+
+  Similar to '-lite-threshold-pct' but specify threshold using absolute function
+  call count. I.e. limit processing to functions executed at least the specified
+  number of times.
+
+- `--lite-threshold-pct=<uint>`
+
+  Threshold (in percent) for selecting functions to process in lite mode. Higher
+  threshold means fewer functions to process. E.g threshold of 90 means only top
+  10 percent of functions with profile will be processed.
+
+- `--mcf-use-rarcs`
+
+  In MCF, consider the possibility of cancelling flow to balance edges
+
+- `--memcpy1-spec=<func1,func2:cs1:cs2,func3:cs1,...>`
+
+  List of functions with call sites for which to specialize memcpy() for size 1
+
+- `--min-branch-clusters`
+
+  Use a modified clustering algorithm geared towards minimizing branches
+
+- `--no-inline`
+
+  Disable all inlining (overrides other inlining options)
+
+- `--no-scan`
+
+  Do not scan cold functions for external references (may result in slower binary)
+
+- `--peepholes=<value>`
+
+  Enable peephole optimizations
+  - `none`: disable peepholes
+  - `double-jumps`: remove double jumps when able
+  - `tailcall-traps`: insert tail call traps
+  - `useless-branches`: remove useless conditional branches
+  - `all`: enable all peephole optimizations
+
+- `--plt=<value>`
+
+  Optimize PLT calls (requires linking with -znow)
+  - `none`: do not optimize PLT calls
+  - `hot`: optimize executed (hot) PLT calls
+  - `all`: optimize all PLT calls
+
+- `--preserve-blocks-alignment`
+
+  Try to preserve basic block alignment
+
+- `--profile-ignore-hash`
+
+  Ignore hash while reading function profile
+
+- `--profile-use-dfs`
+
+  Use DFS order for YAML profile
+
+- `--reg-reassign`
+
+  Reassign registers so as to avoid using REX prefixes in hot code
+
+- `--reorder-blocks=<value>`
+
+  Change layout of basic blocks in a function
+  - `none`: do not reorder basic blocks
+  - `reverse`: layout blocks in reverse order
+  - `normal`: perform optimal layout based on profile
+  - `branch-predictor`: perform optimal layout prioritizing branch predictions
+  - `cache`: perform optimal layout prioritizing I-cache behavior
+  - `cache+`: perform layout optimizing I-cache behavior
+  - `ext-tsp`: perform layout optimizing I-cache behavior
+  - `cluster-shuffle`: perform random layout of clusters
+
+- `--reorder-data=<section1,section2,section3,...>`
+
+  List of sections to reorder
+
+- `--reorder-data-algo=<value>`
+
+  Algorithm used to reorder data sections
+  - `count`: sort hot data by read counts
+  - `funcs`: sort hot data by hot function usage and count
+
+- `--reorder-data-inplace`
+
+  Reorder data sections in place
+
+- `--reorder-data-max-bytes=<uint>`
+
+  Maximum number of bytes to reorder
+
+- `--reorder-data-max-symbols=<uint>`
+
+  Maximum number of symbols to reorder
+
+- `--reorder-functions=<value>`
+
+  Reorder and cluster functions (works only with relocations)
+  - `none`: do not reorder functions
+  - `exec-count`: order by execution count
+  - `hfsort`: use hfsort algorithm
+  - `hfsort+`: use hfsort+ algorithm
+  - `cdsort`: use cache-directed sort
+  - `pettis-hansen`: use Pettis-Hansen algorithm
+  - `random`: reorder functions randomly
+  - `user`: use function order specified by -function-order
+
+- `--reorder-functions-use-hot-size`
+
+  Use a function's hot size when doing clustering
+
+- `--report-bad-layout=<uint>`
+
+  Print top <uint> functions with suboptimal code layout on input
+
+- `--report-stale`
+
+  Print the list of functions with stale profile
+
+- `--runtime-hugify-lib=<string>`
+
+  Specify file name of the runtime hugify library
+
+- `--runtime-instrumentation-lib=<string>`
+
+  Specify file name of the runtime instrumentation library
+
+- `--sctc-mode=<value>`
+
+  Mode for simplify conditional tail calls
+  - `always`: always perform sctc
+  - `preserve`: only perform sctc when branch direction is preserved
+  - `heuristic`: use branch prediction data to control sctc
+
+- `--sequential-disassembly`
+
+  Performs disassembly sequentially
+
+- `--shrink-wrapping-threshold=<uint>`
+
+  Percentage of prologue execution count to use as threshold when evaluating
+  whether a block is cold enough to be profitable to move eligible spills there
+
+- `--simplify-conditional-tail-calls`
+
+  Simplify conditional tail calls by removing unnecessary jumps
+
+- `--simplify-rodata-loads`
+
+  Simplify loads from read-only sections by replacing the memory operand with
+  the constant found in the corresponding section
+
+- `--split-align-threshold=<uint>`
+
+  When deciding to split a function, apply this alignment while doing the size
+  comparison (see -split-threshold). Default value: 2.
+
+- `--split-all-cold`
+
+  Outline as many cold basic blocks as possible
+
+- `--split-eh`
+
+  Split C++ exception handling code
+
+- `--split-functions`
+
+  Split functions into fragments
+
+- `--split-strategy=<value>`
+
+  Strategy used to partition blocks into fragments
+
+  - `profile2`: split each function into a hot and cold fragment using
+  profiling information
+  - `cdsplit`: split each function into a hot, warm, and cold fragment using
+  profiling information
+  - `random2`: split each function into a hot and cold fragment at a randomly
+  chosen split point (ignoring any available profiling information)
+  - `randomN`: split each function into N fragments at randomly chosen split
+  points (ignoring any available profiling information)
+  - `all`: split all basic blocks of each function into fragments such that
+  each fragment contains exactly a single basic block
+
+- `--split-threshold=<uint>`
+
+  Split function only if its main size is reduced by more than given amount of
+  bytes. Default value: 0, i.e. split iff the size is reduced. Note that on
+  some architectures the size can increase after splitting.
+
+- `--stale-matching-max-func-size=<uint>`
+
+  The maximum size of a function to consider for inference.
+
+- `--stale-threshold=<uint>`
+
+  Maximum percentage of stale functions to tolerate (default: 100)
+
+- `--stoke`
+
+  Turn on the stoke analysis
+
+- `--strip-rep-ret`
+
+  Strip 'repz' prefix from 'repz retq' sequence (on by default)
+
+- `--tail-duplication=<value>`
+
+  Duplicate unconditional branches that cross a cache line
+
+  - `none` do not apply
+  - `aggressive` aggressive strategy
+  - `moderate` moderate strategy
+  - `cache` cache-aware duplication strategy
+
+- `--tsp-threshold=<uint>`
+
+  Maximum number of hot basic blocks in a function for which to use a precise TSP solution while re-ordering basic blocks
+
+- `--use-aggr-reg-reassign`
+
+  Use register liveness analysis to try to find more opportunities for -reg-reassign optimization
+
+- `--use-compact-aligner`
+
+  Use compact approach for aligning functions
+
+- `--use-edge-counts`
+
+  Use edge count data when doing clustering
+
+- `--verify-cfg`
+
+  Verify the CFG after every pass
+
+- `--x86-align-branch-boundary-hot-only`
+
+  Only apply branch boundary alignment in hot code
+
+- `--x86-strip-redundant-address-size`
+
+  Remove redundant Address-Size override prefix
+
+### BOLT options in relocation mode
+
+- `-align-macro-fusion=<value>`
+
+  Fix instruction alignment for macro-fusion (x86 relocation mode)
+
+  - `none`: do not insert alignment no-ops for macro-fusion
+  - `hot`: only insert alignment no-ops on hot execution paths (default)
+  - `all`: always align instructions to allow macro-fusion
+
+### BOLT instrumentation options
+
+`llvm-bolt <executable> -instrument [-o outputfile] <instrumented-executable>`
+
+- `--conservative-instrumentation`
+
+  Disable instrumentation optimizations that sacrifice profile accuracy (for
+  debugging, default: false)
+
+- `--instrument-calls`
+
+  Record profile for inter-function control flow activity (default: true)
+
+- `--instrument-hot-only`
+
+  Only insert instrumentation on hot functions (needs profile, default: false)
+
+- `--instrumentation-binpath=<string>`
+
+  Path to instrumented binary in case if /proc/self/map_files is not accessible
+  due to access restriction issues
+
+- `--instrumentation-file=<string>`
+
+  File name where instrumented profile will be saved (default: /tmp/prof.fdata)
+
+- `--instrumentation-file-append-pid`
+
+  Append PID to saved profile file name (default: false)
+
+- `--instrumentation-no-counters-clear`
+
+  Don't clear counters across dumps (use with `instrumentation-sleep-time` option)
+
+- `--instrumentation-sleep-time=<uint>`
+
+  Interval between profile writes (default: 0 = write only at program end).
+  This is useful for service workloads when you want to dump profile every X
+  minutes or if you are killing the program and the profile is not being
+  dumped at the end.
+
+- `--instrumentation-wait-forks`
+
+  Wait until all forks of instrumented process will finish (use with
+  `instrumentation-sleep-time` option)
+
+### Data aggregation options (perf2bolt)
+
+`perf2bolt -p perf.data [-o outputfile] perf.fdata <executable>`
+
+- `--autofdo`
+
+  Generate autofdo textual data instead of bolt data
+
+- `--filter-mem-profile`
+
+  If processing a memory profile, filter out stack or heap accesses that won't
+  be useful for BOLT to reduce profile file size
+
+- `--ignore-build-id`
+
+  Continue even if build-ids in input binary and perf.data mismatch
+
+- `--ignore-interrupt-lbr`
+
+  Ignore kernel interrupt LBR that happens asynchronously
+
+- `--itrace=<string>`
+
+  Generate LBR info with perf itrace argument
+
+- `--nl`
+
+  Aggregate basic samples (without LBR info)
+
+- `--pa`
+
+  Skip perf and read data from a pre-aggregated file format
+
+- `--perfdata=<string>`
+
+  Data file
+
+- `--pid=<ulong>`
+
+  Only use samples from process with specified PID
+
+- `--time-aggr`
+
+  Time BOLT aggregator
+
+- `--use-event-pc`
+
+  Use event PC in combination with LBR sampling
+
+### BOLT printing options
+
+#### Generic options
+
+- `--print-aliases`
+
+  Print aliases when printing objects
+
+- `--print-all`
+
+  Print functions after each stage
+
+- `--print-cfg`
+
+  Print functions after CFG construction
+
+- `--print-debug-info`
+
+  Print debug info when printing functions
+
+- `--print-disasm`
+
+  Print function after disassembly
+
+- `--print-dyno-opcode-stats=<uint>`
+
+  Print per instruction opcode dyno stats and the functionnames:BB offsets of
+  the nth highest execution counts
+
+- `--print-dyno-stats-only`
+
+  While printing functions output dyno-stats and skip instructions
+
+- `--print-exceptions`
+
+  Print exception handling data
+
+- `--print-globals`
+
+  Print global symbols after disassembly
+
+- `--print-jump-tables`
+
+  Print jump tables
+
+- `--print-loops`
+
+  Print loop related information
+
+- `--print-mem-data`
+
+  Print memory data annotations when printing functions
+
+- `--print-normalized`
+
+  Print functions after CFG is normalized
+
+- `--print-only=<func1,func2,func3,...>`
+
+  List of functions to print
+
+- `--print-orc`
+
+  Print ORC unwind information for instructions
+
+- `--print-profile`
+
+  Print functions after attaching profile
+
+- `--print-profile-stats`
+
+  Print profile quality/bias analysis
+
+- `--print-pseudo-probes=<value>`
+
+  Print pseudo probe info
+  - `=decode`: decode probes section from binary
+  - `=address_conversion`: update address2ProbesMap with output block address
+  - `=encoded_probes`: display the encoded probes in binary section
+  - `=all`: enable all debugging printout
+
+- `--print-relocations`
+
+  Print relocations when printing functions/objects
+
+- `--print-reordered-data`
+
+  Print section contents after reordering
+
+- `--print-retpoline-insertion`
+
+  Print functions after retpoline insertion pass
+
+- `--print-sdt`
+
+  Print all SDT markers
+
+- `--print-sections`
+
+  Print all registered sections
+
+- `--print-unknown`
+
+  Print names of functions with unknown control flow
+
+- `--time-opts`
+
+  Print time spent in each optimization
+
+#### Optimization options
+
+- `--print-after-branch-fixup`
+
+  Print function after fixing local branches
+
+- `--print-after-jt-footprint-reduction`
+
+  Print function after jt-footprint-reduction pass
+
+- `--print-after-lowering`
+
+  Print function after instruction lowering
+
+- `--print-cache-metrics`
+
+  Calculate and print various metrics for instruction cache
+
+- `--print-clusters`
+
+  Print clusters
+
+- `--print-finalized`
+
+  Print function after CFG is finalized
+
+- `--print-fix-relaxations`
+
+  Print functions after fix relaxations pass
+
+- `--print-fix-riscv-calls`
+
+  Print functions after fix RISCV calls pass
+
+- `--print-fop`
+
+  Print functions after frame optimizer pass
+
+- `--print-function-statistics=<uint>`
+
+  Print statistics about basic block ordering
+
+- `--print-icf`
+
+  Print functions after ICF optimization
+
+- `--print-icp`
+
+  Print functions after indirect call promotion
+
+- `--print-inline`
+
+  Print functions after inlining optimization
+
+- `--print-longjmp`
+
+  Print functions after longjmp pass
+
+- `--print-optimize-bodyless`
+
+  Print functions after bodyless optimization
+
+- `--print-output-address-range`
+
+  Print output address range for each basic block in the function
+  whenBinaryFunction::print is called
+
+- `--print-peepholes`
+
+  Print functions after peephole optimization
+
+- `--print-plt`
+
+  Print functions after PLT optimization
+
+- `--print-regreassign`
+
+  Print functions after regreassign pass
+
+- `--print-reordered`
+
+  Print functions after layout optimization
+
+- `--print-reordered-functions`
+
+  Print functions after clustering
+
+- `--print-sctc`
+
+  Print functions after conditional tail call simplification
+
+- `--print-simplify-rodata-loads`
+
+  Print functions after simplification of RO data loads
+
+- `--print-sorted-by=<value>`
+
+  Print functions sorted by order of dyno stats
+  - `executed-forward-branches`: executed forward branches
+  - `taken-forward-branches`: taken forward branches
+  - `executed-backward-branches`: executed backward branches
+  - `taken-backward-branches`: taken backward branches
+  - `executed-unconditional-branches`: executed unconditional branches
+  - `all-function-calls`: all function calls
+  - `indirect-calls`: indirect calls
+  - `PLT-calls`: PLT calls
+  - `executed-instructions`: executed instructions
+  - `executed-load-instructions`: executed load instructions
+  - `executed-store-instructions`: executed store instructions
+  - `taken-jump-table-branches`: taken jump table branches
+  - `taken-unknown-indirect-branches`: taken unknown indirect branches
+  - `total-branches`: total branches
+  - `taken-branches`: taken branches
+  - `non-taken-conditional-branches`: non-taken conditional branches
+  - `taken-conditional-branches`: taken conditional branches
+  - `all-conditional-branches`: all conditional branches
+  - `linker-inserted-veneer-calls`: linker-inserted veneer calls
+  - `all`: sorted by all names
+
+- `--print-sorted-by-order=<value>`
+
+  Use ascending or descending order when printing functions ordered by dyno stats
+
+- `--print-split`
+
+  Print functions after code splitting
+
+- `--print-stoke`
+
+  Print functions after stoke analysis
+
+- `--print-uce`
+
+  Print functions after unreachable code elimination
+
+- `--print-veneer-elimination`
+
+  Print functions after veneer elimination pass
+
+- `--time-build`
+
+  Print time spent constructing binary functions
+
+- `--time-rewrite`
+
+  Print time spent in rewriting passes
-- 
GitLab


From 8e00703be9ceb41d9b80c2bc8f024a9610b9aaa1 Mon Sep 17 00:00:00 2001
From: jyu2-git <jennifer.yu@intel.com>
Date: Wed, 15 May 2024 08:20:25 -0700
Subject: [PATCH 0873/1206] [Clang][OpenMP] Fix runtime problem when explicit
 map both pointer and pointee (#92210)

ponter int *p for following map, test currently crash.

  map(p, p[:100]) or map(p, p[1])

Currly IR looks like
// &p, &p, sizeof(int), TARGET_PARAM | TO | FROM
// &p, p[0], 100sizeof(float) TO | FROM

Worrking IR is
// map(p, p[0:100]) to map(p[0:100])
// &p, &p[0], 100*sizeof(float), TARGET_PARAM | TO | FROM | PTR_AND_OBJ

The change is add new argument AreBothBasePtrAndPteeMapped in
generateInfoForComponentList

Use that to skip map for map(p), when processing map(p[:100]) generate
map with right flag.
---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         |  37 +++--
 ...arget_map_both_pointer_pointee_codegen.cpp | 150 ++++++++++++++++++
 .../test/mapping/map_both_pointer_pointee.c   |  42 +++++
 3 files changed, 220 insertions(+), 9 deletions(-)
 create mode 100644 clang/test/OpenMP/target_map_both_pointer_pointee_codegen.cpp
 create mode 100644 offload/test/mapping/map_both_pointer_pointee.c

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index e39c7c58d278..f56af318ff6a 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -6830,7 +6830,8 @@ private:
       const ValueDecl *Mapper = nullptr, bool ForDeviceAddr = false,
       const ValueDecl *BaseDecl = nullptr, const Expr *MapExpr = nullptr,
       ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
-          OverlappedElements = std::nullopt) const {
+          OverlappedElements = std::nullopt,
+      bool AreBothBasePtrAndPteeMapped = false) const {
     // The following summarizes what has to be generated for each map and the
     // types below. The generated information is expressed in this order:
     // base pointer, section pointer, size, flags
@@ -7006,6 +7007,10 @@ private:
     // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(4) | PTR_AND_OBJ | TO
     // (*) the struct this entry pertains to is the 4th element in the list
     //     of arguments, hence MEMBER_OF(4)
+    //
+    // map(p, p[:100])
+    // ===> map(p[:100])
+    // &p, &p[0], 100*sizeof(float), TARGET_PARAM | PTR_AND_OBJ | TO | FROM
 
     // Track if the map information being generated is the first for a capture.
     bool IsCaptureFirstInfo = IsFirstComponentList;
@@ -7029,6 +7034,8 @@ private:
     const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr);
     const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);
 
+    if (AreBothBasePtrAndPteeMapped && std::next(I) == CE)
+      return;
     if (isa<MemberExpr>(AssocExpr)) {
       // The base is the 'this' pointer. The content of the pointer is going
       // to be the base of the field being mapped.
@@ -7071,8 +7078,9 @@ private:
         // can be associated with the combined storage if shared memory mode is
         // active or the base declaration is not global variable.
         const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());
-        if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
-            !VD || VD->hasLocalStorage())
+        if (!AreBothBasePtrAndPteeMapped &&
+            (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
+             !VD || VD->hasLocalStorage()))
           BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
         else
           FirstPointerInComplexData = true;
@@ -7394,11 +7402,13 @@ private:
           // same expression except for the first one. We also need to signal
           // this map is the first one that relates with the current capture
           // (there is a set of entries for each capture).
-          OpenMPOffloadMappingFlags Flags = getMapTypeBits(
-              MapType, MapModifiers, MotionModifiers, IsImplicit,
-              !IsExpressionFirstInfo || RequiresReference ||
-                  FirstPointerInComplexData || IsMemberReference,
-              IsCaptureFirstInfo && !RequiresReference, IsNonContiguous);
+          OpenMPOffloadMappingFlags Flags =
+              getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
+                             !IsExpressionFirstInfo || RequiresReference ||
+                                 FirstPointerInComplexData || IsMemberReference,
+                             AreBothBasePtrAndPteeMapped ||
+                                 (IsCaptureFirstInfo && !RequiresReference),
+                             IsNonContiguous);
 
           if (!IsExpressionFirstInfo || IsMemberReference) {
             // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
@@ -8492,6 +8502,8 @@ public:
     assert(CurDir.is<const OMPExecutableDirective *>() &&
            "Expect a executable directive");
     const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>();
+    bool HasMapBasePtr = false;
+    bool HasMapArraySec = false;
     for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
       const auto *EI = C->getVarRefs().begin();
       for (const auto L : C->decl_component_lists(VD)) {
@@ -8503,6 +8515,11 @@ public:
         assert(VDecl == VD && "We got information for the wrong declaration??");
         assert(!Components.empty() &&
                "Not expecting declaration with no component lists.");
+        if (VD && E && VD->getType()->isAnyPointerType() && isa<DeclRefExpr>(E))
+          HasMapBasePtr = true;
+        if (VD && E && VD->getType()->isAnyPointerType() &&
+            (isa<ArraySectionExpr>(E) || isa<ArraySubscriptExpr>(E)))
+          HasMapArraySec = true;
         DeclComponentLists.emplace_back(Components, C->getMapType(),
                                         C->getMapTypeModifiers(),
                                         C->isImplicit(), Mapper, E);
@@ -8685,7 +8702,9 @@ public:
             MapType, MapModifiers, std::nullopt, Components, CombinedInfo,
             StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,
             IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,
-            /*ForDeviceAddr=*/false, VD, VarRef);
+            /*ForDeviceAddr=*/false, VD, VarRef,
+            /*OverlappedElements*/ std::nullopt,
+            HasMapBasePtr && HasMapArraySec);
       IsFirstComponentList = false;
     }
   }
diff --git a/clang/test/OpenMP/target_map_both_pointer_pointee_codegen.cpp b/clang/test/OpenMP/target_map_both_pointer_pointee_codegen.cpp
new file mode 100644
index 000000000000..e2c27f37f5b9
--- /dev/null
+++ b/clang/test/OpenMP/target_map_both_pointer_pointee_codegen.cpp
@@ -0,0 +1,150 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+
+// expected-no-diagnostics
+#ifndef HEADER
+#define HEADER
+
+extern void *malloc (int __size) throw () __attribute__ ((__malloc__));
+
+void foo() {
+  int *ptr = (int *) malloc(3 * sizeof(int));
+
+  #pragma omp target map(ptr, ptr[0:2])
+  {
+    ptr[1] = 6;
+  }
+  #pragma omp target map(ptr, ptr[2])
+  {
+    ptr[2] = 8;
+  }
+}
+#endif
+// CHECK-LABEL: define {{[^@]+}}@_Z3foov
+// CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[PTR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[DOTOFFLOAD_BASEPTRS:%.*]] = alloca [1 x ptr], align 8
+// CHECK-NEXT:    [[DOTOFFLOAD_PTRS:%.*]] = alloca [1 x ptr], align 8
+// CHECK-NEXT:    [[DOTOFFLOAD_MAPPERS:%.*]] = alloca [1 x ptr], align 8
+// CHECK-NEXT:    [[KERNEL_ARGS:%.*]] = alloca [[STRUCT___TGT_KERNEL_ARGUMENTS:%.*]], align 8
+// CHECK-NEXT:    [[DOTOFFLOAD_BASEPTRS2:%.*]] = alloca [1 x ptr], align 8
+// CHECK-NEXT:    [[DOTOFFLOAD_PTRS3:%.*]] = alloca [1 x ptr], align 8
+// CHECK-NEXT:    [[DOTOFFLOAD_MAPPERS4:%.*]] = alloca [1 x ptr], align 8
+// CHECK-NEXT:    [[KERNEL_ARGS5:%.*]] = alloca [[STRUCT___TGT_KERNEL_ARGUMENTS]], align 8
+// CHECK-NEXT:    [[CALL:%.*]] = call noalias noundef ptr @_Z6malloci(i32 noundef signext 12) #[[ATTR3:[0-9]+]]
+// CHECK-NEXT:    store ptr [[CALL]], ptr [[PTR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 0
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK-NEXT:    store ptr [[PTR]], ptr [[TMP2]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK-NEXT:    store ptr [[ARRAYIDX]], ptr [[TMP3]], align 8
+// CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS]], i64 0, i64 0
+// CHECK-NEXT:    store ptr null, ptr [[TMP4]], align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 0
+// CHECK-NEXT:    store i32 3, ptr [[TMP7]], align 4
+// CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 1
+// CHECK-NEXT:    store i32 1, ptr [[TMP8]], align 4
+// CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 2
+// CHECK-NEXT:    store ptr [[TMP5]], ptr [[TMP9]], align 8
+// CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 3
+// CHECK-NEXT:    store ptr [[TMP6]], ptr [[TMP10]], align 8
+// CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 4
+// CHECK-NEXT:    store ptr @.offload_sizes, ptr [[TMP11]], align 8
+// CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 5
+// CHECK-NEXT:    store ptr @.offload_maptypes, ptr [[TMP12]], align 8
+// CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 6
+// CHECK-NEXT:    store ptr null, ptr [[TMP13]], align 8
+// CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 7
+// CHECK-NEXT:    store ptr null, ptr [[TMP14]], align 8
+// CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 8
+// CHECK-NEXT:    store i64 0, ptr [[TMP15]], align 8
+// CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 9
+// CHECK-NEXT:    store i64 0, ptr [[TMP16]], align 8
+// CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 10
+// CHECK-NEXT:    store [3 x i32] [i32 -1, i32 0, i32 0], ptr [[TMP17]], align 4
+// CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 11
+// CHECK-NEXT:    store [3 x i32] zeroinitializer, ptr [[TMP18]], align 4
+// CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS]], i32 0, i32 12
+// CHECK-NEXT:    store i32 0, ptr [[TMP19]], align 4
+// CHECK-NEXT:    [[TMP20:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1:[0-9]+]], i64 -1, i32 -1, i32 0, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l15.region_id, ptr [[KERNEL_ARGS]])
+// CHECK-NEXT:    [[TMP21:%.*]] = icmp ne i32 [[TMP20]], 0
+// CHECK-NEXT:    br i1 [[TMP21]], label [[OMP_OFFLOAD_FAILED:%.*]], label [[OMP_OFFLOAD_CONT:%.*]]
+// CHECK:       omp_offload.failed:
+// CHECK-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l15(ptr [[TMP0]]) #[[ATTR3]]
+// CHECK-NEXT:    br label [[OMP_OFFLOAD_CONT]]
+// CHECK:       omp_offload.cont:
+// CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[PTR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i64 2
+// CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS2]], i32 0, i32 0
+// CHECK-NEXT:    store ptr [[PTR]], ptr [[TMP24]], align 8
+// CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS3]], i32 0, i32 0
+// CHECK-NEXT:    store ptr [[ARRAYIDX1]], ptr [[TMP25]], align 8
+// CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_MAPPERS4]], i64 0, i64 0
+// CHECK-NEXT:    store ptr null, ptr [[TMP26]], align 8
+// CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_BASEPTRS2]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds [1 x ptr], ptr [[DOTOFFLOAD_PTRS3]], i32 0, i32 0
+// CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 0
+// CHECK-NEXT:    store i32 3, ptr [[TMP29]], align 4
+// CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 1
+// CHECK-NEXT:    store i32 1, ptr [[TMP30]], align 4
+// CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 2
+// CHECK-NEXT:    store ptr [[TMP27]], ptr [[TMP31]], align 8
+// CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 3
+// CHECK-NEXT:    store ptr [[TMP28]], ptr [[TMP32]], align 8
+// CHECK-NEXT:    [[TMP33:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 4
+// CHECK-NEXT:    store ptr @.offload_sizes.1, ptr [[TMP33]], align 8
+// CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 5
+// CHECK-NEXT:    store ptr @.offload_maptypes.2, ptr [[TMP34]], align 8
+// CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 6
+// CHECK-NEXT:    store ptr null, ptr [[TMP35]], align 8
+// CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 7
+// CHECK-NEXT:    store ptr null, ptr [[TMP36]], align 8
+// CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 8
+// CHECK-NEXT:    store i64 0, ptr [[TMP37]], align 8
+// CHECK-NEXT:    [[TMP38:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 9
+// CHECK-NEXT:    store i64 0, ptr [[TMP38]], align 8
+// CHECK-NEXT:    [[TMP39:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 10
+// CHECK-NEXT:    store [3 x i32] [i32 -1, i32 0, i32 0], ptr [[TMP39]], align 4
+// CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 11
+// CHECK-NEXT:    store [3 x i32] zeroinitializer, ptr [[TMP40]], align 4
+// CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS5]], i32 0, i32 12
+// CHECK-NEXT:    store i32 0, ptr [[TMP41]], align 4
+// CHECK-NEXT:    [[TMP42:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 -1, i32 0, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l19.region_id, ptr [[KERNEL_ARGS5]])
+// CHECK-NEXT:    [[TMP43:%.*]] = icmp ne i32 [[TMP42]], 0
+// CHECK-NEXT:    br i1 [[TMP43]], label [[OMP_OFFLOAD_FAILED6:%.*]], label [[OMP_OFFLOAD_CONT7:%.*]]
+// CHECK:       omp_offload.failed6:
+// CHECK-NEXT:    call void @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l19(ptr [[TMP22]]) #[[ATTR3]]
+// CHECK-NEXT:    br label [[OMP_OFFLOAD_CONT7]]
+// CHECK:       omp_offload.cont7:
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l15
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
+// CHECK-NEXT:    store i32 6, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3foov_l19
+// CHECK-SAME: (ptr noundef [[PTR:%.*]]) #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+// CHECK-NEXT:    store i32 8, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    ret void
+//
diff --git a/offload/test/mapping/map_both_pointer_pointee.c b/offload/test/mapping/map_both_pointer_pointee.c
new file mode 100644
index 000000000000..4b724823e7a4
--- /dev/null
+++ b/offload/test/mapping/map_both_pointer_pointee.c
@@ -0,0 +1,42 @@
+// RUN: %libomptarget-compilexx-run-and-check-aarch64-unknown-linux-gnu
+// RUN: %libomptarget-compilexx-run-and-check-powerpc64-ibm-linux-gnu
+// RUN: %libomptarget-compilexx-run-and-check-powerpc64le-ibm-linux-gnu
+// RUN: %libomptarget-compilexx-run-and-check-x86_64-pc-linux-gnu
+// RUN: %libomptarget-compilexx-run-and-check-nvptx64-nvidia-cuda
+
+// REQUIRES: unified_shared_memory
+// UNSUPPORTED: amdgcn-amd-amdhsa
+
+#pragma omp declare target
+int *ptr1;
+#pragma omp end declare target
+
+#include <stdio.h>
+#include <stdlib.h>
+int main() {
+  ptr1 = (int *)malloc(sizeof(int) * 100);
+  int *ptr2;
+  ptr2 = (int *)malloc(sizeof(int) * 100);
+#pragma omp target map(ptr1, ptr1[ : 100])
+  { ptr1[1] = 6; }
+  // CHECK: 6
+  printf(" %d \n", ptr1[1]);
+#pragma omp target data map(ptr1[ : 5])
+  {
+#pragma omp target map(ptr1[2], ptr1, ptr1[3]) map(ptr2, ptr2[2])
+    {
+      ptr1[2] = 7;
+      ptr1[3] = 9;
+      ptr2[2] = 7;
+    }
+  }
+  // CHECK: 7 7 9
+  printf(" %d %d %d \n", ptr2[2], ptr1[2], ptr1[3]);
+  free(ptr1);
+#pragma omp target map(ptr2, ptr2[ : 100])
+  { ptr2[1] = 6; }
+  // CHECK: 6
+  printf(" %d \n", ptr2[1]);
+  free(ptr2);
+  return 0;
+}
-- 
GitLab


From ff313ee70a4f27e3555ee4baef53b9b51c5aa27e Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Wed, 15 May 2024 23:37:31 +0800
Subject: [PATCH 0874/1206] [RISCV] Remove hasSideEffects=1 for vsetvli pseudos
 (#91319)

In a similar vein to #90049, we currently model all of the effects of a
vsetvli pseudo:

* VL and VTYPE are marked as defs
* VL preserving x0,x0 vsetvlis doesn't get emitted until
RISCVInsertVSETVLI, and when they are they have implicit uses on VL
* Regular vector pseudos are fully modelled too: Before
RISCVInsertVSETVLI they can be moved between vsetvli pseudos because we
will eventually insert vsetvlis to correct VL and VTYPE. Afterwards,
they will have implicit uses on VL and VTYPE.

Since we model everything we can remove hasSideEffects=1. This gives us
some improvements like sinking in vsetvli-insert-crossbb.ll.

We need to update RISCVDeadRegisterDefinitions to keep handling vsetvli
pseudos since it only operates on instructions with unmodelled side
effects.
---
 .../RISCV/RISCVDeadRegisterDefinitions.cpp    |   4 +-
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    |   2 +-
 .../CodeGen/RISCV/rvv/calling-conv-fastcc.ll  |  38 +-
 llvm/test/CodeGen/RISCV/rvv/calling-conv.ll   |   8 +-
 ...d-vectors-fnearbyint-constrained-sdnode.ll |  24 +-
 .../RISCV/rvv/fixed-vectors-fp2i-sat.ll       |  24 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll   |  80 ++--
 .../RISCV/rvv/fixed-vectors-int-buildvec.ll   | 154 ++++----
 ...fixed-vectors-interleaved-access-zve32x.ll |  38 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-lrint.ll  |  20 +-
 .../RISCV/rvv/fixed-vectors-masked-gather.ll  | 372 +++++++++---------
 .../RISCV/rvv/fixed-vectors-masked-scatter.ll |  69 ++--
 .../RISCV/rvv/fixed-vectors-nearbyint-vp.ll   |  56 +--
 .../RISCV/rvv/fixed-vectors-vselect.ll        |  24 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-vwadd.ll  |   8 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll |   8 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll  |   2 +-
 .../RISCV/rvv/fixed-vectors-vwmulsu.ll        |   6 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-vwsub.ll  |   8 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-vwsubu.ll |   8 +-
 llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll    | 154 +++++---
 llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll    | 154 +++++---
 .../rvv/fnearbyint-constrained-sdnode.ll      |  30 +-
 .../CodeGen/RISCV/rvv/fnearbyint-sdnode.ll    |  30 +-
 llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll   | 294 ++++++--------
 llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll |  19 +-
 .../test/CodeGen/RISCV/rvv/mscatter-sdnode.ll |   2 +-
 llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll   | 197 +++++-----
 llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll    | 268 ++++++-------
 llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll   |  40 +-
 llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll        |  34 +-
 llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll    |  34 +-
 llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll |  14 +-
 llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll     |   6 +-
 .../RISCV/rvv/vsetvli-insert-crossbb.ll       |  39 +-
 .../CodeGen/RISCV/rvv/vsetvli-regression.ll   |   5 +-
 36 files changed, 1107 insertions(+), 1166 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
index 5e6b7891449f..7de48d8218f0 100644
--- a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
+++ b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
@@ -72,7 +72,9 @@ bool RISCVDeadRegisterDefinitions::runOnMachineFunction(MachineFunction &MF) {
       // are reserved for HINT instructions.
       const MCInstrDesc &Desc = MI.getDesc();
       if (!Desc.mayLoad() && !Desc.mayStore() &&
-          !Desc.hasUnmodeledSideEffects())
+          !Desc.hasUnmodeledSideEffects() &&
+          MI.getOpcode() != RISCV::PseudoVSETVLI &&
+          MI.getOpcode() != RISCV::PseudoVSETIVLI)
         continue;
       // For PseudoVSETVLIX0, Rd = X0 has special meaning.
       if (MI.getOpcode() == RISCV::PseudoVSETVLIX0)
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 4adc26f62891..317a6d7d4c52 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -6181,7 +6181,7 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Size = 0,
 //===----------------------------------------------------------------------===//
 
 // Pseudos.
-let hasSideEffects = 1, mayLoad = 0, mayStore = 0, Defs = [VL, VTYPE] in {
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Defs = [VL, VTYPE] in {
 // Due to rs1=X0 having special meaning, we need a GPRNoX0 register class for
 // the when we aren't using one of the special X0 encodings. Otherwise it could
 // be accidentally be made X0 by MachineIR optimizations. To satisfy the
diff --git a/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll b/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll
index 187f758b7802..0a7fa38b0c8a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/calling-conv-fastcc.ll
@@ -236,11 +236,12 @@ define fastcc <vscale x 32 x i32> @ret_nxv32i32_param_nxv32i32_nxv32i32_nxv32i32
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 4
+; CHECK-NEXT:    li a3, 24
+; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
@@ -248,29 +249,40 @@ define fastcc <vscale x 32 x i32> @ret_nxv32i32_param_nxv32i32_nxv32i32_nxv32i32
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    add a3, a2, a1
-; CHECK-NEXT:    vl8re32.v v8, (a3)
-; CHECK-NEXT:    addi a3, sp, 16
-; CHECK-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
 ; CHECK-NEXT:    add a1, a0, a1
-; CHECK-NEXT:    vl8re32.v v0, (a0)
 ; CHECK-NEXT:    vl8re32.v v8, (a1)
-; CHECK-NEXT:    vl8re32.v v16, (a2)
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT:    vl8re32.v v0, (a0)
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vl8re32.v v8, (a3)
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vl8re32.v v16, (a2)
 ; CHECK-NEXT:    vadd.vv v0, v24, v0
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vadd.vv v8, v24, v8
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vadd.vv v24, v24, v8
 ; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vadd.vv v8, v8, v24
+; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vadd.vv v8, v24, v8
 ; CHECK-NEXT:    vadd.vv v24, v0, v16
 ; CHECK-NEXT:    vadd.vx v16, v8, a4
 ; CHECK-NEXT:    vadd.vx v8, v24, a4
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    li a1, 24
+; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add sp, sp, a0
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
index 647d3158b616..fa62143546df 100644
--- a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
@@ -39,11 +39,11 @@ define <vscale x 32 x i32> @caller_scalable_vector_split_indirect(<vscale x 32 x
 ; RV32-NEXT:    vs8r.v v8, (a0)
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a0, a0, a1
-; RV32-NEXT:    vs8r.v v16, (a0)
+; RV32-NEXT:    add a1, a0, a1
 ; RV32-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
 ; RV32-NEXT:    vmv.v.i v8, 0
 ; RV32-NEXT:    addi a0, sp, 128
+; RV32-NEXT:    vs8r.v v16, (a1)
 ; RV32-NEXT:    vmv.v.i v16, 0
 ; RV32-NEXT:    call callee_scalable_vector_split_indirect
 ; RV32-NEXT:    addi sp, s0, -144
@@ -70,11 +70,11 @@ define <vscale x 32 x i32> @caller_scalable_vector_split_indirect(<vscale x 32 x
 ; RV64-NEXT:    vs8r.v v8, (a0)
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    slli a1, a1, 3
-; RV64-NEXT:    add a0, a0, a1
-; RV64-NEXT:    vs8r.v v16, (a0)
+; RV64-NEXT:    add a1, a0, a1
 ; RV64-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
 ; RV64-NEXT:    vmv.v.i v8, 0
 ; RV64-NEXT:    addi a0, sp, 128
+; RV64-NEXT:    vs8r.v v16, (a1)
 ; RV64-NEXT:    vmv.v.i v16, 0
 ; RV64-NEXT:    call callee_scalable_vector_split_indirect
 ; RV64-NEXT:    addi sp, s0, -144
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fnearbyint-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fnearbyint-constrained-sdnode.ll
index 1b50214bbf16..9e9a8b8a4b64 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fnearbyint-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fnearbyint-constrained-sdnode.ll
@@ -19,9 +19,9 @@ define <2 x half> @nearbyint_v2f16(<2 x half> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <2 x half> @llvm.experimental.constrained.nearbyint.v2f16(<2 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <2 x half> %r
@@ -42,9 +42,9 @@ define <4 x half> @nearbyint_v4f16(<4 x half> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <4 x half> @llvm.experimental.constrained.nearbyint.v4f16(<4 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <4 x half> %r
@@ -65,9 +65,9 @@ define <8 x half> @nearbyint_v8f16(<8 x half> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <8 x half> @llvm.experimental.constrained.nearbyint.v8f16(<8 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <8 x half> %r
@@ -88,9 +88,9 @@ define <16 x half> @nearbyint_v16f16(<16 x half> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <16 x half> @llvm.experimental.constrained.nearbyint.v16f16(<16 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <16 x half> %r
@@ -112,9 +112,9 @@ define <32 x half> @nearbyint_v32f16(<32 x half> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <32 x half> @llvm.experimental.constrained.nearbyint.v32f16(<32 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <32 x half> %r
@@ -135,9 +135,9 @@ define <2 x float> @nearbyint_v2f32(<2 x float> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <2 x float> @llvm.experimental.constrained.nearbyint.v2f32(<2 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <2 x float> %r
@@ -158,9 +158,9 @@ define <4 x float> @nearbyint_v4f32(<4 x float> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <4 x float> @llvm.experimental.constrained.nearbyint.v4f32(<4 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <4 x float> %r
@@ -181,9 +181,9 @@ define <8 x float> @nearbyint_v8f32(<8 x float> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <8 x float> @llvm.experimental.constrained.nearbyint.v8f32(<8 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <8 x float> %r
@@ -204,9 +204,9 @@ define <16 x float> @nearbyint_v16f32(<16 x float> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <16 x float> @llvm.experimental.constrained.nearbyint.v16f32(<16 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <16 x float> %r
@@ -227,9 +227,9 @@ define <2 x double> @nearbyint_v2f64(<2 x double> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(<2 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <2 x double> %r
@@ -250,9 +250,9 @@ define <4 x double> @nearbyint_v4f64(<4 x double> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <4 x double> @llvm.experimental.constrained.nearbyint.v4f64(<4 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <4 x double> %r
@@ -273,9 +273,9 @@ define <8 x double> @nearbyint_v8f64(<8 x double> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <8 x double> @llvm.experimental.constrained.nearbyint.v8f64(<8 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <8 x double> %r
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll
index a8e4af2d7368..6320b07125bb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll
@@ -359,13 +359,13 @@ define void @fp2si_v8f64_v8i8(ptr %x, ptr %y) {
 ; RV32-NEXT:    feq.d a0, fa3, fa3
 ; RV32-NEXT:    fmax.d fa3, fa3, fa5
 ; RV32-NEXT:    fmin.d fa3, fa3, fa4
-; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; RV32-NEXT:    fld fa2, 40(sp)
 ; RV32-NEXT:    fcvt.w.d a2, fa3, rtz
+; RV32-NEXT:    fld fa3, 40(sp)
 ; RV32-NEXT:    neg a0, a0
 ; RV32-NEXT:    and a0, a0, a2
-; RV32-NEXT:    feq.d a2, fa2, fa2
-; RV32-NEXT:    fmax.d fa3, fa2, fa5
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
+; RV32-NEXT:    feq.d a2, fa3, fa3
+; RV32-NEXT:    fmax.d fa3, fa3, fa5
 ; RV32-NEXT:    fmin.d fa3, fa3, fa4
 ; RV32-NEXT:    fcvt.w.d a3, fa3, rtz
 ; RV32-NEXT:    fld fa3, 32(sp)
@@ -460,13 +460,13 @@ define void @fp2si_v8f64_v8i8(ptr %x, ptr %y) {
 ; RV64-NEXT:    feq.d a0, fa3, fa3
 ; RV64-NEXT:    fmax.d fa3, fa3, fa5
 ; RV64-NEXT:    fmin.d fa3, fa3, fa4
-; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; RV64-NEXT:    fld fa2, 40(sp)
 ; RV64-NEXT:    fcvt.l.d a2, fa3, rtz
+; RV64-NEXT:    fld fa3, 40(sp)
 ; RV64-NEXT:    neg a0, a0
 ; RV64-NEXT:    and a0, a0, a2
-; RV64-NEXT:    feq.d a2, fa2, fa2
-; RV64-NEXT:    fmax.d fa3, fa2, fa5
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
+; RV64-NEXT:    feq.d a2, fa3, fa3
+; RV64-NEXT:    fmax.d fa3, fa3, fa5
 ; RV64-NEXT:    fmin.d fa3, fa3, fa4
 ; RV64-NEXT:    fcvt.l.d a3, fa3, rtz
 ; RV64-NEXT:    fld fa3, 32(sp)
@@ -557,7 +557,6 @@ define void @fp2ui_v8f64_v8i8(ptr %x, ptr %y) {
 ; RV32-NEXT:    vslidedown.vi v8, v8, 3
 ; RV32-NEXT:    vfmv.f.s fa4, v8
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
-; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; RV32-NEXT:    fld fa2, 40(sp)
 ; RV32-NEXT:    fmin.d fa4, fa4, fa5
 ; RV32-NEXT:    fcvt.wu.d a0, fa4, rtz
@@ -566,9 +565,10 @@ define void @fp2ui_v8f64_v8i8(ptr %x, ptr %y) {
 ; RV32-NEXT:    fmin.d fa2, fa2, fa5
 ; RV32-NEXT:    fcvt.wu.d a2, fa2, rtz
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
-; RV32-NEXT:    fld fa2, 48(sp)
 ; RV32-NEXT:    fmin.d fa4, fa4, fa5
+; RV32-NEXT:    fld fa2, 48(sp)
 ; RV32-NEXT:    fcvt.wu.d a3, fa4, rtz
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; RV32-NEXT:    vslide1down.vx v8, v10, a0
 ; RV32-NEXT:    fmax.d fa4, fa2, fa3
 ; RV32-NEXT:    fmin.d fa4, fa4, fa5
@@ -633,7 +633,6 @@ define void @fp2ui_v8f64_v8i8(ptr %x, ptr %y) {
 ; RV64-NEXT:    vslidedown.vi v8, v8, 3
 ; RV64-NEXT:    vfmv.f.s fa4, v8
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
-; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; RV64-NEXT:    fld fa2, 40(sp)
 ; RV64-NEXT:    fmin.d fa4, fa4, fa5
 ; RV64-NEXT:    fcvt.lu.d a0, fa4, rtz
@@ -642,9 +641,10 @@ define void @fp2ui_v8f64_v8i8(ptr %x, ptr %y) {
 ; RV64-NEXT:    fmin.d fa2, fa2, fa5
 ; RV64-NEXT:    fcvt.lu.d a2, fa2, rtz
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
-; RV64-NEXT:    fld fa2, 48(sp)
 ; RV64-NEXT:    fmin.d fa4, fa4, fa5
+; RV64-NEXT:    fld fa2, 48(sp)
 ; RV64-NEXT:    fcvt.lu.d a3, fa4, rtz
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; RV64-NEXT:    vslide1down.vx v8, v10, a0
 ; RV64-NEXT:    fmax.d fa4, fa2, fa3
 ; RV64-NEXT:    fmin.d fa4, fa4, fa5
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll
index 6ffa6ac250ed..9c76b83d0974 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll
@@ -132,12 +132,12 @@ define <3 x float> @si2fp_v3i1_v3f32(<3 x i1> %x) {
 define <3 x float> @si2fp_v3i7_v3f32(<3 x i7> %x) {
 ; ZVFH32-LABEL: si2fp_v3i7_v3f32:
 ; ZVFH32:       # %bb.0:
-; ZVFH32-NEXT:    lw a1, 4(a0)
-; ZVFH32-NEXT:    lw a2, 0(a0)
-; ZVFH32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFH32-NEXT:    lw a1, 0(a0)
+; ZVFH32-NEXT:    lw a2, 4(a0)
 ; ZVFH32-NEXT:    lw a0, 8(a0)
-; ZVFH32-NEXT:    vmv.v.x v8, a2
-; ZVFH32-NEXT:    vslide1down.vx v8, v8, a1
+; ZVFH32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFH32-NEXT:    vmv.v.x v8, a1
+; ZVFH32-NEXT:    vslide1down.vx v8, v8, a2
 ; ZVFH32-NEXT:    vslide1down.vx v8, v8, a0
 ; ZVFH32-NEXT:    vslidedown.vi v8, v8, 1
 ; ZVFH32-NEXT:    vadd.vv v8, v8, v8
@@ -149,12 +149,12 @@ define <3 x float> @si2fp_v3i7_v3f32(<3 x i7> %x) {
 ;
 ; ZVFH64-LABEL: si2fp_v3i7_v3f32:
 ; ZVFH64:       # %bb.0:
-; ZVFH64-NEXT:    ld a1, 8(a0)
-; ZVFH64-NEXT:    ld a2, 0(a0)
-; ZVFH64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFH64-NEXT:    ld a1, 0(a0)
+; ZVFH64-NEXT:    ld a2, 8(a0)
 ; ZVFH64-NEXT:    ld a0, 16(a0)
-; ZVFH64-NEXT:    vmv.v.x v8, a2
-; ZVFH64-NEXT:    vslide1down.vx v8, v8, a1
+; ZVFH64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFH64-NEXT:    vmv.v.x v8, a1
+; ZVFH64-NEXT:    vslide1down.vx v8, v8, a2
 ; ZVFH64-NEXT:    vslide1down.vx v8, v8, a0
 ; ZVFH64-NEXT:    vslidedown.vi v8, v8, 1
 ; ZVFH64-NEXT:    vadd.vv v8, v8, v8
@@ -166,12 +166,12 @@ define <3 x float> @si2fp_v3i7_v3f32(<3 x i7> %x) {
 ;
 ; ZVFHMIN32-LABEL: si2fp_v3i7_v3f32:
 ; ZVFHMIN32:       # %bb.0:
-; ZVFHMIN32-NEXT:    lw a1, 4(a0)
-; ZVFHMIN32-NEXT:    lw a2, 0(a0)
-; ZVFHMIN32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFHMIN32-NEXT:    lw a1, 0(a0)
+; ZVFHMIN32-NEXT:    lw a2, 4(a0)
 ; ZVFHMIN32-NEXT:    lw a0, 8(a0)
-; ZVFHMIN32-NEXT:    vmv.v.x v8, a2
-; ZVFHMIN32-NEXT:    vslide1down.vx v8, v8, a1
+; ZVFHMIN32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFHMIN32-NEXT:    vmv.v.x v8, a1
+; ZVFHMIN32-NEXT:    vslide1down.vx v8, v8, a2
 ; ZVFHMIN32-NEXT:    vslide1down.vx v8, v8, a0
 ; ZVFHMIN32-NEXT:    vslidedown.vi v8, v8, 1
 ; ZVFHMIN32-NEXT:    vadd.vv v8, v8, v8
@@ -183,12 +183,12 @@ define <3 x float> @si2fp_v3i7_v3f32(<3 x i7> %x) {
 ;
 ; ZVFHMIN64-LABEL: si2fp_v3i7_v3f32:
 ; ZVFHMIN64:       # %bb.0:
-; ZVFHMIN64-NEXT:    ld a1, 8(a0)
-; ZVFHMIN64-NEXT:    ld a2, 0(a0)
-; ZVFHMIN64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFHMIN64-NEXT:    ld a1, 0(a0)
+; ZVFHMIN64-NEXT:    ld a2, 8(a0)
 ; ZVFHMIN64-NEXT:    ld a0, 16(a0)
-; ZVFHMIN64-NEXT:    vmv.v.x v8, a2
-; ZVFHMIN64-NEXT:    vslide1down.vx v8, v8, a1
+; ZVFHMIN64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFHMIN64-NEXT:    vmv.v.x v8, a1
+; ZVFHMIN64-NEXT:    vslide1down.vx v8, v8, a2
 ; ZVFHMIN64-NEXT:    vslide1down.vx v8, v8, a0
 ; ZVFHMIN64-NEXT:    vslidedown.vi v8, v8, 1
 ; ZVFHMIN64-NEXT:    vadd.vv v8, v8, v8
@@ -205,12 +205,12 @@ define <3 x float> @si2fp_v3i7_v3f32(<3 x i7> %x) {
 define <3 x float> @ui2fp_v3i7_v3f32(<3 x i7> %x) {
 ; ZVFH32-LABEL: ui2fp_v3i7_v3f32:
 ; ZVFH32:       # %bb.0:
-; ZVFH32-NEXT:    lw a1, 4(a0)
-; ZVFH32-NEXT:    lw a2, 0(a0)
-; ZVFH32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFH32-NEXT:    lw a1, 0(a0)
+; ZVFH32-NEXT:    lw a2, 4(a0)
 ; ZVFH32-NEXT:    lw a0, 8(a0)
-; ZVFH32-NEXT:    vmv.v.x v8, a2
-; ZVFH32-NEXT:    vslide1down.vx v8, v8, a1
+; ZVFH32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFH32-NEXT:    vmv.v.x v8, a1
+; ZVFH32-NEXT:    vslide1down.vx v8, v8, a2
 ; ZVFH32-NEXT:    vslide1down.vx v8, v8, a0
 ; ZVFH32-NEXT:    vslidedown.vi v8, v8, 1
 ; ZVFH32-NEXT:    li a0, 127
@@ -222,12 +222,12 @@ define <3 x float> @ui2fp_v3i7_v3f32(<3 x i7> %x) {
 ;
 ; ZVFH64-LABEL: ui2fp_v3i7_v3f32:
 ; ZVFH64:       # %bb.0:
-; ZVFH64-NEXT:    ld a1, 8(a0)
-; ZVFH64-NEXT:    ld a2, 0(a0)
-; ZVFH64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFH64-NEXT:    ld a1, 0(a0)
+; ZVFH64-NEXT:    ld a2, 8(a0)
 ; ZVFH64-NEXT:    ld a0, 16(a0)
-; ZVFH64-NEXT:    vmv.v.x v8, a2
-; ZVFH64-NEXT:    vslide1down.vx v8, v8, a1
+; ZVFH64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFH64-NEXT:    vmv.v.x v8, a1
+; ZVFH64-NEXT:    vslide1down.vx v8, v8, a2
 ; ZVFH64-NEXT:    vslide1down.vx v8, v8, a0
 ; ZVFH64-NEXT:    vslidedown.vi v8, v8, 1
 ; ZVFH64-NEXT:    li a0, 127
@@ -239,12 +239,12 @@ define <3 x float> @ui2fp_v3i7_v3f32(<3 x i7> %x) {
 ;
 ; ZVFHMIN32-LABEL: ui2fp_v3i7_v3f32:
 ; ZVFHMIN32:       # %bb.0:
-; ZVFHMIN32-NEXT:    lw a1, 4(a0)
-; ZVFHMIN32-NEXT:    lw a2, 0(a0)
-; ZVFHMIN32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFHMIN32-NEXT:    lw a1, 0(a0)
+; ZVFHMIN32-NEXT:    lw a2, 4(a0)
 ; ZVFHMIN32-NEXT:    lw a0, 8(a0)
-; ZVFHMIN32-NEXT:    vmv.v.x v8, a2
-; ZVFHMIN32-NEXT:    vslide1down.vx v8, v8, a1
+; ZVFHMIN32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFHMIN32-NEXT:    vmv.v.x v8, a1
+; ZVFHMIN32-NEXT:    vslide1down.vx v8, v8, a2
 ; ZVFHMIN32-NEXT:    vslide1down.vx v8, v8, a0
 ; ZVFHMIN32-NEXT:    vslidedown.vi v8, v8, 1
 ; ZVFHMIN32-NEXT:    li a0, 127
@@ -256,12 +256,12 @@ define <3 x float> @ui2fp_v3i7_v3f32(<3 x i7> %x) {
 ;
 ; ZVFHMIN64-LABEL: ui2fp_v3i7_v3f32:
 ; ZVFHMIN64:       # %bb.0:
-; ZVFHMIN64-NEXT:    ld a1, 8(a0)
-; ZVFHMIN64-NEXT:    ld a2, 0(a0)
-; ZVFHMIN64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFHMIN64-NEXT:    ld a1, 0(a0)
+; ZVFHMIN64-NEXT:    ld a2, 8(a0)
 ; ZVFHMIN64-NEXT:    ld a0, 16(a0)
-; ZVFHMIN64-NEXT:    vmv.v.x v8, a2
-; ZVFHMIN64-NEXT:    vslide1down.vx v8, v8, a1
+; ZVFHMIN64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVFHMIN64-NEXT:    vmv.v.x v8, a1
+; ZVFHMIN64-NEXT:    vslide1down.vx v8, v8, a2
 ; ZVFHMIN64-NEXT:    vslide1down.vx v8, v8, a0
 ; ZVFHMIN64-NEXT:    vslidedown.vi v8, v8, 1
 ; ZVFHMIN64-NEXT:    li a0, 127
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
index 592ce6fc5be0..4f4f0a09de74 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
@@ -1183,38 +1183,38 @@ define <8 x i64> @v8xi64_exact_undef_prefix(i64 %a, i64 %b, i64 %c, i64 %d) vsca
 define <16 x i8> @buildvec_v16i8_loads_contigous(ptr %p) {
 ; CHECK-LABEL: buildvec_v16i8_loads_contigous:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lbu a1, 1(a0)
-; CHECK-NEXT:    lbu a2, 2(a0)
-; CHECK-NEXT:    lbu a3, 3(a0)
-; CHECK-NEXT:    lbu a4, 4(a0)
-; CHECK-NEXT:    lbu a5, 5(a0)
-; CHECK-NEXT:    lbu a6, 6(a0)
-; CHECK-NEXT:    lbu a7, 7(a0)
-; CHECK-NEXT:    lbu t0, 9(a0)
-; CHECK-NEXT:    lbu t1, 10(a0)
-; CHECK-NEXT:    lbu t2, 11(a0)
-; CHECK-NEXT:    lbu t3, 12(a0)
-; CHECK-NEXT:    lbu t4, 13(a0)
-; CHECK-NEXT:    lbu t5, 14(a0)
-; CHECK-NEXT:    lbu t6, 15(a0)
+; CHECK-NEXT:    addi a1, a0, 8
+; CHECK-NEXT:    lbu a2, 1(a0)
+; CHECK-NEXT:    lbu a3, 2(a0)
+; CHECK-NEXT:    lbu a4, 3(a0)
+; CHECK-NEXT:    lbu a5, 4(a0)
+; CHECK-NEXT:    lbu a6, 5(a0)
+; CHECK-NEXT:    lbu a7, 6(a0)
+; CHECK-NEXT:    lbu t0, 7(a0)
+; CHECK-NEXT:    lbu t1, 9(a0)
+; CHECK-NEXT:    lbu t2, 10(a0)
+; CHECK-NEXT:    lbu t3, 11(a0)
+; CHECK-NEXT:    lbu t4, 12(a0)
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vlse8.v v8, (a0), zero
-; CHECK-NEXT:    addi a0, a0, 8
-; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    lbu t5, 13(a0)
+; CHECK-NEXT:    lbu t6, 14(a0)
+; CHECK-NEXT:    lbu a0, 15(a0)
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a2
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a3
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a4
-; CHECK-NEXT:    vlse8.v v9, (a0), zero
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a5
+; CHECK-NEXT:    vlse8.v v9, (a1), zero
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a6
-; CHECK-NEXT:    vslide1down.vx v10, v8, a7
-; CHECK-NEXT:    vslide1down.vx v8, v9, t0
-; CHECK-NEXT:    vslide1down.vx v8, v8, t1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a7
+; CHECK-NEXT:    vslide1down.vx v10, v8, t0
+; CHECK-NEXT:    vslide1down.vx v8, v9, t1
 ; CHECK-NEXT:    vslide1down.vx v8, v8, t2
 ; CHECK-NEXT:    vslide1down.vx v8, v8, t3
 ; CHECK-NEXT:    vslide1down.vx v8, v8, t4
 ; CHECK-NEXT:    vslide1down.vx v8, v8, t5
 ; CHECK-NEXT:    vslide1down.vx v8, v8, t6
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
 ; CHECK-NEXT:    li a0, 255
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a0
@@ -1277,38 +1277,38 @@ define <16 x i8> @buildvec_v16i8_loads_contigous(ptr %p) {
 define <16 x i8> @buildvec_v16i8_loads_gather(ptr %p) {
 ; CHECK-LABEL: buildvec_v16i8_loads_gather:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lbu a1, 1(a0)
-; CHECK-NEXT:    lbu a2, 22(a0)
-; CHECK-NEXT:    lbu a3, 31(a0)
-; CHECK-NEXT:    lbu a4, 44(a0)
-; CHECK-NEXT:    lbu a5, 55(a0)
-; CHECK-NEXT:    lbu a6, 623(a0)
-; CHECK-NEXT:    lbu a7, 75(a0)
-; CHECK-NEXT:    lbu t0, 93(a0)
-; CHECK-NEXT:    lbu t1, 105(a0)
-; CHECK-NEXT:    lbu t2, 161(a0)
-; CHECK-NEXT:    lbu t3, 124(a0)
-; CHECK-NEXT:    lbu t4, 163(a0)
-; CHECK-NEXT:    lbu t5, 144(a0)
-; CHECK-NEXT:    lbu t6, 154(a0)
+; CHECK-NEXT:    addi a1, a0, 82
+; CHECK-NEXT:    lbu a2, 1(a0)
+; CHECK-NEXT:    lbu a3, 22(a0)
+; CHECK-NEXT:    lbu a4, 31(a0)
+; CHECK-NEXT:    lbu a5, 44(a0)
+; CHECK-NEXT:    lbu a6, 55(a0)
+; CHECK-NEXT:    lbu a7, 623(a0)
+; CHECK-NEXT:    lbu t0, 75(a0)
+; CHECK-NEXT:    lbu t1, 93(a0)
+; CHECK-NEXT:    lbu t2, 105(a0)
+; CHECK-NEXT:    lbu t3, 161(a0)
+; CHECK-NEXT:    lbu t4, 124(a0)
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vlse8.v v8, (a0), zero
-; CHECK-NEXT:    addi a0, a0, 82
-; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    lbu t5, 163(a0)
+; CHECK-NEXT:    lbu t6, 144(a0)
+; CHECK-NEXT:    lbu a0, 154(a0)
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a2
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a3
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a4
-; CHECK-NEXT:    vlse8.v v9, (a0), zero
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a5
+; CHECK-NEXT:    vlse8.v v9, (a1), zero
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a6
-; CHECK-NEXT:    vslide1down.vx v10, v8, a7
-; CHECK-NEXT:    vslide1down.vx v8, v9, t0
-; CHECK-NEXT:    vslide1down.vx v8, v8, t1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a7
+; CHECK-NEXT:    vslide1down.vx v10, v8, t0
+; CHECK-NEXT:    vslide1down.vx v8, v9, t1
 ; CHECK-NEXT:    vslide1down.vx v8, v8, t2
 ; CHECK-NEXT:    vslide1down.vx v8, v8, t3
 ; CHECK-NEXT:    vslide1down.vx v8, v8, t4
 ; CHECK-NEXT:    vslide1down.vx v8, v8, t5
 ; CHECK-NEXT:    vslide1down.vx v8, v8, t6
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
 ; CHECK-NEXT:    li a0, 255
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a0
@@ -1375,17 +1375,17 @@ define <16 x i8> @buildvec_v16i8_undef_low_half(ptr %p) {
 ; CHECK-NEXT:    lbu a3, 105(a0)
 ; CHECK-NEXT:    lbu a4, 161(a0)
 ; CHECK-NEXT:    lbu a5, 124(a0)
-; CHECK-NEXT:    lbu a6, 163(a0)
-; CHECK-NEXT:    lbu a7, 144(a0)
-; CHECK-NEXT:    lbu a0, 154(a0)
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vlse8.v v8, (a1), zero
+; CHECK-NEXT:    lbu a1, 163(a0)
+; CHECK-NEXT:    lbu a6, 144(a0)
+; CHECK-NEXT:    lbu a0, 154(a0)
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a2
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a3
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a4
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a5
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a6
-; CHECK-NEXT:    vslide1down.vx v8, v8, a7
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   %p9 = getelementptr i8, ptr %p, i32 82
@@ -1424,18 +1424,18 @@ define <16 x i8> @buildvec_v16i8_undef_high_half(ptr %p) {
 ; CHECK-NEXT:    lbu a2, 22(a0)
 ; CHECK-NEXT:    lbu a3, 31(a0)
 ; CHECK-NEXT:    lbu a4, 44(a0)
-; CHECK-NEXT:    lbu a5, 55(a0)
-; CHECK-NEXT:    lbu a6, 623(a0)
-; CHECK-NEXT:    lbu a7, 75(a0)
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vlse8.v v8, (a0), zero
+; CHECK-NEXT:    lbu a5, 55(a0)
+; CHECK-NEXT:    lbu a6, 623(a0)
+; CHECK-NEXT:    lbu a0, 75(a0)
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a1
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a2
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a3
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a4
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a5
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a6
-; CHECK-NEXT:    vslide1down.vx v8, v8, a7
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 8
 ; CHECK-NEXT:    ret
   %p2 = getelementptr i8, ptr %p, i32 1
@@ -1470,24 +1470,24 @@ define <16 x i8> @buildvec_v16i8_undef_edges(ptr %p) {
 ; CHECK-LABEL: buildvec_v16i8_undef_edges:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi a1, a0, 31
-; CHECK-NEXT:    lbu a2, 44(a0)
-; CHECK-NEXT:    lbu a3, 55(a0)
-; CHECK-NEXT:    lbu a4, 623(a0)
-; CHECK-NEXT:    lbu a5, 75(a0)
-; CHECK-NEXT:    lbu a6, 93(a0)
-; CHECK-NEXT:    lbu a7, 105(a0)
-; CHECK-NEXT:    lbu t0, 161(a0)
+; CHECK-NEXT:    addi a2, a0, 82
+; CHECK-NEXT:    lbu a3, 44(a0)
+; CHECK-NEXT:    lbu a4, 55(a0)
+; CHECK-NEXT:    lbu a5, 623(a0)
+; CHECK-NEXT:    lbu a6, 75(a0)
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vlse8.v v8, (a1), zero
-; CHECK-NEXT:    addi a0, a0, 82
-; CHECK-NEXT:    vslide1down.vx v8, v8, a2
-; CHECK-NEXT:    vlse8.v v9, (a0), zero
+; CHECK-NEXT:    lbu a1, 93(a0)
+; CHECK-NEXT:    lbu a7, 105(a0)
+; CHECK-NEXT:    lbu a0, 161(a0)
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a3
+; CHECK-NEXT:    vlse8.v v9, (a2), zero
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a4
-; CHECK-NEXT:    vslide1down.vx v10, v8, a5
-; CHECK-NEXT:    vslide1down.vx v8, v9, a6
+; CHECK-NEXT:    vslide1down.vx v8, v8, a5
+; CHECK-NEXT:    vslide1down.vx v10, v8, a6
+; CHECK-NEXT:    vslide1down.vx v8, v9, a1
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a7
-; CHECK-NEXT:    vslide1down.vx v8, v8, t0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 4
 ; CHECK-NEXT:    li a0, 255
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
@@ -1530,30 +1530,30 @@ define <16 x i8> @buildvec_v16i8_undef_edges(ptr %p) {
 define <16 x i8> @buildvec_v16i8_loads_undef_scattered(ptr %p) {
 ; CHECK-LABEL: buildvec_v16i8_loads_undef_scattered:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lbu a1, 1(a0)
-; CHECK-NEXT:    lbu a2, 44(a0)
-; CHECK-NEXT:    lbu a3, 55(a0)
-; CHECK-NEXT:    lbu a4, 75(a0)
-; CHECK-NEXT:    lbu a5, 93(a0)
-; CHECK-NEXT:    lbu a6, 124(a0)
-; CHECK-NEXT:    lbu a7, 144(a0)
-; CHECK-NEXT:    lbu t0, 154(a0)
+; CHECK-NEXT:    addi a1, a0, 82
+; CHECK-NEXT:    lbu a2, 1(a0)
+; CHECK-NEXT:    lbu a3, 44(a0)
+; CHECK-NEXT:    lbu a4, 55(a0)
+; CHECK-NEXT:    lbu a5, 75(a0)
+; CHECK-NEXT:    lbu a6, 93(a0)
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vlse8.v v8, (a0), zero
-; CHECK-NEXT:    addi a0, a0, 82
-; CHECK-NEXT:    vslide1down.vx v8, v8, a1
-; CHECK-NEXT:    vslidedown.vi v8, v8, 2
+; CHECK-NEXT:    lbu a7, 124(a0)
+; CHECK-NEXT:    lbu t0, 144(a0)
+; CHECK-NEXT:    lbu a0, 154(a0)
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a2
-; CHECK-NEXT:    vlse8.v v9, (a0), zero
+; CHECK-NEXT:    vslidedown.vi v8, v8, 2
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a3
+; CHECK-NEXT:    vlse8.v v9, (a1), zero
+; CHECK-NEXT:    vslide1down.vx v8, v8, a4
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 1
-; CHECK-NEXT:    vslide1down.vx v10, v8, a4
-; CHECK-NEXT:    vslide1down.vx v8, v9, a5
+; CHECK-NEXT:    vslide1down.vx v10, v8, a5
+; CHECK-NEXT:    vslide1down.vx v8, v9, a6
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 2
-; CHECK-NEXT:    vslide1down.vx v8, v8, a6
-; CHECK-NEXT:    vslidedown.vi v8, v8, 1
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a7
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1
 ; CHECK-NEXT:    vslide1down.vx v8, v8, t0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
 ; CHECK-NEXT:    li a0, 255
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access-zve32x.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access-zve32x.ll
index 8acc70faaa1f..eb95d86e3404 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access-zve32x.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access-zve32x.ll
@@ -7,25 +7,23 @@
 define <4 x i1> @load_large_vector(ptr %p) {
 ; ZVE32X-LABEL: load_large_vector:
 ; ZVE32X:       # %bb.0:
-; ZVE32X-NEXT:    ld a1, 80(a0)
-; ZVE32X-NEXT:    ld a2, 72(a0)
-; ZVE32X-NEXT:    ld a3, 56(a0)
-; ZVE32X-NEXT:    ld a4, 32(a0)
-; ZVE32X-NEXT:    ld a5, 24(a0)
-; ZVE32X-NEXT:    ld a6, 48(a0)
-; ZVE32X-NEXT:    ld a7, 8(a0)
-; ZVE32X-NEXT:    ld a0, 0(a0)
-; ZVE32X-NEXT:    xor a4, a5, a4
-; ZVE32X-NEXT:    snez a4, a4
+; ZVE32X-NEXT:    ld a1, 56(a0)
+; ZVE32X-NEXT:    ld a2, 32(a0)
+; ZVE32X-NEXT:    ld a3, 24(a0)
+; ZVE32X-NEXT:    ld a4, 48(a0)
+; ZVE32X-NEXT:    ld a5, 8(a0)
+; ZVE32X-NEXT:    ld a6, 0(a0)
+; ZVE32X-NEXT:    xor a2, a3, a2
+; ZVE32X-NEXT:    snez a2, a2
 ; ZVE32X-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; ZVE32X-NEXT:    vmv.s.x v8, a4
+; ZVE32X-NEXT:    vmv.s.x v8, a2
 ; ZVE32X-NEXT:    vand.vi v8, v8, 1
 ; ZVE32X-NEXT:    vmsne.vi v0, v8, 0
 ; ZVE32X-NEXT:    vmv.s.x v8, zero
 ; ZVE32X-NEXT:    vmerge.vim v9, v8, 1, v0
-; ZVE32X-NEXT:    xor a0, a0, a7
-; ZVE32X-NEXT:    snez a0, a0
-; ZVE32X-NEXT:    vmv.s.x v10, a0
+; ZVE32X-NEXT:    xor a2, a6, a5
+; ZVE32X-NEXT:    snez a2, a2
+; ZVE32X-NEXT:    vmv.s.x v10, a2
 ; ZVE32X-NEXT:    vand.vi v10, v10, 1
 ; ZVE32X-NEXT:    vmsne.vi v0, v10, 0
 ; ZVE32X-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
@@ -35,21 +33,23 @@ define <4 x i1> @load_large_vector(ptr %p) {
 ; ZVE32X-NEXT:    vslideup.vi v11, v9, 1
 ; ZVE32X-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; ZVE32X-NEXT:    vmsne.vi v0, v11, 0
+; ZVE32X-NEXT:    ld a2, 80(a0)
 ; ZVE32X-NEXT:    vmerge.vim v9, v10, 1, v0
-; ZVE32X-NEXT:    xor a0, a6, a3
-; ZVE32X-NEXT:    snez a0, a0
-; ZVE32X-NEXT:    vmv.s.x v11, a0
+; ZVE32X-NEXT:    xor a1, a4, a1
+; ZVE32X-NEXT:    snez a1, a1
+; ZVE32X-NEXT:    vmv.s.x v11, a1
 ; ZVE32X-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; ZVE32X-NEXT:    vand.vi v11, v11, 1
 ; ZVE32X-NEXT:    vmsne.vi v0, v11, 0
+; ZVE32X-NEXT:    ld a0, 72(a0)
 ; ZVE32X-NEXT:    vmerge.vim v11, v8, 1, v0
 ; ZVE32X-NEXT:    vsetivli zero, 3, e8, mf4, tu, ma
 ; ZVE32X-NEXT:    vslideup.vi v9, v11, 2
 ; ZVE32X-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; ZVE32X-NEXT:    vmsne.vi v0, v9, 0
 ; ZVE32X-NEXT:    vmerge.vim v9, v10, 1, v0
-; ZVE32X-NEXT:    xor a1, a2, a1
-; ZVE32X-NEXT:    snez a0, a1
+; ZVE32X-NEXT:    xor a0, a0, a2
+; ZVE32X-NEXT:    snez a0, a0
 ; ZVE32X-NEXT:    vmv.s.x v10, a0
 ; ZVE32X-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; ZVE32X-NEXT:    vand.vi v10, v10, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
index 35baa6808db6..e2075e074179 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
@@ -812,14 +812,14 @@ define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a0
 ; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    fld fa4, 32(sp)
+; RV32-NEXT:    fld fa5, 32(sp)
+; RV32-NEXT:    vfmv.f.s fa4, v8
 ; RV32-NEXT:    fld fa3, 40(sp)
-; RV32-NEXT:    fcvt.w.d a0, fa5
+; RV32-NEXT:    fcvt.w.d a0, fa4
+; RV32-NEXT:    fcvt.w.d a1, fa5
 ; RV32-NEXT:    fld fa5, 48(sp)
-; RV32-NEXT:    fcvt.w.d a1, fa4
 ; RV32-NEXT:    fcvt.w.d a2, fa3
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32-NEXT:    vslide1down.vx v8, v10, a0
 ; RV32-NEXT:    fcvt.w.d a0, fa5
 ; RV32-NEXT:    fld fa5, 56(sp)
@@ -865,14 +865,14 @@ define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
 ; RV64-i32-NEXT:    vslide1down.vx v10, v10, a0
 ; RV64-i32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
 ; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-i32-NEXT:    fld fa4, 32(sp)
+; RV64-i32-NEXT:    fld fa5, 32(sp)
+; RV64-i32-NEXT:    vfmv.f.s fa4, v8
 ; RV64-i32-NEXT:    fld fa3, 40(sp)
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
+; RV64-i32-NEXT:    fcvt.l.d a0, fa4
+; RV64-i32-NEXT:    fcvt.l.d a1, fa5
 ; RV64-i32-NEXT:    fld fa5, 48(sp)
-; RV64-i32-NEXT:    fcvt.l.d a1, fa4
 ; RV64-i32-NEXT:    fcvt.l.d a2, fa3
+; RV64-i32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV64-i32-NEXT:    vslide1down.vx v8, v10, a0
 ; RV64-i32-NEXT:    fcvt.l.d a0, fa5
 ; RV64-i32-NEXT:    fld fa5, 56(sp)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index f42f32e24658..08cad29ab1b8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -519,17 +519,17 @@ define <4 x i8> @mgather_truemask_v4i8(<4 x ptr> %ptrs, <4 x i8> %passthru) {
 ; RV64ZVE32F-LABEL: mgather_truemask_v4i8:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    ld a1, 8(a0)
-; RV64ZVE32F-NEXT:    ld a2, 16(a0)
+; RV64ZVE32F-NEXT:    ld a2, 0(a0)
 ; RV64ZVE32F-NEXT:    ld a3, 24(a0)
-; RV64ZVE32F-NEXT:    ld a0, 0(a0)
+; RV64ZVE32F-NEXT:    ld a0, 16(a0)
 ; RV64ZVE32F-NEXT:    lbu a1, 0(a1)
-; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    lbu a3, 0(a3)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vlse8.v v8, (a0), zero
+; RV64ZVE32F-NEXT:    vlse8.v v8, (a2), zero
+; RV64ZVE32F-NEXT:    lbu a0, 0(a0)
+; RV64ZVE32F-NEXT:    lbu a2, 0(a3)
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
 ; RV64ZVE32F-NEXT:    ret
   %v = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 1), <4 x i8> %passthru)
   ret <4 x i8> %v
@@ -1208,17 +1208,17 @@ define <4 x i16> @mgather_truemask_v4i16(<4 x ptr> %ptrs, <4 x i16> %passthru) {
 ; RV64ZVE32F-LABEL: mgather_truemask_v4i16:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    ld a1, 8(a0)
-; RV64ZVE32F-NEXT:    ld a2, 16(a0)
+; RV64ZVE32F-NEXT:    ld a2, 0(a0)
 ; RV64ZVE32F-NEXT:    ld a3, 24(a0)
-; RV64ZVE32F-NEXT:    ld a0, 0(a0)
+; RV64ZVE32F-NEXT:    ld a0, 16(a0)
 ; RV64ZVE32F-NEXT:    lh a1, 0(a1)
-; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    lh a3, 0(a3)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
+; RV64ZVE32F-NEXT:    vlse16.v v8, (a2), zero
+; RV64ZVE32F-NEXT:    lh a0, 0(a0)
+; RV64ZVE32F-NEXT:    lh a2, 0(a3)
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
 ; RV64ZVE32F-NEXT:    ret
   %v = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> %ptrs, i32 2, <4 x i1> splat (i1 1), <4 x i16> %passthru)
   ret <4 x i16> %v
@@ -2257,17 +2257,17 @@ define <4 x i32> @mgather_truemask_v4i32(<4 x ptr> %ptrs, <4 x i32> %passthru) {
 ; RV64ZVE32F-LABEL: mgather_truemask_v4i32:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    ld a1, 8(a0)
-; RV64ZVE32F-NEXT:    ld a2, 16(a0)
+; RV64ZVE32F-NEXT:    ld a2, 0(a0)
 ; RV64ZVE32F-NEXT:    ld a3, 24(a0)
-; RV64ZVE32F-NEXT:    ld a0, 0(a0)
+; RV64ZVE32F-NEXT:    ld a0, 16(a0)
 ; RV64ZVE32F-NEXT:    lw a1, 0(a1)
-; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    lw a3, 0(a3)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vlse32.v v8, (a0), zero
+; RV64ZVE32F-NEXT:    vlse32.v v8, (a2), zero
+; RV64ZVE32F-NEXT:    lw a0, 0(a0)
+; RV64ZVE32F-NEXT:    lw a2, 0(a3)
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
 ; RV64ZVE32F-NEXT:    ret
   %v = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 4, <4 x i1> splat (i1 1), <4 x i32> %passthru)
   ret <4 x i32> %v
@@ -6589,16 +6589,16 @@ define <8 x i64> @mgather_baseidx_v8i64(ptr %base, <8 x i64> %idxs, <8 x i1> %m,
 ; RV32ZVE32F-NEXT:    lw a4, 56(a2)
 ; RV32ZVE32F-NEXT:    lw a5, 48(a2)
 ; RV32ZVE32F-NEXT:    lw a6, 40(a2)
-; RV32ZVE32F-NEXT:    lw a7, 32(a2)
-; RV32ZVE32F-NEXT:    lw t0, 24(a2)
-; RV32ZVE32F-NEXT:    lw t1, 16(a2)
-; RV32ZVE32F-NEXT:    lw t2, 8(a2)
+; RV32ZVE32F-NEXT:    lw a7, 8(a2)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vlse32.v v8, (a2), zero
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t2
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t1
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; RV32ZVE32F-NEXT:    lw t0, 16(a2)
+; RV32ZVE32F-NEXT:    lw t1, 24(a2)
+; RV32ZVE32F-NEXT:    lw a2, 32(a2)
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t1
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
@@ -7017,14 +7017,14 @@ define <4 x half> @mgather_truemask_v4f16(<4 x ptr> %ptrs, <4 x half> %passthru)
 ; RV64ZVE32F-LABEL: mgather_truemask_v4f16:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    ld a1, 8(a0)
-; RV64ZVE32F-NEXT:    ld a2, 16(a0)
+; RV64ZVE32F-NEXT:    ld a2, 0(a0)
 ; RV64ZVE32F-NEXT:    ld a3, 24(a0)
-; RV64ZVE32F-NEXT:    ld a0, 0(a0)
+; RV64ZVE32F-NEXT:    ld a0, 16(a0)
 ; RV64ZVE32F-NEXT:    flh fa5, 0(a1)
-; RV64ZVE32F-NEXT:    flh fa4, 0(a2)
-; RV64ZVE32F-NEXT:    flh fa3, 0(a3)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
+; RV64ZVE32F-NEXT:    vlse16.v v8, (a2), zero
+; RV64ZVE32F-NEXT:    flh fa4, 0(a0)
+; RV64ZVE32F-NEXT:    flh fa3, 0(a3)
 ; RV64ZVE32F-NEXT:    vfslide1down.vf v8, v8, fa5
 ; RV64ZVE32F-NEXT:    vfslide1down.vf v8, v8, fa4
 ; RV64ZVE32F-NEXT:    vfslide1down.vf v8, v8, fa3
@@ -7940,14 +7940,14 @@ define <4 x float> @mgather_truemask_v4f32(<4 x ptr> %ptrs, <4 x float> %passthr
 ; RV64ZVE32F-LABEL: mgather_truemask_v4f32:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    ld a1, 8(a0)
-; RV64ZVE32F-NEXT:    ld a2, 16(a0)
+; RV64ZVE32F-NEXT:    ld a2, 0(a0)
 ; RV64ZVE32F-NEXT:    ld a3, 24(a0)
-; RV64ZVE32F-NEXT:    ld a0, 0(a0)
+; RV64ZVE32F-NEXT:    ld a0, 16(a0)
 ; RV64ZVE32F-NEXT:    flw fa5, 0(a1)
-; RV64ZVE32F-NEXT:    flw fa4, 0(a2)
-; RV64ZVE32F-NEXT:    flw fa3, 0(a3)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vlse32.v v8, (a0), zero
+; RV64ZVE32F-NEXT:    vlse32.v v8, (a2), zero
+; RV64ZVE32F-NEXT:    flw fa4, 0(a0)
+; RV64ZVE32F-NEXT:    flw fa3, 0(a3)
 ; RV64ZVE32F-NEXT:    vfslide1down.vf v8, v8, fa5
 ; RV64ZVE32F-NEXT:    vfslide1down.vf v8, v8, fa4
 ; RV64ZVE32F-NEXT:    vfslide1down.vf v8, v8, fa3
@@ -11632,16 +11632,16 @@ define <8 x double> @mgather_baseidx_v8f64(ptr %base, <8 x i64> %idxs, <8 x i1>
 ; RV32ZVE32F-NEXT:    lw a3, 56(a2)
 ; RV32ZVE32F-NEXT:    lw a4, 48(a2)
 ; RV32ZVE32F-NEXT:    lw a5, 40(a2)
-; RV32ZVE32F-NEXT:    lw a6, 32(a2)
-; RV32ZVE32F-NEXT:    lw a7, 24(a2)
-; RV32ZVE32F-NEXT:    lw t0, 16(a2)
-; RV32ZVE32F-NEXT:    lw t1, 8(a2)
+; RV32ZVE32F-NEXT:    lw a6, 8(a2)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vlse32.v v8, (a2), zero
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t1
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV32ZVE32F-NEXT:    lw a7, 16(a2)
+; RV32ZVE32F-NEXT:    lw t0, 24(a2)
+; RV32ZVE32F-NEXT:    lw a2, 32(a2)
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
@@ -12881,22 +12881,22 @@ define <8 x i16> @mgather_strided_2xSEW(ptr %base) {
 ;
 ; RV64ZVE32F-LABEL: mgather_strided_2xSEW:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    lh a1, 2(a0)
-; RV64ZVE32F-NEXT:    lh a2, 8(a0)
-; RV64ZVE32F-NEXT:    lh a3, 10(a0)
-; RV64ZVE32F-NEXT:    lh a4, 18(a0)
-; RV64ZVE32F-NEXT:    lh a5, 24(a0)
-; RV64ZVE32F-NEXT:    lh a6, 26(a0)
+; RV64ZVE32F-NEXT:    addi a1, a0, 16
+; RV64ZVE32F-NEXT:    lh a2, 2(a0)
+; RV64ZVE32F-NEXT:    lh a3, 8(a0)
+; RV64ZVE32F-NEXT:    lh a4, 10(a0)
+; RV64ZVE32F-NEXT:    lh a5, 18(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
-; RV64ZVE32F-NEXT:    addi a0, a0, 16
-; RV64ZVE32F-NEXT:    vlse16.v v9, (a0), zero
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; RV64ZVE32F-NEXT:    lh a6, 24(a0)
+; RV64ZVE32F-NEXT:    lh a0, 26(a0)
+; RV64ZVE32F-NEXT:    vlse16.v v9, (a1), zero
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a5
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
@@ -12925,23 +12925,23 @@ define <8 x i16> @mgather_strided_2xSEW_with_offset(ptr %base) {
 ;
 ; RV64ZVE32F-LABEL: mgather_strided_2xSEW_with_offset:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    addi a1, a0, 4
-; RV64ZVE32F-NEXT:    lh a2, 6(a0)
-; RV64ZVE32F-NEXT:    lh a3, 12(a0)
-; RV64ZVE32F-NEXT:    lh a4, 14(a0)
-; RV64ZVE32F-NEXT:    lh a5, 22(a0)
-; RV64ZVE32F-NEXT:    lh a6, 28(a0)
-; RV64ZVE32F-NEXT:    lh a7, 30(a0)
+; RV64ZVE32F-NEXT:    addi a1, a0, 20
+; RV64ZVE32F-NEXT:    addi a2, a0, 4
+; RV64ZVE32F-NEXT:    lh a3, 6(a0)
+; RV64ZVE32F-NEXT:    lh a4, 12(a0)
+; RV64ZVE32F-NEXT:    lh a5, 14(a0)
+; RV64ZVE32F-NEXT:    lh a6, 22(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
-; RV64ZVE32F-NEXT:    vlse16.v v8, (a1), zero
-; RV64ZVE32F-NEXT:    addi a0, a0, 20
-; RV64ZVE32F-NEXT:    vlse16.v v9, (a0), zero
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vlse16.v v8, (a2), zero
+; RV64ZVE32F-NEXT:    lh a2, 28(a0)
+; RV64ZVE32F-NEXT:    lh a0, 30(a0)
+; RV64ZVE32F-NEXT:    vlse16.v v9, (a1), zero
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
@@ -12970,23 +12970,23 @@ define <8 x i16> @mgather_reverse_unit_strided_2xSEW(ptr %base) {
 ;
 ; RV64ZVE32F-LABEL: mgather_reverse_unit_strided_2xSEW:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    addi a1, a0, 28
-; RV64ZVE32F-NEXT:    lh a2, 30(a0)
-; RV64ZVE32F-NEXT:    lh a3, 24(a0)
-; RV64ZVE32F-NEXT:    lh a4, 26(a0)
-; RV64ZVE32F-NEXT:    lh a5, 22(a0)
-; RV64ZVE32F-NEXT:    lh a6, 16(a0)
-; RV64ZVE32F-NEXT:    lh a7, 18(a0)
+; RV64ZVE32F-NEXT:    addi a1, a0, 20
+; RV64ZVE32F-NEXT:    addi a2, a0, 28
+; RV64ZVE32F-NEXT:    lh a3, 30(a0)
+; RV64ZVE32F-NEXT:    lh a4, 24(a0)
+; RV64ZVE32F-NEXT:    lh a5, 26(a0)
+; RV64ZVE32F-NEXT:    lh a6, 22(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
-; RV64ZVE32F-NEXT:    vlse16.v v8, (a1), zero
-; RV64ZVE32F-NEXT:    addi a0, a0, 20
-; RV64ZVE32F-NEXT:    vlse16.v v9, (a0), zero
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vlse16.v v8, (a2), zero
+; RV64ZVE32F-NEXT:    lh a2, 16(a0)
+; RV64ZVE32F-NEXT:    lh a0, 18(a0)
+; RV64ZVE32F-NEXT:    vlse16.v v9, (a1), zero
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
@@ -13015,23 +13015,23 @@ define <8 x i16> @mgather_reverse_strided_2xSEW(ptr %base) {
 ;
 ; RV64ZVE32F-LABEL: mgather_reverse_strided_2xSEW:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    addi a1, a0, 28
-; RV64ZVE32F-NEXT:    lh a2, 30(a0)
-; RV64ZVE32F-NEXT:    lh a3, 20(a0)
-; RV64ZVE32F-NEXT:    lh a4, 22(a0)
-; RV64ZVE32F-NEXT:    lh a5, 14(a0)
-; RV64ZVE32F-NEXT:    lh a6, 4(a0)
-; RV64ZVE32F-NEXT:    lh a7, 6(a0)
+; RV64ZVE32F-NEXT:    addi a1, a0, 12
+; RV64ZVE32F-NEXT:    addi a2, a0, 28
+; RV64ZVE32F-NEXT:    lh a3, 30(a0)
+; RV64ZVE32F-NEXT:    lh a4, 20(a0)
+; RV64ZVE32F-NEXT:    lh a5, 22(a0)
+; RV64ZVE32F-NEXT:    lh a6, 14(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
-; RV64ZVE32F-NEXT:    vlse16.v v8, (a1), zero
-; RV64ZVE32F-NEXT:    addi a0, a0, 12
-; RV64ZVE32F-NEXT:    vlse16.v v9, (a0), zero
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vlse16.v v8, (a2), zero
+; RV64ZVE32F-NEXT:    lh a2, 4(a0)
+; RV64ZVE32F-NEXT:    lh a0, 6(a0)
+; RV64ZVE32F-NEXT:    vlse16.v v9, (a1), zero
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
@@ -13059,22 +13059,22 @@ define <8 x i16> @mgather_gather_2xSEW(ptr %base) {
 ;
 ; RV64ZVE32F-LABEL: mgather_gather_2xSEW:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    lh a1, 2(a0)
-; RV64ZVE32F-NEXT:    lh a2, 16(a0)
-; RV64ZVE32F-NEXT:    lh a3, 18(a0)
-; RV64ZVE32F-NEXT:    lh a4, 10(a0)
-; RV64ZVE32F-NEXT:    lh a5, 4(a0)
-; RV64ZVE32F-NEXT:    lh a6, 6(a0)
+; RV64ZVE32F-NEXT:    addi a1, a0, 8
+; RV64ZVE32F-NEXT:    lh a2, 2(a0)
+; RV64ZVE32F-NEXT:    lh a3, 16(a0)
+; RV64ZVE32F-NEXT:    lh a4, 18(a0)
+; RV64ZVE32F-NEXT:    lh a5, 10(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
-; RV64ZVE32F-NEXT:    addi a0, a0, 8
-; RV64ZVE32F-NEXT:    vlse16.v v9, (a0), zero
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; RV64ZVE32F-NEXT:    lh a6, 4(a0)
+; RV64ZVE32F-NEXT:    lh a0, 6(a0)
+; RV64ZVE32F-NEXT:    vlse16.v v9, (a1), zero
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a5
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
@@ -13105,22 +13105,22 @@ define <8 x i16> @mgather_gather_2xSEW_unaligned(ptr %base) {
 ;
 ; RV64ZVE32F-LABEL: mgather_gather_2xSEW_unaligned:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    lh a1, 2(a0)
-; RV64ZVE32F-NEXT:    lh a2, 18(a0)
-; RV64ZVE32F-NEXT:    lh a3, 20(a0)
-; RV64ZVE32F-NEXT:    lh a4, 10(a0)
-; RV64ZVE32F-NEXT:    lh a5, 4(a0)
-; RV64ZVE32F-NEXT:    lh a6, 6(a0)
+; RV64ZVE32F-NEXT:    addi a1, a0, 8
+; RV64ZVE32F-NEXT:    lh a2, 2(a0)
+; RV64ZVE32F-NEXT:    lh a3, 18(a0)
+; RV64ZVE32F-NEXT:    lh a4, 20(a0)
+; RV64ZVE32F-NEXT:    lh a5, 10(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
-; RV64ZVE32F-NEXT:    addi a0, a0, 8
-; RV64ZVE32F-NEXT:    vlse16.v v9, (a0), zero
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; RV64ZVE32F-NEXT:    lh a6, 4(a0)
+; RV64ZVE32F-NEXT:    lh a0, 6(a0)
+; RV64ZVE32F-NEXT:    vlse16.v v9, (a1), zero
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a5
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
@@ -13152,22 +13152,22 @@ define <8 x i16> @mgather_gather_2xSEW_unaligned2(ptr %base) {
 ;
 ; RV64ZVE32F-LABEL: mgather_gather_2xSEW_unaligned2:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    addi a1, a0, 2
-; RV64ZVE32F-NEXT:    lh a2, 4(a0)
-; RV64ZVE32F-NEXT:    lh a3, 18(a0)
-; RV64ZVE32F-NEXT:    lh a4, 20(a0)
-; RV64ZVE32F-NEXT:    lh a5, 10(a0)
-; RV64ZVE32F-NEXT:    lh a6, 6(a0)
+; RV64ZVE32F-NEXT:    addi a1, a0, 8
+; RV64ZVE32F-NEXT:    addi a2, a0, 2
+; RV64ZVE32F-NEXT:    lh a3, 4(a0)
+; RV64ZVE32F-NEXT:    lh a4, 18(a0)
+; RV64ZVE32F-NEXT:    lh a5, 20(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
-; RV64ZVE32F-NEXT:    vlse16.v v8, (a1), zero
-; RV64ZVE32F-NEXT:    addi a0, a0, 8
-; RV64ZVE32F-NEXT:    vlse16.v v9, (a0), zero
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vlse16.v v8, (a2), zero
+; RV64ZVE32F-NEXT:    lh a2, 10(a0)
+; RV64ZVE32F-NEXT:    lh a0, 6(a0)
+; RV64ZVE32F-NEXT:    vlse16.v v9, (a1), zero
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
@@ -13202,22 +13202,22 @@ define <8 x i16> @mgather_gather_4xSEW(ptr %base) {
 ;
 ; RV64ZVE32F-LABEL: mgather_gather_4xSEW:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    lh a1, 2(a0)
-; RV64ZVE32F-NEXT:    lh a2, 4(a0)
-; RV64ZVE32F-NEXT:    lh a3, 6(a0)
-; RV64ZVE32F-NEXT:    lh a4, 18(a0)
-; RV64ZVE32F-NEXT:    lh a5, 20(a0)
-; RV64ZVE32F-NEXT:    lh a6, 22(a0)
+; RV64ZVE32F-NEXT:    addi a1, a0, 16
+; RV64ZVE32F-NEXT:    lh a2, 2(a0)
+; RV64ZVE32F-NEXT:    lh a3, 4(a0)
+; RV64ZVE32F-NEXT:    lh a4, 6(a0)
+; RV64ZVE32F-NEXT:    lh a5, 18(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
-; RV64ZVE32F-NEXT:    addi a0, a0, 16
-; RV64ZVE32F-NEXT:    vlse16.v v9, (a0), zero
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; RV64ZVE32F-NEXT:    lh a6, 20(a0)
+; RV64ZVE32F-NEXT:    lh a0, 22(a0)
+; RV64ZVE32F-NEXT:    vlse16.v v9, (a1), zero
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a5
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
@@ -13249,22 +13249,22 @@ define <8 x i16> @mgather_gather_4xSEW_partial_align(ptr %base) {
 ;
 ; RV64ZVE32F-LABEL: mgather_gather_4xSEW_partial_align:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    lh a1, 2(a0)
-; RV64ZVE32F-NEXT:    lh a2, 4(a0)
-; RV64ZVE32F-NEXT:    lh a3, 6(a0)
-; RV64ZVE32F-NEXT:    lh a4, 18(a0)
-; RV64ZVE32F-NEXT:    lh a5, 20(a0)
-; RV64ZVE32F-NEXT:    lh a6, 22(a0)
+; RV64ZVE32F-NEXT:    addi a1, a0, 16
+; RV64ZVE32F-NEXT:    lh a2, 2(a0)
+; RV64ZVE32F-NEXT:    lh a3, 4(a0)
+; RV64ZVE32F-NEXT:    lh a4, 6(a0)
+; RV64ZVE32F-NEXT:    lh a5, 18(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
-; RV64ZVE32F-NEXT:    addi a0, a0, 16
-; RV64ZVE32F-NEXT:    vlse16.v v9, (a0), zero
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; RV64ZVE32F-NEXT:    lh a6, 20(a0)
+; RV64ZVE32F-NEXT:    lh a0, 22(a0)
+; RV64ZVE32F-NEXT:    vlse16.v v9, (a1), zero
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a5
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
@@ -13305,22 +13305,22 @@ define <8 x i16> @mgather_shuffle_rotate(ptr %base) {
 ;
 ; RV64ZVE32F-LABEL: mgather_shuffle_rotate:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    lh a1, 10(a0)
-; RV64ZVE32F-NEXT:    lh a2, 12(a0)
-; RV64ZVE32F-NEXT:    lh a3, 14(a0)
-; RV64ZVE32F-NEXT:    lh a4, 2(a0)
-; RV64ZVE32F-NEXT:    lh a5, 4(a0)
-; RV64ZVE32F-NEXT:    lh a6, 6(a0)
+; RV64ZVE32F-NEXT:    addi a1, a0, 8
+; RV64ZVE32F-NEXT:    lh a2, 10(a0)
+; RV64ZVE32F-NEXT:    lh a3, 12(a0)
+; RV64ZVE32F-NEXT:    lh a4, 14(a0)
+; RV64ZVE32F-NEXT:    lh a5, 2(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
-; RV64ZVE32F-NEXT:    addi a0, a0, 8
-; RV64ZVE32F-NEXT:    vlse16.v v9, (a0), zero
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
+; RV64ZVE32F-NEXT:    lh a6, 4(a0)
+; RV64ZVE32F-NEXT:    lh a0, 6(a0)
+; RV64ZVE32F-NEXT:    vlse16.v v9, (a1), zero
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
 ; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
@@ -13352,22 +13352,22 @@ define <8 x i16> @mgather_shuffle_vrgather(ptr %base) {
 ;
 ; RV64ZVE32F-LABEL: mgather_shuffle_vrgather:
 ; RV64ZVE32F:       # %bb.0:
-; RV64ZVE32F-NEXT:    lh a1, 4(a0)
-; RV64ZVE32F-NEXT:    lh a2, 6(a0)
-; RV64ZVE32F-NEXT:    lh a3, 2(a0)
-; RV64ZVE32F-NEXT:    lh a4, 10(a0)
-; RV64ZVE32F-NEXT:    lh a5, 12(a0)
-; RV64ZVE32F-NEXT:    lh a6, 14(a0)
+; RV64ZVE32F-NEXT:    addi a1, a0, 8
+; RV64ZVE32F-NEXT:    lh a2, 4(a0)
+; RV64ZVE32F-NEXT:    lh a3, 6(a0)
+; RV64ZVE32F-NEXT:    lh a4, 2(a0)
+; RV64ZVE32F-NEXT:    lh a5, 10(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vlse16.v v8, (a0), zero
-; RV64ZVE32F-NEXT:    addi a0, a0, 8
-; RV64ZVE32F-NEXT:    vlse16.v v9, (a0), zero
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; RV64ZVE32F-NEXT:    lh a6, 12(a0)
+; RV64ZVE32F-NEXT:    lh a0, 14(a0)
+; RV64ZVE32F-NEXT:    vlse16.v v9, (a1), zero
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a5
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
@@ -13853,12 +13853,12 @@ define <4 x i32> @masked_gather_widen_sew_negative_stride(ptr %base) {
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    addi a1, a0, 136
 ; RV64ZVE32F-NEXT:    lw a2, 140(a0)
-; RV64ZVE32F-NEXT:    lw a3, 0(a0)
-; RV64ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vlse32.v v8, (a1), zero
+; RV64ZVE32F-NEXT:    lw a1, 0(a0)
+; RV64ZVE32F-NEXT:    lw a0, 4(a0)
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
 ; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr i32, ptr %base, <4 x i64> <i64 34, i64 35, i64 0, i64 1>
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
index aa815e18ac10..42e52436a7da 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
@@ -5598,17 +5598,16 @@ define void @mscatter_baseidx_v8i64(<8 x i64> %val, ptr %base, <8 x i64> %idxs,
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_v8i64:
 ; RV32ZVE32F:       # %bb.0:
-; RV32ZVE32F-NEXT:    addi sp, sp, -48
-; RV32ZVE32F-NEXT:    .cfi_def_cfa_offset 48
-; RV32ZVE32F-NEXT:    sw s0, 44(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s1, 40(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s2, 36(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s3, 32(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s4, 28(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s6, 20(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s7, 16(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s8, 12(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    addi sp, sp, -32
+; RV32ZVE32F-NEXT:    .cfi_def_cfa_offset 32
+; RV32ZVE32F-NEXT:    sw s0, 28(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s1, 24(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s2, 20(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s3, 16(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s4, 12(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s5, 8(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s6, 4(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s7, 0(sp) # 4-byte Folded Spill
 ; RV32ZVE32F-NEXT:    .cfi_offset s0, -4
 ; RV32ZVE32F-NEXT:    .cfi_offset s1, -8
 ; RV32ZVE32F-NEXT:    .cfi_offset s2, -12
@@ -5617,7 +5616,6 @@ define void @mscatter_baseidx_v8i64(<8 x i64> %val, ptr %base, <8 x i64> %idxs,
 ; RV32ZVE32F-NEXT:    .cfi_offset s5, -24
 ; RV32ZVE32F-NEXT:    .cfi_offset s6, -28
 ; RV32ZVE32F-NEXT:    .cfi_offset s7, -32
-; RV32ZVE32F-NEXT:    .cfi_offset s8, -36
 ; RV32ZVE32F-NEXT:    lw a3, 60(a0)
 ; RV32ZVE32F-NEXT:    lw a4, 56(a0)
 ; RV32ZVE32F-NEXT:    lw a5, 52(a0)
@@ -5635,16 +5633,16 @@ define void @mscatter_baseidx_v8i64(<8 x i64> %val, ptr %base, <8 x i64> %idxs,
 ; RV32ZVE32F-NEXT:    lw s2, 56(a2)
 ; RV32ZVE32F-NEXT:    lw s3, 48(a2)
 ; RV32ZVE32F-NEXT:    lw s4, 40(a2)
-; RV32ZVE32F-NEXT:    lw s5, 32(a2)
-; RV32ZVE32F-NEXT:    lw s6, 24(a2)
-; RV32ZVE32F-NEXT:    lw s7, 16(a2)
-; RV32ZVE32F-NEXT:    lw s8, 8(a2)
+; RV32ZVE32F-NEXT:    lw s5, 8(a2)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vlse32.v v8, (a2), zero
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s8
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s7
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s6
+; RV32ZVE32F-NEXT:    lw s6, 16(a2)
+; RV32ZVE32F-NEXT:    lw s7, 24(a2)
+; RV32ZVE32F-NEXT:    lw a2, 32(a2)
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s5
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s6
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s7
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s4
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s3
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s2
@@ -5682,16 +5680,15 @@ define void @mscatter_baseidx_v8i64(<8 x i64> %val, ptr %base, <8 x i64> %idxs,
 ; RV32ZVE32F-NEXT:    sw a4, 0(a0)
 ; RV32ZVE32F-NEXT:    sw a3, 4(a0)
 ; RV32ZVE32F-NEXT:  .LBB51_9: # %else14
-; RV32ZVE32F-NEXT:    lw s0, 44(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s1, 40(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s2, 36(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s3, 32(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s4, 28(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s5, 24(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s6, 20(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s7, 16(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s8, 12(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    addi sp, sp, 48
+; RV32ZVE32F-NEXT:    lw s0, 28(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s1, 24(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s2, 20(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s3, 16(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s4, 12(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s5, 8(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s6, 4(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s7, 0(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    addi sp, sp, 32
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB51_10: # %cond.store
 ; RV32ZVE32F-NEXT:    lw a2, 4(a0)
@@ -10227,16 +10224,16 @@ define void @mscatter_baseidx_v8f64(<8 x double> %val, ptr %base, <8 x i64> %idx
 ; RV32ZVE32F-NEXT:    lw a2, 56(a1)
 ; RV32ZVE32F-NEXT:    lw a3, 48(a1)
 ; RV32ZVE32F-NEXT:    lw a4, 40(a1)
-; RV32ZVE32F-NEXT:    lw a5, 32(a1)
-; RV32ZVE32F-NEXT:    lw a6, 24(a1)
-; RV32ZVE32F-NEXT:    lw a7, 16(a1)
-; RV32ZVE32F-NEXT:    lw t0, 8(a1)
+; RV32ZVE32F-NEXT:    lw a5, 8(a1)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vlse32.v v8, (a1), zero
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
-; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV32ZVE32F-NEXT:    lw a6, 16(a1)
+; RV32ZVE32F-NEXT:    lw a7, 24(a1)
+; RV32ZVE32F-NEXT:    lw a1, 32(a1)
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
 ; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll
index 19f3d3ce19fa..7be015e26b09 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll
@@ -19,9 +19,9 @@ define <2 x half> @vp_nearbyint_v2f16(<2 x half> %va, <2 x i1> %m, i32 zeroext %
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <2 x half> @llvm.vp.nearbyint.v2f16(<2 x half> %va, <2 x i1> %m, i32 %evl)
   ret <2 x half> %v
@@ -38,9 +38,9 @@ define <2 x half> @vp_nearbyint_v2f16_unmasked(<2 x half> %va, i32 zeroext %evl)
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <2 x half> @llvm.vp.nearbyint.v2f16(<2 x half> %va, <2 x i1> splat (i1 true), i32 %evl)
   ret <2 x half> %v
@@ -61,9 +61,9 @@ define <4 x half> @vp_nearbyint_v4f16(<4 x half> %va, <4 x i1> %m, i32 zeroext %
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <4 x half> @llvm.vp.nearbyint.v4f16(<4 x half> %va, <4 x i1> %m, i32 %evl)
   ret <4 x half> %v
@@ -80,9 +80,9 @@ define <4 x half> @vp_nearbyint_v4f16_unmasked(<4 x half> %va, i32 zeroext %evl)
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <4 x half> @llvm.vp.nearbyint.v4f16(<4 x half> %va, <4 x i1> splat (i1 true), i32 %evl)
   ret <4 x half> %v
@@ -103,9 +103,9 @@ define <8 x half> @vp_nearbyint_v8f16(<8 x half> %va, <8 x i1> %m, i32 zeroext %
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <8 x half> @llvm.vp.nearbyint.v8f16(<8 x half> %va, <8 x i1> %m, i32 %evl)
   ret <8 x half> %v
@@ -122,9 +122,9 @@ define <8 x half> @vp_nearbyint_v8f16_unmasked(<8 x half> %va, i32 zeroext %evl)
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <8 x half> @llvm.vp.nearbyint.v8f16(<8 x half> %va, <8 x i1> splat (i1 true), i32 %evl)
   ret <8 x half> %v
@@ -147,9 +147,9 @@ define <16 x half> @vp_nearbyint_v16f16(<16 x half> %va, <16 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <16 x half> @llvm.vp.nearbyint.v16f16(<16 x half> %va, <16 x i1> %m, i32 %evl)
   ret <16 x half> %v
@@ -166,9 +166,9 @@ define <16 x half> @vp_nearbyint_v16f16_unmasked(<16 x half> %va, i32 zeroext %e
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <16 x half> @llvm.vp.nearbyint.v16f16(<16 x half> %va, <16 x i1> splat (i1 true), i32 %evl)
   ret <16 x half> %v
@@ -189,9 +189,9 @@ define <2 x float> @vp_nearbyint_v2f32(<2 x float> %va, <2 x i1> %m, i32 zeroext
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <2 x float> @llvm.vp.nearbyint.v2f32(<2 x float> %va, <2 x i1> %m, i32 %evl)
   ret <2 x float> %v
@@ -208,9 +208,9 @@ define <2 x float> @vp_nearbyint_v2f32_unmasked(<2 x float> %va, i32 zeroext %ev
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <2 x float> @llvm.vp.nearbyint.v2f32(<2 x float> %va, <2 x i1> splat (i1 true), i32 %evl)
   ret <2 x float> %v
@@ -231,9 +231,9 @@ define <4 x float> @vp_nearbyint_v4f32(<4 x float> %va, <4 x i1> %m, i32 zeroext
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <4 x float> @llvm.vp.nearbyint.v4f32(<4 x float> %va, <4 x i1> %m, i32 %evl)
   ret <4 x float> %v
@@ -250,9 +250,9 @@ define <4 x float> @vp_nearbyint_v4f32_unmasked(<4 x float> %va, i32 zeroext %ev
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <4 x float> @llvm.vp.nearbyint.v4f32(<4 x float> %va, <4 x i1> splat (i1 true), i32 %evl)
   ret <4 x float> %v
@@ -275,9 +275,9 @@ define <8 x float> @vp_nearbyint_v8f32(<8 x float> %va, <8 x i1> %m, i32 zeroext
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <8 x float> @llvm.vp.nearbyint.v8f32(<8 x float> %va, <8 x i1> %m, i32 %evl)
   ret <8 x float> %v
@@ -294,9 +294,9 @@ define <8 x float> @vp_nearbyint_v8f32_unmasked(<8 x float> %va, i32 zeroext %ev
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <8 x float> @llvm.vp.nearbyint.v8f32(<8 x float> %va, <8 x i1> splat (i1 true), i32 %evl)
   ret <8 x float> %v
@@ -319,9 +319,9 @@ define <16 x float> @vp_nearbyint_v16f32(<16 x float> %va, <16 x i1> %m, i32 zer
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <16 x float> @llvm.vp.nearbyint.v16f32(<16 x float> %va, <16 x i1> %m, i32 %evl)
   ret <16 x float> %v
@@ -338,9 +338,9 @@ define <16 x float> @vp_nearbyint_v16f32_unmasked(<16 x float> %va, i32 zeroext
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <16 x float> @llvm.vp.nearbyint.v16f32(<16 x float> %va, <16 x i1> splat (i1 true), i32 %evl)
   ret <16 x float> %v
@@ -361,9 +361,9 @@ define <2 x double> @vp_nearbyint_v2f64(<2 x double> %va, <2 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <2 x double> @llvm.vp.nearbyint.v2f64(<2 x double> %va, <2 x i1> %m, i32 %evl)
   ret <2 x double> %v
@@ -380,9 +380,9 @@ define <2 x double> @vp_nearbyint_v2f64_unmasked(<2 x double> %va, i32 zeroext %
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <2 x double> @llvm.vp.nearbyint.v2f64(<2 x double> %va, <2 x i1> splat (i1 true), i32 %evl)
   ret <2 x double> %v
@@ -405,9 +405,9 @@ define <4 x double> @vp_nearbyint_v4f64(<4 x double> %va, <4 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <4 x double> @llvm.vp.nearbyint.v4f64(<4 x double> %va, <4 x i1> %m, i32 %evl)
   ret <4 x double> %v
@@ -424,9 +424,9 @@ define <4 x double> @vp_nearbyint_v4f64_unmasked(<4 x double> %va, i32 zeroext %
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <4 x double> @llvm.vp.nearbyint.v4f64(<4 x double> %va, <4 x i1> splat (i1 true), i32 %evl)
   ret <4 x double> %v
@@ -449,9 +449,9 @@ define <8 x double> @vp_nearbyint_v8f64(<8 x double> %va, <8 x i1> %m, i32 zeroe
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <8 x double> @llvm.vp.nearbyint.v8f64(<8 x double> %va, <8 x i1> %m, i32 %evl)
   ret <8 x double> %v
@@ -468,9 +468,9 @@ define <8 x double> @vp_nearbyint_v8f64_unmasked(<8 x double> %va, i32 zeroext %
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <8 x double> @llvm.vp.nearbyint.v8f64(<8 x double> %va, <8 x i1> splat (i1 true), i32 %evl)
   ret <8 x double> %v
@@ -493,9 +493,9 @@ define <15 x double> @vp_nearbyint_v15f64(<15 x double> %va, <15 x i1> %m, i32 z
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v24, v24, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v24, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <15 x double> @llvm.vp.nearbyint.v15f64(<15 x double> %va, <15 x i1> %m, i32 %evl)
   ret <15 x double> %v
@@ -512,9 +512,9 @@ define <15 x double> @vp_nearbyint_v15f64_unmasked(<15 x double> %va, i32 zeroex
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <15 x double> @llvm.vp.nearbyint.v15f64(<15 x double> %va, <15 x i1> splat (i1 true), i32 %evl)
   ret <15 x double> %v
@@ -537,9 +537,9 @@ define <16 x double> @vp_nearbyint_v16f64(<16 x double> %va, <16 x i1> %m, i32 z
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v24, v24, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v24, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <16 x double> @llvm.vp.nearbyint.v16f64(<16 x double> %va, <16 x i1> %m, i32 %evl)
   ret <16 x double> %v
@@ -556,9 +556,9 @@ define <16 x double> @vp_nearbyint_v16f64_unmasked(<16 x double> %va, i32 zeroex
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <16 x double> @llvm.vp.nearbyint.v16f64(<16 x double> %va, <16 x i1> splat (i1 true), i32 %evl)
   ret <16 x double> %v
@@ -617,9 +617,9 @@ define <32 x double> @vp_nearbyint_v32f64(<32 x double> %va, <32 x i1> %m, i32 z
 ; CHECK-NEXT:    vmv1r.v v0, v7
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v24, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v24, v16, v24, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vmv.v.v v16, v24
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
@@ -660,9 +660,9 @@ define <32 x double> @vp_nearbyint_v32f64_unmasked(<32 x double> %va, i32 zeroex
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v16, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v24, v24, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v16, v24, v16, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <32 x double> @llvm.vp.nearbyint.v32f64(<32 x double> %va, <32 x i1> splat (i1 true), i32 %evl)
   ret <32 x double> %v
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
index 7dcd4c419982..ed2ed2a2ebfa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
@@ -5,8 +5,8 @@
 define void @vselect_vv_v6i32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-LABEL: vselect_vv_v6i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    lbu a2, 0(a2)
+; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a1)
 ; RV32-NEXT:    slli a1, a2, 30
 ; RV32-NEXT:    srli a1, a1, 31
@@ -35,8 +35,8 @@ define void @vselect_vv_v6i32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ;
 ; RV64-LABEL: vselect_vv_v6i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    lbu a2, 0(a2)
+; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    vle32.v v8, (a1)
 ; RV64-NEXT:    slli a1, a2, 62
 ; RV64-NEXT:    srli a1, a1, 63
@@ -73,8 +73,8 @@ define void @vselect_vv_v6i32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 define void @vselect_vx_v6i32(i32 %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-LABEL: vselect_vx_v6i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    lbu a2, 0(a2)
+; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a1)
 ; RV32-NEXT:    slli a1, a2, 30
 ; RV32-NEXT:    srli a1, a1, 31
@@ -104,8 +104,8 @@ define void @vselect_vx_v6i32(i32 %a, ptr %b, ptr %cc, ptr %z) {
 ;
 ; RV64-LABEL: vselect_vx_v6i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    lbu a2, 0(a2)
+; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    vle32.v v8, (a1)
 ; RV64-NEXT:    slli a1, a2, 62
 ; RV64-NEXT:    srli a1, a1, 63
@@ -144,8 +144,8 @@ define void @vselect_vx_v6i32(i32 %a, ptr %b, ptr %cc, ptr %z) {
 define void @vselect_vi_v6i32(ptr %b, ptr %cc, ptr %z) {
 ; RV32-LABEL: vselect_vi_v6i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    lbu a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
 ; RV32-NEXT:    slli a0, a1, 30
 ; RV32-NEXT:    srli a0, a0, 31
@@ -175,8 +175,8 @@ define void @vselect_vi_v6i32(ptr %b, ptr %cc, ptr %z) {
 ;
 ; RV64-LABEL: vselect_vi_v6i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    lbu a1, 0(a1)
+; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    vle32.v v8, (a0)
 ; RV64-NEXT:    slli a0, a1, 62
 ; RV64-NEXT:    srli a0, a0, 63
@@ -214,8 +214,8 @@ define void @vselect_vi_v6i32(ptr %b, ptr %cc, ptr %z) {
 define void @vselect_vv_v6f32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-LABEL: vselect_vv_v6f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    lbu a2, 0(a2)
+; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a1)
 ; RV32-NEXT:    slli a1, a2, 30
 ; RV32-NEXT:    srli a1, a1, 31
@@ -244,8 +244,8 @@ define void @vselect_vv_v6f32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ;
 ; RV64-LABEL: vselect_vv_v6f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    lbu a2, 0(a2)
+; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    vle32.v v8, (a1)
 ; RV64-NEXT:    slli a1, a2, 62
 ; RV64-NEXT:    srli a1, a1, 63
@@ -282,8 +282,8 @@ define void @vselect_vv_v6f32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 define void @vselect_vx_v6f32(float %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-LABEL: vselect_vx_v6f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    lbu a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
 ; RV32-NEXT:    slli a0, a1, 30
 ; RV32-NEXT:    srli a0, a0, 31
@@ -313,8 +313,8 @@ define void @vselect_vx_v6f32(float %a, ptr %b, ptr %cc, ptr %z) {
 ;
 ; RV64-LABEL: vselect_vx_v6f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    lbu a1, 0(a1)
+; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    vle32.v v8, (a0)
 ; RV64-NEXT:    slli a0, a1, 62
 ; RV64-NEXT:    srli a0, a0, 63
@@ -353,8 +353,8 @@ define void @vselect_vx_v6f32(float %a, ptr %b, ptr %cc, ptr %z) {
 define void @vselect_vfpzero_v6f32(ptr %b, ptr %cc, ptr %z) {
 ; RV32-LABEL: vselect_vfpzero_v6f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    lbu a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
 ; RV32-NEXT:    slli a0, a1, 30
 ; RV32-NEXT:    srli a0, a0, 31
@@ -384,8 +384,8 @@ define void @vselect_vfpzero_v6f32(ptr %b, ptr %cc, ptr %z) {
 ;
 ; RV64-LABEL: vselect_vfpzero_v6f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    lbu a1, 0(a1)
+; RV64-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
 ; RV64-NEXT:    vle32.v v8, (a0)
 ; RV64-NEXT:    slli a0, a1, 62
 ; RV64-NEXT:    srli a0, a0, 63
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwadd.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwadd.ll
index a4a5917fd4f9..b1726be941e3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwadd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwadd.ll
@@ -768,8 +768,8 @@ define <4 x i32> @vwadd_vx_v4i32_i32(ptr %x, ptr %y) {
 define <2 x i64> @vwadd_vx_v2i64_i8(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwadd_vx_v2i64_i8:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    lb a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    vmv.v.x v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
@@ -796,8 +796,8 @@ define <2 x i64> @vwadd_vx_v2i64_i8(ptr %x, ptr %y) nounwind {
 define <2 x i64> @vwadd_vx_v2i64_i16(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwadd_vx_v2i64_i16:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    lh a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    vmv.v.x v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
@@ -824,8 +824,8 @@ define <2 x i64> @vwadd_vx_v2i64_i16(ptr %x, ptr %y) nounwind {
 define <2 x i64> @vwadd_vx_v2i64_i32(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwadd_vx_v2i64_i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    vmv.v.x v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
@@ -853,9 +853,9 @@ define <2 x i64> @vwadd_vx_v2i64_i64(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwadd_vx_v2i64_i64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    lw a2, 4(a1)
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll
index bc0bf5dd76ad..f6d9695c5149 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll
@@ -769,8 +769,8 @@ define <2 x i64> @vwaddu_vx_v2i64_i8(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwaddu_vx_v2i64_i8:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    lbu a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
@@ -801,8 +801,8 @@ define <2 x i64> @vwaddu_vx_v2i64_i16(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwaddu_vx_v2i64_i16:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    lhu a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
@@ -833,8 +833,8 @@ define <2 x i64> @vwaddu_vx_v2i64_i32(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwaddu_vx_v2i64_i32:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
@@ -865,9 +865,9 @@ define <2 x i64> @vwaddu_vx_v2i64_i64(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwaddu_vx_v2i64_i64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    lw a2, 4(a1)
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
index 2abd34f01c14..c87584ab6351 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmul.ll
@@ -883,9 +883,9 @@ define <2 x i64> @vwmul_vx_v2i64_i64(ptr %x, ptr %y) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    lw a2, 4(a1)
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
 ; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulsu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulsu.ll
index 921037db2ea9..a56984577ea7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulsu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulsu.ll
@@ -793,8 +793,8 @@ define <2 x i64> @vwmulsu_vx_v2i64_i8(ptr %x, ptr %y) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    lbu a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
 ; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
@@ -827,8 +827,8 @@ define <2 x i64> @vwmulsu_vx_v2i64_i16(ptr %x, ptr %y) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    lhu a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
 ; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
@@ -861,8 +861,8 @@ define <2 x i64> @vwmulsu_vx_v2i64_i32(ptr %x, ptr %y) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
 ; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsub.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsub.ll
index 154093d759d6..2782a5fbb1ea 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsub.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsub.ll
@@ -769,8 +769,8 @@ define <4 x i32> @vwsub_vx_v4i32_i32(ptr %x, ptr %y) {
 define <2 x i64> @vwsub_vx_v2i64_i8(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwsub_vx_v2i64_i8:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    lb a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    vmv.v.x v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
@@ -798,8 +798,8 @@ define <2 x i64> @vwsub_vx_v2i64_i8(ptr %x, ptr %y) nounwind {
 define <2 x i64> @vwsub_vx_v2i64_i16(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwsub_vx_v2i64_i16:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    lh a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    vmv.v.x v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
@@ -827,8 +827,8 @@ define <2 x i64> @vwsub_vx_v2i64_i16(ptr %x, ptr %y) nounwind {
 define <2 x i64> @vwsub_vx_v2i64_i32(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwsub_vx_v2i64_i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    vmv.v.x v8, a1
 ; RV32-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
@@ -856,9 +856,9 @@ define <2 x i64> @vwsub_vx_v2i64_i64(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwsub_vx_v2i64_i64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    lw a2, 4(a1)
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsubu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsubu.ll
index a084b5383b40..ccbc26c84d80 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsubu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwsubu.ll
@@ -770,8 +770,8 @@ define <2 x i64> @vwsubu_vx_v2i64_i8(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwsubu_vx_v2i64_i8:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    lbu a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
@@ -803,8 +803,8 @@ define <2 x i64> @vwsubu_vx_v2i64_i16(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwsubu_vx_v2i64_i16:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    lhu a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
@@ -836,8 +836,8 @@ define <2 x i64> @vwsubu_vx_v2i64_i32(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwsubu_vx_v2i64_i32:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
@@ -868,9 +868,9 @@ define <2 x i64> @vwsubu_vx_v2i64_i64(ptr %x, ptr %y) nounwind {
 ; RV32-LABEL: vwsubu_vx_v2i64_i64:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    lw a2, 4(a1)
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a0)
 ; RV32-NEXT:    sw a2, 12(sp)
 ; RV32-NEXT:    sw a1, 8(sp)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll
index b78b8663eac9..02cfd3de6b4d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fmaximum-vp.ll
@@ -1012,77 +1012,99 @@ define <vscale x 16 x double> @vfmax_vv_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    li a3, 42
+; CHECK-NEXT:    li a3, 36
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x2a, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 42 * vlenb
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x24, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 36 * vlenb
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a3, a1, 3
 ; CHECK-NEXT:    add a3, a0, a3
 ; CHECK-NEXT:    vl8re64.v v24, (a3)
 ; CHECK-NEXT:    csrr a3, vlenb
-; CHECK-NEXT:    slli a3, a3, 4
+; CHECK-NEXT:    li a4, 27
+; CHECK-NEXT:    mul a3, a3, a4
 ; CHECK-NEXT:    add a3, sp, a3
 ; CHECK-NEXT:    addi a3, a3, 16
 ; CHECK-NEXT:    vs8r.v v24, (a3) # Unknown-size Folded Spill
 ; CHECK-NEXT:    srli a3, a1, 3
 ; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrr a4, vlenb
-; CHECK-NEXT:    slli a4, a4, 5
+; CHECK-NEXT:    li a5, 18
+; CHECK-NEXT:    mul a4, a4, a5
 ; CHECK-NEXT:    add a4, sp, a4
 ; CHECK-NEXT:    addi a4, a4, 16
 ; CHECK-NEXT:    vs1r.v v0, (a4) # Unknown-size Folded Spill
-; CHECK-NEXT:    vslidedown.vx v24, v0, a3
+; CHECK-NEXT:    vslidedown.vx v7, v0, a3
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    add a3, sp, a3
+; CHECK-NEXT:    addi a3, a3, 16
+; CHECK-NEXT:    vs1r.v v7, (a3) # Unknown-size Folded Spill
 ; CHECK-NEXT:    sub a3, a2, a1
 ; CHECK-NEXT:    sltu a4, a2, a3
 ; CHECK-NEXT:    addi a4, a4, -1
 ; CHECK-NEXT:    and a3, a4, a3
-; CHECK-NEXT:    vl8re64.v v0, (a0)
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a4, a0, 5
-; CHECK-NEXT:    add a0, a4, a0
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
 ; CHECK-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
-; CHECK-NEXT:    vmv1r.v v7, v24
-; CHECK-NEXT:    vmv1r.v v0, v24
+; CHECK-NEXT:    vmv1r.v v0, v7
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    slli a3, a3, 1
+; CHECK-NEXT:    add a3, sp, a3
+; CHECK-NEXT:    addi a3, a3, 16
+; CHECK-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; CHECK-NEXT:    vmfeq.vv v26, v16, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v26
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    li a4, 27
+; CHECK-NEXT:    mul a3, a3, a4
+; CHECK-NEXT:    add a3, sp, a3
+; CHECK-NEXT:    addi a3, a3, 16
+; CHECK-NEXT:    vl8r.v v24, (a3) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmerge.vvm v24, v16, v24, v0
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    li a4, 10
+; CHECK-NEXT:    mul a3, a3, a4
+; CHECK-NEXT:    add a3, sp, a3
+; CHECK-NEXT:    addi a3, a3, 16
+; CHECK-NEXT:    vs8r.v v24, (a3) # Unknown-size Folded Spill
+; CHECK-NEXT:    vmv1r.v v0, v7
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    li a4, 27
+; CHECK-NEXT:    mul a3, a3, a4
+; CHECK-NEXT:    add a3, sp, a3
+; CHECK-NEXT:    addi a3, a3, 16
+; CHECK-NEXT:    vl8r.v v24, (a3) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmfeq.vv v16, v24, v24, v0.t
+; CHECK-NEXT:    addi a3, sp, 16
+; CHECK-NEXT:    vs1r.v v16, (a3) # Unknown-size Folded Spill
+; CHECK-NEXT:    vl8re64.v v0, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    li a3, 19
+; CHECK-NEXT:    mul a0, a0, a3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vmerge.vvm v16, v16, v24, v0
+; CHECK-NEXT:    vl1r.v v0, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    slli a0, a0, 1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vmv1r.v v0, v7
-; CHECK-NEXT:    vmfeq.vv v17, v24, v24, v0.t
+; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmerge.vvm v8, v24, v8, v0
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a3, 24
-; CHECK-NEXT:    mul a0, a0, a3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vmv1r.v v0, v17
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vmerge.vvm v8, v24, v8, v0
-; CHECK-NEXT:    vmv1r.v v0, v7
+; CHECK-NEXT:    vl1r.v v0, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    li a3, 10
+; CHECK-NEXT:    mul a0, a0, a3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vfmax.vv v8, v8, v16, v0.t
+; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vfmax.vv v8, v8, v24, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    li a3, 10
+; CHECK-NEXT:    mul a0, a0, a3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
@@ -1092,49 +1114,63 @@ define <vscale x 16 x double> @vfmax_vv_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:  .LBB28_2:
 ; CHECK-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl1r.v v7, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vmv1r.v v0, v7
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 24
+; CHECK-NEXT:    li a1, 18
 ; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vl1r.v v24, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vmfeq.vv v25, v16, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a1, a0, 5
-; CHECK-NEXT:    add a0, a1, a0
+; CHECK-NEXT:    li a1, 19
+; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vmerge.vvm v24, v16, v8, v0
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    li a1, 27
+; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vmv1r.v v0, v7
+; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 27
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmerge.vvm v16, v16, v8, v0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vmfeq.vv v25, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v25
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 27
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vmerge.vvm v8, v8, v16, v0
-; CHECK-NEXT:    vmv1r.v v0, v7
+; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    slli a0, a0, 1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vfmax.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    li a1, 10
+; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 42
+; CHECK-NEXT:    li a1, 36
 ; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add sp, sp, a0
 ; CHECK-NEXT:    addi sp, sp, 16
@@ -1162,19 +1198,19 @@ define <vscale x 16 x double> @vfmax_vv_nxv16f64_unmasked(<vscale x 16 x double>
 ; CHECK-NEXT:    slli a3, a1, 3
 ; CHECK-NEXT:    add a3, a0, a3
 ; CHECK-NEXT:    vl8re64.v v24, (a3)
+; CHECK-NEXT:    sub a3, a2, a1
+; CHECK-NEXT:    sltu a4, a2, a3
+; CHECK-NEXT:    addi a4, a4, -1
+; CHECK-NEXT:    and a3, a4, a3
+; CHECK-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
+; CHECK-NEXT:    vmfeq.vv v0, v16, v16
+; CHECK-NEXT:    vmfeq.vv v7, v24, v24
 ; CHECK-NEXT:    vl8re64.v v8, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    sub a0, a2, a1
-; CHECK-NEXT:    sltu a3, a2, a0
-; CHECK-NEXT:    addi a3, a3, -1
-; CHECK-NEXT:    and a0, a3, a0
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v16, v16
-; CHECK-NEXT:    vmfeq.vv v7, v24, v24
 ; CHECK-NEXT:    vmerge.vvm v8, v16, v24, v0
 ; CHECK-NEXT:    vmv1r.v v0, v7
 ; CHECK-NEXT:    vmerge.vvm v16, v24, v16, v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll
index 69c76152910e..72a47ca2a605 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fminimum-vp.ll
@@ -1012,77 +1012,99 @@ define <vscale x 16 x double> @vfmin_vv_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    li a3, 42
+; CHECK-NEXT:    li a3, 36
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x2a, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 42 * vlenb
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x24, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 36 * vlenb
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a3, a1, 3
 ; CHECK-NEXT:    add a3, a0, a3
 ; CHECK-NEXT:    vl8re64.v v24, (a3)
 ; CHECK-NEXT:    csrr a3, vlenb
-; CHECK-NEXT:    slli a3, a3, 4
+; CHECK-NEXT:    li a4, 27
+; CHECK-NEXT:    mul a3, a3, a4
 ; CHECK-NEXT:    add a3, sp, a3
 ; CHECK-NEXT:    addi a3, a3, 16
 ; CHECK-NEXT:    vs8r.v v24, (a3) # Unknown-size Folded Spill
 ; CHECK-NEXT:    srli a3, a1, 3
 ; CHECK-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    csrr a4, vlenb
-; CHECK-NEXT:    slli a4, a4, 5
+; CHECK-NEXT:    li a5, 18
+; CHECK-NEXT:    mul a4, a4, a5
 ; CHECK-NEXT:    add a4, sp, a4
 ; CHECK-NEXT:    addi a4, a4, 16
 ; CHECK-NEXT:    vs1r.v v0, (a4) # Unknown-size Folded Spill
-; CHECK-NEXT:    vslidedown.vx v24, v0, a3
+; CHECK-NEXT:    vslidedown.vx v7, v0, a3
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    add a3, sp, a3
+; CHECK-NEXT:    addi a3, a3, 16
+; CHECK-NEXT:    vs1r.v v7, (a3) # Unknown-size Folded Spill
 ; CHECK-NEXT:    sub a3, a2, a1
 ; CHECK-NEXT:    sltu a4, a2, a3
 ; CHECK-NEXT:    addi a4, a4, -1
 ; CHECK-NEXT:    and a3, a4, a3
-; CHECK-NEXT:    vl8re64.v v0, (a0)
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a4, a0, 5
-; CHECK-NEXT:    add a0, a4, a0
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
 ; CHECK-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
-; CHECK-NEXT:    vmv1r.v v7, v24
-; CHECK-NEXT:    vmv1r.v v0, v24
+; CHECK-NEXT:    vmv1r.v v0, v7
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    slli a3, a3, 1
+; CHECK-NEXT:    add a3, sp, a3
+; CHECK-NEXT:    addi a3, a3, 16
+; CHECK-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; CHECK-NEXT:    vmfeq.vv v26, v16, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v26
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    li a4, 27
+; CHECK-NEXT:    mul a3, a3, a4
+; CHECK-NEXT:    add a3, sp, a3
+; CHECK-NEXT:    addi a3, a3, 16
+; CHECK-NEXT:    vl8r.v v24, (a3) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmerge.vvm v24, v16, v24, v0
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    li a4, 10
+; CHECK-NEXT:    mul a3, a3, a4
+; CHECK-NEXT:    add a3, sp, a3
+; CHECK-NEXT:    addi a3, a3, 16
+; CHECK-NEXT:    vs8r.v v24, (a3) # Unknown-size Folded Spill
+; CHECK-NEXT:    vmv1r.v v0, v7
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    li a4, 27
+; CHECK-NEXT:    mul a3, a3, a4
+; CHECK-NEXT:    add a3, sp, a3
+; CHECK-NEXT:    addi a3, a3, 16
+; CHECK-NEXT:    vl8r.v v24, (a3) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmfeq.vv v16, v24, v24, v0.t
+; CHECK-NEXT:    addi a3, sp, 16
+; CHECK-NEXT:    vs1r.v v16, (a3) # Unknown-size Folded Spill
+; CHECK-NEXT:    vl8re64.v v0, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    li a3, 19
+; CHECK-NEXT:    mul a0, a0, a3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vmerge.vvm v16, v16, v24, v0
+; CHECK-NEXT:    vl1r.v v0, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    slli a0, a0, 1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vmv1r.v v0, v7
-; CHECK-NEXT:    vmfeq.vv v17, v24, v24, v0.t
+; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmerge.vvm v8, v24, v8, v0
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a3, 24
-; CHECK-NEXT:    mul a0, a0, a3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vmv1r.v v0, v17
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vmerge.vvm v8, v24, v8, v0
-; CHECK-NEXT:    vmv1r.v v0, v7
+; CHECK-NEXT:    vl1r.v v0, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    li a3, 10
+; CHECK-NEXT:    mul a0, a0, a3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vfmin.vv v8, v8, v16, v0.t
+; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vfmin.vv v8, v8, v24, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    li a3, 10
+; CHECK-NEXT:    mul a0, a0, a3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
@@ -1092,49 +1114,63 @@ define <vscale x 16 x double> @vfmin_vv_nxv16f64(<vscale x 16 x double> %va, <vs
 ; CHECK-NEXT:  .LBB28_2:
 ; CHECK-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl1r.v v7, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vmv1r.v v0, v7
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 24
+; CHECK-NEXT:    li a1, 18
 ; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vl1r.v v24, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vmfeq.vv v25, v16, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v25
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a1, a0, 5
-; CHECK-NEXT:    add a0, a1, a0
+; CHECK-NEXT:    li a1, 19
+; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vmerge.vvm v24, v16, v8, v0
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    li a1, 27
+; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vmv1r.v v0, v7
+; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 27
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmerge.vvm v16, v16, v8, v0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vmfeq.vv v25, v8, v8, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v25
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    li a1, 27
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vmerge.vvm v8, v8, v16, v0
-; CHECK-NEXT:    vmv1r.v v0, v7
+; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    slli a0, a0, 1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vfmin.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    li a1, 10
+; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 42
+; CHECK-NEXT:    li a1, 36
 ; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add sp, sp, a0
 ; CHECK-NEXT:    addi sp, sp, 16
@@ -1162,19 +1198,19 @@ define <vscale x 16 x double> @vfmin_vv_nxv16f64_unmasked(<vscale x 16 x double>
 ; CHECK-NEXT:    slli a3, a1, 3
 ; CHECK-NEXT:    add a3, a0, a3
 ; CHECK-NEXT:    vl8re64.v v24, (a3)
+; CHECK-NEXT:    sub a3, a2, a1
+; CHECK-NEXT:    sltu a4, a2, a3
+; CHECK-NEXT:    addi a4, a4, -1
+; CHECK-NEXT:    and a3, a4, a3
+; CHECK-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
+; CHECK-NEXT:    vmfeq.vv v0, v16, v16
+; CHECK-NEXT:    vmfeq.vv v7, v24, v24
 ; CHECK-NEXT:    vl8re64.v v8, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    sub a0, a2, a1
-; CHECK-NEXT:    sltu a3, a2, a0
-; CHECK-NEXT:    addi a3, a3, -1
-; CHECK-NEXT:    and a0, a3, a0
-; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    vmfeq.vv v0, v16, v16
-; CHECK-NEXT:    vmfeq.vv v7, v24, v24
 ; CHECK-NEXT:    vmerge.vvm v8, v16, v24, v0
 ; CHECK-NEXT:    vmv1r.v v0, v7
 ; CHECK-NEXT:    vmerge.vvm v16, v24, v16, v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fnearbyint-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-constrained-sdnode.ll
index f90237b8d7e9..f88a9b3081a1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fnearbyint-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-constrained-sdnode.ll
@@ -19,9 +19,9 @@ define <vscale x 1 x half> @nearbyint_nxv1f16(<vscale x 1 x half> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 1 x half> @llvm.experimental.constrained.nearbyint.nxv1f16(<vscale x 1 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 1 x half> %r
@@ -42,9 +42,9 @@ define <vscale x 2 x half> @nearbyint_nxv2f16(<vscale x 2 x half> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 2 x half> @llvm.experimental.constrained.nearbyint.nxv2f16(<vscale x 2 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 2 x half> %r
@@ -65,9 +65,9 @@ define <vscale x 4 x half> @nearbyint_nxv4f16(<vscale x 4 x half> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 4 x half> @llvm.experimental.constrained.nearbyint.nxv4f16(<vscale x 4 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 4 x half> %r
@@ -88,9 +88,9 @@ define <vscale x 8 x half> @nearbyint_nxv8f16(<vscale x 8 x half> %v) strictfp {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 8 x half> @llvm.experimental.constrained.nearbyint.nxv8f16(<vscale x 8 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 8 x half> %r
@@ -111,9 +111,9 @@ define <vscale x 16 x half> @nearbyint_nxv16f16(<vscale x 16 x half> %v) strictf
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 16 x half> @llvm.experimental.constrained.nearbyint.nxv16f16(<vscale x 16 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 16 x half> %r
@@ -134,9 +134,9 @@ define <vscale x 32 x half> @nearbyint_nxv32f16(<vscale x 32 x half> %v) strictf
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 32 x half> @llvm.experimental.constrained.nearbyint.nxv32f16(<vscale x 32 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 32 x half> %r
@@ -157,9 +157,9 @@ define <vscale x 1 x float> @nearbyint_nxv1f32(<vscale x 1 x float> %v) strictfp
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 1 x float> @llvm.experimental.constrained.nearbyint.nxv1f32(<vscale x 1 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 1 x float> %r
@@ -180,9 +180,9 @@ define <vscale x 2 x float> @nearbyint_nxv2f32(<vscale x 2 x float> %v) strictfp
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 2 x float> @llvm.experimental.constrained.nearbyint.nxv2f32(<vscale x 2 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 2 x float> %r
@@ -203,9 +203,9 @@ define <vscale x 4 x float> @nearbyint_nxv4f32(<vscale x 4 x float> %v) strictfp
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 4 x float> @llvm.experimental.constrained.nearbyint.nxv4f32(<vscale x 4 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 4 x float> %r
@@ -226,9 +226,9 @@ define <vscale x 8 x float> @nearbyint_nxv8f32(<vscale x 8 x float> %v) strictfp
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 8 x float> @llvm.experimental.constrained.nearbyint.nxv8f32(<vscale x 8 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 8 x float> %r
@@ -249,9 +249,9 @@ define <vscale x 16 x float> @nearbyint_nxv16f32(<vscale x 16 x float> %v) stric
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 16 x float> @llvm.experimental.constrained.nearbyint.nxv16f32(<vscale x 16 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 16 x float> %r
@@ -272,9 +272,9 @@ define <vscale x 1 x double> @nearbyint_nxv1f64(<vscale x 1 x double> %v) strict
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 1 x double> @llvm.experimental.constrained.nearbyint.nxv1f64(<vscale x 1 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 1 x double> %r
@@ -295,9 +295,9 @@ define <vscale x 2 x double> @nearbyint_nxv2f64(<vscale x 2 x double> %v) strict
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 2 x double> @llvm.experimental.constrained.nearbyint.nxv2f64(<vscale x 2 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 2 x double> %r
@@ -318,9 +318,9 @@ define <vscale x 4 x double> @nearbyint_nxv4f64(<vscale x 4 x double> %v) strict
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 4 x double> @llvm.experimental.constrained.nearbyint.nxv4f64(<vscale x 4 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 4 x double> %r
@@ -341,9 +341,9 @@ define <vscale x 8 x double> @nearbyint_nxv8f64(<vscale x 8 x double> %v) strict
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %r = call <vscale x 8 x double> @llvm.experimental.constrained.nearbyint.nxv8f64(<vscale x 8 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
   ret <vscale x 8 x double> %r
diff --git a/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll
index 9aa356b9b65e..9e14852305ca 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll
@@ -15,9 +15,9 @@ define <vscale x 1 x half> @nearbyint_nxv1f16(<vscale x 1 x half> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 1 x half> @llvm.nearbyint.nxv1f16(<vscale x 1 x half> %x)
   ret <vscale x 1 x half> %a
@@ -35,9 +35,9 @@ define <vscale x 2 x half> @nearbyint_nxv2f16(<vscale x 2 x half> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x half> @llvm.nearbyint.nxv2f16(<vscale x 2 x half> %x)
   ret <vscale x 2 x half> %a
@@ -55,9 +55,9 @@ define <vscale x 4 x half> @nearbyint_nxv4f16(<vscale x 4 x half> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 4 x half> @llvm.nearbyint.nxv4f16(<vscale x 4 x half> %x)
   ret <vscale x 4 x half> %a
@@ -75,9 +75,9 @@ define <vscale x 8 x half> @nearbyint_nxv8f16(<vscale x 8 x half> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 8 x half> @llvm.nearbyint.nxv8f16(<vscale x 8 x half> %x)
   ret <vscale x 8 x half> %a
@@ -95,9 +95,9 @@ define <vscale x 16 x half> @nearbyint_nxv16f16(<vscale x 16 x half> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 16 x half> @llvm.nearbyint.nxv16f16(<vscale x 16 x half> %x)
   ret <vscale x 16 x half> %a
@@ -115,9 +115,9 @@ define <vscale x 32 x half> @nearbyint_nxv32f16(<vscale x 32 x half> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 32 x half> @llvm.nearbyint.nxv32f16(<vscale x 32 x half> %x)
   ret <vscale x 32 x half> %a
@@ -135,9 +135,9 @@ define <vscale x 1 x float> @nearbyint_nxv1f32(<vscale x 1 x float> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 1 x float> @llvm.nearbyint.nxv1f32(<vscale x 1 x float> %x)
   ret <vscale x 1 x float> %a
@@ -155,9 +155,9 @@ define <vscale x 2 x float> @nearbyint_nxv2f32(<vscale x 2 x float> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x float> @llvm.nearbyint.nxv2f32(<vscale x 2 x float> %x)
   ret <vscale x 2 x float> %a
@@ -175,9 +175,9 @@ define <vscale x 4 x float> @nearbyint_nxv4f32(<vscale x 4 x float> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> %x)
   ret <vscale x 4 x float> %a
@@ -195,9 +195,9 @@ define <vscale x 8 x float> @nearbyint_nxv8f32(<vscale x 8 x float> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 8 x float> @llvm.nearbyint.nxv8f32(<vscale x 8 x float> %x)
   ret <vscale x 8 x float> %a
@@ -215,9 +215,9 @@ define <vscale x 16 x float> @nearbyint_nxv16f32(<vscale x 16 x float> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 16 x float> @llvm.nearbyint.nxv16f32(<vscale x 16 x float> %x)
   ret <vscale x 16 x float> %a
@@ -235,9 +235,9 @@ define <vscale x 1 x double> @nearbyint_nxv1f64(<vscale x 1 x double> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 1 x double> @llvm.nearbyint.nxv1f64(<vscale x 1 x double> %x)
   ret <vscale x 1 x double> %a
@@ -255,9 +255,9 @@ define <vscale x 2 x double> @nearbyint_nxv2f64(<vscale x 2 x double> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> %x)
   ret <vscale x 2 x double> %a
@@ -275,9 +275,9 @@ define <vscale x 4 x double> @nearbyint_nxv4f64(<vscale x 4 x double> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 4 x double> @llvm.nearbyint.nxv4f64(<vscale x 4 x double> %x)
   ret <vscale x 4 x double> %a
@@ -295,9 +295,9 @@ define <vscale x 8 x double> @nearbyint_nxv8f64(<vscale x 8 x double> %x) {
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 8 x double> @llvm.nearbyint.nxv8f64(<vscale x 8 x double> %x)
   ret <vscale x 8 x double> %a
diff --git a/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll
index 277cd7dcdabc..249f765971b0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fshr-fshl-vp.ll
@@ -960,108 +960,87 @@ define <vscale x 16 x i64> @fshr_v16i64(<vscale x 16 x i64> %a, <vscale x 16 x i
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    li a3, 56
+; CHECK-NEXT:    li a3, 40
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x38, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 56 * vlenb
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x28, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 40 * vlenb
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    li a3, 24
-; CHECK-NEXT:    mul a1, a1, a3
+; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 5
+; CHECK-NEXT:    li a3, 24
+; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a3, vlenb
 ; CHECK-NEXT:    slli a1, a3, 3
 ; CHECK-NEXT:    add a5, a0, a1
+; CHECK-NEXT:    srli a6, a3, 3
+; CHECK-NEXT:    vsetvli a7, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    add a1, a2, a1
+; CHECK-NEXT:    vl8re64.v v16, (a1)
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT:    sub a1, a4, a3
+; CHECK-NEXT:    sltu a7, a4, a1
+; CHECK-NEXT:    addi a7, a7, -1
+; CHECK-NEXT:    and a7, a7, a1
+; CHECK-NEXT:    li a1, 63
 ; CHECK-NEXT:    vl8re64.v v8, (a5)
 ; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    slli a5, a5, 3
+; CHECK-NEXT:    slli a5, a5, 5
 ; CHECK-NEXT:    add a5, sp, a5
 ; CHECK-NEXT:    addi a5, a5, 16
 ; CHECK-NEXT:    vs8r.v v8, (a5) # Unknown-size Folded Spill
-; CHECK-NEXT:    srli a5, a3, 3
-; CHECK-NEXT:    vsetvli a6, zero, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v0, v0, a5
-; CHECK-NEXT:    add a5, a2, a1
-; CHECK-NEXT:    sub a1, a4, a3
-; CHECK-NEXT:    sltu a6, a4, a1
-; CHECK-NEXT:    addi a6, a6, -1
-; CHECK-NEXT:    and a6, a6, a1
-; CHECK-NEXT:    li a1, 63
-; CHECK-NEXT:    vl8re64.v v8, (a5)
+; CHECK-NEXT:    vslidedown.vx v0, v0, a6
+; CHECK-NEXT:    vsetvli zero, a7, e64, m8, ta, ma
+; CHECK-NEXT:    vand.vx v8, v16, a1, v0.t
 ; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    li a7, 40
-; CHECK-NEXT:    mul a5, a5, a7
+; CHECK-NEXT:    slli a5, a5, 5
 ; CHECK-NEXT:    add a5, sp, a5
 ; CHECK-NEXT:    addi a5, a5, 16
+; CHECK-NEXT:    vl8r.v v16, (a5) # Unknown-size Folded Reload
+; CHECK-NEXT:    vsrl.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    addi a5, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a5) # Unknown-size Folded Spill
-; CHECK-NEXT:    vl8re64.v v16, (a0)
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vl8re64.v v16, (a2)
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 48
-; CHECK-NEXT:    mul a0, a0, a2
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli zero, a6, e64, m8, ta, ma
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 40
-; CHECK-NEXT:    mul a0, a0, a2
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vand.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vsrl.vv v16, v16, v8, v0.t
+; CHECK-NEXT:    csrr a5, vlenb
+; CHECK-NEXT:    slli a5, a5, 3
+; CHECK-NEXT:    add a5, sp, a5
+; CHECK-NEXT:    addi a5, a5, 16
+; CHECK-NEXT:    vl8r.v v8, (a5) # Unknown-size Folded Reload
+; CHECK-NEXT:    vnot.v v8, v8, v0.t
+; CHECK-NEXT:    vand.vx v16, v8, a1, v0.t
+; CHECK-NEXT:    vl8re64.v v8, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vl8re64.v v8, (a2)
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 40
-; CHECK-NEXT:    mul a0, a0, a2
+; CHECK-NEXT:    slli a0, a0, 5
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vnot.v v8, v8, v0.t
-; CHECK-NEXT:    vand.vx v16, v8, a1, v0.t
+; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 24
-; CHECK-NEXT:    mul a0, a0, a2
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vsll.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vsll.vv v8, v8, v16, v0.t
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vor.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 40
-; CHECK-NEXT:    mul a0, a0, a2
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
@@ -1072,66 +1051,50 @@ define <vscale x 16 x i64> @fshr_v16i64(<vscale x 16 x i64> %a, <vscale x 16 x i
 ; CHECK-NEXT:    vsetvli zero, a4, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 48
-; CHECK-NEXT:    mul a0, a0, a2
+; CHECK-NEXT:    slli a0, a0, 5
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vand.vx v8, v8, a1, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 24
-; CHECK-NEXT:    mul a0, a0, a2
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 24
-; CHECK-NEXT:    mul a0, a0, a2
+; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vsrl.vv v8, v8, v16, v0.t
+; CHECK-NEXT:    vsrl.vv v8, v16, v8, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 24
-; CHECK-NEXT:    mul a0, a0, a2
+; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 48
-; CHECK-NEXT:    mul a0, a0, a2
+; CHECK-NEXT:    slli a0, a0, 5
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vnot.v v16, v8, v0.t
 ; CHECK-NEXT:    vand.vx v16, v16, a1, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
+; CHECK-NEXT:    li a1, 24
+; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vsll.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vsll.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 24
-; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vor.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 40
-; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 56
+; CHECK-NEXT:    li a1, 40
 ; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add sp, sp, a0
 ; CHECK-NEXT:    addi sp, sp, 16
@@ -1147,109 +1110,89 @@ define <vscale x 16 x i64> @fshl_v16i64(<vscale x 16 x i64> %a, <vscale x 16 x i
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    li a3, 56
+; CHECK-NEXT:    li a3, 40
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x38, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 56 * vlenb
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x28, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 40 * vlenb
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    slli a1, a1, 5
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 5
+; CHECK-NEXT:    li a3, 24
+; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a3, vlenb
-; CHECK-NEXT:    slli a1, a3, 3
-; CHECK-NEXT:    add a5, a0, a1
-; CHECK-NEXT:    vl8re64.v v8, (a5)
-; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    slli a5, a5, 4
-; CHECK-NEXT:    add a5, sp, a5
-; CHECK-NEXT:    addi a5, a5, 16
-; CHECK-NEXT:    vs8r.v v8, (a5) # Unknown-size Folded Spill
-; CHECK-NEXT:    srli a5, a3, 3
+; CHECK-NEXT:    slli a5, a3, 3
+; CHECK-NEXT:    srli a1, a3, 3
 ; CHECK-NEXT:    vsetvli a6, zero, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v0, v0, a5
-; CHECK-NEXT:    add a5, a2, a1
+; CHECK-NEXT:    vslidedown.vx v0, v0, a1
+; CHECK-NEXT:    add a1, a2, a5
+; CHECK-NEXT:    vl8re64.v v8, (a1)
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 4
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    sub a1, a4, a3
 ; CHECK-NEXT:    sltu a6, a4, a1
 ; CHECK-NEXT:    addi a6, a6, -1
 ; CHECK-NEXT:    and a6, a6, a1
 ; CHECK-NEXT:    li a1, 63
-; CHECK-NEXT:    vl8re64.v v8, (a5)
-; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    li a7, 40
-; CHECK-NEXT:    mul a5, a5, a7
-; CHECK-NEXT:    add a5, sp, a5
-; CHECK-NEXT:    addi a5, a5, 16
-; CHECK-NEXT:    vs8r.v v8, (a5) # Unknown-size Folded Spill
-; CHECK-NEXT:    vl8re64.v v16, (a0)
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a5, 24
-; CHECK-NEXT:    mul a0, a0, a5
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vl8re64.v v16, (a2)
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 48
-; CHECK-NEXT:    mul a0, a0, a2
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; CHECK-NEXT:    vsetvli zero, a6, e64, m8, ta, ma
+; CHECK-NEXT:    vand.vx v8, v8, a1, v0.t
+; CHECK-NEXT:    csrr a6, vlenb
+; CHECK-NEXT:    slli a6, a6, 5
+; CHECK-NEXT:    add a6, sp, a6
+; CHECK-NEXT:    addi a6, a6, 16
+; CHECK-NEXT:    vl8r.v v16, (a6) # Unknown-size Folded Reload
+; CHECK-NEXT:    vsll.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    csrr a6, vlenb
+; CHECK-NEXT:    slli a6, a6, 3
+; CHECK-NEXT:    add a6, sp, a6
+; CHECK-NEXT:    addi a6, a6, 16
+; CHECK-NEXT:    vs8r.v v8, (a6) # Unknown-size Folded Spill
+; CHECK-NEXT:    add a5, a0, a5
+; CHECK-NEXT:    csrr a6, vlenb
+; CHECK-NEXT:    slli a6, a6, 4
+; CHECK-NEXT:    add a6, sp, a6
+; CHECK-NEXT:    addi a6, a6, 16
+; CHECK-NEXT:    vl8r.v v8, (a6) # Unknown-size Folded Reload
+; CHECK-NEXT:    vnot.v v8, v8, v0.t
+; CHECK-NEXT:    vl8re64.v v16, (a5)
+; CHECK-NEXT:    vand.vx v8, v8, a1, v0.t
+; CHECK-NEXT:    addi a5, sp, 16
+; CHECK-NEXT:    vs8r.v v8, (a5) # Unknown-size Folded Spill
+; CHECK-NEXT:    vl8re64.v v8, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 40
-; CHECK-NEXT:    mul a0, a0, a2
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vand.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vl8re64.v v8, (a2)
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    slli a0, a0, 5
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsrl.vi v16, v16, 1, v0.t
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vsll.vv v16, v16, v8, v0.t
+; CHECK-NEXT:    vsrl.vv v16, v16, v8, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 40
-; CHECK-NEXT:    mul a0, a0, a2
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vnot.v v8, v8, v0.t
-; CHECK-NEXT:    vand.vx v16, v8, a1, v0.t
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vsrl.vi v8, v8, 1, v0.t
-; CHECK-NEXT:    vsrl.vv v8, v8, v16, v0.t
+; CHECK-NEXT:    vor.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vor.vv v8, v16, v8, v0.t
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 40
-; CHECK-NEXT:    mul a0, a0, a2
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; CHECK-NEXT:    bltu a4, a3, .LBB47_2
 ; CHECK-NEXT:  # %bb.1:
@@ -1258,63 +1201,56 @@ define <vscale x 16 x i64> @fshl_v16i64(<vscale x 16 x i64> %a, <vscale x 16 x i
 ; CHECK-NEXT:    vsetvli zero, a4, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 48
-; CHECK-NEXT:    mul a0, a0, a2
+; CHECK-NEXT:    slli a0, a0, 5
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vand.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
+; CHECK-NEXT:    li a2, 24
+; CHECK-NEXT:    mul a0, a0, a2
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vsll.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    li a2, 24
+; CHECK-NEXT:    mul a0, a0, a2
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vsll.vv v16, v16, v8, v0.t
+; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 5
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 48
-; CHECK-NEXT:    mul a0, a0, a2
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vnot.v v16, v8, v0.t
-; CHECK-NEXT:    vand.vx v16, v16, a1, v0.t
+; CHECK-NEXT:    vnot.v v8, v8, v0.t
+; CHECK-NEXT:    vand.vx v16, v8, a1, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 24
-; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vsrl.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vsrl.vv v8, v8, v16, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
+; CHECK-NEXT:    li a1, 24
+; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vor.vv v8, v16, v8, v0.t
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 40
-; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 56
+; CHECK-NEXT:    li a1, 40
 ; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add sp, sp, a0
 ; CHECK-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
index e12f1cf7603b..e260ae5344e4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
@@ -1216,35 +1216,20 @@ define void @mgather_nxv16i64(<vscale x 8 x ptr> %ptrs0, <vscale x 8 x ptr> %ptr
 ;
 ; RV64-LABEL: mgather_nxv16i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -16
-; RV64-NEXT:    .cfi_def_cfa_offset 16
-; RV64-NEXT:    csrr a3, vlenb
-; RV64-NEXT:    slli a3, a3, 3
-; RV64-NEXT:    sub sp, sp, a3
-; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
 ; RV64-NEXT:    vl8re64.v v24, (a0)
-; RV64-NEXT:    addi a0, sp, 16
-; RV64-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; RV64-NEXT:    vmv8r.v v16, v8
-; RV64-NEXT:    vl8re64.v v8, (a1)
 ; RV64-NEXT:    vsetvli a0, zero, e64, m8, ta, mu
-; RV64-NEXT:    vluxei64.v v24, (zero), v16, v0.t
+; RV64-NEXT:    vluxei64.v v24, (zero), v8, v0.t
+; RV64-NEXT:    vl8re64.v v8, (a1)
 ; RV64-NEXT:    csrr a0, vlenb
 ; RV64-NEXT:    srli a1, a0, 3
 ; RV64-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vslidedown.vx v0, v0, a1
 ; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, mu
-; RV64-NEXT:    addi a1, sp, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vluxei64.v v8, (zero), v16, v0.t
 ; RV64-NEXT:    slli a0, a0, 3
 ; RV64-NEXT:    add a0, a2, a0
 ; RV64-NEXT:    vs8r.v v8, (a0)
 ; RV64-NEXT:    vs8r.v v24, (a2)
-; RV64-NEXT:    csrr a0, vlenb
-; RV64-NEXT:    slli a0, a0, 3
-; RV64-NEXT:    add sp, sp, a0
-; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    ret
   %p0 = call <vscale x 16 x ptr> @llvm.vector.insert.nxv8p0.nxv16p0(<vscale x 16 x ptr> undef, <vscale x 8 x ptr> %ptrs0, i64 0)
   %p1 = call <vscale x 16 x ptr> @llvm.vector.insert.nxv8p0.nxv16p0(<vscale x 16 x ptr> %p0, <vscale x 8 x ptr> %ptrs1, i64 8)
diff --git a/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
index 0e09f59b6a20..fc8fdf4aaafe 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll
@@ -1714,8 +1714,8 @@ define void @mscatter_nxv16f64(<vscale x 8 x double> %val0, <vscale x 8 x double
 ; RV64-NEXT:    vl8re64.v v24, (a0)
 ; RV64-NEXT:    addi a0, sp, 16
 ; RV64-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; RV64-NEXT:    vl8re64.v v16, (a1)
 ; RV64-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; RV64-NEXT:    vl8re64.v v16, (a1)
 ; RV64-NEXT:    vsoxei64.v v8, (zero), v24, v0.t
 ; RV64-NEXT:    csrr a0, vlenb
 ; RV64-NEXT:    srli a0, a0, 3
diff --git a/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll
index 8bc233428265..ebe89817630d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll
@@ -23,9 +23,9 @@ define <vscale x 1 x half> @vp_nearbyint_nxv1f16(<vscale x 1 x half> %va, <vscal
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv1f16:
@@ -42,11 +42,11 @@ define <vscale x 1 x half> @vp_nearbyint_nxv1f16(<vscale x 1 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v9, v8, v9, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
+; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    ret
   %v = call <vscale x 1 x half> @llvm.vp.nearbyint.nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x i1> %m, i32 %evl)
   ret <vscale x 1 x half> %v
@@ -63,9 +63,9 @@ define <vscale x 1 x half> @vp_nearbyint_nxv1f16_unmasked(<vscale x 1 x half> %v
 ; ZVFH-NEXT:    frflags a0
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv1f16_unmasked:
@@ -80,11 +80,11 @@ define <vscale x 1 x half> @vp_nearbyint_nxv1f16_unmasked(<vscale x 1 x half> %v
 ; ZVFHMIN-NEXT:    frflags a0
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v9, v8, v9, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
+; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    ret
   %v = call <vscale x 1 x half> @llvm.vp.nearbyint.nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 1 x half> %v
@@ -105,9 +105,9 @@ define <vscale x 2 x half> @vp_nearbyint_nxv2f16(<vscale x 2 x half> %va, <vscal
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv2f16:
@@ -124,11 +124,11 @@ define <vscale x 2 x half> @vp_nearbyint_nxv2f16(<vscale x 2 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v9, v8, v9, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
+; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    ret
   %v = call <vscale x 2 x half> @llvm.vp.nearbyint.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x i1> %m, i32 %evl)
   ret <vscale x 2 x half> %v
@@ -145,9 +145,9 @@ define <vscale x 2 x half> @vp_nearbyint_nxv2f16_unmasked(<vscale x 2 x half> %v
 ; ZVFH-NEXT:    frflags a0
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv2f16_unmasked:
@@ -162,11 +162,11 @@ define <vscale x 2 x half> @vp_nearbyint_nxv2f16_unmasked(<vscale x 2 x half> %v
 ; ZVFHMIN-NEXT:    frflags a0
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v9, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v9, v8, v9, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v9
+; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    ret
   %v = call <vscale x 2 x half> @llvm.vp.nearbyint.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 2 x half> %v
@@ -187,9 +187,9 @@ define <vscale x 4 x half> @vp_nearbyint_nxv4f16(<vscale x 4 x half> %va, <vscal
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv4f16:
@@ -208,11 +208,11 @@ define <vscale x 4 x half> @vp_nearbyint_nxv4f16(<vscale x 4 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v9
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v12, v10, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v10, v12, v10, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
+; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    ret
   %v = call <vscale x 4 x half> @llvm.vp.nearbyint.nxv4f16(<vscale x 4 x half> %va, <vscale x 4 x i1> %m, i32 %evl)
   ret <vscale x 4 x half> %v
@@ -229,9 +229,9 @@ define <vscale x 4 x half> @vp_nearbyint_nxv4f16_unmasked(<vscale x 4 x half> %v
 ; ZVFH-NEXT:    frflags a0
 ; ZVFH-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv4f16_unmasked:
@@ -246,11 +246,11 @@ define <vscale x 4 x half> @vp_nearbyint_nxv4f16_unmasked(<vscale x 4 x half> %v
 ; ZVFHMIN-NEXT:    frflags a0
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v10, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v10, v8, v10, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v10
+; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    ret
   %v = call <vscale x 4 x half> @llvm.vp.nearbyint.nxv4f16(<vscale x 4 x half> %va, <vscale x 4 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 4 x half> %v
@@ -273,9 +273,9 @@ define <vscale x 8 x half> @vp_nearbyint_nxv8f16(<vscale x 8 x half> %va, <vscal
 ; ZVFH-NEXT:    vmv1r.v v0, v10
 ; ZVFH-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv8f16:
@@ -294,11 +294,11 @@ define <vscale x 8 x half> @vp_nearbyint_nxv8f16(<vscale x 8 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v10
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v16, v12, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v12, v16, v12, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
+; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    ret
   %v = call <vscale x 8 x half> @llvm.vp.nearbyint.nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x i1> %m, i32 %evl)
   ret <vscale x 8 x half> %v
@@ -315,9 +315,9 @@ define <vscale x 8 x half> @vp_nearbyint_nxv8f16_unmasked(<vscale x 8 x half> %v
 ; ZVFH-NEXT:    frflags a0
 ; ZVFH-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv8f16_unmasked:
@@ -332,11 +332,11 @@ define <vscale x 8 x half> @vp_nearbyint_nxv8f16_unmasked(<vscale x 8 x half> %v
 ; ZVFHMIN-NEXT:    frflags a0
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v12, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v12, v8, v12, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v12
+; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    ret
   %v = call <vscale x 8 x half> @llvm.vp.nearbyint.nxv8f16(<vscale x 8 x half> %va, <vscale x 8 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 8 x half> %v
@@ -359,9 +359,9 @@ define <vscale x 16 x half> @vp_nearbyint_nxv16f16(<vscale x 16 x half> %va, <vs
 ; ZVFH-NEXT:    vmv1r.v v0, v12
 ; ZVFH-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv16f16:
@@ -380,11 +380,11 @@ define <vscale x 16 x half> @vp_nearbyint_nxv16f16(<vscale x 16 x half> %va, <vs
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v12
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v24, v16, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v24, v24, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v16, v24, v16, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v16
+; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    ret
   %v = call <vscale x 16 x half> @llvm.vp.nearbyint.nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x i1> %m, i32 %evl)
   ret <vscale x 16 x half> %v
@@ -401,9 +401,9 @@ define <vscale x 16 x half> @vp_nearbyint_nxv16f16_unmasked(<vscale x 16 x half>
 ; ZVFH-NEXT:    frflags a0
 ; ZVFH-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv16f16_unmasked:
@@ -418,11 +418,11 @@ define <vscale x 16 x half> @vp_nearbyint_nxv16f16_unmasked(<vscale x 16 x half>
 ; ZVFHMIN-NEXT:    frflags a0
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v16, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v16, v8, v16, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v16
+; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    ret
   %v = call <vscale x 16 x half> @llvm.vp.nearbyint.nxv16f16(<vscale x 16 x half> %va, <vscale x 16 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 16 x half> %v
@@ -445,9 +445,9 @@ define <vscale x 32 x half> @vp_nearbyint_nxv32f16(<vscale x 32 x half> %va, <vs
 ; ZVFH-NEXT:    vmv1r.v v0, v16
 ; ZVFH-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v24, v24, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v24, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv32f16:
@@ -458,7 +458,7 @@ define <vscale x 32 x half> @vp_nearbyint_nxv32f16(<vscale x 32 x half> %va, <vs
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
 ; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; ZVFHMIN-NEXT:    vmv1r.v v7, v0
+; ZVFHMIN-NEXT:    vmv1r.v v16, v0
 ; ZVFHMIN-NEXT:    csrr a2, vlenb
 ; ZVFHMIN-NEXT:    slli a1, a2, 1
 ; ZVFHMIN-NEXT:    sub a3, a0, a1
@@ -488,29 +488,30 @@ define <vscale x 32 x half> @vp_nearbyint_nxv32f16(<vscale x 32 x half> %va, <vs
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v24, v8, v24, v0.t
 ; ZVFHMIN-NEXT:    vsetvli a2, zero, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfncvt.f.f.w v12, v24
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v20, v24
 ; ZVFHMIN-NEXT:    bltu a0, a1, .LBB10_2
 ; ZVFHMIN-NEXT:  # %bb.1:
 ; ZVFHMIN-NEXT:    mv a0, a1
 ; ZVFHMIN-NEXT:  .LBB10_2:
 ; ZVFHMIN-NEXT:    addi a1, sp, 16
-; ZVFHMIN-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v16
+; ZVFHMIN-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vmv1r.v v0, v7
-; ZVFHMIN-NEXT:    vfabs.v v16, v24, v0.t
+; ZVFHMIN-NEXT:    vmv1r.v v0, v16
+; ZVFHMIN-NEXT:    vfabs.v v8, v24, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
-; ZVFHMIN-NEXT:    vmflt.vf v7, v16, fa5, v0.t
+; ZVFHMIN-NEXT:    vmflt.vf v16, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    frflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vmv1r.v v0, v7
-; ZVFHMIN-NEXT:    vfcvt.x.f.v v16, v24, v0.t
-; ZVFHMIN-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
+; ZVFHMIN-NEXT:    vmv1r.v v0, v16
+; ZVFHMIN-NEXT:    vfcvt.x.f.v v8, v24, v0.t
+; ZVFHMIN-NEXT:    vfcvt.f.x.v v8, v8, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
-; ZVFHMIN-NEXT:    vfsgnj.vv v24, v16, v24, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v24
+; ZVFHMIN-NEXT:    vfsgnj.vv v24, v8, v24, v0.t
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vfncvt.f.f.w v16, v24
+; ZVFHMIN-NEXT:    fsflags a0
+; ZVFHMIN-NEXT:    vmv8r.v v8, v16
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    slli a0, a0, 3
 ; ZVFHMIN-NEXT:    add sp, sp, a0
@@ -531,9 +532,9 @@ define <vscale x 32 x half> @vp_nearbyint_nxv32f16_unmasked(<vscale x 32 x half>
 ; ZVFH-NEXT:    frflags a0
 ; ZVFH-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; ZVFH-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
 ; ZVFH-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; ZVFH-NEXT:    fsflags a0
 ; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN-LABEL: vp_nearbyint_nxv32f16_unmasked:
@@ -589,11 +590,11 @@ define <vscale x 32 x half> @vp_nearbyint_nxv32f16_unmasked(<vscale x 32 x half>
 ; ZVFHMIN-NEXT:    frflags a0
 ; ZVFHMIN-NEXT:    vfcvt.x.f.v v24, v16, v0.t
 ; ZVFHMIN-NEXT:    vfcvt.f.x.v v24, v24, v0.t
-; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; ZVFHMIN-NEXT:    vfsgnj.vv v16, v24, v16, v0.t
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfncvt.f.f.w v8, v16
+; ZVFHMIN-NEXT:    fsflags a0
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    slli a0, a0, 3
 ; ZVFHMIN-NEXT:    add sp, sp, a0
@@ -618,9 +619,9 @@ define <vscale x 1 x float> @vp_nearbyint_nxv1f32(<vscale x 1 x float> %va, <vsc
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x float> @llvm.vp.nearbyint.nxv1f32(<vscale x 1 x float> %va, <vscale x 1 x i1> %m, i32 %evl)
   ret <vscale x 1 x float> %v
@@ -637,9 +638,9 @@ define <vscale x 1 x float> @vp_nearbyint_nxv1f32_unmasked(<vscale x 1 x float>
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x float> @llvm.vp.nearbyint.nxv1f32(<vscale x 1 x float> %va, <vscale x 1 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 1 x float> %v
@@ -660,9 +661,9 @@ define <vscale x 2 x float> @vp_nearbyint_nxv2f32(<vscale x 2 x float> %va, <vsc
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x float> @llvm.vp.nearbyint.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x i1> %m, i32 %evl)
   ret <vscale x 2 x float> %v
@@ -679,9 +680,9 @@ define <vscale x 2 x float> @vp_nearbyint_nxv2f32_unmasked(<vscale x 2 x float>
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x float> @llvm.vp.nearbyint.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 2 x float> %v
@@ -704,9 +705,9 @@ define <vscale x 4 x float> @vp_nearbyint_nxv4f32(<vscale x 4 x float> %va, <vsc
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x float> @llvm.vp.nearbyint.nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x i1> %m, i32 %evl)
   ret <vscale x 4 x float> %v
@@ -723,9 +724,9 @@ define <vscale x 4 x float> @vp_nearbyint_nxv4f32_unmasked(<vscale x 4 x float>
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x float> @llvm.vp.nearbyint.nxv4f32(<vscale x 4 x float> %va, <vscale x 4 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 4 x float> %v
@@ -748,9 +749,9 @@ define <vscale x 8 x float> @vp_nearbyint_nxv8f32(<vscale x 8 x float> %va, <vsc
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x float> @llvm.vp.nearbyint.nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x i1> %m, i32 %evl)
   ret <vscale x 8 x float> %v
@@ -767,9 +768,9 @@ define <vscale x 8 x float> @vp_nearbyint_nxv8f32_unmasked(<vscale x 8 x float>
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x float> @llvm.vp.nearbyint.nxv8f32(<vscale x 8 x float> %va, <vscale x 8 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 8 x float> %v
@@ -792,9 +793,9 @@ define <vscale x 16 x float> @vp_nearbyint_nxv16f32(<vscale x 16 x float> %va, <
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v24, v24, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v24, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x float> @llvm.vp.nearbyint.nxv16f32(<vscale x 16 x float> %va, <vscale x 16 x i1> %m, i32 %evl)
   ret <vscale x 16 x float> %v
@@ -811,9 +812,9 @@ define <vscale x 16 x float> @vp_nearbyint_nxv16f32_unmasked(<vscale x 16 x floa
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x float> @llvm.vp.nearbyint.nxv16f32(<vscale x 16 x float> %va, <vscale x 16 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 16 x float> %v
@@ -834,9 +835,9 @@ define <vscale x 1 x double> @vp_nearbyint_nxv1f64(<vscale x 1 x double> %va, <v
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x double> @llvm.vp.nearbyint.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x i1> %m, i32 %evl)
   ret <vscale x 1 x double> %v
@@ -853,9 +854,9 @@ define <vscale x 1 x double> @vp_nearbyint_nxv1f64_unmasked(<vscale x 1 x double
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 1 x double> @llvm.vp.nearbyint.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 1 x double> %v
@@ -878,9 +879,9 @@ define <vscale x 2 x double> @vp_nearbyint_nxv2f64(<vscale x 2 x double> %va, <v
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x double> @llvm.vp.nearbyint.nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x i1> %m, i32 %evl)
   ret <vscale x 2 x double> %v
@@ -897,9 +898,9 @@ define <vscale x 2 x double> @vp_nearbyint_nxv2f64_unmasked(<vscale x 2 x double
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 2 x double> @llvm.vp.nearbyint.nxv2f64(<vscale x 2 x double> %va, <vscale x 2 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 2 x double> %v
@@ -922,9 +923,9 @@ define <vscale x 4 x double> @vp_nearbyint_nxv4f64(<vscale x 4 x double> %va, <v
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x double> @llvm.vp.nearbyint.nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x i1> %m, i32 %evl)
   ret <vscale x 4 x double> %v
@@ -941,9 +942,9 @@ define <vscale x 4 x double> @vp_nearbyint_nxv4f64_unmasked(<vscale x 4 x double
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x double> @llvm.vp.nearbyint.nxv4f64(<vscale x 4 x double> %va, <vscale x 4 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 4 x double> %v
@@ -966,9 +967,9 @@ define <vscale x 7 x double> @vp_nearbyint_nxv7f64(<vscale x 7 x double> %va, <v
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v24, v24, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v24, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 7 x double> @llvm.vp.nearbyint.nxv7f64(<vscale x 7 x double> %va, <vscale x 7 x i1> %m, i32 %evl)
   ret <vscale x 7 x double> %v
@@ -985,9 +986,9 @@ define <vscale x 7 x double> @vp_nearbyint_nxv7f64_unmasked(<vscale x 7 x double
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 7 x double> @llvm.vp.nearbyint.nxv7f64(<vscale x 7 x double> %va, <vscale x 7 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 7 x double> %v
@@ -1010,9 +1011,9 @@ define <vscale x 8 x double> @vp_nearbyint_nxv8f64(<vscale x 8 x double> %va, <v
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v24, v24, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v24, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x double> @llvm.vp.nearbyint.nxv8f64(<vscale x 8 x double> %va, <vscale x 8 x i1> %m, i32 %evl)
   ret <vscale x 8 x double> %v
@@ -1029,9 +1030,9 @@ define <vscale x 8 x double> @vp_nearbyint_nxv8f64_unmasked(<vscale x 8 x double
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 8 x double> @llvm.vp.nearbyint.nxv8f64(<vscale x 8 x double> %va, <vscale x 8 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 8 x double> %v
@@ -1046,16 +1047,15 @@ define <vscale x 16 x double> @vp_nearbyint_nxv16f64(<vscale x 16 x double> %va,
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 4
+; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    sub sp, sp, a1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
-; CHECK-NEXT:    vmv1r.v v24, v0
-; CHECK-NEXT:    addi a1, sp, 16
-; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; CHECK-NEXT:    vmv1r.v v7, v0
+; CHECK-NEXT:    vmv8r.v v24, v16
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 3
 ; CHECK-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v25, v0, a2
+; CHECK-NEXT:    vslidedown.vx v6, v0, a2
 ; CHECK-NEXT:    sub a2, a0, a1
 ; CHECK-NEXT:    sltu a3, a0, a2
 ; CHECK-NEXT:    addi a3, a3, -1
@@ -1063,60 +1063,41 @@ define <vscale x 16 x double> @vp_nearbyint_nxv16f64(<vscale x 16 x double> %va,
 ; CHECK-NEXT:    lui a3, %hi(.LCPI32_0)
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI32_0)(a3)
 ; CHECK-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; CHECK-NEXT:    vmv1r.v v0, v25
-; CHECK-NEXT:    vmv8r.v v8, v16
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 3
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; CHECK-NEXT:    vmv1r.v v0, v6
 ; CHECK-NEXT:    vfabs.v v16, v16, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; CHECK-NEXT:    vmflt.vf v25, v16, fa5, v0.t
+; CHECK-NEXT:    vmflt.vf v6, v16, fa5, v0.t
 ; CHECK-NEXT:    frflags a2
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vmv1r.v v0, v25
-; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
-; CHECK-NEXT:    vfcvt.f.x.v v8, v16, v0.t
+; CHECK-NEXT:    vmv1r.v v0, v6
+; CHECK-NEXT:    vfcvt.x.f.v v16, v24, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
 ; CHECK-NEXT:    fsflags a2
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 3
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; CHECK-NEXT:    vfsgnj.vv v16, v8, v16, v0.t
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 3
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; CHECK-NEXT:    vfsgnj.vv v24, v16, v24, v0.t
+; CHECK-NEXT:    addi a2, sp, 16
+; CHECK-NEXT:    vs8r.v v24, (a2) # Unknown-size Folded Spill
 ; CHECK-NEXT:    bltu a0, a1, .LBB32_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:  .LBB32_2:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmv1r.v v0, v7
 ; CHECK-NEXT:    vfabs.v v16, v8, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
-; CHECK-NEXT:    vmflt.vf v24, v16, fa5, v0.t
+; CHECK-NEXT:    vmflt.vf v7, v16, fa5, v0.t
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vmv1r.v v0, v24
+; CHECK-NEXT:    vmv1r.v v0, v7
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add sp, sp, a0
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
@@ -1153,9 +1134,9 @@ define <vscale x 16 x double> @vp_nearbyint_nxv16f64_unmasked(<vscale x 16 x dou
 ; CHECK-NEXT:    frflags a0
 ; CHECK-NEXT:    vfcvt.x.f.v v24, v8, v0.t
 ; CHECK-NEXT:    vfcvt.f.x.v v24, v24, v0.t
-; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
 ; CHECK-NEXT:    vfsgnj.vv v8, v24, v8, v0.t
+; CHECK-NEXT:    fsflags a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x double> @llvm.vp.nearbyint.nxv16f64(<vscale x 16 x double> %va, <vscale x 16 x i1> splat (i1 true), i32 %evl)
   ret <vscale x 16 x double> %v
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
index 897bfdea69f1..cc967396153b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
@@ -2203,17 +2203,17 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFH-NEXT:    add a1, sp, a1
 ; ZVFH-NEXT:    addi a1, a1, 16
 ; ZVFH-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; ZVFH-NEXT:    csrr a1, vlenb
-; ZVFH-NEXT:    vsetvli a3, zero, e8, m1, ta, ma
-; ZVFH-NEXT:    slli a3, a1, 3
-; ZVFH-NEXT:    add a3, a0, a3
-; ZVFH-NEXT:    vl8re16.v v8, (a3)
-; ZVFH-NEXT:    slli a3, a1, 2
+; ZVFH-NEXT:    csrr a3, vlenb
+; ZVFH-NEXT:    srli a1, a3, 1
+; ZVFH-NEXT:    slli a4, a3, 3
+; ZVFH-NEXT:    add a4, a0, a4
+; ZVFH-NEXT:    vl8re16.v v8, (a4)
+; ZVFH-NEXT:    slli a3, a3, 2
 ; ZVFH-NEXT:    sub a4, a2, a3
 ; ZVFH-NEXT:    sltu a5, a2, a4
 ; ZVFH-NEXT:    addi a5, a5, -1
 ; ZVFH-NEXT:    and a4, a5, a4
-; ZVFH-NEXT:    srli a1, a1, 1
+; ZVFH-NEXT:    vsetvli a5, zero, e8, m1, ta, ma
 ; ZVFH-NEXT:    vl8re16.v v0, (a0)
 ; ZVFH-NEXT:    addi a0, sp, 16
 ; ZVFH-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
@@ -2249,152 +2249,133 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    addi sp, sp, -16
 ; ZVFHMIN-NEXT:    .cfi_def_cfa_offset 16
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
-; ZVFHMIN-NEXT:    li a3, 34
-; ZVFHMIN-NEXT:    mul a1, a1, a3
+; ZVFHMIN-NEXT:    slli a1, a1, 5
 ; ZVFHMIN-NEXT:    sub sp, sp, a1
-; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x22, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 34 * vlenb
-; ZVFHMIN-NEXT:    vmv8r.v v24, v16
+; ZVFHMIN-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZVFHMIN-NEXT:    csrr a1, vlenb
+; ZVFHMIN-NEXT:    slli a1, a1, 4
+; ZVFHMIN-NEXT:    add a1, sp, a1
+; ZVFHMIN-NEXT:    addi a1, a1, 16
+; ZVFHMIN-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
-; ZVFHMIN-NEXT:    li a3, 18
-; ZVFHMIN-NEXT:    mul a1, a1, a3
+; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    add a1, sp, a1
 ; ZVFHMIN-NEXT:    addi a1, a1, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    csrr a3, vlenb
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; ZVFHMIN-NEXT:    slli a1, a3, 3
-; ZVFHMIN-NEXT:    add a1, a0, a1
-; ZVFHMIN-NEXT:    vl8re16.v v16, (a1)
+; ZVFHMIN-NEXT:    srli a1, a3, 1
+; ZVFHMIN-NEXT:    vsetvli a4, zero, e8, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv1r.v v17, v0
+; ZVFHMIN-NEXT:    vslidedown.vx v18, v0, a1
+; ZVFHMIN-NEXT:    slli a4, a3, 3
+; ZVFHMIN-NEXT:    add a4, a0, a4
+; ZVFHMIN-NEXT:    vl8re16.v v0, (a4)
 ; ZVFHMIN-NEXT:    slli a5, a3, 2
-; ZVFHMIN-NEXT:    sub a1, a2, a5
-; ZVFHMIN-NEXT:    sltu a4, a2, a1
-; ZVFHMIN-NEXT:    addi a4, a4, -1
-; ZVFHMIN-NEXT:    and a6, a4, a1
+; ZVFHMIN-NEXT:    sub a4, a2, a5
+; ZVFHMIN-NEXT:    sltu a6, a2, a4
+; ZVFHMIN-NEXT:    addi a6, a6, -1
+; ZVFHMIN-NEXT:    and a6, a6, a4
 ; ZVFHMIN-NEXT:    slli a4, a3, 1
-; ZVFHMIN-NEXT:    sub a1, a6, a4
-; ZVFHMIN-NEXT:    sltu a7, a6, a1
-; ZVFHMIN-NEXT:    addi a7, a7, -1
-; ZVFHMIN-NEXT:    and a7, a7, a1
-; ZVFHMIN-NEXT:    srli a1, a3, 1
-; ZVFHMIN-NEXT:    csrr t0, vlenb
-; ZVFHMIN-NEXT:    add t0, sp, t0
-; ZVFHMIN-NEXT:    addi t0, t0, 16
-; ZVFHMIN-NEXT:    vs1r.v v0, (t0) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vslidedown.vx v7, v0, a1
+; ZVFHMIN-NEXT:    sub a7, a6, a4
+; ZVFHMIN-NEXT:    sltu t0, a6, a7
+; ZVFHMIN-NEXT:    addi t0, t0, -1
+; ZVFHMIN-NEXT:    and a7, t0, a7
 ; ZVFHMIN-NEXT:    srli a3, a3, 2
+; ZVFHMIN-NEXT:    vsetvli t0, zero, e8, mf2, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vx v16, v18, a3
+; ZVFHMIN-NEXT:    vsetvli t0, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vl8re16.v v8, (a0)
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
-; ZVFHMIN-NEXT:    li t0, 26
+; ZVFHMIN-NEXT:    li t0, 24
 ; ZVFHMIN-NEXT:    mul a0, a0, t0
 ; ZVFHMIN-NEXT:    add a0, sp, a0
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    addi a0, sp, 16
-; ZVFHMIN-NEXT:    vs1r.v v7, (a0) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vslidedown.vx v0, v7, a3
-; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    csrr a0, vlenb
-; ZVFHMIN-NEXT:    slli a0, a0, 1
-; ZVFHMIN-NEXT:    add a0, sp, a0
-; ZVFHMIN-NEXT:    addi a0, a0, 16
-; ZVFHMIN-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v20
-; ZVFHMIN-NEXT:    vmv4r.v v16, v24
+; ZVFHMIN-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v4
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
-; ZVFHMIN-NEXT:    li t0, 10
-; ZVFHMIN-NEXT:    mul a0, a0, t0
+; ZVFHMIN-NEXT:    slli a0, a0, 4
 ; ZVFHMIN-NEXT:    add a0, sp, a0
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
-; ZVFHMIN-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v28
+; ZVFHMIN-NEXT:    vl8r.v v0, (a0) # Unknown-size Folded Reload
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v4
 ; ZVFHMIN-NEXT:    vsetvli zero, a7, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v26, v16, v8, v0.t
+; ZVFHMIN-NEXT:    vmv1r.v v0, v16
+; ZVFHMIN-NEXT:    vmfeq.vv v20, v8, v24, v0.t
 ; ZVFHMIN-NEXT:    bltu a6, a4, .LBB85_2
 ; ZVFHMIN-NEXT:  # %bb.1:
 ; ZVFHMIN-NEXT:    mv a6, a4
 ; ZVFHMIN-NEXT:  .LBB85_2:
 ; ZVFHMIN-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    addi a0, sp, 16
+; ZVFHMIN-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v24
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
-; ZVFHMIN-NEXT:    slli a0, a0, 1
-; ZVFHMIN-NEXT:    add a0, sp, a0
-; ZVFHMIN-NEXT:    addi a0, a0, 16
-; ZVFHMIN-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v16
-; ZVFHMIN-NEXT:    csrr a0, vlenb
-; ZVFHMIN-NEXT:    li a7, 10
-; ZVFHMIN-NEXT:    mul a0, a0, a7
+; ZVFHMIN-NEXT:    slli a0, a0, 4
 ; ZVFHMIN-NEXT:    add a0, sp, a0
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vl8r.v v0, (a0) # Unknown-size Folded Reload
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v0
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v0
 ; ZVFHMIN-NEXT:    vsetvli zero, a6, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    addi a0, sp, 16
-; ZVFHMIN-NEXT:    vl1r.v v0, (a0) # Unknown-size Folded Reload
-; ZVFHMIN-NEXT:    vmfeq.vv v6, v16, v8, v0.t
+; ZVFHMIN-NEXT:    vmv1r.v v0, v18
+; ZVFHMIN-NEXT:    vmfeq.vv v6, v24, v8, v0.t
 ; ZVFHMIN-NEXT:    add a0, a3, a3
 ; ZVFHMIN-NEXT:    bltu a2, a5, .LBB85_4
 ; ZVFHMIN-NEXT:  # %bb.3:
 ; ZVFHMIN-NEXT:    mv a2, a5
 ; ZVFHMIN-NEXT:  .LBB85_4:
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; ZVFHMIN-NEXT:    vslideup.vx v6, v26, a3
+; ZVFHMIN-NEXT:    vslideup.vx v6, v20, a3
 ; ZVFHMIN-NEXT:    sub a5, a2, a4
 ; ZVFHMIN-NEXT:    sltu a6, a2, a5
 ; ZVFHMIN-NEXT:    addi a6, a6, -1
 ; ZVFHMIN-NEXT:    and a5, a6, a5
 ; ZVFHMIN-NEXT:    vsetvli a6, zero, e8, mf2, ta, ma
-; ZVFHMIN-NEXT:    csrr a6, vlenb
-; ZVFHMIN-NEXT:    add a6, sp, a6
-; ZVFHMIN-NEXT:    addi a6, a6, 16
-; ZVFHMIN-NEXT:    vl1r.v v8, (a6) # Unknown-size Folded Reload
-; ZVFHMIN-NEXT:    vmv1r.v v7, v8
-; ZVFHMIN-NEXT:    vslidedown.vx v0, v8, a3
+; ZVFHMIN-NEXT:    vmv1r.v v7, v17
+; ZVFHMIN-NEXT:    vslidedown.vx v0, v17, a3
 ; ZVFHMIN-NEXT:    vsetvli a6, zero, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    csrr a6, vlenb
-; ZVFHMIN-NEXT:    li a7, 18
-; ZVFHMIN-NEXT:    mul a6, a6, a7
+; ZVFHMIN-NEXT:    slli a6, a6, 3
 ; ZVFHMIN-NEXT:    add a6, sp, a6
 ; ZVFHMIN-NEXT:    addi a6, a6, 16
-; ZVFHMIN-NEXT:    vl8r.v v24, (a6) # Unknown-size Folded Reload
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v28
+; ZVFHMIN-NEXT:    vl8r.v v16, (a6) # Unknown-size Folded Reload
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v20
 ; ZVFHMIN-NEXT:    csrr a6, vlenb
-; ZVFHMIN-NEXT:    li a7, 10
-; ZVFHMIN-NEXT:    mul a6, a6, a7
+; ZVFHMIN-NEXT:    slli a6, a6, 4
 ; ZVFHMIN-NEXT:    add a6, sp, a6
 ; ZVFHMIN-NEXT:    addi a6, a6, 16
 ; ZVFHMIN-NEXT:    vs8r.v v8, (a6) # Unknown-size Folded Spill
 ; ZVFHMIN-NEXT:    csrr a6, vlenb
-; ZVFHMIN-NEXT:    li a7, 26
+; ZVFHMIN-NEXT:    li a7, 24
 ; ZVFHMIN-NEXT:    mul a6, a6, a7
 ; ZVFHMIN-NEXT:    add a6, sp, a6
 ; ZVFHMIN-NEXT:    addi a6, a6, 16
-; ZVFHMIN-NEXT:    vl8r.v v16, (a6) # Unknown-size Folded Reload
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v20
+; ZVFHMIN-NEXT:    vl8r.v v24, (a6) # Unknown-size Folded Reload
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v8, v28
 ; ZVFHMIN-NEXT:    vsetvli zero, a5, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    csrr a5, vlenb
-; ZVFHMIN-NEXT:    li a6, 10
-; ZVFHMIN-NEXT:    mul a5, a5, a6
+; ZVFHMIN-NEXT:    slli a5, a5, 4
 ; ZVFHMIN-NEXT:    add a5, sp, a5
 ; ZVFHMIN-NEXT:    addi a5, a5, 16
-; ZVFHMIN-NEXT:    vl8r.v v16, (a5) # Unknown-size Folded Reload
-; ZVFHMIN-NEXT:    vmfeq.vv v5, v16, v8, v0.t
+; ZVFHMIN-NEXT:    vl8r.v v24, (a5) # Unknown-size Folded Reload
+; ZVFHMIN-NEXT:    vmfeq.vv v5, v24, v8, v0.t
 ; ZVFHMIN-NEXT:    bltu a2, a4, .LBB85_6
 ; ZVFHMIN-NEXT:  # %bb.5:
 ; ZVFHMIN-NEXT:    mv a2, a4
 ; ZVFHMIN-NEXT:  .LBB85_6:
 ; ZVFHMIN-NEXT:    vsetvli a4, zero, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v24
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v16
 ; ZVFHMIN-NEXT:    csrr a4, vlenb
-; ZVFHMIN-NEXT:    li a5, 26
+; ZVFHMIN-NEXT:    li a5, 24
 ; ZVFHMIN-NEXT:    mul a4, a4, a5
 ; ZVFHMIN-NEXT:    add a4, sp, a4
 ; ZVFHMIN-NEXT:    addi a4, a4, 16
 ; ZVFHMIN-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v8
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v8
 ; ZVFHMIN-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmv1r.v v0, v7
-; ZVFHMIN-NEXT:    vmfeq.vv v8, v16, v24, v0.t
+; ZVFHMIN-NEXT:    vmfeq.vv v8, v24, v16, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vslideup.vx v8, v5, a3
 ; ZVFHMIN-NEXT:    add a0, a1, a1
@@ -2402,8 +2383,7 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    vslideup.vx v8, v6, a1
 ; ZVFHMIN-NEXT:    vmv.v.v v0, v8
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
-; ZVFHMIN-NEXT:    li a1, 34
-; ZVFHMIN-NEXT:    mul a0, a0, a1
+; ZVFHMIN-NEXT:    slli a0, a0, 5
 ; ZVFHMIN-NEXT:    add sp, sp, a0
 ; ZVFHMIN-NEXT:    addi sp, sp, 16
 ; ZVFHMIN-NEXT:    ret
@@ -3494,109 +3474,93 @@ define <vscale x 32 x i1> @fcmp_oeq_vv_nxv32f64(<vscale x 32 x double> %va, <vsc
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    li a3, 48
+; CHECK-NEXT:    li a3, 24
 ; CHECK-NEXT:    mul a1, a1, a3
 ; CHECK-NEXT:    sub sp, sp, a1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 48 * vlenb
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
 ; CHECK-NEXT:    vmv1r.v v24, v0
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 5
+; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    li a3, 40
-; CHECK-NEXT:    mul a1, a1, a3
+; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a3, vlenb
 ; CHECK-NEXT:    li a1, 24
 ; CHECK-NEXT:    mul t2, a3, a1
-; CHECK-NEXT:    slli t1, a3, 3
+; CHECK-NEXT:    slli a7, a3, 3
 ; CHECK-NEXT:    srli a4, a3, 2
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v7, v0, a4
 ; CHECK-NEXT:    srli a1, a3, 3
-; CHECK-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
-; CHECK-NEXT:    add a5, a2, t1
-; CHECK-NEXT:    vl8re64.v v8, (a5)
 ; CHECK-NEXT:    slli t0, a3, 4
+; CHECK-NEXT:    add a5, a2, a7
+; CHECK-NEXT:    vl8re64.v v16, (a5)
+; CHECK-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    slli a5, a3, 1
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a1
-; CHECK-NEXT:    mv a7, a6
+; CHECK-NEXT:    mv t1, a6
 ; CHECK-NEXT:    bltu a6, a5, .LBB171_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a7, a5
+; CHECK-NEXT:    mv t1, a5
 ; CHECK-NEXT:  .LBB171_2:
 ; CHECK-NEXT:    add t2, a2, t2
-; CHECK-NEXT:    add t1, a0, t1
+; CHECK-NEXT:    add a7, a0, a7
 ; CHECK-NEXT:    add t0, a2, t0
-; CHECK-NEXT:    vl8re64.v v16, (a2)
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 4
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
-; CHECK-NEXT:    sub a2, a7, a3
-; CHECK-NEXT:    sltu t3, a7, a2
+; CHECK-NEXT:    vl8re64.v v8, (a2)
+; CHECK-NEXT:    addi a2, sp, 16
+; CHECK-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
+; CHECK-NEXT:    sub a2, t1, a3
+; CHECK-NEXT:    sltu t3, t1, a2
 ; CHECK-NEXT:    addi t3, t3, -1
 ; CHECK-NEXT:    and a2, t3, a2
 ; CHECK-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 5
+; CHECK-NEXT:    slli a2, a2, 3
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; CHECK-NEXT:    vmfeq.vv v6, v16, v8, v0.t
-; CHECK-NEXT:    bltu a7, a3, .LBB171_4
+; CHECK-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmfeq.vv v6, v8, v16, v0.t
+; CHECK-NEXT:    bltu t1, a3, .LBB171_4
 ; CHECK-NEXT:  # %bb.3:
-; CHECK-NEXT:    mv a7, a3
+; CHECK-NEXT:    mv t1, a3
 ; CHECK-NEXT:  .LBB171_4:
-; CHECK-NEXT:    vl8re64.v v8, (t2)
+; CHECK-NEXT:    vl8re64.v v16, (t2)
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 5
+; CHECK-NEXT:    slli a2, a2, 3
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
-; CHECK-NEXT:    vl8re64.v v8, (t1)
+; CHECK-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vslidedown.vx v18, v7, a1
+; CHECK-NEXT:    vsetvli zero, t1, e64, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    li t1, 24
-; CHECK-NEXT:    mul a2, a2, t1
+; CHECK-NEXT:    slli a2, a2, 4
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v18, v7, a1
-; CHECK-NEXT:    vl8re64.v v8, (t0)
+; CHECK-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
+; CHECK-NEXT:    addi a2, sp, 16
+; CHECK-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmfeq.vv v17, v24, v8, v0.t
+; CHECK-NEXT:    vl8re64.v v8, (a7)
 ; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 3
+; CHECK-NEXT:    slli a2, a2, 4
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
 ; CHECK-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
-; CHECK-NEXT:    vl8re64.v v8, (a0)
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli zero, a7, e64, m8, ta, ma
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 40
-; CHECK-NEXT:    mul a0, a0, a2
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 4
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT:    vmfeq.vv v17, v24, v8, v0.t
-; CHECK-NEXT:    add a0, a1, a1
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, tu, ma
-; CHECK-NEXT:    sub a0, a6, a5
-; CHECK-NEXT:    sltu a2, a6, a0
-; CHECK-NEXT:    addi a2, a2, -1
-; CHECK-NEXT:    and a0, a2, a0
+; CHECK-NEXT:    vl8re64.v v8, (t0)
+; CHECK-NEXT:    add a2, a1, a1
+; CHECK-NEXT:    vsetvli zero, a2, e8, mf2, tu, ma
+; CHECK-NEXT:    sub a2, a6, a5
+; CHECK-NEXT:    sltu a5, a6, a2
+; CHECK-NEXT:    vl8re64.v v24, (a0)
+; CHECK-NEXT:    addi a0, a5, -1
+; CHECK-NEXT:    and a0, a0, a2
 ; CHECK-NEXT:    vslideup.vx v17, v6, a1
 ; CHECK-NEXT:    mv a2, a0
 ; CHECK-NEXT:    bltu a0, a3, .LBB171_6
@@ -3605,13 +3569,6 @@ define <vscale x 32 x i1> @fcmp_oeq_vv_nxv32f64(<vscale x 32 x double> %va, <vsc
 ; CHECK-NEXT:  .LBB171_6:
 ; CHECK-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v7
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    slli a2, a2, 3
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
-; CHECK-NEXT:    addi a2, sp, 16
-; CHECK-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vmfeq.vv v16, v24, v8, v0.t
 ; CHECK-NEXT:    add a2, a4, a1
 ; CHECK-NEXT:    vsetvli zero, a2, e8, mf2, tu, ma
@@ -3623,13 +3580,12 @@ define <vscale x 32 x i1> @fcmp_oeq_vv_nxv32f64(<vscale x 32 x double> %va, <vsc
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v18
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 5
+; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a2, 24
-; CHECK-NEXT:    mul a0, a0, a2
+; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
@@ -3641,7 +3597,7 @@ define <vscale x 32 x i1> @fcmp_oeq_vv_nxv32f64(<vscale x 32 x double> %va, <vsc
 ; CHECK-NEXT:    vslideup.vx v17, v16, a0
 ; CHECK-NEXT:    vmv1r.v v0, v17
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    li a1, 48
+; CHECK-NEXT:    li a1, 24
 ; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    add sp, sp, a0
 ; CHECK-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
index 7fd77c050b29..85f5ffd784e9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
@@ -1092,7 +1092,7 @@ define <vscale x 128 x i1> @icmp_eq_vv_nxv128i8(<vscale x 128 x i8> %va, <vscale
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
-; CHECK-NEXT:    vmv1r.v v24, v0
+; CHECK-NEXT:    vmv1r.v v7, v0
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    add a1, sp, a1
@@ -1102,23 +1102,23 @@ define <vscale x 128 x i1> @icmp_eq_vv_nxv128i8(<vscale x 128 x i8> %va, <vscale
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    add a4, a0, a1
 ; CHECK-NEXT:    vl8r.v v8, (a4)
-; CHECK-NEXT:    vl8r.v v0, (a0)
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
-; CHECK-NEXT:    sub a0, a3, a1
+; CHECK-NEXT:    vsetvli a4, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vlm.v v0, (a2)
-; CHECK-NEXT:    sltu a2, a3, a0
-; CHECK-NEXT:    addi a2, a2, -1
-; CHECK-NEXT:    and a0, a2, a0
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
-; CHECK-NEXT:    vmseq.vv v7, v16, v8, v0.t
+; CHECK-NEXT:    sub a2, a3, a1
+; CHECK-NEXT:    sltu a4, a3, a2
+; CHECK-NEXT:    addi a4, a4, -1
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    and a2, a4, a2
+; CHECK-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; CHECK-NEXT:    vmseq.vv v6, v16, v8, v0.t
 ; CHECK-NEXT:    bltu a3, a1, .LBB96_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    mv a3, a1
 ; CHECK-NEXT:  .LBB96_2:
 ; CHECK-NEXT:    vsetvli zero, a3, e8, m8, ta, ma
-; CHECK-NEXT:    vmv1r.v v0, v24
+; CHECK-NEXT:    vmv1r.v v0, v7
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
@@ -1128,7 +1128,7 @@ define <vscale x 128 x i1> @icmp_eq_vv_nxv128i8(<vscale x 128 x i8> %va, <vscale
 ; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vmseq.vv v16, v8, v24, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v16
-; CHECK-NEXT:    vmv1r.v v8, v7
+; CHECK-NEXT:    vmv1r.v v8, v6
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add sp, sp, a0
@@ -2248,17 +2248,17 @@ define <vscale x 32 x i1> @icmp_eq_vv_nxv32i32(<vscale x 32 x i32> %va, <vscale
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    slli a3, a1, 3
-; CHECK-NEXT:    add a3, a0, a3
-; CHECK-NEXT:    vl8re32.v v8, (a3)
-; CHECK-NEXT:    slli a3, a1, 1
+; CHECK-NEXT:    csrr a3, vlenb
+; CHECK-NEXT:    srli a1, a3, 2
+; CHECK-NEXT:    slli a4, a3, 3
+; CHECK-NEXT:    add a4, a0, a4
+; CHECK-NEXT:    vl8re32.v v8, (a4)
+; CHECK-NEXT:    slli a3, a3, 1
 ; CHECK-NEXT:    sub a4, a2, a3
 ; CHECK-NEXT:    sltu a5, a2, a4
 ; CHECK-NEXT:    addi a5, a5, -1
 ; CHECK-NEXT:    and a4, a5, a4
-; CHECK-NEXT:    srli a1, a1, 2
+; CHECK-NEXT:    vsetvli a5, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vl8re32.v v0, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll
index a41c26211613..0f47236d6600 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfma-vp.ll
@@ -1125,22 +1125,22 @@ define <vscale x 16 x double> @vfma_vv_nxv16f64(<vscale x 16 x double> %va, <vsc
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
-; CHECK-NEXT:    slli a3, a1, 3
-; CHECK-NEXT:    add a5, a2, a3
+; CHECK-NEXT:    srli a3, a1, 3
+; CHECK-NEXT:    slli a5, a1, 3
+; CHECK-NEXT:    add a6, a2, a5
+; CHECK-NEXT:    vl8re64.v v8, (a6)
+; CHECK-NEXT:    csrr a6, vlenb
+; CHECK-NEXT:    slli a6, a6, 4
+; CHECK-NEXT:    add a6, sp, a6
+; CHECK-NEXT:    addi a6, a6, 16
+; CHECK-NEXT:    vs8r.v v8, (a6) # Unknown-size Folded Spill
+; CHECK-NEXT:    sub a6, a4, a1
+; CHECK-NEXT:    sltu a7, a4, a6
+; CHECK-NEXT:    addi a7, a7, -1
+; CHECK-NEXT:    and a6, a7, a6
+; CHECK-NEXT:    vsetvli a7, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    add a5, a0, a5
 ; CHECK-NEXT:    vl8re64.v v8, (a5)
-; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    slli a5, a5, 4
-; CHECK-NEXT:    add a5, sp, a5
-; CHECK-NEXT:    addi a5, a5, 16
-; CHECK-NEXT:    vs8r.v v8, (a5) # Unknown-size Folded Spill
-; CHECK-NEXT:    sub a5, a4, a1
-; CHECK-NEXT:    sltu a6, a4, a5
-; CHECK-NEXT:    addi a6, a6, -1
-; CHECK-NEXT:    and a5, a6, a5
-; CHECK-NEXT:    srli a6, a1, 3
-; CHECK-NEXT:    add a3, a0, a3
-; CHECK-NEXT:    vl8re64.v v8, (a3)
 ; CHECK-NEXT:    vl8re64.v v16, (a2)
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 3
@@ -1150,8 +1150,8 @@ define <vscale x 16 x double> @vfma_vv_nxv16f64(<vscale x 16 x double> %va, <vsc
 ; CHECK-NEXT:    vl8re64.v v16, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vslidedown.vx v0, v0, a6
-; CHECK-NEXT:    vsetvli zero, a5, e64, m8, ta, ma
+; CHECK-NEXT:    vslidedown.vx v0, v0, a3
+; CHECK-NEXT:    vsetvli zero, a6, e64, m8, ta, ma
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    li a2, 24
 ; CHECK-NEXT:    mul a0, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll
index 292f27794f37..bacf9bae83ed 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmuladd-vp.ll
@@ -1125,22 +1125,22 @@ define <vscale x 16 x double> @vfma_vv_nxv16f64(<vscale x 16 x double> %va, <vsc
 ; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
-; CHECK-NEXT:    slli a3, a1, 3
-; CHECK-NEXT:    add a5, a2, a3
+; CHECK-NEXT:    srli a3, a1, 3
+; CHECK-NEXT:    slli a5, a1, 3
+; CHECK-NEXT:    add a6, a2, a5
+; CHECK-NEXT:    vl8re64.v v8, (a6)
+; CHECK-NEXT:    csrr a6, vlenb
+; CHECK-NEXT:    slli a6, a6, 4
+; CHECK-NEXT:    add a6, sp, a6
+; CHECK-NEXT:    addi a6, a6, 16
+; CHECK-NEXT:    vs8r.v v8, (a6) # Unknown-size Folded Spill
+; CHECK-NEXT:    sub a6, a4, a1
+; CHECK-NEXT:    sltu a7, a4, a6
+; CHECK-NEXT:    addi a7, a7, -1
+; CHECK-NEXT:    and a6, a7, a6
+; CHECK-NEXT:    vsetvli a7, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    add a5, a0, a5
 ; CHECK-NEXT:    vl8re64.v v8, (a5)
-; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    slli a5, a5, 4
-; CHECK-NEXT:    add a5, sp, a5
-; CHECK-NEXT:    addi a5, a5, 16
-; CHECK-NEXT:    vs8r.v v8, (a5) # Unknown-size Folded Spill
-; CHECK-NEXT:    sub a5, a4, a1
-; CHECK-NEXT:    sltu a6, a4, a5
-; CHECK-NEXT:    addi a6, a6, -1
-; CHECK-NEXT:    and a5, a6, a5
-; CHECK-NEXT:    srli a6, a1, 3
-; CHECK-NEXT:    add a3, a0, a3
-; CHECK-NEXT:    vl8re64.v v8, (a3)
 ; CHECK-NEXT:    vl8re64.v v16, (a2)
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 3
@@ -1150,8 +1150,8 @@ define <vscale x 16 x double> @vfma_vv_nxv16f64(<vscale x 16 x double> %va, <vsc
 ; CHECK-NEXT:    vl8re64.v v16, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vslidedown.vx v0, v0, a6
-; CHECK-NEXT:    vsetvli zero, a5, e64, m8, ta, ma
+; CHECK-NEXT:    vslidedown.vx v0, v0, a3
+; CHECK-NEXT:    vsetvli zero, a6, e64, m8, ta, ma
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    li a2, 24
 ; CHECK-NEXT:    mul a0, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
index 76efdda15bf7..26f7c56f05ce 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpmerge-sdnode.ll
@@ -369,14 +369,14 @@ define <vscale x 128 x i8> @vpmerge_vv_nxv128i8(<vscale x 128 x i8> %va, <vscale
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    add a4, a0, a1
 ; CHECK-NEXT:    vl8r.v v16, (a4)
-; CHECK-NEXT:    vl8r.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
-; CHECK-NEXT:    sub a0, a3, a1
+; CHECK-NEXT:    vsetvli a4, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vlm.v v0, (a2)
-; CHECK-NEXT:    sltu a2, a3, a0
-; CHECK-NEXT:    addi a2, a2, -1
-; CHECK-NEXT:    and a0, a2, a0
-; CHECK-NEXT:    vsetvli zero, a0, e8, m8, tu, ma
+; CHECK-NEXT:    sub a2, a3, a1
+; CHECK-NEXT:    sltu a4, a3, a2
+; CHECK-NEXT:    addi a4, a4, -1
+; CHECK-NEXT:    vl8r.v v8, (a0)
+; CHECK-NEXT:    and a2, a4, a2
+; CHECK-NEXT:    vsetvli zero, a2, e8, m8, tu, ma
 ; CHECK-NEXT:    vmerge.vvm v16, v16, v24, v0
 ; CHECK-NEXT:    bltu a3, a1, .LBB28_2
 ; CHECK-NEXT:  # %bb.1:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
index 0a5e50160fbc..312378d39373 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
@@ -373,10 +373,10 @@ define <vscale x 32 x i32> @select_nxv32i32(<vscale x 32 x i1> %a, <vscale x 32
 ; CHECK-NEXT:    addi a5, a5, -1
 ; CHECK-NEXT:    and a4, a5, a4
 ; CHECK-NEXT:    srli a3, a3, 2
+; CHECK-NEXT:    vsetvli a5, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vl8re32.v v0, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v24, a3
 ; CHECK-NEXT:    vsetvli zero, a4, e32, m8, ta, ma
 ; CHECK-NEXT:    vmerge.vvm v16, v8, v16, v0
@@ -430,10 +430,10 @@ define <vscale x 32 x i32> @select_evl_nxv32i32(<vscale x 32 x i1> %a, <vscale x
 ; CHECK-NEXT:    addi a4, a4, -1
 ; CHECK-NEXT:    and a3, a4, a3
 ; CHECK-NEXT:    srli a4, a1, 2
+; CHECK-NEXT:    vsetvli a5, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vl8re32.v v0, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v24, a4
 ; CHECK-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
 ; CHECK-NEXT:    vmerge.vvm v16, v8, v16, v0
@@ -716,10 +716,10 @@ define <vscale x 16 x double> @select_nxv16f64(<vscale x 16 x i1> %a, <vscale x
 ; CHECK-NEXT:    addi a4, a4, -1
 ; CHECK-NEXT:    and a3, a4, a3
 ; CHECK-NEXT:    srli a4, a1, 3
+; CHECK-NEXT:    vsetvli a5, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vl8re64.v v0, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs8r.v v0, (a0) # Unknown-size Folded Spill
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v24, a4
 ; CHECK-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
 ; CHECK-NEXT:    vmerge.vvm v16, v8, v16, v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.ll
index 25aa3a7081a1..9d5ff00fd597 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.ll
@@ -242,7 +242,6 @@ define <vscale x 1 x double> @test6(i64 %avl, i8 zeroext %cond, <vscale x 1 x do
 ; CHECK-NEXT:    andi a1, a1, 2
 ; CHECK-NEXT:    beqz a1, .LBB5_4
 ; CHECK-NEXT:  .LBB5_2: # %if.then4
-; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
 ; CHECK-NEXT:    addi a0, a0, %lo(.LCPI5_0)
 ; CHECK-NEXT:    vlse64.v v9, (a0), zero
@@ -631,22 +630,22 @@ declare <vscale x 2 x i32> @llvm.riscv.vwadd.w.nxv2i32.nxv2i16(<vscale x 2 x i32
 define void @vlmax(i64 %N, ptr %c, ptr %a, ptr %b) {
 ; CHECK-LABEL: vlmax:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a6, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    blez a0, .LBB12_3
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
-; CHECK-NEXT:    li a5, 0
-; CHECK-NEXT:    slli a4, a6, 3
+; CHECK-NEXT:    li a4, 0
+; CHECK-NEXT:    vsetvli a6, zero, e64, m1, ta, ma
+; CHECK-NEXT:    slli a5, a6, 3
 ; CHECK-NEXT:  .LBB12_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vle64.v v8, (a2)
 ; CHECK-NEXT:    vle64.v v9, (a3)
 ; CHECK-NEXT:    vfadd.vv v8, v8, v9
 ; CHECK-NEXT:    vse64.v v8, (a1)
-; CHECK-NEXT:    add a5, a5, a6
-; CHECK-NEXT:    add a1, a1, a4
-; CHECK-NEXT:    add a3, a3, a4
-; CHECK-NEXT:    add a2, a2, a4
-; CHECK-NEXT:    blt a5, a0, .LBB12_2
+; CHECK-NEXT:    add a4, a4, a6
+; CHECK-NEXT:    add a1, a1, a5
+; CHECK-NEXT:    add a3, a3, a5
+; CHECK-NEXT:    add a2, a2, a5
+; CHECK-NEXT:    blt a4, a0, .LBB12_2
 ; CHECK-NEXT:  .LBB12_3: # %for.end
 ; CHECK-NEXT:    ret
 entry:
@@ -678,18 +677,18 @@ for.end:                                          ; preds = %for.body, %entry
 define void @vector_init_vlmax(i64 %N, ptr %c) {
 ; CHECK-LABEL: vector_init_vlmax:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a2, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    blez a0, .LBB13_3
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
-; CHECK-NEXT:    li a3, 0
-; CHECK-NEXT:    slli a4, a2, 3
+; CHECK-NEXT:    li a2, 0
+; CHECK-NEXT:    vsetvli a3, zero, e64, m1, ta, ma
+; CHECK-NEXT:    slli a4, a3, 3
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:  .LBB13_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vse64.v v8, (a1)
-; CHECK-NEXT:    add a3, a3, a2
+; CHECK-NEXT:    add a2, a2, a3
 ; CHECK-NEXT:    add a1, a1, a4
-; CHECK-NEXT:    blt a3, a0, .LBB13_2
+; CHECK-NEXT:    blt a2, a0, .LBB13_2
 ; CHECK-NEXT:  .LBB13_3: # %for.end
 ; CHECK-NEXT:    ret
 entry:
@@ -714,20 +713,20 @@ for.end:                                          ; preds = %for.body, %entry
 define void @vector_init_vsetvli_N(i64 %N, ptr %c) {
 ; CHECK-LABEL: vector_init_vsetvli_N:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a2, a0, e64, m1, ta, ma
 ; CHECK-NEXT:    blez a0, .LBB14_3
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
-; CHECK-NEXT:    li a3, 0
-; CHECK-NEXT:    slli a4, a2, 3
+; CHECK-NEXT:    li a2, 0
+; CHECK-NEXT:    vsetvli a3, a0, e64, m1, ta, ma
+; CHECK-NEXT:    slli a4, a3, 3
 ; CHECK-NEXT:    vsetvli a5, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:  .LBB14_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vsetvli zero, a2, e64, m1, ta, ma
+; CHECK-NEXT:    vsetvli zero, a3, e64, m1, ta, ma
 ; CHECK-NEXT:    vse64.v v8, (a1)
-; CHECK-NEXT:    add a3, a3, a2
+; CHECK-NEXT:    add a2, a2, a3
 ; CHECK-NEXT:    add a1, a1, a4
-; CHECK-NEXT:    blt a3, a0, .LBB14_2
+; CHECK-NEXT:    blt a2, a0, .LBB14_2
 ; CHECK-NEXT:  .LBB14_3: # %for.end
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-regression.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-regression.ll
index f658a2c6b24a..c3b19b59ec3d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-regression.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-regression.ll
@@ -11,9 +11,10 @@ define i32 @illegal_preserve_vl(<vscale x 2 x i32> %a, <vscale x 4 x i64> %x, pt
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vadd.vv v12, v12, v12
-; CHECK-NEXT:    vs4r.v v12, (a0)
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    vmv.x.s a1, v8
+; CHECK-NEXT:    vs4r.v v12, (a0)
+; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:    ret
   %index = add <vscale x 4 x i64> %x, %x
   store <vscale x 4 x i64> %index, ptr %y
-- 
GitLab


From 74218a9c8fc4b0bdb4b2a4839455cf2f211a2a30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Wed, 15 May 2024 15:43:28 +0200
Subject: [PATCH 0875/1206] [clang][Interp] Implement __builtin_convertvector

---
 clang/lib/AST/Interp/ByteCodeExprGen.cpp    |  33 +++
 clang/lib/AST/Interp/ByteCodeExprGen.h      |   1 +
 clang/test/AST/Interp/builtin-functions.cpp | 235 ++++++++++++++++++++
 3 files changed, 269 insertions(+)

diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 7cdc1c6d1947..205e53f02b1a 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -2498,6 +2498,39 @@ bool ByteCodeExprGen<Emitter>::VisitAddrLabelExpr(const AddrLabelExpr *E) {
   return this->emitGetLocal(PT_Ptr, Offset, E);
 }
 
+template <class Emitter>
+bool ByteCodeExprGen<Emitter>::VisitConvertVectorExpr(
+    const ConvertVectorExpr *E) {
+  assert(Initializing);
+  const auto *VT = E->getType()->castAs<VectorType>();
+  QualType ElemType = VT->getElementType();
+  PrimType ElemT = classifyPrim(ElemType);
+  const Expr *Src = E->getSrcExpr();
+  PrimType SrcElemT =
+      classifyPrim(Src->getType()->castAs<VectorType>()->getElementType());
+
+  unsigned SrcOffset = this->allocateLocalPrimitive(Src, PT_Ptr, true, false);
+  if (!this->visit(Src))
+    return false;
+  if (!this->emitSetLocal(PT_Ptr, SrcOffset, E))
+    return false;
+
+  for (unsigned I = 0; I != VT->getNumElements(); ++I) {
+    if (!this->emitGetLocal(PT_Ptr, SrcOffset, E))
+      return false;
+    if (!this->emitArrayElemPop(SrcElemT, I, E))
+      return false;
+    if (SrcElemT != ElemT) {
+      if (!this->emitPrimCast(SrcElemT, ElemT, ElemType, E))
+        return false;
+    }
+    if (!this->emitInitElem(ElemT, I, E))
+      return false;
+  }
+
+  return true;
+}
+
 template <class Emitter> bool ByteCodeExprGen<Emitter>::discard(const Expr *E) {
   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/true,
                              /*NewInitializing=*/false);
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.h b/clang/lib/AST/Interp/ByteCodeExprGen.h
index 6039a54d32a5..fba4e45b9aa2 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.h
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.h
@@ -123,6 +123,7 @@ public:
   bool VisitPackIndexingExpr(const PackIndexingExpr *E);
   bool VisitRecoveryExpr(const RecoveryExpr *E);
   bool VisitAddrLabelExpr(const AddrLabelExpr *E);
+  bool VisitConvertVectorExpr(const ConvertVectorExpr *E);
 
 protected:
   bool visitExpr(const Expr *E) override;
diff --git a/clang/test/AST/Interp/builtin-functions.cpp b/clang/test/AST/Interp/builtin-functions.cpp
index 0cbab1fcd91d..afdfd25527e4 100644
--- a/clang/test/AST/Interp/builtin-functions.cpp
+++ b/clang/test/AST/Interp/builtin-functions.cpp
@@ -639,3 +639,238 @@ void test7(void) {
 /// the actual implementation uses analyze_os_log::computeOSLogBufferLayout(), which
 /// is tested elsewhere.
 static_assert(__builtin_os_log_format_buffer_size("%{mask.xyz}s", "abc") != 0, "");
+
+/// Copied from test/Sema/constant_builtins_vector.cpp.
+/// Some tests are missing since we run this for multiple targets,
+/// some of which do not support _BitInt.
+#ifndef __AVR__
+namespace convertvector {
+  typedef _BitInt(128) BitInt128;
+
+  typedef double vector4double __attribute__((__vector_size__(32)));
+  typedef float vector4float __attribute__((__vector_size__(16)));
+  typedef long long vector4long __attribute__((__vector_size__(32)));
+  typedef int vector4int __attribute__((__vector_size__(16)));
+  typedef short vector4short __attribute__((__vector_size__(8)));
+  typedef char vector4char __attribute__((__vector_size__(4)));
+  typedef BitInt128 vector4BitInt128 __attribute__((__vector_size__(64)));
+  typedef double vector8double __attribute__((__vector_size__(64)));
+  typedef float vector8float __attribute__((__vector_size__(32)));
+  typedef long long vector8long __attribute__((__vector_size__(64)));
+  typedef int vector8int __attribute__((__vector_size__(32)));
+  typedef short vector8short __attribute__((__vector_size__(16)));
+  typedef char vector8char __attribute__((__vector_size__(8)));
+  typedef BitInt128 vector8BitInt128 __attribute__((__vector_size__(128)));
+
+  constexpr vector4double from_vector4double_to_vector4double_var =
+      __builtin_convertvector((vector4double){0, 1, 2, 3}, vector4double);
+  constexpr vector4float from_vector4double_to_vector4float_var =
+      __builtin_convertvector((vector4double){0, 1, 2, 3}, vector4float);
+  constexpr vector4long from_vector4double_to_vector4long_var =
+      __builtin_convertvector((vector4double){0, 1, 2, 3}, vector4long);
+  constexpr vector4int from_vector4double_to_vector4int_var =
+      __builtin_convertvector((vector4double){0, 1, 2, 3}, vector4int);
+  constexpr vector4short from_vector4double_to_vector4short_var =
+      __builtin_convertvector((vector4double){0, 1, 2, 3}, vector4short);
+  constexpr vector4char from_vector4double_to_vector4char_var =
+      __builtin_convertvector((vector4double){0, 1, 2, 3}, vector4char);
+  constexpr vector4BitInt128 from_vector4double_to_vector4BitInt128_var =
+      __builtin_convertvector((vector4double){0, 1, 2, 3}, vector4BitInt128);
+  constexpr vector4double from_vector4float_to_vector4double_var =
+      __builtin_convertvector((vector4float){0, 1, 2, 3}, vector4double);
+  constexpr vector4float from_vector4float_to_vector4float_var =
+      __builtin_convertvector((vector4float){0, 1, 2, 3}, vector4float);
+  constexpr vector4long from_vector4float_to_vector4long_var =
+      __builtin_convertvector((vector4float){0, 1, 2, 3}, vector4long);
+  constexpr vector4int from_vector4float_to_vector4int_var =
+      __builtin_convertvector((vector4float){0, 1, 2, 3}, vector4int);
+  constexpr vector4short from_vector4float_to_vector4short_var =
+      __builtin_convertvector((vector4float){0, 1, 2, 3}, vector4short);
+  constexpr vector4char from_vector4float_to_vector4char_var =
+      __builtin_convertvector((vector4float){0, 1, 2, 3}, vector4char);
+  constexpr vector4BitInt128 from_vector4float_to_vector4BitInt128_var =
+      __builtin_convertvector((vector4float){0, 1, 2, 3}, vector4BitInt128);
+  constexpr vector4double from_vector4long_to_vector4double_var =
+      __builtin_convertvector((vector4long){0, 1, 2, 3}, vector4double);
+  constexpr vector4float from_vector4long_to_vector4float_var =
+      __builtin_convertvector((vector4long){0, 1, 2, 3}, vector4float);
+  constexpr vector4long from_vector4long_to_vector4long_var =
+      __builtin_convertvector((vector4long){0, 1, 2, 3}, vector4long);
+  constexpr vector4int from_vector4long_to_vector4int_var =
+      __builtin_convertvector((vector4long){0, 1, 2, 3}, vector4int);
+  constexpr vector4short from_vector4long_to_vector4short_var =
+      __builtin_convertvector((vector4long){0, 1, 2, 3}, vector4short);
+  constexpr vector4char from_vector4long_to_vector4char_var =
+      __builtin_convertvector((vector4long){0, 1, 2, 3}, vector4char);
+  constexpr vector4BitInt128 from_vector4long_to_vector4BitInt128_var =
+      __builtin_convertvector((vector4long){0, 1, 2, 3}, vector4BitInt128);
+  constexpr vector4double from_vector4int_to_vector4double_var =
+      __builtin_convertvector((vector4int){0, 1, 2, 3}, vector4double);
+  constexpr vector4float from_vector4int_to_vector4float_var =
+      __builtin_convertvector((vector4int){0, 1, 2, 3}, vector4float);
+  constexpr vector4long from_vector4int_to_vector4long_var =
+      __builtin_convertvector((vector4int){0, 1, 2, 3}, vector4long);
+  constexpr vector4int from_vector4int_to_vector4int_var =
+      __builtin_convertvector((vector4int){0, 1, 2, 3}, vector4int);
+  constexpr vector4short from_vector4int_to_vector4short_var =
+      __builtin_convertvector((vector4int){0, 1, 2, 3}, vector4short);
+  constexpr vector4char from_vector4int_to_vector4char_var =
+      __builtin_convertvector((vector4int){0, 1, 2, 3}, vector4char);
+  constexpr vector4BitInt128 from_vector4int_to_vector4BitInt128_var =
+      __builtin_convertvector((vector4int){0, 1, 2, 3}, vector4BitInt128);
+  constexpr vector4double from_vector4short_to_vector4double_var =
+      __builtin_convertvector((vector4short){0, 1, 2, 3}, vector4double);
+  constexpr vector4float from_vector4short_to_vector4float_var =
+      __builtin_convertvector((vector4short){0, 1, 2, 3}, vector4float);
+  constexpr vector4long from_vector4short_to_vector4long_var =
+      __builtin_convertvector((vector4short){0, 1, 2, 3}, vector4long);
+  constexpr vector4int from_vector4short_to_vector4int_var =
+      __builtin_convertvector((vector4short){0, 1, 2, 3}, vector4int);
+  constexpr vector4short from_vector4short_to_vector4short_var =
+      __builtin_convertvector((vector4short){0, 1, 2, 3}, vector4short);
+  constexpr vector4char from_vector4short_to_vector4char_var =
+      __builtin_convertvector((vector4short){0, 1, 2, 3}, vector4char);
+  constexpr vector4BitInt128 from_vector4short_to_vector4BitInt128_var =
+      __builtin_convertvector((vector4short){0, 1, 2, 3}, vector4BitInt128);
+  constexpr vector4double from_vector4char_to_vector4double_var =
+      __builtin_convertvector((vector4char){0, 1, 2, 3}, vector4double);
+  constexpr vector4float from_vector4char_to_vector4float_var =
+      __builtin_convertvector((vector4char){0, 1, 2, 3}, vector4float);
+  constexpr vector4long from_vector4char_to_vector4long_var =
+      __builtin_convertvector((vector4char){0, 1, 2, 3}, vector4long);
+  constexpr vector4int from_vector4char_to_vector4int_var =
+      __builtin_convertvector((vector4char){0, 1, 2, 3}, vector4int);
+  constexpr vector4short from_vector4char_to_vector4short_var =
+      __builtin_convertvector((vector4char){0, 1, 2, 3}, vector4short);
+  constexpr vector4char from_vector4char_to_vector4char_var =
+      __builtin_convertvector((vector4char){0, 1, 2, 3}, vector4char);
+  constexpr vector8double from_vector8double_to_vector8double_var =
+      __builtin_convertvector((vector8double){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8double);
+  constexpr vector8float from_vector8double_to_vector8float_var =
+      __builtin_convertvector((vector8double){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8float);
+  constexpr vector8long from_vector8double_to_vector8long_var =
+      __builtin_convertvector((vector8double){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8long);
+  constexpr vector8int from_vector8double_to_vector8int_var =
+      __builtin_convertvector((vector8double){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8int);
+  constexpr vector8short from_vector8double_to_vector8short_var =
+      __builtin_convertvector((vector8double){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8short);
+  constexpr vector8char from_vector8double_to_vector8char_var =
+      __builtin_convertvector((vector8double){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8char);
+  constexpr vector8BitInt128 from_vector8double_to_vector8BitInt128_var =
+      __builtin_convertvector((vector8double){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8BitInt128);
+  constexpr vector8double from_vector8float_to_vector8double_var =
+      __builtin_convertvector((vector8float){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8double);
+  constexpr vector8float from_vector8float_to_vector8float_var =
+      __builtin_convertvector((vector8float){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8float);
+  constexpr vector8long from_vector8float_to_vector8long_var =
+      __builtin_convertvector((vector8float){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8long);
+  constexpr vector8int from_vector8float_to_vector8int_var =
+      __builtin_convertvector((vector8float){0, 1, 2, 3, 4, 5, 6, 7}, vector8int);
+  constexpr vector8short from_vector8float_to_vector8short_var =
+      __builtin_convertvector((vector8float){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8short);
+  constexpr vector8char from_vector8float_to_vector8char_var =
+      __builtin_convertvector((vector8float){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8char);
+  constexpr vector8BitInt128 from_vector8float_to_vector8BitInt128_var =
+      __builtin_convertvector((vector8float){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8BitInt128);
+  constexpr vector8double from_vector8long_to_vector8double_var =
+      __builtin_convertvector((vector8long){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8double);
+  constexpr vector8float from_vector8long_to_vector8float_var =
+      __builtin_convertvector((vector8long){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8float);
+  constexpr vector8long from_vector8long_to_vector8long_var =
+      __builtin_convertvector((vector8long){0, 1, 2, 3, 4, 5, 6, 7}, vector8long);
+  constexpr vector8int from_vector8long_to_vector8int_var =
+      __builtin_convertvector((vector8long){0, 1, 2, 3, 4, 5, 6, 7}, vector8int);
+  constexpr vector8short from_vector8long_to_vector8short_var =
+      __builtin_convertvector((vector8long){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8short);
+  constexpr vector8char from_vector8long_to_vector8char_var =
+      __builtin_convertvector((vector8long){0, 1, 2, 3, 4, 5, 6, 7}, vector8char);
+  constexpr vector8double from_vector8int_to_vector8double_var =
+      __builtin_convertvector((vector8int){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8double);
+  constexpr vector8float from_vector8int_to_vector8float_var =
+      __builtin_convertvector((vector8int){0, 1, 2, 3, 4, 5, 6, 7}, vector8float);
+  constexpr vector8long from_vector8int_to_vector8long_var =
+      __builtin_convertvector((vector8int){0, 1, 2, 3, 4, 5, 6, 7}, vector8long);
+  constexpr vector8int from_vector8int_to_vector8int_var =
+      __builtin_convertvector((vector8int){0, 1, 2, 3, 4, 5, 6, 7}, vector8int);
+  constexpr vector8short from_vector8int_to_vector8short_var =
+      __builtin_convertvector((vector8int){0, 1, 2, 3, 4, 5, 6, 7}, vector8short);
+  constexpr vector8char from_vector8int_to_vector8char_var =
+      __builtin_convertvector((vector8int){0, 1, 2, 3, 4, 5, 6, 7}, vector8char);
+  constexpr vector8double from_vector8short_to_vector8double_var =
+      __builtin_convertvector((vector8short){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8double);
+  constexpr vector8float from_vector8short_to_vector8float_var =
+      __builtin_convertvector((vector8short){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8float);
+  constexpr vector8long from_vector8short_to_vector8long_var =
+      __builtin_convertvector((vector8short){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8long);
+  constexpr vector8int from_vector8short_to_vector8int_var =
+      __builtin_convertvector((vector8short){0, 1, 2, 3, 4, 5, 6, 7}, vector8int);
+  constexpr vector8short from_vector8short_to_vector8short_var =
+      __builtin_convertvector((vector8short){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8short);
+  constexpr vector8char from_vector8short_to_vector8char_var =
+      __builtin_convertvector((vector8short){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8char);
+
+  constexpr vector8double from_vector8char_to_vector8double_var =
+      __builtin_convertvector((vector8char){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8double);
+  constexpr vector8float from_vector8char_to_vector8float_var =
+      __builtin_convertvector((vector8char){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8float);
+  constexpr vector8long from_vector8char_to_vector8long_var =
+      __builtin_convertvector((vector8char){0, 1, 2, 3, 4, 5, 6, 7}, vector8long);
+  constexpr vector8int from_vector8char_to_vector8int_var =
+      __builtin_convertvector((vector8char){0, 1, 2, 3, 4, 5, 6, 7}, vector8int);
+  constexpr vector8short from_vector8char_to_vector8short_var =
+      __builtin_convertvector((vector8char){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8short);
+  constexpr vector8char from_vector8char_to_vector8char_var =
+      __builtin_convertvector((vector8char){0, 1, 2, 3, 4, 5, 6, 7}, vector8char);
+  constexpr vector8double from_vector8BitInt128_to_vector8double_var =
+      __builtin_convertvector((vector8BitInt128){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8double);
+  constexpr vector8float from_vector8BitInt128_to_vector8float_var =
+      __builtin_convertvector((vector8BitInt128){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8float);
+  constexpr vector8long from_vector8BitInt128_to_vector8long_var =
+      __builtin_convertvector((vector8BitInt128){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8long);
+  constexpr vector8int from_vector8BitInt128_to_vector8int_var =
+      __builtin_convertvector((vector8BitInt128){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8int);
+  constexpr vector8short from_vector8BitInt128_to_vector8short_var =
+      __builtin_convertvector((vector8BitInt128){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8short);
+  constexpr vector8char from_vector8BitInt128_to_vector8char_var =
+      __builtin_convertvector((vector8BitInt128){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8char);
+  constexpr vector8BitInt128 from_vector8BitInt128_to_vector8BitInt128_var =
+      __builtin_convertvector((vector8BitInt128){0, 1, 2, 3, 4, 5, 6, 7},
+                              vector8BitInt128);
+  static_assert(from_vector8BitInt128_to_vector8BitInt128_var[0] == 0, ""); // ref-error {{not an integral constant expression}}
+  static_assert(from_vector8BitInt128_to_vector8BitInt128_var[1] == 1, ""); // ref-error {{not an integral constant expression}}
+  static_assert(from_vector8BitInt128_to_vector8BitInt128_var[2] == 2, ""); // ref-error {{not an integral constant expression}}
+  static_assert(from_vector8BitInt128_to_vector8BitInt128_var[3] == 3, ""); // ref-error {{not an integral constant expression}}
+  static_assert(from_vector8BitInt128_to_vector8BitInt128_var[4] == 4, ""); // ref-error {{not an integral constant expression}}
+}
+#endif
-- 
GitLab


From 3a8d176af519e4385652e762c615ace9b80ef045 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Wed, 15 May 2024 16:50:52 +0100
Subject: [PATCH 0876/1206] [utils][filecheck-lint] Add shebang (#92243)

---
 llvm/utils/filecheck_lint/filecheck_lint.py | 1 +
 1 file changed, 1 insertion(+)
 mode change 100644 => 100755 llvm/utils/filecheck_lint/filecheck_lint.py

diff --git a/llvm/utils/filecheck_lint/filecheck_lint.py b/llvm/utils/filecheck_lint/filecheck_lint.py
old mode 100644
new mode 100755
index dc054ab76a09..837846db8332
--- a/llvm/utils/filecheck_lint/filecheck_lint.py
+++ b/llvm/utils/filecheck_lint/filecheck_lint.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # ===----------------------------------------------------------------------===##
 #
 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-- 
GitLab


From ba3447601c435bb2b24ad9e3c8d146c578f00568 Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Wed, 15 May 2024 17:52:46 +0200
Subject: [PATCH 0877/1206] [clang-tidy] Fix crash in modernize-use-constraints
 (#92019)

Improved modernize-use-constraints check by fixing a crash that occurred
in some scenarios and excluded system headers from analysis.

Problem were with DependentNameTypeLoc having null type location as
getQualifierLoc().getTypeLoc().

Fixes #91872
---
 .../modernize/UseConstraintsCheck.cpp         |  4 +++
 clang-tools-extra/docs/ReleaseNotes.rst       |  4 +++
 .../checks/modernize/use-constraints.rst      |  4 +++
 .../checkers/modernize/use-constraints.cpp    | 32 +++++++++++++++++++
 4 files changed, 44 insertions(+)

diff --git a/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
index 1585925ee996..7a021fe14436 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
@@ -41,6 +41,8 @@ AST_MATCHER(FunctionDecl, hasOtherDeclarations) {
 void UseConstraintsCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       functionTemplateDecl(
+          // Skip external libraries included as system headers
+          unless(isExpansionInSystemHeader()),
           has(functionDecl(unless(hasOtherDeclarations()), isDefinition(),
                            hasReturnTypeLoc(typeLoc().bind("return")))
                   .bind("function")))
@@ -57,6 +59,8 @@ matchEnableIfSpecializationImplTypename(TypeLoc TheType) {
       return std::nullopt;
     }
     TheType = Dep.getQualifierLoc().getTypeLoc();
+    if (TheType.isNull())
+      return std::nullopt;
   }
 
   if (const auto SpecializationLoc =
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 19f830741295..b7ef5a860e8b 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -322,6 +322,10 @@ Changes in existing checks
   don't remove parentheses used in ``sizeof`` calls when they have array index
   accesses as arguments.
 
+- Improved :doc:`modernize-use-constraints
+  <clang-tidy/checks/modernize/use-constraints>` check by fixing a crash that
+  occurred in some scenarios and excluding system headers from analysis.
+
 - Improved :doc:`modernize-use-nullptr
   <clang-tidy/checks/modernize/use-nullptr>` check to include support for C23,
   which also has introduced the ``nullptr`` keyword.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-constraints.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-constraints.rst
index be62dd5823d5..a8b31b80e580 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-constraints.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-constraints.rst
@@ -68,3 +68,7 @@ The tool will replace the above code with,
   // The tool will not emit a diagnostic or attempt to replace the code.
   template <typename T, std::enable_if_t<T::some_trait, int> = 0>
   struct my_class {};
+
+.. note::
+
+  System headers are not analyzed by this check.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-constraints.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-constraints.cpp
index 3ec44be8a1c8..3bcd5cd74024 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-constraints.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-constraints.cpp
@@ -724,3 +724,35 @@ void not_last_param() {
 }
 
 } // namespace enable_if_trailing_type_parameter
+
+
+// Issue fixes:
+
+namespace PR91872 {
+
+enum expression_template_option { value1, value2 };
+
+template <typename T> struct number_category {
+  static const int value = 0;
+};
+
+constexpr int number_kind_complex = 1;
+
+template <typename T, expression_template_option ExpressionTemplates>
+struct number {
+  using type = T;
+};
+
+template <typename T> struct component_type {
+  using type = T;
+};
+
+template <class T, expression_template_option ExpressionTemplates>
+inline typename std::enable_if<
+    number_category<T>::value == number_kind_complex,
+    component_type<number<T, ExpressionTemplates>>>::type::type
+abs(const number<T, ExpressionTemplates> &v) {
+  return {};
+}
+
+}
-- 
GitLab


From 54c6ee922abbaea7d2f138a209f320c414c1657b Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Wed, 15 May 2024 17:53:03 +0200
Subject: [PATCH 0878/1206] [clang-tidy] Add AllowImplicitlyDeletedCopyOrMove
 option to cppcoreguidelines-special-member-functions (#71683)

Improved cppcoreguidelines-special-member-functions check with a new
option AllowImplicitlyDeletedCopyOrMove, which removes the requirement
for explicit copy or move special member functions when they are already
implicitly deleted.

Closes #62392
---
 .../SpecialMemberFunctionsCheck.cpp           | 70 ++++++++++++++-----
 .../SpecialMemberFunctionsCheck.h             |  7 +-
 clang-tools-extra/docs/ReleaseNotes.rst       |  6 ++
 .../special-member-functions.rst              | 28 ++++++--
 .../special-member-functions-relaxed.cpp      | 26 ++++++-
 5 files changed, 107 insertions(+), 30 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.cpp
index d2117c67a76d..ed76ac665049 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.cpp
@@ -25,7 +25,9 @@ SpecialMemberFunctionsCheck::SpecialMemberFunctionsCheck(
                                          "AllowMissingMoveFunctions", false)),
       AllowSoleDefaultDtor(Options.get("AllowSoleDefaultDtor", false)),
       AllowMissingMoveFunctionsWhenCopyIsDeleted(
-          Options.get("AllowMissingMoveFunctionsWhenCopyIsDeleted", false)) {}
+          Options.get("AllowMissingMoveFunctionsWhenCopyIsDeleted", false)),
+      AllowImplicitlyDeletedCopyOrMove(
+          Options.get("AllowImplicitlyDeletedCopyOrMove", false)) {}
 
 void SpecialMemberFunctionsCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
@@ -33,17 +35,34 @@ void SpecialMemberFunctionsCheck::storeOptions(
   Options.store(Opts, "AllowSoleDefaultDtor", AllowSoleDefaultDtor);
   Options.store(Opts, "AllowMissingMoveFunctionsWhenCopyIsDeleted",
                 AllowMissingMoveFunctionsWhenCopyIsDeleted);
+  Options.store(Opts, "AllowImplicitlyDeletedCopyOrMove",
+                AllowImplicitlyDeletedCopyOrMove);
+}
+
+std::optional<TraversalKind>
+SpecialMemberFunctionsCheck::getCheckTraversalKind() const {
+  return AllowImplicitlyDeletedCopyOrMove ? TK_AsIs
+                                          : TK_IgnoreUnlessSpelledInSource;
 }
 
 void SpecialMemberFunctionsCheck::registerMatchers(MatchFinder *Finder) {
+  auto IsNotImplicitOrDeleted = anyOf(unless(isImplicit()), isDeleted());
+
   Finder->addMatcher(
       cxxRecordDecl(
-          eachOf(has(cxxDestructorDecl().bind("dtor")),
-                 has(cxxConstructorDecl(isCopyConstructor()).bind("copy-ctor")),
-                 has(cxxMethodDecl(isCopyAssignmentOperator())
+          unless(isImplicit()),
+          eachOf(has(cxxDestructorDecl(unless(isImplicit())).bind("dtor")),
+                 has(cxxConstructorDecl(isCopyConstructor(),
+                                        IsNotImplicitOrDeleted)
+                         .bind("copy-ctor")),
+                 has(cxxMethodDecl(isCopyAssignmentOperator(),
+                                   IsNotImplicitOrDeleted)
                          .bind("copy-assign")),
-                 has(cxxConstructorDecl(isMoveConstructor()).bind("move-ctor")),
-                 has(cxxMethodDecl(isMoveAssignmentOperator())
+                 has(cxxConstructorDecl(isMoveConstructor(),
+                                        IsNotImplicitOrDeleted)
+                         .bind("move-ctor")),
+                 has(cxxMethodDecl(isMoveAssignmentOperator(),
+                                   IsNotImplicitOrDeleted)
                          .bind("move-assign"))))
           .bind("class-def"),
       this);
@@ -127,7 +146,8 @@ void SpecialMemberFunctionsCheck::check(
   for (const auto &KV : Matchers)
     if (const auto *MethodDecl =
             Result.Nodes.getNodeAs<CXXMethodDecl>(KV.first)) {
-      StoreMember({KV.second, MethodDecl->isDeleted()});
+      StoreMember(
+          {KV.second, MethodDecl->isDeleted(), MethodDecl->isImplicit()});
     }
 }
 
@@ -144,7 +164,13 @@ void SpecialMemberFunctionsCheck::checkForMissingMembers(
 
   auto HasMember = [&](SpecialMemberFunctionKind Kind) {
     return llvm::any_of(DefinedMembers, [Kind](const auto &Data) {
-      return Data.FunctionKind == Kind;
+      return Data.FunctionKind == Kind && !Data.IsImplicit;
+    });
+  };
+
+  auto HasImplicitDeletedMember = [&](SpecialMemberFunctionKind Kind) {
+    return llvm::any_of(DefinedMembers, [Kind](const auto &Data) {
+      return Data.FunctionKind == Kind && Data.IsImplicit && Data.IsDeleted;
     });
   };
 
@@ -154,9 +180,17 @@ void SpecialMemberFunctionsCheck::checkForMissingMembers(
     });
   };
 
-  auto RequireMember = [&](SpecialMemberFunctionKind Kind) {
-    if (!HasMember(Kind))
-      MissingMembers.push_back(Kind);
+  auto RequireMembers = [&](SpecialMemberFunctionKind Kind1,
+                            SpecialMemberFunctionKind Kind2) {
+    if (AllowImplicitlyDeletedCopyOrMove && HasImplicitDeletedMember(Kind1) &&
+        HasImplicitDeletedMember(Kind2))
+      return;
+
+    if (!HasMember(Kind1))
+      MissingMembers.push_back(Kind1);
+
+    if (!HasMember(Kind2))
+      MissingMembers.push_back(Kind2);
   };
 
   bool RequireThree =
@@ -180,8 +214,8 @@ void SpecialMemberFunctionsCheck::checkForMissingMembers(
         !HasMember(SpecialMemberFunctionKind::NonDefaultDestructor))
       MissingMembers.push_back(SpecialMemberFunctionKind::Destructor);
 
-    RequireMember(SpecialMemberFunctionKind::CopyConstructor);
-    RequireMember(SpecialMemberFunctionKind::CopyAssignment);
+    RequireMembers(SpecialMemberFunctionKind::CopyConstructor,
+                   SpecialMemberFunctionKind::CopyAssignment);
   }
 
   if (RequireFive &&
@@ -189,14 +223,16 @@ void SpecialMemberFunctionsCheck::checkForMissingMembers(
         (IsDeleted(SpecialMemberFunctionKind::CopyConstructor) &&
          IsDeleted(SpecialMemberFunctionKind::CopyAssignment)))) {
     assert(RequireThree);
-    RequireMember(SpecialMemberFunctionKind::MoveConstructor);
-    RequireMember(SpecialMemberFunctionKind::MoveAssignment);
+    RequireMembers(SpecialMemberFunctionKind::MoveConstructor,
+                   SpecialMemberFunctionKind::MoveAssignment);
   }
 
   if (!MissingMembers.empty()) {
     llvm::SmallVector<SpecialMemberFunctionKind, 5> DefinedMemberKinds;
-    llvm::transform(DefinedMembers, std::back_inserter(DefinedMemberKinds),
-                    [](const auto &Data) { return Data.FunctionKind; });
+    for (const auto &Data : DefinedMembers) {
+      if (!Data.IsImplicit)
+        DefinedMemberKinds.push_back(Data.FunctionKind);
+    }
     diag(ID.first, "class '%0' defines %1 but does not define %2")
         << ID.second << cppcoreguidelines::join(DefinedMemberKinds, " and ")
         << cppcoreguidelines::join(MissingMembers, " or ");
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h
index 6042f0fd6cb0..9ebc03ed2fa1 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/SpecialMemberFunctionsCheck.h
@@ -30,9 +30,8 @@ public:
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void onEndOfTranslationUnit() override;
-  std::optional<TraversalKind> getCheckTraversalKind() const override {
-    return TK_IgnoreUnlessSpelledInSource;
-  }
+  std::optional<TraversalKind> getCheckTraversalKind() const override;
+
   enum class SpecialMemberFunctionKind : uint8_t {
     Destructor,
     DefaultDestructor,
@@ -46,6 +45,7 @@ public:
   struct SpecialMemberFunctionData {
     SpecialMemberFunctionKind FunctionKind;
     bool IsDeleted;
+    bool IsImplicit = false;
 
     bool operator==(const SpecialMemberFunctionData &Other) const {
       return (Other.FunctionKind == FunctionKind) &&
@@ -67,6 +67,7 @@ private:
   const bool AllowMissingMoveFunctions;
   const bool AllowSoleDefaultDtor;
   const bool AllowMissingMoveFunctionsWhenCopyIsDeleted;
+  const bool AllowImplicitlyDeletedCopyOrMove;
   ClassDefiningSpecialMembersMap ClassWithSpecialMembers;
 };
 
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index b7ef5a860e8b..71734617bf7a 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -262,6 +262,12 @@ Changes in existing checks
   <clang-tidy/checks/cppcoreguidelines/use-default-member-init>`. Fixed
   incorrect hints when using list-initialization.
 
+- Improved :doc:`cppcoreguidelines-special-member-functions
+  <clang-tidy/checks/cppcoreguidelines/special-member-functions>` check with a
+  new option `AllowImplicitlyDeletedCopyOrMove`, which removes the requirement
+  for explicit copy or move special member functions when they are already
+  implicitly deleted.
+
 - Improved :doc:`google-build-namespaces
   <clang-tidy/checks/google/build-namespaces>` check by replacing the local
   option `HeaderFileExtensions` by the global option of the same name.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/special-member-functions.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/special-member-functions.rst
index 176956d6cb2b..20f898fdab93 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/special-member-functions.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/special-member-functions.rst
@@ -45,9 +45,10 @@ Options
 
 .. option:: AllowMissingMoveFunctions
 
-   When set to `true` (default is `false`), this check doesn't flag classes which define no move
-   operations at all. It still flags classes which define only one of either
-   move constructor or move assignment operator. With this option enabled, the following class won't be flagged:
+   When set to `true` (default is `false`), this check doesn't flag classes
+   which define no move operations at all. It still flags classes which define
+   only one of either move constructor or move assignment operator. With this
+   option enabled, the following class won't be flagged:
 
    .. code-block:: c++
 
@@ -59,10 +60,11 @@ Options
 
 .. option:: AllowMissingMoveFunctionsWhenCopyIsDeleted
 
-   When set to `true` (default is `false`), this check doesn't flag classes which define deleted copy
-   operations but don't define move operations. This flag is related to Google C++ Style Guide
-   https://google.github.io/styleguide/cppguide.html#Copyable_Movable_Types. With this option enabled, the
-   following class won't be flagged:
+   When set to `true` (default is `false`), this check doesn't flag classes
+   which define deleted copy operations but don't define move operations. This
+   flag is related to Google C++ Style Guide `Copyable and Movable Types
+   <https://google.github.io/styleguide/cppguide.html#Copyable_Movable_Types>`_.
+   With this option enabled, the following class won't be flagged:
 
    .. code-block:: c++
 
@@ -71,3 +73,15 @@ Options
        A& operator=(const A&) = delete;
        ~A();
      };
+
+.. option:: AllowImplicitlyDeletedCopyOrMove
+
+   When set to `true` (default is `false`), this check doesn't flag classes
+   which implicitly delete copy or move operations.
+   With this option enabled, the following class won't be flagged:
+
+   .. code-block:: c++
+
+      struct A : boost::noncopyable {
+        ~A() { std::cout << "dtor\n"; }
+      };
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/special-member-functions-relaxed.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/special-member-functions-relaxed.cpp
index 0c17f57968a9..26142ccc835f 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/special-member-functions-relaxed.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/special-member-functions-relaxed.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s cppcoreguidelines-special-member-functions %t -- -config="{CheckOptions: {cppcoreguidelines-special-member-functions.AllowMissingMoveFunctions: true, cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor: true}}" --
+// RUN: %check_clang_tidy %s cppcoreguidelines-special-member-functions %t -- -config="{CheckOptions: {cppcoreguidelines-special-member-functions.AllowMissingMoveFunctions: true, cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor: true, cppcoreguidelines-special-member-functions.AllowImplicitlyDeletedCopyOrMove: true}}" --
 
 // Don't warn on destructors without definitions, they might be defaulted in another TU.
 class DeclaresDestructor {
@@ -34,12 +34,13 @@ class DefinesCopyAssignment {
 class DefinesMoveConstructor {
   DefinesMoveConstructor(DefinesMoveConstructor &&);
 };
-// CHECK-MESSAGES: [[@LINE-3]]:7: warning: class 'DefinesMoveConstructor' defines a move constructor but does not define a destructor, a copy constructor, a copy assignment operator or a move assignment operator [cppcoreguidelines-special-member-functions]
+// CHECK-MESSAGES: [[@LINE-3]]:7: warning: class 'DefinesMoveConstructor' defines a move constructor but does not define a destructor or a move assignment operator [cppcoreguidelines-special-member-functions]
 
 class DefinesMoveAssignment {
   DefinesMoveAssignment &operator=(DefinesMoveAssignment &&);
 };
-// CHECK-MESSAGES: [[@LINE-3]]:7: warning: class 'DefinesMoveAssignment' defines a move assignment operator but does not define a destructor, a copy constructor, a copy assignment operator or a move constructor [cppcoreguidelines-special-member-functions]
+// CHECK-MESSAGES: [[@LINE-3]]:7: warning: class 'DefinesMoveAssignment' defines a move assignment operator but does not define a destructor or a move constructor [cppcoreguidelines-special-member-functions]
+
 class DefinesNothing {
 };
 
@@ -81,3 +82,22 @@ struct TemplateClass {
 // This should not cause problems.
 TemplateClass<int> InstantiationWithInt;
 TemplateClass<double> InstantiationWithDouble;
+
+struct NoCopy
+{
+  NoCopy() = default;
+  ~NoCopy() = default;
+
+  NoCopy(const NoCopy&) = delete;
+  NoCopy(NoCopy&&) = delete;
+
+  NoCopy& operator=(const NoCopy&) = delete;
+  NoCopy& operator=(NoCopy&&) = delete;
+};
+
+// CHECK-MESSAGES: [[@LINE+1]]:8: warning: class 'NonCopyable' defines a copy constructor but does not define a destructor or a copy assignment operator [cppcoreguidelines-special-member-functions]
+struct NonCopyable : NoCopy
+{
+  NonCopyable() = default;
+  NonCopyable(const NonCopyable&) = delete;
+};
-- 
GitLab


From 9bbefb7f600019c9d7025281132dd160729bfff2 Mon Sep 17 00:00:00 2001
From: Serge Pavlov <sepavloff@gmail.com>
Date: Wed, 15 May 2024 23:12:57 +0700
Subject: [PATCH 0879/1206] [clang] Store FPOptions earlier when parsing
 function (#92146)

After https://github.com/llvm/llvm-project/pull/85605 ([clang] Set
correct FPOptions if attribute 'optnone' presents) the current FP
options in Sema are saved during parsing function because Sema can
modify them if optnone is present. However they were saved too late, it
caused fails in some cases when precompiled headers are used. This patch
moves the storing earlier.
---
 clang/lib/Parse/Parser.cpp | 4 ++--
 clang/test/PCH/optnone.cpp | 6 ++++++
 2 files changed, 8 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/PCH/optnone.cpp

diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index adcbe5858bc7..869b9c6669c2 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -1439,6 +1439,8 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
     }
   }
 
+  Sema::FPFeaturesStateRAII SaveFPFeatures(Actions);
+
   // Tell the actions module that we have entered a function definition with the
   // specified Declarator for the function.
   SkipBodyInfo SkipBody;
@@ -1497,8 +1499,6 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
     return Actions.ActOnFinishFunctionBody(Res, nullptr, false);
   }
 
-  Sema::FPFeaturesStateRAII SaveFPFeatures(Actions);
-
   if (Tok.is(tok::kw_try))
     return ParseFunctionTryBlock(Res, BodyScope);
 
diff --git a/clang/test/PCH/optnone.cpp b/clang/test/PCH/optnone.cpp
new file mode 100644
index 000000000000..8351bd9de70d
--- /dev/null
+++ b/clang/test/PCH/optnone.cpp
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -emit-pch -x c++-header %s -o %t.pch
+// RUN: %clang_cc1 -emit-llvm -DMAIN -include-pch %t.pch %s -o /dev/null
+
+#ifndef MAIN
+__attribute__((optnone)) void foo() {}
+#endif
-- 
GitLab


From 03c53c69a367008da689f0d2940e2197eb4a955c Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Wed, 15 May 2024 09:18:39 -0700
Subject: [PATCH 0880/1206] [MC] Remove UseAssemblerInfoForParsing

Commit 6c0665e22174d474050e85ca367424f6e02476be
(https://reviews.llvm.org/D45164) enabled certain constant expression
evaluation for `MCObjectStreamer` at parse time (e.g. `.if` directives,
see llvm/test/MC/AsmParser/assembler-expressions.s).

`getUseAssemblerInfoForParsing` was added to make `clang -c` handling
inline assembly similar to `MCAsmStreamer` (e.g. `llvm-mc -filetype=asm`),
where such expression folding (related to
`AttemptToFoldSymbolOffsetDifference`) is unavailable.

I believe this is overly conservative. We can make some parse-time
expression folding work for `clang -c` even if `clang -S` would still
report an error, a MCAsmStreamer issue (we cannot print `.if`
directives) that should not restrict the functionality of
MCObjectStreamer.

```
% cat b.cc
asm(R"(
.pushsection .text,"ax"
.globl _start; _start: ret
.if . -_start == 1
  ret
.endif
.popsection
)");
% gcc -S b.cc && gcc -c b.cc
% clang -S -fno-integrated-as b.cc     # succeeded

% clang -c b.cc     # succeeded with this patch
% clang -S b.cc     # still failed
<inline asm>:4:5: error: expected absolute expression
    4 | .if . -_start == 1
      |     ^
1 error generated.
```

Close #62520
Link: https://discourse.llvm.org/t/rfc-clang-assembly-object-equivalence-for-files-with-inline-assembly/78841

Pull Request: https://github.com/llvm/llvm-project/pull/91082
---
 clang/tools/driver/cc1as_main.cpp                |  3 ---
 llvm/include/llvm/MC/MCStreamer.h                |  7 ++-----
 .../CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp   |  3 ---
 llvm/lib/MC/MCObjectStreamer.cpp                 |  9 +--------
 llvm/lib/MC/MCStreamer.cpp                       |  2 +-
 llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp      |  7 ++-----
 llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp        |  3 ---
 .../AsmParser/assembler-expressions-inlineasm.ll | 16 ++++++++++------
 llvm/tools/llvm-mc/llvm-mc.cpp                   |  3 ---
 llvm/tools/llvm-ml/llvm-ml.cpp                   |  3 ---
 10 files changed, 16 insertions(+), 40 deletions(-)

diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index 86afe22fac24..4eb753a7297a 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -576,9 +576,6 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
     Str.get()->emitZeros(1);
   }
 
-  // Assembly to object compilation should leverage assembly info.
-  Str->setUseAssemblerInfoForParsing(true);
-
   bool Failed = false;
 
   std::unique_ptr<MCAsmParser> Parser(
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 69867620e1bf..50986e6bde88 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -245,8 +245,6 @@ class MCStreamer {
   /// requires.
   unsigned NextWinCFIID = 0;
 
-  bool UseAssemblerInfoForParsing;
-
   /// Is the assembler allowed to insert padding automatically?  For
   /// correctness reasons, we sometimes need to ensure instructions aren't
   /// separated in unexpected ways.  At the moment, this feature is only
@@ -296,11 +294,10 @@ public:
 
   MCContext &getContext() const { return Context; }
 
+  // MCObjectStreamer has an MCAssembler and allows more expression folding at
+  // parse time.
   virtual MCAssembler *getAssemblerPtr() { return nullptr; }
 
-  void setUseAssemblerInfoForParsing(bool v) { UseAssemblerInfoForParsing = v; }
-  bool getUseAssemblerInfoForParsing() { return UseAssemblerInfoForParsing; }
-
   MCTargetStreamer *getTargetStreamer() {
     return TargetStreamer.get();
   }
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
index d0ef3e5a1939..08e3c208ba4d 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
@@ -102,9 +102,6 @@ void AsmPrinter::emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
   std::unique_ptr<MCAsmParser> Parser(
       createMCAsmParser(SrcMgr, OutContext, *OutStreamer, *MAI, BufNum));
 
-  // Do not use assembler-level information for parsing inline assembly.
-  OutStreamer->setUseAssemblerInfoForParsing(false);
-
   // We create a new MCInstrInfo here since we might be at the module level
   // and not have a MachineFunction to initialize the TargetInstrInfo from and
   // we only need MCInstrInfo for asm parsing. We create one unconditionally
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index d2da5d0d3f90..a9003a164b30 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -40,14 +40,7 @@ MCObjectStreamer::MCObjectStreamer(MCContext &Context,
 
 MCObjectStreamer::~MCObjectStreamer() = default;
 
-// AssemblerPtr is used for evaluation of expressions and causes
-// difference between asm and object outputs. Return nullptr to in
-// inline asm mode to limit divergence to assembly inputs.
-MCAssembler *MCObjectStreamer::getAssemblerPtr() {
-  if (getUseAssemblerInfoForParsing())
-    return Assembler.get();
-  return nullptr;
-}
+MCAssembler *MCObjectStreamer::getAssemblerPtr() { return Assembler.get(); }
 
 void MCObjectStreamer::addPendingLabel(MCSymbol* S) {
   MCSection *CurSection = getCurrentSectionOnly();
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index 176d55aa890b..199d865ea349 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -93,7 +93,7 @@ void MCTargetStreamer::emitAssignment(MCSymbol *Symbol, const MCExpr *Value) {}
 
 MCStreamer::MCStreamer(MCContext &Ctx)
     : Context(Ctx), CurrentWinFrameInfo(nullptr),
-      CurrentProcWinFrameInfoStartIndex(0), UseAssemblerInfoForParsing(false) {
+      CurrentProcWinFrameInfoStartIndex(0) {
   SectionStack.push_back(std::pair<MCSectionSubPair, MCSectionSubPair>());
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index b7388ed9e85a..bd48a5f80c82 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -517,12 +517,9 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
 
   DumpCodeInstEmitter = nullptr;
   if (STM.dumpCode()) {
-    // For -dumpcode, get the assembler out of the streamer, even if it does
-    // not really want to let us have it. This only works with -filetype=obj.
-    bool SaveFlag = OutStreamer->getUseAssemblerInfoForParsing();
-    OutStreamer->setUseAssemblerInfoForParsing(true);
+    // For -dumpcode, get the assembler out of the streamer. This only works
+    // with -filetype=obj.
     MCAssembler *Assembler = OutStreamer->getAssemblerPtr();
-    OutStreamer->setUseAssemblerInfoForParsing(SaveFlag);
     if (Assembler)
       DumpCodeInstEmitter = Assembler->getEmitterPtr();
   }
diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
index 2ebe5bdc4771..ad0158086044 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
@@ -114,12 +114,9 @@ void SPIRVAsmPrinter::emitEndOfAsmFile(Module &M) {
   // Bound is an approximation that accounts for the maximum used register
   // number and number of generated OpLabels
   unsigned Bound = 2 * (ST->getBound() + 1) + NLabels;
-  bool FlagToRestore = OutStreamer->getUseAssemblerInfoForParsing();
-  OutStreamer->setUseAssemblerInfoForParsing(true);
   if (MCAssembler *Asm = OutStreamer->getAssemblerPtr())
     Asm->setBuildVersion(static_cast<MachO::PlatformType>(0), Major, Minor,
                          Bound, VersionTuple(Major, Minor, 0, Bound));
-  OutStreamer->setUseAssemblerInfoForParsing(FlagToRestore);
 }
 
 void SPIRVAsmPrinter::emitFunctionHeader() {
diff --git a/llvm/test/MC/AsmParser/assembler-expressions-inlineasm.ll b/llvm/test/MC/AsmParser/assembler-expressions-inlineasm.ll
index 35f110f37e2f..9d9a38f5b5a5 100644
--- a/llvm/test/MC/AsmParser/assembler-expressions-inlineasm.ll
+++ b/llvm/test/MC/AsmParser/assembler-expressions-inlineasm.ll
@@ -1,13 +1,17 @@
-; RUN: not llc -mtriple x86_64-unknown-linux-gnu -o %t.s -filetype=asm %s 2>&1 | FileCheck %s
-; RUN: not llc -mtriple x86_64-unknown-linux-gnu -o %t.o -filetype=obj %s 2>&1 | FileCheck %s
-
-; Assembler-aware expression evaluation should be disabled in inline
-; assembly to prevent differences in behavior between object and
-; assembly output.
+; RUN: not llc -mtriple=x86_64 %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: llc -mtriple=x86_64 -no-integrated-as < %s | FileCheck %s --check-prefix=GAS
+; RUN: llc -mtriple=x86_64 -filetype=obj %s -o - | llvm-objdump -d - |  FileCheck %s --check-prefix=DISASM
 
+; GAS: nop;  .if . - foo==1; nop;.endif
 
 ; CHECK: <inline asm>:1:17: error: expected absolute expression
 
+; DISASM:      <main>:
+; DISASM-NEXT:   nop
+; DISASM-NEXT:   nop
+; DISASM-NEXT:   xorl %eax, %eax
+; DISASM-NEXT:   retq
+
 define i32 @main() local_unnamed_addr {
   tail call void asm sideeffect "foo: nop;  .if . - foo==1;  nop;.endif", "~{dirflag},~{fpsr},~{flags}"()
   ret i32 0
diff --git a/llvm/tools/llvm-mc/llvm-mc.cpp b/llvm/tools/llvm-mc/llvm-mc.cpp
index 807071a7b9a1..506e4f22ef8f 100644
--- a/llvm/tools/llvm-mc/llvm-mc.cpp
+++ b/llvm/tools/llvm-mc/llvm-mc.cpp
@@ -569,9 +569,6 @@ int main(int argc, char **argv) {
       Str->initSections(true, *STI);
   }
 
-  // Use Assembler information for parsing.
-  Str->setUseAssemblerInfoForParsing(true);
-
   int Res = 1;
   bool disassemble = false;
   switch (Action) {
diff --git a/llvm/tools/llvm-ml/llvm-ml.cpp b/llvm/tools/llvm-ml/llvm-ml.cpp
index 1cac576f54e7..f1f39af059aa 100644
--- a/llvm/tools/llvm-ml/llvm-ml.cpp
+++ b/llvm/tools/llvm-ml/llvm-ml.cpp
@@ -428,9 +428,6 @@ int llvm_ml_main(int Argc, char **Argv, const llvm::ToolContext &) {
     Str->emitAssignment(Feat00Sym, MCConstantExpr::create(Feat00Flags, Ctx));
   }
 
-  // Use Assembler information for parsing.
-  Str->setUseAssemblerInfoForParsing(true);
-
   int Res = 1;
   if (InputArgs.hasArg(OPT_as_lex)) {
     // -as-lex; Lex only, and output a stream of tokens
-- 
GitLab


From 141391ad2f22885342935442642c6c892f43e1ed Mon Sep 17 00:00:00 2001
From: Nuri Amari <nuri.amari99@gmail.com>
Date: Wed, 15 May 2024 09:21:02 -0700
Subject: [PATCH 0881/1206] [lld] Fix -ObjC load behavior with LTO (#92162)

When -ObjC is passed, the linker must force load any object files that
contain special sections that store Objective-C / Swift information that
is used at runtime.

This should work regadless if input files are bitcode or native, but it
was not working with bitcode. This is because the sections that identify
an object file that should be loaded were inconsistent when dealing with
a native file vs bitcode file. In particular, bitcode files were not
searched for `__TEXT,__swift` prefixed sections, while native files
were.

This means LLD wasn't loading certain bitcode files and forcing the user
to introduce --force-load to their linker invocation for that archive.

Co-authored-by: Nuri Amari <nuriamari@fb.com>
---
 lld/test/MachO/objc.s                     | 23 ++++++++++++++++++++---
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp |  3 ++-
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/lld/test/MachO/objc.s b/lld/test/MachO/objc.s
index e7074141f011..dbb9f1df2757 100644
--- a/lld/test/MachO/objc.s
+++ b/lld/test/MachO/objc.s
@@ -5,12 +5,14 @@
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/has-objc-category.s -o %t/has-objc-category.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/has-objc-symbol-and-category.s -o %t/has-objc-symbol-and-category.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/has-swift.s -o %t/has-swift.o
+# RUN: llvm-as %t/has-swift-ir-loaded.ll -o %t/has-swift-ir-loaded.o
+# RUN: llvm-as %t/has-swift-ir-not-loaded.ll -o %t/has-swift-ir-not-loaded.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/has-swift-proto.s -o %t/has-swift-proto.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/no-objc.s -o %t/no-objc.o
 ## Make sure we don't mis-parse a 32-bit file as 64-bit
 # RUN: llvm-mc -filetype=obj -triple=armv7-apple-watchos %t/no-objc.s -o %t/wrong-arch.o
-# RUN: llvm-ar rcs %t/libHasSomeObjC.a %t/no-objc.o %t/has-objc-symbol.o %t/has-objc-category.o %t/has-swift.o %t/has-swift-proto.o %t/wrong-arch.o
-# RUN: llvm-ar rcs %t/libHasSomeObjC2.a %t/no-objc.o %t/has-objc-symbol-and-category.o %t/has-swift.o %t/has-swift-proto.o %t/wrong-arch.o
+# RUN: llvm-ar rcs %t/libHasSomeObjC.a %t/no-objc.o %t/has-objc-symbol.o %t/has-objc-category.o %t/has-swift.o %t/has-swift-proto.o %t/has-swift-ir-loaded.o %t/has-swift-ir-not-loaded.o %t/wrong-arch.o
+# RUN: llvm-ar rcs %t/libHasSomeObjC2.a %t/no-objc.o %t/has-objc-symbol-and-category.o %t/has-swift.o %t/has-swift-proto.o %t/has-swift-ir-loaded.o %t/has-swift-ir-not-loaded.o %t/wrong-arch.o
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/test.s -o %t/test.o
 
@@ -20,7 +22,7 @@
 # RUN: %lld -lSystem %t/test.o -o %t/test -L%t -lHasSomeObjC2 -ObjC
 # RUN: llvm-objdump --section-headers --syms %t/test | FileCheck %s --check-prefix=OBJC
 
-# RUN: %no-fatal-warnings-lld -lSystem %t/test.o -o %t/test --start-lib %t/no-objc.o %t/has-objc-symbol.o %t/has-objc-category.o %t/has-swift.o %t/has-swift-proto.o %t/wrong-arch.o --end-lib -ObjC 2>&1 \
+# RUN: %no-fatal-warnings-lld -lSystem %t/test.o -o %t/test --start-lib %t/no-objc.o %t/has-objc-symbol.o %t/has-objc-category.o %t/has-swift.o %t/has-swift-proto.o %t/has-swift-ir-loaded.o %t/has-swift-ir-not-loaded.o %t/wrong-arch.o --end-lib -ObjC 2>&1 \
 # RUN:     | FileCheck -check-prefix=WARNING %s
 # RUN: llvm-objdump --section-headers --syms %t/test | FileCheck %s --check-prefix=OBJC
 
@@ -36,6 +38,7 @@
 # OBJC-NEXT:    4 has_objc_symbol {{.*}}      DATA
 # OBJC-EMPTY:
 # OBJC-NEXT:  SYMBOL TABLE:
+# OBJC-DAG:   g     O __TEXT,__swift _foo
 # OBJC-DAG:   g     F __TEXT,__text _main
 # OBJC-DAG:   g     F __TEXT,__text _OBJC_CLASS_$_MyObject
 # OBJC-DAG:   g     O __TEXT,__swift5_fieldmd $s7somelib4Blah_pMF
@@ -100,6 +103,20 @@ _has_dup:
 .section __TEXT,__swift
 .quad 0x1234
 
+#--- has-swift-ir-loaded.ll
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "x86_64-apple-darwin"
+
+@foo = global i64 1234, section "__TEXT,__swift"
+@llvm.used = appending global [1 x ptr] [ptr @foo]
+
+#--- has-swift-ir-not-loaded.ll
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "x86_64-apple-darwin"
+
+@bar = global i64 1234
+@llvm.used = appending global [1 x ptr] [ptr @bar]
+
 #--- has-swift-proto.s
 .section __TEXT,__swift5_fieldmd
 .globl $s7somelib4Blah_pMF
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 19a15209f8b6..e64051cf5386 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -306,7 +306,8 @@ static Expected<bool> hasObjCCategoryInModule(BitstreamCursor &Stream) {
         return error("Invalid section name record");
       // Check for the i386 and other (x86_64, ARM) conventions
       if (S.find("__DATA,__objc_catlist") != std::string::npos ||
-          S.find("__OBJC,__category") != std::string::npos)
+          S.find("__OBJC,__category") != std::string::npos ||
+          S.find("__TEXT,__swift") != std::string::npos)
         return true;
       break;
     }
-- 
GitLab


From f0e79db215ada7316b4d4046490ab715194a519a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 15 May 2024 08:58:07 -0700
Subject: [PATCH 0882/1206] [RISCV] Fix 80 columns in RISCVMatInt.cpp. NFC

---
 llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
index 0a304d4cb7d9..0a857eb96935 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
@@ -114,11 +114,13 @@ static void generateInstSeqImpl(int64_t Val, const MCSubtargetInfo &STI,
     ShiftAmount = llvm::countr_zero((uint64_t)Val);
     Val >>= ShiftAmount;
 
-    // If the remaining bits don't fit in 12 bits, we might be able to reduce the
-    // shift amount in order to use LUI which will zero the lower 12 bits.
+    // If the remaining bits don't fit in 12 bits, we might be able to reduce
+    // the // shift amount in order to use LUI which will zero the lower 12
+    // bits.
     if (ShiftAmount > 12 && !isInt<12>(Val)) {
       if (isInt<32>((uint64_t)Val << 12)) {
-        // Reduce the shift amount and add zeros to the LSBs so it will match LUI.
+        // Reduce the shift amount and add zeros to the LSBs so it will match
+        // LUI.
         ShiftAmount -= 12;
         Val = (uint64_t)Val << 12;
       } else if (isUInt<32>((uint64_t)Val << 12) &&
-- 
GitLab


From 29c2475f215110d9e6b3955d5eb2832b3f719c2f Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Wed, 15 May 2024 18:34:59 +0200
Subject: [PATCH 0883/1206] [mlir] Fix the build after
 03c53c69a367008da689f0d2940e2197eb4a955c

---
 mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp | 1 -
 mlir/lib/Target/LLVM/ROCDL/Target.cpp                | 1 -
 2 files changed, 2 deletions(-)

diff --git a/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp b/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp
index b07addc77b56..a4f19981eec3 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp
@@ -363,7 +363,6 @@ LogicalResult SerializeToHsacoPass::assembleIsa(const std::string &isa,
       mab->createObjectWriter(os), std::unique_ptr<llvm::MCCodeEmitter>(ce),
       *sti, mcOptions.MCRelaxAll, mcOptions.MCIncrementalLinkerCompatible,
       /*DWARFMustBeAtTheEnd*/ false));
-  mcStreamer->setUseAssemblerInfoForParsing(true);
 
   std::unique_ptr<llvm::MCAsmParser> parser(
       createMCAsmParser(srcMgr, ctx, *mcStreamer, *mai));
diff --git a/mlir/lib/Target/LLVM/ROCDL/Target.cpp b/mlir/lib/Target/LLVM/ROCDL/Target.cpp
index 66593fd8a55f..cc13e5b7436e 100644
--- a/mlir/lib/Target/LLVM/ROCDL/Target.cpp
+++ b/mlir/lib/Target/LLVM/ROCDL/Target.cpp
@@ -299,7 +299,6 @@ SerializeGPUModuleBase::assembleIsa(StringRef isa) {
       mab->createObjectWriter(os), std::unique_ptr<llvm::MCCodeEmitter>(ce),
       *sti, mcOptions.MCRelaxAll, mcOptions.MCIncrementalLinkerCompatible,
       /*DWARFMustBeAtTheEnd*/ false));
-  mcStreamer->setUseAssemblerInfoForParsing(true);
 
   std::unique_ptr<llvm::MCAsmParser> parser(
       createMCAsmParser(srcMgr, ctx, *mcStreamer, *mai));
-- 
GitLab


From 3f954f575156bce8ac81d6b4d94de443786befed Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Wed, 15 May 2024 12:33:54 -0400
Subject: [PATCH 0884/1206] Correct mismatched allocation/deallocation calls

This amends dceaa0f4491ebe30c0b0f1bc7fa5ec365b60ced6 because ASAN
caught an issue where the allocation and deallocation were not properly
paired: https://lab.llvm.org/buildbot/#/builders/239/builds/7001

Use malloc and free throughout this file to ensure that all kinds of
memory buffers use the proper pairing.
---
 llvm/lib/Support/MemoryBuffer.cpp | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Support/MemoryBuffer.cpp b/llvm/lib/Support/MemoryBuffer.cpp
index 50308bd2bf4a..fb7e804fd7e8 100644
--- a/llvm/lib/Support/MemoryBuffer.cpp
+++ b/llvm/lib/Support/MemoryBuffer.cpp
@@ -79,8 +79,16 @@ void *operator new(size_t N, const NamedBufferAlloc &Alloc) {
   SmallString<256> NameBuf;
   StringRef NameRef = Alloc.Name.toStringRef(NameBuf);
 
-  char *Mem = static_cast<char *>(operator new(N + sizeof(size_t) +
-                                               NameRef.size() + 1));
+  // We use malloc() and manually handle it returning null instead of calling
+  // operator new because we need all uses of NamedBufferAlloc to be
+  // deallocated with a call to free() due to needing to use malloc() in
+  // WritableMemoryBuffer::getNewUninitMemBuffer() to work around the out-of-
+  // memory handler installed by default in LLVM. See operator delete() member
+  // functions within this file for the paired call to free().
+  char *Mem =
+      static_cast<char *>(std::malloc(N + sizeof(size_t) + NameRef.size() + 1));
+  if (!Mem)
+    llvm::report_bad_alloc_error("Allocation failed");
   *reinterpret_cast<size_t *>(Mem + N) = NameRef.size();
   CopyStringRef(Mem + N + sizeof(size_t), NameRef);
   return Mem;
@@ -225,7 +233,7 @@ public:
 
   /// Disable sized deallocation for MemoryBufferMMapFile, because it has
   /// tail-allocated data.
-  void operator delete(void *p) { ::operator delete(p); }
+  void operator delete(void *p) { std::free(p); }
 
   StringRef getBufferIdentifier() const override {
     // The name is stored after the class itself.
-- 
GitLab


From 332de4b2677ce7a95cc2df30d761fbb55376fe07 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Wed, 15 May 2024 11:38:41 -0500
Subject: [PATCH 0885/1206] [Offload] Correctly reject building on unsupported
 architectures (#92276)

Summary:
Previously we had this `LIBOMPTARGET_ENABLED` variable which controlled
including `libomptarget`. This is now redundant since it's controlled by
`LLVM_ENABLE_RUNTIMES`. However, this had the extra effect of not
building it when given unsupported targets. THis was lost during the
move to `offload`. This patch moves this logic back and makes the
`offload` target just quit without doing anything if used on an
unsupported architecture.

https://github.com/llvm/llvm-project/issues/91881
https://github.com/llvm/llvm-project/issues/91819

---------

Co-authored-by: Sylvestre Ledru <sylvestre@debian.org>
---
 offload/CMakeLists.txt | 30 ++++++++++--------------------
 openmp/CMakeLists.txt  | 12 ------------
 2 files changed, 10 insertions(+), 32 deletions(-)

diff --git a/offload/CMakeLists.txt b/offload/CMakeLists.txt
index 626df8125063..1d8cab240924 100644
--- a/offload/CMakeLists.txt
+++ b/offload/CMakeLists.txt
@@ -17,26 +17,16 @@ if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   project(offload C CXX ASM)
 endif()
 
-set(ENABLE_LIBOMPTARGET ON)
-# Currently libomptarget cannot be compiled on Windows or MacOS X.
-# Since the device plugins are only supported on Linux anyway,
-# there is no point in trying to compile libomptarget on other OSes.
-# 32-bit systems are not supported either.
-if (APPLE OR WIN32 OR NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
-  set(ENABLE_LIBOMPTARGET OFF)
-endif()
-
-option(OPENMP_ENABLE_LIBOMPTARGET "Enable building libomptarget for offloading."
-       ${ENABLE_LIBOMPTARGET})
-if (OPENMP_ENABLE_LIBOMPTARGET)
-  # Check that the library can actually be built.
-  if (APPLE OR WIN32)
-    message(FATAL_ERROR "libomptarget cannot be built on Windows and MacOS X!")
-  elseif (NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES)
-    message(FATAL_ERROR "Host compiler must support C++17 to build libomptarget!")
-  elseif (NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
-    message(FATAL_ERROR "libomptarget on 32-bit systems are not supported!")
-  endif()
+# Check that the library can actually be built.
+if(APPLE OR WIN32 OR WASM)
+  message(WARNING "libomptarget cannot be built on Windows and MacOS X!")
+  return()
+elseif(NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES)
+  message(WARNING "Host compiler must support C++17 to build libomptarget!")
+  return()
+elseif(NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
+  message(WARNING "libomptarget on 32-bit systems is not supported!")
+  return()
 endif()
 
 if(OPENMP_STANDALONE_BUILD)
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index 9097ca562300..33bfdc8630ef 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -97,18 +97,6 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
-set(ENABLE_LIBOMPTARGET ON)
-# Currently libomptarget cannot be compiled on Windows or MacOS X.
-# Since the device plugins are only supported on Linux anyway,
-# there is no point in trying to compile libomptarget on other OSes.
-# 32-bit systems are not supported either.
-if (APPLE OR WIN32 OR WASM OR NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
-    OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8 OR ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
-  set(ENABLE_LIBOMPTARGET OFF)
-endif()
-
-option(OPENMP_ENABLE_LIBOMPTARGET "Enable building libomptarget for offloading."
-       ${ENABLE_LIBOMPTARGET})
 option(OPENMP_ENABLE_LIBOMP_PROFILING "Enable time profiling for libomp." OFF)
 
 # Header install location
-- 
GitLab


From be10746f3a4381456eb5082a968766201c17ab5d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 15 May 2024 12:43:55 -0400
Subject: [PATCH 0886/1206] [clang] Don't assume location of compiler-rt for
 OpenBSD (#92183)

If the `/usr/lib/...` path where compiler-rt is conventionally installed
on OpenBSD does not exist, fall back to the regular logic to find it.

This is a minimal change to allow OpenBSD cross compilation from a
toolchain that doesn't adopt all of OpenBSD's monorepo's conventions.
---
 clang/lib/Driver/ToolChains/OpenBSD.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Driver/ToolChains/OpenBSD.cpp b/clang/lib/Driver/ToolChains/OpenBSD.cpp
index e20d9fb1cfc4..3770471bae7c 100644
--- a/clang/lib/Driver/ToolChains/OpenBSD.cpp
+++ b/clang/lib/Driver/ToolChains/OpenBSD.cpp
@@ -375,7 +375,8 @@ std::string OpenBSD::getCompilerRT(const ArgList &Args, StringRef Component,
   if (Component == "builtins") {
     SmallString<128> Path(getDriver().SysRoot);
     llvm::sys::path::append(Path, "/usr/lib/libcompiler_rt.a");
-    return std::string(Path);
+    if (getVFS().exists(Path))
+      return std::string(Path);
   }
   SmallString<128> P(getDriver().ResourceDir);
   std::string CRTBasename =
-- 
GitLab


From e2d74a25eb562b117974add098ba2b9dd4cfc7f5 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 15 May 2024 17:46:49 +0100
Subject: [PATCH 0887/1206] [X86] EmitCmp - always use cmpw with foldable loads
 (#92251)

By default, EmitCmp avoids cmpw with i16 immediates due to 66/67h length-changing prefixes causing stalls, instead extending the value to i32 and using a cmpl with a i32 immediate, unless it has the TuningFastImm16 flag or we're building for optsize/minsize.

However, if we're loading the value for comparison, the performance costs of the decode stalls are likely to be exceeded by the impact of the load latency of the folded load, the shorter encoding and not needing an extra register to store the ext-load.

This matches the behaviour of gcc and msvc.

Fixes #90355
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       |   9 +-
 llvm/test/CodeGen/X86/cmp16.ll                | 196 ++++--------------
 .../CodeGen/X86/memcmp-more-load-pairs-x32.ll |   3 +-
 .../CodeGen/X86/memcmp-more-load-pairs.ll     |   3 +-
 llvm/test/CodeGen/X86/memcmp-optsize-x32.ll   |   3 +-
 llvm/test/CodeGen/X86/memcmp-optsize.ll       |   3 +-
 llvm/test/CodeGen/X86/memcmp-pgso-x32.ll      |   3 +-
 llvm/test/CodeGen/X86/memcmp-pgso.ll          |   3 +-
 llvm/test/CodeGen/X86/memcmp-x32.ll           |   3 +-
 llvm/test/CodeGen/X86/memcmp.ll               |   3 +-
 10 files changed, 60 insertions(+), 169 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index a57c10e784d9..e7c70e3872ad 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -22692,11 +22692,14 @@ static SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
           CmpVT == MVT::i32 || CmpVT == MVT::i64) && "Unexpected VT!");
 
   // Only promote the compare up to I32 if it is a 16 bit operation
-  // with an immediate.  16 bit immediates are to be avoided.
+  // with an immediate. 16 bit immediates are to be avoided unless the target
+  // isn't slowed down by length changing prefixes, we're optimizing for
+  // codesize or the comparison is with a folded load.
   if (CmpVT == MVT::i16 && !Subtarget.hasFastImm16() &&
+      !X86::mayFoldLoad(Op0, Subtarget) && !X86::mayFoldLoad(Op1, Subtarget) &&
       !DAG.getMachineFunction().getFunction().hasMinSize()) {
-    ConstantSDNode *COp0 = dyn_cast<ConstantSDNode>(Op0);
-    ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1);
+    auto *COp0 = dyn_cast<ConstantSDNode>(Op0);
+    auto *COp1 = dyn_cast<ConstantSDNode>(Op1);
     // Don't do this if the immediate can fit in 8-bits.
     if ((COp0 && !COp0->getAPIntValue().isSignedIntN(8)) ||
         (COp1 && !COp1->getAPIntValue().isSignedIntN(8))) {
diff --git a/llvm/test/CodeGen/X86/cmp16.ll b/llvm/test/CodeGen/X86/cmp16.ll
index 699ea3e4dd47..fa9e75ff16a5 100644
--- a/llvm/test/CodeGen/X86/cmp16.ll
+++ b/llvm/test/CodeGen/X86/cmp16.ll
@@ -113,8 +113,7 @@ define i1 @cmp16_reg_eq_imm8(i16 %a0) {
 define i1 @cmp16_reg_eq_imm16(i16 %a0) {
 ; X86-GENERIC-LABEL: cmp16_reg_eq_imm16:
 ; X86-GENERIC:       # %bb.0:
-; X86-GENERIC-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-GENERIC-NEXT:    cmpl $1024, %eax # imm = 0x400
+; X86-GENERIC-NEXT:    cmpw $1024, {{[0-9]+}}(%esp) # imm = 0x400
 ; X86-GENERIC-NEXT:    sete %al
 ; X86-GENERIC-NEXT:    retl
 ;
@@ -177,12 +176,11 @@ define i1 @cmp16_reg_eq_imm16_minsize(i16 %a0) minsize {
 }
 
 define i1 @cmp16_reg_eq_imm16_optsize(i16 %a0) optsize {
-; X86-GENERIC-LABEL: cmp16_reg_eq_imm16_optsize:
-; X86-GENERIC:       # %bb.0:
-; X86-GENERIC-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-GENERIC-NEXT:    cmpl $1024, %eax # imm = 0x400
-; X86-GENERIC-NEXT:    sete %al
-; X86-GENERIC-NEXT:    retl
+; X86-LABEL: cmp16_reg_eq_imm16_optsize:
+; X86:       # %bb.0:
+; X86-NEXT:    cmpw $1024, {{[0-9]+}}(%esp) # imm = 0x400
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
 ;
 ; X64-GENERIC-LABEL: cmp16_reg_eq_imm16_optsize:
 ; X64-GENERIC:       # %bb.0:
@@ -191,24 +189,12 @@ define i1 @cmp16_reg_eq_imm16_optsize(i16 %a0) optsize {
 ; X64-GENERIC-NEXT:    sete %al
 ; X64-GENERIC-NEXT:    retq
 ;
-; X86-FAST-LABEL: cmp16_reg_eq_imm16_optsize:
-; X86-FAST:       # %bb.0:
-; X86-FAST-NEXT:    cmpw $1024, {{[0-9]+}}(%esp) # imm = 0x400
-; X86-FAST-NEXT:    sete %al
-; X86-FAST-NEXT:    retl
-;
 ; X64-FAST-LABEL: cmp16_reg_eq_imm16_optsize:
 ; X64-FAST:       # %bb.0:
 ; X64-FAST-NEXT:    cmpw $1024, %di # imm = 0x400
 ; X64-FAST-NEXT:    sete %al
 ; X64-FAST-NEXT:    retq
 ;
-; X86-ATOM-LABEL: cmp16_reg_eq_imm16_optsize:
-; X86-ATOM:       # %bb.0:
-; X86-ATOM-NEXT:    cmpw $1024, {{[0-9]+}}(%esp) # imm = 0x400
-; X86-ATOM-NEXT:    sete %al
-; X86-ATOM-NEXT:    retl
-;
 ; X64-ATOM-LABEL: cmp16_reg_eq_imm16_optsize:
 ; X64-ATOM:       # %bb.0:
 ; X64-ATOM-NEXT:    cmpw $1024, %di # imm = 0x400
@@ -269,8 +255,7 @@ define i1 @cmp16_reg_sgt_imm8(i16 %a0) {
 define i1 @cmp16_reg_sgt_imm16(i16 %a0) {
 ; X86-GENERIC-LABEL: cmp16_reg_sgt_imm16:
 ; X86-GENERIC:       # %bb.0:
-; X86-GENERIC-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-GENERIC-NEXT:    cmpl $-1023, %eax # imm = 0xFC01
+; X86-GENERIC-NEXT:    cmpw $-1023, {{[0-9]+}}(%esp) # imm = 0xFC01
 ; X86-GENERIC-NEXT:    setge %al
 ; X86-GENERIC-NEXT:    retl
 ;
@@ -333,12 +318,11 @@ define i1 @cmp16_reg_sgt_imm16_minsize(i16 %a0) minsize {
 }
 
 define i1 @cmp16_reg_sgt_imm16_optsize(i16 %a0) optsize {
-; X86-GENERIC-LABEL: cmp16_reg_sgt_imm16_optsize:
-; X86-GENERIC:       # %bb.0:
-; X86-GENERIC-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-GENERIC-NEXT:    cmpl $-1023, %eax # imm = 0xFC01
-; X86-GENERIC-NEXT:    setge %al
-; X86-GENERIC-NEXT:    retl
+; X86-LABEL: cmp16_reg_sgt_imm16_optsize:
+; X86:       # %bb.0:
+; X86-NEXT:    cmpw $-1023, {{[0-9]+}}(%esp) # imm = 0xFC01
+; X86-NEXT:    setge %al
+; X86-NEXT:    retl
 ;
 ; X64-GENERIC-LABEL: cmp16_reg_sgt_imm16_optsize:
 ; X64-GENERIC:       # %bb.0:
@@ -347,24 +331,12 @@ define i1 @cmp16_reg_sgt_imm16_optsize(i16 %a0) optsize {
 ; X64-GENERIC-NEXT:    setge %al
 ; X64-GENERIC-NEXT:    retq
 ;
-; X86-FAST-LABEL: cmp16_reg_sgt_imm16_optsize:
-; X86-FAST:       # %bb.0:
-; X86-FAST-NEXT:    cmpw $-1023, {{[0-9]+}}(%esp) # imm = 0xFC01
-; X86-FAST-NEXT:    setge %al
-; X86-FAST-NEXT:    retl
-;
 ; X64-FAST-LABEL: cmp16_reg_sgt_imm16_optsize:
 ; X64-FAST:       # %bb.0:
 ; X64-FAST-NEXT:    cmpw $-1023, %di # imm = 0xFC01
 ; X64-FAST-NEXT:    setge %al
 ; X64-FAST-NEXT:    retq
 ;
-; X86-ATOM-LABEL: cmp16_reg_sgt_imm16_optsize:
-; X86-ATOM:       # %bb.0:
-; X86-ATOM-NEXT:    cmpw $-1023, {{[0-9]+}}(%esp) # imm = 0xFC01
-; X86-ATOM-NEXT:    setge %al
-; X86-ATOM-NEXT:    retl
-;
 ; X64-ATOM-LABEL: cmp16_reg_sgt_imm16_optsize:
 ; X64-ATOM:       # %bb.0:
 ; X64-ATOM-NEXT:    cmpw $-1023, %di # imm = 0xFC01
@@ -377,8 +349,7 @@ define i1 @cmp16_reg_sgt_imm16_optsize(i16 %a0) optsize {
 define i1 @cmp16_reg_uge_imm16(i16 %a0) {
 ; X86-GENERIC-LABEL: cmp16_reg_uge_imm16:
 ; X86-GENERIC:       # %bb.0:
-; X86-GENERIC-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-GENERIC-NEXT:    cmpl $64512, %eax # imm = 0xFC00
+; X86-GENERIC-NEXT:    cmpw $-1024, {{[0-9]+}}(%esp) # imm = 0xFC00
 ; X86-GENERIC-NEXT:    setae %al
 ; X86-GENERIC-NEXT:    retl
 ;
@@ -441,12 +412,11 @@ define i1 @cmp16_reg_uge_imm16_minsize(i16 %a0) minsize {
 }
 
 define i1 @cmp16_reg_uge_imm16_optsize(i16 %a0) optsize {
-; X86-GENERIC-LABEL: cmp16_reg_uge_imm16_optsize:
-; X86-GENERIC:       # %bb.0:
-; X86-GENERIC-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-GENERIC-NEXT:    cmpl $64512, %eax # imm = 0xFC00
-; X86-GENERIC-NEXT:    setae %al
-; X86-GENERIC-NEXT:    retl
+; X86-LABEL: cmp16_reg_uge_imm16_optsize:
+; X86:       # %bb.0:
+; X86-NEXT:    cmpw $-1024, {{[0-9]+}}(%esp) # imm = 0xFC00
+; X86-NEXT:    setae %al
+; X86-NEXT:    retl
 ;
 ; X64-GENERIC-LABEL: cmp16_reg_uge_imm16_optsize:
 ; X64-GENERIC:       # %bb.0:
@@ -455,24 +425,12 @@ define i1 @cmp16_reg_uge_imm16_optsize(i16 %a0) optsize {
 ; X64-GENERIC-NEXT:    setae %al
 ; X64-GENERIC-NEXT:    retq
 ;
-; X86-FAST-LABEL: cmp16_reg_uge_imm16_optsize:
-; X86-FAST:       # %bb.0:
-; X86-FAST-NEXT:    cmpw $-1024, {{[0-9]+}}(%esp) # imm = 0xFC00
-; X86-FAST-NEXT:    setae %al
-; X86-FAST-NEXT:    retl
-;
 ; X64-FAST-LABEL: cmp16_reg_uge_imm16_optsize:
 ; X64-FAST:       # %bb.0:
 ; X64-FAST-NEXT:    cmpw $-1024, %di # imm = 0xFC00
 ; X64-FAST-NEXT:    setae %al
 ; X64-FAST-NEXT:    retq
 ;
-; X86-ATOM-LABEL: cmp16_reg_uge_imm16_optsize:
-; X86-ATOM:       # %bb.0:
-; X86-ATOM-NEXT:    cmpw $-1024, {{[0-9]+}}(%esp) # imm = 0xFC00
-; X86-ATOM-NEXT:    setae %al
-; X86-ATOM-NEXT:    retl
-;
 ; X64-ATOM-LABEL: cmp16_reg_uge_imm16_optsize:
 ; X64-ATOM:       # %bb.0:
 ; X64-ATOM-NEXT:    cmpw $-1024, %di # imm = 0xFC00
@@ -592,15 +550,13 @@ define i1 @cmp16_load_ne_imm16(ptr %p0) {
 ; X86-GENERIC-LABEL: cmp16_load_ne_imm16:
 ; X86-GENERIC:       # %bb.0:
 ; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-GENERIC-NEXT:    movzwl (%eax), %eax
-; X86-GENERIC-NEXT:    cmpl $512, %eax # imm = 0x200
+; X86-GENERIC-NEXT:    cmpw $512, (%eax) # imm = 0x200
 ; X86-GENERIC-NEXT:    setne %al
 ; X86-GENERIC-NEXT:    retl
 ;
 ; X64-GENERIC-LABEL: cmp16_load_ne_imm16:
 ; X64-GENERIC:       # %bb.0:
-; X64-GENERIC-NEXT:    movzwl (%rdi), %eax
-; X64-GENERIC-NEXT:    cmpl $512, %eax # imm = 0x200
+; X64-GENERIC-NEXT:    cmpw $512, (%rdi) # imm = 0x200
 ; X64-GENERIC-NEXT:    setne %al
 ; X64-GENERIC-NEXT:    retq
 ;
@@ -694,15 +650,13 @@ define i1 @cmp16_load_slt_imm16(ptr %p0) {
 ; X86-GENERIC-LABEL: cmp16_load_slt_imm16:
 ; X86-GENERIC:       # %bb.0:
 ; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-GENERIC-NEXT:    movswl (%eax), %eax
-; X86-GENERIC-NEXT:    cmpl $512, %eax # imm = 0x200
+; X86-GENERIC-NEXT:    cmpw $512, (%eax) # imm = 0x200
 ; X86-GENERIC-NEXT:    setl %al
 ; X86-GENERIC-NEXT:    retl
 ;
 ; X64-GENERIC-LABEL: cmp16_load_slt_imm16:
 ; X64-GENERIC:       # %bb.0:
-; X64-GENERIC-NEXT:    movswl (%rdi), %eax
-; X64-GENERIC-NEXT:    cmpl $512, %eax # imm = 0x200
+; X64-GENERIC-NEXT:    cmpw $512, (%rdi) # imm = 0x200
 ; X64-GENERIC-NEXT:    setl %al
 ; X64-GENERIC-NEXT:    retq
 ;
@@ -761,46 +715,18 @@ define i1 @cmp16_load_slt_imm16_minsize(ptr %p0) minsize {
 }
 
 define i1 @cmp16_load_slt_imm16_optsize(ptr %p0) optsize {
-; X86-GENERIC-LABEL: cmp16_load_slt_imm16_optsize:
-; X86-GENERIC:       # %bb.0:
-; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-GENERIC-NEXT:    movswl (%eax), %eax
-; X86-GENERIC-NEXT:    cmpl $512, %eax # imm = 0x200
-; X86-GENERIC-NEXT:    setl %al
-; X86-GENERIC-NEXT:    retl
-;
-; X64-GENERIC-LABEL: cmp16_load_slt_imm16_optsize:
-; X64-GENERIC:       # %bb.0:
-; X64-GENERIC-NEXT:    movswl (%rdi), %eax
-; X64-GENERIC-NEXT:    cmpl $512, %eax # imm = 0x200
-; X64-GENERIC-NEXT:    setl %al
-; X64-GENERIC-NEXT:    retq
-;
-; X86-FAST-LABEL: cmp16_load_slt_imm16_optsize:
-; X86-FAST:       # %bb.0:
-; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT:    cmpw $512, (%eax) # imm = 0x200
-; X86-FAST-NEXT:    setl %al
-; X86-FAST-NEXT:    retl
-;
-; X64-FAST-LABEL: cmp16_load_slt_imm16_optsize:
-; X64-FAST:       # %bb.0:
-; X64-FAST-NEXT:    cmpw $512, (%rdi) # imm = 0x200
-; X64-FAST-NEXT:    setl %al
-; X64-FAST-NEXT:    retq
-;
-; X86-ATOM-LABEL: cmp16_load_slt_imm16_optsize:
-; X86-ATOM:       # %bb.0:
-; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-ATOM-NEXT:    cmpw $512, (%eax) # imm = 0x200
-; X86-ATOM-NEXT:    setl %al
-; X86-ATOM-NEXT:    retl
+; X86-LABEL: cmp16_load_slt_imm16_optsize:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw $512, (%eax) # imm = 0x200
+; X86-NEXT:    setl %al
+; X86-NEXT:    retl
 ;
-; X64-ATOM-LABEL: cmp16_load_slt_imm16_optsize:
-; X64-ATOM:       # %bb.0:
-; X64-ATOM-NEXT:    cmpw $512, (%rdi) # imm = 0x200
-; X64-ATOM-NEXT:    setl %al
-; X64-ATOM-NEXT:    retq
+; X64-LABEL: cmp16_load_slt_imm16_optsize:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpw $512, (%rdi) # imm = 0x200
+; X64-NEXT:    setl %al
+; X64-NEXT:    retq
   %ld = load i16, ptr %p0
   %cmp = icmp slt i16 %ld, 512
   ret i1 %cmp
@@ -860,15 +786,13 @@ define i1 @cmp16_load_ule_imm16(ptr %p0) {
 ; X86-GENERIC-LABEL: cmp16_load_ule_imm16:
 ; X86-GENERIC:       # %bb.0:
 ; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-GENERIC-NEXT:    movzwl (%eax), %eax
-; X86-GENERIC-NEXT:    cmpl $513, %eax # imm = 0x201
+; X86-GENERIC-NEXT:    cmpw $513, (%eax) # imm = 0x201
 ; X86-GENERIC-NEXT:    setb %al
 ; X86-GENERIC-NEXT:    retl
 ;
 ; X64-GENERIC-LABEL: cmp16_load_ule_imm16:
 ; X64-GENERIC:       # %bb.0:
-; X64-GENERIC-NEXT:    movzwl (%rdi), %eax
-; X64-GENERIC-NEXT:    cmpl $513, %eax # imm = 0x201
+; X64-GENERIC-NEXT:    cmpw $513, (%rdi) # imm = 0x201
 ; X64-GENERIC-NEXT:    setb %al
 ; X64-GENERIC-NEXT:    retq
 ;
@@ -927,46 +851,18 @@ define i1 @cmp16_load_ule_imm16_minsize(ptr %p0) minsize {
 }
 
 define i1 @cmp16_load_ule_imm16_optsize(ptr %p0) optsize {
-; X86-GENERIC-LABEL: cmp16_load_ule_imm16_optsize:
-; X86-GENERIC:       # %bb.0:
-; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-GENERIC-NEXT:    movzwl (%eax), %eax
-; X86-GENERIC-NEXT:    cmpl $513, %eax # imm = 0x201
-; X86-GENERIC-NEXT:    setb %al
-; X86-GENERIC-NEXT:    retl
-;
-; X64-GENERIC-LABEL: cmp16_load_ule_imm16_optsize:
-; X64-GENERIC:       # %bb.0:
-; X64-GENERIC-NEXT:    movzwl (%rdi), %eax
-; X64-GENERIC-NEXT:    cmpl $513, %eax # imm = 0x201
-; X64-GENERIC-NEXT:    setb %al
-; X64-GENERIC-NEXT:    retq
-;
-; X86-FAST-LABEL: cmp16_load_ule_imm16_optsize:
-; X86-FAST:       # %bb.0:
-; X86-FAST-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT:    cmpw $513, (%eax) # imm = 0x201
-; X86-FAST-NEXT:    setb %al
-; X86-FAST-NEXT:    retl
-;
-; X64-FAST-LABEL: cmp16_load_ule_imm16_optsize:
-; X64-FAST:       # %bb.0:
-; X64-FAST-NEXT:    cmpw $513, (%rdi) # imm = 0x201
-; X64-FAST-NEXT:    setb %al
-; X64-FAST-NEXT:    retq
-;
-; X86-ATOM-LABEL: cmp16_load_ule_imm16_optsize:
-; X86-ATOM:       # %bb.0:
-; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-ATOM-NEXT:    cmpw $513, (%eax) # imm = 0x201
-; X86-ATOM-NEXT:    setb %al
-; X86-ATOM-NEXT:    retl
+; X86-LABEL: cmp16_load_ule_imm16_optsize:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw $513, (%eax) # imm = 0x201
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
 ;
-; X64-ATOM-LABEL: cmp16_load_ule_imm16_optsize:
-; X64-ATOM:       # %bb.0:
-; X64-ATOM-NEXT:    cmpw $513, (%rdi) # imm = 0x201
-; X64-ATOM-NEXT:    setb %al
-; X64-ATOM-NEXT:    retq
+; X64-LABEL: cmp16_load_ule_imm16_optsize:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpw $513, (%rdi) # imm = 0x201
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
   %ld = load i16, ptr %p0
   %cmp = icmp ule i16 %ld, 512
   ret i1 %cmp
diff --git a/llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll b/llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll
index 0253d1312260..ee5fd78c6437 100644
--- a/llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll
+++ b/llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll
@@ -116,8 +116,7 @@ define i1 @length2_eq_const(ptr %X) nounwind {
 ; X86-LABEL: length2_eq_const:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movzwl (%eax), %eax
-; X86-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X86-NEXT:    cmpw $12849, (%eax) # imm = 0x3231
 ; X86-NEXT:    setne %al
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([513 x i8], ptr @.str, i32 0, i32 1), i32 2) nounwind
diff --git a/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll b/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
index da46ea406557..a46f9ed3d379 100644
--- a/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
+++ b/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
@@ -113,8 +113,7 @@ define i1 @length2_gt(ptr %X, ptr %Y) nounwind {
 define i1 @length2_eq_const(ptr %X) nounwind {
 ; X64-LABEL: length2_eq_const:
 ; X64:       # %bb.0:
-; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X64-NEXT:    cmpw $12849, (%rdi) # imm = 0x3231
 ; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([513 x i8], ptr @.str, i32 0, i32 1), i64 2) nounwind
diff --git a/llvm/test/CodeGen/X86/memcmp-optsize-x32.ll b/llvm/test/CodeGen/X86/memcmp-optsize-x32.ll
index 3db6ae8b76b2..4a9643c0f4fc 100644
--- a/llvm/test/CodeGen/X86/memcmp-optsize-x32.ll
+++ b/llvm/test/CodeGen/X86/memcmp-optsize-x32.ll
@@ -45,8 +45,7 @@ define i1 @length2_eq_const(ptr %X) nounwind optsize {
 ; X86-LABEL: length2_eq_const:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movzwl (%eax), %eax
-; X86-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X86-NEXT:    cmpw $12849, (%eax) # imm = 0x3231
 ; X86-NEXT:    setne %al
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([65 x i8], ptr @.str, i32 0, i32 1), i32 2) nounwind
diff --git a/llvm/test/CodeGen/X86/memcmp-optsize.ll b/llvm/test/CodeGen/X86/memcmp-optsize.ll
index edd61641ad2a..4e27301436c3 100644
--- a/llvm/test/CodeGen/X86/memcmp-optsize.ll
+++ b/llvm/test/CodeGen/X86/memcmp-optsize.ll
@@ -41,8 +41,7 @@ define i1 @length2_eq(ptr %X, ptr %Y) nounwind optsize {
 define i1 @length2_eq_const(ptr %X) nounwind optsize {
 ; X64-LABEL: length2_eq_const:
 ; X64:       # %bb.0:
-; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X64-NEXT:    cmpw $12849, (%rdi) # imm = 0x3231
 ; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([65 x i8], ptr @.str, i32 0, i32 1), i64 2) nounwind
diff --git a/llvm/test/CodeGen/X86/memcmp-pgso-x32.ll b/llvm/test/CodeGen/X86/memcmp-pgso-x32.ll
index 1c301da26bea..bdb50f5b60c4 100644
--- a/llvm/test/CodeGen/X86/memcmp-pgso-x32.ll
+++ b/llvm/test/CodeGen/X86/memcmp-pgso-x32.ll
@@ -45,8 +45,7 @@ define i1 @length2_eq_const(ptr %X) nounwind !prof !14 {
 ; X86-LABEL: length2_eq_const:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movzwl (%eax), %eax
-; X86-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X86-NEXT:    cmpw $12849, (%eax) # imm = 0x3231
 ; X86-NEXT:    setne %al
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([65 x i8], ptr @.str, i32 0, i32 1), i32 2) nounwind
diff --git a/llvm/test/CodeGen/X86/memcmp-pgso.ll b/llvm/test/CodeGen/X86/memcmp-pgso.ll
index 1ee3317b9c96..9347e5422022 100644
--- a/llvm/test/CodeGen/X86/memcmp-pgso.ll
+++ b/llvm/test/CodeGen/X86/memcmp-pgso.ll
@@ -41,8 +41,7 @@ define i1 @length2_eq(ptr %X, ptr %Y) nounwind !prof !14 {
 define i1 @length2_eq_const(ptr %X) nounwind !prof !14 {
 ; X64-LABEL: length2_eq_const:
 ; X64:       # %bb.0:
-; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X64-NEXT:    cmpw $12849, (%rdi) # imm = 0x3231
 ; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([65 x i8], ptr @.str, i32 0, i32 1), i64 2) nounwind
diff --git a/llvm/test/CodeGen/X86/memcmp-x32.ll b/llvm/test/CodeGen/X86/memcmp-x32.ll
index a63402cea209..ad9f2a30d75b 100644
--- a/llvm/test/CodeGen/X86/memcmp-x32.ll
+++ b/llvm/test/CodeGen/X86/memcmp-x32.ll
@@ -144,8 +144,7 @@ define i1 @length2_eq_const(ptr %X) nounwind {
 ; X86-LABEL: length2_eq_const:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movzwl (%eax), %eax
-; X86-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X86-NEXT:    cmpw $12849, (%eax) # imm = 0x3231
 ; X86-NEXT:    setne %al
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([513 x i8], ptr @.str, i32 0, i32 1), i32 2) nounwind
diff --git a/llvm/test/CodeGen/X86/memcmp.ll b/llvm/test/CodeGen/X86/memcmp.ll
index 83cb0d6f973b..8fe1a581cd9c 100644
--- a/llvm/test/CodeGen/X86/memcmp.ll
+++ b/llvm/test/CodeGen/X86/memcmp.ll
@@ -139,8 +139,7 @@ define i1 @length2_gt(ptr %X, ptr %Y) nounwind {
 define i1 @length2_eq_const(ptr %X) nounwind {
 ; X64-LABEL: length2_eq_const:
 ; X64:       # %bb.0:
-; X64-NEXT:    movzwl (%rdi), %eax
-; X64-NEXT:    cmpl $12849, %eax # imm = 0x3231
+; X64-NEXT:    cmpw $12849, (%rdi) # imm = 0x3231
 ; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([513 x i8], ptr @.str, i32 0, i32 1), i64 2) nounwind
-- 
GitLab


From 34f33babc28d240d4ceee69f9afe7d6f5e8ac29b Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Wed, 15 May 2024 20:48:16 +0400
Subject: [PATCH 0888/1206] [lldb] Fixed the TestGdbRemoteCompletion test
 (#92268)

Do not try to run lldb-server on localhost in case of the remote target.
---
 lldb/test/API/tools/lldb-server/TestGdbRemoteCompletion.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lldb/test/API/tools/lldb-server/TestGdbRemoteCompletion.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteCompletion.py
index 04d6abe9d88c..58373d2f85bb 100644
--- a/lldb/test/API/tools/lldb-server/TestGdbRemoteCompletion.py
+++ b/lldb/test/API/tools/lldb-server/TestGdbRemoteCompletion.py
@@ -26,6 +26,7 @@ class GdbRemoteCompletionTestCase(gdbremote_testcase.GdbRemoteTestCaseBase):
     def generate_hex_path(self, target):
         return str(os.path.join(self.getBuildDir(), target)).encode().hex()
 
+    @skipIfRemote
     @add_test_categories(["llgs"])
     def test_autocomplete_path(self):
         self.build()
-- 
GitLab


From fc1df55bcf9b6cc2dec157bcd188b471bc91b945 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Wed, 15 May 2024 20:50:58 +0400
Subject: [PATCH 0889/1206] [lldb][Windows] Fixed the test
 gdb_remote_client/TestGDBRemotePlatformFile (#92088)

The tests `test_file_permissions` and `test_file_permissions_fallback`
are disabled for Windows target. These tests use MockGDBServerResponder
and do not depend on the real target. These tests failed in case of
Windows host and Linux target. Disable them for Windows host too.
---
 .../gdb_remote_client/TestGDBRemotePlatformFile.py        | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemotePlatformFile.py b/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemotePlatformFile.py
index 2be5ae313203..c902722a2f74 100644
--- a/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemotePlatformFile.py
+++ b/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemotePlatformFile.py
@@ -147,7 +147,9 @@ class TestGDBRemotePlatformFile(GDBPlatformClientTestBase):
             log=server2.responder.packetLog,
         )
 
-    @skipIfWindows
+    @expectedFailureAll(
+        hostoslist=["windows"], bugnumber="github.com/llvm/llvm-project/issues/92255"
+    )
     def test_file_permissions(self):
         """Test 'platform get-permissions'"""
 
@@ -167,7 +169,9 @@ class TestGDBRemotePlatformFile(GDBPlatformClientTestBase):
             ]
         )
 
-    @skipIfWindows
+    @expectedFailureAll(
+        hostoslist=["windows"], bugnumber="github.com/llvm/llvm-project/issues/92255"
+    )
     def test_file_permissions_fallback(self):
         """Test 'platform get-permissions' fallback to fstat"""
 
-- 
GitLab


From 2ec85713bd910c5b22ce090798ca00f742d5eb14 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Wed, 15 May 2024 11:57:48 -0500
Subject: [PATCH 0890/1206] [OpenMP] Add back in `ENABLE_LIBOMPTARGET'
 definition

Summary:
Even though we moved `libomptarget` this is still present in `omp.h` and
can't be removed.
---
 openmp/CMakeLists.txt | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index 33bfdc8630ef..9097ca562300 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -97,6 +97,18 @@ set(OPENMP_TEST_FLAGS "" CACHE STRING
 set(OPENMP_TEST_OPENMP_FLAGS ${OPENMP_TEST_COMPILER_OPENMP_FLAGS} CACHE STRING
   "OpenMP compiler flag to use for testing OpenMP runtime libraries.")
 
+set(ENABLE_LIBOMPTARGET ON)
+# Currently libomptarget cannot be compiled on Windows or MacOS X.
+# Since the device plugins are only supported on Linux anyway,
+# there is no point in trying to compile libomptarget on other OSes.
+# 32-bit systems are not supported either.
+if (APPLE OR WIN32 OR WASM OR NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
+    OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8 OR ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+  set(ENABLE_LIBOMPTARGET OFF)
+endif()
+
+option(OPENMP_ENABLE_LIBOMPTARGET "Enable building libomptarget for offloading."
+       ${ENABLE_LIBOMPTARGET})
 option(OPENMP_ENABLE_LIBOMP_PROFILING "Enable time profiling for libomp." OFF)
 
 # Header install location
-- 
GitLab


From 4525f442fadb7cc44cc2eaede2c8ac6ba15bdf78 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 15 May 2024 12:01:16 -0500
Subject: [PATCH 0891/1206] [flang][OpenMP] Don't pass clauses to op-generating
 functions anymore (#90108)

Remove parameter `const List<Clause> &clauses` from functions that take
construct queue. The clauses should now be accessed from the construct
queue.
---
 flang/lib/Lower/OpenMP/OpenMP.cpp | 232 +++++++++++++-----------------
 1 file changed, 103 insertions(+), 129 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index f21acdd64d7c..f05cf1f5120f 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -1233,8 +1233,7 @@ genCriticalOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses, const ConstructQueue &queue,
-              ConstructQueue::iterator item,
+              const ConstructQueue &queue, ConstructQueue::iterator item,
               const std::optional<Fortran::parser::Name> &name) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::FlatSymbolRefAttr nameAttr;
@@ -1245,8 +1244,8 @@ genCriticalOp(Fortran::lower::AbstractConverter &converter,
     auto global = mod.lookupSymbol<mlir::omp::CriticalDeclareOp>(nameStr);
     if (!global) {
       mlir::omp::CriticalClauseOps clauseOps;
-      genCriticalDeclareClauses(converter, semaCtx, clauses, loc, clauseOps,
-                                nameStr);
+      genCriticalDeclareClauses(converter, semaCtx, item->clauses, loc,
+                                clauseOps, nameStr);
 
       mlir::OpBuilder modBuilder(mod.getBodyRegion());
       global = modBuilder.create<mlir::omp::CriticalDeclareOp>(loc, clauseOps);
@@ -1266,8 +1265,7 @@ genDistributeOp(Fortran::lower::AbstractConverter &converter,
                 Fortran::lower::SymMap &symTable,
                 Fortran::semantics::SemanticsContext &semaCtx,
                 Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-                const List<Clause> &clauses, const ConstructQueue &queue,
-                ConstructQueue::iterator item) {
+                const ConstructQueue &queue, ConstructQueue::iterator item) {
   TODO(loc, "Distribute construct");
   return nullptr;
 }
@@ -1277,10 +1275,11 @@ genFlushOp(Fortran::lower::AbstractConverter &converter,
            Fortran::lower::SymMap &symTable,
            Fortran::semantics::SemanticsContext &semaCtx,
            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-           const ObjectList &objects, const List<Clause> &clauses,
-           const ConstructQueue &queue, ConstructQueue::iterator item) {
+           const ObjectList &objects, const ConstructQueue &queue,
+           ConstructQueue::iterator item) {
   llvm::SmallVector<mlir::Value> operandRange;
-  genFlushClauses(converter, semaCtx, objects, clauses, loc, operandRange);
+  genFlushClauses(converter, semaCtx, objects, item->clauses, loc,
+                  operandRange);
 
   return converter.getFirOpBuilder().create<mlir::omp::FlushOp>(
       converter.getCurrentLocation(), operandRange);
@@ -1291,8 +1290,7 @@ genMasterOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const List<Clause> &clauses, const ConstructQueue &queue,
-            ConstructQueue::iterator item) {
+            const ConstructQueue &queue, ConstructQueue::iterator item) {
   return genOpWithBody<mlir::omp::MasterOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_master),
@@ -1304,8 +1302,7 @@ genOrderedOp(Fortran::lower::AbstractConverter &converter,
              Fortran::lower::SymMap &symTable,
              Fortran::semantics::SemanticsContext &semaCtx,
              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-             const List<Clause> &clauses, const ConstructQueue &queue,
-             ConstructQueue::iterator item) {
+             const ConstructQueue &queue, ConstructQueue::iterator item) {
   TODO(loc, "OMPD_ordered");
   return nullptr;
 }
@@ -1315,10 +1312,9 @@ genOrderedRegionOp(Fortran::lower::AbstractConverter &converter,
                    Fortran::lower::SymMap &symTable,
                    Fortran::semantics::SemanticsContext &semaCtx,
                    Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-                   const List<Clause> &clauses, const ConstructQueue &queue,
-                   ConstructQueue::iterator item) {
+                   const ConstructQueue &queue, ConstructQueue::iterator item) {
   mlir::omp::OrderedRegionClauseOps clauseOps;
-  genOrderedRegionClauses(converter, semaCtx, clauses, loc, clauseOps);
+  genOrderedRegionClauses(converter, semaCtx, item->clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::OrderedRegionOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
@@ -1331,15 +1327,15 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses, const ConstructQueue &queue,
-              ConstructQueue::iterator item, bool outerCombined = false) {
+              const ConstructQueue &queue, ConstructQueue::iterator item,
+              bool outerCombined = false) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::ParallelClauseOps clauseOps;
   llvm::SmallVector<const Fortran::semantics::Symbol *> privateSyms;
   llvm::SmallVector<mlir::Type> reductionTypes;
   llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSyms;
-  genParallelClauses(converter, semaCtx, stmtCtx, clauses, loc,
+  genParallelClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
                      /*processReduction=*/!outerCombined, clauseOps,
                      reductionTypes, reductionSyms);
 
@@ -1352,7 +1348,7 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_parallel)
           .setOuterCombined(outerCombined)
-          .setClauses(&clauses)
+          .setClauses(&item->clauses)
           .setReductions(&reductionSyms, &reductionTypes)
           .setGenRegionEntryCb(reductionCallback);
 
@@ -1361,7 +1357,7 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
                                                 clauseOps);
 
   bool privatize = !outerCombined;
-  DataSharingProcessor dsp(converter, semaCtx, clauses, eval,
+  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
                            /*useDelayedPrivatization=*/true, &symTable);
 
   if (privatize)
@@ -1414,14 +1410,13 @@ genSectionOp(Fortran::lower::AbstractConverter &converter,
              Fortran::lower::SymMap &symTable,
              Fortran::semantics::SemanticsContext &semaCtx,
              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-             const List<Clause> &clauses, const ConstructQueue &queue,
-             ConstructQueue::iterator item) {
+             const ConstructQueue &queue, ConstructQueue::iterator item) {
   // Currently only private/firstprivate clause is handled, and
   // all privatization is done within `omp.section` operations.
   return genOpWithBody<mlir::omp::SectionOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_section)
-          .setClauses(&clauses),
+          .setClauses(&item->clauses),
       queue, item);
 }
 
@@ -1430,22 +1425,21 @@ genSectionsOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses, const ConstructQueue &queue,
-              ConstructQueue::iterator item) {
+              const ConstructQueue &queue, ConstructQueue::iterator item) {
   mlir::omp::SectionsClauseOps clauseOps;
-  genSectionsClauses(converter, semaCtx, clauses, loc, clauseOps);
+  genSectionsClauses(converter, semaCtx, item->clauses, loc, clauseOps);
 
   auto &builder = converter.getFirOpBuilder();
 
   // Insert privatizations before SECTIONS
   symTable.pushScope();
-  DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
+  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval);
   dsp.processStep1();
 
   List<Clause> nonDsaClauses;
   List<const clause::Lastprivate *> lastprivates;
 
-  for (const Clause &clause : clauses) {
+  for (const Clause &clause : item->clauses) {
     if (clause.id == llvm::omp::Clause::OMPC_lastprivate) {
       lastprivates.push_back(&std::get<clause::Lastprivate>(clause.u));
     } else {
@@ -1508,18 +1502,18 @@ genSimdOp(Fortran::lower::AbstractConverter &converter,
           Fortran::lower::SymMap &symTable,
           Fortran::semantics::SemanticsContext &semaCtx,
           Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-          const List<Clause> &clauses, const ConstructQueue &queue,
-          ConstructQueue::iterator item) {
+          const ConstructQueue &queue, ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
+  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval);
   dsp.processStep1();
 
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::LoopNestClauseOps loopClauseOps;
   mlir::omp::SimdClauseOps simdClauseOps;
   llvm::SmallVector<const Fortran::semantics::Symbol *> iv;
-  genLoopNestClauses(converter, semaCtx, eval, clauses, loc, loopClauseOps, iv);
-  genSimdClauses(converter, semaCtx, clauses, loc, simdClauseOps);
+  genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
+                     loopClauseOps, iv);
+  genSimdClauses(converter, semaCtx, item->clauses, loc, simdClauseOps);
 
   // Create omp.simd wrapper.
   auto simdOp = firOpBuilder.create<mlir::omp::SimdOp>(loc, simdClauseOps);
@@ -1532,7 +1526,8 @@ genSimdOp(Fortran::lower::AbstractConverter &converter,
   // Create nested omp.loop_nest and fill body with loop contents.
   auto loopOp = firOpBuilder.create<mlir::omp::LoopNestOp>(loc, loopClauseOps);
 
-  auto *nestedEval = getCollapsedLoopEval(eval, getCollapseValue(clauses));
+  auto *nestedEval =
+      getCollapsedLoopEval(eval, getCollapseValue(item->clauses));
 
   auto ivCallback = [&](mlir::Operation *op) {
     genLoopVars(op, converter, loc, iv);
@@ -1542,7 +1537,7 @@ genSimdOp(Fortran::lower::AbstractConverter &converter,
   createBodyOfOp(*loopOp,
                  OpWithBodyGenInfo(converter, symTable, semaCtx, loc,
                                    *nestedEval, llvm::omp::Directive::OMPD_simd)
-                     .setClauses(&clauses)
+                     .setClauses(&item->clauses)
                      .setDataSharingProcessor(&dsp)
                      .setGenRegionEntryCb(ivCallback),
                  queue, item);
@@ -1555,15 +1550,14 @@ genSingleOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const List<Clause> &clauses, const ConstructQueue &queue,
-            ConstructQueue::iterator item) {
+            const ConstructQueue &queue, ConstructQueue::iterator item) {
   mlir::omp::SingleClauseOps clauseOps;
-  genSingleClauses(converter, semaCtx, clauses, loc, clauseOps);
+  genSingleClauses(converter, semaCtx, item->clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::SingleOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_single)
-          .setClauses(&clauses),
+          .setClauses(&item->clauses),
       queue, item, clauseOps);
 }
 
@@ -1572,8 +1566,8 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const List<Clause> &clauses, const ConstructQueue &queue,
-            ConstructQueue::iterator item, bool outerCombined = false) {
+            const ConstructQueue &queue, ConstructQueue::iterator item,
+            bool outerCombined = false) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
 
@@ -1586,7 +1580,7 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
       deviceAddrSyms;
   llvm::SmallVector<mlir::Location> mapLocs, devicePtrLocs, deviceAddrLocs;
   llvm::SmallVector<mlir::Type> mapTypes, devicePtrTypes, deviceAddrTypes;
-  genTargetClauses(converter, semaCtx, stmtCtx, clauses, loc,
+  genTargetClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
                    processHostOnlyClauses, /*processReduction=*/outerCombined,
                    clauseOps, mapSyms, mapLocs, mapTypes, deviceAddrSyms,
                    deviceAddrLocs, deviceAddrTypes, devicePtrSyms,
@@ -1690,15 +1684,14 @@ genTargetDataOp(Fortran::lower::AbstractConverter &converter,
                 Fortran::lower::SymMap &symTable,
                 Fortran::semantics::SemanticsContext &semaCtx,
                 Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-                const List<Clause> &clauses, const ConstructQueue &queue,
-                ConstructQueue::iterator item) {
+                const ConstructQueue &queue, ConstructQueue::iterator item) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TargetDataClauseOps clauseOps;
   llvm::SmallVector<mlir::Type> useDeviceTypes;
   llvm::SmallVector<mlir::Location> useDeviceLocs;
   llvm::SmallVector<const Fortran::semantics::Symbol *> useDeviceSyms;
-  genTargetDataClauses(converter, semaCtx, stmtCtx, clauses, loc, clauseOps,
-                       useDeviceTypes, useDeviceLocs, useDeviceSyms);
+  genTargetDataClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
+                       clauseOps, useDeviceTypes, useDeviceLocs, useDeviceSyms);
 
   auto targetDataOp =
       converter.getFirOpBuilder().create<mlir::omp::TargetDataOp>(loc,
@@ -1714,8 +1707,7 @@ static OpTy
 genTargetEnterExitUpdateDataOp(Fortran::lower::AbstractConverter &converter,
                                Fortran::lower::SymMap &symTable,
                                Fortran::semantics::SemanticsContext &semaCtx,
-                               mlir::Location loc, const List<Clause> &clauses,
-                               const ConstructQueue &queue,
+                               mlir::Location loc, const ConstructQueue &queue,
                                ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
@@ -1733,8 +1725,8 @@ genTargetEnterExitUpdateDataOp(Fortran::lower::AbstractConverter &converter,
   }
 
   mlir::omp::TargetEnterExitUpdateDataClauseOps clauseOps;
-  genTargetEnterExitUpdateDataClauses(converter, semaCtx, stmtCtx, clauses, loc,
-                                      directive, clauseOps);
+  genTargetEnterExitUpdateDataClauses(converter, semaCtx, stmtCtx,
+                                      item->clauses, loc, directive, clauseOps);
 
   return firOpBuilder.create<OpTy>(loc, clauseOps);
 }
@@ -1744,16 +1736,15 @@ genTaskOp(Fortran::lower::AbstractConverter &converter,
           Fortran::lower::SymMap &symTable,
           Fortran::semantics::SemanticsContext &semaCtx,
           Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-          const List<Clause> &clauses, const ConstructQueue &queue,
-          ConstructQueue::iterator item) {
+          const ConstructQueue &queue, ConstructQueue::iterator item) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TaskClauseOps clauseOps;
-  genTaskClauses(converter, semaCtx, stmtCtx, clauses, loc, clauseOps);
+  genTaskClauses(converter, semaCtx, stmtCtx, item->clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::TaskOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_task)
-          .setClauses(&clauses),
+          .setClauses(&item->clauses),
       queue, item, clauseOps);
 }
 
@@ -1762,15 +1753,14 @@ genTaskgroupOp(Fortran::lower::AbstractConverter &converter,
                Fortran::lower::SymMap &symTable,
                Fortran::semantics::SemanticsContext &semaCtx,
                Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-               const List<Clause> &clauses, const ConstructQueue &queue,
-               ConstructQueue::iterator item) {
+               const ConstructQueue &queue, ConstructQueue::iterator item) {
   mlir::omp::TaskgroupClauseOps clauseOps;
-  genTaskgroupClauses(converter, semaCtx, clauses, loc, clauseOps);
+  genTaskgroupClauses(converter, semaCtx, item->clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::TaskgroupOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_taskgroup)
-          .setClauses(&clauses),
+          .setClauses(&item->clauses),
       queue, item, clauseOps);
 }
 
@@ -1779,8 +1769,7 @@ genTaskloopOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses, const ConstructQueue &queue,
-              ConstructQueue::iterator item) {
+              const ConstructQueue &queue, ConstructQueue::iterator item) {
   TODO(loc, "Taskloop construct");
 }
 
@@ -1789,10 +1778,9 @@ genTaskwaitOp(Fortran::lower::AbstractConverter &converter,
               Fortran::lower::SymMap &symTable,
               Fortran::semantics::SemanticsContext &semaCtx,
               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-              const List<Clause> &clauses, const ConstructQueue &queue,
-              ConstructQueue::iterator item) {
+              const ConstructQueue &queue, ConstructQueue::iterator item) {
   mlir::omp::TaskwaitClauseOps clauseOps;
-  genTaskwaitClauses(converter, semaCtx, clauses, loc, clauseOps);
+  genTaskwaitClauses(converter, semaCtx, item->clauses, loc, clauseOps);
   return converter.getFirOpBuilder().create<mlir::omp::TaskwaitOp>(loc,
                                                                    clauseOps);
 }
@@ -1811,17 +1799,17 @@ genTeamsOp(Fortran::lower::AbstractConverter &converter,
            Fortran::lower::SymMap &symTable,
            Fortran::semantics::SemanticsContext &semaCtx,
            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-           const List<Clause> &clauses, const ConstructQueue &queue,
-           ConstructQueue::iterator item, bool outerCombined = false) {
+           const ConstructQueue &queue, ConstructQueue::iterator item,
+           bool outerCombined = false) {
   Fortran::lower::StatementContext stmtCtx;
   mlir::omp::TeamsClauseOps clauseOps;
-  genTeamsClauses(converter, semaCtx, stmtCtx, clauses, loc, clauseOps);
+  genTeamsClauses(converter, semaCtx, stmtCtx, item->clauses, loc, clauseOps);
 
   return genOpWithBody<mlir::omp::TeamsOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_teams)
           .setOuterCombined(outerCombined)
-          .setClauses(&clauses),
+          .setClauses(&item->clauses),
       queue, item, clauseOps);
 }
 
@@ -1830,10 +1818,9 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
             Fortran::lower::SymMap &symTable,
             Fortran::semantics::SemanticsContext &semaCtx,
             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const List<Clause> &clauses, const ConstructQueue &queue,
-            ConstructQueue::iterator item) {
+            const ConstructQueue &queue, ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  DataSharingProcessor dsp(converter, semaCtx, clauses, eval);
+  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval);
   dsp.processStep1();
 
   Fortran::lower::StatementContext stmtCtx;
@@ -1842,8 +1829,9 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
   llvm::SmallVector<const Fortran::semantics::Symbol *> iv;
   llvm::SmallVector<mlir::Type> reductionTypes;
   llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSyms;
-  genLoopNestClauses(converter, semaCtx, eval, clauses, loc, loopClauseOps, iv);
-  genWsloopClauses(converter, semaCtx, stmtCtx, clauses, loc, wsClauseOps,
+  genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
+                     loopClauseOps, iv);
+  genWsloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc, wsClauseOps,
                    reductionTypes, reductionSyms);
 
   // Create omp.wsloop wrapper and populate entry block arguments with reduction
@@ -1858,7 +1846,8 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
   // Create nested omp.loop_nest and fill body with loop contents.
   auto loopOp = firOpBuilder.create<mlir::omp::LoopNestOp>(loc, loopClauseOps);
 
-  auto *nestedEval = getCollapsedLoopEval(eval, getCollapseValue(clauses));
+  auto *nestedEval =
+      getCollapsedLoopEval(eval, getCollapseValue(item->clauses));
 
   auto ivCallback = [&](mlir::Operation *op) {
     genLoopVars(op, converter, loc, iv, reductionSyms,
@@ -1869,7 +1858,7 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
   createBodyOfOp(*loopOp,
                  OpWithBodyGenInfo(converter, symTable, semaCtx, loc,
                                    *nestedEval, llvm::omp::Directive::OMPD_do)
-                     .setClauses(&clauses)
+                     .setClauses(&item->clauses)
                      .setDataSharingProcessor(&dsp)
                      .setReductions(&reductionSyms, &reductionTypes)
                      .setGenRegionEntryCb(ivCallback),
@@ -1886,8 +1875,7 @@ static void genCompositeDistributeParallelDo(
     Fortran::lower::SymMap &symTable,
     Fortran::semantics::SemanticsContext &semaCtx,
     Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-    const List<Clause> &clauses, const ConstructQueue &queue,
-    ConstructQueue::iterator item) {
+    const ConstructQueue &queue, ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE PARALLEL DO");
 }
 
@@ -1896,8 +1884,7 @@ static void genCompositeDistributeParallelDoSimd(
     Fortran::lower::SymMap &symTable,
     Fortran::semantics::SemanticsContext &semaCtx,
     Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-    const List<Clause> &clauses, const ConstructQueue &queue,
-    ConstructQueue::iterator item) {
+    const ConstructQueue &queue, ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE PARALLEL DO SIMD");
 }
 
@@ -1906,8 +1893,7 @@ genCompositeDistributeSimd(Fortran::lower::AbstractConverter &converter,
                            Fortran::lower::SymMap &symTable,
                            Fortran::semantics::SemanticsContext &semaCtx,
                            Fortran::lower::pft::Evaluation &eval,
-                           mlir::Location loc, const List<Clause> &clauses,
-                           const ConstructQueue &queue,
+                           mlir::Location loc, const ConstructQueue &queue,
                            ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE SIMD");
 }
@@ -1916,10 +1902,9 @@ static void genCompositeDoSimd(Fortran::lower::AbstractConverter &converter,
                                Fortran::lower::SymMap &symTable,
                                Fortran::semantics::SemanticsContext &semaCtx,
                                Fortran::lower::pft::Evaluation &eval,
-                               mlir::Location loc, const List<Clause> &clauses,
-                               const ConstructQueue &queue,
+                               mlir::Location loc, const ConstructQueue &queue,
                                ConstructQueue::iterator item) {
-  ClauseProcessor cp(converter, semaCtx, clauses);
+  ClauseProcessor cp(converter, semaCtx, item->clauses);
   cp.processTODO<clause::Aligned, clause::Allocate, clause::Linear,
                  clause::Order, clause::Safelen, clause::Simdlen>(
       loc, llvm::omp::OMPD_do_simd);
@@ -1931,7 +1916,7 @@ static void genCompositeDoSimd(Fortran::lower::AbstractConverter &converter,
   // When support for vectorization is enabled, then we need to add handling of
   // if clause. Currently if clause can be skipped because we always assume
   // SIMD length = 1.
-  genWsloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+  genWsloopOp(converter, symTable, semaCtx, eval, loc, queue, item);
 }
 
 static void
@@ -1939,8 +1924,7 @@ genCompositeTaskloopSimd(Fortran::lower::AbstractConverter &converter,
                          Fortran::lower::SymMap &symTable,
                          Fortran::semantics::SemanticsContext &semaCtx,
                          Fortran::lower::pft::Evaluation &eval,
-                         mlir::Location loc, const List<Clause> &clauses,
-                         const ConstructQueue &queue,
+                         mlir::Location loc, const ConstructQueue &queue,
                          ConstructQueue::iterator item) {
   TODO(loc, "Composite TASKLOOP SIMD");
 }
@@ -1956,18 +1940,16 @@ static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
                            mlir::Location loc, const ConstructQueue &queue,
                            ConstructQueue::iterator item) {
   assert(item != queue.end());
-  const List<Clause> &clauses = item->clauses;
 
   switch (llvm::omp::Directive dir = item->id) {
   case llvm::omp::Directive::OMPD_barrier:
     genBarrierOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_distribute:
-    genDistributeOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                    item);
+    genDistributeOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_do:
-    genWsloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    genWsloopOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_loop:
   case llvm::omp::Directive::OMPD_masked:
@@ -1975,71 +1957,64 @@ static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
                   llvm::omp::getOpenMPDirectiveName(dir) + ")");
     break;
   case llvm::omp::Directive::OMPD_master:
-    genMasterOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    genMasterOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_ordered:
     // Block-associated "ordered" construct.
-    genOrderedRegionOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                       item);
+    genOrderedRegionOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_parallel:
-    genParallelOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item,
+    genParallelOp(converter, symTable, semaCtx, eval, loc, queue, item,
                   /*outerCombined=*/false);
     break;
   case llvm::omp::Directive::OMPD_section:
-    genSectionOp(converter, symTable, semaCtx, eval, loc, /*clauses=*/{}, queue,
-                 item);
+    genSectionOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_sections:
-    genSectionsOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                  item);
+    genSectionsOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_simd:
-    genSimdOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    genSimdOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_single:
-    genSingleOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    genSingleOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_target:
-    genTargetOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item,
+    genTargetOp(converter, symTable, semaCtx, eval, loc, queue, item,
                 /*outerCombined=*/false);
     break;
   case llvm::omp::Directive::OMPD_target_data:
-    genTargetDataOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                    item);
+    genTargetDataOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_target_enter_data:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(
-        converter, symTable, semaCtx, loc, clauses, queue, item);
+        converter, symTable, semaCtx, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_target_exit_data:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(
-        converter, symTable, semaCtx, loc, clauses, queue, item);
+        converter, symTable, semaCtx, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_target_update:
     genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(
-        converter, symTable, semaCtx, loc, clauses, queue, item);
+        converter, symTable, semaCtx, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_task:
-    genTaskOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    genTaskOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_taskgroup:
-    genTaskgroupOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                   item);
+    genTaskgroupOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_taskloop:
-    genTaskloopOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                  item);
+    genTaskloopOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_taskwait:
-    genTaskwaitOp(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                  item);
+    genTaskwaitOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_taskyield:
     genTaskyieldOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_teams:
-    genTeamsOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    genTeamsOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_tile:
   case llvm::omp::Directive::OMPD_unroll:
@@ -2050,29 +2025,28 @@ static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
     // FIXME: Workshare is not a commonly used OpenMP construct, an
     // implementation for this feature will come later. For the codes
     // that use this construct, add a single construct for now.
-    genSingleOp(converter, symTable, semaCtx, eval, loc, clauses, queue, item);
+    genSingleOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
 
   // Composite constructs
   case llvm::omp::Directive::OMPD_distribute_parallel_do:
     genCompositeDistributeParallelDo(converter, symTable, semaCtx, eval, loc,
-                                     clauses, queue, item);
+                                     queue, item);
     break;
   case llvm::omp::Directive::OMPD_distribute_parallel_do_simd:
     genCompositeDistributeParallelDoSimd(converter, symTable, semaCtx, eval,
-                                         loc, clauses, queue, item);
+                                         loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_distribute_simd:
-    genCompositeDistributeSimd(converter, symTable, semaCtx, eval, loc, clauses,
-                               queue, item);
+    genCompositeDistributeSimd(converter, symTable, semaCtx, eval, loc, queue,
+                               item);
     break;
   case llvm::omp::Directive::OMPD_do_simd:
-    genCompositeDoSimd(converter, symTable, semaCtx, eval, loc, clauses, queue,
-                       item);
+    genCompositeDoSimd(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_taskloop_simd:
-    genCompositeTaskloopSimd(converter, symTable, semaCtx, eval, loc, clauses,
-                             queue, item);
+    genCompositeTaskloopSimd(converter, symTable, semaCtx, eval, loc, queue,
+                             item);
     break;
   default:
     break;
@@ -2194,8 +2168,8 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
                           eval, directive.source, directive.v, clauses)};
   if (directive.v == llvm::omp::Directive::OMPD_ordered) {
     // Standalone "ordered" directive.
-    genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
-                 queue, queue.begin());
+    genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, queue,
+                 queue.begin());
   } else {
     // Dispatch handles the "block-associated" variant of "ordered".
     genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
@@ -2227,7 +2201,7 @@ genOMP(Fortran::lower::AbstractConverter &converter,
       converter.getFirOpBuilder().getModule(), semaCtx, eval, verbatim.source,
       llvm::omp::Directive::OMPD_flush, clauses)};
   genFlushOp(converter, symTable, semaCtx, eval, currentLocation, objects,
-             clauses, queue, queue.begin());
+             queue, queue.begin());
 }
 
 static void
@@ -2399,8 +2373,8 @@ genOMP(Fortran::lower::AbstractConverter &converter,
 
   const auto &name = std::get<std::optional<Fortran::parser::Name>>(cd.t);
   mlir::Location currentLocation = converter.getCurrentLocation();
-  genCriticalOp(converter, symTable, semaCtx, eval, currentLocation, clauses,
-                queue, queue.begin(), name);
+  genCriticalOp(converter, symTable, semaCtx, eval, currentLocation, queue,
+                queue.begin(), name);
 }
 
 static void
-- 
GitLab


From eb822dc25853299ea81166f9bb8a43436ab8b0c8 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Wed, 15 May 2024 21:03:15 +0400
Subject: [PATCH 0892/1206] [lldb] Fixed the TestCompletion test running on a
 remote target (#92281)

Install the image to the remote target if necessary.
---
 .../functionalities/completion/TestCompletion.py    | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lldb/test/API/functionalities/completion/TestCompletion.py b/lldb/test/API/functionalities/completion/TestCompletion.py
index 0d6907e0c3d2..63842487fc33 100644
--- a/lldb/test/API/functionalities/completion/TestCompletion.py
+++ b/lldb/test/API/functionalities/completion/TestCompletion.py
@@ -107,9 +107,16 @@ class CommandLineCompletionTestCase(TestBase):
             self, "// Break here", lldb.SBFileSpec("main.cpp")
         )
         err = lldb.SBError()
-        self.process().LoadImage(
-            lldb.SBFileSpec(self.getBuildArtifact("libshared.so")), err
+        local_spec = lldb.SBFileSpec(self.getBuildArtifact("libshared.so"))
+        remote_spec = (
+            lldb.SBFileSpec(
+                lldbutil.append_to_process_working_directory(self, "libshared.so"),
+                False,
+            )
+            if lldb.remote_platform
+            else lldb.SBFileSpec()
         )
+        self.process().LoadImage(local_spec, remote_spec, err)
         self.assertSuccess(err)
 
         self.complete_from_to("process unload ", "process unload 0")
@@ -473,7 +480,7 @@ class CommandLineCompletionTestCase(TestBase):
         self.complete_from_to("my_test_cmd main.cp", ["main.cpp"])
         self.expect("my_test_cmd main.cpp", substrs=["main.cpp"])
 
-    @skipIfWindows
+    @skipIf(hostoslist=["windows"])
     def test_completion_target_create_from_root_dir(self):
         """Tests source file completion by completing ."""
         root_dir = os.path.abspath(os.sep)
-- 
GitLab


From 7645269710493c188d1d270b9e4e085b3e92b9b0 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Wed, 15 May 2024 21:06:30 +0400
Subject: [PATCH 0893/1206] [lldb] Fixed the TestNetBSDCore test (#92285)

TestNetBSDCore.py contains 3 classes with the same test names
test_aarch64 and test_amd64. It causes conflicts because the same build
dir. Add suffixes to avoid conflicts.
---
 .../postmortem/netbsd-core/TestNetBSDCore.py         | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lldb/test/API/functionalities/postmortem/netbsd-core/TestNetBSDCore.py b/lldb/test/API/functionalities/postmortem/netbsd-core/TestNetBSDCore.py
index 756f4d1e81ca..ff1ef21e02e3 100644
--- a/lldb/test/API/functionalities/postmortem/netbsd-core/TestNetBSDCore.py
+++ b/lldb/test/API/functionalities/postmortem/netbsd-core/TestNetBSDCore.py
@@ -147,12 +147,12 @@ class NetBSD1LWPCoreTestCase(NetBSDCoreCommonTestCase):
         self.check_backtrace(thread, filename, backtrace)
 
     @skipIfLLVMTargetMissing("AArch64")
-    def test_aarch64(self):
+    def test_aarch64_single_threaded(self):
         """Test single-threaded aarch64 core dump."""
         self.do_test("1lwp_SIGSEGV.aarch64", pid=8339, region_count=32)
 
     @skipIfLLVMTargetMissing("X86")
-    def test_amd64(self):
+    def test_amd64_single_threaded(self):
         """Test single-threaded amd64 core dump."""
         self.do_test("1lwp_SIGSEGV.amd64", pid=693, region_count=21)
 
@@ -177,12 +177,12 @@ class NetBSD2LWPT2CoreTestCase(NetBSDCoreCommonTestCase):
         self.assertEqual(thread.GetStopReasonDataAtIndex(0), 0)
 
     @skipIfLLVMTargetMissing("AArch64")
-    def test_aarch64(self):
+    def test_aarch64_thread_signaled(self):
         """Test double-threaded aarch64 core dump where thread 2 is signalled."""
         self.do_test("2lwp_t2_SIGSEGV.aarch64", pid=14142, region_count=31)
 
     @skipIfLLVMTargetMissing("X86")
-    def test_amd64(self):
+    def test_amd64_thread_signaled(self):
         """Test double-threaded amd64 core dump where thread 2 is signalled."""
         self.do_test("2lwp_t2_SIGSEGV.amd64", pid=622, region_count=24)
 
@@ -207,11 +207,11 @@ class NetBSD2LWPProcessSigCoreTestCase(NetBSDCoreCommonTestCase):
         self.assertEqual(thread.GetStopReasonDataAtIndex(0), signal.SIGSEGV)
 
     @skipIfLLVMTargetMissing("AArch64")
-    def test_aarch64(self):
+    def test_aarch64_process_signaled(self):
         """Test double-threaded aarch64 core dump where process is signalled."""
         self.do_test("2lwp_process_SIGSEGV.aarch64", pid=1403, region_count=30)
 
     @skipIfLLVMTargetMissing("X86")
-    def test_amd64(self):
+    def test_amd64_process_signaled(self):
         """Test double-threaded amd64 core dump where process is signalled."""
         self.do_test("2lwp_process_SIGSEGV.amd64", pid=665, region_count=24)
-- 
GitLab


From d92c67784f21063d6334a009dbf4f9e0f8217b41 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Wed, 15 May 2024 21:08:35 +0400
Subject: [PATCH 0894/1206] [lldb][Windows] Fixed the
 TestIOHandlerResizeNoEditline test (#92286)

This test caused python crash on Windows x86_64 host with the exit code
0xC0000409 (STATUS_STACK_BUFFER_OVERRUN). Close the input stream before
exit to avoid this crash.
---
 lldb/test/API/iohandler/resize/TestIOHandlerResizeNoEditline.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lldb/test/API/iohandler/resize/TestIOHandlerResizeNoEditline.py b/lldb/test/API/iohandler/resize/TestIOHandlerResizeNoEditline.py
index 3c07554f6caf..bbc2dcbe4e30 100644
--- a/lldb/test/API/iohandler/resize/TestIOHandlerResizeNoEditline.py
+++ b/lldb/test/API/iohandler/resize/TestIOHandlerResizeNoEditline.py
@@ -18,3 +18,4 @@ class TestCase(TestBase):
         dbg.RunCommandInterpreter(True, True, opts, 0, False, False)
         # Try resizing the terminal which shouldn't crash.
         dbg.SetTerminalWidth(47)
+        dbg.GetInputFile().Close()
-- 
GitLab


From 217668f641e82f901645f428ae0d07a3c01e9a8a Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Wed, 15 May 2024 10:34:47 -0700
Subject: [PATCH 0895/1206] [nfc] Allow forwarding `Error` returns from
 `Expected` callers (#92208)

On a few compilers (clang 11/12 for example [1]), the following does not
result in a copy elision, and since `Error`'s copy dtor is elided,
results in a compile error:

```
  Expect<Something> foobar() {
    ...
    if (Error E = aCallReturningError())
      return E;
    ...
  }
```

Moving `E` would, conversely, result in the pessimizing-move warning on
more recent clangs ("moving a local object in a return statement
prevents copy elision")

We just need to make the `Expected` ctor taking an `Error` take it as a
r-value reference.

[1] https://lab.llvm.org/buildbot/#/builders/54/builds/10505
---
 llvm/include/llvm/Support/Error.h             |  2 +-
 llvm/lib/Bitstream/Reader/BitstreamReader.cpp | 12 ++++++------
 llvm/lib/Object/COFFObjectFile.cpp            |  6 +++---
 llvm/lib/Object/WindowsResource.cpp           |  4 ++--
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h
index 894b6484336a..217130ce293a 100644
--- a/llvm/include/llvm/Support/Error.h
+++ b/llvm/include/llvm/Support/Error.h
@@ -493,7 +493,7 @@ private:
 
 public:
   /// Create an Expected<T> error value from the given Error.
-  Expected(Error Err)
+  Expected(Error &&Err)
       : HasError(true)
 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
         // Expected is unchecked upon construction in Debug builds.
diff --git a/llvm/lib/Bitstream/Reader/BitstreamReader.cpp b/llvm/lib/Bitstream/Reader/BitstreamReader.cpp
index 3cc9dfdf7b85..5b2c76350029 100644
--- a/llvm/lib/Bitstream/Reader/BitstreamReader.cpp
+++ b/llvm/lib/Bitstream/Reader/BitstreamReader.cpp
@@ -167,7 +167,7 @@ Expected<unsigned> BitstreamCursor::skipRecord(unsigned AbbrevID) {
         if (Error Err =
                 JumpToBit(GetCurrentBitNo() + static_cast<uint64_t>(NumElts) *
                                                   EltEnc.getEncodingData()))
-          return std::move(Err);
+          return Err;
         break;
       case BitCodeAbbrevOp::VBR:
         assert((unsigned)EltEnc.getEncodingData() <= MaxChunkSize);
@@ -180,7 +180,7 @@ Expected<unsigned> BitstreamCursor::skipRecord(unsigned AbbrevID) {
         break;
       case BitCodeAbbrevOp::Char6:
         if (Error Err = JumpToBit(GetCurrentBitNo() + NumElts * 6))
-          return std::move(Err);
+          return Err;
         break;
       }
       continue;
@@ -206,7 +206,7 @@ Expected<unsigned> BitstreamCursor::skipRecord(unsigned AbbrevID) {
 
     // Skip over the blob.
     if (Error Err = JumpToBit(NewEnd))
-      return std::move(Err);
+      return Err;
   }
   return Code;
 }
@@ -344,7 +344,7 @@ Expected<unsigned> BitstreamCursor::readRecord(unsigned AbbrevID,
     // over tail padding first, in case jumping to NewEnd invalidates the Blob
     // pointer.
     if (Error Err = JumpToBit(NewEnd))
-      return std::move(Err);
+      return Err;
     const char *Ptr = (const char *)getPointerToBit(CurBitPos, NumElts);
 
     // If we can return a reference to the data, do so to avoid copying it.
@@ -421,7 +421,7 @@ Error BitstreamCursor::ReadAbbrevRecord() {
 Expected<std::optional<BitstreamBlockInfo>>
 BitstreamCursor::ReadBlockInfoBlock(bool ReadBlockInfoNames) {
   if (llvm::Error Err = EnterSubBlock(bitc::BLOCKINFO_BLOCK_ID))
-    return std::move(Err);
+    return Err;
 
   BitstreamBlockInfo NewBlockInfo;
 
@@ -452,7 +452,7 @@ BitstreamCursor::ReadBlockInfoBlock(bool ReadBlockInfoNames) {
       if (!CurBlockInfo)
         return std::nullopt;
       if (Error Err = ReadAbbrevRecord())
-        return std::move(Err);
+        return Err;
 
       // ReadAbbrevRecord installs the abbrev in CurAbbrevs.  Move it to the
       // appropriate BlockInfo.
diff --git a/llvm/lib/Object/COFFObjectFile.cpp b/llvm/lib/Object/COFFObjectFile.cpp
index 18506f39f6b5..5a85b8e00c63 100644
--- a/llvm/lib/Object/COFFObjectFile.cpp
+++ b/llvm/lib/Object/COFFObjectFile.cpp
@@ -294,7 +294,7 @@ COFFObjectFile::getSectionContents(DataRefImpl Ref) const {
   const coff_section *Sec = toSec(Ref);
   ArrayRef<uint8_t> Res;
   if (Error E = getSectionContents(Sec, Res))
-    return std::move(E);
+    return E;
   return Res;
 }
 
@@ -807,7 +807,7 @@ Expected<std::unique_ptr<COFFObjectFile>>
 COFFObjectFile::create(MemoryBufferRef Object) {
   std::unique_ptr<COFFObjectFile> Obj(new COFFObjectFile(std::move(Object)));
   if (Error E = Obj->initialize())
-    return std::move(E);
+    return E;
   return std::move(Obj);
 }
 
@@ -1959,7 +1959,7 @@ ResourceSectionRef::getContents(const coff_resource_data_entry &Entry) {
     uint64_t Offset = Entry.DataRVA + Sym->getValue();
     ArrayRef<uint8_t> Contents;
     if (Error E = Obj->getSectionContents(*Section, Contents))
-      return std::move(E);
+      return E;
     if (Offset + Entry.DataSize > Contents.size())
       return createStringError(object_error::parse_failed,
                                "data outside of section");
diff --git a/llvm/lib/Object/WindowsResource.cpp b/llvm/lib/Object/WindowsResource.cpp
index 983c8e30a942..306e8ec54206 100644
--- a/llvm/lib/Object/WindowsResource.cpp
+++ b/llvm/lib/Object/WindowsResource.cpp
@@ -80,7 +80,7 @@ Expected<ResourceEntryRef>
 ResourceEntryRef::create(BinaryStreamRef BSR, const WindowsResource *Owner) {
   auto Ref = ResourceEntryRef(BSR, Owner);
   if (auto E = Ref.loadNext())
-    return std::move(E);
+    return E;
   return Ref;
 }
 
@@ -1006,7 +1006,7 @@ writeWindowsResourceCOFF(COFF::MachineTypes MachineType,
   Error E = Error::success();
   WindowsResourceCOFFWriter Writer(MachineType, Parser, E);
   if (E)
-    return std::move(E);
+    return E;
   return Writer.write(TimeDateStamp);
 }
 
-- 
GitLab


From 0647d1035cb208195e002b38089b82004b6f7b92 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 15 May 2024 10:15:35 -0700
Subject: [PATCH 0896/1206] [RISCV] Remove unneeded casts from int64_t to
 uint64_t in RISCVMatInt.cpp. NFC

Most of these were to avoid undefined behavior if a shift left
changed the sign of the result. I don't think its possible to change
the sign of the result here. We're shifting left by 12 after an arithmetic
right shift by more than 12. The bits we are shifting out with the left
shift are guaranteed to be sign bits.

Also use SignExtend64<32> to force upper bits to all 1s instead of an
Or. We know the value isUInt<32> && !isInt<32> which means bit 31 is set.
---
 .../Target/RISCV/MCTargetDesc/RISCVMatInt.cpp   | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
index 0a857eb96935..fca3362f9a8b 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
@@ -115,30 +115,29 @@ static void generateInstSeqImpl(int64_t Val, const MCSubtargetInfo &STI,
     Val >>= ShiftAmount;
 
     // If the remaining bits don't fit in 12 bits, we might be able to reduce
-    // the // shift amount in order to use LUI which will zero the lower 12
-    // bits.
+    // the shift amount in order to use LUI which will zero the lower 12 bits.
     if (ShiftAmount > 12 && !isInt<12>(Val)) {
-      if (isInt<32>((uint64_t)Val << 12)) {
+      if (isInt<32>(Val << 12)) {
         // Reduce the shift amount and add zeros to the LSBs so it will match
         // LUI.
         ShiftAmount -= 12;
-        Val = (uint64_t)Val << 12;
-      } else if (isUInt<32>((uint64_t)Val << 12) &&
+        Val = Val << 12;
+      } else if (isUInt<32>(Val << 12) &&
                  STI.hasFeature(RISCV::FeatureStdExtZba)) {
         // Reduce the shift amount and add zeros to the LSBs so it will match
         // LUI, then shift left with SLLI.UW to clear the upper 32 set bits.
         ShiftAmount -= 12;
-        Val = ((uint64_t)Val << 12) | (0xffffffffull << 32);
+        Val = SignExtend64<32>(Val << 12);
         Unsigned = true;
       }
     }
 
     // Try to use SLLI_UW for Val when it is uint32 but not int32.
-    if (isUInt<32>((uint64_t)Val) && !isInt<32>((uint64_t)Val) &&
+    if (isUInt<32>(Val) && !isInt<32>(Val) &&
         STI.hasFeature(RISCV::FeatureStdExtZba)) {
-      // Use LUI+ADDI or LUI to compose, then clear the upper 32 bits with
+      // Use LUI+ADDI(W) or LUI to compose, then clear the upper 32 bits with
       // SLLI_UW.
-      Val = ((uint64_t)Val) | (0xffffffffull << 32);
+      Val = SignExtend64<32>(Val);
       Unsigned = true;
     }
   }
-- 
GitLab


From ec36145f58d2cf93d86bc4e3be617ad7d7d8ace7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 15 May 2024 18:54:23 +0100
Subject: [PATCH 0897/1206] [LAA] Add tests with invariant dependences before
 strided ones.

Add extra test coverage for loops with strided and invariant accesses to
the same object.
---
 .../invariant-dependence-before.ll            | 756 ++++++++++++++++++
 1 file changed, 756 insertions(+)
 create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/invariant-dependence-before.ll

diff --git a/llvm/test/Analysis/LoopAccessAnalysis/invariant-dependence-before.ll b/llvm/test/Analysis/LoopAccessAnalysis/invariant-dependence-before.ll
new file mode 100644
index 000000000000..2a210a5a445b
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/invariant-dependence-before.ll
@@ -0,0 +1,756 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck %s
+
+define void @test_invar_dependence_before_positive_strided_access_1(ptr %a) {
+; CHECK-LABEL: 'test_invar_dependence_before_positive_strided_access_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %a, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 4
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %a
+  store i32 %l, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_before_positive_strided_access_2(ptr %a) {
+; CHECK-LABEL: 'test_invar_dependence_before_positive_strided_access_2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %a, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 4
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %gep
+  store i32 %l, ptr %a
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_not_before_positive_strided_access_1(ptr %a) {
+; CHECK-LABEL: 'test_invar_dependence_not_before_positive_strided_access_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %a, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 3
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %a
+  store i32 %l, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_not_before_positive_strided_access_2(ptr %a) {
+; CHECK-LABEL: 'test_invar_dependence_not_before_positive_strided_access_2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %a, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 3
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %gep
+  store i32 %l, ptr %a
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_before_positive_strided_access_1_different_access_sizes(ptr %a) {
+; CHECK-LABEL: 'test_invar_dependence_before_positive_strided_access_1_different_access_sizes'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %a, align 4 ->
+; CHECK-NEXT:            store i8 %t, ptr %gep, align 1
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 4
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %a
+  %t = trunc i32 %l to i8
+  store i8 %t, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_not_before_positive_strided_access_1_different_access_sizes(ptr %a) {
+; CHECK-LABEL: 'test_invar_dependence_not_before_positive_strided_access_1_different_access_sizes'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i64, ptr %a, align 4 ->
+; CHECK-NEXT:            store i32 %t, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 4
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i64, ptr %a
+  %t = trunc i64 %l to i32
+  store i32 %t, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_before_negative_strided_access_1(ptr %a) {
+; CHECK-LABEL: 'test_invar_dependence_before_negative_strided_access_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %a, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i32, ptr %a, i32 100
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %a
+  store i32 %l, ptr %gep
+  %iv.next = sub i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, -100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_before_negative_strided_access_2(ptr %a) {
+; CHECK-LABEL: 'test_invar_dependence_before_negative_strided_access_2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %a, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i32, ptr %a, i32 100
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %gep
+  store i32 %l, ptr %a
+  %iv.next = sub i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, -100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+
+define void @test_invar_dependence_not_before_negative_strided_access_1(ptr %a) {
+; CHECK-LABEL: 'test_invar_dependence_not_before_negative_strided_access_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %a, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i32, ptr %a, i32 99
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %a
+  store i32 %l, ptr %gep
+  %iv.next = sub i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, -100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_not_before_negative_strided_access_2(ptr %a) {
+; CHECK-LABEL: 'test_invar_dependence_not_before_negative_strided_access_2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %a, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i32, ptr %a, i32 99
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %gep
+  store i32 %l, ptr %a
+  %iv.next = sub i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, -100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_both_invar_before_1(ptr %a) {
+; CHECK-LABEL: 'test_both_invar_before_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %a, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep.off, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 4
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %l = load i32, ptr %a
+  store i32 %l, ptr %gep.off
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_both_invar_before_2(ptr %a) {
+; CHECK-LABEL: 'test_both_invar_before_2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep.off, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %a, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 4
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %l = load i32, ptr %gep.off
+  store i32 %l, ptr %a
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_both_invar_not_before_1(ptr %a) {
+; CHECK-LABEL: 'test_both_invar_not_before_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %a, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep.off, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 3
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %l = load i32, ptr %a
+  store i32 %l, ptr %gep.off
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_both_invar_not_before_2(ptr %a) {
+; CHECK-LABEL: 'test_both_invar_not_before_2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep.off, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %a, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 3
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %l = load i32, ptr %gep.off
+  store i32 %l, ptr %a
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_before_via_loop_guard_positive_strided_access_1(ptr %a, i32 %off) {
+; CHECK-LABEL: 'test_invar_dependence_before_via_loop_guard_positive_strided_access_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %a, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 %off
+  %c = icmp sge i32 %off, 4
+  br i1 %c, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %a
+  store i32 %l, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_before_via_loop_guard_positive_strided_access_2(ptr %a, i32 %off) {
+; CHECK-LABEL: 'test_invar_dependence_before_via_loop_guard_positive_strided_access_2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %a, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 %off
+  %c = icmp sge i32 %off, 4
+  br i1 %c, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %gep
+  store i32 %l, ptr %a
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+define void @test_invar_dependence_not_before_via_loop_guard_positive_strided_access_1(ptr %a, i32 %off) {
+; CHECK-LABEL: 'test_invar_dependence_not_before_via_loop_guard_positive_strided_access_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %a, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 %off
+  %c = icmp sge i32 %off, 3
+  br i1 %c, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %a
+  store i32 %l, ptr %gep
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_not_before_via_loop_guard_positive_strided_access_2(ptr %a, i32 %off) {
+; CHECK-LABEL: 'test_invar_dependence_not_before_via_loop_guard_positive_strided_access_2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            %l = load i32, ptr %gep, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %a, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 %off
+  %c = icmp sge i32 %off, 3
+  br i1 %c, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %gep
+  store i32 %l, ptr %a
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_before_positive_strided_access_via_loop_guard_1(ptr %a, i32 %off) {
+; CHECK-LABEL: 'test_invar_dependence_before_positive_strided_access_via_loop_guard_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: could not determine number of loop iterations
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 4
+  %c = icmp sge i32 %off, 0
+  br i1 %c, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %gep
+  store i32 %l, ptr %a
+  %iv.next = add i32 %iv, %off
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_before_positive_strided_access_via_loop_guard_2(ptr %a, i32 %off) {
+; CHECK-LABEL: 'test_invar_dependence_before_positive_strided_access_via_loop_guard_2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: could not determine number of loop iterations
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 4
+  %c = icmp sge i32 %off, 0
+  br i1 %c, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %a
+  store i32 %l, ptr %gep
+  %iv.next = add i32 %iv, %off
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_not_known_beforepositive_strided_access_not_known_via_loop_guard_1(ptr %a, i32 %off) {
+; CHECK-LABEL: 'test_invar_dependence_not_known_beforepositive_strided_access_not_known_via_loop_guard_1'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: could not determine number of loop iterations
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 4
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %a
+  store i32 %l, ptr %gep
+  %iv.next = add i32 %iv, %off
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_invar_dependence_not_known_beforepositive_strided_access_not_known_via_loop_guard_2(ptr %a, i32 %off) {
+; CHECK-LABEL: 'test_invar_dependence_not_known_beforepositive_strided_access_not_known_via_loop_guard_2'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: could not determine number of loop iterations
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.off = getelementptr i8, ptr %a, i32 4
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep = getelementptr i32, ptr %gep.off, i32 %iv
+  %l = load i32, ptr %gep
+  store i32 %l, ptr %a
+  %iv.next = add i32 %iv, %off
+  %ec = icmp eq i32 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
-- 
GitLab


From c19f2c773b0e23fd623502888894add822079f63 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Tue, 14 May 2024 18:11:53 -0700
Subject: [PATCH 0898/1206] Reapply "[ctx_profile] Profile reader and writer"
 (#92199)

This reverts commit 03c7458a3603396d2d0e1dee43399d3d1664a264.

One of the problems was addressed in #92208

The other problem: needed to add `BitstreamReader` to the list of
link deps of `LLVMProfileData`
---
 .../llvm/ProfileData/PGOCtxProfReader.h       |  92 +++++++
 .../llvm/ProfileData/PGOCtxProfWriter.h       |  91 +++++++
 llvm/lib/ProfileData/CMakeLists.txt           |   3 +
 llvm/lib/ProfileData/PGOCtxProfReader.cpp     | 173 ++++++++++++
 llvm/lib/ProfileData/PGOCtxProfWriter.cpp     |  49 ++++
 llvm/unittests/ProfileData/CMakeLists.txt     |   1 +
 .../PGOCtxProfReaderWriterTest.cpp            | 255 ++++++++++++++++++
 7 files changed, 664 insertions(+)
 create mode 100644 llvm/include/llvm/ProfileData/PGOCtxProfReader.h
 create mode 100644 llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
 create mode 100644 llvm/lib/ProfileData/PGOCtxProfReader.cpp
 create mode 100644 llvm/lib/ProfileData/PGOCtxProfWriter.cpp
 create mode 100644 llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp

diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
new file mode 100644
index 000000000000..a19b3f51d642
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
@@ -0,0 +1,92 @@
+//===--- PGOCtxProfReader.h - Contextual profile reader ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// Reader for contextual iFDO profile, which comes in bitstream format.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
+#define LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
+
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/Bitstream/BitstreamReader.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Support/Error.h"
+#include <map>
+#include <vector>
+
+namespace llvm {
+/// The loaded contextual profile, suitable for mutation during IPO passes. We
+/// generally expect a fraction of counters and of callsites to be populated.
+/// We continue to model counters as vectors, but callsites are modeled as a map
+/// of a map. The expectation is that, typically, there is a small number of
+/// indirect targets (usually, 1 for direct calls); but potentially a large
+/// number of callsites, and, as inlining progresses, the callsite count of a
+/// caller will grow.
+class PGOContextualProfile final {
+public:
+  using CallTargetMapTy = std::map<GlobalValue::GUID, PGOContextualProfile>;
+  using CallsiteMapTy = DenseMap<uint32_t, CallTargetMapTy>;
+
+private:
+  friend class PGOCtxProfileReader;
+  GlobalValue::GUID GUID = 0;
+  SmallVector<uint64_t, 16> Counters;
+  CallsiteMapTy Callsites;
+
+  PGOContextualProfile(GlobalValue::GUID G,
+                       SmallVectorImpl<uint64_t> &&Counters)
+      : GUID(G), Counters(std::move(Counters)) {}
+
+  Expected<PGOContextualProfile &>
+  getOrEmplace(uint32_t Index, GlobalValue::GUID G,
+               SmallVectorImpl<uint64_t> &&Counters);
+
+public:
+  PGOContextualProfile(const PGOContextualProfile &) = delete;
+  PGOContextualProfile &operator=(const PGOContextualProfile &) = delete;
+  PGOContextualProfile(PGOContextualProfile &&) = default;
+  PGOContextualProfile &operator=(PGOContextualProfile &&) = default;
+
+  GlobalValue::GUID guid() const { return GUID; }
+  const SmallVectorImpl<uint64_t> &counters() const { return Counters; }
+  const CallsiteMapTy &callsites() const { return Callsites; }
+  CallsiteMapTy &callsites() { return Callsites; }
+
+  bool hasCallsite(uint32_t I) const {
+    return Callsites.find(I) != Callsites.end();
+  }
+
+  const CallTargetMapTy &callsite(uint32_t I) const {
+    assert(hasCallsite(I) && "Callsite not found");
+    return Callsites.find(I)->second;
+  }
+  void getContainedGuids(DenseSet<GlobalValue::GUID> &Guids) const;
+};
+
+class PGOCtxProfileReader final {
+  BitstreamCursor &Cursor;
+  Expected<BitstreamEntry> advance();
+  Error readMetadata();
+  Error wrongValue(const Twine &);
+  Error unsupported(const Twine &);
+
+  Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
+  readContext(bool ExpectIndex);
+  bool canReadContext();
+
+public:
+  PGOCtxProfileReader(BitstreamCursor &Cursor) : Cursor(Cursor) {}
+
+  Expected<std::map<GlobalValue::GUID, PGOContextualProfile>> loadContexts();
+};
+} // namespace llvm
+#endif
diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
new file mode 100644
index 000000000000..15578c51a495
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
@@ -0,0 +1,91 @@
+//===- PGOCtxProfWriter.h - Contextual Profile Writer -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares a utility for writing a contextual profile to bitstream.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
+#define LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
+
+#include "llvm/Bitstream/BitstreamWriter.h"
+#include "llvm/ProfileData/CtxInstrContextNode.h"
+
+namespace llvm {
+enum PGOCtxProfileRecords { Invalid = 0, Version, Guid, CalleeIndex, Counters };
+
+enum PGOCtxProfileBlockIDs {
+  ProfileMetadataBlockID = 100,
+  ContextNodeBlockID = ProfileMetadataBlockID + 1
+};
+
+/// Write one or more ContextNodes to the provided raw_fd_stream.
+/// The caller must destroy the PGOCtxProfileWriter object before closing the
+/// stream.
+/// The design allows serializing a bunch of contexts embedded in some other
+/// file. The overall format is:
+///
+///  [... other data written to the stream...]
+///  SubBlock(ProfileMetadataBlockID)
+///   Version
+///   SubBlock(ContextNodeBlockID)
+///     [RECORDS]
+///     SubBlock(ContextNodeBlockID)
+///       [RECORDS]
+///       [... more SubBlocks]
+///     EndBlock
+///   EndBlock
+///
+/// The "RECORDS" are bitsream records. The IDs are in CtxProfileCodes (except)
+/// for Version, which is just for metadata). All contexts will have Guid and
+/// Counters, and all but the roots have CalleeIndex. The order in which the
+/// records appear does not matter, but they must precede any subcontexts,
+/// because that helps keep the reader code simpler.
+///
+/// Subblock containment captures the context->subcontext relationship. The
+/// "next()" relationship in the raw profile, between call targets of indirect
+/// calls, are just modeled as peer subblocks where the callee index is the
+/// same.
+///
+/// Versioning: the writer may produce additional records not known by the
+/// reader. The version number indicates a more structural change.
+/// The current version, in particular, is set up to expect optional extensions
+/// like value profiling - which would appear as additional records. For
+/// example, value profiling would produce a new record with a new record ID,
+/// containing the profiled values (much like the counters)
+class PGOCtxProfileWriter final {
+  SmallVector<char, 1 << 20> Buff;
+  BitstreamWriter Writer;
+
+  void writeCounters(const ctx_profile::ContextNode &Node);
+  void writeImpl(std::optional<uint32_t> CallerIndex,
+                 const ctx_profile::ContextNode &Node);
+
+public:
+  PGOCtxProfileWriter(raw_fd_stream &Out,
+                      std::optional<unsigned> VersionOverride = std::nullopt)
+      : Writer(Buff, &Out, 0) {
+    Writer.EnterSubblock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID,
+                         CodeLen);
+    const auto Version = VersionOverride ? *VersionOverride : CurrentVersion;
+    Writer.EmitRecord(PGOCtxProfileRecords::Version,
+                      SmallVector<unsigned, 1>({Version}));
+  }
+
+  ~PGOCtxProfileWriter() { Writer.ExitBlock(); }
+
+  void write(const ctx_profile::ContextNode &);
+
+  // constants used in writing which a reader may find useful.
+  static constexpr unsigned CodeLen = 2;
+  static constexpr uint32_t CurrentVersion = 1;
+  static constexpr unsigned VBREncodingBits = 6;
+};
+
+} // namespace llvm
+#endif
diff --git a/llvm/lib/ProfileData/CMakeLists.txt b/llvm/lib/ProfileData/CMakeLists.txt
index 408f9ff01ec8..4fa1b76f0a06 100644
--- a/llvm/lib/ProfileData/CMakeLists.txt
+++ b/llvm/lib/ProfileData/CMakeLists.txt
@@ -7,6 +7,8 @@ add_llvm_component_library(LLVMProfileData
   ItaniumManglingCanonicalizer.cpp
   MemProf.cpp
   MemProfReader.cpp
+  PGOCtxProfReader.cpp
+  PGOCtxProfWriter.cpp
   ProfileSummaryBuilder.cpp
   SampleProf.cpp
   SampleProfReader.cpp
@@ -20,6 +22,7 @@ add_llvm_component_library(LLVMProfileData
   intrinsics_gen
 
   LINK_COMPONENTS
+  BitstreamReader
   Core
   Object
   Support
diff --git a/llvm/lib/ProfileData/PGOCtxProfReader.cpp b/llvm/lib/ProfileData/PGOCtxProfReader.cpp
new file mode 100644
index 000000000000..3710f2e4b818
--- /dev/null
+++ b/llvm/lib/ProfileData/PGOCtxProfReader.cpp
@@ -0,0 +1,173 @@
+//===- PGOCtxProfReader.cpp - Contextual Instrumentation profile reader ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Read a contextual profile into a datastructure suitable for maintenance
+// throughout IPO
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ProfileData/PGOCtxProfReader.h"
+#include "llvm/Bitstream/BitCodeEnums.h"
+#include "llvm/Bitstream/BitstreamReader.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+
+using namespace llvm;
+
+// FIXME(#92054) - these Error handling macros are (re-)invented in a few
+// places.
+#define EXPECT_OR_RET(LHS, RHS)                                                \
+  auto LHS = RHS;                                                              \
+  if (!LHS)                                                                    \
+    return LHS.takeError();
+
+#define RET_ON_ERR(EXPR)                                                       \
+  if (auto Err = (EXPR))                                                       \
+    return Err;
+
+Expected<PGOContextualProfile &>
+PGOContextualProfile::getOrEmplace(uint32_t Index, GlobalValue::GUID G,
+                                   SmallVectorImpl<uint64_t> &&Counters) {
+  auto [Iter, Inserted] = Callsites[Index].insert(
+      {G, PGOContextualProfile(G, std::move(Counters))});
+  if (!Inserted)
+    return make_error<InstrProfError>(instrprof_error::invalid_prof,
+                                      "Duplicate GUID for same callsite.");
+  return Iter->second;
+}
+
+void PGOContextualProfile::getContainedGuids(
+    DenseSet<GlobalValue::GUID> &Guids) const {
+  Guids.insert(GUID);
+  for (const auto &[_, Callsite] : Callsites)
+    for (const auto &[_, Callee] : Callsite)
+      Callee.getContainedGuids(Guids);
+}
+
+Expected<BitstreamEntry> PGOCtxProfileReader::advance() {
+  return Cursor.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs);
+}
+
+Error PGOCtxProfileReader::wrongValue(const Twine &Msg) {
+  return make_error<InstrProfError>(instrprof_error::invalid_prof, Msg);
+}
+
+Error PGOCtxProfileReader::unsupported(const Twine &Msg) {
+  return make_error<InstrProfError>(instrprof_error::unsupported_version, Msg);
+}
+
+bool PGOCtxProfileReader::canReadContext() {
+  auto Blk = advance();
+  if (!Blk) {
+    consumeError(Blk.takeError());
+    return false;
+  }
+  return Blk->Kind == BitstreamEntry::SubBlock &&
+         Blk->ID == PGOCtxProfileBlockIDs::ContextNodeBlockID;
+}
+
+Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
+PGOCtxProfileReader::readContext(bool ExpectIndex) {
+  RET_ON_ERR(Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID));
+
+  std::optional<ctx_profile::GUID> Guid;
+  std::optional<SmallVector<uint64_t, 16>> Counters;
+  std::optional<uint32_t> CallsiteIndex;
+
+  SmallVector<uint64_t, 1> RecordValues;
+
+  // We don't prescribe the order in which the records come in, and we are ok
+  // if other unsupported records appear. We seek in the current subblock until
+  // we get all we know.
+  auto GotAllWeNeed = [&]() {
+    return Guid.has_value() && Counters.has_value() &&
+           (!ExpectIndex || CallsiteIndex.has_value());
+  };
+  while (!GotAllWeNeed()) {
+    RecordValues.clear();
+    EXPECT_OR_RET(Entry, advance());
+    if (Entry->Kind != BitstreamEntry::Record)
+      return wrongValue(
+          "Expected records before encountering more subcontexts");
+    EXPECT_OR_RET(ReadRecord,
+                  Cursor.readRecord(bitc::UNABBREV_RECORD, RecordValues));
+    switch (*ReadRecord) {
+    case PGOCtxProfileRecords::Guid:
+      if (RecordValues.size() != 1)
+        return wrongValue("The GUID record should have exactly one value");
+      Guid = RecordValues[0];
+      break;
+    case PGOCtxProfileRecords::Counters:
+      Counters = std::move(RecordValues);
+      if (Counters->empty())
+        return wrongValue("Empty counters. At least the entry counter (one "
+                          "value) was expected");
+      break;
+    case PGOCtxProfileRecords::CalleeIndex:
+      if (!ExpectIndex)
+        return wrongValue("The root context should not have a callee index");
+      if (RecordValues.size() != 1)
+        return wrongValue("The callee index should have exactly one value");
+      CallsiteIndex = RecordValues[0];
+      break;
+    default:
+      // OK if we see records we do not understand, like records (profile
+      // components) introduced later.
+      break;
+    }
+  }
+
+  PGOContextualProfile Ret(*Guid, std::move(*Counters));
+
+  while (canReadContext()) {
+    EXPECT_OR_RET(SC, readContext(true));
+    auto &Targets = Ret.callsites()[*SC->first];
+    auto [_, Inserted] =
+        Targets.insert({SC->second.guid(), std::move(SC->second)});
+    if (!Inserted)
+      return wrongValue(
+          "Unexpected duplicate target (callee) at the same callsite.");
+  }
+  return std::make_pair(CallsiteIndex, std::move(Ret));
+}
+
+Error PGOCtxProfileReader::readMetadata() {
+  EXPECT_OR_RET(Blk, advance());
+  if (Blk->Kind != BitstreamEntry::SubBlock)
+    return unsupported("Expected Version record");
+  RET_ON_ERR(
+      Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID));
+  EXPECT_OR_RET(MData, advance());
+  if (MData->Kind != BitstreamEntry::Record)
+    return unsupported("Expected Version record");
+
+  SmallVector<uint64_t, 1> Ver;
+  EXPECT_OR_RET(Code, Cursor.readRecord(bitc::UNABBREV_RECORD, Ver));
+  if (*Code != PGOCtxProfileRecords::Version)
+    return unsupported("Expected Version record");
+  if (Ver.size() != 1 || Ver[0] > PGOCtxProfileWriter::CurrentVersion)
+    return unsupported("Version " + Twine(*Code) +
+                       " is higher than supported version " +
+                       Twine(PGOCtxProfileWriter::CurrentVersion));
+  return Error::success();
+}
+
+Expected<std::map<GlobalValue::GUID, PGOContextualProfile>>
+PGOCtxProfileReader::loadContexts() {
+  std::map<GlobalValue::GUID, PGOContextualProfile> Ret;
+  RET_ON_ERR(readMetadata());
+  while (canReadContext()) {
+    EXPECT_OR_RET(E, readContext(false));
+    auto Key = E->second.guid();
+    if (!Ret.insert({Key, std::move(E->second)}).second)
+      return wrongValue("Duplicate roots");
+  }
+  return Ret;
+}
diff --git a/llvm/lib/ProfileData/PGOCtxProfWriter.cpp b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
new file mode 100644
index 000000000000..508179756446
--- /dev/null
+++ b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
@@ -0,0 +1,49 @@
+//===- PGOCtxProfWriter.cpp - Contextual Instrumentation profile writer ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Write a contextual profile to bitstream.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Bitstream/BitCodeEnums.h"
+
+using namespace llvm;
+using namespace llvm::ctx_profile;
+
+void PGOCtxProfileWriter::writeCounters(const ContextNode &Node) {
+  Writer.EmitCode(bitc::UNABBREV_RECORD);
+  Writer.EmitVBR(PGOCtxProfileRecords::Counters, VBREncodingBits);
+  Writer.EmitVBR(Node.counters_size(), VBREncodingBits);
+  for (uint32_t I = 0U; I < Node.counters_size(); ++I)
+    Writer.EmitVBR64(Node.counters()[I], VBREncodingBits);
+}
+
+// recursively write all the subcontexts. We do need to traverse depth first to
+// model the context->subcontext implicitly, and since this captures call
+// stacks, we don't really need to be worried about stack overflow and we can
+// keep the implementation simple.
+void PGOCtxProfileWriter::writeImpl(std::optional<uint32_t> CallerIndex,
+                                    const ContextNode &Node) {
+  Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextNodeBlockID, CodeLen);
+  Writer.EmitRecord(PGOCtxProfileRecords::Guid,
+                    SmallVector<uint64_t, 1>{Node.guid()});
+  if (CallerIndex)
+    Writer.EmitRecord(PGOCtxProfileRecords::CalleeIndex,
+                      SmallVector<uint64_t, 1>{*CallerIndex});
+  writeCounters(Node);
+  for (uint32_t I = 0U; I < Node.callsites_size(); ++I)
+    for (const auto *Subcontext = Node.subContexts()[I]; Subcontext;
+         Subcontext = Subcontext->next())
+      writeImpl(I, *Subcontext);
+  Writer.ExitBlock();
+}
+
+void PGOCtxProfileWriter::write(const ContextNode &RootNode) {
+  writeImpl(std::nullopt, RootNode);
+}
diff --git a/llvm/unittests/ProfileData/CMakeLists.txt b/llvm/unittests/ProfileData/CMakeLists.txt
index ce3a0a45ccf1..c92642ded828 100644
--- a/llvm/unittests/ProfileData/CMakeLists.txt
+++ b/llvm/unittests/ProfileData/CMakeLists.txt
@@ -13,6 +13,7 @@ add_llvm_unittest(ProfileDataTests
   InstrProfTest.cpp
   ItaniumManglingCanonicalizerTest.cpp
   MemProfTest.cpp
+  PGOCtxProfReaderWriterTest.cpp
   SampleProfTest.cpp
   SymbolRemappingReaderTest.cpp
   )
diff --git a/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp b/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp
new file mode 100644
index 000000000000..d2cdbb28e2fc
--- /dev/null
+++ b/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp
@@ -0,0 +1,255 @@
+//===-------------- PGOCtxProfReadWriteTest.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Bitstream/BitstreamReader.h"
+#include "llvm/ProfileData/CtxInstrContextNode.h"
+#include "llvm/ProfileData/PGOCtxProfReader.h"
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Testing/Support/SupportHelpers.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::ctx_profile;
+
+class PGOCtxProfRWTest : public ::testing::Test {
+  std::vector<std::unique_ptr<char[]>> Nodes;
+  std::map<GUID, const ContextNode *> Roots;
+
+public:
+  ContextNode *createNode(GUID Guid, uint32_t NrCounters, uint32_t NrCallsites,
+                          ContextNode *Next = nullptr) {
+    auto AllocSize = ContextNode::getAllocSize(NrCounters, NrCallsites);
+    auto *Mem = Nodes.emplace_back(std::make_unique<char[]>(AllocSize)).get();
+    std::memset(Mem, 0, AllocSize);
+    auto *Ret = new (Mem) ContextNode(Guid, NrCounters, NrCallsites, Next);
+    return Ret;
+  }
+
+  void SetUp() override {
+    // Root (guid 1) has 2 callsites, one used for an indirect call to either
+    // guid 2 or 4.
+    // guid 2 calls guid 5
+    // guid 5 calls guid 2
+    // there's also a second root, guid3.
+    auto *Root1 = createNode(1, 2, 2);
+    Root1->counters()[0] = 10;
+    Root1->counters()[1] = 11;
+    Roots.insert({1, Root1});
+    auto *L1 = createNode(2, 1, 1);
+    L1->counters()[0] = 12;
+    Root1->subContexts()[1] = createNode(4, 3, 1, L1);
+    Root1->subContexts()[1]->counters()[0] = 13;
+    Root1->subContexts()[1]->counters()[1] = 14;
+    Root1->subContexts()[1]->counters()[2] = 15;
+
+    auto *L3 = createNode(5, 6, 3);
+    for (auto I = 0; I < 6; ++I)
+      L3->counters()[I] = 16 + I;
+    L1->subContexts()[0] = L3;
+    L3->subContexts()[2] = createNode(2, 1, 1);
+    L3->subContexts()[2]->counters()[0] = 30;
+    auto *Root2 = createNode(3, 1, 0);
+    Root2->counters()[0] = 40;
+    Roots.insert({3, Root2});
+  }
+
+  const std::map<GUID, const ContextNode *> &roots() const { return Roots; }
+};
+
+void checkSame(const ContextNode &Raw, const PGOContextualProfile &Profile) {
+  EXPECT_EQ(Raw.guid(), Profile.guid());
+  ASSERT_EQ(Raw.counters_size(), Profile.counters().size());
+  for (auto I = 0U; I < Raw.counters_size(); ++I)
+    EXPECT_EQ(Raw.counters()[I], Profile.counters()[I]);
+
+  for (auto I = 0U; I < Raw.callsites_size(); ++I) {
+    if (Raw.subContexts()[I] == nullptr)
+      continue;
+    EXPECT_TRUE(Profile.hasCallsite(I));
+    const auto &ProfileTargets = Profile.callsite(I);
+
+    std::map<GUID, const ContextNode *> Targets;
+    for (const auto *N = Raw.subContexts()[I]; N; N = N->next())
+      EXPECT_TRUE(Targets.insert({N->guid(), N}).second);
+
+    EXPECT_EQ(Targets.size(), ProfileTargets.size());
+    for (auto It : Targets) {
+      auto PIt = ProfileTargets.find(It.second->guid());
+      EXPECT_NE(PIt, ProfileTargets.end());
+      checkSame(*It.second, PIt->second);
+    }
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, RoundTrip) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      for (auto &[_, R] : roots())
+        Writer.write(*R);
+    }
+  }
+  {
+    ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
+        MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    ASSERT_TRUE(!!Expected);
+    auto &Ctxes = *Expected;
+    EXPECT_EQ(Ctxes.size(), roots().size());
+    EXPECT_EQ(Ctxes.size(), 2U);
+    for (auto &[G, R] : roots())
+      checkSame(*R, Ctxes.find(G)->second);
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, InvalidCounters) {
+  auto *R = createNode(1, 0, 1);
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      Writer.write(*R);
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, Empty) {
+  BitstreamCursor Cursor("");
+  PGOCtxProfileReader Reader(Cursor);
+  auto Expected = Reader.loadContexts();
+  EXPECT_FALSE(Expected);
+  consumeError(Expected.takeError());
+}
+
+TEST_F(PGOCtxProfRWTest, Invalid) {
+  BitstreamCursor Cursor("Surely this is not valid");
+  PGOCtxProfileReader Reader(Cursor);
+  auto Expected = Reader.loadContexts();
+  EXPECT_FALSE(Expected);
+  consumeError(Expected.takeError());
+}
+
+TEST_F(PGOCtxProfRWTest, ValidButEmpty) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      // don't write anything - this will just produce the metadata subblock.
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_TRUE(!!Expected);
+    EXPECT_TRUE(Expected->empty());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, WrongVersion) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out, PGOCtxProfileWriter::CurrentVersion + 1);
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, DuplicateRoots) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      Writer.write(*createNode(1, 1, 1));
+      Writer.write(*createNode(1, 1, 1));
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, DuplicateTargets) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      auto *R = createNode(1, 1, 1);
+      auto *L1 = createNode(2, 1, 0);
+      auto *L2 = createNode(2, 1, 0, L1);
+      R->subContexts()[0] = L2;
+      PGOCtxProfileWriter Writer(Out);
+      Writer.write(*R);
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
-- 
GitLab


From df5804aec48f99704ef26c740e19deaa4072fe27 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Wed, 15 May 2024 18:08:23 +0000
Subject: [PATCH 0899/1206] [gn build] Port c19f2c773b0e

---
 llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn       | 2 ++
 llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn | 1 +
 2 files changed, 3 insertions(+)

diff --git a/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
index 9dbfe0f94c1d..c6fa142b3766 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
@@ -17,6 +17,8 @@ static_library("ProfileData") {
     "ItaniumManglingCanonicalizer.cpp",
     "MemProf.cpp",
     "MemProfReader.cpp",
+    "PGOCtxProfReader.cpp",
+    "PGOCtxProfWriter.cpp",
     "ProfileSummaryBuilder.cpp",
     "SampleProf.cpp",
     "SampleProfReader.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
index 4919a8089209..f45542519173 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
@@ -14,6 +14,7 @@ unittest("ProfileDataTests") {
     "InstrProfTest.cpp",
     "ItaniumManglingCanonicalizerTest.cpp",
     "MemProfTest.cpp",
+    "PGOCtxProfReaderWriterTest.cpp",
     "SampleProfTest.cpp",
     "SymbolRemappingReaderTest.cpp",
   ]
-- 
GitLab


From 468357114c64633651ebcc5ef17161990da25a78 Mon Sep 17 00:00:00 2001
From: Pete Steinfeld <47540744+psteinfeld@users.noreply.github.com>
Date: Wed, 15 May 2024 11:30:30 -0700
Subject: [PATCH 0900/1206] =?UTF-8?q?Revert=20"[flang]=20Initial=20debug?=
 =?UTF-8?q?=20info=20support=20for=20local=20variables.=20(#909=E2=80=A6?=
 =?UTF-8?q?=20(#92302)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…05)"

This reverts commit 61da6366d043792d7db280ce9edd2db62516e0e8.

Update #90905 was causing many tests to fail.

See comments in #90905.
---
 .../include/flang/Optimizer/CodeGen/CGOps.td  | 34 -------
 .../flang/Optimizer/CodeGen/CGPasses.td       |  4 -
 .../include/flang/Optimizer/CodeGen/CodeGen.h |  6 +-
 flang/include/flang/Tools/CLOptions.inc       | 11 +--
 flang/lib/Optimizer/CodeGen/CGOps.cpp         |  2 +-
 .../flang => lib}/Optimizer/CodeGen/CGOps.h   |  1 -
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 50 +++-------
 flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp  | 49 ++--------
 .../lib/Optimizer/Transforms/AddDebugInfo.cpp | 56 +-----------
 .../Transforms/DebugTypeGenerator.cpp         | 10 +-
 flang/test/Fir/declare-codegen.fir            | 22 ++---
 flang/test/Fir/dummy-scope-codegen.fir        | 11 +--
 flang/test/Transforms/debug-local-var-2.f90   | 91 -------------------
 flang/test/Transforms/debug-local-var.f90     | 54 -----------
 14 files changed, 47 insertions(+), 354 deletions(-)
 rename flang/{include/flang => lib}/Optimizer/CodeGen/CGOps.h (94%)
 delete mode 100644 flang/test/Transforms/debug-local-var-2.f90
 delete mode 100644 flang/test/Transforms/debug-local-var.f90

diff --git a/flang/include/flang/Optimizer/CodeGen/CGOps.td b/flang/include/flang/Optimizer/CodeGen/CGOps.td
index c375edee1fa7..35e70fa2ffa3 100644
--- a/flang/include/flang/Optimizer/CodeGen/CGOps.td
+++ b/flang/include/flang/Optimizer/CodeGen/CGOps.td
@@ -16,8 +16,6 @@
 
 include "mlir/IR/SymbolInterfaces.td"
 include "flang/Optimizer/Dialect/FIRTypes.td"
-include "flang/Optimizer/Dialect/FIRAttr.td"
-include "mlir/IR/BuiltinAttributes.td"
 
 def fircg_Dialect : Dialect {
   let name = "fircg";
@@ -204,36 +202,4 @@ def fircg_XArrayCoorOp : fircg_Op<"ext_array_coor", [AttrSizedOperandSegments]>
   }];
 }
 
-// Extended Declare operation.
-def fircg_XDeclareOp : fircg_Op<"ext_declare", [AttrSizedOperandSegments]> {
-  let summary = "for internal conversion only";
-
-  let description = [{
-    Prior to lowering to LLVM IR dialect, a DeclareOp will
-    be converted to an extended DeclareOp.
-  }];
-
-  let arguments = (ins
-    AnyRefOrBox:$memref,
-    Variadic<AnyIntegerType>:$shape,
-    Variadic<AnyIntegerType>:$shift,
-    Variadic<AnyIntegerType>:$typeparams,
-    Optional<fir_DummyScopeType>:$dummy_scope,
-    Builtin_StringAttr:$uniq_name
-  );
-  let results = (outs AnyRefOrBox);
-
-  let assemblyFormat = [{
-    $memref (`(` $shape^ `)`)? (`origin` $shift^)? (`typeparams` $typeparams^)?
-    (`dummy_scope` $dummy_scope^)?
-    attr-dict `:` functional-type(operands, results)
-  }];
-
-  let extraClassDeclaration = [{
-    // Shape is optional, but if it exists, it will be at offset 1.
-    unsigned shapeOffset() { return 1; }
-    unsigned shiftOffset() { return shapeOffset() + getShape().size(); }
-  }];
-}
-
 #endif
diff --git a/flang/include/flang/Optimizer/CodeGen/CGPasses.td b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
index 565920e55e6a..f524fb423734 100644
--- a/flang/include/flang/Optimizer/CodeGen/CGPasses.td
+++ b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
@@ -47,10 +47,6 @@ def CodeGenRewrite : Pass<"cg-rewrite", "mlir::ModuleOp"> {
   let dependentDialects = [
     "fir::FIROpsDialect", "fir::FIRCodeGenDialect"
   ];
-  let options = [
-    Option<"preserveDeclare", "preserve-declare", "bool", /*default=*/"false",
-           "Preserve DeclareOp during pre codegen re-write.">
-  ];
   let statistics = [
     Statistic<"numDCE", "num-dce'd", "Number of operations eliminated">
   ];
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 4d2b191b46d0..26097dabf56c 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -30,8 +30,7 @@ struct NameUniquer;
 
 /// Prerequiste pass for code gen. Perform intermediate rewrites to perform
 /// the code gen (to LLVM-IR dialect) conversion.
-std::unique_ptr<mlir::Pass> createFirCodeGenRewritePass(
-    CodeGenRewriteOptions Options = CodeGenRewriteOptions{});
+std::unique_ptr<mlir::Pass> createFirCodeGenRewritePass();
 
 /// FirTargetRewritePass options.
 struct TargetRewriteOptions {
@@ -89,8 +88,7 @@ void populateFIRToLLVMConversionPatterns(fir::LLVMTypeConverter &converter,
                                          fir::FIRToLLVMPassOptions &options);
 
 /// Populate the pattern set with the PreCGRewrite patterns.
-void populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns,
-                                  bool preserveDeclare);
+void populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns);
 
 // declarative passes
 #define GEN_PASS_REGISTRATION
diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc
index 761315e0abc8..cc3431d5b71d 100644
--- a/flang/include/flang/Tools/CLOptions.inc
+++ b/flang/include/flang/Tools/CLOptions.inc
@@ -169,11 +169,9 @@ inline void addMemoryAllocationOpt(mlir::PassManager &pm) {
 }
 
 #if !defined(FLANG_EXCLUDE_CODEGEN)
-inline void addCodeGenRewritePass(mlir::PassManager &pm, bool preserveDeclare) {
-  fir::CodeGenRewriteOptions options;
-  options.preserveDeclare = preserveDeclare;
-  addPassConditionally(pm, disableCodeGenRewrite,
-      [&]() { return fir::createFirCodeGenRewritePass(options); });
+inline void addCodeGenRewritePass(mlir::PassManager &pm) {
+  addPassConditionally(
+      pm, disableCodeGenRewrite, fir::createFirCodeGenRewritePass);
 }
 
 inline void addTargetRewritePass(mlir::PassManager &pm) {
@@ -355,8 +353,7 @@ inline void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
     MLIRToLLVMPassPipelineConfig config, llvm::StringRef inputFilename = {}) {
   fir::addBoxedProcedurePass(pm);
   addNestedPassToAllTopLevelOperations(pm, fir::createAbstractResultOpt);
-  fir::addCodeGenRewritePass(
-      pm, (config.DebugInfo != llvm::codegenoptions::NoDebugInfo));
+  fir::addCodeGenRewritePass(pm);
   fir::addTargetRewritePass(pm);
   fir::addExternalNameConversionPass(pm, config.Underscoring);
   fir::createDebugPasses(pm, config.DebugInfo, config.OptLevel, inputFilename);
diff --git a/flang/lib/Optimizer/CodeGen/CGOps.cpp b/flang/lib/Optimizer/CodeGen/CGOps.cpp
index 6b8ba7452555..44d07d26dd2b 100644
--- a/flang/lib/Optimizer/CodeGen/CGOps.cpp
+++ b/flang/lib/Optimizer/CodeGen/CGOps.cpp
@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Optimizer/CodeGen/CGOps.h"
+#include "CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
diff --git a/flang/include/flang/Optimizer/CodeGen/CGOps.h b/flang/lib/Optimizer/CodeGen/CGOps.h
similarity index 94%
rename from flang/include/flang/Optimizer/CodeGen/CGOps.h
rename to flang/lib/Optimizer/CodeGen/CGOps.h
index df909d9ee81c..b5a6d5bb9a9e 100644
--- a/flang/include/flang/Optimizer/CodeGen/CGOps.h
+++ b/flang/lib/Optimizer/CodeGen/CGOps.h
@@ -13,7 +13,6 @@
 #ifndef OPTIMIZER_CODEGEN_CGOPS_H
 #define OPTIMIZER_CODEGEN_CGOPS_H
 
-#include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 72172f63888e..21154902d23f 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -12,7 +12,7 @@
 
 #include "flang/Optimizer/CodeGen/CodeGen.h"
 
-#include "flang/Optimizer/CodeGen/CGOps.h"
+#include "CGOps.h"
 #include "flang/Optimizer/CodeGen/CodeGenOpenMP.h"
 #include "flang/Optimizer/CodeGen/FIROpPatterns.h"
 #include "flang/Optimizer/CodeGen/TypeConverter.h"
@@ -170,28 +170,6 @@ genAllocationScaleSize(OP op, mlir::Type ity,
   return nullptr;
 }
 
-namespace {
-struct DeclareOpConversion : public fir::FIROpConversion<fir::cg::XDeclareOp> {
-public:
-  using FIROpConversion::FIROpConversion;
-  mlir::LogicalResult
-  matchAndRewrite(fir::cg::XDeclareOp declareOp, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const override {
-    auto memRef = adaptor.getOperands()[0];
-    if (auto fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(declareOp.getLoc())) {
-      if (auto varAttr =
-              mlir::dyn_cast_or_null<mlir::LLVM::DILocalVariableAttr>(
-                  fusedLoc.getMetadata())) {
-        rewriter.create<mlir::LLVM::DbgDeclareOp>(memRef.getLoc(), memRef,
-                                                  varAttr, nullptr);
-      }
-    }
-    rewriter.replaceOp(declareOp, memRef);
-    return mlir::success();
-  }
-};
-} // namespace
-
 namespace {
 /// convert to LLVM IR dialect `alloca`
 struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
@@ -3736,19 +3714,19 @@ void fir::populateFIRToLLVMConversionPatterns(
       BoxOffsetOpConversion, BoxProcHostOpConversion, BoxRankOpConversion,
       BoxTypeCodeOpConversion, BoxTypeDescOpConversion, CallOpConversion,
       CmpcOpConversion, ConstcOpConversion, ConvertOpConversion,
-      CoordinateOpConversion, DTEntryOpConversion, DeclareOpConversion,
-      DivcOpConversion, EmboxOpConversion, EmboxCharOpConversion,
-      EmboxProcOpConversion, ExtractValueOpConversion, FieldIndexOpConversion,
-      FirEndOpConversion, FreeMemOpConversion, GlobalLenOpConversion,
-      GlobalOpConversion, HasValueOpConversion, InsertOnRangeOpConversion,
-      InsertValueOpConversion, IsPresentOpConversion, LenParamIndexOpConversion,
-      LoadOpConversion, MulcOpConversion, NegcOpConversion,
-      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
-      SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
-      ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
-      StoreOpConversion, StringLitOpConversion, SubcOpConversion,
-      TypeDescOpConversion, TypeInfoOpConversion, UnboxCharOpConversion,
-      UnboxProcOpConversion, UndefOpConversion, UnreachableOpConversion,
+      CoordinateOpConversion, DTEntryOpConversion, DivcOpConversion,
+      EmboxOpConversion, EmboxCharOpConversion, EmboxProcOpConversion,
+      ExtractValueOpConversion, FieldIndexOpConversion, FirEndOpConversion,
+      FreeMemOpConversion, GlobalLenOpConversion, GlobalOpConversion,
+      HasValueOpConversion, InsertOnRangeOpConversion, InsertValueOpConversion,
+      IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
+      MulcOpConversion, NegcOpConversion, NoReassocOpConversion,
+      SelectCaseOpConversion, SelectOpConversion, SelectRankOpConversion,
+      SelectTypeOpConversion, ShapeOpConversion, ShapeShiftOpConversion,
+      ShiftOpConversion, SliceOpConversion, StoreOpConversion,
+      StringLitOpConversion, SubcOpConversion, TypeDescOpConversion,
+      TypeInfoOpConversion, UnboxCharOpConversion, UnboxProcOpConversion,
+      UndefOpConversion, UnreachableOpConversion,
       UnrealizedConversionCastOpConversion, XArrayCoorOpConversion,
       XEmboxOpConversion, XReboxOpConversion, ZeroOpConversion>(converter,
                                                                 options);
diff --git a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
index c54a7457db76..5bd3ec8d1845 100644
--- a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
@@ -12,8 +12,8 @@
 
 #include "flang/Optimizer/CodeGen/CodeGen.h"
 
+#include "CGOps.h"
 #include "flang/Optimizer/Builder/Todo.h" // remove when TODO's are done
-#include "flang/Optimizer/CodeGen/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
@@ -270,43 +270,13 @@ public:
 };
 
 class DeclareOpConversion : public mlir::OpRewritePattern<fir::DeclareOp> {
-  bool preserveDeclare;
-
 public:
   using OpRewritePattern::OpRewritePattern;
-  DeclareOpConversion(mlir::MLIRContext *ctx, bool preserveDecl)
-      : OpRewritePattern(ctx), preserveDeclare(preserveDecl) {}
 
   mlir::LogicalResult
   matchAndRewrite(fir::DeclareOp declareOp,
                   mlir::PatternRewriter &rewriter) const override {
-    if (!preserveDeclare) {
-      rewriter.replaceOp(declareOp, declareOp.getMemref());
-      return mlir::success();
-    }
-    auto loc = declareOp.getLoc();
-    llvm::SmallVector<mlir::Value> shapeOpers;
-    llvm::SmallVector<mlir::Value> shiftOpers;
-    if (auto shapeVal = declareOp.getShape()) {
-      if (auto shapeOp = mlir::dyn_cast<fir::ShapeOp>(shapeVal.getDefiningOp()))
-        populateShape(shapeOpers, shapeOp);
-      else if (auto shiftOp =
-                   mlir::dyn_cast<fir::ShapeShiftOp>(shapeVal.getDefiningOp()))
-        populateShapeAndShift(shapeOpers, shiftOpers, shiftOp);
-      else if (auto shiftOp =
-                   mlir::dyn_cast<fir::ShiftOp>(shapeVal.getDefiningOp()))
-        populateShift(shiftOpers, shiftOp);
-      else
-        return mlir::failure();
-    }
-    // FIXME: Add FortranAttrs and CudaAttrs
-    auto xDeclOp = rewriter.create<fir::cg::XDeclareOp>(
-        loc, declareOp.getType(), declareOp.getMemref(), shapeOpers, shiftOpers,
-        declareOp.getTypeparams(), declareOp.getDummyScope(),
-        declareOp.getUniqName());
-    LLVM_DEBUG(llvm::dbgs()
-               << "rewriting " << declareOp << " to " << xDeclOp << '\n');
-    rewriter.replaceOp(declareOp, xDeclOp.getOperation()->getResults());
+    rewriter.replaceOp(declareOp, declareOp.getMemref());
     return mlir::success();
   }
 };
@@ -327,7 +297,6 @@ public:
 
 class CodeGenRewrite : public fir::impl::CodeGenRewriteBase<CodeGenRewrite> {
 public:
-  CodeGenRewrite(fir::CodeGenRewriteOptions opts) : Base(opts) {}
   void runOnOperation() override final {
     mlir::ModuleOp mod = getOperation();
 
@@ -345,7 +314,7 @@ public:
                    mlir::cast<fir::BaseBoxType>(embox.getType()).getEleTy()));
     });
     mlir::RewritePatternSet patterns(&context);
-    fir::populatePreCGRewritePatterns(patterns, preserveDeclare);
+    fir::populatePreCGRewritePatterns(patterns);
     if (mlir::failed(
             mlir::applyPartialConversion(mod, target, std::move(patterns)))) {
       mlir::emitError(mlir::UnknownLoc::get(&context),
@@ -361,14 +330,12 @@ public:
 
 } // namespace
 
-std::unique_ptr<mlir::Pass>
-fir::createFirCodeGenRewritePass(fir::CodeGenRewriteOptions Options) {
-  return std::make_unique<CodeGenRewrite>(Options);
+std::unique_ptr<mlir::Pass> fir::createFirCodeGenRewritePass() {
+  return std::make_unique<CodeGenRewrite>();
 }
 
-void fir::populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns,
-                                       bool preserveDeclare) {
+void fir::populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns) {
   patterns.insert<EmboxConversion, ArrayCoorConversion, ReboxConversion,
-                  DummyScopeOpConversion>(patterns.getContext());
-  patterns.add<DeclareOpConversion>(patterns.getContext(), preserveDeclare);
+                  DeclareOpConversion, DummyScopeOpConversion>(
+      patterns.getContext());
 }
diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index cfad366cb5cb..908c8fc96f63 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -15,7 +15,6 @@
 #include "flang/Common/Version.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/Todo.h"
-#include "flang/Optimizer/CodeGen/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
@@ -46,59 +45,13 @@ namespace fir {
 namespace {
 
 class AddDebugInfoPass : public fir::impl::AddDebugInfoBase<AddDebugInfoPass> {
-  void handleDeclareOp(fir::cg::XDeclareOp declOp,
-                       mlir::LLVM::DIFileAttr fileAttr,
-                       mlir::LLVM::DIScopeAttr scopeAttr,
-                       fir::DebugTypeGenerator &typeGen);
-
 public:
   AddDebugInfoPass(fir::AddDebugInfoOptions options) : Base(options) {}
   void runOnOperation() override;
 };
 
-static uint32_t getLineFromLoc(mlir::Location loc) {
-  uint32_t line = 1;
-  if (auto fileLoc = mlir::dyn_cast<mlir::FileLineColLoc>(loc))
-    line = fileLoc.getLine();
-  return line;
-}
-
 } // namespace
 
-void AddDebugInfoPass::handleDeclareOp(fir::cg::XDeclareOp declOp,
-                                       mlir::LLVM::DIFileAttr fileAttr,
-                                       mlir::LLVM::DIScopeAttr scopeAttr,
-                                       fir::DebugTypeGenerator &typeGen) {
-  mlir::MLIRContext *context = &getContext();
-  mlir::OpBuilder builder(context);
-  auto result = fir::NameUniquer::deconstruct(declOp.getUniqName());
-
-  if (result.first != fir::NameUniquer::NameKind::VARIABLE)
-    return;
-
-  // Only accept local variables.
-  if (result.second.procs.empty())
-    return;
-
-  // FIXME: There may be cases where an argument is processed a bit before
-  // DeclareOp is generated. In that case, DeclareOp may point to an
-  // intermediate op and not to BlockArgument. We need to find those cases and
-  // walk the chain to get to the actual argument.
-
-  unsigned argNo = 0;
-  if (auto Arg = llvm::dyn_cast<mlir::BlockArgument>(declOp.getMemref()))
-    argNo = Arg.getArgNumber() + 1;
-
-  auto tyAttr = typeGen.convertType(fir::unwrapRefType(declOp.getType()),
-                                    fileAttr, scopeAttr, declOp.getLoc());
-
-  auto localVarAttr = mlir::LLVM::DILocalVariableAttr::get(
-      context, scopeAttr, mlir::StringAttr::get(context, result.second.name),
-      fileAttr, getLineFromLoc(declOp.getLoc()), argNo, /* alignInBits*/ 0,
-      tyAttr);
-  declOp->setLoc(builder.getFusedLoc({declOp->getLoc()}, localVarAttr));
-}
-
 void AddDebugInfoPass::runOnOperation() {
   mlir::ModuleOp module = getOperation();
   mlir::MLIRContext *context = &getContext();
@@ -191,15 +144,14 @@ void AddDebugInfoPass::runOnOperation() {
       subprogramFlags =
           subprogramFlags | mlir::LLVM::DISubprogramFlags::Definition;
     }
-    unsigned line = getLineFromLoc(l);
+    unsigned line = 1;
+    if (auto funcLoc = mlir::dyn_cast<mlir::FileLineColLoc>(l))
+      line = funcLoc.getLine();
+
     auto spAttr = mlir::LLVM::DISubprogramAttr::get(
         context, id, compilationUnit, fileAttr, funcName, fullName,
         funcFileAttr, line, line, subprogramFlags, subTypeAttr);
     funcOp->setLoc(builder.getFusedLoc({funcOp->getLoc()}, spAttr));
-
-    funcOp.walk([&](fir::cg::XDeclareOp declOp) {
-      handleDeclareOp(declOp, fileAttr, spAttr, typeGen);
-    });
   });
 }
 
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index 64c6547e06e0..e5b4050dfb24 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -24,6 +24,11 @@ DebugTypeGenerator::DebugTypeGenerator(mlir::ModuleOp m)
   LLVM_DEBUG(llvm::dbgs() << "DITypeAttr generator\n");
 }
 
+static mlir::LLVM::DITypeAttr genPlaceholderType(mlir::MLIRContext *context) {
+  return mlir::LLVM::DIBasicTypeAttr::get(
+      context, llvm::dwarf::DW_TAG_base_type, "void", 32, 1);
+}
+
 static mlir::LLVM::DITypeAttr genBasicType(mlir::MLIRContext *context,
                                            mlir::StringAttr name,
                                            unsigned bitSize,
@@ -32,11 +37,6 @@ static mlir::LLVM::DITypeAttr genBasicType(mlir::MLIRContext *context,
       context, llvm::dwarf::DW_TAG_base_type, name, bitSize, decoding);
 }
 
-static mlir::LLVM::DITypeAttr genPlaceholderType(mlir::MLIRContext *context) {
-  return genBasicType(context, mlir::StringAttr::get(context, "integer"), 32,
-                      llvm::dwarf::DW_ATE_signed);
-}
-
 mlir::LLVM::DITypeAttr
 DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
                                 mlir::LLVM::DIScopeAttr scope,
diff --git a/flang/test/Fir/declare-codegen.fir b/flang/test/Fir/declare-codegen.fir
index c5879facb157..9d68d3b2f9d4 100644
--- a/flang/test/Fir/declare-codegen.fir
+++ b/flang/test/Fir/declare-codegen.fir
@@ -1,7 +1,5 @@
 // Test rewrite of fir.declare. The result is replaced by the memref operand.
-// RUN: fir-opt --cg-rewrite="preserve-declare=true" %s -o - | FileCheck %s --check-prefixes DECL
-// RUN: fir-opt --cg-rewrite="preserve-declare=false" %s -o - | FileCheck %s --check-prefixes NODECL
-// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s --check-prefixes NODECL
+// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s
 
 
 func.func @test(%arg0: !fir.ref<!fir.array<12x23xi32>>) {
@@ -17,14 +15,9 @@ func.func @test(%arg0: !fir.ref<!fir.array<12x23xi32>>) {
 func.func private @bar(%arg0: !fir.ref<!fir.array<12x23xi32>>)
 
 
-// NODECL-LABEL: func.func @test(
-// NODECL-SAME: %[[arg0:.*]]: !fir.ref<!fir.array<12x23xi32>>) {
-// NODECL-NEXT: fir.call @bar(%[[arg0]]) : (!fir.ref<!fir.array<12x23xi32>>) -> ()
-
-// DECL-LABEL: func.func @test(
-// DECL-SAME: %[[arg0:.*]]: !fir.ref<!fir.array<12x23xi32>>) {
-// DECL: fircg.ext_declare
-
+// CHECK-LABEL: func.func @test(
+// CHECK-SAME: %[[arg0:.*]]: !fir.ref<!fir.array<12x23xi32>>) {
+// CHECK-NEXT: fir.call @bar(%[[arg0]]) : (!fir.ref<!fir.array<12x23xi32>>) -> ()
 
 func.func @useless_shape_with_duplicate_extent_operand(%arg0: !fir.ref<!fir.array<3x3xf32>>) {
   %c3 = arith.constant 3 : index
@@ -33,8 +26,5 @@ func.func @useless_shape_with_duplicate_extent_operand(%arg0: !fir.ref<!fir.arra
   return
 }
 
-// NODECL-LABEL: func.func @useless_shape_with_duplicate_extent_operand(
-// NODECL-NEXT: return
-
-// DECL-LABEL: func.func @useless_shape_with_duplicate_extent_operand(
-// DECL: fircg.ext_declare
+// CHECK-LABEL: func.func @useless_shape_with_duplicate_extent_operand(
+// CHECK-NEXT: return
diff --git a/flang/test/Fir/dummy-scope-codegen.fir b/flang/test/Fir/dummy-scope-codegen.fir
index 2e5c091552a7..caef3c1b2578 100644
--- a/flang/test/Fir/dummy-scope-codegen.fir
+++ b/flang/test/Fir/dummy-scope-codegen.fir
@@ -1,14 +1,9 @@
-// RUN: fir-opt --cg-rewrite="preserve-declare=true" %s -o - | FileCheck %s --check-prefixes DECL
-// RUN: fir-opt --cg-rewrite="preserve-declare=false" %s -o - | FileCheck %s --check-prefixes NODECL
-// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s  --check-prefixes NODECL
+// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s
 
 func.func @dummy_scope(%arg0: !fir.ref<f32>) {
   %scope = fir.dummy_scope : !fir.dscope
   %0 = fir.declare %arg0 dummy_scope %scope {uniq_name = "x"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
   return
 }
-// DECL-LABEL: func.func @dummy_scope(
-// DECL: fircg.ext_declare
-
-// NODECL-LABEL: func.func @dummy_scope(
-// NODECL-NEXT: return
\ No newline at end of file
+// CHECK-LABEL: func.func @dummy_scope(
+// CHECK-NEXT: return
diff --git a/flang/test/Transforms/debug-local-var-2.f90 b/flang/test/Transforms/debug-local-var-2.f90
deleted file mode 100644
index 15b9b148492e..000000000000
--- a/flang/test/Transforms/debug-local-var-2.f90
+++ /dev/null
@@ -1,91 +0,0 @@
-! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck %s
-
-! This tests checks the debug information for local variables in llvm IR.
-
-! CHECK-LABEL: define void @_QQmain
-! CHECK-DAG: %[[AL11:.*]] = alloca i32
-! CHECK-DAG: %[[AL12:.*]] = alloca i64
-! CHECK-DAG: %[[AL13:.*]] = alloca i8
-! CHECK-DAG: %[[AL14:.*]] = alloca i32
-! CHECK-DAG: %[[AL15:.*]] = alloca float
-! CHECK-DAG: %[[AL16:.*]] = alloca double
-! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL11]], metadata ![[I4:.*]], metadata !DIExpression())
-! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL12]], metadata ![[I8:.*]], metadata !DIExpression())
-! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL13]], metadata ![[L1:.*]], metadata !DIExpression())
-! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL14]], metadata ![[L4:.*]], metadata !DIExpression())
-! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL15]], metadata ![[R4:.*]], metadata !DIExpression())
-! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL16]], metadata ![[R8:.*]], metadata !DIExpression())
-! CHECK-LABEL: }
-
-! CHECK-LABEL: define {{.*}}i64 @_QFPfn1
-! CHECK-SAME: (ptr %[[ARG1:.*]], ptr %[[ARG2:.*]], ptr %[[ARG3:.*]])
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG1]], metadata ![[A1:.*]], metadata !DIExpression())
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG2]], metadata ![[B1:.*]], metadata !DIExpression())
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG3]], metadata ![[C1:.*]], metadata !DIExpression())
-! CHECK-DAG: %[[AL2:.*]] = alloca i64
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[AL2]], metadata ![[RES1:.*]], metadata !DIExpression())
-! CHECK-LABEL: }
-
-! CHECK-LABEL: define {{.*}}i32 @_QFPfn2
-! CHECK-SAME: (ptr %[[FN2ARG1:.*]], ptr %[[FN2ARG2:.*]], ptr %[[FN2ARG3:.*]])
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG1]], metadata ![[A2:.*]], metadata !DIExpression())
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG2]], metadata ![[B2:.*]], metadata !DIExpression())
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG3]], metadata ![[C2:.*]], metadata !DIExpression())
-! CHECK-DAG: %[[AL3:.*]] = alloca i32
-! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[AL3]], metadata ![[RES2:.*]], metadata !DIExpression())
-! CHECK-LABEL: }
-
-program mn
-! CHECK-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "_QQmain", {{.*}})
-
-! CHECK-DAG: ![[TYI32:.*]] = !DIBasicType(name: "integer", size: 32, encoding: DW_ATE_signed)
-! CHECK-DAG: ![[TYI64:.*]] = !DIBasicType(name: "integer", size: 64, encoding: DW_ATE_signed)
-! CHECK-DAG: ![[TYL8:.*]]  = !DIBasicType(name: "logical", size: 8, encoding: DW_ATE_boolean)
-! CHECK-DAG: ![[TYL32:.*]] = !DIBasicType(name: "logical", size: 32, encoding: DW_ATE_boolean)
-! CHECK-DAG: ![[TYR32:.*]] = !DIBasicType(name: "real", size: 32, encoding: DW_ATE_float)
-! CHECK-DAG: ![[TYR64:.*]] = !DIBasicType(name: "real", size: 64, encoding: DW_ATE_float)
-
-! CHECK-DAG: ![[I4]] = !DILocalVariable(name: "i4", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYI32]])
-! CHECK-DAG: ![[I8]] = !DILocalVariable(name: "i8", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYI64]])
-! CHECK-DAG: ![[R4]] = !DILocalVariable(name: "r4", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYR32]])
-! CHECK-DAG: ![[R8]] = !DILocalVariable(name: "r8", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYR64]])
-! CHECK-DAG: ![[L1]] = !DILocalVariable(name: "l1", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYL8]])
-! CHECK-DAG: ![[L4]] = !DILocalVariable(name: "l4", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYL32]])
-  integer(kind=4) :: i4
-  integer(kind=8) :: i8
-  real(kind=4) :: r4
-  real(kind=8) :: r8
-  logical(kind=1) :: l1
-  logical(kind=4) :: l4
-
-  i8 = fn1(i4, r8, l1)
-  i4 = fn2(i8, r4, l4)
-contains
-! CHECK-DAG: ![[FN1:.*]] = distinct !DISubprogram(name: "fn1", {{.*}})
-! CHECK-DAG: ![[A1]] = !DILocalVariable(name: "a1", arg: 1, scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI32]])
-! CHECK-DAG: ![[B1]] = !DILocalVariable(name: "b1", arg: 2, scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYR64]])
-! CHECK-DAG: ![[C1]] = !DILocalVariable(name: "c1", arg: 3, scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYL8]])
-! CHECK-DAG: ![[RES1]] = !DILocalVariable(name: "res1", scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI64]])
-  function fn1(a1, b1, c1) result (res1)
-    integer(kind=4), intent(in) :: a1
-    real(kind=8), intent(in) :: b1
-    logical(kind=1), intent(in) :: c1
-    integer(kind=8) :: res1
-
-    res1 = a1 + b1
-  end function
-
-! CHECK-DAG: ![[FN2:.*]] = distinct !DISubprogram(name: "fn2", {{.*}})
-! CHECK-DAG: ![[A2]] = !DILocalVariable(name: "a2", arg: 1, scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI64]])
-! CHECK-DAG: ![[B2]] = !DILocalVariable(name: "b2", arg: 2, scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYR32]])
-! CHECK-DAG: ![[C2]] = !DILocalVariable(name: "c2", arg: 3, scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYL32]])
-! CHECK-DAG: ![[RES2]] = !DILocalVariable(name: "res2", scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI32]])
-  function fn2(a2, b2, c2) result (res2)
-    integer(kind=8), intent(in) :: a2
-    real(kind=4), intent(in) :: b2
-    logical(kind=4), intent(in) :: c2
-    integer(kind=4) :: res2
-
-    res2 = a2 + b2
-  end function
-end program
diff --git a/flang/test/Transforms/debug-local-var.f90 b/flang/test/Transforms/debug-local-var.f90
deleted file mode 100644
index 96dc111ad308..000000000000
--- a/flang/test/Transforms/debug-local-var.f90
+++ /dev/null
@@ -1,54 +0,0 @@
-! RUN: %flang_fc1 -emit-fir -debug-info-kind=standalone -mmlir --mlir-print-debuginfo %s -o - | \
-! RUN: fir-opt --cg-rewrite="preserve-declare=true" --mlir-print-debuginfo | fir-opt --add-debug-info --mlir-print-debuginfo | FileCheck %s
-
-! CHECK-DAG: #[[INT8:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 64, encoding = DW_ATE_signed>
-! CHECK-DAG: #[[INT4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 32, encoding = DW_ATE_signed>
-! CHECK-DAG: #[[REAL8:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 64, encoding = DW_ATE_float>
-! CHECK-DAG: #[[LOG1:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "logical", sizeInBits = 8, encoding = DW_ATE_boolean>
-! CHECK-DAG: #[[REAL4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 32, encoding = DW_ATE_float>
-! CHECK-DAG: #[[LOG4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "logical", sizeInBits = 32, encoding = DW_ATE_boolean>
-! CHECK-DAG: #[[MAIN:.*]] = #llvm.di_subprogram<{{.*}}name = "_QQmain"{{.*}}>
-! CHECK-DAG: #[[FN1:.*]] = #llvm.di_subprogram<{{.*}}name = "fn1"{{.*}}>
-! CHECK-DAG: #[[FN2:.*]] = #llvm.di_subprogram<{{.*}}name = "fn2"{{.*}}>
-
-program mn
-! CHECK-DAG: #[[I4:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "i4", file = #{{.*}}, line = [[@LINE+6]], type = #[[INT4]]>
-! CHECK-DAG: #[[I8:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "i8", file = #{{.*}}, line = [[@LINE+6]], type = #[[INT8]]>
-! CHECK-DAG: #[[R4:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "r4", file = #{{.*}}, line = [[@LINE+6]], type = #[[REAL4]]>
-! CHECK-DAG: #[[R8:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "r8", file = #{{.*}}, line = [[@LINE+6]], type = #[[REAL8]]>
-! CHECK-DAG: #[[L1:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "l1", file = #{{.*}}, line = [[@LINE+6]], type = #[[LOG1]]>
-! CHECK-DAG: #[[L4:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "l4", file = #{{.*}}, line = [[@LINE+6]], type = #[[LOG4]]>
-  integer(kind=4) :: i4
-  integer(kind=8) :: i8
-  real(kind=4) :: r4
-  real(kind=8) :: r8
-  logical(kind=1) :: l1
-  logical(kind=4) :: l4
-  i8 = fn1(i4, r8, l1)
-  i4 = fn2(i8, r4, l4)
-contains
-  function fn1(a1, b1, c1) result (res1)
-! CHECK-DAG: #[[A1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "a1", file = #{{.*}}, line = [[@LINE+4]], arg = 1, type = #[[INT4]]>
-! CHECK-DAG: #[[B1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "b1", file = #{{.*}}, line = [[@LINE+4]], arg = 2, type = #[[REAL8]]>
-! CHECK-DAG: #[[C1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "c1", file = #{{.*}}, line = [[@LINE+4]], arg = 3, type = #[[LOG1]]>
-! CHECK-DAG: #[[RES1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "res1", file = #{{.*}}, line = [[@LINE+4]], type = #[[INT8]]>
-    integer(kind=4), intent(in) :: a1
-    real(kind=8), intent(in) :: b1
-    logical(kind=1), intent(in) :: c1
-    integer(kind=8) :: res1
-    res1 = a1 + b1
-  end function
-
-  function fn2(a2, b2, c2) result (res2)
-    implicit none
-! CHECK-DAG: #[[A2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "a2", file = #{{.*}}, line = [[@LINE+4]], arg = 1, type = #[[INT8]]>
-! CHECK-DAG: #[[B2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "b2", file = #{{.*}}, line = [[@LINE+4]], arg = 2, type = #[[REAL4]]>
-! CHECK-DAG: #[[C2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "c2", file = #{{.*}}, line = [[@LINE+4]], arg = 3, type = #[[LOG4]]>
-! CHECK-DAG: #[[RES2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "res2", file = #{{.*}}, line = [[@LINE+4]], type = #[[INT4]]>
-    integer(kind=8), intent(in) :: a2
-    real(kind=4), intent(in) :: b2
-    logical(kind=4), intent(in) :: c2
-    integer(kind=4) :: res2
-    res2 = a2 + b2
-  end function
-end program
-- 
GitLab


From 411bf385ba27f15145c635c7d8ff2701fe8de5b9 Mon Sep 17 00:00:00 2001
From: Walter Erquinigo <a20012251@gmail.com>
Date: Wed, 15 May 2024 20:44:12 +0200
Subject: [PATCH 0901/1206] [lldb-dap] Include npm install in the extension
 installation steps (#92028)

Otherwise the build step fails due to missing dependencies.
---
 lldb/tools/lldb-dap/README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lldb/tools/lldb-dap/README.md b/lldb/tools/lldb-dap/README.md
index 274b1519208a..16ce4672be71 100644
--- a/lldb/tools/lldb-dap/README.md
+++ b/lldb/tools/lldb-dap/README.md
@@ -46,6 +46,7 @@ Installing the plug-in is very straightforward and involves just a few steps.
 
 ```bash
 cd /path/to/lldb/tools/lldb-dap
+npm install
 npm run package # This also compiles the extension.
 npm run vscode-install
 ```
@@ -69,6 +70,7 @@ no effect.
 ```bash
 # Bump version in package.json
 cd /path/to/lldb/tools/lldb-dap
+npm install
 npm run package
 npm run vscode-install
 ```
-- 
GitLab


From 2c54bf497f7d7aecd24f4b849ee08e37a3519611 Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Wed, 15 May 2024 11:44:26 -0700
Subject: [PATCH 0902/1206] Revert "Reapply "[ctx_profile] Profile reader and
 writer" (#92199)"

This reverts commit c19f2c773b0e23fd623502888894add822079f63.

Broke the gcc-7 bot.
---
 .../llvm/ProfileData/PGOCtxProfReader.h       |  92 -------
 .../llvm/ProfileData/PGOCtxProfWriter.h       |  91 -------
 llvm/lib/ProfileData/CMakeLists.txt           |   3 -
 llvm/lib/ProfileData/PGOCtxProfReader.cpp     | 173 ------------
 llvm/lib/ProfileData/PGOCtxProfWriter.cpp     |  49 ----
 llvm/unittests/ProfileData/CMakeLists.txt     |   1 -
 .../PGOCtxProfReaderWriterTest.cpp            | 255 ------------------
 7 files changed, 664 deletions(-)
 delete mode 100644 llvm/include/llvm/ProfileData/PGOCtxProfReader.h
 delete mode 100644 llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
 delete mode 100644 llvm/lib/ProfileData/PGOCtxProfReader.cpp
 delete mode 100644 llvm/lib/ProfileData/PGOCtxProfWriter.cpp
 delete mode 100644 llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp

diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
deleted file mode 100644
index a19b3f51d642..000000000000
--- a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
+++ /dev/null
@@ -1,92 +0,0 @@
-//===--- PGOCtxProfReader.h - Contextual profile reader ---------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-///
-/// Reader for contextual iFDO profile, which comes in bitstream format.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
-#define LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
-
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/Bitstream/BitstreamReader.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/ProfileData/PGOCtxProfWriter.h"
-#include "llvm/Support/Error.h"
-#include <map>
-#include <vector>
-
-namespace llvm {
-/// The loaded contextual profile, suitable for mutation during IPO passes. We
-/// generally expect a fraction of counters and of callsites to be populated.
-/// We continue to model counters as vectors, but callsites are modeled as a map
-/// of a map. The expectation is that, typically, there is a small number of
-/// indirect targets (usually, 1 for direct calls); but potentially a large
-/// number of callsites, and, as inlining progresses, the callsite count of a
-/// caller will grow.
-class PGOContextualProfile final {
-public:
-  using CallTargetMapTy = std::map<GlobalValue::GUID, PGOContextualProfile>;
-  using CallsiteMapTy = DenseMap<uint32_t, CallTargetMapTy>;
-
-private:
-  friend class PGOCtxProfileReader;
-  GlobalValue::GUID GUID = 0;
-  SmallVector<uint64_t, 16> Counters;
-  CallsiteMapTy Callsites;
-
-  PGOContextualProfile(GlobalValue::GUID G,
-                       SmallVectorImpl<uint64_t> &&Counters)
-      : GUID(G), Counters(std::move(Counters)) {}
-
-  Expected<PGOContextualProfile &>
-  getOrEmplace(uint32_t Index, GlobalValue::GUID G,
-               SmallVectorImpl<uint64_t> &&Counters);
-
-public:
-  PGOContextualProfile(const PGOContextualProfile &) = delete;
-  PGOContextualProfile &operator=(const PGOContextualProfile &) = delete;
-  PGOContextualProfile(PGOContextualProfile &&) = default;
-  PGOContextualProfile &operator=(PGOContextualProfile &&) = default;
-
-  GlobalValue::GUID guid() const { return GUID; }
-  const SmallVectorImpl<uint64_t> &counters() const { return Counters; }
-  const CallsiteMapTy &callsites() const { return Callsites; }
-  CallsiteMapTy &callsites() { return Callsites; }
-
-  bool hasCallsite(uint32_t I) const {
-    return Callsites.find(I) != Callsites.end();
-  }
-
-  const CallTargetMapTy &callsite(uint32_t I) const {
-    assert(hasCallsite(I) && "Callsite not found");
-    return Callsites.find(I)->second;
-  }
-  void getContainedGuids(DenseSet<GlobalValue::GUID> &Guids) const;
-};
-
-class PGOCtxProfileReader final {
-  BitstreamCursor &Cursor;
-  Expected<BitstreamEntry> advance();
-  Error readMetadata();
-  Error wrongValue(const Twine &);
-  Error unsupported(const Twine &);
-
-  Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
-  readContext(bool ExpectIndex);
-  bool canReadContext();
-
-public:
-  PGOCtxProfileReader(BitstreamCursor &Cursor) : Cursor(Cursor) {}
-
-  Expected<std::map<GlobalValue::GUID, PGOContextualProfile>> loadContexts();
-};
-} // namespace llvm
-#endif
diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
deleted file mode 100644
index 15578c51a495..000000000000
--- a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
+++ /dev/null
@@ -1,91 +0,0 @@
-//===- PGOCtxProfWriter.h - Contextual Profile Writer -----------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares a utility for writing a contextual profile to bitstream.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
-#define LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
-
-#include "llvm/Bitstream/BitstreamWriter.h"
-#include "llvm/ProfileData/CtxInstrContextNode.h"
-
-namespace llvm {
-enum PGOCtxProfileRecords { Invalid = 0, Version, Guid, CalleeIndex, Counters };
-
-enum PGOCtxProfileBlockIDs {
-  ProfileMetadataBlockID = 100,
-  ContextNodeBlockID = ProfileMetadataBlockID + 1
-};
-
-/// Write one or more ContextNodes to the provided raw_fd_stream.
-/// The caller must destroy the PGOCtxProfileWriter object before closing the
-/// stream.
-/// The design allows serializing a bunch of contexts embedded in some other
-/// file. The overall format is:
-///
-///  [... other data written to the stream...]
-///  SubBlock(ProfileMetadataBlockID)
-///   Version
-///   SubBlock(ContextNodeBlockID)
-///     [RECORDS]
-///     SubBlock(ContextNodeBlockID)
-///       [RECORDS]
-///       [... more SubBlocks]
-///     EndBlock
-///   EndBlock
-///
-/// The "RECORDS" are bitsream records. The IDs are in CtxProfileCodes (except)
-/// for Version, which is just for metadata). All contexts will have Guid and
-/// Counters, and all but the roots have CalleeIndex. The order in which the
-/// records appear does not matter, but they must precede any subcontexts,
-/// because that helps keep the reader code simpler.
-///
-/// Subblock containment captures the context->subcontext relationship. The
-/// "next()" relationship in the raw profile, between call targets of indirect
-/// calls, are just modeled as peer subblocks where the callee index is the
-/// same.
-///
-/// Versioning: the writer may produce additional records not known by the
-/// reader. The version number indicates a more structural change.
-/// The current version, in particular, is set up to expect optional extensions
-/// like value profiling - which would appear as additional records. For
-/// example, value profiling would produce a new record with a new record ID,
-/// containing the profiled values (much like the counters)
-class PGOCtxProfileWriter final {
-  SmallVector<char, 1 << 20> Buff;
-  BitstreamWriter Writer;
-
-  void writeCounters(const ctx_profile::ContextNode &Node);
-  void writeImpl(std::optional<uint32_t> CallerIndex,
-                 const ctx_profile::ContextNode &Node);
-
-public:
-  PGOCtxProfileWriter(raw_fd_stream &Out,
-                      std::optional<unsigned> VersionOverride = std::nullopt)
-      : Writer(Buff, &Out, 0) {
-    Writer.EnterSubblock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID,
-                         CodeLen);
-    const auto Version = VersionOverride ? *VersionOverride : CurrentVersion;
-    Writer.EmitRecord(PGOCtxProfileRecords::Version,
-                      SmallVector<unsigned, 1>({Version}));
-  }
-
-  ~PGOCtxProfileWriter() { Writer.ExitBlock(); }
-
-  void write(const ctx_profile::ContextNode &);
-
-  // constants used in writing which a reader may find useful.
-  static constexpr unsigned CodeLen = 2;
-  static constexpr uint32_t CurrentVersion = 1;
-  static constexpr unsigned VBREncodingBits = 6;
-};
-
-} // namespace llvm
-#endif
diff --git a/llvm/lib/ProfileData/CMakeLists.txt b/llvm/lib/ProfileData/CMakeLists.txt
index 4fa1b76f0a06..408f9ff01ec8 100644
--- a/llvm/lib/ProfileData/CMakeLists.txt
+++ b/llvm/lib/ProfileData/CMakeLists.txt
@@ -7,8 +7,6 @@ add_llvm_component_library(LLVMProfileData
   ItaniumManglingCanonicalizer.cpp
   MemProf.cpp
   MemProfReader.cpp
-  PGOCtxProfReader.cpp
-  PGOCtxProfWriter.cpp
   ProfileSummaryBuilder.cpp
   SampleProf.cpp
   SampleProfReader.cpp
@@ -22,7 +20,6 @@ add_llvm_component_library(LLVMProfileData
   intrinsics_gen
 
   LINK_COMPONENTS
-  BitstreamReader
   Core
   Object
   Support
diff --git a/llvm/lib/ProfileData/PGOCtxProfReader.cpp b/llvm/lib/ProfileData/PGOCtxProfReader.cpp
deleted file mode 100644
index 3710f2e4b818..000000000000
--- a/llvm/lib/ProfileData/PGOCtxProfReader.cpp
+++ /dev/null
@@ -1,173 +0,0 @@
-//===- PGOCtxProfReader.cpp - Contextual Instrumentation profile reader ---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Read a contextual profile into a datastructure suitable for maintenance
-// throughout IPO
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ProfileData/PGOCtxProfReader.h"
-#include "llvm/Bitstream/BitCodeEnums.h"
-#include "llvm/Bitstream/BitstreamReader.h"
-#include "llvm/ProfileData/InstrProf.h"
-#include "llvm/ProfileData/PGOCtxProfWriter.h"
-#include "llvm/Support/Errc.h"
-#include "llvm/Support/Error.h"
-
-using namespace llvm;
-
-// FIXME(#92054) - these Error handling macros are (re-)invented in a few
-// places.
-#define EXPECT_OR_RET(LHS, RHS)                                                \
-  auto LHS = RHS;                                                              \
-  if (!LHS)                                                                    \
-    return LHS.takeError();
-
-#define RET_ON_ERR(EXPR)                                                       \
-  if (auto Err = (EXPR))                                                       \
-    return Err;
-
-Expected<PGOContextualProfile &>
-PGOContextualProfile::getOrEmplace(uint32_t Index, GlobalValue::GUID G,
-                                   SmallVectorImpl<uint64_t> &&Counters) {
-  auto [Iter, Inserted] = Callsites[Index].insert(
-      {G, PGOContextualProfile(G, std::move(Counters))});
-  if (!Inserted)
-    return make_error<InstrProfError>(instrprof_error::invalid_prof,
-                                      "Duplicate GUID for same callsite.");
-  return Iter->second;
-}
-
-void PGOContextualProfile::getContainedGuids(
-    DenseSet<GlobalValue::GUID> &Guids) const {
-  Guids.insert(GUID);
-  for (const auto &[_, Callsite] : Callsites)
-    for (const auto &[_, Callee] : Callsite)
-      Callee.getContainedGuids(Guids);
-}
-
-Expected<BitstreamEntry> PGOCtxProfileReader::advance() {
-  return Cursor.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs);
-}
-
-Error PGOCtxProfileReader::wrongValue(const Twine &Msg) {
-  return make_error<InstrProfError>(instrprof_error::invalid_prof, Msg);
-}
-
-Error PGOCtxProfileReader::unsupported(const Twine &Msg) {
-  return make_error<InstrProfError>(instrprof_error::unsupported_version, Msg);
-}
-
-bool PGOCtxProfileReader::canReadContext() {
-  auto Blk = advance();
-  if (!Blk) {
-    consumeError(Blk.takeError());
-    return false;
-  }
-  return Blk->Kind == BitstreamEntry::SubBlock &&
-         Blk->ID == PGOCtxProfileBlockIDs::ContextNodeBlockID;
-}
-
-Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
-PGOCtxProfileReader::readContext(bool ExpectIndex) {
-  RET_ON_ERR(Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID));
-
-  std::optional<ctx_profile::GUID> Guid;
-  std::optional<SmallVector<uint64_t, 16>> Counters;
-  std::optional<uint32_t> CallsiteIndex;
-
-  SmallVector<uint64_t, 1> RecordValues;
-
-  // We don't prescribe the order in which the records come in, and we are ok
-  // if other unsupported records appear. We seek in the current subblock until
-  // we get all we know.
-  auto GotAllWeNeed = [&]() {
-    return Guid.has_value() && Counters.has_value() &&
-           (!ExpectIndex || CallsiteIndex.has_value());
-  };
-  while (!GotAllWeNeed()) {
-    RecordValues.clear();
-    EXPECT_OR_RET(Entry, advance());
-    if (Entry->Kind != BitstreamEntry::Record)
-      return wrongValue(
-          "Expected records before encountering more subcontexts");
-    EXPECT_OR_RET(ReadRecord,
-                  Cursor.readRecord(bitc::UNABBREV_RECORD, RecordValues));
-    switch (*ReadRecord) {
-    case PGOCtxProfileRecords::Guid:
-      if (RecordValues.size() != 1)
-        return wrongValue("The GUID record should have exactly one value");
-      Guid = RecordValues[0];
-      break;
-    case PGOCtxProfileRecords::Counters:
-      Counters = std::move(RecordValues);
-      if (Counters->empty())
-        return wrongValue("Empty counters. At least the entry counter (one "
-                          "value) was expected");
-      break;
-    case PGOCtxProfileRecords::CalleeIndex:
-      if (!ExpectIndex)
-        return wrongValue("The root context should not have a callee index");
-      if (RecordValues.size() != 1)
-        return wrongValue("The callee index should have exactly one value");
-      CallsiteIndex = RecordValues[0];
-      break;
-    default:
-      // OK if we see records we do not understand, like records (profile
-      // components) introduced later.
-      break;
-    }
-  }
-
-  PGOContextualProfile Ret(*Guid, std::move(*Counters));
-
-  while (canReadContext()) {
-    EXPECT_OR_RET(SC, readContext(true));
-    auto &Targets = Ret.callsites()[*SC->first];
-    auto [_, Inserted] =
-        Targets.insert({SC->second.guid(), std::move(SC->second)});
-    if (!Inserted)
-      return wrongValue(
-          "Unexpected duplicate target (callee) at the same callsite.");
-  }
-  return std::make_pair(CallsiteIndex, std::move(Ret));
-}
-
-Error PGOCtxProfileReader::readMetadata() {
-  EXPECT_OR_RET(Blk, advance());
-  if (Blk->Kind != BitstreamEntry::SubBlock)
-    return unsupported("Expected Version record");
-  RET_ON_ERR(
-      Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID));
-  EXPECT_OR_RET(MData, advance());
-  if (MData->Kind != BitstreamEntry::Record)
-    return unsupported("Expected Version record");
-
-  SmallVector<uint64_t, 1> Ver;
-  EXPECT_OR_RET(Code, Cursor.readRecord(bitc::UNABBREV_RECORD, Ver));
-  if (*Code != PGOCtxProfileRecords::Version)
-    return unsupported("Expected Version record");
-  if (Ver.size() != 1 || Ver[0] > PGOCtxProfileWriter::CurrentVersion)
-    return unsupported("Version " + Twine(*Code) +
-                       " is higher than supported version " +
-                       Twine(PGOCtxProfileWriter::CurrentVersion));
-  return Error::success();
-}
-
-Expected<std::map<GlobalValue::GUID, PGOContextualProfile>>
-PGOCtxProfileReader::loadContexts() {
-  std::map<GlobalValue::GUID, PGOContextualProfile> Ret;
-  RET_ON_ERR(readMetadata());
-  while (canReadContext()) {
-    EXPECT_OR_RET(E, readContext(false));
-    auto Key = E->second.guid();
-    if (!Ret.insert({Key, std::move(E->second)}).second)
-      return wrongValue("Duplicate roots");
-  }
-  return Ret;
-}
diff --git a/llvm/lib/ProfileData/PGOCtxProfWriter.cpp b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
deleted file mode 100644
index 508179756446..000000000000
--- a/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
+++ /dev/null
@@ -1,49 +0,0 @@
-//===- PGOCtxProfWriter.cpp - Contextual Instrumentation profile writer ---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Write a contextual profile to bitstream.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ProfileData/PGOCtxProfWriter.h"
-#include "llvm/Bitstream/BitCodeEnums.h"
-
-using namespace llvm;
-using namespace llvm::ctx_profile;
-
-void PGOCtxProfileWriter::writeCounters(const ContextNode &Node) {
-  Writer.EmitCode(bitc::UNABBREV_RECORD);
-  Writer.EmitVBR(PGOCtxProfileRecords::Counters, VBREncodingBits);
-  Writer.EmitVBR(Node.counters_size(), VBREncodingBits);
-  for (uint32_t I = 0U; I < Node.counters_size(); ++I)
-    Writer.EmitVBR64(Node.counters()[I], VBREncodingBits);
-}
-
-// recursively write all the subcontexts. We do need to traverse depth first to
-// model the context->subcontext implicitly, and since this captures call
-// stacks, we don't really need to be worried about stack overflow and we can
-// keep the implementation simple.
-void PGOCtxProfileWriter::writeImpl(std::optional<uint32_t> CallerIndex,
-                                    const ContextNode &Node) {
-  Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextNodeBlockID, CodeLen);
-  Writer.EmitRecord(PGOCtxProfileRecords::Guid,
-                    SmallVector<uint64_t, 1>{Node.guid()});
-  if (CallerIndex)
-    Writer.EmitRecord(PGOCtxProfileRecords::CalleeIndex,
-                      SmallVector<uint64_t, 1>{*CallerIndex});
-  writeCounters(Node);
-  for (uint32_t I = 0U; I < Node.callsites_size(); ++I)
-    for (const auto *Subcontext = Node.subContexts()[I]; Subcontext;
-         Subcontext = Subcontext->next())
-      writeImpl(I, *Subcontext);
-  Writer.ExitBlock();
-}
-
-void PGOCtxProfileWriter::write(const ContextNode &RootNode) {
-  writeImpl(std::nullopt, RootNode);
-}
diff --git a/llvm/unittests/ProfileData/CMakeLists.txt b/llvm/unittests/ProfileData/CMakeLists.txt
index c92642ded828..ce3a0a45ccf1 100644
--- a/llvm/unittests/ProfileData/CMakeLists.txt
+++ b/llvm/unittests/ProfileData/CMakeLists.txt
@@ -13,7 +13,6 @@ add_llvm_unittest(ProfileDataTests
   InstrProfTest.cpp
   ItaniumManglingCanonicalizerTest.cpp
   MemProfTest.cpp
-  PGOCtxProfReaderWriterTest.cpp
   SampleProfTest.cpp
   SymbolRemappingReaderTest.cpp
   )
diff --git a/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp b/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp
deleted file mode 100644
index d2cdbb28e2fc..000000000000
--- a/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp
+++ /dev/null
@@ -1,255 +0,0 @@
-//===-------------- PGOCtxProfReadWriteTest.cpp ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Bitstream/BitstreamReader.h"
-#include "llvm/ProfileData/CtxInstrContextNode.h"
-#include "llvm/ProfileData/PGOCtxProfReader.h"
-#include "llvm/ProfileData/PGOCtxProfWriter.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Testing/Support/SupportHelpers.h"
-#include "gtest/gtest.h"
-
-using namespace llvm;
-using namespace llvm::ctx_profile;
-
-class PGOCtxProfRWTest : public ::testing::Test {
-  std::vector<std::unique_ptr<char[]>> Nodes;
-  std::map<GUID, const ContextNode *> Roots;
-
-public:
-  ContextNode *createNode(GUID Guid, uint32_t NrCounters, uint32_t NrCallsites,
-                          ContextNode *Next = nullptr) {
-    auto AllocSize = ContextNode::getAllocSize(NrCounters, NrCallsites);
-    auto *Mem = Nodes.emplace_back(std::make_unique<char[]>(AllocSize)).get();
-    std::memset(Mem, 0, AllocSize);
-    auto *Ret = new (Mem) ContextNode(Guid, NrCounters, NrCallsites, Next);
-    return Ret;
-  }
-
-  void SetUp() override {
-    // Root (guid 1) has 2 callsites, one used for an indirect call to either
-    // guid 2 or 4.
-    // guid 2 calls guid 5
-    // guid 5 calls guid 2
-    // there's also a second root, guid3.
-    auto *Root1 = createNode(1, 2, 2);
-    Root1->counters()[0] = 10;
-    Root1->counters()[1] = 11;
-    Roots.insert({1, Root1});
-    auto *L1 = createNode(2, 1, 1);
-    L1->counters()[0] = 12;
-    Root1->subContexts()[1] = createNode(4, 3, 1, L1);
-    Root1->subContexts()[1]->counters()[0] = 13;
-    Root1->subContexts()[1]->counters()[1] = 14;
-    Root1->subContexts()[1]->counters()[2] = 15;
-
-    auto *L3 = createNode(5, 6, 3);
-    for (auto I = 0; I < 6; ++I)
-      L3->counters()[I] = 16 + I;
-    L1->subContexts()[0] = L3;
-    L3->subContexts()[2] = createNode(2, 1, 1);
-    L3->subContexts()[2]->counters()[0] = 30;
-    auto *Root2 = createNode(3, 1, 0);
-    Root2->counters()[0] = 40;
-    Roots.insert({3, Root2});
-  }
-
-  const std::map<GUID, const ContextNode *> &roots() const { return Roots; }
-};
-
-void checkSame(const ContextNode &Raw, const PGOContextualProfile &Profile) {
-  EXPECT_EQ(Raw.guid(), Profile.guid());
-  ASSERT_EQ(Raw.counters_size(), Profile.counters().size());
-  for (auto I = 0U; I < Raw.counters_size(); ++I)
-    EXPECT_EQ(Raw.counters()[I], Profile.counters()[I]);
-
-  for (auto I = 0U; I < Raw.callsites_size(); ++I) {
-    if (Raw.subContexts()[I] == nullptr)
-      continue;
-    EXPECT_TRUE(Profile.hasCallsite(I));
-    const auto &ProfileTargets = Profile.callsite(I);
-
-    std::map<GUID, const ContextNode *> Targets;
-    for (const auto *N = Raw.subContexts()[I]; N; N = N->next())
-      EXPECT_TRUE(Targets.insert({N->guid(), N}).second);
-
-    EXPECT_EQ(Targets.size(), ProfileTargets.size());
-    for (auto It : Targets) {
-      auto PIt = ProfileTargets.find(It.second->guid());
-      EXPECT_NE(PIt, ProfileTargets.end());
-      checkSame(*It.second, PIt->second);
-    }
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, RoundTrip) {
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      PGOCtxProfileWriter Writer(Out);
-      for (auto &[_, R] : roots())
-        Writer.write(*R);
-    }
-  }
-  {
-    ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
-        MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    ASSERT_TRUE(!!Expected);
-    auto &Ctxes = *Expected;
-    EXPECT_EQ(Ctxes.size(), roots().size());
-    EXPECT_EQ(Ctxes.size(), 2U);
-    for (auto &[G, R] : roots())
-      checkSame(*R, Ctxes.find(G)->second);
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, InvalidCounters) {
-  auto *R = createNode(1, 0, 1);
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      PGOCtxProfileWriter Writer(Out);
-      Writer.write(*R);
-    }
-  }
-  {
-    auto MB = MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    EXPECT_FALSE(Expected);
-    consumeError(Expected.takeError());
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, Empty) {
-  BitstreamCursor Cursor("");
-  PGOCtxProfileReader Reader(Cursor);
-  auto Expected = Reader.loadContexts();
-  EXPECT_FALSE(Expected);
-  consumeError(Expected.takeError());
-}
-
-TEST_F(PGOCtxProfRWTest, Invalid) {
-  BitstreamCursor Cursor("Surely this is not valid");
-  PGOCtxProfileReader Reader(Cursor);
-  auto Expected = Reader.loadContexts();
-  EXPECT_FALSE(Expected);
-  consumeError(Expected.takeError());
-}
-
-TEST_F(PGOCtxProfRWTest, ValidButEmpty) {
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      PGOCtxProfileWriter Writer(Out);
-      // don't write anything - this will just produce the metadata subblock.
-    }
-  }
-  {
-    auto MB = MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    EXPECT_TRUE(!!Expected);
-    EXPECT_TRUE(Expected->empty());
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, WrongVersion) {
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      PGOCtxProfileWriter Writer(Out, PGOCtxProfileWriter::CurrentVersion + 1);
-    }
-  }
-  {
-    auto MB = MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    EXPECT_FALSE(Expected);
-    consumeError(Expected.takeError());
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, DuplicateRoots) {
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      PGOCtxProfileWriter Writer(Out);
-      Writer.write(*createNode(1, 1, 1));
-      Writer.write(*createNode(1, 1, 1));
-    }
-  }
-  {
-    auto MB = MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    EXPECT_FALSE(Expected);
-    consumeError(Expected.takeError());
-  }
-}
-
-TEST_F(PGOCtxProfRWTest, DuplicateTargets) {
-  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
-  {
-    std::error_code EC;
-    raw_fd_stream Out(ProfileFile.path(), EC);
-    ASSERT_FALSE(EC);
-    {
-      auto *R = createNode(1, 1, 1);
-      auto *L1 = createNode(2, 1, 0);
-      auto *L2 = createNode(2, 1, 0, L1);
-      R->subContexts()[0] = L2;
-      PGOCtxProfileWriter Writer(Out);
-      Writer.write(*R);
-    }
-  }
-  {
-    auto MB = MemoryBuffer::getFile(ProfileFile.path());
-    ASSERT_TRUE(!!MB);
-    ASSERT_NE(*MB, nullptr);
-    BitstreamCursor Cursor((*MB)->getBuffer());
-    PGOCtxProfileReader Reader(Cursor);
-    auto Expected = Reader.loadContexts();
-    EXPECT_FALSE(Expected);
-    consumeError(Expected.takeError());
-  }
-}
-- 
GitLab


From 9ae2177843f681c70ad89506155a2cb83eeebfd4 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Thu, 16 May 2024 02:41:48 +0800
Subject: [PATCH 0903/1206] [RISCV] Handle undef AVLs in RISCVInsertVSETVLI

Before #91440 a VSETVLIInfo would have had an IMPLICIT_DEF defining
instruction, but now we look up a VNInfo which doesn't exist, which
triggers an assertion failure. Mark these undef AVLs as AVLIsIgnored.
---
 llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp  | 20 +++++++++-------
 llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll | 24 +++++++++++++++++++
 2 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 1c815424bdfa..363007d7b68b 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -48,15 +48,13 @@ static cl::opt<bool> DisableInsertVSETVLPHIOpt(
 namespace {
 
 /// Given a virtual register \p Reg, return the corresponding VNInfo for it.
-/// This should never return nullptr.
+/// This will return nullptr if the virtual register is an implicit_def.
 static VNInfo *getVNInfoFromReg(Register Reg, const MachineInstr &MI,
                                 const LiveIntervals *LIS) {
   assert(Reg.isVirtual());
   auto &LI = LIS->getInterval(Reg);
   SlotIndex SI = LIS->getSlotIndexes()->getInstructionIndex(MI);
-  VNInfo *VNI = LI.getVNInfoBefore(SI);
-  assert(VNI);
-  return VNI;
+  return LI.getVNInfoBefore(SI);
 }
 
 static unsigned getVLOpNum(const MachineInstr &MI) {
@@ -894,8 +892,12 @@ static VSETVLIInfo getInfoForVSETVLI(const MachineInstr &MI,
            "Can't handle X0, X0 vsetvli yet");
     if (AVLReg == RISCV::X0)
       NewInfo.setAVLVLMAX();
-    else
-      NewInfo.setAVLRegDef(getVNInfoFromReg(AVLReg, MI, LIS), AVLReg);
+    else if (VNInfo *VNI = getVNInfoFromReg(AVLReg, MI, LIS))
+      NewInfo.setAVLRegDef(VNI, AVLReg);
+    else {
+      assert(MI.getOperand(1).isUndef());
+      NewInfo.setAVLIgnored();
+    }
   }
   NewInfo.setVTYPE(MI.getOperand(2).getImm());
 
@@ -966,9 +968,11 @@ static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags,
       }
       else
         InstrInfo.setAVLImm(Imm);
+    } else if (VNInfo *VNI = getVNInfoFromReg(VLOp.getReg(), MI, LIS)) {
+      InstrInfo.setAVLRegDef(VNI, VLOp.getReg());
     } else {
-      InstrInfo.setAVLRegDef(getVNInfoFromReg(VLOp.getReg(), MI, LIS),
-                             VLOp.getReg());
+      assert(VLOp.isUndef());
+      InstrInfo.setAVLIgnored();
     }
   } else {
     assert(isScalarExtractInstr(MI));
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
index 12bb4d27b0f9..da0c1cfb5009 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
@@ -699,3 +699,27 @@ declare <vscale x 2 x i1> @llvm.riscv.vmsgt.nxv2i32.i32.i64(<vscale x 2 x i32>,
 declare <vscale x 2 x i1> @llvm.riscv.vmor.nxv2i1.i64(<vscale x 2 x i1>, <vscale x 2 x i1>, i64)
 declare void @llvm.riscv.vse.mask.nxv2i32.i64(<vscale x 2 x i32>, ptr nocapture, <vscale x 2 x i1>, i64)
 declare void @llvm.riscv.vse.nxv2i32.i64(<vscale x 2 x i32>, ptr nocapture, i64)
+
+define <vscale x 2 x i32> @avl_undef1(<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>) {
+; CHECK-LABEL: avl_undef1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, tu, ma
+; CHECK-NEXT:    vadd.vv v8, v9, v10
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x i32> @llvm.riscv.vadd.nxv2i32.nxv2i32(
+    <vscale x 2 x i32> %0,
+    <vscale x 2 x i32> %1,
+    <vscale x 2 x i32> %2,
+    i64 undef
+  )
+  ret <vscale x 2 x i32> %a
+}
+
+define i64 @avl_undef2() {
+; CHECK-LABEL: avl_undef2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, a0, e32, mf2, ta, ma
+; CHECK-NEXT:    ret
+  %1 = tail call i64 @llvm.riscv.vsetvli(i64 poison, i64 2, i64 7)
+  ret i64 %1
+}
-- 
GitLab


From 378c9e952a3d198873677fb2d2afb33695185b72 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Wed, 15 May 2024 18:51:56 +0000
Subject: [PATCH 0904/1206] [gn build] Port 2c54bf497f7d

---
 llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn       | 2 --
 llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn | 1 -
 2 files changed, 3 deletions(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
index c6fa142b3766..9dbfe0f94c1d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
@@ -17,8 +17,6 @@ static_library("ProfileData") {
     "ItaniumManglingCanonicalizer.cpp",
     "MemProf.cpp",
     "MemProfReader.cpp",
-    "PGOCtxProfReader.cpp",
-    "PGOCtxProfWriter.cpp",
     "ProfileSummaryBuilder.cpp",
     "SampleProf.cpp",
     "SampleProfReader.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
index f45542519173..4919a8089209 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
@@ -14,7 +14,6 @@ unittest("ProfileDataTests") {
     "InstrProfTest.cpp",
     "ItaniumManglingCanonicalizerTest.cpp",
     "MemProfTest.cpp",
-    "PGOCtxProfReaderWriterTest.cpp",
     "SampleProfTest.cpp",
     "SymbolRemappingReaderTest.cpp",
   ]
-- 
GitLab


From d542eb7aa830e94490b943a3ea0937506fece15b Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Wed, 15 May 2024 23:12:02 +0400
Subject: [PATCH 0905/1206] [clang] Add tests for CWG issues regarding
 completeness of types (#92113)

This patch covers the following Core issues:
[CWG930](https://cplusplus.github.io/CWG/issues/930.html) "`alignof`
with incomplete array type"
[CWG1110](https://cplusplus.github.io/CWG/issues/1110.html) "Incomplete
return type should be allowed in `decltype` operand"
[CWG1340](https://cplusplus.github.io/CWG/issues/1340.html) "Complete
type in member pointer expressions"
[CWG1352](https://cplusplus.github.io/CWG/issues/1352.html)
"Inconsistent class scope and completeness rules"
[CWG1458](https://cplusplus.github.io/CWG/issues/1458.html) "Address of
incomplete type vs `operator&()`"
[CWG1824](https://cplusplus.github.io/CWG/issues/1824.html)
"Completeness of return type vs point of instantiation"
[CWG1832](https://cplusplus.github.io/CWG/issues/1832.html) "Casting to
incomplete enumeration"
[CWG2304](https://cplusplus.github.io/CWG/issues/2304.html) "Incomplete
type vs overload resolution"
[CWG2310](https://cplusplus.github.io/CWG/issues/2310.html) "Type
completeness and derived-to-base pointer conversions"
[CWG2430](https://cplusplus.github.io/CWG/issues/2430.html)
"Completeness of return and parameter types of member functions"
[CWG2512](https://cplusplus.github.io/CWG/issues/2512.html) "`typeid`
and incomplete class types"
[CWG2630](https://cplusplus.github.io/CWG/issues/2630.html) "Syntactic
specification of class completeness"
[CWG2718](https://cplusplus.github.io/CWG/issues/2718.html) "Type
completeness for derived-to-base conversions"
[CWG2857](https://cplusplus.github.io/CWG/issues/2857.html)
"Argument-dependent lookup with incomplete class types"

Current wording for CWG1110 came from
[P0135R1](https://wg21.link/p0135R1) "Wording for guaranteed copy
elision through simplified value categories".

As a drive-by fix, I fixed incorrect status of CWG1815, test for which
was added in #87933. CC @yronglin
---
 clang/test/CXX/drs/cwg11xx.cpp | 15 +++++++++++
 clang/test/CXX/drs/cwg13xx.cpp | 31 ++++++++++++++++++++++
 clang/test/CXX/drs/cwg14xx.cpp | 17 ++++++++++++
 clang/test/CXX/drs/cwg18xx.cpp | 28 +++++++++++++++++++-
 clang/test/CXX/drs/cwg23xx.cpp | 48 ++++++++++++++++++++++++++++++++--
 clang/test/CXX/drs/cwg24xx.cpp |  6 +++++
 clang/test/CXX/drs/cwg25xx.cpp | 15 ++++++++---
 clang/test/CXX/drs/cwg2630.cpp | 23 ++++++++++++++++
 clang/test/CXX/drs/cwg26xx.cpp |  2 ++
 clang/test/CXX/drs/cwg27xx.cpp | 14 +++++++---
 clang/test/CXX/drs/cwg28xx.cpp | 28 +++++++++++++++++---
 clang/test/CXX/drs/cwg9xx.cpp  |  7 +++++
 clang/www/cxx_dr_status.html   | 46 ++++++++++++++++++++------------
 13 files changed, 250 insertions(+), 30 deletions(-)
 create mode 100644 clang/test/CXX/drs/cwg2630.cpp

diff --git a/clang/test/CXX/drs/cwg11xx.cpp b/clang/test/CXX/drs/cwg11xx.cpp
index 46a0e526be39..8d187041400a 100644
--- a/clang/test/CXX/drs/cwg11xx.cpp
+++ b/clang/test/CXX/drs/cwg11xx.cpp
@@ -4,6 +4,21 @@
 // RUN: %clang_cc1 -std=c++17 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++2a %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors
 
+namespace cwg1110 { // cwg1110: 3.1
+#if __cplusplus >= 201103L
+template <typename T>
+T return_T();
+
+struct A;
+
+template <typename>
+struct B;
+
+decltype(return_T<A>())* a;
+decltype(return_T<B<int>>())* b;
+#endif
+} // namespace cwg1110
+
 namespace cwg1111 { // cwg1111: 3.2
 namespace example1 {
 template <typename> struct set; // #cwg1111-struct-set
diff --git a/clang/test/CXX/drs/cwg13xx.cpp b/clang/test/CXX/drs/cwg13xx.cpp
index a334b6d01acf..416de7c536b1 100644
--- a/clang/test/CXX/drs/cwg13xx.cpp
+++ b/clang/test/CXX/drs/cwg13xx.cpp
@@ -306,6 +306,18 @@ namespace cwg1330 { // cwg1330: 4 c++11
 
 // cwg1334: sup 1719
 
+namespace cwg1340 { // cwg1340: 2.9
+struct A;
+struct B;
+
+void f(B* a, A B::* p) {
+  (*a).*p;
+  // expected-warning@-1 {{expression result unused}}
+  a->*p;
+  // expected-warning@-1 {{expression result unused}}
+}
+} // namespace cwg1340
+
 namespace cwg1341 { // cwg1341: sup P0683R1
 #if __cplusplus >= 202002L
 int a;
@@ -451,6 +463,25 @@ static_assert(!__is_nothrow_constructible(D4, int), "");
 #endif
 } // namespace cwg1350
 
+namespace cwg1352 { // cwg1352: 3.0
+struct A {
+#if __cplusplus >= 201103L
+  int a = sizeof(A);
+#endif
+  void f(int b = sizeof(A));
+};
+
+template <typename T>
+struct B {
+#if __cplusplus >= 201103L
+  int a = sizeof(B) + sizeof(T);
+#endif
+  void f(int b = sizeof(B) + sizeof(T));
+};
+
+template class B<int>;
+} // namespace cwg1352
+
 namespace cwg1358 { // cwg1358: 3.1
 #if __cplusplus >= 201103L
   struct Lit { constexpr operator int() const { return 0; } };
diff --git a/clang/test/CXX/drs/cwg14xx.cpp b/clang/test/CXX/drs/cwg14xx.cpp
index 9ff9a68dc13c..f01d96ad47f3 100644
--- a/clang/test/CXX/drs/cwg14xx.cpp
+++ b/clang/test/CXX/drs/cwg14xx.cpp
@@ -86,6 +86,23 @@ struct A {
 };
 }
 
+namespace cwg1458 { // cwg1458: 3.1
+#if __cplusplus >= 201103L
+struct A;
+
+void f() {
+  constexpr A* a = nullptr;
+  constexpr int p = &*a;
+  // expected-error@-1 {{cannot initialize a variable of type 'const int' with an rvalue of type 'A *'}}
+  constexpr A *p2 = &*a;
+}
+
+struct A {
+  int operator&();
+};
+#endif
+} // namespace cwg1458
+
 namespace cwg1460 { // cwg1460: 3.5
 #if __cplusplus >= 201103L
   namespace DRExample {
diff --git a/clang/test/CXX/drs/cwg18xx.cpp b/clang/test/CXX/drs/cwg18xx.cpp
index 9eb749153e57..89adc2838490 100644
--- a/clang/test/CXX/drs/cwg18xx.cpp
+++ b/clang/test/CXX/drs/cwg18xx.cpp
@@ -206,7 +206,7 @@ namespace cwg1814 { // cwg1814: yes
 #endif
 }
 
-namespace cwg1815 { // cwg1815: yes
+namespace cwg1815 { // cwg1815: 19
 #if __cplusplus >= 201402L
   struct A { int &&r = 0; };
   A a = {};
@@ -303,6 +303,32 @@ namespace cwg1822 { // cwg1822: yes
 #endif
 }
 
+namespace cwg1824 { // cwg1824: 2.7
+template<typename T>
+struct A {
+  T t;
+};
+
+struct S {
+  A<S> f() { return A<S>(); }
+};
+} // namespace cwg1824
+
+namespace cwg1832 { // cwg1832: 3.0
+enum E { // #cwg1832-E
+  a = static_cast<int>(static_cast<E>(0))
+  // expected-error@-1 {{'E' is an incomplete type}}
+  //   expected-note@#cwg1832-E {{definition of 'cwg1832::E' is not complete until the closing '}'}}
+};
+
+#if __cplusplus >= 201103L
+enum E2: decltype(static_cast<E2>(0), 0) {};
+// expected-error@-1 {{unknown type name 'E2'}}
+enum class E3: decltype(static_cast<E3>(0), 0) {};
+// expected-error@-1 {{unknown type name 'E3'}}
+#endif
+} // namespace cwg1832
+
 namespace cwg1837 { // cwg1837: 3.3
 #if __cplusplus >= 201103L
   template <typename T>
diff --git a/clang/test/CXX/drs/cwg23xx.cpp b/clang/test/CXX/drs/cwg23xx.cpp
index db5b7c3cd3c9..ae5ec3b878f5 100644
--- a/clang/test/CXX/drs/cwg23xx.cpp
+++ b/clang/test/CXX/drs/cwg23xx.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -std=c++98 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
-// RUN: %clang_cc1 -std=c++11 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
-// RUN: %clang_cc1 -std=c++14 %s -verify=expected,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx11-14,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
 // RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx11,since-cxx14,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
 // RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11,since-cxx14,since-cxx17,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
 // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11,since-cxx14,since-cxx17,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
@@ -47,6 +47,50 @@ void g() {
 } // namespace cwg2303
 #endif
 
+namespace cwg2304 { // cwg2304: 2.8
+template<typename T> void foo(T, int);
+template<typename T> void foo(T&, ...);
+struct Q; // #cwg2304-Q
+void fn1(Q &data_vectors) {
+  foo(data_vectors, 0);
+  // expected-error@-1 {{argument type 'cwg2304::Q' is incomplete}}
+  //   expected-note@#cwg2304-Q {{forward declaration of 'cwg2304::Q'}}
+}
+} // namespace cwg2304
+
+namespace cwg2310 { // cwg2310: partial
+#if __cplusplus >= 201103L
+template<typename A, typename B>
+struct check_derived_from {
+  static A a;
+  // FIXME: all 3 examples should be rejected in all language modes.
+  // FIXME: we should test this in 98 mode.
+  // FIXME: we accept this when MSVC triple is used
+  static constexpr B *p = &a;
+#if !defined(_WIN32) || defined(__MINGW32__)
+  // cxx11-14-error@-2 {{cannot initialize a variable of type 'cwg2310::X *const' with an rvalue of type 'cwg2310::Z *'}}
+  //   cxx11-14-note@#cwg2310-X {{in instantiation of template class 'cwg2310::check_derived_from<cwg2310::Z, cwg2310::X>' requested here}}
+  // cxx11-14-error@-4 {{cannot initialize a variable of type 'cwg2310::Y *const' with an rvalue of type 'cwg2310::Z *'}}
+  //   cxx11-14-note@#cwg2310-Y {{in instantiation of template class 'cwg2310::check_derived_from<cwg2310::Z, cwg2310::Y>' requested here}}
+#endif
+};
+
+struct W {};
+struct X {};
+struct Y {};
+struct Z : W,
+  X, check_derived_from<Z, X>, // #cwg2310-X
+  check_derived_from<Z, Y>, Y  // #cwg2310-Y
+{  
+  // FIXME: It was properly rejected before, but we're crashing since Clang 11 in C++11 and C++14 modes.
+  //        See https://github.com/llvm/llvm-project/issues/59920
+#if __cplusplus >= 201703L
+  check_derived_from<Z, W> cdf;
+#endif
+};
+#endif
+} // namespace cwg2310
+
 // cwg2331: na
 // cwg2335 is in cwg2335.cxx
 
diff --git a/clang/test/CXX/drs/cwg24xx.cpp b/clang/test/CXX/drs/cwg24xx.cpp
index 9f876cd87083..75e1a614765c 100644
--- a/clang/test/CXX/drs/cwg24xx.cpp
+++ b/clang/test/CXX/drs/cwg24xx.cpp
@@ -45,6 +45,12 @@ void fallthrough(int n) {
 #endif
 }
 
+namespace cwg2430 { // cwg2430: 2.7
+struct S {
+  S f(S s) { return s; }
+};
+} // namespace cwg2430
+
 namespace cwg2450 { // cwg2450: 18
 #if __cplusplus >= 202302L
 struct S {int a;};
diff --git a/clang/test/CXX/drs/cwg25xx.cpp b/clang/test/CXX/drs/cwg25xx.cpp
index 8bca58f44944..0934f0cc19c6 100644
--- a/clang/test/CXX/drs/cwg25xx.cpp
+++ b/clang/test/CXX/drs/cwg25xx.cpp
@@ -6,12 +6,21 @@
 // RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20,since-cxx23 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20,since-cxx23 -fexceptions -fcxx-exceptions -pedantic-errors
 
-#if __cplusplus == 199711L
-// expected-no-diagnostics
-#endif
+namespace std {
+struct type_info{};
+} // namespace std
 
 // cwg2504 is in cwg2504.cpp
 
+namespace cwg2512 { // cwg2512: 2.7
+struct A; // #cwg2512-A
+void foo(A* p) {
+  typeid(*p);
+  // expected-error@-1 {{'typeid' of incomplete type 'A'}}
+  //   expected-note@#cwg2512-A {{forward declaration of 'cwg2512::A'}}
+}
+} // namespace cwg2512
+
 namespace cwg2516 { // cwg2516: 3.0
                    // NB: reusing 1482 test
 #if __cplusplus >= 201103L
diff --git a/clang/test/CXX/drs/cwg2630.cpp b/clang/test/CXX/drs/cwg2630.cpp
new file mode 100644
index 000000000000..0f50dc4f7458
--- /dev/null
+++ b/clang/test/CXX/drs/cwg2630.cpp
@@ -0,0 +1,23 @@
+// RUN: split-file --leading-lines %s %t
+// RUN: %clang_cc1 -std=c++20 -verify -emit-module-interface %t/module.cppm -o %t/module.pcm
+// RUN: %clang_cc1 -std=c++20 -verify -fmodule-file=A=%t/module.pcm %t/main.cpp
+// RUN: %clang_cc1 -std=c++23 -verify -emit-module-interface %t/module.cppm -o %t/module.pcm
+// RUN: %clang_cc1 -std=c++23 -verify -fmodule-file=A=%t/module.pcm %t/main.cpp
+// RUN: %clang_cc1 -std=c++2c -verify -emit-module-interface %t/module.cppm -o %t/module.pcm
+// RUN: %clang_cc1 -std=c++2c -verify -fmodule-file=A=%t/module.pcm %t/main.cpp
+
+//--- module.cppm
+// expected-no-diagnostics
+export module A;
+
+namespace cwg2630 {
+export class X {};
+} // namespace cwg2630
+
+//--- main.cpp
+// expected-no-diagnostics
+import A;
+
+namespace cwg2630 { // cwg2630: 9
+X x;
+} // namespace cwg2630
diff --git a/clang/test/CXX/drs/cwg26xx.cpp b/clang/test/CXX/drs/cwg26xx.cpp
index f7a05b9827a2..d3c5b5bb7b6b 100644
--- a/clang/test/CXX/drs/cwg26xx.cpp
+++ b/clang/test/CXX/drs/cwg26xx.cpp
@@ -49,6 +49,8 @@ void f() {
 #endif
 }
 
+// cwg2630 is in cwg2630.cpp
+
 namespace cwg2631 { // cwg2631: 16
 #if __cplusplus >= 202002L
   constexpr int g();
diff --git a/clang/test/CXX/drs/cwg27xx.cpp b/clang/test/CXX/drs/cwg27xx.cpp
index 0434427d6c92..53ddd566b7db 100644
--- a/clang/test/CXX/drs/cwg27xx.cpp
+++ b/clang/test/CXX/drs/cwg27xx.cpp
@@ -6,9 +6,17 @@
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++23 -verify=expected,since-cxx23 %s
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++2c -verify=expected,since-cxx23,since-cxx26 %s
 
-#if __cplusplus <= 202002L
-// expected-no-diagnostics
-#endif
+namespace cwg2718 { // cwg2718: 2.7
+struct B {};
+struct D;
+
+void f(B b) {
+  static_cast<D&>(b);
+  // expected-error@-1 {{non-const lvalue reference to type 'D' cannot bind to a value of unrelated type 'B'}}
+}
+
+struct D : B {};
+} // namespace cwg2718
 
 namespace cwg2759 { // cwg2759: 19
 #if __cplusplus >= 201103L
diff --git a/clang/test/CXX/drs/cwg28xx.cpp b/clang/test/CXX/drs/cwg28xx.cpp
index be35d366bdd6..696cd1b9c84e 100644
--- a/clang/test/CXX/drs/cwg28xx.cpp
+++ b/clang/test/CXX/drs/cwg28xx.cpp
@@ -6,10 +6,6 @@
 // RUN: %clang_cc1 -std=c++23 -verify=expected,since-cxx20,since-cxx23 %s
 // RUN: %clang_cc1 -std=c++2c -verify=expected,since-cxx20,since-cxx23,since-cxx26 %s
 
-#if __cplusplus < 202002L
-// expected-no-diagnostics
-#endif
-
 namespace cwg2819 { // cwg2819: 19 tentatively ready 2023-12-01
 #if __cpp_constexpr >= 202306L
   constexpr void* p = nullptr;
@@ -67,6 +63,30 @@ void B<int>::g() requires true;
 
 } // namespace cwg2847
 
+namespace cwg2857 { // cwg2857: no
+struct A {};
+template <typename>
+struct D;
+namespace N {
+  struct B {};
+  void adl_only(A*, D<int>*); // #cwg2857-adl_only
+}
+
+void f(A* a, D<int>* d) {
+  adl_only(a, d);
+  // expected-error@-1 {{use of undeclared identifier 'adl_only'; did you mean 'N::adl_only'?}}
+  //   expected-note@#cwg2857-adl_only {{'N::adl_only' declared here}}
+}
+
+#if __cplusplus >= 201103L
+template <typename>
+struct D : N::B {
+  // FIXME: ADL shouldn't associate it's base B and N since D is not complete here
+  decltype(adl_only((A*) nullptr, (D*) nullptr)) f;
+};
+#endif
+} // namespace cwg2857
+
 namespace cwg2858 { // cwg2858: 19 tentatively ready 2024-04-05
 
 #if __cplusplus > 202302L
diff --git a/clang/test/CXX/drs/cwg9xx.cpp b/clang/test/CXX/drs/cwg9xx.cpp
index 8ecb149c355f..2700b0f5662a 100644
--- a/clang/test/CXX/drs/cwg9xx.cpp
+++ b/clang/test/CXX/drs/cwg9xx.cpp
@@ -14,6 +14,13 @@ namespace std {
   };
 }
 
+namespace cwg930 { // cwg930: 2.7
+#if __cplusplus >= 201103L
+static_assert(alignof(int[]) == alignof(int), "");
+static_assert(alignof(int[][2]) == alignof(int[2]), "");
+#endif
+} // namespace cwg930
+
 namespace cwg948 { // cwg948: 3.7
 #if __cplusplus >= 201103L
   class A {
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 92fdcf5556ed..abf5d4ae4676 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -5388,7 +5388,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/930.html">930</a></td>
     <td>CD2</td>
     <td><TT>alignof</TT> with incomplete array type</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.7</td>
   </tr>
   <tr id="931">
     <td><a href="https://cplusplus.github.io/CWG/issues/931.html">931</a></td>
@@ -6468,7 +6468,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1110.html">1110</a></td>
     <td>NAD</td>
     <td>Incomplete return type should be allowed in <TT>decltype</TT> operand</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 3.1</td>
   </tr>
   <tr id="1111">
     <td><a href="https://cplusplus.github.io/CWG/issues/1111.html">1111</a></td>
@@ -7848,7 +7848,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1340.html">1340</a></td>
     <td>CD3</td>
     <td>Complete type in member pointer expressions</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.9</td>
   </tr>
   <tr id="1341">
     <td><a href="https://cplusplus.github.io/CWG/issues/1341.html">1341</a></td>
@@ -7920,7 +7920,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1352.html">1352</a></td>
     <td>CD3</td>
     <td>Inconsistent class scope and completeness rules</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 3.0</td>
   </tr>
   <tr id="1353">
     <td><a href="https://cplusplus.github.io/CWG/issues/1353.html">1353</a></td>
@@ -8556,7 +8556,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1458.html">1458</a></td>
     <td>CD3</td>
     <td>Address of incomplete type vs <TT>operator&amp;()</TT></td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 3.1</td>
   </tr>
   <tr class="open" id="1459">
     <td><a href="https://cplusplus.github.io/CWG/issues/1459.html">1459</a></td>
@@ -10752,7 +10752,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1824.html">1824</a></td>
     <td>CD4</td>
     <td>Completeness of return type vs point of instantiation</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.7</td>
   </tr>
   <tr id="1825">
     <td><a href="https://cplusplus.github.io/CWG/issues/1825.html">1825</a></td>
@@ -10800,7 +10800,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1832.html">1832</a></td>
     <td>CD4</td>
     <td>Casting to incomplete enumeration</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 3.0</td>
   </tr>
   <tr id="1833">
     <td><a href="https://cplusplus.github.io/CWG/issues/1833.html">1833</a></td>
@@ -13632,7 +13632,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/2304.html">2304</a></td>
     <td>NAD</td>
     <td>Incomplete type vs overload resolution</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.8</td>
   </tr>
   <tr id="2305">
     <td><a href="https://cplusplus.github.io/CWG/issues/2305.html">2305</a></td>
@@ -13668,7 +13668,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/2310.html">2310</a></td>
     <td>CD5</td>
     <td>Type completeness and derived-to-base pointer conversions</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="partial" align="center">Partial</td>
   </tr>
   <tr class="open" id="2311">
     <td><a href="https://cplusplus.github.io/CWG/issues/2311.html">2311</a></td>
@@ -14388,7 +14388,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/2430.html">2430</a></td>
     <td>C++20</td>
     <td>Completeness of return and parameter types of member functions</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.7</td>
   </tr>
   <tr id="2431">
     <td><a href="https://cplusplus.github.io/CWG/issues/2431.html">2431</a></td>
@@ -14880,7 +14880,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/2512.html">2512</a></td>
     <td>NAD</td>
     <td><TT>typeid</TT> and incomplete class types</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.7</td>
   </tr>
   <tr class="open" id="2513">
     <td><a href="https://cplusplus.github.io/CWG/issues/2513.html">2513</a></td>
@@ -15588,7 +15588,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/2630.html">2630</a></td>
     <td>C++23</td>
     <td>Syntactic specification of class completeness</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="2631">
     <td><a href="https://cplusplus.github.io/CWG/issues/2631.html">2631</a></td>
@@ -16116,7 +16116,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/2718.html">2718</a></td>
     <td>DRWP</td>
     <td>Type completeness for derived-to-base conversions</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.7</td>
   </tr>
   <tr id="2719">
     <td><a href="https://cplusplus.github.io/CWG/issues/2719.html">2719</a></td>
@@ -16951,7 +16951,7 @@ objects</td>
     <td><a href="https://cplusplus.github.io/CWG/issues/2857.html">2857</a></td>
     <td>DR</td>
     <td>Argument-dependent lookup with incomplete class types</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="none" align="center">No</td>
   </tr>
   <tr class="open" id="2858">
     <td><a href="https://cplusplus.github.io/CWG/issues/2858.html">2858</a></td>
@@ -16985,7 +16985,7 @@ objects</td>
   </tr>
   <tr class="open" id="2863">
     <td><a href="https://cplusplus.github.io/CWG/issues/2863.html">2863</a></td>
-    <td>tentatively ready</td>
+    <td>drafting</td>
     <td>Unclear synchronization requirements for object lifetime rules</td>
     <td align="center">Not resolved</td>
   </tr>
@@ -17021,13 +17021,13 @@ objects</td>
   </tr>
   <tr class="open" id="2869">
     <td><a href="https://cplusplus.github.io/CWG/issues/2869.html">2869</a></td>
-    <td>open</td>
+    <td>review</td>
     <td><TT>this</TT> in local classes</td>
     <td align="center">Not resolved</td>
   </tr>
   <tr class="open" id="2870">
     <td><a href="https://cplusplus.github.io/CWG/issues/2870.html">2870</a></td>
-    <td>open</td>
+    <td>review</td>
     <td>Combining absent <I>encoding-prefix</I>es</td>
     <td align="center">Not resolved</td>
   </tr>
@@ -17138,6 +17138,18 @@ objects</td>
     <td>open</td>
     <td>Missing cases for reference and array types for argument-dependent lookup</td>
     <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2889">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2889.html">2889</a></td>
+    <td>open</td>
+    <td>Requiring an accessible destructor for destroying operator delete</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2890">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2890.html">2890</a></td>
+    <td>open</td>
+    <td>Defining members of local classes</td>
+    <td align="center">Not resolved</td>
   </tr></table>
 
 </div>
-- 
GitLab


From 64b3cdc0220174c1af236a42b227a5226f0f12c5 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Wed, 15 May 2024 14:13:50 -0500
Subject: [PATCH 0906/1206] [libc] Fix GPU handling for unsupported backends
 (#92271)

Summary:
If the user does not have the selected backend enabled, we should still
be able to build the LLVM-IR an ddistribute it. This patch makes logic
to suppress tests if the backend can't build it, as well as removing a
flag for the building that's only present int he NVPTX backend.
---
 libc/cmake/modules/LLVMLibCCompileOptionRules.cmake |  1 -
 libc/cmake/modules/prepare_libc_gpu_build.cmake     | 10 +++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index 5b3a10d55fed..3bf429381d4a 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -101,7 +101,6 @@ function(_get_common_compile_options output_var flags)
 
     if(LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
       list(APPEND compile_options "-Wno-unknown-cuda-version")
-      list(APPEND compile_options "SHELL:-mllvm -nvptx-emit-init-fini-kernel=false")
       list(APPEND compile_options "--cuda-feature=+ptx63")
       if(LIBC_CUDA_ROOT)
         list(APPEND compile_options "--cuda-path=${LIBC_CUDA_ROOT}")
diff --git a/libc/cmake/modules/prepare_libc_gpu_build.cmake b/libc/cmake/modules/prepare_libc_gpu_build.cmake
index 20aca16990fc..88538caaa3bc 100644
--- a/libc/cmake/modules/prepare_libc_gpu_build.cmake
+++ b/libc/cmake/modules/prepare_libc_gpu_build.cmake
@@ -76,7 +76,15 @@ elseif(LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
 endif()
 
 set(gpu_test_architecture "")
-if(LIBC_GPU_TEST_ARCHITECTURE)
+if(DEFINED LLVM_TARGETS_TO_BUILD AND LIBC_TARGET_ARCHITECTURE_IS_AMDGPU
+   AND NOT "AMDGPU" IN_LIST LLVM_TARGETS_TO_BUILD)
+  set(LIBC_GPU_TESTS_DISABLED TRUE)
+  message(STATUS "AMDGPU backend is not available, tests will not be built")
+elseif(DEFINED LLVM_TARGETS_TO_BUILD AND LIBC_TARGET_ARCHITECTURE_IS_AMDGPU
+       AND NOT "NVPTX" IN_LIST LLVM_TARGETS_TO_BUILD)
+  set(LIBC_GPU_TESTS_DISABLED TRUE)
+  message(STATUS "NVPTX backend is not available, tests will not be built")
+elseif(LIBC_GPU_TEST_ARCHITECTURE)
   set(LIBC_GPU_TESTS_DISABLED FALSE)
   set(gpu_test_architecture ${LIBC_GPU_TEST_ARCHITECTURE})
   message(STATUS "Using user-specified GPU architecture for testing: "
-- 
GitLab


From 4ab2ac22d0a481460536f673377b644702cb3372 Mon Sep 17 00:00:00 2001
From: Patrick O'Neill <patrick@rivosinc.com>
Date: Wed, 15 May 2024 12:39:28 -0700
Subject: [PATCH 0907/1206] [DAGCombiner] Mark vectors as not
 AllAddOne/AllSubOne on type mismatch (#92195)

Fixes #92193.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  7 +++++--
 llvm/test/CodeGen/RISCV/pr92193.ll            | 21 +++++++++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/pr92193.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index a044b6dc4838..2b181cd3ab1d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -12142,8 +12142,11 @@ SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
     SDValue N2Elt = N2.getOperand(i);
     if (N1Elt.isUndef() || N2Elt.isUndef())
       continue;
-    if (N1Elt.getValueType() != N2Elt.getValueType())
-      continue;
+    if (N1Elt.getValueType() != N2Elt.getValueType()) {
+      AllAddOne = false;
+      AllSubOne = false;
+      break;
+    }
 
     const APInt &C1 = N1Elt->getAsAPIntVal();
     const APInt &C2 = N2Elt->getAsAPIntVal();
diff --git a/llvm/test/CodeGen/RISCV/pr92193.ll b/llvm/test/CodeGen/RISCV/pr92193.ll
new file mode 100644
index 000000000000..8c8398c4b45f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/pr92193.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=riscv64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32-unknown-linux-gnu < %s | FileCheck %s
+
+; Dag-combine used to improperly combine a vector vselect of 0 and 2 into
+; 2 + condition(0/1) because one of the two args was transformed from an i32->i64.
+
+define i16 @foo() {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+entry:
+  %insert.0 = insertelement <4 x i16> zeroinitializer, i16 2, i64 0
+  %all.two = shufflevector <4 x i16> %insert.0, <4 x i16> zeroinitializer, <4 x i32> zeroinitializer
+  %sel.0 = select <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i16> zeroinitializer, <4 x i16> %all.two
+  %mul.0 = call i16 @llvm.vector.reduce.mul.v4i16(<4 x i16> %sel.0)
+  ret i16 %mul.0
+}
+
+declare i16 @llvm.vector.reduce.mul.v4i32(<4 x i16>)
-- 
GitLab


From fc8775e2142c6bd7876831c27c3fbef0d64860bc Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Wed, 15 May 2024 12:45:50 -0700
Subject: [PATCH 0908/1206] "Reapply "[ctx_profile] Profile reader and writer"
 (#92199)"

This reverts commit 2c54bf497f7d7aecd24f4b849ee08e37a3519611.

Fixed gcc-7 issue.
---
 .../llvm/ProfileData/PGOCtxProfReader.h       |  92 +++++++
 .../llvm/ProfileData/PGOCtxProfWriter.h       |  91 +++++++
 llvm/lib/ProfileData/CMakeLists.txt           |   3 +
 llvm/lib/ProfileData/PGOCtxProfReader.cpp     | 173 ++++++++++++
 llvm/lib/ProfileData/PGOCtxProfWriter.cpp     |  49 ++++
 llvm/unittests/ProfileData/CMakeLists.txt     |   1 +
 .../PGOCtxProfReaderWriterTest.cpp            | 255 ++++++++++++++++++
 7 files changed, 664 insertions(+)
 create mode 100644 llvm/include/llvm/ProfileData/PGOCtxProfReader.h
 create mode 100644 llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
 create mode 100644 llvm/lib/ProfileData/PGOCtxProfReader.cpp
 create mode 100644 llvm/lib/ProfileData/PGOCtxProfWriter.cpp
 create mode 100644 llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp

diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
new file mode 100644
index 000000000000..a19b3f51d642
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
@@ -0,0 +1,92 @@
+//===--- PGOCtxProfReader.h - Contextual profile reader ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// Reader for contextual iFDO profile, which comes in bitstream format.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
+#define LLVM_PROFILEDATA_CTXINSTRPROFILEREADER_H
+
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/Bitstream/BitstreamReader.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Support/Error.h"
+#include <map>
+#include <vector>
+
+namespace llvm {
+/// The loaded contextual profile, suitable for mutation during IPO passes. We
+/// generally expect a fraction of counters and of callsites to be populated.
+/// We continue to model counters as vectors, but callsites are modeled as a map
+/// of a map. The expectation is that, typically, there is a small number of
+/// indirect targets (usually, 1 for direct calls); but potentially a large
+/// number of callsites, and, as inlining progresses, the callsite count of a
+/// caller will grow.
+class PGOContextualProfile final {
+public:
+  using CallTargetMapTy = std::map<GlobalValue::GUID, PGOContextualProfile>;
+  using CallsiteMapTy = DenseMap<uint32_t, CallTargetMapTy>;
+
+private:
+  friend class PGOCtxProfileReader;
+  GlobalValue::GUID GUID = 0;
+  SmallVector<uint64_t, 16> Counters;
+  CallsiteMapTy Callsites;
+
+  PGOContextualProfile(GlobalValue::GUID G,
+                       SmallVectorImpl<uint64_t> &&Counters)
+      : GUID(G), Counters(std::move(Counters)) {}
+
+  Expected<PGOContextualProfile &>
+  getOrEmplace(uint32_t Index, GlobalValue::GUID G,
+               SmallVectorImpl<uint64_t> &&Counters);
+
+public:
+  PGOContextualProfile(const PGOContextualProfile &) = delete;
+  PGOContextualProfile &operator=(const PGOContextualProfile &) = delete;
+  PGOContextualProfile(PGOContextualProfile &&) = default;
+  PGOContextualProfile &operator=(PGOContextualProfile &&) = default;
+
+  GlobalValue::GUID guid() const { return GUID; }
+  const SmallVectorImpl<uint64_t> &counters() const { return Counters; }
+  const CallsiteMapTy &callsites() const { return Callsites; }
+  CallsiteMapTy &callsites() { return Callsites; }
+
+  bool hasCallsite(uint32_t I) const {
+    return Callsites.find(I) != Callsites.end();
+  }
+
+  const CallTargetMapTy &callsite(uint32_t I) const {
+    assert(hasCallsite(I) && "Callsite not found");
+    return Callsites.find(I)->second;
+  }
+  void getContainedGuids(DenseSet<GlobalValue::GUID> &Guids) const;
+};
+
+class PGOCtxProfileReader final {
+  BitstreamCursor &Cursor;
+  Expected<BitstreamEntry> advance();
+  Error readMetadata();
+  Error wrongValue(const Twine &);
+  Error unsupported(const Twine &);
+
+  Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
+  readContext(bool ExpectIndex);
+  bool canReadContext();
+
+public:
+  PGOCtxProfileReader(BitstreamCursor &Cursor) : Cursor(Cursor) {}
+
+  Expected<std::map<GlobalValue::GUID, PGOContextualProfile>> loadContexts();
+};
+} // namespace llvm
+#endif
diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
new file mode 100644
index 000000000000..15578c51a495
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
@@ -0,0 +1,91 @@
+//===- PGOCtxProfWriter.h - Contextual Profile Writer -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares a utility for writing a contextual profile to bitstream.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
+#define LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
+
+#include "llvm/Bitstream/BitstreamWriter.h"
+#include "llvm/ProfileData/CtxInstrContextNode.h"
+
+namespace llvm {
+enum PGOCtxProfileRecords { Invalid = 0, Version, Guid, CalleeIndex, Counters };
+
+enum PGOCtxProfileBlockIDs {
+  ProfileMetadataBlockID = 100,
+  ContextNodeBlockID = ProfileMetadataBlockID + 1
+};
+
+/// Write one or more ContextNodes to the provided raw_fd_stream.
+/// The caller must destroy the PGOCtxProfileWriter object before closing the
+/// stream.
+/// The design allows serializing a bunch of contexts embedded in some other
+/// file. The overall format is:
+///
+///  [... other data written to the stream...]
+///  SubBlock(ProfileMetadataBlockID)
+///   Version
+///   SubBlock(ContextNodeBlockID)
+///     [RECORDS]
+///     SubBlock(ContextNodeBlockID)
+///       [RECORDS]
+///       [... more SubBlocks]
+///     EndBlock
+///   EndBlock
+///
+/// The "RECORDS" are bitsream records. The IDs are in CtxProfileCodes (except)
+/// for Version, which is just for metadata). All contexts will have Guid and
+/// Counters, and all but the roots have CalleeIndex. The order in which the
+/// records appear does not matter, but they must precede any subcontexts,
+/// because that helps keep the reader code simpler.
+///
+/// Subblock containment captures the context->subcontext relationship. The
+/// "next()" relationship in the raw profile, between call targets of indirect
+/// calls, are just modeled as peer subblocks where the callee index is the
+/// same.
+///
+/// Versioning: the writer may produce additional records not known by the
+/// reader. The version number indicates a more structural change.
+/// The current version, in particular, is set up to expect optional extensions
+/// like value profiling - which would appear as additional records. For
+/// example, value profiling would produce a new record with a new record ID,
+/// containing the profiled values (much like the counters)
+class PGOCtxProfileWriter final {
+  SmallVector<char, 1 << 20> Buff;
+  BitstreamWriter Writer;
+
+  void writeCounters(const ctx_profile::ContextNode &Node);
+  void writeImpl(std::optional<uint32_t> CallerIndex,
+                 const ctx_profile::ContextNode &Node);
+
+public:
+  PGOCtxProfileWriter(raw_fd_stream &Out,
+                      std::optional<unsigned> VersionOverride = std::nullopt)
+      : Writer(Buff, &Out, 0) {
+    Writer.EnterSubblock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID,
+                         CodeLen);
+    const auto Version = VersionOverride ? *VersionOverride : CurrentVersion;
+    Writer.EmitRecord(PGOCtxProfileRecords::Version,
+                      SmallVector<unsigned, 1>({Version}));
+  }
+
+  ~PGOCtxProfileWriter() { Writer.ExitBlock(); }
+
+  void write(const ctx_profile::ContextNode &);
+
+  // constants used in writing which a reader may find useful.
+  static constexpr unsigned CodeLen = 2;
+  static constexpr uint32_t CurrentVersion = 1;
+  static constexpr unsigned VBREncodingBits = 6;
+};
+
+} // namespace llvm
+#endif
diff --git a/llvm/lib/ProfileData/CMakeLists.txt b/llvm/lib/ProfileData/CMakeLists.txt
index 408f9ff01ec8..4fa1b76f0a06 100644
--- a/llvm/lib/ProfileData/CMakeLists.txt
+++ b/llvm/lib/ProfileData/CMakeLists.txt
@@ -7,6 +7,8 @@ add_llvm_component_library(LLVMProfileData
   ItaniumManglingCanonicalizer.cpp
   MemProf.cpp
   MemProfReader.cpp
+  PGOCtxProfReader.cpp
+  PGOCtxProfWriter.cpp
   ProfileSummaryBuilder.cpp
   SampleProf.cpp
   SampleProfReader.cpp
@@ -20,6 +22,7 @@ add_llvm_component_library(LLVMProfileData
   intrinsics_gen
 
   LINK_COMPONENTS
+  BitstreamReader
   Core
   Object
   Support
diff --git a/llvm/lib/ProfileData/PGOCtxProfReader.cpp b/llvm/lib/ProfileData/PGOCtxProfReader.cpp
new file mode 100644
index 000000000000..1b42d8c765f2
--- /dev/null
+++ b/llvm/lib/ProfileData/PGOCtxProfReader.cpp
@@ -0,0 +1,173 @@
+//===- PGOCtxProfReader.cpp - Contextual Instrumentation profile reader ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Read a contextual profile into a datastructure suitable for maintenance
+// throughout IPO
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ProfileData/PGOCtxProfReader.h"
+#include "llvm/Bitstream/BitCodeEnums.h"
+#include "llvm/Bitstream/BitstreamReader.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+
+using namespace llvm;
+
+// FIXME(#92054) - these Error handling macros are (re-)invented in a few
+// places.
+#define EXPECT_OR_RET(LHS, RHS)                                                \
+  auto LHS = RHS;                                                              \
+  if (!LHS)                                                                    \
+    return LHS.takeError();
+
+#define RET_ON_ERR(EXPR)                                                       \
+  if (auto Err = (EXPR))                                                       \
+    return Err;
+
+Expected<PGOContextualProfile &>
+PGOContextualProfile::getOrEmplace(uint32_t Index, GlobalValue::GUID G,
+                                   SmallVectorImpl<uint64_t> &&Counters) {
+  auto [Iter, Inserted] = Callsites[Index].insert(
+      {G, PGOContextualProfile(G, std::move(Counters))});
+  if (!Inserted)
+    return make_error<InstrProfError>(instrprof_error::invalid_prof,
+                                      "Duplicate GUID for same callsite.");
+  return Iter->second;
+}
+
+void PGOContextualProfile::getContainedGuids(
+    DenseSet<GlobalValue::GUID> &Guids) const {
+  Guids.insert(GUID);
+  for (const auto &[_, Callsite] : Callsites)
+    for (const auto &[_, Callee] : Callsite)
+      Callee.getContainedGuids(Guids);
+}
+
+Expected<BitstreamEntry> PGOCtxProfileReader::advance() {
+  return Cursor.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs);
+}
+
+Error PGOCtxProfileReader::wrongValue(const Twine &Msg) {
+  return make_error<InstrProfError>(instrprof_error::invalid_prof, Msg);
+}
+
+Error PGOCtxProfileReader::unsupported(const Twine &Msg) {
+  return make_error<InstrProfError>(instrprof_error::unsupported_version, Msg);
+}
+
+bool PGOCtxProfileReader::canReadContext() {
+  auto Blk = advance();
+  if (!Blk) {
+    consumeError(Blk.takeError());
+    return false;
+  }
+  return Blk->Kind == BitstreamEntry::SubBlock &&
+         Blk->ID == PGOCtxProfileBlockIDs::ContextNodeBlockID;
+}
+
+Expected<std::pair<std::optional<uint32_t>, PGOContextualProfile>>
+PGOCtxProfileReader::readContext(bool ExpectIndex) {
+  RET_ON_ERR(Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID));
+
+  std::optional<ctx_profile::GUID> Guid;
+  std::optional<SmallVector<uint64_t, 16>> Counters;
+  std::optional<uint32_t> CallsiteIndex;
+
+  SmallVector<uint64_t, 1> RecordValues;
+
+  // We don't prescribe the order in which the records come in, and we are ok
+  // if other unsupported records appear. We seek in the current subblock until
+  // we get all we know.
+  auto GotAllWeNeed = [&]() {
+    return Guid.has_value() && Counters.has_value() &&
+           (!ExpectIndex || CallsiteIndex.has_value());
+  };
+  while (!GotAllWeNeed()) {
+    RecordValues.clear();
+    EXPECT_OR_RET(Entry, advance());
+    if (Entry->Kind != BitstreamEntry::Record)
+      return wrongValue(
+          "Expected records before encountering more subcontexts");
+    EXPECT_OR_RET(ReadRecord,
+                  Cursor.readRecord(bitc::UNABBREV_RECORD, RecordValues));
+    switch (*ReadRecord) {
+    case PGOCtxProfileRecords::Guid:
+      if (RecordValues.size() != 1)
+        return wrongValue("The GUID record should have exactly one value");
+      Guid = RecordValues[0];
+      break;
+    case PGOCtxProfileRecords::Counters:
+      Counters = std::move(RecordValues);
+      if (Counters->empty())
+        return wrongValue("Empty counters. At least the entry counter (one "
+                          "value) was expected");
+      break;
+    case PGOCtxProfileRecords::CalleeIndex:
+      if (!ExpectIndex)
+        return wrongValue("The root context should not have a callee index");
+      if (RecordValues.size() != 1)
+        return wrongValue("The callee index should have exactly one value");
+      CallsiteIndex = RecordValues[0];
+      break;
+    default:
+      // OK if we see records we do not understand, like records (profile
+      // components) introduced later.
+      break;
+    }
+  }
+
+  PGOContextualProfile Ret(*Guid, std::move(*Counters));
+
+  while (canReadContext()) {
+    EXPECT_OR_RET(SC, readContext(true));
+    auto &Targets = Ret.callsites()[*SC->first];
+    auto [_, Inserted] =
+        Targets.insert({SC->second.guid(), std::move(SC->second)});
+    if (!Inserted)
+      return wrongValue(
+          "Unexpected duplicate target (callee) at the same callsite.");
+  }
+  return std::make_pair(CallsiteIndex, std::move(Ret));
+}
+
+Error PGOCtxProfileReader::readMetadata() {
+  EXPECT_OR_RET(Blk, advance());
+  if (Blk->Kind != BitstreamEntry::SubBlock)
+    return unsupported("Expected Version record");
+  RET_ON_ERR(
+      Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID));
+  EXPECT_OR_RET(MData, advance());
+  if (MData->Kind != BitstreamEntry::Record)
+    return unsupported("Expected Version record");
+
+  SmallVector<uint64_t, 1> Ver;
+  EXPECT_OR_RET(Code, Cursor.readRecord(bitc::UNABBREV_RECORD, Ver));
+  if (*Code != PGOCtxProfileRecords::Version)
+    return unsupported("Expected Version record");
+  if (Ver.size() != 1 || Ver[0] > PGOCtxProfileWriter::CurrentVersion)
+    return unsupported("Version " + Twine(*Code) +
+                       " is higher than supported version " +
+                       Twine(PGOCtxProfileWriter::CurrentVersion));
+  return Error::success();
+}
+
+Expected<std::map<GlobalValue::GUID, PGOContextualProfile>>
+PGOCtxProfileReader::loadContexts() {
+  std::map<GlobalValue::GUID, PGOContextualProfile> Ret;
+  RET_ON_ERR(readMetadata());
+  while (canReadContext()) {
+    EXPECT_OR_RET(E, readContext(false));
+    auto Key = E->second.guid();
+    if (!Ret.insert({Key, std::move(E->second)}).second)
+      return wrongValue("Duplicate roots");
+  }
+  return std::move(Ret);
+}
diff --git a/llvm/lib/ProfileData/PGOCtxProfWriter.cpp b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
new file mode 100644
index 000000000000..508179756446
--- /dev/null
+++ b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
@@ -0,0 +1,49 @@
+//===- PGOCtxProfWriter.cpp - Contextual Instrumentation profile writer ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Write a contextual profile to bitstream.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Bitstream/BitCodeEnums.h"
+
+using namespace llvm;
+using namespace llvm::ctx_profile;
+
+void PGOCtxProfileWriter::writeCounters(const ContextNode &Node) {
+  Writer.EmitCode(bitc::UNABBREV_RECORD);
+  Writer.EmitVBR(PGOCtxProfileRecords::Counters, VBREncodingBits);
+  Writer.EmitVBR(Node.counters_size(), VBREncodingBits);
+  for (uint32_t I = 0U; I < Node.counters_size(); ++I)
+    Writer.EmitVBR64(Node.counters()[I], VBREncodingBits);
+}
+
+// recursively write all the subcontexts. We do need to traverse depth first to
+// model the context->subcontext implicitly, and since this captures call
+// stacks, we don't really need to be worried about stack overflow and we can
+// keep the implementation simple.
+void PGOCtxProfileWriter::writeImpl(std::optional<uint32_t> CallerIndex,
+                                    const ContextNode &Node) {
+  Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextNodeBlockID, CodeLen);
+  Writer.EmitRecord(PGOCtxProfileRecords::Guid,
+                    SmallVector<uint64_t, 1>{Node.guid()});
+  if (CallerIndex)
+    Writer.EmitRecord(PGOCtxProfileRecords::CalleeIndex,
+                      SmallVector<uint64_t, 1>{*CallerIndex});
+  writeCounters(Node);
+  for (uint32_t I = 0U; I < Node.callsites_size(); ++I)
+    for (const auto *Subcontext = Node.subContexts()[I]; Subcontext;
+         Subcontext = Subcontext->next())
+      writeImpl(I, *Subcontext);
+  Writer.ExitBlock();
+}
+
+void PGOCtxProfileWriter::write(const ContextNode &RootNode) {
+  writeImpl(std::nullopt, RootNode);
+}
diff --git a/llvm/unittests/ProfileData/CMakeLists.txt b/llvm/unittests/ProfileData/CMakeLists.txt
index ce3a0a45ccf1..c92642ded828 100644
--- a/llvm/unittests/ProfileData/CMakeLists.txt
+++ b/llvm/unittests/ProfileData/CMakeLists.txt
@@ -13,6 +13,7 @@ add_llvm_unittest(ProfileDataTests
   InstrProfTest.cpp
   ItaniumManglingCanonicalizerTest.cpp
   MemProfTest.cpp
+  PGOCtxProfReaderWriterTest.cpp
   SampleProfTest.cpp
   SymbolRemappingReaderTest.cpp
   )
diff --git a/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp b/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp
new file mode 100644
index 000000000000..d2cdbb28e2fc
--- /dev/null
+++ b/llvm/unittests/ProfileData/PGOCtxProfReaderWriterTest.cpp
@@ -0,0 +1,255 @@
+//===-------------- PGOCtxProfReadWriteTest.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Bitstream/BitstreamReader.h"
+#include "llvm/ProfileData/CtxInstrContextNode.h"
+#include "llvm/ProfileData/PGOCtxProfReader.h"
+#include "llvm/ProfileData/PGOCtxProfWriter.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Testing/Support/SupportHelpers.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::ctx_profile;
+
+class PGOCtxProfRWTest : public ::testing::Test {
+  std::vector<std::unique_ptr<char[]>> Nodes;
+  std::map<GUID, const ContextNode *> Roots;
+
+public:
+  ContextNode *createNode(GUID Guid, uint32_t NrCounters, uint32_t NrCallsites,
+                          ContextNode *Next = nullptr) {
+    auto AllocSize = ContextNode::getAllocSize(NrCounters, NrCallsites);
+    auto *Mem = Nodes.emplace_back(std::make_unique<char[]>(AllocSize)).get();
+    std::memset(Mem, 0, AllocSize);
+    auto *Ret = new (Mem) ContextNode(Guid, NrCounters, NrCallsites, Next);
+    return Ret;
+  }
+
+  void SetUp() override {
+    // Root (guid 1) has 2 callsites, one used for an indirect call to either
+    // guid 2 or 4.
+    // guid 2 calls guid 5
+    // guid 5 calls guid 2
+    // there's also a second root, guid3.
+    auto *Root1 = createNode(1, 2, 2);
+    Root1->counters()[0] = 10;
+    Root1->counters()[1] = 11;
+    Roots.insert({1, Root1});
+    auto *L1 = createNode(2, 1, 1);
+    L1->counters()[0] = 12;
+    Root1->subContexts()[1] = createNode(4, 3, 1, L1);
+    Root1->subContexts()[1]->counters()[0] = 13;
+    Root1->subContexts()[1]->counters()[1] = 14;
+    Root1->subContexts()[1]->counters()[2] = 15;
+
+    auto *L3 = createNode(5, 6, 3);
+    for (auto I = 0; I < 6; ++I)
+      L3->counters()[I] = 16 + I;
+    L1->subContexts()[0] = L3;
+    L3->subContexts()[2] = createNode(2, 1, 1);
+    L3->subContexts()[2]->counters()[0] = 30;
+    auto *Root2 = createNode(3, 1, 0);
+    Root2->counters()[0] = 40;
+    Roots.insert({3, Root2});
+  }
+
+  const std::map<GUID, const ContextNode *> &roots() const { return Roots; }
+};
+
+void checkSame(const ContextNode &Raw, const PGOContextualProfile &Profile) {
+  EXPECT_EQ(Raw.guid(), Profile.guid());
+  ASSERT_EQ(Raw.counters_size(), Profile.counters().size());
+  for (auto I = 0U; I < Raw.counters_size(); ++I)
+    EXPECT_EQ(Raw.counters()[I], Profile.counters()[I]);
+
+  for (auto I = 0U; I < Raw.callsites_size(); ++I) {
+    if (Raw.subContexts()[I] == nullptr)
+      continue;
+    EXPECT_TRUE(Profile.hasCallsite(I));
+    const auto &ProfileTargets = Profile.callsite(I);
+
+    std::map<GUID, const ContextNode *> Targets;
+    for (const auto *N = Raw.subContexts()[I]; N; N = N->next())
+      EXPECT_TRUE(Targets.insert({N->guid(), N}).second);
+
+    EXPECT_EQ(Targets.size(), ProfileTargets.size());
+    for (auto It : Targets) {
+      auto PIt = ProfileTargets.find(It.second->guid());
+      EXPECT_NE(PIt, ProfileTargets.end());
+      checkSame(*It.second, PIt->second);
+    }
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, RoundTrip) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      for (auto &[_, R] : roots())
+        Writer.write(*R);
+    }
+  }
+  {
+    ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
+        MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    ASSERT_TRUE(!!Expected);
+    auto &Ctxes = *Expected;
+    EXPECT_EQ(Ctxes.size(), roots().size());
+    EXPECT_EQ(Ctxes.size(), 2U);
+    for (auto &[G, R] : roots())
+      checkSame(*R, Ctxes.find(G)->second);
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, InvalidCounters) {
+  auto *R = createNode(1, 0, 1);
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      Writer.write(*R);
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, Empty) {
+  BitstreamCursor Cursor("");
+  PGOCtxProfileReader Reader(Cursor);
+  auto Expected = Reader.loadContexts();
+  EXPECT_FALSE(Expected);
+  consumeError(Expected.takeError());
+}
+
+TEST_F(PGOCtxProfRWTest, Invalid) {
+  BitstreamCursor Cursor("Surely this is not valid");
+  PGOCtxProfileReader Reader(Cursor);
+  auto Expected = Reader.loadContexts();
+  EXPECT_FALSE(Expected);
+  consumeError(Expected.takeError());
+}
+
+TEST_F(PGOCtxProfRWTest, ValidButEmpty) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      // don't write anything - this will just produce the metadata subblock.
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_TRUE(!!Expected);
+    EXPECT_TRUE(Expected->empty());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, WrongVersion) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out, PGOCtxProfileWriter::CurrentVersion + 1);
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, DuplicateRoots) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      PGOCtxProfileWriter Writer(Out);
+      Writer.write(*createNode(1, 1, 1));
+      Writer.write(*createNode(1, 1, 1));
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
+
+TEST_F(PGOCtxProfRWTest, DuplicateTargets) {
+  llvm::unittest::TempFile ProfileFile("ctx_profile", "", "", /*Unique*/ true);
+  {
+    std::error_code EC;
+    raw_fd_stream Out(ProfileFile.path(), EC);
+    ASSERT_FALSE(EC);
+    {
+      auto *R = createNode(1, 1, 1);
+      auto *L1 = createNode(2, 1, 0);
+      auto *L2 = createNode(2, 1, 0, L1);
+      R->subContexts()[0] = L2;
+      PGOCtxProfileWriter Writer(Out);
+      Writer.write(*R);
+    }
+  }
+  {
+    auto MB = MemoryBuffer::getFile(ProfileFile.path());
+    ASSERT_TRUE(!!MB);
+    ASSERT_NE(*MB, nullptr);
+    BitstreamCursor Cursor((*MB)->getBuffer());
+    PGOCtxProfileReader Reader(Cursor);
+    auto Expected = Reader.loadContexts();
+    EXPECT_FALSE(Expected);
+    consumeError(Expected.takeError());
+  }
+}
-- 
GitLab


From 2fb92520cba15afff6f25a1f0b959ef39912fa0a Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Wed, 15 May 2024 19:54:54 +0000
Subject: [PATCH 0909/1206] [gn build] Port fc8775e2142c

---
 llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn       | 2 ++
 llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn | 1 +
 2 files changed, 3 insertions(+)

diff --git a/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
index 9dbfe0f94c1d..c6fa142b3766 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
@@ -17,6 +17,8 @@ static_library("ProfileData") {
     "ItaniumManglingCanonicalizer.cpp",
     "MemProf.cpp",
     "MemProfReader.cpp",
+    "PGOCtxProfReader.cpp",
+    "PGOCtxProfWriter.cpp",
     "ProfileSummaryBuilder.cpp",
     "SampleProf.cpp",
     "SampleProfReader.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
index 4919a8089209..f45542519173 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
@@ -14,6 +14,7 @@ unittest("ProfileDataTests") {
     "InstrProfTest.cpp",
     "ItaniumManglingCanonicalizerTest.cpp",
     "MemProfTest.cpp",
+    "PGOCtxProfReaderWriterTest.cpp",
     "SampleProfTest.cpp",
     "SymbolRemappingReaderTest.cpp",
   ]
-- 
GitLab


From 24c39261e62d9f99bab91edf67bb9607a681b038 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Wed, 15 May 2024 21:04:20 +0100
Subject: [PATCH 0910/1206] [RISCV][test] Add tests for parsing profiles using
 RISCVISAInfo::parseArchString

---
 .../TargetParser/RISCVISAInfoTest.cpp         | 63 ++++++++++++++++++-
 1 file changed, 61 insertions(+), 2 deletions(-)

diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 7f2d1eb8c017..d04f21fa2006 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -21,8 +21,8 @@ bool operator==(const RISCVISAUtils::ExtensionVersion &A,
 }
 
 TEST(ParseNormalizedArchString, RejectsInvalidChars) {
-  for (StringRef Input :
-       {"RV32", "rV64", "rv32i2P0", "rv64i2p0_A2p0", "rv32e2.0"}) {
+  for (StringRef Input : {"RV32", "rV64", "rv32i2P0", "rv64i2p0_A2p0",
+                          "rv32e2.0", "rva20u64+zbc"}) {
     EXPECT_EQ(
         toString(RISCVISAInfo::parseNormalizedArchString(Input).takeError()),
         "string may only contain [a-z0-9_]");
@@ -667,6 +667,65 @@ TEST(ParseArchString, RejectsConflictingExtensions) {
   }
 }
 
+TEST(ParseArchString, RejectsUnrecognizedProfileNames) {
+  for (StringRef Input : {"rvi23u99", "rvz23u64", "rva99u32"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "string must begin with rv32{i,e,g}, rv64{i,e,g}, or a supported "
+              "profile name");
+  }
+}
+
+TEST(ParseArchString, RejectsProfilesWithUnseparatedExtraExtensions) {
+  for (StringRef Input : {"rvi20u32m", "rvi20u64c"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "additional extensions must be after separator '_'");
+  }
+}
+
+TEST(ParseArchString, AcceptsBareProfileNames) {
+  auto MaybeRVA20U64 = RISCVISAInfo::parseArchString("rva20u64", true);
+  ASSERT_THAT_EXPECTED(MaybeRVA20U64, Succeeded());
+  const auto &Exts = (*MaybeRVA20U64)->getExtensions();
+  EXPECT_EQ(Exts.size(), 13UL);
+  EXPECT_EQ(Exts.count("i"), 1U);
+  EXPECT_EQ(Exts.count("m"), 1U);
+  EXPECT_EQ(Exts.count("f"), 1U);
+  EXPECT_EQ(Exts.count("a"), 1U);
+  EXPECT_EQ(Exts.count("d"), 1U);
+  EXPECT_EQ(Exts.count("c"), 1U);
+  EXPECT_EQ(Exts.count("za128rs"), 1U);
+  EXPECT_EQ(Exts.count("zicntr"), 1U);
+  EXPECT_EQ(Exts.count("ziccif"), 1U);
+  EXPECT_EQ(Exts.count("zicsr"), 1U);
+  EXPECT_EQ(Exts.count("ziccrse"), 1U);
+  EXPECT_EQ(Exts.count("ziccamoa"), 1U);
+  EXPECT_EQ(Exts.count("zicclsm"), 1U);
+
+  auto MaybeRVA23U64 = RISCVISAInfo::parseArchString("rva23u64", true);
+  ASSERT_THAT_EXPECTED(MaybeRVA23U64, Succeeded());
+  EXPECT_GT((*MaybeRVA23U64)->getExtensions().size(), 13UL);
+}
+
+TEST(ParseArchSTring, AcceptsProfileNamesWithSeparatedAdditionalExtensions) {
+  auto MaybeRVI20U64 = RISCVISAInfo::parseArchString("rvi20u64_m_zba", true);
+  ASSERT_THAT_EXPECTED(MaybeRVI20U64, Succeeded());
+  const auto &Exts = (*MaybeRVI20U64)->getExtensions();
+  EXPECT_EQ(Exts.size(), 3UL);
+  EXPECT_EQ(Exts.count("i"), 1U);
+  EXPECT_EQ(Exts.count("m"), 1U);
+  EXPECT_EQ(Exts.count("zba"), 1U);
+}
+
+TEST(ParseArchString,
+     RejectsProfilesWithAdditionalExtensionsGivenAlreadyInProfile) {
+  // This test was added to document the current behaviour. Discussion isn't
+  // believed to have taken place about if this is desirable or not.
+  EXPECT_EQ(
+      toString(
+          RISCVISAInfo::parseArchString("rva20u64_zicntr", true).takeError()),
+      "duplicated standard user-level extension 'zicntr'");
+}
+
 TEST(ToFeatures, IIsDroppedAndExperimentalExtensionsArePrefixed) {
   auto MaybeISAInfo1 =
       RISCVISAInfo::parseArchString("rv64im_ztso", true, false);
-- 
GitLab


From 891d687137ad9bb3b4efae116f9539addb5be0ea Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Wed, 15 May 2024 21:09:43 +0100
Subject: [PATCH 0911/1206] [RISCV] Gate unratified profiles behind
 -menable-experimental-extensions (#92167)

As discussed in the last sync-up call, because these profiles are not
yet finalised they shouldn't be exposed to users unless they opt-in to
them (much like experimental extensions). We may later want to add a
more specific flag, but reusing `-menable-experimental-extensions`
solves the immediate problem.

This is implemented using the new support for marking profiles s
experimental added in #91993 to move the unratified profiles to
RISCVExperimentalProfile and making the necessary changes to logic in
RISCVISAInfo to handle this.
---
 clang/test/Driver/riscv-profiles.c            | 10 +++++--
 llvm/lib/Target/RISCV/RISCVProfiles.td        | 14 +++++----
 llvm/lib/TargetParser/RISCVISAInfo.cpp        | 29 +++++++++++++++----
 llvm/test/CodeGen/RISCV/attributes.ll         | 10 +++----
 .../TargetParser/RISCVISAInfoTest.cpp         | 13 +++++++--
 5 files changed, 55 insertions(+), 21 deletions(-)

diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c
index 298f301de3fe..55aa5b398cee 100644
--- a/clang/test/Driver/riscv-profiles.c
+++ b/clang/test/Driver/riscv-profiles.c
@@ -111,7 +111,7 @@
 // RVA22S64: "-target-feature" "+svinval"
 // RVA22S64: "-target-feature" "+svpbmt"
 
-// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva23u64 \
+// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva23u64 -menable-experimental-extensions \
 // RUN:   | FileCheck -check-prefix=RVA23U64 %s
 // RVA23U64: "-target-feature" "+m"
 // RVA23U64: "-target-feature" "+a"
@@ -207,7 +207,7 @@
 // RVA23S64: "-target-feature" "+svnapot"
 // RVA23S64: "-target-feature" "+svpbmt"
 
-// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23u64 \
+// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23u64 -menable-experimental-extensions \
 // RUN:   | FileCheck -check-prefix=RVB23U64 %s
 // RVB23U64: "-target-feature" "+m"
 // RVB23U64: "-target-feature" "+a"
@@ -284,7 +284,7 @@
 // RVB23S64: "-target-feature" "+svnapot"
 // RVB23S64: "-target-feature" "+svpbmt"
 
-// RUN: %clang --target=riscv32 -### -c %s 2>&1 -march=rvm23u32 \
+// RUN: %clang --target=riscv32 -### -c %s 2>&1 -march=rvm23u32 -menable-experimental-extensions \
 // RUN:   | FileCheck -check-prefix=RVM23U32 %s
 // RVM23U32: "-target-feature" "+m"
 // RVM23U32: "-target-feature" "+zicbop"
@@ -322,3 +322,7 @@
 
 // RUN: not %clang --target=riscv64 -### -c %s 2>&1 -march=rva22u64zfa | FileCheck -check-prefix=INVALID-ADDITIONAL %s
 // INVALID-ADDITIONAL: error: invalid arch name 'rva22u64zfa', additional extensions must be after separator '_'
+
+// RUN: not %clang --target=riscv64 -### -c %s 2>&1 -march=rva23u64 | FileCheck -check-prefix=EXPERIMENTAL-NOFLAG %s
+// EXPERIMENTAL-NOFLAG: error: invalid arch name 'rva23u64'
+// EXPERIMENTAL-NOFLAG: requires '-menable-experimental-extensions' for profile 'rva23u64'
diff --git a/llvm/lib/Target/RISCV/RISCVProfiles.td b/llvm/lib/Target/RISCV/RISCVProfiles.td
index e56df33bd8cb..c4a64681f5f1 100644
--- a/llvm/lib/Target/RISCV/RISCVProfiles.td
+++ b/llvm/lib/Target/RISCV/RISCVProfiles.td
@@ -13,6 +13,10 @@ class RISCVProfile<string name, list<SubtargetFeature> features>
   // experimental.
   bit Experimental = false;
 }
+class RISCVExperimentalProfile<string name, list<SubtargetFeature> features>
+    : RISCVProfile<"experimental-"#name, features> {
+  let Experimental = true;
+}
 
 defvar RVI20U32Features = [Feature32Bit, FeatureStdExtI];
 defvar RVI20U64Features = [Feature64Bit, FeatureStdExtI];
@@ -201,8 +205,8 @@ def RVA20U64 : RISCVProfile<"rva20u64", RVA20U64Features>;
 def RVA20S64 : RISCVProfile<"rva20s64", RVA20S64Features>;
 def RVA22U64 : RISCVProfile<"rva22u64", RVA22U64Features>;
 def RVA22S64 : RISCVProfile<"rva22s64", RVA22S64Features>;
-def RVA23U64 : RISCVProfile<"rva23u64", RVA23U64Features>;
-def RVA23S64 : RISCVProfile<"rva23s64", RVA23S64Features>;
-def RVB23U64 : RISCVProfile<"rvb23u64", RVB23U64Features>;
-def RVB23S64 : RISCVProfile<"rvb23s64", RVB23S64Features>;
-def RVM23U32 : RISCVProfile<"rvm23u32", RVM23U32Features>;
+def RVA23U64 : RISCVExperimentalProfile<"rva23u64", RVA23U64Features>;
+def RVA23S64 : RISCVExperimentalProfile<"rva23s64", RVA23S64Features>;
+def RVB23U64 : RISCVExperimentalProfile<"rvb23u64", RVB23U64Features>;
+def RVB23S64 : RISCVExperimentalProfile<"rvb23s64", RVB23S64Features>;
+def RVM23U32 : RISCVExperimentalProfile<"rvm23u32", RVM23U32Features>;
diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 575c9dbad515..706b2853cd2c 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -102,6 +102,10 @@ void llvm::riscvExtensionsHelp(StringMap<StringRef> DescMap) {
   for (const auto &P : SupportedProfiles)
     outs().indent(4) << P.Name << "\n";
 
+  outs() << "\nExperimental Profiles\n";
+  for (const auto &P : SupportedExperimentalProfiles)
+    outs().indent(4) << P.Name << "\n";
+
   outs() << "\nUse -march to specify the target's extension.\n"
             "For example, clang -march=rv32i_v1p0\n";
 }
@@ -608,12 +612,25 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
     XLen = 64;
   } else {
     // Try parsing as a profile.
-    auto I = llvm::upper_bound(SupportedProfiles, Arch,
-                               [](StringRef Arch, const RISCVProfile &Profile) {
-                                 return Arch < Profile.Name;
-                               });
-
-    if (I != std::begin(SupportedProfiles) && Arch.starts_with((--I)->Name)) {
+    auto ProfileCmp = [](StringRef Arch, const RISCVProfile &Profile) {
+      return Arch < Profile.Name;
+    };
+    auto I = llvm::upper_bound(SupportedProfiles, Arch, ProfileCmp);
+    bool FoundProfile = I != std::begin(SupportedProfiles) &&
+                        Arch.starts_with(std::prev(I)->Name);
+    if (!FoundProfile) {
+      I = llvm::upper_bound(SupportedExperimentalProfiles, Arch, ProfileCmp);
+      FoundProfile = (I != std::begin(SupportedExperimentalProfiles) &&
+                      Arch.starts_with(std::prev(I)->Name));
+      if (FoundProfile && !EnableExperimentalExtension) {
+        return createStringError(errc::invalid_argument,
+                                 "requires '-menable-experimental-extensions' "
+                                 "for profile '" +
+                                     std::prev(I)->Name + "'");
+      }
+    }
+    if (FoundProfile) {
+      --I;
       std::string NewArch = I->MArch.str();
       StringRef ArchWithoutProfile = Arch.drop_front(I->Name.size());
       if (!ArchWithoutProfile.empty()) {
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 8f49f6648ad2..953ed5ee3795 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -265,11 +265,11 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+rva20s64 %s -o - | FileCheck --check-prefix=RVA20S64 %s
 ; RUN: llc -mtriple=riscv64 -mattr=+rva22u64 %s -o - | FileCheck --check-prefix=RVA22U64 %s
 ; RUN: llc -mtriple=riscv64 -mattr=+rva22s64 %s -o - | FileCheck --check-prefix=RVA22S64 %s
-; RUN: llc -mtriple=riscv64 -mattr=+rva23u64 %s -o - | FileCheck --check-prefix=RVA23U64 %s
-; RUN: llc -mtriple=riscv64 -mattr=+rva23s64 %s -o - | FileCheck --check-prefix=RVA23S64 %s
-; RUN: llc -mtriple=riscv64 -mattr=+rvb23u64 %s -o - | FileCheck --check-prefix=RVB23U64 %s
-; RUN: llc -mtriple=riscv64 -mattr=+rvb23s64 %s -o - | FileCheck --check-prefix=RVB23S64 %s
-; RUN: llc -mtriple=riscv32 -mattr=+rvm23u32 %s -o - | FileCheck --check-prefix=RVM23U32 %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-rva23u64 %s -o - | FileCheck --check-prefix=RVA23U64 %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-rva23s64 %s -o - | FileCheck --check-prefix=RVA23S64 %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-rvb23u64 %s -o - | FileCheck --check-prefix=RVB23U64 %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-rvb23s64 %s -o - | FileCheck --check-prefix=RVB23S64 %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-rvm23u32 %s -o - | FileCheck --check-prefix=RVM23U32 %s
 
 ; CHECK: .attribute 4, 16
 
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index d04f21fa2006..22fe31809319 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -726,6 +726,13 @@ TEST(ParseArchString,
       "duplicated standard user-level extension 'zicntr'");
 }
 
+TEST(ParseArchString,
+     RejectsExperimentalProfilesIfEnableExperimentalExtensionsNotSet) {
+  EXPECT_EQ(
+      toString(RISCVISAInfo::parseArchString("rva23u64", false).takeError()),
+      "requires '-menable-experimental-extensions' for profile 'rva23u64'");
+}
+
 TEST(ToFeatures, IIsDroppedAndExperimentalExtensionsArePrefixed) {
   auto MaybeISAInfo1 =
       RISCVISAInfo::parseArchString("rv64im_ztso", true, false);
@@ -1073,12 +1080,14 @@ Supported Profiles
     rva20u64
     rva22s64
     rva22u64
+    rvi20u32
+    rvi20u64
+
+Experimental Profiles
     rva23s64
     rva23u64
     rvb23s64
     rvb23u64
-    rvi20u32
-    rvi20u64
     rvm23u32
 
 Use -march to specify the target's extension.
-- 
GitLab


From 80d9ae9cbf692a73404995a88665af7166c7e8ad Mon Sep 17 00:00:00 2001
From: Samira Bazuzi <bazuzi@google.com>
Date: Wed, 15 May 2024 16:11:11 -0400
Subject: [PATCH 0912/1206] [clang][dataflow] Fully support Environment
 construction for Stmt analysis. (#91616)

Assume in fewer places that the analysis is of a `FunctionDecl`, and
initialize the `Environment` properly for `Stmt`s.

Moves constructors for `Environment` to header to make it more obvious
that there are only minor differences between them and very little
initialization in the constructors.

Tested with check-clang-tooling.
---
 .../FlowSensitive/DataflowEnvironment.h       | 118 +++++++++++-------
 .../FlowSensitive/DataflowEnvironment.cpp     | 107 ++++++++--------
 .../TypeErasedDataflowAnalysis.cpp            |   2 +-
 .../FlowSensitive/DataflowEnvironmentTest.cpp |  33 +++++
 .../Analysis/FlowSensitive/TestingSupport.h   |   4 +-
 .../TypeErasedDataflowAnalysisTest.cpp        |  32 +++++
 6 files changed, 198 insertions(+), 98 deletions(-)

diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
index cdf89c7def2c..097ff2bdfe7a 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
@@ -19,6 +19,7 @@
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/Type.h"
+#include "clang/Analysis/FlowSensitive/ASTOps.h"
 #include "clang/Analysis/FlowSensitive/DataflowAnalysisContext.h"
 #include "clang/Analysis/FlowSensitive/DataflowLattice.h"
 #include "clang/Analysis/FlowSensitive/Formula.h"
@@ -30,9 +31,11 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
+#include <cassert>
 #include <memory>
 #include <type_traits>
 #include <utility>
+#include <vector>
 
 namespace clang {
 namespace dataflow {
@@ -155,7 +158,28 @@ public:
 
   /// Creates an environment that uses `DACtx` to store objects that encompass
   /// the state of a program.
-  explicit Environment(DataflowAnalysisContext &DACtx);
+  explicit Environment(DataflowAnalysisContext &DACtx)
+      : DACtx(&DACtx),
+        FlowConditionToken(DACtx.arena().makeFlowConditionToken()) {}
+
+  /// Creates an environment that uses `DACtx` to store objects that encompass
+  /// the state of a program, with `S` as the statement to analyze.
+  Environment(DataflowAnalysisContext &DACtx, Stmt &S) : Environment(DACtx) {
+    InitialTargetStmt = &S;
+  }
+
+  /// Creates an environment that uses `DACtx` to store objects that encompass
+  /// the state of a program, with `FD` as the function to analyze.
+  ///
+  /// Requirements:
+  ///
+  ///  The function must have a body, i.e.
+  ///  `FunctionDecl::doesThisDecalarationHaveABody()` must be true.
+  Environment(DataflowAnalysisContext &DACtx, const FunctionDecl &FD)
+      : Environment(DACtx, *FD.getBody()) {
+    assert(FD.doesThisDeclarationHaveABody());
+    InitialTargetFunc = &FD;
+  }
 
   // Copy-constructor is private, Environments should not be copied. See fork().
   Environment &operator=(const Environment &Other) = delete;
@@ -163,24 +187,11 @@ public:
   Environment(Environment &&Other) = default;
   Environment &operator=(Environment &&Other) = default;
 
-  /// Creates an environment that uses `DACtx` to store objects that encompass
-  /// the state of a program.
-  ///
-  /// If `DeclCtx` is a function, initializes the environment with symbolic
-  /// representations of the function parameters.
-  ///
-  /// If `DeclCtx` is a non-static member function, initializes the environment
-  /// with a symbolic representation of the `this` pointee.
-  Environment(DataflowAnalysisContext &DACtx, const DeclContext &DeclCtx);
-
   /// Assigns storage locations and values to all parameters, captures, global
-  /// variables, fields and functions referenced in the function currently being
-  /// analyzed.
-  ///
-  /// Requirements:
+  /// variables, fields and functions referenced in the `Stmt` or `FunctionDecl`
+  /// passed to the constructor.
   ///
-  ///  The function must have a body, i.e.
-  ///  `FunctionDecl::doesThisDecalarationHaveABody()` must be true.
+  /// If no `Stmt` or `FunctionDecl` was supplied, this function does nothing.
   void initialize();
 
   /// Returns a new environment that is a copy of this one.
@@ -193,7 +204,7 @@ public:
   /// forked flow condition references the original).
   Environment fork() const;
 
-  /// Creates and returns an environment to use for an inline analysis  of the
+  /// Creates and returns an environment to use for an inline analysis of the
   /// callee. Uses the storage location from each argument in the `Call` as the
   /// storage location for the corresponding parameter in the callee.
   ///
@@ -365,46 +376,51 @@ public:
   RecordStorageLocation &
   getResultObjectLocation(const Expr &RecordPRValue) const;
 
-  /// Returns the return value of the current function. This can be null if:
+  /// Returns the return value of the function currently being analyzed.
+  /// This can be null if:
   /// - The function has a void return type
   /// - No return value could be determined for the function, for example
   ///   because it calls a function without a body.
   ///
   /// Requirements:
-  ///  The current function must have a non-reference return type.
+  ///  The current analysis target must be a function and must have a
+  ///  non-reference return type.
   Value *getReturnValue() const {
     assert(getCurrentFunc() != nullptr &&
            !getCurrentFunc()->getReturnType()->isReferenceType());
     return ReturnVal;
   }
 
-  /// Returns the storage location for the reference returned by the current
-  /// function. This can be null if function doesn't return a single consistent
-  /// reference.
+  /// Returns the storage location for the reference returned by the function
+  /// currently being analyzed. This can be null if the function doesn't return
+  /// a single consistent reference.
   ///
   /// Requirements:
-  ///  The current function must have a reference return type.
+  ///  The current analysis target must be a function and must have a reference
+  ///  return type.
   StorageLocation *getReturnStorageLocation() const {
     assert(getCurrentFunc() != nullptr &&
            getCurrentFunc()->getReturnType()->isReferenceType());
     return ReturnLoc;
   }
 
-  /// Sets the return value of the current function.
+  /// Sets the return value of the function currently being analyzed.
   ///
   /// Requirements:
-  ///  The current function must have a non-reference return type.
+  ///  The current analysis target must be a function and must have a
+  ///  non-reference return type.
   void setReturnValue(Value *Val) {
     assert(getCurrentFunc() != nullptr &&
            !getCurrentFunc()->getReturnType()->isReferenceType());
     ReturnVal = Val;
   }
 
-  /// Sets the storage location for the reference returned by the current
-  /// function.
+  /// Sets the storage location for the reference returned by the function
+  /// currently being analyzed.
   ///
   /// Requirements:
-  ///  The current function must have a reference return type.
+  ///  The current analysis target must be a function and must have a reference
+  ///  return type.
   void setReturnStorageLocation(StorageLocation *Loc) {
     assert(getCurrentFunc() != nullptr &&
            getCurrentFunc()->getReturnType()->isReferenceType());
@@ -641,23 +657,21 @@ public:
   /// (or the flow condition is overly constraining) or if the solver times out.
   bool allows(const Formula &) const;
 
-  /// Returns the `DeclContext` of the block being analysed, if any. Otherwise,
-  /// returns null.
-  const DeclContext *getDeclCtx() const { return CallStack.back(); }
-
   /// Returns the function currently being analyzed, or null if the code being
   /// analyzed isn't part of a function.
   const FunctionDecl *getCurrentFunc() const {
-    return dyn_cast<FunctionDecl>(getDeclCtx());
+    return CallStack.empty() ? InitialTargetFunc : CallStack.back();
   }
 
-  /// Returns the size of the call stack.
+  /// Returns the size of the call stack, not counting the initial analysis
+  /// target.
   size_t callStackSize() const { return CallStack.size(); }
 
   /// Returns whether this `Environment` can be extended to analyze the given
-  /// `Callee` (i.e. if `pushCall` can be used), with recursion disallowed and a
-  /// given `MaxDepth`.
-  bool canDescend(unsigned MaxDepth, const DeclContext *Callee) const;
+  /// `Callee` (i.e. if `pushCall` can be used).
+  /// Recursion is not allowed. `MaxDepth` is the maximum size of the call stack
+  /// (i.e. the maximum value that `callStackSize()` may assume after the call).
+  bool canDescend(unsigned MaxDepth, const FunctionDecl *Callee) const;
 
   /// Returns the `DataflowAnalysisContext` used by the environment.
   DataflowAnalysisContext &getDataflowAnalysisContext() const { return *DACtx; }
@@ -719,8 +733,8 @@ private:
                         ArrayRef<const Expr *> Args);
 
   /// Assigns storage locations and values to all global variables, fields
-  /// and functions referenced in `FuncDecl`. `FuncDecl` must have a body.
-  void initFieldsGlobalsAndFuncs(const FunctionDecl *FuncDecl);
+  /// and functions in `Referenced`.
+  void initFieldsGlobalsAndFuncs(const ReferencedDecls &Referenced);
 
   static PrValueToResultObject
   buildResultObjectMap(DataflowAnalysisContext *DACtx,
@@ -728,6 +742,11 @@ private:
                        RecordStorageLocation *ThisPointeeLoc,
                        RecordStorageLocation *LocForRecordReturnVal);
 
+  static PrValueToResultObject
+  buildResultObjectMap(DataflowAnalysisContext *DACtx, Stmt *S,
+                       RecordStorageLocation *ThisPointeeLoc,
+                       RecordStorageLocation *LocForRecordReturnVal);
+
   // `DACtx` is not null and not owned by this object.
   DataflowAnalysisContext *DACtx;
 
@@ -736,11 +755,20 @@ private:
   // shared between environments in the same call.
   // https://github.com/llvm/llvm-project/issues/59005
 
-  // `DeclContext` of the block being analysed if provided.
-  std::vector<const DeclContext *> CallStack;
+  // The stack of functions called from the initial analysis target.
+  std::vector<const FunctionDecl *> CallStack;
+
+  // Initial function to analyze, if a function was passed to the constructor.
+  // Null otherwise.
+  const FunctionDecl *InitialTargetFunc = nullptr;
+  // Top-level statement of the initial analysis target.
+  // If a function was passed to the constructor, this is its body.
+  // If a statement was passed to the constructor, this is that statement.
+  // Null if no analysis target was passed to the constructor.
+  Stmt *InitialTargetStmt = nullptr;
 
   // Maps from prvalues of record type to their result objects. Shared between
-  // all environments for the same function.
+  // all environments for the same analysis target.
   // FIXME: It's somewhat unsatisfactory that we have to use a `shared_ptr`
   // here, though the cost is acceptable: The overhead of a `shared_ptr` is
   // incurred when it is copied, and this happens only relatively rarely (when
@@ -749,7 +777,7 @@ private:
   std::shared_ptr<PrValueToResultObject> ResultObjectMap;
 
   // The following three member variables handle various different types of
-  // return values.
+  // return values when the current analysis target is a function.
   // - If the return type is not a reference and not a record: Value returned
   //   by the function.
   Value *ReturnVal = nullptr;
@@ -762,7 +790,7 @@ private:
   RecordStorageLocation *LocForRecordReturnVal = nullptr;
 
   // The storage location of the `this` pointee. Should only be null if the
-  // function being analyzed is only a function and not a method.
+  // analysis target is not a method.
   RecordStorageLocation *ThisPointeeLoc = nullptr;
 
   // Maps from declarations and glvalue expression to storage locations that are
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
index cb6c8b2ef107..338a85525b38 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
@@ -16,17 +16,22 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/AST/Stmt.h"
 #include "clang/AST/Type.h"
 #include "clang/Analysis/FlowSensitive/ASTOps.h"
+#include "clang/Analysis/FlowSensitive/DataflowAnalysisContext.h"
 #include "clang/Analysis/FlowSensitive/DataflowLattice.h"
 #include "clang/Analysis/FlowSensitive/Value.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/Support/ErrorHandling.h"
+#include <algorithm>
 #include <cassert>
+#include <memory>
 #include <utility>
 
 #define DEBUG_TYPE "dataflow"
@@ -290,15 +295,14 @@ widenKeyToValueMap(const llvm::MapVector<Key, Value *> &CurMap,
 namespace {
 
 // Visitor that builds a map from record prvalues to result objects.
-// This traverses the body of the function to be analyzed; for each result
-// object that it encounters, it propagates the storage location of the result
-// object to all record prvalues that can initialize it.
+// For each result object that it encounters, it propagates the storage location
+// of the result object to all record prvalues that can initialize it.
 class ResultObjectVisitor : public RecursiveASTVisitor<ResultObjectVisitor> {
 public:
   // `ResultObjectMap` will be filled with a map from record prvalues to result
-  // object. If the function being analyzed returns a record by value,
-  // `LocForRecordReturnVal` is the location to which this record should be
-  // written; otherwise, it is null.
+  // object. If this visitor will traverse a function that returns a record by
+  // value, `LocForRecordReturnVal` is the location to which this record should
+  // be written; otherwise, it is null.
   explicit ResultObjectVisitor(
       llvm::DenseMap<const Expr *, RecordStorageLocation *> &ResultObjectMap,
       RecordStorageLocation *LocForRecordReturnVal,
@@ -514,39 +518,31 @@ private:
 
 } // namespace
 
-Environment::Environment(DataflowAnalysisContext &DACtx)
-    : DACtx(&DACtx),
-      FlowConditionToken(DACtx.arena().makeFlowConditionToken()) {}
-
-Environment::Environment(DataflowAnalysisContext &DACtx,
-                         const DeclContext &DeclCtx)
-    : Environment(DACtx) {
-  CallStack.push_back(&DeclCtx);
-}
-
 void Environment::initialize() {
-  const DeclContext *DeclCtx = getDeclCtx();
-  if (DeclCtx == nullptr)
+  if (InitialTargetStmt == nullptr)
     return;
 
-  const auto *FuncDecl = dyn_cast<FunctionDecl>(DeclCtx);
-  if (FuncDecl == nullptr)
+  if (InitialTargetFunc == nullptr) {
+    initFieldsGlobalsAndFuncs(getReferencedDecls(*InitialTargetStmt));
+    ResultObjectMap =
+        std::make_shared<PrValueToResultObject>(buildResultObjectMap(
+            DACtx, InitialTargetStmt, getThisPointeeStorageLocation(),
+            /*LocForRecordReturnValue=*/nullptr));
     return;
+  }
 
-  assert(FuncDecl->doesThisDeclarationHaveABody());
-
-  initFieldsGlobalsAndFuncs(FuncDecl);
+  initFieldsGlobalsAndFuncs(getReferencedDecls(*InitialTargetFunc));
 
-  for (const auto *ParamDecl : FuncDecl->parameters()) {
+  for (const auto *ParamDecl : InitialTargetFunc->parameters()) {
     assert(ParamDecl != nullptr);
     setStorageLocation(*ParamDecl, createObject(*ParamDecl, nullptr));
   }
 
-  if (FuncDecl->getReturnType()->isRecordType())
+  if (InitialTargetFunc->getReturnType()->isRecordType())
     LocForRecordReturnVal = &cast<RecordStorageLocation>(
-        createStorageLocation(FuncDecl->getReturnType()));
+        createStorageLocation(InitialTargetFunc->getReturnType()));
 
-  if (const auto *MethodDecl = dyn_cast<CXXMethodDecl>(DeclCtx)) {
+  if (const auto *MethodDecl = dyn_cast<CXXMethodDecl>(InitialTargetFunc)) {
     auto *Parent = MethodDecl->getParent();
     assert(Parent != nullptr);
 
@@ -558,7 +554,7 @@ void Environment::initialize() {
           setStorageLocation(*VarDecl, createObject(*VarDecl, nullptr));
         } else if (Capture.capturesThis()) {
           const auto *SurroundingMethodDecl =
-              cast<CXXMethodDecl>(DeclCtx->getNonClosureAncestor());
+              cast<CXXMethodDecl>(InitialTargetFunc->getNonClosureAncestor());
           QualType ThisPointeeType =
               SurroundingMethodDecl->getFunctionObjectParameterType();
           setThisPointeeStorageLocation(
@@ -580,18 +576,16 @@ void Environment::initialize() {
 
   // We do this below the handling of `CXXMethodDecl` above so that we can
   // be sure that the storage location for `this` has been set.
-  ResultObjectMap = std::make_shared<PrValueToResultObject>(
-      buildResultObjectMap(DACtx, FuncDecl, getThisPointeeStorageLocation(),
-                           LocForRecordReturnVal));
+  ResultObjectMap =
+      std::make_shared<PrValueToResultObject>(buildResultObjectMap(
+          DACtx, InitialTargetFunc, getThisPointeeStorageLocation(),
+          LocForRecordReturnVal));
 }
 
-// FIXME: Add support for resetting globals after function calls to enable
-// the implementation of sound analyses.
-void Environment::initFieldsGlobalsAndFuncs(const FunctionDecl *FuncDecl) {
-  assert(FuncDecl->doesThisDeclarationHaveABody());
-
-  ReferencedDecls Referenced = getReferencedDecls(*FuncDecl);
+// FIXME: Add support for resetting globals after function calls to enable the
+// implementation of sound analyses.
 
+void Environment::initFieldsGlobalsAndFuncs(const ReferencedDecls &Referenced) {
   // These have to be added before the lines that follow to ensure that
   // `create*` work correctly for structs.
   DACtx->addModeledFields(Referenced.Fields);
@@ -602,9 +596,9 @@ void Environment::initFieldsGlobalsAndFuncs(const FunctionDecl *FuncDecl) {
 
     // We don't run transfer functions on the initializers of global variables,
     // so they won't be associated with a value or storage location. We
-    // therefore intentionally don't pass an initializer to `createObject()`;
-    // in particular, this ensures that `createObject()` will initialize the
-    // fields of record-type variables with values.
+    // therefore intentionally don't pass an initializer to `createObject()`; in
+    // particular, this ensures that `createObject()` will initialize the fields
+    // of record-type variables with values.
     setStorageLocation(*D, createObject(*D, nullptr));
   }
 
@@ -623,8 +617,8 @@ Environment Environment::fork() const {
 }
 
 bool Environment::canDescend(unsigned MaxDepth,
-                             const DeclContext *Callee) const {
-  return CallStack.size() <= MaxDepth && !llvm::is_contained(CallStack, Callee);
+                             const FunctionDecl *Callee) const {
+  return CallStack.size() < MaxDepth && !llvm::is_contained(CallStack, Callee);
 }
 
 Environment Environment::pushCall(const CallExpr *Call) const {
@@ -671,7 +665,7 @@ void Environment::pushCallInternal(const FunctionDecl *FuncDecl,
 
   CallStack.push_back(FuncDecl);
 
-  initFieldsGlobalsAndFuncs(FuncDecl);
+  initFieldsGlobalsAndFuncs(getReferencedDecls(*FuncDecl));
 
   const auto *ParamIt = FuncDecl->param_begin();
 
@@ -755,6 +749,8 @@ LatticeEffect Environment::widen(const Environment &PrevEnv,
   assert(ThisPointeeLoc == PrevEnv.ThisPointeeLoc);
   assert(CallStack == PrevEnv.CallStack);
   assert(ResultObjectMap == PrevEnv.ResultObjectMap);
+  assert(InitialTargetFunc == PrevEnv.InitialTargetFunc);
+  assert(InitialTargetStmt == PrevEnv.InitialTargetStmt);
 
   auto Effect = LatticeEffect::Unchanged;
 
@@ -790,6 +786,8 @@ Environment Environment::join(const Environment &EnvA, const Environment &EnvB,
   assert(EnvA.ThisPointeeLoc == EnvB.ThisPointeeLoc);
   assert(EnvA.CallStack == EnvB.CallStack);
   assert(EnvA.ResultObjectMap == EnvB.ResultObjectMap);
+  assert(EnvA.InitialTargetFunc == EnvB.InitialTargetFunc);
+  assert(EnvA.InitialTargetStmt == EnvB.InitialTargetStmt);
 
   Environment JoinedEnv(*EnvA.DACtx);
 
@@ -797,14 +795,13 @@ Environment Environment::join(const Environment &EnvA, const Environment &EnvB,
   JoinedEnv.ResultObjectMap = EnvA.ResultObjectMap;
   JoinedEnv.LocForRecordReturnVal = EnvA.LocForRecordReturnVal;
   JoinedEnv.ThisPointeeLoc = EnvA.ThisPointeeLoc;
+  JoinedEnv.InitialTargetFunc = EnvA.InitialTargetFunc;
+  JoinedEnv.InitialTargetStmt = EnvA.InitialTargetStmt;
 
-  if (EnvA.CallStack.empty()) {
+  const FunctionDecl *Func = EnvA.getCurrentFunc();
+  if (!Func) {
     JoinedEnv.ReturnVal = nullptr;
   } else {
-    // FIXME: Make `CallStack` a vector of `FunctionDecl` so we don't need this
-    // cast.
-    auto *Func = dyn_cast<FunctionDecl>(EnvA.CallStack.back());
-    assert(Func != nullptr);
     JoinedEnv.ReturnVal =
         joinValues(Func->getReturnType(), EnvA.ReturnVal, EnvA, EnvB.ReturnVal,
                    EnvB, JoinedEnv, Model);
@@ -1229,16 +1226,26 @@ Environment::PrValueToResultObject Environment::buildResultObjectMap(
     RecordStorageLocation *LocForRecordReturnVal) {
   assert(FuncDecl->doesThisDeclarationHaveABody());
 
-  PrValueToResultObject Map;
+  PrValueToResultObject Map = buildResultObjectMap(
+      DACtx, FuncDecl->getBody(), ThisPointeeLoc, LocForRecordReturnVal);
 
   ResultObjectVisitor Visitor(Map, LocForRecordReturnVal, *DACtx);
   if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(FuncDecl))
     Visitor.TraverseConstructorInits(Ctor, ThisPointeeLoc);
-  Visitor.TraverseStmt(FuncDecl->getBody());
 
   return Map;
 }
 
+Environment::PrValueToResultObject Environment::buildResultObjectMap(
+    DataflowAnalysisContext *DACtx, Stmt *S,
+    RecordStorageLocation *ThisPointeeLoc,
+    RecordStorageLocation *LocForRecordReturnVal) {
+  PrValueToResultObject Map;
+  ResultObjectVisitor Visitor(Map, LocForRecordReturnVal, *DACtx);
+  Visitor.TraverseStmt(S);
+  return Map;
+}
+
 RecordStorageLocation *getImplicitObjectLocation(const CXXMemberCallExpr &MCE,
                                                  const Environment &Env) {
   Expr *ImplicitObject = MCE.getImplicitObjectArgument();
diff --git a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
index 12eff4dd4b78..675b42550f17 100644
--- a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
+++ b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
@@ -476,7 +476,7 @@ runTypeErasedDataflowAnalysis(
   PrettyStackTraceAnalysis CrashInfo(ACFG, "runTypeErasedDataflowAnalysis");
 
   std::optional<Environment> MaybeStartingEnv;
-  if (InitEnv.callStackSize() == 1) {
+  if (InitEnv.callStackSize() == 0) {
     MaybeStartingEnv = InitEnv.fork();
     MaybeStartingEnv->initialize();
   }
diff --git a/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp b/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp
index 419564816124..bd710a00c47c 100644
--- a/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp
@@ -9,6 +9,8 @@
 #include "clang/Analysis/FlowSensitive/DataflowEnvironment.h"
 #include "TestingSupport.h"
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/Stmt.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Analysis/FlowSensitive/DataflowAnalysisContext.h"
@@ -403,4 +405,35 @@ TEST_F(EnvironmentTest,
               Contains(Member));
 }
 
+TEST_F(EnvironmentTest, Stmt) {
+  using namespace ast_matchers;
+
+  std::string Code = R"cc(
+      struct S { int i; };
+      void foo() {
+        S AnS = S{1};
+      }
+    )cc";
+  auto Unit =
+      tooling::buildASTFromCodeWithArgs(Code, {"-fsyntax-only", "-std=c++11"});
+  auto &Context = Unit->getASTContext();
+
+  ASSERT_EQ(Context.getDiagnostics().getClient()->getNumErrors(), 0U);
+
+  auto *DeclStatement = const_cast<DeclStmt *>(selectFirst<DeclStmt>(
+      "d", match(declStmt(hasSingleDecl(varDecl(hasName("AnS")))).bind("d"),
+                 Context)));
+  ASSERT_THAT(DeclStatement, NotNull());
+  auto *Init = (cast<VarDecl>(*DeclStatement->decl_begin()))->getInit();
+  ASSERT_THAT(Init, NotNull());
+
+  // Verify that we can retrieve the result object location for the initializer
+  // expression when we analyze the DeclStmt for `AnS`.
+  Environment Env(DAContext, *DeclStatement);
+  // Don't crash when initializing.
+  Env.initialize();
+  // And don't crash when retrieving the result object location.
+  Env.getResultObjectLocation(*Init);
+}
+
 } // namespace
diff --git a/clang/unittests/Analysis/FlowSensitive/TestingSupport.h b/clang/unittests/Analysis/FlowSensitive/TestingSupport.h
index 3b0e05ed7222..7348f8b1740d 100644
--- a/clang/unittests/Analysis/FlowSensitive/TestingSupport.h
+++ b/clang/unittests/Analysis/FlowSensitive/TestingSupport.h
@@ -355,8 +355,8 @@ checkDataflow(AnalysisInputs<AnalysisT> AI,
   auto SetupTest = [&StmtToAnnotations,
                     PrevSetupTest = std::move(AI.SetupTest)](
                        AnalysisOutputs &AO) -> llvm::Error {
-    auto MaybeStmtToAnnotations = buildStatementToAnnotationMapping(
-        cast<FunctionDecl>(AO.InitEnv.getDeclCtx()), AO.Code);
+    auto MaybeStmtToAnnotations =
+        buildStatementToAnnotationMapping(AO.InitEnv.getCurrentFunc(), AO.Code);
     if (!MaybeStmtToAnnotations) {
       return MaybeStmtToAnnotations.takeError();
     }
diff --git a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp
index b0b579d2bc19..1a52b82d6566 100644
--- a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp
@@ -146,6 +146,38 @@ TEST_F(DataflowAnalysisTest, DiagnoseFunctionDiagnoserCalledOnEachElement) {
                                    " (Lifetime ends)\n")));
 }
 
+TEST_F(DataflowAnalysisTest, CanAnalyzeStmt) {
+  std::string Code = R"cc(
+      struct S { bool b; };
+      void foo() {
+        S AnS = S{true};
+      }
+    )cc";
+  AST = tooling::buildASTFromCodeWithArgs(Code, {"-std=c++11"});
+  const auto &DeclStatement =
+      matchNode<DeclStmt>(declStmt(hasSingleDecl(varDecl(hasName("AnS")))));
+  const auto &Func = matchNode<FunctionDecl>(functionDecl(hasName("foo")));
+
+  ACFG = std::make_unique<AdornedCFG>(llvm::cantFail(AdornedCFG::build(
+      Func, const_cast<DeclStmt &>(DeclStatement), AST->getASTContext())));
+
+  NoopAnalysis Analysis = NoopAnalysis(AST->getASTContext());
+  DACtx = std::make_unique<DataflowAnalysisContext>(
+      std::make_unique<WatchedLiteralsSolver>());
+  Environment Env(*DACtx, const_cast<DeclStmt &>(DeclStatement));
+
+  llvm::Expected<std::vector<std::optional<DataflowAnalysisState<NoopLattice>>>>
+      Results = runDataflowAnalysis(*ACFG, Analysis, Env);
+
+  ASSERT_THAT_ERROR(Results.takeError(), llvm::Succeeded());
+  const Environment &ExitBlockEnv = Results->front()->Env;
+  BoolValue *BoolFieldValue = cast<BoolValue>(
+      getFieldValue(ExitBlockEnv.get<RecordStorageLocation>(
+                        *cast<VarDecl>((*DeclStatement.decl_begin()))),
+                    "b", AST->getASTContext(), ExitBlockEnv));
+  EXPECT_TRUE(Env.proves(BoolFieldValue->formula()));
+}
+
 // Tests for the statement-to-block map.
 using StmtToBlockTest = DataflowAnalysisTest;
 
-- 
GitLab


From ee765b0c94df7e636d9739216b1646d3a2d3b5db Mon Sep 17 00:00:00 2001
From: Tim Besard <tim.besard@gmail.com>
Date: Wed, 15 May 2024 22:17:29 +0200
Subject: [PATCH 0913/1206] [NewPM] Add pass options for `InternalizePass` to
 preserve GVs. (#91334)

This PR adds a string interface to `InternalizePass`' `MustPreserveGV`
option, which is a callback function to indicate if a GV is not to be
internalized. This is for use in LLVM.jl, the Julia wrapper for LLVM,
which uses the C API and is thus required to use the PassBuilder string
API for building NewPM pipelines.
---
 llvm/lib/Passes/PassBuilder.cpp           | 18 ++++++++++++++++++
 llvm/lib/Passes/PassRegistry.def          | 15 ++++++++++++++-
 llvm/test/Transforms/Internalize/lists.ll |  5 +++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index e4131706aba0..91c5b65c0351 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -1142,6 +1142,24 @@ Expected<GlobalMergeOptions> parseGlobalMergeOptions(StringRef Params) {
   return Result;
 }
 
+Expected<SmallVector<std::string, 0>> parseInternalizeGVs(StringRef Params) {
+  SmallVector<std::string, 1> PreservedGVs;
+  while (!Params.empty()) {
+    StringRef ParamName;
+    std::tie(ParamName, Params) = Params.split(';');
+
+    if (ParamName.consume_front("preserve-gv=")) {
+      PreservedGVs.push_back(ParamName.str());
+    } else {
+      return make_error<StringError>(
+          formatv("invalid Internalize pass parameter '{0}' ", ParamName).str(),
+          inconvertibleErrorCode());
+    }
+  }
+
+  return PreservedGVs;
+}
+
 } // namespace
 
 /// Tests whether a pass name starts with a valid prefix for a default pipeline
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index e5ce6cb7da64..50682ca4970f 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -78,7 +78,6 @@ MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass())
 MODULE_PASS("instrorderfile", InstrOrderFilePass())
 MODULE_PASS("instrprof", InstrProfilingLoweringPass())
 MODULE_PASS("ctx-instr-lower", PGOCtxProfLoweringPass())
-MODULE_PASS("internalize", InternalizePass())
 MODULE_PASS("invalidate<all>", InvalidateAllAnalysesPass())
 MODULE_PASS("iroutliner", IROutlinerPass())
 MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass())
@@ -175,6 +174,20 @@ MODULE_PASS_WITH_PARAMS(
     "hwasan", "HWAddressSanitizerPass",
     [](HWAddressSanitizerOptions Opts) { return HWAddressSanitizerPass(Opts); },
     parseHWASanPassOptions, "kernel;recover")
+MODULE_PASS_WITH_PARAMS(
+    "internalize", "InternalizePass",
+    [](SmallVector<std::string, 0> PreservedGVs) {
+      if (PreservedGVs.empty())
+        return InternalizePass();
+      auto MustPreserveGV = [=](const GlobalValue &GV) {
+        for (auto &PreservedGV : PreservedGVs)
+          if (GV.getName() == PreservedGV)
+            return true;
+        return false;
+      };
+      return InternalizePass(MustPreserveGV);
+    },
+    parseInternalizeGVs, "preserve-gv=GV")
 MODULE_PASS_WITH_PARAMS(
     "ipsccp", "IPSCCPPass", [](IPSCCPOptions Opts) { return IPSCCPPass(Opts); },
     parseIPSCCPOptions, "no-func-spec;func-spec")
diff --git a/llvm/test/Transforms/Internalize/lists.ll b/llvm/test/Transforms/Internalize/lists.ll
index df408f906b78..83dad03d75ea 100644
--- a/llvm/test/Transforms/Internalize/lists.ll
+++ b/llvm/test/Transforms/Internalize/lists.ll
@@ -13,6 +13,11 @@
 ; -file and -list options should be merged, the apifile contains foo and j
 ; RUN: opt < %s -passes=internalize -internalize-public-api-list bar -internalize-public-api-file %S/apifile -S | FileCheck --check-prefix=FOO_J_AND_BAR %s
 
+; specifying through pass builder option
+; RUN: opt < %s -passes='internalize<preserve-gv=foo;preserve-gv=j>' -S | FileCheck --check-prefix=FOO_AND_J %s
+; RUN: opt < %s -passes='internalize<preserve-gv=foo;preserve-gv=bar>' -S | FileCheck --check-prefix=FOO_AND_BAR %s
+; RUN: opt < %s -passes='internalize<preserve-gv=foo;preserve-gv=j;preserve-gv=bar>' -S | FileCheck --check-prefix=FOO_J_AND_BAR %s
+
 ; ALL: @i = internal global
 ; FOO_AND_J: @i = internal global
 ; FOO_AND_BAR: @i = internal global
-- 
GitLab


From ec1f28dc97ce22ba5b3e6f95ff84414dfbda46b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= <nicolai.haehnle@amd.com>
Date: Wed, 15 May 2024 22:23:18 +0200
Subject: [PATCH 0914/1206] AMDGPU/gfx12: avoid crashing on legacy waitcnt
 intrinsics (#92306)

They *are* still accepted by the HW but have a conservative effect.

Leave them untouched since handling them would complicate the logic a
bit, and developers who code to such a low level really need to revisit
what they're doing anyway.
---
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp   |   5 +
 .../CodeGen/AMDGPU/waitcnt-preexisting.mir    | 175 ++++++++++++++++++
 2 files changed, 180 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 839ac927a0ee..5577ce9eb128 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1364,6 +1364,11 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
     unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(II.getOpcode());
     bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
 
+    // Don't crash if the programmer used legacy waitcnt intrinsics, but don't
+    // attempt to do more than that either.
+    if (Opcode == AMDGPU::S_WAITCNT)
+      continue;
+
     if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
       unsigned OldEnc =
           TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting.mir
index 4c01786e45f5..e15814210dfd 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting.mir
@@ -1,5 +1,12 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs -run-pass si-insert-waitcnts -o - %s | FileCheck -check-prefixes=GFX9 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass si-insert-waitcnts -o - %s | FileCheck -check-prefixes=GFX12 %s
+
+# For gfx12+, this test simply ensures that we don't crash in the face of manually
+# inserted waitcnt intrinsics. They are still allowed for compatibility, but
+# their effect in the HW is very conservative and code generation does not attempt
+# to do anything with them. Developers who write code at such a low level should
+# revisit their code for gfx12+ anyway.
 
 ---
 name:            test_waitcnt_preexisting_lgkmcnt_unmodified
@@ -17,6 +24,22 @@ body:             |
     ; GFX9-NEXT: S_WAITCNT 112
     ; GFX9-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX9-NEXT: S_ENDPGM 0
+    ;
+    ; GFX12-LABEL: name: test_waitcnt_preexisting_lgkmcnt_unmodified
+    ; GFX12: liveins: $vgpr0
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: S_WAIT_EXPCNT 0
+    ; GFX12-NEXT: S_WAIT_SAMPLECNT 0
+    ; GFX12-NEXT: S_WAIT_BVHCNT 0
+    ; GFX12-NEXT: S_WAIT_KMCNT 0
+    ; GFX12-NEXT: $vgpr0_vgpr1 = DS_READ2_B32 $vgpr0, 0, 1, 0, implicit $m0, implicit $exec
+    ; GFX12-NEXT: S_WAITCNT 49279
+    ; GFX12-NEXT: S_WAIT_DSCNT 0
+    ; GFX12-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_ENDPGM 0
     $vgpr0_vgpr1 = DS_READ2_B32 $vgpr0, 0, 1, 0, implicit $m0, implicit $exec
     S_WAITCNT 49279
     $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
@@ -40,6 +63,22 @@ body:             |
     ; GFX9-NEXT: S_WAITCNT 112
     ; GFX9-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX9-NEXT: S_ENDPGM 0
+    ;
+    ; GFX12-LABEL: name: test_waitcnt_preexisting_vmcnt_unmodified
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: S_WAIT_EXPCNT 0
+    ; GFX12-NEXT: S_WAIT_SAMPLECNT 0
+    ; GFX12-NEXT: S_WAIT_BVHCNT 0
+    ; GFX12-NEXT: S_WAIT_KMCNT 0
+    ; GFX12-NEXT: $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GFX12-NEXT: S_WAITCNT 3952
+    ; GFX12-NEXT: S_WAIT_LOADCNT 0
+    ; GFX12-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_ENDPGM 0
     $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
     S_WAITCNT 3952
     $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
@@ -65,6 +104,22 @@ body:             |
     ; GFX9-NEXT: S_WAITCNT 112
     ; GFX9-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX9-NEXT: S_ENDPGM 0
+    ;
+    ; GFX12-LABEL: name: test_waitcnt_preexisting_vmcnt_needs_lgkmcnt
+    ; GFX12: liveins: $vgpr0
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: S_WAIT_EXPCNT 0
+    ; GFX12-NEXT: S_WAIT_SAMPLECNT 0
+    ; GFX12-NEXT: S_WAIT_BVHCNT 0
+    ; GFX12-NEXT: S_WAIT_KMCNT 0
+    ; GFX12-NEXT: $vgpr0_vgpr1 = DS_READ2_B32 $vgpr0, 0, 1, 0, implicit $m0, implicit $exec
+    ; GFX12-NEXT: S_WAITCNT 3952
+    ; GFX12-NEXT: S_WAIT_DSCNT 0
+    ; GFX12-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_ENDPGM 0
     $vgpr0_vgpr1 = DS_READ2_B32 $vgpr0, 0, 1, 0, implicit $m0, implicit $exec
     S_WAITCNT 3952
     $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
@@ -88,6 +143,22 @@ body:             |
     ; GFX9-NEXT: S_WAITCNT 112
     ; GFX9-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX9-NEXT: S_ENDPGM 0
+    ;
+    ; GFX12-LABEL: name: test_waitcnt_preexisting_lgkmcnt_needs_vmcnt
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: S_WAIT_EXPCNT 0
+    ; GFX12-NEXT: S_WAIT_SAMPLECNT 0
+    ; GFX12-NEXT: S_WAIT_BVHCNT 0
+    ; GFX12-NEXT: S_WAIT_KMCNT 0
+    ; GFX12-NEXT: $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GFX12-NEXT: S_WAITCNT 49279
+    ; GFX12-NEXT: S_WAIT_LOADCNT 0
+    ; GFX12-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_ENDPGM 0
     $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
     S_WAITCNT 49279
     $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
@@ -115,6 +186,24 @@ body:             |
     ; GFX9-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr4_vgpr5, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX9-NEXT: S_WAITCNT 112
     ; GFX9-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ;
+    ; GFX12-LABEL: name: test_waitcnt_preexisting_apply_all_counters
+    ; GFX12: liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: S_WAIT_EXPCNT 0
+    ; GFX12-NEXT: S_WAIT_SAMPLECNT 0
+    ; GFX12-NEXT: S_WAIT_BVHCNT 0
+    ; GFX12-NEXT: S_WAIT_KMCNT 0
+    ; GFX12-NEXT: $vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GFX12-NEXT: $vgpr6_vgpr7 = DS_READ2_B32 $vgpr2, 0, 1, 0, implicit $m0, implicit $exec
+    ; GFX12-NEXT: S_WAITCNT 0
+    ; GFX12-NEXT: S_WAIT_DSCNT 0
+    ; GFX12-NEXT: $vgpr6 = V_OR_B32_e32 1, killed $vgpr6, implicit $exec
+    ; GFX12-NEXT: S_WAIT_LOADCNT 0
+    ; GFX12-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr4_vgpr5, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
     $vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
     $vgpr6_vgpr7 = DS_READ2_B32 $vgpr2, 0, 1, 0, implicit $m0, implicit $exec
     S_WAITCNT 0
@@ -136,6 +225,24 @@ body:             |
     ; GFX9-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX9-NEXT: S_WAITCNT 0
     ; GFX9-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ;
+    ; GFX12-LABEL: name: test_waitcnt_preexisting_combine_waitcnt
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: S_WAIT_EXPCNT 0
+    ; GFX12-NEXT: S_WAIT_SAMPLECNT 0
+    ; GFX12-NEXT: S_WAIT_BVHCNT 0
+    ; GFX12-NEXT: S_WAIT_KMCNT 0
+    ; GFX12-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_WAITCNT 0
+    ; GFX12-NEXT: S_WAITCNT 0
+    ; GFX12-NEXT: S_WAITCNT 0
+    ; GFX12-NEXT: S_WAITCNT 0
+    ; GFX12-NEXT: S_WAITCNT 0
+    ; GFX12-NEXT: S_WAITCNT 0
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
     $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     S_WAITCNT 0
     S_WAITCNT 0
@@ -159,6 +266,20 @@ body:             |
     ; GFX9-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX9-NEXT: S_WAITCNT 112
     ; GFX9-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ;
+    ; GFX12-LABEL: name: test_waitcnt_preexisting_combine_waitcnt_diff_counters
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: S_WAIT_EXPCNT 0
+    ; GFX12-NEXT: S_WAIT_SAMPLECNT 0
+    ; GFX12-NEXT: S_WAIT_BVHCNT 0
+    ; GFX12-NEXT: S_WAIT_KMCNT 0
+    ; GFX12-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_WAITCNT 49279
+    ; GFX12-NEXT: S_WAITCNT 3952
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
     $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     S_WAITCNT 49279
     S_WAITCNT 3952
@@ -185,6 +306,23 @@ body:             |
     ; GFX9-NEXT: S_NOP 0
     ; GFX9-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX9-NEXT: S_ENDPGM 0
+    ;
+    ; GFX12-LABEL: name: test_waitcnt_preexisting_early_wait
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: S_WAIT_EXPCNT 0
+    ; GFX12-NEXT: S_WAIT_SAMPLECNT 0
+    ; GFX12-NEXT: S_WAIT_BVHCNT 0
+    ; GFX12-NEXT: S_WAIT_KMCNT 0
+    ; GFX12-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_WAITCNT 0
+    ; GFX12-NEXT: S_NOP 0
+    ; GFX12-NEXT: S_NOP 0
+    ; GFX12-NEXT: S_NOP 0
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_ENDPGM 0
     $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     S_WAITCNT 0
     S_NOP 0
@@ -207,6 +345,18 @@ body:             |
     ; GFX9-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX9-NEXT: S_WAITCNT 3952
     ; GFX9-NEXT: KILL $vgpr0
+    ;
+    ; GFX12-LABEL: name: test_waitcnt_preexisting_ignore_kill
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: S_WAIT_EXPCNT 0
+    ; GFX12-NEXT: S_WAIT_SAMPLECNT 0
+    ; GFX12-NEXT: S_WAIT_BVHCNT 0
+    ; GFX12-NEXT: S_WAIT_KMCNT 0
+    ; GFX12-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_WAITCNT 3952
+    ; GFX12-NEXT: KILL $vgpr0
     $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     S_WAITCNT 3952
     KILL $vgpr0
@@ -221,6 +371,15 @@ body:             |
     ; GFX9-LABEL: name: test_waitcnt_preexisting_func_start
     ; GFX9: S_WAITCNT 0
     ; GFX9-NEXT: S_ENDPGM 0
+    ;
+    ; GFX12-LABEL: name: test_waitcnt_preexisting_func_start
+    ; GFX12: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: S_WAIT_EXPCNT 0
+    ; GFX12-NEXT: S_WAIT_SAMPLECNT 0
+    ; GFX12-NEXT: S_WAIT_BVHCNT 0
+    ; GFX12-NEXT: S_WAIT_KMCNT 0
+    ; GFX12-NEXT: S_WAITCNT 0
+    ; GFX12-NEXT: S_ENDPGM 0
     S_WAITCNT 0
     S_ENDPGM 0
 ...
@@ -241,6 +400,22 @@ body:             |
     ; GFX9-NEXT: S_WAITCNT 112
     ; GFX9-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX9-NEXT: S_ENDPGM 0
+    ;
+    ; GFX12-LABEL: name: test_waitcnt_preexisting_buffer_inv
+    ; GFX12: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: S_WAIT_EXPCNT 0
+    ; GFX12-NEXT: S_WAIT_SAMPLECNT 0
+    ; GFX12-NEXT: S_WAIT_BVHCNT 0
+    ; GFX12-NEXT: S_WAIT_KMCNT 0
+    ; GFX12-NEXT: $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GFX12-NEXT: S_WAITCNT 3952
+    ; GFX12-NEXT: BUFFER_INVL2 implicit $exec
+    ; GFX12-NEXT: S_WAIT_LOADCNT 0
+    ; GFX12-NEXT: BUFFER_WBINVL1_VOL implicit $exec
+    ; GFX12-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GFX12-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: S_ENDPGM 0
     $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
     S_WAITCNT 3952
     BUFFER_INVL2 implicit $exec
-- 
GitLab


From 81d20d861e48f5202c9f79b47dee244674fb9121 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Wed, 15 May 2024 15:30:05 -0500
Subject: [PATCH 0915/1206] [Offload][NFC] Fix warning messages in runtime

Summary:
These are lots of random warnings due to inconsistent initialization or
signedness.
---
 offload/plugins-nextgen/amdgpu/src/rtl.cpp           | 11 ++++-------
 .../plugins-nextgen/common/src/PluginInterface.cpp   |  2 +-
 offload/src/LegacyAPI.cpp                            |  6 ++++--
 offload/src/OpenMP/API.cpp                           |  2 +-
 offload/src/OpenMP/Mapping.cpp                       | 12 +++---------
 offload/src/omptarget.cpp                            |  2 +-
 6 files changed, 14 insertions(+), 21 deletions(-)

diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 295685fceaa4..2a9503333c19 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -1670,10 +1670,10 @@ private:
   hsa_agent_t Agent;
 
   /// The maximum number of queues.
-  int MaxNumQueues;
+  uint32_t MaxNumQueues;
 
   /// The size of created queues.
-  int QueueSize;
+  uint32_t QueueSize;
 };
 
 /// Abstract class that holds the common members of the actual kernel devices
@@ -1847,8 +1847,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   // Create an AMDGPU device with a device id and default AMDGPU grid values.
   AMDGPUDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId, int32_t NumDevices,
                  AMDHostDeviceTy &HostDevice, hsa_agent_t Agent)
-      : GenericDeviceTy(Plugin, DeviceId, NumDevices, {0}),
-        AMDGenericDeviceTy(),
+      : GenericDeviceTy(Plugin, DeviceId, NumDevices, {}), AMDGenericDeviceTy(),
         OMPX_NumQueues("LIBOMPTARGET_AMDGPU_NUM_HSA_QUEUES", 4),
         OMPX_QueueSize("LIBOMPTARGET_AMDGPU_HSA_QUEUE_SIZE", 512),
         OMPX_DefaultTeamsPerCU("LIBOMPTARGET_AMDGPU_TEAMS_PER_CU", 4),
@@ -2015,9 +2014,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return Plugin::success();
   }
 
-  const uint64_t getStreamBusyWaitMicroseconds() const {
-    return OMPX_StreamBusyWait;
-  }
+  uint64_t getStreamBusyWaitMicroseconds() const { return OMPX_StreamBusyWait; }
 
   Expected<std::unique_ptr<MemoryBuffer>>
   doJITPostProcessing(std::unique_ptr<MemoryBuffer> MB) const override {
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index a5c8cce63fac..737a8b2a4064 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -593,7 +593,7 @@ void *GenericKernelTy::prepareArgs(
     Args[0] = &Ptrs[0];
   }
 
-  for (int I = KLEOffset; I < NumArgs; ++I) {
+  for (uint32_t I = KLEOffset; I < NumArgs; ++I) {
     Ptrs[I] =
         (void *)((intptr_t)ArgPtrs[I - KLEOffset] + ArgOffsets[I - KLEOffset]);
     Args[I] = &Ptrs[I];
diff --git a/offload/src/LegacyAPI.cpp b/offload/src/LegacyAPI.cpp
index 91d5642e8112..033d7a3ef712 100644
--- a/offload/src/LegacyAPI.cpp
+++ b/offload/src/LegacyAPI.cpp
@@ -88,7 +88,8 @@ EXTERN int __tgt_target_mapper(ident_t *Loc, int64_t DeviceId, void *HostPtr,
   TIMESCOPE_WITH_IDENT(Loc);
   OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
   KernelArgsTy KernelArgs{1,        ArgNum,   ArgsBase,   Args, ArgSizes,
-                          ArgTypes, ArgNames, ArgMappers, 0};
+                          ArgTypes, ArgNames, ArgMappers, 0,    {},
+                          {},       {},       0};
   return __tgt_target_kernel(Loc, DeviceId, -1, -1, HostPtr, &KernelArgs);
 }
 
@@ -132,7 +133,8 @@ EXTERN int __tgt_target_teams_mapper(ident_t *Loc, int64_t DeviceId,
   TIMESCOPE_WITH_IDENT(Loc);
   OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
   KernelArgsTy KernelArgs{1,        ArgNum,   ArgsBase,   Args, ArgSizes,
-                          ArgTypes, ArgNames, ArgMappers, 0};
+                          ArgTypes, ArgNames, ArgMappers, 0,    {},
+                          {},       {},       0};
   return __tgt_target_kernel(Loc, DeviceId, NumTeams, ThreadLimit, HostPtr,
                              &KernelArgs);
 }
diff --git a/offload/src/OpenMP/API.cpp b/offload/src/OpenMP/API.cpp
index c85f9868e37c..374c54163d6a 100644
--- a/offload/src/OpenMP/API.cpp
+++ b/offload/src/OpenMP/API.cpp
@@ -642,7 +642,7 @@ EXTERN void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) {
     return nullptr;
   }
 
-  size_t NumDevices = omp_get_initial_device();
+  int NumDevices = omp_get_initial_device();
   if (DeviceNum == NumDevices) {
     DP("Device %d is initial device, returning Ptr " DPxMOD ".\n",
            DeviceNum, DPxPTR(Ptr));
diff --git a/offload/src/OpenMP/Mapping.cpp b/offload/src/OpenMP/Mapping.cpp
index c6ff3aa54dd6..595e3456ab54 100644
--- a/offload/src/OpenMP/Mapping.cpp
+++ b/offload/src/OpenMP/Mapping.cpp
@@ -314,9 +314,7 @@ TargetPointerResultTy MappingInfoTy::getTargetPointer(
 
     // Notify the plugin about the new mapping.
     if (Device.notifyDataMapped(HstPtrBegin, Size))
-      return {{false /*IsNewEntry=*/, false /*IsHostPointer=*/},
-              nullptr /*Entry=*/,
-              nullptr /*TargetPointer=*/};
+      return TargetPointerResultTy{};
   } else {
     // This entry is not present and we did not create a new entry for it.
     LR.TPR.Flags.IsPresent = false;
@@ -344,9 +342,7 @@ TargetPointerResultTy MappingInfoTy::getTargetPointer(
       LR.TPR.TargetPointer = nullptr;
     } else if (LR.TPR.getEntry()->addEventIfNecessary(Device, AsyncInfo) !=
                OFFLOAD_SUCCESS)
-      return {{false /*IsNewEntry=*/, false /*IsHostPointer=*/},
-              nullptr /*Entry=*/,
-              nullptr /*TargetPointer=*/};
+      return TargetPointerResultTy{};
   } else {
     // If not a host pointer and no present modifier, we need to wait for the
     // event if it exists.
@@ -360,9 +356,7 @@ TargetPointerResultTy MappingInfoTy::getTargetPointer(
           // If it fails to wait for the event, we need to return nullptr in
           // case of any data race.
           REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event));
-          return {{false /*IsNewEntry=*/, false /*IsHostPointer=*/},
-                  nullptr /*Entry=*/,
-                  nullptr /*TargetPointer=*/};
+          return TargetPointerResultTy{};
         }
       }
     }
diff --git a/offload/src/omptarget.cpp b/offload/src/omptarget.cpp
index 5d5c6b05051b..91e1213f175e 100644
--- a/offload/src/omptarget.cpp
+++ b/offload/src/omptarget.cpp
@@ -1750,7 +1750,7 @@ int target_replay(ident_t *Loc, DeviceTy &Device, void *HostPtr,
                                   TARGET_ALLOC_DEFAULT);
   Device.submitData(TgtPtr, DeviceMemory, DeviceMemorySize, AsyncInfo);
 
-  KernelArgsTy KernelArgs = {0};
+  KernelArgsTy KernelArgs{};
   KernelArgs.Version = OMP_KERNEL_ARG_VERSION;
   KernelArgs.NumArgs = NumArgs;
   KernelArgs.Tripcount = LoopTripCount;
-- 
GitLab


From 83f065d582977aca5c037c27a7290f30850bdd35 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb@igalia.com>
Date: Wed, 15 May 2024 21:33:10 +0100
Subject: [PATCH 0916/1206] [RISCV] static_assert SupportedProfiles and
 SupportedExperimentalProfiles are sorted

Just as we do for the arrays of extension names.
---
 llvm/lib/TargetParser/RISCVISAInfo.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 706b2853cd2c..827bc5b44387 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -39,6 +39,10 @@ struct RISCVSupportedExtension {
 struct RISCVProfile {
   StringLiteral Name;
   StringLiteral MArch;
+
+  bool operator<(const RISCVProfile &RHS) const {
+    return StringRef(Name) < StringRef(RHS.Name);
+  }
 };
 
 } // end anonymous namespace
@@ -61,6 +65,10 @@ static void verifyTables() {
            "Extensions are not sorted by name");
     assert(llvm::is_sorted(SupportedExperimentalExtensions) &&
            "Experimental extensions are not sorted by name");
+    assert(llvm::is_sorted(SupportedProfiles) &&
+           "Profiles are not sorted by name");
+    assert(llvm::is_sorted(SupportedExperimentalProfiles) &&
+           "Experimental profiles are not sorted by name");
     TableChecked.store(true, std::memory_order_relaxed);
   }
 #endif
-- 
GitLab


From e1ed138a67a92ef1ff0214ca198094be13045090 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Wed, 15 May 2024 14:38:45 -0700
Subject: [PATCH 0917/1206] [bazel] Port #92199

---
 utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index df5cd276b12f..c469da74fc56 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -1125,6 +1125,8 @@ cc_library(
     ]),
     copts = llvm_copts,
     deps = [
+        ":BitstreamReader",
+        ":BitstreamWriter",
         ":Core",
         ":DebugInfo",
         ":DebugInfoDWARF",
-- 
GitLab


From 8530b1c464ae9af4a5c8be145a8db043798634f6 Mon Sep 17 00:00:00 2001
From: Dave Lee <davelee.com@gmail.com>
Date: Wed, 15 May 2024 14:44:42 -0700
Subject: [PATCH 0918/1206] [lldb] Support custom LLVM formatting for variables
 (#91868)

Re-apply https://github.com/llvm/llvm-project/pull/81196, with a fix that handles the
absence of llvm formatting:
https://github.com/llvm/llvm-project/pull/91868/commits/3ba650e91eded3543764f37921dcce3b
b47d425f
---
 lldb/docs/use/variable.rst                    |  9 +++
 lldb/source/Core/FormatEntity.cpp             | 72 ++++++++++++++++---
 .../custom-printf-summary/Makefile            |  2 +
 .../TestCustomSummaryLLVMFormat.py            | 20 ++++++
 .../custom-printf-summary/main.c              | 13 ++++
 5 files changed, 106 insertions(+), 10 deletions(-)
 create mode 100644 lldb/test/API/functionalities/data-formatter/custom-printf-summary/Makefile
 create mode 100644 lldb/test/API/functionalities/data-formatter/custom-printf-summary/TestCustomSummaryLLVMFormat.py
 create mode 100644 lldb/test/API/functionalities/data-formatter/custom-printf-summary/main.c

diff --git a/lldb/docs/use/variable.rst b/lldb/docs/use/variable.rst
index 8eaed6405315..e9175b25336b 100644
--- a/lldb/docs/use/variable.rst
+++ b/lldb/docs/use/variable.rst
@@ -460,6 +460,15 @@ summary strings, regardless of the format they have applied to their types. To
 do that, you can use %format inside an expression path, as in ${var.x->x%u},
 which would display the value of x as an unsigned integer.
 
+Additionally, custom output can be achieved by using an LLVM format string,
+commencing with the ``:`` marker. To illustrate, compare ``${var.byte%x}`` and
+``${var.byte:x-}``. The former uses lldb's builtin hex formatting (``x``),
+which unconditionally inserts a ``0x`` prefix, and also zero pads the value to
+match the size of the type. The latter uses ``llvm::formatv`` formatting
+(``:x-``), and will print only the hex value, with no ``0x`` prefix, and no
+padding. This raw control is useful when composing multiple pieces into a
+larger whole.
+
 You can also use some other special format markers, not available for formats
 themselves, but which carry a special meaning when used in this context:
 
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index ba62e2625259..07978d388296 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -57,6 +57,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/Regex.h"
 #include "llvm/TargetParser/Triple.h"
 
 #include <cctype>
@@ -658,6 +659,37 @@ static char ConvertValueObjectStyleToChar(
   return '\0';
 }
 
+/// Options supported by format_provider<T> for integral arithmetic types.
+/// See table in FormatProviders.h.
+static llvm::Regex LLVMFormatPattern{"x[-+]?\\d*|n|d", llvm::Regex::IgnoreCase};
+
+static bool DumpValueWithLLVMFormat(Stream &s, llvm::StringRef options,
+                                    ValueObject &valobj) {
+  std::string formatted;
+  std::string llvm_format = ("{0:" + options + "}").str();
+
+  auto type_info = valobj.GetTypeInfo();
+  if ((type_info & eTypeIsInteger) && LLVMFormatPattern.match(options)) {
+    if (type_info & eTypeIsSigned) {
+      bool success = false;
+      int64_t integer = valobj.GetValueAsSigned(0, &success);
+      if (success)
+        formatted = llvm::formatv(llvm_format.data(), integer);
+    } else {
+      bool success = false;
+      uint64_t integer = valobj.GetValueAsUnsigned(0, &success);
+      if (success)
+        formatted = llvm::formatv(llvm_format.data(), integer);
+    }
+  }
+
+  if (formatted.empty())
+    return false;
+
+  s.Write(formatted.data(), formatted.size());
+  return true;
+}
+
 static bool DumpValue(Stream &s, const SymbolContext *sc,
                       const ExecutionContext *exe_ctx,
                       const FormatEntity::Entry &entry, ValueObject *valobj) {
@@ -728,9 +760,12 @@ static bool DumpValue(Stream &s, const SymbolContext *sc,
     return RunScriptFormatKeyword(s, sc, exe_ctx, valobj, entry.string.c_str());
   }
 
-  llvm::StringRef subpath(entry.string);
+  auto split = llvm::StringRef(entry.string).split(':');
+  auto subpath = split.first;
+  auto llvm_format = split.second;
+
   // simplest case ${var}, just print valobj's value
-  if (entry.string.empty()) {
+  if (subpath.empty()) {
     if (entry.printf_format.empty() && entry.fmt == eFormatDefault &&
         entry.number == ValueObject::eValueObjectRepresentationStyleValue)
       was_plain_var = true;
@@ -739,7 +774,7 @@ static bool DumpValue(Stream &s, const SymbolContext *sc,
     target = valobj;
   } else // this is ${var.something} or multiple .something nested
   {
-    if (entry.string[0] == '[')
+    if (subpath[0] == '[')
       was_var_indexed = true;
     ScanBracketedRange(subpath, close_bracket_index,
                        var_name_final_if_array_range, index_lower,
@@ -747,14 +782,11 @@ static bool DumpValue(Stream &s, const SymbolContext *sc,
 
     Status error;
 
-    const std::string &expr_path = entry.string;
-
-    LLDB_LOGF(log, "[Debugger::FormatPrompt] symbol to expand: %s",
-              expr_path.c_str());
+    LLDB_LOG(log, "[Debugger::FormatPrompt] symbol to expand: {0}", subpath);
 
     target =
         valobj
-            ->GetValueForExpressionPath(expr_path.c_str(), &reason_to_stop,
+            ->GetValueForExpressionPath(subpath, &reason_to_stop,
                                         &final_value_type, options, &what_next)
             .get();
 
@@ -883,8 +915,18 @@ static bool DumpValue(Stream &s, const SymbolContext *sc,
   }
 
   if (!is_array_range) {
-    LLDB_LOGF(log,
-              "[Debugger::FormatPrompt] dumping ordinary printable output");
+    if (!llvm_format.empty()) {
+      if (DumpValueWithLLVMFormat(s, llvm_format, *target)) {
+        LLDB_LOGF(log, "dumping using llvm format");
+        return true;
+      } else {
+        LLDB_LOG(
+            log,
+            "empty output using llvm format '{0}' - with type info flags {1}",
+            entry.printf_format, target->GetTypeInfo());
+      }
+    }
+    LLDB_LOGF(log, "dumping ordinary printable output");
     return target->DumpPrintableRepresentation(s, val_obj_display,
                                                custom_format);
   } else {
@@ -2227,6 +2269,16 @@ static Status ParseInternal(llvm::StringRef &format, Entry &parent_entry,
           if (error.Fail())
             return error;
 
+          llvm::StringRef entry_string(entry.string);
+          if (entry_string.contains(':')) {
+            auto [_, llvm_format] = entry_string.split(':');
+            if (!llvm_format.empty() && !LLVMFormatPattern.match(llvm_format)) {
+              error.SetErrorStringWithFormat("invalid llvm format: '%s'",
+                                             llvm_format.data());
+              return error;
+            }
+          }
+
           if (verify_is_thread_id) {
             if (entry.type != Entry::Type::ThreadID &&
                 entry.type != Entry::Type::ThreadProtocolID) {
diff --git a/lldb/test/API/functionalities/data-formatter/custom-printf-summary/Makefile b/lldb/test/API/functionalities/data-formatter/custom-printf-summary/Makefile
new file mode 100644
index 000000000000..c9319d6e6888
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/custom-printf-summary/Makefile
@@ -0,0 +1,2 @@
+C_SOURCES := main.c
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/custom-printf-summary/TestCustomSummaryLLVMFormat.py b/lldb/test/API/functionalities/data-formatter/custom-printf-summary/TestCustomSummaryLLVMFormat.py
new file mode 100644
index 000000000000..d6906a49463b
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/custom-printf-summary/TestCustomSummaryLLVMFormat.py
@@ -0,0 +1,20 @@
+import lldb
+from lldbsuite.test.lldbtest import *
+import lldbsuite.test.lldbutil as lldbutil
+
+
+class TestCase(TestBase):
+    def test_raw_bytes(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c"))
+        self.runCmd("type summary add -s '${var.ubyte:x-2}${var.sbyte:x-2}!' Bytes")
+        self.expect("v bytes", substrs=[" = 3001!"])
+
+    def test_bad_format(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c"))
+        self.expect(
+            "type summary add -s '${var.ubyte:y}!' Bytes",
+            error=True,
+            substrs=["invalid llvm format"],
+        )
diff --git a/lldb/test/API/functionalities/data-formatter/custom-printf-summary/main.c b/lldb/test/API/functionalities/data-formatter/custom-printf-summary/main.c
new file mode 100644
index 000000000000..4164aff7dbf6
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/custom-printf-summary/main.c
@@ -0,0 +1,13 @@
+#include <stdint.h>
+#include <stdio.h>
+
+struct Bytes {
+  uint8_t ubyte;
+  int8_t sbyte;
+};
+
+int main() {
+  struct Bytes bytes = {0x30, 0x01};
+  (void)bytes;
+  printf("break here\n");
+}
-- 
GitLab


From 1daa7fd3fadd17e61d9dfa56f84228617c5514d9 Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara@apple.com>
Date: Wed, 15 May 2024 14:38:18 -0700
Subject: [PATCH 0919/1206] [AArch64][SME] Remove Darwin compile error for ABI
 support routine calls.

These are allowed for Darwin and use the same ABI.
---
 llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp      |  8 ++------
 .../sme-support-routines-calling-convention.ll       | 12 ++++++++++++
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 5a5a18edb12e..d82fa3924f83 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -233,13 +233,9 @@ AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF,
     report_fatal_error(
         "Calling convention SVE_VectorCall is unsupported on Darwin.");
   if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
-    report_fatal_error(
-        "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is "
-        "unsupported on Darwin.");
+    return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
   if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
-    report_fatal_error(
-        "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
-        "unsupported on Darwin.");
+    return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask;
   if (CC == CallingConv::CFGuard_Check)
     report_fatal_error(
         "Calling convention CFGuard_Check is unsupported on Darwin.");
diff --git a/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll b/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll
index d88deec40ce7..7535638137ca 100644
--- a/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll
+++ b/llvm/test/CodeGen/AArch64/sme-support-routines-calling-convention.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-apple-darwin -mattr=+sme -verify-machineinstrs < %s | FileCheck %s --check-prefix=DARWIN
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -verify-machineinstrs -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=CHECK-CSRMASK
+; RUN: llc -mtriple=aarch64-apple-darwin -mattr=+sme -verify-machineinstrs -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=CHECK-CSRMASK
 
 ; Test that the PCS attribute is accepted and uses the correct register mask.
 ;
@@ -11,6 +13,11 @@ define void @test_sme_calling_convention_x0() nounwind {
 ; CHECK-NEXT:    bl __arm_tpidr2_save
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
+; DARWIN-LABEL: test_sme_calling_convention_x0:
+; DARWIN:    stp	x29, x30, [sp, #-16]!
+; DARWIN:    bl ___arm_tpidr2_save
+; DARWIN:    ldp	x29, x30, [sp], #16
+; DARWIN:    ret
 ;
 ; CHECK-CSRMASK-LABEL: name: test_sme_calling_convention_x0
 ; CHECK-CSRMASK: BL @__arm_tpidr2_save, csr_aarch64_sme_abi_support_routines_preservemost_from_x0
@@ -25,6 +32,11 @@ define i64 @test_sme_calling_convention_x2() nounwind {
 ; CHECK-NEXT:    bl __arm_sme_state
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
+; DARWIN-LABEL: test_sme_calling_convention_x2:
+; DARWIN:    stp	x29, x30, [sp, #-16]!
+; DARWIN:    bl ___arm_sme_state
+; DARWIN:    ldp	x29, x30, [sp], #16
+; DARWIN:    ret
 ;
 ; CHECK-CSRMASK-LABEL: name: test_sme_calling_convention_x2
 ; CHECK-CSRMASK: BL @__arm_sme_state, csr_aarch64_sme_abi_support_routines_preservemost_from_x2
-- 
GitLab


From 537a94b2ef67cd96a4b3a9b5612ea726a91c602b Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Wed, 15 May 2024 15:06:08 -0700
Subject: [PATCH 0920/1206] Revert "[NewPM] Add pass options for
 `InternalizePass` to preserve GVs." (#92321)

Reverts llvm/llvm-project#91334

This broke the gcc7 build.
I suspect the issue is a mismatch on user-defined move constructor on
the return: `return PreservedGVs;` does not match the return type of the
function.
---
 llvm/lib/Passes/PassBuilder.cpp           | 18 ------------------
 llvm/lib/Passes/PassRegistry.def          | 15 +--------------
 llvm/test/Transforms/Internalize/lists.ll |  5 -----
 3 files changed, 1 insertion(+), 37 deletions(-)

diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 91c5b65c0351..e4131706aba0 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -1142,24 +1142,6 @@ Expected<GlobalMergeOptions> parseGlobalMergeOptions(StringRef Params) {
   return Result;
 }
 
-Expected<SmallVector<std::string, 0>> parseInternalizeGVs(StringRef Params) {
-  SmallVector<std::string, 1> PreservedGVs;
-  while (!Params.empty()) {
-    StringRef ParamName;
-    std::tie(ParamName, Params) = Params.split(';');
-
-    if (ParamName.consume_front("preserve-gv=")) {
-      PreservedGVs.push_back(ParamName.str());
-    } else {
-      return make_error<StringError>(
-          formatv("invalid Internalize pass parameter '{0}' ", ParamName).str(),
-          inconvertibleErrorCode());
-    }
-  }
-
-  return PreservedGVs;
-}
-
 } // namespace
 
 /// Tests whether a pass name starts with a valid prefix for a default pipeline
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 50682ca4970f..e5ce6cb7da64 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -78,6 +78,7 @@ MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass())
 MODULE_PASS("instrorderfile", InstrOrderFilePass())
 MODULE_PASS("instrprof", InstrProfilingLoweringPass())
 MODULE_PASS("ctx-instr-lower", PGOCtxProfLoweringPass())
+MODULE_PASS("internalize", InternalizePass())
 MODULE_PASS("invalidate<all>", InvalidateAllAnalysesPass())
 MODULE_PASS("iroutliner", IROutlinerPass())
 MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass())
@@ -174,20 +175,6 @@ MODULE_PASS_WITH_PARAMS(
     "hwasan", "HWAddressSanitizerPass",
     [](HWAddressSanitizerOptions Opts) { return HWAddressSanitizerPass(Opts); },
     parseHWASanPassOptions, "kernel;recover")
-MODULE_PASS_WITH_PARAMS(
-    "internalize", "InternalizePass",
-    [](SmallVector<std::string, 0> PreservedGVs) {
-      if (PreservedGVs.empty())
-        return InternalizePass();
-      auto MustPreserveGV = [=](const GlobalValue &GV) {
-        for (auto &PreservedGV : PreservedGVs)
-          if (GV.getName() == PreservedGV)
-            return true;
-        return false;
-      };
-      return InternalizePass(MustPreserveGV);
-    },
-    parseInternalizeGVs, "preserve-gv=GV")
 MODULE_PASS_WITH_PARAMS(
     "ipsccp", "IPSCCPPass", [](IPSCCPOptions Opts) { return IPSCCPPass(Opts); },
     parseIPSCCPOptions, "no-func-spec;func-spec")
diff --git a/llvm/test/Transforms/Internalize/lists.ll b/llvm/test/Transforms/Internalize/lists.ll
index 83dad03d75ea..df408f906b78 100644
--- a/llvm/test/Transforms/Internalize/lists.ll
+++ b/llvm/test/Transforms/Internalize/lists.ll
@@ -13,11 +13,6 @@
 ; -file and -list options should be merged, the apifile contains foo and j
 ; RUN: opt < %s -passes=internalize -internalize-public-api-list bar -internalize-public-api-file %S/apifile -S | FileCheck --check-prefix=FOO_J_AND_BAR %s
 
-; specifying through pass builder option
-; RUN: opt < %s -passes='internalize<preserve-gv=foo;preserve-gv=j>' -S | FileCheck --check-prefix=FOO_AND_J %s
-; RUN: opt < %s -passes='internalize<preserve-gv=foo;preserve-gv=bar>' -S | FileCheck --check-prefix=FOO_AND_BAR %s
-; RUN: opt < %s -passes='internalize<preserve-gv=foo;preserve-gv=j;preserve-gv=bar>' -S | FileCheck --check-prefix=FOO_J_AND_BAR %s
-
 ; ALL: @i = internal global
 ; FOO_AND_J: @i = internal global
 ; FOO_AND_BAR: @i = internal global
-- 
GitLab


From f97f039e0bb7bb60c9cc437f678059c5ee19c8da Mon Sep 17 00:00:00 2001
From: klensy <klensy@users.noreply.github.com>
Date: Thu, 16 May 2024 01:11:14 +0300
Subject: [PATCH 0921/1206] [lld,test] Fix few FileCheck annotation typos
 (#92238)

---
 lld/test/MachO/install-name.s  | 2 +-
 lld/test/MachO/objc-methname.s | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lld/test/MachO/install-name.s b/lld/test/MachO/install-name.s
index 1cf675e278bf..c419c6ca1f95 100644
--- a/lld/test/MachO/install-name.s
+++ b/lld/test/MachO/install-name.s
@@ -31,7 +31,7 @@
 
 # ID:          cmd LC_ID_DYLIB
 # ID-NEXT: cmdsize
-# LID-NEXT:   name foo
+# ID-NEXT:   name foo
 
 .globl _main
 _main:
diff --git a/lld/test/MachO/objc-methname.s b/lld/test/MachO/objc-methname.s
index afc137eac8c2..3d06472971c8 100644
--- a/lld/test/MachO/objc-methname.s
+++ b/lld/test/MachO/objc-methname.s
@@ -16,7 +16,7 @@
 
 # CSTRING: Contents of (__TEXT,__cstring) section
 # CSTRING-NEXT: existing-cstring
-# CSTIRNG-EMPTY:
+# CSTRING-EMPTY:
 
 # METHNAME: Contents of (__TEXT,__objc_methname) section
 # METHNAME-NEXT: existing_methname
-- 
GitLab


From 00179e92c147e16de1f7c653f88c8805aef820c1 Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Wed, 15 May 2024 15:23:29 -0700
Subject: [PATCH 0922/1206] [bazel] Add new dependencies (#92323)

This also fixes building ... on Linux. Seems like target_compatible_with
isn't enough but you also need a manual tag.
---
 utils/bazel/llvm-project-overlay/lldb/BUILD.bazel           | 5 ++++-
 utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index c6fc4e08aa72..ddcaea5184d4 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -916,7 +916,10 @@ apple_genrule(
     srcs = [":debugserver_unsigned"],
     outs = ["debugserver"],
     cmd = "cp $(SRCS) $(OUTS) && xcrun codesign -f -s - --entitlements $(location tools/debugserver/resources/debugserver-macosx-entitlements.plist) $(OUTS)",
-    tags = ["nobuildkite"],
+    tags = [
+        "manual",
+        "nobuildkite",
+    ],
     target_compatible_with = select({
         "@platforms//os:macos": [],
         "//conditions:default": ["@platforms//:incompatible"],
diff --git a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
index 21f0c7092f32..b44489e213a4 100644
--- a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
@@ -617,6 +617,7 @@ cc_test(
         allow_empty = False,
     ),
     deps = [
+        "//llvm:BitstreamReader",
         "//llvm:Core",
         "//llvm:Coverage",
         "//llvm:DebugInfo",
-- 
GitLab


From 050593fc4f9a7f2b9450ee093c4638b8539315b7 Mon Sep 17 00:00:00 2001
From: Andrey Ali Khan Bolshakov <bolsh.andrey@yandex.ru>
Date: Thu, 16 May 2024 01:39:12 +0300
Subject: [PATCH 0923/1206] [Coverage] Handle `CoroutineSuspendExpr` correctly
 (#88898)

This avoids visiting `co_await` or `co_yield` operand 5 times (it is
repeated under transformed awaiter subexpression, and under
`await_ready`, `await_suspend`, and `await_resume` generated call
subexpressions).
---
 clang/lib/CodeGen/CoverageMappingGen.cpp |  4 ++++
 clang/test/CoverageMapping/coroutine.cpp | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp
index ce2f39aeb082..e46560029ab0 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -1439,6 +1439,10 @@ struct CounterCoverageMappingBuilder
     terminateRegion(S);
   }
 
+  void VisitCoroutineSuspendExpr(const CoroutineSuspendExpr *E) {
+    Visit(E->getOperand());
+  }
+
   void VisitCXXThrowExpr(const CXXThrowExpr *E) {
     extendRegion(E);
     if (E->getSubExpr())
diff --git a/clang/test/CoverageMapping/coroutine.cpp b/clang/test/CoverageMapping/coroutine.cpp
index 0105005d198a..d322bc351a72 100644
--- a/clang/test/CoverageMapping/coroutine.cpp
+++ b/clang/test/CoverageMapping/coroutine.cpp
@@ -32,6 +32,7 @@ struct std::coroutine_traits<int, int> {
     suspend_always final_suspend() noexcept;
     void unhandled_exception() noexcept;
     void return_value(int);
+    suspend_always yield_value(int);
   };
 };
 
@@ -45,3 +46,21 @@ int f1(int x) {       // CHECK-NEXT: File 0, [[@LINE]]:15 -> [[@LINE+8]]:2 = #0
   }                   // CHECK-NEXT: File 0, [[@LINE-2]]:10 -> [[@LINE]]:4 = (#0 - #1)
   co_return x;        // CHECK-NEXT: Gap,File 0, [[@LINE-1]]:4 -> [[@LINE]]:3 = #1
 } // CHECK-NEXT: File 0, [[@LINE-1]]:3 -> [[@LINE-1]]:14 = #1
+
+// CHECK-LABEL: _Z2f2i:
+// CHECK-NEXT: File 0, [[@LINE+1]]:15 -> [[@LINE+15]]:2 = #0
+int f2(int x) {
+// CHECK-NEXT: File 0, [[@LINE+5]]:13 -> [[@LINE+5]]:18 = #0
+// CHECK-NEXT: Branch,File 0, [[@LINE+4]]:13 -> [[@LINE+4]]:18 = #1, (#0 - #1)
+// CHECK-NEXT: Gap,File 0, [[@LINE+3]]:20 -> [[@LINE+3]]:21 = #1
+// CHECK-NEXT: File 0, [[@LINE+2]]:21 -> [[@LINE+2]]:37 = #1
+// CHECK-NEXT: File 0, [[@LINE+1]]:40 -> [[@LINE+1]]:56 = (#0 - #1)
+  co_await (x > 0 ? suspend_always{} : suspend_always{});
+// CHECK-NEXT: File 0, [[@LINE+5]]:12 -> [[@LINE+5]]:17 = #0
+// CHECK-NEXT: Branch,File 0, [[@LINE+4]]:12 -> [[@LINE+4]]:17 = #2, (#0 - #2)
+// CHECK-NEXT: Gap,File 0, [[@LINE+3]]:19 -> [[@LINE+3]]:20 = #2
+// CHECK-NEXT: File 0, [[@LINE+2]]:20 -> [[@LINE+2]]:21 = #2
+// CHECK-NEXT: File 0, [[@LINE+1]]:24 -> [[@LINE+1]]:25 = (#0 - #2)
+  co_yield x > 0 ? 1 : 2;
+  co_return 0;
+}
-- 
GitLab


From 5ff6c6542ac451daaed6c417e481e313165d3454 Mon Sep 17 00:00:00 2001
From: Andrey Ali Khan Bolshakov <bolsh.andrey@yandex.ru>
Date: Thu, 16 May 2024 01:40:03 +0300
Subject: [PATCH 0924/1206] [Coverage] Handle array decomposition correctly
 (#88881)

`ArrayInitLoopExpr` AST node has two occurences of its as-written
initializing expression in its subexpressions through a non-unique
`OpaqueValueExpr`. It causes double-visiting of the initializing
expression if not handled explicitly, as discussed in #85837.
---
 clang/lib/CodeGen/CoverageMappingGen.cpp     |  4 ++++
 clang/test/CoverageMapping/decomposition.cpp | 15 +++++++++++++++
 2 files changed, 19 insertions(+)
 create mode 100644 clang/test/CoverageMapping/decomposition.cpp

diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp
index e46560029ab0..cc8ab7a5b436 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -2177,6 +2177,10 @@ struct CounterCoverageMappingBuilder
     // propagate counts into them.
   }
 
+  void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *AILE) {
+    Visit(AILE->getCommonExpr()->getSourceExpr());
+  }
+
   void VisitPseudoObjectExpr(const PseudoObjectExpr *POE) {
     // Just visit syntatic expression as this is what users actually write.
     VisitStmt(POE->getSyntacticForm());
diff --git a/clang/test/CoverageMapping/decomposition.cpp b/clang/test/CoverageMapping/decomposition.cpp
new file mode 100644
index 000000000000..601ea630faee
--- /dev/null
+++ b/clang/test/CoverageMapping/decomposition.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -mllvm -emptyline-comment-coverage=false -triple %itanium_abi_triple -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only %s | FileCheck %s
+
+// CHECK-LABEL:       _Z19array_decompositioni:
+// CHECK-NEXT:          File 0, [[@LINE+6]]:32 -> {{[0-9]+}}:2 = #0
+// CHECK-NEXT:          File 0, [[@LINE+8]]:20 -> [[@LINE+8]]:25 = #0
+// CHECK-NEXT:          Branch,File 0, [[@LINE+7]]:20 -> [[@LINE+7]]:25 = #1, (#0 - #1)
+// CHECK-NEXT:          Gap,File 0, [[@LINE+6]]:27 -> [[@LINE+6]]:28 = #1
+// CHECK-NEXT:          File 0, [[@LINE+5]]:28 -> [[@LINE+5]]:29 = #1
+// CHECK-NEXT:          File 0, [[@LINE+4]]:32 -> [[@LINE+4]]:33 = (#0 - #1)
+int array_decomposition(int i) {
+  int a[] = {1, 2, 3};
+  int b[] = {4, 5, 6};
+  auto [x, y, z] = i > 0 ? a : b;
+  return x + y + z;
+}
-- 
GitLab


From aa889d7783af050ce5d19af67c7225ee119d625e Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Wed, 15 May 2024 15:41:20 -0700
Subject: [PATCH 0925/1206] [ELF,test] Fix FileCheck prefixes

Most violations are stale and should be removed while a few can be
adjusted.

Reported at #92238
---
 lld/test/ELF/arm-exidx-shared.s           |  2 +-
 lld/test/ELF/mips-tls-hilo.s              | 10 ----------
 lld/test/ELF/ppc32-reloc-rel.s            |  3 ++-
 lld/test/ELF/ppc64-pcrel-call-to-extern.s |  5 -----
 lld/test/ELF/ppc64-toc-relax-ifunc.s      | 13 +++++--------
 lld/test/ELF/riscv-gp.s                   |  4 ----
 6 files changed, 8 insertions(+), 29 deletions(-)

diff --git a/lld/test/ELF/arm-exidx-shared.s b/lld/test/ELF/arm-exidx-shared.s
index fce605d6d96a..2e484e5c065f 100644
--- a/lld/test/ELF/arm-exidx-shared.s
+++ b/lld/test/ELF/arm-exidx-shared.s
@@ -2,7 +2,7 @@
 // RUN: llvm-mc -filetype=obj -arm-add-build-attributes -triple=armv7a-none-linux-gnueabi %s -o %t
 // RUN: ld.lld --hash-style=sysv %t --shared -o %t2
 // RUN: llvm-readobj --relocations %t2 | FileCheck %s
-// RUN: llvm-objdump -s --triple=armv7a-none-linux-gnueabi %t2 | FileCheck --check-prefix=CHECK-EXTAB-NEXT %s
+// RUN: llvm-objdump -s --triple=armv7a-none-linux-gnueabi %t2 | FileCheck --check-prefix=CHECK-EXTAB %s
 
 // Check that the relative R_ARM_PREL31 relocation can access a PLT entry
 // for when the personality routine is referenced from a shared library.
diff --git a/lld/test/ELF/mips-tls-hilo.s b/lld/test/ELF/mips-tls-hilo.s
index 6fd2033aac41..9c67f9fe14ba 100644
--- a/lld/test/ELF/mips-tls-hilo.s
+++ b/lld/test/ELF/mips-tls-hilo.s
@@ -28,16 +28,6 @@
 # CHECK-NEXT: ]
 # CHECK-NOT:  Primary GOT
 
-# SO:      Relocations [
-# SO-NEXT: ]
-# SO:      Primary GOT {
-# SO:        Local entries [
-# SO-NEXT:   ]
-# SO-NEXT:   Global entries [
-# SO-NEXT:   ]
-# SO-NEXT:   Number of TLS and multi-GOT entries: 0
-# SO-NEXT: }
-
   .text
   .globl  __start
   .type __start,@function
diff --git a/lld/test/ELF/ppc32-reloc-rel.s b/lld/test/ELF/ppc32-reloc-rel.s
index b89e0b43cb78..d13ebdb7997f 100644
--- a/lld/test/ELF/ppc32-reloc-rel.s
+++ b/lld/test/ELF/ppc32-reloc-rel.s
@@ -6,6 +6,7 @@
 # RUN: llvm-mc -filetype=obj -triple=powerpcle %s -o %t.le.o
 # RUN: ld.lld %t.le.o -o %t
 # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck %s
+# RUN: llvm-objdump -s %t | FileCheck %s --check-prefix=HEX
 
 .section .R_PPC_REL14,"ax",@progbits
   beq 1f
@@ -23,7 +24,7 @@
   .long 1f - .
 1:
 # HEX-LABEL: section .R_PPC_REL32:
-# HEX-NEXT: 10010008 00000004
+# HEX-NEXT:  04000000
 
 .section .R_PPC_PLTREL24,"ax",@progbits
   b 1f@PLT+32768
diff --git a/lld/test/ELF/ppc64-pcrel-call-to-extern.s b/lld/test/ELF/ppc64-pcrel-call-to-extern.s
index e5846e80ce23..de05b733e175 100644
--- a/lld/test/ELF/ppc64-pcrel-call-to-extern.s
+++ b/lld/test/ELF/ppc64-pcrel-call-to-extern.s
@@ -73,9 +73,7 @@
 
 ## DT_PLTGOT points to .plt
 # SEC: .plt              NOBITS          0000000010030168 040168 000028 00  WA  0   0  8
-# SEC-OG: .plt              NOBITS          0000000010030158 040158 000028 00  WA  0   0  8
 # SEC: 0x0000000000000003 (PLTGOT)      0x10030168
-# SEC-OG: 0x0000000000000003 (PLTGOT)      0x10030168
 
 ## DT_PLTGOT points to .plt
 # SEC-NOP10: .plt              NOBITS          0000000010030168 040168 000028 00  WA  0   0  8
@@ -86,11 +84,8 @@
 ## Check that we emit 3 R_PPC64_JMP_SLOT in .rela.plt.
 # REL:      .rela.plt {
 # REL-NEXT:   0x10030178 R_PPC64_JMP_SLOT callee_global_stother0 0x0
-# REL-NEXT-OG:   0x10030168 R_PPC64_JMP_SLOT callee_global_stother0 0x0
 # REL-NEXT:   0x10030180 R_PPC64_JMP_SLOT callee_global_stother1 0x0
-# REL-NEXT-OG:   0x10030170 R_PPC64_JMP_SLOT callee_global_stother1 0x0
 # REL-NEXT:   0x10030188 R_PPC64_JMP_SLOT callee_global_TOC 0x0
-# REL-NEXT-OG:   0x10030178 R_PPC64_JMP_SLOT callee_global_TOC 0x0
 # REL-NEXT: }
 
 # REL-NOP10:      .rela.plt {
diff --git a/lld/test/ELF/ppc64-toc-relax-ifunc.s b/lld/test/ELF/ppc64-toc-relax-ifunc.s
index 9fb1bf0023b6..00a63c7e5b67 100644
--- a/lld/test/ELF/ppc64-toc-relax-ifunc.s
+++ b/lld/test/ELF/ppc64-toc-relax-ifunc.s
@@ -4,7 +4,7 @@
 # RUN: echo '.globl ifunc; .type ifunc, %gnu_indirect_function; ifunc:' | \
 # RUN:   llvm-mc -filetype=obj -triple=powerpc64le - -o %t1.o
 # RUN: ld.lld %t.o %t1.o -o %t
-# RUN: llvm-readelf -S -s %t | FileCheck --check-prefix=SEC %s
+# RUN: llvm-readelf -Ssr %t | FileCheck --check-prefix=SEC %s
 # RUN: llvm-readelf -x .toc %t | FileCheck --check-prefix=HEX %s
 # RUN: llvm-objdump -d %t | FileCheck --check-prefix=DIS %s
 
@@ -13,18 +13,15 @@
 ## still perform toc-indirect to toc-relative relaxation because the distance
 ## to the address of the canonical PLT is fixed.
 
-# SEC: .text PROGBITS 00000000100101e0
-# SEC: .plt  NOBITS   0000000010030200
-# SEC: 00000000100101e8 0 FUNC GLOBAL DEFAULT 3 ifunc
+# SEC: .text PROGBITS [[#%x,TEXT:]]
+# SEC: .plt  NOBITS   [[#%x,PLT:]]
+# SEC: {{0*}}[[#PLT]] {{.+}} R_PPC64_IRELATIVE [[#TEXT+8]]
+# SEC: {{0*}}[[#TEXT+8]] 0 FUNC GLOBAL DEFAULT 3 ifunc
 
 ## .toc[0] stores the address of the canonical PLT.
 # HEX:      section '.toc':
 # HEX-NEXT: 0x100201f8 e8010110 00000000
 
-# REL:      .rela.dyn {
-# REL-NEXT:   0x100301f8 R_PPC64_IRELATIVE - 0x100101e8
-# REL-NEXT: }
-
 # DIS: addi 3, 3,
 
 addis 3, 2, .toc@toc@ha
diff --git a/lld/test/ELF/riscv-gp.s b/lld/test/ELF/riscv-gp.s
index 29411d19b019..e82e36ee9a7a 100644
--- a/lld/test/ELF/riscv-gp.s
+++ b/lld/test/ELF/riscv-gp.s
@@ -16,10 +16,6 @@
 # SEC64: [ [[#SDATA:]]] .sdata PROGBITS {{0*}}000032e0
 # SEC64: {{0*}}00003ae0 0 NOTYPE GLOBAL DEFAULT [[#SDATA]] __global_pointer$
 
-## __global_pointer$ - 0x1000 = 4096*3-2048
-# DIS:      1000: auipc gp, 3
-# DIS-NEXT:       addi gp, gp, -2048
-
 # ERR: error: relocation R_RISCV_PCREL_HI20 cannot be used against symbol '__global_pointer$'; recompile with -fPIC
 
 ## -r mode does not define __global_pointer$.
-- 
GitLab


From 0585eed9409c1362f7deaabc42c1d3c3f55c4b6c Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Wed, 15 May 2024 15:44:05 -0700
Subject: [PATCH 0926/1206] [lldb-dap] Support publishing to the VSCode market
 place (#92320)

Update the publisher and add a publish script that we can use from
Github actions.
---
 lldb/tools/lldb-dap/package.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lldb/tools/lldb-dap/package.json b/lldb/tools/lldb-dap/package.json
index 2e8ad074256b..aeb24445551c 100644
--- a/lldb/tools/lldb-dap/package.json
+++ b/lldb/tools/lldb-dap/package.json
@@ -2,7 +2,7 @@
   "name": "lldb-dap",
   "displayName": "LLDB DAP",
   "version": "0.2.0",
-  "publisher": "llvm",
+  "publisher": "llvm-vs-code-extensions",
   "homepage": "https://lldb.llvm.org",
   "description": "LLDB debugging from VSCode",
   "license": "Apache 2.0 License with LLVM exceptions",
@@ -42,6 +42,7 @@
     "watch": "tsc -watch -p ./",
     "format": "npx prettier './src-ts/' --write",
     "package": "vsce package --out ./out/lldb-dap.vsix",
+    "publish": "vsce publish",
     "vscode-uninstall": "code --uninstall-extension llvm.lldb-dap",
     "vscode-install": "code --install-extension ./out/lldb-dap.vsix"
   },
-- 
GitLab


From e00a3ccf43563209b71c5b68f56d83f4052dca63 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Wed, 15 May 2024 15:44:37 -0700
Subject: [PATCH 0927/1206] [flang] New -fdebug-unparse-with-modules option
 (#91660)

This option is a compilation action that parses a source file and
performs semantic analysis on it, like the existing -fdebug-unparse
option does. Its output, however, is preceded by the effective contents
of all of the non-intrinsic modules on which it depends but does not
define, transitively preceded by the closure of all of those modules'
dependencies.

The output from this option is therefore the analyzed parse tree for a
source file encapsulated with all of its non-intrinsic module
dependencies. This output may be useful for extracting code from large
applications for use as an attachment to a bug report, or as input to a
test case reduction tool for problem isolation.
---
 clang/include/clang/Driver/Options.td         |  4 +-
 .../include/flang/Frontend/FrontendActions.h  |  4 ++
 .../include/flang/Frontend/FrontendOptions.h  |  4 ++
 .../flang/Semantics/unparse-with-symbols.h    |  4 ++
 flang/lib/Frontend/CompilerInvocation.cpp     |  3 ++
 flang/lib/Frontend/FrontendActions.cpp        |  9 +++++
 .../ExecuteCompilerInvocation.cpp             |  2 +
 flang/lib/Semantics/mod-file.cpp              | 20 +++++++++-
 flang/lib/Semantics/mod-file.h                |  3 ++
 flang/lib/Semantics/unparse-with-symbols.cpp  | 38 +++++++++++++++++++
 flang/test/Driver/unparse-with-modules.f90    | 34 +++++++++++++++++
 11 files changed, 122 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Driver/unparse-with-modules.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index c54eb543d658..e579f1a0a366 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -6647,7 +6647,9 @@ def fdebug_unparse : Flag<["-"], "fdebug-unparse">, Group<Action_Group>,
   DocBrief<[{Run the parser and the semantic checks. Then unparse the
 parse-tree and output the generated Fortran source file.}]>;
 def fdebug_unparse_with_symbols : Flag<["-"], "fdebug-unparse-with-symbols">, Group<Action_Group>,
-  HelpText<"Unparse and stop.">;
+  HelpText<"Unparse with symbols and stop.">;
+def fdebug_unparse_with_modules : Flag<["-"], "fdebug-unparse-with-modules">, Group<Action_Group>,
+  HelpText<"Unparse with dependent modules and stop.">;
 def fdebug_dump_symbols : Flag<["-"], "fdebug-dump-symbols">, Group<Action_Group>,
   HelpText<"Dump symbols after the semantic analysis">;
 def fdebug_dump_parse_tree : Flag<["-"], "fdebug-dump-parse-tree">, Group<Action_Group>,
diff --git a/flang/include/flang/Frontend/FrontendActions.h b/flang/include/flang/Frontend/FrontendActions.h
index e2e859f3a81b..7823565eb815 100644
--- a/flang/include/flang/Frontend/FrontendActions.h
+++ b/flang/include/flang/Frontend/FrontendActions.h
@@ -108,6 +108,10 @@ class DebugUnparseWithSymbolsAction : public PrescanAndSemaAction {
   void executeAction() override;
 };
 
+class DebugUnparseWithModulesAction : public PrescanAndSemaAction {
+  void executeAction() override;
+};
+
 class DebugUnparseAction : public PrescanAndSemaAction {
   void executeAction() override;
 };
diff --git a/flang/include/flang/Frontend/FrontendOptions.h b/flang/include/flang/Frontend/FrontendOptions.h
index 06b1318f243b..82ca99672ec6 100644
--- a/flang/include/flang/Frontend/FrontendOptions.h
+++ b/flang/include/flang/Frontend/FrontendOptions.h
@@ -63,6 +63,10 @@ enum ActionKind {
   /// Fortran source file
   DebugUnparseWithSymbols,
 
+  /// Parse, run semantics, and output a Fortran source file preceded
+  /// by all the necessary modules (transitively)
+  DebugUnparseWithModules,
+
   /// Parse, run semantics and then output symbols from semantics
   DebugDumpSymbols,
 
diff --git a/flang/include/flang/Semantics/unparse-with-symbols.h b/flang/include/flang/Semantics/unparse-with-symbols.h
index d70110245e2b..5e18b3fc3063 100644
--- a/flang/include/flang/Semantics/unparse-with-symbols.h
+++ b/flang/include/flang/Semantics/unparse-with-symbols.h
@@ -21,8 +21,12 @@ struct Program;
 }
 
 namespace Fortran::semantics {
+class SemanticsContext;
 void UnparseWithSymbols(llvm::raw_ostream &, const parser::Program &,
     parser::Encoding encoding = parser::Encoding::UTF_8);
+void UnparseWithModules(llvm::raw_ostream &, SemanticsContext &,
+    const parser::Program &,
+    parser::Encoding encoding = parser::Encoding::UTF_8);
 }
 
 #endif // FORTRAN_SEMANTICS_UNPARSE_WITH_SYMBOLS_H_
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index db7fd3cccc7a..e8a8c90045d9 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -488,6 +488,9 @@ static bool parseFrontendArgs(FrontendOptions &opts, llvm::opt::ArgList &args,
     case clang::driver::options::OPT_fdebug_unparse_with_symbols:
       opts.programAction = DebugUnparseWithSymbols;
       break;
+    case clang::driver::options::OPT_fdebug_unparse_with_modules:
+      opts.programAction = DebugUnparseWithModules;
+      break;
     case clang::driver::options::OPT_fdebug_dump_symbols:
       opts.programAction = DebugDumpSymbols;
       break;
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index 2f65ab6102f4..4341c104a69d 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -477,6 +477,15 @@ void DebugUnparseWithSymbolsAction::executeAction() {
   reportFatalSemanticErrors();
 }
 
+void DebugUnparseWithModulesAction::executeAction() {
+  auto &parseTree{*getInstance().getParsing().parseTree()};
+  CompilerInstance &ci{getInstance()};
+  Fortran::semantics::UnparseWithModules(
+      llvm::outs(), ci.getSemantics().context(), parseTree,
+      /*encoding=*/Fortran::parser::Encoding::UTF_8);
+  reportFatalSemanticErrors();
+}
+
 void DebugDumpSymbolsAction::executeAction() {
   CompilerInstance &ci = this->getInstance();
 
diff --git a/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index 4cad640562c6..e2cbd5112d6e 100644
--- a/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -59,6 +59,8 @@ createFrontendAction(CompilerInstance &ci) {
     return std::make_unique<DebugUnparseNoSemaAction>();
   case DebugUnparseWithSymbols:
     return std::make_unique<DebugUnparseWithSymbolsAction>();
+  case DebugUnparseWithModules:
+    return std::make_unique<DebugUnparseWithModulesAction>();
   case DebugDumpSymbols:
     return std::make_unique<DebugDumpSymbolsAction>();
   case DebugDumpParseTree:
diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp
index e9aebe5b08f2..bb8c6c7567b8 100644
--- a/flang/lib/Semantics/mod-file.cpp
+++ b/flang/lib/Semantics/mod-file.cpp
@@ -132,11 +132,11 @@ static std::string ModFileName(const SourceName &name,
 
 // Write the module file for symbol, which must be a module or submodule.
 void ModFileWriter::Write(const Symbol &symbol) {
-  auto &module{symbol.get<ModuleDetails>()};
+  const auto &module{symbol.get<ModuleDetails>()};
   if (module.moduleFileHash()) {
     return; // already written
   }
-  auto *ancestor{module.ancestor()};
+  const auto *ancestor{module.ancestor()};
   isSubmodule_ = ancestor != nullptr;
   auto ancestorName{ancestor ? ancestor->GetName().value().ToString() : ""s};
   auto path{context_.moduleDirectory() + '/' +
@@ -151,6 +151,21 @@ void ModFileWriter::Write(const Symbol &symbol) {
   const_cast<ModuleDetails &>(module).set_moduleFileHash(checkSum);
 }
 
+void ModFileWriter::WriteClosure(llvm::raw_ostream &out, const Symbol &symbol,
+    UnorderedSymbolSet &nonIntrinsicModulesWritten) {
+  if (!symbol.has<ModuleDetails>() || symbol.owner().IsIntrinsicModules() ||
+      !nonIntrinsicModulesWritten.insert(symbol).second) {
+    return;
+  }
+  PutSymbols(DEREF(symbol.scope()));
+  needsBuf_.clear(); // omit module checksums
+  auto str{GetAsString(symbol)};
+  for (auto depRef : std::move(usedNonIntrinsicModules_)) {
+    WriteClosure(out, *depRef, nonIntrinsicModulesWritten);
+  }
+  out << std::move(str);
+}
+
 // Return the entire body of the module file
 // and clear saved uses, decls, and contains.
 std::string ModFileWriter::GetAsString(const Symbol &symbol) {
@@ -710,6 +725,7 @@ void ModFileWriter::PutUse(const Symbol &symbol) {
     uses_ << "use,intrinsic::";
   } else {
     uses_ << "use ";
+    usedNonIntrinsicModules_.insert(module);
   }
   uses_ << module.name() << ",only:";
   PutGenericName(uses_, symbol);
diff --git a/flang/lib/Semantics/mod-file.h b/flang/lib/Semantics/mod-file.h
index b4ece4018c05..739add32c2e0 100644
--- a/flang/lib/Semantics/mod-file.h
+++ b/flang/lib/Semantics/mod-file.h
@@ -35,6 +35,8 @@ class ModFileWriter {
 public:
   explicit ModFileWriter(SemanticsContext &context) : context_{context} {}
   bool WriteAll();
+  void WriteClosure(llvm::raw_ostream &, const Symbol &,
+      UnorderedSymbolSet &nonIntrinsicModulesWritten);
 
 private:
   SemanticsContext &context_;
@@ -46,6 +48,7 @@ private:
   std::string containsBuf_;
   // Tracks nested DEC structures and fields of that type
   UnorderedSymbolSet emittedDECStructures_, emittedDECFields_;
+  UnorderedSymbolSet usedNonIntrinsicModules_;
 
   llvm::raw_string_ostream needs_{needsBuf_};
   llvm::raw_string_ostream uses_{usesBuf_};
diff --git a/flang/lib/Semantics/unparse-with-symbols.cpp b/flang/lib/Semantics/unparse-with-symbols.cpp
index 67016e85777c..c451f885c062 100644
--- a/flang/lib/Semantics/unparse-with-symbols.cpp
+++ b/flang/lib/Semantics/unparse-with-symbols.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Semantics/unparse-with-symbols.h"
+#include "mod-file.h"
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Parser/unparse.h"
@@ -98,4 +99,41 @@ void UnparseWithSymbols(llvm::raw_ostream &out, const parser::Program &program,
           int indent) { visitor.PrintSymbols(location, out, indent); }};
   parser::Unparse(out, program, encoding, false, true, &preStatement);
 }
+
+// UnparseWithModules()
+
+class UsedModuleVisitor {
+public:
+  UnorderedSymbolSet &modulesUsed() { return modulesUsed_; }
+  UnorderedSymbolSet &modulesDefined() { return modulesDefined_; }
+  template <typename T> bool Pre(const T &) { return true; }
+  template <typename T> void Post(const T &) {}
+  void Post(const parser::ModuleStmt &module) {
+    if (module.v.symbol) {
+      modulesDefined_.insert(*module.v.symbol);
+    }
+  }
+  void Post(const parser::UseStmt &use) {
+    if (use.moduleName.symbol) {
+      modulesUsed_.insert(*use.moduleName.symbol);
+    }
+  }
+
+private:
+  UnorderedSymbolSet modulesUsed_;
+  UnorderedSymbolSet modulesDefined_;
+};
+
+void UnparseWithModules(llvm::raw_ostream &out, SemanticsContext &context,
+    const parser::Program &program, parser::Encoding encoding) {
+  UsedModuleVisitor visitor;
+  parser::Walk(program, visitor);
+  UnorderedSymbolSet nonIntrinsicModulesWritten{
+      std::move(visitor.modulesDefined())};
+  ModFileWriter writer{context};
+  for (SymbolRef moduleRef : visitor.modulesUsed()) {
+    writer.WriteClosure(out, *moduleRef, nonIntrinsicModulesWritten);
+  }
+  parser::Unparse(out, program, encoding, false, true);
+}
 } // namespace Fortran::semantics
diff --git a/flang/test/Driver/unparse-with-modules.f90 b/flang/test/Driver/unparse-with-modules.f90
new file mode 100644
index 000000000000..53997f7804ef
--- /dev/null
+++ b/flang/test/Driver/unparse-with-modules.f90
@@ -0,0 +1,34 @@
+! RUN: %flang_fc1 -I %S/Inputs/module-dir -fdebug-unparse-with-modules %s | FileCheck %s
+module m1
+  use iso_fortran_env
+  use BasicTestModuleTwo
+  implicit none
+  type(t2) y
+  real(real32) x
+end
+
+program test
+  use m1
+  use BasicTestModuleTwo
+  implicit none
+  x = 123.
+  y = t2()
+end
+
+!CHECK-NOT: module iso_fortran_env
+!CHECK: module basictestmoduletwo
+!CHECK: type::t2
+!CHECK: end type
+!CHECK: end
+!CHECK: module m1
+!CHECK:  use :: iso_fortran_env
+!CHECK:  implicit none
+!CHECK:  real(kind=real32) x
+!CHECK: end module
+!CHECK: program test
+!CHECK:  use :: m1
+!CHECK:  use :: basictestmoduletwo
+!CHECK:  implicit none
+!CHECK:  x = 123.
+!CHECK:  y = t2()
+!CHECK: end program
-- 
GitLab


From 667d12f86e626173726e87e101626a9060b8d967 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Wed, 15 May 2024 18:55:53 -0400
Subject: [PATCH 0928/1206] [Clang][Sema] Do not mark template parameters in
 the exception specification as used during partial ordering (#91534)

We do not deduce template arguments from the exception specification
when determining the primary template of a function template
specialization or when taking the address of a function template.
Therefore, this patch changes `isAtLeastAsSpecializedAs` such that we do
not mark template parameters in the exception specification as 'used'
during partial ordering (per [temp.deduct.partial]
p12) to prevent the following from being ambiguous:

```
template<typename T, typename U>
void f(U) noexcept(noexcept(T())); // #1

template<typename T>
void f(T*) noexcept; // #2

template<>
void f<int>(int*) noexcept; // currently ambiguous, selects #2 with this patch applied
```

Although there is no corresponding wording in the standard (see core issue filed here
https://github.com/cplusplus/CWG/issues/537), this seems
to be the intended behavior given the definition of _deduction
substitution loci_ in [temp.deduct.general] p7 (and EDG does the same thing).
---
 clang/docs/ReleaseNotes.rst                   |  3 +
 clang/lib/Sema/SemaTemplateDeduction.cpp      | 36 +++++++---
 .../temp.deduct/temp.deduct.partial/p3.cpp    | 72 +++++++++++++++++++
 3 files changed, 103 insertions(+), 8 deletions(-)
 create mode 100644 clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.partial/p3.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index ae699ebfc603..6f7e54252150 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -713,6 +713,9 @@ Bug Fixes to C++ Support
 - Correctly treat the compound statement of an ``if consteval`` as an immediate context. Fixes (#GH91509).
 - When partial ordering alias templates against template template parameters,
   allow pack expansions when the alias has a fixed-size parameter list. Fixes (#GH62529).
+- Clang now ignores template parameters only used within the exception specification of candidate function
+  templates during partial ordering when deducing template arguments from a function declaration or when
+  taking the address of a function template.
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 853c0e1b5061..b5d405111fe4 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -5453,7 +5453,7 @@ static bool isAtLeastAsSpecializedAs(Sema &S, SourceLocation Loc,
     //     is used.
     if (DeduceTemplateArgumentsByTypeMatch(
             S, TemplateParams, FD2->getType(), FD1->getType(), Info, Deduced,
-            TDF_None,
+            TDF_AllowCompatibleFunctionType,
             /*PartialOrdering=*/true) != TemplateDeductionResult::Success)
       return false;
     break;
@@ -5485,20 +5485,40 @@ static bool isAtLeastAsSpecializedAs(Sema &S, SourceLocation Loc,
   switch (TPOC) {
   case TPOC_Call:
     for (unsigned I = 0, N = Args2.size(); I != N; ++I)
-      ::MarkUsedTemplateParameters(S.Context, Args2[I], false,
-                                   TemplateParams->getDepth(),
-                                   UsedParameters);
+      ::MarkUsedTemplateParameters(S.Context, Args2[I], /*OnlyDeduced=*/false,
+                                   TemplateParams->getDepth(), UsedParameters);
     break;
 
   case TPOC_Conversion:
-    ::MarkUsedTemplateParameters(S.Context, Proto2->getReturnType(), false,
+    ::MarkUsedTemplateParameters(S.Context, Proto2->getReturnType(),
+                                 /*OnlyDeduced=*/false,
                                  TemplateParams->getDepth(), UsedParameters);
     break;
 
   case TPOC_Other:
-    ::MarkUsedTemplateParameters(S.Context, FD2->getType(), false,
-                                 TemplateParams->getDepth(),
-                                 UsedParameters);
+    // We do not deduce template arguments from the exception specification
+    // when determining the primary template of a function template
+    // specialization or when taking the address of a function template.
+    // Therefore, we do not mark template parameters in the exception
+    // specification as used during partial ordering to prevent the following
+    // from being ambiguous:
+    //
+    //   template<typename T, typename U>
+    //   void f(U) noexcept(noexcept(T())); // #1
+    //
+    //   template<typename T>
+    //   void f(T*) noexcept; // #2
+    //
+    //   template<>
+    //   void f<int>(int*) noexcept; // explicit specialization of #2
+    //
+    // Although there is no corresponding wording in the standard, this seems
+    // to be the intended behavior given the definition of
+    // 'deduction substitution loci' in [temp.deduct].
+    ::MarkUsedTemplateParameters(
+        S.Context,
+        S.Context.getFunctionTypeWithExceptionSpec(FD2->getType(), EST_None),
+        /*OnlyDeduced=*/false, TemplateParams->getDepth(), UsedParameters);
     break;
   }
 
diff --git a/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.partial/p3.cpp b/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.partial/p3.cpp
new file mode 100644
index 000000000000..cc1d4ecda2ec
--- /dev/null
+++ b/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.partial/p3.cpp
@@ -0,0 +1,72 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+template<bool B>
+struct A { };
+
+constexpr A<false> a;
+constexpr A<false> b;
+
+constexpr int* x = nullptr;
+constexpr short* y = nullptr;
+
+namespace ExplicitArgs {
+  template<typename T, typename U>
+  constexpr int f(U) noexcept(noexcept(T())) {
+    return 0;
+  }
+
+  template<typename T>
+  constexpr int f(T*) noexcept {
+    return 1;
+  }
+
+  template<>
+  constexpr int f<int>(int*) noexcept {
+    return 2;
+  }
+
+  static_assert(f<int>(1) == 0);
+  static_assert(f<short>(y) == 1);
+  static_assert(f<int>(x) == 2);
+
+  template<typename T, typename U>
+  constexpr int g(U*) noexcept(noexcept(T())) {
+    return 3;
+  }
+
+  template<typename T>
+  constexpr int g(T) noexcept {
+    return 4;
+  }
+
+  template<>
+  constexpr int g<int>(int*) noexcept {
+    return 5;
+  }
+
+  static_assert(g<int>(y) == 3);
+  static_assert(g<short>(1) == 4);
+  static_assert(g<int>(x) == 5);
+} // namespace ExplicitArgs
+
+namespace DeducedArgs {
+  template<typename T, bool B>
+  constexpr int f(T, A<B>) noexcept(B) {
+    return 0;
+  }
+
+  template<typename T, bool B>
+  constexpr int f(T*, A<B>) noexcept(B && B) {
+    return 1;
+  }
+
+  template<>
+  constexpr int f(int*, A<false>) {
+    return 2;
+  }
+
+  static_assert(f<int*>(x, a) == 0);
+  static_assert(f<short>(y, a) == 1);
+  static_assert(f<int>(x, a) == 2);
+} // namespace DeducedArgs
-- 
GitLab


From 325d1d0b73aa6bff0ce4174b45a7601f6b32a793 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Wed, 15 May 2024 15:58:20 -0700
Subject: [PATCH 0929/1206] [flang] Fix purity checking for internal
 subprograms (#91759)

ELEMENTAL internal subprograms are pure unless explicitly IMPURE.
---
 flang/lib/Semantics/check-purity.cpp | 10 +++--
 flang/test/Semantics/pure02.f90      | 59 ++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Semantics/pure02.f90

diff --git a/flang/lib/Semantics/check-purity.cpp b/flang/lib/Semantics/check-purity.cpp
index 5176390f366b..55a9a2f10738 100644
--- a/flang/lib/Semantics/check-purity.cpp
+++ b/flang/lib/Semantics/check-purity.cpp
@@ -39,12 +39,16 @@ bool PurityChecker::InPureSubprogram() const {
 
 bool PurityChecker::HasPurePrefix(
     const std::list<parser::PrefixSpec> &prefixes) const {
+  bool result{false};
   for (const parser::PrefixSpec &prefix : prefixes) {
-    if (std::holds_alternative<parser::PrefixSpec::Pure>(prefix.u)) {
-      return true;
+    if (std::holds_alternative<parser::PrefixSpec::Impure>(prefix.u)) {
+      return false;
+    } else if (std::holds_alternative<parser::PrefixSpec::Pure>(prefix.u) ||
+        std::holds_alternative<parser::PrefixSpec::Elemental>(prefix.u)) {
+      result = true;
     }
   }
-  return false;
+  return result;
 }
 
 void PurityChecker::Entered(
diff --git a/flang/test/Semantics/pure02.f90 b/flang/test/Semantics/pure02.f90
new file mode 100644
index 000000000000..11dc0fd26829
--- /dev/null
+++ b/flang/test/Semantics/pure02.f90
@@ -0,0 +1,59 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+pure subroutine s1
+ contains
+  !ERROR: An internal subprogram of a pure subprogram must also be pure
+  subroutine t1
+  end
+  pure subroutine t2 ! ok
+  end
+  elemental subroutine t3(k) ! ok
+    integer, intent(in) :: k
+  end
+  !ERROR: An internal subprogram of a pure subprogram must also be pure
+  impure elemental subroutine t4(k)
+    integer, intent(in) :: k
+  end
+  !ERROR: An internal subprogram of a pure subprogram must also be pure
+  elemental impure subroutine t5(k)
+    integer, intent(in) :: k
+  end
+end
+
+elemental subroutine s2(j)
+  integer, intent(in) :: j
+ contains
+  !ERROR: An internal subprogram of a pure subprogram must also be pure
+  subroutine t1
+  end
+  pure subroutine t2 ! ok
+  end
+  elemental subroutine t3(k) ! ok
+    integer, intent(in) :: k
+  end
+  !ERROR: An internal subprogram of a pure subprogram must also be pure
+  impure elemental subroutine t4(k)
+    integer, intent(in) :: k
+  end
+  !ERROR: An internal subprogram of a pure subprogram must also be pure
+  elemental impure subroutine t5(k)
+    integer, intent(in) :: k
+  end
+end
+
+impure elemental subroutine s3(j)
+  integer, intent(in) :: j
+ contains
+  subroutine t1
+  end
+  pure subroutine t2
+  end
+  elemental subroutine t3(k)
+    integer, intent(in) :: k
+  end
+  impure elemental subroutine t4(k)
+    integer, intent(in) :: k
+  end
+  elemental impure subroutine t5(k)
+    integer, intent(in) :: k
+  end
+end
-- 
GitLab


From c227bf1b217598066acd32de8c9a75c2e0928f89 Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Wed, 15 May 2024 20:01:17 -0300
Subject: [PATCH 0930/1206] [clang] Create new warning group for deprecation of
 '-fno-relaxed-template-template-args' (#92324)

This allows the warning to be disabled in isolation, as it helps when
treating them as errors.
---
 clang/docs/ReleaseNotes.rst                           |  3 ++-
 clang/include/clang/Basic/DiagnosticDriverKinds.td    |  3 +++
 clang/include/clang/Basic/DiagnosticGroups.td         |  2 ++
 clang/lib/Driver/ToolChains/Clang.cpp                 | 10 +++++++---
 clang/test/Driver/frelaxed-template-template-args.cpp |  4 +++-
 5 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 6f7e54252150..089a85c8cb36 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -51,7 +51,8 @@ C++ Specific Potentially Breaking Changes
 - The behavior controlled by the `-frelaxed-template-template-args` flag is now
   on by default, and the flag is deprecated. Until the flag is finally removed,
   it's negative spelling can be used to obtain compatibility with previous
-  versions of clang.
+  versions of clang. The deprecation warning for the negative spelling can be
+  disabled with `-Wno-deprecated-no-relaxed-template-template-args`.
 
 - Clang now rejects pointer to member from parenthesized expression in unevaluated context such as ``decltype(&(foo::bar))``. (#GH40906).
 
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 9781fcaa4ff5..9d97a75f696f 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -436,6 +436,9 @@ def warn_drv_clang_unsupported : Warning<
   "the clang compiler does not support '%0'">;
 def warn_drv_deprecated_arg : Warning<
   "argument '%0' is deprecated%select{|, use '%2' instead}1">, InGroup<Deprecated>;
+def warn_drv_deprecated_arg_no_relaxed_template_template_args : Warning<
+  "argument '-fno-relaxed-template-template-args' is deprecated">,
+  InGroup<DeprecatedNoRelaxedTemplateTemplateArgs>;
 def warn_drv_deprecated_custom : Warning<
   "argument '%0' is deprecated, %1">, InGroup<Deprecated>;
 def warn_drv_assuming_mfloat_abi_is : Warning<
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 2beb1d45124b..4cb4f3d999f7 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -104,6 +104,7 @@ def EnumConversion : DiagGroup<"enum-conversion",
                                [EnumEnumConversion,
                                 EnumFloatConversion,
                                 EnumCompareConditional]>;
+def DeprecatedNoRelaxedTemplateTemplateArgs : DiagGroup<"deprecated-no-relaxed-template-template-args">;
 def ObjCSignedCharBoolImplicitIntConversion :
   DiagGroup<"objc-signed-char-bool-implicit-int-conversion">;
 def Shorten64To32 : DiagGroup<"shorten-64-to-32">;
@@ -228,6 +229,7 @@ def Deprecated : DiagGroup<"deprecated", [DeprecatedAnonEnumEnumConversion,
                                           DeprecatedLiteralOperator,
                                           DeprecatedPragma,
                                           DeprecatedRegister,
+                                          DeprecatedNoRelaxedTemplateTemplateArgs,
                                           DeprecatedThisCapture,
                                           DeprecatedType,
                                           DeprecatedVolatile,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 42feb1650574..c3e6d563f3bd 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -7253,10 +7253,14 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Arg *A =
           Args.getLastArg(options::OPT_frelaxed_template_template_args,
                           options::OPT_fno_relaxed_template_template_args)) {
-    D.Diag(diag::warn_drv_deprecated_arg)
-        << A->getAsString(Args) << /*hasReplacement=*/false;
-    if (A->getOption().matches(options::OPT_fno_relaxed_template_template_args))
+    if (A->getOption().matches(
+            options::OPT_fno_relaxed_template_template_args)) {
+      D.Diag(diag::warn_drv_deprecated_arg_no_relaxed_template_template_args);
       CmdArgs.push_back("-fno-relaxed-template-template-args");
+    } else {
+      D.Diag(diag::warn_drv_deprecated_arg)
+          << A->getAsString(Args) << /*hasReplacement=*/false;
+    }
   }
 
   // -fsized-deallocation is off by default, as it is an ABI-breaking change for
diff --git a/clang/test/Driver/frelaxed-template-template-args.cpp b/clang/test/Driver/frelaxed-template-template-args.cpp
index 57fc4e3da6e5..7a7fd6f0bbc8 100644
--- a/clang/test/Driver/frelaxed-template-template-args.cpp
+++ b/clang/test/Driver/frelaxed-template-template-args.cpp
@@ -1,7 +1,9 @@
 // RUN: %clang -fsyntax-only -### %s 2>&1 | FileCheck --check-prefix=CHECK-DEF %s
 // RUN: %clang -fsyntax-only -frelaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-ON %s
 // RUN: %clang -fsyntax-only -fno-relaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-OFF %s
+// RUN: %clang -fsyntax-only -fno-relaxed-template-template-args -Wno-deprecated-no-relaxed-template-template-args %s 2>&1 | FileCheck --check-prefix=CHECK-DIS --allow-empty %s
 
 // CHECK-DEF-NOT: "-cc1"{{.*}} "-fno-relaxed-template-template-args"
 // CHECK-ON:  warning: argument '-frelaxed-template-template-args' is deprecated [-Wdeprecated]
-// CHECK-OFF: warning: argument '-fno-relaxed-template-template-args' is deprecated [-Wdeprecated]
+// CHECK-OFF: warning: argument '-fno-relaxed-template-template-args' is deprecated [-Wdeprecated-no-relaxed-template-template-args]
+// CHECK-DIS-NOT: warning: argument '-fno-relaxed-template-template-args' is deprecated
-- 
GitLab


From 7605ad8a2f95e3b37de83e7fb3d320efc74e0ccc Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Wed, 15 May 2024 16:08:06 -0700
Subject: [PATCH 0931/1206] [flang] Always check procedure characterizability
 (#92008)

When a procedure is defined with a subprogram but never referenced in a
compilation unit, it may not be characterized until lowering, and any
errors in characterization then may crash the compiler. So always ensure
that procedure definitions are characterizable in declaration checking.

Fixes https://github.com/llvm/llvm-project/issues/91845.
---
 flang/lib/Semantics/check-declarations.cpp |  9 +++++++++
 flang/lib/Semantics/resolve-names.cpp      |  3 +--
 flang/test/Semantics/entry01.f90           |  2 ++
 flang/test/Semantics/resolve102.f90        | 23 +++++-----------------
 4 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index ce7870b8d54e..8d17989ac279 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -1357,6 +1357,15 @@ bool CheckHelper::IsResultOkToDiffer(const FunctionResult &result) {
 
 void CheckHelper::CheckSubprogram(
     const Symbol &symbol, const SubprogramDetails &details) {
+  // Evaluate a procedure definition's characteristics to flush out
+  // any errors that analysis might expose, in case this subprogram hasn't
+  // had any calls in this compilation unit that would have validated them.
+  if (!context_.HasError(symbol) && !details.isDummy() &&
+      !details.isInterface() && !details.stmtFunction()) {
+    if (!Procedure::Characterize(symbol, foldingContext_)) {
+      context_.SetError(symbol);
+    }
+  }
   if (const Symbol *iface{FindSeparateModuleSubprogramInterface(&symbol)}) {
     SubprogramMatchHelper{*this}.Check(symbol, *iface);
   }
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index e2875081b732..5626f2a8be97 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -5013,8 +5013,7 @@ bool DeclarationVisitor::HasCycle(
     if (procsInCycle.count(*interface) > 0) {
       for (const auto &procInCycle : procsInCycle) {
         Say(procInCycle->name(),
-            "The interface for procedure '%s' is recursively "
-            "defined"_err_en_US,
+            "The interface for procedure '%s' is recursively defined"_err_en_US,
             procInCycle->name());
         context().SetError(*procInCycle);
       }
diff --git a/flang/test/Semantics/entry01.f90 b/flang/test/Semantics/entry01.f90
index 970cd109921a..765b18c2e81a 100644
--- a/flang/test/Semantics/entry01.f90
+++ b/flang/test/Semantics/entry01.f90
@@ -83,6 +83,7 @@ function ifunc()
   !ERROR: 'ibad1' is already declared in this scoping unit
   entry ibad1() result(ibad1res) ! C1570
   !ERROR: 'ibad2' is already declared in this scoping unit
+  !ERROR: Procedure 'ibad2' is referenced before being sufficiently defined in a context where it must be so
   entry ibad2()
   !ERROR: ENTRY in a function may not have an alternate return dummy argument
   entry ibadalt(*) ! C1573
@@ -91,6 +92,7 @@ function ifunc()
   entry iok()
   !ERROR: Explicit RESULT('iok') of function 'isameres2' cannot have the same name as a distinct ENTRY into the same scope
   entry isameres2() result(iok) ! C1574
+  !ERROR: Procedure 'iok2' is referenced before being sufficiently defined in a context where it must be so
   !ERROR: Explicit RESULT('iok2') of function 'isameres3' cannot have the same name as a distinct ENTRY into the same scope
   entry isameres3() result(iok2) ! C1574
   !ERROR: 'iok2' is already declared in this scoping unit
diff --git a/flang/test/Semantics/resolve102.f90 b/flang/test/Semantics/resolve102.f90
index 8f6e2246a57e..33cf6fa245ea 100644
--- a/flang/test/Semantics/resolve102.f90
+++ b/flang/test/Semantics/resolve102.f90
@@ -4,17 +4,12 @@
 !ERROR: Procedure 'sub' is recursively defined.  Procedures in the cycle: 'sub', 'p2'
 subroutine sub(p2)
   PROCEDURE(sub) :: p2
-
-  call sub()
 end subroutine
 
 subroutine circular
-  !ERROR: Procedure 'p' is recursively defined.  Procedures in the cycle: 'p', 'sub', 'p2'
   procedure(sub) :: p
-
-  call p(sub)
-
   contains
+    !ERROR: Procedure 'sub' is recursively defined.  Procedures in the cycle: 'p', 'sub', 'p2'
     subroutine sub(p2)
       procedure(p) :: p2
     end subroutine
@@ -41,11 +36,10 @@ end subroutine
 
 subroutine mutual
   Procedure(sub1) :: p
-
-  Call p(sub)
-
   contains
     !ERROR: Procedure 'sub1' is recursively defined.  Procedures in the cycle: 'p', 'sub1', 'arg'
+    !ERROR: Procedure 'sub1' is recursively defined.  Procedures in the cycle: 'sub1', 'arg', 'sub', 'p2'
+    !ERROR: Procedure 'sub1' is recursively defined.  Procedures in the cycle: 'sub1', 'arg'
     Subroutine sub1(arg)
       procedure(sub1) :: arg
     End Subroutine
@@ -57,15 +51,14 @@ End subroutine
 
 subroutine mutual1
   Procedure(sub1) :: p
-
-  Call p(sub)
-
   contains
     !ERROR: Procedure 'sub1' is recursively defined.  Procedures in the cycle: 'p', 'sub1', 'arg', 'sub', 'p2'
+    !ERROR: Procedure 'sub1' is recursively defined.  Procedures in the cycle: 'sub1', 'arg', 'sub', 'p2'
     Subroutine sub1(arg)
       procedure(sub) :: arg
     End Subroutine
 
+    !ERROR: Procedure 'sub' is recursively defined.  Procedures in the cycle: 'sub1', 'arg', 'sub', 'p2'
     Subroutine sub(p2)
       Procedure(sub1) :: p2
     End Subroutine
@@ -76,8 +69,6 @@ subroutine twoCycle
   !ERROR: The interface for procedure 'p2' is recursively defined
   procedure(p1) p2
   procedure(p2) p1
-  call p1
-  call p2
 end subroutine
 
 subroutine threeCycle
@@ -87,9 +78,6 @@ subroutine threeCycle
   !ERROR: The interface for procedure 'p3' is recursively defined
   procedure(p2) p3
   procedure(p3) p1
-  call p1
-  call p2
-  call p3
 end subroutine
 
 module mutualSpecExprs
@@ -118,4 +106,3 @@ module genericInSpec
     ifunc = x
   end
 end
-
-- 
GitLab


From 463f58a564a8d136b3e5d56d23bb86b99ab75245 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Wed, 15 May 2024 16:18:47 -0700
Subject: [PATCH 0932/1206] [flang] Further work on relaxing BIND(C)
 enforcement (#92029)

When a BIND(C) interface or subprogram has a dummy argument whose
derived type is not BIND(C) but meets the constraints and requirements
of a BIND(C) type, accept it with a warning.
---
 flang/lib/Semantics/check-declarations.cpp | 16 +++++---
 flang/test/Semantics/bind-c15.f90          | 45 ++++++++++++++++++++++
 2 files changed, 56 insertions(+), 5 deletions(-)
 create mode 100644 flang/test/Semantics/bind-c15.f90

diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index 8d17989ac279..527a1a9539aa 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -2891,7 +2891,8 @@ parser::Messages CheckHelper::WhyNotInteroperableDerivedType(
       } else {
         bool interoperableParent{true};
         if (parent->symbol()) {
-          auto bad{WhyNotInteroperableDerivedType(*parent->symbol(), false)};
+          auto bad{WhyNotInteroperableDerivedType(
+              *parent->symbol(), /*isError=*/false)};
           if (bad.AnyFatalError()) {
             auto &msg{msgs.Say(symbol.name(),
                 "The parent of an interoperable type is not interoperable"_err_en_US)};
@@ -2981,6 +2982,9 @@ parser::Messages CheckHelper::WhyNotInteroperableDerivedType(
       }
     }
   }
+  if (msgs.AnyFatalError()) {
+    examinedByWhyNotInteroperableDerivedType_.erase(symbol);
+  }
   return msgs;
 }
 
@@ -3068,8 +3072,8 @@ void CheckHelper::CheckBindC(const Symbol &symbol) {
           }
           context_.SetError(symbol);
         } else if (auto bad{WhyNotInteroperableDerivedType(
-                       derived->typeSymbol(), false)};
-                   !bad.empty()) {
+                       derived->typeSymbol(), /*isError=*/false)};
+                   bad.AnyFatalError()) {
           if (auto *msg{messages_.Say(symbol.name(),
                   "The derived type of an interoperable object must be interoperable, but is not"_err_en_US)}) {
             msg->Attach(
@@ -3077,7 +3081,9 @@ void CheckHelper::CheckBindC(const Symbol &symbol) {
             bad.AttachTo(*msg, parser::Severity::None);
           }
           context_.SetError(symbol);
-        } else {
+        } else if (context_.ShouldWarn(
+                       common::LanguageFeature::NonBindCInteroperability) &&
+            !InModuleFile()) {
           if (auto *msg{messages_.Say(symbol.name(),
                   "The derived type of an interoperable object should be BIND(C)"_warn_en_US)}) {
             msg->Attach(derived->typeSymbol().name(), "Non-BIND(C) type"_en_US);
@@ -3151,7 +3157,7 @@ void CheckHelper::CheckBindC(const Symbol &symbol) {
       }
     }
   } else if (symbol.has<DerivedTypeDetails>()) {
-    if (auto msgs{WhyNotInteroperableDerivedType(symbol, false)};
+    if (auto msgs{WhyNotInteroperableDerivedType(symbol, /*isError=*/false)};
         !msgs.empty()) {
       bool anyFatal{msgs.AnyFatalError()};
       if (msgs.AnyFatalError() ||
diff --git a/flang/test/Semantics/bind-c15.f90 b/flang/test/Semantics/bind-c15.f90
new file mode 100644
index 000000000000..9aaad52cc0e0
--- /dev/null
+++ b/flang/test/Semantics/bind-c15.f90
@@ -0,0 +1,45 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
+
+module m
+  type, bind(c) :: explicit_bind_c
+    real a
+  end type
+  type :: interoperable1
+    type(explicit_bind_c) a
+  end type
+  type, extends(interoperable1) :: interoperable2
+    real b
+  end type
+  type :: non_interoperable1
+    real, allocatable :: a
+  end type
+  type :: non_interoperable2
+    type(non_interoperable1) b
+  end type
+  interface
+    subroutine sub_bind_c_1(x_bind_c) bind(c)
+      import explicit_bind_c
+      type(explicit_bind_c), intent(in) :: x_bind_c
+    end
+    subroutine sub_bind_c_2(x_interop1) bind(c)
+      import interoperable1
+      !WARNING: The derived type of an interoperable object should be BIND(C)
+      type(interoperable1), intent(in) :: x_interop1
+    end
+    subroutine sub_bind_c_3(x_interop2) bind(c)
+      import interoperable2
+      !WARNING: The derived type of an interoperable object should be BIND(C)
+      type(interoperable2), intent(in) :: x_interop2
+    end
+    subroutine sub_bind_c_4(x_non_interop1) bind(c)
+      import non_interoperable1
+      !ERROR: The derived type of an interoperable object must be interoperable, but is not
+      type(non_interoperable1), intent(in) :: x_non_interop1
+    end
+    subroutine sub_bind_c_5(x_non_interop2) bind(c)
+      import non_interoperable2
+      !ERROR: The derived type of an interoperable object must be interoperable, but is not
+      type(non_interoperable2), intent(in) :: x_non_interop2
+    end
+  end interface
+end
-- 
GitLab


From 5bbb63bd6d6d3929de643fcd88babbda20c97b69 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Wed, 15 May 2024 16:28:58 -0700
Subject: [PATCH 0933/1206] [flang] Parse REDUCE clauses in !$CUF KERNEL DO
 (#92154)

A !$CUF KERNEL DO directive is allowed to have advisory REDUCE clauses
similar to those in OpenACC and DO CONCURRENT. Parse and represent them.
Semantic validation will follow.
---
 flang/include/flang/Parser/dump-parse-tree.h |  1 +
 flang/include/flang/Parser/parse-tree.h      | 18 ++++-
 flang/lib/Parser/executable-parsers.cpp      | 23 +++++--
 flang/lib/Parser/openacc-parsers.cpp         |  6 +-
 flang/lib/Parser/unparse.cpp                 | 36 +++++++++-
 flang/lib/Semantics/check-cuda.cpp           | 44 ++++++++++++
 flang/lib/Semantics/resolve-directives.h     |  2 +-
 flang/lib/Semantics/resolve-names.cpp        |  2 +-
 flang/test/Parser/cuf-sanity-common          |  7 ++
 flang/test/Parser/cuf-sanity-unparse.CUF     |  6 ++
 flang/test/Semantics/reduce.cuf              | 72 ++++++++++++++++++++
 11 files changed, 199 insertions(+), 18 deletions(-)
 create mode 100644 flang/test/Semantics/reduce.cuf

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 477d391277ee..68ae50c312cd 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -236,6 +236,7 @@ public:
   NODE(parser, CUFKernelDoConstruct)
   NODE(CUFKernelDoConstruct, StarOrExpr)
   NODE(CUFKernelDoConstruct, Directive)
+  NODE(parser, CUFReduction)
   NODE(parser, CycleStmt)
   NODE(parser, DataComponentDefStmt)
   NODE(parser, DataIDoObject)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index c06354458379..0a40aa8b8f61 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -4303,12 +4303,23 @@ struct OpenACCConstruct {
 };
 
 // CUF-kernel-do-construct ->
-//     !$CUF KERNEL DO [ (scalar-int-constant-expr) ] <<< grid, block [, stream]
-//     >>> do-construct
+//   !$CUF KERNEL DO [ (scalar-int-constant-expr) ]
+//      <<< grid, block [, stream] >>>
+//      [ cuf-reduction... ]
+//      do-construct
 // star-or-expr -> * | scalar-int-expr
 // grid -> * | scalar-int-expr | ( star-or-expr-list )
 // block -> * | scalar-int-expr | ( star-or-expr-list )
 // stream -> 0, scalar-int-expr | STREAM = scalar-int-expr
+// cuf-reduction -> [ REDUCE | REDUCTION ] (
+//                  acc-reduction-op : scalar-variable-list )
+
+struct CUFReduction {
+  TUPLE_CLASS_BOILERPLATE(CUFReduction);
+  using Operator = AccReductionOperator;
+  std::tuple<Operator, std::list<Scalar<Variable>>> t;
+};
+
 struct CUFKernelDoConstruct {
   TUPLE_CLASS_BOILERPLATE(CUFKernelDoConstruct);
   WRAPPER_CLASS(StarOrExpr, std::optional<ScalarIntExpr>);
@@ -4316,7 +4327,8 @@ struct CUFKernelDoConstruct {
     TUPLE_CLASS_BOILERPLATE(Directive);
     CharBlock source;
     std::tuple<std::optional<ScalarIntConstantExpr>, std::list<StarOrExpr>,
-        std::list<StarOrExpr>, std::optional<ScalarIntExpr>>
+        std::list<StarOrExpr>, std::optional<ScalarIntExpr>,
+        std::list<CUFReduction>>
         t;
   };
   std::tuple<Directive, std::optional<DoConstruct>> t;
diff --git a/flang/lib/Parser/executable-parsers.cpp b/flang/lib/Parser/executable-parsers.cpp
index 07a570bd61e9..382a59341687 100644
--- a/flang/lib/Parser/executable-parsers.cpp
+++ b/flang/lib/Parser/executable-parsers.cpp
@@ -538,25 +538,34 @@ TYPE_CONTEXT_PARSER("UNLOCK statement"_en_US,
     construct<UnlockStmt>("UNLOCK (" >> lockVariable,
         defaulted("," >> nonemptyList(statOrErrmsg)) / ")"))
 
-// CUF-kernel-do-construct -> CUF-kernel-do-directive do-construct
-// CUF-kernel-do-directive ->
-//     !$CUF KERNEL DO [ (scalar-int-constant-expr) ] <<< grid, block [, stream]
-//     >>> do-construct
+// CUF-kernel-do-construct ->
+//   !$CUF KERNEL DO [ (scalar-int-constant-expr) ]
+//      <<< grid, block [, stream] >>>
+//      [ cuf-reduction... ]
+//      do-construct
 // star-or-expr -> * | scalar-int-expr
 // grid -> * | scalar-int-expr | ( star-or-expr-list )
 // block -> * | scalar-int-expr | ( star-or-expr-list )
-// stream -> ( 0, | STREAM = ) scalar-int-expr
+// stream -> 0, scalar-int-expr | STREAM = scalar-int-expr
+// cuf-reduction -> [ REDUCTION | REDUCE ] (
+//                  acc-reduction-op : scalar-variable-list )
+
 constexpr auto starOrExpr{construct<CUFKernelDoConstruct::StarOrExpr>(
     "*" >> pure<std::optional<ScalarIntExpr>>() ||
     applyFunction(presentOptional<ScalarIntExpr>, scalarIntExpr))};
 constexpr auto gridOrBlock{parenthesized(nonemptyList(starOrExpr)) ||
     applyFunction(singletonList<CUFKernelDoConstruct::StarOrExpr>, starOrExpr)};
+
+TYPE_PARSER(("REDUCTION"_tok || "REDUCE"_tok) >>
+    parenthesized(construct<CUFReduction>(Parser<CUFReduction::Operator>{},
+        ":" >> nonemptyList(scalar(variable)))))
+
 TYPE_PARSER(sourced(beginDirective >> "$CUF KERNEL DO"_tok >>
     construct<CUFKernelDoConstruct::Directive>(
         maybe(parenthesized(scalarIntConstantExpr)), "<<<" >> gridOrBlock,
         "," >> gridOrBlock,
-        maybe((", 0 ,"_tok || ", STREAM ="_tok) >> scalarIntExpr) / ">>>" /
-            endDirective)))
+        maybe((", 0 ,"_tok || ", STREAM ="_tok) >> scalarIntExpr) / ">>>",
+        many(Parser<CUFReduction>{}) / endDirective)))
 TYPE_CONTEXT_PARSER("!$CUF KERNEL DO construct"_en_US,
     extension<LanguageFeature::CUDA>(construct<CUFKernelDoConstruct>(
         Parser<CUFKernelDoConstruct::Directive>{},
diff --git a/flang/lib/Parser/openacc-parsers.cpp b/flang/lib/Parser/openacc-parsers.cpp
index 946b33d0084a..3d919e29a248 100644
--- a/flang/lib/Parser/openacc-parsers.cpp
+++ b/flang/lib/Parser/openacc-parsers.cpp
@@ -19,9 +19,9 @@
 // OpenACC Directives and Clauses
 namespace Fortran::parser {
 
-constexpr auto startAccLine = skipStuffBeforeStatement >>
-    ("!$ACC "_sptok || "C$ACC "_sptok || "*$ACC "_sptok);
-constexpr auto endAccLine = space >> endOfLine;
+constexpr auto startAccLine{skipStuffBeforeStatement >>
+    ("!$ACC "_sptok || "C$ACC "_sptok || "*$ACC "_sptok)};
+constexpr auto endAccLine{space >> endOfLine};
 
 // Autogenerated clauses parser. Information is taken from ACC.td and the
 // parser is generated by tablegen.
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 3398b395f198..1639e900903f 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2705,7 +2705,6 @@ public:
   void Unparse(const CLASS::ENUM &x) { Word(CLASS::EnumToString(x)); }
   WALK_NESTED_ENUM(AccDataModifier, Modifier)
   WALK_NESTED_ENUM(AccessSpec, Kind) // R807
-  WALK_NESTED_ENUM(AccReductionOperator, Operator)
   WALK_NESTED_ENUM(common, TypeParamAttr) // R734
   WALK_NESTED_ENUM(common, CUDADataAttr) // CUDA
   WALK_NESTED_ENUM(common, CUDASubprogramAttrs) // CUDA
@@ -2736,6 +2735,31 @@ public:
   WALK_NESTED_ENUM(OmpOrderClause, Type) // OMP order-type
   WALK_NESTED_ENUM(OmpOrderModifier, Kind) // OMP order-modifier
 #undef WALK_NESTED_ENUM
+  void Unparse(const AccReductionOperator::Operator x) {
+    switch (x) {
+    case AccReductionOperator::Operator::Plus:
+      Word("+");
+      break;
+    case AccReductionOperator::Operator::Multiply:
+      Word("*");
+      break;
+    case AccReductionOperator::Operator::And:
+      Word(".AND.");
+      break;
+    case AccReductionOperator::Operator::Or:
+      Word(".OR.");
+      break;
+    case AccReductionOperator::Operator::Eqv:
+      Word(".EQV.");
+      break;
+    case AccReductionOperator::Operator::Neqv:
+      Word(".NEQV.");
+      break;
+    default:
+      Word(AccReductionOperator::EnumToString(x));
+      break;
+    }
+  }
 
   void Unparse(const CUFKernelDoConstruct::StarOrExpr &x) {
     if (x.v) {
@@ -2768,13 +2792,19 @@ public:
     if (const auto &stream{std::get<3>(x.t)}) {
       Word(",STREAM="), Walk(*stream);
     }
-    Word(">>>\n");
+    Word(">>>");
+    Walk(" ", std::get<std::list<CUFReduction>>(x.t), " ");
+    Word("\n");
   }
-
   void Unparse(const CUFKernelDoConstruct &x) {
     Walk(std::get<CUFKernelDoConstruct::Directive>(x.t));
     Walk(std::get<std::optional<DoConstruct>>(x.t));
   }
+  void Unparse(const CUFReduction &x) {
+    Word("REDUCE(");
+    Walk(std::get<CUFReduction::Operator>(x.t));
+    Walk(":", std::get<std::list<Scalar<Variable>>>(x.t), ",", ")");
+  }
 
   void Done() const { CHECK(indent_ == 0); }
 
diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp
index 96ab90239263..45217ed2e3cc 100644
--- a/flang/lib/Semantics/check-cuda.cpp
+++ b/flang/lib/Semantics/check-cuda.cpp
@@ -463,6 +463,46 @@ static int DoConstructTightNesting(
   return 1;
 }
 
+static void CheckReduce(
+    SemanticsContext &context, const parser::CUFReduction &reduce) {
+  auto op{std::get<parser::CUFReduction::Operator>(reduce.t).v};
+  for (const auto &var :
+      std::get<std::list<parser::Scalar<parser::Variable>>>(reduce.t)) {
+    if (const auto &typedExprPtr{var.thing.typedExpr};
+        typedExprPtr && typedExprPtr->v) {
+      const auto &expr{*typedExprPtr->v};
+      if (auto type{expr.GetType()}) {
+        auto cat{type->category()};
+        bool isOk{false};
+        switch (op) {
+        case parser::AccReductionOperator::Operator::Plus:
+        case parser::AccReductionOperator::Operator::Multiply:
+        case parser::AccReductionOperator::Operator::Max:
+        case parser::AccReductionOperator::Operator::Min:
+          isOk = cat == TypeCategory::Integer || cat == TypeCategory::Real;
+          break;
+        case parser::AccReductionOperator::Operator::Iand:
+        case parser::AccReductionOperator::Operator::Ior:
+        case parser::AccReductionOperator::Operator::Ieor:
+          isOk = cat == TypeCategory::Integer;
+          break;
+        case parser::AccReductionOperator::Operator::And:
+        case parser::AccReductionOperator::Operator::Or:
+        case parser::AccReductionOperator::Operator::Eqv:
+        case parser::AccReductionOperator::Operator::Neqv:
+          isOk = cat == TypeCategory::Logical;
+          break;
+        }
+        if (!isOk) {
+          context.Say(var.thing.GetSource(),
+              "!$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type %s"_err_en_US,
+              type->AsFortran());
+        }
+      }
+    }
+  }
+}
+
 void CUDAChecker::Enter(const parser::CUFKernelDoConstruct &x) {
   auto source{std::get<parser::CUFKernelDoConstruct::Directive>(x.t).source};
   const auto &directive{std::get<parser::CUFKernelDoConstruct::Directive>(x.t)};
@@ -489,6 +529,10 @@ void CUDAChecker::Enter(const parser::CUFKernelDoConstruct &x) {
   if (innerBlock) {
     DeviceContextChecker<true>{context_}.Check(*innerBlock);
   }
+  for (const auto &reduce :
+      std::get<std::list<parser::CUFReduction>>(directive.t)) {
+    CheckReduce(context_, reduce);
+  }
 }
 
 void CUDAChecker::Enter(const parser::AssignmentStmt &x) {
diff --git a/flang/lib/Semantics/resolve-directives.h b/flang/lib/Semantics/resolve-directives.h
index 4aef8ad6c400..5a890c26aa33 100644
--- a/flang/lib/Semantics/resolve-directives.h
+++ b/flang/lib/Semantics/resolve-directives.h
@@ -21,7 +21,7 @@ class SemanticsContext;
 
 // Name resolution for OpenACC and OpenMP directives
 void ResolveAccParts(
-    SemanticsContext &, const parser::ProgramUnit &, Scope *topScope = {});
+    SemanticsContext &, const parser::ProgramUnit &, Scope *topScope);
 void ResolveOmpParts(SemanticsContext &, const parser::ProgramUnit &);
 void ResolveOmpTopLevelParts(SemanticsContext &, const parser::Program &);
 
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 5626f2a8be97..40eee89de131 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -8940,7 +8940,7 @@ bool ResolveNamesVisitor::Pre(const parser::ProgramUnit &x) {
   FinishSpecificationParts(root);
   ResolveExecutionParts(root);
   FinishExecutionParts(root);
-  ResolveAccParts(context(), x);
+  ResolveAccParts(context(), x, /*topScope=*/nullptr);
   ResolveOmpParts(context(), x);
   return false;
 }
diff --git a/flang/test/Parser/cuf-sanity-common b/flang/test/Parser/cuf-sanity-common
index b097a6aa3004..9d73204e3f5f 100644
--- a/flang/test/Parser/cuf-sanity-common
+++ b/flang/test/Parser/cuf-sanity-common
@@ -23,12 +23,19 @@ module m
   end subroutine
   subroutine test
     logical isPinned
+    real a(10), x, y, z
     !$cuf kernel do(1) <<<*, *, stream = 1>>>
     do j = 1, 10
     end do
     !$cuf kernel do <<<1, (2, 3), stream = 1>>>
     do j = 1, 10
     end do
+    !$cuf kernel do <<<*, *>>> reduce(+:x,y) reduce(*:z)
+    do j = 1, 10
+      x = x + a(j)
+      y = y + a(j)
+      z = z * a(j)
+    end do
     call globalsub<<<1, 2>>>
     call globalsub<<<1, 2, 3>>>
     call globalsub<<<1, 2, 3, 4>>>
diff --git a/flang/test/Parser/cuf-sanity-unparse.CUF b/flang/test/Parser/cuf-sanity-unparse.CUF
index b6921e74fc05..d4be347dd044 100644
--- a/flang/test/Parser/cuf-sanity-unparse.CUF
+++ b/flang/test/Parser/cuf-sanity-unparse.CUF
@@ -34,6 +34,12 @@ include "cuf-sanity-common"
 !CHECK:   !$CUF KERNEL DO <<<1_4,(2_4,3_4),STREAM=1_4>>>
 !CHECK:   DO j=1_4,10_4
 !CHECK:   END DO
+!CHECK:   !$CUF KERNEL DO <<<*,*>>> REDUCE(+:x,y) REDUCE(*:z)
+!CHECK:   DO j=1_4,10_4
+!CHECK:    x=x+a(int(j,kind=8))
+!CHECK:    y=y+a(int(j,kind=8))
+!CHECK:    z=z*a(int(j,kind=8))
+!CHECK:   END DO
 !CHECK:    CALL globalsub<<<1_4,2_4>>>()
 !CHECK:    CALL globalsub<<<1_4,2_4,3_4>>>()
 !CHECK:    CALL globalsub<<<1_4,2_4,3_4,4_4>>>()
diff --git a/flang/test/Semantics/reduce.cuf b/flang/test/Semantics/reduce.cuf
new file mode 100644
index 000000000000..95ff2e87c09b
--- /dev/null
+++ b/flang/test/Semantics/reduce.cuf
@@ -0,0 +1,72 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+subroutine s(n,m,a,l)
+  integer, intent(in) :: n
+  integer, intent(in) :: m(n)
+  real, intent(in) :: a(n)
+  logical, intent(in) :: l(n)
+  integer j, mr
+  real ar
+  logical lr
+!$cuf kernel do <<<*,*>>> reduce (+:mr,ar)
+  do j=1,n; mr = mr + m(j); ar = ar + a(j); end do
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type LOGICAL(4)
+!$cuf kernel do <<<*,*>>> reduce (+:lr)
+  do j=1,n; end do
+!$cuf kernel do <<<*,*>>> reduce (*:mr,ar)
+  do j=1,n; mr = mr * m(j); ar = ar * a(j); end do
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type LOGICAL(4)
+!$cuf kernel do <<<*,*>>> reduce (*:lr)
+  do j=1,n; end do
+!$cuf kernel do <<<*,*>>> reduce (max:mr,ar)
+  do j=1,n; mr = max(mr,m(j)); ar = max(ar,a(j)); end do
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type LOGICAL(4)
+!$cuf kernel do <<<*,*>>> reduce (max:lr)
+  do j=1,n; end do
+!$cuf kernel do <<<*,*>>> reduce (min:mr,ar)
+  do j=1,n; mr = min(mr,m(j)); ar = min(ar,a(j)); end do
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type LOGICAL(4)
+!$cuf kernel do <<<*,*>>> reduce (min:lr)
+  do j=1,n; end do
+!$cuf kernel do <<<*,*>>> reduce (iand:mr)
+  do j=1,n; mr = iand(mr,m(j)); end do
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type REAL(4)
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type LOGICAL(4)
+!$cuf kernel do <<<*,*>>> reduce (iand:ar,lr)
+  do j=1,n; end do
+!$cuf kernel do <<<*,*>>> reduce (ieor:mr)
+  do j=1,n; mr = ieor(mr,m(j)); end do
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type REAL(4)
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type LOGICAL(4)
+!$cuf kernel do <<<*,*>>> reduce (ieor:ar,lr)
+  do j=1,n; end do
+!$cuf kernel do <<<*,*>>> reduce (ior:mr)
+  do j=1,n; mr = ior(mr,m(j)); end do
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type REAL(4)
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type LOGICAL(4)
+!$cuf kernel do <<<*,*>>> reduce (ior:ar,lr)
+  do j=1,n; end do
+!$cuf kernel do <<<*,*>>> reduce (.and.:lr)
+  do j=1,n; lr = lr .and. l(j); end do
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type INTEGER(4)
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type REAL(4)
+!$cuf kernel do <<<*,*>>> reduce (.and.:mr,ar)
+  do j=1,n; end do
+!$cuf kernel do <<<*,*>>> reduce (.eqv.:lr)
+  do j=1,n; lr = lr .eqv. l(j); end do
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type INTEGER(4)
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type REAL(4)
+!$cuf kernel do <<<*,*>>> reduce (.eqv.:mr,ar)
+  do j=1,n; end do
+!$cuf kernel do <<<*,*>>> reduce (.neqv.:lr)
+  do j=1,n; lr = lr .neqv. l(j); end do
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type INTEGER(4)
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type REAL(4)
+!$cuf kernel do <<<*,*>>> reduce (.neqv.:mr,ar)
+  do j=1,n; end do
+!$cuf kernel do <<<*,*>>> reduce (.or.:lr)
+  do j=1,n; lr = lr .or. l(j); end do
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type INTEGER(4)
+!ERROR: !$CUF KERNEL DO REDUCE operation is not acceptable for a variable with type REAL(4)
+!$cuf kernel do <<<*,*>>> reduce (.or.:mr,ar)
+  do j=1,n; end do
+end
-- 
GitLab


From 3ddfb6807e905868a3a9df71fa5ea87309181270 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Wed, 15 May 2024 16:41:12 -0700
Subject: [PATCH 0934/1206] [flang] Prevent crash from unfoldable TRANSFER()
 (#92282)

When the MOLD= argument's type is polymorphic, the type of the result
cannot be known at compilation time, so the call cannot be folded even
when the SOURCE= is constant.

Fixes https://github.com/llvm/llvm-project/issues/92264.
---
 flang/lib/Evaluate/fold.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang/lib/Evaluate/fold.cpp b/flang/lib/Evaluate/fold.cpp
index ed8829581998..cf6262d9a7c6 100644
--- a/flang/lib/Evaluate/fold.cpp
+++ b/flang/lib/Evaluate/fold.cpp
@@ -272,6 +272,7 @@ std::optional<Expr<SomeType>> FoldTransfer(
     }
   }
   if (sourceBytes && IsActuallyConstant(*source) && moldType && extents &&
+      !moldType->IsPolymorphic() &&
       (moldLength || moldType->category() != TypeCategory::Character)) {
     std::size_t elements{
         extents->empty() ? 1 : static_cast<std::size_t>((*extents)[0])};
-- 
GitLab


From c87b1ca4edefe3c267a20f28eaf79f6b83d36c66 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellis.sparky.hoag@gmail.com>
Date: Wed, 15 May 2024 18:41:25 -0500
Subject: [PATCH 0935/1206] [InstrProf] Fix bug when clearing traces with
 samples (#92310)

The `--temporal-profile-max-trace-length=0` flag in the `llvm-profdata
merge` command is used to remove traces from a profile. There was a bug
where traces would not be cleared if the profile was already sampled.
This patch fixes that.
---
 llvm/lib/ProfileData/InstrProfWriter.cpp           | 11 ++++++-----
 llvm/test/tools/llvm-profdata/trace-limit.proftext |  6 +++++-
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index b61c59aacc0f..c941b9d89df3 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -320,11 +320,8 @@ void InstrProfWriter::addBinaryIds(ArrayRef<llvm::object::BuildID> BIs) {
 }
 
 void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) {
-  if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
-    Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength);
-  if (Trace.FunctionNameRefs.empty())
-    return;
-
+  assert(Trace.FunctionNameRefs.size() <= MaxTemporalProfTraceLength);
+  assert(!Trace.FunctionNameRefs.empty());
   if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) {
     // Simply append the trace if we have not yet hit our reservoir size limit.
     TemporalProfTraces.push_back(std::move(Trace));
@@ -341,6 +338,10 @@ void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) {
 
 void InstrProfWriter::addTemporalProfileTraces(
     SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
+  for (auto &Trace : SrcTraces)
+    if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
+      Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength);
+  llvm::erase_if(SrcTraces, [](auto &T) { return T.FunctionNameRefs.empty(); });
   // Assume that the source has the same reservoir size as the destination to
   // avoid needing to record it in the indexed profile format.
   bool IsDestSampled =
diff --git a/llvm/test/tools/llvm-profdata/trace-limit.proftext b/llvm/test/tools/llvm-profdata/trace-limit.proftext
index cf6edd648b23..e246ee890ba3 100644
--- a/llvm/test/tools/llvm-profdata/trace-limit.proftext
+++ b/llvm/test/tools/llvm-profdata/trace-limit.proftext
@@ -1,13 +1,17 @@
 # RUN: llvm-profdata merge --temporal-profile-max-trace-length=0 %s -o %t.profdata
 # RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefix=NONE
 
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s %s %s %s -o %t.profdata
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 --temporal-profile-max-trace-length=0 %t.profdata -o %t.profdata
+# RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefix=NONE
+
 # RUN: llvm-profdata merge --temporal-profile-max-trace-length=2 %s -o %t.profdata
 # RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefixes=CHECK,SOME
 
 # RUN: llvm-profdata merge --temporal-profile-max-trace-length=1000 %s -o %t.profdata
 # RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefixes=CHECK,ALL
 
-# NONE: Temporal Profile Traces (samples=0 seen=0):
+# NONE: Temporal Profile Traces (samples=0
 # CHECK: Temporal Profile Traces (samples=1 seen=1):
 # SOME:   Trace 0 (weight=1 count=2):
 # ALL:    Trace 0 (weight=1 count=3):
-- 
GitLab


From c00e012bcf5da384a3e7339dc2e046779b339063 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Wed, 15 May 2024 17:03:09 -0700
Subject: [PATCH 0936/1206] [ctx_profile] Follow the pattern elsewhere for
 choosing the block IDs

This was an oversight in #91859. Using the subblock ID mechanism
other places that use the bitstream APIs (e.g. `BitstreamRemarkSerializer`) use.
---
 llvm/include/llvm/ProfileData/PGOCtxProfWriter.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
index 15578c51a495..edcf02c09469 100644
--- a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
+++ b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
 #define LLVM_PROFILEDATA_PGOCTXPROFWRITER_H_
 
+#include "llvm/Bitstream/BitCodeEnums.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
 #include "llvm/ProfileData/CtxInstrContextNode.h"
 
@@ -20,7 +21,7 @@ namespace llvm {
 enum PGOCtxProfileRecords { Invalid = 0, Version, Guid, CalleeIndex, Counters };
 
 enum PGOCtxProfileBlockIDs {
-  ProfileMetadataBlockID = 100,
+  ProfileMetadataBlockID = bitc::FIRST_APPLICATION_BLOCKID,
   ContextNodeBlockID = ProfileMetadataBlockID + 1
 };
 
-- 
GitLab


From 772b1b0cb26c66804d0a7e416dc7a5742b7f8db2 Mon Sep 17 00:00:00 2001
From: ChiaHungDuan <chiahungduan@google.com>
Date: Wed, 15 May 2024 17:13:08 -0700
Subject: [PATCH 0937/1206] [scudo] Move the chunk update into functions
 (#83493)

The code paths for mte enabled and disabled were interleaving and which
increases the difficulty of reading each path in both source level and
assembly level. In this change, we move the parts that they have
different logic into functions and minor refactors on the code
structure.
---
 compiler-rt/lib/scudo/standalone/combined.h | 371 ++++++++++++--------
 1 file changed, 221 insertions(+), 150 deletions(-)

diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index 927513dea92d..15a199ae0349 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -410,133 +410,18 @@ public:
       reportOutOfMemory(NeededSize);
     }
 
-    const uptr BlockUptr = reinterpret_cast<uptr>(Block);
-    const uptr UnalignedUserPtr = BlockUptr + Chunk::getHeaderSize();
-    const uptr UserPtr = roundUp(UnalignedUserPtr, Alignment);
-
-    void *Ptr = reinterpret_cast<void *>(UserPtr);
-    void *TaggedPtr = Ptr;
-    if (LIKELY(ClassId)) {
-      // We only need to zero or tag the contents for Primary backed
-      // allocations. We only set tags for primary allocations in order to avoid
-      // faulting potentially large numbers of pages for large secondary
-      // allocations. We assume that guard pages are enough to protect these
-      // allocations.
-      //
-      // FIXME: When the kernel provides a way to set the background tag of a
-      // mapping, we should be able to tag secondary allocations as well.
-      //
-      // When memory tagging is enabled, zeroing the contents is done as part of
-      // setting the tag.
-      if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
-        uptr PrevUserPtr;
-        Chunk::UnpackedHeader Header;
-        const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
-        const uptr BlockEnd = BlockUptr + BlockSize;
-        // If possible, try to reuse the UAF tag that was set by deallocate().
-        // For simplicity, only reuse tags if we have the same start address as
-        // the previous allocation. This handles the majority of cases since
-        // most allocations will not be more aligned than the minimum alignment.
-        //
-        // We need to handle situations involving reclaimed chunks, and retag
-        // the reclaimed portions if necessary. In the case where the chunk is
-        // fully reclaimed, the chunk's header will be zero, which will trigger
-        // the code path for new mappings and invalid chunks that prepares the
-        // chunk from scratch. There are three possibilities for partial
-        // reclaiming:
-        //
-        // (1) Header was reclaimed, data was partially reclaimed.
-        // (2) Header was not reclaimed, all data was reclaimed (e.g. because
-        //     data started on a page boundary).
-        // (3) Header was not reclaimed, data was partially reclaimed.
-        //
-        // Case (1) will be handled in the same way as for full reclaiming,
-        // since the header will be zero.
-        //
-        // We can detect case (2) by loading the tag from the start
-        // of the chunk. If it is zero, it means that either all data was
-        // reclaimed (since we never use zero as the chunk tag), or that the
-        // previous allocation was of size zero. Either way, we need to prepare
-        // a new chunk from scratch.
-        //
-        // We can detect case (3) by moving to the next page (if covered by the
-        // chunk) and loading the tag of its first granule. If it is zero, it
-        // means that all following pages may need to be retagged. On the other
-        // hand, if it is nonzero, we can assume that all following pages are
-        // still tagged, according to the logic that if any of the pages
-        // following the next page were reclaimed, the next page would have been
-        // reclaimed as well.
-        uptr TaggedUserPtr;
-        if (getChunkFromBlock(BlockUptr, &PrevUserPtr, &Header) &&
-            PrevUserPtr == UserPtr &&
-            (TaggedUserPtr = loadTag(UserPtr)) != UserPtr) {
-          uptr PrevEnd = TaggedUserPtr + Header.SizeOrUnusedBytes;
-          const uptr NextPage = roundUp(TaggedUserPtr, getPageSizeCached());
-          if (NextPage < PrevEnd && loadTag(NextPage) != NextPage)
-            PrevEnd = NextPage;
-          TaggedPtr = reinterpret_cast<void *>(TaggedUserPtr);
-          resizeTaggedChunk(PrevEnd, TaggedUserPtr + Size, Size, BlockEnd);
-          if (UNLIKELY(FillContents != NoFill && !Header.OriginOrWasZeroed)) {
-            // If an allocation needs to be zeroed (i.e. calloc) we can normally
-            // avoid zeroing the memory now since we can rely on memory having
-            // been zeroed on free, as this is normally done while setting the
-            // UAF tag. But if tagging was disabled per-thread when the memory
-            // was freed, it would not have been retagged and thus zeroed, and
-            // therefore it needs to be zeroed now.
-            memset(TaggedPtr, 0,
-                   Min(Size, roundUp(PrevEnd - TaggedUserPtr,
-                                     archMemoryTagGranuleSize())));
-          } else if (Size) {
-            // Clear any stack metadata that may have previously been stored in
-            // the chunk data.
-            memset(TaggedPtr, 0, archMemoryTagGranuleSize());
-          }
-        } else {
-          const uptr OddEvenMask =
-              computeOddEvenMaskForPointerMaybe(Options, BlockUptr, ClassId);
-          TaggedPtr = prepareTaggedChunk(Ptr, Size, OddEvenMask, BlockEnd);
-        }
-        storePrimaryAllocationStackMaybe(Options, Ptr);
-      } else {
-        Block = addHeaderTag(Block);
-        Ptr = addHeaderTag(Ptr);
-        if (UNLIKELY(FillContents != NoFill)) {
-          // This condition is not necessarily unlikely, but since memset is
-          // costly, we might as well mark it as such.
-          memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
-                 PrimaryT::getSizeByClassId(ClassId));
-        }
-      }
-    } else {
-      Block = addHeaderTag(Block);
-      Ptr = addHeaderTag(Ptr);
-      if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
-        storeTags(reinterpret_cast<uptr>(Block), reinterpret_cast<uptr>(Ptr));
-        storeSecondaryAllocationStackMaybe(Options, Ptr, Size);
-      }
+    const uptr UserPtr = roundUp(
+        reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(), Alignment);
+    const uptr SizeOrUnusedBytes =
+        ClassId ? Size : SecondaryBlockEnd - (UserPtr + Size);
+
+    if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {
+      return initChunk(ClassId, Origin, Block, UserPtr, SizeOrUnusedBytes,
+                       FillContents);
     }
 
-    Chunk::UnpackedHeader Header = {};
-    if (UNLIKELY(UnalignedUserPtr != UserPtr)) {
-      const uptr Offset = UserPtr - UnalignedUserPtr;
-      DCHECK_GE(Offset, 2 * sizeof(u32));
-      // The BlockMarker has no security purpose, but is specifically meant for
-      // the chunk iteration function that can be used in debugging situations.
-      // It is the only situation where we have to locate the start of a chunk
-      // based on its block address.
-      reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
-      reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
-      Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
-    }
-    Header.ClassId = ClassId & Chunk::ClassIdMask;
-    Header.State = Chunk::State::Allocated;
-    Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;
-    Header.SizeOrUnusedBytes =
-        (ClassId ? Size : SecondaryBlockEnd - (UserPtr + Size)) &
-        Chunk::SizeOrUnusedBytesMask;
-    Chunk::storeHeader(Cookie, Ptr, &Header);
-
-    return TaggedPtr;
+    return initChunkWithMemoryTagging(ClassId, Origin, Block, UserPtr, Size,
+                                      SizeOrUnusedBytes, FillContents);
   }
 
   NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0,
@@ -1163,6 +1048,175 @@ private:
            reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
   }
 
+  ALWAYS_INLINE void *initChunk(const uptr ClassId, const Chunk::Origin Origin,
+                                void *Block, const uptr UserPtr,
+                                const uptr SizeOrUnusedBytes,
+                                const FillContentsMode FillContents) {
+    Block = addHeaderTag(Block);
+    // Only do content fill when it's from primary allocator because secondary
+    // allocator has filled the content.
+    if (ClassId != 0 && UNLIKELY(FillContents != NoFill)) {
+      // This condition is not necessarily unlikely, but since memset is
+      // costly, we might as well mark it as such.
+      memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
+             PrimaryT::getSizeByClassId(ClassId));
+    }
+
+    Chunk::UnpackedHeader Header = {};
+
+    const uptr DefaultAlignedPtr =
+        reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
+    if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
+      const uptr Offset = UserPtr - DefaultAlignedPtr;
+      DCHECK_GE(Offset, 2 * sizeof(u32));
+      // The BlockMarker has no security purpose, but is specifically meant for
+      // the chunk iteration function that can be used in debugging situations.
+      // It is the only situation where we have to locate the start of a chunk
+      // based on its block address.
+      reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
+      reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
+      Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
+    }
+
+    Header.ClassId = ClassId & Chunk::ClassIdMask;
+    Header.State = Chunk::State::Allocated;
+    Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;
+    Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
+    Chunk::storeHeader(Cookie, reinterpret_cast<void *>(addHeaderTag(UserPtr)),
+                       &Header);
+
+    return reinterpret_cast<void *>(UserPtr);
+  }
+
+  NOINLINE void *
+  initChunkWithMemoryTagging(const uptr ClassId, const Chunk::Origin Origin,
+                             void *Block, const uptr UserPtr, const uptr Size,
+                             const uptr SizeOrUnusedBytes,
+                             const FillContentsMode FillContents) {
+    const Options Options = Primary.Options.load();
+    DCHECK(useMemoryTagging<AllocatorConfig>(Options));
+
+    void *Ptr = reinterpret_cast<void *>(UserPtr);
+    void *TaggedPtr = Ptr;
+
+    if (LIKELY(ClassId)) {
+      // Init the primary chunk.
+      //
+      // We only need to zero or tag the contents for Primary backed
+      // allocations. We only set tags for primary allocations in order to avoid
+      // faulting potentially large numbers of pages for large secondary
+      // allocations. We assume that guard pages are enough to protect these
+      // allocations.
+      //
+      // FIXME: When the kernel provides a way to set the background tag of a
+      // mapping, we should be able to tag secondary allocations as well.
+      //
+      // When memory tagging is enabled, zeroing the contents is done as part of
+      // setting the tag.
+
+      Chunk::UnpackedHeader Header;
+      const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
+      const uptr BlockUptr = reinterpret_cast<uptr>(Block);
+      const uptr BlockEnd = BlockUptr + BlockSize;
+      // If possible, try to reuse the UAF tag that was set by deallocate().
+      // For simplicity, only reuse tags if we have the same start address as
+      // the previous allocation. This handles the majority of cases since
+      // most allocations will not be more aligned than the minimum alignment.
+      //
+      // We need to handle situations involving reclaimed chunks, and retag
+      // the reclaimed portions if necessary. In the case where the chunk is
+      // fully reclaimed, the chunk's header will be zero, which will trigger
+      // the code path for new mappings and invalid chunks that prepares the
+      // chunk from scratch. There are three possibilities for partial
+      // reclaiming:
+      //
+      // (1) Header was reclaimed, data was partially reclaimed.
+      // (2) Header was not reclaimed, all data was reclaimed (e.g. because
+      //     data started on a page boundary).
+      // (3) Header was not reclaimed, data was partially reclaimed.
+      //
+      // Case (1) will be handled in the same way as for full reclaiming,
+      // since the header will be zero.
+      //
+      // We can detect case (2) by loading the tag from the start
+      // of the chunk. If it is zero, it means that either all data was
+      // reclaimed (since we never use zero as the chunk tag), or that the
+      // previous allocation was of size zero. Either way, we need to prepare
+      // a new chunk from scratch.
+      //
+      // We can detect case (3) by moving to the next page (if covered by the
+      // chunk) and loading the tag of its first granule. If it is zero, it
+      // means that all following pages may need to be retagged. On the other
+      // hand, if it is nonzero, we can assume that all following pages are
+      // still tagged, according to the logic that if any of the pages
+      // following the next page were reclaimed, the next page would have been
+      // reclaimed as well.
+      uptr TaggedUserPtr;
+      uptr PrevUserPtr;
+      if (getChunkFromBlock(BlockUptr, &PrevUserPtr, &Header) &&
+          PrevUserPtr == UserPtr &&
+          (TaggedUserPtr = loadTag(UserPtr)) != UserPtr) {
+        uptr PrevEnd = TaggedUserPtr + Header.SizeOrUnusedBytes;
+        const uptr NextPage = roundUp(TaggedUserPtr, getPageSizeCached());
+        if (NextPage < PrevEnd && loadTag(NextPage) != NextPage)
+          PrevEnd = NextPage;
+        TaggedPtr = reinterpret_cast<void *>(TaggedUserPtr);
+        resizeTaggedChunk(PrevEnd, TaggedUserPtr + Size, Size, BlockEnd);
+        if (UNLIKELY(FillContents != NoFill && !Header.OriginOrWasZeroed)) {
+          // If an allocation needs to be zeroed (i.e. calloc) we can normally
+          // avoid zeroing the memory now since we can rely on memory having
+          // been zeroed on free, as this is normally done while setting the
+          // UAF tag. But if tagging was disabled per-thread when the memory
+          // was freed, it would not have been retagged and thus zeroed, and
+          // therefore it needs to be zeroed now.
+          memset(TaggedPtr, 0,
+                 Min(Size, roundUp(PrevEnd - TaggedUserPtr,
+                                   archMemoryTagGranuleSize())));
+        } else if (Size) {
+          // Clear any stack metadata that may have previously been stored in
+          // the chunk data.
+          memset(TaggedPtr, 0, archMemoryTagGranuleSize());
+        }
+      } else {
+        const uptr OddEvenMask =
+            computeOddEvenMaskForPointerMaybe(Options, BlockUptr, ClassId);
+        TaggedPtr = prepareTaggedChunk(Ptr, Size, OddEvenMask, BlockEnd);
+      }
+      storePrimaryAllocationStackMaybe(Options, Ptr);
+    } else {
+      // Init the secondary chunk.
+
+      Block = addHeaderTag(Block);
+      Ptr = addHeaderTag(Ptr);
+      storeTags(reinterpret_cast<uptr>(Block), reinterpret_cast<uptr>(Ptr));
+      storeSecondaryAllocationStackMaybe(Options, Ptr, Size);
+    }
+
+    Chunk::UnpackedHeader Header = {};
+
+    const uptr DefaultAlignedPtr =
+        reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
+    if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
+      const uptr Offset = UserPtr - DefaultAlignedPtr;
+      DCHECK_GE(Offset, 2 * sizeof(u32));
+      // The BlockMarker has no security purpose, but is specifically meant for
+      // the chunk iteration function that can be used in debugging situations.
+      // It is the only situation where we have to locate the start of a chunk
+      // based on its block address.
+      reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
+      reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
+      Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
+    }
+
+    Header.ClassId = ClassId & Chunk::ClassIdMask;
+    Header.State = Chunk::State::Allocated;
+    Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;
+    Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
+    Chunk::storeHeader(Cookie, Ptr, &Header);
+
+    return TaggedPtr;
+  }
+
   void quarantineOrDeallocateChunk(const Options &Options, void *TaggedPtr,
                                    Chunk::UnpackedHeader *Header,
                                    uptr Size) NO_THREAD_SAFETY_ANALYSIS {
@@ -1177,31 +1231,23 @@ private:
       Header->State = Chunk::State::Available;
     else
       Header->State = Chunk::State::Quarantined;
-    Header->OriginOrWasZeroed = useMemoryTagging<AllocatorConfig>(Options) &&
-                                Header->ClassId &&
-                                !TSDRegistry.getDisableMemInit();
-    Chunk::storeHeader(Cookie, Ptr, Header);
 
-    if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
-      u8 PrevTag = extractTag(reinterpret_cast<uptr>(TaggedPtr));
-      storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size);
-      if (Header->ClassId) {
-        if (!TSDRegistry.getDisableMemInit()) {
-          uptr TaggedBegin, TaggedEnd;
-          const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
-              Options, reinterpret_cast<uptr>(getBlockBegin(Ptr, Header)),
-              Header->ClassId);
-          // Exclude the previous tag so that immediate use after free is
-          // detected 100% of the time.
-          setRandomTag(Ptr, Size, OddEvenMask | (1UL << PrevTag), &TaggedBegin,
-                       &TaggedEnd);
-        }
-      }
+    void *BlockBegin;
+    if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {
+      Header->OriginOrWasZeroed = 0U;
+      if (BypassQuarantine && allocatorSupportsMemoryTagging<AllocatorConfig>())
+        Ptr = untagPointer(Ptr);
+      BlockBegin = getBlockBegin(Ptr, Header);
+    } else {
+      Header->OriginOrWasZeroed =
+          Header->ClassId && !TSDRegistry.getDisableMemInit();
+      BlockBegin =
+          retagBlock(Options, TaggedPtr, Ptr, Header, Size, BypassQuarantine);
     }
+
+    Chunk::storeHeader(Cookie, Ptr, Header);
+
     if (BypassQuarantine) {
-      if (allocatorSupportsMemoryTagging<AllocatorConfig>())
-        Ptr = untagPointer(Ptr);
-      void *BlockBegin = getBlockBegin(Ptr, Header);
       const uptr ClassId = Header->ClassId;
       if (LIKELY(ClassId)) {
         bool CacheDrained;
@@ -1216,9 +1262,6 @@ private:
         if (CacheDrained)
           Primary.tryReleaseToOS(ClassId, ReleaseToOS::Normal);
       } else {
-        if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options)))
-          storeTags(reinterpret_cast<uptr>(BlockBegin),
-                    reinterpret_cast<uptr>(Ptr));
         Secondary.deallocate(Options, BlockBegin);
       }
     } else {
@@ -1228,6 +1271,34 @@ private:
     }
   }
 
+  NOINLINE void *retagBlock(const Options &Options, void *TaggedPtr, void *&Ptr,
+                            Chunk::UnpackedHeader *Header, const uptr Size,
+                            bool BypassQuarantine) {
+    DCHECK(useMemoryTagging<AllocatorConfig>(Options));
+
+    const u8 PrevTag = extractTag(reinterpret_cast<uptr>(TaggedPtr));
+    storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size);
+    if (Header->ClassId && !TSDRegistry.getDisableMemInit()) {
+      uptr TaggedBegin, TaggedEnd;
+      const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
+          Options, reinterpret_cast<uptr>(getBlockBegin(Ptr, Header)),
+          Header->ClassId);
+      // Exclude the previous tag so that immediate use after free is
+      // detected 100% of the time.
+      setRandomTag(Ptr, Size, OddEvenMask | (1UL << PrevTag), &TaggedBegin,
+                   &TaggedEnd);
+    }
+
+    Ptr = untagPointer(Ptr);
+    void *BlockBegin = getBlockBegin(Ptr, Header);
+    if (BypassQuarantine && !Header->ClassId) {
+      storeTags(reinterpret_cast<uptr>(BlockBegin),
+                reinterpret_cast<uptr>(Ptr));
+    }
+
+    return BlockBegin;
+  }
+
   bool getChunkFromBlock(uptr Block, uptr *Chunk,
                          Chunk::UnpackedHeader *Header) {
     *Chunk =
-- 
GitLab


From c6e787f771d1f9d6a846b2d9b8db6adcd87e8dba Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Wed, 15 May 2024 17:52:59 -0700
Subject: [PATCH 0938/1206] [MCAsmParser] .rept/.irp/.irpc: remove excess tail
 EOL in expansion

```
.irp foo,1
nop
.endr
nop
```

expands to an excess EOL between two nop lines. Remove the excess EOL.
---
 llvm/lib/MC/MCParser/AsmParser.cpp  | 31 +++++++++++------------------
 llvm/test/MC/AsmParser/macro-rept.s | 14 ++++++-------
 2 files changed, 19 insertions(+), 26 deletions(-)

diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 8d9acd54e879..46c1caa940c5 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -5629,27 +5629,20 @@ MCAsmMacro *AsmParser::parseMacroLikeBody(SMLoc DirectiveLoc) {
       return nullptr;
     }
 
-    if (Lexer.is(AsmToken::Identifier) &&
-        (getTok().getIdentifier() == ".rep" ||
-         getTok().getIdentifier() == ".rept" ||
-         getTok().getIdentifier() == ".irp" ||
-         getTok().getIdentifier() == ".irpc")) {
-      ++NestLevel;
-    }
-
-    // Otherwise, check whether we have reached the .endr.
-    if (Lexer.is(AsmToken::Identifier) && getTok().getIdentifier() == ".endr") {
-      if (NestLevel == 0) {
-        EndToken = getTok();
-        Lex();
-        if (Lexer.isNot(AsmToken::EndOfStatement)) {
-          printError(getTok().getLoc(),
-                     "unexpected token in '.endr' directive");
-          return nullptr;
+    if (Lexer.is(AsmToken::Identifier)) {
+      StringRef Ident = getTok().getIdentifier();
+      if (Ident == ".rep" || Ident == ".rept" || Ident == ".irp" ||
+          Ident == ".irpc") {
+        ++NestLevel;
+      } else if (Ident == ".endr") {
+        if (NestLevel == 0) {
+          EndToken = getTok();
+          Lex();
+          if (!parseEOL())
+            break;
         }
-        break;
+        --NestLevel;
       }
-      --NestLevel;
     }
 
     // Otherwise, scan till the end of the statement.
diff --git a/llvm/test/MC/AsmParser/macro-rept.s b/llvm/test/MC/AsmParser/macro-rept.s
index 1dc8060e1d87..2a6a4070bff5 100644
--- a/llvm/test/MC/AsmParser/macro-rept.s
+++ b/llvm/test/MC/AsmParser/macro-rept.s
@@ -13,10 +13,10 @@
 // CHECK: .long	1
 // CHECK: .long	1
 
-// CHECK: .long	0
-// CHECK: .long	0
-// CHECK: .long	0
-
-// CHECK: .long	0
-// CHECK: .long	0
-// CHECK: .long	0
+// CHECK:      .long	0
+// CHECK-NEXT: .long	0
+// CHECK-NEXT: .long	0
+// CHECK-NEXT: .long	0
+// CHECK-NEXT: .long	0
+// CHECK-NEXT: .long	0
+// CHECK-EMPTY:
-- 
GitLab


From 26fabdded34f8cea490060a70188a07ad6b76b8b Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 15 May 2024 17:53:28 -0700
Subject: [PATCH 0939/1206] [memprof] Pass FrameIdConverter and
 CallStackIdConverter by reference (#92327)

CallStackIdConverter sets LastUnmappedId when a mapping failure
occurs.  Now, since toMemProfRecord takes an instance of
CallStackIdConverter by value, namely std::function, the caller of
toMemProfRecord never receives the mapping failure that occurs inside
toMemProfRecord.  The same problem applies to FrameIdConverter.

The patch fixes the problem by passing FrameIdConverter and
CallStackIdConverter by reference, namely llvm::function_ref.

While I am it, this patch deletes the copy constructor and copy
assignment operator to avoid accidental copies.
---
 llvm/include/llvm/ProfileData/MemProf.h    | 21 +++++--
 llvm/lib/ProfileData/MemProf.cpp           |  4 +-
 llvm/unittests/ProfileData/MemProfTest.cpp | 66 ++++++++++++++++++++++
 3 files changed, 85 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index 3ef6ca8586fb..60bff76d0e46 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -426,8 +426,8 @@ struct IndexedMemProfRecord {
   // Convert IndexedMemProfRecord to MemProfRecord.  Callback is used to
   // translate CallStackId to call stacks with frames inline.
   MemProfRecord toMemProfRecord(
-      std::function<const llvm::SmallVector<Frame>(const CallStackId)> Callback)
-      const;
+      llvm::function_ref<const llvm::SmallVector<Frame>(const CallStackId)>
+          Callback) const;
 
   // Returns the GUID for the function name after canonicalization. For
   // memprof, we remove any .llvm suffix added by LTO. MemProfRecords are
@@ -784,6 +784,12 @@ template <typename MapTy> struct FrameIdConverter {
   FrameIdConverter() = delete;
   FrameIdConverter(MapTy &Map) : Map(Map) {}
 
+  // Delete the copy constructor and copy assignment operator to avoid a
+  // situation where a copy of FrameIdConverter gets an error in LastUnmappedId
+  // while the original instance doesn't.
+  FrameIdConverter(const FrameIdConverter &) = delete;
+  FrameIdConverter &operator=(const FrameIdConverter &) = delete;
+
   Frame operator()(FrameId Id) {
     auto Iter = Map.find(Id);
     if (Iter == Map.end()) {
@@ -798,12 +804,19 @@ template <typename MapTy> struct FrameIdConverter {
 template <typename MapTy> struct CallStackIdConverter {
   std::optional<CallStackId> LastUnmappedId;
   MapTy &Map;
-  std::function<Frame(FrameId)> FrameIdToFrame;
+  llvm::function_ref<Frame(FrameId)> FrameIdToFrame;
 
   CallStackIdConverter() = delete;
-  CallStackIdConverter(MapTy &Map, std::function<Frame(FrameId)> FrameIdToFrame)
+  CallStackIdConverter(MapTy &Map,
+                       llvm::function_ref<Frame(FrameId)> FrameIdToFrame)
       : Map(Map), FrameIdToFrame(FrameIdToFrame) {}
 
+  // Delete the copy constructor and copy assignment operator to avoid a
+  // situation where a copy of CallStackIdConverter gets an error in
+  // LastUnmappedId while the original instance doesn't.
+  CallStackIdConverter(const CallStackIdConverter &) = delete;
+  CallStackIdConverter &operator=(const CallStackIdConverter &) = delete;
+
   llvm::SmallVector<Frame> operator()(CallStackId CSId) {
     llvm::SmallVector<Frame> Frames;
     auto CSIter = Map.find(CSId);
diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp
index 4667778ca11d..f5789186094c 100644
--- a/llvm/lib/ProfileData/MemProf.cpp
+++ b/llvm/lib/ProfileData/MemProf.cpp
@@ -243,8 +243,8 @@ IndexedMemProfRecord::deserialize(const MemProfSchema &Schema,
 }
 
 MemProfRecord IndexedMemProfRecord::toMemProfRecord(
-    std::function<const llvm::SmallVector<Frame>(const CallStackId)> Callback)
-    const {
+    llvm::function_ref<const llvm::SmallVector<Frame>(const CallStackId)>
+        Callback) const {
   MemProfRecord Record;
 
   for (const memprof::IndexedAllocationInfo &IndexedAI : AllocSites) {
diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp
index 8b97866e403f..a913718d0fe0 100644
--- a/llvm/unittests/ProfileData/MemProfTest.cpp
+++ b/llvm/unittests/ProfileData/MemProfTest.cpp
@@ -596,4 +596,70 @@ TEST(MemProf, IndexedMemProfRecordToMemProfRecord) {
   EXPECT_EQ(Record.CallSites[1][0].hash(), F2.hash());
   EXPECT_EQ(Record.CallSites[1][1].hash(), F4.hash());
 }
+
+using FrameIdMapTy =
+    llvm::DenseMap<::llvm::memprof::FrameId, ::llvm::memprof::Frame>;
+using CallStackIdMapTy =
+    llvm::DenseMap<::llvm::memprof::CallStackId,
+                   ::llvm::SmallVector<::llvm::memprof::FrameId>>;
+
+// Populate those fields returned by getHotColdSchema.
+MemInfoBlock makePartialMIB() {
+  MemInfoBlock MIB;
+  MIB.AllocCount = 1;
+  MIB.TotalSize = 5;
+  MIB.TotalLifetime = 10;
+  MIB.TotalLifetimeAccessDensity = 23;
+  return MIB;
+}
+
+TEST(MemProf, MissingCallStackId) {
+  // Use a non-existent CallStackId to trigger a mapping error in
+  // toMemProfRecord.
+  llvm::memprof::IndexedAllocationInfo AI({}, 0xdeadbeefU, makePartialMIB(),
+                                          llvm::memprof::getHotColdSchema());
+
+  IndexedMemProfRecord IndexedMR;
+  IndexedMR.AllocSites.push_back(AI);
+
+  // Create empty maps.
+  const FrameIdMapTy IdToFrameMap;
+  const CallStackIdMapTy CSIdToCallStackMap;
+  llvm::memprof::FrameIdConverter<decltype(IdToFrameMap)> FrameIdConv(
+      IdToFrameMap);
+  llvm::memprof::CallStackIdConverter<decltype(CSIdToCallStackMap)> CSIdConv(
+      CSIdToCallStackMap, FrameIdConv);
+
+  // We are only interested in errors, not the return value.
+  (void)IndexedMR.toMemProfRecord(CSIdConv);
+
+  ASSERT_TRUE(CSIdConv.LastUnmappedId.has_value());
+  EXPECT_EQ(*CSIdConv.LastUnmappedId, 0xdeadbeefU);
+  EXPECT_EQ(FrameIdConv.LastUnmappedId, std::nullopt);
+}
+
+TEST(MemProf, MissingFrameId) {
+  llvm::memprof::IndexedAllocationInfo AI({}, 0x222, makePartialMIB(),
+                                          llvm::memprof::getHotColdSchema());
+
+  IndexedMemProfRecord IndexedMR;
+  IndexedMR.AllocSites.push_back(AI);
+
+  // An empty map to trigger a mapping error.
+  const FrameIdMapTy IdToFrameMap;
+  CallStackIdMapTy CSIdToCallStackMap;
+  CSIdToCallStackMap.insert({0x222, {2, 3}});
+
+  llvm::memprof::FrameIdConverter<decltype(IdToFrameMap)> FrameIdConv(
+      IdToFrameMap);
+  llvm::memprof::CallStackIdConverter<decltype(CSIdToCallStackMap)> CSIdConv(
+      CSIdToCallStackMap, FrameIdConv);
+
+  // We are only interested in errors, not the return value.
+  (void)IndexedMR.toMemProfRecord(CSIdConv);
+
+  EXPECT_EQ(CSIdConv.LastUnmappedId, std::nullopt);
+  ASSERT_TRUE(FrameIdConv.LastUnmappedId.has_value());
+  EXPECT_EQ(*FrameIdConv.LastUnmappedId, 3U);
+}
 } // namespace
-- 
GitLab


From fa750f09be6966de7423ddce1af7d1eaf817182c Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 16 May 2024 09:49:34 +0900
Subject: [PATCH 0940/1206] Revert "[MC] Remove UseAssemblerInfoForParsing"

This reverts commit 03c53c69a367008da689f0d2940e2197eb4a955c.

This causes very large compile-time regressions in some cases,
e.g. sqlite3 at O0 regresses by 5%.
---
 clang/tools/driver/cc1as_main.cpp                |  3 +++
 llvm/include/llvm/MC/MCStreamer.h                |  7 +++++--
 .../CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp   |  3 +++
 llvm/lib/MC/MCObjectStreamer.cpp                 |  9 ++++++++-
 llvm/lib/MC/MCStreamer.cpp                       |  2 +-
 llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp      |  7 +++++--
 llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp        |  3 +++
 .../AsmParser/assembler-expressions-inlineasm.ll | 16 ++++++----------
 llvm/tools/llvm-mc/llvm-mc.cpp                   |  3 +++
 llvm/tools/llvm-ml/llvm-ml.cpp                   |  3 +++
 10 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index 4eb753a7297a..86afe22fac24 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -576,6 +576,9 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
     Str.get()->emitZeros(1);
   }
 
+  // Assembly to object compilation should leverage assembly info.
+  Str->setUseAssemblerInfoForParsing(true);
+
   bool Failed = false;
 
   std::unique_ptr<MCAsmParser> Parser(
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 50986e6bde88..69867620e1bf 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -245,6 +245,8 @@ class MCStreamer {
   /// requires.
   unsigned NextWinCFIID = 0;
 
+  bool UseAssemblerInfoForParsing;
+
   /// Is the assembler allowed to insert padding automatically?  For
   /// correctness reasons, we sometimes need to ensure instructions aren't
   /// separated in unexpected ways.  At the moment, this feature is only
@@ -294,10 +296,11 @@ public:
 
   MCContext &getContext() const { return Context; }
 
-  // MCObjectStreamer has an MCAssembler and allows more expression folding at
-  // parse time.
   virtual MCAssembler *getAssemblerPtr() { return nullptr; }
 
+  void setUseAssemblerInfoForParsing(bool v) { UseAssemblerInfoForParsing = v; }
+  bool getUseAssemblerInfoForParsing() { return UseAssemblerInfoForParsing; }
+
   MCTargetStreamer *getTargetStreamer() {
     return TargetStreamer.get();
   }
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
index 08e3c208ba4d..d0ef3e5a1939 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
@@ -102,6 +102,9 @@ void AsmPrinter::emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
   std::unique_ptr<MCAsmParser> Parser(
       createMCAsmParser(SrcMgr, OutContext, *OutStreamer, *MAI, BufNum));
 
+  // Do not use assembler-level information for parsing inline assembly.
+  OutStreamer->setUseAssemblerInfoForParsing(false);
+
   // We create a new MCInstrInfo here since we might be at the module level
   // and not have a MachineFunction to initialize the TargetInstrInfo from and
   // we only need MCInstrInfo for asm parsing. We create one unconditionally
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index a9003a164b30..d2da5d0d3f90 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -40,7 +40,14 @@ MCObjectStreamer::MCObjectStreamer(MCContext &Context,
 
 MCObjectStreamer::~MCObjectStreamer() = default;
 
-MCAssembler *MCObjectStreamer::getAssemblerPtr() { return Assembler.get(); }
+// AssemblerPtr is used for evaluation of expressions and causes
+// difference between asm and object outputs. Return nullptr to in
+// inline asm mode to limit divergence to assembly inputs.
+MCAssembler *MCObjectStreamer::getAssemblerPtr() {
+  if (getUseAssemblerInfoForParsing())
+    return Assembler.get();
+  return nullptr;
+}
 
 void MCObjectStreamer::addPendingLabel(MCSymbol* S) {
   MCSection *CurSection = getCurrentSectionOnly();
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index 199d865ea349..176d55aa890b 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -93,7 +93,7 @@ void MCTargetStreamer::emitAssignment(MCSymbol *Symbol, const MCExpr *Value) {}
 
 MCStreamer::MCStreamer(MCContext &Ctx)
     : Context(Ctx), CurrentWinFrameInfo(nullptr),
-      CurrentProcWinFrameInfoStartIndex(0) {
+      CurrentProcWinFrameInfoStartIndex(0), UseAssemblerInfoForParsing(false) {
   SectionStack.push_back(std::pair<MCSectionSubPair, MCSectionSubPair>());
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index bd48a5f80c82..b7388ed9e85a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -517,9 +517,12 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
 
   DumpCodeInstEmitter = nullptr;
   if (STM.dumpCode()) {
-    // For -dumpcode, get the assembler out of the streamer. This only works
-    // with -filetype=obj.
+    // For -dumpcode, get the assembler out of the streamer, even if it does
+    // not really want to let us have it. This only works with -filetype=obj.
+    bool SaveFlag = OutStreamer->getUseAssemblerInfoForParsing();
+    OutStreamer->setUseAssemblerInfoForParsing(true);
     MCAssembler *Assembler = OutStreamer->getAssemblerPtr();
+    OutStreamer->setUseAssemblerInfoForParsing(SaveFlag);
     if (Assembler)
       DumpCodeInstEmitter = Assembler->getEmitterPtr();
   }
diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
index ad0158086044..2ebe5bdc4771 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
@@ -114,9 +114,12 @@ void SPIRVAsmPrinter::emitEndOfAsmFile(Module &M) {
   // Bound is an approximation that accounts for the maximum used register
   // number and number of generated OpLabels
   unsigned Bound = 2 * (ST->getBound() + 1) + NLabels;
+  bool FlagToRestore = OutStreamer->getUseAssemblerInfoForParsing();
+  OutStreamer->setUseAssemblerInfoForParsing(true);
   if (MCAssembler *Asm = OutStreamer->getAssemblerPtr())
     Asm->setBuildVersion(static_cast<MachO::PlatformType>(0), Major, Minor,
                          Bound, VersionTuple(Major, Minor, 0, Bound));
+  OutStreamer->setUseAssemblerInfoForParsing(FlagToRestore);
 }
 
 void SPIRVAsmPrinter::emitFunctionHeader() {
diff --git a/llvm/test/MC/AsmParser/assembler-expressions-inlineasm.ll b/llvm/test/MC/AsmParser/assembler-expressions-inlineasm.ll
index 9d9a38f5b5a5..35f110f37e2f 100644
--- a/llvm/test/MC/AsmParser/assembler-expressions-inlineasm.ll
+++ b/llvm/test/MC/AsmParser/assembler-expressions-inlineasm.ll
@@ -1,16 +1,12 @@
-; RUN: not llc -mtriple=x86_64 %s -o /dev/null 2>&1 | FileCheck %s
-; RUN: llc -mtriple=x86_64 -no-integrated-as < %s | FileCheck %s --check-prefix=GAS
-; RUN: llc -mtriple=x86_64 -filetype=obj %s -o - | llvm-objdump -d - |  FileCheck %s --check-prefix=DISASM
+; RUN: not llc -mtriple x86_64-unknown-linux-gnu -o %t.s -filetype=asm %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple x86_64-unknown-linux-gnu -o %t.o -filetype=obj %s 2>&1 | FileCheck %s
 
-; GAS: nop;  .if . - foo==1; nop;.endif
+; Assembler-aware expression evaluation should be disabled in inline
+; assembly to prevent differences in behavior between object and
+; assembly output.
 
-; CHECK: <inline asm>:1:17: error: expected absolute expression
 
-; DISASM:      <main>:
-; DISASM-NEXT:   nop
-; DISASM-NEXT:   nop
-; DISASM-NEXT:   xorl %eax, %eax
-; DISASM-NEXT:   retq
+; CHECK: <inline asm>:1:17: error: expected absolute expression
 
 define i32 @main() local_unnamed_addr {
   tail call void asm sideeffect "foo: nop;  .if . - foo==1;  nop;.endif", "~{dirflag},~{fpsr},~{flags}"()
diff --git a/llvm/tools/llvm-mc/llvm-mc.cpp b/llvm/tools/llvm-mc/llvm-mc.cpp
index 506e4f22ef8f..807071a7b9a1 100644
--- a/llvm/tools/llvm-mc/llvm-mc.cpp
+++ b/llvm/tools/llvm-mc/llvm-mc.cpp
@@ -569,6 +569,9 @@ int main(int argc, char **argv) {
       Str->initSections(true, *STI);
   }
 
+  // Use Assembler information for parsing.
+  Str->setUseAssemblerInfoForParsing(true);
+
   int Res = 1;
   bool disassemble = false;
   switch (Action) {
diff --git a/llvm/tools/llvm-ml/llvm-ml.cpp b/llvm/tools/llvm-ml/llvm-ml.cpp
index f1f39af059aa..1cac576f54e7 100644
--- a/llvm/tools/llvm-ml/llvm-ml.cpp
+++ b/llvm/tools/llvm-ml/llvm-ml.cpp
@@ -428,6 +428,9 @@ int llvm_ml_main(int Argc, char **Argv, const llvm::ToolContext &) {
     Str->emitAssignment(Feat00Sym, MCConstantExpr::create(Feat00Flags, Ctx));
   }
 
+  // Use Assembler information for parsing.
+  Str->setUseAssemblerInfoForParsing(true);
+
   int Res = 1;
   if (InputArgs.hasArg(OPT_as_lex)) {
     // -as-lex; Lex only, and output a stream of tokens
-- 
GitLab


From a9763deb2f3f20d789b947ec69360c258377db6a Mon Sep 17 00:00:00 2001
From: Shubham Sandeep Rastogi <srastogi22@apple.com>
Date: Wed, 15 May 2024 18:15:40 -0700
Subject: [PATCH 0941/1206] Merge sourcelocation in
 CSEMIRBuilder::getDominatingInstrForID. (#90922)

Make sure to merge the sourcelocation of the Dominating Instruction that
is hoisted in a basic block in the CSEMIRBuilder in the legalizer pass.

If this is not done, we can have a incorrect line table entry that makes
the instruction pointer jump around.

For example the line table without this patch looks like:

```
Address            Line   Column File   ISA Discriminator OpIndex Flags
------------------ ------ ------ ------ --- ------------- ------- -------------
0x0000000000000000      0      0      1   0             0       0  is_stmt
0x0000000000000010     11     14      1   0             0       0  is_stmt prologue_end
0x0000000000000028     12      1      1   0             0       0  is_stmt
0x000000000000002c     12     15      1   0             0       0
0x000000000000004c     12     13      1   0             0       0
0x000000000000005c     13      1      1   0             0       0  is_stmt
0x0000000000000064     12     13      1   0             0       0  is_stmt
0x000000000000007c     13      7      1   0             0       0  is_stmt
0x00000000000000c8     13      1      1   0             0       0
0x00000000000000e8     13      1      1   0             0       0  epilogue_begin
0x00000000000000f8     13      1      1   0             0       0  end_sequence
```

The line table entry for 0x000000000000005c should be 0

After this patch, the line table looks like:

```
Address            Line   Column File   ISA Discriminator OpIndex Flags
------------------ ------ ------ ------ --- ------------- ------- -------------
0x0000000000000000      0      0      1   0             0       0  is_stmt
0x0000000000000010     11     14      1   0             0       0  is_stmt prologue_end
0x0000000000000028     12      1      1   0             0       0  is_stmt
0x000000000000002c     12     15      1   0             0       0
0x000000000000004c     12     13      1   0             0       0
0x000000000000005c      0      0      1   0             0       0
0x0000000000000064     12     13      1   0             0       0
0x000000000000007c     13      7      1   0             0       0  is_stmt
0x00000000000000c8     13      1      1   0             0       0
0x00000000000000e8     13      1      1   0             0       0  epilogue_begin
0x00000000000000f8     13      1      1   0             0       0  end_sequence
```
---
 .../GlobalISel/LegalizationArtifactCombiner.h |  6 ++++
 llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp |  5 ++++
 .../AArch64/merge-locations-legalizer.mir     | 30 +++++++++++++++++++
 3 files changed, 41 insertions(+)
 create mode 100644 llvm/test/DebugInfo/AArch64/merge-locations-legalizer.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
index 305bef7dd3ea..2efc48e3be4c 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/Register.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "legalizer"
@@ -99,6 +100,11 @@ public:
       const LLT DstTy = MRI.getType(DstReg);
       if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {
         auto &CstVal = SrcMI->getOperand(1);
+        auto *MergedLocation = DILocation::getMergedLocation(
+            MI.getDebugLoc().get(), SrcMI->getDebugLoc().get());
+        // Set the debug location to the merged location of the SrcMI and the MI
+        // if the aext fold is successful.
+        Builder.setDebugLoc(MergedLocation);
         Builder.buildConstant(
             DstReg, CstVal.getCImm()->getValue().sext(DstTy.getSizeInBits()));
         UpdatedDefs.push_back(DstReg);
diff --git a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
index 551ba1e6036c..547529bbe699 100644
--- a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
@@ -51,6 +51,11 @@ CSEMIRBuilder::getDominatingInstrForID(FoldingSetNodeID &ID,
       // this builder will have the def ready.
       setInsertPt(*CurMBB, std::next(MII));
     } else if (!dominates(MI, CurrPos)) {
+      // Update the spliced machineinstr's debug location by merging it with the
+      // debug location of the instruction at the insertion point.
+      auto *Loc = DILocation::getMergedLocation(getDebugLoc().get(),
+                                                MI->getDebugLoc().get());
+      MI->setDebugLoc(Loc);
       CurMBB->splice(CurrPos, CurMBB, MI);
     }
     return MachineInstrBuilder(getMF(), MI);
diff --git a/llvm/test/DebugInfo/AArch64/merge-locations-legalizer.mir b/llvm/test/DebugInfo/AArch64/merge-locations-legalizer.mir
new file mode 100644
index 000000000000..3bdf87cea0e5
--- /dev/null
+++ b/llvm/test/DebugInfo/AArch64/merge-locations-legalizer.mir
@@ -0,0 +1,30 @@
+# This test checks to make sure that when an instruction (%3 in the test) is 
+# moved due to matching a result of a fold of two other instructions
+# (%1, and %2 in the test) in the legalizer, the DILocation of the 
+# instruction that is moved (%3) is updated appropriately.
+
+# RUN: llc %s -run-pass=legalizer -mtriple=aarch64 -o - | FileCheck %s 
+# CHECK-NOT: %2:_(s32) = G_CONSTANT i32 0, debug-location !DILocation(line: 13
+# CHECK: %2:_(s32) = G_CONSTANT i32 0, debug-location !DILocation(line: 0,
+--- |
+  
+  define i32 @main(i32 %0, ptr %1) #0 !dbg !57 {
+  entry:
+    ret i32 0, !dbg !71
+  }
+  !3 = !DIFile(filename: "main.swift", directory: "/Volumes/Data/swift")
+  !23 = distinct !DICompileUnit(language: DW_LANG_Swift, file: !3, sdk: "blah.sdk")
+  !57 = distinct !DISubprogram(name: "main", unit: !23)
+  !64 = distinct !DILexicalBlock(scope: !57, column: 1)
+  !66 = distinct !DILexicalBlock(scope: !64, column: 1)
+  !68 = !DILocation(line: 12, scope: !66)
+  !70 = distinct !DILexicalBlock(scope: !66, column: 1)
+  !71 = !DILocation(line: 13, scope: !70)
+name:            main
+body:             |
+  bb.0:
+    %1:_(s8) = G_CONSTANT i8 0, debug-location !68
+    %2:_(s32) = G_ANYEXT %1(s8), debug-location !68
+    $w2 = COPY %2(s32), debug-location !68
+    %3:_(s32) = G_CONSTANT i32 0, debug-location !71
+    $w0 = COPY %3(s32), debug-location !71
-- 
GitLab


From 72200fcc346bee1830d9e640e42d717a55acd74c Mon Sep 17 00:00:00 2001
From: Ryosuke Niwa <rniwa@webkit.org>
Date: Wed, 15 May 2024 18:16:39 -0700
Subject: [PATCH 0942/1206] [analyzer] Check C++ base or member initializer in
 WebKit checkers. (#92220)

Co-authored-by: Ryosuke Niwa <rniwa@apple.com>
---
 .../Checkers/WebKit/PtrTypesSemantics.cpp     | 10 ++++++++-
 .../Checkers/WebKit/uncounted-obj-arg.cpp     | 21 +++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index 950d35a090a3..5c797d523308 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -525,11 +525,19 @@ bool TrivialFunctionAnalysis::isTrivialImpl(
   if (!IsNew)
     return It->second;
 
+  TrivialFunctionAnalysisVisitor V(Cache);
+
+  if (auto *CtorDecl = dyn_cast<CXXConstructorDecl>(D)) {
+    for (auto *CtorInit : CtorDecl->inits()) {
+      if (!V.Visit(CtorInit->getInit()))
+        return false;
+    }
+  }
+
   const Stmt *Body = D->getBody();
   if (!Body)
     return false;
 
-  TrivialFunctionAnalysisVisitor V(Cache);
   bool Result = V.Visit(Body);
   if (Result)
     Cache[D] = true;
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
index ed37671df3d3..96986631726f 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
@@ -159,10 +159,13 @@ private:
   StorageType m_storage { 0 };
 };
 
+int atoi(const char* str);
+
 class Number {
 public:
   Number(int v) : v(v) { }
   Number(double);
+  Number(const char* str) : v(atoi(str)) { }
   Number operator+(const Number&);
   Number& operator++() { ++v; return *this; }
   Number operator++(int) { Number returnValue(v); ++v; return returnValue; }
@@ -173,9 +176,16 @@ private:
   int v;
 };
 
+class DerivedNumber : public Number {
+public:
+  DerivedNumber(char c) : Number(c - '0') { }
+  DerivedNumber(const char* str) : Number(atoi(str)) { }
+};
+
 class ComplexNumber {
 public:
   ComplexNumber() : realPart(0), complexPart(0) { }
+  ComplexNumber(int real, const char* str) : realPart(real), complexPart(str) { }
   ComplexNumber(const ComplexNumber&);
   ComplexNumber& operator++() { realPart.someMethod(); return *this; }
   ComplexNumber operator++(int);
@@ -311,6 +321,7 @@ public:
     return;
   }
   unsigned trivial60() { return ObjectWithNonTrivialDestructor { 5 }.value(); }
+  unsigned trivial61() { return DerivedNumber('7').value(); }
 
   static RefCounted& singleton() {
     static RefCounted s_RefCounted;
@@ -391,6 +402,9 @@ public:
   ComplexNumber nonTrivial18() { return +complex; }
   ComplexNumber* nonTrivial19() { return new ComplexNumber(complex); }
   unsigned nonTrivial20() { return ObjectWithMutatingDestructor { 7 }.value(); }
+  unsigned nonTrivial21() { return Number("123").value(); }
+  unsigned nonTrivial22() { return ComplexNumber(123, "456").real().value(); }
+  unsigned nonTrivial23() { return DerivedNumber("123").value(); }
 
   static unsigned s_v;
   unsigned v { 0 };
@@ -479,6 +493,7 @@ public:
     getFieldTrivial().trivial58(); // no-warning
     getFieldTrivial().trivial59(); // no-warning
     getFieldTrivial().trivial60(); // no-warning
+    getFieldTrivial().trivial61(); // no-warning
 
     RefCounted::singleton().trivial18(); // no-warning
     RefCounted::singleton().someFunction(); // no-warning
@@ -525,6 +540,12 @@ public:
     // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
     getFieldTrivial().nonTrivial20();
     // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    getFieldTrivial().nonTrivial21();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    getFieldTrivial().nonTrivial22();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    getFieldTrivial().nonTrivial23();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
   }
 };
 
-- 
GitLab


From f0b3654701bde1cf7821d60698b42383edaff9f3 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 16 May 2024 10:21:22 +0900
Subject: [PATCH 0943/1206] [LoopUnroll] Clamp PartialThreshold for large
 LoopMicroOpBufferSize (#67657)

The znver3/znver4 scheduler models are outliers, specifying very large
LoopMicroOpBufferSizes at 512, while typical values for other subtargets
are on the order of ~50. Even if this information is
micro-architecturally correct (*), this does not mean that we want to
runtime unroll all loops to a size that completely fills the loop
buffer. Unless this is the single hot loop in the entire application,
the massive code size increase will bust the micro-op and instruction
caches.

Protect against this by clamping to the default PartialThreshold of 150,
which is the same as the default full-unroll threshold and half the
aggressive full-unroll threshold. Allowing more partial unrolling than
full unrolling certainly does not make sense.

(*) I strongly doubt that this is actually correct -- I believe this may
derive from an incorrect reading of Agner Fog's micro-architecture
guide. The number 4096 that was originally used here is the size of the
general micro-op cache, not that of a loop buffer. A separate loop
buffer is not listed for the Zen microarchitecture. Comparing this to
the listing for Skylake, it has a 1536 micro-op buffer, but only a 64
micro-op loopback buffer, with a note that it's rarely fully utilized.
Our scheduling model specifies LoopMicroOpBufferSize of 50 in that case.
---
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      |   8 +-
 llvm/test/Transforms/LoopUnroll/X86/znver3.ll | 764 +-----------------
 2 files changed, 30 insertions(+), 742 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 2091432d4fe2..8dba6a641285 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -612,7 +612,13 @@ public:
     if (PartialUnrollingThreshold.getNumOccurrences() > 0)
       MaxOps = PartialUnrollingThreshold;
     else if (ST->getSchedModel().LoopMicroOpBufferSize > 0)
-      MaxOps = ST->getSchedModel().LoopMicroOpBufferSize;
+      // Upper bound by the default PartialThreshold, which is the same as
+      // the default full-unroll Threshold. Even if the loop micro-op buffer
+      // is very large, this does not mean that we want to unroll all loops
+      // to that length, as it would increase code size beyond the limits of
+      // what unrolling normally allows.
+      MaxOps = std::min(ST->getSchedModel().LoopMicroOpBufferSize,
+                        UP.PartialThreshold);
     else
       return;
 
diff --git a/llvm/test/Transforms/LoopUnroll/X86/znver3.ll b/llvm/test/Transforms/LoopUnroll/X86/znver3.ll
index 30389062a096..467c57906d88 100644
--- a/llvm/test/Transforms/LoopUnroll/X86/znver3.ll
+++ b/llvm/test/Transforms/LoopUnroll/X86/znver3.ll
@@ -9,8 +9,8 @@ define i32 @test(ptr %ary) "target-cpu"="znver3" {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_127:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_NEXT_127:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_31:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_NEXT_31:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[SUM_NEXT:%.*]] = add nsw i32 [[VAL]], [[SUM]]
@@ -137,396 +137,12 @@ define i32 @test(ptr %ary) "target-cpu"="znver3" {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_30:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 31
 ; CHECK-NEXT:    [[ARRAYIDX_31:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_30]]
 ; CHECK-NEXT:    [[VAL_31:%.*]] = load i32, ptr [[ARRAYIDX_31]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_31:%.*]] = add nsw i32 [[VAL_31]], [[SUM_NEXT_30]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_31:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 32
-; CHECK-NEXT:    [[ARRAYIDX_32:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_31]]
-; CHECK-NEXT:    [[VAL_32:%.*]] = load i32, ptr [[ARRAYIDX_32]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_32:%.*]] = add nsw i32 [[VAL_32]], [[SUM_NEXT_31]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_32:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 33
-; CHECK-NEXT:    [[ARRAYIDX_33:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_32]]
-; CHECK-NEXT:    [[VAL_33:%.*]] = load i32, ptr [[ARRAYIDX_33]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_33:%.*]] = add nsw i32 [[VAL_33]], [[SUM_NEXT_32]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_33:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 34
-; CHECK-NEXT:    [[ARRAYIDX_34:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_33]]
-; CHECK-NEXT:    [[VAL_34:%.*]] = load i32, ptr [[ARRAYIDX_34]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_34:%.*]] = add nsw i32 [[VAL_34]], [[SUM_NEXT_33]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_34:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 35
-; CHECK-NEXT:    [[ARRAYIDX_35:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_34]]
-; CHECK-NEXT:    [[VAL_35:%.*]] = load i32, ptr [[ARRAYIDX_35]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_35:%.*]] = add nsw i32 [[VAL_35]], [[SUM_NEXT_34]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_35:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 36
-; CHECK-NEXT:    [[ARRAYIDX_36:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_35]]
-; CHECK-NEXT:    [[VAL_36:%.*]] = load i32, ptr [[ARRAYIDX_36]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_36:%.*]] = add nsw i32 [[VAL_36]], [[SUM_NEXT_35]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_36:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 37
-; CHECK-NEXT:    [[ARRAYIDX_37:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_36]]
-; CHECK-NEXT:    [[VAL_37:%.*]] = load i32, ptr [[ARRAYIDX_37]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_37:%.*]] = add nsw i32 [[VAL_37]], [[SUM_NEXT_36]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_37:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 38
-; CHECK-NEXT:    [[ARRAYIDX_38:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_37]]
-; CHECK-NEXT:    [[VAL_38:%.*]] = load i32, ptr [[ARRAYIDX_38]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_38:%.*]] = add nsw i32 [[VAL_38]], [[SUM_NEXT_37]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_38:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 39
-; CHECK-NEXT:    [[ARRAYIDX_39:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_38]]
-; CHECK-NEXT:    [[VAL_39:%.*]] = load i32, ptr [[ARRAYIDX_39]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_39:%.*]] = add nsw i32 [[VAL_39]], [[SUM_NEXT_38]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_39:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 40
-; CHECK-NEXT:    [[ARRAYIDX_40:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_39]]
-; CHECK-NEXT:    [[VAL_40:%.*]] = load i32, ptr [[ARRAYIDX_40]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_40:%.*]] = add nsw i32 [[VAL_40]], [[SUM_NEXT_39]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_40:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 41
-; CHECK-NEXT:    [[ARRAYIDX_41:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_40]]
-; CHECK-NEXT:    [[VAL_41:%.*]] = load i32, ptr [[ARRAYIDX_41]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_41:%.*]] = add nsw i32 [[VAL_41]], [[SUM_NEXT_40]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_41:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 42
-; CHECK-NEXT:    [[ARRAYIDX_42:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_41]]
-; CHECK-NEXT:    [[VAL_42:%.*]] = load i32, ptr [[ARRAYIDX_42]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_42:%.*]] = add nsw i32 [[VAL_42]], [[SUM_NEXT_41]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_42:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 43
-; CHECK-NEXT:    [[ARRAYIDX_43:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_42]]
-; CHECK-NEXT:    [[VAL_43:%.*]] = load i32, ptr [[ARRAYIDX_43]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_43:%.*]] = add nsw i32 [[VAL_43]], [[SUM_NEXT_42]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_43:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 44
-; CHECK-NEXT:    [[ARRAYIDX_44:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_43]]
-; CHECK-NEXT:    [[VAL_44:%.*]] = load i32, ptr [[ARRAYIDX_44]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_44:%.*]] = add nsw i32 [[VAL_44]], [[SUM_NEXT_43]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_44:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 45
-; CHECK-NEXT:    [[ARRAYIDX_45:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_44]]
-; CHECK-NEXT:    [[VAL_45:%.*]] = load i32, ptr [[ARRAYIDX_45]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_45:%.*]] = add nsw i32 [[VAL_45]], [[SUM_NEXT_44]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_45:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 46
-; CHECK-NEXT:    [[ARRAYIDX_46:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_45]]
-; CHECK-NEXT:    [[VAL_46:%.*]] = load i32, ptr [[ARRAYIDX_46]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_46:%.*]] = add nsw i32 [[VAL_46]], [[SUM_NEXT_45]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_46:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 47
-; CHECK-NEXT:    [[ARRAYIDX_47:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_46]]
-; CHECK-NEXT:    [[VAL_47:%.*]] = load i32, ptr [[ARRAYIDX_47]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_47:%.*]] = add nsw i32 [[VAL_47]], [[SUM_NEXT_46]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_47:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 48
-; CHECK-NEXT:    [[ARRAYIDX_48:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_47]]
-; CHECK-NEXT:    [[VAL_48:%.*]] = load i32, ptr [[ARRAYIDX_48]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_48:%.*]] = add nsw i32 [[VAL_48]], [[SUM_NEXT_47]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_48:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 49
-; CHECK-NEXT:    [[ARRAYIDX_49:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_48]]
-; CHECK-NEXT:    [[VAL_49:%.*]] = load i32, ptr [[ARRAYIDX_49]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_49:%.*]] = add nsw i32 [[VAL_49]], [[SUM_NEXT_48]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_49:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 50
-; CHECK-NEXT:    [[ARRAYIDX_50:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_49]]
-; CHECK-NEXT:    [[VAL_50:%.*]] = load i32, ptr [[ARRAYIDX_50]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_50:%.*]] = add nsw i32 [[VAL_50]], [[SUM_NEXT_49]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_50:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 51
-; CHECK-NEXT:    [[ARRAYIDX_51:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_50]]
-; CHECK-NEXT:    [[VAL_51:%.*]] = load i32, ptr [[ARRAYIDX_51]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_51:%.*]] = add nsw i32 [[VAL_51]], [[SUM_NEXT_50]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_51:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 52
-; CHECK-NEXT:    [[ARRAYIDX_52:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_51]]
-; CHECK-NEXT:    [[VAL_52:%.*]] = load i32, ptr [[ARRAYIDX_52]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_52:%.*]] = add nsw i32 [[VAL_52]], [[SUM_NEXT_51]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_52:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 53
-; CHECK-NEXT:    [[ARRAYIDX_53:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_52]]
-; CHECK-NEXT:    [[VAL_53:%.*]] = load i32, ptr [[ARRAYIDX_53]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_53:%.*]] = add nsw i32 [[VAL_53]], [[SUM_NEXT_52]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_53:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 54
-; CHECK-NEXT:    [[ARRAYIDX_54:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_53]]
-; CHECK-NEXT:    [[VAL_54:%.*]] = load i32, ptr [[ARRAYIDX_54]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_54:%.*]] = add nsw i32 [[VAL_54]], [[SUM_NEXT_53]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_54:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 55
-; CHECK-NEXT:    [[ARRAYIDX_55:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_54]]
-; CHECK-NEXT:    [[VAL_55:%.*]] = load i32, ptr [[ARRAYIDX_55]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_55:%.*]] = add nsw i32 [[VAL_55]], [[SUM_NEXT_54]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_55:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 56
-; CHECK-NEXT:    [[ARRAYIDX_56:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_55]]
-; CHECK-NEXT:    [[VAL_56:%.*]] = load i32, ptr [[ARRAYIDX_56]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_56:%.*]] = add nsw i32 [[VAL_56]], [[SUM_NEXT_55]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_56:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 57
-; CHECK-NEXT:    [[ARRAYIDX_57:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_56]]
-; CHECK-NEXT:    [[VAL_57:%.*]] = load i32, ptr [[ARRAYIDX_57]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_57:%.*]] = add nsw i32 [[VAL_57]], [[SUM_NEXT_56]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_57:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 58
-; CHECK-NEXT:    [[ARRAYIDX_58:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_57]]
-; CHECK-NEXT:    [[VAL_58:%.*]] = load i32, ptr [[ARRAYIDX_58]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_58:%.*]] = add nsw i32 [[VAL_58]], [[SUM_NEXT_57]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_58:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 59
-; CHECK-NEXT:    [[ARRAYIDX_59:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_58]]
-; CHECK-NEXT:    [[VAL_59:%.*]] = load i32, ptr [[ARRAYIDX_59]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_59:%.*]] = add nsw i32 [[VAL_59]], [[SUM_NEXT_58]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_59:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 60
-; CHECK-NEXT:    [[ARRAYIDX_60:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_59]]
-; CHECK-NEXT:    [[VAL_60:%.*]] = load i32, ptr [[ARRAYIDX_60]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_60:%.*]] = add nsw i32 [[VAL_60]], [[SUM_NEXT_59]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_60:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 61
-; CHECK-NEXT:    [[ARRAYIDX_61:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_60]]
-; CHECK-NEXT:    [[VAL_61:%.*]] = load i32, ptr [[ARRAYIDX_61]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_61:%.*]] = add nsw i32 [[VAL_61]], [[SUM_NEXT_60]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_61:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 62
-; CHECK-NEXT:    [[ARRAYIDX_62:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_61]]
-; CHECK-NEXT:    [[VAL_62:%.*]] = load i32, ptr [[ARRAYIDX_62]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_62:%.*]] = add nsw i32 [[VAL_62]], [[SUM_NEXT_61]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_62:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 63
-; CHECK-NEXT:    [[ARRAYIDX_63:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_62]]
-; CHECK-NEXT:    [[VAL_63:%.*]] = load i32, ptr [[ARRAYIDX_63]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_63:%.*]] = add nsw i32 [[VAL_63]], [[SUM_NEXT_62]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_63:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 64
-; CHECK-NEXT:    [[ARRAYIDX_64:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_63]]
-; CHECK-NEXT:    [[VAL_64:%.*]] = load i32, ptr [[ARRAYIDX_64]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_64:%.*]] = add nsw i32 [[VAL_64]], [[SUM_NEXT_63]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_64:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 65
-; CHECK-NEXT:    [[ARRAYIDX_65:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_64]]
-; CHECK-NEXT:    [[VAL_65:%.*]] = load i32, ptr [[ARRAYIDX_65]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_65:%.*]] = add nsw i32 [[VAL_65]], [[SUM_NEXT_64]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_65:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 66
-; CHECK-NEXT:    [[ARRAYIDX_66:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_65]]
-; CHECK-NEXT:    [[VAL_66:%.*]] = load i32, ptr [[ARRAYIDX_66]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_66:%.*]] = add nsw i32 [[VAL_66]], [[SUM_NEXT_65]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_66:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 67
-; CHECK-NEXT:    [[ARRAYIDX_67:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_66]]
-; CHECK-NEXT:    [[VAL_67:%.*]] = load i32, ptr [[ARRAYIDX_67]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_67:%.*]] = add nsw i32 [[VAL_67]], [[SUM_NEXT_66]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_67:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 68
-; CHECK-NEXT:    [[ARRAYIDX_68:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_67]]
-; CHECK-NEXT:    [[VAL_68:%.*]] = load i32, ptr [[ARRAYIDX_68]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_68:%.*]] = add nsw i32 [[VAL_68]], [[SUM_NEXT_67]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_68:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 69
-; CHECK-NEXT:    [[ARRAYIDX_69:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_68]]
-; CHECK-NEXT:    [[VAL_69:%.*]] = load i32, ptr [[ARRAYIDX_69]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_69:%.*]] = add nsw i32 [[VAL_69]], [[SUM_NEXT_68]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_69:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 70
-; CHECK-NEXT:    [[ARRAYIDX_70:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_69]]
-; CHECK-NEXT:    [[VAL_70:%.*]] = load i32, ptr [[ARRAYIDX_70]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_70:%.*]] = add nsw i32 [[VAL_70]], [[SUM_NEXT_69]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_70:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 71
-; CHECK-NEXT:    [[ARRAYIDX_71:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_70]]
-; CHECK-NEXT:    [[VAL_71:%.*]] = load i32, ptr [[ARRAYIDX_71]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_71:%.*]] = add nsw i32 [[VAL_71]], [[SUM_NEXT_70]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_71:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 72
-; CHECK-NEXT:    [[ARRAYIDX_72:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_71]]
-; CHECK-NEXT:    [[VAL_72:%.*]] = load i32, ptr [[ARRAYIDX_72]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_72:%.*]] = add nsw i32 [[VAL_72]], [[SUM_NEXT_71]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_72:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 73
-; CHECK-NEXT:    [[ARRAYIDX_73:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_72]]
-; CHECK-NEXT:    [[VAL_73:%.*]] = load i32, ptr [[ARRAYIDX_73]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_73:%.*]] = add nsw i32 [[VAL_73]], [[SUM_NEXT_72]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_73:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 74
-; CHECK-NEXT:    [[ARRAYIDX_74:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_73]]
-; CHECK-NEXT:    [[VAL_74:%.*]] = load i32, ptr [[ARRAYIDX_74]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_74:%.*]] = add nsw i32 [[VAL_74]], [[SUM_NEXT_73]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_74:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 75
-; CHECK-NEXT:    [[ARRAYIDX_75:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_74]]
-; CHECK-NEXT:    [[VAL_75:%.*]] = load i32, ptr [[ARRAYIDX_75]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_75:%.*]] = add nsw i32 [[VAL_75]], [[SUM_NEXT_74]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_75:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 76
-; CHECK-NEXT:    [[ARRAYIDX_76:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_75]]
-; CHECK-NEXT:    [[VAL_76:%.*]] = load i32, ptr [[ARRAYIDX_76]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_76:%.*]] = add nsw i32 [[VAL_76]], [[SUM_NEXT_75]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_76:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 77
-; CHECK-NEXT:    [[ARRAYIDX_77:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_76]]
-; CHECK-NEXT:    [[VAL_77:%.*]] = load i32, ptr [[ARRAYIDX_77]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_77:%.*]] = add nsw i32 [[VAL_77]], [[SUM_NEXT_76]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_77:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 78
-; CHECK-NEXT:    [[ARRAYIDX_78:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_77]]
-; CHECK-NEXT:    [[VAL_78:%.*]] = load i32, ptr [[ARRAYIDX_78]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_78:%.*]] = add nsw i32 [[VAL_78]], [[SUM_NEXT_77]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_78:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 79
-; CHECK-NEXT:    [[ARRAYIDX_79:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_78]]
-; CHECK-NEXT:    [[VAL_79:%.*]] = load i32, ptr [[ARRAYIDX_79]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_79:%.*]] = add nsw i32 [[VAL_79]], [[SUM_NEXT_78]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_79:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 80
-; CHECK-NEXT:    [[ARRAYIDX_80:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_79]]
-; CHECK-NEXT:    [[VAL_80:%.*]] = load i32, ptr [[ARRAYIDX_80]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_80:%.*]] = add nsw i32 [[VAL_80]], [[SUM_NEXT_79]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_80:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 81
-; CHECK-NEXT:    [[ARRAYIDX_81:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_80]]
-; CHECK-NEXT:    [[VAL_81:%.*]] = load i32, ptr [[ARRAYIDX_81]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_81:%.*]] = add nsw i32 [[VAL_81]], [[SUM_NEXT_80]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_81:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 82
-; CHECK-NEXT:    [[ARRAYIDX_82:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_81]]
-; CHECK-NEXT:    [[VAL_82:%.*]] = load i32, ptr [[ARRAYIDX_82]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_82:%.*]] = add nsw i32 [[VAL_82]], [[SUM_NEXT_81]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_82:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 83
-; CHECK-NEXT:    [[ARRAYIDX_83:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_82]]
-; CHECK-NEXT:    [[VAL_83:%.*]] = load i32, ptr [[ARRAYIDX_83]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_83:%.*]] = add nsw i32 [[VAL_83]], [[SUM_NEXT_82]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_83:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 84
-; CHECK-NEXT:    [[ARRAYIDX_84:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_83]]
-; CHECK-NEXT:    [[VAL_84:%.*]] = load i32, ptr [[ARRAYIDX_84]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_84:%.*]] = add nsw i32 [[VAL_84]], [[SUM_NEXT_83]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_84:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 85
-; CHECK-NEXT:    [[ARRAYIDX_85:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_84]]
-; CHECK-NEXT:    [[VAL_85:%.*]] = load i32, ptr [[ARRAYIDX_85]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_85:%.*]] = add nsw i32 [[VAL_85]], [[SUM_NEXT_84]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_85:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 86
-; CHECK-NEXT:    [[ARRAYIDX_86:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_85]]
-; CHECK-NEXT:    [[VAL_86:%.*]] = load i32, ptr [[ARRAYIDX_86]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_86:%.*]] = add nsw i32 [[VAL_86]], [[SUM_NEXT_85]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_86:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 87
-; CHECK-NEXT:    [[ARRAYIDX_87:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_86]]
-; CHECK-NEXT:    [[VAL_87:%.*]] = load i32, ptr [[ARRAYIDX_87]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_87:%.*]] = add nsw i32 [[VAL_87]], [[SUM_NEXT_86]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_87:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 88
-; CHECK-NEXT:    [[ARRAYIDX_88:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_87]]
-; CHECK-NEXT:    [[VAL_88:%.*]] = load i32, ptr [[ARRAYIDX_88]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_88:%.*]] = add nsw i32 [[VAL_88]], [[SUM_NEXT_87]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_88:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 89
-; CHECK-NEXT:    [[ARRAYIDX_89:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_88]]
-; CHECK-NEXT:    [[VAL_89:%.*]] = load i32, ptr [[ARRAYIDX_89]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_89:%.*]] = add nsw i32 [[VAL_89]], [[SUM_NEXT_88]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_89:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 90
-; CHECK-NEXT:    [[ARRAYIDX_90:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_89]]
-; CHECK-NEXT:    [[VAL_90:%.*]] = load i32, ptr [[ARRAYIDX_90]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_90:%.*]] = add nsw i32 [[VAL_90]], [[SUM_NEXT_89]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_90:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 91
-; CHECK-NEXT:    [[ARRAYIDX_91:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_90]]
-; CHECK-NEXT:    [[VAL_91:%.*]] = load i32, ptr [[ARRAYIDX_91]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_91:%.*]] = add nsw i32 [[VAL_91]], [[SUM_NEXT_90]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_91:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 92
-; CHECK-NEXT:    [[ARRAYIDX_92:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_91]]
-; CHECK-NEXT:    [[VAL_92:%.*]] = load i32, ptr [[ARRAYIDX_92]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_92:%.*]] = add nsw i32 [[VAL_92]], [[SUM_NEXT_91]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_92:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 93
-; CHECK-NEXT:    [[ARRAYIDX_93:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_92]]
-; CHECK-NEXT:    [[VAL_93:%.*]] = load i32, ptr [[ARRAYIDX_93]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_93:%.*]] = add nsw i32 [[VAL_93]], [[SUM_NEXT_92]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_93:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 94
-; CHECK-NEXT:    [[ARRAYIDX_94:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_93]]
-; CHECK-NEXT:    [[VAL_94:%.*]] = load i32, ptr [[ARRAYIDX_94]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_94:%.*]] = add nsw i32 [[VAL_94]], [[SUM_NEXT_93]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_94:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 95
-; CHECK-NEXT:    [[ARRAYIDX_95:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_94]]
-; CHECK-NEXT:    [[VAL_95:%.*]] = load i32, ptr [[ARRAYIDX_95]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_95:%.*]] = add nsw i32 [[VAL_95]], [[SUM_NEXT_94]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_95:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 96
-; CHECK-NEXT:    [[ARRAYIDX_96:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_95]]
-; CHECK-NEXT:    [[VAL_96:%.*]] = load i32, ptr [[ARRAYIDX_96]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_96:%.*]] = add nsw i32 [[VAL_96]], [[SUM_NEXT_95]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_96:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 97
-; CHECK-NEXT:    [[ARRAYIDX_97:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_96]]
-; CHECK-NEXT:    [[VAL_97:%.*]] = load i32, ptr [[ARRAYIDX_97]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_97:%.*]] = add nsw i32 [[VAL_97]], [[SUM_NEXT_96]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_97:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 98
-; CHECK-NEXT:    [[ARRAYIDX_98:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_97]]
-; CHECK-NEXT:    [[VAL_98:%.*]] = load i32, ptr [[ARRAYIDX_98]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_98:%.*]] = add nsw i32 [[VAL_98]], [[SUM_NEXT_97]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_98:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 99
-; CHECK-NEXT:    [[ARRAYIDX_99:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_98]]
-; CHECK-NEXT:    [[VAL_99:%.*]] = load i32, ptr [[ARRAYIDX_99]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_99:%.*]] = add nsw i32 [[VAL_99]], [[SUM_NEXT_98]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_99:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 100
-; CHECK-NEXT:    [[ARRAYIDX_100:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_99]]
-; CHECK-NEXT:    [[VAL_100:%.*]] = load i32, ptr [[ARRAYIDX_100]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_100:%.*]] = add nsw i32 [[VAL_100]], [[SUM_NEXT_99]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_100:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 101
-; CHECK-NEXT:    [[ARRAYIDX_101:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_100]]
-; CHECK-NEXT:    [[VAL_101:%.*]] = load i32, ptr [[ARRAYIDX_101]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_101:%.*]] = add nsw i32 [[VAL_101]], [[SUM_NEXT_100]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_101:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 102
-; CHECK-NEXT:    [[ARRAYIDX_102:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_101]]
-; CHECK-NEXT:    [[VAL_102:%.*]] = load i32, ptr [[ARRAYIDX_102]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_102:%.*]] = add nsw i32 [[VAL_102]], [[SUM_NEXT_101]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_102:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 103
-; CHECK-NEXT:    [[ARRAYIDX_103:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_102]]
-; CHECK-NEXT:    [[VAL_103:%.*]] = load i32, ptr [[ARRAYIDX_103]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_103:%.*]] = add nsw i32 [[VAL_103]], [[SUM_NEXT_102]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_103:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 104
-; CHECK-NEXT:    [[ARRAYIDX_104:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_103]]
-; CHECK-NEXT:    [[VAL_104:%.*]] = load i32, ptr [[ARRAYIDX_104]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_104:%.*]] = add nsw i32 [[VAL_104]], [[SUM_NEXT_103]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_104:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 105
-; CHECK-NEXT:    [[ARRAYIDX_105:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_104]]
-; CHECK-NEXT:    [[VAL_105:%.*]] = load i32, ptr [[ARRAYIDX_105]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_105:%.*]] = add nsw i32 [[VAL_105]], [[SUM_NEXT_104]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_105:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 106
-; CHECK-NEXT:    [[ARRAYIDX_106:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_105]]
-; CHECK-NEXT:    [[VAL_106:%.*]] = load i32, ptr [[ARRAYIDX_106]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_106:%.*]] = add nsw i32 [[VAL_106]], [[SUM_NEXT_105]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_106:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 107
-; CHECK-NEXT:    [[ARRAYIDX_107:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_106]]
-; CHECK-NEXT:    [[VAL_107:%.*]] = load i32, ptr [[ARRAYIDX_107]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_107:%.*]] = add nsw i32 [[VAL_107]], [[SUM_NEXT_106]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_107:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 108
-; CHECK-NEXT:    [[ARRAYIDX_108:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_107]]
-; CHECK-NEXT:    [[VAL_108:%.*]] = load i32, ptr [[ARRAYIDX_108]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_108:%.*]] = add nsw i32 [[VAL_108]], [[SUM_NEXT_107]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_108:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 109
-; CHECK-NEXT:    [[ARRAYIDX_109:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_108]]
-; CHECK-NEXT:    [[VAL_109:%.*]] = load i32, ptr [[ARRAYIDX_109]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_109:%.*]] = add nsw i32 [[VAL_109]], [[SUM_NEXT_108]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_109:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 110
-; CHECK-NEXT:    [[ARRAYIDX_110:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_109]]
-; CHECK-NEXT:    [[VAL_110:%.*]] = load i32, ptr [[ARRAYIDX_110]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_110:%.*]] = add nsw i32 [[VAL_110]], [[SUM_NEXT_109]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_110:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 111
-; CHECK-NEXT:    [[ARRAYIDX_111:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_110]]
-; CHECK-NEXT:    [[VAL_111:%.*]] = load i32, ptr [[ARRAYIDX_111]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_111:%.*]] = add nsw i32 [[VAL_111]], [[SUM_NEXT_110]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_111:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 112
-; CHECK-NEXT:    [[ARRAYIDX_112:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_111]]
-; CHECK-NEXT:    [[VAL_112:%.*]] = load i32, ptr [[ARRAYIDX_112]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_112:%.*]] = add nsw i32 [[VAL_112]], [[SUM_NEXT_111]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_112:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 113
-; CHECK-NEXT:    [[ARRAYIDX_113:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_112]]
-; CHECK-NEXT:    [[VAL_113:%.*]] = load i32, ptr [[ARRAYIDX_113]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_113:%.*]] = add nsw i32 [[VAL_113]], [[SUM_NEXT_112]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_113:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 114
-; CHECK-NEXT:    [[ARRAYIDX_114:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_113]]
-; CHECK-NEXT:    [[VAL_114:%.*]] = load i32, ptr [[ARRAYIDX_114]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_114:%.*]] = add nsw i32 [[VAL_114]], [[SUM_NEXT_113]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_114:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 115
-; CHECK-NEXT:    [[ARRAYIDX_115:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_114]]
-; CHECK-NEXT:    [[VAL_115:%.*]] = load i32, ptr [[ARRAYIDX_115]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_115:%.*]] = add nsw i32 [[VAL_115]], [[SUM_NEXT_114]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_115:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 116
-; CHECK-NEXT:    [[ARRAYIDX_116:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_115]]
-; CHECK-NEXT:    [[VAL_116:%.*]] = load i32, ptr [[ARRAYIDX_116]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_116:%.*]] = add nsw i32 [[VAL_116]], [[SUM_NEXT_115]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_116:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 117
-; CHECK-NEXT:    [[ARRAYIDX_117:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_116]]
-; CHECK-NEXT:    [[VAL_117:%.*]] = load i32, ptr [[ARRAYIDX_117]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_117:%.*]] = add nsw i32 [[VAL_117]], [[SUM_NEXT_116]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_117:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 118
-; CHECK-NEXT:    [[ARRAYIDX_118:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_117]]
-; CHECK-NEXT:    [[VAL_118:%.*]] = load i32, ptr [[ARRAYIDX_118]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_118:%.*]] = add nsw i32 [[VAL_118]], [[SUM_NEXT_117]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_118:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 119
-; CHECK-NEXT:    [[ARRAYIDX_119:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_118]]
-; CHECK-NEXT:    [[VAL_119:%.*]] = load i32, ptr [[ARRAYIDX_119]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_119:%.*]] = add nsw i32 [[VAL_119]], [[SUM_NEXT_118]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_119:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 120
-; CHECK-NEXT:    [[ARRAYIDX_120:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_119]]
-; CHECK-NEXT:    [[VAL_120:%.*]] = load i32, ptr [[ARRAYIDX_120]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_120:%.*]] = add nsw i32 [[VAL_120]], [[SUM_NEXT_119]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_120:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 121
-; CHECK-NEXT:    [[ARRAYIDX_121:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_120]]
-; CHECK-NEXT:    [[VAL_121:%.*]] = load i32, ptr [[ARRAYIDX_121]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_121:%.*]] = add nsw i32 [[VAL_121]], [[SUM_NEXT_120]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_121:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 122
-; CHECK-NEXT:    [[ARRAYIDX_122:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_121]]
-; CHECK-NEXT:    [[VAL_122:%.*]] = load i32, ptr [[ARRAYIDX_122]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_122:%.*]] = add nsw i32 [[VAL_122]], [[SUM_NEXT_121]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_122:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 123
-; CHECK-NEXT:    [[ARRAYIDX_123:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_122]]
-; CHECK-NEXT:    [[VAL_123:%.*]] = load i32, ptr [[ARRAYIDX_123]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_123:%.*]] = add nsw i32 [[VAL_123]], [[SUM_NEXT_122]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_123:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 124
-; CHECK-NEXT:    [[ARRAYIDX_124:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_123]]
-; CHECK-NEXT:    [[VAL_124:%.*]] = load i32, ptr [[ARRAYIDX_124]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_124:%.*]] = add nsw i32 [[VAL_124]], [[SUM_NEXT_123]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_124:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 125
-; CHECK-NEXT:    [[ARRAYIDX_125:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_124]]
-; CHECK-NEXT:    [[VAL_125:%.*]] = load i32, ptr [[ARRAYIDX_125]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_125:%.*]] = add nsw i32 [[VAL_125]], [[SUM_NEXT_124]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_125:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 126
-; CHECK-NEXT:    [[ARRAYIDX_126:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_125]]
-; CHECK-NEXT:    [[VAL_126:%.*]] = load i32, ptr [[ARRAYIDX_126]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_126:%.*]] = add nsw i32 [[VAL_126]], [[SUM_NEXT_125]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_126:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 127
-; CHECK-NEXT:    [[ARRAYIDX_127:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_126]]
-; CHECK-NEXT:    [[VAL_127:%.*]] = load i32, ptr [[ARRAYIDX_127]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_127]] = add nsw i32 [[VAL_127]], [[SUM_NEXT_126]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_127]] = add nuw nsw i64 [[INDVARS_IV]], 128
-; CHECK-NEXT:    [[EXITCOND_NOT_127:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_127]], 8192
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT_127]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    [[SUM_NEXT_31]] = add nsw i32 [[VAL_31]], [[SUM_NEXT_30]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_31]] = add nuw nsw i64 [[INDVARS_IV]], 32
+; CHECK-NEXT:    [[EXITCOND_NOT_31:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_31]], 8192
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT_31]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT_127]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT_31]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_NEXT_LCSSA]]
 ;
 entry:
@@ -551,16 +167,16 @@ define i32 @test2(ptr %ary, i64 %n) "target-cpu"="znver3" {
 ; CHECK-SAME: ptr [[ARY:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -1
-; CHECK-NEXT:    [[XTRAITER:%.*]] = and i64 [[N]], 7
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 7
+; CHECK-NEXT:    [[XTRAITER:%.*]] = and i64 [[N]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[FOR_COND_CLEANUP_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]]
 ; CHECK:       entry.new:
 ; CHECK-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[INDVARS_IV_NEXT_7:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[ENTRY_NEW]] ], [ [[SUM_NEXT_7:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[NITER:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[NITER_NEXT_7:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[ENTRY_NEW]] ], [ [[SUM_NEXT_1:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[NITER:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[DUMMY1:%.*]] = mul i32 [[VAL]], [[VAL]]
@@ -667,339 +283,15 @@ define i32 @test2(ptr %ary, i64 %n) "target-cpu"="znver3" {
 ; CHECK-NEXT:    [[DUMMY48_1:%.*]] = mul i32 [[DUMMY47_1]], [[DUMMY47_1]]
 ; CHECK-NEXT:    [[DUMMY49_1:%.*]] = mul i32 [[DUMMY48_1]], [[DUMMY48_1]]
 ; CHECK-NEXT:    [[DUMMY50_1:%.*]] = mul i32 [[DUMMY49_1]], [[DUMMY49_1]]
-; CHECK-NEXT:    [[SUM_NEXT_1:%.*]] = add nsw i32 [[DUMMY50_1]], [[SUM_NEXT]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
-; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_1]]
-; CHECK-NEXT:    [[VAL_2:%.*]] = load i32, ptr [[ARRAYIDX_2]], align 4
-; CHECK-NEXT:    [[DUMMY1_2:%.*]] = mul i32 [[VAL_2]], [[VAL_2]]
-; CHECK-NEXT:    [[DUMMY2_2:%.*]] = mul i32 [[DUMMY1_2]], [[DUMMY1_2]]
-; CHECK-NEXT:    [[DUMMY3_2:%.*]] = mul i32 [[DUMMY2_2]], [[DUMMY2_2]]
-; CHECK-NEXT:    [[DUMMY4_2:%.*]] = mul i32 [[DUMMY3_2]], [[DUMMY3_2]]
-; CHECK-NEXT:    [[DUMMY5_2:%.*]] = mul i32 [[DUMMY4_2]], [[DUMMY4_2]]
-; CHECK-NEXT:    [[DUMMY6_2:%.*]] = mul i32 [[DUMMY5_2]], [[DUMMY5_2]]
-; CHECK-NEXT:    [[DUMMY7_2:%.*]] = mul i32 [[DUMMY6_2]], [[DUMMY6_2]]
-; CHECK-NEXT:    [[DUMMY8_2:%.*]] = mul i32 [[DUMMY7_2]], [[DUMMY7_2]]
-; CHECK-NEXT:    [[DUMMY9_2:%.*]] = mul i32 [[DUMMY8_2]], [[DUMMY8_2]]
-; CHECK-NEXT:    [[DUMMY10_2:%.*]] = mul i32 [[DUMMY9_2]], [[DUMMY9_2]]
-; CHECK-NEXT:    [[DUMMY11_2:%.*]] = mul i32 [[DUMMY10_2]], [[DUMMY10_2]]
-; CHECK-NEXT:    [[DUMMY12_2:%.*]] = mul i32 [[DUMMY11_2]], [[DUMMY11_2]]
-; CHECK-NEXT:    [[DUMMY13_2:%.*]] = mul i32 [[DUMMY12_2]], [[DUMMY12_2]]
-; CHECK-NEXT:    [[DUMMY14_2:%.*]] = mul i32 [[DUMMY13_2]], [[DUMMY13_2]]
-; CHECK-NEXT:    [[DUMMY15_2:%.*]] = mul i32 [[DUMMY14_2]], [[DUMMY14_2]]
-; CHECK-NEXT:    [[DUMMY16_2:%.*]] = mul i32 [[DUMMY15_2]], [[DUMMY15_2]]
-; CHECK-NEXT:    [[DUMMY17_2:%.*]] = mul i32 [[DUMMY16_2]], [[DUMMY16_2]]
-; CHECK-NEXT:    [[DUMMY18_2:%.*]] = mul i32 [[DUMMY17_2]], [[DUMMY17_2]]
-; CHECK-NEXT:    [[DUMMY19_2:%.*]] = mul i32 [[DUMMY18_2]], [[DUMMY18_2]]
-; CHECK-NEXT:    [[DUMMY20_2:%.*]] = mul i32 [[DUMMY19_2]], [[DUMMY19_2]]
-; CHECK-NEXT:    [[DUMMY21_2:%.*]] = mul i32 [[DUMMY20_2]], [[DUMMY20_2]]
-; CHECK-NEXT:    [[DUMMY22_2:%.*]] = mul i32 [[DUMMY21_2]], [[DUMMY21_2]]
-; CHECK-NEXT:    [[DUMMY23_2:%.*]] = mul i32 [[DUMMY22_2]], [[DUMMY22_2]]
-; CHECK-NEXT:    [[DUMMY24_2:%.*]] = mul i32 [[DUMMY23_2]], [[DUMMY23_2]]
-; CHECK-NEXT:    [[DUMMY25_2:%.*]] = mul i32 [[DUMMY24_2]], [[DUMMY24_2]]
-; CHECK-NEXT:    [[DUMMY26_2:%.*]] = mul i32 [[DUMMY25_2]], [[DUMMY25_2]]
-; CHECK-NEXT:    [[DUMMY27_2:%.*]] = mul i32 [[DUMMY26_2]], [[DUMMY26_2]]
-; CHECK-NEXT:    [[DUMMY28_2:%.*]] = mul i32 [[DUMMY27_2]], [[DUMMY27_2]]
-; CHECK-NEXT:    [[DUMMY29_2:%.*]] = mul i32 [[DUMMY28_2]], [[DUMMY28_2]]
-; CHECK-NEXT:    [[DUMMY30_2:%.*]] = mul i32 [[DUMMY29_2]], [[DUMMY29_2]]
-; CHECK-NEXT:    [[DUMMY31_2:%.*]] = mul i32 [[DUMMY30_2]], [[DUMMY30_2]]
-; CHECK-NEXT:    [[DUMMY32_2:%.*]] = mul i32 [[DUMMY31_2]], [[DUMMY31_2]]
-; CHECK-NEXT:    [[DUMMY33_2:%.*]] = mul i32 [[DUMMY32_2]], [[DUMMY32_2]]
-; CHECK-NEXT:    [[DUMMY34_2:%.*]] = mul i32 [[DUMMY33_2]], [[DUMMY33_2]]
-; CHECK-NEXT:    [[DUMMY35_2:%.*]] = mul i32 [[DUMMY34_2]], [[DUMMY34_2]]
-; CHECK-NEXT:    [[DUMMY36_2:%.*]] = mul i32 [[DUMMY35_2]], [[DUMMY35_2]]
-; CHECK-NEXT:    [[DUMMY37_2:%.*]] = mul i32 [[DUMMY36_2]], [[DUMMY36_2]]
-; CHECK-NEXT:    [[DUMMY38_2:%.*]] = mul i32 [[DUMMY37_2]], [[DUMMY37_2]]
-; CHECK-NEXT:    [[DUMMY39_2:%.*]] = mul i32 [[DUMMY38_2]], [[DUMMY38_2]]
-; CHECK-NEXT:    [[DUMMY40_2:%.*]] = mul i32 [[DUMMY39_2]], [[DUMMY39_2]]
-; CHECK-NEXT:    [[DUMMY41_2:%.*]] = mul i32 [[DUMMY40_2]], [[DUMMY40_2]]
-; CHECK-NEXT:    [[DUMMY42_2:%.*]] = mul i32 [[DUMMY41_2]], [[DUMMY41_2]]
-; CHECK-NEXT:    [[DUMMY43_2:%.*]] = mul i32 [[DUMMY42_2]], [[DUMMY42_2]]
-; CHECK-NEXT:    [[DUMMY44_2:%.*]] = mul i32 [[DUMMY43_2]], [[DUMMY43_2]]
-; CHECK-NEXT:    [[DUMMY45_2:%.*]] = mul i32 [[DUMMY44_2]], [[DUMMY44_2]]
-; CHECK-NEXT:    [[DUMMY46_2:%.*]] = mul i32 [[DUMMY45_2]], [[DUMMY45_2]]
-; CHECK-NEXT:    [[DUMMY47_2:%.*]] = mul i32 [[DUMMY46_2]], [[DUMMY46_2]]
-; CHECK-NEXT:    [[DUMMY48_2:%.*]] = mul i32 [[DUMMY47_2]], [[DUMMY47_2]]
-; CHECK-NEXT:    [[DUMMY49_2:%.*]] = mul i32 [[DUMMY48_2]], [[DUMMY48_2]]
-; CHECK-NEXT:    [[DUMMY50_2:%.*]] = mul i32 [[DUMMY49_2]], [[DUMMY49_2]]
-; CHECK-NEXT:    [[SUM_NEXT_2:%.*]] = add nsw i32 [[DUMMY50_2]], [[SUM_NEXT_1]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
-; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_2]]
-; CHECK-NEXT:    [[VAL_3:%.*]] = load i32, ptr [[ARRAYIDX_3]], align 4
-; CHECK-NEXT:    [[DUMMY1_3:%.*]] = mul i32 [[VAL_3]], [[VAL_3]]
-; CHECK-NEXT:    [[DUMMY2_3:%.*]] = mul i32 [[DUMMY1_3]], [[DUMMY1_3]]
-; CHECK-NEXT:    [[DUMMY3_3:%.*]] = mul i32 [[DUMMY2_3]], [[DUMMY2_3]]
-; CHECK-NEXT:    [[DUMMY4_3:%.*]] = mul i32 [[DUMMY3_3]], [[DUMMY3_3]]
-; CHECK-NEXT:    [[DUMMY5_3:%.*]] = mul i32 [[DUMMY4_3]], [[DUMMY4_3]]
-; CHECK-NEXT:    [[DUMMY6_3:%.*]] = mul i32 [[DUMMY5_3]], [[DUMMY5_3]]
-; CHECK-NEXT:    [[DUMMY7_3:%.*]] = mul i32 [[DUMMY6_3]], [[DUMMY6_3]]
-; CHECK-NEXT:    [[DUMMY8_3:%.*]] = mul i32 [[DUMMY7_3]], [[DUMMY7_3]]
-; CHECK-NEXT:    [[DUMMY9_3:%.*]] = mul i32 [[DUMMY8_3]], [[DUMMY8_3]]
-; CHECK-NEXT:    [[DUMMY10_3:%.*]] = mul i32 [[DUMMY9_3]], [[DUMMY9_3]]
-; CHECK-NEXT:    [[DUMMY11_3:%.*]] = mul i32 [[DUMMY10_3]], [[DUMMY10_3]]
-; CHECK-NEXT:    [[DUMMY12_3:%.*]] = mul i32 [[DUMMY11_3]], [[DUMMY11_3]]
-; CHECK-NEXT:    [[DUMMY13_3:%.*]] = mul i32 [[DUMMY12_3]], [[DUMMY12_3]]
-; CHECK-NEXT:    [[DUMMY14_3:%.*]] = mul i32 [[DUMMY13_3]], [[DUMMY13_3]]
-; CHECK-NEXT:    [[DUMMY15_3:%.*]] = mul i32 [[DUMMY14_3]], [[DUMMY14_3]]
-; CHECK-NEXT:    [[DUMMY16_3:%.*]] = mul i32 [[DUMMY15_3]], [[DUMMY15_3]]
-; CHECK-NEXT:    [[DUMMY17_3:%.*]] = mul i32 [[DUMMY16_3]], [[DUMMY16_3]]
-; CHECK-NEXT:    [[DUMMY18_3:%.*]] = mul i32 [[DUMMY17_3]], [[DUMMY17_3]]
-; CHECK-NEXT:    [[DUMMY19_3:%.*]] = mul i32 [[DUMMY18_3]], [[DUMMY18_3]]
-; CHECK-NEXT:    [[DUMMY20_3:%.*]] = mul i32 [[DUMMY19_3]], [[DUMMY19_3]]
-; CHECK-NEXT:    [[DUMMY21_3:%.*]] = mul i32 [[DUMMY20_3]], [[DUMMY20_3]]
-; CHECK-NEXT:    [[DUMMY22_3:%.*]] = mul i32 [[DUMMY21_3]], [[DUMMY21_3]]
-; CHECK-NEXT:    [[DUMMY23_3:%.*]] = mul i32 [[DUMMY22_3]], [[DUMMY22_3]]
-; CHECK-NEXT:    [[DUMMY24_3:%.*]] = mul i32 [[DUMMY23_3]], [[DUMMY23_3]]
-; CHECK-NEXT:    [[DUMMY25_3:%.*]] = mul i32 [[DUMMY24_3]], [[DUMMY24_3]]
-; CHECK-NEXT:    [[DUMMY26_3:%.*]] = mul i32 [[DUMMY25_3]], [[DUMMY25_3]]
-; CHECK-NEXT:    [[DUMMY27_3:%.*]] = mul i32 [[DUMMY26_3]], [[DUMMY26_3]]
-; CHECK-NEXT:    [[DUMMY28_3:%.*]] = mul i32 [[DUMMY27_3]], [[DUMMY27_3]]
-; CHECK-NEXT:    [[DUMMY29_3:%.*]] = mul i32 [[DUMMY28_3]], [[DUMMY28_3]]
-; CHECK-NEXT:    [[DUMMY30_3:%.*]] = mul i32 [[DUMMY29_3]], [[DUMMY29_3]]
-; CHECK-NEXT:    [[DUMMY31_3:%.*]] = mul i32 [[DUMMY30_3]], [[DUMMY30_3]]
-; CHECK-NEXT:    [[DUMMY32_3:%.*]] = mul i32 [[DUMMY31_3]], [[DUMMY31_3]]
-; CHECK-NEXT:    [[DUMMY33_3:%.*]] = mul i32 [[DUMMY32_3]], [[DUMMY32_3]]
-; CHECK-NEXT:    [[DUMMY34_3:%.*]] = mul i32 [[DUMMY33_3]], [[DUMMY33_3]]
-; CHECK-NEXT:    [[DUMMY35_3:%.*]] = mul i32 [[DUMMY34_3]], [[DUMMY34_3]]
-; CHECK-NEXT:    [[DUMMY36_3:%.*]] = mul i32 [[DUMMY35_3]], [[DUMMY35_3]]
-; CHECK-NEXT:    [[DUMMY37_3:%.*]] = mul i32 [[DUMMY36_3]], [[DUMMY36_3]]
-; CHECK-NEXT:    [[DUMMY38_3:%.*]] = mul i32 [[DUMMY37_3]], [[DUMMY37_3]]
-; CHECK-NEXT:    [[DUMMY39_3:%.*]] = mul i32 [[DUMMY38_3]], [[DUMMY38_3]]
-; CHECK-NEXT:    [[DUMMY40_3:%.*]] = mul i32 [[DUMMY39_3]], [[DUMMY39_3]]
-; CHECK-NEXT:    [[DUMMY41_3:%.*]] = mul i32 [[DUMMY40_3]], [[DUMMY40_3]]
-; CHECK-NEXT:    [[DUMMY42_3:%.*]] = mul i32 [[DUMMY41_3]], [[DUMMY41_3]]
-; CHECK-NEXT:    [[DUMMY43_3:%.*]] = mul i32 [[DUMMY42_3]], [[DUMMY42_3]]
-; CHECK-NEXT:    [[DUMMY44_3:%.*]] = mul i32 [[DUMMY43_3]], [[DUMMY43_3]]
-; CHECK-NEXT:    [[DUMMY45_3:%.*]] = mul i32 [[DUMMY44_3]], [[DUMMY44_3]]
-; CHECK-NEXT:    [[DUMMY46_3:%.*]] = mul i32 [[DUMMY45_3]], [[DUMMY45_3]]
-; CHECK-NEXT:    [[DUMMY47_3:%.*]] = mul i32 [[DUMMY46_3]], [[DUMMY46_3]]
-; CHECK-NEXT:    [[DUMMY48_3:%.*]] = mul i32 [[DUMMY47_3]], [[DUMMY47_3]]
-; CHECK-NEXT:    [[DUMMY49_3:%.*]] = mul i32 [[DUMMY48_3]], [[DUMMY48_3]]
-; CHECK-NEXT:    [[DUMMY50_3:%.*]] = mul i32 [[DUMMY49_3]], [[DUMMY49_3]]
-; CHECK-NEXT:    [[SUM_NEXT_3:%.*]] = add nsw i32 [[DUMMY50_3]], [[SUM_NEXT_2]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_3:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
-; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_3]]
-; CHECK-NEXT:    [[VAL_4:%.*]] = load i32, ptr [[ARRAYIDX_4]], align 4
-; CHECK-NEXT:    [[DUMMY1_4:%.*]] = mul i32 [[VAL_4]], [[VAL_4]]
-; CHECK-NEXT:    [[DUMMY2_4:%.*]] = mul i32 [[DUMMY1_4]], [[DUMMY1_4]]
-; CHECK-NEXT:    [[DUMMY3_4:%.*]] = mul i32 [[DUMMY2_4]], [[DUMMY2_4]]
-; CHECK-NEXT:    [[DUMMY4_4:%.*]] = mul i32 [[DUMMY3_4]], [[DUMMY3_4]]
-; CHECK-NEXT:    [[DUMMY5_4:%.*]] = mul i32 [[DUMMY4_4]], [[DUMMY4_4]]
-; CHECK-NEXT:    [[DUMMY6_4:%.*]] = mul i32 [[DUMMY5_4]], [[DUMMY5_4]]
-; CHECK-NEXT:    [[DUMMY7_4:%.*]] = mul i32 [[DUMMY6_4]], [[DUMMY6_4]]
-; CHECK-NEXT:    [[DUMMY8_4:%.*]] = mul i32 [[DUMMY7_4]], [[DUMMY7_4]]
-; CHECK-NEXT:    [[DUMMY9_4:%.*]] = mul i32 [[DUMMY8_4]], [[DUMMY8_4]]
-; CHECK-NEXT:    [[DUMMY10_4:%.*]] = mul i32 [[DUMMY9_4]], [[DUMMY9_4]]
-; CHECK-NEXT:    [[DUMMY11_4:%.*]] = mul i32 [[DUMMY10_4]], [[DUMMY10_4]]
-; CHECK-NEXT:    [[DUMMY12_4:%.*]] = mul i32 [[DUMMY11_4]], [[DUMMY11_4]]
-; CHECK-NEXT:    [[DUMMY13_4:%.*]] = mul i32 [[DUMMY12_4]], [[DUMMY12_4]]
-; CHECK-NEXT:    [[DUMMY14_4:%.*]] = mul i32 [[DUMMY13_4]], [[DUMMY13_4]]
-; CHECK-NEXT:    [[DUMMY15_4:%.*]] = mul i32 [[DUMMY14_4]], [[DUMMY14_4]]
-; CHECK-NEXT:    [[DUMMY16_4:%.*]] = mul i32 [[DUMMY15_4]], [[DUMMY15_4]]
-; CHECK-NEXT:    [[DUMMY17_4:%.*]] = mul i32 [[DUMMY16_4]], [[DUMMY16_4]]
-; CHECK-NEXT:    [[DUMMY18_4:%.*]] = mul i32 [[DUMMY17_4]], [[DUMMY17_4]]
-; CHECK-NEXT:    [[DUMMY19_4:%.*]] = mul i32 [[DUMMY18_4]], [[DUMMY18_4]]
-; CHECK-NEXT:    [[DUMMY20_4:%.*]] = mul i32 [[DUMMY19_4]], [[DUMMY19_4]]
-; CHECK-NEXT:    [[DUMMY21_4:%.*]] = mul i32 [[DUMMY20_4]], [[DUMMY20_4]]
-; CHECK-NEXT:    [[DUMMY22_4:%.*]] = mul i32 [[DUMMY21_4]], [[DUMMY21_4]]
-; CHECK-NEXT:    [[DUMMY23_4:%.*]] = mul i32 [[DUMMY22_4]], [[DUMMY22_4]]
-; CHECK-NEXT:    [[DUMMY24_4:%.*]] = mul i32 [[DUMMY23_4]], [[DUMMY23_4]]
-; CHECK-NEXT:    [[DUMMY25_4:%.*]] = mul i32 [[DUMMY24_4]], [[DUMMY24_4]]
-; CHECK-NEXT:    [[DUMMY26_4:%.*]] = mul i32 [[DUMMY25_4]], [[DUMMY25_4]]
-; CHECK-NEXT:    [[DUMMY27_4:%.*]] = mul i32 [[DUMMY26_4]], [[DUMMY26_4]]
-; CHECK-NEXT:    [[DUMMY28_4:%.*]] = mul i32 [[DUMMY27_4]], [[DUMMY27_4]]
-; CHECK-NEXT:    [[DUMMY29_4:%.*]] = mul i32 [[DUMMY28_4]], [[DUMMY28_4]]
-; CHECK-NEXT:    [[DUMMY30_4:%.*]] = mul i32 [[DUMMY29_4]], [[DUMMY29_4]]
-; CHECK-NEXT:    [[DUMMY31_4:%.*]] = mul i32 [[DUMMY30_4]], [[DUMMY30_4]]
-; CHECK-NEXT:    [[DUMMY32_4:%.*]] = mul i32 [[DUMMY31_4]], [[DUMMY31_4]]
-; CHECK-NEXT:    [[DUMMY33_4:%.*]] = mul i32 [[DUMMY32_4]], [[DUMMY32_4]]
-; CHECK-NEXT:    [[DUMMY34_4:%.*]] = mul i32 [[DUMMY33_4]], [[DUMMY33_4]]
-; CHECK-NEXT:    [[DUMMY35_4:%.*]] = mul i32 [[DUMMY34_4]], [[DUMMY34_4]]
-; CHECK-NEXT:    [[DUMMY36_4:%.*]] = mul i32 [[DUMMY35_4]], [[DUMMY35_4]]
-; CHECK-NEXT:    [[DUMMY37_4:%.*]] = mul i32 [[DUMMY36_4]], [[DUMMY36_4]]
-; CHECK-NEXT:    [[DUMMY38_4:%.*]] = mul i32 [[DUMMY37_4]], [[DUMMY37_4]]
-; CHECK-NEXT:    [[DUMMY39_4:%.*]] = mul i32 [[DUMMY38_4]], [[DUMMY38_4]]
-; CHECK-NEXT:    [[DUMMY40_4:%.*]] = mul i32 [[DUMMY39_4]], [[DUMMY39_4]]
-; CHECK-NEXT:    [[DUMMY41_4:%.*]] = mul i32 [[DUMMY40_4]], [[DUMMY40_4]]
-; CHECK-NEXT:    [[DUMMY42_4:%.*]] = mul i32 [[DUMMY41_4]], [[DUMMY41_4]]
-; CHECK-NEXT:    [[DUMMY43_4:%.*]] = mul i32 [[DUMMY42_4]], [[DUMMY42_4]]
-; CHECK-NEXT:    [[DUMMY44_4:%.*]] = mul i32 [[DUMMY43_4]], [[DUMMY43_4]]
-; CHECK-NEXT:    [[DUMMY45_4:%.*]] = mul i32 [[DUMMY44_4]], [[DUMMY44_4]]
-; CHECK-NEXT:    [[DUMMY46_4:%.*]] = mul i32 [[DUMMY45_4]], [[DUMMY45_4]]
-; CHECK-NEXT:    [[DUMMY47_4:%.*]] = mul i32 [[DUMMY46_4]], [[DUMMY46_4]]
-; CHECK-NEXT:    [[DUMMY48_4:%.*]] = mul i32 [[DUMMY47_4]], [[DUMMY47_4]]
-; CHECK-NEXT:    [[DUMMY49_4:%.*]] = mul i32 [[DUMMY48_4]], [[DUMMY48_4]]
-; CHECK-NEXT:    [[DUMMY50_4:%.*]] = mul i32 [[DUMMY49_4]], [[DUMMY49_4]]
-; CHECK-NEXT:    [[SUM_NEXT_4:%.*]] = add nsw i32 [[DUMMY50_4]], [[SUM_NEXT_3]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_4:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 5
-; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_4]]
-; CHECK-NEXT:    [[VAL_5:%.*]] = load i32, ptr [[ARRAYIDX_5]], align 4
-; CHECK-NEXT:    [[DUMMY1_5:%.*]] = mul i32 [[VAL_5]], [[VAL_5]]
-; CHECK-NEXT:    [[DUMMY2_5:%.*]] = mul i32 [[DUMMY1_5]], [[DUMMY1_5]]
-; CHECK-NEXT:    [[DUMMY3_5:%.*]] = mul i32 [[DUMMY2_5]], [[DUMMY2_5]]
-; CHECK-NEXT:    [[DUMMY4_5:%.*]] = mul i32 [[DUMMY3_5]], [[DUMMY3_5]]
-; CHECK-NEXT:    [[DUMMY5_5:%.*]] = mul i32 [[DUMMY4_5]], [[DUMMY4_5]]
-; CHECK-NEXT:    [[DUMMY6_5:%.*]] = mul i32 [[DUMMY5_5]], [[DUMMY5_5]]
-; CHECK-NEXT:    [[DUMMY7_5:%.*]] = mul i32 [[DUMMY6_5]], [[DUMMY6_5]]
-; CHECK-NEXT:    [[DUMMY8_5:%.*]] = mul i32 [[DUMMY7_5]], [[DUMMY7_5]]
-; CHECK-NEXT:    [[DUMMY9_5:%.*]] = mul i32 [[DUMMY8_5]], [[DUMMY8_5]]
-; CHECK-NEXT:    [[DUMMY10_5:%.*]] = mul i32 [[DUMMY9_5]], [[DUMMY9_5]]
-; CHECK-NEXT:    [[DUMMY11_5:%.*]] = mul i32 [[DUMMY10_5]], [[DUMMY10_5]]
-; CHECK-NEXT:    [[DUMMY12_5:%.*]] = mul i32 [[DUMMY11_5]], [[DUMMY11_5]]
-; CHECK-NEXT:    [[DUMMY13_5:%.*]] = mul i32 [[DUMMY12_5]], [[DUMMY12_5]]
-; CHECK-NEXT:    [[DUMMY14_5:%.*]] = mul i32 [[DUMMY13_5]], [[DUMMY13_5]]
-; CHECK-NEXT:    [[DUMMY15_5:%.*]] = mul i32 [[DUMMY14_5]], [[DUMMY14_5]]
-; CHECK-NEXT:    [[DUMMY16_5:%.*]] = mul i32 [[DUMMY15_5]], [[DUMMY15_5]]
-; CHECK-NEXT:    [[DUMMY17_5:%.*]] = mul i32 [[DUMMY16_5]], [[DUMMY16_5]]
-; CHECK-NEXT:    [[DUMMY18_5:%.*]] = mul i32 [[DUMMY17_5]], [[DUMMY17_5]]
-; CHECK-NEXT:    [[DUMMY19_5:%.*]] = mul i32 [[DUMMY18_5]], [[DUMMY18_5]]
-; CHECK-NEXT:    [[DUMMY20_5:%.*]] = mul i32 [[DUMMY19_5]], [[DUMMY19_5]]
-; CHECK-NEXT:    [[DUMMY21_5:%.*]] = mul i32 [[DUMMY20_5]], [[DUMMY20_5]]
-; CHECK-NEXT:    [[DUMMY22_5:%.*]] = mul i32 [[DUMMY21_5]], [[DUMMY21_5]]
-; CHECK-NEXT:    [[DUMMY23_5:%.*]] = mul i32 [[DUMMY22_5]], [[DUMMY22_5]]
-; CHECK-NEXT:    [[DUMMY24_5:%.*]] = mul i32 [[DUMMY23_5]], [[DUMMY23_5]]
-; CHECK-NEXT:    [[DUMMY25_5:%.*]] = mul i32 [[DUMMY24_5]], [[DUMMY24_5]]
-; CHECK-NEXT:    [[DUMMY26_5:%.*]] = mul i32 [[DUMMY25_5]], [[DUMMY25_5]]
-; CHECK-NEXT:    [[DUMMY27_5:%.*]] = mul i32 [[DUMMY26_5]], [[DUMMY26_5]]
-; CHECK-NEXT:    [[DUMMY28_5:%.*]] = mul i32 [[DUMMY27_5]], [[DUMMY27_5]]
-; CHECK-NEXT:    [[DUMMY29_5:%.*]] = mul i32 [[DUMMY28_5]], [[DUMMY28_5]]
-; CHECK-NEXT:    [[DUMMY30_5:%.*]] = mul i32 [[DUMMY29_5]], [[DUMMY29_5]]
-; CHECK-NEXT:    [[DUMMY31_5:%.*]] = mul i32 [[DUMMY30_5]], [[DUMMY30_5]]
-; CHECK-NEXT:    [[DUMMY32_5:%.*]] = mul i32 [[DUMMY31_5]], [[DUMMY31_5]]
-; CHECK-NEXT:    [[DUMMY33_5:%.*]] = mul i32 [[DUMMY32_5]], [[DUMMY32_5]]
-; CHECK-NEXT:    [[DUMMY34_5:%.*]] = mul i32 [[DUMMY33_5]], [[DUMMY33_5]]
-; CHECK-NEXT:    [[DUMMY35_5:%.*]] = mul i32 [[DUMMY34_5]], [[DUMMY34_5]]
-; CHECK-NEXT:    [[DUMMY36_5:%.*]] = mul i32 [[DUMMY35_5]], [[DUMMY35_5]]
-; CHECK-NEXT:    [[DUMMY37_5:%.*]] = mul i32 [[DUMMY36_5]], [[DUMMY36_5]]
-; CHECK-NEXT:    [[DUMMY38_5:%.*]] = mul i32 [[DUMMY37_5]], [[DUMMY37_5]]
-; CHECK-NEXT:    [[DUMMY39_5:%.*]] = mul i32 [[DUMMY38_5]], [[DUMMY38_5]]
-; CHECK-NEXT:    [[DUMMY40_5:%.*]] = mul i32 [[DUMMY39_5]], [[DUMMY39_5]]
-; CHECK-NEXT:    [[DUMMY41_5:%.*]] = mul i32 [[DUMMY40_5]], [[DUMMY40_5]]
-; CHECK-NEXT:    [[DUMMY42_5:%.*]] = mul i32 [[DUMMY41_5]], [[DUMMY41_5]]
-; CHECK-NEXT:    [[DUMMY43_5:%.*]] = mul i32 [[DUMMY42_5]], [[DUMMY42_5]]
-; CHECK-NEXT:    [[DUMMY44_5:%.*]] = mul i32 [[DUMMY43_5]], [[DUMMY43_5]]
-; CHECK-NEXT:    [[DUMMY45_5:%.*]] = mul i32 [[DUMMY44_5]], [[DUMMY44_5]]
-; CHECK-NEXT:    [[DUMMY46_5:%.*]] = mul i32 [[DUMMY45_5]], [[DUMMY45_5]]
-; CHECK-NEXT:    [[DUMMY47_5:%.*]] = mul i32 [[DUMMY46_5]], [[DUMMY46_5]]
-; CHECK-NEXT:    [[DUMMY48_5:%.*]] = mul i32 [[DUMMY47_5]], [[DUMMY47_5]]
-; CHECK-NEXT:    [[DUMMY49_5:%.*]] = mul i32 [[DUMMY48_5]], [[DUMMY48_5]]
-; CHECK-NEXT:    [[DUMMY50_5:%.*]] = mul i32 [[DUMMY49_5]], [[DUMMY49_5]]
-; CHECK-NEXT:    [[SUM_NEXT_5:%.*]] = add nsw i32 [[DUMMY50_5]], [[SUM_NEXT_4]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_5:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 6
-; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_5]]
-; CHECK-NEXT:    [[VAL_6:%.*]] = load i32, ptr [[ARRAYIDX_6]], align 4
-; CHECK-NEXT:    [[DUMMY1_6:%.*]] = mul i32 [[VAL_6]], [[VAL_6]]
-; CHECK-NEXT:    [[DUMMY2_6:%.*]] = mul i32 [[DUMMY1_6]], [[DUMMY1_6]]
-; CHECK-NEXT:    [[DUMMY3_6:%.*]] = mul i32 [[DUMMY2_6]], [[DUMMY2_6]]
-; CHECK-NEXT:    [[DUMMY4_6:%.*]] = mul i32 [[DUMMY3_6]], [[DUMMY3_6]]
-; CHECK-NEXT:    [[DUMMY5_6:%.*]] = mul i32 [[DUMMY4_6]], [[DUMMY4_6]]
-; CHECK-NEXT:    [[DUMMY6_6:%.*]] = mul i32 [[DUMMY5_6]], [[DUMMY5_6]]
-; CHECK-NEXT:    [[DUMMY7_6:%.*]] = mul i32 [[DUMMY6_6]], [[DUMMY6_6]]
-; CHECK-NEXT:    [[DUMMY8_6:%.*]] = mul i32 [[DUMMY7_6]], [[DUMMY7_6]]
-; CHECK-NEXT:    [[DUMMY9_6:%.*]] = mul i32 [[DUMMY8_6]], [[DUMMY8_6]]
-; CHECK-NEXT:    [[DUMMY10_6:%.*]] = mul i32 [[DUMMY9_6]], [[DUMMY9_6]]
-; CHECK-NEXT:    [[DUMMY11_6:%.*]] = mul i32 [[DUMMY10_6]], [[DUMMY10_6]]
-; CHECK-NEXT:    [[DUMMY12_6:%.*]] = mul i32 [[DUMMY11_6]], [[DUMMY11_6]]
-; CHECK-NEXT:    [[DUMMY13_6:%.*]] = mul i32 [[DUMMY12_6]], [[DUMMY12_6]]
-; CHECK-NEXT:    [[DUMMY14_6:%.*]] = mul i32 [[DUMMY13_6]], [[DUMMY13_6]]
-; CHECK-NEXT:    [[DUMMY15_6:%.*]] = mul i32 [[DUMMY14_6]], [[DUMMY14_6]]
-; CHECK-NEXT:    [[DUMMY16_6:%.*]] = mul i32 [[DUMMY15_6]], [[DUMMY15_6]]
-; CHECK-NEXT:    [[DUMMY17_6:%.*]] = mul i32 [[DUMMY16_6]], [[DUMMY16_6]]
-; CHECK-NEXT:    [[DUMMY18_6:%.*]] = mul i32 [[DUMMY17_6]], [[DUMMY17_6]]
-; CHECK-NEXT:    [[DUMMY19_6:%.*]] = mul i32 [[DUMMY18_6]], [[DUMMY18_6]]
-; CHECK-NEXT:    [[DUMMY20_6:%.*]] = mul i32 [[DUMMY19_6]], [[DUMMY19_6]]
-; CHECK-NEXT:    [[DUMMY21_6:%.*]] = mul i32 [[DUMMY20_6]], [[DUMMY20_6]]
-; CHECK-NEXT:    [[DUMMY22_6:%.*]] = mul i32 [[DUMMY21_6]], [[DUMMY21_6]]
-; CHECK-NEXT:    [[DUMMY23_6:%.*]] = mul i32 [[DUMMY22_6]], [[DUMMY22_6]]
-; CHECK-NEXT:    [[DUMMY24_6:%.*]] = mul i32 [[DUMMY23_6]], [[DUMMY23_6]]
-; CHECK-NEXT:    [[DUMMY25_6:%.*]] = mul i32 [[DUMMY24_6]], [[DUMMY24_6]]
-; CHECK-NEXT:    [[DUMMY26_6:%.*]] = mul i32 [[DUMMY25_6]], [[DUMMY25_6]]
-; CHECK-NEXT:    [[DUMMY27_6:%.*]] = mul i32 [[DUMMY26_6]], [[DUMMY26_6]]
-; CHECK-NEXT:    [[DUMMY28_6:%.*]] = mul i32 [[DUMMY27_6]], [[DUMMY27_6]]
-; CHECK-NEXT:    [[DUMMY29_6:%.*]] = mul i32 [[DUMMY28_6]], [[DUMMY28_6]]
-; CHECK-NEXT:    [[DUMMY30_6:%.*]] = mul i32 [[DUMMY29_6]], [[DUMMY29_6]]
-; CHECK-NEXT:    [[DUMMY31_6:%.*]] = mul i32 [[DUMMY30_6]], [[DUMMY30_6]]
-; CHECK-NEXT:    [[DUMMY32_6:%.*]] = mul i32 [[DUMMY31_6]], [[DUMMY31_6]]
-; CHECK-NEXT:    [[DUMMY33_6:%.*]] = mul i32 [[DUMMY32_6]], [[DUMMY32_6]]
-; CHECK-NEXT:    [[DUMMY34_6:%.*]] = mul i32 [[DUMMY33_6]], [[DUMMY33_6]]
-; CHECK-NEXT:    [[DUMMY35_6:%.*]] = mul i32 [[DUMMY34_6]], [[DUMMY34_6]]
-; CHECK-NEXT:    [[DUMMY36_6:%.*]] = mul i32 [[DUMMY35_6]], [[DUMMY35_6]]
-; CHECK-NEXT:    [[DUMMY37_6:%.*]] = mul i32 [[DUMMY36_6]], [[DUMMY36_6]]
-; CHECK-NEXT:    [[DUMMY38_6:%.*]] = mul i32 [[DUMMY37_6]], [[DUMMY37_6]]
-; CHECK-NEXT:    [[DUMMY39_6:%.*]] = mul i32 [[DUMMY38_6]], [[DUMMY38_6]]
-; CHECK-NEXT:    [[DUMMY40_6:%.*]] = mul i32 [[DUMMY39_6]], [[DUMMY39_6]]
-; CHECK-NEXT:    [[DUMMY41_6:%.*]] = mul i32 [[DUMMY40_6]], [[DUMMY40_6]]
-; CHECK-NEXT:    [[DUMMY42_6:%.*]] = mul i32 [[DUMMY41_6]], [[DUMMY41_6]]
-; CHECK-NEXT:    [[DUMMY43_6:%.*]] = mul i32 [[DUMMY42_6]], [[DUMMY42_6]]
-; CHECK-NEXT:    [[DUMMY44_6:%.*]] = mul i32 [[DUMMY43_6]], [[DUMMY43_6]]
-; CHECK-NEXT:    [[DUMMY45_6:%.*]] = mul i32 [[DUMMY44_6]], [[DUMMY44_6]]
-; CHECK-NEXT:    [[DUMMY46_6:%.*]] = mul i32 [[DUMMY45_6]], [[DUMMY45_6]]
-; CHECK-NEXT:    [[DUMMY47_6:%.*]] = mul i32 [[DUMMY46_6]], [[DUMMY46_6]]
-; CHECK-NEXT:    [[DUMMY48_6:%.*]] = mul i32 [[DUMMY47_6]], [[DUMMY47_6]]
-; CHECK-NEXT:    [[DUMMY49_6:%.*]] = mul i32 [[DUMMY48_6]], [[DUMMY48_6]]
-; CHECK-NEXT:    [[DUMMY50_6:%.*]] = mul i32 [[DUMMY49_6]], [[DUMMY49_6]]
-; CHECK-NEXT:    [[SUM_NEXT_6:%.*]] = add nsw i32 [[DUMMY50_6]], [[SUM_NEXT_5]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_6:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 7
-; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_6]]
-; CHECK-NEXT:    [[VAL_7:%.*]] = load i32, ptr [[ARRAYIDX_7]], align 4
-; CHECK-NEXT:    [[DUMMY1_7:%.*]] = mul i32 [[VAL_7]], [[VAL_7]]
-; CHECK-NEXT:    [[DUMMY2_7:%.*]] = mul i32 [[DUMMY1_7]], [[DUMMY1_7]]
-; CHECK-NEXT:    [[DUMMY3_7:%.*]] = mul i32 [[DUMMY2_7]], [[DUMMY2_7]]
-; CHECK-NEXT:    [[DUMMY4_7:%.*]] = mul i32 [[DUMMY3_7]], [[DUMMY3_7]]
-; CHECK-NEXT:    [[DUMMY5_7:%.*]] = mul i32 [[DUMMY4_7]], [[DUMMY4_7]]
-; CHECK-NEXT:    [[DUMMY6_7:%.*]] = mul i32 [[DUMMY5_7]], [[DUMMY5_7]]
-; CHECK-NEXT:    [[DUMMY7_7:%.*]] = mul i32 [[DUMMY6_7]], [[DUMMY6_7]]
-; CHECK-NEXT:    [[DUMMY8_7:%.*]] = mul i32 [[DUMMY7_7]], [[DUMMY7_7]]
-; CHECK-NEXT:    [[DUMMY9_7:%.*]] = mul i32 [[DUMMY8_7]], [[DUMMY8_7]]
-; CHECK-NEXT:    [[DUMMY10_7:%.*]] = mul i32 [[DUMMY9_7]], [[DUMMY9_7]]
-; CHECK-NEXT:    [[DUMMY11_7:%.*]] = mul i32 [[DUMMY10_7]], [[DUMMY10_7]]
-; CHECK-NEXT:    [[DUMMY12_7:%.*]] = mul i32 [[DUMMY11_7]], [[DUMMY11_7]]
-; CHECK-NEXT:    [[DUMMY13_7:%.*]] = mul i32 [[DUMMY12_7]], [[DUMMY12_7]]
-; CHECK-NEXT:    [[DUMMY14_7:%.*]] = mul i32 [[DUMMY13_7]], [[DUMMY13_7]]
-; CHECK-NEXT:    [[DUMMY15_7:%.*]] = mul i32 [[DUMMY14_7]], [[DUMMY14_7]]
-; CHECK-NEXT:    [[DUMMY16_7:%.*]] = mul i32 [[DUMMY15_7]], [[DUMMY15_7]]
-; CHECK-NEXT:    [[DUMMY17_7:%.*]] = mul i32 [[DUMMY16_7]], [[DUMMY16_7]]
-; CHECK-NEXT:    [[DUMMY18_7:%.*]] = mul i32 [[DUMMY17_7]], [[DUMMY17_7]]
-; CHECK-NEXT:    [[DUMMY19_7:%.*]] = mul i32 [[DUMMY18_7]], [[DUMMY18_7]]
-; CHECK-NEXT:    [[DUMMY20_7:%.*]] = mul i32 [[DUMMY19_7]], [[DUMMY19_7]]
-; CHECK-NEXT:    [[DUMMY21_7:%.*]] = mul i32 [[DUMMY20_7]], [[DUMMY20_7]]
-; CHECK-NEXT:    [[DUMMY22_7:%.*]] = mul i32 [[DUMMY21_7]], [[DUMMY21_7]]
-; CHECK-NEXT:    [[DUMMY23_7:%.*]] = mul i32 [[DUMMY22_7]], [[DUMMY22_7]]
-; CHECK-NEXT:    [[DUMMY24_7:%.*]] = mul i32 [[DUMMY23_7]], [[DUMMY23_7]]
-; CHECK-NEXT:    [[DUMMY25_7:%.*]] = mul i32 [[DUMMY24_7]], [[DUMMY24_7]]
-; CHECK-NEXT:    [[DUMMY26_7:%.*]] = mul i32 [[DUMMY25_7]], [[DUMMY25_7]]
-; CHECK-NEXT:    [[DUMMY27_7:%.*]] = mul i32 [[DUMMY26_7]], [[DUMMY26_7]]
-; CHECK-NEXT:    [[DUMMY28_7:%.*]] = mul i32 [[DUMMY27_7]], [[DUMMY27_7]]
-; CHECK-NEXT:    [[DUMMY29_7:%.*]] = mul i32 [[DUMMY28_7]], [[DUMMY28_7]]
-; CHECK-NEXT:    [[DUMMY30_7:%.*]] = mul i32 [[DUMMY29_7]], [[DUMMY29_7]]
-; CHECK-NEXT:    [[DUMMY31_7:%.*]] = mul i32 [[DUMMY30_7]], [[DUMMY30_7]]
-; CHECK-NEXT:    [[DUMMY32_7:%.*]] = mul i32 [[DUMMY31_7]], [[DUMMY31_7]]
-; CHECK-NEXT:    [[DUMMY33_7:%.*]] = mul i32 [[DUMMY32_7]], [[DUMMY32_7]]
-; CHECK-NEXT:    [[DUMMY34_7:%.*]] = mul i32 [[DUMMY33_7]], [[DUMMY33_7]]
-; CHECK-NEXT:    [[DUMMY35_7:%.*]] = mul i32 [[DUMMY34_7]], [[DUMMY34_7]]
-; CHECK-NEXT:    [[DUMMY36_7:%.*]] = mul i32 [[DUMMY35_7]], [[DUMMY35_7]]
-; CHECK-NEXT:    [[DUMMY37_7:%.*]] = mul i32 [[DUMMY36_7]], [[DUMMY36_7]]
-; CHECK-NEXT:    [[DUMMY38_7:%.*]] = mul i32 [[DUMMY37_7]], [[DUMMY37_7]]
-; CHECK-NEXT:    [[DUMMY39_7:%.*]] = mul i32 [[DUMMY38_7]], [[DUMMY38_7]]
-; CHECK-NEXT:    [[DUMMY40_7:%.*]] = mul i32 [[DUMMY39_7]], [[DUMMY39_7]]
-; CHECK-NEXT:    [[DUMMY41_7:%.*]] = mul i32 [[DUMMY40_7]], [[DUMMY40_7]]
-; CHECK-NEXT:    [[DUMMY42_7:%.*]] = mul i32 [[DUMMY41_7]], [[DUMMY41_7]]
-; CHECK-NEXT:    [[DUMMY43_7:%.*]] = mul i32 [[DUMMY42_7]], [[DUMMY42_7]]
-; CHECK-NEXT:    [[DUMMY44_7:%.*]] = mul i32 [[DUMMY43_7]], [[DUMMY43_7]]
-; CHECK-NEXT:    [[DUMMY45_7:%.*]] = mul i32 [[DUMMY44_7]], [[DUMMY44_7]]
-; CHECK-NEXT:    [[DUMMY46_7:%.*]] = mul i32 [[DUMMY45_7]], [[DUMMY45_7]]
-; CHECK-NEXT:    [[DUMMY47_7:%.*]] = mul i32 [[DUMMY46_7]], [[DUMMY46_7]]
-; CHECK-NEXT:    [[DUMMY48_7:%.*]] = mul i32 [[DUMMY47_7]], [[DUMMY47_7]]
-; CHECK-NEXT:    [[DUMMY49_7:%.*]] = mul i32 [[DUMMY48_7]], [[DUMMY48_7]]
-; CHECK-NEXT:    [[DUMMY50_7:%.*]] = mul i32 [[DUMMY49_7]], [[DUMMY49_7]]
-; CHECK-NEXT:    [[SUM_NEXT_7]] = add nsw i32 [[DUMMY50_7]], [[SUM_NEXT_6]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_7]] = add nuw nsw i64 [[INDVARS_IV]], 8
-; CHECK-NEXT:    [[NITER_NEXT_7]] = add i64 [[NITER]], 8
-; CHECK-NEXT:    [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]]
-; CHECK-NEXT:    br i1 [[NITER_NCMP_7]], label [[FOR_COND_CLEANUP_UNR_LCSSA_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    [[SUM_NEXT_1]] = add nsw i32 [[DUMMY50_1]], [[SUM_NEXT]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; CHECK-NEXT:    [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; CHECK-NEXT:    br i1 [[NITER_NCMP_1]], label [[FOR_COND_CLEANUP_UNR_LCSSA_LOOPEXIT:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.cond.cleanup.unr-lcssa.loopexit:
-; CHECK-NEXT:    [[SUM_NEXT_LCSSA_PH_PH:%.*]] = phi i32 [ [[SUM_NEXT_7]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_7]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_UNR_PH:%.*]] = phi i32 [ [[SUM_NEXT_7]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_NEXT_LCSSA_PH_PH:%.*]] = phi i32 [ [[SUM_NEXT_1]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_UNR_PH:%.*]] = phi i32 [ [[SUM_NEXT_1]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_UNR_LCSSA]]
 ; CHECK:       for.cond.cleanup.unr-lcssa:
 ; CHECK-NEXT:    [[SUM_NEXT_LCSSA_PH:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[SUM_NEXT_LCSSA_PH_PH]], [[FOR_COND_CLEANUP_UNR_LCSSA_LOOPEXIT]] ]
@@ -1010,10 +302,7 @@ define i32 @test2(ptr %ary, i64 %n) "target-cpu"="znver3" {
 ; CHECK:       for.body.epil.preheader:
 ; CHECK-NEXT:    br label [[FOR_BODY_EPIL:%.*]]
 ; CHECK:       for.body.epil:
-; CHECK-NEXT:    [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_UNR]], [[FOR_BODY_EPIL_PREHEADER]] ], [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL]] ]
-; CHECK-NEXT:    [[SUM_EPIL:%.*]] = phi i32 [ [[SUM_UNR]], [[FOR_BODY_EPIL_PREHEADER]] ], [ [[SUM_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL]] ]
-; CHECK-NEXT:    [[EPIL_ITER:%.*]] = phi i64 [ 0, [[FOR_BODY_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[FOR_BODY_EPIL]] ]
-; CHECK-NEXT:    [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_EPIL]]
+; CHECK-NEXT:    [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_UNR]]
 ; CHECK-NEXT:    [[VAL_EPIL:%.*]] = load i32, ptr [[ARRAYIDX_EPIL]], align 4
 ; CHECK-NEXT:    [[DUMMY1_EPIL:%.*]] = mul i32 [[VAL_EPIL]], [[VAL_EPIL]]
 ; CHECK-NEXT:    [[DUMMY2_EPIL:%.*]] = mul i32 [[DUMMY1_EPIL]], [[DUMMY1_EPIL]]
@@ -1065,17 +354,10 @@ define i32 @test2(ptr %ary, i64 %n) "target-cpu"="znver3" {
 ; CHECK-NEXT:    [[DUMMY48_EPIL:%.*]] = mul i32 [[DUMMY47_EPIL]], [[DUMMY47_EPIL]]
 ; CHECK-NEXT:    [[DUMMY49_EPIL:%.*]] = mul i32 [[DUMMY48_EPIL]], [[DUMMY48_EPIL]]
 ; CHECK-NEXT:    [[DUMMY50_EPIL:%.*]] = mul i32 [[DUMMY49_EPIL]], [[DUMMY49_EPIL]]
-; CHECK-NEXT:    [[SUM_NEXT_EPIL]] = add nsw i32 [[DUMMY50_EPIL]], [[SUM_EPIL]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_EPIL]] = add nuw nsw i64 [[INDVARS_IV_EPIL]], 1
-; CHECK-NEXT:    [[EXITCOND_NOT_EPIL:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_EPIL]], [[N]]
-; CHECK-NEXT:    [[EPIL_ITER_NEXT]] = add i64 [[EPIL_ITER]], 1
-; CHECK-NEXT:    [[EPIL_ITER_CMP:%.*]] = icmp ne i64 [[EPIL_ITER_NEXT]], [[XTRAITER]]
-; CHECK-NEXT:    br i1 [[EPIL_ITER_CMP]], label [[FOR_BODY_EPIL]], label [[FOR_COND_CLEANUP_EPILOG_LCSSA:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK:       for.cond.cleanup.epilog-lcssa:
-; CHECK-NEXT:    [[SUM_NEXT_LCSSA_PH1:%.*]] = phi i32 [ [[SUM_NEXT_EPIL]], [[FOR_BODY_EPIL]] ]
+; CHECK-NEXT:    [[SUM_NEXT_EPIL:%.*]] = add nsw i32 [[DUMMY50_EPIL]], [[SUM_UNR]]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT_LCSSA_PH]], [[FOR_COND_CLEANUP_UNR_LCSSA]] ], [ [[SUM_NEXT_LCSSA_PH1]], [[FOR_COND_CLEANUP_EPILOG_LCSSA]] ]
+; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT_LCSSA_PH]], [[FOR_COND_CLEANUP_UNR_LCSSA]] ], [ [[SUM_NEXT_EPIL]], [[FOR_BODY_EPIL]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_NEXT_LCSSA]]
 ;
 entry:
-- 
GitLab


From f60c699d37c41c46dd0be4ec98e5b4d74e73b2b7 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Wed, 15 May 2024 20:44:02 -0500
Subject: [PATCH 0944/1206] [OpenMP] Fix intermediate header locations for
 OpenMP

Summary:
A previous patch moved the code here and accidentally overrwrote the
include path that the LSP interface used. This caused incorrect errors
when using clangd with the offload project. This patch removes the
unnecessary header and makes sure we include the correct folder.
---
 openmp/runtime/src/CMakeLists.txt | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index a2468d04e60a..94eeea63b804 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -20,7 +20,7 @@ endif()
 # Configure omp.h, kmp_config.h and omp-tools.h if necessary
 configure_file(${LIBOMP_INC_DIR}/omp.h.var ${LIBOMP_HEADERS_INTDIR}/omp.h @ONLY)
 configure_file(${LIBOMP_INC_DIR}/ompx.h.var ${LIBOMP_HEADERS_INTDIR}/ompx.h @ONLY)
-configure_file(kmp_config.h.cmake ${LIBOMP_HEADERS_INTDIR}/kmp_config.h @ONLY)
+configure_file(kmp_config.h.cmake kmp_config.h @ONLY)
 if(${LIBOMP_OMPT_SUPPORT})
   configure_file(${LIBOMP_INC_DIR}/omp-tools.h.var ${LIBOMP_HEADERS_INTDIR}/omp-tools.h @ONLY)
 endif()
@@ -55,7 +55,6 @@ include_directories(
   ${LIBOMP_SRC_DIR}/i18n
   ${LIBOMP_INC_DIR}
   ${LIBOMP_SRC_DIR}/thirdparty/ittnotify
-  ${LIBOMP_HEADERS_INTDIR}
 )
 
 # Building with time profiling support requires LLVM directory includes.
@@ -441,7 +440,7 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH})
   # install under legacy name ompt.h
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
-  set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${LIBOMP_HEADERS_INTDIR} PARENT_SCOPE)
+  set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
 if(${BUILD_FORTRAN_MODULES})
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
-- 
GitLab


From 1595988ee6f9732e7ea79928af8a470ad5ef7dbe Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Wed, 15 May 2024 21:52:59 -0400
Subject: [PATCH 0945/1206] Reapply "[Clang][Sema] Earlier type checking for
 builtin unary operators (#90500)" (#92283)

This patch reapplies #90500, addressing a bug which caused binary
operators with dependent operands to be incorrectly rebuilt by
`TreeTransform`.
---
 clang/docs/ReleaseNotes.rst                   |   3 +
 clang/include/clang/AST/Type.h                |   5 +-
 clang/lib/Sema/SemaExpr.cpp                   | 363 +++++++++---------
 clang/lib/Sema/TreeTransform.h                |  17 +-
 clang/test/AST/ast-dump-expr-json.cpp         |   4 +-
 clang/test/AST/ast-dump-expr.cpp              |   2 +-
 clang/test/AST/ast-dump-lambda.cpp            |   2 +-
 .../expr/expr.unary/expr.unary.general/p1.cpp |  65 ++++
 clang/test/CXX/over/over.built/ast.cpp        | 158 ++++++--
 clang/test/CXX/over/over.built/p10.cpp        |   2 +-
 clang/test/CXX/over/over.built/p11.cpp        |   2 +-
 .../over/over.oper/over.oper.general/p1.cpp   | 173 +++++++++
 .../temp.res/temp.dep/temp.dep.type/p4.cpp    |  25 +-
 clang/test/Frontend/noderef_templates.cpp     |   4 +-
 clang/test/SemaCXX/cxx2b-deducing-this.cpp    |   6 +-
 .../test/SemaTemplate/class-template-spec.cpp |  12 +-
 .../ASTMatchers/ASTMatchersNarrowingTest.cpp  |   6 +-
 17 files changed, 586 insertions(+), 263 deletions(-)
 create mode 100644 clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp
 create mode 100644 clang/test/CXX/over/over.oper/over.oper.general/p1.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 089a85c8cb36..11812c355f8d 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -56,6 +56,9 @@ C++ Specific Potentially Breaking Changes
 
 - Clang now rejects pointer to member from parenthesized expression in unevaluated context such as ``decltype(&(foo::bar))``. (#GH40906).
 
+- Clang now performs semantic analysis for unary operators with dependent operands
+  that are known to be of non-class non-enumeration type prior to instantiation.
+
 ABI Changes in This Version
 ---------------------------
 - Fixed Microsoft name mangling of implicitly defined variables used for thread
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index e6643469e0b3..da3834f19ca0 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -8044,7 +8044,10 @@ inline bool Type::isUndeducedType() const {
 /// Determines whether this is a type for which one can define
 /// an overloaded operator.
 inline bool Type::isOverloadableType() const {
-  return isDependentType() || isRecordType() || isEnumeralType();
+  if (!CanonicalType->isDependentType())
+    return isRecordType() || isEnumeralType();
+  return !isArrayType() && !isFunctionType() && !isAnyPointerType() &&
+         !isMemberPointerType();
 }
 
 /// Determines whether this type is written as a typedef-name.
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index ec84798e4ce6..50569c1cd536 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -672,12 +672,12 @@ ExprResult Sema::DefaultLvalueConversion(Expr *E) {
 
   // We don't want to throw lvalue-to-rvalue casts on top of
   // expressions of certain types in C++.
-  if (getLangOpts().CPlusPlus &&
-      (E->getType() == Context.OverloadTy ||
-       // FIXME: This is a hack! We want the lvalue-to-rvalue conversion applied
-       // to pointer types even if the pointee type is dependent.
-       (T->isDependentType() && !T->isPointerType()) || T->isRecordType()))
-    return E;
+  if (getLangOpts().CPlusPlus) {
+    if (T == Context.OverloadTy || T->isRecordType() ||
+        (T->isDependentType() && !T->isAnyPointerType() &&
+         !T->isMemberPointerType()))
+      return E;
+  }
 
   // The C standard is actually really unclear on this point, and
   // DR106 tells us what the result should be but not why.  It's
@@ -10827,7 +10827,7 @@ static bool checkArithmeticIncompletePointerType(Sema &S, SourceLocation Loc,
   if (const AtomicType *ResAtomicType = ResType->getAs<AtomicType>())
     ResType = ResAtomicType->getValueType();
 
-  assert(ResType->isAnyPointerType() && !ResType->isDependentType());
+  assert(ResType->isAnyPointerType());
   QualType PointeeTy = ResType->getPointeeType();
   return S.RequireCompleteSizedType(
       Loc, PointeeTy,
@@ -13957,9 +13957,6 @@ static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
                                                ExprObjectKind &OK,
                                                SourceLocation OpLoc, bool IsInc,
                                                bool IsPrefix) {
-  if (Op->isTypeDependent())
-    return S.Context.DependentTy;
-
   QualType ResType = Op->getType();
   // Atomic types can be used for increment / decrement where the non-atomic
   // versions can, so ignore the _Atomic() specifier for the purpose of
@@ -14410,9 +14407,6 @@ static void RecordModifiableNonNullParam(Sema &S, const Expr *Exp) {
 static QualType CheckIndirectionOperand(Sema &S, Expr *Op, ExprValueKind &VK,
                                         SourceLocation OpLoc,
                                         bool IsAfterAmp = false) {
-  if (Op->isTypeDependent())
-    return S.Context.DependentTy;
-
   ExprResult ConvResult = S.UsualUnaryConversions(Op);
   if (ConvResult.isInvalid())
     return QualType();
@@ -15368,14 +15362,10 @@ ExprResult Sema::BuildBinOp(Scope *S, SourceLocation OpLoc,
   }
 
   if (getLangOpts().CPlusPlus) {
-    // If either expression is type-dependent, always build an
-    // overloaded op.
-    if (LHSExpr->isTypeDependent() || RHSExpr->isTypeDependent())
-      return BuildOverloadedBinOp(*this, S, OpLoc, Opc, LHSExpr, RHSExpr);
-
-    // Otherwise, build an overloaded op if either expression has an
-    // overloadable type.
-    if (LHSExpr->getType()->isOverloadableType() ||
+    // Otherwise, build an overloaded op if either expression is type-dependent
+    // or has an overloadable type.
+    if (LHSExpr->isTypeDependent() || RHSExpr->isTypeDependent() ||
+        LHSExpr->getType()->isOverloadableType() ||
         RHSExpr->getType()->isOverloadableType())
       return BuildOverloadedBinOp(*this, S, OpLoc, Opc, LHSExpr, RHSExpr);
   }
@@ -15466,190 +15456,191 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
       return ExprError(Diag(OpLoc, diag::err_hlsl_operator_unsupported) << 1);
   }
 
-  switch (Opc) {
-  case UO_PreInc:
-  case UO_PreDec:
-  case UO_PostInc:
-  case UO_PostDec:
-    resultType =
-        CheckIncrementDecrementOperand(*this, Input.get(), VK, OK, OpLoc,
-                                       Opc == UO_PreInc || Opc == UO_PostInc,
-                                       Opc == UO_PreInc || Opc == UO_PreDec);
-    CanOverflow = isOverflowingIntegerType(Context, resultType);
-    break;
-  case UO_AddrOf:
-    resultType = CheckAddressOfOperand(Input, OpLoc);
-    CheckAddressOfNoDeref(InputExpr);
-    RecordModifiableNonNullParam(*this, InputExpr);
-    break;
-  case UO_Deref: {
-    Input = DefaultFunctionArrayLvalueConversion(Input.get());
-    if (Input.isInvalid())
-      return ExprError();
-    resultType =
-        CheckIndirectionOperand(*this, Input.get(), VK, OpLoc, IsAfterAmp);
-    break;
-  }
-  case UO_Plus:
-  case UO_Minus:
-    CanOverflow = Opc == UO_Minus &&
-                  isOverflowingIntegerType(Context, Input.get()->getType());
-    Input = UsualUnaryConversions(Input.get());
-    if (Input.isInvalid())
-      return ExprError();
-    // Unary plus and minus require promoting an operand of half vector to a
-    // float vector and truncating the result back to a half vector. For now, we
-    // do this only when HalfArgsAndReturns is set (that is, when the target is
-    // arm or arm64).
-    ConvertHalfVec = needsConversionOfHalfVec(true, Context, Input.get());
-
-    // If the operand is a half vector, promote it to a float vector.
-    if (ConvertHalfVec)
-      Input = convertVector(Input.get(), Context.FloatTy, *this);
-    resultType = Input.get()->getType();
-    if (resultType->isDependentType())
-      break;
-    if (resultType->isArithmeticType()) // C99 6.5.3.3p1
-      break;
-    else if (resultType->isVectorType() &&
-             // The z vector extensions don't allow + or - with bool vectors.
-             (!Context.getLangOpts().ZVector ||
-              resultType->castAs<VectorType>()->getVectorKind() !=
-                  VectorKind::AltiVecBool))
-      break;
-    else if (resultType->isSveVLSBuiltinType()) // SVE vectors allow + and -
-      break;
-    else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
-             Opc == UO_Plus && resultType->isPointerType())
+  if (InputExpr->isTypeDependent() &&
+      InputExpr->getType()->isSpecificBuiltinType(BuiltinType::Dependent)) {
+    resultType = Context.DependentTy;
+  } else {
+    switch (Opc) {
+    case UO_PreInc:
+    case UO_PreDec:
+    case UO_PostInc:
+    case UO_PostDec:
+      resultType =
+          CheckIncrementDecrementOperand(*this, Input.get(), VK, OK, OpLoc,
+                                         Opc == UO_PreInc || Opc == UO_PostInc,
+                                         Opc == UO_PreInc || Opc == UO_PreDec);
+      CanOverflow = isOverflowingIntegerType(Context, resultType);
       break;
-
-    return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                     << resultType << Input.get()->getSourceRange());
-
-  case UO_Not: // bitwise complement
-    Input = UsualUnaryConversions(Input.get());
-    if (Input.isInvalid())
-      return ExprError();
-    resultType = Input.get()->getType();
-    if (resultType->isDependentType())
+    case UO_AddrOf:
+      resultType = CheckAddressOfOperand(Input, OpLoc);
+      CheckAddressOfNoDeref(InputExpr);
+      RecordModifiableNonNullParam(*this, InputExpr);
       break;
-    // C99 6.5.3.3p1. We allow complex int and float as a GCC extension.
-    if (resultType->isComplexType() || resultType->isComplexIntegerType())
-      // C99 does not support '~' for complex conjugation.
-      Diag(OpLoc, diag::ext_integer_complement_complex)
-          << resultType << Input.get()->getSourceRange();
-    else if (resultType->hasIntegerRepresentation())
+    case UO_Deref: {
+      Input = DefaultFunctionArrayLvalueConversion(Input.get());
+      if (Input.isInvalid())
+        return ExprError();
+      resultType =
+          CheckIndirectionOperand(*this, Input.get(), VK, OpLoc, IsAfterAmp);
       break;
-    else if (resultType->isExtVectorType() && Context.getLangOpts().OpenCL) {
-      // OpenCL v1.1 s6.3.f: The bitwise operator not (~) does not operate
-      // on vector float types.
-      QualType T = resultType->castAs<ExtVectorType>()->getElementType();
-      if (!T->isIntegerType())
-        return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                         << resultType << Input.get()->getSourceRange());
-    } else {
-      return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                       << resultType << Input.get()->getSourceRange());
-    }
-    break;
-
-  case UO_LNot: // logical negation
-    // Unlike +/-/~, integer promotions aren't done here (C99 6.5.3.3p5).
-    Input = DefaultFunctionArrayLvalueConversion(Input.get());
-    if (Input.isInvalid())
-      return ExprError();
-    resultType = Input.get()->getType();
-
-    // Though we still have to promote half FP to float...
-    if (resultType->isHalfType() && !Context.getLangOpts().NativeHalfType) {
-      Input = ImpCastExprToType(Input.get(), Context.FloatTy, CK_FloatingCast)
-                  .get();
-      resultType = Context.FloatTy;
     }
+    case UO_Plus:
+    case UO_Minus:
+      CanOverflow = Opc == UO_Minus &&
+                    isOverflowingIntegerType(Context, Input.get()->getType());
+      Input = UsualUnaryConversions(Input.get());
+      if (Input.isInvalid())
+        return ExprError();
+      // Unary plus and minus require promoting an operand of half vector to a
+      // float vector and truncating the result back to a half vector. For now,
+      // we do this only when HalfArgsAndReturns is set (that is, when the
+      // target is arm or arm64).
+      ConvertHalfVec = needsConversionOfHalfVec(true, Context, Input.get());
+
+      // If the operand is a half vector, promote it to a float vector.
+      if (ConvertHalfVec)
+        Input = convertVector(Input.get(), Context.FloatTy, *this);
+      resultType = Input.get()->getType();
+      if (resultType->isArithmeticType()) // C99 6.5.3.3p1
+        break;
+      else if (resultType->isVectorType() &&
+               // The z vector extensions don't allow + or - with bool vectors.
+               (!Context.getLangOpts().ZVector ||
+                resultType->castAs<VectorType>()->getVectorKind() !=
+                    VectorKind::AltiVecBool))
+        break;
+      else if (resultType->isSveVLSBuiltinType()) // SVE vectors allow + and -
+        break;
+      else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
+               Opc == UO_Plus && resultType->isPointerType())
+        break;
 
-    // WebAsembly tables can't be used in unary expressions.
-    if (resultType->isPointerType() &&
-        resultType->getPointeeType().isWebAssemblyReferenceType()) {
       return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
                        << resultType << Input.get()->getSourceRange());
-    }
 
-    if (resultType->isDependentType())
-      break;
-    if (resultType->isScalarType() && !isScopedEnumerationType(resultType)) {
-      // C99 6.5.3.3p1: ok, fallthrough;
-      if (Context.getLangOpts().CPlusPlus) {
-        // C++03 [expr.unary.op]p8, C++0x [expr.unary.op]p9:
-        // operand contextually converted to bool.
-        Input = ImpCastExprToType(Input.get(), Context.BoolTy,
-                                  ScalarTypeToBooleanCastKind(resultType));
-      } else if (Context.getLangOpts().OpenCL &&
-                 Context.getLangOpts().OpenCLVersion < 120) {
-        // OpenCL v1.1 6.3.h: The logical operator not (!) does not
-        // operate on scalar float types.
-        if (!resultType->isIntegerType() && !resultType->isPointerType())
-          return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                           << resultType << Input.get()->getSourceRange());
-      }
-    } else if (resultType->isExtVectorType()) {
-      if (Context.getLangOpts().OpenCL &&
-          Context.getLangOpts().getOpenCLCompatibleVersion() < 120) {
-        // OpenCL v1.1 6.3.h: The logical operator not (!) does not
-        // operate on vector float types.
+    case UO_Not: // bitwise complement
+      Input = UsualUnaryConversions(Input.get());
+      if (Input.isInvalid())
+        return ExprError();
+      resultType = Input.get()->getType();
+      // C99 6.5.3.3p1. We allow complex int and float as a GCC extension.
+      if (resultType->isComplexType() || resultType->isComplexIntegerType())
+        // C99 does not support '~' for complex conjugation.
+        Diag(OpLoc, diag::ext_integer_complement_complex)
+            << resultType << Input.get()->getSourceRange();
+      else if (resultType->hasIntegerRepresentation())
+        break;
+      else if (resultType->isExtVectorType() && Context.getLangOpts().OpenCL) {
+        // OpenCL v1.1 s6.3.f: The bitwise operator not (~) does not operate
+        // on vector float types.
         QualType T = resultType->castAs<ExtVectorType>()->getElementType();
         if (!T->isIntegerType())
           return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
                            << resultType << Input.get()->getSourceRange());
+      } else {
+        return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                         << resultType << Input.get()->getSourceRange());
       }
-      // Vector logical not returns the signed variant of the operand type.
-      resultType = GetSignedVectorType(resultType);
       break;
-    } else if (Context.getLangOpts().CPlusPlus && resultType->isVectorType()) {
-      const VectorType *VTy = resultType->castAs<VectorType>();
-      if (VTy->getVectorKind() != VectorKind::Generic)
+
+    case UO_LNot: // logical negation
+      // Unlike +/-/~, integer promotions aren't done here (C99 6.5.3.3p5).
+      Input = DefaultFunctionArrayLvalueConversion(Input.get());
+      if (Input.isInvalid())
+        return ExprError();
+      resultType = Input.get()->getType();
+
+      // Though we still have to promote half FP to float...
+      if (resultType->isHalfType() && !Context.getLangOpts().NativeHalfType) {
+        Input = ImpCastExprToType(Input.get(), Context.FloatTy, CK_FloatingCast)
+                    .get();
+        resultType = Context.FloatTy;
+      }
+
+      // WebAsembly tables can't be used in unary expressions.
+      if (resultType->isPointerType() &&
+          resultType->getPointeeType().isWebAssemblyReferenceType()) {
         return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
                          << resultType << Input.get()->getSourceRange());
+      }
 
-      // Vector logical not returns the signed variant of the operand type.
-      resultType = GetSignedVectorType(resultType);
-      break;
-    } else {
-      return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
-                       << resultType << Input.get()->getSourceRange());
-    }
+      if (resultType->isScalarType() && !isScopedEnumerationType(resultType)) {
+        // C99 6.5.3.3p1: ok, fallthrough;
+        if (Context.getLangOpts().CPlusPlus) {
+          // C++03 [expr.unary.op]p8, C++0x [expr.unary.op]p9:
+          // operand contextually converted to bool.
+          Input = ImpCastExprToType(Input.get(), Context.BoolTy,
+                                    ScalarTypeToBooleanCastKind(resultType));
+        } else if (Context.getLangOpts().OpenCL &&
+                   Context.getLangOpts().OpenCLVersion < 120) {
+          // OpenCL v1.1 6.3.h: The logical operator not (!) does not
+          // operate on scalar float types.
+          if (!resultType->isIntegerType() && !resultType->isPointerType())
+            return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                             << resultType << Input.get()->getSourceRange());
+        }
+      } else if (resultType->isExtVectorType()) {
+        if (Context.getLangOpts().OpenCL &&
+            Context.getLangOpts().getOpenCLCompatibleVersion() < 120) {
+          // OpenCL v1.1 6.3.h: The logical operator not (!) does not
+          // operate on vector float types.
+          QualType T = resultType->castAs<ExtVectorType>()->getElementType();
+          if (!T->isIntegerType())
+            return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                             << resultType << Input.get()->getSourceRange());
+        }
+        // Vector logical not returns the signed variant of the operand type.
+        resultType = GetSignedVectorType(resultType);
+        break;
+      } else if (Context.getLangOpts().CPlusPlus &&
+                 resultType->isVectorType()) {
+        const VectorType *VTy = resultType->castAs<VectorType>();
+        if (VTy->getVectorKind() != VectorKind::Generic)
+          return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                           << resultType << Input.get()->getSourceRange());
 
-    // LNot always has type int. C99 6.5.3.3p5.
-    // In C++, it's bool. C++ 5.3.1p8
-    resultType = Context.getLogicalOperationType();
-    break;
-  case UO_Real:
-  case UO_Imag:
-    resultType = CheckRealImagOperand(*this, Input, OpLoc, Opc == UO_Real);
-    // _Real maps ordinary l-values into ordinary l-values. _Imag maps ordinary
-    // complex l-values to ordinary l-values and all other values to r-values.
-    if (Input.isInvalid())
-      return ExprError();
-    if (Opc == UO_Real || Input.get()->getType()->isAnyComplexType()) {
-      if (Input.get()->isGLValue() &&
-          Input.get()->getObjectKind() == OK_Ordinary)
-        VK = Input.get()->getValueKind();
-    } else if (!getLangOpts().CPlusPlus) {
-      // In C, a volatile scalar is read by __imag. In C++, it is not.
-      Input = DefaultLvalueConversion(Input.get());
+        // Vector logical not returns the signed variant of the operand type.
+        resultType = GetSignedVectorType(resultType);
+        break;
+      } else {
+        return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
+                         << resultType << Input.get()->getSourceRange());
+      }
+
+      // LNot always has type int. C99 6.5.3.3p5.
+      // In C++, it's bool. C++ 5.3.1p8
+      resultType = Context.getLogicalOperationType();
+      break;
+    case UO_Real:
+    case UO_Imag:
+      resultType = CheckRealImagOperand(*this, Input, OpLoc, Opc == UO_Real);
+      // _Real maps ordinary l-values into ordinary l-values. _Imag maps
+      // ordinary complex l-values to ordinary l-values and all other values to
+      // r-values.
+      if (Input.isInvalid())
+        return ExprError();
+      if (Opc == UO_Real || Input.get()->getType()->isAnyComplexType()) {
+        if (Input.get()->isGLValue() &&
+            Input.get()->getObjectKind() == OK_Ordinary)
+          VK = Input.get()->getValueKind();
+      } else if (!getLangOpts().CPlusPlus) {
+        // In C, a volatile scalar is read by __imag. In C++, it is not.
+        Input = DefaultLvalueConversion(Input.get());
+      }
+      break;
+    case UO_Extension:
+      resultType = Input.get()->getType();
+      VK = Input.get()->getValueKind();
+      OK = Input.get()->getObjectKind();
+      break;
+    case UO_Coawait:
+      // It's unnecessary to represent the pass-through operator co_await in the
+      // AST; just return the input expression instead.
+      assert(!Input.get()->getType()->isDependentType() &&
+             "the co_await expression must be non-dependant before "
+             "building operator co_await");
+      return Input;
     }
-    break;
-  case UO_Extension:
-    resultType = Input.get()->getType();
-    VK = Input.get()->getValueKind();
-    OK = Input.get()->getObjectKind();
-    break;
-  case UO_Coawait:
-    // It's unnecessary to represent the pass-through operator co_await in the
-    // AST; just return the input expression instead.
-    assert(!Input.get()->getType()->isDependentType() &&
-           "the co_await expression must be non-dependant before "
-           "building operator co_await");
-    return Input;
   }
   if (resultType.isNull() || Input.isInvalid())
     return ExprError();
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index c039b95293af..b10e5ba65eb1 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -16236,10 +16236,11 @@ ExprResult TreeTransform<Derived>::RebuildCXXOperatorCallExpr(
       return getSema().CreateBuiltinUnaryOp(OpLoc, Opc, First);
     }
   } else {
-    if (!First->getType()->isOverloadableType() &&
+    if (!First->isTypeDependent() && !Second->isTypeDependent() &&
+        !First->getType()->isOverloadableType() &&
         !Second->getType()->isOverloadableType()) {
-      // Neither of the arguments is an overloadable type, so try to
-      // create a built-in binary operation.
+      // Neither of the arguments is type-dependent or has an overloadable
+      // type, so try to create a built-in binary operation.
       BinaryOperatorKind Opc = BinaryOperator::getOverloadedOpcode(Op);
       ExprResult Result
         = SemaRef.CreateBuiltinBinOp(OpLoc, Opc, First, Second);
@@ -16250,12 +16251,8 @@ ExprResult TreeTransform<Derived>::RebuildCXXOperatorCallExpr(
     }
   }
 
-  // Add any functions found via argument-dependent lookup.
-  Expr *Args[2] = { First, Second };
-  unsigned NumArgs = 1 + (Second != nullptr);
-
   // Create the overloaded operator invocation for unary operators.
-  if (NumArgs == 1 || isPostIncDec) {
+  if (!Second || isPostIncDec) {
     UnaryOperatorKind Opc
       = UnaryOperator::getOverloadedOpcode(Op, isPostIncDec);
     return SemaRef.CreateOverloadedUnaryOp(OpLoc, Opc, Functions, First,
@@ -16264,8 +16261,8 @@ ExprResult TreeTransform<Derived>::RebuildCXXOperatorCallExpr(
 
   // Create the overloaded operator invocation for binary operators.
   BinaryOperatorKind Opc = BinaryOperator::getOverloadedOpcode(Op);
-  ExprResult Result = SemaRef.CreateOverloadedBinOp(
-      OpLoc, Opc, Functions, Args[0], Args[1], RequiresADL);
+  ExprResult Result = SemaRef.CreateOverloadedBinOp(OpLoc, Opc, Functions,
+                                                    First, Second, RequiresADL);
   if (Result.isInvalid())
     return ExprError();
 
diff --git a/clang/test/AST/ast-dump-expr-json.cpp b/clang/test/AST/ast-dump-expr-json.cpp
index 0fb07b0b434c..4b7365e554cb 100644
--- a/clang/test/AST/ast-dump-expr-json.cpp
+++ b/clang/test/AST/ast-dump-expr-json.cpp
@@ -4261,9 +4261,9 @@ void TestNonADLCall3() {
 // CHECK-NEXT:                     }
 // CHECK-NEXT:                    },
 // CHECK-NEXT:                    "type": {
-// CHECK-NEXT:                     "qualType": "<dependent type>"
+// CHECK-NEXT:                     "qualType": "V"
 // CHECK-NEXT:                    },
-// CHECK-NEXT:                    "valueCategory": "prvalue",
+// CHECK-NEXT:                    "valueCategory": "lvalue",
 // CHECK-NEXT:                    "isPostfix": false,
 // CHECK-NEXT:                    "opcode": "*",
 // CHECK-NEXT:                    "canOverflow": false,
diff --git a/clang/test/AST/ast-dump-expr.cpp b/clang/test/AST/ast-dump-expr.cpp
index 69e65e22d61d..4df5ba4276ab 100644
--- a/clang/test/AST/ast-dump-expr.cpp
+++ b/clang/test/AST/ast-dump-expr.cpp
@@ -282,7 +282,7 @@ void PrimaryExpressions(Ts... a) {
       // CHECK-NEXT: CompoundStmt
       // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <col:8> col:8 implicit 'V'
       // CHECK-NEXT: ParenListExpr 0x{{[^ ]*}} <col:8> 'NULL TYPE'
-      // CHECK-NEXT: UnaryOperator 0x{{[^ ]*}} <col:8> '<dependent type>' prefix '*' cannot overflow
+      // CHECK-NEXT: UnaryOperator 0x{{[^ ]*}} <col:8> 'V' lvalue prefix '*' cannot overflow
       // CHECK-NEXT: CXXThisExpr 0x{{[^ ]*}} <col:8> 'V *' this
     }
   };
diff --git a/clang/test/AST/ast-dump-lambda.cpp b/clang/test/AST/ast-dump-lambda.cpp
index ef8789cd97d3..a4d3fe4fbda5 100644
--- a/clang/test/AST/ast-dump-lambda.cpp
+++ b/clang/test/AST/ast-dump-lambda.cpp
@@ -81,7 +81,7 @@ template <typename... Ts> void test(Ts... a) {
 // CHECK-NEXT:    |         | | `-CompoundStmt {{.*}} <col:15, col:16>
 // CHECK-NEXT:    |         | `-FieldDecl {{.*}} <col:8> col:8{{( imported)?}} implicit 'V'
 // CHECK-NEXT:    |         |-ParenListExpr {{.*}} <col:8> 'NULL TYPE'
-// CHECK-NEXT:    |         | `-UnaryOperator {{.*}} <col:8> '<dependent type>' prefix '*' cannot overflow
+// CHECK-NEXT:    |         | `-UnaryOperator {{.*}} <col:8> 'V' lvalue prefix '*' cannot overflow
 // CHECK-NEXT:    |         |   `-CXXThisExpr {{.*}} <col:8> 'V *' this
 // CHECK-NEXT:    |         `-CompoundStmt {{.*}} <col:15, col:16>
 // CHECK-NEXT:    |-DeclStmt {{.*}} <line:22:3, col:11>
diff --git a/clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp b/clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp
new file mode 100644
index 000000000000..6744ce1cad17
--- /dev/null
+++ b/clang/test/CXX/expr/expr.unary/expr.unary.general/p1.cpp
@@ -0,0 +1,65 @@
+// RUN: %clang_cc1 -Wno-unused -fsyntax-only %s -verify
+
+struct A {
+  void operator*();
+  void operator+();
+  void operator-();
+  void operator!();
+  void operator~();
+  void operator&();
+  void operator++();
+  void operator--();
+};
+
+struct B { };
+
+template<typename T, typename U>
+void dependent(T t, T* pt, T U::* mpt, T(&ft)(), T(&at)[4]) {
+  *t;
+  +t;
+  -t;
+  !t;
+  ~t;
+  &t;
+  ++t;
+  --t;
+
+  *pt;
+  +pt;
+  -pt; // expected-error {{invalid argument type 'T *' to unary expression}}
+  !pt;
+  ~pt; // expected-error {{invalid argument type 'T *' to unary expression}}
+  &pt;
+  ++pt;
+  --pt;
+
+  *mpt; // expected-error {{indirection requires pointer operand ('T U::*' invalid)}}
+  +mpt; // expected-error {{invalid argument type 'T U::*' to unary expression}}
+  -mpt; // expected-error {{invalid argument type 'T U::*' to unary expression}}
+  !mpt;
+  ~mpt; // expected-error {{invalid argument type 'T U::*' to unary expression}}
+  &mpt;
+  ++mpt; // expected-error {{cannot increment value of type 'T U::*'}}
+  --mpt; // expected-error {{cannot decrement value of type 'T U::*'}}
+
+  *ft;
+  +ft;
+  -ft; // expected-error {{invalid argument type 'T (*)()' to unary expression}}
+  !ft;
+  ~ft; // expected-error {{invalid argument type 'T (*)()' to unary expression}}
+  &ft;
+  ++ft; // expected-error {{cannot increment value of type 'T ()'}}
+  --ft; // expected-error {{cannot decrement value of type 'T ()'}}
+
+  *at;
+  +at;
+  -at; // expected-error {{invalid argument type 'T *' to unary expression}}
+  !at;
+  ~at; // expected-error {{invalid argument type 'T *' to unary expression}}
+  &at;
+  ++at; // expected-error {{cannot increment value of type 'T[4]'}}
+  --at; // expected-error {{cannot decrement value of type 'T[4]'}}
+}
+
+// Make sure we only emit diagnostics once.
+template void dependent(A t, A* pt, A B::* mpt, A(&ft)(), A(&at)[4]);
diff --git a/clang/test/CXX/over/over.built/ast.cpp b/clang/test/CXX/over/over.built/ast.cpp
index 56a63431269f..78f86edb1e96 100644
--- a/clang/test/CXX/over/over.built/ast.cpp
+++ b/clang/test/CXX/over/over.built/ast.cpp
@@ -1,41 +1,139 @@
-// RUN: %clang_cc1 -std=c++17 -ast-dump %s -ast-dump-filter Test | FileCheck %s
+// RUN: %clang_cc1 -std=c++17 -Wno-unused -ast-dump %s -ast-dump-filter Test | FileCheck %s
 
-struct A{};
+namespace Test {
+  template<typename T, typename U>
+  void Unary(T t, T* pt, T U::* mpt, T(&ft)(), T(&at)[4]) {
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '*' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    *t;
 
-template <typename T, typename U>
-auto Test(T* pt, U* pu) {
-  // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '*'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  (void)*pt;
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '+' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    +t;
 
-  // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '++'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  (void)(++pt);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '-' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    -t;
 
-  // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '+'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  (void)(+pt);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '!' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    !t;
 
-  // CHECK: BinaryOperator {{.*}} '<dependent type>' '+'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 3
-  (void)(pt + 3);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '~' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    ~t;
 
-  // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  (void)(pt - pt);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    &t;
 
-  // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
-  (void)(pt - pu);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '++' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    ++t;
 
-  // CHECK: BinaryOperator {{.*}} '<dependent type>' '=='
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
-  // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
-  (void)(pt == pu);
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '--' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T' lvalue ParmVar {{.*}} 't' 'T'
+    --t;
 
-}
+    // CHECK: UnaryOperator {{.*}} 'T' lvalue prefix '*' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <LValueToRValue>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    *pt;
 
+    // CHECK: UnaryOperator {{.*}} 'T *' prefix '+' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <LValueToRValue>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    +pt;
 
+    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <PointerToBoolean>
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <LValueToRValue>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    !pt;
+
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    &pt;
+
+    // CHECK: UnaryOperator {{.*}} 'T *' lvalue prefix '++' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    ++pt;
+
+    // CHECK: UnaryOperator {{.*}} 'T *' lvalue prefix '--' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    --pt;
+
+    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <MemberPointerToBoolean>
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T U::*' <LValueToRValue>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T U::*' lvalue ParmVar {{.*}} 'mpt' 'T U::*'
+    !mpt;
+
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T U::*' lvalue ParmVar {{.*}} 'mpt' 'T U::*'
+    &mpt;
+
+    // CHECK: UnaryOperator {{.*}} 'T ()' lvalue prefix '*' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T (*)()' <FunctionToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
+    *ft;
+
+    // CHECK: UnaryOperator {{.*}} 'T (*)()' prefix '+' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T (*)()' <FunctionToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
+    +ft;
+
+    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <PointerToBoolean>
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T (*)()' <FunctionToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
+    !ft;
+
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T ()' lvalue ParmVar {{.*}} 'ft' 'T (&)()'
+    &ft;
+
+    // CHECK: UnaryOperator {{.*}} 'T' lvalue prefix '*' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <ArrayToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
+    *at;
+
+    // CHECK: UnaryOperator {{.*}} 'T *' prefix '+' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <ArrayToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
+    +at;
+
+    // CHECK: UnaryOperator {{.*}} 'bool' prefix '!' cannot overflow
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool' <PointerToBoolean>
+    // CHECK-NEXT: ImplicitCastExpr {{.*}} 'T *' <ArrayToPointerDecay>
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
+    !at;
+
+    // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '&' cannot overflow
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T[4]' lvalue ParmVar {{.*}} 'at' 'T (&)[4]'
+    &at;
+  }
+
+  template<typename T, typename U>
+  void Binary(T* pt, U* pu) {
+    // CHECK: BinaryOperator {{.*}} '<dependent type>' '+'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 3
+    pt + 3;
+
+    // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    pt - pt;
+
+    // CHECK: BinaryOperator {{.*}} '<dependent type>' '-'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
+    pt - pu;
+
+    // CHECK: BinaryOperator {{.*}} '<dependent type>' '=='
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
+    // CHECK-NEXT: DeclRefExpr {{.*}} 'U *' lvalue ParmVar {{.*}} 'pu' 'U *'
+    pt == pu;
+  }
+} // namespace Test
diff --git a/clang/test/CXX/over/over.built/p10.cpp b/clang/test/CXX/over/over.built/p10.cpp
index 678056da5820..8ff2396d0b6f 100644
--- a/clang/test/CXX/over/over.built/p10.cpp
+++ b/clang/test/CXX/over/over.built/p10.cpp
@@ -15,6 +15,6 @@ void f(int i, float f, bool b, char c, int* pi, A* pa, T* pt) {
 
   (void)-pi; // expected-error {{invalid argument type}}
   (void)-pa; // expected-error {{invalid argument type}}
-  (void)-pt; // FIXME: we should be able to give an error here.
+  (void)-pt; // expected-error {{invalid argument type}}
 }
 
diff --git a/clang/test/CXX/over/over.built/p11.cpp b/clang/test/CXX/over/over.built/p11.cpp
index 7ebf16b95439..f7a741db726d 100644
--- a/clang/test/CXX/over/over.built/p11.cpp
+++ b/clang/test/CXX/over/over.built/p11.cpp
@@ -7,6 +7,6 @@ void f(int i, float f, bool b, char c, int* pi, T* pt) {
   (void)~b;
   (void)~c;
   (void)~pi; // expected-error {{invalid argument type}}
-  (void)~pt; // FIXME: we should be able to give an error here.
+  (void)~pt; // expected-error {{invalid argument type}}
 }
 
diff --git a/clang/test/CXX/over/over.oper/over.oper.general/p1.cpp b/clang/test/CXX/over/over.oper/over.oper.general/p1.cpp
new file mode 100644
index 000000000000..d49fb0645751
--- /dev/null
+++ b/clang/test/CXX/over/over.oper/over.oper.general/p1.cpp
@@ -0,0 +1,173 @@
+// RUN: %clang_cc1 -std=c++20 -verify -Wno-unused %s
+
+template<typename T, typename U>
+void operator->*(T, U);
+
+template<typename T, typename U>
+void operator+(T, U);
+
+template<typename T, typename U>
+void operator-(T, U);
+
+template<typename T, typename U>
+void operator*(T, U);
+
+template<typename T, typename U>
+void operator/(T, U);
+
+template<typename T, typename U>
+void operator%(T, U);
+
+template<typename T, typename U>
+void operator^(T, U);
+
+template<typename T, typename U>
+void operator&(T, U);
+
+template<typename T, typename U>
+void operator|(T, U);
+
+template<typename T, typename U>
+void operator+=(T, U);
+
+template<typename T, typename U>
+void operator-=(T, U);
+
+template<typename T, typename U>
+void operator*=(T, U);
+
+template<typename T, typename U>
+void operator/=(T, U);
+
+template<typename T, typename U>
+void operator%=(T, U);
+
+template<typename T, typename U>
+void operator^=(T, U);
+
+template<typename T, typename U>
+void operator&=(T, U);
+
+template<typename T, typename U>
+void operator|=(T, U);
+
+template<typename T, typename U>
+void operator==(T, U);
+
+template<typename T, typename U>
+void operator!=(T, U);
+
+template<typename T, typename U>
+void operator<(T, U);
+
+template<typename T, typename U>
+void operator>(T, U);
+
+template<typename T, typename U>
+void operator<=(T, U);
+
+template<typename T, typename U>
+void operator>=(T, U);
+
+template<typename T, typename U>
+void operator<=>(T, U);
+
+template<typename T, typename U>
+void operator&&(T, U);
+
+template<typename T, typename U>
+void operator||(T, U);
+
+template<typename T, typename U>
+void operator<<(T, U);
+
+template<typename T, typename U>
+void operator>>(T, U);
+
+template<typename T, typename U>
+void operator<<=(T, U);
+
+template<typename T, typename U>
+void operator>>=(T, U);
+
+template<typename T, typename U>
+void operator,(T, U);
+
+template<typename T>
+void operator*(T);
+
+template<typename T>
+void operator&(T);
+
+template<typename T>
+void operator+(T);
+
+template<typename T>
+void operator-(T);
+
+template<typename T>
+void operator!(T);
+
+template<typename T>
+void operator~(T);
+
+template<typename T>
+void operator++(T);
+
+template<typename T>
+void operator--(T);
+
+template<typename T>
+void operator++(T, int);
+
+template<typename T>
+void operator--(T, int);
+
+template<typename T>
+void f(int *x) {
+  [&](auto *y) {
+    *y;
+    &y;
+    +y;
+    -y; // expected-error {{invalid argument type 'auto *' to unary expression}}
+    !y;
+    ~y; // expected-error {{invalid argument type 'auto *' to unary expression}}
+    ++y;
+    --y;
+    y++;
+    y--;
+    y->*x;
+    y + x;
+    y - x;
+    y * x;
+    y / x;
+    y % x;
+    y ^ x;
+    y & x;
+    y | x;
+    y += x;
+    y -= x;
+    y *= x;
+    y /= x;
+    y %= x;
+    y ^= x;
+    y &= x;
+    y |= x;
+    y == x;
+    y != x;
+    y < x;
+    y > x;
+    y <= x;
+    y >= x;
+    y <=> x;
+    y && x;
+    y || x;
+    y << x;
+    y >> x;
+    y <<= x;
+    y >>= x;
+    y, x;
+  };
+}
+
+template void f<int>(int*);
diff --git a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
index 3ca7c6c7eb8e..982e5372f5b0 100644
--- a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
@@ -357,17 +357,14 @@ namespace N0 {
       a->A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
       a->B::A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
 
-      // FIXME: An overloaded unary 'operator*' is built for these
-      // even though the operand is a pointer (to a dependent type).
-      // Type::isOverloadableType should return false for such cases.
-      (*this).x4;
-      (*this).B::x4;
-      (*this).A::x4;
-      (*this).B::A::x4;
-      (*this).f4();
-      (*this).B::f4();
-      (*this).A::f4();
-      (*this).B::A::f4();
+      (*this).x4; // expected-error{{no member named 'x4' in 'B<T>'}}
+      (*this).B::x4; // expected-error{{no member named 'x4' in 'B<T>'}}
+      (*this).A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
+      (*this).B::A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
+      (*this).f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
+      (*this).B::f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
+      (*this).A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
+      (*this).B::A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
 
       b.x4; // expected-error{{no member named 'x4' in 'B<T>'}}
       b.B::x4; // expected-error{{no member named 'x4' in 'B<T>'}}
@@ -399,15 +396,13 @@ namespace N1 {
       f<0>();
       this->f<0>();
       a->f<0>();
-      // FIXME: This should not require 'template'!
-      (*this).f<0>(); // expected-error{{missing 'template' keyword prior to dependent template name 'f'}}
+      (*this).f<0>();
       b.f<0>();
 
       x.f<0>();
       this->x.f<0>();
       a->x.f<0>();
-      // FIXME: This should not require 'template'!
-      (*this).x.f<0>(); // expected-error{{missing 'template' keyword prior to dependent template name 'f'}}
+      (*this).x.f<0>();
       b.x.f<0>();
 
       // FIXME: None of these should require 'template'!
diff --git a/clang/test/Frontend/noderef_templates.cpp b/clang/test/Frontend/noderef_templates.cpp
index 5fde6efd87c7..9e54cd5d7889 100644
--- a/clang/test/Frontend/noderef_templates.cpp
+++ b/clang/test/Frontend/noderef_templates.cpp
@@ -3,8 +3,8 @@
 #define NODEREF __attribute__((noderef))
 
 template <typename T>
-int func(T NODEREF *a) { // expected-note 2 {{a declared here}}
-  return *a + 1;         // expected-warning 2 {{dereferencing a; was declared with a 'noderef' type}}
+int func(T NODEREF *a) { // expected-note 3 {{a declared here}}
+  return *a + 1;         // expected-warning 3 {{dereferencing a; was declared with a 'noderef' type}}
 }
 
 void func() {
diff --git a/clang/test/SemaCXX/cxx2b-deducing-this.cpp b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
index 5f29a955e053..aa64530bd5be 100644
--- a/clang/test/SemaCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
@@ -19,7 +19,7 @@ struct S {
     // new and delete are implicitly static
     void *operator new(this unsigned long); // expected-error{{an explicit object parameter cannot appear in a static function}}
     void operator delete(this void*); // expected-error{{an explicit object parameter cannot appear in a static function}}
-    
+
     void g(this auto) const; // expected-error{{explicit object member function cannot have 'const' qualifier}}
     void h(this auto) &; // expected-error{{explicit object member function cannot have '&' qualifier}}
     void i(this auto) &&; // expected-error{{explicit object member function cannot have '&&' qualifier}}
@@ -198,9 +198,7 @@ void func(int i) {
 void TestMutationInLambda() {
     [i = 0](this auto &&){ i++; }();
     [i = 0](this auto){ i++; }();
-    [i = 0](this const auto&){ i++; }();
-    // expected-error@-1 {{cannot assign to a variable captured by copy in a non-mutable lambda}}
-    // expected-note@-2 {{in instantiation of}}
+    [i = 0](this const auto&){ i++; }(); // expected-error {{cannot assign to a variable captured by copy in a non-mutable lambda}}
 
     int x;
     const auto l1 = [x](this auto&) { x = 42; }; // expected-error {{cannot assign to a variable captured by copy in a non-mutable lambda}}
diff --git a/clang/test/SemaTemplate/class-template-spec.cpp b/clang/test/SemaTemplate/class-template-spec.cpp
index 56b8207bd9a4..faa54c367538 100644
--- a/clang/test/SemaTemplate/class-template-spec.cpp
+++ b/clang/test/SemaTemplate/class-template-spec.cpp
@@ -18,7 +18,7 @@ int test_specs(A<float, float> *a1, A<float, int> *a2) {
   return a1->x + a2->y;
 }
 
-int test_incomplete_specs(A<double, double> *a1, 
+int test_incomplete_specs(A<double, double> *a1,
                           A<double> *a2)
 {
   (void)a1->x; // expected-error{{member access into incomplete type}}
@@ -39,7 +39,7 @@ template <> struct X<int, int> { int foo(); }; // #1
 template <> struct X<float> { int bar(); };  // #2
 
 typedef int int_type;
-void testme(X<int_type> *x1, X<float, int> *x2) { 
+void testme(X<int_type> *x1, X<float, int> *x2) {
   (void)x1->foo(); // okay: refers to #1
   (void)x2->bar(); // okay: refers to #2
 }
@@ -53,7 +53,7 @@ struct A<char> {
 A<char>::A() { }
 
 // Make sure we can see specializations defined before the primary template.
-namespace N{ 
+namespace N{
   template<typename T> struct A0;
 }
 
@@ -97,7 +97,7 @@ namespace M {
   template<> struct ::A<long double>; // expected-error{{must occur at global scope}}
 }
 
-template<> struct N::B<char> { 
+template<> struct N::B<char> {
   int testf(int x) { return f(x); }
 };
 
@@ -138,9 +138,9 @@ namespace PR18009 {
 
   template <typename T> struct C {
     template <int N, int M> struct S;
-    template <int N> struct S<N, N ? **(T(*)[N])0 : 0> {}; // expected-error {{depends on a template parameter of the partial specialization}}
+    template <int N> struct S<N, N ? **(T(*)[N])0 : 0> {}; // ok
   };
-  C<int> c; // expected-note {{in instantiation of}}
+  C<int> c;
 
   template<int A> struct outer {
     template<int B, int C> struct inner {};
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index c08deb903f12..f26140675fd4 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -1572,9 +1572,9 @@ TEST_P(ASTMatchersTest, IsArrow_MatchesMemberVariablesViaArrow) {
   EXPECT_TRUE(
       matches("template <class T> class Y { void x() { this->m; } int m; };",
               memberExpr(isArrow())));
-  EXPECT_TRUE(
-      notMatches("template <class T> class Y { void x() { (*this).m; } };",
-                 cxxDependentScopeMemberExpr(isArrow())));
+  EXPECT_TRUE(notMatches(
+      "template <class T> class Y { void x() { (*this).m; } int m; };",
+      memberExpr(isArrow())));
 }
 
 TEST_P(ASTMatchersTest, IsArrow_MatchesStaticMemberVariablesViaArrow) {
-- 
GitLab


From 3a4c1b9b4428b08d4475decf74c11e0d328c5842 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Thu, 16 May 2024 09:55:36 +0800
Subject: [PATCH 0946/1206] [Serialization] Read the initializer for
 interesting static variables before consuming it (#92218)

Close https://github.com/llvm/llvm-project/issues/91418

Since we load the variable's initializers lazily, it'd be problematic if
the initializers dependent on each other.

For example,

```
SomeType a = ...;
SomeType b = a;
```

Previously, when we load variable `b`, we need to load the initializer,
then we need to load `a`. We can only mark the variable `b` as loaded
after we load `a`. Then `a` is always initialized before `b`. However,
it is not true after we implement lazy loading for initializers.

So here we try to load the initializers of static variables to make sure
they are passed to code generator by order. If we read any thing
interesting, we would consume that before emitting the current
declaration.
---
 clang/lib/Serialization/ASTReaderDecl.cpp    |  29 ++-
 clang/test/Modules/pr91418.cppm              |  67 +++++
 clang/test/OpenMP/nvptx_lambda_capturing.cpp | 246 +++++++++----------
 3 files changed, 216 insertions(+), 126 deletions(-)
 create mode 100644 clang/test/Modules/pr91418.cppm

diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 0c647086e304..a6254b70560c 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -4186,12 +4186,35 @@ void ASTReader::PassInterestingDeclsToConsumer() {
     GetDecl(ID);
   EagerlyDeserializedDecls.clear();
 
-  while (!PotentiallyInterestingDecls.empty()) {
-    Decl *D = PotentiallyInterestingDecls.front();
-    PotentiallyInterestingDecls.pop_front();
+  auto ConsumingPotentialInterestingDecls = [this]() {
+    while (!PotentiallyInterestingDecls.empty()) {
+      Decl *D = PotentiallyInterestingDecls.front();
+      PotentiallyInterestingDecls.pop_front();
+      if (isConsumerInterestedIn(D))
+        PassInterestingDeclToConsumer(D);
+    }
+  };
+  std::deque<Decl *> MaybeInterestingDecls =
+      std::move(PotentiallyInterestingDecls);
+  assert(PotentiallyInterestingDecls.empty());
+  while (!MaybeInterestingDecls.empty()) {
+    Decl *D = MaybeInterestingDecls.front();
+    MaybeInterestingDecls.pop_front();
+    // Since we load the variable's initializers lazily, it'd be problematic
+    // if the initializers dependent on each other. So here we try to load the
+    // initializers of static variables to make sure they are passed to code
+    // generator by order. If we read anything interesting, we would consume
+    // that before emitting the current declaration.
+    if (auto *VD = dyn_cast<VarDecl>(D);
+        VD && VD->isFileVarDecl() && !VD->isExternallyVisible())
+      VD->getInit();
+    ConsumingPotentialInterestingDecls();
     if (isConsumerInterestedIn(D))
       PassInterestingDeclToConsumer(D);
   }
+
+  // If we add any new potential interesting decl in the last call, consume it.
+  ConsumingPotentialInterestingDecls();
 }
 
 void ASTReader::loadDeclUpdateRecords(PendingUpdateRecord &Record) {
diff --git a/clang/test/Modules/pr91418.cppm b/clang/test/Modules/pr91418.cppm
new file mode 100644
index 000000000000..33fec992439d
--- /dev/null
+++ b/clang/test/Modules/pr91418.cppm
@@ -0,0 +1,67 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 -x c++-header %t/foo.h \
+// RUN:     -emit-pch -o %t/foo.pch
+// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/use.cpp -include-pch \
+// RUN:     %t/foo.pch -emit-llvm -o - | FileCheck %t/use.cpp
+
+//--- foo.h
+#ifndef FOO_H
+#define FOO_H
+typedef float __m128 __attribute__((__vector_size__(16), __aligned__(16)));
+
+static __inline__ __m128 __attribute__((__always_inline__, __min_vector_width__(128)))
+_mm_setr_ps(float __z, float __y, float __x, float __w)
+{
+  return __extension__ (__m128){ __z, __y, __x, __w };
+}
+
+typedef __m128 VR;
+
+inline VR MakeVR( float X, float Y, float Z, float W )
+{
+ return _mm_setr_ps( X, Y, Z, W );
+}
+
+extern "C" float sqrtf(float);
+
+namespace VectorSinConstantsSSE
+{
+  float a = (16 * sqrtf(0.225f));
+  VR A = MakeVR(a, a, a, a);
+  static const float b = (16 * sqrtf(0.225f));
+  static const VR B = MakeVR(b, b, b, b);
+}
+
+#endif // FOO_H
+
+//--- use.cpp
+#include "foo.h"
+float use() {
+    return VectorSinConstantsSSE::A[0] + VectorSinConstantsSSE::A[1] +
+           VectorSinConstantsSSE::A[2] + VectorSinConstantsSSE::A[3] +
+           VectorSinConstantsSSE::B[0] + VectorSinConstantsSSE::B[1] +
+           VectorSinConstantsSSE::B[2] + VectorSinConstantsSSE::B[3];
+}
+
+// CHECK: define{{.*}}@__cxx_global_var_init(
+// CHECK: store{{.*}}[[a_RESULT:%[a-zA-Z0-9]+]], ptr @_ZN21VectorSinConstantsSSE1aE
+
+// CHECK: define{{.*}}@__cxx_global_var_init.1(
+// CHECK: [[A_CALL:%[a-zA-Z0-9]+]] = call{{.*}}@_Z6MakeVRffff(
+// CHECK: store{{.*}}[[A_CALL]], ptr @_ZN21VectorSinConstantsSSE1AE
+
+// CHECK: define{{.*}}@__cxx_global_var_init.2(
+// CHECK: [[B_CALL:%[a-zA-Z0-9]+]] = call{{.*}}@_Z6MakeVRffff(
+// CHECK: store{{.*}}[[B_CALL]], ptr @_ZN21VectorSinConstantsSSEL1BE
+
+// CHECK: define{{.*}}@__cxx_global_var_init.3(
+// CHECK: store{{.*}}[[b_RESULT:%[a-zA-Z0-9]+]], ptr @_ZN21VectorSinConstantsSSEL1bE
+
+// CHECK: @_GLOBAL__sub_I_use.cpp
+// CHECK: call{{.*}}@__cxx_global_var_init(
+// CHECK: call{{.*}}@__cxx_global_var_init.1(
+// CHECK: call{{.*}}@__cxx_global_var_init.3(
+// CHECK: call{{.*}}@__cxx_global_var_init.2(
diff --git a/clang/test/OpenMP/nvptx_lambda_capturing.cpp b/clang/test/OpenMP/nvptx_lambda_capturing.cpp
index 641fbc38dd6b..efea8d4a0561 100644
--- a/clang/test/OpenMP/nvptx_lambda_capturing.cpp
+++ b/clang/test/OpenMP/nvptx_lambda_capturing.cpp
@@ -1165,8 +1165,113 @@ int main(int argc, char **argv) {
 // CHECK2-NEXT:    ret void
 //
 //
+// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l27
+// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK3-NEXT:  entry:
+// CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[L1:%.*]] = alloca [[CLASS_ANON:%.*]], align 8
+// CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
+// CHECK3-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l27_kernel_environment, ptr [[DYN_PTR]])
+// CHECK3-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP2]], -1
+// CHECK3-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
+// CHECK3:       user_code.entry:
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[L1]], ptr align 8 [[TMP3]], i64 8, i1 false)
+// CHECK3-NEXT:    store ptr [[L1]], ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP4]], i32 0, i32 0
+// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP5]], align 8
+// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i32 @_ZZN1S3fooEvENKUlvE_clEv(ptr noundef nonnull align 8 dereferenceable(8) [[TMP6]]) #[[ATTR7:[0-9]+]]
+// CHECK3-NEXT:    call void @__kmpc_target_deinit()
+// CHECK3-NEXT:    ret void
+// CHECK3:       worker.exit:
+// CHECK3-NEXT:    ret void
+//
+//
+// CHECK3-LABEL: define {{[^@]+}}@_ZZN1S3fooEvENKUlvE_clEv
+// CHECK3-SAME: (ptr noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]]) #[[ATTR2:[0-9]+]] comdat align 2 {
+// CHECK3-NEXT:  entry:
+// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[CLASS_ANON:%.*]], ptr [[THIS1]], i32 0, i32 0
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[TMP1]], i32 0, i32 0
+// CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+// CHECK3-NEXT:    ret i32 [[TMP2]]
+//
+//
+// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29
+// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR3:[0-9]+]] {
+// CHECK3-NEXT:  entry:
+// CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [2 x ptr], align 8
+// CHECK3-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
+// CHECK3-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_kernel_environment, ptr [[DYN_PTR]])
+// CHECK3-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP2]], -1
+// CHECK3-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
+// CHECK3:       user_code.entry:
+// CHECK3-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1:[0-9]+]])
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [2 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 0
+// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP5]], align 8
+// CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [2 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 1
+// CHECK3-NEXT:    store ptr [[TMP4]], ptr [[TMP6]], align 8
+// CHECK3-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP3]], i32 1, i32 -1, i32 -1, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_omp_outlined, ptr null, ptr [[CAPTURED_VARS_ADDRS]], i64 2)
+// CHECK3-NEXT:    call void @__kmpc_target_deinit()
+// CHECK3-NEXT:    ret void
+// CHECK3:       worker.exit:
+// CHECK3-NEXT:    ret void
+//
+//
+// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_omp_outlined
+// CHECK3-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR4:[0-9]+]] {
+// CHECK3-NEXT:  entry:
+// CHECK3-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[L1:%.*]] = alloca [[CLASS_ANON:%.*]], align 8
+// CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[L1]], ptr align 8 [[TMP2]], i64 8, i1 false)
+// CHECK3-NEXT:    store ptr [[L1]], ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP3]], i32 0, i32 0
+// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP4]], align 8
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i32 @_ZZN1S3fooEvENKUlvE_clEv(ptr noundef nonnull align 8 dereferenceable(8) [[TMP5]]) #[[ATTR7]]
+// CHECK3-NEXT:    ret void
+//
+//
 // CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}_main_l41
-// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], i64 noundef [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], i64 noundef [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR0]] {
 // CHECK3-NEXT:  entry:
 // CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[ARGC_ADDR:%.*]] = alloca i64, align 8
@@ -1178,7 +1283,7 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[_TMP1:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L3:%.*]] = alloca [[CLASS_ANON:%.*]], align 8
+// CHECK3-NEXT:    [[L3:%.*]] = alloca [[CLASS_ANON_1:%.*]], align 8
 // CHECK3-NEXT:    [[_TMP4:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[B5:%.*]] = alloca i32, align 4
 // CHECK3-NEXT:    [[_TMP6:%.*]] = alloca ptr, align 8
@@ -1214,20 +1319,20 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    store i32 [[TMP9]], ptr [[C7]], align 4
 // CHECK3-NEXT:    store ptr [[C7]], ptr [[_TMP8]], align 8
 // CHECK3-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[_TMP4]], align 8
-// CHECK3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP10]], i32 0, i32 0
+// CHECK3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP10]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[ARGC_ADDR]], ptr [[TMP11]], align 8
-// CHECK3-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP10]], i32 0, i32 1
+// CHECK3-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP10]], i32 0, i32 1
 // CHECK3-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[_TMP6]], align 8
 // CHECK3-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
-// CHECK3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP10]], i32 0, i32 2
+// CHECK3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP10]], i32 0, i32 2
 // CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP8]], align 8
 // CHECK3-NEXT:    store ptr [[TMP15]], ptr [[TMP14]], align 8
-// CHECK3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP10]], i32 0, i32 3
+// CHECK3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP10]], i32 0, i32 3
 // CHECK3-NEXT:    store ptr [[D_ADDR]], ptr [[TMP16]], align 8
-// CHECK3-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP10]], i32 0, i32 4
+// CHECK3-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP10]], i32 0, i32 4
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP17]], align 8
 // CHECK3-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP4]], align 8
-// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i64 @"_ZZ4mainENK3$_0clEv"(ptr noundef nonnull align 8 dereferenceable(40) [[TMP18]]) #[[ATTR7:[0-9]+]]
+// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i64 @"_ZZ4mainENK3$_0clEv"(ptr noundef nonnull align 8 dereferenceable(40) [[TMP18]]) #[[ATTR7]]
 // CHECK3-NEXT:    call void @__kmpc_target_deinit()
 // CHECK3-NEXT:    ret void
 // CHECK3:       worker.exit:
@@ -1235,7 +1340,7 @@ int main(int argc, char **argv) {
 //
 //
 // CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}_main_l43
-// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR3:[0-9]+]] {
+// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR3]] {
 // CHECK3-NEXT:  entry:
 // CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[ARGC_ADDR:%.*]] = alloca ptr, align 8
@@ -1267,7 +1372,7 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP5]], -1
 // CHECK3-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
 // CHECK3:       user_code.entry:
-// CHECK3-NEXT:    [[TMP6:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1:[0-9]+]])
+// CHECK3-NEXT:    [[TMP6:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
 // CHECK3-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[TMP]], align 8
 // CHECK3-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 8
 // CHECK3-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[D_ADDR]], align 8
@@ -1292,7 +1397,7 @@ int main(int argc, char **argv) {
 //
 //
 // CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}_main_l43_omp_outlined
-// CHECK3-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR4:[0-9]+]] {
+// CHECK3-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR4]] {
 // CHECK3-NEXT:  entry:
 // CHECK3-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
@@ -1305,7 +1410,7 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[_TMP1:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L3:%.*]] = alloca [[CLASS_ANON:%.*]], align 8
+// CHECK3-NEXT:    [[L3:%.*]] = alloca [[CLASS_ANON_1:%.*]], align 8
 // CHECK3-NEXT:    [[_TMP4:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[ARGC5:%.*]] = alloca i32, align 4
 // CHECK3-NEXT:    [[B6:%.*]] = alloca i32, align 4
@@ -1345,128 +1450,23 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK3-NEXT:    store i32 [[TMP11]], ptr [[A10]], align 4
 // CHECK3-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[_TMP4]], align 8
-// CHECK3-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP12]], i32 0, i32 0
+// CHECK3-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP12]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[ARGC5]], ptr [[TMP13]], align 8
-// CHECK3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP12]], i32 0, i32 1
+// CHECK3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP12]], i32 0, i32 1
 // CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8
 // CHECK3-NEXT:    store ptr [[TMP15]], ptr [[TMP14]], align 8
-// CHECK3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP12]], i32 0, i32 2
+// CHECK3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP12]], i32 0, i32 2
 // CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP9]], align 8
 // CHECK3-NEXT:    store ptr [[TMP17]], ptr [[TMP16]], align 8
-// CHECK3-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP12]], i32 0, i32 3
+// CHECK3-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP12]], i32 0, i32 3
 // CHECK3-NEXT:    store ptr [[D_ADDR]], ptr [[TMP18]], align 8
-// CHECK3-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP12]], i32 0, i32 4
+// CHECK3-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP12]], i32 0, i32 4
 // CHECK3-NEXT:    store ptr [[A10]], ptr [[TMP19]], align 8
 // CHECK3-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[_TMP4]], align 8
 // CHECK3-NEXT:    [[CALL:%.*]] = call noundef i64 @"_ZZ4mainENK3$_0clEv"(ptr noundef nonnull align 8 dereferenceable(40) [[TMP20]]) #[[ATTR7]]
 // CHECK3-NEXT:    ret void
 //
 //
-// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l27
-// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR0]] {
-// CHECK3-NEXT:  entry:
-// CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L1:%.*]] = alloca [[CLASS_ANON_1:%.*]], align 8
-// CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
-// CHECK3-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l27_kernel_environment, ptr [[DYN_PTR]])
-// CHECK3-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP2]], -1
-// CHECK3-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
-// CHECK3:       user_code.entry:
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[L1]], ptr align 8 [[TMP3]], i64 8, i1 false)
-// CHECK3-NEXT:    store ptr [[L1]], ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP4]], i32 0, i32 0
-// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP5]], align 8
-// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i32 @_ZZN1S3fooEvENKUlvE_clEv(ptr noundef nonnull align 8 dereferenceable(8) [[TMP6]]) #[[ATTR7]]
-// CHECK3-NEXT:    call void @__kmpc_target_deinit()
-// CHECK3-NEXT:    ret void
-// CHECK3:       worker.exit:
-// CHECK3-NEXT:    ret void
-//
-//
-// CHECK3-LABEL: define {{[^@]+}}@_ZZN1S3fooEvENKUlvE_clEv
-// CHECK3-SAME: (ptr noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]]) #[[ATTR2:[0-9]+]] comdat align 2 {
-// CHECK3-NEXT:  entry:
-// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[CLASS_ANON_1:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
-// CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[TMP1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
-// CHECK3-NEXT:    ret i32 [[TMP2]]
-//
-//
-// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29
-// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR3]] {
-// CHECK3-NEXT:  entry:
-// CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [2 x ptr], align 8
-// CHECK3-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
-// CHECK3-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_kernel_environment, ptr [[DYN_PTR]])
-// CHECK3-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP2]], -1
-// CHECK3-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
-// CHECK3:       user_code.entry:
-// CHECK3-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [2 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 0
-// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP5]], align 8
-// CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [2 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 1
-// CHECK3-NEXT:    store ptr [[TMP4]], ptr [[TMP6]], align 8
-// CHECK3-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP3]], i32 1, i32 -1, i32 -1, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_omp_outlined, ptr null, ptr [[CAPTURED_VARS_ADDRS]], i64 2)
-// CHECK3-NEXT:    call void @__kmpc_target_deinit()
-// CHECK3-NEXT:    ret void
-// CHECK3:       worker.exit:
-// CHECK3-NEXT:    ret void
-//
-//
-// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_omp_outlined
-// CHECK3-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR4]] {
-// CHECK3-NEXT:  entry:
-// CHECK3-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L1:%.*]] = alloca [[CLASS_ANON_1:%.*]], align 8
-// CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[L1]], ptr align 8 [[TMP2]], i64 8, i1 false)
-// CHECK3-NEXT:    store ptr [[L1]], ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP3]], i32 0, i32 0
-// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP4]], align 8
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i32 @_ZZN1S3fooEvENKUlvE_clEv(ptr noundef nonnull align 8 dereferenceable(8) [[TMP5]]) #[[ATTR7]]
-// CHECK3-NEXT:    ret void
-//
-//
 // CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooIZN1S3fooEvEUlvE_EiRKT__l18
 // CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[T:%.*]]) #[[ATTR3]] {
 // CHECK3-NEXT:  entry:
@@ -1500,7 +1500,7 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[T_ADDR:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[T1:%.*]] = alloca [[CLASS_ANON_1:%.*]], align 8
+// CHECK3-NEXT:    [[T1:%.*]] = alloca [[CLASS_ANON:%.*]], align 8
 // CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK3-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
-- 
GitLab


From 1dd0d3cf40f21b842dbee107b3d203db9fbaa4ae Mon Sep 17 00:00:00 2001
From: Dhruv Chawla <dhruvc@nvidia.com>
Date: Thu, 16 May 2024 08:08:06 +0530
Subject: [PATCH 0947/1206] [AArch64][GISel] Fold COPY(y:gpr, DUP(x:fpr, i)) ->
 UMOV(y:gpr, x:fpr, i) (#89017)

This patch adds a peephole to AArch64PostSelectOptimize for codegen
that is caused by RegBankSelect limiting G_EXTRACT_VECTOR_ELT
only to FPR registers in both the input and output registers. This can
cause a generation of COPY from FPR to GPR when, for example, the
output register of the G_EXTRACT_VECTOR_ELT is used in a branch
condition.

This was noticed when looking at codegen differences between SDAG and GI
for the s1279 kernel in the TSVC benchmark.
---
 .../GISel/AArch64PostSelectOptimize.cpp       |  68 ++++++++-
 llvm/test/CodeGen/AArch64/aarch64-mulv.ll     | 117 +++++----------
 llvm/test/CodeGen/AArch64/aarch64-smull.ll    | 133 ++++++++----------
 llvm/test/CodeGen/AArch64/arm64-neon-copy.ll  |  12 +-
 llvm/test/CodeGen/AArch64/bitcast.ll          |  24 ++--
 llvm/test/CodeGen/AArch64/insertextract.ll    |  13 +-
 llvm/test/CodeGen/AArch64/ptradd.ll           |   7 +-
 llvm/test/CodeGen/AArch64/reduce-and.ll       |  42 +++---
 llvm/test/CodeGen/AArch64/reduce-or.ll        |  42 +++---
 llvm/test/CodeGen/AArch64/reduce-xor.ll       |  42 +++---
 10 files changed, 238 insertions(+), 262 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostSelectOptimize.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostSelectOptimize.cpp
index 11866f2dd186..e9aed60595e6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostSelectOptimize.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostSelectOptimize.cpp
@@ -48,6 +48,7 @@ private:
   bool doPeepholeOpts(MachineBasicBlock &MBB);
   /// Look for cross regclass copies that can be trivially eliminated.
   bool foldSimpleCrossClassCopies(MachineInstr &MI);
+  bool foldCopyDup(MachineInstr &MI);
 };
 } // end anonymous namespace
 
@@ -105,7 +106,10 @@ unsigned getNonFlagSettingVariant(unsigned Opc) {
 bool AArch64PostSelectOptimize::doPeepholeOpts(MachineBasicBlock &MBB) {
   bool Changed = false;
   for (auto &MI : make_early_inc_range(make_range(MBB.begin(), MBB.end()))) {
-    Changed |= foldSimpleCrossClassCopies(MI);
+    bool CurrentIterChanged = foldSimpleCrossClassCopies(MI);
+    if (!CurrentIterChanged)
+      CurrentIterChanged |= foldCopyDup(MI);
+    Changed |= CurrentIterChanged;
   }
   return Changed;
 }
@@ -158,6 +162,68 @@ bool AArch64PostSelectOptimize::foldSimpleCrossClassCopies(MachineInstr &MI) {
   return true;
 }
 
+bool AArch64PostSelectOptimize::foldCopyDup(MachineInstr &MI) {
+  if (!MI.isCopy())
+    return false;
+
+  auto *MF = MI.getMF();
+  auto &MRI = MF->getRegInfo();
+  auto *TII = MF->getSubtarget().getInstrInfo();
+
+  // Optimize COPY(y:GPR, DUP(x:FPR, i)) -> UMOV(y:GPR, x:FPR, i).
+  // Here Dst is y and Src is the result of DUP.
+  Register Dst = MI.getOperand(0).getReg();
+  Register Src = MI.getOperand(1).getReg();
+
+  if (!Dst.isVirtual() || !Src.isVirtual())
+    return false;
+
+  auto TryMatchDUP = [&](const TargetRegisterClass *GPRRegClass,
+                         const TargetRegisterClass *FPRRegClass, unsigned DUP,
+                         unsigned UMOV) {
+    if (MRI.getRegClassOrNull(Dst) != GPRRegClass ||
+        MRI.getRegClassOrNull(Src) != FPRRegClass)
+      return false;
+
+    // There is a special case when one of the uses is COPY(z:FPR, y:GPR).
+    // In this case, we get COPY(z:FPR, COPY(y:GPR, DUP(x:FPR, i))), which can
+    // be folded by peephole-opt into just DUP(z:FPR, i), so this transform is
+    // not worthwhile in that case.
+    for (auto &Use : MRI.use_nodbg_instructions(Dst)) {
+      if (!Use.isCopy())
+        continue;
+
+      Register UseOp0 = Use.getOperand(0).getReg();
+      Register UseOp1 = Use.getOperand(1).getReg();
+      if (UseOp0.isPhysical() || UseOp1.isPhysical())
+        return false;
+
+      if (MRI.getRegClassOrNull(UseOp0) == FPRRegClass &&
+          MRI.getRegClassOrNull(UseOp1) == GPRRegClass)
+        return false;
+    }
+
+    MachineInstr *SrcMI = MRI.getUniqueVRegDef(Src);
+    if (!SrcMI || SrcMI->getOpcode() != DUP || !MRI.hasOneNonDBGUse(Src))
+      return false;
+
+    Register DupSrc = SrcMI->getOperand(1).getReg();
+    int64_t DupImm = SrcMI->getOperand(2).getImm();
+
+    BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(UMOV), Dst)
+        .addReg(DupSrc)
+        .addImm(DupImm);
+    SrcMI->eraseFromParent();
+    MI.eraseFromParent();
+    return true;
+  };
+
+  return TryMatchDUP(&AArch64::GPR32RegClass, &AArch64::FPR32RegClass,
+                     AArch64::DUPi32, AArch64::UMOVvi32) ||
+         TryMatchDUP(&AArch64::GPR64RegClass, &AArch64::FPR64RegClass,
+                     AArch64::DUPi64, AArch64::UMOVvi64);
+}
+
 bool AArch64PostSelectOptimize::optimizeNZCVDefs(MachineBasicBlock &MBB) {
   // If we find a dead NZCV implicit-def, we
   // - try to convert the operation to a non-flag-setting equivalent
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
index 7b7ca9d8ffc2..e11ae9a25159 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
@@ -25,22 +25,13 @@ declare i64 @llvm.vector.reduce.mul.v4i64(<4 x i64>)
 declare i128 @llvm.vector.reduce.mul.v2i128(<2 x i128>)
 
 define i8 @mulv_v2i8(<2 x i8> %a) {
-; CHECK-SD-LABEL: mulv_v2i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    mul w0, w9, w8
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: mulv_v2i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    mul w0, w8, w9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: mulv_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mul w0, w9, w8
+; CHECK-NEXT:    ret
 entry:
   %arg1 = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> %a)
   ret i8 %arg1
@@ -230,22 +221,13 @@ entry:
 }
 
 define i16 @mulv_v2i16(<2 x i16> %a) {
-; CHECK-SD-LABEL: mulv_v2i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    mul w0, w9, w8
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: mulv_v2i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    mul w0, w8, w9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: mulv_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mul w0, w9, w8
+; CHECK-NEXT:    ret
 entry:
   %arg1 = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> %a)
   ret i16 %arg1
@@ -372,22 +354,13 @@ entry:
 }
 
 define i32 @mulv_v2i32(<2 x i32> %a) {
-; CHECK-SD-LABEL: mulv_v2i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    mul w0, w9, w8
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: mulv_v2i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    mul w0, w8, w9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: mulv_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mul w0, w9, w8
+; CHECK-NEXT:    ret
 entry:
   %arg1 = call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> %a)
   ret i32 %arg1
@@ -424,10 +397,9 @@ define i32 @mulv_v4i32(<4 x i32> %a) {
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
 ; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    mov w8, v0.s[1]
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mul w0, w9, w8
 ; CHECK-GI-NEXT:    ret
 entry:
   %arg1 = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %a)
@@ -452,10 +424,9 @@ define i32 @mulv_v8i32(<8 x i32> %a) {
 ; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v2.2s
 ; CHECK-GI-NEXT:    mul v1.2s, v1.2s, v3.2s
 ; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    mov w8, v0.s[1]
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mul w0, w9, w8
 ; CHECK-GI-NEXT:    ret
 entry:
   %arg1 = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> %a)
@@ -463,20 +434,12 @@ entry:
 }
 
 define i64 @mulv_v2i64(<2 x i64> %a) {
-; CHECK-SD-LABEL: mulv_v2i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    fmov x9, d0
-; CHECK-SD-NEXT:    mul x0, x9, x8
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: mulv_v2i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mul x0, x8, x9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: mulv_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    fmov x9, d0
+; CHECK-NEXT:    mul x0, x9, x8
+; CHECK-NEXT:    ret
 entry:
   %arg1 = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> %a)
   ret i64 %arg1
@@ -522,14 +485,12 @@ define i64 @mulv_v4i64(<4 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: mulv_v4i64:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    fmov x10, d3
-; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mul x9, x9, x10
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    mul x8, x10, x8
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    mul x9, x10, x9
 ; CHECK-GI-NEXT:    mul x0, x8, x9
 ; CHECK-GI-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
index 540471a05901..307aa397eabb 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-smull.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
@@ -279,17 +279,15 @@ define <2 x i64> @smull_zext_v2i32_v2i64(ptr %A, ptr %B) nounwind {
 ; CHECK-GI-NEXT:    ldr d0, [x1]
 ; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
 ; CHECK-GI-NEXT:    fmov d1, x8
-; CHECK-GI-NEXT:    mov d3, v0.d[1]
+; CHECK-GI-NEXT:    fmov x11, d0
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
-; CHECK-GI-NEXT:    fmov x9, d0
-; CHECK-GI-NEXT:    fmov x10, d3
-; CHECK-GI-NEXT:    mov d2, v1.d[1]
-; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    mov x9, v0.d[1]
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    mul x9, x9, x10
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
   %load.A = load <2 x i16>, ptr %A
   %load.B = load <2 x i32>, ptr %B
@@ -324,16 +322,14 @@ define <2 x i64> @smull_zext_and_v2i32_v2i64(ptr %A, ptr %B) nounwind {
 ; CHECK-GI-NEXT:    ldr d1, [x1]
 ; CHECK-GI-NEXT:    sshll v1.2d, v1.2s, #0
 ; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-GI-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    fmov x10, d3
-; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    mul x9, x9, x10
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
   %load.A = load <2 x i32>, ptr %A
   %and.A = and <2 x i32> %load.A, <i32 u0x7FFFFFFF, i32 u0x7FFFFFFF>
@@ -1052,16 +1048,14 @@ define <2 x i64> @smull_extvec_v2i32_v2i64(<2 x i32> %arg) nounwind {
 ; CHECK-GI-NEXT:    adrp x8, .LCPI36_0
 ; CHECK-GI-NEXT:    sshll v0.2d, v0.2s, #0
 ; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI36_0]
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    fmov x10, d3
-; CHECK-GI-NEXT:    mul x9, x9, x10
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
   %tmp3 = sext <2 x i32> %arg to <2 x i64>
   %tmp4 = mul <2 x i64> %tmp3, <i64 -1234, i64 -1234>
@@ -1169,16 +1163,14 @@ define <2 x i64> @umull_extvec_v2i32_v2i64(<2 x i32> %arg) nounwind {
 ; CHECK-GI-NEXT:    adrp x8, .LCPI40_0
 ; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
 ; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI40_0]
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    fmov x10, d3
-; CHECK-GI-NEXT:    mul x9, x9, x10
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
   %tmp3 = zext <2 x i32> %arg to <2 x i64>
   %tmp4 = mul <2 x i64> %tmp3, <i64 1234, i64 1234>
@@ -1272,17 +1264,15 @@ define <2 x i64> @amull_extvec_v2i32_v2i64(<2 x i32> %arg) nounwind {
 ; CHECK-GI-NEXT:    adrp x8, .LCPI43_0
 ; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
 ; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI43_0]
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
 ; CHECK-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    fmov x10, d3
-; CHECK-GI-NEXT:    mul x9, x9, x10
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-GI-NEXT:    ret
   %tmp3 = zext <2 x i32> %arg to <2 x i64>
@@ -1901,17 +1891,15 @@ define <2 x i64> @umull_and_v2i64(<2 x i32> %src1, <2 x i64> %src2) {
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    movi v2.2d, #0x000000000000ff
 ; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
 ; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    fmov x10, d3
-; CHECK-GI-NEXT:    mul x9, x9, x10
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    fmov d0, x10
+; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    ret
 entry:
   %in1 = zext <2 x i32> %src1 to <2 x i64>
@@ -1947,26 +1935,22 @@ define <4 x i64> @umull_and_v4i64(<4 x i32> %src1, <4 x i64> %src2) {
 ; CHECK-GI-NEXT:    ushll v4.2d, v0.2s, #0
 ; CHECK-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
 ; CHECK-GI-NEXT:    fmov x8, d4
+; CHECK-GI-NEXT:    mov x10, v4.d[1]
+; CHECK-GI-NEXT:    mov x13, v0.d[1]
 ; CHECK-GI-NEXT:    and v1.16b, v1.16b, v3.16b
 ; CHECK-GI-NEXT:    and v2.16b, v2.16b, v3.16b
-; CHECK-GI-NEXT:    mov d3, v4.d[1]
 ; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mov d4, v1.d[1]
-; CHECK-GI-NEXT:    fmov x10, d2
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    fmov x12, d2
+; CHECK-GI-NEXT:    mov x11, v1.d[1]
+; CHECK-GI-NEXT:    mov x14, v2.d[1]
 ; CHECK-GI-NEXT:    mul x8, x8, x9
 ; CHECK-GI-NEXT:    fmov x9, d0
-; CHECK-GI-NEXT:    mov d0, v2.d[1]
-; CHECK-GI-NEXT:    fmov x11, d4
-; CHECK-GI-NEXT:    mul x9, x9, x10
-; CHECK-GI-NEXT:    fmov x10, d3
-; CHECK-GI-NEXT:    fmov x12, d0
-; CHECK-GI-NEXT:    fmov d0, x8
 ; CHECK-GI-NEXT:    mul x10, x10, x11
-; CHECK-GI-NEXT:    fmov x11, d1
-; CHECK-GI-NEXT:    fmov d1, x9
-; CHECK-GI-NEXT:    mul x11, x11, x12
+; CHECK-GI-NEXT:    mul x9, x9, x12
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    mul x11, x13, x14
 ; CHECK-GI-NEXT:    mov v0.d[1], x10
+; CHECK-GI-NEXT:    fmov d1, x9
 ; CHECK-GI-NEXT:    mov v1.d[1], x11
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -1999,20 +1983,17 @@ define <4 x i64> @umull_and_v4i64_dup(<4 x i32> %src1, i64 %src2) {
 ; CHECK-GI-NEXT:    ushll v1.2d, v0.2s, #0
 ; CHECK-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
 ; CHECK-GI-NEXT:    dup v2.2d, x8
-; CHECK-GI-NEXT:    mov d3, v1.d[1]
 ; CHECK-GI-NEXT:    fmov x8, d1
-; CHECK-GI-NEXT:    fmov x10, d0
-; CHECK-GI-NEXT:    mov d1, v2.d[1]
+; CHECK-GI-NEXT:    fmov x12, d0
+; CHECK-GI-NEXT:    mov x10, v1.d[1]
 ; CHECK-GI-NEXT:    fmov x9, d2
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    mov x11, v2.d[1]
+; CHECK-GI-NEXT:    mov x13, v0.d[1]
 ; CHECK-GI-NEXT:    mul x8, x8, x9
-; CHECK-GI-NEXT:    fmov x11, d1
-; CHECK-GI-NEXT:    fmov x12, d2
-; CHECK-GI-NEXT:    mul x9, x10, x9
-; CHECK-GI-NEXT:    fmov x10, d3
+; CHECK-GI-NEXT:    mul x9, x12, x9
 ; CHECK-GI-NEXT:    mul x10, x10, x11
 ; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    mul x11, x12, x11
+; CHECK-GI-NEXT:    mul x11, x13, x11
 ; CHECK-GI-NEXT:    fmov d1, x9
 ; CHECK-GI-NEXT:    mov v0.d[1], x10
 ; CHECK-GI-NEXT:    mov v1.d[1], x11
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
index 749d6071c98d..43d5ab5ab54e 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll
@@ -1488,8 +1488,7 @@ define <4 x i16> @test_dup_v2i32_v4i16(<2 x i32> %a) {
 ; CHECK-GI-LABEL: test_dup_v2i32_v4i16:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov s0, v0.s[1]
-; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov w8, v0.s[1]
 ; CHECK-GI-NEXT:    dup v0.4h, w8
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -1510,8 +1509,7 @@ define <8 x i16> @test_dup_v4i32_v8i16(<4 x i32> %a) {
 ;
 ; CHECK-GI-LABEL: test_dup_v4i32_v8i16:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov s0, v0.s[3]
-; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov w8, v0.s[3]
 ; CHECK-GI-NEXT:    dup v0.8h, w8
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -1578,8 +1576,7 @@ define <8 x i16> @test_dup_v2i64_v8i16(<2 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: test_dup_v2i64_v8i16:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov d0, v0.d[1]
-; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
 ; CHECK-GI-NEXT:    dup v0.8h, w8
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -1626,8 +1623,7 @@ define <4 x i16> @test_dup_v4i32_v4i16(<4 x i32> %a) {
 ;
 ; CHECK-GI-LABEL: test_dup_v4i32_v4i16:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov s0, v0.s[1]
-; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov w8, v0.s[1]
 ; CHECK-GI-NEXT:    dup v0.4h, w8
 ; CHECK-GI-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/AArch64/bitcast.ll b/llvm/test/CodeGen/AArch64/bitcast.ll
index e0851fd8739e..5de99586f7fc 100644
--- a/llvm/test/CodeGen/AArch64/bitcast.ll
+++ b/llvm/test/CodeGen/AArch64/bitcast.ll
@@ -517,10 +517,8 @@ define <4 x i64> @bitcast_v8i32_v4i64(<8 x i32> %a, <8 x i32> %b){
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-NEXT:    fmov x8, d2
-; CHECK-GI-NEXT:    fmov x9, d3
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
 ; CHECK-GI-NEXT:    ret
@@ -578,10 +576,8 @@ define <4 x i64> @bitcast_v16i16_v4i64(<16 x i16> %a, <16 x i16> %b){
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    add v0.8h, v0.8h, v2.8h
 ; CHECK-GI-NEXT:    add v1.8h, v1.8h, v3.8h
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-NEXT:    fmov x8, d2
-; CHECK-GI-NEXT:    fmov x9, d3
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
 ; CHECK-GI-NEXT:    ret
@@ -622,14 +618,10 @@ define <8 x i64> @bitcast_v16i32_v8i64(<16 x i32> %a, <16 x i32> %b){
 ; CHECK-GI-NEXT:    add v1.4s, v1.4s, v5.4s
 ; CHECK-GI-NEXT:    add v2.4s, v2.4s, v6.4s
 ; CHECK-GI-NEXT:    add v3.4s, v3.4s, v7.4s
-; CHECK-GI-NEXT:    mov d4, v0.d[1]
-; CHECK-GI-NEXT:    mov d5, v1.d[1]
-; CHECK-GI-NEXT:    mov d6, v2.d[1]
-; CHECK-GI-NEXT:    mov d7, v3.d[1]
-; CHECK-GI-NEXT:    fmov x8, d4
-; CHECK-GI-NEXT:    fmov x9, d5
-; CHECK-GI-NEXT:    fmov x10, d6
-; CHECK-GI-NEXT:    fmov x11, d7
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    mov x10, v2.d[1]
+; CHECK-GI-NEXT:    mov x11, v3.d[1]
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
 ; CHECK-GI-NEXT:    mov v2.d[1], x10
diff --git a/llvm/test/CodeGen/AArch64/insertextract.ll b/llvm/test/CodeGen/AArch64/insertextract.ll
index c6b2d07231bf..8b82004388b0 100644
--- a/llvm/test/CodeGen/AArch64/insertextract.ll
+++ b/llvm/test/CodeGen/AArch64/insertextract.ll
@@ -983,13 +983,12 @@ define <3 x i32> @insert_v3i32_0(<3 x i32> %a, i32 %b, i32 %c) {
 ;
 ; CHECK-GI-LABEL: insert_v3i32_0:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NEXT:    mov s2, v0.s[2]
-; CHECK-GI-NEXT:    fmov s0, w0
-; CHECK-GI-NEXT:    fmov w8, s1
-; CHECK-GI-NEXT:    mov v0.s[1], w8
-; CHECK-GI-NEXT:    fmov w8, s2
-; CHECK-GI-NEXT:    mov v0.s[2], w8
+; CHECK-GI-NEXT:    mov w8, v0.s[1]
+; CHECK-GI-NEXT:    fmov s1, w0
+; CHECK-GI-NEXT:    mov w9, v0.s[2]
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    mov v1.s[2], w9
+; CHECK-GI-NEXT:    mov v0.16b, v1.16b
 ; CHECK-GI-NEXT:    ret
 entry:
   %d = insertelement <3 x i32> %a, i32 %b, i32 0
diff --git a/llvm/test/CodeGen/AArch64/ptradd.ll b/llvm/test/CodeGen/AArch64/ptradd.ll
index 107db8723c64..af283f6a093e 100644
--- a/llvm/test/CodeGen/AArch64/ptradd.ll
+++ b/llvm/test/CodeGen/AArch64/ptradd.ll
@@ -81,13 +81,12 @@ define void @vector_gep_v3i32(<3 x ptr> %b, <3 x i32> %off, ptr %p) {
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
 ; CHECK-GI-NEXT:    smov x9, v3.s[1]
-; CHECK-GI-NEXT:    mov s3, v3.s[2]
 ; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-GI-NEXT:    fmov d1, x8
-; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    mov w8, v3.s[2]
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
-; CHECK-GI-NEXT:    fmov w9, s3
-; CHECK-GI-NEXT:    add x8, x8, w9, sxtw
+; CHECK-GI-NEXT:    fmov x9, d2
+; CHECK-GI-NEXT:    add x8, x9, w8, sxtw
 ; CHECK-GI-NEXT:    add v0.2d, v0.2d, v1.2d
 ; CHECK-GI-NEXT:    str x8, [x0, #16]
 ; CHECK-GI-NEXT:    str q0, [x0]
diff --git a/llvm/test/CodeGen/AArch64/reduce-and.ll b/llvm/test/CodeGen/AArch64/reduce-and.ll
index 62ad45b21296..8ca521327c2e 100644
--- a/llvm/test/CodeGen/AArch64/reduce-and.ll
+++ b/llvm/test/CodeGen/AArch64/reduce-and.ll
@@ -30,10 +30,9 @@ define i1 @test_redand_v2i1(<2 x i1> %a) {
 ; GISEL-LABEL: test_redand_v2i1:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    and w8, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    and w8, w9, w8
 ; GISEL-NEXT:    and w0, w8, #0x1
 ; GISEL-NEXT:    ret
   %or_result = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> %a)
@@ -457,10 +456,9 @@ define i32 @test_redand_v2i32(<2 x i32> %a) {
 ; GISEL-LABEL: test_redand_v2i32:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    and w0, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    and w0, w9, w8
 ; GISEL-NEXT:    ret
   %and_result = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> %a)
   ret i32 %and_result
@@ -480,10 +478,9 @@ define i32 @test_redand_v4i32(<4 x i32> %a) {
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    mov d1, v0.d[1]
 ; GISEL-NEXT:    and v0.8b, v0.8b, v1.8b
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    and w0, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    and w0, w9, w8
 ; GISEL-NEXT:    ret
   %and_result = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %a)
   ret i32 %and_result
@@ -505,10 +502,9 @@ define i32 @test_redand_v8i32(<8 x i32> %a) {
 ; GISEL-NEXT:    and v0.16b, v0.16b, v1.16b
 ; GISEL-NEXT:    mov d1, v0.d[1]
 ; GISEL-NEXT:    and v0.8b, v0.8b, v1.8b
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    and w0, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    and w0, w9, w8
 ; GISEL-NEXT:    ret
   %and_result = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> %a)
   ret i32 %and_result
@@ -524,10 +520,9 @@ define i64 @test_redand_v2i64(<2 x i64> %a) {
 ;
 ; GISEL-LABEL: test_redand_v2i64:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    mov d1, v0.d[1]
-; GISEL-NEXT:    fmov x8, d0
-; GISEL-NEXT:    fmov x9, d1
-; GISEL-NEXT:    and x0, x8, x9
+; GISEL-NEXT:    mov x8, v0.d[1]
+; GISEL-NEXT:    fmov x9, d0
+; GISEL-NEXT:    and x0, x9, x8
 ; GISEL-NEXT:    ret
   %and_result = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> %a)
   ret i64 %and_result
@@ -545,10 +540,9 @@ define i64 @test_redand_v4i64(<4 x i64> %a) {
 ; GISEL-LABEL: test_redand_v4i64:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    and v0.16b, v0.16b, v1.16b
-; GISEL-NEXT:    mov d1, v0.d[1]
-; GISEL-NEXT:    fmov x8, d0
-; GISEL-NEXT:    fmov x9, d1
-; GISEL-NEXT:    and x0, x8, x9
+; GISEL-NEXT:    mov x8, v0.d[1]
+; GISEL-NEXT:    fmov x9, d0
+; GISEL-NEXT:    and x0, x9, x8
 ; GISEL-NEXT:    ret
   %and_result = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> %a)
   ret i64 %and_result
diff --git a/llvm/test/CodeGen/AArch64/reduce-or.ll b/llvm/test/CodeGen/AArch64/reduce-or.ll
index 20c498d36fde..aac31ce8b71b 100644
--- a/llvm/test/CodeGen/AArch64/reduce-or.ll
+++ b/llvm/test/CodeGen/AArch64/reduce-or.ll
@@ -30,10 +30,9 @@ define i1 @test_redor_v2i1(<2 x i1> %a) {
 ; GISEL-LABEL: test_redor_v2i1:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    orr w8, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    orr w8, w9, w8
 ; GISEL-NEXT:    and w0, w8, #0x1
 ; GISEL-NEXT:    ret
   %or_result = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> %a)
@@ -459,10 +458,9 @@ define i32 @test_redor_v2i32(<2 x i32> %a) {
 ; GISEL-LABEL: test_redor_v2i32:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    orr w0, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    orr w0, w9, w8
 ; GISEL-NEXT:    ret
   %or_result = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> %a)
   ret i32 %or_result
@@ -482,10 +480,9 @@ define i32 @test_redor_v4i32(<4 x i32> %a) {
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    mov d1, v0.d[1]
 ; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    orr w0, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    orr w0, w9, w8
 ; GISEL-NEXT:    ret
   %or_result = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> %a)
   ret i32 %or_result
@@ -507,10 +504,9 @@ define i32 @test_redor_v8i32(<8 x i32> %a) {
 ; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; GISEL-NEXT:    mov d1, v0.d[1]
 ; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    orr w0, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    orr w0, w9, w8
 ; GISEL-NEXT:    ret
   %or_result = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %a)
   ret i32 %or_result
@@ -526,10 +522,9 @@ define i64 @test_redor_v2i64(<2 x i64> %a) {
 ;
 ; GISEL-LABEL: test_redor_v2i64:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    mov d1, v0.d[1]
-; GISEL-NEXT:    fmov x8, d0
-; GISEL-NEXT:    fmov x9, d1
-; GISEL-NEXT:    orr x0, x8, x9
+; GISEL-NEXT:    mov x8, v0.d[1]
+; GISEL-NEXT:    fmov x9, d0
+; GISEL-NEXT:    orr x0, x9, x8
 ; GISEL-NEXT:    ret
   %or_result = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> %a)
   ret i64 %or_result
@@ -547,10 +542,9 @@ define i64 @test_redor_v4i64(<4 x i64> %a) {
 ; GISEL-LABEL: test_redor_v4i64:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
-; GISEL-NEXT:    mov d1, v0.d[1]
-; GISEL-NEXT:    fmov x8, d0
-; GISEL-NEXT:    fmov x9, d1
-; GISEL-NEXT:    orr x0, x8, x9
+; GISEL-NEXT:    mov x8, v0.d[1]
+; GISEL-NEXT:    fmov x9, d0
+; GISEL-NEXT:    orr x0, x9, x8
 ; GISEL-NEXT:    ret
   %or_result = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> %a)
   ret i64 %or_result
diff --git a/llvm/test/CodeGen/AArch64/reduce-xor.ll b/llvm/test/CodeGen/AArch64/reduce-xor.ll
index b8ca99e003b6..9a00172f9476 100644
--- a/llvm/test/CodeGen/AArch64/reduce-xor.ll
+++ b/llvm/test/CodeGen/AArch64/reduce-xor.ll
@@ -27,10 +27,9 @@ define i1 @test_redxor_v2i1(<2 x i1> %a) {
 ; GISEL-LABEL: test_redxor_v2i1:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    eor w8, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    eor w8, w9, w8
 ; GISEL-NEXT:    and w0, w8, #0x1
 ; GISEL-NEXT:    ret
   %or_result = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %a)
@@ -448,10 +447,9 @@ define i32 @test_redxor_v2i32(<2 x i32> %a) {
 ; GISEL-LABEL: test_redxor_v2i32:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    eor w0, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    eor w0, w9, w8
 ; GISEL-NEXT:    ret
   %xor_result = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> %a)
   ret i32 %xor_result
@@ -471,10 +469,9 @@ define i32 @test_redxor_v4i32(<4 x i32> %a) {
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    mov d1, v0.d[1]
 ; GISEL-NEXT:    eor v0.8b, v0.8b, v1.8b
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    eor w0, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    eor w0, w9, w8
 ; GISEL-NEXT:    ret
   %xor_result = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> %a)
   ret i32 %xor_result
@@ -496,10 +493,9 @@ define i32 @test_redxor_v8i32(<8 x i32> %a) {
 ; GISEL-NEXT:    eor v0.16b, v0.16b, v1.16b
 ; GISEL-NEXT:    mov d1, v0.d[1]
 ; GISEL-NEXT:    eor v0.8b, v0.8b, v1.8b
-; GISEL-NEXT:    mov s1, v0.s[1]
-; GISEL-NEXT:    fmov w8, s0
-; GISEL-NEXT:    fmov w9, s1
-; GISEL-NEXT:    eor w0, w8, w9
+; GISEL-NEXT:    mov w8, v0.s[1]
+; GISEL-NEXT:    fmov w9, s0
+; GISEL-NEXT:    eor w0, w9, w8
 ; GISEL-NEXT:    ret
   %xor_result = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> %a)
   ret i32 %xor_result
@@ -515,10 +511,9 @@ define i64 @test_redxor_v2i64(<2 x i64> %a) {
 ;
 ; GISEL-LABEL: test_redxor_v2i64:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    mov d1, v0.d[1]
-; GISEL-NEXT:    fmov x8, d0
-; GISEL-NEXT:    fmov x9, d1
-; GISEL-NEXT:    eor x0, x8, x9
+; GISEL-NEXT:    mov x8, v0.d[1]
+; GISEL-NEXT:    fmov x9, d0
+; GISEL-NEXT:    eor x0, x9, x8
 ; GISEL-NEXT:    ret
   %xor_result = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> %a)
   ret i64 %xor_result
@@ -536,10 +531,9 @@ define i64 @test_redxor_v4i64(<4 x i64> %a) {
 ; GISEL-LABEL: test_redxor_v4i64:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    eor v0.16b, v0.16b, v1.16b
-; GISEL-NEXT:    mov d1, v0.d[1]
-; GISEL-NEXT:    fmov x8, d0
-; GISEL-NEXT:    fmov x9, d1
-; GISEL-NEXT:    eor x0, x8, x9
+; GISEL-NEXT:    mov x8, v0.d[1]
+; GISEL-NEXT:    fmov x9, d0
+; GISEL-NEXT:    eor x0, x9, x8
 ; GISEL-NEXT:    ret
   %xor_result = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> %a)
   ret i64 %xor_result
-- 
GitLab


From 31c903890a905d203de3303eaaa63063754ffbca Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 16 May 2024 11:53:11 +0900
Subject: [PATCH 0948/1206] [SeparateConstOffsetFromGEP] Add additional
 inbounds preservation tests (NFC)

Adding these for NVPTX because for AMDGPU the problematic -1 case
does not get reordered in the first place.
---
 .../NVPTX/lower-gep-reorder.ll                | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll
index 43dda1ae1517..ec1cbb9e61c0 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll
@@ -63,3 +63,44 @@ end:
   call void asm sideeffect "; use $0", "v"(ptr  %idx3)
   ret void
 }
+
+define void @inboundsPossiblyNegative1(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @inboundsPossiblyNegative1(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) {
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <2 x i8>, ptr [[IN_PTR]], i64 [[IN_IDX1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <2 x i8>, ptr [[TMP0]], i64 1
+; CHECK-NEXT:    ret void
+;
+  %const1 = getelementptr inbounds <2 x i8>, ptr %in.ptr, i64 1
+  %idx1 = getelementptr inbounds <2 x i8>, ptr %const1, i64 %in.idx1
+  ret void
+}
+
+define void @inboundsPossiblyNegative2(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @inboundsPossiblyNegative2(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) {
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <2 x i8>, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <2 x i8>, ptr [[TMP0]], i64 -1
+; CHECK-NEXT:    ret void
+;
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr inbounds <2 x i8>, ptr %in.ptr, i64 -1
+  %idx1 = getelementptr inbounds <2 x i8>, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
+define void @inboundsNonNegative(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @inboundsNonNegative(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) {
+; CHECK-NEXT:    [[IDXPROM:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <2 x i8>, ptr [[IN_PTR]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <2 x i8>, ptr [[TMP0]], i64 1
+; CHECK-NEXT:    ret void
+;
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr inbounds <2 x i8>, ptr %in.ptr, i64 1
+  %idx1 = getelementptr inbounds <2 x i8>, ptr %const1, i64 %in.idx1.nneg
+  ret void
+}
+
-- 
GitLab


From b4d1a606c7492d827aff6ff0c1c109adff1253b9 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 16 May 2024 11:54:38 +0900
Subject: [PATCH 0949/1206] [SeparateConstOffsetFromGEP] Check correct index
 for non-negativity

We were checking the index of GEP twice, instead of checking both
GEP and PtrGEP.
---
 llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp     | 2 +-
 .../SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 1a9eaf28f6e4..7ac1f43b7b6a 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -1001,7 +1001,7 @@ bool SeparateConstOffsetFromGEP::reorderGEP(GetElementPtrInst *GEP,
     auto KnownGEPIdx = computeKnownBits(GEPIdx->get(), *DL);
     IsChainInBounds &= KnownGEPIdx.isNonNegative();
     if (IsChainInBounds) {
-      auto PtrGEPIdx = GEP->indices().begin();
+      auto PtrGEPIdx = PtrGEP->indices().begin();
       auto KnownPtrGEPIdx = computeKnownBits(PtrGEPIdx->get(), *DL);
       IsChainInBounds &= KnownPtrGEPIdx.isNonNegative();
     }
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll
index ec1cbb9e61c0..23b4a4f788ae 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lower-gep-reorder.ll
@@ -80,8 +80,8 @@ define void @inboundsPossiblyNegative2(ptr %in.ptr, i64 %in.idx1) {
 ; CHECK-LABEL: define void @inboundsPossiblyNegative2(
 ; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) {
 ; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <2 x i8>, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <2 x i8>, ptr [[TMP0]], i64 -1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <2 x i8>, ptr [[IN_PTR]], i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <2 x i8>, ptr [[TMP1]], i64 -1
 ; CHECK-NEXT:    ret void
 ;
   %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
-- 
GitLab


From 83e61d03deaaa8f4dd8395cfa753af7b38f74b24 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 16 May 2024 12:20:18 +0900
Subject: [PATCH 0950/1206] [SeparateConstOffsetFromGEP] Add tests for multiple
 indices (NFC)

---
 .../AMDGPU/reorder-gep.ll                     | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll
index b4119f0b50b4..a7ca5b93c361 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll
@@ -284,3 +284,29 @@ entry:
   %idx3 = getelementptr i8, ptr addrspace(3) %const3, i64 %in.idx2
   ret void
 }
+
+define void @multiple_index_maybe_neg(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @multiple_index_maybe_neg(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[CONST1:%.*]] = getelementptr inbounds [2 x <2 x i8>], ptr [[IN_PTR]], i64 0, i64 1
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds [2 x <2 x i8>], ptr [[CONST1]], i64 0, i64 [[IN_IDX1]]
+; CHECK-NEXT:    ret void
+;
+  %const1 = getelementptr inbounds [2 x <2 x i8>], ptr %in.ptr, i64 0, i64 1
+  %idx1 = getelementptr inbounds [2 x <2 x i8>], ptr %const1, i64 0, i64 %in.idx1
+  ret void
+}
+
+define void @multiple_index_nonneg(ptr %in.ptr, i64 %in.idx1) {
+; CHECK-LABEL: define void @multiple_index_nonneg(
+; CHECK-SAME: ptr [[IN_PTR:%.*]], i64 [[IN_IDX1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
+; CHECK-NEXT:    [[CONST1:%.*]] = getelementptr inbounds [2 x <2 x i8>], ptr [[IN_PTR]], i64 0, i64 1
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds [2 x <2 x i8>], ptr [[CONST1]], i64 0, i64 [[IN_IDX1_NNEG]]
+; CHECK-NEXT:    ret void
+;
+  %in.idx1.nneg = and i64 %in.idx1, 9223372036854775807
+  %const1 = getelementptr inbounds [2 x <2 x i8>], ptr %in.ptr, i64 0, i64 1
+  %idx1 = getelementptr inbounds [2 x <2 x i8>], ptr %const1, i64 0, i64 %in.idx1.nneg
+  ret void
+}
-- 
GitLab


From e91ea1b5d88805ebf7657da57ca6a7577374e4ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jim=20M=2E=20R=2E=20Teichgr=C3=A4ber?=
 <a@teichgraeber.digital>
Date: Thu, 16 May 2024 05:38:15 +0200
Subject: [PATCH 0951/1206] [Clang] Disallow VLA type compound literals
 (#91891)

C99-C23 6.5.2.5 says: The type name shall specify an object type or an
array of unknown size, but not a variable length array type.

Fixes #89835.
---
 clang/docs/ReleaseNotes.rst                   |  3 +++
 .../clang/Basic/DiagnosticSemaKinds.td        |  2 ++
 clang/lib/Sema/SemaExpr.cpp                   | 19 +++++++++++++------
 clang/test/C/C2x/n2900_n3011.c                |  8 +++++++-
 clang/test/C/C2x/n2900_n3011_2.c              | 16 ----------------
 clang/test/Sema/compound-literal.c            | 13 ++++++++++++-
 6 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 11812c355f8d..be4cded27632 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -570,6 +570,9 @@ Bug Fixes in This Version
 - Clang will no longer emit a duplicate -Wunused-value warning for an expression
   `(A, B)` which evaluates to glvalue `B` that can be converted to non ODR-use. (#GH45783)
 
+- Clang now correctly disallows VLA type compound literals, e.g. ``(int[size]){}``,
+  as the C standard mandates. (#GH89835)
+
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 6100fba51005..e648b503ac03 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -3371,6 +3371,8 @@ def err_field_with_address_space : Error<
   "field may not be qualified with an address space">;
 def err_compound_literal_with_address_space : Error<
   "compound literal in function scope may not be qualified with an address space">;
+def err_compound_literal_with_vla_type : Error<
+  "compound literal cannot be of variable-length array type">;
 def err_address_space_mismatch_templ_inst : Error<
   "conflicting address space qualifiers are provided between types %0 and %1">;
 def err_attr_objc_ownership_redundant : Error<
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 50569c1cd536..cc507524e2fc 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -7130,12 +7130,19 @@ Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
       // init a VLA in C++ in all cases (such as with non-trivial constructors).
       // FIXME: should we allow this construct in C++ when it makes sense to do
       // so?
-      std::optional<unsigned> NumInits;
-      if (const auto *ILE = dyn_cast<InitListExpr>(LiteralExpr))
-        NumInits = ILE->getNumInits();
-      if ((LangOpts.CPlusPlus || NumInits.value_or(0)) &&
-          !tryToFixVariablyModifiedVarType(TInfo, literalType, LParenLoc,
-                                           diag::err_variable_object_no_init))
+      //
+      // But: C99-C23 6.5.2.5 Compound literals constraint 1: The type name
+      // shall specify an object type or an array of unknown size, but not a
+      // variable length array type. This seems odd, as it allows int a[size] =
+      // {}; but forbids int a[size] = (int[size]){}; As this is what the
+      // standard says, this is what's implemented here for C (except for the
+      // extension that permits constant foldable size arrays)
+
+      auto diagID = LangOpts.CPlusPlus
+                        ? diag::err_variable_object_no_init
+                        : diag::err_compound_literal_with_vla_type;
+      if (!tryToFixVariablyModifiedVarType(TInfo, literalType, LParenLoc,
+                                           diagID))
         return ExprError();
     }
   } else if (!literalType->isDependentType() &&
diff --git a/clang/test/C/C2x/n2900_n3011.c b/clang/test/C/C2x/n2900_n3011.c
index 4350aa140691..82a3b16c8acd 100644
--- a/clang/test/C/C2x/n2900_n3011.c
+++ b/clang/test/C/C2x/n2900_n3011.c
@@ -27,8 +27,14 @@ void test(void) {
                               compat-warning {{use of an empty initializer is incompatible with C standards before C23}}
   int vla[i] = {}; // compat-warning {{use of an empty initializer is incompatible with C standards before C23}} \
                       pedantic-warning {{use of an empty initializer is a C23 extension}}
+  // C99 6.5.2.5 Compound literals constraint 1: The type name shall specify an
+  // object type or an array of unknown size, but not a variable length array
+  // type.
   int *compound_literal_vla = (int[i]){}; // compat-warning {{use of an empty initializer is incompatible with C standards before C23}} \
-                                             pedantic-warning {{use of an empty initializer is a C23 extension}}
+                                             pedantic-warning {{use of an empty initializer is a C23 extension}}\
+                                             compat-error {{compound literal cannot be of variable-length array type}} \
+                                             pedantic-error {{compound literal cannot be of variable-length array type}}\
+
 
   struct T {
 	int i;
diff --git a/clang/test/C/C2x/n2900_n3011_2.c b/clang/test/C/C2x/n2900_n3011_2.c
index eb15fbf905c8..ab659d636d15 100644
--- a/clang/test/C/C2x/n2900_n3011_2.c
+++ b/clang/test/C/C2x/n2900_n3011_2.c
@@ -76,22 +76,6 @@ void test_zero_size_vla() {
   // CHECK-NEXT: call void @llvm.memset.p0.i64(ptr {{.*}} %[[VLA]], i8 0, i64 %[[BYTES_TO_COPY]], i1 false)
 }
 
-void test_compound_literal_vla() {
-  int num_elts = 12;
-  int *compound_literal_vla = (int[num_elts]){};
-  // CHECK: define {{.*}} void @test_compound_literal_vla
-  // CHECK-NEXT: entry:
-  // CHECK-NEXT: %[[NUM_ELTS_PTR:.+]] = alloca i32
-  // CHECK-NEXT: %[[COMP_LIT_VLA:.+]] = alloca ptr
-  // CHECK-NEXT: %[[COMP_LIT:.+]] = alloca i32
-  // CHECK-NEXT: store i32 12, ptr %[[NUM_ELTS_PTR]]
-  // CHECK-NEXT: %[[NUM_ELTS:.+]] = load i32, ptr %[[NUM_ELTS_PTR]]
-  // CHECK-NEXT: %[[NUM_ELTS_EXT:.+]] = zext i32 %[[NUM_ELTS]] to i64
-  // CHECK-NEXT: %[[BYTES_TO_COPY:.+]] = mul nuw i64 %[[NUM_ELTS_EXT]], 4
-  // CHECK-NEXT: call void @llvm.memset.p0.i64(ptr {{.*}} %[[COMP_LIT]], i8 0, i64 %[[BYTES_TO_COPY]], i1 false)
-  // CHECK-NEXT: store ptr %[[COMP_LIT]], ptr %[[COMP_LIT_VLA]]
-}
-
 void test_nested_structs() {
   struct T t1 = { 1, {} };
   struct T t2 = { 1, { 2, {} } };
diff --git a/clang/test/Sema/compound-literal.c b/clang/test/Sema/compound-literal.c
index a64b6f9e5dfa..3ed53d670d38 100644
--- a/clang/test/Sema/compound-literal.c
+++ b/clang/test/Sema/compound-literal.c
@@ -29,7 +29,7 @@ int main(int argc, char **argv) {
 struct Incomplete; // expected-note{{forward declaration of 'struct Incomplete'}}
 struct Incomplete* I1 = &(struct Incomplete){1, 2, 3}; // expected-error {{variable has incomplete type}}
 void IncompleteFunc(unsigned x) {
-  struct Incomplete* I2 = (struct foo[x]){1, 2, 3}; // expected-error {{variable-sized object may not be initialized}}
+  struct Incomplete* I2 = (struct foo[x]){1, 2, 3}; // expected-error {{compound literal cannot be of variable-length array type}}
   (void){1,2,3}; // expected-error {{variable has incomplete type}}
   (void(void)) { 0 }; // expected-error{{illegal initializer type 'void (void)'}}
 }
@@ -42,3 +42,14 @@ int (^block)(int) = ^(int i) {
   int *array = (int[]) {i, i + 2, i + 4};
   return array[i];
 };
+
+// C99 6.5.2.5 Compound literals constraint 1: The type name shall specify an object type or an array of unknown size, but not a variable length array type.
+// So check that VLA type compound literals are rejected (see https://github.com/llvm/llvm-project/issues/89835).
+void vla(int n) {
+  int size = 5;
+  (void)(int[size]){}; // expected-warning {{use of an empty initializer is a C23 extension}}
+                       // expected-error@-1 {{compound literal cannot be of variable-length array type}}
+  (void)(int[size]){1}; // expected-error {{compound literal cannot be of variable-length array type}}
+  (void)(int[size]){1,2,3}; // expected-error {{compound literal cannot be of variable-length array type}}
+  (void)(int[size]){1,2,3,4,5}; // expected-error {{compound literal cannot be of variable-length array type}}
+}
-- 
GitLab


From 90fbc5bbcdc7d35d57157e4cc0459470d473f2ae Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Wed, 15 May 2024 20:38:55 -0700
Subject: [PATCH 0952/1206] [MCAsmParser] Simplify. NFC

---
 llvm/lib/MC/MCParser/AsmParser.cpp   | 22 ++++++----------------
 llvm/lib/MC/MCParser/MCAsmParser.cpp |  1 -
 2 files changed, 6 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 46c1caa940c5..009465d11d78 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -833,11 +833,8 @@ AsmParser::~AsmParser() {
 
 void AsmParser::printMacroInstantiations() {
   // Print the active macro instantiation stack.
-  for (std::vector<MacroInstantiation *>::const_reverse_iterator
-           it = ActiveMacros.rbegin(),
-           ie = ActiveMacros.rend();
-       it != ie; ++it)
-    printMessage((*it)->InstantiationLoc, SourceMgr::DK_Note,
+  for (MacroInstantiation *M : reverse(ActiveMacros))
+    printMessage(M->InstantiationLoc, SourceMgr::DK_Note,
                  "while in macro instantiation");
 }
 
@@ -1510,9 +1507,7 @@ bool AsmParser::parseExpression(const MCExpr *&Res, SMLoc &EndLoc) {
   // As a special case, we support 'a op b @ modifier' by rewriting the
   // expression to include the modifier. This is inefficient, but in general we
   // expect users to use 'a@modifier op b'.
-  if (Lexer.getKind() == AsmToken::At) {
-    Lex();
-
+  if (parseOptionalToken(AsmToken::At)) {
     if (Lexer.isNot(AsmToken::Identifier))
       return TokError("unexpected symbol modifier following '@'");
 
@@ -2708,10 +2703,8 @@ bool AsmParser::parseMacroArgument(MCAsmMacroArgument &MA, bool Vararg) {
       if (Lexer.is(AsmToken::Comma))
         break;
 
-      if (Lexer.is(AsmToken::Space)) {
+      if (parseOptionalToken(AsmToken::Space))
         SpaceEaten = true;
-        Lexer.Lex(); // Eat spaces
-      }
 
       // Spaces can delimit parameters, but could also be part an expression.
       // If the token after a space is an operator, add the token and the next
@@ -2722,9 +2715,7 @@ bool AsmParser::parseMacroArgument(MCAsmMacroArgument &MA, bool Vararg) {
           Lexer.Lex();
 
           // Whitespace after an operator can be ignored.
-          if (Lexer.is(AsmToken::Space))
-            Lexer.Lex();
-
+          parseOptionalToken(AsmToken::Space);
           continue;
         }
       }
@@ -2865,8 +2856,7 @@ bool AsmParser::parseMacroArguments(const MCAsmMacro *M,
       return Failure;
     }
 
-    if (Lexer.is(AsmToken::Comma))
-      Lex();
+    parseOptionalToken(AsmToken::Comma);
   }
 
   return TokError("too many positional arguments");
diff --git a/llvm/lib/MC/MCParser/MCAsmParser.cpp b/llvm/lib/MC/MCParser/MCAsmParser.cpp
index bfeba3108cb4..236585fc9082 100644
--- a/llvm/lib/MC/MCParser/MCAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/MCAsmParser.cpp
@@ -99,7 +99,6 @@ bool MCAsmParser::TokError(const Twine &Msg, SMRange Range) {
 }
 
 bool MCAsmParser::Error(SMLoc L, const Twine &Msg, SMRange Range) {
-
   MCPendingError PErr;
   PErr.Loc = L;
   Msg.toVector(PErr.Msg);
-- 
GitLab


From ce961c5607dd5c2d181117938720e410b406a49f Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Thu, 16 May 2024 07:44:08 +0400
Subject: [PATCH 0953/1206] [lldb] Fixed the TestFdLeak test (#92273)

Use `os.devnull` instead of `/dev/null`.
---
 lldb/test/API/functionalities/avoids-fd-leak/TestFdLeak.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/test/API/functionalities/avoids-fd-leak/TestFdLeak.py b/lldb/test/API/functionalities/avoids-fd-leak/TestFdLeak.py
index e4f5cd3a03f8..c840d38df5c7 100644
--- a/lldb/test/API/functionalities/avoids-fd-leak/TestFdLeak.py
+++ b/lldb/test/API/functionalities/avoids-fd-leak/TestFdLeak.py
@@ -26,7 +26,7 @@ class AvoidsFdLeakTestCase(TestBase):
     @skipIfTargetAndroid()  # Android have some other file descriptors open by the shell
     @skipIfDarwinEmbedded  # <rdar://problem/33888742>  # debugserver on ios has an extra fd open on launch
     def test_fd_leak_log(self):
-        self.do_test(["log enable -f '/dev/null' lldb commands"])
+        self.do_test(["log enable -f '{}' lldb commands".format(os.devnull)])
 
     def do_test(self, commands):
         self.build()
-- 
GitLab


From b11a6607cb6522c58dfbd5f54239e7daa281368e Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Wed, 15 May 2024 21:01:57 -0700
Subject: [PATCH 0954/1206] [clang-format][NFC] Reformat with 18.1.5

---
 clang/lib/Format/UnwrappedLineParser.cpp | 3 ++-
 clang/tools/clang-format/ClangFormat.cpp | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 4f1c2c5114e9..2236a49e4b76 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -47,7 +47,8 @@ void printLine(llvm::raw_ostream &OS, const UnwrappedLine &Line,
       OS << Prefix;
       NewLine = false;
     }
-    OS << I->Tok->Tok.getName() << "[" << "T=" << (unsigned)I->Tok->getType()
+    OS << I->Tok->Tok.getName() << "["
+       << "T=" << (unsigned)I->Tok->getType()
        << ", OC=" << I->Tok->OriginalColumn << ", \"" << I->Tok->TokenText
        << "\"] ";
     for (SmallVectorImpl<UnwrappedLine>::const_iterator
diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp
index 01f7c6047726..3fa5f81a3576 100644
--- a/clang/tools/clang-format/ClangFormat.cpp
+++ b/clang/tools/clang-format/ClangFormat.cpp
@@ -336,7 +336,8 @@ static void outputReplacementXML(StringRef Text) {
 
 static void outputReplacementsXML(const Replacements &Replaces) {
   for (const auto &R : Replaces) {
-    outs() << "<replacement " << "offset='" << R.getOffset() << "' "
+    outs() << "<replacement "
+           << "offset='" << R.getOffset() << "' "
            << "length='" << R.getLength() << "'>";
     outputReplacementXML(R.getReplacementText());
     outs() << "</replacement>\n";
-- 
GitLab


From 526553b25131a69d9d6426e17c7b69c2ba27144f Mon Sep 17 00:00:00 2001
From: Yusuke MINATO <minato.yusuke@fujitsu.com>
Date: Thu, 16 May 2024 13:16:07 +0900
Subject: [PATCH 0955/1206] [flang] Add nsw flag to do-variable increment with
 a new option (#91579)

This patch adds nsw flag to the increment of do-variables when a new
option is enabled.
NOTE 11.10 in the Fortran 2018 standard says they never overflow.

See also the discussion in #74709 and the following discourse post.
https://discourse.llvm.org/t/rfc-add-nsw-flags-to-arithmetic-integer-operations-using-the-option-fno-wrapv/77584/5
---
 clang/include/clang/Driver/Options.td         |   4 +
 clang/lib/Driver/ToolChains/Flang.cpp         |   1 +
 flang/include/flang/Lower/LoweringOptions.def |   4 +
 .../flang/Optimizer/Transforms/Passes.h       |   4 +-
 .../flang/Optimizer/Transforms/Passes.td      |   5 +-
 flang/include/flang/Tools/CLOptions.inc       |  13 +-
 flang/include/flang/Tools/CrossToolHelpers.h  |   1 +
 flang/lib/Frontend/CompilerInvocation.cpp     |   6 +
 flang/lib/Frontend/FrontendActions.cpp        |   3 +
 flang/lib/Lower/Bridge.cpp                    |  12 +-
 flang/lib/Lower/IO.cpp                        |   9 +-
 .../Transforms/ControlFlowConverter.cpp       |  44 +++-
 flang/test/Driver/frontend-forwarding.f90     |   2 +
 flang/test/Fir/loop01.fir                     | 211 ++++++++++++++++++
 flang/test/Lower/array-substring.f90          |  40 ++++
 flang/test/Lower/do_loop.f90                  |  42 ++++
 flang/test/Lower/do_loop_unstructured.f90     | 189 +++++++++++++++-
 flang/test/Lower/infinite_loop.f90            |  34 +++
 flang/test/Lower/io-implied-do-fixes.f90      |  51 ++++-
 flang/tools/bbc/bbc.cpp                       |   7 +
 20 files changed, 659 insertions(+), 23 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index e579f1a0a366..7bb781667e92 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -6550,6 +6550,10 @@ def flang_deprecated_no_hlfir : Flag<["-"], "flang-deprecated-no-hlfir">,
   Flags<[HelpHidden]>, Visibility<[FlangOption, FC1Option]>,
   HelpText<"Do not use HLFIR lowering (deprecated)">;
 
+def flang_experimental_integer_overflow : Flag<["-"], "flang-experimental-integer-overflow">,
+  Flags<[HelpHidden]>, Visibility<[FlangOption, FC1Option]>,
+  HelpText<"Add nsw flag to internal operations such as do-variable increment (experimental)">;
+
 //===----------------------------------------------------------------------===//
 // FLangOption + CoreOption + NoXarchOption
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index d275528b6905..42ca060186fd 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -139,6 +139,7 @@ void Flang::addCodegenOptions(const ArgList &Args,
 
   Args.addAllArgs(CmdArgs, {options::OPT_flang_experimental_hlfir,
                             options::OPT_flang_deprecated_no_hlfir,
+                            options::OPT_flang_experimental_integer_overflow,
                             options::OPT_fno_ppc_native_vec_elem_order,
                             options::OPT_fppc_native_vec_elem_order});
 }
diff --git a/flang/include/flang/Lower/LoweringOptions.def b/flang/include/flang/Lower/LoweringOptions.def
index be080a4d29d7..7594a57a2629 100644
--- a/flang/include/flang/Lower/LoweringOptions.def
+++ b/flang/include/flang/Lower/LoweringOptions.def
@@ -34,5 +34,9 @@ ENUM_LOWERINGOPT(NoPPCNativeVecElemOrder, unsigned, 1, 0)
 /// On by default.
 ENUM_LOWERINGOPT(Underscoring, unsigned, 1, 1)
 
+/// If true, add nsw flags to loop variable increments.
+/// Off by default.
+ENUM_LOWERINGOPT(NSWOnLoopVarInc, unsigned, 1, 0)
+
 #undef LOWERINGOPT
 #undef ENUM_LOWERINGOPT
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index ae1d72a3526b..25fe61488f4f 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -54,6 +54,7 @@ namespace fir {
 std::unique_ptr<mlir::Pass> createAffineDemotionPass();
 std::unique_ptr<mlir::Pass>
 createArrayValueCopyPass(fir::ArrayValueCopyOptions options = {});
+std::unique_ptr<mlir::Pass> createCFGConversionPassWithNSW();
 std::unique_ptr<mlir::Pass> createExternalNameConversionPass();
 std::unique_ptr<mlir::Pass>
 createExternalNameConversionPass(bool appendUnderscore);
@@ -89,7 +90,8 @@ createFunctionAttrPass(FunctionAttrTypes &functionAttr, bool noInfsFPMath,
                        bool noSignedZerosFPMath, bool unsafeFPMath);
 
 void populateCfgConversionRewrites(mlir::RewritePatternSet &patterns,
-                                   bool forceLoopToExecuteOnce = false);
+                                   bool forceLoopToExecuteOnce = false,
+                                   bool setNSW = false);
 
 // declarative passes
 #define GEN_PASS_REGISTRATION
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index e22c1b5f338b..622c9465754c 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -151,7 +151,10 @@ def CFGConversion : Pass<"cfg-conversion"> {
   let options = [
     Option<"forceLoopToExecuteOnce", "always-execute-loop-body", "bool",
            /*default=*/"false",
-           "force the body of a loop to execute at least once">
+           "force the body of a loop to execute at least once">,
+    Option<"setNSW", "set-nsw", "bool",
+           /*default=*/"false",
+           "set nsw on loop variable increment">
   ];
 }
 
diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc
index cc3431d5b71d..1817dd6ca4a7 100644
--- a/flang/include/flang/Tools/CLOptions.inc
+++ b/flang/include/flang/Tools/CLOptions.inc
@@ -148,9 +148,14 @@ static void addCanonicalizerPassWithoutRegionSimplification(
   pm.addPass(mlir::createCanonicalizerPass(config));
 }
 
-inline void addCfgConversionPass(mlir::PassManager &pm) {
-  addNestedPassToAllTopLevelOperationsConditionally(
-      pm, disableCfgConversion, fir::createCFGConversion);
+inline void addCfgConversionPass(
+    mlir::PassManager &pm, const MLIRToLLVMPassPipelineConfig &config) {
+  if (config.NSWOnLoopVarInc)
+    addNestedPassToAllTopLevelOperationsConditionally(
+        pm, disableCfgConversion, fir::createCFGConversionPassWithNSW);
+  else
+    addNestedPassToAllTopLevelOperationsConditionally(
+        pm, disableCfgConversion, fir::createCFGConversion);
 }
 
 inline void addAVC(
@@ -290,7 +295,7 @@ inline void createDefaultFIROptimizerPassPipeline(
     pm.addPass(fir::createAliasTagsPass());
 
   // convert control flow to CFG form
-  fir::addCfgConversionPass(pm);
+  fir::addCfgConversionPass(pm, pc);
   pm.addPass(mlir::createConvertSCFToCFPass());
 
   pm.addPass(mlir::createCanonicalizerPass(config));
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index f79520707714..77b68fc6187f 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -122,6 +122,7 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
   bool NoSignedZerosFPMath =
       false; ///< Set no-signed-zeros-fp-math attribute for functions.
   bool UnsafeFPMath = false; ///< Set unsafe-fp-math attribute for functions.
+  bool NSWOnLoopVarInc = false; ///< Add nsw flag to loop variable increments.
 };
 
 struct OffloadModuleOpts {
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index e8a8c90045d9..50c3e8b0113b 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1206,6 +1206,12 @@ bool CompilerInvocation::createFromArgs(
     invoc.loweringOpts.setNoPPCNativeVecElemOrder(true);
   }
 
+  // -flang-experimental-integer-overflow
+  if (args.hasArg(
+          clang::driver::options::OPT_flang_experimental_integer_overflow)) {
+    invoc.loweringOpts.setNSWOnLoopVarInc(true);
+  }
+
   // Preserve all the remark options requested, i.e. -Rpass, -Rpass-missed or
   // -Rpass-analysis. This will be used later when processing and outputting the
   // remarks generated by LLVM in ExecuteCompilerInvocation.cpp.
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index 4341c104a69d..b1b6391f1439 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -818,6 +818,9 @@ void CodeGenAction::generateLLVMIR() {
     config.VScaleMax = vsr->second;
   }
 
+  if (ci.getInvocation().getLoweringOpts().getNSWOnLoopVarInc())
+    config.NSWOnLoopVarInc = true;
+
   // Create the pass pipeline
   fir::createMLIRToLLVMPassPipeline(pm, config, getCurrentFile());
   (void)mlir::applyPassManagerCLOptions(pm);
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 596049fcfc92..afbc1122de86 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2007,6 +2007,11 @@ private:
   void genFIRIncrementLoopEnd(IncrementLoopNestInfo &incrementLoopNestInfo) {
     assert(!incrementLoopNestInfo.empty() && "empty loop nest");
     mlir::Location loc = toLocation();
+    mlir::arith::IntegerOverflowFlags flags{};
+    if (getLoweringOptions().getNSWOnLoopVarInc())
+      flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
+    auto iofAttr = mlir::arith::IntegerOverflowFlagsAttr::get(
+        builder->getContext(), flags);
     for (auto it = incrementLoopNestInfo.rbegin(),
               rend = incrementLoopNestInfo.rend();
          it != rend; ++it) {
@@ -2021,7 +2026,8 @@ private:
         builder->setInsertionPointToEnd(info.doLoop.getBody());
         llvm::SmallVector<mlir::Value, 2> results;
         results.push_back(builder->create<mlir::arith::AddIOp>(
-            loc, info.doLoop.getInductionVar(), info.doLoop.getStep()));
+            loc, info.doLoop.getInductionVar(), info.doLoop.getStep(),
+            iofAttr));
         // Step loopVariable to help optimizations such as vectorization.
         // Induction variable elimination will clean up as necessary.
         mlir::Value step = builder->createConvert(
@@ -2029,7 +2035,7 @@ private:
         mlir::Value loopVar =
             builder->create<fir::LoadOp>(loc, info.loopVariable);
         results.push_back(
-            builder->create<mlir::arith::AddIOp>(loc, loopVar, step));
+            builder->create<mlir::arith::AddIOp>(loc, loopVar, step, iofAttr));
         builder->create<fir::ResultOp>(loc, results);
         builder->setInsertionPointAfter(info.doLoop);
         // The loop control variable may be used after the loop.
@@ -2054,7 +2060,7 @@ private:
       if (info.hasRealControl)
         value = builder->create<mlir::arith::AddFOp>(loc, value, step);
       else
-        value = builder->create<mlir::arith::AddIOp>(loc, value, step);
+        value = builder->create<mlir::arith::AddIOp>(loc, value, step, iofAttr);
       builder->create<fir::StoreOp>(loc, value, info.loopVariable);
 
       genBranch(info.headerBlock);
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index ed0afad9197d..97ef991cb399 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -928,6 +928,11 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   Fortran::lower::StatementContext stmtCtx;
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   mlir::Location loc = converter.getCurrentLocation();
+  mlir::arith::IntegerOverflowFlags flags{};
+  if (converter.getLoweringOptions().getNSWOnLoopVarInc())
+    flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
+  auto iofAttr =
+      mlir::arith::IntegerOverflowFlagsAttr::get(builder.getContext(), flags);
   makeNextConditionalOn(builder, loc, checkResult, ok, inLoop);
   const auto &itemList = std::get<0>(ioImpliedDo.t);
   const auto &control = std::get<1>(ioImpliedDo.t);
@@ -965,7 +970,7 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
     genItemList(ioImpliedDo);
     builder.setInsertionPointToEnd(doLoopOp.getBody());
     mlir::Value result = builder.create<mlir::arith::AddIOp>(
-        loc, doLoopOp.getInductionVar(), doLoopOp.getStep());
+        loc, doLoopOp.getInductionVar(), doLoopOp.getStep(), iofAttr);
     builder.create<fir::ResultOp>(loc, result);
     builder.setInsertionPointAfter(doLoopOp);
     // The loop control variable may be used after the loop.
@@ -1007,7 +1012,7 @@ static void genIoLoop(Fortran::lower::AbstractConverter &converter,
   mlir::OpResult iterateResult = builder.getBlock()->back().getResult(0);
   mlir::Value inductionResult0 = iterWhileOp.getInductionVar();
   auto inductionResult1 = builder.create<mlir::arith::AddIOp>(
-      loc, inductionResult0, iterWhileOp.getStep());
+      loc, inductionResult0, iterWhileOp.getStep(), iofAttr);
   auto inductionResult = builder.create<mlir::arith::SelectOp>(
       loc, iterateResult, inductionResult1, inductionResult0);
   llvm::SmallVector<mlir::Value> results = {inductionResult, iterateResult};
diff --git a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
index a62f6cde0e09..a233e7fbdcd1 100644
--- a/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
+++ b/flang/lib/Optimizer/Transforms/ControlFlowConverter.cpp
@@ -43,14 +43,19 @@ class CfgLoopConv : public mlir::OpRewritePattern<fir::DoLoopOp> {
 public:
   using OpRewritePattern::OpRewritePattern;
 
-  CfgLoopConv(mlir::MLIRContext *ctx, bool forceLoopToExecuteOnce)
+  CfgLoopConv(mlir::MLIRContext *ctx, bool forceLoopToExecuteOnce, bool setNSW)
       : mlir::OpRewritePattern<fir::DoLoopOp>(ctx),
-        forceLoopToExecuteOnce(forceLoopToExecuteOnce) {}
+        forceLoopToExecuteOnce(forceLoopToExecuteOnce), setNSW(setNSW) {}
 
   mlir::LogicalResult
   matchAndRewrite(DoLoopOp loop,
                   mlir::PatternRewriter &rewriter) const override {
     auto loc = loop.getLoc();
+    mlir::arith::IntegerOverflowFlags flags{};
+    if (setNSW)
+      flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
+    auto iofAttr = mlir::arith::IntegerOverflowFlagsAttr::get(
+        rewriter.getContext(), flags);
 
     // Create the start and end blocks that will wrap the DoLoopOp with an
     // initalizer and an end point
@@ -104,7 +109,7 @@ public:
     rewriter.setInsertionPointToEnd(lastBlock);
     auto iv = conditionalBlock->getArgument(0);
     mlir::Value steppedIndex =
-        rewriter.create<mlir::arith::AddIOp>(loc, iv, step);
+        rewriter.create<mlir::arith::AddIOp>(loc, iv, step, iofAttr);
     assert(steppedIndex && "must be a Value");
     auto lastArg = conditionalBlock->getNumArguments() - 1;
     auto itersLeft = conditionalBlock->getArgument(lastArg);
@@ -142,6 +147,7 @@ public:
 
 private:
   bool forceLoopToExecuteOnce;
+  bool setNSW;
 };
 
 /// Convert `fir.if` to control-flow
@@ -149,7 +155,7 @@ class CfgIfConv : public mlir::OpRewritePattern<fir::IfOp> {
 public:
   using OpRewritePattern::OpRewritePattern;
 
-  CfgIfConv(mlir::MLIRContext *ctx, bool forceLoopToExecuteOnce)
+  CfgIfConv(mlir::MLIRContext *ctx, bool forceLoopToExecuteOnce, bool setNSW)
       : mlir::OpRewritePattern<fir::IfOp>(ctx) {}
 
   mlir::LogicalResult
@@ -214,13 +220,19 @@ class CfgIterWhileConv : public mlir::OpRewritePattern<fir::IterWhileOp> {
 public:
   using OpRewritePattern::OpRewritePattern;
 
-  CfgIterWhileConv(mlir::MLIRContext *ctx, bool forceLoopToExecuteOnce)
-      : mlir::OpRewritePattern<fir::IterWhileOp>(ctx) {}
+  CfgIterWhileConv(mlir::MLIRContext *ctx, bool forceLoopToExecuteOnce,
+                   bool setNSW)
+      : mlir::OpRewritePattern<fir::IterWhileOp>(ctx), setNSW(setNSW) {}
 
   mlir::LogicalResult
   matchAndRewrite(fir::IterWhileOp whileOp,
                   mlir::PatternRewriter &rewriter) const override {
     auto loc = whileOp.getLoc();
+    mlir::arith::IntegerOverflowFlags flags{};
+    if (setNSW)
+      flags = bitEnumSet(flags, mlir::arith::IntegerOverflowFlags::nsw);
+    auto iofAttr = mlir::arith::IntegerOverflowFlagsAttr::get(
+        rewriter.getContext(), flags);
 
     // Start by splitting the block containing the 'fir.do_loop' into two parts.
     // The part before will get the init code, the part after will be the end
@@ -248,7 +260,8 @@ public:
     auto *terminator = lastBodyBlock->getTerminator();
     rewriter.setInsertionPointToEnd(lastBodyBlock);
     auto step = whileOp.getStep();
-    mlir::Value stepped = rewriter.create<mlir::arith::AddIOp>(loc, iv, step);
+    mlir::Value stepped =
+        rewriter.create<mlir::arith::AddIOp>(loc, iv, step, iofAttr);
     assert(stepped && "must be a Value");
 
     llvm::SmallVector<mlir::Value> loopCarried;
@@ -305,6 +318,9 @@ public:
     rewriter.replaceOp(whileOp, args);
     return success();
   }
+
+private:
+  bool setNSW;
 };
 
 /// Convert FIR structured control flow ops to CFG ops.
@@ -312,10 +328,13 @@ class CfgConversion : public fir::impl::CFGConversionBase<CfgConversion> {
 public:
   using CFGConversionBase<CfgConversion>::CFGConversionBase;
 
+  CfgConversion(bool setNSW) { this->setNSW = setNSW; }
+
   void runOnOperation() override {
     auto *context = &this->getContext();
     mlir::RewritePatternSet patterns(context);
-    fir::populateCfgConversionRewrites(patterns, this->forceLoopToExecuteOnce);
+    fir::populateCfgConversionRewrites(patterns, this->forceLoopToExecuteOnce,
+                                       this->setNSW);
     mlir::ConversionTarget target(*context);
     target.addLegalDialect<mlir::affine::AffineDialect,
                            mlir::cf::ControlFlowDialect, FIROpsDialect,
@@ -337,7 +356,12 @@ public:
 
 /// Expose conversion rewriters to other passes
 void fir::populateCfgConversionRewrites(mlir::RewritePatternSet &patterns,
-                                        bool forceLoopToExecuteOnce) {
+                                        bool forceLoopToExecuteOnce,
+                                        bool setNSW) {
   patterns.insert<CfgLoopConv, CfgIfConv, CfgIterWhileConv>(
-      patterns.getContext(), forceLoopToExecuteOnce);
+      patterns.getContext(), forceLoopToExecuteOnce, setNSW);
+}
+
+std::unique_ptr<mlir::Pass> fir::createCFGConversionPassWithNSW() {
+  return std::make_unique<CfgConversion>(true);
 }
diff --git a/flang/test/Driver/frontend-forwarding.f90 b/flang/test/Driver/frontend-forwarding.f90
index eac9773ce25c..35adb47b5686 100644
--- a/flang/test/Driver/frontend-forwarding.f90
+++ b/flang/test/Driver/frontend-forwarding.f90
@@ -19,6 +19,7 @@
 ! RUN:     -fversion-loops-for-stride \
 ! RUN:     -flang-experimental-hlfir \
 ! RUN:     -flang-deprecated-no-hlfir \
+! RUN:     -flang-experimental-integer-overflow \
 ! RUN:     -fno-ppc-native-vector-element-order \
 ! RUN:     -fppc-native-vector-element-order \
 ! RUN:     -mllvm -print-before-all \
@@ -50,6 +51,7 @@
 ! CHECK: "-fversion-loops-for-stride"
 ! CHECK: "-flang-experimental-hlfir"
 ! CHECK: "-flang-deprecated-no-hlfir"
+! CHECK: "-flang-experimental-integer-overflow"
 ! CHECK: "-fno-ppc-native-vector-element-order"
 ! CHECK: "-fppc-native-vector-element-order"
 ! CHECK: "-Rpass"
diff --git a/flang/test/Fir/loop01.fir b/flang/test/Fir/loop01.fir
index 72ca1c3989e4..c1cbb522c378 100644
--- a/flang/test/Fir/loop01.fir
+++ b/flang/test/Fir/loop01.fir
@@ -1,4 +1,5 @@
 // RUN: fir-opt --split-input-file --cfg-conversion %s | FileCheck %s
+// RUN: fir-opt --split-input-file --cfg-conversion="set-nsw=true" %s | FileCheck %s --check-prefix=NSW
 
 func.func @x(%lb : index, %ub : index, %step : index, %b : i1, %addr : !fir.ref<index>) {
   fir.do_loop %iv = %lb to %ub step %step unordered {
@@ -43,6 +44,34 @@ func.func private @f2() -> i1
 // CHECK:     }
 // CHECK:     func private @f2() -> i1
 
+// NSW:     func @x(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: i1, %[[VAL_4:.*]]: !fir.ref<index>) {
+// NSW:       %[[VAL_5:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
+// NSW:       %[[VAL_6:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] : index
+// NSW:       %[[VAL_7:.*]] = arith.divsi %[[VAL_6]], %[[VAL_2]] : index
+// NSW:       br ^bb1(%[[VAL_0]], %[[VAL_7]] : index, index)
+// NSW:     ^bb1(%[[VAL_8:.*]]: index, %[[VAL_9:.*]]: index):
+// NSW:       %[[VAL_10:.*]] = arith.constant 0 : index
+// NSW:       %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
+// NSW:       cond_br %[[VAL_11]], ^bb2, ^bb6
+// NSW:     ^bb2:
+// NSW:       cond_br %[[VAL_3]], ^bb3, ^bb4
+// NSW:     ^bb3:
+// NSW:       fir.store %[[VAL_8]] to %[[VAL_4]] : !fir.ref<index>
+// NSW:       br ^bb5
+// NSW:     ^bb4:
+// NSW:       %[[VAL_12:.*]] = arith.constant 0 : index
+// NSW:       fir.store %[[VAL_12]] to %[[VAL_4]] : !fir.ref<index>
+// NSW:       br ^bb5
+// NSW:     ^bb5:
+// NSW:       %[[VAL_13:.*]] = arith.addi %[[VAL_8]], %[[VAL_2]] overflow<nsw> : index
+// NSW:       %[[VAL_14:.*]] = arith.constant 1 : index
+// NSW:       %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
+// NSW:       br ^bb1(%[[VAL_13]], %[[VAL_15]] : index, index)
+// NSW:     ^bb6:
+// NSW:       return
+// NSW:     }
+// NSW:     func private @f2() -> i1
+
 // -----
 
 func.func @x2(%lo : index, %up : index, %ok : i1) {
@@ -79,6 +108,29 @@ func.func private @f3(i16)
 // CHECK:   }
 // CHECK:   func private @f3(i16)
 
+// NSW:   func @x2(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: i1) {
+// NSW:     %[[VAL_3:.*]] = arith.constant 1 : index
+// NSW:     br ^bb1(%[[VAL_0]], %[[VAL_2]] : index, i1)
+// NSW:   ^bb1(%[[VAL_4:.*]]: index, %[[VAL_5:.*]]: i1):
+// NSW:     %[[VAL_6:.*]] = arith.constant 0 : index
+// NSW:     %[[VAL_7:.*]] = arith.cmpi slt, %[[VAL_6]], %[[VAL_3]] : index
+// NSW:     %[[VAL_8:.*]] = arith.cmpi sle, %[[VAL_4]], %[[VAL_1]] : index
+// NSW:     %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_3]], %[[VAL_6]] : index
+// NSW:     %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_4]] : index
+// NSW:     %[[VAL_11:.*]] = arith.andi %[[VAL_7]], %[[VAL_8]] : i1
+// NSW:     %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
+// NSW:     %[[VAL_13:.*]] = arith.ori %[[VAL_11]], %[[VAL_12]] : i1
+// NSW:     %[[VAL_14:.*]] = arith.andi %[[VAL_5]], %[[VAL_13]] : i1
+// NSW:     cond_br %[[VAL_14]], ^bb2, ^bb3
+// NSW:   ^bb2:
+// NSW:     %[[VAL_15:.*]] = fir.call @f2() : () -> i1
+// NSW:     %[[VAL_16:.*]] = arith.addi %[[VAL_4]], %[[VAL_3]] overflow<nsw> : index
+// NSW:     br ^bb1(%[[VAL_16]], %[[VAL_15]] : index, i1)
+// NSW:   ^bb3:
+// NSW:     return
+// NSW:   }
+// NSW:   func private @f3(i16)
+
 // -----
 
 // do_loop with an extra loop-carried value
@@ -115,6 +167,29 @@ func.func @x3(%lo : index, %up : index) -> i1 {
 // CHECK:           return %[[VAL_8]] : i1
 // CHECK:         }
 
+// NSW-LABEL:   func @x3(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> i1 {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.constant true
+// NSW:           %[[VAL_4:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
+// NSW:           %[[VAL_5:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] : index
+// NSW:           %[[VAL_6:.*]] = arith.divsi %[[VAL_5]], %[[VAL_2]] : index
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_6]] : index, i1, index)
+// NSW:         ^bb1(%[[VAL_7:.*]]: index, %[[VAL_8:.*]]: i1, %[[VAL_9:.*]]: index):
+// NSW:           %[[VAL_10:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
+// NSW:           cond_br %[[VAL_11]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
+// NSW:           %[[VAL_13:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           %[[VAL_14:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_14]] : index
+// NSW:           br ^bb1(%[[VAL_13]], %[[VAL_12]], %[[VAL_15]] : index, i1, index)
+// NSW:         ^bb3:
+// NSW:           return %[[VAL_8]] : i1
+// NSW:         }
+
 // -----
 
 // iterate_while with an extra loop-carried value
@@ -160,6 +235,34 @@ func.func private @f4(i32) -> i1
 // CHECK:         }
 // CHECK:         func private @f4(i32) -> i1
 
+// NSW-LABEL:   func @y3(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> i1 {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.constant true
+// NSW:           %[[VAL_4:.*]] = fir.call @f2() : () -> i1
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_4]] : index, i1, i1)
+// NSW:         ^bb1(%[[VAL_5:.*]]: index, %[[VAL_6:.*]]: i1, %[[VAL_7:.*]]: i1):
+// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_8]], %[[VAL_2]] : index
+// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_5]], %[[VAL_1]] : index
+// NSW:           %[[VAL_11:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_8]] : index
+// NSW:           %[[VAL_12:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_5]] : index
+// NSW:           %[[VAL_13:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
+// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_11]], %[[VAL_12]] : i1
+// NSW:           %[[VAL_15:.*]] = arith.ori %[[VAL_13]], %[[VAL_14]] : i1
+// NSW:           %[[VAL_16:.*]] = arith.andi %[[VAL_6]], %[[VAL_15]] : i1
+// NSW:           cond_br %[[VAL_16]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
+// NSW:           %[[VAL_18:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           br ^bb1(%[[VAL_18]], %[[VAL_6]], %[[VAL_17]] : index, i1, i1)
+// NSW:         ^bb3:
+// NSW:           %[[VAL_19:.*]] = arith.andi %[[VAL_6]], %[[VAL_7]] : i1
+// NSW:           return %[[VAL_19]] : i1
+// NSW:         }
+// NSW:         func private @f4(i32) -> i1
+
 // -----
 
 // do_loop that returns the final value of the induction
@@ -196,6 +299,29 @@ func.func @x4(%lo : index, %up : index) -> index {
 // CHECK:           return %[[VAL_6]] : index
 // CHECK:         }
 
+// NSW-LABEL:   func @x4(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
+// NSW:           %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_2]] : index
+// NSW:           %[[VAL_5:.*]] = arith.divsi %[[VAL_4]], %[[VAL_2]] : index
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_5]] : index, index)
+// NSW:         ^bb1(%[[VAL_6:.*]]: index, %[[VAL_7:.*]]: index):
+// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_7]], %[[VAL_8]] : index
+// NSW:           cond_br %[[VAL_9]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_10:.*]] = fir.convert %[[VAL_6]] : (index) -> i32
+// NSW:           %[[VAL_11:.*]] = fir.call @f4(%[[VAL_10]]) : (i32) -> i1
+// NSW:           %[[VAL_12:.*]] = arith.addi %[[VAL_6]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           %[[VAL_13:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_14:.*]] = arith.subi %[[VAL_7]], %[[VAL_13]] : index
+// NSW:           br ^bb1(%[[VAL_12]], %[[VAL_14]] : index, index)
+// NSW:         ^bb3:
+// NSW:           return %[[VAL_6]] : index
+// NSW:         }
+
 // -----
 
 // iterate_while that returns the final value of both inductions
@@ -236,6 +362,32 @@ func.func @y4(%lo : index, %up : index) -> index {
 // CHECK:           return %[[VAL_4]] : index
 // CHECK:         }
 
+// NSW-LABEL:   func @y4(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.constant true
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]] : index, i1)
+// NSW:         ^bb1(%[[VAL_4:.*]]: index, %[[VAL_5:.*]]: i1):
+// NSW:           %[[VAL_6:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_7:.*]] = arith.cmpi slt, %[[VAL_6]], %[[VAL_2]] : index
+// NSW:           %[[VAL_8:.*]] = arith.cmpi sle, %[[VAL_4]], %[[VAL_1]] : index
+// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_6]] : index
+// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_4]] : index
+// NSW:           %[[VAL_11:.*]] = arith.andi %[[VAL_7]], %[[VAL_8]] : i1
+// NSW:           %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
+// NSW:           %[[VAL_13:.*]] = arith.ori %[[VAL_11]], %[[VAL_12]] : i1
+// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_5]], %[[VAL_13]] : i1
+// NSW:           cond_br %[[VAL_14]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_15:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
+// NSW:           %[[VAL_16:.*]] = fir.call @f4(%[[VAL_15]]) : (i32) -> i1
+// NSW:           %[[VAL_17:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           br ^bb1(%[[VAL_17]], %[[VAL_16]] : index, i1)
+// NSW:         ^bb3:
+// NSW:           return %[[VAL_4]] : index
+// NSW:         }
+
 // -----
 
 // do_loop that returns the final induction value
@@ -277,6 +429,31 @@ func.func @x5(%lo : index, %up : index) -> index {
 // CHECK:           return %[[VAL_7]] : index
 // CHECK:         }
 
+// NSW-LABEL:   func @x5(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.constant 42 : i16
+// NSW:           %[[VAL_4:.*]] = arith.subi %[[VAL_1]], %[[VAL_0]] : index
+// NSW:           %[[VAL_5:.*]] = arith.addi %[[VAL_4]], %[[VAL_2]] : index
+// NSW:           %[[VAL_6:.*]] = arith.divsi %[[VAL_5]], %[[VAL_2]] : index
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_3]], %[[VAL_6]] : index, i16, index)
+// NSW:         ^bb1(%[[VAL_7:.*]]: index, %[[VAL_8:.*]]: i16, %[[VAL_9:.*]]: index):
+// NSW:           %[[VAL_10:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_10]] : index
+// NSW:           cond_br %[[VAL_11]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_12:.*]] = fir.call @f2() : () -> i1
+// NSW:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i1) -> i16
+// NSW:           %[[VAL_14:.*]] = arith.addi %[[VAL_7]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           %[[VAL_15:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_16:.*]] = arith.subi %[[VAL_9]], %[[VAL_15]] : index
+// NSW:           br ^bb1(%[[VAL_14]], %[[VAL_13]], %[[VAL_16]] : index, i16, index)
+// NSW:         ^bb3:
+// NSW:           fir.call @f3(%[[VAL_8]]) : (i16) -> ()
+// NSW:           return %[[VAL_7]] : index
+// NSW:         }
+
 // -----
 
 // iterate_while that returns the both induction values
@@ -331,3 +508,37 @@ func.func @y5(%lo : index, %up : index) -> index {
 // CHECK:           fir.call @f3(%[[VAL_7]]) : (i16) -> ()
 // CHECK:           return %[[VAL_5]] : index
 // CHECK:         }
+
+// NSW-LABEL:   func @y5(
+// NSW-SAME:             %[[VAL_0:.*]]: index,
+// NSW-SAME:             %[[VAL_1:.*]]: index) -> index {
+// NSW:           %[[VAL_2:.*]] = arith.constant 1 : index
+// NSW:           %[[VAL_3:.*]] = arith.constant 42 : i16
+// NSW:           %[[VAL_4:.*]] = arith.constant true
+// NSW:           br ^bb1(%[[VAL_0]], %[[VAL_4]], %[[VAL_3]] : index, i1, i16)
+// NSW:         ^bb1(%[[VAL_5:.*]]: index, %[[VAL_6:.*]]: i1, %[[VAL_7:.*]]: i16):
+// NSW:           %[[VAL_8:.*]] = arith.constant 0 : index
+// NSW:           %[[VAL_9:.*]] = arith.cmpi slt, %[[VAL_8]], %[[VAL_2]] : index
+// NSW:           %[[VAL_10:.*]] = arith.cmpi sle, %[[VAL_5]], %[[VAL_1]] : index
+// NSW:           %[[VAL_11:.*]] = arith.cmpi slt, %[[VAL_2]], %[[VAL_8]] : index
+// NSW:           %[[VAL_12:.*]] = arith.cmpi sle, %[[VAL_1]], %[[VAL_5]] : index
+// NSW:           %[[VAL_13:.*]] = arith.andi %[[VAL_9]], %[[VAL_10]] : i1
+// NSW:           %[[VAL_14:.*]] = arith.andi %[[VAL_11]], %[[VAL_12]] : i1
+// NSW:           %[[VAL_15:.*]] = arith.ori %[[VAL_13]], %[[VAL_14]] : i1
+// NSW:           %[[VAL_16:.*]] = arith.andi %[[VAL_6]], %[[VAL_15]] : i1
+// NSW:           cond_br %[[VAL_16]], ^bb2, ^bb3
+// NSW:         ^bb2:
+// NSW:           %[[VAL_17:.*]] = fir.call @f2() : () -> i1
+// NSW:           %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i1) -> i16
+// NSW:           %[[VAL_19:.*]] = arith.addi %[[VAL_5]], %[[VAL_2]] overflow<nsw> : index
+// NSW:           br ^bb1(%[[VAL_19]], %[[VAL_17]], %[[VAL_18]] : index, i1, i16)
+// NSW:         ^bb3:
+// NSW:           cond_br %[[VAL_6]], ^bb4, ^bb5
+// NSW:         ^bb4:
+// NSW:           %[[VAL_20:.*]] = arith.constant 0 : i32
+// NSW:           %[[VAL_21:.*]] = fir.call @f4(%[[VAL_20]]) : (i32) -> i1
+// NSW:           br ^bb5
+// NSW:         ^bb5:
+// NSW:           fir.call @f3(%[[VAL_7]]) : (i16) -> ()
+// NSW:           return %[[VAL_5]] : index
+// NSW:         }
diff --git a/flang/test/Lower/array-substring.f90 b/flang/test/Lower/array-substring.f90
index 421c4b28ac8f..2e283997e3e0 100644
--- a/flang/test/Lower/array-substring.f90
+++ b/flang/test/Lower/array-substring.f90
@@ -1,4 +1,5 @@
 ! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -hlfir=false -integer-overflow %s -o - | FileCheck %s --check-prefix=NSW
 
 ! CHECK-LABEL: func @_QPtest(
 ! CHECK-SAME:     %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}) -> !fir.array<1x!fir.logical<4>> {
@@ -45,3 +46,42 @@ function test(C)
 
   test = C(1:1)(1:8) == (/'ABCDabcd'/) 
 end function test
+
+! NSW-LABEL: func @_QPtest(
+! NSW-SAME:     %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}) -> !fir.array<1x!fir.logical<4>> {
+! NSW-DAG:         %[[VAL_1:.*]] = arith.constant 1 : index
+! NSW-DAG:         %[[VAL_2:.*]] = arith.constant 0 : index
+! NSW-DAG:         %[[VAL_3:.*]] = arith.constant 0 : i32
+! NSW-DAG:         %[[VAL_4:.*]] = arith.constant 8 : index
+! NSW:         %[[VAL_6:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! NSW:         %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<1x!fir.char<1,12>>>
+! NSW:         %[[VAL_8:.*]] = fir.alloca !fir.array<1x!fir.logical<4>> {bindc_name = "test", uniq_name = "_QFtestEtest"}
+! NSW:         %[[VAL_9:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+! NSW:         %[[VAL_10:.*]] = fir.slice %[[VAL_1]], %[[VAL_1]], %[[VAL_1]] : (index, index, index) -> !fir.slice<1>
+! NSW:         %[[VAL_11:.*]] = fir.address_of(@_QQ{{.*}}) : !fir.ref<!fir.array<1x!fir.char<1,8>>>
+! NSW:         br ^bb1(%[[VAL_2]], %[[VAL_1]] : index, index)
+! NSW:       ^bb1(%[[VAL_12:.*]]: index, %[[VAL_13:.*]]: index):
+! NSW:         %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_13]], %[[VAL_2]] : index
+! NSW:         cond_br %[[VAL_14]], ^bb2, ^bb3
+! NSW:       ^bb2:
+! NSW:         %[[VAL_15:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : index
+! NSW:         %[[VAL_16:.*]] = fir.array_coor %[[VAL_7]](%[[VAL_9]]) {{\[}}%[[VAL_10]]] %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.char<1,12>>>, !fir.shape<1>, !fir.slice<1>, index) -> !fir.ref<!fir.char<1,12>>
+! NSW:         %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (!fir.ref<!fir.char<1,12>>) -> !fir.ref<!fir.array<12x!fir.char<1>>>
+! NSW:         %[[VAL_18:.*]] = fir.coordinate_of %[[VAL_17]], %[[VAL_2]] : (!fir.ref<!fir.array<12x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
+! NSW:         %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
+! NSW:         %[[VAL_20:.*]] = fir.array_coor %[[VAL_11]](%[[VAL_9]]) %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.char<1,8>>>, !fir.shape<1>, index) -> !fir.ref<!fir.char<1,8>>
+! NSW:         %[[VAL_21:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
+! NSW:         %[[VAL_22:.*]] = fir.convert %[[VAL_20]] : (!fir.ref<!fir.char<1,8>>) -> !fir.ref<i8>
+! NSW:         %[[VAL_23:.*]] = fir.convert %[[VAL_4]] : (index) -> i64
+! NSW:         %[[VAL_24:.*]] = fir.call @_FortranACharacterCompareScalar1(%[[VAL_21]], %[[VAL_22]], %[[VAL_23]], %[[VAL_23]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64, i64) -> i32
+! NSW:         %[[VAL_25:.*]] = arith.cmpi eq, %[[VAL_24]], %[[VAL_3]] : i32
+! NSW:         %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (i1) -> !fir.logical<4>
+! NSW:         %[[VAL_27:.*]] = fir.array_coor %[[VAL_8]](%[[VAL_9]]) %[[VAL_15]] : (!fir.ref<!fir.array<1x!fir.logical<4>>>, !fir.shape<1>, index) -> !fir.ref<!fir.logical<4>>
+! NSW:         fir.store %[[VAL_26]] to %[[VAL_27]] : !fir.ref<!fir.logical<4>>
+! NSW:         %[[VAL_15_NSW:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] overflow<nsw> : index
+! NSW:         %[[VAL_28:.*]] = arith.subi %[[VAL_13]], %[[VAL_1]] : index
+! NSW:         br ^bb1(%[[VAL_15_NSW]], %[[VAL_28]] : index, index)
+! NSW:       ^bb3:
+! NSW:         %[[VAL_29:.*]] = fir.load %[[VAL_8]] : !fir.ref<!fir.array<1x!fir.logical<4>>>
+! NSW:         return %[[VAL_29]] : !fir.array<1x!fir.logical<4>>
+! NSW:       }
diff --git a/flang/test/Lower/do_loop.f90 b/flang/test/Lower/do_loop.f90
index d9c83658ee25..a46e6c947391 100644
--- a/flang/test/Lower/do_loop.f90
+++ b/flang/test/Lower/do_loop.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -mllvm --use-desc-for-alloc=false -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
+! RUN: %flang_fc1 -mllvm --use-desc-for-alloc=false -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
 
 ! Simple tests for structured ordered loops with loop-control.
 ! Tests the structure of the loop, storage to index variable and return and 
@@ -7,8 +8,10 @@
 
 ! Test a simple loop with the final value of the index variable read outside the loop
 ! CHECK-LABEL: simple_loop
+! NSW-LABEL:   simple_loop
 subroutine simple_loop
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_loopEi"}
+  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_loopEi"}
   integer :: i
 
   ! CHECK: %[[C1:.*]] = arith.constant 1 : i32
@@ -18,14 +21,18 @@ subroutine simple_loop
   ! CHECK: %[[C1:.*]] = arith.constant 1 : index
   ! CHECK: %[[LB:.*]] = fir.convert %[[C1_CVT]] : (index) -> i32
   ! CHECK: %[[LI_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+  ! NSW:   %[[LI_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[C1_CVT]] to %[[C5_CVT]] step %[[C1]]
   ! CHECK-SAME: iter_args(%[[IV:.*]] = %[[LB]]) -> (index, i32) {
   do i=1,5
   ! CHECK:   fir.store %[[IV]] to %[[I_REF]] : !fir.ref<i32>
   ! CHECK:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[C1]] : index
+  ! NSW:     %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[C1:.*]] overflow<nsw> : index
   ! CHECK:   %[[STEPCAST:.*]] = fir.convert %[[C1]] : (index) -> i32
   ! CHECK:   %[[IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
+  ! NSW:     %[[IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
   ! CHECK:   %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]] : i32
+  ! NSW:     %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST:.*]] overflow<nsw> : i32
   ! CHECK:  fir.result %[[LI_NEXT]], %[[IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -37,11 +44,14 @@ end subroutine
 
 ! Test a 2-nested loop with a body composed of a reduction. Values are read from a 2d array.
 ! CHECK-LABEL: nested_loop
+! NSW-LABEL:   nested_loop
 subroutine nested_loop
   ! CHECK: %[[ARR_REF:.*]] = fir.alloca !fir.array<5x5xi32> {bindc_name = "arr", uniq_name = "_QFnested_loopEarr"}
   ! CHECK: %[[ASUM_REF:.*]] = fir.alloca i32 {bindc_name = "asum", uniq_name = "_QFnested_loopEasum"}
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_loopEi"}
+  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_loopEi"}
   ! CHECK: %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_loopEj"}
+  ! NSW:   %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_loopEj"}
   integer :: asum, arr(5,5)
   integer :: i, j
   asum = 0
@@ -52,6 +62,7 @@ subroutine nested_loop
   ! CHECK: %[[ST_I:.*]] = arith.constant 1 : index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_I_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_I_CVT]] to %[[E_I_CVT]] step %[[ST_I]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=1,5
@@ -63,6 +74,7 @@ subroutine nested_loop
     ! CHECK: %[[ST_J:.*]] = arith.constant 1 : index
     ! CHECK: %[[J_LB:.*]] = fir.convert %[[S_J_CVT]] : (index) -> i32
     ! CHECK: %[[J_RES:.*]]:2 = fir.do_loop %[[LJ:[^ ]*]] =
+    ! NSW:   %[[J_RES:.*]]:2 = fir.do_loop %[[LJ:[^ ]*]] =
     ! CHECK-SAME: %[[S_J_CVT]] to %[[E_J_CVT]] step %[[ST_J]]
     ! CHECK-SAME: iter_args(%[[J_IV:.*]] = %[[J_LB]]) -> (index, i32) {
     do j=1,5
@@ -82,17 +94,23 @@ subroutine nested_loop
       ! CHECK: fir.store %[[ASUM_NEW]] to %[[ASUM_REF]] : !fir.ref<i32>
       asum = asum + arr(i,j)
       ! CHECK: %[[LJ_NEXT:.*]] = arith.addi %[[LJ]], %[[ST_J]] : index
+      ! NSW:   %[[LJ_NEXT:.*]] = arith.addi %[[LJ]], %[[ST_J:.*]] overflow<nsw> : index
       ! CHECK: %[[J_STEPCAST:.*]] = fir.convert %[[ST_J]] : (index) -> i32
       ! CHECK: %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
+      ! NSW:   %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
       ! CHECK: %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] : i32
+      ! NSW:   %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST:.*]] overflow<nsw> : i32
       ! CHECK: fir.result %[[LJ_NEXT]], %[[J_IVINC]] : index, i32
     ! CHECK: }
     end do
     ! CHECK: fir.store %[[J_RES]]#1 to %[[J_REF]] : !fir.ref<i32>
     ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_I]] : index
+    ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_I:.*]] overflow<nsw> : index
     ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_I]] : (index) -> i32
     ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
+    ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
     ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
+    ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
     ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -101,9 +119,11 @@ end subroutine
 
 ! Test a downcounting loop
 ! CHECK-LABEL: down_counting_loop
+! NSW-LABEL:   down_counting_loop
 subroutine down_counting_loop()
   integer :: i
   ! CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFdown_counting_loopEi"}
+  ! NSW:   %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFdown_counting_loopEi"}
 
   ! CHECK: %[[C5:.*]] = arith.constant 5 : i32
   ! CHECK: %[[C5_CVT:.*]] = fir.convert %[[C5]] : (i32) -> index
@@ -113,14 +133,18 @@ subroutine down_counting_loop()
   ! CHECK: %[[CMINUS1_STEP_CVT:.*]] = fir.convert %[[CMINUS1]] : (i32) -> index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[C5_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[C5_CVT]] to %[[C1_CVT]] step %[[CMINUS1_STEP_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=5,1,-1
   ! CHECK: fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i32>
   ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[CMINUS1_STEP_CVT]] : index
+  ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[CMINUS1_STEP_CVT:.*]] overflow<nsw> : index
   ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[CMINUS1_STEP_CVT]] : (index) -> i32
   ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
+  ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
   ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
+  ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
   ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -129,6 +153,7 @@ end subroutine
 
 ! Test a general loop with a variable step
 ! CHECK-LABEL: loop_with_variable_step
+! NSW-LABEL:   loop_with_variable_step
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}, %[[E_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "e"}, %[[ST_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "st"}) {
 subroutine loop_with_variable_step(s,e,st)
   integer :: s, e, st
@@ -141,14 +166,18 @@ subroutine loop_with_variable_step(s,e,st)
   ! CHECK: %[[ST_CVT:.*]] = fir.convert %[[ST]] : (i32) -> index
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i32
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do i=s,e,st
   ! CHECK:  fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i32>
   ! CHECK:  %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
+  ! NSW:    %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
   ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i32
   ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
+  ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
   ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
+  ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
   ! CHECK:  fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   ! CHECK: }
   end do
@@ -157,11 +186,13 @@ end subroutine
 
 ! Test usage of pointer variables as index, start, end and step variables
 ! CHECK-LABEL: loop_with_pointer_variables
+! NSW-LABEL:   loop_with_pointer_variables
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "s", fir.target}, %[[E_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "e", fir.target}, %[[ST_REF:.*]]: !fir.ref<i32> {fir.bindc_name = "st", fir.target}) {
 subroutine loop_with_pointer_variables(s,e,st)
 ! CHECK:  %[[E_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEeptr.addr"}
 ! CHECK:  %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", fir.target, uniq_name = "_QFloop_with_pointer_variablesEi"}
 ! CHECK:  %[[I_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEiptr.addr"}
+! NSW:    %[[I_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEiptr.addr"}
 ! CHECK:  %[[S_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEsptr.addr"}
 ! CHECK:  %[[ST_PTR_REF:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFloop_with_pointer_variablesEstptr.addr"}
   integer, target :: i
@@ -182,6 +213,7 @@ subroutine loop_with_pointer_variables(s,e,st)
   stptr => st
 
 ! CHECK:  %[[I_PTR:.*]] = fir.load %[[I_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
+! NSW:    %[[I_PTR:.*]] = fir.load %[[I_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
 ! CHECK:  %[[S_PTR:.*]] = fir.load %[[S_PTR_REF]] : !fir.ref<!fir.ptr<i32>>
 ! CHECK:  %[[S:.*]] = fir.load %[[S_PTR]] : !fir.ptr<i32>
 ! CHECK:  %[[S_CVT:.*]] = fir.convert %[[S]] : (i32) -> index
@@ -193,14 +225,18 @@ subroutine loop_with_pointer_variables(s,e,st)
 ! CHECK:  %[[ST_CVT:.*]] = fir.convert %[[ST]] : (i32) -> index
 ! CHECK:  %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i32
 ! CHECK:  %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+! NSW:    %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
 ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
 ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i32) {
   do iptr=sptr,eptr,stptr
 ! CHECK:    fir.store %[[I_IV]] to %[[I_PTR]] : !fir.ptr<i32>
 ! CHECK:    %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
+! NSW:      %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
 ! CHECK:    %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i32
 ! CHECK:    %[[I_IVLOAD:.*]] = fir.load %[[I_PTR]] : !fir.ptr<i32>
+! NSW:      %[[I_IVLOAD:.*]] = fir.load %[[I_PTR]] : !fir.ptr<i32>
 ! CHECK:    %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i32
+! NSW:      %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i32
 ! CHECK:    fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i32
   end do
 ! CHECK:  }
@@ -209,9 +245,11 @@ end subroutine
 
 ! Test usage of non-default integer kind for loop control and loop index variable
 ! CHECK-LABEL: loop_with_non_default_integer
+! NSW-LABEL:   loop_with_non_default_integer
 ! CHECK-SAME: (%[[S_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "s"}, %[[E_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "e"}, %[[ST_REF:.*]]: !fir.ref<i64> {fir.bindc_name = "st"}) {
 subroutine loop_with_non_default_integer(s,e,st)
   ! CHECK: %[[I_REF:.*]] = fir.alloca i64 {bindc_name = "i", uniq_name = "_QFloop_with_non_default_integerEi"}
+  ! NSW:   %[[I_REF:.*]] = fir.alloca i64 {bindc_name = "i", uniq_name = "_QFloop_with_non_default_integerEi"}
   integer(kind=8):: i
   ! CHECK: %[[S:.*]] = fir.load %[[S_REF]] : !fir.ref<i64>
   ! CHECK: %[[S_CVT:.*]] = fir.convert %[[S]] : (i64) -> index
@@ -223,14 +261,18 @@ subroutine loop_with_non_default_integer(s,e,st)
 
   ! CHECK: %[[I_LB:.*]] = fir.convert %[[S_CVT]] : (index) -> i64
   ! CHECK: %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
+  ! NSW:   %[[I_RES:.*]]:2 = fir.do_loop %[[LI:[^ ]*]] =
   ! CHECK-SAME: %[[S_CVT]] to %[[E_CVT]] step %[[ST_CVT]]
   ! CHECK-SAME: iter_args(%[[I_IV:.*]] = %[[I_LB]]) -> (index, i64) {
   do i=s,e,st
     ! CHECK: fir.store %[[I_IV]] to %[[I_REF]] : !fir.ref<i64>
     ! CHECK: %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT]] : index
+    ! NSW:   %[[LI_NEXT:.*]] = arith.addi %[[LI]], %[[ST_CVT:.*]] overflow<nsw> : index
     ! CHECK: %[[I_STEPCAST:.*]] = fir.convert %[[ST_CVT]] : (index) -> i64
     ! CHECK: %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i64>
+    ! NSW:   %[[I_IVLOAD:.*]] = fir.load %[[I_REF]] : !fir.ref<i64>
     ! CHECK: %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST]] : i64
+    ! NSW:   %[[I_IVINC:.*]] = arith.addi %[[I_IVLOAD]], %[[I_STEPCAST:.*]] overflow<nsw> : i64
     ! CHECK: fir.result %[[LI_NEXT]], %[[I_IVINC]] : index, i64
   end do
   ! CHECK: }
diff --git a/flang/test/Lower/do_loop_unstructured.f90 b/flang/test/Lower/do_loop_unstructured.f90
index c6bdd4b64ce3..e1a669e09c9a 100644
--- a/flang/test/Lower/do_loop_unstructured.f90
+++ b/flang/test/Lower/do_loop_unstructured.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
 
 ! Tests for unstructured loops.
 
@@ -44,6 +45,36 @@ end subroutine
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
 
+! NSW-LABEL: simple_unstructured
+! NSW:   %[[TRIP_VAR_REF:.*]] = fir.alloca i32
+! NSW:   %[[LOOP_VAR_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_unstructuredEi"}
+! NSW:   %[[ONE:.*]] = arith.constant 1 : i32
+! NSW:   %[[HUNDRED:.*]] = arith.constant 100 : i32
+! NSW:   %[[STEP_ONE:.*]] = arith.constant 1 : i32
+! NSW:   %[[TMP1:.*]] = arith.subi %[[HUNDRED]], %[[ONE]] : i32
+! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[STEP_ONE]] : i32
+! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[STEP_ONE]] : i32
+! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[ONE]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER:.*]]
+! NSW: ^[[HEADER]]:
+! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
+! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
+! NSW: ^[[BODY]]:
+! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_NEXT:.*]] = arith.subi %[[TRIP_VAR]], %[[ONE_1]] : i32
+! NSW:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[STEP_ONE_2:.*]] = arith.constant 1 : i32
+! NSW:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_ONE_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER]]
+! NSW: ^[[EXIT]]:
+! NSW:   return
+
 ! Test an unstructured loop with a step. Mostly similar to the previous one.
 ! Only difference is a non-unit step.
 subroutine simple_unstructured_with_step()
@@ -83,6 +114,36 @@ end subroutine
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
 
+! NSW-LABEL: simple_unstructured_with_step
+! NSW:   %[[TRIP_VAR_REF:.*]] = fir.alloca i32
+! NSW:   %[[LOOP_VAR_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_unstructured_with_stepEi"}
+! NSW:   %[[ONE:.*]] = arith.constant 1 : i32
+! NSW:   %[[HUNDRED:.*]] = arith.constant 100 : i32
+! NSW:   %[[STEP:.*]] = arith.constant 2 : i32
+! NSW:   %[[TMP1:.*]] = arith.subi %[[HUNDRED]], %[[ONE]] : i32
+! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[STEP]] : i32
+! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[STEP]] : i32
+! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[ONE]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER:.*]]
+! NSW: ^[[HEADER]]:
+! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
+! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
+! NSW: ^[[BODY]]:
+! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_NEXT:.*]] = arith.subi %[[TRIP_VAR]], %[[ONE_1]] : i32
+! NSW:   fir.store %[[TRIP_VAR_NEXT]] to %[[TRIP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR:.*]] = fir.load %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   %[[STEP_2:.*]] = arith.constant 2 : i32
+! NSW:   %[[LOOP_VAR_NEXT:.*]] = arith.addi %[[LOOP_VAR]], %[[STEP_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_NEXT]] to %[[LOOP_VAR_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER]]
+! NSW: ^[[EXIT]]:
+! NSW:   return
+
 ! Test a three nested unstructured loop. Three nesting is the basic case where
 ! we have loops that are neither innermost or outermost.
 subroutine nested_unstructured()
@@ -180,6 +241,90 @@ end subroutine
 ! CHECK: ^[[EXIT_I]]:
 ! CHECK:   return
 
+! NSW-LABEL: nested_unstructured
+! NSW:   %[[TRIP_VAR_K_REF:.*]] = fir.alloca i32
+! NSW:   %[[TRIP_VAR_J_REF:.*]] = fir.alloca i32
+! NSW:   %[[TRIP_VAR_I_REF:.*]] = fir.alloca i32
+! NSW:   %[[LOOP_VAR_I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_unstructuredEi"}
+! NSW:   %[[LOOP_VAR_J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_unstructuredEj"}
+! NSW:   %[[LOOP_VAR_K_REF:.*]] = fir.alloca i32 {bindc_name = "k", uniq_name = "_QFnested_unstructuredEk"}
+! NSW:   %[[I_START:.*]] = arith.constant 1 : i32
+! NSW:   %[[I_END:.*]] = arith.constant 100 : i32
+! NSW:   %[[I_STEP:.*]] = arith.constant 1 : i32
+! NSW:   %[[TMP1:.*]] = arith.subi %[[I_END]], %[[I_START]] : i32
+! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[I_STEP]] : i32
+! NSW:   %[[TRIP_COUNT_I:.*]] = arith.divsi %[[TMP2]], %[[I_STEP]] : i32
+! NSW:   fir.store %[[TRIP_COUNT_I]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[I_START]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_I:.*]]
+! NSW: ^[[HEADER_I]]:
+! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO_1:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND_I:.*]] = arith.cmpi sgt, %[[TRIP_VAR_I]], %[[ZERO_1]] : i32
+! NSW:   cf.cond_br %[[COND_I]], ^[[BODY_I:.*]], ^[[EXIT_I:.*]]
+! NSW: ^[[BODY_I]]:
+! NSW:   %[[J_START:.*]] = arith.constant 1 : i32
+! NSW:   %[[J_END:.*]] = arith.constant 200 : i32
+! NSW:   %[[J_STEP:.*]] = arith.constant 1 : i32
+! NSW:   %[[TMP3:.*]] = arith.subi %[[J_END]], %[[J_START]] : i32
+! NSW:   %[[TMP4:.*]] = arith.addi %[[TMP3]], %[[J_STEP]] : i32
+! NSW:   %[[TRIP_COUNT_J:.*]] = arith.divsi %[[TMP4]], %[[J_STEP]] : i32
+! NSW:   fir.store %[[TRIP_COUNT_J]] to %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[J_START]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_J:.*]]
+! NSW: ^[[HEADER_J]]:
+! NSW:   %[[TRIP_VAR_J:.*]] = fir.load %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO_2:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND_J:.*]] = arith.cmpi sgt, %[[TRIP_VAR_J]], %[[ZERO_2]] : i32
+! NSW:   cf.cond_br %[[COND_J]], ^[[BODY_J:.*]], ^[[EXIT_J:.*]]
+! NSW: ^[[BODY_J]]:
+! NSW:   %[[K_START:.*]] = arith.constant 1 : i32
+! NSW:   %[[K_END:.*]] = arith.constant 300 : i32
+! NSW:   %[[K_STEP:.*]] = arith.constant 1 : i32
+! NSW:   %[[TMP3:.*]] = arith.subi %[[K_END]], %[[K_START]] : i32
+! NSW:   %[[TMP4:.*]] = arith.addi %[[TMP3]], %[[K_STEP]] : i32
+! NSW:   %[[TRIP_COUNT_K:.*]] = arith.divsi %[[TMP4]], %[[K_STEP]] : i32
+! NSW:   fir.store %[[TRIP_COUNT_K]] to %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[K_START]] to %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_K:.*]]
+! NSW: ^[[HEADER_K]]:
+! NSW:   %[[TRIP_VAR_K:.*]] = fir.load %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO_2:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND_K:.*]] = arith.cmpi sgt, %[[TRIP_VAR_K]], %[[ZERO_2]] : i32
+! NSW:   cf.cond_br %[[COND_K]], ^[[BODY_K:.*]], ^[[EXIT_K:.*]]
+! NSW: ^[[BODY_K]]:
+! NSW:   %[[TRIP_VAR_K:.*]] = fir.load %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_K_NEXT:.*]] = arith.subi %[[TRIP_VAR_K]], %[[ONE_1]] : i32
+! NSW:   fir.store %[[TRIP_VAR_K_NEXT]] to %[[TRIP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR_K:.*]] = fir.load %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   %[[K_STEP_2:.*]] = arith.constant 1 : i32
+! NSW:   %[[LOOP_VAR_K_NEXT:.*]] = arith.addi %[[LOOP_VAR_K]], %[[K_STEP_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_K_NEXT]] to %[[LOOP_VAR_K_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_K]]
+! NSW: ^[[EXIT_K]]:
+! NSW:   %[[TRIP_VAR_J:.*]] = fir.load %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_J_NEXT:.*]] = arith.subi %[[TRIP_VAR_J]], %[[ONE_1]] : i32
+! NSW:   fir.store %[[TRIP_VAR_J_NEXT]] to %[[TRIP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   %[[J_STEP_2:.*]] = arith.constant 1 : i32
+! NSW:   %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %[[J_STEP_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_J_NEXT]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_J]]
+! NSW: ^[[EXIT_J]]:
+! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[ONE_1:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_I_NEXT:.*]] = arith.subi %[[TRIP_VAR_I]], %[[ONE_1]] : i32
+! NSW:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
+! NSW:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER_I]]
+! NSW: ^[[EXIT_I]]:
+! NSW:   return
+
 ! Test the existence of a structured loop inside an unstructured loop.
 ! Only minimal checks are inserted for the structured loop.
 subroutine nested_structured_in_unstructured()
@@ -211,9 +356,12 @@ end subroutine
 ! CHECK:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
 ! CHECK: ^[[BODY]]:
 ! CHECK:   %{{.*}} = fir.do_loop %[[J_INDEX:[^ ]*]] =
-! CHECK-SAME: %{{.*}} to %{{.*}} step %{{[^ ]*}}
+! CHECK-SAME: %{{.*}} to %{{.*}} step %[[ST:[^ ]*]]
 ! CHECK-SAME: iter_args(%[[J_IV:.*]] = %{{.*}}) -> (index, i32) {
 ! CHECK:     fir.store %[[J_IV]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! CHECK:     %[[J_INDEX_NEXT:.*]] = arith.addi %[[J_INDEX]], %[[ST]] : index
+! CHECK:     %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! CHECK:     %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %{{[^ ]*}} : i32
 ! CHECK:   }
 ! CHECK:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
 ! CHECK:   %[[C1_3:.*]] = arith.constant 1 : i32
@@ -226,3 +374,42 @@ end subroutine
 ! CHECK:   cf.br ^[[HEADER]]
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
+
+! NSW-LABEL: nested_structured_in_unstructured
+! NSW:   %[[TRIP_VAR_I_REF:.*]] = fir.alloca i32
+! NSW:   %[[LOOP_VAR_I_REF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFnested_structured_in_unstructuredEi"}
+! NSW:   %[[LOOP_VAR_J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFnested_structured_in_unstructuredEj"}
+! NSW:   %[[I_START:.*]] = arith.constant 1 : i32
+! NSW:   %[[I_END:.*]] = arith.constant 100 : i32
+! NSW:   %[[I_STEP:.*]] = arith.constant 1 : i32
+! NSW:   %[[TMP1:.*]] = arith.subi %[[I_END]], %[[I_START]] : i32
+! NSW:   %[[TMP2:.*]] = arith.addi %[[TMP1]], %[[I_STEP]] : i32
+! NSW:   %[[TRIP_COUNT:.*]] = arith.divsi %[[TMP2]], %[[I_STEP]] : i32
+! NSW:   fir.store %[[TRIP_COUNT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   fir.store %[[I_START]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER:.*]]
+! NSW: ^[[HEADER]]:
+! NSW:   %[[TRIP_VAR:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[ZERO:.*]] = arith.constant 0 : i32
+! NSW:   %[[COND:.*]] = arith.cmpi sgt, %[[TRIP_VAR]], %[[ZERO]] : i32
+! NSW:   cf.cond_br %[[COND]], ^[[BODY:.*]], ^[[EXIT:.*]]
+! NSW: ^[[BODY]]:
+! NSW:   %{{.*}} = fir.do_loop %[[J_INDEX:[^ ]*]] =
+! NSW-SAME: %{{.*}} to %{{.*}} step %[[ST:[^ ]*]]
+! NSW-SAME: iter_args(%[[J_IV:.*]] = %{{.*}}) -> (index, i32) {
+! NSW:     fir.store %[[J_IV]] to %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:     %[[J_INDEX_NEXT:.*]] = arith.addi %[[J_INDEX]], %[[ST]] overflow<nsw> : index
+! NSW:     %[[LOOP_VAR_J:.*]] = fir.load %[[LOOP_VAR_J_REF]] : !fir.ref<i32>
+! NSW:     %[[LOOP_VAR_J_NEXT:.*]] = arith.addi %[[LOOP_VAR_J]], %{{[^ ]*}} overflow<nsw> : i32
+! NSW:   }
+! NSW:   %[[TRIP_VAR_I:.*]] = fir.load %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[C1_3:.*]] = arith.constant 1 : i32
+! NSW:   %[[TRIP_VAR_I_NEXT:.*]] = arith.subi %[[TRIP_VAR_I]], %[[C1_3]] : i32
+! NSW:   fir.store %[[TRIP_VAR_I_NEXT]] to %[[TRIP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[LOOP_VAR_I:.*]] = fir.load %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   %[[I_STEP_2:.*]] = arith.constant 1 : i32
+! NSW:   %[[LOOP_VAR_I_NEXT:.*]] = arith.addi %[[LOOP_VAR_I]], %[[I_STEP_2]] overflow<nsw> : i32
+! NSW:   fir.store %[[LOOP_VAR_I_NEXT]] to %[[LOOP_VAR_I_REF]] : !fir.ref<i32>
+! NSW:   cf.br ^[[HEADER]]
+! NSW: ^[[EXIT]]:
+! NSW:   return
diff --git a/flang/test/Lower/infinite_loop.f90 b/flang/test/Lower/infinite_loop.f90
index 0450e2c4485f..6942dda8d7a2 100644
--- a/flang/test/Lower/infinite_loop.f90
+++ b/flang/test/Lower/infinite_loop.f90
@@ -1,5 +1,6 @@
 ! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -o - %s | FileCheck %s
+! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -flang-experimental-integer-overflow -o - %s | FileCheck %s --check-prefix=NSW
 
 ! Tests for infinite loop.
 
@@ -106,6 +107,39 @@ end subroutine
 ! CHECK: ^[[RETURN]]:
 ! CHECK:   return
 
+! NSW-LABEL: structured_loop_in_infinite
+! NSW-SAME: %[[I_REF:.*]]: !fir.ref<i32>
+! NSW:  %[[J_REF:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFstructured_loop_in_infiniteEj"}
+! NSW:  cf.br ^[[BODY1:.*]]
+! NSW: ^[[BODY1]]:
+! NSW:  %[[I:.*]] = fir.load %[[I_REF]] : !fir.ref<i32>
+! NSW:  %[[C100:.*]] = arith.constant 100 : i32
+! NSW:  %[[COND:.*]] = arith.cmpi sgt, %[[I]], %[[C100]] : i32
+! NSW:  cf.cond_br %[[COND]], ^[[EXIT:.*]], ^[[BODY2:.*]]
+! NSW: ^[[EXIT]]:
+! NSW:  cf.br ^[[RETURN:.*]]
+! NSW: ^[[BODY2:.*]]:
+! NSW:  %[[C1:.*]] = arith.constant 1 : i32
+! NSW:  %[[C1_INDEX:.*]] = fir.convert %[[C1]] : (i32) -> index
+! NSW:  %[[C10:.*]] = arith.constant 10 : i32
+! NSW:  %[[C10_INDEX:.*]] = fir.convert %[[C10]] : (i32) -> index
+! NSW:  %[[C1_1:.*]] = arith.constant 1 : index
+! NSW:  %[[J_LB:.*]] = fir.convert %[[C1_INDEX]] : (index) -> i32
+! NSW:  %[[J_FINAL:.*]]:2 = fir.do_loop %[[J:[^ ]*]] =
+! NSW-SAME: %[[C1_INDEX]] to %[[C10_INDEX]] step %[[C1_1]]
+! NSW-SAME: iter_args(%[[J_IV:.*]] = %[[J_LB]]) -> (index, i32) {
+! NSW:    fir.store %[[J_IV]] to %[[J_REF]] : !fir.ref<i32>
+! NSW:    %[[J_NEXT:.*]] = arith.addi %[[J]], %[[C1_1]] overflow<nsw> : index
+! NSW:    %[[J_STEPCAST:.*]] = fir.convert %[[C1_1]] : (index) -> i32
+! NSW:    %[[J_IVLOAD:.*]] = fir.load %[[J_REF]] : !fir.ref<i32>
+! NSW:    %[[J_IVINC:.*]] = arith.addi %[[J_IVLOAD]], %[[J_STEPCAST]] overflow<nsw> : i32
+! NSW:    fir.result %[[J_NEXT]], %[[J_IVINC]] : index, i32
+! NSW:  }
+! NSW:  fir.store %[[J_FINAL]]#1 to %[[J_REF]] : !fir.ref<i32>
+! NSW:  cf.br ^[[BODY1]]
+! NSW: ^[[RETURN]]:
+! NSW:   return
+
 subroutine empty_infinite_in_while(i)
   integer :: i
   do while (i .gt. 50)
diff --git a/flang/test/Lower/io-implied-do-fixes.f90 b/flang/test/Lower/io-implied-do-fixes.f90
index a309efa17f12..a6c115fa80de 100644
--- a/flang/test/Lower/io-implied-do-fixes.f90
+++ b/flang/test/Lower/io-implied-do-fixes.f90
@@ -1,4 +1,5 @@
 ! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false -integer-overflow %s -o - | FileCheck %s --check-prefix=NSW
 ! UNSUPPORTED: system-windows
 
 ! CHECK-LABEL: func @_QPido1
@@ -7,9 +8,23 @@
 ! CHECK: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
 ! CHECK:   %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
 ! CHECK:   fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.ptr<i32>
+! CHECK:   %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
+! CHECK:   fir.result %[[J_VAL_NEXT]] : index
 ! CHECK: }
 ! CHECK: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.ptr<i32>
+
+! NSW-LABEL: func @_QPido1
+! NSW: %[[J_REF_ADDR:.*]] = fir.alloca !fir.ptr<i32> {uniq_name = "_QFido1Eiptr.addr"}
+! NSW: %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.ptr<i32>>
+! NSW: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
+! NSW:   %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
+! NSW:   fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.ptr<i32>
+! NSW:   %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
+! NSW:   fir.result %[[J_VAL_NEXT]] : index
+! NSW: }
+! NSW: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
+! NSW: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.ptr<i32>
 subroutine ido1
   integer, pointer :: iptr
   integer, target :: itgt
@@ -23,9 +38,23 @@ end subroutine
 ! CHECK: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
 ! CHECK: %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
+! CHECK: %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
+! CHECK: fir.result %[[J_VAL_NEXT]] : index
 ! CHECK: }
 ! CHECK: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
 ! CHECK: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32>
+
+! NSW-LABEL: func @_QPido2
+! NSW: %[[J_REF_ADDR:.*]] = fir.alloca !fir.heap<i32> {uniq_name = "_QFido2Eiptr.addr"}
+! NSW: %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.heap<i32>>
+! NSW: %[[J_VAL_FINAL:.*]] = fir.do_loop %[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}} -> index {
+! NSW: %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
+! NSW: fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
+! NSW: %[[J_VAL_NEXT:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
+! NSW: fir.result %[[J_VAL_NEXT]] : index
+! NSW: }
+! NSW: %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]] : (index) -> i32
+! NSW: fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32>
 subroutine ido2
   integer, allocatable :: iptr
   allocate(iptr)
@@ -35,12 +64,32 @@ end subroutine
 ! CHECK-LABEL: func @_QPido3
 ! CHECK:  %[[J_REF_ADDR:.*]] = fir.alloca !fir.heap<i32> {uniq_name = "_QFido3Ej.addr"}
 ! CHECK:  %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.heap<i32>>
-! CHECK:  %[[J_VAL_FINAL:.*]]:2 = fir.iterate_while (%[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}}) and ({{.*}}) -> (index, i1) {
+! CHECK:  %[[J_VAL_FINAL:.*]]:2 = fir.iterate_while (%[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}}) and (%[[OK:.*]] = {{.*}}) -> (index, i1) {
 ! CHECK:    %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
 ! CHECK:    fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
+! CHECK:    %[[RES:.*]] = fir.if %[[OK]] -> (i1) {
+! CHECK:    }
+! CHECK:   %[[J_VAL_INC:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} : index
+! CHECK:   %[[J_VAL_NEXT:.*]] = arith.select %[[RES]], %[[J_VAL_INC]], %[[J_VAL]] : index
+! CHECK:   fir.result %[[J_VAL_NEXT]], %[[RES]] : index, i1
 ! CHECK:  }
 ! CHECK:  %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]]#0 : (index) -> i32
 ! CHECK:  fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32
+
+! NSW-LABEL: func @_QPido3
+! NSW:  %[[J_REF_ADDR:.*]] = fir.alloca !fir.heap<i32> {uniq_name = "_QFido3Ej.addr"}
+! NSW:  %[[J_ADDR:.*]] = fir.load %[[J_REF_ADDR]] : !fir.ref<!fir.heap<i32>>
+! NSW:  %[[J_VAL_FINAL:.*]]:2 = fir.iterate_while (%[[J_VAL:.*]] = %{{.*}} to %{{.*}} step %{{.*}}) and (%[[OK:.*]] = {{.*}}) -> (index, i1) {
+! NSW:    %[[J_VAL_CVT1:.*]] = fir.convert %[[J_VAL]] : (index) -> i32
+! NSW:    fir.store %[[J_VAL_CVT1]] to %[[J_ADDR]] : !fir.heap<i32>
+! NSW:    %[[RES:.*]] = fir.if %[[OK]] -> (i1) {
+! NSW:    }
+! NSW:   %[[J_VAL_INC:.*]] = arith.addi %[[J_VAL]], %{{[^ ]*}} overflow<nsw> : index
+! NSW:   %[[J_VAL_NEXT:.*]] = arith.select %[[RES]], %[[J_VAL_INC]], %[[J_VAL]] : index
+! NSW:   fir.result %[[J_VAL_NEXT]], %[[RES]] : index, i1
+! NSW:  }
+! NSW:  %[[J_VAL_CVT2:.*]] = fir.convert %[[J_VAL_FINAL]]#0 : (index) -> i32
+! NSW:  fir.store %[[J_VAL_CVT2]] to %[[J_ADDR]] : !fir.heap<i32
 subroutine ido3
   integer, allocatable :: j
   allocate(j)
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index f7092d35eeb5..bab21338cef2 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -216,6 +216,11 @@ static llvm::cl::opt<std::string>
                          llvm::cl::desc("Override host target triple"),
                          llvm::cl::init(""));
 
+static llvm::cl::opt<bool>
+    setNSW("integer-overflow",
+           llvm::cl::desc("add nsw flag to internal operations"),
+           llvm::cl::init(false));
+
 #define FLANG_EXCLUDE_CODEGEN
 #include "flang/Tools/CLOptions.inc"
 
@@ -355,6 +360,7 @@ static mlir::LogicalResult convertFortranSourceToMLIR(
   Fortran::lower::LoweringOptions loweringOptions{};
   loweringOptions.setNoPPCNativeVecElemOrder(enableNoPPCNativeVecElemOrder);
   loweringOptions.setLowerToHighLevelFIR(useHLFIR || emitHLFIR);
+  loweringOptions.setNSWOnLoopVarInc(setNSW);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
   auto burnside = Fortran::lower::LoweringBridge::create(
       ctx, semanticsContext, defKinds, semanticsContext.intrinsics(),
@@ -432,6 +438,7 @@ static mlir::LogicalResult convertFortranSourceToMLIR(
 
     // Add O2 optimizer pass pipeline.
     MLIRToLLVMPassPipelineConfig config(llvm::OptimizationLevel::O2);
+    config.NSWOnLoopVarInc = setNSW;
     fir::registerDefaultInlinerPass(config);
     fir::createDefaultFIROptimizerPassPipeline(pm, config);
   }
-- 
GitLab


From 3cc445a6608dc0e88f7d5f16501ef827199cf0c4 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Wed, 15 May 2024 21:40:58 -0700
Subject: [PATCH 0956/1206] [MCAsmParser] Simplify expandMacro

The error checking is only for .macro directives. Move it to the .macro
parser to remove one parameter.
---
 llvm/lib/MC/MCParser/AsmParser.cpp | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 009465d11d78..33287c6529ca 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -295,10 +295,9 @@ private:
 
   void checkForBadMacro(SMLoc DirectiveLoc, StringRef Name, StringRef Body,
                         ArrayRef<MCAsmMacroParameter> Parameters);
-  bool expandMacro(raw_svector_ostream &OS, StringRef Body,
+  bool expandMacro(raw_svector_ostream &OS, const MCAsmMacro &Macro,
                    ArrayRef<MCAsmMacroParameter> Parameters,
-                   ArrayRef<MCAsmMacroArgument> A, bool EnableAtPseudoVariable,
-                   SMLoc L);
+                   ArrayRef<MCAsmMacroArgument> A, bool EnableAtPseudoVariable);
 
   /// Are macros enabled in the parser?
   bool areMacrosEnabled() {return MacrosEnabledFlag;}
@@ -2496,17 +2495,16 @@ static bool isIdentifierChar(char c) {
          c == '.';
 }
 
-bool AsmParser::expandMacro(raw_svector_ostream &OS, StringRef Body,
+bool AsmParser::expandMacro(raw_svector_ostream &OS, const MCAsmMacro &Macro,
                             ArrayRef<MCAsmMacroParameter> Parameters,
                             ArrayRef<MCAsmMacroArgument> A,
-                            bool EnableAtPseudoVariable, SMLoc L) {
+                            bool EnableAtPseudoVariable) {
   unsigned NParameters = Parameters.size();
   bool HasVararg = NParameters ? Parameters.back().Vararg : false;
-  if ((!IsDarwin || NParameters != 0) && NParameters != A.size())
-    return Error(L, "Wrong number of arguments");
 
   // A macro without parameters is handled differently on Darwin:
   // gas accepts no arguments and does no substitutions
+  StringRef Body = Macro.Body;
   while (!Body.empty()) {
     // Scan for the next substitution.
     std::size_t End = Body.size(), Pos = 0;
@@ -2882,10 +2880,11 @@ bool AsmParser::handleMacroEntry(const MCAsmMacro *M, SMLoc NameLoc) {
   // Macro instantiation is lexical, unfortunately. We construct a new buffer
   // to hold the macro body with substitutions.
   SmallString<256> Buf;
-  StringRef Body = M->Body;
   raw_svector_ostream OS(Buf);
 
-  if (expandMacro(OS, Body, M->Parameters, A, true, getTok().getLoc()))
+  if ((!IsDarwin || M->Parameters.size()) && M->Parameters.size() != A.size())
+    return Error(getTok().getLoc(), "Wrong number of arguments");
+  if (expandMacro(OS, *M, M->Parameters, A, true))
     return true;
 
   // We include the .endmacro in the buffer as our cue to exit the macro
@@ -5694,8 +5693,7 @@ bool AsmParser::parseDirectiveRept(SMLoc DirectiveLoc, StringRef Dir) {
   raw_svector_ostream OS(Buf);
   while (Count--) {
     // Note that the AtPseudoVariable is disabled for instantiations of .rep(t).
-    if (expandMacro(OS, M->Body, std::nullopt, std::nullopt, false,
-                    getTok().getLoc()))
+    if (expandMacro(OS, *M, std::nullopt, std::nullopt, false))
       return true;
   }
   instantiateMacroLikeBody(M, DirectiveLoc, OS);
@@ -5726,7 +5724,7 @@ bool AsmParser::parseDirectiveIrp(SMLoc DirectiveLoc) {
   for (const MCAsmMacroArgument &Arg : A) {
     // Note that the AtPseudoVariable is enabled for instantiations of .irp.
     // This is undocumented, but GAS seems to support it.
-    if (expandMacro(OS, M->Body, Parameter, Arg, true, getTok().getLoc()))
+    if (expandMacro(OS, *M, Parameter, Arg, true))
       return true;
   }
 
@@ -5768,7 +5766,7 @@ bool AsmParser::parseDirectiveIrpc(SMLoc DirectiveLoc) {
 
     // Note that the AtPseudoVariable is enabled for instantiations of .irpc.
     // This is undocumented, but GAS seems to support it.
-    if (expandMacro(OS, M->Body, Parameter, Arg, true, getTok().getLoc()))
+    if (expandMacro(OS, *M, Parameter, Arg, true))
       return true;
   }
 
-- 
GitLab


From 245b7b65cb341ac5499fabf62f28fdbbc39bc7d7 Mon Sep 17 00:00:00 2001
From: jiajie zhang <56027356+JumpMasterJJ@users.noreply.github.com>
Date: Thu, 16 May 2024 12:42:01 +0800
Subject: [PATCH 0957/1206] [flang] Add ETIME runtime and lowering intrinsics
 implementation (#90578)

This patch add support of intrinsics GNU extension ETIME
https://github.com/llvm/llvm-project/issues/84205. Some usage info and
example has been added to `flang/docs/Intrinsics.md`. The patch contains
both the lowering and the runtime code and works on both Windows and
Linux.


|   System  |   Implmentation  |
|-----------|--------------------|
| Windows| GetProcessTimes |
| Linux      |times                     |
---
 etime-function.mlir                           | 25 +++++
 flang/docs/Intrinsics.md                      | 52 ++++++++++
 .../flang/Optimizer/Builder/IntrinsicCall.h   | 12 ++-
 .../Optimizer/Builder/Runtime/Intrinsics.h    |  2 +
 flang/include/flang/Runtime/time-intrinsic.h  |  3 +
 flang/lib/Evaluate/intrinsics.cpp             | 26 ++++-
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 99 +++++++++++++++++++
 .../Optimizer/Builder/Runtime/Intrinsics.cpp  | 14 +++
 flang/runtime/time-intrinsic.cpp              | 70 ++++++++++++-
 flang/runtime/tools.h                         |  9 ++
 .../test/Lower/Intrinsics/etime-function.f90  | 24 +++++
 flang/test/Lower/Intrinsics/etime.f90         | 21 ++++
 flang/test/Semantics/etime.f90                | 30 ++++++
 13 files changed, 382 insertions(+), 5 deletions(-)
 create mode 100644 etime-function.mlir
 create mode 100644 flang/test/Lower/Intrinsics/etime-function.f90
 create mode 100644 flang/test/Lower/Intrinsics/etime.f90
 create mode 100644 flang/test/Semantics/etime.f90

diff --git a/etime-function.mlir b/etime-function.mlir
new file mode 100644
index 000000000000..740dfd4866aa
--- /dev/null
+++ b/etime-function.mlir
@@ -0,0 +1,25 @@
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+  func.func @_QPetime_test(%arg0: !fir.ref<!fir.array<2xf32>> {fir.bindc_name = "values"}, %arg1: !fir.ref<f32> {fir.bindc_name = "time"}) {
+    %c9_i32 = arith.constant 9 : i32
+    %c2 = arith.constant 2 : index
+    %0 = fir.alloca f32
+    %1 = fir.declare %arg1 {uniq_name = "_QFetime_testEtime"} : (!fir.ref<f32>) -> !fir.ref<f32>
+    %2 = fir.shape %c2 : (index) -> !fir.shape<1>
+    %3 = fir.declare %arg0(%2) {uniq_name = "_QFetime_testEvalues"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<2xf32>>
+    %4 = fir.embox %3(%2) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
+    %5 = fir.embox %0 : (!fir.ref<f32>) -> !fir.box<f32>
+    %6 = fir.address_of(@_QQclX116781708dcf8f012d7ec1e40d743d97) : !fir.ref<!fir.char<1,71>>
+    %7 = fir.convert %4 : (!fir.box<!fir.array<2xf32>>) -> !fir.box<none>
+    %8 = fir.convert %5 : (!fir.box<f32>) -> !fir.box<none>
+    %9 = fir.convert %6 : (!fir.ref<!fir.char<1,71>>) -> !fir.ref<i8>
+    %10 = fir.call @_FortranAEtime(%7, %8, %9, %c9_i32) fastmath<contract> : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+    %11 = fir.load %0 : !fir.ref<f32>
+    fir.store %11 to %1 : !fir.ref<f32>
+    return
+  }
+  func.func private @_FortranAEtime(!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none attributes {fir.runtime}
+  fir.global linkonce @_QQclX116781708dcf8f012d7ec1e40d743d97 constant : !fir.char<1,71> {
+    %0 = fir.string_lit "/home/jump/llvm-project/flang/test/Lower/Intrinsics/etime-function.f90\00"(71) : !fir.char<1,71>
+    fir.has_value %0 : !fir.char<1,71>
+  }
+}
diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index 848619cb65d9..41129b10083b 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -916,3 +916,55 @@ used in constant expressions have currently no folding support at all.
   - If a condition occurs that would assign a nonzero value to `CMDSTAT` but the `CMDSTAT` variable is not present, error termination is initiated.
     - On POSIX-compatible systems, the child process (async process) will be terminated with no effect on the parent process (continues).
     - On Windows, error termination is not initiated.
+
+### Non-Standard Intrinsics: ETIME
+
+#### Description
+`ETIME(VALUES, TIME)` returns the number of seconds of runtime since the start of the process’s execution in *TIME*. *VALUES* returns the user and system components of this time in `VALUES(1)` and `VALUES(2)` respectively. *TIME* is equal to `VALUES(1) + VALUES(2)`.
+
+On some systems, the underlying timings are represented using types with sufficiently small limits that overflows (wrap around) are possible, such as 32-bit types. Therefore, the values returned by this intrinsic might be, or become, negative, or numerically less than previous values, during a single run of the compiled program.
+
+This intrinsic is provided in both subroutine and function forms; however, only one form can be used in any given program unit.
+
+*VALUES* and *TIME* are `INTENT(OUT)` and provide the following:
+
+
+|               |                                   |
+|---------------|-----------------------------------|
+| `VALUES(1)`   | User time in seconds.             |
+| `VALUES(2)`   | System time in seconds.           |
+| `TIME`        | Run time since start in seconds.  |
+
+#### Usage and Info
+
+- **Standard:** GNU extension
+- **Class:** Subroutine, function
+- **Syntax:** `CALL ETIME(VALUES, TIME)`
+- **Arguments:**
+- **Return value** Elapsed time in seconds since the start of program execution.
+
+| Argument   | Description                                                           |
+|------------|-----------------------------------------------------------------------|
+| `VALUES`   | The type shall be REAL(4), DIMENSION(2).                              |
+| `TIME`     | The type shall be REAL(4).                                            |
+
+#### Example
+Here is an example usage from [Gfortran ETIME](https://gcc.gnu.org/onlinedocs/gfortran/ETIME.html)
+```Fortran
+program test_etime
+    integer(8) :: i, j
+    real, dimension(2) :: tarray
+    real :: result
+    call ETIME(tarray, result)
+    print *, result
+    print *, tarray(1)
+    print *, tarray(2)   
+    do i=1,100000000    ! Just a delay
+        j = i * i - i
+    end do
+    call ETIME(tarray, result)
+    print *, result
+    print *, tarray(1)
+    print *, tarray(2)
+end program test_etime
+```
\ No newline at end of file
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index b7d060926761..977a69af5281 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -222,6 +222,8 @@ struct IntrinsicLibrary {
   fir::ExtendedValue genEoshift(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   void genExit(llvm::ArrayRef<fir::ExtendedValue>);
   void genExecuteCommandLine(mlir::ArrayRef<fir::ExtendedValue> args);
+  fir::ExtendedValue genEtime(std::optional<mlir::Type>,
+                              mlir::ArrayRef<fir::ExtendedValue> args);
   mlir::Value genExponent(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genExtendsTypeOf(mlir::Type,
                                       llvm::ArrayRef<fir::ExtendedValue>);
@@ -400,8 +402,10 @@ struct IntrinsicLibrary {
   using ElementalGenerator = decltype(&IntrinsicLibrary::genAbs);
   using ExtendedGenerator = decltype(&IntrinsicLibrary::genLenTrim);
   using SubroutineGenerator = decltype(&IntrinsicLibrary::genDateAndTime);
-  using Generator =
-      std::variant<ElementalGenerator, ExtendedGenerator, SubroutineGenerator>;
+  /// The generator for intrinsic that has both function and subroutine form.
+  using DualGenerator = decltype(&IntrinsicLibrary::genEtime);
+  using Generator = std::variant<ElementalGenerator, ExtendedGenerator,
+                                 SubroutineGenerator, DualGenerator>;
 
   /// All generators can be outlined. This will build a function named
   /// "fir."+ <generic name> + "." + <result type code> and generate the
@@ -442,6 +446,10 @@ struct IntrinsicLibrary {
                               llvm::ArrayRef<mlir::Value> args);
   mlir::Value invokeGenerator(SubroutineGenerator generator,
                               llvm::ArrayRef<mlir::Value> args);
+  mlir::Value invokeGenerator(DualGenerator generator,
+                              llvm::ArrayRef<mlir::Value> args);
+  mlir::Value invokeGenerator(DualGenerator generator, mlir::Type resultType,
+                              llvm::ArrayRef<mlir::Value> args);
 
   /// Get pointer to unrestricted intrinsic. Generate the related unrestricted
   /// intrinsic if it is not defined yet.
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
index 737c631e45c1..7497a4bc3564 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
@@ -44,6 +44,8 @@ void genDateAndTime(fir::FirOpBuilder &, mlir::Location,
                     std::optional<fir::CharBoxValue> date,
                     std::optional<fir::CharBoxValue> time,
                     std::optional<fir::CharBoxValue> zone, mlir::Value values);
+void genEtime(fir::FirOpBuilder &builder, mlir::Location loc,
+              mlir::Value values, mlir::Value time);
 
 void genRandomInit(fir::FirOpBuilder &, mlir::Location, mlir::Value repeatable,
                    mlir::Value imageDistinct);
diff --git a/flang/include/flang/Runtime/time-intrinsic.h b/flang/include/flang/Runtime/time-intrinsic.h
index 650c02436ee4..80490a17e455 100644
--- a/flang/include/flang/Runtime/time-intrinsic.h
+++ b/flang/include/flang/Runtime/time-intrinsic.h
@@ -43,6 +43,9 @@ void RTNAME(DateAndTime)(char *date, std::size_t dateChars, char *time,
     const char *source = nullptr, int line = 0,
     const Descriptor *values = nullptr);
 
+void RTNAME(Etime)(const Descriptor *values, const Descriptor *time,
+    const char *sourceFile, int line);
+
 } // extern "C"
 } // namespace Fortran::runtime
 #endif // FORTRAN_RUNTIME_TIME_INTRINSIC_H_
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 441a762c930d..ded277877f49 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -454,6 +454,10 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"erf", {{"x", SameReal}}, SameReal},
     {"erfc", {{"x", SameReal}}, SameReal},
     {"erfc_scaled", {{"x", SameReal}}, SameReal},
+    {"etime",
+        {{"values", TypePattern{RealType, KindCode::exactKind, 4}, Rank::vector,
+            Optionality::required, common::Intent::Out}},
+        TypePattern{RealType, KindCode::exactKind, 4}},
     {"exp", {{"x", SameFloating}}, SameFloating},
     {"exp", {{"x", SameFloating}}, SameFloating},
     {"exponent", {{"x", AnyReal}}, DefaultInt},
@@ -1342,6 +1346,12 @@ static const IntrinsicInterface intrinsicSubroutine[]{
             {"values", AnyInt, Rank::vector, Optionality::optional,
                 common::Intent::Out}},
         {}, Rank::elemental, IntrinsicClass::impureSubroutine},
+    {"etime",
+        {{"values", TypePattern{RealType, KindCode::exactKind, 4}, Rank::vector,
+             Optionality::required, common::Intent::Out},
+            {"time", TypePattern{RealType, KindCode::exactKind, 4},
+                Rank::scalar, Optionality::required, common::Intent::Out}},
+        {}, Rank::elemental, IntrinsicClass::impureSubroutine},
     {"execute_command_line",
         {{"command", DefaultChar, Rank::scalar},
             {"wait", AnyLogical, Rank::scalar, Optionality::optional},
@@ -2484,6 +2494,7 @@ public:
   bool IsIntrinsic(const std::string &) const;
   bool IsIntrinsicFunction(const std::string &) const;
   bool IsIntrinsicSubroutine(const std::string &) const;
+  bool IsDualIntrinsic(const std::string &) const;
 
   IntrinsicClass GetIntrinsicClass(const std::string &) const;
   std::string GetGenericIntrinsicName(const std::string &) const;
@@ -2545,6 +2556,17 @@ bool IntrinsicProcTable::Implementation::IsIntrinsic(
     const std::string &name) const {
   return IsIntrinsicFunction(name) || IsIntrinsicSubroutine(name);
 }
+bool IntrinsicProcTable::Implementation::IsDualIntrinsic(
+    const std::string &name) const {
+  // Collection for some intrinsics with function and subroutine form,
+  // in order to pass the semantic check.
+  static const std::string dualIntrinsic[]{{"etime"}};
+
+  return std::find_if(std::begin(dualIntrinsic), std::end(dualIntrinsic),
+             [&name](const std::string &dualName) {
+               return dualName == name;
+             }) != std::end(dualIntrinsic);
+}
 
 IntrinsicClass IntrinsicProcTable::Implementation::GetIntrinsicClass(
     const std::string &name) const {
@@ -3083,7 +3105,7 @@ std::optional<SpecificCall> IntrinsicProcTable::Implementation::Probe(
         return specificCall;
       }
     }
-    if (IsIntrinsicFunction(call.name)) {
+    if (IsIntrinsicFunction(call.name) && !IsDualIntrinsic(call.name)) {
       context.messages().Say(
           "Cannot use intrinsic function '%s' as a subroutine"_err_en_US,
           call.name);
@@ -3218,7 +3240,7 @@ std::optional<SpecificCall> IntrinsicProcTable::Implementation::Probe(
   }
 
   if (specificBuffer.empty() && genericBuffer.empty() &&
-      IsIntrinsicSubroutine(call.name)) {
+      IsIntrinsicSubroutine(call.name) && !IsDualIntrinsic(call.name)) {
     context.messages().Say(
         "Cannot use intrinsic subroutine '%s' as a function"_err_en_US,
         call.name);
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 58064d23eb08..ae7e65098744 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -35,6 +35,7 @@
 #include "flang/Optimizer/Builder/Runtime/Stop.h"
 #include "flang/Optimizer/Builder/Runtime/Transformational.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Support/FatalError.h"
@@ -49,6 +50,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include <mlir/IR/Value.h>
 #include <optional>
 
 #define DEBUG_TYPE "flang-lower-intrinsic"
@@ -222,6 +224,10 @@ static constexpr IntrinsicHandler handlers[]{
        {"boundary", asBox, handleDynamicOptional},
        {"dim", asValue}}},
      /*isElemental=*/false},
+    {"etime",
+     &I::genEtime,
+     {{{"values", asBox}, {"time", asBox}}},
+     /*isElemental=*/false},
     {"execute_command_line",
      &I::genExecuteCommandLine,
      {{{"command", asBox},
@@ -1682,6 +1688,24 @@ IntrinsicLibrary::genElementalCall<IntrinsicLibrary::SubroutineGenerator>(
   return mlir::Value();
 }
 
+template <>
+fir::ExtendedValue
+IntrinsicLibrary::genElementalCall<IntrinsicLibrary::DualGenerator>(
+    DualGenerator generator, llvm::StringRef name, mlir::Type resultType,
+    llvm::ArrayRef<fir::ExtendedValue> args, bool outline) {
+  assert(resultType.getImpl() && "expect elemental intrinsic to be functions");
+
+  for (const fir::ExtendedValue &arg : args)
+    if (!arg.getUnboxed() && !arg.getCharBox())
+      // fir::emitFatalError(loc, "nonscalar intrinsic argument");
+      crashOnMissingIntrinsic(loc, name);
+  if (outline)
+    return outlineInExtendedWrapper(generator, name, resultType, args);
+
+  return std::invoke(generator, *this, std::optional<mlir::Type>{resultType},
+                     args);
+}
+
 static fir::ExtendedValue
 invokeHandler(IntrinsicLibrary::ElementalGenerator generator,
               const IntrinsicHandler &handler,
@@ -1725,6 +1749,22 @@ invokeHandler(IntrinsicLibrary::SubroutineGenerator generator,
   return mlir::Value{};
 }
 
+static fir::ExtendedValue
+invokeHandler(IntrinsicLibrary::DualGenerator generator,
+              const IntrinsicHandler &handler,
+              std::optional<mlir::Type> resultType,
+              llvm::ArrayRef<fir::ExtendedValue> args, bool outline,
+              IntrinsicLibrary &lib) {
+  if (handler.isElemental)
+    return lib.genElementalCall(generator, handler.name, mlir::Type{}, args,
+                                outline);
+  if (outline)
+    return lib.outlineInExtendedWrapper(generator, handler.name, resultType,
+                                        args);
+
+  return std::invoke(generator, lib, resultType, args);
+}
+
 std::pair<fir::ExtendedValue, bool>
 IntrinsicLibrary::genIntrinsicCall(llvm::StringRef specificName,
                                    std::optional<mlir::Type> resultType,
@@ -1820,6 +1860,34 @@ IntrinsicLibrary::invokeGenerator(SubroutineGenerator generator,
   return {};
 }
 
+mlir::Value
+IntrinsicLibrary::invokeGenerator(DualGenerator generator,
+                                  llvm::ArrayRef<mlir::Value> args) {
+  llvm::SmallVector<fir::ExtendedValue> extendedArgs;
+  for (mlir::Value arg : args)
+    extendedArgs.emplace_back(toExtendedValue(arg, builder, loc));
+  std::invoke(generator, *this, std::optional<mlir::Type>{}, extendedArgs);
+  return {};
+}
+
+mlir::Value
+IntrinsicLibrary::invokeGenerator(DualGenerator generator,
+                                  mlir::Type resultType,
+                                  llvm::ArrayRef<mlir::Value> args) {
+  llvm::SmallVector<fir::ExtendedValue> extendedArgs;
+  for (mlir::Value arg : args)
+    extendedArgs.emplace_back(toExtendedValue(arg, builder, loc));
+
+  if (resultType.getImpl() == nullptr) {
+    // TODO:
+    assert(false && "result type is null");
+  }
+
+  auto extendedResult = std::invoke(
+      generator, *this, std::optional<mlir::Type>{resultType}, extendedArgs);
+  return toValue(extendedResult, builder, loc);
+}
+
 //===----------------------------------------------------------------------===//
 // Intrinsic Procedure Mangling
 //===----------------------------------------------------------------------===//
@@ -3235,6 +3303,37 @@ void IntrinsicLibrary::genExecuteCommandLine(
                                       exitstatBox, cmdstatBox, cmdmsgBox);
 }
 
+// ETIME
+fir::ExtendedValue
+IntrinsicLibrary::genEtime(std::optional<mlir::Type> resultType,
+                           llvm::ArrayRef<fir::ExtendedValue> args) {
+  assert((args.size() == 2 && !resultType.has_value()) ||
+         (args.size() == 1 && resultType.has_value()));
+
+  mlir::Value values = fir::getBase(args[0]);
+  if (resultType.has_value()) {
+    // function form
+    if (!values)
+      fir::emitFatalError(loc, "expected VALUES parameter");
+
+    auto timeAddr = builder.createTemporary(loc, *resultType);
+    auto timeBox = builder.createBox(loc, timeAddr);
+    fir::runtime::genEtime(builder, loc, values, timeBox);
+    return builder.create<fir::LoadOp>(loc, timeAddr);
+  } else {
+    // subroutine form
+    mlir::Value time = fir::getBase(args[1]);
+    if (!values)
+      fir::emitFatalError(loc, "expected VALUES parameter");
+    if (!time)
+      fir::emitFatalError(loc, "expected TIME parameter");
+
+    fir::runtime::genEtime(builder, loc, values, time);
+    return {};
+  }
+  return {};
+}
+
 // EXIT
 void IntrinsicLibrary::genExit(llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 1);
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 8b78a1688c73..3f36d639861b 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -106,6 +106,20 @@ void fir::runtime::genDateAndTime(fir::FirOpBuilder &builder,
   builder.create<fir::CallOp>(loc, callee, args);
 }
 
+void fir::runtime::genEtime(fir::FirOpBuilder &builder, mlir::Location loc,
+                            mlir::Value values, mlir::Value time) {
+  auto runtimeFunc = fir::runtime::getRuntimeFunc<mkRTKey(Etime)>(loc, builder);
+  mlir::FunctionType runtimeFuncTy = runtimeFunc.getFunctionType();
+
+  mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
+  mlir::Value sourceLine =
+      fir::factory::locationToLineNo(builder, loc, runtimeFuncTy.getInput(3));
+
+  llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
+      builder, loc, runtimeFuncTy, values, time, sourceFile, sourceLine);
+  builder.create<fir::CallOp>(loc, runtimeFunc, args);
+}
+
 void fir::runtime::genRandomInit(fir::FirOpBuilder &builder, mlir::Location loc,
                                  mlir::Value repeatable,
                                  mlir::Value imageDistinct) {
diff --git a/flang/runtime/time-intrinsic.cpp b/flang/runtime/time-intrinsic.cpp
index 68d63253139f..989d4f804c5f 100644
--- a/flang/runtime/time-intrinsic.cpp
+++ b/flang/runtime/time-intrinsic.cpp
@@ -19,8 +19,12 @@
 #include <cstdlib>
 #include <cstring>
 #include <ctime>
-#ifndef _WIN32
+#ifdef _WIN32
+#include "flang/Common/windows-include.h"
+#else
 #include <sys/time.h> // gettimeofday
+#include <sys/times.h>
+#include <unistd.h>
 #endif
 
 // CPU_TIME (Fortran 2018 16.9.57)
@@ -370,5 +374,69 @@ void RTNAME(DateAndTime)(char *date, std::size_t dateChars, char *time,
       terminator, date, dateChars, time, timeChars, zone, zoneChars, values);
 }
 
+void RTNAME(Etime)(const Descriptor *values, const Descriptor *time,
+    const char *sourceFile, int line) {
+  Fortran::runtime::Terminator terminator{sourceFile, line};
+
+  double usrTime = -1.0, sysTime = -1.0, realTime = -1.0;
+
+#ifdef _WIN32
+  FILETIME creationTime;
+  FILETIME exitTime;
+  FILETIME kernelTime;
+  FILETIME userTime;
+
+  if (GetProcessTimes(GetCurrentProcess(), &creationTime, &exitTime,
+          &kernelTime, &userTime) == 0) {
+    ULARGE_INTEGER userSystemTime;
+    ULARGE_INTEGER kernelSystemTime;
+
+    memcpy(&userSystemTime, &userTime, sizeof(FILETIME));
+    memcpy(&kernelSystemTime, &kernelTime, sizeof(FILETIME));
+
+    usrTime = ((double)(userSystemTime.QuadPart)) / 10000000.0;
+    sysTime = ((double)(kernelSystemTime.QuadPart)) / 10000000.0;
+    realTime = usrTime + sysTime;
+  }
+#else
+  struct tms tms;
+  if (times(&tms) != -1) {
+    usrTime = ((double)(tms.tms_utime)) / sysconf(_SC_CLK_TCK);
+    sysTime = ((double)(tms.tms_stime)) / sysconf(_SC_CLK_TCK);
+    realTime = usrTime + sysTime;
+  }
+#endif
+
+  if (values) {
+    auto typeCode{values->type().GetCategoryAndKind()};
+    // ETIME values argument must have decimal range == 2.
+    RUNTIME_CHECK(terminator,
+        values->rank() == 1 && values->GetDimension(0).Extent() == 2 &&
+            typeCode && typeCode->first == Fortran::common::TypeCategory::Real);
+    // Only accept KIND=4 here.
+    int kind{typeCode->second};
+    RUNTIME_CHECK(terminator, kind == 4);
+
+    ApplyFloatingPointKind<StoreFloatingPointAt, void>(
+        kind, terminator, *values, /* atIndex = */ 0, usrTime);
+    ApplyFloatingPointKind<StoreFloatingPointAt, void>(
+        kind, terminator, *values, /* atIndex = */ 1, sysTime);
+  }
+
+  if (time) {
+    auto typeCode{time->type().GetCategoryAndKind()};
+    // ETIME time argument must have decimal range == 0.
+    RUNTIME_CHECK(terminator,
+        time->rank() == 0 && typeCode &&
+            typeCode->first == Fortran::common::TypeCategory::Real);
+    // Only accept KIND=4 here.
+    int kind{typeCode->second};
+    RUNTIME_CHECK(terminator, kind == 4);
+
+    ApplyFloatingPointKind<StoreFloatingPointAt, void>(
+        kind, terminator, *time, /* atIndex = */ 0, realTime);
+  }
+}
+
 } // extern "C"
 } // namespace Fortran::runtime
diff --git a/flang/runtime/tools.h b/flang/runtime/tools.h
index 52049c511f13..dc12e5c4533e 100644
--- a/flang/runtime/tools.h
+++ b/flang/runtime/tools.h
@@ -99,6 +99,15 @@ template <int KIND> struct StoreIntegerAt {
   }
 };
 
+// Helper to store floating value in result[at].
+template <int KIND> struct StoreFloatingPointAt {
+  RT_API_ATTRS void operator()(const Fortran::runtime::Descriptor &result,
+      std::size_t at, std::double_t value) const {
+    *result.ZeroBasedIndexedElement<Fortran::runtime::CppTypeFor<
+        Fortran::common::TypeCategory::Real, KIND>>(at) = value;
+  }
+};
+
 // Validate a KIND= argument
 RT_API_ATTRS void CheckIntegerKind(
     Terminator &, int kind, const char *intrinsic);
diff --git a/flang/test/Lower/Intrinsics/etime-function.f90 b/flang/test/Lower/Intrinsics/etime-function.f90
new file mode 100644
index 000000000000..c47d509af535
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/etime-function.f90
@@ -0,0 +1,24 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+
+! CHECK-LABEL: func.func @_QPetime_test(
+! CHECK-SAME: %[[valuesArg:.*]]: !fir.ref<!fir.array<2xf32>> {fir.bindc_name = "values"},
+! CHECK-SAME: %[[timeArg:.*]]: !fir.ref<f32> {fir.bindc_name = "time"}) {
+subroutine etime_test(values, time)
+  REAL(4), DIMENSION(2) :: values
+  REAL(4) :: time
+  time = etime(values)
+  ! CHECK-NEXT:        %[[c9:.*]] = arith.constant 9 : i32
+  ! CHECK-NEXT:        %[[c2:.*]] = arith.constant 2 : index
+  ! CHECK-NEXT:        %[[timeTmpAddr:.*]] = fir.alloca f32
+  ! CHECK-NEXT:        %[[timeDeclare:.*]] = fir.declare %[[timeArg]] {uniq_name = "_QFetime_testEtime"} : (!fir.ref<f32>) -> !fir.ref<f32>
+  ! CHECK-NEXT:        %[[shape:.*]] = fir.shape %[[c2]] : (index) -> !fir.shape<1>
+  ! CHECK-NEXT:        %[[valuesDeclare:.*]] = fir.declare %[[valuesArg]](%[[shape]]) {uniq_name = "_QFetime_testEvalues"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<2xf32>>
+  ! CHECK-NEXT:        %[[valuesBox:.*]] = fir.embox %[[valuesDeclare]](%[[shape]]) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
+  ! CHECK-NEXT:        %[[timeTmpBox:.*]] = fir.embox %[[timeTmpAddr]] : (!fir.ref<f32>) -> !fir.box<f32>
+  ! CHECK:             %[[values:.*]] = fir.convert %[[valuesBox]] : (!fir.box<!fir.array<2xf32>>) -> !fir.box<none>
+  ! CHECK:             %[[timeTmp:.*]] = fir.convert %[[timeTmpBox]] : (!fir.box<f32>) -> !fir.box<none>
+  ! CHECK:             %[[VAL_9:.*]] = fir.call @_FortranAEtime(%[[values]], %[[timeTmp]], %[[VAL_7:.*]], %[[c9]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+  ! CHECK-NEXT:        %[[timeValue:.*]] = fir.load %[[timeTmpAddr]] : !fir.ref<f32>
+  ! CHECK-NEXT:        fir.store %[[timeValue]] to %[[timeDeclare]] : !fir.ref<f32>
+  ! CHECK-NEXT:        return
+end subroutine etime_test
\ No newline at end of file
diff --git a/flang/test/Lower/Intrinsics/etime.f90 b/flang/test/Lower/Intrinsics/etime.f90
new file mode 100644
index 000000000000..e5e7984a340c
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/etime.f90
@@ -0,0 +1,21 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+
+! CHECK-LABEL: func.func @_QPetime_test(
+! CHECK-SAME: %[[valuesArg:.*]]: !fir.ref<!fir.array<2xf32>> {fir.bindc_name = "values"},
+! CHECK-SAME: %[[timeArg:.*]]: !fir.ref<f32> {fir.bindc_name = "time"}) {
+subroutine etime_test(values, time)
+  REAL(4), DIMENSION(2) :: values
+  REAL(4) :: time
+  call etime(values, time)
+  ! CHECK-NEXT:        %[[c9:.*]] = arith.constant 9 : i32
+  ! CHECK-NEXT:        %[[c2:.*]] = arith.constant 2 : index
+  ! CHECK-NEXT:        %[[timeDeclare:.*]] = fir.declare %[[timeArg]] {uniq_name = "_QFetime_testEtime"} : (!fir.ref<f32>) -> !fir.ref<f32>
+  ! CHECK-NEXT:        %[[shape:.*]] = fir.shape %[[c2]] : (index) -> !fir.shape<1>
+  ! CHECK-NEXT:        %[[valuesDeclare:.*]] = fir.declare %[[valuesArg]](%[[shape]]) {uniq_name = "_QFetime_testEvalues"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<2xf32>>
+  ! CHECK-NEXT:        %[[valuesBox:.*]] = fir.embox %[[valuesDeclare]](%[[shape]]) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
+  ! CHECK-NEXT:        %[[timeBox:.*]] = fir.embox %[[timeDeclare]] : (!fir.ref<f32>) -> !fir.box<f32>
+  ! CHECK:             %[[values:.*]] = fir.convert %[[valuesBox]] : (!fir.box<!fir.array<2xf32>>) -> !fir.box<none>
+  ! CHECK:             %[[time:.*]] = fir.convert %[[timeBox]] : (!fir.box<f32>) -> !fir.box<none>
+  ! CHECK:             %[[VAL_9:.*]] = fir.call @_FortranAEtime(%[[values]], %[[time]], %[[VAL_7:.*]], %[[c9]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+  ! CHECK-NEXT:        return
+end subroutine etime_test
\ No newline at end of file
diff --git a/flang/test/Semantics/etime.f90 b/flang/test/Semantics/etime.f90
new file mode 100644
index 000000000000..28735c2a7aac
--- /dev/null
+++ b/flang/test/Semantics/etime.f90
@@ -0,0 +1,30 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
+! Tests for the ETIME intrinsics
+
+subroutine bad_kind_error(values, time)
+  REAL(KIND=8), DIMENSION(2) :: values
+  REAL(KIND=8) :: time
+  !ERROR: Actual argument for 'values=' has bad type or kind 'REAL(8)'
+  call etime(values, time)
+end subroutine bad_kind_error
+  
+subroutine bad_args_error(values)
+  REAL(KIND=4), DIMENSION(2) :: values
+  !ERROR: missing mandatory 'time=' argument
+  call etime(values)
+end subroutine bad_args_error
+
+subroutine bad_apply_form(values)
+  REAL(KIND=4), DIMENSION(2) :: values
+  REAL(KIND=4) :: time
+  !Declaration of 'etime'
+  call etime(values, time)
+  !ERROR: Cannot call subroutine 'etime' like a function
+  time = etime(values)
+end subroutine bad_apply_form
+
+subroutine good_kind_equal(values, time)
+  REAL(KIND=4), DIMENSION(2) :: values
+  REAL(KIND=4) :: time
+  call etime(values, time)
+end subroutine good_kind_equal
\ No newline at end of file
-- 
GitLab


From f2d74002fdad2171b62392eaedf38aac7e4fb50d Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 15 May 2024 21:46:31 -0700
Subject: [PATCH 0958/1206] [LegalizeVectorOps][X86] Add ISD::ABDS/ABSDU to the
 list of opcodes handled by LegalizeVectorOps. (#92332)

The expand code is present, but we were missing the type query code
so the nodes would be ignored until LegalizeDAG.
---
 .../SelectionDAG/LegalizeVectorOps.cpp        |  2 ++
 llvm/test/CodeGen/X86/midpoint-int-vec-128.ll | 32 +++++++++----------
 llvm/test/CodeGen/X86/midpoint-int-vec-256.ll | 32 +++++++++----------
 3 files changed, 34 insertions(+), 32 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 423df9ae6b2a..6acbc044d673 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -367,6 +367,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::ROTL:
   case ISD::ROTR:
   case ISD::ABS:
+  case ISD::ABDS:
+  case ISD::ABDU:
   case ISD::BSWAP:
   case ISD::BITREVERSE:
   case ISD::CTLZ:
diff --git a/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll b/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
index 5a6375e08bca..c6e8b7532505 100644
--- a/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
@@ -1991,14 +1991,14 @@ define <8 x i16> @vec128_i16_unsigned_reg_reg(<8 x i16> %a1, <8 x i16> %a2) noun
 ;
 ; AVX512VL-FALLBACK-LABEL: vec128_i16_unsigned_reg_reg:
 ; AVX512VL-FALLBACK:       # %bb.0:
-; AVX512VL-FALLBACK-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm2
-; AVX512VL-FALLBACK-NEXT:    vpminuw %xmm1, %xmm0, %xmm1
-; AVX512VL-FALLBACK-NEXT:    vpsubw %xmm1, %xmm2, %xmm2
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %xmm2, %xmm2
-; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %xmm1, %xmm1, %xmm1
-; AVX512VL-FALLBACK-NEXT:    vpxor %xmm1, %xmm2, %xmm2
-; AVX512VL-FALLBACK-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512VL-FALLBACK-NEXT:    vpminuw %xmm1, %xmm0, %xmm2
+; AVX512VL-FALLBACK-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm1
+; AVX512VL-FALLBACK-NEXT:    vpsubw %xmm2, %xmm1, %xmm1
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %xmm1, %xmm1
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %xmm2, %xmm2, %xmm2
+; AVX512VL-FALLBACK-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; AVX512VL-FALLBACK-NEXT:    vpsubw %xmm2, %xmm1, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddw %xmm0, %xmm1, %xmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
@@ -2784,14 +2784,14 @@ define <16 x i8> @vec128_i8_unsigned_reg_reg(<16 x i8> %a1, <16 x i8> %a2) nounw
 ;
 ; AVX512VL-FALLBACK-LABEL: vec128_i8_unsigned_reg_reg:
 ; AVX512VL-FALLBACK:       # %bb.0:
-; AVX512VL-FALLBACK-NEXT:    vpmaxub %xmm1, %xmm0, %xmm2
-; AVX512VL-FALLBACK-NEXT:    vpminub %xmm1, %xmm0, %xmm1
-; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %xmm2, %xmm2
-; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %xmm1, %xmm1, %xmm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm2
-; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; AVX512VL-FALLBACK-NEXT:    vpminub %xmm1, %xmm0, %xmm2
+; AVX512VL-FALLBACK-NEXT:    vpmaxub %xmm1, %xmm0, %xmm1
+; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %xmm1, %xmm1
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %xmm2, %xmm2, %xmm2
+; AVX512VL-FALLBACK-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm1
+; AVX512VL-FALLBACK-NEXT:    vpsubb %xmm2, %xmm1, %xmm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll b/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
index e880a1acc9e8..cc08396ae8c7 100644
--- a/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int-vec-256.ll
@@ -1445,14 +1445,14 @@ define <16 x i16> @vec256_i16_unsigned_reg_reg(<16 x i16> %a1, <16 x i16> %a2) n
 ;
 ; AVX512VL-FALLBACK-LABEL: vec256_i16_unsigned_reg_reg:
 ; AVX512VL-FALLBACK:       # %bb.0:
-; AVX512VL-FALLBACK-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpminuw %ymm1, %ymm0, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm1, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm1, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpxor %ymm1, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm1, %ymm2, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpminuw %ymm1, %ymm0, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm2, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpxor %ymm2, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsubw %ymm2, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
@@ -2210,14 +2210,14 @@ define <32 x i8> @vec256_i8_unsigned_reg_reg(<32 x i8> %a1, <32 x i8> %a2) nounw
 ;
 ; AVX512VL-FALLBACK-LABEL: vec256_i8_unsigned_reg_reg:
 ; AVX512VL-FALLBACK:       # %bb.0:
-; AVX512VL-FALLBACK-NEXT:    vpmaxub %ymm1, %ymm0, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpminub %ymm1, %ymm0, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm1, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm2, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm1, %ymm1, %ymm1
-; AVX512VL-FALLBACK-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm2
-; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm1, %ymm2, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpminub %ymm1, %ymm0, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpmaxub %ymm1, %ymm0, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsrlw $1, %ymm1, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpternlogq $15, %ymm2, %ymm2, %ymm2
+; AVX512VL-FALLBACK-NEXT:    vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm1
+; AVX512VL-FALLBACK-NEXT:    vpsubb %ymm2, %ymm1, %ymm1
 ; AVX512VL-FALLBACK-NEXT:    vpaddb %ymm0, %ymm1, %ymm0
 ; AVX512VL-FALLBACK-NEXT:    retq
 ;
-- 
GitLab


From 487b43cdc9fff9e370b8ea948c0cc19ca817aa86 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 15 May 2024 21:47:29 -0700
Subject: [PATCH 0959/1206] [RISCV] Pass subvector type to
 isLegalInterleavedAccessType in getInterleavedMemoryOpCost. (#91825)

isLegalInterleavedAccessType expects the subvector type, but
getInterleavedMemoryOpCost is called with the full vector type. So we
need to divide by Factor.
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  19 +-
 .../RISCV/interleaved-accesses.ll             | 566 ++++++++----------
 .../LoopVectorize/RISCV/interleaved-cost.ll   |   4 +-
 3 files changed, 259 insertions(+), 330 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 4d2479fc233f..b73ed208ed74 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -613,14 +613,19 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
     std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(VTy);
     // Need to make sure type has't been scalarized
     if (LT.second.isVector()) {
-      auto *LegalVTy = VectorType::get(VTy->getElementType(),
-                                       LT.second.getVectorElementCount());
-      // FIXME: We use the memory op cost of the *legalized* type here, becuase
-      // it's getMemoryOpCost returns a really expensive cost for types like
-      // <6 x i8>, which show up when doing interleaves of Factor=3 etc.
-      // Should the memory op cost of these be cheaper?
-      if (TLI->isLegalInterleavedAccessType(LegalVTy, Factor, Alignment,
+      auto *SubVecTy =
+          VectorType::get(VTy->getElementType(),
+                          VTy->getElementCount().divideCoefficientBy(Factor));
+
+      if (VTy->getElementCount().isKnownMultipleOf(Factor) &&
+          TLI->isLegalInterleavedAccessType(SubVecTy, Factor, Alignment,
                                             AddressSpace, DL)) {
+        // FIXME: We use the memory op cost of the *legalized* type here,
+        // because it's getMemoryOpCost returns a really expensive cost for
+        // types like <6 x i8>, which show up when doing interleaves of
+        // Factor=3 etc. Should the memory op cost of these be cheaper?
+        auto *LegalVTy = VectorType::get(VTy->getElementType(),
+                                         LT.second.getVectorElementCount());
         InstructionCost LegalMemCost = getMemoryOpCost(
             Opcode, LegalVTy, Alignment, AddressSpace, CostKind);
         return LT.first + LegalMemCost;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index 576dc0833fa3..87bc77cb7767 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -393,23 +393,23 @@ define void @load_store_factor3_i32(ptr %p) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[TMP2]], i32 0
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <6 x i32>, ptr [[TMP3]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <6 x i32> [[WIDE_VEC]], <6 x i32> poison, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <6 x i32> [[WIDE_VEC]], <6 x i32> poison, <2 x i32> <i32 1, i32 4>
-; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <6 x i32> [[WIDE_VEC]], <6 x i32> poison, <2 x i32> <i32 2, i32 5>
-; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i32> [[STRIDED_VEC]], <i32 1, i32 1>
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i32> [[STRIDED_VEC1]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[STRIDED_VEC1]], <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP5]], 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add <2 x i32> [[STRIDED_VEC2]], <i32 3, i32 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[STRIDED_VEC2]], <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[TMP8]], i32 -2
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> [[TMP12]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <6 x i32> [[TMP13]], <6 x i32> poison, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
-; CHECK-NEXT:    store <6 x i32> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP13]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; CHECK-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
@@ -451,23 +451,23 @@ define void @load_store_factor3_i32(ptr %p) {
 ; FIXED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
 ; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP1]]
 ; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[TMP2]], i32 0
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <6 x i32>, ptr [[TMP3]], align 4
-; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <6 x i32> [[WIDE_VEC]], <6 x i32> poison, <2 x i32> <i32 0, i32 3>
-; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <6 x i32> [[WIDE_VEC]], <6 x i32> poison, <2 x i32> <i32 1, i32 4>
-; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <6 x i32> [[WIDE_VEC]], <6 x i32> poison, <2 x i32> <i32 2, i32 5>
-; FIXED-NEXT:    [[TMP4:%.*]] = add <2 x i32> [[STRIDED_VEC]], <i32 1, i32 1>
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[TMP3]], align 4
+; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; FIXED-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
 ; FIXED-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 1
-; FIXED-NEXT:    [[TMP6:%.*]] = add <2 x i32> [[STRIDED_VEC1]], <i32 2, i32 2>
+; FIXED-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[STRIDED_VEC1]], <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
 ; FIXED-NEXT:    [[TMP7:%.*]] = add i64 [[TMP5]], 1
 ; FIXED-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP7]]
-; FIXED-NEXT:    [[TMP9:%.*]] = add <2 x i32> [[STRIDED_VEC2]], <i32 3, i32 3>
+; FIXED-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[STRIDED_VEC2]], <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
 ; FIXED-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[TMP8]], i32 -2
-; FIXED-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FIXED-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; FIXED-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> [[TMP12]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <6 x i32> [[TMP13]], <6 x i32> poison, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
-; FIXED-NEXT:    store <6 x i32> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; FIXED-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; FIXED-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; FIXED-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP13]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; FIXED-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; FIXED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; FIXED:       middle.block:
@@ -509,23 +509,23 @@ define void @load_store_factor3_i32(ptr %p) {
 ; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP1]]
 ; SCALABLE-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[TMP2]], i32 0
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <6 x i32>, ptr [[TMP3]], align 4
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <6 x i32> [[WIDE_VEC]], <6 x i32> poison, <2 x i32> <i32 0, i32 3>
-; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <6 x i32> [[WIDE_VEC]], <6 x i32> poison, <2 x i32> <i32 1, i32 4>
-; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <6 x i32> [[WIDE_VEC]], <6 x i32> poison, <2 x i32> <i32 2, i32 5>
-; SCALABLE-NEXT:    [[TMP4:%.*]] = add <2 x i32> [[STRIDED_VEC]], <i32 1, i32 1>
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i32>, ptr [[TMP3]], align 4
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <24 x i32> [[WIDE_VEC]], <24 x i32> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; SCALABLE-NEXT:    [[TMP4:%.*]] = add <8 x i32> [[STRIDED_VEC]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
 ; SCALABLE-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 1
-; SCALABLE-NEXT:    [[TMP6:%.*]] = add <2 x i32> [[STRIDED_VEC1]], <i32 2, i32 2>
+; SCALABLE-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[STRIDED_VEC1]], <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
 ; SCALABLE-NEXT:    [[TMP7:%.*]] = add i64 [[TMP5]], 1
 ; SCALABLE-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP7]]
-; SCALABLE-NEXT:    [[TMP9:%.*]] = add <2 x i32> [[STRIDED_VEC2]], <i32 3, i32 3>
+; SCALABLE-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[STRIDED_VEC2]], <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[TMP8]], i32 -2
-; SCALABLE-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SCALABLE-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SCALABLE-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> [[TMP12]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <6 x i32> [[TMP13]], <6 x i32> poison, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
-; SCALABLE-NEXT:    store <6 x i32> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; SCALABLE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SCALABLE-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SCALABLE-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <24 x i32> [[TMP13]], <24 x i32> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; SCALABLE-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; SCALABLE:       middle.block:
@@ -589,54 +589,38 @@ exit:
 define void @load_store_factor3_i64(ptr %p) {
 ; CHECK-LABEL: @load_store_factor3_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = add <vscale x 2 x i64> [[TMP6]], zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 2 x i64> [[TMP7]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP8]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 1, [[TMP10]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP12:%.*]] = mul <vscale x 2 x i64> [[VEC_IND]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 3, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[P:%.*]], <vscale x 2 x i64> [[TMP12]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP13]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x ptr> [[TMP13]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 2 x i64> [[TMP12]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP15]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP17:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER1]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 2, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP17]], <vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP18:%.*]] = add <vscale x 2 x i64> [[TMP15]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP18]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER2]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 3, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP20]], <vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], <i64 1, i64 1, i64 1, i64 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[STRIDED_VEC1]], <i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP5]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i64> [[STRIDED_VEC2]], <i64 3, i64 3, i64 3, i64 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[TMP8]], i32 -2
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> [[TMP12]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP13]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -668,26 +652,29 @@ define void @load_store_factor3_i64(ptr %p) {
 ; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], <i64 3, i64 3, i64 3, i64 3>
-; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], <4 x i64> [[TMP0]]
-; FIXED-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP1]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
-; FIXED-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[WIDE_MASKED_GATHER]], <i64 1, i64 1, i64 1, i64 1>
-; FIXED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP2]], <4 x ptr> [[TMP1]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; FIXED-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[TMP0]], <i64 1, i64 1, i64 1, i64 1>
-; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP3]]
-; FIXED-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP4]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
-; FIXED-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[WIDE_MASKED_GATHER1]], <i64 2, i64 2, i64 2, i64 2>
-; FIXED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP5]], <4 x ptr> [[TMP4]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; FIXED-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[TMP3]], <i64 1, i64 1, i64 1, i64 1>
-; FIXED-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP6]]
-; FIXED-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP7]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
-; FIXED-NEXT:    [[TMP8:%.*]] = add <4 x i64> [[WIDE_MASKED_GATHER2]], <i64 3, i64 3, i64 3, i64 3>
-; FIXED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP8]], <4 x ptr> [[TMP7]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; FIXED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
+; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
+; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP3]], align 8
+; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; FIXED-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], <i64 1, i64 1, i64 1, i64 1>
+; FIXED-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 1
+; FIXED-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[STRIDED_VEC1]], <i64 2, i64 2, i64 2, i64 2>
+; FIXED-NEXT:    [[TMP7:%.*]] = add i64 [[TMP5]], 1
+; FIXED-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
+; FIXED-NEXT:    [[TMP9:%.*]] = add <4 x i64> [[STRIDED_VEC2]], <i64 3, i64 3, i64 3, i64 3>
+; FIXED-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[TMP8]], i32 -2
+; FIXED-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; FIXED-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; FIXED-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> [[TMP12]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP13]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; FIXED-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 8
 ; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; FIXED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
-; FIXED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; FIXED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
@@ -718,54 +705,38 @@ define void @load_store_factor3_i64(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor3_i64(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
-; SCALABLE-NEXT:    [[TMP7:%.*]] = add <vscale x 2 x i64> [[TMP6]], zeroinitializer
-; SCALABLE-NEXT:    [[TMP8:%.*]] = mul <vscale x 2 x i64> [[TMP7]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP8]]
-; SCALABLE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
-; SCALABLE-NEXT:    [[TMP11:%.*]] = mul i64 1, [[TMP10]]
-; SCALABLE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
-; SCALABLE-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP12:%.*]] = mul <vscale x 2 x i64> [[VEC_IND]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 3, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[P:%.*]], <vscale x 2 x i64> [[TMP12]]
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP13]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x ptr> [[TMP13]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 2 x i64> [[TMP12]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[TMP16:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP15]]
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; SCALABLE-NEXT:    [[TMP17:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER1]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 2, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP17]], <vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SCALABLE-NEXT:    [[TMP18:%.*]] = add <vscale x 2 x i64> [[TMP15]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP18]]
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; SCALABLE-NEXT:    [[TMP20:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER2]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 3, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP20]], <vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; SCALABLE-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 3
+; SCALABLE-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
+; SCALABLE-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <12 x i64>, ptr [[TMP3]], align 8
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i64> [[WIDE_VEC]], <12 x i64> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; SCALABLE-NEXT:    [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], <i64 1, i64 1, i64 1, i64 1>
+; SCALABLE-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 1
+; SCALABLE-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[STRIDED_VEC1]], <i64 2, i64 2, i64 2, i64 2>
+; SCALABLE-NEXT:    [[TMP7:%.*]] = add i64 [[TMP5]], 1
+; SCALABLE-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
+; SCALABLE-NEXT:    [[TMP9:%.*]] = add <4 x i64> [[STRIDED_VEC2]], <i64 3, i64 3, i64 3, i64 3>
+; SCALABLE-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[TMP8]], i32 -2
+; SCALABLE-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SCALABLE-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SCALABLE-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> [[TMP12]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i64> [[TMP13]], <12 x i64> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; SCALABLE-NEXT:    store <12 x i64> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 8
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
 ; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -823,79 +794,57 @@ exit:
 define void @load_store_factor8(ptr %p) {
 ; CHECK-LABEL: @load_store_factor8(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = add <vscale x 2 x i64> [[TMP6]], zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 2 x i64> [[TMP7]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP8]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 1, [[TMP10]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP12:%.*]] = shl <vscale x 2 x i64> [[VEC_IND]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 3, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[P:%.*]], <vscale x 2 x i64> [[TMP12]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP13]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x ptr> [[TMP13]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 2 x i64> [[TMP12]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP15]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP17:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER1]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 2, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP17]], <vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP18:%.*]] = add <vscale x 2 x i64> [[TMP15]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP18]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER2]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 3, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP20]], <vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP18]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP21]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP22]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP23:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER3]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 4, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP23]], <vscale x 2 x ptr> [[TMP22]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP24:%.*]] = add <vscale x 2 x i64> [[TMP21]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP24]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP25]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP26:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER4]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 5, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP26]], <vscale x 2 x ptr> [[TMP25]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP27:%.*]] = add <vscale x 2 x i64> [[TMP24]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP27]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP28]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP29:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER5]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 6, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP29]], <vscale x 2 x ptr> [[TMP28]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP30:%.*]] = add <vscale x 2 x i64> [[TMP27]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP30]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP31]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP32:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER6]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 7, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP32]], <vscale x 2 x ptr> [[TMP31]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP33:%.*]] = add <vscale x 2 x i64> [[TMP30]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP33]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP34]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP35:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER7]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 8, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP35]], <vscale x 2 x ptr> [[TMP34]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 0, i32 8>
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 1, i32 9>
+; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 2, i32 10>
+; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 3, i32 11>
+; CHECK-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 4, i32 12>
+; CHECK-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 5, i32 13>
+; CHECK-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 6, i32 14>
+; CHECK-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 7, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC1]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP5]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC2]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add <2 x i64> [[STRIDED_VEC3]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP9]], 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add <2 x i64> [[STRIDED_VEC4]], <i64 5, i64 5>
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP11]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add <2 x i64> [[STRIDED_VEC5]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP13]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = add <2 x i64> [[STRIDED_VEC6]], <i64 7, i64 7>
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = add <2 x i64> [[STRIDED_VEC7]], <i64 8, i64 8>
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i64, ptr [[TMP18]], i32 -7
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x i64> [[TMP16]], <2 x i64> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i64> [[TMP21]], <4 x i64> [[TMP22]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i64> [[TMP23]], <4 x i64> [[TMP24]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <8 x i64> [[TMP25]], <8 x i64> [[TMP26]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i64> [[TMP27]], <16 x i64> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    store <16 x i64> [[INTERLEAVED_VEC]], ptr [[TMP20]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -952,51 +901,48 @@ define void @load_store_factor8(ptr %p) {
 ; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = shl <4 x i64> [[VEC_IND]], <i64 3, i64 3, i64 3, i64 3>
-; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], <4 x i64> [[TMP0]]
-; FIXED-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP1]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
-; FIXED-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[WIDE_MASKED_GATHER]], <i64 1, i64 1, i64 1, i64 1>
-; FIXED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP2]], <4 x ptr> [[TMP1]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; FIXED-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[TMP0]], <i64 1, i64 1, i64 1, i64 1>
-; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP3]]
-; FIXED-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP4]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
-; FIXED-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[WIDE_MASKED_GATHER1]], <i64 2, i64 2, i64 2, i64 2>
-; FIXED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP5]], <4 x ptr> [[TMP4]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; FIXED-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[TMP3]], <i64 1, i64 1, i64 1, i64 1>
-; FIXED-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP6]]
-; FIXED-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP7]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
-; FIXED-NEXT:    [[TMP8:%.*]] = add <4 x i64> [[WIDE_MASKED_GATHER2]], <i64 3, i64 3, i64 3, i64 3>
-; FIXED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP8]], <4 x ptr> [[TMP7]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; FIXED-NEXT:    [[TMP9:%.*]] = add <4 x i64> [[TMP6]], <i64 1, i64 1, i64 1, i64 1>
-; FIXED-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP9]]
-; FIXED-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP10]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
-; FIXED-NEXT:    [[TMP11:%.*]] = add <4 x i64> [[WIDE_MASKED_GATHER3]], <i64 4, i64 4, i64 4, i64 4>
-; FIXED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP11]], <4 x ptr> [[TMP10]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; FIXED-NEXT:    [[TMP12:%.*]] = add <4 x i64> [[TMP9]], <i64 1, i64 1, i64 1, i64 1>
-; FIXED-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP12]]
-; FIXED-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP13]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
-; FIXED-NEXT:    [[TMP14:%.*]] = add <4 x i64> [[WIDE_MASKED_GATHER4]], <i64 5, i64 5, i64 5, i64 5>
-; FIXED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP14]], <4 x ptr> [[TMP13]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; FIXED-NEXT:    [[TMP15:%.*]] = add <4 x i64> [[TMP12]], <i64 1, i64 1, i64 1, i64 1>
-; FIXED-NEXT:    [[TMP16:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP15]]
-; FIXED-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP16]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
-; FIXED-NEXT:    [[TMP17:%.*]] = add <4 x i64> [[WIDE_MASKED_GATHER5]], <i64 6, i64 6, i64 6, i64 6>
-; FIXED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP17]], <4 x ptr> [[TMP16]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; FIXED-NEXT:    [[TMP18:%.*]] = add <4 x i64> [[TMP15]], <i64 1, i64 1, i64 1, i64 1>
-; FIXED-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP18]]
-; FIXED-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP19]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
-; FIXED-NEXT:    [[TMP20:%.*]] = add <4 x i64> [[WIDE_MASKED_GATHER6]], <i64 7, i64 7, i64 7, i64 7>
-; FIXED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP20]], <4 x ptr> [[TMP19]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; FIXED-NEXT:    [[TMP21:%.*]] = add <4 x i64> [[TMP18]], <i64 1, i64 1, i64 1, i64 1>
-; FIXED-NEXT:    [[TMP22:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP21]]
-; FIXED-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP22]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
-; FIXED-NEXT:    [[TMP23:%.*]] = add <4 x i64> [[WIDE_MASKED_GATHER7]], <i64 8, i64 8, i64 8, i64 8>
-; FIXED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP23]], <4 x ptr> [[TMP22]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; FIXED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
-; FIXED-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; FIXED-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 3
+; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
+; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0
+; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[TMP3]], align 8
+; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 0, i32 8>
+; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 1, i32 9>
+; FIXED-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 2, i32 10>
+; FIXED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 3, i32 11>
+; FIXED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 4, i32 12>
+; FIXED-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 5, i32 13>
+; FIXED-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 6, i32 14>
+; FIXED-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 7, i32 15>
+; FIXED-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC]], <i64 1, i64 1>
+; FIXED-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 1
+; FIXED-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC1]], <i64 2, i64 2>
+; FIXED-NEXT:    [[TMP7:%.*]] = add i64 [[TMP5]], 1
+; FIXED-NEXT:    [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC2]], <i64 3, i64 3>
+; FIXED-NEXT:    [[TMP9:%.*]] = add i64 [[TMP7]], 1
+; FIXED-NEXT:    [[TMP10:%.*]] = add <2 x i64> [[STRIDED_VEC3]], <i64 4, i64 4>
+; FIXED-NEXT:    [[TMP11:%.*]] = add i64 [[TMP9]], 1
+; FIXED-NEXT:    [[TMP12:%.*]] = add <2 x i64> [[STRIDED_VEC4]], <i64 5, i64 5>
+; FIXED-NEXT:    [[TMP13:%.*]] = add i64 [[TMP11]], 1
+; FIXED-NEXT:    [[TMP14:%.*]] = add <2 x i64> [[STRIDED_VEC5]], <i64 6, i64 6>
+; FIXED-NEXT:    [[TMP15:%.*]] = add i64 [[TMP13]], 1
+; FIXED-NEXT:    [[TMP16:%.*]] = add <2 x i64> [[STRIDED_VEC6]], <i64 7, i64 7>
+; FIXED-NEXT:    [[TMP17:%.*]] = add i64 [[TMP15]], 1
+; FIXED-NEXT:    [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
+; FIXED-NEXT:    [[TMP19:%.*]] = add <2 x i64> [[STRIDED_VEC7]], <i64 8, i64 8>
+; FIXED-NEXT:    [[TMP20:%.*]] = getelementptr i64, ptr [[TMP18]], i32 -7
+; FIXED-NEXT:    [[TMP21:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP22:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP24:%.*]] = shufflevector <2 x i64> [[TMP16]], <2 x i64> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; FIXED-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i64> [[TMP21]], <4 x i64> [[TMP22]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; FIXED-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i64> [[TMP23]], <4 x i64> [[TMP24]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; FIXED-NEXT:    [[TMP27:%.*]] = shufflevector <8 x i64> [[TMP25]], <8 x i64> [[TMP26]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; FIXED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i64> [[TMP27]], <16 x i64> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; FIXED-NEXT:    store <16 x i64> [[INTERLEAVED_VEC]], ptr [[TMP20]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; FIXED-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; FIXED-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; FIXED:       scalar.ph:
@@ -1052,79 +998,57 @@ define void @load_store_factor8(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor8(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
-; SCALABLE-NEXT:    [[TMP7:%.*]] = add <vscale x 2 x i64> [[TMP6]], zeroinitializer
-; SCALABLE-NEXT:    [[TMP8:%.*]] = mul <vscale x 2 x i64> [[TMP7]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP8]]
-; SCALABLE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
-; SCALABLE-NEXT:    [[TMP11:%.*]] = mul i64 1, [[TMP10]]
-; SCALABLE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
-; SCALABLE-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP12:%.*]] = shl <vscale x 2 x i64> [[VEC_IND]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 3, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[P:%.*]], <vscale x 2 x i64> [[TMP12]]
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP13]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x ptr> [[TMP13]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 2 x i64> [[TMP12]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[TMP16:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP15]]
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; SCALABLE-NEXT:    [[TMP17:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER1]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 2, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP17]], <vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SCALABLE-NEXT:    [[TMP18:%.*]] = add <vscale x 2 x i64> [[TMP15]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP18]]
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; SCALABLE-NEXT:    [[TMP20:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER2]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 3, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP20]], <vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SCALABLE-NEXT:    [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP18]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[TMP22:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP21]]
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP22]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; SCALABLE-NEXT:    [[TMP23:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER3]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 4, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP23]], <vscale x 2 x ptr> [[TMP22]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SCALABLE-NEXT:    [[TMP24:%.*]] = add <vscale x 2 x i64> [[TMP21]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[TMP25:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP24]]
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP25]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; SCALABLE-NEXT:    [[TMP26:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER4]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 5, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP26]], <vscale x 2 x ptr> [[TMP25]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SCALABLE-NEXT:    [[TMP27:%.*]] = add <vscale x 2 x i64> [[TMP24]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP27]]
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP28]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; SCALABLE-NEXT:    [[TMP29:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER5]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 6, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP29]], <vscale x 2 x ptr> [[TMP28]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SCALABLE-NEXT:    [[TMP30:%.*]] = add <vscale x 2 x i64> [[TMP27]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[TMP31:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP30]]
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP31]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; SCALABLE-NEXT:    [[TMP32:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER6]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 7, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP32]], <vscale x 2 x ptr> [[TMP31]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SCALABLE-NEXT:    [[TMP33:%.*]] = add <vscale x 2 x i64> [[TMP30]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    [[TMP34:%.*]] = getelementptr i64, ptr [[P]], <vscale x 2 x i64> [[TMP33]]
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP34]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), <vscale x 2 x i64> poison)
-; SCALABLE-NEXT:    [[TMP35:%.*]] = add <vscale x 2 x i64> [[WIDE_MASKED_GATHER7]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 8, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP35]], <vscale x 2 x ptr> [[TMP34]], i32 8, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; SCALABLE-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; SCALABLE-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 3
+; SCALABLE-NEXT:    [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
+; SCALABLE-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0
+; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[TMP3]], align 8
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 0, i32 8>
+; SCALABLE-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 1, i32 9>
+; SCALABLE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 2, i32 10>
+; SCALABLE-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 3, i32 11>
+; SCALABLE-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 4, i32 12>
+; SCALABLE-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 5, i32 13>
+; SCALABLE-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 6, i32 14>
+; SCALABLE-NEXT:    [[STRIDED_VEC7:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <2 x i32> <i32 7, i32 15>
+; SCALABLE-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[STRIDED_VEC]], <i64 1, i64 1>
+; SCALABLE-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 1
+; SCALABLE-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[STRIDED_VEC1]], <i64 2, i64 2>
+; SCALABLE-NEXT:    [[TMP7:%.*]] = add i64 [[TMP5]], 1
+; SCALABLE-NEXT:    [[TMP8:%.*]] = add <2 x i64> [[STRIDED_VEC2]], <i64 3, i64 3>
+; SCALABLE-NEXT:    [[TMP9:%.*]] = add i64 [[TMP7]], 1
+; SCALABLE-NEXT:    [[TMP10:%.*]] = add <2 x i64> [[STRIDED_VEC3]], <i64 4, i64 4>
+; SCALABLE-NEXT:    [[TMP11:%.*]] = add i64 [[TMP9]], 1
+; SCALABLE-NEXT:    [[TMP12:%.*]] = add <2 x i64> [[STRIDED_VEC4]], <i64 5, i64 5>
+; SCALABLE-NEXT:    [[TMP13:%.*]] = add i64 [[TMP11]], 1
+; SCALABLE-NEXT:    [[TMP14:%.*]] = add <2 x i64> [[STRIDED_VEC5]], <i64 6, i64 6>
+; SCALABLE-NEXT:    [[TMP15:%.*]] = add i64 [[TMP13]], 1
+; SCALABLE-NEXT:    [[TMP16:%.*]] = add <2 x i64> [[STRIDED_VEC6]], <i64 7, i64 7>
+; SCALABLE-NEXT:    [[TMP17:%.*]] = add i64 [[TMP15]], 1
+; SCALABLE-NEXT:    [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
+; SCALABLE-NEXT:    [[TMP19:%.*]] = add <2 x i64> [[STRIDED_VEC7]], <i64 8, i64 8>
+; SCALABLE-NEXT:    [[TMP20:%.*]] = getelementptr i64, ptr [[TMP18]], i32 -7
+; SCALABLE-NEXT:    [[TMP21:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SCALABLE-NEXT:    [[TMP22:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SCALABLE-NEXT:    [[TMP23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SCALABLE-NEXT:    [[TMP24:%.*]] = shufflevector <2 x i64> [[TMP16]], <2 x i64> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SCALABLE-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i64> [[TMP21]], <4 x i64> [[TMP22]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SCALABLE-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i64> [[TMP23]], <4 x i64> [[TMP24]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SCALABLE-NEXT:    [[TMP27:%.*]] = shufflevector <8 x i64> [[TMP25]], <8 x i64> [[TMP26]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i64> [[TMP27]], <16 x i64> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; SCALABLE-NEXT:    store <16 x i64> [[INTERLEAVED_VEC]], ptr [[TMP20]], align 8
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; SCALABLE-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
 ; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
index a724ef87abb3..7bfd2eaad574 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
@@ -72,12 +72,12 @@ entry:
 ; VF_8:       Found an estimated cost of 0 for VF 8 For instruction:   store i8 %a1, ptr %p1, align 1
 ; VF_8-NEXT:  Found an estimated cost of 3 for VF 8 For instruction:   store i8 %a2, ptr %p2, align 1
 ; VF_16-LABEL: Checking a loop in 'i8_factor_3'
-; VF_16:       Found an estimated cost of 48 for VF 16 For instruction:   %l0 = load i8, ptr %p0, align 1
+; VF_16:       Found an estimated cost of 5 for VF 16 For instruction:   %l0 = load i8, ptr %p0, align 1
 ; VF_16-NEXT:  Found an estimated cost of 0 for VF 16 For instruction:   %l1 = load i8, ptr %p1, align 1
 ; VF_16-NEXT:  Found an estimated cost of 0 for VF 16 For instruction:   %l2 = load i8, ptr %p2, align 1
 ; VF_16:       Found an estimated cost of 0 for VF 16 For instruction:   store i8 %a0, ptr %p0, align 1
 ; VF_16:       Found an estimated cost of 0 for VF 16 For instruction:   store i8 %a1, ptr %p1, align 1
-; VF_16-NEXT:  Found an estimated cost of 48 for VF 16 For instruction:   store i8 %a2, ptr %p2, align 1
+; VF_16-NEXT:  Found an estimated cost of 5 for VF 16 For instruction:   store i8 %a2, ptr %p2, align 1
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 0
-- 
GitLab


From 6bf185920bd6831efc151d7d4158d6390006c50b Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Thu, 16 May 2024 14:44:58 +1000
Subject: [PATCH 0960/1206] [ORC] Support visionOS in LC_BUILD_VERSIONs for
 JITDylibs.

rdar://127846581
---
 llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index 2b397b2d48e7..b477a48af290 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -277,6 +277,10 @@ MachOPlatform::HeaderOptions::BuildVersionOpts::fromTriple(const Triple &TT,
     Platform = TT.isSimulatorEnvironment() ? MachO::PLATFORM_WATCHOSSIMULATOR
                                            : MachO::PLATFORM_WATCHOS;
     break;
+  case Triple::XROS:
+    Platform = TT.isSimulatorEnvironment() ? MachO::PLATFORM_XROS_SIMULATOR
+                                           : MachO::PLATFORM_XROS;
+    break;
   default:
     return std::nullopt;
   }
-- 
GitLab


From 566fbb450092bf8c9f966a6ab1b0381626e3e535 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Thu, 16 May 2024 12:57:28 +0800
Subject: [PATCH 0961/1206] [RISCV] Defer creating RISCVInsertVSETVLI to avoid
 leak with -stop-after (#92303)

As noted in
https://github.com/llvm/llvm-project/pull/91440#discussion_r1601976425,
if the pass pipeline stops early because of -stop-after any allocated
passes added with insertPass will not be freed if they haven't already
been added.

This was showing up as a failure on the address sanitizer buildbots. We
can fix it by instead passing the pass ID instead so that allocation is
deferred.
---
 llvm/lib/Target/RISCV/RISCV.h                | 1 +
 llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 1 +
 llvm/lib/Target/RISCV/RISCVTargetMachine.cpp | 4 ++--
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h
index d405395dcf9e..2b8688c5de61 100644
--- a/llvm/lib/Target/RISCV/RISCV.h
+++ b/llvm/lib/Target/RISCV/RISCV.h
@@ -60,6 +60,7 @@ void initializeRISCVExpandAtomicPseudoPass(PassRegistry &);
 
 FunctionPass *createRISCVInsertVSETVLIPass();
 void initializeRISCVInsertVSETVLIPass(PassRegistry &);
+extern char &RISCVInsertVSETVLIID;
 
 FunctionPass *createRISCVCoalesceVSETVLIPass();
 void initializeRISCVCoalesceVSETVLIPass(PassRegistry &);
diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 363007d7b68b..324ce5cb5ed7 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -868,6 +868,7 @@ private:
 } // end anonymous namespace
 
 char RISCVInsertVSETVLI::ID = 0;
+char &llvm::RISCVInsertVSETVLIID = RISCVInsertVSETVLI::ID;
 
 INITIALIZE_PASS(RISCVInsertVSETVLI, DEBUG_TYPE, RISCV_INSERT_VSETVLI_NAME,
                 false, false)
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 5d598a275a00..5aab138dae40 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -548,9 +548,9 @@ void RISCVPassConfig::addPreRegAlloc() {
   // Run RISCVInsertVSETVLI after PHI elimination. On O1 and above do it after
   // register coalescing so needVSETVLIPHI doesn't need to look through COPYs.
   if (TM->getOptLevel() == CodeGenOptLevel::None)
-    insertPass(&PHIEliminationID, createRISCVInsertVSETVLIPass());
+    insertPass(&PHIEliminationID, &RISCVInsertVSETVLIID);
   else
-    insertPass(&RegisterCoalescerID, createRISCVInsertVSETVLIPass());
+    insertPass(&RegisterCoalescerID, &RISCVInsertVSETVLIID);
 }
 
 void RISCVPassConfig::addFastRegAlloc() {
-- 
GitLab


From 70a926cfb1d4af326be5afe6419991aeff8f44b2 Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Thu, 16 May 2024 02:39:04 -0300
Subject: [PATCH 0962/1206] [clang] NFC: Add a few more interesting test cases
 for CWG2398

---
 clang/test/SemaTemplate/cwg2398.cpp | 58 +++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/clang/test/SemaTemplate/cwg2398.cpp b/clang/test/SemaTemplate/cwg2398.cpp
index a20155486b12..d163354b2e5f 100644
--- a/clang/test/SemaTemplate/cwg2398.cpp
+++ b/clang/test/SemaTemplate/cwg2398.cpp
@@ -137,3 +137,61 @@ namespace ttp_defaults {
   // old-error@-2 {{template template argument has different template parameters}}
   // old-error@-3 {{explicit instantiation of 'f' does not refer to a function template}}
 } // namespace ttp_defaults
+
+namespace ttp_only {
+  template <template <class...    > class TT1> struct A      { static constexpr int V = 0; };
+  // new-note@-1 2{{template is declared here}}
+  template <template <class       > class TT2> struct A<TT2> { static constexpr int V = 1; };
+  // new-error@-1 {{not more specialized than the primary template}}
+  // new-note@-2 {{partial specialization matches}}
+  template <template <class, class> class TT3> struct A<TT3> { static constexpr int V = 2; };
+  // new-error@-1 {{not more specialized than the primary template}}
+  // new-note@-2 {{partial specialization matches}}
+
+  template <class ...          > struct B;
+  template <class              > struct C;
+  template <class, class       > struct D;
+  template <class, class, class> struct E;
+
+  static_assert(A<B>::V == 0); // new-error {{ambiguous partial specializations}}
+  static_assert(A<C>::V == 1);
+  static_assert(A<D>::V == 2);
+  static_assert(A<E>::V == 0);
+} // namespace ttp_only
+
+namespace consistency {
+  template<class T> struct nondeduced { using type = T; };
+  template<class T8, class T9 = float> struct B;
+
+  namespace t1 {
+    template<class T1, class T2, class T3> struct A;
+
+    template<template<class, class> class TT1,
+             class T1, class T2, class T3, class T4>
+    struct A<TT1<T1, T2>, TT1<T3, T4>, typename nondeduced<TT1<T1, T2>>::type> {};
+
+    template<template<class> class UU1,
+             template<class> class UU2,
+             class U1, class U2>
+    struct A<UU1<U1>, UU2<U2>, typename nondeduced<UU1<U1>>::type>;
+
+    template struct A<B<int>, B<int>, B<int>>;
+  } // namespace t1
+  namespace t2 {
+    template<class T1, class T2, class T3> struct A;
+
+    template<template<class, class> class TT1,
+             class T1, class T2, class T3, class T4>
+    struct A<TT1<T1, T2>, TT1<T3, T4>, typename nondeduced<TT1<T1, T4>>::type> {};
+    // new-note@-1 {{partial specialization matches}}
+
+    template<template<class> class UU1,
+             template<class> class UU2,
+             class U1, class U2>
+    struct A<UU1<U1>, UU2<U2>, typename nondeduced<UU1<U1>>::type>;
+    // new-note@-1 {{partial specialization matches}}
+
+    template struct A<B<int>, B<int>, B<int>>;
+    // new-error@-1 {{ambiguous partial specializations}}
+  } // namespace t1
+} // namespace consistency
-- 
GitLab


From 5bd8091f4bde53de48329e14d5b955314e6518f4 Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Wed, 15 May 2024 22:58:03 -0700
Subject: [PATCH 0963/1206] [mlir] fix polynomial docs for MLIR website
 (#92348)

I built it and confirmed this fixes the issue locally.

Co-authored-by: Jeremy Kun <j2kun@users.noreply.github.com>
---
 mlir/docs/Dialects/Polynomial.md                       | 2 --
 mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt | 1 -
 2 files changed, 3 deletions(-)
 delete mode 100644 mlir/docs/Dialects/Polynomial.md

diff --git a/mlir/docs/Dialects/Polynomial.md b/mlir/docs/Dialects/Polynomial.md
deleted file mode 100644
index 6fdd74ab30aa..000000000000
--- a/mlir/docs/Dialects/Polynomial.md
+++ /dev/null
@@ -1,2 +0,0 @@
-[include "Dialects/PolynomialDialect.td"]
-[include "Dialects/PolynomialOps.td"]
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
index c8655ba99870..e8f48f13be2b 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
@@ -1,6 +1,5 @@
 add_mlir_dialect(Polynomial polynomial)
 add_mlir_doc(Polynomial PolynomialDialect Dialects/ -gen-dialect-doc -dialect=polynomial)
-add_mlir_doc(Polynomial PolynomialOps Dialects/ -gen-op-doc)
 
 set(LLVM_TARGET_DEFINITIONS Polynomial.td)
 mlir_tablegen(PolynomialAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=polynomial)
-- 
GitLab


From afce597baa9c6cf46feb30aba97cf832bff32673 Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Wed, 15 May 2024 22:58:26 -0700
Subject: [PATCH 0964/1206] [mlir] fix IRDL dialect docs (#92349)

Currently the irdl dialect page has no content beyond the header.

By referring to the Ops.td in the CMake config, it pulls in all the
types, attributes, etc., so that the doc generation can include them all
in the page.

Rendered locally to confirm it fixes the issue


![image](https://github.com/llvm/llvm-project/assets/2467754/8758f324-6bc3-4f0e-8fa9-8962cdb0177f)

Co-authored-by: Jeremy Kun <j2kun@users.noreply.github.com>
---
 mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt
index e255fc632110..ec3e06bedae0 100644
--- a/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt
@@ -1,5 +1,5 @@
 add_mlir_dialect(IRDL irdl)
-add_mlir_doc(IRDL IRDL Dialects/ -gen-dialect-doc)
+add_mlir_doc(IRDLOps IRDL Dialects/ -gen-dialect-doc)
 
 # Add IRDL interfaces
 set(LLVM_TARGET_DEFINITIONS IRDLInterfaces.td)
-- 
GitLab


From 3c2638dae58466f7eb4384bb7f26c9af904bf94c Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Thu, 16 May 2024 13:45:07 +0800
Subject: [PATCH 0965/1206] Revert "[Serialization] Read the initializer for
 interesting static variables before consuming it (#92218)"

This reverts commit 3a4c1b9b4428b08d4475decf74c11e0d328c5842.

This breaks a bot on clang-s390x-linux
---
 clang/lib/Serialization/ASTReaderDecl.cpp    |  29 +--
 clang/test/Modules/pr91418.cppm              |  67 -----
 clang/test/OpenMP/nvptx_lambda_capturing.cpp | 246 +++++++++----------
 3 files changed, 126 insertions(+), 216 deletions(-)
 delete mode 100644 clang/test/Modules/pr91418.cppm

diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index a6254b70560c..0c647086e304 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -4186,35 +4186,12 @@ void ASTReader::PassInterestingDeclsToConsumer() {
     GetDecl(ID);
   EagerlyDeserializedDecls.clear();
 
-  auto ConsumingPotentialInterestingDecls = [this]() {
-    while (!PotentiallyInterestingDecls.empty()) {
-      Decl *D = PotentiallyInterestingDecls.front();
-      PotentiallyInterestingDecls.pop_front();
-      if (isConsumerInterestedIn(D))
-        PassInterestingDeclToConsumer(D);
-    }
-  };
-  std::deque<Decl *> MaybeInterestingDecls =
-      std::move(PotentiallyInterestingDecls);
-  assert(PotentiallyInterestingDecls.empty());
-  while (!MaybeInterestingDecls.empty()) {
-    Decl *D = MaybeInterestingDecls.front();
-    MaybeInterestingDecls.pop_front();
-    // Since we load the variable's initializers lazily, it'd be problematic
-    // if the initializers dependent on each other. So here we try to load the
-    // initializers of static variables to make sure they are passed to code
-    // generator by order. If we read anything interesting, we would consume
-    // that before emitting the current declaration.
-    if (auto *VD = dyn_cast<VarDecl>(D);
-        VD && VD->isFileVarDecl() && !VD->isExternallyVisible())
-      VD->getInit();
-    ConsumingPotentialInterestingDecls();
+  while (!PotentiallyInterestingDecls.empty()) {
+    Decl *D = PotentiallyInterestingDecls.front();
+    PotentiallyInterestingDecls.pop_front();
     if (isConsumerInterestedIn(D))
       PassInterestingDeclToConsumer(D);
   }
-
-  // If we add any new potential interesting decl in the last call, consume it.
-  ConsumingPotentialInterestingDecls();
 }
 
 void ASTReader::loadDeclUpdateRecords(PendingUpdateRecord &Record) {
diff --git a/clang/test/Modules/pr91418.cppm b/clang/test/Modules/pr91418.cppm
deleted file mode 100644
index 33fec992439d..000000000000
--- a/clang/test/Modules/pr91418.cppm
+++ /dev/null
@@ -1,67 +0,0 @@
-// RUN: rm -rf %t
-// RUN: mkdir -p %t
-// RUN: split-file %s %t
-//
-// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 -x c++-header %t/foo.h \
-// RUN:     -emit-pch -o %t/foo.pch
-// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/use.cpp -include-pch \
-// RUN:     %t/foo.pch -emit-llvm -o - | FileCheck %t/use.cpp
-
-//--- foo.h
-#ifndef FOO_H
-#define FOO_H
-typedef float __m128 __attribute__((__vector_size__(16), __aligned__(16)));
-
-static __inline__ __m128 __attribute__((__always_inline__, __min_vector_width__(128)))
-_mm_setr_ps(float __z, float __y, float __x, float __w)
-{
-  return __extension__ (__m128){ __z, __y, __x, __w };
-}
-
-typedef __m128 VR;
-
-inline VR MakeVR( float X, float Y, float Z, float W )
-{
- return _mm_setr_ps( X, Y, Z, W );
-}
-
-extern "C" float sqrtf(float);
-
-namespace VectorSinConstantsSSE
-{
-  float a = (16 * sqrtf(0.225f));
-  VR A = MakeVR(a, a, a, a);
-  static const float b = (16 * sqrtf(0.225f));
-  static const VR B = MakeVR(b, b, b, b);
-}
-
-#endif // FOO_H
-
-//--- use.cpp
-#include "foo.h"
-float use() {
-    return VectorSinConstantsSSE::A[0] + VectorSinConstantsSSE::A[1] +
-           VectorSinConstantsSSE::A[2] + VectorSinConstantsSSE::A[3] +
-           VectorSinConstantsSSE::B[0] + VectorSinConstantsSSE::B[1] +
-           VectorSinConstantsSSE::B[2] + VectorSinConstantsSSE::B[3];
-}
-
-// CHECK: define{{.*}}@__cxx_global_var_init(
-// CHECK: store{{.*}}[[a_RESULT:%[a-zA-Z0-9]+]], ptr @_ZN21VectorSinConstantsSSE1aE
-
-// CHECK: define{{.*}}@__cxx_global_var_init.1(
-// CHECK: [[A_CALL:%[a-zA-Z0-9]+]] = call{{.*}}@_Z6MakeVRffff(
-// CHECK: store{{.*}}[[A_CALL]], ptr @_ZN21VectorSinConstantsSSE1AE
-
-// CHECK: define{{.*}}@__cxx_global_var_init.2(
-// CHECK: [[B_CALL:%[a-zA-Z0-9]+]] = call{{.*}}@_Z6MakeVRffff(
-// CHECK: store{{.*}}[[B_CALL]], ptr @_ZN21VectorSinConstantsSSEL1BE
-
-// CHECK: define{{.*}}@__cxx_global_var_init.3(
-// CHECK: store{{.*}}[[b_RESULT:%[a-zA-Z0-9]+]], ptr @_ZN21VectorSinConstantsSSEL1bE
-
-// CHECK: @_GLOBAL__sub_I_use.cpp
-// CHECK: call{{.*}}@__cxx_global_var_init(
-// CHECK: call{{.*}}@__cxx_global_var_init.1(
-// CHECK: call{{.*}}@__cxx_global_var_init.3(
-// CHECK: call{{.*}}@__cxx_global_var_init.2(
diff --git a/clang/test/OpenMP/nvptx_lambda_capturing.cpp b/clang/test/OpenMP/nvptx_lambda_capturing.cpp
index efea8d4a0561..641fbc38dd6b 100644
--- a/clang/test/OpenMP/nvptx_lambda_capturing.cpp
+++ b/clang/test/OpenMP/nvptx_lambda_capturing.cpp
@@ -1165,113 +1165,8 @@ int main(int argc, char **argv) {
 // CHECK2-NEXT:    ret void
 //
 //
-// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l27
-// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR0:[0-9]+]] {
-// CHECK3-NEXT:  entry:
-// CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L1:%.*]] = alloca [[CLASS_ANON:%.*]], align 8
-// CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
-// CHECK3-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l27_kernel_environment, ptr [[DYN_PTR]])
-// CHECK3-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP2]], -1
-// CHECK3-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
-// CHECK3:       user_code.entry:
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[L1]], ptr align 8 [[TMP3]], i64 8, i1 false)
-// CHECK3-NEXT:    store ptr [[L1]], ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP4]], i32 0, i32 0
-// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP5]], align 8
-// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i32 @_ZZN1S3fooEvENKUlvE_clEv(ptr noundef nonnull align 8 dereferenceable(8) [[TMP6]]) #[[ATTR7:[0-9]+]]
-// CHECK3-NEXT:    call void @__kmpc_target_deinit()
-// CHECK3-NEXT:    ret void
-// CHECK3:       worker.exit:
-// CHECK3-NEXT:    ret void
-//
-//
-// CHECK3-LABEL: define {{[^@]+}}@_ZZN1S3fooEvENKUlvE_clEv
-// CHECK3-SAME: (ptr noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]]) #[[ATTR2:[0-9]+]] comdat align 2 {
-// CHECK3-NEXT:  entry:
-// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[CLASS_ANON:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
-// CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[TMP1]], i32 0, i32 0
-// CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
-// CHECK3-NEXT:    ret i32 [[TMP2]]
-//
-//
-// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29
-// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR3:[0-9]+]] {
-// CHECK3-NEXT:  entry:
-// CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [2 x ptr], align 8
-// CHECK3-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
-// CHECK3-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_kernel_environment, ptr [[DYN_PTR]])
-// CHECK3-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP2]], -1
-// CHECK3-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
-// CHECK3:       user_code.entry:
-// CHECK3-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1:[0-9]+]])
-// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [2 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 0
-// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP5]], align 8
-// CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [2 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 1
-// CHECK3-NEXT:    store ptr [[TMP4]], ptr [[TMP6]], align 8
-// CHECK3-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP3]], i32 1, i32 -1, i32 -1, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_omp_outlined, ptr null, ptr [[CAPTURED_VARS_ADDRS]], i64 2)
-// CHECK3-NEXT:    call void @__kmpc_target_deinit()
-// CHECK3-NEXT:    ret void
-// CHECK3:       worker.exit:
-// CHECK3-NEXT:    ret void
-//
-//
-// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_omp_outlined
-// CHECK3-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR4:[0-9]+]] {
-// CHECK3-NEXT:  entry:
-// CHECK3-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L1:%.*]] = alloca [[CLASS_ANON:%.*]], align 8
-// CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
-// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
-// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[L1]], ptr align 8 [[TMP2]], i64 8, i1 false)
-// CHECK3-NEXT:    store ptr [[L1]], ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP3]], i32 0, i32 0
-// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP4]], align 8
-// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP2]], align 8
-// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i32 @_ZZN1S3fooEvENKUlvE_clEv(ptr noundef nonnull align 8 dereferenceable(8) [[TMP5]]) #[[ATTR7]]
-// CHECK3-NEXT:    ret void
-//
-//
 // CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}_main_l41
-// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], i64 noundef [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR0]] {
+// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], i64 noundef [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR0:[0-9]+]] {
 // CHECK3-NEXT:  entry:
 // CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[ARGC_ADDR:%.*]] = alloca i64, align 8
@@ -1283,7 +1178,7 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[_TMP1:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L3:%.*]] = alloca [[CLASS_ANON_1:%.*]], align 8
+// CHECK3-NEXT:    [[L3:%.*]] = alloca [[CLASS_ANON:%.*]], align 8
 // CHECK3-NEXT:    [[_TMP4:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[B5:%.*]] = alloca i32, align 4
 // CHECK3-NEXT:    [[_TMP6:%.*]] = alloca ptr, align 8
@@ -1319,20 +1214,20 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    store i32 [[TMP9]], ptr [[C7]], align 4
 // CHECK3-NEXT:    store ptr [[C7]], ptr [[_TMP8]], align 8
 // CHECK3-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[_TMP4]], align 8
-// CHECK3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP10]], i32 0, i32 0
+// CHECK3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP10]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[ARGC_ADDR]], ptr [[TMP11]], align 8
-// CHECK3-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP10]], i32 0, i32 1
+// CHECK3-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP10]], i32 0, i32 1
 // CHECK3-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[_TMP6]], align 8
 // CHECK3-NEXT:    store ptr [[TMP13]], ptr [[TMP12]], align 8
-// CHECK3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP10]], i32 0, i32 2
+// CHECK3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP10]], i32 0, i32 2
 // CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP8]], align 8
 // CHECK3-NEXT:    store ptr [[TMP15]], ptr [[TMP14]], align 8
-// CHECK3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP10]], i32 0, i32 3
+// CHECK3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP10]], i32 0, i32 3
 // CHECK3-NEXT:    store ptr [[D_ADDR]], ptr [[TMP16]], align 8
-// CHECK3-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP10]], i32 0, i32 4
+// CHECK3-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP10]], i32 0, i32 4
 // CHECK3-NEXT:    store ptr [[TMP2]], ptr [[TMP17]], align 8
 // CHECK3-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[_TMP4]], align 8
-// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i64 @"_ZZ4mainENK3$_0clEv"(ptr noundef nonnull align 8 dereferenceable(40) [[TMP18]]) #[[ATTR7]]
+// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i64 @"_ZZ4mainENK3$_0clEv"(ptr noundef nonnull align 8 dereferenceable(40) [[TMP18]]) #[[ATTR7:[0-9]+]]
 // CHECK3-NEXT:    call void @__kmpc_target_deinit()
 // CHECK3-NEXT:    ret void
 // CHECK3:       worker.exit:
@@ -1340,7 +1235,7 @@ int main(int argc, char **argv) {
 //
 //
 // CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}_main_l43
-// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR3]] {
+// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR3:[0-9]+]] {
 // CHECK3-NEXT:  entry:
 // CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[ARGC_ADDR:%.*]] = alloca ptr, align 8
@@ -1372,7 +1267,7 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP5]], -1
 // CHECK3-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
 // CHECK3:       user_code.entry:
-// CHECK3-NEXT:    [[TMP6:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
+// CHECK3-NEXT:    [[TMP6:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1:[0-9]+]])
 // CHECK3-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[TMP]], align 8
 // CHECK3-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 8
 // CHECK3-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[D_ADDR]], align 8
@@ -1397,7 +1292,7 @@ int main(int argc, char **argv) {
 //
 //
 // CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}_main_l43_omp_outlined
-// CHECK3-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR4]] {
+// CHECK3-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[ARGC:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[B:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[C:%.*]], ptr noundef [[D:%.*]], ptr noundef nonnull align 4 dereferenceable(4) [[A:%.*]], ptr noundef nonnull align 8 dereferenceable(40) [[L:%.*]]) #[[ATTR4:[0-9]+]] {
 // CHECK3-NEXT:  entry:
 // CHECK3-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
@@ -1410,7 +1305,7 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[_TMP1:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[L3:%.*]] = alloca [[CLASS_ANON_1:%.*]], align 8
+// CHECK3-NEXT:    [[L3:%.*]] = alloca [[CLASS_ANON:%.*]], align 8
 // CHECK3-NEXT:    [[_TMP4:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[ARGC5:%.*]] = alloca i32, align 4
 // CHECK3-NEXT:    [[B6:%.*]] = alloca i32, align 4
@@ -1450,23 +1345,128 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP3]], align 4
 // CHECK3-NEXT:    store i32 [[TMP11]], ptr [[A10]], align 4
 // CHECK3-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[_TMP4]], align 8
-// CHECK3-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP12]], i32 0, i32 0
+// CHECK3-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP12]], i32 0, i32 0
 // CHECK3-NEXT:    store ptr [[ARGC5]], ptr [[TMP13]], align 8
-// CHECK3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP12]], i32 0, i32 1
+// CHECK3-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP12]], i32 0, i32 1
 // CHECK3-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8
 // CHECK3-NEXT:    store ptr [[TMP15]], ptr [[TMP14]], align 8
-// CHECK3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP12]], i32 0, i32 2
+// CHECK3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP12]], i32 0, i32 2
 // CHECK3-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[_TMP9]], align 8
 // CHECK3-NEXT:    store ptr [[TMP17]], ptr [[TMP16]], align 8
-// CHECK3-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP12]], i32 0, i32 3
+// CHECK3-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP12]], i32 0, i32 3
 // CHECK3-NEXT:    store ptr [[D_ADDR]], ptr [[TMP18]], align 8
-// CHECK3-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP12]], i32 0, i32 4
+// CHECK3-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[TMP12]], i32 0, i32 4
 // CHECK3-NEXT:    store ptr [[A10]], ptr [[TMP19]], align 8
 // CHECK3-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[_TMP4]], align 8
 // CHECK3-NEXT:    [[CALL:%.*]] = call noundef i64 @"_ZZ4mainENK3$_0clEv"(ptr noundef nonnull align 8 dereferenceable(40) [[TMP20]]) #[[ATTR7]]
 // CHECK3-NEXT:    ret void
 //
 //
+// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l27
+// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR0]] {
+// CHECK3-NEXT:  entry:
+// CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[L1:%.*]] = alloca [[CLASS_ANON_1:%.*]], align 8
+// CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
+// CHECK3-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l27_kernel_environment, ptr [[DYN_PTR]])
+// CHECK3-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP2]], -1
+// CHECK3-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
+// CHECK3:       user_code.entry:
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[L1]], ptr align 8 [[TMP3]], i64 8, i1 false)
+// CHECK3-NEXT:    store ptr [[L1]], ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP4]], i32 0, i32 0
+// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP5]], align 8
+// CHECK3-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i32 @_ZZN1S3fooEvENKUlvE_clEv(ptr noundef nonnull align 8 dereferenceable(8) [[TMP6]]) #[[ATTR7]]
+// CHECK3-NEXT:    call void @__kmpc_target_deinit()
+// CHECK3-NEXT:    ret void
+// CHECK3:       worker.exit:
+// CHECK3-NEXT:    ret void
+//
+//
+// CHECK3-LABEL: define {{[^@]+}}@_ZZN1S3fooEvENKUlvE_clEv
+// CHECK3-SAME: (ptr noundef nonnull align 8 dereferenceable(8) [[THIS:%.*]]) #[[ATTR2:[0-9]+]] comdat align 2 {
+// CHECK3-NEXT:  entry:
+// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[CLASS_ANON_1:%.*]], ptr [[THIS1]], i32 0, i32 0
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// CHECK3-NEXT:    [[A:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[TMP1]], i32 0, i32 0
+// CHECK3-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+// CHECK3-NEXT:    ret i32 [[TMP2]]
+//
+//
+// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29
+// CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR3]] {
+// CHECK3-NEXT:  entry:
+// CHECK3-NEXT:    [[DYN_PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[CAPTURED_VARS_ADDRS:%.*]] = alloca [2 x ptr], align 8
+// CHECK3-NEXT:    store ptr [[DYN_PTR]], ptr [[DYN_PTR_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
+// CHECK3-NEXT:    [[TMP2:%.*]] = call i32 @__kmpc_target_init(ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_kernel_environment, ptr [[DYN_PTR]])
+// CHECK3-NEXT:    [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP2]], -1
+// CHECK3-NEXT:    br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
+// CHECK3:       user_code.entry:
+// CHECK3-NEXT:    [[TMP3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
+// CHECK3-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK3-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [2 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 0
+// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP5]], align 8
+// CHECK3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [2 x ptr], ptr [[CAPTURED_VARS_ADDRS]], i64 0, i64 1
+// CHECK3-NEXT:    store ptr [[TMP4]], ptr [[TMP6]], align 8
+// CHECK3-NEXT:    call void @__kmpc_parallel_51(ptr @[[GLOB1]], i32 [[TMP3]], i32 1, i32 -1, i32 -1, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_omp_outlined, ptr null, ptr [[CAPTURED_VARS_ADDRS]], i64 2)
+// CHECK3-NEXT:    call void @__kmpc_target_deinit()
+// CHECK3-NEXT:    ret void
+// CHECK3:       worker.exit:
+// CHECK3-NEXT:    ret void
+//
+//
+// CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__ZN1S3fooEv_l29_omp_outlined
+// CHECK3-SAME: (ptr noalias noundef [[DOTGLOBAL_TID_:%.*]], ptr noalias noundef [[DOTBOUND_TID_:%.*]], ptr noundef [[THIS:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[L:%.*]]) #[[ATTR4]] {
+// CHECK3-NEXT:  entry:
+// CHECK3-NEXT:    [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[L_ADDR:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    [[L1:%.*]] = alloca [[CLASS_ANON_1:%.*]], align 8
+// CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
+// CHECK3-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[L]], ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK3-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[L_ADDR]], align 8
+// CHECK3-NEXT:    store ptr [[TMP1]], ptr [[TMP]], align 8
+// CHECK3-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK3-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[L1]], ptr align 8 [[TMP2]], i64 8, i1 false)
+// CHECK3-NEXT:    store ptr [[L1]], ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[CLASS_ANON_1]], ptr [[TMP3]], i32 0, i32 0
+// CHECK3-NEXT:    store ptr [[TMP0]], ptr [[TMP4]], align 8
+// CHECK3-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK3-NEXT:    [[CALL:%.*]] = call noundef i32 @_ZZN1S3fooEvENKUlvE_clEv(ptr noundef nonnull align 8 dereferenceable(8) [[TMP5]]) #[[ATTR7]]
+// CHECK3-NEXT:    ret void
+//
+//
 // CHECK3-LABEL: define {{[^@]+}}@{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooIZN1S3fooEvEUlvE_EiRKT__l18
 // CHECK3-SAME: (ptr noalias noundef [[DYN_PTR:%.*]], ptr noundef nonnull align 8 dereferenceable(8) [[T:%.*]]) #[[ATTR3]] {
 // CHECK3-NEXT:  entry:
@@ -1500,7 +1500,7 @@ int main(int argc, char **argv) {
 // CHECK3-NEXT:    [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[T_ADDR:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    [[TMP:%.*]] = alloca ptr, align 8
-// CHECK3-NEXT:    [[T1:%.*]] = alloca [[CLASS_ANON:%.*]], align 8
+// CHECK3-NEXT:    [[T1:%.*]] = alloca [[CLASS_ANON_1:%.*]], align 8
 // CHECK3-NEXT:    [[_TMP2:%.*]] = alloca ptr, align 8
 // CHECK3-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK3-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
-- 
GitLab


From 5c35b63da3c2e3c773580f748129fdec76354ede Mon Sep 17 00:00:00 2001
From: William G Hatch <william@hatch.uno>
Date: Thu, 16 May 2024 00:24:56 -0600
Subject: [PATCH 0966/1206] [MLIR][LLVM] add dwarfAddressSpace to DIDerivedType
 (#92043)

This field is present in LLVM, but was missing from the MLIR wrapper
type. This addition allows MLIR languages to add proper DWARF info for
GPU programs.
---
 mlir/include/mlir-c/Dialect/LLVM.h               |  9 +++++++--
 mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td |  1 +
 mlir/lib/CAPI/Dialect/LLVM.cpp                   | 14 ++++++++------
 mlir/lib/Target/LLVMIR/DebugImporter.cpp         |  2 +-
 mlir/lib/Target/LLVMIR/DebugTranslation.cpp      |  2 +-
 mlir/test/CAPI/llvm.c                            |  8 +++++++-
 mlir/test/Dialect/LLVMIR/debuginfo.mlir          | 11 +++++++++--
 mlir/test/Target/LLVMIR/Import/debug-info.ll     |  6 ++++--
 mlir/test/Target/LLVMIR/llvmir-debug.mlir        | 10 ++++++++--
 9 files changed, 46 insertions(+), 17 deletions(-)

diff --git a/mlir/include/mlir-c/Dialect/LLVM.h b/mlir/include/mlir-c/Dialect/LLVM.h
index b3e64bd68f7b..446d364342ac 100644
--- a/mlir/include/mlir-c/Dialect/LLVM.h
+++ b/mlir/include/mlir-c/Dialect/LLVM.h
@@ -241,11 +241,16 @@ MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDICompositeTypeAttrGet(
     MlirAttribute baseType, int64_t flags, uint64_t sizeInBits,
     uint64_t alignInBits, intptr_t nElements, MlirAttribute const *elements);
 
-/// Creates a LLVM DIDerivedType attribute.
+/// Creates a LLVM DIDerivedType attribute.  Note that `dwarfAddressSpace` is an
+/// optional field, where `MLIR_CAPI_DWARF_ADDRESS_SPACE_NULL` indicates null
+/// and non-negative values indicate a value present.
 MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIDerivedTypeAttrGet(
     MlirContext ctx, unsigned int tag, MlirAttribute name,
     MlirAttribute baseType, uint64_t sizeInBits, uint32_t alignInBits,
-    uint64_t offsetInBits, MlirAttribute extraData);
+    uint64_t offsetInBits, int64_t dwarfAddressSpace, MlirAttribute extraData);
+
+/// Constant to represent std::nullopt for dwarfAddressSpace to omit the field.
+#define MLIR_CAPI_DWARF_ADDRESS_SPACE_NULL -1
 
 /// Gets the base type from a LLVM DIDerivedType attribute.
 MLIR_CAPI_EXPORTED MlirAttribute
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index cc849cb7c978..8965f4f652a2 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -418,6 +418,7 @@ def LLVM_DIDerivedTypeAttr : LLVM_Attr<"DIDerivedType", "di_derived_type",
     OptionalParameter<"uint64_t">:$sizeInBits,
     OptionalParameter<"uint32_t">:$alignInBits,
     OptionalParameter<"uint64_t">:$offsetInBits,
+    OptionalParameter<"std::optional<unsigned>">:$dwarfAddressSpace,
     OptionalParameter<"DINodeAttr">:$extraData
   );
   let assemblyFormat = "`<` struct(params) `>`";
diff --git a/mlir/lib/CAPI/Dialect/LLVM.cpp b/mlir/lib/CAPI/Dialect/LLVM.cpp
index 108ebe5367d5..9a28749631f5 100644
--- a/mlir/lib/CAPI/Dialect/LLVM.cpp
+++ b/mlir/lib/CAPI/Dialect/LLVM.cpp
@@ -176,15 +176,17 @@ MlirAttribute mlirLLVMDICompositeTypeAttrGet(
                           [](Attribute a) { return cast<DINodeAttr>(a); })));
 }
 
-MlirAttribute
-mlirLLVMDIDerivedTypeAttrGet(MlirContext ctx, unsigned int tag,
-                             MlirAttribute name, MlirAttribute baseType,
-                             uint64_t sizeInBits, uint32_t alignInBits,
-                             uint64_t offsetInBits, MlirAttribute extraData) {
+MlirAttribute mlirLLVMDIDerivedTypeAttrGet(
+    MlirContext ctx, unsigned int tag, MlirAttribute name,
+    MlirAttribute baseType, uint64_t sizeInBits, uint32_t alignInBits,
+    uint64_t offsetInBits, int64_t dwarfAddressSpace, MlirAttribute extraData) {
+  std::optional<unsigned> addressSpace = std::nullopt;
+  if (dwarfAddressSpace >= 0)
+    addressSpace = (unsigned)dwarfAddressSpace;
   return wrap(DIDerivedTypeAttr::get(
       unwrap(ctx), tag, cast<StringAttr>(unwrap(name)),
       cast<DITypeAttr>(unwrap(baseType)), sizeInBits, alignInBits, offsetInBits,
-      cast<DINodeAttr>(unwrap(extraData))));
+      addressSpace, cast<DINodeAttr>(unwrap(extraData))));
 }
 
 MlirAttribute
diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
index 1ab55b079b52..b40bfa4f4dbb 100644
--- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
@@ -106,7 +106,7 @@ DIDerivedTypeAttr DebugImporter::translateImpl(llvm::DIDerivedType *node) {
   return DIDerivedTypeAttr::get(
       context, node->getTag(), getStringAttrOrNull(node->getRawName()),
       baseType, node->getSizeInBits(), node->getAlignInBits(),
-      node->getOffsetInBits(), extraData);
+      node->getOffsetInBits(), node->getDWARFAddressSpace(), extraData);
 }
 
 DIFileAttr DebugImporter::translateImpl(llvm::DIFile *node) {
diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
index 2aa1b6b85ac0..815943c40bde 100644
--- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
@@ -165,7 +165,7 @@ llvm::DIDerivedType *DebugTranslation::translateImpl(DIDerivedTypeAttr attr) {
       /*File=*/nullptr, /*Line=*/0,
       /*Scope=*/nullptr, translate(attr.getBaseType()), attr.getSizeInBits(),
       attr.getAlignInBits(), attr.getOffsetInBits(),
-      /*DWARFAddressSpace=*/std::nullopt, /*PtrAuthData=*/std::nullopt,
+      attr.getDwarfAddressSpace(), /*PtrAuthData=*/std::nullopt,
       /*Flags=*/llvm::DINode::FlagZero, translate(attr.getExtraData()));
 }
 
diff --git a/mlir/test/CAPI/llvm.c b/mlir/test/CAPI/llvm.c
index 25f900e521cf..0c589a45f9df 100644
--- a/mlir/test/CAPI/llvm.c
+++ b/mlir/test/CAPI/llvm.c
@@ -296,8 +296,14 @@ static void testDebugInfoAttributes(MlirContext ctx) {
   mlirAttributeDump(mlirLLVMDILocalVariableAttrGet(ctx, compile_unit, foo, file,
                                                    1, 0, 8, di_type));
   // CHECK: #llvm.di_derived_type<{{.*}}>
+  // CHECK-NOT: dwarfAddressSpace
+  mlirAttributeDump(mlirLLVMDIDerivedTypeAttrGet(
+      ctx, 0, bar, di_type, 64, 8, 0, MLIR_CAPI_DWARF_ADDRESS_SPACE_NULL,
+      di_type));
+
+  // CHECK: #llvm.di_derived_type<{{.*}} dwarfAddressSpace = 3{{.*}}>
   mlirAttributeDump(
-      mlirLLVMDIDerivedTypeAttrGet(ctx, 0, bar, di_type, 64, 8, 0, di_type));
+      mlirLLVMDIDerivedTypeAttrGet(ctx, 0, bar, di_type, 64, 8, 0, 3, di_type));
 
   // CHECK: #llvm.di_composite_type<{{.*}}>
   mlirAttributeDump(mlirLLVMDICompositeTypeAttrGet(
diff --git a/mlir/test/Dialect/LLVMIR/debuginfo.mlir b/mlir/test/Dialect/LLVMIR/debuginfo.mlir
index 94bb2bb06229..6d9b7799c391 100644
--- a/mlir/test/Dialect/LLVMIR/debuginfo.mlir
+++ b/mlir/test/Dialect/LLVMIR/debuginfo.mlir
@@ -37,6 +37,13 @@
   tag = DW_TAG_pointer_type, name = "ptr1"
 >
 
+// CHECK-DAG: #[[PTR2:.*]] = #llvm.di_derived_type<tag = DW_TAG_pointer_type, baseType = #[[INT0]], sizeInBits = 64, alignInBits = 32, offsetInBits = 4, dwarfAddressSpace = 3, extraData = #[[INT1]]>
+#ptr2 = #llvm.di_derived_type<
+  tag = DW_TAG_pointer_type, baseType = #int0,
+  sizeInBits = 64, alignInBits = 32, offsetInBits = 4,
+  dwarfAddressSpace = 3, extraData = #int1
+>
+
 // CHECK-DAG: #[[COMP0:.*]] = #llvm.di_composite_type<tag = DW_TAG_array_type, name = "array0", line = 10, sizeInBits = 128, alignInBits = 32>
 #comp0 = #llvm.di_composite_type<
   tag = DW_TAG_array_type, name = "array0",
@@ -73,9 +80,9 @@
   flags = "TypePassByReference|NonTrivial"
 >
 
-// CHECK-DAG: #[[SPTYPE0:.*]] = #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #[[NULL]], #[[INT0]], #[[PTR0]], #[[PTR1]], #[[COMP0:.*]], #[[COMP1:.*]], #[[COMP2:.*]]>
+// CHECK-DAG: #[[SPTYPE0:.*]] = #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #[[NULL]], #[[INT0]], #[[PTR0]], #[[PTR1]], #[[PTR2]], #[[COMP0:.*]], #[[COMP1:.*]], #[[COMP2:.*]]>
 #spType0 = #llvm.di_subroutine_type<
-  callingConvention = DW_CC_normal, types = #null, #int0, #ptr0, #ptr1, #comp0, #comp1, #comp2
+  callingConvention = DW_CC_normal, types = #null, #int0, #ptr0, #ptr1, #ptr2, #comp0, #comp1, #comp2
 >
 
 // CHECK-DAG: #[[SPTYPE1:.*]] = #llvm.di_subroutine_type<types = #[[INT1]], #[[INT1]]>
diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll
index 245cf300d2c1..72c828448d08 100644
--- a/mlir/test/Target/LLVMIR/Import/debug-info.ll
+++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll
@@ -137,7 +137,8 @@ define void @basic_type() !dbg !3 {
 ; CHECK: #[[INT:.+]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "int">
 ; CHECK: #[[PTR1:.+]] = #llvm.di_derived_type<tag = DW_TAG_pointer_type, baseType = #[[INT]]>
 ; CHECK: #[[PTR2:.+]] = #llvm.di_derived_type<tag = DW_TAG_pointer_type, name = "mypointer", baseType = #[[INT]], sizeInBits = 64, alignInBits = 32, offsetInBits = 4, extraData = #[[INT]]>
-; CHECK: #llvm.di_subroutine_type<types = #[[PTR1]], #[[PTR2]]>
+; CHECK: #[[PTR3:.+]] = #llvm.di_derived_type<tag = DW_TAG_pointer_type, baseType = #[[INT]], dwarfAddressSpace = 3>
+; CHECK: #llvm.di_subroutine_type<types = #[[PTR1]], #[[PTR2]], #[[PTR3]]>
 
 define void @derived_type() !dbg !3 {
   ret void
@@ -150,10 +151,11 @@ define void @derived_type() !dbg !3 {
 !2 = !DIFile(filename: "debug-info.ll", directory: "/")
 !3 = distinct !DISubprogram(name: "derived_type", scope: !2, file: !2, spFlags: DISPFlagDefinition, unit: !1, type: !4)
 !4 = !DISubroutineType(types: !5)
-!5 = !{!7, !8}
+!5 = !{!7, !8, !9}
 !6 = !DIBasicType(name: "int")
 !7 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !6)
 !8 = !DIDerivedType(name: "mypointer", tag: DW_TAG_pointer_type, baseType: !6, size: 64, align: 32, offset: 4, extraData: !6)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !6, dwarfAddressSpace: 3)
 
 ; // -----
 
diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
index 15d4b8ccd88b..1cb94bca169d 100644
--- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
@@ -35,6 +35,11 @@ llvm.func @func_no_debug() {
   // Specify the name parameter.
   tag = DW_TAG_pointer_type, name = "named", baseType = #si32
 >
+#ptrWithAddressSpace = #llvm.di_derived_type<
+  tag = DW_TAG_pointer_type, baseType = #si32,
+  sizeInBits = 64, alignInBits = 32, offsetInBits = 8,
+  dwarfAddressSpace = 3
+>
 #cu = #llvm.di_compile_unit<
   id = distinct[0]<>, sourceLanguage = DW_LANG_C, file = #file,
   producer = "MLIR", isOptimized = true, emissionKind = Full,
@@ -51,7 +56,7 @@ llvm.func @func_no_debug() {
   elements = #llvm.di_subrange<lowerBound = 0, upperBound = 4, stride = 1>
 >
 #null = #llvm.di_null_type
-#spType0 = #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #null, #si64, #ptr, #named, #composite, #vector>
+#spType0 = #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #null, #si64, #ptr, #named, #ptrWithAddressSpace, #composite, #vector>
 #toplevel_namespace = #llvm.di_namespace<
   name = "toplevel", exportSymbols = true
 >
@@ -138,11 +143,12 @@ llvm.func @empty_types() {
 // CHECK: ![[NESTED_NAMESPACE]] = !DINamespace(name: "nested", scope: ![[TOPLEVEL_NAMESPACE:.*]])
 // CHECK: ![[TOPLEVEL_NAMESPACE]] = !DINamespace(name: "toplevel", scope: null, exportSymbols: true)
 // CHECK: ![[FUNC_TYPE]] = !DISubroutineType(cc: DW_CC_normal, types: ![[FUNC_ARGS:.*]])
-// CHECK: ![[FUNC_ARGS]] = !{null, ![[ARG_TYPE:.*]], ![[PTR_TYPE:.*]], ![[NAMED_TYPE:.*]], ![[COMPOSITE_TYPE:.*]], ![[VECTOR_TYPE:.*]]}
+// CHECK: ![[FUNC_ARGS]] = !{null, ![[ARG_TYPE:.*]], ![[PTR_TYPE:.*]], ![[NAMED_TYPE:.*]], ![[PTR_WITH_ADDR_SPACE:.*]], ![[COMPOSITE_TYPE:.*]], ![[VECTOR_TYPE:.*]]}
 // CHECK: ![[ARG_TYPE]] = !DIBasicType(name: "si64")
 // CHECK: ![[PTR_TYPE]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[BASE_TYPE:.*]], size: 64, align: 32, offset: 8, extraData: ![[BASE_TYPE]])
 // CHECK: ![[BASE_TYPE]] = !DIBasicType(name: "si32", size: 32, encoding: DW_ATE_signed)
 // CHECK: ![[NAMED_TYPE]] = !DIDerivedType(tag: DW_TAG_pointer_type, name: "named", baseType: ![[BASE_TYPE:.*]])
+// CHECK: ![[PTR_WITH_ADDR_SPACE]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[BASE_TYPE:.*]], size: 64, align: 32, offset: 8, dwarfAddressSpace: 3)
 // CHECK: ![[COMPOSITE_TYPE]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "composite", file: ![[CU_FILE_LOC]], line: 42, size: 64, align: 32, elements: ![[COMPOSITE_ELEMENTS:.*]])
 // CHECK: ![[COMPOSITE_ELEMENTS]] = !{![[COMPOSITE_ELEMENT:.*]]}
 // CHECK: ![[COMPOSITE_ELEMENT]] = !DISubrange(count: 4)
-- 
GitLab


From 6706aebefac3c89d8df55bade022f3aeb55cfeb9 Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Wed, 15 May 2024 23:26:07 -0700
Subject: [PATCH 0967/1206] Revert "[flang] Add ETIME runtime and lowering
 intrinsics implementation" (#92354)

Reverts llvm/llvm-project#90578

This broke the premerge linux buildbot.
---
 etime-function.mlir                           | 25 -----
 flang/docs/Intrinsics.md                      | 52 ----------
 .../flang/Optimizer/Builder/IntrinsicCall.h   | 12 +--
 .../Optimizer/Builder/Runtime/Intrinsics.h    |  2 -
 flang/include/flang/Runtime/time-intrinsic.h  |  3 -
 flang/lib/Evaluate/intrinsics.cpp             | 26 +----
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 99 -------------------
 .../Optimizer/Builder/Runtime/Intrinsics.cpp  | 14 ---
 flang/runtime/time-intrinsic.cpp              | 70 +------------
 flang/runtime/tools.h                         |  9 --
 .../test/Lower/Intrinsics/etime-function.f90  | 24 -----
 flang/test/Lower/Intrinsics/etime.f90         | 21 ----
 flang/test/Semantics/etime.f90                | 30 ------
 13 files changed, 5 insertions(+), 382 deletions(-)
 delete mode 100644 etime-function.mlir
 delete mode 100644 flang/test/Lower/Intrinsics/etime-function.f90
 delete mode 100644 flang/test/Lower/Intrinsics/etime.f90
 delete mode 100644 flang/test/Semantics/etime.f90

diff --git a/etime-function.mlir b/etime-function.mlir
deleted file mode 100644
index 740dfd4866aa..000000000000
--- a/etime-function.mlir
+++ /dev/null
@@ -1,25 +0,0 @@
-module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
-  func.func @_QPetime_test(%arg0: !fir.ref<!fir.array<2xf32>> {fir.bindc_name = "values"}, %arg1: !fir.ref<f32> {fir.bindc_name = "time"}) {
-    %c9_i32 = arith.constant 9 : i32
-    %c2 = arith.constant 2 : index
-    %0 = fir.alloca f32
-    %1 = fir.declare %arg1 {uniq_name = "_QFetime_testEtime"} : (!fir.ref<f32>) -> !fir.ref<f32>
-    %2 = fir.shape %c2 : (index) -> !fir.shape<1>
-    %3 = fir.declare %arg0(%2) {uniq_name = "_QFetime_testEvalues"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<2xf32>>
-    %4 = fir.embox %3(%2) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
-    %5 = fir.embox %0 : (!fir.ref<f32>) -> !fir.box<f32>
-    %6 = fir.address_of(@_QQclX116781708dcf8f012d7ec1e40d743d97) : !fir.ref<!fir.char<1,71>>
-    %7 = fir.convert %4 : (!fir.box<!fir.array<2xf32>>) -> !fir.box<none>
-    %8 = fir.convert %5 : (!fir.box<f32>) -> !fir.box<none>
-    %9 = fir.convert %6 : (!fir.ref<!fir.char<1,71>>) -> !fir.ref<i8>
-    %10 = fir.call @_FortranAEtime(%7, %8, %9, %c9_i32) fastmath<contract> : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
-    %11 = fir.load %0 : !fir.ref<f32>
-    fir.store %11 to %1 : !fir.ref<f32>
-    return
-  }
-  func.func private @_FortranAEtime(!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none attributes {fir.runtime}
-  fir.global linkonce @_QQclX116781708dcf8f012d7ec1e40d743d97 constant : !fir.char<1,71> {
-    %0 = fir.string_lit "/home/jump/llvm-project/flang/test/Lower/Intrinsics/etime-function.f90\00"(71) : !fir.char<1,71>
-    fir.has_value %0 : !fir.char<1,71>
-  }
-}
diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index 41129b10083b..848619cb65d9 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -916,55 +916,3 @@ used in constant expressions have currently no folding support at all.
   - If a condition occurs that would assign a nonzero value to `CMDSTAT` but the `CMDSTAT` variable is not present, error termination is initiated.
     - On POSIX-compatible systems, the child process (async process) will be terminated with no effect on the parent process (continues).
     - On Windows, error termination is not initiated.
-
-### Non-Standard Intrinsics: ETIME
-
-#### Description
-`ETIME(VALUES, TIME)` returns the number of seconds of runtime since the start of the process’s execution in *TIME*. *VALUES* returns the user and system components of this time in `VALUES(1)` and `VALUES(2)` respectively. *TIME* is equal to `VALUES(1) + VALUES(2)`.
-
-On some systems, the underlying timings are represented using types with sufficiently small limits that overflows (wrap around) are possible, such as 32-bit types. Therefore, the values returned by this intrinsic might be, or become, negative, or numerically less than previous values, during a single run of the compiled program.
-
-This intrinsic is provided in both subroutine and function forms; however, only one form can be used in any given program unit.
-
-*VALUES* and *TIME* are `INTENT(OUT)` and provide the following:
-
-
-|               |                                   |
-|---------------|-----------------------------------|
-| `VALUES(1)`   | User time in seconds.             |
-| `VALUES(2)`   | System time in seconds.           |
-| `TIME`        | Run time since start in seconds.  |
-
-#### Usage and Info
-
-- **Standard:** GNU extension
-- **Class:** Subroutine, function
-- **Syntax:** `CALL ETIME(VALUES, TIME)`
-- **Arguments:**
-- **Return value** Elapsed time in seconds since the start of program execution.
-
-| Argument   | Description                                                           |
-|------------|-----------------------------------------------------------------------|
-| `VALUES`   | The type shall be REAL(4), DIMENSION(2).                              |
-| `TIME`     | The type shall be REAL(4).                                            |
-
-#### Example
-Here is an example usage from [Gfortran ETIME](https://gcc.gnu.org/onlinedocs/gfortran/ETIME.html)
-```Fortran
-program test_etime
-    integer(8) :: i, j
-    real, dimension(2) :: tarray
-    real :: result
-    call ETIME(tarray, result)
-    print *, result
-    print *, tarray(1)
-    print *, tarray(2)   
-    do i=1,100000000    ! Just a delay
-        j = i * i - i
-    end do
-    call ETIME(tarray, result)
-    print *, result
-    print *, tarray(1)
-    print *, tarray(2)
-end program test_etime
-```
\ No newline at end of file
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index 977a69af5281..b7d060926761 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -222,8 +222,6 @@ struct IntrinsicLibrary {
   fir::ExtendedValue genEoshift(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   void genExit(llvm::ArrayRef<fir::ExtendedValue>);
   void genExecuteCommandLine(mlir::ArrayRef<fir::ExtendedValue> args);
-  fir::ExtendedValue genEtime(std::optional<mlir::Type>,
-                              mlir::ArrayRef<fir::ExtendedValue> args);
   mlir::Value genExponent(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genExtendsTypeOf(mlir::Type,
                                       llvm::ArrayRef<fir::ExtendedValue>);
@@ -402,10 +400,8 @@ struct IntrinsicLibrary {
   using ElementalGenerator = decltype(&IntrinsicLibrary::genAbs);
   using ExtendedGenerator = decltype(&IntrinsicLibrary::genLenTrim);
   using SubroutineGenerator = decltype(&IntrinsicLibrary::genDateAndTime);
-  /// The generator for intrinsic that has both function and subroutine form.
-  using DualGenerator = decltype(&IntrinsicLibrary::genEtime);
-  using Generator = std::variant<ElementalGenerator, ExtendedGenerator,
-                                 SubroutineGenerator, DualGenerator>;
+  using Generator =
+      std::variant<ElementalGenerator, ExtendedGenerator, SubroutineGenerator>;
 
   /// All generators can be outlined. This will build a function named
   /// "fir."+ <generic name> + "." + <result type code> and generate the
@@ -446,10 +442,6 @@ struct IntrinsicLibrary {
                               llvm::ArrayRef<mlir::Value> args);
   mlir::Value invokeGenerator(SubroutineGenerator generator,
                               llvm::ArrayRef<mlir::Value> args);
-  mlir::Value invokeGenerator(DualGenerator generator,
-                              llvm::ArrayRef<mlir::Value> args);
-  mlir::Value invokeGenerator(DualGenerator generator, mlir::Type resultType,
-                              llvm::ArrayRef<mlir::Value> args);
 
   /// Get pointer to unrestricted intrinsic. Generate the related unrestricted
   /// intrinsic if it is not defined yet.
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
index 7497a4bc3564..737c631e45c1 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
@@ -44,8 +44,6 @@ void genDateAndTime(fir::FirOpBuilder &, mlir::Location,
                     std::optional<fir::CharBoxValue> date,
                     std::optional<fir::CharBoxValue> time,
                     std::optional<fir::CharBoxValue> zone, mlir::Value values);
-void genEtime(fir::FirOpBuilder &builder, mlir::Location loc,
-              mlir::Value values, mlir::Value time);
 
 void genRandomInit(fir::FirOpBuilder &, mlir::Location, mlir::Value repeatable,
                    mlir::Value imageDistinct);
diff --git a/flang/include/flang/Runtime/time-intrinsic.h b/flang/include/flang/Runtime/time-intrinsic.h
index 80490a17e455..650c02436ee4 100644
--- a/flang/include/flang/Runtime/time-intrinsic.h
+++ b/flang/include/flang/Runtime/time-intrinsic.h
@@ -43,9 +43,6 @@ void RTNAME(DateAndTime)(char *date, std::size_t dateChars, char *time,
     const char *source = nullptr, int line = 0,
     const Descriptor *values = nullptr);
 
-void RTNAME(Etime)(const Descriptor *values, const Descriptor *time,
-    const char *sourceFile, int line);
-
 } // extern "C"
 } // namespace Fortran::runtime
 #endif // FORTRAN_RUNTIME_TIME_INTRINSIC_H_
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index ded277877f49..441a762c930d 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -454,10 +454,6 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"erf", {{"x", SameReal}}, SameReal},
     {"erfc", {{"x", SameReal}}, SameReal},
     {"erfc_scaled", {{"x", SameReal}}, SameReal},
-    {"etime",
-        {{"values", TypePattern{RealType, KindCode::exactKind, 4}, Rank::vector,
-            Optionality::required, common::Intent::Out}},
-        TypePattern{RealType, KindCode::exactKind, 4}},
     {"exp", {{"x", SameFloating}}, SameFloating},
     {"exp", {{"x", SameFloating}}, SameFloating},
     {"exponent", {{"x", AnyReal}}, DefaultInt},
@@ -1346,12 +1342,6 @@ static const IntrinsicInterface intrinsicSubroutine[]{
             {"values", AnyInt, Rank::vector, Optionality::optional,
                 common::Intent::Out}},
         {}, Rank::elemental, IntrinsicClass::impureSubroutine},
-    {"etime",
-        {{"values", TypePattern{RealType, KindCode::exactKind, 4}, Rank::vector,
-             Optionality::required, common::Intent::Out},
-            {"time", TypePattern{RealType, KindCode::exactKind, 4},
-                Rank::scalar, Optionality::required, common::Intent::Out}},
-        {}, Rank::elemental, IntrinsicClass::impureSubroutine},
     {"execute_command_line",
         {{"command", DefaultChar, Rank::scalar},
             {"wait", AnyLogical, Rank::scalar, Optionality::optional},
@@ -2494,7 +2484,6 @@ public:
   bool IsIntrinsic(const std::string &) const;
   bool IsIntrinsicFunction(const std::string &) const;
   bool IsIntrinsicSubroutine(const std::string &) const;
-  bool IsDualIntrinsic(const std::string &) const;
 
   IntrinsicClass GetIntrinsicClass(const std::string &) const;
   std::string GetGenericIntrinsicName(const std::string &) const;
@@ -2556,17 +2545,6 @@ bool IntrinsicProcTable::Implementation::IsIntrinsic(
     const std::string &name) const {
   return IsIntrinsicFunction(name) || IsIntrinsicSubroutine(name);
 }
-bool IntrinsicProcTable::Implementation::IsDualIntrinsic(
-    const std::string &name) const {
-  // Collection for some intrinsics with function and subroutine form,
-  // in order to pass the semantic check.
-  static const std::string dualIntrinsic[]{{"etime"}};
-
-  return std::find_if(std::begin(dualIntrinsic), std::end(dualIntrinsic),
-             [&name](const std::string &dualName) {
-               return dualName == name;
-             }) != std::end(dualIntrinsic);
-}
 
 IntrinsicClass IntrinsicProcTable::Implementation::GetIntrinsicClass(
     const std::string &name) const {
@@ -3105,7 +3083,7 @@ std::optional<SpecificCall> IntrinsicProcTable::Implementation::Probe(
         return specificCall;
       }
     }
-    if (IsIntrinsicFunction(call.name) && !IsDualIntrinsic(call.name)) {
+    if (IsIntrinsicFunction(call.name)) {
       context.messages().Say(
           "Cannot use intrinsic function '%s' as a subroutine"_err_en_US,
           call.name);
@@ -3240,7 +3218,7 @@ std::optional<SpecificCall> IntrinsicProcTable::Implementation::Probe(
   }
 
   if (specificBuffer.empty() && genericBuffer.empty() &&
-      IsIntrinsicSubroutine(call.name) && !IsDualIntrinsic(call.name)) {
+      IsIntrinsicSubroutine(call.name)) {
     context.messages().Say(
         "Cannot use intrinsic subroutine '%s' as a function"_err_en_US,
         call.name);
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index ae7e65098744..58064d23eb08 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -35,7 +35,6 @@
 #include "flang/Optimizer/Builder/Runtime/Stop.h"
 #include "flang/Optimizer/Builder/Runtime/Transformational.h"
 #include "flang/Optimizer/Builder/Todo.h"
-#include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Support/FatalError.h"
@@ -50,7 +49,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
-#include <mlir/IR/Value.h>
 #include <optional>
 
 #define DEBUG_TYPE "flang-lower-intrinsic"
@@ -224,10 +222,6 @@ static constexpr IntrinsicHandler handlers[]{
        {"boundary", asBox, handleDynamicOptional},
        {"dim", asValue}}},
      /*isElemental=*/false},
-    {"etime",
-     &I::genEtime,
-     {{{"values", asBox}, {"time", asBox}}},
-     /*isElemental=*/false},
     {"execute_command_line",
      &I::genExecuteCommandLine,
      {{{"command", asBox},
@@ -1688,24 +1682,6 @@ IntrinsicLibrary::genElementalCall<IntrinsicLibrary::SubroutineGenerator>(
   return mlir::Value();
 }
 
-template <>
-fir::ExtendedValue
-IntrinsicLibrary::genElementalCall<IntrinsicLibrary::DualGenerator>(
-    DualGenerator generator, llvm::StringRef name, mlir::Type resultType,
-    llvm::ArrayRef<fir::ExtendedValue> args, bool outline) {
-  assert(resultType.getImpl() && "expect elemental intrinsic to be functions");
-
-  for (const fir::ExtendedValue &arg : args)
-    if (!arg.getUnboxed() && !arg.getCharBox())
-      // fir::emitFatalError(loc, "nonscalar intrinsic argument");
-      crashOnMissingIntrinsic(loc, name);
-  if (outline)
-    return outlineInExtendedWrapper(generator, name, resultType, args);
-
-  return std::invoke(generator, *this, std::optional<mlir::Type>{resultType},
-                     args);
-}
-
 static fir::ExtendedValue
 invokeHandler(IntrinsicLibrary::ElementalGenerator generator,
               const IntrinsicHandler &handler,
@@ -1749,22 +1725,6 @@ invokeHandler(IntrinsicLibrary::SubroutineGenerator generator,
   return mlir::Value{};
 }
 
-static fir::ExtendedValue
-invokeHandler(IntrinsicLibrary::DualGenerator generator,
-              const IntrinsicHandler &handler,
-              std::optional<mlir::Type> resultType,
-              llvm::ArrayRef<fir::ExtendedValue> args, bool outline,
-              IntrinsicLibrary &lib) {
-  if (handler.isElemental)
-    return lib.genElementalCall(generator, handler.name, mlir::Type{}, args,
-                                outline);
-  if (outline)
-    return lib.outlineInExtendedWrapper(generator, handler.name, resultType,
-                                        args);
-
-  return std::invoke(generator, lib, resultType, args);
-}
-
 std::pair<fir::ExtendedValue, bool>
 IntrinsicLibrary::genIntrinsicCall(llvm::StringRef specificName,
                                    std::optional<mlir::Type> resultType,
@@ -1860,34 +1820,6 @@ IntrinsicLibrary::invokeGenerator(SubroutineGenerator generator,
   return {};
 }
 
-mlir::Value
-IntrinsicLibrary::invokeGenerator(DualGenerator generator,
-                                  llvm::ArrayRef<mlir::Value> args) {
-  llvm::SmallVector<fir::ExtendedValue> extendedArgs;
-  for (mlir::Value arg : args)
-    extendedArgs.emplace_back(toExtendedValue(arg, builder, loc));
-  std::invoke(generator, *this, std::optional<mlir::Type>{}, extendedArgs);
-  return {};
-}
-
-mlir::Value
-IntrinsicLibrary::invokeGenerator(DualGenerator generator,
-                                  mlir::Type resultType,
-                                  llvm::ArrayRef<mlir::Value> args) {
-  llvm::SmallVector<fir::ExtendedValue> extendedArgs;
-  for (mlir::Value arg : args)
-    extendedArgs.emplace_back(toExtendedValue(arg, builder, loc));
-
-  if (resultType.getImpl() == nullptr) {
-    // TODO:
-    assert(false && "result type is null");
-  }
-
-  auto extendedResult = std::invoke(
-      generator, *this, std::optional<mlir::Type>{resultType}, extendedArgs);
-  return toValue(extendedResult, builder, loc);
-}
-
 //===----------------------------------------------------------------------===//
 // Intrinsic Procedure Mangling
 //===----------------------------------------------------------------------===//
@@ -3303,37 +3235,6 @@ void IntrinsicLibrary::genExecuteCommandLine(
                                       exitstatBox, cmdstatBox, cmdmsgBox);
 }
 
-// ETIME
-fir::ExtendedValue
-IntrinsicLibrary::genEtime(std::optional<mlir::Type> resultType,
-                           llvm::ArrayRef<fir::ExtendedValue> args) {
-  assert((args.size() == 2 && !resultType.has_value()) ||
-         (args.size() == 1 && resultType.has_value()));
-
-  mlir::Value values = fir::getBase(args[0]);
-  if (resultType.has_value()) {
-    // function form
-    if (!values)
-      fir::emitFatalError(loc, "expected VALUES parameter");
-
-    auto timeAddr = builder.createTemporary(loc, *resultType);
-    auto timeBox = builder.createBox(loc, timeAddr);
-    fir::runtime::genEtime(builder, loc, values, timeBox);
-    return builder.create<fir::LoadOp>(loc, timeAddr);
-  } else {
-    // subroutine form
-    mlir::Value time = fir::getBase(args[1]);
-    if (!values)
-      fir::emitFatalError(loc, "expected VALUES parameter");
-    if (!time)
-      fir::emitFatalError(loc, "expected TIME parameter");
-
-    fir::runtime::genEtime(builder, loc, values, time);
-    return {};
-  }
-  return {};
-}
-
 // EXIT
 void IntrinsicLibrary::genExit(llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 1);
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 3f36d639861b..8b78a1688c73 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -106,20 +106,6 @@ void fir::runtime::genDateAndTime(fir::FirOpBuilder &builder,
   builder.create<fir::CallOp>(loc, callee, args);
 }
 
-void fir::runtime::genEtime(fir::FirOpBuilder &builder, mlir::Location loc,
-                            mlir::Value values, mlir::Value time) {
-  auto runtimeFunc = fir::runtime::getRuntimeFunc<mkRTKey(Etime)>(loc, builder);
-  mlir::FunctionType runtimeFuncTy = runtimeFunc.getFunctionType();
-
-  mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
-  mlir::Value sourceLine =
-      fir::factory::locationToLineNo(builder, loc, runtimeFuncTy.getInput(3));
-
-  llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
-      builder, loc, runtimeFuncTy, values, time, sourceFile, sourceLine);
-  builder.create<fir::CallOp>(loc, runtimeFunc, args);
-}
-
 void fir::runtime::genRandomInit(fir::FirOpBuilder &builder, mlir::Location loc,
                                  mlir::Value repeatable,
                                  mlir::Value imageDistinct) {
diff --git a/flang/runtime/time-intrinsic.cpp b/flang/runtime/time-intrinsic.cpp
index 989d4f804c5f..68d63253139f 100644
--- a/flang/runtime/time-intrinsic.cpp
+++ b/flang/runtime/time-intrinsic.cpp
@@ -19,12 +19,8 @@
 #include <cstdlib>
 #include <cstring>
 #include <ctime>
-#ifdef _WIN32
-#include "flang/Common/windows-include.h"
-#else
+#ifndef _WIN32
 #include <sys/time.h> // gettimeofday
-#include <sys/times.h>
-#include <unistd.h>
 #endif
 
 // CPU_TIME (Fortran 2018 16.9.57)
@@ -374,69 +370,5 @@ void RTNAME(DateAndTime)(char *date, std::size_t dateChars, char *time,
       terminator, date, dateChars, time, timeChars, zone, zoneChars, values);
 }
 
-void RTNAME(Etime)(const Descriptor *values, const Descriptor *time,
-    const char *sourceFile, int line) {
-  Fortran::runtime::Terminator terminator{sourceFile, line};
-
-  double usrTime = -1.0, sysTime = -1.0, realTime = -1.0;
-
-#ifdef _WIN32
-  FILETIME creationTime;
-  FILETIME exitTime;
-  FILETIME kernelTime;
-  FILETIME userTime;
-
-  if (GetProcessTimes(GetCurrentProcess(), &creationTime, &exitTime,
-          &kernelTime, &userTime) == 0) {
-    ULARGE_INTEGER userSystemTime;
-    ULARGE_INTEGER kernelSystemTime;
-
-    memcpy(&userSystemTime, &userTime, sizeof(FILETIME));
-    memcpy(&kernelSystemTime, &kernelTime, sizeof(FILETIME));
-
-    usrTime = ((double)(userSystemTime.QuadPart)) / 10000000.0;
-    sysTime = ((double)(kernelSystemTime.QuadPart)) / 10000000.0;
-    realTime = usrTime + sysTime;
-  }
-#else
-  struct tms tms;
-  if (times(&tms) != -1) {
-    usrTime = ((double)(tms.tms_utime)) / sysconf(_SC_CLK_TCK);
-    sysTime = ((double)(tms.tms_stime)) / sysconf(_SC_CLK_TCK);
-    realTime = usrTime + sysTime;
-  }
-#endif
-
-  if (values) {
-    auto typeCode{values->type().GetCategoryAndKind()};
-    // ETIME values argument must have decimal range == 2.
-    RUNTIME_CHECK(terminator,
-        values->rank() == 1 && values->GetDimension(0).Extent() == 2 &&
-            typeCode && typeCode->first == Fortran::common::TypeCategory::Real);
-    // Only accept KIND=4 here.
-    int kind{typeCode->second};
-    RUNTIME_CHECK(terminator, kind == 4);
-
-    ApplyFloatingPointKind<StoreFloatingPointAt, void>(
-        kind, terminator, *values, /* atIndex = */ 0, usrTime);
-    ApplyFloatingPointKind<StoreFloatingPointAt, void>(
-        kind, terminator, *values, /* atIndex = */ 1, sysTime);
-  }
-
-  if (time) {
-    auto typeCode{time->type().GetCategoryAndKind()};
-    // ETIME time argument must have decimal range == 0.
-    RUNTIME_CHECK(terminator,
-        time->rank() == 0 && typeCode &&
-            typeCode->first == Fortran::common::TypeCategory::Real);
-    // Only accept KIND=4 here.
-    int kind{typeCode->second};
-    RUNTIME_CHECK(terminator, kind == 4);
-
-    ApplyFloatingPointKind<StoreFloatingPointAt, void>(
-        kind, terminator, *time, /* atIndex = */ 0, realTime);
-  }
-}
-
 } // extern "C"
 } // namespace Fortran::runtime
diff --git a/flang/runtime/tools.h b/flang/runtime/tools.h
index dc12e5c4533e..52049c511f13 100644
--- a/flang/runtime/tools.h
+++ b/flang/runtime/tools.h
@@ -99,15 +99,6 @@ template <int KIND> struct StoreIntegerAt {
   }
 };
 
-// Helper to store floating value in result[at].
-template <int KIND> struct StoreFloatingPointAt {
-  RT_API_ATTRS void operator()(const Fortran::runtime::Descriptor &result,
-      std::size_t at, std::double_t value) const {
-    *result.ZeroBasedIndexedElement<Fortran::runtime::CppTypeFor<
-        Fortran::common::TypeCategory::Real, KIND>>(at) = value;
-  }
-};
-
 // Validate a KIND= argument
 RT_API_ATTRS void CheckIntegerKind(
     Terminator &, int kind, const char *intrinsic);
diff --git a/flang/test/Lower/Intrinsics/etime-function.f90 b/flang/test/Lower/Intrinsics/etime-function.f90
deleted file mode 100644
index c47d509af535..000000000000
--- a/flang/test/Lower/Intrinsics/etime-function.f90
+++ /dev/null
@@ -1,24 +0,0 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s
-
-! CHECK-LABEL: func.func @_QPetime_test(
-! CHECK-SAME: %[[valuesArg:.*]]: !fir.ref<!fir.array<2xf32>> {fir.bindc_name = "values"},
-! CHECK-SAME: %[[timeArg:.*]]: !fir.ref<f32> {fir.bindc_name = "time"}) {
-subroutine etime_test(values, time)
-  REAL(4), DIMENSION(2) :: values
-  REAL(4) :: time
-  time = etime(values)
-  ! CHECK-NEXT:        %[[c9:.*]] = arith.constant 9 : i32
-  ! CHECK-NEXT:        %[[c2:.*]] = arith.constant 2 : index
-  ! CHECK-NEXT:        %[[timeTmpAddr:.*]] = fir.alloca f32
-  ! CHECK-NEXT:        %[[timeDeclare:.*]] = fir.declare %[[timeArg]] {uniq_name = "_QFetime_testEtime"} : (!fir.ref<f32>) -> !fir.ref<f32>
-  ! CHECK-NEXT:        %[[shape:.*]] = fir.shape %[[c2]] : (index) -> !fir.shape<1>
-  ! CHECK-NEXT:        %[[valuesDeclare:.*]] = fir.declare %[[valuesArg]](%[[shape]]) {uniq_name = "_QFetime_testEvalues"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<2xf32>>
-  ! CHECK-NEXT:        %[[valuesBox:.*]] = fir.embox %[[valuesDeclare]](%[[shape]]) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
-  ! CHECK-NEXT:        %[[timeTmpBox:.*]] = fir.embox %[[timeTmpAddr]] : (!fir.ref<f32>) -> !fir.box<f32>
-  ! CHECK:             %[[values:.*]] = fir.convert %[[valuesBox]] : (!fir.box<!fir.array<2xf32>>) -> !fir.box<none>
-  ! CHECK:             %[[timeTmp:.*]] = fir.convert %[[timeTmpBox]] : (!fir.box<f32>) -> !fir.box<none>
-  ! CHECK:             %[[VAL_9:.*]] = fir.call @_FortranAEtime(%[[values]], %[[timeTmp]], %[[VAL_7:.*]], %[[c9]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
-  ! CHECK-NEXT:        %[[timeValue:.*]] = fir.load %[[timeTmpAddr]] : !fir.ref<f32>
-  ! CHECK-NEXT:        fir.store %[[timeValue]] to %[[timeDeclare]] : !fir.ref<f32>
-  ! CHECK-NEXT:        return
-end subroutine etime_test
\ No newline at end of file
diff --git a/flang/test/Lower/Intrinsics/etime.f90 b/flang/test/Lower/Intrinsics/etime.f90
deleted file mode 100644
index e5e7984a340c..000000000000
--- a/flang/test/Lower/Intrinsics/etime.f90
+++ /dev/null
@@ -1,21 +0,0 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s
-
-! CHECK-LABEL: func.func @_QPetime_test(
-! CHECK-SAME: %[[valuesArg:.*]]: !fir.ref<!fir.array<2xf32>> {fir.bindc_name = "values"},
-! CHECK-SAME: %[[timeArg:.*]]: !fir.ref<f32> {fir.bindc_name = "time"}) {
-subroutine etime_test(values, time)
-  REAL(4), DIMENSION(2) :: values
-  REAL(4) :: time
-  call etime(values, time)
-  ! CHECK-NEXT:        %[[c9:.*]] = arith.constant 9 : i32
-  ! CHECK-NEXT:        %[[c2:.*]] = arith.constant 2 : index
-  ! CHECK-NEXT:        %[[timeDeclare:.*]] = fir.declare %[[timeArg]] {uniq_name = "_QFetime_testEtime"} : (!fir.ref<f32>) -> !fir.ref<f32>
-  ! CHECK-NEXT:        %[[shape:.*]] = fir.shape %[[c2]] : (index) -> !fir.shape<1>
-  ! CHECK-NEXT:        %[[valuesDeclare:.*]] = fir.declare %[[valuesArg]](%[[shape]]) {uniq_name = "_QFetime_testEvalues"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<2xf32>>
-  ! CHECK-NEXT:        %[[valuesBox:.*]] = fir.embox %[[valuesDeclare]](%[[shape]]) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
-  ! CHECK-NEXT:        %[[timeBox:.*]] = fir.embox %[[timeDeclare]] : (!fir.ref<f32>) -> !fir.box<f32>
-  ! CHECK:             %[[values:.*]] = fir.convert %[[valuesBox]] : (!fir.box<!fir.array<2xf32>>) -> !fir.box<none>
-  ! CHECK:             %[[time:.*]] = fir.convert %[[timeBox]] : (!fir.box<f32>) -> !fir.box<none>
-  ! CHECK:             %[[VAL_9:.*]] = fir.call @_FortranAEtime(%[[values]], %[[time]], %[[VAL_7:.*]], %[[c9]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
-  ! CHECK-NEXT:        return
-end subroutine etime_test
\ No newline at end of file
diff --git a/flang/test/Semantics/etime.f90 b/flang/test/Semantics/etime.f90
deleted file mode 100644
index 28735c2a7aac..000000000000
--- a/flang/test/Semantics/etime.f90
+++ /dev/null
@@ -1,30 +0,0 @@
-! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
-! Tests for the ETIME intrinsics
-
-subroutine bad_kind_error(values, time)
-  REAL(KIND=8), DIMENSION(2) :: values
-  REAL(KIND=8) :: time
-  !ERROR: Actual argument for 'values=' has bad type or kind 'REAL(8)'
-  call etime(values, time)
-end subroutine bad_kind_error
-  
-subroutine bad_args_error(values)
-  REAL(KIND=4), DIMENSION(2) :: values
-  !ERROR: missing mandatory 'time=' argument
-  call etime(values)
-end subroutine bad_args_error
-
-subroutine bad_apply_form(values)
-  REAL(KIND=4), DIMENSION(2) :: values
-  REAL(KIND=4) :: time
-  !Declaration of 'etime'
-  call etime(values, time)
-  !ERROR: Cannot call subroutine 'etime' like a function
-  time = etime(values)
-end subroutine bad_apply_form
-
-subroutine good_kind_equal(values, time)
-  REAL(KIND=4), DIMENSION(2) :: values
-  REAL(KIND=4) :: time
-  call etime(values, time)
-end subroutine good_kind_equal
\ No newline at end of file
-- 
GitLab


From 45cc6bdea97be10793e4cd5f0ef11abd9d19adea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Thu, 16 May 2024 07:23:00 +0200
Subject: [PATCH 0968/1206] [clang][Interp] Implement __builtin_shufflevector

---
 .github/CODEOWNERS                          |  2 +-
 clang/lib/AST/Interp/ByteCodeExprGen.cpp    | 45 ++++++++++++++
 clang/lib/AST/Interp/ByteCodeExprGen.h      |  1 +
 clang/test/AST/Interp/builtin-functions.cpp | 66 +++++++++++++++------
 4 files changed, 96 insertions(+), 18 deletions(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 6adae58832e1..e25b2f50b1b4 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -132,7 +132,7 @@ clang/test/AST/Interp/ @tbaederr
 /bolt/ @aaupov @maksfb @rafaelauler @ayermolo @dcci
 
 # Bazel build system.
-/utils/bazel/ @rupprecht @keith
+/utils/bazel/ @rupprecht
 
 # InstallAPI and TextAPI
 /llvm/**/TextAPI/ @cyndyishida
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 205e53f02b1a..70328c1f52af 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -2531,6 +2531,51 @@ bool ByteCodeExprGen<Emitter>::VisitConvertVectorExpr(
   return true;
 }
 
+template <class Emitter>
+bool ByteCodeExprGen<Emitter>::VisitShuffleVectorExpr(
+    const ShuffleVectorExpr *E) {
+  assert(Initializing);
+  assert(E->getNumSubExprs() > 2);
+
+  const Expr *Vecs[] = {E->getExpr(0), E->getExpr(1)};
+  assert(Vecs[0]->getType() == Vecs[1]->getType());
+
+  const VectorType *VT = Vecs[0]->getType()->castAs<VectorType>();
+  PrimType ElemT = classifyPrim(VT->getElementType());
+  unsigned NumInputElems = VT->getNumElements();
+  unsigned NumOutputElems = E->getNumSubExprs() - 2;
+  assert(NumOutputElems > 0);
+
+  // Save both input vectors to a local variable.
+  unsigned VectorOffsets[2];
+  for (unsigned I = 0; I != 2; ++I) {
+    VectorOffsets[I] = this->allocateLocalPrimitive(
+        Vecs[I], PT_Ptr, /*IsConst=*/true, /*IsExtended=*/false);
+    if (!this->visit(Vecs[I]))
+      return false;
+    if (!this->emitSetLocal(PT_Ptr, VectorOffsets[I], E))
+      return false;
+  }
+  for (unsigned I = 0; I != NumOutputElems; ++I) {
+    APSInt ShuffleIndex = E->getShuffleMaskIdx(Ctx.getASTContext(), I);
+    if (ShuffleIndex == -1)
+      return this->emitInvalid(E); // FIXME: Better diagnostic.
+
+    assert(ShuffleIndex < (NumInputElems * 2));
+    if (!this->emitGetLocal(PT_Ptr,
+                            VectorOffsets[ShuffleIndex >= NumInputElems], E))
+      return false;
+    unsigned InputVectorIndex = ShuffleIndex.getZExtValue() % NumInputElems;
+    if (!this->emitArrayElemPop(ElemT, InputVectorIndex, E))
+      return false;
+
+    if (!this->emitInitElem(ElemT, I, E))
+      return false;
+  }
+
+  return true;
+}
+
 template <class Emitter> bool ByteCodeExprGen<Emitter>::discard(const Expr *E) {
   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/true,
                              /*NewInitializing=*/false);
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.h b/clang/lib/AST/Interp/ByteCodeExprGen.h
index fba4e45b9aa2..e73a2f0334cf 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.h
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.h
@@ -124,6 +124,7 @@ public:
   bool VisitRecoveryExpr(const RecoveryExpr *E);
   bool VisitAddrLabelExpr(const AddrLabelExpr *E);
   bool VisitConvertVectorExpr(const ConvertVectorExpr *E);
+  bool VisitShuffleVectorExpr(const ShuffleVectorExpr *E);
 
 protected:
   bool visitExpr(const Expr *E) override;
diff --git a/clang/test/AST/Interp/builtin-functions.cpp b/clang/test/AST/Interp/builtin-functions.cpp
index afdfd25527e4..fbe76aba73c9 100644
--- a/clang/test/AST/Interp/builtin-functions.cpp
+++ b/clang/test/AST/Interp/builtin-functions.cpp
@@ -644,24 +644,23 @@ static_assert(__builtin_os_log_format_buffer_size("%{mask.xyz}s", "abc") != 0, "
 /// Some tests are missing since we run this for multiple targets,
 /// some of which do not support _BitInt.
 #ifndef __AVR__
-namespace convertvector {
-  typedef _BitInt(128) BitInt128;
-
-  typedef double vector4double __attribute__((__vector_size__(32)));
-  typedef float vector4float __attribute__((__vector_size__(16)));
-  typedef long long vector4long __attribute__((__vector_size__(32)));
-  typedef int vector4int __attribute__((__vector_size__(16)));
-  typedef short vector4short __attribute__((__vector_size__(8)));
-  typedef char vector4char __attribute__((__vector_size__(4)));
-  typedef BitInt128 vector4BitInt128 __attribute__((__vector_size__(64)));
-  typedef double vector8double __attribute__((__vector_size__(64)));
-  typedef float vector8float __attribute__((__vector_size__(32)));
-  typedef long long vector8long __attribute__((__vector_size__(64)));
-  typedef int vector8int __attribute__((__vector_size__(32)));
-  typedef short vector8short __attribute__((__vector_size__(16)));
-  typedef char vector8char __attribute__((__vector_size__(8)));
-  typedef BitInt128 vector8BitInt128 __attribute__((__vector_size__(128)));
+typedef _BitInt(128) BitInt128;
+typedef double vector4double __attribute__((__vector_size__(32)));
+typedef float vector4float __attribute__((__vector_size__(16)));
+typedef long long vector4long __attribute__((__vector_size__(32)));
+typedef int vector4int __attribute__((__vector_size__(16)));
+typedef short vector4short __attribute__((__vector_size__(8)));
+typedef char vector4char __attribute__((__vector_size__(4)));
+typedef BitInt128 vector4BitInt128 __attribute__((__vector_size__(64)));
+typedef double vector8double __attribute__((__vector_size__(64)));
+typedef float vector8float __attribute__((__vector_size__(32)));
+typedef long long vector8long __attribute__((__vector_size__(64)));
+typedef int vector8int __attribute__((__vector_size__(32)));
+typedef short vector8short __attribute__((__vector_size__(16)));
+typedef char vector8char __attribute__((__vector_size__(8)));
+typedef BitInt128 vector8BitInt128 __attribute__((__vector_size__(128)));
 
+namespace convertvector {
   constexpr vector4double from_vector4double_to_vector4double_var =
       __builtin_convertvector((vector4double){0, 1, 2, 3}, vector4double);
   constexpr vector4float from_vector4double_to_vector4float_var =
@@ -873,4 +872,37 @@ namespace convertvector {
   static_assert(from_vector8BitInt128_to_vector8BitInt128_var[3] == 3, ""); // ref-error {{not an integral constant expression}}
   static_assert(from_vector8BitInt128_to_vector8BitInt128_var[4] == 4, ""); // ref-error {{not an integral constant expression}}
 }
+
+namespace shufflevector {
+  constexpr vector4char vector4charConst1 = {0, 1, 2, 3};
+  constexpr vector4char vector4charConst2 = {4, 5, 6, 7};
+  constexpr vector8char vector8intConst = {8, 9, 10, 11, 12, 13, 14, 15};
+  constexpr vector4char vectorShuffle1 =
+      __builtin_shufflevector(vector4charConst1, vector4charConst2, 0, 1, 2, 3);
+  constexpr vector4char vectorShuffle2 =
+      __builtin_shufflevector(vector4charConst1, vector4charConst2, 4, 5, 6, 7);
+  constexpr vector4char vectorShuffle3 =
+      __builtin_shufflevector(vector4charConst1, vector4charConst2, 0, 2, 4, 6);
+  constexpr vector8char vectorShuffle4 = __builtin_shufflevector(
+      vector8intConst, vector8intConst, 0, 2, 4, 6, 8, 10, 12, 14);
+  constexpr vector4char vectorShuffle5 =
+      __builtin_shufflevector(vector8intConst, vector8intConst, 0, 2, 4, 6);
+  constexpr vector8char vectorShuffle6 = __builtin_shufflevector(
+      vector4charConst1, vector4charConst2, 0, 2, 4, 6, 1, 3, 5, 7);
+
+  static_assert(vectorShuffle6[0] == 0, "");// ref-error {{not an integral constant expression}}
+  static_assert(vectorShuffle6[1] == 2, "");// ref-error {{not an integral constant expression}}
+  static_assert(vectorShuffle6[2] == 4, "");// ref-error {{not an integral constant expression}}
+  static_assert(vectorShuffle6[3] == 6, "");// ref-error {{not an integral constant expression}}
+  static_assert(vectorShuffle6[4] == 1, "");// ref-error {{not an integral constant expression}}
+  static_assert(vectorShuffle6[5] == 3, "");// ref-error {{not an integral constant expression}}
+  static_assert(vectorShuffle6[6] == 5, "");// ref-error {{not an integral constant expression}}
+  static_assert(vectorShuffle6[7] == 7, "");// ref-error {{not an integral constant expression}}
+
+  constexpr vector4char  vectorShuffleFail1 = __builtin_shufflevector( // both-error {{must be initialized by a constant expression}}\
+                                                                       // ref-error {{index for __builtin_shufflevector not within the bounds of the input vectors; index of -1 found at position 0 not permitted in a constexpr context.}}
+          vector4charConst1,
+          vector4charConst2, -1, -1, -1, -1);
+}
+
 #endif
-- 
GitLab


From e27f9bb31984927d10995e3d7e5a8fcde60cf5d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Thu, 16 May 2024 08:28:25 +0200
Subject: [PATCH 0969/1206] [github] Add keith back to bazel codeowners

Wrongly removed in 45cc6bdea97be10793e4cd5f0ef11abd9d19adea.
---
 .github/CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index e25b2f50b1b4..6adae58832e1 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -132,7 +132,7 @@ clang/test/AST/Interp/ @tbaederr
 /bolt/ @aaupov @maksfb @rafaelauler @ayermolo @dcci
 
 # Bazel build system.
-/utils/bazel/ @rupprecht
+/utils/bazel/ @rupprecht @keith
 
 # InstallAPI and TextAPI
 /llvm/**/TextAPI/ @cyndyishida
-- 
GitLab


From 7c956293d856224dd6a1b633820ef53009f7ef1c Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 16 May 2024 00:00:58 -0700
Subject: [PATCH 0970/1206] MCAsmParser: Support \+

In .macro, \+ expands to the per-macro invocation count.
https://sourceware.org/pipermail/binutils/2024-May/134009.html

\+ counts from 0 for .irp/.irpc/.rept .

Note: We currently prints \q for `.print "\q"` while gas doesn't. This
patch does not change this behavior.
---
 llvm/include/llvm/MC/MCAsmMacro.h             |  1 +
 llvm/include/llvm/MC/MCContext.h              |  2 +-
 llvm/lib/MC/MCParser/AsmParser.cpp            | 30 ++++++-----
 .../MC/AsmParser/macro-at-pseudo-variable.s   | 51 +++++++++++++++----
 4 files changed, 61 insertions(+), 23 deletions(-)

diff --git a/llvm/include/llvm/MC/MCAsmMacro.h b/llvm/include/llvm/MC/MCAsmMacro.h
index e2989c09017a..bdf55345adeb 100644
--- a/llvm/include/llvm/MC/MCAsmMacro.h
+++ b/llvm/include/llvm/MC/MCAsmMacro.h
@@ -145,6 +145,7 @@ struct MCAsmMacro {
   MCAsmMacroParameters Parameters;
   std::vector<std::string> Locals;
   bool IsFunction = false;
+  unsigned Count = 0;
 
 public:
   MCAsmMacro(StringRef N, StringRef B, MCAsmMacroParameters P)
diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index 3de41b6a6ea7..b0ac432a065b 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -856,7 +856,7 @@ public:
   void reportError(SMLoc L, const Twine &Msg);
   void reportWarning(SMLoc L, const Twine &Msg);
 
-  const MCAsmMacro *lookupMacro(StringRef Name) {
+  MCAsmMacro *lookupMacro(StringRef Name) {
     StringMap<MCAsmMacro>::iterator I = MacroMap.find(Name);
     return (I == MacroMap.end()) ? nullptr : &I->getValue();
   }
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 33287c6529ca..56da5e2f9a40 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -295,7 +295,7 @@ private:
 
   void checkForBadMacro(SMLoc DirectiveLoc, StringRef Name, StringRef Body,
                         ArrayRef<MCAsmMacroParameter> Parameters);
-  bool expandMacro(raw_svector_ostream &OS, const MCAsmMacro &Macro,
+  bool expandMacro(raw_svector_ostream &OS, MCAsmMacro &Macro,
                    ArrayRef<MCAsmMacroParameter> Parameters,
                    ArrayRef<MCAsmMacroArgument> A, bool EnableAtPseudoVariable);
 
@@ -312,7 +312,7 @@ private:
   ///
   /// \param M The macro.
   /// \param NameLoc Instantiation location.
-  bool handleMacroEntry(const MCAsmMacro *M, SMLoc NameLoc);
+  bool handleMacroEntry(MCAsmMacro *M, SMLoc NameLoc);
 
   /// Handle exit from macro instantiation.
   void handleMacroExit();
@@ -1980,9 +1980,8 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
 
   // If macros are enabled, check to see if this is a macro instantiation.
   if (areMacrosEnabled())
-    if (const MCAsmMacro *M = getContext().lookupMacro(IDVal)) {
+    if (MCAsmMacro *M = getContext().lookupMacro(IDVal))
       return handleMacroEntry(M, IDLoc);
-    }
 
   // Otherwise, we have a normal instruction or directive.
 
@@ -2495,7 +2494,7 @@ static bool isIdentifierChar(char c) {
          c == '.';
 }
 
-bool AsmParser::expandMacro(raw_svector_ostream &OS, const MCAsmMacro &Macro,
+bool AsmParser::expandMacro(raw_svector_ostream &OS, MCAsmMacro &Macro,
                             ArrayRef<MCAsmMacroParameter> Parameters,
                             ArrayRef<MCAsmMacroArgument> A,
                             bool EnableAtPseudoVariable) {
@@ -2560,14 +2559,18 @@ bool AsmParser::expandMacro(raw_svector_ostream &OS, const MCAsmMacro &Macro,
       }
       Pos += 2;
     } else {
+      // Check for \@ and \+ pseudo variables.
       unsigned I = Pos + 1;
-
-      // Check for the \@ pseudo-variable.
-      if (EnableAtPseudoVariable && Body[I] == '@' && I + 1 != End)
-        ++I;
-      else
-        while (isIdentifierChar(Body[I]) && I + 1 != End)
+      if (I + 1 != End) {
+        if (EnableAtPseudoVariable && Body[I] == '@') {
+          ++I;
+        } else if (Body[I] == '+') {
           ++I;
+        } else {
+          while (isIdentifierChar(Body[I]) && I + 1 != End)
+            ++I;
+        }
+      }
 
       const char *Begin = Body.data() + Pos + 1;
       StringRef Argument(Begin, I - (Pos + 1));
@@ -2576,6 +2579,9 @@ bool AsmParser::expandMacro(raw_svector_ostream &OS, const MCAsmMacro &Macro,
       if (Argument == "@") {
         OS << NumOfMacroInstantiations;
         Pos += 2;
+      } else if (Argument == "+") {
+        OS << Macro.Count++;
+        Pos += 2;
       } else {
         for (; Index < NParameters; ++Index)
           if (Parameters[Index].Name == Argument)
@@ -2860,7 +2866,7 @@ bool AsmParser::parseMacroArguments(const MCAsmMacro *M,
   return TokError("too many positional arguments");
 }
 
-bool AsmParser::handleMacroEntry(const MCAsmMacro *M, SMLoc NameLoc) {
+bool AsmParser::handleMacroEntry(MCAsmMacro *M, SMLoc NameLoc) {
   // Arbitrarily limit macro nesting depth (default matches 'as'). We can
   // eliminate this, although we should protect against infinite loops.
   unsigned MaxNestingDepth = AsmMacroMaxNestingDepth;
diff --git a/llvm/test/MC/AsmParser/macro-at-pseudo-variable.s b/llvm/test/MC/AsmParser/macro-at-pseudo-variable.s
index b0966e137170..a083b17aa54f 100644
--- a/llvm/test/MC/AsmParser/macro-at-pseudo-variable.s
+++ b/llvm/test/MC/AsmParser/macro-at-pseudo-variable.s
@@ -1,35 +1,44 @@
-# RUN: llvm-mc -triple i386-unknown-unknown %s | FileCheck %s
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -triple i386 a.s | FileCheck %s
+# RUN: llvm-mc -triple i386 b.s | FileCheck %s --check-prefix=CHECK2
 
+#--- a.s
 .macro A
   add  $1\@, %eax
+  add  $2\+, %eax
 .endm
 
 .macro B
   sub  $1\@, %eax
+  sub  $2\+, %eax
 .endm
 
   A
-# CHECK: addl  $10, %eax
+# CHECK:      addl  $10, %eax
+# CHECK-NEXT: addl  $20, %eax
   A
-# CHECK: addl  $11, %eax
+# CHECK:      addl  $11, %eax
+# CHECK-NEXT: addl  $21, %eax
   B
-# CHECK: subl  $12, %eax
+# CHECK:      subl  $12, %eax
+# CHECK-NEXT: subl  $20, %eax
   B
-# CHECK: subl  $13, %eax
+# CHECK:      subl  $13, %eax
+# CHECK-NEXT: subl  $21, %eax
 
 # The following uses of \@ are undocumented, but valid:
 .irpc foo,234
   add  $\foo\@, %eax
 .endr
-# CHECK: addl  $24, %eax
-# CHECK: addl  $34, %eax
-# CHECK: addl  $44, %eax
+# CHECK:      addl  $24, %eax
+# CHECK-NEXT: addl  $34, %eax
+# CHECK-NEXT: addl  $44, %eax
 
 .irp reg,%eax,%ebx
   sub  $2\@, \reg
 .endr
-# CHECK: subl  $24, %eax
-# CHECK: subl  $24, %ebx
+# CHECK:      subl  $24, %eax
+# CHECK-NEXT: subl  $24, %ebx
 
 # Test that .irp(c) and .rep(t) do not increase \@.
 # Only the use of A should increase \@, so we can test that it increases by 1
@@ -62,3 +71,25 @@
 
   A
 # CHECK: addl  $17, %eax
+
+#--- b.s
+.rept 2
+  .print "r\+"
+.endr
+.irpc foo,12
+  .print "\+i"
+.endr
+# CHECK2:      r0
+# CHECK2-NEXT: r1
+# CHECK2-NEXT: 0i
+# CHECK2-NEXT: 1i
+
+.rept 2
+  .rept 2
+    .print "n\+"
+  .endr
+.endr
+# CHECK2:      n0
+# CHECK2-NEXT: n0
+# CHECK2-NEXT: n1
+# CHECK2-NEXT: n1
-- 
GitLab


From 89ee3ae2bd1d5705a3e775e3f30bf0ec6d8d863a Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan@arm.com>
Date: Thu, 16 May 2024 08:04:40 +0100
Subject: [PATCH 0971/1206] [Flang][OpenMP] Fix update operation not found
 issue (#92165)

If there is only one non-terminator operation in the update region then
the update operation can be found and we can try to generate an
atomicrmw instruction. Otherwise use the cmpxchg loop.

Fixes #91929
---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 44 ++++++++++---------
 mlir/test/Target/LLVMIR/openmp-llvm.mlir      | 22 ++++++++++
 2 files changed, 45 insertions(+), 21 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index a7294632d666..ed9fb44cf08e 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -1623,31 +1623,33 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
 
   // Convert values and types.
   auto &innerOpList = opInst.getRegion().front().getOperations();
-  bool isRegionArgUsed{false}, isXBinopExpr{false};
+  bool isXBinopExpr{false};
   llvm::AtomicRMWInst::BinOp binop;
   mlir::Value mlirExpr;
-  // Find the binary update operation that uses the region argument
-  // and get the expression to update
-  for (Operation &innerOp : innerOpList) {
-    if (innerOp.getNumOperands() == 2) {
-      binop = convertBinOpToAtomic(innerOp);
-      if (!llvm::is_contained(innerOp.getOperands(),
-                              opInst.getRegion().getArgument(0)))
-        continue;
-      isRegionArgUsed = true;
-      isXBinopExpr = innerOp.getNumOperands() > 0 &&
-                     innerOp.getOperand(0) == opInst.getRegion().getArgument(0);
-      mlirExpr = (isXBinopExpr ? innerOp.getOperand(1) : innerOp.getOperand(0));
-      break;
+  llvm::Value *llvmExpr = nullptr;
+  llvm::Value *llvmX = nullptr;
+  llvm::Type *llvmXElementType = nullptr;
+  if (innerOpList.size() == 2) {
+    // The two operations here are the update and the terminator.
+    // Since we can identify the update operation, there is a possibility
+    // that we can generate the atomicrmw instruction.
+    mlir::Operation &innerOp = *opInst.getRegion().front().begin();
+    if (!llvm::is_contained(innerOp.getOperands(),
+                            opInst.getRegion().getArgument(0))) {
+      return opInst.emitError("no atomic update operation with region argument"
+                              " as operand found inside atomic.update region");
     }
+    binop = convertBinOpToAtomic(innerOp);
+    isXBinopExpr = innerOp.getOperand(0) == opInst.getRegion().getArgument(0);
+    mlirExpr = (isXBinopExpr ? innerOp.getOperand(1) : innerOp.getOperand(0));
+    llvmExpr = moduleTranslation.lookupValue(mlirExpr);
+  } else {
+    // Since the update region includes more than one operation
+    // we will resort to generating a cmpxchg loop.
+    binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
   }
-  if (!isRegionArgUsed)
-    return opInst.emitError("no atomic update operation with region argument"
-                            " as operand found inside atomic.update region");
-
-  llvm::Value *llvmExpr = moduleTranslation.lookupValue(mlirExpr);
-  llvm::Value *llvmX = moduleTranslation.lookupValue(opInst.getX());
-  llvm::Type *llvmXElementType = moduleTranslation.convertType(
+  llvmX = moduleTranslation.lookupValue(opInst.getX());
+  llvmXElementType = moduleTranslation.convertType(
       opInst.getRegion().getArgument(0).getType());
   llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,
                                                       /*isSigned=*/false,
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index ad40ca26bec9..8654899efefd 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -1467,6 +1467,28 @@ llvm.func @omp_atomic_update_intrinsic(%x:!llvm.ptr, %expr: i32) {
 
 // -----
 
+// CHECK-LABEL: @atomic_update_cmpxchg
+// CHECK-SAME: (ptr %[[X:.*]], ptr %[[EXPR:.*]]) {
+// CHECK:  %[[AT_LOAD_VAL:.*]] = load atomic i32, ptr %[[X]] monotonic, align 4
+// CHECK:  %[[LOAD_VAL_PHI:.*]] = phi i32 [ %[[AT_LOAD_VAL]], %entry ], [ %[[LOAD_VAL:.*]], %.atomic.cont ]
+// CHECK:  %[[VAL_SUCCESS:.*]] = cmpxchg ptr %[[X]], i32 %[[LOAD_VAL_PHI]], i32 %{{.*}} monotonic monotonic, align 4
+// CHECK:  %[[LOAD_VAL]] = extractvalue { i32, i1 } %[[VAL_SUCCESS]], 0
+// CHECK:  br i1 %{{.*}}, label %.atomic.exit, label %.atomic.cont
+
+llvm.func @atomic_update_cmpxchg(%arg0: !llvm.ptr, %arg1: !llvm.ptr) {
+  %0 = llvm.load %arg1 : !llvm.ptr -> f32
+  omp.atomic.update %arg0 : !llvm.ptr {
+  ^bb0(%arg2: i32):
+    %1 = llvm.sitofp %arg2 : i32 to f32
+    %2 = llvm.fadd %1, %0 : f32
+    %3 = llvm.fptosi %2 : f32 to i32
+    omp.yield(%3 : i32)
+  }
+  llvm.return
+}
+
+// -----
+
 // CHECK-LABEL: @omp_atomic_capture_prefix_update
 // CHECK-SAME: (ptr %[[x:.*]], ptr %[[v:.*]], i32 %[[expr:.*]], ptr %[[xf:.*]], ptr %[[vf:.*]], float %[[exprf:.*]])
 llvm.func @omp_atomic_capture_prefix_update(
-- 
GitLab


From ca1f0d41b8c5f42ca6cdcbdf9732bf4df2fcc80f Mon Sep 17 00:00:00 2001
From: Daniil Kovalev <dkovalev@accesssoftek.com>
Date: Thu, 16 May 2024 10:04:58 +0300
Subject: [PATCH 0972/1206] [lld][AArch64][ELF][PAC] Support `.relr.auth.dyn`
 section (#87635)

Support `R_AARCH64_AUTH_RELATIVE` relocation compression as described in
https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#relocation-compression
---
 lld/ELF/Arch/AArch64.cpp           |  13 +++
 lld/ELF/Relocations.cpp            |  10 +-
 lld/ELF/SyntheticSections.cpp      |  24 +++--
 lld/ELF/SyntheticSections.h        |  15 ++-
 lld/ELF/Writer.cpp                 |  35 ++++++
 lld/test/ELF/aarch64-reloc-pauth.s | 166 +++++++++++++++++++++++------
 6 files changed, 216 insertions(+), 47 deletions(-)

diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp
index 2bf6e2c6c851..b0aa0b9b8e9d 100644
--- a/lld/ELF/Arch/AArch64.cpp
+++ b/lld/ELF/Arch/AArch64.cpp
@@ -428,6 +428,19 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
   case R_AARCH64_PREL64:
     write64(loc, val);
     break;
+  case R_AARCH64_AUTH_ABS64:
+    // If val is wider than 32 bits, the relocation must have been moved from
+    // .relr.auth.dyn to .rela.dyn, and the addend write is not needed.
+    //
+    // If val fits in 32 bits, we have two potential scenarios:
+    // * True RELR: Write the 32-bit `val`.
+    // * RELA: Even if the value now fits in 32 bits, it might have been
+    //   converted from RELR during an iteration in
+    //   finalizeAddressDependentContent(). Writing the value is harmless
+    //   because dynamic linking ignores it.
+    if (isInt<32>(val))
+      write32(loc, val);
+    break;
   case R_AARCH64_ADD_ABS_LO12_NC:
     or32AArch64Imm(loc, val);
     break;
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 04db413a6609..2c02c2e572bf 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -898,9 +898,9 @@ static void addRelativeReloc(InputSectionBase &isec, uint64_t offsetInSec,
     isec.addReloc({expr, type, offsetInSec, addend, &sym});
     if (shard)
       part.relrDyn->relocsVec[parallel::getThreadIndex()].push_back(
-          {&isec, offsetInSec});
+          {&isec, isec.relocs().size() - 1});
     else
-      part.relrDyn->relocs.push_back({&isec, offsetInSec});
+      part.relrDyn->relocs.push_back({&isec, isec.relocs().size() - 1});
     return;
   }
   part.relaDyn->addRelativeReloc<shard>(target->relativeRel, isec, offsetInSec,
@@ -1154,6 +1154,12 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset,
         // relative relocation. Use a symbolic relocation instead.
         if (sym.isPreemptible) {
           part.relaDyn->addSymbolReloc(type, *sec, offset, sym, addend, type);
+        } else if (part.relrAuthDyn && sec->addralign >= 2 && offset % 2 == 0) {
+          // When symbol values are determined in
+          // finalizeAddressDependentContent, some .relr.auth.dyn relocations
+          // may be moved to .rela.dyn.
+          sec->addReloc({expr, type, offset, addend, &sym});
+          part.relrAuthDyn->relocs.push_back({sec, sec->relocs().size() - 1});
         } else {
           part.relaDyn->addReloc({R_AARCH64_AUTH_RELATIVE, sec, offset,
                                   DynamicReloc::AddendOnlyWithTargetVA, sym,
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 298c714adb3b..22bfed0852bc 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -1420,6 +1420,12 @@ DynamicSection<ELFT>::computeContents() {
     addInt(config->useAndroidRelrTags ? DT_ANDROID_RELRENT : DT_RELRENT,
            sizeof(Elf_Relr));
   }
+  if (part.relrAuthDyn && part.relrAuthDyn->getParent() &&
+      !part.relrAuthDyn->relocs.empty()) {
+    addInSec(DT_AARCH64_AUTH_RELR, *part.relrAuthDyn);
+    addInt(DT_AARCH64_AUTH_RELRSZ, part.relrAuthDyn->getParent()->size);
+    addInt(DT_AARCH64_AUTH_RELRENT, sizeof(Elf_Relr));
+  }
   if (isMain && in.relaPlt->isNeeded()) {
     addInSec(DT_JMPREL, *in.relaPlt);
     entries.emplace_back(DT_PLTRELSZ, addPltRelSz());
@@ -1731,10 +1737,13 @@ template <class ELFT> void RelocationSection<ELFT>::writeTo(uint8_t *buf) {
   }
 }
 
-RelrBaseSection::RelrBaseSection(unsigned concurrency)
-    : SyntheticSection(SHF_ALLOC,
-                       config->useAndroidRelrTags ? SHT_ANDROID_RELR : SHT_RELR,
-                       config->wordsize, ".relr.dyn"),
+RelrBaseSection::RelrBaseSection(unsigned concurrency, bool isAArch64Auth)
+    : SyntheticSection(
+          SHF_ALLOC,
+          isAArch64Auth
+              ? SHT_AARCH64_AUTH_RELR
+              : (config->useAndroidRelrTags ? SHT_ANDROID_RELR : SHT_RELR),
+          config->wordsize, isAArch64Auth ? ".relr.auth.dyn" : ".relr.dyn"),
       relocsVec(concurrency) {}
 
 void RelrBaseSection::mergeRels() {
@@ -2002,8 +2011,8 @@ bool AndroidPackedRelocationSection<ELFT>::updateAllocSize() {
 }
 
 template <class ELFT>
-RelrSection<ELFT>::RelrSection(unsigned concurrency)
-    : RelrBaseSection(concurrency) {
+RelrSection<ELFT>::RelrSection(unsigned concurrency, bool isAArch64Auth)
+    : RelrBaseSection(concurrency, isAArch64Auth) {
   this->entsize = config->wordsize;
 }
 
@@ -4774,6 +4783,9 @@ template <class ELFT> void elf::createSyntheticSections() {
     if (config->relrPackDynRelocs) {
       part.relrDyn = std::make_unique<RelrSection<ELFT>>(threadCount);
       add(*part.relrDyn);
+      part.relrAuthDyn = std::make_unique<RelrSection<ELFT>>(
+          threadCount, /*isAArch64Auth=*/true);
+      add(*part.relrAuthDyn);
     }
 
     if (!config->relocatable) {
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index 34949025a45f..eaa09ea7194f 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -548,7 +548,9 @@ public:
   static bool classof(const SectionBase *d) {
     return SyntheticSection::classof(d) &&
            (d->type == llvm::ELF::SHT_RELA || d->type == llvm::ELF::SHT_REL ||
-            d->type == llvm::ELF::SHT_RELR);
+            d->type == llvm::ELF::SHT_RELR ||
+            (d->type == llvm::ELF::SHT_AARCH64_AUTH_RELR &&
+             config->emachine == llvm::ELF::EM_AARCH64));
   }
   int32_t dynamicTag, sizeDynamicTag;
   SmallVector<DynamicReloc, 0> relocs;
@@ -596,15 +598,17 @@ private:
 };
 
 struct RelativeReloc {
-  uint64_t getOffset() const { return inputSec->getVA(offsetInSec); }
+  uint64_t getOffset() const {
+    return inputSec->getVA(inputSec->relocs()[relocIdx].offset);
+  }
 
   const InputSectionBase *inputSec;
-  uint64_t offsetInSec;
+  size_t relocIdx;
 };
 
 class RelrBaseSection : public SyntheticSection {
 public:
-  RelrBaseSection(unsigned concurrency);
+  RelrBaseSection(unsigned concurrency, bool isAArch64Auth = false);
   void mergeRels();
   bool isNeeded() const override {
     return !relocs.empty() ||
@@ -622,7 +626,7 @@ template <class ELFT> class RelrSection final : public RelrBaseSection {
   using Elf_Relr = typename ELFT::Relr;
 
 public:
-  RelrSection(unsigned concurrency);
+  RelrSection(unsigned concurrency, bool isAArch64Auth = false);
 
   bool updateAllocSize() override;
   size_t getSize() const override { return relrRelocs.size() * this->entsize; }
@@ -1460,6 +1464,7 @@ struct Partition {
   std::unique_ptr<PackageMetadataNote> packageMetadataNote;
   std::unique_ptr<RelocationBaseSection> relaDyn;
   std::unique_ptr<RelrBaseSection> relrDyn;
+  std::unique_ptr<RelrBaseSection> relrAuthDyn;
   std::unique_ptr<VersionDefinitionSection> verDef;
   std::unique_ptr<SyntheticSection> verNeed;
   std::unique_ptr<VersionTableSection> verSym;
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index e8a7b19a95ee..fe2e1900520a 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -1454,9 +1454,32 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
       in.mipsGot->updateAllocSize();
 
     for (Partition &part : partitions) {
+      // The R_AARCH64_AUTH_RELATIVE has a smaller addend field as bits [63:32]
+      // encode the signing schema. We've put relocations in .relr.auth.dyn
+      // during RelocationScanner::processAux, but the target VA for some of
+      // them might be wider than 32 bits. We can only know the final VA at this
+      // point, so move relocations with large values from .relr.auth.dyn to
+      // .rela.dyn. See also AArch64::relocate.
+      if (part.relrAuthDyn) {
+        auto it = llvm::remove_if(
+            part.relrAuthDyn->relocs, [&part](const RelativeReloc &elem) {
+              const Relocation &reloc = elem.inputSec->relocs()[elem.relocIdx];
+              if (isInt<32>(reloc.sym->getVA(reloc.addend)))
+                return false;
+              part.relaDyn->addReloc({R_AARCH64_AUTH_RELATIVE, elem.inputSec,
+                                      reloc.offset,
+                                      DynamicReloc::AddendOnlyWithTargetVA,
+                                      *reloc.sym, reloc.addend, R_ABS});
+              return true;
+            });
+        changed |= (it != part.relrAuthDyn->relocs.end());
+        part.relrAuthDyn->relocs.erase(it, part.relrAuthDyn->relocs.end());
+      }
       changed |= part.relaDyn->updateAllocSize();
       if (part.relrDyn)
         changed |= part.relrDyn->updateAllocSize();
+      if (part.relrAuthDyn)
+        changed |= part.relrAuthDyn->updateAllocSize();
       if (part.memtagGlobalDescriptors)
         changed |= part.memtagGlobalDescriptors->updateAllocSize();
     }
@@ -1614,6 +1637,14 @@ static void removeUnusedSyntheticSections() {
         auto *sec = cast<SyntheticSection>(s);
         if (sec->getParent() && sec->isNeeded())
           return false;
+        // .relr.auth.dyn relocations may be moved to .rela.dyn in
+        // finalizeAddressDependentContent, making .rela.dyn no longer empty.
+        // Conservatively keep .rela.dyn. .relr.auth.dyn can be made empty, but
+        // we would fail to remove it here.
+        if (config->emachine == EM_AARCH64 && config->relrPackDynRelocs)
+          if (auto *relSec = dyn_cast<RelocationBaseSection>(sec))
+            if (relSec == mainPart->relaDyn.get())
+              return false;
         unused.insert(sec);
         return true;
       });
@@ -1926,6 +1957,10 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
         part.relrDyn->mergeRels();
         finalizeSynthetic(part.relrDyn.get());
       }
+      if (part.relrAuthDyn) {
+        part.relrAuthDyn->mergeRels();
+        finalizeSynthetic(part.relrAuthDyn.get());
+      }
 
       finalizeSynthetic(part.dynSymTab.get());
       finalizeSynthetic(part.gnuHashTab.get());
diff --git a/lld/test/ELF/aarch64-reloc-pauth.s b/lld/test/ELF/aarch64-reloc-pauth.s
index b603d8ffdcab..f1ce29eaae46 100644
--- a/lld/test/ELF/aarch64-reloc-pauth.s
+++ b/lld/test/ELF/aarch64-reloc-pauth.s
@@ -1,11 +1,13 @@
 # REQUIRES: aarch64
 
-# RUN: llvm-mc -filetype=obj -triple=aarch64 %p/Inputs/shared2.s -o %t.a.o
-# RUN: ld.lld -shared %t.a.o -soname=so -o %t.a.so
-# RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t.o
+# RUN: rm -rf %t && split-file %s %t && cd %t
 
-# RUN: ld.lld -pie %t.o %t.a.so -o %t
-# RUN: llvm-readobj -r %t | FileCheck --check-prefix=UNPACKED %s
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %p/Inputs/shared2.s -o a.o
+# RUN: ld.lld -shared a.o -soname=so -o a.so
+# RUN: llvm-mc -filetype=obj -triple=aarch64 main.s -o main.o
+
+# RUN: ld.lld -pie main.o a.so -o main
+# RUN: llvm-readobj -r main | FileCheck --check-prefix=UNPACKED %s
 
 # UNPACKED:          Section ({{.+}}) .rela.dyn {
 # UNPACKED-NEXT:       0x30470 R_AARCH64_AUTH_RELATIVE - 0x1
@@ -21,8 +23,8 @@
 # UNPACKED-NEXT:       0x304B0 R_AARCH64_AUTH_ABS64 bar2 0x0
 # UNPACKED-NEXT:     }
 
-# RUN: ld.lld %t.o %t.a.so -o %t.nopie
-# RUN: llvm-readobj -r %t.nopie | FileCheck --check-prefix=NOPIE %s
+# RUN: ld.lld main.o a.so -o main.nopie
+# RUN: llvm-readobj -r main.nopie | FileCheck --check-prefix=NOPIE %s
 
 # NOPIE:      Section ({{.+}}) .rela.dyn {
 # NOPIE:        0x230460 R_AARCH64_AUTH_RELATIVE - 0x200001
@@ -38,67 +40,95 @@
 # NOPIE-NEXT:   0x2304A0 R_AARCH64_AUTH_ABS64 bar2 0x0
 # NOPIE-NEXT: }
 
-# RUN: ld.lld -pie %t.o %t.a.so -o %t.pie
-# RUN: llvm-readelf -S -d -r -x .test %t.pie | FileCheck --check-prefixes=PIE,HEX %s
-
-# PIE:      Section Headers:
-# PIE-NEXT: Name Type Address Off Size ES Flg Lk Inf Al
-# PIE:      .rela.dyn RELA {{0*}}[[#%x,ADDR1:]]
-# PIE-SAME:                                     {{0*}}[[#ADDR1]] 000108 18 A 1 0 8
-
-# PIE:      Relocation section '.rela.dyn' at offset 0x[[#ADDR1]] contains 11 entries:
-# PIE-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
-# PIE-NEXT: 0000000000030470  0000000000000411 R_AARCH64_AUTH_RELATIVE 1
-# PIE-NEXT: 0000000000030478  0000000000000411 R_AARCH64_AUTH_RELATIVE 30472
-# PIE-NEXT: 0000000000030480  0000000000000411 R_AARCH64_AUTH_RELATIVE fffffffffffffffd
-# PIE-NEXT: 0000000000030488  0000000000000411 R_AARCH64_AUTH_RELATIVE 12345678
-# PIE-NEXT: 0000000000030490  0000000000000411 R_AARCH64_AUTH_RELATIVE 123456789a
-# PIE-NEXT: 0000000000030498  0000000000000411 R_AARCH64_AUTH_RELATIVE ffffffedcba98766
-# PIE-NEXT: 00000000000304a0  0000000000000411 R_AARCH64_AUTH_RELATIVE 8003046f
-# PIE-NEXT: 00000000000304b9  0000000000000411 R_AARCH64_AUTH_RELATIVE 4
-# PIE-NEXT: 00000000000304c2  0000000000000411 R_AARCH64_AUTH_RELATIVE 30475
-# PIE-NEXT: 00000000000304a8  0000000100000244 R_AARCH64_AUTH_ABS64   0000000000000000 zed2 + 1111
-# PIE-NEXT: 00000000000304b0  0000000200000244 R_AARCH64_AUTH_ABS64   0000000000000000 bar2 + 0
+# RUN: ld.lld -pie -z pack-relative-relocs main.o a.so -o main.pie
+# RUN: llvm-readelf -S -d -r -x .test main.pie | FileCheck --check-prefixes=RELR,HEX %s
+
+# RELR:      Section Headers:
+# RELR-NEXT: Name Type Address Off Size ES Flg Lk Inf Al
+# RELR:      .rela.dyn RELA {{0*}}[[ADDR1:.+]] {{0*}}[[ADDR1]] 000090 18 A 1 0 8
+# RELR:      .relr.auth.dyn AARCH64_AUTH_RELR {{0*}}[[ADDR2:.+]] {{0*}}[[ADDR2]] 000018 08 A 0 0 8
+
+# RELR:      Dynamic section at offset {{.+}} contains 16 entries
+# RELR:      0x0000000070000012 (AARCH64_AUTH_RELR) 0x[[ADDR2]]
+# RELR-NEXT: 0x0000000070000011 (AARCH64_AUTH_RELRSZ) 24 (bytes)
+# RELR-NEXT: 0x0000000070000013 (AARCH64_AUTH_RELRENT) 8 (bytes)
+
+## Decoded SHT_RELR section is same as UNPACKED,
+## but contains only the relative relocations.
+## Any relative relocations with odd offset or value wider than 32 bits stay in SHT_RELA.
+
+# RELR:      Relocation section '.rela.dyn' at offset 0x[[ADDR1]] contains 6 entries:
+# RELR-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+# RELR-NEXT: 0000000000030460  0000000000000411 R_AARCH64_AUTH_RELATIVE           123456789a
+# RELR-NEXT: 0000000000030468  0000000000000411 R_AARCH64_AUTH_RELATIVE           ffffffedcba98766
+# RELR-NEXT: 0000000000030470  0000000000000411 R_AARCH64_AUTH_RELATIVE           8003043f
+# RELR-NEXT: 0000000000030489  0000000000000411 R_AARCH64_AUTH_RELATIVE           4
+# RELR-NEXT: 0000000000030478  0000000100000244 R_AARCH64_AUTH_ABS64   0000000000000000 zed2 + 1111
+# RELR-NEXT: 0000000000030480  0000000200000244 R_AARCH64_AUTH_ABS64   0000000000000000 bar2 + 0
+# RELR-EMPTY:
+# RELR-NEXT: Relocation section '.relr.auth.dyn' at offset 0x[[ADDR2]] contains 5 entries:
+# RELR-NEXT: Index: Entry Address Symbolic Address
+# RELR-NEXT: 0000: 0000000000030440 0000000000030440 $d.0
+# RELR-NEXT: 0001: 000000000000000f 0000000000030448 $d.0 + 0x8
+# RELR-NEXT:  0000000000030450 $d.0 + 0x10
+# RELR-NEXT:  0000000000030458 $d.0 + 0x18
+# RELR-NEXT: 0002: 0000000000030492 0000000000030492 $d.0 + 0x52
 
 # HEX:      Hex dump of section '.test':
-# HEX-NEXT: 0x00030470 00000000 2a000020 00000000 2b000000
+# HEX-NEXT: 0x00030440 01000000 2a000020 42040300 2b000000
+##                     ^^^^^^^^ Implicit val = 1 = __ehdr_start + 1
 ##                              ^^^^ Discr = 42
 ##                                    ^^ Key (bits 5..6) = DA
+##                                       ^^^^^^^^ Implicit val = 0x30442 = 0x30440 + 2 = .test + 2
 ##                                                ^^^^ Discr = 43
 ##                                                      ^^ Key (bits 5..6) = IA
-# HEX-NEXT: 0x00030480 00000000 2c000080 00000000 2d000020
+# HEX-NEXT: 0x00030450 fdffffff 2c000080 78563412 2d000020
+##                     ^^^^^^^^ Implicit val = -3 = __ehdr_start - 3
 ##                              ^^^^ Discr = 44
 ##                                    ^^ Key (bits 5..6) = IA
 ##                                    ^^ Addr diversity (bit 7) = true
+##                                       ^^^^^^^^ Implicit val = 0x12345678 = __ehdr_start + 0x12345678
 ##                                                ^^^^ Discr = 45
 ##                                                      ^^ Key (bits 5..6) = DA
-# HEX-NEXT: 0x00030490 00000000 2e000020 00000000 2f000020
+# HEX-NEXT: 0x00030460 00000000 2e000020 00000000 2f000020
+##                     ^^^^^^^^ No implicit val (rela reloc due val wider than 32 bits)
 ##                              ^^^^ Discr = 46
 ##                                    ^^ Key (bits 5..6) = DA
+##                                       ^^^^^^^^ No implicit val (rela reloc due to val wider than 32 bits)
 ##                                                ^^^^ Discr = 47
 ##                                                      ^^ Key (bits 5..6) = DA
-# HEX-NEXT: 0x000304a0 00000000 30000020 00000000 31000020
+# HEX-NEXT: 0x00030470 00000000 30000020 00000000 31000020
+##                     ^^^^^^^^ No implicit val (rela reloc due val wider than 32 bits)
 ##                              ^^^^ Discr = 48
 ##                                    ^^ Key (bits 5..6) = DA
+##                                       ^^^^^^^^ No implicit val (rela reloc due to a preemptible symbol)
 ##                                                ^^^^ Discr = 49
 ##                                                      ^^ Key (bits 5..6) = DA
-# HEX-NEXT: 0x000304b0 00000000 32000000 77000000 00330000
+# HEX-NEXT: 0x00030480 00000000 32000000 77000000 00330000
+##                     ^^^^^^^^ No implicit val (rela reloc due to a preemptible symbol)
 ##                              ^^^^ Discr = 50
 ##                                    ^^ Key (bits 5..6) = IA
+##                                         ^^^^^^ ^^ No implicit val (rela reloc due to odd offset)
 ##                                                  ^^^^ Discr = 51
-# HEX-NEXT: 0x000304c0 20770000 00003400 0020{{\ }}
+# HEX-NEXT: 0x00030490 20774504 03003400 0020{{\ }}
 ##                     ^^ Key (bits 5..6) = DA
+##                         ^^^^ ^^^^ Implicit val = 0x30445 = 0x30440 + 5 = .test + 5
 ##                                  ^^^^ Discr = 52
 ##                                         ^^ Key (bits 5..6) = DA
 
+#--- main.s
+
 .section .test, "aw"
 .p2align 3
 .quad (__ehdr_start + 1)@AUTH(da,42)
 .quad (.test + 2)@AUTH(ia,43)
 .quad (__ehdr_start - 3)@AUTH(ia,44,addr)
 .quad (__ehdr_start + 0x12345678)@AUTH(da,45)
+## Addend wider than 32 bits, not enough room for storing implicitly, would go to rela
 .quad (__ehdr_start + 0x123456789A)@AUTH(da,46)
+## Negative addend wider than 32 bits, not enough room for storing implicitly, would go to rela
 .quad (__ehdr_start - 0x123456789A)@AUTH(da,47)
+## INT32_MAX plus non-zero .test is wider than 32 bits, not enough room for storing implicitly, would go to rela
 .quad (.test + 0x7FFFFFFF)@AUTH(da,48)
 .quad (zed2 + 0x1111)@AUTH(da,49)
 .quad bar2@AUTH(ia,50)
@@ -106,3 +136,71 @@
 .quad (__ehdr_start + 4)@AUTH(da,51)
 .byte 0x77
 .quad (.test + 5)@AUTH(da,52)
+
+#--- empty-relr.s
+
+## .relr.auth.dyn relocations that do not fit 32 bits are moved to .rela.dyn.
+## In this case .relr.auth.dyn will be made empty, but
+## removeUnusedSyntheticSections fails to remove the section.
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64 empty-relr.s -o empty-relr.o
+# RUN: ld.lld -pie -z pack-relative-relocs empty-relr.o -o empty-relr
+# RUN: llvm-readelf -S -d -r empty-relr | FileCheck --check-prefixes=EMPTY-RELR %s
+
+# EMPTY-RELR:      Section Headers:
+# EMPTY-RELR-NEXT: Name Type Address Off Size ES Flg Lk Inf Al
+# EMPTY-RELR:      .rela.dyn RELA {{0*}}[[ADDR1:.+]] {{0*}}[[ADDR1]] 000018 18 A 0 0 8
+# EMPTY-RELR:      .relr.auth.dyn AARCH64_AUTH_RELR {{0*}}[[ADDR2:.+]] {{0*}}[[ADDR2]] 000000 08 A 0 0 8
+
+# EMPTY-RELR:      Dynamic section at offset {{.+}} contains 12 entries
+# EMPTY-RELR-NOT:  (AARCH64_AUTH_RELR)
+# EMPTY-RELR-NOT:  (AARCH64_AUTH_RELRSZ)
+# EMPTY-RELR-NOT:  (AARCH64_AUTH_RELRENT)
+# EMPTY-RELR:      0x0000000000000007 (RELA) 0x[[ADDR1]]
+# EMPTY-RELR-NEXT: 0x0000000000000008 (RELASZ) 24 (bytes)
+# EMPTY-RELR-NEXT: 0x0000000000000009 (RELAENT) 24 (bytes)
+
+# EMPTY-RELR:      Relocation section '.rela.dyn' at offset {{.+}} contains 1 entries:
+# EMPTY-RELR-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+# EMPTY-RELR-NEXT: 0000000000030320  0000000000000411 R_AARCH64_AUTH_RELATIVE           8003031f
+# EMPTY-RELR-EMPTY:
+# EMPTY-RELR-NEXT: Relocation section '.relr.auth.dyn' at offset {{.+}} contains 0 entries:
+# EMPTY-RELR-NEXT: Index: Entry Address Symbolic Address
+
+.section .test, "aw"
+.p2align 3
+.quad (.test + 0x7FFFFFFF)@AUTH(da,42)
+
+#--- empty-rela.s
+
+## .relr.auth.dyn relocations that do not fit 32 bits are moved to .rela.dyn.
+## If this scenario does not happen, .rela.dyn will remain empty,
+## but removeUnusedSyntheticSections fails to remove the section.
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64 empty-rela.s -o empty-rela.o
+# RUN: ld.lld -pie -z pack-relative-relocs empty-rela.o -o empty-rela
+# RUN: llvm-readelf -S -d -r empty-rela | FileCheck --check-prefixes=EMPTY-RELA %s
+
+# EMPTY-RELA:      Section Headers:
+# EMPTY-RELA-NEXT: Name Type Address Off Size ES Flg Lk Inf Al
+# EMPTY-RELA:      .rela.dyn RELA {{0*}}[[ADDR1:.+]] {{0*}}[[ADDR1]] 000000 18 A 0 0 8
+# EMPTY-RELA:      .relr.auth.dyn AARCH64_AUTH_RELR {{0*}}[[ADDR2:.+]] {{0*}}[[ADDR2]] 000008 08 A 0 0 8
+
+# EMPTY-RELA:      Dynamic section at offset {{.+}} contains 12 entries
+# EMPTY-RELA-NOT:  (RELR)
+# EMPTY-RELA-NOT:  (RELRSZ)
+# EMPTY-RELA-NOT:  (RELRENT)
+# EMPTY-RELA:      0x0000000070000012 (AARCH64_AUTH_RELR) 0x[[ADDR2]]
+# EMPTY-RELA-NEXT: 0x0000000070000011 (AARCH64_AUTH_RELRSZ) 8 (bytes)
+# EMPTY-RELA-NEXT: 0x0000000070000013 (AARCH64_AUTH_RELRENT) 8 (bytes)
+
+# EMPTY-RELA:      Relocation section '.rela.dyn' at offset {{.+}} contains 0 entries:
+# EMPTY-RELA-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name
+# EMPTY-RELA-EMPTY:
+# EMPTY-RELA-NEXT: Relocation section '.relr.auth.dyn' at offset {{.+}} contains 1 entries:
+# EMPTY-RELA-NEXT: Index: Entry Address Symbolic Address
+# EMPTY-RELA-NEXT: 0000: 0000000000030310 0000000000030310 $d.0
+
+.section .test, "aw"
+.p2align 3
+.quad (.test + 0x12345678)@AUTH(da,42)
-- 
GitLab


From ce1ce5d30c7696fddcb0c040909a9b786fe7cb06 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 16 May 2024 09:09:02 +0200
Subject: [PATCH 0973/1206] InstCombine: Try to use exp10 intrinsic instead of
 libcall (#92287)

Addresses old TODO about the exp10 intrinsic not existing.
---
 .../lib/Transforms/Utils/SimplifyLibCalls.cpp |  11 +-
 llvm/test/Transforms/InstCombine/pow-1.ll     | 131 ++++++++++++++++++
 2 files changed, 140 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 9cb8e20b4806..69b0f78530d7 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -2134,12 +2134,19 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
   }
 
   // pow(10.0, x) -> exp10(x)
-  // TODO: There is no exp10() intrinsic yet, but some day there shall be one.
   if (match(Base, m_SpecificFP(10.0)) &&
-      hasFloatFn(M, TLI, Ty, LibFunc_exp10, LibFunc_exp10f, LibFunc_exp10l))
+      hasFloatFn(M, TLI, Ty, LibFunc_exp10, LibFunc_exp10f, LibFunc_exp10l)) {
+
+    if (Pow->doesNotAccessMemory()) {
+      CallInst *NewExp10 =
+          B.CreateIntrinsic(Intrinsic::exp10, {Ty}, {Expo}, Pow, "exp10");
+      return copyFlags(*Pow, NewExp10);
+    }
+
     return copyFlags(*Pow, emitUnaryFloatFnCall(Expo, TLI, LibFunc_exp10,
                                                 LibFunc_exp10f, LibFunc_exp10l,
                                                 B, NoAttrs));
+  }
 
   // pow(x, y) -> exp2(log2(x) * y)
   if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
diff --git a/llvm/test/Transforms/InstCombine/pow-1.ll b/llvm/test/Transforms/InstCombine/pow-1.ll
index 7a72902126f8..c52b97c38893 100644
--- a/llvm/test/Transforms/InstCombine/pow-1.ll
+++ b/llvm/test/Transforms/InstCombine/pow-1.ll
@@ -893,3 +893,134 @@ define double @test_simplify19(double %x) {
   %retval = call double @pow(double 10.0, double %x)
   ret double %retval
 }
+
+define float @test_libcall_powf_10_f32_noerrno(float %x) {
+; CHECK-EXP10-LABEL: define float @test_libcall_powf_10_f32_noerrno(
+; CHECK-EXP10-SAME: float [[X:%.*]]) {
+; CHECK-EXP10-NEXT:    [[__EXP10F:%.*]] = call float @llvm.exp10.f32(float [[X]])
+; CHECK-EXP10-NEXT:    ret float [[__EXP10F]]
+;
+; CHECK-NO-EXP10-LABEL: define float @test_libcall_powf_10_f32_noerrno(
+; CHECK-NO-EXP10-SAME: float [[X:%.*]]) {
+; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call float @powf(float 1.000000e+01, float [[X]]) #[[ATTR3:[0-9]+]]
+; CHECK-NO-EXP10-NEXT:    ret float [[RETVAL]]
+;
+  %retval = call float @powf(float 10.0, float %x) memory(none)
+  ret float %retval
+}
+
+define double @test_libcall_pow_10_f64_noerrno(double %x) {
+; CHECK-EXP10-LABEL: define double @test_libcall_pow_10_f64_noerrno(
+; CHECK-EXP10-SAME: double [[X:%.*]]) {
+; CHECK-EXP10-NEXT:    [[__EXP10:%.*]] = call double @llvm.exp10.f64(double [[X]])
+; CHECK-EXP10-NEXT:    ret double [[__EXP10]]
+;
+; CHECK-NO-EXP10-LABEL: define double @test_libcall_pow_10_f64_noerrno(
+; CHECK-NO-EXP10-SAME: double [[X:%.*]]) {
+; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call double @pow(double 1.000000e+01, double [[X]]) #[[ATTR3]]
+; CHECK-NO-EXP10-NEXT:    ret double [[RETVAL]]
+;
+  %retval = call double @pow(double 10.0, double %x) memory(none)
+  ret double %retval
+}
+
+define half @test_pow_10_f16(half %x) {
+; CHECK-LABEL: define half @test_pow_10_f16(
+; CHECK-SAME: half [[X:%.*]]) {
+; CHECK-NEXT:    [[RETVAL:%.*]] = call half @llvm.pow.f16(half 0xH4900, half [[X]])
+; CHECK-NEXT:    ret half [[RETVAL]]
+;
+  %retval = call half @llvm.pow.f16(half 10.0, half %x)
+  ret half %retval
+}
+
+define float @test_pow_10_f32(float %x) {
+; CHECK-EXP10-LABEL: define float @test_pow_10_f32(
+; CHECK-EXP10-SAME: float [[X:%.*]]) {
+; CHECK-EXP10-NEXT:    [[__EXP10F:%.*]] = call float @llvm.exp10.f32(float [[X]])
+; CHECK-EXP10-NEXT:    ret float [[__EXP10F]]
+;
+; CHECK-NO-EXP10-LABEL: define float @test_pow_10_f32(
+; CHECK-NO-EXP10-SAME: float [[X:%.*]]) {
+; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call float @llvm.pow.f32(float 1.000000e+01, float [[X]])
+; CHECK-NO-EXP10-NEXT:    ret float [[RETVAL]]
+;
+  %retval = call float @llvm.pow.f32(float 10.0, float %x)
+  ret float %retval
+}
+
+define double @test_pow_10_f64(double %x) {
+; CHECK-EXP10-LABEL: define double @test_pow_10_f64(
+; CHECK-EXP10-SAME: double [[X:%.*]]) {
+; CHECK-EXP10-NEXT:    [[__EXP10:%.*]] = call double @llvm.exp10.f64(double [[X]])
+; CHECK-EXP10-NEXT:    ret double [[__EXP10]]
+;
+; CHECK-NO-EXP10-LABEL: define double @test_pow_10_f64(
+; CHECK-NO-EXP10-SAME: double [[X:%.*]]) {
+; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call double @llvm.pow.f64(double 1.000000e+01, double [[X]])
+; CHECK-NO-EXP10-NEXT:    ret double [[RETVAL]]
+;
+  %retval = call double @llvm.pow.f64(double 10.0, double %x)
+  ret double %retval
+}
+
+define fp128 @test_pow_10_fp128(fp128 %x) {
+; CHECK-LABEL: define fp128 @test_pow_10_fp128(
+; CHECK-SAME: fp128 [[X:%.*]]) {
+; CHECK-NEXT:    [[RETVAL:%.*]] = call fp128 @llvm.pow.f128(fp128 0xL00000000000000004002400000000000, fp128 [[X]])
+; CHECK-NEXT:    ret fp128 [[RETVAL]]
+;
+  %ten = fpext double 10.0 to fp128
+  %retval = call fp128 @llvm.pow.fp128(fp128 %ten, fp128 %x)
+  ret fp128 %retval
+}
+
+define bfloat @test_pow_10_bf16(bfloat %x) {
+; CHECK-LABEL: define bfloat @test_pow_10_bf16(
+; CHECK-SAME: bfloat [[X:%.*]]) {
+; CHECK-NEXT:    [[RETVAL:%.*]] = call bfloat @llvm.pow.bf16(bfloat 0xR4120, bfloat [[X]])
+; CHECK-NEXT:    ret bfloat [[RETVAL]]
+;
+  %retval = call bfloat @llvm.pow.bf16(bfloat 10.0, bfloat %x)
+  ret bfloat %retval
+}
+
+define <2 x half> @test_pow_10_v2f16(<2 x half> %x) {
+; CHECK-LABEL: define <2 x half> @test_pow_10_v2f16(
+; CHECK-SAME: <2 x half> [[X:%.*]]) {
+; CHECK-NEXT:    [[RETVAL:%.*]] = call <2 x half> @llvm.pow.v2f16(<2 x half> <half 0xH4900, half 0xH4900>, <2 x half> [[X]])
+; CHECK-NEXT:    ret <2 x half> [[RETVAL]]
+;
+  %retval = call <2 x half> @llvm.pow.v2f16(<2 x half> <half 10.0, half 10.0>, <2 x half> %x)
+  ret <2 x half> %retval
+}
+
+define <2 x float> @test_pow_10_v2f32(<2 x float> %x) {
+; CHECK-LABEL: define <2 x float> @test_pow_10_v2f32(
+; CHECK-SAME: <2 x float> [[X:%.*]]) {
+; CHECK-NEXT:    [[RETVAL:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 1.000000e+01, float 1.000000e+01>, <2 x float> [[X]])
+; CHECK-NEXT:    ret <2 x float> [[RETVAL]]
+;
+  %retval = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 10.0, float 10.0>, <2 x float> %x)
+  ret <2 x float> %retval
+}
+
+define <2 x double> @test_pow_10_v2f64(<2 x double> %x) {
+; CHECK-LABEL: define <2 x double> @test_pow_10_v2f64(
+; CHECK-SAME: <2 x double> [[X:%.*]]) {
+; CHECK-NEXT:    [[RETVAL:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 1.000000e+01, double 1.000000e+01>, <2 x double> [[X]])
+; CHECK-NEXT:    ret <2 x double> [[RETVAL]]
+;
+  %retval = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 10.0, double 10.0>, <2 x double> %x)
+  ret <2 x double> %retval
+}
+
+define <2 x bfloat> @test_pow_10_v2bf16(<2 x bfloat> %x) {
+; CHECK-LABEL: define <2 x bfloat> @test_pow_10_v2bf16(
+; CHECK-SAME: <2 x bfloat> [[X:%.*]]) {
+; CHECK-NEXT:    [[RETVAL:%.*]] = call <2 x bfloat> @llvm.pow.v2bf16(<2 x bfloat> <bfloat 0xR4120, bfloat 0xR4120>, <2 x bfloat> [[X]])
+; CHECK-NEXT:    ret <2 x bfloat> [[RETVAL]]
+;
+  %retval = call <2 x bfloat> @llvm.pow.v2bf16(<2 x bfloat> <bfloat 10.0, bfloat 10.0>, <2 x bfloat> %x)
+  ret <2 x bfloat> %retval
+}
-- 
GitLab


From b5107bdda332234d632c9749e01d60dd0f01bf39 Mon Sep 17 00:00:00 2001
From: Cullen Rhodes <cullen.rhodes@arm.com>
Date: Thu, 16 May 2024 08:17:37 +0100
Subject: [PATCH 0974/1206] [mlir][ArmSME] Verify ops on tile types post LLVM
 conversion  (#92076)

Unsupported ops on tile types can become dead after
`-convert-arm-sme-to-llvm` resulting in incorrect results. Verify such
operations don't exist post-conversion and fail if they do.

Based on discussion from
https://discourse.llvm.org/t/on-improving-arm-sme-lowering-resilience-in-mlir/78543
---
 .../Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp  |  19 +++
 .../ArmSMEToLLVM/arm-sme-to-llvm.mlir         | 149 ++++++++++++------
 .../ArmSMEToLLVM/tile-spills-and-fills.mlir   |   2 +-
 .../Conversion/ArmSMEToLLVM/unsupported.mlir  |  14 +-
 4 files changed, 130 insertions(+), 54 deletions(-)

diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index 3dbc8e9916df..7e7706793674 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/ArmSME/IR/ArmSME.h"
 #include "mlir/Dialect/ArmSME/Transforms/Transforms.h"
 #include "mlir/Dialect/ArmSME/Utils/Utils.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
@@ -877,6 +878,24 @@ struct ConvertArmSMEToLLVMPass
 
     if (failed(applyPartialConversion(function, target, std::move(patterns))))
       signalPassFailure();
+
+    // Walk the function and fail if there are unexpected operations on SME
+    // tile types after conversion.
+    function->walk([&](Operation *op) {
+      // These ops are legal post conversion, skip these.
+      if (isa<arm_sme::CopyTileOp, arm_sme::GetTileOp, cf::BranchOp>(op) ||
+          !op->isRegistered())
+        return;
+      auto isSMETileType = [](Type type) {
+        return arm_sme::isValidSMETileVectorType(type);
+      };
+      if (llvm::any_of(op->getResultTypes(), isSMETileType) ||
+          llvm::any_of(op->getOperandTypes(), isSMETileType)) {
+        op->emitOpError("unexpected operation with SME tile type after "
+                        "conversion to LLVM");
+        signalPassFailure();
+      }
+    });
   }
 };
 
diff --git a/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir b/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir
index 14b1f323da3a..ef85f3d069d7 100644
--- a/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir
+++ b/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir
@@ -629,18 +629,18 @@ func.func @arm_sme_streaming_vl_double_words() -> index {
 
 // CHECK-LABEL: arm_sme_fmopa_2way_f16f16_to_f32
 // CHECK: "arm_sme.intr.mopa.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xf16>, vector<[8]xf16>) -> ()
-func.func @arm_sme_fmopa_2way_f16f16_to_f32(%vecA: vector<[8]xf16>, %vecB: vector<[8]xf16>) -> vector<[4]x[4]xf32> {
+func.func @arm_sme_fmopa_2way_f16f16_to_f32(%vecA: vector<[8]xf16>, %vecB: vector<[8]xf16>) {
   %result = arm_sme.fmopa_2way %vecA, %vecB : vector<[8]xf16>, vector<[8]xf16> into vector<[4]x[4]xf32>
-  return %result : vector<[4]x[4]xf32>
+  "test.some_use"(%result) : (vector<[4]x[4]xf32>) -> ()
 }
 
 // -----
 
 // CHECK-LABEL: arm_sme_fmopa_2way_bf16bf16_to_f32
 // CHECK: "arm_sme.intr.mopa.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xbf16>, vector<[8]xbf16>) -> ()
-func.func @arm_sme_fmopa_2way_bf16bf16_to_f32(%vecA: vector<[8]xbf16>, %vecB: vector<[8]xbf16>) -> vector<[4]x[4]xf32> {
+func.func @arm_sme_fmopa_2way_bf16bf16_to_f32(%vecA: vector<[8]xbf16>, %vecB: vector<[8]xbf16>) {
   %result = arm_sme.fmopa_2way %vecA, %vecB : vector<[8]xbf16>, vector<[8]xbf16> into vector<[4]x[4]xf32>
-  return %result : vector<[4]x[4]xf32>
+  "test.some_use"(%result) : (vector<[4]x[4]xf32>) -> ()
 }
 
 //===----------------------------------------------------------------------===//
@@ -651,18 +651,18 @@ func.func @arm_sme_fmopa_2way_bf16bf16_to_f32(%vecA: vector<[8]xbf16>, %vecB: ve
 
 // CHECK-LABEL: arm_sme_fmops_2way_f16f16_to_f32
 // CHECK: "arm_sme.intr.mops.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xf16>, vector<[8]xf16>) -> ()
-func.func @arm_sme_fmops_2way_f16f16_to_f32(%vecA: vector<[8]xf16>, %vecB: vector<[8]xf16>) -> vector<[4]x[4]xf32> {
+func.func @arm_sme_fmops_2way_f16f16_to_f32(%vecA: vector<[8]xf16>, %vecB: vector<[8]xf16>) {
   %result = arm_sme.fmops_2way %vecA, %vecB : vector<[8]xf16>, vector<[8]xf16> into vector<[4]x[4]xf32>
-  return %result : vector<[4]x[4]xf32>
+  "test.some_use"(%result) : (vector<[4]x[4]xf32>) -> ()
 }
 
 // -----
 
 // CHECK-LABEL: arm_sme_fmops_2way_bf16bf16_to_f32
 // CHECK: "arm_sme.intr.mops.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xbf16>, vector<[8]xbf16>) -> ()
-func.func @arm_sme_fmops_2way_bf16bf16_to_f32(%vecA: vector<[8]xbf16>, %vecB: vector<[8]xbf16>) -> vector<[4]x[4]xf32> {
+func.func @arm_sme_fmops_2way_bf16bf16_to_f32(%vecA: vector<[8]xbf16>, %vecB: vector<[8]xbf16>) {
   %result = arm_sme.fmops_2way %vecA, %vecB : vector<[8]xbf16>, vector<[8]xbf16> into vector<[4]x[4]xf32>
-  return %result : vector<[4]x[4]xf32>
+  "test.some_use"(%result) : (vector<[4]x[4]xf32>) -> ()
 }
 
 //===----------------------------------------------------------------------===//
@@ -673,9 +673,9 @@ func.func @arm_sme_fmops_2way_bf16bf16_to_f32(%vecA: vector<[8]xbf16>, %vecB: ve
 
 // CHECK-LABEL: arm_sme_smopa_2way_i16i16_to_i32
 // CHECK: "arm_sme.intr.smopa.za32"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_smopa_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[4]x[4]xi32> {
+func.func @arm_sme_smopa_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
   %result = arm_sme.smopa_2way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[4]x[4]xi32>
-  return %result : vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
 }
 
 //===----------------------------------------------------------------------===//
@@ -686,9 +686,9 @@ func.func @arm_sme_smopa_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vecto
 
 // CHECK-LABEL: arm_sme_smops_2way_i16i16_to_i32
 // CHECK: "arm_sme.intr.smops.za32"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_smops_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[4]x[4]xi32> {
+func.func @arm_sme_smops_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
   %result = arm_sme.smops_2way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[4]x[4]xi32>
-  return %result : vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
 }
 
 //===----------------------------------------------------------------------===//
@@ -699,9 +699,10 @@ func.func @arm_sme_smops_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vecto
 
 // CHECK-LABEL: arm_sme_umopa_2way_i16i16_to_i32
 // CHECK: "arm_sme.intr.umopa.za32"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_umopa_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[4]x[4]xi32> {
+func.func @arm_sme_umopa_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
   %result = arm_sme.umopa_2way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[4]x[4]xi32>
-  return %result : vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -712,9 +713,10 @@ func.func @arm_sme_umopa_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vecto
 
 // CHECK-LABEL: arm_sme_umops_2way_i16i16_to_i32
 // CHECK: "arm_sme.intr.umops.za32"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_umops_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[4]x[4]xi32> {
+func.func @arm_sme_umops_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
   %result = arm_sme.umops_2way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[4]x[4]xi32>
-  return %result : vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -725,18 +727,20 @@ func.func @arm_sme_umops_2way_i16i16_to_i32(%vecA: vector<[8]xi16>, %vecB: vecto
 
 // CHECK-LABEL: arm_sme_smopa_4way_i8i8_to_i32
 // CHECK: "arm_sme.intr.smopa.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[16]xi1>, vector<[16]xi1>, vector<[16]xi8>, vector<[16]xi8>) -> ()
-func.func @arm_sme_smopa_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) -> vector<[4]x[4]xi32> {
+func.func @arm_sme_smopa_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) {
   %result = arm_sme.smopa_4way %vecA, %vecB : vector<[16]xi8>, vector<[16]xi8> into vector<[4]x[4]xi32>
-  return %result : vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
+  return
 }
 
 // -----
 
 // CHECK-LABEL: arm_sme_smopa_4way_i16i16_to_i64
 // CHECK: "arm_sme.intr.smopa.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_smopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[2]x[2]xi64> {
+func.func @arm_sme_smopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
   %result = arm_sme.smopa_4way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[2]x[2]xi64>
-  return %result : vector<[2]x[2]xi64>
+  "test.some_use"(%result) : (vector<[2]x[2]xi64>) -> ()
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -747,18 +751,20 @@ func.func @arm_sme_smopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vecto
 
 // CHECK-LABEL: arm_sme_smops_4way_i8i8_to_i32
 // CHECK: "arm_sme.intr.smops.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[16]xi1>, vector<[16]xi1>, vector<[16]xi8>, vector<[16]xi8>) -> ()
-func.func @arm_sme_smops_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) -> vector<[4]x[4]xi32> {
+func.func @arm_sme_smops_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) {
   %result = arm_sme.smops_4way %vecA, %vecB : vector<[16]xi8>, vector<[16]xi8> into vector<[4]x[4]xi32>
-  return %result : vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
+  return
 }
 
 // -----
 
 // CHECK-LABEL: arm_sme_smops_4way_i16i16_to_i64
 // CHECK: "arm_sme.intr.smops.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_smops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[2]x[2]xi64> {
+func.func @arm_sme_smops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
   %result = arm_sme.smops_4way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[2]x[2]xi64>
-  return %result : vector<[2]x[2]xi64>
+  "test.some_use"(%result) : (vector<[2]x[2]xi64>) -> ()
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -769,18 +775,20 @@ func.func @arm_sme_smops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vecto
 
 // CHECK-LABEL: arm_sme_umopa_4way_i8i8_to_i32
 // CHECK: "arm_sme.intr.umopa.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[16]xi1>, vector<[16]xi1>, vector<[16]xi8>, vector<[16]xi8>) -> ()
-func.func @arm_sme_umopa_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) -> vector<[4]x[4]xi32> {
+func.func @arm_sme_umopa_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) {
   %result = arm_sme.umopa_4way %vecA, %vecB : vector<[16]xi8>, vector<[16]xi8> into vector<[4]x[4]xi32>
-  return %result : vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
+  return
 }
 
 // -----
 
 // CHECK-LABEL: arm_sme_umopa_4way_i16i16_to_i64
 // CHECK: "arm_sme.intr.umopa.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_umopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[2]x[2]xi64> {
+func.func @arm_sme_umopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
   %result = arm_sme.umopa_4way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[2]x[2]xi64>
-  return %result : vector<[2]x[2]xi64>
+  "test.some_use"(%result) : (vector<[2]x[2]xi64>) -> ()
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -791,18 +799,20 @@ func.func @arm_sme_umopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vecto
 
 // CHECK-LABEL: arm_sme_umops_4way_i8i8_to_i32
 // CHECK: "arm_sme.intr.umops.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[16]xi1>, vector<[16]xi1>, vector<[16]xi8>, vector<[16]xi8>) -> ()
-func.func @arm_sme_umops_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) -> vector<[4]x[4]xi32> {
+func.func @arm_sme_umops_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) {
   %result = arm_sme.umops_4way %vecA, %vecB : vector<[16]xi8>, vector<[16]xi8> into vector<[4]x[4]xi32>
-  return %result : vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
+  return
 }
 
 // -----
 
 // CHECK-LABEL: arm_sme_umops_4way_i16i16_to_i64
 // CHECK: "arm_sme.intr.umops.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_umops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[2]x[2]xi64> {
+func.func @arm_sme_umops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
   %result = arm_sme.umops_4way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[2]x[2]xi64>
-  return %result : vector<[2]x[2]xi64>
+  "test.some_use"(%result) : (vector<[2]x[2]xi64>) -> ()
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -813,18 +823,20 @@ func.func @arm_sme_umops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vecto
 
 // CHECK-LABEL: arm_sme_sumopa_4way_i8i8_to_i32
 // CHECK: "arm_sme.intr.sumopa.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[16]xi1>, vector<[16]xi1>, vector<[16]xi8>, vector<[16]xi8>) -> ()
-func.func @arm_sme_sumopa_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) -> vector<[4]x[4]xi32> {
+func.func @arm_sme_sumopa_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) {
   %result = arm_sme.sumopa_4way %vecA, %vecB : vector<[16]xi8>, vector<[16]xi8> into vector<[4]x[4]xi32>
-  return %result : vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
+  return
 }
 
 // -----
 
 // CHECK-LABEL: arm_sme_sumopa_4way_i16i16_to_i64
 // CHECK: "arm_sme.intr.sumopa.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_sumopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[2]x[2]xi64> {
+func.func @arm_sme_sumopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
   %result = arm_sme.sumopa_4way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[2]x[2]xi64>
-  return %result : vector<[2]x[2]xi64>
+  "test.some_use"(%result) : (vector<[2]x[2]xi64>) -> ()
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -835,18 +847,20 @@ func.func @arm_sme_sumopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vect
 
 // CHECK-LABEL: arm_sme_sumops_4way_i8i8_to_i32
 // CHECK: "arm_sme.intr.sumops.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[16]xi1>, vector<[16]xi1>, vector<[16]xi8>, vector<[16]xi8>) -> ()
-func.func @arm_sme_sumops_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) -> vector<[4]x[4]xi32> {
+func.func @arm_sme_sumops_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) {
   %result = arm_sme.sumops_4way %vecA, %vecB : vector<[16]xi8>, vector<[16]xi8> into vector<[4]x[4]xi32>
-  return %result : vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
+  return
 }
 
 // -----
 
 // CHECK-LABEL: arm_sme_sumops_4way_i16i16_to_i64
 // CHECK: "arm_sme.intr.sumops.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_sumops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[2]x[2]xi64> {
+func.func @arm_sme_sumops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
   %result = arm_sme.sumops_4way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[2]x[2]xi64>
-  return %result : vector<[2]x[2]xi64>
+  "test.some_use"(%result) : (vector<[2]x[2]xi64>) -> ()
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -857,18 +871,20 @@ func.func @arm_sme_sumops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vect
 
 // CHECK-LABEL: arm_sme_usmopa_4way_i8i8_to_i32
 // CHECK: "arm_sme.intr.usmopa.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[16]xi1>, vector<[16]xi1>, vector<[16]xi8>, vector<[16]xi8>) -> ()
-func.func @arm_sme_usmopa_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) -> vector<[4]x[4]xi32> {
-  %reuslt = arm_sme.usmopa_4way %vecA, %vecB : vector<[16]xi8>, vector<[16]xi8> into vector<[4]x[4]xi32>
-  return %reuslt : vector<[4]x[4]xi32>
+func.func @arm_sme_usmopa_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) {
+  %result = arm_sme.usmopa_4way %vecA, %vecB : vector<[16]xi8>, vector<[16]xi8> into vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
+  return
 }
 
 // -----
 
 // CHECK-LABEL: arm_sme_usmopa_4way_i16i16_to_i64
 // CHECK: "arm_sme.intr.usmopa.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_usmopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[2]x[2]xi64> {
-  %reuslt = arm_sme.usmopa_4way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[2]x[2]xi64>
-  return %reuslt : vector<[2]x[2]xi64>
+func.func @arm_sme_usmopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
+  %result = arm_sme.usmopa_4way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[2]x[2]xi64>
+  "test.some_use"(%result) : (vector<[2]x[2]xi64>) -> ()
+  return
 }
 
 //===----------------------------------------------------------------------===//
@@ -879,16 +895,45 @@ func.func @arm_sme_usmopa_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vect
 
 // CHECK-LABEL: arm_sme_usmops_4way_i8i8_to_i32
 // CHECK: "arm_sme.intr.usmops.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[16]xi1>, vector<[16]xi1>, vector<[16]xi8>, vector<[16]xi8>) -> ()
-func.func @arm_sme_usmops_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) -> vector<[4]x[4]xi32> {
-  %reuslt = arm_sme.usmops_4way %vecA, %vecB : vector<[16]xi8>, vector<[16]xi8> into vector<[4]x[4]xi32>
-  return %reuslt : vector<[4]x[4]xi32>
+func.func @arm_sme_usmops_4way_i8i8_to_i32(%vecA: vector<[16]xi8>, %vecB: vector<[16]xi8>) {
+  %result = arm_sme.usmops_4way %vecA, %vecB : vector<[16]xi8>, vector<[16]xi8> into vector<[4]x[4]xi32>
+  "test.some_use"(%result) : (vector<[4]x[4]xi32>) -> ()
+  return
 }
 
 // -----
 
 // CHECK-LABEL: arm_sme_usmops_4way_i16i16_to_i64
 // CHECK: "arm_sme.intr.usmops.wide"({{.*}}) <{tile_id = 0 : i32}> : (vector<[8]xi1>, vector<[8]xi1>, vector<[8]xi16>, vector<[8]xi16>) -> ()
-func.func @arm_sme_usmops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) -> vector<[2]x[2]xi64> {
-  %reuslt = arm_sme.usmops_4way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[2]x[2]xi64>
-  return %reuslt : vector<[2]x[2]xi64>
+func.func @arm_sme_usmops_4way_i16i16_to_i64(%vecA: vector<[8]xi16>, %vecB: vector<[8]xi16>) {
+  %result = arm_sme.usmops_4way %vecA, %vecB : vector<[8]xi16>, vector<[8]xi16> into vector<[2]x[2]xi64>
+  "test.some_use"(%result) : (vector<[2]x[2]xi64>) -> ()
+  return
+}
+
+//===----------------------------------------------------------------------===//
+// Operations on SME tile types allowed after conversion
+//===----------------------------------------------------------------------===//
+
+// -----
+
+// The following operations on SME tile types are permitted after conversion:
+//
+//   - arm_sme.copy_tile
+//   - arm_sme.get_tile
+//   - cf.br
+//   - any unregistered op such as 'test.some_use'.
+//
+// this test verifies this. Conversion will fail for operations with SME tile
+// types not in this list, this is tested in 'unsupported.mlir'.
+
+func.func @ops_on_tiles_legal_post_conversion(%ub : index) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %tile = arm_sme.get_tile : vector<[4]x[4]xf32>
+  %copy = arm_sme.copy_tile %tile : vector<[4]x[4]xf32>
+  cf.br ^bb1(%copy : vector<[4]x[4]xf32>)
+^bb1(%x : vector<[4]x[4]xf32>):
+  "test.some_use"(%x) : (vector<[4]x[4]xf32>) -> ()
+  return
 }
diff --git a/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir b/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
index 2c3868d7f25c..91c1b92b0122 100644
--- a/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
+++ b/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
@@ -141,7 +141,7 @@ func.func @very_excessive_spills(%useAllTiles : vector<[16]x[16]xi8>, %memref: m
   // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
   %loadSlice = arm_sme.load_tile_slice %memref[%c0, %c0], %mask, %tile, %c0 : memref<?x?xf32>, vector<[4]xi1>, vector<[4]x[4]xf32>
   "test.some_use"(%useAllTiles) : (vector<[16]x[16]xi8>) -> ()
-  return %loadSlice : vector<[4]x[4]xf32>
+  "test.some_use"(%loadSlice) : (vector<[4]x[4]xf32>) -> ()
 }
 // AFTER-TILE-ALLOC-LABEL: @very_excessive_spills
 //      AFTER-TILE-ALLOC: arm_sme.load_tile_slice
diff --git a/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir b/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir
index a62ca080ab8d..b2c41f284fb8 100644
--- a/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir
+++ b/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(func.func(convert-arm-sme-to-llvm))" -verify-diagnostics
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(func.func(convert-arm-sme-to-llvm))" -verify-diagnostics -split-input-file
 
 //===----------------------------------------------------------------------===//
 // arm_sme.outerproduct
@@ -6,9 +6,21 @@
 
 func.func @arm_sme_outerproduct_unsupported_type(%lhs : vector<[16]xi8>, %rhs : vector<[16]xi8>) {
   %acc = arm_sme.get_tile : vector<[16]x[16]xi8>
+  // expected-error@below {{unexpected operation with SME tile type after conversion to LLVM}}
   // expected-error@+2 {{failed to legalize operation 'arm_sme.outerproduct'}}
   // expected-error@+1 {{unsupported type}}
   %0 = arm_sme.outerproduct %lhs, %rhs  acc(%acc) : vector<[16]xi8>, vector<[16]xi8>
   "test.some_use"(%0) : (vector<[16]x[16]xi8>) -> ()
 }
 
+//===----------------------------------------------------------------------===//
+// Unsupported operations on SME tile types
+//===----------------------------------------------------------------------===//
+
+// -----
+
+func.func @unsupported_arith_op(%a : vector<[4]x[4]xf32>, %b : vector<[4]x[4]xf32>) {
+  // expected-error@below {{unexpected operation with SME tile type after conversion to LLVM}}
+  %0 = arith.addf %a, %b : vector<[4]x[4]xf32>
+  "test.some_use"(%0) : (vector<[4]x[4]xf32>) -> ()
+}
-- 
GitLab


From 8389177710c2cb7a7e0cf5c6c9b34aabb8048f39 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 16 May 2024 09:20:18 +0200
Subject: [PATCH 0975/1206] SimplifyLibCalls: Use IRBuilder helpers for
 creating intrinsics (#92288)

---
 .../lib/Transforms/Utils/SimplifyLibCalls.cpp | 82 +++++++------------
 1 file changed, 29 insertions(+), 53 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 69b0f78530d7..134eeba3996a 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -1855,14 +1855,7 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
 // Replace a libcall \p CI with a call to intrinsic \p IID
 static Value *replaceUnaryCall(CallInst *CI, IRBuilderBase &B,
                                Intrinsic::ID IID) {
-  // Propagate fast-math flags from the existing call to the new call.
-  IRBuilderBase::FastMathFlagGuard Guard(B);
-  B.setFastMathFlags(CI->getFastMathFlags());
-
-  Module *M = CI->getModule();
-  Value *V = CI->getArgOperand(0);
-  Function *F = Intrinsic::getDeclaration(M, IID, CI->getType());
-  CallInst *NewCall = B.CreateCall(F, V);
+  CallInst *NewCall = B.CreateUnaryIntrinsic(IID, CI->getArgOperand(0), CI);
   NewCall->takeName(CI);
   return copyFlags(*CI, NewCall);
 }
@@ -1987,10 +1980,9 @@ Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
   Value *RealReal = B.CreateFMul(Real, Real);
   Value *ImagImag = B.CreateFMul(Imag, Imag);
 
-  Function *FSqrt = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::sqrt,
-                                              CI->getType());
-  return copyFlags(
-      *CI, B.CreateCall(FSqrt, B.CreateFAdd(RealReal, ImagImag), "cabs"));
+  return copyFlags(*CI, B.CreateUnaryIntrinsic(Intrinsic::sqrt,
+                                               B.CreateFAdd(RealReal, ImagImag),
+                                               nullptr, "cabs"));
 }
 
 // Return a properly extended integer (DstWidth bits wide) if the operation is
@@ -2016,7 +2008,6 @@ static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
 Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
   Module *M = Pow->getModule();
   Value *Base = Pow->getArgOperand(0), *Expo = Pow->getArgOperand(1);
-  Module *Mod = Pow->getModule();
   Type *Ty = Pow->getType();
   bool Ignored;
 
@@ -2073,11 +2064,10 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
       // Create new exp{,2}() with the product as its argument.
       Value *FMul = B.CreateFMul(BaseFn->getArgOperand(0), Expo, "mul");
       ExpFn = BaseFn->doesNotAccessMemory()
-              ? B.CreateCall(Intrinsic::getDeclaration(Mod, ID, Ty),
-                             FMul, ExpName)
-              : emitUnaryFloatFnCall(FMul, TLI, LibFnDouble, LibFnFloat,
-                                     LibFnLongDouble, B,
-                                     BaseFn->getAttributes());
+                  ? B.CreateUnaryIntrinsic(ID, FMul, nullptr, ExpName)
+                  : emitUnaryFloatFnCall(FMul, TLI, LibFnDouble, LibFnFloat,
+                                         LibFnLongDouble, B,
+                                         BaseFn->getAttributes());
 
       // Since the new exp{,2}() is different from the original one, dead code
       // elimination cannot be trusted to remove it, since it may have side
@@ -2123,9 +2113,8 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
       double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
       Value *FMul = B.CreateFMul(Expo, ConstantFP::get(Ty, N), "mul");
       if (Pow->doesNotAccessMemory())
-        return copyFlags(*Pow, B.CreateCall(Intrinsic::getDeclaration(
-                                                Mod, Intrinsic::exp2, Ty),
-                                            FMul, "exp2"));
+        return copyFlags(*Pow, B.CreateUnaryIntrinsic(Intrinsic::exp2, FMul,
+                                                      nullptr, "exp2"));
       else
         return copyFlags(*Pow, emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2,
                                                     LibFunc_exp2f,
@@ -2165,9 +2154,8 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
     if (Log) {
       Value *FMul = B.CreateFMul(Log, Expo, "mul");
       if (Pow->doesNotAccessMemory())
-        return copyFlags(*Pow, B.CreateCall(Intrinsic::getDeclaration(
-                                                Mod, Intrinsic::exp2, Ty),
-                                            FMul, "exp2"));
+        return copyFlags(*Pow, B.CreateUnaryIntrinsic(Intrinsic::exp2, FMul,
+                                                      nullptr, "exp2"));
       else if (hasFloatFn(M, TLI, Ty, LibFunc_exp2, LibFunc_exp2f,
                           LibFunc_exp2l))
         return copyFlags(*Pow, emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2,
@@ -2183,11 +2171,8 @@ static Value *getSqrtCall(Value *V, AttributeList Attrs, bool NoErrno,
                           Module *M, IRBuilderBase &B,
                           const TargetLibraryInfo *TLI) {
   // If errno is never set, then use the intrinsic for sqrt().
-  if (NoErrno) {
-    Function *SqrtFn =
-        Intrinsic::getDeclaration(M, Intrinsic::sqrt, V->getType());
-    return B.CreateCall(SqrtFn, V, "sqrt");
-  }
+  if (NoErrno)
+    return B.CreateUnaryIntrinsic(Intrinsic::sqrt, V, nullptr, "sqrt");
 
   // Otherwise, use the libcall for sqrt().
   if (hasFloatFn(M, TLI, V->getType(), LibFunc_sqrt, LibFunc_sqrtf,
@@ -2232,10 +2217,8 @@ Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
     return nullptr;
 
   // Handle signed zero base by expanding to fabs(sqrt(x)).
-  if (!Pow->hasNoSignedZeros()) {
-    Function *FAbsFn = Intrinsic::getDeclaration(Mod, Intrinsic::fabs, Ty);
-    Sqrt = B.CreateCall(FAbsFn, Sqrt, "abs");
-  }
+  if (!Pow->hasNoSignedZeros())
+    Sqrt = B.CreateUnaryIntrinsic(Intrinsic::fabs, Sqrt, nullptr, "abs");
 
   Sqrt = copyFlags(*Pow, Sqrt);
 
@@ -2259,8 +2242,7 @@ static Value *createPowWithIntegerExponent(Value *Base, Value *Expo, Module *M,
                                            IRBuilderBase &B) {
   Value *Args[] = {Base, Expo};
   Type *Types[] = {Base->getType(), Expo->getType()};
-  Function *F = Intrinsic::getDeclaration(M, Intrinsic::powi, Types);
-  return B.CreateCall(F, Args);
+  return B.CreateIntrinsic(Intrinsic::powi, Types, Args);
 }
 
 Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
@@ -2442,9 +2424,8 @@ Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B) {
 
   Intrinsic::ID IID = Callee->getName().starts_with("fmin") ? Intrinsic::minnum
                                                             : Intrinsic::maxnum;
-  Function *F = Intrinsic::getDeclaration(CI->getModule(), IID, CI->getType());
-  return copyFlags(
-      *CI, B.CreateCall(F, {CI->getArgOperand(0), CI->getArgOperand(1)}));
+  return copyFlags(*CI, B.CreateBinaryIntrinsic(IID, CI->getArgOperand(0),
+                                                CI->getArgOperand(1)));
 }
 
 Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
@@ -2563,8 +2544,7 @@ Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
   if (ArgLb == PowLb || ArgID == Intrinsic::pow || ArgID == Intrinsic::powi) {
     Value *LogX =
         Log->doesNotAccessMemory()
-            ? B.CreateCall(Intrinsic::getDeclaration(Mod, LogID, Ty),
-                           Arg->getOperand(0), "log")
+            ? B.CreateUnaryIntrinsic(LogID, Arg->getOperand(0), nullptr, "log")
             : emitUnaryFloatFnCall(Arg->getOperand(0), TLI, LogNm, B, NoAttrs);
     Value *Y = Arg->getArgOperand(1);
     // Cast exponent to FP if integer.
@@ -2590,8 +2570,7 @@ Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
     else
       Eul = ConstantFP::get(Log->getType(), 10.0);
     Value *LogE = Log->doesNotAccessMemory()
-                      ? B.CreateCall(Intrinsic::getDeclaration(Mod, LogID, Ty),
-                                     Eul, "log")
+                      ? B.CreateUnaryIntrinsic(LogID, Eul, nullptr, "log")
                       : emitUnaryFloatFnCall(Eul, TLI, LogNm, B, NoAttrs);
     Value *MulY = B.CreateFMul(Arg->getArgOperand(0), LogE, "mul");
     // Since exp() may have side effects, e.g. errno,
@@ -2725,15 +2704,14 @@ Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
 
   // If we found a repeated factor, hoist it out of the square root and
   // replace it with the fabs of that factor.
-  Type *ArgType = I->getType();
-  Function *Fabs = Intrinsic::getDeclaration(M, Intrinsic::fabs, ArgType);
-  Value *FabsCall = B.CreateCall(Fabs, RepeatOp, "fabs");
+  Value *FabsCall =
+      B.CreateUnaryIntrinsic(Intrinsic::fabs, RepeatOp, nullptr, "fabs");
   if (OtherOp) {
     // If we found a non-repeated factor, we still need to get its square
     // root. We then multiply that by the value that was simplified out
     // of the square root calculation.
-    Function *Sqrt = Intrinsic::getDeclaration(M, Intrinsic::sqrt, ArgType);
-    Value *SqrtCall = B.CreateCall(Sqrt, OtherOp, "sqrt");
+    Value *SqrtCall =
+        B.CreateUnaryIntrinsic(Intrinsic::sqrt, OtherOp, nullptr, "sqrt");
     return copyFlags(*CI, B.CreateFMul(FabsCall, SqrtCall));
   }
   return copyFlags(*CI, FabsCall);
@@ -3002,9 +2980,8 @@ Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilderBase &B) {
   Type *RetType = CI->getType();
   Value *Op = CI->getArgOperand(0);
   Type *ArgType = Op->getType();
-  Function *F = Intrinsic::getDeclaration(CI->getCalledFunction()->getParent(),
-                                          Intrinsic::cttz, ArgType);
-  Value *V = B.CreateCall(F, {Op, B.getTrue()}, "cttz");
+  Value *V = B.CreateIntrinsic(Intrinsic::cttz, {ArgType}, {Op, B.getTrue()},
+                               nullptr, "cttz");
   V = B.CreateAdd(V, ConstantInt::get(V->getType(), 1));
   V = B.CreateIntCast(V, RetType, false);
 
@@ -3017,9 +2994,8 @@ Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilderBase &B) {
   // fls{,l,ll}(x) -> (int)(sizeInBits(x) - llvm.ctlz(x, false))
   Value *Op = CI->getArgOperand(0);
   Type *ArgType = Op->getType();
-  Function *F = Intrinsic::getDeclaration(CI->getCalledFunction()->getParent(),
-                                          Intrinsic::ctlz, ArgType);
-  Value *V = B.CreateCall(F, {Op, B.getFalse()}, "ctlz");
+  Value *V = B.CreateIntrinsic(Intrinsic::ctlz, {ArgType}, {Op, B.getFalse()},
+                               nullptr, "ctlz");
   V = B.CreateSub(ConstantInt::get(V->getType(), ArgType->getIntegerBitWidth()),
                   V);
   return B.CreateIntCast(V, CI->getType(), false);
-- 
GitLab


From de483ad513895c0adf7f21c7001c30f031998ea3 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 16 May 2024 00:38:37 -0700
Subject: [PATCH 0976/1206] [ADT] Deprecate StringRef::equals (#92351)

This patch deprecates StringRef::equals.  Note that I've migrated all
known users to operator==(StringRef, StringRef).
---
 llvm/include/llvm/ADT/StringRef.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h
index 8ed8e424cfe1..2120c04530b2 100644
--- a/llvm/include/llvm/ADT/StringRef.h
+++ b/llvm/include/llvm/ADT/StringRef.h
@@ -161,7 +161,8 @@ namespace llvm {
 
     /// equals - Check for string equality, this is more efficient than
     /// compare() when the relative ordering of inequal strings isn't needed.
-    [[nodiscard]] bool equals(StringRef RHS) const {
+    [[nodiscard]] LLVM_DEPRECATED("Use == instead",
+                                  "==") bool equals(StringRef RHS) const {
       return (Length == RHS.Length &&
               compareMemory(Data, RHS.Data, RHS.Length) == 0);
     }
-- 
GitLab


From 8f711aa32488beb3382de101a290e11b41ceb876 Mon Sep 17 00:00:00 2001
From: Mital Ashok <mital@mitalashok.co.uk>
Date: Thu, 16 May 2024 08:45:45 +0100
Subject: [PATCH 0977/1206] [libc++][test] `__cpp_lib_within_lifetime` ->
 `__cpp_lib_is_within_lifetime` (#91896)

The feature test macro was renamed when
[P2641R4](https://wg21.link/P2641R4) was adopted into the standard:
https://github.com/cplusplus/draft/commit/0facada4cadd97e1ba15bfaea76a804f1dc5c309
https://wg21.link/version.syn#lib:__cpp_lib_is_constant_evaluated
---
 libcxx/docs/FeatureTestMacroTable.rst         |  4 +-
 libcxx/include/version                        |  4 +-
 .../type_traits.version.compile.pass.cpp      | 68 +++++++++----------
 .../version.version.compile.pass.cpp          | 68 +++++++++----------
 .../generate_feature_test_macro_components.py | 18 ++---
 5 files changed, 82 insertions(+), 80 deletions(-)

diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 17b3476d2c86..17d2da907692 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -436,6 +436,8 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_hazard_pointer``                               *unimplemented*
     ---------------------------------------------------------- -----------------
+    ``__cpp_lib_is_within_lifetime``                           *unimplemented*
+    ---------------------------------------------------------- -----------------
     ``__cpp_lib_linalg``                                       *unimplemented*
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_out_ptr``                                      *unimplemented*
@@ -465,7 +467,5 @@ Status
     ``__cpp_lib_to_chars``                                     *unimplemented*
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_tuple_like``                                   *unimplemented*
-    ---------------------------------------------------------- -----------------
-    ``__cpp_lib_within_lifetime``                              *unimplemented*
     ========================================================== =================
 
diff --git a/libcxx/include/version b/libcxx/include/version
index ba116957b033..69556d731f1c 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -143,6 +143,7 @@ __cpp_lib_is_null_pointer                               201309L <type_traits>
 __cpp_lib_is_pointer_interconvertible                   201907L <type_traits>
 __cpp_lib_is_scoped_enum                                202011L <type_traits>
 __cpp_lib_is_swappable                                  201603L <type_traits>
+__cpp_lib_is_within_lifetime                            202306L <type_traits>
 __cpp_lib_jthread                                       201911L <stop_token> <thread>
 __cpp_lib_latch                                         201907L <latch>
 __cpp_lib_launder                                       201606L <new>
@@ -251,7 +252,6 @@ __cpp_lib_unreachable                                   202202L <utility>
 __cpp_lib_unwrap_ref                                    201811L <functional>
 __cpp_lib_variant                                       202102L <variant>
 __cpp_lib_void_t                                        201411L <type_traits>
-__cpp_lib_within_lifetime                               202306L <type_traits>
 
 */
 
@@ -520,6 +520,7 @@ __cpp_lib_within_lifetime                               202306L <type_traits>
 // # define __cpp_lib_function_ref                         202306L
 // # define __cpp_lib_generate_random                      202403L
 // # define __cpp_lib_hazard_pointer                       202306L
+// # define __cpp_lib_is_within_lifetime                   202306L
 // # define __cpp_lib_linalg                               202311L
 # undef  __cpp_lib_out_ptr
 // # define __cpp_lib_out_ptr                              202311L
@@ -538,7 +539,6 @@ __cpp_lib_within_lifetime                               202306L <type_traits>
 // # define __cpp_lib_to_chars                             202306L
 # undef  __cpp_lib_tuple_like
 // # define __cpp_lib_tuple_like                           202311L
-// # define __cpp_lib_within_lifetime                      202306L
 #endif
 
 // clang-format on
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
index b1dd76b98720..98277f832f5a 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
@@ -30,6 +30,7 @@
     __cpp_lib_is_pointer_interconvertible          201907L [C++20]
     __cpp_lib_is_scoped_enum                       202011L [C++23]
     __cpp_lib_is_swappable                         201603L [C++17]
+    __cpp_lib_is_within_lifetime                   202306L [C++26]
     __cpp_lib_logical_traits                       201510L [C++17]
     __cpp_lib_reference_from_temporary             202202L [C++23]
     __cpp_lib_remove_cvref                         201711L [C++20]
@@ -38,7 +39,6 @@
     __cpp_lib_type_identity                        201806L [C++20]
     __cpp_lib_type_trait_variable_templates        201510L [C++17]
     __cpp_lib_void_t                               201411L [C++17]
-    __cpp_lib_within_lifetime                      202306L [C++26]
 */
 
 #include <type_traits>
@@ -102,6 +102,10 @@
 #   error "__cpp_lib_is_swappable should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_is_within_lifetime
+#   error "__cpp_lib_is_within_lifetime should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_logical_traits
 #   error "__cpp_lib_logical_traits should not be defined before c++17"
 # endif
@@ -134,10 +138,6 @@
 #   error "__cpp_lib_void_t should not be defined before c++17"
 # endif
 
-# ifdef __cpp_lib_within_lifetime
-#   error "__cpp_lib_within_lifetime should not be defined before c++26"
-# endif
-
 #elif TEST_STD_VER == 14
 
 # ifdef __cpp_lib_bool_constant
@@ -205,6 +205,10 @@
 #   error "__cpp_lib_is_swappable should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_is_within_lifetime
+#   error "__cpp_lib_is_within_lifetime should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_logical_traits
 #   error "__cpp_lib_logical_traits should not be defined before c++17"
 # endif
@@ -243,10 +247,6 @@
 #   error "__cpp_lib_void_t should not be defined before c++17"
 # endif
 
-# ifdef __cpp_lib_within_lifetime
-#   error "__cpp_lib_within_lifetime should not be defined before c++26"
-# endif
-
 #elif TEST_STD_VER == 17
 
 # ifndef __cpp_lib_bool_constant
@@ -329,6 +329,10 @@
 #   error "__cpp_lib_is_swappable should have the value 201603L in c++17"
 # endif
 
+# ifdef __cpp_lib_is_within_lifetime
+#   error "__cpp_lib_is_within_lifetime should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_logical_traits
 #   error "__cpp_lib_logical_traits should be defined in c++17"
 # endif
@@ -376,10 +380,6 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++17"
 # endif
 
-# ifdef __cpp_lib_within_lifetime
-#   error "__cpp_lib_within_lifetime should not be defined before c++26"
-# endif
-
 #elif TEST_STD_VER == 20
 
 # ifndef __cpp_lib_bool_constant
@@ -489,6 +489,10 @@
 #   error "__cpp_lib_is_swappable should have the value 201603L in c++20"
 # endif
 
+# ifdef __cpp_lib_is_within_lifetime
+#   error "__cpp_lib_is_within_lifetime should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_logical_traits
 #   error "__cpp_lib_logical_traits should be defined in c++20"
 # endif
@@ -542,10 +546,6 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++20"
 # endif
 
-# ifdef __cpp_lib_within_lifetime
-#   error "__cpp_lib_within_lifetime should not be defined before c++26"
-# endif
-
 #elif TEST_STD_VER == 23
 
 # ifndef __cpp_lib_bool_constant
@@ -658,6 +658,10 @@
 #   error "__cpp_lib_is_swappable should have the value 201603L in c++23"
 # endif
 
+# ifdef __cpp_lib_is_within_lifetime
+#   error "__cpp_lib_is_within_lifetime should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_logical_traits
 #   error "__cpp_lib_logical_traits should be defined in c++23"
 # endif
@@ -720,10 +724,6 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++23"
 # endif
 
-# ifdef __cpp_lib_within_lifetime
-#   error "__cpp_lib_within_lifetime should not be defined before c++26"
-# endif
-
 #elif TEST_STD_VER > 23
 
 # ifndef __cpp_lib_bool_constant
@@ -836,6 +836,19 @@
 #   error "__cpp_lib_is_swappable should have the value 201603L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_is_within_lifetime
+#     error "__cpp_lib_is_within_lifetime should be defined in c++26"
+#   endif
+#   if __cpp_lib_is_within_lifetime != 202306L
+#     error "__cpp_lib_is_within_lifetime should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_is_within_lifetime
+#     error "__cpp_lib_is_within_lifetime should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_logical_traits
 #   error "__cpp_lib_logical_traits should be defined in c++26"
 # endif
@@ -898,18 +911,5 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++26"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_within_lifetime
-#     error "__cpp_lib_within_lifetime should be defined in c++26"
-#   endif
-#   if __cpp_lib_within_lifetime != 202306L
-#     error "__cpp_lib_within_lifetime should have the value 202306L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_within_lifetime
-#     error "__cpp_lib_within_lifetime should not be defined because it is unimplemented in libc++!"
-#   endif
-# endif
-
 #endif // TEST_STD_VER > 23
 
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index d7035d7e5e3a..7829e06f9076 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -130,6 +130,7 @@
     __cpp_lib_is_pointer_interconvertible                   201907L [C++20]
     __cpp_lib_is_scoped_enum                                202011L [C++23]
     __cpp_lib_is_swappable                                  201603L [C++17]
+    __cpp_lib_is_within_lifetime                            202306L [C++26]
     __cpp_lib_jthread                                       201911L [C++20]
     __cpp_lib_latch                                         201907L [C++20]
     __cpp_lib_launder                                       201606L [C++17]
@@ -232,7 +233,6 @@
     __cpp_lib_unwrap_ref                                    201811L [C++20]
     __cpp_lib_variant                                       202102L [C++17]
     __cpp_lib_void_t                                        201411L [C++17]
-    __cpp_lib_within_lifetime                               202306L [C++26]
 */
 
 #include <version>
@@ -680,6 +680,10 @@
 #   error "__cpp_lib_is_swappable should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_is_within_lifetime
+#   error "__cpp_lib_is_within_lifetime should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_jthread
 #   error "__cpp_lib_jthread should not be defined before c++20"
 # endif
@@ -1060,10 +1064,6 @@
 #   error "__cpp_lib_void_t should not be defined before c++17"
 # endif
 
-# ifdef __cpp_lib_within_lifetime
-#   error "__cpp_lib_within_lifetime should not be defined before c++26"
-# endif
-
 #elif TEST_STD_VER == 14
 
 # ifdef __cpp_lib_adaptor_iterator_pair_constructor
@@ -1530,6 +1530,10 @@
 #   error "__cpp_lib_is_swappable should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_is_within_lifetime
+#   error "__cpp_lib_is_within_lifetime should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_jthread
 #   error "__cpp_lib_jthread should not be defined before c++20"
 # endif
@@ -1958,10 +1962,6 @@
 #   error "__cpp_lib_void_t should not be defined before c++17"
 # endif
 
-# ifdef __cpp_lib_within_lifetime
-#   error "__cpp_lib_within_lifetime should not be defined before c++26"
-# endif
-
 #elif TEST_STD_VER == 17
 
 # ifdef __cpp_lib_adaptor_iterator_pair_constructor
@@ -2518,6 +2518,10 @@
 #   error "__cpp_lib_is_swappable should have the value 201603L in c++17"
 # endif
 
+# ifdef __cpp_lib_is_within_lifetime
+#   error "__cpp_lib_is_within_lifetime should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_jthread
 #   error "__cpp_lib_jthread should not be defined before c++20"
 # endif
@@ -3054,10 +3058,6 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++17"
 # endif
 
-# ifdef __cpp_lib_within_lifetime
-#   error "__cpp_lib_within_lifetime should not be defined before c++26"
-# endif
-
 #elif TEST_STD_VER == 20
 
 # ifdef __cpp_lib_adaptor_iterator_pair_constructor
@@ -3797,6 +3797,10 @@
 #   error "__cpp_lib_is_swappable should have the value 201603L in c++20"
 # endif
 
+# ifdef __cpp_lib_is_within_lifetime
+#   error "__cpp_lib_is_within_lifetime should not be defined before c++26"
+# endif
+
 # if !defined(_LIBCPP_HAS_NO_THREADS) && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && (!defined(_LIBCPP_VERSION) || _LIBCPP_AVAILABILITY_HAS_SYNC)
 #   ifndef __cpp_lib_jthread
 #     error "__cpp_lib_jthread should be defined in c++20"
@@ -4438,10 +4442,6 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++20"
 # endif
 
-# ifdef __cpp_lib_within_lifetime
-#   error "__cpp_lib_within_lifetime should not be defined before c++26"
-# endif
-
 #elif TEST_STD_VER == 23
 
 # ifndef __cpp_lib_adaptor_iterator_pair_constructor
@@ -5259,6 +5259,10 @@
 #   error "__cpp_lib_is_swappable should have the value 201603L in c++23"
 # endif
 
+# ifdef __cpp_lib_is_within_lifetime
+#   error "__cpp_lib_is_within_lifetime should not be defined before c++26"
+# endif
+
 # if !defined(_LIBCPP_HAS_NO_THREADS) && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && (!defined(_LIBCPP_VERSION) || _LIBCPP_AVAILABILITY_HAS_SYNC)
 #   ifndef __cpp_lib_jthread
 #     error "__cpp_lib_jthread should be defined in c++23"
@@ -6056,10 +6060,6 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++23"
 # endif
 
-# ifdef __cpp_lib_within_lifetime
-#   error "__cpp_lib_within_lifetime should not be defined before c++26"
-# endif
-
 #elif TEST_STD_VER > 23
 
 # ifndef __cpp_lib_adaptor_iterator_pair_constructor
@@ -7042,6 +7042,19 @@
 #   error "__cpp_lib_is_swappable should have the value 201603L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_is_within_lifetime
+#     error "__cpp_lib_is_within_lifetime should be defined in c++26"
+#   endif
+#   if __cpp_lib_is_within_lifetime != 202306L
+#     error "__cpp_lib_is_within_lifetime should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_is_within_lifetime
+#     error "__cpp_lib_is_within_lifetime should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # if !defined(_LIBCPP_HAS_NO_THREADS) && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN) && (!defined(_LIBCPP_VERSION) || _LIBCPP_AVAILABILITY_HAS_SYNC)
 #   ifndef __cpp_lib_jthread
 #     error "__cpp_lib_jthread should be defined in c++26"
@@ -7911,18 +7924,5 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++26"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_within_lifetime
-#     error "__cpp_lib_within_lifetime should be defined in c++26"
-#   endif
-#   if __cpp_lib_within_lifetime != 202306L
-#     error "__cpp_lib_within_lifetime should have the value 202306L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_within_lifetime
-#     error "__cpp_lib_within_lifetime should not be defined because it is unimplemented in libc++!"
-#   endif
-# endif
-
 #endif // TEST_STD_VER > 23
 
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index d0171c84acbc..b04cb4f51155 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -767,6 +767,16 @@ feature_test_macros = [
             "values": {"c++17": 201603},
             "headers": ["type_traits"],
         },
+        {
+            "name": "__cpp_lib_is_within_lifetime",
+            # Note this name was changed from "__cpp_lib_within_lifetime" when the paper was adopted
+            # https://github.com/cplusplus/draft/commit/0facada4cadd97e1ba15bfaea76a804f1dc5c309
+            "values": {
+                "c++26": 202306  # P2641R4 Checking if a union alternative is active
+            },
+            "headers": ["type_traits"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_jthread",
             "values": {"c++20": 201911},
@@ -1342,14 +1352,6 @@ feature_test_macros = [
             "values": {"c++17": 201411},
             "headers": ["type_traits"],
         },
-        {
-            "name": "__cpp_lib_within_lifetime",
-            "values": {
-                "c++26": 202306  # P2641R4 Checking if a union alternative is active
-            },
-            "headers": ["type_traits"],
-            "unimplemented": True,
-        },
     ]
 ]
 
-- 
GitLab


From dcd32bd65f16e80db2485e6e02b62d6a91c3cddf Mon Sep 17 00:00:00 2001
From: Adam Siemieniuk <adam.siemieniuk@intel.com>
Date: Thu, 16 May 2024 10:05:12 +0200
Subject: [PATCH 0978/1206] [mlir][tensor] Fold pack-unpack with unbalanced
 outer_dims_perm attr (#92234)

Extends pack/unpack perm attribute checker to account for cases when the
optional outer_dims_perm attribute might be missing in one operation and
the other one has explicit identity permutation. This enables
canonicalizer to fold more unpack(pack(x)) variants.
---
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp   |  8 ++++++-
 mlir/test/Dialect/Tensor/canonicalize.mlir | 26 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index d3b1754cbe1c..8a6df82abb31 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -4112,7 +4112,13 @@ Speculation::Speculatability PackOp::getSpeculatability() {
 static bool hasSameInnerOuterAttribute(PackOp packOp, UnPackOp unPackOp) {
   if (packOp.getInnerDimsPos() != unPackOp.getInnerDimsPos())
     return false;
-  return packOp.getOuterDimsPerm() == unPackOp.getOuterDimsPerm();
+  if (packOp.getOuterDimsPerm() == unPackOp.getOuterDimsPerm())
+    return true;
+  // Outer dims permutation is optional.
+  // To compare unbalanced pack-unpack pair, treat no permutation as equal to
+  // identity permutation.
+  return isIdentityPermutation(packOp.getOuterDimsPerm()) &&
+         isIdentityPermutation(unPackOp.getOuterDimsPerm());
 }
 
 // Return true if pack and unpack have the same tiles.
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
index 8036d996d232..b5a82eb3e903 100644
--- a/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -2252,6 +2252,32 @@ func.func @pack_unpack_dynamic_with_padding(%t: tensor<?x?x?x?xf32>, %dim1: inde
 
 // -----
 
+// CHECK: func.func @pack_outer_dims_unpack_no_outer_dims(
+// CHECK-SAME: %[[T:.+]]: tensor<16x16x?x?xf32>,
+// CHECK: return %[[T]] : tensor<16x16x?x?xf32>
+func.func @pack_outer_dims_unpack_no_outer_dims(%t: tensor<16x16x?x?xf32>, %tile1: index, %tile2: index) -> tensor<16x16x?x?xf32> {
+  %tensor_empty = tensor.empty() : tensor<128x128xf32>
+  %unpacked = tensor.unpack %t inner_dims_pos = [0, 1] inner_tiles = [%tile1, %tile2] into %tensor_empty : tensor<16x16x?x?xf32> -> tensor<128x128xf32>
+  %tensor_empty1 = tensor.empty(%tile1, %tile2) : tensor<16x16x?x?xf32>
+  %packed = tensor.pack %unpacked outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [%tile1, %tile2] into %tensor_empty1 : tensor<128x128xf32> -> tensor<16x16x?x?xf32>
+  return %packed : tensor<16x16x?x?xf32>
+}
+
+// -----
+
+// CHECK: func.func @pack_no_outer_dims_unpack_outer_dims(
+// CHECK-SAME: %[[T:.+]]: tensor<16x16x?x?xf32>,
+// CHECK: return %[[T]] : tensor<16x16x?x?xf32>
+func.func @pack_no_outer_dims_unpack_outer_dims(%t: tensor<16x16x?x?xf32>, %tile1: index, %tile2: index) -> tensor<16x16x?x?xf32> {
+  %tensor_empty = tensor.empty() : tensor<128x128xf32>
+  %unpacked = tensor.unpack %t outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [%tile1, %tile2] into %tensor_empty : tensor<16x16x?x?xf32> -> tensor<128x128xf32>
+  %tensor_empty1 = tensor.empty(%tile1, %tile2) : tensor<16x16x?x?xf32>
+  %packed = tensor.pack %unpacked inner_dims_pos = [0, 1] inner_tiles = [%tile1, %tile2] into %tensor_empty1 : tensor<128x128xf32> -> tensor<16x16x?x?xf32>
+  return %packed : tensor<16x16x?x?xf32>
+}
+
+// -----
+
 // CHECK: func.func @invalid_empty_negative_size
 // CHECK: %[[IDX:.*]] = index.constant
 // CHECK: %[[T:.*]] = tensor.empty(%[[IDX]]) : tensor<4x5x?xf32>
-- 
GitLab


From cd5ee2715e89ad31982f91cc85fc3939977f2f4e Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer@amd.com>
Date: Thu, 16 May 2024 09:10:59 +0100
Subject: [PATCH 0979/1206] [reland][flang] Initial debug info support for
 local variables (#92304)

This is same as #90905 with an added fix. The issue was that we
generated variable info even when user asked for line-tables-only. This
caused llvm dwarf generation code to fail an assertion as it expected an
empty variable list.

Fixed by not generating debug info for variables when user wants only
line table. I also updated a test check for this case.
---
 .../flang}/Optimizer/CodeGen/CGOps.h          |  1 +
 .../include/flang/Optimizer/CodeGen/CGOps.td  | 34 +++++++
 .../flang/Optimizer/CodeGen/CGPasses.td       |  4 +
 .../include/flang/Optimizer/CodeGen/CodeGen.h |  6 +-
 flang/include/flang/Tools/CLOptions.inc       | 11 ++-
 flang/lib/Optimizer/CodeGen/CGOps.cpp         |  2 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 50 +++++++---
 flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp  | 49 ++++++++--
 .../lib/Optimizer/Transforms/AddDebugInfo.cpp | 60 +++++++++++-
 .../Transforms/DebugTypeGenerator.cpp         | 10 +-
 flang/test/Fir/declare-codegen.fir            | 22 +++--
 flang/test/Fir/dummy-scope-codegen.fir        | 11 ++-
 flang/test/Transforms/debug-local-var-2.f90   | 94 +++++++++++++++++++
 flang/test/Transforms/debug-local-var.f90     | 54 +++++++++++
 14 files changed, 361 insertions(+), 47 deletions(-)
 rename flang/{lib => include/flang}/Optimizer/CodeGen/CGOps.h (94%)
 create mode 100644 flang/test/Transforms/debug-local-var-2.f90
 create mode 100644 flang/test/Transforms/debug-local-var.f90

diff --git a/flang/lib/Optimizer/CodeGen/CGOps.h b/flang/include/flang/Optimizer/CodeGen/CGOps.h
similarity index 94%
rename from flang/lib/Optimizer/CodeGen/CGOps.h
rename to flang/include/flang/Optimizer/CodeGen/CGOps.h
index b5a6d5bb9a9e..df909d9ee81c 100644
--- a/flang/lib/Optimizer/CodeGen/CGOps.h
+++ b/flang/include/flang/Optimizer/CodeGen/CGOps.h
@@ -13,6 +13,7 @@
 #ifndef OPTIMIZER_CODEGEN_CGOPS_H
 #define OPTIMIZER_CODEGEN_CGOPS_H
 
+#include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 
diff --git a/flang/include/flang/Optimizer/CodeGen/CGOps.td b/flang/include/flang/Optimizer/CodeGen/CGOps.td
index 35e70fa2ffa3..c375edee1fa7 100644
--- a/flang/include/flang/Optimizer/CodeGen/CGOps.td
+++ b/flang/include/flang/Optimizer/CodeGen/CGOps.td
@@ -16,6 +16,8 @@
 
 include "mlir/IR/SymbolInterfaces.td"
 include "flang/Optimizer/Dialect/FIRTypes.td"
+include "flang/Optimizer/Dialect/FIRAttr.td"
+include "mlir/IR/BuiltinAttributes.td"
 
 def fircg_Dialect : Dialect {
   let name = "fircg";
@@ -202,4 +204,36 @@ def fircg_XArrayCoorOp : fircg_Op<"ext_array_coor", [AttrSizedOperandSegments]>
   }];
 }
 
+// Extended Declare operation.
+def fircg_XDeclareOp : fircg_Op<"ext_declare", [AttrSizedOperandSegments]> {
+  let summary = "for internal conversion only";
+
+  let description = [{
+    Prior to lowering to LLVM IR dialect, a DeclareOp will
+    be converted to an extended DeclareOp.
+  }];
+
+  let arguments = (ins
+    AnyRefOrBox:$memref,
+    Variadic<AnyIntegerType>:$shape,
+    Variadic<AnyIntegerType>:$shift,
+    Variadic<AnyIntegerType>:$typeparams,
+    Optional<fir_DummyScopeType>:$dummy_scope,
+    Builtin_StringAttr:$uniq_name
+  );
+  let results = (outs AnyRefOrBox);
+
+  let assemblyFormat = [{
+    $memref (`(` $shape^ `)`)? (`origin` $shift^)? (`typeparams` $typeparams^)?
+    (`dummy_scope` $dummy_scope^)?
+    attr-dict `:` functional-type(operands, results)
+  }];
+
+  let extraClassDeclaration = [{
+    // Shape is optional, but if it exists, it will be at offset 1.
+    unsigned shapeOffset() { return 1; }
+    unsigned shiftOffset() { return shapeOffset() + getShape().size(); }
+  }];
+}
+
 #endif
diff --git a/flang/include/flang/Optimizer/CodeGen/CGPasses.td b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
index f524fb423734..565920e55e6a 100644
--- a/flang/include/flang/Optimizer/CodeGen/CGPasses.td
+++ b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
@@ -47,6 +47,10 @@ def CodeGenRewrite : Pass<"cg-rewrite", "mlir::ModuleOp"> {
   let dependentDialects = [
     "fir::FIROpsDialect", "fir::FIRCodeGenDialect"
   ];
+  let options = [
+    Option<"preserveDeclare", "preserve-declare", "bool", /*default=*/"false",
+           "Preserve DeclareOp during pre codegen re-write.">
+  ];
   let statistics = [
     Statistic<"numDCE", "num-dce'd", "Number of operations eliminated">
   ];
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 26097dabf56c..4d2b191b46d0 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -30,7 +30,8 @@ struct NameUniquer;
 
 /// Prerequiste pass for code gen. Perform intermediate rewrites to perform
 /// the code gen (to LLVM-IR dialect) conversion.
-std::unique_ptr<mlir::Pass> createFirCodeGenRewritePass();
+std::unique_ptr<mlir::Pass> createFirCodeGenRewritePass(
+    CodeGenRewriteOptions Options = CodeGenRewriteOptions{});
 
 /// FirTargetRewritePass options.
 struct TargetRewriteOptions {
@@ -88,7 +89,8 @@ void populateFIRToLLVMConversionPatterns(fir::LLVMTypeConverter &converter,
                                          fir::FIRToLLVMPassOptions &options);
 
 /// Populate the pattern set with the PreCGRewrite patterns.
-void populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns);
+void populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns,
+                                  bool preserveDeclare);
 
 // declarative passes
 #define GEN_PASS_REGISTRATION
diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc
index 1817dd6ca4a7..9c4a30dcf301 100644
--- a/flang/include/flang/Tools/CLOptions.inc
+++ b/flang/include/flang/Tools/CLOptions.inc
@@ -174,9 +174,11 @@ inline void addMemoryAllocationOpt(mlir::PassManager &pm) {
 }
 
 #if !defined(FLANG_EXCLUDE_CODEGEN)
-inline void addCodeGenRewritePass(mlir::PassManager &pm) {
-  addPassConditionally(
-      pm, disableCodeGenRewrite, fir::createFirCodeGenRewritePass);
+inline void addCodeGenRewritePass(mlir::PassManager &pm, bool preserveDeclare) {
+  fir::CodeGenRewriteOptions options;
+  options.preserveDeclare = preserveDeclare;
+  addPassConditionally(pm, disableCodeGenRewrite,
+      [&]() { return fir::createFirCodeGenRewritePass(options); });
 }
 
 inline void addTargetRewritePass(mlir::PassManager &pm) {
@@ -358,7 +360,8 @@ inline void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
     MLIRToLLVMPassPipelineConfig config, llvm::StringRef inputFilename = {}) {
   fir::addBoxedProcedurePass(pm);
   addNestedPassToAllTopLevelOperations(pm, fir::createAbstractResultOpt);
-  fir::addCodeGenRewritePass(pm);
+  fir::addCodeGenRewritePass(
+      pm, (config.DebugInfo != llvm::codegenoptions::NoDebugInfo));
   fir::addTargetRewritePass(pm);
   fir::addExternalNameConversionPass(pm, config.Underscoring);
   fir::createDebugPasses(pm, config.DebugInfo, config.OptLevel, inputFilename);
diff --git a/flang/lib/Optimizer/CodeGen/CGOps.cpp b/flang/lib/Optimizer/CodeGen/CGOps.cpp
index 44d07d26dd2b..6b8ba7452555 100644
--- a/flang/lib/Optimizer/CodeGen/CGOps.cpp
+++ b/flang/lib/Optimizer/CodeGen/CGOps.cpp
@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CGOps.h"
+#include "flang/Optimizer/CodeGen/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 21154902d23f..72172f63888e 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -12,7 +12,7 @@
 
 #include "flang/Optimizer/CodeGen/CodeGen.h"
 
-#include "CGOps.h"
+#include "flang/Optimizer/CodeGen/CGOps.h"
 #include "flang/Optimizer/CodeGen/CodeGenOpenMP.h"
 #include "flang/Optimizer/CodeGen/FIROpPatterns.h"
 #include "flang/Optimizer/CodeGen/TypeConverter.h"
@@ -170,6 +170,28 @@ genAllocationScaleSize(OP op, mlir::Type ity,
   return nullptr;
 }
 
+namespace {
+struct DeclareOpConversion : public fir::FIROpConversion<fir::cg::XDeclareOp> {
+public:
+  using FIROpConversion::FIROpConversion;
+  mlir::LogicalResult
+  matchAndRewrite(fir::cg::XDeclareOp declareOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    auto memRef = adaptor.getOperands()[0];
+    if (auto fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(declareOp.getLoc())) {
+      if (auto varAttr =
+              mlir::dyn_cast_or_null<mlir::LLVM::DILocalVariableAttr>(
+                  fusedLoc.getMetadata())) {
+        rewriter.create<mlir::LLVM::DbgDeclareOp>(memRef.getLoc(), memRef,
+                                                  varAttr, nullptr);
+      }
+    }
+    rewriter.replaceOp(declareOp, memRef);
+    return mlir::success();
+  }
+};
+} // namespace
+
 namespace {
 /// convert to LLVM IR dialect `alloca`
 struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
@@ -3714,19 +3736,19 @@ void fir::populateFIRToLLVMConversionPatterns(
       BoxOffsetOpConversion, BoxProcHostOpConversion, BoxRankOpConversion,
       BoxTypeCodeOpConversion, BoxTypeDescOpConversion, CallOpConversion,
       CmpcOpConversion, ConstcOpConversion, ConvertOpConversion,
-      CoordinateOpConversion, DTEntryOpConversion, DivcOpConversion,
-      EmboxOpConversion, EmboxCharOpConversion, EmboxProcOpConversion,
-      ExtractValueOpConversion, FieldIndexOpConversion, FirEndOpConversion,
-      FreeMemOpConversion, GlobalLenOpConversion, GlobalOpConversion,
-      HasValueOpConversion, InsertOnRangeOpConversion, InsertValueOpConversion,
-      IsPresentOpConversion, LenParamIndexOpConversion, LoadOpConversion,
-      MulcOpConversion, NegcOpConversion, NoReassocOpConversion,
-      SelectCaseOpConversion, SelectOpConversion, SelectRankOpConversion,
-      SelectTypeOpConversion, ShapeOpConversion, ShapeShiftOpConversion,
-      ShiftOpConversion, SliceOpConversion, StoreOpConversion,
-      StringLitOpConversion, SubcOpConversion, TypeDescOpConversion,
-      TypeInfoOpConversion, UnboxCharOpConversion, UnboxProcOpConversion,
-      UndefOpConversion, UnreachableOpConversion,
+      CoordinateOpConversion, DTEntryOpConversion, DeclareOpConversion,
+      DivcOpConversion, EmboxOpConversion, EmboxCharOpConversion,
+      EmboxProcOpConversion, ExtractValueOpConversion, FieldIndexOpConversion,
+      FirEndOpConversion, FreeMemOpConversion, GlobalLenOpConversion,
+      GlobalOpConversion, HasValueOpConversion, InsertOnRangeOpConversion,
+      InsertValueOpConversion, IsPresentOpConversion, LenParamIndexOpConversion,
+      LoadOpConversion, MulcOpConversion, NegcOpConversion,
+      NoReassocOpConversion, SelectCaseOpConversion, SelectOpConversion,
+      SelectRankOpConversion, SelectTypeOpConversion, ShapeOpConversion,
+      ShapeShiftOpConversion, ShiftOpConversion, SliceOpConversion,
+      StoreOpConversion, StringLitOpConversion, SubcOpConversion,
+      TypeDescOpConversion, TypeInfoOpConversion, UnboxCharOpConversion,
+      UnboxProcOpConversion, UndefOpConversion, UnreachableOpConversion,
       UnrealizedConversionCastOpConversion, XArrayCoorOpConversion,
       XEmboxOpConversion, XReboxOpConversion, ZeroOpConversion>(converter,
                                                                 options);
diff --git a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
index 5bd3ec8d1845..c54a7457db76 100644
--- a/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/PreCGRewrite.cpp
@@ -12,8 +12,8 @@
 
 #include "flang/Optimizer/CodeGen/CodeGen.h"
 
-#include "CGOps.h"
 #include "flang/Optimizer/Builder/Todo.h" // remove when TODO's are done
+#include "flang/Optimizer/CodeGen/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
@@ -270,13 +270,43 @@ public:
 };
 
 class DeclareOpConversion : public mlir::OpRewritePattern<fir::DeclareOp> {
+  bool preserveDeclare;
+
 public:
   using OpRewritePattern::OpRewritePattern;
+  DeclareOpConversion(mlir::MLIRContext *ctx, bool preserveDecl)
+      : OpRewritePattern(ctx), preserveDeclare(preserveDecl) {}
 
   mlir::LogicalResult
   matchAndRewrite(fir::DeclareOp declareOp,
                   mlir::PatternRewriter &rewriter) const override {
-    rewriter.replaceOp(declareOp, declareOp.getMemref());
+    if (!preserveDeclare) {
+      rewriter.replaceOp(declareOp, declareOp.getMemref());
+      return mlir::success();
+    }
+    auto loc = declareOp.getLoc();
+    llvm::SmallVector<mlir::Value> shapeOpers;
+    llvm::SmallVector<mlir::Value> shiftOpers;
+    if (auto shapeVal = declareOp.getShape()) {
+      if (auto shapeOp = mlir::dyn_cast<fir::ShapeOp>(shapeVal.getDefiningOp()))
+        populateShape(shapeOpers, shapeOp);
+      else if (auto shiftOp =
+                   mlir::dyn_cast<fir::ShapeShiftOp>(shapeVal.getDefiningOp()))
+        populateShapeAndShift(shapeOpers, shiftOpers, shiftOp);
+      else if (auto shiftOp =
+                   mlir::dyn_cast<fir::ShiftOp>(shapeVal.getDefiningOp()))
+        populateShift(shiftOpers, shiftOp);
+      else
+        return mlir::failure();
+    }
+    // FIXME: Add FortranAttrs and CudaAttrs
+    auto xDeclOp = rewriter.create<fir::cg::XDeclareOp>(
+        loc, declareOp.getType(), declareOp.getMemref(), shapeOpers, shiftOpers,
+        declareOp.getTypeparams(), declareOp.getDummyScope(),
+        declareOp.getUniqName());
+    LLVM_DEBUG(llvm::dbgs()
+               << "rewriting " << declareOp << " to " << xDeclOp << '\n');
+    rewriter.replaceOp(declareOp, xDeclOp.getOperation()->getResults());
     return mlir::success();
   }
 };
@@ -297,6 +327,7 @@ public:
 
 class CodeGenRewrite : public fir::impl::CodeGenRewriteBase<CodeGenRewrite> {
 public:
+  CodeGenRewrite(fir::CodeGenRewriteOptions opts) : Base(opts) {}
   void runOnOperation() override final {
     mlir::ModuleOp mod = getOperation();
 
@@ -314,7 +345,7 @@ public:
                    mlir::cast<fir::BaseBoxType>(embox.getType()).getEleTy()));
     });
     mlir::RewritePatternSet patterns(&context);
-    fir::populatePreCGRewritePatterns(patterns);
+    fir::populatePreCGRewritePatterns(patterns, preserveDeclare);
     if (mlir::failed(
             mlir::applyPartialConversion(mod, target, std::move(patterns)))) {
       mlir::emitError(mlir::UnknownLoc::get(&context),
@@ -330,12 +361,14 @@ public:
 
 } // namespace
 
-std::unique_ptr<mlir::Pass> fir::createFirCodeGenRewritePass() {
-  return std::make_unique<CodeGenRewrite>();
+std::unique_ptr<mlir::Pass>
+fir::createFirCodeGenRewritePass(fir::CodeGenRewriteOptions Options) {
+  return std::make_unique<CodeGenRewrite>(Options);
 }
 
-void fir::populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns) {
+void fir::populatePreCGRewritePatterns(mlir::RewritePatternSet &patterns,
+                                       bool preserveDeclare) {
   patterns.insert<EmboxConversion, ArrayCoorConversion, ReboxConversion,
-                  DeclareOpConversion, DummyScopeOpConversion>(
-      patterns.getContext());
+                  DummyScopeOpConversion>(patterns.getContext());
+  patterns.add<DeclareOpConversion>(patterns.getContext(), preserveDeclare);
 }
diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 908c8fc96f63..07e8aed4cd07 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -15,6 +15,7 @@
 #include "flang/Common/Version.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/CodeGen/CGOps.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
@@ -45,13 +46,59 @@ namespace fir {
 namespace {
 
 class AddDebugInfoPass : public fir::impl::AddDebugInfoBase<AddDebugInfoPass> {
+  void handleDeclareOp(fir::cg::XDeclareOp declOp,
+                       mlir::LLVM::DIFileAttr fileAttr,
+                       mlir::LLVM::DIScopeAttr scopeAttr,
+                       fir::DebugTypeGenerator &typeGen);
+
 public:
   AddDebugInfoPass(fir::AddDebugInfoOptions options) : Base(options) {}
   void runOnOperation() override;
 };
 
+static uint32_t getLineFromLoc(mlir::Location loc) {
+  uint32_t line = 1;
+  if (auto fileLoc = mlir::dyn_cast<mlir::FileLineColLoc>(loc))
+    line = fileLoc.getLine();
+  return line;
+}
+
 } // namespace
 
+void AddDebugInfoPass::handleDeclareOp(fir::cg::XDeclareOp declOp,
+                                       mlir::LLVM::DIFileAttr fileAttr,
+                                       mlir::LLVM::DIScopeAttr scopeAttr,
+                                       fir::DebugTypeGenerator &typeGen) {
+  mlir::MLIRContext *context = &getContext();
+  mlir::OpBuilder builder(context);
+  auto result = fir::NameUniquer::deconstruct(declOp.getUniqName());
+
+  if (result.first != fir::NameUniquer::NameKind::VARIABLE)
+    return;
+
+  // Only accept local variables.
+  if (result.second.procs.empty())
+    return;
+
+  // FIXME: There may be cases where an argument is processed a bit before
+  // DeclareOp is generated. In that case, DeclareOp may point to an
+  // intermediate op and not to BlockArgument. We need to find those cases and
+  // walk the chain to get to the actual argument.
+
+  unsigned argNo = 0;
+  if (auto Arg = llvm::dyn_cast<mlir::BlockArgument>(declOp.getMemref()))
+    argNo = Arg.getArgNumber() + 1;
+
+  auto tyAttr = typeGen.convertType(fir::unwrapRefType(declOp.getType()),
+                                    fileAttr, scopeAttr, declOp.getLoc());
+
+  auto localVarAttr = mlir::LLVM::DILocalVariableAttr::get(
+      context, scopeAttr, mlir::StringAttr::get(context, result.second.name),
+      fileAttr, getLineFromLoc(declOp.getLoc()), argNo, /* alignInBits*/ 0,
+      tyAttr);
+  declOp->setLoc(builder.getFusedLoc({declOp->getLoc()}, localVarAttr));
+}
+
 void AddDebugInfoPass::runOnOperation() {
   mlir::ModuleOp module = getOperation();
   mlir::MLIRContext *context = &getContext();
@@ -144,14 +191,19 @@ void AddDebugInfoPass::runOnOperation() {
       subprogramFlags =
           subprogramFlags | mlir::LLVM::DISubprogramFlags::Definition;
     }
-    unsigned line = 1;
-    if (auto funcLoc = mlir::dyn_cast<mlir::FileLineColLoc>(l))
-      line = funcLoc.getLine();
-
+    unsigned line = getLineFromLoc(l);
     auto spAttr = mlir::LLVM::DISubprogramAttr::get(
         context, id, compilationUnit, fileAttr, funcName, fullName,
         funcFileAttr, line, line, subprogramFlags, subTypeAttr);
     funcOp->setLoc(builder.getFusedLoc({funcOp->getLoc()}, spAttr));
+
+    // Don't process variables if user asked for line tables only.
+    if (debugLevel == mlir::LLVM::DIEmissionKind::LineTablesOnly)
+      return;
+
+    funcOp.walk([&](fir::cg::XDeclareOp declOp) {
+      handleDeclareOp(declOp, fileAttr, spAttr, typeGen);
+    });
   });
 }
 
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index e5b4050dfb24..64c6547e06e0 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -24,11 +24,6 @@ DebugTypeGenerator::DebugTypeGenerator(mlir::ModuleOp m)
   LLVM_DEBUG(llvm::dbgs() << "DITypeAttr generator\n");
 }
 
-static mlir::LLVM::DITypeAttr genPlaceholderType(mlir::MLIRContext *context) {
-  return mlir::LLVM::DIBasicTypeAttr::get(
-      context, llvm::dwarf::DW_TAG_base_type, "void", 32, 1);
-}
-
 static mlir::LLVM::DITypeAttr genBasicType(mlir::MLIRContext *context,
                                            mlir::StringAttr name,
                                            unsigned bitSize,
@@ -37,6 +32,11 @@ static mlir::LLVM::DITypeAttr genBasicType(mlir::MLIRContext *context,
       context, llvm::dwarf::DW_TAG_base_type, name, bitSize, decoding);
 }
 
+static mlir::LLVM::DITypeAttr genPlaceholderType(mlir::MLIRContext *context) {
+  return genBasicType(context, mlir::StringAttr::get(context, "integer"), 32,
+                      llvm::dwarf::DW_ATE_signed);
+}
+
 mlir::LLVM::DITypeAttr
 DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
                                 mlir::LLVM::DIScopeAttr scope,
diff --git a/flang/test/Fir/declare-codegen.fir b/flang/test/Fir/declare-codegen.fir
index 9d68d3b2f9d4..c5879facb157 100644
--- a/flang/test/Fir/declare-codegen.fir
+++ b/flang/test/Fir/declare-codegen.fir
@@ -1,5 +1,7 @@
 // Test rewrite of fir.declare. The result is replaced by the memref operand.
-// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s
+// RUN: fir-opt --cg-rewrite="preserve-declare=true" %s -o - | FileCheck %s --check-prefixes DECL
+// RUN: fir-opt --cg-rewrite="preserve-declare=false" %s -o - | FileCheck %s --check-prefixes NODECL
+// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s --check-prefixes NODECL
 
 
 func.func @test(%arg0: !fir.ref<!fir.array<12x23xi32>>) {
@@ -15,9 +17,14 @@ func.func @test(%arg0: !fir.ref<!fir.array<12x23xi32>>) {
 func.func private @bar(%arg0: !fir.ref<!fir.array<12x23xi32>>)
 
 
-// CHECK-LABEL: func.func @test(
-// CHECK-SAME: %[[arg0:.*]]: !fir.ref<!fir.array<12x23xi32>>) {
-// CHECK-NEXT: fir.call @bar(%[[arg0]]) : (!fir.ref<!fir.array<12x23xi32>>) -> ()
+// NODECL-LABEL: func.func @test(
+// NODECL-SAME: %[[arg0:.*]]: !fir.ref<!fir.array<12x23xi32>>) {
+// NODECL-NEXT: fir.call @bar(%[[arg0]]) : (!fir.ref<!fir.array<12x23xi32>>) -> ()
+
+// DECL-LABEL: func.func @test(
+// DECL-SAME: %[[arg0:.*]]: !fir.ref<!fir.array<12x23xi32>>) {
+// DECL: fircg.ext_declare
+
 
 func.func @useless_shape_with_duplicate_extent_operand(%arg0: !fir.ref<!fir.array<3x3xf32>>) {
   %c3 = arith.constant 3 : index
@@ -26,5 +33,8 @@ func.func @useless_shape_with_duplicate_extent_operand(%arg0: !fir.ref<!fir.arra
   return
 }
 
-// CHECK-LABEL: func.func @useless_shape_with_duplicate_extent_operand(
-// CHECK-NEXT: return
+// NODECL-LABEL: func.func @useless_shape_with_duplicate_extent_operand(
+// NODECL-NEXT: return
+
+// DECL-LABEL: func.func @useless_shape_with_duplicate_extent_operand(
+// DECL: fircg.ext_declare
diff --git a/flang/test/Fir/dummy-scope-codegen.fir b/flang/test/Fir/dummy-scope-codegen.fir
index caef3c1b2578..2e5c091552a7 100644
--- a/flang/test/Fir/dummy-scope-codegen.fir
+++ b/flang/test/Fir/dummy-scope-codegen.fir
@@ -1,9 +1,14 @@
-// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s
+// RUN: fir-opt --cg-rewrite="preserve-declare=true" %s -o - | FileCheck %s --check-prefixes DECL
+// RUN: fir-opt --cg-rewrite="preserve-declare=false" %s -o - | FileCheck %s --check-prefixes NODECL
+// RUN: fir-opt --cg-rewrite %s -o - | FileCheck %s  --check-prefixes NODECL
 
 func.func @dummy_scope(%arg0: !fir.ref<f32>) {
   %scope = fir.dummy_scope : !fir.dscope
   %0 = fir.declare %arg0 dummy_scope %scope {uniq_name = "x"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
   return
 }
-// CHECK-LABEL: func.func @dummy_scope(
-// CHECK-NEXT: return
+// DECL-LABEL: func.func @dummy_scope(
+// DECL: fircg.ext_declare
+
+// NODECL-LABEL: func.func @dummy_scope(
+// NODECL-NEXT: return
\ No newline at end of file
diff --git a/flang/test/Transforms/debug-local-var-2.f90 b/flang/test/Transforms/debug-local-var-2.f90
new file mode 100644
index 000000000000..0fe1b81c27e6
--- /dev/null
+++ b/flang/test/Transforms/debug-local-var-2.f90
@@ -0,0 +1,94 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=line-tables-only %s -o - | FileCheck --check-prefix=LINEONLY %s
+
+! This tests checks the debug information for local variables in llvm IR.
+
+! CHECK-LABEL: define void @_QQmain
+! CHECK-DAG: %[[AL11:.*]] = alloca i32
+! CHECK-DAG: %[[AL12:.*]] = alloca i64
+! CHECK-DAG: %[[AL13:.*]] = alloca i8
+! CHECK-DAG: %[[AL14:.*]] = alloca i32
+! CHECK-DAG: %[[AL15:.*]] = alloca float
+! CHECK-DAG: %[[AL16:.*]] = alloca double
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL11]], metadata ![[I4:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL12]], metadata ![[I8:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL13]], metadata ![[L1:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL14]], metadata ![[L4:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL15]], metadata ![[R4:.*]], metadata !DIExpression())
+! CHECK-DAG: call void @llvm.dbg.declare(metadata ptr %[[AL16]], metadata ![[R8:.*]], metadata !DIExpression())
+! CHECK-LABEL: }
+
+! CHECK-LABEL: define {{.*}}i64 @_QFPfn1
+! CHECK-SAME: (ptr %[[ARG1:.*]], ptr %[[ARG2:.*]], ptr %[[ARG3:.*]])
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG1]], metadata ![[A1:.*]], metadata !DIExpression())
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG2]], metadata ![[B1:.*]], metadata !DIExpression())
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[ARG3]], metadata ![[C1:.*]], metadata !DIExpression())
+! CHECK-DAG: %[[AL2:.*]] = alloca i64
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[AL2]], metadata ![[RES1:.*]], metadata !DIExpression())
+! CHECK-LABEL: }
+
+! CHECK-LABEL: define {{.*}}i32 @_QFPfn2
+! CHECK-SAME: (ptr %[[FN2ARG1:.*]], ptr %[[FN2ARG2:.*]], ptr %[[FN2ARG3:.*]])
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG1]], metadata ![[A2:.*]], metadata !DIExpression())
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG2]], metadata ![[B2:.*]], metadata !DIExpression())
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[FN2ARG3]], metadata ![[C2:.*]], metadata !DIExpression())
+! CHECK-DAG: %[[AL3:.*]] = alloca i32
+! CHECK-DAG: tail call void @llvm.dbg.declare(metadata ptr %[[AL3]], metadata ![[RES2:.*]], metadata !DIExpression())
+! CHECK-LABEL: }
+
+program mn
+! CHECK-DAG: ![[MAIN:.*]] = distinct !DISubprogram(name: "_QQmain", {{.*}})
+
+! CHECK-DAG: ![[TYI32:.*]] = !DIBasicType(name: "integer", size: 32, encoding: DW_ATE_signed)
+! CHECK-DAG: ![[TYI64:.*]] = !DIBasicType(name: "integer", size: 64, encoding: DW_ATE_signed)
+! CHECK-DAG: ![[TYL8:.*]]  = !DIBasicType(name: "logical", size: 8, encoding: DW_ATE_boolean)
+! CHECK-DAG: ![[TYL32:.*]] = !DIBasicType(name: "logical", size: 32, encoding: DW_ATE_boolean)
+! CHECK-DAG: ![[TYR32:.*]] = !DIBasicType(name: "real", size: 32, encoding: DW_ATE_float)
+! CHECK-DAG: ![[TYR64:.*]] = !DIBasicType(name: "real", size: 64, encoding: DW_ATE_float)
+
+! CHECK-DAG: ![[I4]] = !DILocalVariable(name: "i4", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYI32]])
+! CHECK-DAG: ![[I8]] = !DILocalVariable(name: "i8", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYI64]])
+! CHECK-DAG: ![[R4]] = !DILocalVariable(name: "r4", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYR32]])
+! CHECK-DAG: ![[R8]] = !DILocalVariable(name: "r8", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYR64]])
+! CHECK-DAG: ![[L1]] = !DILocalVariable(name: "l1", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYL8]])
+! CHECK-DAG: ![[L4]] = !DILocalVariable(name: "l4", scope: ![[MAIN]], file: !{{.*}}, line: [[@LINE+6]], type: ![[TYL32]])
+  integer(kind=4) :: i4
+  integer(kind=8) :: i8
+  real(kind=4) :: r4
+  real(kind=8) :: r8
+  logical(kind=1) :: l1
+  logical(kind=4) :: l4
+
+  i8 = fn1(i4, r8, l1)
+  i4 = fn2(i8, r4, l4)
+contains
+! CHECK-DAG: ![[FN1:.*]] = distinct !DISubprogram(name: "fn1", {{.*}})
+! CHECK-DAG: ![[A1]] = !DILocalVariable(name: "a1", arg: 1, scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI32]])
+! CHECK-DAG: ![[B1]] = !DILocalVariable(name: "b1", arg: 2, scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYR64]])
+! CHECK-DAG: ![[C1]] = !DILocalVariable(name: "c1", arg: 3, scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYL8]])
+! CHECK-DAG: ![[RES1]] = !DILocalVariable(name: "res1", scope: ![[FN1]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI64]])
+  function fn1(a1, b1, c1) result (res1)
+    integer(kind=4), intent(in) :: a1
+    real(kind=8), intent(in) :: b1
+    logical(kind=1), intent(in) :: c1
+    integer(kind=8) :: res1
+
+    res1 = a1 + b1
+  end function
+
+! CHECK-DAG: ![[FN2:.*]] = distinct !DISubprogram(name: "fn2", {{.*}})
+! CHECK-DAG: ![[A2]] = !DILocalVariable(name: "a2", arg: 1, scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI64]])
+! CHECK-DAG: ![[B2]] = !DILocalVariable(name: "b2", arg: 2, scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYR32]])
+! CHECK-DAG: ![[C2]] = !DILocalVariable(name: "c2", arg: 3, scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYL32]])
+! CHECK-DAG: ![[RES2]] = !DILocalVariable(name: "res2", scope: ![[FN2]], file: !{{.*}}, line: [[@LINE+5]], type: ![[TYI32]])
+  function fn2(a2, b2, c2) result (res2)
+    integer(kind=8), intent(in) :: a2
+    real(kind=4), intent(in) :: b2
+    logical(kind=4), intent(in) :: c2
+    integer(kind=4) :: res2
+
+    res2 = a2 + b2
+  end function
+end program
+
+LINEONLY-NOT: DILocalVariable
diff --git a/flang/test/Transforms/debug-local-var.f90 b/flang/test/Transforms/debug-local-var.f90
new file mode 100644
index 000000000000..96dc111ad308
--- /dev/null
+++ b/flang/test/Transforms/debug-local-var.f90
@@ -0,0 +1,54 @@
+! RUN: %flang_fc1 -emit-fir -debug-info-kind=standalone -mmlir --mlir-print-debuginfo %s -o - | \
+! RUN: fir-opt --cg-rewrite="preserve-declare=true" --mlir-print-debuginfo | fir-opt --add-debug-info --mlir-print-debuginfo | FileCheck %s
+
+! CHECK-DAG: #[[INT8:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 64, encoding = DW_ATE_signed>
+! CHECK-DAG: #[[INT4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 32, encoding = DW_ATE_signed>
+! CHECK-DAG: #[[REAL8:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 64, encoding = DW_ATE_float>
+! CHECK-DAG: #[[LOG1:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "logical", sizeInBits = 8, encoding = DW_ATE_boolean>
+! CHECK-DAG: #[[REAL4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "real", sizeInBits = 32, encoding = DW_ATE_float>
+! CHECK-DAG: #[[LOG4:.*]] = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "logical", sizeInBits = 32, encoding = DW_ATE_boolean>
+! CHECK-DAG: #[[MAIN:.*]] = #llvm.di_subprogram<{{.*}}name = "_QQmain"{{.*}}>
+! CHECK-DAG: #[[FN1:.*]] = #llvm.di_subprogram<{{.*}}name = "fn1"{{.*}}>
+! CHECK-DAG: #[[FN2:.*]] = #llvm.di_subprogram<{{.*}}name = "fn2"{{.*}}>
+
+program mn
+! CHECK-DAG: #[[I4:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "i4", file = #{{.*}}, line = [[@LINE+6]], type = #[[INT4]]>
+! CHECK-DAG: #[[I8:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "i8", file = #{{.*}}, line = [[@LINE+6]], type = #[[INT8]]>
+! CHECK-DAG: #[[R4:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "r4", file = #{{.*}}, line = [[@LINE+6]], type = #[[REAL4]]>
+! CHECK-DAG: #[[R8:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "r8", file = #{{.*}}, line = [[@LINE+6]], type = #[[REAL8]]>
+! CHECK-DAG: #[[L1:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "l1", file = #{{.*}}, line = [[@LINE+6]], type = #[[LOG1]]>
+! CHECK-DAG: #[[L4:.*]] = #llvm.di_local_variable<scope = #[[MAIN]], name = "l4", file = #{{.*}}, line = [[@LINE+6]], type = #[[LOG4]]>
+  integer(kind=4) :: i4
+  integer(kind=8) :: i8
+  real(kind=4) :: r4
+  real(kind=8) :: r8
+  logical(kind=1) :: l1
+  logical(kind=4) :: l4
+  i8 = fn1(i4, r8, l1)
+  i4 = fn2(i8, r4, l4)
+contains
+  function fn1(a1, b1, c1) result (res1)
+! CHECK-DAG: #[[A1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "a1", file = #{{.*}}, line = [[@LINE+4]], arg = 1, type = #[[INT4]]>
+! CHECK-DAG: #[[B1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "b1", file = #{{.*}}, line = [[@LINE+4]], arg = 2, type = #[[REAL8]]>
+! CHECK-DAG: #[[C1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "c1", file = #{{.*}}, line = [[@LINE+4]], arg = 3, type = #[[LOG1]]>
+! CHECK-DAG: #[[RES1:.*]] = #llvm.di_local_variable<scope = #[[FN1]], name = "res1", file = #{{.*}}, line = [[@LINE+4]], type = #[[INT8]]>
+    integer(kind=4), intent(in) :: a1
+    real(kind=8), intent(in) :: b1
+    logical(kind=1), intent(in) :: c1
+    integer(kind=8) :: res1
+    res1 = a1 + b1
+  end function
+
+  function fn2(a2, b2, c2) result (res2)
+    implicit none
+! CHECK-DAG: #[[A2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "a2", file = #{{.*}}, line = [[@LINE+4]], arg = 1, type = #[[INT8]]>
+! CHECK-DAG: #[[B2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "b2", file = #{{.*}}, line = [[@LINE+4]], arg = 2, type = #[[REAL4]]>
+! CHECK-DAG: #[[C2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "c2", file = #{{.*}}, line = [[@LINE+4]], arg = 3, type = #[[LOG4]]>
+! CHECK-DAG: #[[RES2:.*]] = #llvm.di_local_variable<scope = #[[FN2]], name = "res2", file = #{{.*}}, line = [[@LINE+4]], type = #[[INT4]]>
+    integer(kind=8), intent(in) :: a2
+    real(kind=4), intent(in) :: b2
+    logical(kind=4), intent(in) :: c2
+    integer(kind=4) :: res2
+    res2 = a2 + b2
+  end function
+end program
-- 
GitLab


From 0ea178b085ba386d9d7bfea5afa072aa37f633fd Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 16 May 2024 10:24:56 +0200
Subject: [PATCH 0980/1206] SimplifyLibCalls: Emit vector ldexp intrinsics in
 exp2->ldexp combine (#92219)

Co-authored-by: Nikita Popov <github@npopov.com>
---
 .../lib/Transforms/Utils/SimplifyLibCalls.cpp | 17 +++++-----
 llvm/test/Transforms/InstCombine/exp2-1.ll    | 13 +++-----
 .../Transforms/InstCombine/exp2-to-ldexp.ll   | 32 +++++++++++++------
 .../test/Transforms/InstCombine/pow_fp_int.ll |  4 +--
 4 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 134eeba3996a..c9f6839e4b94 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -1992,11 +1992,12 @@ static Value *getIntToFPVal(Value *I2F, IRBuilderBase &B, unsigned DstWidth) {
     Value *Op = cast<Instruction>(I2F)->getOperand(0);
     // Make sure that the exponent fits inside an "int" of size DstWidth,
     // thus avoiding any range issues that FP has not.
-    unsigned BitWidth = Op->getType()->getPrimitiveSizeInBits();
-    if (BitWidth < DstWidth ||
-        (BitWidth == DstWidth && isa<SIToFPInst>(I2F)))
-      return isa<SIToFPInst>(I2F) ? B.CreateSExt(Op, B.getIntNTy(DstWidth))
-                                  : B.CreateZExt(Op, B.getIntNTy(DstWidth));
+    unsigned BitWidth = Op->getType()->getScalarSizeInBits();
+    if (BitWidth < DstWidth || (BitWidth == DstWidth && isa<SIToFPInst>(I2F))) {
+      Type *IntTy = Op->getType()->getWithNewBitWidth(DstWidth);
+      return isa<SIToFPInst>(I2F) ? B.CreateSExt(Op, IntTy)
+                                  : B.CreateZExt(Op, IntTy);
+    }
   }
 
   return nullptr;
@@ -2366,10 +2367,10 @@ Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
       hasFloatVersion(M, Name))
     Ret = optimizeUnaryDoubleFP(CI, B, TLI, true);
 
+  const bool UseIntrinsic = CI->doesNotAccessMemory();
   // Bail out for vectors because the code below only expects scalars.
-  // TODO: This could be allowed if we had a ldexp intrinsic (D14327).
   Type *Ty = CI->getType();
-  if (Ty->isVectorTy())
+  if (!UseIntrinsic && Ty->isVectorTy())
     return Ret;
 
   // exp2(sitofp(x)) -> ldexp(1.0, sext(x))  if sizeof(x) <= IntSize
@@ -2382,7 +2383,7 @@ Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilderBase &B) {
 
       // TODO: Emitting the intrinsic should not depend on whether the libcall
       // is available.
-      if (CI->doesNotAccessMemory()) {
+      if (UseIntrinsic) {
         return copyFlags(*CI, B.CreateIntrinsic(Intrinsic::ldexp,
                                                 {Ty, Exp->getType()},
                                                 {One, Exp}, CI));
diff --git a/llvm/test/Transforms/InstCombine/exp2-1.ll b/llvm/test/Transforms/InstCombine/exp2-1.ll
index 5bf70320d9ec..2dff0b08ecf9 100644
--- a/llvm/test/Transforms/InstCombine/exp2-1.ll
+++ b/llvm/test/Transforms/InstCombine/exp2-1.ll
@@ -308,20 +308,15 @@ define float @sitofp_scalar_intrinsic_with_FMF(i8 %x) {
 
 define <2 x float> @sitofp_vector_intrinsic_with_FMF(<2 x i8> %x) {
 ; LDEXP32-LABEL: @sitofp_vector_intrinsic_with_FMF(
-; LDEXP32-NEXT:    [[S:%.*]] = sitofp <2 x i8> [[X:%.*]] to <2 x float>
-; LDEXP32-NEXT:    [[R:%.*]] = call nnan <2 x float> @llvm.exp2.v2f32(<2 x float> [[S]])
+; LDEXP32-NEXT:    [[TMP1:%.*]] = sext <2 x i8> [[X:%.*]] to <2 x i32>
+; LDEXP32-NEXT:    [[R:%.*]] = call nnan <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x i32> [[TMP1]])
 ; LDEXP32-NEXT:    ret <2 x float> [[R]]
 ;
 ; LDEXP16-LABEL: @sitofp_vector_intrinsic_with_FMF(
-; LDEXP16-NEXT:    [[S:%.*]] = sitofp <2 x i8> [[X:%.*]] to <2 x float>
-; LDEXP16-NEXT:    [[R:%.*]] = call nnan <2 x float> @llvm.exp2.v2f32(<2 x float> [[S]])
+; LDEXP16-NEXT:    [[TMP1:%.*]] = sext <2 x i8> [[X:%.*]] to <2 x i16>
+; LDEXP16-NEXT:    [[R:%.*]] = call nnan <2 x float> @llvm.ldexp.v2f32.v2i16(<2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x i16> [[TMP1]])
 ; LDEXP16-NEXT:    ret <2 x float> [[R]]
 ;
-; NOLDEXPF-LABEL: @sitofp_vector_intrinsic_with_FMF(
-; NOLDEXPF-NEXT:    [[S:%.*]] = sitofp <2 x i8> [[X:%.*]] to <2 x float>
-; NOLDEXPF-NEXT:    [[R:%.*]] = call nnan <2 x float> @llvm.exp2.v2f32(<2 x float> [[S]])
-; NOLDEXPF-NEXT:    ret <2 x float> [[R]]
-;
 ; NOLDEXP-LABEL: @sitofp_vector_intrinsic_with_FMF(
 ; NOLDEXP-NEXT:    [[S:%.*]] = sitofp <2 x i8> [[X:%.*]] to <2 x float>
 ; NOLDEXP-NEXT:    [[R:%.*]] = call nnan <2 x float> @llvm.exp2.v2f32(<2 x float> [[S]])
diff --git a/llvm/test/Transforms/InstCombine/exp2-to-ldexp.ll b/llvm/test/Transforms/InstCombine/exp2-to-ldexp.ll
index 3069ee65e238..6e5be5a19d6d 100644
--- a/llvm/test/Transforms/InstCombine/exp2-to-ldexp.ll
+++ b/llvm/test/Transforms/InstCombine/exp2-to-ldexp.ll
@@ -39,11 +39,17 @@ define float @exp2_f32_sitofp_i8_flags(i8 %x) {
 }
 
 define <2 x float> @exp2_v2f32_sitofp_v2i8(<2 x i8> %x) {
-; CHECK-LABEL: define <2 x float> @exp2_v2f32_sitofp_v2i8(
-; CHECK-SAME: <2 x i8> [[X:%.*]]) {
-; CHECK-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i8> [[X]] to <2 x float>
-; CHECK-NEXT:    [[EXP2:%.*]] = call <2 x float> @llvm.exp2.v2f32(<2 x float> [[ITOFP]])
-; CHECK-NEXT:    ret <2 x float> [[EXP2]]
+; LDEXP-LABEL: define <2 x float> @exp2_v2f32_sitofp_v2i8(
+; LDEXP-SAME: <2 x i8> [[X:%.*]]) {
+; LDEXP-NEXT:    [[TMP1:%.*]] = sext <2 x i8> [[X]] to <2 x i32>
+; LDEXP-NEXT:    [[EXP2:%.*]] = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x i32> [[TMP1]])
+; LDEXP-NEXT:    ret <2 x float> [[EXP2]]
+;
+; NOLDEXP-LABEL: define <2 x float> @exp2_v2f32_sitofp_v2i8(
+; NOLDEXP-SAME: <2 x i8> [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i8> [[X]] to <2 x float>
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = call <2 x float> @llvm.exp2.v2f32(<2 x float> [[ITOFP]])
+; NOLDEXP-NEXT:    ret <2 x float> [[EXP2]]
 ;
   %itofp = sitofp <2 x i8> %x to <2 x float>
   %exp2 = call <2 x float> @llvm.exp2.v2f32(<2 x float> %itofp)
@@ -117,11 +123,17 @@ define fp128 @exp2_fp128_sitofp_i8(i8 %x) {
 }
 
 define <vscale x 4 x float> @exp2_nxv4f32_sitofp_i8(<vscale x 4 x i8> %x) {
-; CHECK-LABEL: define <vscale x 4 x float> @exp2_nxv4f32_sitofp_i8(
-; CHECK-SAME: <vscale x 4 x i8> [[X:%.*]]) {
-; CHECK-NEXT:    [[ITOFP:%.*]] = sitofp <vscale x 4 x i8> [[X]] to <vscale x 4 x float>
-; CHECK-NEXT:    [[EXP2:%.*]] = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> [[ITOFP]])
-; CHECK-NEXT:    ret <vscale x 4 x float> [[EXP2]]
+; LDEXP-LABEL: define <vscale x 4 x float> @exp2_nxv4f32_sitofp_i8(
+; LDEXP-SAME: <vscale x 4 x i8> [[X:%.*]]) {
+; LDEXP-NEXT:    [[TMP1:%.*]] = sext <vscale x 4 x i8> [[X]] to <vscale x 4 x i32>
+; LDEXP-NEXT:    [[EXP2:%.*]] = call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i32(<vscale x 4 x float> shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 1.000000e+00, i64 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> [[TMP1]])
+; LDEXP-NEXT:    ret <vscale x 4 x float> [[EXP2]]
+;
+; NOLDEXP-LABEL: define <vscale x 4 x float> @exp2_nxv4f32_sitofp_i8(
+; NOLDEXP-SAME: <vscale x 4 x i8> [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp <vscale x 4 x i8> [[X]] to <vscale x 4 x float>
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> [[ITOFP]])
+; NOLDEXP-NEXT:    ret <vscale x 4 x float> [[EXP2]]
 ;
   %itofp = sitofp <vscale x 4 x i8> %x to <vscale x 4 x float>
   %exp2 = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %itofp)
diff --git a/llvm/test/Transforms/InstCombine/pow_fp_int.ll b/llvm/test/Transforms/InstCombine/pow_fp_int.ll
index 9c1fa88f3183..7b194b3f8925 100644
--- a/llvm/test/Transforms/InstCombine/pow_fp_int.ll
+++ b/llvm/test/Transforms/InstCombine/pow_fp_int.ll
@@ -530,8 +530,8 @@ define double @powf_exp_const2_int_no_fast(double %base) {
 define <2 x float> @pow_sitofp_const_base_2_no_fast_vector(<2 x i8> %x) {
 ; CHECK-LABEL: define <2 x float> @pow_sitofp_const_base_2_no_fast_vector(
 ; CHECK-SAME: <2 x i8> [[X:%.*]]) {
-; CHECK-NEXT:    [[S:%.*]] = sitofp <2 x i8> [[X]] to <2 x float>
-; CHECK-NEXT:    [[EXP2:%.*]] = call <2 x float> @llvm.exp2.v2f32(<2 x float> [[S]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <2 x i8> [[X]] to <2 x i32>
+; CHECK-NEXT:    [[EXP2:%.*]] = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x i32> [[TMP1]])
 ; CHECK-NEXT:    ret <2 x float> [[EXP2]]
 ;
   %s = sitofp <2 x i8> %x to <2 x float>
-- 
GitLab


From 710f4bd054a857dd25012c9fef0cb66104f17ec0 Mon Sep 17 00:00:00 2001
From: Alex Zinenko <ftynse@gmail.com>
Date: Thu, 16 May 2024 10:20:57 +0200
Subject: [PATCH 0981/1206] [mlir] add missing include to MemRefTransformOps.h

The ODS-generated code has the following:

```
::mlir::TypedValue<::mlir::transform::OperationType> getAlloca() {
  return ::llvm::cast<::mlir::TypedValue<::mlir::transform::OperationType>>(
      *getODSOperands(0).begin());
}
```

that may require the compiler seing the definition of `OperationType` so
include the corresponding header.
---
 .../mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.h        | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.h b/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.h
index a87767acdd3b..5ef901ade91d 100644
--- a/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.h
+++ b/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.h
@@ -10,6 +10,7 @@
 #define MLIR_DIALECT_MEMREF_TRANSFORMOPS_MEMREFTRANSFORMOPS_H
 
 #include "mlir/Bytecode/BytecodeOpInterface.h"
+#include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/IR/OpImplementation.h"
 
-- 
GitLab


From 51403ada789e63e98e0dd5ac0ceba351465490d1 Mon Sep 17 00:00:00 2001
From: Alex Zinenko <ftynse@gmail.com>
Date: Thu, 16 May 2024 10:25:46 +0200
Subject: [PATCH 0982/1206] [mlir] fix orphaned IRDLExtensionOps documentation

.md files should be included into other .md files so they don't show up in the
top-level menu.
---
 mlir/docs/Dialects/Transform.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/mlir/docs/Dialects/Transform.md b/mlir/docs/Dialects/Transform.md
index 02f4a766ff42..37fcc0c88803 100644
--- a/mlir/docs/Dialects/Transform.md
+++ b/mlir/docs/Dialects/Transform.md
@@ -427,6 +427,10 @@ ops rather than having the methods directly act on the payload IR.
 
 [include "Dialects/DebugExtensionOps.md"]
 
+## IRDL (extension) Transform Operations
+
+[include "Dialects/IRDLExtensionOps.md"]
+
 ## Func Transform Operations
 
 [include "Dialects/FuncTransformOps.md"]
-- 
GitLab


From 72a895336b1e4df1ec2f485f8786ead6c356edb8 Mon Sep 17 00:00:00 2001
From: Tomas Matheson <Tomas.Matheson@arm.com>
Date: Thu, 16 May 2024 09:45:14 +0100
Subject: [PATCH 0983/1206] [AArch64][TargetParser] move ArchInfo into tablegen
 [NFC] (#92037)

This moves the architecture version, profile and extension information
into tablegen, and generates the TargetParser ArchInfo objects from
this data.

There are two lists of "dependencies" defined for each architecture: the
SubtargetFeature::Implies which controls which features are
automatically enabled in the backend when the corresponding architecture
SubtargetFeature is enabled; and the list of Extensions which are
enabled by default for this architecture. As far as I can tell, the idea
here is that the SubtargetFeature models the mandatory dependencies (although
they can still be disabled if desired) while the default extensions models
the typical use case for that architecture.
---
 .../llvm/TargetParser/AArch64TargetParser.h   |  48 +----
 llvm/lib/Target/AArch64/AArch64Features.td    | 174 ++++++++++--------
 llvm/utils/TableGen/ARMTargetDefEmitter.cpp   |  65 +++++++
 3 files changed, 163 insertions(+), 124 deletions(-)

diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index b662a74e5fc6..b3fff3c99025 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -239,8 +239,8 @@ enum ArchProfile { AProfile = 'A', RProfile = 'R', InvalidProfile = '?' };
 struct ArchInfo {
   VersionTuple Version;  // Architecture version, major + minor.
   ArchProfile Profile;   // Architecuture profile
-  StringRef Name;        // Human readable name, e.g. "armv8.1-a"
-  StringRef ArchFeature; // Command line feature flag, e.g. +v8a
+  StringRef Name;        // Name as supplied to -march e.g. "armv8.1-a"
+  StringRef ArchFeature; // Name as supplied to -target-feature, e.g. "+v8a"
   AArch64::ExtensionBitset
       DefaultExts; // bitfield of default extensions ArchExtKind
 
@@ -289,48 +289,8 @@ struct ArchInfo {
   static std::optional<ArchInfo> findBySubArch(StringRef SubArch);
 };
 
-// clang-format off
-inline constexpr ArchInfo ARMV8A    = { VersionTuple{8, 0}, AProfile, "armv8-a", "+v8a", (
-                                        AArch64::ExtensionBitset({AArch64::AEK_FP, AArch64::AEK_SIMD})), };
-inline constexpr ArchInfo ARMV8_1A  = { VersionTuple{8, 1}, AProfile, "armv8.1-a", "+v8.1a", (ARMV8A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_CRC, AArch64::AEK_LSE, AArch64::AEK_RDM}))};
-inline constexpr ArchInfo ARMV8_2A  = { VersionTuple{8, 2}, AProfile, "armv8.2-a", "+v8.2a", (ARMV8_1A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_RAS}))};
-inline constexpr ArchInfo ARMV8_3A  = { VersionTuple{8, 3}, AProfile, "armv8.3-a", "+v8.3a", (ARMV8_2A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_FCMA, AArch64::AEK_JSCVT, AArch64::AEK_PAUTH, AArch64::AEK_RCPC}))};
-inline constexpr ArchInfo ARMV8_4A  = { VersionTuple{8, 4}, AProfile, "armv8.4-a", "+v8.4a", (ARMV8_3A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_DOTPROD}))};
-inline constexpr ArchInfo ARMV8_5A  = { VersionTuple{8, 5}, AProfile, "armv8.5-a", "+v8.5a", (ARMV8_4A.DefaultExts)};
-inline constexpr ArchInfo ARMV8_6A  = { VersionTuple{8, 6}, AProfile, "armv8.6-a", "+v8.6a", (ARMV8_5A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_BF16, AArch64::AEK_I8MM}))};
-inline constexpr ArchInfo ARMV8_7A  = { VersionTuple{8, 7}, AProfile, "armv8.7-a", "+v8.7a", (ARMV8_6A.DefaultExts)};
-inline constexpr ArchInfo ARMV8_8A  = { VersionTuple{8, 8}, AProfile, "armv8.8-a", "+v8.8a", (ARMV8_7A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_MOPS, AArch64::AEK_HBC}))};
-inline constexpr ArchInfo ARMV8_9A  = { VersionTuple{8, 9}, AProfile, "armv8.9-a", "+v8.9a", (ARMV8_8A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_SPECRES2, AArch64::AEK_CSSC, AArch64::AEK_RASV2}))};
-inline constexpr ArchInfo ARMV9A    = { VersionTuple{9, 0}, AProfile, "armv9-a", "+v9a", (ARMV8_5A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_FP16, AArch64::AEK_SVE, AArch64::AEK_SVE2}))};
-inline constexpr ArchInfo ARMV9_1A  = { VersionTuple{9, 1}, AProfile, "armv9.1-a", "+v9.1a", (ARMV9A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_BF16, AArch64::AEK_I8MM}))};
-inline constexpr ArchInfo ARMV9_2A  = { VersionTuple{9, 2}, AProfile, "armv9.2-a", "+v9.2a", (ARMV9_1A.DefaultExts)};
-inline constexpr ArchInfo ARMV9_3A  = { VersionTuple{9, 3}, AProfile, "armv9.3-a", "+v9.3a", (ARMV9_2A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_MOPS, AArch64::AEK_HBC}))};
-inline constexpr ArchInfo ARMV9_4A  = { VersionTuple{9, 4}, AProfile, "armv9.4-a", "+v9.4a", (ARMV9_3A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_SPECRES2, AArch64::AEK_CSSC, AArch64::AEK_RASV2}))};
-inline constexpr ArchInfo ARMV9_5A  = { VersionTuple{9, 5}, AProfile, "armv9.5-a", "+v9.5a", (ARMV9_4A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_CPA}))};
-// For v8-R, we do not enable crypto and align with GCC that enables a more minimal set of optional architecture extensions.
-inline constexpr ArchInfo ARMV8R    = { VersionTuple{8, 0}, RProfile, "armv8-r", "+v8r", (ARMV8_5A.DefaultExts |
-                                        AArch64::ExtensionBitset({AArch64::AEK_SSBS,
-                                        AArch64::AEK_FP16, AArch64::AEK_FP16FML, AArch64::AEK_SB}).flip(AArch64::AEK_LSE))};
-// clang-format on
-
-// The set of all architectures
-static constexpr std::array<const ArchInfo *, 17> ArchInfos = {
-    &ARMV8A,   &ARMV8_1A, &ARMV8_2A, &ARMV8_3A, &ARMV8_4A, &ARMV8_5A,
-    &ARMV8_6A, &ARMV8_7A, &ARMV8_8A, &ARMV8_9A, &ARMV9A,   &ARMV9_1A,
-    &ARMV9_2A, &ARMV9_3A, &ARMV9_4A, &ARMV9_5A, &ARMV8R,
-};
+#define EMIT_ARCHITECTURES
+#include "llvm/TargetParser/AArch64TargetParserDef.inc"
 
 // Details of a specific CPU.
 struct CpuInfo {
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index b9c26e99ae03..ba0b760ce3d7 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -792,90 +792,104 @@ def FeatureTLBIW : Extension<"tlbiw", "TLBIW",
 //===----------------------------------------------------------------------===//
 // Architectures.
 //
-def HasV8_0aOps : SubtargetFeature<"v8a", "HasV8_0aOps", "true",
-  "Support ARM v8.0a instructions", [FeatureEL2VMSA, FeatureEL3]>;
-
-def HasV8_1aOps : SubtargetFeature<"v8.1a", "HasV8_1aOps", "true",
-  "Support ARM v8.1a instructions", [HasV8_0aOps, FeatureCRC, FeatureLSE,
-  FeatureRDM, FeaturePAN, FeatureLOR, FeatureVH]>;
-
-def HasV8_2aOps : SubtargetFeature<"v8.2a", "HasV8_2aOps", "true",
-  "Support ARM v8.2a instructions", [HasV8_1aOps, FeaturePsUAO,
-  FeaturePAN_RWV, FeatureRAS, FeatureCCPP]>;
-
-def HasV8_3aOps : SubtargetFeature<"v8.3a", "HasV8_3aOps", "true",
-  "Support ARM v8.3a instructions", [HasV8_2aOps, FeatureRCPC, FeaturePAuth,
-  FeatureJS, FeatureCCIDX, FeatureComplxNum]>;
-
-def HasV8_4aOps : SubtargetFeature<"v8.4a", "HasV8_4aOps", "true",
-  "Support ARM v8.4a instructions", [HasV8_3aOps, FeatureDotProd,
-  FeatureNV, FeatureMPAM, FeatureDIT,
-  FeatureTRACEV8_4, FeatureAM, FeatureSEL2, FeatureTLB_RMI,
-  FeatureFlagM, FeatureRCPC_IMMO, FeatureLSE2]>;
+class Architecture64<
+  int major, int minor, string profile,
+  string target_feature_name,
+  list<SubtargetFeature> implied_features,
+  list<Extension> default_extensions
+> : SubtargetFeature<target_feature_name,
+    "HasV" # major # "_" # minor # profile # "Ops", "true",
+    "Support ARM " # target_feature_name # " architecture",
+    implied_features
+> {
+  int Major = major;
+  int Minor = minor;
+  string Profile = profile;
+
+  // Extensions enabled by default. Not the same as implied SubtargetFeatures.
+  list<Extension> DefaultExts = default_extensions;
+}
 
-def HasV8_5aOps : SubtargetFeature<
-  "v8.5a", "HasV8_5aOps", "true", "Support ARM v8.5a instructions",
+def HasV8_0aOps : Architecture64<8, 0, "a", "v8a",
+  [FeatureEL2VMSA, FeatureEL3],
+  [FeatureFPARMv8, FeatureNEON]>;
+def HasV8_1aOps : Architecture64<8, 1, "a", "v8.1a",
+  [HasV8_0aOps, FeatureCRC, FeatureLSE, FeatureRDM, FeaturePAN, FeatureLOR,
+   FeatureVH],
+  !listconcat(HasV8_0aOps.DefaultExts, [FeatureCRC, FeatureLSE, FeatureRDM])>;
+def HasV8_2aOps : Architecture64<8, 2, "a", "v8.2a",
+  [HasV8_1aOps, FeaturePsUAO, FeaturePAN_RWV, FeatureRAS, FeatureCCPP],
+  !listconcat(HasV8_1aOps.DefaultExts, [FeatureRAS])>;
+def HasV8_3aOps : Architecture64<8, 3, "a", "v8.3a",
+  [HasV8_2aOps, FeatureRCPC, FeaturePAuth, FeatureJS, FeatureCCIDX,
+    FeatureComplxNum],
+  !listconcat(HasV8_2aOps.DefaultExts, [FeatureComplxNum, FeatureJS,
+    FeaturePAuth, FeatureRCPC])>;
+def HasV8_4aOps : Architecture64<8, 4, "a", "v8.4a",
+  [HasV8_3aOps, FeatureDotProd, FeatureNV, FeatureMPAM, FeatureDIT,
+    FeatureTRACEV8_4, FeatureAM, FeatureSEL2, FeatureTLB_RMI, FeatureFlagM,
+    FeatureRCPC_IMMO, FeatureLSE2],
+  !listconcat(HasV8_3aOps.DefaultExts, [FeatureDotProd])>;
+def HasV8_5aOps : Architecture64<8, 5, "a", "v8.5a",
   [HasV8_4aOps, FeatureAltFPCmp, FeatureFRInt3264, FeatureSpecRestrict,
-   FeatureSSBS, FeatureSB, FeaturePredRes, FeatureCacheDeepPersist,
-   FeatureBranchTargetId]>;
-
-def HasV8_6aOps : SubtargetFeature<
-  "v8.6a", "HasV8_6aOps", "true", "Support ARM v8.6a instructions",
+    FeatureSSBS, FeatureSB, FeaturePredRes, FeatureCacheDeepPersist,
+    FeatureBranchTargetId],
+  !listconcat(HasV8_4aOps.DefaultExts, [])>;
+def HasV8_6aOps : Architecture64<8, 6, "a", "v8.6a",
   [HasV8_5aOps, FeatureAMVS, FeatureBF16, FeatureFineGrainedTraps,
-   FeatureEnhancedCounterVirtualization, FeatureMatMulInt8]>;
-
-def HasV8_7aOps : SubtargetFeature<
-  "v8.7a", "HasV8_7aOps", "true", "Support ARM v8.7a instructions",
-  [HasV8_6aOps, FeatureXS, FeatureWFxT, FeatureHCX]>;
-
-def HasV8_8aOps : SubtargetFeature<
-  "v8.8a", "HasV8_8aOps", "true", "Support ARM v8.8a instructions",
-  [HasV8_7aOps, FeatureHBC, FeatureMOPS, FeatureNMI]>;
-
-def HasV8_9aOps : SubtargetFeature<
-  "v8.9a", "HasV8_9aOps", "true", "Support ARM v8.9a instructions",
+    FeatureEnhancedCounterVirtualization, FeatureMatMulInt8],
+  !listconcat(HasV8_5aOps.DefaultExts, [FeatureBF16, FeatureMatMulInt8])>;
+def HasV8_7aOps : Architecture64<8, 7, "a", "v8.7a",
+  [HasV8_6aOps, FeatureXS, FeatureWFxT, FeatureHCX],
+  !listconcat(HasV8_6aOps.DefaultExts, [])>;
+def HasV8_8aOps : Architecture64<8, 8, "a", "v8.8a",
+  [HasV8_7aOps, FeatureHBC, FeatureMOPS, FeatureNMI],
+  !listconcat(HasV8_7aOps.DefaultExts, [FeatureMOPS, FeatureHBC])>;
+def HasV8_9aOps : Architecture64<8, 9, "a", "v8.9a",
   [HasV8_8aOps, FeatureCLRBHB, FeaturePRFM_SLC, FeatureSPECRES2,
-   FeatureCSSC, FeatureRASv2, FeatureCHK]>;
-
-def HasV9_0aOps : SubtargetFeature<
-  "v9a", "HasV9_0aOps", "true", "Support ARM v9a instructions",
-  [HasV8_5aOps, FeatureMEC, FeatureSVE2]>;
-
-def HasV9_1aOps : SubtargetFeature<
-  "v9.1a", "HasV9_1aOps", "true", "Support ARM v9.1a instructions",
-  [HasV8_6aOps, HasV9_0aOps]>;
-
-def HasV9_2aOps : SubtargetFeature<
-  "v9.2a", "HasV9_2aOps", "true", "Support ARM v9.2a instructions",
-  [HasV8_7aOps, HasV9_1aOps]>;
-
-def HasV9_3aOps : SubtargetFeature<
-  "v9.3a", "HasV9_3aOps", "true", "Support ARM v9.3a instructions",
-  [HasV8_8aOps, HasV9_2aOps]>;
-
-def HasV9_4aOps : SubtargetFeature<
-  "v9.4a", "HasV9_4aOps", "true", "Support ARM v9.4a instructions",
-  [HasV8_9aOps, HasV9_3aOps]>;
-
-def HasV9_5aOps : SubtargetFeature<
-  "v9.5a", "HasV9_5aOps", "true", "Support ARM v9.5a instructions",
-  [HasV9_4aOps, FeatureCPA]>;
-
-def HasV8_0rOps : SubtargetFeature<
-  "v8r", "HasV8_0rOps", "true", "Support ARM v8r instructions",
-  [//v8.1
-  FeatureCRC, FeaturePAN, FeatureLSE, FeatureCONTEXTIDREL2,
-  //v8.2
-  FeatureRAS, FeaturePsUAO, FeatureCCPP, FeaturePAN_RWV,
-  //v8.3
-  FeatureCCIDX, FeaturePAuth, FeatureRCPC,
-  //v8.4
-  FeatureTRACEV8_4, FeatureTLB_RMI, FeatureFlagM, FeatureDIT, FeatureSEL2,
-  FeatureRCPC_IMMO,
-  // Not mandatory in v8.0-R, but included here on the grounds that it
-  // only enables names of system registers
-  FeatureSpecRestrict
-  ]>;
+   FeatureCSSC, FeatureRASv2, FeatureCHK],
+  !listconcat(HasV8_8aOps.DefaultExts, [FeatureSPECRES2, FeatureCSSC,
+    FeatureRASv2])>;
+def HasV9_0aOps : Architecture64<9, 0, "a", "v9a",
+  [HasV8_5aOps, FeatureMEC, FeatureSVE2],
+  !listconcat(HasV8_5aOps.DefaultExts, [FeatureFullFP16, FeatureSVE,
+    FeatureSVE2])>;
+def HasV9_1aOps : Architecture64<9, 1, "a", "v9.1a",
+  [HasV8_6aOps, HasV9_0aOps],
+  !listconcat(HasV9_0aOps.DefaultExts, [FeatureBF16, FeatureMatMulInt8])>;
+def HasV9_2aOps : Architecture64<9, 2, "a", "v9.2a",
+  [HasV8_7aOps, HasV9_1aOps],
+  !listconcat(HasV9_1aOps.DefaultExts, [])>;
+def HasV9_3aOps : Architecture64<9, 3, "a", "v9.3a",
+  [HasV8_8aOps, HasV9_2aOps],
+  !listconcat(HasV9_2aOps.DefaultExts, [FeatureMOPS, FeatureHBC])>;
+def HasV9_4aOps : Architecture64<9, 4, "a", "v9.4a",
+  [HasV8_9aOps, HasV9_3aOps],
+  !listconcat(HasV9_3aOps.DefaultExts, [FeatureSPECRES2, FeatureCSSC,
+    FeatureRASv2])>;
+def HasV9_5aOps : Architecture64<9, 5, "a", "v9.5a",
+  [HasV9_4aOps, FeatureCPA],
+  !listconcat(HasV9_4aOps.DefaultExts, [FeatureCPA])>;
+def HasV8_0rOps : Architecture64<8, 0, "r", "v8r",
+  [ //v8.1
+    FeatureCRC, FeaturePAN, FeatureLSE, FeatureCONTEXTIDREL2,
+    //v8.2
+    FeatureRAS, FeaturePsUAO, FeatureCCPP, FeaturePAN_RWV,
+    //v8.3
+    FeatureCCIDX, FeaturePAuth, FeatureRCPC,
+    //v8.4
+    FeatureTRACEV8_4, FeatureTLB_RMI, FeatureFlagM, FeatureDIT, FeatureSEL2,
+    FeatureRCPC_IMMO,
+    // Not mandatory in v8.0-R, but included here on the grounds that it
+    // only enables names of system registers
+    FeatureSpecRestrict
+  ],
+  // For v8-R, we do not enable crypto and align with GCC that enables a more
+  // minimal set of optional architecture extensions.
+  !listconcat(
+    !listremove(HasV8_5aOps.DefaultExts, [FeatureLSE]),
+    [FeatureSSBS, FeatureFullFP16, FeatureFP16FML, FeatureSB]
+  )>;
 
 //===----------------------------------------------------------------------===//
 // Access to privileged registers
diff --git a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
index 0e90f57af493..491011643bbf 100644
--- a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
@@ -13,9 +13,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Format.h"
+#include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cstdint>
+#include <string>
 
 using namespace llvm;
 
@@ -112,6 +115,68 @@ static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
      << "#undef EMIT_EXTENSIONS\n"
      << "#endif // EMIT_EXTENSIONS\n"
      << "\n";
+
+  // Emit architecture information
+  OS << "#ifdef EMIT_ARCHITECTURES\n";
+
+  auto Architectures = RK.getAllDerivedDefinitionsIfDefined("Architecture64");
+  std::vector<std::string> CppSpellings;
+  for (const Record *Rec : Architectures) {
+    const int Major = Rec->getValueAsInt("Major");
+    const int Minor = Rec->getValueAsInt("Minor");
+    const std::string ProfileLower = Rec->getValueAsString("Profile").str();
+    const std::string ProfileUpper = Rec->getValueAsString("Profile").upper();
+
+    if (ProfileLower != "a" && ProfileLower != "r")
+      PrintFatalError(Rec->getLoc(),
+                      "error: Profile must be one of 'a' or 'r', got '" +
+                          ProfileLower + "'");
+
+    // Name of the object in C++
+    const std::string CppSpelling =
+        Minor == 0 ? "ARMV" + std::to_string(Major) + ProfileUpper.c_str()
+                   : "ARMV" + std::to_string(Major) + "_" +
+                         std::to_string(Minor) + ProfileUpper.c_str();
+    OS << "inline constexpr ArchInfo " << CppSpelling << " = {\n";
+    CppSpellings.push_back(CppSpelling);
+
+    OS << llvm::format("  VersionTuple{%d, %d},\n", Major, Minor);
+    OS << llvm::format("  %sProfile,\n", ProfileUpper.c_str());
+
+    // Name as spelled for -march.
+    if (Minor == 0)
+      OS << llvm::format("  \"armv%d-%s\",\n", Major, ProfileLower.c_str());
+    else
+      OS << llvm::format("  \"armv%d.%d-%s\",\n", Major, Minor,
+                         ProfileLower.c_str());
+
+    // SubtargetFeature::Name, used for -target-feature. Here the "+" is added.
+    const auto TargetFeatureName = Rec->getValueAsString("Name");
+    OS << "  \"+" << TargetFeatureName << "\",\n";
+
+    // Construct the list of default extensions
+    OS << "  (AArch64::ExtensionBitset({";
+    for (auto *E : Rec->getValueAsListOfDefs("DefaultExts")) {
+      // Only process subclasses of Extension
+      OS << "AArch64::" << E->getValueAsString("ArchExtKindSpelling").upper()
+         << ", ";
+    }
+    OS << "}))\n";
+
+    OS << "};\n";
+  }
+
+  OS << "\n"
+     << "/// The set of all architectures\n"
+     << "static constexpr std::array<const ArchInfo *, " << CppSpellings.size()
+     << "> ArchInfos = {\n";
+  for (auto CppSpelling : CppSpellings)
+    OS << "  &" << CppSpelling << ",\n";
+  OS << "};\n";
+
+  OS << "#undef EMIT_ARCHITECTURES\n"
+     << "#endif // EMIT_ARCHITECTURES\n"
+     << "\n";
 }
 
 static TableGen::Emitter::Opt
-- 
GitLab


From ca02f36bacaec58071a26ff65329fbab5526d1d7 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Thu, 16 May 2024 09:52:21 +0100
Subject: [PATCH 0984/1206] [mlir][vector] Teach `TransferOptimization` to
 forward masked stores (#87794)

This only handles one case (that's fairly common in practice*), storing
a masked constant splat, then reloading again with the same mask and a
padding value that matches the splat.

* For SVE/SME (without peeling) this occurs when you have a
`linalg.fill` preceding a `linalg.matmul`.
---
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  37 ++++-
 .../Dialect/Vector/vector-transferop-opt.mlir | 127 +++++++++++++++++-
 2 files changed, 160 insertions(+), 4 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index d10a31941db4..58951641d33c 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -170,12 +170,43 @@ AffineMap mlir::vector::getTransferMinorIdentityMap(ShapedType shapedType,
       shapedType.getContext());
 }
 
+/// Check if `write` is of a constant splat and the masked `read` is padded with
+/// the same splat value -- meaning it could be the same value as the initial
+/// constant splat.
+static bool isSplatWriteConsistentWithMaskedRead(vector::TransferWriteOp write,
+                                                 vector::TransferReadOp read) {
+  auto readMask = read.getMask();
+  auto writeMask = write.getMask();
+  // Check if the masks are consistent. The splat value could be the same if the
+  // read is masked (and padded with the splat value), and the write is unmasked
+  // or has the same mask. Note this does not allow the case where the write is
+  // masked and the read is unmasked, as then the read could be of more elements
+  // than the write (which may not be the same value).
+  bool couldBeSameSplat = readMask && (!writeMask || writeMask == readMask);
+  if (!couldBeSameSplat)
+    return false;
+  // Check for constant splat (as the source of the write).
+  DenseElementsAttr splatAttr;
+  if (!matchPattern(write.getVector(),
+                    m_Constant<DenseElementsAttr>(&splatAttr)) ||
+      !splatAttr.isSplat()) {
+    return false;
+  }
+  // The padding of the read and the constant splat value must be the same.
+  Attribute padAttr;
+  if (!matchPattern(read.getPadding(), m_Constant(&padAttr)))
+    return false;
+  return padAttr == splatAttr.getSplatValue<Attribute>();
+}
+
 bool mlir::vector::checkSameValueRAW(vector::TransferWriteOp defWrite,
                                      vector::TransferReadOp read) {
-  return !defWrite.hasOutOfBoundsDim() && !defWrite.getMask() &&
-         !read.getMask() && defWrite.getIndices() == read.getIndices() &&
+  return !defWrite.hasOutOfBoundsDim() &&
+         defWrite.getIndices() == read.getIndices() &&
          defWrite.getVectorType() == read.getVectorType() &&
-         defWrite.getPermutationMap() == read.getPermutationMap();
+         defWrite.getPermutationMap() == read.getPermutationMap() &&
+         ((!defWrite.getMask() && !read.getMask()) ||
+          isSplatWriteConsistentWithMaskedRead(defWrite, read));
 }
 
 bool mlir::vector::checkSameValueWAW(vector::TransferWriteOp write,
diff --git a/mlir/test/Dialect/Vector/vector-transferop-opt.mlir b/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
index 13957af014b8..0719c0dd1742 100644
--- a/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
+++ b/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
@@ -222,7 +222,7 @@ func.func @forward_dead_store_negative(%arg0: i1, %arg1 : memref<4x4xf32>,
 // `vector.transfer_write` would not be safe:
 //         %1 = vector.transfer_read %subview
 //         vector.transfer_write %1, %alloca
-//         vector.transfer_write %vec, %collapse_shape 
+//         vector.transfer_write %vec, %collapse_shape
 //         %2 = vector.transfer_read %alloca
 //         vector.transfer_write %1, %subview
 // Indeed, %alloca and %collapse_shape alias and hence %2 != %1. Instead, the
@@ -360,3 +360,128 @@ func.func @forward_dead_store_dynamic_non_overlap_trailing_dim(
   vector.transfer_write %x, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
   return
 }
+
+// CHECK-LABEL: func @forward_dead_constant_splat_store_with_masking
+//       CHECK:   %[[SPLAT:.*]] = arith.constant dense<0.000000e+00> : vector<[8]x[8]xf32>
+//   CHECK-NOT:   vector.transfer_write
+//   CHECK-NOT:   vector.transfer_read
+//       CHECK:   scf.for
+//  CHECK-SAME:     iter_args(%{{.*}} = %[[SPLAT]])
+//       CHECK:   }
+//       CHECK:   vector.transfer_write
+//       CHECK:   return
+func.func @forward_dead_constant_splat_store_with_masking(%buffer : memref<?x?xf32>, %mask: vector<[8]x[8]xi1>) {
+  %zero_splat = arith.constant dense<0.0> : vector<[8]x[8]xf32>
+  %read_padding = arith.constant 0.0 : f32
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %c512 = arith.constant 512 : index
+  vector.transfer_write %zero_splat, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
+  %0 = vector.transfer_read %buffer[%c0, %c0], %read_padding, %mask {in_bounds = [true, true]} : memref<?x?xf32>, vector<[8]x[8]xf32>
+  %x = scf.for %arg2 = %c0 to %c512 step %c1 iter_args(%acc = %0) -> (vector<[8]x[8]xf32>) {
+    %1 = arith.addf %acc, %acc : vector<[8]x[8]xf32>
+    scf.yield %1 : vector<[8]x[8]xf32>
+  }
+  vector.transfer_write %x, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
+  return
+}
+
+// Here the read can be eliminated but not the write (due to mismatched masks).
+// CHECK-LABEL: func @forward_dead_constant_splat_store_with_masking_unmasked_write
+//       CHECK:   %[[SPLAT:.*]] = arith.constant dense<0.000000e+00> : vector<[8]x[8]xf32>
+//       CHECK:   vector.transfer_write %[[SPLAT]]
+//       CHECK:   scf.for
+//  CHECK-SAME:     iter_args(%{{.*}} = %[[SPLAT]])
+//       CHECK:   }
+//       CHECK:   vector.transfer_write
+//       CHECK:   return
+func.func @forward_dead_constant_splat_store_with_masking_unmasked_write(%buffer : memref<?x?xf32>, %mask: vector<[8]x[8]xi1>) {
+  %zero_splat = arith.constant dense<0.0> : vector<[8]x[8]xf32>
+  %read_padding = arith.constant 0.0 : f32
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %c512 = arith.constant 512 : index
+  vector.transfer_write %zero_splat, %buffer[%c0, %c0] {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
+  %0 = vector.transfer_read %buffer[%c0, %c0], %read_padding, %mask {in_bounds = [true, true]} : memref<?x?xf32>, vector<[8]x[8]xf32>
+  %x = scf.for %arg2 = %c0 to %c512 step %c1 iter_args(%acc = %0) -> (vector<[8]x[8]xf32>) {
+    %1 = arith.addf %acc, %acc : vector<[8]x[8]xf32>
+    scf.yield %1 : vector<[8]x[8]xf32>
+  }
+  vector.transfer_write %x, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
+  return
+}
+
+// Negative test, the padding does not match the constant splat, so we can't
+// forward the store.
+// CHECK-LABEL: func @forward_dead_constant_splat_store_with_masking_negative_0
+//       CHECK:   vector.transfer_write
+//       CHECK:   vector.transfer_read
+//       CHECK:   scf.for
+//       CHECK:   }
+//       CHECK:   vector.transfer_write
+//       CHECK:   return
+func.func @forward_dead_constant_splat_store_with_masking_negative_0(%buffer : memref<?x?xf32>, %mask: vector<[8]x[8]xi1>) {
+  %zero_splat = arith.constant dense<0.0> : vector<[8]x[8]xf32>
+  %read_padding = arith.constant 1.0 : f32
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %c512 = arith.constant 512 : index
+  vector.transfer_write %zero_splat, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
+  %0 = vector.transfer_read %buffer[%c0, %c0], %read_padding, %mask {in_bounds = [true, true]} : memref<?x?xf32>, vector<[8]x[8]xf32>
+  %x = scf.for %arg2 = %c0 to %c512 step %c1 iter_args(%acc = %0) -> (vector<[8]x[8]xf32>) {
+    %1 = arith.addf %acc, %acc : vector<[8]x[8]xf32>
+    scf.yield %1 : vector<[8]x[8]xf32>
+  }
+  vector.transfer_write %x, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
+  return
+}
+
+// Negative test, the masks don't match between the read and write, so we can't
+// foward the store.
+// CHECK-LABEL: func @forward_dead_constant_splat_store_with_masking_negative_1
+//       CHECK:   vector.transfer_write
+//       CHECK:   vector.transfer_read
+//       CHECK:   scf.for
+//       CHECK:   }
+//       CHECK:   vector.transfer_write
+//       CHECK:   return
+func.func @forward_dead_constant_splat_store_with_masking_negative_1(%buffer : memref<?x?xf32>, %mask_a: vector<[8]x[8]xi1>, %mask_b: vector<[8]x[8]xi1>) {
+  %zero_splat = arith.constant dense<0.0> : vector<[8]x[8]xf32>
+  %read_padding = arith.constant 0.0 : f32
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %c512 = arith.constant 512 : index
+  vector.transfer_write %zero_splat, %buffer[%c0, %c0], %mask_a {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
+  %0 = vector.transfer_read %buffer[%c0, %c0], %read_padding, %mask_b {in_bounds = [true, true]} : memref<?x?xf32>, vector<[8]x[8]xf32>
+  %x = scf.for %arg2 = %c0 to %c512 step %c1 iter_args(%acc = %0) -> (vector<[8]x[8]xf32>) {
+    %1 = arith.addf %acc, %acc : vector<[8]x[8]xf32>
+    scf.yield %1 : vector<[8]x[8]xf32>
+  }
+  vector.transfer_write %x, %buffer[%c0, %c0], %mask_a {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
+  return
+}
+
+// Negative test, here the write is masked but the read is unmasked. We can't
+// forward the store (as the write could be of less elements then the read).
+// CHECK-LABEL: func @forward_dead_constant_splat_store_with_masking_negative_3
+//       CHECK:   vector.transfer_write
+//       CHECK:   vector.transfer_read
+//       CHECK:   scf.for
+//       CHECK:   }
+//       CHECK:   vector.transfer_write
+//       CHECK:   return
+func.func @forward_dead_constant_splat_store_with_masking_negative_3(%buffer : memref<?x?xf32>, %mask: vector<[8]x[8]xi1>) {
+  %zero_splat = arith.constant dense<0.0> : vector<[8]x[8]xf32>
+  %read_padding = arith.constant 0.0 : f32
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %c512 = arith.constant 512 : index
+  vector.transfer_write %zero_splat, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
+  %0 = vector.transfer_read %buffer[%c0, %c0], %read_padding {in_bounds = [true, true]} : memref<?x?xf32>, vector<[8]x[8]xf32>
+  %x = scf.for %arg2 = %c0 to %c512 step %c1 iter_args(%acc = %0) -> (vector<[8]x[8]xf32>) {
+    %1 = arith.addf %acc, %acc : vector<[8]x[8]xf32>
+    scf.yield %1 : vector<[8]x[8]xf32>
+  }
+  vector.transfer_write %x, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
+  return
+}
-- 
GitLab


From 30410018d32b2cc19552c6d6a3b7b6de84d99382 Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Thu, 16 May 2024 16:54:18 +0800
Subject: [PATCH 0985/1206] [LoongArch] Enable all -target-abi options

This is a pre-commit for modifying `computeTargetABI` logic.

This patch will provide warning prompts when using those ABIs that have
not yet been standardized.

Reviewed By: xen0n, SixWeining

Pull Request: https://github.com/llvm/llvm-project/pull/92222
---
 .../LoongArch/LoongArchISelLowering.cpp       |  5 ++-
 .../MCTargetDesc/LoongArchBaseInfo.cpp        | 36 ++++++++++++++++---
 llvm/test/CodeGen/LoongArch/target-abi.ll     | 26 ++++++++++++++
 3 files changed, 60 insertions(+), 7 deletions(-)
 create mode 100644 llvm/test/CodeGen/LoongArch/target-abi.ll

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 21d520656091..6740563b90b4 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -3575,15 +3575,14 @@ static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI,
   switch (ABI) {
   default:
     llvm_unreachable("Unexpected ABI");
-  case LoongArchABI::ABI_ILP32S:
+    break;
   case LoongArchABI::ABI_ILP32F:
   case LoongArchABI::ABI_LP64F:
-    report_fatal_error("Unimplemented ABI");
-    break;
   case LoongArchABI::ABI_ILP32D:
   case LoongArchABI::ABI_LP64D:
     UseGPRForFloat = !IsFixed;
     break;
+  case LoongArchABI::ABI_ILP32S:
   case LoongArchABI::ABI_LP64S:
     break;
   }
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp
index 928adb03f098..6d75e575cd3c 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp
@@ -14,6 +14,7 @@
 #include "LoongArchBaseInfo.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
 
@@ -21,6 +22,33 @@ namespace llvm {
 
 namespace LoongArchABI {
 
+// Check if ABI has been standardized; issue a warning if it hasn't.
+// FIXME: Once all ABIs are standardized, this will be removed.
+static ABI checkABIStandardized(ABI Abi) {
+  StringRef ABIName;
+  switch (Abi) {
+  case ABI_ILP32S:
+    ABIName = "ilp32s";
+    break;
+  case ABI_ILP32F:
+    ABIName = "ilp32f";
+    break;
+  case ABI_ILP32D:
+    ABIName = "ilp32d";
+    break;
+  case ABI_LP64F:
+    ABIName = "lp64f";
+    break;
+  case ABI_LP64S:
+  case ABI_LP64D:
+    return Abi;
+  default:
+    llvm_unreachable("");
+  }
+  errs() << "warning: '" << ABIName << "' has not been standardized\n";
+  return Abi;
+}
+
 ABI computeTargetABI(const Triple &TT, StringRef ABIName) {
   ABI ArgProvidedABI = getTargetABI(ABIName);
   bool Is64Bit = TT.isArch64Bit();
@@ -50,7 +78,7 @@ ABI computeTargetABI(const Triple &TT, StringRef ABIName) {
       errs() << "'" << ABIName
              << "' is not a recognized ABI for this target, ignoring and using "
                 "triple-implied ABI\n";
-    return TripleABI;
+    return checkABIStandardized(TripleABI);
 
   case LoongArchABI::ABI_ILP32S:
   case LoongArchABI::ABI_ILP32F:
@@ -58,7 +86,7 @@ ABI computeTargetABI(const Triple &TT, StringRef ABIName) {
     if (Is64Bit) {
       errs() << "32-bit ABIs are not supported for 64-bit targets, ignoring "
                 "target-abi and using triple-implied ABI\n";
-      return TripleABI;
+      return checkABIStandardized(TripleABI);
     }
     break;
 
@@ -68,7 +96,7 @@ ABI computeTargetABI(const Triple &TT, StringRef ABIName) {
     if (!Is64Bit) {
       errs() << "64-bit ABIs are not supported for 32-bit targets, ignoring "
                 "target-abi and using triple-implied ABI\n";
-      return TripleABI;
+      return checkABIStandardized(TripleABI);
     }
     break;
   }
@@ -77,7 +105,7 @@ ABI computeTargetABI(const Triple &TT, StringRef ABIName) {
     errs() << "warning: triple-implied ABI conflicts with provided target-abi '"
            << ABIName << "', using target-abi\n";
 
-  return ArgProvidedABI;
+  return checkABIStandardized(ArgProvidedABI);
 }
 
 ABI getTargetABI(StringRef ABIName) {
diff --git a/llvm/test/CodeGen/LoongArch/target-abi.ll b/llvm/test/CodeGen/LoongArch/target-abi.ll
new file mode 100644
index 000000000000..de834c152b18
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/target-abi.ll
@@ -0,0 +1,26 @@
+; RUN: llc --mtriple=loongarch32 --mattr=+d --target-abi=ilp32s < %s 2>&1 \
+; RUN:   | FileCheck %s -DABI=ilp32s --check-prefixes=CHECK,WARNING
+; RUN: llc --mtriple=loongarch32 --mattr=+d --target-abi=ilp32f < %s 2>&1 \
+; RUN:   | FileCheck %s -DABI=ilp32f --check-prefixes=CHECK,WARNING
+; RUN: llc --mtriple=loongarch32 --mattr=+d --target-abi=ilp32d < %s 2>&1 \
+; RUN:   | FileCheck %s -DABI=ilp32d --check-prefixes=CHECK,WARNING
+; RUN: llc --mtriple=loongarch64 --mattr=+d --target-abi=lp64f < %s 2>&1 \
+; RUN:   | FileCheck %s -DABI=lp64f --check-prefixes=CHECK,WARNING
+
+; RUN: llc --mtriple=loongarch64 --mattr=+d --target-abi=lp64s < %s 2>&1 \
+; RUN:   | FileCheck %s --check-prefixes=CHECK,NO-WARNING
+; RUN: llc --mtriple=loongarch64 --mattr=+d --target-abi=lp64d < %s 2>&1 \
+; RUN:   | FileCheck %s --check-prefixes=CHECK,NO-WARNING
+
+;; Check if the ABI has been standardized; issue a warning if it hasn't.
+
+; WARNING: warning: '[[ABI]]' has not been standardized
+
+; NO-WARNING-NOT: warning
+
+define void @nothing() nounwind {
+; CHECK-LABEL: nothing:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:  ret
+  ret void
+}
-- 
GitLab


From 70608c24fae8943cb93276db4d0358722705507b Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Thu, 16 May 2024 17:15:21 +0800
Subject: [PATCH 0986/1206] [LoongArch] Refactor LoongArchABI::computeTargetABI

The previous logic did not consider whether the architectural features
meet the requirements of the ABI, resulting in the generation of
incorrect object files in some cases. For example:

```
llc -mtriple=loongarch64 -filetype=obj test/CodeGen/LoongArch/ir-instruction/fadd.ll -o t.o
llvm-readelf -h t.o
```
The object file indicates the ABI as lp64d, however, the generated code
is lp64s.

The new logic introduces the `feature-implied` ABI. When both target-abi
and triple-implied ABI are invalid, the feature-implied ABI is used.

Reviewed By: SixWeining, xen0n

Pull Request: https://github.com/llvm/llvm-project/pull/92223
---
 .../Target/LoongArch/LoongArchSubtarget.cpp   |   2 +-
 .../MCTargetDesc/LoongArchBaseInfo.cpp        | 139 +++++++++++++-----
 .../MCTargetDesc/LoongArchBaseInfo.h          |   3 +-
 .../MCTargetDesc/LoongArchELFStreamer.cpp     |   3 +-
 llvm/test/CodeGen/LoongArch/e_flags.ll        |   6 +-
 .../target-abi-from-triple-edge-cases.ll      |  35 ++++-
 6 files changed, 142 insertions(+), 46 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchSubtarget.cpp b/llvm/lib/Target/LoongArch/LoongArchSubtarget.cpp
index ffcde7dd1fa7..57babfc917f8 100644
--- a/llvm/lib/Target/LoongArch/LoongArchSubtarget.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchSubtarget.cpp
@@ -50,7 +50,7 @@ LoongArchSubtarget &LoongArchSubtarget::initializeSubtargetDependencies(
   if (!Is64Bit && HasLA64)
     report_fatal_error("Feature 64bit should be used for loongarch64 target.");
 
-  TargetABI = LoongArchABI::computeTargetABI(TT, ABIName);
+  TargetABI = LoongArchABI::computeTargetABI(TT, getFeatureBits(), ABIName);
 
   return *this;
 }
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp
index 6d75e575cd3c..17bcacedfc51 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "LoongArchBaseInfo.h"
+#include "LoongArchMCTargetDesc.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -49,63 +50,127 @@ static ABI checkABIStandardized(ABI Abi) {
   return Abi;
 }
 
-ABI computeTargetABI(const Triple &TT, StringRef ABIName) {
-  ABI ArgProvidedABI = getTargetABI(ABIName);
+static ABI getTripleABI(const Triple &TT) {
   bool Is64Bit = TT.isArch64Bit();
   ABI TripleABI;
-
-  // Figure out the ABI explicitly requested via the triple's environment type.
   switch (TT.getEnvironment()) {
   case llvm::Triple::EnvironmentType::GNUSF:
-    TripleABI = Is64Bit ? LoongArchABI::ABI_LP64S : LoongArchABI::ABI_ILP32S;
+    TripleABI = Is64Bit ? ABI_LP64S : ABI_ILP32S;
     break;
   case llvm::Triple::EnvironmentType::GNUF32:
-    TripleABI = Is64Bit ? LoongArchABI::ABI_LP64F : LoongArchABI::ABI_ILP32F;
+    TripleABI = Is64Bit ? ABI_LP64F : ABI_ILP32F;
     break;
-
   // Let the fallback case behave like {ILP32,LP64}D.
   case llvm::Triple::EnvironmentType::GNUF64:
   default:
-    TripleABI = Is64Bit ? LoongArchABI::ABI_LP64D : LoongArchABI::ABI_ILP32D;
+    TripleABI = Is64Bit ? ABI_LP64D : ABI_ILP32D;
     break;
   }
+  return TripleABI;
+}
 
-  switch (ArgProvidedABI) {
-  case LoongArchABI::ABI_Unknown:
-    // Fallback to the triple-implied ABI if ABI name is not specified or
-    // invalid.
-    if (!ABIName.empty())
-      errs() << "'" << ABIName
-             << "' is not a recognized ABI for this target, ignoring and using "
-                "triple-implied ABI\n";
-    return checkABIStandardized(TripleABI);
+ABI computeTargetABI(const Triple &TT, const FeatureBitset &FeatureBits,
+                     StringRef ABIName) {
+  bool Is64Bit = TT.isArch64Bit();
+  ABI ArgProvidedABI = getTargetABI(ABIName);
+  ABI TripleABI = getTripleABI(TT);
+
+  auto IsABIValidForFeature = [=](ABI Abi) {
+    switch (Abi) {
+    default:
+      return false;
+    case ABI_ILP32S:
+      return !Is64Bit;
+    case ABI_ILP32F:
+      return !Is64Bit && FeatureBits[LoongArch::FeatureBasicF];
+    case ABI_ILP32D:
+      return !Is64Bit && FeatureBits[LoongArch::FeatureBasicD];
+    case ABI_LP64S:
+      return Is64Bit;
+    case ABI_LP64F:
+      return Is64Bit && FeatureBits[LoongArch::FeatureBasicF];
+    case ABI_LP64D:
+      return Is64Bit && FeatureBits[LoongArch::FeatureBasicD];
+    }
+  };
+
+  // 1. If the '-target-abi' is valid, use it.
+  if (IsABIValidForFeature(ArgProvidedABI)) {
+    if (TT.hasEnvironment() && ArgProvidedABI != TripleABI)
+      errs()
+          << "warning: triple-implied ABI conflicts with provided target-abi '"
+          << ABIName << "', using target-abi\n";
+    return checkABIStandardized(ArgProvidedABI);
+  }
 
-  case LoongArchABI::ABI_ILP32S:
-  case LoongArchABI::ABI_ILP32F:
-  case LoongArchABI::ABI_ILP32D:
-    if (Is64Bit) {
-      errs() << "32-bit ABIs are not supported for 64-bit targets, ignoring "
-                "target-abi and using triple-implied ABI\n";
+  // 2. If the triple-implied ABI is valid, use it.
+  if (IsABIValidForFeature(TripleABI)) {
+    // If target-abi is not specified, use the valid triple-implied ABI.
+    if (ABIName.empty())
       return checkABIStandardized(TripleABI);
-    }
-    break;
 
-  case LoongArchABI::ABI_LP64S:
-  case LoongArchABI::ABI_LP64F:
-  case LoongArchABI::ABI_LP64D:
-    if (!Is64Bit) {
-      errs() << "64-bit ABIs are not supported for 32-bit targets, ignoring "
-                "target-abi and using triple-implied ABI\n";
+    switch (ArgProvidedABI) {
+    case ABI_Unknown:
+      // Fallback to the triple-implied ABI if ABI name is specified but
+      // invalid.
+      errs() << "warning: the '" << ABIName
+             << "' is not a recognized ABI for this target, ignoring and "
+                "using triple-implied ABI\n";
       return checkABIStandardized(TripleABI);
+    case ABI_ILP32S:
+    case ABI_ILP32F:
+    case ABI_ILP32D:
+      if (Is64Bit) {
+        errs() << "warning: 32-bit ABIs are not supported for 64-bit targets, "
+                  "ignoring and using triple-implied ABI\n";
+        return checkABIStandardized(TripleABI);
+      }
+      break;
+    case ABI_LP64S:
+    case ABI_LP64F:
+    case ABI_LP64D:
+      if (!Is64Bit) {
+        errs() << "warning: 64-bit ABIs are not supported for 32-bit targets, "
+                  "ignoring and using triple-implied ABI\n";
+        return checkABIStandardized(TripleABI);
+      }
+      break;
     }
-    break;
-  }
 
-  if (!ABIName.empty() && TT.hasEnvironment() && ArgProvidedABI != TripleABI)
-    errs() << "warning: triple-implied ABI conflicts with provided target-abi '"
-           << ABIName << "', using target-abi\n";
+    switch (ArgProvidedABI) {
+    case ABI_ILP32F:
+    case ABI_LP64F:
+      errs() << "warning: the '" << ABIName
+             << "' ABI can't be used for a target that doesn't support the 'F' "
+                "instruction set, ignoring and using triple-implied ABI\n";
+      break;
+    case ABI_ILP32D:
+    case ABI_LP64D:
+      errs() << "warning: the '" << ABIName
+             << "' ABI can't be used for a target that doesn't support the 'D' "
+                "instruction set, ignoring and using triple-implied ABI\n";
+      break;
+    default:
+      llvm_unreachable("");
+    }
+    return checkABIStandardized(TripleABI);
+  }
 
-  return checkABIStandardized(ArgProvidedABI);
+  // 3. Parse the 'feature-abi', and use it.
+  auto GetFeatureABI = [=]() {
+    if (FeatureBits[LoongArch::FeatureBasicD])
+      return Is64Bit ? ABI_LP64D : ABI_ILP32D;
+    if (FeatureBits[LoongArch::FeatureBasicF])
+      return Is64Bit ? ABI_LP64F : ABI_ILP32F;
+    return Is64Bit ? ABI_LP64S : ABI_ILP32S;
+  };
+  if (ABIName.empty())
+    errs() << "warning: the triple-implied ABI is invalid, ignoring and using "
+              "feature-implied ABI\n";
+  else
+    errs() << "warning: both target-abi and the triple-implied ABI are "
+              "invalid, ignoring and using feature-implied ABI\n";
+  return checkABIStandardized(GetFeatureABI());
 }
 
 ABI getTargetABI(StringRef ABIName) {
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.h
index 3c3fed7d43ed..1a12fb492a60 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.h
@@ -69,7 +69,8 @@ enum ABI {
   ABI_Unknown
 };
 
-ABI computeTargetABI(const Triple &TT, StringRef ABIName);
+ABI computeTargetABI(const Triple &TT, const FeatureBitset &FeatureBits,
+                     StringRef ABIName);
 ABI getTargetABI(StringRef ABIName);
 
 // Returns the register used to hold the stack pointer after realignment.
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp
index 9e56333e5fd9..a8a4f1ed327d 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFStreamer.cpp
@@ -27,7 +27,8 @@ LoongArchTargetELFStreamer::LoongArchTargetELFStreamer(
   auto &MAB = static_cast<LoongArchAsmBackend &>(
       getStreamer().getAssembler().getBackend());
   setTargetABI(LoongArchABI::computeTargetABI(
-      STI.getTargetTriple(), MAB.getTargetOptions().getABIName()));
+      STI.getTargetTriple(), STI.getFeatureBits(),
+      MAB.getTargetOptions().getABIName()));
 }
 
 MCELFStreamer &LoongArchTargetELFStreamer::getStreamer() {
diff --git a/llvm/test/CodeGen/LoongArch/e_flags.ll b/llvm/test/CodeGen/LoongArch/e_flags.ll
index 25f3285bdf32..2feb9d832bca 100644
--- a/llvm/test/CodeGen/LoongArch/e_flags.ll
+++ b/llvm/test/CodeGen/LoongArch/e_flags.ll
@@ -4,10 +4,10 @@
 ; RUN: llc --mtriple=loongarch32 --filetype=obj %s --target-abi=ilp32s -o %t-ilp32s
 ; RUN: llvm-readelf -h %t-ilp32s | FileCheck %s --check-prefixes=ILP32,ABI-S --match-full-lines
 
-; RUN: llc --mtriple=loongarch32 --filetype=obj %s --target-abi=ilp32f -o %t-ilp32f
+; RUN: llc --mtriple=loongarch32 -mattr=+f --filetype=obj %s --target-abi=ilp32f -o %t-ilp32f
 ; RUN: llvm-readelf -h %t-ilp32f | FileCheck %s --check-prefixes=ILP32,ABI-F --match-full-lines
 
-; RUN: llc --mtriple=loongarch32 --filetype=obj %s --target-abi=ilp32d -o %t-ilp32d
+; RUN: llc --mtriple=loongarch32 -mattr=+d --filetype=obj %s --target-abi=ilp32d -o %t-ilp32d
 ; RUN: llvm-readelf -h %t-ilp32d | FileCheck %s --check-prefixes=ILP32,ABI-D --match-full-lines
 
 ; RUN: llc --mtriple=loongarch64 -mattr=+d --filetype=obj %s -o %t-la64
@@ -16,7 +16,7 @@
 ; RUN: llc --mtriple=loongarch64 --filetype=obj %s --target-abi=lp64s -o %t-lp64s
 ; RUN: llvm-readelf -h %t-lp64s | FileCheck %s --check-prefixes=LP64,ABI-S --match-full-lines
 
-; RUN: llc --mtriple=loongarch64 --filetype=obj %s --target-abi=lp64f -o %t-lp64f
+; RUN: llc --mtriple=loongarch64 -mattr=+f --filetype=obj %s --target-abi=lp64f -o %t-lp64f
 ; RUN: llvm-readelf -h %t-lp64f | FileCheck %s --check-prefixes=LP64,ABI-F --match-full-lines
 
 ; RUN: llc --mtriple=loongarch64 --filetype=obj %s --mattr=+d --target-abi=lp64d -o %t-lp64d
diff --git a/llvm/test/CodeGen/LoongArch/target-abi-from-triple-edge-cases.ll b/llvm/test/CodeGen/LoongArch/target-abi-from-triple-edge-cases.ll
index 1d5ed089c69f..36b78ea2ea02 100644
--- a/llvm/test/CodeGen/LoongArch/target-abi-from-triple-edge-cases.ll
+++ b/llvm/test/CodeGen/LoongArch/target-abi-from-triple-edge-cases.ll
@@ -5,7 +5,7 @@
 ; RUN: llc --mtriple=loongarch64-linux-gnu --target-abi=foo --mattr=+d < %s 2>&1 \
 ; RUN:   | FileCheck %s --check-prefixes=LP64D,UNKNOWN
 
-; UNKNOWN: 'foo' is not a recognized ABI for this target, ignoring and using triple-implied ABI
+; UNKNOWN: warning: the 'foo' is not a recognized ABI for this target, ignoring and using triple-implied ABI
 
 ;; Check that --target-abi takes precedence over triple-supplied ABI modifiers.
 ; RUN: llc --mtriple=loongarch32-linux-gnusf --target-abi=ilp32d --mattr=+d < %s 2>&1 \
@@ -29,8 +29,31 @@
 ; RUN: llc --mtriple=loongarch32 --target-abi=lp64d --mattr=+d < %s 2>&1 \
 ; RUN:   | FileCheck %s --check-prefixes=ILP32D,64ON32
 
-; 32ON64: 32-bit ABIs are not supported for 64-bit targets, ignoring target-abi and using triple-implied ABI
-; 64ON32: 64-bit ABIs are not supported for 32-bit targets, ignoring target-abi and using triple-implied ABI
+; 32ON64: warning: 32-bit ABIs are not supported for 64-bit targets, ignoring and using triple-implied ABI
+; 64ON32: warning: 64-bit ABIs are not supported for 32-bit targets, ignoring and using triple-implied ABI
+
+;; Check that target-abi is invalid but triple-implied ABI is valid, use triple-implied ABI
+; RUN: llc --mtriple=loongarch64-linux-gnusf --target-abi=lp64f --mattr=-f < %s 2>&1 \
+; RUN:   | FileCheck %s --check-prefixes=LP64S,LP64S-LP64F-NOF
+; RUN: llc --mtriple=loongarch64-linux-gnusf --target-abi=lp64d --mattr=-d < %s 2>&1 \
+; RUN:   | FileCheck %s --check-prefixes=LP64S,LP64S-LP64D-NOD
+
+; LP64S-LP64F-NOF: warning: the 'lp64f' ABI can't be used for a target that doesn't support the 'F' instruction set, ignoring and using triple-implied ABI
+; LP64S-LP64D-NOD: warning: the 'lp64d' ABI can't be used for a target that doesn't support the 'D' instruction set, ignoring and using triple-implied ABI
+
+;; Check that both target-abi and triple-implied ABI are invalid, use feature-implied ABI
+; RUN: llc --mtriple=loongarch64-linux-gnuf64 --target-abi=lp64f --mattr=-f < %s 2>&1 \
+; RUN:   | FileCheck %s --check-prefixes=LP64S,LP64D-LP64F-NOF
+; RUN: llc --mtriple=loongarch64 --target-abi=lp64f --mattr=-f < %s 2>&1 \
+; RUN:   | FileCheck %s --check-prefixes=LP64S,LP64D-LP64F-NOF
+
+; LP64D-LP64F-NOF: warning: both target-abi and the triple-implied ABI are invalid, ignoring and using feature-implied ABI
+
+;; Check that triple-implied ABI are invalid, use feature-implied ABI
+; RUN: llc --mtriple=loongarch64 --mattr=-f < %s 2>&1 \
+; RUN:   | FileCheck %s --check-prefixes=LP64S,LP64D-NONE-NOF
+
+; LP64D-NONE-NOF: warning: the triple-implied ABI is invalid, ignoring and using feature-implied ABI
 
 define float @f(float %a) {
 ; ILP32D-LABEL: f:
@@ -48,6 +71,9 @@ define float @f(float %a) {
 ; LP64D-NEXT:    ffint.s.w $fa1, $fa1
 ; LP64D-NEXT:    fadd.s $fa0, $fa0, $fa1
 ; LP64D-NEXT:    ret
+;
+; LP64S-LABEL: f:
+; LP64S:         bl %plt(__addsf3)
   %1 = fadd float %a, 1.0
   ret float %1
 }
@@ -69,6 +95,9 @@ define double @g(double %a) {
 ; LP64D-NEXT:    ffint.d.l $fa1, $fa1
 ; LP64D-NEXT:    fadd.d $fa0, $fa0, $fa1
 ; LP64D-NEXT:    ret
+;
+; LP64S-LABEL: g:
+; LP64S:         bl %plt(__adddf3)
   %1 = fadd double %a, 1.0
   ret double %1
 }
-- 
GitLab


From 53bdcee9d7a921a0ea5e0c8fa3f889f16f1d268f Mon Sep 17 00:00:00 2001
From: Oleg Shyshkov <shyshkov@google.com>
Date: Thu, 16 May 2024 11:37:55 +0200
Subject: [PATCH 0987/1206] [mlir][bazel] Fix bazel build.

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 751cd94d5ff1..62ac56f8b46e 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -2217,6 +2217,7 @@ cc_library(
         ":ArithDialect",
         ":ArmSMEDialect",
         ":ArmSMETransforms",
+        ":ControlFlowDialect",
         ":ConversionPassIncGen",
         ":FuncDialect",
 	":FunctionInterfaces",
-- 
GitLab


From cfcbe3ae55144635cac4ab00c017e3e93dad1c4b Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Thu, 16 May 2024 17:49:10 +0800
Subject: [PATCH 0988/1206] [lld][test] Fix test failure after #92223

https://lab.llvm.org/buildbot/#/builders/272/builds/16745
---
 lld/test/ELF/emulation-loongarch.s | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lld/test/ELF/emulation-loongarch.s b/lld/test/ELF/emulation-loongarch.s
index 343e836274a3..28b879f75846 100644
--- a/lld/test/ELF/emulation-loongarch.s
+++ b/lld/test/ELF/emulation-loongarch.s
@@ -25,8 +25,8 @@
 # LA32-NEXT:   Entry:
 # LA32-NEXT:   ProgramHeaderOffset: 0x34
 # LA32-NEXT:   SectionHeaderOffset:
-# LA32-NEXT:   Flags [ (0x43)
-# LA32-NEXT:     EF_LOONGARCH_ABI_DOUBLE_FLOAT (0x3)
+# LA32-NEXT:   Flags [ (0x41)
+# LA32-NEXT:     EF_LOONGARCH_ABI_SOFT_FLOAT (0x1)
 # LA32-NEXT:     EF_LOONGARCH_OBJABI_V1 (0x40)
 # LA32-NEXT:   ]
 # LA32-NEXT:   HeaderSize: 52
-- 
GitLab


From 90d2f8c630e1ddddd034e4a0e575929c08dd26bf Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Thu, 16 May 2024 10:53:14 +0100
Subject: [PATCH 0989/1206] [mlir][vector] Teach `TransferOptimization` to look
 through trivial aliases (#87805)

This allows `TransferOptimization` to eliminate and forward stores that
are to trivial aliases (rather than just to identical memref values).

A trivial aliases is (currently) defined as:

  1. A `memref.cast`
  2. A `memref.subview` with a zero offset and unit strides
  3. A chain of 1 and 2
---
 .../mlir/Dialect/MemRef/Utils/MemRefUtils.h   | 17 ++++++++++
 mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp | 30 +++++++++++++++++
 .../Transforms/VectorTransferOpTransforms.cpp | 33 +++++++++----------
 .../Dialect/Vector/vector-transferop-opt.mlir | 30 +++++++++++++++++
 4 files changed, 92 insertions(+), 18 deletions(-)

diff --git a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
index 46003ed84686..1c094c1c1328 100644
--- a/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
+++ b/mlir/include/mlir/Dialect/MemRef/Utils/MemRefUtils.h
@@ -22,6 +22,9 @@ namespace mlir {
 
 class MemRefType;
 
+/// A value with a memref type.
+using MemrefValue = TypedValue<BaseMemRefType>;
+
 namespace memref {
 
 /// Returns true, if the memref type has static shapes and represents a
@@ -93,6 +96,20 @@ computeStridesIRBlock(Location loc, OpBuilder &builder,
   return computeSuffixProductIRBlock(loc, builder, sizes);
 }
 
+/// Walk up the source chain until an operation that changes/defines the view of
+/// memory is found (i.e. skip operations that alias the entire view).
+MemrefValue skipFullyAliasingOperations(MemrefValue source);
+
+/// Checks if two (memref) values are the same or are statically known to alias
+/// the same region of memory.
+inline bool isSameViewOrTrivialAlias(MemrefValue a, MemrefValue b) {
+  return skipFullyAliasingOperations(a) == skipFullyAliasingOperations(b);
+}
+
+/// Walk up the source chain until something an op other than a `memref.subview`
+/// or `memref.cast` is found.
+MemrefValue skipSubViewsAndCasts(MemrefValue source);
+
 } // namespace memref
 } // namespace mlir
 
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
index c93e5a9dcd39..05d5ca2ce12f 100644
--- a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -178,5 +178,35 @@ computeSuffixProductIRBlock(Location loc, OpBuilder &builder,
   return computeSuffixProductIRBlockImpl(loc, builder, sizes, unit);
 }
 
+MemrefValue skipFullyAliasingOperations(MemrefValue source) {
+  while (auto op = source.getDefiningOp()) {
+    if (auto subViewOp = dyn_cast<memref::SubViewOp>(op);
+        subViewOp && subViewOp.hasZeroOffset() && subViewOp.hasUnitStride()) {
+      // A `memref.subview` with an all zero offset, and all unit strides, still
+      // points to the same memory.
+      source = cast<MemrefValue>(subViewOp.getSource());
+    } else if (auto castOp = dyn_cast<memref::CastOp>(op)) {
+      // A `memref.cast` still points to the same memory.
+      source = castOp.getSource();
+    } else {
+      return source;
+    }
+  }
+  return source;
+}
+
+MemrefValue skipSubViewsAndCasts(MemrefValue source) {
+  while (auto op = source.getDefiningOp()) {
+    if (auto subView = dyn_cast<memref::SubViewOp>(op)) {
+      source = cast<MemrefValue>(subView.getSource());
+    } else if (auto cast = dyn_cast<memref::CastOp>(op)) {
+      source = cast.getSource();
+    } else {
+      return source;
+    }
+  }
+  return source;
+}
+
 } // namespace memref
 } // namespace mlir
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index 0ffef6aabccc..997b56a1ce14 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
@@ -104,10 +105,8 @@ void TransferOptimization::deadStoreOp(vector::TransferWriteOp write) {
                     << "\n");
   llvm::SmallVector<Operation *, 8> blockingAccesses;
   Operation *firstOverwriteCandidate = nullptr;
-  Value source = write.getSource();
-  // Skip subview ops.
-  while (auto subView = source.getDefiningOp<memref::SubViewOp>())
-    source = subView.getSource();
+  Value source =
+      memref::skipSubViewsAndCasts(cast<MemrefValue>(write.getSource()));
   llvm::SmallVector<Operation *, 32> users(source.getUsers().begin(),
                                            source.getUsers().end());
   llvm::SmallDenseSet<Operation *, 32> processed;
@@ -116,8 +115,8 @@ void TransferOptimization::deadStoreOp(vector::TransferWriteOp write) {
     // If the user has already been processed skip.
     if (!processed.insert(user).second)
       continue;
-    if (auto subView = dyn_cast<memref::SubViewOp>(user)) {
-      users.append(subView->getUsers().begin(), subView->getUsers().end());
+    if (isa<memref::SubViewOp, memref::CastOp>(user)) {
+      users.append(user->getUsers().begin(), user->getUsers().end());
       continue;
     }
     if (isMemoryEffectFree(user))
@@ -126,7 +125,9 @@ void TransferOptimization::deadStoreOp(vector::TransferWriteOp write) {
       continue;
     if (auto nextWrite = dyn_cast<vector::TransferWriteOp>(user)) {
       // Check candidate that can override the store.
-      if (write.getSource() == nextWrite.getSource() &&
+      if (memref::isSameViewOrTrivialAlias(
+              cast<MemrefValue>(nextWrite.getSource()),
+              cast<MemrefValue>(write.getSource())) &&
           checkSameValueWAW(nextWrite, write) &&
           postDominators.postDominates(nextWrite, write)) {
         if (firstOverwriteCandidate == nullptr ||
@@ -191,10 +192,8 @@ void TransferOptimization::storeToLoadForwarding(vector::TransferReadOp read) {
                     << "\n");
   SmallVector<Operation *, 8> blockingWrites;
   vector::TransferWriteOp lastwrite = nullptr;
-  Value source = read.getSource();
-  // Skip subview ops.
-  while (auto subView = source.getDefiningOp<memref::SubViewOp>())
-    source = subView.getSource();
+  Value source =
+      memref::skipSubViewsAndCasts(cast<MemrefValue>(read.getSource()));
   llvm::SmallVector<Operation *, 32> users(source.getUsers().begin(),
                                            source.getUsers().end());
   llvm::SmallDenseSet<Operation *, 32> processed;
@@ -203,12 +202,8 @@ void TransferOptimization::storeToLoadForwarding(vector::TransferReadOp read) {
     // If the user has already been processed skip.
     if (!processed.insert(user).second)
       continue;
-    if (auto subView = dyn_cast<memref::SubViewOp>(user)) {
-      users.append(subView->getUsers().begin(), subView->getUsers().end());
-      continue;
-    }
-    if (auto collapsed = dyn_cast<memref::CollapseShapeOp>(user)) {
-      users.append(collapsed->getUsers().begin(), collapsed->getUsers().end());
+    if (isa<memref::SubViewOp, memref::CollapseShapeOp, memref::CastOp>(user)) {
+      users.append(user->getUsers().begin(), user->getUsers().end());
       continue;
     }
     if (isMemoryEffectFree(user) || isa<vector::TransferReadOp>(user))
@@ -221,7 +216,9 @@ void TransferOptimization::storeToLoadForwarding(vector::TransferReadOp read) {
               cast<VectorTransferOpInterface>(read.getOperation()),
               /*testDynamicValueUsingBounds=*/true))
         continue;
-      if (write.getSource() == read.getSource() &&
+      if (memref::isSameViewOrTrivialAlias(
+              cast<MemrefValue>(read.getSource()),
+              cast<MemrefValue>(write.getSource())) &&
           dominators.dominates(write, read) && checkSameValueRAW(write, read)) {
         if (lastwrite == nullptr || dominators.dominates(lastwrite, write))
           lastwrite = write;
diff --git a/mlir/test/Dialect/Vector/vector-transferop-opt.mlir b/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
index 0719c0dd1742..3ddfacf40cf6 100644
--- a/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
+++ b/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
@@ -485,3 +485,33 @@ func.func @forward_dead_constant_splat_store_with_masking_negative_3(%buffer : m
   vector.transfer_write %x, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
   return
 }
+
+// Here each read/write is to a different subview, but they all point to exact
+// same bit of memory (just through casts and subviews with unit strides and
+// zero offsets).
+// CHECK-LABEL: func @forward_and_eliminate_stores_through_trivial_aliases
+//   CHECK-NOT:   vector.transfer_write
+//   CHECK-NOT:   vector.transfer_read
+//       CHECK:   scf.for
+//       CHECK:   }
+//       CHECK:   vector.transfer_write
+//       CHECK:   return
+func.func @forward_and_eliminate_stores_through_trivial_aliases(
+  %buffer : memref<?x?xf32>, %vec: vector<[8]x[8]xf32>, %size: index, %a_size: index, %another_size: index
+) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c32 = arith.constant 32 : index
+  %cst = arith.constant 0.0 : f32
+  vector.transfer_write %vec, %buffer[%c0, %c0] {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>
+  %direct_subview = memref.subview %buffer[0, 0] [%a_size, %a_size] [1, 1] : memref<?x?xf32> to memref<?x?xf32, strided<[?, 1], offset: ?>>
+  %cast = memref.cast %direct_subview : memref<?x?xf32, strided<[?, 1], offset: ?>> to memref<?x?xf32>
+  %subview_of_cast = memref.subview %cast[0, 0] [%another_size, %another_size] [1, 1] : memref<?x?xf32> to memref<?x?xf32, strided<[?, 1], offset: ?>>
+  %21 = vector.transfer_read %direct_subview[%c0, %c0], %cst {in_bounds = [true, true]} : memref<?x?xf32, strided<[?, 1], offset: ?>>, vector<[8]x[8]xf32>
+  %23 = scf.for %arg2 = %c0 to %c32 step %c1 iter_args(%arg3 = %21) -> (vector<[8]x[8]xf32>) {
+    %24 = arith.addf %arg3, %arg3 : vector<[8]x[8]xf32>
+    scf.yield %24 : vector<[8]x[8]xf32>
+  }
+  vector.transfer_write %23, %subview_of_cast[%c0, %c0] {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32, strided<[?, 1], offset: ?>>
+  return
+}
-- 
GitLab


From 1795fa58bac5092f758bf27e1b27a2b333307bab Mon Sep 17 00:00:00 2001
From: Sirraide <aeternalmail@gmail.com>
Date: Thu, 16 May 2024 12:08:39 +0200
Subject: [PATCH 0990/1206] [NFC] Update example in comment

---
 clang/lib/Sema/SemaExpr.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index cc507524e2fc..151cda5d68b0 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -7133,10 +7133,10 @@ Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
       //
       // But: C99-C23 6.5.2.5 Compound literals constraint 1: The type name
       // shall specify an object type or an array of unknown size, but not a
-      // variable length array type. This seems odd, as it allows int a[size] =
-      // {}; but forbids int a[size] = (int[size]){}; As this is what the
-      // standard says, this is what's implemented here for C (except for the
-      // extension that permits constant foldable size arrays)
+      // variable length array type. This seems odd, as it allows 'int a[size] =
+      // {}', but forbids 'int *a = (int[size]){}'. As this is what the standard
+      // says, this is what's implemented here for C (except for the extension
+      // that permits constant foldable size arrays)
 
       auto diagID = LangOpts.CPlusPlus
                         ? diag::err_variable_object_no_init
-- 
GitLab


From 5ffd154cf61390c1ed32a1b0eb134929f78c0fbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Don=C3=A1t=20Nagy?= <donat.nagy@ericsson.com>
Date: Thu, 16 May 2024 12:10:56 +0200
Subject: [PATCH 0991/1206] [analyzer] Clean up list of taint propagation
 functions (#91635)

This commit refactors GenericTaintChecker and performs various
improvements in the list of taint propagation functions:

1. The matching mode (usually `CDM::CLibrary` or
`CDM::CLibraryMaybeHardened`) was specified to avoid matching e.g. C++
methods or functions from a user-defined namespace that happen to share
the name of a well-known library function.
2. With these matching modes, a `CallDescription` can automatically
match builtin variants of the functions, so entries that explicitly
specified a builtin function were removed. This eliminated
inconsistencies where the "normal" and the builtin variant of the same
function was handled differently (e.g. `__builtin_strlcat` was covered,
while plain `strlcat` wasn't; while `__builtin_memcpy` and `memcpy` were
both on the list with different propagation rules).
3. The modeling of the functions `strlcat` and `strncat` was updated to
propagate taint from the first argument (index 0), because a tainted
string should remain tainted even if we append something else to it.
Note that this was already applied to `strcat` and `wcsncat` by commit
6ceb1c0ef9f544be0eed65e46cc7d99941a001bf.
4. Some functions were updated to propagate taint from a size/length
argument to the result: e.g. `memcmp(p, q, get_tainted_int())` will now
return a tainted value (because the attacker can manipulate it). This
principle was already present in some propagation rules (e.g.
`__builtin_memcpy` was handled this way), and even after this commit
there are still some functions where it isn't applied. (I only aimed for
consistency within the same function family.)
5. Functions that have hardened `__FOO_chk()` variants are matched in
`CDM:CLibraryMaybeHardened` to ensure consistent handling of the
"normal" and the hardened variant. I added special handling for the
hardened variants of "sprintf" and "snprintf" because there the extra
parameters are inserted into the middle of the parameter list.
6. Modeling of `sscanf_s` was added, to complete the group of `fscanf`,
`fscanf_s` and `sscanf`.
7. The `Source()` specifications for `gets`, `gets_s` and `wgetch` were
ill-formed: they were specifying variadic arguments starting at argument
index `ReturnValueIndex`. (That is, in addition to the return value they
were propagating taint to all arguments.)
8. Functions that were related to each other were grouped together. (I
know that this makes the diff harder to read, but I felt that the full
list is unreadable without some reorganization.)
9. I spotted and removed some redundant curly braces. Perhaps would be
good to switch to a cleaner layout with less nested braces...
10. I updated some obsolete comments and added two TODOs for issues that
should be fixed in followup commits.
---
 .../Checkers/GenericTaintChecker.cpp          | 370 ++++++++++--------
 1 file changed, 204 insertions(+), 166 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
index a0190c30bfd2..eb40a4cd3d90 100644
--- a/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
@@ -400,17 +400,14 @@ private:
   void taintUnsafeSocketProtocol(const CallEvent &Call,
                                  CheckerContext &C) const;
 
-  /// Default taint rules are initalized with the help of a CheckerContext to
-  /// access the names of built-in functions like memcpy.
+  /// The taint rules are initalized with the help of a CheckerContext to
+  /// access user-provided configuration.
   void initTaintRules(CheckerContext &C) const;
 
-  /// CallDescription currently cannot restrict matches to the global namespace
-  /// only, which is why multiple CallDescriptionMaps are used, as we want to
-  /// disambiguate global C functions from functions inside user-defined
-  /// namespaces.
-  // TODO: Remove separation to simplify matching logic once CallDescriptions
-  // are more expressive.
-
+  // TODO: The two separate `CallDescriptionMap`s were introduced when
+  // `CallDescription` was unable to restrict matches to the global namespace
+  // only. This limitation no longer exists, so the following two maps should
+  // be unified.
   mutable std::optional<RuleLookupTy> StaticTaintRules;
   mutable std::optional<RuleLookupTy> DynamicTaintRules;
 };
@@ -506,7 +503,8 @@ void GenericTaintRuleParser::consumeRulesFromConfig(const Config &C,
                                                     GenericTaintRule &&Rule,
                                                     RulesContTy &Rules) {
   NamePartsTy NameParts = parseNameParts(C);
-  Rules.emplace_back(CallDescription(NameParts), std::move(Rule));
+  Rules.emplace_back(CallDescription(CDM::Unspecified, NameParts),
+                     std::move(Rule));
 }
 
 void GenericTaintRuleParser::parseConfig(const std::string &Option,
@@ -572,196 +570,236 @@ void GenericTaintChecker::initTaintRules(CheckerContext &C) const {
       std::vector<std::pair<CallDescription, GenericTaintRule>>;
   using TR = GenericTaintRule;
 
-  const Builtin::Context &BI = C.getASTContext().BuiltinInfo;
-
   RulesConstructionTy GlobalCRules{
       // Sources
-      {{{"fdopen"}}, TR::Source({{ReturnValueIndex}})},
-      {{{"fopen"}}, TR::Source({{ReturnValueIndex}})},
-      {{{"freopen"}}, TR::Source({{ReturnValueIndex}})},
-      {{{"getch"}}, TR::Source({{ReturnValueIndex}})},
-      {{{"getchar"}}, TR::Source({{ReturnValueIndex}})},
-      {{{"getchar_unlocked"}}, TR::Source({{ReturnValueIndex}})},
-      {{{"gets"}}, TR::Source({{0}, ReturnValueIndex})},
-      {{{"gets_s"}}, TR::Source({{0}, ReturnValueIndex})},
-      {{{"scanf"}}, TR::Source({{}, 1})},
-      {{{"scanf_s"}}, TR::Source({{}, {1}})},
-      {{{"wgetch"}}, TR::Source({{}, ReturnValueIndex})},
+      {{CDM::CLibrary, {"fdopen"}}, TR::Source({{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"fopen"}}, TR::Source({{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"freopen"}}, TR::Source({{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"getch"}}, TR::Source({{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"getchar"}}, TR::Source({{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"getchar_unlocked"}}, TR::Source({{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"gets"}}, TR::Source({{0, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"gets_s"}}, TR::Source({{0, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"scanf"}}, TR::Source({{}, 1})},
+      {{CDM::CLibrary, {"scanf_s"}}, TR::Source({{}, 1})},
+      {{CDM::CLibrary, {"wgetch"}}, TR::Source({{ReturnValueIndex}})},
       // Sometimes the line between taint sources and propagators is blurry.
       // _IO_getc is choosen to be a source, but could also be a propagator.
       // This way it is simpler, as modeling it as a propagator would require
       // to model the possible sources of _IO_FILE * values, which the _IO_getc
       // function takes as parameters.
-      {{{"_IO_getc"}}, TR::Source({{ReturnValueIndex}})},
-      {{{"getcwd"}}, TR::Source({{0, ReturnValueIndex}})},
-      {{{"getwd"}}, TR::Source({{0, ReturnValueIndex}})},
-      {{{"readlink"}}, TR::Source({{1, ReturnValueIndex}})},
-      {{{"readlinkat"}}, TR::Source({{2, ReturnValueIndex}})},
-      {{{"get_current_dir_name"}}, TR::Source({{ReturnValueIndex}})},
-      {{{"gethostname"}}, TR::Source({{0}})},
-      {{{"getnameinfo"}}, TR::Source({{2, 4}})},
-      {{{"getseuserbyname"}}, TR::Source({{1, 2}})},
-      {{{"getgroups"}}, TR::Source({{1, ReturnValueIndex}})},
-      {{{"getlogin"}}, TR::Source({{ReturnValueIndex}})},
-      {{{"getlogin_r"}}, TR::Source({{0}})},
+      {{CDM::CLibrary, {"_IO_getc"}}, TR::Source({{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"getcwd"}}, TR::Source({{0, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"getwd"}}, TR::Source({{0, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"readlink"}}, TR::Source({{1, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"readlinkat"}}, TR::Source({{2, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"get_current_dir_name"}},
+       TR::Source({{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"gethostname"}}, TR::Source({{0}})},
+      {{CDM::CLibrary, {"getnameinfo"}}, TR::Source({{2, 4}})},
+      {{CDM::CLibrary, {"getseuserbyname"}}, TR::Source({{1, 2}})},
+      {{CDM::CLibrary, {"getgroups"}}, TR::Source({{1, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"getlogin"}}, TR::Source({{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"getlogin_r"}}, TR::Source({{0}})},
 
       // Props
-      {{{"accept"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"atoi"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"atol"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"atoll"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"fgetc"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"fgetln"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"fgets"}}, TR::Prop({{2}}, {{0, ReturnValueIndex}})},
-      {{{"fgetws"}}, TR::Prop({{2}}, {{0, ReturnValueIndex}})},
-      {{{"fscanf"}}, TR::Prop({{0}}, {{}, 2})},
-      {{{"fscanf_s"}}, TR::Prop({{0}}, {{}, {2}})},
-      {{{"sscanf"}}, TR::Prop({{0}}, {{}, 2})},
-
-      {{{"getc"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"getc_unlocked"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"getdelim"}}, TR::Prop({{3}}, {{0}})},
-      {{{"getline"}}, TR::Prop({{2}}, {{0}})},
-      {{{"getw"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"pread"}}, TR::Prop({{0, 1, 2, 3}}, {{1, ReturnValueIndex}})},
-      {{{"read"}}, TR::Prop({{0, 2}}, {{1, ReturnValueIndex}})},
-      {{{"strchr"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"strrchr"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"tolower"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"toupper"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"fread"}}, TR::Prop({{3}}, {{0, ReturnValueIndex}})},
-      {{{"recv"}}, TR::Prop({{0}}, {{1, ReturnValueIndex}})},
-      {{{"recvfrom"}}, TR::Prop({{0}}, {{1, ReturnValueIndex}})},
-
-      {{{"ttyname"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"ttyname_r"}}, TR::Prop({{0}}, {{1, ReturnValueIndex}})},
-
-      {{{"basename"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"dirname"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"fnmatch"}}, TR::Prop({{1}}, {{ReturnValueIndex}})},
-      {{{"memchr"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"memrchr"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"rawmemchr"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-
-      {{{"mbtowc"}}, TR::Prop({{1}}, {{0, ReturnValueIndex}})},
-      {{{"wctomb"}}, TR::Prop({{1}}, {{0, ReturnValueIndex}})},
-      {{{"wcwidth"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-
-      {{{"memcmp"}}, TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
-      {{{"memcpy"}}, TR::Prop({{1}}, {{0, ReturnValueIndex}})},
-      {{{"memmove"}}, TR::Prop({{1}}, {{0, ReturnValueIndex}})},
-      // If memmem was called with a tainted needle and the search was
-      // successful, that would mean that the value pointed by the return value
-      // has the same content as the needle. If we choose to go by the policy of
-      // content equivalence implies taintedness equivalence, that would mean
-      // haystack should be considered a propagation source argument.
-      {{{"memmem"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-
-      // The comment for memmem above also applies to strstr.
-      {{{"strstr"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"strcasestr"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-
-      {{{"strchrnul"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-
-      {{{"index"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"rindex"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"accept"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"atoi"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"atol"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"atoll"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"fgetc"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"fgetln"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"fgets"}},
+       TR::Prop({{2}}, {{0, ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"fgetws"}},
+       TR::Prop({{2}}, {{0, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"fscanf"}}, TR::Prop({{0}}, {{}, 2})},
+      {{CDM::CLibrary, {"fscanf_s"}}, TR::Prop({{0}}, {{}, 2})},
+      {{CDM::CLibrary, {"sscanf"}}, TR::Prop({{0}}, {{}, 2})},
+      {{CDM::CLibrary, {"sscanf_s"}}, TR::Prop({{0}}, {{}, 2})},
+
+      {{CDM::CLibrary, {"getc"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"getc_unlocked"}},
+       TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"getdelim"}}, TR::Prop({{3}}, {{0}})},
+      // TODO: this intends to match the C function `getline()`, but the call
+      // description also matches the C++ function `std::getline()`; it should
+      // be ruled out by some additional logic.
+      {{CDM::CLibrary, {"getline"}}, TR::Prop({{2}}, {{0}})},
+      {{CDM::CLibrary, {"getw"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"pread"}},
+       TR::Prop({{0, 1, 2, 3}}, {{1, ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"read"}},
+       TR::Prop({{0, 2}}, {{1, ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"fread"}},
+       TR::Prop({{3}}, {{0, ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"recv"}},
+       TR::Prop({{0}}, {{1, ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"recvfrom"}},
+       TR::Prop({{0}}, {{1, ReturnValueIndex}})},
+
+      {{CDM::CLibrary, {"ttyname"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"ttyname_r"}},
+       TR::Prop({{0}}, {{1, ReturnValueIndex}})},
+
+      {{CDM::CLibrary, {"basename"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"dirname"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"fnmatch"}}, TR::Prop({{1}}, {{ReturnValueIndex}})},
+
+      {{CDM::CLibrary, {"mbtowc"}}, TR::Prop({{1}}, {{0, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"wctomb"}}, TR::Prop({{1}}, {{0, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"wcwidth"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+
+      {{CDM::CLibrary, {"memcmp"}},
+       TR::Prop({{0, 1, 2}}, {{ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"memcpy"}},
+       TR::Prop({{1, 2}}, {{0, ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"memmove"}},
+       TR::Prop({{1, 2}}, {{0, ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"bcopy"}}, TR::Prop({{0, 2}}, {{1}})},
+
+      // Note: "memmem" and its variants search for a byte sequence ("needle")
+      // in a larger area ("haystack"). Currently we only propagate taint from
+      // the haystack to the result, but in theory tampering with the needle
+      // could also produce incorrect results.
+      {{CDM::CLibrary, {"memmem"}}, TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strstr"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strcasestr"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+
+      // Analogously, the following functions search for a byte within a buffer
+      // and we only propagate taint from the buffer to the result.
+      {{CDM::CLibraryMaybeHardened, {"memchr"}},
+       TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"memrchr"}},
+       TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"rawmemchr"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"strchr"}},
+       TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"strrchr"}},
+       TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"strchrnul"}},
+       TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"index"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"rindex"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
 
       // FIXME: In case of arrays, only the first element of the array gets
       // tainted.
-      {{{"qsort"}}, TR::Prop({{0}}, {{0}})},
-      {{{"qsort_r"}}, TR::Prop({{0}}, {{0}})},
-
-      {{{"strcmp"}}, TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
-      {{{"strcasecmp"}}, TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
-      {{{"strncmp"}}, TR::Prop({{0, 1, 2}}, {{ReturnValueIndex}})},
-      {{{"strncasecmp"}}, TR::Prop({{0, 1, 2}}, {{ReturnValueIndex}})},
-      {{{"strspn"}}, TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
-      {{{"strcspn"}}, TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
-      {{{"strpbrk"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"strndup"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"strndupa"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"qsort"}}, TR::Prop({{0}}, {{0}})},
+      {{CDM::CLibrary, {"qsort_r"}}, TR::Prop({{0}}, {{0}})},
+
+      {{CDM::CLibrary, {"strcmp"}}, TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strcasecmp"}},
+       TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strncmp"}},
+       TR::Prop({{0, 1, 2}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strncasecmp"}},
+       TR::Prop({{0, 1, 2}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strspn"}}, TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strcspn"}}, TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strpbrk"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+
+      {{CDM::CLibrary, {"strndup"}}, TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strndupa"}}, TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strdup"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strdupa"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"wcsdup"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
 
       // strlen, wcslen, strnlen and alike intentionally don't propagate taint.
       // See the details here: https://github.com/llvm/llvm-project/pull/66086
 
-      {{{"strtol"}}, TR::Prop({{0}}, {{1, ReturnValueIndex}})},
-      {{{"strtoll"}}, TR::Prop({{0}}, {{1, ReturnValueIndex}})},
-      {{{"strtoul"}}, TR::Prop({{0}}, {{1, ReturnValueIndex}})},
-      {{{"strtoull"}}, TR::Prop({{0}}, {{1, ReturnValueIndex}})},
-
-      {{{"isalnum"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"isalpha"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"isascii"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"isblank"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"iscntrl"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"isdigit"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"isgraph"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"islower"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"isprint"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"ispunct"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"isspace"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"isupper"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{{"isxdigit"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-
-      {{CDM::CLibraryMaybeHardened, {BI.getName(Builtin::BIstrncat)}},
-       TR::Prop({{1, 2}}, {{0, ReturnValueIndex}})},
-      {{CDM::CLibraryMaybeHardened, {BI.getName(Builtin::BIstrlcpy)}},
-       TR::Prop({{1, 2}}, {{0}})},
-      {{CDM::CLibraryMaybeHardened, {BI.getName(Builtin::BIstrlcat)}},
-       TR::Prop({{1, 2}}, {{0}})},
-      {{CDM::CLibraryMaybeHardened, {{"snprintf"}}},
-       TR::Prop({{1}, 3}, {{0, ReturnValueIndex}})},
-      {{CDM::CLibraryMaybeHardened, {{"sprintf"}}},
-       TR::Prop({{1}, 2}, {{0, ReturnValueIndex}})},
-      {{CDM::CLibraryMaybeHardened, {{"strcpy"}}},
+      {{CDM::CLibrary, {"strtol"}}, TR::Prop({{0}}, {{1, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strtoll"}}, TR::Prop({{0}}, {{1, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strtoul"}}, TR::Prop({{0}}, {{1, ReturnValueIndex}})},
+      {{CDM::CLibrary, {"strtoull"}}, TR::Prop({{0}}, {{1, ReturnValueIndex}})},
+
+      {{CDM::CLibrary, {"tolower"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"toupper"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+
+      {{CDM::CLibrary, {"isalnum"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"isalpha"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"isascii"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"isblank"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"iscntrl"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"isdigit"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"isgraph"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"islower"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"isprint"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"ispunct"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"isspace"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"isupper"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+      {{CDM::CLibrary, {"isxdigit"}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
+
+      {{CDM::CLibraryMaybeHardened, {"strcpy"}},
        TR::Prop({{1}}, {{0, ReturnValueIndex}})},
-      {{CDM::CLibraryMaybeHardened, {{"stpcpy"}}},
+      {{CDM::CLibraryMaybeHardened, {"stpcpy"}},
        TR::Prop({{1}}, {{0, ReturnValueIndex}})},
-      {{CDM::CLibraryMaybeHardened, {{"strcat"}}},
+      {{CDM::CLibraryMaybeHardened, {"strcat"}},
        TR::Prop({{0, 1}}, {{0, ReturnValueIndex}})},
-      {{CDM::CLibraryMaybeHardened, {{"wcsncat"}}},
+      {{CDM::CLibraryMaybeHardened, {"wcsncat"}},
        TR::Prop({{0, 1}}, {{0, ReturnValueIndex}})},
-      {{CDM::CLibrary, {{"strdup"}}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{CDM::CLibrary, {{"strdupa"}}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{CDM::CLibrary, {{"wcsdup"}}}, TR::Prop({{0}}, {{ReturnValueIndex}})},
-      {{CDM::CLibrary, BI.getName(Builtin::BImemcpy)},
-       TR::Prop({{1, 2}}, {{0, ReturnValueIndex}})},
-      {{CDM::CLibrary, {BI.getName(Builtin::BImemmove)}},
+      {{CDM::CLibraryMaybeHardened, {"strncpy"}},
        TR::Prop({{1, 2}}, {{0, ReturnValueIndex}})},
-      {{CDM::CLibrary, {BI.getName(Builtin::BIstrncpy)}},
-       TR::Prop({{1, 2}}, {{0, ReturnValueIndex}})},
-      {{CDM::CLibrary, {BI.getName(Builtin::BIstrndup)}},
-       TR::Prop({{0, 1}}, {{ReturnValueIndex}})},
-      {{CDM::CLibrary, {"bcopy"}}, TR::Prop({{0, 2}}, {{1}})},
+      {{CDM::CLibraryMaybeHardened, {"strncat"}},
+       TR::Prop({{0, 1, 2}}, {{0, ReturnValueIndex}})},
+      {{CDM::CLibraryMaybeHardened, {"strlcpy"}}, TR::Prop({{1, 2}}, {{0}})},
+      {{CDM::CLibraryMaybeHardened, {"strlcat"}}, TR::Prop({{0, 1, 2}}, {{0}})},
+
+      // Usually the matching mode `CDM::CLibraryMaybeHardened` is sufficient
+      // for unified handling of a function `FOO()` and its hardened variant
+      // `__FOO_chk()`, but in the "sprintf" family the extra parameters of the
+      // hardened variants are inserted into the middle of the parameter list,
+      // so that would not work in their case.
+      // int snprintf(char * str, size_t maxlen, const char * format, ...);
+      {{CDM::CLibrary, {"snprintf"}},
+       TR::Prop({{1, 2}, 3}, {{0, ReturnValueIndex}})},
+      // int sprintf(char * str, const char * format, ...);
+      {{CDM::CLibrary, {"sprintf"}},
+       TR::Prop({{1}, 2}, {{0, ReturnValueIndex}})},
+      // int __snprintf_chk(char * str, size_t maxlen, int flag, size_t strlen,
+      //                    const char * format, ...);
+      {{CDM::CLibrary, {"__snprintf_chk"}},
+       TR::Prop({{1, 4}, 5}, {{0, ReturnValueIndex}})},
+      // int __sprintf_chk(char * str, int flag, size_t strlen, const char *
+      //                   format, ...);
+      {{CDM::CLibrary, {"__sprintf_chk"}},
+       TR::Prop({{3}, 4}, {{0, ReturnValueIndex}})},
 
       // Sinks
-      {{{"system"}}, TR::Sink({{0}}, MsgSanitizeSystemArgs)},
-      {{{"popen"}}, TR::Sink({{0}}, MsgSanitizeSystemArgs)},
-      {{{"execl"}}, TR::Sink({{}, {0}}, MsgSanitizeSystemArgs)},
-      {{{"execle"}}, TR::Sink({{}, {0}}, MsgSanitizeSystemArgs)},
-      {{{"execlp"}}, TR::Sink({{}, {0}}, MsgSanitizeSystemArgs)},
-      {{{"execv"}}, TR::Sink({{0, 1}}, MsgSanitizeSystemArgs)},
-      {{{"execve"}}, TR::Sink({{0, 1, 2}}, MsgSanitizeSystemArgs)},
-      {{{"fexecve"}}, TR::Sink({{0, 1, 2}}, MsgSanitizeSystemArgs)},
-      {{{"execvp"}}, TR::Sink({{0, 1}}, MsgSanitizeSystemArgs)},
-      {{{"execvpe"}}, TR::Sink({{0, 1, 2}}, MsgSanitizeSystemArgs)},
-      {{{"dlopen"}}, TR::Sink({{0}}, MsgSanitizeSystemArgs)},
+      {{CDM::CLibrary, {"system"}}, TR::Sink({{0}}, MsgSanitizeSystemArgs)},
+      {{CDM::CLibrary, {"popen"}}, TR::Sink({{0}}, MsgSanitizeSystemArgs)},
+      {{CDM::CLibrary, {"execl"}}, TR::Sink({{}, {0}}, MsgSanitizeSystemArgs)},
+      {{CDM::CLibrary, {"execle"}}, TR::Sink({{}, {0}}, MsgSanitizeSystemArgs)},
+      {{CDM::CLibrary, {"execlp"}}, TR::Sink({{}, {0}}, MsgSanitizeSystemArgs)},
+      {{CDM::CLibrary, {"execv"}}, TR::Sink({{0, 1}}, MsgSanitizeSystemArgs)},
+      {{CDM::CLibrary, {"execve"}},
+       TR::Sink({{0, 1, 2}}, MsgSanitizeSystemArgs)},
+      {{CDM::CLibrary, {"fexecve"}},
+       TR::Sink({{0, 1, 2}}, MsgSanitizeSystemArgs)},
+      {{CDM::CLibrary, {"execvp"}}, TR::Sink({{0, 1}}, MsgSanitizeSystemArgs)},
+      {{CDM::CLibrary, {"execvpe"}},
+       TR::Sink({{0, 1, 2}}, MsgSanitizeSystemArgs)},
+      {{CDM::CLibrary, {"dlopen"}}, TR::Sink({{0}}, MsgSanitizeSystemArgs)},
 
       // malloc, calloc, alloca, realloc, memccpy
       // are intentionally not marked as taint sinks because unconditional
       // reporting for these functions generates many false positives.
       // These taint sinks should be implemented in other checkers with more
       // sophisticated sanitation heuristics.
-      {{{{"setproctitle"}}}, TR::Sink({{0}, 1}, MsgUncontrolledFormatString)},
-      {{{{"setproctitle_fast"}}},
+
+      {{CDM::CLibrary, {"setproctitle"}},
+       TR::Sink({{0}, 1}, MsgUncontrolledFormatString)},
+      {{CDM::CLibrary, {"setproctitle_fast"}},
        TR::Sink({{0}, 1}, MsgUncontrolledFormatString)}};
 
-  // `getenv` returns taint only in untrusted environments.
   if (TR::UntrustedEnv(C)) {
     // void setproctitle_init(int argc, char *argv[], char *envp[])
+    // TODO: replace `MsgCustomSink` with a message that fits this situation.
+    GlobalCRules.push_back({{CDM::CLibrary, {"setproctitle_init"}},
+                            TR::Sink({{1, 2}}, MsgCustomSink)});
+
+    // `getenv` returns taint only in untrusted environments.
     GlobalCRules.push_back(
-        {{{"setproctitle_init"}}, TR::Sink({{1, 2}}, MsgCustomSink)});
-    GlobalCRules.push_back({{{"getenv"}}, TR::Source({{ReturnValueIndex}})});
+        {{CDM::CLibrary, {"getenv"}}, TR::Source({{ReturnValueIndex}})});
   }
 
   StaticTaintRules.emplace(std::make_move_iterator(GlobalCRules.begin()),
-- 
GitLab


From 83974a4b92d1fd33b8e21d7a868862893d9430e9 Mon Sep 17 00:00:00 2001
From: Dmitri Gribenko <gribozavr@gmail.com>
Date: Thu, 16 May 2024 12:05:13 +0200
Subject: [PATCH 0992/1206] Revert "[LoopUnroll] Clamp PartialThreshold for
 large LoopMicroOpBufferSize (#67657)"

This reverts commit f0b3654701bde1cf7821d60698b42383edaff9f3.

This commit triggers UB by reading an uninitialized variable.

`UP.PartialThreshold` is used uninitialized in `getUnrollingPreferences()` when
it is called from `LoopVectorizationPlanner::executePlan()`. In this case the
`UP` variable is created on the stack and its fields are not initialized.

```
==8802==WARNING: MemorySanitizer: use-of-uninitialized-value
    #0 0x557c0b081b99 in llvm::BasicTTIImplBase<llvm::X86TTIImpl>::getUnrollingPreferences(llvm::Loop*, llvm::ScalarEvolution&, llvm::TargetTransformInfo::UnrollingPreferences&, llvm::OptimizationRemarkEmitter*) llvm-project/llvm/include/llvm/CodeGen/BasicTTIImpl.h
    #1 0x557c0b07a40c in llvm::TargetTransformInfo::Model<llvm::X86TTIImpl>::getUnrollingPreferences(llvm::Loop*, llvm::ScalarEvolution&, llvm::TargetTransformInfo::UnrollingPreferences&, llvm::OptimizationRemarkEmitter*) llvm-project/llvm/include/llvm/Analysis/TargetTransformInfo.h:2277:17
    #2 0x557c0f5d69ee in llvm::TargetTransformInfo::getUnrollingPreferences(llvm::Loop*, llvm::ScalarEvolution&, llvm::TargetTransformInfo::UnrollingPreferences&, llvm::OptimizationRemarkEmitter*) const llvm-project/llvm/lib/Analysis/TargetTransformInfo.cpp:387:19
    #3 0x557c0e6b96a0 in llvm::LoopVectorizationPlanner::executePlan(llvm::ElementCount, unsigned int, llvm::VPlan&, llvm::InnerLoopVectorizer&, llvm::DominatorTree*, bool, llvm::DenseMap<llvm::SCEV const*, llvm::Value*, llvm::DenseMapInfo<llvm::SCEV const*, void>, llvm::detail::DenseMapPair<llvm::SCEV const*, llvm::Value*>> const*) llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp:7624:7
    #4 0x557c0e6e4b63 in llvm::LoopVectorizePass::processLoop(llvm::Loop*) llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp:10253:13
    #5 0x557c0e6f2429 in llvm::LoopVectorizePass::runImpl(llvm::Function&, llvm::ScalarEvolution&, llvm::LoopInfo&, llvm::TargetTransformInfo&, llvm::DominatorTree&, llvm::BlockFrequencyInfo*, llvm::TargetLibraryInfo*, llvm::DemandedBits&, llvm::AssumptionCache&, llvm::LoopAccessInfoManager&, llvm::OptimizationRemarkEmitter&, llvm::ProfileSummaryInfo*) llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp:10344:30
    #6 0x557c0e6f2f97 in llvm::LoopVectorizePass::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp:10383:9

[...]

  Uninitialized value was created by an allocation of 'UP' in the stack frame
    #0 0x557c0e6b961e in llvm::LoopVectorizationPlanner::executePlan(llvm::ElementCount, unsigned int, llvm::VPlan&, llvm::InnerLoopVectorizer&, llvm::DominatorTree*, bool, llvm::DenseMap<llvm::SCEV const*, llvm::Value*, llvm::DenseMapInfo<llvm::SCEV const*, void>, llvm::detail::DenseMapPair<llvm::SCEV const*, llvm::Value*>> const*) llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp:7623:3
```
---
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      |   8 +-
 llvm/test/Transforms/LoopUnroll/X86/znver3.ll | 764 +++++++++++++++++-
 2 files changed, 742 insertions(+), 30 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 8dba6a641285..2091432d4fe2 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -612,13 +612,7 @@ public:
     if (PartialUnrollingThreshold.getNumOccurrences() > 0)
       MaxOps = PartialUnrollingThreshold;
     else if (ST->getSchedModel().LoopMicroOpBufferSize > 0)
-      // Upper bound by the default PartialThreshold, which is the same as
-      // the default full-unroll Threshold. Even if the loop micro-op buffer
-      // is very large, this does not mean that we want to unroll all loops
-      // to that length, as it would increase code size beyond the limits of
-      // what unrolling normally allows.
-      MaxOps = std::min(ST->getSchedModel().LoopMicroOpBufferSize,
-                        UP.PartialThreshold);
+      MaxOps = ST->getSchedModel().LoopMicroOpBufferSize;
     else
       return;
 
diff --git a/llvm/test/Transforms/LoopUnroll/X86/znver3.ll b/llvm/test/Transforms/LoopUnroll/X86/znver3.ll
index 467c57906d88..30389062a096 100644
--- a/llvm/test/Transforms/LoopUnroll/X86/znver3.ll
+++ b/llvm/test/Transforms/LoopUnroll/X86/znver3.ll
@@ -9,8 +9,8 @@ define i32 @test(ptr %ary) "target-cpu"="znver3" {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_31:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_NEXT_31:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_127:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_NEXT_127:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[SUM_NEXT:%.*]] = add nsw i32 [[VAL]], [[SUM]]
@@ -137,12 +137,396 @@ define i32 @test(ptr %ary) "target-cpu"="znver3" {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_30:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 31
 ; CHECK-NEXT:    [[ARRAYIDX_31:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_30]]
 ; CHECK-NEXT:    [[VAL_31:%.*]] = load i32, ptr [[ARRAYIDX_31]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_31]] = add nsw i32 [[VAL_31]], [[SUM_NEXT_30]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_31]] = add nuw nsw i64 [[INDVARS_IV]], 32
-; CHECK-NEXT:    [[EXITCOND_NOT_31:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_31]], 8192
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT_31]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    [[SUM_NEXT_31:%.*]] = add nsw i32 [[VAL_31]], [[SUM_NEXT_30]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_31:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 32
+; CHECK-NEXT:    [[ARRAYIDX_32:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_31]]
+; CHECK-NEXT:    [[VAL_32:%.*]] = load i32, ptr [[ARRAYIDX_32]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_32:%.*]] = add nsw i32 [[VAL_32]], [[SUM_NEXT_31]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_32:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 33
+; CHECK-NEXT:    [[ARRAYIDX_33:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_32]]
+; CHECK-NEXT:    [[VAL_33:%.*]] = load i32, ptr [[ARRAYIDX_33]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_33:%.*]] = add nsw i32 [[VAL_33]], [[SUM_NEXT_32]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_33:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 34
+; CHECK-NEXT:    [[ARRAYIDX_34:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_33]]
+; CHECK-NEXT:    [[VAL_34:%.*]] = load i32, ptr [[ARRAYIDX_34]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_34:%.*]] = add nsw i32 [[VAL_34]], [[SUM_NEXT_33]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_34:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 35
+; CHECK-NEXT:    [[ARRAYIDX_35:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_34]]
+; CHECK-NEXT:    [[VAL_35:%.*]] = load i32, ptr [[ARRAYIDX_35]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_35:%.*]] = add nsw i32 [[VAL_35]], [[SUM_NEXT_34]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_35:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 36
+; CHECK-NEXT:    [[ARRAYIDX_36:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_35]]
+; CHECK-NEXT:    [[VAL_36:%.*]] = load i32, ptr [[ARRAYIDX_36]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_36:%.*]] = add nsw i32 [[VAL_36]], [[SUM_NEXT_35]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_36:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 37
+; CHECK-NEXT:    [[ARRAYIDX_37:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_36]]
+; CHECK-NEXT:    [[VAL_37:%.*]] = load i32, ptr [[ARRAYIDX_37]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_37:%.*]] = add nsw i32 [[VAL_37]], [[SUM_NEXT_36]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_37:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 38
+; CHECK-NEXT:    [[ARRAYIDX_38:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_37]]
+; CHECK-NEXT:    [[VAL_38:%.*]] = load i32, ptr [[ARRAYIDX_38]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_38:%.*]] = add nsw i32 [[VAL_38]], [[SUM_NEXT_37]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_38:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 39
+; CHECK-NEXT:    [[ARRAYIDX_39:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_38]]
+; CHECK-NEXT:    [[VAL_39:%.*]] = load i32, ptr [[ARRAYIDX_39]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_39:%.*]] = add nsw i32 [[VAL_39]], [[SUM_NEXT_38]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_39:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 40
+; CHECK-NEXT:    [[ARRAYIDX_40:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_39]]
+; CHECK-NEXT:    [[VAL_40:%.*]] = load i32, ptr [[ARRAYIDX_40]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_40:%.*]] = add nsw i32 [[VAL_40]], [[SUM_NEXT_39]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_40:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 41
+; CHECK-NEXT:    [[ARRAYIDX_41:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_40]]
+; CHECK-NEXT:    [[VAL_41:%.*]] = load i32, ptr [[ARRAYIDX_41]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_41:%.*]] = add nsw i32 [[VAL_41]], [[SUM_NEXT_40]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_41:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 42
+; CHECK-NEXT:    [[ARRAYIDX_42:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_41]]
+; CHECK-NEXT:    [[VAL_42:%.*]] = load i32, ptr [[ARRAYIDX_42]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_42:%.*]] = add nsw i32 [[VAL_42]], [[SUM_NEXT_41]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_42:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 43
+; CHECK-NEXT:    [[ARRAYIDX_43:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_42]]
+; CHECK-NEXT:    [[VAL_43:%.*]] = load i32, ptr [[ARRAYIDX_43]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_43:%.*]] = add nsw i32 [[VAL_43]], [[SUM_NEXT_42]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_43:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 44
+; CHECK-NEXT:    [[ARRAYIDX_44:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_43]]
+; CHECK-NEXT:    [[VAL_44:%.*]] = load i32, ptr [[ARRAYIDX_44]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_44:%.*]] = add nsw i32 [[VAL_44]], [[SUM_NEXT_43]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_44:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 45
+; CHECK-NEXT:    [[ARRAYIDX_45:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_44]]
+; CHECK-NEXT:    [[VAL_45:%.*]] = load i32, ptr [[ARRAYIDX_45]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_45:%.*]] = add nsw i32 [[VAL_45]], [[SUM_NEXT_44]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_45:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 46
+; CHECK-NEXT:    [[ARRAYIDX_46:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_45]]
+; CHECK-NEXT:    [[VAL_46:%.*]] = load i32, ptr [[ARRAYIDX_46]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_46:%.*]] = add nsw i32 [[VAL_46]], [[SUM_NEXT_45]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_46:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 47
+; CHECK-NEXT:    [[ARRAYIDX_47:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_46]]
+; CHECK-NEXT:    [[VAL_47:%.*]] = load i32, ptr [[ARRAYIDX_47]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_47:%.*]] = add nsw i32 [[VAL_47]], [[SUM_NEXT_46]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_47:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 48
+; CHECK-NEXT:    [[ARRAYIDX_48:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_47]]
+; CHECK-NEXT:    [[VAL_48:%.*]] = load i32, ptr [[ARRAYIDX_48]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_48:%.*]] = add nsw i32 [[VAL_48]], [[SUM_NEXT_47]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_48:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 49
+; CHECK-NEXT:    [[ARRAYIDX_49:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_48]]
+; CHECK-NEXT:    [[VAL_49:%.*]] = load i32, ptr [[ARRAYIDX_49]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_49:%.*]] = add nsw i32 [[VAL_49]], [[SUM_NEXT_48]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_49:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 50
+; CHECK-NEXT:    [[ARRAYIDX_50:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_49]]
+; CHECK-NEXT:    [[VAL_50:%.*]] = load i32, ptr [[ARRAYIDX_50]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_50:%.*]] = add nsw i32 [[VAL_50]], [[SUM_NEXT_49]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_50:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 51
+; CHECK-NEXT:    [[ARRAYIDX_51:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_50]]
+; CHECK-NEXT:    [[VAL_51:%.*]] = load i32, ptr [[ARRAYIDX_51]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_51:%.*]] = add nsw i32 [[VAL_51]], [[SUM_NEXT_50]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_51:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 52
+; CHECK-NEXT:    [[ARRAYIDX_52:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_51]]
+; CHECK-NEXT:    [[VAL_52:%.*]] = load i32, ptr [[ARRAYIDX_52]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_52:%.*]] = add nsw i32 [[VAL_52]], [[SUM_NEXT_51]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_52:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 53
+; CHECK-NEXT:    [[ARRAYIDX_53:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_52]]
+; CHECK-NEXT:    [[VAL_53:%.*]] = load i32, ptr [[ARRAYIDX_53]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_53:%.*]] = add nsw i32 [[VAL_53]], [[SUM_NEXT_52]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_53:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 54
+; CHECK-NEXT:    [[ARRAYIDX_54:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_53]]
+; CHECK-NEXT:    [[VAL_54:%.*]] = load i32, ptr [[ARRAYIDX_54]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_54:%.*]] = add nsw i32 [[VAL_54]], [[SUM_NEXT_53]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_54:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 55
+; CHECK-NEXT:    [[ARRAYIDX_55:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_54]]
+; CHECK-NEXT:    [[VAL_55:%.*]] = load i32, ptr [[ARRAYIDX_55]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_55:%.*]] = add nsw i32 [[VAL_55]], [[SUM_NEXT_54]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_55:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 56
+; CHECK-NEXT:    [[ARRAYIDX_56:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_55]]
+; CHECK-NEXT:    [[VAL_56:%.*]] = load i32, ptr [[ARRAYIDX_56]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_56:%.*]] = add nsw i32 [[VAL_56]], [[SUM_NEXT_55]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_56:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 57
+; CHECK-NEXT:    [[ARRAYIDX_57:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_56]]
+; CHECK-NEXT:    [[VAL_57:%.*]] = load i32, ptr [[ARRAYIDX_57]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_57:%.*]] = add nsw i32 [[VAL_57]], [[SUM_NEXT_56]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_57:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 58
+; CHECK-NEXT:    [[ARRAYIDX_58:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_57]]
+; CHECK-NEXT:    [[VAL_58:%.*]] = load i32, ptr [[ARRAYIDX_58]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_58:%.*]] = add nsw i32 [[VAL_58]], [[SUM_NEXT_57]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_58:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 59
+; CHECK-NEXT:    [[ARRAYIDX_59:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_58]]
+; CHECK-NEXT:    [[VAL_59:%.*]] = load i32, ptr [[ARRAYIDX_59]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_59:%.*]] = add nsw i32 [[VAL_59]], [[SUM_NEXT_58]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_59:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 60
+; CHECK-NEXT:    [[ARRAYIDX_60:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_59]]
+; CHECK-NEXT:    [[VAL_60:%.*]] = load i32, ptr [[ARRAYIDX_60]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_60:%.*]] = add nsw i32 [[VAL_60]], [[SUM_NEXT_59]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_60:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 61
+; CHECK-NEXT:    [[ARRAYIDX_61:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_60]]
+; CHECK-NEXT:    [[VAL_61:%.*]] = load i32, ptr [[ARRAYIDX_61]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_61:%.*]] = add nsw i32 [[VAL_61]], [[SUM_NEXT_60]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_61:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 62
+; CHECK-NEXT:    [[ARRAYIDX_62:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_61]]
+; CHECK-NEXT:    [[VAL_62:%.*]] = load i32, ptr [[ARRAYIDX_62]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_62:%.*]] = add nsw i32 [[VAL_62]], [[SUM_NEXT_61]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_62:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 63
+; CHECK-NEXT:    [[ARRAYIDX_63:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_62]]
+; CHECK-NEXT:    [[VAL_63:%.*]] = load i32, ptr [[ARRAYIDX_63]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_63:%.*]] = add nsw i32 [[VAL_63]], [[SUM_NEXT_62]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_63:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 64
+; CHECK-NEXT:    [[ARRAYIDX_64:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_63]]
+; CHECK-NEXT:    [[VAL_64:%.*]] = load i32, ptr [[ARRAYIDX_64]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_64:%.*]] = add nsw i32 [[VAL_64]], [[SUM_NEXT_63]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_64:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 65
+; CHECK-NEXT:    [[ARRAYIDX_65:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_64]]
+; CHECK-NEXT:    [[VAL_65:%.*]] = load i32, ptr [[ARRAYIDX_65]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_65:%.*]] = add nsw i32 [[VAL_65]], [[SUM_NEXT_64]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_65:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 66
+; CHECK-NEXT:    [[ARRAYIDX_66:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_65]]
+; CHECK-NEXT:    [[VAL_66:%.*]] = load i32, ptr [[ARRAYIDX_66]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_66:%.*]] = add nsw i32 [[VAL_66]], [[SUM_NEXT_65]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_66:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 67
+; CHECK-NEXT:    [[ARRAYIDX_67:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_66]]
+; CHECK-NEXT:    [[VAL_67:%.*]] = load i32, ptr [[ARRAYIDX_67]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_67:%.*]] = add nsw i32 [[VAL_67]], [[SUM_NEXT_66]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_67:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 68
+; CHECK-NEXT:    [[ARRAYIDX_68:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_67]]
+; CHECK-NEXT:    [[VAL_68:%.*]] = load i32, ptr [[ARRAYIDX_68]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_68:%.*]] = add nsw i32 [[VAL_68]], [[SUM_NEXT_67]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_68:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 69
+; CHECK-NEXT:    [[ARRAYIDX_69:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_68]]
+; CHECK-NEXT:    [[VAL_69:%.*]] = load i32, ptr [[ARRAYIDX_69]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_69:%.*]] = add nsw i32 [[VAL_69]], [[SUM_NEXT_68]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_69:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 70
+; CHECK-NEXT:    [[ARRAYIDX_70:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_69]]
+; CHECK-NEXT:    [[VAL_70:%.*]] = load i32, ptr [[ARRAYIDX_70]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_70:%.*]] = add nsw i32 [[VAL_70]], [[SUM_NEXT_69]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_70:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 71
+; CHECK-NEXT:    [[ARRAYIDX_71:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_70]]
+; CHECK-NEXT:    [[VAL_71:%.*]] = load i32, ptr [[ARRAYIDX_71]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_71:%.*]] = add nsw i32 [[VAL_71]], [[SUM_NEXT_70]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_71:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 72
+; CHECK-NEXT:    [[ARRAYIDX_72:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_71]]
+; CHECK-NEXT:    [[VAL_72:%.*]] = load i32, ptr [[ARRAYIDX_72]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_72:%.*]] = add nsw i32 [[VAL_72]], [[SUM_NEXT_71]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_72:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 73
+; CHECK-NEXT:    [[ARRAYIDX_73:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_72]]
+; CHECK-NEXT:    [[VAL_73:%.*]] = load i32, ptr [[ARRAYIDX_73]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_73:%.*]] = add nsw i32 [[VAL_73]], [[SUM_NEXT_72]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_73:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 74
+; CHECK-NEXT:    [[ARRAYIDX_74:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_73]]
+; CHECK-NEXT:    [[VAL_74:%.*]] = load i32, ptr [[ARRAYIDX_74]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_74:%.*]] = add nsw i32 [[VAL_74]], [[SUM_NEXT_73]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_74:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 75
+; CHECK-NEXT:    [[ARRAYIDX_75:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_74]]
+; CHECK-NEXT:    [[VAL_75:%.*]] = load i32, ptr [[ARRAYIDX_75]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_75:%.*]] = add nsw i32 [[VAL_75]], [[SUM_NEXT_74]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_75:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 76
+; CHECK-NEXT:    [[ARRAYIDX_76:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_75]]
+; CHECK-NEXT:    [[VAL_76:%.*]] = load i32, ptr [[ARRAYIDX_76]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_76:%.*]] = add nsw i32 [[VAL_76]], [[SUM_NEXT_75]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_76:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 77
+; CHECK-NEXT:    [[ARRAYIDX_77:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_76]]
+; CHECK-NEXT:    [[VAL_77:%.*]] = load i32, ptr [[ARRAYIDX_77]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_77:%.*]] = add nsw i32 [[VAL_77]], [[SUM_NEXT_76]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_77:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 78
+; CHECK-NEXT:    [[ARRAYIDX_78:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_77]]
+; CHECK-NEXT:    [[VAL_78:%.*]] = load i32, ptr [[ARRAYIDX_78]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_78:%.*]] = add nsw i32 [[VAL_78]], [[SUM_NEXT_77]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_78:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 79
+; CHECK-NEXT:    [[ARRAYIDX_79:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_78]]
+; CHECK-NEXT:    [[VAL_79:%.*]] = load i32, ptr [[ARRAYIDX_79]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_79:%.*]] = add nsw i32 [[VAL_79]], [[SUM_NEXT_78]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_79:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 80
+; CHECK-NEXT:    [[ARRAYIDX_80:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_79]]
+; CHECK-NEXT:    [[VAL_80:%.*]] = load i32, ptr [[ARRAYIDX_80]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_80:%.*]] = add nsw i32 [[VAL_80]], [[SUM_NEXT_79]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_80:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 81
+; CHECK-NEXT:    [[ARRAYIDX_81:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_80]]
+; CHECK-NEXT:    [[VAL_81:%.*]] = load i32, ptr [[ARRAYIDX_81]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_81:%.*]] = add nsw i32 [[VAL_81]], [[SUM_NEXT_80]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_81:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 82
+; CHECK-NEXT:    [[ARRAYIDX_82:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_81]]
+; CHECK-NEXT:    [[VAL_82:%.*]] = load i32, ptr [[ARRAYIDX_82]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_82:%.*]] = add nsw i32 [[VAL_82]], [[SUM_NEXT_81]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_82:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 83
+; CHECK-NEXT:    [[ARRAYIDX_83:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_82]]
+; CHECK-NEXT:    [[VAL_83:%.*]] = load i32, ptr [[ARRAYIDX_83]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_83:%.*]] = add nsw i32 [[VAL_83]], [[SUM_NEXT_82]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_83:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 84
+; CHECK-NEXT:    [[ARRAYIDX_84:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_83]]
+; CHECK-NEXT:    [[VAL_84:%.*]] = load i32, ptr [[ARRAYIDX_84]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_84:%.*]] = add nsw i32 [[VAL_84]], [[SUM_NEXT_83]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_84:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 85
+; CHECK-NEXT:    [[ARRAYIDX_85:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_84]]
+; CHECK-NEXT:    [[VAL_85:%.*]] = load i32, ptr [[ARRAYIDX_85]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_85:%.*]] = add nsw i32 [[VAL_85]], [[SUM_NEXT_84]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_85:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 86
+; CHECK-NEXT:    [[ARRAYIDX_86:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_85]]
+; CHECK-NEXT:    [[VAL_86:%.*]] = load i32, ptr [[ARRAYIDX_86]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_86:%.*]] = add nsw i32 [[VAL_86]], [[SUM_NEXT_85]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_86:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 87
+; CHECK-NEXT:    [[ARRAYIDX_87:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_86]]
+; CHECK-NEXT:    [[VAL_87:%.*]] = load i32, ptr [[ARRAYIDX_87]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_87:%.*]] = add nsw i32 [[VAL_87]], [[SUM_NEXT_86]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_87:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 88
+; CHECK-NEXT:    [[ARRAYIDX_88:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_87]]
+; CHECK-NEXT:    [[VAL_88:%.*]] = load i32, ptr [[ARRAYIDX_88]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_88:%.*]] = add nsw i32 [[VAL_88]], [[SUM_NEXT_87]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_88:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 89
+; CHECK-NEXT:    [[ARRAYIDX_89:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_88]]
+; CHECK-NEXT:    [[VAL_89:%.*]] = load i32, ptr [[ARRAYIDX_89]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_89:%.*]] = add nsw i32 [[VAL_89]], [[SUM_NEXT_88]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_89:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 90
+; CHECK-NEXT:    [[ARRAYIDX_90:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_89]]
+; CHECK-NEXT:    [[VAL_90:%.*]] = load i32, ptr [[ARRAYIDX_90]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_90:%.*]] = add nsw i32 [[VAL_90]], [[SUM_NEXT_89]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_90:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 91
+; CHECK-NEXT:    [[ARRAYIDX_91:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_90]]
+; CHECK-NEXT:    [[VAL_91:%.*]] = load i32, ptr [[ARRAYIDX_91]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_91:%.*]] = add nsw i32 [[VAL_91]], [[SUM_NEXT_90]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_91:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 92
+; CHECK-NEXT:    [[ARRAYIDX_92:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_91]]
+; CHECK-NEXT:    [[VAL_92:%.*]] = load i32, ptr [[ARRAYIDX_92]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_92:%.*]] = add nsw i32 [[VAL_92]], [[SUM_NEXT_91]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_92:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 93
+; CHECK-NEXT:    [[ARRAYIDX_93:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_92]]
+; CHECK-NEXT:    [[VAL_93:%.*]] = load i32, ptr [[ARRAYIDX_93]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_93:%.*]] = add nsw i32 [[VAL_93]], [[SUM_NEXT_92]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_93:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 94
+; CHECK-NEXT:    [[ARRAYIDX_94:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_93]]
+; CHECK-NEXT:    [[VAL_94:%.*]] = load i32, ptr [[ARRAYIDX_94]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_94:%.*]] = add nsw i32 [[VAL_94]], [[SUM_NEXT_93]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_94:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 95
+; CHECK-NEXT:    [[ARRAYIDX_95:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_94]]
+; CHECK-NEXT:    [[VAL_95:%.*]] = load i32, ptr [[ARRAYIDX_95]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_95:%.*]] = add nsw i32 [[VAL_95]], [[SUM_NEXT_94]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_95:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 96
+; CHECK-NEXT:    [[ARRAYIDX_96:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_95]]
+; CHECK-NEXT:    [[VAL_96:%.*]] = load i32, ptr [[ARRAYIDX_96]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_96:%.*]] = add nsw i32 [[VAL_96]], [[SUM_NEXT_95]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_96:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 97
+; CHECK-NEXT:    [[ARRAYIDX_97:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_96]]
+; CHECK-NEXT:    [[VAL_97:%.*]] = load i32, ptr [[ARRAYIDX_97]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_97:%.*]] = add nsw i32 [[VAL_97]], [[SUM_NEXT_96]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_97:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 98
+; CHECK-NEXT:    [[ARRAYIDX_98:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_97]]
+; CHECK-NEXT:    [[VAL_98:%.*]] = load i32, ptr [[ARRAYIDX_98]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_98:%.*]] = add nsw i32 [[VAL_98]], [[SUM_NEXT_97]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_98:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 99
+; CHECK-NEXT:    [[ARRAYIDX_99:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_98]]
+; CHECK-NEXT:    [[VAL_99:%.*]] = load i32, ptr [[ARRAYIDX_99]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_99:%.*]] = add nsw i32 [[VAL_99]], [[SUM_NEXT_98]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_99:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 100
+; CHECK-NEXT:    [[ARRAYIDX_100:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_99]]
+; CHECK-NEXT:    [[VAL_100:%.*]] = load i32, ptr [[ARRAYIDX_100]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_100:%.*]] = add nsw i32 [[VAL_100]], [[SUM_NEXT_99]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_100:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 101
+; CHECK-NEXT:    [[ARRAYIDX_101:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_100]]
+; CHECK-NEXT:    [[VAL_101:%.*]] = load i32, ptr [[ARRAYIDX_101]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_101:%.*]] = add nsw i32 [[VAL_101]], [[SUM_NEXT_100]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_101:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 102
+; CHECK-NEXT:    [[ARRAYIDX_102:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_101]]
+; CHECK-NEXT:    [[VAL_102:%.*]] = load i32, ptr [[ARRAYIDX_102]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_102:%.*]] = add nsw i32 [[VAL_102]], [[SUM_NEXT_101]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_102:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 103
+; CHECK-NEXT:    [[ARRAYIDX_103:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_102]]
+; CHECK-NEXT:    [[VAL_103:%.*]] = load i32, ptr [[ARRAYIDX_103]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_103:%.*]] = add nsw i32 [[VAL_103]], [[SUM_NEXT_102]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_103:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 104
+; CHECK-NEXT:    [[ARRAYIDX_104:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_103]]
+; CHECK-NEXT:    [[VAL_104:%.*]] = load i32, ptr [[ARRAYIDX_104]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_104:%.*]] = add nsw i32 [[VAL_104]], [[SUM_NEXT_103]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_104:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 105
+; CHECK-NEXT:    [[ARRAYIDX_105:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_104]]
+; CHECK-NEXT:    [[VAL_105:%.*]] = load i32, ptr [[ARRAYIDX_105]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_105:%.*]] = add nsw i32 [[VAL_105]], [[SUM_NEXT_104]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_105:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 106
+; CHECK-NEXT:    [[ARRAYIDX_106:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_105]]
+; CHECK-NEXT:    [[VAL_106:%.*]] = load i32, ptr [[ARRAYIDX_106]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_106:%.*]] = add nsw i32 [[VAL_106]], [[SUM_NEXT_105]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_106:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 107
+; CHECK-NEXT:    [[ARRAYIDX_107:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_106]]
+; CHECK-NEXT:    [[VAL_107:%.*]] = load i32, ptr [[ARRAYIDX_107]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_107:%.*]] = add nsw i32 [[VAL_107]], [[SUM_NEXT_106]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_107:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 108
+; CHECK-NEXT:    [[ARRAYIDX_108:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_107]]
+; CHECK-NEXT:    [[VAL_108:%.*]] = load i32, ptr [[ARRAYIDX_108]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_108:%.*]] = add nsw i32 [[VAL_108]], [[SUM_NEXT_107]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_108:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 109
+; CHECK-NEXT:    [[ARRAYIDX_109:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_108]]
+; CHECK-NEXT:    [[VAL_109:%.*]] = load i32, ptr [[ARRAYIDX_109]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_109:%.*]] = add nsw i32 [[VAL_109]], [[SUM_NEXT_108]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_109:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 110
+; CHECK-NEXT:    [[ARRAYIDX_110:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_109]]
+; CHECK-NEXT:    [[VAL_110:%.*]] = load i32, ptr [[ARRAYIDX_110]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_110:%.*]] = add nsw i32 [[VAL_110]], [[SUM_NEXT_109]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_110:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 111
+; CHECK-NEXT:    [[ARRAYIDX_111:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_110]]
+; CHECK-NEXT:    [[VAL_111:%.*]] = load i32, ptr [[ARRAYIDX_111]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_111:%.*]] = add nsw i32 [[VAL_111]], [[SUM_NEXT_110]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_111:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 112
+; CHECK-NEXT:    [[ARRAYIDX_112:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_111]]
+; CHECK-NEXT:    [[VAL_112:%.*]] = load i32, ptr [[ARRAYIDX_112]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_112:%.*]] = add nsw i32 [[VAL_112]], [[SUM_NEXT_111]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_112:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 113
+; CHECK-NEXT:    [[ARRAYIDX_113:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_112]]
+; CHECK-NEXT:    [[VAL_113:%.*]] = load i32, ptr [[ARRAYIDX_113]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_113:%.*]] = add nsw i32 [[VAL_113]], [[SUM_NEXT_112]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_113:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 114
+; CHECK-NEXT:    [[ARRAYIDX_114:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_113]]
+; CHECK-NEXT:    [[VAL_114:%.*]] = load i32, ptr [[ARRAYIDX_114]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_114:%.*]] = add nsw i32 [[VAL_114]], [[SUM_NEXT_113]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_114:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 115
+; CHECK-NEXT:    [[ARRAYIDX_115:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_114]]
+; CHECK-NEXT:    [[VAL_115:%.*]] = load i32, ptr [[ARRAYIDX_115]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_115:%.*]] = add nsw i32 [[VAL_115]], [[SUM_NEXT_114]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_115:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 116
+; CHECK-NEXT:    [[ARRAYIDX_116:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_115]]
+; CHECK-NEXT:    [[VAL_116:%.*]] = load i32, ptr [[ARRAYIDX_116]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_116:%.*]] = add nsw i32 [[VAL_116]], [[SUM_NEXT_115]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_116:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 117
+; CHECK-NEXT:    [[ARRAYIDX_117:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_116]]
+; CHECK-NEXT:    [[VAL_117:%.*]] = load i32, ptr [[ARRAYIDX_117]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_117:%.*]] = add nsw i32 [[VAL_117]], [[SUM_NEXT_116]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_117:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 118
+; CHECK-NEXT:    [[ARRAYIDX_118:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_117]]
+; CHECK-NEXT:    [[VAL_118:%.*]] = load i32, ptr [[ARRAYIDX_118]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_118:%.*]] = add nsw i32 [[VAL_118]], [[SUM_NEXT_117]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_118:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 119
+; CHECK-NEXT:    [[ARRAYIDX_119:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_118]]
+; CHECK-NEXT:    [[VAL_119:%.*]] = load i32, ptr [[ARRAYIDX_119]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_119:%.*]] = add nsw i32 [[VAL_119]], [[SUM_NEXT_118]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_119:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 120
+; CHECK-NEXT:    [[ARRAYIDX_120:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_119]]
+; CHECK-NEXT:    [[VAL_120:%.*]] = load i32, ptr [[ARRAYIDX_120]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_120:%.*]] = add nsw i32 [[VAL_120]], [[SUM_NEXT_119]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_120:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 121
+; CHECK-NEXT:    [[ARRAYIDX_121:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_120]]
+; CHECK-NEXT:    [[VAL_121:%.*]] = load i32, ptr [[ARRAYIDX_121]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_121:%.*]] = add nsw i32 [[VAL_121]], [[SUM_NEXT_120]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_121:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 122
+; CHECK-NEXT:    [[ARRAYIDX_122:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_121]]
+; CHECK-NEXT:    [[VAL_122:%.*]] = load i32, ptr [[ARRAYIDX_122]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_122:%.*]] = add nsw i32 [[VAL_122]], [[SUM_NEXT_121]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_122:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 123
+; CHECK-NEXT:    [[ARRAYIDX_123:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_122]]
+; CHECK-NEXT:    [[VAL_123:%.*]] = load i32, ptr [[ARRAYIDX_123]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_123:%.*]] = add nsw i32 [[VAL_123]], [[SUM_NEXT_122]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_123:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 124
+; CHECK-NEXT:    [[ARRAYIDX_124:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_123]]
+; CHECK-NEXT:    [[VAL_124:%.*]] = load i32, ptr [[ARRAYIDX_124]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_124:%.*]] = add nsw i32 [[VAL_124]], [[SUM_NEXT_123]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_124:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 125
+; CHECK-NEXT:    [[ARRAYIDX_125:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_124]]
+; CHECK-NEXT:    [[VAL_125:%.*]] = load i32, ptr [[ARRAYIDX_125]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_125:%.*]] = add nsw i32 [[VAL_125]], [[SUM_NEXT_124]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_125:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 126
+; CHECK-NEXT:    [[ARRAYIDX_126:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_125]]
+; CHECK-NEXT:    [[VAL_126:%.*]] = load i32, ptr [[ARRAYIDX_126]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_126:%.*]] = add nsw i32 [[VAL_126]], [[SUM_NEXT_125]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_126:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 127
+; CHECK-NEXT:    [[ARRAYIDX_127:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_126]]
+; CHECK-NEXT:    [[VAL_127:%.*]] = load i32, ptr [[ARRAYIDX_127]], align 4
+; CHECK-NEXT:    [[SUM_NEXT_127]] = add nsw i32 [[VAL_127]], [[SUM_NEXT_126]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_127]] = add nuw nsw i64 [[INDVARS_IV]], 128
+; CHECK-NEXT:    [[EXITCOND_NOT_127:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_127]], 8192
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT_127]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT_31]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT_127]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_NEXT_LCSSA]]
 ;
 entry:
@@ -167,16 +551,16 @@ define i32 @test2(ptr %ary, i64 %n) "target-cpu"="znver3" {
 ; CHECK-SAME: ptr [[ARY:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -1
-; CHECK-NEXT:    [[XTRAITER:%.*]] = and i64 [[N]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
+; CHECK-NEXT:    [[XTRAITER:%.*]] = and i64 [[N]], 7
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 7
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[FOR_COND_CLEANUP_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]]
 ; CHECK:       entry.new:
 ; CHECK-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[ENTRY_NEW]] ], [ [[SUM_NEXT_1:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[NITER:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[INDVARS_IV_NEXT_7:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[ENTRY_NEW]] ], [ [[SUM_NEXT_7:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[NITER:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[NITER_NEXT_7:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[DUMMY1:%.*]] = mul i32 [[VAL]], [[VAL]]
@@ -283,15 +667,339 @@ define i32 @test2(ptr %ary, i64 %n) "target-cpu"="znver3" {
 ; CHECK-NEXT:    [[DUMMY48_1:%.*]] = mul i32 [[DUMMY47_1]], [[DUMMY47_1]]
 ; CHECK-NEXT:    [[DUMMY49_1:%.*]] = mul i32 [[DUMMY48_1]], [[DUMMY48_1]]
 ; CHECK-NEXT:    [[DUMMY50_1:%.*]] = mul i32 [[DUMMY49_1]], [[DUMMY49_1]]
-; CHECK-NEXT:    [[SUM_NEXT_1]] = add nsw i32 [[DUMMY50_1]], [[SUM_NEXT]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2
-; CHECK-NEXT:    [[NITER_NEXT_1]] = add i64 [[NITER]], 2
-; CHECK-NEXT:    [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
-; CHECK-NEXT:    br i1 [[NITER_NCMP_1]], label [[FOR_COND_CLEANUP_UNR_LCSSA_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    [[SUM_NEXT_1:%.*]] = add nsw i32 [[DUMMY50_1]], [[SUM_NEXT]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-NEXT:    [[VAL_2:%.*]] = load i32, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[DUMMY1_2:%.*]] = mul i32 [[VAL_2]], [[VAL_2]]
+; CHECK-NEXT:    [[DUMMY2_2:%.*]] = mul i32 [[DUMMY1_2]], [[DUMMY1_2]]
+; CHECK-NEXT:    [[DUMMY3_2:%.*]] = mul i32 [[DUMMY2_2]], [[DUMMY2_2]]
+; CHECK-NEXT:    [[DUMMY4_2:%.*]] = mul i32 [[DUMMY3_2]], [[DUMMY3_2]]
+; CHECK-NEXT:    [[DUMMY5_2:%.*]] = mul i32 [[DUMMY4_2]], [[DUMMY4_2]]
+; CHECK-NEXT:    [[DUMMY6_2:%.*]] = mul i32 [[DUMMY5_2]], [[DUMMY5_2]]
+; CHECK-NEXT:    [[DUMMY7_2:%.*]] = mul i32 [[DUMMY6_2]], [[DUMMY6_2]]
+; CHECK-NEXT:    [[DUMMY8_2:%.*]] = mul i32 [[DUMMY7_2]], [[DUMMY7_2]]
+; CHECK-NEXT:    [[DUMMY9_2:%.*]] = mul i32 [[DUMMY8_2]], [[DUMMY8_2]]
+; CHECK-NEXT:    [[DUMMY10_2:%.*]] = mul i32 [[DUMMY9_2]], [[DUMMY9_2]]
+; CHECK-NEXT:    [[DUMMY11_2:%.*]] = mul i32 [[DUMMY10_2]], [[DUMMY10_2]]
+; CHECK-NEXT:    [[DUMMY12_2:%.*]] = mul i32 [[DUMMY11_2]], [[DUMMY11_2]]
+; CHECK-NEXT:    [[DUMMY13_2:%.*]] = mul i32 [[DUMMY12_2]], [[DUMMY12_2]]
+; CHECK-NEXT:    [[DUMMY14_2:%.*]] = mul i32 [[DUMMY13_2]], [[DUMMY13_2]]
+; CHECK-NEXT:    [[DUMMY15_2:%.*]] = mul i32 [[DUMMY14_2]], [[DUMMY14_2]]
+; CHECK-NEXT:    [[DUMMY16_2:%.*]] = mul i32 [[DUMMY15_2]], [[DUMMY15_2]]
+; CHECK-NEXT:    [[DUMMY17_2:%.*]] = mul i32 [[DUMMY16_2]], [[DUMMY16_2]]
+; CHECK-NEXT:    [[DUMMY18_2:%.*]] = mul i32 [[DUMMY17_2]], [[DUMMY17_2]]
+; CHECK-NEXT:    [[DUMMY19_2:%.*]] = mul i32 [[DUMMY18_2]], [[DUMMY18_2]]
+; CHECK-NEXT:    [[DUMMY20_2:%.*]] = mul i32 [[DUMMY19_2]], [[DUMMY19_2]]
+; CHECK-NEXT:    [[DUMMY21_2:%.*]] = mul i32 [[DUMMY20_2]], [[DUMMY20_2]]
+; CHECK-NEXT:    [[DUMMY22_2:%.*]] = mul i32 [[DUMMY21_2]], [[DUMMY21_2]]
+; CHECK-NEXT:    [[DUMMY23_2:%.*]] = mul i32 [[DUMMY22_2]], [[DUMMY22_2]]
+; CHECK-NEXT:    [[DUMMY24_2:%.*]] = mul i32 [[DUMMY23_2]], [[DUMMY23_2]]
+; CHECK-NEXT:    [[DUMMY25_2:%.*]] = mul i32 [[DUMMY24_2]], [[DUMMY24_2]]
+; CHECK-NEXT:    [[DUMMY26_2:%.*]] = mul i32 [[DUMMY25_2]], [[DUMMY25_2]]
+; CHECK-NEXT:    [[DUMMY27_2:%.*]] = mul i32 [[DUMMY26_2]], [[DUMMY26_2]]
+; CHECK-NEXT:    [[DUMMY28_2:%.*]] = mul i32 [[DUMMY27_2]], [[DUMMY27_2]]
+; CHECK-NEXT:    [[DUMMY29_2:%.*]] = mul i32 [[DUMMY28_2]], [[DUMMY28_2]]
+; CHECK-NEXT:    [[DUMMY30_2:%.*]] = mul i32 [[DUMMY29_2]], [[DUMMY29_2]]
+; CHECK-NEXT:    [[DUMMY31_2:%.*]] = mul i32 [[DUMMY30_2]], [[DUMMY30_2]]
+; CHECK-NEXT:    [[DUMMY32_2:%.*]] = mul i32 [[DUMMY31_2]], [[DUMMY31_2]]
+; CHECK-NEXT:    [[DUMMY33_2:%.*]] = mul i32 [[DUMMY32_2]], [[DUMMY32_2]]
+; CHECK-NEXT:    [[DUMMY34_2:%.*]] = mul i32 [[DUMMY33_2]], [[DUMMY33_2]]
+; CHECK-NEXT:    [[DUMMY35_2:%.*]] = mul i32 [[DUMMY34_2]], [[DUMMY34_2]]
+; CHECK-NEXT:    [[DUMMY36_2:%.*]] = mul i32 [[DUMMY35_2]], [[DUMMY35_2]]
+; CHECK-NEXT:    [[DUMMY37_2:%.*]] = mul i32 [[DUMMY36_2]], [[DUMMY36_2]]
+; CHECK-NEXT:    [[DUMMY38_2:%.*]] = mul i32 [[DUMMY37_2]], [[DUMMY37_2]]
+; CHECK-NEXT:    [[DUMMY39_2:%.*]] = mul i32 [[DUMMY38_2]], [[DUMMY38_2]]
+; CHECK-NEXT:    [[DUMMY40_2:%.*]] = mul i32 [[DUMMY39_2]], [[DUMMY39_2]]
+; CHECK-NEXT:    [[DUMMY41_2:%.*]] = mul i32 [[DUMMY40_2]], [[DUMMY40_2]]
+; CHECK-NEXT:    [[DUMMY42_2:%.*]] = mul i32 [[DUMMY41_2]], [[DUMMY41_2]]
+; CHECK-NEXT:    [[DUMMY43_2:%.*]] = mul i32 [[DUMMY42_2]], [[DUMMY42_2]]
+; CHECK-NEXT:    [[DUMMY44_2:%.*]] = mul i32 [[DUMMY43_2]], [[DUMMY43_2]]
+; CHECK-NEXT:    [[DUMMY45_2:%.*]] = mul i32 [[DUMMY44_2]], [[DUMMY44_2]]
+; CHECK-NEXT:    [[DUMMY46_2:%.*]] = mul i32 [[DUMMY45_2]], [[DUMMY45_2]]
+; CHECK-NEXT:    [[DUMMY47_2:%.*]] = mul i32 [[DUMMY46_2]], [[DUMMY46_2]]
+; CHECK-NEXT:    [[DUMMY48_2:%.*]] = mul i32 [[DUMMY47_2]], [[DUMMY47_2]]
+; CHECK-NEXT:    [[DUMMY49_2:%.*]] = mul i32 [[DUMMY48_2]], [[DUMMY48_2]]
+; CHECK-NEXT:    [[DUMMY50_2:%.*]] = mul i32 [[DUMMY49_2]], [[DUMMY49_2]]
+; CHECK-NEXT:    [[SUM_NEXT_2:%.*]] = add nsw i32 [[DUMMY50_2]], [[SUM_NEXT_1]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT:    [[VAL_3:%.*]] = load i32, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[DUMMY1_3:%.*]] = mul i32 [[VAL_3]], [[VAL_3]]
+; CHECK-NEXT:    [[DUMMY2_3:%.*]] = mul i32 [[DUMMY1_3]], [[DUMMY1_3]]
+; CHECK-NEXT:    [[DUMMY3_3:%.*]] = mul i32 [[DUMMY2_3]], [[DUMMY2_3]]
+; CHECK-NEXT:    [[DUMMY4_3:%.*]] = mul i32 [[DUMMY3_3]], [[DUMMY3_3]]
+; CHECK-NEXT:    [[DUMMY5_3:%.*]] = mul i32 [[DUMMY4_3]], [[DUMMY4_3]]
+; CHECK-NEXT:    [[DUMMY6_3:%.*]] = mul i32 [[DUMMY5_3]], [[DUMMY5_3]]
+; CHECK-NEXT:    [[DUMMY7_3:%.*]] = mul i32 [[DUMMY6_3]], [[DUMMY6_3]]
+; CHECK-NEXT:    [[DUMMY8_3:%.*]] = mul i32 [[DUMMY7_3]], [[DUMMY7_3]]
+; CHECK-NEXT:    [[DUMMY9_3:%.*]] = mul i32 [[DUMMY8_3]], [[DUMMY8_3]]
+; CHECK-NEXT:    [[DUMMY10_3:%.*]] = mul i32 [[DUMMY9_3]], [[DUMMY9_3]]
+; CHECK-NEXT:    [[DUMMY11_3:%.*]] = mul i32 [[DUMMY10_3]], [[DUMMY10_3]]
+; CHECK-NEXT:    [[DUMMY12_3:%.*]] = mul i32 [[DUMMY11_3]], [[DUMMY11_3]]
+; CHECK-NEXT:    [[DUMMY13_3:%.*]] = mul i32 [[DUMMY12_3]], [[DUMMY12_3]]
+; CHECK-NEXT:    [[DUMMY14_3:%.*]] = mul i32 [[DUMMY13_3]], [[DUMMY13_3]]
+; CHECK-NEXT:    [[DUMMY15_3:%.*]] = mul i32 [[DUMMY14_3]], [[DUMMY14_3]]
+; CHECK-NEXT:    [[DUMMY16_3:%.*]] = mul i32 [[DUMMY15_3]], [[DUMMY15_3]]
+; CHECK-NEXT:    [[DUMMY17_3:%.*]] = mul i32 [[DUMMY16_3]], [[DUMMY16_3]]
+; CHECK-NEXT:    [[DUMMY18_3:%.*]] = mul i32 [[DUMMY17_3]], [[DUMMY17_3]]
+; CHECK-NEXT:    [[DUMMY19_3:%.*]] = mul i32 [[DUMMY18_3]], [[DUMMY18_3]]
+; CHECK-NEXT:    [[DUMMY20_3:%.*]] = mul i32 [[DUMMY19_3]], [[DUMMY19_3]]
+; CHECK-NEXT:    [[DUMMY21_3:%.*]] = mul i32 [[DUMMY20_3]], [[DUMMY20_3]]
+; CHECK-NEXT:    [[DUMMY22_3:%.*]] = mul i32 [[DUMMY21_3]], [[DUMMY21_3]]
+; CHECK-NEXT:    [[DUMMY23_3:%.*]] = mul i32 [[DUMMY22_3]], [[DUMMY22_3]]
+; CHECK-NEXT:    [[DUMMY24_3:%.*]] = mul i32 [[DUMMY23_3]], [[DUMMY23_3]]
+; CHECK-NEXT:    [[DUMMY25_3:%.*]] = mul i32 [[DUMMY24_3]], [[DUMMY24_3]]
+; CHECK-NEXT:    [[DUMMY26_3:%.*]] = mul i32 [[DUMMY25_3]], [[DUMMY25_3]]
+; CHECK-NEXT:    [[DUMMY27_3:%.*]] = mul i32 [[DUMMY26_3]], [[DUMMY26_3]]
+; CHECK-NEXT:    [[DUMMY28_3:%.*]] = mul i32 [[DUMMY27_3]], [[DUMMY27_3]]
+; CHECK-NEXT:    [[DUMMY29_3:%.*]] = mul i32 [[DUMMY28_3]], [[DUMMY28_3]]
+; CHECK-NEXT:    [[DUMMY30_3:%.*]] = mul i32 [[DUMMY29_3]], [[DUMMY29_3]]
+; CHECK-NEXT:    [[DUMMY31_3:%.*]] = mul i32 [[DUMMY30_3]], [[DUMMY30_3]]
+; CHECK-NEXT:    [[DUMMY32_3:%.*]] = mul i32 [[DUMMY31_3]], [[DUMMY31_3]]
+; CHECK-NEXT:    [[DUMMY33_3:%.*]] = mul i32 [[DUMMY32_3]], [[DUMMY32_3]]
+; CHECK-NEXT:    [[DUMMY34_3:%.*]] = mul i32 [[DUMMY33_3]], [[DUMMY33_3]]
+; CHECK-NEXT:    [[DUMMY35_3:%.*]] = mul i32 [[DUMMY34_3]], [[DUMMY34_3]]
+; CHECK-NEXT:    [[DUMMY36_3:%.*]] = mul i32 [[DUMMY35_3]], [[DUMMY35_3]]
+; CHECK-NEXT:    [[DUMMY37_3:%.*]] = mul i32 [[DUMMY36_3]], [[DUMMY36_3]]
+; CHECK-NEXT:    [[DUMMY38_3:%.*]] = mul i32 [[DUMMY37_3]], [[DUMMY37_3]]
+; CHECK-NEXT:    [[DUMMY39_3:%.*]] = mul i32 [[DUMMY38_3]], [[DUMMY38_3]]
+; CHECK-NEXT:    [[DUMMY40_3:%.*]] = mul i32 [[DUMMY39_3]], [[DUMMY39_3]]
+; CHECK-NEXT:    [[DUMMY41_3:%.*]] = mul i32 [[DUMMY40_3]], [[DUMMY40_3]]
+; CHECK-NEXT:    [[DUMMY42_3:%.*]] = mul i32 [[DUMMY41_3]], [[DUMMY41_3]]
+; CHECK-NEXT:    [[DUMMY43_3:%.*]] = mul i32 [[DUMMY42_3]], [[DUMMY42_3]]
+; CHECK-NEXT:    [[DUMMY44_3:%.*]] = mul i32 [[DUMMY43_3]], [[DUMMY43_3]]
+; CHECK-NEXT:    [[DUMMY45_3:%.*]] = mul i32 [[DUMMY44_3]], [[DUMMY44_3]]
+; CHECK-NEXT:    [[DUMMY46_3:%.*]] = mul i32 [[DUMMY45_3]], [[DUMMY45_3]]
+; CHECK-NEXT:    [[DUMMY47_3:%.*]] = mul i32 [[DUMMY46_3]], [[DUMMY46_3]]
+; CHECK-NEXT:    [[DUMMY48_3:%.*]] = mul i32 [[DUMMY47_3]], [[DUMMY47_3]]
+; CHECK-NEXT:    [[DUMMY49_3:%.*]] = mul i32 [[DUMMY48_3]], [[DUMMY48_3]]
+; CHECK-NEXT:    [[DUMMY50_3:%.*]] = mul i32 [[DUMMY49_3]], [[DUMMY49_3]]
+; CHECK-NEXT:    [[SUM_NEXT_3:%.*]] = add nsw i32 [[DUMMY50_3]], [[SUM_NEXT_2]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_3:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_3]]
+; CHECK-NEXT:    [[VAL_4:%.*]] = load i32, ptr [[ARRAYIDX_4]], align 4
+; CHECK-NEXT:    [[DUMMY1_4:%.*]] = mul i32 [[VAL_4]], [[VAL_4]]
+; CHECK-NEXT:    [[DUMMY2_4:%.*]] = mul i32 [[DUMMY1_4]], [[DUMMY1_4]]
+; CHECK-NEXT:    [[DUMMY3_4:%.*]] = mul i32 [[DUMMY2_4]], [[DUMMY2_4]]
+; CHECK-NEXT:    [[DUMMY4_4:%.*]] = mul i32 [[DUMMY3_4]], [[DUMMY3_4]]
+; CHECK-NEXT:    [[DUMMY5_4:%.*]] = mul i32 [[DUMMY4_4]], [[DUMMY4_4]]
+; CHECK-NEXT:    [[DUMMY6_4:%.*]] = mul i32 [[DUMMY5_4]], [[DUMMY5_4]]
+; CHECK-NEXT:    [[DUMMY7_4:%.*]] = mul i32 [[DUMMY6_4]], [[DUMMY6_4]]
+; CHECK-NEXT:    [[DUMMY8_4:%.*]] = mul i32 [[DUMMY7_4]], [[DUMMY7_4]]
+; CHECK-NEXT:    [[DUMMY9_4:%.*]] = mul i32 [[DUMMY8_4]], [[DUMMY8_4]]
+; CHECK-NEXT:    [[DUMMY10_4:%.*]] = mul i32 [[DUMMY9_4]], [[DUMMY9_4]]
+; CHECK-NEXT:    [[DUMMY11_4:%.*]] = mul i32 [[DUMMY10_4]], [[DUMMY10_4]]
+; CHECK-NEXT:    [[DUMMY12_4:%.*]] = mul i32 [[DUMMY11_4]], [[DUMMY11_4]]
+; CHECK-NEXT:    [[DUMMY13_4:%.*]] = mul i32 [[DUMMY12_4]], [[DUMMY12_4]]
+; CHECK-NEXT:    [[DUMMY14_4:%.*]] = mul i32 [[DUMMY13_4]], [[DUMMY13_4]]
+; CHECK-NEXT:    [[DUMMY15_4:%.*]] = mul i32 [[DUMMY14_4]], [[DUMMY14_4]]
+; CHECK-NEXT:    [[DUMMY16_4:%.*]] = mul i32 [[DUMMY15_4]], [[DUMMY15_4]]
+; CHECK-NEXT:    [[DUMMY17_4:%.*]] = mul i32 [[DUMMY16_4]], [[DUMMY16_4]]
+; CHECK-NEXT:    [[DUMMY18_4:%.*]] = mul i32 [[DUMMY17_4]], [[DUMMY17_4]]
+; CHECK-NEXT:    [[DUMMY19_4:%.*]] = mul i32 [[DUMMY18_4]], [[DUMMY18_4]]
+; CHECK-NEXT:    [[DUMMY20_4:%.*]] = mul i32 [[DUMMY19_4]], [[DUMMY19_4]]
+; CHECK-NEXT:    [[DUMMY21_4:%.*]] = mul i32 [[DUMMY20_4]], [[DUMMY20_4]]
+; CHECK-NEXT:    [[DUMMY22_4:%.*]] = mul i32 [[DUMMY21_4]], [[DUMMY21_4]]
+; CHECK-NEXT:    [[DUMMY23_4:%.*]] = mul i32 [[DUMMY22_4]], [[DUMMY22_4]]
+; CHECK-NEXT:    [[DUMMY24_4:%.*]] = mul i32 [[DUMMY23_4]], [[DUMMY23_4]]
+; CHECK-NEXT:    [[DUMMY25_4:%.*]] = mul i32 [[DUMMY24_4]], [[DUMMY24_4]]
+; CHECK-NEXT:    [[DUMMY26_4:%.*]] = mul i32 [[DUMMY25_4]], [[DUMMY25_4]]
+; CHECK-NEXT:    [[DUMMY27_4:%.*]] = mul i32 [[DUMMY26_4]], [[DUMMY26_4]]
+; CHECK-NEXT:    [[DUMMY28_4:%.*]] = mul i32 [[DUMMY27_4]], [[DUMMY27_4]]
+; CHECK-NEXT:    [[DUMMY29_4:%.*]] = mul i32 [[DUMMY28_4]], [[DUMMY28_4]]
+; CHECK-NEXT:    [[DUMMY30_4:%.*]] = mul i32 [[DUMMY29_4]], [[DUMMY29_4]]
+; CHECK-NEXT:    [[DUMMY31_4:%.*]] = mul i32 [[DUMMY30_4]], [[DUMMY30_4]]
+; CHECK-NEXT:    [[DUMMY32_4:%.*]] = mul i32 [[DUMMY31_4]], [[DUMMY31_4]]
+; CHECK-NEXT:    [[DUMMY33_4:%.*]] = mul i32 [[DUMMY32_4]], [[DUMMY32_4]]
+; CHECK-NEXT:    [[DUMMY34_4:%.*]] = mul i32 [[DUMMY33_4]], [[DUMMY33_4]]
+; CHECK-NEXT:    [[DUMMY35_4:%.*]] = mul i32 [[DUMMY34_4]], [[DUMMY34_4]]
+; CHECK-NEXT:    [[DUMMY36_4:%.*]] = mul i32 [[DUMMY35_4]], [[DUMMY35_4]]
+; CHECK-NEXT:    [[DUMMY37_4:%.*]] = mul i32 [[DUMMY36_4]], [[DUMMY36_4]]
+; CHECK-NEXT:    [[DUMMY38_4:%.*]] = mul i32 [[DUMMY37_4]], [[DUMMY37_4]]
+; CHECK-NEXT:    [[DUMMY39_4:%.*]] = mul i32 [[DUMMY38_4]], [[DUMMY38_4]]
+; CHECK-NEXT:    [[DUMMY40_4:%.*]] = mul i32 [[DUMMY39_4]], [[DUMMY39_4]]
+; CHECK-NEXT:    [[DUMMY41_4:%.*]] = mul i32 [[DUMMY40_4]], [[DUMMY40_4]]
+; CHECK-NEXT:    [[DUMMY42_4:%.*]] = mul i32 [[DUMMY41_4]], [[DUMMY41_4]]
+; CHECK-NEXT:    [[DUMMY43_4:%.*]] = mul i32 [[DUMMY42_4]], [[DUMMY42_4]]
+; CHECK-NEXT:    [[DUMMY44_4:%.*]] = mul i32 [[DUMMY43_4]], [[DUMMY43_4]]
+; CHECK-NEXT:    [[DUMMY45_4:%.*]] = mul i32 [[DUMMY44_4]], [[DUMMY44_4]]
+; CHECK-NEXT:    [[DUMMY46_4:%.*]] = mul i32 [[DUMMY45_4]], [[DUMMY45_4]]
+; CHECK-NEXT:    [[DUMMY47_4:%.*]] = mul i32 [[DUMMY46_4]], [[DUMMY46_4]]
+; CHECK-NEXT:    [[DUMMY48_4:%.*]] = mul i32 [[DUMMY47_4]], [[DUMMY47_4]]
+; CHECK-NEXT:    [[DUMMY49_4:%.*]] = mul i32 [[DUMMY48_4]], [[DUMMY48_4]]
+; CHECK-NEXT:    [[DUMMY50_4:%.*]] = mul i32 [[DUMMY49_4]], [[DUMMY49_4]]
+; CHECK-NEXT:    [[SUM_NEXT_4:%.*]] = add nsw i32 [[DUMMY50_4]], [[SUM_NEXT_3]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_4:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 5
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_4]]
+; CHECK-NEXT:    [[VAL_5:%.*]] = load i32, ptr [[ARRAYIDX_5]], align 4
+; CHECK-NEXT:    [[DUMMY1_5:%.*]] = mul i32 [[VAL_5]], [[VAL_5]]
+; CHECK-NEXT:    [[DUMMY2_5:%.*]] = mul i32 [[DUMMY1_5]], [[DUMMY1_5]]
+; CHECK-NEXT:    [[DUMMY3_5:%.*]] = mul i32 [[DUMMY2_5]], [[DUMMY2_5]]
+; CHECK-NEXT:    [[DUMMY4_5:%.*]] = mul i32 [[DUMMY3_5]], [[DUMMY3_5]]
+; CHECK-NEXT:    [[DUMMY5_5:%.*]] = mul i32 [[DUMMY4_5]], [[DUMMY4_5]]
+; CHECK-NEXT:    [[DUMMY6_5:%.*]] = mul i32 [[DUMMY5_5]], [[DUMMY5_5]]
+; CHECK-NEXT:    [[DUMMY7_5:%.*]] = mul i32 [[DUMMY6_5]], [[DUMMY6_5]]
+; CHECK-NEXT:    [[DUMMY8_5:%.*]] = mul i32 [[DUMMY7_5]], [[DUMMY7_5]]
+; CHECK-NEXT:    [[DUMMY9_5:%.*]] = mul i32 [[DUMMY8_5]], [[DUMMY8_5]]
+; CHECK-NEXT:    [[DUMMY10_5:%.*]] = mul i32 [[DUMMY9_5]], [[DUMMY9_5]]
+; CHECK-NEXT:    [[DUMMY11_5:%.*]] = mul i32 [[DUMMY10_5]], [[DUMMY10_5]]
+; CHECK-NEXT:    [[DUMMY12_5:%.*]] = mul i32 [[DUMMY11_5]], [[DUMMY11_5]]
+; CHECK-NEXT:    [[DUMMY13_5:%.*]] = mul i32 [[DUMMY12_5]], [[DUMMY12_5]]
+; CHECK-NEXT:    [[DUMMY14_5:%.*]] = mul i32 [[DUMMY13_5]], [[DUMMY13_5]]
+; CHECK-NEXT:    [[DUMMY15_5:%.*]] = mul i32 [[DUMMY14_5]], [[DUMMY14_5]]
+; CHECK-NEXT:    [[DUMMY16_5:%.*]] = mul i32 [[DUMMY15_5]], [[DUMMY15_5]]
+; CHECK-NEXT:    [[DUMMY17_5:%.*]] = mul i32 [[DUMMY16_5]], [[DUMMY16_5]]
+; CHECK-NEXT:    [[DUMMY18_5:%.*]] = mul i32 [[DUMMY17_5]], [[DUMMY17_5]]
+; CHECK-NEXT:    [[DUMMY19_5:%.*]] = mul i32 [[DUMMY18_5]], [[DUMMY18_5]]
+; CHECK-NEXT:    [[DUMMY20_5:%.*]] = mul i32 [[DUMMY19_5]], [[DUMMY19_5]]
+; CHECK-NEXT:    [[DUMMY21_5:%.*]] = mul i32 [[DUMMY20_5]], [[DUMMY20_5]]
+; CHECK-NEXT:    [[DUMMY22_5:%.*]] = mul i32 [[DUMMY21_5]], [[DUMMY21_5]]
+; CHECK-NEXT:    [[DUMMY23_5:%.*]] = mul i32 [[DUMMY22_5]], [[DUMMY22_5]]
+; CHECK-NEXT:    [[DUMMY24_5:%.*]] = mul i32 [[DUMMY23_5]], [[DUMMY23_5]]
+; CHECK-NEXT:    [[DUMMY25_5:%.*]] = mul i32 [[DUMMY24_5]], [[DUMMY24_5]]
+; CHECK-NEXT:    [[DUMMY26_5:%.*]] = mul i32 [[DUMMY25_5]], [[DUMMY25_5]]
+; CHECK-NEXT:    [[DUMMY27_5:%.*]] = mul i32 [[DUMMY26_5]], [[DUMMY26_5]]
+; CHECK-NEXT:    [[DUMMY28_5:%.*]] = mul i32 [[DUMMY27_5]], [[DUMMY27_5]]
+; CHECK-NEXT:    [[DUMMY29_5:%.*]] = mul i32 [[DUMMY28_5]], [[DUMMY28_5]]
+; CHECK-NEXT:    [[DUMMY30_5:%.*]] = mul i32 [[DUMMY29_5]], [[DUMMY29_5]]
+; CHECK-NEXT:    [[DUMMY31_5:%.*]] = mul i32 [[DUMMY30_5]], [[DUMMY30_5]]
+; CHECK-NEXT:    [[DUMMY32_5:%.*]] = mul i32 [[DUMMY31_5]], [[DUMMY31_5]]
+; CHECK-NEXT:    [[DUMMY33_5:%.*]] = mul i32 [[DUMMY32_5]], [[DUMMY32_5]]
+; CHECK-NEXT:    [[DUMMY34_5:%.*]] = mul i32 [[DUMMY33_5]], [[DUMMY33_5]]
+; CHECK-NEXT:    [[DUMMY35_5:%.*]] = mul i32 [[DUMMY34_5]], [[DUMMY34_5]]
+; CHECK-NEXT:    [[DUMMY36_5:%.*]] = mul i32 [[DUMMY35_5]], [[DUMMY35_5]]
+; CHECK-NEXT:    [[DUMMY37_5:%.*]] = mul i32 [[DUMMY36_5]], [[DUMMY36_5]]
+; CHECK-NEXT:    [[DUMMY38_5:%.*]] = mul i32 [[DUMMY37_5]], [[DUMMY37_5]]
+; CHECK-NEXT:    [[DUMMY39_5:%.*]] = mul i32 [[DUMMY38_5]], [[DUMMY38_5]]
+; CHECK-NEXT:    [[DUMMY40_5:%.*]] = mul i32 [[DUMMY39_5]], [[DUMMY39_5]]
+; CHECK-NEXT:    [[DUMMY41_5:%.*]] = mul i32 [[DUMMY40_5]], [[DUMMY40_5]]
+; CHECK-NEXT:    [[DUMMY42_5:%.*]] = mul i32 [[DUMMY41_5]], [[DUMMY41_5]]
+; CHECK-NEXT:    [[DUMMY43_5:%.*]] = mul i32 [[DUMMY42_5]], [[DUMMY42_5]]
+; CHECK-NEXT:    [[DUMMY44_5:%.*]] = mul i32 [[DUMMY43_5]], [[DUMMY43_5]]
+; CHECK-NEXT:    [[DUMMY45_5:%.*]] = mul i32 [[DUMMY44_5]], [[DUMMY44_5]]
+; CHECK-NEXT:    [[DUMMY46_5:%.*]] = mul i32 [[DUMMY45_5]], [[DUMMY45_5]]
+; CHECK-NEXT:    [[DUMMY47_5:%.*]] = mul i32 [[DUMMY46_5]], [[DUMMY46_5]]
+; CHECK-NEXT:    [[DUMMY48_5:%.*]] = mul i32 [[DUMMY47_5]], [[DUMMY47_5]]
+; CHECK-NEXT:    [[DUMMY49_5:%.*]] = mul i32 [[DUMMY48_5]], [[DUMMY48_5]]
+; CHECK-NEXT:    [[DUMMY50_5:%.*]] = mul i32 [[DUMMY49_5]], [[DUMMY49_5]]
+; CHECK-NEXT:    [[SUM_NEXT_5:%.*]] = add nsw i32 [[DUMMY50_5]], [[SUM_NEXT_4]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_5:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 6
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_5]]
+; CHECK-NEXT:    [[VAL_6:%.*]] = load i32, ptr [[ARRAYIDX_6]], align 4
+; CHECK-NEXT:    [[DUMMY1_6:%.*]] = mul i32 [[VAL_6]], [[VAL_6]]
+; CHECK-NEXT:    [[DUMMY2_6:%.*]] = mul i32 [[DUMMY1_6]], [[DUMMY1_6]]
+; CHECK-NEXT:    [[DUMMY3_6:%.*]] = mul i32 [[DUMMY2_6]], [[DUMMY2_6]]
+; CHECK-NEXT:    [[DUMMY4_6:%.*]] = mul i32 [[DUMMY3_6]], [[DUMMY3_6]]
+; CHECK-NEXT:    [[DUMMY5_6:%.*]] = mul i32 [[DUMMY4_6]], [[DUMMY4_6]]
+; CHECK-NEXT:    [[DUMMY6_6:%.*]] = mul i32 [[DUMMY5_6]], [[DUMMY5_6]]
+; CHECK-NEXT:    [[DUMMY7_6:%.*]] = mul i32 [[DUMMY6_6]], [[DUMMY6_6]]
+; CHECK-NEXT:    [[DUMMY8_6:%.*]] = mul i32 [[DUMMY7_6]], [[DUMMY7_6]]
+; CHECK-NEXT:    [[DUMMY9_6:%.*]] = mul i32 [[DUMMY8_6]], [[DUMMY8_6]]
+; CHECK-NEXT:    [[DUMMY10_6:%.*]] = mul i32 [[DUMMY9_6]], [[DUMMY9_6]]
+; CHECK-NEXT:    [[DUMMY11_6:%.*]] = mul i32 [[DUMMY10_6]], [[DUMMY10_6]]
+; CHECK-NEXT:    [[DUMMY12_6:%.*]] = mul i32 [[DUMMY11_6]], [[DUMMY11_6]]
+; CHECK-NEXT:    [[DUMMY13_6:%.*]] = mul i32 [[DUMMY12_6]], [[DUMMY12_6]]
+; CHECK-NEXT:    [[DUMMY14_6:%.*]] = mul i32 [[DUMMY13_6]], [[DUMMY13_6]]
+; CHECK-NEXT:    [[DUMMY15_6:%.*]] = mul i32 [[DUMMY14_6]], [[DUMMY14_6]]
+; CHECK-NEXT:    [[DUMMY16_6:%.*]] = mul i32 [[DUMMY15_6]], [[DUMMY15_6]]
+; CHECK-NEXT:    [[DUMMY17_6:%.*]] = mul i32 [[DUMMY16_6]], [[DUMMY16_6]]
+; CHECK-NEXT:    [[DUMMY18_6:%.*]] = mul i32 [[DUMMY17_6]], [[DUMMY17_6]]
+; CHECK-NEXT:    [[DUMMY19_6:%.*]] = mul i32 [[DUMMY18_6]], [[DUMMY18_6]]
+; CHECK-NEXT:    [[DUMMY20_6:%.*]] = mul i32 [[DUMMY19_6]], [[DUMMY19_6]]
+; CHECK-NEXT:    [[DUMMY21_6:%.*]] = mul i32 [[DUMMY20_6]], [[DUMMY20_6]]
+; CHECK-NEXT:    [[DUMMY22_6:%.*]] = mul i32 [[DUMMY21_6]], [[DUMMY21_6]]
+; CHECK-NEXT:    [[DUMMY23_6:%.*]] = mul i32 [[DUMMY22_6]], [[DUMMY22_6]]
+; CHECK-NEXT:    [[DUMMY24_6:%.*]] = mul i32 [[DUMMY23_6]], [[DUMMY23_6]]
+; CHECK-NEXT:    [[DUMMY25_6:%.*]] = mul i32 [[DUMMY24_6]], [[DUMMY24_6]]
+; CHECK-NEXT:    [[DUMMY26_6:%.*]] = mul i32 [[DUMMY25_6]], [[DUMMY25_6]]
+; CHECK-NEXT:    [[DUMMY27_6:%.*]] = mul i32 [[DUMMY26_6]], [[DUMMY26_6]]
+; CHECK-NEXT:    [[DUMMY28_6:%.*]] = mul i32 [[DUMMY27_6]], [[DUMMY27_6]]
+; CHECK-NEXT:    [[DUMMY29_6:%.*]] = mul i32 [[DUMMY28_6]], [[DUMMY28_6]]
+; CHECK-NEXT:    [[DUMMY30_6:%.*]] = mul i32 [[DUMMY29_6]], [[DUMMY29_6]]
+; CHECK-NEXT:    [[DUMMY31_6:%.*]] = mul i32 [[DUMMY30_6]], [[DUMMY30_6]]
+; CHECK-NEXT:    [[DUMMY32_6:%.*]] = mul i32 [[DUMMY31_6]], [[DUMMY31_6]]
+; CHECK-NEXT:    [[DUMMY33_6:%.*]] = mul i32 [[DUMMY32_6]], [[DUMMY32_6]]
+; CHECK-NEXT:    [[DUMMY34_6:%.*]] = mul i32 [[DUMMY33_6]], [[DUMMY33_6]]
+; CHECK-NEXT:    [[DUMMY35_6:%.*]] = mul i32 [[DUMMY34_6]], [[DUMMY34_6]]
+; CHECK-NEXT:    [[DUMMY36_6:%.*]] = mul i32 [[DUMMY35_6]], [[DUMMY35_6]]
+; CHECK-NEXT:    [[DUMMY37_6:%.*]] = mul i32 [[DUMMY36_6]], [[DUMMY36_6]]
+; CHECK-NEXT:    [[DUMMY38_6:%.*]] = mul i32 [[DUMMY37_6]], [[DUMMY37_6]]
+; CHECK-NEXT:    [[DUMMY39_6:%.*]] = mul i32 [[DUMMY38_6]], [[DUMMY38_6]]
+; CHECK-NEXT:    [[DUMMY40_6:%.*]] = mul i32 [[DUMMY39_6]], [[DUMMY39_6]]
+; CHECK-NEXT:    [[DUMMY41_6:%.*]] = mul i32 [[DUMMY40_6]], [[DUMMY40_6]]
+; CHECK-NEXT:    [[DUMMY42_6:%.*]] = mul i32 [[DUMMY41_6]], [[DUMMY41_6]]
+; CHECK-NEXT:    [[DUMMY43_6:%.*]] = mul i32 [[DUMMY42_6]], [[DUMMY42_6]]
+; CHECK-NEXT:    [[DUMMY44_6:%.*]] = mul i32 [[DUMMY43_6]], [[DUMMY43_6]]
+; CHECK-NEXT:    [[DUMMY45_6:%.*]] = mul i32 [[DUMMY44_6]], [[DUMMY44_6]]
+; CHECK-NEXT:    [[DUMMY46_6:%.*]] = mul i32 [[DUMMY45_6]], [[DUMMY45_6]]
+; CHECK-NEXT:    [[DUMMY47_6:%.*]] = mul i32 [[DUMMY46_6]], [[DUMMY46_6]]
+; CHECK-NEXT:    [[DUMMY48_6:%.*]] = mul i32 [[DUMMY47_6]], [[DUMMY47_6]]
+; CHECK-NEXT:    [[DUMMY49_6:%.*]] = mul i32 [[DUMMY48_6]], [[DUMMY48_6]]
+; CHECK-NEXT:    [[DUMMY50_6:%.*]] = mul i32 [[DUMMY49_6]], [[DUMMY49_6]]
+; CHECK-NEXT:    [[SUM_NEXT_6:%.*]] = add nsw i32 [[DUMMY50_6]], [[SUM_NEXT_5]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_6:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 7
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_6]]
+; CHECK-NEXT:    [[VAL_7:%.*]] = load i32, ptr [[ARRAYIDX_7]], align 4
+; CHECK-NEXT:    [[DUMMY1_7:%.*]] = mul i32 [[VAL_7]], [[VAL_7]]
+; CHECK-NEXT:    [[DUMMY2_7:%.*]] = mul i32 [[DUMMY1_7]], [[DUMMY1_7]]
+; CHECK-NEXT:    [[DUMMY3_7:%.*]] = mul i32 [[DUMMY2_7]], [[DUMMY2_7]]
+; CHECK-NEXT:    [[DUMMY4_7:%.*]] = mul i32 [[DUMMY3_7]], [[DUMMY3_7]]
+; CHECK-NEXT:    [[DUMMY5_7:%.*]] = mul i32 [[DUMMY4_7]], [[DUMMY4_7]]
+; CHECK-NEXT:    [[DUMMY6_7:%.*]] = mul i32 [[DUMMY5_7]], [[DUMMY5_7]]
+; CHECK-NEXT:    [[DUMMY7_7:%.*]] = mul i32 [[DUMMY6_7]], [[DUMMY6_7]]
+; CHECK-NEXT:    [[DUMMY8_7:%.*]] = mul i32 [[DUMMY7_7]], [[DUMMY7_7]]
+; CHECK-NEXT:    [[DUMMY9_7:%.*]] = mul i32 [[DUMMY8_7]], [[DUMMY8_7]]
+; CHECK-NEXT:    [[DUMMY10_7:%.*]] = mul i32 [[DUMMY9_7]], [[DUMMY9_7]]
+; CHECK-NEXT:    [[DUMMY11_7:%.*]] = mul i32 [[DUMMY10_7]], [[DUMMY10_7]]
+; CHECK-NEXT:    [[DUMMY12_7:%.*]] = mul i32 [[DUMMY11_7]], [[DUMMY11_7]]
+; CHECK-NEXT:    [[DUMMY13_7:%.*]] = mul i32 [[DUMMY12_7]], [[DUMMY12_7]]
+; CHECK-NEXT:    [[DUMMY14_7:%.*]] = mul i32 [[DUMMY13_7]], [[DUMMY13_7]]
+; CHECK-NEXT:    [[DUMMY15_7:%.*]] = mul i32 [[DUMMY14_7]], [[DUMMY14_7]]
+; CHECK-NEXT:    [[DUMMY16_7:%.*]] = mul i32 [[DUMMY15_7]], [[DUMMY15_7]]
+; CHECK-NEXT:    [[DUMMY17_7:%.*]] = mul i32 [[DUMMY16_7]], [[DUMMY16_7]]
+; CHECK-NEXT:    [[DUMMY18_7:%.*]] = mul i32 [[DUMMY17_7]], [[DUMMY17_7]]
+; CHECK-NEXT:    [[DUMMY19_7:%.*]] = mul i32 [[DUMMY18_7]], [[DUMMY18_7]]
+; CHECK-NEXT:    [[DUMMY20_7:%.*]] = mul i32 [[DUMMY19_7]], [[DUMMY19_7]]
+; CHECK-NEXT:    [[DUMMY21_7:%.*]] = mul i32 [[DUMMY20_7]], [[DUMMY20_7]]
+; CHECK-NEXT:    [[DUMMY22_7:%.*]] = mul i32 [[DUMMY21_7]], [[DUMMY21_7]]
+; CHECK-NEXT:    [[DUMMY23_7:%.*]] = mul i32 [[DUMMY22_7]], [[DUMMY22_7]]
+; CHECK-NEXT:    [[DUMMY24_7:%.*]] = mul i32 [[DUMMY23_7]], [[DUMMY23_7]]
+; CHECK-NEXT:    [[DUMMY25_7:%.*]] = mul i32 [[DUMMY24_7]], [[DUMMY24_7]]
+; CHECK-NEXT:    [[DUMMY26_7:%.*]] = mul i32 [[DUMMY25_7]], [[DUMMY25_7]]
+; CHECK-NEXT:    [[DUMMY27_7:%.*]] = mul i32 [[DUMMY26_7]], [[DUMMY26_7]]
+; CHECK-NEXT:    [[DUMMY28_7:%.*]] = mul i32 [[DUMMY27_7]], [[DUMMY27_7]]
+; CHECK-NEXT:    [[DUMMY29_7:%.*]] = mul i32 [[DUMMY28_7]], [[DUMMY28_7]]
+; CHECK-NEXT:    [[DUMMY30_7:%.*]] = mul i32 [[DUMMY29_7]], [[DUMMY29_7]]
+; CHECK-NEXT:    [[DUMMY31_7:%.*]] = mul i32 [[DUMMY30_7]], [[DUMMY30_7]]
+; CHECK-NEXT:    [[DUMMY32_7:%.*]] = mul i32 [[DUMMY31_7]], [[DUMMY31_7]]
+; CHECK-NEXT:    [[DUMMY33_7:%.*]] = mul i32 [[DUMMY32_7]], [[DUMMY32_7]]
+; CHECK-NEXT:    [[DUMMY34_7:%.*]] = mul i32 [[DUMMY33_7]], [[DUMMY33_7]]
+; CHECK-NEXT:    [[DUMMY35_7:%.*]] = mul i32 [[DUMMY34_7]], [[DUMMY34_7]]
+; CHECK-NEXT:    [[DUMMY36_7:%.*]] = mul i32 [[DUMMY35_7]], [[DUMMY35_7]]
+; CHECK-NEXT:    [[DUMMY37_7:%.*]] = mul i32 [[DUMMY36_7]], [[DUMMY36_7]]
+; CHECK-NEXT:    [[DUMMY38_7:%.*]] = mul i32 [[DUMMY37_7]], [[DUMMY37_7]]
+; CHECK-NEXT:    [[DUMMY39_7:%.*]] = mul i32 [[DUMMY38_7]], [[DUMMY38_7]]
+; CHECK-NEXT:    [[DUMMY40_7:%.*]] = mul i32 [[DUMMY39_7]], [[DUMMY39_7]]
+; CHECK-NEXT:    [[DUMMY41_7:%.*]] = mul i32 [[DUMMY40_7]], [[DUMMY40_7]]
+; CHECK-NEXT:    [[DUMMY42_7:%.*]] = mul i32 [[DUMMY41_7]], [[DUMMY41_7]]
+; CHECK-NEXT:    [[DUMMY43_7:%.*]] = mul i32 [[DUMMY42_7]], [[DUMMY42_7]]
+; CHECK-NEXT:    [[DUMMY44_7:%.*]] = mul i32 [[DUMMY43_7]], [[DUMMY43_7]]
+; CHECK-NEXT:    [[DUMMY45_7:%.*]] = mul i32 [[DUMMY44_7]], [[DUMMY44_7]]
+; CHECK-NEXT:    [[DUMMY46_7:%.*]] = mul i32 [[DUMMY45_7]], [[DUMMY45_7]]
+; CHECK-NEXT:    [[DUMMY47_7:%.*]] = mul i32 [[DUMMY46_7]], [[DUMMY46_7]]
+; CHECK-NEXT:    [[DUMMY48_7:%.*]] = mul i32 [[DUMMY47_7]], [[DUMMY47_7]]
+; CHECK-NEXT:    [[DUMMY49_7:%.*]] = mul i32 [[DUMMY48_7]], [[DUMMY48_7]]
+; CHECK-NEXT:    [[DUMMY50_7:%.*]] = mul i32 [[DUMMY49_7]], [[DUMMY49_7]]
+; CHECK-NEXT:    [[SUM_NEXT_7]] = add nsw i32 [[DUMMY50_7]], [[SUM_NEXT_6]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_7]] = add nuw nsw i64 [[INDVARS_IV]], 8
+; CHECK-NEXT:    [[NITER_NEXT_7]] = add i64 [[NITER]], 8
+; CHECK-NEXT:    [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]]
+; CHECK-NEXT:    br i1 [[NITER_NCMP_7]], label [[FOR_COND_CLEANUP_UNR_LCSSA_LOOPEXIT:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.cond.cleanup.unr-lcssa.loopexit:
-; CHECK-NEXT:    [[SUM_NEXT_LCSSA_PH_PH:%.*]] = phi i32 [ [[SUM_NEXT_1]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_UNR_PH:%.*]] = phi i32 [ [[SUM_NEXT_1]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_NEXT_LCSSA_PH_PH:%.*]] = phi i32 [ [[SUM_NEXT_7]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_7]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_UNR_PH:%.*]] = phi i32 [ [[SUM_NEXT_7]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_UNR_LCSSA]]
 ; CHECK:       for.cond.cleanup.unr-lcssa:
 ; CHECK-NEXT:    [[SUM_NEXT_LCSSA_PH:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[SUM_NEXT_LCSSA_PH_PH]], [[FOR_COND_CLEANUP_UNR_LCSSA_LOOPEXIT]] ]
@@ -302,7 +1010,10 @@ define i32 @test2(ptr %ary, i64 %n) "target-cpu"="znver3" {
 ; CHECK:       for.body.epil.preheader:
 ; CHECK-NEXT:    br label [[FOR_BODY_EPIL:%.*]]
 ; CHECK:       for.body.epil:
-; CHECK-NEXT:    [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_UNR]]
+; CHECK-NEXT:    [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_UNR]], [[FOR_BODY_EPIL_PREHEADER]] ], [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL]] ]
+; CHECK-NEXT:    [[SUM_EPIL:%.*]] = phi i32 [ [[SUM_UNR]], [[FOR_BODY_EPIL_PREHEADER]] ], [ [[SUM_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL]] ]
+; CHECK-NEXT:    [[EPIL_ITER:%.*]] = phi i64 [ 0, [[FOR_BODY_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[FOR_BODY_EPIL]] ]
+; CHECK-NEXT:    [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_EPIL]]
 ; CHECK-NEXT:    [[VAL_EPIL:%.*]] = load i32, ptr [[ARRAYIDX_EPIL]], align 4
 ; CHECK-NEXT:    [[DUMMY1_EPIL:%.*]] = mul i32 [[VAL_EPIL]], [[VAL_EPIL]]
 ; CHECK-NEXT:    [[DUMMY2_EPIL:%.*]] = mul i32 [[DUMMY1_EPIL]], [[DUMMY1_EPIL]]
@@ -354,10 +1065,17 @@ define i32 @test2(ptr %ary, i64 %n) "target-cpu"="znver3" {
 ; CHECK-NEXT:    [[DUMMY48_EPIL:%.*]] = mul i32 [[DUMMY47_EPIL]], [[DUMMY47_EPIL]]
 ; CHECK-NEXT:    [[DUMMY49_EPIL:%.*]] = mul i32 [[DUMMY48_EPIL]], [[DUMMY48_EPIL]]
 ; CHECK-NEXT:    [[DUMMY50_EPIL:%.*]] = mul i32 [[DUMMY49_EPIL]], [[DUMMY49_EPIL]]
-; CHECK-NEXT:    [[SUM_NEXT_EPIL:%.*]] = add nsw i32 [[DUMMY50_EPIL]], [[SUM_UNR]]
+; CHECK-NEXT:    [[SUM_NEXT_EPIL]] = add nsw i32 [[DUMMY50_EPIL]], [[SUM_EPIL]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_EPIL]] = add nuw nsw i64 [[INDVARS_IV_EPIL]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT_EPIL:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_EPIL]], [[N]]
+; CHECK-NEXT:    [[EPIL_ITER_NEXT]] = add i64 [[EPIL_ITER]], 1
+; CHECK-NEXT:    [[EPIL_ITER_CMP:%.*]] = icmp ne i64 [[EPIL_ITER_NEXT]], [[XTRAITER]]
+; CHECK-NEXT:    br i1 [[EPIL_ITER_CMP]], label [[FOR_BODY_EPIL]], label [[FOR_COND_CLEANUP_EPILOG_LCSSA:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       for.cond.cleanup.epilog-lcssa:
+; CHECK-NEXT:    [[SUM_NEXT_LCSSA_PH1:%.*]] = phi i32 [ [[SUM_NEXT_EPIL]], [[FOR_BODY_EPIL]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT_LCSSA_PH]], [[FOR_COND_CLEANUP_UNR_LCSSA]] ], [ [[SUM_NEXT_EPIL]], [[FOR_BODY_EPIL]] ]
+; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT_LCSSA_PH]], [[FOR_COND_CLEANUP_UNR_LCSSA]] ], [ [[SUM_NEXT_LCSSA_PH1]], [[FOR_COND_CLEANUP_EPILOG_LCSSA]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_NEXT_LCSSA]]
 ;
 entry:
-- 
GitLab


From ba2e4fe4e7f79e49fcac54ea20f5b899dc687cfc Mon Sep 17 00:00:00 2001
From: Arseniy Zaostrovnykh <necto.ne@gmail.com>
Date: Thu, 16 May 2024 12:14:53 +0200
Subject: [PATCH 0993/1206] [clang] Fix CXXNewExpr end source location for 'new
 struct S' (#92266)

Currently, `new struct S` fails to set any valid end source location
because the token corresponding to `S` is consumed in
`ParseClassSpecifier` and is not accessible in the
`ParseDeclarationSpecifiers` that normally sets the end source location.

Fixes #35300
---
 .../test/clang-tidy/checkers/modernize/make-unique.cpp     | 7 ++-----
 clang/lib/Parse/ParseDeclCXX.cpp                           | 1 +
 clang/test/AST/ast-dump-expr.cpp                           | 7 +++++++
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/make-unique.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/make-unique.cpp
index 7934c6e93ffb..fe512a8f3bf3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/make-unique.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/make-unique.cpp
@@ -606,11 +606,8 @@ void invoke_template() {
   template_fun(foo);
 }
 
-void no_fix_for_invalid_new_loc() {
-  // FIXME: Although the code is valid, the end location of `new struct Base` is
-  // invalid. Correct it once https://bugs.llvm.org/show_bug.cgi?id=35952 is
-  // fixed.
+void fix_for_c_style_struct() {
   auto T = std::unique_ptr<Base>(new struct Base);
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: use std::make_unique instead
-  // CHECK-FIXES: auto T = std::unique_ptr<Base>(new struct Base);
+  // CHECK-FIXES: auto T = std::make_unique<Base>();
 }
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 65ddebca49bc..32c4e923243a 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -1883,6 +1883,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   if (Tok.is(tok::identifier)) {
     Name = Tok.getIdentifierInfo();
     NameLoc = ConsumeToken();
+    DS.SetRangeEnd(NameLoc);
 
     if (Tok.is(tok::less) && getLangOpts().CPlusPlus) {
       // The name was supposed to refer to a template, but didn't.
diff --git a/clang/test/AST/ast-dump-expr.cpp b/clang/test/AST/ast-dump-expr.cpp
index 4df5ba4276ab..604868103dab 100644
--- a/clang/test/AST/ast-dump-expr.cpp
+++ b/clang/test/AST/ast-dump-expr.cpp
@@ -583,3 +583,10 @@ void NonADLCall3() {
   f(x);
 }
 } // namespace test_adl_call_three
+
+namespace GH35300 {
+struct Sock {};
+void leakNewFn() { new struct Sock; }
+// CHECK: CXXNewExpr {{.*}} <col:20, col:31> 'struct Sock *'
+}
+
-- 
GitLab


From 239f8b9eb0725b4c6ff8342717b6abbfc9a7915a Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Thu, 16 May 2024 12:44:47 +0200
Subject: [PATCH 0994/1206] [AST] RecursiveASTVisitor: Don't traverse the alias
 deduction guides in the default mode. (#91454)

By default (`shouldVisitImplicitCode()` returns `false`), RAV should not
traverse AST nodes that are not spelled in the source code. Deduction
guides for alias templates are always synthesized, so they should not be
traversed.

This is usually done by checking the implicit bit of the Decl. However,
this doesn't work deduction guides that are synthesized from explicit
user-defined deduction guides, as we must maintain the explicit bit to
ensure correct overload resolution.
---
 clang/include/clang/AST/RecursiveASTVisitor.h | 28 ++++--
 .../DeductionGuide.cpp                        | 85 +++++++++++++++++++
 2 files changed, 106 insertions(+), 7 deletions(-)
 create mode 100644 clang/unittests/Tooling/RecursiveASTVisitorTests/DeductionGuide.cpp

diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 782f60844506..f5cefedb07e0 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -736,13 +736,27 @@ bool RecursiveASTVisitor<Derived>::TraverseDecl(Decl *D) {
 
   // As a syntax visitor, by default we want to ignore declarations for
   // implicit declarations (ones not typed explicitly by the user).
-  if (!getDerived().shouldVisitImplicitCode() && D->isImplicit()) {
-    // For an implicit template type parameter, its type constraints are not
-    // implicit and are not represented anywhere else. We still need to visit
-    // them.
-    if (auto *TTPD = dyn_cast<TemplateTypeParmDecl>(D))
-      return TraverseTemplateTypeParamDeclConstraints(TTPD);
-    return true;
+  if (!getDerived().shouldVisitImplicitCode()) {
+    if (D->isImplicit()) {
+      // For an implicit template type parameter, its type constraints are not
+      // implicit and are not represented anywhere else. We still need to visit
+      // them.
+      if (auto *TTPD = dyn_cast<TemplateTypeParmDecl>(D))
+        return TraverseTemplateTypeParamDeclConstraints(TTPD);
+      return true;
+    }
+
+    // Deduction guides for alias templates are always synthesized, so they
+    // should not be traversed unless shouldVisitImplicitCode() returns true.
+    //
+    // It's important to note that checking the implicit bit is not efficient
+    // for the alias case. For deduction guides synthesized from explicit
+    // user-defined deduction guides, we must maintain the explicit bit to
+    // ensure correct overload resolution.
+    if (auto *FTD = dyn_cast<FunctionTemplateDecl>(D))
+      if (llvm::isa_and_present<TypeAliasTemplateDecl>(
+              FTD->getDeclName().getCXXDeductionGuideTemplate()))
+        return true;
   }
 
   switch (D->getKind()) {
diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/DeductionGuide.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/DeductionGuide.cpp
new file mode 100644
index 000000000000..274f275ea66a
--- /dev/null
+++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/DeductionGuide.cpp
@@ -0,0 +1,85 @@
+//===- unittest/Tooling/RecursiveASTVisitorTests/DeductionGuide.cpp -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestVisitor.h"
+#include <string>
+
+using namespace clang;
+
+namespace {
+
+class DeductionGuideVisitor
+    : public ExpectedLocationVisitor<DeductionGuideVisitor> {
+public:
+  DeductionGuideVisitor(bool ShouldVisitImplicitCode)
+      : ShouldVisitImplicitCode(ShouldVisitImplicitCode) {}
+  bool VisitCXXDeductionGuideDecl(CXXDeductionGuideDecl *D) {
+    std::string Storage;
+    llvm::raw_string_ostream Stream(Storage);
+    D->print(Stream);
+    Match(Stream.str(), D->getLocation());
+    return true;
+  }
+
+  bool shouldVisitTemplateInstantiations() const { return false; }
+
+  bool shouldVisitImplicitCode() const { return ShouldVisitImplicitCode; }
+  bool ShouldVisitImplicitCode;
+};
+
+TEST(RecursiveASTVisitor, DeductionGuideNonImplicitMode) {
+  DeductionGuideVisitor Visitor(/*ShouldVisitImplicitCode*/ false);
+  // Verify that the synthezied deduction guide for alias is not visited in
+  // RAV's implicit mode.
+  Visitor.ExpectMatch("Foo(T) -> Foo<int>", 11, 1);
+  Visitor.DisallowMatch("Bar(type-parameter-0-0) -> Foo<int>", 14, 1);
+  EXPECT_TRUE(Visitor.runOver(
+      R"cpp(
+template <typename T>
+concept False = true;
+
+template <typename T> 
+struct Foo { 
+  Foo(T);
+};
+
+template<typename T> requires False<T>
+Foo(T) -> Foo<int>;
+
+template <typename U>
+using Bar = Foo<U>;
+Bar s(1); 
+   )cpp",
+      DeductionGuideVisitor::Lang_CXX2a));
+}
+
+TEST(RecursiveASTVisitor, DeductionGuideImplicitMode) {
+  DeductionGuideVisitor Visitor(/*ShouldVisitImplicitCode*/ true);
+  Visitor.ExpectMatch("Foo(T) -> Foo<int>", 11, 1);
+  Visitor.ExpectMatch("Bar(type-parameter-0-0) -> Foo<int>", 14, 1);
+  EXPECT_TRUE(Visitor.runOver(
+      R"cpp(
+template <typename T>
+concept False = true;
+
+template <typename T> 
+struct Foo { 
+  Foo(T);
+};
+
+template<typename T> requires False<T>
+Foo(T) -> Foo<int>;
+
+template <typename U>
+using Bar = Foo<U>;
+Bar s(1); 
+   )cpp",
+      DeductionGuideVisitor::Lang_CXX2a));
+}
+
+} // end anonymous namespace
-- 
GitLab


From a9605730a481707623358d3b12220f05cfdc50a8 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Thu, 16 May 2024 13:01:18 +0200
Subject: [PATCH 0995/1206] [clang] CTAD: implement the missing IsDeducible
 constraint for alias templates  (#89358)

Fixes https://github.com/llvm/llvm-project/issues/85192
Fixes https://github.com/llvm/llvm-project/issues/84492

This patch implements the "IsDeducible" constraint where the template
arguments of the alias template can be deduced from the returned type of
the synthesized deduction guide, per C++ [over.match.class.deduct]p4. In
the implementation, we perform the deduction directly, which is more
efficient than the way specified in the standard.

Also update relevant CTAD tests which were incorrectly compiled due to
the missing constraint.
---
 clang/include/clang/Basic/TokenKinds.def     |  3 +
 clang/include/clang/Sema/Sema.h              |  9 ++
 clang/lib/Basic/TypeTraits.cpp               | 10 +++
 clang/lib/Sema/SemaExprCXX.cpp               | 10 ++-
 clang/lib/Sema/SemaTemplate.cpp              | 88 ++++++++++++++++++--
 clang/lib/Sema/SemaTemplateDeduction.cpp     | 86 +++++++++++++++++++
 clang/test/AST/ast-dump-ctad-alias.cpp       | 20 +++--
 clang/test/SemaCXX/cxx20-ctad-type-alias.cpp | 27 ++++--
 clang/test/SemaTemplate/deduction-guide.cpp  | 16 ++--
 clang/www/cxx_status.html                    |  4 +-
 10 files changed, 241 insertions(+), 32 deletions(-)

diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 56c4b17f769d..b5a0e9df9f7a 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -538,6 +538,9 @@ TYPE_TRAIT_1(__can_pass_in_regs, CanPassInRegs, KEYCXX)
 TYPE_TRAIT_2(__reference_binds_to_temporary, ReferenceBindsToTemporary, KEYCXX)
 TYPE_TRAIT_2(__reference_constructs_from_temporary, ReferenceConstructsFromTemporary, KEYCXX)
 TYPE_TRAIT_2(__reference_converts_from_temporary, ReferenceConvertsFromTemporary, KEYCXX)
+// IsDeducible is only used internally by clang for CTAD implementation and
+// is not exposed to users.
+TYPE_TRAIT_2(/*EmptySpellingName*/, IsDeducible, KEYCXX)
 
 // Embarcadero Expression Traits
 EXPRESSION_TRAIT(__is_lvalue_expr, IsLValueExpr, KEYCXX)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 6a414aa57f32..66d5e2d4a4ad 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -9492,6 +9492,15 @@ public:
                           ArrayRef<TemplateArgument> TemplateArgs,
                           sema::TemplateDeductionInfo &Info);
 
+  /// Deduce the template arguments of the given template from \p FromType.
+  /// Used to implement the IsDeducible constraint for alias CTAD per C++
+  /// [over.match.class.deduct]p4.
+  ///
+  /// It only supports class or type alias templates.
+  TemplateDeductionResult
+  DeduceTemplateArgumentsFromType(TemplateDecl *TD, QualType FromType,
+                                  sema::TemplateDeductionInfo &Info);
+
   TemplateDeductionResult DeduceTemplateArguments(
       TemplateParameterList *TemplateParams, ArrayRef<TemplateArgument> Ps,
       ArrayRef<TemplateArgument> As, sema::TemplateDeductionInfo &Info,
diff --git a/clang/lib/Basic/TypeTraits.cpp b/clang/lib/Basic/TypeTraits.cpp
index 4dbf678dc395..8d6794223cca 100644
--- a/clang/lib/Basic/TypeTraits.cpp
+++ b/clang/lib/Basic/TypeTraits.cpp
@@ -13,6 +13,7 @@
 #include "clang/Basic/TypeTraits.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <cassert>
+#include <cstring>
 using namespace clang;
 
 static constexpr const char *TypeTraitNames[] = {
@@ -81,6 +82,15 @@ const char *clang::getTraitName(UnaryExprOrTypeTrait T) {
 
 const char *clang::getTraitSpelling(TypeTrait T) {
   assert(T <= TT_Last && "invalid enum value!");
+  if (T == BTT_IsDeducible) {
+    // The __is_deducible is an internal-only type trait. To hide it from
+    // external users, we define it with an empty spelling name, preventing the
+    // clang parser from recognizing its token kind.
+    // However, other components such as the AST dump still require the real
+    // type trait name. Therefore, we return the real name when needed.
+    assert(std::strlen(TypeTraitSpellings[T]) == 0);
+    return "__is_deducible";
+  }
   return TypeTraitSpellings[T];
 }
 
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 34e12078a8c9..e4601f7d6c47 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -6143,7 +6143,15 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, const TypeSourceI
 
     return Self.IsPointerInterconvertibleBaseOf(Lhs, Rhs);
   }
-    default: llvm_unreachable("not a BTT");
+  case BTT_IsDeducible: {
+    const auto *TSTToBeDeduced = cast<DeducedTemplateSpecializationType>(LhsT);
+    sema::TemplateDeductionInfo Info(KeyLoc);
+    return Self.DeduceTemplateArgumentsFromType(
+               TSTToBeDeduced->getTemplateName().getAsTemplateDecl(), RhsT,
+               Info) == TemplateDeductionResult::Success;
+  }
+  default:
+    llvm_unreachable("not a BTT");
   }
   llvm_unreachable("Unknown type trait or not implemented");
 }
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index c7aac068e264..a5350ceb59cb 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -2782,17 +2782,24 @@ NamedDecl *transformTemplateParameter(Sema &SemaRef, DeclContext *DC,
   llvm_unreachable("Unhandled template parameter types");
 }
 
-// Transform the require-clause of F if any.
+// Build the associated constraints for the alias deduction guides.
+// C++ [over.match.class.deduct]p3.3:
+//   The associated constraints ([temp.constr.decl]) are the conjunction of the
+//   associated constraints of g and a constraint that is satisfied if and only
+//   if the arguments of A are deducible (see below) from the return type.
+//
 // The return result is expected to be the require-clause for the synthesized
 // alias deduction guide.
-Expr *transformRequireClause(Sema &SemaRef, FunctionTemplateDecl *F,
-                             TypeAliasTemplateDecl *AliasTemplate,
-                             ArrayRef<DeducedTemplateArgument> DeduceResults) {
+Expr *
+buildAssociatedConstraints(Sema &SemaRef, FunctionTemplateDecl *F,
+                           TypeAliasTemplateDecl *AliasTemplate,
+                           ArrayRef<DeducedTemplateArgument> DeduceResults,
+                           Expr *IsDeducible) {
   Expr *RC = F->getTemplateParameters()->getRequiresClause();
   if (!RC)
-    return nullptr;
+    return IsDeducible;
 
-  auto &Context = SemaRef.Context;
+  ASTContext &Context = SemaRef.Context;
   LocalInstantiationScope Scope(SemaRef);
 
   // In the clang AST, constraint nodes are deliberately not instantiated unless
@@ -2903,7 +2910,68 @@ Expr *transformRequireClause(Sema &SemaRef, FunctionTemplateDecl *F,
   ExprResult E = SemaRef.SubstExpr(RC, ArgsForBuildingRC);
   if (E.isInvalid())
     return nullptr;
-  return E.getAs<Expr>();
+
+  auto Conjunction =
+      SemaRef.BuildBinOp(SemaRef.getCurScope(), SourceLocation{},
+                         BinaryOperatorKind::BO_LAnd, E.get(), IsDeducible);
+  if (Conjunction.isInvalid())
+    return nullptr;
+  return Conjunction.getAs<Expr>();
+}
+// Build the is_deducible constraint for the alias deduction guides.
+// [over.match.class.deduct]p3.3:
+//    ... and a constraint that is satisfied if and only if the arguments
+//    of A are deducible (see below) from the return type.
+Expr *buildIsDeducibleConstraint(Sema &SemaRef,
+                                 TypeAliasTemplateDecl *AliasTemplate,
+                                 QualType ReturnType,
+                                 SmallVector<NamedDecl *> TemplateParams) {
+  ASTContext &Context = SemaRef.Context;
+  // Constraint AST nodes must use uninstantiated depth.
+  if (auto *PrimaryTemplate =
+          AliasTemplate->getInstantiatedFromMemberTemplate()) {
+    LocalInstantiationScope Scope(SemaRef);
+
+    // Adjust the depth for TemplateParams.
+    unsigned AdjustDepth = PrimaryTemplate->getTemplateDepth();
+    SmallVector<TemplateArgument> TransformedTemplateArgs;
+    for (auto *TP : TemplateParams) {
+      // Rebuild any internal references to earlier parameters and reindex
+      // as we go.
+      MultiLevelTemplateArgumentList Args;
+      Args.setKind(TemplateSubstitutionKind::Rewrite);
+      Args.addOuterTemplateArguments(TransformedTemplateArgs);
+      NamedDecl *NewParam = transformTemplateParameter(
+          SemaRef, AliasTemplate->getDeclContext(), TP, Args,
+          /*NewIndex=*/TransformedTemplateArgs.size(),
+          getTemplateParameterDepth(TP) + AdjustDepth);
+
+      auto NewTemplateArgument = Context.getCanonicalTemplateArgument(
+          Context.getInjectedTemplateArg(NewParam));
+      TransformedTemplateArgs.push_back(NewTemplateArgument);
+    }
+    // Transformed the ReturnType to restore the uninstantiated depth.
+    MultiLevelTemplateArgumentList Args;
+    Args.setKind(TemplateSubstitutionKind::Rewrite);
+    Args.addOuterTemplateArguments(TransformedTemplateArgs);
+    ReturnType = SemaRef.SubstType(
+        ReturnType, Args, AliasTemplate->getLocation(),
+        Context.DeclarationNames.getCXXDeductionGuideName(AliasTemplate));
+  };
+
+  SmallVector<TypeSourceInfo *> IsDeducibleTypeTraitArgs = {
+      Context.getTrivialTypeSourceInfo(
+          Context.getDeducedTemplateSpecializationType(
+              TemplateName(AliasTemplate), /*DeducedType=*/QualType(),
+              /*IsDependent=*/true)), // template specialization type whose
+                                      // arguments will be deduced.
+      Context.getTrivialTypeSourceInfo(
+          ReturnType), // type from which template arguments are deduced.
+  };
+  return TypeTraitExpr::Create(
+      Context, Context.getLogicalOperationType(), AliasTemplate->getLocation(),
+      TypeTrait::BTT_IsDeducible, IsDeducibleTypeTraitArgs,
+      AliasTemplate->getLocation(), /*Value*/ false);
 }
 
 std::pair<TemplateDecl *, llvm::ArrayRef<TemplateArgument>>
@@ -3112,8 +3180,10 @@ BuildDeductionGuideForTypeAlias(Sema &SemaRef,
           Sema::CodeSynthesisContext::BuildingDeductionGuides)) {
     auto *GG = cast<CXXDeductionGuideDecl>(FPrime);
 
-    Expr *RequiresClause =
-        transformRequireClause(SemaRef, F, AliasTemplate, DeduceResults);
+    Expr *IsDeducible = buildIsDeducibleConstraint(
+        SemaRef, AliasTemplate, FPrime->getReturnType(), FPrimeTemplateParams);
+    Expr *RequiresClause = buildAssociatedConstraints(
+        SemaRef, F, AliasTemplate, DeduceResults, IsDeducible);
 
     // FIXME: implement the is_deducible constraint per C++
     // [over.match.class.deduct]p3.3:
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index b5d405111fe4..4d89fcfc9b11 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -3237,6 +3237,40 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
 
   return TemplateDeductionResult::Success;
 }
+/// Complete template argument deduction for DeduceTemplateArgumentsFromType.
+/// FIXME: this is mostly duplicated with the above two versions. Deduplicate
+/// the three implementations.
+static TemplateDeductionResult FinishTemplateArgumentDeduction(
+    Sema &S, TemplateDecl *TD,
+    SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+    TemplateDeductionInfo &Info) {
+  // Unevaluated SFINAE context.
+  EnterExpressionEvaluationContext Unevaluated(
+      S, Sema::ExpressionEvaluationContext::Unevaluated);
+  Sema::SFINAETrap Trap(S);
+
+  Sema::ContextRAII SavedContext(S, getAsDeclContextOrEnclosing(TD));
+
+  // C++ [temp.deduct.type]p2:
+  //   [...] or if any template argument remains neither deduced nor
+  //   explicitly specified, template argument deduction fails.
+  SmallVector<TemplateArgument, 4> SugaredBuilder, CanonicalBuilder;
+  if (auto Result = ConvertDeducedTemplateArguments(
+          S, TD, /*IsPartialOrdering=*/false, Deduced, Info, SugaredBuilder,
+          CanonicalBuilder);
+      Result != TemplateDeductionResult::Success)
+    return Result;
+
+  if (Trap.hasErrorOccurred())
+    return TemplateDeductionResult::SubstitutionFailure;
+
+  if (auto Result = CheckDeducedArgumentConstraints(S, TD, SugaredBuilder,
+                                                    CanonicalBuilder, Info);
+      Result != TemplateDeductionResult::Success)
+    return Result;
+
+  return TemplateDeductionResult::Success;
+}
 
 /// Perform template argument deduction to determine whether the given template
 /// arguments match the given class or variable template partial specialization
@@ -3305,6 +3339,58 @@ Sema::DeduceTemplateArguments(VarTemplatePartialSpecializationDecl *Partial,
   return ::DeduceTemplateArguments(*this, Partial, TemplateArgs, Info);
 }
 
+TemplateDeductionResult
+Sema::DeduceTemplateArgumentsFromType(TemplateDecl *TD, QualType FromType,
+                                      sema::TemplateDeductionInfo &Info) {
+  if (TD->isInvalidDecl())
+    return TemplateDeductionResult::Invalid;
+
+  QualType PType;
+  if (const auto *CTD = dyn_cast<ClassTemplateDecl>(TD)) {
+    // Use the InjectedClassNameType.
+    PType = Context.getTypeDeclType(CTD->getTemplatedDecl());
+  } else if (const auto *AliasTemplate = dyn_cast<TypeAliasTemplateDecl>(TD)) {
+    PType = AliasTemplate->getTemplatedDecl()
+                ->getUnderlyingType()
+                .getCanonicalType();
+  } else {
+    assert(false && "Expected a class or alias template");
+  }
+
+  // Unevaluated SFINAE context.
+  EnterExpressionEvaluationContext Unevaluated(
+      *this, Sema::ExpressionEvaluationContext::Unevaluated);
+  SFINAETrap Trap(*this);
+
+  // This deduction has no relation to any outer instantiation we might be
+  // performing.
+  LocalInstantiationScope InstantiationScope(*this);
+
+  SmallVector<DeducedTemplateArgument> Deduced(
+      TD->getTemplateParameters()->size());
+  SmallVector<TemplateArgument> PArgs = {TemplateArgument(PType)};
+  SmallVector<TemplateArgument> AArgs = {TemplateArgument(FromType)};
+  if (auto DeducedResult = DeduceTemplateArguments(
+          TD->getTemplateParameters(), PArgs, AArgs, Info, Deduced, false);
+      DeducedResult != TemplateDeductionResult::Success) {
+    return DeducedResult;
+  }
+
+  SmallVector<TemplateArgument, 4> DeducedArgs(Deduced.begin(), Deduced.end());
+  InstantiatingTemplate Inst(*this, Info.getLocation(), TD, DeducedArgs, Info);
+  if (Inst.isInvalid())
+    return TemplateDeductionResult::InstantiationDepth;
+
+  if (Trap.hasErrorOccurred())
+    return TemplateDeductionResult::SubstitutionFailure;
+
+  TemplateDeductionResult Result;
+  runWithSufficientStackSpace(Info.getLocation(), [&] {
+    Result = ::FinishTemplateArgumentDeduction(*this, TD, Deduced, Info);
+  });
+  return Result;
+}
+
 /// Determine whether the given type T is a simple-template-id type.
 static bool isSimpleTemplateIdType(QualType T) {
   if (const TemplateSpecializationType *Spec
diff --git a/clang/test/AST/ast-dump-ctad-alias.cpp b/clang/test/AST/ast-dump-ctad-alias.cpp
index 423c3454ccb7..7fe6c05621ee 100644
--- a/clang/test/AST/ast-dump-ctad-alias.cpp
+++ b/clang/test/AST/ast-dump-ctad-alias.cpp
@@ -24,14 +24,24 @@ Out2<double>::AInner t(1.0);
 // Verify that the require-clause of alias deduction guide is transformed correctly:
 //   - Occurrence T should be replaced with `int`;
 //   - Occurrence V should be replaced with the Y with depth 1
+//   - Depth of occurrence Y in the __is_deducible constraint should be 1
 //
 // CHECK:      |   `-FunctionTemplateDecl {{.*}} <deduction guide for AInner>
 // CHECK-NEXT: |     |-TemplateTypeParmDecl {{.*}} typename depth 0 index 0 Y
-// CHECK-NEXT: |     |-UnresolvedLookupExpr {{.*}} '<dependent type>' lvalue (no ADL) = 'Concept' 
-// CHECK-NEXT: |     | |-TemplateArgument type 'int'
-// CHECK-NEXT: |     | | `-BuiltinType {{.*}} 'int'
-// CHECK-NEXT: |     | `-TemplateArgument type 'type-parameter-1-0'
-// CHECK-NEXT: |     |   `-TemplateTypeParmType {{.*}} 'type-parameter-1-0' dependent depth 1 index 0
+// CHECK-NEXT: |     |-BinaryOperator {{.*}} '<dependent type>' '&&'
+// CHECK-NEXT: |     | |-UnresolvedLookupExpr {{.*}} '<dependent type>' lvalue (no ADL) = 'Concept' 
+// CHECK-NEXT: |     | | |-TemplateArgument type 'int'
+// CHECK-NEXT: |     | | | `-BuiltinType {{.*}} 'int'
+// CHECK-NEXT: |     | | `-TemplateArgument type 'type-parameter-1-0'
+// CHECK-NEXT: |     | |   `-TemplateTypeParmType {{.*}} 'type-parameter-1-0' dependent depth 1 index 0
+// CHECK-NEXT: |     | `-TypeTraitExpr {{.*}} 'bool' __is_deducible
+// CHECK-NEXT: |     |   |-DeducedTemplateSpecializationType {{.*}} 'AInner' dependent
+// CHECK-NEXT: |     |   `-ElaboratedType {{.*}} 'Inner<type-parameter-1-0>' sugar dependent
+// CHECK-NEXT: |     |     `-TemplateSpecializationType {{.*}} 'Inner<type-parameter-1-0>' dependent Inner
+// CHECK-NEXT: |     |       `-TemplateArgument type 'type-parameter-1-0'
+// CHECK-NEXT: |     |         `-SubstTemplateTypeParmType {{.*}} 'type-parameter-1-0' 
+// CHECK-NEXT: |     |           |-FunctionTemplate {{.*}} '<deduction guide for Inner>'
+// CHECK-NEXT: |     |           `-TemplateTypeParmType {{.*}} 'type-parameter-1-0' dependent depth 1 index 0
 // CHECK-NEXT: |     |-CXXDeductionGuideDecl {{.*}} <deduction guide for AInner> 'auto (type-parameter-0-0) -> Inner<type-parameter-0-0>'
 // CHECK-NEXT: |     | `-ParmVarDecl {{.*}} 'type-parameter-0-0'
 // CHECK-NEXT: |     `-CXXDeductionGuideDecl {{.*}} used <deduction guide for AInner> 'auto (double) -> Inner<double>' implicit_instantiation
diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
index 4c5595e409f2..7c186dc379c7 100644
--- a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
+++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
@@ -109,10 +109,12 @@ struct Foo {
 };
 
 template <typename X, int Y>
-using Bar = Foo<X, sizeof(X)>;
+using Bar = Foo<X, sizeof(X)>; // expected-note {{candidate template ignored: couldn't infer template argument 'X'}} \
+                               // expected-note {{candidate template ignored: constraints not satisfied [with X = int]}} \
+                               // expected-note {{because '__is_deducible}}
 
-// FIXME: we should reject this case? GCC rejects it, MSVC accepts it.
-Bar s = {{1}};
+
+Bar s = {{1}}; // expected-error {{no viable constructor or deduction guide }}
 }  // namespace test9
 
 namespace test10 {
@@ -133,9 +135,13 @@ A a(2);  // Foo<int*>
 namespace test11 {
 struct A {};
 template<class T> struct Foo { T c; };
-template<class X, class Y=A> using AFoo = Foo<Y>;
+template<class X, class Y=A>
+using AFoo = Foo<Y>; // expected-note {{candidate template ignored: could not match 'Foo<type-parameter-0-0>' against 'int'}} \
+                    // expected-note {{candidate template ignored: constraints not satisfied [with Y = int]}} \
+                    // expected-note {{because '__is_deducible(AFoo, Foo<int>)' evaluated to false}} \
+                    // expected-note {{candidate function template not viable: requires 0 arguments, but 1 was provided}}
 
-AFoo s = {1};
+AFoo s = {1}; // expected-error {{no viable constructor or deduction guide for deduction of template arguments of 'AFoo'}}
 } // namespace test11
 
 namespace test12 {
@@ -190,13 +196,16 @@ template <class T> struct Foo { Foo(T); };
 
 template<class V> using AFoo = Foo<V *>;
 template<typename> concept False = false;
-template<False W> using BFoo = AFoo<W>;
+// FIXME: emit a more descriptive diagnostic for "__is_deducible" constraint failure.
+template<False W>
+using BFoo = AFoo<W>; // expected-note {{candidate template ignored: constraints not satisfied [with V = int]}} \
+                      // expected-note {{because '__is_deducible(BFoo, Foo<int *>)' evaluated to false}} \
+                      // expected-note {{candidate template ignored: could not match 'Foo<type-parameter-0-0 *>' against 'int *'}}
 int i = 0;
 AFoo a1(&i); // OK, deduce Foo<int *>
 
-// FIXME: we should reject this case as the W is not deduced from the deduced
-// type Foo<int *>.
-BFoo b2(&i); 
+// the W is not deduced from the deduced type Foo<int *>.
+BFoo b2(&i); // expected-error {{no viable constructor or deduction guide for deduction of template arguments of 'BFoo'}}
 } // namespace test15
 
 namespace test16 {
diff --git a/clang/test/SemaTemplate/deduction-guide.cpp b/clang/test/SemaTemplate/deduction-guide.cpp
index 51e1eb49c5de..a91ab5ec7bcc 100644
--- a/clang/test/SemaTemplate/deduction-guide.cpp
+++ b/clang/test/SemaTemplate/deduction-guide.cpp
@@ -282,12 +282,16 @@ using AFoo = Foo<G<U>>;
 // CHECK-LABEL: Dumping <deduction guide for AFoo>
 // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for AFoo>
 // CHECK-NEXT: |-TemplateTypeParmDecl {{.*}} typename depth 0 index 0 U
-// CHECK-NEXT: |-ParenExpr {{.*}} 'bool'
-// CHECK-NEXT: | `-BinaryOperator {{.*}} 'bool' '=='
-// CHECK-NEXT: |   |-UnaryExprOrTypeTraitExpr {{.*}} 'G<type-parameter-0-0>'
-// CHECK-NEXT: |   `-ImplicitCastExpr {{.*}}
-// CHECK-NEXT: |     `-IntegerLiteral {{.*}}
-// CHECK-NEXT: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for AFoo> 'auto (G<type-parameter-0-0>) -> Foo<G<type-parameter-0-0>>'
+// CHECK-NEXT: |-BinaryOperator {{.*}} '&&'
+// CHECK-NEXT: | |-ParenExpr {{.*}} 'bool'
+// CHECK-NEXT: | | `-BinaryOperator {{.*}} 'bool' '=='
+// CHECK-NEXT: | |   |-UnaryExprOrTypeTraitExpr {{.*}} 'G<type-parameter-0-0>'
+// CHECK-NEXT: | |   `-ImplicitCastExpr {{.*}}
+// CHECK-NEXT: | |     `-IntegerLiteral {{.*}}
+// CHECK-NEXT: | `-TypeTraitExpr {{.*}} 'bool' __is_deducible
+// CHECK-NEXT: |   |-DeducedTemplateSpecializationType {{.*}} 'AFoo' dependent
+// CHECK-NEXT: |   `-TemplateSpecializationType {{.*}} 'Foo<G<type-parameter-0-0>>' dependent Foo
+// CHECK:      |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for AFoo> 'auto (G<type-parameter-0-0>) -> Foo<G<type-parameter-0-0>>'
 // CHECK-NEXT: | `-ParmVarDecl {{.*}} 'G<type-parameter-0-0>'
 // CHECK-NEXT: `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for AFoo> 'auto (G<int>) -> Foo<G<int>>' implicit_instantiation
 // CHECK-NEXT:   |-TemplateArgument type 'int'
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index 06777eaa6df6..6e2fd745e666 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -870,8 +870,8 @@ C++23, informally referred to as C++26.</p>
       <td class="partial" align="center">
         <details>
           <summary>Clang 19 (Partial)</summary>
-          The associated constraints (over.match.class.deduct#3.3) for the
-          synthesized deduction guides are not yet implemented.
+          This feature has been initially completed, but the feature macro
+          __cpp_deduction_guides has not been updated.
         </details>
       </td>
     </tr>
-- 
GitLab


From 46bc54f4e688870da8356a9b85257ffb12d47e1f Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Thu, 16 May 2024 19:14:30 +0800
Subject: [PATCH 0996/1206] [ValueTracking] Fix assertion failure when
 `computeKnownFPClass` returns fcNone (#92355)

Fixes
https://github.com/llvm/llvm-project/pull/92084#issuecomment-2114083188.
---
 llvm/lib/Analysis/ValueTracking.cpp           |  4 ++
 .../test/Transforms/InstCombine/known-bits.ll | 40 +++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index c8c527a2d4d2..e07d14ccbb85 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1126,6 +1126,10 @@ static void computeKnownBitsFromOperator(const Operator *I,
       KnownFPClass Result = computeKnownFPClass(V, fcAllFlags, Depth + 1, Q);
       FPClassTest FPClasses = Result.KnownFPClasses;
 
+      // TODO: Treat it as zero/poison if the use of I is unreachable.
+      if (FPClasses == fcNone)
+        break;
+
       if (Result.isKnownNever(fcNormal | fcSubnormal | fcNan)) {
         Known.Zero.setAllBits();
         Known.One.setAllBits();
diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index 7a3802050051..cb404c0b7f88 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1658,5 +1658,45 @@ define i32 @test_qnan_only(float nofpclass(snan sub norm zero inf) %x) {
   ret i32 %and
 }
 
+; Make sure that we don't crash when the use of x is unreachable.
+define i64 @pr92084(double %x) {
+; CHECK-LABEL: @pr92084(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp uno double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN1:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then1:
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_ELSE]], label [[IF_THEN2:%.*]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast double [[X]] to i64
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[CAST]], 1
+; CHECK-NEXT:    ret i64 [[AND]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i64 0
+;
+  %cmp = fcmp uno double %x, 0.000000e+00
+  br i1 %cmp, label %if.then1, label %if.else
+
+if.then1:
+  br i1 %cmp, label %if.else, label %if.then2
+
+if.then2:
+  %cast = bitcast double %x to i64
+  %and = and i64 %cast, 1
+  ret i64 %and
+
+if.else:
+  ret i64 0
+}
+
+define i32 @test_none(float nofpclass(all) %x) {
+; CHECK-LABEL: @test_none(
+; CHECK-NEXT:    [[Y:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], 4194304
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %y = bitcast float %x to i32
+  %and = and i32 %y, 4194304
+  ret i32 %and
+}
+
 declare void @use(i1)
 declare void @sink(i8)
-- 
GitLab


From 0bc1ec5cda8d68a681534bda9fd121b7f58fb495 Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Thu, 16 May 2024 13:34:22 +0200
Subject: [PATCH 0997/1206] [GlobalISel] Reduce KnownBits usage in matcher
 combines (#92381)

Two icmp/and combines forced computation of KnownBits on all operands
everytime. We can avoid computing KnownBits on the LHS by exploiting a
couple of properties:
- Constants are always on the RHS for those instructions. If we have no
KnownBits on the RHS, we can bail out early and avoid computing LHS
knownbits.
- For icmp uge/ult 0, we don't need to know the KBs of the LHS to infer
the result

This allows to save some KnownBits calls, which are very expensive,
without affecting codegen.
---
 .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 102 ++++++++++++------
 1 file changed, 67 insertions(+), 35 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 9999776b9826..809f13abcadd 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -3220,8 +3220,15 @@ bool CombinerHelper::matchRedundantAnd(MachineInstr &MI,
   Register AndDst = MI.getOperand(0).getReg();
   Register LHS = MI.getOperand(1).getReg();
   Register RHS = MI.getOperand(2).getReg();
-  KnownBits LHSBits = KB->getKnownBits(LHS);
+
+  // Check the RHS (maybe a constant) first, and if we have no KnownBits there,
+  // we can't do anything. If we do, then it depends on whether we have
+  // KnownBits on the LHS.
   KnownBits RHSBits = KB->getKnownBits(RHS);
+  if (RHSBits.isUnknown())
+    return false;
+
+  KnownBits LHSBits = KB->getKnownBits(LHS);
 
   // Check that x & Mask == x.
   // x & 1 == x, always
@@ -3260,6 +3267,7 @@ bool CombinerHelper::matchRedundantOr(MachineInstr &MI, Register &Replacement) {
   Register OrDst = MI.getOperand(0).getReg();
   Register LHS = MI.getOperand(1).getReg();
   Register RHS = MI.getOperand(2).getReg();
+
   KnownBits LHSBits = KB->getKnownBits(LHS);
   KnownBits RHSBits = KB->getKnownBits(RHS);
 
@@ -4253,43 +4261,67 @@ bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
                                                    int64_t &MatchInfo) {
   assert(MI.getOpcode() == TargetOpcode::G_ICMP);
   auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
-  auto KnownLHS = KB->getKnownBits(MI.getOperand(2).getReg());
+
+  // We want to avoid calling KnownBits on the LHS if possible, as this combine
+  // has no filter and runs on every G_ICMP instruction. We can avoid calling
+  // KnownBits on the LHS in two cases:
+  //
+  //  - The RHS is unknown: Constants are always on RHS. If the RHS is unknown
+  //  we cannot do any transforms so we can safely bail out early.
+  //  - The RHS is zero: we don't need to know the LHS to do unsigned <0 and
+  //  >=0.
   auto KnownRHS = KB->getKnownBits(MI.getOperand(3).getReg());
+  if (KnownRHS.isUnknown())
+    return false;
+
   std::optional<bool> KnownVal;
-  switch (Pred) {
-  default:
-    llvm_unreachable("Unexpected G_ICMP predicate?");
-  case CmpInst::ICMP_EQ:
-    KnownVal = KnownBits::eq(KnownLHS, KnownRHS);
-    break;
-  case CmpInst::ICMP_NE:
-    KnownVal = KnownBits::ne(KnownLHS, KnownRHS);
-    break;
-  case CmpInst::ICMP_SGE:
-    KnownVal = KnownBits::sge(KnownLHS, KnownRHS);
-    break;
-  case CmpInst::ICMP_SGT:
-    KnownVal = KnownBits::sgt(KnownLHS, KnownRHS);
-    break;
-  case CmpInst::ICMP_SLE:
-    KnownVal = KnownBits::sle(KnownLHS, KnownRHS);
-    break;
-  case CmpInst::ICMP_SLT:
-    KnownVal = KnownBits::slt(KnownLHS, KnownRHS);
-    break;
-  case CmpInst::ICMP_UGE:
-    KnownVal = KnownBits::uge(KnownLHS, KnownRHS);
-    break;
-  case CmpInst::ICMP_UGT:
-    KnownVal = KnownBits::ugt(KnownLHS, KnownRHS);
-    break;
-  case CmpInst::ICMP_ULE:
-    KnownVal = KnownBits::ule(KnownLHS, KnownRHS);
-    break;
-  case CmpInst::ICMP_ULT:
-    KnownVal = KnownBits::ult(KnownLHS, KnownRHS);
-    break;
+  if (KnownRHS.isZero()) {
+    // ? uge 0 -> always true
+    // ? ult 0 -> always false
+    if (Pred == CmpInst::ICMP_UGE)
+      KnownVal = true;
+    else if (Pred == CmpInst::ICMP_ULT)
+      KnownVal = false;
   }
+
+  if (!KnownVal) {
+    auto KnownLHS = KB->getKnownBits(MI.getOperand(2).getReg());
+    switch (Pred) {
+    default:
+      llvm_unreachable("Unexpected G_ICMP predicate?");
+    case CmpInst::ICMP_EQ:
+      KnownVal = KnownBits::eq(KnownLHS, KnownRHS);
+      break;
+    case CmpInst::ICMP_NE:
+      KnownVal = KnownBits::ne(KnownLHS, KnownRHS);
+      break;
+    case CmpInst::ICMP_SGE:
+      KnownVal = KnownBits::sge(KnownLHS, KnownRHS);
+      break;
+    case CmpInst::ICMP_SGT:
+      KnownVal = KnownBits::sgt(KnownLHS, KnownRHS);
+      break;
+    case CmpInst::ICMP_SLE:
+      KnownVal = KnownBits::sle(KnownLHS, KnownRHS);
+      break;
+    case CmpInst::ICMP_SLT:
+      KnownVal = KnownBits::slt(KnownLHS, KnownRHS);
+      break;
+    case CmpInst::ICMP_UGE:
+      KnownVal = KnownBits::uge(KnownLHS, KnownRHS);
+      break;
+    case CmpInst::ICMP_UGT:
+      KnownVal = KnownBits::ugt(KnownLHS, KnownRHS);
+      break;
+    case CmpInst::ICMP_ULE:
+      KnownVal = KnownBits::ule(KnownLHS, KnownRHS);
+      break;
+    case CmpInst::ICMP_ULT:
+      KnownVal = KnownBits::ult(KnownLHS, KnownRHS);
+      break;
+    }
+  }
+
   if (!KnownVal)
     return false;
   MatchInfo =
-- 
GitLab


From 7d81062352f75cf328d91d4900af52c1842b950e Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Thu, 16 May 2024 13:39:00 +0200
Subject: [PATCH 0998/1206] [GlobalISel] Refactor Combiner MatchData & Apply
 C++ Code Handling (#92239)

Combiners that use C++ code in their "apply" pattern only use that. They
never mix it with MIR patterns as that has little added value.

This patch restricts C++ apply code so that if C++ is used, we cannot
use MIR patterns or builtins with it. Adding this restriction allows us
to merge calls to match and apply C++ code together, which in turns
makes it so we can just have MatchData variables on the stack.

So before, we would have
```
  GIM_CheckCxxInsnPredicate // match
  GIM_CheckCxxInsnPredicate // apply
  GIR_Done
```
Alongside a massive C++ struct holding the MatchData of all rules
possible (which was a big space/perf issue).

Now we just have
```
GIR_DoneWithCustomAction
```

And the function being ran just does
```
unsigned SomeMatchData;
if (match(SomeMatchData))
  apply(SomeMatchData)
```

This approach solves multiple issues in one:
- MatchData handling is greatly simplified and more efficient, "don't
pay for what you don't use"
  - We reduce the size of the match table
- Calling C++ code has a certain overhead (we need a switch), and this
overhead is only paid once now.

Handling of C++ code inside PatFrags is unchanged though, that still
emits a `GIM_CheckCxxInsnPredicate`. This is completely fine as they
can't use MatchDatas.
---
 llvm/docs/GlobalISel/MIRPatterns.rst          |  28 ++-
 .../CodeGen/GlobalISel/GIMatchTableExecutor.h |  15 +-
 .../GlobalISel/GIMatchTableExecutorImpl.h     |  14 +-
 .../include/llvm/Target/GlobalISel/Combine.td |  10 +-
 .../match-table-cxx.td                        | 143 ++++++++++++++
 .../match-table-permutations.td               | 104 +++-------
 .../match-table-variadics.td                  |  38 ++--
 .../GlobalISelCombinerEmitter/match-table.td  | 133 ++++++-------
 .../patfrag-errors.td                         |   2 +-
 .../pattern-errors.td                         |  17 +-
 .../pattern-parsing.td                        |  13 +-
 llvm/test/TableGen/GlobalISelEmitter.td       |   2 +-
 .../test/TableGen/GlobalISelEmitterHwModes.td |   2 +-
 llvm/utils/TableGen/Common/CMakeLists.txt     |   1 -
 .../Common/GlobalISel/CXXPredicates.cpp       |   2 +-
 .../Common/GlobalISel/CXXPredicates.h         |  10 +-
 .../GlobalISel/GlobalISelMatchTable.cpp       |  63 +++---
 .../Common/GlobalISel/GlobalISelMatchTable.h  |  23 +--
 .../GlobalISelMatchTableExecutorEmitter.cpp   |   7 +-
 .../GlobalISelMatchTableExecutorEmitter.h     |   4 -
 .../Common/GlobalISel/MatchDataInfo.cpp       |  49 -----
 .../Common/GlobalISel/MatchDataInfo.h         |  90 ---------
 .../TableGen/Common/GlobalISel/Patterns.cpp   |   4 +-
 .../TableGen/Common/GlobalISel/Patterns.h     |   4 +
 .../TableGen/GlobalISelCombinerEmitter.cpp    | 184 +++++++++++-------
 llvm/utils/TableGen/GlobalISelEmitter.cpp     |   2 +-
 26 files changed, 498 insertions(+), 466 deletions(-)
 create mode 100644 llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
 delete mode 100644 llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp
 delete mode 100644 llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h

diff --git a/llvm/docs/GlobalISel/MIRPatterns.rst b/llvm/docs/GlobalISel/MIRPatterns.rst
index d7dce1b978cd..7e6d88683d49 100644
--- a/llvm/docs/GlobalISel/MIRPatterns.rst
+++ b/llvm/docs/GlobalISel/MIRPatterns.rst
@@ -274,8 +274,32 @@ it's less verbose.
 
 
 Combine Rules also allow mixing C++ code with MIR patterns, so that you
-may perform additional checks when matching, or run additional code after
-rewriting a pattern.
+may perform additional checks when matching, or run a C++ action after
+matching.
+
+Note that C++ code in ``apply`` pattern is mutually exclusive with
+other patterns. However, you can freely mix C++ code with other
+types of patterns in ``match`` patterns.
+C++ code in ``match`` patterns is always run last, after all other
+patterns matched.
+
+.. code-block:: text
+  :caption: Apply Pattern Examples with C++ code
+
+  // Valid
+  def Foo : GICombineRule<
+    (defs root:$root),
+    (match (G_ZEXT $tmp, (i32 0)),
+           (G_STORE $tmp, $ptr):$root,
+           "return myFinalCheck()"),
+    (apply "runMyAction(${root})")>;
+
+  // error: 'apply' patterns cannot mix C++ code with other types of patterns
+  def Bar : GICombineRule<
+    (defs root:$dst),
+    (match (G_ZEXT $dst, $src):$mi),
+    (apply (G_MUL $dst, $src, $src),
+           "runMyAction(${root})")>;
 
 The following expansions are available for MIR patterns:
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
index 72c63ecba529..371c5c5a0a1e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
@@ -471,16 +471,11 @@ enum {
   /// - RendererFnID(2) - Custom renderer function to call
   GIR_CustomRenderer,
 
-  /// Calls a C++ function to perform an action when a match is complete.
-  /// The MatcherState is passed to the function to allow it to modify
-  /// instructions.
-  /// This is less constrained than a custom renderer and can update
-  /// instructions
-  /// in the state.
+  /// Calls a C++ function that concludes the current match.
+  /// The C++ function is free to return false and reject the match, or
+  /// return true and mutate the instruction(s) (or do nothing, even).
   /// - FnID(2) - The function to call.
-  /// TODO: Remove this at some point when combiners aren't reliant on it. It's
-  /// a bit of a hack.
-  GIR_CustomAction,
+  GIR_DoneWithCustomAction,
 
   /// Render operands to the specified instruction using a custom function,
   /// reading from a specific operand.
@@ -688,7 +683,7 @@ protected:
     llvm_unreachable("Subclass does not implement testSimplePredicate!");
   }
 
-  virtual void runCustomAction(unsigned, const MatcherState &State,
+  virtual bool runCustomAction(unsigned, const MatcherState &State,
                                NewMIVector &OutMIs) const {
     llvm_unreachable("Subclass does not implement runCustomAction!");
   }
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
index 4d147bf20c26..2ea9d11779f0 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
@@ -1335,13 +1335,19 @@ bool GIMatchTableExecutor::executeMatchTable(
           -1); // Not a source operand of the old instruction.
       break;
     }
-    case GIR_CustomAction: {
+    case GIR_DoneWithCustomAction: {
       uint16_t FnID = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
-                      dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID
-                             << ")\n");
+                      dbgs() << CurrentIdx << ": GIR_DoneWithCustomAction(FnID="
+                             << FnID << ")\n");
       assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID");
-      runCustomAction(FnID, State, OutMIs);
+      if (runCustomAction(FnID, State, OutMIs)) {
+        propagateFlags();
+        return true;
+      }
+
+      if (handleReject() == RejectAndGiveUp)
+        return false;
       break;
     }
     case GIR_CustomOperandRenderer: {
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 98d266c8c0b4..d61a5759d5a9 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -338,7 +338,7 @@ def bitreverse_shl : GICombineRule<
 
 // Combine bitreverse(lshr (bitreverse x), y)) -> (shl x, y)
 def bitreverse_lshr : GICombineRule<
-  (defs root:$d, build_fn_matchinfo:$matchinfo),
+  (defs root:$d),
   (match (G_BITREVERSE $rev, $val),
          (G_LSHR $src, $rev, $amt):$mi,
          (G_BITREVERSE $d, $src),
@@ -1352,7 +1352,7 @@ def match_extract_of_element : GICombineRule<
   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
 
 def extract_vector_element_not_const : GICombineRule<
-   (defs root:$root, build_fn_matchinfo:$matchinfo),
+   (defs root:$root),
    (match (G_INSERT_VECTOR_ELT $src, $x, $value, $idx),
           (G_EXTRACT_VECTOR_ELT $root, $src, $idx)),
    (apply (GIReplaceReg $root, $value))>;
@@ -1567,20 +1567,20 @@ def combine_shuffle_concat : GICombineRule<
   (apply [{ Helper.applyCombineShuffleConcat(*${root}, ${matchinfo}); }])>;
 
 def insert_vector_element_idx_undef : GICombineRule<
-   (defs root:$root, build_fn_matchinfo:$matchinfo),
+   (defs root:$root),
    (match (G_IMPLICIT_DEF $idx),
           (G_INSERT_VECTOR_ELT $root, $src, $elt, $idx)),
    (apply (G_IMPLICIT_DEF $root))>;
 
 def insert_vector_element_elt_undef : GICombineRule<
-   (defs root:$root, build_fn_matchinfo:$matchinfo),
+   (defs root:$root),
    (match (G_IMPLICIT_DEF $elt),
           (G_INSERT_VECTOR_ELT $root, $src, $elt, $idx),
           [{ return isGuaranteedNotToBePoison(${src}.getReg(), MRI); }]),
    (apply (GIReplaceReg $root, $src))>;
 
 def insert_vector_element_extract_vector_element : GICombineRule<
-   (defs root:$root, build_fn_matchinfo:$matchinfo),
+   (defs root:$root),
    (match (G_EXTRACT_VECTOR_ELT $elt, $src, $idx),
           (G_INSERT_VECTOR_ELT $root, $src, $elt, $idx)),
    (apply (GIReplaceReg $root, $src))>;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
new file mode 100644
index 000000000000..bb1acde29642
--- /dev/null
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
@@ -0,0 +1,143 @@
+// RUN: llvm-tblgen -I %p/../../../include -gen-global-isel-combiner \
+// RUN:     -combiners=MyCombiner %s | \
+// RUN: FileCheck %s
+
+include "llvm/Target/Target.td"
+include "llvm/Target/GlobalISel/Combine.td"
+
+def MyTargetISA : InstrInfo;
+def MyTarget : Target { let InstructionSet = MyTargetISA; }
+
+def OneMatchOneApply : GICombineRule<
+  (defs root:$a),
+  (match (G_FABS $a, $b), "return MATCH0;"),
+  (apply "APPLY0")>;
+
+def TwoMatchTwoApply : GICombineRule<
+  (defs root:$a),
+  (match (G_FNEG $a, $b), "return MATCH0;", "return MATCH1;"),
+  (apply "APPLY0", "APPLY1")>;
+
+def TwoMatchNoApply : GICombineRule<
+  (defs root:$a),
+  (match (G_STORE $x, $y):$a, "return MATCH0;", "return MATCH1;"),
+  (apply (GIEraseRoot))>;
+
+def NoMatchTwoApply : GICombineRule<
+  (defs root:$a),
+  (match (G_SEXT $a, $y)),
+  (apply "APPLY0", "APPLY1")>;
+
+def MyCombiner: GICombiner<"GenMyCombiner", [
+  OneMatchOneApply,
+  TwoMatchTwoApply,
+  TwoMatchNoApply,
+  NoMatchTwoApply
+]>;
+
+// CHECK:      bool GenMyCombiner::testMIPredicate_MI(unsigned PredicateID, const MachineInstr & MI, const MatcherState &State) const {
+// CHECK-NEXT:   switch (PredicateID) {
+// CHECK-NEXT:   case GICXXPred_MI_Predicate_GICombiner0: {
+// CHECK-NEXT:     return MATCH0;
+// CHECK-NEXT:   }
+// CHECK-NEXT:   case GICXXPred_MI_Predicate_GICombiner1: {
+// CHECK-NEXT:     return MATCH1;
+// CHECK-NEXT:   }
+// CHECK-NEXT:   }
+// CHECK-NEXT:   llvm_unreachable("Unknown predicate");
+// CHECK-NEXT:   return false;
+// CHECK-NEXT: }
+
+// CHECK:      bool GenMyCombiner::runCustomAction(unsigned ApplyID, const MatcherState &State, NewMIVector &OutMIs) const {
+// CHECK-NEXT:   Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]);
+// CHECK-NEXT:   switch(ApplyID) {
+// CHECK-NEXT:   case GICXXCustomAction_GICombiner0:{
+// CHECK-NEXT:     // Match Patterns
+// CHECK-NEXT:     if(![&](){return MATCH0;}()) {
+// CHECK-NEXT:       return false;
+// CHECK-NEXT:     }
+// CHECK-NEXT:     // Apply Patterns
+// CHECK-NEXT:     APPLY0
+// CHECK-NEXT:     return true;
+// CHECK-NEXT:   }
+// CHECK-NEXT:   case GICXXCustomAction_GICombiner1:{
+// CHECK-NEXT:     // Match Patterns
+// CHECK-NEXT:     if(![&](){return MATCH0;}()) {
+// CHECK-NEXT:       return false;
+// CHECK-NEXT:     }
+// CHECK-NEXT:     if(![&](){return MATCH1;}()) {
+// CHECK-NEXT:       return false;
+// CHECK-NEXT:     }
+// CHECK-NEXT:     // Apply Patterns
+// CHECK-NEXT:     APPLY0
+// CHECK-NEXT:     APPLY1
+// CHECK-NEXT:     return true;
+// CHECK-NEXT:   }
+// CHECK-NEXT:   case GICXXCustomAction_GICombiner2:{
+// CHECK-NEXT:     // Apply Patterns
+// CHECK-NEXT:     APPLY0
+// CHECK-NEXT:     APPLY1
+// CHECK-NEXT:     return true;
+// CHECK-NEXT:   }
+// CHECK-NEXT:   }
+// CHECK-NEXT:   llvm_unreachable("Unknown Apply Action");
+// CHECK-NEXT: }
+
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(94), GIMT_Encode2(194), /*)*//*default:*//*Label 4*/ GIMT_Encode4(464),
+// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4(410), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4(428), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 2*/ GIMT_Encode4(440), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_FABS*//*Label 3*/ GIMT_Encode4(452),
+// CHECK-NEXT:     // Label 0: @410
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(427), // Rule ID 2 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
+// CHECK-NEXT:       // MIs[0] x
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // MIs[0] y
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner0),
+// CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner1),
+// CHECK-NEXT:       // Combiner Rule #2: TwoMatchNoApply
+// CHECK-NEXT:       GIR_EraseRootFromParent_Done,
+// CHECK-NEXT:     // Label 5: @427
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     // Label 1: @428
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(439), // Rule ID 3 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
+// CHECK-NEXT:       // MIs[0] a
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // MIs[0] y
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // Combiner Rule #3: NoMatchTwoApply
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner2),
+// CHECK-NEXT:     // Label 6: @439
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     // Label 2: @440
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(451), // Rule ID 1 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
+// CHECK-NEXT:       // MIs[0] a
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // MIs[0] b
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // Combiner Rule #1: TwoMatchTwoApply
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1),
+// CHECK-NEXT:     // Label 7: @451
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     // Label 3: @452
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(463), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
+// CHECK-NEXT:       // MIs[0] a
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // MIs[0] b
+// CHECK-NEXT:       // No operand predicates
+// CHECK-NEXT:       // Combiner Rule #0: OneMatchOneApply
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:     // Label 8: @463
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     // Label 4: @464
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     }; // Size: 465 bytes
+// CHECK-NEXT:   return MatchTable0;
+// CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td
index fda57d5b64e0..4a908e782348 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td
@@ -24,7 +24,7 @@ def Test0 : GICombineRule<
          (MatchFooPerms $cst1, (i32 20)):$b,
          (MatchFooPerms $cst2, (i32 30)):$c
   ),
-  (apply (COPY $dst, (i32 0)), "APPLY ${cst0}")>;
+  (apply "APPLY ${cst0}")>;
 
 def MyCombiner: GICombiner<"GenMyCombiner", [
   Test0
@@ -159,9 +159,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(738),
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(562),
 // CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_AND),
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4(88), // Rule ID 7 //
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4(66), // Rule ID 7 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -187,16 +187,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner22),
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner23),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*NumInsns*/4,
-// CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[1], c[1]]
-// CHECK-NEXT:         GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
-// CHECK-NEXT:         GIR_RootToRootCopy, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:         GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:       // Label 1: @88
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 2*/ GIMT_Encode4(175), // Rule ID 6 //
+// CHECK-NEXT:         GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:       // Label 1: @66
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 2*/ GIMT_Encode4(131), // Rule ID 6 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -225,16 +219,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner19),
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner20),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*NumInsns*/5,
-// CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[1], c[0]]
-// CHECK-NEXT:         GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
-// CHECK-NEXT:         GIR_RootToRootCopy, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:         GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:       // Label 2: @175
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(262), // Rule ID 5 //
+// CHECK-NEXT:         GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:       // Label 2: @131
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(196), // Rule ID 5 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -263,16 +251,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner16),
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner17),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*NumInsns*/5,
-// CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[0], c[1]]
-// CHECK-NEXT:         GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
-// CHECK-NEXT:         GIR_RootToRootCopy, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:         GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:       // Label 3: @262
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(357), // Rule ID 4 //
+// CHECK-NEXT:         GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:       // Label 3: @196
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(269), // Rule ID 4 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -304,16 +286,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner13),
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner14),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*NumInsns*/6,
-// CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[0], c[0]]
-// CHECK-NEXT:         GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
-// CHECK-NEXT:         GIR_RootToRootCopy, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:         GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:       // Label 4: @357
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(444), // Rule ID 3 //
+// CHECK-NEXT:         GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:       // Label 4: @269
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(334), // Rule ID 3 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -342,16 +318,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner10),
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner11),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*NumInsns*/5,
-// CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[1], c[1]]
-// CHECK-NEXT:         GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
-// CHECK-NEXT:         GIR_RootToRootCopy, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:         GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:       // Label 5: @444
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(539), // Rule ID 2 //
+// CHECK-NEXT:         GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:       // Label 5: @334
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(407), // Rule ID 2 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -383,16 +353,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner7),
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner8),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*NumInsns*/6,
-// CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[1], c[0]]
-// CHECK-NEXT:         GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
-// CHECK-NEXT:         GIR_RootToRootCopy, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:         GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:       // Label 6: @539
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(634), // Rule ID 1 //
+// CHECK-NEXT:         GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:       // Label 6: @407
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(480), // Rule ID 1 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -424,16 +388,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner4),
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner5),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*NumInsns*/6,
-// CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[0], c[1]]
-// CHECK-NEXT:         GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
-// CHECK-NEXT:         GIR_RootToRootCopy, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:         GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:       // Label 7: @634
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(737), // Rule ID 0 //
+// CHECK-NEXT:         GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:       // Label 7: @480
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(561), // Rule ID 0 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -468,18 +426,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner1),
 // CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner2),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*NumInsns*/7,
-// CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[0], c[0]]
-// CHECK-NEXT:         GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
-// CHECK-NEXT:         GIR_RootToRootCopy, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:         GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:       // Label 8: @737
+// CHECK-NEXT:         GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:       // Label 8: @561
 // CHECK-NEXT:       GIM_Reject,
-// CHECK-NEXT:     // Label 0: @738
+// CHECK-NEXT:     // Label 0: @562
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     }; // Size: 739 bytes
+// CHECK-NEXT:     }; // Size: 563 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
index 83b77519bc73..35bddf912a54 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
@@ -37,11 +37,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(69), GIMT_Encode2(73), /*)*//*default:*//*Label 2*/ GIMT_Encode4(88),
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(69), GIMT_Encode2(73), /*)*//*default:*//*Label 2*/ GIMT_Encode4(84),
 // CHECK-NEXT:     /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(26), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_BUILD_VECTOR*//*Label 1*/ GIMT_Encode4(57),
+// CHECK-NEXT:     /*TargetOpcode::G_BUILD_VECTOR*//*Label 1*/ GIMT_Encode4(55),
 // CHECK-NEXT:     // Label 0: @26
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(41), // Rule ID 2 //
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(40), // Rule ID 2 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
 // CHECK-NEXT:       // MIs[0] a
@@ -49,10 +49,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // MIs[0] b
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #2: InstTest2
-// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 3: @41
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(56), // Rule ID 3 //
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:     // Label 3: @40
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(54), // Rule ID 3 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/4,
 // CHECK-NEXT:       // MIs[0] a
@@ -64,12 +63,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // MIs[0] d
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #3: InstTest3
-// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 4: @56
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:     // Label 4: @54
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @57
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(72), // Rule ID 1 //
+// CHECK-NEXT:     // Label 1: @55
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(69), // Rule ID 1 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
 // CHECK-NEXT:       // MIs[0] a
@@ -77,10 +75,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // MIs[0] b
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #1: InstTest1
-// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 5: @72
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(87), // Rule ID 0 //
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:     // Label 5: @69
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(83), // Rule ID 0 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/4,
 // CHECK-NEXT:       // MIs[0] a
@@ -92,12 +89,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // MIs[0] d
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #0: InstTest0
-// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 6: @87
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:     // Label 6: @83
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @88
+// CHECK-NEXT:     // Label 2: @84
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     }; // Size: 89 bytes
+// CHECK-NEXT:     }; // Size: 85 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
index 1052e31b2d05..513a86754d6d 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
@@ -41,7 +41,7 @@ def InOutInstTest0 : GICombineRule<
   (defs root:$root),
   (match (G_ZEXT $tmp, $ext),
          (G_STORE $tmp, $ptr):$root),
-  (apply (G_STORE $ext, $ptr):$root, "APPLY ${ext} ${ptr} ${root}")>;
+  (apply (G_STORE $ext, $ptr):$root)>;
 
 // Imm operand of G_CONSTANT should match a literal int, while the second
 // should match a constant.
@@ -76,16 +76,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // We have at most 2 registers used by one rule at a time, so we should only have 2 registers MDInfos.
 
-// CHECK:      struct MatchInfosTy {
-// CHECK-NEXT:   Register MDInfo0, MDInfo1;
-// CHECK-NEXT: };
-
 // Check predicates
 // CHECK:      switch (PredicateID) {
 // CHECK-NEXT: case GICXXPred_MI_Predicate_GICombiner0: {
-// CHECK-NEXT:   return CHECK State.MIs[0]->getOperand(0), State.MIs[0]->getOperand(1), State.MIs[1]->getOperand(1), State.MIs[0]
-// CHECK-NEXT: }
-// CHECK-NEXT: case GICXXPred_MI_Predicate_GICombiner1: {
 // CHECK-NEXT:   return matchIConstant(State.MIs[0]->getOperand(1), 0);
 // CHECK-NEXT: }
 
@@ -96,8 +89,6 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:   B.setInstrAndDebugLoc(I);
 // CHECK-NEXT:   State.MIs.clear();
 // CHECK-NEXT:   State.MIs.push_back(&I);
-// CHECK-NEXT:   MatchInfos = MatchInfosTy();
-// CHECK-EMPTY:
 // CHECK-NEXT:   if (executeMatchTable(*this, State, ExecInfo, B, getMatchTable(), *ST.getInstrInfo(), MRI, *MRI.getTargetRegisterInfo(), *ST.getRegBankInfo(), AvailableFeatures, /*CoverageInfo*/ nullptr))
 // CHECK-NEXT:     return true;
 // CHECK-NEXT:   }
@@ -107,43 +98,52 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // Check apply
 // CHECK:      enum {
-// CHECK-NEXT:   GICXXCustomAction_CombineApplyGICombiner0 = GICXXCustomAction_Invalid + 1,
-// CHECK-NEXT:   GICXXCustomAction_CombineApplyGICombiner1,
-// CHECK-NEXT:   GICXXCustomAction_CombineApplyGICombiner2,
+// CHECK-NEXT:   GICXXCustomAction_GICombiner0 = GICXXCustomAction_Invalid + 1,
+// CHECK-NEXT:   GICXXCustomAction_GICombiner1,
+// CHECK-NEXT:   GICXXCustomAction_GICombiner2,
 // CHECK-NEXT: };
-// CHECK-NEXT: void GenMyCombiner::runCustomAction(unsigned ApplyID, const MatcherState &State, NewMIVector &OutMIs) const {
+
+// CHECK:      bool GenMyCombiner::runCustomAction(unsigned ApplyID, const MatcherState &State, NewMIVector &OutMIs) const {
+// CHECK-NEXT:   Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]);
 // CHECK-NEXT:   switch(ApplyID) {
-// CHECK-NEXT:   case GICXXCustomAction_CombineApplyGICombiner0:{
-// CHECK-NEXT:     Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]);
+// CHECK-NEXT:   case GICXXCustomAction_GICombiner0:{
+// CHECK-NEXT:     // Apply Patterns
 // CHECK-NEXT:     APPLY
-// CHECK-NEXT:     return;
+// CHECK-NEXT:     return true;
 // CHECK-NEXT:   }
-// CHECK-NEXT:   case GICXXCustomAction_CombineApplyGICombiner1:{
-// CHECK-NEXT:     Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]);
-// CHECK-NEXT:     APPLY MatchInfos.MDInfo0, MatchInfos.MDInfo1
-// CHECK-NEXT:     return;
+// CHECK-NEXT:   case GICXXCustomAction_GICombiner1:{
+// CHECK-NEXT:     Register   GIMatchData_r0;
+// CHECK-NEXT:     Register   GIMatchData_r1;
+// CHECK-NEXT:     // Apply Patterns
+// CHECK-NEXT:     APPLY GIMatchData_r0, GIMatchData_r1
+// CHECK-NEXT:     return true;
+// CHECK-NEXT:   }
+// CHECK-NEXT:   case GICXXCustomAction_GICombiner2:{
+// CHECK-NEXT:     Register   GIMatchData_r0;
+// CHECK-NEXT:     // Match Patterns
+// CHECK-NEXT:     if(![&](){return CHECK State.MIs[0]->getOperand(0), State.MIs[0]->getOperand(1), State.MIs[1]->getOperand(1), State.MIs[0]}()) {
+// CHECK-NEXT:       return false;
+// CHECK-NEXT:     }
+// CHECK-NEXT:     // Apply Patterns
+// CHECK-NEXT:     APPLY
+// CHECK-NEXT:     return true;
 // CHECK-NEXT:   }
-// CHECK-NEXT:   case GICXXCustomAction_CombineApplyGICombiner2:{
-// CHECK-NEXT:     Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]);
-// CHECK-NEXT:     APPLY State.MIs[1]->getOperand(1) State.MIs[0]->getOperand(1) OutMIs[0]
-// CHECK-NEXT:     return;
 // CHECK-NEXT:   }
-// CHECK-NEXT: }
 // CHECK-NEXT:   llvm_unreachable("Unknown Apply Action");
 // CHECK-NEXT: }
 
 // Verify match table.
 // CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label 6*/ GIMT_Encode4([[L677:[0-9]+]]),
-// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4([[L462:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4([[L504:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4([[L557:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4([[L599:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4([[L624:[0-9]+]]), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4([[L637:[0-9]+]]),
-// CHECK-NEXT:     // Label 0: @[[L462]]
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4([[L491:[0-9]+]]), // Rule ID 4 //
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(133), /*)*//*default:*//*Label 6*/ GIMT_Encode4(653),
+// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(466), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(502), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(549), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(583), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(606), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(618),
+// CHECK-NEXT:     // Label 0: @466
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(490), // Rule ID 4 //
 // CHECK-NEXT:       GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasAnswerToEverything),
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
 // CHECK-NEXT:       // MIs[0] a
@@ -153,25 +153,22 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:       // MIs[1] c
 // CHECK-NEXT:       // No operand predicates
-// CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner0),
 // CHECK-NEXT:       GIM_CheckIsSafeToFold, /*NumInsns*/1,
 // CHECK-NEXT:       // Combiner Rule #3: InstTest1
-// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 7: @[[L491]]
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4([[L503:[0-9]+]]), // Rule ID 3 //
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner2),
+// CHECK-NEXT:     // Label 7: @490
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(501), // Rule ID 3 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] b
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #2: InstTest0
-// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner1),
-// CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 8: @[[L503]]
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1),
+// CHECK-NEXT:     // Label 8: @501
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @[[L504]]
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4([[L556:[0-9]+]]), // Rule ID 6 //
+// CHECK-NEXT:     // Label 1: @502
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(548), // Rule ID 6 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule5Enabled),
 // CHECK-NEXT:       GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
 // CHECK-NEXT:       // MIs[0] dst
@@ -188,10 +185,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_RootToRootCopy, /*OpIdx*/0, // dst
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z
 // CHECK-NEXT:       GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:     // Label 9: @[[L556]]
+// CHECK-NEXT:     // Label 9: @548
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @[[L557]]
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4([[L598:[0-9]+]]), // Rule ID 5 //
+// CHECK-NEXT:     // Label 2: @549
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(582), // Rule ID 5 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled),
 // CHECK-NEXT:       // MIs[0] tmp
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1]
@@ -206,40 +203,36 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // ext
 // CHECK-NEXT:       GIR_RootToRootCopy, /*OpIdx*/1, // ptr
 // CHECK-NEXT:       GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/2, /*MergeInsnID's*/0, 1,
-// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner2),
 // CHECK-NEXT:       GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:     // Label 10: @[[L598]]
+// CHECK-NEXT:     // Label 10: @582
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @[[L599]]
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4([[L611:[0-9]+]]), // Rule ID 0 //
+// CHECK-NEXT:     // Label 3: @583
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(594), // Rule ID 0 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC'
-// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 11: @[[L611]]
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4([[L623:[0-9]+]]), // Rule ID 1 //
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:     // Label 11: @594
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(605), // Rule ID 1 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC'
-// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 12: @[[L623]]
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:     // Label 12: @605
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 4: @[[L624]]
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4([[L636:[0-9]+]]), // Rule ID 2 //
+// CHECK-NEXT:     // Label 4: @606
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(617), // Rule ID 2 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT'
-// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
-// CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 13: @[[L636]]
+// CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
+// CHECK-NEXT:     // Label 13: @617
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 5: @[[L637]]
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4([[L676:[0-9]+]]), // Rule ID 7 //
+// CHECK-NEXT:     // Label 5: @618
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(652), // Rule ID 7 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule6Enabled),
 // CHECK-NEXT:       // MIs[0] dst
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] cst
 // CHECK-NEXT:       // No operand predicates
-// CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner1),
+// CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner0),
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
 // CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:       // Combiner Rule #6: PatFragTest0 @ [__PatFragTest0_match_1[0]]
@@ -247,10 +240,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_RootToRootCopy, /*OpIdx*/0, // dst
 // CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:     // Label 14: @[[L676]]
+// CHECK-NEXT:     // Label 14: @652
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 6: @[[L677]]
+// CHECK-NEXT:     // Label 6: @653
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     }; // Size: {{[0-9]+}} bytes
+// CHECK-NEXT:     }; // Size: 654 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/patfrag-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/patfrag-errors.td
index 6f5c2b93609f..8a3c501a1b4e 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/patfrag-errors.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/patfrag-errors.td
@@ -28,7 +28,7 @@ def too_many_perms : GICombineRule<
          (MatchFooPerms $cst, (i32 0)):$b,
          (MatchFooPerms $cst, (i32 0)):$c
   ),
-  (apply (COPY $dst, (i32 0)), "APPLY ${src}")>;
+  (apply (COPY $dst, (i32 0)))>;
 
 def DummyCXXPF: GICombinePatFrag<
     (outs),
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td
index 4b214d2ca89e..5017672f5cbd 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-errors.td
@@ -270,6 +270,19 @@ def badflagref_in_apply : GICombineRule<
   (match (G_ZEXT $dst, $src):$mi),
   (apply (G_MUL $dst, $src, $src, (MIFlags $impostor)))>;
 
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: 'apply' patterns cannot mix C++ code with other types of patterns
+def mixed_cxx_apply : GICombineRule<
+  (defs root:$dst),
+  (match (G_ZEXT $dst, $src):$mi),
+  (apply (G_MUL $dst, $src, $src), "APPLY")>;
+
+// CHECK: :[[@LINE+2]]:{{[0-9]+}}: error: GIDefMatchData can only be used if 'apply' in entirely written in C++
+def dummy_md : GIDefMatchData<"unsigned">;
+def matchdata_without_cxx_apply : GICombineRule<
+  (defs root:$dst, dummy_md:$md),
+  (match (G_ZEXT $dst, $src):$mi),
+  (apply (G_MUL $dst, $src, $src))>;
+
 // CHECK: error: Failed to parse one or more rules
 
 def MyCombiner: GICombiner<"GenMyCombiner", [
@@ -311,5 +324,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   miflags_in_builtin,
   miflags_in_intrin,
   using_flagref_in_match,
-  badflagref_in_apply
+  badflagref_in_apply,
+  mixed_cxx_apply,
+  matchdata_without_cxx_apply
 ]>;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td
index c261ec457545..85075359df73 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/pattern-parsing.td
@@ -65,7 +65,7 @@ def InstTest0 : GICombineRule<
 
 // CHECK:      (CombineRule name:InstTest1 id:3 root:d
 // CHECK-NEXT:   (MatchDatas
-// CHECK-NEXT:      (MatchDataInfo pattern_symbol:r0 type:'Register' var_name:MDInfo0)
+// CHECK-NEXT:      (MatchDataDef symbol:r0 type:Register)
 // CHECK-NEXT:   )
 // CHECK-NEXT:   (MatchPats
 // CHECK-NEXT:     <match_root>d:(CodeGenInstructionPattern COPY operands:[<def>$a, i32:$b])
@@ -90,7 +90,7 @@ def InstTest1 : GICombineRule<
 
 // CHECK:      (CombineRule name:InstTest2 id:4 root:d
 // CHECK-NEXT:   (MatchDatas
-// CHECK-NEXT:     (MatchDataInfo pattern_symbol:r0 type:'Register' var_name:MDInfo0)
+// CHECK-NEXT:     (MatchDataDef symbol:r0 type:Register)
 // CHECK-NEXT:   )
 // CHECK-NEXT:   (MatchPats
 // CHECK-NEXT:     <match_root>__InstTest2_match_0:(CodeGenInstructionPattern COPY operands:[<def>$d, (i32 0):$x])
@@ -116,7 +116,6 @@ def InstTest2 : GICombineRule<
 // CHECK-NEXT:   )
 // CHECK-NEXT:   (ApplyPats
 // CHECK-NEXT:     <apply_root>__InOutInstTest0_apply_0:(CodeGenInstructionPattern G_TRUNC operands:[<def>$dst, $src])
-// CHECK-NEXT:     __InOutInstTest0_apply_1:(CXXPattern apply code:"APPLY ${src}")
 // CHECK-NEXT:   )
 // CHECK-NEXT:   (OperandTable MatchPats
 // CHECK-NEXT:     dst -> __InOutInstTest0_match_0
@@ -132,7 +131,7 @@ def InOutInstTest0 : GICombineRule<
   (defs root:$dst),
   (match (COPY $dst, $tmp),
          (G_ZEXT $tmp, $src)),
-  (apply (G_TRUNC $dst, $src), "APPLY ${src}")>;
+  (apply (G_TRUNC $dst, $src))>;
 
 def MatchICst: GICombinePatFrag<
     (outs),
@@ -156,7 +155,6 @@ def MatchICst: GICombinePatFrag<
 // CHECK-NEXT:   )
 // CHECK-NEXT:   (ApplyPats
 // CHECK-NEXT:     <apply_root>__PatFragTest0_apply_0:(CodeGenInstructionPattern COPY operands:[<def>$dst, (i32 0)])
-// CHECK-NEXT:     __PatFragTest0_apply_1:(CXXPattern apply code:"APPLY ${src}")
 // CHECK-NEXT:   )
 // CHECK-NEXT:   (OperandTable MatchPats
 // CHECK-NEXT:     cst -> <live-in>
@@ -169,7 +167,7 @@ def MatchICst: GICombinePatFrag<
 def PatFragTest0 : GICombineRule<
   (defs root:$dst),
   (match (G_ZEXT $dst, $cst), (MatchICst $cst, (i32 0))),
-  (apply (COPY $dst, (i32 0)), "APPLY ${src}")>;
+  (apply (COPY $dst, (i32 0)))>;
 
 def MatchFooPerms: GICombinePatFrag<
     (outs),
@@ -205,7 +203,6 @@ def MatchFooPerms: GICombinePatFrag<
 // CHECK-NEXT:   )
 // CHECK-NEXT:   (ApplyPats
 // CHECK-NEXT:     <apply_root>__PatFragTest1_apply_0:(CodeGenInstructionPattern COPY operands:[<def>$dst, (i32 0)])
-// CHECK-NEXT:     __PatFragTest1_apply_1:(CXXPattern apply code:"APPLY ${src}")
 // CHECK-NEXT:   )
 // CHECK-NEXT:   (OperandTable MatchPats
 // CHECK-NEXT:     cst -> <live-in>
@@ -252,7 +249,7 @@ def PatFragTest1 : GICombineRule<
          (MatchFooPerms $cst, (i32 0)):$b,
          (MatchFooPerms $cst, (i32 0)):$c
   ),
-  (apply (COPY $dst, (i32 0)), "APPLY ${src}")>;
+  (apply (COPY $dst, (i32 0)))>;
 
 // CHECK:      (CombineRule name:VariadicsInTest id:8 root:dst
 // CHECK-NEXT:   (MatchPats
diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td
index 23b3d6f59b38..5d5bf92664a7 100644
--- a/llvm/test/TableGen/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter.td
@@ -83,7 +83,7 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; }
 // CHECK-NEXT:    const uint8_t *getMatchTable() const override;
 // CHECK-NEXT:    bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI, const MatcherState &State) const override;
 // CHECK-NEXT:    bool testSimplePredicate(unsigned PredicateID) const override;
-// CHECK-NEXT:    void runCustomAction(unsigned FnID, const MatcherState &State, NewMIVector &OutMIs) const override;
+// CHECK-NEXT:    bool runCustomAction(unsigned FnID, const MatcherState &State, NewMIVector &OutMIs) const override;
 // CHECK-NEXT:  #endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL
 
 // CHECK-LABEL: #ifdef GET_GLOBALISEL_TEMPORARIES_INIT
diff --git a/llvm/test/TableGen/GlobalISelEmitterHwModes.td b/llvm/test/TableGen/GlobalISelEmitterHwModes.td
index 7c8340a154e8..9d235f5f07a7 100644
--- a/llvm/test/TableGen/GlobalISelEmitterHwModes.td
+++ b/llvm/test/TableGen/GlobalISelEmitterHwModes.td
@@ -55,7 +55,7 @@ class I<dag OOps, dag IOps, list<dag> Pat>
 // CHECK-NEXT:    const uint8_t *getMatchTable() const override;
 // CHECK-NEXT:    bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI, const MatcherState &State) const override;
 // CHECK-NEXT:    bool testSimplePredicate(unsigned PredicateID) const override;
-// CHECK-NEXT:    void runCustomAction(unsigned FnID, const MatcherState &State, NewMIVector &OutMIs) const override;
+// CHECK-NEXT:    bool runCustomAction(unsigned FnID, const MatcherState &State, NewMIVector &OutMIs) const override;
 // CHECK-NEXT:  #endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL
 
 // CHECK-LABEL: #ifdef GET_GLOBALISEL_TEMPORARIES_INIT
diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt
index c31ed5a1de69..30f188ae48a2 100644
--- a/llvm/utils/TableGen/Common/CMakeLists.txt
+++ b/llvm/utils/TableGen/Common/CMakeLists.txt
@@ -16,7 +16,6 @@ add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL
   GlobalISel/CXXPredicates.cpp
   GlobalISel/GlobalISelMatchTable.cpp
   GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp
-  GlobalISel/MatchDataInfo.cpp
   GlobalISel/PatternParser.cpp
   GlobalISel/Patterns.cpp
 
diff --git a/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp
index e39293ebfe7a..cbf70278a281 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp
@@ -45,7 +45,7 @@ CXXPredicateCode::CXXPredicateCode(std::string Code, unsigned ID)
     : Code(Code), ID(ID), BaseEnumName("GICombiner" + std::to_string(ID)) {}
 
 CXXPredicateCode::CXXPredicateCodePool CXXPredicateCode::AllCXXMatchCode;
-CXXPredicateCode::CXXPredicateCodePool CXXPredicateCode::AllCXXApplyCode;
+CXXPredicateCode::CXXPredicateCodePool CXXPredicateCode::AllCXXCustomActionCode;
 
 } // namespace gi
 } // namespace llvm
diff --git a/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h
index 01610a13110d..01a4cb46a22a 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h
+++ b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h
@@ -37,7 +37,7 @@ class CXXPredicateCode {
   using CXXPredicateCodePool =
       DenseMap<hash_code, std::unique_ptr<CXXPredicateCode>>;
   static CXXPredicateCodePool AllCXXMatchCode;
-  static CXXPredicateCodePool AllCXXApplyCode;
+  static CXXPredicateCodePool AllCXXCustomActionCode;
 
   /// Sorts a `CXXPredicateCodePool` by their IDs and returns it.
   static std::vector<const CXXPredicateCode *>
@@ -55,16 +55,16 @@ public:
     return get(AllCXXMatchCode, std::move(Code));
   }
 
-  static const CXXPredicateCode &getApplyCode(std::string Code) {
-    return get(AllCXXApplyCode, std::move(Code));
+  static const CXXPredicateCode &getCustomActionCode(std::string Code) {
+    return get(AllCXXCustomActionCode, std::move(Code));
   }
 
   static std::vector<const CXXPredicateCode *> getAllMatchCode() {
     return getSorted(AllCXXMatchCode);
   }
 
-  static std::vector<const CXXPredicateCode *> getAllApplyCode() {
-    return getSorted(AllCXXApplyCode);
+  static std::vector<const CXXPredicateCode *> getAllCustomActionsCode() {
+    return getSorted(AllCXXCustomActionCode);
   }
 
   const std::string Code;
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
index 6d03eecae672..e011e78c67b1 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
@@ -1034,32 +1034,47 @@ void RuleMatcher::emit(MatchTable &Table) {
   for (const auto &PM : EpilogueMatchers)
     PM->emitPredicateOpcodes(Table, *this);
 
-  // Emit all actions except the last one, then emit coverage and emit the
-  // final action.
-  //
-  // This is because some actions, such as GIR_EraseRootFromParent_Done, also
-  // double as a GIR_Done and terminate execution of the rule.
-  if (!Actions.empty()) {
-    for (const auto &MA : drop_end(Actions))
+  if (!CustomCXXAction.empty()) {
+    /// Handle combiners relying on custom C++ code instead of actions.
+    assert(Table.isCombiner() && "CustomCXXAction is only for combiners!");
+    // We cannot have actions other than debug comments.
+    assert(none_of(Actions, [](auto &A) {
+      return A->getKind() != MatchAction::AK_DebugComment;
+    }));
+    for (const auto &MA : Actions)
       MA->emitActionOpcodes(Table, *this);
-  }
-
-  assert((Table.isWithCoverage() ? !Table.isCombiner() : true) &&
-         "Combiner tables don't support coverage!");
-  if (Table.isWithCoverage())
-    Table << MatchTable::Opcode("GIR_Coverage")
-          << MatchTable::IntValue(4, RuleID) << MatchTable::LineBreak;
-  else if (!Table.isCombiner())
-    Table << MatchTable::Comment(("GIR_Coverage, " + Twine(RuleID) + ",").str())
+    Table << MatchTable::Opcode("GIR_DoneWithCustomAction", -1)
+          << MatchTable::Comment("Fn")
+          << MatchTable::NamedValue(2, CustomCXXAction)
           << MatchTable::LineBreak;
+  } else {
+    // Emit all actions except the last one, then emit coverage and emit the
+    // final action.
+    //
+    // This is because some actions, such as GIR_EraseRootFromParent_Done, also
+    // double as a GIR_Done and terminate execution of the rule.
+    if (!Actions.empty()) {
+      for (const auto &MA : drop_end(Actions))
+        MA->emitActionOpcodes(Table, *this);
+    }
+
+    assert((Table.isWithCoverage() ? !Table.isCombiner() : true) &&
+           "Combiner tables don't support coverage!");
+    if (Table.isWithCoverage())
+      Table << MatchTable::Opcode("GIR_Coverage")
+            << MatchTable::IntValue(4, RuleID) << MatchTable::LineBreak;
+    else if (!Table.isCombiner())
+      Table << MatchTable::Comment(
+                   ("GIR_Coverage, " + Twine(RuleID) + ",").str())
+            << MatchTable::LineBreak;
 
-  if (Actions.empty() ||
-      !Actions.back()->emitActionOpcodesAndDone(Table, *this)) {
-    Table << MatchTable::Opcode("GIR_Done", -1) << MatchTable::LineBreak;
+    if (Actions.empty() ||
+        !Actions.back()->emitActionOpcodesAndDone(Table, *this)) {
+      Table << MatchTable::Opcode("GIR_Done", -1) << MatchTable::LineBreak;
+    }
   }
 
   Table << MatchTable::Label(LabelID);
-
   ++NumPatternEmitted;
 }
 
@@ -2108,14 +2123,6 @@ void CustomOperandRenderer::emitRenderOpcodes(MatchTable &Table,
         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
 }
 
-//===- CustomCXXAction ----------------------------------------------------===//
-
-void CustomCXXAction::emitActionOpcodes(MatchTable &Table,
-                                        RuleMatcher &Rule) const {
-  Table << MatchTable::Opcode("GIR_CustomAction")
-        << MatchTable::NamedValue(2, FnEnumName) << MatchTable::LineBreak;
-}
-
 //===- BuildMIAction ------------------------------------------------------===//
 
 bool BuildMIAction::canMutate(RuleMatcher &Rule,
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
index 30301c28ce6c..5fe3f9a32c01 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
@@ -473,6 +473,10 @@ protected:
   /// have succeeded.
   ActionList Actions;
 
+  /// Combiners can sometimes just run C++ code to finish matching a rule &
+  /// mutate instructions instead of relying on MatchActions. Empty if unused.
+  std::string CustomCXXAction;
+
   using DefinedInsnVariablesMap = std::map<InstructionMatcher *, unsigned>;
 
   /// A map of instruction matchers to the local variables
@@ -563,6 +567,10 @@ public:
   /// should NOT be emitted.
   bool tryEraseInsnID(unsigned ID) { return ErasedInsnIDs.insert(ID).second; }
 
+  void setCustomCXXAction(StringRef FnEnumName) {
+    CustomCXXAction = FnEnumName.str();
+  }
+
   // Emplaces an action of the specified Kind at the end of the action list.
   //
   // Returns a reference to the newly created action.
@@ -2206,7 +2214,6 @@ class MatchAction {
 public:
   enum ActionKind {
     AK_DebugComment,
-    AK_CustomCXX,
     AK_BuildMI,
     AK_BuildConstantMI,
     AK_EraseInst,
@@ -2261,20 +2268,6 @@ public:
   }
 };
 
-class CustomCXXAction : public MatchAction {
-  std::string FnEnumName;
-
-public:
-  CustomCXXAction(StringRef FnEnumName)
-      : MatchAction(AK_CustomCXX), FnEnumName(FnEnumName.str()) {}
-
-  static bool classof(const MatchAction *A) {
-    return A->getKind() == AK_CustomCXX;
-  }
-
-  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
-};
-
 /// Generates code to build an instruction or mutate an existing instruction
 /// into the desired instruction when this is possible.
 class BuildMIAction : public MatchAction {
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp
index 5697899a915a..331da97a6f7c 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp
@@ -236,11 +236,10 @@ void GlobalISelMatchTableExecutorEmitter::emitTemporariesDecl(
         ", const MatcherState &State) "
         "const override;\n"
      << "  bool testSimplePredicate(unsigned PredicateID) const override;\n"
-     << "  void runCustomAction(unsigned FnID, const MatcherState &State, "
+     << "  bool runCustomAction(unsigned FnID, const MatcherState &State, "
         "NewMIVector &OutMIs) "
-        "const override;\n";
-  emitAdditionalTemporariesDecl(OS, "  ");
-  OS << "#endif // ifdef " << IfDefName << "\n\n";
+        "const override;\n"
+     << "#endif // ifdef " << IfDefName << "\n\n";
 }
 
 void GlobalISelMatchTableExecutorEmitter::emitTemporariesInit(
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h
index d2b6a74c7577..13dcfc0e55e1 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h
@@ -182,10 +182,6 @@ public:
   /// Emit additional content in emitExecutorImpl
   virtual void emitAdditionalImpl(raw_ostream &OS) {}
 
-  /// Emit additional content in emitTemporariesDecl.
-  virtual void emitAdditionalTemporariesDecl(raw_ostream &OS,
-                                             StringRef Indent) {}
-
   /// Emit additional content in emitTemporariesInit.
   virtual void emitAdditionalTemporariesInit(raw_ostream &OS) {}
 
diff --git a/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp
deleted file mode 100644
index b5c9e4f8c248..000000000000
--- a/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp
+++ /dev/null
@@ -1,49 +0,0 @@
-//===- MatchDataInfo.cpp ----------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//
-//===----------------------------------------------------------------------===//
-
-#include "MatchDataInfo.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-
-namespace llvm {
-namespace gi {
-
-StringMap<std::vector<std::string>> AllMatchDataVars;
-
-StringRef MatchDataInfo::getVariableName() const {
-  assert(hasVariableName());
-  return VarName;
-}
-
-void MatchDataInfo::print(raw_ostream &OS) const {
-  OS << "(MatchDataInfo pattern_symbol:" << PatternSymbol << " type:'" << Type
-     << "' var_name:" << (VarName.empty() ? "<unassigned>" : VarName) << ")";
-}
-
-void MatchDataInfo::dump() const { print(dbgs()); }
-
-void AssignMatchDataVariables(MutableArrayRef<MatchDataInfo> Infos) {
-  static unsigned NextVarID = 0;
-
-  StringMap<unsigned> SeenTypes;
-  for (auto &Info : Infos) {
-    unsigned &NumSeen = SeenTypes[Info.getType()];
-    auto &ExistingVars = AllMatchDataVars[Info.getType()];
-
-    if (NumSeen == ExistingVars.size())
-      ExistingVars.push_back("MDInfo" + std::to_string(NextVarID++));
-
-    Info.setVariableName(ExistingVars[NumSeen++]);
-  }
-}
-
-} // namespace gi
-} // namespace llvm
diff --git a/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h
deleted file mode 100644
index abe1245bc67d..000000000000
--- a/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h
+++ /dev/null
@@ -1,90 +0,0 @@
-//===- MatchDataInfo.h ------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file Contains utilities related to handling "match data" for GlobalISel
-///  Combiners. Match data allows for setting some arbitrary data in the "match"
-///  phase and pass it down to the "apply" phase.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_UTILS_MIRPATTERNS_MATCHDATAINFO_H
-#define LLVM_UTILS_MIRPATTERNS_MATCHDATAINFO_H
-
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/StringRef.h"
-#include <string>
-#include <vector>
-
-namespace llvm {
-
-class raw_ostream;
-
-namespace gi {
-
-/// Represents MatchData defined by the match stage and required by the apply
-/// stage.
-///
-/// This allows the plumbing of arbitrary data from C++ predicates between the
-/// stages.
-///
-/// When this class is initially created, it only has a pattern symbol and a
-/// type. When all of the MatchDatas declarations of a given pattern have been
-/// parsed, `AssignVariables` must be called to assign storage variable names to
-/// each MatchDataInfo.
-class MatchDataInfo {
-  StringRef PatternSymbol;
-  StringRef Type;
-  std::string VarName;
-
-public:
-  static constexpr StringLiteral StructTypeName = "MatchInfosTy";
-  static constexpr StringLiteral StructName = "MatchInfos";
-
-  MatchDataInfo(StringRef PatternSymbol, StringRef Type)
-      : PatternSymbol(PatternSymbol), Type(Type.trim()) {}
-
-  StringRef getPatternSymbol() const { return PatternSymbol; };
-  StringRef getType() const { return Type; };
-
-  bool hasVariableName() const { return !VarName.empty(); }
-  void setVariableName(StringRef Name) { VarName = Name; }
-  StringRef getVariableName() const;
-
-  std::string getQualifiedVariableName() const {
-    return StructName.str() + "." + getVariableName().str();
-  }
-
-  void print(raw_ostream &OS) const;
-  void dump() const;
-};
-
-/// Pool of type -> variables used to emit MatchData variables declarations.
-///
-/// e.g. if the map contains "int64_t" -> ["MD0", "MD1"], then two variable
-/// declarations must be emitted: `int64_t MD0` and `int64_t MD1`.
-///
-/// This has a static lifetime and will outlive all the `MatchDataInfo` objects
-/// by design. It needs a static lifetime so the backends can emit variable
-/// declarations after processing all the inputs.
-extern StringMap<std::vector<std::string>> AllMatchDataVars;
-
-/// Assign variable names to all MatchDatas used by a pattern. This must be
-/// called after all MatchData decls have been parsed for a given processing
-/// unit (e.g. a combine rule)
-///
-/// Requires an array of MatchDataInfo so we can handle cases where a pattern
-/// uses multiple instances of the same MatchData type.
-///
-/// Writes to \ref AllMatchDataVars.
-void AssignMatchDataVariables(MutableArrayRef<MatchDataInfo> Infos);
-
-} // namespace gi
-} // end namespace llvm
-
-#endif // ifndef LLVM_UTILS_MIRPATTERNS_MATCHDATAINFO_H
diff --git a/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp
index 388bf7e9e833..28a9bcc2568c 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp
@@ -145,6 +145,8 @@ CXXPattern::CXXPattern(const StringInit &Code, StringRef Name)
 const CXXPredicateCode &
 CXXPattern::expandCode(const CodeExpansions &CE, ArrayRef<SMLoc> Locs,
                        function_ref<void(raw_ostream &)> AddComment) const {
+  assert(!IsApply && "'apply' CXX patterns should be handled differently!");
+
   std::string Result;
   raw_string_ostream OS(Result);
 
@@ -153,8 +155,6 @@ CXXPattern::expandCode(const CodeExpansions &CE, ArrayRef<SMLoc> Locs,
 
   CodeExpander Expander(RawCode, CE, Locs, /*ShowExpansions*/ false);
   Expander.emit(OS);
-  if (IsApply)
-    return CXXPredicateCode::getApplyCode(std::move(Result));
   return CXXPredicateCode::getMatchCode(std::move(Result));
 }
 
diff --git a/llvm/utils/TableGen/Common/GlobalISel/Patterns.h b/llvm/utils/TableGen/Common/GlobalISel/Patterns.h
index dac092556548..76d018bdbd71 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/Patterns.h
+++ b/llvm/utils/TableGen/Common/GlobalISel/Patterns.h
@@ -205,6 +205,10 @@ public:
   /// Expands raw code, replacing things such as `${foo}` with their
   /// substitution in \p CE.
   ///
+  /// Can only be used on 'match' CXX Patterns. 'apply' CXX pattern emission
+  /// is handled differently as we emit both the 'match' and 'apply' part
+  /// together in a single Custom CXX Action.
+  ///
   /// \param CE     Map of Code Expansions
   /// \param Locs   SMLocs for the Code Expander, in case it needs to emit
   ///               diagnostics.
diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
index ef9e9ff04f85..085841ed963e 100644
--- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
@@ -35,7 +35,6 @@
 #include "Common/GlobalISel/CombinerUtils.h"
 #include "Common/GlobalISel/GlobalISelMatchTable.h"
 #include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
-#include "Common/GlobalISel/MatchDataInfo.h"
 #include "Common/GlobalISel/PatternParser.h"
 #include "Common/GlobalISel/Patterns.h"
 #include "Common/SubtargetFeatureInfo.h"
@@ -45,6 +44,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -79,8 +79,9 @@ cl::opt<bool> DebugTypeInfer("gicombiner-debug-typeinfer",
                              cl::desc("Print type inference debug logs"),
                              cl::cat(GICombinerEmitterCat));
 
-constexpr StringLiteral CXXApplyPrefix = "GICXXCustomAction_CombineApply";
+constexpr StringLiteral CXXCustomActionPrefix = "GICXXCustomAction_";
 constexpr StringLiteral CXXPredPrefix = "GICXXPred_MI_Predicate_";
+constexpr StringLiteral MatchDataClassName = "GIDefMatchData";
 
 //===- CodeExpansions Helpers  --------------------------------------------===//
 
@@ -605,6 +606,21 @@ CombineRuleOperandTypeChecker::getRuleEqClasses() const {
   return TECs;
 }
 
+//===- MatchData Handling -------------------------------------------------===//
+struct MatchDataDef {
+  MatchDataDef(StringRef Symbol, StringRef Type) : Symbol(Symbol), Type(Type) {}
+
+  StringRef Symbol;
+  StringRef Type;
+
+  /// \returns the desired variable name for this MatchData.
+  std::string getVarName() const {
+    // Add a prefix in case the symbol name is very generic and conflicts with
+    // something else.
+    return "GIMatchData_" + Symbol.str();
+  }
+};
+
 //===- CombineRuleBuilder -------------------------------------------------===//
 
 /// Parses combine rule and builds a small intermediate representation to tie
@@ -671,10 +687,6 @@ private:
   void addCXXPredicate(RuleMatcher &M, const CodeExpansions &CE,
                        const CXXPattern &P, const PatternAlternatives &Alts);
 
-  /// Adds an apply \p P to \p IM, expanding its code using \p CE.
-  void addCXXAction(RuleMatcher &M, const CodeExpansions &CE,
-                    const CXXPattern &P);
-
   bool hasOnlyCXXApplyPatterns() const;
   bool hasEraseRoot() const;
 
@@ -721,6 +733,8 @@ private:
                                DenseSet<const Pattern *> &SeenPats);
 
   bool emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M);
+  bool emitCXXMatchApply(CodeExpansions &CE, RuleMatcher &M,
+                         ArrayRef<CXXPattern *> Matchers);
 
   // Recursively visits InstructionPatterns from P to build up the
   // RuleMatcher actions.
@@ -777,7 +791,7 @@ private:
   Pattern *MatchRoot = nullptr;
   SmallDenseSet<InstructionPattern *, 2> ApplyRoots;
 
-  SmallVector<MatchDataInfo, 2> MatchDatas;
+  SmallVector<MatchDataDef, 2> MatchDatas;
   SmallVector<PatternAlternatives, 1> PermutationsToEmit;
 };
 
@@ -811,7 +825,6 @@ bool CombineRuleBuilder::emitRuleMatchers() {
 
   assert(MatchRoot);
   CodeExpansions CE;
-  declareAllMatchDatasExpansions(CE);
 
   assert(!PermutationsToEmit.empty());
   for (const auto &Alts : PermutationsToEmit) {
@@ -845,9 +858,8 @@ void CombineRuleBuilder::print(raw_ostream &OS) const {
   if (!MatchDatas.empty()) {
     OS << "  (MatchDatas\n";
     for (const auto &MD : MatchDatas) {
-      OS << "    ";
-      MD.print(OS);
-      OS << '\n';
+      OS << "    (MatchDataDef symbol:" << MD.Symbol << " type:" << MD.Type
+         << ")\n";
     }
     OS << "  )\n";
   }
@@ -1008,7 +1020,7 @@ bool CombineRuleBuilder::addMatchPattern(std::unique_ptr<Pattern> Pat) {
 void CombineRuleBuilder::declareAllMatchDatasExpansions(
     CodeExpansions &CE) const {
   for (const auto &MD : MatchDatas)
-    CE.declare(MD.getPatternSymbol(), MD.getQualifiedVariableName());
+    CE.declare(MD.Symbol, MD.getVarName());
 }
 
 void CombineRuleBuilder::addCXXPredicate(RuleMatcher &M,
@@ -1030,13 +1042,6 @@ void CombineRuleBuilder::addCXXPredicate(RuleMatcher &M,
       ExpandedCode.getEnumNameWithPrefix(CXXPredPrefix));
 }
 
-void CombineRuleBuilder::addCXXAction(RuleMatcher &M, const CodeExpansions &CE,
-                                      const CXXPattern &P) {
-  const auto &ExpandedCode = P.expandCode(CE, RuleDef.getLoc());
-  M.addAction<CustomCXXAction>(
-      ExpandedCode.getEnumNameWithPrefix(CXXApplyPrefix));
-}
-
 bool CombineRuleBuilder::hasOnlyCXXApplyPatterns() const {
   return all_of(ApplyPats, [&](auto &Entry) {
     return isa<CXXPattern>(Entry.second.get());
@@ -1175,9 +1180,20 @@ bool CombineRuleBuilder::checkSemantics() {
     UsesWipMatchOpcode = true;
   }
 
+  std::optional<bool> IsUsingCXXPatterns;
   for (const auto &Apply : ApplyPats) {
-    assert(Apply.second.get());
-    const auto *IP = dyn_cast<InstructionPattern>(Apply.second.get());
+    Pattern *Pat = Apply.second.get();
+    if (IsUsingCXXPatterns) {
+      if (*IsUsingCXXPatterns != isa<CXXPattern>(Pat)) {
+        PrintError("'apply' patterns cannot mix C++ code with other types of "
+                   "patterns");
+        return false;
+      }
+    } else
+      IsUsingCXXPatterns = isa<CXXPattern>(Pat);
+
+    assert(Pat);
+    const auto *IP = dyn_cast<InstructionPattern>(Pat);
     if (!IP)
       continue;
 
@@ -1260,6 +1276,12 @@ bool CombineRuleBuilder::checkSemantics() {
     }
   }
 
+  if (!hasOnlyCXXApplyPatterns() && !MatchDatas.empty()) {
+    PrintError(MatchDataClassName +
+               " can only be used if 'apply' in entirely written in C++");
+    return false;
+  }
+
   return true;
 }
 
@@ -1440,7 +1462,7 @@ bool CombineRuleBuilder::parseDefs(const DagInit &Def) {
     // data from the match stage to the apply stage, and ensure that the
     // generated matcher has a suitable variable for it to do so.
     if (Record *MatchDataRec =
-            getDefOfSubClass(*Def.getArg(I), "GIDefMatchData")) {
+            getDefOfSubClass(*Def.getArg(I), MatchDataClassName)) {
       MatchDatas.emplace_back(Def.getArgNameStr(I),
                               MatchDataRec->getValueAsString("Type"));
       continue;
@@ -1463,9 +1485,6 @@ bool CombineRuleBuilder::parseDefs(const DagInit &Def) {
   }
 
   RootName = Roots.front();
-
-  // Assign variables to all MatchDatas.
-  AssignMatchDataVariables(MatchDatas);
   return true;
 }
 
@@ -1502,6 +1521,8 @@ bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
     llvm_unreachable("Unknown kind of InstructionPattern!");
 
   // Emit remaining patterns
+  const bool IsUsingCustomCXXAction = hasOnlyCXXApplyPatterns();
+  SmallVector<CXXPattern *, 2> CXXMatchers;
   for (auto &Pat : values(MatchPats)) {
     if (SeenPats.contains(Pat.get()))
       continue;
@@ -1523,7 +1544,11 @@ bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
       cast<InstructionPattern>(Pat.get())->reportUnreachable(RuleDef.getLoc());
       return false;
     case Pattern::K_CXX: {
-      addCXXPredicate(M, CE, *cast<CXXPattern>(Pat.get()), Alts);
+      // Delay emission for top-level C++ matchers (which can use MatchDatas).
+      if (IsUsingCustomCXXAction)
+        CXXMatchers.push_back(cast<CXXPattern>(Pat.get()));
+      else
+        addCXXPredicate(M, CE, *cast<CXXPattern>(Pat.get()), Alts);
       continue;
     }
     default:
@@ -1531,7 +1556,8 @@ bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
     }
   }
 
-  return emitApplyPatterns(CE, M);
+  return IsUsingCustomCXXAction ? emitCXXMatchApply(CE, M, CXXMatchers)
+                                : emitApplyPatterns(CE, M);
 }
 
 bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
@@ -1539,6 +1565,7 @@ bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
                                           const AnyOpcodePattern &AOP) {
   auto StackTrace = PrettyStackTraceEmit(RuleDef, &AOP);
 
+  const bool IsUsingCustomCXXAction = hasOnlyCXXApplyPatterns();
   for (const CodeGenInstruction *CGI : AOP.insts()) {
     auto &M = addRuleMatcher(Alts, "wip_match_opcode '" +
                                        CGI->TheDef->getName() + "'");
@@ -1552,6 +1579,7 @@ bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
     IM.addPredicate<InstructionOpcodeMatcher>(CGI);
 
     // Emit remaining patterns.
+    SmallVector<CXXPattern *, 2> CXXMatchers;
     for (auto &Pat : values(MatchPats)) {
       if (Pat.get() == &AOP)
         continue;
@@ -1576,7 +1604,11 @@ bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
             RuleDef.getLoc());
         return false;
       case Pattern::K_CXX: {
-        addCXXPredicate(M, CE, *cast<CXXPattern>(Pat.get()), Alts);
+        // Delay emission for top-level C++ matchers (which can use MatchDatas).
+        if (IsUsingCustomCXXAction)
+          CXXMatchers.push_back(cast<CXXPattern>(Pat.get()));
+        else
+          addCXXPredicate(M, CE, *cast<CXXPattern>(Pat.get()), Alts);
         break;
       }
       default:
@@ -1584,7 +1616,10 @@ bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
       }
     }
 
-    if (!emitApplyPatterns(CE, M))
+    const bool Res = IsUsingCustomCXXAction
+                         ? emitCXXMatchApply(CE, M, CXXMatchers)
+                         : emitApplyPatterns(CE, M);
+    if (!Res)
       return false;
   }
 
@@ -1727,11 +1762,7 @@ bool CombineRuleBuilder::emitPatFragMatchPattern(
 }
 
 bool CombineRuleBuilder::emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M) {
-  if (hasOnlyCXXApplyPatterns()) {
-    for (auto &Pat : values(ApplyPats))
-      addCXXAction(M, CE, *cast<CXXPattern>(Pat.get()));
-    return true;
-  }
+  assert(MatchDatas.empty());
 
   DenseSet<const Pattern *> SeenPats;
   StringMap<unsigned> OperandToTempRegID;
@@ -1764,8 +1795,8 @@ bool CombineRuleBuilder::emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M) {
       cast<CodeGenInstructionPattern>(*Pat).reportUnreachable(RuleDef.getLoc());
       return false;
     case Pattern::K_CXX: {
-      addCXXAction(M, CE, *cast<CXXPattern>(Pat.get()));
-      continue;
+      llvm_unreachable(
+          "CXX Pattern Emission should have been handled earlier!");
     }
     default:
       llvm_unreachable("unknown pattern kind!");
@@ -1780,6 +1811,44 @@ bool CombineRuleBuilder::emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M) {
   return true;
 }
 
+bool CombineRuleBuilder::emitCXXMatchApply(CodeExpansions &CE, RuleMatcher &M,
+                                           ArrayRef<CXXPattern *> Matchers) {
+  assert(hasOnlyCXXApplyPatterns());
+  declareAllMatchDatasExpansions(CE);
+
+  std::string CodeStr;
+  raw_string_ostream OS(CodeStr);
+
+  for (auto &MD : MatchDatas)
+    OS << MD.Type << " " << MD.getVarName() << ";\n";
+
+  if (!Matchers.empty()) {
+    OS << "// Match Patterns\n";
+    for (auto *M : Matchers) {
+      OS << "if(![&](){";
+      CodeExpander Expander(M->getRawCode(), CE, RuleDef.getLoc(),
+                            /*ShowExpansions=*/false);
+      Expander.emit(OS);
+      OS << "}()) {\n"
+         << "  return false;\n}\n";
+    }
+  }
+
+  OS << "// Apply Patterns\n";
+  ListSeparator LS("\n");
+  for (auto &Pat : ApplyPats) {
+    auto *CXXPat = cast<CXXPattern>(Pat.second.get());
+    CodeExpander Expander(CXXPat->getRawCode(), CE, RuleDef.getLoc(),
+                          /*ShowExpansions=*/ false);
+    OS << LS;
+    Expander.emit(OS);
+  }
+
+  const auto &Code = CXXPredicateCode::getCustomActionCode(CodeStr);
+  M.setCustomCXXAction(Code.getEnumNameWithPrefix(CXXCustomActionPrefix));
+  return true;
+}
+
 bool CombineRuleBuilder::emitInstructionApplyPattern(
     CodeExpansions &CE, RuleMatcher &M, const InstructionPattern &P,
     DenseSet<const Pattern *> &SeenPats,
@@ -2226,9 +2295,6 @@ class GICombinerEmitter final : public GlobalISelMatchTableExecutorEmitter {
   void emitTestSimplePredicate(raw_ostream &OS) override;
   void emitRunCustomAction(raw_ostream &OS) override;
 
-  void emitAdditionalTemporariesDecl(raw_ostream &OS,
-                                     StringRef Indent) override;
-
   const CodeGenTarget &getTarget() const override { return Target; }
   StringRef getClassName() const override {
     return Combiner->getValueAsString("Classname");
@@ -2371,8 +2437,6 @@ void GICombinerEmitter::emitAdditionalImpl(raw_ostream &OS) {
      << "  B.setInstrAndDebugLoc(I);\n"
      << "  State.MIs.clear();\n"
      << "  State.MIs.push_back(&I);\n"
-     << "  " << MatchDataInfo::StructName << " = "
-     << MatchDataInfo::StructTypeName << "();\n\n"
      << "  if (executeMatchTable(*this, State, ExecInfo, B"
      << ", getMatchTable(), *ST.getInstrInfo(), MRI, "
         "*MRI.getTargetRegisterInfo(), *ST.getRegBankInfo(), AvailableFeatures"
@@ -2438,50 +2502,38 @@ void GICombinerEmitter::emitTestSimplePredicate(raw_ostream &OS) {
 }
 
 void GICombinerEmitter::emitRunCustomAction(raw_ostream &OS) {
-  const auto ApplyCode = CXXPredicateCode::getAllApplyCode();
+  const auto CustomActionsCode = CXXPredicateCode::getAllCustomActionsCode();
 
-  if (!ApplyCode.empty()) {
+  if (!CustomActionsCode.empty()) {
     OS << "enum {\n";
     std::string EnumeratorSeparator = " = GICXXCustomAction_Invalid + 1,\n";
-    for (const auto &Apply : ApplyCode) {
-      OS << "  " << Apply->getEnumNameWithPrefix(CXXApplyPrefix)
+    for (const auto &CA : CustomActionsCode) {
+      OS << "  " << CA->getEnumNameWithPrefix(CXXCustomActionPrefix)
          << EnumeratorSeparator;
       EnumeratorSeparator = ",\n";
     }
     OS << "};\n";
   }
 
-  OS << "void " << getClassName()
+  OS << "bool " << getClassName()
      << "::runCustomAction(unsigned ApplyID, const MatcherState &State, "
         "NewMIVector &OutMIs) const "
-        "{\n";
-  if (!ApplyCode.empty()) {
+        "{\n  Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]);\n";
+  if (!CustomActionsCode.empty()) {
     OS << "  switch(ApplyID) {\n";
-    for (const auto &Apply : ApplyCode) {
-      OS << "  case " << Apply->getEnumNameWithPrefix(CXXApplyPrefix) << ":{\n"
-         << "    Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]);\n"
-         << "    " << join(split(Apply->Code, '\n'), "\n    ") << '\n'
-         << "    return;\n";
+    for (const auto &CA : CustomActionsCode) {
+      OS << "  case " << CA->getEnumNameWithPrefix(CXXCustomActionPrefix)
+         << ":{\n"
+         << "    " << join(split(CA->Code, '\n'), "\n    ") << '\n'
+         << "    return true;\n";
       OS << "  }\n";
     }
-    OS << "}\n";
+    OS << "  }\n";
   }
   OS << "  llvm_unreachable(\"Unknown Apply Action\");\n"
      << "}\n";
 }
 
-void GICombinerEmitter::emitAdditionalTemporariesDecl(raw_ostream &OS,
-                                                      StringRef Indent) {
-  OS << Indent << "struct " << MatchDataInfo::StructTypeName << " {\n";
-  for (const auto &[Type, VarNames] : AllMatchDataVars) {
-    assert(!VarNames.empty() && "Cannot have no vars for this type!");
-    OS << Indent << "  " << Type << " " << join(VarNames, ", ") << ";\n";
-  }
-  OS << Indent << "};\n"
-     << Indent << "mutable " << MatchDataInfo::StructTypeName << " "
-     << MatchDataInfo::StructName << ";\n\n";
-}
-
 GICombinerEmitter::GICombinerEmitter(RecordKeeper &RK,
                                      const CodeGenTarget &Target,
                                      StringRef Name, Record *Combiner)
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index cf7e4398741c..9b356148cc17 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -2353,7 +2353,7 @@ void GlobalISelEmitter::emitTestSimplePredicate(raw_ostream &OS) {
 }
 
 void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) {
-  OS << "void " << getClassName()
+  OS << "bool " << getClassName()
      << "::runCustomAction(unsigned, const MatcherState&, NewMIVector &) const "
         "{\n"
      << "    llvm_unreachable(\"" + getClassName() +
-- 
GitLab


From cf92e51f5b66afc315f2b43c238291ca87a5602e Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Thu, 16 May 2024 11:40:02 +0000
Subject: [PATCH 0999/1206] [gn build] Port 7d81062352f7

---
 llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn
index daa3278d56d7..9e19ec9df4d3 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn
@@ -21,7 +21,6 @@ static_library("Common") {
     "GlobalISel/CombinerUtils.cpp",
     "GlobalISel/GlobalISelMatchTable.cpp",
     "GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp",
-    "GlobalISel/MatchDataInfo.cpp",
     "GlobalISel/PatternParser.cpp",
     "GlobalISel/Patterns.cpp",
     "InfoByHwMode.cpp",
-- 
GitLab


From 311339e25cd4d431902f93b1cbc6b67ac3fd2abf Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 16 May 2024 13:05:22 +0100
Subject: [PATCH 1000/1206] [DAG] SimplifyDemandedBits - ISD::AND - only
 request DemandedElts when looking for a splat constant

Limit the isConstOrConstSplat call to the vector elements we care about

Noticed while investigating regressions in #92096
---
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |   2 +-
 .../Thumb2/mve-laneinterleaving-cost.ll       | 138 +++++++++---------
 2 files changed, 66 insertions(+), 74 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 9ddb14e11dab..38583de03d9c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -1380,7 +1380,7 @@ bool TargetLowering::SimplifyDemandedBits(
     // using the bits from the RHS.  Below, we use knowledge about the RHS to
     // simplify the LHS, here we're using information from the LHS to simplify
     // the RHS.
-    if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
+    if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1, DemandedElts)) {
       // Do not increment Depth here; that can cause an infinite loop.
       KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
       // If the LHS already has zeros where RHSC does, this 'and' is dead.
diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
index 203ce1f88118..c2511a4992cf 100644
--- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll
@@ -6,39 +6,36 @@ define arm_aapcs_vfpcc <4 x i32> @loads_i32(ptr %A, ptr %B, ptr %C) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vmov.i64 q2, #0xffffffff
-; CHECK-NEXT:    vmov.f32 s0, s6
-; CHECK-NEXT:    vmov.f32 s2, s7
-; CHECK-NEXT:    vand q0, q0, q2
-; CHECK-NEXT:    vmov.f32 s6, s5
-; CHECK-NEXT:    vmov r4, r5, d0
-; CHECK-NEXT:    vmov r3, r1, d1
+; CHECK-NEXT:    vldrw.u32 q2, [r1]
+; CHECK-NEXT:    vmov.i64 q1, #0xffffffff
+; CHECK-NEXT:    vmov.f32 s0, s10
+; CHECK-NEXT:    vmov.f32 s2, s11
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    vmov.f32 s10, s9
+; CHECK-NEXT:    vmov r3, r4, d0
+; CHECK-NEXT:    vand q2, q2, q1
+; CHECK-NEXT:    vmov r5, r1, d1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vmov lr, r12, d5
 ; CHECK-NEXT:    vmov.f32 s12, s2
 ; CHECK-NEXT:    vmov.f32 s2, s3
 ; CHECK-NEXT:    vmov r0, s12
-; CHECK-NEXT:    vand q3, q1, q2
-; CHECK-NEXT:    vldrw.u32 q1, [r2]
-; CHECK-NEXT:    vmov lr, r12, d7
-; CHECK-NEXT:    vmov.f32 s16, s6
-; CHECK-NEXT:    vmov.f32 s18, s7
-; CHECK-NEXT:    vand q2, q4, q2
+; CHECK-NEXT:    vmov.f32 s12, s6
+; CHECK-NEXT:    vmov.f32 s6, s7
 ; CHECK-NEXT:    asrs r2, r0, #31
-; CHECK-NEXT:    adds r0, r0, r4
-; CHECK-NEXT:    adcs r5, r2
-; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    asrl r0, r5, r2
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adc.w r3, r2, r4
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    asrl r0, r3, r2
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    vmov.f32 s2, s1
-; CHECK-NEXT:    asrs r4, r2, #31
-; CHECK-NEXT:    adds r2, r2, r3
-; CHECK-NEXT:    adcs r1, r4
-; CHECK-NEXT:    vmov r3, s10
+; CHECK-NEXT:    asrs r3, r2, #31
+; CHECK-NEXT:    adds r2, r2, r5
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vmov r3, s6
 ; CHECK-NEXT:    asrl r2, r1, r3
-; CHECK-NEXT:    vmov r4, r5, d6
+; CHECK-NEXT:    vmov r4, r5, d4
 ; CHECK-NEXT:    vmov r1, s2
 ; CHECK-NEXT:    vmov.f32 s2, s5
 ; CHECK-NEXT:    adds.w r6, r1, lr
@@ -54,7 +51,6 @@ define arm_aapcs_vfpcc <4 x i32> @loads_i32(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    asrl r4, r1, r3
 ; CHECK-NEXT:    vmov q0[2], q0[0], r4, r0
 ; CHECK-NEXT:    vmov q0[3], q0[1], r6, r2
-; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %a = load <4 x i32>, ptr %A, align 4
@@ -138,62 +134,58 @@ entry:
 define arm_aapcs_vfpcc void @load_store_i32(ptr %A, ptr %B, ptr %C, ptr %D) {
 ; CHECK-LABEL: load_store_i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    vmov.i64 q4, #0xffffffff
-; CHECK-NEXT:    vmov.f32 s4, s2
-; CHECK-NEXT:    vmov.f32 s2, s1
-; CHECK-NEXT:    vmov.f32 s6, s3
-; CHECK-NEXT:    vand q2, q0, q4
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    .vsave {d8}
+; CHECK-NEXT:    vpush {d8}
+; CHECK-NEXT:    vldrw.u32 q2, [r1]
+; CHECK-NEXT:    vmov.i64 q0, #0xffffffff
+; CHECK-NEXT:    vmov.f32 s4, s10
+; CHECK-NEXT:    vmov.f32 s6, s11
+; CHECK-NEXT:    vmov.f32 s10, s9
+; CHECK-NEXT:    vand q1, q1, q0
+; CHECK-NEXT:    vand q2, q2, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vand q1, q1, q4
-; CHECK-NEXT:    vmov r5, r1, d3
+; CHECK-NEXT:    vmov r6, r4, d3
 ; CHECK-NEXT:    vmov.f32 s12, s2
 ; CHECK-NEXT:    vmov.f32 s2, s3
-; CHECK-NEXT:    vmov r0, r12, d2
+; CHECK-NEXT:    vmov lr, r12, d2
 ; CHECK-NEXT:    vldrw.u32 q1, [r2]
-; CHECK-NEXT:    vmov r4, lr, d5
-; CHECK-NEXT:    vmov.f32 s20, s6
-; CHECK-NEXT:    vmov.f32 s6, s1
-; CHECK-NEXT:    vmov.f32 s22, s7
-; CHECK-NEXT:    vand q4, q5, q4
-; CHECK-NEXT:    vmov r6, s2
+; CHECK-NEXT:    vmov r5, r1, d5
+; CHECK-NEXT:    vmov.f32 s16, s6
+; CHECK-NEXT:    vmov.f32 s6, s7
+; CHECK-NEXT:    vmov.f32 s10, s1
+; CHECK-NEXT:    vmov r0, s2
 ; CHECK-NEXT:    vmov.f32 s2, s5
-; CHECK-NEXT:    adds r2, r6, r5
-; CHECK-NEXT:    vmov r5, s18
-; CHECK-NEXT:    asr.w r7, r6, #31
-; CHECK-NEXT:    adcs r1, r7
-; CHECK-NEXT:    asrl r2, r1, r5
-; CHECK-NEXT:    vmov r7, s2
-; CHECK-NEXT:    vmov r1, s6
-; CHECK-NEXT:    adds r4, r4, r1
-; CHECK-NEXT:    asr.w r5, r1, #31
-; CHECK-NEXT:    adc.w r1, r5, lr
-; CHECK-NEXT:    asrl r4, r1, r7
-; CHECK-NEXT:    vmov r6, r5, d4
+; CHECK-NEXT:    adds.w r8, r0, r6
+; CHECK-NEXT:    asr.w r2, r0, #31
+; CHECK-NEXT:    adc.w r7, r2, r4
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    asrl r8, r7, r2
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    asrs r4, r2, #31
+; CHECK-NEXT:    adds r2, r2, r5
+; CHECK-NEXT:    adcs r1, r4
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    asrl r2, r1, r4
+; CHECK-NEXT:    vmov r5, r7, d4
 ; CHECK-NEXT:    vmov r1, s12
-; CHECK-NEXT:    adds r0, r0, r1
-; CHECK-NEXT:    asr.w r7, r1, #31
-; CHECK-NEXT:    adc.w r1, r7, r12
-; CHECK-NEXT:    vmov r7, s16
-; CHECK-NEXT:    asrl r0, r1, r7
+; CHECK-NEXT:    adds.w r6, r1, lr
+; CHECK-NEXT:    asr.w r4, r1, #31
+; CHECK-NEXT:    adc.w r1, r4, r12
+; CHECK-NEXT:    vmov r4, s16
+; CHECK-NEXT:    asrl r6, r1, r4
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    adds r6, r6, r1
-; CHECK-NEXT:    asr.w r7, r1, #31
-; CHECK-NEXT:    adc.w r1, r7, r5
+; CHECK-NEXT:    adds r0, r1, r5
+; CHECK-NEXT:    asr.w r4, r1, #31
+; CHECK-NEXT:    adc.w r1, r4, r7
 ; CHECK-NEXT:    vmov r7, s4
-; CHECK-NEXT:    asrl r6, r1, r7
-; CHECK-NEXT:    vmov q0[2], q0[0], r6, r0
-; CHECK-NEXT:    vmov q0[3], q0[1], r4, r2
+; CHECK-NEXT:    asrl r0, r1, r7
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r6
+; CHECK-NEXT:    vmov q0[3], q0[1], r2, r8
 ; CHECK-NEXT:    vstrw.32 q0, [r3]
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    vpop {d8}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
 entry:
   %a = load <4 x i32>, ptr %A, align 4
   %b = load <4 x i32>, ptr %B, align 4
-- 
GitLab


From fdaad738753cde2bba6480c2ee5d1e9fb45064b9 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy@outlook.com>
Date: Thu, 16 May 2024 20:13:39 +0800
Subject: [PATCH 1001/1206] [Reg2Mem] Handle CallBr instructions (#90953)

Fixes #90900
---
 .../lib/Transforms/Utils/DemoteRegToStack.cpp | 18 +++++++++--
 llvm/test/Transforms/Reg2Mem/callbr-crash.ll  | 32 +++++++++++++++++++
 2 files changed, 48 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/Transforms/Reg2Mem/callbr-crash.ll

diff --git a/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp b/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
index b2a88eadd3de..f03de2f25a41 100644
--- a/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
+++ b/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
@@ -50,6 +50,15 @@ AllocaInst *llvm::DemoteRegToStack(Instruction &I, bool VolatileLoads,
       assert(BB && "Unable to split critical edge.");
       (void)BB;
     }
+  } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(&I)) {
+    for (int i = 0; i < CBI->getNumSuccessors(); i++) {
+      auto *Succ = CBI->getSuccessor(i);
+      if (!Succ->getSinglePredecessor()) {
+        assert(isCriticalEdge(II, i) && "Expected a critical edge!");
+        BasicBlock *BB = SplitCriticalEdge(II, i);
+        assert(BB && "Unable to split critical edge.");
+      }
+    }
   }
 
   // Change all of the users of the instruction to read from the stack slot.
@@ -102,9 +111,14 @@ AllocaInst *llvm::DemoteRegToStack(Instruction &I, bool VolatileLoads,
         new StoreInst(&I, Slot, Handler->getFirstInsertionPt());
       return Slot;
     }
+  } else if (InvokeInst *II = dyn_cast<InvokeInst>(&I)) {
+    InsertPt = II->getNormalDest()->getFirstInsertionPt();
+  } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(&I)) {
+    for (BasicBlock *Succ : successors(CBI))
+      new StoreInst(CBI, Slot, Succ->getFirstInsertionPt());
+    return Slot;
   } else {
-    InvokeInst &II = cast<InvokeInst>(I);
-    InsertPt = II.getNormalDest()->getFirstInsertionPt();
+    llvm_unreachable("Unsupported terminator for Reg2Mem");
   }
 
   new StoreInst(&I, Slot, InsertPt);
diff --git a/llvm/test/Transforms/Reg2Mem/callbr-crash.ll b/llvm/test/Transforms/Reg2Mem/callbr-crash.ll
new file mode 100644
index 000000000000..2abb01b14b0e
--- /dev/null
+++ b/llvm/test/Transforms/Reg2Mem/callbr-crash.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=reg2mem -S < %s | FileCheck %s
+
+define void @crash() {
+; CHECK-LABEL: @crash(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_REG2MEM:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    %"reg2mem alloca point" = bitcast i32 0 to i32
+; CHECK-NEXT:    [[A:%.*]] = callbr i64 asm "", "=r,r,!i"(i64 0)
+; CHECK-NEXT:            to label [[THEN:%.*]] [label %entry.else_crit_edge]
+; CHECK:       entry.else_crit_edge:
+; CHECK-NEXT:    store i64 [[A]], ptr [[A_REG2MEM]], align 4
+; CHECK-NEXT:    br label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    store i64 [[A]], ptr [[A_REG2MEM]], align 4
+; CHECK-NEXT:    [[A_RELOAD:%.*]] = load i64, ptr [[A_REG2MEM]], align 4
+; CHECK-NEXT:    [[B:%.*]] = inttoptr i64 [[A_RELOAD]] to ptr
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a = callbr i64 asm "", "=r,r,!i"(i64 0)
+  to label %then [label %else]
+
+then:
+  %b = inttoptr i64 %a to ptr
+  br label %else
+
+else:
+  ret void
+}
-- 
GitLab


From 911207338add39b8439a09fd7f9d97bbe4a335bd Mon Sep 17 00:00:00 2001
From: Orlando Cazalet-Hyams <orlando.hyams@sony.com>
Date: Thu, 16 May 2024 13:16:44 +0100
Subject: [PATCH 1002/1206] [RemoveDIs][NFC] Fix rotten green C API test
 (#92362)

`llvm_test_dibuilder(/*NewDebugInfoMode=*/true)` isn't currently executed in
`return llvm_test_dibuilder(false) && llvm_test_dibuilder(true);`
because `llvm_test_dibuilder` returns 0 for success.

Split the llvm-c-test flag `--test-dibuilder` into two, one for the old and
one for the new debug info format. Add another lit test for the new format.

Now that the test actually runs, it crashes using the new format with
`llvm/lib/IR/LLVMContextImpl.cpp:53:llvm::LLVMContextImpl::~LLVMContextImpl(): Assertion 'TrailingDbgRecords.empty() && "DbgRecords in blocks not cleaned"' failed. Aborted`.

Insert terminators into the blocks so that we don't leave the debug records
trailing, unattached to any instructions, which fixes that.
---
 llvm/test/Bindings/llvm-c/debug_info.ll       |  6 +-
 .../Bindings/llvm-c/debug_info_new_format.ll  | 66 +++++++++++++++++++
 llvm/tools/llvm-c-test/debuginfo.c            | 16 +++++
 llvm/tools/llvm-c-test/main.c                 |  8 ++-
 4 files changed, 92 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/Bindings/llvm-c/debug_info_new_format.ll

diff --git a/llvm/test/Bindings/llvm-c/debug_info.ll b/llvm/test/Bindings/llvm-c/debug_info.ll
index a7fcd8a999ef..9358bac59bd2 100644
--- a/llvm/test/Bindings/llvm-c/debug_info.ll
+++ b/llvm/test/Bindings/llvm-c/debug_info.ll
@@ -1,4 +1,4 @@
-; RUN: llvm-c-test --test-dibuilder | FileCheck %s
+; RUN: llvm-c-test --test-dibuilder-old-debuginfo-format | FileCheck %s
 
 ; CHECK: ; ModuleID = 'debuginfo.c'
 ; CHECK-NEXT: source_filename = "debuginfo.c"
@@ -8,8 +8,10 @@
 ; CHECK-NEXT:   call void @llvm.dbg.declare(metadata i64 0, metadata !38, metadata !DIExpression()), !dbg !43
 ; CHECK-NEXT:   call void @llvm.dbg.declare(metadata i64 0, metadata !39, metadata !DIExpression()), !dbg !43
 ; CHECK-NEXT:   call void @llvm.dbg.declare(metadata i64 0, metadata !40, metadata !DIExpression()), !dbg !43
-; CHECK:      vars:                                             ; No predecessors!
+; CHECK-NEXT:   br label %vars
+; CHECK:      vars:
 ; CHECK-NEXT:   call void @llvm.dbg.value(metadata i64 0, metadata !41, metadata !DIExpression(DW_OP_constu, 0, DW_OP_stack_value)), !dbg !44
+; CHECK-NEXT:   ret i64 0
 ; CHECK-NEXT: }
 
 ; CHECK:      ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
diff --git a/llvm/test/Bindings/llvm-c/debug_info_new_format.ll b/llvm/test/Bindings/llvm-c/debug_info_new_format.ll
new file mode 100644
index 000000000000..05b6ef4de9ad
--- /dev/null
+++ b/llvm/test/Bindings/llvm-c/debug_info_new_format.ll
@@ -0,0 +1,66 @@
+; RUN: llvm-c-test --test-dibuilder-new-debuginfo-format | FileCheck %s
+;; Duplicate of debug_info.ll using debug records instead of intrinsics.
+
+; CHECK: ; ModuleID = 'debuginfo.c'
+; CHECK-NEXT: source_filename = "debuginfo.c"
+
+; CHECK:      define i64 @foo(i64 %0, i64 %1, <10 x i64> %2) !dbg !31 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:     #dbg_declare(i64 0, !38, !DIExpression(), !43)
+; CHECK-NEXT:     #dbg_declare(i64 0, !39, !DIExpression(), !43)
+; CHECK-NEXT:     #dbg_declare(i64 0, !40, !DIExpression(), !43)
+; CHECK-NEXT:   br label %vars
+; CHECK:      vars:
+; CHECK-NEXT:     #dbg_value(i64 0, !41, !DIExpression(DW_OP_constu, 0, DW_OP_stack_value), !44)
+; CHECK-NEXT:   ret i64 0
+; CHECK-NEXT: }
+
+; CHECK:      !llvm.dbg.cu = !{!0}
+; CHECK-NEXT: !FooType = !{!28}
+; CHECK-NEXT: !EnumTest = !{!3}
+
+; CHECK:      !0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "llvm-c-test", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !11, imports: !19, macros: !23, splitDebugInlining: false, sysroot: "/")
+; CHECK-NEXT: !1 = !DIFile(filename: "debuginfo.c", directory: ".")
+; CHECK-NEXT: !2 = !{!3}
+; CHECK-NEXT: !3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "EnumTest", scope: !4, file: !1, baseType: !6, size: 64, elements: !7)
+; CHECK-NEXT: !4 = !DINamespace(name: "NameSpace", scope: !5)
+; CHECK-NEXT: !5 = !DIModule(scope: null, name: "llvm-c-test", includePath: "/test/include/llvm-c-test.h")
+; CHECK-NEXT: !6 = !DIBasicType(name: "Int64", size: 64)
+; CHECK-NEXT: !7 = !{!8, !9, !10}
+; CHECK-NEXT: !8 = !DIEnumerator(name: "Test_A", value: 0, isUnsigned: true)
+; CHECK-NEXT: !9 = !DIEnumerator(name: "Test_B", value: 1, isUnsigned: true)
+; CHECK-NEXT: !10 = !DIEnumerator(name: "Test_B", value: 2, isUnsigned: true)
+; CHECK-NEXT: !11 = !{!12, !16}
+; CHECK-NEXT: !12 = !DIGlobalVariableExpression(var: !13, expr: !DIExpression(DW_OP_constu, 0, DW_OP_stack_value))
+; CHECK-NEXT: !13 = distinct !DIGlobalVariable(name: "globalClass", scope: !5, file: !1, line: 1, type: !14, isLocal: true, isDefinition: true)
+; CHECK-NEXT: !14 = !DICompositeType(tag: DW_TAG_structure_type, name: "TestClass", scope: !1, file: !1, line: 42, size: 64, flags: DIFlagObjcClassComplete, elements: !15)
+; CHECK-NEXT: !15 = !{}
+; CHECK-NEXT: !16 = !DIGlobalVariableExpression(var: !17, expr: !DIExpression(DW_OP_constu, 0, DW_OP_stack_value))
+; CHECK-NEXT: !17 = distinct !DIGlobalVariable(name: "global", scope: !5, file: !1, line: 1, type: !18, isLocal: true, isDefinition: true)
+; CHECK-NEXT: !18 = !DIDerivedType(tag: DW_TAG_typedef, name: "int64_t", scope: !1, file: !1, line: 42, baseType: !6)
+; CHECK-NEXT: !19 = !{!20, !22}
+; CHECK-NEXT: !20 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !5, entity: !21, file: !1, line: 42)
+; CHECK-NEXT: !21 = !DIModule(scope: null, name: "llvm-c-test-import", includePath: "/test/include/llvm-c-test-import.h")
+; CHECK-NEXT: !22 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !5, entity: !20, file: !1, line: 42)
+; CHECK-NEXT: !23 = !{!24}
+; CHECK-NEXT: !24 = !DIMacroFile(file: !1, nodes: !25)
+; CHECK-NEXT: !25 = !{!26, !27}
+; CHECK-NEXT: !26 = !DIMacro(type: DW_MACINFO_define, name: "SIMPLE_DEFINE")
+; CHECK-NEXT: !27 = !DIMacro(type: DW_MACINFO_define, name: "VALUE_DEFINE", value: "1")
+; CHECK-NEXT: !28 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !29, size: 192, dwarfAddressSpace: 0)
+; CHECK-NEXT: !29 = !DICompositeType(tag: DW_TAG_structure_type, name: "MyStruct", scope: !4, file: !1, size: 192, elements: !30, runtimeLang: DW_LANG_C89, identifier: "MyStruct")
+; CHECK-NEXT: !30 = !{!6, !6, !6}
+; CHECK-NEXT: !31 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: !1, file: !1, line: 42, type: !32, scopeLine: 42, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, retainedNodes: !37)
+; CHECK-NEXT: !32 = !DISubroutineType(types: !33)
+; CHECK-NEXT: !33 = !{!6, !6, !34}
+; CHECK-NEXT: !34 = !DICompositeType(tag: DW_TAG_array_type, baseType: !6, size: 640, flags: DIFlagVector, elements: !35)
+; CHECK-NEXT: !35 = !{!36}
+; CHECK-NEXT: !36 = !DISubrange(count: 10, lowerBound: 0)
+; CHECK-NEXT: !37 = !{!38, !39, !40, !41}
+; CHECK-NEXT: !38 = !DILocalVariable(name: "a", arg: 1, scope: !31, file: !1, line: 42, type: !6)
+; CHECK-NEXT: !39 = !DILocalVariable(name: "b", arg: 2, scope: !31, file: !1, line: 42, type: !6)
+; CHECK-NEXT: !40 = !DILocalVariable(name: "c", arg: 3, scope: !31, file: !1, line: 42, type: !34)
+; CHECK-NEXT: !41 = !DILocalVariable(name: "d", scope: !42, file: !1, line: 43, type: !6)
+; CHECK-NEXT: !42 = distinct !DILexicalBlock(scope: !31, file: !1, line: 42)
+; CHECK-NEXT: !43 = !DILocation(line: 42, scope: !31)
+; CHECK-NEXT: !44 = !DILocation(line: 43, scope: !31)
diff --git a/llvm/tools/llvm-c-test/debuginfo.c b/llvm/tools/llvm-c-test/debuginfo.c
index 9b5c37b05d90..35c65f885af3 100644
--- a/llvm/tools/llvm-c-test/debuginfo.c
+++ b/llvm/tools/llvm-c-test/debuginfo.c
@@ -215,10 +215,26 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) {
 
   LLVMDIBuilderFinalize(DIB);
 
+  // Using the new debug format, debug records get attached to instructions.
+  // Insert a `br` and `ret` now to absorb the debug records which are
+  // currently "trailing", meaning that they're associated with a block
+  // but no particular instruction, which is only valid as a transient state.
+  LLVMContextRef Ctx = LLVMGetModuleContext(M);
+  LLVMBuilderRef Builder = LLVMCreateBuilderInContext(Ctx);
+  LLVMPositionBuilderAtEnd(Builder, FooEntryBlock);
+  // Build `br label %vars` in entry.
+  LLVMBuildBr(Builder, FooVarBlock);
+  // Build `ret i64 0` in vars.
+  LLVMPositionBuilderAtEnd(Builder, FooVarBlock);
+  LLVMTypeRef I64 = LLVMInt64TypeInContext(Ctx);
+  LLVMValueRef Zero = LLVMConstInt(I64, 0, false);
+  LLVMBuildRet(Builder, Zero);
+
   char *MStr = LLVMPrintModuleToString(M);
   puts(MStr);
   LLVMDisposeMessage(MStr);
 
+  LLVMDisposeBuilder(Builder);
   LLVMDisposeDIBuilder(DIB);
   LLVMDisposeModule(M);
 
diff --git a/llvm/tools/llvm-c-test/main.c b/llvm/tools/llvm-c-test/main.c
index c4748d342fba..9db6e05d6350 100644
--- a/llvm/tools/llvm-c-test/main.c
+++ b/llvm/tools/llvm-c-test/main.c
@@ -109,8 +109,12 @@ int main(int argc, char **argv) {
     return llvm_echo();
   } else if (argc == 2 && !strcmp(argv[1], "--test-diagnostic-handler")) {
     return llvm_test_diagnostic_handler();
-  } else if (argc == 2 && !strcmp(argv[1], "--test-dibuilder")) {
-    return llvm_test_dibuilder(false) && llvm_test_dibuilder(true);
+  } else if (argc == 2 &&
+             !strcmp(argv[1], "--test-dibuilder-old-debuginfo-format")) {
+    return llvm_test_dibuilder(false);
+  } else if (argc == 2 &&
+             !strcmp(argv[1], "--test-dibuilder-new-debuginfo-format")) {
+    return llvm_test_dibuilder(true);
   } else {
     print_usage();
   }
-- 
GitLab


From 03d8e613911bdef52fec23924a4e5b4235997bb8 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Thu, 16 May 2024 20:29:54 +0800
Subject: [PATCH 1003/1206] [Transforms] Fix -Wsign-compare in
 DemoteRegToStack.cpp (NFC)

llvm-project/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp:54:23:
error: comparison of integers of different signs: 'int' and 'unsigned int' [-Werror,-Wsign-compare]
    for (int i = 0; i < CBI->getNumSuccessors(); i++) {
                    ~ ^ ~~~~~~~~~~~~~~~~~~~~~~~
1 error generated.
---
 llvm/lib/Transforms/Utils/DemoteRegToStack.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp b/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
index f03de2f25a41..653ea8bbb72b 100644
--- a/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
+++ b/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
@@ -51,7 +51,7 @@ AllocaInst *llvm::DemoteRegToStack(Instruction &I, bool VolatileLoads,
       (void)BB;
     }
   } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(&I)) {
-    for (int i = 0; i < CBI->getNumSuccessors(); i++) {
+    for (unsigned i = 0; i < CBI->getNumSuccessors(); i++) {
       auto *Succ = CBI->getSuccessor(i);
       if (!Succ->getSinglePredecessor()) {
         assert(isCriticalEdge(II, i) && "Expected a critical edge!");
-- 
GitLab


From 99de3a685170c24208b00ef83fe0416373387acc Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 16 May 2024 13:34:04 +0100
Subject: [PATCH 1004/1206] [VPlan] Document current status. (#85689)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Update VectorizationPlan.rst to include a section about the current
status of VPlan and its use in LoopVectorize, based on "VPlan: Status
Update and Future Roadmap", LLVM Developers’ Meeting 2023,
https://www.youtube.com/watch?v=SzGP4PgMuLE

PR: https://github.com/llvm/llvm-project/pull/85689
---
 llvm/docs/VectorizationPlan.rst        |  63 +++++++++++++++++++++----
 llvm/docs/conf.py                      |   1 +
 llvm/docs/vplan-scope.png              | Bin 0 -> 679990 bytes
 llvm/docs/vplan-transform-pipeline.png | Bin 0 -> 555223 bytes
 4 files changed, 54 insertions(+), 10 deletions(-)
 create mode 100644 llvm/docs/vplan-scope.png
 create mode 100644 llvm/docs/vplan-transform-pipeline.png

diff --git a/llvm/docs/VectorizationPlan.rst b/llvm/docs/VectorizationPlan.rst
index 418c52eeef7f..6496263cf8d9 100644
--- a/llvm/docs/VectorizationPlan.rst
+++ b/llvm/docs/VectorizationPlan.rst
@@ -19,6 +19,49 @@ candidates. It serves for both optimizing candidates including estimating their
 cost reliably, and for performing their final translation into IR. This
 facilitates dealing with multiple vectorization candidates.
 
+Current Status
+==============
+VPlan is currently used to drive code-generation in LoopVectorize. VPlans are
+constructed after all cost-based and most legality-related decisions have been
+taken. As def-use chains between recipes are now fully modeled in VPlan,
+VPlan-based analyses and transformations are used to simplify and modularize
+the vectorization process [10]_. Those include transformations to
+
+1. Legalize the initial VPlan, e.g. by introducing specialized recipes for
+   reductions and interleave groups.
+
+2. Optimize the legalized VPlan, e.g. by removing redundant recipes or
+   introducing active-lane-masks.
+
+3. Apply unroll- and vectorization-factor specific optimizations, e.g. removing
+   the backedge to reiterate the vector loop based on VF & UF.
+
+Refer to :numref:`fig-vplan-transform-pipeline` for an overview of the current
+transformation pipeline.
+
+Note that some legality checks are already done in VPlan, including checking if
+all users of a fixed-order recurrence can be re-ordered. This is implemented as
+a VPlan-to-VPlan transformation that either applies a valid re-ordering or
+bails out marking the VPlan as invalid.
+
+.. _fig-vplan-transform-pipeline:
+.. figure:: ./vplan-transform-pipeline.png
+   :width: 800 px
+
+   VPlan Transformation Pipeline in 2024
+
+
+VPlan currently models the complete vector loop, as well as additional parts of
+the vectorization skeleton. Refer to :numref:`fig-vplan-scope` for an overview
+of the scope covered by VPlan.
+
+.. _fig-vplan-scope:
+.. figure:: ./vplan-scope.png
+   :width: 800 px
+
+   Scope modeled in VPlan in 2024
+
+
 High-level Design
 =================
 
@@ -160,13 +203,10 @@ The low-level design of VPlan comprises of the following classes.
   VPValues.
 
 :VPInstruction:
-  A VPInstruction is both a VPRecipe and a VPUser. It models a single
-  VPlan-level instruction to be generated if the VPlan is executed, including
-  its opcode and possibly additional characteristics. It is the basis for
-  writing instruction-level analyses and optimizations in VPlan as creating,
-  replacing or moving VPInstructions record both def-use and scheduling
-  decisions. VPInstructions also extend LLVM IR's opcodes with idiomatic
-  operations that enrich the Vectorizer's semantics.
+  A VPInstruction is a recipe characterized by a single opcode and optional
+  flags, free of ingredients or other meta-data. VPInstructions also extend
+  LLVM IR's opcodes with idiomatic operations that enrich the Vectorizer's
+  semantics.
 
 :VPTransformState:
   Stores information used for generating output IR, passed from
@@ -177,9 +217,9 @@ The Planning Process and VPlan Roadmap
 ======================================
 
 Transforming the Loop Vectorizer to use VPlan follows a staged approach. First,
-VPlan is used to record the final vectorization decisions, and to execute them:
-the Hierarchical CFG models the planned control-flow, and Recipes capture
-decisions taken inside basic-blocks. Next, VPlan will be used also as the basis
+VPlan was only used to record the final vectorization decisions, and to execute
+them: the Hierarchical CFG models the planned control-flow, and Recipes capture
+decisions taken inside basic-blocks. Currently, VPlan is used also as the basis
 for taking these decisions, effectively turning them into a series of
 VPlan-to-VPlan algorithms. Finally, VPlan will support the planning process
 itself including cost-based analyses for making these decisions, to fully
@@ -248,3 +288,6 @@ References
 
 .. [9] "Extending LoopVectorizer: OpenMP4.5 SIMD and Outer Loop
     Auto-Vectorization", Intel Vectorizer Team, LLVM Developers' Meeting 2016.
+
+.. [10] "VPlan: Status Update and Future Roadmap", Florian Hahn, LLVM
+         Developers' Meeting 2023, https://www.youtube.com/watch?v=SzGP4PgMuLE
diff --git a/llvm/docs/conf.py b/llvm/docs/conf.py
index acbddc7b2f12..270f5b33e985 100644
--- a/llvm/docs/conf.py
+++ b/llvm/docs/conf.py
@@ -222,6 +222,7 @@ latex_documents = [
 # If false, no module index is generated.
 # latex_domain_indices = True
 
+numfig = True
 
 # -- Options for manual page output --------------------------------------------
 
diff --git a/llvm/docs/vplan-scope.png b/llvm/docs/vplan-scope.png
new file mode 100644
index 0000000000000000000000000000000000000000..d60b58b55b8eab8ddd9824a49f7c1ceb804d4c98
GIT binary patch
literal 679990
zcmeFZbzD^6+CGecQVI$Jf)dh_0@5u?cXx+$*HBWTBHfLEfOHN$gc3t{BS?4m5bxr9
z&Uwyx-}jH_c+Q`{AD_Y5?%8|KUTd#)-`9Oz*W#0cocIGwVoW3?qz94`Z<LUb?vEiM
z-AhBi2lmiXC=Y=hf)=8p3X-Ct<O+^<W){|_NJtW&V%5<!l;H$v+DgwWk&u;M+xdTt
zK~j4C?oPJb)8`lurM~!O7>N_$>tN9nlzuIs_?&7czQy>tk%i!Kj7yn;`mgNw#fYxc
z6~s!P?+wpkI{Rwhj3*k>F`iBtH`*u?xq1xCCyWzwD(c*q60=Bm#opiRY!2$~#;eH8
zY`w!6uszpZ{}d^*N}a?+>t^ew<ZU!d=i;6BZ$whs`xlpAdp$&I_)HtOgd!ZXzoLBD
z#&(}!Ag&qdxgOmQB1|3X9~PJm>b7`<KR(ppBbC#~Gr*CAx7n76?l#=d`#OeJ7X6Xs
z@tusm_Y<^SZfs-=Dj#h!LZ28V3bJpbrG9C$$a!GlSW6_T!Aj`y91Z>OC(q3KjXCeN
z8@x9X`=Zoel|va9mok~XXS!$$Zpc=M=}<z)M1gC>^ic@?+|l!+qGir2Dm}w+;)8>U
zm*&ERMzn%l<oAt4KL)(vdGuRKkV*H+ProO{51o@<rsJWNk^aQG>)}@>l`I<tXO+@_
z=Uo@rt8}nhgIlYgSb{dR<sPSW<C60}YoucYlYJ5QdCqHVMRJ*s?=>D6bu?^Ey!_Ui
z&?@8lGb8j3F=fac^1>58x(WTS;rQ9o{zfq;-)cPq4%;|QF?k+IH3$i@Grx%>D~#=k
z?Z8>}*CqI<u^+(e-;ll6QoDEej^F26<5^@yuR^8O0PZ;<F;;@y&rX>!CpC%Wv>NTn
zB3n`F4-1}!&@TPB`+cBexb0lg;WPJsE2HhZ7z$D^B6KwChe%I^F|CLb?w7wWZLI!E
z`0EA4ABj*x_})V#G-`Hk3dLWP{^J@7=+}V~NCzL<&$~VEW*{Bt2WzyA-pTIv#9E>(
zZ&Ci}B=V%w?^NlWZZM)6bq|%T)z^KswJTtMrI~*7=Jb*rk!mPHqXU;f6JQ%bWBTOx
z6Uhjj_2avcr}Kg}grwLdKBc^iGBd{Zq;uOC@)#t6;(?oia;@8vtrT#1#jv}n3PKWz
zA(doKe%d0k>N8}gA2t+-#@imeN7+HXDz(>No5U}-Hk~OieCacEJ$>KDomh(RGv3{A
z_p{Cvt$j$po8_+XW>a(@ho39{OyZ2OJ5lKGzn&B_NynKkO=l_#nz*yY=!(l1PSQe*
zgu)lbVPrF*NJOfoE5B4=HK6^4Kg~Li@-n!og_fb)Bu%+UIeeJ#tv<{kX^Z5+M0DFQ
zb?cWNuX|WNk5KQ2eO%@C|Gl-eh5UXCk?HQ;_$khp>)|YPcji(EX-`n_<xBfBK0?T|
z_uf192rlnc>x8}GugJ}{G-)bI8T0m&K}d7B8}_;DD8>fZ*7IH|eoyz1eaIgM`b#`S
zi)wlE#G6KE7562659*;X1|3RwGkqzxHL7+qcIneCv<LI(+!%fDL+06Puy<NVhwtjW
z7u>`45_=dJIPj1+GdzJj?xVoMmzR&os2*a78%3B<;lqXBe0-evsYkpk!qJ4JoanpA
z$#dCG$tC_FzZo%73V}|mCA`mH^Yb_@1GT<VD$;bl{xQJqjGW{rCxSI-<cv{wuSm@8
z>+zc85e0-~`;C6a)stqGm%2TVothJ#JPrA&RO&3_-`g%#hFM5l@FVHB6YR5#QR9So
z%Ew`cdpk(#_UNSHhkGy1ti4!a*mHj4i$!Oum(PC%`%(^|e?ay0r}Yz+pfGuugDZuO
zA7~RO8aQmv;_~R%qfc*iGdTuHcG^sg2psM?usR4h;OwK81)L>J$WuffNuE&IDN%ir
z?xKy27>ja^42v#~sFNV;q&LAUXXp8t@|qwrq!WKBb;)>%Vaa@{|M7#*)NRqYlE0O6
zl;f13N=Yx`>2zcIRb=xyMw5xfdo%m8iocF2$}8_G&gJFhC46_!dsBd}x{_b0I9cE*
zxBRtBT$M?cPZhf1uKIlgcXT_SR>iIW_I2=`+AAR~Or~^bpqeP(TTfv`hM2mO%3|LB
z=t`by{%C=&qUU!x#cCDv@A4`vDkTa+89tJY>H(RC-&)4zEqliDM&1q8d><^49DnwW
z=zC&8t%g(Xibt<4Q&e26S_XlBfreGc*o&eL=P$N$=EELG=SM|H!$-_Enb0auaX#sK
z8P96hde`)@xUdj8|GrnLQmI+IsdWj(vCQIJ;%?t}d}g24rGFrLVEh2@zV@p^+lg4i
z-Qy3N=j#|pQpYE2y5orK{sEkk{xSXHxeVVqB(75SkDLjXr7iQ7BD?n6zx952{E!ab
zCG+Wc$Wh7ug*C;nrmeKMbgpxPZbHJka%?R|v81fHEZre)Uuw~~U4X_Jj|}fCTq=5z
zrk{xSJ^^KL*@wZdHFl0y4_IT9Y&mVXuB^>$2<+5MpdDK<p3xMGG+WqELt8_C>rbP?
zHklBu5Wcv_(mAF%%a+|0al1u@MYu%-8XOuf+MAkAg;+&aMZ1#&)?PL_6Drd>71UE~
zHWfDG(|S<RZJlklZTfBI$*dFGlh=sj6B0^vN`A4AVk+S^>lqu(j&$~UF79i2n@@H$
zmL}%U9JAXxV<Nl7(@4&H;iE4)EDDBa6}7Tni{pn{k{(Y@_9d?Btb5tKvT+_*<H>p|
z-sD*3_Nd0%zIIv4jbyW~xvK5BAh|yIXHj2K$84*>Xk*tEjdC(kn@$h^s>t5nX~Lqz
z&&90VWOLZ6Pk&Rxvxf3USBOh^87bwH3YQ*ZJ9fL`Q?p$;vJdk4{WR%v`pWvnc2&ES
zLQX=kInE|?Mr}0$E5glhP0>w1Zz66yp|qX^h&M<C51%Oz50K{JKOw7O-V=$?R1j0J
zQP3~7+WRt(_dGD7G-CNV7mwI8(%cWjgd3#143!)*9E_aSx)WLU_4bnPhZ@e-YZLg!
zyKgnbK2i`<h)~fwzFoSa+su2R7@YNKP;oGJP~KRqt>+8RQ|a*3$Mx(dx+%UhLpAmL
zoclv(aj3_E`xqP;yLc$L-FSS_Vl=%{bqUrHj54+|&%aS+TT8E#5pvs())em-FG{!1
zB^U@8P6Zy~J)l{U;pdg~#&sL=Q+aJ^w9z-hG9zKjQ7sf2trk1hVSK1|sJC}@#&9lx
zY$sAXuUY2?@qoB}aBKhkB~50o=W~zVbL|?rGkOLpE8{W5R_E6CBnr<b9#2Pj?e|()
z3MNHpo=f@ul0`J%&yxjQht<dZFA8#wBwMMb4<Z^cJVMW>_QYYbmXud-FG`zpGs@?G
zj0Ap1%dwEdleJ2=<?<dpG7l07YU*t^-!rT1d6mjzxAH@?P&3(_vSQe*x`#DgnLpRc
zc5jB{Fz>K@^-6C^MMdw()o0;y#VBi`wk_Nj<N6!)P0<@AM<>5=|4^*`pw^)65HS|5
z`ZetrO-HX2l{Mc^#<w-J5&DoEZzk!Z#bvWt=x;!sf3PjE4VFLc_Kx4B8)sx;^ldQO
z`g{@lCH5P`Tyaw6hF6euTl?t9=+J1YRqObx;wp`L>+N`##LZ&$3-z7vY=v4X$36nS
zSFsO#o^Egw+b~X_Sw!0p7xlIzZx$5gGe93!KeqO$<1pNuoJ(jLwPv4CgKlrS4xaRp
zu<~hH%~)Bw67T3}g^WpAdAf$qxp=rW@3iv<*Y$cjpT=z8_59|UzAHWJX6!q09f#6`
z;)2G6qehZRMqebpkvo3MVZ!OmNyFv(Du}b{l_v)o@0Pne;?Xc=thh@Uvw)a@nwRyw
z`|tg0yal{QT6Q@>+r$yaQ<b6bRz-tF+qpmF()sY5S4K3)CKp#5`bl_pJz(p|hp&IU
ztqwW1DYgmral5)wsMq$qqCS-ljTd0%a{IE*?b>$Q1uwXx-mkN$Q|~UY0O<%lds=N9
z)->r6;#+`l?b}v&!`$(L@tu<X)?4=uMd0JyCsVmGM0BLdRrh7#ppZfZK6^W#vhMDO
z(0a(jobiRe2^6`T9TetF*CHh375bM~7t`DtA2yxMXdmRRq^q*n=#gG_@0LJe*|1{u
zD0P$)!lpsrimRgT>A35>YqVX*Id32R3z3^2Vc|XD=0emykQ0v$!l8H$eVZmuUo3>H
z9l4u<a8IoLauf2C!gml}q>^eBVd;DCvHQ(O96leWH;{j3FL>s66NqpA#=`+61N*v3
z{Is<xIT>kd7BgI5LEoDnN%0lZ0zVQj<Vb;pzX0AJTmCw5^}Xz%{;=b4bJL5@?5j8q
zH#5kI`B-PQSA;7>UM@_PH9DZ6_-v{nX(lU+L=U#nkx=gtBi#jCcfhaUou~h^Ep~?v
z3FWW-$Vf=R7D%XnKSmC`-~L2^-`jKk_Kp(u0SOKK4;TEpXCVLM==)<CDF4{Lmj*sV
z5>XbFlmzd}#*U_@wodQtoU2<`m%$DUdkIY^BqRdr+uu8qN>qE``V$r^8qONBGQ7ri
zHq3@5c1EVm?l$(f*FoZU=LMTKrp|`s?l#u8PQ2~{6n`DT3$|}}vrv%#b%?W-0ELFE
z0=cN2qbWHj^GoKJ6oQ!K<mCL0CT6@!Z^Zt7I{2Re#XDzbdtMe6H#awCH+E(_M{^cd
z9v&W+muxI-Y)s$?CMOSDXG3=;TPMoDUF09vd1LBi>}X-{Y++|hetTU*BRdyo0Sb!S
zFZ!RKzul*)yTw1h$=2!b%>oT%x&4HNmH8#h|6CiK%743;SHZ&F)LQe6g$?L4@EwAz
zoIJ1i|2pCS_t8JU@?TEX_~)tYY}}mxdg{M?^#46o)ydRR)XoNcsk7ieAK2ed{?`xx
zej-21ZQK8)FaFlif9(Z5Er`j_@;^^a5Od}-?+bX4&n(`^tAKYf%x-^B<iS65e|rbp
zD1GVOoBZcUNWw^xZ$wnw?`$<<T*{2zocVr~n53LWd;45T>oG1<%uwb>{dbW{FTAHK
zX{9P9Fki%Wr769ZtjER0|E45~^?I`0l<(#+>BiivJz@EmXJ2aK>GYu>>262QOs%<R
z+G->F?kS>Sr{KMkR5BSSx-b&*!}mz2<Vbh^)6FeqRLq1yT#X8OS)PEP|LRh=H}en~
zb!`65e{(~BHK|!p@_loXx8oGitN;31NPZmL|9SuXqlZi7amh=~(o<aTJ@}9N`XAR4
zHh6&jkL~~b&bFda(F-SOb5lh`|LalkP$K~4zZ^)QK?tfl$W|wC3-f{h{#sxd{nvx>
z{r??C|9e>YKWP{Vx1&FPq4Xw}MPKjgd^Oz?ak5Qh760Ks3ZKFhAk<N1whI1B7n`GM
zL9M^@I<IAf37lxSEJcdvw+3Mqknp+YeUpyQCt^3t>mc=%@jjh(>zm4=_?I*P{muDU
z5YY4_9iII&FurF&2A5YuLBm#w*LBtGa@4Zoa9omb>E#&DW739Z^T%;L|4-t~duh<b
znrunkf0@pYkYgFt)tpxQRPh<qj$};wQh4&;h7fF|`(E$sd_einzUKjm8`f)z1piEC
z1}w;TF&`@o=g2C97R+omI`ip4)~Q|2m*aE!zMTGNUxNpl!`&i5&50@%OaCoLE;U~&
zj*+M)DU5_KzohA!*WLs3HTg6uCLwd7R6KK0SImnFMDDx6Ov~9i7t7;~QOnVM)eUf<
z(sZ?5zpJ_h8Mt3zE#%pHH}wEC963*&auhk9<;TZx+H@+<<SUOD;r)fS(H1na!mc=`
zryeom+E8o7&G8cTr<~Rw`Mb<#+e4oouAw)&)k{QDL|V(m(yJ;3KXd#zRc0KlRb~{V
z1@o%as&)J}RqK>{+I6rNc`z%k^YMaw&X0<YWy9b7xzGWLgjK4zxMcO8b%<YsFi5Wr
zFQv~L!f(#|eMjt?uFN>P!0i-hR+wpyiWs7sd=?JE#2=BQP0c3euzY;ZbFr4C`U#hs
zpM&h_y9yIHE{FHUQD-<KuW0~?(-91)1dJLPVZ0l8&9g)tmN6SMwNCQ~k9!3DrmCz8
z%zBgF`Yd^BwSC4Py+-V&Lyd8T%beGhsIlSwm*<D0p1Y0lxdAqTZ|hyRM?mitQbmYq
zNW7FRw;3<ijhrXOC_i5e7owO<r;$r}g7H#Pjb{8*8maM%eo91hEw9Ul`}NLYztBxy
z5Eh}B|61FpaKQ@+@0+XBT0xTwv_DJNOV5$}P^@-pS4mxd2NCj$41AL&JRVio=lR}k
zMZKEt3)>hih#&u<OiCMq<ax4{c#wc6$1U=Ji0utNy$ap3gJHvqkP#DBpRIl{BfgvU
zr6djpd7_R5veu6zTA`XK+Em-lj@%{G$;ekP*{Sp_N7p1El~3kmI$#G^>rCBE<$P^=
zJKgwk>C_4}s1|?vAhuV_e*OxEyWznXTH>nFU)lZQKhSiWvdH~@u-whJc{x_3O(~L%
zt8=iDY?ohUHO^bWSNNIA{*AxIQm$SjWUN=>Uh-||!N8-*8`tw>%9<MuaE0mAI&Mr^
zm4teoMv^c1-^IFDWLX00!P8V+vcR@@eV^KKgojP=8O5ETs506w+mq($sy{wFz{;}R
z2>g@8EDU>|(JPU1mGG9&EVMd#d#8Tab&RFN*bB_$S?6CPj>EEC-?ndV5WYA4nx4h<
zY;N$e!<F7)Z^X%{Y%=F4%^K}Ccf*0Ka|%Xb*-jn&@{uJO-f^<`PVv+8*l)G$O>s^$
zGn%tg4np{aniXPm4jm+QV9KTtVhhb36y$yhu}ZW;OL}Fouo@M6q=A?5gM$%A=f~|7
z=X2S}mVIwNRO@`+4o3cQR=8lk?7XVn#bD-JtgGX3{rYDIdM=6NPX#;<Omr*EdQ(gk
zs%>=zyw2<!O}pb7FG$<N<OR>)V7ZNtm&=Tr`&%64ruz`zOPX~#*_q|*KY)+$4`9y)
z+asa<!OI}st(ob49Fz8rUb8V`2@LzbGvo{qYbRX!l=S?S^VN!ypf|NoH1Kc?Kv;q_
zzrRyw)b`n;18>RywL$2#)633h3hh_VIw_2!@6ca6cdc!L+#rVRY0^UDwB-|Kh^hQc
zVoSIqj3iZOlU}{Jr;4Ei#j3dBK&sJtl8yFoyE>{jnQOCy%vXoZc|e5c_b1egFGTF^
zyDtmF;+yfKnN+07?U;4KvH7-MUX5wnX@<gc_CU{9qMj@xV+nb`Bzj=;Vf#CyAIB%5
zlFJ^vj@E~N(tISegGLlvG&;AUIBN`MsLeFg)VWOO>f%_}nNO($%)cEAMRh&&!cVU!
z>s$&*d7bkPk)9%3&7=kBR919XK#n&cQ<WBj!!)#{t38QX;Q5%yk5gU3i-q=reSPfb
zkA~%|VJ9{dr6Pok8WEdnVfV&j7}VwDlU~6gX}T(Vvu<-1@QA1)Cvksu>X$~_Q&r3#
z@bbbyhFe#b<Ds^?A9#hYuJd)dq4xx8+=1<l*sk7ez*W)t`uZ*t7c06aW$A}q!pS<Z
zA?BUmNZ9lL_G{@m+Kl|z$48VYMGtsSXPnrf5?i2mUB-6P1Uw-dZhDx7pIXlmyG<?t
zvj~LtcYhK#2t_yzV7Y}k^k20<wHtZLW&0(Kp6B!oG;uXCDCcbbmwZC&D`jkn_g}Y9
z2F1yE5NVO$9d;SEQ=IyEH2XUVcE@vMlbu)lQrUN8)`z|w)24gj&ZQPM`Ye2+WxM9_
z-D^SH{DenqX43;FPUH6|k2{ASA39#1?ZZFJI}7=7c%gM3JNaGzru&0-&I@C6&->tN
zRoiY`KOaz*7l<vpt2mr1?>4pz(u_N3=zZ{PiTPX$qrbcc;6n(_X})y<esej}4~8T^
z#B%9q(Wze`e*bOtxkVZg);Pnny_Mz$kHf;^pm2ec3BgmID8iY#^OfYH6^^aeenCVb
zvr3+V*vt#n{FpR8H`@u(1KlRnL9?!J`KkpQ`o34AZe;AL2B7_QA_4b^7JtWk7N`}m
z1--`8Slq7Y-hJlCujBAXGno8_TVcK_+Jbb(=I!Ulir{KW)KW2}#m)c>fZ3}xcAQ<1
zaf>Rz;F2P>sV;$CNF~12Ny}&T<FM>WU`-3_Z3{5#Pb=yZL~NngXI!!Sacor2`Cw`B
zD-}B|wp)%BYO1*?JV6#ge@v$!oQNbm+UR*o-d3SxcDDCBU%T42t8k@93h@f_4*HTW
za=eb?Ct?AJbB3QS81<ti`ud4?5q@3DbkC60^*p|@wIu#l*(n|KLqVg*Ls}|rZ!DnK
zEYF4jh&qZvn^Jn3%u{OK8S#AMVzUINigvj>UY$leoOj_P<!hS`=m7{!d5d9ua&Tj2
z8<5?V=X?-FY(y}&p&~)=i#(8hOtYDY{r)In=utoH()H>!k}`LjF9Q8V=lu!~TK|iH
zNsD`PSKxG}qW;$eC$2z$H<49lvkZA8XUHahI8Pc1k=6C8A^%goRnDh)Dp|`rXC4lo
zD~aZvI(Uaf&9!mUtHg)VT&t_UgUPU+J+_9vNzd;-uE7ms0FPmq&u!=atMo~_k*}h`
z1JBsFum@;L`Jetc7+t}W=YK?u0w1HCyE*JSS<g-10Ej-gowCz~cI}vz&WUX-(8=aL
z+i8p;^B+WcFrdiINzikR3H96!2V8_aV>ba#TEqHE>U}iWIQ5_|%XNDy4UtTSY!x^R
zpwuxyO}id5WrV26R;@f}#zcqw&)QdP_e5CmcocUc#Yr)o`xcbB1zYfo&wKA+2tMPo
zO)1$-g@0^ydC}p2oy2LA58#l#{$av)0GI7d-sWh*>I`*stg`RPr1>Oi+Ug17cyl#H
zq{aX4F^Km3r_a@~UrpXktw4wL3U1_O9i41Xvw!~3ObjzdJ&tv(2|(d~|8Kuaad;l~
znGiCl$=FR*!r}g3!MLM73Q_=*uPPeGtWz_fu74<P5Ze5d&E#WAnYcXLUyaQ2^0*&d
zVls3AqJD}FSGnpZ#to2Y%rXuMN?P|Zi|u>uuqDV_oBdFZLAq4Z+W>g!D@)Bha<MoM
zfr@gT!t<-`>babhdL3^kF!i*2$Q(T6W_@q2YI$b@uRwRl+YV)aQoc8?<0RV(J!tI%
zZR9h3^pgpLJB7!|tli;}#*fnwa+uORyFYreC)8+|mu{siEmxd|t*E0=8|}@{<As`8
zU9k+gJqs^4>u^;SK)Ns`aHfHDpThn=Pdu2-k*!7&SdIC~(l^mu*9JsF$%KBXbIBTn
zUVvtJ#eNd1`TOMGulIQ1H-O%7buztCSkZ@BbS<N5T`oyii(MuHm>~3M;crs0BQE2*
zxx}yDWyqK{b)NF#on@@_r<0wtii}9XcFUFt{F3cpnh&15B(LUO&f)?{LisjUhU3C<
z@a<&DVQfUi(PS|JyNp_u5+qJNtX-(XTBpU<ljTx<(`=?)8so8{n6R5`Pt{zxRO?rb
z^}PipFfD<@9`<^D^Nmk)bBgSvEK`3r?~5OzbCintGq#73Y?>6(rnzTcTpsqPj^OJ{
zf*dv4&C>L&iEm&Nz%(r;jnZbP$x8x_Lohbzqwk0&TE9&rdZ@8Tgpam@?sr?yDQ1Z+
zOW1AI8QLb2YOXPd1Wo?E%2!0jb-(_BBb0|rE%nBY$5W>fQWZspq4Zs^v5qa#A!U?A
zJJO3bes=|$3}?#}yjY`j>3Wenp%xo^0P=G9#&mUw%QQg3%D9C~5O$)Xm`m(3hea{!
zCc+qc)h-vj`c13B%pk~a<Ed*?-%>Py=nNP;FLdjiX_7-L-x|oaM-G$!vvmJoBcm`W
z<wHvY+1o-ABrFwxO^eEAK)tj9%F4=OTE)AR;$bSOye?8laV6g6X9_!J+veSNejIGB
zO-^qOLdlG|ISJ8w64}K$tS1I6)UQDlIPT?|%L9{-<A72#W$EHY$V<~fhbqfa3L%%x
zF&Cfe%j#bc=~Ms`<T^zEszt~nsxfM?Gb4KZqXbp}LQ?IHW14;-?r#;wy?{=gZbzu=
z9BDaTJiBwM!yjpo06^Ez{QP3w`uyKNg0)8<>1+7==(+z6lGqz!X}ZKcw~n88hBH&P
zFo0TD!Yi`51e8Sipmd}=9+nrnTP(`f*m)XrTagq~Lg$5UF0<}EAkL;j$(Z3c1hBP!
zF7Cd%IH^ZA@AcW@Xi@dAd@ALzg6&YQUkWx;l@6+@C6FaCY)8>x^5ng!xuO7D)G<_Q
zah`vlU_tWp4ie7u<3M&8%9h+{IP9xfd4aXiS|eRK;|BpbPL81Dcy~^J)?+ml*1kbe
zYIc2jzLDzoOLEdSlH#pOC^5$f$T7vY0GYN$4o*B%YABjr)O3BpF!|hbi?-AZ=UGFi
z1F|DFlU~DwCd<t=!j8jvO@YIt^I?_aa#!WC)IGnQ*?Mt;**FGu=IwnwoOdI6kx2rc
z(D5HU*nd$Z2dsV|3yBLx>zIsbRG5X<flx3CD&P_-X|1~LFYJ8^fR9iEk5tMnFZZ+~
zoUDW@#B4woSqFyL;WO71gI%3yyL6Da<$YJqr2)b__1e(45XihAno1&@slTgtxZ`fq
z4Gb45QvIvZvjDME&+=|xpi4I4rO4IX%~d+!CC*6^c#RPmNj_H>FEDJQ=BBNyQ~BKT
zMR5dFu0XDWs-DyPf(rFTVgPyaktXt&d)UvOWV>hI_v7#zRDAePp67pMt_{d2$Qfo{
zBWjO^q<e{Z5uhTlnVR`F?jZz<{C9}HtLhH`fAqQC$QXpWAk0KyC9b`xIWAzbxP%Ze
z9WBA5i>BpOT4cFqmulr{N@*1FVJC?hNd4PE>wGT3ZcK1|Xd~qKVA-rZpqE0@`>>a5
z6O>bT9mg5#K-KSE^{diCxtzvpJKkX#)A%Ctrk`>pfv4SX#&y1jz5nQ#C~9kdFiU(h
z07r1*bTtHSlqLHi!DRGeils?30m*63`y=mEwVl5H_BY-$5OhWW65N|1jIMf{eBbqZ
z{?YL-z;<laZm*qXh0~CCs%wA3z6K@Vk*0YnejZNfQ!dw&)YUj`Ta1xlI;^Jfg}Ku?
z-*og(x!$1hd7NAcd(PJ70Wt<EVjWxYk2+RQ%#ja&T>kr22RL$z5<8G@5G0JD@{Ha#
zGqyQy{fExeHr2IjN!NfPMLA}A>c_Eflw7aA3D_0h-w%sSel>Wcc|6p>+u?a4G}i!K
zyaEk-!%+=+rD@KePWgT1Hc3Mk+e6j7BU^Dz;`2#<RUF>)+52(lS*J914Z39_tMP|8
zgPn}UI3y*)mS+b<Za*K1qi%EnrX!d7$<MYEz+>FQ+>6|W3tb&WhcK@_q7*}2P{@@d
z**tOEn)pi1O_PVCv;zQX^-i$LOY4c31wLma0q`Tdnz^RS18FMs<Yr<)(Mw3jKKCC1
zmG>yu&*=8_4OX5vY=fMuwzBhc>_I{gV;5syc`+^2-)+uYJ+V1zLf0=1=gCXqS;%G)
ztUIZhn8U2+$=tFL)_oFQmBhCfW%XV{Tv%waaG$Sf?h6>EJGeXbUYW>dkJ8xO%;k>8
zND)2$BBnd-jU9qbEIqFUIIPAPo7#}cSQ<>c#7SIY=(VARY<EoZP2f?H;!d=VrvC7e
zTQW(lSf_S;5d`jU0371iZ0j)5Jp(LKF~|!99x<zce^Q`fmkw+xH|>tOyae52xieFH
z%W>E8#$;ddE=7n^7)wIlcX0L0?v4T=y|}-26r9)=XSp%*eIp|PTgulyit-jj(SurP
zX?D*wgbT1qYRg@*$}|NnJK`R;1gZrfMZDlKH}WN5(JPdBY3!bg`E})W5a#zfmrb!g
z3?i$oa2-A<i05GNkX0G}_|VxOMJ+wTHRqLOQs}X-vma6_lgL(I@-&zfl1C4+qDfy{
zHz+C?JYqIG0F1j-OO0K`dQA85{7ZyWzSsGIR58h*IFG#SOP!zF;BLf7ds{7z+X(f!
zYhfm)%?O38gL8c_dIbbGmk#?(R8?pKG=3^OjG`LnftvjV-owb%wazEYd|z~e18f}y
z*fpQW*4R`eT1Qx(Y)wiPf8|VuH)rL@r|;B%1_@7GHj%B(t@aGrktDhw>0Ab@Jk@zf
z(sHFIu?v;{?h9D`7jD4Iwso%d?xd2Yb-Kl&k-9hkh=k*;kC&v^Z5G!@k<I@msfEH0
z`}n#{W{>Qh4@9%E37e8>?0)<1judJtCq8}h1~z<YS8{4@p6(5en)|j~{|i*MTa5-Y
zk$_QEq~SB1(bIeVzsAM?*8z&icYE+^jqUQZ0bd$hdsyZ2ROtFFi_I+sT1>0eatG~M
z1}GZw+`lSUl6u@Ho`_`d;}|T{fvkVq_!h7C8CD4aX^v0z=K5-QERqn}>ht3lnV>{3
zlH=QcMIFKqDC?4+>Ey|b*_@LiBR@akQHJpU(4q4I!BU*S8Eri`hDZQXsa0wqGPD4C
zB%^MtyjvxlR6wOjt160=@TUFZcyqMUVmPMO(=#xLvAGv(dcT8=3<I-#y#Q6FlS0~Q
z(=pldgx+&8luHbG@dWatBMhKrasMY?kN3UvP`gvS4&agR#V{<b2Qox5f?AtyuIm9G
z9X=9F=JVuSv(EYZE-X-2ctw-o25LR2lnUAS9y!m~WFw^z%GejK#|{Hgp-J^jP_J(p
zz)1dq1_@)_XH+f5-#ImFd|U1kzVUy(+z^!!=$u^Y4uIPT0JHhO+!sSTSKR<6^wRea
zFe%oqvixT9s0nrWY?kVYC^8N;lEF-kgCF5<gbml1#brb<YsWLZ8cPg{h<<dIWK^h*
z%c!bf0L#xJ=a#U8D`hY~+56=mX2Bf^czkJPld=V`j+9w`z%N)iEJsSQ(d!Pn7)G&J
z8s<$HLGFfPlQ*p-S&gvw^Dov59q1MTDiqe;Mi{Xhfs15SApjch=YO_19U}~BYKs5V
zMC0_%OEpd37ckZNVA9DUzr_LYV{={$L+kjkar96a?-`v4|H0%#lcz#H4I2YrMYHMr
zI7HBu?)^b3Mnb+u@7&l?@a-WfIR+TJhu0C&%b4aore3i(3KPw_7j5%4nU4-_BVN<p
zauzKbPOC5tUzs&&F58J#F(44~+=vGpu5Af`=4n$>oGWh-rgF)kGEMhB(X{b5EJP&a
zAxBpU%LAAZf@b!$_>h&#a=0HB!UE$qOqWq3wjprm{atQ3O7Fd0HE!v|brEPoos?|7
z&F+Wfvq#N8HnT4f<B-(Daxv$7BRi2Zo@zE|P3Ikb&UWY36r;izO+=5Cn-_^WGIM2Z
zmhZc((~Xpr4w^iLWF&h1t5Znn2*au#fDNKgH*WyS@z|DYU6e8VknN2p57ZG?8#Z-n
z#!Z)8jKF~v?CU91UK*S3&6G(}bAq1DTfjScnnH&X-6oMdPi%wiQVCazJa)KPJ_0nd
zcrfnTK1lON{APIQ1bLzFz5;@k)}|Y^!vgQ&SJ|A}wcmbZ<~<2t<+*<cX4{;bJ{#LV
znC&n_oJ`xLh7Gp4nc}T&*UQsz%O&3r<fNl8ZS*0N<4UJ_shXUjw0WIyUl^vD@Ls6E
zb=j@hdGYWN1~)PrC98$qyPEBl7h3jo9+34peTbH|o!Fd|q58J#cxTqrO?Qyj<Nm1~
z`CbsqpRjjG*h@TUY{S0+44JcJRj4ej?Jlo)D}r+G{zo;d<2Cbr^Gl2cqq2<6W@62s
zRTjfE3^Ym;CJ}qD)B$5XoGpoGQ~z;*MOgET&){1^4<mwI#qx0<#6w@N%w}Q?W(-R1
zR+Hrcn#1VfecDA_=Nr&_WZ|8sn|-N#!JJ~64OSIEh3od1lWE7i0{Ly)xm#T``uFSP
zFG_tKk$s5(xUEmy*2|XYHBPDf29Pp__@{#etPAt<ST%mYP%2hz$1JFAlqfl{lOYmN
z33p#i5v}9lN{p|B3*%0w4y5%bbFr+Jgzq;TKn!Gmw+0=9XtCubM=fkVlBcNY!08hf
z)o42hYaFeSZKuy&H3rmC(FJ9Y>^;Fb?<lLf5ZPjUxCUa@7m#Q5Yf;78)x{-(K)z(O
zucwjz`U_AG-c|8o-P(4I_%%X<{$(Nz>r+?<K0sBgd{BKh=YdfO@0nxt@8K{_c>1jQ
z&L6xwxv)*l$fF`#+g3J;`ziK7b)f4k+5I{<gZ2&cy|GS>dpM+9fuSZ0QXMq$$W~NX
zKPSqJQ@;qjqqb|<{{V1>qn7P@O&0>H?PU;}(gEnO$|6n+4>++sP{MeL>InsX!Xx>9
zP1}M*?(}Z4qL#hxi*&i_xoOna*=Kx)4InG^O2(lz^_c#yGr8af>|jbL?IgFU@9EC$
zF$m>XuEX&Co6Ey+*e(olbSr6D7aF3D9c@jrs}O=Z4#yktP<+M_mN2Wa!tc2<i3=vz
zmueJuh=3&8)yb{Qx_6+bhg}%iqrBU27&f=Q?{W!rLs5(`hiqKyVYTZy$zGw<*9PO0
zl2pdgUP7gl?O%u%5e<>=Ij3d&0-PJJ8dzG$p_A(-G9bt9&#;_(mBIPxSEJHdAv{q1
zCLgGB*0Sg4c&I-qrAWF+bt1d@POM^Zb5hS`FU6Xda@cvxpFE9bKd;}yX7`?l#OpA2
z>Vx_$`tHvmGQw%lQ<aQoSP-@)P|*WkSZ96^6k$4AoVB<b!wnuzjjr2Tw2ROp6F@Iq
z3`w)db~C5CL9aeODD(hayiLKOiXMTTp3nL6n6Z2ZO(y~f{ztDqsG!x}$NoYl`t@~?
zGq8`L+&pui_mit3^V}-C0P|En=sD%Cq+0yT046}aBQb?kIT<4Me&D1s4Y?6%P|9JG
z3G!2ZT$&mz*5#wc?nu`j9Pdry7zTsYO{_h-@Dcp;5*vz`vvo1qp`}46Dsv5bo&=|F
z8N&B+UtAZIP70|H*h+3|F>5YwA)$j#%G7X4*v+(cKmXyE5<z?>Efu>t*lZ45V$ycD
z?%Z;`+pG3$W*=|M2*EcDnEch0763(!1-Ob~hg3=I=cJEcJ+S~aULD|$Jy|JBY1n-$
zBwnR&1ql3EoZ4c?HnGh}_d57=OhNa3K08`&%Bn0<-;nxUOWO&MNVJcs)lsZW$`Tp3
z%4g+3p;77O_`EbnIaf|@2g$aZ0os6?S!FZL$<(B>(=kT4<ngLhjLp%+F}m&JgQuL8
z#GdrbB6IbicrRit)T)vSu1#QHo6w1kZvFsLwWgD!s^83{&=*FDDAijOUR<}HO^RXW
z=<uuNejyR7QDAhLN6{OC)@l$UW2}O`?Cc|7v~FrBk<f}51GmV1)0E=kyB@5iIl_L(
z5MZ(RKEUTI@9z#|%M@j&$pUqj+nNjY`h{X<;P8IT3+48tdQ_})hkB4&$84nT;^r;q
z>aui9ndL_NNWRaAhb}#Vue@9Q0Z5XyTBZ_j$bK84=@UP$^R)FUrlt<~148R!%BWCU
z#bnemlN~#PPiGZ4*MGq90xq~zo;#veq^p&la2nWK;_X`^$qdCqiI+1OE&v}<x*=VB
zZw$Q~NRe9X2e?n7wwsfzN*c!?DTb=0rriXy*^<%CVUs7)1a_oedo5EZa{D3|&qKIF
z@EHi)w|t7x^untiaDQ}6;Iz@|W6D11A|G{dL_625cYPb2$R3V+-vQJcd`|0ucya^X
z@hts``)#go+Bc{(SV#G%Ak5a;KIdLN?KC1^ivzx29IZ!(e$BQy1o_o%#f6^MzagFw
z#j4JEJu1%Q2^#-iGYSu1xX8*c7b1+j>e)-Pu=T;LV4G+N;NHIWSe+j%7a?fyK|-L6
zxvovSd2@UlyVUM4@tI{pbhqjOY(qY^fk#nmmH(#RK=b2BL?pkZxZVm7`ojT?2L^Bi
zuJCW8!73A2S)}rg1Psta%^_)-chyVuim>qMO-AE&unW%@LRd0IA6!m=@~G0Q9L#B~
zB`0bW5w=c8^?faXT9*^nQY6%J$FFcT*g>&e63i!L3vjewd+Nua5bvnT&QVUt(Yr<<
z-7JrsSwE3H`J(fSYGR=ki=}c!_nQ<xlV-4^2auIgi^2EKm(`&czR;@7y9N-x;)CQ9
zl|l{HmjM$_Ep5n%mm(5zc@x(wU55<>8p`D+lC*OV0(z=U`XhEJYuVgt$HGDG<3Jx1
z=sN3l<#Al<6zD?pU^VHC7+5))wrhGY^i80v(r!+$@;Yc;0Z^XDK$ky#5;8|v0`zG*
z<M<5PWP_uMK3)ckCiFu1%p7CuQG8vZ_#Y%nkX1V^(AcJHd7M_xbP<)~`@qXH4p<q9
zx;?s7`;KQ$J$f$pCiemJjJ6|M8D9HLgWb6O3%SM#9p$7!rm^fvLQ!*)(l;rKT6Sl>
ziIv{u_&8$<7}OZlNa??TPsfJCuUmRRxz=$RIaJ^g&a^kE=vC&+sOah3WSRhnb?Eh8
z7SOeYgtasai;#3L*H_??l*$8Tw<(RoDsIy@NNEn>%^U{R{KxMKnUSqXA!}b1+Lv>O
z0Q-NJ*muii;cPLy|9rk1#F=%#jB12)xXv)AtyVQBfjV*~T(2)YmD|CnulFk!j`e1Q
zf%`PMz&W!6Aong75Lc&j+Wig*B6hbm#?G8^xWG!tr)`}&`ng6=x6QKP*zHYF5Ms_$
z?=N=brr;o3u_3sy)L}|EhadiJx#uro$fL`z$Mpm{ZHv_~PY0a`Oj5ezz9KyaFu4$E
ztIn^2$`ErV_}z^oPpKIIT7HvnupqThU7^TUAFxS?x6RVrt$#Jt+Ra&`L0vZ0rs)g2
z=WhAd4ibST9U-X?iN4q89T1s){LakZ1;uqz@h_BV%-c6&QyJA<9ab;hX$%{EfZ&C~
z?z8bdw{=5}?pTW43W!_Y;p3d8W`MrdVnbbSbp`U^1*z|dH{9R=u&1l7FNaP<kiR=_
zUqs%5r1vVwyXCc#W5nlzkB`GUZf;JanRV+l<hwz@^>T=v_a-Fbv>M6#U7#!EwBz3m
zdjFZ$&E=Q2{xUQY=Lo>G&lRQq1%<+AY4%KkhryzvaB_sD8FIR_?bYM65@yVf;39PY
z@$}CO639Fc^AzTTc!Eh>6ON7Oz8e5_)(^4w%X3%r)DWa@t#ZN%srYU`4hHq&$`xnO
z0Xe2Iq0LEvqSUba)i4S1N-UfQ5`auM+pi9p6P7}#4{*Drbgi)gY*hLo-_-C}oXk5L
z)WLe*7wfrJn!?0FxwR*dpC2E6-A6B2VDIDp!KF?(zERp9o&U?KxVHUN!v@7l>6mtS
zSJFTRL`iS<h{G+wp7uJs6-_;T^-dwf;e(4TT-uLAZJva{2VfZ|OY9$xKdc&Y8KIz_
zvuXvl#YC23=$-pRhUfe&DtQqlu8oe7i&PU+uF=AVLN+U{W#Rk>AGb@Zbdq(#zuEt8
zy#V?Z#!U5QNt2*Wx^*K}DNTj%q5bxgV^-fkOI#6;(Ab7LNkf8e^)dI~ij^5A?&uT}
zfaa+TVSEMz3&)I;MZ}g-8B{hx(aSfp1OPYUw>HP_z}=a%eA}^}JGC8_I-f8>*}+?`
zAz$uhJyXLv_*@OKw6pHBTWT|EfQ@epTq#=NnL8h@CTg8lzeGj`)s`z9g1I&%067|v
zO?ovL^_*sl6;jj$P?FrvMmlT8jm;Vul&5W_z#IqxStoHPC;H?Xs0F%EYfb&dUU?u<
z4DMz^_M3nkN$#vetsFS&ZoNtZ!25aYQW5~XVk2M!CPWL<q}yXbDmVv8cNG$`h+eoa
zl!}W^5a{B=sGK2uN5Eo;>gM61R42!PU4$h-UyGLeHwVB3hT+0opjxofq>Lf;%)a@~
z*OTFI$+-56li#eXfkeSaRv{}k2HFGZViq@$UAZy)J*2;3bp1zisXaMB4(D>e0BV3*
zWl}8T2plQQVP*C4ZZgu^0NV2$HZAA;8X)H0SarSiz-W^XlCO3qyeH8MHKEpt$7fW+
zR=_lIQ8_F(T)rphDsyldA&NbCD@o~<8u+<r4`Px6B-}Xl$&Ty9rBWv$d~>X5xIBZe
zr^viN4c1Q5-F)n%8Axy3H%1d8MaZP}STltsM09iHdo%|p3+yaY`F`th?P}UE5SIqQ
z6mI~!7Swh$KNhwer+sv*xtRJ@7HorLdDJ6Y4y%~~o~0bX(Ynl*8H;-lNz$$-aae79
zc<^)s$jh4`hOdKyu*UipaSo?-(637SU=>C(D)^M3Ftbg=sk&AJQ61yx6e$eJsNCmk
z7Q==s%==y0R&i?<-3kp!(&vz^mYkUJ#;<{{xZ3mV!}5oC$g<O18%>jwy9LWh0=PRZ
zu3H5LVao-@Sx?&q#t|(+Y3piGdwiJccV67P@BwsbWr$4D!IjXUn`_CHgmcOzDc-9L
zJ>)##pwW<b68`H^M!thhcJTj^mM||2b0t3vh;AEV1{83eht|i(40}M%u$`@(F@yoj
z3v4=UTl~laK*0teDvhO>lxcvVJnY!zY!Dh!?<Jk%hlx+0O_JPG@p273fM^*>kP4x>
zoLN4a@<#rcoOZtom-(aa`V6*x0%F7%AOnU;c%28X5T|ZQK_Yc89Twi`FNTE)6?Om`
zlKf8M)^?ONeU@105FnZdM+?;Z(F*{#&8<1YFtVhzKOl<pt+7}tBMtSJ#0kx`ESt2)
zeio4CAppEVd$*Zc?r`8b$LDVYz55;f^NMA(LE){2Z_N%BG6XfXPWNiJs=6*dyxDoe
zF!(KU9+j{nK{E@mGmMtHjZ?xT_{xlBj&+cAb*#yJ&c3cP8XxdH_j3e5q1tw8lVV%t
zzD`3tFnBESK3^3Ds$nIn2u3`xrC7l;T>-)MS4|CpEiiC{l9#`J!zj(MEDVderdiSq
z7o%N<U?k3RhI3Lp^YBHw>h~%8{WO!^%Gq+Fl+9QXfSjsAtaDqX%5OYu#y1@}$Nhk}
z$zulxfYF=eAaykEu-ZQ@XcvWD?;U`wF;RRsgz?=GLuQC8b=Lbi*JzUT5I+tmR&^#G
z0?_jA3o+}>#Bd7ZibmkUvOqz??X6~9shGX>3dqZ3ZBIi9_4NoE$BVSNY{*O=)uWQt
zPEpxY@#g5bGpZFP=m9FX;5SdnD$w=E0IM_rgkF<(GgO{YA+pH$$-u^0kPf<P$ZEW}
z%Jw=hD@<y-7`5&860PA)Lr~Prnu(pdP3C@^V0;aBo2ggZ)Wa#Llb4|0WB$-s9|pV@
z?xZ(o`-^VzSQksc(t13jSBO{Msq0T@!zqMSWjyVA#b1Ip285iEVe>|>aHmALnE!p<
z{<nJhC1CE3tqLDq)-HTzKrY(?xUuqcb2yIBrF{nUswXx?M*YBJBez$EjKOajINEUd
zamwyebBp_%>V~z@;N_e2@8g}BA`qIpzZz4+W~s$Dy87mf=ds=X482Zt`!%|nm<)z8
z?v%}%7#`npDYZ?G9X^EGAZ`m(C(706U={Mrzeh=##lHY1fuo-<hW$9o-*K!#ErxR{
zCux*=YfVzhCU0jbqWJv#)F+tvu$tE-NE8<Plv-|e;+hR~EhNR9LLoTA$GvTcBJFC|
zD4sU_bdzI=_tA{Zac$q3+nHQGqTxO?uD6+k=~e^ds6rD+aK_UOw^4Bjx=kO@)p+_V
z8Cvw_kg|oF8QK;?=xHnTr8()TZB2_4La{^7V;P@NG<@W_pZ%w^_TA_V;c0^$Fik|H
z_jJ8x_4KFj85V0*b)7Fblz4{=FqeS&=GG19@qzJ6Nt%YcSOnbr9OG=2ZG9FK{kP-;
zlct%DHSU*3l}(}_?w9Sn2Duoydbeth%xlljMF?>B_z<DSFxqSGa}s%5vw5TF-KP(T
zU)l4j!KdCubF6_9>SPqm#qvY05O)@j-v1!(@dXs<H^D+&l&1Pq<8EQYCMR|2LNf}^
zQPo5GQH&UXe!y*iF}YLz;b;z1?$(%^!bYNT_!Cbam2`nJoPWVW2ENh&Mz-H{7(onD
znaOd-Zg^aocM7mxu5ub9$Xhr#L$HkzOjz;8n|(p6QFp0ItfSgiqNJGz9n+@E`Hn}F
zQ|}{y#9y(|4V;yxqd4%4sLyXflq}i^C%}eWzd@V;R_Sd)Iphc;>q?^G1oW1irDr7O
zmszL*k{@2GYXkm!_(b%=vI4%a4Sem-ZTR+$L%nqYM7RR@a?)8e9e^D*?~mK^1FIgu
zty4aAm&gfNqYGCUwQlw8`y)OPf~!hIaB?WH;?mtK5XT6M+H@P(4<!|t%J+WZ1(fBx
zwsTfZfZ{Wnl|L8wY2}&7F9$35Ld?+x@fyTz?Hr@HQz26A_RFJ#%k6IuL<S<S-NX84
z)w9X0-<FH;PKI6mc`fA;uvJ+nhoGj_#Bi!3TM=_w=LsRU*sTNyF)vN<K=SDY39}>|
zV0B?3sEBI${FLh<o{ZUaTugk>0;sA_8j)T?(wsL&;BUE({*rlc24R8wXERRyJVg3A
z^ydfNELDsKP(ue$b7;G{0kEc~DUf{w6v$B!?r~<~0Ydw>M4sql%{6@}-u|M+8F$2@
z{sAdJ_p*DI9H4|EGcw*aijXcRApoGA!WK~aB~1`DuJ1dWUvzzi&<^G5l0e3A+nFJ%
zVFBwL8lrvPAn=x0S?-N_0feqLV~vt&(%N*f!B8NGnpj{R&jV<<TiO)`L8@&JB*BZV
zihigOx68)J1U-sC69|!o7xSiaIwr?;CsS6UySedYKLPW}0q~9y`OYUy$%jQD84DAF
zx3wn(xbZ(FN!Qzfs;Lev<bc)UX+$?RY5wdJ=ubrZv*6V^j%{LR4=N$)`ZK!OJM89t
zN0l!ncaWOE${Od<Z{GTBf;Ifl%Skkp0qt-LW0|lX>Dn%KgrD}@1~KN<emh;XXZG(o
z%rrGtt8_QpW$uFMzCWXv=>s=v2h?(39YYi~wc^#}FWT`qG%~M&+hRwUz-5Z@;%Yts
zr?7rHOB{#xkjHgvqS-rgGYxf>%1NR)6q<~xM_n-2?}IT>Itx2nz>@cNzz4>fUSi#1
z9Ugg`93uyy5>9~U0s~~`3k1dL0nnIiM%4oDJy3#1TBhvp!oBX!f+aBXg=G6z2N%30
zcgH%9h5f%YhOrDf03Qc7uW5a^B)Mj0-QlJnMv}|^6+h+xCti|$91_E{EHe9#G<L!k
zTDKVo@wKa?A0lidbv-J#t84&pu~;>(gy*cR<~%Ih0<6ccIYy&ZzttX;KbsIsR%qp1
z08jGuxz1`)C5r)!Gt8X8dop2MG#%BpcFTZF$ED-ZMEh~Ht^<Z)d}swk@TZzJD->9-
zDN20^a(6%kTllaY2M5~DI~|~@Sgn#l)}BurT50O+uFiBFl>rCP7_gyNlA6;nL=+O>
z%FGAHns{MC;!oBlsPH<T2R~aVm^JaXo`ZQ;P6>zK`je)@fly7UkXjZ5JgP~R*CxhN
zl}&P~yk(>JATU6`>SjwsR$hUJ;k26SmSe*(sDZbgEa2&y!P-Fp0lSn=R)G#2CNOjA
z1h4sh>LvARYz+of;R!dg%)8(z8>T*@p9;?i_h2eD1M9h7T<vv<lsimXgCf9b2&d$#
zcg2A=)j2BeesD1-e12i!nR8^zet9=DmN%*RmnVjyJH_=on2qUrB+<Ksc(ihrq{@}Z
zJn`1-N7Ngh;2K%7XFivJ2ovzernQW3PTIQar5N-o4L|=Q^#<q#>%y*Rw`$XwUd&_t
z3_$Hz5C*|X5^bvC(X_cX9lMeX6s2PARltB5BjkH^9KXFM5bq1Zx0eG&X*>H=r(fdz
z!do4@q*i9d7x?KeAF#FjRRl%Ds5Y}gu5JUnV```LbgAJFF9#c7d_CS1ygoD5x5-I*
zis2d)Gzruh`3y4n%gDT2cjOEt#ZlL_tdnm_2>#lr-U9>T{78e+j|Av3Qk|T<q(b4$
z#m^6T67qYW+BFPfmY8Tts5Z@=g5{4Eqv<BLh;8wK#X~KFUO<BEnp}ghVaj3W$b{9!
z&k}0|{6kAgHg_3v_JEHum-^+8y8)m|;mcVukS}RG(W3R1<RFD(ThzPk{t`Iy$sra!
z5`eF*`Fq9$&)jcOHL$7-KcLgx@d1qQJ9ZdZKksH?a|4jhcEH3CnCEfw`!lBMeAo~9
zCRO7wBGm@0rP%=9BdFyJ<9&6rJK9~mK(q690Y0{G{4}%4w~L~HWg>ovWM$JEPb2GA
zV=9ngCbL#cc;PheTV?<Ib^q+Ezs&mFXlvl_1pMu90r(txd1a#XA{&_!RM0>e3J^W7
zMO5=uz6<#Lf^0Gs2m<?$Om70~4`mWIGS2Cv_2JP(^E3<Px7mdwtN_9mK&GHJ1$v&_
z90j~=suL#Tia=dVw*&-kPEw_12P2(2aQ~mEbO64kD9vMq2q;5*YiXW4ieLqm?1`(~
z7FVME2(S(pUL1gY^&PNfUHM@Xtw3aD(=;%0JNLO|e+r(9e#mCLbsz&fwK8e+j%<an
zRw^L2*Rc;TVLIZnVoyv7`aGGV*O~zb{Q}P*dX4FJiRmaX5C-mgOr8VBqn7@v(KHWW
z&nd?vC%k>q=Innwu-uz(@%hXxForso2TUMn#fRT>)06<^v8SR3OHbS)Neh#89iWca
z$po(7<Sm<i#J4bP5?*&f!E52@a~XUVJ$@fI7{t4eXZ3dZ8r^x=l2J03T^cXII<Xj1
zj*7ZJQIdIL<Ys^sOUkDM!-?Imq02<R%<^@yN0xXvH2(**6eB{F#yEz}?<Ew7-^qpU
zU8u$dlF>9g-&vrz(fi8B>%+9J@vEx`Bx<nkd$Yhs$mRzZ2@$`TEl@9MUVef8C+i3O
zy#0jd`BsU!$4sAcSS}%KoA&SefD|2b1y=JVTmw07G2|T5cpKP&-Vn2m`ujJ%acD_P
zpnE9>?m;gdi)kTiNO={8^<can<ohj{b_S-LDh##($MHRYiNyPXllKOwcT<c=)vnBw
zuT0sDd!ywA*F6RCQqgV$D(AC)#Nq4qq_eW>cZ08+K$#5z03fX+F%+fRZmwANVMJ~%
zryt3fF8^&=S%u%v9$m7`d<{*;FmmAS173>>sPt4f&o+f|uTAyrit`rRsiSw*ELe1s
zSgG&$pkF>&^D@=BFp)d{K&w3o5XfT~CBOu#X%REWK9+z)!<5cb47G!ui*>uP$N$B0
zjLov&c_N31dFQo~b7$=8m5AKeQ+CC<EQHC9j6uvXcP4&9MpCES(Yq^VGHQn3fKtzr
zJ%_xy<5UM$>8cgNHH>fE(BcIZ2oL{sW)JyoVdnz{?E)m3y(;b(gDrqF)fj4%0_?Mt
z%Fl7ZXd6n4D^6N5AI!gkC$m1R!A`E0SVOt;og&lDed1RV$$W3XS_yT8BTZodJCVjB
z{Yzng;B{>P>+c#lKQ7X+$456KX@|kPZml<vrmGWa&uc-DRN?u14NYR9#1_+^%Dcz1
zY76RD_o?^>`fE2#Qvjlr8;e=JqG6EMIspv7%*n^fZFeH4EmqNB>|_2u6Eso)1SAo+
zHa9AUINDQBjLgr=n-?B>ps-uZ2#Bit9o!4-{pQQkKG=n<lhjqHf;CgO;DbL;Y-Ng*
zTdJMb<!qm(vy4VLi=&H1_Wz;mD}bup*8YzzAaE1`5djrx0R`z)1Vq@<T_Pp1N$C<5
zQBYDqT0)TSPC=SYvuTjpbhp(1dC$G~yXSoWd%u}8!#IpG0_$DxdY<Q3E8@*)yMcc2
ziUeZs+`1A^`c4x{H4$16D%uC`>HRfe!itv3J-G%BBgHxV2inX3F6?7-egCx+AkD7q
zaz5-<;5)yioqf;hXZ3EaTUF(JYFZH1()Phr7<3NjHLBbXML*auWhVjDqJcAjs_YCl
z?@FPjhbKfJK53tpg2Vj4FO)R1P!lNs_cj3yAaImeP7(%4^APIE&@*d|qgMoMj|9eO
zB|c=E+JY#&^y&9ku+)a3A7|Kv!(N@fnshbov@o7XBq|e~G8nL<bsGB#KsB_$+^>tr
zm99iX7GTwM?sAv1dhR}0ep<ec(|(Vi{S5XI8NRCi%3wiSh{PUOcD;CEo?7-qA577U
z=A+MMbYERJ=V@{#(KNo&u6_Dz4}ch(?(|pp5xF-MWmV1AvX&NOefW9AD5arYb|{+O
zdGFfCqE(!2HwLZS(b(-vC9T4^Gk*{8Uk^DoKqCX)=s~9Z;DTd`uNB8#*l9x*emi;k
zAn%MDzNq;Nu?kHQbpmeUu5i?Zqbn65KWRoJxGL%fUBY~Mc6J1GC%Sp@%cYK=e){IM
z3v0q=82JTNqZcIhd81&5iIx78r=Khv8?!AKbI&A5*#%}%)X-ak%tE^lR03GjD24vI
z2ZOuT=S3oyK&g*v8RG`O#-+*qr3p}lW}x3vGr53gntmzr^u4KM^!ikjRKxr;;jZ5M
zPf>Rw*Y%%bV9LF8K_W<DW2U_iMj}GY3V;xFCIkD+*&Nyu?E+1^OegdTH9>JIHhE>q
z>|hdxtpwFfEt8?Gk?HB3d?*cXxQX^|7vViK1(QW^<wdpNs><VrSn_fs_rK}VW4PAn
z&M?JxP)Lvt2I^!w7Kw5tD9ZyCtIV^(#KOBzvz#Y~@Su}FhkYc60ySg$u7N-{u18VU
z&be>#ON<jMK;GL~>i63nvZf(NAO-4Dre*0IJyf^n70|k8Ijd1IoDhP}OWH|T`srH?
zyk%8i)5lm6vi*av(?+6U&Eh_EZxbDlxo)5a-3QKMZ-Z6{O?e-e1xO9I;w7db;LG2*
zu_z?Ivl}T<`3|(qC#&@P{pXm(U`1)y6`y`IU~mEU`L*XGg~#PYw4mOdTpB8p?JC_F
z@g-*`+B!&6N(tI&OXO8J*`nV-y}bd4!SkWtI{U{TS#|QwM3Ky2nJQBQXF!tnUAk=g
zV~=BpY5c99_<wJ<IBkF3>b^07m%%#V++;5>UCTBw@{Fd*s%lx+E$_S*E`f=2JGC#w
z%84%<2LTFQfPv>51!q)WEjhIX0C7yg;`RG(z~S&@s%K`N&{;@<vti0_5rJ-Pmc0sK
zaAlh2tAGY8EonM)MW-POGbwz0V4G&$e9f!VW9Wjp&dEsgQ)V?0`b{=Z%cP8QA8yz*
z-tg{MiME}F0b}xwKI-1lo$ZTiBr@+0E~(Lu<<$Gr&|G*Bq!4sB1Ayr`7li6!nDg8j
z0ZB(zXm$LZHr5oylS<}A@T0!LRy~{)26Iz^!9Cjosh4e_=DRk3g$OnYE05**T)eRt
z-<#j_(hW&I#Z>Mcii8s{HvG-W_<WtG9P?I1V+vpf_p!H^<Z%E=jHLmI(6DU%VT?Q|
z@GGd>TzVHC*Oy;a<8)Zo(k?JnX3)xou3SBcUgq;pRqrU?nCgh(VxyAMv!_!eb8SI?
zCs75u-nQs#90re?HwL+XPQm%`MBU?eYQ0Z6nASYN{N(DH)PFbUW>>!q>vyYG;fie+
z_*Ej~B%7h-EWq1W0kUbK)HdVfNwf0ycLo(!)8^1mRPaRoY&`f~O%cd{IEgG7u4u1n
zBj`a*>!(Z{Yhes@iDy=x%Si3HQ)*_l%s{#EGhyoQ2G$)Gb2q0DN}zOhU>f2XErs?W
zZpQ;rB$OQhUdFo=i++j$iK-p&<O)ZRlKOT_Cb_B{{$5Q2X9Ek-<=qugjF>%7o}5;O
zR*6Cgu*6o9FpPYa!A6WsW1{0}?IOwuS4L-c-m7b6zyvy9^m%3yEO%EqjXumw=DB@Q
zUrmW|o@eS!mZdQyYvh~rQ+NC(Lv|*u)L_)`sKOk{C}A&OeN}p}-coTb#<t<!znb}~
zR#Q>g20DRn(1y<gb0xK1ICE&@j%cAa*4`7D4;5xO1ImA?qCj>9CZdw7f{xkX-OVH+
zqPLQWcAR)GwGbifR?6WSAT=htQxB;V$^a{MQ-yBr0CpA+4RKWUVSsJ>=~X81{8yd=
zqAZzXKzYK!#a+l{+r0Oie8Q&&@Ii1v75>za=xhtjaP#TZME+YHE)&ta#(8#&{ddQW
zGh2H>$!nh4R+0<sL@fEhpiKQ$L$b<q7CMz?o3ov@icWiiG$2??)+eDd;E-k{xezqV
zI<^mO@>OBi;uV|DlQ8is0lw>B*=p-gCmpI}b9gWOVbs=PP6M?f{3tKng;2fUb=~*G
zm9un&dT*GFnT?bdMDW|04o<$hwhGgGDk2Jb)Hz}(b3LQ<=+}Zn716@`C=x_K7BtqP
zWsbN&I0h^~;Z3}d*KX#Q`7MuQfDeY0MHESO)|9J#L3pbdv<NDhsnTlgFpkE_{Isvx
z>BsxKjeFa0IQF!_=}MOD%lCYr*`+9awfCO^$RsaS7QZEW|KsVn!>N!DTt;#Ep&vxy
zWVUa+*-c?~OaH#kXrmg*=-w4v3AND(!u@mmIz&H@W*ce*sv%?0I-KIBOcMC?@~<Xe
z=JuV8)@A2G4;u7lA+!BbFuMNg-feDzZ{3BH4o7V84yENt>yg|J$=3}sWE4Rgq^}7_
zvSmr{+tgnrxtD?%TvmQ|Un_y)+)?GtP(~E~W#V3i^l>4cyOeW<6gIQ(Fo>_qz6><@
z4;FM^T%+G@D2#OZ6lN_g5%>I1PvmVNg_Zw_>3&~;7lriAYhwiFXLjiR9*$RiqKaq=
zHcI!}Bm3V<S_&}sAm>eitN@ABVmNSm!3K7@m_H9I1y2Qqu#uh%k5uo3abh364TK}$
z8>mPXK}9DcRTy!nP{t@d@WUg-ub^yntLg$`9)W^fv_Auyy>-nx>W1JP#4@+b?s<w>
zP!1C-qDy7{<D!bp4`mY3uzaGDg#&0}qBq^xEgPMRK|#sxZM1ZyDOAse;TWP}zJZf%
zbl{4C&MnFh7zF-iYFZ97&BLhC+Lck4^_aX8fNfbZW%3i1`@A)Zk5g|C$XjLQ3=OVv
z7ajY@g~?#tYiWt2-~(WwQXX&$7m2~gJ?AORF3C$(v1h^0ZaV{($bzE_K9^unkL2*A
zF$G^hr#6LK`BLFf541ebdT|g?&;u}$rLtsv4M1)0T*)3Ku|}deMp!lS8<TuK8A2PE
z21PRSQ^T&4El??sbIBuqQxvI7_bY$_sMjhKKRAb8`lrdCB9i{%0Qplw)Bk*W-T3@e
zYqSXrogNkLht6p(I}yJHj7;vq_?R)zai6Yifg_K!c|)s7jh5mpn2S;{H${b>R{w_R
z+02?=UBnjaFJKJnBY-<p=ewxs2rk@T0IhWZND~3<e^KRBkn|>C6)CcUYY(;4R9M}h
z<j&p{_gSpFQ|NcSy-AIvt>nsrfsWhqP0C<{^N&K%i5|yC_wz>*KHypOFO)ZO;XraS
zkkEc#C*Q2EZH{+(xM*bRSBzf)yc2W!@cp_AgEMK?R-_B5hB819Dz$HT@_u@%$m9pk
zAS{(gsM~k}YdgeNbaBd(+&1EJFi6kY*9ut?gzh-9_u3D#(6KX!g{6Iirhcyno<Tp`
z8E3XJ66w&}bFSBdoLcNTzppciETBI(JG=t~q~G4!f1RFDiJV!fk+0>&<SqS=BLz38
zbVt4arbzy$s2^W|CJBdW&o?15g*~?%uyQrVvjkV%KL(+fyYGs1d5NaoEI~2MC5_J}
zA>?51$Yd}-1<@9;K}ghdUs#-&l<KRt#+K+@yv4f>_jTT>UaD3Gy{)MGX%o&uFjAJ-
zVb&)cll+imCVgeR1Fg0Y1GmtH_$B1+N!(&E<LTZ<6<!_En@3zqbG&5^&}}mW3tdOY
z!nghWLBwQejWZc4SO-!&wsw(LF<+N{=&kZ0&#;wPSLxvguBvO&&}l~B`gwz$4+`|D
z3IOV#zDCKJLM{6BS>LaT|5Pxkg!Gq`C2Xae%Z$DgbW!NCukbSK&m3N|q}#3A20PXF
zt%W}JPnFJm?dQ|S1&YXJdC$uJO^wJHZ-C}0=!#JX&dg(3h)7&}7_*mMOGMyyBg_~2
zNR}5m!Z$cc)g~vFC*h3r1@<tN)N<@W1L^f_dV#*%O)m?e@K=k#1}+Za(P^{BdzVAV
z2<C6QX7|S`^g~M^QIj2b8<tZ~Z41AIWUjeX5^I$R9P>~5!=P*KCsM518Fj&1@LMX2
zok6grWzY&ocWpO;%lfvTEa{hN<6#?fgbQn63m@<yV~u!m0OWACG9^s)D2<}AHJtY(
zUOHZqB4oJ){<Z`o(;3R=uM~pL+<D$%b7iJAUflQ6X4hCI(3y>Mc5#kt6V9LmKjIJk
zISZk2ttM4(s<d>q4aj`$RXL83((EL1Sp48A%D&QeKb+Rj(C_ap>2K$*@ds$$AqGu&
z1yI|w5vij7ecBGu)QDayxAOtaXzh&;0G{~Nrt}m8uKTOeqoV$~0Xgs<s_1~%omp-u
z#|x&~HZC*pLF9D*+~0$P7(MM5=39B113-Q_7M~H^R?EI_C<2l>5*NoWGndLInP(?J
zctP0X{X7ov5xQ$xu4Djei~4~)jaTnb2gkYn8IF)y*fNr3RJ%E#S~A(61n`@m@kC~h
zezueFvmfuO_5H|XX6%9RPVheK{G#IJ=v!5`PcXm)NbqN!#y(2=0T!Eclm@R95cQ0G
z#f#}<`0qb|Nj`F*a+(Tk%k)|o@bk@pU%Tzst#Z1bk|zVS(V%ecg<r&Wu;IGS?`px(
zE5rA3J{+y2e{U87;{BWr@dQg&`LhDH1rS_!AfSsf6?jJ-B*<)GU@fVRK<pCD?-6EC
zK)%Y|Elv259#EaoqJcBEcen!4#vCP<{iFeny^7nX8rb&IL+#LGX%^WcU^JRx#Bur%
zf`K%)=#+o904{B!Ty)u<RL%}Cam^iZ4htu}dFJeT<@y*P0X5GJ;|t=oNpZP1Nf_lJ
z!B(Z9pJN(&)a&l<;$I!{lzZB%Is#EAfw%e$<<nORib*n^(3rQB*h+G!E)N#4-1k3R
zvI)Wbfu%tiT<ddul`!paE3A)l?;5oB7Oz5IEd_zr-LT7$<~0^V4q#87i)L@WtNZt~
zcrQ<d5=~Kys%IPLp$Bs7k{lDw0jlH3{o)H&j5AoM&)v<w+~`3y0qcc~B8{#4_`pR*
zn_(fOMNh~9YH7ATWA7Och{3_SxdIf?wQW7s7>HeZ^qWCyj-y+<(cKkT5XJ3rkbV?U
z&%hfOWiXRNxESP_NO1c-hXbh^EgGUXZqQ9Wk4&LPzeazh$<+pA$7KKJlYaA}VH@c|
zZiSCrA{8+D-wC!60{G($!pfRk;XM`t0;LqjRz8<eP_UoFb6&CxGY}C#2)2fPU>!XO
z8Td3ReM?KRkA5b&wv`Nm+ck;q{D|_g`N9o2qQ*+lGauJmDgWkn6BUK+HhljaRigIk
zhSS<p1%QNoV4c00O>gVT1;~!A9No=nL~X#rek<71SK#A6Rt%JfrqlkHL&f)ll<9xg
z@f(&v7Sudg57c(nKtUiq;J2xL^0JmWC=;cIRG3xZv?`c?d;9YfWe<S25IsR5Mq~1(
z=Eu!xV9TWWMi~jhKyj*)-ht!^3KmeJMYeN5o-h9a>kopR$3y<0DAp{X4JU&K5LOu7
zf0y2cZGb7p{g(;r{pZ2-s4l>alm1AVTBL@9_2WVL7FH8UKJhb%2s-m~ofA5R#=G3Z
z!COu3?kCM&rF?1u*%3v}#Ct@r|D=snMCX?fws+>`fkh6C@aL#4;oEM)FEw%Tmpkiy
zVel9e5BsaR1KZLQgoo*+4}SqO11`}r=P=*t$sap*sFlJnobykeuxB|=6!d8Y2O{ZU
zaYo>1mxIf!8uyajC4G!3$Hw3<=0FNQy-9d)qM_O>lj8s0?bbw7^uMb=ipW@kj<~&F
z<Tf0hT2C?1pjv$WG|VKpJhisv3{_TTNs{0nPf1>qiZuFUS_7ad^GK6m!w~I&h~8|%
z9LP0+d+yrK08~JOY-Z3x*KI3A7*?XYFzoq(LL0Z-5-=@~2kb$|x(Un++B*d?4S=|^
z;}e?hYQkdhbwX8wunIOcdoT>Ff}8uA`n3S_4^7*#c35@9L0N!fCw-U25!*EAvkYp9
zyo@18tyCMdk6-%g4CzU}%DU#K`V_V>Es$<_I<iuWFda<Ly4fe_80jc1fi>l4n0Zf5
zj`j-_mqBpEJNU<CNC?F9R*+YN9_x_j^OqkWcA)-7ymo;u?ZfFOrGt=n#i^YnL&4zY
zyOM@6L)0J9Lq=_kEm42YQkyz+4$0PF4bPpnjD!*({VS~(1e%eyRSe>L&4hXiXlVa5
zr_SEhGq&>I_LgjO`_m)+4>m4Icr1<JFiiec=K5`=>7=uyRW}v@Hrz=wgtwf=l&w@o
z+6xCeb;pUFODoj>$ForZyH%87SCIJ8rShMKaULAW235m{M?Nf>9Z0&T@~zd4SodE|
zd6T)keRcE$JOGl2v@c0GAz6B*d*C%^K$jsCuAP8_=kWWg7js=EL%x2p)JQ!>26%=g
zFgfqX41r9C_LN$#!F%uvcMEOZcP)nZB0Ac#tJ0CIg?qr(WFN?lfSaB?P1o!ox6Rp<
zU0vi>H5#UU(1gQ?cZNSJXp5l{d1X+#nV?FDwxzQTN<(odAdxI8Gm^j4pqnpH3-<S}
z(%$w;={tIq{eEeuufGjQ<U!k^reBN$RpyzpIfsEWK_8`sSAq({Fo)n0nh|i|o{1_9
zxBb)NdfuU8gzj}}r$*`fy__n}>FMf-erwm=aqnOK`u>=vvHhAT%XyshtN)9kb!tcW
z*n1lRnvqfzS;$z@TC?`Sei0gHRlot8)FSD#CBn~4s4Nje+yZdKkR4-AcGT{1l+a4m
znqJybY^`rdMxB%{#!Fxsnc=W9+H7!dUD*pSRlZ$lCyIlD+ciYm-U=GE9(Wc(6%`Hc
z8^q~CI{R*&(86vtvt;eE7WS>tGm3{l4XW7QALuMZYi>bO<{s0;fm&X*0+8@s>J{|Y
zB|y`pK}aL4<pxofF$-MkGCf><S_zpzVH@T)II@K;S3`wQ6Idzd9MKEhL1C%j$yqw_
zbC2AopA`%L?S^6Q<F5ir%9&c~LI<n0slXeU_@nHq;aIu}v6JtVyi)*9O@(<mQLug?
zQsiLuH3T&T{{pRA7g=C?9C*E$He_e=8=Uz>%elUL>JTX-E7OV}<G1Akt1#JvsFPbo
zdP15sawOLv?&uqo&TpW=yH5rDl1sVqeogz<WSlqkx-0dqg)lrnxh@>g&%sndFRGd%
z5&$91BI2L@lGu>wG6_`FhbYB4KY|T}`Ib@ftww$RpwMjKv!QeZ>L*oTVjzno{<TcD
zH;1R+`_c7^EO_e!%}Q5Z9kV{ojYyfO^F7D9oUa9Dk6vl^3WLKDrb0!8d2n8DV$gRm
zTr9ymzqQFnG8PBw11(mqVmVSKg<D4%qFIPp-U4uS!`#5Hw=Hv@h-=wwxxD}I{q)k3
z-4bfxeK8Lv`fZ5Si>+vt)G8$S@7Lu%@pkqwHtkdyIoak(ZI2b{0X3p>6OMWbVQ<?n
zEH6&{+4Q~_RNWXzOY1?o$&X+cvZxrrfrW7jp*Dkbd5EK3AI5>yI^@8;!fC^(-H`YH
z%&rMAImt(<Mtj|$v+(w4+Y~lSR|I%E2g7tJXrr5zo}xhc65X27!wFM%0fcybiiv`J
ziNLzK^;sL*?0Pzshn>`KeUR34NV@L02@Y4<?_8I(o^cpGBs+%IF1C5NP8)$H))nCo
zyV*GJR4`+Rfyc5(A2UiE{PgUccNk{aS|pcI?HBHuW|ez|Gn@%GJtYtfTu1q_C<X+}
zUoFa(xETn3eq~kfUo^}k{QVtT?4;AJfXA<X!?i_I?yF2W!I;@&7^Y2z7y7BhAqRSC
zSt@8nW76<Bv*`!%g5|}fz6A2J`R*9H1BXHB(+6hJB2Y90aX6g%$Q`?$1Lx-BSLUeg
zh))e)T@-PH2EXKp1y^5cz_e9b8`2b-ERz)TsN*wwC^^PC6$x_aDx|=jD4Zem<kD+<
z@~d?E7birg8FILM72);2cH_GbJ+0A~(NXB3AHPb|M%%bu5wJJEwW^WnWy5)5Pb;Lo
zN8gW+{?bpzu^lb+rRS_;dNO4B)yg_AI>iGcPZ<14I_D{{821wdNb$EixCiLRil#Pb
z$qS)_mJnumy)AyS6oQ36=N1poULT63ViIVK`~yS@;5um|7W;)qB!Kdj<Dq~U_UxnD
z7`#RYmAy*l?_=vc%y`A2Gz07ei_?-8@fQyRBNrP#(%kwMPIz*k|HR(VFo5lux31SM
zgvOpUGiZfHRe&aDjo;M|iA?^lf!qdOFB9_{n_|Ih^uy>wd*g;oaw3l>7k2V0p!iSP
zVpX#MeC2u#!_!y8r^`1jN1o)L3^sX)1P_o=m|EB0F6}l7^yD63%x|*)l8lNZ6YANU
zWJ-<-vQL-aF8rwLx>gr6<Wm?v+X-;V(&IOF@ui#<mT<IS^_%MOWw=SkY!S}zEypLA
zI2K^$m5mfgHp}}>GIEioZRoDU(vF}`@KT-AzCxUObt>kh${~`92#lF%qzqA9zCK5?
zBXwF7lC|{7-5^bY<jc6(L5*m0@!${L;Jd8rdsdM4^E^0EHnN%*O?Gn*&<Zm?e;Jk2
zsu+{t&ek2r+G0P*)JeM=o=zXrpvQq#D==IRoiE9Fk#1gP_KtmSu<^(K`UpoAnm&N;
zjwXprJYw2lzLd9nC;u1bS%0-Y`52%SUr~4Do*%ZT)X^`*l7`7d`F3&}5TUtAu7n!b
zU!~&oFWKX{it}>%?&;QjBG~%gYJ(i;9QQ6XcUG0{^TndJCES3}H`{ahIEFdRGASx-
zRqfAyEpj)nX82*-%3K1JlHuwZ{}K~l?(pJ*68S)Go*>@?_DKM}yKxd?8b>P6EzTyP
zo@)L4g?bsVviA9Q#h^ht(J;LQojn1Y)&Hr!cOaU2Q6B**J`&4DmSoeBe8e7{z7ZkA
zDZ;^&6lNR%QQXE?LZ-^BvKqs#Gg;`soO+fND?iQ?6puMx?v|82<rT74U4Z_9vMDr>
zQBfvU<yKlfWE!3vE8$ii(W^8*X|Dvq8?IbyQ0>0KIkd+dOmMD+OE&jsqqcs9MR=py
zWC43&L-_G33!gEai%%@(bk#$S`A_V+qp~?syeb^G%b__@Um>7?uEJB@Al`?ZojN_c
zR)PMCOp`;;K|VCBZKg6HV^fA1);ebYKsV=^_>|v75Cz*Ty@Se=Pll||-AWXm<aVBq
zEzg!p`mJQrqXV^!&+6c9^vBEL`SkRy$ZgJc!Qgx=P{4#}r5#$(KNSaBgI4culZ6B1
z?yk7Ya_Bv_R}_gfAPAJX0W5*yl%%`JJ)T$cK|<~A^oRFOaJc&{3Uia^nk+A0kUe>k
z7i6&^B^2+1%suWGLY^q2?Z&6;;!Gp~)ld-Qpl2;j{3-z9{m_TzC9W_yCm;z-RT+=$
z;~z$9WlfcmmVGU=ucuy^DJi+(rOYiUf-d1#Ir1yF7~wa3^*Bg{DLiKCSTRUh@re8S
z5GWOB_J+*59^j1zR2gn94XK7x8htgF^}(0ki%d2M<={DzlGB{mMlhmOP>u(+CUsdr
zbKeWI+^5`t*kg2JI-{6nbIGk4rFhKnx5wH}(;@Fi9~aAV%tb^d(YeaH8-o~o=29iY
zfaIY8t*n~Vt7u3BP+G4Ijd^LX7P675mh%jcn3~)CGY8MG^e(l)e+!BCESJ2Vyz)Dp
zIsr!k5-meve82wcSI&@}`lyH}c_$5;TXyYXOR(k2MV(*QKN`Ox!s_&E3*-aj?_OsM
z;6Cf;tEUGW*RwlFd@}PJbIO7^3C3{Sn$tzV@i!jqQkl6ot8IV;`B%&WBYWO+*48<^
zugegw#<l90FloQ_s6OJUViXHdDJ=@_L*^XVhwZ+Q95};wo1U@fSLsu-)u|F|m$^Hz
z+1!ZzVu%H6h|gI))wL1jIGZ?li@;Nt8ltZrDSr*UJ(^<|Azr}bzw;}};Bm-pUoZ)c
zCLHw_PQ(I-FeWrsM`CMBk>yM$==Xsue5c{`1YMNx>=Tmo@x23o9pWzSa~PENcB&#r
zg?CPP<k`humHgO}UyL5AG6mt)l5RcnlIkC&+f<t*^iMh}aD{9Nm0(p<6zWom0VVV^
zKtCCL=ac{_k_y_u*>Bns-BbuXM|YnZc=lEdl*Msu1N}%Ie0~&$5{DzG{1uz-vFB7(
zvX{WkV=Cpl2Pv+ehSkRf)5acGG;vj@zF))r$9dB4S|d6D2`wEI(J19dk`0=7X+%H(
zY5o)i<yd<zsK6N!n{c%Z%c~%sL^Gmd^IZhEhdAP#((UjvlF~(mZ+0X0Luk(i(`C6U
zrej*F)K$N?EQFmMs*ZsB9ZZ9p8n>)m`;3=YQxNFRu@W=#a6{=14rbL%!xOLjoQU>$
zPnx;1P%?p8?2d)b<C610ge(FQwbrHe<Qh`F8TufrHurd>+u(k-O)9kEPqfbO=e_=w
zw{q3C`r1#?XXjc+nzlgg6|CdSW8VJ-RfyK@8&vWjzDFTh&_t9g>6UrYEdF)C%019a
zjV63F*pPjSdd42K)-1bk*jcb70TbFG-`Won#V3(XKj!9!{p|(}3La@BYso(I|0v9m
zjq|Xz<Pf2oX0Qb1KGTg_#+DcUFhF!dEZ}1RdU@zpYYndE&HFZKC7|p#6ERzh1Kp`d
zFK%m9$v(v#3)*A2`xZDUR<^){7~V78&jXk3I0!Y_`Onc<O#H`OZ_(@LPLekF(^O3p
z`9XrpheJw3#&$#(#OnK;lT|*|BhiIwHsFRB$Eamk6i0YY5<oQ?%WdHqsO4|QWq=G)
z0!-bL3##*>xmMFXKr9vj{Z!W#L2^5Uz=fXfS0_CQX-VNLbx+p%N>7|R>uAEf&>#rN
zljlq?h;hn;#QVc{7oVI#(nAPauk)_}+0+5tt6{2<&4iHNNV+=b!kR?G(nKKOex{J&
z_aQP&0WjXFVaRGO4IQJjaf{@)OGTn;qP578UF0AOPf_F?n=oa{Yo{Nc<xADzepYZl
z;4$rK?3=Mjm2m`!bu^8w4<bje?2Rk|xiprR;i!rP@34LsZgN-fGy%cy;IGM{NxS7J
z76NiM8Lo=JoheKG+%LzVi0KWo6SMfa4l0m&V2>YepJ}@K;gmg5<1w!S>)Eg`=k_#X
zXxdWT8$4i6AOUO<bBxOY-2ziPUDNYPW8r81A-QZGk|DN;H^FnC)=he=C?9-YYQ0nf
z+cJX4HlQ|s<-nGIgU56|vI#Pf+}eY+tU`o;QvDlgJMdT>!7JC=IOJKS-KHbF^uGU$
zvhuz{!DYImXYZ%w1$|U*=ZGA?txAr6SK%|bLw2n2@oT}kuh{-GUpI49@s4g?x)G|@
zX6vpxW3&vxska?)qb0#RP8I7B;d!w;E4H{NMr}I>2BK&G=i_Rq=i7{83*7*RhBHHu
z1ZO|8T@;bcoIW>7ZT4X30wp)F2)mgL#6I|D5i>$8TF-&X>=~w!h_su4{Iy9-kNqJ#
zm$3yfp&c~R`rlg^uOC18(s>hik?lz_=<Oo5d29WI_NIc@1UFh0ZHKT4+4SXo$iy#%
zdjXj<J+24Q$8Wt)j#=&W8c{(!T8m@Rh<)17n!WH5=y<hXD84tudhwa;Z{)=urUxk-
zwUI0f#}tlg)GvW_EPN=%;8jh;<4>lM$vR$KH^{++@&0%#%cHlDb|hQLk$BP6@<rXA
zs#^C(tPm-7)0OgaEJ=(RF~w&t&O4C$FkCeO1WM*Sry(PM<5lF+kRUs*_-uM}kxkc<
zAD5&*nP){rRY4${y7wiP^tp-|>D?pRtmGsBVg!gP43vUNH|%w(YY0`>-<mpK!WRm7
zQ&XC*H`@`H!uPFV=cmcg1)j`#v^suWn3LEnMFv*CbSB1ST}8V)e>M7vs>Oq_B3Tgq
zD_|Nl(&SY`Bb5ovAgG|MbT}Z5Wm_o6oOc<Nw#oAw5_+heL$`(ZTK!4eAtxvT{{Ih8
zb+%I&k7~#=eBP^Tpi)*XL5c36)t!YO`lu#imEHlM4^_%QwsxB3Rf@t9xK{;bTVd=)
zBLQtWw%2G0JH4Q-q`3RDBTKicFy7BX=rcYy!co@&wwF5OasWyF6k%nW)@RCk1Jo_b
z*XkWT&ydY_<zUmY>4!(yhfkE0d4tR4P7E-3Wd?GMFUId)j@$Z?^C@p1P=W~vr7g*9
z(BQ>o3NxoO^*PUVlVTIgdi2eO2+V5sE-wGv2SB1%zi+<O4|<<8>ADb;naY}onB&Ei
z?4+5Y>r2CTo|H9Kr-yiU(29=!?sI_|^8?V{$f8uoUS(BQb=3rXvl_oQRh%KqE`zT4
zW*Qq>jSG6Npy;&TC1PYHuIG}ZE`(T-;hk{MD3?rZ6O0KM5RMCUrpIuQVq04u=;)z}
z^&V%;pcqdw^0)%+yavYrwJtO|){K|%gk+RC%nN!|xK>nm-8szU)PmVIHe!zjyY~UJ
zc&z))7u3hw{jKP?lssPMZ>hISHC}3(RhHS7Sczq34$DaEVOn>}mcep0!FGz{`hWW9
zcz2<7dnJUP7$a~?cUYoHJ1$|0O`UL{X^QBfe7}UvOu^f95zDVqdZ<q?SYmjj2TYoX
ze|AEHret4f6n$eat}w%`EY|~IO!rTlq4K@VNBdRB0)r51l%|n{4M!rF7@0<Wf_%S-
z8KN@MOf)TEu<{GiiDK4}R4poKVJmwGR1jKxs5#Qaj}P};V7uI@88u{*933xmI43)8
zv=BIV!=uSRSl;H`KJ|b*7bTU5yX$6m?9si)%YK@eXQb_smlYo(&A_MOz1#`It9+Bt
zGn3GXa5(G<C2a_AgSed^8nk%e@A&cCBlRA!`Mh)AYtgme6QXBIf8PejLj*C?0ZQ9E
zr*1G*7GEL0>S!wq?V)d`qen^r>DLLJWSZs|>FV&5Zz~U^OIfRIdW#&~7&OnrFl&DK
zZcA5TC@v+-+6no(JwI7{3x6eM(?4cApN<|kNP%VGd5oXNq9M$y1))`>m`Lt!<c1VF
z2X9oQ(pGU|Lj+^V?3ejzd;(W6(a3k5QX4n)7kb;CG|p+tLI`+R#{S)m>>7M|8-`H}
z7k`#M>ZUUlovY)u2?2A?MDSEL!93Yx8-mKoTrFZgz|0eh?H-Rp+NIn^CN%&S@;04H
zm=?`|-%gx7U3u~I9lZznkj|52^QNk*Njx0U%Es_lI8K4Hk?L@}sEU=k_ejw77)cr~
zs$vtA!Z(YjvhR;u&8?jjIa>3d1#Ox+P1vycjmJI?gOv&W&_5-BLO;_@V=y>-MK2PX
zBlp3n+{vJth*q?D>0DALw8*OVR7TH*Jy#F)RffFQ;?RT`mVjZaTM>dzHlH>P!b^k4
z_k$DrY3(#-0?Bn<t@Gh^v7;W@=66-4tjt4E!vDp{H1#O5hnvl{p78iwru`#)apw`^
zp{IvXX_Ki<70Q37<5`o6R<SVH6?r<kql8FQhQ_=-jy7VcR0;rclx#v($4|YzNL5c>
zb%n=5>oP9Quz36}Wqe{Wi(TjHp<x%N=6t@<H42Vvxl!E1HZ82^<8!i+q1<dy!Q*8c
z`Fap_6h78K@5Lp*dw2tVzUny2e$d1uxY<wIqV=NJg|CG0B=WHuW(6qxnPEt2A&<6c
zX6l4<DS+E03l{7%3;x(z4=8t$FceemVVl$Ef4?sJ5_u~B{^VY`XmNAZ+%cUviFWz1
z!uEuA1*g91!KPX_rx5lHx3s(32an_ZG2?RyPZWcEV<rJwEsE9G*3O_di`Y11o?_aQ
zzY`<u=Byf7e4~Yde7phlRv&^-)@|RmpRQoB9LWASHPOzyS>B;o3zG+FEJTEcJ78Le
zj!lyeXC*~^f_-gV=I;y@?t`}T$GD{w*M2mQWrcG(lR541M?R)0(_;DEsE%0y9KE>G
z8eB3&?(MoYWd#m2?!%FrTWX4$4>r{lBQ@!yd&?RkYFxA&$M~QvbJ=to2R!pb@XdYR
za!Jzdv>1Qq^trwEnDOH@ZHk()OlPB-QfFb3K~c&^(RPFD<*_(IsT&W(SBj**yW>o2
zW_<FuF$+2LB{v?=bLa`hF}z>(D4&|xf<q53=dgN!<L}x|cGQyHRwHpvCKS34uO;^y
z7@|lZpZE)Il2JJ6ajf^K0$jK-&0Dftp25?ZxXK&@<*m$)hIypVbVk}jBQQLCRol2=
zA<h0soLu{!j>CxEWQUo=ZW}EV_49oXD+kvsWeZ)g_OmxE;~2uG9?b7T$MGTfRY~DB
z`nn;)BBmk_uQ@4saU;sN)K<Zf{7U&*i2WJo92B`@keQ(6&!xR&U1AkgF@86zA)<q<
z;94!H_pf*x=DqC@K#e*FziQ@n)s5o1RTo;)GLs>|l-{I)PH%{K5uxFdUIH2;Q;Xzs
z?h8WL7t{}QB>btV7<iIR8o4*NlRLYE;*;CFhsNzn+T~$g!`pEm+q|~SRR2@0K0)!>
zkh4R82%vW2l~r*gA>B7LIOqA(a%$bSUR++>$jwoct77sM-cwI>)&MG02+GK`kc5g|
zWlSe@r>;qRt!%K>PWuQ{TE6FqskMUYYWO<7HOR>Y4%g*pHYhYT>p-(#+(i&EZ@mbE
z_y_jV*?8U(PHVa{)gq^w?HX}aDStCN^pG<-_Vi@7zf`-noxt`4sTU1*t?`>g<MdMg
zDMQ*h&)Y~qj;}+4W{hk&2OZLimwgpwTiIm^Ps&aze8iZZg2@Q_dR19Xk(Z|O_;95`
zirpk+@v-&{p2sAm+gi-O3-}kSOqU)qjyq<?Uxvqi+|{pF+Vlu2kK{BjnGQ`^_*Rge
z6flq|89Cg3U226iWr+25>`s5Q7YN_pPG#<`0ecYZq#wNM@dLUiHUtxyOf()(^<$X}
z=SE_c9v>l>X;4+F;vF+wh^_5nGbQGEcAkFxcp-$0ZEBC(Mx#qpJZjibWJ{N~viycw
zN!w&F^>54;uUAXB=PK^GLk1lu50%C94PXNsq-3YoOw%${)=FRN?^d#9xaHM*O>8|o
zULDbR-UpYL87%95?zmm2Jl(c0m#^gNYSJF2VVcsaPpGbI5Gc;k7nf~>8*7!gjJ;Li
zWn75bJz{j#4aX+cO&+x$?}>yjoF_45byra1%v<uU*zn<T7SluBxIJzw<LDvftic?a
zGcjq2n{h%1Zz^5$Zl^&Cs^^y}c|DXfF0ur(6v-X@|2Kbs{7qw%=t?!7t`_$*Bsh2i
zbOjX+mA|P#=L_pNb>5-)g2)vQ;D_wqs~7*|UuYP~v=G%ptvI&hdr=`-nm8|NZOr|W
zX&^Z+$2#G{*LEs*DyUsNSPsSq41!S{?7U14*Lr1Qp>`pWDLeiC&fw8<bj7M0*TtO+
zmmrG!f`jat4`LTpzZb{&e`HL@+b3UUE`u!U`nc5;om#jc;i|WQId)?=O}#HE(&FkE
zm@c>YqguDhq0XTz<lmItP|foY;>&`ng=>xel1(3$EKNeG4dT5bxu4|ths`5R`O@ds
zSG;OnOo}<39inEY*=}YwtZn|_caEPe#%?R9bqfd|_YGEtc2{zAxN!V_B#UJ086OQG
zR|+&Ap^ugwsv=8VcQ0IYu0^#xRE6%d(t9F2{f4VZMB&KedOLW_W=ZK?W74)=%B;pc
z)H1tHhK(IAJE%XjW^~5)i?d4X8jn3bFuqnvlJX;m0f$$y%I$of{5VXYaCO9Wm+cuS
zCY6`kfbJaGc2nfdYDUw_WEK8iS#11K?u~*8doQLdQcOo)<|RU+wv-+&8jG};2SeM=
z=&-lUmX0^$#7Jf~Rb?2ePQUXx&*sX}N+?&)pgsnYg8~bhi@S8bLV3$*wR?ldl3Dx=
z)exzwMq=^g({#^=NwIbjb+O&$stC6>meE<17uOm8)g6(8N{S<HmB|1u%Uz3#z!^(5
z2LLe@cepapH4ZPp!7jSL#*{uM6g!Z|VN}lj>A!5QZ+A{roMYCyr@b;>RCS1+Cr=@r
zuyYy!Y|LRVpeA~~)Xal2C)#Q!KVLvl1)s$IA)&5MpZ-|Q8T)bXVdMgpE=*DTPL{_Z
z3z0~1Xc>wA{z)$3y*7tR#kCy!t(rEuYYN1DtCNpaUC!W()MNE^OI@hO*+PcQ^H7@p
z^p+J{Djzd+2A++d42JQ5WfT}84)*%F<w^EYon7liP)7TT1&UU@4M<x}P~kdbT({rz
zZf0DFbyLL-Vjh%TJH`3@t=8&6!$IwTm5Z5(T;krdDuM-bB|Vhh;rN3bt83%0LRXKp
zmwsoXCno2}fNw%*$GB)o=O35sqkA~7x`x=4mc|X*j^Wn8dZbfEB(;UdFEyJg@3jab
z(JQzdWsY&kfk`cP^EjBKQO#{`b$XCAKTS56#v+z?8LMT}M5oR7AxW!H0N)Is58q*n
zkUZ=S!BZrDsnqEWfex4!CU`ds2cmh-s+oKZF?FuW9P;FOE?xm6Py~TvcX7>3)&pUo
z?U)sj{E)&RPh6|$=uDy&$r~H;9taotZerPso~$wUPPbn%)kCSRfXX+vWa&;EHU{r1
zyhA>&X{^Pwu3yGoD${*Ud@(Jg*eTRXIi_Y|^?3BLkF?KDLRk!t805B@We#9Zp}wbu
zHmrsVT$AQ2(jYK<&@LHwF5t1Z&C03i?Yx-o4L`mEtb&DR6&(AV#QWq`yW-31Bid@+
z@5cQ2X<QjYS&T!u4?$kHRGzxsS3rh+6G>o5IDROM7UB0Ns7je4EFilyO^Q93X7@;W
zKE}UdCR?GR7918}ndGJO1K4KlS9@*9Eic=o)Z43o<Vz4%-L(@DAjb|vD+|GTEa@g~
z?Kq*W(WhnW|8b(aeu_2PGgP>^hh6H7Mx<(xr<G)()l3nnpgDxWa5}O2x&ubMQG4(9
z)l+i2cMm}q@DYsm7cKKUC9GqG-KJPe<Pt6osoTZKJ-y6!<$MXALoUL^Oj=#E%MdGe
zoSsk1@~fp<;vTlD%_U#G7wl5}g>E=w4T3+Pvz&E?M0)AvRb8zY7un?S9nQJ~K0YCG
zFiyM%5G;+iKoEAp&+%g}XqKHOSPT2(GiGn*BR3(CRISz!Sg-lp)ivzF5nlIw0>m>u
z@bk9u9`uthTL?fwup27rovn#@wjfTXo|y6bZLO&7sbJZ9#IY_2*i0dfLhYHU@sj~^
za!QU>+Q!j#rI<KJku;}D05pEU(QT=DcLd~s{o6dWT8<Y73-4Smaw-<RecAraWECJb
z6JVps!U?=m$&1Bp?L0nP%5bVqoO|3YWA}i)oWVfoEkSzCzNGBs?ncR-sn?vwNQ*P;
zM!~51Na)3E$MA996lvp|m9Sinz#=Wi#&(TNN7qsD{_%}Q=CpK$PG_G!iWr{UZ2HT)
z_FEFHUhNtXs=@B!>g2^~9Azz`r>rdU_IQ$Ka)(?<TP?5S0m1b4NoB$p#U&kQDa`|h
z>Yc>tO@{D!<nWis2D2jK!eQ>;u}~MA?PSWoKVw_EYd4B|)w~rnG6GfaEZs+HQ`Ynn
z$eQ4A$iUV_Sn0=dVby52#$KK{WiwRJQ7^EVErxUVT1hi?8*64s#}hXl-><F_zPFzM
zHR#<ZnPN!tn4mfMd5)_9%_zF$7pJ9%YCHs%UNJ8#J88$w!g70ue4jz_zYO*H6we2#
z(&h-%?svz)msGHxic^gQ!CPs@IlqoFRPkr};jJ}zc79@6gy7bYI|z<FfG27Q(nw1d
zMy8!w2TM-I#T5Ww2DT?nXZM(fx$YMCIQI?L@JDXlz1(seno@K_2^cK6G1m6RdZ@fY
zRKhk(RMEJF_xMD1hvEYVG8wM<&*^<*S`^>d&ahg4`BxUesqq!9Sts*~Ev#>?xXiev
zYjOV)$f!+NnhC^{$0xpl)ShWjC41?(_8-3+>a(ZS_XAM42RSVn?|QgmXA-Q$LV;_&
zZZ{z3N_li-%&O^M69GUEpWTpGV8J&2U}H@L5>$}wV&`;$r?QcQXmSJJthDshxJz_7
zdgn)e3|l&qU{%^YsJ90v=pBS4=ZrREO2;FAuPSg)Q4htNwec1XZQ*W~;TOzi=XoTU
z!qmOE#tw9?p~Y!?W==_CaZ9N!SH7DDw6v{mF3_%;jb=f3g`jPP-UG(-gvUG<J=VRp
z6?vw57~qC=Y>7F>CB#4yYook0bOE-Ak+YLf9ZP!3@_v?cj5oGU=ik8S{~)BuCto5t
ze=Cptp7!g(KxI?%g~pc`8buw9MD;$F+Zm6VOOO8fi=8#kTEM!t61=5>JM>yt4v~@_
z5lSGu#h4D)Q@p@!*S-ObzrnJL4kcG=HvOiWhQ;}cPs1S4SOaMV4VW-7YoG|SA;+s|
z#8FApLYH!8(P?P;y~&8YT5#{iBHY6hKBS`eY*wCb^VdF{Ty-S$GVjDJ>!;r$cY}8G
zZSo>_2HWpCKth9YCda0lEkIhvPue3oWW2cI8Uo8>v=HBxOx7tP-s=m$PnJ*PtH<=>
zn>|>YDyrPd;N3L1W>W>A(2uxt{x)K>DJ4&_(f)D$?3U++68lO;#^1_MzwOv>2lVSI
za$c^%s7}5E0YgldqMT`P*cTFu)5F1|BE`NjJ;*IXyfPM7&739ol~)(uY7z@0m2wf2
zc2exFh_nXVhhT#LJS9DEx{yD0KjiR|(&vJ*KbHLEg-@{NsFe3ncLM*~sW2Km1$}LJ
z-zTJ*w+3;GVpW(R4EY>k1Qoqx*w}3~*qm19_R8b1qt7Un)i%;}51DNpz++5(ymEY4
zHOYtjobXz+n0J50<Jbjf6Zbm<rsz&4fF5$?oJOsm3w6%Uz+vqO|GPg@-W!mf1ZQ9S
z2J#!GEy}WqNA|8bD-+gnnH;aLUuAnItC_S>T+pW@u=QCiM^a64bQNl0!PYaa#fv^<
znU%lh%j+hX6q5{MUe_%!3lj4+U#&Sa4afWw>{RO-MoEptkCVL*sC_L{O0Vg4e0<2F
zjVRIjZm$A|R0T;N$0E#rhO7ihTh7Ss9+RuW5nOMcrFYaiXAEfqfX2-HW}*D>_*d3(
zs#9davRTRWdfRKc4t0yYiMe0~*=5(prK}D7@S1%+tW^_Tm{I|8-M-@7h8MXGFM_m9
zg4TlFjEAEExs-7Yes2nx;;%c1xRhomT_W(}I%1cTdeTbE@L!I-OBB!FzivEKP?0Oh
z`R8*}3O(V)6~fm<!tj@84KV!g_)P0>VK(63a%mP=E~d53u?n=+i7C}k%c~>PG_Vkm
zaRhgwO;K!-<ws^X8g3gC0_CoYJ38A^t_K>`@D?2D^*449>$krKB{`<ll~DoD&q<8&
zLQ|f3l8IBe6X=!7UR;BCSR-mh%@wg|-y|P2=#Zy|r^VJZ*AAT8;aK9gyrp%W+Szmf
zx{Jw<vhP2?zn<)nmMVTKc=@7N{L5_1I4efW$PpnlLif8eE`fFU*Mj5EYEQa-N<Ws%
zufGz4G(PQ{ai9&nU*1`Ww<Fa9{&Z}rBoo|l#z_hIY1DhY%C6bw7r`p1k>J<yXC-%L
zk*ga7?t)hVBXj^FIy<MT8ka@{t}2UeBLRfmBNHh~C0W~o_#56aA*mP*#zN4+y>33r
zSQyS6w{slh1xR%w{U&f9b7);OWq54DKNlQ1q5k^OLwCwfU8+~N##C_mpAW9Xl>$Hq
zSM-EnC;eZZWrr380#XjzC7>sf=!~EhsTt@Cc{f9YxB>j9q}y<?SjoJMg~}-Mu9kcK
zEd=6aFABdHO|DsxAjQ5RaIE#JGC8ghKlWNm7Q$&ig*MSc#C8By2WdcXxq2cJAFc#8
zM5%uYw!a`Glo{y7b-Z(YG$xbWvCxgrk{+;d&G}6@q(|fUAFN4#bq~3WeKQdZkuj->
z=b|4Mb#`xHK(c4eh~IqGe$;8^QkI-(D3=xhMsyp=RQe?~5$RhqAFF=8BH$>Z^3g$t
zt+z)mFavTF>7w*7Jt>Nlpo*v(^38s5;wK~sL`z;nrqMN!2E;f;nV_S&EFjC+ut>F>
z6W^>$WMPUFdwE}Z1Gy?ZtK_4Y@B>+CS)nZ@u1NF99k4OTj#$jDrFX(~bqttqK<Q!n
z1#<sBeU!7X??*x7TmK30hn}_LCPha8YgYi5$Bo6^=zo5Z*9FG(K(7a6g3Yh~@&FT!
zpF+57IC=0}0+c`*8V=p?4J4Bmo`jBYRyMs1f!A3tkGP%pa)Oue&(@q-oqj8KFB@`w
zLPosX_LUq*zSTt3{4#Tb`->Q$VF&1350`_3N9F6;Lb~~eyOFXGX!uripH0+9tt5CH
za9ghweK*bVZiCMUxLxsGs~r+;Lz+RmtBmYl&#NXQR14IA;wJS0Sf^y;iX@ug4iL6L
z8Ye687N%LLA@AjDs#1Cwu)JI$hT9z=45);KBYoy*XK9FssN(cAVcb*JEY&QH9pmW`
zRX=vHik72_P>O27N8t$r_+~z~cc7)*1w9P&P@xxvFd$|Y?e1$w_S5i$H(l-PdU9n9
z|HNhfxe7<zJ9RZliZZFs{d;RSA04v16+EZ#&73WV_*U|x|FXC*WY1#3WM0D+Pv;Nv
zb>(Z$XBaIIQm4?MvgRFY(Qs~Ud0h)>%nWP;tm48XbmrRj`(B5STozMmA_j~iGcZmA
zhG8H3)bu|(!Zq?A9UzC*5gPL5r8tJmd|#;~uY{@o5prn6rdnPJK1h)~rkW$t_Mz5x
zN_92cT0$O!tB4<nlmg7Cw{5Aw$^Afl;r!7wl;l8_3g?yCrfo!1&sm`ZqPa_1KoYXr
z9jzX5KdpWlwAby2(nB%pUGt$pLJ=L4qwP~0?_vrEt>gOQL98s|=6xh&4;fKw&_c`g
zx#6$0BL<Rd&he=RKxG|KB!v^0;j3%eVz?&3<zJ7m^cIRN=~9-HuM#AM|8g+=dDQB-
zO!3?&`r88e_N%X`e?EpUg|L59FsRYurazxP)#<^Way>}8i?NR2V9Cpy3$JjX7B7yE
zAprPr1J3RG@mC9(21RA~W}z@CJ3`;$FJ{T`M{V|D?^zyxRIvR=(N4XY3^P;G2K|t(
zLz}qjFlB`~^he8(5N99UChLw&-c#kkB0K2hA8vs_0Qjvtw^h~he$A(;mS;*7NvUFK
z+G5M~ppaXNAUzQ05vV>DjETv2RZD5`%@ptiJ_v%VlX*M6r((l(T<8c)C=1ymZR`*3
zKCB>cg993r$pV(lI|Z->+vO|b0J|&$_K%k6jHCK%_&aX4U8gD9|MJdXKZOePOzMCC
z_W0#>^Fj50cAL${(A7B6@;&=sN5${>g1K)&p`UzM)@CJ-TrnHNZ^rmJ<(4!8uT{{$
z(a~WZa32-?2*}Zy69787rRuWMnz5#*<THdoM6{r7(PgabMrEKwpofnCgE~GWD>i8Z
zjz5+*I~_d?9C@m1&OSRC>ajDVZeH;PHSS1Z3ooqNtz#dYVQG!s>VH=gF|mV9U)t{3
zg?fDZ2*wqTrCfS*VChLCRb0#oZfA=kVF?E4QnV}MtA5C8nlB}e>nj8kHqL?vJpnJh
zdf(wUEDQRmyh;Rt8kP42oYtrEYb_{u5Ej)|8-5#gqi{oxqA*+l58Gu*%d(}yHE^tR
z`e)s_ulKRN-ASDTCH&{N`_G?8+&GnY=)w?m?$kecoI7a*x4gJIp>0v;tOe0K2^~2}
zPkH$nygzY09h9eod><uD#SivqyFr7Hu;RAjg6<9~+m-}Q!g$^lWaRJIz`ogIQb!w%
zP+!fA)V*J45auTVX__Ciy;|yWuBky7Ct#|+W->4hzYn_U(_<3yK@duGJmj8oJ@yG^
z4zGxJ;)W~S7)v???M4wSQT};wz8=^nI}s@?cS>zBFqIn6E%Se1DenjDKLLdC!1blj
zY}o7rqWTaSS1|iD92W;~pdaDK#v?pCTEW{IzV$uG0w<cpXAdGMN;pHVmA%Tba~H@|
zq`~ZQs@V&?{44Rv!be)<KCjuZssqFMOz)kz2ivuRdZYO_%Co$7iXOj3cv7EB9L6YT
zhW9}_WgN-LsPhXDaI7$x_&l=&_n1nzJy6P8bQ6U42AMYR=|bNHz49ttse5~Y;?l=!
ze7Q?@xRXt95JkjQ{uj?jZ*!ptW*QNw4W0e{Y<LS;O_$v3NL3Bm6kFavcg6GAN7j}c
z<(&84eSc_70vTLt9Nxp6ZE^XCn``KI4)T9;e+a$WP5;$zAL26O6K5Ii1|dm;qskuz
zY-r+7R=bkywgf+ZW;+dRV`4)^YdgDoOX5da=OQ2X_2Aft<63`Uz1-4G#jBMgplS$O
zld^tt8QV8|gXxxUO_j!5Xb9;Cfh1HU7h1~ij;HnGy=fN__$MOpf40k}uxIH=I3wSS
zg=had|0;M-Ij66w*U#|%rJoJ(T&MU9{xJ?0hVvEQFI?k=0lHpF@6ou#5*2l=AR?xi
zJ#OEc2OjO6&=a<U4>K@x`xI(Q5nb)o$AGb)`dorDF>rmcOC8JbeCgd3)H=T&z;+H_
zut=gdW2@NgVxuyqn~`;B%LA9bQ7O`gE;S&RJSVm7fH$O|U9rhM42ZhTB!wnt=4b|{
z)<+{N&VC0oBFE~z!rnWB2$JSN9uyyxidBE#fQ);f4WVfpv2gSl0sYT><x;i>@GKU~
zg+L=86@yQnaKN5UxVJU9PanxmQdqltOPtnvmWt8rTLI8$E;?KdivCVw{(49UZ~#IM
zL#ijZUEtUG?L2D74wkRf?5U;QR{+1MFGA^?=~!m$>z6bOw_8p8uk-F-tIHuLcJ@o}
zOPq9<j%#P}wtZU*GH=_Ty^J*&Y9yrmUrRo50qIP_$<>!Z5`rG15e6w<%7!4!F{B<<
zYG$D9uyTe7s5Ow*cB;tb0gtCrj+vNjbs2@T4H<c2pwCRz)HC2M><fPO61=!|QkIm~
zT^pvZi#ZMT(?_zAs>Ojos;ap)O@cO3Jy5$#AJvrWbx|ry3>u>Lva+NtfRejNl*A=&
zw5Zo2=c|Tfwv03Ihp4E{VTCp4(lqVT+~`Efcf47<21u&_eC_Bipvux*P=jn!*X7hU
zFk>yySO~epH&sB@HYT2nO4VvHygo;Gi|7oEQ!QXA6!4lX+l^SPxzh>R8d@BImgRl$
zsO#0jrHR9D4I7Zjetko&W>0C3I~0m|>((1QC-Emz{#@ZL2sp%e7KWbSNo4F?mc_n-
zlb90(2Njc6!inKb>1%uZtmO4gUVmwf{yjp75*z;JO8h<9-P2rmO8$?!^@V{`Zuw?n
zI?W@3Ne~O5J;pA_HvhTmzr4_TiQ)>Mjlp=;13G`W2tmFa06oLeI3zDUwVx5%*2_D$
z3wjjjjvx5*sgq}8H;$vA7QnSs58oTFy#tElCXx7W>J1l{-^Fr4U$H0<CP9Whn3MHr
zHjQya)BsBw!=sTFr6w>Yu;bTl;Dp)%%*p@EY<Y;`X3cWE9pw;-E>urifDj)Rd%}hc
zAm>WRoOc?p&a12C!FYfpTYKZS3wP{H!tTgPEllTbaDoX=GSRfJFL(oDz{k~5ecZyE
zI;u@aUK<RSjV$Ghb2Sm9Drz?XuJ`5iU0(MygWI{$6+>_fvtY>+d^4j;U8Nv~jn$yP
zA{74?ZF#;m`Vz;|eweFNn;Z6zFY?4SzpRPCnG(-<Z~yxX;om%Qm}(GE#W)Ggfmi7=
zTtcuN-mMDAxK9-ih9$iOLX%l^WNCAIU@LcUN?`9m((LdgVckPS?T(#TBlg&>ZJplm
zA!26D?a)W<ZEV5y6fNelaset*PoNJyZnYDH1WVY(=Ei_6G`Yx@0t7thobc#%G~93{
z2ynCV;#G6dNC}d!4=*S~=g*ht{D>vrWR}&aKU+_^K%Z_#f5vGYaM*qGnZ3w8JH`ji
zl<UqBOv0HRc^aeHpzkm$-Bi-UAYuSWG}#J2Q{&V?azqqsRfD&N96zm-%-iy6oEwMn
z{dKl%|GnL$rI-!;sIqC8D*F6iN5JeTw3~A}mU}qXzcgGG0;iy`G?o|MOS_H(toz%1
zS!epz8^WSzdKeh|VogfBp9oA3W)xsG!du?r;hQlFj9nej#~{<&fCS?D-QQV>9jXVH
z%zK0kh2aN4xsn$0s+l%ZFD@XYJ&dwQ;*w(_2<=|)7JxBn{rNxBklnTYn&#koFV}~3
zNj(fy?`t!MtoTEF&|^-ZxCavF?QiEBbNRy_Lv(*)+cz%1d^2j*b>vkQd}F`&nwZ{b
zuZkw?sv*(%ic?8^9;vF<UBuPK*>{m6yZR&nkk7I(bsz0iwGWS#<H0^fq)kXr0&eZH
zxi1YoC)X-Q`?V<F|MD8Yys-J`tg}IavRexCmBB~<>^}kG2V}NEJ%$fCX#eL!)XV@G
zI-UAHCp{4W4N2NBix;p-_J+oeLg&dOs3F)a+#cFf<ETD;88!EWkKD~#I{sXo?M?8O
z!p#Nu?up~9;ASJELV3qA<gJY;`G4$PWk6MH*1mQipn`y)pduv*0@A4{2uMk{bc=L@
zSO^Fx9n#$(9S4+@?vRv@LrBMg!?(6GH)iHK-^~2Jf9~~q(X-Fq`+eVMt@VUy=q<1!
z8JhAQbus_|{d57+xtlh`BS@OPhi^D!!42H${sK+6H$;v`jbup=!_H?HjX25|X;CeH
zV3)EZ?tK{-1`w4y;4#!QZ1A7YL)!Q;3|7O??j35F57UDR-4Eohb3;?e;vFiS^Q;Po
zt~HBYG>_CC)ES5$0*i+2S3R(Zy2+&Vp(;FVnOIr4j`ioub*bT#>A`DB@-u(K2A?t*
zbAFJSjuEi2UbiFi_4lqZ));yIN8~Uoaaa_ePR8<(v3wb?X@AbFFV9kneHcKk0XQWe
z%}6==zd`yI-A2RAtKiAy2_4;%ZV9u$Q?)|4Z<<d6kR~z%(qU>IqT-k7L_b`IeabGh
zqM4=X9vmMUgm5r%m_j?I1MG<2af?G)?nA<S&1PV>BR(GK1KX&qd`ZZmL9nHlNPz&t
zD~X?czP8!dvUeR&;u1`iy44YPf;hMYE*ljpN(7F~LO;qU7~wR0t=>9~oauq<uI*Y;
zlloUWU>HS8#dpt|M00PvY;^z0Iy6J$?d7J|;G2~~i6xt?I53@x!vUXLA2SQ&niPBd
zfakAh=UlFZ1EQ>LdU+86({uN)TZL7)V1$ufYI@z624NJKVU!mwdzz!y0|_&8>qvVT
zI@!(PD^b-Fq&89Uh20aCc|4zQNqOQ7$=PARkusg>VoMoFp%Ry~+=F+pCU79zWjW>o
zj?YC>!$XKM-esF^nba=h%NGZ((=j%x?SXryDW>c_mr&TLG4m!w-NMrc*?&YZ;H~uX
zuWFwK)avX19KgQ5R~6>L7snAqA$AYyX_aXP1n2|SN|r>j)8b)9#$=<SDEzvOZEQ_~
zsv3F(oxMp&!<H&9xix^W_a5}36zB6DeR-j25f9?Xc-^3}W!sU_pqx)InWaFeUN@B4
z-=2)7zpHA13o#wpb83w$(JOy^jeg~(-Ij#<aKbb*>>rtGKLDyb5mBgF;nz4=CiE&u
z>zBLn=~ahfl2WVBf+JOY$d0X_1)F-gx=Gm=XBb0DM$)J~2)Btfqy4h;wat}?%C3Y-
zc@rF&eZWTWoBztIb6R1KSD_Oo3_gQW`JH0)1sBF&&P*MD#WwY_B5VYfXU;3>D*@{d
z1Ik!oEjkiD8CiinN<^F?_uk;Esj1T^8KIG52#{1#-JG^!0N5kk4{?Yq{GeE<F0S%^
z8oNFfqN!o8yt-X4fXz)?E7m>6cUv3!&~E`Ay2+PsNT2=q`8b&zubWmK(nfN8;t5fS
zvi;6N<jAx<+{^vPaidJjo^E?`xePIV%5S`pE>~t)yEY18*<tr19q3Or!2+^P&ASbB
z?rcHEg?pWOYSu#*^@&STRFu^isqKiMjiM}n8{HOGWN3P=8r*JZO@dRDVc7k-6JXxg
zLbos;&9Rr#YJ~t46gce`^NuYeIZV38kVbv}Y@B!S+w7S)<!uxjL%ufWetlTKl?`XF
zwA;9cmrhHvn{XZf>9Ka@NS^g^Mi}H3cV4ACnL2QTA%IN{3=ieAvpbh<0&oiR*i}PT
zaBk*H&sWgcz07Xf?Nabxw+$$5dF;Qg6j1nJuy}HOta-|dFM!;HrSJVZw_FFeS6`oO
z8?3bEi3g52^Q#sNaCodtHUzL1$uzx$!n_WW51NLG)NQ{e2KVSQ1`t~6K1oj=$n0FC
zMuq5-E6$t}HK$>_YygBB2l>ky9%tlAD^V^L2{}pCxpuDa36Ck5y(cuZ)sPt0q~U9m
zLutll`fFITY)yd%D9tF+;o<`?eU>6(YEz(hs*mV*dMhlk8%Kt&E6Cc8BBazFpf)+M
z7f6Z&L>|(&KX~96Wb(z<WOy`-F$6$9xjBmZdaeFVy<T~%rn0)bKO4$<vc_117h5=Y
z%vTZqV^Q!2>}5E9#uWMq7c}ikfVd$)9Gs`3C9$2g3Cac4+N4N>53n)K$Sh`G=#|@4
z6^w_+Pex<6>?Z(Nd9!eCJ^&UM_14f2#uidXdzBy@Mzy|-Q3b-@=5>UmXoW&pcm)=$
z#JvwgZ(Uup7ux*DB8%QeDoiUUQ6>wEPBO8OA}+)B7*4_qy;jA*)g4dhEBlD)Wq8Gw
zU9oFi4h=o|{I-HZJ*k_?a0p`Lr3%~^jgS1~OSw}wKC3f~tE0tGmAVA|gk%Uf(lF{w
zGr;@{7fw3ENO=Vu)zh(9a|MPBWjI*I!Nf}51$PYzSBM54ZCKNft}@m!Bg{b(Awt>X
z;|%V=KEN`V?tcozy7zfu5Y}vavt02;M%dSisndL?mk`o{a^G|F8If0PtSBLiM<Y3~
zH7YI+WdMrqOObB&2l~>HM>EbBDqHDmE<wAvr?3{y=vo0wst)j{TW_p98x~qb4U1ao
z_GQR)&`T1`f<KiK(q5oYWaU#tfMHtLKy8H~@z3rCdE+nJ<j=DQiFY?OU;nHhtQ#}K
za(0PV^NIa>hlt58|5LaPr<6wk)*;ib>`Nw?^{0nyiu?D#SJ#Z9ZwGKgW9$1R7&$Fo
z8kht%EqX}q0*V}1@8!r|W3534H(@h2<Sl5yi!xTAij#eBHZV$#+y{gasD^%b0FCkd
z8)$uoj4*f=#*w(tEhQm$VHFMI%Y86<+=j_R%g{6+0)@WC7GQs@Ko&M@k>Zmoq}D@F
z%nGD{%B;9<+|z0S^6n+6NNzTAm>^&B@-Q)r9{KnKy(pp*TNaXk$_q4neEb{x*`le+
zr}{H{|1>rIt<(tZ`5}j~?CYvSQbHw0FKhy0*Y>_i<S`;h%Epl2epio=2vj>r(##Lx
zHs|_w0Q<}5iRUPVYhsnIOg@Eb%qXwGr-?p;3kaI5Avoyl>k|&g6cCEVeRPU0k57~V
zyo7D}_Rwx4v%G|GyDklU^lwH%xAyWt`FTG_qF!fsHY)&22G4qk`6iODO>VTy8Scdj
zdGtEJg|3*E78{ixctRP?jWBT+Nzr0MfoJV0wDRU%D$;+#R#RZlBC2w;0|h9N9v&>w
zW|Y7-%~C|+yTVbqBQfZTWLB>r$f^>iShXI3yUxH<Zc;ZeD6`Cl=*O?!i-f{R_snKk
zL(A>%;LF2dVmK^!@lC-5->4k=eDWv(hv-AT;9`*0r^xxbygny-Y}tH-H}iI8r!B)|
zT5vt^!kz@Lm?oNX5*^h4OsvBM9m}$gfL}Y7|4czY3p{-U9+_n13U^n!9i%`ocn1>J
zPIZh(#**1?Z83u{P5H`P!&O9ypARGo;Zf{nQGCKL^kQB~aWIp<-%#5o)9EC%4n3vb
za#x)5v!~nvV=)cD@t8iajwlxrl(N(GN=>GpN>bPQ%9}CLg?mFb`z3IlC`p@I)d#gM
z06&FbZgKlI63(ZO0}1aWNM_f6L0S}Ij*8y?QF(Dd5-HfnaZ5)AGlg3j2?i;&bISCH
zk4Ah#QMjXNC-?8P`ac|tf-&xgPn(dwG3og;u=DpH%*zt&lobzLe;|87F#SUB2QUNb
z0Be=Cv@?w@mwH3iKLl_O0%L-B5nh+nuD93!<sSa+JT`()>Xds+`IG<sZ!Q4-m!Shp
zt#6Lj;QbJ8{P}nJC-uM|zvdMQfNkMk7XSW3{scGtxY(OaF*!+75&ehG|LiCJ?cbYl
ziZ(-3mf81DAmcyC9sYR1zkQdOsbhq_`OXA9f`3#-{OOPW_GkX^3-{Y_-;8e5{TU|u
z{~tttIrKt;&9zTF_`_fM|6lX3Km3jwn8Omaod4UE2NT0Yd{tuMmp@Jm{{L(K_11JA
zCG1T|B@y}cV*c|#32#^XD!i;J6`q%`{O8~H4<8^H4)^W$g^B<AJ^gX9DjM*zHgV)0
z`PoYPw{P3o2Lv!mOwzvezg>9~4sC|f4z=yGfBzvrTL^zG^8e+w|Fy{fTI9d(TmRXf
z@z)~%&+gMd?Nfi9<bR#y{}l}T>m>gJtohIH++T40FS!2S0m#4L`u|cT3H}Aw|EkIV
zs~!5Un*6^2+7h(w&>S`d)q}5sicum=4eoS6>pmVKt3ooA$~t%@-eW?oKUxR9x(K*%
zDHANA)UW;hf!vNpm?S0yPWS$c9>naG?~%*SG~A$=i32T0X=Z3rb&Vv`-S;iPZl_R(
ztl19mZ5c2!23EqCYu@+J8F`ZPcwhuv-2r4L+2o7Lk)NPdea<YmmM9fxe}dxR4Prsi
zZG_UB1o{g3FpQodUts@{odfXdJD|NC#+^w|NvvWRJImy@Bd&T#ZNTxa`w-BPLfBK&
zED)Ypyt>^mn^oAsw%@^<yF3c42vMj5A%zlug_e6f;${r_=}OkX<TYsJfYHP((7&2K
zXz-3f2i99bSubc@X0c5O^U~HhX$<C4USNRlSGv@R82bR>PcjVc48FB-5c$et<v8#X
zr2Y3m8uJ9R1ugFTU=k}hSc!}T&1@%5Hg_Jjuw2J7|JE=6uYciA-6ROAqzUl#Wx~Q8
z8CQ9YP(fFKAu>cixB*wz!aE5x`C>(KSyqj_GT@Z!ar<57DeU8cB)_DvdSdG9eEe*a
zBmvjn4=v)6^A<A>qChHYRV3R~@)71*dRc2={^N^-!B%4X4jMl&Jw)ue>ACB`o2>@n
zv`?2-VKnn1d5~%8G`5#}45s1De60yzGJECil%9Z9(h6eo&IFUfl0g%8D@^=!{O60m
z3_X5q&MC1OeP(g-!{5nDf0(WPC--3%_af3i=GCftiAQ{|Z}pnmTNF?j)=SYB1IVwk
z;M>9E0JUTv&SoW=O&QjT*u^-ZtkQ|WKtV~bP&4#<vcP`oJbO8~aIyAXJS}!wnGuFU
zz|66SkYS_f${e+sdXO&Mv~tn{D=>4ceqVY{(zD3*&>w6my}@KFCx=B=Id6Tz>O#Y6
z#{k*(pOdry&6F1Y7wYzr*pM(J5BKT5RR{%=FHk3bvp_Dp5qRHAC3TA6r?8YuO_=ra
zaBzkeu)<cr1x2B!M8IOaCZnge{*u*0Olc!%E`s*z-S$K<d&+&LGZ$80+CcN`p_W8W
zB(sh%uk-fzk7j{Awsm%GnA<Z?ES7?OIKMj1ov@;bT@(7rAoM2^h~G%kr2<F5!w9#J
zx6h^+hF$3upDJhu!SZTrQW7p1u^pHN+oV8@MlM2=cfPr0SFsj3WP{K!7SU-5Inx<=
zt;W#;i~H9J_g_ZIt^yTcw4Asb{AHelV9|}-97rf7i-EnibvA8IaM4yvg}7W_3HyGh
z68*+0`VW607<Og>*61sABpO=`<iZV7w^wY#RzBbjF`}siyh+Sth2mRMwzTmzERfHe
z)%svO(xb(P=!Bw*R!<*6^c>BL7pQ47cBWA=GOyS^?*Q$h%?u1&Zf81;q;qlfIuyDc
zp}{}?-yk6OC?b0>9EnR%d^0p$3<H&TMDNl8l-hHMHoOCbo)^G~y?jw~<T(;)8?50t
zSE#o-ZSvX(VQV|C&vvVGE#y?kjqXizz|<l`@(K`f>U_vTO51TchR!a6l|gEkW78LS
z9Jdj#I2JoQ;9vte#ix0$KvGFBNIniYy6jdf2p1oToo7i_WYvOfYZ=a>fDz3{#AUGR
z<6*$Xj`%o@xlpCIAie^I2m4#*nh5D93kbuTeacb@K@?&7NJJttHxUpEIM3%-eF485
zmT;hS;V?yvGjlS)A5F!qHfds6Y@A7U5HdQlJY|!S=~)M2{nu@zr3)mTn|aMFsa<Wb
zC(i;SNC%UDj7aH<%9E0*IM{yfK~RgsI?U#BTvU1k5ax?aXP$De?!+nR^v{1}2^tkW
zO!B|)t%ml;+4~>AxJ!c@<_A<-$sssAyIr`JiYHh?=pQ~f{q#|YSs|J#BDX#qJlUBv
zz*}J)A=x43Lr$-BN^E;{B$k(Rsq-x!=_+#8ijyP9nmq^~nIihWSFnIm!G8*evJj+H
zc<2;x3d>mlOP)@xlJSSg*y~k~0YAHrA6spG`_2w94MQCmV5zm_&z48EX0JJBh<f2#
z^_9W?REsc_0!lQs?ml`DQs&!=nJ6HAq0sn@+{C~4_J4s6j)vd)7U#|ACG2}Ctb{(6
z;|0R0w{i@93R5=`5poCJX<lX2yO%4m91*QTmNKsSh_@SXaWrN`sj`L4d5o$-6*J)2
z6$YbIj|Ne9v3z)q?MM{ub325|?vEZ|w{Vzvco8{ehuguHn;l8W<~6#zUdW(4U$i5X
zS%<}Gn+wj*0Vg*lu(XJWguQ$-0U-BwY}&kvdXDImW<h%42}T(eg;CW0CS8Nc27Dl|
z)rkXoRfVpl(k2l4eaS)X<Zl!yq6SpF4&ddDI9RK;(Id*vb#NHTtj;%gL95ElUc7Eb
z0wYd+d@3z0Ow0Mqa`7hMwF;hfKS4fwD@$3U`7}hjqu-?`<S+*CeGmsxk^S`gt*h&S
zK{j}ZQKt;poy5a-k>PSVXj91wK0rj(tUw-P`HBWKi%$nJVvp8kNrHC!-}3b)EX`Jm
zAnsz1zW8%DL4*RqA}~8syBd9((Ye#$*5ud9Iq`N#esV+d0+BP|vjNAF26a!*qbf>*
zdSRy>f5K1r56(-&=Vy2Ucf?LSYt{m-1jC>tNCmtMnnT~0__mweu~av0s7!bOq`4OM
z!3<q&zKw3ym9QK@Vu2Zybo~vg2~AzS0GIsDh&FlXBpcKY5Zzwk6tFsy)tbgHAkuP+
zNA>|Fs482xt+LSzOuZE8w06IG0gUYcy+PG>CnpM`aqN_`wre5QMEavIC_c*VgG&V{
z*o@^G*m3FyyZm4mTUr6erwLGx2x}+FXP8`9yy3heG|<hO08^hE$`zgPM#p{VCXezz
z8~{78RJ%?K$8w+gTyD}+^4GS_q|dFW_{c8wPUWsI(*|gmb*9@@Oya7L6#Pw9{~w+Y
zfBZ2)^JYQ7mg5P;ff*)I6Y$=ZjiafBv>uG)0L}=^0Q<#<x!*$8InUAX=`~+c{+uVB
zpWi?o5!g>O1I9{Q6#^|cz&lh&bopTz9SRT{SThGx099pr6_^ZC4XZ(*Xb+-BB?dCM
zAwrfX+4cy~Xbp2jOrF9nbOrNj!dompkh;C!w6^4OFg~0)GF-|`qayaI=u=b79~iZN
zyzKw?*Pjr%p5=S1l{bi1bsT+5NQ&N}8Z>m89~psVQj6GxZv&GqiYv~`0MVoKLX1n`
zHc_hu9F<O4ZDOz3h&AlB^dBLa3!K?@QMVQW#EYq85uo_PH*7AWDm<S&2$8|zHMOx1
zny$djQ~ij_*SpZ`Y}MKwpQB&%<{Z^@Eu{iLNW4Wk`_a{Y)Us;1Q%#u&`}|AU$Nwvo
z0yK61BH~-<oo&wOPyw&C+(H4OR%OuS)rQ@QQ8@%0PGYn8kj+@!8HZR5oJ>bdLJShC
z!ZQm8Zg3PMbP#>UwbPg=h^XzJ`gqzzl6x+}f8QRIq}&-gko?XNHUk^-n`}Ui{wt?V
zD!2^h!9Jmvz4lw|On#M9Q2mli_W1C`2w%k!$%;7qTFH|RKl4BOe{in5GV2jgUK*@y
zv{1G60SNHQE>VM`Nj#W|+ybPaK^f^_t^l->bo<00g&SP0RXa8y0=W>-y|DwVyon5)
zZq?iz>P+eMZ$n*vd5^$^ip~fqk73IEuy3-odBAfkokzYSQp0qQ7%C`fz1m!)0Dddg
zAP;OD6L7?q=$0e!m)f=fEG5pb!w&{+6A3^3z?qA3aojC0*jHY&l9npOm>!JSuRi{#
z5yii6o6XX>!#01L8zJ^^RQ79o-$xjb)9d4>Kr^bi4j7QwZ~prpcwoNu$C=}KD-IGZ
z1RY$iyP#oOu`}JFd9d48)zn-+d3dqe2yvuC%;fO8s*4fX9r9DxYe_pG4Mf5I>zGB=
z%TpcaL@sq`sm&0BvC*A%P*zxz4`aCjzJT5#8-XuX&K$$mvg?SW)I4Y!^IN&m_mIQy
zHV9%f?hn*aShwHl07aZMi`3s}(Q&%YB+N*W2nb6&Qi0cUZ+k_iv--hG%H|B%4(M2e
z1~<PFPih*tXw5qil`=dVC@VR4%WbR{tT^Jp=lOz05<m5^hX~pF>{Nlv{PW@4EC`Dl
z5}%qgUs+Mn4So5bfqp5kpiJcm=z*0ud)m2GO|<dlyB_e#N`B}WhY)a1lN$+z76N=q
zmhkc)(Tf9_RH?2CTqBb1Ml~U{iD-xop{Sp`wI~3PVQ_S!lZ6*l(6n?|Pxb-Bc)TVg
z<);JjgwaxCtt^JWDjqmTj5fsCCDY-CaODz+1?R7zERPDMdmnMuLC_sIm(6sTyRJo4
zHTl<nC#M)e;~v$QKNO_r`1WVau>a@HM<gobt)D=dclz>)1h_xH$QN|v8+ASEBM^=P
ztvMBX-V@FPZ#(hL_*+YeLp&G+0T0fqq-u4FylF#e4RN^y%ZmH+w#RWv32ADthhd1R
zbr*G{;Dj-FPuQvpvAa_IVUBwaXp=&tyhna)pZV$Z8%)~}-aj5{!1_r%0_*U)I$s0T
zF;5p;wW`e5`<9%Kdqh9NOiaNJOs4ZQO+FtZejA`2hSvp|)#qtySuGyD?bdR(kF5{S
z+E8KzaYH@gV)=nkHMpP1j}tab*1^*DY4W@S{oJ`>c3^8A`Z-(cPuG&5)G3hdmGI%o
z4d&HBlO!I(#S_UTXJfCIY@@34qh@oHs`0EH-xKx*F6d|lx93UC?#Iy!m5`hGf|?jn
zi&_yNZq|sSGA*6lgUA_B*7+bJeSagq4$z-T68fCv>wHVbk!9H2oE&Qq19l@$({D7k
zO%>3&`cM*dfIoNqSACoi&k#IB1tp5l9~<J2_$yma>ox;7k?``<Plr;wK00Rn0CbK~
zVL*iqB(%wAiXdFRw~yKKLs0K5EIBg@iYPUxa5|(wTv-?ui)ggrjyq;JfLJXKC}F5-
z=MC2t!2Xm2Ae&FqFoVZO;E1;ir9)>$HX@@zN+H8n7ZeQ9hiK`Jn??t3>Vhb*1Bl}k
zuPVuoAI?jA*7)&*@yvBXXOjQ9ZoJKVWXchg(Ohy?>N=M0))odwFGqnG(hHFIDb#+#
z#zMzec-C#p-&FR-LHhT=j0xO4mkUbrRQ%Yk#v|y_f{=$-02GAcmSKbgHlfMe_a7aV
zWpNwh{W70B?Jv$~q&f{WG^>@gfhdoS#7HNod8`ImjQGqFH<Y?c>LdJdd$jIl^08Ba
zW>b{{(t(l;KC_`$ppVw3X0Awc2}CnsBi)N79YrnMl$R|GPhH^j+%vfI)6!l)_RMuV
zPH&{Z{UM7-HVDr0&HH9;Zf)GK_t!V5Pmd!F5XmLlBnkaktInN1kczXnBb@WhP<J33
zGa`<8<3OXV19X$!M#QiB4|iCQLl3w=^#?lCe7(9F^sZp(Xu}m^M7%}JPI8*y#?HYA
z=mgc35++vwf=le<slyN3X4RC?uQhwUVnf;*qU1P~<c_nk{-!jjN<wNo`wviEJaJc6
z097QJ#2`B&FTg|NfQEhmjP{4M_K>atVge!Q7~_dZSn;~5z*lDU11osEh&m>DrQaM{
zXyh{by5xdCsh(?B%Nktnc~_t2A4Fb%Y}Vu{9EX$AcN@^r^7j{Y&?cOKQ)zU|V|cyV
zae%7`pmNPvt4FQS_8p~;t_&O3B)w|RzvJ~3sv~1}E5^$zB%^q_G=LfHJc9n*=2Va$
zulM2L_YZ03VU=F43P(~>woR(>+BTE|Ms-?1IiuWh@B(px!^?`)M2PlOXc?t0)XYqV
z5?CF*4Q2xg6(wV<9zaxXXME}MA*k);{71(tnZV1+4IHosg3+UuZVjd0l~%N-^n06x
zy=BnZBNn!?mjh8KR~Jb8;Ap$9@$~aCtHF}u7QW??EvAv-`Qc15c-9}~fBNP)S{!%X
z%!{_!z8e3191@DVviSP5##-MR3C(`4BVMXF@>@eMh5Cp$W2xbw`ePoqLZn6Hk~Isr
zpeH}pA#&{xx1fymd-TChbfkc<oju>vZ=bvbB0R{bkk-fLU?x`!i1-T5KttKU5S3WH
zCRS;_kRzQ;JVt$|M>n6w309!7+{bJ%aWPJgU)cUw{46$yAAzb-t?(LMH38kUe*G;X
z)Bw0i4X*J)BP@>Ya`X~3dluI&tg7yXg3;{IL>JV%eHMvy*_)tVEL=Quk>$Ex8o9~o
z=?&m>ACrXLUX2$Vw5AFzv<At`wp_gJ{Oq^B2uD7vu>Ny4E^>rvgt})O_604Fg45<>
zC;fd|Hpg|t#m*Ot<05|Zu5Pm+&*8QI{>^^arq8EZoj(NBTJ~`82WwpnM*jGd)=$8*
zjdDEH1X8oOo24Z;or6=B@{TfT{C4j${oPTU1N0lL4qB5??rN+UtWgCk#pNi?Pq^*(
zxdr{GQlJbORpJHV0TtJr+00;uijozmvs%wsCmSFg@xehhbWt=&Zb}z<2Fo{FZ)7;k
zC58<2X=XjXWoiJecasQ#^yJHYVEJHI5e4WUY`mSwrV4V6_;p%tTnt?tJrT}J@@Ev5
zS=_@7wKE;i3-8oYgg5-VA*66!u45a&zba%BxYz&a<NRdP7Hvs{QiDBZsmM2wl?vs9
zW+BLQ;n@+qH;mvf?7t#>c9tOr>Z*H?^8GkaE%Y3f-O;!FaFou1=b>L#@wnIQYqBS^
zyiT>*2tj`KguwP4YryHcjClxy8Eg%ns=kJbWejhP%C6ZPx?;qywnD!fBFC$H9Zm*F
zkvgDem;M+F3Krpp`wmnSK7TlRV=4|KT=R7hx^r12B1OJ6RQm>!Oi?)qIuy-5R~ygG
zE9DsX#y?7y`yVjs^;g@!L)6{<W+PEQo(^zMiRMDBz5HPg?=C&lVU`V$V}Z8hv4D+A
zA|+y0141#e$bFExy+Ps$e^^D3o5WS_FPbACwG6;?aLcTg@&)G2IZ8x<N5QK9FmU}^
z2bLS%09#Z%E_?qKn^n*TD)nq35!+3BSTh9u!igJM8acx98@?Ql1OY)@!c$5s$4mlb
zv@r{3Zi^Z(9knN@2qT;4)FaQ1Fadxr?C!aKOjgF;?_@P7Lx}-hkJ5)_+ei6(9*_gn
zxQBRzG)z8?@>t_L1C1fmb@-^DW?7PRBgu$^a}lc9IQ_f&3ZP46rG!Q1a>Z8o%9rbU
zFttzs1n!n8l`IYcg}ehQw-9dC?F#3A#hV<Ty%A;MrV*l9vW5inqROQ`U8rWv(1-?b
zMpT0!RPUHvL`mMepH{}}ac8azoVPB`R=R|=FNZr{(`V$|<we{s2J!9A<DkT^kLK+5
z2_0|n(^&x*^`2NrYYppk_24k0jBvOJG93c3RoP2PAjxKp0s&7Ne`PhM6QrN#5fNlq
z6ZJ6%i`hf5{$pE}2QKA^ur`Z+nMA@0FJJ8V-8M-CEnzArMXYD;;qNcrY?Xt(dtmT-
zz7#Z*oLtIy0Ul79x4>Gr%ofNRw_Pi@IbcVGHkVr)5HBR#h;OJ)uD()8CG9Chb^z&v
zGl));D!SfV<a+I`3vLynjPr~d`I8QhoMk>{m6Mx*zncn++sv6}fD=@MbmX~6Z%18Z
zg&Ra%<F!1z_$X7tHq|E3SteC;duv|jxS3|Sugqsk*c-GNsqjciHdJjjl(0uGe>mY=
zqVCtKHwl92aW;g_AhxOkQzOTW)O}v5(<WlP-Xbr_%W=%w-_(PwnMTr!#*5=CmX}V>
zlUVKPXL}emT5Lzsm?5@ejZ2yeerydb5!?(Bgd~Y(ab{|dc>DO50ft6$YXV;Kxs~3B
zTwnLZNlJ)f9&|15dBOz7>eYBW8M>%3%ue|jXlZULCk?o;xGgT9TL^E>nzQ|KYF_;2
zPdjDcVl4&T7AE@up3qtlTIub<6XKIxdP`ysUB3bPWK=0yeC0c(!*(oneiOp~_n=rP
zH5n}Ia;h{q=<st5+=@`a<VR!IljW!A_O9$fhDpKAXi!1APNv&FSh{hsjk*4k{Gzw>
z=7_U#S!bd6(%r$ad{M_hOa9xRN52W-aU}?qT>{@mRE4@fdCj=A(nZE7v885G8WfS^
zf!Xs62TcYS&4;Wz4N|V1V;{~aOGdO6H}Y~>{9QlnzSJg2+uKH#wo~~HoIb;g(m3Zt
zizo9Qf!z+`gaZ%xZARgMamnBUSTu{a;p_QvTymq8ZX~_SEI6>G49aj(zl{rKMQai{
zURw~;Qg6jEcPue9dD5YiK<xf><eJ8kjT8u<_siyzBE)=)ZfQfv_zEa4AAz8f#H+6H
zvQ8p^GU(?w?BY-=fL3jD5j66l!>w2_LNo>Auw&ER_0qqvL8W}qHsbP0Tqg2>LO_kX
z^WObq7Z#3}J(I~XA@tcGrqAs=d#f&`gVxj2E1ZOYEcXt>IUM1)DJO03f1)+MO)9sb
z{{9;6<!QHywKh|I{bVzg)0XQr)lxUB!N}Ue!pPUP=7t@Qt$OR#olnV2&htyp?5(V1
zLrAV1e{)Ft*`NI@CQBzKm%yZ0w3$y{JH}P}(<r68Bj2NCHpkVJsIw^)`>3%Nj?v~J
z6Q8K3oH})9%)7pjn{>y&h<Zapp5F$xP%wll9K7T<yoU~&5KX9gx?l6uiYLTzdHV!a
zF+0Z#6<!Uh$n6t+5+y`v(a7RWwh$7|xrBf{bJ3YyboLBi1K;7_FU-)AIY#5z#PMNS
zWsvyE`ib(%)NIJbel%!{y6jLXvgJPh8Y%0S?4Z)Jh$vf&c%<EeSob-W??z=h8Cg_A
zD~(LK4l3hucF*!?A}oI;6jQZNvX5QwJ$i85wAGVse(J`Pc9a5@<$YX42(7#DT8G(0
z8;NF(x1E59dDa|?^Cr)8$u=2p=L!!>0Xpp*i4e_ZYnUFh`ed!{)U)yV7-Qj4mYMKB
zyPJQ%#WCdC&uU1=i^<1Bby8d}p;N8Q1`At;In4XABI9*Q&D#l%?9WgQ#XoF|;yco`
zk9M3GF0-=(!y?@52OJK<QoQTI=z{y+M6H01f>rCuI{b<wdF=2q(vw|W0r!pESV|}L
zyT>QdQ;F|m2Z{C43ie!GJ$`qZ#W`qlldP-7YOZ>+6*y?zu*jmzGx}h1-Xy%vVSK;F
z;o7h7fxpf-fbIB0_@|_hbe7aDKIG8(uJ1mF_@aM&3(>4Ma-!T2OM9Q8I-Y1;T-+yZ
zA#hSoyrS-%y>rXr1$pcEVp)G5f{`q0%hq}Zg&uvK_Yp?DsmFz29;L}m(Rwz!rJ-_+
z?ie?st+gn#o*@qZF-EKjCJwnom?e8tiesjRbc3<Na;ovm8U`R5w}q#(GR32JY`MZK
zu{C@f`I85B0Jn~hP+sr7?#53Tpu`H)j4x};EeU<$x@#&Gg)&7Sqsu-LlN5ilsQOOz
z>(~3!x4n_<$o1a9zJTqTyb+xk7tE%wzdqzZP9lc9B~IJW5P7v#7KSMW<T4rDkRTqy
z=ssoSb26}aLMl|Iv$aEn;xeu56uE7`mb)bz;)W~Ce^nm63?owq)F3phsI%1$9C+8;
zg@%)}u#+?b7Tqz`m2WLZ)+D%&KLCTaZK~WP_BL@@*KFCXPHnH6@XozA0(-w+s{6p3
zLmRXoZl}C2#}aw+XwJu0jgnc&DI?)CltmPvK<u6`aey1CIBA<EzJ2FriH|cSJPZKi
zHAOa0*FjQ>!~1}`mD0#nMY{za3sbTUx;M9e&s%fvno??eH!|xiS%lz_vns5dh@S+z
zCC}w1Ai1n9g&ZqlEUym-K|PDwy~mEwmV28i<!_$JU;-<K`vl^Q_sv;9O!~Qa@S5TS
zZHeI32N|N+OE;)<g_jUg%(zi9JdQ1f+$L^`(VSX0sW=xc!Cwv?J^L^Ja8J7On84)i
zV(wLG`N1nA&Hz>19UfkJ_RC@R&oycg{iF{M!-~1^t3H`d^xawC!Q-UUaN}+2Ll~5`
zP4U?s9z%3D3%gcbWcEj7+Es3I8wp*UhK|;4c8hxTCvLH<i1&R7jAVz64h|N&q<!`+
zZgVv8C$=N@X?4`t$>~=j3$jWrNNPO9MW<<OiaM4!VBdd!ae^$ud7{-3n%T`@T>J#k
z&A8?(i=*KQ&m=OKkv>J%5)7DcLrc;8N%;G^b2&a=n<=p0J%(Br0~A`I^Z719|LCEk
zY4q?Fn_&~O<SD?t=wtRa?8k5a?PeIBK3rI@spIti7MW?z9c>#kznl6C{ke7Q;daXx
z5v!SL7_HwH^1zcAYTZ0*3KJ&vJcx%c-&o&jhEl<wV>BWb!^|F5gGBX|V7f~MGAq3y
zM2*(GpkHxtBDaGtIpf6vxk5KPxO^b?-mjswk%}WqZq4&#tD`Qto@GEmtpo6vQzyXG
z5}o<6@kSl{Jq~1~?$fF`^c`s)sae!saPUUV6p(a|nU*d6S?mDJt<7Bb(V8<$?q5%g
zg+@b+!V+)9!bB$Nj<M5oc0^Y^`7tG|aLZtA*lzUxP0N{m%;<q_r0K+7zm2hjyF`Y?
zvSwXR$L24(rd29K<NZ5cQ-FpUY*iz2bO{A4USMrSE&T?UpsQ}NRc603UMzoj?SjbW
zn7F%xC;<cq-cHXf3~zW;0nY+)gj@6Kvang`XEt{oI5W)C%QyKwX0*0*oWEQmG1;tb
zULTaDo<lxa)g$|>-%mp=$$>Pl?F)JKp;Qh<?A$EF^apB4yUOgAzf2V-vutfWOL=86
z)1az2BR=H{s@V|+jdO;!hj(u)3ST{P_$2a<{_{U>pK~|9-n#(Yr)j5Aj0W|&{cR3N
zYJ7-xq@O!6=Cb*<KEm4VfzR$3Rnc8TU9k1I1@)WRlO^z})9dCGK*#9F3SnWi(zss)
ze$`>eaEL-vWs}3<1+FrCmQCSnY$mP!!X-{8$+#i!9YqT2?#6_d>(J(LDq;Im_`C0C
zq5HxdVfxV%Q(e~k9z$kcJ{AtCyDftm9hw_LbI^(nPWN#AE`Q`jMm5-tEn%OgY-~_&
zBC^#l6r5(+Om&-+{`8x$j@YecT5XaqC~h`H3kyU+i2cA4{<lrh*lBo_Y{u)~;jEY`
zq~Ce~9{tTQqt)p?IVw;?)jM0y-NVdV4r^^`r+2ToCOk{MtaO<Sm3;dP8anyRn{!D^
z`3G*sPh%*i_e6QCJ@7e@sQBz)j;z<@FlKPiMYT-^*Y=3eZxfi5;{tsEFaGrzuvjhO
z@b0^D`V1H#e-~PX(b*Gfsi+Sn$HA}t-CRe!$P<?Xr<)u=9<zvleSd^abGe~4l821#
zTBkv17N%6VUn5vs><haSnB%a3=>jV<sKH2e?cUjinGVb5sZwO%aO(Ca(HjMDyOx|7
z%Y7U9p+#wx*He54_e;QW^<-dbOeM%4ZkkMNQ{tK<opG4swBGNuxg%<8;c4bxh2>#3
zY`>BRO*E04jV)bl)h6wUb-Z9&5-vkY>0;1cXgm~mUAsrf=MFNPA1dr2?XYoyWAV%2
zFp@=J$h!$+0h*1{q_&4_FW-<Z$UbX9a>xp3IhNtXwGnVbT~yJG?if%wAe<;IOxFq{
z4ciCi)22cTs!@K1x+$jg#{2a4JD!nP-NsV{)S=KmXul~OO<`=^S7fdkFKn;zaLzG%
ztC%#bd$+yNOjReEFcJbI`O;oLK@vpC(s!3)byj>cLnma8@UhjbRMG^s?*dn+Y+2ML
zx}7VE!(3x}+$M8=w6e0Sg1;!}ZQ62`(F<Qi&%BYCZ7-9B_&Ui=y19`wNVMfzvT2Wq
zT{iZ(Wcg;$P^52bXKjWIWw(y91Txl=O_wUi61Zh6`aERp9%b2O9LN304TG%mhA)aP
zc(c28zpbin|7R6a!@zjBkm~PmurB>9<<4jq{A6p&eRqdlGRe{9jbVN?pZEo7Lv0@{
z{AKZ?FGz>b=H@QMeWQN6<e8Y+T{1;C0V=+|E8^NdMdp{6o5NU#F|SJxl9s=SzF0ao
zA7$J<>Q*|MSZUF9{IXwfSFSyNo9Li$6OKei2m7jbsYr_6NKEN$G`Vv;x5EMCVv27q
zcQSaAXejqMONeLCB{G)!jSUOohs0M-`JXqbLPKncefubn^6_Cb-6Xah+6z8Q^?Db_
z&$EXMY5R~*Ct#wcF2TE2^2r=;?4vh)jFY^Z6~>L@R7mDg(SsvdRZU^SF<d8g=hK?*
z=8zw+`Gr~V%-JJ<{x3n?nrHiW+6&!q@c5iI8PFxUQldCZSF{>OCYkI<3u9Gd?qMiD
z=(s&=QS;q|m|8eX9u5xF+4=TLkjYVCChjD$q&qBkv)__PcYLw~Wt^hizS$9QohHo{
zur8aK2Ks`Bj#~{b1CA8!6H}p)xJ}h3BADBCf*Yn2;A{6$YTRnUZ^V7kk9YOUm&HW_
zx+CbX6j7H@B0zf(m<}4#lnE}h$kieHCLFAq;bGQ{?yu|0#T4RNSz!hN$6CsN6tJI}
z#%!4d<XNKoDJIr>JFr#EI%L%|nKMBpz&D#}ToX2pQ(f2{vGt<I5DJ?UZ1u1+cn-Hx
z_a5!tpf-)ys;BPbEX*}uC>-H;xVgN1m))p}2X`smO#0o<rF`?zk&coze8*)yE|J0&
zFP#AD{Rm;-!Iw<+?ASIref!G7m6tkBuPsfrC3U}I!(c2Ya5>^-a4E&0;Yg*j=tL>?
zOY|iUGqPh`>|>NRZV+s13sbvM&eG#GK~vG@Wc$g%b3ryh+~Q$B#AvY(lve&O*L@q>
z`r?DqFs09DaLmp>Tww06`P_ZpbeU{v)5{_vA-K3=F_OVj6X~;!+hSks4s~v&+`Viz
z?psMxPmL|r>rYnYDeRDWtZZM!E@@%9YWx|)kndZYRSEeS-0UU1YK!GY5x2(sqxwE5
zWU5?nW?&a>%;l?^3c45-+xB3Pwlf!1qBl^F41NXAA<ToM#dViqf!TpeGX~LO3z|K-
zr-GJB3uVH1$dyRKb51S!x{Md=kO|cVXYbEDs$|{4c=HfVhivg2in2fd>x=hQM{v~R
zqca7=I?wwhW@;CKp3#KO&&;ogA{<;IUdp!HF%6sKc@Bg%;b*5*G6#&q1A56pH)uV;
zH}KtM>EmKQkuJek0zaO{dt3D47uPu+%9Rw{YNPHAv^4Y_oF9=NVxBjT5yNHYqPMM-
z*~k^;3`^H0z_5gv5tv+SD!>o>qV~G7qEv*KZ8i$!hEgFZ!guvdGZtO5j;5LGH`SIf
z-9d5=#tZAoolF-BRbs4yH%9J0x~v?$pGTe^o|jf~nyMo8)Rk7^f47>f{M=Q<GZ3SC
zoQmXR3WK2XaSDJ%#Sf=wJ!wCDd!NXhPr9DW*1gduG6C!|8&iC;uz7i(VutIv)6i0-
zO&b}zerm8<C`mXzD$>QcRL6;u&Ukd{HZM>O%Wsrq8-IV|MEXUJY2R>`VqM3d)I+$3
z>4TCLY=vAvbbjWxR%1bxwH=@yxA|T#id5wTIP>|su;?yeyF<T5e<H%laO??Xkd|Oh
zuwB{gO9Xp72i)%-c$Z0)>MJYi+PG#exxM#Y6s5FqG1G|_oA0NYJ672BlJvl%X)9@i
z()E#$73Q9!hz|<2;gGp!hcoE(XoO4dt{{;UmCeqH{Wldz6y_5rn3*TD4{e2{qhBuG
zp)0srRM}d1f~}rEwO8}ue6fp4a(vAxH?FZ2W18MsQf(*q28^jB4+&df+p3OJZw)Et
zdiTRqV%K{)eb1Y8leF*`a4p{EntG7YA^GtG`YD44fyw&CRNd^17yjm^=DeQEA>aAw
z`7)2c;eO4~V2z!~N9kM${E#q7sm^rzA|bx=`K4;lzj>ffJR&eLWs>tf@ADC_(4(@(
z@x7AlPTR6rWi)NH)q`KR$b@>q<q_27K_`RWZ0S@q9UV`TkmUMff9(^41X|iGHnE;;
zUIF=a-$ZS0QsoNce`&Ag^txy*#k`1pwKx|S8aHef_znvh5w%Y=AHsHB9DCwBA<sLd
zfP*N!(HA#+*^SG-?+@jh%p-h!`G+IC`&ex!MzKAo=1QjXCaWjK!(|$3NQb)~h&9X9
zB9PR8e+$d`nD=1cPc-1Rk$=#%gh?gC>g(!1ZM^*WP50@GM|vJ)$Ou(r2r<Le4kL7h
z9B&9Q;OydK;)9TP+V@xHPI(W6kWQmL>Qe4V4fo|hIgnE8>u8)Bw51X}Vq5-#R#CeJ
zAm!ABu$>LX%3b;R2)YuQJ>8c41pH-Qsqou$PcvYD7}fTp`JhBVEaHAxh+x-3!{>VW
zKJo5ArJI|`a&_??l+CQj?$eSnng+zp#oZ<_@Eb*UfS=;)LR<kE)~0}wYN=oLp*rxV
z@1h?(NIPC^;X`Z<1%a?n`XP6J)cA3>@4XvWjR)(Zl3$;kcQ693hu*&3r$Z-hH?FYX
zzRfw7tbOA>rS}@JJT7p)Qs!r>YT(2kE<c+8)IX`nmlR(wHDu(3s80l8yhY#9mQnD*
z(cX$rXnhn_hraBskq2=qKmy+7IQ5oEvx4{5<tTe(07i9~DlFBa{QRl?gQ}?~U@NmE
z%UnGrBGKoh-2!Yyk{o|k&d+x}*^uImP->*SxQ3LfEop;vf+^#YC)*hL*xppVPr5I{
zz$DONpu;wzARG>(tR_Fa<-CpYaMPnQGe~Viu=6OxEppf8qeKIzT$U2qo*BQbA!Qrd
z0aj&I4{$#Y>wLKA$=0&AJXUP~n1AK(hq!bba}H2Nc%q$GMd$63^DjVK9Y@5?NiVk(
zRe<OmGQX5h?N*e+KE?@Toyyf<2Uai944FuTfs_5_xrR9W2UEWQklt|yq7@XbVRN*q
zS(H7TMp{erzz=g&#*fnid2cKgq?T&&uJ=xAdYjPYQm1jOq*!HonCq2D+N0ko5hN<@
zm5V{!=EEzuu9JO%8YZiw^%@y#0}Q6&wRecD-_v~eJoS4M?q7O^?kc`WnKQc$h4$qg
za~Sa84;e7A=T5`7#|w-Xu0y7a^QiKtI<6aM{cS4{n(3#C+ZNy_&x3N@XoTLEnW_kG
z>&6i)`*!B!#dy-Do_N6_2d=3=*~2}IirPMkDXwRb9q9uY^nR6*qTWq@?Q8svoP2#!
z-0i$Dn`zvp1vva4OWOoiQbLqS56{pn$DHVnsKDJh&tPMc^g4W`f9CrA1qG+rbx-5w
z=-|iaZJdo?BEFYjR;|A|8lxPSJO_FsW~9D!aFH`>zqIO>5h>wQ+`imh?Oo(Uy4Tbx
zKzpiQHEMkq%(r$qtQgC`SM@I<?6&@dquK8me9AYXxWETbhroKBP?G9`4RqZK8Aa>m
zQuW5J-(3zBcWLqeMLt!h6pWRQ^c+9~J<@oj!o|tp^Ydfwi@pGvY8JaA9u<FnPr3;-
z^vYA$pII*reKos8uyjY_RESFObWAeG>l19f9hg@1Rq^PrVx~AW3r1yN9DIG#H$Hh_
z9<mGE=EE8V%E1EHeF|%XTFFnIFm{l;_6cs<+&a3CsShWNpEc-Nyos~@msRCUo81I&
z_g^%A$KO4tDGYhI)`4GQUM9m7wRUkX&3v>Xb%v(A(()mCccx@lGrM^Fv|G<*s`EP`
zB0gxV{CC``98?EqH#-fyAKHure!G!}^r3wDVPZWS*Y85d#kEvc3#|C*Q9Wfa<t9!!
zdX~>dMO1%xPo5>wQPNZFK?E2)FEA61E6tbgF8DZB&VE1~<|QN8&4wK3c`_Z1yv6V&
z&_*)3=L$n2{O<%kPC&=`>O|V~TI|E)MrRgLJF*FD`y33Rxh=uZv{Zi=FY8_murfzG
zwtT+7>yCNg!xulIIkx3}ek#6qtxa6~ECZWuf0Wai-{G^g@fq-Zr(C{se{5M&v}^uO
zZo7v!LXK+%W8XkI)5l?0-?fTmg=@LOqj?(~Lfv+&g7@zdB*I!5QioPs?4t>qjkCT`
z1oN%%ln9WotFHr^_hh=I?OCyw_7+aQS&ce`p7W3|;dQ#Owgk2yEBL(0I)~i$=gVVO
zDMrIWYxRZ$oStklUbSuFyF5xW*HY`l5Hi-r5FMVi`a{y$0_p?QhfV&+4#S~6E$#^#
zXHKfLvi&>Y!dD|$aNcA*qua1ufG|QS{<X*0>u1KdE){9EEPu^2&^UibWnus(huxa$
zcD$)tC8R=aK!(WiA-zY_lTH-;icB?xHFr8ayTTHoDl5G}%vrxBdLoo|+kIn))K=^r
zPf8=Hr-&Kfenc@!W|ogwaR%8r&(>@t<RdbWjeD`=L4nQ$MHC0lLFvc4ETBtD7*m&~
zF`;o#IA%c=pm(vHMlFqb8CYZ&`zN7@YmzfxROM6M_DIoq*_|gU_wZE!g`DJ7f!_4k
zkFICsu1R#qB<CbiU3;~EgG=4dlg;=Y0eN={R|E6V+FMc`C!fZNLJS0-6ZZRP8$be3
zE~aDVL9t?9LITsGU4R68gUOzN*}!cYI!~&!T=8C-V@~ymlH<7VP4TLjBC{bhI8C3U
zmF;UxAlK>zL*(+UkEBu;259*{D8*|imybcHmZg`=bBiGh1<Q)_w0lCq8qZH>N^#13
zI=C$6sVgQ%m*1SfQO(DqC#ruY1WX052uM6pLw%-`p)0RQ+K75P|5X=WSzk2)cq>|D
z@20y+-wil_XJRvXQpL^GM77p#Xg8d~CR6&k;Sjs=<5aYto3y3>dGql?qvjgY9vg(?
zkP(r!Ja5z{@I}eTb=;TAxT4k5O?q6m+p&29`-i&lb}fmtAHlBVoZA(zgEBuiHT<x4
zA<4iRwT0Xp=QJC3Ww!LnUFI;!{(Jj;d3991hQoI$ZlBF6BYz$1Q9i3BwmWz=l}6Gf
zx|@59Wv^?DJQy|6nGQ?1H)sMHElZl0NcvDz`<b%Mqmlz2<hBG@%+;J12fg)fqiplb
zC(OzCChOCkn6pc1xb43`=7<&FYT78O<efMksXEL`tFPJ9FWev*L_+8$x}`-Qq!e!=
z*o$d!)#S^+YSwNph&Ai1tKJ$zt!|vn%QuS;roY2v+|DeLIU2gQ-X39zZv=<3(=XV4
z1JR84irX5&m1lg=TC_fMjTJk-y>FO*aitu$8p|a%+++TiQ|h<*R!j~_N=OZ9<kok5
zw_WDjbmUdIEr>&D&4zq0NGT3ip3)UP>LtcL3unfBh5x+9oL<nwj1Iq2(TzU8F{|n3
zQ=Iq^2m{sIYGc7ezP*$_qJwwcx8yOnTQ_1gPWsOi1Sze-kd^95{OWpl{r9J^iV~kB
zQ#|Ww66uaPjyE59MZ8T4tFsHfs+Z-}o<#K#ydo{&2NrS34(Upi?;I185w9~plJE10
zIX(G(ipIP3qbNu;pJ5xPO?qG8INWY?HS6lL^B#e}_*-A<Q6j1sIrdFF^nn;dMxBL3
zG(Vp-CjV$Ka0=;juwXkZ+tvcK35tGUPj;hv;KYX<E0i%mCpt*$&>}z^wTJsAu28;H
zX?s7Spoj)iEda^$+%G=lPfF;3@ph7zVjtUj=NRazDq!r&nOnEESIguoBh4kFr|6_n
z%zT&aya_Lp%M<yukYX{aUSvL&+ws0N+R+;azu<em998zN2YB}<)|qhLkw!gDwa`}k
za3l7sfbQ3N5GI88<*dRi*k@QohpLy>lQ}as(!Zuk2%3o6epHR`Yr?Cr6>nK|$zT7v
zV+$_UKO+~8(YV&FlZf^0+NuL>YTx39EJ|Y5RU;N(@B#PoE?)Ywo+GVih17g3?e3rw
zzK*EVJBddnQz}W7=0WAzZlxZH_fG6zj!N9i@MP0a7SrzIyE$yoEwG=dkyY7bEg-uU
z8>AFdnP;OZIioa_*S=h<;nK8|wwu~#z(voj(fW~ThMzWlGh5&TW$QWEhr2D#Jhen`
zPp-bI9kHL;)Bb6LW`pR+MxfFf61vs?L{h3ieE0Jf%ouPoq7#$fe(yS$$Dg_k)9KR-
z^iO#9-=rj84_?#kBv=|BFL~t2W+WxDHgDB*v*>8~jgFN5LB*@tm6+k>!y97U<omyn
zyAI4KsG0@Y`!V0hr5490bflnXZbqU_-QEm`L@l_23G%ig{@YR5#8YwQ1BSYvDW5vy
zcq3`V!+1TTj%0)YvhB^#z)NS&k@`v<bg^Io0vpc+j=5>3#+r4@3T%*XMwwDbeiNI2
zb=~*?>mb%GV@@ISVq6fHSj^>lnw#4z-}Eg8b;_oFsVO3r5X&qINfI%!cXCH?*#x)n
z;apeM-siDhAbGGYjelP>ntmkLW^sTmyN@(z>Lz|=n&Pbk#3aApW`FR;d*PL-{W75C
zvzpBW$t=PAFKLXTdsZE#X^#<i#HGoapQ{l?dxYGl>gC=3_TpI3SWv2oF_(Y`>SOt8
zmj)SquCNqN)}&J@+%*j*L!&G(o&WBUyoPWVSoi}2=OXhgQNkPC$;pzK6I3C3qTl6?
z+~Z5^15(a8_9d63y7$;SgR?H-`iWBjiq_5NzW|4)PAsEgclhd%&U^on3D|r2J=uni
zyz#=BK1clPyZH<AS>huwZ>tHacGiCT>+sibmAQQqKPD!^@F>Gf#IL{mC!14DYP}#I
zbj;O|Zn?YgGvkxQHolIV3l|L;P};R#YC4#d4K^tSW4T3&?r$v6;NE*0Z?ti1Tr+D0
zy75gie2sh`ppoc+IlauotDagD4oHkvR=pej(oH%}q}bPo83kIho%E@?ciMDmj0jKW
zu{xFYy-d&vNM;~2phrb3khQ_L|KhxIx!#x}8@nHAAZ<Q4$JRd~W=sw>qH^Se*`@oi
zANZLUNwM7sjyx|DD4n<^rgp6wC`^}WMWjaMA^|hdaHP`Yv`fCF5ExmZ;b@NFV9IF(
zE`)!&2y=as8N$jcN53lyP?^0P+dq-kvsh}k92N43s82|`IOGIdes6$GfUm?US?rw#
z6;U)M1iqdLRw6Q?hI`}T6ooHjAvR-Hmh;+IV5_<WHtg8yr-7fs3nZai<rWH3{AsfS
zzo*(h{8AIzS2HC`T@|#YtAqzC3d(cOn=nF!^>Cxf@{9~x4nW<EvXr0ifTsNjL{S_`
zmKSXsvq2Jcl#<m5c1=$%i#L40-swyD&YdVp?*L%R5?7Wb!wt>|Vbgt5`dsRpT%Bt;
zG!;>f^QpP@<7N|XRJpG!clK94asFE2A)A2_v!FIEvp*3f1C8*HVXh%NbGs>CeoTZ}
z`Jp#B|F*F5ad#BYxXABZT!|aQYSfwV#mQkz0}oNxeA(EWxw&^83+W@RbA8#GNp52!
zcMv{b`9AUt+)G1D;s9zQegNeYn`BNCFf`Ks_I+yRrpZDd43Un{Cn<TQ1!%VkATFNo
z(K|)BG`+f>;^t0l=Qr@%1P+`RuvMNnkuK*){*}=|P3COJMl9$ozGWv(P(`E^B|$ji
zv0w)RF+<t5OJgps(D(%20UAQl$7tyuN-g8#i}4y88h#0i9XZm?^USRs!&UC5og}{r
zHR@{PmTy5Q%U5cw(>>0p3CCN~Ok&v2(d`#zoWKoyhr=Rp;r&J_$Q_qY6iPVyy&}L1
zyhNm!Y}v|Wr`-Y^VhT?E4gTN)uw&iCHBBc^{rDt(qso&_wBeP}6r$*HFAaomPhf&f
z?{zaZ!!?IipMFTM4=%Qun8qi)P5YF4o!?uPgIDuZGj<G+ZsVZBsmLIbIOy4nHcJWY
zTWh#f)l1J>Z#&f@$rKJ0h_LC|UFP#4Z~K)z*_3sV?0}P(yeq7TwMDfh_6(k0_0906
z{2i6w21-rtYbA5W+*XV8=pS1|LY?6J>0`qQhc8L}n@4^~JLV&^w7Dl65O?ry>uVO8
zhUE<yr%w%sbjz;qSHlP<x=5096PbS>P*eG&yK5HjO}|rO1>#Wo1Rjr}BQReMjjD)c
z$l%|=UQH7@Ui(42jVzc+b14JXQEEN;lV6;mR8D*{vGa=Tixhwb+rg*To}YgR^gv0G
zWeNZQl{dalLZr>wgCRw#GJA|C`hI=oXHbuZscH_sBI~RKWe2ur*do#ci8IKFER>g`
z8Ph@>dsW-gHLqril}jI)`!1+0p~v4Am@`8`NIbUI^Y=;@+w*~8hG#G8459S5myM_U
znj$tendBCb6q2&6A_*>y>OS0!d1<i{FN*_URuk(oW&_ItD{+}Bo9I!<_YFZ&{#T~U
znnhgHy@Z_so}+2+R-ne>1v+b3+Zxb?DBk39SxrxxskT5h!jJI)dOlk=|Eq1`Oblep
zHdf-$+{ooI=4t5~Jvn9)a9%gQ*BtsZRKvnORK^;d79lm^jn3pEUs>@@et$%UzR3Eu
z7`-$Rr9-{siNMG=Z{5Fyg_#?>dr|!U$p2(4BRx!#QdC1)D^+|4gJd5Ri$ML=Cb;TD
zx~%UY=EOx{0)ZI>DFO2+m~L<Zb$|g=nSrX=9(5W^P8oe#jq9+nkxjvY1&#MnQtMtJ
zZHb~M!-x6eXz8eiPV(>RT1ED8Qo=z%iA86XF?QMB&;ju|o)QgJZ^}^09Ur-#*+P@P
ztgtO~w%br~7As;mu&@e|p$&_aL*CEMU+K<k`B8xBFAbHrZy9q|9Ht*Um6=`Rp5nMM
z*F{tAIO6bjfYC@JTLX0oRaDJ4X8$8jOjq5%QCE4_K)cL#@q@$Kv_ta}nn>03T#kY{
zGxknyKrQ=T)Ov`_tbOMWK8J?lHR<5ppi;?10uxuzsT=U)QH}0+K>zkffF%Zx2Qekl
zJ7DOY5$s)a1li*@-6blgmuRJV>tYW9bu00ktY#Q^%7}}*6>4L@P2e7Y#j&{aqqVlM
zS(H1zCNSA+NHPjm4uO7h;4c9-s+n7$YTj+QblwCjP!GC$g?Bnvr!yk3C4T3Ge?P9J
z(LXk36*RNh{u`Me4?K<Ni2UF=j`W3=hlk0I{QeI7%X#hoNGFDz0Co#pXww;f(-^pD
z@Zp@{b&{aT^HZcy)kHn;+DR79lfk3}IRG<BQ?-gO`z4tl(rZ1dgLFz*BbVW^0c56a
z#p7F-o&hKEGu>Sgp?pq-@mr#5M+WYW%k`0Lyq=)Iu7*?|<?R;iA)9WCUN!~BWy#c7
z)Q3Hk_%P`8M)V!BSq)mP*@vyW)VRjDJqX?h_De6Uq$T5SbjQ{tq>O<W!U8Ge`%7P<
z6+GwU71etn*g*UP0KD>ygX4IY`cvwsl##*ep<M(sNN4}39QJ_OfzPnjc&hUCXn<Jw
zjp^|0ek4awd+VZPMuY#t#Vix%u7W_N_^oI+TDIIIXc*;=)GEnaMbrV8>E0ue{tUY7
zy)V&f4RcSmTYwy*z8QKGXj{ud2WPjqApfdil#KqONT3hjAD2)8F;K9&N4UmZs<^KH
zjjUyuK=Mn)`)^II@j(+;f`6>Pe|a3pqBJCDmWIXv@ZfBT7}Y{ePNW=fg00<giF^Q9
z*CDD&@4*>+DwwJkMT)_A0AS(kvK7_qo&P`f-a8)4zx^LCsf=t9A`K!WdnJk#A~P$p
z_s9+*NwTx|9wB>gva|QdN><sjvwp`}z3;p3yYA2X-S^+$<KYin*L7a!`Fb6%V?Un<
zGrob}*a6Zxz(*JXhH+Y54F<fJ?|M!4?ubA^#jM%C&87(N05Sx=dOU$Pkg-Oc4d~JE
zk~`ijijL+%#DHNqCk81D9%B^mLWkH)XoYJ7`CB|WQ!#4MEhY}{KxtruHtAvR{V2mc
zzH}aF0U7=bB<+(0Fgi}ue0A^qY`)0M*^~-3gN5^Yd&ye*s%fZR3c1?b*3o2gPrF{8
z6*$c$X!M}fX8c+8eJat@lEuAOy^HK!2oezrfcxb=qAx~WM<MYqrQ^lc*T~>QC%szZ
zrtChJu<oHu<bacUD^b1ql4dD*Of;s=d^F?q2KGqj&_R;%T@r=DUR#7V1!ZUxa$&0#
zn1grO`iwkl!vN$_`avb|cG`;Leo|yOw0T~|JueB6{6NXJDWt*{a}9ck=ytd`p$w0*
zgP<N+mITIL)i~P*xiizSXKMqH;Kclm{@LWuY?+{aoGb&gP8AwI+K1jJl%d&o6e<_>
zxFhoJBEsPBPp|Nt{NJ*`>WWxUP7`d?HYWU;ucubi-RTb9NsD-lM|XZN1|S<wO?CXd
z)O|E6)#U^c;F~Slzm4Lx<DNCGB7+`AmRrXZ)YCvOj%q`f<eJt9yXba)9=03X+a)e4
z8Du6y@gs>lDUwBPYDFtkK)e)2TajnPIXD5(w8k={BBw=Q7V)(if}+pZHq1)#<%wEu
z1p0;u5D+GF8*U=1udP>3aY8;6k-;+YmX+X;4BH|U{-X|>AF=i|;{lD=7anORtI^MH
z^lmaRc5mw-<s#id%&5t3b9X%s*Cg)DLTNnqvA?(g3dc5xeaRkVtGk%P?eUD1Skfi8
z<k<)1l#LP?5Sk!F5{svGHKG5>Y|k=dw+8wQSfYc$b8oLqfq60Qnxh3TS0Su0Il2+Q
zN0eiqJr~52EmUs3?Ro?MXtkxygck<}Gup*I#6>u2^dL=*s=xs3qWyE35AwTONcJ<j
zZr}cbmuu!x7(eJRZRFp?U^NIM5~dwXIf0$TY;Z1KWpuX&!GzkJnO%R&_i?w5+<+54
z=VS``<0W8h*jFwRez`xjZwlSBV;Z~co*DvwmFd88@*CAv0c!!K&WBnz-vh+QzcUud
z^(x;|oa;;k&NoHubWY<?(Yf)hYQis)1h+;-|E<aYGvt=Jcx;v}n|k;Z(z0T`^u~oO
z_yM#I9vunq?rz<yf$FS$$A%bHhH#p&;{nv0EcYcIuEiI(EhPy1qpbGIU!AkTxd?!r
zRt{e`8p$b_==y7BlZ`Dq>rF$D(muVuhR5Ma&eQi6e^tb+2;89~bu{Bt88bn}<}=fM
zBcLQd&)P$tWh!V4G*9sq(#U6A!4api?0s=jR@iJdzm`waNTvcdhzDW8aO|!svaNjw
z@D1rXT)9G#R-6YrD1n#N;_lg|`G(s*_>2Q{pWW;zs-c$A64h%1zSb)7CuHt*Xklv!
zPC5s$I8`~6%?T@#fY)dR0|O#<3ZZq5P^R)BvuKK&!1!nx#wT=r7!7yW88Rk?F&byH
zsrSsA&)kKME1X9<<;T7>(^!<@LoerxG+v2FK({(+I86H9!f?;N;lp9HhhUi{qQTU2
zi42Pki8n(t-@)*XO8=80!M2AV-0_>%Ng}n*=r7Nl0rx+9V4<3K0vWNCJ@;jke1BBH
zhs~wjXD`T8<JJ#b{zJ+V5rEb@{#IF<ZZRGdw^gCvzU>fG$9;_$6#@greElgXLlPHn
zJZTPQv?fzY?JRX1fQ~kd5?3gIFn-~%E_*(tw~`B0Bkm)|PBiW*0U{3bor&W1l!~~z
z*cUcQbfW!nh$XBcal)oEn4?uZh%|f~G7I-`VRBrNy^f&P^wVf6!0bj3WeOo9*LxP?
zW@CiPXBp1kkCNN+qY{lkYH!(Ik9t+*?xt?yfMOT6Siaad53paG^$I{xnbmC*7|0wT
z=BDt@dEiL19^Dw9xC&4Xlp$SpIC$d2;^-$tI+iydO)(031sZIp#3Su!<_K(W?*XL3
z@JFgQ0wemk)^KKzCPBbsjDt4%xK0s#FXsMYD4p5NTs1IZCNs*DTnx*cbCkL(0+8c$
zxtp7^<x3yU>09UO$xKcdT7Pr;8KvDg_s7B6px3o?R3IIL9?`wJ;t7Q^i?neSyPhLL
zMY{=~Y(>)0H+Q9VeH4YoM|0q>?U5yiU{X5UvDdDo2Qlr;?eR=1r52w)WI`0Qo)sc<
zX1zXPDStm!E>p#~A#QZ7+8N)}uc@EeJ_rdgvJju+Wa$Xd2jIzX0-xIL-QoUgYaERT
z^k*RB9=K7wlYwE+LZ;h4uPu$AQWN~$_KQ&-;}jNqf1z%4En%^i@!!VcKYj03MnnM!
zWv3nU-ya8Qz-E2v0MsxT6>>z2iJB`P>OyUoJESZ2oTKKIo0mB74RVw*tIJED*|iiw
zeGZemUDA!sw%QQ@Gz87)o7nO4J8y6tUKWWd?xu_{LVF~K<G``{z?c|K|A$=TsE4Vl
zKXMbOwDT=2)lpvZ{n*B{)Usd)g(at)>S?-rX%7*Ch$`-V`xM_5WLz}e-SbdSuGHx7
zFMtl^()dl=o|q^T2WSv@6?NW`{8anK{-t`ocX@hF-3iAw_hUP133|tYQ)Gj997(V;
z2@4Hb%=XZ}98N*;v-L;cJQ2Ho7iy)oMKT|}ZSz0l%rYh67J|BOuPo-N08T1Gndgp<
zwwbQ}!~wfOK^GDPtCp7$8lj`T@llW=fB|ENTx~_9@Sl&B^sK;kD-=D9jW@9Kdz<O7
z{`yG@wK&g0UagT|6J3AK@+KNpVItLmGH^^dX$bF}=-yO5h)D>}vlpa_xx~e?ShTY~
zH#6qK6BT5b8dY2^kPfXs{<yMtsn{B9u2h;uuMVh-{&st5j&Ul8ebh9~ZQn1d+phj?
zTSS=Mi5_$HR%ojwGtMLqxtDp`CCZWQ#yubM{p((7)({mm=1kf_{eMT!+TRW#qH0{n
zOuw7hcQKoKD1Qa8foKs!%g=1lX(vxZ8Zr3}o4c3(d<BdU;uK*;$AK<uEB%^-v~^OT
z$Wu{wDy#%AfcxR&am|8Cz~iN0&C2?`^}lF$v!uLwYc5IU+HU#XNXWf8*=GPbTU+6n
zL->%&d6Aw$@cxd>rQD(7c}8~1U=JCLY@+N)zGiWdJPxzWOxXuD{cDZ(rmjRS*vq<i
zgO?X4E^9mr;&Xp_n|t!5fZ34Ud4w9!P+;G9q=XBa6p9K{cTz`R`{B0)Jc`>)ca-Sr
zgUFmj;ej$v&ucG7`8-|UZh(|Gu20+{BPBg;G;^lM{7!8DWyftmGOiNOc)u<zN~U1T
zJAm#0zK-r-b_qrzVU_(on+ZrwpFdc>qStld)h!>2qmd8Vsb}*+JT%v#z5XV3=BIKt
z^;)!B!omA$(9>zmXfDa#8$rCjjU@27B*zHWZIYuRs-3+VQ`#im?B3n#nF+Q{MP}Ir
z1>Z@Xla^u@@6S*^@^&sY-qy@RghuABs>oIB&60Q{eK@FZ>&MlA?9dmzUU=rIR^S1}
z6Q=s#nRf{~rj8GlC$kw({AaiIw+`$MLR7;VSq-v+qpAAQp2mhpYOBif(088M=E8W<
zn;F`6`Bye*zxXyAe)h=bm6;40yl6=_#L34&PLVBqX<WgrdxLhgo}Ir<NoT4lu>RV}
z@PLq`O>>vF=V`940n`^&`k!;k?LrXBwR(0D_JWnzPeX;7T#$lj1;65yyft_*+K^|*
zYBBJ1gQ871LbkT}j=h?7<I_a37W8ScfhXK5wsT9nHB8IU=iI(R@d`BNz0u_xRrP1}
zY!V?e6p&^CR+M4=cblLyeZw;X@rERZX*P@mdde;hs$3pE)u{CHihyb2mOH7T5A8G*
zCz+A<5KplSrW5ZrL^Nbvmw`m$c+uSR8DbtIvP!ZcyXV<n)~))l(}U>1Miwx^e6v&@
zB<m20v=SB0Lodt~Xg?28C@@qoRvB&tjvifc-gO!4?2KFNopl2Vr{m}RM^^!M&}yV_
zXLjY%5FfBjP}Ng&ZR?QOC<7i~8W0ie1r295zi1L%@LZTWpB0sX$BnH_@ZtHxtM8s2
z(;%Q|0l##DUCrS<`alAahTR3`g2T2dSZgZ1>jUiGZAw9`0!t=CXd@q2oA?Gs)mwm{
zMoz<@-}72&p2<+DZ|A!7#Qr!;jPlAZPukX*@-pDD_kQLI3Uzd#mQcg4;0lC(>?Pip
zg4Qe6aK6mv@$zr!XptTuF8U@3Lp%aS8+j#WXr5`x+-vs<q*|=3VqlU20=K18L%6(I
zjy6!ow_jwtGXc~XObC!>)KY4xj@)YC&DWlag*^g@Crb<#qF+rQg>>T}G3+a;c8@yN
zyQ%A3bozTN=K9iG0q66%f^VQ!rT;<pRl5ztO;SC7!r@hQI}K)b(<>XCfRK0WhQ#IY
zXM}Xmsa@8dL-Oh^EOyD30hOjWqNCF(qcDZmPcky@Ike4Ox}wy-04d(fSK7ln)@2(=
zC=fyxyM00)sA^pKvfFFz4>V0J0~9SSn-!bo51FPtS=z1?%mkNh_eRBTW?yq%b6Cb%
zooVO5xwaLv<KVjoS+XS+rRwPI)#+CAF@Z1l-N_8ie97*PxAP6o<=z{kBfR^$i>-y)
zW9DSyW~vxXOZZZB>|Ah$#{Tsg!TwWqNn#(&+Yy?pSqkWo-qf-0vvVGeW=HP}KT$UO
zMT5Y^Y!xPxZZ6Q*K#(^)u-3N+piK3(k@%O_`)%zXK(|Sox?LhGJW~dlIXBUgV#-o|
z0WsUbtmpZEwR-;>O2Nu<c#rfA--*9KQ2#38{%9MVsPT^!9G2fY+;UUwb$?KioipNR
zNoolh5f=IsZCCD2F~+-_8rHlY_);G@)hdTWV%ufS<Q$b;3C<x=xk)DW)T4Z_JP)pC
zWk!b1EdvwTZfs8LM(6I%m+Wg`@V(iCo{qP9k(HkMUPHBBRZO-7UpuZc%!?XG!;?P@
z)c30RhN|R?C3-9Knhw-P5BneN@%Eht>W$(rgY{KM*F)k*Nem~3hBrth8A%J?JXDRc
z?aIO`T=l+tcV%<amSBKJ?Z(v5^IDrJm%Xw_d0sEc*FVR+T5#=GE3tAQBL&v8Nrg(7
zndW}Ygk7>LTlkglR8p&qeUE{VqG$3Zk@B27NsqI5kiB5<8}@<rxqRvr>{+JssizOF
zmcK%x&jfY#V8HJ<iZDFj#W~57JcH$^Y+yM|O-E$$0^tStV4BjOpng1DH(b6zN7w(7
z`^e2vD|S2@vuO5g>Z}a0I8%f+2vMWqjZ$knbv%7$6|<$jVur%6twzn(tIzw;TT0?^
z*3~}|D{9KaS)V<%c$hA;E0$!ocLjO|+FH|lDGW5zRtfhr{9Zq*eIb?ssE(=KZ1IK1
z4e#$iy!^Y}jd^nF(Cb6=#z}0x^;8kkqy4|N3GGZ0UeIpJI{#u?yRP-p+Wmo7{WE@t
zcI^;PTcm_fC%SP7Au`@SC*2U}nG0Z=z`U^=K74xiL-G9jk-2Fd!vqH6fFZw1JtQhB
zi7qn^?Rz<eyqCE|KHetl8R*_P_I}`1q1@H_010zw<lrvY^c=1zt0f&G5R`Pgm}=KW
zDLvFJZR|KcusB*?s$o}j`_j2AL$H)>AUq^WQI(H}n%!7{z@lNpEKtw>pukh^falpl
zFU`Y@ayQVnMG32~geD4HsWXf2WEM0$ZPd&vD#|LK;4H8Xe@bdHG>U(E$X`adA*p`5
zpg;0XLjW|aHWUJ7CKu8Rn8cj#hmVPH6y0g)gtVt-ay#l!>0Rm?Md%$pcfI&Il)6h6
z=Sl+?QZ9981kR7-8}#2XCEC1}Ui)|`+%tZ1F3lNVz-Wp8D7kBflZZqs_kfCd#94vo
zoF$jFc(v^-3(DIx-?>!WqG2jXxzN|h-!PMD(&gkQFTK}(kY?Is8$tHFWNEV0i9^a%
zA6q!Cq~I_K|IYP;jeSXYWJ)9r!gM+HC51q!;+7x1KrTAWye>X*W{q_oD7+Grb9Q$m
z1ZvKEb5@IGNrsvtbP6-cPP%8Y2=;bTdL9+y8&Hdd5gL#<xIgZrFbG(Fe@4H!fs`tk
z{(e!%1Lao(DjHDYvDmlUTwL%x#Z<iwp#jQH=QBU!B=N{uXzaU|JrgU%G|6ZUP@IQ8
zr2(~3@C_WFQ}$-@48T0z=DoJAZ4Qw=6}w<R!G3Q%wlMSh&fqEI<n+zA2*K6_u-TeU
zkn>nsJjWuNPIf&-Wp@(`iBIjK27w5|eRKw}REyN+H_K=8?^1Q8+{yw}X6Z-!XFie*
zNh+fhkKP}ifxGbFX59?Q7k@NVq*s0s=9{8kOmWSFL~|norN$5uqKC~dSE}nRB$}bS
zF5mA+CD9tm!|}O{UZsq4oh4>qRK%inzW22pb*>q8-`dg`pIP)cEQfsjYSLsA*t>Qc
zXuVwZ45C-~kgO9f<)2DNe&)+HQ^>t@BKe5;x!XN4OW}a@PmHISbWicA*hE<E5U??k
zY#9&Iv}GFbL}q>_!L~uNte)Jt&4tfx5rnw0_hx@t(L~oKrlSw1`%aN%&FVVEXB>b%
zK~c6wD`iH6>XbP27_h(9{?wJ&20pOD+O_H(%@T*o+=s?og}*-D{CoBM9tTnPcdCD3
zAikOk)-#8P2HFP_@Q`RfOgxf16mq8|Lq*+Xz?xHkt+;fX|6u|Z`amry3*>R`lYj%p
z#%@CgSdNr%$l7&!v9uz2zgD0@GBT{08RizVPmq0m4=|S;1)W)T_X}KcHSEf-6kIo)
z7RcolAsMoB{?z@-2asm2*>7*vvzu8qlkho|Wn;Cl1lPHv)uPh~8C3^om)C_97!8F%
zJ+Y^Jkxu2L$<!%QQqnuXTj6Pxyy#FcP>Zq2YvSMPX{4&+ty~)v+s<4bY$%nSEi711
zjT@AA*Jf}udRU`VeEnJ;)~wfpFI53HJsQTx`_SNB_mRgn!_A!~k~o|b|9-ajD?@d!
zY~J`O?}tF@QBOTSnWVJ_ohD=D8$%a(H2h13Ii*i<Bwp_ibUJdpB-*6}y9}zUi*{qw
z<tNu{LM&PeW+*m!Q#KQaWeO_fK5|QYU+=F2PNbeadgHN*@@#aTMX+p(W`qw|v_A5t
zOmx35zi8BTx3$N|{ols=-@mbXA97>!nrKtO`!9g+chK)FKP5wU5;nH4z_)kA=sM-q
zhQ(oO3;Y5UjC>}>69vH8c>Zz!jT!!a$*^Gf($=}h=6PQO0~hp{z`#Ll@GG~1@(KS0
z+DrLv&c*qY-#`6dmyN+jg+hk2f8*J&&n0u`No|;Wr_`1I0(pLYIY1I#W>%LX{PoN5
zg{%0K*-18$Cf-C}ALq(pd;^SCb)#QDuI_1gnY1CY{M_$kzQ+lk*76}f7b<cECwJs&
znG`XEevPVT#(<ZF?zrFh<;!qGx+*B(+_M35jPha(<UPJt=Z0C&e~YnLb>McwRtIMW
zez~E#+^{#Sk>)qWe)%%8!}@nrp$(G>`0LtxXCeQ@ii~A94#WHQ_+h#HJ`}typd6#l
zBPFm<=nekYi~qedM99v#<nojM*Gs>9ykCf6>Sd@=_r4uG`@0=G^nbs=&j3;+jj7|r
z{cqX-=fBA)gY2aG=l)Lg|IR}>DdK{75w)RM{^C*p_A(`xfKbU^)N%cFX#G1`{P~gd
zDPdt5B96}fy|@4Rt0XKz;!?%iuzxF2f8(L_ufxJBa$@!z{@o$}8%rBA3&JfE8A1Nr
zqxkdV|MGcyp2EWBKO{>24N&}VTmn=`?@^F@ZZ00dzj52YdrR_BSXljWdWqw|);xbp
ze4J7-LGCThtX}+$L-M<~NMORk_EamxVE=N-{??KlLXjl~xu2b!$NhiV;{RWv{MWYs
zzbO=_r>Nqbi$zcSbhN&jrzZ4avmv8Ey0U62e|c#&Rrg!Jlh6OUk$>;2(`Q8L9Hd1M
z`FB4F`!3H_An*qlf=?sb5{;GpI(+)l1<+S6s<dD2@;|=X3oz@{A}%%{!~eHW@xQJf
zmV&r1YqjPBXp|EE$kCY8i<SD5qal%re0nZ)stDXBTCn`xgzco3zxE9Ny}G~p&54>0
z*2bULQs(qw)L9A8rrcLkPUHXIhVj48d`||V0qu;FNl?&O`4D$vjBDn_pJ?|OJWzvr
zf{(4d;9H)7D}e5F#qGbp@Nei387W`_5R*3ks4UFWgscw+Uh?Lg5vIpDAxd@0pDg_o
zehthXJAtM84Xp$C{F93kOZ`gt24yU*bbqpFtRzmhCLc#M#Zgb*JmgeD40p!>Uf-GO
z)0BU3xplZ3tn#@fPg6lO$_|;MvwOD4<9pqiMON}}tk;ik;b$RA-h4jVy47R$5d>BP
zW!<41_>1aMyk~!W+uyw~e+hp%$C@~?Di-LA81C;^<2J2T`5Sirzup4D!un`OsizE~
z3*11!g>Sncf`P;SN2j137kN}KwWezzuTC^hww0rP@&uNBp{{@T@qT<C8w0Wp2BgtV
zXP@xr;(y!I)=Lpw@_%5FX9Z$L$!z-@6h6?vx>2Bh+f!aa+J-;yzW7-PzV7L~g;^2o
zY0dXtc<$_zdiGB)mxt`B_ci@##C7WZF1@`zv>SiqHc(+8Ckx+Z0CW*9O*nt~0~6si
zTBT$O+<z>*upKg@$im4r`|cs)s`u?tsTok@y?@-Rzk4d!z8)<D><A2GPfvc|Q}#Tr
zhkvlAVHwD}8JFlh4?F*azZ|`1XFaGk%+^ER`}nVv^#A^!lQtzHZsBieCvoxkW8Bcb
zU5y(5g$%*}{^YM;-T(Te2i6Imh)8(ReLGsD92EmRf3gjPXOX+}pj0D3&i%RX5_e8b
zAolMr{g3aH5JvRnPud*xrHkN^xj%8<`Nhx>i3{UT!iO(;@_kpIasA12vT}XZ<dfoP
zCxt+hf4`O`S;^^Be{i{VSa#6ln=PiApvil_pL%JJ+p`{jBE3m)63pEZ-KK~bKv=ST
ze7&9{zZQW1Qno)iHOyblUkh#;a>QliV@5LzA$sC`LPPN<S8HNKp0k-z$PG4t-}Jk5
z6cwq`{lT%q)fN1@2QTvGN#SPDzMq7uPueJda&KWzzn)TznJ3)mAiD!50$I4$^WIz{
ze{gfNB471*@7oWma1Mkqzn@a_D49Q4IHzgauV;QnVJaEUJkfVc&mcr@`P&50|MI)P
zu3O}zWqQQ%PNn&F<Wo=GnaB8(g=;$hbyMR7X^~C+^nFv;PQ`xBWc|}CCg)UDI3G(~
zT=8`yknUGIimbhpCis_r_VVuk^f0i}I*DE`P7v-5Zu+h;o-jd+oNKHvxS4tv9$Ee4
z<0W$goMw?^De7;hJP?Umaw>!q{pI}=YMB42pv1UAny<T7{8aDFucxjGaqb_ffsZ}`
z1?hi$wRwmW0wM!YzIoxvMc|>Iyy!i0*o`flmHvMS#IN@#EDAm^wiF?$@$E)n#mb<E
zH)&;lkmaB1I0-8!#XD(wy2aIbOU(Uwb8B9DL#9r8-QzsnB_rp$Qy1{NbG-KYZA0}2
z^+WY^^M_gU?i(wzrm<peJn(paGU+B}!zr)1-hG3!OkQ(Sm6?mE2VVolWV*u*rkPvy
z19F?GnR2h*gaq2=AKWWorITYH=s%Wgc9SmG%!V#kUBMt%-NqpIv9MCEx}H+5h3WBJ
zL$%|%lSAU#ox2&@ogB&|3;K9K(OJ(!2dPhJFaF1lWz%H|OgFZ4!<?4XIeXQTv^}>^
zIeo8^YEpBBpVIDxCh>oj<Qqo=o~@1y_qOaZ+J=~1d(hCbS5&BAOz==&4O!YY6v(p!
z`|UrCqnvUuFHrT#<a8R(vDeRku3^`%d}GvFa6~EeUi!gMH~*l|%kNS_<5Trn%iBh)
z$!xVzS+s=(hk>m*DCpI2kre413wOq~mV*P3B3$$z&G6sx75T~0WlB4ZXN(zIe8ly2
z_Ln9Ll0*xibNfw7e6=uP{#6|2_rgr$gF{T?iNZ~$IZtw}={pL^O;XNgN#GmIE+?4X
z{)b=M@%U(JuWq5(5YhDVS)<lYojeY5?k*m&RWD9&#HmbXbh<glrQOo&(aPJ;$Sz70
z%<(2kF}iczQs|N+E7-Bo5Z-?r+{h9nTf~nLvAb%w3%m=FcD24@kS>bDCd-pOkA@ti
zm{gPp_9b1z4_9ui))pn!(@HytCGy3;#*n$ib3QxCTb$bU>96(JI_*mf=kjYlA@M(G
z^_U9O=Qw9n9vrXj{C=zat(X@^r0Lg7-|8E%hhPe2sDD{_S29hWO!@6pPT@YX`OIwo
z_HHH11IO%~$q$#dNtDf|!b}f?WKa0F$Nr{*(v5q&<jp9n_Y=h*oU4wqqasFS@5<IW
zZNIA{8X?M0QR=i~V#l!cjoM0WPj)AgmG=;us|+0b`uLwP9<x{{Zr8LEhrJ~T3bMcr
z{Ni~zMat0dcG_9A-#8miq{Q&uDigA=Y=0C1)uSSMWf<Qd3j!EV{x04+^@?!TJl5$o
zQSOj4k;54wvsqZ*p4;;L!-J1?)3caFVS#k+JPBrwLd0^I1sITIA|O&jT!de#z+@(;
z1hRRyd$Os{Kkf=1N*$|;7tM2pzwU=$+bO}$-Nk`JZz#iMd*q{R%aigU%~8i;cMoE&
zY3IUwA$P|f=vC}$CWi7DB!+S{FpW9ncG2H|A954ZOF_L~p`<oMOO$-4J>+CFd2Thk
zic0oykyi+L1Jedyu36%Uhkb9+KGWAN?g{4iAgccuoqa%RcuSAeL|0cWU_Ql>L5ZY0
zRTi^5HJ>ScDDR4q;i69I2E(0FY=%3o-t`u<?^NyN4b<+<jjEPx8eZY;H@Q;IVWhWK
zd~f#3dS~Kv;YCLs_~WBQTl>cwo2_3~;F#AAWdVQfuh#|r>$*r@zvT4ey11dZu&apC
zd%FB<U5<TSm&Y8R49PAWINZ6IYr{8WTiiIF?|7NR{~|pbBX4O-*}*gh$MvMfxPcR2
zzs|`IHNZ3X&8A{lMQI*8&%#i>;riYOzOm7eis6WziKE&TVz>;NH2pdjXLmG5EWrAA
zdCC9If7O*i?$k?>$?B(2FBBC+uPiKu|K7h2lN74oBCYnGaa=;!T)nkb^kQ^QX<41^
z(}Q@8;IJJb1}lHP&-YZ1eUm0O$`}em2MaT$-1c#!ktkz~^e$JE%qs#oArB<X#JH}-
zW2Rr6dj+!PZ(K|I*JAv*Qt&_ic#I1pswO?|+2%hka^r=U`78QK)$-_nt$icqOQ0!h
zw)VGLPzvGmp(}e822!$GWhC0d;3%(ILA#8;PlD|s`)#L^Up(;>_Tein=bC(0kfimR
z=;(p~Z-2zLBc`XHNbZS52O*uo<8y>aJ#1Xt@Tcwj;YMX-pODSVPHF4>@IOMTs?CkK
zfq}1z3_Oxc7bKdj{htTb-?z29#iF{zeze#43rlm0Y^9>^wXov2m<3XeDd30go(Duo
zz3FL$J~$j|^=v|H7^6BsL4eTl>-EWNV6DPHsPMv3+t=@z2M#$&9er15vQ)1I@W&Py
z+tmsK8Km`n4So1q(b5VfMa}EA$nPACC=$!OO7XeOHgjetT_JS*d{#jF$ETO-nvNp1
zQCwoWU+Xa>Gx7eI-FoZb`5(L#7mVfdatWrs?|Et`5sZM(Ba<!7JRt(&`oYDv%|Tq#
z^&Ui_b|&`?31pc~`gG|wmd3`eLaTQYKvs>(i5OK39ky!DgaOh%r*k$ys{z_VSf!Kh
zFd<V=jMo8b*LeTYxJ$8~5N>iFKQza-LI+6;!pS?-3*$p{kbdx)B$DefDVTX#<bAsz
z1FW<gT^7XV*az)0iyhb0S7%m<k|-Q#qdTD%dZQDLC!tuw^B;ESM^nXbiCXkVg<NOj
z2V25Vi}Kin@urW_ckdx)7^TjEV^D9bub?nYJoNQyT?5?$!588XYnArp7yB)V_pIt*
z7-{$sY);B7f9QY?SGF3Yx{}#P?eL{LI?VU{tz)VA>U9|LDguC+I1|#2iV}#=3G)eC
zPs-iLqj7laM<s;LAtI&09}|Vj{;n*MM9uD@k{$0*pi=$#(}@Z;IR0YyO9$FN*4|K%
z&jD|eia_c=j>y0Mlu$TqoF&&6-1UR&L*sgAY^XW*Hp%xzkhtsw+CQ!0o^$ckU|Uji
zDpqLfN*h^oE7z^Z2HK9r%-vt|bGo3Zll0^I&Gj%ggP>2N<#y>ZOgR*@c$2}EyIP|;
z(5e|0{J??B<QfZZ^0O)cb{If!PmtUi^jq?-!i1seR+Gytw(GN6Fqh3B7Y2KU1u-av
z0f2rOdgd-pr@V?fH=gWv{CeRc*0j4FcbHe9Zs4ce6``eF5k+&4S>{%^+UwP?#j`9#
z1d1Rn0vE3RHc5Gju?lOWi7fZW(w)f3_jYzEPYr#(ugnx7&0482*n{2e3UTeyNF|-r
z`q$k-p{owN>D$Im{(5`MB7lc^NETtY3dRI!bp`xb6bIc#KJ+F}18XV!!cbv*x_OuI
z#ZaKTJkJryY6g^U;Tqmt)LGD^E!_fm0KyPC0|MvcIP8FUL6<Su{ye1(z{4xhQ;ON5
z6Ss0SV42l&wkJnMaLEVIa-G8<&|0*a0XM|2ZEKnXuIjkEe2GQMkVb3VO%IgSIeT?L
zXIR<!z5{-ekqIAem2wWtQRp${W`XM`%ZBu$qH<dps$D+Q-D$`|Iixe$>>nJ;;+R)8
zP-r|U@lnS@^K1)!DF}?##l)|E^IwYc2MZokhBB<mn0Ybrhg7Bn`r(Q~xf_AZf1Ln%
zryG^w^Xu{K(;OTYEqOfaGUi_nW4eo{b^?PEGIqLZAK^F;Pq4M4qoWmi%d|hfHQ~bn
z#PQ*d_vW_%YGWAX@I|jX{uYjPB*4MKkSVc-!noNymrbt&WCv-1ZpCp}K2n$vNGDgq
zJAhEyQw6n52jYJ0&y2PKL{sy<ytY*!c~&m??9QMYD0`0yY4wG{MjlGF?kDM50BJl<
z9YUz2Ej=vW?}^a0r2*wZXlmud@f+q_<F4Uf$sxa_5i-W8_!kVQV_LgFY#0uNU#t#O
z%$Ft2TJv$Nnug6=>jWy;eD>`1G_m}y&4JHIQH;ex#|TDX%4_g<Woik(LmUI5A2hy)
zcwg;H>Nzk1saJB53?34dHquBmTKg9)D26b#TQiN=u_pxWgC=QxX+IM__#oqsGKZc^
zGSIqA$EX-|J%=f3wI_g*AxrVB8{4}q7)hWFI7Vgw(P~e(go>p$5R(9x-kX*%w&ut1
zbcc^q>)%2EU>Am~txOYZp96w21iIOq7OtmSP0~Fept^P+aA8N6ZVTTQIZ+0<c#P2?
zto@72FzhZA>3w2Bh`jInO&$P7ULHA<&)p(h73Y#1RUFnoiL6eLS1^eZ%~&f*H|khS
zc|Z570loU{X2|ctJN-Lg71^wABxC<zJn%rT`tYj}efJD#I&uP;=z{uH>zeK--lzya
zZKrK#{&{;r+91m?R6~5ouEJ(!{bb5qQaUa`$Bh8i!n4{$msx8J%E)>Z`s%~(rO&&q
zVF9_!7P}UqlRMhlY;QBd_L8lY36RaNJd5<m#J(fl`))90W*7!gw()tu@GpSAV6_0A
z>=X6Owsoe;<0zYHl2sTl6$(14#Lezigqgbn!0-27ldNaO*<cdUqqXRqE9KXq?n@3L
zGaBt0-qO|SE)nzS5%DRp@GahJ_xRjLH_5_nqM`kNG(=HKP=JCCZt#`*!Cg#F00m8#
zB0Bg@K?mGbMPj18V})!JLJaKP6iV9ng8EiOwlCeA_RDeL_|%o|Lb78f*X<CvcC5O8
z&o-9CuI58d59{UZX;>gu0ApIsUxP7C`pDSjvMC_A^t}>af3$kZ35-z~%!loLl%6sQ
zge@)a-#DA&iU!e}whtkL7go!Bp`o<yV;F*rx5*urTYto&Ui+fzDs>JftChN!zU|iC
zccE&;*0<Bw;+9UEZ&|IjaF{<W@6+7#6dyTE7a$91og6c%-z9G)9FT_}3&8);Iu#<;
zi9rM@6@#*t5mI&KRSM@D9(l{KyXtdq05zOSyBb35r#5edG6M(g)<s&sj7&azP{469
z!hck$bxkKIG#6V*HG-MR#UJA;$+}Q346hk{_UfbOE1-c40iemF*E1@@X>TR3bP1wX
zFCmQ{2en237o7(57(0XFE+gJbj59X#M1#-Wv@%;V1Q>5V_mR60nP`$kLVG8M2;YDZ
zMQ(KH?b=YjrT&yjvAS>Z6wVtJgB%B;S43?XT1fsXErRg*zwGeOv3|e<I1W=BiGn})
zuT2b~AIv<YD83zsK)iuCN4nKu^vEMFj4G2M!khF=6Z<8#WqcYpRTZQo8$XXESdwYf
z@0GrW{Xe^MMULLO#k91A<4h}kv>0bf)dMH!bFR6%-Xp!Qu=ofFep{ijn^?DJF`Y0_
zKKIQrvJiv>Qi*EC5+HcffUpz2-%$AwR)GcHXKmaE{^x{up+SG!Y2YE%2F46_AR%Q&
zyfXxpfPkS}VE*%nW3d;$5bz(L$%UCt)M+aR#3^s~5!g@9dfw9}>0(lV=%@h`5Bm8+
zV8!V`u+v|y=d9NR@YY!O>kSp84|PAsM}cN@hW>p79s?65{9%GrTNXEPKo950Ji*89
zIdn}=w>LF#8`|^3l(azQO+#DkTQP$KXvdp=Mj(ELL8*WhnQk#lq6x%?+Kn&IYQy{$
zRv4$~Wk=6_e3n<R2ATX<D2G>96~SfRvi7OF6#$o7(`}J+mor@sT*i*nUHCd<N*fs8
zJ$NIa<G%_~rHh{-Z577S##w_5xDmRHfLpc!9?}XPtGaR7ijX;U3C{$czeVy23mI#I
zz~W#=*3|gsG~lFL4m6gozK>7tW8Y8mdntrPDRLZ>QY=w9QDf)O(d$JoUc|+qxVuol
zXgn%ONJyCA9A}Xr|B<pw67`da?>XF;xVJNRA~)yT=36$ztT@6ax%L|)9NKnutaTzn
zXRcnn8<o(}A+c>U*?t=HZd_a(&E?D9(ARqtkSrJXA5ZFRE!W`a!BWf;VpCkbjYY~T
z(Y<l5tg^+GuSVpBn_Da}(b5Ws4-E{+>vtuf_ogca!|cTKH(gtt9i>D1yAnksH)1XB
zms%Og4*Q3NiNMSlDffBfGd7cWsj{zw6tCHTv1n>(k?9tB#wX37nD<h6AN%u++mVC)
z?9#j4Nn%q=YL6ZjU46c5*I2o-v$5d6etK_wOscv4Jq#6024GhXiQ@s#NLrREYieqC
z03y(!@G9fcK>cH#PZv4~2@Q!92WOYGA7fi)g!6}zOiRhNbxyx~vF(278C%6UFH`<Q
zs6YPfvu8^D#Rc%=|M@9{Q6D<<2EXq(@zeVs-#*m8aP65_?DscQ;-IFEQmH)KqpCN?
zIqvW_VJf#vxBvN_rj-cq+QmnsmmW^NOE8*|u(cLS=%!7J@E3~47By9DkHiRyLveT$
z{a|gTs@e@ES9YeB!hEf6aALHsVP1%fufuFjBJNe7%u#)VqD*GxC!en$c)4BW>CNN8
zmz8tm3<KNGd4thS-F9+vl6qfdPNQ6s7iK~d%3(r*MR8T${W^Vf0ILV)R-N@?w*mx1
z#Ja#l)0X=7GZq=U1h`9@y8s^v+TpSIDDk>r_uMXwvEWxKFpL%QB2@^?<!J!lQjID@
z&DZtb)-n+6bO7IvR<BC<8^(9J@<Qm)cz=;-Mz;p$o?IU$b_L1@NeCbfF~D7zrJEfo
z<xrp?OJpQEpQSsuyS*xMufmw2@<=gW-*WcHj#%fBJyo?9fLz{=zvYtzrUVbO((tH~
zM((n6rH-OCk&+QtCIBVbUAQ)8l^i&t!LZT}tH-|Jy+}-Cp8XLa(6!k9tYxQc#}>^O
ztz2Ru*`SIy0+8{Zy!b{~#6(V^lvNnz(*YR4zy#yw76ndrwQ5w_%Uvh+@#!yxa$8=?
zet(gcR#;k{ytkz6ex-x*UU`0oUfa0+#8NoEJdK9vlOk~WW(RX$r3Jlw$xkI36o4W?
zG_bssmtNJd3INaGXwe0eCk2;1X5}9z?6w&+-}f@N+o!jqE4voUY3cB0V^f4@dS}m1
zw_7APr#{d;uYmVM#o0q@UM8Fxgt&(e|MsVm?eMT_I~zAE>QC>9#;dX{ZI{rX{Qkj4
z0;in9qh0O-R1q7dQ#Y`)V<sC+x<B<{4K1D)=Xu+Jk#Rr#aSpxvq6x`|he7kx-T4kE
zBc7Y|M?@tLSDwy5lijK_w7M-hd@BH5|G}LZ;sO}_VuOLs)W%oo!bVU3KDb%g=I~>P
zoV!eK4l*_~@YE$FCMswT-6=hvrWSkW^}TdWK7e`zij&cQ$UXsNIfaB*mOb`1P%zKS
z_rzk&f2a}X6j=_)dQN^rp0moyg#tW7AMuzN&j!ZXT>m<l$q=KdU^HkdF!xeJ_Khn^
z(AZ(64@5fMYMm`GCq}=Jv~)0Q0<CnN6QiJB+Z45QuL%G1r8Jn=jgNKe3O{!rCz(&=
zgDvxUZ`?M8=!_tQ&J2cD+UIlg@G2LZ-4x?-&ziu-tmK@WoWwnIFj8u*#HOZ3Q7V)z
z?eW@aBx}oDRdg+GEnBCZLVvg@`;M)We9hBxpxfx8k|grjn&<M!etD6JNpjHTzEL?t
znj7s&{$}i;n5>5l8EiX+T3zAFtver@xVYtbJcVq|9a0mET8?ExdvfTvKaZjh8E4r=
zGl8%B!`3^M;a6GOsi4R4e*fUsbN)wyhtdy96Ple9Uy<n6<eX6C;_(tcL&r(P-j*SD
zP6I`^bp@aB^oK32_iUJ5pC$_Zl&7ckZz9_&xDSH)`Z5DxHG>#~>GlxS4A@y5D%uAk
zFk`P&aV#h+35X^%eU6mA){SKPJS$(rm`W$kM!&3T9Z*!nnJSftVLBG%q)Z7etad0v
zH$9QMHHlKJOp25R1hT0&i%<5^kl!zQc)cwbPVFVpM9;!`TwrY_p`})R<AdLqp)3lv
zUdGRC4pYsGnX;1J5r#w|WWa{p>RDIZ_!xUrWxKh|vj`+)WC*o%W)7?eKl7-H4#&-B
zmNQ_U%DaKHf5*I1<PiWy+-}mf4f_~+wMNj_NsCl!H3ucW%Qv8VdzGgznLP9y{m1ck
z>N9*%u3DTA-#-!KplY^_5?`GW54|fS^xWdmhow@wv**q|-8$o%mPTJ1M`e}&VT{G7
zv85#eRF23+IxExh_VRO=WM+*<%M`e!2E%<apRrv%W7V!EdEz(`Ubg?q1EJ3=nw<$+
zx<dsFE4vIwR=Le4UuYHVD$6q4IJQUfyt<HHaFzI)<0F=`byHN!d82*%JhprHlAO3l
zBl(3nyrPNiW~hyM%qDLuDO0u|N2N?#Ijni&Fbd_N-~QN~1rN_A*0tcL_q-=VDKg%~
z((y$3{y`fmTuO@zN|^!pjA5lPr-z3{uWsJr-HP<FsmTMoZEWkR!F=JJj96B)u$4NU
zTjc8w%hZv)<J*z>geS9J*F!4o#e=NrtjF7Cv+wd>bF*2UC1E!Vxs%%w57w32^cjPi
z*Si6sNdQa51FcgEl?}4%_K>GQhUa!x9_+u?6h7RPrjSd+&VH|z=h1SEg{LITea|eq
zqon~fXBn%*5=Dc%U=*#uZr46CZi{M7O%4g2LySkugj>Vd&J!`ydSBi3bByG+4A5&#
zho#^3a{z?zrw?@=pKsa|oyh7CK%ak9J2ii~dK<V+UuH0fYy+C;?9io8yYCbojvKlu
z4M&pkpr<S5ix?F+9ylO`hFW2Df-p-f@G>n>OZi1h5`D-@_EL5SC{|p7IT4_uJa)Y=
zt%KCZuhooU?q6Qr<wSjPW*%f6-$U$@Tt=FVztaBYG^IlE;F(rb$Dj<A@>J|Ml>&hF
zlIxCov0MQ}0kZwQU)JXmyRk6W10*u;b17!2I_)}KvMJM=A+~yweeyAFnq+-6mpOIz
z1mVGRnsFdCU|i8D7evoBKV7Pquh%V<^1-5cOpi$1?0rSPJ(tnE3x@0xM%>ZLTF!-S
z$IJ`Dv6tm6th}w>kz9J@0xpZn8+Y;RKf5eHxhYlV;HHe2({~d5q<bED9Lg9fze%+B
z^qWq4GK0RmEGO)KX?!51t~1$`*!*Lujo#W`wMEIvr7$I@`28};>G^k>EfKf%dzpI1
zJ6tz3yQ;9=u;(TVI;3Q(uusq*dLLZJ1xmT{Dj)}T&3G$oRT4!`Dr|u2Z4UMuWV(UA
zPnkx-rXMg8oQQaPFkUb^;FFZCEw_uSt3R(bdVbu^TIp(1if9*%cfsq9I}04@Go(~m
z@1H(Ul|#W~moIwrrmuOv0>95<X;k?l3JHhtX|GdEH$tgR73a1(U{*%hHTx39(L6nA
zb>Hg_$D5a)a6Dscx)#Of7zJdmDLhdk3dRMaxP4hgPx;2)E#)&8hJ>85Y1v{)dgevO
zAqBo<U_yW4d_lFijL1N-`Kvba+i%;Otu_}&dJ=>Q@;zYa=i3SGIqRp6#0rfKW5=no
zw$kB$B2tCx`B~FIvlgf>JaXX<I#EADU!;$BfIVoIBS|MOxGN@K(;Z&I?mo+go*j~M
zt=LlRn3uy@VQp@nTXRb1rsz;``?lO!>kt2^`~Ty2r~CMjfV}@yfaM4Ka2R*_?BV-`
zJGku6QNF$3=_<awZ47E)<$2;DHlhjb1Tx);&nL#!{e@n6KN{_=Q|B!713OW{BlT1y
z*t|uxiHA1tv38i!M2X^{l*~iW;$4ujO6-V-A%8jH;EQ*FdzPq#QCXXTYE`IK=ob(`
zNF`cwpC;}!Tle+aZf~|*-lu)BSrrr%tZwBKX2_(<xLv+qq*_>!j&c{>U}ItE7OZU_
z)*EB$mWU&m;qqJYSddqY1(dj)cJ@}Re75>K_0NG>Nr(-ZsK>ILhePz~89N-@I3U<b
znW9e^%r^)Q)8-%(mBY>NaK(ddEqun-G7HoJ3E)=R7%p99NK@m8gfuW7&)GptfRBc>
zdAV$Q@L^V91XYvXIu|tDLMt(lOm*?1w7h(XVN_VyrO2SI!ZsDv%|!o;cV{bJ&yU7g
zzL0$PMM}P=vY);smkRB}_HsqeDj|#P4Nc92VMnS3-8rA?uvaP=@z*CjXD-{BS5gEE
z-0c&7S=ZLqx$%X+!Wx}lenPq8Bl%C!z)wf<5bEbUkiuK_#3ld96L`{({=Q^n4k*;W
zAJIpiRT1f(GP&Ye4PPd_u@~ABPUd!f3XW!`8LB9#j@RIxZRyN^JHs+cMMTVnIYLU^
z5s1mvmO7O>3@1^X7Bnyq!~G%&Zq|pdV5CEw)xx_@plT4Gcc^lDfUOfNC?gYKk?R}!
z;jRNuAB^E9W3~CLsm0A+s5p)xmu2xRd$93bw{$gBX65ib&gYGT!xaZ%l#+I~fW-Q2
zZ+;!?!}t7Q%v$a(&5UYT9x3O}owHa~k8PQ54ez~5q_QHqBa*C;OT7mKPa=NqVOiSX
z($_Zh`M>F|4uc${a3QIV`Q$`ngEabv;02N5i-n^^eLcyN0-MECW7R6<wp=4e7fh(-
z?m37ukinKrG%aQK9rl<3+9e}ytND<rJfoyNN%<OeW$U)r-ff+V4GiIjUVdVOL-YuX
zf&18h9Gsv2Y{#v_vTA=Q$oR7b6+<j2welL)cMBRtbw$gPu-Am>!=SiWxpHZMz*wtj
zeEx&P?$$@na`g`rrEm39D>>Rl3$=S%474Vi5BBU;go7?j?2wW-KiJ#s&aB)Qfr#nK
zis;7afx0;T-qc!gMsQLBZJ+=^X|+M4kaMA~p`rQe{zQ|<u)Y@FN}dK*yB#paygQrr
z0gfSs7X`0kYr9uf4eUgn2eCAO^m>XPP6B=}0l?vnWsnx=0Md+fI-LzD9t7HKQ>6*t
zzWuU6g@?&187Q?p!el}Ha$7G(gcT6dPd+bw7kd0WxX~t@efPlwc5oeVYDBh#lqfNq
zvsFR9+?EYSc1ot-12>u5;^WJ(4ei>YY6Q`X=OGrb495+bp%<7i!qDHnKf0Q&HXfn1
zoR}rF&084Z%-(vAmuQamO{B&rpK8)kpSsHW4x>Ql0Og!w@>MB(qy5VQ)-CjOH{>;h
zYpwOr3?4U-o_G37+<!U@-~P#;fu?)LUET`(9ous!>F)LDyy$5S=b)vfeS7H)$<Lc7
zjBMTov9+V$H*b_``AxrX&#tdTCbnOwT8tMRv+mS9>z7p@!%T0j*XFQwI`|Ep<|Zyf
z`bpD9_oKu$9*E!Tj`Egw#~5&V{$Ljy10&9Zm{AHUD^oA&@96;u*HcGZTgnCF<VP^>
zQo82rv$B?o+n{!cy<qKAR<lpqfX4p>1)O^xX6KimWF4y}T|c6EUY)j0^=uOoDaUJ%
z=763;AojX|5*EK$H+E*9f;JWi`e8WUHL6d&>4{m|Gi{Nzl^RhPKsA;KrP-7bI4)k`
ze<UR>+$<VJ0$BA$9UwN)S9IHg8TTa?dXdYLq0Ux3#>1SXFgYho_S7@B_@rABrLWvm
zb~Ndj4<=;7cGNCCV=H*Sb)DiFo5MrgD@T7aAV0+$hjAYYR&@$IL^EiaBxTbVtXw<@
zq^XLyr5Q8TEGGfcUVi8c=O4H2Nduy~CT@40{Y!OuRq;F!?5G(zUAa5^_6^QbbKDe5
zx9$!ONkHV(+o*-jsgLh2r1u=}sdzoH7tPJ=64bh!*6d<c0Ge8S)noQFkO&AuajWlM
z@CutjA1>x`&aCC>&r{9JZH}Ln<fY4PExFC7J-N>);h`C%r95+Wl@<|kS45oE*LKD8
zaW%%=r{BxR$K{Y@6ZS1z3)2Z^57H2aMiKQvZq5~9akZcZ2s+gQ3*G|^19c4z+RK;4
zBPRe-?l$uB>PdPRHW;CZmj0k}RB!YC(yM3(skv=uh*ImeN@r;Zh1A6Q9n@Ht9qY&!
z*p1|a#1HJKNUBL)13qWsR)_6s`1t-$SBwh9o8qa#nRCTG3THEF>8=`&<|&-)J5^h8
z3{9=I-YdLEmH~5RQKSDytMNG$9JST$L#mIyZ`(uXDuXkGuP)oy4C{`ErXHxgjg?*P
z(p!-GWHB*zy4^zm6){LS71?7!%QI*N-CZHPX*%Z?-o5;kC9Z<O;KBxEG?WVi*#cIB
zG1rf9LvSR!S@sAUlaZ5OUZZvu=#D#vi_N{zFFF<)4&DGW?3WMa^gFz^YaRKjD;OT~
zoDrO+PZuk8jFq>a96q88xjkma^XHx2Pj-9cE|$z)=62XEdN&O`1+maVASyQG&M&{}
z;XdWTSV5^B5v)p^X4;EhKf>e@m-U>4ikeyq>}zdcRFwGcHruH!VZl-7#H{J;28wQU
z?RhWNPwGe2ym+NX+fDkeJ)?e1F!hj<=v~E^Ee14$(^5^9H_!||`|t|=Pn92x#%YU&
zt>#5?mXC0>Oe+_+J4?&!`mT<r(Qpx7H7N^q?T^b{CJ5;}6%-HS?;BZ#iTVW_=atSF
z+UAY+49if^C#_>z9ZiFuEI!0o(;J>v4)1wvaZWawaPnbJ`I$0dox7SmY5u`CZdH>m
zZZ`U!mjSC~iDt#8%7j(f(ssC-^gO8}QQw`YtD(q9`3b>=T}U$1E0^2KGPr1X>A0}n
z<y|LCm(RX2HJ9?@l?xlBq7xDNiBD^g^S%xPz5Nv1gT2|VeuR<ODPSyTw|03FEVCss
z4F!SAU%ZU-lxtodD;!?Jkun?U^tTzjioU0pW*{2tF|)(VXt*Ds+P()}i@ddEoWn(?
z&LV-A5?QlL_h63Ni=)>>R3#<N%yRr>3L)*C06FT^PjR#I((4(Nip!#&O>gc^e%>~;
zE$I|PGcbI!;(Qg&VD~+xD#lL=`9I><_b;BryS~Tm3wbNt5Y`$lcns5^1(tyY^XGNH
z1?xVl;Su%Mx_6_NRFVp1mdOowk4&dl##d^0x-01A85teZvgx3#@5~n-xqL?YEwdB$
z%+RMr)Xb7uPJ+H=j}0i;O3XAeh8R3ZcFn4PQ6KHX1|-072<hJ}Q{G9v&bXuCSBfz_
z%}Ub6Fo2xDz<k~Z`ehv#J{D*dPQ-xZh`6w=ee=0Q7+d^$LRUGj=MInW_nc@?7bxQ{
z=r2f1D)yYCQ3JlQRObLc9fRieyfz8NO-3X8^sMK~Rs%-2@Gj}k=}o{wQ)#E6q8Sk4
z7Zg5LQMNC8ws1rGa%u0%c5f#aY5U&kb8Gr%A&RT>X!~EPgK`Op;@F-GskLfLYt?1f
z^OJEHKh^Lecy{*e*;<@oQg*}Jkmt47T2@o6bl{zuoaEGVQ~y!XyMP)wi9hgHxaWwE
z#z4f6*Lj%mBmsY96rGUrj>~qhU}nQq$L#i}JlpHsBSk@*cFf)xViX_O(OpjBJS;j`
zpc{+HBXq9ewxPA4PAdiuRXy+rXK}%$iGdW_YcTAdG`qs`8a8*V7Keo&N07UKtI`3H
zeeyl0uj@S!I}5XvsfZpAV1iHc3WO|KoGv9Lg%A&KJj<)R!YsT!gByw+1@Csj<BW%i
zi+9C_JeK!1>Etq!&1Y<>L;^av5NFXDDz1$AnET}heQBX#1;pdFVqvV39?<?Hwz9GU
z_7TE#ZwiMkH7a2+BMFF=)@#09Y&P`~YzZN*>NUBo_$+F^YsZ7q0$8KvI3nm$E*Kea
zmo?4E-I~z_>5f|!rhdkD?TzR4AJxNu{!Y1x`q;*qm!94e;qb@6!b$`}@Q(Glesez4
ziwxUp_<WQz$1}ii`cF*VZ0t;xrF)m7g}D;Xu^La|_qt5)%qgzWpa%Et!}lQbl_HfW
z0`(d}Q&Urml@B+IqbbO0kC4L)9z^z~FG$(-ljpq7Xcy>pKSweT&9_i~2sNtM4?j>T
z9Ev(@_Far(mhsWB&$_?Bh`D{%)6Ek-0ENUmS3+ny^h1!|WjgCylrGP7mx#7GUbWmg
zVx$~+olg5z_sS84;;6KE1OJ0EPGS0t4@Gl9Qy=Q$=)A$P?Fr4QR~QRA=MC99Zz6j6
z_i3Su`_L-U8FSsa$D+>tB;-#-_1`IILYoQ_ALJ{7Xhe=~+WqAi4K3fe4y7U4?hHW+
z`!z{~9=JDk|3cchFclS*aQ2Y~7G~xbbtv@DvraYzF^FCikLYm2qD-R=t0vul3`KVj
zU>4V%i$Avt0>n$oYn!L&u>AhIen*T`*v8ES7>JPvX`aY|Nw{vm{Y9fzXGfngn;G)_
z7m=z^S4v9as%{rG9xjS?aO7NayBO7heuC>R8Mo!EAmpm0a^=gUV{aT~OZP&EZP%c{
zovbjAQGesC-8fW1l7PQXNNyeyS7#iyoPcJR$<lb&Q-DrH3iI*l@zvRaA}Q=!Q>&Mv
zACO1JpG`A;(ec^^f{~Mq+{b^4K7KrpxQCp|2$rX@tcV`Nd_W(_UETs>s3073&h}F$
zK3o?^iM(p#2l=*}5A4@ZYH4vfhEy8#U+%omXtJ%<fK7Vo-kHfNbuZwyjf9boco^6i
z_hhf=>FHVQZkcW`&v)PDv-}v;1@p#B;0$RwmbsH>hShu1ZH`%2rtqz9&9q0q(rOMC
z&GwQCCc9)hS^<=$IgATu92zyZ-``fss8u+Tplsv2X#$s9t2*Kpc!`@7N;j9p!r8@w
zfbb|7vGhPusYdkpPEKxaAbq%!N~>?fJvlFyg$-!t(ye}m&OW5eQd3j&>3yuT%xi51
zqXJsvm@x+)ejMI4+{A>00h(JxKcu9d1U=!%9Lk@d#QyHaAN4v^soV6FIQL3D{R_7j
z*m?N%S@9?Or`V@19pkpVN2_~vICLp{EazDjwO2v<+nAOU){1^L!B&`{7IVv8vg&U)
zjy+>TGJS35n^u&{9%Yv`=-WiIu(JojkU=tiC=pA^8lO$0ErAfoCF`XoRI^LkQ7=O5
z!@mnnz@$`<Vg4|;X}I6g!UDM7$soM5SLQ>yAknnFE7;m2XuG*62FYz-4-?wK-TO}}
zOM~7>En6=RbIlH9pWg#H43eGN^o%|tlW|{xA)L!R>G5d1w(dFbZj*%lsZ+Shl1X_6
zR#yp(-s@mNFA1qw)K!IWhKc=iY1xx`Rtt*TgKwLwjqdS;dz|3xn?wnVW^>A_Z^uSU
zI`Vyqera}tP|MbzssK6sCzvvj0P%_vHf@)-n!BePpU(a@(M*rymv&!PORC0V9M{09
zuUz?XR*WPY<?K>3luu8y1#1lz9^5In<GY2_Y-C4Z{1G1J-Ms5OL4KLd6I^b`Zxo>L
z88UgewXn(OF$=zM(&YQ#bDg!Y_sEh-jGh}XaMh>m=$JevI=SL}20IAnNlQH0Gu7$q
z?;xMX)qJkrd>{C$Qv}`cx%=+7jyiLBT-L36au^!cWSR|q+1sKpVW1DKBCxG{aW{TX
zX?;CH9r(+AmZerFYQL*DN<K82bHtyH)nqWyKgNqAHfT8If4lyIC$03gmvJHkCE<Q&
z>?E@qJS)Ql2*z$FMb5D(kK%z}Y+3sF4plQ+oxgJjcnu{hnVa+mkH_U}Hd_~qCa}Zp
z-Wp@U0C$<@M|hM<-fSEuqj|O~5o9*q+0U1DMxbS5b~q1jI*I;W&2||--WX=y33^Q~
z$VprdAE_8z4?p5TXuI|?mQHTb)r^|wEe$3aZD8Ie{-D?MrEbONZUh$<8Pt(*Z~Mjp
zL>wuaZ~baB&diTOXDUhaG^(1HT{Wpyl#iR6Bv($nZ(p$+F%0bqI?Kg1_(@%4K1SU@
zl4jB4sHCEAJw)PDI!$J$Rgyi3yK_ziiPMR2_0P1gK?){uhll5ScGe_hU*u977t)mq
zk)rtahk~|=&(?ID^7N!h=)jRmZZuY*?5_HVFNB<NxDNBF-Vw|f;L8ip8)l!N_v5mh
z4aitpTeJ8Ab7WssR?b^U=j+R=7rut->0E^~V$d||8yb|X=(-_9rLx+VjM92YZGM`^
zDvgjuOIy2XjGPE%MO~@jX(gIL6alstbSHL;`4Yd-P{ZTtzj9=se9GnjqwKAts@}Tq
zVMS2FICKjF(jf>4g3>A7NSBl#B`u*KjY>Dt-AIRY3DTt~p>%_U67M=*z1Jt+XMBJE
zFpg&^p7Yszuf5h>bI#Rv;N%zkoodtN@wY9U%YL)J;^&OU93n*>>stO6qprj6VDMje
zx(nTO2<d8?cu4Ai(WkW6UV<|p4TlrxwH~)Wu=0&-yQ1#QRRLCFIWAGyVj5|^3m1rw
z6e20kFbFN_601)zes$Yl&7hTyr<Z!1hi)0MS1Kr*js2#f-e2h^AD;w&wvx8%yEkvH
zg!NX>#|-L1YudsUvJk{~@(#A^X1-jIZ=~qdI^OGj@%5xkvSZ*J-c-+pZey9JF#viF
zwTF=QhQGlzB1|<+S~9VB7t;#YV>+FP<tY6p&3pR@!+EY|>X8!x?zx|voc;!Cb2E$2
z4O)X>sz3v88NAxAm=jwMV;N|m1k|&<L@9U_ix=~RjTyJzk=O4*uxv_5&&@AS_|^A&
zARq3Ppi}ad4!vRxlOP8ta6_jebP|IKTx;C4Qc)V+8$_yfYe}?i^Or{QqIC!{)H}d$
zHnuNO*nC0OKvuIvZ~)pEktud>WMyR17Z=TRwDiy|HF#RsN}?H6rEgW#qc$$SpUr|x
zzK0PpT1A&%LZ?vDq18pf{g7zu_>qLS*wqSx2D{<*QfA<b9MoD6GcWXpGJ^?RkqK9=
zx#R0o7<0B7<yJcl3?Vv?_OST3>wI8kUq6+(YJAgj6xd;(?iegXQ|TU%H{MKnRC|dz
z14GtULxSN8T9?Pkp{Z(c#~}nq1H~bjgusd>-R-_+1#M&b8nD${!*}CsT-FgPpC&Bp
z?^Rofes4N76k-j46ILk27a@(g3m48R{~OlvuYZ(4CA)*_L{nEhMkkQ|slqB=BQ}`g
zorXYyL5lxYN{Emd6UsX-ALnN}S{!W_K5LV2!t;tINJN<O6~A3gr7C(#h>d;CP`a>m
zWIvT0u4-#Fre!Gu*<71^jN&QX5Akax(uKAwoh9>DGa6|)I3ucKBoo8qNeew!?E3Vo
z1o0o$%JS9KWKGII0kQKwftOl6R(YBJhv)R$M~2_pe?&aYD;Oo0(CTF6#%ENa)Wv_V
zsR0w8?|{>PjYao?d&0p%kxtq310gz;KQC1yl%xi0Cw>y?zjj?9iuX%}EgZ{o8}E`8
zgpotYtFOX0(yBUbY99IuVq&OnSm2>)`&v`n9K2NSe4m8nh%;nR>ELdJTNCWBuJaU%
zQx|rvv7+j)ti(#za#yXtG@AINNzzObR)`C?;~Qbsm%dzeH&=aH*f#xqeKptOxt}G{
zj2i$0&LhR{Yj+nFx@BNy@pE5})lFn>vL?~W*tcIdVc@$FQpTk0@f=F7q1B1946Jab
zo#`KYtHZ59I=ooN-@j4oZgvmicc}5L?*j`w4Jw3DwG!41?yVz3paiK|tJFwSH6=i*
zq?)T%KvGg0jdLWX(Xx}H*Vy17Wk(xV?Yxz$v%q1}$p6Ob7m`q1+wueTL&qc78!wek
z@P#$PJy+Q|{&;wQ;#@ypSN+7soP4=vW@eC;R<3S+jrqeg9#QPtP3h-#nB-WN@V{H&
zG}C9K2IN~K@+8(H@+2oG&px{*VZ0MWnp&DFs`y;7MuC(Oty_)dCAMx3B=W8`GHoT|
z(dG#;owFW1Rd+q)LcO0p%~xL}yX($V50?0)x5OSQZB+ysW~i-yI5NbHeq%8r9T=%i
z{8>rE2(R<v*tf1{c8y1`SGpsa63Q{_s7M9)InMhgdjnt~1*UJ6xsRAynEMnbwZ>*6
zv0Tu=>g<o5-+{{SGPUkZ6Y*DNh=+<IBwR-6%S!7PgHF(CrejcUOVDR?M_D&^+Z*qy
zLBqZRVvo>%m)YCO@}}A3B3_N1_eR-QR-PJeDW-2{-)!7}XV9T6k=b8LY-mf$<_ohT
z2H@UPQGX81ed!A*>V9Po`3BzS*{ISkCE4VcEX=>_`#9<Lz`Ze&u-_IB$&pP=6NSF-
zB1}(JljXmbZ!i`E3FY#4$9at%;CN@VQm(2Dl$zYr`BA7-I|%W}<SJhT6YI?y=}goh
zn(@bJz+hNrS6hZjEkRJH2qL>pD`Q1SYj%}NoIcO>>V=40J3dinkMycM8X6i63}Sfe
zYNXNq`*GS1{woaL3oc88*D){QTZmzU|E~=61(gGLtSlK=-GG#rVb+?6Q9}KFeX4fK
zoNA@ivI`ny<P|$UMj-`1e(+~!7+u1$1WQv!F-`YJ@%#5km6Vhg-+G^G{kFG0mB-RO
z##74zb7FNo)PsWpHRb)p7@0Ze7A_st+j8CTBzUt~QhHr!?)@%2E&oOexM%I(XiX3X
zg+nBMA5`w0ELFVu{5{2o%12p%{|o^a&=~*3x`Z145laiWUoXN8Q>C)%q&>cD0!TE^
zUrGf}QJMK-$y|VWTB(SfPz0DrZs>dAF?@pbuC*~SY6j?*O64Z88zBzAkxj>X1I^pV
z8<EEUj~Lr}Vevdzs-_L3NqiuSroh<|i5VVN(Y%U_OD#%ra}i0x9Jv+5#xt64t~^<0
z`a)IjXtPc2Mg@2OXu$`wfmAVvlOvZJ02RJIyF%a}<EDO}w6)u8VtXcZ^vU%bHxvrr
zK5!UOTn=+rcz92txj6#1m2^s=38{5&C|9z__nd0#3nf5O2@DD0dao)I_eAI58jZlT
z9bUvSu&kSw&ruiZAv;LG(x{eygZpf8$0hy>&tcnH4;$KJpMyK3m-x#%CDuBveO9*5
zJ-!^rD)TLb_vlO_MLKAg{s{{_w9{QHjvdq~ct5M}w;xbjU8!?_1mV@8&jeAq8kERH
zAi6fyAi6RHZ|#u{c+Svlj=f@z<Dl}ryymzrx*7~a;KOeQ%q><|?y{+9?N!>k+zxrd
zJHbI}j<5+5gaH{DeCLDQ4}9Gl=-xeLB#@@W(?O~5;YxPj4#4!$ksN37^~Gflp7wO7
z_^fwMpbRM0&$VfS(d>_czd%~3wD}EseM;iu;)Pj#g<O%0s$oGHCC+|ryhr>^<gY>t
znq2zosz|Vg#si*<u`qYmx*wJnETd7eF)~Wul{Y+DT3Xt6694hA>jNLVg)8bCx9$?N
z$0r>*ock>*=$3U)ZrDu-_;6LD`aSrsIoYo_r<ssYcP&%?M3pV|8>=BT%PJ}-qnT#k
zRwkZFOg^XebjV3H>DexF+RrmDgZe@h*TXD@&^$)T@aFQAVk;}d{jEzycvVmZS$u9t
zTx1fo%G+(a(%+YK$C4}5L>#yv=8#fyXqoq5cyy{%TI=*IS4%TBlaOCr(@$0RIP<NI
zP!Az+W6q_l*iTIQ$SW5b^~g5o5@vehnOgLb9Az!zrG3xCQj_5Vty@;A#HQF+itOl~
zJ#hC}Y%{#>bd{qtdl9Z0TpeOQrJA|5kG%HTKr3vw-6Z@Ib-8v{Ubm8u4Bhvy?=6Rc
z_kuG>_aT;1@K<qzV4E<54x(!j0eFS%s|2`aRu5MkWK^K;M@UvBiIjC@fm!cR+mpn)
zU1RH&#Yy9pM;k+nv9IR4xecTHOqBP3SWnUm{j8C{gm~RFkp1K;VTtIZKnd~;N%_FY
z*WmBI(MB^_%6JZ154@ss5>WFpP^tV8+y9L7KVN$vU7udOR#n>^SCyyE0P_a&$aA%;
zOSM`0a|Mmdtv*C)yi7x9)9^GvFjZQAsEO2gXGnbUyvO(V+>GCg-S(|5KGjvrsieVl
z``2g557xZGhPfV%6_UezwJz-kbhN%)+VYpJuAV~Z7A{xnkYm~}%_z>yx5>wzikG;t
z*CZzZt(VC4TY)Pt^6IZp+)ZeILHelBdYS*)HE)28L?$LC<~gvnY8)6~La}Oe2Zkd#
zi~)JA+K_fpH#RgeSVJ3^7+WeKw3bAoPMGr4ULt+Yj4a}xKBAxj9URWR=040H7my`#
zC^{Y%E()R8VsqgQ;nimIHumS0r3pLuV?>%)B{O=^6eP31o|f>DTr#12*@#-oajD$U
z@%9$xURR}aEbe<WC>N98J4Z)drJr#qw!VbB*o}6D;YmX1b0pp^h9uvJeU&6kJZAQk
zc&)Xd3b>(D@(+CS?oXeU;iSAL`j@!zXAD8U7ED5SUOw*zJ;nfXS>|&KQ!s)mYQiFJ
zN=;+hfVeN2Pa-l;rQ{tqmZ^{ZB{ctNO&Q??Q`fz8Wa^XK5C(PjB}T>O1HoiH>KhH`
zjb5)$x%C4|5n<t2dB1~1vx~E#!<Vb1WS%#MP8tx0s8kaBl$8=d!Ukh+nhu|zx4yIs
zrQJX%Ip1*U^tAqWD9bajLM=9o7@ur5U3mO>M4VL=rL`mw{|5mM4wa6Rw%Jq0Ux)iA
zz)ymbRAW8n5I~O%jd(mL*HVmen$Zb~!WVrL^;0=*-I`;ROwKXzqol^5IQ6s7XycBT
zF@$G^%A{areZ$KAQX>-d{4XMku>SxG%l)Nr<xf&6&@0@Hl|-(n;8h@0giCmttvjJ6
zxiSQw4`Euj(H)z!TkN{sZfXUri_3h%DP!osy5K5bB>Lk}{s$R~U*xUwt#Pe#Ga*p9
zmht95Z#Vbw{545~SH7Abg&f@4K7@Q-{=ss7CN)eXB_%~Il=frNvx+-eq|gDqaM?`T
zoJScK$&<|0sw|94D+ip&t<Z-{+HC{i?)sj_+4o8HZt0}*x*v6bX(0v>*_+8z=(2f<
z2NR{tmpMQ9v6n11=@2Z#*WFPJ5bpzn@JSTxKhLHfVsrR$E!%t`I}%@hOW^&MGBwNu
zg`u2iiOi7ag^sZXWy>iUhS9@sLHvkKb(C1*=kB&VxA)b#pRl5Gxm6mw;)j%3hO^sK
z<)})eJhw+&Uh7KtCdVNk7Q{v_PnG<cW|IBb6kku=n7>#^wme-0mK<igSW=~-W%lJQ
zp$d!tNB+d-@TEP_ef4L`#(zoYEJawN1#?&_b|EN5xoJ0u?8TlTdWq~Orf8U$?wK2`
zet2mljfEd7wc8Yr+00G{y;gPgczyLKz$xO(Wajp5K9oF{nA0aaT`Wmc$&w>CQoDM(
zePm?Rll!B{VIN%K2La(Mps{~L&SOI<8_Uu5akTu@@AB1D7{!r&$Xrf*`(cc$N-)~_
zIQ~OrSCEekyZZlm(hj1$vI;*u;j;`rk3y*sgHE3s?N5KrBY+)KKY~BaEqFpqoQ3dl
zwcOIUprHYH6X4;Y@XqcjZ?l}cKGltVdXb}VsWqM|$xO!H11HPmj?vw$qLXkD^PKbF
zKOafDF4}f9<H&DMmn_K2WJKBz(rbD0^@g`(KjE$AHvh~OAuboBvhpDa#(*ICmE`K=
zZbnoTsDiI;aJ`B%Z74pcRyh6U_D!vw)`y9bMwxWF!q%4K6G1CNconxZX$&y@lY9L6
z&Hj4r9O{*2T@1|nIPOQE^5bK~pBmyd+2kxsr{ytnanu+N@X!oL6^k#ae!dI1CnU_7
z?}yVJj|_1}HG(&yjCyHHJPFqBENR$ZU&o9LVlhqN+Q>v)-SPs?7INO1`Cj|688l7&
zbxfZ=dsaa58G;m&Ss&{O(0i_WQIb@-F<J5{x7DBzWVe<h1ph+KXt*|=v*P&q9YGp=
z*aK4UP4qIvb6-|PS|=h^B5EHFr5i@wFM`<vjD2oosyq~IsIopF6?vDXkKG@jC(}&C
z5XO#SgjW)rabtctgNz|!tM$|6cpRY@-w*_{x|ex{!%O#8IOC3m7^i|4YIzzkAew0C
zSI#?QU|ue%ihaOd|BL_3V)qKkmG}OLJN;t`9EGCEjZIC5VCwi1$l0qm5G+EJ6c~ST
zbk`1CsrY&X$=9V{KGjRgEwgvHyDs<YDoJx3)susgoj>E>zr9BDMlJ+XDAxmq=iB`s
zV~pB_?=BE}qC~iUCprJjTUbQIif7cHyY$|n1pb!@!sb_b>P5?Z1L+TWM{Ojg+;(4s
zguu*VGIZHTByQ^tLq7ZM(bjH^&65i;<E^c&{ea><lz$iU@mC)*n8leQCW&YcFup7(
zcNC%hl5rp&W!5g>w#N*RSMT^`&wp?x;GOv*kHL<~HW6g4l=7&;K}+O@+PLjRzMR*4
zP?a=nKb(F}%#s3%y7Mc+`<Q%fF^T#Pwo3?VZDYJKqE~9&nliXf?0&5*mm1N}ovJ6)
z>|yOJYhPaNOwZX!?_V-%aB{c(I{78sMA5;s2jO>dd@9TG7&STYDVf~w)QEDL4%JW^
zt-tiyb$aHHTi=dOtYUW`jEO!TElfxo>GnSehWqrcNQn2px7hv+=af#U{Ewg21COfw
z@rqjbVjF)n6u3jsqOtBP_BThPiP{46)<62%?jsSRi@d29zFtF#CZ1dEoR!fS%6G9>
z)GBB9=gwqB>V)ru2)ZplKm36~h>Ogp37~x|LfM&5`h{wt8-GTFdQSvOw=f1ODy8_y
z6=xU*2>KY_n&X8&80akrz+WT^K>DK9tVD}xFKD|}j~yngX{o5Gb3Bf};}H=_cD$f)
zT4eq5U<}l$fA1Bsr^s~(LS<g1K|jFu*5A5Y?eJ8QOy7rECVeS+mVmlaEtI|8l|2Jx
ztw3T%!Yv@j%PUq;yhePHs+BzeN^)at)3d?Lh%`(iA3(cP4Ju>uW$g9P?u?pB@_cjQ
z`^Z#ld+5&ng#C78g=g|i<cZGGaHM5;MWV>k6mj2%Wx+3bk%<P_a{rYPQi)7|!)VLd
z!RiyM+1S6AD$CS6h6+m(bWv1bQW5(5i~9Qia`FHDWq!TT4?^yI7(bW@RIv=wr`PG}
zeEes@OJd;b!u2L$(!?_o4(%kw>ONe(@m%U?T>KZ3^j@q_>Dc9)kL2Q8O_ch^FSYNN
z8_7<LYhd=9IfaFX2Z%8wp|85P%qAA!n0rQK5)*&xky0#&i5V0}!`1f)rw;h70ro|f
z)zxt*@1Jac=~61kB|^|2n!?G3dVC2Gr_6Wngm>Fz3ei)3-hqF`%PR4=%42JVA<yr_
zr<W1T#}Z_AWydrM7A6$Ch)YaP-=jg3hHhlc-yIW*BOHs9agN~zI<wI0X+&oq2LvTy
zLRIuMfBlE>FRV`L6GT?|pX$^07kCm~BGUSpFypOZQR+?L5hop#m7$n<c&vG4ch!Rs
zlr}y-I0gLLPc>*3={HS__C+cekMtUxKG|}z{mxzmJ+QM?Zu%2Ee3AKDe~8i6k=E|!
zUBBC7vi&-QRd%5jL*%9Pl5mT&HH(e?`wIL?xR}O(ysYUe8JSFuc@7>KUVglqnpzaz
z^YpB&m2La@VuxbI@3yS@hrV&QPG4jfc1i`a)Yj7HzY67w=f*JbKe^V9Y)pX0tDAGX
zJR~S6ZG5~m^~xNPu&^jY7sFMcSbR|{)E1JzJ2Am8j~iNH^&zk{5|5lbgY<M*FDqNE
z;Qst~>vnF82T&7W7vQaTM=gQ`It5@Z&1#%#Uew%w=wpJfDB*tCqdJSlhk-(i!tDR5
znFy|C4h(%2gW$evv@fZ|?qj!aR*1%)!NeFjJ*9a2)Jp(YzeR5Ze_kz=*V-;vQQKgg
zT|9setLY+NxAu#hqx%cTO#b$6`4a3T+}E3_l^5n}J<M<S62&vVh?cnBYeZM58(+2j
zvCLD8wtj>~@grULTx&nsuiO!n=d8!YZ(XJIHATL~jSe{TaH+dcWGq`$yM~T(2{*4G
zB^tL4FaG(s|Kqg_^>i2MDl<opT0vaGOU1heM|=<75xD}P^PwCk2u|RdmQhrEwcX_6
zB~@hyJVL{0qMp~dqyFocX3w5$T>he>vD0(e;)DVsP*!VL<e7;+RRdW-K$qTbG{)1g
zWoiaAlwRU>=vb>9`q~rClC|s@H)ZTBP~H?C%JmV7F&%RNua-}kjMgT~;^R2Yb7@9c
zJnzDSW@JtUz?8J!)$_(j^cq{px~=2@Y@q&$w{O{hR<l<zGALi3q8@l+1^H^-ZjDJW
zD8I~3G83#w1lNJ6ZouGKJhg;Vd2Fz3zU&=A^9-+rChnu5%By#>;gaIEAp__}VWL;D
zObYe7QW6rLmKBMaWL`Peqd$J6`$@<D#YvnPGSQ#AP<n)!m^VfXvCeGC8wc;|%#qXK
zjv_<(ZVzYUq`b`eUmwg}_O0p|pJVCD=nZe{_MLM?_BP+htgO9b*pR6|RyWbPKfb}A
zcg4@w+k$VyZ;b=3@*cDjO5$Y2#c98-?Bnz%3+Sm6kdw<Io9mBPhr8%Cu=h{!Q{Dl!
zM+W#<!y^kmiW){r=JbY_uHiDM7xI@7b?5?V4&5>iluKeZsv0dQuhY|Mlr4`Bpv9cd
z)(|5OAY&G2WVM2)T{+gJ$s$<@^RPGk35~<Td>8$7jQMe!TUr9p_}}SbkOVV&8pV1;
zyX_vx7*%s?-Xz|Aa{C*2*cik^VK)G@G4fHN-;~Oqk?R#*(T65n)H+#8nh2kB!8YEC
zqBjilLhg+bg@|0tN*fnyR}rWYeVw71U6~bpA_RB*pi)RGk>gW?6fC|b`obF=y@yIf
z;nY1N9IxJ6v&Ry<kX#Kpju1@q6E@op{P=mO+L2d+XE?pT{VTD@((G*Co365FEjb(h
zHdzvkH(LacCqJhuOOz#!iv6aK(g&kj3}d7cucYbNy-&DvH?kov-zm?4c0C~ti(15U
ziVp6MZ=L$O|0`PQQz1_aG@i16&8(ncQ!GwqC|>Whp^I!6<8^SF5BjV(o_b(jGGTJ-
z!Dng{+4VR!N9wfOGh=2o()@)Y3Y0rY5{t#n#lGYgKld|Lj{|y7Pb~c%dab@M&Ps-<
z<B4Z5F)yaSeogrustGe-qL+s3$v=F^4DCCqcrnNpI5T|HLKTR%7<)m(Q0}~Ch<)Yd
zM?49NLLO*J8J-QR^=^|viqyG=*1ZSAZI5s}UJTp;rCF@?qg&P=C4o%#W}uIM9ta#c
zfNpRvQ)2zGAqDTf4If*Z^uWGw_8y8*6Ac<<zpw6zmMc=Wgx**^pN477*-X?I{?(9F
znd2qOGiEn264ayUGmV`^_%-7=?VWbvMlw`brq;wd#)Tck{$gC{l=?w$b-q0_pBLY0
zrma{W@9vSj)j!KNXf|VN9uYO$n9Sx>C>!XD)_=>*kiNq3b%D<4SDosYBF7ZfVq5A4
zE$Qc;q5iBW1XJh~YP2b3PZ{F6KhGSwyL?L~Tb}qrn~|ld$5dq>%DSs?Y<2<KuxZb{
zbN;$L|NBob7HBdxvFbpdw~*`XgDoGG1j)5vUikn9t)zIIM&~W6DG>i7#T|JJfY52a
z=0sqOK{g;N%ZvS(*~y1%wQ8u=>CsLC=Un_vF`0Q%EKTjfh9_=1>~sKEh-b+st1UFS
z?A6S1Xk{RY(Y*HaL_t#rKOpZKydrQyFJ;y-++JM^u=CK5jm8b}_uq3ttjb#t!}sEH
z;w2Oe_1w46kgCwn2!=naZYE@JtdismvG3C}L59uL!Lx*_XN+(<B`*+PLtZT!=+rAQ
ztsDuVti;oU-JQND1dgc+h)(QGf&f@Uj@ssN+h9fCo2xX`zU=Kaw+d(4j)k{`Hir_k
zHMsl<)`Dy;9Y2gd=r12&Xr0CJyt1wvU%rMTz2QN;G-!SNm?r4FqhxvnO~W&v-<ODQ
z@)<V68z!=NBl*-PS$)2n9;Y;u<;@jqDYXk`gwI9ruOLgpbBF=7|5*}-du2T;hNf-c
zXuhThkmlcyl+P8mDh5TdECSfi{UP3$E09w}1$V&iYY(#NYgIAxj{J59i9HvrMcpHa
zNL<yUiyf65Q|y;!WHgMD0vIwK@*Z;G?`oc3{;=PsjbS-1wE)c~6`(E1Zj8=-e!Q>a
z_~O}Pk(C5iotl0)fdyEcyucWK8Ok5WEc$05@({7P`?xmb+RMoIsN*GJVnE(_b@@gh
zrT9cCLYz7{AP|Ey$XJ7nx;sdMgb=55L8obxa+>2TL5t)CB7(XYRF2%uwl9blbFF3~
z$Fj~X&o%EJW`&YXY<k7pUZV;0OM{^$FO8S^7#wWWqwak>P6?|WqjgV;ucHbix!buu
z($~_Y6U&)ziqp!~r<ij7uXAvg?rmxu#%EEb`pm*a>MKWKQqO8>!n`R+CeYWD(ySc3
znc0xf9ZO;0MqDfXZ!duB|8ot(ylz<+6Edj<BRf!2NgnqXWq0imTnuF)PREy02}nPZ
z-EoZ2Ftdd&Vr{M2bmfBssX8&6-V$;DClO0A?Eo=dV9_jXO}+J_%kQeIf5XxCJtUFK
zN?qwYVFuaMm;RFZh*gmZPmtKhk^UegKL4s{on3%|ea+LF+_GxUd?<6Ehi$)-?*z#b
z29ahv@0V@xCZs<G)3@Q8$Cmb2x-SQ=vGG+X139*>g`yAKh5tDlAOavATMKO$iTweD
z1TO0vw6?)iVXw4EFSEDXU3&>-`NN~4&>y%$Scpw*?_uM#X)#QL^c(GhgIx0uLKV@C
zaM~qjf)V}Mp&V3wa_q4cgyVYytLzuios$J6=?CULrFqHdXIwA$UQL$%%qGKTBV6)P
zJ3gWSA!+X!ni&)3eYu+6*OR<2wX&8D8*2%CI_a^&zJPNCL_+I!_wSRC-*(8RUu$T2
zVv_Y^bJIqX?#`XWgS*^--x#9+t~)3ZM6KhPTkS&a_nqG%=%n88^kYoCy@SvL4Tbs3
z2C_oDN-=c*|LZ6C>iTp6>FU|pkYXg211VIM=Rc$o6BivfsRQ;73Cx>wsIjFjl?19p
zXQyN$%60GxsOhAkqx(Q^1hx2ZwY_PcofDa5M0?#MC@NDS{T*1V{LI=>sE~r<>$7QL
z7-%Qj>Z9-DdmTAgdQfGm^hdA2>5+(mfk8X}z&pxeF=4~>(E+h{Q@P*x999L9M2W(<
zyntWG+q)5#b_#YwR3ub8zpdSFXlQ5#lAqy19X_2aKvGEuzlc|}b90XmCI5`s<FwNS
zD~=!1@awfb4tY5y7tgd<v*#kj7Z64`eZ}a{eQyfWr=gpU-=V#cy=dEVl~5aH$($r5
z$i_!%6vK#_+|W)%@nVpXwf0Sg`;4dZj6gN;x``6c8kOE}IJ)op3qbEZ<x%x$O!zkU
z;%jw1x65XU0!gLd#}D@G+IN#>sa>xREE!F(RtRz-I`vSXlFtC)UYF;<z`&r%F?4gw
zpZsvF@i?xz7p~RlQD0Zt7YSqtq{P-KM@KQ+0rcXJ2lC2H-zz~EJh0i?hyshjo8<9R
zvm<pym6OY3s9C`Vtz`k_b6GNeqIPbfj^hlNJ2GdQ<M^pXFZ{1mWLgKMp{nWXHBl}A
za3fLhJc>I}B0w!X0QC?OD2jOba(XIpS7?K|VkUCi&L{2iX-{8Y)T#Rohr;KJqXnca
z1afy)@zTe(iRhV_v<m$@J35t9pd?lS6_(+xG?@1sa&q$Hk^JKjkAG6`*OG|Mw#RNl
zXvECBYi-mahS01q#A`=YO2=xB$;LT5d#aeP=gOMRe`BLro>IB)Wl>cl5xJp^Fj2V3
zMZQiNhRf{CF$OcilB7F!^fR|b*?muypWfLeM9B+{daYFQJ=$ddVRw2Cwen)~(o+_^
zuAVnt(->__U4$}0y;U~F95*j=K9ON$NsgTtwo*BJ_A@|a&A@TJ_IPQ~7lvkX$K!T(
zcD{zEKLqj_bLhHpuh7Sm`eL~Pl5T?2OyV183;A%;1tqj_FXseNAdxFJI<-uF_Z--m
zPapVVz#!*zn{B}?D=|&D5Q*noI+&sV1Bd+QdNd#v?hP}m@@cK@?SnuzgC<ozVuZe%
zWJv8{6pNlXa~+Hg`X^FtQlR*LtGT04XGvy_)0QpA4)}>MrO^xmx2cf5nf>`rOP~ne
zoItY5yH?_}C6}KeYS1qbX<cTgHIPY_f>Pc<<yA1EgXX9wh?j>H?JIns6Jh<Cb)sCA
z7Bh3l^`eL0wb3p1tZnPM!Qf_Uf`8l2s+slCTw`GRgEe211X6?33YqT@oiP&yREx!=
ztXDa53C^GZBQ68ljf%h#(n(s|oM|x!Hmu`{h=vj2B_R!?<V(_+=e?!5o-Ka65;*jP
zr4<v~+DC_!!$(UN=WCS>=DQx=$q<H=+<2%rZE=bIsF?p~L;pI~UPTzw=WB}set6V@
zmYGp86;-uRI~Mp7;xPG=bEHTwnVE)bxwWbf=n5m8RDVA3L;1IQMITzIcjedvdD<6=
zj2MlugJJ81Ez?8%E*`pAGzyOEXue)}t;L&SbirNhHBB+wcz#6p3}l+Wk#)bLr*{i<
z0oI7~)3?Fj1{@^5QO4b3k5mg(oi8inAIS6EeKlqoUc7lGu8t$n;xpmu=w%aAVo;SY
zaq8WufBbC28pd^;U_(ZJjzy;?qPbbyNnh!~51`Es!7R`Mn#Wq`mJU1)7|5S5=>?JC
zp+fStnc*BKc1%pi4bP{@Qt}xeozneW%Z2*7q%<tVC<>P<x+UiDOtbNyNyk6O?PpCG
zNCgfmY)2BuNG4HNvWN7xVn}MHq6wGbsYz0aJ~&|vSJ}=rUqN&GW2>SCd))%$&KE7l
zmzCL_{n%&=>of=_^ae*ErV-~i6J5%!z#VWsMxYb5o^GcH5e|$Wi(%?HG!q$lY_nW@
zIpZE~6g<i>KmYhy4_hfd>eU!(h@Fqn`A5@;*0hs_Zp|Gw+?&Bns^ShHU?cV{;f`Y*
zvpaoC#nL9Y5}i2|II*<5FGU!&!KQssa?3-Il9Mw+E{R`CHCJVzcOF_K8AxI}(vGRC
ztIGk#J9hep#&KKa=9wZ0=>Rj1_~w8{b=Dx{I<`HqwLp3U8!*=iO)<K`V7Nhxz9f5i
z>g__+S@K5%EVl};^V^0xIwfUqBmg4YNBk>I`p>=h^PMkDNIM$!qr1cjzFf(e1|jVJ
zguEALXF@_2Zed_xAcgaXg5z|!EtG%mJyzV!*2ivUC1>JJ@0H{A7wgl~w6=?-^xPB{
zLu|5m2Rn!z8R9ia4g?{ZikXlnm73fz{RXYXJe_~^`@y!LF>S)a^9^@&i7r7mWfOkc
z8w_Pb+g`uVRU6uA*N-|rF(chBjw|kj_mRD4Rp}F)JQn@fh`zW{O|rlZEC3P?V{{Xf
zrL`QCbLY(aUkh>C&Wd7Q=C0)|&@6inngHWBMu<_3Yl=51^A!~-XR}b9w*oALWrniF
z(C-{*&`*6BKT33R>Rwg3S#FxDl}d88MH!mE3qjvQkt*<*E5GP=|F-dd{gLuBoLRR9
z3X~#w!37N?lHNsmBaS@mD6ZI8&0X-;8X7Ios=rI|FF4cm-LvvKm5*wbFj;S|&q)OF
z<8*f8p&%ofqc6L7Q}8U2J-~%_cH2A{oPTK!HtFGC#BZiMqn#rAg66d^`QNkO>7^up
zc)}Ym0{#y4Cl;5d>aK~#VDLsz_3`B}h+ilRc1V49(N>=MnGd3avW;h=`G8j9$cPU6
z?ndzkV!k^&YVAAxd=3g4=%Qj`$cr)O;Y8^&D>REj#Y4$*GOfhgUr@+Hi)9R7HDBWu
z6a!#WQ!MT$672;)&js+~pigskI@e<ishLfS-WEI&uz!i@M8%MmQ7G1kpo(a1#uH6x
z{`Mw@$JCdQj<Jumklh<knZrxE?YvU;hjTfGk%GBYqFmLsnI#|b9qZhR9&v2B^iEB8
z*_YQEr8PVO1^Bq>Gn{{}hd-~1zg}-*p;R@k6Z$pfg)6Abe_3E#PW^Huenx<WYO?WE
z9W=B&NHRm-j^M(d`?Z=5iBxW!G7Ak{Z~GvOPB~3=*5egV068-2T|{m9#Vn|tO?C(w
zXw}3Ek6fD7&xV#Gly(aG(CD8OoiOllap^w|913UiVbJ`o!xDT&Q*0+kBI`mY7I?@0
z+}ojr#gFgKTIItwGU`R;{qr;Y`|D|Sq$I59vY$Hb&T?MSU<5<5qMdZ0a|%lkWPV7e
z6eOSO``(8~=0JthB_>S#rTzMOZjp_{!Vl3$1S0s10U1u%fv9J7tpr%w=-s6_s#}UT
zuWw~4Qc{cE_?caZU!-p1pnAge>17^%SP-f<bT8ok!LtRZnY*O*VA?2!^UKg|^Vu`P
z`uDgc|3Y~2tWoOdn|#K?%ol3v3S_1W>>Xw@H;(I{6}SHgX8)6<`X7<yO8|HwEnrZP
z{DT$4-|>=%=UDTkFUl-eQ=J*(g#smddPW>7n<JUwyJn*B@L)s>JqmG~RHoo6JuSvx
zkA52Y=vzlaqoK6eaK{$9L~*f1HJJa`2BCLA^*H7YZ?BJ&8JlZeMt8aRiC<#D)<Ud?
z7pNWIH;oql$1Q@1g1kl0B{1y&G~%LPh6dHt$wV%_ML!u^&iC1!h}dfh`gkcp<Oada
zw|Lu$R!Gn|0#>hv>r*-={GG~p(oE-A;=27HKnM<3+@t?Ss+nNO)Lh?wmF~;0r}+>C
zj;vYxINs!u&UxaCNfU&RlGE<o>`5KNg3r1UWAx&GeO4mmXVt%OaR}{?+=)I6`C0Ft
z<Vk*ipOzvPmcqxE7|D>J7Z$wEwt6O<-iN(T_5t?AAYB=ARh||XZR<%|+Aj)fnS9i#
ztiOsUJZ9t;u&o$tYerdIif+kXur$ECdTp2+{nu_)Ku1saKmDwIpvlORm?THMbX93M
zrs`msOb7k)GkGV_fhNzk>;L=K{tbvA--pQeQQs+<`2L@Tj2GlR$j#8YfW6x})aHKY
zNpfg(xNIvMgH%coUh|i1VIl^1cHUNcDk76S{Py$==e;)CWR$nR|EKA5KR@^5TtJ_*
z?LXgGJ`!dRWI($>s$QLP_2+l<^2DwpP8MM@%^C{iHFse8#CQ66X~n+W6MOIT1jP8p
zj`KJFBJKZYjQRPF6gYis{k_0{?YPsQVMJ&yXh@mu`Qh4S(pEPyW(xxeBG5R+$TeI?
z<yggp+>2hAh@c&bsq^%G{rw=%pnSTH+#^n=#8koi_mA}G?@Xo<ahY9}X}V@pVjF#J
z`LB&~5Oszx^`zlukjD{sV#Vo8p;+Qi<N}R-1D~eyo_^-@;X*qlGQ9HNFQ%W_5hgEk
zuY9>cRQr=c^?M!Zk%#Zt*I;J&N3Ts0yOca`GXX&jItUf)49;G}GT|xkHweRf70e(X
zg(Y-xtgCBla<~6)$1^WK<Uyoi8v-5I>-GYXY~3OK+z&i4xu1wdg_;nPUwj!R|B6mu
zfSs3}bSGBUZ~wty%9Lig)4j|i&)K^D)%#aYY(gEaGR5);V8^dG8$J48JDw8xExayV
zQvOpLPH*?~bnY@4>@LU5avz!msyg6r1<9cs+4(DWM-*t%v^2+(EkXD;FS9iMJ6z&%
zAa~0a4S{1>b^U$Blp60#L#FdeIpczi06V2hqv8bFG*h<<*b?L2XZ8~I^`@wuLiO`6
zK+K_@e}(vpKs%3v=I`k(oQ#Wj>*;KHw9r+Z%Jy?dcpuTd4b3@v@kNb!`>j?<|2ek3
zmx<Ragek)#O(;)JwH*_?B&oY4Zz)({Dq55-ohG+Y;b4r}FMb~(tNAW}S?6}N2%)^G
z+f-yb_8m`t3BI^TiVV++GqcIeSN|8j;e8#dEVR9TgV{e%o|r%KN?;m6PhH+eRZO*c
zZr~lv7>3PuZVL-;yuZ)nG$t~!sNx`4_VQ}PK0wO39F_*{?h_LetK@62zmS6tM#yl5
zmDYr*ni_$x4?0>*_zjb8JS3CvejXJ24&VpJRUnGgHUgZh8dg2&0aj3)&YSvq71R=;
z=_N)Txra#91cbZU=JKLIQ4<HdpvsoQ)!z38)Ob$?uTgoyNxUip9fqAMeYJl++utAZ
z?-#GZHOUbl?O!S%%IQi}o2m6mzw^IzH)p)5q|S`diftDgWyzu<t8^qQviq4BH&RP{
zy9m3WxLtK7l+Blmf7G(5{=u90sOC8M`~y%`6)b<f-hyusBBb#WjMJW102L=XrJn#5
zllg2y+VM_5$P1D?(8~VQvi(QIh=lttj~?-__D_x1y;d9>?{LNt{SLNZbi&OL=10zF
zMOEyw^T1-N=1n)!fz1KPI9!Qi{_y$K3$>x?Mpl{0Z`w!Z3Uu{?NexWP`$+0r<)g~n
z=;(`D&2hlWmX?#t<S^;Z)=36)tmV;ydq5#l+CiN4P8B}0V0Ows2!>P(woUo}BTxtm
zB6krZW?m;$xH+({YKI9;d7scwQOB8w9%N=@41hgExhBWSb`P5}bnA!RbzN-0MK>Zb
zv0$Dz2yC)L0YlDG`vJsoGXTvj?vu{=#XzT}l<(2niP_x83j!Ko7Ramvc7h)=NEY5J
z09SIPJk8(av$T%Y-3~V0I!WO6D+ZQ)Q$_*Yj1!DXEhb8@Ba2>eB=V=Y&!$xoj6_bs
z14p~i^i!FsJW_nE)8IZ~PSMoV#A((~iS!;Et^qDiCWvoy%|(BG-`B5k!FN$e;AxD&
z?}yKdeE8ZOxNdV-y@5eF2!0mX+#vK!>)B61ssz~THlI#94yH>v%(h~1^<qE5v;>Nr
zxcj5~7(mFm?l`?A>I@cnOJJs?R6F})0Hp6KU>XOj6BZVh4K5GJV{zO4;D(rlL|zNC
zMzI%AD0sO-(9!YmQFtVNEWCPN+%E6}>%niEoBz7&86oVcD{6>2ZhVr34j1;}`7L?u
z0h@$plJTW)GPDh4NKa|6(<-Vs;F+$VVCg5V9;Gj~ecS+3Az@PPhy5dE(fUpAa~kE$
zvxJ}zkO8I8_(P=N73r*2EhF<DLD?!4w-xfk46$=-frOOE>|hg{6RcD1-DFo=yTTnz
z3+`KlI-cT-TcqIFa-7TvoV<pe!SJT!n+OU#h735;3RYXO%6HrNWxpPDwCQAYTy+b+
zOvW?4x2kE(4PgRwxhz|luR~Fa(7W@Dx1*kMoWK}}06i7(xBn|<HVcDQgzr~LLxJC`
zWrKM8Mrwv9bUf%I_O$PV@_lJgk4uU?zi69t^8|$XeS8)=)*Wkan&O*Dda4D!FQ%1S
zPu*VItPXWrn~aQN)yb`(0K*L4?*$4>HP{1~mg70FcHFJ|TfDpLQ}N&$!#3BQ4A#tH
zz;DQ)DM9n55_&tZADWnf*!F`W`LKmN?eE*-U-Fc{a$Bz+WU78Q7d#R$3}lt(g1#SB
z+`Uln^lgmMKHIGs{GHUahgg=dP4g?h*64YDi*!H8dT`r+Q=|)`tDbsGPVs{{C~aaJ
zN7ikhBR<)jJK+<q+Bc&hu)J`W!H4S&XERBL%vcs{%J7XxEJ0VuZ_|(~yyJo;LJ>GQ
zpx%1s<mUZi2?;Nc<f&3S8ryMg2F4CG#Lx<OLie(aDc_)y?BwcZ&xKVDBjUX^!sfUO
zSBu|?ll<yqLL}3!Bpes<3;30f3#QO`S%wR{Xp|_k;y5qoWThLGDcxabf5oVpD+)!w
zbkr0J8`TiB8Q#=Egda>1??oWb1M@;3RWb}QvGjZYxDNb5cLC?gS-WL5DGErL@=yg;
zokkxx=;sq`x$mt`q-C;D-#6$5H@cj-s{QOsNX_b<tyWx%Q8=n!3{`+}q>7AK&2m2@
zR`dTu#Pa`C@;pd37f<b?nJ8Tx&IpT0j@wK#+Mz<^idjqGSElSToBWhBWrMyHZ=;yh
zL!C43n9dvo%sGrDI(2PTvF^`GY7CVaN!+h@eN(NKV~p(yol7j=6+fYa(QK1Ya?A2&
zl7M#6rtxHUi4T>*e8i`q$jAieko#OB&pi<!b@^#Qa(o01X;LJIeUj4h@>yV(6Moe-
z+t)nxyz5q5v>7}BR@oy^F{BgUe9&2QbU1fm<LJyMj-v&E(^pSMYj)&}bd=H779KSs
zM)|=v`1<Yizw#l>8_4CFe&(0+d%3RtTrOvYl6$PUNQ3U7EO}*cYZyTe5J>@nCRl6)
zYD}f!M<RU}z$de)u;vz+{2CA3mg(r|PzFQJfXY3TT4Bq5D_~l2Cyw2VpcjJPcmLy|
zIv0Crd`Gj&#~(nL2f3Yx#i-}$4?VbXUd2@egCz?R$Ay6aH0l{Fp9#kTts>kX3cW7I
zdCNN~@PgTg>%>L-2Gg&4>_6^B<ogOpfHSa}(GYKxli*N%s|VafBn6Il!|@VrOgy&q
z*aS<KpMoWp2y<b{MJBc4;`UVy>f2#d(cs+_2S?jNQzyDmukr3N*icfvr>9oHB*(C<
zk^@(bf`mJtJbqYRb9a6f$!T{41WZXnM9=Z8Uv1PCu}E9cn0hek)>Va-Pi~(Zf#4PU
zfT^7wwcqH4fH?S&-S(JdchO^}0J(AS#bm_vdl#fV7&I`Jox#K{?aJNhNal0th+qLN
zu=6P82fz;LSfZ*>0sZjhfw@KtEw@)lD-E!U&%>-iLU-V`G+3|;X%s_Z`1lCzjhBtl
z4-Dc~)psI0K@%#_b2RaHlg^VGasy93s+cRkuO}<y^)%T*;U>Qeb8(gv)M79`3q0tE
zTwQ(0Bno8o*ACS)zL61cQC`+OUSX7aydrCG>GR5*TMxK&D0j@cjnupCXMXu&%q1TX
z0eauti^ZfNIFl7tmSZN9gY5Hg#kW391!F-i^Fr&|$1AjAoM1P@!3HdEOhYo>Q&?yl
z3HiH+GzHq#O5}djz@y=Z*0X%HA1XSY+F$n<$W#8$_t(2nF7Ob?y6S#VfwZChm=To0
z?1k)SLta=K7sPD@&O6QcUsz%@2C#V?@&_u$+7AsjUS1O!&WFo+Sq)tUYa216N>~I&
zDMM$)#KaUqhIiFfOj!>xePJHymv?0ZGNPi(yq+iShsB8kp7(r-^1!g#f>;8iaZgBi
zEh3KGH$GIpa8|u1kpi)%^~rgEzr}~=A0KoYFyvN4hiY;arUksR=63pM>&Mujc=t5=
zq*`?4R(=+^W@dle$LNS_ueQmbD091Vo%Z@5*rY6iJj=q<|JMZ}=z+{JJSffvU-=1E
zK+5|1x|by`%ca&JD(~Xr8{fgLIbcK8yajBS%pgLx2DNVri*Z>Dzm}kQqQcBSB%&3f
zAv&0^d2_bSfcY65S$=}gp7Xvh%K;=;<}_?YpLKKL-vC23M6I~M&bym-O|%)TY}xi%
zHm~Nz!XEkpD?@#q56d~?tuXOP?|7{f8{|f@V06n&=w@gTLX|sH^Dab;C-f<>Hmn=L
zX5JM>GK>_0L8lmhCA7wJT>QNR4X*#p#WnxWxwx;YF|gEz!KIb!b9mllPAFF($J_GW
zPRXq8uT0~Q{bnwNEwCkIePH@j*mp00jF(Z3C*P>PIYr6ZG7r;IX3kAS1L>Dg;|Ly9
z>F=6(6dW8l!AEQ|4rluaM%oT$d~I2e)^(M-`|(q`3+@}s4|0hU;DRBAJsC_Ppj+6Q
zLlB5wWYZ-~e7TO~I^X0t){#=vnBEgvp(df!Kq2(jP$|V;xM3V*0YyVPG}Q3Ry-T{I
zh1q+*;$e>4wYT^}FH}vb=74)T3<hf5&>n-$p3ojlEOInW{+mv>9glQv`+=_S{&#F+
zIjbFSW=^TAf>BI;2&0$$$rGk7)!pa?v6L0}PDSza3nr$Hx}EKbFQcNwPxn1fO~(qv
zmn<jC&09l<)=?dwG{T6CU?5hQE#?-}($YR3XOUoXha*b*#(HY9kiN62m>$I&MeOHo
zsQv$P8;+Jd$Ev5wSCrsl&=ReI7~YCPsFea8tkx@MOc1gc4QB*Q)*7iLe2O!zS$(;L
z``_Mi5z(L~x19AUeon{e#|}#n3~qWbA&K@y8Fum32F$`p?-Y(VW$U|QLE(91JEG{T
zUhuKbrL1?kZjzdB--gHYg#Q?@ZjQcL27-Xi<zS>v(FGeDyY))6=4#uOmBVWlw}cfG
zULGB7Z9G+LEj15EjF#WTvIJ~g;!35bhR4Y#*B!`k&BLs30~-LbelJf<*Gt6UQjzo_
zblQ~^6kg}MK_*_k0pYjiga@0#Q{-KaPn_rXUGwVFukdS3uI}8gkL%#{c(=FlfhoZ*
zuCtua6VD)I@c9A<loG%mD%$MV@qWtkGsFruAmFq%=y$ySJMVTxpPp>JOk_;Ri?Vv~
zZhr+lKEDn&WMyRy!TF>iPo8+wuD0o0Ov}Tgaa(mC$bF-D`P;y5UV>=}G)eJHGl@>0
zQjo>&z7+WzVDfu@saI%VTVSqY%%AZ%Nv_{HW0N}h=^%xfvYt9o43W|D3=Kab_vvdq
zhE%Gjc#47w3ZjUwY_baJnAu)lk-3!Fc-R^HrRb8lFIkK4vBjVAI@!^MKMa#~48J~c
zyza(!(|)1iaEdS7rRQ)PiX^>|vE`(ln)pT&+F`oBXM$DOZ=D>>`oKUkg_n&8|JTLx
zU$WUV=rO397aVcL>Js6cDD=ib+DP~eI`aXLG!qnQ8*<2WkQ9=+E#c^Z*X$>Lmq>+~
zebu~(s<%3|V7Hses&jiMhZE0<FEsQjpLAPL&;ivYz9*}hz#FQ~n`f6Nv;4ldlEukp
zm#Ca6r`U}70w2d#7{gL7$S8!`H`hoOY_iNwPmcGza|7tG1DNlZcd|RPxozJIKX>6W
z=RJQS@gSpjGC%r&`4Sl(G=Np2JTOHg#|{jNJCQ6}QTHa^XNRzgIqa<xJ1WJ#J~6n>
z%G&p4LKLaBj&Bbom55|gEv=kc^qkn%)703>o~R%g?QFJP&=>+rS{%F4b4Jg_egj5L
zj=}sqr!RzeV6KanW3E)uXtMuHQ3Yi2r_*Zg2FQDN>`l6<h9^OHPn<8pkQ9HA`ddV~
zK(>ss43@T@Q#Idqk+3}|p?dF@{C4!|k~*`o)+I&y<Zn?Ujp&&TS`6&w!(mZD4?I&C
ztFsFnp1j75>RQM6Uh&4GZLDvL11;P85>D3xpi+^CbWNGA{`kqjeJvm6?+ti#^H0&R
zwy%sFnX=3}B7O3aERnCe;XtEvHUYy032@AN4cjnm^+z}fr`c<dR!|g+FK@UOJ_eV-
zdr-M6S!td2E!^*OWfmx<_7sUg7Eg2?E|E*yTd<WC;BWPQ96EBcIONNJcL4`IT?OU4
z=JfjaPvE}gG|_qQJhWwK&KW!WPiuRrYhD7yZkbt-Jg3hfvtH;fxfM=#5QB-II6Heu
z_a%|1d)Fr(+A>Emq>xGH%XSx`2fB&F$dbq}hQz10^)C~Uh^iJ92Cf+z$oh(+TYfJo
z7SDRkJIaG$DLhsp&Xcd#7z&D|rp8P&1#E9gAngV}L%rK(`}Grt$;A|$XnZ=tLwr%E
zF2rcWw6eGa)$}X9N*H{GRpib2;MDb4^TW%!q6hCY33Z*leQ7HTvW`BnX;&40kCZvQ
z;6Oc*@Ld30No$@%@)2i+g^T^Zlc(WyozG$@GsH#CF&~B6cEV*b#V?^R2wfai6dQYq
zSKv~>k9Saa%ODeOW!@EUbn7n^4Se872A0jGNlHKNYoxhYjTd{9TUxX-MJtGd4Y~Q^
z1(Cx~-6?^~dtU^7WM=hup}SM|cIlz8><s^}1B)7Q=F^9Bfw&?6?4Va?KFNlmBSw^&
zy{W_k6|SGmkR0kkG9~39BkqBaN>FS4c3ovi$7w3#Br59GBi-(o*Uka*B6Hj;ccL4u
zf*p9Iqje&uo~>|gq@v0Ri_ocXB&GYp9E!IMmDk2%e*&0s3ft@6VEp^4`!6qUU`$_G
zr44lEyRTuCP`p*Ng4S2^pzmY#a2PosjoIlkun9rLeZSJ9o}m*_f=u!kIcc$V!QSM`
zgCu`O01?C>1y^+mc$`1bOhVi<%+TbJZ!3u>JW?Vq*s$Hrw2FRHmp$?Y(zn+e1r6!b
z`yi<{5fh)WAMisNKWgk?u;GfWc=UO_kSCxxQ`9(}cj2^z`rf*B^~Wf#Ddl`q;)O2^
z;W^-AGOC@t|COQ>vO(ExtjF6gjzTBc<Eq+OrIGa#)U724Ozk;fz5FEld?TNR`O|c5
z8odq}$*D?g?sR(cWOZ0sn>>g}R@D49vR>tM`o!Pe`8EB0i{Iq|>KFVt@{*Z7D{J6_
zQ{>UR0~gWl9`ih3u6EyXv$=LzJbe7GD(z$EY%hmd;#I(fsNEae=Zi%cadV(G05@{H
zeaunu@~^@?Oc;WNREl5C-&c}1t(Q}{Ag-9(uXhkcC<mdJFLnr4q8P}&4oJQ8UsmSt
zstf(b*(2|%n+m1|mr9B=sR01#wMP@^;~}&32j4s)C_~esNEoco*ibd}UHed>{FN~>
z@%~ZZluVNO2eq$|I|i&b)KV5`);?#9$a}rKI{79@^y7>?G*wZ}4MK+RUF{%ARe)$(
z#Bt(wgeqo?xP?z0kPVKUV&D7M_K?;Q0A|K6h#FHj$}BMuhjP8@%hGJLUD;zn+L<21
zRN8X;1-a|X*jvMA2;P3|u;vlx@VRQcYOON=iDMU_628jtw~By1N19~<B;9wtT5~Ti
zWuolC(1&xM_UIjRRPzW5bn7R{y6+MU`riwo!A|RRNvv^L5{806)eo3>JRl5623#~f
z^aUAjswh{ftc5EK5;X_of(JKY>#>tglh07^OM>-)PCD25)5Kawr5EaM{lHVsA1VHw
z#|vHa+TVMvFP8TAx&+X`x`;j%;rjLb(K*h#OI#qfp^En_rhPT705pc5&6ofBGc4%L
z$=mhVavbozl&avAuEfzW=uM~|PBpO~Ib7o*&5)0l<Mpc(F!o>4S88x~!677(nx(f}
z9%krvAJgHF;pBANd*})tC-k>($F+8eC>fGlelO*dBN3rPxEqAjC&QYc-vbcCa#V&m
zvh(PD5?4?XO#9*e$_^S5KVP<e;vnC9)%N72YImJCz3f~|wt_8BPRZ^_7bJPaiQ>)1
zahUeyCphG<vnc^9arn;u$23$f*(9^(xrnZp_Z)T`?Y>GQwYg?JQ7p>|l~=IC`Qcpm
ziLLH1#g^8-XfrGgZ8OP)wciB;ZIV=yyDq!q0?fHNB7Wl~v<NBQ(RX_K+n<YVsYfK!
zjY3wOMd>+Rcg+HGb#<mZ3m1?Va2psJ8x>~RT>Mo@ou!5q@JbOC`FjP<Qo;(H5fsJ!
zeKJsyCj({p3@0FRXDCxK{^!Y{{dqEU=sHLUP_%13LI6<ct07Rk?{DFCE9@b66s=@Z
zxBr|`K>aN_IM8DSl1<AR)K=-T@lk<<Y;v=PsVkSsZ%3TJNJN>n1fH20n#}9{QV9aB
z1ESR*KlZTRN^sl=eSj^(tJ9iQu%yvqR8i7zxEzr){Kir}dHMT&dc6jB2S{AAW>Z-6
zkq)+9(4?|wf4)uFSv0S=pjkPKM{}9W)u6SlO%YDh6688>2<#^XY^KjdGHVWYmhC*6
zOlM%xtsBot_(n;NRooie$v7PVP!JifokmdGhxcLEvFgEjWfw-!F*(tUp-}a^f^R3T
z!`G5gCZ%+ABtfe?vaFD|Bg7UQA(1dUSY#)Cx%+aZ=AMs1!j}YpJLCEbj0^_7PY*vP
z*lt)E8_247`EYTFSF=Gxl?6^n7Ear}Me8pS($Q>sDJdcX3f3aC!oH}#RtzH7{@x#v
z>kJBH1Ww`U{USQQ&&*ZinR#KQbpLrVEyJ$DpFSM_ad^VKk!OZ@dy>mCd;Z*&R;Je(
z=UhpP&X_LR<BYSNJt>qDCGFL$OG%XGFzyrto4@ptJkKmJE6SVoo9Ln*WUWIe3PU7O
zJqZY>%aNRErkTBz$3tBb$~Bvx8ZNL-S0g9<!3J;C_+88^xng=+_4SYQUjv@?Hv3}r
z8m;t!4nZE7jh1;Fep7c{9gyfXa^j<63}D7K$K(VcP#ic}4(h`UMc@P3eJEDM&a3%a
z(A#<Tx=s#+UFp?p!bFx~-?QYCQk^a|-b;S6M+}+d2iM80W2mecuSKEDt=Dec9chR3
z&73^Q{O&>HX&U4a@)%|?%tm;&ih2ny3#6VNMt3L6KRLMH2~$suF40w2$etw_&3=9;
zGTII!2XjRqfw+5cVqR^%YfRUzm+e)UCNbZy6Ef`!iA4R|roS~4-o|vWBE}M-mwz>0
z-_pSu+QU#TC45>;JMk(_<bR!^%@;qTShQV!j^YsB-R!XE<RK+J$=#$axi-bz9fu=n
zx!~j#pI>dE$2-dTsOb_>q?m&Z2;`OrGqAuJa{_Okf45&m+KbZA2e_zZhHu5w1it$t
zO|(b)0AS447%;S;48R?|nB{T&=}SehEvl-9(G1>lC>M$ov}J}1EN|JLG{W?`uVm?O
zYu&+SOakMcgCn>*Ar(b%1C1iYAZIL~BK@{L<p=frTtz;Y@~C%yq(j}vO40l|uE?9g
zZxU8N_gIV1TBL{hp2B<#Gswp|VT2Q0B4M_7(UXx#o1I#B99Q0b!z{UErPMeW4I4xw
zuWbp~c>2dDqCV8KsgTGp5Yl`48R3sgPKA(;mt=}ZH1-~7-@O%72)+iQQ+n%i*u+$`
zBR$Ci0*NpyNcS$subY9g1a<RMl7?OO-_3c%MHJ^-80S^X|DwFWxuJukEdAz1ao|Iu
zK4YBxlb-VTUH|Ka_ZghG6ytYVv|B>Ct`r-Ej8Y##UgEI4Yp(oGf9|IFV7emJnNK!K
z6mrN3>JI3n%sRcXH~sFHKQxb4L455@N(2XeDKbG((QpQ3T2`wx@MIb&)TwpmQyhsZ
zQ7KUJ1%YU0=nDz~IY391Oq@1Ck!4<<yPg1;V+cK&C}}6+RaEfhvQRkR3tTcZ)&7~4
zZ*EL<_H34d?;GE}@V^efZC*t-X5(Y^vz?29N`?AOPmvA;S$2gNF2Ium_xOnh9-M&x
zkFl?Ss%q`JRs<=7P)d;Q5)>o_kwzM6L_!dc(%m5HK}uRWL_kVHr9qJHP64Hn5<ybn
zU)!sWtMB{2;~T>}#(~Q@`|Q1+{j9a-nsctA0B(zkaSx5%@gB2o=+cDdunr>nl#%yz
zB%;-Y>2Qk`ptqRnH5|(P>?QPWH`ckUpSHc(;HC=)L^KTNo#X|6)aE@l)ru0AeOC_P
zqazR%kqot07>sNpHZ#e{RAdNNtx?+gV0T(ELFk4-+~?MDuY>PcG&#8Z?yr9Y|CfRA
z%%~6hP1*i@L}==8o!u_uuBDhb|9q^T-o-^4QZF}0RyQ7A)3(bnKeGQ5sQ(Wl6jFSP
zV_;V7(>vciJXUU-V`Ss4#9g&bhZu-aGg$~bap6OA_tMBm@$Vm8Y<qXs=lTn97<3Eg
zrI9H<a+soYY6Mudd1K|75)bh{$GgXiI6T42AG)T*q^aHe%1Ug&kA+XX%a2d<YFH*m
zZB2pn_R2(c5_p9%mOt@YCuz5+@x^K=*k6Z5k>PV};uE>-^9!DW>k6(gE;7EQM3~@{
zEZl&*T5M6Qc1gC(wps&x$jpRP>7es-_pb5w0t)~kpcw?Pk|v0{r3Mu=>m<$+Gb?w&
zNZ<v4H!}c(ZOkS{2Y&9){xLE|&~{oSAyIUTy5B^#9d<W_U<ir;e#cvd<rSGP%yY!X
ziqWlE`QA%sURfE26AiS@mGV5zhXD;JUyhvSXFT+X?Ca94cSl-c*rTh0`ksQF=we$6
z<&Wyf`xQJrE9|ThznMF-r!dB$IC}22@w+<xDCX05u<Ch0jIVzZ2Zhl{#QD3D`Y&LJ
zxC)N*p3IlN18he^$j@}HO%pw-Ea|)+$InT#e(P+f>;vdptJ6^ollr`KC&5AE2iw+n
z%O4#$U(tWaNfb?<An6<~tbG1#SGBOKQz}3o1tvjC2ugyCL3;0fZ)QZPs5c~Zo9><k
zi`%L9Da<rfm<6GbD|G{UYkq2f(W)vPTxSHokyV<X9_vgc6IF!Rk_KhIV5E%`^xKcJ
z=@h29$V>7+K26OZQn}2bU-C53y^xjEsFC5|-d5YijrPkEb=h<sf|CT{0aDb`yxXnX
zjE1u}C53#hBPG}1tKI394nqND=oDNuh>GIUkq;3)csBeZaEjZp|E22l<7ZjbBxgoT
z#vfA$r(K?K@8yW@3SQ8rm)x#CcNwd*Ksr847fAu2D?%h<<!BJ=f#w%-`FaNbMT^Sy
z>(?dTH@N%^9A9GK@$ry;==(cuYZiYuS}w<HAHwxJH72@B^X9PJknSl~4xS(1(QE#X
ze*drk@`6U-*W<GXjjAEm*`vhn;t%=k=G0nAI3rjj(g@-H0CWzO7DF(V%TRrT2I|ZB
za>=Mhg|-bc%0iHdaA4ZV2#N)k2yVl1k-64jv^`Xptx_j_8v0n(Ghqf*mBzQyKpYgn
zUT(S8HaUc!0&s3I_%|)6Ein=?hru7b6tOuyQeg6UBsyfOF<8CZSgM(+<tb(ltz!-=
zSL%s9wl6Z;Mc&Blhn=+H_ZMMMi<<aABVthkhJLBQE6Eb#6$%Y#cC7z8(+9?xBV`WR
zLeUW+K%ih(4Z0al%B7zSk)sR1q#H%LrZ4tLqZ432x2E9SbX$yi=?9H$O~pu2rxlKw
z_PEZuPYlr}?GH|0kwpUN-9myUzOgcwfGgyCxc&izl6-mgKlA9P8W_%2%4}Ere!H0C
zFdVg7W%60VzW@7~zzi0@d+1dCE_9=s5qbdTk0^fMy#MmzY0mH40V0KYA8c)H6Ro2a
z+miv8K%nj2)(NvmGNsm|<i}_z0>AH21@raP_~8;~LM<+BxOt$JqkR?3^_@P8!MGKm
z1+Y3%S~$x;g(+_Z(%GyfN!Ek|IN|-(uhMnjpw<#?j)Igs4g67>Ss9bbfPMs*Ka}3X
zNk`E6BaKv4pef-g?qiyzFnO%r-ht=bMt;%Vel}N_&fH!!?#Ea5RG>MKN;vAg>pX48
zq|&gUY#-mF<4w|F@Xt^>2h*6cvbW!y4ehoiSwK5Wt6guCx;ykmbtK>D!Ei8I?Erjf
zub9QCQK~k($qSWkT)ICJ0n-R*lSqe8s{(4Z04^1G#ZpesY`pePNL<2#cB;6l>RLTF
z?whFvcYrt}^fd5E>Pi(R)%u>xoSvzgS`qpBbA8jiM3Mjv-^^bHCk=rtvW;O3pC0+m
zGs2<#v5ooAFL)_HGobSX-+y^Je&@|JI=H{KF?CR=gua8(7-96{6T=14W!W&x3yS)I
zGq|!h0XDk>sZk*+y~rGjGj8F5d?jNi`yNVES}4pxKVDxjyRo1yV#eu$!)>Gpy8FnF
zveyowBPr|05e6;oYd}d#n&ggVo(uyEm4(!s#n{6IOuW2dtz^Clh`0?<mAN;;S9YNL
zs#fFcw4PG)8u(<dO`C~4MyB{rvO6QJ7{()IdzLS`o97MRLTBzdk4ZxpG${9N^;uz*
z3Y0Nc99hk+%P&%31hwl@LE|0HS4JqX5v(kIG(&+$%5_h){S=?eLR-v5Hf8A>{ih2;
zZF6)kB%#Q!^ILvss*PpI*d(^2SrJD3GhrK6eRP7n1b2-~{{10O3Dp(%XE$NEwEDfs
zXIE6Pf7b2pQ1Lq&vZS2&g_?`o5`NjV4AVCe7yJDb91zMbAFj+%=e~=A3MOBqo8s4B
z|F^)88Sh=}g{i#kb&FKh8%O{3QZlH^VHLghZmWTZ<fWjn;rvH;n;n8Ja@0t*mzcl8
zqYbZbEoX^n{cuQf<Ro$Eq@TWab25I&x3A%`VBRv_EH9qSi?KmFm`;h}Sbi98q6l4c
zq)v!{)Qyi+i_1_mkAwFzOw+vZ3NgoD5>oFMQMxmC1Ro_Y9re-p5rOHKk2M>}_zsOA
zQ;|F_uF%j>r4DXfzA#Qbent}Tm1dTbL~GY^y(-{#)Y0NSZ*u`MtEj<QN$uBw0NhN%
zD$%DXt8=FiJrF?L@z*{(eD3qVvbS_Gvf5(ZEbC^jZUrwq2OoK)GTjLq(BcizU_naq
zYzw9KkPkthpby#uoH7=eICvMwQRMC+?CA2|3|vyM2)^A$v-|?l<>;W6WcYOL_OE)x
z=Lj*}4Wf^EUH{|(NLuBMleX!#g-_5R$#=|1F*QGh=zrY}2I`|$hd7BNhwm(LP<OwW
zlGQ#I$45~H^QHMw<kiN4OFk>%!;(OK2oyN?^~3v`+k+{1C2Zsg8_#(4AaltL#h_#M
zO;WZSN_wwY+5CGY_^4?NV5|4{<}NAJNJH~K9r|ZI&{kBWl>xV*dB8%Ho<8RKXr04j
z8+=|={&q`*oJtz4M->!$W#nZo7$G@2g(~{`FtdFz7;SuIXPVe`qg!T7a|i5@=5ve3
zQvo(C6E!Tex7fYk&&H;apDg6H^YyIjq<_QfC+i;=5oCR|cr_V<i11kjNLc*ynP65n
zTQz{K5OSC@coR=0x_^~}5vH_j%c--by~iDMUVf@J3Ho3KyrCChsH^5rR9`AQb#R$E
z)@ioWP@H#SQuD#DlD=6A>19Y!?Ejav3AqEeK;%HIop|Sef?pU)M~=ECh6)J}3}u=$
zDjjvmAv6lIefKfMeP>E|!1;X46a?O{(;e*vhZaPu+P%fp!`i|$Uwv-jpz2g*C{-w%
zrH6~=&~sH&0@SW<P+TKQsT#^5Kns<@KP3Yy^`sk={AKN_QGkti0gBtsi@S_Q0Ahw0
zpLPbnV?#j(OtLlYH`bqOUxPJ8OPd76vg7LqSZtAOT3@vrqluy80<6AWG@Xk`p4`Cv
z+!Aju2!J!xfl-nMP)q_IDO=h=QpsDy&O$Rs`;$h8D~)d?ObC2Z)RV~<D(2<6Xu(!@
zpGwU0LmNBnV1T+cifq1pPKC0MsS_GN$3@)ONMBJc&yx0&2337BBzFsz`9G=rKBfrP
zWqK_r_`mP(G|3<%#AQ*s^p6_lH!=3l7e371M?^cXzsZ*tzQIsz5;PYo*ui3SQg}cX
z?{;G>z@zNUgrZ-eW_}!*&jdC$Sk#R}I!!t_sjtkT;|}w9=so6pH(~KIOrAjz4aPu0
z9)@!d#`#)^N`X)}6^7ZVf_YQISE+d(&+pC|6?K#dbTaO~t~jF-zK~b0y=$zT!=dxm
zot*F9kjfm((cqGvGn~2_GX^Gul!Vlb3<5}|2{vdQ`4OK^A-)jc3B^C!AIwi@=3nj+
zhd_{&_u4RSI@T3P3opv<E1rYE(&E(^3ps$w4BL(PJTWGI=~B!fDq;Td3TJX6rJpH`
zxH#^|O3`G2?q59Hzm_Y*xuZC!u#<d3zeDSf;+aqI4*8KI;z#5puWJ0-XMPeC>j=fY
z6U{Bt^BZnX15|N_n0Ttx7q32KYJG{ZmW#<^ND3KfrvcE5vjfn7uC?pEzVu*geMSK$
zml$Xlxj9`b&x{Bdj5bBbAv=LKechB=C578oe*U$1O-Ye719eZsZ^DMYvcxAW9->8W
zx-IjK!1(Ubx5O<GC}PL3OVQTE15HC>up=N7>X)*nX(W-RpQG`PZqio^H*YG~(MPC^
zIlkZKIvClDw)G?|Cb9YcQuhrTpXG;Vz+p0f%egND6jJEQr;c(+;A#+k>HwZX=Iz#u
zb|$F{09>Hi6%)!F`tg7WBji67gbpB*FC7NAF%U|FmCYNX!J;N<eM_j|GCp{Df_Fcg
zu3x=S#Q|`uk3@1AjNG$@A&45u;z!qq9ArLNfo*#2aG_<;@EU(Q<afPL9vZ}@aK;s@
zL}uA`Qf#e3td9KHM148cIf(9;;SYv5>@P>X(M*3Wd=Vnqg{u?ixha{E|3O2JleS@=
za}XV4D(=tq*Ps6JBa{jR$#XH;pwq=%I9Ic)HhwT?lq=btu@iV(iwbten8MqwOp)<D
zfWc{(bg9kAC485I;=rtt;(XZU##l%dy1|e;z9IQ^8&5XXx@^*HZqV+QSKOo1)RxWb
z9)oN--EV4Dwdq&bV0hO@1?$8^NKhEJOI*KYRf#0M#n#a#CMUPZi$V61dgaRHWxcpA
zJ32i%=|h|1`y2BnAw95cf|ibW4FJWtF6G6&;k{vhv(oT<Ghl*i&}T7?mAG8XodV*<
z&WAm40JsWJL8{IH?C<i{kv!L=<*8|FX-lu9YTa~s$fOjXxj8chC5A?-q|Lb9tQE%e
z#n^}lCrZINia9h$_og?;&osue0h>-qX!1Bi=TU~A%Nz#_%7W7mb>5NvdHH63Tz6Kd
zS?k-7^L!Ggc_C)9d}vF2O5KO&f3joUFmSr;I;Cs}^K=yWs52MPom={S=ewxIybUEu
z>F8_}HI0Je6|zi%HYh9INQDP9te54f5a#Zc{SEb2!NCHd;C9i<_oD|^TgqYivuA_{
z$ZT;<a6W;pS*VPR47>SRd|4vqv)L+jt{tOQBm-o(@0~e&;{h`0hudk@GQaSPlW*hM
zU@B4hPSHaVh{P>=q;glHuXwmJa+NwR$A(`}h&N*^+D-#u_iO7<64kq`Ahbq0N!Ieb
zsLug>mc_XomdO4DLc3NDBOH|cv&I3c=Wm~%TofD*@oH!Q_tn&5s8>L`O1c0eF6eDg
z5<j;S9~Goqq!NQ0_-kGNb5+%td?L<oB=DM_)B?sg0T-s)?nRQ?SZf`Bvi~9N@RosV
zxsHB>*p5Ik8NuIm0bdAvVKl@(eSAbA2}Vlck#O7@=og->TLZ#o_Ip@~DhJ<pZd|Co
zvbw0Le-{=@M#Qo5=T1fW=@p|{z~Ms-CSx#;MH`i|oaQiJF5ll{R@AoJ-C#YhMNxEU
zeRiM%W=CDJAG#3GQ7;V`K?cC0Wiu<By>i;`<DcJ^3@v$a33e4p9B}WTmycu0?cEh_
z@r?zGo*2pRHbA?zYRCTe0a98Es1NQcGNwNVa#RmuW)C9?p^rE8qM@qe9{)MOfMbx%
zn~3=ws^^s2X6mMT86EX!uQ#FhD@-GL9d5e~KQK|O$ZGn9Z4w|CmLLY_#9tSd<;R7U
z$T%5|Tv#UL!fGB8D?Yo9{pc5l`a5U;_lq@L8gcF`#M9p5wglEh1NnuLlqe_|?l>Bx
z$r5I17tqbL#e^#rO?5&GXaUY=b|)gAKD7vv)ZXRMVmZk>kXqB&xcs{DM+wl%uz{_`
z(EO?*em+h#)|hIpT}~fmQy7sa?n{!myjO#S_S9jl_{l#b&A-HaM|=ptZn}~nj^G4y
zKY@Vrmm`sA4vVr05b6rzqSQ{}=>ITkMq-<{GJ=8;GoAS}pb^M2{J1c(9RCPckJ3Kv
zNh$yg*)%5k54Yk!zk`pk9As!tVOMsQ+A#qEiMvXhb=;5drHkmtkzXJvErKJU2|E75
zIT;>`%b(s=!aLW8<T5^Kuzz%FL#!>6Pyv_R-?iyqFFpnni+hh#C1ISPi6jzu7tF^9
zD6%a7lP6h68)G%fNUVo{-n5VSM=Of=Byey4^-)9$G%v6{n4|DNBN>1=$VfNLfr;#+
z&3Na}mGZ~?J_ZvZhpi9kQ0F-c<DkgV7K%YE0t0E2Gu{e60TLffB$c=-$?|7%Me`IB
zgH?218266KS>(LJt|MC`ZI0YmqN~ULy~6%_?G$?iV`F9GMDk&u{5!TB!sDt018D;<
zycd6I+LCZSR*o@WXZe+J|9t;b*&l}s)&8fij3M#Q;c%OHuhRc@h=2d+6owp*!DAyK
z78h7E_xQ6qYGaX%*^HsY@%K&VMivj&sbAR>T)`jz7-D|hOJ15wKh9l_=AXsGU}Tfb
z_Wv)e#P1M@d*8g%c1${S7f0k<pLzrzN^`w7n#({Qu!D;szj;ycJvCtw75BWm^5>_C
zC*!7VB(I9EOZ?zGd_3(FaS#Yh<D`D?!hfIKU$1GAMN2ElNp|Lsj}st#=T=U9it?`%
z5c3&wa;GkEUHZ{>@g)3bp?=pr#qz7;{MY9rKN38Fe1h~zkuT40u<Xa<cM+DQ@cO%d
zxsoEkhH?lwk#{{3DCEr_sehhG^*8UH|MKBKt}^_dRu(xC0amg<%hwVDI1zjT%D>_+
zX2g#ZVd40v8u0x2#C_K%VELCz?cawZa79kU)y|jj&l6#X6DcRE`K|K&XDx`+{BuVN
zCG>ur2=~tunK<k7_3uyqbr#5p$RH<jnURd(k2_L_R5#ADPk+Pn|Gb4j{Kpda&g}Ye
z1Gg3bS>kJMa>@TgS}vXmnfcAkcASP^P{BW6XbpL6k)8e5YX0X1X7G=zMala5#{+gi
z{PVUO5@-JPb^di8Gz7-T)iNgb{q)Djd7p#z>vRlD=dWvUl>TYYgQ{c4@lO8Ojp%;v
zM*iAoGBJMk#{YT;@*{yRauD5hpYA^oB8(hF2FvBIxA@TgI7mJ2GV<r*#=j1tb2jtO
zEAhws2-J{+sP+2P{BsaL0Uu~OL?ct;EP9&6&JrObDq$)o+2r$vFBn&8U=%A-X7ooS
z9>Um}K+B0LD)emkaU2F&h;z7f68{cYM+s!VVM3)5J%5-)(U!cL7RcVO%ytA0l!k**
z<KFNHg9cb-MkE)G|K552<Ao0cay!Gmr2I3rNt}(&deFeB8$j@ds=ZZa#_R&ZAgaqz
z0B1x#VM!ZT*q;pGo(FCrL!u+!q3~NkK5npuw=)UK(h#`Abvu2<WaHmA5^`828m}sM
zjVgV?k+&w;&;;}HG*G(WTLs*A#YlF|G=p-d-i3?i-ziSxA)!rNo{k{U{2t4iPd-zG
z*)fwp^_+ivec3ZdG3Oca&auHwPm{m^zqS{U^#E$TbK{n;K%{<^yI?Xy9TvUSctzG}
zLYw$29=d|5Fs_+Cc|W<Jj&K_&pf6F;d5^Mu;{-gv3n0W+@Bkwkq`CX>qz~jDy)Zpj
zLH7o|S}gY6fix(o9=G$Ewnha#0rGGnH_|Quzw2Ep)5xy+bmgQ(ND5^EEn;#pYZ3hl
z&}$FmeZ*Ox2X~;<$wC1ravB+nY|skn>;7GOO8luxQA+cZCcom)k5luRX8xl6{*eaI
z1d$m#h8S_;$30M26`}0!Kji+tGiVr2jp>SXqkRbI2%60A;TS{-o`J5(nW6u2p>VVy
zp~WAJ!LPI#%+-rd^w=E6#Ad2q;+{O3pdSJz+sfcX)tkZUwz&l07d5NQ5d$mi!^uhD
z1GV#3tst;Ff?*XMg41Ugl&}d|3gTZtC(<S&dZjXw*wxVUo9YNUI9PL39CNUhFBnv*
z2>+{r#sjf}0YZXVaC0=(;9?2XxxfpEb3c0roq4Sx<<*Tm)!7=p=89s}V29~v*PFpF
zR>74l>kxz*3I-w553hwSy3IstWnPcr)yUFdwYzC=zmG`p@JTq50S)Pf0fEF0a?p0#
z_+ZzmT4dIq2~E8SK|8YyN7GZL3DDt9_@P~eL5#gF%GikP1FC7^gn{*!7Y#xjs>WMf
zbleJ$JA|z%fs-k97T=_9gGkJ~4q#th15fnJ3Z>=D?cq+&Ncw)sYFuQG<wPeUU+jik
ziost9bn`@HaBNW}6gAz@D>6H8GS@-CqM1VmgCP?n+l&~_O&m*71J>y={^ZO|7FTFR
zMUY8RM4Hb6%oENb+YBsG1i%%l8Vb$2@v4FI97^kCywlcUWytE<!skSi0!3pO8rk1h
zq_*PEp;I^%S72jhF~9o;^?5wzf{|};QQbsUp;@~;RMsiL(Nq&W_;#C~+#~H1YPi7e
zvFlA3_wX=JzjV@S-AMKP>ar1!BH0h$k>5Ms2OEs9&o3xnx@#KzUdu<3#<wchmqOIL
zULh)zI;6YthsTQeT?PU)Jd<`0;F&bXHu>H8{aGdaYft*(5BZOKt)Ijd0)jOkw6R84
z-jmT8vFCb|PC-lp?;Cbb87NjnN$KZ933$=akIC!%8n)>6y9Z{K0;e{;=4-I!d^<c8
zCo#Sb8oIG4uT+=`0~s|P<TnC*Hi!FLy(?qg9A`p5Daht~<VT@+966IJ+<@D-3%te!
zg!~8=xB_n$I(^Wz19=8>ebwm8(}Ps|a<)|6?o_VZYFpn14V~D7Z3gpjV!8AyZrwZt
zG$3;R{hK8arb>vqZM@!if8h)TR#zA|-wXilw%_pfTdLJqJCh)XE9ECAYDDl(LV9Tx
z;57xUWugjKD9Asna|Il(!=9*8?YaNm*85IVH4WQVJA56~&CA?Y89Q3cfh*u{rzfv5
z0DZ?N0Ra~mOGXK6kSTqf8K%i1+?$N|1wXxB9P#?u8dACqS~l^}P{yE_vt}8JN$I|0
z33xl!7oa8o=9GOmjG_rc-<RO@SNRAnn-;~UXL#p0Z+=&=n3FMX<lHJQ`6L3PRyZ_l
z+YVn0Dah+T8*)_TzJN)>sVk{0vgUX9`t0Gfzt_USUaJ}nrJAHHvoVZ>atkY|)Sjz0
z3yOUW^jt|pNx&o*5ZiIU32o1b*m7@G=U5Ro>Pp#H*@H!?kn3E6Qv?n;O1{oIw|A^<
zM(4e<oTq6Jh3FGR5crK_{e_1?I<O!<>&mYe@dm9;^CXD{o5$+Yt6st<9DQwZjc)q7
z1@7uH>wRe((LY0`2*${phLLL$7;$Ns9_7S+vJ;cWc*PWp9JipPp?O0j+j5bg@>#Q(
zaMOfR=pO~fUu6!>J*3>cm%%lC9EQzIX=_h0-7!3PX4yW_*f6jwx6!Ar+<jW?`$u~Z
z#6yu&uUMtw1i9eE)AA+L!Pvw7Rj(`9b5D97mt2(%JI~$Npr&(BzXyAMmdL&hC}~-p
z2%bln^qmR~GEpAnDgN+fhCTX*7`RtoF4`X^M+Oh>%*N^0e@tC`66Yfp%p$M-p6g<m
zy#maQ)=I@1W{Qy1HaSVL#gGGinG;a-x86~5n(iq<y}d%>(+6s*7nC>>%?9EF{jq#r
zYRBJ<Z@jPwjojLqs4@|T(OZw*Z9JWULK1P#A1{5*k}_Lyw=d$umRPibTHQ=T+<FLR
zb3k~ookDO_ko#grcl3&CNK-<-8Nkf)nJ-Wf3}fpUS!oMGxLy_01rzhcoPYkBJvBu|
z0l@M6a#Qkkuf<tK#z#av5^7;M*fYa|08X?tSW*U?68O)|MSKX5bqe-IFV*4J6X;N^
zPb5|=OpnXo4KsfzKon`p6H~rtxDr$cawZbaEWd!AZLKG{_ZQvTUFQ}nz$hSVE?Ugv
zNzpLS=RP~<mxy=YU%V%n$Y;m&#%kiGIb%trAVG~XOxD`bF={bLSS+<?=)Co*7+gc=
zY>hnqwl*&$iJ$nhx(y5U&PD~jK`CRz)nR(L5%tX<eAIj{cMBhb=IID7!z}qZrM79S
zr(c<cej#xGxp%>DaP*K!Ir<X6)JMU3TflML8uLpT4ne%{&7F4#tvkDeJKwP6lGoso
z*hnco%lr)CS~nKbV?+Q}FEmpsoT>HdJg4DX{YA_YbdYO)>yuCUQ~K}OWL3@EY~qAF
z&T`0<`Jt)m#>DGb4$>jQs6vyPs(&fy)YLn-ok%akaNBX{ehE@<I!cF}SG1~FllE;#
zQVz&V+HP4W*HX-AzgNN!+xT`aXz$7)n5C<CP$NZN)_9FFArJ6;>OaLA-;4-KKr8k5
z9K)7v(1A>XO+7s@eEuqF<Tv)kAwAxB{{dUVQ$&}U<?fQ@%BpNjIi6tCO(T18mK2Ay
zOieAG1(_c?>=5@lUpk@hK`-|Cm1^gG>q@is1cS6Rf9zqN&!E-Q$Xy7*KJWaFo0OGs
z5+)YM_oO=<dFnWrca|$9ye@QVwR%h4pD1)Txeiy>Pf<>eZZb~j+r_LKi?j+Cm58MZ
zcz|dGUV2z6g{x2W)b}>&S}qN|MDr*_aV!*N_KtbmO*dWr;i36?BdmAVsq&DBJ;xu7
zR@qAZr173UpCg-I=2Y+JiNj?5QoFU0Y(>HNf#|jPY>k&h!1Zew>q_`4X%C|KOOfUa
zzQBHy&(=D;nxT|HXA{e+^)|l2{KTUqvC6#-CPFi&i0PP$$cJwj6&johiMj*P>v(5a
z#6gtBWOK+9wzgm7HQ~8Z&HFIS`eMqW;=IWCC^4Pi-&1TfF;?_ZIdcYB{0EpG%nX^K
zo*y+$UIckSp4O-IF3J3!Kw(&QV#@^1|9QKjL^Xs=*^t*X(kadNW`r0{RoE{O-5;+Q
zzMpV{oVV+g<4acc*_#Jb72l03gC5*|r6%Z@ur^Bp1Og`TeP<51As@5c!abotkSFYP
zk~62^IPQcS_f|6=jZO5^T+9(t?wj%Q(OkKngonVTF2a@(zU$8-)v`UQpNg$nU@V37
zLWJyJo5q9P0=FDZ)hju}$ExGqv1rdUDyz1%VAh-hr;>ZaymyE4Q}&w*RAKx(D|xTw
z6wTbXjR8HY7_2lZ8%m<aqdQ4$(QPm2$xZYMjIsFcbcwGy#48+BB5F?|$E7<#*R&+!
zV4?Pa)8_Og8r;b!hl#`712)+)ipp`I4k8kGb(h5&Kvb6}&xu~<7xFsrwe>;NiQsv8
zEvsaM7Ii});Hk`R*P@}M#bjO)jI=@Z&Q?MHVT1PSJfuZDoo=Juiv|M$n0e()rKG_{
zEa|sWd}nV5jH-bE&LF^51ehcjAy10wJa;(l_<g&Yf$Ln46Uz%=UG{)0<bCR}8nG-_
zhneGH&upGtQ-v5Gp>n}Qx|DQ7Opu~e$)fie51qgO!z1gDpU<$W`5`KFIt`w%pt%)A
z#wT+3xDiD+VwCT{PI)ocTz|_r=Lu|#(A7`h&40_(>F&Y>>252fE9fIEK1Z<jW!lhp
z*yV^3R)B5BD94<oOrm0Ky7&RcW>y1V(+Ej-R>_1c?6V?Hl6^6Z3M56WJHdg0foagu
z?rgIwz6cgTh-b}mM&O=R<L3RoR4Il2>OA_1HM*~$-H7T?({o{JE@FMbP*I!_5mV>z
zI}=1+h<_#cGOKGs<wG(gQh!*@lk)mYBlSz(!;|U<D={j*dWsT$*5@Gx^AURIGx4?a
z=l#Iq{(A_f;Xr~fM+PSY_I(^wL)_6%bP<^#j1_O(ZK4l(ibKSpqQy)oj1~^e*r>s#
z77(n2+U(lCuji%m(h`LZ-!rKKggt3z8won)FnQ<*Kv0+NvadFxB||k<h<RFppny;3
zeJ;C3=EpN<?sLwSt-4YTtRtf+{O4=+h@!nAZ;Txb2R<$jvbZ8{dhXl~Q4DC3j@N8i
ze}fi~$F^%7Z>2nO#ibC3b46dsW)gGEV{2^oh-3s~&9ac}2N9#;(DiK1eQAi!ysu>I
zcp%4r6Jfh_1yZYx-Y`a`!D@$5{=37e4_Py9BKS8~$`-j|n`*YHZaUX2^o3Bi(fI7H
zguyzDKU)xoV}lXN;Cyq%jeeL#$TfsWX=kJ=uo8&tStV_Un3k`=C#FhM`K>LO1C$e8
zUg&3263wgGt2!l3DJfZT<E8EGjGgGWCOUU$YQ4b*JEpz9u0ico73q6Ue`i_0Q@c8x
z7}rXRYdLiidQ5}!Zk9^Wy#-Bv#YA)%x8W5q)}ef3(c5qk-^JdtowyURYE^1p-!7d|
zhl7$VVd>E2RZ8%yF1ZRFa~bm(y_oENukTm+p5L8s<Pq-BSI;x}+FF|!CXnR67#tdi
zTAM|ioy{bFaUZOLE+AuYGq_mHJK{gbu;?i&G%)Bx5km4R5$z#)hs9VsL9&Ft<|f?-
zMO-A&Ga|7$`A4Gnv;vdJHWzo`&PPpRj>0=2MxfD0_zZa$AKUx|bN_plrcpqUR^U^o
z+Q45PuM|F%-^V<a#_##S>o$QOpDkgM2Y;lHgKKled0eP!%lJH?ZZJz6CwzQ{kXYcu
zaamj~2JI1bTavUducqv<Er<yQWc7$Han|})xU7>MeW$;=UER2E`;ma~ai9(}VbQZm
z2Fum4n$;d)IlNMK8xt~oY^NH>8JsGBb_KG12Q!Dvqx@NnIWoF*KX4CN6=`S=uf#{C
z;oA`IZcl~C>7UI(8W}D1SL>9W=wR+RI+$9_a$!Uo+%vZ3Q%zYjDP^I#X_R-!9?hlx
zc4>IB^cA}6R%=u{G`G(dm6e3aesq$k*{!|b7`(g`F^xlEJ{@sc`FwHS=^B!OzHF_K
zs5J^cO*GFT06}V3_h;)}hC+`x$5@pzC79+0&~QhnflRT#B277Z+>?DM^Zuy$Jb1<^
z_FVw6t^yyb(7Pth>{1cT>|m57=7K4|(Utel)z^Us2;FO0jSQ#cFtiX*XYOu_m^7UD
z+?Ddcg_28{E|uu&6HN!ScOgn^ZZ7K8ASgfxkcyzB<infYJX#{=mtY^@uled`?lVD5
z9{z9f<0UrFUm0ZDhKO#GaT*l%jXbo?TANM2QD`dnW+i7La#EYVgAbP3M8s`uUJph(
z*Rz8>PD<8gCkb!7wdjpK9i;oU!a@HP*UV5$EMY;+v6!GNt&p#~V>fbj`9?~7VitUC
z>|h@)Dp3*~Io&8)QT%u+$EfX(<z{V;yZu?0cwV(Om#Is#8$1fdOJIl~(+6{cso(6(
z|4a*I*8stbp3D#!=28(Jm_PLZrV8C}<sMx9^lrTze|nBSIQ}+YBS>5TFk6|T*&rLM
zw3Xj};PucDOKQ}PkwBYLxB~aGJ{D`0z`(pxJI$?xno*Frgclp`4@%9DZUQ=xefmcJ
z8v!lt1VzWCp{qa`;B9ptwW&NMqeGuL8u$E3go9&FK{r@CXmq`PPGzzhu6!j0Dz?~R
zT)}~7ST1v*^h{1yj13FQP;C?s$i8YUg+K6?xM%_*C?Au-7GAkJB$4f?<PxrP<m7VL
zr_c92zRMm=HA!itIN+jsv4)ynjPm7&pqv?z<;(Ymo_Ze4kj~z2d!BRn7-%P8H&vHC
z$(A!*;c^lGVOBkR^w0^LGJ$3;2A8k1d~vcYWQL^o26D*T<HVABzT9s5o+Rw1DToDP
z8$D87|J)LMPMxpBA@$lRbQDbq!o{|B@@+?8>AeSsB#j#}HFoA5YOe-eLrY}kkVI8V
z`<i`dl7%2`j_SRzJ$gt8H>ibsP0_bhBRvl4spR9Mh{7e$2wEq@+|W1n>jRt=^}*a>
z+^2ZLf)DH>o}6(IXoq%PXmxEY$S8zLA2n}<ptbI$x^IL5q$vZC++_vlz2FN;PCe;k
zS3%F0_#8<7ao@(4o3;wjkqpgurVn0eR>sP#P3oUl514IFOVd!byfnERp%OOt>DXzq
zt(b_jS36;xLAc7Be1VXy&HQIfCy4&&v`GYu+m;W!%VnC5I&vIPZvqU2J>(<zj^Mus
z%72R1cjo}jbb0>3?tp-xF&3AYDU*%p!a0sciiKpr9@tbg9n22N#5_e1p?Vw9Wn^+U
z`6)UP8XJq3^%$5ofnTB|xATRCM$R|1ew(}(!Ec~zb)(I0WK9z5gk4@8rP$Y|_bvYK
zIHrx}fUBMcuJGAKYF-<esx4z5Ik3mg+oK}w0yS6Rf;eNGa#GA%9BLn|S!?1AKFc0u
zC+?*fcFQKWnSj=xne-TEPWNq5+!tYVphEhGPGqqSW<tb>P*=&N+2KnqK6e*zcD^yP
zr$mWa(&i2eHh+FccXT_C>k=eFDb!vc-?`H(#e)D)iBUcxIh^CvI-~!ynXfMtB4|yd
zi<}`*p9gEW3nAZvNe1SJoNTO!jC5$iH}(K~j#YeudMGf=5OL-3yB%U3r*l2H{tbO!
z*oSaitIu&vFDDshwAG)d$9fU=qMcVb0-o3t0iO_ix%GiDSA$A$yx_({{_vBhQ`98>
z^K+fihL4g=+JSi{FUGw;?D=X$`<D50k)=}(pL_*ec9NhM>v`VFY>-W76(CZ(yR!P8
zco&655B&)TC<W}F%15C!Iz2vdOPnLDf;vK2qlJ9pl4P;=Hoe3^YczLb_TBxe=x{pC
zNJH@|*kCIP2Tl;io)>m)b3bSnv{~DM;lVleeN_VW1{K!b3wu`J>^Hq!dE+t=u_*6$
zr@kmX;L;a(g=w?vE?`w@Q2hygrbFv-!C#>fGvMcP)4u5M2;Jh^7;sbjB;rOU{6`b$
z-x1VXJc|Yz-5NaY&9<SBOxguj6eN7_b4+W?0m^+^cLKrO09%b)R_IzkcDgSO)DSFl
z0o4KmV{A{zE?dTj7d_GGzTIejX1Kjn`cBdJmTJuH-B^Zv0T9WW?IZI(fNMmYQ<d0A
zb)&LO)N?`_P!=@NbX$UO^f5oZXWtl`yr!E%XA3f@Y`(zr7viGZU_IYO3)Tq)bHm$W
z7I#kLquDfbWLNYhu7kQ3Yr>C@e|Mv|S;-Q2_K3(52bdCm9?Z5X8;Cy)IhMRWy{RHp
zG7Lbf(bXaOL{}<ZC5?nRQ;5z-7q$J)m^8TOY`5YOU&g~t++uE~c-1h)tI1M;|7h=)
z9jxX1%s0-Yjuu!-Va^bk4-1eT^3UAHK4*1w(^kyCRRf%j0@KNy!qXLHR~A$h=)Myj
zeWnC<EIi*9z@lWg@+|4oFh=^F^9w7Rt|uEQO09y1&ysR2+O@5oT2;YfOcrRszyr$*
z`tN>x;7|DCV1I8>AsMI)>N72o?IvDMk{bxBM<L>TA;|bsTKQbwCvEj-as0CrhOlax
zZ?|h#7uC(kCl)1!MVvjV1HSt0tJGuF0{YE%@A>@Enst0apy}q`fJ|fITCpDCH-814
z6bY>Mxui4SXHkyf7Nir3Kn<_L*9PsFZ2X+a3#58P7*>*QWjW|d$VKu_zjq9PeJnxf
z#IGrS|LjvM0zT8WxjuUBR7rPFz$Qf_i6m#K#1D-AXkkLx@b7=#Ou&Lvq;n~zlW<w$
zp~t!5cg|wkd`6UEYmUn2uq-wLHC`cjnB^JYAwQ6dIUGE9OV5Kb(T~iBLIb)S77z_1
z7%vLv4+)Po_;mIQ5;y~+1*Yw=ugEJ6Mc7FI;cuQu*n=VBgpuNFo$Jo&3eg?Fux=z6
zFr71Qp!6@f8qZqs0=UH8q3>m!OQaK^&!h@w1{xpV;Mg&+jNJ*jv%h<?#ic>d&9=ja
zN;-}%$7N?GdI9kGtosJl3ejAb96oD!+_9vprFdSIvGZDJ0OkQCd47)0mliC^FADH!
zop06BI&YVMbc9v+HOs3=uXx=6v$bx}g3RAwnbe^5c>qQrLW1o6)}O(ZQz<!WeF1?`
ztzBs^+JR2S9^{zo+N)%>6od}smt`;b@sSS;&cSxc!PIJLxQ5vHJqKE=?rHjQP3Hr!
zv|%$e6N!3wWJlYSK94jds?J7MUwC7GFh|Eux5U~g^)Z;8y`;Pg8p5%+z?wSX?R=P>
zFD-duvMM#{0noQ%E?b@V%HW?uU8s_22&T%QeAL#wW;d5tzCuSxxexug^C-NFMj7GN
zwG=M=WD;akIuxW%aR#qfQh^-$nJM8dU0tZ#8;kJt*{zT_-WjXhJ6AGchIiu4&x5b1
zJwWI&kfpQ-d@nW2TVUMb*b=R3X+D?;N=<>8Rv#R502=Dtjvs>HBcEN#$9cy75zek@
z$KAoacAbWA!f(_3s(4qdm7#ICMP9f4lk4(NK>XN2D9bfSQPTgRR8YDTGPx8*7(vsO
zNFDggyWzivQyd`Dm=P{bREGfge1Ab7+ef!n+#zaQSH66OLh2qcEB!_dYn(W#15Mr+
zm{knFW~O}v{bXUpyA4Q#P3I+0Dg-h(Hs=eNaJvi&P5t>=r>Yh<jOQ#(nB!kBxZC9r
z7Cczwo(y>JBS7V>@F<{}?1p4n0<578h<lkPxFlDg-LxS7ZkH8yFczw!lB*@CQQ91y
zfj$nB0kmueNST({F8XJa83%+jE2+Kgc`*-72&MLCwG_hG%ImDPvFAnIY*ErR%KK*q
zz(?-Mv}e~;sAlYCR20LSZ|}ox7aGe8lWHbGtygTmVm9l>T{Kq;TjhRl@iNq<5;1eV
zzbAM-=}EvlDM_<U)-<5*J}r^P6q2J}j+pW*NUz#?M8%Tmzbc-xN5y}StP+T4yHAhp
zRZs4A#x|xDh67`-%;f<VZo}=DM$I0KmfAJ)U~kC1JXSua?^VE45Fz>)=UXj1xVtyJ
za3_Bh%$Kd1oB9|fcCVw_y-_h<fG3lfw)P6)_ClQIhp_3GX%txO@_B%3tJxa#_Sm{(
zV~mCgZ9a!^+|5x*5t*Y1&=+I3KZMzyiMB%p8tUaBO!%~#Vtm)|bHz}~1A`-<b$hJ)
z!W=0vjEhKbY=50mh~UdEsSwOa{9gIB`I+)YSA=?yz(9!tU3?_UAAk@i-7ThCwH;5q
zEdYwGldcB-2^6nz_}oI0v)PUgZdb_qaqfe&j^F0@TlRm{3V-&nX^t4RQ|TVt+g#1)
z;b6HC9{gQ{JnC3$9@k?0ZLvZt27>Pp^UCfNT1MuZQR1VPuFfMH8l?vKjt$q`7QL}&
z$oJgPTp){mhtHFcY&$l`_Y5xA+cmE=YD2ZgU@An#c{yOb%pstTkcO5y_(rwOku7@4
zce!CfRM&Mw1!Cqmz|ynp+G$Z!F@?T>W5JC<=Q5Pb$Ak>94d1uL6h-7q4>WC331@QP
zW7^x2isbt7k#aE!B()Tz%P5B*GLS5z-9lf<=~L$ER~c{8*9kkg`d4jF;v2@{+$n{r
zrriLLn7&>pOstEMnw7#R0V&9G__46Xk8cZvuhq*(ZrqoE9jT1yacLBOUBM^qive-G
zT&7?omW6dyn?6_=`1zG9(xrRoV?oT(f%Bqn^9Oq?7r9kG>kXE>OrBb8;x2uX*&aJx
zCYeK!Gd->SzL&(!{~_Pf@iUgqY&oG1L3<S4=TlGmDFRfOQglOIFF_iilzKLSx@o!l
z`5}nz@lgzG-v=`nlJaLk%x7y$LrnP0owS6>CWy$1)JD2U?EI1x$7#`Vs&#+2Pf4HJ
zn2gB|YUgYfWBHAoV(OK~Oj{xlV=Za;T-6!-R#z0sY~Tjr+Jph(uLOfk&^p&8i&_{P
z+jtx$1=uInhl|!cjJ*!6xr{ehjH{sECy<2qo*oBPYu$6f^bZmO4lPEFBLqoB97X;|
zKsF(;MUwqe)*A2sx;y>+<+~VYlg7I+40e0Mo}m7ie)yWAi^<eGm92slceMm-H_iG6
z=?@l0?aq%)f^gj9^;MRl=S$2Q6NBz+PY*853{Fn~>%S3~dNWJorNp@%(*54AZ_2|2
zoi-`Q3^zT6z(?scC%F`)uLjo|W*_9wg4IDfK)LFGbvh_M2@ZC|YvWhec4Wxc(YFK!
zNd`v5D}(BE%HzOUHjP4d_8_t=AL6}~k2aWLjoT0nT-b{#I?+hM+6yJ+E^tcQCVRAy
z1|bYXG`sos-Y^RjQ-jFO0%O7cR}=3<285y4B|$A(c>vK<m`<Cp#`hB4V)g0r_vrA}
z1o`(d4iye8npdOtlqJ=HJey|dwSNO^v{ipEtgbQ>Vz-&94M8mL?I$jjO;qaI^x^MW
zxwwM43WWVoJzGhWoz$SIXN==i8Ve`NAJ5ghJjkZhNOxYieM{SnL~a3b6x6M9FTHZ9
zxUge$nfNgsy+NgGWL@M!%UKewhuTD{57F!zlRMFel9Hx4S<c<|IR!=NAVrv?aMSBz
zRXm|1gytJ^??Mv~A3<FJek(<t*RA=}O@b<2yuey3AntquniqEQ+|&J<@vYM_;XEeN
zZdD>QJx@ip7?el45OVT!;2A7lvt#cR`<8IK^QBHvI#KCo3@m(-H26<NLnFd9Z}EHO
z-e%UO=i$CnLcQKOAigT#e&vXf$E2U|U|ZY2{8%ao!fGLjstJ|o-7^Kd{ofCwold)W
zYUS$o6ZDTHd0jiO!U0pXrk9=q=Yf0mpJb9QIsy(Pfcg!Hm|Q{vs0<Q7<A?)s{-{^~
zTBz|jG~%7OGKEwc#*KJ8d|Q5Dm=y{{-TuXC$ENBlXOtLcv`q*HoGBg?*0P&3UKb4_
zKfx{ty1EyDOg85kT0mkC^4bY!2L^9va12=cT6z1K@0mSuslZz&X&kk-yp}L{VSG?n
zE+68Sa_QdC1(asRJ}kV<vfVmZi*owvIn9*eqU-$Fe54#2P1(1a-UM3{oEL(>mUnD^
zY4SB%97X6dpGJw|Dl`cab+M)CsiiQm>|r<r1?4#O*<aeuw3Oi@IkkU4N5XXw#e!GL
zV_n})&gZIh=c<@pg6`0MfR+L>Qmh3_Vxl;TVs?d}WI#i$mcnG;N#T@H&~Q0Lp-0Em
zgjbZYSq=F`cVfvla5VzCtr`EPg_|Ug6IF?Dtf~)5ZNGSck5Rj8%4_v(P0pF?X^r;4
z#)i+1zH*{v09KZmm*j^iaO6+kh>Vb~w_u~Ali*9K_L>%7C7qn>J@3fWvjAPt)M!Ib
z9qO{NC))H8n3{vH3^O22e`azv<&j9XZw$b1Ex~+JZF|aotaKmO`AW+w(B5BPW+bZh
z*SjnqUfK5EEwT%!I-G8opzNO>7W3G(cLmVX0?J9V!xhSjh}LH~pEx?j2>twTX@{W=
zUKl!1u{CZJq$J>&NN(G%O=4xNda<%&sG}%4SBbygObP!cEI%*~vc{Lh!W-HX!IJ1r
zAlSS-cu{fV41=*^_8r;cbC!C5v5f{ARs0b*f#&EtqkF6vmKh3JW;CyHpl0|Fj@`ct
z26!zl0*G$FV@z^?hL4cE#uhgQo)c~MVF(WXb|%p3%$M&wh|3=g6q`KD0;&NjL0g8K
zRmCTkwVyZKp}qr2bwZY`H=IX(WNUx`9%8MRxNQLiyGEs}0{|CR3v(~TdY>=(B|m?_
zxUH{ms2|=xBm1cTp#f%{;yZPDS#5d~V%n%E{;}Zl+<1-G>raRV^lupjiQXO7GEy|A
zL;;sXVbb{)!?}~>U%hBQanRI^TiFooDo|D|TRU0nRC5vMgSd7kjE3%nlN=?A4Zzdj
z8(6OujyPWkThN~BDe=%bS87Wmce=^bDU0{}+Dsew!b8%V7h0TmB5pZmqn8m0Nug=8
zGarzPq~1__4lS)f=AQu^r+Ri$JaRhkh&^p)n<`$a>-J{e#UlmM>mikAT){|3h(RB~
zg80mAt-*9uix0z1FCY-6z_`;Ufm1%6MS!g(R=Bi<0->zt(=F54<Bl=_VeY(AEjoVx
zmJGjPIGPBE3<gCd0!#29KCQbUzjBeVT4yI9TMwKo6ps!A%S*OSu<USk&iz%@MB8x}
znUO-Ftmaaf*opY;zq!*bA&RX(*Z`WP$b5N8J}^W~0|vx7MR}R{C($fQw#x$y6O5V#
z8G(k0q?34H5+vd}9V(MkFDEHC*}&APE%$YQ=%qSy(RJeq?zZC+rr4{?w7i65|BRLW
zwM_g3eZ+qtYHg&Lk3g{flmmjS97)?e=KtErh?Id1>u>%zwgdBZegn?Pn{F(U+mfrn
zZ!wqjTI^``D6$QMR2R!vtLSaS*O=v!%LV9V^K%)-s_iQPthw8h<}cPBnoT2BeCHBf
zuEHdX^>g}4p28jVLaT`~IaO6YBKyk(TLGZ!D_Fng`dl&3;Po9IXlgziP8=r}b-e@t
z!Kml6M;`=7=L-vOV5Hn}49v%8(x*bD1(p!LzCRkZ6&Roc{Y|{<QFf;qDW>!<JmiDq
zHyz?%woOx(bCIFgE;#qyCZi2Cz5m0sowGkz?=jR0uR==`4)^yq)xgLZ5g)HLa9^D`
z%dT;0uo^R%IXZv#Yh>$dk8{`w>k(z;htqz&2F)EJg&h?h%JcXUw{{Tga#zPNS(A)O
zHp!-N<`CW0g`RZv{fP=2b=(_Vy78g*ZYlX^=Yl|zysi=$H$U5cZs}GSPv*6YRH&s&
zFeCp`Nv0djhm2@E+p48a{eX#h%EHKfB-+QN<HK71!9dPV_(Q%oZN^TnGIi8~c6e?k
z2h?U)hrpoZfl!W@Hl>;dC2C2|FYW*=CfADW>d>F`N4JAxt8ZPY+#Er*)JXXmvkmX8
zj+QHn4#Hf6TdP44_<{X8O$jEE0`v7_!Mz47LWqToC>SztT*0^NVbceF*IR)ZrY+zv
z#(1DVuB;yOvJ-ufE#fpC;|gODiUOAes@4N-Cm%6k=-Rwzbx>x_IH>QJiFe8A{NQ3Y
z$J+hW(!LH&D9`dT<h{e<vI|El%YQIMzC099Hb8yUoWggA0J^2@GO`Mn*^dnV_7M2H
zY=C_5E5iT%s(yz5SYIuL=q+W56CPJoMgjC<e@rmA$?#gQ=OPE?4(RQJj!|EU%$LUX
z4g6^R?d7>%n|C)yhKsGJ;W7tl=IJlIh_K0mNdOE!tz|p8dT+Z954OaLzX6eWs&5vi
z#|e-LQrFkjm2h$o761&4V=#N_gJ6iHv~&+-S5k!JyqWydUm5Nk0IBujLq1;q0rBo9
zk?fhC`@3tqM6?=+0jWE#=gN3xxl4RkaLJ*5xo1L8hO*To9VeJ$TK9?OOgLs_G^d^>
zR$B4B;V>+K#UBeM%zdjrMKzhAUQ6Ma(=E3W2dMPt_uiI@jTCzrqlNwJyLgg~bhR#E
zAfTb%Gb4QLco%GS?djiK2gz~$-`kf8zu^S7B$ZhsyUroo+Q~4Wb%bo`(=@#F<8wRU
zwG^tNH#X*04PF|?=(c}1@(;bH>I2G+#8kfN1)iI;=jTGywYGv;;M=B$agcSdb9Zn7
zJM@d|n4BnU9=jA?QNFZ8?4$1>wq%h`WR6{UF{E8)yMuR@Ma6s6K2T|P6)+mJqd7%I
zQD{9q(Xo~#uUbEU#8g&*ZDb3WlM+s24$(2y;F(B7o1oNFT{0pvnn$${c$<*k{h1h6
zOA8^n;kFORJO%rxkt!&aFIjidgdZ+k%!ssm_Tj<620$$ATDib_(k-&+U#;$a{{|Je
zX!SAK=ZZgO0tIf(F^Hod({9QB8AsDGF@D6+3kFBX_#r|TBAu%LgFhI?#`md*J&Cwz
z@mn9Z*C@-`jS4MTpjaoWtM+@a<4}>`U9@Hq*eA`boTw&&V?+Igk@afJ$`}I%-q@pp
zH`Lj)RHDs-g7jBwzGi>FwoSTcaLg}gcTJ2@R?qYHL@dLt;RYqzGB1-PZD*nyOL7kH
zf{jDqUiRkb6y@}{d%Slszf65@%5hGc{-I0|g%naY>R15~E6))Iy9sZ+WZE^7fgJ6a
z<a}v)rGpLiLZS1SEjO>MRP9+ZM;~Qn;}SYRr&L^`-cYA21cEDua9>}+D|eIo;K~;&
zAPH`mtrjP$?ed+Lbt#uPU$_J{Uv!2nN9i_(Ls#&F5h>^NUJ~wFP6H;^QAogx_Bu>0
zYqjlQ<nD8XtaQ&7+!`2+8L*y+mF4RxXX0g=cIgzF8rg{Fczkc*^B--1G_ehZw-F4&
zcOQj`L%MjovqYF>%3h=44xB60W<-|?mjoBafR2oqklyPwB7HSlYym@~vjV@p-^~dy
zqrC-M<D87u`6{$nYHHSK<I_hsT^AS;dg=oqFu^&)r9Tw!MVNv<7x1cCr@~AmK!5Sc
zwJ3MP2@pMgi`X5HG6#JgV)O?<aU(mk5^86NQz^{GoJ>L7>#)@Ut@i;M?34ZXKa}ur
znmJu^Igkspk<9OnMA??CHj%q=e+~;=4C^%F_&^*@q9c=MaoH=RbzLW8ywVM5Dkd97
zLzZjf=ijP7_W7fUAl^iX6M;JU;jV<`T^<^QT+VpJND-}c;t{C({&P?J^&3Q5b;*Bv
z><Nuz{MDSR^@*DXGU>-;8Nc-F%CyKG$+@zs%RSWCQegJM`QicJP;D%91=zbK!0pNf
zY`2!u4);DD>PwRIpL%p4fh84(Rk=jw8*#@Cm)=`TUpq(64ieed6c5nMo2wJAd>eCT
zDwfULV<Xe5cKf|>@%SH#4CqQ922(B;1VL-h3g;{(R-0&}1DS&eE(!_&smZpd!cLY&
zc{scz+&9~5DQqrYz=#!X&j>d;>GW;E^bibhD!sAD1{Pm|X8ZnFSwE$+z*3;O>M;A*
z@NmufLr#;G{%lf^x(`@v!wP<WiiA0iyZXCYDW||$ye=PrLBjPe){Yfh4z~r%_G9$9
zem7&FnSyZ4z$F<reT7v!pGJ<r6O3LKfae>*r=VFc>qkPD6;`35H0euAM@T;mX2y5B
zDI7I%HPwRomgrvW1%OWg{4sWRdGZ@Q5JVE$JdDyqUyODb!_J*L!q3quPUtg#2|cDR
zDL~!^_0O|}(0fbT>f!wNhCbRwPqNK&+~Q3<{Av+U4&1@+XrQW?ge(gR-yP1B2}xw=
z<R5$;Ira)>6HLb6w}0U$jxdt~bd`mYimCl+G!or}5MvfP&WS0Z3HuK{<noobMQ&Mr
zDCwA3NRX=<6k&8(neI#qQ9jpMJpkr#=)GFePnDwo(LML+qI;_GPub~i{G;srKjF3x
zjTOwlyZ|Efe3?&tW&bE!Lz@q*PqrInulbG}I;2giE!VspNX*sI?W3K`-LVB0t7~-~
zl^U~(94?CEXbFP4eYS!EbVGWgis_mS4x!TtT!}<L!1++)A4HhjB%5A?!#rlW6OLM7
zs+tx)7E#WUjE^Se^d49d3-*6;I^ZrUSA?FAtiGjw_@aX|(SWQdm{HA9j2(WWe=L|>
z{)kAj$)(w_)L6<lm``m}U5p9^RL%Qk8nR-SUri+E3>PB0j~Au+a&(l0J~x2h+4pQl
zw;|-542UP}%Z|I`$y-+$x0`$JEnj}apMb|`-IZ^2jcD#nLfzP0Ef(QbLFSBi_AAy1
zx^@FUF<;s>deH?k=C;LNE^5SGms6*(j`E|GJNLkC^FS44Iw5wKPSM$-X)r6XbwuHQ
z!4kknp`jb_USHef2%d~lkU%*aKfFrIYiEX(NcG-@-fpj8wa`>u?Q5R}mqPN>RZ=AE
z!H(3f%{3=lJGp-|=B@mNm+Sh)bYDf>W;t$mgbj=I|3p+=a-6Eb`_Xche7&k`2SNRa
z@Sf1Ls}mJJuiD>@afmKl)Ca5c$+M(BGw?_z<VjV9bb>j0`_A4ze08*zj5+#_vTyn8
z_25ftdG3NlT4~nGfWGdXF#i6>HUrA6u+KlL$XMmT<46^Wbe=}mA9Vk(&J(mfo$j1O
zm;2+r8zd}uMd8xES@Wd0xDkQfa`c6!*6G0%g&p(;;Y$f^^-|m0-?F#q7<v|_c&OEr
z#y$E{FiBUzGKA=<<#R}~)M1v|0vME)JsHg<NxO4ULp0#N|821Et)kw{IBjp0u<K<6
z5T^{^Kpn?w4cS;P;P7Eia(yy4Y%_)o4R22n9?Z#b+HfFi<UhE5sM80&_Z}k_?5R(y
zhZvtr`cKAN31CzJWsl+1Ng)>^d5cDh_Eu^mkY~HhH-*wiFxrv074x4;8z-_~dtAjd
zn2XsSKpe*KsTyPxZyY%17KZwRW&smsu(8ZfHy0o?@a?%Vxg+B8nZdvOu|UaoY+bv|
z%FzT7e^JU>iWT&ngThlPl&J_ELl|F5FZ5!?mDgeu1Z?OkzuR8!;2(&NIM-PFTtyiw
zgUgf;r)N#HUNR~~t4v7;J>b6wEz&LZ78socrd8bUI|`%|qv>$e{O80rdl+YyAUn-X
zZm)*yZ=D-M#tYnP^PzjJTeXLVSwWNmTc5ijU{wU`CEobj7M`{jZ61!}pdapjxdJjJ
zv$qk{yEQ5>)bN&X=XF977}ZnYzS=x>tF6%_(<y1(OwW}W2q5y>=~dXvTN7e3helU#
z8VEUmN#>e$9DSic8Y|*bxE~*s^vdS*u;*06$>N#!ti+)2kL3*flbBfhK-?I~+=<Vx
z2dILc)9)1ckw}<xUHG4Kj;$4-fV3>Y6zx4T@Jw`={(Ae=w(y#~^`oE%YCF|zavv-t
z>k{m87J!7N`0PRr8NS!STU){Y?7??r`bWg-Ma`g>cjks@ONA$)c?754Ky&HLOthh9
z&G%VvvWOff-0$C_;;~pCy`$?2=8NIIor)A@fTb>7D<~kYoni>a!DVEn4`w8a;8;KG
zUm8Y4PW9Qd_Eh?gvpF&6AM&Y{0-nuaOr217w8?cW(T@-DRtFT0wfpqqsXO4;%7Chg
zX7Cw)<op=>j$yM^^*%pz^8X)uZyi@<+O~}<f`a4%rKF@w5fP<BX+atmT@q4K(jiKB
zDJd-=pfm^)N{h6Vl+s9d3;SG-&ht9o^Uk;T|9gIpKbFkMy6<(}*L9x9c|?2c!s#o~
zm+zb<jKx^w7T){z)cp1Sa3tYfPRS|r(O|6f^mMbagY8)lHml%M%@;=KyW;~=fq>3@
zN9?Tq4<J>?fRN-uK^9Per^f2jU!K5tquS29JwoJ&4zyVh`k%6dRxxcB<wc=?hJ3!b
z?D#utE&}jCWdKNsQM{O1+Mjmk9N~u;-L!fk$XcY|mAn=OJdbi^NJ5$D*t3%u!m_v@
z+U68YFL(mvqs(phle27Ker_$yS)b4({G)NQprsP)5mB$a`%xp2{5+$ewT`bhSTkk;
zDp4`B*q<l)Y3O6~yl5>`;7Z+z5R+}ryC4|1XS`Tidbub~CO@gbTVec(I9KUq(8m)X
znxx))!S|7+;KBFals*3bl_w2GS{0rz>`FH^S!C}`)(FZno^L>D?*9JeeT_B(`SXbD
zp*PBbt?RNVAgd8^XK~Q}4UiB3(S5S%%?|=}-kHz<?hg4nve}Ng=IaZmp}l9Q-^R;C
zxr#vi_rskf=>P*_sp+r)9#D}<3~$_{bBsV%IlNL1-eB7WmMA~42S2sfxol?H9f0!g
z=BBn2#ghwkF~h{B8KvKQtBCLR+D0?VNsY?VOLN2;8nJ)~a{5!j_ucuPr!WP!(DeeZ
zy|!2muj;C>MTIe-$vPQ?u4&d8oKY=M>&Qtx&4zPRv&aw?1o_pov#2MGo3BN^g&d1C
zmiJR6S2p5)$B)sVcXCa<aXOys8}~9jmgrThR9M^wZvtka*jJz@oZ?4&gvunm5U|qu
zi~&%og@e)@nYESS%4ir4$E1FKijxVm`5O>dXn$;+ZKk+Odl78&>Yl*y$rOC48wk=;
zvp^75s5xADcc8#)gX~IVx^0s!7D(tYAMw`6jOB4?!RxHAS7~c?6Yd~0tObHX179xj
z3V5mS4O`pA5_vxh5OW}$BSEnEY7v^ow6+ooSQ;H4?yjQ3)#s+Z(Svta+zF6is~neQ
zHYf(9DI7tqA_I+4pT;^WHDGHMeBEMo8bj_I&Alqx8n!5CVq+tlBzBtJA_HgCbJS#(
zC<ETf<e4iqo1OeyCrdBRJ-#6R*>ACxmrfcTT^tKd1~2LYfw+Q#EV{Ve0Rd6Y<F!jd
zzVfHV5jgrH7^0Cmk7dH1Ow(xC-6O=aKF&>E{u1BeR^e9RRQ-Cieak`JNmz%A|JLB@
zK%qfHK5Z<|6K&DsCH}6S$L=pL-|<_kxkH;BZ`C@d6G<yhNvqlAT4_J0R<DY<yAY%F
zL~hKJzQWl$oL+9|A08Fm==GtRPtCX!D9TZ%E&<=qz0f!6k0bbZjsNQpBWTeJML5I@
zvp-6d6*g<*$+9;hzbOYq#(elRsgG#7Z)*F5z2sSW?ePK|%=TZ%mk>6gcE9_2`tK~i
zC=rkJZC<jyjm`qG9E=0fIrNb%04=P3_B7-gi##o>30YKWl2k7A^!CNwUV|%R1A8wM
z2}4EF35S&5)|0=|s`K<XfHoTW4PhFp*EQ!JcF(3ua2yc;5|vnN#(c!-Hf)aS)bw-G
zJ`6Sbn#1^v$C1~2WWRbIUMaIW%z>FJA2)1U@kdmp8uq96^sh)N%`Acd8^|8pxonEA
zVwQ%=EfMTIUf{1^-8Rg1e+3YDd0w90$o6;Fgl&&_u=CCj2WuQALnPrb$fJ8gk#wPT
zTfKKrH=aA2;uVh?BIczO)s`d{+%hQm3^;WT{M857(zOr+pvmd4I;OuJhKZr1CBs&q
zfFh%#6gJp(Vu{hbggq8FdZE|LK^9F89NqQF1GiY;eibmFSIwkE)Xi@}x=Bv!wSysh
zwp}a4bgPc|*ON;(%tSyp>2IpI=v?0z*f+lYh3&>w+eVJcI+Oy4)#z(vD_1b_m2_w&
z-WB79m2yYZPd=|x0a?eRu9B44R%cW|tf|H!p{=ZtJT+KzvdPcNBx6AAuKCQ%SxSM5
z?Ab4OzWzbRpmD_Vxm0$x$K25AY7ZB5h(o9jaj6s6^<OQ~f=&_7e?ZWRgU_F=vG)lk
zlk|h1-p1#s-MepmU~>tl6wV@Ur!uwmuA@ggi?}`{j#JF6PCR!YQ!(b4Xk<{x-uA4i
zDpIC+Xn!rTAx2ow^R7=1)KUzrSY@HO6cd@Z_s>6PoIwrTU=Qcb)v71gSZ`+y;i#Jy
z?_TETY1`wAgVfb<-_GMXWo(Pm*C<;k*RA<9{QjGe2cQB}DOSB&0SNy}Ah(^qbOk5!
zbtau;xRQ(RkvpUianWITj)N*d7yDyZ5^qK`UHT=2hD#RQ<ySd)b{rRbB6dznC>~@=
zMbZk<u}SqMz|Nd#&8#A2OOSY7PWWD;1jK5k+EaXJ<qBLg2Y8Wn!-sD!K^?IbM@!j7
zE+&KAoKe3j%sJmBJjfVaIQv}7It{pCiIjHRpFMwkbRqLHmxdIxhWTgex@A>v-+LLj
z@x530)h4bvS=Vf<L2g;1>RtEC3JzR(`XS56dh)>|`M5O<$$S^SW>>Om{!nu6bnvsb
z74^ub-6qSV3&W@&&<%w8wz9zXO+s7%cW*iaX0eXfnX@?bfvi(3=NG=#3D@wuzqRto
zME9n}!HE&er%e&x>5@boqq16b6<fexQG!h^U%y9zbJ1SDN^LNFuX}t28v7iV&BJ94
znI(EO)h_xcHIVU>T%xnt6e|1TrmDw^^K-e^EzZ{Ruetm^|AZcvh%mX$ant)7kG0eN
zVYexghNE;SiDc{iza6FQXvkF4wD<Q2u>#`iuc}qHe%19_dw;qtV8B|Fc$@X~lPjN-
z2W#A%5^l*8U*}hbea8%9H{ZzGmlcZT?g!BgpvwSTDGRFCNK6xlC}sj$fk<I=H%11j
zi2&|_s!>dA3-XejMxk1m^G3R}!I+y#@ZIF@Ug)jk@UwXCO#)raVJSTKe36+3B%aRa
zeQt-IrJMR%=t=jo>)QpQ{a}%4mV89Hqy{~m^z)l{9WH*?O5FmCQVGa1^B^k1RbkiY
zM7h8x+<ai(QD^ON30s^I!nI(_3>H(koKA)i71*9uR>>*c-J|l@QcSEfXIpt4@D5td
z<<1Df<WF3tUEL`b1HDys1G$T0>HMQEb1xj2B%j!DEZ_R!#UNRnqx|kBjJ^9Gv^<(7
zD?)kY(PQtP(ecV<m1IxndC(wBXfO{AM4$g<D`W2(vBWi5p#GDnzR~`|#h(>Eb1+)s
z#9`bK)zmNlln$qc_(pw1fhaEIZvd5TmdCj6<fHrGRDfpr$1Esh+}h)LPn1knlWSOA
z{e`J*Zp4I~qfOldd3-)GlWg4k`h>6nTl3+_08S^>*XD8`pI~hAl8*=);7_33sX#YB
zV03lAxWa=`CMJ$O{!5oO$vf==_{cVLj5<$1RQpQ3OBb)RN*P5KUS{uZYfSOlp^t14
zJ7PwYK=1`&X2G1yMCE3z)<=&V#VH9?B<`2IgVB>9UpIN7?aZlB$iY$-D~4J!B7hYl
znsaDd1LTLI#WbSLEIveyo=oe;juXC3pnT>Q6<~2y`>hBnjk1^_rC^7C(6^7|w1TEI
zEL}FPK-)i<jv`BlUrTp<G))qgE`-H`X_HZM_uc<dEB;+H{`H0TVjS#=IxW%E6KQz?
z+!?}!#OR54^*U12ln)fFX8g^tCfm7>sw0qgREFNZVp&DmARftMzAk>A#N==`$Os|Q
zn*L2e33hN5dP3`uOVqcvrsO|67p*qv_P>_mu=D%Gu!g_;M7Lnlz!dzy{HmN5JAYhj
zIccdE?@RUkQMlBXuMrLD8M8Tb*g@h)>CwDla_(ju=T*U3)p+6&M40#OJ!^ild7E&c
zHKKjCPrJF&Gf3{)mP^?)YB5M-NIR!hnhF*cc~ddRmuYD<+^+_AzC3h2%GIkrrF)pt
z<(X?6e<YBx<NUk}jO0JoZ#3_YPOW*72$eWL-VuMxh}{f7V&TCs-_C^r<ma>l7Kq&G
zT<lUmlrN#V?Sq)egvhzi5tpA<26fQh;+=kU0(6X`lu=`#N!``qb1I+J6LUd%g6{3@
z6;{#G#+%3`7QN`+*FyiXmm=oR`<}KLDPu?3kt`Ve?*5MF(cU`!Zf##2?38y2i-Im*
z8#q$WRS6UmAifvXdaZ%?dWZ|9gvqgZ{`Bhbg$dLwZA>SI(U`Aq!0Qmyw`Z|iMiB|4
zXRCK%^T?p>325!Es+(tAB2(%gI6n5;QS8s{XGdgJ7@LL#9F9+z@u!V?jmeSqWy|Y5
zJe~2`A(y}2-Yqxrx&!GSdzsQjZ+%4zg4c|ya!h=nP$n>fK~i85mZ;M|s2G2|d>;)9
z$A$2Ntsxe(LB}~l?*`4YZ)Crhm7ZR<yZn?SA$U!kHr5#uqSO38wfb(7<hc88l*VRA
z`6Tq!?_JfQXhLPkz_jwwG>ik>CDO4nOKMrZD&AN+Q$(joK|dUL4;45YO3Hbg+OPB?
zKUfhcMOr&x1QOFLDme5`bsseVer<C4GxYN)_uwqvo5kNMFeqMrAj1{8R|{AM%2Zvw
zq>6j+^22KTc6(^;E^AnFAOylFcqiWwoN?R0eKAP_ByCQ&DPGJjBlVO58}vYYVTwC&
z{LlpA5wWA(dBwGzC)b(SPj)q0GHPDleXbOqEGU&;hb}{#zaK@%$6Fvp%{^3Gd+eU1
z8YyvfWqemvcY|It6cb;d>dgiiAI}$Pw$BFJ=mRSwtd$D#V=^HWnk`?5%>WljotZR8
zH8idwq2dOiM((eUXn-2B;E6<8i_o{947$`QLC;iA9?y`!p8w2ee-+eD$^ugchp4VS
z@)??^-<RQ7H#?H!q?oQ6g=p<dz-mpuZfrfv+W9UNX*)}>{IyO`JQnL#P)I<OPuk$N
z{-eIJgsl@$tIoDspLt#K!1!F(@&jW=PP{)hhyXdX59tFmnsSz%ms|mHkKhB+L^R(l
zR?Qz>=bt;t+b}GB@9{?h`v@x6U&mWpu9XWLwcFJ^?n`RMqZm3hob%xHt%|s#O(rvH
z3-t^k;tBn$4edHlneMdfw~sD=&94KGVph%4oZQQf#K*_yxXp&%8;Bl1X78cCzGi5?
z?|1}(Ys!)7)|W#vY~s!lHi_1LQ6%PFT>q-MqgH0dFRQ!-^5LG(^1@cCT5oMG_g9G5
zMwWwBsjzWtgyJ(SNhIAxR2mCv4rJ^Fh{P5&?8p{ZQ^05`E~bVHsc7w%Cb-dF=LG{E
zR9v1o)dAdWGoY!-P)z5L8lA%z+=`e~0K6{G)Ya0Jjp;t3Vl9#x|DDrnd7EF=XRt#g
z2No-I!_hidUueh?m%mHrdcaRB;tIYoeWU)xYd@&M{5HYOd0`>!^+KHo!9xC(hJjk`
z5#gh?rx~8i=O8^JTg$q#iM7gV<Yv`!`AGg$Upd`m*@~cR4u3*KRiollR+Su)caF>F
zU6}RU*D(S~X(^5!r(FHE)H+3xMobr8cpwb!#b0mgb-+Avlou3-dF!KA++jK?ar0yT
zvjNBq{s90`A$~1}u<W})*N%85D{aif+I<NUEDIEQmx16JGjDv!eOr%(k(wB(#u%Vv
zydX48=6l4-epqD#1T@vYlwRYuJ(psd;kI2e7YPD{mZmHmNx>ySfBK?{SR&R)ewOJL
z$=#Pw>T?w3tM8WS{67rFelsgj2ip5b5~r}Y-jraq*r4Pij5LP%ez=#EfKKyD#2Za!
ziD8m0a3SLa!bfgezDvF+hQpxfEfd3>mX0v)Ow#xgE5$;4KH5xDibawJ8~ZG1{^Y<U
zr2?IpmJk)Z+xF=E6{btxU_jpkWqRH)(Z|-J49RB->_xPZ_ZQ#hq5MUn0`{hbfcwc!
z=rs-#@7KtpCt$uwpcX^!<w0^f50g+_0Rpdt`a7Y)H3Po`a3s3HP`IUjE9uB}dqEK#
zL;I1CG^`<#ASzyu@CIVy-3HEQ3vH8?g8g!2e1H1Oy$X9=m%|;P7QK)~ND1ys0i3`9
zZXEJS<^R>|{-gE%>*Y6TxJLqk-+u4p?nKbC(Z1F$iYmU}VR$h@>K<VcCW{qO>+>s@
zAd)^sWoMvRji!8kszn*zFxWbCR>jdip4*8_z;ggZ`+pdf^s=As`uToLvdPbYwT_eo
zD(jQZ^2Wc;-U2T_#&8Wx^W*#LO%JkpLj$CkeXnb6ZR)}km_{qXM880tVzB=$xFX!9
zc3(T)^y*c3hK~<=u3lYjP`jOu_j`TikW<Mc$~c1Lf4WdD20b?IBNxl7e3M1}6nrHE
z2mvBUM<+J<qbK|y*MPbn8YL7&phR!*rxj<q<f*RZ9wY2e|G2yN(D+bgKTqYipKL_J
zyRc9jqwWa)u`~V$a{IaYNijm%A1j;M^LIwT&vi`i4ObC;3SadfZ?B#<H3PM+B8L=y
zer!jCj|-bQ$dmr@xBl_cdx!>ZY!N5^JjNfo7If5yC*ol9xAuVMB__J7g!C(+G@ajX
z+50`ZuE%!&dkmYKlc*D&M~U=bHt<v4t%R6cEf1@a;p`U3KRlJZ8wuaryBU5=1C8om
zPpZfbEcj`GChrPye@|p1oYD3U^(MbClr#Ep2Qv7({^JJ!@iIad9qq^ZC60mU2ZY^!
z-aYioaAh=)ZODJGN*eK>R~Ctvrug@jrJ}Fx)};o}>cXcKK<6v^`x}3rNq^nR0CsP{
zCz^w|TZ{dEhc*879VzzH0u@ZpDSn!U{=6`;XpvOY7_)Nhk-y(=Ak*Wz6NY3RLyOzU
z=>M70_1~V$k76|4)OG8Ig7J)}JN~TB3DBIa@zF4||6UYIKB%8{;X462$}08eFHD(J
zUMJW=%##p6_5Xid|8G97Q^|_y(j?`-ZYs?>`Wo2_XUCo#NPxVNQ_uK&z5cao|NLDn
z;*@T_BmtxBA45Em7|d^#f-?FB#DA|pY4o*_D0=st-}|;A-Q%wLoHV%@1567-U)*Q^
zbNBq`*EIW>-<n)EiX`s-y`WHEL?B@`$g}hI{r#K;FrF635R1NQ`3sH^nZ)|W#kanc
zt_Tw(GX-C{e_i4<r{S@sk&OdrRf_W`uNy3il_9}y9t;k%p~`-V)a$<X>u!iWh5ac*
z^y1CGmtvI6sV-(|hJb@#Z<2r#Z7(DYZ?I7lvd)X974U!Gi+}8~KVL=#!hK{SGP?n`
zlLXvo!2Qa4rJSt<WF?hLAgs=SgB0bVo<6{clTG0bWR+L@9neuNGy=}}0K5PYs|r9Y
zNAc`0kdS0s%7Z8Ly<`Ehbw_;y)Q`Q*0Q-T0^sl%t_y|Sp2;Xzb*!4>M^dg0u35c<~
zgnuV1uM9k)6fxjV`Nxy@=cf*vGrZf=RJkyAYee@3c<b|(iqXmWPvDiv3{!~<)YPl3
zEusrd3NK|YYL=LUGB`_1Nu@#s(hDkiI=;66U@5~?nLSbXc#lpH;tK~j9e<uTl~}NF
zDg!hAJ~F%&Fu##8T#)%Y#N?y=sg$0NT>=WhV8NXJ)c%hp@sI6<;0*ejq2O7@(>ky<
zNZs%TkS{X8%;CB-Q&@Wn?R#W}^$Q_is~AiOgNjD+fp3jDMALfhb&Ikprr(dz<m&J|
zT5myxjSk8ZN%tjy_^by$721InsQ&MH{U{5&Zqhzo{6JxMMNn#30tLx?$I!r?j!6ys
z&$@VmfIEHdHqUQDA<G@O0G5U^y*!8?qI>H8uWJ&4=3GQYN91h{p9X!92x>8cSz`=D
zV@P^L4#co4J^_`|Tx%T_p7q;_v#!f!gE^Yp3o1poF=R+$!^$nkZbSEz4x|ttph-nD
z<J4D~b|v3%Tmr(}OptMg^ZF!Sr$0Vr7Oi+NaxS9?0@kR8i;b}_0;lQv3E<QU(1zqs
zNiE?VLXxKXxmv@7Ai65&by~1!cy{yQI0e>b;ZMs~0|Xpy$hUu;$v;2Ic+}?>FOos_
zds~h^C6v*CPCPS7Fd#;Dt^E(g@XzJ-SVRd69(X0JbRk%&GDM`j=Jea&$67AdG=`{c
zf@g;^WTq)sS$rHvA((PNS`b8eaE#p47~(JT{NZX2&IJI{wkA_L2!qH3RL}A+l3h@#
zu<|eoDqWDuOaNt|<`F)`Ih*yQ5A2&Q3+xOTZQ%y{B3IL+LdxY92T+Pp^?SXyP>3RE
z^%bH1lrtJ_EuOji%5j-b`W7BGYT*&UhEw5>h12w(UBDN1(<lrs@?YMa-i4?w$nw78
zYrxi3*6Vrx^|^Y1dahnvW`|EZlieExq@Q63BGR#Haqo-${cM0}Pzey<;m=t{itbwr
z!XUjA%#m0>M7oZCyfBCq4&^xjH^dz2&si(9qB~+Ln-C|Y9DDUe<$VgM-@~DtH0}ps
z#p@|V&pkU#i_9~?tF*z+agG2sV<vPnNRQygPMFHbbD6a3D>I50v7lu^Y=U^bfF)`>
z5NSnU8>PC=dg1R92nt(g32BKTv){;WJq_&Z*Yp|gD1u@6>FWu9oyq_GckfJs_rcr0
zqih2x(Yge8DNta=BPn{C&AE;Ya9~q{;rY+d9<VL^vS0~K;RA@38L&-#0g<AQfd>34
zh}2IM<NSp+vLcBQse1+;fbC0yZD_Atp4E?oZB#^cSal|q<>=c;n2pL`_e7AR60^5J
zr_b2WzJOYWB1qI_5nsLVcf|}vDO?s~x7qKqzMdX+S>EE6JU-ltXL0`N752v_go=kN
z`a6cKNK_wyroJq>OZ{Yl+R8Anse)V!zUMjDfot;=HjvCP=s?9${!ULFQ7NO@kcVJb
zNc?=EFcl<jEPdrxS~5~ng$qH;f^NGu?1?<jVv|74RB1V09p5&tHejxo{;AZgr+z_?
z-jUB~MK;BKBhXj|{U&WGkQjsEKjufx=>avNH;sN!dj*mD1D+sG;Z9Vn58PmiLH%|o
zzGnV7%~{=R&|EdYz#98?pn5OHz+!&$Ik2Vhuk~%149~EUMGzELulGeB{_~vvp9kCf
zU|7m-mqmY`BbhR2;A^LDNFJ!Rxp9}Hhkvztiqs#%dh6<VZQC!BAHpsRz<!HM*HbXD
zPzDb#)$*Yu+hK^a?vcDw6n3=$sKYx!D-tiC`r|1UrHgNW&w2Rn(V_&{0NgK2d7mEJ
zOYs6WHbI01KTM@dfnRp<wIrD_4Ym8;X>h0IH?;CG%+ap2V?D2!Jn8~lq<NZ(XgmQX
zN+1Dyyd*iF12YU(f}r8Nlk==vd}FrbE-IH1Z01;n1?*>KHymj~%b$ZNjL&U{-2;Td
zH*CI%Kn$X@Z1{z)WlllC`QHgz+7(dg*p1y2Jo}e(IKdQ@Q<Pdn=mv6@9p_yC#a8+h
znl2nyW7t?fLv3kP33)(xz;G1N%8kM`j3_{HlquwKkW-DS41l6TIk$Wc#s8>joTN~j
zu*oEU>Ur$q$GUpghNk}K2G@@dzGZ}p43$x0i4iwpFmJ`Zzf{uI3rVErl#+&Xs4T2l
zJrhR4c>hYvvGQlu5E(8Nhi87L`6gKZ0JEP75T8i30!>^4IR*1H5;K$|OoE_RlXeud
z7~#wBsG-N0$c4KyDp?@)>b~@_R&;-JX6pGQGt?h&z4ud1L8Ez9uS)i;-axkei*(3R
z4iz$)n1EK6_Opo3YdGucMGXGae$m3MBSeH;`)%-zV1lPEfKDz-6gCqd{AYjfvj1h}
z(Hx-8`h$=jG|bEk-ux?Xp8}2Eh#NFhUq1tbJ%37spD!WsLyxdn;RH)>1W*wMR1r$|
zy}+E{23T`2SUZDrOb*mpG2tZMt6)|vj)odDUQm7o5wYw9^WrmYV}J-5X93+NLQHud
zIOYzB0xQUso`)}Y+IC|(6G=Y$Un&wyk8)TV>bdT+HLqwfTKx70B*VP~dX2)zD27+e
zlJ|KZ9o(=QFAap_Q?@r*N86B0x|uRQ`v~Jav#kgeMf1Ssw=ac6d3|D2oY`ug^Yr<V
zUB18~5%0gK(*d%F9GEA2W;o{raI@!6Jo+x<Zy!fDfSDq~0_g14bhNytcw-mPV`V{w
zS-MgG1@YBo?ZY`aN&@vGpup!qyx>F}re-;Cicpy?dAJ73rBVnG{@8R^1zw@?WD)`=
zY`A0=uHKb|e)jxVv%-d<+H7_mQiPm+d+SsEwMRKXt`&p>(}7EJyK=qx(s!6dwfQ%N
zldvOG-$>@!JbnRe5Czz#UlXFLkI(LM=yzdUqkbEqvqEqfEEZgpot^9&WH;L`JxNmk
z4aL?0e%Wwv&JSQ1^1EBXt|xRzQy;JNQsxyO%31aJa`_|k;nG)CUV-N><rjWqd_XQ~
z28vd_Al*S!iWcf9SgGE;H>Nt{<#GnoBH_k^+u)p4-u9_RYsEO~er;0<{YH=6t&8fr
zUs1&>V{Q0rlN6btIH}of5LTswwl8nzC4Ll>Cqqx^^jCQK{X(!h+Dm{rkx#he>DHyc
zXP8DgHpV{E1ip^>UFyZx?&GaxdC}P^tOmGxXH2foc}@9k_a%x4poaKwh`F=`Dq7lA
zQvVYxaT5-dq7Uz{{yP2}3836=>Ar8H-dj~DVfN{g`q7y)d#Bt_37w*TjQ{@Jw=-*}
z|Ie2+{RgM&khhM0G#tuTpZMZuT2t?Jc^w;(VpZSJc!@p*_Qcrxx-TuhS5=D)=UtT)
zDGFksrzfpOj_7;i&S`{dz5~A!)op&K6-(_JN4frzXG+{acbVw-+xc+G`t1kvPkF^Z
zLarz17q-?HT(SCWzxd@W2b%g#XT^6#P-nQY4ZL-j1-vc<BAv+C(J<fXba~S6#F2Ep
zJ9wsVbA!CVbAF#tj9p}nk7K!bPXb@s3UD%u`<+Q&Of|;ji3i>;7OsQQF*<t+XhMwO
zd2r-VJOuu=-FsIE<kKRykN2?YfZhL)$m&z9$Hys*y|cmmmdm6Pp!=g653^43<!Xkt
zRw>BMe9LHDtISUIy(463TW?4E(&t(NxQR>5p)s(ewtd{6ryhE1-1$zZ>Tbi==XB(}
z2aS^{k3D)Fk!u+X-*0N{O{s>9Q{5cFI*aXG-4-B7T=J!jtQq!-)sj)z`10*~S9F}l
zW|&**FD+JkNX{nGVUV#`*~bhT1rC%$;9WC}m2`A^GDiicFeN+>Q9;m^*mnK!zvfvu
zgXf`NdqH_tg%WbLM5H64yn!rsbQ4K$YB^HgQFVf(?Do^S5};Si!<qWSY22kH_bNXe
z&s!SD0H)f<9DY4Ho-0Xu(6tb-4(<~?Z66;k3oTxIP>9*`ZC)#EW1=unLc^<TDT+FD
zN1yAF@HxZdYhJ%rON6%2W77+Jns)o-!`=lSP|L`B%^muUiS?jzuOd*)?Qjfx1Qh=m
zLZh1bFlkId?Ek;5wcjhTHx>U|6W{pe<e1&?_GyOvqSr*Tt>-R(ZL#X_d&6PS0UnN{
zY{iw}Nf|7Vr_n-NJ;6+7Zcy>;du2mjBJb^|T|#`~9OcH{<vQ;w>9H)dZ;B%udZ=*_
zj68`&lwTZQ{y-UYn4UyW@lCY~6F-4?uAV$<ii@p$!me9p1)CHvn*~<MKZB1S4nh=t
z@!cI{12Ncf&R2vDO-b>o32${mVfek`Ug6G+fTVm+R{j;qcBv>iy=A*CobuJ=Ewgqf
z$m+%oyJpnNoe}NUb?sg<-zzcg+Jo#(9@oe>xku_^53h5aj<)ux0HTMRf+MX@aoRO7
z@1rUW+mgq&aL_|pt*bt4kK;Tj9d=yk`<NGEQ(VD;A&#CwHg5$DSqL`CRpbuO?eJKe
z06KOuT3Tgk`$BWl<`qG=5HN8Y`c!UBr!)d{TXLAiRojAaO>1$*An4jGIKFX$$KJOJ
z4vP48#R_ofmQk2#-Fozme77j01t%<Qda!m5iJCm<4WP{iw#uVlNiLs$3*AUq6@^Ee
zY325%fp;HlaKnHsGYVoIiXAw?s3dt2XHga|5=YOROOjfSg3avSKR)ewxM+dX)81h3
zHtb;%ob~nn&7-rwwwm<iGtLs;;o>7L`qV;af9^hnb7%HK?om{j?Q30A3{47G8~JFU
zE?GKP9`Roe-oL9pnn#4+NESW$kKvd(5cDU4qPR#N!kZfv&GW-HCJo@>h<!w1=C4l5
z#*+5I3pY|YVC}XwF!d5L&SP8TIIWJa2<(@HUzokWGJJhB{!O&qJ;y1jCg%y)D6aj(
z)xeC$Ircl$P}F7soe-Re8J50udT7~$8A>LcF8p2Rcn#O_Ku6rjTIO-+*cQpfbn%K(
zV>=O}?VHn@FD?1q`94U|gxTCd`5`WgZ7kW9Vi}4?np-Y{s2CdNN#DH%<Xe<`jyV;E
zLrSejKohrG3(o;&&GycS>p_6;#l{}td1yAv+V(~a3~+6y2?W18fWwtx+7oqKVwD`;
z%H3qQ_zc<SyeD*i_)KB66RJW<IK-YkP<{l^x#XA|)-;{L?*?8?11C`_yX4ko;)lo?
zA0lj${He^PR_+1FGd30m_#`O1_PFsXwnBif07^>FJs@I5$?utTe!8ngd@>;8O*gRE
zUrB6}&u+8WSRO48ri{JQz?4Kyn7*1Ik$2#c;p;+V2Z&!DD5|4ee;Q08R~NN-qIsm+
z7RQp2;S5vY`QWZP8o^`0wNl{knqzKIG%*8DrWFCI!H{^a0J`+Io6g}FYHhnZw-_{Q
z%(+$bbUPF!s8OjAJoQ*(wk?#pM}Y+=E&$_zILZvhSyeKP<?NGm#7W&&6=NpD)9dr=
zch6#r1z^}iWNOx9tKvexVSx?IXjP)uZ)V_J%~{XE7Vz0OUB`O_Ga`L{yC^!N9-!px
zm$xPx-OOHC`sUVwSL@YgDght(cd#FRl;Tt-e7|+EF`c5|rZu=|PX;LGR)=_<IkbK$
z^NeC<dh>CUB>iB`J@&^cgTmOopxkS#kzP^E^*W++N*2g{lR-c&C2YL1EvV8qvcEc1
zbZ1{bbCo6e2c~1l+>ZWF58S_&G7SR?x?sr3_zf3g-h&<V0~v!U7I7!K)ygIo(Ua$a
zGiVzN_$TjwZ<GK1z(##ns!ZrT#{jO^$=Wf|s@KGsS0m@>4MTr?+9jea@(bMyMNS1^
zM5Na43ws?Ou}*r6NflKT1k02@5pdZct$wU2-W+k8T}m-qqkVGMlQj~9KK<#B6ftWs
zyYt-hUVNu@SE-SFOtHqe1fY!@$e8<UXb&$|iqL9<D5g!nDCnwj8Al|C(&X?2#z}cC
zX*0xYhe9qwuNg2pi!27Ow(t_4AU)h+=U_iplA;y9-<3SZM{UWz62Q&qrH8Pf=kpab
z3|qm$X1s09gcOEJ;LDEOE5Tf}CR!C&6k{!|O$zlHXsrs@J`_q)*x~pT6+3@;3m*A*
ztQVnJZ1drzv$lZPDkVjoT=fF$LY@!I3aO7TTXE>sj_S0QRX|hnI8srikzDKbM8j9y
zS6*pi`3AOuBiHdK!9HQV*h(B(EW=284v%7um1!5v?Ytv(4CXDu%aWF_rcW!<CLXPN
zop6CNY^MMDPP9!qut;uX-Fsoanr`H^oMI6Q(2xJ<b3wyU$-z*VS8INWax2xIvZLMu
z1b}j|?#9(fTcjCbX$M3Fd^v!!ho*Tg1e1<^Vc@kz;k72U0R=^$qcPK^9*E|wp5ET4
zi|nXYesbB<$TS6Jypo*6rhJ_vpB4aSSiF>{_GN{Eioo<ppn(~<uL}Ibx|tQn$zK?R
zd#yg`E_j}8JTf@`wMuAsJpe0StC-%pRf^&D^k==7b_l)o9n)tNp3es82Cg*>p@kjm
z3;plQ>Q|NMZR2r@V_<@7;sB>Dn<@oDfCX)Rq9RDf$+0^78`Ho%Y;OwR-y@95(B{Hv
zH<A#rD&c9nz^2yzmYjN(H9D18i>bLq%sDlBOW<QY`N*EJws_D5+bykKPT+$ygE@x7
zXLGmj8^(8t=N7MjEQ6vx?*qcgvI>8WV&UF-wlSt`Ql<~cYUiTMM!j=)zEt^gn_!>A
z<7>jD^-Q>Fm0+!F_0p6=lqKKs$+kGUT+~($khQ554||WHR$VlEnz2d$Ds-|b_!7lM
zi7yVj+Gh>I3;}AB0McB8b;E@LGr}woMpuvndbRuD?weH><<Y?iYvG8u^;$@-(QK<O
z7b1huh4?%-yB}`P5r{E32z0b@rl_f(kjba?`igIvRNT^kf67C~O5Dsuc=e&47&S#&
zt1bR#F=MyAH3O7<!EdWSB~)bJuEwsBcnX3j-yA<!bZ_0{puLFj_jTjgUpZW@S+bpG
zd79z8zntwtZBzagu!_1FW9{qVvHQBF@{sGeqm>PstAV84=X>WQzpra}Cir)^e#RG{
zo@Wz;^fo*<h^eu;K!i|C<?Wffe@{H7wa1!boBJ}^ElG>>cOU0iF+B>N+v(fHdEQqk
z;nB|hs}IOzq<+tJ{)n4<pY7nhmoKc7H%!#;O&u)v`F*$v#(lQzHe9Qu4f2Z~Mc2Fk
z`(lasYR0`$Pw8$G`u?0!1^(9Z$Y2d~oI3is;6DA7hpLJNw6kZxE>0^R2!x}HwVdEL
zQiA`+uC|R`2j~)v+6MkxBUkegMiA_ZyfgU%Y`@59kBdsyN9$H=58IDNJY_DN@3Q-7
z%Z~jqoRWXG+bqYr@+<=_P2dH3kMBJh{&_94A7?&2zeG53dDPSOyu}j2qVZvjP2&^u
zo4jC<D~3&Yw7m+YoNd>sirv}7hkk;@UgeKG!8dn_CzLI=fIb&D@EaXMU3MZKJIsnM
zG6F(+BE`NuucZ<;k;v~6A$-F?N;q>TykW51;d^g0d?xG$QA-f;Gy0H{A>VdeW0*`x
zz`-C?M}vrZH8D->RZ%w7lzazZ{2=sDc<y}iV6E5#Tl}_bC=#wer5^i|3vDiR&`GwT
z8E@zB<0d-Mo8JZWv7vY$3>-?#jKV)9eM+#f_gm$K+Wm%@v(c(#&LOK|XuoI)7J>vL
zmt*jI`RfrC10C)m_qInIo-aT5WJsPVzq}kwV*(BCg6EIrifA0f^ZkX`?1xw!Zj}&9
zTW>iqR?W8)@|YpgU&ZbjU_6NpVPjbZ9tivzpr&Whv@(|WbsU$RP&W*kCU9s)wlzG7
z>-(Wi%p&}la_{u7RnwS&|BVboh|EGL@c2E`1gevK{2dCqu-6vghvX?55KM&gf8SS-
zJ1`FO_i8ToN`EN5DIbW;AUXAEa&paw+eA+1Ye8Xix%qx4|LWGkyH-hRk4%|3VZxAi
zD^;KJLNqb0+2H<PEx7){9Yl6HcudY$-r%e0b$+#Jz%AE}ql*>qPwRUoKEql$3{q5E
zsqNcEl-6X3?P_oXYKD16XmIS+qVCxoPH+|}LkfceDpwC&|0hyOv7US7Lkw*M=ADUh
zh8eJPq&g8*ZzK;qxMnaZ#*vlcv7^x_&fSY2*m|GDGQgRMa|%lZr7ppX^uxKT&8Y+o
z`j9%O6%}n+4&`?X{$dG{i)vlUskP{QOpcA^<-!K_J`u<xFm8C$8OQCm3v-@ie#20W
zn{fnpc5m6wwyV`;npB4ea4R_u6&k4VepdmS%In&pf_@(#3s41fwR@I`2BKw*L@fip
z)NXIN3>8h_PR;p?tfcV7INo`r@hrq*^c>>r1!jf!J>@mj#n4{xP63cs)@&Tbh1;YT
zs;-!H-!gf-o_AnZx#-FX08$sDgi)4I<dJ<b0V3YR&x+zcQuwwTfeaOG0PB92=Nm{Y
zX#-cfYncysHc_C@!2y`hX&pje;_$PZG<^}x8F;PJnyJqtUcIuemGq)YqUy(p924Ng
zN~T--51-S)>spp}yV^@pgdzDi>>xI94&XSA<Td#<pT8gpZ>fD00Q@NyKnupsnYY!V
z|B#{lf%AL!Nk7)hKV05JMZ)Y-+=B^1qtI#HRCzE46h2;5q&Sh>0<tinBo5}QscP#X
z#W3g<ZsSOfLrWPzGH#!oHq7if73%Qq{b|W~V5<8c2%kJL*E<eTJ<6Kgf@X@Rt+4w^
zvgg4Awr~up#z@2bDFsKUf2}ktE|*+R2SO~N<R`3`xvG97rJn)m&Nuq<O1FluXudIW
zZc(2h_LZZ$YTggjWK~_yV<yimXb$B)a>Mf|`@-D8-NE5UUP@EVyWN1c=D_COh;}wt
zyYjlN4*|*~=LZDML8j8#x*xMSM5Rt&g@TA;4HBWUrVZP;!+rzQXURoJ^0Oy-1$F6o
zlzh(nQF>DbNzQQPQtrCha51sE0Q!57Mp}U^L`TRPZwCgWDuw!@FkCZ~+L5E?Z#Tve
zVlr+{{S;e3fc;fUC|W*6$SV16J-{eGgNe`B{Sa;#B$3?5G>^pKL5WfGC?tPe4$C*9
z9C3oX@UUi^*ff6d<U}GQ6<FKjx{Y%}lBaDqC)f3Ah~*&wVT1MX{TY0MV8(X$tAnrI
zZW$KE@XazS-mn;8FRKG79XTWzg;H^A8az9Kfq~wO=)(b*<P3I@`w!U9c3ys!7~=r*
znr=)gHf$DcllJxClkgJ<fk!K&UkdiYau_AI-K(}g&XzxkM>a|VQFUAERv!_2{^c;o
zX;$lQDDP(Cl)W6Mt4a@BL4lBC(vff+L?qdFIqWUuR5is9@`QVhK-dM0Uk%d&zq+O+
z=@%tb;h~)Eg6#X)T)}^&&l9FYdz%wDFXdB2DN6z(XFE~~R)k58Yed-FOIJATU20dY
zdR2OkckAn7*qp9Huc^tbv!>yfL5T<^{+{Nqx$ZJZQFY|C+o_{cCHo*{n>IdhZbz!u
z=gRM?thW?ue48m%kN11<{~R6xx=&oM29QXizQ(yD&zoh8E?Uh6|Cbz?Z9<rJ_3=B+
z^J1dYbpx#yW#{$UaN@j$_NOM*AK(q<d_{`bK#qfWz)aLPfCWGfvoa=rT5B7Q@>@C1
z>xt=1`41ZE$@f3wo_sA<%`5cfrj7X4%0-_*i+Eu}kEWaXLhA~Z6HZ&p2QY4ODcZiH
zcDTGBeqYaX|7Ca-OCm$?oOo*{N&X282q7?vZc>U01s1b<-|kGRY+vV+QjQY8;k2TS
z8o79!Pv$YiMX8~=uRZnnMu4(L>C14#-yOCW0+uEft4>u&EVX!0D=e&gK=HhG;@U#g
z+QXXTbpk4_f~6P^z2uvgqpjLQ(tD<853d_<)!tm9nHl`;1#rZTvbkjtC{7_M2M5D!
z_;;ivx)i27`$#06SI4vpBmijExR$vPe9IOf!lBp;;yyX?IIOz?wOk6f7;=*l+hS{*
z!;omVY^5Zq%>mPbXAa;62q2nQ{czdAs53*DR1S#1iZi-y%iJ*C$w~0VVkeDqOJsS>
zg~W06LFGNR1ritnuy}RkEUt$KDK6FZp}f~NT+;`U77yAbsg<A0vY)rJr4kk@EHs(&
z3stGT?UGjsg8#xUN7-V6rt7Wzw6`NI`e)K*3PCBjIuyJ&`f#Z5@)cy^`>K=I$dC02
z<j20;hWC3bzV;f0mzOmckk3>Ox3z<{mo$Z_A8bz&K!D%;Vv^d9duc!!yqr}d0oz%x
z#f%Do(ZwBr#3yTbhAX^;pRsBvjn#=ouq?;2sV9qBGDwRhZB)`rFZAU(RSe57gacuc
z1)=mR@nQLq*Wgv+ff&C+##fe#f4igtGV}gjQy=fcdG4R9SY#MASQ$z%!n8275JHhk
zf0L*F&o9<!ACoNVqJm~0_&o;IiC|4FUwZ&I;fv-n&7|EUEDLzSc-^Z$;h3W7wpP?U
zV=e7~!*l?7m8_*$J|~Z1J=w)Fh710CP}9w?w@_6?NcV*!i3S2}W<Le8?@VA4)w|MR
zi-AkH2-Cf!RH#0a?&GX9=1sjOi=0_F`x;sM1Z6CYy*iV`uTf8KdNJ>D=kowb*g<ch
zTod?6K2ioM0QiXgDN_vmE7&4si&KyBiLuS}E{aBxkujF4i6QMH=nkYjqJuvi$@hZl
zGA4&bNJ!|$9Z{J|x`DXm3&v@IV$|oK`^4{S>+jr4w?`)&h(lP&=E0-6wO629GYfW=
zYU23{Kq;T`!k*=z2_&YI_R%7ICKDA9CG=xUnGa>E@G(Jr=KEnga3*AxJ@-B&a0K4Z
zpc~715_oi(79W9o(_*0A%KO|6>m#0p-j~7rT}JY6D(%=idpEtF&1uo{<C0ycSDYlm
z-(2V>EAorc?{u*`%oSf4cGOEe`&zp)$h<FS;$0z|`m71GNA#Y{c4h?|*Sq-3A7+C+
zTMnk!CT~^9oM2-fp=p2=oLKi3`kw8v#RzISGSm~=_JjX`YHdgvNXxfAanyC!?2hJH
zYtT9;fL*}M$7o6MiR(W?RASN;eDrD$47=DAQcpp}5dbyJ?3xy)#ZpY+Z}JL_O$`E;
z(xO!}r|!>H<K1@~z>y+u<p(-Qy3!dPievBJb?E<&EJv7oKxJ_0_;}(6)A&g}sv#*X
z^c&=Or}V@igi4Ta+U?mJ6B|KkinI?f#mMKBuky5u!7R95y*DbbSZoiHPS<A>n5;MX
zu_OSUt!_i#YyvUY;u}^im;~t$-Q>A_cJR@om|dViA&PNz-&*HAAcc++yVAO_NPJR&
zL+BQ=bYe0|-*UY4OT>E6GJvP#6mrdcO#DzFUPo5zC-J$oMRGU_9KwdH+AMbukBCBG
z>jPkaRw#VrlV5g6BL#<U^-#Zh$qz6klB}i-leurvaIO5|S1Pqj?BraakFc!Nc!E0|
zR_8<t4J_WLWZ`>|%KEug)k|ZKAZiRsj($!d;?BL-EeHz=;~CA}L25T4$$Z-oy`x^w
z@miF?plVo3-&!&sR!PY)TxW4xMVD}}wF;GFIp?QJo)YHzVuOcJ(@nJACViT*gj`;v
zd~kaYqZlj$aIQ8n==|u}^qOradx|0JSBRw|9AnRZQ_<>e!ZqX#k|;Uij(Cr$lqj`)
zBtoeM#8qS8%%ohUOpJwkm-BE&J{)X&k+IvHH&4WnA!hnAVWyai7i^+!McQoNC78W|
zj58@~Nx$^WG^}MItOxipsa9tv+3YtV%Yi&wKBauPVzp48S$gM8nU(O|YP`<I@=Re$
zRm#_EUU9;|%1bG$pOXqp^G%H{5+Q7WowYfHP@B@mBjN$l563U<zJ^@+_KJd0cgJ8I
z>;E<v!*B->WIdT=_4(&F_-}o=XLDy^CD2SC=37ygYVuPkAg5ThOU)EH4zmiItvbL?
z&mNk>Sm}IdW6qCARfGJQ^-9fz8NKe*x>HR+Ve_~rI~W#-x*Vxq$>j_|ArOvf;@}uW
z=|Rn7lghmUS-8xiPo<l#&BB1|#b<UfBNXz~IR?jhypbqj;8$^68dg&(hxA?IO|_sH
zsiyI42zRL>UQ0Y~>*1@X3J$P<v9MrlxAbAnIQdygTuu~esa(fRIGmc_+BU+asH>tT
zX|5JL6vln%^KCCu3AsB<d8pNs=Q&}qE3I?ne;+JJQB`u};fqj&Jiu|TOEoqVPa?rQ
zr~j&Cdr@0Z_w8nr^Y-n<v<n*)4|7@oMddn+Pi2|J@6^e8k}G->RpMnpT9xSXCy!E4
zM4pwrV7J|m(PjJy0tsYdNZ_Bo=XJG-_2k&T5CS6ab02h$nraN+Rdedezx#(Q7PU}u
z@tDWhu1|`83aiX$neUvPR|StAk25=YFu7ehU%fk@5uUqo>6X<b-DeZ<HB?E)eB_X<
zE!@k*(5^__UTl}L(Za4!+wL0>y|aW^Pb0>uY@wfQfXTFmCuyr1edk1_;IdGOc)hUx
znd8&H=AtY!sBy*W%b3+Pojt0b3j=crx?8+<rpg~oaXAv5Oin4Zqq>%-!`Hqk9Zsj7
zuoPmR5ejfa`<F%J-8XjzI6DWkID=%f>?+hY*cN>YhIA?ha^QB`ao(J)s4@?H!A739
zo?3~~-y;EU@_n!oY9ud{{ZgjJyIxOTQ#vdaQ<$ay;mJza#`s4fI6Ov|TY?I+U|5er
zBl2yRhl(03l3jOKtk#9W?j#b(BrRW&0EK5ReA-$XuNkquca#Q%Jon}#fz9kMN$$0+
zPY!`lnLt*a0_32SSju+~ec$QjBRP%7#yx?pLOblY1+K}`<%*=uF)Yf2V}4@aBA7{{
zRO*&r%n2Z*M%_0%0*hCPt(i9sur1m({SZ>f&EC5l-wmtfuH6RryuwzB+<h*EMsl}4
z1d9#`@-Hrn-8zCJ80jwNoQCL3;J!NpC5<QvckS_kMV@8OF~jj;J69K}osQYabJ1~L
zx1Ic;gh`hU^pX^ia!kS5S4#Vpd8GX8@zJJyhGK?<!LIsns869pGiew+Ai9B43?8v&
zqR>cliXBbI-_OCq8pJ_#4J{7u&T!3*?r*B~Ln}N6r<K1gOa-1f?Qa;f)rZ$Q1?yZX
zFZL2bk@_C;@7xsJ1<=Xb2>d^C1xL1Wi~=>cJ~E{+7sZ;^m{CntQg%J=+2R=T^ZGde
zJ-sJl_!Dk@(RhHj@WJUP^daKl0moL)%%wLcw~=QI?xgD~`7N3;Q)`h3&7Rr(z1IHg
z4F1tRFq^X9?ALGm?w@ZtCjfQ$dCZr!{7R4ASc?7{h*AW1MAPOgvOUugCZx-cY?`J(
z7iIWD&uth=3ZZmjQq?}M*oP}BsIlu4fEdCf#tjqs8}3+%<6b8oO>&+`fKKZgJ(r?z
zO=qYIFuj3GS=20a3=EK_Vr3{oE`Dz?_X!PD5N*Ur+cRP|rcd{&`GGyhK?>}Ypg4~)
z|EK2|(8BpKf~(RNF@|Nt(tzw(6{_im<ri>{zDn+GeiFhWMC`l;A=-HhRD|9#ac)bB
z*RhLKwfNe7d2jF)9FxFm&>nOQrxL`uvPF35!~?t#<UZl^p0rNT$<UZU>ge~+Ev%Q!
zgtmCs524pf)G4#}oedMy27ABTt=eUBU9U$iJg~YPGJr>T4d@gkYISvoEBuSD7x~j9
z@{5AwGRI+#-(WsyZ+U>6Raj{OsEep1i9OCQfb|ito)+TXcUD@%JjkSQX%|LW*XO)5
zYWZ4~qX^>uyQPfrKpW1;$S){gaslL?8xEkXB@GEHewagX<*L0Ve=hg#N9Gc0A$Uu#
zQ}_9v{3=C6O5IVo`)jj(;-&hEzZOC-YCJAIMe^X}Y2n%WgbOZdqz@$hYKE$c^o6cb
z|F;$Jd*nmA2zv(APkbbVff~Y~6Wt81tzOZs=E>aMDJIZj8oX}TkW$#J#ASn_SD)bI
zJn>F&pU|;&(%elZq|gLdfsRB7#6Xe&nJ0{J0%M2=q{K)O#`C3ayVe&qnq$3Wrg2`-
zqdhf^hxT~kv-#q8Y|ZXe>_D^T&l13^(8(SPUZ^uZ{n)g!kqR(ZwOa+{b?}@vS}qEW
z*?qdwR$j|EO{36=tHTjv1$}Y|7$K5Fbcc%7JayHT^(LTMxQP>OnDX<|4lqw7t8v--
z&vC?26WRx!qLU5{_0kRu$45SdWRu+LBT_Dt5ELY=GvW-RWy(iQyAE((#gD6wK-b0J
zc3{MGECjuys^n(HCCwU>P*iSvgyp_P1DD=P<vBbaajo90QVzqL>#M*l^6fJ*q#FQL
zE8(T0X4C>$xdhf4MG9?0YV3Ll8`;@8e_m|B3%JEvv*sGP&_<wEGXAmx!cE(~wabgn
zFwQ<b(&G`+ddpC~Q8&s#YdAuz7BP}Md0R)Ax7^vf#ookjk8JpH%J0Yj?axv}_N~y<
zjWx2L^+$vNs@9mEpvV=z1CPI)6bH+lN6az7udyjMz4!lj0?sL+h?}TV<8!mXq-&$V
z!G<@iy@cSW9sM8t*vr9B!v4L}Zn6GQ5aBmnjU_!~hC0RO08|@5Q!cZ3=q@1}=u_BS
z8>j7;7!$yKRa=4w+((cZ(S7pL2BBiZ1m1oF0$}hl$gEME`PHQY+rnKuB1h@v`EV&^
z&T97$%1Hb7{)G}p%IObtrf*)fyMe2BY}ExVY~q8Fer2~t3b)G4C8y=#e)Bty^y!M3
zBtv%{r~}^^Fa$rVq*L?<!Stfj)$P4$&B09$R9xM;*L05{T~l>L_R?v(fl|vay$e}O
z0rNC>{BTZ>NCR!S$o-3*WQ_>IB2zxgD`5)7r33e{(*Ts2LWin$!sNjvT}C%W5~)$>
zmNEokPV%8pNmL%1-u%>p)eMn2IEa9cq&McItRef?00r;ji6(KAj`KRLBqdNI3^2@h
z`NPH;q`SP{A}qEdo?=;u%`3w>w7m-126!yFQG`R^&%O-@$N`V%Iherb6=9x$=_)xZ
zc{tk~c%|F*0Y*{5M>osBmq=A-d5PP6ntAb~y!##6YpBTJQ0@H619EDwCQ?pApGXk)
zK6VsXmY5>#T6P`}_S5^ocy#_32O??#RaD?J^!qDN3!(q4G}5O~kU7XV4lFcD1{y2h
zubsvEHwA|h(`nD5m5QnA_XDfEC^=ey2O5Z<(*PbpQqCCHgMgfR^5JyFhJX`K0=mb+
zcjwOM!cX54al|p{+N^YbtAXPZ!C_LqfpX)Z;H>9+PDwn_4M1+eMy3a`LAUV9(G+uo
zA>P8?)@PI8YD-?H<WUD8lSszvpQ9-9165{6Q@ik2fj#}Z2cT7as%oE9H{o|_9=2@D
z+XMGnP|5{n@J9)|evDVUKYPg)qzx*9`{QK*fD%b6tf>o_?}O}wF>GG6dQZ3Pa=yzt
zKlDtW9wuIw{NWPjGYUMd+w84&F&^K%&#^yeA1bn3ynQ_zK=PS8(TRJH_nw%tDrI6?
z4HbHC%yqReOsTll$I<x^NG`6iQ*9K5gkz9sk6Y@MgIN3}aP#~u>0+LsVrD=pnw#DW
zd9KG&pbJ)DR&BUSXMfFZT8dSxJin{zmyJ;49Ph{b2u!f}eb2|SgikGGNqIOD+4TcR
zGaC;!4@}d}*_Ps`G_k4Vq|Sc!=uPc>@veJs-R@-V1vtZ?Qm4JSPm~tS|Dr&wo=+M9
zyfopO!IP=#HR1QQI@Qj$lE-&BYQ%s~OFS3<V~`2Bl+rM?>-N_j0k(299MfW7iZ>oj
zRUJRNT)FB2HxFeV^~&u@XZXN#I=UW!NacZ+virqHTTf{gI>l|5<R+#qsDL%+lHE6?
zqYjk|8xa;}A14_G_MUP=4inYK9}-MqjA8&O_4x}`z<8PumwYRM`vl6P%VBS3O7id9
z`EU)_dDRWN#Ge#bj4$lD!wn;sP8r)hU7~sWHrh-_;^3<3dCA_i7w%r{A9gu6iPKl<
zOgzZ4-L0*`HK<)`A_Wvu6Ideu;j0T_eIsjPBEApGoyQ`wMBg~tX^v^B`dh%@9om~j
z!>YQEUwB>QV>mPR-z>l$Q3@j8X-lDWs|hZ?be?F-5MORlEBBRksvBm$q^vX#@6U9Q
z6aJf~AyX5@hBwc~qde!Giaw41nB;2GoG`jcahqKX?&QM*CXH@OdD<j09vhmE^{zks
z?`&3Awhk2(W|3f9Xj=?<U|T?ZdIz?eaxuX;?4;kr<h8J??C0?GkPQfn99(Q%ZJo<u
zS!w8uY7Li{z(fH7G{fFk(d_vK2r1@3x`FN~c>#km4#UT=ey_*mBG~BAqQP6F(ud=0
za1x&4)3dk~!CfBQ12EGi-<%+AIvK6pH0OM$Gt`K<0~V_*xeEu|mtYf?!Q;|IX2fP3
z5)DW(XsmHypFE1<t2-droZn2d%!V8q&|h62Y_#{SL-_^0TfWJ08OdCJ^b8(5Jg5l*
z<lh|&m^FgOu+VK<n?SXwMQ}l_WfvnA1N_E@+o-(GwVyl}QYyb@0i&@Q4UC23N(hq+
zP2!Aqfv2ssi^0|a686_9`~L%+@2YP~9hT-5iQ@$CfrO**89j^?qpK<cq$F1u_@rN0
zsB0jnH1kt8m$eWiR%5y1xYozYo@>M;`-B=FlLF(qucDQX#Qp%ty$8-4XZf$9?SvFH
zcLP}m3iKYD&`-%DN?|_lMvo@Fo08eFKKC&?mZvJHYdj8w2RiY_ATfB=?aO@^iJX%8
z;KQ9E8J?vp=%&&7rm;o&T<t0<8EV+ah;26%+830DE#G~8*`Q(c*yKgwa=`o>vmV+v
z6!x-AVJVqt7-RLEyevYDvJjO}FPZiAljG*>6_r|d7j=)-(7t!>->-+?8RYyFfGmWV
zY5Lt*qzw3<z5F7g_cs&m25A1w2#2za(Kg(C-8ln}!dUCvY+~d!-#K1^sUgd(Kd9Ut
zw=r-x7;rb_J@NVKSL>#KBpr=gc%6%h`(SfHVcSg|T8URGuatuDCCgL|eV~kD<78B1
zBEuZ26EWFd=%?31dfz9E;K)WyL}r5L3{tPuC|OQNcpaGin;8FJnEf<zh=`^xLAUJ<
z<W=i#7#V(tK^Dq1j|EW=Rl@7zx7Js#4Di2stq0~7`JBO|6fdVGQV<pnH!hrBR7b6O
z`oPx$o<3lHR$4@xCbgCLmPg}$$0KL2y)`FFG}^#~%RP#jAvO6ivBDIGK_*HtQ{t<y
zKV|%k-)_Ny2VsFkZtf8o>b{I27yzr?1hE);>El!Ev`>K-oi`;SJZgv@eKJV#K$4cE
zY4f&AU=l6%Cn)A}Wdk0g5}fetuV)|bEYAqo3*l`{1{#V7^R*4(i}`G!2S<#4{3b<G
zBSow0lW1MLZ^GRSIwf1x5|6-yNK!5D;hS`{Wko?!^vsA=sT(0?Oo*l<a6O!1mRiow
zYV?O=-I~&Eg+kaf$(MUywOm_S{7wjRvyH7F!u9Gk4)d_%N#LQAQbX|dBi9`b#?{>S
zE_OB2ee@do^opuJ(R)b1MPHszk^i=+H2X<by`Y)@jn0dL3On2s0o$mJ>5hz&J~dhc
zP#4ul#sd2|YEdLTVxC^mqH76FJmR`9cgbr-ioLm0HEWaDHy-rTTQQ247xTP=fd>6%
zdWM3uBw8T;40)L(SCqw-7~8&F)!^qgF-#sOT9vFeF)6@zk6^D3gxtaeulPgIjP(LP
zSOcf**_&cB8T44dkFxNNLX&1a8n-T1Izs9EYNf^d8_2Qx*{;WwUaqI>qThW)<}WBM
zRtkUkrG(;825BGf%bk>I8q32v2q#d+oBxNs_YR6G>(+-AK@mies00BKC7Q57K!PYB
zC^=`4OcRtSK@b!r2uczq=g{OR8DS(!QlQBoIS0v^-#Y5dICE#-y7&I|)mOJ_YN{Kd
z`<%1)Ug25KdKQmYD>?dT%Um`W<K&6oE$$Oy&vs7l+p;?V7qTZWIjh2Hdva36b_haI
zjPZ*td*%9&i4Q^*M<$EA+=b|ImP*o`hdhmJv{S6d8pz3uIAun$LIrV;rCjK!p$Eox
zw#)y^w*`yxfS0<U*KzKo&Tqc$2;m{{_W9t!?K$mOT}vaVdDEQUmS4TpwV*;$rdYni
zl$}CZu^+Tf#tR<Pv*Na1fq=pY={M=?;^kSQ8<Xc`qsH!feGw+9^t^58kE`ozEy|Gg
zaOs2WD1N-HuJ7xBb@axO!~ZGM`jM$&RuArtydPSi8*m-oz2r>4x(!kmcrQU=^_wG^
zR&$0+@%xNhuU_Kn!3dzkV!IDUmxXOJxBCE1kR-kM8?>>lvi|1!pg`@-ePD*iDWBHb
zFtZh*8uL7?Ww1egN%1uQ;avBbJ4Vy#O`&Ds0=rPKaO|ekp72CzXoOrn<~Ki*8=g5c
z3>=kgWt0l;Nh?k5HIhL4u_bbp#vWGoDJ7jphky^gMtkG8-0PJ(vd=1g1~oUwpHo&j
zb&q+X#>JIo{ZvL2YXo3=%Q6*>N0oWVjAr43%CDA^L@F&7w-%OT3AybHtf6h>DW#l;
zxmfRVK8I}Vvr3$U9on75L|T(4KXW<f&%CSkX*{Mi8NF3m75Zc@H>;*8<Z&@j(hq)`
zDR}hIv*gT)Hz0+URvSw<vQxBdNJ~^}UYTzc*_s5^CiLR_VYP(e%#TORZf({0s3<s?
z-#UKX`*n+Bsc~(N`TRNDRSpVlax5rwDCvz`F0Y!XTT9HR_i^)(ytK8yuj@Ow2PE<<
zFyL&115Nj0BV_yj)tA7{I+9-d*|1M>HgIIp&3ex^dm_^5V)ElDx=ZspX;Z~CQ#D4K
z6l$slt5VR*{P<0Wo{_HaMbj+LT1a&CKP(k%Nto~Q=iYUZSUozIxMw8N$S_X)XJqd0
zD4GIDz^YlI{iuDA%+`g9r%vZ7>|ZEwlWj9wKi0aiR088>#`#dv`#E&b@^pRNc}{SD
z^Oa+!-hxr|V~Va*G$`))hjmUNh}QONo=j`#o4f+}17)j#SlB9tQmO_RETdVMDtbSa
za5&wM35vUyg$`q^?@6V;RJtgvvih`Jx_+5GExU$dmS0|CjgzUzP}jFwbf7xnHr?3{
z!+0nNcSOzfA16e=)Y+HgiJMt=V+fX3!<5&%@n`V-L8$nrf{5FlZGe-0OFBCzpGD~;
zP(4U*63$yscyI-yKO_CrvDVg|8$w{&DqO$17->xU4}2-@>ix>hEK#ZAJwPEBxs^lh
z3SW{fGo=;5;73i-#@I-~gvh~BQ}tJZud`|_CV+&(urE1;K8fv;^TtO%I{6V&vzPDy
zUi4W$E?eS$Kt<WC-Rfz^_;4=g!#JcI>n6(2P<9q)LqFT?n&-c+%(K9171+G19E_&Q
z%P3$Grr9fyHxPagbllgkXV83=1;9O~<x9-dDIg*GCY2>Hxm+do45yO93c4lx=43oj
zG^eXkL*wV)W{GZ9pM3cel<s(^d5&M>9q>ND0gC~{x!4Cjk1W5Z<z2DmGFhVRrID=9
ze9~b3u*GYn63^-^@z(J2rI!yVyEbMl^OnYV$1Dd?&i$`!!y2^BNr?ell<c~Ulrn6z
z;wAmg8HNj3*I;97GtBqKUN-hG4*hm3hhxp4acxH03~c*HqsQMwT0Rbn&gQ0#SD*B6
zch7QKDC5@Q(G*BP)K$xj(t_5GLOSs~#ZqO_?GAt31_>5gE_vc7uK7s}Z896%!0?W9
zcG+KVW%nG}WjfJQG)jvr)^P2*a8h$l0g%#Po*r0gu3q+$OY=M2J#<I<U@e38N*0iz
z>6M(L-pr_$AE+C27E@DUoO?iKC}xE4u-`w{-t8%}PKpx{lRBPSekm+F;E1m8z7ak?
zdMDS5d|xts7ymrQe>uh0?2Dz>Gmb+-|B-*AaQDgn#@zdbo%Gi(Ekn2v@K2wW0sMFv
zZ=yX<m9yb3!y)^DulvIrf}G#zs!diVKO-<I@qYE&H|$!%`7`ZG7dCroS)gG3(KX0E
zn2Vb|4Wy`@)Vbn`{FH{7Ar_g0O!NJcH{~TMnsg*%^;5Y8TBvyqBc3Ne4(f=1N#xJY
zdR}1OYiRgzpV`l;nR*lLd**3bX~pBs-=r$`cV1lY@l*B(rZdn`n1ScjuvRaBfdz=r
zT4T-zi`*(CuGGo4$b_5~P>@oqt<PQteE1}xBh33<^)k(XIhR1!d4!RPai^*{%w~Ny
z>Ie;y@aFTRhws|pk5s#9t~`TbiuWN`+?*VFm8{-r;n!AL7y)f&6Lrh>mtFUqO}mo)
z=g$L-5HeplavZul(<)eXCzkJ%4dbc#IY#i=D}~d%V{xYM6iT$2O4BkM%n!3*eHdt~
zQEDv9Yd7j?%gz^~*_m5rkz^E=aH7^Pe>}<qb#0?VnK?qF``VCdNVVe1!9i_(f%erA
zV=v?1p0C2&OXr=}icIaBLOT@3noEK@;5^90lx`{z`4Qgk9NiiZx}q80G5>g-JISQY
zRcUkf*~&yuzCwu_YPoAwzBAa#t54o$*h8j0$H5w1$}7lW&e=QeGMf}7_ri}heF$Nk
zUNiV|Zbrp!_#MNHrb1+fI>?Y3|MrH%8Ex?Dt2~=hMl^4}sqA5%f`mtmZZn~$i!G+E
z8W=CO#!N4JP^xxIl-Q36ECErZjoPyGCdg!+)l?3-^Hp?2^LawkN`@VvIXA_mBhFF;
zr^J@>zVs$zzeA&VSD8suc}Hgc5O4j&otx#E+Am*)FeG?An^?Pc#rG8KzVA)`@jN(S
z&!K&SLoKh)rzOIEK%B;)gH7A$!HVkerAN`nrZrzQGfmgERMYr(M!4}4<3AO;xmR{;
z;N(;ubafPiCY=}Kjnp(~I3*U-??^HM+8t8<pShl3x8Z}@uwNwlh(nP|xPy0OrJ7PZ
zd9WV(8g7BxVWjX(w!dQ)(UnQ-$9xZn;^+3N;EX+fQIYoi@Uc@mEpio6^?a8d=+0Fv
z)H|Imid2oiwNp*(mqX+5R;6#auI&iyeyHJ1-wK7c(Bgvhm-KwVusi3sHqFemP=8)3
z+;|KCF3Wb$LCt3Yqq}dj+QkP-zJ+j4{-#i2pi5N0t4Jbd5Mz2~K~-+|K;Y#Az~hP$
z_6WioQh*l5b1HYcG5%XGrMhL~Be1I1No_7iB@G)<I{3U*8mPpJU?%&o%#1&NawiOF
zeKl`p6UD>X!$;<#UjK6Bp#o_D-lv=meQ0n!M~M4cNC>~ruGd`Rlr>1s=HJo0t(?Ow
z@FrlQ3SX7%3o;A>Fp!VM?_i=&K=vnhk_eZLp(Dfiv|E=A3%HG1B-p5m_UkSM6v?_S
zB*m!Pr3(0!%y5}1=Fv~ou$*+?@z0%Xtd^*Ipx!*2m+V&bT6k}fW#YNzpxAzYfpSNL
zOgatf(yZF+-!5ckZ^SGfv(8pe=;JXS=;t~~?(;>UBs_Aj(f`#m?HA#vC;jeDO!Y9x
z-?N`m!y%8#zulewhMTe}L*8rqw=!?e{ZU@y<%?U!69;Ww)Y82b-V2?{zf{Ic?HVl6
zqKfgGdAOI?LoTAHmk`1|8!x^7wyOM{*?Znz<)=eeYcoE|7iqsyH)!i=E_TAmXQ#bO
z*G=bp967J4fANNytGq6i>=cEqREulx81>Y9y(EQ!@$>n~I)*3JLtQtt9G2Mmvqmr3
z_7y3}I_3(S3{YlX=1$+}3m`RWA8IztXHgbiKbumW)ZCWkI&z1?ieD>+J)d3e3CW`?
z4f%{mScX16G;MyYX4^nkzH9AD+du-z!oCCatc|d7v6k#w6nPzf!=Z~}<MXB56Q5}b
zSDtv4pPY&-<hW;ahF6MK*-~~wYuvGAP_2sWpu~HJy_wzY=LM10-YCT(g^fckb14tL
zv>N<m@)RD&zde1rIx0)Fe6(`UgEsPx0Yf=^-TFOZEX;U%cpZ1tN6-a^hopLYUb!8l
zD-F6I1p_A^=##Skrp^2MOTGLeqwqUk+v(Sy!p=-5<#%B2!?gXQ$1cpS534|dVl^gz
z%s5j*1&+$hIXfP?0+RO08)f8I+?^}komKs$mNLBJsvRF+c@@4+hTXWIG54(4*4gX0
zGAJLgdYxJzb9K9wg*4Ewju994Mc;gC3~bu1%N>>sY2O4rKo!O4DW^Ci=w*D_h=Dm-
z-|faf{{MD_Q2gEo?W$kw@BAel9Fot*GjGxr7N5IMQ!nbz?SM?66ad;@%uz60tzrcz
z3R#l!3R$lE0^k{}?)Y3f+v89C5=If=6Af!nd46`?*E@=?8MNi&uRD9h*89^bc6}Bu
zYp=U1fxZ-|G>L~AsyNT@><;y==VlqcGIcmhh@NLlW8@Zpcb!$A6x(Q|OQb9>-s2iL
zQ&yo@!Mj%=if8J4OKafO$koG|YsA<_4BOyJ=!Ybp^T(B4%-bxsj6+ze-YLx-r@)RX
z*^Sno2KLsw+<UY>UqzpBAGYVB?F?Ki^FV2IGIjCTZEg``@m=3l<R`Vr28zs(rLHk~
zw|eqf$nLD}2sYqQaHMsA$vaG57v3GZAI$FjjRMOozw=wP(p_$qs?dssR7r=c@ya_1
zAFK3|QdX^&d!3zv%l!|Bu#I7in_ZWyT|b-NRgv!`&#wvJ-xQN-)nnsIS!(HbBqd^2
zuTt#nyQWo$CBf1>j*g^Cirhc_LJF91xvb1U{n8jbW^_Ih5|TlNG+Fw^Lw^G0vA5~v
z7ASU)qDKjDp5u)%BGcp#XS#cR*utE1IHNg`1L91ot`i|ojIJqo9k2#H%NOrLv>d$#
zT(-@FFUX>n$Abr5<+;SqnR?}=2I9AXwU0aT*dl6dGJmATH@A5g1Wu+u-Ay~z>36$o
zz9&C5nB%k4IjcpGN=d)bLhH4)1QbQy*xvG`2DWqlmN(?DJwtt|f#f(D0x>0PWLnbW
z<tA=ul#75gCh)dBR=nn{_J;r^l4qzrBa$kreX|GWO^}&@RD$h)$b<Z0dH;OjmQ3o|
zPFE2(Abah15u$rS<x>h2&JZNKQFhjT2H$b#Y!?WM-$_J7F4B8!gT4yIQ-e@yG{9t{
zIbNRKa#f)<iotJ%K9U~o(>{qk)^-v-U+SwiP<Il2+lan_P)VnY5z9Ub?7ODhOcyzw
zYqjh@hGH_Hl;ltFi9Wft#YaIF>PgQ~>;<|WKBoJ=>1WfrlbT)C<8GiXGRM-Kf9p57
zq-gXpRMnR4+i6bm?Wa8Vo8CqwW*K6<RRWd2O-Qw=?kTA{eWc>KB<^VbVEdBgR+vEl
zW0|q0*Pf_Nt7V>C>DueVaz)z8$<`*{YKxsrWwOoQk+}|io0!VA&~1`Do5m8iF|+Av
z(|!1^N^X(3Bn4JqUuk?N^Q-Z-6YQvp`a<iJ`zwZ<Vcln=tead@6J|Jco4|N7+t(cP
zu4vqk)iatLGMFJBj$}_eeYT3~8&T&Dx3)edwp|t`E$%OypF~HN<ij{52`i5I;AD?S
zy^s9)?>F@~Bwjm*orAdoov&s-o#x-S-B*$9CC(Oo!0@LqnUDe)#r{}9U0<6X?NRRl
zrn{Isr<Y1%j6f(^KG#r^JYNwPhooQm>@I919}ue)0#IC8)`j2P>C9c5Kp;tJ8o=nI
z5PjJ-gaw5wNYbwdb?X%%@_erGH4GB<+W8E;-r|{z5`}o;`jE*BAXvM6ZEjhmhK+C!
z1lE)HfC%D;5SX_({N>N#)aMwCQ<836)A`icD7<YH=}^PqoD~5A6C3MCl;bH_*kXy^
zo$%t71J4}b$qY03PI%$R01Uc@v4K)ts#O4szp%j8BqL{Wd|c4_O=yEeMn7VYo;19D
zRXE+?`Uk(;>Z<vs!HTgC%j4Ia*32LxoD~j)i{QO|%Fn(2QI_MX>6mh{xR)9S&RZez
zvubnzzv;U$dA*&hp|)JXSk3qGkngHRy`H>ochj$Vz0Z5&&h0zA;$&f<-ylJ!)@yte
z{n1F|L3|tVF~wf-ue}vmDGI)mcI3nvg@uUt4-ok)JTr>&2)rUcf5B?EzD!~-7^Y2X
zszVc)@-n-7OjB6XrAT}%hb!R9p7l-zKoww<)!SN`>evubI42waOyqt*-vT1-z*Ixx
zAMny(VW4bUi*3C({<b`2$$f2W|4U;$Hj~5vUP^cGgANBXFgr&rUNENB1L?qavbA!F
z+(aKJ_IXg{ik_=4D*?X1v;fP=%p9YZjO%II6_KM+ee0vecO|^rjcnw$o)lYvQfHy6
z&VN*EyB`2Kd+lgmwER!uAL6TSvoxsDNb04B4jl5HakLZr2~9?qn5h+)$7AoSHsM`O
zlzr+VV51?DTwjfo@SGL%GUF4WsLHdf4yN15k(VgR>#Jgkm~>@Gj)uA{7G4c}Aws@N
zIG){H-_2pPpD@=~5U;jr-A2UT(`uWUedqq+_iQZMi7ypiF}4&2-lQ(7;A;sRv{8RC
z<Nftk(j%pnMqV>)urrTk56XiQLW!h)%EsdR+}er9u$!M~yp5W5C%6gGB__JP@iGDp
z4TPF2J5l9U!F$(T)QHmc3)mG{q@`*vhh#q*pB!Uj%{(o}*w9G0Xwy%sVV|9vfnUHn
z-aL;%UsTYr8oow5fiDygYTf^G>fGQB`<Lby4SE9_M(ZTTtjE6cTf3aIJlV_dP@T3d
z<c||HP_Xf>E`cp?Cth_O8{R#+7XBrp_k!z$@pY2?J93|5@iZsmHJ=`T*dKhMrGcf`
zcjit0bm#<SmhnViM;*a}f*#8v1Uf7SXeE*I)%X8!nTS)2$fN;4SP5Zf2!KI53!YUm
zd`#L;q8YWrd2PM*-E!G+LfW(S#E^R5#p!(Vp(pGGNY`3vb2q2{O*??Ir)E``-?h0&
zttFIwVju6EYrvU->575)oU`Jwj&+nGWb7(3M-{Cm?8GL6tPby&pQ{Rjev+P4hdG~U
zkw@2zTR+T%^WXZ2f?|>!Vd>mv<aXp|i5o3I%6J1DTG6v-wm^2rnc!l)ANy=boiMLg
z>ja>ZSQ-fzC>jMF5DWSCp4?v+ky>l3lD|z^|4v_^I>~DMnrjq^cD5M$q^v*wu^a&5
zi<x3$&hR^)GopN4kgr#JJOFQt7dqs;*A^)QaPLZnUSpUGyn0c!-BOyn@4?eh>4*!l
zc-b7~bnGC9(eh1%>Fi#LQhKOzJ{;Tlg)iFQ#Kj@Em+jQ0kMWqW{RU0epMk(wwiO<J
z8bodsI`^8mKT+yzNT0<I+J226kHe&0MOAd93>079Xggs;HqXE`_WF>|&$jTmty;DL
zC7$vZJM-<@#Pv5W@6CI$0By&;u64sV9*vy3n{03iQ|$+GFgZrs{U<<~KK9y4`JTAM
z`p37Pxt*0*3As(i8l`zvYc8a`p&_5`Y!*2i$-coANQu@u1V=rj4^)LNzKi+Ks)TIN
zu_mq5XMF6>@B^wuBqAVI>s9q!;t!E7+4Dd4OcO}y1q+E3yKb$R1mKfI`fL~s<=hr{
zeGZ4%X*5^}xEaLsvD6&rWDeu(y!Pajt5v>@=ap!x9ev54;P#!bY?9&EKYL@o&g?+R
zFZCnK1*wg_`%eJB_kZyD{&8)tS%)u|R2wf2)4Ph_<~&RAE#w@lPk2b(Q8oSH)U>25
z(T2MFjwM~UW7qDSivr`zMZ@o?Ad>sqiew5=r2`G2Y@v==D+^#C?zap?e2_$~@6+uQ
zGvox#TX^K0yKTTS^yUt$c#gp*sx$LBmJHjkhBNR_s(PyzSQMvshXhG5#}cbn2{yc8
zQX6+2sD13m$vFWE!JHGJ=<(IZH{QNm=#H9jiYN-5DV@;6D8wZayH&{nN#YtFicDh(
zLk6t#FJz9)9<1*%j_pO(MuWG4?`M?E^X}U!MlqA2V8)Z{H^5EXkMmB_@7pF`AfRJz
z9LB>Zo4i95X|j1A51nTbk*Px-|9GvtHvLY5iS8W4YVUzUG8h$4{~G5essmIuEWqm-
z$|(lj+m5A|g((7GaO5ft<es9a5Ji+@%_N%|x)gKL1&P@k(5rL~Z}uuj903>1eU09>
zeq$ma*$ZNC-?3hlMc?(_v^^US_l(jdiGADlKn-^krf|7kUYg05&DYvD{hpo(xtqce
z>!~L`JttOYs1Ug)Ln2Ml^f;M?&$2T~f`SL#QJW4|xN-35pB(tF|06CrK$li4)qMD*
z(1C-v#NYo{r~YKHclo^D@9#ZtM$1&h#UDnKNjZR4`A*n3Tha=P@%0|S1)^-)3g0Ma
zRhBpivZ}<%JafNVT-1#w5G@7FmJ56{LUwa%tH6B1RaZDgk5U=oh--a??Z`2>o5?oy
zaG=y7ksf3T(*w8SFuBYhV`N@nPn#7Vbl%(9Ow@zz;z7NIFSS4NN-zx=IBPki>@WlC
zsD>MDX8N$g8wGN_OqQ&3y+Gh~(2RMdTNw&HX<d*R&9`JaK#%vh(!@7D`A8bOF5NZI
z#oH>AzVYqZ>TGwLEA2!emw8Vicijw4YD$0)t0S`+Sz0qE$b-qR#>*MM@3ZoWaGXxr
z%b47mah8ICc?+oxq>8vjP(0ohfoTh(d?PP8cRITxlTE#9-t<7sb&ot;PW{SL6o`;H
z%3Wq?+he;N<E^UoL{j(FxRi%~s3``Ma6rtot)(nOl`4vWwf=gSf~|2<gq_vX)0Hj@
z4wqHp1T=4z&Lp|(W8UO^7NsuwluD(N#gdnVc4s{&g5rQJOv8POGJFC`Vs@!T#_D{t
zS+{!fMP?BAgK=Gb=8LUNXIJ}79;**@vwtEQPALv?c`hF&{6MzzmgRt1Whkmsqm8Wm
z%W4~%suQz=C_!ejt;h=c5SThjb~GrgidLfV2v-L>fkU!h>bN}FH>okrPd})Xygupt
z{?pD7&e)l>&$6j|*Y<TkoYFaV`S7VjFAvrn(0g*_AKte2T>0vp73sP4szq>fC*<$H
zB15G|KW%TfYV9X6Li6y<^%FloNybYL*=yH_h$i-iD?&+0#Bad#beCG-;D)d8wLp@-
zYeMI`^RTK&yw^4TLhTtRvoAXXMMdEE0>-VY&Uk^HN@$wF)gY1b2#<G&kRUKw=Hbr2
z&=f~0MSjX&;KH5P58PQy+9FI(9F=SnfOZgOlVbUIXU(ZYJC71EHBBgp-l2gOY?EbP
zx7cUaPwl^yN)<Ah3$!q<%Jvd)CFKY&8Vq|-YTdc+=KV~5mvQLXT@tAeEJL`~BvRy_
zt>)JiT>{?Nvqn8@o|vlzrHH_Bdv~tW6YFo-up^4fn!$O3DVM@cg}geHwaY};uRNNS
zdQCsn_bjS7%nR;?eMmt^{JPsS*11&qDMpisiEwMFU@U20XZwUy)_D_ZsROY(y<Pf&
zk^ve5o1=kswtO)!;&&gga)>mCIUe(HNnZ}$;#`5xE)sODH0NQ6ekLEmJcN5cV66-)
zzW&?uxaffK4#EqUNA}X|j#6%Zn(D#WCTX1zgIPT-q<xeNMe>?#Cc9EhnA<>-{qp8B
zY6kil#X~z&1m18(z1QN8bluw&U!JLk$>?n^6szUK&zu6rA4B`+)eQ9+dw0~{RjJi?
z@td|%;igJ^HvF4_qmxSEMZ|BqZ_n^|I#^X%?Bz&(V4u@4)k+Fg8-{@ls}pUcNmeS@
zRj1yg0u<ByyoFMc%Dd~D7gPf4*0jyn6rJS`1wa1>!-W{LcC~{OPgm<6+wWyj^Y|Y9
z)xr@+9jJI;Yt;1She;Enw|5iX^^3(YgRfrVPV>eC5?C(Oi;5ouNi!b=HNsd)t`D8C
zws^Uodgpq~@iv!a-WQ!!tk~q~Pe6iNXc}&=0FzciuD#RzAuv#ix89;+*RDb=m>A(7
zR$v3$y-`xc)pp$ml&)tK>mGve5&%twkD=X-YvR0X#r;|;beb-x9_4G_tUXHn4pV>|
zX3bfgV&&6m6OLISzCmFbdM5u>bJN#7u8vN)uyfk09;j{%udrzw7wCi-L+Ep>#+x(c
zlcrTYQ{wUniX#(qNmsIis(g1%y+Y2D^h{30{_aAewYI$EgqWTb`BisGQNSEhG_9LZ
z4i31TvS|}ginww?g`(K;T{-mK&$~+9X$XIPbJ95_Ldv)@{Ic)h<91oGZWs!;)9LXj
zKrLIeBOtt0Ln>8iH3<Sk!*F|`<5@5fTb@@eldBGCx0{Ht4<(&lEK$5Un%}9ltX;c#
zdpl<LTWC#ehHA0lGnC@VFD-97!ym+|i&Q(&I^RoIi8I0Yal(9$uV#D8?NJ5}7*<G9
z6T(2gV}ty`%5u(#A762Ye$_ZK>$B--$7Y}clW}en7vXNVmAly*FiE<3XVOMm7I4jH
zwfDC|_IWpMcdBcK*m8Ue#W<yyswntiRN`jJGCoVyebH9X1K!)3HCx(d#7Swn!HKJi
zQl_oZZYd2RtjsA>rIS0Oy#%XgA{Ol432H?5v0;hRjtVZUoc$*^`9z3=<m=N_HJh9+
z#h+eM=zyCy`PcO|Z~j%9>pvVT6NgOlpsbwbbr6!vr+%bb_#nnr?;FGimIOqz1-%wr
z4omuj2C3-<R+ht9qGl~ZLM+Y0k0-@-4<zy4lT-Pij&)KvC0h9unkXGL6BorE@TF8z
zIZPg2(P<+ZhbBtfk~3iY_`W_l<Q~|R!aOn<opC@C^g#m?lLlHNS;B`oCKb3ZCt3Ne
ze<6E$eW7%Dr&Crs)*V&+Z5KjVFE#cYZSGn&nwV8geN~DLspD^@I3mzc6kTx<j{qsu
zRrqatHLvoo_vsy>wUOybq<i%BVA~_s>clG!otpNl&;*k^AKCm}8*P(Z$<s8YG87E+
zjVTQgju2^A)2#DR2IO^eo*eSln2fI147YYvFP3lI=RP%Ne_c9!n%%;<T0I1ZVyl~o
zFbppQ`mJ}<*w{z%vu1p_C0`XAP8l^4`cJAUR=Di8?n)D3jnl;Na`G(F#oW&9Y9u3t
z6_D~B6_DQ5hPp=}4sWr=%vMB7!r>?~&(GQMT$&58X%1DWS5|Wrbe+{Zu(0&$5_i&c
zK*dXWgHN=}E@BOVSvtOE<mU+w(~)mkv)+6sY+7_J2_q?YX9hTrwugLVKOyamC#V2I
z4jETq^a(}fR6rEvPpu=Yg@cP;8oqD%$J+lM8~uoy=v@v9sJt2s=lEswyv_s*G{IK0
z^V2L5zYvOX5fdlECKp!%=wgoGwvruJ+QFte%s}rXp5=!qUR%r@04?*9gP8HJ`Ea>w
zirj5ZlEDTF_4M-cA^{+q+#1D{-ymX|C8|rI*n?jHq#QNKu=*}~oRWeH*;jq}E>Uiy
zR1Z8f+3;jr(yNMlfYwNo(_Td>=D+1M@)PtCzCuu|kJhcEQk;?L@WT`U4IO3T(61o?
zwBEdql=~tN==f}>gXGCQ^T^4gZw*C(V`>z%Oi5M1qf<O6cZ$A??Ac;-FzYJwR|QhZ
z30XmI%pJO8&qd}w!D%xjpwqDLw~fgSx|JoWC?T-3cx!*7iZvtt`C65W)-L(S#ZiDW
zV0>forn(%{DG!0Gs9k54px8wE8qke9&Fcg=)+wtg&6)o-|K4(nX(P}Z0}ATEQJ8~p
z?eaFKV%0Z8UI+;SG0n3@6OoB%F;n?P8No2El4(q8-?Z$&OqQ4;UoSedY)2f^6W?m6
zaF+-R%DDsSk#^7mzU_m6{@7IIOHSkOjCRV+r0sf4F1!zY%N;dKmgoGfnzAgzT@d_G
z@mD^j7`AgN@-Y`8e-r&9H2yP4hU4GE_~FP*{)9$$(x09unC-*DT(HwTkJXQ@BBA;-
zb_UV?iGe3HY0g>qs|n=s$oh%r6(vqx)_LMf!}Eo!OXm=w5{&Z0-q@OqvjZyfR=8Rx
z=$I{39CwOsI9y5nF@MK(e_PuX+pg1e^g_4}AXD!JHKoajO?yF)#U6S`&nrgDBw3KG
zW7?nLp<xuHW-rh+`^FUAgt>DskQ`mNZQsw+Cb&B(E4dw=C09T}ktX{@ANs7iDSX2-
zFr2`4TuXWdJhJ8+%lG#4Ba;c9WBk(cv1%3-?zeaayZty#nD8897Trs2ILkW`sx^0G
z{KF}f;_Fu5rAq1*R1`OtZ6BssK2TDT%Vwk;Gtb-y^WuMdSz-;SUhkD!8W7mHBpsT0
z-W}ERLW8Z^c-E}+TaEZO&Dt=OORsp0v|lT*$#^;;O>VBxU4w>Ex|v{!tOq`I(Ftyj
zlCDn(Q7hds?6>XY=n{ZjByZk~T-Lplb<Q)R1cZcd^$3{s&8SVHcEgi-<r1+#u8&6>
zb>%C3N;>>N;hp%wM&SY-*jqTelsC?pv#plVhs<@Rt9Pb<#KHiA_-@b>Fj#*q<VL5t
zu&Sk;j0C;HbJ7W!FLWdmULEkG`|ElCeLqNOJvo8ByGNG&6xjh|RLOUL*8=#NX`QNg
z&x!vt1c@fYBUt5=FwLq-jf~d$;EqQ)3o>6NP8%%(+i6pV_W1zf6>--f4Y)*{%<bRL
zzlE9-dC>H=OLZgZjD9+ji{~xz4wX)G4+z}m$MC+u*dznvK536=OLjR`5T(MX=b?$Q
zj$3z}NuN;aRsqM~FUj^g)|;xbwNdv4t>JfDkH(OZrCl7A_8h8zc9gVFf@~mB0W|_J
zhy7Fa%+e+AlrR3BWY?$rigNiVz&6;*lM|lCq*S~OW-fhPw<^1xAX5*}@Z4coCH)RX
zWl8l?h%N0talAjbnsgYgKx72kH{#0*>mpes!Ptb|_7*dSUV{+Lai#HZ*mqPeh(3gQ
zn0dj&3v9_r4-`%7>eRXXYJoc60|C;Ig@l_AbVN{^<yPwWx{JGR_YUS{@TuIq$0>$~
z-dm2^FVth7A>eF$FqIGziuoL005ykRRilV8`_#}k7*p~byv!om8Q-3Y<jpb{#WEye
zD+@7sMZle6)rySFBQ`1!lItOm<3${Qj+&N|>=PYH@*$OD|Gd7$o_Aq^M*<A_{ygFM
z3Yw4}r&`no{QQ~B4J5P-pi28Z1MS)+e36cf`S#Z^k*#25))U;hk)V&e-kUs)2jo@e
z62mN(-jmvAiE_W55na;g0j_MHH*4s7Ol2s**_%V*8U?3OuY@@XS{`vca_Qg(A_yM@
zLuK_Ag$Q?XZ>BJk#;T5C4Hj%W%n8nY-{m)~cqu@fZBKtomSz|yRA-VDF@88iLH8pX
zSbat65mio_rmrxe4_7;sO9a^}AIsMv$oV*JZlWf*x<nc8&Sstonzj}L26dWBDm))D
zsb#fAs+_Sm>OBa9Ofm<;(ZuczQawkKV_4?+ge;{PHON=#y)f(g=!b{V*TypnhrRPO
zB_0cQ!Pc95dP3k%SSBxR5iXjHcNQ9)${=!fu$1Vq%m4~<8o8%JHi(p*u}Worytg^w
zU@1mO1_oDx+9>84fE6a^Z2l3cfUqoq2@Gz6tw0y`FMhZRcLV00dkFmeQL@R;)i|oe
z&;^@b*iuUXCUNu&JwLWYgq?hD(0?PsvSc*i?ku1_OO!r*Dx84%$S3$~s4doPVqgcO
zVxaq@$n4_C4G8=z_(5rI?*>AxjCfdcdVU51-CnYB83P&=o;zaG1zv5tB7Lm1kTuR5
zqD0jS%uEO<@xp2_!pAW(BPhHgM0O9vuS<Fp2m~>$@_`*vrH?<YC4<Oc>2h;?ch7rE
zO9FC-bl}a~f%43wOXe_r*ABeXRwm)MbAAv5aH`ac55*91_qDog&vEl@&*nlqitb(v
zb6(O{NlC=4b@=l_b9Pq$Q7t-e+O$njDqR||jPS`?R6+uU6c~IuQMPxF5FCSoxM(M?
z;@726R+_oIHmStrAWM0VXz;z=>9cX`h4&;~Fkcmd56H$0#Bt8iwOLWps|1#iTmKT*
z{3Sd_orM)aRW*zL3Lo7?;G;i-B~Nk01?m%ECuM9#Y@pX;CvFP_lKL(;!I;_kwK0-3
zhqZ2_bjZWzchscZj^oi|+J^D5&JcwqPM<Z!sI`KSLAc!}LQw^3RMGOdnuB4mqS8al
zGwWQ@m#d*;d!{UHDK6Z`y70ZAV>tl1>Sz74KvTZ!(1{ygGx_paUkfrkzkmsdyX@P8
z3;+ZozxJe(H2<nX32gEmlj3X_P+7JRp|&+GoqoRG^Tl0|y2r@FXTD<J#dY1j*POe;
zJ&2bMRydyE*4|47>1&Dlib+zm_RuaSA6R8FaP}b{`uVZb!%7Ff6*zr3c%(2U6X|O`
zk%9EQ&a0$L=v1j&GC=VDO{@Y_4`XBvn<6BHJxr^QVJ9b41v|CmPmKX`w4|jp^KkKN
zB^QtgTxt0oSF)fW)!?e>c>yAAfH;~#>1SH$VL+I$wU+$YX^$LCn~i3B%Uj_<$5#*s
znD<O0!Q}ofoB93ce!^v9MqRjpOT@5(M<{atbp;ua6^v=-`gH}RkriCNZ}Ia?d5V2s
zK|-v?gG#(a>ftgj5HViw-7D*aBc68;0`^joG+-i|bllnT;t}v-RN;iNck*~AQFyY5
zJZ`?QyM?~;Mjg84#_~&0sBdyQy{VF><I9!IdV!$nQB$e}!gmvlk0H23a5BYRvQLuS
z8sTvSx~XMhq`JzJ8+<Tj&+>u7Ik>l)>FOcLsupF@c*1W1veHoD14Vfd?qTy<wB%)k
z+zw_UplM1~V3ihl86f@?G=?!PJp})hhe$F<Ww6p!z7FSR$k6e1d9%VfhSH=YYnLHz
zBHX_LW<tD@*m9td&ZsHW0NCwz-0G98cVLm?{gO28x~mhVqqK<2<)wJ4VIPiVrh?-W
zXW(`fMn&R2MHTx2^cuManmL8&8#8IS0jyJ#p=qX95g~9?!Q0YF3U#xgy&b?GIZfO%
zd=UX;Iqd>U564%{1+<A1=Y_J)b6*%2{ynt#88rO<*7P1YBJ|5hBE~<@-|Ji?!?;)d
z@~0yrjzcnxdI;=n1Ci`v&!6bj<qul$Mf<iFE2;W#WSSn);@esLZLabN-HMjd)D;LR
zOjlnZP`1{iA&-~asjSBlj8srMLy_G3bXqrf5;Nc?><RSSv`N|xc~BNJP<g1e=e4A8
z2W}IRZHfbF9`Hz)T)?(E`oBOjX!e5EPGwZ7Q$>$RKA6<wu)Cx#sjo5~YS?UlXF^a?
zmnxGb*{a7R$>9>GzyJiX7Qs}OiEvCWPfbYfVuoW`rM37#m*Ey$%cHikT9_FU%1Y;8
z4k3jdKobs^c54gWQB)(Cy`1{<VTHX)GB-P)gLm56jx}W?=xn!<H;9H3*LDfC1zxak
z+@xiYhUrY>$~R=A#XEw@_U@R&Vum{})dObwelx6H0Qa}cd(T*Jd|q9EwObBpWWr6*
zX-o*wn!cE8H74=Cs<C9Dd?u4|Rs6CFV-kd^-BBW)FAy<rqTWXlRMG@O<QUdh*wpnm
z@DO?-!{@W)9QQc6>qip_E{*qv<dwU}A#=16x=-<o$NF_?qtD}Q=#-sb?dJLgn=d-L
z=;UM})#&jP_xeUaHf<jS5U!b6@ZGHO)>|hei&b4<^br<%+g>Vi+sKzd;NJs-wT(cJ
zZ%wP1TkuG#U9x*U048(NouvIV{V;Cj{iQU52EAAO<1fUk+?k3QcD&g#7CSegBwncN
zmlZaK3bfe4a+OI?PRlcqDyr0DVF!FJP)yfHqBZ+lZqKcod1}e>7S$w=P?c{C6LOQq
z?II9|dWSJ+-Ve+t@B^map6-}89*W*-8*rR~k}>mq0q_QTK*wRHiM1>Dx8r<W5L1q=
zd+>j5@xwg4X4+mCmDUPlwh4DS9j$6<?cY{i(?V)){a4n&`#Ncwyn;r2ZoE?noG;zh
z(hA7qZs$NC?1COG)75hk_Lq!vZKmWpg>S}&T3{S);0$h+d;>X%q7K;|zn}}eA-ohG
zSK9$DC6!+d%u;I9k%T7mIRGZtxl^;-p$@SPgwWmc_6r1!la62W*)mJ7vfP+jO_dAq
z`-CQFFarc>KEHbA?qh`w(N5RQ6vq(EBa!@jl~f#{-fq2aBs$WI23##0Y7_%br?-+{
zk3Ofin3QVUu3>mVGptlk5YauQ@OI_Nq2r4eQdt!*%S=nxEye?^aGI+!{st~ScGRsp
z?U(5C*Vso$5`VGMQ8GDK@|WGSL<*qOPU&0fpCRk;N#^zi=7+dw1#>+pn}{EDM`j+_
z#O(tUzU_m}@(ScPJJBBQ?8V}mGmC08<Y=;iRxTv6x6DNbqEKHmehp(xs}|Zw=UQw#
zHI0lSY~KuKva%E8CS!yPlxD$3bVNgLj}@fDthE+C)RWJONmQFA+Q8hm>U&u0G4Gew
zEV%Eyt`XlHEy0a>L~0~oC^oTS>AD?O(T(5}Ae3wdWCa)P#d-iY(BOB2=E}F{UEbq}
z5E#dEDGtfv1?hO+A@?LZj!&&lK+N1oe(dn1Q8I9y#e;oH=RTfJo9fEe(;o`~5sw=t
zQDwWSUSfEn(B@}Ah1M$h>Q2}f^>877T_sbCpSv;kNpPplV?r&$QWt&-PL()x#}PVb
z9fHb7l5V#YGmaA8Hz`foYEx1-A8WFJc`9+##4CI_($_MZV>%IyWk9}Ov!sReNb!sX
z#hdaF+(VglwL%Q|s$F%rv)dgND$CH&o^@X(;eu;IGbvu2f^8id2Ee4NbJPRGsc=g2
zU_B96*M~VJVllzYO)WaY)ibnx+>y$fkZ5$wsgV3zq9FqIT~F7L@t3UtM0ZFus%xu$
zPc+UUp)HA6mMAX9^R*fW!YHv2Io1C$Ei9e<YU^I|))Qq-p-)m!zb&?3$VhVWVN{9p
zol0D3e^Lw068#O-1HS8+^0*QC2Mk>oOBbtJ7hiBoXUTb4d{xe>LIZ0|-<bSx|L88*
z@khC1JINUOD3`4ib;S#|4E^yL4ZNuCwLWWUCkV^}?QRaSdYo4960J=S+gCg)%8gBJ
z5LJ;kIPS~fn{vJPv{AS&fjn-QJ1&g(xuz&8mvDmiDb9T@kJq3KgKK2}^i;~sqM?lO
z-J5wj>L9b6qV~ei7OH_3*ChmcC@+zHZK6}kt`B=dl_qO&-uIhEFXO8l!-t2g-c>+>
z$AT$J2ftojO3;WvAy@+3jG`3**RF;sog)9;PI_OI?`ItYdXKjXL$_nTpLpMbq?rkr
zdl8&xqIjtoLjOLQS2YQ1(%UW|wx|XLfgJ~1$pOewcREv~tK4NVt?}!O9%W=*y&yG{
z4o=>LA?2Pfk0l>OJJ&3;dfsZG=@46QzwH>&Q#Ig9_Cel&O4CSSf5)1;&<>{3gt4wo
zo+<@vcx?u<pY^K~wGfO=l-@LKWCkzKc!KnoSnDr{QYa;W%%OruW)J@24Z4v8Cyco5
z=O0C%#|0R{+ad$&Tdma#_Iwt>-n9%LD^x$#$#+i<8|hR-bF>jPU~v7V+kAVR=JG?3
zu2*rNYVtH$4?IP_I{S+J3IZPHdnd7SZ0*mQNU_|2{l&9h&|SX0TDDk4SOCACB&o!7
zL@_pbD)I6Y;e#oX9d0Yy;O*r>jtImRf?2MQ<|ER+kOGB}DJ&(D&ph-+VJ~}Jgjro0
z36l4qW(t}>v_)y|R|!mU*?ev{hhM!(e(xHEBB*9397b1-*SB+<wllv!E0O5n-iCA*
z^Fc}LsiwsktA{x<qdk5v)R_UjTRKJcT3Y@IQ`>4$S0$AuHKOc5sMK0-u&-Doq4<U@
zqrSZgN9^5rXC6#&DR!eZXO#bhQi&r_C*a0A^!A<0bA>tagL%+v%xkkuK%J7KmL=?O
z*xvGTfYi2m$^2c-6HMgiVRCvE(oAmkTfawJzj|o+*%AdTOKH8zFaPY;g$Ju*XT|0D
z(?|2*BR(4Z5^p>}=|~0N2wLD9hwz(-L?=Zp71i&ULz0lEsrbeLeAK;Q6Dt$Uv&%Xk
z4!rjkZr?$jOp~1yCx7HV2g6dJ;L+QtU=WuWR>cAspli(&@dtFE4Wl>H`#!TNoTrUQ
zC=ZAzQE5WYa8xnR$du(D-J2+fBA$c!639p7#QE{EqM2fCv8uHe#Z%|o92)ak?%6i5
zB)v_GRqLw^h4Ej-iplQVaZ#CYncQ89BrS?Xw>xCjfK=DE2TFB4tIaFR3bQ)Cb_zF1
z6k~ap>KLtiEEOwzzzfWRyKLdz$E0RVD-BDeN0a48V^Ar>T`&{m6v}5X3Y{fNm|WZl
zwa;CH)!UUf6dw|xXVTwCs@G3MKRr$m7LHAtR^C!=Yq3#OgCsLG$NY^rl4M#4!vx%4
zo!=idKN&e3=PWI`q^A)#|6Bli1+a~`R3~r$_>mhG1>yxU4bi4J1Kg_%g(H6R+}jh5
zk5b6dHiuRL`eCH(l<d9`{=l8qaWckYQ5V1v=`dw-F@&vAEWV4>Im0sifL_saqW@#2
z9U#HIP%s}6Sl>%pU7zV}d7Q#}OpNB?Gj5i-Y_LHv_=~Otj*5B7T*2M2atJ}*W16Az
zY7R+;anDMYA0lyuu^Da4o7b>BCasz2E{Ul7xMrX<&*!itB!+bZ>Ni`*dXwN_AGB%B
z7cJJjz#%4JPXmNxFDCrau-Clrii_AiX|l8;sMYneUeO)t?L~(F{DwGN#0HmTgr7Mf
zxbvU{NWkWh-`wL3sq}zhRXPL$#u`TG4kGL^THf}4rGmLfZ3atC+=J&LQst%jI}|n(
z$--PL5S(`glC!C;Nlmu0$C^b}3F;jZorfdZ%p^Nv08f}~hH?ly;o+*-H?&-4Z?`#P
zrZ8KvGc}O4pFx!jx&oYD(wUAV4@A8iq<ibPZ@xLnDB1-_ZCZ<3F3HF-{mCf6z9U+C
z20=Xv^=6RRCK8Ag9qt?bTx^TBS0o5GAWWI!jn-QUt>S|`ZAI(3fN9!PMKY^CG|y#2
zjietSn&Ov3b*;fico9FU!X%x_Ae5Cz^@!d3kL~eiK7_pbh@`U*nRAJLiI^}K0F&=n
z)-(C}GiE;_o*lnf0vzC1z6XC)$eo(%=*^0rOx%{Upf<~LGtaK3uwbHAWO0DfHgv~1
z6u<;pxdHRuM<c$HF$p@ohGFJ?I*M7D3X!Q2p5RE*FF$v`1GHNER8w~C@=^smPAfsO
zJ-G*xwb^q8y~VZB)3kJtSNzq>Zq>?1Boc5D97zrca2G)gDIW#u%xbvOBe(OCp@l<K
znk8h@a<6Z7rfCRY^cW25>rBqh`E;>F1*#vC43*GS-`8)6hsN|Gpb|WsR7iHah;Fz5
z3Y_r?W))#dreiN=&Uj;NS>ogMTJvQm<j%iiJw8?}Kel)#VkR5=d=nqr{yg`iz(3;w
zLL!6^af(E_96$3%pc4HCAJP#H9@*LRKTat5`40=s<aSkr4+^0LA)D}G;Ro$D7<Xn2
zePwIf%ecs2MqkD$6{D=lmO(S9$TN#$pZ?2wk%>dx<AeLj^cR=N91VFbC2^o4!;iHU
zs>I!(P0$G<&yEG^Z4ZPp=^&=J_ib&u<I@?Z*(>`f>F)-cfboNqjV=vgK7~8VBS4@3
zpUue=A%9#*x$b18QT%a>2i$N7;2sZNQ6>Lrgvjna8I&auRV`pM?aoP<s-U&+``mne
z<d`~)+1b^L;I@q9GY54A7rOuHS6_Xc1>KxEYQOAHV&>=%&@Mty$ffjq#)m(8@%2=$
z2xRm9+2IpAP|8dPQA3j#g=XDXXS%ZEBqMlH=2W!b@34dND}V9Ua}SL}m7&6>ITwHZ
z<X@tTXndS^0xFlP##NYpHMC?ZPr{T4k2OsH^m+6~Zug=%miQ!^UmEa_Aft2?5=4)b
zK+%dCDOORTFlnzMo+6&oRPM?8WN5qeAXjB56qEnvGk<^U7C=(BQt2o@$ISPaJrIM_
zmq*ngeBtM{e5C(zzg|PQ?e3DyD!C1;sVZfTtFy4LvbYDG+b)*OUJZlsBph8%b|+!s
zlFML&g<y*qRpiwso|gUM8vb+=@ahXLJn%>^&L5BJ_dwbTzFf^J=^5=0xFz~N{8m97
zY--M9SI<l$ObxHwRq1kZ5~yp)({y`;`{<QlEl+1Uaaz5i_vY{S_UHcj{>qK&`>hhq
zk%Wo-auIGfa0^XPC)f>t{?N^xK!3~3AGVi7Q^Svss=Eu1vb4}!fU`0*mNfTopZmw#
zXh&oTGY3Ldss8vGe9<uR*TjzNLEVRcKFeoEe|&z`$H^ZN4gHye192k{n{C+uiwM5|
zJ5>J5=J<ZO(a({GrFE<1qv0>7QHcJ;K?}8}Lq4j%2cXe#73$El?k<;m^~YvgBKgYC
zdnqHf<z80e<VUk#ZvNNn{FlcDzv8L=B+PDv+sNm~0*#)9gM3lqThpzq^8)CCXa8b5
z{xMqk?gY+%kSxiT&1Lw%tNXua_n&s)|Lwc~JF~w!!~e4=i0S^nQ}O?yQ}KE#vA@gC
zo#^*iSPKW*aT?Grh8g7tDnqBKa}NJYy!Wq8IC>ON=b%0o-4{QD4^{xB*kmh>=vCw&
z$SM5}X#7j`{9k=>dnsJVFQC;X0Ypj4X<N&rX({Cl=)Xr9|9V}&|AT}SK&AB&BzE}U
zO}ASbR8VqA$ej<nqwp>mc>K4o>*rw>Y9b^~OWCykaOTGkfR|Sp>RgQx6#W0^<NLm-
z-sffJxgS#!`H>^_LLTUJLfMhm()&L>l>4tX?mzwm6ASWzA?JAcAVnc(l$q~@5z^$G
zK|C`5)9nY#+#ny_XSsE;CAUVF;fav>N#=Dldu3?PRg!M$|HY>fgMQ|ESoy6Jp;GYI
z^yl5IDr!j5yiDk`PycT|4bXw2!4*v!6c)i4M&Ussw_4<T=wEBI{~p&u>Ok`S;_e1T
z^a~$wBlvl%1gGn0ekRlZqwPG#`h68<97YNu#kND1^^K0?+^H9{f63|o<4XSWg}31Y
zBnMBvkVLwJh<${LwDpc+KVE&=h5z4jihuhWvb*0`^$qn5lIU3dyj2a+gUf&W#sAh@
zp?U)3;+P0q{9uoW{Jd3>V|vH`t%UL)42|2;_f`0`-uO@-pI(L<w_%s+ZdM}KOW`#n
zm;4WZ`Op7)7O!D9aiQ*z5f1X;p8kArmp!|1|5p!A`ul^6q(LJ@(;UJdtGagO1I;gM
z_1}7Eh;f#w!WnaEvp?FFi-tD`;f*hoiu}Q+Z*Tp?O8@zm{`nz9hkw6QD*<~NY8?Ea
z@*j5^w0BDX#IGsh@22~oKjhDUrH4l~xDqkc`VPrU$TXw-GP7~f-U83M{;OO0Z`?51
z4?B`bJM;a)9sTj(3Qm5o`CqIlF|!F+j;WIPZ{0k&#LP}V%m~XiL*v1J#>(K56)+$6
zcL(<sr(>-9V6BAA5a;Y|u5R|jb^ZFJW4}MhNk$_AWR0AEe)`_2`-K0#$nBoss;G30
z;@~fI!3(m^KC>?T+pjaReK!g0H~64ShMeK2VZQQ`=gz-912+jW@bq3r5qhEr+zRpV
zu^%~_*Z#J04XQBfjTeDES!wTIvE0gNc2S(tQ9*8t`HI_pExo%#GDopNRjN94fBVZ{
z-ZHzx<)4ktk$@|q7XhbEfPd-lyC(Vhhe0Y-&YW6)&bx(DJy2ZJ)Izh?NDe}4R#ri}
zdhhJ|K(1x^E?4HRF2cT#4#)L==*z)G=Z;FS;p(FNx0^hOO8~XD#3ygl2;iDMfdoZ*
z@Ya86f>b1%>`CxJfp{8#@wufqTpp!Uy@c&BEUvM9Ynshzyl4XR=o&lHe-!jWh#@mO
zfmt{!k*tMa```0wp@R@!ksa_?L^iN0yw*Q>|LWh(f&;<MoTY2-nf{XPyJJ*YEFUEY
zD}{XDACeZ>%|nzs2JW7LC*|f%#{3ag5Htkw+ETaf;4jbiZze1GIXr69ui4D-MX%%F
zHBHQ0qF=Wk{3+m2r5IGDNDiJE#=6a3C0evMQB893*donATvpX+-p;O`ucBZS!<qhr
zQFLR)2PWSG#z*8rssQw|Dm|Gosg86!4xjPs)d7?wmx{%;XJplp4-PAj!$A$KB`b}H
zFG6$~d{HJCZDxKIkh`#EitT?x^5Hn0!Rf<~{K;KAjawLL_(7NC?@v;M1V2@cv`@kG
z!}+MN@?1RAJX%Z#ZoY#b@<p#(Pz79SC(}}VjeH;fGm9U&Zu%n!EpS^7dZ{8$i3YxC
z{K-f5znwFDG8KCHj?k}(rB5U?Bl(~9$F%HkZ;W6IY|&_Jn(E|=&V>$Bcd8p=2izo}
zo(27TPe#Q4w0wVi75#<)o*@VS!&%_{1ce{yYzyFHnkD}0%?Wwn>!+&8vmeHEkllT=
zdKN+@C{)U;x6o|+!j8O-L#g#B2I*79``+3C6UXzTXY>s`NRyQ_BSDay#l{0PZgUWY
zXd5;2sXo)x^x20K>ZDjTbb6kqR*t?GNOZBLR}j-eZ&5ysbMFE6E@g2sl#2S^T+%+n
z%MsNK42j>6zN<PBsPr0vGT6)TGh8tsnY-4d_sT=kYWq0PC~+7_htb-$%W^$h{93mu
zCAe!71`{(zSH3<&=AP(NV3Br#UZ5xacD_Sp5DG-5z-?rnT#IrTKg~B4A9%JCr1Le}
zTF?UKAb0z{%?oN7X3!$O22|D_Ah65=zv`0_s?p~!enKJW0Jpnx`_#D3Nug*@91Uq7
z&rg`YTbMff*XWMWW2wns&S5$fynH>th`a==7xb-utQB7G-~smyuNR7NgpScb$@R+z
z4V@p)5ow^|U;J<=hK+{DDjvvm^vH8Wc!G1lUnc192YLj{)eVwjp)(QR%j_G&KK^&x
zp{)QqxF~q|JC8zrE{kQ7q{Hv}uxH&eb$%nl(Slwo&B;*^Q}3tZGBd8&S@ManYGC1l
zS!k`B?Jzc21s?jy*(4`YwLD{85P6STzdHAAf45@ah}t|)R?St?t}#k*FqGR;t5s75
zKgE7wz<(ylp}aCQE11PNK`5v>qWQw#4nMu>6Y9!9{zpiCK|vADtm2r;@xLC~18&oV
zA2suzo1!`djnWQ6lr9Y>dox1b53D}x9e9lwF}qJ|3SD4xkxuhR`4?gGZ!ah-((tAi
z*t~Jd8iyT`e?_n6cBCItyCeE=yP5dv(9d=sk9$|%6G($-TfYgAXb2uK69?WoC)_z^
zi^*y7JjqxJBz;`;-Z>V9teQhgDAwJb3aEN&QNH@yDfEK{=RAfS(<=N2{2VVC(SHa2
zqgintDyPS&Nh&szd;%L%1lb?UI%nfIh$2;y*E4O@jt_9rSVG$r3nLA3G&Ggz6GWyJ
zPh(<AZop_>d6Bmoy|C>pV1dfg%hq~gd{!0)2Q7-mDD2e=bbJjFp4}Xrkz<-6o$pxc
z;vDd?J1mNmI=<K~=shoHz(A#Dx9H^FtH113dZDc6R}gTT40)7p^r}REJ?sZ?6v;eO
zO^G^8F^}AleuM@s_`U9^`^SmlwJN;kiRxd}1EX2;6vdm0NSpQB^>Xas@|A^|<#@q-
zMtUp92UUW5-%flY%Q1#ACJ4EH{w#1-E*6ajyyQNY4$V9TFo(5GPk~WoIqBXNca+*P
zw63SYv~Vs^=TDXw68y_G$;g8J)sjsLdn$6k?JNO2@3R+E|87O;kGK_LurlfAUj$Zy
z&6M#+;*_O4>sEMu5neZp-5YnI!9|<8W*L6{;P%}_1R}(n9_a2N<>9|R1Z2H@wv58F
zU1Ne#DD)c`=ZAHT0GcC0G&r?21p{bO?n@?$W{pcnAo2@q7owcMtij^kOL8;$%k~he
z#s{NX9d3G@>>#c;`-j!Mx({i8)-ivWV6gAr-*-7rIIiJMaIuI<%}$Cu1cuv+_&S_C
zGarrueEuL9KkTuVx(vL&0$?Hv+7w%O)A3m8PhR6iBP%m^0-I;r(O}pTNz3yVII?=c
zO>;&}9zy(1*CLr+j)d}V2Sw@eQIa&(?E4;5y_bOW#%GBVf5j;JG7EU<{3(}KMOY4_
zn+3MbdUc)>$=P>2roAQ+p$i|!1x>Q;7VE3<)pQQD9mn3eqm(sm6~8zFT~7xw*S=gm
zb)df>2)YQGlU!xZx6kSVW4ZadU#Ww`UjzLHO*s3Xc8^&*L#`46uBI@)`}kiL8n)$U
zoazj<Tlt~JPKij|f{Q+N2%fFbJA&k458+d~lH?a`mhEWLVK4<2Xpg->2kvibddFD{
znxjV0*L7)Yyi?VP7Chm*bT~<CKrrrtK7H90a|)+J-xpt055}?zWCOK@-KNkP2><gf
z_a;9I86h=3xXJgRD0A3x-?1m=))DAu3?1~pn<dJLQ11QR{K1O0f-M}%1mzldUR7&t
zGhl8z&7kjMUF$qQtNwMJ4^QIDHMAi#p|NhfVh3gW=NX_7rS!0+lH+~fec+&4Ox~DA
zMzu&z1pT$0T%j0oB3t-hH^7O=fD<8Ps{b#=%;ZR+Y-E=wI!<^b!0+uK4rHHa93Nf_
z<3Vqa1F#|bvm{tk-Py!wZG#Y%5ug~Nkk+041_~gFs86R^d=Oa%@f<94!8qE#%X=rs
zYf(7#ysm);2*uef?C}Li$IUyVND~<GnvJx3Xnz73lzTCeK%(pcZ7w5()V*u+F7g<i
zKp%9M2XXnh-vmmFuoO8XGB>_8`Xi`t&B1UEyIT~MM9HQetY*-J%{K*<=&p0`Ehk%J
z{M$;8p1862q-=9MY~WLX<Db3^+7GtsvFe#O+)>|~U7#;ArZz=NF;;HlygbsM$`288
zOfI`dVQaceY{Pjigk|;Ng*#n}<aWG_Mgej@W1xNJ#voAAunJ|&7^qQq*g?Nua{bCL
zq;=R_<cuGDzmi*H5bTIK2R7u{L%EKB2_NC8T9Gj;!RUyN(MLE)BnyG2j^xikvq3Xj
z1^Sfk7xb8>%9I1!s-!~~6p}29rvyW|K;*uSXcV{@E!Vf<N(_O7bu3~6RL|{h5nPC{
z34jWAL;s7`(}yAmN>+f^lfH3sQp#y-aDT7!`1P+(@?30_13XyO^wBW9ARpZL^JXlF
zECt@og0P^5=m6d}!KTIl6<tCXE0anf3t8N9Y_>D-mi2Fl{NzH=|3A*YIxNd|`Fe|>
zgb0d&l+q{&h;$<e(jX-b3Mk#3f(VMzAYFoVcO$5jbT=y9($e2NctZC%=lkRLj~8Co
z-txZBbKf&-)~s1$8xO*aFS2~=pHhq}&Y2+d`6grK1!yAN>cNc%GMwXK#U1&CknNN*
z%=5)eV7-~7z9W3VTB}aQA?`$+KhknKH9*t6l%n<mfPXA#<?#fH5q5t*PC)z#LP=Rj
z3>Ng5g_^DvsqdoZ;*#NKOx*NWy8b;Indg#q#RXO%NZEemseHhfX@E75j`F$zH7mPN
zvX8z8mXIIXlofWsI9gAt6B()r-z&#!3_u)|nQLhj8wTHeLofg?33DpXFk@!7WfBC&
zk>RHrY3F5FRU{L6n<9gGP}iRT&_NC2q$Kf05LEmFFjSgS`|G>jyq7PAU!&y7keO4|
z!IQcI(Car$>?uj`RqThPzyT>QWJm=7C=~2!gntiG{dhNQku#yR=9<q2Kn5v#TDdw&
zf&5ZR6p}$TIknzNaI+6DR^AiEXL+Zds_T!!4&kM4;rf-mQcyEYs+XCYfoUDOR+z5L
z4XJJ(_>NQ`K)f~tKsHGJ3+$^IiDomp%p(&A$-I`N4cR(7!dG{4Tm?el+l-c%@Ux@%
zoC*;Ev&2%xa^aU{Smv;GFraAzlaP)Dh$R&$sg;$6&C13@_5m8SJb0<w4hB?apnsC6
zttW)w&F+?r{wF|7*GFFSbWvvkRgji0*uoU{t7tG9iMhFN!|XH7_IkscrJKpWJ+RY5
z;@NZgh7$5X!UmiSEC9>eB4KH@;!!7pkc=_<?&YWQ@K%FnWE2J-%s|MLl+|)28$*jc
zumjPnL!QB_M6HjOfBVi(MInDSQZJwYrYXeK%|U|5;jlW^wE*E+GE01_oeLKHfqPih
zZjZ+sO!%_x?DpNFgnOF{EyYH?*&@Cd-6!FRoDTS$sLLTfm%TFR8Kab{n{<j4UhHrX
zO2NdBrox5!Et8fp%<GgQA#&;apY&V0IPw^pdpqt@1r<HZOhFW)M#tc4T&~lT()rNP
z1H+R4=!@6WKhi#6sxU91q|@+xrGxeWB239$h!$pCQp;3ugyhjYM+maJDZm`452kaf
zyOJ57<cm$p4iy?mDZUTvQwgY#h6$7|TTQssH&u&Zz0?S-2r`4=R&0L!ql4Wt4f6^b
z+Yx}N%diU!Cyt;*Qz<_cV7EQc6*S7p4^)$3oshXQso69jz%tfOK2{?a5ENEGMRpni
zQ{27SJ*<^DPj6EUze{Ne?22g_bYZcXl^%r|#p(#YKPe`B1|&kdEM~cVh18YXpQj^e
z#J?|nFWlQ`h6)$HO!vYegRaPd)U%}p9Rj|;>sAw$P%||UB>&!;b{Ge=GT4yNjX$bA
zzK)Oz^yq#1`Lh7gOAtE&ke%s6v@C-}t^rmOG9Hl{COIXPfntwvUI;X(!`#r(H;ucf
z$}nE2>uHu>e{NOg1Wc@KPvnV9we#}i5(IW&9tD}qiAEpDhO0dQ{y`qsz2quw7asG(
z=hN;Y77?Z)lW9M?K)H1^_<koP#EZRfuwn>qh_Kv8P>RZ2dnDue8f=_i$bcVQ^e~v4
z#&a-&pHzXzr%U%!j+V*Xwz|!~iZLk6kiO{jl1U1H=N(1Agi2B$Ey$ipl1;?tXgS?k
zS!#)lnuFB^QOX?mAHf4)DL~oav>NjBM}Rlg(q!X82ExUuO@V7E18;Y>j0D1xhj3_C
z=K_|DR#n<w{R!Tm32CDA=O6&7yyMC!z(#9eloT&M4c0@Vm@uj_y(%j#66Cy>EU21V
z3d-*x2&ASZ{VGy%)3xq^Q*vPJEvdH<%>==K4pp%HKtjX4SBA3$1CLfed)2aqCZfDP
zq3?wWrD|v;PW>RvKCs7)=I_8>QdKYzYoP6qu9d_ZxIz8?FHuz_ATL~Rs!XC9f_ctw
z;rWPA(*y<php1m;4y2#JLWq1AT;et2RxWr@{sb;_5n>5z6rIU;^LCm=$&g{Pe@N=3
z?7h(BOh}|C%r#pIWt0z;*s09E!BA)s<Rb2tc)~w!nGg-aSkT08H;`^>Sk&O}J0%Hx
zbOi?(%p#DHPr(7R7DUsOvnhLwbYcF8RNt?p8zWKblhfNF0=TTtA#7W7y(T>iuLze?
z9<M$cpFySE<Bf_CZ6@FaEqho)<|{kkKc5M{yzt2Och}ZbvM@AM+%QS10k{y@CGSVF
z=n3z>)|Ck}wAyE2R87ug%@n}vZ!n*hSEf{Ez%bdLDiePE2+ojAJsM%8I}`p(2;>2E
z>0bPp>xSIQUYMoe0e`CKL*{>aGvz^(zBrg?v43vpaT&r|V8={X#lc`lJ95dpK8ppA
z8iA)RdfeL_R>gcV3fh&t!7Cg}@)<DpSPFYV)PeY|mPiWR9~c9EV?^ne{}S0hy#PG@
z{yXAK-(QCoX{DB#dI%i6r?3C}!HY#vK0SC`otH<Uh4lA>N2$sR^NzfW>B+)7vw687
z=pIy3qsd(G*#MocfSGlwZaYj@jm7lfFSqnvbNz7_e8WBpKaFBD3W2paEM~;@(QF~_
z?3L8>#UcSDWZr(X0=KvjW{4<^dca9A2b@_Y+$LTl)@l&yt0N==AAomATgCjl*UKPy
zF5%x9qB0`NSpy4d`>eZYbhfKw!&k&Hu<XI4VF0EayAl?@?qjZ3XX3{gQ>rSCPbrs6
z4zd#eaXp_sd!a(UqX}BvY5ti@p1;Px4TkSR^yrn$r*&YM{j;P2%Ff)=^%U}~vVH>c
zI*}CVX@69kzKUc%Tu+Eau?`bX9JcdaK{b0>SzxVz<g+>+h4`C~kR{})ThRuX1AH)%
zUpnc2@SZVrB&n4Y5S+w{y^aTnt4jgIq{&dJhKecl1xQ{PNZ<bg!;l1vnO2jq(<ucp
zAuBKgT1&#sxxn(9@F?7<TfyQfGK%+=@@K(vvcv=23hbfG83hlFkhef1n2h@|B6)8z
z?`w>^EWTGShawWbMDHkPOR#^L+lFAQ3lLjQ;Pq|hKOT<!01~?f(#q1$-_K7?ucj%o
zhJIuMW^P)A2FpM7lmF_LSYb_{hJLFAic;UdP-cL4NG0u{Y=W$a?v0Khf~TGFJILni
z<#rt{m{Sd4R4bg$3kZ9vi~%cheCb{EO%NwSelQ|QD$3Ysj5f)Q;_%m%A(nW+OV6U(
z`(BFB1-_SvmIqkrn+)YAA&EQ=u!T*GpojZ>HQ2vI#DRW2ske^^Kf3}ZIaZpf0*k?v
zrvY+IL~(VdP(Ms(W~>3^F56xi9RwK9Z~*>%A*l;!@IWN|U=~6-{`p0oX7{7*K{f}T
zKf;NpBt9U;X#P!v^)RbkNL`Jva0U5usq5Z`*J>AiR`%l&pW5vEI)_QjdyWu!S0J0@
z?%037IN;dyZmN5ZZlhJjvufmgR3VDnPG)Nh&(nRlxOgi~P=y8GdAd;3H`i*s-$t|e
z+Mr>FCqdW0G>KWKhr{@}#L&7*THunA-InTa%Hdx?>^s)9I+Acd3lf?l)R`ir5~uiw
z0Z8*99}XkfUUoyK4SP@r);FoRQc78p`Lh5Vi$U9c%Q)Qq1S~boGQmh4eZV{8)8xZf
zkaneSf#ngzr0bXyxmvt2EP6h?E8cjr`2b;9ZEK;X8wLLk;j|1^dR4sapNJ8%Nq?sD
zG!NU|Im5%B>qs7kN?LuS`jl9Mln)%LO`a7(KENmfReS8kKLT}P2pav0*bn5MBdcZP
zoa)?<mciE&CvMhm(WW?GG97YAiWYDh6Q;p|$aFNrnt9o{eJLp1M_{gRsWVI7zZQ2I
zre_Q5pP#St#P)c6O)i?lZ1I@a5fJUhS@!zR_?cZ1&*yD0$U4DU;!14nj;0RnJL4J{
zFLL=f0#iyP>JLU?D#J&|X$F7Ipz*$&b|$m%Fe0}<skVGc&{Mar9OVduul6v)A5$w&
zr2%w^yli;Ywp#C*^Q2$%WbuRT+>-@%xuNb}L2O&+0;gJlCb57Isxgv^ET=~Z=doiT
zpL&mk-+LTRU0;ppyrosyhiy@_*})4$nDn3-5c)<-==S<p1I32j4LL)uupYQdIl9Kc
zV}F5ech2bhr&p_vx!#@n^efJbaVAKw=Fh5)5v4)}AtBAUlb7L&hD8fioHv<{`zm+=
zOcOmNNzcmWH|~+xIJzumq-DCA+0rX{=k+a&L%~{@JBV@V9#b9$6snk7zMJQlnr9Gu
zjV=R)&MLg@SDL<?_lLXexbRIVBNis>Jm&9YjDqgoe459a1uM5R<I3d<rn$gs?iD#p
zxA!|^7-sjbO#&WJErvPOyc)`~ER3wYTIcT49Gf37W_#Zp{I@%I%T|<LW5mf#4ekx7
zc+}=^E~1(UN5sM2?Yh{_rp+#=YFV?UiZh?$CNqks?CAKBdwINHd>twz_nxiCHR+2#
zJ%Ep)bXb5gR+i`O?ZI>bI#s-i30FEzyMATmiNN%u<b+B8|1^yBv?e;cUuN3pF=PFv
ze##PsG592#l*>}L(L<02U-9z3gNrH&>l5ilgKeh|F#3xJtFW;&_;NZiD5zue#Tznk
z<E(1x^)%p@;O$q-B%GNUiDmu*7K*|5pm3oqvjK!PQ&|~1YeWC^zWEp4Yi^CeuMfoz
z%(b&QZA^P_iqaAd#x3#0S<q|`+CUla81vF~8CZY8I)@9N-Nyi-QtsYQ_^bu>PH=Ax
z#(uW(hi7MqcE>RW9E<4qd33BpEi<|3iUUmBG1jQT9WT_1_w$aQ-qC|Wh3%#Sx6=AG
z+W+N|*(`7_Hj^D=u5qpTgpI6eIpFG}PhhL*?e8f)_|QUq$hzdcMfk}|`lb*9@9CwU
z`LQp|#lRo5?*|lL!C2G90PfkVU|5LGPqwepwmM!_)vJ_gm+a3W8N<vaH}dJk?w)dX
zciFo|k|+I2WKUsO^hAU5s2*cR4a^<KFdod&mpt~z`&j^3`h8yZ2mZ`zR%k&~55mhw
zm^B<N^XcZ`RnE6QvTZ$48IS!lFLHa*dSJ(xR~j{DYCtQf7-<y2S$OC{6#uXM^9wW{
z-uycDgG?f6@!HG~;zgzSIkFGw<wVoF-YY?7DJEdFva1CK3SVbOBDL-?m^)ZNNSBx|
zjN;jjdXf2FKWpa!cx;sQ*rpzJ7L(0=v04VR+zWA^F$R?OCU+k(D7}w?11Mk`GMc96
z2=trTb8L>6v2?~>0&?@PGJ%^Srb}&WFIeDy>p<*Z;n3uzF7&cm+{hf*fMk<uPq<wI
z*NLyASC+TkcEZ{$^Jj8rZG*bf=V&k??1&Xo8=_2d%S0&#QZ5?r5Hn8uVqCZ*Kcnwv
zBg@~utkPSd(oA-xtc>jS58Xi1JVHxX6%i%r$$(cc6cykp37&)&?`<cp6&({QKN6V~
zPq(1Gpm9IdkG<o%qez5=Rwp=u5bVA8RmzqSun1Q#c=8dOQ#xC<fYLVhmRjMboJeFM
z&|CK59PEGV%+_PceB2In-6C_AMXnYs!ko63h0$G)3B5RrI&>45cvJAWCb?oKqm=z%
zCPV4`Tx05O*}An@n+^#LT0638+1hw882h_zYvP?oAsQ9ca)5P*9?a(w(xcmuZs6Wp
z6HH)maaEmY;`0NeS<*G5Br4W#p4A)sL2SPAxnT0sSSv9*j&Dq(c{+7nE1%Q~#KPTX
zI}>zr^d5YUhj|^JYf=LECQ9Q!ArE&L=}6{0_NQN+?p4XvjI#97OM3DE-vk-=bmjQg
z!3#?zy3A0Utj*lDUk&z-xYiEsw*KVz-E4Du@}GXQ4UIyl<i~C8hDI#6H&~5At9l=@
zBtsVpI9wD88o6Cyb|YQ-^ywR41yWXN0B^+`1@U3Tin9Q}msxL;p45d=CY)ci--T;@
zL$M@8!f8%%PyoN>_V@tCov7ea42(zs(26N3wbIP7BpWhw9NGMvZLFylggNbunV5b7
z(Bg8xGdCObj&wrXWyJu-VjoXef~m1dNBhm-UM!ZollL+$7B84&rWYax2ykmiIfj+i
zr{Te(Lvj^U&bIQ>L1=Xg4#baU8Qh(>bzJ0PtIm&E@F<HCvqLQ1&xwZ@TRkJST|i4e
zFdt(86!!#m2v{u*0ri>CN9cODKRWST=O3BKvksNMpusCM+8rt?4+_+HGQ<iQ!v1;Z
zW9Q7y#N+iqkHh};x~F9m&RN_#Bf$fC0gwYbk+HijBiYFMf3W+Ubn@Tb1upR<_YMQ7
zKuo8E={RBVOaQX%n}h?%=Vnh~o)SmPmuH)3417)=Wy)VRwNlA$G0?Dny`SARx&|du
z2Q+qd*503Rx&E}|oemN?9PEfl83Ox><+`2tbT}1DV;(0*+7BC>6tYx9t)5cRBBBiQ
zl7zk%I3o&kUuH@+XQa(%XqEGBHmSh*u;vRS#rp#)KNH%+x~GgJ-n|5u0BD@~fjS1?
z1oRjVhxzuvHb3%|#2nxXzh(-BYlkz(mhT)e77Vtv3%zNhFkuwJVFp_^!5%OVc;GRx
zCqX;|ePuTxK}fQ#=s@2r^p;{veYEG<^OcWs^n(R;PTg1w`oM0kxCNT+cjh#td-oN2
zt+5Aef&78>S&^0H{~*F2)&ZE5yR<(I)KG!*_Tmd!@c*3miH2e8&Qh?;=rZU-MuSJ7
zV4zH61r+%k?oGevDOqz#^F$Ws;Fd{SBnwg^r(wT<(slC6xpai0EJhHGHlVX&_ajej
zo#p~?;YgK1tTbeF{z~rbt4@{ggu3S%anOUI`@J8d-*N*EYM_l$toRfw$+k+^J3l$H
z@2U?yf&r+MyY>RPzN5Q9oCrx>fs>Y$5O>3|2kO5Rr|rkEQ4cAmirw?3!XIj5998E5
zr?~?tE2e=d11w(B7E8yWN1B__aC={K9Z>F@3s?(MDo;9Mqbf5~21niw#EopBk+D8$
z%Lm*46j@9+EubM@7+ks26ET6T9RTk9Uax9^F@VfccnO?F*3frtcqB_hegMslF=p)^
zuKuT`4`lS@^b$!m-fPQ!S=Y3d{)4W7%7^)@QkqpA`5>=?TfoknO>WFn$~AEj@v#n;
zR*Nv0`^i;@gy_DjmR{@~gxo%5V7HXnIw&?bO7-#K;?^oSOQ{Vm9bRIyc9JpdOMfuW
z1LKAT%`>qUS<;9xNYWQ4fe!4|S1^{zXHC{o#_7Yf=BoE}`i|_2An$;tQL&-qftZY$
z#{`;ricxgdcCd>4OF47>cG#i~Txlm)WTv}2pfdxYl9r5REq00F=uU?yIAl@oeuK?`
zvaXPZlX~-@p%%N-jxpfPgzN&LWz1t)In?9*ri3oJ--%&I-986sZE((&%DBgMVm*8@
zS+Rh2i>q%z2|aV8fQ$3sUhZJU`VHF}*{`(LL^76{XJ~908I+}VUQbH=7eE$$^=ya#
z*H`H$#*aO|kIcd^ytZ#4epEGvp^<Vl%Cg8x=vIr*Q5TR5CTU5du!nK-vh4O)v6&%J
zTD0W+9iZ05vmtyWec_^b+hwq|cMtp(dfr-j31HBDxLnzm{vf$$HjszTqGnrVo+s9?
zi#w01eD(O~z&BSePF8fe%Z`V_-y81@hE_)gJ!z%1Wekr)1sffDI2V{<U+y48r-}XX
zYN--x4%%2Dna5z27M##>u1tA;PIhLtSu3K&s);PZ{!kW|{+p73pJp!blSW^->Z6<Y
z;`|zn5=yvq+(XKtwx^2V{^pI>Q~JO#CR$ymuSDsHy+qD5Vv_McAIUF|b6kLH%%3H=
zIG-JyEU>G5Qz`%20B~)Z=bYDIfBIZSZq~gyz1v~)ufEB|b`k0&HtQt<&AHDoC&gS7
z)^(jfIoD|rdIZzgMc6481D`|u(Qcf94Sl#3?NDS1P8CK+TIACz{&kO&y>G{ZT6BDx
zBFh?Qz;M}pqlJ#AuR3~jr8rpc_Rq~AlYH88=~Q^>cTNJ-!gbq0*7G(JO7F(aycpd(
zafGP`0?a?Ze*C*d`;9sl`3z1ki{frEj2Frc(Te>vE0Y0Gtv?*f#z@<j8mXS>e5I(x
zX&Z&0Dr+LfC8a@Zcnjq*rLo71htN6l?(hIIEv0>JB?g7!kr|X>!CxG|I0wGCIB=s6
zZ=levr5n7X8wjMTK?kE2NTZQI3yl@=`OPm}wQP+x?b|HCoOwp1Z<|6j_6jJO7tva_
z-J(WcMA1xQjirr}h%I6;Jz=Xnmo16xOj5C#<Cp%OCdr!kft}69<A}*Mj#$Nxn|}u`
z&sc(%u3Bael-CpU&sJX8WT`!Ug+g0bUt;#69+(tO(w~6#$n#Xhm+%296mzJTF7n@=
zi`hgP7fHR)!#C?FTYVvGllgcE;cdE%7H#PVh5+v1^VIw!`*#^>rzv88bU!(ASTh(&
z#=B4eW(XWPlNqXd&|;Kc!m+PD-d~z^4Nc;n{4nnTeWd->0v>$I!`ghhjY@Dc?WHAi
z{ZY^1e)tmt=f?`q`dGo#5iqnzBv0lJKp(bMtbd3l`Pz<g;8^4%Fbh$`GE|0xC2d1C
zpJAcmf-KJhHHegUL-tL+V9}}S;qY(C&#&@uszoZ^y<w|fDmyp&5b*^JF5>p2u`yS9
z8H@1CQJXSR!M(|}hPO;irz^8Jv6E(9r%d}6d-e&LGz~la*=7&J1<llreetqc=FSE7
zncJ9C=0n;~+kUzPQIX?GTV(NYY$pAxz<ZhW`T$_IRbjJ?$nJ*<iKfz_$FBh<yeGFY
zH^+#ZRoX9@yA#%B=fys&6oH{NkRh$tflZmz3%<lkHnqVVw$VIx9d;`xkhG}g9wEwU
zgp>O9hr9#|M4ez?Uc;`$W}}(g-@u>==g9O&h!j&n;%mHG*kXKmXKQi^acw7Jed2A+
z6&~4+k)I;N`~4-=vgi40;!C?c$U>#wr)C*42}S&1Ss^hFp3&m_x0tyKK(>(DN&hig
zV6zHlse5S+4ldj%e}<P`0Dy>MXhy(+etQT!B6|0W?6{Pz%pmN_#k6O94_)6>ajVU;
zi5juqYZIH}=$(@t?oeLjm~vf#8fjy?B`IbM_-Zq-0Yum?_8s;e=tf5r$HK!*lXO-{
z(4J>af(8y#&g4=gtm)y#cdw2(o`U-4oBJ`ig>;90wWHe3Z#{C3cCN1eyRs&c1AShg
z>RQKBHAEG<z>0gynF?*A#{MNUfIr4>Z#v*4X$Mm=@136vF$1`H`JuOi`7_j149%6*
z;b1Pwn;O%VtbX&nNpUCURW4=0v6Ev^5R3vnY1YAcM~p0Bv`)X}@~+r6kl63r(*$#u
z0+)xQQ8@PXc99uq$m;DJRf7DJV8zN)nWR1X%J@!~%t>Xi?%u{ss$jlJ1!8f}R9SHM
zK*Ntw5qgaW3`Jl@!)uKX8n0li+1q!&VWvgPU$lv~Tj=>54^|Pev;EQ?W$h!c=;9-L
z55q~bTWrT$F_`sf$2GeFLj~RRuvcbREduNktyt1%d*o*fz|p)hQMJ$dZz6)38}coj
zUQ4WJ#-|fTwmvgpGCH<rTtOV)OQA=32snYVPUt=>B6OHls#~AK5m19t>4!m&&Cy*Q
zo!rn7M0I16IZkc2JKy~=9tg`6ff~!q7eip*6*-{O93Xt&<O<3N$Z~q@A0(74vJR$~
zE~Lg(CNQMR4$e7Pw_S;aR%$mP{tGk_g)&9OJXSQ5>3OBmS7YpM7y*i}=*%hw9xg$X
zT)_i0Z5bTqh`mxzbv@w&oo6uCoc_t?cYJrsAAxl#qJthT*}{r683|ZH+WZ;NSWysl
zMD(70l%62?tlt3=%WKuj3}-+IZ3-;h(2snia-<$zi82M1iL#pAfC2cN8oq%2@tkfk
zhRQLu-J1}*vlqXMYUTETO`)3bnLNboeL7p$7v)+-u_Q$A^qy)=Wv4q81PrRlX7pT}
zc^*d$)|*N>uTuWJr_g1mfyW~7^JIDayz3Ra{iby~ehUA0=%k61i7(0Rarry_to@wN
zV-lDqKIQ#vP5175%d6do^||QI1oCIrQD^uVhD1d_`=Fto!SX_(s5zO78XuPC;4O;L
zcPN@+&3iZ&HF@}CmUHdZ@a>{WK6UN^!B6izFJPCc_G<%{z5zhjY-5}S3|lLpOB-FR
zfsGhcT_EM^Qa9Z;NyMbsKd)Qd`tl^DX+TJxI?eLD$reu4^z<|vk#)X{6>&hfU;tkF
zM#-@iS<d%mwhA;8ghClKu97`?qf!OQ=|sq15ntEDRCco0x$ZDMIiRpIDo+-;*okAH
z$w;<Km<>f!z~px{rp?KVT}IsSZ@k~X{(4y$2w-7NF<F!|s3FSFbp7XAAt5YTcilaY
z=4wf{>>?s^0T`;wIqFkv{#RLq2^iI<i$_Y_plW>bIR?zG?&#LN1wF$BB1$;X_8)v#
z&ft6>yt@3TR%bVziPj}wtN^~b5fKQ|<ab+JB^UZKLJbYV<iT9K%Ddr;b%OXglZ}my
zwmPLzP_Hzt%a=3;`{4f`af9?xT1d0IrE+$&w<f|*Y{8SU1>9i_IO^cWSkEW<?mGk@
zH|i8=R#+Q^+Giy(K4%AqR_uf8R|-46IQy%>!-~7UeC&S?9`WH%IbvAYn7d=K!&rr8
zVn5;ZRISdvCeI?foeRyxOF4dEvmCDU^ji&cBWp=OvKOx276J>w*R%)u#OMR6m*WSK
zHn-JTsoy^#?n4ywUAgaz_paXhl_n&pKk!$~>zI16cXQ6Yg#T1%2)aIwHxL=|9(3Lf
zz|Fk9u-~@_pke^XZ}9Gxxq$T$l$sYYMmp=B`w$gpc7Z|CVqL4Ixdb>c_ZbuH*YCfT
zMm34fB!kVsh59WasSFKW?>rtUp7cCP?*-Sn87Km<MvgpA4iZ3i#b+p(1vVR6xBT*}
z#qOvkpS9ip;lu8j3E!dm_ZGfTAy&EDZEB)xJpN#(M3`9M!Q2+bdLQxJxq0grTh>%+
zTH5Ht7<{DoTN5)o#xf(yR;zV&%A)5uNRUdLF$AIG9V-b5iA0K@BMoHP-IjjmoEMjt
z+?rmN5P7i|*o)o^Ihr$C*19D2+a!X}Pt-B@Nu0?CL;oy`y%ouc;;p=U8AuN(Rq-y0
z?s!5VDGvUto*1VZJk)z$x$6p@5I?=q3;v4t1RZ~XFRxFJ7TC<vBg$n`hcPRP?Yyk4
zPyn$?h#PJcFwH~H+11q4tid7a3;sh*in@dGHDTh=evlc_bpIfhs*i@GdY6qYI68Xd
z_J_Z0oR78-82Vo6wlgIuyy#=@sVB}+DKa1D*%Y}gd-k2Nf9Qpk-`{$WG>ZB43lcuQ
zvK18-vu>zvC2}#G79bj_YzD3IEi8gt`FVHd@}Qyi3}0}WEY6FLc&OXOq&;qHvI426
zlG~5B(ga^kjBX}uO}q>c3<>SsTR5siBzD8T13%uJ*mcEit&BbdwZTj|V(an%ajn#^
z@p)YQ*x$!ImNw0g-PsM~GMs7c>f^ahE5HUXLwlp}(Z_R}f!7v}x@M(iVOViQy``hW
zp5Nl=M4;=s+PlXxHn~E*wJpb+#Tn>~zkMO<fM=s?2L-loZ)VtqL@CloRnRn=MCnh5
zNja;$m}4lC?78zMZ_8Ms{dR4S@tZt#2^mgp{}lxV6B?QvSp`L7QH3QMn$JIb?5C7%
zwX5?gvqN|LHP?*3)t&wh3iQ6gG)HGAopu8~ZI2$m+Qo-jnJfK~dF-`|Jt-ZAd07$>
zmFjx699qU%tZ!y6O!b%Js7KMTD>ptgC{>D>6IXW1RSKV#P*%&W$iww{YpE6Tjp%;1
z7Tf6=yNXzHbzSf*1vh3OLJGlx_#ElXYSkJO^i0h@*r_D@pl#i?0x?B)s#qSNY67<(
zYl1*T*jP7U(Ea3$`Lh4R4M^nKY9E7C=_tw?m)(m)yCBDk018Acw|3tc;gdcHnEL}a
zSgXdEnwUtg%jxzt=b#RFuxW6+x_wZNW`f9EyM2E$#%+D8bdS7ihL{kt>L=qX#HMCu
z(s%oAGjcnmx%m`nomBuO3(g!<yW7I<m@Mzgborz>nh8BUeOujytkw3Y38y)-ZXCjL
z89?l8g5_Tanrc^5LS;jMejS+5??C##h0yY6H^LdP98i1Ok{l>7k}?Xiw{~*%P<#N_
z_ABbc;1k%u<E49Q$Seyq4*riw^>9O2$9`sTnB=)~sxzQR1E5iMWFr5Z{|J%&u=1Ru
zWut2W4ssX><=HBB5Vy(snrwh-c`ZCgoLR>%!)Bm#4SsW#<()*t+jDZ!%gu&encPrT
z8x~5g6&Q(s&cL)(S6Bbw$FVe+7lEX|mSYrlZ(dbUvP?!&5TEE7eSVa~x6H<kCcPz)
zdhX#EZJMtCM1*Cf;9_Z={J_FqTP$}`_w|Ero59wvP)}&zsJeZ?6<3yf!9GE4?fRnI
zy%UKO9WLmMldvzj-L1L4C<v1eNVvv2c`{k3$o|dunzJ~vk#Z?Qob`}TCMBz-;x{PY
zrTSrI_o$ISxkQP1WyB)<u!DrAtoKRF%8!`oE9T+Z)sI;(;N?89Bt`I#=MZ66kgTXg
zZ5W|lP+?A7%Jg7|LUqOKFpDMJgXb1A=%F)2_ZgxZD!F00!Mc$3^#2jZ@FvbK2*|^U
z^M(AVek5|znj}CCA6U)+G%zX=1a-z-;)1SSbsip@QrFv%)cp>pYTNS=yr?y**DOub
zar$hmt>4sD20u=zUTWKXc=qV=e=%OZEb)&QCUNrBZ-RF-0ddn~RhrUSaU6ynw{ENe
zr@sKsiU1AIPtGsU&vvK)r~8=VvJ?ttW;wstug{uienZ1%L5+?*7H@?uu<ZwSo~7#+
zHkbp3;n$Cz6Sv2)@!5bDo?{rPVS-*WRi;Z+_1}ioY*AcR?@H!f5Dov?b1|-DZ-|zT
z6KZ7UsPMD9H*GL6%*HGG56_Y#4q_*kC`^!b!O&AUibYTC!%A_q|Afc(N$o(#{`m+*
zVk|a#Dck7Nx5^KliVv@;{FbM12T{!H*6uHT>S1Mguh&`9V-&HL;#Ri(ggx4jEN1HT
zty0Ge**WnJ@9W_V7n*Yp7rH+z8j*PyXHdvD_456aSU&CB+;e)TQy7+449)W!t?p<4
z3kLnhV2WQoQ-<S1{Y8W>$1odTpV!v#kZ_-#fq{mBA*-dN7Rs_2pq-6-RwofHR5JvT
zsmWW2Wu6-Ma0X?eiXTD-$Cx^RDi7^~0pf`DPC5H14bf|6$C9NR10p%@O*o#edvyy&
zA#`a%Y03P3z~xpsfN!IjHVcfBX?X@_Wg*|@X~Lct8IG)s43>`qp7d@Y<kKo1kxInu
zBQ6J7j6&$qFv&Yexq`Qf;a-mVb{QT+PaE?plW|{5)mIST-9{YG^`R9h!_og?2;pE;
zv`C7L13X?Bc(I?!K;hWfWFyD&|2+wKBS|2buRg4gXZTS>e~&>RbIQ@$fTBNb$j~z4
z<SYL8OpbBUN>hB{`clpG;F?3|er2(Dm-ySKqy@GGFr(Siuls8xEdVM4QX^F)k^gq<
zkW!jD=h^6ak&uO?UNFqp2hZ%gbvVV91wP!x`_}nJ(71epW{7R^Q;xa%hZoV~t}ZV(
zn0*Q3gGsI@>;^7X9UvGw-srx6eUJ0YScT0Dv?0uTa<M_(9|1n=tsu~E+kWxl#k+dH
zI&fh>b6{d{H3t{3H3k0xQ~6gfTCI}BO<)6=oC6@J#49~~$hm-*{tdK&;b7mz+C|^}
zqYH6`^>NC+2&_vuOmQeU?<^a!vpXwGVPjUvbYB=SM6<~8p5h1(3yVC%jE#pDGX}e>
zR-jmEPwau_H7#*&k5HgWGB+d`GKZYOukh>4i{o`=s4fnoxyM&-<o`Ar?paP?dEh2^
zneJ_rBJKD&*IQb<KVw_jg8zDwaQfT=Q2J-MH#P^)PNy;N1ZNg%Ho33oQ%1#GeQMo?
zVNr&MURI}%w;bFotY#klm-+gjuv4Ux>MM)*LK^->F;oor@tLxEY^hD=h2*1>=w_s;
zd5#<uWDec(A~2|Nc2zQ~5bpc6GlQUfc?hRoN<vN!r(Bx^ie9r(js*}?PtWoQUg+1G
zFA$C@rf-K1^z4Xo*0Eu}*x6e~Mn)T;+l*4SJv=KqAj|fW)$xa}NkL1Nk0>Y>^nC6i
za(mjP93NQpBE;tSj^g@3ANW)vdgVNcCOd%mdv9}yu*(r)iMr`lW>OLe@7Rp73?2b*
zv;hLHb1RvXSe}`iKP|4_Qq|iYydA0S8yicSb@OYM_AZdLoGetTctH%hRDyD63%h=E
z3K0@Uzqhn|#Kf;K^X7}qba{7FNbv9$jvX~BE#2u!#U(`PSKi*}Ii!zryI*lwUCZ>O
zi8N<-J`(L^f#;>{R|QxNmZ#AI1!lA`mW3Jq<8+e;wW09YZfG9$^(pNEP9=TLII!Cc
zJQ1_{IBlO1%oaUCr;_iFmfuY}LSkVzCW~yS;`Y_FW6zNi1%H)n3_D`5x>mb%|14{J
z6jad`d&@<gDCX8}EKQN&=FwGh2OXK2nQcD;eEbj7<Uigx9vzMV<%HuNZ0Rwvz%+7?
z(6SFMF&V7f>Bp76d)G7U?c2(o-tcc+ga>zH=?`(U3m7UoH#hsneM^43nV*V;QamQt
z@<lS=)ZEb!iw&nCB0@F(zUT6y0IBa^(aLLSF?R}T-wc$=3d)JmW7fr6*qy&u6(1}B
zGtR*TPfkw%u}7ESP!fpQd;OPdANChc95E9x$`d!?BdUo!pNv(}23GO4no*-g_|kWs
z;|)S&6HdDyV=ft)ifq&!QOFACd`se=kPU+2IVCc?Xd;*vj(Aieo}t;M-}~s~4`8{4
z0VrKORk%x`RJKiqR+$vcnOrJr^`DZ}RGD`~mNB3$UkbP_DFriWi4Z>~vP<zSEya?x
zeUXj0Ms~^eWj0YDc#d_Njq=~GVnl>AND(aI=H?b^4s}-$7srpyzCUZzPc=d*?vrw?
zex1ionx(M^yQD@D_Y-MX>mG0+O)+v$v{uQ*nc&e@gZehijGR|-c>aAZw#%`PT>ZUu
zWn8tG#gH*k>Ku@nrqpc3{$n}cZ|{Ws>x~a0Y3GQs32teWW+&BK$YzBZroW@<1>utc
zDA__HA_zBTXJ%%s!B+L<ITPV1YisMazdB(@0Ewi7P6FsdwGjG9tqgVnGxkMgXWuFc
z>44})v9QpX<|!QM1&@o6I5!IBVnO-E;?3_|X=peD!sGDIuL*h0rwGKX#+ofeO}yA4
z3>HGd5-JuBIZ5d^c0W0o`~m{6sXF?vg`*W)i~UlF!zDM2oSro|vCss&{rTNC*pRf+
zUJDF_fL|gYFl@<t=YJ!UzN?U5u3Lkif>v%V)kEi%?rO<=tJlpRPFus5g>_d3ORPun
z-*0^OEeOti^~uFnSTmi22g8fq47Kyo>3<YW2ckhI?XABvlAuEgx%<pl)<16p+H!Qn
zlIUzkj?=*9p%>Gql*aSB_^pc6A>2~A%!3EPNH_N#zAal{*VsasqAW{bQj*`!$|$dS
zZvO+&!cl|jI7BZ&{^5LuQAq+!+ev`VIR(bq6jW7*^f#lUiHG5r%pf7Ou%j+1F;R#K
zHfbQTQ+-5Drn90kk^ks9RMkxv$c?VHm@Pu=bOl)$W3Er~fdK4C)CZQ*+}BDAI@MfJ
zFj6+Zr`8KJGUdoB0cPR8dY*Ybam_gZjG-pE6^(4g*QLq^aXT}Wv!-FX6?Uaob&eQ(
z7=-)Y3XS0&!#M5xHM7W&OtSq!15_qAkO@qn*CDwTAuI1>M(~n66VhH;;&cE0GElI*
zH+PTTNV72Xy{VvX`uP#dGjK3^z;DBOZ^oBrZzY0fxG+Pz)FLPJ9pkn_<@itf+=z`H
z=ZqVywQoFxA1e1r*@dsGD>Jh<n#h!_8@tM?GhFH$=p#9CXFJALGif8y+2+xi%dpK{
zdD5+@*G=zkTvPgF+4S@RWD5C}-H6(zZ!GLsIuP<>N#4BK=o%edBF$dQmv{4Bc>C5G
z(fg+<;+5;i!BJ6^`vM{1p_Iei@egwx-^N(D;-`q;^}TX)Ig*}?H%&Y(PQf5;H6tW^
zh2Qb2)wY}NsvG{skGj^j-Qe4+mBUq-b1t?;I34F?fRbUONIGp_lM_IBI14?FvlnHq
z$Adrm=9$6n-?GNPVp&i*gm&W?6JFg9e{~1>o)askN45#l5-+ONIF}g8r>F=P^t4K3
zy)(4CT0@skL5MA4s7%No;BE%<M&b7@C;3=edmaCZed5El<+=rxVqZF}Q!0d)7&G_o
zTXJ6Qe(NDU6O;LZ`KT!`HlNXpl*2<;B#G#M_QO|r@Sr<;9TjUpHHon19Gsv8qN}|o
z?>wWyp;=l|N-D;VkLNt$*Eu6gD(Bx27j7QPw|%!dmprzLg~^JZs=D=?3%8{O-E|le
z>U5|kv(v-ZR0f(WmF)`ciq?&u(*C-k)hNXI3&V6W_Dd18RXCENXU@NuDvSH5$Y|F7
zu*CX^o2Kwv)xcz@`2rf&po^_kV5hx5%=EDB=wke#2lx|Vf4>Kp(#ksNDlLjTO42!#
zLk>L_FO)+*e{!a(-|ldtKU%~3!SOe$J2PtzGkz)rhV5HXmr|T;te{z-P;yvax+2g#
z7Hy-eaav}T;&&y^bv+`op1;m&ka(%ss9i}#fw@ffMmP<Yf6r61+b^d0E3tjz@_ngQ
zL-Qlv;{AOL<=>h}@&Yu+`LLnEh$1OXSn*ZhV7BK(>@w(m=^w6U$U(qm@*Fs<O?W(Q
z*il7$zfn85_xran^Sp@9a!Re6fe%~v`@N0SFSPia#&H=`=N;M5sm`_#ypHCOF|W72
zf=?&am~dnmBlh=ceP0@im`o&7@#IsFL*YnW^CR~6^(wz-0!YE@Oppv5>%86ws==fF
zG!Z_B5NmjU5NqYA>iY-oPgMLgU379utsXyl?NexA6W3Og*KnaPjNtxWXe%Qz3(4u%
z`IPlcMTz&3>DxCITpB#~vS!^zx7>0i&1=(@GU=0d<NJGd=LzD-Nzf42`LLH4|Mqcy
z|10imxEFq(vl=3C(V4)dt^7f;%a2XAk?*9xO$mC=V-!#C9>tDbsW7mQUv@DRD>|P^
z5e=6$Xb(q4TvTmKAa-oov+#7MfKUU5{#;6PI#%Iyv6PLvQvK{yS(d?fA{qs4;<(+s
zx7GLQ#zbEy;Q?DHLijxCxATrv7{C8jXdXTr=D1%x`lk@cw5pZF@dfbNc5*SOeygR)
zHBs7++ywJ(i<-y8Fdyy8S?u6ytI6Nom0^ltj9HAXVI=wWkHYJ}$eY?K$~oJduUHse
z^CkRxf3tELft199>5uH~Zh_`IE%|qgkm&OIb;lo{<j)U=%YZzyaE5v=T*G+yV6#r5
z$((|6$w_T2zm=ngB%ZT}wRz#jsmzA%=}&qC?KbkhNE+;s^qbj)km8?SHUO^bPAkz>
z*6*bP`~EVCi(%7b{nq*UQl4gS?5^QlcOk7N)M5=ZE$+Yl!++jPKr5NZ$D!vuFG6mL
z0u<R>*jRXeY_i&EGo`wk?<)^672Su<-sa)Lr@y7}5q0vu$z9eWh3SC`r@}P~qITj!
zm5iw4vO6~!0}gd|ECxQzzd8Nd&nSXo5-1~1vb2h_23-&BOAgI!UnwbztsXJ<?HV%a
z+rk68*A$if`<wgoc|8+~gv)v5iM&0G3rH})hxrpU30+3_-Z*fV^lP40bF%*QDvILu
z{CWSi+EVFwm+CvyX7j%Sx{wA2sfXBGqWo@hqx9aMxy<QCD;kCCNv1M4*1D4d!rgPg
z6{Xi%Wasba==V=!Mh;hzLuf7tW?F;b;5cPo7rJUMnHm<+FZE)NX4ci|)zgY<>FSm3
zO9x-?3TvawA_Dt=JaSKT<bxiUo7m_td&dR^)@~2-RP3}}-UIbc^Jv$vNkh+4f+3qa
z+bAAhIiA%>;tlyH5H^a;1EgQ9$WJ7TT)RQ0!3f}|-+TGr(d$2d8WDzoYnIf(a!H8%
z5?b`ZrfpMh#zx!I6_M1T9gVn4#Vie+?^S-JPVlZ_d{sR|hCMqZ!abs`ZM>vszJ{@G
znCSaQIQPOrJ#ag*OqOgHyw-|UOJxzbon<nUspw}g<x?3{mV`_(!ec9-?0SDcbBu#o
z$9^@GU?a$!REJWEK;^}lU|&EzeE4ykqXsI75_|#6X#ajJ|MAniTkw!?Jh$Nj)<M;C
z_Ly&_KqH#3AUnUEf6?F6pRRZcV_Fs8hRs$F&6cfA56yx8O;7;)nvq3%wjPtqbKY=#
zZ~KwM?|1(C_mj`ZC_kz_`AQXPFp!1T!^BLJGL^IcPAr#LnNq#L=ht`jbi_BDyeXQ>
z*c>6?^4`}L)1dMCb0UvE`{5s2X5vR8E7LpI+@OAaQAx7@LIT=nk>tbw;gRv%y}$Pm
z-a+^FtRbA0a^$RxJQa%Ot4>*Y&K_J%Y1cSMU22wLqN$el;4_)VjVAPs$RskCmrW$A
z1rHax1(d_R?J1qchyHB|C8T&JZE{z!*ZBQt*FsO0j-f|^jtF~1g66Hyqd((HGw%Fi
z_19^G9!63+Fkax-ESk+yr_xlU?tgSkb872a4m9lEosfy<)9%lb{yEqBXHaNBPWf%}
zLM(WIA8-v<J9|B!_s&erVda}73CY;iTrn$9?zzcIzhHQyiDWFJ%K&@HkgJJsfW?ci
z*2&CbUT+&~?mVx$qp|6~=gM%OqkzVD&ACgXN3YA5!r$>|QN3cz`!RjLcZw(-@}@i;
zFLkxD?_S+4w_n9mZ{BKgq1SJeY4ne|EhD?uUFqV*{z%xo(H!FI+1I(hJ?Fni4qRR&
zpv9Tgi$id$MGpT~BGsMgQm4Ijw;l|h{Y82?!=?+Pk@ci<25~-J0q30g?p|A&TA0IL
z8S>&)DQ_CdP35^;&&Bx9<3o;tigHw|1CAJlyRSB>X=og`qjKs7TF{zIBm7AgLxf%{
zo=$&>b9f^BLVW~Q%EyZtcjan292u&of383w!+!U`r-!1;rMQHwo>+>~IS<~%)Arzh
zMuz{m0`SS-AlZwGwhBh-QV5WD@KLro?B+>_lyQ+h-(IVq^KdP@5Di&YVSt)d6UlUe
zFz*x9CcLqN27xDrO$6UFusfeV2w4xY{8uUx5=S|%TT4qBXmO`{Dphr{4sE<PvrwCf
zH$$~nvxC{}Zo&L%=))~TaaoOEa$s(aXv+>uoXg)?Qs>_NL5b0Hbm3&z@Sq-JV2J%4
zt6V-UPY3D$?;bZG_xSenLSDGX>c~ANvp0S#UCGI5RKvCY1M7+foS>lw4i)$kk7Zxr
zcXi_iayCz8l=sTJH5Jv+4e7mbTr~bDsIv1fME)t}*~!|$_<5!XZIOu!3vA|fg-S?}
zPpP&2mD0)M(PX(;rE=r1jU443&_=XWA26u9RUWT(?~q;2E-<<G6xcZ1SZ>7Q?tjI3
zZ#B5LkIor5z`ZR-?yYkVx0?0moFum{a-#>_>yfzWd4$8YIkNPUM+^&BJxMK;$ymLU
z>&YyxGlcb~7X}ne^sbHv7Gxhh`4`Uke)q#Mz%$t>(%Z9>A#Wex+s^0RN)Nk9NL)wF
z@(W`IQKJkB1~J?ZY{PUeHK5tPGy8#ZxUz!DP^awT&aUB!JF3Z{C}o{=KJ7*$>7U};
zKW{H3`PZGUn<&0|5%3*_6fjYpUkc8Km{wR?SEX!yO}Uhhy7;EE;|fU<8ADKS`S$=i
zc;HrP6wRl9rc7b%vTYRv)c=d$!o7Ir;3#xLBtpEbMg?_zCH4B!bhgOrl;TJ>EBRYI
zr;m~PjgYy!AK(6yq~$(-_IYmp1Irg<Rx)Z`?&f66Yu(33DC41<Yagv00l#_=hSvOs
zs{j5Lgu+nAuJMQ(p_872TOsb~DTK4$xu+2jYhx$dkibj#67o3Imr_>2D%~u#Z#w6d
ze$vkx&X7_))6?16DHu6qSYj(CIk-9Zyui#(hMVcvi#EHdJ1*GL!5nf{Opx+sM*T_U
zF+SDnQ#8eG8i0sb;8De(FMFu^Nb3Y$-x^}?-?rZWO1bCc_G4E$ElzE%+jHQ2B^_b<
zb+d1H0E-2rc7E$?{|*sO57vV#mM-&GK0mU&0YQX32@Qq+li*(Ut-W5S;gDV?(L(<Q
zc5bJeA;G~RT^D0XQC?9dUOybJl{w4fPC>?+IAyfyZNpa|@26m4Kvq;oH?OFr#!NHH
zfA9m<;T}B$`U-XHF8bUlZ5iZ<GH?`H>sv#qweyp29;3D}Q6W{+@bL17Ums5>1XJ@w
zD3_G~4#WH~bG+isGFihtmsHqQGcV-1+Jd%tU*jhrFz?O~{{P=gxq^ng6sxPG2%Hf&
z^ql4UB-e>+G<mZheKbW}Huq9F%X3b#pR|Lj94K7z_Hlzk0i0pUOK+Gb3mxK0)#~GH
zg?TZK_#0bj0)*_s9~d>EuY`>X^J3%MRQEf)u^V}^-EGXrGuhRd-VC?iHf~*6E1XL#
z7ULgpk0Nnb?#^g<t7jTNb%s$%uE-6>ox|-T#u)YE!Cft!4~T&yfS~U}iF$KNPLI`+
zre~{8D2R?6ex$qkrMzE@G4JGxwvl7HC_eT0C928y?{`@Kr~4#)!5@V1_NFHz3{KP1
zTZ~VwB8P9JA#30@tozuqc~4=_s@9XSR_hg;<Tv3B7@@BkUT65E_rkuXXIzlH`wn|4
z1Z_(!L($j}N*DAd4T&c7iAa3=wr2rpIZ59G3QTj#8U*}>`Mm8L+~S6=pB_F_%=x2G
zt&6T2&a0buyUqx(m1th8O9nZ|uNy`lG>@W3H{A^2(JOv7wyM=&R#0PSVqxbEl?uE@
zH1WiXs3w=^A6wJ^@4!YK`fOArP2c*fCXi^J;+WJQMGmu<fk2tgh}J_hcg449Gv>kW
zcW_<V!Q`6|P)TocWgk&cq;fJqP>8owqOD6!nA2IVczIb7!={1|*(lCWdgq+8{dgx<
zs_`9+Ge5Em@0{*X!Nox#Mh)zzH{p~!^1UFSm#RNq?lJH1WtNAi#v_<W-T%Bz<3CV2
zHQLs3$Joq^9;^GiBz&XpS)qbBV*~Fd&yK}gQGevG5ZA7IyBwn{Bh@#A1cX0B-M{}j
zqWNJZNihZ|;0zLwFepEW91dtTdj*JnCe6uChJFOYf?<^!gt_vmg{N8BdOr5HE^^ni
zzRP01uvd3@G(_xq1C$D{)eM?46ZLA|MmURb*Di8X5`Vq9d(JxBQqJjB8FAq71nx+8
ztz8MwycbVzU)L?qG=W=Tz7{tl2IF%syWzZizfhmnC|Q?GaGtE^i1TgB*a5xQPBLO7
z03WM`^7j6B(~5bqsZ{A8qYWpT!mWx-@|x&edzZ#~{)$xXU~>vpJflZDe{*L0TZ{}+
zLXpM!dd+FMAjSHtJ|KK+6yofg^Sa)4ZMqQ6^<G%FJv8Hd&ifr}o2}So#Pv&s|G_=;
zzGb2_s0}oU_0b_Kc4DGusKTbO<_&S1icEdPI$Qhy=>=fJmaXS%6obFGUhHxZxy05r
zg|RcwbCj8dHF#qfky`}up+u?O+*rgRYr{L<4<)YH^|`ZWswUI+*R6g1>kD0OaZ8TH
zUZc%-M^lQq8Wry5CO`Q{Zz>qSxm_+8jbTK}-19((YfsmxMAJ^vs_48(e(JFPrST67
zwY5iy>#l)1b+7kiI^Ef${?sS@`)&XF>1Fz}-m~S7?(+l>Zrg*MCJRkaG`+$QOMiBV
zo=RW$vjCOR?o94lOrCfHxzA)4FPoMJ%L@b2>NuN2r;aJYB87Etqf-|MnaR|IUA>;k
z#9W{}u&&Z>bme>Z*8uzxmO1`<Qid@kD?Sn|P^qVQ#mt%pzvU4@7yXOZb?c}m*;;jb
zlk`rOk`<;ubIrcInJF7AunP%O+MS-RK2JX_K!yA1pE<%Gjj1*xF4p;x{ghZ8&5=Xj
zOZU+fM~%(LSdi7pM;*K2tGlVfSBD9lBgs4e|M!n;btdh+<@#Y4<99<+2zw#I<cwMj
zD=E<waXx<B#A8O9>`mF9lFcT_1J&G`NS4>zTs{hjm5{AudC<f|t@BtXv9eOTn9_@=
zc3rv6Dm`1V2Y1XUy?iPoH~CBgrp<v5CVzp)&Knp{x~!%PG|k2q-AYXCWB+v<i*CK}
zjd`;fh2r;4*Ne>xiwHO_@X|p4GJ?={iPh&y%YMHZ2?{2U_-WL-M1U-~*9S2@YoLO*
zHdBJzj+%F1^Xk;y#HVX)QJdP2x{@YbtzK2L)zKRYx}5i7xA!kiaH!0)xUR-GIRzT>
zx{$WNdrAFy8SZI*SU97}CusYci}*Ye9Mo?H1>X#roWL7dR~}#y=B$+uP#IrWhVNH9
zg|W_dao2I*X<uLe@`%yQ`$smY?<nP!jlAvONb<(x20#|`;^whn)Yvqb5Rwx$61kB!
zmJR4gtx0p|oP%wh(#We;nrU+MPbD8Z+<NtQcss6__sf5qatQn46@{-bUfnrbIx>?G
zINP#zVtDv&`i|R<yT7LOp?$=)Yy=91h<pWNJ_vLeyR$91niUFQ9Fjmv*P(U}zudvf
z^8#P<PZJ8JUXs^Y)B84WdJ@_rkz$7dQ!8$DX%vU7bN44e?>y?w{~h9yD07L>()lmK
zPMmSdfW|X$Q#`K{bop`Zt>ow;Ur?9_8{dJi-@eU2$3?`lsdqJ}c?xU)>L^LS{hd4L
zE1!NCJT*lBiON?`Vj<pyJ{|dp%tGjX)?Ap&C?CbO<OCo4wtU%M%fP&G041V^O}=dx
zP^_5pn-XoK_9X9i6%Caq$zF?9poOwul%_Z$JdU!{V^q7)X3S%*{9xpc+U+dL`JW5O
zJEX4s05&Z&zNFSKouVOk9T{`39f}2XzO9pGmt3fn#-G`?7M?UyP`l(D&f~?-lxUXU
zirkVL|HoT`E3R<nC^5^&p6ywvnXA}+5#A@CNiFQ+NR!(!S5(7o+YC%<hz5*ES6f(w
zd5soUuP4eVn9GuQMqcic<WjLC#h!P}Tx2WWd+QeN!1gKm=#6C{V|j9Yyw3{1+dHjG
zRH?c|7f{T|^E<n2D@-2!(Di38&J)NDMv9Q9UyQR;UshlH;d>!!$JV{eEpX&%%;AQJ
zR+qaiyq_*q^V+>9h$BFIM}L1m((m;yrKlOv1dbs_$19TTgY3BC%!1s=d9umyWWSk@
zgIUjaig=)(OZ|ANA9*IW|JP?CM81roqT8hSs7X<@DYG<m1$)}DfoB4fPvzzPL8s@u
zDnm_}oAWyvMvt!3^WOW*^EOr=Q9>XyGm9`sseAxrM)E+q;^ZZMvmFE0Vm++Y$WgTe
zmJt)-lUUuzPQP&q+$t22fR3OOmphKnKA&8eIJ5|FE6>n$S5;i9qrdfKif*PkDq^l2
zj(4ZA{#fv+)&U}sh^G57X4Bb%K^LoQuB4`Rb9i{Tee_Jts5J6Gx|=?GUU$wB+P^66
zK(@5Z`^8|OeJ0m!nE@$FN8Bfn=b`_9e;($j3UA~d|AcB<s7cW*?STOqYvfeMhu$u8
zKn*#&W@{L`DXERvXU~d~8=*lVbrLZw!dr_Q@Bl#GS2k~l>MtrcnlRU|9<{qasp4Fo
zT2Uk!{!S+0($2Nsj794HIQ4KUshWT+J+)GsKtTOw;Dk2QXxY{G*$rIdo0_%D{L~-q
z0-d2VsWX55FZ*#KhD$;pz%382ZW0}E>+XoC-t^ot|NV~IR{s3`1($y7g_qxhd_k|T
zpP6oJ3z1XM@&6h*5w54pm}~FFjvsepbxlaX1D5;Gd4?<Ey8`qK$Sf?IFt_Kk&Ur;b
zK~sumY_jOzB|DYxPv_l>JHo=Sc$I>TAvXE$wYW0+jmVWJLruL+i)o(Zw5XIcw_T;*
zYTA^o&-VFcq@-&DtksN<Ql{uv&KU}JP;{F-n2f&I!_;|%{R0@qndk4rz;J`hMJN&R
zNP8|?-45!DG6sV2^=pofK4SutHNtV{x&oI<lov(XT49gID*FX{wzPx&ot5u<*97Z{
z55o2T?y34Ga3ueZ;HU3<NgAKJx&BeQT_O;d+VzIu=1)|;j_Q~V2%w>bMSw7F<)Os=
zgP+2DvpjO!WJQckM`yoW5@5843{8QlM526m>v;hFd+a1qIdov~92sVR3UdQ=zR`4H
z<*Hj7ZeZT`0M2qnZDwzK;c9e@u@$+g!kG<?H=M6;OOtT+wLDjB)o*FY3=#|@na}5U
zNo$GS#XOs1+{aP9tI{ox11f26#N_zrt>hXyP?Zw|=1t8Id7yr;%i=*jO2$sZVG6;X
zeDQXXRuxZU@~rIc_Db@bIhR9JljVmYPX8M%BU#}k78n1s*K@w+m+nvf^(&)%Di9J&
z71zF-`9P)#@5A~FOWR=$gZraxPS~qg&3M)p2J=l{xVKpb76>rJbjgx>Xx!eSzMP)@
zL#h2{8C;mpq#88HIl*f)Lf<e<Y*GYXY6p0!Mt57S@Pf(8N@}k+Q;PgYg{Bo>D}SJx
z-10dMkmff+J16oAqNBH+=YL37>=p10nzfeNhv*&TTCaSRM7`w+=6CYvaITp$wtoNo
z+Mqp}z1AQt3VdLi!HaZ3;|Rt9TSMgp7BuW*x~Gu+#CI&e3W4=Vo&J+2(M_6QBVG;e
zMunh_Eo;{rT&O*dO{~BAa92xSo?P%~D>PfDPFve~fyU|E3&NCAj-ThJ`VBhycTH`x
z!TY9dtc+uzudf+Q;7D?IbsRw(^@d!_@Jt!Z9FLCeK09+(AEe$Pph4O@K3JOpQ?pQ*
z8*KT)T(?|z@azlNhzG$(<NnHHrsRuk++9*sR5XD3=sS-;p(75A`k;#8=v;y^#31D?
zm8e0^D-2D~*=;_9SqwS2J+YJ_o*iv*yo0fbZRs?apM`?4juc3>-d9&^a+nU0L+?0~
z52S~)k<ZCmZ>lP)s@{Nc|4PJ=yr<}?TuXCv5Ok_l5Gj|_O!z~6SPEKi-@cD}$?<SS
zc-X)pxi9kz+d}qbr==HNt(}*$tvpSiD<daK<kkQEB$4CFle032T4O1s#Bm1o%F8s7
zd|yB3@c&qQ3#cmFtzB3UDFqQhC8QAn5v3cHMndThX^@m|1S#nhBqgOA3Bf=br5jOc
zK{}-W`GCIrefPh=bIv&XjIjm-Wv%txG4EOTyspWocQu`f`Qy1I)o)i>7bnlIURek}
zFv+2#W~H6eQOD(?Acklr7nM*<l(Zl*UFmTfo0I-it2&Eha9z5(4OzSNeQoR9`3N0}
z88)KLKawpw5Tj$SvW`9ApK#(IdL78kvK5pVaPBDNqY}$xmhFy>#mf0#qZoKnL&m+i
z9xQ8L9T#8R4)aQX3qbKHfF`XJq(B9XCaeo0QNT0O3WbSoc`2TU&aA25%rS5EzHxPo
z=YE`p01Z$EIM@)tUQ_5*#?dM(E0c39gTOU~5J+hW)7G5DG0+6CsF7ETMI0?HZR88{
z{U`lF^!)t%CIQ*N)oKe>hi!l^)(k6u_z}ArAikm&V_DD&yX~AW*Jk&vHYjXlBA)7|
zi1u`9sn=3gj#X58dWovoYOu!{r+j0+w|Q(II5A^|ak)hR^r+{-%=J@2Sm+|1Y&7@d
z87}+Vqo+@5Y^wo#Oj`8V-oC<w>meZjD4#lNV!Um8e);vGw<foXLF@vT@s*g_<ht7T
z6=4;TfQq<4Qgp03p8Yo)Ztp$jh9o<<#Tgf_=n%J<V$M2^*p_!{8Y_*hZkW!^Vr>~1
z+&88BaXIQVRhKjhPQM~?+Ev#4N%EfCr+JcZh0!I#y-su-o3CAdmuxy-7}4DAEpuOo
zNQ)W+DPHbr=HUyxKNEhqsQr@2<l`yOthB=F+|D(vW_g&uhv7Ns&Zqi`bn9+oQ)lSK
zd%8f07Xyv1TO8$y`MTR^-!T-JoFT+9QSAkE4Fkd^dFe$R0;zjRs9Anj;(9L>Ofl7j
z_%S_^fj-7jfU=h=50bsEko17#5%4Y8Ld86<!=O;0GP7wX!3|YbAc$}Vf`d$edr^|l
zK-m<#50@;7FMd8Z@RdzLW!YsM9w->a9Wb#^hEBlo!n?pNEs{4dSAoMZzn$xs2{|OT
z9e_O&e*(JQslGQ4YSCMuZT7yH#SNO|x6Xr8Eov$?{-!Ceb@G-CJ=gDCc`s6Y4K3rk
zgAEcK-Tl8e@u$3fFG$=LDCqr!`P0OZ6ldH73THUOA?vPxc6DeMwD!dxtB>?8ii(tn
z=#si*-@Bz)Qw1{Wy5Wv%5FweYoXYe&i7YxFNiDYh>=r4>mfeI^n{(>*{XF>gx`TN8
zyBPzz^3M`T1!Su6+RbUoD>8@4FvlJ7DPHLNK9Id;xbk3$eCudtQ;)@?PXSjvoya>2
zRmWq;FujcygWRpS4$1&!cwIDr2LxjWY9~c~sCftp2?LdbU*%wGw%*ArHM!qJ=GJZf
zOqq5E3PNdESrve5-E$r~&u?QUU&h%tsU3}ui=&%xTMw}VwAzCzAZcg=jmWT@yZ00L
zsM0qpDk^N>qF@A|bD_!rm%&<6N(E9nSz^>V2qc)*=;1EgOFvah#Bll>1r^H-+b?x5
zq+N571zm1pC#}T=)GAjQ%co{%+mUMGGiT1!{}c-&`}txph&?ki)5xTCaPC%b>^tHk
z%KpB-0PQHS4XX{fBPt)*#mSsA5GF5Ag~!8W`Q7N_DVqm*6VITejQtJ!{b};Uo=t#R
zM<Q$uU+Yb!&v{Qa8MHRmj^`;dxmKtja%6F3Tm(yxuh7TCJw8}<o+5b}2pUUtBxnA?
zs`5{72jWhun_o9h-ApF05hV>deVnjIeo$g&WO4X9<QIn&Jq+c1uCF|>itfPojWj(n
zRLQj2h5fxiJ%`mVIwmbaBP5Sea^t?r!sW_U<<{GbbI;esFq_ZKvEgi++{#Wv4Ko(R
zp-dil>QLxOp%O|quYL7eVxeGBQ4B_QbbI8}Pm<~WaRd@6D9N{cdP@Y|4+*L_G-umz
zWJkM`>rer?HZiSpv>SK+E3n_PG1mMd-ZuhF3!VDM4+&&0bCg7a9(5H%Q=3r77mko2
z4ITnx>+^5n2n*(57OQ%Qs4yilLrE4GTD4068fW5g?zdU^RUj#hrygj~7ES+heITRX
z-mXJEq&Wd${za-43qGb6ft628w0@6O?*WC&A)((iF1!y`UMAvo$S=x9=P7)%r49v>
zF6qUefU|)P)JcvBS1a_BW~_zcaSfTmqAjUg#2g6mxqH78n`%bChH}2Do-#(y*EYZc
zP$MLTh-nFZM=^N1(mK;Uao{H436P3aTcCk`_GzIHppWZ&Q$T!F0x-^^y0svL96n?u
z|I87v`YmkxyBtlDAB9Kar@Otloq0ua4IC0_q|Za^C(9hwEzl`#K!0Jm^-VC4{?^=A
zyK2`5!W2whYK6OXVxM>)x~3}0j6eo~(zF*H@sIC(T;~U(-z~;gzLR|9|By!zYCMxj
z_V(kta8*FM&0m-3P@E;-E3$^2&%mqv@hV)mD0SQx=1w+b?EtUe>N*`Ouc&Q>R9ICQ
z;Z`fl$mn!tz|spO*`$$LDdxjTW%$J_l+IT_FMra!>ZxlTWwKz>3)+J%`|~9t-xpOp
z6sfnz?_W#tyx(iEy)J0UtmSgWR3vWxgY#@lOq}%N`F9Lx#AodgAjGaz?lr9tNgoox
zNIfE0DYjY#sKqmMrTqEg54cfCCvK=59|1yQwQAeUBAL@Pz=-0nM%g%M@}k@86O?1R
zGZCY8n^AgaL9HkFY5P6b1Jct{zyssD?>G$fq|>0kx0PtR8R&>rcK|$bdTZP2oJz5n
z50iRKlK>70hjbr%S4NC^UzfJzG*oC`!kP7DIfI=)S1kv4^mFUJ#9dg^ss;0tg0eXE
zq_!=Tq5V+^lnjC1kX*GzU=D;CVj)d;Q*}F2j`odrK6@TcPV{RI+>h|9%a_dSKZXlW
zv7?+pO31jE32Tyvn{e!H+v-O(_f#RwRmCq^{b8pwAyDH~-z-<$<U#wtOO(F;lPI-e
z!~H9NMpG>0A|{-9mC-ILiV=>8Z6&&WFo&FKv)T4W_uEh<JvxoNpQ?J~ll{MZ=7GE>
zd<xa-MqwloXH0vTc}S$dm-(mtS80m64lo+`9)F6<qhnJfFR7d+MB()Q4Qr9miAxjq
zZv&@zL8H<Lu=q42oR-lWc5-t_M+MHjY?J%cU2H}jSGvu}2QmonvXy^jGbOx&lVpO~
zlwkY#ab|vpM9Yqh55{>*f|_4IjFI7abXcgQyi~tuS;i7tk*7;&lkc%hT&NqOEC4+S
zLahdCCN9eHbyty^Gzp5l^v1*S^JUq=P_`o}Seqo5!ZOr}SVXo(Rm=L_nNa_}WkMY7
zdG@Mn1rO5+J@)C2j;Ux3fJo<X+^t=`dKPf$(GX`mw*bWsHP^EmpQXQJEe<g$CGV8K
zSqO|F=x<;7qM`Nwa#tw2sJw%vsmt{hKbn%(-;NDAh2)}zxgnWLICgsva`Fdt@5sza
z3f@mLNp!umDY<}(DWZ}J>4fA1$4ovB#4##}Q}LXmh*w*Bm*7ZVty;J<8a;;_o&NF7
zTfhsGC<#>qn0X3Ljg_so`UN9EX4E?zto#B|U)I=4>D~lFoHcfGyRJKH&?Vzn?CDPF
zBpxL|x6f%zD?Xh9@?A56mk3+L7;q@n|3vKsx+P&YHUk;p6?^$;^pg6SAnP;c`=6Hs
zs6)du@5QMLWB<y%6ZZZ4!|q#wvH%ne2c88wU7aJbqqjlIbUk0?4Gl9r2RXF1$g`EV
zc7TLWvw0(~VPBNY=_`s(3efpUUXx2+1PR*rVJD{2K-xm_e9c2|2k`lKjqLsq4y!MV
z8b;Sj(AdgzzbRF{f})1dAHZMPd0lxBv6Eq(|COD@Y)lk#2vz=cn_=oo8cEs9(4zOD
zRh^ZKuU@BnlgU*;1qN~*R>rLLiRCUYq=>8AJH7BlQk^v3@Og+=YjRa<by<&Tl--Pz
z9Q+baVf@Kd<epHTjR`Gr0d9Jf@<JD0kSA>df;SBn*fp6bW=mZKzhXD`z!>-_)DRLZ
zw@%<I%6l~a<M7=E0KVh^DmF7JR;2?lOsnC9)$=*~K$8}YP%o%o-5P|o-_R5bzVeMk
z2ckq7R~?@Z!Te70ii$KpN2?s?hF=fYClw+HrD>oaH65czk6oP1fAuQD;pd1X#0(29
zxV($EoPj{o-JQ&H-=_O*%+VBi{xFaW8vs4j;W#nV=OkVggfm5FblGE@cLz+6TYCph
zte5gj%{2w_!KmE4i#s1=H4tS5<TH|uO?kcU?t63pMRo=Zm5@9Z<f5q!CLs-{3<-t_
zs9JOH>^9>iUV>Ax>%_3+G{X1os@xDAbySzEygRi`X#qYI^fA<Dd~pKst8EjJ@vn2Z
zz)9l_71ZwSNj&x}rzz&==Z!`|7cqm5*(x~CmL8*exZ*E}rcP)(N*kw11&+T?)FGID
zlXN`Mqr(G3h@NG3pXO0&z9V*}jgE}OZ~grFylH^CmN6id>g!*LVeuS&jk3zLXd+YX
zo`D8PG6?W#;Q~q7mQA7d8+9PLylAHj6ez61y2{PY>octgg|+BJ9>1!Bx;g_f%~_7l
zL~ey+^r~2KlXyfpE9A!4N_S<E<ad$J?Ti1Hw*Y^fLQEeo<@Z<wBXHlNpU?S$2U1l7
zIv@C(phs3O-uG2BSGyGA5xu(<5ollBryf?%SSjGoM{~tp$3g^FLSF_1Cu|>JH5Hc(
z0l>!syzT(G{rYk6-E%DQVzK5_{nZpWrbr!W=L^m~$FrlhG|<KgMf5aDnluDnaSzhP
zc4{hVRMTU00ka~aqi<1)VGfn|kBNX)wTseWuIo<aBhwrTe5qn1Ao2mM#TB~=H1j9;
zw2J<9?+Kjs>@fx(=jP_erPtJ|jFmbFeS~@{4sGOyMEvr9xi$edD3j+8IUx-QcZT7i
z*$+djN<)b{N>F)r%{E%g?(!uY>UdgBJW8jM6GKl+y3(+Z#a*{O8w|E@U*~Qz2_2yt
z3Xa*Mq2fE`*a5P{yRfzXef|CM?Zh?RD<QIw+wZ9`SJ{E%-C>x01T+OTDIp}BJofgF
z5blj>1fLtVdH60tLWLj@x{>hr-Hx7z(-jNo#$9C1E}GF^nYi<k3(~zoL!`NZZ$roB
zr{m>DIZdlLqmOM7B>t|gvMeLc!Qp@99KeCYKbw&iSk8*dl^n#KoXb5>N7&IF9;zhS
ztPs;&`^e8N*rGFtH<=?ZMR`J!t~{!7LTxGg@LS#EskVDr)s7FDP;pTa+D1*F{Hw3<
zzv}QDp~mk4W^Go2sN`hJLZnGRqFs+<y@xF8)-72u0#h?HQWh2k3fo2%f^ZvR3)tel
zN+K4606;-?RFN*lgKh&JLMCX0BHZ!qm1wQQ&kO+A&slGd;quH;OgIcwU{<E_7q0;W
zKpU{}`dIW_TlhJ-4^(v-zzuSzxdZ?g?GQHnToC_AOn!#7lVEO`S7xSeR=d??(vOGR
zaKeJ}XM7Ml<!#Y|C`4(U`tOt$B$q`*DEL1pn>Oc1PO4Qt;T`@W6iSy5H?2_S!j_uu
zP43<w9$FDH%qdR8lqAJ_q1xoZ$b9x^sF+UXrIr$(ycKxu7GO>gB%oWiJ(5BKs4v50
zQ%ICrnbFQjg>4&j^O)3dNa(h((=h)5To5Mx(bm#PRtQ)UcOBTly8=#(LM!eu)^ETk
zzuBk20&(ob3ZS3{uakb)_{N#u@6F?|Xi{0IULgfZ^o2p{m6!a*_^rA!&<#&C54}LO
z<Sv^?|5&Sl?06-#eE?H3RrBHkwO9bStMjnbu}Gd&edEo@2WiNQ#l63~ed?(t7w}Xg
zftwY1L}9=bFCMD})$b6+d!a<dU7LFLYRUNj!~D~+TZ`w2N)%t&6-v~#ie?S<EajcQ
z%m~`x&Xc>kvW;iJWnxS0D3PF>K|nf!@YfdPAT0Y7BKR>~<xISR{YV7p{SA^~ZAm(d
z8-e@uI#7U@4_DeRD9-~;lEk+_z+s6Y;a5r=x>R|t1Xq@VEO%yiJ!Io3!TJJ`7J|l%
z2hV6YyQW6{X3ZQGY*NSq$AA>XbIJ+y%*<8F{`<Ll6gX$pqLjx={A2SS_VWC!fNFN$
z9J}ISxTWs-d=V<pjfTl?Ym(giD<3*n2`^u6UmPy$6jIjGO0w#BXS6ukSKmzLnqLPE
z2n%2AX#Ci80~*YPM^Ht9g@xmR`k~pNmEDn~q2;x7yLLfrLkPXYe0OmWa+h+W<Jcc!
z`tST9BzeH$-Eo6jSKrODB8jN3+dA|@%kA8m=S5q1q21z)^K6PxgrL@py``#`as%gQ
z#t3e7W7&cL*$rb{SIWRMT$=;fCMyPD3+u;C?1>A|5PeX|4@Z6t{JxJ>Uo4Ek2WRm-
zbkm80^tKd&h3&rotk({h2A}!K<j;eT7z^I!dl0%M$a9yv?buG(Dk;Sv+_`Iz9QZY|
zZwyqUO5DJ*=H7_};s~y@-mr=Y=UJjn6H)z5u=lY@x1}o5AHhq)+lDec+64lWmwLJ>
zLX?e5uHi&WC$P1u+qqX@C{ta??_O!Y-AvkRaE^1_E=oF?uUfKJ!N)yuWEKcGB2VLT
z2?6uXY0I7h4bC0l0-@!sqC95<jbU1#+u4f^I@MlZUvxq5^}^POvO>1Dt!nAd{Mz(Y
zI%+?9()9@6C7?f3ef}89>uLNKdA;BE{UuE*p-p*EC}B^Db^&sU)vX(b{+R7`x<&Gq
z9|QkzQ_cPpg5eU4N3`|`MzuDEo#pG>b1L03dAqGGJ`*;`h07RPIclIEe@_n|Z5A6^
zR7CfRyJHKbe+%!j?-qYnnm+<L4CP&#<DEhfy~MGvUcFkaP-CrKEPZBz`4VV1jiUS*
z*lwint)$4y%OeOj<U(d57nn<A$|F0WXPKMDK(Oa#BIMbJ^P46K#)>P;wApEFfwx!v
z*&4)oZ4bn+P7~v=*;jS~IaUiG`-$UCx@Ml7y@`3Huo|IafakCLywmKu@(u&(LFZ;+
z$-4Dc>a`EMix#KF2*Jt<u?rH*pDsMxXF{G`;Q`kuF9$o1Tn)hx-TI)t6|N{L8Mv`H
ztWf?k@<^n(NypdF(|f^qs;%4e)0&lKIiJ21pwb;h(>b!wJYkn*vFKewjB3JvX;i@=
zG^PnPxc5SpIQGMbQ!hO7qm0RN@ec0o?2$!Matg-I@)VX@T%8^g$V2YJy>(F$G6dxM
zL8iv^R~k@3$c&5=7uKywRYR{p68jyX3ZEiB`sMEiXX9mzkst0=2I3)ZAea&nzBsUf
zK;}vPD-QpV$f&dFaKfDtf(fm*!cj9}e~zS-An=$IJ7CMV{Ya?7X7*U4E9ka5cawlS
zWT{y6o6aHBRnuTK**{3YO}x=9Q*eT}3PWaq+QOwf=;oGW-VQj2?5}ci!ck76BUfdH
zzgE6&S%PzzFGlE2T)=h-Vt)h06*Ek9Gtp!an&HJP{oN1bhhG~4)d<C!C&pSE%);GW
zaZU|4>*-uJ5x8G0wr8r83Xb_izIsNbO9%A+m4ax7<fj<puN<A!KwTLM!jVPPvI+X=
zq&=7SzFQ7^`x#|YQw5iE7iB`%5dw-4gQtiE*Zr?8I0c?R>fRNU*xKYytKJCD1Idt%
zln0deQt~3zt@OgkYBO6ewqyH*nma8Q$jd8|my=(`v&O%|?^2E!TN!2ZD^1!JnUeme
zaMaU8KU^RtF#S~Syl#Xj_i0E!*^MuPMUT`E<!-77J8GM5k86k4y=LgD`-9v}=1&tr
zy^ZEyn(S*f8w?M6BNf(f`Nm9W3NquIj9*kO((26YpCb)!bd@fdyJmX@RS0zDEja@I
z7`wc!qCl(Gp%hvnixGz8<ESf?WR;gIa#)U;C^MFB&2HrqhJX`57ZVeMQhOmEtL1LG
z3I)L64r_CLt{gq9l|8v!?yuP@QHIj_|Gv}&64Afw_JqBC!dk;U`7a#k%4j9E)G{lz
zA_a53+jXYIFH$__wmr>xBx?D#<&L!NrApk2%SS3{%P`sGgg)FNesul1iDorq$xN)%
zs}(#v>Jc6_gw4sfd>)c{50#(q0G*D{)@Qi)1UAyailAribm|p`5S|$+5xKfvUspdc
z0th{eNf*W7lA)nt<ck*zBjUpJ&@bSTykxU!I+9k-*DZ6Ya23rpK{vO{M>4;O_5*4%
ze*Ajx(W)T?e^JiMKGb{wEmB)oo5`8MElV~qG~~8;SZOYr4F9WzkarD*>4s3_FrI_!
z>(Ak%v$YMgaWQ@EE8~?x^>Y3!u;QZhF7Am)+=BOi7`HH8YH4qeqCQ-~z}hm#;<aDA
zvT~)tVi>=8d{9+~`CjmO!RYAdUkT6ut>tilmN&I*TQrIBw5Wv_q2cCQLpFoqp-;@(
zr22M)pXcYtJEhy<=LSpoU0kd8_m0|rD;|VkEzxzUy516KwF(j1g{rZ!cp$fU?3peW
z!ot{XLeb<2WZ*v1DFRy16^&NCB7^(*f(^tkLG6r~Fymn_sBR<!Z-kdoIdE{qdSvml
z9b#@<r012AE%dWI1+G<|dj)q<zXK0Yc4zS4{{S2Z(<A0ZI3^leT1@IY(+Jufl6z8b
zY{KY=G6qds!YSX?u&WoLWC6=LhuK;CJ3h>inYj3-@M?&wvCFt+?N5$spyvw4yBk1{
zHmM5%)x^P)<IMA$z<a-5^W%VbuiU7U7|9np%b1%bpkMgueze_i)C#mG<{YM{#81n=
zyLO;my;QT%VF?@yMxrU-zkk<#Q6hkQ=F@>|Xd+PdEVKx^7pdR8dSQ2yPIa=4Xfp-?
zJ+hxr{@3>~-<7_$Ku#=~-$lgW*KS(4O5BaWmil4(e`e!_ew{gVa*5qPkli-B_>JpF
z5y`@o;CYpnm*R|>P8uucI~Hy?%e6(YGCLlfTB%CuZ+T5!<*y&Gc-lJvli<>&HXv#;
zLBA~@EH6#$(yjXuvPJ6dd<B*1?;?RruKO1W13gj-!J1eKmtD%syj{3#D`|JgaNZ~?
zDaF5w)V&iXwAY<00OXI6K$x)Gbb!!y?Valr92jBa>IXWs2ebE4g$^gr6Xcid=5%`O
z{iM6fV;9cUME)A-B1}L?sFvp}nyQ3){*5w)J4RORlKxiGG|R$F*XcYob4~8_dT1ur
z*UC-t&sqPj!})V1AykKL_oGizjMURj#P4YA`$vRVM$}+p5jrQIXpxgjaEg6Y>Sp71
zootQ*rwfJD2&ivH@IGY`zXmOPf9L1|if2o>soX5axJjSD=u{l}-J(ku?EInX4sOM+
z16NYuSG{5T$g==jQA$3{aKHbQQJk>dzsUd4Pp0_VbyG!Ny!=1}8~c~?`cDPu0gm1h
ziU_HHou|qzkH3#kLd=`>=I7XTx~H6~`UGX+hDFNleAM^XS2fZ+xdtZ$8F@vMk1F2)
z;`vvRS-IPGg<+|#GB_QVkP01D1b>~Ua;>D9P%6CgWu3O{T&)ZP<K4U)*$>goA1>_q
z>B>CPIxluya{pIa|F8SK-%E)%i5iiEGzPrVFn8{6sBBnAOp04eBcw#Ulyq+sW;o|D
zY;+QI43%@#8fq$ccrlAV3X*q5O&r~3eVYL6#9xXc?uAZ#>3T~F8;AD+fW;4geWzLJ
zTGG;_IDKBqk^ckdC6(6A+343>t2&b4QDs=fVGWcQ2#Wu2s<?%{t&dv~k$RlIW$^dK
zgL<^r8!0~x`pyg&ln#|v%O<<OmcJw*PbsQ#_lR*jjnr<4KAvU1zAjbk^A&lTZ`kW-
z{8z3bE2oT!mv4H8>_w%-UxmgNR`stN&$XccG+7WTZt}Zux(p`pmFgDSKc4vS+wA=d
zAJh1_HQ-={arqzNJeE~2TI<aXP3~_UH_S`aip=W^W3tI}!s9M0O4%!wao|)x_Shp(
z+n1q}lq`5WZj^jGbkFkoW5m6N`ngL0xoLc_BvE)<(@lYwd-Lu#o&}Vl2hUZ>1XZy!
zQ`{@QR#_hh?-|(CW&6id|LvHZLbaj73q^Xn1V%mUy!8+D0|*69zUOq}2lT9OiD$FU
z$11l28g2@zN0WG;+H}5@+ls&)AFN&~0a!;cX}OHlvlrXmx`Roj2uW3d*IQ^1iDXS?
zlH+b_F(5STXrb?KURc4Ln#pc?Lim%misEC8$i!$7Pv%gCTq-C))K}$N&@ryEY_{cp
zGv*t=$2@6u$`4BF|MhN3X6yDvJ|dS#E4niFN^c&-XGzVF#OMtW$C$n3=KnRNI`6pg
zJ=MSNsEH-sRY&x=eil&(6l!Q<I*Ck|$A@mt&C3cDiNp#F?yo(DykU=61@NKf*gn4X
zzussOd=nXDqiK2^QD^@_V$^!wx|wySbYhv6RL*_W%D!|ZLdtpTo7C{Gd6IsEMB!`a
zAA$xeM69&b*<+ixG&lxZ=MJ7_ybR7|FT9_B0u06WUUZ;madb|z8@t`V>9+ai@bl>^
zN&EH3?01WsVmN-lrJhJq;_1H;-~ET<OcNwze%#3ZLH$`SXqx&jO(dc0jd7?#fshy2
z?k*_zxAn3zOnr1#l2;n8yhC`<wR97so{4{ZC$*L9fP^eT{btz2H?p^@6Mn}4C#dp&
zpu6ZxO5|uE+oka0u}XEvzH$2(UaSGM=@2t0%6~9Dqx-Kd`cJ<FUh9fPgix}48TAv!
z-!*jPgRkA3XERkA*d|_i?kiVxylJ@m#j`;D$0&!~BuSKq31^wsD^jNs;m8E#1@rT4
z3hJ87h~vrhVkE0ZHuZM*34&Ot{0tCP#L&eON&(PtAuTWOhhd)pljj5tSYML=GwX{l
znM$i<o}(NkF@chJ_HROmNt5@sE)3hyA0YB0p1B5xrwL1U?`?mT(%@1fANlDz#0YeF
zpM7w@O=mMsr(BLf|0({PK?h&y>n?`v^jw769XYGjM5$`~yX_OKz#1lmQ3LJ+Dtqsx
zJ3uFLq0w;gKP}t8{;!qNy6=Lx1oXYS%cGXt@7pMH42k}0B`IdkAo2~E(Z~spz;SHm
z0e^?G^=DIF-C+_^W-BGyw@X!H8y;L;KRgGh4MYW6pZx=7w5ww8PiC+ZEINN!|D&9B
zfuE<#<q5mDBGo8xBbL@LUj6GS|E~RzKLGd{n9iYdpyUe9JNkc0=xr+9qb|#3;aG{P
zL&hhei*e*XrWPkIOPboachw);Sor({G~0EBsgf$}%=Kk?pX8o0aAmFzMxXgPJMiw1
z?B@h{$qf=yU@AnuJg5G5<Mq#DK;tcR{dfn_CNLAJ{=S10c*3Y#=YJZ_M7N^oA2OK|
zDcuqOwbmQ6_W7BV&tLefKq~8l)77>cE%~Y03P2U3k9#As;Dv^Ef!O(no?%;p*WVF)
zMzk<P*R8tS;{Q5c|LHdgRPX!8hY_p(0+;CD52Fx;AKJcP`KbF>+j;h>&Tug&gv)Aw
zu`PVJ=MT}qr7O2N?iD7)lJwAzaaK#ujO$#G*Qt_4V9NE7!YA&IhxRL1jYB~XVcqz;
z)fD?SyYtlvJ`)%*_a|8knaO?rX;vSqzb!Ds={I}0+jAR;6aF@43d%G4f}_!q>U4V{
ziA+Cv_AhO?<uNQw7tvj6ZwoVfvX%K8p0>)+<#~jF7wQd+GST>-R_F}2Rr_C-w7Bwx
z_+TRaC&p}<j6=v=2dqyht2p!+M!RI~s(<fm?@Zsx^8(CK4p)f=<HFt{pq<;)T{r2(
zN2X_j>Wz$z??FN1PE14uWqgxLjYnqoJs0(a3?iLZy9o&luPTe8qX}*|ly^QKY&9#B
zRg8=O`vi`~GSvbNp_j%OaGr?kR@YH6zBxFoZC^j|D>ttoMtw3*tijAIqo!tLTL}OI
zyGq&*D8Y4?S64Ty-^Fb_cOol#1ehDcSx=HYBHzd*8jpopI;A}3nTqYSUyIWl1u6@k
z?3?8$a3jFLYq?~!`7mFW@V^RD>$57yUws1Q*OTbKB7Sy*%mU}hXAU*|ZdY2>i~Q!H
zP$7lIfNWK2c8BTjt=dMmYKfVW4iRG!C7}<ud6!<NFTVbO+XIz9ra%ovFg5!^FFps?
z*XQITf0K8i_t++w#;!Ick47tHDjy9~z3@Cxf~)RFezy2q8QiyzPk9a|mwTn@;5#d+
zus3o3Y<~86X45?vE88n4o?zJA9idG6y)h`>TS@X$Up}7cDRWepNOxXK#*Vj9P?K;M
zo}U6@h|#s-tovvmM*VeVocM=H0GyNNHA<p^Z)@FlqN5t0o;vy!{H(o^0)dWl5Oy)J
zA$QV90*!|xjktj2!ylyEzjPX=h=New#3PpX><AGsBn_qFAB_Lu+L*zHfzz`WDA(X5
z-F&nAbW88Ml1^WXAsyj8$qU~SX0%W59`8hBvJm)D)$7A<^JOY75<YvDh6Rb9b`gpy
zPy6^YWD#PB+bI^<fw%IL0L2^=l|<<!#De|$O$lu}#_S0LjSwH^O4w(o*HXa<=S4uh
zm!Gi%#}hVpm+e^Sbt)+(CAER^?!rG@rxO)=3UzaHa}5ddbNllQfi$JN3+Fd)c^c+x
z_saQq;@8^qRnBH;X`5_+VAp&+Es_47IL7>LS~dcd^b!ijHpxsdJ$v_je>6n?ztSt)
z{g%x>xQJOUce_>6=l4<m4AzyAkUr)vNIWz4Whk-g;U7kk@=RFqNB7*1!RfFHmX?K!
z!$m0@9H{KWYHB5%LVu@!`r|?kp)oKp8dh8=R@>=UL4;r9!*6POnKNSoqZyxN!no)a
zEE8h}?S%bf5n=hE@XT-${cbX@2;0ee7YoIay|UfgcbgyPEf2DAVTfvuY3UKwyP-6>
ziM2&`^+PZgH;`{*IjeEg!)j(Ba>Ca9uH^TDrLY5gm!{Eu5tdNG!28J)RHE|jFFodi
zTCZfIS}rF%{`W|V;?9}!pw5H}bHk6>DQ!@ab#dbc_c8T`_W(95a;u9akB59{qO-GX
zi~k4&0oRfrQ0cg9UR>j_J9e&wQVY)PE~ng7FrJl!vnov1hT!pp721WDCK5x_>clyX
zJvS_pfM{u8r-U8$W#wow3B&7R|5nrIn;PDk=~O+Ucp`V`gL(*x9q->5s{Q;WxkaJd
z_+$2UxxNOu6|A9RpQltkia}38UV8l!%5aP|G(Vhuv1mXS6(2orz}e~{9v44Wz5OX^
zLYk#DX+4R$u4I*p0tFv=dH<q$>uTqgd^t1-O~MQ9ySEUa7c%e7e(F(X?aRo>X#5b)
zg?{|bllTAa=XFjL%ty**=r3sR#kSPf4RN`}xLQAwK&25vrMW+i_TJet=+&D4k%Go<
zQp|-dr?YGPEI*d_4BCA6i${y8u)<FvFGWJ{vYYME8stUvMDgDQ$&+8HnTcuEs%+4y
zNZ&|wwUc(}kiCj^Uv)Cz=~Lv3@dQx5481r0n1K8Pc~Cr>Q&^$&$k=7%+i1|e72R2e
z(u6VQ^Is-xl$T?luX+7i6)Lqt8u;whTSpsou(7debdIoBRTODNpFVx!BD(8u2$}Y1
zU9C84mv`njetJGnVxKiG2@yN~{*$Nt_47VET1P7B0PR4hU<YSLMizGLE7gG<y&L9p
zEHX#IhAuU?FGnPU3HjCD5e=M{tpg3}6=p=9IVukr`<w%41@|csGGP2qbczIYqfxK|
zLOpnx+tu`>At@fUm89->T|}A%HpktZm+IY1t(a#^sZpETPNo`jH&t0SXl%z$hUnIH
z)|UrN$tu^kafcc4BAeRYzY%XXJ6aq}IrWtNknh*=R#IJiOQO-(GUSEqE4&z<o?7mX
zc@ob{iEO(s<^2Y2{`vmAIJQS!shawJ5=M}sDuul7aHV_g;#jr9*3!5}Vf|3R?~k8+
z$zMP7(I|Sn-)1F$($QV|?sXbB93U{oW8QaeIo@_$(i~1(zs&YqY67OJO&tmdoR<zr
z=dGm;-o4@|WQ?G*YLEH&_$*f9?{EL^Cp0K9zqUPV-ijilKuz4+vm7+_>5eN6GpEB#
z;&ay{UF}2wA}~!Qq$g8dTyg=~?C1+$Lkfb=4E`PyMKb0J*;P4XAN}JA=F)aODC@K1
zpSeP<{)mQ}UjBCH(LiBLf1XOaUkFLaB1=TMvDW_}*vu#f4W>Y{+LI+iV%l9-ogw<%
z{j!>E!``8*TFoOhXaBbjCwn4!m!fAjt70Lmg>_lvW+BQ=gCe4J>*U2D`AC`#rz<CK
z3Y18Fhw{p{_roxYwY7nonRPy1{}`)5*y1>$#si!!!g+pT3zHIRnAze`sWI%0*<CNr
zN;BJ+&d*Q20889!d(M~Fkn|QZZIL7)F)_alo<k?o!cTM&L$L)K(~yIYVmBYaWa;nv
zYlr^vODY3aMqW-9Ab4y6@hmwh7D7ebrjA3*vb|802|WW|I^;_q%X>H<THE><LA-hL
z^ZOd10+jp@@v3Y2S#0c*>$aup7e7CHqhQ;3+Tiqw^2neKS7H^t`C8!K%^KQ!X44rW
zZiy{*4o6*YZhuzP$Ia*4HYsC@>!yRbZN!&~BXbi&=H~F;`StI)2_tjkRwBKQ%&pN^
zNLDuV9M?R}D~&44p@R?IQ`tHm5lkADzT<v`lE-q4Az5khhYR2zbA-R9vS5*Ry)Loi
z`5q8}8n`%CanHzYClfBGvswJx+#`-PP9v80ngC~Ki*y7=Q(vTqv0?OrM5Wcuh$`(d
z=BSC5ICoe2V*1M>98|}u4)fV55jJD(SbAa0Pm$;$3Lnh$i`dpDQJwc^H0fK4>Rgqb
z!6z<Rx3|N*4<2#o;!_v}@kNkK;uJSB9%NGCLYf!9^79nRoyQPI*7!E9S4+Xc-gaDm
z$BMkGutb?|Hx;(oLl@;C?&G%~j}rOZ$xDb0i%r>ECd_git=;;Or$;@2vxLK6#hvJ<
z`)@E6;eZ99?=={yR-SvqUpv;dwY1g}9k|0{=vJSvrKGv!F%<s$m@T3ukgE%Sx{^R|
zterq#@k1mzt!Z&N#CUzGf^NN0!8w6ge~_?R{DuB3iwoNI{6MmMTOy!iWw2J7lx5`j
z&c8VR9*QL_IBDD20jFsYRq`B<YcAC%qroUwQtsZ_VNY-bZtTKIDKnC3smcFKb^i00
zRf$v(#-5+g^)uiqg3@0FP+_b_pQ54M$S=Pg`$RZeg(-+X!*l6`m)r|u`|`CsvJLSL
zOz%W=9Fz;yC#zrgJd}-)S<^J+!$;&XhYwXoqd$C7PB!j~u|Z1&|6Re0xEw47impE@
zHFthQJ<;udSjj{Tl-W3i#17JWGVFZwd<HOsIsC67Of`JNB)`{O<Qb^SZmL!fM7}6&
zFohIZ@3Z#9jX(8Ns>;z}tsAdr{o6F(|7anEPy?kY`1!R$dAC$e;dY_r!}XgtV@(sP
z{k|uJ=csxkD;Vh~RG$2Tit#PAzWKd{55k|{`JY}cIwEYBgNV6`{F&iqDOIPD0@Wx-
zjBjtW{fU(4VtMa|C}fI`Dw_6E%8b|-QYI-%^p+diEeYqvUTXDGnO1()#j3tNXJ*UY
zxPOa2Kal(35;Fc_*uB_A<LAxC(nk}X0~S-ZkG0w6*#7KbvRU>?K>OpV@OU$d4DVF*
z_r%11WbZE_JpP5|A_}HCMX`6+H2`-R{3tgqHC!0b?SCC|A{CODBJ0(rE`BZS+k??(
zt=b~{`sPlzO)N*qc*eF2gwU4iKNEMbV2&8)<tufJ6Sv)VqG2AS-f^xOqmxgyKg<43
z=#IB|ke!-hPCHe^!^zLow7K2%-5FfD^n~XJ?(sEwJy?2|dq{LI--T23$ZLoi86*vs
zcXW@hVUzq|3EPppf&z``GFE{5^B_Sjxq=8P)7>%}8Q|B?bRA@6Q9#>{2+k*HZM)Q(
z8&5S{A)lcfde|=+!LzB5Vd<l2+QlVGOcu?6K^B?2$q-B5i6yuM4eg)P>8G@@E>X+c
zGvwV!*q&=6e$2A$|31M$pq7sVL`RAetZb(4jO=e!MDY-{PFy@V80T1Y^stcX0HW-a
zn`-g#cf$1GhM47`o!RhnGXI?go8uY$>!ANxY#IiXj;KF6uuFI-C52f*;2?UP<_`VI
zM`$p-RrxsiUuq<btv_Ni?HAxtJ00o67WU+1AHHDi*S}O1`x3!;uR(#(h*b5)Qn!t@
zDi$vqwmJS*pUqb|T&swQapJy6it&4^e}Re`Nx{%Tbfi!zehXYqvdZ!@Wt6o*D?`Jk
zj`+wTS?WI~{kQM<#}||^LX9VO%Qy<8YXej$-o`CQGrUMS!98i}1M}yeC&DDugO9m2
zR!)VH6sb-n6Q+8DspeB0@qAsJqA%@e<nh!SV})$={PtJJZPo^vwb-7ios%#8sQc@K
z|M{>xMT;y~P_WIPbbo_IN5RMR5h^B6sVjMk60Y=T;l)6#LocX0+(UG%jo8-=kAn;-
zU%ew<;V{k<&+b}DaM@hj>VFVz#nY(lW_{uH?z<A}Mx9T{pi83pbbC=fwz@ybiP^z_
zAyhn@)y-8Jyy)Ju|9q!sY9$5jA<3WbYdq_XPuTrP)gd}xYfKn>-K?p-_&qVkLD+})
z1pQJ310sZUywsqgLeZWa{F(|W;L*OoXLMpjohZQruS>)V^+ERUp?IpkKW@((jjB{$
zuN<Q+9)Bmk<kZdA5yzW*Yyp4I`1tq85FsfkfRv`vP&Trkz04Gu*&a(f`fc6m<V{jo
zXs(vTG7@UswHZ6zXQ#a?+i5D$Piu)rG3}igxPV|GNgJHu4Gj#nsto_d2?e^+CL8$|
zhs`mbRM+!gxy_7$m<mN+z}%XQ;;f|M9p)oF8MZGK4Et`<P25;yWK`D>f0BElgB7l|
z_?5OnYNnr=_C9}g?P1NMf2#fy-A)sJ11-FICffV-G`Nq}{25THykoFtFkJFD+x)8K
ziHJ#PqP|Oh(eyos#cZbSo%oH2Yd<|>Hx}zu7oWJF2u-*bwfFnFN5jQ0qxLt~yXC`m
ziOd!;M2_@%BZeBXimz=2S7X32rJ=)HY^Jk%B<Zo{>~t~!8yWCCUJLM(k|IYyF+eC+
zOoM+im^DMs2fgqB@f`&A_8&v_v}3^soZn;qw~5@piU|mAfH);Hz*VI^|Cq&SReLnO
zLWZxJrxuaT>0=GW;H~>vM&DVQ{?*5xoPIOzoN;FPqIq>mNv6Q*r6Eo%#j~iBO9HG@
zHrhKKQ{nbtAtD34vT2kY3axIEsE|@>9GOrjM-ofgIBf_P?)OUQ$xO4(l7Uc02A>=P
z+TqrjZzpPmh5+SD`o$l1HpR$*@h*5%^EgCXYSq?MLeI45F!r_|aAT^uV0}EY|G(x@
zq(kNA=C-XgI+Br-bL3xq`s`9`k72uJ)#H;72t7PL#HFFvnW%M$*>tFJGzw!~Fp7Gm
zIrbRL2glB1(GbEZjr?J^rpq97D&ExLuwS$CRk+dKh_bw5xSIEu?`8A{VT?z0_f-{k
zI;s(ekcRsA&N!4hbDb<RmMG#A@@SFXKo-zfCKc}PZNG5OqS>Ry^E(#q1CCv9yEW*4
z+>>(CadnsNAJZ;5n96JK{K?zu>-YwF#!fab+zIUoKW9O1dTU{our0Aw^hgNDz0TcH
zn%X7Y*4#J#Y8JUae61U>guUJ;AbzPyG$sX!jEAK~>*%nMts`6Xm_>-;=NTV^iBG#V
z{?XTe(hAh39oazI9ow8Cg?k&~-*AJIL(bGH%W>ZSL@B31aFTRC`<V|d;xZLukxrCl
zlMYlCm>={r<itZ(=8q!z_gxX9I5xUAWfRJU(M1hAO%|-}CmKnD!U#>?Jb;bwmJ%_i
zWDGb58QMOIuX-s&&A_aY9<&!0cvi^M<QWASm^^$EEUje}i2hwFf=#9+R=Gw*kiBPv
znO(FvIU(1xM*6|gLaAWgN_`VbTA~<gpb$)6gzH`_w^&%St0q`i{+cKD>qQx=85tr+
zc50g6Bfe8DyF5sfcn~4Z+7>`?$t6R#LE=vR&2`bO<#h?|!o#1NEnoP_TE~AzXpQoh
zKGq0a9QqznKyuGuGz&kyXey&EJ%|&PV4!i7=o_M{@Iy}}Ib2+dwt0K9tNp&<f%Q*`
ztlxsU<`Q~Cl}1~&aZ17Y*NG|C(=#(|{?L#!s+33g9w^m+Ge;-?1UMMfY<3j&<tj1i
zl3;(O*nTHY-nukid$p?Uk2{R$$kZR0G_}2s{Snj2$g&F23=8UiGklO6F*$z?*{UJD
z)LzR+uS->aKPKlo95F_z@Sn)LDGo7xgK~JxKanEd?mkW;^@&pb_Yrw>QFXyM=0|I*
z76P7M7$U^s{MSAU#o$XgWG^#3ZBGJ^Owkp!6`UMBnTm!+EHDRhUb3|%#vS~>-!J40
zf}}-e_DC2pbF24GJy67e&Tr<G2@oW;M1K}|HgWoE%Zc*;_n#D@n29zcraiB1^qVhG
zns;UZ4BP-C>%`<=Y$=SP!Z|MuawMN2>qGELLr}@~N4CUI9(wtEM`i=E7y4QY<Ei3@
zsfgl%7c#Rwm_V`}Px4-`VU2_tP6Y98TtdRb{Oqitj5+eym)gPVkDf<FXblzw4i_x6
zetPlS`Gg@$3wdG_8QqNda=IaGWn?JhC^~ws=1@;fJ~&f6$X-oz>VT19P2z#K1bBnk
zThFGJw{LDwyBHh^ge{IPM2wA-4d;2fIye~&hT;!bbW-zfS}io;r_1uOO3jo!P>t#^
zj%x<T{60O}T)77Q^XiMg_buQ#NYn*t{o}|rz#MhF#vul@quo_%5q7-6{~>JlLc4p>
zD+m2MG3TD)yu8}RxTEC@HCbfku|!jCq7+Ss3;k9dCGr$adtUk-W@5KdCZ?T4naFkt
z)#0^_r=z~Wp)J+32;ARnc&OHpC35(yV(8cC%jxo=3#rA|93Bm`!aLb1g|f&0$B_Ru
zzC?ex$7OA6Zv#Uko&mL5hXOaqpN5XAo@Cqp%`Cw`ze@k{e?m#N0SL~!T%Xq+H~)GX
zGXP1W+1xutWXQ8A5tSZ#FX}^>Y}A&P*%dXD?+$NCul#lhVFJFAr-m(*Z_vjYPF!w9
zLX+7uqVnNZQ$1K^7|LhGKu74>ojCvRDPZ`wC`zz@e^?O(4oTiA9!JUJfER?{@ZS7r
zc9{8chw@~QAIZN<=CLay@iyiyN<ziV28EirQD-uf`Qfh*75v8s1y<6;NjG<che_**
z<=h>dM)tqCq7n|Mc>VjoEFdb(7dpZ=9KD7vmeazWU7SU=F?8&lM1yj1Ci-)N-0&e9
z7x;WuTof7HH07;gPfyd4@>lx3OaR1>mJ_reYr6ZNl?sLTSF}%D2X>}oIT8Aq08nTz
zCt@Y7Dmhjtxd}oL{9t7pwamvENI^R5W#DI1c*5`3eG4p#;W@{$v(vp0n=h}U(%;Cb
zyx!*im0%#6<SGf`&UnQk1Eu}X1C7AXiBDp>e2(k}J^iN(=naQ6Bs&(<Z4$MH?*GGJ
zQE07IM=NdJJ|~CX;3EsG<28+3Y;S)&_UU+bIoPP3qXXfr{zh%|myIOG(G`0YKXQK+
zK=j)2>-&qhg+FQ6c1F+q&Logd_=Kc)KTy3WPUj;GNj3>-RJ#$Gj`gMoNF)itm>}$W
z)Dh`ACVWROhIi;y9*J$sankdZIXMzvJZ7)b_tBojihmxFk5;Ob>OK`AuKw(9;nB*6
z^N}ej<fUYce^!9yf`r_?D8MdUH?6TN{q2q<a+CxkLu7~%noTyfU1b{Xnar@mX;gew
zJM-yZl0`v@Ws;8;FPMJ+3uzLaphvs$k2G1Tr|fB%HoV7v1tPO}Ua#YSPd7IJ6oe95
za_zB&zJ8HwGg@IP_4IkWMf=U=hW5nj$5(%E^zUb+x?G$~Mg0i?QZck|K=cy<rHFJd
zIesV7giu?BQXPx5d$R1!n}vs8n?|`Ee!TqW$j01BHBYgto;iFpZyj8As+gtet^4mu
zBZ}kYMb$0$i2d0>z8y;ky_e)t9Zhn2G96u|`e2cM+y#@+KP|n^b+T3XAu#YjhbAs<
zL=G@%yd+-5Y1&0(WM!dS_C=cZX1DJ)9NPX%rZN3bv8azh>1hve$$0%>R4I%GJ`sl0
z#4|K_xZWKd(x#B<5bBrt@jNKswS4H}^{`^2#i0`0&3=_;59_f%zGEX0V_=ZqIE``6
znp6og=+bTrOdkF=86QBpyjAf{CTL|We~DL;jx+Md*6L0C?I09j-bmKC&McTf#~5p!
zJmT@4KELGRH*;k40pV~0sL`5!{5dp|NP3^9YYYe3vc~YY?_?5MT2Eu}ET!GQDnFr^
zjmzcuM|Y?4p@jmxg^1HKvD{*GpkxQ>5D_P~lobI+b06U;0D23W)OjTpEkjL56cSZD
zd+<O+uF~@&X)F-uhp3olwk7DCl7kd<vRfY^zX2U<vt@1|E{1o`#hg(JQA?+stVD+g
zS>Ok~+4I7EyzOMr9E(3((obDm71xriRWtVOLiFr(?7`2^7wKEpB|PdS^y<a2&x!<L
z+rPRq@eJ%+{=H2((Hl=AABKI>(tqVyv!Q$SUYPh%@y20bb>D-u;Nz@P+OboTGdb&k
zA3%$;?&@FCw@jfLdyr{zOTxXtQ#EVgRvi74ln3l9*@o;r3N%{m@8*fr3)_(dN6W^|
zvQp7f^UN!r?B>}L1&IW$ud+Su6~L-1zM8l$`h!!Jy-nXaagJnySHP7~N1Lof&cR|f
z4ll~F)U)Y(9{cA?=gMkWlK?Cfo}W!<jJ(K+K-K^ok_0vHF^5r*%~0g_rslPWx^RIJ
z8&c0g9p+x+4(q>3nb;{cvlZ8G05@^$4D#C|7(h#Ju}H(@19lSIK`qBKinWuWya<#X
zW%@>d0E7)MKS%SQ1a&l=-Y5rq?UXVvc*v%<pIvU36Fgdc9ddF3B!g;<+ZjzlmT$@x
zFIF~jz!0GJrhMqv-bC9&wW=Cb<aPlPPFKFin=Cy$PWcMmz(PE{8^a}5_!0!Dc;lp9
zY)ZIhu?(Ffec_iZ%q3ogerd>GoXZW}UfgK8$50s0awp~ko8;DKRcv#o2+(bfi6E8w
zp<HQ>)&Mh+P>91aS@14UD-%O*yP3Z8V2M0Ma`JXdV`}~DUzq#v#omZ`f{K$Q{Oq~c
z=0Tn7wk6V;y3xJ(&NSGb8e8YbR)~D#okcy^-G~6V36H}Y)rP_SU%hgF6a@U;5)CjU
zT+|v8fFc~r{nbXFznb<{iW~CG3b-Z@ET1e#Og4u(E^hi>l(oJ=4xAteknwMyZ<YO;
zLmd{KUTbH=*TL?&Pg!R4+2QvF1)zI3$;eOF&zehadk~NCV=Lzn8F=hxNnGl<!4}Kt
zNzYQ+Dw1aIa0jtoN>UWVN%eUIHyveBZDdcW*^H1yTod1vjv}MWB5Akl48Mr;!7(e_
z>T?Ve3EM|)_DX}J;E(<9gW3Ckq0Xj72^oS5UC2A_heOf^Y^&Fx-aiKVubJ13QVqYh
zNzVsVU;W{(xr~3_j&;;f4_osDtVo9C8<)t3W1(&JjbCS`7<=nnILDmQ>(<&BC8d4*
zm6gyC?0v}ly&Rj5dgt(<L-wGK5__WWIJ2*$lOP~Sgu*CAjTSzn*SqgsTU=F)^Ij*@
z()z>}76ndwJXYf4tzmhc207m04d#Yupf&0>z1aTS>IDcvk<%2ET1P=s+7_U0(>A1T
zZ)=MLtexUg-TJ3gHM9KF!HTMnp)q<JQd|!Ss<;?N74_QhF6=aL)8Uq~0YpLf0pWh(
zmISsrYIyO9xl0A^Fs~96x}+(mMqY=d=rDo9Asdpjh!RhIiME>@Pq{6ia?fm#Y5mo8
ztCV{UjFGRFXWBc$uno3xYj9%Ys7JxIY-IK}aNG^mStBLFY9#lD<mFW3W0t9@vn-kV
zDoW#u$I1U*$Emjb)_zVC<Uq@-D!k*wdI4hWdkrj}kQiVtVgRrk%bnz^M5EqaMOACJ
zX8#>9O@fE3=aF{Ox^$BFBje-g_qLa-NF9b(N6#<&kyxe{H{m`lO22z*xdim6$ljW^
zCwMOB8GYGlX)}Y|M7dpYpR!L=t5p|(#@75YcAQLaeDc;1;GmFOvtD%R*;{T0Zn(e9
zXzC9vQs^1)mpTJvn(XM0q5pFG^V|Bl2D*UEjstY46yU3=M?5p$Cjff!4gzeXe%%3W
zOpW%*jIEyqgwrV*Vw->8vy)f7I5F#=ei91i5TimVN9?=Vl;zj!!0F&D{BiBrTMW47
z%_bxJxHlkyWo{%N+rAc}Qny28Z|V4O<^|=;Dmly?#}Qu}g_p6;FApWMXPLc@qD@N4
z`g$aw{N^s03}pEw+L&e~x;*gfGk>n07CPZ--w^C<l0rc1IOZlUR<a*#Kc+`}8J4{D
zZ8a~F?Sml1VPJLI1)%7*^R#hNK{i&nWHM-$hohC2G4FgZ+QA05@km9oW12fG4t;ZC
z(~U`vNONYc6VnT!zII8>TPVJ`X_V+?NP$j+05-o$(M^i{cNyrhc=VmgtSY-(^9pY$
zXufu@ktKQ2g$N_#M>*@AFg$Gg?aXCWsW#>Ljt5Oo%#XE=6&kdtzlMM&G(C2vJ-)Z(
z5)33}Jg746c>XP@Eq&PcO~dwnUbKp;{uWS)(BX~p?My|LD)@ACQPJIR@NlQ5-h$I)
zu=%6^iD5a8S`{Uld0$QhT&{_G9PSto*Ev6a>Zm~Yo|qKD2QfhJd9eX&>N&ZmVKn==
z!16|2pk5IPCtIr0BTjfrC!ig;h$x*>K1L~J6{CAwLeFl!6;(i^Qr17?cm^Yt0<{<X
zS<4_eY||K5fE8&sy~Cnub>jGYPf>V%-cF0t{Jzcq#l^s?VXyEj@6BW9^5vaWk4Z)6
z{g4sDpu&1H(}=pw<%6~U<}>|0tE|Mc-iV*2h~s^lm@^j1j*)xp(tr2rE3w|S>`L{v
z6IhejflzNaf1Mki<lRVn>YHst8SB)`2E*30tL-Tt(bZ1*YgMW^Jfg=DNqQ|oMeKY-
zK%bgeD45ZepIpE0qJta+pKq|bOsf|61)(u8%cG$Kb=@}64hts}vjd5nTHAB-N6n`f
zU?~x@$WC<02E-&zM}-Nl>Q#ReGs>6TjnWpy;6f<epk=)Mvi9s#VO&!&JsIv#XUMMY
zund^&0iT`(IktJ*ol@HukPfJ>tL8lRo;Xmrs*;^|=R?PrzP!=+a<n4c418_J)*Ft6
z^&C0Jqvt|GKDyKnkMRfg7j%vy!3KZQ?svku(+2Q`pRaMAf=!9zuoyvmbOfOEshWkt
zbw<aF14220`R&l@ku}yMWJ6T+DZ7y;X5AX}HzuNb;p8Ziuap<-wmY94nTDdbxCL}`
zL}Ndv2L_=leB~Ilpd6smcZZBRi{0FP`WxrSzT<8m0p*THDEG{j`J7K@WD?j`H_zyy
zix4A*6PVD{XFeq@j9zM$f|5MMQDp&uIrXy(Tc%Y&Zqx!2Oab^F@NBb&IrrZ9d7%vo
z_~Iz<jZPfSwAcWX#v1S=Y2@t|Fl$swhp%2I^--OK#@Lb6{i`V7QvnS7cJSQ#6<}VA
z(t(0CKo(M`e}*!k35V<$1v&oymRn2Zo!SoFqL*8N23s)w%AO}M=}p_kEn6b|F<#<)
zYB!#;kJmb0Mw9>1mF8okO~J*2v+O<Lc{I+&j4A*0u>#o#*G;h9w*-Vs$TiO@ds8&(
zH5_ccSQT*I8a6h}Lw}8J2+-aJ-7h1SP7QjAubY5bOKOsJ3;3Z}6AOWWGi00g3-j4a
zH^OI=U2k4P<=T7xE8!%M=z#WVFgR+!HO^ixYSZ;0*EWGUNg)z`d0q5Y_8$z-zBbcx
zr?c&{jQht(s-4dlol95pzBb%W+UjIEmXOP+bPHpBK%4WZqez6RI54ZXiazIE9qP<@
zmTU?cvlACCNd0xZxi8FGh1G%@BaZYbZ;d8v@T-?2`AdzQcUT^Ye#2vU^UTDfNqP6{
zhatUJvkq6-)!^3K3$0BO0uya(>-*0ObqZHkK1FvwnyA!DCfQ7U5b8hcv@!s;1wf&^
zK#~--AHb~n`o7D?+&7ij4(4-uopcQO^c9G;JzPFI{Os0BizO0rigiE)BeS8nDgAvy
zHMr^+7SgWY_<6FNhL3N`f%fKcr;~M0ARW&V#-~CZUu#?YM=d9zJNWCjl4_T$XUSAB
z_!j#lRZr9H(<@|gg<_>_f&;?NVl~8==y|xa_QiF1_*uvxp68fduZ+iNqA&YB_FXE{
z#>-6V23-5#3XBxtXp7ua)prQ??+xQy3!t>V&cML(Y|<?7Mmu~#HYxBBp-jFXn|SMe
z`gB{t8LKy&vFC_b8r2Q*g}u6^SmR$CGQC^2OgzuEBAh`O3O<^OpO0YaV_$Q+ACAvd
zw4Fb;{J^(-fi#?+WwqiIVUV@>q%^D07a04E{~L{E@m31tF)__<E$d(2P+d?Pw86bk
z{*sMTIO`RH1GFg9;r8Wg5qUhH8P`T)RrihV$4Xuf^R4B;ozG6Cs+i_K`t#`I_*Q*w
zwC-Bthu$$bV?%(Z@@)hFxYR)7SI1`cg)jdaK%&`w-KoWb!*##s9_`NxBEcTU1~;Gy
z^@CgX`@t4_{Tx8W{IEjg)gkv=1vk=I2YF1C1-&Q`U!CrVFI9e8d;FIiOz(8p%y#{w
z0;USvuLGCk4|ZHj>h?{NJL2*AOoR;HlyV%$4>%}j65LuoiO>D2Us$7YbX)IMVw-gw
zc<%1ZK4|^^XsekPBH?|!Id?I?v6Zk#+!^*E0T?m0&H4s~YAi)Am<j2tL?pMTq#l9K
zMf-lHP$IPk9~{l?4!J9WcWzd1ME115rEZTa0yvv%+ovSntIQ#=Ko6RNiPA@<47qCN
z1XCN_cBbP(H=_!QrBx^SJw$Fb3cC|yiM;UAQtd4_RiJJYDc_D-Jb3pmTb24vg3irZ
z7JZq!iMDXZ@}12l9I>ijrA|F44s&dM=yiRfteq(Hi@pEviw3+4C|p|~9+1V$pXXb7
z8*Y*0%Wkp({DCcJE^B`M|CoBqsHomITv+L507+?(?vQQ-EW)CD=o%U%q#2bIX%GYi
zl%bVwkPZ<Ty1TpUJmc?w-t(^I7hRyTXYc*o_myMx%%8*Q-*!6P7y8a)++d(r>A0;O
zMF|OE?7<aE)$QX#1ZH9`5FVvxWH`N2VvFpCAd}4XKwez4!{408MT%tQj<0#<OWuJQ
z6l;s7j`3|*UmxLE7#a>PN{H(v;3OvC*$oYTE1qZUyB~VAIXW=Ivt$Q_q6A<3u1`v^
zI@dmK{*n&6X5{`jWME~1Z$~5M$Ww0FCsg+J^QsQxf;<)(`^Q&0!CS9p{Vxj{?KA`m
zh+0yL3~On|zaVsjg9J+*gU6OZ)Yd0(Z|HqQM+(Ye?oWhL8c~(-rlCR8sinP2s&FrZ
zaP1h~2*d=Cp!ythJQ|k%_rEEdG+LhF+Hv{F@V@H7vDj><(}iKezr}a?!5>$X8fl@5
z3Mf>C?Z<e=YV7{NO4p?M$x|j#(dFL*DCL#2T2rr{Ih`Iy;PXdm4e1!7V`AkaIs<n`
zA2Uke=Gc9@U=p+F`Xg2TfLp3;*9`NV-h1yy{XyT)CCbrPOTrIHifD&U1xebw1;M(>
z$^n+#w;HsxG%=A_lGM@u7xdp_H^G3Em}x~GOAt;(w^0xrJ_*04Lx(`!BzkyuKU#H*
zQ_PQW?W8YOhEG?R!bRc$6sX;2Lh4lvGvMIl6Qo)YSkcspgIL7@B1`-^Ei3AVuOeM@
zYa{u4oMcX0$v>hthA-NB=RY_s^8i;~+8LZUxGT;vfvg5f6x7hT()&W$CPui)^jj59
zv$BDI{$5C(pt4NB=sr<EQ!F^U@cC}XJ)}v|2FMA9K;b2NKSM+vgYIoMAT~cz@+Saq
zx^eDrU5W}2%Z!9P1*v&Ot>^C!mbzA+HZD7J!LL=)dDFMnvZE7#epB4x6G-fM23E9{
zD&DjfppwH^CRER&EcaXOPe+ADlR=j_ixvLB_J31iixk_NJ{J(wYnn5_0|%?5H+Wf~
zSEz6y6y|bZ7+PI?h(K|IT`Qc+ZaZ&pZ`9$9GUyxD1|;o+nZje5z8|Y{lgeQ_L==d;
z;#DGPLWk;$iDq^?);zr{d0sbsFhD8z0RA9_HeKr;2Vg|;s7<MB`fFGCYGV|k?my7`
zN#?9B^+9x^-D5q;sq>afh6c{h?2cXWgm%Zey#Frir-%R<yU63cDU659&KDkJ!%3Is
z6*O@eiL5bK7%3te@t3NjpZaBnSR}`?6r-hG{EB*0(x$jp4}9BIFv>n~FQ$sWxcd)W
z0z})|Pwp4B9uDKb>=H@hNi#ts_F4+_)LUkPys&j)0wyBal3<zP@`_+|)!7EWWgPdP
zn_kMQ>o6~D1vh2o2iG7M86);|?#ReUY9l{82fT0pX$xoSs^`A0fMGYHfkYG%E5>ar
zYx?{<WF5<o;{$dtxgeDJZlV&g%rKl4Q}ym%69cm<pJ9y%Vux5_2_msucX+k2aF-(l
zvji+WG!dnal0!}3(dzU>oR}W{epF^OfXV#E@a_7y7g@TpTs<}GP`Na5lT<OM1e9f8
zCwvV*ir6Ih=-Au)5;|QYX8~S}2lL?!H7YhDG>JFC*v%kZu<lo<6P=>qP$2qc@JphE
zxfgkkhU0ZMaXYT*(l=*|rjP7GTCvj<A%x__%lopeqOu@VhoyYz13CBA$?geStj_!v
zl%}7b_7lu+PO=T%$6)o0B+?F3UFGBOV{-d$b09VrE_-b{mHx^kJpcEZZ&Crtvfv5G
zA%9|OdOo}f|L}1?i1Vhbtc)O+N-WJ>VCR4;s$L^Q7H!4rcJ%v8)j5Fx0zFigP`%G7
z&8S?Fg>Pw_lxct3_(ZOgPHnL3qurVMNLO6G?O{%W>nq^9Z<lStyu|N-P>ble{V6r?
zpjmumT;&KMxd7m^nAbebL{7$l-DqKaI0OCs1}k~LIO#VcfUdgaimRh;Nf_H5Ex<mg
zS1g4yfM4(2CffeZRg8gDq~g~{Ly@z;&K$^!=1;jp1jyK)L7*0}GO(~B-Hk9BE9bX}
zgUwGtKYC8lO@D=ciT5|~$#<jR$gNfzf<aQ`+$k&VEXYFlKvivQE4lea&W1w9=EPj<
zsLEO;aP#Jdqn{0{KLp9_9hnj!k*h$AF<ObGciCj7`}tmwb#jt+laottdDTd;z$(b1
zg^dN@M2$)m0u>DPA$b(}6MScfIWb=WS5haibFroQ1(L691yQe*2r&G#8lZ2q1=+rM
zotEg>#J%+sgs!zic^bsk{6C+1m_8RDk7<$MPnvA0`x$52Pk2}hbOP$BkwP!q%=#B=
zUbjgn4P8~Imb4dUXfC5t;mbaS{^;079gu+Wdz4u&%f6m9@HCAeiw16IjrwB<=UwQJ
z^VQ8?ETkj%D7==IRS%ZxZzqd`w$)BMFHm0*BaV?)hSX7Yq8w9RIHG%^8GJ3E9~u8}
z^l|1}NWlHI%+0c;FzoUNGOe3rH2twk2EtcBGHgKBWo0kkyHH5f|FBW7V^SB;%Zyz(
ze{8e~zP1=|-X6S7-8>KVx<hQKr2F=54M?_?Xt4Eass~8#tX&?tGH2+bmj=1`ndABj
zd;K?)@v>!?P!-0=w!wzP+=uVhzJ_BEhfWhPr|vIy^cZ?hFS{Cf7TiZ%MF{urecaKF
zjSvL^>@gtErwJysFMEX@bFcVLQ>`rlCUK1uaQ(C3*_V_a@6OJ8aqPUo4#J1uXO#B>
zum_=OoYrW|?9{siSvOA?9p}y3tYy^3@YBI3-y@EB*6wqs5$GCGcCOnh>$$3_FcECy
zf*Az{YY6D)Y7H`J2Ix+v&WcR@a$<+ad{9dtHE@m0$wycEVEo8w7bLTUN6?6NU$)E}
zU*-FO4uPdlPx+k3Hh2{)4`1aAdF`2u=LDsjtn`3@Kin1?c*%_4DV@z=);GH$)KCgD
z4C7`EHt#`2>A<qO`umBhZhGnFWx&$Es>-@yJylhDb-nwJQu$@5cpEXdQPa^ys@%JV
z`MRGb&1g=#RJScmnEiB)T4Ep?h8Hd#F@Pm5-&E35ZEA@KWV8dgNLXj5wZX*9sYkP2
zI<7z1A|u!8R%Cl?P97*L_Z`dZ+Br%15C1+7&EFciy;9|?b;~yjZ%An<`ol_NCTkq-
zT-V~`Mw>xP^zWtL!uaPU<`5)Da@@CmmY?{g_15qgi~sJ!Sduj$1V(`xLDg|E!sMrW
z`l}$388^A~j`|#G^a&yzyX<gcgY(B&to)`lz6Wh61uO4l#O&7mP1vu>=!BB?Ey_b-
zoQEzTm$?e>NpB7Ef&WMIDE2868f3+wk*W!#H*t)R_&^^71oA&LQlbiRwGitimj+2H
zCuWPk^zY-y3Pgv_4(BojU<c*9!Qh!u@^Vz#AR|O~x-!I7E<{^B%$vLS5G=%9N#@PF
zws5zpHl@ykpmW>;rWpL7{xL!eh`<=(XpKISVK_7&H!yfBAtAxCx0bmlF3u0EsX^Y1
zy`~Djff5a<&}x-GWO*0jJ75h*+Qb=;gV0+GjKh8IValZJP!nF-r<SJA!5WOYAehEb
zJAa5DA!rP5dM@snG~l!GJuf0a@Z`cPl0d#nWeYWU&^T-_dqpCQ&uCkxbex?_8I|s>
zl&8%^=ih<^K)jW_ZHgy4!8~T5Y$An32Y*)-gQ;aIC6x&hHAyu!QSzVAABFf8nXQ<M
zb|%IZ{_&ZqeB1bsFPCS<s7Z_AW0@`DrA`hU;GMo0FR9R>ujA78zb@@8#QUdsVYI=)
z=+QHmQG03PT7fl4Xu)9DzTQtmiz{pKHDcUWkz~PxRLux&JL*ZCYCG~1TVLzIkx|TU
ztWVfh{DZ@8N*$>g8-Nt<TEQ0qQV90}<mIs??|3D@HVdWY`uV}yJ5Drqap}&Fx$_F!
zAI*(E^K7Zgw95R@Q`@gjv<ocVnU>3ip6t%@QyRL(E(Tj)R?V&5f`Oh&i@#nb{=~||
z&aTAd22Eml_@yuP6Z=%929XfajAswSA`GYU(Zax{_W&y8g9+W1XNzHbz9&zoYUpug
z;od`JHd#>!0;symbRMxgu8<?24fRbJ>%KI0$FYNOdfWt^|J{nQ{03q^Zm(NobLz$)
z`$|QaV74T@xe&O|n!uxLCa#?3(%p4|_dp<m?2U)L%6r+RFms};qMBswpF_x6-TULk
zjUYX>hqrQyS!`3u-qoq0yMZ)%%DE%Gs3nE$70%I4ZQPMBeg5;Fb}E6~zt>8C!oo+J
zF`zdsG|`eL1mxm@P%8_m9W?1D7P@5zmGLghc`S+_vKf5?m4GF9IvzG&`*;0JD<Q*}
z5tXQDCt~I#wcEIMpG1Tt9%)5%6&XBOzq->C=h`nTg@}-Xh)@4emqZ>~$oOv_6NjfH
z8olk3!YCB+*|A^0&urcYa<SOK7Mi`GW18EqyjxP{x;dRu=>{UQT)^uK>)Jp8nU#wz
zMhIR+i%v*HHIpNkt9|LtSNmIdCP7{TR1DD2*93=WVAuGRF3;dT6PBG7*Y{fMhU<Dq
z^*@vt&U+V%AJ&AXE7>9_9;=Zo^czNV3#Qaon}velSwA^z6`{Ods&cD=hyInf`QzPb
zX`^5jpb;_&b5bSZ9;84f)dra0EPMdylo+()UO8df0zi}8mEb|M0k16}rB_FQ%c)fj
zdj%Imgw5{a+{}OGL|>(7L9b;Jz{U_!*R_uE9(zm>{7jx(^r=u8WSsD>FC$;+ePu#N
zMe5#QCiBASsR@%=oGY>@82}H4S$7ywQm4V5E8F6nnXNjW+$F&jdN59y93n`(_E~{b
zhmFZj&as8Yjt)v5v`0AZ%|xJB0ZlW8M?u<fvIL=QAJ|bzWRVj0E|VCkYA!Ykwtp8D
zlhAhXnh>vO#=7@7;<N=-yXFlDS8nO!yjf)Zp%{=1f~vI`-<Hw@=_zEoPuS$?evEYY
zeZzuI)E_qj7CQm44+x7+8cR{%cF;BB6LdY!D<(FI^U0il^&giDyk%^#d|4mmIZ5_b
zKmQ8Z#GHRpqbdwNIwPfaUp-aZWu1&pET?+wH(#aQ`i#bFdweUX>triYETFs)1SkF%
z+nbI~d(l$n9{<Nchd*4D%FJ@}QorVyFm2XV>cf=GhMo6?qi%WKSmBeFgqJ#TC3#Wk
z?BL<1L_OSJ-W!V?dt&?LJF179n2{5g?+b|1U}4J8(@rv^EC$M?am74dQF}d6X*<$g
zE7h7fl})r!8^Mv#EHn>-+^1nvl~RsN-vv4#nQOr+N9ix6IyP&K-pYH4M|EZ|3ySA3
znibeAJG)?i1QGz6LGeD$2YagR_-WCg&jAHqFPZeIn?co^`+uT&xv&5IEvm;O6eN9F
zZ1v59PR7WIjM`-$VotZsglJY8EPq{RlHz`=qRV+35z+gE5=qsc`0cXeZfXIT<ii#*
z={Hz{zEAJTQxjjz^UOiveB(Kf$!6KaLjmVF?iJ}b^&=}~Fk989GF9r=*6dj3)szrY
zXMlYp3x8~2qb<laY~S=<6-8kfffnoqtcL)KW2Yj$;E;9M;e$!=tQGwT<ZRtHBPxgs
zt-h5~f{rQ3buPHQS6LX@sn3lZy{>)(S_@bwD|aEvVw=eN)C|{qQP`tQQGI=L5y71a
zs-6-P`oAmD1!-O8OfZ~vKPK=-Zm(Eg?RB`%-TQ`|qND{C!d8r(`bE^mAD(=`j8YVB
zeiF^z$T;hC1-TS|lrDOy+NM6dZ^8GM(EIX`faD&nb_^4ChD;|tR1{|`hO@J8E++_~
zl4}~FnqN*v&_V1xx<+{<$@4rh<j~shHk%$Hogp!k@cx(|jOy+U;(Dl2y@X8s@$P8f
z$oxQ;rR_Z@<&~`elv!;FcZ8a%EY5!JP%DUn&~Oe2L6yXf{$_1qtwc?>tk5TMiFknP
ztwu#HI$pxKqT@^2YxpsS<nLgK=SscTK)9Di_4da?amp&A+wFQc6%4=;A8u;YYuouJ
z`221y(;jY_mAEcY`qKr6nlyHPj;KawZ>1$%(cW4S;Tvu?NO&1$i%}(h4#j}tir~Bu
z$XIhL5_m^yUo^PIJJFJY&sWuiLjXN-;@g+~eTAd^L_Msx8>h@n3|<A+pqzXTFN`&l
z7Me7c-f$TZ9vRfD^&aY&Ar^~g7t@Eo>rl&NN4XL{)%(0|`RGgYYKguI-xm6%i%>72
z`IoWBO~I#apkUxS1(OVE!q?|8b8ozBYNbms<HNxJl=1QlkS5`Avda^9+C4o#w&QQx
zH}g8N|Csko44@o{*Um-82B&-r>swqLJ?+WAJbf=ak@xSp{>2P&M5b6PnO3A&lUclD
zjY-0=wHDf$%*f}oGX8pZxAkY_=98t1IM1Fa-Up@?Y7w^gkgV<A3mkv}T}T29dp-Lq
zh*nHMO)d?>q2WKgI6g+;c;z?1<`qZ7=NnWM-aw9tbS<9s3?c4SPQCqzD$TNt-Os!~
zP9O6+3s|KauxC-hvt?U^4I2&Qo9If?MaHG}-=|=U&sB!yUn1_PdWeUgT?~GxuJ$1u
z>;_^`KflmgGd#OzrvV30M5)}f*;>zH5<jw7QxNrKLx4#l2gVsa8L@29i#&hypmlW(
zneA7oR+qZs)-%dvlV0n2h@zW#U9bw%@Xf2UJNqPM*Ni`Sm?;&1-fhNEMhBrZiE~VT
z7ReIo2Jdng(L+c<dk>j3;$bL-cm~X|YN4Au@@sNX#QIR4ok~WaDIsNtr8~0%G70=J
z)DBG2LGlX##t{o)@tQ`-N2hBmnJQp!ajpUJ(j`MvINVE7?sw@CxY!EHA=bp`p{8$_
zm$g~h^D3Gd?la|<G~6Nx!-yI9qGj<+pRa2zWWD?bR4qe<cRRr1={s_HyqTxv-0}Eu
zh4)F1^u-_lf;sHE)NxEMh=<Hg8=26;eE2o3Fl3!#+-qVJ-S6d5AdeW_cHOeWW`(=J
z{0{9eK!>YuuL3~8yL^yfm3r5wF4}J7hsEdlD`W7Jt0cdi^)haH+{~rcygCAcNVzoD
z#`<`kf;7(d8+xBDL4}hC8Jw*Wu9i<360x7c|GYm99X-csdW5Msi=Va4TvliUIlKlW
z6{)X(i)MLMiTuPbxheRL?Y*$G4j+Gp<B=A3oDNo4pc0`hu`>ep1pHR@zCm~uay}=1
ztKG+~EW?zZ0h}adWa7GwiFEcP5A~xxQ)YUQ1xf#bjjRxTG@OaS5>uto^f^y#|6693
ze3`>QyKKh$dz1K$rjl-JkeoN;Y`vk3b4|bz(u3Di`i8oN##KlBA(Xq>NjZ;z*ir_1
z6-brGMR>@0_S{~q*Um(?$rkcvK4}Z9`EJ@yI)Sdjp3jV9(r#-SJk`{3351mqj77{g
zb*kBmVLU(uMQ{B^?O-fJDotU3(XLi1@OuPy_nXj(sREx)O(ZXaSwp@q=nj`hYg(d2
zrO9mAXjOkqU)y7YKyNe*H9X7a5K+jM2@x9!uU-pH?{7r`D=~vsWVPL;2eeQ@p1PLl
zQUgS#U46LRp#41Ly8xgWjO2)*<jM2aw9@Nqt916tlWO1dgUpN`z`A>q*JG32=$tmY
zqh6ul1ZVp5>QUQ2jn^MzF6l2n7NGxx>LJWAxpxOt-@Q<k^mv^SB(N<Pz1nquU^!Gb
zbp(a*GM?kS@<7s97CdgSgR3f|kw1Wq(~Z0Im6Qccf4$JZFpQmYK{|4B5xf?5t^nt@
z`0581pfZBL{C93siOb6Ckeo3R7cFoG^^(Gg-eY^i)`-}DWP5kVer@d>Gd;?=_I{Bw
z`~DgV((5hB8y}RZnHbN6|E7;js4@gl_7>Mx^H?u}#OHJ?{$Y5<zQne^RG9D$h!y3X
zu07tfxd)ghQ+OESxJ6eEcs`&9OBz3aXm14&ki(SUw?Sm~yI+{X>C@6&cd8G7#zV?u
z|7<b32k?r;lXa`;#Ph+1wI}L~IyT>^<p~DE4?0Sii~@r1q~She3aU4>XnYtdy5j#-
zWU~{O{pb?_bdWp;`e3V`4m0W2&i9g}8}EWIt#P~R?vo)o@Zk|ARd1M%<llUP5mK8o
z-g_K;)-@SP3tC~yzy{sHhiWqT7q30JXHH4dmc|@GLv1&_-(oj&xVvH<B<a;SUvG~W
z2kR5DaVfEbKE^SQ8Y)fcVAz@n5~k=U32|Eag~N`<BK+Ai!2k9wol2NpvbJLgLf#QA
z3-BR>5RGIy`QdN`kNQJ-2At=q4@cHUr=ke-9d{XqJA#=vj~RH(`%qN_0MGhURA0wr
z)MAy*xg~UJ!?!;pmZn)ZLYVi_gh<&^iYM|=$}A+hv6?W78YTja@A0~?coUmtSLYZX
z{cq}aLd)Cqq!$a^wc;XGJ$dq<=|SEe7lRCdUpPXqn{cI5u-Lb%U03z|6KhWCdH#g>
z6g2V1b0P#!iN0d{Cstj1L}Hr~DpRUAatjiN%$`|>-NVRj@*C$<Y@X8UN#HdM)xZ%#
zi;aylZRqbq$qvrS@2@=BLc@mQgxx8y|B{U$FI%|ENe(i%08iE!I&t!%M^BW!V+ZLf
z$?A|}b6HH1`*a93WJqC*Vte3@Yh$UA%2OS5(-)wK3oY!Ugl#WFHFc4xm<I%@Q>h>j
z#?$VBPyQ3@SWp#<Byw87DrmsDpc>Z>a<&SzLgjCc^~KQZk`$Z}_rpOMNRmyLUijvJ
zaK!$jJIR;oQp@)`6y|j)vQmcrJva9PQBerNMTGnd67;{K{Y^6JPB(5er(OPOR#`;@
zyZiN8?L{B^N$k(5Wlmx2(d{I3*rpT(FkLo#R#Ap+<LmNkaRKgu!|KB#O2elQo+w@^
z(X*i2YRM|P>1zC|xogC*@f1g^k~<FFo&bb6qz@tBPsqd1SAX<^W%K#>=$cp?%$6XN
zFzj`tWG8K@+xm+RHken3l0Ow&NUTL$AHocecIW}k`@_P@8JB_`J;y#B^jFIyNO_)+
z-g^SdJ!z8Uig&G)Dj-8zcdo!;ew*;#B`x(Vo+c>%P2qLFT?stfKazX@Hy63X&(a9t
zq$;%5ARRnKD)GBXSfzgf2khA3W~f(;Sx(v}0_%p6X(k<S^?;VTa~W$@6$-d((6}}i
z&t?Jf+H<?SWRn=dPDd|!Sw-;3>61or!lUdY8^<;mMKRc{Y}u&G37MB#zIHgm(iG%i
zF9MltuS^{#X|y!A-TOw7)|Fnwm}EV`qj~mdHPod4d0QxtvVNt5aNIoAf+UVv8Y-?^
zIEyni<EeU-*og<|o%y5?jCKCOs=z!XzR@YTXpGK^5r*pPD13c|Bu1_|!C(p%>^k`z
znaK9AlH6C~7PQefhpo|aRMgp1I=YkWmy)}KUH)O-zThSR0zDq`>{^S*NmuK6XjcDs
z0{izgFv6QaPbfFeRiHi9pa@1tlfh#FrA+LrFYRe+j|$5|MIo6c8E3AwqrZbounblQ
zGWc<L%*DjVQ*}K`>5jK3HmG8LlPB9`Tc{_)1y`;5kap9QV=U${M6axz$qu$|=`_Y8
zOS-&o`j+=Lt6{d`WgwbXh_MIKt)Z1SZsPhOe~}I04HBE5qMv8JS<nU45{}nU8&eN+
zQT;7I`xKgiBAA6yijc<SiywIE8Y;Xp=B~}eFfs$11d^CIJ`G*r2t79SAGHNbY476V
zAy*R9(?g&OqS3^>Yn>+a4D%KApNcd_ehSj$IU-C~R%#kBc_h)xo>KT<^hPtzk&Lhs
zJ+;1+4tb*ktSu+Mofy{`CEhVdi;w>mHg5Z4|B|zS=l^B_JZe{B4-~+WH$eaDR&_G>
z+Q$O96a*{iaF+oZv5O-Z3|#T|@=fbsKTGUvP!Xa45wtva>8*m<%m&Go)S~kzm|s<1
z@D%^T8RP*M;~<k5R;bGzb9bRvcz{$|jmQ$x!+01*9Z8BtpJz|sd;y#A0BVjNAPZcY
z49U-eI4RrambBYz)0;+)KTLZ-dvdzGa&guO$~XO)12z2+M)gnhDpg)RExvb`IFxA#
z{z{(IR*rlcJUokor5;;<1|ggno0MDgCBYX}M$YvzpaQNrg=!K=BRZ|>%vks-ByO(`
zmxGCAamJBGcg?8||5D9fK1lPAP>a7ss>ORfUB5w1r%G;?yOZp79DcV<G+U^`4j<9R
zQIzP50UbVx>PXVY=GU{MU=irJ5hQ{d;Vz$rU~hT-aYVJmQHrvs0CB_<zym}U&Hr%)
z08w)7zWxN=Pj#tnk#lL~5K(37bkR-j?T$-o3UDk?atY9S9mp^GQczOf4@%Z#PW+yt
zU^N#I*e-s$JL?3GXAHu(Xg{a>O8q{<)LR2XIg|?hIW3*)Grp01dIx%`MAWM=h6AI2
zWOoLR+=epv9lUQMiT%%54~8=^+pK)i!tS3w3(htaAxKsSV>$KD0<a^=Lok?V-srZ_
zh)<Zy&i^PVI>7u6yZ??u>w|u+bz*8W=hk)0-PztC(+pLC&9H8cS0i9-oB2cQ1S;Pj
zAS9j!XEVA!7D&xa_DV;Kd8MpF<_JqMS^UxMP)dPh<OcD27{ZkUW3W(Bpfn<d^59t}
zVf!w;XGi;?%#6TQZ3+&Ma}b@2n&$X_%Jp|>fCNr-J-Q5yo0qx>r%C(oHg?27ne%S@
z3%P%7zRmbvGa}q`d+7D0Zr|X``KPC-DJ%a3gNxgauzcjz<<rPJYK*LbJv+S#b~~q7
z?}=V}@A5?(%-AK`k8AGBTYo(v-`obSxfFp*fO4q41^RXI3^Qm-l}>rJJp6B^lW}Wl
z-SBAxB9Ow%P4=5I-2g~Nc4P@w9&axbh3-;GTbF3agG1caGOCP%+*pZ?NPK@@d=I-B
zk^l&_g!xtmj(8Ke0T|v7K!brwU8}Ij#mSNe+x?Dc&TrOub^`q%C;hBFYE(So^wl$S
zb>rrJ3?h#CiH+9<-`wx_>6oV}M@t46@PaK2zta2LNI9MzV~!$Qyk-zq+R%8p)uo$p
z=ZY%`aSx#_f7ebE&7*ge9{Ye5suYfwts0$>>U5K;;TZ<61&Z8qrM|K(RAJ%_DDnD>
z7Cq<>axW~%pNu{;jXb}(FX@*5EJ=VV^UJS0Q6%Oa4Y`kIxZC@*vYs)nY|{nhtY*BM
znRwmw7EnB53%L)QFK@~G?f|Z56=N9xjb_0c-oUu{<NEif98vhoC19O~Hb|_u#rf{a
z;e<?OjixL%E&a}oOuhGBJ4jTunA*2OVjMUSG5X2=F|EvUb}~4Y&sOPN=~js{k_G<Q
zQ-mF2ZoD1X#W6hmQeYCqFkI{i_78WdMQPy42!yi0ZmL&lX6@3WHqpszcPD=J{C{%e
zEVZUDB;q=woqC5LQkq)V9<Zv>Ca;worL=eNenr!s^PC0qX9k4Y4x~@pLphoa2+CH&
zgs$GS?QUuk9e%xd-|hijkU!Dyjn^d%x_em3-ybSdK#2BS#9?9>=ZCq2m0_{1DdEaM
zkj2rjs9(~X;O=j}+GOXFB(f}pAQ>}p`~9-?5TT+dCU!=u`Ro-CeGI$@VnCTOzKA<g
zK(_y=E%LvK<sq6^zsr{%*Hj>W>$zm`x3+8k_W*0E%q4re=J+%ol?5@{F*G(9-?(eI
z3D04T(~12<23jC+_kYVk($hfn2Z3Ddu+h#GmlJYb(@67q?NFCfOrMq?_XpxZ;>c=w
zYbp31Rt%uZ2$OxDoSYmsXEN6mC}h3IXj7c5PQVTu8Zw^leELM%1*ju?K%oiMpZPj(
z<AQ8ujcnQ|OkG{w7n+dkVQ0e{H$mFTea*8Z^C-a+G65=mg-{`+FM_y^(AX;)&(2Y#
z%x|oQc=G}6O>y-K%{t(QXba4p#xfObYF{+9w_xj6dV{81Ik<p%B%a+XUlYC5hYm|X
zw<l)Y-sci;sbTX{vL9ra*nVznRgBsOH3ugj>2XWDW0;f_D}+Nu*>uH&<r&}5-v=Et
zjNH$>d#7%}?aT@Of_L(i9`l%KQ4OGODBvv;4ce=az&<nu_PF({(@3%2o-y`U`xV@D
z$)y6S?|G3fvA;hyj-KjkyJ|;06|b42=@{F;33(Z9)9}R|Q<1K?P8r%9bndcM+D~l<
zIj&rFzDUlB$H&J%s{cZlz(PS!rO=Cx&t|fZRC?<=9VjS2%kc~0Hm$v+uTUu~1r(~s
zR1{}MwS`G7oNgZ%Z$V=S9<7qK5s0xkeDZGxY-1D9X^fMX-h@ITcgJf-|KGj)><%?A
zs)DV#?y-27JSEO2A(th!bD$w;v02c@j`9VA0oj!iu|=oI;ljt1zF$EU1reg`U%awV
zqR=3TQkVm1SZUt{5XY_T6q{i2DUgTYMg4Y!5Ga@gM+|$+69(ytTd0+u1V&J!YQ27S
z>bwI408g_r+QV83#vsWex&o%L2>pcGC^{*u;mV4_wYNXPXf(js`F^?ucP#i0pDL|$
zNd0%`C|D7g8jbyysSGfvlVV0BwNTqK&Hy8GxjrZn0^h>yQ@Sg<(#xdwv(Z!Yt7&Uk
z89-Ky4dGTX$K#G?i2A)2{O#ZV#Y_#Tmdxw!Jv6ga@I~kGEMy}i+Q!@3VU#!}A<y)i
z3DhN!5>n-S!pJ1nDwOm#u#G7u+f^{wO})MAWW%7BH9hVh=LH0tDDR!j0McdRo@NEW
zz^8l$={foE#@f*ScPyBt{~L3zAE?}6g**(Tv0>I|jAxm?2xCv9_qHl}MxVjpal5Y*
z+mhlX%}q-NpbkY6;8tGB!$(V5J-&O6yTR(@&I!hbfo0Ikj=u&2wE|&EG7p<fr+d{%
zG)vPkVaV3ttIeWDBU|gE=bK4swLAq{Os0{t1Y$mTt9Pr0bR28e*}PGm@)05ylcL#S
zZ^!cT<79fL!cXHt390*EpFf#^jLF=*NjAX~Mb~;BnEk9S*)y#$V$t}bXW;L8CyNph
z3Fr2H=ETxb1)d!rp1zLoi_6oX22d})3_fn6YPmc-2+|;{h2{tv)%yKQWv^bg{WWO8
z7Gl<%m?UJ7HK2c88fr6-^AP61SP*LiMgph&lJ;3e<993hrB_<%h#(28W(gBuBm#+D
z*zr5<_K}Im<G+1+VVm#fTQIp05=fpS{r;DxefwO>N}GPw)kQ?LK*jtUL{xlpUR0?`
zAbq7%ZgookJ$=y~&4d2y>5o!J;nBVaAucHCn;*{IV!xJ@zr39lA&93U{J`S)pF>mL
zjI0U<a2xcmLt1s&v#;miYX37J(4uwLyq~o?^YaFJfw20Q(5|N45s>B|HoX$BZ6qj!
z5k7QhT!l#P3f34vJ^!<O^gU3(s~_O#ID?)B&?3@|{#d7@6ri3;nq;`#c9%LuEMuWB
z+buE9<a3drXasAEzW?03BZ`otl48oGhZhYjo?N7LZ8MZcP9Tbt<_vA{^9V|ff~lK4
zg#Z=cL5S)%!CqFN=LX3<dTfnvXY_Xc0n6quzwoi>IQcF__^DG-|2wFaRqCS#3vpdv
zpv(d?27&&@;&KfA_X$9ruf>=J^k`&_;NCwmo81JJqp0d2zv4b}oaU|3O_OUQ0`Q-p
z;5V|zfR`oe8`oOUw_eidwj);95DGCVWa{G0dLvGBfz|>}yHJpQMF~IrCj$W^xe_Yg
zAxMInz14tV1^Vw#P1rRn6+YBeXMR~4=5~KkqFquq1Nh(~S?9{F=}`9U{{*Oa+0jb7
zQOmIUPhBuELL%h1Yd8-kdL{qnct@IyL^$|(yj@zLm$Dc=X}kWSYWlo$s22})l0i^*
zuHZxG$q(2yUR6ZkL^(J(hzK9)d}98~+x4}MYs<dgQfyt_by2@N@?k7Oi7KG?zO1nH
zq)bXzmKwwvw)>^NOicS$1dmKv#!J)o)gTr*De_DR3E$VGjbe$V5=yK48bsh~+f7S$
zRyc|iSnIhh)E?oyJDJFBm8L~8SmxW3@<y`mo-Tk$EQ4(BcWH7L7QMa`qL!wr!iW&W
zMs+mxJfbv&f;||}p=$010_Q9jJXzfd9alogwf9pc`~2kLUOqwzHF5W5Uu(0m3l2!w
zMo|m5Lv+Vaod$HsN=sBcU6x;KccZrH-3hPRq!yY8i*UTpd$M6adX^R(mgqQ(p4Xic
z4l&0!QN@~0GoLck>_n^!vpyhqsPoI@MLu+x;+Py6WYjKpGFUuQy0?7fR;);`PKY8I
z+i9dR%A~qX7}Nn<E%NY0&a<BuK-keQMFY(fsSV}i<c#UODr{wBf{8kI7q#B{&PXy_
zrMp*zkfo2^LngVkbg3<gud(tj$w)rd;35793{Vr0DmA*Q*uuYEGjBQ+7k$EqR1*Hj
z*F?zc>7~SK-6L98PsW|{h?o@vJxzr?x@B#JAGD%obzuwqmxcF!)|Bo8m0bcD^4Q&e
zGm2(CKqxd(;XI027c&S}TLhgGF+k12IK>ukM67;u|Dk<ML<B*jAE*i)?bI<FX!-bT
z2vx%+A41yOv<V_r3G;)_$t$aa0=iA_$^RS%oL)c*L`ME<4hs2dKV2hgLSF`wB^fys
za=&PTo17Zd>6l~1YF;Zy2hYiSC?(Lt13+-=R%*m3<e_+<t|=d;Zjc})NiPQiS4EXz
zKjP)drpYMf+mJJ&7Ny#GzmZx`r$#fV_AO9aT!GVN_m@Ws-3^#-aDb|}&NJ41ic|A7
zP@{_o^eHxafDgfd8w5UxDBJoIJ~2)MeYFRJstghipFo&eb<{xV-G8<+;KEM3(|}D_
zJ%`reyfxcsJoERC`@e^$gsf`!T+ij=ck;<Xz-TBYef9cJnS;&feabdrlM<eP=ROxJ
zqyQn>`{rdRdweZhAO_o?oqQ4ML*1wxt1}-ANxHvzPAfexE0-l5p1_4bwABA3(fiMU
z3t(AJ3hwn&J(QbpV<dc_+BBTjMXf2=M)_xOaJ@|=ji`$s+zN^7i9#59C^O#8Ps&Xw
z>;)50dvQdDFTW|5+iL{r<g_dvdXwUX8Qv?&2k$Krg3mGxXJLzkr0+{|0U8vxh(_<9
zidEVK>eP5<eGi|@AMMbH+NyoEOcHFIs5_lurIsG^elfK*Q1cV-jp8GYpA`jL?z7_Y
z>$!cYJi6*rszKocNVV+UHT1iA;qxc-C^ka@<Mj|2w0Y!i&a%&ivK|ykKtlxy`ybzb
zElx^Uj?d3bfm~w*+-g9a=)np&zW8$g<Yt+NlaXi5EC@G?z`wzyVQa7tLDkN!DiV8{
z=!DCCSD1&(Rf|!=sTaJfeW?YyB~P;LH+}&jC7lw%qITB_TE7xxey$!PRmP{{XnS&0
zr}DPkZ_?xYti0Ula1gHV)^_*dgmfbaCrW6Ypko7eZ5|En$KzqAYdO+|(i~vGuvMNj
zyDo}NUcVAJc}}jR)b!+k2NAIRRMdi3_Hx7J4jD9vu*0$8+>(VOMBfn3xnFw$>?Zj6
zL9;3F+%wQFBLh-WVwiH7TRvL@(e@7oiUb(A&)wUliOQlTRx?^`jK|%9dg+{}K%q0R
zG*yAG*C8KY^->e%>Ok3B<wcJ^Hj#h^Rbp_*Li=GUsz4iYx&$yEJnP_U{Jd&<sJZL{
zoMj2v76`3_H;+-^guF!CLzq3z0~Li<FilBdDFpr@P|Sc`ln~t+m53VbN-K-bq+bza
z{34I#TpL(CIraV*U#RFCn!wFYwkTHzu{7f}tGp&oU<(piaY~bwBay})%^2y8m_!P>
zqIzLIKjo`#^Ifm>1yIeuN;d9R{0Kr_%b2cC6f!lefaK~8og_R(_w|s%G~?n?UOC(f
zIg2P~pfpCAYUJJ70;;KcyTqwi%mmn#{#{n?z!Ss2OYz^`AwGgRk@O83Z1rVwP?<NV
zXXY*`<o!R|WBxS=m(V0Z3oah`q<^C;45frGx?OkGTKxXhcJIYY9kPRa<!C5TI0n0X
zo?-EjU$IfWZ|0m%Acx2N7?9^~Cce(H8!fs^eBan-TeN!PM?u~6dSs>ksbdt$May_b
zJy8}Kvt*_Xi)z%7k3&LgPjkw}m0@Jm_H(}Z!SMH^+6Fq=>9a5;XiG|*y5wWa-aSjz
zL0ZOx>6S+>ip69R9HA>Zg-?|qh-8H<q<-wTo@bJddSFnF_om_e<bG>4(^N&g?tobT
z-*m-*pHGEPG0MyPC7(j3kpjODTh=I-04$1+Y#R%U5FL;fe!-8~*Ppd5>B-p$#ux_3
zqj)ynSpVybE?{j`jABZ|yJ^9XrY$fdgUbYsXDS3QO+Su1>O5*XG8Qu>Oyh5O!$cWn
zT+jv7Ci81*5PM_C$+{qR@lh`;6B0u*bw<ZkGQ~7H8r(|9?_)ekt~mW-{k1mg8B!|`
zUA;kPVo%w(p1Rh-#dKlM3QNSgo#Tl<pHZDojoZ3e+CX}(=VQNmB6Ac5wGS8MOAGVZ
z2mnqxYKNKZ9v#>}#UN>H(>EH0nVoz)`VM9Hm`5Rw|6?UaNSk&Y;dTO)A$Pc%c|L=>
zZV9?T^1b^c;|e5Htuz76Qw;QYo@;-akHuI*AUH(}mgrandjpmRdi$t6R+C2V+-J$m
zQX;88!1QVdgoJG1bzW6>w(t@60qG+TWAVLk^ZbtyO<xI4y{~3GCG(ONt2N6<#~<O+
zNqL9_jr??+mdy0QDNYj#kSCa9ZBxL8bQ@%nj5E;(D!_x9nD_x+FCRv49}QI}yk-vg
z829|IH|1WMgln$(9whH!XB55y#f+y%aKJnW%%a5MeAe8L@_lW!@-Fg4?dPUCmU)uz
zrL?5nVfDxCOxmWfx2m=DY_Ch<Z~9cxc11D6pssk&En>?EQZ?oPh?Jt_G`43+jAAoh
z*-q;8M1x7I9%L5^&#+8Wr-2wqlM+p(ZXjlmcGnnZjuFFU10OIV%d81)5C3;HuSn%H
zOP?NHrp1}^VYv$ZcM6eG9V<kEgotUqrK+1&u~3&u37;dekBgQRC<)LV+%X(wDHJj`
zcIOaf22#dg%<{j<rd^*BK00jx`N7L4hbPLz@9JIwIkQneMhCE~$CQA4F9QZyU#}K=
zZC3P-;3j<jDoyDP!(vkOMN<)aqHuFBDtkY#nM?(Axp0gl)94*9P@;Wlf#*Ci09@ow
z5$MF9=DqHkMvV+M+aH1+yB^NGG_!`!z`9D4Y~qVqL0${NzC3Tdy_C~0{KJ0@{FujT
zp`vgRIQDXGC{4IFT(}j%x7&EyKtLnic`7;l;dMz+gt(VAO|&H)dpmYsGB7S;(*0;$
zj8bJ#oMi=Av&)}{V{3_}D(tsh%!q2xnrd-%wm-pbN%9gSi*ED$_f5Cmn|aI&xQIES
zid($oojOp)Y`tT3M$%|A#AZI`b*qu_P$ob7lkp;i{~1YYBAhDfnK*@5xfMNL0;lD$
zYc_abd)i)l$uT{BEfD-N__&^)$DA<D9`XqJ6W9x0!5rFKdJpaVZ(4K%TT+Dn)~P0z
z&i(t`SeXN#iXM1?Dul&9?D;HQv}*rLwAoj<ldR3)yQiRlj1Wm2*Z~-tc&?KC{4hmh
zV}{R;#G*XYPZ*i~T%|N^PdWlfe^pkr@*K!HQTB%=VDUU~CwZn5%qSurW%b3I&7LW=
zcu`#EJFt-oVh30VC}Dw5L0W*3EKL7?ny~na6m{7=Xf+-0<eCQIRV*(FY0pRuS{;+n
z+v@1cW+w>I?m%<fcU{GT=el~+KKfNthj2i0Ld+qWW1$Ks9^xv{&gajci{6ybHOpJ}
za-?>YR|In4qtS>3$Dnt7;2DLaO4R55p$HO%ZZ=OLra`~%D!ZdcPuRbi>dWeY>?Myq
zNrK<(pns&doSo(ayW2M_NEZZ4YHXZBPDr&@tsk((D6F<2i;bOnM@{JK0E-$mDsa(O
z-Ri#hIl1)R?TwIMjo6=0ez#ur?`~1%BIAfsPf#BgRId%R#0Jgh<@y|^k-7|v!8m=}
z7ebpddrjzJF@Jpfz-^B@hs*#;!j2*3T&7sBs-7f1_#X!IgEO$m@9*;Xsf!~0^^J72
zfqx4h5YDnf4gYr<aj-xN|BZ0%pHUw0W%fTg-aoE9jSRcGiPEwwPv5R)${b8@wfTI}
zFpa!^b|8xS*Ob-30B5Gm=7*wxy1a+AOrz!$GA7a0$ee+<#cz;Ns{*vsIe&O)<Czk`
z@(%E#to-}~Tis6Y!?WNV6Iij_#|pK1Im$$TurJ{9Hex<)zfFktfMK8v+C{oJ&wdfu
zdV5T>e*j>UU!M05L_<Uw-W|Mu!cj-@y#$N+55o}lx@B%m<Mp<aO3uqlIu_fn=MdUO
ze~nNLcnWoqE++RrFGBa*sOJ4$)9*^ZPg5jvfQ_g@E&XKi(UQ6WHxj!K1u@OK+c=pw
zJQPAk__lZ63o7eR@5v~|N;XCiFN--&aBNTh@{IiX@j4yWf*TY*l&f&G{CT+@VgwAH
zy1a^}k=n?Ggi@^Z783Mb!eZ|n&s|v|z&-pO4aA3Pv>aQ@dW9<m#KEoC{(L=XzkfSI
zRJTj&+m7K%+kA4lqjbXQbbX#exLNf@1|2)|0X#=~<9yX`w~_Z^9F@p4gHR{ph50dl
zvI3Foe)?7e$t->&$t_(3cV}cysWRloi1PTAHwi6a2lBbhRP@gbMWFC(*AMl?w}+d9
z)_LL7T1wl;KQ`?^#MWl=Dk{d86+f`&l`lcK4Ez0Jb=t4~PU64JssFok>~T$umcecy
zDik^Pn;6VIzDSmALI1`{hxNbT+IJl`tz?CKlNvJ+$4m%$m1R(wI-zHe(Ee6}rBxrx
zi`l3FGEQg(DwMgro@U1p56%`e^Q6r-Ay44&s4YqW8kbY3*4^A((kQD4@73s*YzaC#
zY%R-O?FE|<VWWk?c%-BzJIjVa6c6mj<pu`+9kfBxHUk`w7bN%;IoKWH5$wyNy@w_<
zJoE(P@v){4^uo@?^9HC4D3cOfj}45t2PL+?NG6i=4E>y3HD<k3rg%_!>Ri>50z0V$
z*o`?oU5nuV4j40%ro+^)G+S+5jClS^8T<d7TTQHc2b5eJa_ti0pLDT%v>7JZ7ga+~
z^}8pc{?YK7)40n2aWALb?>jWm23Z7-PI#(M0#lfz3_t|a6s&tE=6In{u?*j?=|26}
zD;9Jk^g1jv55hGi=jcge_wy?)g}@8G&uhlxW?NOx^pVy}-G=2{UJo5+wnkT`J!jn!
z#=tigElGdo5vbx+w;!g;uqTNeMUbvoM?BnE+SMUmRV{~|9bW=fnIy8WyZhkTThC9P
zbt{c;J3&lNy7{%{s!^z@zxi<ZILWaqc+n(AP4*Zv<og_%)|)d_PPu%PFmqTeET$Hv
z-K>e!4;7VFzt|Jcs&Mpc&+<LyqRZ1;AF)PQv+6|4V&G7@0zONc=6dMG_~PJ}$`_#f
z#2$91DNyPCwe`I%Tg$f9c6B)ML$frfDG+OSyg6KH6O*ahN!u{rLajuzH=-Y7%niwn
zk_<OW6TE6XcXxCKY8c#=_ZLysyj2xo4R#wBzgH(I;|&5G=5>tIzexk6O0?blOExHE
z7e~I%%PVu<oT&M=DJ|}}Dc<MaN0c3&E0f^y;rG?~A>C#cVY{}b&e}Gp+5thT(kX)6
znUN}9$ADWtaUgn<{s3(wWpNN*Oz&5<G|EP$<bHLKpB+E`OO+*5+^F307WvImWyr$<
zyGY%>XtAuKfN#_B)atV~C6=`M4aJgZlO1Dh@m+GHm)k=3KJhF!>G_zrM}wF*;U10J
ziCrcrlcVn=-MUN<Cht3%KN~M(Mr(YFq~yxPwH^X}IMJ58<!=rP7sb9yR$%`uy=^}d
z&7b>VIk7*PXo=WAhR=p3HU8z{@^>dapR!{kpWUx6%v8_m7|T{Aj>h`yyeR}dPc-&l
zmkc+HdK79zPktS;)l=94Jhjrx=Y2B3CRxd6*aOr;Cz%A!(>7srqbFY<K1-JzL8oWC
zfB6C=m(}3ZTpQnBB-+iM`2D{41@e%iME^!Q9y*B=^`R|ZjNj|`7qNb(=9rYHR|PW_
zpO!$5mzV~t3JX>5sD_G0ZNK`ncz!q_M^{-*Zu^jH^t$b$uKb<<zyi1~0i*K2i|+PT
zSM_z66ovV6w<lh8@+TJs0)M2mYlR5T`RPhf&6gvBFPBisQ)vpLfjTU%Q<^^r(W2Rb
zsfJ43p207x2$unB?>J8(WArnl@LkV&U*fjY-^%8u*H>=(8b8g;q9~bqoL~HB{}M*q
zNZXvKl$5S@d|E*W$MXg>-#Hx;i0mFcA`}9q^m72_<e-qj&TpMlu#tG}$8S|?iv3MZ
z169bkw`pQ`|L29Z_47se2*0?>AEeG0!hv%M2$YTHmniHE@{oX_8{X|v{ZU?KE-pjY
zKGAQ~R^1or`Gdt7qxKYFkG`iIC=K|JWw4?Ol`ZL<MpK9xysOWT$#Z*mgAeW3vLe21
z`K#?Gsvj+>e^$YivdP-Vj6M_|n_}JJ69Mo15rl6n<N7FF2s*Mc@<Qp<H0lZ%W#fPp
zQ|D;@2g&Of>6e7`*COa2D=W(e8*jg;?9*=1|1Jl6ZqR<Jx$aNv*nFzH*>M&<2~QdN
zhE1aeX~{MuEii>DwcA~ecLAZw-S-Nblt^%6_|qh?PTVpA)Qa<GnUPDZB$-E8$9L+O
z7k(rVh_+*6WLNsEMea?V3PV&CHBBzbfiiQ2r5AC@)ygV~S69AyeAnH+$;lB!_4P$M
zr7fLLf?W%UXxo3m*J=}Ok4{$J%EFAaT))$6=c%V(T?iZN2&ZK3IKP#_{^<MO1HSiR
zljPV`+tB-4K29G1C(y$>tucyn=aZUAslWQ)UVCbV6SK7TS6@&&OP6(e5o>%9CAE4g
zL*ysm^zfK>L`(k0pmD~;(WtTUrbdz8caMpMh3IVI+I{i4Sl4;Q0KFT4{#iaCij^k2
zK`}RB)Qs6~0CMvN*n%y5a~Yp-B+a~?bol$jqUSI28~Uhrvk)@#Al0eC#upj?3!=53
z8LNvIYvJq*clnUt1w1yPDk8xCTR?boJ|>5`+Sj!w@j>qJIvl424EY(wKP0`VG7MLa
ztsI+<gE_iJ`ao8Zqg7f52n~<1$KV?!@HkW7fZmSyNbHRMhcW{bL%=yYl<N1?2T9!?
z(1*PxHU~g9>c)|MsSY3)<6>mNDtik60vLTM)bUYzcfXFX_W_g|B&|5xtD8R?7J@Fy
zQ{WLkS@6P<^4t+w`P+SXNv4aCyWA4C87)dTpH&Y%+r9*(S+_-^A;lY{_E8{WAgMTl
zl?8w0p-5F!en5u}6;ltaaKW?TzL*a-xByA+f|a2WP1}=|VjmUxTwPrsp{I4C^cy0F
z48q8&Y_}dA^NK%n{6Z?Lj4WLoId|Vg3<&q$4>hNuQW1FP*Ugz1@^U&q__Vvqc@cv;
z?)B&8iGE(7<}mg(?Tt1s8dyJ^m|!YmAP=Wbu0bpt0I2~$Hq>a*l0G)4Mg0Hvw>yD(
zv)!@gmx<@TRa(^k%SOq<-!5eG+!mW%P~QRnEdQt+DWLy6r%g8lXhQD=qG}oBI=rtr
zf=ALpFyl_WwMGe-bLI2jB1|13JB_!5y6i^WO~hn35&teeoEqiLA5?VvnrxZ_5_WeG
zh4d>bg=J`cH>T3V?sgH_9*_vPtxQF7-<s4f=M5ue4G2G9KnsV>VG|Rl6PUdR49;^R
ziJb2zNz5t;mF#zR4$+>e6wBK^aa$?lH7nB@mMPBUNb4iH!Dmtybk}k|o@nfkocUo_
z1Eo_veBo$(%&K=V`IjJ%NaD}XW@t1I7Oq8zwGQ{;#F4Q{dGECjW;)u0WkqcxuxA7p
z`o41D;RSKf2hetNmhkZ4*OnCD$PCq@_YMm^<sUM<-fT-BFOvPyF!J>I=y0Q5B2R9S
z+7}l0`0=6M%mu=xVp+9gsdsb!%-mAjI`zq?MW75lMN~BS)h7#KVOS@egvDdx$AvcG
z=RbD|-+!yQ>M~0m;aokOd6QCBbb_Vvios&H@o8Tr5yhWodH~!Ve$;w>W2pVAJL}PG
zw*ofZXF(hlbgK4Q3E6^8je%_Ad?2zoeE(9<-!?OZ=#PNL;<OES?Jd}SEY9}kXJ%*B
zrQSB$24rA4p1;lwPWBKhVRNo~LU!it+}=bpt%8j|?J?=_zEAl2Sv8b2^;I{a*wh+f
zN%zkl(BW(v)LJC0EA}q6Mr%ay#|-P}>y*G>?Oj*tsqEjq!-s@yb%l9pmc+KNPc4iO
zJ%3<-U*CP#w=wefM@hU$uicu1ocl(tQGjmlBG?-Zd$-iHZi&2}K1uL2)bhI-&*)De
zdnPai^ca8XCABgdg>AazEO)DFX?+v@!U^xQN#Za&;ybua*`^&Z2L#xmQKl%A`3tP~
zTlgH6cC_`5R~*{<|7#}1`5r)YCqvTxhxM)UM%L-4w;I)lS*J%d0x>%Z1T}@S2>DCb
zqd*QR{(`>i)FO>-{J-Ykd|+MumzIaM$+7J|IPYb1nvHw6Sd9WYE?4}AMwS7*azy_s
zuE^IiL9ZIPZ2vykMoRi+c6G+G)MJ}kYre_7qyKA^5kvU?``&aZ&&sIunn4m(TBb%3
z;=fcvP&g=*<`lIFKehBA9=v?QTb}&DCj6)UyTLLWxrIMpmA)y36V1GvuqrvUqN79n
zP{6eeo1@YA>c?pL#%_0Ba3b&5Vd)h*!m##DFQPAtm`|_9?R%r-;wFfhD&(W~e8#qh
zF8@=5o#3OMEu$9X<4tN-pz(zO+ho@D7)_J=$ZPG#T3TA|&qXs;iWk8fc6<}PzsfWW
z!>{Ut?2+p2o8I1io5~5G_4fsf;f#{*&x{|rzbSOG{hq!}w=v0<HL+f8)VTK4XXuAo
zYIySHm#~TbMU~c#Vr-=D7Vru939VB&5Z_u}Ky^SDTcG;%{m#W9yk0-}wAl>;uX-zo
z@FY_05vofuU#qECl4XRLxXAhN2jS1#c@PnTxJVa6Rn}>uz4OcdMmwh3F=%0Qw0U)i
zMc(tNxYA;V1Imek3_&P%5`ij1jR1w(5%=6cDpWQAz0BLBW_-)B%5~+(xVUQ6_jVf#
zMU0<ldkI&rQPzs`N?I!hKUSyrcNs}Kcy_sVKI`ZCBMtnJS%KIU-@x>G00OPv`+xtU
z)7!st+lBTjeG@1agpKWuc?YB&zR)LaG5*bay^*-NPI2wxyZj>_)W#TFJH6c9dmqQ@
z&-R9u_@$qJ`gt~}S(=`&=YPfkk_*X>V1s<?-*3p<0UXGmAeWS^BguxIVT*?Vnh+<_
zyPQ%>Pc^c=g+nzG<c`~Dls`MlKizofemyf#neguXV0ZcROZ8Y3pS6GQ@8>esnK!p-
zra={M6jYOxi$*4WXHvoma=+}q->~>gD!QrUP3s<Y6{8M7F{R%AQDd`rajW;<5!Oj=
z{~Q9+_bzH;V^Q<xWmmOiI@!(gZzEaxwRUNGM9d@KnbC=7Yb*^5<s`^alfHE4nKr_1
zzG=s}f~L@Q%H<N_3_7$&(WM&vGC#lGKKe-Ow~*VP;y$dU>b(lG$1Wp8Yl{254o{?Z
z)oT4to>%7{)3OQf6DL&~VPmcF$P&vOy(=@u7G`WQxP6)XY#(dx%v<erqT1DJFz?>x
zmR7?z$He^q!`oXxRk^lnqk<D@X%LVQ=~NJ;LkR)tnsi9F2uQcmCEZBJq)Ta$PC=M-
zC=DVC()B+Rbbagl*4caiXN<GY7!C$1@xD*o_jT8Gg~VnA*Ku0GS%<}-wG4skV`lXP
z^y9g^;|nZj2Yml_(`#GFDB7d>4W?=;RVe>WVf0d<lbQtics6XmCr29%CY|5L5BbV;
zZ{_8{QtGgrCF`lvUD>5<>1A%)t{ayMk<PwTmz~zm8p?R0nfu7sBZgvr4lRwl_~dW1
z#JJne{d{Z1{XO<0HX5wwj&Q!@HKd1-Th$HEgf>3^{Iwo>JlX|jc#<pD_lkK6ZdOV+
zrWmAmAAn&Ntp~S^UXpm?r&C_Aa{ch`UidK5np!q@^W$iTyK=7Zx`J8p<ZLdWY{(L0
z&d+eJukN0Ip3INkpLEaAFG+XthQ(}sjRzP{n<K5$PzCb^>1#sIxo2IS(<@9@ck+Yf
zeoD<~P{`Xju3%1QAyJVfj+T0!P<+(|95SZql1H^+<PYWZ6f)Rl<JqIn1UAAR#eoKb
zmfdvo#PxT_#%kD#KN6PSd0&j@yvvvsFA}~3x=?XSrszrRl+9qnq6Jx55qjx>DCBN-
zKY`%xz-KY;@pCLjOcM6ku7TLS<Ci*y8=q^<7s73eL${+{<j#`Z0@{xO0gXDK*=iaM
zAe+ws^;u5KQO51{NdDqC-ooBpYAfpvFH;y25c3hFW;{RZn!t2w6N9v6A<F=aonNId
zOCt1isk$uL%=1lo%Qss^$&2%|rhIrhpbDVx?u8WR=tR4nmuT1af(pW&mvgT9{TU+j
zA|Byot0^4z)e)zp6&^UAX{OeD=w1Bep*NI{szwFDYmsJTrno)wlz|CLsQNEW)mo$^
zn`|e$EXQexn*}F5v`b%83ED%acAGOBXhA<FoK21gP<nWCJB({6>iq0DCEs<U1*-N%
za_&2s;%=A7`7oyGNdEUJ;(^t(m3ii~gS%g%j=7wM+TFb6K-#Q{A@;bXYdMaGpPgHd
z4^OmJ4GGywr$n>&)gNihzm$ayiKxrzVcd_KanT}0_3lQmt`f_LZEjRhuR9zJ2<~3{
zbf;fEX7IHW!W!B@DldD4EVKKCf#N}U>&F=F-blA;vl1q~2Hin^<EB`<ftBv})A6mz
zGm|e&A-MM9z%(YYR023PQmnyYryd08v1BwglOEm;8pO!DAR-|#n{$1r>_|^SAwCbL
z&W1m%k*WR!$+|J#$KhL_rZi!`to)zFK`*@VWTnXtNM(i$LceeT&nt|zrjley2HD=O
zdZ;&@(n^*_`uO^N$4*RcJsaeEr@5eYp8gXP-k^#KZg7jjs2&Nc4z+HB<Ne9fRTQx`
zYw`1uHpm!)tExc{UaHFihOg%qtv-o3>o!^HZg9^5F+Mx`L_k&)bmKakezV}Hg8BG0
z-ZUhcvoT#Wt`TY=@RzRRPBsD0SD@OqQ!#s?jEU|1GePKmCpj1h8mP_n<;l&>tu*N)
z-nKJ8JKj70a&TBvP|%I}iy4TcfC9f8lKk;DJpEkzT^H$#!)w*UH#OKL9$)i?qD}CZ
zPy#YW=q<%3|FkOxc9@?Ss_HuUs6cLu_e+@M&NS*v52!^w{^`M5_|8fd9;sMT=`|Qa
zSo9Hn)D|KkOHheHD}ct!_+v4H=gs_##_np{^NR{4`X<8gp;xshhAkM`g6d*YIGc<0
znh0mfPhjxLHf`9!0OBmW5RBBcyvBo)_n|B{0!M_!b?abv|FchMbl_fmOUf@ddbA==
zQq>2x=}Eb!Js5T=W~h`jQq{c=*H<)l3k~Cvl$N_+nz~s$-nXEO{3XzRTg{AQ^1El|
zJHzjfT-_#RIL?-{abo;2lT!|Y+2U>3W1ca1f6YxBEv=cEUu`X#RQPo|-hU*icPNav
z_G{|CXO2{J5?RAY>gcKWm(BssU$dHX&H2veEH5eMKIbGYCO;55j;mVfYf(R4$W<Vl
zk>Y=-&~KE@vXWr^=F7I{MsAu~by=9ml9byMj+Wz<%E;9|?P9pw`savk28!gRq=VyM
znOmG@XY_oF{(O2<1!vPeHcd|Ph3#R?Zs(Y0UhFF4w8*lisT2BB4@Nu*meVRq#U<r4
zHn-frAW<x>=%o#~zt9<|bA9<;YIo^lP4L2dzQ?FfLIp+mo0Rj;%Ebn`*6Veqa-Z0K
zbM;xx{9x&JUL~B!QUy<NS=Uc>jD27c#Tv=ClcTn|^d_?9$4KcKT&Z-uqD8aV6)pqg
z$d^xH+vTA}{z8l;Fx3~Te^FKjWyucDWgmB(*DqZ&HMQqV+S|Oyd1=-+PA;1pjgFP<
zJ<`hTU2>6-P!_E}cs@1#64tYAq8r+H(A*Wpl+pclgwtZU+_#^?Ehm9|d&#tTHNNDH
z{(xM6ztgT*+H%`q{kC&eak|dZnlzV*#H6}~jfV)&V7fu&`9+0s?y1`x^0rRh{&#wj
z&FZpAy88+VrQZ(ru{0ZMbfz9!&~8)BxAf|BQ|eag*dBRA*KMb*)bloE57NZikL*$%
zuSiu^v<W|ssCyCM{Yg^l-A*?@(4#egSf{BgeTQW^{6A<A-vfuAU}fr#9omwf!70}f
z=Yq@G9Pe)Bd0=}?4@)nX-D3>!BYZ^nksB^cdB%2ql)zFoi9X(qRy5;ewSJb-^8+|u
zhuQ9SXzRD3%9jV_vp6Sw!8=eCit)m1)PJ7;j_jM!Vm6pfH(Ca+D@dQ+3~nWBk`^3?
zzy-}lni9!&gGEIM!_pc=DbM%aw<pRdUnKsr8yk`RbSrR7yEpfM{Bu+F+mqUO@7sl9
zd}OClhrG}C^7Z@XW5oFED5j-ua`|!xixNiD<!a|_PgA*`+_amQ1Tyf&8}HoC`$UeE
z=@YME>dujmXt=E~_&NMKK%p-;%|tK#wA_ETAKkLgEyia$T(xm@u^7X#lxwUwD8=wA
zrgR`BsT_^K%g~QQn1d;9@u^}U`>L}wgZKL9kM7%N_fQYtqer+3k;ZbJbY0IWkbV}E
zLlxZ)DYM4KrXOBDy*|~KadWHm`F(VbD%+)BL&hT;PI|{A7d(nnJe<33o2~;F!uQ0u
z^5tbAzjUTdMBZ;L*ecI|E7qw`{ds<yKr{voHhe81o_?X7B~se`rCbGix-{gMxqbgT
z|N6uG^m9jnDT{+T35moW>pz}7HIUoMTxql`Jr(nc$LbwkNF>Mk_KiX#vSd}{1B7va
zX|6ysN-h8PCec{gg+|y%2`=G-k<R^ZtmDa<x9y}?P9}P&`Vu--u3I$=xGK`L^c;Ji
zH;nCsD&ULpCC}d%*;qR|*%N7n7_&`PYwpPLktFZrQ68-4vMl#b>bn+?u5%JRRju)L
zeUXT6f7EOpO7iZP1ZSqy=bERHY<hRb2adG7<9_COvESQU3UsqLqdM#i-0??;O%9C~
z-?(>lUVr8v+MGZxA`4kQUH(Do5P_a9^Mu~ZIgM9N1`VdfOnj+J9)$sHde)34k+N;&
zKV2}I3-YfX`#a#+();pkZS=i(fzjcSW~rOr*zojwL+)rjcJL)goW%V09bwL6n%LaI
z7jHo%K{_tG^d8QwI@ald-TWb8$EMTrkw)|*qXw$|D$3;w41P<+F4{_jvv;BwHIVyb
z(>t<p;>1cYn+ihIy**{fs*7*JNaL_zKeJdGgYcp+IJp7^Pq!Q{PxHH9pGmFy?T+U|
zDjoO^2R2%h8RQJYy`p}!;r6E#%lSN3GYC)OTPgP=rXT31G#ZJIjgy|QbH;)do>MC;
zus`I=J@n<?PHs)UuKj=uRldJE%a>a!7u=pt2h<$qj}>66Eu;PXDiqMq`t=XW&hH%`
z-N1$&O<2?EEws<l`?Tr*Iu*HCd*M0&$zJ1}hUgJ;k1Zs&`<f+Pi;a21wQLT;kH_^^
zA?Snmayeg-9U00Pgrgst7jwaBvsHz48pPxaF<?w@=BY;16qNRQtO8q;GiZ^m8vRr7
z<+y#h%~CWi97&%cTX{_vhbC0d5~>f0uyU8L7!yEMM56U)&*iwD4Gak%Kb=pWwB%2k
zwB>g{*HaDGeEBsNti)LU?U??u@A~Tl$Jtds*0QPwb*2}u#gN;1-P&&SSH5ob8fPb|
zu{!O#Rp~un?)t+$5rdI1U1Tdk*{T5*S&gLDWR9jSCv|GOTJn+!<1?ysd}ub;^5MDL
zSE-liuyq6H(KLcS-!g?V5UBmiK`NA{FJBQLfL48)R=jRC2p&He8$VrF-#w!>Ip;3N
z<r(=xSni}H@xF>Vb!rLYQn&rzK0HDD#PBf=9ET*x;Jbnfofm<^q9e_xC#^=B^#&=P
zcR}<4aDyeC;^)gkE{nSJqieoe7~kegd<E|}Wp*}QtRBnle7&n}Im13qKJh*9tWm-h
z<|mRq%z;^kFuV)M5I97>7YVOhM${~DePN)D#^EFuC&wmVe(u~Yd*VV$*96+Y6TC<#
zLeywys&!=ak=|~$kYcL0Ih^}bJ1f2CZmrEuBppkW{(;>TRDQ%$MHqCBesr^wB$!=r
z<f?i)-AVt6mUOYaBI{AQ{NR%pqE+-+Tyr_!1xyC`k2)cr)*RG_n@5V2u2+>&>g2p@
ze3X91em_NSd_`{g&{Ioo+lt4v3lg(t%~&6{_O7fNWPqB+sjLQA1vqHUs@84&oNlB{
zx)%*v?R&mh*5XQ$PpWIWq5j|@SCNY=Pt_0Dq_PrG`Dd<{q}$mi8(&d8gC2e!s2m8!
zslJ#iY)azm6m(-us;#sxZmjeOuX8PnA6PDcuUxORGttxMw&)w+sZTZYa|WTa_x$$2
zeFl27VZL?<1NJ9Pf5-WyouTdV?jp8ePmXUhVq}NGrYX63tagc<>kcl4&A+%W)Y$0H
zB)F==1{bpW=uGJ;wo{KorB9qyk{X+sE9~?M6rEf?M4SlQZP=w1hTfoGkQmxr)zBv)
z^Y-3P2Qf?UTl)FAH?RxzymOvJ&)uiEpN>$!z1s#*LGIqLo87Lj4MtAgpn6UfHQnY#
zp$UsgOcI}@=NR8!<E|`^+Q2DSfsHSEH-J6)@C=1S(%9Zwn}dxp%v5Q3B;~X8$(L>p
zaOx)ZU&lZ8_i77&nSNv%H>VFInKBP#Nyxh6Bg*>Qv4==3EBzDL2End*az>Z%b4G}!
zVV+t%YW_uK)O@#&b=OqAd(DuXRXZvG5xbN#Aaf5O+OXZT`kK+G70G>1=xbx=&WgIY
z@&r1=dX^owg5Eevztu7WCXkB^VFJH>+8XxrV*43#4Ve!CYzNo)?jtv#u|ms5>z%hY
zxX=6K4CIuQhS{gfk&z%@?~0OBWYp*7piwX6`ILNS?oCs(g%jsKx{T`o{y_t>7V|up
zo(UEC1I7bO9R)pIMaE-G?W~;#4%(zo5cs=|J<0+xiG=Px5G?|kA)H9l#0HIJv0rB4
zq0O&dD**C1iU7z1+3Hkli3ka@@f1W$PLh4Tc->kHpp7bFe`_F&$~5VL#hV$Yaa~O+
ze+pdDRtlsk^OqZM-hkUCh;oIMLd<NGE&mZK1ciVlRVrixQ;u^)YXZCq=LhdSHl_Re
zSeE<eHcdX(+v6lw6Z?sSZ=^;>PBJY4Tq<a)l$zqLV!rVu8$Fo27<3#%ls=w&lXN>j
zM|Tu45f?s8$`yv}%k9Os0~XYYzBO}}KYO|L()gf#i@s*_i}Gg$7ucz&7f9OqYOw;v
z4mV|QfT-f%>=5{CD0n#H+i!4#Dq1@(##GrGOAFNuR$?9AujJFgZJvlDz5z|rU{%v9
zv4sAXMugur<Qjs*471(b`RA_iv<n`uD#6pP0l1ArnmO3(U1G4;5cpwDHw!Y_Xwo%*
z=gqMA>vw^v1BXTmxq8^Wy=xF*p$31iSuY<e=|7(e!U}vb)c(mM>Q3aB!f6~*=>}A+
zu0!vXU61D~Z?Pyjg2gh}M*k*=XcYS<<UR;aWdla)3s(Zt7qV5l1j=xaL;3fo8>#lF
zhSOlRvc&&CUoDXq=2$=Vy%OYvGZ+8<3|+(7*N82=2Gqh%FBgkfB?3HE4FT_uAFp@D
zA8Kp8WY??HmEWqEtCuS!usQhXHJ2r*?%PV%LhLH>_h|2Om-e6d@VG@DPLP82?|%RL
z2Ji*aUBBAmzY+)FuRan!!laltrP10|4Ki--Ex+e`1s_SUDUCJM9q-}!Y6+8nFH`Ml
zms~9PT<`bkEXPDJpYLNoik-@Q+ui=EBu82Wd$zkV)pN~Fv&qAK1=}I@621KUcKU3s
zaLHh6ry)HzxcWg3w4&r`cU|N|!Tv&M4K}Mo9L&^21#jQ@&tnGiy97wb#v1`~{xrB2
zzUPkt`3AvGiAmzasz;JI(+0Dz+h@=xnja46mhvKwE#kmTK(ctw51WJ8a$kxN7(UZ1
zrBI^bqE7&lhc<7vye5V;W4oAfp^M`*z~w2LA=jo16>wWSy>9otidedRuh}@>{KQ8G
zejIKlVg*wMnYG<<4V?K1fB%NRts<_LuXS_X%_0T*>RZNGNRUe6TSpmkOCcAwrnB42
zQgz*jn>;}N;{HqIUooqL>0DPwT2b5A`un1#dj$1CgRRF4zAe8__~Y~CJmsVxqDf9E
zI`mWE@OhYUsehPfU@Zfw>RkG6ROFkO6Sl(H@NzyKIYK~)4(|PHa5L`197DpcA<Lr!
zkXpG8tE@ql_f@*}d&B;AMX3EndFHiI#~5Pnu)rj*`J!Ryc)D}(?Y^Ohrv`dP<#6xb
zf0C0Scq8s9-sM815_GQ!vtzxY`WZ^^uw1fl@Qs9$Ip%^wo9y+sya9i(lJj>`DlpE4
z4PEKV=~=nOgmLYmt<JloHLnN0SZ0}DZpMAW%<u@qToq%GOp<AZTg1tk#gz!zV}!N+
zZ4EDPIru;dsXUtFFAz?`(lq1-iFw6^i2NOsg<w~Bs=i2(cwUQ|?d5)dQb-qY9|Z3V
zeubzFO6B{=;w?VE8M}o~g6|i5JT~v_?Pht|Vr?Q$;%@2y6vaEEyb&Kt5T8s<g~rQj
z&~UVGpmBWd8g`wuxR`2~*2filZlwo^74yA?@kz3)e1wS;8I6|?oXg(4s$MuHFN&n$
zKW(5d7V@Nr9ffMf)7P+aG9(BM<B!7=6bI3R8}QT&@23?WC(AsWJhUtFK3+Rl`833C
zSh7S7c<R^;;VpTkx${JI&9RhII{X};UTVw^Ety{89CjA@w#xFqy|6VBB{`ay6t5Fp
zkLm_Djd$~U`^lr;s30BpyyzY3xw;4iaiTm2&ujR_S_Qam^c3_iYUB0_a>-ei8Fb;A
zbv-6oh|8qG^Ni@!vu#JlKn`l@=Rv(tj=1cW*eM^!#D_1}%!m=V<c~7>-M}Ts{&k%e
z2yeX-`-fM7aH1LSHaRJJa*|fI0hbt**!mqSYDQI=>0|xtmp;br-J7%xwO%{cQ$hSa
zw^1((K%&6bm(~mukkr0eIU2_9t_6NM!Q!8fWd)u|P2%@6(Sv8I!pMIIo+<TT&vcs%
z<)1(SGJ<BzWsW$O4<-$W5XUm}+4zl}NE}QUpMI@<%IUXmBg8hVx55?D<;bI_mF=Cc
zZr==neMvZcDE#<nxOQz-^6d0hX9r?;AO3T95qGBO&z&IvcSiG-iVnCl@>h3;Xv_CS
z!i|4EobLl<u{~c1xYP8ZFo5;ZFm7{ZzMa`Ei-)HU9hE;S<u&nlBFBHhf~)#pJy;l@
ztAGDlL0hf#;D|MFE_s!t#S7a6@fyfhZOQ~o*O3BIbgU+BmokUEMqD^tE~KfoL^a+H
zu=cA6NI2=JK1|8`(n68?A13F?U0hYFCtx!zD!ZA%W@ff#W-RqkQ4ZCuoHhk(FL&-f
z)B>il4q2=Q?>!re*=40wv{eXcd}?i81pX+36yB|+^e|O}=CdA4oo!69`1DzMM!t0?
z^WjOra$IY2`6%Q_wAB!4oQx;1I0T?n@PzG8eR%jyiR7Pyyg45N!|;HlA?lzm86ny~
zoXZFrWDFG_T;OEZN)T?*Fy8Y}eMWk+x&lvC@URR${n9H3uuOl6sG^Oyt?Y-%t9xOu
z&|1TD{JP~qg5fGpt@`MehRqv|j2WX@dW~R^julau3{(4YfZoUA%WYVQzT@lj5xJDx
z`lbdE$Pa@@2ucF}rJp(VE2ucXK(3W9w&pK3$4PB?z=-<~_mYE$R)j-~IC`4Kmq%|Q
zwJ)VRQ9D`l#d-gkiNk@NOv!XqqPB15RM1SkU)7a&SxEP>viTKP#}k9sVs2DY;NBV6
z`}rUq+*!5DJFD{l{&GxRUuDO}xWw>(to5aRw}AL<Tt^4qPmWj<q(<-v-x*u0ykD5g
zgQto(teUzmMF5ERU#wu(W=gL_OQpB7$1jDH<+sxxk0ueuPv7%@bN!0uAI4<pS$c0A
zHnp(D<$p(CRq49>M2|u9W>I)SO#gGmt5a;mx}Zhw`8|=|;mB^QibD9rAnEj|(|*ty
z0aiIa|8E|_{h<Mq!-7tDYGjLTOylVq@uhL0F5sc5%U$?(1##XnARccDC49O5=RgV3
z-bRe}(ta@j8vlRn@bX(?hf|Ai7+=fYPWVn|K*@i(!#Q~2OL^|vP!`LPK5dg3I&$FP
zm@ge1)BmCgocU?ojCDKNY~V<a#3r_|3>W%LqW8HGxWf^lAJ%{0VR>+e-F`5IFTth0
z`Eq}rXHM9ZiVLV@N8sz%RsV}7P?uvxHbZQ`HF<N0EqdK9L8{#GXO9DTIFEKlrYi&B
zE`@kFfO{OeRF2deH9z+*Czv7Lq9|J&*=hqV{v(Z#XpN5*wzgA`OW+j@szx!-&eru_
zq68KO%@P-N3uzqxALgW8jJiNLD)ZR*tARWzUXXvQCxN!iwikAZ#MyEZ9hf9Qwt_w3
zlgBCw?|y!tSLo{d2hrcbB@g)M6~4QDpoDVSnZ636ti2QKWPu3LQwJW4p8dP~|DXKq
z|FtP`F;|vd_bL}EU)lLWzEnj@gvYAuQF&XH9FOY042WLARcM{V11?tho%iGS2V?Q-
zjb$RmQL+&X<((m_Qo#`TC?0Dd2$3v-gQ(qj^YiAV1MoG+1m%i`^KQiZ;f_+@oVs30
z_YdC{<S(k|NiDY5)BYiqlLi;nw=18h8%c-^8tS(G343Qb9|DI)JEYzMes$5v-tFr9
zQxOjTyDRju>kl?}gMqJQwLPx>A&)f^5N_e)q#$JU-{Jf_1eRC(jpa=dSiaak1jIyx
zzp;Flkp1_OOwk&`?xCRsFvhC3rK_pQ1Vr^>0GCJ)sC5^?8&2~;>*@&TgD4B~I@(JB
zluSMSNnA~Pr+<U~!0lu)|Mta7PPFqNrp|7S;lju%r~<3o_MxNlN2ra!_inql_-IAp
zXlie}v99+1CK}lLmqG=;r#6<pHKxrc&M9FV3I^dsg%SAZu)@30o8W23kZWI;xUtM#
zx?h0E6Cr>IBJZ9_VB;mm2R_UaH*hOGe4iRyusD^7_xrm=L?a1JTwc(WRkBAFm`hp*
z+Gw#r+g&C<0WcHyH66?}JU*&v?da%GhC<gICz)<vJ<o6J$MO4JzK5zjf_%Mm&rtns
z^(J&)Ya)Y6msPfz9_)YYVmt7Ow@Lgz|MrQgz$ZH9f5%?h-k!d0rOrOhe)kxLW1@?Q
zw6M%}aTWg~NC23Jo#)D9i&BpuW939R?nS!-_Z|EjFHQ@QZ+)0+N_6Z*jC=JCa=n5<
zD~z^HY4gm;fuPaTBu+ZbV{ylDs%L<l?W7y9sHP_TM7H`462Co}G%cqc<e+$h<jXy%
ze~l3LZ_uO*<;taYKDP}C6t#xv9l{i+s;f}pVH+)aU<cf-pJqWnftO>9PZPX)Ib2f&
zWXc-cTkav<-=2ZPCfLm=9vR57m&zhrIlLX@!wX9CDKQKsMM1zea8{`~5ij&qEm{e@
zP=%<XKd<$Hbv3H{0dfk+R^5JR{KTk|@WqBqxaB>hn}8e#u*#5Z&-S<fu;za@K+E+k
z+&OV0>wK@DLks()lfB%uYsU_1his)aDR3n8!0X>byF0|cF$}IsG&e{}JX*-*z{pk8
z1{b9JF1#-UR1=Ip)Q>;JE()L6BwY=-+GAdJ;NfPyE{=El84#~jGRdA#B@SMRed_}C
z+O>;|0cA1OKXh*2(oho#JEmT>%+jr*pv%0$qJI-YjYbc7sv)5iW-#c7_rXVRSK?Pu
zW&vbR<DL|*qF9p=ZBbAGHQ#FaLSBpffU1JR(9`XH+wJd9w+jPv8(7A*xjzeyIc>-i
zM0@N>Ng<?3IwThmQ^*y|17TNM+JJ*GB(N2<T81z@TaiJ03GD)@uULW}8CMy441EXI
ziyQCGOc&2urHft4z&JMpg@7Zjv@9k|f#=D7rTK8yJrO+@g-o-dbWV3T4DFn;5Oe`m
zy6&28W6D+DMvw`Ua+y9AF9BMgcWNxhLVo0<UXJNpPCFtdy{@6>;3Ji-h&Cq2NYgpP
zSitO;H8K@P2^}}*=Vwxicg(1ob$QB$UW9*a9idwo6UuVb{xI_U*;~CKr_Pb3?gsUZ
zRo!Q4(}Omy4qBUA(vwNHErD{c2sDcgMrix+?ADoJ((!W!LQh)rAv3nt!Zw^KU|Jcx
zswgs(f)8e)N;Br<H85AsDcWi?!W`YhTsLpsNEQ1{bfhggg5oYlZaF=&bcKt#;lQ^^
zJlAJC)q14F@XRv-CRp$-l3!PI(0_e=B803r;OoN85aR~XM97z$L-NkznAEHS>|z(g
zUas7kHkqc!uu9c=vp!YqF3wpK>~y96M2#eZ`(s*&zsk1WqkKXaP_&pe|FWrIV`>=M
zIw&u+r?Db>nrzIc42@wG`rsDj6N&zAcEV%En&(PW%Lk&Ie2qjo-|?Vg2fUa`fX{S}
z8cmFvZOf#K2;<}jLME7R&K<c3-mhp!V8~OpmeZuX`iY^OkH|h9|AJ3FqEBRA1-%YR
zV0i)9g)AyM51NK<T#cn(ZB&%c1-c72>esQ<u#4cTSG0;eXdd-~Dn+U<NZ_e+rYYYh
zLf#YC8zXj!MsvE$?TXGKO0)<cyjHy_Ak<qscsl-lqCr6a_(67gCz0}f6rddY)F6os
z?0S&TbT}JfsM*!NfR8=mU~6IuA3{kkd#mMJizRMfzD&?y^E;24@;v`aD1ir#oiR2c
zoRH+_5AaW>@Na16L27kaTCn?zI9u}-&l>RI?-X|K>RG%4L~h;ByC@id3SEIUV2uG}
zROFiF24WVfuT1{!Ex~jd-=7odpaY`>&z1@?g>}cusLj0M&aa)KZvYIw1Pmj}!7JcA
zsxl8f`g0z;F3+PLi=6P$M{pjSn~p{rm|k(`-iI;4n6Nikff)0pG_rE6&vYpA=Nxfo
zsfQLWS9+ya2k1$v&vcb0&5*HIN|-fJnt5O`BF<wyK@G-}5@u#*+P~E2Jdu@+0mA~)
zK&mCza;&KSHZ*3k$-Pdt=+G~$BiSeO4y~2>d~*}4RyAYuQ~aP;wLT>0Kz^EmMIH7d
zRYPp7P@^pUfffAe_#Hr7!ecvAaAE_zF$&$D;0zS@d`C)bIDi|_kSS+NVvFfE-8Nfa
zUmvM7ku{-Py*A5W1&95tF1b~=&}=o%j*!5G&Xq%VXA<wrOcRlL2bF{F(VS{1P@6XE
zL-g8zO^)hI0J72m{OPmbQgQribANwgfcE=96_P}Pe=3wSSSfm12<-uKgmp(elU1R+
zggrB#^=9wBc^?kD6j(+W3>u`C+3aF<g&2!Iup=i2%Y8lJV1dJb${5O7D4Hex{^3zs
zkQqI~M0&c)pwBs9qJOXbCF|^SWZNOrV8Iw7C(a!}mlmz0AY2#@Gr$vQCo`2hC(_Wl
z0~F<)ZDv3t^YT)R0{r>6b&;gwU_={TsRbUI6+2=o{I>Ii;Yg{jK%r(;o|>1ud<3yY
ze=@f$rRx&j5onmj1fC@Tp#hFi_W<3`@jzvkQB01Ah>cdLTV?Bfb4UasG#7)=&PI^n
znNLE!01FW=+O5`J%PI^Ce#fA}@gH1W8v34U;9HGzU#5G2`F}(M8h=x^-p5K*2rBLk
z3Wx4H$~9WtvoC`h!KAHbt##6A?cqbxgRi_v!k%uUg5T21El2D0TMEwXB2kgLNX=3W
zUDE)=NWC|XlDg!hzMFnVH=TI%FZg!AfRWZu*{=}m2VDl-Sl+QV0o)9!_I?}rZ%4pH
zS7E6*cfrXVjP*z&xOfHduL3nqkGHT~tlPYbyS`&_>TKGD9k+-|o;Lz%EltpgO{@`4
zDWnC26cX=6gtFH^LC`v{PgG=ATaL+4&-rQtE}-vbVbefKeB(La%KQ6B#X_(BLskjl
zqN=}sGGsuGbzk}WllJL*?!_TWPrVGaV1y~PKb22~S+xN7a8OZ6?ikD#b#SK-$5VM7
zigBvyy}ch0Hkz^HqU^~r>$X`^!SVqYVLSfy$Cpzov)~2wjYN$Xob)1;;NB{yw60ol
z=6bv;p9~JVk>>D&r)SDv8uPz2V-8~vlFsWI%IQxsJoqZ^ATGD*p1{xuW8%N8NDd`Z
z9SKSvcQEVM2mi(OmPiW=$V4I5Zqqd|5g_gsIbO^DB38?SCW~kU#3p-Cesn{gK|Yz*
zpS^*#ZQeyy#Sb?~7a<7#f#%GfL`Spi=8@$-CV>GMX^4EP>^cW38kKhQ%|F)emr@J+
zT)Tlx#Wr#O=eO-vitT=-K}4e{f)e%HY{PgCnt(76SaJBcH;&3o%#L3E6-C=XH!zL2
zH}KPSKg@@yn+yoCVZb9oi;EpE=r~J(jb`dc)11a7^%{#bs{^xQ=Y|T40*i9_{)WL}
zcE%k=KX;(Ty0r7bU$+l2Vqwx7%cLGb_uPmTo(VI+gc$+B4=6d{C!qcm!23fi)p6RU
zt_Q$LE2DRmckIaooMFE<##mU>t2_wMe)b4#H)i+wyn5?*n{@+09pM1x<X%ZR{SxWi
z01~d*C8*&_#8#=x0<X9MzUPF(pVtFgpeLZaQMcp}h?wX$pC7ZtjUp)?{o0<_uNzJ(
z+jRq64hW{u{JQ#^qrwjvG%QWvJMGD;v5(gNpnyLH!w(X|k7|_s4S2YoNGagEN9lhY
zY0<%$sQ9`J?bNpEr;YWAS6J8H#QA6(1R2{<ZT|TQg)*&n2Wv8MpN%_vkps9VWuPx`
zFVDtL)uH_T5$Yk%-#&sG1^rzU<GlfCW4wInk9YqDhD5(YX3_`nUFmm?IPu$(Ki(8j
zmE+8P#TFB|j1fd+^j<!Vzh~zCi?MI9$>|tc_4glz340#5S0^Hel4p3IzkgsmGeLi?
zJCacZrql0Jm{bl}7kXOG_c++9(J^1@7#ka}an5W+(tfbsOKLg;Oa{K}UC|}YqixCL
zwrD647T%1X&ajr}wS3BN*tl%Z9?(`fhe>NC>>yOk)<W%?kLWEWsD0{HL!D-h%CsBg
zZ(6!U>3|`Moi<F4fK{U0(-C}lf3lE(HC9T<9B{Ndz@(+fZr&=s=n5xi07k5ah-0b;
z@WC++F(Jtx?JVEz^sYX1774O6Fho^1HR1!GNaTY)@O@u0QlW0|gVI^f{i6y8z%?z^
z9t_Bt7kp8T+l3AwuCER*D!M3HUc}yGiTD43)h4iIeb(YWWNSmgNcoc-Mm<)l!eDun
zH{;ctj4!<V51(@lS+C!hnyPHYIEsV<IQI>L#2R)L%~2<SVn)qVPOGT6q#b)?S^+q#
ztc~2Gxw0R=s&or=?vAfQEOsV0Zf#rpr^TkkaCA`t1Z5GXiD4z<&)@gkk^2(TE$nBt
zK46NLNTtU{__e?O1QXc1^T;kG^@sL!BNV~4orX*9Z3m9QROD?VMhkVNoB`i&jc@t6
z@6O||0ixC6<PR-3=>e4h-nxOr3J?m(aNL?2!@!iaV-dl<gIjcgBk2L;^h{cPkkpUO
z835gXmEhi}%CH~Jso&&{)8|O}Q#fBBuqxg(fLP+^NckeH9)@y*z&lRg>qsJ*(xW`5
zgOTxdof_vc&ev2FOk$|l3WV5-UA>L$DW|2Gm3Q}JO?s_5c`X*PQQ*oTPL&Pj%L(!2
zrnUHR@fQGr#Ps*C#mpG);nY%b5}<Wu-G2<aJnG^Rf|j{5aWr6yce0N@7-2zJ0hwqB
zF)MEK%!?s3qIhD)yRfXcw9a-=og&4zj~MT{5X(WYTM2++U~QK(|E-II#~11*^90e6
zKYbSR!5q->h~7Eic4H^eb9oy~yo+{U8EB=8UNciD<A`GBrM_e9*Bu=3Hy3}2vqW1%
z`EZvO$5OF&vH+LS6Hnx-@$vXySO?Cj6ZXZ%7mJjEZ*yf4JH1}(mx~}k0)Koi&<?jK
zb@iHFpLodGj7RkL5v`9I6kDZ^U^t!#S#QL4YoWx@Yeu+q-13-BVpTH|x&n3JBd`ne
zRAF!81pcxMVCRR?-U+49acVAi7}K&EogR&6Sh6)lYb54=KLW<>`Vt_O4}n7vim?cp
zROY9-nDigM992A923qu?$v5B+XjlUa#E)dS-()0kAOyIVmW*XVER`!w0IfzJV~#S+
z%miX1T3$d7ufSxqOkW={qzXWZX}MNyFG@;c8O&OgSG75{k_VqvL{*tu7}iv6KKzK9
zu(Bbl{Kn6PwrM*?8rmq8g^LB0#*e8f<@BoY()j(d`Zs!q1_olV8qaXlK8n}&X54?4
zm+H{V6yL&!;4#a+;5*~?t`?Egq_i9@c#9#)@<{zDoXKzi5W<4o$U{HF?mLRjO$I6r
zz%uZ%!66$2u(-n8MYb<{sTPMOFUn()Lz(1JpHF}paUhUV@gWn}#T^d_=)TGOdc)+t
z>Ae7xgNbL6JwSDmXeBVzZEvFoD2~yo6{~#~!V59rKw7mQ?vr{#?c8iV-+Z^I{0XM<
z)XO0A3qZY)`(kylo2HEdD3w<N>W*b9&GRI)bS8ij48@bpDdTyiObEh7cD5(05?URI
zff4vyI0`=Z@gaFZ-Q88pN4g{d!se)B$X7ggs{JQ*C6#!SId01dMV=x#qw@E%m3(F2
z@p|Cw@W4O7KFw$T`>;#RX$O!^e&eKbW6qwzY75Ho2ZH$K&>&2M>VyBrG!*xA0}^X_
zK<Jc3Zlul5ftct5lLQhE%E1^eBDA)7-bU&IsLGj!2gc>?*oZ;K=9Ov@lnM8BQc0#Z
zKwdf?KRb2+OWZ9Ex(z-B?cxH>?7Weiw}M3%eht(aX_vSnG%*2%c;(KAV2jW7HCD?&
zQTbEo9w3G-E_nmg&@ZAvqE7RtoQ~?YA11q^(VHIiTk`U}Uw|B8pwP88MEEh;^y^5L
zqnWma?TD?Ps*N%Tx<EaWDDcu2%F8}gXUBm{W&R;B`^@#ob*lFhm?CC12yrlES^^V;
zgwX5H`Kt?6x+3ngl_wq+sFdOdv}(>~<J(QV1M<&aZ&VZn*gg>%;yL~*zmRzh^?!~U
zKYV*CBQ1?hA)Jzp%WRMkYLA#Co&iMJ0R09A&{Z6Wv>A+bd`BocB=H(%VJk8^0zoP*
zA21qAWg+_jK!n(7y9V9TcN;Pk(p)t}K5iheqHSX)gmttUg*yjruU+)0&~H9}@3v=Y
zF;!it11M{hBt(6&#A}NkILFN3>mcEG)AwE1cg5SCt752>1#E<qAMJ&Kj*aV4Zo=KE
z$Xe0ST7bXJ-V}@@)aPfwuqdA4jUjqEbN@BFp>!d<=WBqh>2Iz8gwMaD>8F$5P->GQ
zr&0#{H%uz%XPb1psNSiKQVN;OrDd)t;;6om3?1{BEDI%Jh->Wx#wuys82JyTjYJ2=
zDosh=7+w!>$A+3!*>N>=@uy-4i$G&2_cqz0OzV@seyvFYBJO-l_urqs6x(`f#IO;N
zi_ve<GkA-PxwZ8jnC&>=UbyaYnP~qiBROVy>%%2HvcLXW$^fN(ecxm&hR0^QyEl#{
zO>Csa+gmpR?`>jXxuD4`Jr@_(qbMJ&_|s<bscPG7h5=TqJH6f|_+z(Sm!gq3S)4Tr
zmFbS00i$MS0E0Yg^dR@R&f>2KH=&nvg6jo4_xuQPA0L5iQ;dSh*#k?mheN;(=%st^
zYYvP|o^uVIYN+;6<Dz-q-^66=yuVD2(w=z#8N{`oFZQ}Di?bOC66v}3>Cb#0Y*s8V
zM57Z5qpsx#tb93Eb)`Gos23%#==MEAPC(F-Zd3<-W+&24353C|1AGdNIzl*2dI|Y<
z!?+0PKfidJoDN305JA#w!BNuor??5``IzXj)l%p+{4Y0hO6XqEDU#so3LPymIuUDj
zB7hEpunhVJ(EsU%)-S9tdGC#ln}5)RB0C{?Z+p+~N@>weOydk=9(S!%ZydQF&HLsF
zOyP)81USi~VyYkN00(@LBOS9;*~k7uGSXF7PIJ*0w@9T+o;jkoTU~f6Y&#K*e=(T4
z-S`g!7W*d$rLynl!$yzfjeqFy%;uz8d{lDZvRrM)A^S!glLDGmc@O4o7@Rid>Qrqg
z^2GnPV=_JnJGM?rQ|_|1@<Xf13}{Ns+#=Gr^@_VgBL3)izQhOT3d_p<H(1t^6JG$k
zRI7&B!1qR#XRXBdv{>l1c$IIqfbpyM&45X79~z1(nUgSZAqDD;QTsq&{f!i*E3^x<
zUHxJ;zehzuB=;3+5WxT&XqMErLl5GA>Y63NidNO&u-uEzI&5Cd=KBa{5W8JHXozfR
zxE%g)4ozojss&Il*-W;#zYu8Kz=CSBL%qOE8YUgg6-*K?1FHYpO0KOy-a`ErFP@7Z
zDzdR(%r5|YUv*8|kBKiDTF{tcb@Yj25yHFrnr@R!cO}C<WsyOBr0GcvD>~ulAcgc~
zTtLeckg67R+yiI|8!~-NMXYO}+40>#ZM`5nLRqgCc@$8lokg{dJbjQ59epR1m`xe}
ze5%mk?t$|9*dbLe<e_{Yn=_{J+tZ!pP!7``pkIA!r!p<KxI~SFU57D{!Wr5+uHFP~
zBNGpBDg@GPfesk8RBi8(uHOha9U~RiFIQEo%B!SVHurAv%9kV>d}Xrm&iqrRuOK42
z0|^)_B)q9|=$13a+j&5A@Fi&AQhec$-2lN&B}l39;Q}1o>*ig4M!P_Q{2bkCG-5HL
z$^3NWG+@e*1fCesLsR~9gVJ!S<F8P~=OH4?rCwGEA;y`#tyiN2U6d?1Y-5Bm3D@=~
z{XRe*QmtDOz82Orp}cZrh&cN%jD*jyux}mT&6Si86P*EW@kEwq6TAM|!>{Rh$cBE;
zarMDINK$uJXnz(x&5m1G0NiPuN*^O|OGR6=(gp?XF+`x)!OLx|0R>O!Ego3WlxWqw
z0eX~@#;+cr0`(*9I>~toG`JujS2z!H#3dJFC5-g%NS0TE%@L|YclR#N8D<+O$Vb>w
zSnx___m9O~e4DEUs=b#(h#Y`m=ycQF!7XB|2WHm=ZtVi!7u6HPAeAF~0|yFA5^{GQ
zEfTff0rp1{+kf?eiBiE3V3v55y}o#xIJDJlbubOtip75a=RC-Z*)&O^iXUwQV&jA^
zDsMqtqeYBR-Zhm5f0x`^vNEw}it36FHhI)D`QHO)pB1sV)oUbYN*K!h#C@w~y!S<K
zveJ`E;_m00wKmFSwz@s9J~6LR>m`@K`V-&fogM0k(W*czh+vXqY9?D#HL_sZ)iz}D
z?<ua!6>s~UUV92%c0o`DRzjMmmYVP3V}ERW!WzZ8$(~%^d5_7)H(cWY?K9=SwyE(B
zh;^#n)EIG1NtPmt$<NXp#nwXvW<VaANg%oggC6Aef8LCV-0B%2hy6+DgRE(b$SpcP
zFZIZO?RMKA@VjihgvbFROyZOG__5_4Ag&uJ(SBHb4ia5aM>67Ht1WlHN<9Gr%_S~_
zF;NmE+06H(qQ{Cgtnhp}7x0w4fT~-L$EzL7`z@M-k-SHo#%qeMK)1Oqgb#!~%YZ&d
zoqGr;v<2j``4)Vxw|#Z<ysvP7{T3m7qK^YsZn_C)dy_<(m{_wq4j5c*&S=%oK%t1n
zyX;T^b{2%`3t>GH#lq`DVTCWDbO^pZ7dxWzJd92Ssj`YU@%1gb&}XZ0`LzTk%DEwc
zT3E!CBE#$0;{AGO3i4KMavDd52d5fBHHL*EKpJFg#}|i$&nxURft9jcRqkMaeKOv1
zP6zCMItDiiB;wXXkP)m^zC#7o#pw8`X(0G`0vVI5BDJE?_-l%bEwL4~xn_fNNo*e@
z$?!R)P(;iHlW)TFNWj)sE10YCW??|=(--&aSycRgkma8&7o+^|@`FmMjmaGF{B%-2
zZOMn44d_-+*>mZxF7t=~xXeJuC*`@4&ilmN8y#^XyFi%k%1r#gyG&#y{Wpm$J$Xq}
zQ`26+tCj1AUjd0DsyD<n7M(GIHDQtSJf(CyG;HXXu)0Qhdf@K>-o7&c!Sl`s9PlGI
zq!Jc;nQKuqXgEfuxzA>tR@3faQhz7_Vmxnv+I(QiiIi<9F?$r?Uhf?I<d56XL1wNI
z_C2#4NVc<3SK9MK$9)xP;{ADV#6RZb<aCU^Hqd%O#G+2vI1{mqOl)QFmX<d8-g6LB
zAu!{E1nHMp1n`N&22HE`;jQl|_#BLXaJFKQWH8I}tcfg6)!MKig3op|U64Wr@ll(2
zs9V4q1`QcY1v@>6mX|g7fUJ4R14Qr)^gy)7_hTO4<P%S@JE+RYa2%1YkNmcX)A4QU
zy{;v^zgh}gAcMyy`PceqhBXq)i;o*Gggp-JSxS$v0*xSk^1??;Xe`p|H0l50xTAce
zAdSh_e_G0SW}Dt82S!?%9yhX_Ui0N`4hzShXjV_tWV8$x$AsEOGeCao_deO79sZ<D
z>m7<jl)WMuD%C&aNXOrO#oaSP@Yn6kxrVr%xuPWHA%DR^S_3>pI>NZJceAG(kSmA|
z@-AXE9a6!cg*6%h4HCfgXSa6j4cHrz`$&;Ubd1Vw22wf~u<A8VSZ43Gv-Z9`o-Qjv
zIoW8}YtEJYgoj-5J&XoM3JS-KfVyd)b?*q+@}&R*`Jf*MBX<od^89;86fu`>erN;h
zLcneri4Rw;2;dgu+)%918q1W3nb2HWS^^%=9fm=~Y(5-5JLE-)3ui25DZ0&YK`)ow
zTclR}Z8&B_kQ$bhyFoZOKcE$A;e!W<FlldPP^v-w@7wmG<v@>5lr28jhWD^<;AL@o
z+(`WnBFtby__ryF4^$?^v@qXR6EM<0NK=iA=?8;;2f5B@X%NvTQ+>X`Tp3CLiMDlP
zR0_u^m}Zlt+Q+OMkkef3l=HiX_G9fYH}?CJOx`~r{&{VjE65<k-u6}1yQS$6i%dsH
z1H13U!gOg}RqX!Fx^}rcGK+}RpHcD%rT<AM_=i4B`dhL}ks7gKL})&E|N5QO!GFx@
zp79bW;@P2_BfpE4Ak8fYx?ApY?9GQpg-Y44ttTrb01`JFZeJQ53MH04+23MoF=rTl
z#r-Ap*JD4Fn8%qO&(1n1r{$o}z9uaKTum!Q;S}3b4UI4)wo*km{aTt)>4MzH5M1xm
zkt~zk(sagMq!DQ`+#VZU62QG2+d=l(Ymut1ypvXuFZ1HSWJ}F2-E!kLAALS%6*RlZ
z{BT<p?l~h)XD%lSec^qN$OC!Mq9PBFgd!L*31E@bJF8cg{^MyH33x0NO&B`W1#o1c
zjPrUCc<@B8hh2W{5@TbDQ}3Sps}>R3-eD3EQ6S^7rUmXHhNbk~qt2$oEIKC6Wl*YA
z2bhunMbBS0?N2aINdCKw$5i{86H$r9r;F;vyemr>%M82MMyAsBm3YF%De~Ul@?Va#
zl^oftQ7V6DH+y>9@bh9YOzob<n`QGk90plWD9)b?=Sxe!aDbXkD>+p3yT%EAr6>fc
z&a6tDzPK?x45n36WIa(XR#b48K0R!@+9KwG`6tmq{!LbdbY5(okm?eG%?;q(cJWBJ
zQDR+wM=W}7*>ed@XV8X0%#9szyF3ej;0)+*Rsz|!@=TB}*GA|<MP&e6RAZ4{>dn}C
zQiV{BgbTrD)IfMdBoD1}Kd=?^=B0q#_GY{z#i~(ft4sH0;uhM?TB=VU@Y~3sRl&=k
zLa4Y6G0O4=Y=w9%n$c%Uwt&wR!8J9-e6yX!)wS0_9{HwvtbrU;f8pz;mE|XK%xWkc
z+t1!3dRhkRW{kouKc9otvUH2r8O+$=;$#~a9srUw0xzY*leu9o!}9Wc%Y;Xc$@tLq
zoIFT0N^gi<*UfTALi@E{-J<P}vS<NLh%`N@z7lYMbpbEc19%S~o!2LNm{gdW`F_oM
zIPWGqxHYezk3A+2QO^V%%$Xt@haM>Vt{7vb8fALXGmp80HopPE*jNOYuQvP$uoz=j
zcN1#lg^B-rh`k{2+Ziqjv|8h$$0{KgHr}huX@D_;?H~V&f3^Kf|HJkNJxp{NlD_Pp
zxH6q6_&-kkm3*(=^8FXkc?0U#pl}<b)^VTo$ot~Jo6~-w_0#du0`m#GZhg0!hAthj
zLhnD~%QQc*n|l<`q5oax5Qy7FaSig`rIDY1q~#dg{lvA^a?lS2lg5vl+1KC>3kwSk
z7A2^^B9xyt?=>WvE%(OZeN8JMWmK@K=)}MI%801e698TJR*Vqc<;+Sp#um_iAwi@s
z8U*<glET$SM$r{p9O2D&BCAzvy&hlv$q=`+5h(kN6e!(g!KwBHVm3Qlb@N(YC~4-Q
z+*PIm+#LLxV)1%;@@YlyU&p<j5_&!&u8Xr6c^~p2MvUMGj*ghi4J9qsmmjeoc5@90
z65ySJgWL-$GXyIwpdwtTUZa)CR@?St5~x6VKy^+!gbrrz;t2lmnU&2TF|N1~ofSGF
zKIgS*L+bKH6rrPB3w`x2PX@P^+jfT65RY)j$6wK6BuCnz|9%%laP$H&-Nu0;Uk@lI
zvx@w~TVDH(Z?W88L7ucyL6lt$jinEKNWkuzm7(x}y7AMqf0fsNy!N5m9xgmRUE6n`
z_~r5+{S;SLLlgrSv9}nGNX1Y7b(V;KU+s`*oIAfhD<ptg+9xeRJLYhd6aTTcb4hIS
z8;!A0iKLk5%gmkgmSt9lAn@@qoW|WziZ1p}=V>qJm;f8POLq&J*;tVpK@fg5SMkLd
zihNo9j*Kg)pCKDYTp+#xh;0w4H9nBW%=`GIt|PRD4qNJ1LBbS`f|N;g*E<8@@||5U
zmPbDSF0wp+Bwryz(IZa3IT_o->l^evkdSzE9#MrLWW*%+MiErxaXKue^h$1j2@%n<
zb`&`Hjg5csK2sBHBdfNX8nq}AjD5}d__WS;ntm2=)_9$4J)}_o<gEqea(8A-E<rk&
zb1xG-T=0K;Z(0+S*y2z!bCm%KFW-qi)uU5=?$t@1j_4*Rf`u4X<vHf|9X@&vKyhKK
zW%#n+9yy;w&W!iPS^PtJ>YD<Og4^#MS@98hZ{&YoQcXdjV0oG*dV3!Tk4Q^#-S9uU
zJGf6FKwG8G=Kd>aS0;|Rryd|m_3&iFvlgIB#Rt@|Ojg2@+)g!}l(#O*@G)jV-2>0H
zV-e}jh|%6rFD#C9_~NLDf&KjwzAK2JzV?JX|Cisq42F8qkXE<3+^eL*ZW*>X3LOB9
zRfCIYaLK{CblD?OO+c%q-u((AZ~S|}>k=uyyed^U|1q!YPg~LU{huMRf0Z0E!*IL1
zyQ4sl#-o~AUaGJV<+4ZIuY>>f4AdRyS9Jij+cDA0eF+JBmcE|A<$Yw_o~~0UKhztN
z)GXeVK-73$kJs}*ewA$S>A1<-UPm`J3^bqbmNOcm8{G~Dbg9nozg70dMu_#WTR(cO
zZP%n0N_2ZTZY*j~=1@`JbvhPh!?NcM-AJKIeu28x9N5&TG2A#-eRQGN_Qwl2N@9ZF
zfjfx-HQ`}mZpHHgB?Mc#ZPiiN%j0iVL%qP5QsMD22uR%&`lbl#_!i>O$8>If-oOv2
zLS0VoydwprTye67K;WkLQ_8>M!&^h-0k5;%ec$Zx3jpjsc1V1=|FM)NfV*~od$vI%
zf`5aNZK<4r8C35Z0o9QMNWla)+AQn4e0jA4svd!Dqmy+nn&Y0;zh;Fc8r_R9^hGh@
zo0VorqvbMVHVnbAj{pQnCw-tYOb`zlFQ+CL_>epbl$`PK9yqQket=@WSG4#Eg^AUJ
zMw4!ngrXrW`_q(0>hvmDZ$KpxR7$43D5?ji3kXT|<|gP_9$=&vpDLZt3Axj8aj8^R
zRoz|u^e2D+2h}~r|6zHvy_heFa}wS90s3K-C^JVG>-u-bn}*>QkCm=Wi|7BXX%S_@
zWz^841&v*!g&FKMmif|q{SOM0lGxR(a-`!dEtG5Al2w&A&6(BZF;5F?6o{Df3PqwF
zmOqbDMQoJmBpb)72&9-Ayi|z>s$`z&d4tcxxn<vq=s-$`ji=r6l&M>3erH%R?IGpw
zc~garzNGVicz@KVl2W}|+#;B3*f%ghqO-aJDShFvLnl?T_3^lQ=9|ZnSxZ5S$7G9w
zv3ERwQ437S^3H;7v{1vS(%IHTJVQh6a^F7NQKiYVbnEHZ{9UwBzl*CPS*#keZH#*G
zQ+1+o<E2oJW=0L2=kGTtpNcRAlAFX*`M%$fM91X8{YtomQ2o+;XaBaBJ~5x|!)T$3
z3Gg|UfXX_h3{4!htZ^qq^>t=f^!M(&k+A^Jyzc1gSn7#E6g1;SY|ni0cjF`?-}4WF
zxNhwSXg|eZ{W;_ESuQ@8jOFbcF&+0UryZd77TBmWh`SBadJ#d<*#SSZ&v&WB$nQJ~
z(qQ}qI#-Use;^<69hPaRtFSk((fB5Iy?YktL=q&L{J_Gf@z-!Q74jXlS2L-^@P#i@
z0x`-<Wg-``@jzIJPL3BuFs79NP%BEm{p#ns%u+FC%l4zUM+xT{$~#c|rS|RNVPTQg
zE<vgSL7^-V09>XtiN4`SPSGV^-P@Cp9r>LGK_tEGPXIMXWGkvOq>Um#GH~P>4h{Pv
z_27qz0`-!josion1*ieC&X*N0z$i%>0!k9k7dzLR_3CWXM4zHq@sa4*`wC%h-gcTU
zWQC71w;EyNBS`x=pLK>AXnkgOT4$ufQ!mwF0{ukg7>^p;({xpHWh;qDy4m#6Uq-N1
z+cW$myN~kkwJy<0g5L+-c(vmgfZ28_>Npa~&hbavMnw7LouifI$mF2Ll}9P~SC5>_
z<JQQ7HB~P!L^ae+@J~93zSiE3Jth|g5pjz8LN+K9*-NI)o>bM6j#R6?eqbGc2b&BP
z!X3dafwIzp_E!v=t8mQ<r_Q=NX0Le1cwk3I^QTlPtNt4nS(26BJSWpTR_o(2+*g4v
zqu$>R9XdlQb%yu;dpHtwbG9Ugn%qg{#0btB7)SilHl8jbs8PJucA5rM{GtIEmzk=z
zxGPkD5Op1g|DN&k$M_YXN|{h=GcC!6rT*CKY)_*mI$%nojaOUCrR324mGyjuZYWEU
zx|$y<g)I$A<i@8wo0Nih!>6_TcVQ!fKMx6ps|h}w#^Jua@M44ca;JtlkRtjgZ6vyw
z7&(sylenZbrPj%!o+WN1iFSCu!hgsLN)O45#T)~1igb`Zag5vEhv%s4QzgatSb-u|
z)yhUmC*K<sLhmBKz_M`o^6I)84focqHfr}++-3g*5&LR3YWdf3v7kU7Kys^V3<z}K
zSMuicabi~!!HRq<a0mD!tI*hBN!-&)T?VW%HPVE8Q!Y0mFiBWrBhvUK(WLI_-IL_H
zqlzxypBQTppHDbduziEhtL;{azP_eHB~8FMDspXcX?^c#FMpfg=b=z9__KBy%d|wn
zSgF{Z>;4_RG8x|{npHsS4Aw!ILtLpd6r`uZi(XtW9#pS@OH+UGj)h{ub1)VSh%iHG
zS@U;!P#}>T_5(HJ-IyGW1`><t|8ejp=@-Us2Z1&$lZJ;AZumcNu6%HlA_!DI=ZRlg
z*rWgH)FB#s(-DomkCB~lWD-gl;u7SAdSy>am#ZbnaH`DI|2k*VPp|o<yqaI>ZY-->
zo*t?+`8@TbqfA=IN&TvQmRjszhl`eY0>~3--4y)MYS4!qIy*4((&g)^B^`zjFSXvw
zc04I_Jjqr|ImU-TRyg_y<<Zl_nsL2NOX<!^%4~vw5gPrc1t_UK10%Qr>r{X)k_AC)
zAFQ)b0ooXPX$BQ~@nM`7mNLQF9`_KPxY;W>m^8qFR_$Po@v5$t%WUdcSB$QUOrFHK
z<W~`s^I@*TOHvSR9_qxku@MEnxbE-xL;#Z|XiOA--p+Bj1Eg5tS=8+hT$8e;{|{Sl
z9aiPmtq%**-6h=;QX(lSf|N9il<p9a?r!N2X#|Nyi+~{A4N7;nl+y9dWxVhCo$vg!
zFZaH-%39BS#vJ3Wkx>G@X(=878Fx{d1I9f@(d@+D#ZZjf1om}SwGqqJ{X5l{Pt|ou
zBQ&TH7-F8@x@D)>b|S9K2nVbi3oj82B{qBfViEW`(ryCaZ2x=RcVJyh6$Yg+|JH+^
zZ^_l2Y;SU(J(cB8%6y9#kdM&6SSg8Wy;1=QY|C`tAgx~xc@VIkCTVg}@iXv9A5v>G
z{b~%V<&-h^sh?aS6SR=uRu^?o{zqpGdp=5^|6wT8KQKRMB3w(h9H7&we*L&Y2piC3
zob{oie@_qn131xzyqnwW)E@0%lLz*0k`ZUoFwo?oV)pa>#{%1X5o4Q;Xk(xAY^CQ3
z?1@WW{%hiu9>dFRZ1#M#zopCi+VIN7ZavR;)Pno>0uXv)xN;%~f=!b(@ZB%~mec9Y
z>z&Vu842t_i5Cm>EYD-`+t1Rh%i_$lyfwJTcmj6sZ2Px=73~YkQUcs8dk%y>DoaS{
z7fue4@~l_1W23w8@R*uPg=2l%YhmdUL}d{^HvKW@4tCadTcPv3#n9z|X%R@D?3Ia;
zezuVer|k#gz2A~uf=&k)Fb?zuq2ebx#Aid`mvP-^p9YvBd<FU&#u$;yLb}NoQ%zG$
z^PzP6?^1O*?O)!e$-2)(IN@h^$l6^S+HNidzAx_IS_hl{G~Wsz%iz!dvLK$=68Sui
zOZzaaQt8O;@(vL4@qps5Pv-OZX;HtPtP|j16a0>QPhUgP|M(=)U@MAeDD2^!_)idz
z+#k;%_W^X|8dkgXv<H63BGq6AgHv|&XMQ1<y)YdJJ_)`PG&eWn8k}W#YiTKm)$W)S
zAXq43vmqS+=x@!IX7Rl+&IYjR%$=M->HQC$f&IcS=AQsKW~AyNy;1?{xwqkmSAw<F
zzfrU1LnEQTEg7{I4CLHD+mr^viuG^p9<gGiblFetlJZ_Mb9~fOQ~q-Lh-1BfmZ&F@
zi^%5DzrV&$$bWCo>mAQi9g&OXo8xxkX)8X*vvZ6;5VwpEygvI1^H6UOOhD5F2I*eS
z-MO6YiWhi4wa<${{*r1^O?(?aBsZ1YG(xRJdvHvxqpv$LE2%~k$b6OkD;Oy?>m5zm
zmq?>Jul$kt`&2OdH8Gv8$+N(6J6)G)(<;CsyG5Wzs#Y3F*}mA!jeHdExL)Ca)Te};
zay;w$ns_4nUYPasrvkD&-^&8~4Khzv^<=UOTZ*m$_T#u3&DRGw5%MsSbzkF3M%mbP
z({rJ##n-IRPBMbm6I$lIU~hFP6k7NKFEp@*EX>qEnTayAu3Bv|A90pSChyVrX6<+H
zj$v8rK#^zOk7gKma|gO1D><nK*MrZU(LR?T?z9&ukXQ&=+sG7KSQI`T@884>6Fd_E
zi|t}Eiw4iY(!2yfg0c2L2`x5*#Ag@Y_)hAzuYtWO@wE$H{Z9(+?=R*+S30aKk>JZf
zwtp{>ctEZ3O7EzE4!R!PYPP%z*Po;!iH`Tv?ep!veo@A!^G%feA8^W_8Xe-N=|rtR
z*qiEc1nNWV48nC2CwGk}Q`KoD0&J3}mHUAG@2QF@|MxINaId@rv^BLMe1)RZM7Av0
z@kp4j-`*2Uhicw~@Spl`8qw0l>wY|1a<%>y^~$2NK>nBT1CWBh#P|CM{OFZ|6tuvf
zS0r6uvb=iq<&f6>=GP?pJg-99V_X!aM{2&IpR9>-f`e|TddMc8z!G6Ax++0~76hA1
zPTmMS_4qlm0?)w9nTC^*Qh)nGOPaKd=wvvRi*A`a9d+ah<jS8n4W<SMi#eHjttT*8
zv5D&i>c=WorD2yqhSoFP$wFmG9T|V-JfFbJVoW83;9mAOX@G9%wJ!f!fG7hNedAtC
zJq19{!)6XWmTg&i+$Dp%MapkVXy?JQgDH7$1+{@{gUNF(XsEny1<l>@J9AONjJZ{%
zPdie*%0)wXKzi^eRbz{zl};Laog`MZ!t5uHA98?ODTzihRAzr*Mnm{n|3dRCpW9!W
z?y_1ra1GDM+4?cSN@64|a%RB9VKdYuVaMS-<9I1@@d2jJm8GX>mL-GdLXlS#y?J9f
z0ev^N<rj}Tz1EK{dP^VkF!C7yM9Ti^a(BfjY}Pr{z2^WDtTd0t{~-VJK2%~T`Q7mz
zu+FjR)^QT(_Edw9A<M|>uje#gCWd+upOHZa%W;P7tw42PF(FKi`Y+}ND-G1a|2R&x
z9AkAb2I-=}N{HMyRl%NzFt#ghObO!t7rB*@fWf<0Hy2=oH2pPor7ZoDH2kjz`TdA|
zYVkhzS!MJ4AlDNA<7fsmRdBGr(y4fNJts0<A$Z&QE|??k;h$lvk$!2M7OPqQIi=6}
zGJKpgxZUyrSPSa|*`N7UAWPt);79CG%lJ{l_N-_dBxGcimX3)X`ZZ_t6-zq^q6isu
zBH8*^&@wT79O*P=L1)8}UBV8a2#bN?FgLYe9iqghjy9^H0^$w4Vc*VpiD_#9B249N
zRD{*#Z&>TRt1;c=-2fKDK0_6(iqu^Iq4)IQwwmz8IRWMNMWi>i&2)K(kd!lyrZ`l+
zRGU{^w4DXl&3&IyF}<*qUI5WqshRE4J+3TJ({zPrw8mN!hUVCP12_~j&~~1%cy37k
zyFdRZmzL=M0jhMLY|u~>E#ZA$k!+t01&?oAe*4%Dlj>C$0l7pNX%{e5{h_D)SAZkv
zpl+qq)TsW?SKrZf-x$o1^fK8R8IS8tEKCJXsv=aH;1PLISB&5M=l8**nA|0BP|446
zlncLOn5{J^xBo+T`m0gHzIsu`=LHgXm2c^nG~vOQ54xcX2rtLuP}5pX9w5wu1xUYo
zKkV{fi2nD@L(lck++(B$1*??A;D1*dID&_SNkrBv#SIXx-#>vrw|q;<qC;=c$iZ+7
z!36y`3u8T*7HNdOBw~SsG+D>((xiOH@ev~gD&9DSe-17@8CrNP&tucELN@o43-5T<
zpO}*gg{Dw6SmioT@Y9Uszvd_19;P%ek)akqZRt}4JF3^i=-$Qx+!i&6#~F)XY;4ae
z&0w6*W{z2bjE{&-HybEd|1o`kezN#~y)(nSPck^IO2V*@P~}#bq+g~e9$gd;===n#
z{|b*6a1K|W&iwuuZak;&(80YME!O>G-E?bJ{exu*#{2TKk5{J~{RPhMRklt|VAcnt
z|K@J<ywEYdbxhB~{6&2J%5^oztJ4Gg>8vD<X{-JFE}w@r(CwVQbMa|XjJszFZftDG
z>FY0%cYCF8BDyq9PV%e!UchneZh7~o0%k^x7C}+q!lc!xFy0Xy?4)fT^JFy#won9N
zrdXoy4Lf;)-CR$fD#?3zq<tuI(*0O6%YVzA`0BZUvzqVv;YYTSBu#j2Cn!Wffkgwa
zNa>+MF*6rCLQPT$xRu{M8n@2hZXWout<qqZMm|EV$D;mFzjZ$%zy5fQ!qeoIPaw0}
zIONY&uz-7Lg&Ru6ME5msARQDCG+IvY%QxF#co!d4b87;Cr3@*chRGdB0i^stWGmp;
zjZA+>mWA3_0Wc0HBEdBSR0y-6Q`wA7Jo;gOiD%Gn+ra;%W&d<6jigrt;Lj7oE&M<<
zq3`+1^kQ;J)+4S=Q|U7peHqXl*J84q5F<@BZ<1JdHP?HGY{i2kH4(X}0Be;<uT-x8
zRG`Lw5wGNm35fci*Qj#AgzL*F-+SZ}kfOikJ49kw&In3sQ)JTY3nnMb{0i8~Y%_h3
zj7B?k)_ssrNmp%>g*P$w=S`ref_1y+4&2-q9+AAXaCdls3=`&RlT{E2+Eh*xG9Mxr
zIO@PRT@<NtLjV71V-w;3k!pwtZ1G@`H3_WVns8CGj<TWSeNXL!F=`jEEBtT8!2H{>
z%NR!U8E4hUioIhR`79|=@2abA`5n5YVddy_yxu)6a6I_ackSc!bk&BvsbI)&TvZn(
zF>#zsyToA|0Vz%l7xl3)zuwzDb^W(b^8?7oOVX=i@oiri8wEr4Gg+~9YoyWN;LG(+
z{rQVK6<~frpY$j;yVC*KGL~m8DAp1p%jP(9Y_C4c19dI!dp>(C>@~uFO~t>yG?Dt>
zCu}XKhOHYmqHhU~J8FNLqsalZp_bRE5jZ^h;CZ+GU(b7RQ`)}kxx-pOl8aj{j5Jc}
zLl`2{Rkk~KC4Bg*$pYOV<xyYUOQt)ztmd;An^}1-g@>-Jkxc%Hv`Q)())u9V`5}m`
z@7oZ#v1L;5hs&tFhs(3sW)fLd<f^Jq)~9hj*S{WJMJ8h7|3Tv1fPDS;=lq%c*a=jM
z*L3#`2YbJIhC;tjK_}MMH+psKn;*7^%(DYgifey};eR@!zrRr0`>Qx8n0hb(501HP
z&^>?v#njUW@rI*W$?t#tBAt5u8|w5AKp}tK0ymZ`g-(IFs0TVk4(1nqA?79~GGJI_
zU}vv*atcVUxTl_{A8b4!NAV0x-JXxLsN`HQ7*xoDdBaDHbn%#x;MZBh<}TFKx4rE;
zB{rMv?VIF%otnLV?)YeZVXhsy;G}^)5`tRTqhBVW^B<!bxeUV)SdEw6anB1-D^_vC
z<;ditq^gpxx_g-6Qla*zD!zs_a)+xXCj1Ze!AU1z@(1@A1PD`@fb}3gf{^w16$;3g
zsyEvzz?B?z4TzLZE#VW!|6O0}{>`Ivgyzq{0r<cOI{>aP9nh@NSPQvo&VV59_>bBO
zjrI@-kNV=IDU0$9pQ^WqV4};$S1q@PViRAT?E&>MyM-tPYKLTth)enrU>=r#%7$Bu
zh%(yqDMJz(z1!20;NW?!KlVzNtEJF=A+RI9pwKy4+jqDc|8aC>viXE8i{-v#hyuO8
z5tGs!?06JRHN+yu`my?xy|Z+jSdtyPD~?d9cKKpG2l0<=8jf2|*^8&px!A*v#Jz#_
zDcK)T+-kzwV-n&R#*sn2{a-Z=_OH~=|5~V>SYrk7o90kn8-93@aagf}GeMxPWNfts
z1Sf0V#*TP?ZaF|5p|MVFbq<==(%k?>16yNog{d_AcvJj>%>+PpuIuPjkLx17M?(4H
z-YU$|BiM4#mZqHdycg_rVMigDfH1q^A;ZeT#Wjg^cZv!Du+N|hM>6<>TDIQnS}d1m
zMvB@F^9dJ3O*5%>{lHKE2CH3uDF_k$R+y#ZZqsrDkq8mbvt?@@lfJfPAbqXJ(CVIR
z_v=`f)MMz8Rx=C7=!d93v;Pv!-*Pu5IWQ3Me|`QIrhl3nCdf1f9`gx89VhoK%~F0v
z7c00;zy7&RdtVp;V;jtlrj-<!9|$y{JzpMeeE_Z9G1z*jHM#-J@i0wS*^LYQ6R?ix
z+hE_&Q1Zf8&#Th#{259#Ym3SPHDm)c!XLlii3Hbsz{w3e0T-!at3;&SaNe`2|FQtI
z!>x9B<!wHmd*$c6yWQ|172=lrY`Y?%L4}PA?B23TVh&oRIW=i2`x-=w@XDwr6ACHT
z2PJIYNNQ1JKQ`(e6f!ZCSJKpd9nL8D6fs~8g#XI#b`|tYL`<$53jfOUpCbL|pRs9w
zzXwL%hgB1xJSf8Jl5ErCUCp2yMo$Kisp<eE7`P@A5&$}X9~OV1u?|{aWPb>w{@GzW
zc7KIoJ9G<n^v^HGy@6HL5%5^M8~cmRuUOQJ*Cq>+WgD2GITZzJNcw0*JAc0h>VIED
z;=O0Q!(M87RwcpEaQ;1vyDbGzlTlZJaC&21*c;AF09Rf*F`})I^Tj`0Cc6XW(}A|)
z53es)ZOK1Bv*a8^e^k{~HKOPYfSu_U@xNv9P4>#4m9qcJ`+x8U06xOz%?dg2|6hyy
zq0_A%bmyX2N63K#fAH_X7wUig0_&w&G%#$0-<7-6i)>{Fnt*Q8LH!(7>VN{_d&Dd-
z0c%y6ggfuo;-gtd)Ji{ueM&d!RA0xl)u(C0F}*?}0I6JPG92`h$rfLI<9?b)hN@HS
zZhtL8>gk)j*RR58{aad;jez22TN5&2C7b;Xf`Cd8Czju5>9~KBv<__z-{-#H&&!wI
zq&iWa%lnnq1ug-01&FmFG-`J&clY)L8FIt@@5}5r$a3(vo<$w_SNqc|JqNG1$u)%@
ziJ^i6`hN<%ygs~3C5n|h*ay-l$MIFoAQ;?_k8mVNYTb@jIOC#^&c2NfMG#px9ssu+
zGxW$u?Se=8_dA^0Xq?oRwzmIZV!#~1YC<ugGHXJ{g6aHe!}0p)`?H=Ds+(0FG8-)L
zv$EhE_Q77@4-)p(Z|QMpM^1jCIi^(UtJVnmuykx1XdTRm2$^<kMEI@JbwOY^)Dq^u
z*Y`mitcCi2Te9De`sKgZ_odA=5gv2@@7H(X<uiz?3OKZ$e-CX+3s^VDh|efr!Ez4j
zaxcsXt=BCG1HVjuZjEYib`1cng0bG!QBYgj%Dy`6Lq1%<-N>4|af{hFF+P6#5+F~t
z_7EN_T;^BeCO;c^*TNpHuh@<Ei8`&9p}Y3)nASUdGz1q09NuBPpC+tYclGNV>eDN6
zBarb9$4cLo1H$PYuIukYO=EtLXtU$LYStA7`<zN@|E@3PY3`^0=VQG4hn3x-R7=6^
zBe2JKLm5+1@Bxvx;1s5I@DXO@@CQ33`vKDMLASm|un7uf14QuKrLB#o^A&UfU^Fjf
z&Jc8tzc|^{nY3f-CU*i0<))Nju@JWF1rot%fb8NH&xAQj0cSDCUT;lu`~!^VLuUXf
zN+RR6LV41nSMQiFpCzIPR3a?(^@!!7!oa?~9xI83%DM}-IZe2?^Dr?<>ztuRF4d)i
zES9y+?pjiFph9zA0NR8)m~UPMpk^?We2scGOfq3Mkh<LS^b*MIqSv9-LlKmPUaiXj
z0h?Tgsb5+Vi7jZ<0crLD2#>z#m1veH15Qm6$lX@zAf0S%t$vWUG+>eMY4N|G4*-)f
zu<HEj6U|cX{O*3>vMkw6H&6z;1!owJZicyG0n4{|d;q-4s<8)vQ^Xt8=suXKDOo!4
z<5~bo2e32+>S6G9Yx&$>m{XX4FI46RL^G>zR4tV+2Ju89$1D9uV2OF_=6x)0Zrhp1
ztkJW*)8<+1(PX?;xs5lcvzJZ=?uYdGr~t7X-0WVXhzIkd{bGfg2rZ|P{O`V}PUrD~
z@3iA^_I#DKrx$jbgZaSkV-QR2pQ{Zk?Fjxnf~fSk{yQ~eYptl`A5_nd9VV?V-(Rig
z?4ET{Wa0h+AtIjtYjer83jTK^^SiG(r2E^>9?=<p4oB#tg?khg)jT;L#GB9XpI(Nm
z9u;P+R}|c)FNv>!fI3sDYi3pe5-3LTR>cVJ8%l^{q2IuayugU#9DNBez8;1YU0Sf<
z@lCWjcF>a1Kq^;RE?!3{^(wF}CV<XsDpPoc+$anvHlLp}t-w{Ph|^Nq9ACjC2KAen
zyJ^3G&S?#FV8j&;PFvDH&DqYtSJ>Ojl*NW~4BPjPyF%mn1$1p_O_8#F0>Qk4@A*<H
zjl27@#RH8r;BEwfoYwu<xTbNJ1CoK4G4!#`)Cj4GUfw*+E*D29-Nh~={z^jFX~y!S
z)&>7dU<<cqPp;ocvqmGBAa|W}fW;_iYe98t6(dcLkB`}aJ`yM#;(<o-1@KnB;q1`E
zQSc|s5QPDazn|gzf6tKS^~6#H_gdvSm0)w`Wz3xKy4{b{9|^bSJTlbds08D`>)R!Y
z!A-v)9Av)XuG4BP@0Edcteaa@lT%D1pO3GwGU=a2EIaI9?WWB&b^P7N{;ysXj_R)x
zfs~5YEI}q)9&f3w=e*%*T1CO9|MAlE!Lc2O`nA@j=zH{zll2D^>obGwowV>r6-^Gn
zZm%^tsyHNkBJYow0?zw$^tXpP;JTt79LtxZYfGp6nC=+Sc+~X(zV)RrR$CB_WJnLN
z8ZV}f9%%)2#^%-q1o6V~6|ihnN(UvPHXUd$l+hiGI%A|EPB5+lkf~H=qx9N7`GA7t
zF@FoJ_oCY4ZP(P~{t&PTJPZ58RX<yl6Up#~Q6G5}0^D$j8eVg{U}jhcpiLZT(Ho?`
z8G`p4?-OR0&}uH&xFnSW82_MO_&czz>F-RGV=5pBkwRVK>EFN(C#T*#?EgHM)PK)~
z8a7tC+gUmP8VQflZv9d1dZ*mbc#u4o9<a)d1goft1paNmU#|Eanq*1%PC4ql^q#}b
z;@%hL-ArY@LqjRMSNrh{=;vapjZfR>U~iaM$DiqH?fn0NVE+4*nEt+94ihnxppF;4
z_}xhGfq)1tg#Q+0KL7jJ!b<4K6|q!y;Hz{T5l2QvKp&k0(#N&NlcDMWu-%an?Cvae
z-WP6V{EX8#70iKCJMf<GB@mty7b$@_1YnjM1OHdW2jFcTzctK!iw;>o2PM+kdtV%8
z#0HjYw6{I5DqVwYEyoQL^n$%$q63S%z!yxFEJeBM3j8haRlS3@Xo%>ZS%6(u@{zpA
z<gfkARKOnG3gKG2d6wqe%M!By5sSGx2jJs`6<AKggr^c9A7)+j?ji<M!uDekknvFJ
z1c6Rho5v8bYh`jN%Bn(Ep&$vR!303du*weHnR-E6s5h@y!kPMjf+GDz=`KvDR9z|j
z9kq+*(gWNDUpQjGol^wd6i*|ty3eCb#Zj}~!*V>R%N)LdP!KBz`D(L4Fb5C0fk2wD
zldZg%KA@trlnQmG3oDr;2kVYsw<2mJ!o<KjW0k^g@y7_)e{O0lxj#2`H35@Skz-oa
zT~)|qt7KeFIfJ5&h5nm|#SRUXors}v|7j*<(_s-<P;Qzs_oel7_DL9^5a{_M(OYYj
ziLx}xUE+7UtV9pHqF-MPnl)1q6jjN~!~Sch%sq}`FkrH{p*8@k@iYjU;7aa-CWPP9
zPdC<l#-`E|x!}Az(do7PUjEmk(l6i3^mND`TV%gVUOfp=P(F7LpMPL!Y3YHi?$vF8
z+YaQvKS^ervoy4gRgGn|82qFR&qc~~YRmMW1s|+}>Qw)zj->=TV#lW4cE&}NxDNzY
zqRj@%_%J}|=%1Kl#hyzJ)-j`k2A)*Z6a~0RAlI|@$0gdFSPlwR{GcW39iyDcGk^z<
z>KXvg&qufc!KFnBW3u!iLbL1PRTl~$XBJE)I=LtW#H@G3W()`uqSRP06BOhW2Sbb?
zfIBMiS2TF_7j>VLXSCf%VW0$i9^JtitFdUK=Lh1)^-khf-%kp<4q&+jpv7v}C2|86
z3hZ?tp{!joH8BZFGDPfffbV~pqU#t=h@W)_i$ur-nikGvaLptE!4}$van${0Fj~pE
zxO_fxe0mGiX@VlO;x=!9?-$ZWfD+00GLsFRh;3xTpK}wov)q5=zC_~OobF`fSBy`S
z-`3D^M-=$x>0y0eUH028*C^rvO;Y4hoJ)Z;>#+~vL>p?Q+T&;;>m(>j)Kgqo%`dY&
zc8X9U6<p_?Q(&Y6qxO`;;ZiGZTM|$<zIgOyi<bS8I}QjyU$>gWNq>xFfcjHotnjQ-
zdst6Z7!EifKLN|v2vCNWI48pwX8}XXsdcz@A4WTzTyVJ~GXht}EFV-W0-xmAf>2=h
zSk|G&XyLAE{S>Z&UdY{tqf7n3jjn!vYGp9+=)dPP+0Ti_*_}7#tRrz}tjN=ABWuyq
zJu9lo+P+sKEgv=;12z<fiG$>_p(}AMiKrB$DLu>vpKW47OIgk|Di1?t9P|(EZL<&T
z>S$<pfL`Ij=LBMX>g8Y=S`HgZk1d+7Zlfk5pqT+GbJqdWsY+moz0B83{kY#|m3Pdu
zxe+F42-CIqCDXo%;B5TxS}<W@n`(cCl6ceY#S_Q19t-8|lxGNF(EH%}RtU72hz&E#
zorErVp1Y-drh8MHzuXN=gutTiuiq$%)}K2udv1;7jLETg@#`kdi?Od^)_Z9+UE8|@
znZqx1$R5UvO%8a-nqSJ{$IO|RWPFw^%ZmdSR+>ZmR^(0juYMS9cI18m)Tq4GB-Z|i
z@RT1@54mtPtIZThd94zY5Ee__l$(JZzP~&hpDl=!W2jPynMkf<wSojLs|Y55K##4s
z@y{UD|IiE=N{<{wk#<CZ#>8eIcrCR6L>tiU;`H{04_iGX$}wm~Aa6PW!3*XW;3QkG
zZwiYq3{hxgIP6WHdZH11S@MbRxgm_IEZK(jfL~t<NLi_M%Uw`cQVX+rw7_FuFK4O~
zga*SE<`wb1h26?tz)%z1M<mo>#1-wZl@;Lf>#;+^epsca;$1bq&#eTNQO+R9jQU8d
z@yr^q%j@62T0wukgLh@yXq6~(qaeyim3jbXGa_LjK?<U`b3hrQXzy)%N$xCFk7H-p
zJyqUx3C3Y0VQZJI5%JCNVVX-SV^Q2t^IIS{hr~#-iNO?{y0lWl_w_-3izDceiYStV
zSTnZ`J9BNP1kONbG!fZ0N2DM@eE~vTc15^sKHv;5Zg`HVnQRQa3&_aNs~7rFCwZ`9
z)8|Av-`9P{kQrF%$F3i`NCbL>`~`-3P^heIkV9%O6ubD(!Jh$!O8sz%4RJ{n@H{+=
ze24S)a`iSy%J@i-(_*|hsq+Swb>W$#-Dt{5i3&s>e**!Zh(H173xICeq%(K;0A={8
z-Ehd!=7z02C}9f|5%q*ja03FV0%(bW0%sNn&!}~Q`8z~CX*1LF(N1G?_`O|Pmu2`a
zPjEqwr>{HWGwg*eqJkl#`ub4b-e*%Qvo%4x#;O-gOKm~?b`t`Jv#VezAs$d@U3R^X
zz<N&cAm&6Ca^DagIyN=`&}yM`Xq+<S0+aLcujf;+aS~^<V>K65W5i`(c=^n?5kAAM
zk42W;{OM5xWwMdcO<BABlPJ-K_Br8=$#g8IaFPdItz=^$yryJPhg9?m*LYE6F|6Kw
zfbiK>Q;1miqvTe5-=F4>%PQc5+{=J%U@G#PL;;oWHP^tfZgv?M+qhFMpNc{2xlu+r
zYc0ta&@o6jTXc`N@c&a^m^O9}S^NNc4ir%oG3-n)ziLV<JR?ruh7GARO+u!VE_Xfz
z^L93f_(mB28czQ)Y8>0K3Xjsqq(Y@3k1e$-q(pc*7;4jNR5yI_7gvhMEa-lR+xV*i
z3gmZ@8U2A=G{nAe=?nZU0iCnqg4bUrzrpR8OlHnoiQD}OV*|QlTVM-t3b^$ru;YkJ
zCpA!}B+{Ru(3hnE<DU!1?lsUMByVO>m6WLuzH!iCPkMDYcU+|i{b?tZApR+v7zxQ(
zr)V7YjZ98aoI#ifq%t?T*<y5f40tz7yfW#j{LKc_Q1y!iyVJL)qaQqxUG(ubU1)Si
z)ucpksqCTTfL5+%eiQFG0r5mzQrSbCs!dvjX(js4R)YDPHdpk19t<Q*_DbGEuj6_$
z1P1@YB*#<HCSPi3>zS=#CpN^U@y+&F#WBzb76eC+>Jx-YebUSRu{*(%LqZge+)R3u
zJM&1XQYywHEd@EpoEk+)j1uetR$e6?CSup_0jKbTtNosK5#CO&<hoGvU3JxIu^uzk
zWEN@dFuKd(8=ubrEcMV!emJ`y*7*Q(&#3PX$hm&zUPCM^T4*PR-k(0!=>Hyma~tTH
zSdG2Ey|)R`y=MlA#ffPTxCHJ<Ic1_;$v1HRA<O=bG#D@m!b*_@VC$_Qg+zR*R|i8G
zLf>f<wQ-MBT33gJ4!$>3cBxKU-1rXiPHtUEefkOco>R&@jVE$8M=}|#Jmgh@f;#bh
zDY>Xqnf?1vy`uTA@<-h)@>1xTI{Fjg`auDD2{fr;E>v%lk6@$Uws|`4mQ&K;h)!tV
z)@WYFs4grUw8Ui-lO57}-p(JjPOVS+#3uDNu&$^0t6-(??cV-eqiMB)^ej*?F<X6x
z7-&r$Sk)d&$N0!bI^+>vj;FkO5wlvMTAd{M$%5;~fSrR<oYo~XWC=AAEK*cL?yruh
zQG`|<rUxHP1f&|^YVh!${xsJ&<*`0x6Ic$yhkBt2`p1??WABYnoKLhUHh(hFz~K$~
zDEP0GIA_8{WMY1R+qg8Q2PD-Ya&*-wS{Txv5pfuP?jS16HgDKW48l8}0S^X;kwk!0
zEQoHHM+<dBegN%otbh*$HY;HvXjIFW{BO${hzoioXP{W@V!oJrLfGn+=KU=?fNdqU
z(cvcu{{jgsJJHYN7kbVHc!nRRD96+;`ZO%~wIWCe<9d`EdF4@M1Q2$#3EzM6Muu?{
z%5=#xfkaX0>f!OlKp*GCsS60tWbAK^$_BY`9ni)M$`Dpjo1LQrr9Ofnt}fjjx-M4e
z4z&n*7iExv3oDOc%PCdYe!76u&jPG2q2x}oAf1zktv#vb?e=RHSXsPY8&QG53^v;0
z;8OkDiF7lY^>XUKQel^H{T<h(7zfH8D(Db5F1;Bp^Qu4okWuBK)%y<TXW=_k(^VH^
zIcB=&|A7~nx!@~_RQRkiX*hf|B#Sk$8(+JYi~zcoyKoW<+?ur2IvT%0XE?kt2^c;B
zVmpoIv2)eBUG<&5_ZO8}Rf#&jdtY62N^<Hn+7PW7LksmcCEW|MCs+Ae>{8J5b4)He
zW%nr4^GD_Wb1LR<3Xs6y{HY|0M~xx=Mp0P;Gpnr!b~Ue$0gi|H%;h?Um>tL-ur6nu
zIB`$LAGujh7=a)>c)1IC5*Hy5G<qmEv<j5xh{*_T(uv!xVi}XbPcb+yF>E0;(`iKr
z6<B5<b=;wlf~?L-jg?-Dk@_E<0c2u3rEDrG_7N|t0bA?Ny|*BW*6Gv<pKj#NdJ>wf
zdigVaitt;9qfnRj)S~n{A7GJdKNuP!Q@UkCm5J#0nsY1o;oCCY*yY4J4B5DrI{977
zqt+`>OuC}Bt&qo%P`&SHn_a)t2BRl-CPUhCVk;OsyjHs~`}%ZPr-2j$%A$1(28SAG
z7<Fh-FwsO%ANJiNPASC8X4~2GAYO<X%$+NPSi{`X$X|Vy(#!h>BPphGq}lf_2pW~@
zRWWN8v--BW3(0XlkB!t{)sU!G@{nXI|MAG}e1nT@^itWUf&n!Fi79l47jKAHs7mfY
zYa+<N<|AU|$adSKlB~7ojVVqFrR!wvKtr{LPabP)&Vuhu3#LAuNkbzQbqZ=0;FI<`
z(0el~HDTWMp@xbEDgd;gQGrsm9l^XOKXb`d%idQMcwHPHgGxW+;>%kkkNFxSN|X-i
zF;@%8*(D6Cpw!BegjCCF`Bk00ct+pfzJ`K*RUeMSLe<7AEc(m|C+W?>DBa#kip&M@
zJ{PMM<(A5>G#R=UOl*X4I@!Gi)<reiclhfdit-Z^rjYBSHVJAr6!A|aF;^;F{EA$D
zlJ)A$&{4MYtmc0Zc(Y}-Grf`vjNuZ!1OdE3q85o68e$(tK>Bz|>tGsh0`Sw#e(6GG
zLv9b?AooExx=+YQgWiS5=1+z!M}erN4k(Y%UmJ*wm`{en7}Z*Yg2MX(${fzW;!jny
zpFg7n?LU+~DK!?jYSmz|43G3;^ii$T_uE76RXDC}P+xcJ;s=qxuAsFgPOQs!zSWSR
zlBoLxqJwHp7$`TO=E4s;_}R&&W*d>=$sv76l67v=0-Ht#yyQ;zTtWxE_L@~`xC5u4
z)X2M!PcP*83^AW<v|b@X$U}6ReZCqjv{>>I?a$UZ<e4gy5RDq__t<{R4%q2ET%-TP
z#U?mGb2Q9+INhy#(&EG-a7o;Z8R|C{SRBSLGg}+h`(tjyS?yS)9UCh19UCC7e_Mn@
z9=O`z?@wQrQ)9m^C(=C0@---Lyd7w&Tj9q0ja7>o{YJ7*-w*El#6oOUIX;_KJ}<Z7
z>7P{-f~xLz9HU(z<d!N&-ai`a+Xft(Edx}5X~1)GNKd8Q=W89&=q*Uf2ncR7Gq+4^
znCdjbB+@Y&HSL+UCi@w^k}sKfSJ5yZ%v+jR(IwBlf<nyNkec%u0kSp)c4mvI-DbT(
z7~!iPx7#wiW>;P*_5^t?91Qji`%eTE*>U*h!&AIQiH4tBe}@IT!n(hp`-+jg5Vl*O
zbb8S#JB2!&NL?lVmH&M?+v<|k0}*=|KrV<(ZonSTYYNCc?<eAEN9$aKC`;=CLGb%C
zpV6@zl<rnBlEXEiz;973g$ZdPN5iba(l9$TBID4+cM!ck{91+K^oBY-$)Sm?gaZxg
z-?>W-MMbfI%D)+dLCc({s}An<vXBJI^Ur`b)Srs9@brMhwDj)-IBfQ7;O2!@?Kd_6
z(nLN6+v5-2(aYoCo&oSQWZ%SUn3*1m>WvrI+1@9pCOnKu6UpnZi(Ul@?4;Fpvl{;<
zSa~V9hF_de<Rpxahl3ocKG5%xp<zzeLmJ8d#D0cNA?=BBq`;W^>m@t$zD|dND(L-P
zr*QtCqcgnL40@ZSnRpGKjgDk<vC!?Qm3Zv{P6xtC6rA3vNV6>fJYOYy2<h|qZcfwZ
z05$n50R(cpwzSvaM=*lzyyrZseo&$Cp05|q8EeG!wv=t64%M1G8pYJQ{B;lCRIC`a
z$naOsGpt#uY0B|OQ1J+zNNOn8MH@E>8{2ay939Cw7R7dN!y->=T7lGVQ9LJgG)M@z
z(%*bn@+zqq33oP&G`*PK^}f3~>j&w)`}z|XYkZ9TekaI~$%z-~>r~{Son%plQR|1J
zq|B&`yCoXsV~djj+O3MfeoQ~Y&9-F?hJk_3SH3Kfyj|MyhH$M<pY7P8-^d%{oGnie
z@`LDTdbgiE|3Gsd>(9{$eRu#g|0!+n`KN987V$f~@qm}T3kkJ?%!r)L0YGhwx>o%)
z;L+AQQ|u!&v)}zH_tg>y1qFXVp<ZB306axjw6W0PZRCJ-_$1g2T7jzKAW$B}d%9Rw
z$hy%-oy^=b9n1tEgIMjFmFhn0m!aGS;?8YBsECA`E6b-IMhOoFY>DX6wsW3Hw+Ioe
z>W5cZzE`~t7&HO$OpEvqVIz?p00`E$<qZ}Yx3gcRI#n|on|@GK8U_xnsbfd{32Hp(
zkb&0&WVuuD-aJXU42yO&=<VT$q<~!q?*|xSL}>pO30FU)g*JNo8wd}c%tQE^oe#Z<
z^)(+H$6F$KYx|n{d;s*oGiH*flct901S*Bqdx-tNN+IMQK=cgL9U76&J9RoHiWCcd
zdcmE2;xgTYSbnzs2h#W>odj3~;yiYv&C9BF#Oj}514K=6uvE}g@8GrpTss7Z)3D1P
zoc}7Tah6qE3ZPrxa~i5-zs6<@e86eo*Lbn<UZc^1=@fDB^fkz|3=2(hHEXRXS_I}P
zT-^~0W?#V8!&~#eX2cq*M1KV(_MI|>g1GE3GXYQS#x0M#r@v^so~Q>9bAWMQ!)vp5
zr85#=|N6&3;N${M%M=zWJiI%WDpAxSJTVp#Uq!4Nkak>t0sNT9N8i1rw^BI?ALWzL
zAJvJ4*5B}1j8MC~U-!osKpz-FmrQwG*dGfqNkIJdC}`5SusBGL>N{Lhy+4Yxp>(=V
z+SV=<u)S2Pn5g_QT`hfJ6-r!DNv^M(tMH~*>K;cEjhKUAm?-rZzb&NXxC(qawicMX
zfcp^sS1!q-kbgN}20<h~^=-tYA(P6U8R?+mz-qPsL4#ZRjgv`9y!Ov@IuNK6_2qsM
z&!QXqghh!N9TkeT=uY@>Kk(HdE>@mjFI|x?E>DdglbGfV4xw=z8%7V*OS0||?1~VS
zBWAufRA{Oy=2X|weEk^&v3W}Qek6%m!c7+5yM_1#NFfe+a?jxOU%u^C(TNw@-K+2<
zY~ry4rql~ioIWX`VpJYdKtXz(rN27$$4K*9b<Z^5E0Wa*W>3EqZv$qkJtQ(nl>JNP
z!+N3#Hnq02rSnDLXZuiGfmGW0v};e^T04d-(wuQs-|I-q9pJ}pnR+-O6mrJio9#w}
zP2O%qaLnsA*Fe7hXwmCLp=ELXF1C|wG&dz6nAa-Bm;t^~Es_nMbTt!k6rm$;WB5Wh
zIJE*NtW$RsN+LkaMA&B(jkoK(_kGmL_VF@0JdMu<(C<Jg>=oem1S9oqEARgRS?M)=
zLA-8AZZhFNUi${~q{c}%xa_wDQJq6qPu8z7c7s~)4oTZx8y%L6p1nQYe2I@G8n$li
zjl9HTOWgWi7;sIyoDQ^5q-ocbCr5|S14e!EtVg<aN{d-aOy6!nv!d(NIy8mzqZJLO
zNmXfx9i%E0+fyuu9%NVyHKE9bBBQsSp^H2~6!5Y?-CAhBWLAQx;&6pr`Tv#3pZh<Y
z1(Y#$TX5Vh2_oQquf_BBKEXHlR>UPaq$0P3h;}XWh3P@5VuEELM%&{hDZRN$-2nL$
z2OV{N*ry33`c@o>ey8rsKlSH~z2XDR*Iic}<;r~;Ezfxux@dwZuxZbLqHy@&(RBLP
z%=6Nyi@Jbx?@U~WBmGS;Q}3A{hgW3Hq0ycomKTBcUBDboC^;TT!cHTEd`xAaEP25^
zd7;H30|~6B8Zaf}bNORWAIKMheLIE+RA_xXY+M{F5im=>myi4cjfFltgFU7eC^$6m
zd{5P+E?=o1!haj@cQSoFoKeM5vV{DVz{+Q!_{;p15u?$6)Er_(c+3_w&Oj!?ecMkq
zR;cyd|Cg8NoXEw&>qkm#I0SSs1%z0@-|nO|`${}OU9o6u#z=v<W`w_PVXU8G@F)C2
z^*FqK<2{u&#MQ%iG@*cc0XFg)q7Y+k8vcf}-IZ700|wgX<DUn;qv%$+2Mtc`4DTv0
z>;`nqPH?FpmHMkKMyrX!9AaD|Ui$(LeR!NogHS%U<l1KW&-x3+x9P`bYjQ%30Dkqs
z7^X@hDFC9OGQ!SB&OP`XQp?owEkB$p3Ju9&0jpoWV0i0uYH*nu3+z1#C-zNNCH!(!
z@Bq;$3Ic)_n!Xw1H_S}GD(sGUdAGvkCp9cLF1OwlO;+3ne}mn6A4+5Ru@i#D;YOqO
zI6wlU=c^U&Q<q;c=6v#Nkq&;r10?;H6OLO3;eFNS<(Y;Jl1P%dl!tAegEKWbxHea}
z-B>ZF^Tc^(*~0s6WNcj`JB4|)xf4aI(j0h&WliRT9}y!74+W|Zo>;y_@rUc6m|=-R
zRoZ|5vyUF;NRNtf{@zAbgYt?vdwEHejpStlD!?hk8$MeZV!F^{F5ID2L5vCa_%BqW
z0oA8n>;=Dmw*5`B;AQvOGb5};<@3GSc|b_kjN^rA<O#3^+ztEQ((%m~y0l;<$D9sd
zEb@(gl|Hk0k$WjX914k3OZaTNSU`N*jfVhnpkTrQ#e~RO{_bN{Cz<p&s9cn07zjb{
z_j12JV)}$g$SOh31ZOzdRLIqQbt(jmm)74o8<?OZ;{w^APqjZ^+xJ_AJ$>6m9PJ}3
zdYApujKE+Tr7dYv-*ZsgFJIKC^?*wFz68Pk8=Q&sGARB|P9fUE#ER!yKJDf@Lh4)Y
z_lFW{7d-m45}OY6-8+*EvwR_kZD;v<YgRAAI_0Q}^vq_9I<5S#)!}wdu$9?=+=cUh
zEqt!^j%!g=bnu4DM4c@z3M~xXP#Ee#di&Lbf}>(DzOeo5$2;?M%U_~ntJ7%S1lGt#
zVb;`PtvADo&4%sIe|b+zyuNc1WNWi3_+eXcci%E`o!WW0H2`X1x#w<S7(KH;#gAr<
zfl7G;gzUd_{;mhoZ0Y=~z+Mfyo6*@nb;G6xzWI)Z*{}WJ^?>F3XfjUZZ(tIl2w9v$
zX-DF)8$vh4^f6yP3;ee1n&^#%e8Lv4ZQ-)9?V?BCMs}^=da&q5zHw%tK3>;`Vtsgp
z-t262gT>>X(lz=B+S=&$DOxxESKq+ErzbUH5Rf|2s9L4-%yGlA>_}&-NlO>}7rw|=
z##iKAACExPua-VyxWQoq$w{4AJBiI3klu^DEudl&AktaDQn7$p3&MlGhKDvS`8Lu0
z>r+yVD2bhHU=Bc_h-p^-(s(PWP*Y7HqESh)@NU7aOHu2mp5mU22>Q`#^q{$3$40D}
z1cIV6dfK6J=l0>M>z5gJScU_TKob0-oNuxgaNlSrLwv#y()~%NOhauT8JY&V&;M}0
zVOcQ!euRf;SaVkqp(?udTYqZmL!o9!5HHe;*NEi$J%+CK#`*0C@9A6)yA(I0$NMoq
zUI8>6A<`1xA-RGP;!}xk9gtKaZm2*zueRb)$w}4F|H^%xP<AgKz;RQ-ypXzu23LDM
z4oDdcqlZu!8XB&A2;WIaV5}e*1w?Aa>nlW;D&Jp8nhbfmV$f)ZsQIl7j=({=+9f5z
zz1Jv_@OBOOD<{XG4Gay(?jo+(^8GPaKf|Kv;_}wR;pP)e-X77iN1=hXSwR9zXy1)j
zxRc!D0+B25)`3T;=yEQhQa|o)4t)3ZxrPA+#16E)pV~z4fOJSX`sFJ1mvA$!mY~@f
zm@yKd6Nth0KF1p5DZ)TAiS>gG$CG3Fr_9e=A;C4IAKUx4XAkQ1d~fARo(bU`5UjC{
zeter+PV``H=L$_<c2<uKv~BnMF;TiD{2-48K(f#sld7mg4FsQ+YHww_Dj#nGK`%?h
z8;t?#@W=(|2y9Fl^521Pq05Bg7YZxO`CG0?>o@e{nZT-#MkGbBR)J#Feu=5lB;ig$
zIPsFRmIxw-OF}XgM<@~XEd4ViFNnA6LK}i_L*GRAqwz%X+gjaQ5R8%^#H;*jb4~8<
zb`wnm-|mrNByX{f`9M1E=WlkOuJ5wGvci3`(!P6@w=w!tc1{QPII%KJmVtvzraY86
zXW*HB$iQt-{KIEC%sDs`!+Q_Qm1U)zMzgn{47XqSHrEWz0MF>z^k%x}S^ZhMcT>!d
z%nAK=`_iuyBG$gmzXB6*x-_8l*55rwXt65ja1uYMy)tSTRe>&x(U@ue{94WN6i8-1
z;~cpTo_~}KnX)Wm6*nBU17E*V$wZp)GT9j5=p*-Ab<e~Nt}KsVs#PU2j?#``FW!l<
z4_j}OiA=oMw5n0+^QU{ec#6DGyZgN|*S42cfk_V9D?PJ%>F(6=K4_8PXb&VnXM>0*
zomM6A^c!N3Fh~qCc!ZMiHR^7UZcQ<4QG(1l@QtF01s&N7QO>~@pviQ3C5L23RQTx-
zKmO-Pd0jVTbOHX)U6R_^2ux%#e__WGirqqhQcR$f$i4N-L%4dF*B~eH8L@k1cVuuR
zXUg_trb;CD4ljvfHo^|FC4VUkJTA7&k$gE~RtxbkUa){ggJn{K-tv?2O~I;gemo@b
z!kKLfclc0;q&YmAK?3R@f$`mB<=wkgzU38MYaF}t!)4{j^QrONj6MMfGtMG!M?kn3
zPPc`-ul(ZYh-uz|?r>E205KZetrHW(9WUGZx=ML3d+etUa<}UFD+{TuZ~^=krmvl4
zCJ_B4=<XqhMb6ap6Azoe^1{^$7hF^qpg}=_-wyMmGXCi;$ucH->nt<3;(dN;=kbG5
zxilO-5djAQ0ZyD672(=(<<MAIhIW<|Uk%?nt@zsLcu(6Uz!Q+0-o4H9?xYUyQ3Y+j
zs_`!^VM^QcwHJFZWxR=>%}?F@bl*jk^~Ug_5)^o_C3#=86%56eb&&B-3uaINuKrgY
zD-N08(P$Ci$2VfL-3|x&79riI6wv4TKz8AbMpc|Wb?-hY!q9sP_jPyv;WidEs0^$7
z`kUAn*&~&nd_wRaTb7t3PtT_AZm7l$LhX%TdEfBXtRoUdO^5T^vCcbhjrbq75`G&3
zNfzeFzrJit6xsqYhviS910_pE;^9_!mj&tR%H#BJN<aD^T@hzZ4CR8KCBHQgStxK_
z=xuB_KVlaVRn5^6d-9FVtIDkVG#sxde0ED~g<l9)t|XiUOB*2J%P1QEqHmYc0>S09
zs`OQD^qH2zq0;RFg7Px0()wO|(dN_wZ4Yyv^~M*E17tTD^h%6Tm|cf^luiR&8zWBW
z=5QYq0>9p6b=qP-Bju=q&_?hA4Mr}Q?;P~s>;c_~iV4^7KrlxDb$~i#|4FR6)+JYc
zubrY%v}`Xy$bE_6{EYs_xbSuy>j8E|btqQ{S;DR848u#Pxc=?RrtTS=VCcC4w$z7b
z7^w>815qjnKg6l{{l^7XNY#SW7Te(BL{3xKbqab2HWmVK^m!}IT=+F|OLaagsgE<t
zL*F%n5v(x3BJ`39qGW^_^Ra+JP}%<*rUvV&3I=>-0aAjp@?C#2hZDs?`i|uhf%V>n
zTeAA(B`4{K@Nk_=7J7Xf?L?6MqfbVd-GN*)MUeYdF(<B)AR~aVl`Z|#CeqCTCf{6k
z+r6`z7yK&k!cK6l&o6|&Nqy}P;@Bc8Ihin{ZTRs(;@T&^(Fjw`ysY6=+xl9Z%15~5
zH2WaJ+>PlfdNJ~?`kMH>tZ`gY3Y)%!^X~Lw4Q;8%)n{AzQ=y4p^L=aJ6+}XvHp7P>
zY8uc#HBs}tNlU2w$Z-|mM;%*_5@u|&F_H0f(<d7Dl;2d82M0)oxD2s6r-X#fr$MZ1
z0A4rnnA<3_21qwbk>HRt-wa4dDV;%kP8uLl9oW@sy6tfp&J$V}iEcMC-5txsl^cM~
zS3_r@(Kk@ekq8djn9t$1AsBVY%2%}ICNZ{Vf*R<51z9Xy5<~}<o+6b~1RKRG(;x*z
z4A@X=1L_}6h;4h2mGnMnmyk@&3o{2*_lZKL@H84yrx}}<2_IX<5nv0nKEk#$>1Xd4
z?($DyZJZo{RQFv++Kx0JF_C6}<1auXoWglCxWru(BQOqBByDq*5Kdf_6%zJ?nC)2&
zE~YTjCtTuUILrv}>Q;%_4$W$Lj_bc3dqN9c?661I&Fyj&cB`#xPN}hMDt915=<w44
zeyTM{Lo~S>3G6i->lk?OK2f9N*sL#%AvNU^3<qb~^9^pDlU?sEtuwE$QB)qpSD}C!
zbws<Af*(q?Dt<&8m-c<XgX}{rUME@R-eQw+D6Q;w6L_@es^#Mi`)@xWW0QXn)x5P7
za3+3MiPS*`#Y1XG$zU-W6uhPWut3n}<FhRb^*?US4c^<c@GO1tU7U>MxyF<4h<X>B
zvc*nsOlq8hxIP0%Oy6K{WjcF~8C-s9<v~n7{)0@50IAQ0drxd2FC{}7pT#^qC21A^
z#vufvxU_QCrL5jK<`o{(-Il+D7>gjRk|>@KfBGsG?4%0a9?e5GTz~esIKMv`HdemY
zNp$`(_fFH>D@8OYS$UI~^I}<;ahzXsnUBx=Y~NM6WQM7D<sR}v%5>c1r)FRz)uJ*|
zm^<Ac=J731Reaj%BfJ0dVv*p>Me=*pjt7;wQuomG)l1ZBX4Zjoy>GZBHi(BSAj#IF
zA|n1MGe)|M8y9Rcc)wJb9W{HrYu{MbLzH^Qe;h_vdQE=}N8-RA&YQq^BLj9@<$C;R
zkKyyT-~!-WMnV6pNJu;1weaqyMLq#mj0l2vP@gWp3V#k@1g;)c)@M}-`A`SFCksvh
zFQi^1(O{8o{jpc5B8fNBB^IA*2boJDp9xdpwwOAT7EcFRhDn&Z*lpaWFX-urjFr`X
z+jLkVc&=W27l?2NKn1lb*@0(d9_WMcG0^Ra&$Sz)wKKPZs=%fn<o-I2aoB_6(cYhD
z5fGmDgx35*h0(S;wJPJq*BFRBx#?kbu`jW`B3{Iv(W|!szcF6=`?eUgoIBC$u=Z`{
z_bdj&!L0cZ)jURHiuW?bVZ6g4AKxQya*`Th9#@$hsjzs4QL-wxEc#w$DDoc#6;^@G
zA#RLoE-Ox79wX9D@OO#w8DVP3{&07GS;9u`nFTWDmEDg7#XO}O%#moi{WpVA`V=Y?
z$>vx3q@ge3j4;D^i`j!7poBH|vP^h&pMMJ4(c;I#N%FJKE$GV05yrACR1>||&GSj8
zfnBRA^@X0>B-$wOfLs13ciNuY*i17ShnS%#oL&ty(N3g0l_g<~P1RVZ)U)Y`Q^)@6
zHK@gtB~anD!hZJ}V}E*$)1)@V*DY-fsm<p=Qih6F&)LXij9UcY^e#kc-2L)~Av9Mu
z{Dn%tKq@3>vw>c+K-zOKL|bEdFJ4MPD-)<=U&KnLv-H}%TdZ@S8wd7#y*Awukd&}V
z#I`8l>1~Eqi4-iRHG11P@WRG{ZjgVIws|Qyo}=+lqTFJf*V;~$9FLyJE2Zi|6Ic%P
zf$Nj#r*^lHPUk&$854a%^tfdmS+Er@!Uwp4tjaJ>1gFZ?isA)h7WdBg9}(#y1n@ua
zzcRl3tSR4ZDEm?#2L}PoUo56+eS609aCu~Q!OMgAs{rQ}<A*^03iruRx`YS;G4~j8
z_lkLQHSn(mQ}B@R;hMX)GVdO}e0tUCOYCvetM~Z$-JT~gn%Qog3ar-=q=wazSF7TJ
z1}o;2Y(~+JA?RF{8`_J6);PT6`jxUb0DQ`=ib<-;_PZA<(vb8}^?4lD_r5K!<&yFy
zBlUpdH(jLW#Rg7Vl^X|$N>JsFw2Jmu;u>AGo1?aiDp1shGn%4e_2Q8y`r<+q3gv#t
z@-@V(GA@11Am|Ohk3|H5ax&Zh$X6|bMM=Yf=NA_<pJu43lAz0u%t4fUD2sYWX!h&i
zrROyzF-OE9>0n`8+F@Kt1uF$ugZ4htXUFnO?kpjT-3?Q`h(@m@DNIzdE6BNxl?!kF
z!<!MlwRhYOHvqS2e1Aw_ll;nbT?T7&Y;3MTnF#u6A=eZgXnWMv_^;qJI=ZE~y<`HH
zFXn(@{%P3mE%5CS=(E2}L1mS7*&gfUWm8;{T=s)|S%EJ|AUgHk_HyCAyopC3VN^EJ
zJ&>UAoz4%t_W>XLE?zN9?8?z~lz1?sa>M@T*{zGs>7u9P-MNV9g>bA$9n~oQ@{^(>
zkU$eWhp3UwoCt)_5Vw$+>5^<ed@Pi1j94!u(e^&M<phW_t`$<HTG0rSTu7FGmaiE@
z);Gl%St6=DAjY-Im~C(^*S_I-=L1i#!tDHTm3AnDFGcKo#e2pHzv~<xEcwIMyG8l$
zkJn-IWunU4_wd*oiEK$EimN9aXwuM8Uh%$BMEWd~fx@m-&!uer=G)2*&YXQ>`6t)i
zh{Rhz0uJ23iU8COBo0a6keIu5@#jV(r+f2~yzjE%Hp%l7TfM8dVHvTc>(C`ajxx(m
zGRBD@TXfb53qm?4#*<nXBg~!VmI&Skm-lA!35*J~g@n;0ToU&?aP0?XTh>M{5isU4
zj6cvk=g=8Kv#qbc2-p_caxd|wmxPL&xbZbSf@Yg!a(qm{yez<izA-N&4uag5m@JBb
z6dPZF{WC8BOBcz4ovv1)VpHcZ2J|_S)Rs(E)WzA}@6CyXFkf1c)b!ak);+E?!$AIa
z5oDysLSUUOqw#S_nT2AFcx}-noOg)-i>}8*m=EnJ!a@%jm%<i%yIsjB3K!jty!t_U
zr=u1H&g3X@`!@E$3kJF5H69utP2s}JU?9UD^m$)Hr3}7Bx(+gQvT0OKc%MQ4b6zHV
zgTj1^05J_3>bG&Ng?7zdVFuZ8-uWc_uc^f|W=RM-xk6E;Oh^a>D<Dd=A&YYNkuWL(
z^g#?vW6y}^goB}r*i>z|sperAA&1&Tv<gb1R?bG$#rCs-7tn`W_DHe*drlS*`Fl>X
z`WtFy4m$T3CvM)!LSLBVaIkF|-Tm;p43dS;bH+3U5<9cL%G}JH;>1o$NWgcc3GSKu
z__h?0gT#<&bP0OlT`q|uZb%r!#4)}I0_clgE1~iwqf{apR`#UoX;p#2-3piV%6@M*
zwa0AGG7Z7x0rS6V^h9}}`kg@i&%yjPc^nyI1Wvx2+u`x8!<p0%Z*6ssvBsJeXC;(M
zd!L}Mv?SX)Gp`dO%;9jtO44e`+jM*c?)#u7Srm>p@>R2_moDFke6kxC-|^Tn16m!P
z!#G_ucvu7Z1Lb_ZV|mgo_J<q8`{t}EntdpvLiA$hT?B8+p)`NyyFh$)jWOWqVKK*5
zb?zvczY|)tIy=t}h=HW`p*z%-yi*(9JQXo?j>ZALkSs+O=E~>!WJa5VhHpS@O9@@`
zEzD%JlNbH8>5V#6oD!&J%rx5Flnn>aN4_PO>Yz}0rk4cH_6K_R$7x;A7mI$cb1+T=
znLRv(f}ee-{y)CnGAatL4Hs5GN>UnzMp7h4x<Qa`hHeoM1PPJuP5}w&kQ#&ml#pgf
z2|=VArBk}Ujql@G-}!U?>2lU`5qt0ZzOo}i<-;G3pU0Gol?|~8zID%nhb)PjySzf^
zr2~rZbsKuekCZh`65U5w-q3jN)US?igsDDAlG9x@bl;xqEGrl<Y{?OC3r$`EIH!U<
ztxvRYXwYk5d-y4SZ6+t|9sJnvBb=RTlwzR(w3sMzM4O77dKCAle$l}pqL*qlKLde1
zAE`dI1oS0wF8AP7ptroeFLv6Bx?yR}8+=b*mQ4gH{|APRLML>GtXhtF>%y^HFzmI-
z>)wYGoSH5V<i1~nY!~tq)Jm~L58ok>>F$PY9U}9(kwiK-r}DVsv+mWj9@FRZr9H-K
zCwd|`N+ipMZ25!U6JOy)ExA8|`7W~!^%)YgH$hvN@^}Iw9(#8%I7+0YjhAbFs6}C;
zv&8Tyf9G>vJZnEli_YElKG?1$SWq7}i!KFCRNBOmu#WZ3M`06K7Im5u9i_QpG<_S?
zi&g%Y7)!-JGM1&KNbDsmg^wrBbv+~4^(PPVFRH`ff)8}~tGWZOX@g-0kJ={9I1-;X
z;S_{~zf%uSZoQTPIjLH;t%N97YUEFuL&em5&JXt^A8({XX+nk=xY~5!!?;K5McL0g
z)4%3Bg}=Ge&8lMY!&&+HZw7eX`2k#ibmHS2`VuB2Z|0>;DlEtzktI+F>sT(~iErt*
z>t4wi{{ljW(R8FQKyrC-pruAP9(Db|S`{(4mE^u?soK75y1A^j_q7<v7S*M-PG}DQ
z)c8kH?(nFu<$!YPEif^R+5&Bv7w^Y#sO8+}^K4)*H}LW1-ACRpd&c7iH}lX_3xx$8
zhafr#0ttt0i!$HW{4qA6nZqwTm_5I$5G(V_9jf`@S=9f#Ict+P66z7xX4mGv#OEs`
z$<s5ig`~XfDrSv@I&$~c`_nIsL&8m6c<%=3MKWSqaiT1dQ$&}Z*hgccQ%>qdRu2kC
z<-D}OxsN(pLUd+Kpk_NU6XtNZ>&wu81Ed5!i5~v-0`V%*i&O#EVlM{QIDHNK7v#{R
z&CN+zL%jwnq;<%x&@3Y!P!iDGf4q2-&GX3L%5<VLBd%5sa@_eL-ZY-4k(lr2t}(M9
zg^QM&z<7q?#ibWB+t?+@a=pB`OiI0PXO4&MGDlZRZV$A*2CdzY59Qb8evv}j^C&ca
zm(H6L^+BDGc;EGr!g3{;Aa1FYd&eu|(GT>Qi<B#$xWh=kouBNg4T?V*cAxMkGpH(~
zGH)a4cQb!X9D0_?7$?fR(#{Gxu2=c)-?h0wtj?)6R%NT5_yCw^m$F^C?2-tl0GGHD
zm)}8)8JTG`<2SqeDEl+{K`UwGk=i-!x?dr#l_S*x$y1rjMY`bfQOlLF{83}~abx&C
z#cHAhmB;wAUzwP@s=ZyDi`8E<6|a=8l&HdI(f1ycZkqDeoe9w}B|0{Q{Z7uBlqYqM
zzmXufc<uknlDzDB@e4u`q9@vo_en2fWeP}v)k?Ry21G?!&UPqL(=*Ei1NF3J<}yCb
z*-k_lk4Th(Ed#?fU4koQf5Z&Ku9ZwxepmaG(d3Ei(r?D$&C6WYnm0oxnTt-J-paMq
zo%#Gqkbij+EUjF;-)IBAce)-ik=I|}%wH4u(cvIaZHj>)HW#G&@*ruc!I#!kYVWcf
zfO;Wwv^jpF++9AS?1+VsX^43GS%~QD?jX8Id)R5dbH9gv0JAq2#>!1vF~J7#N1mSm
zKGa~}q6j3d>xynEK$hZ+&V}c@IVM9+WpdnRIV(bhy8HOqIlV5H@h7l~-hfEEP>-J?
zj~Q)9N$1vr;ytfDI-7%+Ih6I^)u%|De|S{tz;y8i<=6C2mQmAbsLqR6e?6*4?_Zk$
zXOqThwXkf$uORA&?c}`VM!vc!h>_nqG&0Ih7B}lY2Jw^(H_()QN`(={YR!XWtL$R0
ziGBTS%o}SKyQ8hIoi5+CnZq)LT%I=bpP@3CMvy1U{t0o88uyK*=3yW#=B2^e4_hzx
zI9(PGi%6P}rdFn_oTb7>SbBy_{E?rHs^N~a!XfYlbI@7)!~wCr-*CGeGv#M%3zd=m
zN*Rp#GL8p+X*j8_5X7HVH<Ot4FG>2G-xO0KFiJ%!mxJhad8<CX?X>;GgJAZS=xz<M
z=!`1Z5!f#{AaxiN^<#ZBXY+1=OT=%&PMrAMd4!2X9_UW;_iW@72r7^ploIDmmH=aO
z0YySfIh@tXaA#aHjvM7*Tws@<HyV2t{x1uFDgTRdPIP1tbxhu;YCfgWBbf8_&=3gQ
zV^yBSdc=+=`Yk~`oQm{jNVcGu`}kDz^Q)t@3~}IDw`J??05;xSS~{k7+lL}^)qL-^
zkECo!cinoI+BZx!j@q$P&X&w<cz$V8bP*J^6)5_yPUR1#9g~&eD8q+2`J>so>a7mg
z7A<d{lf4`vqH0-@v>51KXdLzN9BUel8dYMSmSmXk)=cv?Upn3x>i^u1%zk!q4t;8+
znf9<Ep*f)!g<f|Quf<sY6?fQB>?Ky*ext&(a1!Y}2J?_<0wPkQWwVC~@jj~fEiJEI
zT#P#yvQxdjP!ZWkD_6b29g32h2nzLBoFQ0JsM&d|@vivd?Qj5rZyB>^3s%@XrK&PF
zKJ}&mrtkC=ZYp;P%cARcOR|SXSnw|28E0v^SoX(8xHNHlV+bF9YPQ+Cc&v|K8d-J2
z$*g+FNKY&r=NvJq?62XVQZbiJ>*GXFSfdl4i{VNr7$?H^X4}$iAbhruf?pY^E_cQo
zb$snK9|F;BXOx!kg(N3YN9S+#3KFS<Ff`}r_b1(l4qh;HT<WM)Iyy{Zf`A(;jfiwp
zWZ6s4iJeUPjS&w%huNgeI4b;?Jj1=(T`&4kdhbcmeIpRWDq|k}kYZi<ehC|;O{#T>
zr$O=P0sl^lo%W|d6I&rtMG55zCH%t~n$4;VvRi0bl0X6F?sX*v-{A+~aBW;^Tv+v3
zh!#nVOM}U%XcfTtK#2Z^x2=r{QA4o7TO1Dm#S<Q;?HDPa=O(AdRFMoIdC9sk6no<j
z5N-ctcLvB=5s|8(l*-OkmpK(Pz!{HXzwYjIpD?n7Xi4&%xjQ2AC`7XU*!D-Podmn;
z!lkae<7g}Nj__>W0xMuJ30BkA8xyxZL6hN_&ys>Ltim}0B{!7)N5&0Fr^~Fna>Oqx
zv6n#Rw&IT>HTif}nb{Vg>~#gM-RXGcPlJ`O>9t>^vRWxY--#zto*!>3DQ`t8PH*2S
z`yykD{@hOY6{8<n6)&dlRM^xr0PlwPVvQ+!`dyv@huK_Us!{&Z-UYt4Vr3rVRQ4Bw
ztCJRq2O`PLnZ!g9YoP6u9-f22_Zv#sbTQ|3xchbIj}gg8FlQv^l*V>`EVhILbLX0o
zyq#E;jkJN`H0Rc+in<WHos1WR|3!TAqp6jWqAWY=B0G#yMY1;y6R!GppTC0<LBH4h
zw;$_x>zyBra3|z1VK>NToG}8u*zdfzs5H9`JMPka!p^i#0$>#)1WsA5ela902<ej9
z`0mY5UkB~-RXbU|CSVuu`%nzxcdB~pACv&5=g^$>W;4T)%S403L6&XL$&{!$d8g^b
zq#b4{z&uvWgZswv@+|Nt*<%ZmP%B()p`(p)JzIL(o4N&*^FQCX5+l(!M9X1?-Q>F4
z%17Q%?&tCkN;M0s*d;34vFn~&>%#>b%&*WvL1-Rv(i(OatGsV#I%i{~4O9y$`4Yj)
zrn?WF)s_PLQ{@-0A)2ys)pY)6Y-4wFfdB3RaeTD7+$?CH`ygZg732sC*F0BAj{D6+
zoR<*^eUX1$SKcG)o<kZdj!YV#R6W9OUQ0YQR4~;oungmjBR?=r7jk)C;P*Qid|JK5
zpwTKKgv_F6U_fP3FiRrrJX#ZO4kabuJ5AHG6&HgmepEd_EUJuz7F*UZW`286W7!>B
zDQ)(eqNZXn`j2>rC4+AHmk%Z(LMdv=Iik9EOM4@sm4JEpLH$N@Tk)hvSz7Z-NQ9<j
zb45)S@ie<lq-Z>Hctv~?8zn8;-_(W9cc9mb_x-y!;2ee1wZ<kS1d)3U;D3v9F7#0I
zI<jH5>FtMp@Zxpe94AkRuuc)RN4sr2CmzW3Sh$*F9T2YQ6m}o9r-#ga;|U8Eddxb&
zZ$B2zeS+D!+avn%+IS;*4mcj++HKpmmZ}6TWP}35(z)nC^9AZ+pQ(Jgq}Ta1HhUs@
zP8}=+`qz2SWJ<zcF3fkSGk1EUuRPfsV0bUq%Dw>n5idrq`yWb0pr3{Q!CZl+667Tf
zZQteDwbnaCn0APBJ(mRAYK^L8Y$lnTW_J%!l0s(aynK(JbNaPPH+unI&}$sxO+@D|
ze?O{p3531S9Y9_eFF<6=vDTRQN{+gox(g3zec+^+m(1CUdF$GLCH0enoU+54d&NAV
z`2#58m^czaSz+WkX}jN9{1&yw(j@5{CpMIJ<Cqi4dCNE{dx+2C9Mk|$rPEiYRg~a=
zBpZ6<WX&2y&EY`V?d~XoRyxWmMLSU5MT>icbcN&iidqPW9qeWz9%Ed46JUI4)k`N<
z2dLLAXU0LW=LPJH>T(+s%LHm$l@mt~S$Gy<oO$aM+K>P4jC7btsn*SW8(vzGD;oSI
zD-p}bL9ide60E7Fw`#CFF|U(-9udw^ybVsZvNkpYm5LCLZ#Nx!Dw#{wtBC5y8QGa#
zk<iilgNNP*-Ld%Mpj10Jj#XG(lQt2In-z|oVPP8ib5HFy@B}v(m(j{n)j<)NpmO-i
zK&BeY)fJPCW>$A&`&S14RLN6y7qDf1kTLxNhvh@U%X`SP*>%bYD%hWAD%NHK>kZKc
zX5z=<TtHntfV91}mL7+*v#d<Iz|Z9`{1f>h=9ZH{=yKYjd5FPBU;?YBALHCW{XJ8E
zuy(p!KiH9m8cWCimZBoQIAeO(;!iid^_j5>R76WyGj<P{$CWmgi7H~{O}+Q;1>9K~
zy9PNIy-c}}`Q0|Ok01*j?-@d!tAOfNvj7qwrD=W|e<38YQN7_mNJW-PqBkmU=iT@?
ze6c>5k4s#0NgsN;Stng%d7<iQ(MlBQPgIsk%rqM~6BM^rOq(e1W2o4&Y^ppn_1T0B
z=)8+tm1rU(jik_^f86?(9Pi)`<Z8Zv)~^QBI3#@RK$n0*%9B;ExzSU550R$#vH+OL
zzct|hO2!;!mUVf4qZmUeQ1$8SHmk(|`wmh2(0oz0DxDy6tE48X{s1|ILo(e}oUxI+
zh{m+;_48}d#M?wsm#C>&a-Mgdhv9-Z5(oWsz$W+tW2>0T0<wplIHRWbQ~x)24{e!-
zelFlhMh8Fu5ob5LZha20B}+h_$Qpim`7r+ciQ~+R*<A?PUL$D|OsmCLY!D@=@1~la
z6hRrDtjZulM?6K%F%vEsflO2;KRlwNq;z3?wtf;kxJXC8<CVoEsK{hNXc6NZsgDRP
z)wxfKKpzx2_X+vdb98gqT9n4>{f1!=X1-vA5=L#=h+Rg>;)H{v&&ISBPsHGRxW@0!
z+$QwZVWKkY8IJP8uVavj{kqgpVBA-KdmfYoM?SLYiU}mJmu$YZBgcL?_|qQ6#k$Ix
zNDj#mc3peX<kv36u5W=mMW5wBZZO!=;9dw)FiI^i+gutw_APu8zTNMrHX6lK0Eiqf
zpXIr-@Nz$&YpC}n6@}8#sSAyW+2qB#$FsH)3?^kuj`tNg88Z-<)V*V1q%O)DH?lnH
z=4vD`J_*Ncq1{vMtzh30P3czEog%~@)Gg*Ssh=nn&<)Ig(}844#9~2&>P5wRK2a6~
zH>#9$YlRR&%ZgQCaCwp(LC<qEBo8n7=iPB~LLC08YU0$PoA+dkPL!C`3H5ZVIwzKm
zhL(=REj)=lk2FKPQ{`$cW<fW**bN1T?DYYkhck*Ut;$9w?Li(adI=~=2WfMcaeNt<
zYAqh2o}s%iz=GK*I*h0}g4o;D;TIN!+7=<*Z;|&co;jzB&BId*NK1r41`Sc)gB3je
zeesny_hkd$a(TO`yBuJ3mZCna3dd?@2|!COE(hhJoDhq4mg;yM1&O-(`ctATvi{jX
zOeWHo{~)k0`pSM5vD$wDd4I{(%|4CnyPlt671I^=O(CR%LJK5J-^YPe;9f&}ffGQp
z=zuIhKgfUH^3a(jC2w(FJ0@!0<z%M-KTIA1TbiT0`*(69Kr<rT`xeo$yWa|}rtr{Y
z8{-clvYhbZF2zYzbrFe^TrmjOo3UFe6<=~q4e1bOzFw@0CfOk9{6q3>^4Vj(!bp5t
zG82#7sW2CTjFnszi<`ATe24U3Ie`xk<%#UC2YLf4iIzydhZaV|EY+pDM!;M$_~leV
zt4SD=;3j5CBDvb<jcx_63j29n6SB5h(fsj%q@*LbrzK`I(|zJ{WwXNLNRmLRQANb6
zLxXSRdT$?}_U<eWUa|URPlcJZBF_@A{uO{|yF|@)r>NFt2;}qP#7uJLt9^4Khd<+{
zt9uyBw!yv+xwzDXeg*k!W$FEti4EWoU*R;W`_8!1Zal)FG>1%(<@OZ3E0-XvDK}kr
zG-g<Gb9Ggc9)*H@_b{qNEnDOr1KY!Qj=bgz<hT#fzXm`o6ubmq#Uer`D@urBB{22W
zHQzCo7T^J3)`#j`AwMZeTlN{+4!BMy(u__FSZjZOr-{WV6Acu!>D~sCvY)D}JR?;c
zV|17qm^cXZZq>p^@mhPAQ@BHHkfPx)vpwXD^08(*F*M%BtJI20bfXXL>Bh9JQ3~FM
zHby`TAWB;tp{i-^NNv{Ukb#m`BKk366U0Yxjxll$OvH*I^H|1s)jj##i>N%hu~v<1
zO7|prG4o3|Ezx$x&#xNyL)p?d_@D`6!Xz#%((qxaOQtbNb1r>(oFDV4I3Fy3+$?xk
zSsCVo8a`#@WJfS`Ivm_HCR>qEOw>ya{I2Hf^ltZV_7*2{-m?0~rp=FN%}GxTkm*jc
zPTtanDqRe_F~SwEnvO^l*F0ewFLa2?VL|S0-PCW`LxedbQA89GD@j@%anj-4$A0w$
z_N30r35d_95l~qtY!SwLkx+1}q2up9zDGZ{M#=bD^OdTqDhia48g2IlP(C`GT7?7I
zb|<SQ$>X&*eMWN}gmUmU=qmQ4Qv-@K5t^zcDS|meUvxPSX8AqAL7i~V9uHX|l$Nvf
z==TG(D%`uy(Kt_IE}&1*P-C_p-6P#<yZmY_hVGJ;IzC*w)w_5b6cTves39KvMnHRW
zf12;TV)&IW0S-65U3MbE^v#aU=X>|vDLRc--6|}L=Nr3AXVuyEs{p(Afb{Jj-piF%
z(bNM0cevMWZoAx^dXyLv8@bAOuuT&HNg0WO1-<Pl(?`&_5UW~K%8&gt;v}_-3cJ69
zny?Ns3YluUJwoLaa!i8W%GLOAbp$$}&e~;5M3B&|FMpT@Ia1|xy#)47iBLbgPRT>y
z2Q959n`WEDWkr#K<o~|n@uA{)F3F?A%nZ){J;{a76{X+zW0bt+>YWCC8$KQB4slDh
zkRt;)*W8$Y6a8;>>%l|Q<aye1?LZ=YygRom-!e3%)-?V6<U+%}hMTPrTq6aPcUv+9
zHF;<X8n)yJZ*Hznh@cLh@6!dN>Ii_6@gk4nL`ZDWomu3(ZE?sJP$ZVnIHmHs7*s0V
zlXiB~Dd|!8q)?4n8_E$6{viBzGUi|r+Nyvd0{b_t@3D*>%4qj36nld1wp1{?5MtO-
zQxlZ7(>ej0+K$p?N#rl!($|2cQu=@h2u#}2nJr*i%|};$bXN#VH3lT)7J<J{a~8Yd
z#l{HL*Fy}Zf!87-p6HQy>Jn<9Eq>isHVzK+BblPbAeO1P+4zo35%=QcEh)I7wU`-+
zj39x`U9>L%ejfeP_+HT#i*~<Km_fWNkQM#Gpq{CZoYPK?h+!F3;;AJhRYxV51AoH5
za%HvD?G*TAMnHx~fo|`QXCXwtE~OWX>1d}DS?zSu?y(`S5uaJZp&nzPr-HXjPhuE<
z#!o^F<WHRKJ^k9=ZAxx;O{4qm`-bp_cC&<2E9@Sx>S-T~GJKs_7v9De&kV*pr9y4f
zis=WKV(!w3%wuchGZ9cmotylf6^S97$2f?OrO>AjmEdg$$IE_xHJ7`&<RL%#2Eu$r
zY5frS;ca<Nk<g=#GaMfMT;p64d%y<JNYAIT2L<6}JSladwdnI>=g93(hjOq#i_Wy4
zNGKu}VI@a3fBv{O!2`g><|NSIkPQPJUAjLVIU7qxxw^HwS7f%S$2KI~8BR>Hg370&
zUAeLlk@mKqU8-=YMpXUDKDACtrof-7!_N`zl?aeIK8%bY8pzeOFKOfWm{$uMjAW{>
zuOEX}y8w57+5T)&3tc4Csl8z-l0!MEk8QJ$qoO7jig?OlMh|9H@1jbEg-meq46=<D
z?KRu=Yr=V0JU-wd)7LMvVon@dwn3Vg`bH4*T;K!y{;O|inPLJUH;%b6hIy>gAfe1+
zz(QB4v}mb+x+X#=$qu$3M9{c!a?jOukY|h$s|wdw4Scz*(gwIZZ$IS`*y9LR;XeAu
zr@?Xx%>pIg|Bp{|`X8UxROr+kwwy`Pf`x5IM{5><dpQ{RAV$j>bw6+>?!>itX=|Z=
z+6w1>^@0Fn50PV(`2tnDWt`a4+t_iXq&-ElgJGBz7P}B^;f{;3YKJ%LiVqMLGNHT8
z0f6-P{n&b@NBw9cXw4~1yr!3+e{NXcCNr7qaRm__cP7Z5F_F^r9Z<#Oy?-fLNZGdz
zX82fx35>M1Hz6Rwv_x3tc{A0)WU5Gao`z!qb^nm%90&s-<7{uIjG!Q;*_f%XjuiPb
z&HI{bMR(7Tl&MnO=N#4|&odNJ9|}s}<WagOJB-2)zA9Dy$`E|k^PB%EUASdUqr}Ho
zK^-$<&!2VPE<?}I@XB=FiKoUSD>ZG0T)2<anNKz`coASc*)>Qh`)}e1lzdJXAIa08
zRPow%nrNTo>ajyj+E|5nDuv{;$M^nXMOwE)M1@X|_>b`qlR3S^?;_AQ?xczuR=waU
zH{AV%{a~Ac@)<gZGLX0ICJ-%(eg1R$FtbCon_kzcq*T{F3<VQn`wc+^*#aE1u4)a}
z{+Hlm7p7tjQ0~jMaSoE*l9pLUvISoG29VfCpYVxmE-t5Lld8anLloY0QYO*TE;6fK
z_m<vl8`cgl%0pCPhyh0f3bQBK?fe{UdDx?mmVEAXVsLN!m+`cP$+CWBvN$cqbi=V!
zUDb1;Vdtg0l6{0!t)eZ)hrJLf;g9!O5Pg|ZxUWG%n}gtG$k>r@5{ryAXHi?N=F2{q
z&h_y+3gDi+l+wW*A{E!A>Ik#^b&o^kV|&i~fK_zb&TVESr3mG{`7nzCtu5|X=Pf{)
zx!@oDCDQ#q5AER%q6g>d;zX}c`6KQJ{CR|8l<(b0l|I28H5OTpv4uwZLvz}7wJgzR
zgx{TTl#*ciGJJ@S{y=p&!Sj`!gEwuzLhfKiQDXoZ<&3F%Iwiez$Rj$xQB3P|NVu5B
z{{+}y3LQf%MQLFHKh5+6EpRh>-EIsCI@5Py2y`E!X!@dyou(8Jee}#NBT~XC+`lw!
z@O3z~jjjQjzKy5TbHOOA$d6aRRoZa~iqZsu(sFEh1hUzfjieN%J=XD3p*D!<oVnZ3
zkVX96PA{7Sf_U1kLC{Glx_ivy^I+NMah#fahsf)ji*rN$unt_zTn|!nswZ%p&UBfd
z;t(l-zW!{!ienxV{Z{09c3&EfsMSJ>fy~1OkP26B1^A~7kWwZ>5-^4x-#V!5`%&$;
z4OQs%s_D(fMjotMkp1Hl%UEz~t2m!^Bvf&mYsPh0jTk^MsH7IWm$XTMq7E@-z)%en
ze`7K5fO%|GXE^u9U09*54e!?@Yojb+7iC+07#tEX>W!C}>309&?{Jn#C^qF8kr~=0
zXv4NwWo}hh8b%9%OG`o+gEVG=>aa~xMlk^!t#a>P_#KZ|81#2tQs{J%nv819_%L!5
zm#Ne8(7L}8O}3R;iL|;0SW%rxlE3b*-z!Z`#a9bmGaqRch4}t3yUUaK>Jh26-W0!i
zwL??`^39$8H_P)|lhs&#$PgYgqp$7`M_&o%^8wU&Kt0xIP0vsP;xyB#0!DCGj3vpH
zLJNqv)<3*z)+cmT7WDQCVlRWa(-9zf4CZKm)NWg3;rs}+xMjSVP^i5i(r<A8yyp_F
zl#orys7ZVBs(+{ZQ8=pMy8*et`?tLK?IvLxA<5pgwh{Ke0N<~?kAw4r+MgXc+R--{
zK7Yn&PNH7x{)d>*|2y33cSl4CtkxIfU*VMXH33`XN~OA?9DGryF?vxZ{j3%0_^52s
zwO-qJs?r&iQ@=4d$KXzf;*nD-6uqjIQ2E%u`nfr=J-2<p&oNUk2UuhT)RUpgE}1=2
z!KyFe)3jU^nZ}jpQTdiJ;FtM<P;HZ3<Tk`ZY!4995GTmd*z{<V{2RBeBS1=7Una20
z-)r;1g&8vs$0^f+5^*EL@So66Ilx6N;0>Al#WEJp;3#(aUFHzbg=BUi?F#H?CzTfT
zqc+jf?VxRlWFL)o4HP~1AS?CZby{5w!!;7WUHfS*q#g$?H@i312EN_#haWsF(!Y*>
zO}sR&-WajO$fzrdtWLn@F#G~HSlsp1<!_T$(80S7I*bEnz?2RH(e()hwLO2D{e3f2
zS3-@A=6DgD$r1+-vrB(6gU^tqWQVSH)^g&m8R*w#Rv3Mf2g^4U9a(>&1(#Z2j1`n-
zvm#Gs$*4XNIc0SSw*+nSOmGw1+62a@BRVrTsiqcmHcLS+yrFTMr#Z24^IbWQp|&GT
zF2z}D;4ZXovnvIL=P#cuF|b9+tG6}`)P>t7D8b0hvv@Eht~DVq7vI_jBGpTLeN=d5
zRi~U=Fq7d{%0#3Fdz{gf<BO<p+22PdNy;;h<{B*S(!qMvQeqfoiIhPHXbRN(@U?Jg
z!*v4f=Wajco6Yy~)WWah>R;XJc)33<xhwU%(7iYE6PwC#+y+yA4*q<T4Ssjd0e{-G
zV9z~wndZtWc8Gp-0RI&TlPK=MrFz1R`1mo>yptE4(gA22+@Z!oCHz^D>F_-=BQs+)
zD~w$Xy4RGbXPYL>0Tf!ubUbR#OK@MDrClUx!NhuD<DZ6+l!qZs$C0pbJ(f?&yi0aW
znQLNjBz*p@Ym-X0_^c*=6rO$^qu`CC#t;k`!HDm6Lb|6xR8;^d%q8NOFwiX(es^(d
zrBYUG>^OT2+8sFpYup0gkh(ODek(8A@T<O$K)*WY?^-|nq26+yM#2Arz&tu60G6*{
z4Z(Y!#Q!tkk`N6g(tjKl_vS)&0Bj_(KN4^E^!T}3*|O)jT3-EueY66<7#0gSZ@2$I
zj&kY{CHHl2po5&)Bgc;EJvHV(X6I|_!;Y=|TV{tTqP3%gU85}Oo97(#_omP2?jMIN
z^QuJ)0A;=`7GTT&dwc^W_Xrh!l3yLB_Knu4AoWLZdtY_$7KV8_ya8D0e@MUs*;X!=
zSnU`LEK){?ptb-ij>b0bx9;V;$aEz^-R;)rru$#Did4^fsHWWFF;%~)i5(GF74!?b
zG6cD-$|X7~lUHyqZH!lBlQ8Lk$&PvS;qL;$3Z2sD=!g|5_sURV|2DsV5zDnmM?IK_
zIYuORx4=T2&>~6}NHIGE$d7!8vk0ug6^qFJaAT=1>%#{n*tR$!)y;nQXzxnAAIK;B
z`oJED=?fv#WjYm(RYg>Mt8UkqZ@1JZ<8p`>i!%FE>w)_=h9RBKgrEzk?5s;3sbmQ9
zYypK}$nur6Nx+^mbKE2~Z1~CW91%~v+XAtjX)ZMfrSv__xBJt3g!p*fP9E}2`nXIP
z1ej8=XHHZiQBfp#0ics?xD1XN+#Mx^U$cboZZu@xuVv-*eiwvIYHx|SZ;wXOFUH<E
z!@Qv3She)J>Ik5*Lvcj67}(G$>+q2a#}Xy_Lul_r8}3QBMRDP?h;r`LkvJNeZSXKZ
zSdkX_{)ei_0GgIz{Qc;BepTgE?tq1v%xF2b&Ak}bFNvpQKofG;glgrJ`|*ep*Kmx$
ztksc3B8$rNrFJ0=B;~Ji^++h4L|#B8i7aBpl3B_9m-9Hec97$eXi>XlB+CBAD?gEU
z`S>>JC+<5l6v+@>PdSNvT_4;q{%t&;`{6Vu?jz4wN|K)vq94^dZ0bg>t+|;pjL&)9
z_@D@lIrhAIHpn#+n0C3DKjAmvBa+s04@IB%7=)+T@sd5TX1%WnpX@+VTB>8BJo{lw
zSJZU6L~t~dO9RS*%g;SVr$djF;X*YZ8}3<pI68hhaOXu*{?G%j@N*Edq5VmeNM6Y2
zPy%!*nG&z};dfFVvUz-o5&F58qg~@T>zUk)6(r7@XJS40(VdEZ%t<)9`;el9e(W|o
z)RnlRCW?imF8hti>qsapz_V|(B!MM!nbkDcJ4sQKK_OrihzSSa$Kx%2uCt7>h05?=
zTn^AZ@tdtGh0l?7(+8{}-sybp^??RNx<_(fb&!-)CCcGY)x%$c;r|4_1U&C~ftl!k
z0^hN}0^iUi>D$LH;83cy{tgjw`o5Ud%EFJLSk25ty4c}A*Qp!2T6?EIGtAmvrHI@M
z*pgewum4%RLikYZuE)x}$l5YtLL6UK0ID|+cx3&{lp=4p_1p6<hgo13z2kaBKl&!8
zwralXGe&ZiDQO$09Lepf-umY6)B4k6!7Ct2P`Vt3T1YPDGgiF{<+ng$n!aAth9ijV
zD+A9#B~yp0^VAuxsuKo&6=ge5w`>VL+_WWYVJ&>qWp$varJB!y_c_ZLU7`6_E2tSA
zXCacCsbrmQaI+VJIE{vqEbuJ`$y>R(8{<#czS6^6RCn>0)4nThg!*v8)|Z5OAd+P2
z%s2%)HeaZe@>FPHWrbN4&VyIe^;QU=(6*KjuUN7?0zZA-5ms7n&tehOpK}{G2Bg%Z
zi*=N3nP+!p-&6jyFV^`{`*Kd%VXB6n)xOo$5N6YP1(G_3!Lnb1iK^}$&=;ez;NKVH
zIIZh6PVFsvF@NKmHzjYos)vTXU;SF=r0fL<@BDN^yuvi9dE4?>$Y8zKeGqmoQR&yh
zpyMAMg74L7O=j#$C}oc1k`c;Hde^@(>sts^&-TovKApqs*On5NI1iI$vk+DXcdSW@
z{TI`QN!R^U&W{B%kK!QLKn*_%a<{#>?M(^o>s|@0d;}6Lrx&5Za^(ho%Zy^zGp;`d
z(AqKjaNF<otAN}8<hx#|K`BYg;90V&qiYp*_8P5VkeaoPQ;mH=Tmu&<n)XyxfjBkG
zp9^!*gJHr$97JM;@PbA4y3|NW8d(Qu=;ZSl<AhRmWcvBh=mCZ{wlvuAy}L8tgc`!y
zdKA!8JPZ$M-G9>(&DHR`z-w3FlN;4whIx||sdXrJ-b49ew48n??#tOzxU*7);GjML
z<!(gcpINsuQ^=U2ybh2OBn5pBC7O0}qsuV;L2^td55KuebFrVrZ;S0D=vaTPtfbS8
zjRBv}5EB3wVBsfXUxl7=tu);U39$Q~FN!AyhW%s1rJbNk9Z*K<R7XIha3co@(U=B&
zMJJCizN>jaHYzK8WyM9N^H_Glu7(aN_|}zm`wphTo9z&*ds4qPJ>1P=N3Qf;y#J~W
zU(w_~*<)pXY!XLM*s(D?GUGaFokBR5y@mdezVK7mQFiTHqMud|l?6~IAM9dw`7ht5
z<sd0h!(Uco-v!!_%k;!v_PxT&KuLc>g6<wO?N6qbvfSU}HUFL|^+rsySgNQnDeFeV
zTF0?Q*4I$b!L<Q?t@od+YjSD+2d*bP$t`%$5ly!e^LNw5`ft-U=G>u9$ws8<a!Qa3
z7=1@Yb2hv0nfdJOzxT2VH9$LN__K@RNb36jeDY(lw|wQMjJt}H!&3!Gw+9;gzY`lD
z$xCLPp$@n0i-b;P#;&C&v5w`)J|teev%%L@*(agfFEE%Th>wDn(Z5YfKXxk~jjL4j
z;RSzwq1F#8MeRkd<7_f%nXy+XCYMr3!KL(pIih<H`?%rB0`WD_WtLh5O5W9>vP8we
zi`KluyE^+N9hTUUa(K7}N_p){8tOu7vx6*c4ps)rwM>wdM3{=aT+OEp6`pt`%oj=A
zWjMijBh#<qG?7?>??{mgJuy!V5~=+OFUB|x*Dt@U{;0A20bi@pI*ap${{ZRaG;M^F
z<VjHRvs0o>F>$PWl&o_|-gxJQ0=!O2%i=<Dno@dZqbkE<Sy2$UJ6i7|YuiVIp&jW6
zvX2VQtRze!7kw0U#~RW3O&B~vJgb!<A=4<L2$kyA_XEMmYf!6RtW5Fhw`1_Z^y}BJ
zZ!@7G<-IE5dJJ%xvQ*^zV*1$Ih?9F^e1ZyAK*KTg<}1U#cc-1!<KMMKukQ7bp*l@f
zh#w1QE)arkI{p<CfgunX^UUG+UAZNa$f?3jZ(yJhkIzqX?7mn%p}nLPs+3BH4!86@
z$nm_5&V!&jDt$UdMCgIY%L#5h3*5eu7C%2Z$yRaRFE55IbbSTmp5?9C^OFJ@lU!1D
zEp~b<EkXRoI9OXNbF-uj6uP*_n)!b{Oe9G^*o4^K=Jab6Uz7LYhrZ;_(s9~C(i2UB
zxcGM3%YetqDf1T66yMJx9md^>m&Dx~7>*ZrU>_r()n`?X{dDKI^?15_(V`NrYu{NJ
zoPZpH9}yZ+wRk<m%3gmLMJ^LW{?cJ<Mc?;0;bl@k)QOx~2&V&({^#^Z2=quHnNXZX
z7-KnufIxcmh42ZPSpB>@bg=Dd3!ed@9<_eNTTBajZD++4w(@qm!uOm8tnDFJ%tfWR
zU1E)fOqRF%0mPiUE5GGgS01tbX&2Csey3n#9NU9Yc8LM@ug@x6VM1QiVcI0;;)M^x
zj%j?u@z$hDp<=Fy>s3CSwR%<(9snfyep++P%wF|G_P37gbUvFW-FlHS8*dPNLof3(
z5(dS>u<9GWsiUdMFw_<?UIa?U6Z{9-)0uNv6QWg7`~~f||AO|oY$1Oq+Rm^$zBf}X
zoD(j8K-(xGDMq{W3yaC1f299$lr3+6*I#Gt15Ex4c?!xsck3Y$E0SYOQ7Om-Lt&yA
z&|CbE<#!=e_)RY6YXU8GuB&aBZ_;zf?<G31e(e9w<?m-ADgFZBLNvsDvLT8z?BpsR
z0)uk~{47*#h}^_2RDjXoMPJg*{!wKZ)$Y;OBww}#UXP%GF+FlH^8jGC{;X!Ei&vkC
zS;t@;<@SD(P$K<8yXW8v2=qMKkZbKE*Tw3#Tjcg`z3stxt44-5*bmawb;g&joY}`*
zx2K@<eS3%{|2)Es!x&?bSaS%j*wl_x!(t^#dUU)Vd78IY%}!2v-T>si6;wPU%w8xy
z177<w_h^M$hv`1qP`}D|I8#o@hBv#ajPlXi1D;2vKjFMV$qS6nry`&7$(mR(vcB+_
zbOL@AC$x{w39lEz-uat`uHOGu0#fx8A|6_6*CdRFSH8Q<gu9#f==v5Krg~go9^lJ~
zOF?u^24W{YnR0?;G0@DBN%PCP^IUCbWZNTI?m%0Z_*%KEFC+*ZZ)BE@%zme_UeTd1
zZSsY5KC}O=G*_2fCY{^RWijynlxAmSa(^yEs{e~6egG7>ziK#wMMCTFe_sQjFYo-Y
zo5oAI41M1vI^izfPEn@2fi|y!wLMSv4@(R?1ieJ|TG<0AYu(J-Rcc*`W5M7H3ze6h
zlm%G#vO?~YZmf#Uz|~1FT+Trna!E1^(Lg>{Q3UkpR6V5@pwM*7${B*gvL)xuw|n(x
zkxi=%YF0PUxWEo!NBV5F79)PnG{0Qu?du)j35%@5dhotUW!(pD*cycYJrG&0{k#)k
zm!Q=&*g!tV6A4{m!(#xuJq#6&tqPxXXd_cXbfMiD{z4Ha)bKnUC`_$&-PY|0?PnBT
z)Qy9umV*g+;4rz=-$`Y55OjWQ6#Y>3Uf6-La$WYLa+<UakC3Pasub=}Vzc2*r%R=y
zv8*+oa{wCZpG=-nK$^^$;VnD5xVT{LO1Ps+VOcAW=0!m1nbZ1}aS<zV66?jnbYm|7
zE%=l$figRf)+Y3*EsTcg&@X~=gFx2LvY4WHSnyf*dkft01>k-o;hCISo>|Hy-Tl(l
ze6b?}?OO8b8}$ppCP~~tRWYg}LF2{x_v|ulF;#$Pj(`dr&*&HxX5)SP=$-;zC6oW*
zyz*!Udx5m4D+~Png^L2`muCl=DZd04{?86ulZx79l=gRr4QNgF>C@P~CK26D6?NvC
z^-AWpysHV2<)OZmDDYq0B2a;np>*+mJf5w^ilbYBRapv+$ET{4M@GnmPQFSb;{VvJ
zFn?23NC^c0kF{`+sz34fnrB+e)9EOklq&Ex<ji6i^;7pw%GLr%=;prSrKlH1_iI5x
z$LTMe*lkBtV?T1C_&SMR2ltC%2fMykk64DSsG0J4iI%}@@YHJyVWF}@U=QBwB#g=`
zv6tPk)RB@XETfYyE39C}oZLdgw>gj<Ah2(b{|>l3W%bqw8+F^s8n(EgeZ}Pst`d?z
zpG)N%zaAa+C3A2T9Rm!0Z{yG!elEZteY83Dqa#ifjwwf{?LiSPH@}bZiLu_N-5)<=
zI+tmgKmJE$sraKY>;tcpokHHiwAS`*nzF%qwkUsB?pt-G>wYKcb?Pr)kZ7GWlBk#y
z@25Twi?{(lK}Itg2l}ZdFMO$eteX)BofC=ti~tpm#Rn7aDsS`DLgM<LRsU}7l!Hr&
zlRY+4aB6&l{(z##Dq;$x_sXcd@&2w&TvmnvZ5I>k)o7{CH|#f^BzoULlCW}o>NL-8
zvr@{{rs<elJU{67v7!Dz9X|;+KTEvcC|HnW`0~M!L~Z--Oubfuwlca6FJi?Npk5`y
z0hNrY{pc3Y8FWGajRC^@p3bNC`-&l(^C~guKGY-KE@5UJV#N0Gk4OBLj|AZR#H(f>
zBXTpqG@l@zhBS>%JgVb*DsBd_BXa40WU9f@^4eKSUCiNq&#QapJaG|tx)N<CCv^=N
zO@PbSsFh1h2*-m6=5#$M3i2LR@%9(KSAW+npA+fxV>^qTJ``rzM{H#gV=J>3wF`(c
z7VoTXP#J9vjTa;Rl*n>J%xvE<KUQO_#T*^@E2n!NRaE@_(mBueP$KML%SHDBm?(ba
z%j08|2+MYawa~~#OFBxO>RjoN6Tm01dGV&6GknY%7C5EAozLzrWq}%NtKra3FEnR%
zUqV!E0ACuxBxBs*icmm?TMq5aqWvMy3X{bJc-7aBH)&}Mo#~)eC}?A}#L~Kq4oTTq
zCb|B70edwYbc3hQ(?9rtaRE9g)+~I-c|uQGKoe5BjQxEJU~pFEwNe*wcn&Ob^GM!k
zIZV@B3%Rmw0}33yN0<rg$Bsm0qOVN|SXuugy|li+?(w-DT221KQ9tFST$ablHNGGI
z7XR;|jPa)*lf)&RN9Eq-HE&1z%u2q}fULp~_}|joygkt`_VE?oLz_TJN(QH!ZzLt#
z5}#${;9D`|HBoaOc6#t{`flQ2<<-^6A^ry!S7H{tj&5Gr`Qv(I{96ohF?8Xzw*TRc
zM`*&oV~@C%r!XcjOKr^*-6UtH!dys2G$;>@5i9SM+Ew2_{KGazry@8<A#V^*GNXGC
z8bqTlZgzmJ0AKG0s015GQHa=kk{Ax{c>>u-us89*$9(_vviv|jspeIj$R{l&55mBI
zBDLh0^4M4!(322V`8!3|=(v<fH>=>jj{G=LyUrG`AxGVB#R}>?3S*LyZ?Df*^G6Ak
zA$#RUb!GvY#-|{#W?;RTvj-0_Awh*%sNH@9G){v9pPG&h+%aiBJq`#Tosg16slSV)
z^k$VgSFr%Ps+{`8^F8b{^Gk4xltH5I2L_2%IK2xC`m?MiDpfBebgVQc4l&Nv42pZZ
z{^CWMlXfV@L!4?(1z%^bYhY>2qpKx5&jQ4^AseSYq4Tof?@_I!yePNFU3l6Sm7ckB
zo(oQ(HGr#4XvXn2b3m%ReL~zl6=TsEtN<(8{SKg3<pr2b$J_)rI-DopRshNCXEy8T
z)uK!KI<U@-`8bMzU=QeL$64X?2Q+wJ(j<lo04r%sJ8>=DTLOag`_zPyL*3Ut7ir!p
zN*sEhYC>MmHv!5nIUVI4ri8)9TS?j%0&)-1&Z<dzV!SQ!<5sb;Mb`w9ozL7*gp=FB
zw;-BFm%svdG~RyzEHI9Fxg=ZarrDXJ%Y^rxQ8QK+MQITPP`lC874H2JN00H1`IZ*(
z&fhn?aW{p)zcv+woS_x-<PTUYjLwyzod)DRPX~v^z?M+(IYzsaWTgZu&#V5N`-Gj8
znd^D#a4AJaM4=K4(J9JgKEurn_T11o2C8^|o-xXcKi_Uxbp%`>_E9{fvSO?n`dvqU
zW0N+B(naD;Vs}Ydkh{vh8<K4@hvVzcy*SI1C3P1M?Wqid6pof9>zpeH5fCRo35ex2
zx(24YB+y!s>(N`h=`l9Vb2cV=LpQ{*+ncHhr#NsFjGd63wOVq$A9YfdAIqKrzIyMy
zL5o_$S1QqU7*NULkBhTC6BxAKIX%x-CSE3Lz}v*=T)Ip7M*ZZ#RrkW0>i^9=-6p}@
zrZo76(C#(SB?vVv4U`m^5dZh$=DiTQr-Z7NoytN4v>yRX1RiGx?5-meg<`uN1t;$3
zEBQ?TIR5<?I4%)kEBf!%t-PI%XV2eN9e%f;%H<-BbSc^``tUf3M9yN7arN(0AO7#v
zT}r0Utl;QO*M49>Av8>`Z43m~E9v8;C!@vsT4yYS9|TfBnb)pK+ktTh_HHCBKO;kl
zHRRo9MT@v;U3*3)Abe0M+}r#4EmXqrsMkOZ6T-~jk`scIQkFvC{ZZU>YQ6*?hI8+`
zM_a^TlXcwy@t3yhHe#pT`0Eo+v#R^y{9QTeKk76XU0^w!x-uAStO6|KuaRHm(Rh^s
zV6jyMT;xtWpE}=4f8Lq;oH$jxD+!#n4#18b@HtOBTXtG*x{_*CTKAKIbvjmC$sssm
z)fn_<R(i*GujjeX?%0u?_UOWbey~HrX19QzQ;+=#-Q#c)Y7`)E%3#2guJ4!t5zDu~
zGLh4-ZcRJ}vt461bqQRz5LLrLtVsEBoO?#xcZ&yaUgqLdI?fuf>$B$>IseqrlS^=L
zQmO(IZ;|tI*CAh$d1~qse}=?BE-Z-KCew}dF74M`(HG-%Fk}^AJ>w)PgA`ME;$<ZB
zoF9xupGO8jX%Oi9pB<*(Ums-Oq!u$8?%asmkmLojjy<yKOt&OKg!Y=wxU`zjLt_-|
zK6%y7yuROb+8(8Q#Yk|D{JPVil+b7NYAKO!xj$1R<${;BJ6YX_EL=zZ9zaV>M;+)q
zGM(D7#)ObD?kVAu7plQw4H)4DbmI;BfKoM`ZLsFhDJLz_tM%gDhJ-_#JWufA$ml)K
z*8?Ik)0KsCl)ro+xqEh3%k&w`6@S{G*eYa*Rq}Nk?2hz%+nYLyqZS6h5`eVjqSY)*
zut-15g9VBI0wxVjhu)bm&d~+XhG2BT_8DKAy-Vq916h}ByzcX%tW2)PD>Pq>!<Gs^
z?F(AKxm*F9@C%04#ah(!!Hkmgds^3=hEEO-ro|&OQg@$<o3o6kZ{DMK<*)!yaO$B=
zf2yiYMgl~ryI}Y8kqiQT+p2B%K5I&EOxNDbQj;c8ZvW&O=lOe3ZBTYSV<aWv4WRPh
zUR>=*50%G0eJI0%ski;ngiCqEzsfx=97XTgo+sFMGA6e74}^XHKgzRZP9V0by7za9
zRr$Y`*xUD3{NF%qYdV;*5QVcs`yU1+o^BXpM?<L3*0(*jY49pH#09U;xUazltgls{
z9u&6zHx~!U(k1`XczTHJzC7KebG!Ds)X71`Y9?0vyW-Lk{O=77!0&wYg*EaW4E1MP
z43z-0v2H#rsJH1XL><mE7;(?pDLTbA3HMCfHkhc7-vSrHsK>L|@eGZ}x;yd@@VfGa
zVL1RnbC5&^Gh#!$hf%d?%VlDctdAOq);0E78veW;Re6j7L(njlfk4l&z7#H65|mPk
z)oj0Jx5okkdTh+8Keyj(&#7d>-ZIiG7CcL2)+pPOaX`gu=-N`Pb%F?RjMN(otX59>
zfC%*$pp9!qJNFA6hd;kr?X75C_m>|iF!=K`N-Sfm2=8u@@IB4IAAF<vU~PHNWnP@x
z2r|bG<NS<T#}xtYXv9pHHydziRtrWsIWa@v&5h7ZtqgOU#^{BE_TRCT_xrSQMM^n}
z&#PHGDaCJqQMXG6-nM-!Tx{!|v-NPFA+{&LbFO_gpLOE88`u&BOF+REqFWW({`kRB
z^Y`b(6aw@`!jvCCz`06zvS;2=ERYCUqQJa&+>Ey9d_xAe&;ainq+`?Bc$tx`KZd`s
z!sAAc7Qae1i5GJ|P#jm0#}TP_c=p25O@FehAg*2Xru`uJ*is-KZFL~iCPB#=H<;&_
z1Yyt=PM3z)mH*ZY;?@jj*>I?2Et<okI6OUu`!`N@M7K;B-a~=_H>_tdnYU1|rW_E|
zET@!66ziJ9L1b00|07hsbxXf9UR^kXkkttf$h(GkfkrZ(3IT+-pk$`h??pElK!*jU
zuXk1&17EHl<1oTNR*mhi@C04qo>@?#_H#lUbPpLSd)6`&RR`qran?Xt*8*}DpOYCm
z1!#`6LA87vtNp_*^;URJEFOhXl8(Ccz2~%7E!!vFy$6D?ss<lcS%yO~qYiBbQ`?Z~
zd=KiOCEUKOV{~maVxHf_1>`3-LPp7hZOC`sRAs1T*a-TP-8*3Zbu-FrW8@YCE2l_G
zWPrFAE77R(MQu07t!So-R{Nh0qgYU>9xdRd%wHYG@_#yvzpvr#?pT2TPaZjy!pD-z
z*D>=y<}_S#owxD9N-3osJzs_4V_|n9hv`RgWZc+Gtt&bwuT}mB4@naA5Tj`dJ=Oc9
z8z^q5JxhfBXmGB<on|}Eq<{Q?rk?eG><xj`U-C2-BVp$GTo(P<LW<Z-704(9{z}KF
z*U?&YFBY?}J)2%HXMNnALRgB3NIex<M~w()4{bgJ?RXsIh)^8~JnL?S{+)qLiIi&^
za2xgEy_xH^!x25TFt^hRZ`itnT}a{vuU3}0JZaHTb*J-ceI_^;xf=d+;87>#CTO+P
zE>WER!t<U2NF<J*xEDZmUVamSRCEKcuLzN!9+Ca)_PatO+}9J2%rIWDDw`d>B}h~i
zJJOb$VH=1fKR#R?$d->$bu1XYzf_b83%86FdRoG2<hfqBf&L57weB_^zi>B5xe17K
zJWyY2h?apCV$EaLm|o}?Hm&O-ZAVG=pm(TA%(10f@+8inbWek0H;9S*I)7ig)U~A8
zO1HwKJY#M)O9Bn2qjYC@yCqn2-kH&o?z=J&EQ2HuI?)W1MR;a3WrCve^=afC1-2+M
z&X3q69I~BXK@&mx#ir4@^)Xk?+k@xrJX4y$d$$%Z)A#zyo`i4)?I`pkTzbWINtY=h
zO=9}1lw9SVNoI|p4nbm?8e1FpLm!Y}UYYkj5?Xc!d&3a%z<$99cSQc|wX#R`kpu$l
znr4i;h;-<EYj)d~5bDX3kqn_011l!ha>@0@)cm*zc8R4{^4oM0OMz?<O6SoNimMh4
zLk)8#2vmHJ4Ak)uHh;KwAc&+i^XgjP5`4uWu@g)wUoX>#7zzDkOR(j8O<^U5Hrmuq
zxx?sry!DLkp$EnJzDB;h{KEi<<NTN8<&45ENrG!hQCXqy?~|PA62&p`Tc49@tG`6o
zYy(QdE*k<%ES+rkBTB>XsbA;4uW&!x&Gt3y$gPbc(<d|19dFjYeb;;1<rgHj8vaIy
zHB@<ZP%%*Q&RzFTTR?hg`F!ck@BM~QRz9CjkS|rHy1e<e%ax!RT-UT)1VBC><^W)?
z%P|hVFCGyA1?g|#VPaFqEa}_jts5_B5&dKlqxu45?w5)?@$FTpH>3O%Fa6RR%4Ts2
zn|Il3m1;O{u(w5q6OOs9;%VEHK03`L`Mq&axTZnqgL#e<UF$(Axnw(4gT?V-?TrGz
zcji0Y_ivYD1Tu!@sr?RcYvq|gFEMNRqJgZ2dQTHcMX8&V4<E`D!qf|*3jE{gnF%F+
zm5hX`JKoZkSL=L5KhnUXbyo_HJ^14#^8oW@G_5A-E(c}C0K6ZSgzIZ=#<-D7Opyi6
z#&6J;^Bb}5L7ftLrvx$5|E>@%(8<Iki(xQVEr=ag!Btq|e7CvjqPT4zmvyYJARG$+
zh-M_<Fa^;ATqdKbN9`*Q3mGlhB<(bS>`3-7fB#1f=d`_V2iZ@-f3u&e{{PhQUzsEA
zKjTIXmyIg7+DDtc48314aHkj%3c_`totwEs$L&{seWIpspzHRL<04}I@a6VtFS#y&
z|M#@F$<h6^JWSyz)vCu{`Ozp6oAP8zc~g(pwT0E@@8C^<Ci-_h{<$1rLqBqwM?aSO
zC>@I*S|vxFN{~=q-x!qXjliz|0rr!-AnZ(EsA`cCQaP)Od^5KyYSLmZMr4ntQ?&Y)
zb<9lXBv;0C(TVUkx?AE2x5L9XyG<t}U;wwJW~t5(hbk*FxO+cY*DKe2xEu~u4^8bF
zj2k@1kXj38jO|%fpvGtXYJuC1M<WVL$hMfZnM!(g)7pj_th2m0la6ga+wfWQMZgFO
zq+vQ^zDx0W-_1t?-L&~{s^@eIK=07jK>BJmMqsWWOV6}C$k}|sAMXz-BxBnSM_}HY
zuL%e{bzNaiO{irysej7uZ?s}tY|EO5RF|&HP0gPvH-v|L(0~UVNR0Y_#}6;WKw0ZH
z0qM1lsEd(mRo53ME!kNfOA+27ZR8Vx$JrEx<SbfcMoal1ixpCj$CH1$0)3Z_94b(2
zK<!=idTb;z<5Mkg86<C}<FHJQk8k*hY*a<I`8j-7W93$yYW#kZFgoE}I$=eUtfBgc
zd<cM9u(0APsUy&xrRt?h9#4$K$;0oq<>Qt;pZrG4af9y1;VFJM>@WmAv26V?)h+N`
z)3Tb;OglpZBf+1}Eg&3?<!up&*y&>{GB6(2Qj;hgZOd3}jFoco0k=(g#oIN%+g?XN
zSSdA6gG5Phth3KCEfJ$F5d>F{!xKAg)it~<TJI!>1diSSG+0u-@iku*jHo%a*aA0D
zW^o%>h9e`N&>*@>v1hEI--}s|@c%!;-U2G>u4^A2x}~MN5hRC_RFII69=dZ#1qCEU
z=?<kqN|2FmK|o57kWLY#1VOs%f94kN=lQ<x{oenYwPd*jXMTrs_St)1*S@YNLO}=b
zZk0u?YZ#;LD#xu$pPdW6GbyRPVab*@dB!kWC!2vm_2tX9%QmnXDHgFOwEJBdLBVnX
zH1LX7F|2E>-^2H7D4YIj0Zf$c1LWSIuAeg>^sx<q%UvmX_G6BBs+w35tk^vdAMObf
z*<S7~>+3{+4@fqEcz-J>W9pf4{h-EW-HGcA(t-~T+Vo@S7ThRk4pVB$fP;Y($RQ_4
z|F_ZXC0hUiM<Ry^3GfP||3k=TD)<MyL1H+=Sl2x$>CG>xq^jRji)B}#;x|D@m;yaU
zra-v)(8ddOc3DFGqG1W*2~WGq(~Fu9lA!~G5ZlS@7=K&Gxae9^_hYlPlGjHgO?78+
zoVIF>){YF$GT$yEk1dh{jq(qmd*Br(t2r!)O}k!U=s8I)kfErW*WlG&JO#bXdmUWY
z8$7mW5)Nf}5e~)fsiS-1)fNT-2J>Z)4wM5D#G_4HH=dXG8by)~<_KP+g^Ij2dLh83
z^IB{d_$ibD;J*ewAt)RV6Vj}}tPgBLx6}Gyad7+`@za$U{^>Z|+m^I34TnQ?&O=-J
zJQMnyd>Blbs(%l-YL>e7aVz*I>LVPzy4Y$4&VE%*Ziy@F9}wEIL;<lJ^(@J7$B7Uv
zlJo~>>76C(!S!g0o>oe4t-5dc+E2$fd@||?lF_57GHEA&LR4GVBH}y{tBeS$5>C<x
zA!I1UmKhT0(R62V10*a`C0gKu9A(X>evy~`;;}Rt<!nah>Xk^gy4WT9w<xi36xELv
zq|VGZ-djO1wCB+RMKC#xPD-7p;Ql9<=4l(~BmLw&{UWUSy<rWvTqh2$URM(tT$M6T
zK(-epNH!fc@~PF))dPd@ynAOiR0O_sEY9B40j2Gzg)X-_#xlj;IpB*?0D~tA@x+Q4
zIfS@b{2A`yIWQ#LaSw+o;p5)8XG(on)Hon`!0@<i0K5FL2E)3nr~nm_U*5=iWdwT0
zT#du|p!llsZ!C^G!9Mhg-Fd-C&q<&|Q{;ODF86^;4XhZGtHKwb*IsByzllM3>1Tx4
zJq=_Y@96qoas{~BU*Fs_iOdOyKBO3S^(jh}2bF>W@Wj-)Bi%SK<lEuBxP#@$T*UGA
zoB!D+`q-Us^X^cA^9KNA|FG4p!PuiP9Vc{n-u<xQ6wL69o~h&>v^@PT?UUe+g!QGy
z^f!vf%FMFa-DJjdP>!DA#I!ID-=I5vw{9gcezODgRD^6*-wLJFYdu6u{W-hRKel;z
znEmtYN<e%k92!Qu9=zBc(Ba28ji2r~+-S1M*Oz(;a`?4ZMlPC*uWu||PJkIAPoUe1
z!!@3gA{*y8R9PS)yoOiNmBTW_$Z{>@BiCiv%Z_bL`j3#$^x;23K3cf{f~I~%*ZQL)
zA2Y<;BvVtX9-Agwfq!<B%W?Q6xWE75L_fU-D^ti*=G(A;$%T&I7M8pDmv!9aPuHYk
zbj%=~*r9vuI&TuwlMXvdzT3>$_lllLwtrT*ewROtrvtC~th0TYJAmZ7J{nJ_68$<g
zc{z+U<544?JH(C>vEQ#A0yKyz+ES+-9{vcd;wW4`{Qd1M8NAj-&7WS5SW4YI7whd-
z0W6q&Rp#_3OR}klt*3#<<;K|pZKwvhVTCQ-t#Qr~V!a;OPHJl)dUbc_Q@v40k#5Pj
z<Jc43Ce}*;CB{VH)mcCv7v;A$(Du@TVmL2>P368W1vb=;zXhl$ke#3%k%#)Vc9a9T
z>XA?ee^}1a0!-%7l9J<4u6Qz!gVH1uyHuRk&7U)Me4DckLfkHDh_%nF9yFDgK9T{N
z#6@U+6d3~`!AifX%;>>J2RuiGSjk;%6QI+kQ*2pKWGAmLG$C}<ayF?++aW4h=k4uX
zY7@M-4Q{U~W%>@%Z}HhcxwaG?X9C-a1M3EDQI^_RoVZ`LKsD3~7!5JU;m5P%bED81
zyyVjS5KWGk6+l05(s-5nIc5(Ga9YgHES%r4iH^M87>6UW?6h#Om!t6_Wxpx}DK;=#
zby$8k<n;oTX{{4iGIvkQ0cEW<*z<fz!9Ws%er_bC1u(olfQ@5^>h5OU_rhBS6?r*>
z0l}C%<nWE~Jc9G@R#)8vE4*Y=?2r1_y>)n(oM+_;Wp>N8CL#)&hTh!6I(>n|&A!Fx
zBe#r?9&GI_O=_<(FYtZo$J$%0;cp1*CE+-yy%K_{Uo=lAV>izhCXYw&Sk1k0`PL|&
z$2V+6q%RjSQn=@Ep_X%Xq=n!zsAPS=c$5gVj@<Uy$fnjZ9uFwj``Rbkr*{>qXR4^B
z#%&6#ftL3hw`UU*n<=67Ez@PE%+;|9kvE5sQ(vSU>N?hRcD~`X!BN0-d4lT^12)~z
zK{TbuseK(dIm<^eG{)1a=SjfM=ib8<v~a=>gORJ{7NL%dbsYtgwh!tSs-*BPH^bS0
z?W>o%2&y}We(S@_*+T}E`mvr0+0~aJBR~zJOCD&8ZH)q=g=I}P%cr+{9JT2<PzL|~
zH@Gw!1Ry^k!|ddl&jv__LZfJS*A%xxm&1tH<miO}nQNTrod$7rvPT9)pOyaB!7Oq~
zs^E0VN+ZXf8Ca>jKa~kEv*7WiA}_;#4%IN_U%ZCkhLBB4%|ESRbYlCn!Ag&-FXuz!
z?fxm{ujzj_d}EP+Vgf4RS0+%-f`15zvP9V2sKxYo49}`aYV6I%Nk3kFaJb_q{-n%~
z@^2M9AM+3XN>4cZn9mMyAMdwvj;D+*$stmu{TeZFX(twv%mAfj`S=c?11@{OF|b>2
zDA^%1hDZEnRkj5EsY0Q^-1KH(f--RgDvFp#-IDGhjfhiG817h!zJ3RJ9R7~$!l#!J
z#@b}wJuif1?InTlJC~^F>llf3&VqUr4*xj6!U}EZWdql#_1!70WJR&Jbq*sTfJdZ5
z6Wn_viD_jpyDUtb@eZzS@m;eY2Y4u)BFLs`4@u}PHqxl7!fbh?xf;B8QZGfDs*p+=
zF_ODV3Y4MKbU(Uf^7V;Oy%!!QX9)z3Vj!qRT~rw($ALg=^HpG6Y{f6onjTQ(S07AJ
zDn$?KQ@9>Q7o<?`X#j1ANx)B2x<j4%0ZgOEH()CbT&M7WNh*ImZe?e5uk-~h@uMe;
z_ycY9#(i$ZI;l7MH=~j}aq<qojy&B6^A!IzfQ+M!DoLn+<0uI-j$UNFSK0&;PJ<QK
zvsoXku;OlA7ZL*3Rf1d}>SrMG6G@DuS>LIjs3CGAfg|uLS^o4e{(;%A(e^;bd75iz
zaM7rZl^N_FPCc-#Dq#ypyzVVs*xv<NiGi5w-*3PBcVcYurls2Hc=QsF8u8DeI!Tjf
z8s3Am(NvY`rry?X?NXc?kprmd>l(e>*NqcnWZw1t_#sB)`Kw1+kdK#3x2aE}WNzjA
zu(MM9vqspAsQ%*a`96}^jQd)wayehIDLbi2<H6J(2_Pg~@9h740k#>}RsicN+ycn+
zpverRF%`m}tPk%3ioMAVHrwkPMvn2$zdG8lUjDoCB<@6XfL-8oCT`&UO~(Qd5wt~s
zy1@0rrbfxW>;74Cnv`dH(VTqwA}*tcS84Y)c%07Y>+K&w!=Y?Oq@;)rM7)XY%FuBt
z*y-TPyR848b(XvLSFj+YEJ=~*GjfRqDWQJ^3-ZbSF0btA-|3yVe&UarJE%FBAMpU`
zk@-?N@?xvldy@B>-M*_+i-Ob{Gatx&7driCVFam=`@_U1Id3Lwn|<6@@DU+FBV3Wa
zozwdt`KFx7f7-)ii*gkxtT=e64pFTMY!8Y+Xyp#g85b7cQ%#fh^PygaRbCU7|9hLr
zuKdjxQe%sfklA?PKlL!(F>=ECFIfvb#{aX=ja|_d9!86Y>P}RDwHz^em_65##^E|6
zmmNDr%2Bw{KnwQBGe-@Tzk9?Qxkt>rPMACf@yMl^&L3^ipSGmGfGy%A{Gom@VqR~N
zuy{XlUTkY9DnR}Sf;q>nJs-#Y%{cy{g#O`@5dYOHRM2r!z7IaORuuT$g_QHm5p!h%
zxnY&Jdj9=xgMV3NE4Mn*_TSU=7^E?8Po^gqDV$YiH9Y${y}AF5<C#>u^++dJ`zf6O
zExkYcD`=iMZe8HCN!?~Gej!;B<=o<vsZY-8lR^GRntw`)R=R(<Wm5F@tEL0Es8i9c
z2?Y;|U0hr!Ul3)2WM31}^P8Qi;96(v-;xJCh*|b}<3!|qPIv*r3mFQ?)C|xWDdb2%
z15_bSSNR{BIx_5k=KkA@5V2qy&v*T%G%!pS^!EFERW+m6JAL1zzSRzCnEl?vPgJ7-
zs?Ijr7s>|L!RdJ<a<AJ`B)vh&<9>!Jwww+oOe(>I$x`kgd@%6N^}@RiHfNBdSJDKj
zXG&-SE^o0sj_M!@n7@bzN+&7ba+KEo#=Ac^(Ld)^N#HuhB>woJvJbY&LwO|qfEXq7
zhxIu%i&lnOxapsT0#70F?^9U)=P6dZ^$WGryh-7|H!7oT&jCY1`z2_js7_hm`g_={
z1(Da5hO1@>X6zpOcwbsmtV;_l44OI{!@7lhPhX_&YyA37&M{F!(*N)&ua`ZH@j5k*
z&*;elPY?dzAC##*H55G`0Oy$X{?zqIJ@VUSuG3@x>Zs8UrjE4HdF~2K${_uZMRy4_
z_%;nbrSnaz0MXhlD)#i-m>{4Ro+egaM;LSp<be&4sS<A9Hz3!5toBc8h}O=mQ{Dfo
zIaJRA1hf_3b)y~4elhqP%C6~*|Mex}=5+<lH9h>m5qSu?k>N%6^G)X_<Q_?fEC1W%
z#+LruB%gUt1Rt(*VYPFJX#MqF?~T>fZ;-0`UFw})P=E##H~x=S+g^5~zi-Q!NUU@~
z>F2vRyg#iEj<G_^@3(Lx5bzRb;+IUoFE4`ddW(}W+)b#w-ZlTj7vjHJOo9+{C4*G&
zGZJ@yz8Siz$(iI|u)jX2>hhgde0T%X2Pk0?+sFN@a`IeOfRU(Lpb=e2)E}nLaytR3
zo<&nAhKhc1p&1749D2vvA9de+DOQ*3=vXVu=I4|p9&c(utB|UQ3I25JD>Y9Jo4*;3
z2y(0Z%7jJCoxgd<5%$GjMpkOz+mF9X|4;{e|EE!w$hl|S<=!RzbW2&07O$6tX`)__
z?}qAu*TZd+-b}F2{~aR#>u<*ZDE~+B>sVhfTp77Gq$ohUgX8qFU~__U!7PTA{qM5M
z5<yVqX!3xxM_?R*<r9gti7ku7@7BKLeNt<c<d$lUqc4_Ntcbkq9Can$m5NrZ^j6h<
zWr%zMv*L!@qos4bcKW-KgH#ofZN#DOSGk%p;GkR@KMHJwg8!nhj;*JKk%$caxBo|G
z6OT@&{fTTWhb5s8!&&BwP>19`Yu$I<!8z50T%rGe4`eO!_xw)TtNaG<*vc44-o3!M
zX2To%IWHKi`7YTYwCu8Pi8Hmq_OS-L3uu!HR%|5L{e4WvOk(V5N|Q>~p(A)RIs64o
z+m}}h{LVs~^E2LHBjuVeHtk(4`)%I37LMH--llLDu%NFFdY@vRWc~L6QNAO0pvkY4
zg#F)9j#($Bd)+9f6ApI~JLt;D6C)eXWF7`(mm_kTqi%|1|M)r!SHzS-D)D48b$zFG
zp6lPO{9i}sI-3B$pFcAOT<Scm&fr6D69&BqgDL|2j9_B*V0{q(?*B`j?K&v~mON3i
zU%P%LfX4rY*<FP4ga}7Rn#X&p9{p+N?Y0d^4;XJDzR#qL#*`{tAGe}QmaICcKj}g3
z-_0Xi$b!5&!tzvxCk?={Iw&C=E{GPflSoPk0G|WIHeegWrPoN})Fkc0zE>dxdcbX(
z|Nc0LYUkLXp%I3pTpND$6$e9(nTK27&CXL3Xu+r!^TRSzDT#~y5iUNySa!8EdEh0f
zB8Z7|8{3QJE1<f)Gi#CV&z8vs;sgpzJfz&nt3T-Pm4k}%jtqIZj7BWNyl1}*oEg#6
zDYa*tnO>xFX$U^<U&&IPMG9_OPO%OC`#cgX$TO<rkh?YRMGDXH!0v4KsLpAT!3C+2
zQRWiYdtyqNGU_q0*)xhAHV)^)X!uL8mj(aFz5GTIIfs0}KECw+juM3^*?;Y2OJca{
z$#C?a4ctAT6N`l@^PkiW$&T3uvuj#gbIoF;$HMZ}ZVPuog^mHHpv=}`m)NMOfvrG4
zL=Fs6<ZBfi-%%?<Uv}=z1A?4GVES5!=`5jJSTTq7S%?4W+FK6*M?X@(cW#Dr`dvdt
z6R@2f0G-Pr>)Aio1MBna!_Jyuhl9+p*J4|b;P8i%mTD!2-qitSr^%fqGC~b%wxv_E
ziQz_|t*OfMVaed{eWzrH{#@k#>)ZGCH`f8Kq$lCj6A`**FBcF1o>!t)qDR}1WxHo<
ztjJK!)l%&(smMqn!Ciy^Io1EPks43mu&phA2kJrTqV)MO-FOx3BQpt0W<=Pys5@cM
z!i=@M<vVHBDl1lY;93bz|2KM|$bnnYey&OU0?b3z;@gMr=$rbdx&Y<!8Sg1_^kIr1
zpnaSGxZoPxCTXudtET`dMgu6dWNk)m;G$yO%K?m4;YUlKeMOIcO=u7|?wZ7rf;|!E
z{x?VOLgA!BmZ$pYAC0s#Rgy#SOr78+cVZ7Wqy3ueOTB;sqQ?Jz#sqD`KN>8nA;iH1
z`|5?WwxQmGuiqJV^!TR79H#bJwbvqA?IPc4FFVf3gv~J2h1a?yD1I5W_=+i_efvLK
zBQ*w@>2O0;zP`X1ld_m@Tw|<}Q4fP=fp#4@aGZkZBy8g;_~fE(UGQP<)Y!~odP02D
zKWR3&13C3eY1`g>0OzIJZjjy()UwXNdnhq7X}C%Molf}WVO@*n!TOj3P}i;YIdGh=
zbv9BNN&4JOT>r<fAzMF)|FN@ez1={D^4o0x#-sj6x<uG4U&}4-oraIy0b~lGHc2BI
zQ(FHcg1P-IVBqbK>6;m>t<}=z0+V8Gz|cnQrB((D6aeLwcV$2!fEV;vRZe>?OPhe?
zjy8jyf}mf1&j4VT$pWxnAQbt@gYfVFEmA2}dKy8LpCnRx0UKE<QmcNdae6}RePCA@
z_P?)wWoS}PnZD1N7EVRsv0*+DJc96<viLpXV@qYvcd$<J0(z4aykD-ln?}Q+1L$!(
z@;9eFn$Zvw2iL#VV=)rv@+f!*0MyPo8O*0n1ezaw)*5=j$$UKtqC6uVvC=C*(tT)*
zPE!_}_2;TJcT-vA)!XS_AMLsXq@Q6U*Jhg=cumg3`8Z&mzcgOeOqk|5j1(GlPXfc<
z?zQdq*f5FE;D?;@Y$`ubu+Bb8&CfPGUTav*s>+U-4&D1jw?S#&$D4(Qb*ODR$39(d
zf%YC~6afnL2h@0saur`dnN-y(HLBY`NRx^8e#LYz<SS73N!YpsKGrGIb*`?N*?=er
zhSmEdV(LZ!T$3ylycR~xVn3X(yyNm<&9k~;rGIc)n0Z+gXS$+tEBzN3Nh>(pYRLw3
zxen?I&T{r%K(-q3nDP1k+DPc(=8RqD-ZV-$x+luLQ(*RUM{2K6=)^@ie=c5RdII>d
zJ6xWh2z&2ZE2#ER7(|g0N*^=Q`Rxy_bN9})1YG53#>lY;wnNt$yybw+UzZtst^dWT
z$7aohR_GyJ(SkC;-m)8DdF+FUh@tI;_BG&)l^={{bqmH)3g|oC#G#pUL=DzukI(#B
z<*m0epU%B;ZMhJ6^r=?VZC-X65v~1ur$e1#aNz-8vvnlz<!=BqsQ_Qbd}RI)_;0kL
zFYK3lb&=W(pZ2f>6`4c)tE_v%e8kpoux85ln+NU&&p)0pdCKNfQnsKjwt(IU2&-#J
z+UjpGyP-K!&yK!EQH?l^77gLXf?!O>*^Fe(YqbLd80w=-@SaJs&mTT`U+khdA)|!r
zJX0I}tOYpT1<C@piPNH8SXT_4A~+Ykf^OS97Cd0Evt3q699YLj@G!hoF1jC0?J=Sz
zIs}f88rUt`nhGj+g>8Cue1I3;Pk=<`0M!)=_Ts1yNOSu_u(hHAPlD{7QQhs$*@{H}
zjQu>VII25OoPxe+m+0{p*bDpY|Ezc4-<)|3sKfamRE#?3o2#B~&AD&AMj}^nmw%((
zDGR$e0Gxv4iH>bOJ-M~nug+sG=4usPbH9DqejabSGM1g1!7eo6^t-3yJ*X<!UzzOk
z{4Heig^(%Xmt;OUv&)6e2Q{u{VTKbb8fNdnHTvVd^u^rt$O%qz(}m2l2^3CfMNKHP
zZv1UVR0{G7@^a?hd+2SZn#6JulAVY0ou%gO9NZB(oH;NQ3>1uP&M+OkW)_wBIk~mw
z&47=o8BPiPwu#Ljo;`gn-m4~(>SHS?l$x>S+ueleZ)~7K#{2u?uCM2SNX64S|LXcG
z>zR(OgJH$@DAzCGHX~cP`qQ#|l{8P^s|Ek}3@M4e?t{VCZK|$jCjQ{Ng665iC}8d4
z1=Qw?H4fxUR(6%EXyY0Z3>N6pmK|=~%*|VgI|r14f*;r4l{_-Brqq18IX$L|#YlPU
zD~2dAYv$9;jN;y_C=goK7{9X3fN=l<rm;@mgHa1G7F^3(Ze+1yGh1PLN2TjqmF?iG
zz)ZPdOl2!yBUDZQb718{+>9L-A4`I#xsx~6nnuB4(~Jq}2V~DLb$u+d%#eg~2m%Nz
z5mbOs{c*{=2@snVfafPZZzwj8d#o_*AS!7o9fA+>vUx9an*IGSl&NV9-cnQlq}|FB
zD<pUO({DwUBu*wdY}zNY_3o%VDC#KY7(A3j285@iB^O7V-&nx@<Oq@hq8%RWR6Eb+
zn!IyC++H~_a%oV&XNXxHYjOJ4sEeu!o1ie!(eFzUU~xo++-yFahNE6k55hb&xqe)7
zuSeF2!f&EvE!Mv_^(jt~&ygxy<x@H<@wk_Nr6>>Ru3|JZ?we^fKbCqq(kJRK$2mry
z^bcRf&`UAnl5yI19Y}yic|N*n&4-dqqLaN&-%A`3PHDE_9WZLuH$FJK3t{Zy2Dn;9
zJ?PLPUP}&6l(Nwg(V$yPRp4i;c8wL^42<?U`pt{0-SP;2<%;7p3Isz?a`cE$ZV`2P
zjq-7DaM<dUWAVcHvp<ElVSRP^rT6{?gVkON)lW$G3(QZg)KDLMDnWT+*Ez`oPb{Sd
zb-Hq%;xSB>U5V(aK|uIa`9iE=``YA5Dy7nVOz-|IH^I9WmqlP3Utw{wyS!ta?CxGZ
z@NBK%wf%3ddBZ1&ML@o0(jY1^k}vO9OA*s_ny!}<cAEUm8@5RuSPluW<n2J6=aHRm
zIp4)I1N$|oUie{)zn^B&LnfR<Oh%%4Pu}7Q+1x`}k{XUgAA+B>HJH_y32%bje__M4
zmELRQ6~ww)9J9Z2`2o#xHwfN<KF&~;WOt1D+|l?;5Kdr!{mQSxtaOeExe(v@+Pw5t
z4s|>uBdb<t>Z5<hJLwC(P19zD?<3Totq34o*c~~q^Gn|1lnHX|nbxW~^^7K~8^L9-
zUyRArN{6&2`IO!Uu;mLu>@8#19fu+6ui&MGMX+=e*6yi9-r5Bp;r`q`U_FS1c<mwm
z$w&-xn+Fi`_igg~7V_w1SoA_bN1Zw(@h)|6JLfDnaQP<ZpWh>CUmC7%sGj`zAn(e4
zu8#`VzGFkbLb%83I1UkR%>jcuS5ed<uC04M!cXvw)WN^JNQ_hDGgkvfP;4tNei0VN
zz<9(n#L(dN0<CvZHMYIE@Dbs<nB!y<>9O`^<@xnJK<?b3mZtlAt#kT*RIC*b3LYlR
zoiR*i4i9!OYnfoimTK-Kbno><KLgWL_E5<7C4ICvmitOShq>DOY1^wog7T1=tK7)j
z5x9sTrcIzbD1)+ShfR@sFrer?LLE48v@yk!MqF<T<``C-rBQ$^pRx>1;}4w^mwlGd
zCEt_J0uU$gvrG?_*#zfAo(hT!ROw}?ED_^#W8igRjwjvHS3d`{8wufTg30BVOHzuL
zk5~HQRk_TGBiI#-Kkc$$c4Dk-Z!fSUO*EiJnfO~1tU)k0;s}0;#z|}8Ay5`kC!x3$
zx1BUYim|J3%UUiQt*Zt2I>b=WNKXxdfjdWX$@!oGGMV!NBPSS?W39{wz%Muw1r4*~
zv)cD@YHT<oKH?30G&vMsO{yF-iVrM$Q+@Sif#p@u1xegYo?>i~?4sM+-XZ>8E3g7`
z*wp*1FUPOt#28YF$Jg}GiP79#B=Wr(75Q>}!m1Q4gV((OLg<@FPu-8c-aTDj<5!sq
z^W(}SwCzVtF<(zWqRzZp3x$t|0iNiDr^}JbJ+Kem93g$mkdgAfY#B2ElZl^eGiVnD
z$!`X+I%=@L)SWPJ0K1x16HVqlm%;#9U!ChjMrrX!wT-Ely6QgR6JA|x1eFtg?t(ih
zMdHk)ze>L#pgGsCxB=?Vwc?G(I42!N%X76lScjn<<ZnAJvtCndzS{hyc%sJ5M^>^o
zO8r@fAj-r@g^YXS;%*cQ0b2Uaf*_~#NskYX|DGx9NX!H&#YGlWX?CxkHZ-Q8{6{_?
z@qLnwz$1L6{;W>GiVJI>MrpN`jrFq}ok#nz&Ga#?K5wE>H!e1_CyS6oe{rcGNhZL*
z;_<|7%?$ixrfn4ypjF(ufige>*bxTCk9OFs2})Jbz+k+kfb+DlQF(TF8*0ca?iZUk
zFXl^ptG{C<qDj&}LxtZbwu+8#k*dAWPonLRxpE>0X^e?{!)C=x>qD<tN_hT5OLlm;
z>ZFt>oSGJOu9ZVA&AORw@Po%1bF}??(XZ+drPdNJ9)Td}{Z<dic<Ze*kV0}k=s2jG
z2rk<l@}IziSqYARE?PJS@|F}Dk~=S?m|mMjRM_76ra99MCdQ7;5o@wiW1qC{PGLA)
z6>GnZ0MJv`>9ovkHeyR)tu2z*Zj1ttqHPL<nwvqyx;9VRg|N_!AhE5(2^y6Th?PET
zUA2eah&+8g9_wYxS{{=1R1~K!?%=lBa^3-ba!l{@^O<S~QgTWGKk5nzV=`OM^0!2n
z%dgtwSweAOi=E?nKdw|mdJk}BEq=e!vYfsA+IdRwOTIvbLlI-`9I*IN=VtPhIX#kG
zQj5QsLz0Z2ZPpCsPL`ILeCV>a53-PmGSub*SzR>>1c}HCfH`|j01FNQi`Jen$i)^j
zqY;3tEd`HrHUJ_^j-<DMX(i$dT;g)un$K_ctL+K$@1j9u18ai2TbS;JhOg9p&x$|@
zNf1U!J+_y=rC}~#GJkUA*M=&PK8dO!zD#CFD#AEVKxwo)Bm<SO+Qe=~+fI4E3s$XH
zBG;et=(|@QO;B8yvN_(IZu#6x1gzpNgz={$1>lY+=cEA`xwjr>W|>*60B0-{j!R6m
zvdS^6v<??rj1t;?C)VxpF?<?Lc{Hm};2UM*t|6L1K8jdsaFsY;!|#1(%ZVq5o#6*f
zT|&1V;+kz^62rUY`-xLwPfcw9lQ4OV%hvV}aU}I|d^B5keozy@C(c!x_UG1E%AE4>
zXZM(@#vE2bTO22S`OAHduOnJxj{63kYKi5lgGHsY<ri(6^*-cv_2)%W6l6U(<n~Fz
z#l&Hdgh4`nzaV*d&*p2@{AuBM;JB=wBjZpjwVqZbM8q^*Pfq#?`!i{Csr)ndu6?Q>
zpCqgUOPwe&qPB0|HclzAyxJT5p6jtDC`drrhK4AAcI0A|27GawieMGFo7Db1hSf?-
zEUkoFvc}%a$*8IldU9!cg>=89GK8$64<uk>NdkyCCsBTNc@ub+iT0@U+geU<OXZP}
zZx3HiHhPxeFD~<5$XuPoLdruH`?8=$l847u<A`PPU=(JTuNdWieHr2-o3FgUaH~8Z
zNUYJX<~&zR;g>T8UOlDm2#WSrYH}I6Kq$l`h_{E!T;iR6S(bse*YXu;^klSk7+U_A
z1QRSBp2iPbsrSu(%W+Qvg^6V;J2S1P?ExsXQ*t4fv@~7s)zXA<u=oe&zyLW)7uhSM
z{|ZbrZUE?T7MxNZ2xU9~BHnj8?%rS$@IOaSl|BvCCMz$jr8)$(8uIo~!l#&f=h4!}
z@5rRJiVkBj5kaA+Ya?&J*mX>g_#ZD|?M%bVaLUCtnThDpF~9!Q2jW>tIMfSbE~gSZ
z9i(isVc9aNVT**6$V!s0jQKqv_@T5uibOMME2nHtvPZqpFM8P46fMTDZd$+nbTMuC
z%H}n+x`N5=j#o6F5+vtCJd0w;$XlU&G5fEK;`D*>n0{okabQqL19jW5)=8P^;hnJF
zE1;)F^4;$!vvKLTr1w+Wavl+w11a<unj@Cj<;Y4*B<M;z=s0g(jb}=HYy)&RP!Uj~
zz}hrXR~Os-eZ>QjI3=&HuTPuumExHbG}nMTJAbf+lP|DN>HmHz4`)oJb`y-mr<2f`
zEe(PS=+3>>U=X*1b+-V|jxf+do1x@!oG9g$25ra;cqOdJ9mIntsK0`pl29>l!<wHR
zCfvJ&k>w3)u;tlaPhEJQ=|1V>kU02Tpi%l|&px2l@rC&FDOr9qWWj&)Lb=Yc8go|#
zO<Wu@&Xcz>c52s?P(Te!RE&-<l8Tex>{*JZ_NX;PGepA~Z~aMLAYZXK=EG(<ESgcF
zwlUee)jo8$;Ia|iU<Ek;`we#5F?`bTG_pAe`s`_F$C&t<!-HgSebqcp9ME`iJAGU>
z&B0s}1&-SV88<d{fNM1ozfI-#rQn!h7x_*;f%2bIIc$v9la;qH?{n_sYTj>wI&KWs
z$ze|0W2YK3hK_(057thtD*Rs=o$-~2402u-^?oFg?++KC%>M(#tXD0IC)$OoR6C&8
zMq_(^_qnCH>au?8&3G9Z$yc|EHS<CMzRz{F?}<)nw;wCREhPVLbRAlvF!oM2uj?hT
z|M!o63V4FPgLbcsF+zvVK)p)|y^RLffnX#BE<DR+#YbpvT0wOoGdPnpO&#dtIqqDK
zBp~qI<fotz0dl<%OU)YjDVgK>VEZ<cGH}Q>zm*UF{Pa5bW3_;NJzEYX{7FjPY#?)e
zaUhiLr)o0!#cB{1J^!M(1_VJR=~WJ>Rj>A5MTH)#qoiPjbn$>@7CF$#ozMOlV-)UP
ztmAO1WWmWwU2eNES@Gl1dfn_*Tf{z4cYR<^5v-O$W$M<3f!dnByZnWwPKTX<S~w1I
zV_@5~CEM@NU<Yh4+e>xE5Pjp8qnJCt#oV`ZzOfL=-i73#69@W1GQ>k6-r&BPcswYj
zsEXz)ueJ^be{Px-$3R|HKqNiEaPNbg^dGwv-_A8?u(<x5^Uhw}(L{xZ31FFcwEI{h
z#Kk?Mpb!BRSdt9GSn@6wC8l2#hJ>-APnmuB&so6X%r$HA_r3E2`ysWg4U&ZJO?W{%
ze~Z@;t#~+)4n|6HFB~Y4!kcPhK_f;mN2Zj>If1g$Ylp<Whm9iAd4^dg<~566`pMo(
zBrbV-s}e-H{N{f9=qGRM-ks_Z^+dQa^kZmPd|?QI*bTQ8b=0$)oG_w$#jU4@AkmC4
zl;ZETKy=auc{4OUg1kr42{jjg(ttPrDytroSz!yNGJf<44XP@POAv-)lGI=&6iHDj
z?(AzIYjXUt)*_n%`5h8RS5`xE7YX=>$yy8qVvWsok?OVZXH%i>o(`+rl`vJrhuc-)
z7Wwm%m4GwOto+nk-i>s<VpI&Jue7sMGJfp1H1|d^Dd}79;Q@=@!DPv3F31Y!wb4jL
zl5z=rw<vzxC$Gb;3>dPsX9VZuTRHNa{&`L1BJVB?TwDQ*VA1lG&kJ$(Is#H=m{kRt
zo>Iu`?HtZH)k=L^IrqOVuX&%hXFEU^jsnbYv6CmxojBBOG^h_MheUXNA?PCRWPQ_z
z-nQ-r1T&7+Z-gW|MMW#^K;UFn{hSf+4+TJQVgt&-Axn}NkUSvw7yhF&;;|0Z93H1R
z+W5odZVgI4DmF<WT(=Qigyu{tz8{h00z*L9V{ESl8z=4{X!BXhLD0l582uEM#nGZ5
z>&i`_yYET2wjgb4PVh_R)tnNc><bCc^)TQSukbzzecXw|Jh}p71q0-9QPBJRIDNaQ
z<Ba$CT|o2V-uM^J1RktFdvj9v=h7LiX`m;-Cf${mfQ}HrT?ZG-K*5{a6cI5}K8HlH
zQa5{=mUL*Nde!upSfo!WP7R1q$_sy$nCOagWtN5bvi+`KjA2ynz=vauy8(CdVrC%E
zKq^!Uh4f<BQmdmKwU#bH-@5s}_%&P*$$LA+7{dBUh1Ko;a#)MAVckuTqj}t~xDcU&
zbACFFrs?Rx4C}nfsVoD588-XWsATSz`8#@6Sa#wXxqzBFvIw9?CV0fRCiXN_?I6Q|
z6X+Vo_R|ozq^o5jTmeUx9eE5!A%W1A^V1Sq^&Q$9BT-O+F+$k|5Y=2l+9k29d{({8
zF>^>cSSWu7y&+Wfn_{O#6S!%qB}Og5ow|90p*Fz0omRI&Fo?By6$rGb`0jL4lsZE0
z(8@w7OqFt|qmrvv+Nc3mA>%<%#^*?ouZ8AC*#0=Sg7m>KP_eM@O4AVuWaBG4cxS0x
zgMKKcHkuWsc=PlXax+I_x;PoZGW;y$1fK3%2prYjvut1AhrEk2*3wGiB!9xVpylBX
z-`uqP60*-(R^tnfMJV3?eswe&cb@<Fbu!nQmN^p>@eA&x@x`_WoQOl`@_;-AQ0KS4
z#bkHk;pMquWnu+Il<U$;{qXsf(v;#%IwLR@);Rq4P#6>=4+TufVsnWN`DZ@zhh^c6
zbi#4m<^&VaItWdfS_vb=okS@HH@}gKB3aJ7eoOzulhxdBCQnmaPVzyI^hZ14r!H@_
z7D~=3jbmwst=#%-|J`Ejq2Dwj^QL(A5Wkc_C<af|jU7<qMAP%`DK?nTZ+61kZMui7
zy()}>*-ne>#kUSj#GB2ookb2-LYURJ{uhswEd``zK}|`r|K{#nQLBC!xY*Pg3G*CL
zZ3rESxT)YbFD(vbN<|qs)H4{Vp^X^<_i2L##wDG)Vmo10nE8oU7h3_u=0br`)xZd1
zN+<T!jK?ulP<r!znX7XamDe}Q9()UgZd(t^QhHR}vpzNksv#9XnLx<3ERi9c4%c*_
zs!!NvmrQfZ?4T)Eem1d{+VZj7rTtaDz&Mco5ajnUkZh$6d~d$gk1B$aUuC73vKF=_
z9?S0wj(Reu<`4P3@#CG}=0GeidiewkZ-UA$g9(78u7!(|ph)wuYR9xhC^MB80@q@Z
zRuULl#_0(ta@0ilDIa|Vmf{24*nw|CNW2!nV5nfripR66^uUUb>46t-0f}ljhZZ6B
zl_|zW@Y$ggvaYN%(kx5nHPQp*!oH?}z*AD#_jBBolG`6P1ck(8{ehsFXuz+;V<@Ch
z3_-zY|1=N-C`PjffZJ_~r06L(N>c=}+f95W;37nfcatYhRy&a9BFq|bvaD}~Ph9CL
zXhm>JCEEevwv<HX=abbAiO0c^a-g!8v|&mad!Yo(-&?1748PYDqTD+es-9EUPnGl*
ztQaxQjZoY~6_LBOmt@qS>B>)@<+EPAvY93IEFy;PPFhRV@)FP=B?rHfD#SqT=8XKB
z*1}2JFHX7I8nS05CF8$asrmVG2Pc}tx@4$#`PVJ{rawG&RquM@G+F+p)rTt$);$15
zetZ0Bc@Z<72&VGe^U_gv_*M;OZH5Y!zbT2?)iXGeKw`@fDg6z<S|H$R+X&0s-itbl
z6mg!$j@r^yFTOi~gZKIJ44@{FNHwx&ZC*mAk{n0^X{!uB<?{ogy{`Z`VPnMm6%Wc{
z+&%?=bmy4bI$Q`Gi27EVK9cHAYi~T<aU7WmyCB;Wt&<@^r_t3wbxud)%>3rPnpscI
zG<#JZv#`p-9QTo^?)rvIt-qDa;z&T7M_pgP<DNS+Pue%_YWh^Ck0~%3eVS;}_FF`g
z!05^0U_!3W)VXcLth$R#z=u|Uw?<@x#OD9jAo;{~(it}eVuW)51G(r$Hs<(}LQBWT
z++}SYQ!=^LVrZNJbxM7BAari4J$FGF9iiNEv>>VN0UCzhg-V$f|GA2vL*q8Ysp0I1
z%qDc?pP*1izVd@HwFnRk65V*^JEuRX{&d?*?HU;hQ1;iMg-mlvT-Ib72qE<f>hFtl
zmCIi`9d70+-XLV`7aEjomyv3Qrw8$HBvf=&q7d_ycL@s@(6#DXe|CX}RasNG;|3d1
zzWc=a$_C8Q<vOKST7vx2w@31mTgAk)Eh^j74j5^t(p(hLcE<oLPnJq_&MOZ`c2Ws*
z?`@`k<7AaxEWB4e=PPhiXZiYSjguXulVV9y{$13`*iknu?7>(^ce-xKj!;U{YzGb^
zd^~HbX)nerg%V!gsakmUBN!yJJYCcSgv&uptOfyzeoW5VBMYILGD>_Gw3vz<R%(E+
z-3f3wgq&rT>xEqjr*0Z09N!&nTs4DVQ`p`sbt)8vtRBF8;wpVCQJF|N2%*{9_tvz=
zUC(%(Fh$fUVeJzrf-2yH^@Z-6y~MW+7958Z>8DZ}YzLu4A9c}ubF#{6+StL`ARwW%
ztiGbA4k*!iPeP0qQozIimDjMk$iY75&LB~JJ^=h^S%1{W;A`E|WqyUmZJ&h#fB3Lh
zhJ!;9VcamR*JbuOrAeA9Iq&M}`mC}Cn|ZP>?oA%fHd{ziMc__Pq%sbVh)7^eyi9i;
zy(q$rkDFLqhX}<BEsAKt`kwVqW1}3ub8OkP_u<Y;xHVg@=n@YRQvQ~|_)V!H!HHu^
zG=UURSJ|YnV<5a18|5e&aCWpwVRqwGE3lccYu%JN2-*Q_-vnBNMDAys{cI^KKw~HZ
zefyEDipR)G{||rE$Wy9z6xKal3Uv|IZLUI~h-L{$52gx(iO(RTZO0#FgHUZRD_UEz
zv8{Xq#UT<FC`G8PFP%swuVhHA2v!pFs+C_ri|$yd*%)~4?*R04xDXTJ3UR9`{RTV<
zG=n@<gi-7p{LT&^o!c;~SC<dX;pM*U2C+hV{V1OHYUd2h-TcT;RNwukfl^prE0g8=
zctid<*Uei*WP{%u(0wqE@p2r?v6t(b>QWSh=tqG!W;g)6A3i_ndnT>EP5&HPo_H{F
zdTw>m;JRhH1569BqvR^k%{Nog58KXre+^=^@WnB*S8EE(9j6M}sg#SZwm2-_cb5pF
z0`L<#u`^b50f{W)rV8=V`(I{dmspIA=;QYbbIO+PqW$Rrsos8<n+GCAA~mNbclNQ1
zJC}IW2d??8Xhnj!Xw(^msZR$l>{FRvRZc#qQQJ&hkwWIC<+t8_A=X_0>uAayrirVw
z)009Ktu4QKrv6vuXn-~)qL^*W+d?VaP6G?}+~lokI&$25*I(&(5w=OYab=GL2DXcX
zl+zz5`YErn-oDkM`#XGh%jyD|OH|BdHD3PBC8qL_xkPzv$8^sF^X}l9#GC&#qJliW
z%9^6U>wn9k{HG~2e`@oI7&DbW@C6c(O5)OH&z*emiih|uX<$1yN22DgkDz*OFBr1r
zO=7ARY<~XQ27-RB@*@zM$N9jLC#MCjM{K25^Ugn#k)RHYs=4411}jifNOW2w<nFUB
za@1F86n$vBk_dwGmj??-HL2eszyr-PzVO#({!>tQ9K_nUZebV6CxK9vI~Q2^M7MA_
z4?f}XrJSM!LnH(8%EZ~Q%L~y^;FGre*j5C)FWfJet@8(WcJK{TZHT%+xQMI>$yyy|
z!+}{@FJU53%F!WrUaj<}Kj@;or293QBZns7@Pm~+YFFHMsT_2rdWy{X>h?j>4N?ap
z8bUlDC0)x`*r<r0At6~OezMh75PSwm%_(+oQIf6#-|VhX0c8HmEID8l1T_~y_AWZ-
z+k*R@S|Monu@2K*=lIp<*XUZ~eRaM@r;ALtJ|8F#zcc}WQ)l5lzWblAdP_=^oNzM;
z@mJqG?PDV1XZHidB449*imt^z%;z-CJ_mxWPPXM?iTvgFYhunz)LA~cu;-xRHwR-+
z)3tJ!U%6ERItMYbY29c2^=TYO7bW%&nY&il>ji8_)ReMTQuwpUFT(marFY1J2GvW{
z5gd|V4J#K87ay4OBfWQ)sKd&{WAPDfo*fML@oACWq2r}R4g7naF>kb92hdTWc%6f`
zk(=zqA~4zH0n>oLePa0x6ER3%n2poLWJj>1II>@fOpVHPNd#hS)5?<u%0qJ9s7I_m
zc8Pg^6%0{)z>-*MWfFY=jU-hyZSr!*IZlwp#s#Pi3(Dcmo3jiD?VK2Q1y03fE%nN8
zh1SrcOhP{Q#0Gj|z(X0e-sx~vWCea#z$F)%`7S=7_<(Tn)(X}w7)IbDNw_zgdZApP
z|8^GR!ShgJ%NU{cwWNS(Eb<Lq^MMEvvl8@A0U%A{nq&DYz5kQ_e&za2|G52&oU#m-
zW{4p6M6ou<PH(^!&F$MJ?RU`U*m7}T%iGeQp7XfwLyFPzVb3W~`LAXK1xdmlr6!8T
zQp2p?F~8xsKInIEy(Jx?7P~LZKVL_xFh*@LU#VZSux3;-#hKIJtDu-*`9;V>UD;jB
z#?@S{#OIFuyFh7E3HP}@>OnE5|4ScOSDlX$Q(byUdpUiap1glxq?KBOjctv+wmxdJ
zkaamHag6%2Ac4KP+BRl?Z8&c|rU(9iTFLvP0DHHWX-(T7jk=xuPi`PP=h({E+68<{
z|9~O8dg*PCu979oC(b5XRA=5++I*RgYT?!?Vx%I>D1NSq+DD=nw0c=JC><#>w;Rq9
zO8B5K4%C9MV6QBp^7~XKFR}Zy@0|nAT$X!=Xj$Vrw}9imeU&tjZ6N_orpK6(r!*lE
zMxVd<3^7|1h3ORW)6xHCjGoPiS2d4rB4+MN!$C!0hLcb_%X~la9^v0igO?KX8-^Y^
zCEb$OgFidkAO`}UX_HEGd6^`(2u<pKGIIizGuhcFAkc8tj>#YOzKylYY0Au6-mLAR
zh7u?~E>P)bG6AAo@SaG3KI*j-`hZ(PMPYlp))hKkoavVf!Er`jNWM9Ln22y=iglo&
zy6Qt7poPW;`6_Ok;su(=y<F|8$?>Ex?R+KTiq}n<?ddo6F~TvKwV$$FF-HT6WQr)H
zHaOmg$Q>2_ky=oGRt}Oa-|OB@XGdvsk0Q=DS4GymA6Ym6b+Y_iZ9%U~mDk*KiEDQi
z9tNOT&ejq@o<VEpAvqbTrNm-n&qdA1ZO1nZasgW=VePZ9TjBY$k7qGImHN<KP|XR@
z^Z{vJ7IQOs_Bq>T8hJU?Y4YI^VrtXY)SM{({_r~^mGM^~QBj-2^1=wLf5rPd=li=$
zzth8b`m28Q+>F4nwO?hSn$_oF4#X@h-WaRTqt5EkF~=F{A?ipt$@liqCD7J*wHf-%
z)KYqH_YK7;1!2!&51uUgDPvUadq##2A9d6qzfrNd(_l3b{O0{$R~JWl7#iybL1<V+
zRs@022nGBo9DGU<EKLK{vCG-IsN3vkco0{wj~P)+#%{ODG5f_lPjxg)Y<YvA_rwU3
zTTI{<YA7jK{1~wOT%5pckd%=tM3511{K8jN&b_i&Qr9;p2qcSO(ijV9dyYXex{koT
z>hg;6t`Mnk<b(-32p}Xht8CRO0Ace+7P%r~^^k<d5+!(N>3!WCiH7t^zCm3mz0YUN
zk7(FWfZoNfbm0AeaHGj`rVG7@U9{P|*8Ca!8$%<fl=GEj$q8?~B>GaQXko!GLP)kZ
zYG+*XgpzNjuvUabv(ZU-5N!?5cD<b0J-Ufw8+B;hrNT#ilnH9c^xKK50q6NNr@4*q
z-VUdgG#>r_u%*&$v{j#J2HKLlCwvbBVK!n0Eb}Y8M)KaM!*t@l@=JVS-UC@O5N#M3
z=3SXI3u9YLh~%>%k8JPIdki?F5hr`Ait-qY*l>w`-B*C6<&9cZp4H}~99}~r7WyLF
z`IpJ8X3Llv0V3!=3Ey@6`2jD|2FR&Df#kQU`ugAM8@d>;=|GRcJCQKmJbgKUmTA0c
zeof2@Kg~T?tRYQwa<vrmPRFhcOJ=&*h{}O$*-q3K66><mMe82|nvdn#$4y=vT=Nb<
zTOkt~=lpoT*p8eiTYVRE9AV@wUtLti!kf-EGZ8!K{DUr|+6iL9`98>Fq_x4d^Z2h8
zz@Sl6b7@`^L9avEukj~eYQy^tMg2<ug9!IfD|OOt+HEHFNQ~M{fQ$uD8458W<y4c8
zvyUylq6qGX_x1cCH0bnU#TRG{-ZG!52UgL2pW3`Uc9(iqTq(QSWCv!|gf{BUQ}EEl
zZhF0*yLbOo;j@s;62Q;NkWy*i4?V5T39z2{Y_0q<RfNFJTi+t8?pl%yjH9EBGj4W9
zQs6E1(}Z1Mzpxl1UUkJ7XWV$P6K;oClh4X6=;aD7DQhsK=F<R<<AGc+Fc^vJb_L2Y
zZCGfG-8Igh;osAbNtNyf>h?-qklVz)H(1&jQ3Fva9r7RgVc~D^O;<bI@*JttF!Rhz
z!XYAQRpNhCi;D(7o<8Q><N_(SNE=B~1*SJ2u%plW!Xml<pgh{$^RnqA*LKrzmttB*
z8nivo5nTPT&+ChRE1=G;8aSN_-Zmt7DsEgEOWaD0x`9&vS}DFY4Xp!a6=kM$hqD~>
zej8GI(f|Q$C?iOFZasU5GJ_vXO>_jDiK_}C;SlJ2!^=MQE^5l=PY*7^Dj{wUn|T!~
z(%d$x?Q81dz@0?BpW<uT-ldkS)rsqSi%pUL>#V1~QL4=P4%G#{s7uD^1nAJ+M$+^Q
zVgP6puB|SvAj^?*Px@>_Xb4y;coXhOGozVCIqP9EcnkG1R13Y3WAa7+Y`Mp7*~>p2
zfQO&-<#lX=%KjIKl$gP!8$k{V-`bUTfG!_P03@>YQyc}V?aipViWg_uJe8Y@)~d5H
zA{zHY)X&MwGpd{R)Ll#;Un&L@QlBRuKMxc&e|7f?G;dcZ5ab!pKZaJ%qUYSh3>4og
zW076>(s9fWx-8<gXrm$G)EmM!*odeTeOxY~SDF%^g5GrCIR5yEU(TOm1{)@ajnu&P
zgDU8B6)(DDXyECK4d-H}prdFBglt60(-mwgrCeQns!+*|SZv!G&>f7Qq#|&pDd(m3
zbrD8p%<S=ZDMys3Zt1@#0}^P0w2=3d_b5kkO{6s7onVT1tnU6ck6ZQ+7G^8Sbn(Z9
z;#Yl9!4;Tm=Dujyv^Mw77;;}rKn_rj4@gYEGt)0gF*w<9Tkl?~nD-xunu~?Uf!~wy
zKETQb@Z4A-vOPcph#eq#aux((DVkTHFxQ5mj1F`_d{QMm-(_qyTz)2jtCsij2Y$34
z%6+7U&B!v&NPF`JL0sre8lbqf`awvzALcZoZL5Ot=-hfY3Cqg#TjcsXSslv!XaNsE
z1s!K~UQKZoxNmSqKK^Rl8;eI8^3o!sDx@mQ$`2A({;I=~QS%vN_;jMO)24@`AzF6`
zW+~hN7U(TWT=`5<sV_Zu(y7&{YX7MHhdu2ig$$3xX8HzM<SL!Id<AJ*2xx^+VNAhS
z#8SOfM``23Jtu6&|A_$qL;*Vj>ls?VE^!QnmI^jW-H(nv#6zcM)Ud_keAaIrres@C
zHZHtO#JKDeht^p4BFyp08A$*fJlovo`%muEHzv!yVlM-K+c%bTyWvmn<0wo-V7_=T
zBm_uOAf^-#>HK~iq`2|4$1QrZ6Q~w*NV6>3z5;EoMjwML(WhBrM@^5#Q-o2CD=c50
z(3oe;xDo>6Xi-UZ2%<-^Yi#4aq-ZO($N;hdUStRqolE2EAaM?pMe*(|tRVzvX_ORc
z3h33Sa5o`pA{OS{LA<i2-H4yF4Wuvi-lMhq2Zqe+2COF)TgQbMo#ZpoMlqq-R)YB8
zb!$KC+4Ni9@|Cw0GU4pe>6|R8r_Zh~2$yhS!H1zi9zgF3Jqpd@_y*B)vS}*+hA&@!
zunE4X#cjPY)6<%LW>$S@-9=rjKOB%PeMt!w<D*pg{SEJ7Y%k8lR>PUgP7g=68<9hP
zxQMRvYd&@J=;Vs~4_k`iXydJ%Px~}vdL)OE=ZQb^qb$a-e{z0rkT}VqXHFop8G7SH
zMc6e9-Mk<0*kt2y3{LfmwR9nYKlJH+x7+wxunH>XpSog(*stAAX^|a8c#7aMMeZ`F
zQ22f3{@J2(*Ln#-3E!saT7_8jgzek?EVn3KL?jDfZcrw37rd&#?COLxpzv6cy^3+9
zwYpe50=OK9>9b9!gE8Kd$Pf0H3#)|l86Vafuz#UahiRDmHf6$8^Wz-r&YM#Mp&Xsb
zlu>hoDCuO}cL!y<)_{-!lcVAGehU!z<HwH7{Wf?01OpaLeal;vtb9eWmr?SPk~c4{
z>$pTE(EsyOUYk!gd6&LW>ej?9j=ivoOLf%o@|;21{8hS(RY1v^|FS~dHEX{4^YU<}
za?72d8K#>N1vS2M{djVJ#p;kxh=M7I86uSRNX4e{-6I1v^Y8D*-WlG-xPxJ(%+@2T
z5G{M)R*8@KfY<wt#eS?nV)j)Vx|`z=syW(+Y9w^AzoVGz_U2R8#TGEjA%O^3r^A9=
z)*ZA1bfwx?d5vnxf-wma9t%Tm>6>^LI0d{w`}wIMD_mb=mB@;yON@<4!0!|vrye_v
zKJbotIk1_FB8Fy%b*6=RJ8q_p?bEl?Z{6z{e-e*=irz01zmORP{aFM`aFVAF12Exn
z>5>Pd+9a^<3x+`%SO$NY0%_;N{oCJGh`CrN?%oUQ98?25ira0qed?k7kdtcXnQ`9w
zP3k#9ADUk`2BKH3ZeRyK1To{}ZR;J-UD8{{Tj{6d0AUbUhxLO;*tjX2tzP`Tu-_dV
zYNb(nrQAd7gog^5`Qx(V@SNT?jE>fxw#+-nntfR&OcmzTl+I+6n6fD>(&hXN8k|h(
zWd>h`BC*Q8artL2um}w5kOYjtB^PI(|L=^;QiHcSKQ91gh#9!`fo4<n8fpnI_xlUA
zoKuE6nn#<YPKi`Z!QTcTscnB=)}@$Q0*Yq_cWg>AJ$wH2U9?J2IR0Ys$iOkCbKYQb
zKdW*<AiV$}YziDWD-ZSOJC7xCa(XyEp6@<bekqO~Je!w5fToH@RN~I)-vSgVa{=K}
z?0`3+f}MfNl19(80cG5+utdL%*<1~Jx%LZtQ4Jyp^9M6w$^4ccVFa*yAx)t@Kb`n0
z4s(tb*Xnk~_}+i!5g2o&yi8Ezp&C6RAC%mSTJ}HIL4iw*V33CfL9tQL9^5Jkr9pd+
zI=)}{Rpm_ukkE56#qr>2E~8n$I|bU6t=e_u#=_>UkSaRY`UT8j;`=rb*O~3<e)?8T
z*B)3^=4sW?m1GdK!5krH%3lq<ig+@M`y`%ha6&AtP;P~wj?b%v1w!j0fYJJ%vzek-
z2%qjd)|a~8kT@F~gu8Ur_6-*st(|0LR8)f?=+E_gAvc(;hF5Tu#52&Nd@bw;2T6XT
z+W;D;|6rCBC(ba(g$$4ocL!=hp+Y;}Azs^~12*3KLy8!!hI$pxmBAE^GX2J{&oe(4
zAuD8(x1H+IYq9ukU~KpZ98HGr)>if9ArDr>18-#epW;-!Agi>tlAQOj=?Gg84ylRT
zd~VkadWZ;;AUleX{cu$8mqAeO0LcK3OfrV@E(fP3>mHIGE%W01O<~Ey`A14lGqomy
zqD7bXcxed?ly(AhX}z^8)GqX2bUla8;rjs(AS&Ff(u^y@Q8V65>gEK!kgg>4Oo(dq
z?STgRMUBnJvs7UqyimL=@wtrWp0pYONfGv;5In|_-gch$eMU?OI;CQ(7<nwh=s{~*
zV!LdE$NwShE#sm<+cxeQV(1Qsp(Q0G2Pp|bN<<LE0TG5qIz+k~DMge{1%W{Z1O+9A
z7A2&H4rv8xl;*v-ckkzY-Y?I)-*)%sF09Kn=Xsp}<2Y<F2A_zJUu1@4i^QNfEfcMs
zcm~f~)JQrS|IP@nxFVa%a!I~Jaq{DW5X;bRVa~^IhHXpjV&S)xQn>hftWhKvMBQ=_
zcK9*>hfMW2b`+08ulU1o&jsys5ZP5L@gTgaJ@HEX5R8I<9^2j)*zLjKM}%GWH&xRS
zEZjb^=2#twz!Mq}xkS|$T-;e=Q7%Z=by*L;g{@2iNEv7j9Z%SN67ILFhE<r43g=bF
ztuDmTQEy(ZBBBiRtwhUV&qHN5Pj(mO5k2v{EB%)MrdKdY9(kCVERi<k$>HS*!9YLf
z3ax<8S5kLW2aE2xUf6$+DU3!Y_65ikZv-wy*;D(9#|>>{G5`DU<G=LZ?c|{@!G<bX
zH@#>mOv`R+<r(m`k6((Oy~9KBJxVikgWfAR(d~5YrFr2yzeM|N&Ho=AWo1NJuR+SJ
zYFu;ZxHTiDgK!jRCeo+8yYe%VSl04?@2~%#N8#03GnzC!*)6~39I(2Ceq6oZCR$Pj
z5`^b&`}JkZ4XRa)q+b^MR=oSq=<<b`>a|tUpZ8C`G6((g|M%LWLr96eavxhS>)(D#
z=m{j3(sY`;Z?dMA35#^$Ebmbnc3ZB*3xVRnD!)d`1y~Hn4$i=!&LG|sE}j)39rjxJ
z<II!tP=UW(K#|`wZ>PV8wHtQ$FipQLj>jEP<z}VHm2f3b4?Vp8WHJ<~QBZ#t{TqJ2
zLzm(b<6Dh=YdBpidiFMuLcB)-I9)YU*zvUcTfmqD`(`R=s*y~sM(h=u{obq>2(N-U
zlZ)@{3!6Pr`qZJOurW@k!F+B-;FKT;&<LE?`xKO9Y=ZiSW+qgG4U~&8CE?qK4N9fi
zE+5|7MSyod2U$jyzH?zDv>hMka?Yo``jN-P4ZUq^iWH&9+Sn?Mefnp&IuW%j=A*=@
zT^^QCo*a*$7;a*mLO@{{l9+;7nrgiMbb$~b*>Y!rjB7}i8r>^v)&6M;-F}1f7n~T?
zcIpIZ1tLs+%IE0S>CYe6-2%^n(#&+>cJSjAutX<vp?hHj_(-a47ij7U;ZpPuvE_Kc
z1w92sk+w`EK(b8(?Zk^bg6aezRN)<$4e}AMoq|3LoQ0g9eiWfuVOa!ulQ-om8S|Ao
z`l)km72mrOz`~3x9%NY_@2?4j5xQC$dQfrkKx*dcf}b2X+$LNj6?V$mH?QtzLGk2~
z9ld|F%MWdHhX^(j>C%9Qo^N5h&<+a%iFND*cf&obtO4HrrChFR$Fg^kFWo|dQ|J#r
z^4$8JsqU_i;$;@6*zt(G^T)*aIhN+7mk(^14Dk0!9jI_fqT~JaJU9M_yHb&BAmAoB
z^bFKrULuPm7EC0DEP@MKM2OvV703xENO%VEcqL+TH|YmD393SdqlO^AkMdfxjBv;Y
zy?=0RdvEBDfYBvN%#Sl-@vAEJ+giYywRn^<L?CPqEQ2-*pWno|$tu3$*0U$*gmS$a
z1z*&=k|u}8#*knTLB+>n*6<Z3BeVD5-8qo8ADU#zQb8J7w&G-0z9VQwOwGe<dx3b!
ztB+b%3vnEI#}|Bl>Qm{k!?7&++M<~9Jvo_#_b;Pym~tYh*&OuGu<cuyn=kxyN!w1X
ziFq+SwFvIyyMY2WEvS<o+4D&1EG{ZuN1FC;wAa7n>V}uSNgY7QE=zxR`A#-J&)@Z?
zVderMzhRY=p<q%qft#aK$@0<2=IT3s<>Qz8$@*2gydI&`-5Cd{Bv>%$xNfl<{g0vn
zNQ;;YS4Pu+4!CH5-?0%qQTE)u$~mDOL|ggS^6G8==1bPc6al(`Dy)G8I{v$5q-D4;
zWHmndc}kJB1E4#)16lHx=<7<lKh%umDU`5D2;|pg;_*jBFZz~7Z7*1KK%+DM_fcRB
z`sw)(!~Qx8CUaGcC-~^?JJ~>e=yl8zMYTy<{2|Wt!((N8Yk+}e^kyj}!yWrlZ${hy
z94I1teWTjG-&OLDFkjPoFmK~WGrr1}HkmPguIn=!+#J6T1aq{9u0q=ez|-3u5PZ(2
z<cVVZ!s$9$2e9%CgFIyUsMZQ`@V8KRv4*X1szq_HWnCztYKa#5cIDp}RXJrc#1eN_
z6Ht%F=xLt)Jblrq<7IjlhUDH6391<9h@EW=@YzOnb@iOs)4wOC<l5$-tOqj8mmxn3
zza(Y7PDH5VoB?YeI~N_mat~=npC)&DpLJ{M*eXdl;$)>_A*i7`ZEHgdfRdaYSaaK!
zV>m6p&j8o!d$@xdoT@rZ3ZjCJI*h={(w1*u4+ifE7v;VCd#7BoSlH6r2Or<*5`;hs
zZ!}w1N#WZ`Fk#O;e-6Hp(;mOp0O-Pv3E;=~C}dH(C(r|42{9HNq8sHk7yA3~DGiE{
zX1^??;~PpUxYc*yttF3PV-?&fdx3`1?Ht&~JTA`3ZHWw&kfM87Q}jv80X5;oV@Ba3
z=tRhBN<n)({w_j<5DsX(S6YO2p$x|F%&w^3fBQiHSpstML&ooBfP{np!e3aAeZY!e
zTN)xT#QLhz{YNXkZsdvIiQD)5Fd}Mqcu-Fx3`PwX+;H|5;Csz~Sy)BZ_}7>kY>gpV
zc_3X-w-d2&sQF1;i{wD|r8AfNgQ%VM6dSA#j)k!^**7u_DS7qYN3%mudO_wprrimI
zc1*&Poxq03cTvV}Q159=#`wdV|J9HFn<4S6qz#Z1h|S`cXY?s08!@rt#a^bNB1V39
zk1x%U^?;`^yivVZ8F)88_=4}<Ru!^?o2Ud6)^Nv8xA!#oCYYFT^Y{E=c=vP%oM#i~
z8Yyi&(GJU+1(ptKw)#Sch?$V<#VEjd2~?=v1I?cj;k8m;AuLY9d0d>_%lg#`nyyKz
z)b`0+ia_U5i4&yK7yPdGk4`7?L?_c;n+{}{zPsf0j$~?ns}}rAyPG-|INw?KS`zPW
z%r@UKb8m@SHqBUo$t^DmqN%z7la5#al?)`#ZAo78>i?Rss?`Wx<e8cNDqey`uiUGk
zX6MbP3Um5u{6$RYKF#JreOigv=jr*CXICFza}6`}?FGAn6`SyaY2h*{K1=EUHIV=#
zi^jzr^;%t1%E4yI-orMv;-u@mxsC;(JwZrkLUdixDn|S1g>G<%>Og{e1c&%6`QF9q
z{pi)kkRNS#fxF)&Y6@!^fg{O$%p91~72{Vjw~)>$X5HtT@rRZ3Y-=D|Isc}xGG*RY
znC8VlM`c4A(pC-JRI|D^z^;3NMl5eSD{22Dg7(Vez5jVKw4!0s*0&{V5PGTOH_(59
z8IeptFj*CVQNn}iblhFf|IQ1)tGjVgHlhM~=-?%qchKAVLfmuF$A-wSQTOrr+36q`
z<=S5^0ylh5Qbg}(DaYZInZ=0Lmc)eI=86eOw0>P4Tx*9#!x1g-Ar<{z3Ln}2=eL3)
zRsW0btTL*ECG?(>%_{A7pTcIHF`fU$)S(|wIPIueKq3{2@H-LZ`mHpfo0dku#C1p(
z2;0A+B4)uPJmJ^Xmt#cE=MdtLOf9_o{zdwor2)rqdv%$eE-{Sg0s&sy)oPiO1`F%V
z5}&v6@G^!@Pl1z(5b205FSKJ!SZS{+bq}hF6wrffI@jG^bX()BxgQh5DYMT^V@;((
z0Ek5<?!jwDXTyef;y@ViP><c!DvNA2^RDq&=G=43w`}U@t%{@Qc;dDP0@zOQWqrB@
zQAyFIr)B1t{xP0&%t;yEE2E=wq(!i)OhU~eJxMt_54I$(PDG)4>yW}T$^}a<P9w<J
z^93?*$(r+`c5qmXyO;EJTUE)qwHp=TeqqU4y+KJU^7N7zy?x%pp}l;XqI*wdQ{Ukb
z>$BH&j(s{)cyR7vPe=@@RP4a-bKIX;@-iWzd$iWm24psCfLP$tVBK4Dnf#dL7xpkJ
z1B!}KO@iBl)sRB!>)@MP>WK(@5T8h7O_5N|{kHWhlW;q5QvdpfGSwIi34DZ}!$If!
z6dq()ZuceIw<iaJ^YA>73naTZh0!et3j(L$!;!!<K!dm`S>xSZSHNL~!Q0_vsVP?h
z-D_%4HP~%u`tvJl7nJBtjLG4QcNdbCb!51(FXzJM4r$iDO}c*W?Sx<R+i@epl$Q)^
zTPi5w4ffVq>?pm4ykyJ?-!6X>Yhvtmbt{I^WroZIeSBpjPc%e-v<4amHSev=ys2?d
zSyt$wL!&7#Ls+d(Th?Z$#PM7fSmi>>PFmNdTe9EwK)`(*K<&D}kt#fzC(0Cv=&$U1
z%&<L4g#V%wsr5Hmqi5Xp@M5J|OqdJsqc8~J*5JVMf6_iZ+P{tCYkl|deI`85)tdt0
zQ4#R>qk07F$KyQTc`~wwWK#GN0leAtL&K0+2v3M>h<Y4MCgFMZ+EO+f3D%BE0&gVO
zfy6YWn}NCF&eS*BQ<Q|t9bm>Eu7x!q-OW!I+7;Rw8v_GMQjR{K2f2TQOBsO)qSEhN
z4%j-k@`|WS|L1lA=w;1|$HQVAld?KB$%>Nt5kk`O)F(jd+3~<b$yVP+8n57t6Y~Ui
zjdiO3*fh6Fp~{^F;B0)~uK14g7AXk8z5a86UIPv(o>L=NEanJ}>6c~>jE!lDGu)!f
zp|Z*TJNV;c@6(JoDqCzaz!dc@ZC%kD5qPtFX=B=wr`f@B&l_l+VPgnfj7#90U6}p1
zD2gA`3dej*UNw2OhJTz)NA>@~LM0Grm0p?%>e#lJ$gPPQ^8d^lToMSa+d{yjpAv1A
z&C13wcio-d?|!sT5NX0?%hx1D;*i^1zRDuN6AVeYAjJzie<EKOEC%}fDE^)DvQ6g-
zfwWhJ|MvQZiV(%!D>tTSSb#kPLxG-INFxCm_sqxHvt-4}HrASla8yyk_d==cG3&!~
zVrB+N2I^gXCpBOU;NQJ@RHCIzUx#B*wRcZAp}OB?mIGdjpn<yZy91btdb+hCY5sSX
z5$PWZa?-1Q5=B>2%oE|IAJN0A=GYGb*Gp+cjpY^P{lij95if^U2^lE9{onyYeB*(7
z?luLJkTblLoXk@t@6R_B;lhk2DP;>iBj}Pz`f~*pszRvI2KM9A4@dmrRd#Fm5Mb+_
zcxxZ4I5C<C!6&dtMVA$r(yAO=b;Ntr-hBe})IM>R1f;m0(&Jcux+`T(9#AKd=xH=N
zL6MF=klo3vm>%{xz6$Cl!&Lc~<{_PA<Sy6aUQ$gw`uY6L^E8bS4blWn2=R@%!)DK=
z)Sn4(CmCAn+i&))FhB7XS-v}Q0=;L+63oXI4m^+^@ppb?ov;MD;EprrnD&>VAuI7!
zup<&Q8Cnsq43}N*%W)E>AA3Jp<EplARFK_c^Z|FIIfV$dJjk(u_o+}pY)>R#DX2P%
zZ-k_Wrs^3aAOSGWOPWfu{0gfYq$SNydj;}jJC;d0=HMqerbTL^DJa0}%tS4E4?TM2
zhKSxy%QcQ{5F()hF@mCN@8ODlz*Cd;Dg_T(t!&PenDwy62jMD6pQ(W<%mx53#rC5p
z&~~5FAleOQ3ul!C!o=3IG`M@s^EQ|a2}0di?-Pw=(ip!dOJA9YCccPmvUA&gC1(<v
z7&CESuqVj5=#{WL`MlZmAP^W_mNy$NP+JXd*T#M0Z2+3Ii_k-A7P<z!TocaUTKlqL
zHItX!dpU%5F7+#?$JU)@unS4n95Xd^iK4J7)gW*rfm2yknp;MDEwi1avwxG#G(gOv
zDLcH3z7l+jD?8rON&nZ=dL?Z*4iH*|2tiZDJa{eU1vce+Nl-=ep+ocPsCVAjdkQbr
zqu2{u5as{Pb{`dgCV$#BE3#4QIY!uUw#Cse?&mFaaUZqUtz-%?udV63f5CVES6|4G
zuNo;e)SQc8R)T#dQ*v+H6;UDz0WI{dsyio$22&s*e*56*JW&&;G?8+xLHZ?-pi#OR
zl6-}c2l>P~_MadB8rJr`ehrv{i14`Yv>Z()Aa;U#&g1^A<wuZc3FG8GS7hjqADlLr
z7%)30z$I?ru-rqh@8l^Bet-V0A`Gd+VBB^Oe_K%#Tu6KeM?C)V3MTPU$wHB4KL7CJ
zjriQSOq#l$40obb?dIT9T|!HEa7)|KlbtOKfJT@^fxzKgYWlukW}1Ss2>8FVN9HK{
zRgZo4skDhwnG<V*3j#q&bb<KL>jEbKlA@pSkjx4a`>M*0ktK|dlkzt~XB$c?ppVTX
zml!?_68Q&!Jw3JzOn-{bx87@%a;{!6m?uFQrY9I(5;e6Uq^bJo2*YEQa-!oMzPZH`
z-%L|7c*8vCy8)`dC|_{Rlp+|wZC2Hfv+g{&UYU;fnYcBTM7;iSL`ObQ0qC?JZ_ea2
z%qrV0BcGxST9z-f1U^|82XjH*^aMi!(z0zWWp(l|;Wk8q?JzB#NjToS{Rz&ygHD$n
z?P#e@gtcRpz`x+NzcO_;wp%5ME9^7;DYx=Rvu==)qd!OpD2rx&mC=-RKGHO-^a6z<
zXiOMM2=n5-G|FJ8%C@irPY3Lc&L<^^brh1Oy<8<Ibh0_B=r<kSvHe#k==Ji=RX%#l
znN!N#K68n305FC(_f4H3PLV{~#SBlqKz&SyRKJoF{+wiIm4-ylffHK~VY^?a&uwHN
zxrKHPRf06ADOg3y`b+cn$*+l!t-gR-pM<?<cWRz#uzf098h8k=YH({?{rvFrXTg*~
z<zG>aqRa`s4yp7nt|?n@_dLqZRniy@CBjenmC$kVjyvJf=QVSKj~;CULNsf^#u=UH
ze-;68!f@HXh#NHw{<=c>9#ocD*(#6Mu>k@5kTfM+O3(bnPQ#C5w89owODkb`9h4Xw
zm^kR7-_liq)zoTht_5#XeZ-cJ2yHX1C6|(bVj{iSJJmw<gU|fvf2t&2MMWfoU?W`B
zO)`<Gbd_Jf<d-9W7bhP06%GgQ^J49y%{UL^+4xer2koYO2udWBD~jvWN6(MR7N2g)
z{+#FPrN)rOf#s<85*F5DWL<jO_v9}j#v{Od-LW<ro9p$KwhF$V($8$M?G-5nm1S}V
z4M@c45?;dY?$~hnB-a;f_TNQ$p7C~=XXM)3dH+5W9V4UTkEm?YU`iYoH?qy2vU%I2
zB%JB6Xbnn=yR?H1;|A>i=a$(O5+6%qvGZg@1~VJHz2OxwRBduWYX*s|Pyb76{?B~C
zujibsboAC2RHA&5)d_0&Fe<onsllS<pyj8*RY@Di$``;ffRF-k)h#>9{%-NtVV6EW
zei>-99hJPZRrX~E6fne5v&(PwW0aJ@U1uINr+HTJB+!jHdCL<xA!M8d`E}FLFWEIr
zs~XBP!}hgJA3-=Xf0T;F;>x8Ie-)maCgz2|es;+ciS2y27AN@{NSdBvVNGTKjz6?i
z`fZupV363Ul46)0N0D^P%7>kakHH9gfyyTIUg7Z;Ez9bw7jrCT2Ot2vov+<9{<aL9
zKEl!p?Cr&>%ri`uZxKdd?_HmI7PEpt_NDxUX|I%(zoQ2!2Qo$NgqZOg8lzyjR^_&l
z-*VW*+e0uvZ*IB>8PT&iy;h2Jx5lVK0nDe3E*8^0b>ma%cvO<ZpUEH#T~p!%yLo3~
zjV#W0#s*VxG;^-vj;4Y;K|<|W#BuB>Lh)0eQK^?ce^VFNO?#{-0aDqkA^=hPjC<li
zIxV08!r)ZCsfU0h2P7%P0qWlCEH1P~e94RpRLu3>6zH6*2k9(c!)}9`pF=(}04GL<
zpE6!MjDUD?7csLCXM+J|Vckeokl<4LW@QKUQ(#Al&`I_-BmrWj{aQ8*2^bB)OlgZ3
z$ILx@eMyB(v4-G@`}i$=YAll_4ag)rg^COqlSQFjUTeHGk+P4YSTl2I6zw1A^or$v
z!(5eoL%;Z4yKcNsCeo(H^w7pOkcNaF;`JtCu90Qk{_a~`kw!8-c<j>TTae?I%zq1Y
zg$$nqzcEvhM&k1p%hw4W1jlM`ovux^?X3+1hQ0-Eq4Wxez|tk~Y_TVw!wtnZs;7W8
zjsP~UbHnP_@FNciHV^+e`9d0{!^PC+1r!1BYB5v#`=jn*PImQ=vTvOCeTEN3Gw4f+
z5#cglN=JKTi06_3gIRqGhpjE|!kTd>VmJhrNbJ3aUz*JA0<I7G<5^p~^eLzlo{sPy
zpUM3fMS=C6bPIJE1{}E%=?FIxo%QRmfY0!SW&!=2EVOfkm)M#t85dcl_Vm2X5Hi67
zeW`(BmXvbjM;26Nt6`J0%i;HC<NZH`>$eI+`ndIFiB#Oz{Gm7t@wG!bfcc)y2?TaN
zCcqCjez`OJ*7Bm9*g6ji`C_(dAQE^iZZi3{DWVy{Yg*&Lf^0OJ1<r6Rs*J8v3{dJK
z!)dH^5QRWBcZSp|jSn9kA2%>~96!+Oxur3h(rwaT;D!l_AkDp}_+Dt+-SNP9)hBoV
ztC&Cxi<6OHFKXdU#o~-tK;9Bt#Cn{7<DKYT{6#Jj$U9m+@Ru0ab6p}5@LeD7N<ZA?
z-X5#?d^gV{fvvGqDhcs_vsjNmm&C_Exx>6~bhtA8H!j&?c7=t%($((xi0-q+7|Y<l
z?#kc<HSiPsu17a(3{5nGF_8-33p{BhsSvJgxI0IvCr_rNl5=Y&Y8y(eOTR^%aV4CF
z3PRFS#MVIVMA!Oj+<?&OaWVk1=E0HDuh0*<bo}$lFb)AIxE;EI>%`t8{S)27O-&{L
zc@^A69J!eDqrK<<32-WIeD86Z>MdeW7)ihgXThbA&zxF-OTHg5>^)=_J15U-3T-Ed
zsN)PiP@OJ*_>tAMK{cFFwqcfs?HVrsIUmR789%lYOF;py7A<RIRo3;SI}XT=4>S4+
z$jyvbg%a86P=egYKL>9@hTG_=P!T>BHFT8c<MSK8H{71S2@$OYADq%(BdhG><G~q~
zIP(4zcoK|`3}d2&Ls<XyQDN*vynZBg^rorWb42H7S`ffL?OqN{ytnVqm1%Q=V_=q;
zFsbM+jy0`AL7v_S66$ycoouwVQJ4gOf~1K#62fqzfUh#G#rn8YQ&jHza$u^1x}JE<
zlL>el5H-BilzQ@^bKzpWHV=P3rV;|)Y8f<Z%&t6u+sBFs1Vx_R#iCU`Xrm|W3b_gJ
zMpW&n1(-TP3PC7&Gl2&4(&;ojU-0rEF}zR6SuxRR0Pf^YV;wH{XK)iZ<Kq$b!?Z)B
z?H-;>djxp2NLzX_uR2#o)jUA68Qo*sc7Tgn$lCK$s01`1$ru_SGq2KgcI4zS>GILU
zP4jve+s7Oguc@=FkIwv^nW=@*f4vA7Hb9~ppUTs=2T)J6Jm_db&eErgfjlRdv%d1w
z3$-bMWtYd+&4Fi%$?G03_EBiuWicX+#pmN~Z*S2)Y{sYI!Ej>DtS~W?1TI9lKJRB4
zq2IdO<A^ZaPn0)8z@ZzRcWgr_9fSG$<_Z?J#=mCPI!)<Ct731yzD_<R(O;~+o(8fd
zls`C)9}q5(=kkR*pu}H7L_CJ)XV|H8;je%ES?}fh&NpVyrOKwl2iU2#$lEm^gfv9r
zQg}3}Z{hjO-NL$waaUe~297?N%I*CI`3V*$A_@G|udFH!sMGM^$)z#@l+rYj3Qv!p
zh~5^R#huP7kOVfUowI-XbXPU6+@iTDUDB4^B7QAB)tVR|xf(Dn--}{+#qY&=<saCo
zJ1FrnIe?ws!7`^zwHm(vJxDOXr8FIP1oUs-{=DO#Xm2*g70}Bjz>sWy!BJ}dFOG5!
z^<AaVIqPm|IrX_Hf9CWG6>Z3QN8^0?%nvtMsfY!p;bI^-by{C={KvlW-+`cb^qc*s
z+s;7p@`}_{^=(IhZc{(QY@tTRwgUnV=^xvbzT5L3zJ@;{a;5_Jisx^jR2c1-cHW=#
zs8C_Zi}Tjro)Z;e1Tp+=3yzADI43+gacd@*3>@AzDqg&mGp)>-+ApuXkVapZz;c62
z>gwLv{!kRowoswE^!*<IQ`g^q;gi5IDWPS<vE}G`){j{&{%EH!;*y*Xeb*{B2HUQ`
zf@-+&G_z_yspqcpTG6xG3^0LwftDLMf4WyvOFu{<1gHm#;0)^dWpd!ZX!0L%G^j<4
zd@DggsoqljvM9K5^L`8!Icn(Xjl~F|_5-qohb(%5cI<TPY4^<wkdFjR$fpuupadC?
zYW;pU%|r_p>oeo0skpyrcv=1TjdOb#w)3q@11>YF9SDCn3MKG!9O(u>dI;76m{*Sv
z`}(s^#SmDSe^6ap8zXfT(<0d!;Vdo_h;FyspL)A>P{|HdS}1#5W5E}m7pSP3nCXVX
zZ)zTIPJX_Rs~aI7QyG8u7aW0;4X*%Wcg-rkg7rMXPhClwj`nt0_>P?Jv3A!`6CKLr
z<l?<qVwaTO#vxLxqT%viD4vR#>USd43Fc_m1<Fir*!0YrR^L3V5PpBl<KXw?ho^Vn
z#L0DYI|z*Ia61-_&$m+WusF$*hlVDKxrrQ$&IjuBz#y-U-kF)a^lB?kEWFDhg->^J
zUgflATBa*0;wTS8s>t$xPBbuq$n(vm3duYD(H*1+BMXY{Mp-ZBk#-_%C~%YS`>)&o
z3Hh@@Q!EHi%tOtqAD*zw`|4N2<e)}M@wr=E<S41m>kmhME?(aVskZiTXc9|4h`3t3
z+)Q)4#E!L_3k{p^GxkCBC(uHapRAK^Svf;h)-Gy<prtzJLHo@zD#9gPVaR6Mr<~C=
z<kx~arL8bzgh_JQlLJ$u(nl7qp~y9SMbSz<eVW?OX0!U8N8KpH3-j)JrS0+&q5mP!
z5cv$XMMzK=H!9zsORstlX^y?dlD|EHzJ<*xU_!osN<Rlu=`A4gu(G~|`J;egmxzDv
z@bg6i2!zj+ICS=Sqw&tK^DGFLIyhmRN!j(hn2p3d>hhvC?szF`s{dyA<eP|mT5Qmx
z-b`eCX|CjSjbCc^XGqPROF(eYxe^wNv?k-NaKy01NWj0Ee7pcFPY%2C90Loo&^hIv
zU$69`$<7nKxm8>N-PHL>!?xvjGC|lgM(XuHgYuH*sz=|TH)JL_AuWb?^_zXaW%X|V
z6RVKnjIVtM!a=H$`c*o?+as9g-F^I)Ka&xGDh+tcJFSDajBWr2`3l`XAN&JOAV%(J
z^?$|4u~8tjUbq+vdG9+{&d+BmG`P^4hxjHRSf-qB-YOLd`B2H3Sg~nTX0-AKnKF*}
z-dWHx*}t|VmBTM1{u#Y{0-`Hrgvr`=Cf%B^x2lN|%q@^NsOeY>^S~N9eps_4lmd4s
zuNvVpiGz-r6rNYDP5w4MlnM1d@X!<L*6K=7gaKAs3SY?%*{{zj`@d#CDFhang?9t$
zy#xNg7vEU~-McWLUq`0g$7SY+BJjEYgl%Q)^mXp*b$Eb-)ZllPm-m{_oAePM(nLy?
zhG#+OCPB2V$Qzxs=ecc`M5NHV1B?B$o3KFcs(?|~N4^`<b1QY`T|AmegGc1tMxWv!
z_)yg%ASjrRXa4<#uee3cUz~op082Nm$gSzcJ>Hp%pos@5C>9Sd3)d_{oa9+yFK2KF
z#(AJCr6%x{sU$TUH3dw}@(kuxX=Lnn5MrK1$Avt%EaW!FA&4HSYUEl3AM%k1w2h=x
zocj?taqy!PxmDp#=KeqVh?(_AUR;4Z-U7R@m*;XVsL=ZAMfC9Y=>fQ@Uy6zK?nss$
zPg?HD<>_nYqRjsxolV&(j=#P?JQQhBWGwptnhwcf>_30raf@hzJ2o9n#&SiStB`vh
zq>_+6SMR~{W24pX`dv6hW2;~WU<eTfw--gdn>|_A?jW)=FIi=$ix6lWe-0~4iS$0n
z)}AYo=fELe*e~~FT($@%ko_LACGHk<XS&X#h{(HL&~(sM86*kK_qr~oXi}{a%&~-m
zHnztlx*59b4zIpcV}I1vqQOqhbg{lPFslCr!<Lrww`I}76YZr6?Jn=DCe@@pU8f-(
zJ4~LO&;&NToEN~L3qgdg$I`P^EqZ`(fK0JJ*b=wLy5Bm!2*1n<)Ct?6M@3YDFKaL4
zOS*(L(b(F_m)0nX4iK7TOGNd|u+;d2U)azWD(l|YajCIY#ZUo0XkEI+JebP3#R+bG
zlEkBfNO6%iqZ0DZES78Kyq41!gbIVRH2P`QKrYF;gc7(N*bPF4Tf13QR`g7ogB#_*
z|9JZ&^FS)DA%+Ultx=J7UdAk(#-@ro93`+6UX^?Bts*}GY(96p^^5e<WmV-xZv+{*
zHGy8PJ63xEA|J=&E(noVWZA?r%zHN*76NbN8$!w=;>j=)myc{QAaeGZP56(SicN0P
zytEBjj92bPF1KGd!f$cxk_lvUHAmD%-5Ew=*)2mJXlr2}d^E4-DX74qtrb7b1A2=A
zm`7q2CbQWntf}}uxt_`spCu~XWS20%02_D>0wlNbYF)rLhDYj`IR|@m))TA0N>09X
zGrs=nuOYpg%}MftxlZpDet%s85tEkxMoju&<b`qm@4PT;Rl>Mx7o9inKK)TPA8syA
zv47eBVTLc5_4lSiy8R99x(pCd)bJ3kto83T(_ukBFTR>+*Z)wrB(4H1W`!Z=XLj9|
z`OvMNJZ3e<-C~b$nkNUV4*QHTG)yN|oh;_(c*K_qm^7CEBVu^_6gbGR(L-%xBzH0|
zZei5U>YW~jL0~!XNfdBQ!aE80oc=1maG3rq4FflndE#O0p9YzV@WAz+7d|rYe1ww3
z*GV1_Q?MFF-O#)_UcD4KG^CU*J&d{cZEv`L8!$7D2#sB2vnKcigS>in&VqE*y3-xJ
zz{8gCNHdwEqA<nhV8mwZkvBD+sA$7NtMA7A(=-zQcOf9TURTZ4-mDunw03<+MGisR
z#?rsJ$(4ZAV(4rEa2k(wUAPcMn}X7c4~<0rYh|oPnhXazQPR^wi%f9gIkaYWAewT!
zh%7<0<WAnkS6KL;JeBBvG_!lW4Tc+?xX!azvPlvYB!P$2MK$?dhT?rAjs#hH!sUoH
zs*S_i7~msiF3$JzG&%g#8-R@*kAAlzE(k!;mKL_QnHCZn4v{eRK2rL<IW%_@7*yLq
zA^HX-Vw)AiCe@u4R54N86JQa9OjnUuOj8O?cjni~wviRV5;)m>HZb3MNnQC<m5A$d
zXdKtn+33TNoP!@RoL2l^q7tD&z>`&6AT-~6^V7;6hK%iv{ry^h=T1a>+YHB_1I%gE
zce(1*Ypz`A+i&9Wi-u3mR*RD3Ij;MsA3`I^F7yeHRG?dYk$`{lOHc5%>~Ts9VUQwB
zwoH$}#{AU8;1I7HqsDCX<bU!i8(2rT!0(pmvGbT|Rj<yOxJ;L&9iAK_+mkX(2na&C
z!&L<+llHSjAQl7jXl33?cpwQiZzqC)%l({0oE8GDJarjnSx|uIVF;cGLT0k*p_qav
z&FP(wm93^n)z6)DMZTb2;x*x@h{r0XhVKIFYk)s^n%C)T(BF&^5cg4Q(+N2c%f$JO
zT1ba%A86Jx`flVThz!i{dP58b1fEepCWh|9@MPwP<te4-Pd(@t_=uK>zfnSp2UOmz
zU2~>ox>E+(CxskB*s8t?S3c(MY+G4+!-Yd6lzJ`qA@}nH0nj2XKAUS)5U$I*78Lp}
znws5_u=2vwMUQi=xb=Nk()MfYA(4ND`p*Z%0ZqP1)`=8&S=r=ED8dh3SadbS$ba1S
z)pma@4+!qbR68!3CHOL*O~`Y3fVjOyo<3XxG5h;O7Po+mOK>=`)Fek=y*PWUmTmLi
z{R9_VVE?GEF`T>>9ZdBhn`=E9@A+%@0GaYnj9I${1G}Nq+}Hr*LzkXhMd(|AIluLz
zJ1w#v8KSVsCRo0sAj0t}*nT2y63CLR|H_hknV@My=~K^tP2XUH35U_!PQK82*mxd9
zX7ChWYrd1W_9UKoXeg$4%oOizp=|p1ZpA)+<>HqZNarW26$y4ORD(@K{<ia0!&4e^
zkpr!$<a<@G4NLS|O}l{N6i2>i*?uKIF2)|~u+wqLLQAZxE}6#s01Z*&!m7^|{k1##
zo_O2GdHEcCX3}-zzPF27*yDgL|0cXtw$1l6+HwR8*kX-K1=--o`1w+6S02cU`}#JD
z;?m1cVBcGGI)X$e=eUONUETTd+Xdzq6=4u&5oSQ4#M@THLS!3J043s#-d3gl1hU!-
zab3KD4i_^eaf0!bV$|1Ij*xg}Q6c3{H36+lGZ;(C8qX!244H(l7<1$6&vzPZPa#1f
zQh3~g$dR6?q<On?AZ2u@39*LZ0V+m`p%V8^G<W<W`!Qs7$l=!O>mX`<vI~duL*?~?
z4@I&QwZkmwN0+ZVw1t)eRbppBc_;Lh!?m_3)FJOtus)h9m#-W;1)>Itb19U=CEkGp
zcyZG<C3nc`F_283bH0-J0AqsJb3IHjaWI7Qp?kuiE#TC~dlq`_A-ysVu~G-?Pue>9
zr3t<SJO6%lU-*ut?j4e<K8wB`-v7L~C3gF67upgCk1PvC)`J(9y750Ru5L#l&bIGG
z(a?6d$Tbs{)21z-`m@J)LS0cBr7f;;v#N|2@~x{ZkR;!Ba*XN;xYxUv8dsW%mKm4-
zGGT)^w}@X_Qz+-!$T;^Y2HA;u`_+E$;7;g+3J~X;oHs|_F3v-wXn@P?vlGE8rss3-
zmYps}&?wP@1oz5E4a`d1otf59)cX|t=V3O7^RRZS?_hlY9M0B^UX=A{RsX<VGtI7?
zH1WQB>4Vcj3oz|Y(pFR)>-;H<4ecbx4CDdGsXp1$O!de?rJ38@_CiuZ)myz0y!YwF
zvv}I`lhfo2lg{-PF5k`889k@6k3Ai}v~x@|HuohB84Cf^X*F)~O+709$8~lrXx*ay
z_FiF(BsXBgMCbN?2wRI14~w&iU%UM6E7<4+HIs93^^ROa+A9w}U;;^A9bdisv3>Bp
zO=)0%hr)g7mL@XXwUU7V&XwlWnIXM5e`_zI_%CNp-IAENiYQM+(*Gjh*|{9`^vkk%
z;a9y69)>gGcrE^LpSU;`ZyzhdETMM|J35EKj-sFU*^UwLWNt`6p&N=gNpx<GfPhvL
zAPyBDi*mDYh@#Vv&fMV5%;&qoxIh4LGcve)qOjS-GeIv~!P~gI@E7m$U}?$|+*8zT
zN`~AALczcyuSBg(A4}(DCmmLx41eo5TITY{K+y*tFdb?Etb!9OR$Q(%iU*(SD<AEF
zzrT`MrGZEChV`RU?`XE{x?0~YI8wgT?_PtH{Zk8l{9GhV<H3A=n1NF6IM?Nt(uiis
zNoJ1~dWJr~n)@$Zcle<qP)}QE8`Ia#4*i&*Hp4p=dImdW*SI9dOeUsB^zfVk@5p^S
z<<yl4P=t2K+9o>Byz%~P39DD4Hy>}FF;@*JUKPvt`#gx4AGgnCI;2>9HM*5Za6Da~
z_L1!MFdY|P5lzNVgL78xIb+DP@K_?e*&UCB`W`WTNqUqN$iJZ~vP;~&ul0aDEokzO
z%g*&LE{V7}bWq$Mf|QiCwI$ln1<N}eNAFN!GXV+yCeShTcc?JiOk15|`6%Y)?*)>Y
zw!<yD1yQ&kxI_1J$7Q)T7)ZK!!h>`ZVTFm!UjTIK3ydmv4V`kH;#KS(%^fi8#vy8%
zLSZVIURSRQ#xq}ot@V_}SJXJY;-nK@+rU1&6~lQEjwkj{I9|~xu(j<E{4ZOZEVDH2
z;`OT!_qLa67;Y~8no;q<uvP`@YF>nZ%_w5u6#omM*Lb|WbJHZ;uQ9NAs2x`1a+Z>i
z90RFsm&{f{Li0mk$-&&`5K4-zzLP=+Dr9XN4&lkjvG$m*dx1o){GYSt)Z~@kK3GbS
z>f5?!+#Xgy*$__5v|0?Z2<J5>R+IU~DSguwM_Oaqa}Q7mq$6*b=*gsi<TLUB`uFBO
zxoui}S|M_vUc$wEbjyR2^CoC9-~~-1<n}wuZh;A(pDG<4?~9PlK^okitd9+>E8AJU
zWPp2LP$BccD^O)d73>%sTKu+42n%ze(2ryyT`AUve9sLz^CPqI_`LaqsxtzVoH7GB
z>;0t5?nF<9R0lY2!-p6u-*bldl5=_t&_E{Q_#vB9nz<qp{EKNsfQcsLZ&+{frK$!U
z)pAOmLgy|Yr}-PH{r$cWYmsP_g4By`DlOFqt?z3_cd0kSA^NSX*E=6F*HGb}7<(X8
zAw+2)r<ZEppSm(a+#eT%A`K`^AQ4>5RJ|%zYy}3^5;K|SNf*}Sl)F^Ctf*8$6)NG5
z|FIsME$hn7Pg8BW-+e8SS*_I;&__t4NNbP?vO}J#?lrZrx}219Zju3*CV_NwrS(~n
zrutc6%?HVyN`r-jDG9v(jdJ5kx{)&DTH9oA^3Y!>Ld(QyKwOv(6~Ht^6HY3dju4*>
z-ik7D^MZS|kl<~yM}lje?(?zB#{!|q9C=|7c(6=@B3xk*-|EUts93KHWFf>95X_$A
zP6@aQxC-G;j45o}xcLEmL{;>V@ywUXlDt14c=m-ANE{sb%@Q^Me#r7fkKvSj^%m}I
zk27y)UF#MfNGOC}_U(qy!TET^&y4-uK3NcK*PjHoE44ug7^2v--%ictupf&JBs|_W
z|6X#(?ZK(f=D6rA>zsc3l`YsT@XLD8<g)q;%*(sXHa$c<gxQ8c;VZSHpE3_4N=cGb
z>8}7H*Y;*Xxe*mLMl~b7{>4zMhk~3oE`4>G5w>Zs_EwMYR+^-DRO&19;BKNg*){TF
z@jcDgROs96o-gTdWn-*QPTa=o42(QOVb*WbCXew-d}}ItDw!!9TYc+%V-cTMTnX&x
z(Ow%<eslrx;PA#T-o_^x*ROP}iwhk~76&vy%X^~E%pEc-40Mu(Z4FzL<L+nV>bjsN
zo1k<n5(Y9TM5|H`{!E^0tCS9jvx_`|3-g7lbsD%toesSeeK2wN2PZjdm*Ui@XuC!h
z?o@ZqM}TU3RB$f0)@7>~vp_j<t04uY#U;2)H=*_!aQkLdzXtd#Hk|qLLRMLR9Dfhy
z!<(uSdoeZ%U~_3XN9RODa@5myS%jYdx_=R`g3GlO;hKR@oHvGNUn&TmrKc?c^MqY3
zIeq9Qy#Opf?)23oi<i(iyL>?bh(k}7CJv!??mlNU8smi(>>j_WpsMgAt1N9Ex*R=3
zns<d+q9^0WKH0|QS2@xoX{~0IDOq>Or0{P&3(u)Jl@#rqAx@q2D5F|T%4{)!r?)tw
z8-0Q~Rb3$;Lr;5LD%mzAPgftP3U}k*#1m>l&;+e#A70G`(BVaNZq&pae>}PBz<vv2
zmZLs9gzZ~!2cONQ7hze6vn|x9r}J4*<riNlDCG_~^7*`h({JLZf$V>?0RH3pyL;a&
zCl#-9<d}bV<Ye9%I{;6RF3EYAaw`7XtzbcL9Qgk{ZhLPYUP-d?V($R+Yy1DT^Qx|E
z58dP}dijdJ<IY?o?C;%&_gu2ZK6ahYBnAj|*JpyIw?^0BQAzH1sUgTn0X{wt2X`ia
z@Th)khw-oq)55YQA@~EPz}5OQe|WASbck5yxPssVqu3qm0~FzAm2ssr{vdDzq-=)8
zPihh;SzqrY#6a7={ax%x2-C#xWv_rX;uHuQyb<dQb=qoay!%FnP5PI_M;szhga{4G
zHL&gmO2bXyvS^KdY;ZZWg}`Ogw$3dvp5~r-D}^Ji?d=5=33%H)fp?Pl@uns<n=F^I
z?s&rujW|+qss<K9krmDFxsYbu7+<DrUN4REiGDC+w55qiizYJA*9=^cF!|c`oB(OF
zcj|u5XJ>+r^$)WYN1NAV<Bs_$ky~{5N=SaRgb1_X((_wapY`)6Oj(gBC0L)-Ah_$n
zmn^}tkmk(>bc=U5O-Hdu1BFpUh92;}o4yTtmjISrF{<}-aN-l&#SRcGog#w1T?p8q
zOaHJzh7z3a8B$BJ`^y{G1Jqkih{u}lFgG%>JPvaiz9K#)NwxZ>BIAT-mTAso>K3G-
zQ{h?l$la{BC?}bG?_}*zH8K#a-im%fI$&q6^e7u9E}m{x*nRK*LtATaC@tbAgfUSO
z%TE|lNT69RSzeQ%m~gAI@lWPsMp{u$GkdVH(^JsB?+w1ce0TBN;R5LoC)qBA)U3YN
z1Z0`>=n=3k`a{|Y=QbGOZ7omz%*oRqU|;3`N`5XFgx-YU2%U~&*lpe(ZSvc5_;%GI
z&P>(-SBTTP#tc3rcsTxuk$*oL%~XaH!)dL1NpMwQoln17T1_Z|wf$zddo4SqE__vV
znhI0p%jrdE<!%lk#7T6;3A_|PZvN%8=ohlVMy43L;tsJxW6A#z5x<>h9H!euLB=#0
zFvl&<eu&0d)LQbEFQ=+`)-|_h0b1^!;ipN1&$Ts2@{U75aL&W7c!$Fs@AbpSzmLkw
zcz&*ybJr6B^%LGBDYEM1zIgZCqTSVsk>#)cwc!36a7rz-v@c`s#&x3KpPzTW7(ChF
zn@l?rWdDJIZQgPx5C0Ha60)kBbWJi_2cATVGj*JP_1^BvH*2$W<WtHaF^qNNZ~vq-
z0&B+=1@1qXlXca4eneNtke<TrCV^cP37U4y2_#MOLF}4N|4iD6mlq(}8ZOdP9NK8B
zc&<BHYqsVRE(`a)VhFoxsdYN|R4n5+O=H^?GKyBzf@L0W?ESNgwx}{gj)wBKE6Fbx
z{{u3&e^Cvkyfm(blU&zE_ao*6)-4EMn^yBmkY^Cyh#h16(``RuHYz*#9G7Vkj69y*
z`$;65e@dr(`7ct;1#P#2<hNFYk;+VP<P%(tAz_e}j_<rdtpdR-`&}8M0AtOIN4ovC
zVTN$=x7KH^N)uFz!(|*QUN0pO`vlgoKX1ax$4;>v(osgn=YvO0IXW5J&way#Auq2A
zk~mEbycU2^6^5ZJZd!(lEzcGe@DzHl$#`;7dr`sNIRe%tq+~<OAed^1yEXAVs!Lgs
z6J=k8F>j3TT;w<z`tYRV<_n(KIk^|>LV|VUW85B#4qD-JMb|@Achc2%Wq(;R@F=V@
z_rLT`s*UnGv;-zQ>3J(mL7npX&ikoZ@9x$H8|#W^_til4N5s5TW~%rrD;h-KZ|sbk
zLXO!_o4eMiSHX}H$S@^`V(i_O7*TcfGV=$eR$~9;Y!XGSSw0&r9#+hc?Ou_$XxA>b
zN?tQN?9_q+1pZ??h<x)ANIWU2T`>zga?r;7*M`}S*DEPTrko)J?@H&%HX1`2w7gs!
z!wc1~EY*me-r{VGe7elWe9wt7wi}b01bYtV+)>&8W6n*v`ll<{n~J}a^Zv^n$|fHl
zEwh}|;85qQkHMj$iX`$OtLVj}qJ#v(LXgl``W|qqc*rla6edC$*+L=^h}sh#Lw)p5
z@H&3`9TY)4!nQwZC7pJBJX4@XGEjbBtciVp6^1D_uE-=Js=FB;_KHdOg6vJ!KS0rK
z(?v@@cyE`8dW4k$SF>@^0KZ-e6k&3uF%OO2Yp@Uwu7-!KfWn-gTXgwQcoMFTkduGA
zpjC)LK<@=i67SUK<$K~X{i$}hdYED`x+<C>KPuVSfd)?0!P6y)h0Xkegu7SE{jg?0
z6`^@a-Q4=lh(0jvsfsO7VY49mBnzzNxCfF6W#5&*cWo=05OZ>F)AjXwNwms^A@Oyx
zG0<;7bGDt(i{PA!86c!gC}e?g=~nZSg5;KK-=UpK%<yI3#Pd2D7@~Zxh&z`ZzF@f?
zrpk>fz=Z#SY>8&-p?$+>lD;<@ikn+bh3hpYlum^rwQjLcDOvF!;~%(@!PWf9#^NxX
zKZ{rv{NFt52{3p0=gRwhJY3%8LL>n3`~$d58>nO5%s-6}#a2pllJ<jiYxE7mUn-;?
z?Y_9<-_YsbdK=yT5Zve{Ma7G;RiA!GFOi_6%7>03gS5Y4T4jtGp9jMaC!IHK5ZsbU
zqP(N6FL`?n?WQB1DTs@FHKUsh<9(?(T)I^0L9_M{?zIpk^Aa+*dv?6}8AkD38Nw{y
z^CX@K|M-{lMAM-qMin8h_2AQ<m_d~l@q^Q$u&QEC=w{1ftAPgjYkQTCRg@Rq#zXB+
zJ!qn<tveG8TG&D=aEMjTPJA%<_F))$_sjX}OUk)8L024ND1U_X_meHh^WT#n<5*qm
z3{H-OSsr!K!e?5g<SDPD<vYK=RCG)8Y2P6Y{CnQx?=q(1yZ&v_R2^TQ@z~Bj`&#0h
zK=S3;tAI0cm<t}0kzt;~K!)+FEzh7ysA$U_j(GjLwN?BhJxf;S?n#6T!{+@eqJ9y|
zw47rhw&eXynKN|qA_xV(#1m`Fl^MBu{n@^<(;pE6e7hY+)pX3BP`Czpew+2$07_rN
z)R?R}$>-bQ?xipD{oMM-shzjm@f8n(mIzBY#}tHA=lI?!kEL>vOBWx__B~WsQh3K?
z(?!Wd>QvDjZZm)R^!;t(U*RWC^)aC%<#`Cw6i77L8VA`5@W$SHq)h2m_R_`?*9}$&
zrLh=6;|4ggzGA>uLPXT9PdO)VQv^%HdK++f)+bRCvI&!?u?(mPKVsbNYpV4m-)DsX
zhFonTzr{x*el`n;Fi$g|Zvtsxf?cgjBHF_uquwyF^>g;0!v!k(@1;muvzxvPlzV^r
zanM`<RVLn9@)VN4rR-@=Y}THh*Qw>sk<E8V2Z2KuYc?)f_tWm)ilY}mObe)UeNUrT
zbiRFQ?hmslj@%y8TC_SVZ&xO1prU!Mb~ZM;C+z+XpBSp#w|`!@y}%0@a+5dzagdcZ
zdGd7gdhm*2UmFo#h1=--f!pVM)z!PF4qf8z@i+91fNB=F*?LVL;(59{oB0={RW2gr
z5a>&mRbq;HDk8R&?$VjTL=_T6a-ppa6kWRL!{@|>N0qZ!^J<Ass}9v*;}-cR+w(W)
zUz0QOOypgK6R;0m5uH1!*7MX2+AI0D6{*vr)EgvUfvlNv@<TobDyhM2KUb*dvL&^D
z#si64-64o4Dh=DK=G)tm^o75E5{vj;zB4&I%WZsBFRvD^vkDG7R2NAWY(c~S*hPh@
zRy{;3r7L$wz@Kjwg*n<Y(w>IYP725`RG8{e+x+_NZmyR#JFe6>??U`Q>R*4V3Tw=f
z(F;CQp^TuYHvb8uM!$Hl9kg71EH<)IL%9_@;pngjY*X6F4g=PxhC@Uc1Rh!iI5|k*
zK|9hnKFVGzP!}Qyn+tV_meRQUfsmDct0Nzm3CMnni~dQFP4$&ng^aQng>Fwu{^$x}
z69Wc4ACBz-BF+vJJuxQC@;c?)7nc;%<P(H&@}AS)qaIIoP{866MkC=$Kue@TW6YBZ
ztjcN#E^ZoT_#$qXOL49Ww<Y*;^gi{PB1buhQxp0A_mCybd~2ZWI!i$dVdcH=_CF4L
z$-l%dKg?KYAtmK3HQ9Q`)L+Z6{)ERbG4j5XFewZNhnsy0N=X7Ux(6lvE@aNCKXj{%
za?`8Np0r<FDYDq~gun^HL8z8CGDgu4Q;{%k<8V5JUTqsQ2K@T2q|$2mv@L*Amyy28
z8q+zjP%vY=t53SW2Jvx%^D~RxskR95C$@6HcfFKO9HCaCPKf8C{@NY>q<xI-w*Faf
z*0rqt$Pzo>wImpM{7XH_%98)c2{s?{|I^*ZHGt4!#PW}aBHn%c=OPa5|A41|+%CWS
zvbg=7ec)O$@~`#uY$i%;TxG*e(oDqOGCtPW2NR;lSERzj8X*2i-?kZ~Yku33r%9>L
z8e?(7RHeWI!+A!xGLTdaQcxo)VCG$0db{E-Gw>AeY8DdU8f(`~--is0*5A^nZo6I*
z31~!}U>2to>KRt0lPyAd3MV;Gk>k7B{MQ`fr0ZoLQ2Sd*YIEDR8HZ5c1BM<fB(lJQ
z!{Wr(&~{*^bSro=48Kw)z+ETA{)+DBl`0E4-_`upzn02mAG$KfD{&HE|0)NH3FWap
z*=V`)z-!q_>(ZJ##29ii|F;r#*rFA9aINhoF@cR-N2bii_U7px$Dw?cf~6beSbm!{
zS591AyC4$yhzPmzJtJG+EWWNE9nVvC%1CD8NB4SEg&||TKv8M@S$n(<QlCoMg_JJi
z{6ocLtjU5NYdP}``Hl%YRmw048J$1e2n)H8hVo*1R9gVSmP&G?gzP5i<*`|i7jSKS
zB)ay|=xbWn&91Sc8ig=y7x_p31Cc^A4My2wTvLn%l&}>x=yipa)m^J$Z}8jv`S@pQ
z+48u`fUt-D_laJ=Bfa~YbdZr&RbQR<)F@t%xh%0PKQ{I^4ME{$=60R?RiT&3jfv7`
z-jE9BOQ(Gpp0;h4OT9MU0RC8XF@Lae+QOfqB+$+4=GmVO+9c&fqlcp6Aj@{2<BoM6
zN~DLhPf@m}+2-tHuD7#a(g}(np`k9Vsi1QTe%|pz(uDg(DMHQ&)1ruaT5fuf>z@&h
z^}DTabvv2k`s%|}orv=^%!YIAh*j?+S_%RI($p%M6tR4QH=@If$;}p%fjKxco{c3k
z2L6Ma3fCcibCu11Vi$kKa}|7-xYgGzC(48Pse%a;Qk+3zLv|Lm?~j05$joT!AoV{(
zzfSCJveU@t?dVsWRm~fF<4IXjvCheID0b^C&7Oy>dG`zCL+uByZYSu`k46q`(Ek!X
zxcwvQ2TP4j_Z=(l(s_T6Uu6_A&M$9b2IAkFn(0-3!CLd;^UpX#4xX9DyYUxLk~?v|
zxMKCO^}&*1*M#vg#l0J}!D?43R38=R`1%<5z_cpU!Ds$MaYG8`x4y%&3O&bU6dR2^
zogXI(`yhtwyEp6nVrT@P5{6%%8)1vR?<J|q`jsw0h2Sz5KDgkiBkxaj#M^;TRH`9W
z!$fy~c0LP3tz77eJyQXHMXJl^W35r^y>ErmlhOl8HLU$c52kvM<%$iqL~y7EmoaI(
zzu8mo)2h~-wTr)EPbyu{pOFiMmf>oN2C?}@33JhsWUq9oRz@mT9vrExdT5RsJAtj%
z@(9tJ0<C@@(0Kj(?*MK2-eg6Oy{pQrs}CIjv*9nUgr))2NXj23!iJuuy*~jynyAIV
zOZ5aV?<sw>9OuJ_PF=na+r8J|FmSna*?OriYD9Y*O38^zWEN}Bta$KICyh&XB%O*=
zrT}GS^fzo5&?$yJ+Dj5za4u`X2sK7F4h#l3E{JX<NXi~&N!;})_^={W1uH+bdSh;T
zfFqxeekNEjkBwqrbeECOLo+GyT)FWkhQ1G&ErtuG<`;6evsrkmB-xGsAJ*P7tjev6
z8m8HFBOuZx-7VeS9nxKblypcph~TC>1StgsrAz5X1f@e-QsP}(&w1i~zW4n4etDhk
zxi0YD>t1urF~=OEd{QrZ+R}ZLtQIxHOlZ8^%2`suiIp5Xf5?)5Q5sCtXT?cqf<P1(
zJWJlI6WNtHg=)!5xv&T1>G?~DT_n+ba?tNgNU3CctXQ@n&H$Ezusse;qf^x63{)qE
z;;Z(6-5p=Gf`hDWSL}iNwI+(FWOTg}#xc==Clwzl8%Z6C1E<{mjB@wa$h$1jQU*6#
zhN3l`BnfuBA4MuyLf*y0(I!J6)|mzbY*Gnb6Rs0O!~G-nqBuIQ2plj_zu|>RRFl_~
zs^>B21>)#b>jWZbB1Qx@35zc=<_IQyH-!`jv7DS&Y1O!YNdKT9Vu_{jrAmL*RR2`!
z$4;c9h_8AW4=K5uBn3jj28bJ`5e$3JRic%Dm6xGktAR0rbDC3KY*`DYpApMBwuqru
zHU-C({wq3lDC^(Omw?ioPgmN!>AN6d_LuapR<wo#m8`)L2&2JYoW4u|ZOrR>C8=|Y
z?<!S7&aLKqWNVj<y^rz_bA>#g`$m0yl+UhOw;}0a8k4?~H+?A%489(S`f8-2OL%oy
zQX52)(?z*s_E3Ia{U#8}Z%&<DUoQTLHitO26}XTFUEc3<G1U<1TEq_8&18Q1q4_B$
z5>xl!;Ac@~j1Skb6gzjg-_I4iRhVZ)&!Macx&bd$d$TiG^fhj-u<Emm-x)6t#`5f0
zETOK(X?G(KYS1GaGALus=p+e7bfH6K<P*uaqwT|Xi@mln=x(b_YTnfL(N9M;In8Lb
zf7+?&j&{@5(kdnp38F<c_NCFGb<as6<;fn(sLfwaBI(Du_0(8GwUzfs&y=l3>oI5P
zA#@#w6bsT0Sh|1qsjM%G9n@+cs_bWzl+Wlj_JM>*gg?`0P^o(}D$ymF5b4PI!s52_
zu_O~>)p*X@Wv?ZFI11LWvCKn*;f43Tj_rvyAKA<LRLM}4_wUsVJMrJMFuHm9rJMX!
zM(ATWd~`FDu!N7%4P<vs$MrQILHhF`$TixDktI=(_C+n%C<wDTwZAkf=!^HEY-4;5
zL~UUbRKuvK#I>Cy<m%7;o`@1iL_>VIvVr^U1n^<B)!?2Lj`!33Ij=;D<2kH{?rErx
zt$`kRV<GzCTNs9kbgU0`a&boJ;&UJJDz|&g318TPtiBoRS5+BnK3TG^xWWcoiK5=+
z_3Hty`doRt_r;+2PMo8yX65fHHb;}h*<WA`L{{P)8BAgB=n5=f2TKi>@evJh^l=1U
z;M*y+-F?SoyY{C}ARbl1b&|@uBPlKELol$V$<q}gN#o>qD4jMJDyhH5TF3Be>l7gl
z5?}d3+GpS7n|5?1@{C`a75k9kba!^tqXjxfieNSz>dVKtT0ISo#fw%)Qc53E<zxRO
zoyo6MXq4#`k-gS-_4q+v!dba)f0FG)xmZOIT~=rj#)Rs*EgQ4F?sQMT`dE}(#5<BG
zT|cvk`2@Sl00GZoW*j$Cc)1X~Xd;9<idjq3;A;0xfAC2MDZ;z6L3JE4?J@o%Mz@?o
zv~IPz(ij_SHOkGyI22D_XYg`=!-e<8ehNva&eA7He@^pgnUBEf4L(RxUL~ImPZG<6
zRQTF_$9Fm`Kux@G)wj_;HF9Z?t`1ET7qjg|LqjP;ug1neP~r?PL?m>{=PYD3i{>-<
zirJI**!!UU{0h(E&B6nyQ8ngE6VG8jfAQ}IZkzUpKt;G)RcLD$IpvvQ5Ki>$Xk48%
znivH8=|{aVuHLuP5e@|-?uqQ-fTbhN>GBVjYDuejv~mhj)I_MqGA~xc(CPieUp+v2
zb6j|<DZ*0m0d)dp%48}q3L?lfZ))yui{taThm27c82Mx#uUeK9m2o}SsWmetj+rKx
zR5N7I6U4H0x9^Uf@5_kfnyEH>O)k_3)#&Jyx97VJDG%qGE)TP+L0nDqGkHTh(@}?Q
z70z8wYay%85{^$sIG&E|iLZN9Hz&7BI((p3C)i-Z2uf2#oJ4a^wmpHP&fO1&*NXu^
z;`>e{nzCT`X9vVL22St#Ik#JdRtbdVRd^d5w`fq5g=RR&q_%!9eElJ$oAy>Z^f;hE
zCNg786X$mIT(vo|W8T$-oH(R~?s_Cln2Fe$38Y)-K)-$r{wT`va`tegt;b@Pc7~4C
z7Tyx>u*U>GI_V<EsVPLJhpZEs5eb2;4h192ek5L_8oo}BRKbnp6KA5ut#tyGo5Xx9
z+ZE*7hdvX&jh=i~lvC%3?*So?opVpn%OYQj1YTv?c~Nv3))foqT7AdA>2i?fJ$W49
z3>tN)d~X1)!t}+`iku=t_FjUmsRlv!wPWLEX8RA?1sr_hrArW^q2OEFQE058dvg3N
zJJZEmoA@f~&iA0lNAM@wL1w^BtK_(!QSF9ATXveq(i62oNDp!7VwY*Ymab%>0~3T-
zj7GILj*l{I?k($8)4_a-WOr-XM2!n+P{#6%fcT@s#`d(e#dEdEX6pW%<KC+dS5a*j
zPoCN8<HHwtkC8}14XAN<J=kVW`_X-BGQlTMYEEkt>YX{-JeL(;I;;Elp_BbW3^OF7
z{0BS;*tm~3HBhOtQhoU(9hhe(yM3u=I=Gh0UBAgP!p?cPI8-Lxf{3pgoqXsG0#q}F
zo&p?jzL|14t7%lG>=43YOlBE5T`VGskr{>P&M2p%g_u{zz~bYyl}7eQ3H;aeJz1mb
z0T&Ze*XxTTUh9Em=X;NQ_D#BCCx+gjQ|XDm<-z5MN9S{A2jO8IM17L+9hJ|A-lS6M
zsQkc8l$7fwt^P6J=r9?=YTHZ3I7^KlIuv_khEe*KK%*cE{yVOd#v?J?uz=%nc}l~#
zC~A7msBc!NK39YhL~By<KI|Uq206=`QBUGB)FhR(KtfgCFH-^L#kO8SJDq|$aJc8;
zSz1_*M}XkA93(EmQwn*wg*`^9$N2%%&7h0BTSha+_3>mc=d5hUdm6(q8zF*g3|}41
zRC&;xs!_}39BIO4s3Xsx8wfmp*hTahi6$Iw<3nrA<+_R3AR<p419#JtilbP#jg5R}
zgl_se6;@1S8XOp+<3!jL{_9%wtjWXK55A_+Xf|N4G*j0ufKMrXdtsiREcY#OSx^3h
z&8Ls*MJbQ8AD$|FBb&i8W|+Ld+57m_%_K$bX5m3XfVAn<V}aZX%YO!#ZD%%`6=(LD
z{)eyoAqP1}jzWP@D@)ZjGrz@}W?29)zvhwEGBACe{UItZitF@d!5UcjB0?K-&bx$d
zyrcPS>2m?lAIp~vuV&$nP|M<u%tt6>1&0D?eMt+%y;15>^n|saCnddG@~;y1YGl*M
zhqE}9$F=)rP<t!wfvk4-<_E6jh^{4;Uf%ny!Z^Ah5O`l}VyI0qcilPkw1cebQP&Gr
zLmGWHUb{<>s#*?o#QvOmLSg|IZ-U63QDtIaany3Ft*|{v2`!#FTM%eJRcF;7;<W*X
zn;L(fO%&4u?ccTHsY9V7Dk9=AGpQ*@!<2Zz^#!PguXUd5v}Ii%10GWo*hA`ijk$o$
z=bPv#y&+3?VorSje9`A#1)sO<LmIt%$a<{SHL95_F^id#tKw~455S_>c8WX&7eh?k
zjlF2m<=2PoYJ37=;QJ=0`Ic@#;3So>llZkb<5g15xGzOH|H<zc^3REY9UX3neJtO*
z-AJA~xu_NB;$XJ=^h-V9Cr_6^#<v6Q2@@Fx=OI1>5B|^X!1XraoHW#cMx*A5YlRG=
zY<8<^T7_1V=*oR3R%628r@@C;5B*OE3A|>^%jMkcE9zSuMp{>{>ZV63(z2%Z3#HRj
zT#4?#;(ULk(ljt^(lFF*{QGcPMoLuoHyO-QvMjckflnX_5)^ePr<o#38n*~%91C~)
zo1708@0()?+<%q{?h0s*1X(*C{E{6w-Qq}dLQ^={@~CF28p4&VpJi+<hrt?}8W@RJ
zuTBX2?5i}`4o^(tPynq~n1vmvbmHyDLoEKOU@>zX$Qtq~pEadDuv7(|3Z#;42w+Ju
zf6#tUS&XqwG0Oq%ZGS&vT_4&*NbyU+mv8NPOI0hXsg;Txw55&73(m3}N0~+UVzbN`
zgy7R`ZH*VT3afKIC8?r0`!O%QK9rHB39O50a7{nIb)05y{ZaqP`VfSONrNQ-ZpJ#%
zTfhx(gFWC>54#a)LBJ(~4cTxHSv&>_rIaCXOS;)_QYbf!bPP{1KqSg$)~z(K4L!z$
z*l+0r0C+uNfENa~17w#2P7-om=iXQ#;j1z6uqc5rDNo*j3-^!C9cXHFLsvhyrPCPI
z^`a!-k<@a!KE#l%1`cbe?O=}0EM%C#K^c6_^%d+U_-BKRMqEhxk?pvz+{t_Gww$M~
zIN(B<qsEOjAmE4~n?`-F$Ra=~QgkG$;j+Q@`>yz1Ou-S6u*QAKydl#X*MeuGQmv@~
zeP+yh9ntcr=^L&Xl7Q>ffcop3Md`?ApMRgH=OOx@RA$42hkh1FmZ1{2VJ+oGd!jlA
zvDTGAh7^^PbTP3n1|GzfTEADr)qVSVG=H_9yFzhy%{kLfS7IScg-qlEiS6an1?B|~
z=cSny9af|l?O(`ChMiWT46VS8Gt!dfr03`R3HHMzs&S)Fn~8U?tRh8bVV@M>h0v_6
zMasGmPzZUw5Z1wHr)qYg#!LLXy-H5tL=yn-aU_HTgJ8jG?b-QW<_#z0K)S;-oW&zQ
zdI&ZSaml~rkP+g+^gWM+$Qu*-{Ua>;RjNaUk)aLk{2VcS;PK<MpNxo5znTLM5B?yd
z@zm=G&<zL!$<)$6`3?7)>1ROxUA+72scOT?%I16vm1;C})iDTd-3Bn4FLkPI=eIYq
zLK@)Z13p%gYSDb&NcdJQb9o{@>`%AGBvC{_yZi=q#f*uEKwu@ypy&!A3~MS_0;-8J
zk(%Xf#V{jC{{uMGB)oN=FN|ei>>>Mh0nXX3U)bK`-VqR$yfa=9&8JZuri7=H@)(#R
zmnvm2Z^v3`R0_L)gIgcXGw%c0#4lj>oad{1>b$w**ZYKSb~#_V!PE1GAkohB+%J$j
z?fj3<)9?4Ah5;+^#NlXhNs$adiD_oAtT4db5~*WKHu`nkaf_%^>P#BA&y4r0$Hf*_
zm^xUH+OiRk9M$@&f;@!cLmnWMuoF7+W-@Na@@7t{7B2ezc+%sxDE@0?;`#b^qvm!0
z*}Lg&Ey+@&98uo&SC+W~*UL9{+eh!U99GfoWM0-a#A&SRogw8Z(yEXB&{y|cnsLW~
zaU5B$8`-)POWePIITj?#M>V~zw+7!wPQfP<_DI6ZnR)w^C70RA$~U|39DcfQT6Gq#
z(ue}4Lj|NLeTu)Iq5yg0S^leG9nc)VvU4^=x=+CKImHAag!y3eeY?NF8zlH`mGN7Z
zOgv`qR;k{etD^;E=_r)-OTQVu(|-H8nLltlk$&s&ts^uj_K{GEL0W36l>3V)#P<2y
zn{)H9V?{^D%4-}NRgiJN4T6@^6tfJh<XC^!r5H2-R>To_TWzV(^n%H<+;o+Grb7-c
zFc)0K>1ugu*=!CoHqJ_q;UlvIIiJPA02nVFnyy=8k_F2Ye99CfAlFOA^i;BrCEfJN
zAbq3XiS45-0?T1jpRcl@E6B}doJ%ye#2VrfsWS2CCHe{WQa>@bdhaQLjk2*S#0s3X
z55A3m3Jk@l=(`wDV8W;$GtBg`6v#C#|KmX@Rn3w6+PYkm@-RYXFeDUl3%E1Hs@Rw~
zn$YI61w_iWJsI@2{7%X$(cSmw@JOuvA+-Y3NZ@Pn6h@SFN?uTnUqtgheZ?^DS3;nj
zz`(|q=IuN*7Q<VWSxgYkuJ?j5X6w@)J;YJeY~EIKfpm#B1AD)zX0kH(*>mFj_s<F6
z#7=bUT_?+hyT(x3DT4CDxA2eUrvrnMsNhrW&~Zh>(9wQ#+Rw9>X?mI-1TR4Gxh}+=
z-2L6C5yEbXSRFlK08bi}eg@&JK?L_%Fg8MJSB8y?o*=KlVl9opWL79^xh~6KvRIMe
z3d9E8O!#^vgNg$<BnM#e>d%wEbqHn^+x?y1-z2b+DQv*mK(t8&&M=ki6Tq6^9;8M6
z1S`%;S_u^2zrd3gg9VmjNW^^>sYIcq=_A14OMslc0dFBa8=zdOw^6<VMeJuGlnHq|
z#>E>LW5|FBc;-Wi8VzP%enV>*G=c2Uya7$HHp~z8!_g?1&u)W==}bwm1M-5I&TjO1
zE>-U%vv;nhSqD8a1UewG%c}0lM-UX;Xwi-G=o(lQ6)&|v+XcVzXJ&(#eApF~TBw=R
z8*pdU`}yGGof&-@5CTnt2)#;5)GsX?*-rM~yCVI0`)G=WFlj;vARI%`@+j&(aNVf|
zUj5XZXG*uk!e6=V7tF_GHl-r!0^SP*J5v>Di=zVr`9QEb27cVSN83%zALzSR?fDFL
ze_+GQR>$}NkQ4roAZI{UqV+3#GvURLOl9VJX_r@fi_&n8%OGEDLgW@ex`}toK~<C|
z*ni+J_uufBJr865y#OS=jY?p;JWrl+V#+yLX3(#lHq&=f$*zR_QE|mM36aCZh~TF<
z9gZNah#i3jA002Fg{~yE$(QhqTb~`kjg4hQHyI%mis4DNi)^t~Y817-MeSpA|I^`<
zWzZ;{HvdDZI)f0hrNoKN8<?=_x!l!=9o{954^dA3z=-p329&`|1A<kI_$qZ#7zDZz
zI&ln}2pryjLN4l?JR*J<kcFmps?jp{*2Q>03YhyaOq0CP)oZlZ43qwtLyu6`r;_?o
z$ddsarr{c6?tSpemw}bX-R;kbr~VPciH12a31tabHCtqe3ML$(oKi{`qV^?Z(g=}A
z);I(0%{T}js>D8Z(b7s4{RmQgGA#sWscd0MdFob6p{d9N+MEnCFUQdWF|{xUusO@S
z30V!2B@Es)07H@q5Kr`)5q|zVNJ*YanodkX3gu*)k!WE^|1QeRt;#u(m}P)yNw{C7
z!EZA}7$%j@ovakk%)u<0YD}Bu0IVo=N|cD?r`bS`Og6tuWm)@C-+i<-X&a_Z0Pysq
zc<n~W!-n*kc4uoTJJBRa_2zpVU&)0}!V&hTOIm!2E<zV@lSJXky~o&-E$VkXw*U1#
z+AJyNS}S-?N;I>8=~76*p;|Wa`TBH&d38gOY}7u;C-&9kFqQ?VTd0u7e4TgL(2`3Y
zb#4EEQT@TCe|KJp7eZ)%*TLWAP=X(~830bSq0Toye|iaRhOzx=vwlM-YwEu!d_%U?
zM{-oW2rVbLx-|?6ya^FBs?vMGmvJ(W5f%)Ovd%|cvdOTbd7WMqk1v@P+!Jfub*%l4
zyp;%l=E&uJi5Jy$k5Sw+%vME|h`GWhU@4%*{`PDo;&o2@;ef~0&zwnb#X@2`fQ{t-
z`M`O#icn(ti@A@xTCpd$*493m<7@e1Ajf;=c_r8n=oAxK6Zu4`-(8v7pRSBOYMzul
zaj#GJYgh)%x9@8C?zT++VEJR-Q2N6n@cJtvS*ilOdksMezB#?8``Rc&Ef<qr096YQ
zdN>NUKHGPN5)2TV2+PSHhAR!L3VbcbMtn=PB7uk2^I-01wf`wmMo14e!ia7!Sp)MF
zk-mu}70W!Sf+l6aVQ4NLiO_E3J+rY4x#SD5LRt)hZ+9-Mz4MhoJ~s0sM}WeoN|f*2
z80@`_;iIZb=P+MN>L{70bg40Eu?Kyma0=x6K=?0?Y+~#>2A};!uZu$qaao4swfN$O
zwKZ#@=OHf<5QWxP;y7MJJbZtvnkV)(X>Rt7g|ra}NHKo_T%yN;X!a*(`Tg(Ya}RMP
z$L&G)rc)?`_9%}4+-OM;L|#0{BrV(wN5tqG2BE!M7GK}{Fg89iWBYTYW*bKLECEFV
z4GH<z4Yr2V)`Iglu>KdO`U_em&p~|oP*<j*DwaiquakHYWMelAcTSN)`{au&*xp`Z
zED-S1d@XhVBo1sk-&U(KCfoC~DwmzE+zqZSF~8+?NmH@1ovZ4dYr4A_S0S5b%m^|}
zZzsy^3pR)#d)uf#GT%&-bJZl`I14TBh4Gnq{{Wpr9wF2Q8m2p%=32k9zdU!~z0mvB
zd+mY|tp#H=!Sz&v5mPlN^cy;PZvTc(U)P(#8sJur&(b)LBoX4sw09*RfGp`qaln~i
z_cBcAU>7YwBnH6cRS^ZM%j5OQS9d)G>SL4Y;;$)R`Sit;N+*dPN1MB=(ayY7B+}U0
z9xDhp&r3jcuNJ`PGpOr}PHeb=)gItIdfFp?dqHdZbXxbS&JWePG~vDgel+n={#&GJ
z`i9dV^Jz-~C#sKpK^@krL${LP3`bW4iLR$xkIy5s48k$N8f_tCUVzr9)6&tgKUe17
zh9J4+Fvw_;hq+amVL@zQ%CgTYcQGEgW)jVs{Pts1*_;+(!kBY2Rr*Q5JXw)yzeB?;
zg+_Mm!de9?;AW<{pt+4Z3zDHfo;ax_1;)=OJvw`7{zltU6;xt-;GpEc1pM6<@tr^`
zM%V3L8*U%t!TbNySvkjtuiK}{P(IQxQmn|0!p*gRJ{lH$_i)M{$xjZ?!Tb6J^@ju5
zw~VsNYlVKsN)eIsxerKrtDiCN;WG=*^D$PX6eWInMPBgrOM|(&_ZKH{m(+j#e~%7B
zzd>}$Q=>}nv{-!+%vD4y$QJnce*@||kzeHf5Cf`?kpklU2*|Nx7`F`Hd?HE=QsRZ2
z-9hl}Wt3~nI&qj%^t|ItwW7v&xmHOm1+x6l^mIbuCu`9Z#LkR((R}v5fbE$qXtt7;
z62PEM`DxOu*)vs~ZfqTX3jZzYif?eT=tb+imRJy77@@Ny%XGo(y7+RY*1R^VBIoLK
z_tP$FCCQLMogDS|fM?^f_Y|iIN}hy4!Tvp6JclhGeliP)!;;;Zt<5v*1el3#&=$1h
z{x28mv{*Zz-gO}2j8$mgaxXWJNRlIt%aiUz`NwK@%JmxJIjKPfB|#b)18)82=erPe
z1pwHA<G%*j^82hv8}#G3f#>bz=41#wqW#6;_n<vs#Dv%6Fxkg}iF)(vJabb)R$_ev
z6vk4Ry(seb7g$kJk+9b$t~TijKAX1#2u?mD9kg`fF#^Xgv6!W)bT&rwmN|wV)}dqp
zL@5nwdd8(Bos?lSiUmMpVy9c<()<0pd4CYR*{ImNIDp`vARu_B>A8w|%dPq)6C7@c
z{qEiVx@Z3L4+%@!zp7MB0>e8o5r~A}Mrx~}Otx@B8X7uZjD4A!^~B6+tU{WcQ?ME%
z(^OAaDLb27M&*Ww?rHtO#MYo$MY?~_EB-uvVS!;oeG^s^`{nqoT4>>{T+SPH!bk&v
z$aN2FYYf-F|DhG?q^M?AjjDH(i1tl5Ks7~@&tD!0n>vLLUgq_5{2rtLV}x$roB!x(
zS^w#g7{lig1pnk@1t(O-9h*(Rh8)ZdlYdMR7sgbB8Opf*3D};I16D_8Rm^N`s>1Sj
z0PHr}d?K!p8v{YCt;$S`5?r13Kn_RC^wz)-Sep$f)W8|`Zb5P6-KQ-01s%gbTiStd
z@F(pB$)r>X;+_^tJ=~PoXOt%c;PJX*zZiM|Sf11#QmZupKh#0od@;Crq*G&R+Ruy}
zm)csmVm`RZSHmB5l(QFbgSN~NKXZx;x-K72Qh<H)*4|w1`JE#>=<3Q~xt8hzLD#-y
z+}6=S&Q0N-NE6}p;J%55d4oRPdMBF$NP)5Lvx8-I^+%_K!+GL)5-SonAeOG=L*TE}
z6wEM8(A-n$rZej*aeQe;#Fqe6ku~nZ0%$(08b2hRkqf#@h$qgdeR{!mGk^zkD9H=H
zo!333GBq$gX%_HEBEyo22izFmPN)(V1e@vui6V(Y5;w4lu(8tn7|1e>k6YrN6ebvY
z)Cd~i)M_h+SQV(CX6n#2`p*XjcuqR408AU3t$umuRtEYCZSWK5g^+JFp6`-vekqSF
zT%l>_1Jf%Nt~6V)pQ=Q}tV4xs4m_QOyKby_O*TnbJZfLQ)+mr_bed&HP*f#_C<)4&
z2OLF`I{}*%DOhMP#8E<{UGupUq7oO#6xWpuGF~&Rw~jF<A#^QO$&vze7}nlWyK!Qs
zMi#nv@O$bRARB%O1P7b?dBPn{ma<!#M@MfG`gnx#mU33Uo4DWqO_4)T;r~T%LCS)m
zQQ!^v{TKiH2M6AvBX&=f5Glv6KAT<vgnQ|qR5STi3YG82+*umG_QqfL?!SwEg-vn&
zi`B(|ddfZ^me+Wubkgun{glgT_f00o_^IiMP)+ce6In~|?pe~Wd<b9bLNJ$>XA$;|
zr=GBSX<3$4;WKA&#lH9_xHbc!i_w=#u2F`%jkco9KO1oAuf~XJ7=wz9aMdx4unW4X
zT?4VeNWtp}t7r4|HV$c2M9zfR!*vXEV7fw@P=)A72&4a=?o16neflYrgE1LBz6aBa
z+BTBR3i>q&tZ6s^`p{dk9PH>cdTncc;Vq!iuLVr2ejPVcxF|5l)6;xm;qF@tgvk_k
z;Dn>~DhN*Sw9LVyA@4}Z#qri{z(5&TjU6>8*%RBBF0sUmH9&kqJU~6q1?9mFEOfE8
zR24i8fAb<cn=F|qf)=+mzETT5h7|#pJwO!sd5y(ND|r$YFy#ogHMj)VZb9e3hNef=
zVVDJ4gjBQzHwI(Fb=f)4O754n5wM!lbtUv<yAOQ?(tQU27bF|P;!(RyJ_L*~zSAV>
zD+F33WVU;FfK8#DI+!}!{|aOr%<cCrv2x#0?Y<n{HV?j`SCqGvrPq~jb1niZp9r5a
zUm?mx?B7iFBF$p~hxH9BJKkJW*I#|xzsJ1)dqTkfdr0;dX@1liY+@Ebl?=_?T2bah
z02u!1D>}_Vj2{oSSBlm{Un_{DDSi<^=Od#`0YtUe)q%uyO<|z)dBD$s2&vEqT#@<a
zR;YRT$PGX43G&%^2Kb&P_&H3acTrD@f91#^4)S8KZdG7-16Z#1e}Za!2rjOJ%Cu$}
zH)ewMcFck2hR(|vub<(PWd9z%A?12vTOc{(DIjfX;5q@SneG?7-K_zEb6a+P<0CE#
zxiAw%Ekr`r&xsSAm7zmjz>C^h*;2{c66*>pRgSZlgqFR-5)3eYUZ0Hab^-<?d_?jo
zq$g_4`q#8=J~!g#)iYpOXo)ZpZMdg^s{Dhqt}ojFF+SfC0U5W*C>c-WL`=a!qv{ge
zpTnB){ZzhzphS92>*^!-7$WCC0IhmzrK;W$gonV%y<K8`uw0Lbm19B*i=M=g!$7RI
ztDAmC!ViO)N23wl256wmfR}TKgfqHL2YjUviIr4}E~mG8j<9Z}TqN_owLHMg{lXhx
z1jD-9Fk}pf40Tb$0}=i&NJ3VADGNz|t_GS&j?6y{axvbUSB+0j>UhXT!xgML*Lh$x
z;(r#%fBhFO`d?o<+O%1sIwsNxOU^6DsZrW=`_vK|th2PV?}-_{-fQ9oa)|V>?|X8{
zOGz2?P7KX^UHN$ImpD-a^6lS@X3>^aG3Huj((Ju5OZq)D-H&c6WzUR^IKGioyv_R5
z1io*hf07!Ig}(d9p>%InsXp;%<jRBVjfE(Jjdqbg2&~;iC>0MPLxl&GPUs<<4mZGu
zea||9F*C$NM1X%RYJIx9qeQ#W3UlQ}M8V7lGoc=GBKLq#Z12XRD1%6FDVSjn!>J07
zGe<Cb$poLjbDABgA*<9b!^R@xjdp#%#}D#0Lvr~aJu(#@2Vt@?AAslowvL-ldNduF
zgJoc_H;1U@idMWs6`{)9o2v_dvN^+0)t5x6E)D!hl&!=Rl#~iV>b6D<KD{;J)d_%G
zFVij`gh#+5An7Pb3~?`j8+{vee)*u>yn+Gwo%x2c_*=lQRe<W%R~+=q4A<%EY){#(
z?dZ9A%ede`jpZ=t4(V>^P<Wt><tdo=`Q!6dtocnaU|VfN3_#i?ApeD8m3#?kzf(KL
zf#4#f(ZRtXHR1@ge?uo^$sL4`WvgL9FvZMzd18UEL154<R-fba8HilWQi|KB9$+Y#
z3U$wwAna;?ey4Hc5hp}g#+euvU{G!FO-5j<oS4ImRZ&jFXWt7GVStasN3yz$#NjPS
zI~<EZBUlSW=t}h0vk<M)C_n&bcoa&%#kKFUxGLA}+Y)r+Z>3?fUlnA`w|~@6`3~Lt
z!3)TNT8&6A>F+8PO@Q<-l>?O;)vE3P3(R!@p8e1C1=?LkWAQ#$1ZP}!`b6XPbHiBP
zb>_GZq{aqPnP6ukM=<vv&Zc*0bo05c92X2F$=m%nrx%8P@8r(4_Sn@dN3Hu=@kD2P
z%E9K1I#Kb+&2d#_xNFp+cl?XMOk!TK2|VxBtjaI{#Iwrek53Qs?uXNa;*}=P*X9%|
zszW&=`wlGrbdMirDc-TF{l1b|EEB?bh>`9C%0ZX1uzfg_Babef*Lf~kOG^tjb({F^
z4~A_ZShCVyfRTL%@e<CO_&Tbx7tn?eX{A56U1Fl+0*s^^rDckwb|o~Q@2v&I+KQNP
zb*mucq8L2wT~m}!`ILYVU0d5yihhyrv>Nr+ij`U==%LUQ(77RGp*2To@p@>{>O^XI
zgTA!HpJ)h~6mgBL%oYsPK$DQ$4xIB-$s=}Adk7nsRWJP|g&w1x%eOpPr`w>KBNq=c
z!Pf?`EpKlr)LtK?*dREd%nkzLX&YR5quwo*a7-AX(G||XsLczQsZ?|~F9AJE4)~5e
z0#lnke&0BARR69%{9nH@{VqxW4N_Ln;aTIKf|(O2aIfr7ccw-2DD~~4?sr41asIlS
z2F=_AeRy%1jA-snH?uyBQ=o<LU1Aue@fVKg{+ROkIcB&}KO<jkn5(vnT(B_}13e%1
z|M7!ns#*65oDPP#u&NabG@qxeXmT@F_hf8}zPE=a4qo*L18%;|=gVBh%p7113jU!K
zNl+oAePmaf?sY>G3CGxepifa?`lUX=ZEvDq+T_nSkE2iej=BIBR*Xa}6A(nQ|9tab
z{O+3~<(I>EK`@8#G|cOG1Yd^-z%hE0C@@fA`AT{vBOg=BQHA0{WUCePYuvTNso-p&
zKs-7jd(X7E45*@gLc$U#NbN+xgp;xBy*4z5*!2(ToA~c59<e`XCF1|%g71OAe?A?~
z5y`Cic<UebiA}5+mSPP<_)4m(37S>SMzE@n|991|cQesD!-In_l*$nGs-PNWlFc5<
zTticg9|#&=%7jC0|LBd5pFlOHAWK&FwS_cJ+}~WkEo08g9lq7a0KVPOE>duK%}#AB
zKBnyF8~Z&`@dgNA3m~SfYVMBSYj5|Dj=l^pdIE!pDE|ZzZy}A%)Etja)GY2K;)=|P
zOU7wUlMH)$+%*2@9Rq_xWH(+2KY(|Y{3raa5Q(fZkjL&{0-j(g7(3~S^B;)*`qA)n
z&MIlU6Z8*7;0oUDZ|jCYl_8vU44Zj-Kod5mt^TOQ=O;jEC4jeA&?=xH>HqtAljqU6
zlKrlnZz2C?OkLy!Q~wuihYCu3`wKgA<kJJr2Ldb6R8T~4-l>jo{~++J9%)Xyre=u(
z6P<Q5;*?UgRcmGwOjn&4JH{ltiJ7N{blFcomQL1nZ^Fg(t?2OCr>GEY5T<bbCFVnl
z>NPNEXGz^!AB}xp?LV4`xifwZDd`O{hE0*5zyB|4{?O8-yAXqv^m~Q8uVygkkNU})
z$2Ly{!9T}0N#coF;W_cWqT_q(XoeTk0u&{D%9GNiab-v;<eMu6d0>hd)zIOhz=VX~
zDKv#uCl^Yo7?uEf)5_Ov3G4`91X2L9{Y-0rg5?ul$0;WN%^V+l7~9~hq=f$61XN+&
z>V-7E@6p#_l0P$fgiY(tPTkRy4fT5qvjw*qN-wzY)|LX)4nG7o{+j<EMr`|n*=@|>
zemW;({QV)=bS(t!-%SoDUk=W1_<BefDiw*YnZc~f1Q^gKH7gyVh`B7A#DM8n%4Ams
zzY2wgtM$QBJD8i^r~O8Rpb^^l6e=DZB<y>b-}Yj1l3+JHTi>8ZTS#N&xf0uk$oauC
z$0*Rt5gko&xGD!wg=`Yn)E3Id4eer#66*ye?X~!QkGy9O2BBh2WF>~uyqiTrUY}mS
z>v`*8A_9A!VSiO9--(CpJzNMj%xf>eRG1$hOobUWr<K!~eC*8y`q$-ozK4DsxvCg<
zHqBGB>pr<lux%*Px(CrVcjb@_zRUVcr&fo&af&QkVH;kY92k5uPr!THyH&LVyQRke
z`<7}WnPq3vjb(qW8#hyJEDs#U@~UclH9xj=rP3>FwF8E(yDz~@;`9A_E-*F@pmpyL
zxt?<J?MzNsY~;?lGV|fJZuM2hjqza5y`m@Qoc`MjV4x@4SNQ(oiwFrA;OeHHVLGIh
zd~@U&rqz`4Zd?|QG6_6)Mc|C~!Myh(qzyE_Abgf2gTp-c5Z~=BfL;-<A1-Y@IVpJ%
zHZc)Yst5hbS^?tt31lut-p)BMbsQ#&6}!r(^9lakYia9f>k_|f13uipzi$ON!Il^c
z8h7yMCkWI<B{R)~gudULJKHk2EWQb~{rplAL_0G)5DyZqP6eastx!~o{CnmB;~uww
z2+tBT{}8~9Gk{}%KaG-H`XwhCmp**iEKrDN0|mVtOz7gGzhArUu-$l3)(e|fP5|dk
z)ryJ=i_<*-pGpDx&lL9BqN;WCq-IplEe7+VNg(0h0&UyKTb`dlU8(?BtLdsvdE@G2
z%9`&KQhroO*0IlkS}yA?bI<n~s|*|r?8kV-Q*l>5KjtJ2qB~y0m|d^_2R;||m%$?%
zHdWkpaG!xi$@(1tR0BdRME8=6<2^y=P^gw8tO*>cf!H%PjQqv=bkr?y%nAcZwLJd2
zuisr0H`ndpY<z>=>mQjSQx>jdg++ZqATNDZ50v~|4ph?N&Y=EEv!>IypAWv7^<5Wc
zSa?uA4JJSqzau$#=lWF%us?->x=8@z1}j}?5hpG=Fp-fl&_Ucp9u+DoDJdEB3r!Ng
zHK;U-_yy&8G5Bj-9srbh?Ng2_bw0-#+^VtCN#Jw^YqpPry7`C_LrJ23`LFcENH$nG
z_sn5uTo~~=X%=)v${6jTJs0v}>L_{ueyt+FRU)xP(b9x>fPhK|pb>_TTy1~rbFc*e
zom?W~yuLs>N(=P;PX)~aPj!J-C?n{zNW6sT-$SRP2vs#F4852dJw`<^-E@V@&?~R2
z4Y-!(3>Gg7SZ~Q>^NE*D7KGao)N_nGdVNemMpj&>E1`&KcN|^Ur=P^2bU+vOz7c=Q
zo#xLgBK6D%(6I~Pz52fz(KfQ#8kb4A5sVukcdP*gW*X<3u%HyUNBjiXXHp;IULU&N
zZ)GSsjmBF`SO0@D%k+;DGhIJB>#<wcNnT%2^AE*#F09cle)1Sm3>;Tmy`!gDXz~n%
zExo|_jd|*sa_trR&b}%%rD^^pIm%FA?>&i186bgmpWn;m5?nsTso};OA6C%?scDnp
zVC4#4E%g%vNLz1eeB2opV&eV{5Ddrw(z8*q=OPyV)%c6aFY^C>lYZd-s|Q4UALMSb
zLgNmW{?HJHy_hwn*c$Wgv5fM9KuoFQ@`v_aqOZqmgVH=m@Za7VyV3s;F>QIEwnQb!
z6h0vh5ShcTvpweMcW8-l(GXyvQ=FonSj=7WDSC;dR%rFZjy=Q8>8#mzI&qB=r}8lk
zV?nvQgW$WOU;7{Pj?ugH+&@)<Y!vZrAQ();9`Bq)O@imbR1yKBh}mb66IE#aFoNWf
zuiPsVTyF5fF-5Z((ct=+`K=SZB43ipVA7O2qks4?iE3so^)x&lSp3}^1G(0wx=(;R
z%XqP3TG-lP*Yceam5}Ras8G0EVgNULKV0%-_XJ&{9?sL0KA-ICk&ozzN7ld}b;3}5
z?q&4xbFBNe9n9MOl%aBsIy6M3Z~MHMF_CzxJO~r-z>MvLs6?wOwJCM#QsBjIURZf!
zGkgA+mCgNLW+<Q2Ok*s*MW1lOFBy;^o1wNOe)Cg1xh=G_8MUD8krXa>9dku+OepZh
z+W-2j9JqEy^hVC%qlOy0w#cTkW88aa>30`+zV|NmkU0L3sJe{mENsjVgmNk&lX(h@
zcDuL3g5Gt2amWoBkC~Gr<(ELIc9a8tz-E@TFo`8nA&{KtRhuaGA`PmM4un$qvzxUU
zARo;{!3E*8fswKfNXSKG$ojeT{$SG7XW0g_YAQz+^f&mpkYqSA){;uLb;O|0R=yv5
zXZcLAT{In9y;M$66&!Udq<lZKY*!|GS+7icA**Kx@!bq%=sj_Esnb_qhB@rFXIWK?
zZgUc2mY>{S()w8g$l?Fr0~KGs8VmM@!1vz%0#MMQmoF=nYhqnUea3lw`!WGQrj>i}
z0daoYlQcj&H+B$MMO-!>^6K?HXzVhDBio53G;ULDMc<mxynmGx-4&E>Oc+0Y8fH*5
zb{O_}70lxt3EZs0C4gRa9rYTtA_UR=ozCEdMr5#lyxOUHtSAV@6STwaK<x#^!tme#
zXaqQNM1I08DGDMH)}gRG2f19YU?QY#-cZl``^<B7NYrGbTQa^rvFk$LT7RR90qp;n
zGUNn3S-OycW4Q{~iBD3NW7Jr$FYE$2BdWmA%3~1?*P&D`M+x+48GeT&$aZ}iD0b7s
zQU!&v2B8$s$q(9pv40Ib|9brLZVeMDU#uuc`)Q<2%Zy#5btLICEL}`SbcN=(o5Qad
zV@?)&U6>4)r+uz#?g*p`w)G?-82xCi;O!3ed{~%mk%V2uWz|RWe#MuvCp+<h$j&w;
z7#SH8AwRO<_W`+5k_WNBMxDX*KU3#|GwREOUwBSlbEgl)uZLH0hL8(3gCvu~CM4QO
z(wVPE9Q1syUomxKrC6RP47zCm)OIs@nMhdIm)uR0BU>$-{F$knXL}{C3Wa{giwKDE
zTv@xCDd$2u3`FI_2%PvKma1&8Ngh>wD4Kmmw6aVL^+~pYipu9VmgYX{yLD0$>Ql3w
zE<JP4`Ea)26#8{Nxmg-+_Wo1?m<p9PspiOND@zhWylb;khonE}jArpfEiAhumY~y+
z=hDk{{~9io#fZe8nC^RghcQD!`=x}B9a=YOKzQ(89w??!ctM6-s|phtj4&TLcz)vS
zWA3@K7Ls;vsj3x2SEy?902FTdX772DP+ysjDk0q34eJ%WA{l$_T)M*a9X~G`)xEw!
zol+^w%5m#ryNMAtQsETa*BV9h??hT->0{hS2-*{-Ig|?pJ}_5u1X#@GWk%)YCo$yJ
zE3Y^}zxtLkKl!gkfp6>0r8-XWfoQUC2Gwr#BFLtO9r^lrI)24VT+_Ha<z4^$#)B-)
zkDK@Cm&vnqehh>_8YAz%7MZQutBn+h%Q`<df=8$XjbN0&Vzp+zUDTgbj^SpN1g96P
zqNrE^rp($viqyMP8$39zT^F|F@m;3txAIIvO%ARqcBLhnXpG?NU-!R^c+z4Jxc>!x
z{@ovjd!DndFXkqZ@x|uiR$Z~Y67i`qc}=E71hRZe!Nfz^ZoDDOt^({?O~mMyV7*0-
z*=YA!CcyUpdP=88<n*oi9JDoCm)zXk#YU>lN|gSRZ3F_4MQWfj1<?f|XCZ>{srSU}
zSID{oWOWOUX}q*}gMEiN425T;>5&`}yit11lC7#SM=sspu+ivXyT3LjU6^e1^Mpyu
zUKfTxSIrg8@<07i^6RiF;;$yHN?PAou^KOyF9d;@%@X+0VGDR2c2l0xUIRUAPmR?H
z?Tp|EJ^s~*$Xk*qiL)$zm+V*gG^uX^>;JPmNnER*3emBYkI-fD=oj|FWxtwm;8kN=
zVig+S{qpjcE?Z*-5xoySgJ3D{#j)V0x4QmBB@gZ=V!mf)tFdDxL&(r;fFA{D@GE)s
zgR1(endBi7uj4f|X6;he2ao)xl4+b~&FUCfOgnNbWt`3ZFX_KiiBnO<aOHzZ1Mg+R
z;8lC!%HbFD1P&^9GrECSx|KS3Bkn^9Cdz0;1^1B(!X6wDJvn;ciy)|RFu|Gza(3C!
zf?*vInAuLi3|5QW_hHL<t4ifd-^6YXyKzpB_U1Gts~>KtS9b=+ozp&bsJs$zTap55
zp|UhaHLWJ)N;H~&8>IF7MLBp@T^YmK<KShB?`#`|^kBl5i%F_t%QV5An6CMj+4W|_
zc?nXf{aq3SL+STlX*OE3S3ZnJ(<lt3Q62u2#uN;H=TqN&5K$W{kCRiftY?Sk025_s
zGjfI0uX`aq|DY4S;Byi@+pyya)Xl92S5}QSE?ys`VOT2g|LRf5XAj@^mm69A(B=#S
zlM__hsq!m>*K*XI#-K9cH5a?$w+nxS&~IYhh^)tvmN}^0^_<pW*+Ju$dG5_|LnUt`
ziK8*1I1waayxN!ot`i~Ecii9Xugkap&~*ETOZ;gb*RZlp(0}*r<MJK<QhF99Vb5;S
zBOs#_=3if)Oah$w)AAm>iNb8-2RUJC4)eEAG69Eb7KF!K037f~3o5!$39_nZam$W6
zH6b$El&6sBzwJ8$mf=nriyWA1ua<+p&<zr1b%l#&zT684-uX3=vcyr)jfh*J4i)^)
z$;TLnf&iL~A~s`w?E<OD)17aJAkey2S>C34&aM$AT<?2}{Q#*&I;)vIh``3gsEW7~
zZE1-8<Mk7S6j|Y6tC_{~8!He(j5N^&wp6!YYS4s)I>XV&%TokJq^6z>a6Y{WJlPZQ
z8s=NMb+>PRK8`Ams%w5@&{7SWhSaguH-e9{@+=dWHVbZ3^sC{ww?LDet{zVsl$RuP
z!>FEDh@q76q~r|Ea+sydPat#j!2>HCLi^9&OMaV!T=`%L__&P_AET3GSN!QQp{`nm
z%CGYS0lS0+F6e>eS$7jHii(h2;c*s^-2_m&u=Uj~UoV7t(aOc=KkE^EY3wbxi>JT@
zI`l%YllQAcT~Tp-h(96>)0Qq23|~_mXKP;1;>h2D5jDw^i-Q%D>@Ou+DtTf#txxwD
zuBfkQoI$SDPk<OJ>|6L|YiV>Z059qilX+jjwUCU@X={@*0@9%5{m?@-R7D@y-|YCg
zsSAIcx@0&;7>5~8#^6*2tr8C{!d4kXchfDf$<<b=II|RGsl;G}&{U172xX}9?p=1%
zZuy63>E*c<M0n9rk7dcH2J<j&>5~GvFrRd=o()@u&>;-HcnfBU%YIECU3D06Yrq>T
zD{!}b=kPA9S=HS4J2IK5?;9nlrXEkb-l2?ib3EG~zT<bzXTMH&8PTHrrRu_+NZ0a!
zibxG*)v@bBz-M1Hr`bxyZE^Gb9xYkh0dXi;$Vuj8_>y1&BGodiR)M5!P2jfToKn=c
z$?3(GX@SYT#3yv-{jQ_rWjLsRPi)F?gdQdNs%ZCC`fX`R=4#4hD@`{po5alcw2|#E
z<x1XvQMMb)D<WV=XC%<vPC>1qg4<(zZ3K-aym!wJ^5s2C_`b{2bgd+r+w(|J*s{0(
zOV7};>oKg<)Hf%<)7htbQvl&BkD7Ljw?7f1#m$2dvcHI1x_Zl$@tWky=TlvG*3|1G
z3*lq)wdE6<$QQ3X*tJ}uty_GJ)fXEG6s?<`<)|PCG2h6^6ag#Vi}kzf(G~cQg|CkL
zNPh|#qubb;)}Q%+*uP;IZlG7QTrF<Mlm3gN)$OKPv$P%r-F?``8Q_7Aan}&aT}WI&
z1n!hdbq#j->KTynI>w)UiJBBRKL0VVjT6O<20aSHqxu<PSqAp~y9%CFQXtt#CO^yP
z{x*|8bOji71sYuPg#>GwcQzjW2e?}cO)SGNxukt&WCn3W4@1268>`mA%U=YxXeK!n
zft=F;V9+3o9SQtCFDVrU8*6AuS+2iQDn*!3Ha{-*0!#p)bDFUpJ%6Sm$!qVYt9+%&
zlqOh5%w{wO^v{o#VjZ3h&zFM!U}AhBoocOsvQDlz@i{1U@$;0QiP_$H6Yeaw0KwTn
z1u2nJ$bg-chl23W1wy!f^W+-$nuN1O&(ZP2@7)QKnqN_=t)_=`cN&LU%|X)??*II$
zC(Hp&ac!WLCYmDX!h%e|75WtA)mX21Y+XwWKR+$$Kzy@$7nfr7HbXR@SobfaK2?t4
za)a>OWtssDFkkl=V+QquLGdXGWjqSEEYOw3OJ0fP@=-k@!y2XKQ5+yi+9Vzzr20XE
z710Vvm_Cx8UtAJ!o2XWIU_%Uo{>(Q@6UCAY62e(^1XL;sVGwQUo{d7_BtFP(Z5by{
zCA%UfAvDn=do@(t@$70^-#mkk>=aj7RW$?Y9T1{)7wTv>*o)T`0lLO0X%shNEPW^=
zJe9S8F#4P#c~fFan@*!0Zpi{Ej0!G;A%2;vD|`xJnwJG}{=sO@N^cyhj8%2tsgG6J
z{nJhHTsGgn=B7LB8~MD^U(ozxq(ZPZh>P~uA_oJ0ph_b*2JwjObb*`|)kFsu8{VFb
zW2p`)c%J|jD<4Uc5^a!E<+ATCFXn#{DV5mz@%k69fAKEYQ1l!31n}aJ?flP+qXu3a
zdD%BD<IQtvT%i_sJY3?Y>a5t2y`EB}Rnrkt(W|7Pb2Q2F(UamZn%tSRT+D5{zAij*
zvaMp~ghOw8mFx4a2mjJlw0Y45MJEjIVgD|F_MrTksQsb5h5vJjWr+$m!On}OO~fyp
zA+rlcol5-t4+ywb)XismFI1VMT}Sx1yMiDssyot3fKVh|k`1Y<vAdc)hEJ;d0gaw^
zAzG;@;-;n`^lbZ!e(@A0Qmn{+b2nZEOlyi{_dz*AE0j<bH&W#lzv{+0(WWD!ouq6j
zd?$+9g~k^TsY4H4JVx(GIC3Enq(FCUua5>{(KmgnJHUOsG0Z|F#SDqEDktdR+%Ocn
zlWzJ3xU@5t6W~N&3<S(qk%kXCT7u51=PUiV2f>Rb4U@|09*4YvDGXE1bb7$xRQa`0
zQ)(0lk0mUA1$3;H_WfVfk<r#+xoYNcG7k&y#;ImL=RW!(&oA-GGvx;cv^(&E_5<kD
z6MrQGf0H5UJSjXU4MBw`8YHb?b@Jyeh>uDdGBcR*_-e+8WVi|z1*YP@p3Iu3YqfBP
zVv-C6U3`D$IKIOCXdX62me?7ib@7fbmZ$Q6huTBtXWNr!m)rE=opUw;w_`ZTvJ+Zi
z@fj)0y&H3gWdieInz|V14<iZ@KKtB&cQYfKEQYx94!LEpJS@WF8_0Z|UJc)q_-+f6
zhqtPb-dvtstR^Yr>%0*qr|?6{7v=X@`7~9msBK8vhbvqCP^Ub(=4IqA@ep5?9$NjY
z&iII)BPl=IXi2}m<oLr|cr7=5Np|f6N~)idk$uh|ua7bKTo(;+yrtr+So@yuty~SU
z^{e@1j$;WdQpG_1C7(YiUT*O$!<>JA?&iOiQg9qNkR0Jd3N{qovCfGOmVG&xftP!H
zxKs&I9ZH05gD@30lz7LisP_O7SHEB);x$=qw{CT)BVFHYkreeIsU&{)dy*iZH(Us2
ze;d7+ov~>SF&lHeE0tNLozSVspv!$7GvR;dz5gXpuX3D;X87&tw&iVyI2I)rf-O>C
zphg6l>E7p~;|3q~@rx+hs!G_v=<R<GHlll)iItidv;u9KvK6R5t#czRQFCrxhD~kn
zUnioQoSBkgwd}ntYCM86;jcQ2o(g2zW7=BJgR;^=jbJ7cB}{5f&d*jl_~Z9`)Ylkp
zBJ>+Cf*D#eB3^VK&%&-x8M!Z6v-0n$T0}y~*8{~rdxVhpX<P#kM&}~wz0PpdZBS^Q
zEaD5Q0ga})I!@U!;BmM59Z{=Fq#b1EK~+(4nyF5Sa(8@xx$!EJ;3-BI`r5@X)}}yt
zkAwqaC0PnLuJW5!5omlJ{(EjFh6_BW5zt)M%T-GTgI3~o>d+pTvfE4C!Kd0EzR*C}
zdZ^d(mb<zd+XjBvCY?^```P|GjFM!`sx^2E_pv-7&(hH-4sAgYk)E;PG!8uK5GRDu
zIPsly`~|-jCJ4$q*GYbR@Ak3ih5oxK7M~)~!WyopDPTaTPy1@C#6d)(ta)?)*&1IQ
z?xGI~&Mp2LD&yI-K1R8VvEZtc{>i9va4Abo8;!MI@~7L26ZJ|6GShMbTrYh*Dium6
z_rxf=lmcm&?^>UP)HYKz-vuP3o^t0dS%=46`}wOzFn}hARYygxkS&*tzEaB-R&;-Y
zdZ~Jf#f)ySel%5~of!L~CtIIkd$SjaLL<Wud6jYfbnfc38|}wmJ{0O`p;Y@kfi!QG
zsA1LuOq?56af2Hmax?`U-n(yKdkxZOMhp;<ye4o8uqG%+vo@}iGkcdMMZZewzY@cU
zz%ncsty5h=9nE)4mkL(3NhQ5Q{iUcS-)L<HcXYrb;lG-H5nI>pkj8Mu7b_-l)EyJb
zG<{-L@OHyWFMw()TiC~2joec!YLa&vxpS)Ay&mL0<?op2*@RU`*U8TVMUGeDbM$WF
z9b%1At+TH_%i}OspE{Z)+G(M&A|P1k;r@x}hx1I3m?Wroq-FDdzR=WH9X>^$**1nC
zN%<rQs09wL`blaBm@3(mX)=cTFv1KF>qF)b)t-B4;SKxTe7xKs3=7LM{EB&?C7(oI
z5*CTR(_36ztijnt_9$hplP|y{Bkpwf4#aHJ;dWMIvMk;n^{e5n?>5h|8n&iW5FT)B
z`0!R;Z(15COB%e^MBI@%d+v!YbpVF$KM0I^-nuR4GV4{_<E0V(1efU+lIx;0$w9$Y
z^lrnG5<;a>g|uSg>VWujkJD$=RY>R0fF3DxR?DSMxSou*jv&>5w$|tM2cSgL#g8Up
z*C<QxAv4eRSqju|)Bn2H>Je)|%M)AA?>5h>I6{&(2iD~NYClP9uZU{b*M>9WDyIi&
zV|bg8z(_QWj@cu~vfh3KI#TSi%i~2e+-g3tJtk>8-m}|uZ*-4YmtUa$T{DT0u5I@5
zqN0Cr*I@M1L9m*X(fVKWPjF3a=rr~1iZvJ|%tk-)iw}7RSrmE$cN$O~QY59FWUZC7
zA;0x)spk$0FUDgAFIrP6;*wj%qYezqLZp$}z^)wkfS|LH30(utZ~5W92WxWwQmF_*
zLV-HDJ;-NB^3YTNH)I{)NAUj0`1QAO2i2FH1WWkr!VYWAA}^k_D*kTCbAZ<pjp{DS
z^2)Oeh|K4Kcc1zr|DPIBSHx-=1~DcxG`5@kt_unpG4KVpvF=lBn)Ssmh<SX(qD=W>
z&R@Czcm^_P&|;CzNTD(Z#3OF*(eJ;7X_f1+Ea=}^C1IV@&U`x$uBV$>D+I5;vM#!n
zF>i0Shz%|Tqhs1RHGoGV+0FWajC<?eVoQUS78V2z&<B|^WafChE>~Wy-4X`ev!pxC
zz<zbbgc5)7`WFtR3G@@dY~w(uD@D|OL_rI;jwcx0<!<YK1I!tEzP&^*K{w&xgo-zC
zV|9U`jc0l2iQateY!Hji^cTrX&Khq+TSE_qQikN|rvshUoCRH{%JDjdbmuE|St=yM
zKcQh4&xVz+zQ22Rx<fymrX$Q~Y+BdnlO}6aS8COth}48?^BYGSglPMKso2zOH3ihv
zd;;wVd??YJ9X63uGCw`NEOTtRB#O;cm2oEcXGd+78^6d-nkk%bu+;>cEf1OyXYr?)
z(>ssmu(q|=e>ogkwrUAFJLwB4B1n42@Kv(5sT)*CR<w4{D_JH`Ju5#6j+5>sn`*Ki
z#BW_ePyW(K45XZ&ralJWh1iB%8@)E-Gxsee*`zE1bSzSF1A6ntN?hPm`#P$Xz>{GC
zJw;jy6ryx688xfj1S+4@9@`SJ0!Hp1&U0@hMb#y%$K~oC`<i;s+eoe3l;rL+hI)E>
zsD3v^m65}Zx>4<>44y^n`sQA+6JFQJg6M&~+EZSZTZkV<?#$0LAfbq&_kSpR%c!oi
z_U&8gkOpb^NvE`QcXuO5cZYN%-GYLobW3-Ggmg+bNVhcHo4KyJV?FD6@qdMD&8(SO
zI(vW5eVoVfIly9o$M?R$_aW%I-ih)y!L-RJ+T!FI_va*tYFCl<l4MZ2S9J1Xy?XG}
zi~}`5%!yX$XkDYvt|CR11hID<9O{_&r4wYW<3hQg=pcLk{B+C0f$zU1(VNrGSQOQ^
zL+C_3(2%^J0|+d8X)Bq09;|C~sis2nSJ+>g=UE;D^?dS(6suHt=elcs!QN+omgwwH
z?X??r3M}2nXF*XVug9w)Y7%pm=@nx)n{l$cTd?)B5CX(4t5Fhz2W?YRBqRBNxCoBE
z*n)|T*w{ER@eS8faC5_&$yQ3#h_*fzK91rX<31y!_-CtihP|SMxc<!$aruPp@a-i1
zEgEMB_T}CuE7$X|M(CAmf>D%uiv_<x_G&}zCW%~(1xD*h+QTSu5mY0XE}<;b(3Xp-
zg%IK$?;Dm=!joaun8{{o3ouzY{lBOWF)O{uIxCct>7G5LbpSvzM_8Po7v#Utn=#_a
zcH(zbr1KGEGQZCXo8lrw>bB_Jz%5NpjL=BeLnDtHLvTMN+B1oh7VC(DA#6@UBE==0
z0C(L#+c@w8@(#iM5@WplQoo&dZo2nB2B|b_gx?PtF8{G=it%td=>LR|Y!UJ3+!;Ok
zeTp|?rIa*~PN-^^x`9BHT}C8To7DR>mL1TA6h@`Q-r@2-zrSK{7Hfc*QNi6_W(%4N
z)@*k5_tPLfR*_-1MBDfKeRvO8)Le<3Nnj;Xy>Z$y<;?EzpRSAkn+U-R7<y6dRQKte
zDa}8rGajd8u^=z6C@W5vhTme7Y?$rjV6NJ!)`vbMD=y>)lY*N17f#DD25Oab<9Xp^
zp*g4+Erhu6i92w)O=^J07$Wk^!XX!s_TQW?AMQ2UaxbKJw`}g=crQ#NR>aeUkT46j
zo2v~-G;$9o4`@x~7M`9wV7V8#DrRf0q=om+rcmm8G(Ol5=N1G9ajE4*4Z?uxKpYVG
zc6Aen?b$_LZ^PrQ4e}o{gt3r8;}=sXW3saGzYTRn9gD6mBb%`p9>@j7uTgeQyHt{8
zvY0Uk+jlhpDgYA_t%XOlPU^9VaTpopk1&c{IKriVeENM@rE+`FY!6$S6-Fv(?KdFg
z{Q0}k99}wq^V6g58$4<ODtIuQnIZM2>i#j#zS!u=MM`o|)BJRo4GOjPlvV;QW<gKV
zietV~h%Uuc!TjE7colR=^D``?B`D1N%jlTF1$bW-NzlpSFAJ92xKT3VL{OVSOWR%2
zWvfstdQpLCG3_}we7oZOgk?B?{yVMDJ*x5)R8IE4<`zl{-FeZ^S8F}{cS-Z{vO=p;
zq^;Z1XVNLaA-7);a-Q1fA<bN@E&CTOHx*<KOZF&=;Es7)J&{;HRXl<J!bI0q;+{by
zY#PY4q>#`HTKFCkdcCr<Kp2`kDsr~ohnaNXpfF7OtTehgVBgOkA!Q|ppn`Iw*kVpF
zfc}$xXI@~$^mrLEV`2Q&^HKNxPhhqOIsifQCkr4-0s*bL)cf&EA;N4w11FLzHx2VP
zrr5`lTlIye5b4s+J0PUfEMvcx(bvxBK)9mxRbA!p+|z**wpn?6xJXx8KX|sbO<)b4
zM}TSsN~t)d&R?vD563l1ozG&09>VJ8K!4eMraIH{mfF8B^?1jTL@~8Isws;E%J6)*
z916Pk0xZ)OtP^aOeP-Db;C&$!ZT3X6zvD%g$!6!}{pN+F9G3|X3&S<5qq|8z2XJ><
zef!pEmK6FQq*f17fD~x%)+jTqP1SlY401c@1TR~<`LAt+Sypg)7*+fsDBoQaDy5>U
zPXo~u=*@<oPxJr%gm{}6@If^v`5GOkCyI1Bh+^%-zlR;5)X<^yeZ*?4GiHw!+*}=r
z{&`{S5d2>g!uSY^+KWfU(ufM}e8|tCD^(<f8B(r*R`5mfyzpJ~hm+LgDoGh`sA|}r
zNR|d3l<)UlEE6l)3O$1`Mg)qog}6y69?4Cv&HOeyQ_nTYIH@E6kC1>&L4<6I)8;9}
z9C-U8ZEINJs~}*?)>@3~)zIQH8dL(M+pe@A;0!fQILR28rIw|^|1Q%DjNve%p}W3O
zAYIegL+Fg4fMQyKLBBFv#Hc3cjWz2abufdXusp|k@p3cM?GPru6Y2>}fwBdH^jnDs
zvl*?J&rketL7-qxj|QlRD^Pz*Vmkp|n_QJBb?tc6#65|Jkf!)!y&M7b>Q%kn5W`w`
z8vj-Lu%@RhmjIl-bb>+uDt}fQO`#&n&jD(hwR^o}DQ-{GxK`*5b@UpgH0hhum!Wsi
zSEZ_#3{%LVw}2Bc7|ZJ*efXs0w;N1SEWg&%ZHa44(l5r$wXIR8B#emze9}_wZ|~GE
zlz0Q45aEv}L=cCpY}jW}YQr@_Auj1tR@cU7Eme@rp07Ln2tGWd4S{%g$orE8vH59+
zc)wP#kxUUdV=LpWGE=mizu-M}Ym}zEOK$=)=8O(Krm5Rqc2QwiqbaGG`#Zom5pvxF
zxuBWRr-DpxI`y?7Td46D6wKNo6&dg(?*3HrsU`{pSA9yqUV6ykOW^@EqnYPb6KuA9
zt`d~$#tO)?BU<rkP=#}pXb$YOA15cnyo-8i7IaQ3@Yee6Y1>P^;B@|P!o<N&I8yLy
zc*|u!qlqQcn_cRL9zo_^D&%zw?$={Rnyl$l+N_sM8x2b}ma5RZl&UeA?vl!%DiMqm
zS|Gi5qfser6w>_>VOeLv&Andzo`u$yFV->MtkeQI#Z1=(e06il27n^Rcw1g2+f+AH
z?>4OC3D|vz#+{9J(7{&g!H7%<c=s!Q9C|SB>zQ4o$4!v$%^ZclYOI!Irn1VP<amx-
z&M5*UUNg}Z>DH4xob9vHy&wx`N^y%eod&rF<x)#vIl1>CJGLxSnL|;3T5LSGcwWhU
z)aC8=F)p|^qncTr3pAJGN=`^18(@2`j!K6&rcJ5YA6;VKTzivI{E@(V^&u_xzot71
z%=KQG?eD25!O!+oq3C>x2MU(ZY&8AwDI6ZBKMW6bbdgfkmf}3=JmKT<(HLPQP}1u-
zp6iR>)%MRQ`jQkpjwo6MJ9Pt#p2fWF|3}P&By8x^s99^3QE6zR0UA%R#Og=@Y~hZv
zi0y2)Ep|Hx$$|UliS!EkKR-5qQ$}ub5jN|aPDdFu-tRLbqb(c5h>)$;Iu|AiQ<>zf
z^9d&vs|VG^g>wjJuOa+<RY2DNGZ$q4|IY=yoLXzdB^B1Ft`x))Yh%8D#@;NHGrXo_
z94>DW$sH%U(D&V?44lkJ0!h8O`uXb2<{~-IlU3pD2-b*ak58a8JvIukYf)(uFaZRh
zRkB>Oai2o5Mj4%{iHVlFF+Lm)e5n+Q<h)U@QpV8{^`xkw@xkxPgc`^I-0Qp#)6O**
zp9Xqwx=vkY4(5#V?iUkERsVJ&=b=I7Ly}H}N&JPwp-qUVt+}Q_u3^~a_QDeX_#)Of
zMhuR-kv8qx8j*Smn9YVO5V)AWw=<Wj9J>MQT$%2#={@kH<V)o3=Y90Fdw5hLDTnoF
z+!DHN!#ac(b;4RVd|i~GDecjDWQ%sc*HGbqfMkXu^*Z`R5gd8H9<y<^LnbN#JI%uN
zY@u?{JKOoqSsX6AAR;X3L<;t7guD?PY`HA6yjLdqzhK5CQc`mByMk;TeT;CtL*_-%
z__0}xP_PBt9&hQDzr1@a!<NB-OmzqaLnlMLIvu8KZ6}j~T2B#s_;^h}6Q@+_s8AXp
z`j%*4{VDsmzxtK2DTDynGvsN$Z{g6am0Dk|jEdmEh@n!8l^)bRNM`??D)nG%^+i|g
zqMDJ*RCo`YqP!ojL><{+gd&ZZuWB?5ESd-~+^&y77=Bp_(Bd?LSkS|zbJe-k+(*B}
zShy<N8T)@&$#DZ?r6I?(Qn3_Iz`(3B-%2L77$N;+7)OceXM)&T+x&*nr{rTe@O8%R
z=@3LTrdKmPLCFFJu{OT>bNE@W=f=Wf1OhnNNT65APDg`<$?hn$`qNssWG<V4+E!qz
zX3I$z@>yfqQ9D`)>s*V+IYO=)y4C5jf1qNzfH-8QGamL6D=GT10+KGxcEIN9_5-~|
z9|M_CgviaH@6*G_(>6^;f0|rCkfEVke_!r-DJ%Bbs&>!yZR*=20~(8>$CUF-`o0MC
ze5sz|6-d0$sT6+su0t3HUAtrC$Ggb4G8kV{{!+M0-|N5NHqhRyRMV+;LkWO^!;vV&
zn8u8p4e0S*u&tj<&{f0#9Vi9eB^bsrShal13d`b4$<Lr>Bb0pAB~<G1w{hp9gpn77
zgGes$DXi{MRW_m{x=+zW0#L8=l#^t70ZQ(N=zkp{2lE8KxK1a5<N;D6fkHW9>^Mv#
zSih)3-gEZnH`gIzMmt!jMzH(YxeA10y7i{TV>tK=g{w5~Mps0rq#%t<X9##*VYgoY
zK^#QT>+2Ja4v;?hS=}*3^dIGYM5tq!lifHrCr_u_ibG<`BNHPG4JPaNoB<80Thl_y
zIz*?{{z`k#`B0h1h1tBwkjKOC)yK5x(CX+pwITlhj{res+?(3K$GG^v^AkcqUzHT&
z<pT0URY~N=KaUKqS@a)w{JUmVbRhL5#>?qY9-Lz+T}pFkeGd%Q<Vm`Lmw9@PP69YE
zYZN=bTmaZ~5_k?7t2;cFBce`0538)?AmA{;Zi<o=?q=!be8f-dGe=g5#DZ3a8Oa95
zH}nDZ<C-)F)Lx_~$gtM5Jp*YmHb@3x_oNw-X~{7~BrPbX=@7rqOJ_Aj0-h{c5*gN!
z#2oKwGXPSKEX>iW6!_d-LHDcSJ#E^{eNH9R#uRq18ThHZ60Pa^4-Rzcoc84i<G0~%
zS^Rb=CU4LQGG2qFt*%seR51{yo}|~Jc#{F%?l84sqmEHwB1kJIOwJb<iFN;Q=40UN
z-AppoII7n_qqKq~-s&|l%W8LwanW!--2P&{J;_0%yn!D{G7AaJO1}^ns(i6UAC!Rf
z^JV>3;k`mVq#i$gHv5wUuut#L;b5RpqoOusHbRoJ;sBCQZP6Iaz^|o^P}Bs0*|j9r
zuVKY;l;#_F7hmqt{)F;aF+ti&=JXy!qYJszAfIU=yNfM_s0Kvg4l=(3IRp)wDYc2j
zz<y{bB?epIj|~(Hay}pwdKKMSi){YD)PAqR<02^nD@r%R-mFqVDi(EsZr$XH74y_~
z`jk?TnT??s#e6|?*W3X!O26eF<IEIZ@g@Q;eE({t9+6e@4ZkUXLF_^cfB(UX^2N;a
zGl3B^9K-u2F+Q)I{?lH$j!Cwj0flFt!?`YbrA4cM7Z!qL=c`T4YE>QtN?x~vA45hI
zGyS0v$M=4xH@Gh5vAnxL$U*Lh4?IcZcBbI&IDvczi|e;>Q`m93$Q^9Nd#1RWjlUi4
z!UQ62CwrV?@q(Dm0)@=Y8{VBfLM13LLpyDRunB%1|FF>)`-MWm146dr%@=xdv?b;$
zyX|-bKPczEGAT#+1dsIn9Ri-+tDbcLj6$lKOmq=0nz+3#c>xno&<^ciLO3nNX)85V
z15p#@N1dhrnTbP!Y3*R2mz(=%%wNf%SdG>r$_uBRTO|JGa_BWLBprqH@hc=7CH|O|
zHT)e>*pi&E=K|$`rACCp>t1fuGtJTg?9t>aFu<{BMpTkUd;w*Ts$VJ&kdK~R)|B~n
zlcYslttp7PoqE!eV)0+f{>yo*if2J!P@}+AZ<ka39xq0aouyo8vC3=xDs{VQ5UVJq
z#-sKh*4O@*yvfrXyjQ??b}a_UlOXZDjQk#aH&(>>BoHm}S1xage6GFj@F!cr;hM`o
zYXu4dD7&`ol)KlK(*znNEaW47Q9=h8G<poU(Wxq53F_qB7QX_F;BMPrNy{C3?%1j0
z%?B3VjZ)({^3Uk@TXTNDVw5%-v?L%T7$@`GQv(nzfd00^66mqPEa|bsF7a6x^<)MY
z$A^B0=gh}H!X>%~Ov`PU#jHxjO8UXxI@dol$Os3_fv;jyN@>K;a|p)hFnvV}8PVb~
zmi(-M6_eHve_%hst)bVlBq$NFR<&&mzU`G*6Y{hiyU^|-sC9FpBF3Uu3&O4hyX>r+
zAGa4EV-uXCmn`~cvC-M57+bg+apx=|Sbcu2bk|<z!2oFsuT;;@t^87P-){;CfxD}(
zaMBr6GrUqRCKY9Kq4|j59?}oP+HST9_W%+_OAI@jnN6_*b7TiRg;AA=0-#{hej8e-
zA2Ww@RJ}Nbj~P^w#OHCL!Fj8v1-q|XIue_7vBpOv3qFy$Vpvr~tW(;EHkvb+1qHKM
z`@F2W3^>+fFBaJ#?P_IoX~if$iP4i!CKH12Puccu51+P2pHz6L;8ovGREH7E+iqDi
zO1a!=?XjG-mq!=1F7#Pl)b<u%r~Z^Je?jdW_veR98!OG*&YQo$mq6F=b_W1d(r2kW
ziScAJ{@$O^+VW%#AoDpsPKBDstc~++R#`V~upY-E*)}*4E!Z(&+s}*c@wqEu&kvX|
zccVF%mE<D^UI&vF8nfFk>rIi~KL&NZo-)nKh1%IsGI?2$eo7r@f>f|`Ks)1t6~Qq$
zlE%%bE>h!ch35yu4O=Weh#qTf1LB@`y5FNDN{foBcr;2ooiRr7zVbXl;=Za6t93#W
zK6pzPrL7a{tG!JUSvXp0j|H7^yew@)Up8d9dX%0pTLl52if+l*>tpK&J;9&nxsumz
z_Xxh<7i9hPWi<5>&ys3yu&is?-A6X&jCZC`FBxh4KeUVurEKsI_TEhcQy)_mYz8i$
zb0`WSRhU})-5^S9XDy5M3<Dh`W8dmNsnNSPC^+<raxoONM8zLV>J+80Yc96e->)#L
z3^TjPAE|=H20~i0>Og`Bcb|EK9U;2~`V22t#EUf~JYhq$J?d1`Xu*ZLhqpMYLVL!O
z=>0(b>C>m1vFJ%1T(|mUBtOFueKi9@wWj;n{wJ_`Wa2QRbm^N9HwV?DGJ!tK;spn4
z<*oaI^bun}y_m4AqYeX_l-hkCHm@X#7)+zExk&gN;yxixr+o~z(k%Rl++OPKpz!ZT
zvyumKi*6@hM=9r&c0x~$T$-=Icsli9y7}W5r5ub+eDU15HkP0s_Lqd@aY_hN_8D@c
zqYMn#3e~>6A$u>%AE=MG6#!cyRK|+Xo-<?4Y*Xr=Lx_p@l5=HjXep}ZS$_PAv12R&
zsYB3H;SA64Y(`Hi`=i%+KC~<6nQKi~rjz!1;iqJw8})JQ9KJdElucjndNk3w)qi2^
z73resuUJvPy~Z7g3MQx>e8#dnjtkqt{Cf+;1)=}D<b)CY81fut3$P*oaOE{IY@!Z0
z0dq$-uaGPj8(1CycBHA9P(zf-M|d}jUW(~)m)gp03+H>@JiG(f7vWBdug}Ovw=kT^
z=*(ZDTgs*Z5a=B#QwP4b3l(cjMW7VriP8|!t;t!s{OEqvsU)}pGBw4xc+!rUE|Ai*
zRmMYk!f{T2z1Py<IwVe__wEK{t{;%r54~5<8v5%<9zXPAJ`i@LrVfTLsbkQsiy{C@
zp`>uY3mF)*pfh*`mBB9cQstXzV9=T{y1Y_$!nx-2s0nvL(L7|55`{4L$jrUiAwta_
z^mt#ZYbzQ;xDkC9ibk5xs4Mh4Jbf*PQp4FE?C1>0qprzCNFoJ8V~LVclO5DR`){)W
z$wmP1<%kjSzV<MRmhhi1=7dIFgbWwnu#5S7|1cjVkrq-$v<IA%po9RV+GRSU=!j_S
z%6#o=vl{o)*zW|kVt1l(Ln=LZ=Il}6hPeqvLX>pF8&O6UetRPwmF@;!x*co8BYhIk
z3Jpxce8K0B{8!7euxDGE!|~Gb72tW1JW}Vt6G~eJU&p+0X}|a-{>`KRJq!*rJ@o5i
z#dNx~^I<W`r1_@^0oPmX;Z$x}Ojyk_ojlBtT3BuPT#_=l1S_aH)zaN<dfB9}u1`1o
zt76vGDkg`aDtgYcr2*9(M16M`M;jj>*RWRrZ{<b~363aCAJ2f`9d@z9_tT$$b+&Gf
z+x<cY7kDQymSqA)=oHzzglKe6wxC*{lqfrYBN}BS+5Y~~SwNA~+vN!M0S5Ilblm`*
zzv=>F4${3}vr@b<!0%A9+J@5ax7t;~%)VOqYMei4X0tkjXzZPq{VCifyYG+(5&DGs
z50{$jDM*d({U=5Mt8^E1PRIFl^2b{5R`!uhMoJQWW{cXWA=&LjMM(dn8BL&MIic`o
zii?2LN^Y+8YMif-UH~{orXFH=Y3l9M*xnHav!jO{W_<a1!R^r=18EJsFSfa<+L|hK
zBzOH<cIcL^<*Y0dVSukh({;|TC+d|gHMd?|Li44`-8<wt0p&%2<e?FvvNmHm61u`N
z8nG`seaulQF|FC+?&x8#o=#Pvl))M^^XIn8z%S`|?Ygz&Cj!jBFO0z2$zLblH_qdf
zuIC_@G#{N<c<m8B@I4@C9;7_QQZL(cC$s&&if?RPcA55~2GGw_O|l)o&d)VD<Sa@|
z&IN%cz#7Wsw$H$9H%#9|kOKiv*!*que=Z9rxaZM9`;tqgn4?s~IOJU@3sj}>i%IjR
zS$e@sLT&$qa~?lIS;b$ueaA9+c?LUjDkvkL60iCWVyY{-_jSs+77W{T`l{yiILXm@
z|GVkzDX$-~t^-u|#pgWjnfw1m$o<D$R0D*}<%=}gXP}@2#2oT}?u+n<>+NCPKMAtM
zmLt(daAoEd888J()t$<YfX6aHOXvtm1T~M^pHe1^jeCx<dr}#$j1o-3fCf-RAzTlJ
zWGb$3dT3!-)Itq7_&K1=P2+}G3fZ2096jP^zdv1LSLgcJ?ZK`B2ZbIclxj)$Zub4h
zbhX77c3L>33QZ1Fy^`dmeD;#dLi!sKdltNDh--K@?{bVloA}=v0l#f;h1hK07hOts
zf3?I7y4APDy;f^}y~c+DnME2+hCZanpI$Utwn|AaDy`;gER&_ND<ziRKpKU#DH7ne
za>jF>sEfzS(?2NGIyi79z=#wn;w>B~f^jk?|G^c9+PLZ9NbxIk0Ng9QEL>=j1rQ7@
zBrPFs<rm1JmU9>8azxV@PuAuBB#&wQ^@-6;s$?YfO@9qC4oV!FunH?7w(7F)oh`l%
z17bIsK-vJaad6{{rH3U!GHgSbq6$#z9;kzWAjoXYp>b*4BI?Ok-%VVfXEdA?*bh0q
z5UI>BvG5p6B)`jhV7X9O8P0wCzG(vNZBTaaKT_`Im@&2@@u_Ap7_*~K2&t9mxAH78
zV!O+HW-yFO9wY-i)p~D*Yf5Z*h&ll3NJlY|_f|`zV`Jpk&!!Rdr36xVsw9XX+!`ja
zsTe}M*VA)t=Bhk#lnicwOX<?MHfZOoO0g&)1`Ego_xc-j+MpD{N(pHA2qVA^LrNA$
zljIe+Jt1GY8NwcYXiYfCre;3)ntq{<Jtcq+uVPDq0yB@H$d{ezhyDp5FC>ZvgCEx)
zVZ(8b4LS&ox#nM&Q&Lh6Tz#`HZ<OE>D?Z&hUF)XAr&@F>I}7mV0>&o5PN*d*d+Qv9
z0cf&g-J{dQX$#YV-sMM6QZzG#0gUhCn5~gmrva9>>l2}e@LLYTBd-?w24xr4FKZWg
zSCnD8gJj~R%3B-t*n8dQ8W=|EL<nL5#qa2n!G2)iW6=jL|Eb-n=Z*=T%x^AvgWJh6
z>@xAsDCcsb%cUut-I}K(L0Q6NWDbH%vf*ToU5XigLt_{x1}M#sLjYXEw0WEHzM4Jy
z@oLp^zWEpb_>djnuSy~3K5%|k(<GlI0{S0h|40NH7S4F&=sqxm8bk823JhYG<x+0y
z+)^BhSNz~lm4AcI;5((;+2hwG$9KtJS0mxKW&e+xNx~@3a=MCgZ9{>5C`Kt~&zzbu
z1m=HxYXh!DAb{Km=KXbKVLOz6Mu1B3j-#vGZv(}kgvlsU?Yg$RXd47zXOORfLmiR6
z^II91<yF$fc_<?USG=zn-;?(<xxtw;M{DKCkC-B{Du6snP&th~Ly7mz8QernsX9-r
zcA1E+hL_MDHE@7~brf&$FRh9NAFdhWOB++L607b(vw{G1A<6dI?{7JDM|ko6@?rDE
z@<Lz;SvzT<f0qAm974Cs$kwZvLo8(QYdWb)e<Mu>|J4F0D6-ZdhKakkwz7|2%bW64
zb0!IM=_ZK>tupzoo@U#voc6~a1xO{DcpS{WYw|n+IDr3xhhF2Yl|nL7Iu8gfee~X$
z6=)B20X+ou*w*)~YeB%#K&SqBn^A`bUJh-v;Vrt6JVZ{J`;#*>QM$0)pDsEHe9K*!
zdzM||p%Vc#BgtK%Ao6ObnX4*%qiij<bWOG6`P!13djq$*t%yIbwcZu-`haiW_U@8W
zo#AEp!@&#E5jnC*&JQ)%)?>-Fo9|Fi`#YW}+DHR7;j!1OG!pOy?unP#j?mQvoSdCo
z^N`yRZMyG8Jr+;|ivG6T5Ec=>p_Wh0l@iFt&du<>wQK_GcrNC?37gkXayBs{JEVw}
z64bBEE#kNpxNop|iu;sh-pd&v*P-N1;4s*%a?>Ca`&_VW@6wky1P+#Kv8BFkbN81)
zmMS&w+D@%~3if__Rm1SrBV0u32FWxag5z<G!SWRL;a>Y#uq)K__+j!)Ffj35m>1+U
zdj*~az6ah91nCH15T5SI1PQ=f3&hM~YhLfAyj0!?pXROL=$)r(L@b0&T;6jG>!{nH
z#mG}x3KJ&uMPP$6r!wf0Hpwl3RmTied4+9U`iUo?0hh5CFI{u>8qH)I02_+Dv;PD-
znVJfV_qose!#J8sMB~dtH>L-PgsFNRFJ1i^EGfT;eCKwc)wG!)OV%~RWE2el$3O@#
z)F{^ODT@qFy=2f1GIQVCLtv{U7?7Gr2nVHMXIDP=1lspLy*0(jKKp%U`m5+t*mmeI
zt{6ehv8lQXovgNzeCfDpaKu*Fm84QL>f!LM{lZ{Rs;8?f`WzBe9=d}yYEKz=iVCcY
zJ6ZZe3+i5+9i}CZn|uMBJYag~H}s3rzpf&P{vvgLbwM^?Gr}$1@g^8k4?T<*CQI+U
z$7@<Eo$+M<XItUj-?Oh5A*~nVKYY~wbbOgC=dT`BE=Ph2^oPVXAaLCf>J;iUmFbXM
zab(9Op9cyZp!+-g4dFDK%=@^l<OV+3cvBqk?~@i~IBpfhSbbe`(x-2FJtKK5w!Mwh
zD>tbTDCnT<cta&};=^GpsW3&N+xqO}mG&-7=n)W}|0NLc;|PKtDKLLnnaIoWMGfYq
zun@WwD$GF}b-J8J?{`5dwH{$#<&xOyAV#WjB;*F4necyE0Fmyn*Ag|Z`x@<Z4?KsO
zAVH^iPK1pv5rt<+0>2<UpQ;tK4^(_|bTcv*uIEhVmvo-$s8zn^*(^ioc@Xm|`r{zu
zcX&A~FUd9pGm^pf?Wl2wB(U3~=2cJ1bKxju>_A`2K^%fQI!yo`9Z$u%b{4_n>gNh|
zA{PXC-?6r1hxL$+oM~yz&|VEJ>o`HD2)IhPkwUF$lHivmlN^IEl*E<4-@NFvE3H}q
z^#$T}MVb{EgL*4|M4z$0DDAvWe@HWR)Tf{MKx$S1Wg1MgVt>9pYze3}+erJpEjs)f
z+^-~JevOQ1*eWO};Go)H3uO9w3f=;oW!Gz{p47<{ysVWP;Ja>e*%i?mFd$rZEX6cX
zQ_EFM(WNn5Me8XH^?kzd0~AZMuN}nlr640jq^gM!P{zpxIRils6XgS>hClr5VH{WR
z?fvGiXA0*ZMM^8pGE!2C(&|r%HZsW$vWfK2eV!^?K^xg%?(b}DYc)Spe=Px{VZ5g6
z?~#{fY9;N0T|M%gwvSz~Q`ztDR4zHs$2j}>))s^X5!VsaCQo9L`ieCx6k5Hm-Yt9q
z=_0%V?K4Rdi!0IihQY-4$p)G}pvDaE)MsT0)v{PDfLd{jJJeEy&D!cwfo}rlP9^C#
zVH`ve`+|PLn@G0Y+}w+8uN}lzWTPtOn7_XY<9ITKLKfNEc>+tzg68A4NTI3dWPB^5
z(ZWqfuL8M@p+T2|B1cV#V+YJ{m9LH)KgVx&;HmF*0&flj3xjTTYyleGiU=Dqe1d>!
z!lw`pb=#Y%^?=n**ZWEo+xiuZd@q4O&pCMWvEksqQ}I6RvNVcDh3@*QxTKuvS_c~s
zT#E0^DB_y4v=EI9S3gGL_iv?lT#?U5aL@=~Gm#ppQ!_Yy@Wnm#P%d@6*`_jdvKtO)
z05xgY>#43P*x}nh$R`ma*bif*3QBDZbdT6uBVM&T>mo9y^@^q9heXH)Jv`ivK}RA&
z1$JQcnT)BDmncu~jjSBEB=ufwCwf`#&Bj++AI!5gV*PWRDCnUS9zVTCg05G$7AvE|
zo^aAf4?}@!y)~V<jNMeKp0SK!WB7fT9r|{(U~V#vsbPbwnkx)_;CejTt?qkq@pdjf
zo%4Vux|A5qq8n<o1UZ(xQ8X4fx9$Jab8FI5AeYiLQq$Ui^pms-dH~-+PVX1}g{n+3
z(#ryMu2|92t`VKuGwK<ClgTDm>t-6?**IT7mS`P`Bhj~Q!LXgL8DM&a`eXjFAT!2%
zLejF)iEDcWL}<%-Uqa__u6RbOB;j(6m(KZ3oc3i30mV2A>=7qe-5yJhDfzRaI`BrW
zm3&`vi!2OKXUc)TnW<11YSn$Zy^Tiwopi4Z4}J$p3~POlq+=9dpw*+C>EOnTiR-BF
zbk#4)oaB5@%+(BJH;FPkQY|X(7!)W7HwqC~Tn5QK`*iVp)g`}lXXY(hnm!@npH@Kr
z(-<wiOw+xCG_(Luo|%)o$ki=(%Z6pw;1KP>*uL^{6aLqa%YIKjO?$;9r3HaUgb^GC
z=NucmQE-)JC##(<Pta)B{7+tgJ=3qIOUzsRn*5cDstCdu15meS_Tez>zj{~!mfXb<
z+o#E9kA#n4z|51vc5s<37N;R3vO9bwCKac4kK&%a=uo&_zp(C7t^jv`Jv-JA)-rCE
z)ECEuopMrI^Ff@Uq)e+Z5S4#ZsV^8KNXxk<MXfQM1J}T$&JRzqL7~)eT@oJoR1|md
zqXD1U;~v3E6sEvAIsH(;=e+((o&?#2sHj(YO`mRO3YC|bo}PCEOKe}N=rmdVhHoNK
zpU@C8e%|?koZdK49f#Qgw^UIHW@3~n?f4tJCFbW%BZ+llbZZb&w~Y}ra~4R~9;e2X
zm6fFzZvss^0fF%IJUxgtj-?t`9S>+@Am;<cl(c^h8W{55h<<V)fTMd34x~E`(nhES
zRW1QvN#p2_7ivB%;b$8m@0X{z6jEZMcg|{8M|ro{IB)u6sb1<b0jzy+3{SD7QdJ&@
zp7vj5-?-4MmY~Y0wkMu4V3eduBJ$DGuK#2-OjO=>`sgndQ5i^RMmh81sam`8)q|_Y
ztI>5wpIEj#!Hu7u?nKpzW2C~utU45o@9og7GuB(Mjr1zxNuww(SjL%`P|=LeO!%qJ
z>CW5y`u#=@xLw3{`sed3J}XW{`N1rvLOWV3{6wvsQv049ny)NLHAeJPhMl-6SM~f|
z-kn4<`}!6^<i&*GFK;V$rMw$+kDLC{Qx#6yPzX9BjMGBic(rk{YR>i|-sjHOqqN`G
z6p!=p6Ay{`4(F>`P3V>X^XWu@NxMfGE*G&YnhDd1CLVN>M-O`o^*(kuD*$a%X+a?%
zVk>^D+0YhoiSq+znU0!An>`u6VAJ%7gd6T6O2`f6&_X!4Y=nXl8V-pFq9v#ZdQz*<
zt=Mz_=jT6Lj1^QTl|$u^sr&&xkX4)>Mb?sFoCE{UUCMJVl~A!Dt!kCL@EV{C9e&j^
zrpy-NrUaF(u)GI9y8i&5N1lMs?Mb{c8b77h5E4ltsm5h!x2FdRstc%DAtCRbjFGdr
z*{^7fV6OJLb4y~^aC`!<zt$~C({BDZd2z(g=YCIjnL0iM0c9m^q+SP=`XR(Q_YPLI
zswKkrLgv##D%f3IwQ7HDnxZs6_IW|8sAt*ERfRmq(w>=7g)kYSI41RrPO@WXH)-Q#
zu#=;~$7vjC%b}uxBuIw;*)Z#qO1rO*eI@?^yMAF>c-a*^5`DwR?fuT|>}zxxl%f{A
z4((>m3S%x{IcPBp)UpdP{nEWlZj21?-g3UC#`Vf*7~jnCRIdfV2(~K;c_VR|^1<?7
zjWSQx5_^`f$|Psh5Lq@G;7F8tH7F2GACWXkG$yYDvx+f>d>?Pjr<ecSgY1(yFze+4
zL^d392J_C?%+ucm`>DzeLp?J-wok;~C!H`!P}JM1rNX7yTNkg?1FG<Dn<JS(;a-I>
zz9Fc3yB$k9^cFAeJU48i$D)4B>St|wKat`;g_MBsD9l17C}?01G5?N}Q?rO8`@w&$
z=<TYvuwEEE^o=uc#+WHTy95O>@oyjcNW4BHkNiG+QmXeKROLEXEnVwy-u?mIOl}nI
zw#Rm4siEN}#i|IFf{nwTNlhOGBb+A=LzP8noU-^u{fOia>Gc2tADA%?F977Ic(Q+&
zlO#6HWkMRJmh`vl?0ZR&=F6Y&8+}H70|&_7bw7Emni!9Kxrg~RuM4OdOEnZdO!GQw
zyf~F4Rkb4EmGLZrmrMmYn0du%W;Ji>9bu&|u0H_ysJbV9n?7;n*L{oPa_n<8QK+1g
zM)9r25XETNKo6YrXIM)($C6&ru?&^O_IBO7iS4UnX1j%Px1>I10oat2@V?Bfc~>h(
z758>;I4Hred{Uo_c<EQ??_JFmX=nI2zE;0uj!ZTh)|M0V1S2JaPu9ERebJld^WA|b
zfud#88VsXAA)0?L1RVt$)tdb@Fu~8C`)W_om0D^{324}SQWZpCKQ}*V7mO_YfV-^4
z0tRD;K%v9iBKGSbs|c+fiVTi}Gg$G9HS0TzGhFyW8Yx`M*$nN;cpvb0qQfL0zNGdO
ztuCq8E#~;HVH{eehXu1J`NA55#LJB8y^HfZ;;C$#V1mz9Fa37-+_eR4SM(hH&|FB<
zPiQ5;Bwvz_VnA>uipMs4R6tLQ1SK8mzc7+LZeg-WG4nZ8DTfrQo{GlRvkC|1pL+)#
z{P>6-CQf?z_xC`0sw~}qrV)53x0OT%suH7dQVP~}fZNUj5q26$@2Q-^LMhsbj!NRc
zAZe!pV7Kn04j;M)w*j}IwaM(t7~NVUMR*mvINf<T@r~OLnRDP1XDn$5)hGqDA(dZa
zZ$yhcBDZHItw@o$wDO)|NYq?3UwVHu-G^b2({SKu<&J@Wlsu$Ii&#I~>;%*v6mEWD
z)XgV4TZ5FLD3sM#I@9mF(5NUBD5Ov2f!u4{TY$u_tw`Tl9YRKS`7xsqcd3O_fR9B#
zmA)oL$Z4CK-r`h$r9rzR!u49x3XrnWoKytTH1|1=>SA^c14YA~8pI#i!=((OFJH%e
zf^>GUYi!4Okhv5^LGqwQ_z~zt!p$7?NUjFBHrl*OlD>VDEzzv_TN0t6HgS18d+ZWe
zteOjV22E5}`^KkaJwYH=u&&&oJ!1vSEf>{@$M0Mi1tI776*i_2MP)UJZ!Ic~o=7<9
zR>H}U0odvWnbr!z`>y^>!CYuoG3KD06Y#rgrw<wwpe~PBR`}|UIuHTlzI{Vg!b2%R
z_+DZD(s}0kw8j1d|B82#@{H6Sp>^x`Y^|JAxDh*gP@G9LNH_0reEO;<^10*cr5Fqv
z$m@wk6Vf$BDQOM-`ju0orazrw5Y7+lRVdsIN2P4?cn@0&x7q!-Fem0a$jOr1R6c}r
zjD&Skg^wr5nwe3}ad(I|A_mWxXZj@nS{bXT6{d$4k9kfeV}$)G?#~Wykl5ccB+6$C
zzZDa_6po5L`jCWM^A=*`wmZ?Qf-EY15JCTT$C*|p9BqS?cYmpw^Pxkma;Q{}4euu!
z*YUfebFic>FwgWzp0Bkx-_HAZaz$F0^1<e{f~|62(JNij%AbX{K=g>?Sw`O1YUH~%
zIzSE~D4eIQrCy@+sxO59oHAHa__{PILaW=OJ8lVHZXRZBnGrhR5P)%&df3S2-*azR
z41FChrSKpPNTZ@c2o~+Ivk4Q&B;(-RHc8#ir==t)lbVc}wiNNE64NO~Law)SWnp{~
z{!=zrd!U3vb5#cm=Tndg1A-XzOD^B#Yj+%;D*_E^hEUrT6_FJ`7ymYr*36mmLO%N9
zFVL*vbk8qjAu)u{nza&BR%C(NP>4|FkQcCHzF$nn2XM_)4m~@cB(N+cHdf{&yGfR?
zJ~t3q5`+_s9C>NY4un?(#$Yk&4!tzqmX-sz0hw5b?ar75J1ZDl79mp>QcI~>Q@aBi
z;4l66{4NSZUi;K67*l_=dRz3*<2;TC;kS<>57@qo`^EhU)0HMJMAl&R1J!FuWc^_E
zJ{PL8TTLTb4!mf=>hVGZsanRah(T;=R0j|_8td8ebSIHnN7=7mRm!Z?#p@7pBr_`j
z1D5}KHE6KtEj|6OQE9@-;HSrCecJL5NtPwwLt{v-kf#FyZo0!|UH6AW0NJ$7a?2bO
zQWz&-CN=VKdR|yS;rCg|jDOO!58c@V^6A{%kC>)|iWb8uy;%eMVV8RaEmVeP1mOMA
z3k-7L3KHd&l5{ZQ9Fmfv=5(EBZ%RrMiqq%&nxthc_h5wdRUY>@7n0K;p-||guKhKs
z@3nB*vVAy7;u3JgD%IPysjq96YGq65#+&I^o5{PiEU#($aq$86MlVx1kPZ{GVzL~Y
zz*c)I;*W2D4Jtq+Tcf{kX%Lprr)&Qb5VNfGO6cyT=osUDjQ_z0%pUfmmdg`A2fEOg
z(4v$B2<YKW!7E<f<94lw8e87wm@aGZ6cK%M_zQQz81vmJY-Yt*oXj58zTVj&avA(r
zK@*3LT8S6YETAWKR4}t6f}=B>BTUe1#%9kYShycL5a27-56d*t5t1btMgrVjyAY`h
zOZ{QzC1_l@SqdXKER-^H{5|_))HxSd3$iezbu^4nCK?jbGk)c0Ib+m6+&&ufVb~fC
zbMK^yg2GpA3iW83=f+Qjk@?#jm^Caq?s$SQCtoZ3S9{9*WaNcnsgfRD03!7f?#;wq
z)qB7{gV`d8OWJ0$lwN5DMKnsF0fu}K>0#T3s*db!!sU-X8=zxIP2*dCha3y}Abx7_
zO6iaM2z?#OhMo=vG?u_K4IR&>l=gDy{wRloP&$2z-QD|Vl%V*ss)^*=pF@J?#UDA8
zb_XGJA}<-lX}7Y7G=fBib?w{o<^>CbYV1(N6IN$HqldUnp*r#)88ce!+TSAY^g3jT
z`_*<O%s{L0N}m|U>9^3Qq>TolXJnuIdhRj&t<qifcHULytJ{@(eu}pS)iKeI5q=nh
zuED3Trsz&bioVFV(=4l<@;yU^f<LY>R%2Sn?e@o$Hi(pVa!vxs$S}+uazmlSQ|R6*
z{?FH9Ug)8fzXQHP&}pH-Yo9AUabQp(Tl5LU09?3O)2*f%Gs!h`UEO#WAALx4SZ5`3
zQ#f1W+-y)*GMnF{N-Lrl+8|tT^CuqpRMM%J;Ey$ad5KV3E^wOoz`lD9U1Qzc(r)Un
z^!=YR`hiSIf0{tKgeChku`q-VVX81)#AkFcH*sW??h8`ZTfIjGH2#m0FXeCc`~}o^
z>)ckl30_gRA@L}**jz5EwtSWxe1p)#|6h@g7OnE9Jb3J<{7;8{mmcB5YcO~YTM1D@
zSz$Oxc<<(Y89LVzzX7&$*4RTgFzd>ouHPARr46rL5RLpWdMelu<Op<h;Y!@z-u@K1
z{jn7YDKM^hrA8z}+g0;Jtyhbp5Y=F|IK!VY#MX3wggl@qpasp1?a6>O<4v5y@BO4t
zKdN^C`C|XeJxd=^PMZ8F^+Vvjcd>qJQxl04Iy3prDw|?Txm#LTXazjldUm59Fy;}T
zFpjr2YaNo(H$2DQKaVK1<a<1zoXu<%odvn$D7uXa!(*g%uQ^(1`ayrq`tx-pV#ZS)
z0NO?oJ|opUoZgWRI=&|jU3g~_0$r$}jY2;zsytxj@J<jo%cp1;*K0w>rei`ux|3i<
zQ=vy8n=*1vx%IuJq$`b1dXEeO!P8ELSB%g)qPc^Ah;dO9H<9~N`+gxeJ?k8I+_wmh
z<(?&tQt+s0N6@lrPMjthD-W*zqHknMXD4{M5bjdLqyAx$@GBn_X8gOUUEmp+3C*jE
zNb8HaoY@haa7vOH>cuk8@Sw^*7;Jc0sB%Rv$>fL$3k&ChXbBI0H;Yb{Ah`E>$&0@M
ziY_CxCaVi>?uF&E_<%Dt!wcE}G<czhAWl!nZfg5oC0wy7#M%OS?QPmb7;2;~pkJwz
z9`NoMQr6YwTYWwD?mbw0%vzcL3$`<uYAsOvwvu*}j+Yv=fN_vxVBktX5iF&qk@fxb
z=wxmC-2Ccj{~oFy0^17I5i6qC?kG%;uc)qs<H-|FrvUv2XIvQzZ)Ye&-bXuKA)W;q
zGc<6ns35I)MBw5?R0e?FU^a??*bu0gr$mzY>KofQ&({F|2QbhmCsF|_l#(dlerYvP
zQEp0n&_R~Hf5a%|7d@n-`gX^=>WIE*3*6y%1A@UsU=t0Z)*w-$HidesC#sIT@hZwP
zDWE4mBOE)g>jp4=#sLPlygy^B^>oG&>1XDadg)6Rg26YCN<)?x{Ey08n-|$<LHmV4
ziM=ldbrr)ii4Yn%;4$qK@`WeWwrZv;t$B!lQlti<%H~`|b$sGxVP0x!6Qdos5KqnX
z!&M(tPr=%-lcr514!Dh0-EI3)qB3jDrdMQT_+~WrK|^fP?`>{lJMMVZjkucLrp!b_
z*gyJs5>h}_F?h_&=XuKIR<VdcIHLc=#q7FVEALJnHoW3#k>oLLxL|x|?DU4;OZVj(
zVG--MrK=x5d;dEFcsFh^`51wD9R1C|#~itmiXfN(rAQa+GY>!5DEwE65A^3n==mP&
z=5?^CFieT_pJxR0%jRQg>O}R*KWLc5j&XEnXJ-hgx6E8ZBI3qf!H82q#Gg@S@z8ib
zkqSFtM)!b&PDfto<Rq(%I-w;mkS3qXKHviOAIoBHQ&&Yr!I6i4#Had3lGmcB{8OVC
z??a&wUjzpzp;kj~KDb_AT(D=D$Yq<#eo+Cf02aU^&V$&_sAl!N!XD{-Ng9)`5hTjC
z2mh-c-TE2i%T}uKS_p5-V56c%l94UFv9S@s2rqM`#v@Tim3J)g#Z-lXAaU2C(bj|4
zSG8b2Eos5AETO=vjin+BoT!efHN2ABZbzOm_Lriwf+tM^f4jeA$`ktn&?dX%x)?Eo
znQpiCjVzlW7eLbBZ-w{Bk)@tyje!*%wjuIy<@9RU0}(xzM^KThl4c;LD%Y=}p<F@p
z2WeuoeG9l=FGqj}PVM_0kVx}chLzPDm06P1$3<+A5`qguXL+&tSF%L#+H{kj8PReX
z7j~qm0^rYoWW|gIq}AKpIGsOq-}y0P#b?oAkHi@FU1s{;4j^pXKE*8$CP4^qb+e36
z=HD+)DkUc;=WDD=dd@$-zhc^|8;O=u7VUHNvc_CyKn(Kd@;Ed3l`_^0!D9Fjf>H8j
zSn8ziXC>Xts<%fNM^4HQmKS2ucHxdsnGsNxaIO+dwQI{8_zu5>@>Mq}d?i;A^t)tK
zLDgH#?}0@~d6dSb<U%DtDhDEk+u1MCnQKo5$Si%jBrNnq3CeByc&uM+&@S-Y>jz;t
zy$jubd5o;4LB=9pY8Zmh!!|FA*7`8Q>#|fkJ0~G&0A<QtSZHw<7G9*pPZt?ratX>#
z*cyvM!rM8B<0OQ1^aeK)K72BN0=&Px&uSV-tAqEwCJSwRa4!QGztV#G39xGEp+L7U
zgX_xFQ0@APv6(Cc9RL-<p3f79`}FH=a}=`xT}Ad&s8#=A{fffe+?<3{8)4pOluVCp
zeDN9~gFpG%?Uf0&r>Gwo*0=BW%sjgDaBO=CI3zCr`i+y#{NV}AMLi=zD4RKjmpa1Z
zJ)Uoj%<%Y&-v#wDfX<62TB`0bf+JVzoBL=Q!61SP<1*U^EDJpNKsjg?lztBIzSsU|
zOUFZ<n7W<KBmYq)D0!h!xa3f+%$H=^zRO1rN(X~?j>$mD2GjB}LtpIMQ`w?|F`_$R
zUVqYyfJvs|@B8rT*?YC6)p%PK*KMsNI-RyJ9?P^f<g7KZCqLCv*059r|9$<zftKn&
zt2Fdi|Gg%d*|*Xl==f0F>Z*l?)oNB#d`YSG(mBcS1<K!0f3_bGee9EJ{^<T^nELOS
zX3k$pjRp!d(+3h%b#*WoWKR?pp1XURZ0aiik!?c^|L>ze4c+>?bm2JX?=nQ0^b8x=
z=PlhoFNY3d7^88?W_~xPg6)Kef9|V+0RN?{+{A%mtqb@8f37fW2em1<MH255VHwj<
zIl1!TOwQQVVB8{YFuhh3xM%-91K+z(azFPPilBu#c0k5-EHtLx;RAFhEUHy!7{&9i
zAY;4dQEVg6`}_0aX8hv&UvmQ_xdgmj-Mk0hkvQvy4rZtnn1-f@ppvL)pYUC&Is6Wl
zRXc;AL!r-hN&&Z3M3baum9qhtDIC|EZp3;^)fJhunU-w5;2j~_E-j{h-zoHtR{zAR
zX(3d9E9~vsgIB02VD$Z-T&ANOaU~uk>H!Qxvf|yX@Y0=n)9Iq6+SQ*F1a3@pfa9v@
zWCkk;9!+CpXj_iAeN3xyskkQS1fYS6NR{C!2+{wNxpu*yQgCw44iPA9L102yxspX`
z#nsdOo+p3L%V|wf*pDtGH>ieW33ly5YSna+zbQYxgLj1=m+FI$x1}l3g4Kb5^H~K{
zynFDQm9gx;fwIh|(svwvkb4K#f$L5ocv7%3j-U2RQAP<fdY@r0s>1ah$ibp<KWg5T
zeL(HI^(?f7R8py&Y5lm!3Wib%K5BowqoAL8<w>D6#+r#tG1E|{6?8=#HwLTD6Q|H!
zq~2+`u4reF%4uyj6-!z}+s>uhLd(XCO4bHe{w}>+npyN}e>W2qQwK%dIeoYOVn=wy
zr&7&8Kt)WviawMfl@j7mD%I4=vez;HKJ6A4sg%+wGWzxlF|CWs>o4hBlnH;}6J#~j
z)a2aF+U_SLN5m5FtUvI*P(cX{;(PV|+fp-d(r_i#AQkx5uxUWQyq?x<@Y#YHHwSHI
zBc3Tr8hpBv#IBz`NUoO#HaJUevpvSwP!Uui2J4ONLk+xpU;G3f4pcd2?n0-3e>Z$8
zz3GEsX`cXT0!uADC*GiDGAhI<+A&nBb}Jj<z1<AfD8*EU0@oT$Qt61nPJlTyMYGU)
zWgCf|HQkK6Dk=9FG#&VA{#dHtj<ly}v3A}Wsm)0C5$<4l9`QdY<e$?<78&lBsnyS+
zKE73uI~#~8qs;wm@yMXEAYL75`6N(-`Oi1_7-6dTo5(Cn<Mvc!l?*+OMD_`6Gk=Yb
zGotOn0VwZ~vA9w|>m0z|5pJ4GJkIv<x?a)$4lOsf=4x5!&>o89v6p}H;^)r&VaF!F
z{-{uqzM-zlf#D^$%?+tN|L?xn6fSlV7lPj2A4v1-f5zYc^KC~LMBq{{!>CQYZv3c%
zxr1=xQqS^uVt2{1-RUtHy|W}`t2!C=P*Z~~v++>lvL15l{tJoqx0y-)iQuwX-VY;2
zWhyMAD(A4Ol3b-ED_WQ{x#>OH2syo0Go<OlL>bq;0wo|z?zNmJ@J0qW_SKUf6>yTS
zi1OBZz>|+8NDYp;2_Q9fDR|?!q`A)-MW`ORwJf60N{T>M>zpmv2~woOI4U7ie+c(x
zVDc0c>;CsN0$D|cs{E6)M8eys>MSk(eB%Uc3UwTLO$?Z*Gu#H52!ZaUk&GN7D%t5!
z^Vy@;WBn}?{XLy9EWqMgpI+1StT@9aHv8jVZM>t<6p+p#c>5<ug=MWLd^&e6t*Gql
z6Veix2je1GdGsiB3r~b9yOki!MtHqv%$BRdR7HHoRL=931>@#e0y%O+?#@|BC7LRk
z%RbNTqGn$|@Xkz^a#jb*oo_SVYxMo4g`k7#(In5SBVvY>u#wL3zm{*=sDupTx#C1E
zJN6a}_uIX1ki_+Gft4w`XOlA@;Y06T^$6du;|F9(r$E&MW-85#A6zUOhQzEUg(`+f
z8ZS%Z1V9V$Ip&d{@S|PR*<cBF_)yF05(uxp<$t)Cbh+#(p~dl|RoTua=5<p6tS(uM
zK&lg7=NWDDZ)?B@F<@!1%a7U~AbJ4$rLEwCyTpJWVtAv2Shez}*GtDGvYC4z3&IJ9
z3!gDt@6MTLaP*4@gdjM+63+y-n%D}0!xhrWrxRzc*>O3e=u=h|iXH+Qj#B;B2m*E+
zleVXhy*N=++Uo8fZxuc2EJVlO;(tQIz-l+c+T$|*`D(+Z!~Xf*(7f_wqp>?4jURdg
z3c?XKB?`95XYg(eMuH=y)KUVsppAxmao?97`61z=ZiGvcc>HrC>Hhd2KtJQcd?!^O
z&Ow9sc}{_6mtqN}q-uTX<r;gOOAb90CSpI%d!COh`?1!LG9i4jjG+RU9iMQuwWzz;
z&^?gIq>R%i=b*2I%UXNrdw{3iDlREk8O`Tlyk{ZS0+oW;re<dbeMy}Ic7lnD^OB{X
zjpf-KNR8>DVn+nN-66h3Cc(>5lPli1`URQCugs!&=XpW%C@Szonb{qP{Tmd#|DY3x
z$cS{J)kQ3P37RMBLG?R0-Qf@&-^Lb@Q5GYKBd84sj;{Mv@W*HsKsg1r=6N>=HE(B0
zFN1VaG@z$Uot$++brO^BExs}QuY0KCJ|7b<sM2p+4s47;THPDIe2RKyDSjilfN9W_
z5F(slYnuPQz>c`RvVcD2HwBc_9x@Z3VwuiXs%=S|^B=)q#X1{=8-&Lz@N)X}$dzEv
z%6}k~*xm0X+xyLL_Nio9@PZdyLWIw+mO$<lw&VMdJ@&aRiAtjcxTf-HS?y>uiTENZ
zXbz<3WrLIr%)FaS!9M9gHhIix(FT=UbneCfIhzm!H9n~70-T9&ibr|9&R>ZcwnP2X
zV*^qD^k@qAg?}ky=_X*xi+4hRLiGRd6mbB~ImOhUh28(*Io}jY`(v*^-m+mBimzZ%
zh;26WdsPeML7lshjN?<+UNq8~oB#}c$TL1Y@2ewt!YjvS@PkugCy{)@7UcNo@H%9&
z@Ee>u4A)AxhX&!4hY-ZSa~E|J%f?XY^2JNI1qV3E|Hax@MpfCpeM)zCcO#8-hlKPY
zC8QBly1P51MNm>&Qc6l%L>i=|OAzU<*+=nt{uA$qnR&nHS}gRO``&wB`-)$T4Kew=
zfIm#7!hB1O{AmPh@I>7@pym2x)&-aokeGhiLql?tL)7&1q-2~7ai_)tYDOCyrCu^5
z`DJlomds7ED3Xg@e!Dlz6Q9SXg!Iq#mxU!HJYI+c5&jIIf95n81}6ZdQQ+Ha!pu84
zwZ!0SJw9s+S1Vh0_+nW7XqF|-OaPtFCv9IMT7be&#^pB?p!>lHWjK9!KC8n*4i&b5
zOeyPQ&<vmBMG3yHe^$Uce~P#U+`%OR`S6P=+~*vR#^NPNJ_(7VW8OHgx1ZP7f<*bp
zqt++A?)U1<H#m%?9=xS7O>=%?&RvEW?d%n;JcVRhwCm`54e`#Vg~!d}kEImz!tQ(U
zat2J<t{{~OL^C6j-E6h?6)iVSN`wq?A$q~>>;32Rr&P!BA%=xKQKMLZ+RdiRQb0N9
zUkVk3<^vw{1sU)K_}I3?h`|ksZ=TjhvZ8*o23lxat!Z+eW;5eY!wzBv`~}6rwGOIN
z$C@f4_{;S2Fs?fQ@A^1rhI9z;4TJcqH4B0WjO)V%(Bys4=5g_4DJXQjt|jLx&<5xz
zy8N2c|3EXjL2|w#zIkE^D4kIefhPz<UR;`Z5H<C*7H?0_bwQp4;l;j%ptmd&_}S0q
zy({e(#0y40Bq_GMG{{a2=cjxDV%&9pgc&iO>DA~*cTUE8R7-YIIDv<~>9FQ#0vJl<
z70m}xwr{nY4717X4{+=x?@d>l2B$&HnZE=;Y5<m~vRm?!d-)niUXVi~;sG`8B1tbl
zGn`{y(R(b<UqWb+gTzDV&FdMOXGpA9#JkZi5l=>LQkhp{9;dd6y!E2|&TFIp_=Bw+
z+bc<A8PtYC;7!ZiGPq-{PoMxX!?}`_`US@3N2%j|5~Fp(L92O$@H;>Ct+OmoymhYj
zGlS2ftV!!qCk!R$tHZe86Sq;fBJviEO2v-Ov_eP9gqAd#t99UFI4g+t;hL0Oz{Y@6
zq>|;1NhuGxR`1Ff7<kl?P7Ar#kgttL;tibe2s(!kKE3YR)h3^~d2K_=AA}0Yjv{y;
zpqKZh_H7D^iIt^l0-;Ro_n7fjU2KQ2fXB|%=}AIcnQg^88g`ik)L}VjZk{$v&w^#)
zc)oV%%PHy8hRG$Af%yV;7_!TcEGamuzMPXR&vSQZc=>1W03XOh6R`VkY3OHxL?M1E
zi=SnL#YSHC_U@lJa(})4sQF2AvLdum$$!_uKmi;wWYDO5`G4D}#O#WMg8aPVvKZhn
z21A(Mc$h6doizvD3@NFv1674~;uSxym%^zMotlq&y7IFc4Qhm~Ge=O&T|ER(k%W1+
zx7Vi47~B&K-uN7#0E@==PJR<zq}cGNsHl(whxbUu@)Ha}=u3+qfhx1@*t=MqjM7@?
zQ`G{W;E7_>^h$VBSk+iGLU#3y{)4<o{u!Wwh~xw=DEhYmy7n=7OF_S%*0DeS9(me=
z@2Sxk&CDa53It>N3Yi+u`Hy+K5}DttX0CurI9%`<MdxSIep&4joe#dZ0}3$b@l+CT
z21T%x*l*mx6OJ}GzV%Rh$EC4-Z(>b*>-gg;mGBu(goMX}-FT5HqQ_Qo;U;*@2l3@1
z4nd*PK|?vZ1gXJ`_su(V(p>Nyx*JxnuiI`fB_y=VUfZ`$7y5MK2t1!rh}GZkFPYCC
ztv?5O6Xx@ohRD&;SL|zY7dnj7pO&U<sVCdH=N?Z*MDhbxHa)f)K>od$^dHHUgHKFN
zax(a`+#^0NGk(~;W0QYIc3$$jz)fl^z)VH|86Y}6Dii8?bP&b%MG?S|)aZBLzRH=|
z_rD?Kvmw^J5AQ1ptX^V8&x9gD;V&0NJGWh?ld|>vA$qNE`u2tz!NNr_wGSQI6b?B&
z1@_+Z2qG5c0DZz`i`?Oo#EUCIojBnIjhAo)b_WNOqxO$V&{E@G_{XB->ovdbxZV=h
zDSb*t@X#~CIcE?-Q}KiC#OAx#FL(&-vIh!3dw)sg$eZ`|8UsG(bS3^eI)rSd<vOzT
z6sAN;1Y=`k5dWN^z>w%Fl8z3pH#`ydTQ5q{w)Qn8N?6*3cC+^DntS*OCcP15(Q2Q6
zLZ`=BMHzs4RriHb2laFW#l~l`(sbu$271O&h@^4=<=2RPtQdRQ)yu9O#Gd+g@nOjm
zo<8o243H##yB&Nj@OJoCeyo;tX8O^Q+-IN|>(Gk-R9*XXk$)vmnAfegUfcuobr(iD
zH%@{H>Pd^5j49csox0g^cbQ^ICuwYErn_=QWkYn!Rl_0CTPRxp-KJ+LfE-bemKkdf
zm@n`!Fua7G*+85#wRo4NvsCgeH$(ZN;zaoOS@=f(ac*HwaoTGKkKUp^{aNb$qhdie
zhmpJkr24||@k|gXn2hVvnY6S__BE-Y2&UaTJW-EQS+N2*xoQ6oCpUX^Z%D#!%9G>k
z_Hy5t?Hl~2p9f!T#)~xKN}dA1fD?J=Ck_dJfr(OGlj8AoCGPYOhZhUyxB{xGSr1}4
zJ|2xmK+Jp`vvw!SAE!|QE$NHJV61V}BL$eQ3?IhHtB)M<iu}IG9YIOSKp(vja*rnk
zHTZqDzop6YMDcFJ3MSL%RMgT){tIc424fRt!}5WFWRPuwo)3)!;3foEXk<k)M7i|r
zniRXk`@^76Pq$A~#7(|@e+9q`?iqX2@V4U(8u#w?INRPB9E2ah;iG3KhFnXIVK!e}
z0r;wNVU}Blc#H>H$2ZTP$Q9ddHv7_ylrG4mCM=U&D4krl?gM8vl6$9Wq4n?}{dfeN
zVB^H<;%#pOBy=*NCNnMPvaNyAXc3-G?7-VjhgEVC-=~W-^O&n@9j)MnB?g`?)hQg?
z0htv9dGJ<`p^X+}6=2}xwLla7x_@1l5$)#U>H=f?N8lwDb4>^O=&K%IB*<sJ^)-!s
z6bPWBpdZ{6AI=+(P4!k(1VO!N1N6+hz_if$2f*qcGOR4mYwH@xsb}3I>C_Ze#*}~c
z)nxY)D}^mVU{AES6rdzkiyhDVt;b=zM6g>j)1}H@eI+K@u{SoROtP360WFnH>>^);
z+BW%_)=M$afR*+~UrV<!0ODTbX9%c18S2H6{A@bxg~lyjBATSK`kYpw?_P>hDyf-E
zXBS)xY=0WM&5IY4R}<Y2@$9FbeB!G7eSSF+3+vX`o^RW7mnCHOgPpsA8Ab$FXFB-_
z=$mA$@blUxyARa0AKb_Uq7TW7l3k1)4k6)8dfA_G!QUWT8!j&dT*hCL$JBxy#)!CT
z`nA1jSMG~9Dy!ZI<D`c(eGz8hJ)cet0eA*vDJjGdqCA@`GIjryxV4*Ym)Pe|C}LNB
zj1}VbJPte=jpiK5|D3e>g|iL4nZ7qD!0s*&-*QFUoI_%4)Oo;!va_&})9qhHih*ad
zKEziiQolrjhX)<ew@YSse|qY2QK0W~fba@ZgBfqy4SxI|_vjV&RAYHa>+RV+Uw&JO
z>zcBu-3;;&`S!twQoT;grP(W8UkGbL4_qlY-DUb;V&9!d#I6iLiyxCV4t0dNHN{%Z
zh~fjl9=2-C#QNXeJdp4w#N15<l4DM#w&;K367+E@FW<QE=o3WWWS6<d^m;f9gePUX
zHGLTxDg-tatk%3tYy3b8Ci#$boIU|FnPAVpJpe}Oeh>YWLq~c$n<)ViHoW9=ppz;W
zk~otkvz=bqjJWgqqXP0uK;`0L7^}B8k$WPPp5^c%197V%97RMuGSJ+{6C-Z0qNRU}
z$k<;|@F0#{sJi*H(iN)wJqu3e+C@KUQ{*sfruO?fS+fm(a3VqyB@t9%W&x;_BUqW+
zQvCsmaxE9f*JJe-5vSO3%hbQvO<F}QW*mP08qNvU*Ac8pT@ylTv>8Xr8$b>2f_t>M
zM{0A^MY>W_uxWJ}%r<#A=~B5YIzL9hrc*TLVUokARpBSbI|Zoq<;fNUcsRcnji`7m
zB2Z(m>YvR&qSn{LKxJ6bLr_>)sGs*z{*Gstu6~(L6`Daf!~Wbxg7f1~9$!7v$JqE)
zxDs#3u<uDP#l!xZX?Y#SqsrO57ep7D_ax=Duq!Ktb4-q2u(vnx;!15alJ%4fK-Y0|
z<>*7$W?79ttrl-H0eA;=9Yfg2QmoRe%}c8L-K1GUZ`wd5Eh|Q6K+<nuMKI_M+s_0P
zv^3A8Ebq@h2WIZ$YkhPo>^WAj9hGBsBWhJ)Y%Qsu@$RR~2A&Y_5|00xsp?PAMOXe5
zB$`cp3~Y9owOQvbqqvwYuR*6Ceu9blS4y{}JnVTrAWO0~`HAnLFLabV4wSV6)*k*M
zXy|1tRM#;`sfva1hs4elPJJ|iQXmOl<w+>5y#|slO0Mcuz1t^1hg<{IStkbWi@}yb
zI|})W>E_7G_hEq=6IE(eJ?GcAV7?e*AD7M2dqN+vVZ&?ttps3))r0N{LUP6%sfo%8
z_pVkR=YmqNsyt$q5!nL=aARbLJ{z7*$@Ui$Ubl7I8|X?O-X7sRP5sK2Yo@QD|HclS
znuvRWF^_Wd)IR5rAzEmp&66}gQHOs?|7YTMSL7;2-GhkTUKJ#?lGMEQyZ&{LK>pU5
z<43~yLZ=&n^K$8vG*vmnPQ&}_)m_m0Qu+=tA1o2^d|g6TmoG<N=Kr>%wJJIH-N42y
zR#h9%?uffiuRvHpAPuNtm1NFTIk|Fec-m#k&hAcC<<XKo!Gw#ubQol3@9O^rih40v
z2OaUmtS*pv=8_+7e=B{6bT$psh%nfDR&Vu{l(nMMo7Fp$n(8WeOibY(ph4yWhm#_I
z2ST;F(@{###;pgiHb~w_n>h*@9<7Vd+ML#u4uyLBng<S>j7~jNuvAxpmsr7ImPo04
zD`h<=byd#W(VZ^<1f%uqgu!^*UY7&(l$1vTSy9Fn%iOwf9;$=Z4+<+0HGEzR`ny%j
z*d&|Fq7^A5iu^Y)<Bq@)BTi8%QfK4Inl@`#S$<M3#77YRc7mk;IWV=5tXy<b{oU<r
zkZdm(!WLt4o=f2^mP2w+nTkRct9@q7eg6sVS5fBCy|%C0)0fIngcE$A6Rmo^DL5^*
zh__$zJL8oK2g=m(cb%;0NB*5xfR(3%$NYIZJhu>SGgFUrW<vA)JM)6>TBF#7_Y?;P
z7=gI`fOPTGOhVwd60N1_pe@R30iK3wkgqTv1=EER{RK=dV+ZOTjl77@TE>=VN&Fow
z3BvvzeGnjOl3N=_>D9a@u(O54uiz?={2Q;ctLd^M_-i(}0y{cOPry{0tqrJ&=T>)W
zKHXjb$-)IVAuY!Oe2QMjTXMjiva~i`p&L8}_08S>b`f<Gaus^6(a75BtBi6#{WCmi
z1?|^mp>FtI&Og;vZsa`$eu|F;Y(Xj?=|+o<3i(Pj9<M&jjcm-n|7T(ZC0|uZJ^6Nm
zYtXfz9c@L#BK@`bhulHz8toDEWhnkPzl@)t>!%uVE?pM6@{vLA_pSEl`l5cqJjqqn
zW9Mn7U-d4u%`CQ+ad!3LVi?(rY#{Q?{aGn;VmNL`q{uJ$B1N->>3p|tTpe415(=Ol
zapu8rt{*8DxaKqqQ1UQc)sBwV4}F&q$u~tf(xlyu)0-CX`#8(uL5J?e6FE<lXCK67
z)F~D8l#doUMngKzStn$_GiFRF&)l7R@E5T66Fa9Y_}O^#7T2JyDSfh>Lu0|9Gz~v`
zxsaEcB772r0GzBd)~Nz5oNpm|S~x|@Z%|^PZLZJreT!Bf`K4YBW5wYc|IL;oWbVdf
zQ8L5@3co)GLi{f_7DBAxnCS=(%u~?HC<$?pXYT`USX;Szc)j(HqV5KKpyB2P5AwNS
zrI%^FT^Z}rWyNJ!Jy2`dZ-%@HrS^jQepmawE@0@aqa&C<4?ZM;8k%9`Zdi<tDsLth
zwib-@9mN)uKS>KG95CZ2WgM1#X+?7z440n%d-q0$NvclZm&|gd*nU&aA$8R(Vl177
zRAfm{*ygKi-ykhxbgV)YbIQnRnU^iB4j=G_j`Bv1j(NV8nlr{NDE4Pb?tm34zEZR2
zE_<}*&%$!XjorcV$Uc_t{9ll8V3d6O%5~Yu<Fe-EB{=eJe?7hIkZ(A9dvy&{owqS7
zxzfv_`l1k*fX#LoEQ+b#$2vQ6ZD}^Ulvq0}{s<q>sr=Q}Ru(X^cBXWsKi2KLm4??n
zW6o(oI55)I7z((dtoyP6U=K#Z1dbS;YMxE3u%4fJO>rD%h4`k)Nak!nYJ+!@^D+nx
z_HL~mqa@icZ0dVe9Z?JLm4Am3RH=U_45%>7gS^Kqlf(!F-5G*yrkbMvW?Xw(WuK^s
zzM)jT)080KwI0kLG-|SW`><&_ip^%V?_~-pUU4&cuU!(*l1p#H;^Mn->{k6$4r`mU
zv-ZW##&gD(R{W)7>2X&3Sn<i~zdu_kGQem-odxEl67(C|AzyubL6HQ1!u6vPmH+9?
z_y-pT8t?_Ez0Q`x5yDWV+b_h(OOU}}MY=#kV<S{gOQR1AZGR)Rq5DvN);B~6DXp?1
z2;cC}PNYV;!)UXhJV|5yjUZbD!mocZ+6_8C=#~23>UEahUn7bzQ^EaiQv7GA3*AjI
z$Y(&2W~SWeW2!-|&C3|=6yZ=QYB8}ax7~S_>FH*4p~onkQZqoUW2V8;JU!h>|MxRR
z$w6cIcIKl)x4Dz5jj!(}K0LjB^T>&ZNvq06;n&?lNVdPX#}4@&A(^pYL9L(XD(XI(
zJmGhI>g+k1)Bt{MQAYAA&R4I?YY03x>#pn#2celt(MQ$|1YlDS*eV%6W6__~ULvPp
za&!eV7XA(Tfy8$NQ!Vr-FWG;8f&%e>5KM{H&Gppf%Jdr(v+TLV9s*BjA~O~A{WNIf
z>Mp%*7lMt_<MobnqGU`h`~0{*HA8^NEY_JGPeR_nfD~ZRK0-d&@{6K|XnWXy+^OrT
zNFRs(%S1|;`gO^<j3h;`_JW|_4@!Rz*3DN27T+X_8AM;lxc0c6*<ziY8AoF!mN(Q1
z9>@_3j@{pRKkk|uvEIS-LiO-C-KG0qa{YbA7iHfc7P&Z}9BwL0-92RD-@7Pr?}0+f
zpSQ^S@46;8CMuFw0n(Vu`YUUmiXQ4od_uxernf$cUHqvffbR2$i5n!DOQ-&tLP}Na
z7*a!ufPioWaP_ZytLgsU7hNLIy$tA!c)K2o=f6Lu2pxIq7=u9LHhCPkzqm2W$@XV<
zne9iG8YLLZ)t@;>W!K4X8sCocptA61-4+n+x0xxa+<%u`i!0rnjE{B8w$zPXX^AR&
z<EcWi4fhv{M9%<A8NV;^@Zg1G*`TN9#H7~1Y$8O5QZ+Ae+2y8pwga7&V<0~l?#kZ(
z0l}{?IdI-3qc>MuweGslXFk@XM&a29L_*&q&VT=V#MSt7lK!Lc`e&b{-4U}%(n<il
z_s?5R2KcGg9P-cd<ox_C5w-$GG9NN-WcW$1l_$1d=+M84z)SdZL4n_w#K*aZ!{l%0
zJM$rjWk>H@W?hAyB7@uyjREfJ{O(;rp3GdFegN;@*ti?9VB0OPsVWHJBEr;X$t1PB
z{1p4)_x3lX^kpgfQ@%;!(*B`WYvIl#7XL|8Eam@=a6G!2wEqDB`}2?PwgWhFxOWS_
zqWOWI<j-*kbfSLpO(2<WnoC0r@^M;Xyy&}V$(R{K0e7SP??eoGY!u+(qd=9q^LDb<
zRO8{yn{PqUKeazcIxd07=m~gSU2{n}`6=^^byIM^n!rq@c@Bl;VaM-zFf`e!a-+%_
z{lqYk$BA%vbtp*RU7$)Gv((zS&>Sp#;9q#U`nr;m?BCP+zb}oUG&BU&uUi<Fq|)E*
zW_3U^zX)@pC*!kKvgPI%cA?=Hxl#jLSj>i4h~@8&r$>eLcAv~`ZZxu48O4NXW)hB>
zMK3r$S~~eM09FFY<xgR)6T!dl#FS4CAXj%eK+nwY*vl2)iVyi4dvX5oV^~n^gCUB)
zV<QjHwi5ecQO!ztE%l0x1?O+f`X}=I^DGW1cd^8=*dQ_HZf%9I9ZKzybwk7ptpUAv
zT|Fta%2gCom!py6@Apq83dFEe-i+-%9bV<cOcr2iJFD8tyXHv=t^{unYrF4mDvw&l
zX#6WAna&jZ?_H_)V-a_KKQ-uXb3l0R7NEqUUZLa<egFQy5oSt9mgB$PM%Zw7WG6;M
z?bJUZ7#kiex6T4K>+8B(#lqLOk3(-N9(xs01HSUzWIzEGetn2NSC?`|CVB(EQS+_M
z>BxKhYFtLMQ?arohvm}G4nakKwE&>qjsL?UfZLBNzW(nDWGB+xodt%Ea+*fJ_sg??
z8yW3&v`+u^=?2%x;v{**V`m=jqG}78KQ9;?oV<&$_GhE%51qTe3%UH*5lUvI<qzEJ
ztGZjFL|S!CX}*iKmZKYI@8X8cs|=gt5k*wrff7sb7-&oX<w*v&m?@?GKgXCs<_|ri
zP{}Pizv*|to}&F{^8`MZTg?TZm@J(Q>`7dvOv5!mSM3@M_aP9-V4HYyzm)1-K>Fve
zU)00ZmdabePjGExV+{Bl6=!F+XN3UVCp9WGZ(N?te1u=Zta!Jqg0k)9t)DWUuMigV
zZI;`P=R_Bc>p9+$&ENz3gWJK#=cvo|{AicdY9@IkcjRGO6*KPFAW#xjVy^%9KIa+D
zUO!vK@csI^jt$W(&QP8AhKfL8K!={JEx-v{*ZjTFppgv11^dm=83TjtM7W@^^C@2_
zG+$xLa!I5O*L4`7_m-k&{(G&}COB!{oz{2A4HE*J((c=@rQ$Iv<h4sX?`!=N-in%$
ze||n4HvhIdMkPUziUMDMxyeHyO`9SOBC8&f+kY7q{+~AFBlybE)1x0_=Tp=Z#BH)r
z`m2%e@r0G&G|t~*^zIH~B@G0(Jk;ZfMMPwlxWx;p!vFf1$Q6hj<VG`{qXDbhJMEC{
z(B!1Mm&6Scv{sl<@wHs`rqfW?<WysEO_e1Wta3xhBTqJE8*^in=il`}x}7%si-M9B
zs0a9ZN1W^)4#V8&dIS_jT&F}wi9$3VK?XX$F|cDU?(bF2IDPWS><Qm8dGsk|;O@Jb
zg-N2ii=Fk_85cdbWvycgc}`jp`Z7blnOPpSz#7}q0@u>2P~8plwp2O_-Jl#`i^>4D
zi0qwar&aNIvkL16+t>0WBx~DaQH}njP8*aLC!N=ERSq@rZkdg;$dS-(7PopgGkoLm
z$njqq^Y6qEoTrX(<)D9(9d7x%14S%DKmGfs*x&xWP_VyA3bPxs4oQ9kvjQ-Jk_5sN
z{V$&ZK(fo?y;}s{Pi3Go2a^XnU?Y%fpzj4B!?v%VOemBr8+-43`0WoXycGZaj1HO6
zo@mfSGTc>^y?Bc?<!-?g9>KqR^hz3dEJYOU0p1af5fDSi2~Cj%O9GNWpcoixHJGVi
zvgoi3Omk6z%wWYhzYq;Fgzs*rYK?d(+}y3-uX8>lC+`PFm)PD+X(KxKIo^FXA<zDi
zg!**4nN}{9#at6t<!?}RK!Cw^!2Ff2s&2mRE6@1BDpy5+C#&#exF{|2D_+C#waz4u
za`WQvyMg$>?QWol;dob2FaqopA6;Lb7OupJ#7Qdblh|Of)Pgyw7mzo83MwY}T$Ox8
zS_FvFqEA>#9_UIx1gZ@&d`&A3z~E5O$aO;7rH8~?y;P?h;**Eiv)lIQ{rOciCqxX`
zh2UshbcSK%4Lo{dbUFi?ii*JAZY12a9^f+Hfw^7j1ra{}F!0|c<QW72N>zP*{jH`R
zc8f)9B492c57d;OMU)bFU(}AN=2f)aG(l*9u7^Buc4BpH<;mO}E5!Qlu@Zaq_Vy|q
z$Ra((lAzP_JC|#*2aXh;uV_H$2ElcAJ_~T6lXwbg{@mh4G053!!WWxw3|+4xN2mkd
z?QwN<hWC)T8eBt1eE`wEC_>{6U2h07-_y~M$+dE}<D1?2me;nEWyP(Ii6tF4kad8s
z%7I!m#0f4V+s@YAr(kB!Z(ARDH<H)OOwcba&Ji70<kC%`i{0AN)gKNW$-nseIl1Bi
z?&Ciz-i`r3w@KEoR1(-RP_aOjtrospDAbnvJaZ8dF){4V9R-*HD6m;x@3d|Mr(FhE
zUMayaB??Y*sea~vaWpb{Ww}5Bs-<vO%)bD>I5K~JBGA4GME+Y?x3f|hhG3<^|I^bk
zi@7VSxx*ByfB%&F{l6D2_4`0{EkVKKwg*35w}#(~iG)>kG7r7`QBE?km|xVnylm#R
z5-Wf&1jJUm&~a@y0EasljiaHx*$0HaQJ~W~U|#Pgi3rtu!f5uQR&%TQ^)Uk%-NpZS
zEvy;{D`os@K2)7)axLAJPG|wwf~K2Wou}nMIycmfFTHokO^4k?wS2Z#;}bBKQw6~h
zg#^LV($cc@@(E(~qWznipJztv!6vt-Y!uD|+^N|6HPM$q_Eyw(y1GE0Ir(<R?tRWu
zXZY(4)-Qv2YNAqD;#b%O1^F4fS;yDjfcP}di<+2%2MqRDt~*xA6UtPD*6^7$7|MRX
z?)NmqfT30p5DqWJehv60hRPxtDAxcJI9|hqXE%HlMyXWvg$m?j8TGUrqrmO@18r+A
z7qNu&NZ->oaw38yX>za4obfV!UTwc3jb(izwD|UTi5mp~6=^n+^uY=wHkw9{=57Ya
zB}jp-w#lJQT?*yJ&1Iy^pwPxJkhCV3f-8U|?3Q1L<-yZeQvFTvbp?g)60ag``2Jae
zOeYOOu{|a2Udy*a1qsGvwc$GV&D>)Dw2GkHRuZ{Zm@oE)S8kMMwWGmL>lgR4?Iq@@
zG-?k{MsO@?1IeMeZ5bYDl>g7kSv%+JtuqmOcoxrH#MyDG%V0xRS60xSf#?Z0Xx1D3
zo2NJiO4eT52vVB=w6NX_?3(&bWCfQAsK)<vEONh730CM|bu|FL)MO`Rd3j-?lG=?T
zHSX@t!912$?724I^NBQQM+`TD+U*2T|HnW<Yk%=hK3ESfxBqm(Uk%JiEc$ZOZ;btD
zL~EaXk}enpxE}d7w#i_yZ`5_%;`II1BEy@=r$eEtB^ZQG{*7sRa3ORwlQy{V6gTT7
z=Ot!0AA*xCnsM#!X});D0W21;%ifKq>`zHi`RoO1TBD#mNi^P^t>=W!O$<eDFAGac
zcO`Uwg=s?aCqbV?u0#khVpD|2flLkgQwR#v@9d=%xy;~0ltG<s!G#(&E-)om0WEtc
z{&*XD#9l{i{9t{cqkz#-S$|6*aw4NqhJpye&AJ3iDGJ90K119$09Qr?)Npj9Cmnzi
zb6JABU>Zf9CRWn3j4@#yXZ)In*#;kMkAQ-L`djn(0$gK@MbJmo-b$cm(q@$Th)giX
zDx`gegDUQaKncY_l}lSwTb-Sp1JT_dA$Y3UfU2hf!Yy2IUzN;X83WUOhK4`!L=qpH
zS#4qX`V8I63&O9Y+)K>EGh6#UBy9d`G}{$DD*sCw#+<vPV3{EBg)i^suZ>c0&Fc+w
z^009n48x_ZKODzOEsT#V<HIds|9u%9-4%=cR){MFe43b2)G*lCX0eB_=_xXu2k8F|
zWoYoZ-yZ^~Hle%50dy+R9~1CBKLyYcY)+9Pz2Y$?t{gJDyAo<4VkBi*c`^6NO>$OY
zEnh--V?truKZ)O(!Djio`9DSmL?+KLCfcEKxnRAOwy4?<l(OC33_rQjAI6gL=fn)4
zD<k-Pgtq;_5e+r;oh#AH(}H@^1rp;#-ZyEDnW+_5IOlq?&=bY)Dud@Nwfxx%SB-h>
zoXimYun%{xvsBDqrtqy_(^}@w9y(awv&LX&j<WpxbVMw_3}DS$f@}s(wO~PE@Sc1v
z00e?9VAQvvg>Z1gX9N1Swlj5NkgQjN`$(N6Q=|$fzYE6*9N6t|x5}}k4<^O0AQgTs
zm8_23tOl=Y8h=)QN&hJrv_ht8X)agwZa|MwNjB!(>-2~DoFk<dh3~+~QQPe`r$*GS
z?qDbyk>+DcEt|ByU`*i%oie>8B6&>sKy_rv;`IzW!~@F}6jFX${2YvDqxq4V^yONG
zby0x)BYuq0=y|xVuJ~%Brn=guBV<U=bRXiX8o|%Vw>IPWN3qUHTb}l=_u2p`GM8bD
z_qBesu<#cv@rwnan`E+>P-N4%6M{T5y&nIc+XR;?(3W3o^9@nSXqB~vyx}X*n{O%^
zQoV3jiF}?}eKJxP+28H-C)~LI2{(jvA|%D#U%n?e(Zj=PJ(vW@(aYSErwy^Ui}qCV
zCMqZLh{ws8steEmww#b*_d}uKCgv5FTQx8APu#oDcZj<i>VOS2Xh%rJJup#!zk9mT
zndv(qKZC2AzJv8peV_>BX~&x*!Nz{)mbSn~NT4=;cm#_35ch3jAh<Ou*UKAfQYqus
zmT-C5JPZ8bl8u25P62C;H5S`o<(Li-qS%B^Jd~$d9xzBUz?TRE!KbnAquEr(kHoKz
zw!g(gh3DHUKVj;45ZF09pFP=uq!GADlPHMlH`~_SI3M$WGn4@{1T<3eQ>0Ybx{7X4
z20Wat`+U|3s~EZq(3NQihlfvZZ>~$&^<I4h#U6dR7|2+_exk0)1ZH+b7!<<e-sIC!
z2h<af-tfJeap#&JGc(rk1(W*_clAfu3d+iKSEbKNx4&dn=)pfIGf~4J<L5yOtm7KF
zWD3^f9VM4TuIg!SQTzyJrb5LyX`|!@Ajm(U(3ZXn4uh_)jJlnHYP71J5NS79>>@za
zz38bdiCH3BsqDnm+F1}V@!xs4Dv>(jvxiZ<1LgZvv|cDtA%3^AvKj@7Qrf~7S}%ZF
z*3qyG76Jy<*CddXo2KaHy=t3%`tf(c8Z&#GbaWS5GU?P)TWMmyd3#=Ola9C6RP8A@
z81A1A>r&0gyv1A~;wUowXO@SH8(32@m^+K8i?RK<oUPtL2j}~^JrM!X58i@$%O82l
zg|yal(5{)@#@hc{$p*5JrRybGnaqjDrt4`v`~TWqI{Vy<<j71?V{FOM;5vfc0^nYt
zV8v$xBlQTTalL8#{b4lHhjfs1aNCe)UTx;ZZw8@p>=~a{ntk=Pm0L-`AvaDJzs3Sp
z{cD_Pv1-w)1#^Iip=PyNR(z5O5W?ZCGNi)JoJ^Kr0^6^FZV@6m+i8(T&M2L}etZRM
zrGA4#*o*!}<jX5PFm)uGKO8;*b2n9DGZi;C!f^f~iL7|Rp!I=Q<|?Wuc*ro=Hc(nu
z6W}x_vyK9Ge7O=<VV9@>5bxVIu=li#jo;|kKt<V3C`86rB74i-D~$P`&+r|3Evkat
zdJ_n#zq`F`yVXilntHtiE|<cQyoX4LDJx1*_h!1HIhrC~v|YRDe7I;z%%z@)<A>b_
zUrPb#c*fy9Cm7$uo#v~O#|jJ4OX_pyt*#Dczc(5{DF8T@%dC^^2DK40lrxj${>W8~
z)z;ZOp+TBIpy(tu?oVqsm%sAB<!_=*wQ+2Q;6XAdddPkOKK?p;WI@zse|K>ov6Vtb
zUtS*3_uw;=3t&@t279DKZkkDX!=v!Z4*|`z)?o?OIC;tP_|!OI0q^@CZ7;@={p5q+
z;Vl6jIk~`|`0NR$k=sKX?dNP9NGIN#k=tlCTi=k%3N20k)?*67@LpM|iox?GO0V>9
z(B?x+Hjy}g6)XjxhBdMK_0qg7o<akh{U(e5trkmfZhcTtFe`A?`(WPcOMb1+1CT?F
ze_*G|(TTD6=l{os&kcr#wX#V*&la&Ns5B_}eab9UrIO#(BmIBmQuNEu0f3${&t?0a
z5QCBs9H!iB0sFdgOdSG{H9l}vk0KUzZ#UMWoaeKfNxlIS>TJinVm-KrWkZaQxV8gf
zMUTg|jsOH4>3;MveFWx^*lFf5zL8@$0iV;VLYuQ-KomMMt~f7wh{kQ;H}rh>1>bM(
zEPZd>csdJq%JlG3^^(@UY)1@Am!ZHumCh6fis?T<x>g9}MT!n?HY9Fy8l5*vuJKd_
z1O3jHkUq2lJa-4{bYLBT$phkPt8i>>b_f2zo@gK>_{fwx`4f#(R|L+Mo*z66b-)Sc
z4&_g1jsyQ%-Byn^;R+d+gL)ARq$3$Hs;uDICil+5ZwrAS+%TYyFT)xn$~Mzt7lEDC
zl@a==?CH1ENPc5HptLVtCwWgPALK-+6rQUuQL$r~_^S|CRODig{BB(>5(NoKDrdSP
z7rcoK6=+1(UL75|T!1%%vhw)3I%)SZ5`k$rxfAkz{bpIqta^@kqM!!dlnUDZWt}&F
z<|->XE%X$B`q+Whbc?n2Htw>*E&E2X5jv|##p`d-4xbKIgj#ZDcj5o-{NN%7s)~(0
z?dzSOqx~j1vJ<A|Hj+F2$<i+KWCR6XOPXi`L@nKczk)D!E1ciWz|sRUc+yR|$`29`
zSLw$KjI%||9rn2v_7(w&E)F?w2^wktHqjvTup__nczGiIjIGtmSXhtL7Z}0lxo<=A
zfDoitT|lw<Hy^9^GFVoI7(OF~P4|L;3=>lS-R<>4qdHPAxidQ8MNRA;IQCLJ)cYy|
zW$$`!RGqR6@$a#JZ#n(GoniJ>V8KTQhyj(nykD4QlAs2aGy~$;rQO=)1}Qh^8&ndB
zU+mki<;A>D*tIc&^^&@~)%NN3si`x=pri>oRaSx#Z?p4i%klN>V90g-&XlTaC+nnM
z1V2iN9vs74Kx=>xh3|^T7sYxu6&{6F>2BMgYmg&X7+S7McBzN*xRkCv5{s1sbOmug
zy#$)FlRv2wN`5T?`UL^XI>2Vj)H>;dp=62{?L3;L;X~2eoCtp4;g}7&6lwj=pA#8e
zxI`{siUF)1B-SJJj1w+fF7O*t@C6xm6)g|gr_o*oGJpGEryDn`n1Z1i`q6!t@d*sm
zGT;!rvV8<Jk?M3^JDAtO9oLDwACu5y4<N|NHH^P>0jCRvc*!roC9GKz-AVdDvL&ZA
z7sFN@h(76tYnMQBoCk%%7tiLJ^?6|(S@`d(wCv)gM-+}XxpWE6yO|%*CC6sF0aZ<9
zR(AGWfXN<)+7|%*D9CPX(lQW$6u`5K5s2Re8Vp;nkVv>mbgbNi<eI~}MMesgi4L<w
zld~31xvTJOb=qz&N@*SN2}zR<HZO-4o?Zef7sxS>&eb0$obTP_mG^j0=g1&UFfN{V
zhy;oI0wM04+%(~ZkM8!iAWt;8jFzluJ0S(ynU9bCsdMMNzTCH>dcK3^669ebaru+i
zmw2QOn7Ed36d`Smy*Cn3T5+d*j=B|iV%Z%<fKx!ANDXjhI5R5U88a~i<g?n3KOaB!
z-pt924Bzdh)y5h}dhS9W=44g|CnI{qV&lEk5O;}jHSz*jagIOpAMwv^lOmhRn0;d;
z;?6qJd^+zPpY5{T1$FF5o>3>dlJV-6lY^G{+C+0i*a~#~$&Ey%^-tjx4cRVo-VmAi
za8tOBYizaLi?iX}N6(A%cz3e9P>~llmn+eAh-ZT(aln<N;j<BrK_-8Dxo|u1q#f6a
z!aw`1tO(ga++r2QyhO-D!=f*s(O#OZUP;FF<lcuB?}<t&I7au~CI8EDF<4XRDt$_1
zcf~ZX)G0SO&w6Owu;8Kl0l;kgd?Xyc8z6;zYJlBJ%J~U;RCLKCgv7eNp4;X15MjZd
zmZtq4vI&pb%>@GuZe_8GFOLf^>cc6$B5p3WIVt$xJ&x)2*<v}-H<t?JY(47`S5GJ<
zwlno1Mz{0De9o23_aUdH_Yk9b56ip+ko|eAJdZ{|scbhlnEGw66%d%lY+u408#)CM
zbLvI%gR@Zpj%T|5l2C_x!h~dMBQWWk06U+wcF1GhRS;gD;>FSpji$`}8RqcHIoNNd
za+Jg2rq9lSP)?}NZO|GxI&<>2on&IMZSGKuk|g|!3Wom4KNC7!1Ob-9-*x2Yn1HcX
z)2Vv?%}P2?*H2uC)=N^l8_}97#F?wrca`P=&Aqh@rukAr92D1^0otbhaU&@yhna^+
zDzd6g(7e5J@Sn(cNz9wEuSp99qi-Nz4RmG1aCK#@WW31hSoFtZ*8T{BpStNs%=h@z
z04RUg3c-e7h7DBEsm9VB1|-!`BvBVbEt<)YPR;kfuX$|rJ8=xRg=a60kUsTO4sFCq
z+<07oR>N1?4n-nWMe0|wX4JHjt_X2uVV+P{0iz%ZA}_YxpvbMy*ef!&dsrkq^m^*O
z8>t*d{rQB9We!{E_k7mY(o#@m@=?T2rp%8(mY4BqX%VHj8YF9RUxkdjGYXVbH6sf{
zW->~By#-@?HzS$eAjeoyb~X|bce?$Yl5#jhw+!48bZ8xq%gb5rS~(ZlZPpUn$$Qoy
zke-iWxdic%?)KjACOY>E#<rTfO{OM!%7fO&2W@>iDWIp!gT9oi2V=i2h!~K4AiQWL
zM>_Ij)9gp}V)L<BNKHi>Z6b~q+a&c(<Ih#{&1Z2}*<NrPhABw5hW?$$@JN#mh#Ss=
z{6P&k{fiNpg$ja@&Okk!f&`q|mqziBRm_#X8zRgc5YP$vpU(_DpeBDhWR6QoP76GC
z-eMH`Ydc>!PqGUqqM%UEHiNAV|E!jT<_L$S@P#0*Ke!EAlN_s&tS5PgM3X2z7GYdX
zG}zk?y!U(2jsdoMm@Zyanwmh)OsAXF?hNc!76vlG&M0kiSck1{kz#FF7sawu3H#@i
zVwZC@{V*i58GhHQD^KJv-FfOA4G@%jF+S{!6~@yp1RB~{Z<<S0-@~EJltTECtfk6A
zFko<_NfewpRqcs3VXrRHm$F#62rm({aNdJKhOlGfa#Pa~D`~XoVq^@Jn;-9up-@ia
z!pUuRLnvKuIqQgiBwJ$1Sn#Dnbur-iLGtH*8ZmMfgpe}EGUkXIqPPq7U`YwsLbuW!
ztPWn74$9<_LBSRD!|Fk*{4a-|?57~*di_m$=Se2AwM)W@Q~IXOx7$1l#3k2n?-%0`
zCPkrASzhK-Wb3|Ajy~Y?zGXOSRJ%T9Dbdi1SN5%PIhR6}76(0xBO~<c(+)JwGZw4O
zGu9#^f0NLd(yect*)J=zgvX#hA$uco_g=w=q@{V1qdBp2g`6{dL88v~0agMoK=3Pk
zhfV6NDdAF9<jVCXr>>96q9`UNMqMfyc7}!N@FNvHosnOR<6dT9{{8^(P`a(pCtNv+
z7Y|oKaZ&E<fmOEp*vKzM9P^2lm{{`b*%`;zu&$**xq;+q=vm7_KU>Ncb)Q9NIF_!3
z$T1iEYJ5jFXJu;*O(MD{X^3jG3^Ckg>rTvK(ZK{^H>pqC86oH19&xofc6UakaX0D8
z?mOZ_J&d6TTUfaXbs(?MdRZ>Xj%rO!2wHQ#A#Q?fXwj5hs9U`fjK~rmZ}*G|h=?vt
z(4FuV%A~OAA9pxvh>yc9uti$@$;grvrWEymcUKg2;4_afdf&WKjK``V#KGs3s0<gU
zXyweU#8vnkFBBfYUtkTqn|=EpSK|XEMY>v28}p9CPJVnGGabq*D(G8w3d7yudbqm@
z{1_RHmDtuml(`r9_P{ngg+(Oi+-;_fvs#x8idW<Lh{=KZq8MN5@BfbtA4I(vM(3*I
zhI-xbpw^tdIu!n5|10ixe1+6vqSo(}9)f(=9{t%mJ-3O)3ZLm@;RQIXEJ!q2@Z?ri
zi4AzKg^!7Z*NMwWe=Xf=+Hi@eg>@W&BX4Fsfc9utvOa>}0sVVTVE_+L%_Ay-vIC&Q
zdlXR$>og$kR(CYY10h*TM?umo-T7XVR_&Ca7Ue`8441!@Y(L%>pi6?}hzPL9ymDnv
zJG)5V6weKIWeBH)>G6)EA?eO-AFY2+=UNi<Kjm;+Kg6LKh&FXwyyq=a(QJPqSh6=s
zE?U+#s#wf*#j{4_w)b_Un@j6~G+CP^!(;$DOt?+i(f3MIIMv5#$UizXlWPv3(ot2Y
z*4Z<#1(aF-P#*p`qa->EiZTh()<_K!bpGALeQRd<i^?xu;d>L6<+|ILia;oR!z&z5
zBwGZBXFvjC&?IGAuo6-G(gC(*sfVn{I{wYP9i`jNE@`S>Fu{b0|24mxU-R6_&EthO
z3K4f^>n{%dQXcL^_YL~i1R(hV49Jq4El0FzReU!WJ=?5i2!4WRP+|ham+b()&qVL@
z!cfC7<q7HW56d409-H?{r_i5lK$z)5c#C3NIx(BVc>-l#zf9a0dA`Nhv^Mz|*Ui$+
zs8DupS>NY4MYmXKg4#a>!|4`Wln>S#d<@$B*2nyoBbhE<S_LoNW3s4yEimU0oRj=1
zunaKf(ixnj{9ue+kF}Mqzc@9Gzt5pYGE+G_ReM+`?teiMN-BMQu@uQ~Lnl?#IF)N?
z`GAiVE+2E{fS6<?wfyl9S{==RL@Cb!>d6?c*|!J4`SXc(BF=7nww5Y_9}MB@O`fW!
zr(LoCt@E=;TI4+B*vy`V!<%<dkrb6A{Bl$7cKxm>K`Or^)2E4*4F1e(?Zrl=j0njP
z3Lx^vT6q3u&YEPsbD{4u(HFs!N}gWaAl3}+oKB+t`oai)`+%-k>w!-AE~JSXSlSBH
zN%Bgp!!?sG%meF8KjG<PPGFKEDv~o}VpsZXb?vP`=pd+ci>@^75_M(V9!%-C`KyVa
z)5n*_(_o8SvCG~7Wyv_ekKzoxBJ2kma-9}q(Z{iq6@esk%;oh1lew>HJi>>F2#GWq
zby84F#Wl0|aej89U)4BQ6FkKaAt|>t4>{?nl8t3DQ)#&LM`qKJmqvV0$>qmi#R$2^
zLG^`wGe$CItzJX;l~_*nkP9Pu|2nW{jb<`n6{Eu_{DdOro6_3S<@w5~#O47ThrnCj
z1hdL}l2%rTY4o9kR%Yhewpe>MWA8pF(K{?5bOC9>Qvw$GZ?{?QVR)@7bTku^kq*F>
z6(SeA;afz^04VRsmt5vo7@&!K%uGklLXhn~?-|a~un2od@?qZpNagGX4)9cg$NU}U
z5Pl#(=1+|;0{ij=gZLu$Jdcpy_C-R=j>D^i2^dnBl$>T8BGvc*jKn|3+=JL%WKw$l
zlKtk}IfF|&?E8>~J6Pe;CM)ilc`!7c;t2f9?b99QW!~3hgr-y9kgxXCp7#%q;C=WM
z?Pd05r`BH0qCy8Cxg`7E2cyMhXZ}wBVCv+<iX<mXUBw)#$tgU$lbz$?$onCBEAYA~
ze{e;2@U)cg0<X!g13Bz-pTTLPEuzoHXE|pPCktfo9~P@Y*>8#*-a(8o-K1qs@4wzR
zQvtnC-%?pN6ILuBG%QHpD7Rt%eSLK`eQmOa{GrR1tomSOhPIs5ryvBSwUso0cOepy
zVqNL8ucAfD6c2v2WHBjk6s1tb$Tq7GR%nf@U4_WX;BXR*XC(M#Axmw-4uVBgHts5z
z3+pVn!d)L)FUxPYAH>~!QO6_xtyGtD4ST)_r<7!h_}49-rFvOlQx~u+?I?|OU%_wY
zsJ<{=zLE<2)PK-+ee&Am7j>2%ilI$mS*h0o(6+Wsc~0@U=|F7RueSu|;pq{skzm{G
zZi7+zKzV2QOa}{hdPihIU+c6BhXniENOO<2r7H}XsrQEVMPyUcpup_0)e%TR)$A%=
zSfT23^dj)FyJ6^N7{4z1Rfo9(90p##d_c>)Z?VvNo!nizT$=L5vHP(fo%#<GLHReG
zN|O7zhH$Rb%oFMUUy+$5`lC-jM-kW}u;++^!9(I`3c4U%b57VEF&ppFtI>C3<7_o~
z_I!~>!s_JDKuP>qDP_Rp_WZjHTtecOFPaRIPWVIQpM_4Dm%rMEN(1nPjceZrSO|Cz
zk{^o~J$`R5-+uk=e1(TR1aPDV@+05%9Clz`gZvdgRS(XR-%ccNjD6j@G&?)Mmdmta
z9Bv)M1@z$9!mTKn9H0ZP(UhO)ls_)5W9`eG@}u}2UbF(@<JXJrlluIk^wLEQR0L{V
z1k(95j<Og0KpeeY8Hef;-%b#f$nR&kxlMb3GHo0cF*YiF;TyM%5exnHdw~eM?DO3Q
zAJ7TNzF4j;Vx}N4kHPb1>JBMXMyEPY^2vPnr?`@4WHdD-8W<M;gkw_7fgsuj9$c&c
z8Q@(VY2nKV_xZCWy}>ja#e5&mb973r^(o;hnMJEeZCBhU0AJ~}&8Iq9su9t`NfhZ5
zy7@Q|DW2@3R=6io=3rz=A?k0}Mv<Njc3aiHsx|jCKl#n%RZbtwapTuHLiid@`kW_$
zXPNvew{Uc@m7~6{mGt6f(mD&Rz1&uFH%_mkZnq}Ow*_wC)_JW4kV^)M(r-hFLj8Jz
z)^^v0uzSJO(ntxMge6@aaTHCl@#{?D<n!!<;>)+VJ*5CD1PN_Z<h!Bf&Gp?b?)98U
zhdq3FR?+JAUN6|b2<meg8+bK(L|JC|Au$^nNI*AeYIo|v?O?L+k>UCt0r`Yytr9vN
zOvu{Q?~B}KUvB~NZTv`u<Ar(WoCjHFm(O_G4%H{PUlS#NySyPv^I*oHg5%;1a75;5
zBsC!kuWPeu3&4nY5g7J!&j;EODCjv3G+kMm`cn!n!{D(5|1R`WC;}BO!+1Hq<bHfx
zK}z^uWm{>pUtHwrDW+I1GMOCgKX@<2NXcUZmZO{GjPdNJXKkSvEu{giS5MlgFkhaG
z_`O`E-ugXE901e}DDt%XC^BYwdF=&&(82~rmfC5lP6WK_4M=>O`7!Hc_2G#vgEt*e
zn-^}lx4HTop#KM#xeN~@Kwex5QO?=bodyAn&|@=S2hduQW;wc{BS&Dh_`4_eCxA%>
zeNtKh<DS7E3nm0)CMWhS=kE+!#0`$m&gd8?4IL<%>;=W<dsU-l!q8dV-TFPZt=;C$
zf@KIZaix+YyM(@EoP$=pXSZ`OSHM%*vU?Jy9q@M8L7*l7KxDJJa}vk_q|7LQ@t}$;
z`6^1GyP!hnQ{Lg!ip7c7kLxv(k{uF0XqCYP!W<8$7Il#(OJt#Ncv(6bQ9Y}9byW$}
z{6dl5M16l$7MPbwMp%@wcc9EU_Pl<fUy*ER;pY1Csdy=0py&sOCF!*@Pp%3<OxQu-
z*`AjPPsf6UPmrx1tle$e>P32J$)@`r^+$5i*zI+^MA7AKjXOnv2E^!e6v%(&=oXKb
zG6Q+{!rX|63w;foV_jJ(cpP;J1h!H5?jQtNxnx#~WYx`<dTd;;>DxD_>t6?7SXsFz
zDe856RI+kgpE&Gq&T84nVvNOkd_<KWeiM4o40;=|5+UDf9r3hFja6j7WQo*Ib(61Y
zhdE(Uy&n%8pICh;D8^~dgWhM(m5g5moKiX6mfncCzn@BJ2Ig=b++Ntb@30CraIpQ8
zJ864^S8~WeC)XTDM9*5Dyz>W+Esk0hDH^gab?GOiG!B)%4H}&*4Tg?mDCLdWX>o3_
ztcx^8g6#$glRO&+ZXRE?8Q7gLg@gvL+Ff5nQhc6xtTygXS2{mlrY*AU+4=raaidPI
zEXopjJa-=Y$2@2q%NMWc1H>S(oehCrmFz-8mV2ab*o9iApvKR874JiZrFY;|-wlb)
zMt@eYZG9;Q>$tQUmo>cvPo^SL14Qh+Pk+HopuO<-9J8_1V9@8h3e}%A8AN^6byiQ8
z`I-3XrTfXx>G8Vfj|XK_@frhN!iDbTYl~`0qQjHGml${q>Zwe$0kmkD8Gol2+ameO
z6NV(kkVEkWsTYb&xJBP;hEO&D>G=uuu<7$7bP-H6y!LaObBSApj*#YzpPcK*>FLB@
z^pd5r9dv7LxP@Z(3buh8E;eis#&JMSTVt^UHF^*4I$Mml_N`1fCdx$ck4M7rk#LBx
zd!QLX2dT8!s=n6B+Y4|h?;>550Ap0f$+`H)IAndknvSRp$t^%)EeqQFh!SQ^K@hz1
z-}{8*!i(on7RHTQotd>W{S_FK6r892{E_ZXWw};M<$v{ZVs6Xvqkk`P3U6-zRTbhV
zAM_Y7ev)7Cy`-o&!BLe?#qj7@njg%rLrCG##^SpGWOt#VhuuEtL5MSDR$nuYK*`Mg
zG`mrP9m((TYv552nf(;Q{r3oL9Lu@3xBd<OW-am7mw?S*Y}OqW;#qwgAi!*mS*#Pj
zSA>8p+_x~OE*_Keq;oEKBZG%|7y-{pTOUGvv=UvGXl|XTcSyL9*kI5rad85|qzwYz
zuVE|dAxSwIaV9t|vVrZe)-peHYwFGjYmZ?S>_#&ZD*js-Cb@wl3FYgT>J{p?(l^5g
z&t~g(@X7ExCz7sFee1y*Wqd#|b)P6nnKLexollq8^9eqn?>ke^`DlnZT0y}=E|_fy
zvLUYMuww&oU0PXuL5WzN+|6&_*6&eP&Ag%rKJ2Nv7b2Mq<4itjn~o-hiO4NL4mSwY
zcB{|QqP0VUE*GlId%FaKyxwGZ2aO@HAeUHkJ%R1!82EVh>tIksrbs=fW0|s5L_4CG
zI>KaA_wB){3cqfSt`EN>ti%u%s&{OlCtAObPB1L)L67iL(9m86%8#()W$+r~oBok=
z3uft$LM}q>`t15<WzuCKp{ANEkRJJ#s$Z};SayRp{1x+o9>=PT29Bd^qsTV!e6YHB
zuuGz3dsCOTjxel`xs?@cjGH)L65hIsTL@A}(q-ce|M@jISe1$)c^RhAdC?a`b<t%8
zN$m^8*F_<RF(=`Z);~f6W=~DE!xVf7AelBo%g@6L$mEr)T!i>wN21$zAERl!xeDY&
z4OaHnTJum+LbkWDJW@oQY7kotsv3LC4@(LY8z2w;v<OW2G6{piT3Jsti`<2}20S~}
zH+v(oDSTCo1;pDP;P=`m05$ORvx2csqp$VhvZq7Ahfy=Pb$iekXMf=TVfWHtp^~Cn
zQ&U$^>=TW;5QbNQ-dNs4EK**p82$XOuLWL<Fux}16x6SK_R2v|M+%&KmEE@6)(hr1
z>%q)-<Y7@Unq{9b{J&c`Fj*TVmjmi9IdjsKXRA8R#&IW*A4mr-fs#u{Q`u57ONK1Y
zeZ>AbwAAEp_iZChq&`K?v6+2^&Q^fqT^>wwJg8l5nVrvU_LmttK9O2PCzD2CIZy&O
zb_o$I#QHq?;wOWS&Pou^1+VRLKG!YC@ra<3`aoOjBbN;$chqVka$1cxH-`>^%qz*b
zcGbBh;!G#7=G$2Puh<QmBPzXx$zKLz^b4woCaipOO7lN{+SNrW$TLeq&Rmt}-s0tY
zUgy)>;JAWl`|X$T<8ZHh7)vtdxgt;4WBb{6u@0$S*G~WdG$S^zA|6G^9|g(%BgNHH
z`VAZ&+EPCIqagOo3Sc{in<>}i=Xdd=FqG~?$*n2lM6JzyCYSpZ$-4MJVE>{|EXD59
zPm;(_+(ej-fYwZaxy$O05*vSWZH*u{#^&W_MJ3@UDmGRAdioY_+m^4$bL3*$>jiwJ
zA*{xZ6SkeO`>#iVzF2Xyb|Hs*K%i<MqX=&RM!gUA2kjJ)WRV<w3d|2kF!G}(pBp5N
z`FH>=pOJ4SzU5(+5lBc^?FSj3h2H#}t&dkq2{KF1L>t}!fb<<=@o@~nmd-|U)w6y{
z1?crgL9iV}(2k9@P0ehR-__(u1;n#CzZN;LtD#gOniH;~VWStE+W_MYX}Mw5Mg(WJ
zi2Kx9tidH10PJ>+AgJsrH9h^-n?rpX-T4-@z6ukU^WssoTVfjZjKko>$bX~&47u)6
zJKX09Pf1K!qpKk&swC%kD2XAo!o_?JT{Zam)S_w042*BGr>3HUrV8Kd$5VIt1XU9c
zNw{LIJ`zpZf!Ih=8`M2x*+^4CAd+Pq7sKW4#ah=JTqx<nQ50q^)2@;?g1f0`nN&+~
z+d+<X=V9@?YS>Q}Ffry;^6?Rs@`c|na=?I(-I_=jr4P?n-_!35G3Ny4;^n#bEi=T0
zT{bI=y^v^;A<DEgLLT~c<M|(6><BZiO!6fuM4H8tOgja+nlH3cD=0RNRs9dX-a0A@
zwTl~-?rx+(QlybC>23y;E@h+}K{};Fx<sS~WCSFoJ4J8^K|oSMy6Zl8&UwG@u65V_
zA7L$F_Otgdi<C;0Ee+F_g^|bc$2KgO(jWLiN*H2fde~W66(u)Ed>F=qhQr}uQmb|w
z&L9gkx_y9nrB%XZ@o5M{Ol>u5@2v66<CNAyj3&uNWeL5dNuopS{*(|(huFZ!x}um$
z+O^539HO_DKRd6;qJwiHd@)p=876T)vM~p~57j`Hi9sg9c6Y7KG}@JE@na)#Jli<9
zn^x4p6-2=<d3aNwM-+Eo)6a7F%;X#%$))d26&Nx9Y#RM{j<_5%2HCUc6&K=;X-lZ3
zg&)QmS|0OV6S`bV!BmL=dmBT-t<*x@{glKJ{0o^3%&VqGLfLo2BTi-1T5P|%?we2r
zBqM1$5r{mdIhOo)G~ErtNA$>MBD(85Inf^+alJ}$VH443Y94CemxC#e+5Y`HOdO>8
z-ISyjnA_#(TF)T^Y*1dO05mwDZa2lzCM}g_8`?d{U(^UOiV>P_ydwE|Jv|+p@tu!c
z$K6XlVe<9uwKp&rr1%6!)iX{<)5cdAcH#dK^<}f0t0#t5GyBb5hz#a2H&f9JqD(u^
zIk?D#>Pc_j6<7>lQQdrXNsK(K<s{Tc4Xa@ncSCEdl=UmWxgm|hrg%j28OTS>{BpCq
zshOXAj26nxfD~0DkKY^ZXHW~Cv|abP@APFUH6E_p?Bz(*6BEdy8I8DFgq?zDm-4lf
z1>)!w7euIl;00>bQx`g;hz6&_Z0wTw4hMSe-Hy>M&G_Rej6UaWee;MkdeRRKe?CA+
zNJy+dl<i?eV>Yie8@JDOALpNamk&W{&h1F@mlVf%dMTYF3x6dsX_$knisdp`FBaAb
zm!R_P=Li;TlJeO2dM`W=pitd9I!FrbG{|TL!_UczEC*9Lo-pvezoQ$KL0ib9I!pc(
z-lE$j8vLImGgM^mr>j3RO$niN{$I`I)kJ#~Zmhq&zo7qP{IQ~X)29)jv2LL5YklMs
z60|h6FhPupCSVEGJZ-=hakHxk3zpN7kbVWTrfj}=I>6|4jwVgNe-3!2PdJ-Cc$<RS
zb2?`w0Lc`^aj<5RWFEio)DV0}3-<9ZQtc6>TyF{t$jxkw7h#n%bMUbvVy%<YJEe;O
z(xxXftvww!apUtk#Kq|IGf{y+k**(=_SfCi?+#()D0LF-;p{_slFclQ)LS?b^k9!w
zR{esxB$F{S7K2;9D^eDX#Y9dLt8Xgvq0FSNoGTjpkP=fCO++hADm9s9KsW7MASnQd
z?^~8%-)5at6kgBcqUPnzAvg)xJ1lY(tX|u4Pz|YB#dYF_rC2D$FY;T(VS9=j1nO^s
zMB!raU<r$qF5An(;H3x8+((40G+kp8L+(E}f}!!ASd_;zB!$?x7E`yPSkM;keSB8T
z?>OHi^!<B>2|lN5Cn@9#X^RyWTHuwb+vSG^H7aopX+=r?{KJG-#F^bOKxm)Ku*F(P
z>rL6aP*nU?#^DDHxe^l<>7qA)Sb5dyzx001MrBkYJei2pId!QK`S_7Iau#-||DV=N
zr+uQ;%`pcE#JQw<4;PK9ied_7%k}$Bxh{ll`LZ_K;hE}zh9gb=E~eKQZpjsx6w9ft
ztyTZ-EWaaVM8+bJiVk&~M{m4yyo#H<-^EJKKKObfA~pWu-!HnG7|}8>;~olnc_9tv
z^t9N#;(xU%>Q_{9OKU1J_WVcKQIDWP`_5(cpa`)!8X22Oybm<en`ZWI81WKiG!-(C
zJxrbvI&u_>*SktmtFoLNiIo~C$OFS8JXFfzZfO0p+z^^#${#Sl(;L7c8&TK$g)KHu
zK9xZKMLxHp<Vwk9s{2^u?*K{cAOd5GYG8=7e)pxB1VL4;o%ZOK)N5Pxazzso=wdb6
zI;X7}2O<b^_`AfVI#u)4^gyG+8*<L<>V#lyHuWptbK8)V=I=0@bkD7+%x>(keQ`xK
ziNc`J+`Y;{q8Lc%o=ZHA1~^f({3<!NA{)M?47myNEWRD4?^t|zKTe#j=q0ZRdcGYQ
zg96z>7K9nt)J<2D76A8L21=Z(@O?1XZC0m)eb4rmj`APPC7@&}G!ec_=$i2atBIyc
zgIQ$gqe&&1tl-0+AmQZs>Y#yK5?1_nI}Rrit+<EaVW9hB_7S=`mZFtm9p_QH<&((h
z8#+!rz2Az4Mn_%H`{rsx={)su0KusL%A@^@kH(=8+p^qdrZ<sd+=()wyod+1P%aDA
z#UUw;H#8jkYsC{xu805RVbRO}^al4hyf5EpZaYx>=ZneGB-5{@t>}HL)6)<g#mP3~
z3A?a!Ng;IP$!g0dWz>_nA{ya5-_*sDlu;TMOh<EPWVM0RjXj3M!Vo#G$<kY69!Dyq
zlqmYw6PK&h*Z8k*WEclE<5)!k%Dm9h9t7U*PQp`)^mvp@z~_}WWv1Z;^LLnzz)DV(
z>l@!El-`ltPwO!cvNE+P*<v6?As+xHr}RQZz2Zg`Xd|Q<)orcZQ7xB%)qvES*X$<m
z)Elz@GMR}GdRJv$&vK*5`N+$)a_UOe;D9sl0fR~ye;E^sTgdPN$IrYkuS(T_^D~K-
znss+~v#4Dxc7%RNjxwbrWR~e1M(?2*by_nQ_<eEz3RNqO0Wtg9Dkwq9Lg{s)+>@Cg
zBouW6oG>(`dt18VDKSDVe@MQ+JtXND*BmtC#49x_$Y)m*Ns>XsX=v-yvHb2j3eMh$
z=m`gtqhv^^gp5O{J2$Bc%2qHCH+)jn@Qr`)gC75u;YS#B>wLzz$PZ7z$Al27O^`TN
zDGd4&3KQ+WcopYfl0j{5Q8Fo@SCd(RSEcJ!3E#De(IT8|EbILX4-SA~1BV|IGjovl
z*7901@Wy|0qC_iZ3Qs%KZyVkh)=7@(qXc>3yjq?nxVp}9XQY3#Ei)ZXT>1$#IjH!v
zspAcp7L#pK!m!Ey9De6EEj<qEpRU^qXz5OCE98<O+Fj?622o^^`M-8ioP0}G^8Q&X
zQj*0sME?DBE^;Fm=HZFD*}XFtCR>RMUlE(&IQ4DIE1IG^5JwRYpP2gJmEt1Aw1;SD
znY^&`PTtI%kZMYMye9byeHUr&I{sttLUmE`LPG$(p=~S>xAsK8Wj6HxgrTSweRmA$
z#J7)aVe2maIw&sC)7dQz$cd+$?51Y(Kk@`0{>4EBh9aKWZ_T!XdB5WtMtQyvv77J<
zTsQR2GcHdTs(HNP)_|$(ejY!60mcdy3Q~QYJPS{S+_g&v-&gE5^1(*4ef;XnKj0$n
z)MDsNbU6c$!s{pN%a1*IzMQ>d;E>Ny|Fr0`b^*$6hPOBjVCTnE=D!tx21}<bala^m
zSQA4<9#>f5@kff`(qO`en)$c8af|6}T`pkH%089ex*C3$p<yHOUA5?;21=9q?o&NG
z{fDbQB~lo4A2C5Xu0V{pEE_RN**)=lewKqe^D%^&_rY&c%k@Az2FbYFncq|7K`T(W
zp#D-ep_(NgU7XH*Is^W8r-RRh_TfPLFDHr4DaJtDzC@1w5KI%jjuB>3E^K~RjEH@J
zw|j<}o;@)x(=853)%I7HZ+t7G3>Qe8_({hv?_oLT?VkF%7sK+`-cr|x<%4l%rLyPn
z6&CF7vR((;X~sE+hC6NJ)`K7J7Ux4e6T>&&sc=q{_fd}1&s)!cfK?AL9wSV@=2Y`|
zo#lOyh3JPrUN3BD^~6wU2;X|_%%*`FGg8ZPu?AXXAFu9DhmC$WL4pTe1k%%JcnuM>
zhlQw#qBXa)t=U$1{bKp>q8hh^ERk_zyumD@9QeNaV6uW!4Q8j^BSYsm@-Oajsuruh
zB_5Mr$v3gW!wd&TzQ=DwK|Fb(qMezG2**(QQ*D=E%Nu}#@hq9B;$jmU*5#*iB9xOC
zNbD|`eUU#ov4?i1rcH0i<ImMZMhn4g!dFL<ZPm&+NQy%oaZWo_$*j-aSd&;M`$O=;
zEE1(G1&kTjF1Av*y4x*pK)Do!rHHuS;>{&iLm0no_f>dwhT?8_*26vDEiGA2g#XKk
z`R2zh!l<HN>s6+9w@R1mBiWG!ij-C7WKiv9bvaAiO-T-+)gvCg!~I#efO{`~aGMgM
z`kidQK#xx4y~o6uns3qH=0web=<a=I4tHf;=P|S3H^<XbX6GLtI`jrkEEq&@Xn4-u
zDsGz)xn}C(3#&4XogGnL#(N=6l7%+j+s<muPH)=SnOecpKq~>XTIqxQLDc<6rcIc?
zq%RafpG4`S6nD4PL6v6j9df8q;$;he=_fFiMu!9Err~$m*;mOD{nHF{*_?rI^$X<?
ztpTrw(uTY@zl#%o{5kMUipAr<d!c|3^Z4a+PdzVcL8*_}e|-E;;^j^FZd;ZE0-pRI
z2t)w+_^DkH!RtW^;*i$)l#Shm(P)>#ym3a738ZhRVCnDT?D`J?F(ooncrgJkhEDm#
zNCUJo$cx(qH?Q%gNoh@tQ;<nUD}0d!01p>i=EQLd#ozN{)MH1A&(_m_Vy3cZXSSLz
zdB3Y0y4`D4^Dr{ifWf;I&F{XMAaWt0%TqP^gHNh$skrif1Iay;4e*6V{v9%CtEJV5
z<N+RSTiKy5d@V&wT8Op>sC|`3vg<KnFDdg$iJqXvr9wh|ruV5h2|69)7UZ24;Bnq`
zA7$4^2Pl^1;2*3WxL%&^`xs_EE93bu3jpTIjL0d>5p^&9RqZw~cyC>+WuqHMYqgZs
z?DeN~c4tnWoNzdfX9c2Cj*}>Z>RANIcEjtnVJZRpT{5Vc>n!pEFH*DK?v!3TGY-*A
zVaF_$mq<7J>fJsoNlM@5IEXZNwCx%|sOqrQ6Ai5HGzs4W%vHiR_sVgBGnbP&D!-k&
z*GQ<Jnbc`IoPMpztki&_YLZEoPIy=tYUiKz^4pH{+0AK34V3s;m*=T#MM=!kR-Tdj
z2nM8eRb$k?MZu-v06_c(?VSbXUQhM)WG>%cox5-6WQsqGyd6yAV$&13mzN}B%*Lfn
zZ5q^x>J>7U&SU;5FhP1B)qtJgJ90(>S+faoYI6&J?PF5*Fl$&6`|Ry&4cI=aRieRO
zY<04=Od^V}0L_oj0pdMww~1@kO1JSVD;q#Fw0nP((G4|}^c`j$=)LhIu0Tsr>8|RQ
zFWAZPZeS&as-_CkKU*C#;>CKj^uZF3t-lFI_kpHoX)YG(bS&e)7<-)HC(PjgXRZ+?
zPXXaSH(Cf0h+h{BM@*RqDx5Cd20bPH@d^OD#lVe{eJb<0`g37<3RSyE@a8<dfL+G9
zcqJFWtVg=3SqWUbxGmADfeja(?LkzBs<F+?r_~lI4^~9Mwf%@q54E8YjxZr@g+XQS
z9+N(6Ncm#h?ah6fhcouWt0X{={*jqh;ewZknk^a96(f{`HXGPWAE;fT>GQKWI}ldb
zV^gs@l5LbYF=u_T8v;S;p(e^c-2DoFBJ=e8Nrd-@C?QTkOiu<xxOGNKB<~AZMA$|G
zudDWpzWo}K1}-(6o-D^7#yZ@r5jBgt{AMI=*a^cWgQS&8!(-Z)x`zY^l0yB6&axpO
zQ1P8V$)XbE;80;#37*c-eRVkUxXirmHFDrMZmSv${g|dA;C0qo`P;KU^YyzG&pc*s
z+OBWz%L?JD3PHL;j;syFWoc!7?hDunTINJnJP*~tv$D4Cxxj>yZ<&-Syjjax&$YXT
z3x|unPe(r%No{;SZlx$C1z_*bAfFWLlcw6g(?CLjVe)d{NNKH5^Ym;%Y1Xzipi5!%
zCuhf>x2^y8Wf9zd)`p|~^7vtukNWM&L%i1)?=eX>sdlmwTQlu28_wBF)tUDVv|Zn=
z_B%W7&1z<YnLzjmdErJcNHxtne{R>&Bj;pivOCL#@B7l-#leLU9NWsPy8HI4&xgmq
z#M+J&3)I!^vTBEOAvJ+h3QZ-2ug7zKIu;$?jSGHsWG84eKuUk7CkJ0!0%7&G$PA=~
zx9d%fi?=d?`hzPi7oVMj{M+_A)s=I8aa60w^T{l8cm$d0beAK*!S&l-Uq_|nO4{Ss
zIZpJ(Vnp&JQoz=XVDJS48u=s*7^!Zn92&_K!0<_OGA%<OdS%HC>FwfeT^xvkE8hV^
zptmgm`JN(ZZA>3gK2E5^bOM;7$)Jmb>*1F5m)VeD<r&Zy9d;3Mk&|4M4+)(bhH-cN
z4AkDf_uB$I{~Rj(HZ-LX7Ta^a>$FT-@<GW+y<Ec|-!Wgd9Ym%-AeHT(#gHAH4qOas
zr}5T;*Vxf{aS}{@#n}2@G1M{c&>bTbM?(f>)yh)`hOBhVNa=F*%?*!{SPRkMP|s?9
zJg2+0^t#^J`a7DJ3tr!%O~6|$Jmcwcv$TQeb`niLvxtuApeKHj__Sdp{z*&>ODe<H
z$|mkd^=$yV;q(-VwR&_UZlP9NYZWKQ?x`d~RzzQpP;g-I3)@nbOu@pRGM&<txaGZ>
zCxuGc*JUX|>J(RtFTRM0O?6Yt<<&nQXJSI0l49LW<23$811&Rv?T_`b*R|aaY2A#!
z+kEd(5RY(y!{HoRUDq)}{I@D8t#(!;*PKJu2}5srN)%yI%P+l(KS!KQ(W}ZCc5=$e
z!aoRdAqwUx4rf0aD_f25!=-B1g51j>q4plEpzN6q0Xh8CYb2ztG^e+|`5cdcxWoYI
ztKS$4f|5chfsR|b3K-SbA?_<H=VTH7hs8mV31HP2<W+jAqP^+YphO7$2?W$MDNCF@
zjMsp7n$Z57yaPH@)g7Flc)4-nD3=KVOlePx#A-yC6_`MDecvB@<CQe|1pq+C0U?-t
ztdquRAgQyIB%j)9=bEcuI(*ME2OP50Bv9TP196$(_>ORpSj(%cY)n{DPI-Wagpayk
zV?Ty1r_nZLPQb3`@~DvXyNn{LZ?o^A9f%3@WYrjC%PubxlJvyu+ovOe?mfIO@tk+;
z+SJ!UzsX(u_)|V}<*MpO_kVehFPoKz<9{DgY9xD9Im!Fg(G0>Q9yL7Z`5$-|v_#LR
zxE%wu{`cflp#@{)k0~UyWDV7c5ef!XBR!lHm=$6S5E%K3CGyApv*H)|*^_dwj<0^j
zl$S&{=i!jC<!T~<(ml_UX%;ret#|~`L@XRsT4nH!8>|Swy1%X0DBeFJ*UB|HV$^O`
zCC_T84uW=gIe$?;eGC%tjS`Rk!2%XR*@p>Ja=O7qu?<7JfKU1%K%6>SqUL{^`|m3p
z)=aTkdHb>=*grn{ND2k^u9JH`iPkVCWP^BRJaG*p80FCTduzlK0?AZ$Qxul4XQJ*g
zoUrxz@<L9+moDYu0vUqzJ$>}+9$eUwlQKP7gxy(sB^>ruIXST&7(;;Ho<Pc|$jtBv
zi5B_;Ew^aNl~O}v+P2{NF`*gU&rvmToW!j3O;1a*xTzX50wLO`8FH7tfk!6+sjbnw
zQvMivhNQIf2sU`{i5as}Yyn7`jvmv-$=F}dSHz31t~m!G)!mx&p#Mj}%I238r50Jn
zHktwhArzk%u4w$8Q&uhgQL!RSms#h5A(ERREK}nA6)OUDr-i4vX7Fbq=@j?l5j|hs
z6j}(p`E_;hw&j<2_b|Mv(R^tw2A%=MW1V1RG{-d8PK~iQE*a@yFUSWCur6+_gln0%
zF~?QX_Mj{SZ)Ti+3~g1u>Y!&|f<(4}5a}4*vFypc?ksnE<L|@TXEF{A<bO1;!4*Sc
zLv0WPJSj?<kd$Hh<#O`1=Cv|kD>GdqrXBm>7$JExP@~v%Zs-0;)sU^G`>lBS-di=1
zxHt_eIC_kbAz2lO%3y>?<o>uXgbeS^FdlKsVzYC+GI}^-Q<gx>fn$@-uMU%CdShH!
zVq9bQmh!-33&GuDSjHmt@abT87V7}nPZ`+2*rp|WamqwPX1cMJnjxa|>7${7x7q%F
zO#7%~eJ5K;3NRf$CMLw5*dw-&%c;fk@Ln-CucIJ{z(%F)+8KQ}%l!(gXwH&(qW68E
zbs(!gXSN%c(8G!ofyofyF(TyoVkApQwovy}r>c@`P+lbehf^DO!El+{^+69h`~&(&
zp-Gcc)ilnZL{KwZt2pIpCBVl>&byCEmX))olF612hshfRiz*~`wE=^zA4pk#4w&b%
zL1q#D3*JfZRG@h_4+CX?fis_MUjSDm>=8Tdn@46X{f7hQusnC!Qj698(eB4ZHz2!n
z9goDY%%&Uev;XVPP@MoOsidHu$eq)p@x5V7X;ye^Y!(<jMq4>dIKPSjuR0EnF&h1<
zp~LYCZw9k3DUA~B8_)i|+S_A-e$*^o6MdY}g3doCk~bmvtX@ESzw3~sX03AU|2pj%
z3H7*f0z`pY<1JaFxC4Dj5eCl-t2nY$b}?_j9>h@n#~ws<K>uS8!kCp1MH<MCru084
zJ^=79466eFZVPzEh`Av;^dsV%kKdaTP3{ZAQhfoOxZJIqVu)Iil6muQ;=4QE2>;(#
zy7AhDvWnzKTBylJ@@t^eLS2`)1^>II<+=6xCSFP^u-~2a?~T%pfCd;OeQX+3*nWzb
z&~NoWM;&OdqVbyIw;z3&^JQ@uhAYl!tz=(eY+$pfZ?6O!=x!#n#&|AL+@<~g^k+c&
z#nJCBP4Db7-3MhKTywA<hhs;6grJb+0kc_*puiw~sar-jb(Q{m7J|n#EbukOiMmsu
z6%+RI@_s{hK}QNHphPKgnTUPuy}iNbcL(MHgGNp0qVJ)Xs-{eNCa)#(d|!O%Ag^7%
z|AESJ(jraNe3Q;V_j~z}kTAEH@Yw<2)OSF%fft9VMFqw;=>IFM=I~Qy(kd!w%zcsC
zNCa)Y&5IU_ZKm0eS}J-&VSG+a!4Nc}B;sW?48hCSmat_&wqy%PTno`4gdzl%dZS3|
z8TmogHft25Pm(m{D*;bBs4Yl%`{|wh=j|ECH!ho#96C~orF*+811b6?UbKVh()p-y
zu2HFbggHbP!a2>d#JEoP!|tNA!<l~rw{2+AhadK};uo!^=lYDvr`$yo%t5ZyH0ajD
zPz-AWx{@LWgo|{o2}^gDGz9HlIaRr)B@y(Yc6-^3L&q|~X^3k~f^_eo3tkv;j8N5&
z^VaK22>e4TZs&RWNA0{jf|9)FIIj$0JGuHz!ElF=<V_OV826kk@)|v#PC>}qb9)e7
zqhIf28X~9qo;6Bnxi|Z8Y>Go4W;_}r<S-J%|E$SF#iDAEBv!gAdrJT5C#!{sIM;WN
zIvhC%9u<okvWA#gPUv$W`cLUxQtOjz42%KOF!Lb5&NMazeTiNo!r(SrI<Qq`Q}0DZ
z5fdV6EC>85m$(o)qOT&{Z1mzU<;xfm41qkPP~HX3HyIFt@Mt0Q=K{uX^spZ;jv%)Q
zRY=74MH!@sl^vE1f8F}c5e#wt3PWxKZbR69BBYBmYEWi6g44<Y&-yhgNlhVtX&xD;
z27CV^T`OCO4W0h)Qf5-=NL(r_JAp_t*+Pag{b?C_BtAtd#CK3CSadcG`lzyiyog2O
zkStX#X@Kb7!judB7LDcF6@YhXf)Zmqxe=}^EmC3wTP*D624wQKxTM5TOIWm?1B117
z+<aSLAOLqtMQ<j%(8ow$>LV!(J?HyR{NQr;QvVe{2p~I+MNqw2cwgVU<{7^J&%NDP
z(+ZUd(EHyOK#g=^X&}pKIBSaar;biV(0Nu=0~Y=5=GRF8giQ2bQlPh@=Q&6Gqew-D
z3nEHME@^|ZE^*<eY3s*11$bOM4Pko=gBvNaZ67Yw!T0UQ{5u#7C;vAX%y_s0ZtP!A
zg*ckI2m6_McM8R}mi<dnoM1LoK49`SUOnkr`mPK^8n>RVwj}oOQg@r}nQJvz`My=v
ztSORG#>=oSL)$f~4s07Az>ol^G;bM$%DF)FB(vvSm*yWA^l%k=Zu;FnW^Ye+ey~-Q
zNtcAj;ELOPsx+<7qn%CRLM+lv%GrH#i}F}vlBEkiWTMgGGDsHhv|c}z3-4q>xD<k>
zIvFlQn9KoHFX*saqD;@o09&>?;a>ae>z&6<zkBc7s%~4%hve>&KYF$yJ1DP>8iYLh
zCcuoeTuyO+q!#`}{Et?i#7qnM02U2~Y}Ri?Wp->YRqTaTs>@2KK;0(=915KYI}L^W
zI?kCzshqSWK^$b{R~8eC^$X;qnXK?$Wz^rnQRpFxBA-D62B%RK&H4cT*0WUTeQ_>C
zhBck-+CdOY*!=hx!AWPyipf$qx<Tx>;w2fn*+p$`Wx=3H<$L0ZT^xsQlsQ612cz3{
z4xQu1<e2y;?Jq$9Str#wKo19EgcGoWDpON97Bo2E=<k`fTW{vvrvps?u<-B;?9DXU
za!D>+`XU!Dx!u>7quV|y2pah2{|VwBEQ(&PqHwEp>n^B<>=mZk=R-GNO$@cs@t5cw
z%pQBMJp;4ow3TFyV0ULmpprNgwxcmw_+@CrIE=pybJoQO>3vWy1e{6s=nw6WU3Eu-
zfD3Rn*Z;y*93#ZgEa)^h{VY%ripU}7G21pafZ0&kgGD>W_Is=Pvl!c<w{H$mDHZ{K
zQ^*v09*9tXMcg+^dlhT(Ewz({m9^LXQx*fl|0-JarY|-=2;A{hkXP(#AaGgZ$LA02
zsp&v~cbg!65taA);)Q|46f**Ix$Wv(x0phV(089W3qeBw>m!pw8-%q}Ynca{3<a^B
ztDZK;J0Qwmj7o#2gLP7<E@HRbkZA;1$)0Gd9d%ZN;agLpZS>G26^oH~=x~=kB^{$R
zQ_SrQho?oR3-Vht?iPfcVClwr#+?(Lx+)Nx^9CZ-y|hiVv57`yTm{r5S$;vkSCPrk
zA(_874G>7Jojuuqyhpv~Xtr!MI``S}K1l=0z&nN|D~(Frb&dy!MbXf!3WG9-QR(t3
z28U1|0ymndv2h!-FDUz)x$1(JMMLlYeyFMpQRnjTo|2&JB&0vwMR_TcC3#HqWn7>s
z&s(@>GX4K>c4U~il)34%o}+(E-ol?~O-x&s2*G>Fiif->042z2z|xA%saZ?2?}V}o
zsl9v6Et&QPcF)TUnZ2z>^d9FI2&n;!gSxUO?;XC6E%y(;&+$KezjQ)`E8wU?9Bwy%
zfOFIxm)@ts^JV|*Ux1$)=_AoUXy4#YWLHHJ1ZBP8uDSa8BYIj<x6}a^Tg-UbCF&?_
zmu;h0oz|y6kx<a56)7}NOmD2k_ADPSEN%O*hRcp4hauUz&;ZaQ_JZ;tYCgG=1#`O`
zw=+JiJ4n7$2d0w=5{TlQT7TQsw7N7nerv!oD1R6&#K1uI=oth}%S)oiwm(gv#P^GA
zD6Tq;PvTO-2=*K3jJlTbnk{%tYWuq<-jImkT2Jl#aB3bAZVbmE*UhK6luzZs<lAVT
zr}tAPGxdKB;J)PJK9C@%3;)&AeYOS6;GgE<OA_p)jtw5>0mo#Ld+jKlg_PIR<-~+e
z3!NGFexr_NcAh!DDuUi02Rc#fr-88`mYvf*n_fRdIYAC=(wvp#RaRvB1t31bJf2fh
zt&kt}8>yE>8x8AZz2Qf4;~`=q$*a9CL9cDd)GrG<Cj=~$3Vl!^DY0K>!xs=s-E);W
zjs`c%q@1EIqygz^_0=^2f*DMRvyA!TJn4Z!+e%a?^gv<eGv{acR(LVNuHmmMaLY+t
zk_l`n`&=NSX+yhbD{C9Kz`Y&wS^qZ(QizlbP!7->TD}RmEVJR*jS|x4_u2eiGJJxO
zxlilmqp3v#rX@`0t*#Z`<d=f-58I4+x8r?Z_+?=nriu~-xzR#T@6?b?L&dPBsjz6N
zjtEbjNqP&-7$NNo5(nbmYBy#-v=aJ1_F&Cl4)hQsfzI;7(GRf>cBc!OuK_Q%L@sIE
zu*er(8-{G~DRF?OWq>6Lu(p|x3oe3>UpA0a)CUjLs}u4X-hZJA<GVmXe9=U`av7q<
zTPlE{SjG9>_>@5oIk_e&)r@36Artkep#=LZwg;;_`Nn**k=5jUoH<0!iuolO+07@2
z7_a`cQvpZ8{ByEHB8vVl*h*RW?@RepRy4P;2ngTE6|MpIBVqn8p8%s8JB`CuCdTsp
z)6<r=vVhBzsx4tykzyd<EQ}SQ-#{KMn@g0f14<ed6$N_R90E&iq(EeahiKZ>+~)E>
zfPF)n!NNZvICb!mZ2QiPOZnoD$=`I+MDtbh^rzAa3;rIx|8M+Fh7kEwDlH{%J70Xm
zVX^P`LSAu)Rz85!WFYU>QtlLs{r>^>BZOx2{{i-)`G-2!<TKBnoVJQ5y&aR>rB<9D
z2CsK{$zMJn5O0$9zCYrrwVd}S%M|F5<7RD&PFXK$3%6AptU7%CPEO>TpZ>-7@8pwV
zGVPAsM0b2XJ^n{UD?FMFL%3MTcP7U7zz`CiOiW@7N5TK%v-38($>TS%j5hp3<s_Gl
za^qHYS%x)UM#v$^kgB%!b*D=C&L{kGj-3V`63wTn!llG*@EJIAiS(TT!xKiM?iH-t
z*K~Yhsh<N<6R`Y)Nw$g8xdg;zD<qh7VM}LA$@KDeM*Z}lS%iHT>yH*1M5qfVG$q?J
zeXEuMQR>#|N{M>E7@q_dk5ux4JveN3ghulKL^6s>9w=*Rnuv+L9%vr#8knu(xy;Q8
z07_`_y9JFiS;Pv%StPrwfYlRZltj5#_#r(Cd-7AJ0f5b)idLB1OI)6X)+`;Y4!LFx
zS2H)dVJfuU+I?d|M(S5f>CQUNdA~$CxD0bmVTCnRKeQ2M$H7z@`~|`wvS1I%$|OO@
z@H?|<_-m})45{--dfoTKD91>(^;uel+TpIdb{hiw`+X1vIkDy^xuv!hhIcVP_0QKN
z6FVdDX;lp>429NN)1;9tf1-;zfR)-V=eY<`H?<*7(g!<(o9<%*x<Je#xzeIoiZhDS
z*;*$J2K4v!m3DFlJ7S)(X4^$lun06JJ@zNfj&lvcq34%@N?wpWKI;fa{b3}-@9be+
zS7~(UXbs85s%6Q#4rEYJSL|rP@>EEO6P2T~jRg_Ayg_{X(O0T5A1gMAy4Ql9_5P5)
zzo1RO_^(W&7h;-A-Zf0-#;lqsWyP@g&_;>rMRju(Lg*YRH2zSaDX6q6MGSk5SI;p{
zAh@Q{Etz0cpg125zD8mQBU&rsnSMcc=py393KoYNJG~!Xjw7(SsF!vO=_Ft}xK98%
zQMgr_8tEZ9<`Td^*#h56+V3sO!n_W$e5EI`lyR))JCTwCO>fMqfm&O8oT-TbEhz&i
zCFE9gSV9S*dy=xmQZ_@rDlj-F;(c*BWq%6;d&t9g7x}j@2D&L(cYdAI$NP~FVy(&_
zU-g`p187RN92xzXa+p?IBAhXxv;~5<@BnTX9$Xr-PJ_EpLafE2TQ&EZ=><r;!wad!
zw7SF?)0N=Sr)Bmv)GyVREnUJyqcwA0y3FUBlQ#n`M3!!w?nfYDbasuFn*zk|{`xMS
z`Mg6R{U1*5*$8HX<bCYPfXGeZku~fwu6g1Y9Y=fq-OotAJ^nuzJsHE-cY4BpuR@yX
z$IE-xe`h`=0WY%ZW$4|<f8~E4e<m~E=l}3^78{SxOJ7_PnrG21@@NExofdNxY4KZj
zmWTem)-jR&Mh`w03g-R%Ri^hw3f078g{PZjzKxJ4&o|Gk$%q8{?&GmX^k1+Z>0gF^
z?zKn@b+pj*65Z&xaBMRE<__@bC`A{4Yu@BB(6Z)%eEbHBWD#}r<2<es^W#2#k;Be1
zqjQ!oN8i+}PR&b>m>a9Sb_`!Ir!@a*eN+9)n?2OsD2V@F-?dBF^`jd(I4ed)f3^Mh
z*}5%E%@<DS$zd)hwIZ)7_~}6MKsfd#oItZrzmJdU8D2N_7#Y;5MW=J{?6GxEmP-yw
z=nuxZG3{#~po#1GvwwbY(4>9Qn020SEtCo(^B~*AueYb+HUl9H!IufQr>+8zCJ7b`
zwgu^AOW7fbmxQNDTh7W=E=a#j@}&aKS`%7r*%X#2`$fmHA19cARc;NXPl2$bVEUI`
z2#iHPNT26~5B>qp3`fVmyo1!vyJm>XX<QP+zG<?D*04kw@ruZ)*=DZ5QQ>xyT!^z0
z#tXNrblBO1r0-$@SBEMOSOf&-LH3Po;hglHFaL-}+`!~e=@qU&l;I1I;{8bq{UhhQ
zID+JywZjht)k;DvWOYDTQ0vw+7<5%g4)jkba-^i;1z(9R6el_xshfMpG6mQcy=wiR
z_pC5VIHv~tMDVTH6#l93aft`<3FUicwBx5?2Wi0|>+P}!qN1wmHev-RMUTL#OO)9K
z4Mx<RR*<`Un2CqRBof(mE}2^RQFa_l#JUPIvc*or9e9cKf#Oe8$W8KTy$WqKmJib7
zx>AwBLkFQe^LRQ*hY`dHjfl+<Yv5@R)5Kt!M1-zI2t>fCCHj^nuV%L`{|B?&<!<1u
z8d&|xjMG5{ZIoAsSo_FV@)(xTLqLOubOl!LfTY{Ox7=I>ZcK=vGh@Eb?@DoIKf{Zx
z@JOHyWVyf(+_H}@o2x#9ZHNY)E=}SV)er9=+6ng3u|5VDPy8kUF%KbhrTeyLgNKP3
zVY6=khR<{baC)Uu#zVinq^x`C)hRdHWmhc1XQu#VGxx6Hr(QQTQ9tFac4`j4Z49&p
zXsnR=SR-{P<ab7Gy0A!C8+vPgq85%d#qZ2D3S0xVHG-i`3RDnt^#DOYVJuZV#JV=~
z$0@zeIq$8EO7E5C;0x*N;R#k*(P0)Q{D@sU2tg{h9)gB~iKxh_uz!gXC4cWKd<5ld
zKVUyd-ct`NdD%Tcv;ZA<$Q%?8XtGa<5N*m(91^3+npW_%)&|~6;m34v;Y99MPnWij
z><yE+zabK!vBFDjGz+-vaec|-&dphqdY&hdDIbc_mdmR88uA$DJ7;Fv8GaBvTF6Ir
z<R2ta<%Oh&#J}?1J$$5k`zNcKU8Nr3534dIC&hw4_n3O%6*=2UpcatwD9>EopzX%&
z<$}!PfBX|+iUn3cU+6^lvqbbqYc9p_pNd#nkY(I)+b_#^5W>TMA%sm0wc1oAFx?1`
zF>z_K(`V<B=j(EuC6eW9N^;!sik$3@R20AK(Sz9*92*>zSJUFC6jxI3vpM;WAlR$k
zwgkz1#glt_?!|hkIe~W{`xNTGG=d8mXx@4owPKt0gCOz)$%H|KKtK0x%V9jM&Si0b
zyg)V-O|)H;@kxhXgtwG=jTbkWRUfmhOI9RvvYtdCWMWnN!vx|@i?}#VV+-#JPRZBK
z7Lw0ypJ!VG#OM7FM>_YvauA5Hb|z8VFs<@}1mG9t{mq|$h#GYvQD95@j5oZ^HQJgk
zodt_!PxE1DZKAjq9Z}yN3V_>%nTDWoOVZLk_;rmCTBWRnG%jjl(ue%XXzU1=TQzmn
z_g60pAfZ+BfP#1faePpT7azz_D^qvx8rANsH#+5Pd0#FZHGabI639J%Q+XA1-;?#(
zMg&N6l|O6Q*oNJ+diMKUu>${mv-jgk&8xCbJ|g@WW#{b-K2!zHX%K#lIU4&mIht=*
zSSiE>`CV4|8?jE~aTPeabkpu!W6CE1Sw)MEq~17sC6F#oxax8-5k_WD4nKI)xqI|m
zPcEq^{*0LnngOxSB9LH`qjrkFWuVf?>W<Imq=*+-$z>!elLxSv#xPHC7=ASFjUl^v
z-r`rfDvou(?(m&OQC$3D&47FH8`?&)rq>|bR6Wa$8DU}XyRAz=4}o6GQcAp*kR=R}
zIdB`C%1@U%#72@Jh{W%yfy4*Czm#Tmv`i0RgMuJyfZ20JX|GtLQQP-WCt{xb`Jr}x
zC`+Y&^4$gVnU7*8mNX6t=D6b{{`1ezlb9H3vnp-#AE#plRAzp)r(9!X_br|r<Ws>2
zV@RY<3+Uuz)gE*cSQLJB@)DnEi{4@9*W0)HFe+#CU}SRTz9bWn6>xTb!=rRqe|wL$
z1ecPxU8u|g?H%{iE6oi=lw2{)gNkpikfpF6<X(qBx}fw0A&oWBwD1VpX&iEPhR$KD
z@6q5@|CtpgvfCY_0vP({EMtXfHoCwZsmtve8_P<Hynqwq!mEfYOB`cQ92Qw|9DNw0
zAuKuw;(ij#Qbf5!&jD8~9fXU;XGk@=a4_1KY_WhDhxhAq(lt>a+^PHGU`}*>VP3|o
z0oa@?RT??_(NG1YEVl1_Hoe+=e2rTIejxW@JLJO|b?@_+FJaV9R`YL;chj6+_fQ*?
z@I(s*n7<m?HrJBQJ?W;NWFQFCk~QBR!RCi#SKL%m<FlwIQr(^{Cj@=yY9h8~a$g(v
zpYLPYAhR-afm$clJ2kG_;pazJpY%-EQ)4cvj(e;QlIgd#>i+>EZ}H%l8PSKjtM1In
zOd5KkI`}+E8thhht<9An-!X=iV-V03-)A1fR!*7Hi4Gl1raPWQmioZ06u<FL0YMKY
zT4Czi`sdb!71}7HT^n5!0^0uw%a$kLD$-uBTGB6N&)XUGdK$M(8pv`Kp+3s7A#OyC
z>~newY{&=AqQ5EQ@m%&sX?I!vVitE2NO5GJD$VG_+js{G>M;We^ZC5-B>%vgp(o~Q
zqR37QARa9VgxiRWyLG=JZwdZ-M{k5hw?paI3-L{TP9IU+|F*?)E&Lm%hIv~FD3CK~
z`PkhG6dUw+PAU9vPG$cYlD$2(q?-yEIlFoI<#<J9iWg)^|1&A4|C>{<IWlP$a@6Oh
zhvPGB@gmj8tY&Lp$Z>QzQCMuh2VL6N3Aw#o8Sz0;6oV>;gwPw1j?$$vpw>gZLVz*f
z5sI-qDJU}B8GPgIb$}}@G8NLIotm4D8>mRsT}kJ%2VVSi%=)*{3W(g+&<UgbP;Y0W
zV~o#+bqr)?v`I$zqiP}v`%gzM@HR_)i~`c^YH62Pyd!dtNpH+J5kb73Aohf#q0Co|
z2uj-9?+n8B_1nw>&hSl!m5$G5%MB`qp>oG7jqAf>(pP(-8}$^Ap7DO1$-TNhm5_bc
zF3BzzEMJ8C9w$*um6V6^;P$Q=JVhRZA3w@|rpA7GJY%4nUVOeQnq*m#DhE;c9yxLd
zW(<dx=};|gtphK)AdBS&RT@o-*`xhBBCD3!GCmnHl+o7Jq4X5n<qW6M&4Fkkey&nO
z3kpy1@0{##Tw1ZzT&0@qW9h)97Rs^vkDUaEs5e&%BkA4&(o76JG*ohsuBt<YHXi$1
z217Tb?e^y@IZyVCYRtZ|1S#qPY;kW^HK|l9yiI&24D?8cTeZ1I)*1fcUWn#+)=<F_
zo*->Zwt4?^$c}2p%WMu<{{e`?MlG!l>PPDw0pd;{T(&@O`oVEZ8@#d)E6%rk)_q^y
zB`_@U@r6s(WhIOJjvf5f;sdvH2-vceaaSVqB<Y1G?z!T`I4$=jc<ZlX1CN7TPy&B!
z3V7qwFHUx#YO2q>2RWqqKG<f`z%Fhx#*0wCX}<_}$7MwX4QCL8zk&z3BhGxEj>j7h
zDKJ;46Z3bJoj~ma1(cFY*86LL?CUxlH8fOsZ|Pz312xlVp{UO<XLNx;g80P7is2_2
zd;9x`dAIhXpV$x#?{%YOL_DYxlb0S6{7T84_g&AGdw~|C{LNooaOXi@q1O@$NJMDg
zz?>z-x6sAYIJxyF0%R)jocNPQGsRVrz*5#$kRC5*I#Si*G=&O}_`&_vzt&^MO|}Zn
zc+i0%`N{l7-mB`o6P`yhEsD2Zx(ziS3P<@2{iMfMBm?>7%5Zl`9j*(%VR{Ni3o#WW
zL<=eO$9WKOetF*5<}`0&m{x8}-p2*TN5#p`Wx2nw`{r+O{gIXFmk|f{y~=lLT{WEH
z9(MXu6iNSws(KBn@ctIMN;xCEDjd5sm$OKfz7J$R23^hwyo=8ke)*nS2#Wf0kk@_9
zeDY7z2~(iI7D})-RsZbjP}<`@`U`;LjBF65=)m4SP$TF2$jnpxy0n!rj0v%rSsy$V
zVg-S0w%LuC<fZ4`<%^os930WyHv%Z|kC%bNg-Qvu4+y{S<e@Ct{rLyd347{9tNAa_
z^6l?lwd}O3J399k?VSdnn0{1=K*?Ugq*on-Me+eRS+63`fPNzP=@*e66Osk@$q3=C
zsgLym&Pm@|Hm5$0?Ha7uEkmA)eIej7YiXF{Wsp|{!4hwthvpIvC>-y+Q+Pw%D&1bq
z5tg1Eyd)W^TZM~;Ogg$FY=up>tNJD3bnQ%%AmVg$8dd4(5p&S`W^8WwGn|-p&b*IK
zW6w2UuHn2I&P_nqO{D}<v46=A5phX>`Sj*3;_GSy6XAw{1cB$DysJJ@=#sSmF{uWf
zXRqJDz67`&<H@ET|Gfr@|H4Ku{|6g+OJ}9{;TdW&oUCSx-8{tGEs7R8+1RM~t4MFB
z{r7e}Qd4=Avy_UxuD`#jE<(5u6_m<Tmgo>CxP>N*y1{jEUb{HFta&4<jNI_&bF*-i
zkZA^l`P6z$9&5WLzgx3qvyaaPyBNzq!X0+p#NJ527ZCE$^0Wxa>*>!=wp?isjpf3u
zn}TA!2Kt%<o!>oBOULfK*J)YuWmCvOmt0x6-{Rd3z*u?`<TNz{&pPC;eS+u9p^@4!
zS)}UwVqWx-$<aYmXZOdtvpU0b%uAmhA`1gOZeh*(D7qzc`Ez%THPi=0Tr+YJnhR~y
zBZQ;gHz!By1on^h_@cLb)`pFdkyxs)ua{)6s@<obEVQhWuO>e1>M|zdyoWv)qD%yX
zh4}9g$iJ=Sy}F0n=PDvjVd;tS{l*Z8tfd){qC?z*IMg{i9vv--)X5>;TOp$CEUIJ!
zf-e{LFagF?75!`3s8IJ7{-u>s&1A$)LbVE;Rt)`<rC}V5Hgxrk&V=Ib<uQ80VmU_d
zevD}Q`xPpyi}cs}*GYsq<ZZV6guVRAy!|+~-SHbUHTwy(YYi-gCX4UetsWry&ruA>
zJ&1IaxAU+-=EU1&j0KQ1d5~~0HTbC?IogHcabsEN8FX9v$q%a%FWZjuCkBkEs-ZD&
znESQw3WJj$sG?8Oi+@g_5-_|HN%$<&)|=5eXC;&tDAJuJ1u){*OK*&6F-PBZEO%vn
zmeP8K%1aUA;($vk(4|+R`K>&y;>GfZI8^(1P!NvI_7voRyqlFKg`>xx|HyzCmEPM;
zeI74BpR(gc7{v(EBuF&}8JuNDeu3_3i(|o?Drm%m&AG-Jh7i>BjvaxuI$(j052dWQ
zXMH8hVGH6$-|co$As<rYO|bbsN=9W^*XPnNm0{>wGJdz2uuoqvrJ1P7+qRgg^ZBJQ
z$kA>NQ<W3WGnYqPVOYYwT(>SEr|kBHi}oZ4;uz)7E0D%SFG~pL;u{RM&q!gUhc((P
zT@}i>ou99BzzV7rVU|FJ8D8WIgm&tYUSU#W@;RIaoH%K`fiHd0m3F?_4q~r>8#d!G
zTn~53dgj3PGgJ~5_chQ$fR+^MvI~L*SQ0<NJdxH8*G8AN+TJt;Eg~yrR>6i5A2guM
z554*EGd%cJYK`UOkdxKyI%0&9h@oLI)Y#oK3Lwyyjl4F6iKMi1XCW{Mq;Yevy*M&W
z)P)Npxvs0cCKo3%0~(0Rpo^ks)$iZ4z@^$V!mKb6IohbYs-FC*S3k-PxUvA3eyrQ?
zE}fauLFl7=<^5-t;~TOl1$6wSaShyrS-Pt+<&Eedqpiqo1^it|kVUMoTIWvD?aisH
zEDVcIjUoFox|s81Rs3B55Kvk2QIzRIhGOy1VG+9-O7S#sGybu{gHM)S=!Z4OsP1Js
zZ7ZjAiORf29HA)7-O(gHO3<4Iko+M3oUHE+GvY%j^BQU0P3SCxG6yegrrJzr`Cc|&
zEp%#>2!&|ij5*3R-`_z>Pe#3q6}5WNGwqn}PxhFKKQC^Wc49~O)xcEsBUIzvEU~E!
z-_n~%oYK&2H>N0|*U3U$=RGya3-><?WS%qkxxxmMXPGf8rsq1Hr7rBg)q9v5NxS|G
z<hc^~muA08-b(z}T$5;c2i2KKUC>D{&$~N4{Y5EFG2gY(OofBga@w{=ZDwwa%R&k}
zp7>^Fp`cJPE7|HC^mRFW>6)2TInDBVx7HW`CwEv)Bf0RWB<v;5@3F7KzHuvS@e;=D
z(SiftaNSdlqOG1r!sjcuazvB;d<)Z~NAZvSOEU{M({;E)*Ee%Q!%Zt9|9@urN2dKh
z@_*I0tjJm|N!fp1nedzDzKf;yh+)0jjh)yD+PV|*mBMcBApK|ifBAdX|MB-5C(ktf
zYCi%R?_0mX(Lj*ixRH-3(_h5E1r0PY{yn~5bG%rXcCiXyeye?oiOoctbVc#qz9RIS
z@y|3EpLqIglIay+@w?u@Y0Kd~irSz2G#AJ`ODTg|^BwD~+o*5oowsp4&w$ouT4m%`
zs%zI~S6Yocog~5vjg+-o{8LJ<6-L)HbXg12j2IzhHL;k0-eVmPdgAzSMD7{@iUgSZ
zF%4Tt=?<P>L*pvtlFs0K(*xtw{F0+flz8E&wIAwkkbE!SZ2L8WITkjL6<YbM(XF#X
z%ZxyMIKSd{n3_I*&cR!5%adoUP+ibvQO55onvEUjHW}F}OczEpy0#F&lXJVMjuEN)
zI3%m?S-Z86_)~jy+2Lvdd(9e@Kca?Y><m=}h-_geV*EzKPxv%bOQ|#9RbD|EQRX64
zF=SkHP0zlxsTA{UFduAodFbt~s-%17N-e%^Kk}BlK>Gen!`dw}mwV9BFB>rbJ84AS
zGRVq5;Pi0&pVT+q;!ah11<}5%fKw~I$1gwIcjG46zS>|mkap0>5+r(02Q@1km30Xo
zBpA+5jxClND!RhyDfx~L*9iwrL}a2wNuYQfu^4GjI8#809BJy9PJtw1)K!KOf5`j&
zXSx1njl^J^3j*B|1>5+W|0LX+Eej9<F?O%IB~a`gWl0UNFv4+LVN%xIXk-<s(;(?k
zLE8k2H_fG8fr-$G)5xym58(;wSOuj313zlf8vXH&0Mw0v{;p*jp*Aoo>u@hqj!ciP
zGO$udpWSyPzan8E><%LKZ_YGWdc^x6a0Dh=!Rr$rym)whdi4;_N60eB+lcA@9c-|j
z+IQMs7T$A4dg-UUw_`b?Z)ApD_!lqEND|y3E1OD8#y$pE%jgIxhQSch-!{M>qF19K
zL)s(8MRUl7=>H1lAt4mxyUVGtly3O5?ZuP}U3^HHm9GZRES%w08ke~l1Do1!E#ozd
z>#eHI55L4|lk(I03p>#e@@w>gN0?+BChBM^UM))z_mS7=3~pU!>AT?;N?&ZK-Y=sa
z$(AZ}==ngBjHdJ{1Gxsc%5R)DJtGNd?N-%Uc`%G!W;bhu0!2=Oza|?-)bU~VU7Kww
zH6E;d$Rk(kiXr3rMxIy1*90HGm+BB=V4X7<e$19?FC${~1bvG~p<G^K@=%b=G<fBc
z_pX@$hsc)Y^)_2Yc6}`5hAgu)@MnwfWDG6ucUloOE7MIIRd{conTGaCFjG0ni$RXK
zRI|PsG8?AvKb0{CpF6!Db~!lM{Cih-a@61#{nZKxBEJA7&^5ALi1*wlWh%4_m-e8&
zh&A|B#D@n>=~5fr^O-*n>bbcwsOQiiu5ViN_!}*8`jQj(r?{ncdad9~{jgp2!4?)o
z(95{vJpRX4c!>92_i0KSHhqMMpnId}3pf^8kRm^sS$aTC?C@#6p{cdfFe%7teQ~#N
zK8XHTW$gdzK6mk*cIg=srbi$n#qpK&ZRc!Nf09TPfChW<=>s(G6vl2ucM4<2lK&LO
zg!qiUM~M@HH}ez!Ut<Yif<~84(k>UF2O)DEMvY8Vxm}y{C`R;88j)X55BVCoi`3m5
zj!84VA|?fwc8ri0c_P|6kxV;gL-hP=wg@paQJ6-OP2(afhTL*pdP#vSG05~JEcX*V
zsMfltK+qgiqLu$HNA%1Ts|0L5Vm~XZKATqCOf5b{FX;f~HyNjLKzU|aVsA){&n^{M
zz39B8=hB3P)YEs4vsNRg#3e8`<ch4T!TvqCJz2cI4GdW}BQ3!O*oVwgU#q9rM}WPI
zUCOy@gwp$H_C=i-BT8riE^tIYQG(M$zO70hZK<>mQsLhgp?;q@^9jbX&J|ziwMkh>
zdGLlFAzKce_*{u5`s!*a7H|EBdC9e^=N3Fgv?KwfD#OvKx-#iB7Z|9}o>~mp-mGWP
z904F3Yc+Rq4>9)IDo!0u+^)B~8&eMFLXH0!Ea{wP^oNT{t(X%!n(GxN87m=U^|HT<
z)I_6X;cOyt=o7Iu152dEJ&d6<=~pMX9Yn~Z!vvC(xuHWh^s0apsh%dIwHDCj*HPUp
zS`JIN7xjq2TaH69L+$dTX>-J%hGqOh5x^%r#eeieIu-o%43%>AE4f?lj^<Z`84cFb
zH?A1&15cgt_@j~=h#NbI&^R>>!{I@kCI^ur*rXbre2Gcbz5V&pZMaUX4tc07C~jz8
zt#}IMvMdtK#bUH2wp4)`w^!t;3FqSyMA(=aALXb>pD8BLk(?Y#^Km56{XUQ4;-HK%
zbvT81TWLJ3mti#-Wl<IP>|BOz8WmU|@!gJP2~DIyLj6M6F|Ak-GMZ~jQIId&m=*hT
zT$3(2oy&fJC6%$#Lub3y+is_i4vswpDjy<J`eN$*I`W+4bDcbyxM9_ob!%uP*8s=4
zpYGp6PX8dMSq2?b-+8$<<bqz`B?6a94j3lwd)^%)Y`UZ{WRaF-s>X8rq&Yp0`uUy`
zM-2|^w_}yJH933h%NE5!n7uyUM0<Y}?(=oKx<1ya<E_e|Usb7hnx~;9msR92R8LBq
z+b3|ktz|`&UReHV_p=20x(~@XG#iV`7`Ex3TOvTNNf0(2f@tBfY}8P<jXR0e&{cxG
z%++y)$wf-fDK!l4wr-exKQm}ZW=)Cyy4w-6m?}{$EgU2X<=i`<=~ZC=)u1|2!AvSx
z_?bC&%XR;oLWmy?;&a)5l0+fTao7Kp%J=W2NuGbGHN#`<yhb)YQ}=P)U^-WR<TfR0
zZ_cx*BPVp9pVcEHgz5sV{|R9mMxP?r5@l;PK(+c<(q%?S7q$wevfq+(cT>0IS2H6P
zFAH=(W8eeELBqNI0}exyakK@e?aO->0euBzP~ctJyg%;W?e3|XqaWnO$(C2))7muh
z+vw7H`v0Nqt)r^!y02kHlopVZMvyM)E=dUisY7>5H_{yf(t>n%Nw;)&cXxvl65l@D
z&-47^ecwO6G0tEN7>A3)b?v?OTyw28=f|1sZU^XC<HNtpq>IQ}x&JDYt{jQmeJcI6
zx0&UhWr{I8HnJ@{w~Sn$I7*q2wzn4nkY6iS_#^oz_#g5werZ^^hIhqjsEl?surwv3
zH?vLQ=4WVWDYUW*v|Vujlk@hDG{N(<2&q03m@+gnrVMq`5j5buTIp^8%GT%vf7{A|
zj(R|rUfAxb!G|SVdXn<i{h2V-mnDt`<WnZrOM#hh2)2;06&Z_T+Y<!_!HoP3m@ya8
zY_n0DE7ylp0@Y>%P!?2UkRmj%F4?v=c<{&i5HkjsG`f)V*-&efOQxk?Rp&(k(H_;Y
znm~3}oxr=&7|$~d^Qd*=1Ghw898Zgt7_~U3g}^?svQ=(JEx&7h*uaWm&*3FL*v=pQ
zh9q3AzwooLIKSj}K%J>5(i^4a1S_V91#b0n?KwX+;P!cG_O5<oR3bJoNj-i=Z#=f}
z+VQN5AzW@usHM|#331mtm`ws*AC5<G$>Z;#t+DIRvaMhe*Va>_p_#vz$MMf?uJ<XY
zdIVp0cZTBNy!>NL;0!vNzG|M_9eM(_DW5SoNBTfJd$S3(6Ea2ufmL`_pqHnN1Wx>0
zp)^Sp)8~G{9(P)q+_(C+Q*?4`C-`%^bCuf8D(BvM+l~{@UZTGvnGM;t;sCRJau2~~
zuw@wZT^q&8R<T3B#Y{(4R=vVcgSHvrpN3-7Gh<1N+$TE|1b!-#j}6p^CrBDyhMGYz
z6fRtS|FGeG8-Pm8p!mWECVPi0Y3da991cELITt>51<YUOdEC9&)9=yGmW=J-6x4j1
zKZGY5{@}NMT9on|*ZZ5vC0%)!+_s!wnUpO5k2^b@HaL$@)IB_4>M^tWz>6qcH+dY`
zg4N%J&*$4__p!^n!Dfe|Lh|80u9LtuBxr-ve%LqL$g{kw{D>V%Nl;x6Wt(hr#EsRV
zPq991`%Est{)FKDmw3hq<Kprhre9~L=U*behmPjTrXb&`V~j;3zGRLBK1xVYQh60C
z_d|p{`EJj()N!wKMasoCeXGuZX&Ggi&$=wT;|YRRIXO8w!a&kgA4oAt>VzZV!p%AL
zEOGjXXcYJ<m=er7Lt-attd<SJ9pZG*lh>{z9N_1W?)c27=m-SL*dg&<U0vKc=C917
z{7(ct2U+P4Bg{$oz24%{2RQTrkFyOq0wrgMm|rjg*PtmzSugP6C6n$N8djCkQ(H-t
za`*R7^EK+ubc>^lu3a2nR&+DANE;gWsgqGm<}vFmUVuP*Hup|U^OZQKsgDM7MOL&f
zAD8_OkMvQ4P?0<OP{`|)Xfp9|%pYPaF5Ib33M$JjKz}RgOgdu!elge30(dIWNR;Id
z&i|_>YT<70_OF_#1Ru1Ix-U6Eeispf#62%ujA5d_AiDkJ=3^p>j(AaUXedSjTRL-x
z5sxIHihrk57+Bpwz9`R5n&4V+J})(LZCuL5%9@3bzAv&Vc-G9Ls`M7LSt3v$m#qJ7
zQS$zTVa{6K)vy%n2t*4a{ksNfR}z2yS0y!MY_nk(R8nJU{;kmREQkaBUkWWdBjh|y
ze{8g-N9=NwGd=jCG}`lifLu=g_k=k3_$2xyY1?#G^YZJv^eNm`JHk3EGmR5laR(*O
z@vlT)-EmzfnZN!|Gx))OHDn4m&C8Pf0m3U-FD<P#e138|UIvVB);sKX2#D&zf5mc7
z6@2AIM(eTD7T#|SFC!s=7H)ENif)Ew{b@P}y><*;P`jI9iDX)%(ODjiVFk8Of{fhO
zD3{=Ny{1CyAgyJ)Qn9-?ft&?v803YVT0Y6-p(jVX4KE4PfqP}}seIJB(RS?&*1MyV
z(I<iPG)!1K1Uvz1hdXuW_Gdo!6;z!zKUL3~TCb1R-OA^$tA6jv*XfCh;yN8Kd}ye)
z%B-4q`V>`u?qRC@xi4|{c;?G}RK%auNW-D_k1e7V#+jgbYP85xC9%04Fc9bsE%8I0
zHYgv06K~r+sT~FRe$az5Rwcb9-)6J(FEs=10Yt9c1nXzC?KVFl8&pF~E+?(HT(TSM
zZb&xG$~+x5!?v?4w@KRr)Mu(a0mfW|zHY>E5%YO+e*~ijgg#ytp*L%kB)xwwk?kI^
zPEkfnm2^<p-6iMIrT?(hR2kbVgK)4C&Yb?)+AOFyK2TWl8NwmvdIYy=s1|I<W{+51
z1uQq+Avbxi!)vS8PA2nux5x}#2IWoNYo^blVFZ`joI7j%i)Wp_o#DKTQM)>?`n^A%
zE`#5}_nR`M+aQC9Ldtr9Kulvn#qKG5fDArc)*0`8`Tm|3(L;md=iI9KY6YLsc%SXa
zP~MbFymJYLk3V1MT2$#LSw%&bCW=qEv*GBt%O@$xNpAGY;M5(b82Y?{7Y=kO`nw%T
zz#EZAG_=RXyYwf&&Wnkz4Wd^3B;rz%%?#1M=vI|w{64wCWc!bWz0NW}clY_ahdyti
zO9vjuLLCX?@N<8QEts$B5Y-fMX7w%fuXbOZzAy5wvEnFhy2cc0BrtCeQf{paFEEHZ
z9cKw+`v-Z|*0~U&*YyZ|pPtZ=BT-jw6DZ7gft~KN(<n^VD+*7Fm85#krFEZH`#pk*
zk*vk`mrdPbg!SKc#f3TE?f8hW%_@k{oD*pz`?*`A`@C}BFEKRygnw%Rh@cmyFP;*3
z>oz~vQWj1XUT)_NyPh-KvD$HZSBW~;Yvz2EK0MzkvvH5(`H>(0#7*d-Y%XQon3Eyc
z7%7EGnP-qnqc2vSmlEj&($j}1yVEV1Z^RK{aVr?+p@C@T7r=M7CN+`!@Z{%UGr`^S
z-}}G6*2~PgxWyDs3-V4gr=wcTN$|X85uNzJ>FIrfYLtv~f_wc&`*q60>lS66AqT!+
z4gsp`YpSWCN-p6V7rkfaEL_K(=Xw_DYj6Y9NtSuP{X+sX=NomB6^d*wT)IS`WPKE!
zX}cT%4baaos^r)rf`fIRk8<nfT7R>3Jek8W=QDIpp&sL$>bMGYu&u;p_Yu4<o@()Y
zP|WYhuWWqmNW)cPaG&^Rq}01V5G0D}f{RXhmgB9*D!<6xv(-uIzcWlP{JOl@t`u$!
zhxjf41N7F8zgXe8`scWLtc7fmh_694yHjd7lVU21axQYya4|`rxW!zarY<u3l?y06
z%o=%kBE~~-xcayQrr|4hV#1bEz74a@DuoV@v(!+S>4nis#vkWE8`=u~gVahWrbML7
zcWgD#;aM$G!vZ|k`O8bSY`s&OH$@=7dv~IxAtXD3k1);m5vI}n2c|`ipN+H0zz{0>
zNL`OL1y?%fc+3)f4ZSaV%!k{A{;QuAq^n7BBf5{}4J>c%L~UW2(32sP+W=mLqXqkl
z_oA-WyRNG`kd2<FH;sA-XXWR*6poyJfmW%L>L#~CcNz7bl||wxh?}`ZN(J)2oCG-D
za5`SRb|_Zzr(n;=I@xNP?vr6Ptx?2I9It;mN%JjmePz;(V$S9aS(C>+L1KOClc?$E
zOuROE)yBr{TjB?faF2yAkOl3T5_F~BXsIM$n+odFw^iTIBd!**Kik~YYB}_MYUahd
zG4Vle_rxmSO8qlk5zX1nw)d0bO>oH%N<x$)M(wDjQgu;IW-@I~$>kP%!aJ9p?-Ir`
zffTT3Pmj`P`XeaE!p;*K+o#BKFj0yX5{^%ow(~w25ieGmpbYZRVUe*rn2Gt%UK7;F
zE&Gdyk6VN)hss8%&<Q^?SKjEK;n3vHT&~w)qvww!IXbS*f4-@^ZEW!M%I;<NxF>cg
z)}^ZwgZQ0UN08mE&^_V3me7Oktj=F6Q`1+$a!1RkMA?=L&2t$z-U!oES@=@e=y!}P
z$G3_3YV`HvJG5M=#A(5?A^qWAG}fsuw3a;XWrj6*BTh2=at#^&R4lYZnD?KfR6n4J
zmeOh`dRTrXq#cc{r94evuD7_@t!^S+80N)c-L3;2HV@(DcJ7k`*uh_lL$R}n6DQH{
zP%o{|#^&f*9Rrh)Jhto7<kQ^<WdA()ZV!Dr#FY|Rzz@O6DRp5G<ehv-U9VdyT!ibm
zODNtSXS;U6V9b1IUu{{H=E7MR;QZPYw~n_ddP}MDIji@}TT?5Kc`-%$R3@C0Ci)M#
z`LhR`q>DWLn3=<Pia6U>zPH5pl;Jr?JftP`4M%Drz0tx;tA5j`ykm|>Eo)0no021e
z4uyTpc6Hx8Yc}^yZ@e?t?hE$uG2lI@Sj?(?y43?)HZ?|?-c#dg)l|S6D}Jv(fgTB+
z4kluvLRl<c8g8tSmYTJa3tuYrOz|`dqiX%H;e_7F((%%4`&e+W#~Zvtcss}wCFn#S
zciuA|+yChS4{Pum4#9Da`3{3|+oy>)!T_g(`eW?AuS+rQuvHmOK}_yLAyIp0>^bV0
zMfW-BmiWx7x>|4>iTc8i>IeAW5TZc!3(4Pz6F=7575~;GH6*c9q`b-s;p3=zchvjm
z!F&6(`ym##^qx(BH(%0l6D9Kw-){bRh5uVBLZG_KQYF9V$!)70EPJgZxhrcmaw@LQ
z$t;^E1J>LA4C%a({h0~Z&@Mae!=GZ@lPzCl3|+s?_x3;t3I-u`L4+h5Y3^9}J)W2m
zk<I(nw$2%mzY9kBZj;cBDm@R5k)F%b#idF~&1=$bdD^6gvfK6(;kD{x{&8vFvllo0
zs@?pQr0&4bL$g&xf>>!d{>N681r@`(@=!yAd<q;fR#kt;FAmx3aKv!j{EXoJ&fL2&
zkvvQ9#tWPXGJk;-0pE4Uk-tAS8{g%Bu&7UWe)B9}+CHxLnW3jT$(mOG@X4~;#OB#m
zs=(c}Rc=J}5&EU4i6cr?7oIm#XD6CSqojs5L@EWxW6ebf_v}F+1*dtYSym&lYnPpx
zuGc+p{0)Wvq(PGIw2e!1C!P@b!1bk8Q!vFr%z0aC@ty)-Nxa!Zlk`-<<@6_Xm4{Jy
z4H@`Ki}^Sp?oTLZNF^a8L|3)-PkB14nxDJ*U#%%wM4;`jJ>SPlA@|G4UGFUak!MKr
zwZyC1WZ_*NEwUAnA{XH0lrNjgA3n<mygT0yGIDxROkJRvwIbr=r3s>QzZwr=yA6+)
z!7wM?Wy?p}SE<{${HRN#oXsFkIFAH-<W57$H!DzM{%O30Ps%ryKR=kQgi5pd2ekPn
zpY$J<kf2K!XW~34%Lk9)z4!a)*fz^;!{V&yeA92z#bw3&6@8~=S9(x%o~qxo1L$_8
zPh4<Vb*VbNYh0bz!E7|JRV5Tju1-=_1L+SW5TjDpe*tBJE$u&}hQ0ri7XkxE!a!>5
zMME>R++jchn^Uc$C$s9Kktr2(ylPCfZK3JsOJ7iXH7t;K79><reTz*v)C6_3D}Gy^
z)A}tkrZ`~LepuINC`Gir-Z8xtZ(48Lz_ChH#4+Wt$`)vSjORR^G%$8uIJfyu*y!uI
zo-p=?s3UO{3>dk^iQYdLihmX@QtjwS>~on-&}y8$EPr-sYL*|wW*9Oqj#3~$RUut;
z?VLf!^Ptp`xOs@L;J+}IK$NZ4zXv}(_#4Cg-bn^a-g8xFXxB-zts~Jl3i9*M3yP_r
z<hxB-#D_8#K7wgolBvdLdMK2Aljx!iKZ4op5(J;kPn6?g=^0xc1clEa&B)*X@gIA0
z-$F7q5t2P{n=1EeEd&Ja*bhasy+apuKMf;I?G|RA0_k+=qF_<%HSw1l%ig{G))#??
zEp=Fhe^;cj&2Bs>rs$%FE@}(F{P*`(k*IKTq|p1+1}TodyVgvjH!2LO(uQzN1F3cC
zAPx&OVbzSw7U{y>R3DUSr7iQs+|3!?JJYI+tsl;hFdpoLn*@Emf8dUOj#+Mm$TQ*#
z@%=jbknSjNcteU_?@;b@K^E<Z*4^`6OE1bN6`~N2SK-^d_Yes2&i<1M=J#PS%<-nN
zM-?ADEhmj`%;HI>Skt(l+y1!o#GwsjO`;~;vD&KBAsBw1@z-{%hqxp07)5Vz*#iIF
zn>P_uSVtC{b!oE%0_9udH{Zpr#s55{^iPc|Quw#VF_%-YTYY4D(79LlTvL2fZX|M7
zz$^^AwX^8V@ZECYd7%pfRV-~oD=~+V!M@4Bb5v!j565S`FiDfQ#^h6<b=t(ymG9BA
z*DC0_ZGE`$I}*F~q{h?cVy}sYF(ka6Eq0xYkdy~};Hv#~xlI(3+kg^92pawrBiOcy
zMUH#yyh*1HbfYvKKVnX!<DRQcXGf(?H<)p`R^fGH`Jv9<9~O5c4iyhv!`b8>3VePu
z)$y)8hd|Px+LJaV;#n|T67568h3CSTpkOw9QEI^=mx@nZ{Ko$D`uER%eL^$;9t5u!
z_D*CQx%s15(Niw5DfsK@O@6q^H)!VU_9kX)5%1Q&qus3S4U%$I%5ne}aU1ol^uGI5
zG~_4fTSfE~xk?Hqb)QxXV%z*q;QQGdClzcIsS@+ImaFV@51AGXrcW@GbW&MH_g_-D
zy$KKMjiuqUXqLNtTE2D6j&dE>Au#L-(|x~{+MM}bUp%DN#Iu9(J~S0_P8r1JCeP>j
z!V3QFl*aJ7T$lh=e(aqCe{QR!GOwklg-cklz?rB19sAA9+o_2+Up*t)17&$VBWXTz
z1hMgI-*))%e$PXFaPi-c+33JyB>~Oi7G^+DUdG%m0rnH=!%`vpXna@WlMFO?;Wo~m
zB?TY{`fJ3qlpQw3^WMevxao4hjF{U#NHUI&y6dM4lRxFZCM^*Q(=#7AJ<wSg2(A5A
zQy^y$F`j;Tjq~Gi(o!MnT+?S)od_B@z42sRmh8py)m|7kfcYIWP_j8O9P7XX0ceVv
zv=)$*q)+<W(*S|_=KJpV39nI?d`)AImK{C)z?7Ww_K)UzVMt=}=W$T{Tk>!GTTHlg
z{jYhhG|2A2MLW&y%F*Xs*_%9l=|;OJ{mGz75mHbf3vH%G{-;S%<Uarymj_9HXyK}F
z^|R9?-^aj9P1~1g$B+8c;Qt}}FS&-iV7Bm+eS83-1z!m2n4Q-jNT;T1e>2V?IOLIU
zHr7VSn@m{FksI4V4W8*G;_qNAnf**NHBeuN!uI8eyl?iNVwJU!v>CYcRq7pVvqlE9
zy<4^J7cbNsJYfHw&#p)_Q>L~=lm%g_*G1_Y;E-deMH5~XL^sDiDY;jgMNsX!DAyY_
zRW6Dgx^Vg#)k^uBygVmOs7UBuJc1w{%n^<X97t=wLFE}g_Y7*GSba-7b=$vP^byo>
z9`rl$Uhjq69zQDwWlxRe7|D%33L4m4bIu~C3aTk&UtwOF_JH<exgsX89P-f$BZ9uT
z+1-z>OBbn<@go7nu>5ZUw&)n<MImg1)Uw#h4d|kOMn5~|v=|552jPUU9Y~{=8gDA1
zz<DZ^Z*9fJ_|uYxbe=xw7~jAFqw)8I*`AaqwYxnuj60sQB^Dp9Z&0ug1OJG(@UR%a
zNWR_CJWKBU*h(^W@Gh85Kx^8Mo9{2o=H37-SNn)X04lzi0J2MH&Z)CFUcmz#?NsHH
z@&FVU1idiPawA&>$25+{{m3pU^J?aDqg42ZFQKjnX{o~p4{hFB=;p|scuX6dtZV%J
zFE6wYxvAEM^x$r|pPvQW#e7!r(nBd!(d^%NeJ}g_fifY6DWMm)ti^>PFC{*hP3&RW
z{tS~Xm431i+6_6ZN@}`?(23^@0WILRlAo`S>oGon!vH`jTTHG8k_R}*;wGxHn@@2w
zQ3oE{1*bgG#RHMR8wt8vxK2E(3=-|v;d0+Ug|J0Ip4on-!lKuWTBLu`d&H!ygR7`s
zFqC(F%uca*70fnB1(<+Vw9{?o-lL;VTOyZ$SZyuijF3jM^Uhm?SLP5f|3j;t|L5Uc
z$BnrY4>$C|Hhuo*S!_PU&fFZsQ*med8Fpu)nKWp#2~YI=CJ+=J0{Qs^%N+B*_Hw<s
z9X8`Faz|or-9oEsqAXRs&@XAKsdxEk?n+e;9A93-QcuNfwb^Q9-tFIRQciU+mA`m#
z8+pU2T;wt;SA;us=<DdSN@SbY<$gC)cMjW`h$Y>sct!RgkF7Tde}p5)`iUvh^Wfq+
z=&J^Ym76XZq<l*z8v+`3FQnM+ty_>WG{DWgVL`q^(v#s3E)OWRxH<ho!)oc5_w94L
zpx&UgvU54V=L-ukNN+a!yAv_A68-WbTS_rJ)>a1$0b}K6^?qvkC6e|8JUX5HQzxHk
zm{^oH=)&E4K6zjI?rE=Nk?BAPTkC5So7X^iMh(l{?8g;Z4^k22JUbn7jx(KWDU_mc
zB~bZ5L@_myO5N)UUnoOnT}5)T8LvI|4Anl`jw2!y0i!wVXtCkTC-;BaTe_kpcSS(n
z3)OZeQG$jzBRY@hB|LMqR6^Td$4OUB3hFncAi4=rmoyyJ8|*pN5@BEnTXzimF~SQO
zi5KNV%1IY(`{{+=914Ijpu~?Uyr!NkUkgFELOST_`UF%q@`1XgeF(>zkZR<S<Ps|^
zyU7TNE3g?DoKUN=l<|C^=OB8=IF~e0qWR++|5%B9=8I$jPpG_z3<l-LEYN@%@|w?8
zgt9!xdbMo=^eNSEC7O4%AY#$Le~trvf55vm>-u~<beMhe<FwSrjznN#B@fDr`9R;t
zhsxvT!qoc?NHpyLh2ktEoKH1Rsc+yW{A0YodYq)W9cPt@+71liXH=Su`ivN#l)C=j
zvj@75@9$2mK(BTsFyoepLWWl_C88DJFcXpEXlo%A4H@x%24@2P1gr570dSN^h2I-r
zg{r6ktqQao7o)sP{A!?zCacJ#<a+a(2<D_I0JZu@(-c|omJCmBFq<WbX4e-e9xnek
z9#-;kQ-4iQJ_eQE;(}4V=2%JivRUF{DcV#g$xBXz%Gbzz^L9NTvN;#pw&U{Hw!=RD
zkL*T0T90;+lkK8nPnBGoZuS6Ez4a2?_y1twm;WWZkxnti=~c>T4yshRFy4vR=kRbw
ziWK4{Nf(1_hKdz*n6Ns|rSZY9!U?kz1RkV+OcmwO!=cI_Wpw6&dpdD=|2JRoO_5XV
zK5d~OohFTk^FVhCf|74L&%jCTqU#we&r~1r^U~Gi9XsP*(&O`GmV=#!F*#N0BtCql
zL8P^>1H|1RE8d0Y;cG7obuTF*P;tE(+YQArRq|QD!jgO*d%lr_sL^bNP0to}0*(ZE
zp`!fZ@Oo<o700M`n(fmyt_lTFibaV)Q@@kZfDQ%wB*I`_l<%D1_@dF{J(VQZ_RVD_
zjGt1?c5J;|XkueH%_5;Bu_RM|-h&wUe0@3y<JR}omCTYlT>aNq+Wp4xV{$wJ8jh{n
z1p1+uB)lXR5K~7OMpQV@NQQesf65P>F6)g5Ns08AYqO?LULHfa3NvZE=(lvlhP{cg
z6a(sl<&y<)n4Hw7u!D(bES<#=q4}%8%cAE<NZWpJF4EXrr_{ACf_^PhTM8#xOZK{^
zOp>NO@n$fZ8B)mgZ-e9St9zpV=}#t7vs7hLuYMU!5z(i1xd3z=&K^giVK`N980FCI
zNH+~-!)?x^7O@2%0+sx}y!)20bt2|1A8w5t-Ytz4vOH6&PI4|B9gSY*H+S^XDO_zY
zZ{9KC7_2gxUYw4{J2P-HZ2OS%MkxN`&EUP~y4}iBu%GxYzIUqeZJZe=X{}DDiYWqf
zy)G?^3#W&vj%oL8XDZ7UO!yDN0i?|L9|)Z2qc3!9iOL6_CwUSSuIenbr5!{MGN&*N
zF)itmyD$VHBqF_&)oJ$fhK=k@5>KPC9~Hpa*r+4n*kWu^-G)IU=-}0>e6UGDde`zx
zkb=7VyoE_O#N6DOh|6U)gVRb4@~Wf*?*17}A4S(I<?!`5Z^Z9<4kuMg2*JFJKM_%c
z{oDq>+~Qi)bOwc<#ckRC9TcanLm9Hxn85g56^Ag4$}6tJL{O6z)Q`nHN9@;TcT3mX
zh08$~RQp-%FY!k0&d;(L-nu;4n)1oG?$k7-Tj|~*_l&1Cr2^W-gex8|u?QjL;p)hJ
zhF#}yt8q5kEj_7rw%SF;Q5_=Ib52`DB%!MqH)J>2lW_!6Xu`}m-LZ)DdB_-*u9rF8
zf18JlD<0fA9Fd!rb8BiT_xJUceqyZI5qS_zF$EPcKN&q5H#T7`6!?`Epn%r#(HrB!
z#jSE8=tso*6!Ustqob|nrCpuw>&w><eR9KwU$N)}v`Z193cUqNz*yCD=U*L-t!Cy0
zcf;q6X+c^~&wb!!FgY1F(EUCsc~%yZe#QIsTZqhBi3ChO<E{4O(tH6>cC2REqmVym
zZ|7ghe*rUh#lMn&XP^I44?_wV$o*0XJJ&4a5dn`2fjfcq*3m=3`~eD~G(Zz5lyPTN
zoLD9fmAwO(*SV~gh?rRR`uyBUX8lVYVfj||yk%zUb(0RKa*j0aT#c1Br_i%DFUUKy
z399<_a2}Wvb%ECFhe&av^uO1O;vh1DCcIHFp}E^jnOaGKW8G}I$nCn9p9}vsgGs;Z
z`=#L2YaYj<Ypx^=H1esESo}4T+mUqP-G^Bo)F;#-7AUEM4Rp9~C0?N^`XH*HU1WGk
zd=eBCQ54L8eU484V+DZ<_XFvN6Q3ZTsJB#NQBiM$y)uCRvc`=6AW!MXVddk218LdI
z&%+0PEk~VuB_-TSzkeUtUnboAMJ65^4djz@L%jmpU<KQg30T-HK(npC@{6dk`wq1o
z=G41ka01c1f2ne)y1?psVe*Dld#fO1-j>Cm4d{Z(`9ZC;tY;a;eQqwsWp}HlwKDa)
zix^RDRA;vYM>9lCfzgK}DanuaJqqyx0`||Z-GOJBS;AC+mM+Qz{pXo7??hJ9j6^AE
z%+UpZc&lNeeW}8UXEP@>-s+a2Y1slZf9I1NW}1~@G@HLY*9~Lg^pMN>%}W<(Y*6g;
zd7aUvns!t>z}Lo0PzP-#Fy~daorj+z``!|&X7)PVq5$jjzwuvL*r}d9MT6IXxf)`=
zT=34V<JBE(fmdu{@xgY>z7xD1URIq*XJ1;`&^CUV`4qhKEsWabE3qUcH35S65DbY_
zPzHn*Bq@c3R>N=KLBN#B!Di!FhQqI7jSU(p3%mO_;mft%0$ZC-Ya9soXZ`X_$c_)8
zoeQ(*<}rk~aIi2zf<Rbn*X~UQsO*NkuDa#1=YRS;ui}je&3oxqD}AWqM>2;XHN2OX
z_Nx_9y<2P}^NNw%>*H}C$hRG@-<c-(1h!F_gn}X<D(V$=)UqRW$wR_>`<_u8m(xPr
zUOM6+ReoHSEm(Y#CotgULbh@=cXHaCp_t?I2e173dbH^-YhfCnRi==66D)y08(pjO
zsy|N576Jl-r2V%E#X@VXxk{|#=F6ooeWVY02UPvnZ;Do_=zu)sbLeQOZ4iSUHWQHa
z^nZbZmo+sS0%^aO3Eyc$H(Mb8LpYSg6%EXU3zY|Dbx~5IBlvHEfJ;$Yc=ux0Tl8{g
z(2&^C7I7;P-j)cao=1SE_K_nM_US7)Y<}S5HEf}2j5Y9TN;-)%Qp;r%tHFd}=qB1w
zr9=ZvHn~cK<-F}r-u%5ktwK7wbDh-x2k01=VMO-Yc56hG1*9@gm1s(VtEl{-Bqx^#
z((k5VR>TaK-Ay=c{u}2WU*A?I@5lPPzom4p86^&ckqzdR(OXn!PoEZV>&`YsRv}h|
zwQQhGUG*ola>9hPCWZOuD>o%^C~@K6{&0`pDP)_fQQ+cX==Zsk5*xzcg7*@%Mr(D7
ztJ_BB9o$4PzZYFe|1)}V<90Z*6X@BO_O!Uz>hDGAs6H$7L7=^ctvn+C%2~J)gZ%Y8
zOy1`S{8fM;(7iY548iJ~-%I+P$FQV=s%G+gkFCS1kr((tsU@c3E9(w279g*tN<ECo
zWLx$Pze%@3JNYiigpM1s{H@O_*x}D5<#U6nmql#s7IRsRi%~cOB;HTpVX>cjsFxm|
zBQQFGc^YjU8JNuHPtkfXymxj?HZ67=pD;*^oUV1+-cFhQj5>7snkhrZh{M!Rxz73G
z!1oEf7x-`~UJd5$d8(%q6+bOHdKdIJOxOsjUi3OamrNKQTLv(gQO0^vMx_r1RrkEE
zHVNTSi4-sE?x03mQ%)2`bzqYJFhzEvOK*Kj-6|W1g%paWh!m}g-T_WE)lAqD=#K|{
zOy(A?S+GS?$|K1%@6Y6_iz_n%!@Uz#=F>@iv+D5-ni-($RtcR*saQ>^Zn;nC1Lj6<
zSwQAO*+@Hk`Bh-1zg-MY_=|Q%h1DS8>fHq6l=O$YW6x?){`qRGDNe^eMgwlw3o4rl
zOl<BCOz}nCYk1iCZa}cILc+;1psY*DAO~pE`EvRoBTYbHm9v8Gq?E3c%1i<T3QIm^
z1)Vi&izJsQ6~<{|8MQIC*IBRn*z)gB<m)>nIRQZy(S!AI$s<3u0f3Vyp-zkiM!*49
zs93#5xtVd|{pYK%t#HdY>3ubJ8`jgN<9&*F4~=YnRX@!yTHen%Y@QiTzp#FrhwxhU
z*(x<Wyg4s<pyL}Eoq;50v!xGz(@lFf4Ad^F|JI<Ja4o(y(00S^Y5oR%I9}LK-Hc5~
znLLnr!&rnvTi=Y>ITP^xt@!hUg55Jo(h8-L9}RA3;N2&*Cr{B%h#%uu3RyB2`6gW$
z9ZjUg<3$A1FoJ@D(tv0jEz<S%paU?os!WLT2RkSWgHm?P=9G-J1qHO^B(ry)dq#UI
zT8`(0aiiUzo=3~_%U_m1h4B{i!kIinU1_8Wc%YDqr_Suqk9v=P&y<4mE{W5_XRC=L
z0yw{6n<#WR3q66YKseFE*D7oGyiEF-#F<W)(#QW585TscGwIgHh?T%A3;zOq|Id#W
ztJ3B7O<eb%9mXS^k-Vsp&AJoF+IHc6mlolbS(XFtH#FNY1L9A5UhcZd+zy|@Y+#YY
zP1=HS)>Xi8wn$gMVgwJyR32RN=Yj7`?F_D8<#h*vtZl3mtnEtcQ$3p=pmRGxq#AU(
z3KVD->m4SU>sBQp?HL$<RQk4QN*=&a?I+Mqh{+WBc^F;f%8;Tul6Ry0_SKYPiAKUp
zVt(o8h?w7m<vDE><ISgSwPt$tjnQS+yTT@2Z*L))Vkmt*J#zhSRjbkvySp2QO_sK=
zs=V#NoosY?YdezF+uQw&Em?uY9y2G_=QKsskg_z=n{q7ca_3!;LFItS9W6rCzx_Z%
zOKEKx*+o6)ITE%UP~+IJEJ-4xJ6#U3(DS?jmyk9!<zg(d8cBQF1h$(a41rM}40~}F
zd6BOCFzCcOBqb)Nq@?8+yLwrb=XwlDeh?e|%I3e|_^$rTEBBVsn`>*wFt82i+tNyO
z3{d=XNl1!LV+iK_S-p+5zGgRP+F7n7{hDc}4ij^fv32Yzk?!B6puJh^7yRQ-0DV-I
z@GCr0q2Z_cE2c@fYOpW85s0y~{;G?ZzjVG==h$iArA~CPMuWanu$Ap;@fQTL;mDz+
zs0tN$^S1~aATFj$do9Ol*4vwI7Q1;q+;N=#AdG}Afruw9y(yVeHmvv_b`&j(uMi%u
z@whxL(BAdK>BY&44S$!1`{&XPunU^_dV1|9i>Dcw_{LbC4S0i2_+usmKaZw^>&zl-
z^&bzqygZdsA}HhtNso!a>`d*}L&ko00&}ZwStrWH4iv@6UxgjEd1xb<bKW3Es8ED!
zjaQl~@RWP=c6p#-YgsE2&2LCTe!5-T?u})Ks>PB;EP!4y``>0qd6EMGne%07-Rzt3
zWk&b6*N*Q|u*G<<H=?VB6>3yhe(AlPH=zQG1u9#a%4eeinPmZ;z$Mr&;#F+tEE<H0
z70l%~2;W8_WRAvz^<eY$rTzUapm4F!(K}Sv?0$_d8Lr`QpDttl9U#Q;tY!$!BRyz|
zegs$`Deu*6XawhRUntvWG9jBRYHYPS$r3-d+gTl<#6#o2?m1v_J6B;CN^3`T$?W~0
z{^=W$v&a_|dC3a@IYrsD3CzORgR^XSZFr2SR014U(1oC|%EP};Iii4M0tfR(V&y(g
zo>e!Y+V$+rpUzC1_7^=NeeRFX2ceA5W^rtxVP=HX>zIz7yDP#w*T%>MofDRy=Tl5b
zJmNILNl!^|r8~ybLKXr79Eb#hg1%l;O@5Kv62>1t%+mX@MZrE^0bJyNyu1Y|b>dJv
z=u)NA&O5H4zew^Ley5JTv964S&9t!THK!D;!0zY<P4W#nC8FfJlhbJ~PDFD^3{)9*
za}%F=?n49-p9v7iw};AzAgLIv?XUwo&~z{?-9vg-1x}G;+e);L1cQE}?7%Tpse8Rv
zKKCQx+)rav|G@3r`y=qEJs&(oCCDszPdhLw%_eE;e$X<Kkv;?Fmr00NeX2+a+HHwY
zw<7ec6B5-c`@e@ZwA*acX+M!5RHp#O6$UN?b7wHenp%Hch$5~#IFP_%db0A1p<(;H
zXFGbhQ2Okg=JNN33)WPbb+7<2^Y5d@A2T`1YZC0Ymz2?kEy-X~f+jM`fX_n}OZ@Gc
z1b@jZ@z}MUPs}I;oN~b;<d=kY#%*{;{qjDX@1vz}Ro5hD@Yz3q6YX)APCFw+ue{nP
z#Z19C>Dh&E*7_k<dfA7b&)py<3|Yt2?7_r?9y^%-K6X}eegG%b)X;jvk!&rP<U2B{
zpDA(UGVB+f@!khW0PH-0H$ewz2b*xcyLFtCr4k+)X44WO7fYb7MP<*~7)F!3)V&n*
zHA@%O<FoRFH(8W=?GkRk6%)haQzjNA_!5dn|Mv@R2X^4h`}<}zf_=(s0q9?W1xjqw
z0j^=0CVd42quzxs4;M1rD2FjSd++*W-mXv;<*U~)l&AS+D)Qc4j7!QAY;J-Oo&)5y
z(HTd!!AKl~*-TkFDe0;SJ9FJDxSsX!>#w*hCSXF`%%;wEb7!H>Zm)Uj#k8Srgfx%i
zLE7Kr7VV?@0mA1fLw;-o<bM80Ha0dPgy9$ck+qBD*re~CC7cOE+Vzcp$-B8VG9zOP
z&mwYe%zU=9Y=_AQ0y){v`%}deK-+enhJV%(b85kH-Xa9@q41pF{RT~ZCo-6<&RPVA
zjJmW9FZ_i9(9Fx03O7+~bhhd18`Ekz1%eav2vwXg6F@y9#eC+r#u%T_Xe{dkE69d4
zet$*+Wta~DVJU<*|2^UvY@rp`MH%+*R2opuxwx>0no1@=ks=a4;(>8P*)-2@=$Mvk
zj_cbvz;hQBd9P=K^2y}YyNVZW3h&XToF!i#zE5dQY!}TJ*(JcLWg$_}s{5GA_`$(Z
zf_~h#zE`sOius%S8#w4jqzC!YScd-XYiB&;0Bmoew8fuHtoBiptD9yWu9<(KG4-GA
zdisY4eQZgI`HWz9fSUU7J%sE7$>SCN=WYgxls$~mwcx;6FD$q0S7K<~_5pe~=Dv^y
z*@t;0zdjY4^8)v$29S7>x!;CE87n$E8xeNam{ZmrIN?vmGDML80~yV{F)$!a{{Dry
zPu*_LkC+N@N~eG#zYM_8nY8RkKi#Q|OvkbifI+HEXFXsYfx%sT{jr&^=<8~qr^aoG
zfF~URe!of>bkfe?9GG-h!KZU}2K#oh9h&5rP|(GE+s%yNAH#Cr0y%fb`BujvaMPuc
zxuUB9oAjG+CWWDafytlcJbALUbFy*ZT$A6>b?i}*Vv&NQD`Q}gMJgE*QV~MPsU`==
zUW-AVMWa(hBwGVu0-K;||5|m((;OIFmMkdoHvoe<xy!+sGOQ^A9>=fEH+!^7B9w;$
zpwFOEuM@E@l$RKu)dXAgxX7CEBj5QK3ckgaf}pin7762N6Op3cRg4gIu_;04KDahK
z+1_-`9t`L8JD-UQro(rsw`m=y(rO>TVLgBS|2(W;JoTX33e+#IJp9S(nkoAcUZuF6
z^Tp#1T}k?HA)4&}XNYcMuRO#SW|STEC?`3ub!HEBWo?!WMqsD)^?e$XI`~T-(k^ep
zi6<i>;`a(AM>>JIFNjMo;HyrXPhf>%|Ck~Ap_e~9!JsIGHljHcWP^Aeqd5-nnki7|
z+G_@Z^fZ)3SEkBr@_;nE>?FhKc&S9XV3UyBJ`FTfD=rSB>dF8_XtLbwzW17URzgZD
z6Bx0}CT``26_;&+*;++lqFnRm>Zb|*rpabs?8Qd3Y`ulC`Wr^DLtonMj*(9;fNW*b
zMAj@l8YTQ?2Z*j}Xt^NmRQSPeVLM46;L&(C8wbbfgCZWu%rc8;t}(mWLVu4h+(!)K
z9}+OSW)&irZ#(xC+1WliP%F1e>d^N!Cp7Fv3v7>RyRw1(IL-uuFf~^tk_(J0(RXGR
zx(PsPkZU;Jm1S+(M}KoaH6;1_{xM*x!JZ^%i*{czov;oe)8D2FmPf5t-pHYc1XXf>
z(g>TG6)rEVr}}5dS$`DV*J|sr%VNpRIzVs3`vCJ8w?R<-A1?s?|3Od`BQAQk(1?tN
z411x4pV83}|4>s%6`Q}}VROQ<Kekhq;`}R(Lx4EE4Q~nD&W5qgiF^gqo^NmJ>a`mj
z=|PtEk=l555EL9Hxv!~<R?9!8@@o=?7KRD=z8D2jccR@D#MI<KvcOs?kQL>i;IrpV
z!YF|Spl1M!?ZC_rUH#Uu$xz=Bi_&oU=reT)3ougLTW)EYx3TjFC{3v~O9f&FQpWLN
zfrq00<|RpRNmW(Vj!|#NgPHVcy1fJmor4UYiP|v76PTl;<CY4M%0yU6*`LP?kC^6*
zSbDWF6w-YD7q;c~{$cqtSJsnZFRpz+o?|EaQbA)18vn%ppYiWrcIA}6oYlnbY0a}k
zXBBYP#cx&<(fo5)Ybnwk7<D(e#u5}h{Xx_Xd+YrOjiH$LzrRRE5Wa6d135Ov4lk1j
zG&;h+fDtP}k(m=6|AB<Km-y!*lEdUT%=CVzJ+2UWJQT4uTHk{lL2j=h#BD5Jf$|2d
z)EMsjw?!_y+3_YI9x`mxpgNNlC9oL3gd&1O>x93VWfV+Ikyoja9c_uA=7VdhC%LGD
zHZ_nckec%4h)a|AlY#;@_Z&^!2yIb5fWwEgKgHwSn_`m?%uBReKJqrNb%quiUEMA?
zOlx*@e{&O0(?5pb@&3A-dqXt3-xR&GKTN2|mhsQMT}~GEJm~GggO}|_)eXzt7<mZ`
z>wT?`IDY6r%BD&n@Li~hxPGf5910}P{$D`i#1jUo1}n#`s_Hl*{BSs}c~Eni`1*m4
z={@u&ouK}={1y34QRLgl^!0x&X#g9zpZ;_sF}gN@Nv7;Vg+{Sof~`AMx<g%UiiIhE
ztkMTWVL!-aOu5y(xqQE>vl6##H}Y!gWCohbv6)XP1}1#_CP<gF`usBKOaU@pVJLM2
zc;AWgCLr{}s@DPUusqpyb-Hc}l!5ki`OwYz?=BQv!JzLXdkH(|`@uGUByfbFm*;>%
z>yA+^1Hfax0;^s6=*Gd4wx|w>z{b)j^&L7NgLyx|NaU#vPotX?kZpZlsEs6Y*+6&T
zEn#=I1gb)EsGfgjjVFMworAU|)a3jENFnrS?#+~w6>BxAEpq6$%R}rA=h*ZjXO=x5
zibcAEBO+89Tmadf2VArE_}Lq!R=mYCO%K|9Fpv0q@Com!eP2{MHhc8!u+266^l|tg
zxbFg)VyaLUsy>a=G}X3xsT-#CDGv_jBfLFYK_8Ao9?cg4``B>e0ed~~9_?o2q?a%K
zp;2N*@X503OTn{Rq!bs}pe+yaw3}q~%9TP=%Wi)T);du?MCiNHLZN2|x-nK#RK0cP
ztN$5#{&{SIUjGh2tu}gBj{yi1_Nx~DQ6ByWyB4Y<^Kxso0)MUDB#FN@#<0(6Jrn(S
zbO!-r{&)g>1OK-B-h54IeQGL**bAqCL@dy^A(xh;HLxm^$Po^FX;#jn8<q+2c`gP*
zl=6J-oQ_RIaS*wH9~)5WB*hl%sRWd}8E{cw_`ScN3~7J1X+jP1=hCh3#9t1W)P5R;
zsTl*=2v+A49S7~<7oq7_8oeE<=IW!NbWX8$u<<a@i_k>@k0AssDV&s)=z+7^-^m?0
z4Nk*8OxNqp_{nwKOpsDk8uY%ZlLDV78`!cG{5ASKgJ>QV6_xiPn!-$xAiNnG>KZH|
z_WHDDKY4-($KfZgtuSS7@<>Fx%Xo!&%HgX#3(YV3`M0M(WKNqLE?G~|!Ke+lSD=!a
zV;SIE&Bp%uy@fWlL+<ZNvf?MWx~5yXWi1~PGS&?5*8J8#r_r`O__=!<UM`9}Jaj{N
zNVZ1flL;h;Ob2c<a`<VbQgOrM!U$7hr8y3FnGJF9ExtcACm*Sz#U6C!eEbcPm6S&y
z@P(O^eDS9k%Kyf6(1gvq_1}5g3PvX~G~|(-AUNAv?C{Zq*P{fFv$<eTe|10Neq$TP
zpixg-hcOkysGY+9aBDZY1SXJWd7O?EFV7Vq3Ka$)*77KrlX;xN3l#G+#Kt2xt6l8v
zS<9V$x5=YfbWw=ccX&lZAld;5adBe6g3O+_dIl<22pGq)O42DvqjPzEsj)8^Z233w
z9H%9hK*);0Tg^u2NXz<NitJv1`>+@i5{9&mEIUeS_PG(wK?D^B^yu_~<Jm@U$c%_G
z-aDNqgULK9M|*uB3oj2lh>468g7TBed|#A*T8WRUfnj75P{^p8<Gc?m(QJqVlIQV-
zDHcLN{`P|hr8*t9=s}g$GXFe{<9u~nqPEAizu82du2YgIE5RGdob&Br3~{k~+szQV
z_m<30!xhix8l5#W@Qz$}D-G1%@y~8jZEXKi^#^4jPFk@9RCp6U)c&KeBBgH@jfN_7
z(?5hEnemu=-d;#B4>egNd?yl>p_x-_G5FZ7x0dBz7t0Gs62IMWP4K#f9%Cm9Ov=LR
z>sd@UFw8yu<m6&@S5VyLtz0eSwYzN^RVVfcHi2gYf0))`WK8$c$Un!H7XmbKs|)l}
zc$G@|e|_L`B>zgxsor%HK$E@JW#|JB3VNH^)ckJpDq?0N@+$0>MNBi74N1XaLJW%!
zmH_9F!)&65AtZnY{}CzJ7+{Bwkd6ae+n~&@T1kRKU;o$Dy|HBh{_sF;QofJ@QsxCr
zZ8DE|EX!uRGq+}PZi680cR#Q40k+taTL_kEYio<iqXI*QpXRJ5V!==I58bO)sfxh-
z+c(>n8o@p^N(CaoaV~RaCMiA+(Bm=9_c!Lk!b62+TOefQ0PZ>8-PSLGh|e|eZ#X^y
zdhm8%owQps=(V7CnuHQ69T`mXP{)?J7S9%o$i+WUv~NCLVumaJyBw=0AFhI90m0m|
z`PxYIz>r5P1<IBZ%IGe`!VrGx9z(Li8o?S3Yv8Hp@O{DW8pfOqS12#G(aSdHm*47#
z|G1I!ABzq(2a<nX?cmKXqTchNSqr5J{iJd=799#J2{o(KdCyFroT#puzun&&D--ib
z{|dZ1+34ya-d+u5+O2!xmCHB;d-Kj$uqJ*j?^5E@$_Q_YJkrrU!BQ-~>?w5Lzma`}
zg?z9{QEwjPVV?Y-4;2Gmu~aKrES%uO13FTdWy7808iyY-r?pocy}H<K*YSn@@NEVl
zJms)!2UGaeFR74l7>ekckx@}8J?8B`eR>rJnq2_(ojx!2PMP(Qouhb-tN-yaw41>4
zHnA^gP$yx#EBGN|PI&C5(MOa{=Zt2>mne))E-)HZ7YABg!op}m+Ajl!`TrJi%Qvfh
z*-9`@3V|nbzmQjJcB@N93&gJn=QUsI;WoDvRRgD9#7`3{fXQ<K=AU4A^>TiQQC9|7
z8y<A^MvAIcnh1UQbFiulY~mWv5N!+y5|m4|WI)9!rh&ho(4TGL6jX=BfpftSR&Y?}
z%ko;^%%43S+o@zocYqJ5kWfic(Upm&kTIFBwlLGaRb7Q=r7N?MzIFT6hBKeXLZGw*
zE&2j@+P&9z54?r)f#=0udDEB6SD}WW0+t!q8wMJS=70CIOT|#7U*E2MgtS*ft&`Ns
z_52USdQ(BYSHsW|t4u#gFg-)}qHn_4Y(b2N)4O;S{{!~k>=jM&Uj-^zHc*>M&}S1V
zvU$7K;10@ET!{ad(C_1QB0ARnH{?^kt)4KMW>b`C@`GLMRff)w3pd=Escv@Y8GOa_
zRMztAt^3C%ywnu)C!kw~{~+;$7hxCeC3ZS#XVq~aL&{(V<sJoEODw`aY%9s}W65ek
zxRpBL((Wk|^lARjSGtTBj`1LXmTT+lgn}nl?YbA8INDTJzRezY_3bu?i?y|!YAqgo
zlc1Yv<40|kIwWF?F0b-W>i6#wz+NN=n8NRy5>-0<ivm&``TqKm0z&g{)WE+o9KeXf
z$_kRWTT4VTt}r<p*{}Ta!`Ns3$@~4u^qcn(g-R3o0=aabaTjD_{v@c!$!`d#`q_zY
zulTBzYLR0TbLQ|I)%u1LaJ?t6>11~YUJk&GfC6|LVA?2RV8@oNw5r;Y7p?9PEExat
zd48C9dbdNF>h`|n{)RS|K~oKV3z>lP8!4HcxYpRKsfi+01(B?tCWaMI&mzaJ?&(-d
z2X<$Ge7)Tn&D@|%lu}g0Bm~qTPzuGa@(>dXcr|DYsGAGG)5}d3C<PA-uvh`ip%%db
z=_q&DS8ia|KH$WdvTh-$3rI!hG%mmOygw(#V>W=tV$@o&1XYBv3<L~{nb=;!67>=d
zt~&Gh2tpoZ^n#xE*nON0%f`RLS(uP&{sQ@+eBi@3<`-3}Rwao(dSV<yMtfQdEgo#T
zv{mbYtTo0MYo2-D0^5>H?C0t!FjNlrxY?zw2lXsHz;WI~;buciAb2D+8L!p9gwKN<
zYazgb573@wc3Z!ORe(B*Qh(Vh?$hR%&AePohjXUt`w6(%c(ZIDrjcCEZNsV$G3K{k
zjkprU@!_f#lFU2zmsFN=1eS?0Qcpa4iUPA3*d<?zezZK&e3fjV#@owp7M8;JEUd%^
z9-1ZHhI>(Rh<R`z5Ti&8vpwN;w`U%aNrAs`CJ%Q-w)14HwAN#Gf7f}hMI}p^$pl>@
zew>fhgiADQ$$%?3gz^8Aa_Yf&s<q>IJH4(*()=V+sawxj7JD#>OTopwYEdXG88CNh
zAcci`iP(dbf3XcuO9W81o&`vMY9DGSKPqPIn%LvK3jEj7MA-TM_Hi3=c{gg|t2qUy
zqq7kDR~=Cfwk6J%E~W{14BBhsve}}a_?P8?Ga{sxL}mL0A|y?xA#J&udKteNSfp2r
zCeCjUr{)5hqrgD}!MqkmTrz>*y<Tm4F(A`|UKa(anoyMI=zxc)1+ZH?z^K87PWcZH
zD4)Z>NkZ^QV(($tpo*k>zh)BF{IR+X58J%VB#3Ne=L6l#e4wFF(!*Y10g6PJHsK4G
zZ;jbORm*NJMpSFr?L6$r7{ye?%AO?LV_lMf!h&mSAX=1*pj?7>7tmUqy8v4|dI}K8
z1a_)Qq@d@yiNo><R6m(CCor68^%I)j4aLf@pWFP6S6qmp$Se&_An;iJ&jbPw77+(E
zQ}!ih_6+A2tU7&+aOLsrd!?JpPebN&0GDvi{o;STi7bz?$c2B2;KySuI!A7;#)S=8
z#QaF>VNblpOFsKPVmVoGy5wpp(y)tH_u1njiTnqP!gG?}ogU~(|F2jCzV&|Zwfufj
zoz6MB1@pYE3aF>jTE%&6Cp#6v;FPyAa*>O=oO6a6EMx;dXsyP!#%xkD-}Q7&baQa+
zXzuhBolI=R?-Zkl^hJmM8Wq=Vmz}Xy>6AX>_q~55?$V!`aG-}LydsQ|27Z0>Ybkdz
zB=6#1G32al%bt;9g*aIw_lLMI)B_!VgPhyCGLg$}OAZ`L3n|j!Iz5pPqk-?~Q<JZz
zth3{_JraQVeAbjxRk`snQ8tnI?{Cxs{%pJfpwS@+64Np2bdi-U75@p;FsNv0jb(CD
z9E~Q+p=`-_sO}B$rGZo0IVvD7>5q&6ZKk^gh%V}ocEafbrBKk4qDRSV!$x8>klGHQ
z<OtaB8cGWN*LFkbW}F#0DIJXPP^<2kppQp;`{4N}1GDLI(4TrBZQDU{k1sMNn>KV8
zIZ-Z4cUPlS8KjZjl3c{~P>g@EdVyKD&(5)qcUX;~`0oE$2SXm1MW4r|kbZ+2PirU!
zl3buxIUZY4AbYWN_UZ7p-__K);mQ=%BX8nGj1EOH5dp{J={ryqBP)x*eeTuz*l}R@
zbv~{TCuigW{fLm(5+xl5x@x?<B>#hBXg+=ZhkW@TU%e>2-}@s{cVEjaG86LJe>0md
zC_XsL=myxYM|dLZ0|qFB{ip|W_3EJ4Ho5-zGZ{B`;$tGeF)FT9J_Pc62YRv4h=^Pu
z$B5qYT;qctTMtk4Qu=CpAkTiE1!zKiZcqQ+&b&vK=AR|8P6>I3{J11cVT?5Kd9kJ?
zu3Cuuv#fv|fOrL$4VCZM_y+pM&?CMIC*%=pG)Ywe`_dE`4-6G>)6**CAlt-XCqM=o
z!NF8lxM3txl)z>lXQYTnB(O|u>xbKHWiRcf-ObK<GXVO3j45Gi0U@yq+`=6`>(xB0
z_@dHO*F^*c!8hmw@*oY^%Ftv4iiP1(w;`%}`=da)(*{QuEwZV%<mA+&@=?*zhX>q8
zip6TNH<!n<5qy7MpH}Oy^{~1f^T~@CEtKmAuIr{A@gSNXHC<}eZ>M}QY4bw}k&y8C
zxS0k)I|G##ngbT*N>K+R_(v0F0Ixz0mQ05(?-p>7D*ECXQwv|FUIH&5>S3$~r{f`q
z1<j`qVdX19nH`u68k{N)_q<T)+&*JoHYIJ=XproE@_m7VsZQX~J-htPLLAT-M86N`
zRp`0xwveHcE{ok_mN3Ald9b*OH2;^d{6v-}t5Wmg{g<~+NYmtEqHLdgi_9zc<yV6!
zVJlMtv-$on04YFl_iVM&%?Ujuq<X2Wp<rnAg|^9;u$M;_OPIUY%+`iTo6umg5&_Fw
z$8^^1a-}EjTJm&dH<ZPjpO@U)*jFO4ilJt_eN@Xe(LbgBl17s=TEHsEh5+@?W33DF
zSnGnOOw15|^zU$wtM$J#R9-dkCwjHuj7CS&fMiuTpXy~7TAwCrc_`RPOIOaIW;!0u
zje=D2FyNO}+E%C6ZP)l4#%7bt9Giv^JxUsy5dQls{c^9e<e8uMM@7bOk}5MB6EsA6
z(F>(xNyC*D>TC_iFv!K{zH!y?0CK6~Tl6w>@pjlmMRCj4KSh@-TFS3T&?x#o*h_W#
zh0#>K6f(*WY}r*2KUbN5YXPK`@+gnw=Bn?YzNx(4uc6z#ob1P`*x8GvlawwS344qF
zT()tTi>>p$=^8PTHS<_!3o?>nNVW;Vb4r%zWA2XXE7PgAhoT0v5`qrT+wdo$xL5a~
zWIi2TU1Lz~Jj#zrXxsFEVYhTUpjq3I_;pI0{~8^Yj(j+eyf{%kYZcU}7TT$<m?eQO
zDIr0ET~W3}B`V@~aoOQ>3A(O>cZgQ|TEMxOP@>heH{H&lSzmePLM@-+cXkbqNwj2s
z_qmv1Qi0>L*8!atjf3cB{~vE}9aYuZb`J}pf{2u)NQ0#0MnJl|Q@W9qE-C45X{4k(
zB%}nCP`W`vLQ;^H{@n}E=e*B(-|x?F42FlH2ln1;-Rp{Z%{i}dBHqn`-C!NTuel$$
z4bpdHg<_(n(9!j}9jHm8shF8V)#xxO2z9dbTdCM&PBG!e=8^s6Ka%PxWjnLSQ9Y7r
zz^;E4_p-oN)OTvXe={eEx<u9*wT?fTXYg@CMYYKf*yQMw_q!rcMtsippN!Z$6WgHU
zvVY+3N{qIMaJy02i*Re5$E+t&F>!eJqj_w!1qrv#ZT@hSA&88~47KUE@Zf=$n;psx
z*=ct9fewV<aBc%Ne)+GA4%5EN{>bhBIuNL`u%PI>3Ef1mzo}4guI)87Ad~%%KzIYu
zS;`B?qY)V8cWdNMh+PlDgSoXW4gid<rm7V<f&&I9nZO%{s$<cU&3Z<vhsZ-seIgsn
zg_4;$n&)_hY;8OvE6?1r$NDi5vtc0XyTo1iuZo_h{6_5p>_%9DWqLxf3#KWv2&E0T
zw2G)W+%}7L-mF8L;Z>0H+^@0NyM(uqp{9XfqNT{(r+p}0{Xyfl2S1G*<B9$g)IWnl
zZ#1%(+yesEdol`{<u1$)m8An|p>USAm{2`g{SWwK_Yr|F%=JYEA!03wT$K=R@i&dw
z`T1#Hf@1Ypfgyhb@BD|^?DsBTP`X9O3eg9jnQ|EhFF5mr@26Wm&V$D~HFC=r<Z`V>
zT#n8^4O<uihp!W(zdoLEzUZ|gvmviQ0{eeoNmC5nC^1lq4|ATfeFvGe;AE36iOo`?
zX4*0Nxh24Tx|734UT>gykl6yS+t{bXyBbx^V3@a5{OMJ~Y~EQrrfh%X;f&-dwsm+C
z2Ib7bX}q(+4set=1?<G|A5v4xHqwCZSnsrz_UOjv<A*PS<={vlte}kY!&tI)QzBS>
zZT<8%c9y-zj<_%_%X#rOtLs<O4+{;L=;-{=Zf@=)Lz8lx9J=`gZgcELP-fPx)zjbt
zo-&4(NP$@ivWjxMMjwNA!%9JraUD&Oyji*D(@+qEHRe5jOHX+xTvxb4{<tjAXGeGl
zS6PDcUegNr3zs&kMw*?9`rhG#zhY;2qmjQfl)WA1{v@dPEq2Bbm()zHh#UI@)=H9e
zK*TNlUuyqUgfv%@H^+^X^I5B6r3X@()?+~lDPmXE8;Lk{n*=d<C{rK9qrMNyu?e}m
zR>5Swf%5#k<RU<K6o#jC`O>^((2%8F{Z?-F+f=oCSG=Epj8{&5TMA2?B&-4R6j4BJ
z)#j+%hz8_VFsxm{{~m<-cm?!qah#~vusogW6exEF!+$;-rlO}SdNEprx4$|RkRy{U
za%*8h`61iG?jJMmZl=R|165!Nr=}w}{ahy#ovF)0*jT>Ycha@fC6T_gMkPIIK%!g)
z!$LFFUeZyE$*LfLJY)=iH+lKPEV<QB&U9kXXm5B(HviM^kDe^B%8^Q+tXj^%Cu`*N
z{A5cB7%f+;XlIU;p<|B{ob0P|{M^_z$oMLS3`nf;pF{Bk0VUVW)4NRXiU&i?*oJ$V
zZ^n@l94uchJ|JO*aR+n^4EfB9X|6%EDd>xs9MRASgFZP;N|N%iVh+!kx8#$VdXWzs
zXm{+<p$F=A=)dtq8Tcr9p^#>3fw%(ETFZ|ay~nH{L`*zzR58gf(IW3ivN-@5liVT1
z?8WrnQ6^nEV?b7XMRJIKe9o07&W*40E_9E}A?WkfZGCc?5N%XGQm?`@21D=ppI^l7
zNM%(1pO{9;V_|O?%_a&cEN2=f=o5j@(%?*^+el7PZ!M@iY`ZP!SWWxA95=sFOVpxf
zZh;NbfKf>4OK&|sTybEn>@xl!qz7U_fL%mj&&KC-uRCt2+<R!C%ZoAlCNGF}tk$D;
zGFxY!i}6v!R!?5a)|OdVRFrmv5P<y0z*_RBE04<#IYcd`b_2VzUL0&ptl3yKgY>)x
zcGyx?_IiG7^#kr+I@6i>Y;3?DH;IDuo0=~Y`krX$>TTr`O}{6awdn^&_F1|gU+;lC
zN^3De`PNh8{Ni*M6@tXD>D2ht)zg~&##d^06lSl8jtyULSzcX|uOzyo)gMPpH}hx_
zK3u4JnT-hU;j4zqT`^_Q2oUaX_NJ>FSoN3RbWs{MT=;bW<7Ce}@xTT{09#g(QJPM+
znF`vU>pfV90W;lTf<v4h!3S~t_D1|F1YeuiaJVza9M{x~1R=G42|~C;Z_pGFPM7={
zv(pVKTu9~eOmTb8G8vT{MjE{mJX_q$&tQ|y(<yg;aG?-{U7j_uOE?(1m3Y}iXi}Fo
z#6Rz*6mgmRiDNw=H7H3U<Irc$G`i-Wl(11sCGe4nfRCc#tGl=omDN6=nL;6~Rw;;R
z?!5n6jBc<D@s5Fz5Q3z%VY#i;rp9i{g>f(yCjplKEsnHoPy!6ZQY+>ebc8$h7<m$)
z_%Bxi`V|OzyM=*ktpi%@V0qu=+{`HXVh9xW$7WNi`u9iQ;q@ffq=QK_O-GyWb3b8f
zZnpy^R+y#{V1ae>^yygZ%l84^vIyu5`>gaK^kztsp)q^R*SPU`e;re_`NeQvlq-*I
zXTZRW`-y2};IB%9fI!a0<!e?wZ-?J^z^?Y<6GLNMpPaj|65}{9e6bI&lmp*7R%_t5
zaCQ8>h3CG#O=R8bF+4WbO3J@^I0Bm-<`3{<f)G6Se;25_ku8xyxANRe)wGHa&LU+>
z?Pap<?Mi9uk3}f19>3Ge$L~n=2KOBdZ>sIqPvW%5@>(RsqcNj&QW-DEtXA@aIj>%v
z@NYgFY{T?2zWU4m{D}sS0L_7uo;sx!SREbC(kj*4<55V2j_$=#(JKqZVhht5WB_jL
z03b@Fvgzsd+($1L$uAhzr4CmP-T?r~V6Dx<3fT6Z38aQ{fC&Deyc;YJqoSl7-4qT0
zC4VMR1#tdsQ`WPvphE$;90;@`06Ok0c{6kYuKQ|n&5T0k$Cri#CNBU|It-S2+jP=A
zJp#L);_`SbB9=9a)Z}gGq=GEk;L{}Ip9I+i3^cgx#&)t=#Auw$R;|x8e)zt^0LV32
zK#F3lfs_S$94d6%^YasHw&EOxcM{ZRwjaSjPV&Wz&j5>?ZE8&DLnzSx$+pzcy~1u0
z5#(NLkvPYmLU6LQa2|b?{I+qP;9gekzRXbabH&}+DlSe7&3Jt0v%(S0rjB7=2j$vz
zg~jKA{(HSf>e$rruI2N@wYciOrUsE^oHo7<RZs->c62_!HSt-)@tu1k2dGuVL9K#I
zj`k2oB&3O6YKeB;4ei*&0KajRHi&s!!O16E{t*-$;p4rUzoFfJ!XNrg6@gU9aHkv@
zSs`n*b*d2Xw&TRPi5f(xyN+iI^4jPMuHFYb+Mlvz5RT6AzeDT|1SNQg6SeF4HaB4f
zx4H-{6@{Q#XwU)1D;e&j!9h}qiHS;}a{wlq&4y80B|^XfV?f3>8>acavX3VVj78)G
z1YT>o&*ZCDQn~}U(0<CYCWN~%-R}#y<WjR9C-m0!(sU0JIUQ&q(QBqvIEqzgCbZE-
z6@8=(QJB`|1ZY;}46|Qm2OC;b=Tad1YP<b}z0(D8mQ}aqM)i&;k-JbI*X}(gS~6-H
z8i_r3up^*e*pkit*dD;3f%Sb48--#sZXwU$7#Mtv_5<@eNoYMtVU|h2i2(^}Ultg!
zW+J_<Ou>6hNjV6rC2?>l?P;3^hu%(HgUU>99%Udfg7!`~1?qgrxRKrIr+0N9a8JVg
z@p%Y@8$RFrLoSB*zEOH}vJRFyN-z1YaW*H!v~uR5gNJ5T<J081@*+^+n7p`D8qt3k
zY?{$@<=7h)<pvMvO$4jvXVJ#ibVBm-$An%@nyDhzD899yZ|Exl)x`~OFG?tR!l9u1
z9CE^8kA$XaMJ74)KQ0iLt?6$W4YJ?3G2E)n%drAw@5TSb+Z`f`8_<OHe|$;9gE0b<
z!^XILaV7Zl!oM=H*)60J7`;V8aWlH<L4p(kIWSVNI7O?l%sBV?+lUlspsHFb19TvL
z`731(kStCHebv8h)Yr)T*^j=rvIIt=w9~+ml-1z#J;<rS5oT^cjjCQ}`;Pe>$Rh;5
zgie>=f5h1j3LW7GsRVG*RUKGQExw7RzI7HzL_OpN66WeNP|@ar=hYu{<mrHt#R?Wg
zj2z`=XUhTN!^-IqAotD8nOp$h%|as}3Zk+cFXNwnS}zYez4<(Tt_WQ8ebJqGM+hSU
zW0@R!jrX0T&7D1$+|X@gDqIApHU+3Jcs$eeLxWH-Nt4A2rl^9=#v-yc%!TE$dmW6^
z4uTx7BVBqtpTF8o`yDY#i2rcl5ijG`)CHK%9+UeX-CO2WhbFiQ+|YCt?|&&-QIV>)
z@B{^C!pEkQ$i^B59(U1Z4A`T>V_iLFuLPi?)ty=09wQQgo#5!<)_R0eTTBYuDJSSd
z@FRCWrC+Ab{=Zcao~a4p0(2Dq$JZ<dkWHszaDSdG6c&oClK^m^N@hL#!>+EF$N3=L
z%}4Gp`4Bkxe_}6*Rwk|t*=JI*2~*1&s#KMI4rbkjlx7?u>T0p1L#jIN!k}}G0Z0#6
z^ARUTOLzNwC++)I9FH%X2S0iHA@v{-d{>$0IxPcbUrv+zNw6uS<w{PnRVEn03;_Z6
z3K+#d#y)+S#B6jIT2b1~c6c$71Os%h#}Ps>0+MBbjY+T)e@g$2DjKZEA7U_nc`t+X
z*Ezm3xnh)m9<TyEeQX|&EcbpayQj+3D5pw2fkR-{GA=qb$AR9DCUqXP%2&+By(sUm
z$_FkosQ9Uzv8xFVC8bg2c-^gfKUZO?6_8V3(mW?kxr>dGB_P~N_5Q{M8Hoi-)=B~+
zNh`_xtUVcND;1Z(;{3ay@85$r)fVR`zh3m6$A5SldleZGFyTB(lu0`SxoE#+T9_CG
zS)${@6dN8sWf{sWlI#`3k$<j8-`yPhP^fE+KGwcRAP8yR<-!&lXbOdK`iuwacq>-E
zZY3VQe={*#GVpOvz3Yo6ge+<OBXMs`=tkTI+!ik!q|ii6*`8-h4XK@4swW;|xvFmD
z)HpOw4mMZgbYf2=At52>Js+Hi$>gWLE_kZv7{XiU0_l-2d&4y}BiGJVoedmvdWdp^
zSKLZwiSvi<6+k7%4a61lzb%i<T;3Hi^B?0k5(zJ1Dgyo-5CmS|U|=G)#oEq>mZAd3
z{CD>mgv6R`Qi0l1GAJlWBC+zKGZ6N^L#@^hpxslXxl>?h$_kCc*2)2yMk~;@py1^F
zLuj99<H7N|{%tO1CRA-3a+RbscAMA0=~u?wxARjn#J*vb+9NyBIXQan+Ay>bo0e7*
zXaG(W5410aN=3-)2;`DLvgH(j!z@KY3;%!-faoZTK>Gmjk$hD`fXRDe*N(sbm0sX<
zX!L<z*aoQcH#5L_9|Bv&sX~UE-dmnLXVH(EY`OFB8o1K!UV*>tA6ihLhu^CHqu$@=
z$OqwN7#reO(UI!3fF-nl_ceTkh`*BIa*CfwInB^&Bd4>Z@+69rrO7Bfc*~ibj(^$e
z^YNfjQOt0v78*-)zWhb3RcHIDAF;tk5YS0%>Q8!~t^SH4v)>-15M1ClLW@M`Rt-uI
zx}X)qvc@Wf4JJYrI?1DwrKw~Z`}>$9(9tH&`_FT)M1(AL`|{oYqXB9V&>byO2kSOk
zHX@mYK+JpxX2r{E<^XAhm}cg6ayE22<tb{XnQ`p;J%D8rOG9^Bn-s|hGWw8NCZPd;
z2e^OPo0Vb|oX=0$*U1}?ek4n@<GXZ{0=DI}(dM_h*eDY1+4WEDeKT|xQ7?cj37YV|
zo7HRc9~dt;_?fvvWsQeD7z><k<-vHuHY`G?1f&$E`u$OjcjEJQd#%U}dEi8ZpCx0J
ze}0cTV+9sc(rebr1(rz^yp03&YJRl^#JeX~!PZ0Y#<@*12No{k<psTgIF8-|o4g+|
zyJ1=^AL+yTA6V=U(+qG>vYl~zrj^uu*M^(<_FXgmDDOAP%$7)gG1xM_YJcXW;pquI
z5+HiTF$hj5NETlkNMc_@q(*9MVEE}!_%!$46u;EmN45c0eG#rtIMG`i_Pi4;xy<>V
zU(D_I?_O+#UG+S3wAUKmmlDTop&3Mf&(HNYKn)Br34lB&1WB#^qIG3*q)@3qiCRLk
z^5W6%VizW<nMC+_7FazFjgNbRG{{Cb&>V`s;DAC#5=j4sEXYtRZY)<gug1;yJlGUR
zHj;O)-JLa1Z`fHYEDh#l-n>6>IO{13m|)eI%VJZ;O9a%*4(QaPIiS&31p5=HV6Zsb
zb{<<L=J;yypw1yk{s>>b|COb$AIu6LYzGTV@ko!)o5H{xwz}}KlLcTf2u`tCN`(PR
zj8c(W)aM8ySt#-%BE&A}_?L;K-Ganb;Okrr+#63WT3pQQdxQ2J|L5Dc@WrsZw(bAu
zAdAze8>{6}!ReJn=vG3^RbRiWm~Z>tf1?N>Yj(3@IkAS0cBEH<>Gwmp^X*^g+Y9MZ
zuD95XzoAXuH8Z?;*!27L6YT~nrzjt<C-CJqdczlC>}InpDyr`4tMe0du1!6#W1ziN
z_q_^?Euwz~v3mnZf={m#_umMIHiy72&}~^WozpuH)c5Zo?(oTF?Rs*Me*&mBitX5-
zf;CI5L<v``R1_+TQV0d(gnppN+A1<DsrdcdSwcbzLjtdm*e|Q!H{jU?{d=e`@jQ3S
zB(l*giA8f!QNacaaiYPvMA-9mTeO5{M}g7!D=z+1D~tC*jK@1)iidNgH6+BvGeG4t
zw7t+l{p5)Z`iWXQ`}^Q{W1GDcuwz|b>YnIKxk4QXfGc1zV(_E1Sy1ET*R$hv%QNjy
z*clIj!@Fgh$I*%}uTEt%Sp}0-Qv^j1K8I~K#@S=G(a~lL?#CfG3^{4sZuJDo45-(W
zr7v|^3|HrK8R2U^8+%RorJnP$gpfs=!~aRmvGslk8n{3F&%ix?leBUF@-{{w&?tqa
zJB?vw&OhaFTB&F5vwr=SeTDPUx20*u#gb#uD{8nG8ysHs=l5@(c}r_sA*Ldk)moot
zU<{a9^ufR1N2{&mCl5=&v!<#boNBbC4$(G^5%WCDIcuv;V@kh&d7`{*{ytmyCK1^G
zx{JXGVAllw4DK@-scD*nxj<?#{Zf>FS%rbE))~nN=4{HDAM>y@E!4m>r4#`|mm;G1
zrs82W+XFr6XtdG5cINwoXrsUMHq8Mc(@cn<aIfuHC!BZ}WSv0`>qcP!k!f<vzWDv?
zJFCsS(8%Yfv#$gps`=ug;X4bUwPLkAw&x$3^aMn(7vNFt2BuOE2SJM>iXIhp*OG7y
zEJ}_w00P7e(3_e}RR`wruPVKhz=|I*tDOg(ZlEv^b2Sr*zH~OO28figrd&WM8=I@e
zePPm<@=mWUU?NocAkDMxs=7H6h3AGSS~n2IpmAU}_w`c<uF4NI5EBW)jWq_qEpz{0
zicU<VvRdQGT)%swCFmZylczQ`J?<=t=UB+<*T~*!fS18A{!~;Q#<;Vs;EIGC*@<8k
z;P}@O!hiDZuen&a8LqGkO<QwxU{~mg3`ZLo=l^(lkA5foW#sbk*sR?D?|cLw%Li(=
zl|#i`Dz2|IMRkrp@yo4KZc(Rzxw1G2deyx$rzPsSR*yKr$_xjch34Qg;{dc#wVpwS
z=2HRS_|R_}WtRs`Er3^0;s!kOx|ya(QB*PeJhG@<BBLIB9G(2&6IYZmpuV~r_UPFg
z@P7%euC7)jW}Lo@*UqF?%*nF^d*9u_&L)cKS{p(7WNtS$loejSRnHUDsnzDN^FqRf
zFfdu@kCAo6*3g53Fc=2K*`9)GBj<D2L)s~Yo(dq<E4Rj4`2`pLO>mxz8d;#7%?@;v
zRYFCKb_B@C;d?pIkPRpe1V;*$YjzazpP$G|N-F;76)4QM7>LpSFjHc3iL-{*EF)iM
zw#`eRbr#d?s<H%s;*xloSD`;|Dl7!ups3z|x<P_g{KjlKR0JtXxf}s@X{`CAoj9_4
z8ck7#8F6+Brc2)*%v)Yzsc;v>b9825KRXF-v58nAPJ?HTNT{&<U6>_3{H?qG^2_Jt
z!!CSu+wTY3qxQ5hGLxDwUiXhtJsZB{xUl@~4`2LH_on|>Dzk!j%)HMu0{Xx~fT!jG
zl4F(AERS-n%}PH>{ci6whmS8q)!JG^!Ar>*=bVU5&@p~zD$i*r)l5|t&Q1gdbE|7L
zfZ`hLm2=?*_W&>I<mnV>^}|4W$qzcupeJq>7@}_y!Je#d?;bsSuLvws?1H4!=!Sqo
zOZYyE(T<_(Qk?2ATSLZpqE+?omvVyg0Z`uLdz>9lI27w+b9m;?XbT&*jieGPAJ0~}
zRFa-{v8^!|0x5ztwt@Sc_nnIM@xzNxpTpXtoYf4MBCeq~PeWg(OXw{R9YAkrX5Scf
zu0Q)PHaO6sOOLah09bS|^B<JTgOsw|o1nQTCPI8bO(<ebPcP&&^KeyuOSRN+px{tp
zwel0S{ZE}!YBvP*FsO8E5rTg%Z*K7+3O?D<a&jE|+4Q;QDX~k<@9n5rgq0^G&%>Sx
zuxkVQEcYnc8%}N<LE+#4H-yEx&3pLM^id_+9~%QiZwy>gnxx3nmo%B7vC?3B47}pM
z;^HC#``{O#8fYiIY0er8nHKDUbWb&)qvo4Q{+;uc(Kv1J4MEiVK<`%r*j=2d>^7~I
zyKn*z+M;v}CmYRiV5Z}8fTs+&M$!O$eZu{F&PQf_YrNb?+vCV+vQANG#3ttUeFCw{
znbdb?YHA3(0AW+q*Viw31BhW*Fw03LYGCt71*B3VB{I1(sVjgJtJ1BOr%?PHW(3IV
zT1utojZX}jGw*(|N;NlaXR?VTyB`$lCsr|O{IoR*Xr}#UFwjlwxsE_(uzsX^oBX_u
zG#f|fRn%yy4nxQ(^V3Tn=w}8JJvr#pqv=Cw%omqU3g<ciHOSl;Gc6@M$h!$}(HH&y
z0M1Ut$BSQs+AUHRAu}_`UpgcgTejVq>op)cLxHha<BdcIYQ3jjOvj@PlB+bN0l%3!
zS__-gyCqb(jiwpMTIgYdP`!$u_075+JCZjJ|1H<7bpy*H=et}y>yYa_^#ilt_6Z&Y
zZofFM^A`Ts!=wTa2o*U1OiZ4ipO1~ZJ%&*#59}!*x|CA285WS*b${*74923Nk{{iF
z`I53Ik;5(r<4CB`-8Ycg4)_cJP(;NMhD@)%1Gm32AXu3A1uUv^QfcP{*~aO)319`X
z+AfL-3(Xe^0DaX9fV~92q>w6V{r#d(HaPDGkT3QY#1xkqgY>os)_jhStPj3Jrq^lG
zq&A2`4+&$3Dt};>p<2l9Pa#7Wi{3>l#M01%%vbBF3?pL$J3{Hs&9w+xtB(jMfe(Pu
z>05AA!lAwlh)Z57y#=JP0c3^-zVh;Nm8qTd6|z;=Xwm`*19+E{(@o})oP4QRTBu29
zWP~mbA_<h%1Q79Z?p{J_H{f}ocqq&Ga8u~NIAOg!n2!0770Dut_d%on%8|`2Dw^)K
z-+T7#9jM0rIT3>^Uq<p{U*dVrq}d8RgMRz3Sg)BpSlXo1$o&B`%_yYlolJ_>JoNxv
zvm--R<>sNv73XMe*LpwWusVKo?4t|REZeW0mq^P4L{pCVD{mD)S>zvA+J(QBBuKvK
zYEAleftrI9Q|W%dokp~@@qk8>>j8ZSU_L?I-TG&-t@wr5?Z>mBd-n?5O)NPbHxeYD
zfsItT9Cqx_!yG1V9`Y&(^|%13sMdQMCt@BRZS=kdu05mRpq?}R_Nrl4v#An#8(*BM
z;X+l0B%lX*kVbDi@xiH-v;d&&o|c6ZmZeze>r-6Ec?v(I#M<R;A^I1A37kC8DsD_Z
zoJeG|ObVo8QDa3fLYRM3{QO6fPgWx&h|B=1;)Q}RI0A$;Lgaq_O4J6p97fkN(2{d>
zoW1&T<@fVHClhEGWsKr~!XpU?kM4+5e5z1*4E;xVyk?yLg7co*3eO(<(V=+Mjqb8Z
z&(bEq>mI{D^)z=N&3bTkgGk-wJ<`pq2zHG4o6`&>b7bWiLxx=8H~R1PvI|u8&BCmJ
zD=3a0<$}4vTnRd`7Z$VY_8WX6RK)F<3IXEu|J<Rxq41CYO<3S-!ZaNzzyebpdAvJc
zblLOM#N_*n-JU|S*?q6@3kkijA{tBWhL`+7nH3=YyNGqQwY3G2HG~2}>~Fqe2CVsW
zYrsDja^P?~DAID5bu3URmKe&F%@rjozpG<QDK=bba*3Lmy@!N3KpH%6Q*OGn>~jaG
zadNUL3ci14G5!jQ!Q0f19%uWika1UR9?M`E#(>E9_r2|ch#;~x=cJ<6B%F~S^v|o(
z0YI5|(hH<ty`>!<)r1PNjMVxM!*zFm*35F6{}?m6MlWD=_b%%#0bc+<`=fpIdvX2I
zz4{PA4bL*{&p8MRLjQkY7nLW1$M5#u^Fzoo14dvNa@Kii$AAI1fpDJHE@%BKk`PEG
zhZ(@K+P9FnH)fq^e{PRWK{-d;!qr`N+3>2sH-$g<F@<FucF`)ahIXsm%e|{imZz|f
zSfE1t+Nu;&oI)am$VyrQ+a~1KMCOB}6JzFg<O?1j$%`0oLJxKh#^1QO_QIR#`Xv2D
zFA-hg(Sb3=d>jx&EGL=k>yz4K#{gh{ne#`1mmlnz<K}}&tOFp26)T>?{+?$D*9m$X
z2m=m&`Qzgq!t@WDP0!-%0EbNreQJ?2GT=3M3Tfe!K~1M#9H_Uy5l2=;ty)3=v3Y0;
z&|&PYl0!Mts}LP_Y8{Z^M8Pm!ac$e811STo&sz~NUQL1HMmO6<fUwX`!gEtyLG;mk
z*sb{Jmi~A(7OOn^D1Ao&zBFS1y#guofH|o|KSB4`uS#ETzG%A|$@1w3ltcH0uM7h!
zE{a|;UbkLooec1`V7lY-(*r<EMw0)40RITD4B#Ek9||)xDtgf<e{tKxFU|lglYU-L
zlYux%_1Iq#!swG_<CLpr*wbR9^9x;1>0)WHhjZRVN9X3Ezspsp;!dR!T-<otgEpm3
z+V{0;Q$xs$U}jmkPF)?FV%hS%gU5zqS$akkVnKKllg}0A!2(rcc!>z9J(O*2M0;F%
zn;EysXyj{aeFK6tPe0S02Qf#9il`rwWwlXBScuvXN<Sd;tsY<^jX=q&bzX0VzZG;B
z8|g9jz^~>^inhGp(%+RTq-qs9l37g9H;47cUR*c|<!@7(jNgV{vElDYNE`!!{9k2^
zD*{k(3otb9Meld;tTJNZf{juV1Rp<%DFFBG@h(?X3c&WWwqNW-2Xecx@p#*Z=iX6|
z%J089>JS;Y0J69|&_l<jq-@BMY<(CQ80gmQ2qhHUzP0d^bW|x{!OyuN1sJJH0Nzaw
z0LoU^s-Q_bz_R6nFeuu;31KqTfC31~(-z@VM@I;}c?MpT^9I?dTApyon`6?^Krzll
zb@eJWp+!8<Za?`LS<<zvNO8qY+CjQme+9{xU5?>Dle7JuqDc{htcHZvs4D=nn&Xwb
z+~v$CWm;w<j33;KUxM%rzYl^u%7@)eQ^s)mkH7CRU6=RZX(7VY<^Op{F?w~ug*w=D
zv|~E!GQr(|^FgRekpyOPFD391BG&%<YTkgiuL;D3Oh@8;N#W2hTxyTn_JCn+!ICRz
zhr<HZ$cCWN0N@c;0gYLKW?jW_d}o`2Pv&N+PIDSCMH0qg&{q1=Q2ue?#f<X;3b0^h
zKBGZ+oRVGUf2gjs3-ZvzwR+JALU0w~Ve=?Y-*>;T{ThnP^cvbnmjVblYHBP%^O(V3
zK@+nlRHSFqSEOEP3Q#VIc4E}}{STDaQSA}&y~`q3Ao4P*2?bDJb-i9*st_`v+(7tn
z3`8}Ne^9rsm1}%F7*x+mL+cwKi2h6qlb^A88zIO)_V+^YixC4A;#&rpckdKqsJ}DY
zAM!f~Gl3zvH!;PZdYV~`mpB3Y8qWILm^ml5fZdY$j>VG`@`iNw2OAs(z>L=@2=TE<
z^p#sco?5y7-bQ8LAQVDsrd6~Os)7Z^9YvmVKH@$=2C~0+AmlO0ex|q%R8Ma}ND{T2
ztjYkAJvYM-0J&pzSOb_~ezevJD69It;BG?R0gg*)8z2R}(FR(J>U=`Iu;(0sY}bS;
z(yD~J(7QySgK%p&<Qq5th!l7!!13}-!-ePo`u|Hvxq~1PkDB|WuR~!ySD%XQNr%nG
z9l<uVtGNmI6F!m;Sog3!k$s0%sj$+DXTa-tXU?9<5Q@`?obb<kDad+h*eQHAv%Ha@
zp6Q=oe0=D&z}fEwFzoG4II8AtzHoeP)fnf;%CQPT@xUNM4h-%g84IM0&d<;1z<RB_
z&U<kb0TYNw_yVx4xIA`%yYVx!TqM4}f<4X!Z<7J5#2eoI6kw)9VvnT%<NA)yJp$p=
zEx=s6*o+%ha_S#|BT3PK<lTL~mI%ZLPY6YRy0t4LmoO<6Xx0*i-4ld<+2t1^0z5R5
zsW-~6GPb#^_2D%l%==|8qX_iRFGcvi)EQn<!dU7dQ^&vg>FS$bSa7Q7=C4M0kC7f-
zTaWnOclr7N>OM~6p;_=nDv8LSxr(<qXhtr$tRZU6KY`Id?-Lct7g<{REJuECg@tLj
z%;kZwFiCJ2Q5xmn>GA1nK2V%GT8ZcG6Xgs>Obt>a!<n7VH|U~47b6nq?|s>|1;We!
z24NseN${BJz_m9!rg6DMgU;cNYNaVEd>!HyAQ)rrMRonbIG_W?L&VOBaP^_Ri637c
zDBb_-KrJJ%2TNDpXByrKGOcfv$$lGw-S+o_{lfA#IAJjz>UgoYw5BAKyNzYPcUcAS
zkp6k(21Os=oBkii=<-OhQxN;XvZ>7J1k`pP`od(yGZ;&+?R>h{U-c7Seny}o1AX73
zJ9*jPf?QV;Tx$*d^U7ifMES=^{SSx&AJ`)OG(fS<SYD|{{=xJJW3)spc7|o--zPxe
zb;}a6`rdBuy~#s&&Y!2usZUY`O_L(ow;`DhpV;55g%?dkB!&Oq7yHz|1)^jE<Xxe|
zClE$+*COHCh7K<#_m17w;U&D}`0$UJ!e93~#9*vDW{Sdo<SYi#!u>D*XjreA9tezZ
z*prZ#D9tP(e_VmETwOQ1(tCY&ECwtYy;FLxt8oO>49g+V&G&?<J~19R8dA4EP@+6@
zA&rrF^d)=Y4hu;(QXWgtWsS@?^S5sPI_1J={@-UDBxrOwDAS6^o_$yf0?EO(+RTL|
zcoYc2y{;Zk8UH}is~;o6gh)#ZdVi7;8mQxHMNXEfM?tJ6`v-gZ&zJ^8W9Hzzap~_8
zLdlt*+VU7LXz^vX{o3;QJ~9R~JP;|r6{tVoT6Ccwlxfh~&YRhp#~2(;865avDoRg+
zkdWkntd&X=q4WDs#@<i7y+>dCg}L;)Jr^C(1U|)H-AV!f8zCCMiKy#@CsZ7Ty%OlQ
z`&wOc9G-zut%Zp(nvtPe;O-u4d8$=Ll@i7UU2Kv-Y!aRAHp9~Gfqfz5X|am_eRJ&k
zH<7|)C2Casrwf6s7uwwzmV&oolvf?<+-t8lc3pv$cV)zJyyP#b!GDtt^vxV5JU}y-
z?+?T)tAmrk^N}z*^;+!{{TMC=s)68mFDKP~Iey=-9|67=OaSKc%4Ur2mJyWyyQBIW
zh&su<u~B)Y04818q0*cBOf<R1YQL5n|8M()fCERr9DQk)@NvJc#p3S)zy5}}A2F6r
zJsM(_-hT#sJ%Gb$;ikb&gs-Gm66{d*-PezxyNjp{Uw*5I;X+y4pNhCMMJKdVd~<nm
z;!aUO+=6uz2b){OLp9;1iqS#?ZRPN;0}ib6TYpaVoF%+5ce9IIN`|oxIM)@7|DM<V
zy<}i;2j*g~*~+vL#TLKkk?WneGdq3`Tsizfv(MVU=k;Dm-!0?IIP&+WufGBALRUw#
zS2XB)rqPIy=FSoBdOu=FV|vf^jRi@#*|?t0UHfd^j&eexKB<tLn{OR5V%tgA^!_9K
zF~!7XF%EUSMX-;5Y2o_6sHy+n2}W<PTzPY};?D@b881cA;Jxr)ClI62%xL1akkx+&
zIPf<Jc-OunukSJBT!qEQa%Ez;#gnSB^T--xJ-=VwptS$}MKz-TT}(})E)wORTW62R
zrxxS6{2W}<@a})diodUxz}GSa74-)2uM)1`uQxH-Rp@0lmlu!x{>lEo&&B05`E}^^
zb>#QQ8M@5BuSoh0)Cl&SDK0Wc7mWs&!)JsaH+xbsQ2xy~0?{{oVMk7r(7-(JA5G_V
zdG@zb@DfJ|6)EPg!jEZbyLVkdfJ<?CSmV?~B>rBF|Gv3shJOs*m>Uu<#}A9h-h5%t
zWA<dJIe)3+ba#h#;#jR*W?}HD;1e(3&jBD<{QvfT9(t#>Ev!t$-+XK)ZTy_|r^eYm
z3LW|+>#zNn(o*QAGs7X8?`&<n00bfV|7BM5Hw<GUt%O0HHl;uS_4Qu@HOEx}R*4;l
zfTu0>FRD6pFNr&^vpS}sGt=b*-1+p<I|`C<zar7TyLz#@GAg?IcDC^3g73-+2b_&x
zituQoi?cz{3@MBG|KmBO#OQxmeS5%6%J=>s=o#FYtKP$yo(Ad9z4?0xufO4oM8CF%
z>ingDNhpmhMQ{xZ%-R$@<2CD<XmePp`6;~<hsAka%L13i8+y{`sr*psLG$lg_CGnF
z8P4hQ0W_RryyD~iFApGJ6Gooa2e$2IfZPlL;sX^K8JSLpPw!P@)*wnuM;E%;eBlA=
zaEwP>&XItWw%FLKtNzp*2+e`f26`No{A<7$7YF0qfX(-LxW5s=W=_7E8CTjz9Japq
z3E26tLnwYd_QM08m+JbUetdCNZZbyjeb*n&gWbA&Y(1Q$<Jk)woJ-|hlU#QFDbNtH
zz?f4f+B<6Yv*E+*sorNq=k=3u4acyaaKu*ptaiCeSG%i6Rf=@(351g+AmAHcKfJ_%
zN)1rTQ)gW$-h2vu;qBE2HcNBFXghzuRqmz8ZtZt?0Q~B+z(i{ZSUSDp!M4A4VbOvI
z2yK<iuCdu(%gt_7$8S@hJ;?_&jM(Vt`RQK0X$o)P6gdcnK6z$w1*19&!aaB_#{D4-
zO)C{B_Yv-R1A!o<cCI#?PHI(?B-{K1ytDWr7%MwFdoF?87!IC9=GuoBjrUQ)rAjwJ
zCjBsveT}W1H)~Y0**HU6x^%FPw^Hiu(|fP^r~v)+B6|1f&QyYh&-Fkb7tY$9$n8r>
z`2@e3K`hSNbx#OlidQ~@rZ5yu6q(vC75>mOMgc{WC?eFa&y_G@Bblhteld9i7WNRB
z7TN7zoA;@f*gm*Anh0!g*a{Wt3d)Mcvbol?O%UDw&cPzQ^Pc1zx(tXwlk^cdstN*h
zuSYrj^&ClmvBuM5ppUebHXSe1V+FQ=noCdtjmY)Khmepz6!)?jNcQ3tEymlmz;LgD
z%P>1V!So$CVu33@Vc@VLYYyvTy9AkUAcUg-8a*tp*}zYJ<O>sV$+)~qSMM%CqD6Ta
zU#O4?izW2B<KmOMEJBaATZ)7!_>u5p-~lW!=Qj-KmuXt1rxklyqTzDI&#O1O%WuLz
zM*tdCA+WNxx-|OihkN;Jo!EdNhg0mkIC@bkrRsuUXz;!?gd6SGO?Uu>WdXuoK@uij
zm*k_Q-&7jy7r*UJzmyx;1Eq680KAcpUktbFUMK7u7+xrB%{1D@Wwks-IM|w%b((gs
z!4MYn*kt!SKk|kwlxOe3H-Z2SfSAEZ62XenSdOF`S)Q|4PVRH(rqHh0XDFDn9|40i
z6N5scAN;;R9o)D$znZd>6Bp#A?ZhiXv+iYhyUTcBjlROd8D8JCq9bpv(iktW6<+tf
zn7Zu=qAvn;IgwgAtqm$ruAL(d5E}N~H|RfnHol`5Ca;}$Y1(y_PI53`hC*RAhs>Y<
zPgs?>46D3^3DtSzktDpoT5h&m=e}9}u+@|Wh?Ct~5xc<BBObDhdB6O$@0kc09<K*i
z`8cQ3T}<JI->Eoz>VY^#964<Ps6%_JLytm6&XQo}G{`qc;2%E({jB{`H`Xzrx32(w
zTy|;mdD0V|rj6_zM|t1^vI0DrM@zC7Rsb5Z(&Bxy&g{MUSU6xKXT4z&C9~cE$TJi7
z?pGEwHxJA+RNm9{&66*m$s)J}04CZ;7_NNU^413oUeX9({o?n;%aQH)xJw9}hDP!g
z13)Ab5EmEU1F$j1+cua0+WZR$w}4rUN2-))uI3{(TOC3xiv*>KFq&_4_C>9K2hztR
zFrHYchY$DS19DR)1HiavrBdDE7!!Uo;!8arU?8Ao*YU_{v0#1c7T|tmL3Tp-hDb>b
zq$tB9(S!2=i>TXaP|Imr*r+>JepCMUWM}IT@UiVOAb-^y9~~_KNWWPc4}q0zI*2nt
zt_O!Y5JLjcb~`QIlmJ@_va|w7R0z_z4)0M@%&mSg6?kBUl^c~MBCloR2|UnX!69K}
zA;0auHz$DE$nnp^S<{IMBM5h@2VQ7p-d;M_iRN?ZoGv4D*U`u?-2;?C0@Q3cJA36J
zy%q0{$NFo)JT@v#hm5b87zkpjHk%jkg`KidPF-sU8AVz&W;<po%q<+zMsb(?j{j4S
z=yJk!T-H^b`nZ1@<@C#`?9PHWC}H67m_yijG<Fl$sHZ9%kgLH=pidFda>&bHicn&I
z{PnQR@)B+XOkDk++6NM>$*@;!$(cdlV(LaDfYk@kmW?MV)wtXzsBY!yI!LjbCrbF_
z=y;)?s18Q%+VfO!4KHJH7}muJ<h_KMz%*VEc&L(m$<{hKhg`-W0g~mK##1|NLXD~1
zoE!{>m1ltWUQ*wC7eql(U?zkWQ1sIcfO-TJEwc(40)8ui$J?#ZOhHXZhFs;Wr=Lf`
zvvun903H2FPTHAzx%%Q)Pellc-R#R>1S=CbNP3U_WztT=zz~Hx#D_-J7lw+Y7p4{c
zmZq0O9a7NVZpEiXJ00soRP%!|<h@m1HyJ%5_B>j`DUi?d-}Uj(?4FMTk%uN^aA{1V
zzP4nnRJAX^e>fsHdO73Mr%wYQ=j0XFGAvj>Hb3?}C_#|RS-bFjyYzh`z9L>Vy9!A-
znl4U3TsXrb!(+eZof2twtQ_?_Q+~sU>}XPnkPVFx3Ui}bLPD2>i$l9jKvVzG^DXh&
zR(Q?VLyei~-s3tz4*l_MYx&iyGnMC)FWznLf`ut7A?ViYBZax2g`h(S<~4p{!1I7s
z8JpXx#Sv)of*Tybbt>DY;5!gb2fMZ)FI5_aY(&*oQ%}M^bxQW9>7PPyOlE`Yly;zP
zdH~3aO~UmEXxURV+W?z@@IP-bH2i``!DDXUc9y0f_C)Z;C7v542-L7+X3EPX2S)wy
zuf2#{5V<BxZ*Y&3?JApx6CTLriinyX%E<4<_){YqK7jkM9WOK0Rm1d1^gZ~WiNVkG
zq6I30l|lBYBQl;GcKDL5$k4BW{^jyNwZUF0#pu=es(Nwl2ypy-XlTGmks!=#dNDdC
z;*J$ZoS{h>L(^}4$Ff-q6+PfMEP>a?g}&4a*qr(90r&mws7?|bHU|@VPJanD56_L4
z2EyJ{YUA+mEZEUjYaVcvZi_$D?*ZBy37x)VlOLsyr|#25T}daL7r^`wL&Z3t(w2Rw
z9iJ?P0n<))YsE4Up$*ggRj(U}e!*%K9pt0VyL|=i_$><g3KXuupfDd0OAwZ|iVjk-
z#Q8|9GR$)y!pww#rL9=Jwl3ycOW!z{xD5FrJ7fYMEs5>hg@t{|hbWll%rcqCajM2?
z))N))NKJ1ket1pRtgN>-B+L5_oBp|>u?>YaKQR@2SR#Q6Fws}6>&aCTl<3M<Wr$P`
zb24&NF5>^hf864>7ff3sw`K#>w~A*<a&or6x&Wtr5NNqNVfVPtw^O(a5tTsr3d9Y9
zM1>8d%kmkmeh)~9G4xR|?$q5jbCP5Cks$6w0M<Cc>^evZ?&x{Sh1k%DK7Yx(og2*L
zlb$(Q9TZ1OQuM_$5S@W%wLs?}tkYq>2|cMU*U9;0+czQ~*c@<{S={oL5^gExK&$y-
z$c%NbOY`d1h4RU(6eqs9rDT3w#Vx_>POkvtsLdB2=_P@8Twu_;_jvZRJ;B7?tGmxG
zNUU;ZQ#9zH^31>yf9SdXKaU0?d}83`;YgBdK#H#Zb7mrY(8`iEZFja?=yRhjZok#B
zv0LgE2H^g?Z1N)g1U@-SYPCG#R4$i6hLoY3>}gt_?vLUq=K+F{3}!TI@n0cEDxkd{
zrtP}Eb$RM>55Jjh^_=sM7l4|`jZ^7+g=#k+^OgD($c#n5xuD*P;^~??yK^uEr4CSb
zmA6YYuZtE9Y%4Q<@itP;o%8G#1gYrWn$(735x*4KsJhC~$KZ1{Wv&!z{T?cnqHTd~
zz(T<cl+{<q@ivl-moGLq&xcPl5;e=?Jp=z@Pd{r{^eXL{ETO}*3&Z6#vv@7+0h$$%
z-*W&}c>u7*3I_c$hd{DSbKnn@@8YY6rsfl$Koza{z+B(An*8B21U>=vFL!m4UPq+{
z!O8`xPqH-ulY6qxAQ#^=!;bUSA|sTlgH7#$Km~JV>iCS;>KpqzO-%8hj0Z^_<O`X?
zAq8{99YPXCLFDd8WQgsU@Yap{*bYBOrT0IKGI5FhULP$1YmFlEKOr%y(1OhkaA6d>
z08U>`b(#r!+?idX{s6A#2j91(n66Al@$$>3xf31nwA2%$di4$O&>FMRusA-97&f#M
zA9~9>#oufCx8&H!{}n9)VKUp%zrcZ*!+@udT+Y0Uvy?fAWh<XwQJ1IIrRj@Bb{9)L
zJzD@tDP0$Si8hii=*W40x*g$MG&Vqa$!=LDU9-+5IUw*Xjmo<@1^=Qf>rjg(o1E+_
z@<jeb*VU19EL$+kD-P(@{~|H95#MpY@^dqb30JIR@CqvfcC2H!VD32?XX2XtML5wu
z+W8}nm5jENzQHCflNYFx4i}C4bdSE#Mo&*Pd5hh;`nW$3$EBZe968UuQ50tjh<QVA
z@Yq8;_4!nss&>B0<FSqb3)GES*-w&hlzL5wZ@1*{f;i?Ojzyysn4XSqN{qbNimF}8
z?`{n2*N*$YO%|uj0Fh_Xmv%TEn)dd%+s2xC(|px8d%J!+jVP<im+OBTcDR>e-vE?D
z1?vFSWnQ?w`5WEb`B$~=5)B~HsLI88x6~g01~I7$Y;SO1so5tRO@FDaQEN)VKS7?t
zeY}#jH{-UI__3lwKTN%oeUe(aAeOY|?2ric^Tf__pR6e=apT9A7bk~Xr3c%~AP;4K
zZ}U$Ms{(#Ad!S-W81wn5%j5&uue`f?D#qT%iwKH6k|a(Cv$8pkV}g`nz_4m{U`^XG
z4T8zFeQe|eWYzG`k~Hx!^cFYsN0?yTSS~^q#r+--gAUhgtbt>t@T=hk)JQ^pkKx-!
z#co@5s*kK@%Ts}=VK#%`x7NA~Aa`TxV7qglARSOzkE@d-qbk4q0a~A!X9~)eh6=SU
z;9DXZxKsJus)T$5n$CV`EJPoQ!YlE-b)H8`;&ywl6U%JW-4E&J!A7dxXU$H1gaw^<
z<#J`Vg)9&yB_uM3m27xW;385Sk*-g)P5$%f(CY|v3oflBeKKrEVJtp+<#qRlZY<Kl
z)Xvawv3l_Fc?!~m<eS%g;X*A|r|)Pt7N-)z$uzFg8k`AYdjW5_a(%Cv$tw<I9>E{1
z6i5u<I8I=|edrQAq*=@$ednZDzB83Cmgt$^dGvu=wkfpucwKvfoZ|Xd@Dp<)49fh-
z&2syq(7kDEzw#E@x%uAX#j^?mt!VTPuyR4CU5_ID>YeiS3b$C<c6D{!{4ndcz1QpQ
zDjsm~sI#rqu3<m}I0e3(4;__Y&M@W$cfoS1J}jC+Vl+Un2JDJp62FD`s*55%@=*|_
z+H2y`1etuUo4CE`)nNH1uw>4WPPXsrNn{BCoy42X1|B&5p3a(CHc>&aaN-fn>YJKQ
z@2}AaH3M-J%w(r&m&fkF8um@)s#BbP&xazl1sKVwDLJzf^5Z$MDnJf2YEV+G0^949
z)A<69poSiZb1bp3$*V!rk{nDc=MKrfT)1D}X-@_FV5oL*OT1S`fv_Wq-9}%xzm}aX
z5T-4F{;a<@1x8Er3BQP|bZa(SJZ2vLS-#;Yf6JJxb1j2dI*RAHczn5QoZ6_!jouSb
zMHxkZ<&Mq9qOa|Y;Mze1y7lK+9)K=w54M6GD=$Juhjn%tttNZloju1nmdEnseXzg2
z?aNM@H`Bcn%@_y=4Dxp*pXL&H>DB{?0wT66EP7*PB@m+_!Q+yl5OO7OI&FSkOBK%+
z@AbABc>ilAK<_lwb`ep_t_Ox-D@~OQ=G4HAq<f1wma!S2%FTA8KHz<!dpPlxcZUyK
zhmoDU_EJRnmUPp`T;}tOBZ`LW8*7TdmzQs6r|tSm7emWg{0EwuTz#ydNxov%gUQ_*
zx<TAyt1c}577i%orY6N$8*qkO+`6i+Qb^DW&)HR0ISW(K%v0tmzTfgD2ms}lo(Qc*
z*&+{(+XN>~TtgH_jG<qC=sflI^^aAs!RIO<51tn*n;sIW0%87gM^)MO;v#_}F@??L
z;j93!79%OcLpXl+BsAA4T%iTG_b#rs+h*PJ3J8A#0pT@3jQb9>TR{sS_v!dKB0oma
zDcD7TOgoIA9Gp5eAzvNcoMl$FV-G~09LCsH><y$Tk6b<~74bVvG-)_n1;RSlLGo7h
zMbgFY20P;)V4Ali>0|1Lc?%k!-*Wj0*6u0N#xJ}Fo}Sa>qu+ouSHkGVl8?`0C3$#D
zaLnvyT$c0SPY?Pds%8oyJ#P^zT~pO6-sDx$_XLu#7vE}ayv0BFl*g(0h&qmbO_WDB
zPnUqDY>{jd1xf?<KFgNgH!o_e9##atBK0r9uo<h=^HZN7*od!M%!{YwQlEk33}Rck
z0!sxXGk3Y{vT&B$_i*`c-np-oPxrh^jQWdOjBo1<fVp}UpKG;o=}pufuN7h?Vvw8c
z*-!^W^!L<!BT&HV!+=d~3VbwKe^b21eTTUf)QFF276&ZcF#BviX59TfZV*AO+%o?u
z(d)DSwh9&4*Vx(*Rz2V^f{u7kub)1!-_3C=<JHk_;uwDpq1923k@m06bn6>7_ZJz_
z)<prI`Na!5mM4^mwSit5cEnaaUA1a3af!?FtycJkZ_>JTmt=&uoFc_8BCHk+f?geb
z%-Z`>!|We4b(<P10`Kxv<^Nx&YMUBww;L+t&~iaCBow(N{yIQH`8b3|voTcJ%uzVb
zR|o^)y40_D22btQJva-tA)i~ux!hL%a8#>W=xl52hzb)IzkEk2(BXu70mo+Kh1Jxo
z5iLe}|C&RP(elBgvWkCS>50^1=KFvpEF?ULcrUQlpu?m6eku}8BZ;6tl~F*;%ZzYX
z`DdyK06EkN6$=s4yeiwx8~Rq-%$6EnV?8VLvLUp+sx_pSDDM^5xq;bihI{`CWx0sf
ze!tT!y}fd|G1#E8_o{h0EE!V`{>dU)1djk%Q{^#-wW~HJikZrstadY3C$dzR*p<BH
zCb2L7rzL=<+_!=4tv<A>{itNhpQn2ymCP0fK-hE)03Jjr*m%>l08P3Sj+Q<8Qg>>y
z<Jk%7&++mYZO=0YMS5cUlJBKTg(_h)uIuzxL}3lyb(^nH3w2`B)O-$VOb(yWXuQ{~
z_XRp+zW85u-nI14Z>uKIs1zmGmiXJkeO=&?)T-wa00uc3I<C$taysLX@dNMaubZg=
z``Q~;lpO%iNnuL!Q|n17gBc>hm%m4E$34oUQ);X#pt-Y&6<d-Xj@gY5x*Fv|TW7XB
zdapSH(Yr&dUP1`W(QIhlkYd4o#nc?lJjZ!~y#MPxG;{5T=lB<kOs<^6H-j->rUY>M
zj@Jp#0c~U!NDu{5py1Sj(yrMD0MDfZxn-0@JkQ&>Tkp}+PVyBSgM|t3gH4HrwQQE+
zlp;s&0vdH`prvBY3R`p&p&T{e$=GdW_&hPc%5UFt=Mo2UzT`)wT7XnC>0o?v!V*F|
z`ptE{xWKx4lu4~QbRC#JqJTYjI-}|6KWBm7rC{ssy`o(X_v5PB4;5ljq{EGFP6YsK
z*J?lWpil|oZ$|Jd1AsERSceL;q;7o_VOE6udhye`Vmx;uPAMNX#ZIYKi6W%Y$G2$U
z{7FaCwdf3Ot-BPK^S5OsB!U$R)B#g7G2UVM;~gUMIkaw8$J+ZwC1gc|-`js}Xn7nh
z{cxLEeRf8uascEWp@~b7&1#-vy51`g46%8_wtX|^_mCqVr87n=D!alny>xvwB>fcq
zGr42V7Vk`r^A=|{<m8%Ok7JRNJM`HNQ^J)ovWL*~)CMTq2_qz<)$8&xQn~HkoPRIJ
z{k%<l4t%;NUoZ{&-BBkrC@+MqCJ3f4`I3&#9Zl6(zt!RHe{#4n5%c6{3ntJ%FFRp=
z*6Z|skdMDmDCgfq0_uQJBe-;t_uj5N14Fs3cr=+QI0yuPP=NcvPx!C10X&*+Ntt+F
zUApZL3^V%y$}leREtF`#02pBf7Q(m7+Gp2h$5N^zs`$ESVnNU!A?l#fGN~S7DUvY|
zcLq75lL-nCoW+LXl*8mYhh!93PYIY)KF32v|0O`cCnHF_FDE|ve&e=Eh2HD15NuhQ
z-Y2b;;z&^Y7`gDL5!`gr<7%{{QOe^3rs}LyT|l0rk3sY`rr~D5X}~icixmBfGQeBS
zwu}8FoHxyNvQa6*?s)2%_frtTq*F{9)7;*fod<0C7AF#^8Zjh6Lk)a1TT`hoo!y@F
z>S(9j=Z6X<Ypk*h2@-pKs%pUlAW0zAI#<z0X?<eo7YV|7E;7zfq$0KH8Xev?iU^j3
zvm=XoU_sWi2(2%N{C*cYB9fPjo+YU*_`T+IT1^Sp1g`Gu?o{r4#S|FNUac@<G3XVM
z6N)=O`VsbYMsDrK@woww+L(^t_k%)Z{eBgYxKh(C@$}HG`;i$E%;0I7+IU}T5_Z0@
zKWQ#Oz=#-Y9+bLem<8mM1q)X-?}0|O=UTI(2W{#)qT6A)KFTwhwmGT$q85dEarW6Z
zmc{1EC!+qQcV6nERDqF6LNLAQGBJwRR|HW7+%;4mNh%d5PiVCQFv;D6AA8*2TV`iL
z<Z%#y@(|lu{E<)k=zB|z%5zXYgJtrR2q@;<NVff;a{Jm^q*e3AOP$piAQq>(G8<9Z
z4~rh%EWCN=K(V>$EcRxzBX&4BH&P4}*oU^!kxhv^Nh8JvZFv9k<bzUbInM>zEMYz(
z7An^-ue(`k;zCv9cZgoK2vt3K!_uVHFk4h$6%{S|y~>?5ZkB|K_+cO+AhE;l6Pzoz
z<EzN^zZnUFDdtM&p1?SIH9WNrT7H79SNXv1#yhu-@S?&nh_oxAEBWd4%7rX}NnC=!
zEoDw(O%y{vTr@O6ycrbYBO&vyt}Y47Ot4jJb%ysg$Q#ImH3#`5SmtaNtii2V?Y|MW
zJPN(Z;XXACF2}3nW;lCu>Pq2s10y+to#_Df(Q{9ry<4=mIPH3jnJ=Aur{|bK>%G!+
zmfTW#8iQsn<4-*YHm;{@X|{TVn|q9yH671=^4&wX-v6paF>bst;WkM?J8z<?p&TsW
z+Yv=y65O}JCv?Av_x-Jv-sU&yu|J{;ZCgt|)bV4njzmW`utJ+m$2|gSTN)ToYFe+(
z?1f6P#t}N1L#<d1XsJ{eVgQnTCk!K4?WR83@x93Ka@5y&$7f>q?xu~gB&K)<Q?1Wr
zeNgy(4-3cr<^yxx(PJOFV_@BuH~TnazT1VLSmC<-q#U%BWWVl0h~SBbhb>$0!fr<A
z&3{t&k@FcW)^JU#pG_4okvZw6af7;c!pkxt<o_xYn3@Zg-=%zxgC6$Co8U?23axUz
zCApE?z2B6TX><|J8tjYtr%sr^>rM8dd*}zz!=Y`TcbfS%ilX;bnd5~1lo15~C_QVO
zdOr#V#QIRDjHSO%PhBg1>e+T%rAs$DWMnSeU-_{3_DC?8n5d?<1`od+?x0zU!ViQY
zoyMt^j;5gNhiRj`fgYe?XWuUh+xuSb(RTuEw;Qf2$Ls5Z7>?giSZT{|k2QtBmFMU*
zH>Fg_zxl9PFUfIR+m-rRXyJr(1gRwg8D~0Il3JG59;p7KVSzBJ4?C=^o2=byf)7s!
z!48y!pJvB_m|xQwZ{87dzX)S=r5W$M)sD|@-x`Eh=+g$jJ7zNH_2{k++0Qx8?_Lub
zf_Ij@T+WYfql@I`b4#0i{jJ3D%luOatvZJRN{j`F0CTUAPXn!qdsfQCvNlm1&Ioe9
zhu$-JNoBTjY%t;hXyy}O@UzNO+IE6>;Gx!J!7=iqmeZ3gyk0$Lw@^G*F>;5fB$q8x
z9L!`g1D%djW;{;I<JEjH35pe$W%hsudG!lP&59tH%e>IOgGAldj;~CCTDb9bPqB@O
z1*y{Kw8Mk-%}LFg$Ntzrdh`v72kaxkp9tZ&-V#4Hg-p7Ux|Mm}A&jUURSc+1fz=ZW
zyo91IyW|C;=-c+RD$+iEY?iK~a_?wj)Uu-Y`pv)1j#HzpFjMP?QJUC(8!{PEIzO(D
zDOM?xN5$avgapfK`$&&hKRm*w;Xj<6jgHyREaDp#dsz44l>e@8YnUSVoNFwT_7^4u
zCrbM1_^ij=lz9Fw7pyOAHI&+u$W?gOhH@j!Fl(1NJq?oiE|~OodeF&M5>4K2Ym>Qx
z&x`c}{jnqCz>&b_VX<+v-;v?-=lCS3Xq8oZDG18w#hdV%^bf{=5sc#n!sOPyU!1x-
zeY&_OhD!Nj@`?Nx7}c9s+n;b=-Og{tpyJ?2d|yr~$KJ+b)?glr;r+_{_|g2fS-N-N
z4PjxbUn?dvt{40T{1^X^uD6VfLW|nIDG8;!LAraSOS&6IQ0W{6q*1!Nq@_EC21P_c
zX{1X5X_S=ieD`?te(v|f{pox;zu_F1z4uzz`d`;VvM9r+#W@Nfi-3F*x6H!xxCI-4
zf1N(NvrM-MC4#}haW}{y7X>-wk9_(k*nfx}^>#Gda1)%!M%3vM34STPy@$ehu@1Cg
z4g+cf`>%)bwj05C^QG15ejW|JtIG=l<TWHb@X2yQ;y8elYnUqUQb^cJX862LAD;0f
zA6^D^DZ)w|^&JZ(4)m|wD2{`H%@9_i=aNi0f8OH#2qa91@Q>8qkW3W)r2@EI+2@ZU
zE_+;(KlnmVu3tT%b&8#MgfGI1`8~GH4LQq)YW#&2=YLY$B6cA3xTj9_+oRtx-WCHx
zk>K@~$zm`Q_9n+oLd)@L%j>SWa>5e7?2Yem_Ud#0?~(~TQ*2*R$)n-i8i^xu-WeaF
z7&Rw!(i|GiXZkBGp8wU_%uYY>&CiP>tmFU%&xkE0u55rBorVdnR6n5~fo9F0eWZPM
zyzbr_C6(o8?u-;#cndbJmc}uj7ZSq=&tP?)-xI*WA9Cu6`>hTyFT(3^MCkcI@-pHq
z-fDMF@&`BWasGQCo$)2wArry!#FkTK>$>gY!wT;k-fF6=7ft65KHx`Krx*m$s?FjB
z$U8H2T5XYL-n;(dn39P!8_F7HH{$t?X6BxERPakOl;kx3WO*WhV)M5v{8)Rmpo?o0
zpSESc`$FjXjL_5bwwTD+3@w?&#YQ}rSfURnGq6ADVQ6F><r8AKC$$^{y#1qs+8TM<
z`48QMewKZD65_Y%T|<a*5bnBOYc40Mq8WetPO38EUK7Sv!w*DHdN?R7N9x;+%`)}l
z+n3vyGS{)u-71GQ!v1X$KW=2}DL!ki&_B472}GNsgT0Q=lNcx~6j6|`=Z&La`50fF
zO_b`r^MR~$`-_W}hiK@)zOk40(EtQ6>_+PN!}QBa5IZS<`0v!`3=v+^r^#+GXH&dm
zo!e+9vKz(Rd^Hr~%SIaNx{jyLS=dw7a|qwIX7)ZG!-gjue{b21U6Q;LM2U$LSP}u@
zZ7ax;ejHrr3i@@_;9~I;0$2?8o}auaZLGC$7v)hdDmwO_;bhc%_l4%G-@@u}`{k@Z
ztJ4%#n+Q2aq$yk5(26BIK@rVQuy1RKJFY6baVVU%qcZoZGz!9<UW92;x`{VH1zQbV
zLqlUsa72k@28(W@<2yS(znLGMh@S}c+XO(kfx>L#3q}x;IH^t(<xW4D9NbT9M2dUr
zHN5;-<PwZUu&ka)-$TL5IWjV(*-VF`LdrmU^(I9Mn}lXH_=YdJBBP1A!Dg@ku=TIB
zyqjxstSc%Xbv!!VuMS{%3ZkF6N_|OC`lu{LW4u@9^4bL~I;pTxU%Et}FA8Y!SQh7R
zRUal!4cs&LkO>0H1VQ-0Pn{qG%&L{P%IcO(86kcZJJ|!_c_A#IQpoIlNk<!wG@rtj
zI67Z*1vgpF{W8Dz&jXo!uSmb7lQQv(t7&Q)u2Pg$P}n3tx}B@@WtE(~WJlpQR>zM(
z|0{7umf#X9{b{+q<dDW~j~%s&ZjWsrYSibbeHCLf<5=A=RNDBqJ65lmU&It7gZrn-
zuiW~-6wZNFb6Qn(@OjH%D%d9{u6#hEnXg7QSX^LzQ+JQ$+ux@f{DU4TFj?~F`tTi%
zja8ksp+vEAFIShT<J5nxgY^IJamO{>?~dA-sU5pT?8U=NDG1(ZxAyWVhk;1nB7noZ
zSz5HiEgkCgIiIlD^LIRZ&GTVf;P54rcoei_!!9}#DtHPTPa7_`z&V)j9jerxADQgh
zoHvYdJHLv6_y|0Rf^75cK=5IRojE^+5ZMQ{KM&cb$g3lt2xLHm9onZ2#ih4in&0c`
z2WwCc^&2}W1(Cnv8fv!1){XCE)4yr2BMy7z6GcNSd)MUSOUO0gb+*qk)aLxnu$9a#
zCw)nNoD73<zLc7QZs&}Tx4Xbquz)_d%H%EEn$@BA-K0zd?<H)`_Es?~%qr8ir=npe
z{-30O_PtT+3W`eB<ymG-_^hqepy6!e+#2$_E_*PYKX~Wkt`rej$Q>2rp(RB6ofeBm
z0oCTzG4qlyz065qg9A{+)ANK<NUWgmyI|4Z^FQS0S~k2ezdGBWsD2{uj>PJk(8{0#
z=Z}DJ#>k6+(dajP<@=-GuTK|hvSuL^Rv@{+JiDpmCu$R#$jBE_GWmyh63JLQ6~xic
z5e>_pUIu2GBfO^Vpr5mX+c+wd=d0T{6iqbhiZcNE=#rPJafPuG*LADO=vgmI6R~J&
z%L}#Y<$ZUP7H8jL)m^x)v+Su@1HIdi3htnUyyw6Ytd1#SN|$N!W9|Nx`j>=nTURj(
z1yIybE;igUuOBx$JZ$XpmY~Sm57%oR_k1pNZo{Nm=11o{1<ZDXMTG`%exllK8EzB~
zQ@*lh^&E*u$Kn?^E4~d$lf_`~2;MPi0#QxNCdXvk5AdguO?k)F_vS-09lh>@XW#ke
z4&!C$T@X908=p-J2*$Z%_;Po%uS=V;kzqdm9#nSkr=Yb@M^7A;8a?wD9x$6Wd}5+Z
z!ogP;{~L{yA&{*Bz#i_@J9zUQ&AgwncYE^Hlg5YDg`yzJWV={J#j~3rR0XS3PaG8&
zFk8hw5-cO<3&5IlG;NzZtsq!~91M)FnY&7lN6szjR7JdMFgSRYjU^tH-C#sLv#X>O
z+&1-`bFY?Z2<?#pfRaE@z*HT%0Fn+h8e2>kU-#$B<AblfMykAC{&ICZgdGimjk55-
zjg~Us9e*P@H2>naGNv5|a=3q9%bjJ)3*Qvh+l=l`u~s@p2%B?Yvwc!=Rm!6LCzIxc
z7D1hDfaOICsbH}JXz)K3TG283e`(P6F8)#u-#DdHYLbsJAuX#!idlK57`Y?iKn4K(
zz<o2}o3Te?=MPF&dJPd*3$koG1xnp!yk$yt!wgnkH~Uk<r}(`uFy#fkp|?Zwn};^C
z98h`xvA!3z%Ax;mh`@9PXNfL3xtFx1yK5m&^Z7G9-PG@KNB#u~tVGeaS<<q6BoaL1
zcBeqrMT$&Pp!b<{0t%utkK4x>vPTb8eC1AIBGUGV=Eo8jP+kn<%pZr3`+=o7gKiAd
zqQ1X`es+JetWnT_n;&3dA5XuxWaXTCysp7tkAo@ToHR*V!4m})OiFpiXV5-*<GAG6
z`q0yohP%Tg*8(T$VC45ZVbFhyhz14ZV{&BeW;EEY_jA@10h*|TS1Tb<CTlEpIttQ4
z{rRo4jKqrrG&%c$56v2IWOkLxC~6U}0M%5s@~C|f>QRw068n}4ZnZT0S@axK&a7p#
z#GL3#j)k~({$<0!=aq7SO?}l-Wr$m9CSp}0yz*WbCw9CQ_?oaPFFN)S8+VGraGF0>
zJd=b+?PYCT<x0^kP&95`rb7##A0+opl(<v*mQkDCUTpCy2#uZl9rVe+T+e@NlMbtX
z9|j8ia$ce@mxkfO!HK$X@AdmUg@GDifJaRTi@B(=wunvXUHN+dDu-oiJ;4zl=4MQ9
zzIfxgF_z)7_8@EW_*y9)he}vGen%3spYqXXp}KBkQkXZ$<M+Kiv3-gsl5O9a={Nl5
zxBp;fliw~&!imSt$JeS$8K-aC4Ajez?A1&_s<c6RkjkA&_5|L&XLmTN*}EuNnC(%$
zxd|;|b>ZgA<wLiim!V*pBEQ2N71mktHFw&itQUyuy(T1hgSQ@1pV{YdKq7tu111A^
zE2Caq5&n+gGY~Kqaw<G8I7*4hk5J<`RzC~w{`UVU9z`B-LWp%AZFog+SgJ#X{MzzD
zz&kztIF}*%*}|tM$Ak2D9&7KTq|EvIR87wf`t%EL5IMV{S6I?R9imgAD_SwDLqWOH
z3A8e`nAX5Gr?X1tOLI2}G~RfZI}=NFUE&lg*?~V;)27fw{Ys7xF;Fo4Faq|-ce0;p
z2-BX(!jbC~sEDm)4xBsqq?*?Yq6dm(%+T&rJlJY@H1S@~7V=a8RpbY9Uo!wdqk2<?
z`D<<uJ5T+oq{W+YlV~o^Vkl=QBEHuG^BP;1W}|7@AatQr&F{&>SSpBNG@i&HQBgf8
zPrg{~s)<ziEm4+dsfcSh#lKWy4+2<lr~MeAlqP{Q47amNG(+Sgj_&h6rS~&}_GG*3
z61cWt9#YDO5n*WuD)!er)#uEyFMqfupRJ{)gBzfe9Krew&HNwJ`=Rvdp)+{K_~Sn#
zD=P}LEKS<CUQE*Nuh>=mU-fBAdhAN5%?fAdOSaCnU2@UP_^0aPdb1s~-HLb9^=W#o
zOFwPr`z?)^ZPTTHyxXXBO>ce6u-v6O+1Xb2v%uLYY!`{g=U%V&N2dC}*U^+L(x-=K
z7`fQb_Or<})6x@uc6m)c(4u{solMr-)z*21)mks16Ig@>GWcZ^m7|HZ@|KHTbovC|
zmu=s94`ht@<?0G6!7mgYqE07ghfBwfMz8AcS!gbd$|fb~{Z%AIsX;Bn%u0v1n!N&J
z1IR~SXzKw#_U@+9Q}0KjIUD%Snc;^Le*T)75`E6CT8j(iwvzf_)-*Qs7*S;A>akL5
zF4^sLE<VFPMk2RWGT?n50sz>eIL2#4nzMq`=u)8$_)?TTxjd1`ddN=5Ho<Xm69u^v
zzd52X=@Dgw6k5HU^uagH3&J0G>hl8sIJiS3hH+tjLL2`1J~e7prRi|Sp(q6AA<UBj
zB)HFgTv~9^7h_qkRLsNI3KR;A#D!!RY0BOQKrOrIzAfoTr)-Vx6hQE6^9-H4NYY#h
z$wMx&aX-*kA*Pp$t2E`Z5Ey=C*7`~5^2V3LlfifMMZnU|k&+7*{ID6Iu0*QPm-T%=
zcA64E<Ctpiy9Qf(YtiqsVIy{iv1+aJq8NFDwKG(&zkUDwMvXcO|Bl6=-1FH?Oj&p2
zsQXI;ZUZM%bevNi`@)pb%yo)s82$IP)dH3k^78NIg6-zc#>1G2oP2Y3abTgO_2%I-
zx`RdGLSf{i+f7d<Ix6Q^vOVq}E8iNXnLAlI1pw|wVj=#X#RMADJ46;Qnfi0o*#*mZ
z$zMuh^7Qx<jSe^!-m+a$M_`6nd3F&m3-``|!L;eK@L{@-bmT3w!FEkB!V`O>WWrVi
zw+`jbHuTVjV;RUwTi^ss`QN;&CV;`CAiZ>_N7$#wV5eRY$kz|!(EWaxZX$adYTH}t
zP(%xKG<p0O-#y3pqdAmil4O0E^bm^j^D&9UmqE0zuK0rvzTF{voOiJZ;{3{My@rLH
z=CghcWmm?{omt|;^1N&s>j5djc8~a5o>&1a*cNjmQ~;m$jbMgM+bzgzEACax|1z5$
zet~l!=X)TiqbV)OflFpL{!1fD1eaNc9~HUmL6yyNYxlzQu1pXG#>m)h+U75_hp{ih
zh%RvN!ZM*21*c7Kuhk+yK&IZx?7MROsHZGV|Jh)HiRZ<4@`eECc)$C`!7cdnkF>eD
z9#w>#-;8ejp8F6!>rL0h7DN?tKfC&wD+`K88$njtNehJawV+O_@gF@Np9W=d^IKam
z7UcTgp78W{d<S^jc_C*p@6-~i?Zy&1d;db8r{M{r;y3&w6Xa-qiro-FlnRK^v~!pY
z{9*-Kyao5@e#U3h(uw#NH3=fz4)Ez-)A>q8&bWQ0gXq!^bKM*eb3aN@Y^xH{V8<xF
zjHFbLWcy3?lw%TeC(5LK>z8YyS}UgV?kdt%Z;6~LX?0DS_q`9H;TIb7bZb0UmG~<%
zY{&ndE*=c+H{F4g;*@gzmI1>ew;)t8I3^KQ-rM{>9x%2xh`Q%&7&7;whbF2qz*j`D
z?eIjrq>wt9Z;^hZ6U2F2R^P;UQ5e7jUu~ZJ7FQO}Q#2;UzYqFIQNgpZ;;&v^J6i*m
z9}6QPC2BOgc1p60F{e>8>G@|#yH_TR3k4u^dY$R|Eja1waC4Uk52R362I`8XtSUWR
zCdu?pN~aruqviv`dW1I~?t;{n`I^;tbxRv9)~+AS;T%`WW~!#-Kp=o#NL>T;J=5LF
zUd>a$P8bbGb<+tyT)0I<jXfad4n&cMh&XlJZ-CY<|A>26Zm=p1_MQBc!D+P-R6&&&
z$i4+ZAQSyR6Af*<-3;OO$X%3-p@=9J&{tg-Fvf$(f{eBP6iy`sY7}4Ba0V9e&Ep7{
zS;ZC8mVODnd;X*cnfl9Y6KqX#H(#?=Kys6CNL_*2Nyak*qUdYuCpK<!>s;*YM>D((
ztHH4~Al6VF^NeiH&dmb>or013Bx0Yfq6hdIO@DmVE{XL2Ts`|_w$)pZM#PyO+L)d#
z_W>hcUD{vxP!M&JzmlCnCs%4-kCvryt{|(R${g&C=!lYgGN7j|hRQ$g0JQt}7P0=f
zHVJ%7Tr{zPMM(8QZ@0DJk9rOGFMeq=89p&$odv%m9iYCn&b`^cTB4Qf<2DX9rtVGp
z?l7)(qQVjubQJM6H7TipSE)7hKmlwyAvS*tP!-it%u~|M0Qv%%MBEB;5Co=M!K+M;
z#+~ru)7f%##9<^8eZ(9390$1@yUS7Xhz_~1#SH$PeejD*hd<)9(<L_nzfYq?MfU(u
z@;zlg+E}rBOK-CHsquMfU-n~C-xV)_;P4WkqepS<I|c$3CY5{b#IK68_StkxXcDKv
za-+}La5{hXBzNor^{5pJ1AgzI@MK#bwLis%7c`bF79<a9&|j>)i+L0Ff}mR|)h3ze
z7R<y{6TjpM{0x(2uEcBW0r{}I#OaV*(==`n!`eciAJ&N$sEyeS!C0QkQZI7>+CKuu
zUB*JjcW`VGOr)hskT2$xza4~7+ia!glwj55=B0I$Er*hNOQa%q2pR8U#EsP}w5rx3
z%7ZK-wC!%w7Nr04gk$skdh_puTOHsRW!lbjDQlQey`Ip_{}MkAcBqcwsaFJ!#UMWI
z4=bGT8h@mcb?zIp^b><Owtj`R@AYm9@6W|wCc+rJm6Q@w<O%+}NP1=e3WM~!)VGSr
zwih6+g%hI+QfWkW7ed`xapQ7z_QGS2x5V(c4dlL_IiQHE;p%8URmRawckUJ%kV|`F
zs6imz%E~a4Wn-b?=)p4?XE90y`*lDbOmgS!oiAwRKtR%|-t40~i#s5XZ=PU+ekx|Q
z<&IynjRJ<#O<8kwJPDdFE^go`J}iOMaBXLD3u|8Rpr?NsOz3{K#+GYoz(>77rYAdt
zKyXvmngg9?5NkR#I5V%c2U>^T$s;^aA)h}UETEuhcowc3ldoDZ3emkuRAJGQI=51%
zp^_~@vB&8Iiw+lset<O{2ACq<AYr}<Uf>$)aL;)N-JEBnPDr~BJ#Mb$w_R3V!AOY)
zm9Mh*!RmedML2pr*~MM*i^zwvK)=Se2PDf3!}I78i&$X`#Rv#uP6`HQAff7;4SyIt
zQg?sev;7e{JoY>H%haJb=hD-aX}O0j$1U4RaRDmueMQ+1w}4B%B5ZFk_ucKFctunB
z3Hloh;EjfkFCuX@z$05o;ag4AMadb-5u2h%SjaS$=l7}p)tTgnz=d~dk1CU%pIj5e
znkNq1+_ma{^@518B0HVK>*!C2jaVBIftu|o_Hg;gWom`C!~JU3;s>Y#JHS(%&9uhG
z72!-j3F1;7{ltS%AKB;no@6jcFSM^J;uGlfOibt(KS1xfwIde8?#lc&cwij@Mb*96
zZSK`(SNZt8z#YcA#V3M^3!;>Bu~fpkAd}3!5!)#c53>%33L?-$DaMO0zqR0H8Ftva
zgH^;j#dXM-WL@NE@|LfTMReMSWfbq})2d8MYgHIUo<roxs-WeSBy37u0o}FY<%U|1
z-!T1@Pj{f(y|Jb1o?EY*ydjK-*!qn>__#;|_)RMf@%DD97hFRd@n|JvK(?j}Bb5qc
zID;9@y*q|F#4r?sM9V)1&>~uf_Ngs$2qo8nNlC%2_}pODH!hVz9aPN`I!>JM;+^2v
zWnGM;yc{w0vHri&d_|aCNwHoKQK_im_iAE(_az!XcK$ym@q#ujd%p7jX>@_gEO>Ps
z@>|tI{`xmV6+%wZxvE=fSZ`P~-=yiA{E-El<&ZcYJTX<*b$Fo3iT5*82UTy-F0r+6
zu)Y2-REwYXzfRsSLH`?{;+Vh&FivBvQbx^6Zjc%-8Z_n0%_~4IrD7{LbNRgn)@cCE
zzPz*6;-#L)QFt^(veM#Xy7hn>l$k4j5?F#)P&f&!Ld>JNAt^AyKnTd{Ai2#l{u(b-
zhOj@Q2|ODP3l|ujnTbwq&s!#Gq+CM$Y{PdJqi`#9Z@!&POxWv9I?PVF9e*l^TFM^-
zI6EET=VCF8VABD6=+V2DWj)jZDqb%()k*2z?#VUFHs$OaPpf8bp`lZ_PH{z525kV~
ztb#<Ja=mNw$A!M%bFW~;_Qk-ZA_SX)$1qVn;L+!iiFhAvq^7hbf720$%Up&+%mq!D
z_c_!s4BklQ$-cc2?;GAZ>@SC!9swcIRt61ANC9p?2!3K;2ksAVHtnBEL@@2tiG<F~
zeIRJtvg}5HpBkDWogB7!Mina?3EP|~x0!%)Or>Y+X@bb745V+-c_G%2tcHq*(3(EK
zRZKgJSU=Lry21w&l5kRG_<JD4UF)0T8v>#ud`6D!DU`hkOgK1=EpU_xMmeSepbe=i
zvKQwI?VazpRpI5PZBo8~()m^8VpR8wk8^6>Ga`JUfl)3AI*1Z7jVxLx(&lu0-Q&C7
z%OMC-V6ce<Ez`axvv;f{gsKa?*Jnn1_;AaxY8{YjKRsUFN38X7T&#nd;o*X(i372Q
zwGahh$<6gOxUo;`&?$y<Z@J_7ogREeA|xdKfP%9-{L53=XhquXf?m{;di8B!^2KO4
z-RODou<&bi_7lbXQ~1i^;MZ;QIA*FbhJ>o-Ucq-elr=L9tBPozmfncZza74GVv#b7
zV_-cBddU2HIhyxDt_ro>`3CEaFQHnn3lw|@2f6rYG0JgnYqbO0ywlg;KD-`^QXK4;
z=3XGcoB(by#@0i(+z{JO`t0|o&a{(0%KRIA*ZR0{A)X)^33{Z58o@T|WgBEBp_!1j
zqEYc{t&!ZyYu{AW)FA~uBjQ-&CYwnu`=d@@r49J)OU`}gt)jRfnJ|LhZ1Hz1PRwF@
zplQZkL1^Z|&lUTvHIz>N5H%4PfAN^oQ#8S#bX!6&`3@9Op#QOI{#EHv?_F9Fb$)~Y
z>7%E=@28Lsh!q=FxDr1?`5z~ZC{keeDbv^rFVm>Eb#_&?P`xRK2A!|&ef%AoY*e!(
zj=yFbEByc0zWOsNw(<wPJlZzT>Fi%eB!Ywx!rAbsfIshZ;B?Cu2}6spD4Yl(?JjN3
z=tKByW)rLo995f_i97An<aDl$3N_HfCenk5DCmkFBfn~p!NMD1TianIL1NhagJHkZ
zIZqbZK8cJ#n%{iYuMx{Tv6#>+C66ccgn-6+H~~Dn>%h2TP&si)1dE7-WN&ofNegw@
z6w|Ox<<Z!SZx&d@%EI5zxk>ndZ{|j#67A?Cg{C`+1KKhC1;Niy#?Yz5Uf@}nYe&n{
z0{yEdL!uW+yEm}h<5f>~m^A#PUPz8IKE#0e4jN7I_hlTaB&ekh{PQ726f_tfM`ziU
z`6ZO=J>`d$9>KbjC*tG#vjlZvB!!LE@^#(0&=vE^+(-Nd@KI~Wk-Wf{im>|4l;Qok
z4y!k;q0mCv{`RSw+|{=}=Ok(4fCb!1W61fMmj1nHA5Rg^Pu%`8V?R%TxJ>2%gf3kW
zpfE5`K}p^xeBKzhSBI+z9lF$`hVK9a${U@%ox$o(N=Od0S^08LZ#x1~l(o1Nl0yIt
zN!jT0kLY^>nSwkGxwgT$$;gu5H=bWJh>&#t4)9Gjns?;t=RZ_ngw5@4apy{c&E9i9
zhl7n?`^yp0YfA7XD1{DOwnUqgYPiDs;@srD1M?(YXS_y&-g0;LofZMX3uH8Fe3rM4
zbDLy5g&4qqV0K(VmN7D#Gbz<9{}dj87x8r8aqVH9VsKpDy>31Vehs*e{oY6Ql~3*y
z;)Ux9pL*ysvS4y=9J-2x@xP4tHRWdvn*EfCk^8~NCKkkZw@a<nH2DP!1|E%7S7wr<
z)qVR_?SBU<{0rCQ9|RN_p6DlpT4(7evP|cvg(vc`!bGr^?-4sZfbX+TK!Z=cm?L+#
zwtWJZf-i^0I&gH?7v~$MR;^6Id&D5Wcy%~b$mykNdpy&$q2JZPY!1}s7uh~ka@1V-
z+Sz1A(BbHi-#-^t{<(^l2qF~xV;)<kIG$0ShqqsG|El|2f&%*F>g1cO(0a$iD|2QS
zXHtLnsHeWmBE{6n)yysvOhtc1CQbP~<$bix>zA_3=Ct<L-%W!qJjE0jhDEz@ad|^y
z!Y22-{;Agg@1!lH$}}TQ%TTnvadL1bx&~{9+}8KZITGk_O|YW)P?nX!cHMQ%TqDiT
za1PgT4V-nrkJ=PR>G!`$H_~X)xL76ndA5<OcE(r?eceu%{;{o3P#(T-^Um@Tx21M0
zbV=}C2(-x|g#mrZ$PeWkh_hwY1{)T+@5ej|2AZ`TlZ;vTCzExax^8XwMx|;Xbua;J
zQTr4_+>o`Q1;tq|<CWkvI_?6{w>O_0?0L&%az{YW*SCv&Vn|?jUCRX>P5q8ZAlAA`
z0<=C-v!(Ga#6!ycP7S)Fx7Q0Rib?aAcnMkp)7!rj87G@hFgeNJThlknM3z}cr*;oo
zWhs7=Oadz&tLp<+gtEpy?y<A(BFrMk<>>IYA(wavsa97cfT&po&d)=k**)+3Q6~zL
z2zuJ*1^w=b|AK~Kw>;kg%}M8K!^+XUB9jsVSc&?Q3#A?KJ_(j|QsVyztIr(C_MEd`
zuGzwKA<S|rZ78gQcH?UUBFqw{*Y4I?7n5l!JqmKR-*+laUZ==Db>#D~IG9it3oXjl
zP%sj*DSPNFeJK|BgM28@*loEJtKr&Kzs_cE#{ID-aZvN^g3Kh(2d>8i^{eijI%Id;
zvY{OnwinkJV<AK?DtzsRe0?=Qno*QHT#i4zz4>^#9xBT_vOZdvAw)70$PLZ(Sj&?L
z^rPlb;5(xFot_urM}e-87RcLbCUnQ~_V((Mmm6U{1*vxV+g%!H$G3g?l=mVaHT+A+
zD|8b06wFfrLJhA@9XWdC>E~|I<95LkXN}iV8CFvM(tM*oS;!)Jv@LutHcCkdNLIef
zpq9{cb$zh6h<&jp2?kg$Ejuk+j|023>caHU%xOB0W72hhJez<vQ56m-I)tBl_Jg4T
zocMknX8ZdGjdh?B9ZA_PvWXE<u2RPd`T?vS1w6RPLws-P`10BqM%EYTdeM$e2aB30
zk&Bwj1Ybvp68>|2CDN-E|M~i3${=mA3pwN{#~=Mv!1~%1r`>R%gzunhzcM3;%&hDW
zt%%7tel4sMuRj)!CD9?Ru-=!dWZ#8nADFfZ1!tR@eq=S(1oK`dm;}xdeC`HTnuhqJ
z*>si(v-vI49HsQ0!@hrqI`RIZjsLSje!~sp!w4(7Lb(C{zkmt8!vUZF5^>Ab&}q3@
zVy~Zkj^^RWo8jWI?H6l)E>fCK+5Z)`KxTSFKLYoa`ceU~B*ha=$(-f<IGi}`Z%P=U
z@O*sO!-tGyfBwHBI1!^JhHEdR_J`LzolRaoYhZqAA{vl?lm1|9qFk^h=u@Ddm96^s
z@<t(!p?)C{Be!{m*ck`hm=(SvTv~twfAIWACvH}MSM&wn3-jKD`<pGVr9r=6{CHEm
zfj<n>tu=<J3Pj|dmA5R)8Gw9xhDe97&lMZac9SRI9l&@l7%zPz@)0DOEAr($kHM)c
z$^F&@5vfDZcxS%c!Ac~}zmMUK!fA~N3OEAG0y2MY{a_Yn0|V^DYJf_2cT(<-Gy+rM
z5%Bg?Hd`(7PCQ+V!Lz>igK6$2ktrExaWDQPQAY`u;DA_V=qxc6b{Tb2ild7Cdq$3c
zMK;mFv3UvI_+4D~?li6Mk<QEAxzm9Q&+YGt_)@wS8ZbxGm-=+NbzzZ^GHFZ?=K@@z
z*U*)Ziy_d4tn6v`n`-u{z3WU-&Ig^8Z0gw5A_>41|4^Gk*G34|6z$h!^3JjG<*@@V
z<(i+~Cm-+4nWjnxJz0t`rG~5fF8)Hx`Cn~f!+ptnO`Db0#)Rq=rTw+tus{<^%EeUo
zD>8@rOW6nRy6!icMNtrzx^Ajnw;6i{`D0qKnE<Q&+~@KGCsB}S(v0I<5Y4%6m~ix%
zPh^UW+jv4b@{R5LgD^r<#UupH0~Bt;ZZto@(Z!#<d(olz@>hnffI6uRMZQ=a!pOh5
zS!ocBR{jet1!pPqw+{}MQWo;Mg+FOarGz{y(id~tm$|#?MkD6N7j$r3k!Dy7+RM$d
zw<eVqI7-F&z;#BSV%@oiD;UX{Ww+qxOR_IpDhiw8Vln<P7~Xw|VIcZ?uL@P6PJs?v
zcmJNc%)<pZcFKXr3vJs@NxAdRh-wRlUcl9>v!(KwC^pTpOnL1qnVwM9#AFe4q@&3b
zwPGw%liF$HhgeJ#vrKND>YJrQX^hHs6+W)Lz8j!zvr4#mD$it%ST#!AhqG}(vEqGn
zrt@;6D=j(5kn+}shzRZB{lCud{C@VeKNV^(7AD)nLCN#=#I?JfsoD)9vGe8+2TR<S
zupowyU*W10yb@QaHWu_)$D$OEA&bUG6tLL$0~U%s*Sm5>ij}b*Al$!s&I{e}5K5Z0
zukT&$0pW9@{m&+Dj_+J3jhm_fAHzvan-f(xH{aC!+nJiZ4#&FM-`f08s+%}=s~FNB
zxk9@+yF|`y%0x6tfXov3_sLzx4Kx=1S)A5QQc(18p10Q=AKJDpL}NVp2E;_-(Zy>H
znbW`i`e%~U$&6r1mZWH*$oEb@Xq(3OcxomxDD+lT2Tt}aC0~H)Egu&x2p~}fBF!j1
z`xrJg0srR*yIVDrQ6r{x91-TUV5h)`b#QeGbtts}jttu-R`tUsbDGH0>FIcA$Iv_o
z1J>_HO-#*yvC<tvQv1=*pL$SE<ITLR>$#j>9(xyPS>NEgY9Gn^yyGJ5qFlXGdIs#i
zAcVf@td~s`lg_yX5()8PHUZV}!)NU9GTr3aVydvWSWR@#>9b|5`*75jNzRs>uiSR$
zlj9B8g>E>WJNo-SpDw5l?j(%;EKA8mUVO5wfi%=j{}q}o?{A(0h_2(kLy@GHN9*P`
z;U~xi$i(_)&@P$~k+R|dtD-@r>JeLIcE*6_Vw|6KzoHLKUFibWa6PWh_&r3lYJ{wq
zr}9EQEHL&Fu!e|-e|6i<%eGf{!WV$gbXEb&`!@hxN<p4V=J11FgQ<(~81kh%aBaMB
zx1-o~Jqk97ESqO#DiR%0SjNx4v?gGQ+o5VZ9;6Jl;Vbe6xqgzo#h>RyPRsv-8Z%tX
zZh=EYgMFyo<d8N_=YO?igr-*)HeAqa^u(JPFfsGIF}m=jOANVmRas(~vlGOg*!-!j
zb*(ZnOlGfLoFvT47CMR6k2;#pGDT05GgKlie5Il-+Z#u!2gb?iq}C5>-<n;&`Si;z
zdeS;y@z@Mt+NW)aVJ)0Tpa&`QsEbj-8ij&cG)eidOxVb`I!EkNW-ot#fQH(**h$k1
z{OW@xoXWJKdCQn&2v{dCCpp@B&U@Q-t_aZ8b(d)=lEAb}OuP){oj#p+Z|lf#wSte)
zf056Eszi%FV#cbOQ{-a(HFG#djG`;|1C$%PC{K-hI9!L3Aps8|8z}T5B^nLLAVonc
z&*sT@(LG6EZ}+^)bY1FIsZR5WU-!s7^yrWY>v@H<DwyybWueWBbkt~NS|Y84*qy8G
z9$yaX`{_toeoUqz1P?i?FMjgCCWbw)J0zu~3oX$3pT^sK(ElH6S<i?l!#@erxg+P7
zJliIANcA5EY`32M;Y-^P%2@XP2iD#?PxtOmzyokqREyCaAlg*~hyxzOm}dZkvxhQ%
zM^K+)`JZC%Fiz0Nl}*~M{s(``tYTUfNx%KdY`8J=lVY1vF)ZEXUvK+;bt$yTFaI9w
zhLC_E{3N${9*;U!Srt-%#+>j!ny(x|nTxd}rBoGgNRUjU9h{JS8$bE^4MTyEHajaA
z<v#0w?XxT5U4G~*fr<X8b|F;kaqB17+6qC#^>DBfSx;2t!N-r~E|fG<J|f39%K>i@
zjvAoU--C3dx*TERkn<%NsrY5{9yU@9w;0fVcs2Yl3qZBH$nf#_662h4BqZWBK^J9B
z8^!PMpMrVdRuh~_E@q);pzfoP{TSIU>khV;Zxm2PL7Wb~EnD_K?2>i|K*xs%$X4+a
z$b%jM;Li@MDC~x_lC8Drm97KeIDK!wQAZ*!IXeOZ9%eh>&|otic2EQH0$Avl6KhGY
zUr(woc?z$PoW}OZG~NBJfX#rHB_<mXIkAX|JBd7w^?G(i$x>SPlt1A|z2`2i*sjFW
z*xZgr$CX{P*sH~XX<(53w38pX9fDOb<i>K@Xf{dnq}wcR4dJ%HptD+gL0+0IuI1=a
zvUgg7w6XnmRtBz;x6%`*m;VJ&a8K*8vSfhWqa>;661D5j>wu2)8er`}2vXp9QV)N$
zO4E9ut9KmF!mmcXh%SMFN2^ZKZ=~#bt5uv`Kybws#0IJYezk#t**o?J6f7b6Yh$O<
zPwdH^7z0jKM=X8EzT2SJp{o*KUj{=hCA=niYDj@U=DB}92@Kl;$AQ|gF04|*(Z@=2
ztb`*=*uM;F|0S4gE<!jHYWYJL@7u8!3z0#zh@auzvIxjMvW(x`vrcW+U703N*b8*w
z?79=`;?UdF(U`^c8?XYMU$&FJeccvI3-w}#{dMVxt8Xr2Obr7hp>{FGw9<yVP{mbD
zdleC@XiBJE42DibxgC1j`Kl>N<FOy|^7MWtad6N^v==JI9;h)+1Tsvuc3X-~$|mUc
z^_BrmEkoRM+w5oF0f&j1o4tXY0!C4u%4?R4F60fOYxhP(qQ{r!CiCDh8ybD%`87N=
zx&*EK6Q*DBNb@zS8iVU&SRust;y-{#>84<HHCkU+`4bc8fQ<R5M#o3G^OITqG?n)k
z-a_MRm(rJq!zAY<6<LYom|qR=sKwBn?k>=&=#%{M4RIK41{~hT^=u>(?(O5EBoggT
z9=YE){%AwOIqc3@77>MolF0D)Wlu~s`SPEQ3dHc`|4O|)(6MEiNai4#J{_^!Y>FYf
zrs=v1@xP{}$?wiwBZ3>dYLQQ7qP~KPHkjX~nOT#@v-(fH0z_9}1!U@bJ|^Ocr;*;9
zg|a=2#pQK?P-6X;^MNmlGJ4dLSVUXt#@IJY*`03xHYd>+sN0~PMq$Ofm-<}`VywQn
zySdqr#!aeSu`$J^pbUA-)~E2_lon89;Yhs@MrmHcL>HV12g4ydwI00G1LnoKXd~7&
zH<@0n|7KMPI-|H9SVgt(-0(Dp#eVa|jS!$8Xs=E3554=d985j?(<;yBJjd6zV@hx;
z%RB100%@4}M^6~V-6t(c@V}2*YE!aT7($I$sjoy`Pq2TgF5->!=MQ&uW>!$>!Iwhb
z?tU*%#$aF%8$QpFs}v1F1I~2*$r@8>JGWwJ2d>Z4Te+xE3ylRo@YNQXC4gzS15l#Y
zcqBWuo|1X|=T}ho^6<?h>R#7Qu*^IG1z~M-U?fBnlSlI))a9NsahO<)^o=qGZig?F
zeHEl})oYMyE;<+ObaLF``{7^qDZq+I`PpZ8ky6sb3c&!YV?b!GG}_baw)0amXG%S%
zlmt~jaukF%hfHnv3fV$~3cP^+<c2|b0}EPpEGdzYJL5#K`L!0eov|(-!}(upC7|S1
zEO_8A`^h^AOGWG!45;|(&^5P*aN=h4pc)FrT{1pmf-6xF`XwAoj2+6SKkwY?dP6rh
zVL$ahyjW4ZRm<=>>i_zK_Cf;29y)3`iPZ$7Ug1?=aXQ|a<Ub-^5(tK_)IjB;tO!*B
z;4zsBEkyB7hw7=Y`{g~Id3my3j_D?%SQ|=Eku0l#%p{t9yCK9f#q?{kPi)4)B35$?
zj|D@&fz3rkmFqR?B{3{!21o^?z6IUvq#%_iTxuT(gZE=Nml~G^8P{IlUMn$X_@F{?
zs*G3ft+a)Eg2gh2P+#~1Q?p{qn^_3jeRK{5ZHDn<;C`gbws=@IxkuZa*u(4PH|mi;
zLKtb^MOlOsys4%b3E`kIJlA1*?v$TvTg<-r01S2<=C5c|$T%jpcWfLaEVCl6oB(S}
z@_9c$oc7>=6$jh>wh54hpc*3c1wlcHtdzF*5sayEgAL@{OgnC8<_rKx5W}NspV&q3
z7jK}!v!S*$&le3Go#?x_E^Bg0j@k-)VptLcPJhqZz6#%;#pqWG^G;7bgzHw;$6CF|
zwBW=1!nZ2@W_Dik)N8gy?9B~cw~O|pt$s@;iEk}el+6Hc!_UbES>>fv@x@viLzpNx
ze_#6;|BY4UKVzKjNEwSyXuxsF@ld00Y-3ITM+FHcWcNs`nA_hfOS|x^+f&gR2aTr-
z5pt#-sik%sk(au5lUk)!rPc`D?Xe7fx<f2vS|%tK#JBW}FLZw)aKL{$;UMJjAISmk
z1I-IdR(4?aWVM!o#@yu(XJ!HV`HLu-#nq)rPb~xijuqX#A%rJ6_lul+jjsA%FAr*S
za91=@t`_INzN?diPt-V-9JkG(1(Wh`f2sPrr}DL{82xI<pX%U69D>}q@xsTF4{W<v
z^<w!r+meIaZdj0)O}o3@VGz7WjbG%QodJ_v8RN<kN3aj}ZdjL&i20c_N{aaY!u}cN
z+IITJQIQztmLu*J>Q>cRIDBWeATC#s|L4WbV~?-G*7y{8Cj<k4W88c8Go{S31iG@s
zMOaoIYk|{`F&PAo3Ww5y+0Wk-<wU-qHvG(Eeeu4tQ$7f_K5baukz>jS#9mA^vRCqY
z&w9*}6T*;~lGzqJv3q*J`m4*@<6n4dYoKgY5+%lvGQz|z)n>aVQ{2vnSMy8yQF#Gc
zl5s2YchTcKhA&HJsV0s(Q5YALZ|Tb<j#u?6o@*Tv!fa|S27R^RI;2_9L#)c#M)Nz=
zjnWdNpCcfCQIMVts2-bS9lBLP8b;eQDHG>2bmZQlMJrr|Kmc^4pBMTnRYJ$>t<@gR
zHOXdtHS+g;P|uyyl)v3r*&C61qXnYntfL)J=|5O2R6c4bp^IBzgY37r(80F@zg6{U
z4+b4xycd%0q8vIy(8eLCFp9;ZtFdU_jNf>x$zX5VAY}-`&i=9hfV5thw`NfgPr<$k
z=YC<@Utry+AXXEK>~rp9WF!1@2h$!aET<WJd;1V6UP`%ZVE^vXiS3kO=$}G#>o1<f
zc>eHtM*@RhE4*=BxcL72d0!H<K`w3*&cq`W94o6B$<cQ!HyD%`gWjgYg)PDp5MEh>
zJ5eH^)xq%VG&B6JKt4YNGRe!9%(eF^c)Iu}StK6#lZxz`cV*^R<HKa^<mjr$pS*LF
zoNF(7#)k5XS6k-e?di^JWlp&nxi@3U?BbI^Pwe>*=Z9-rp4Xj`L?Qe<^TDiIIqII0
zjI1McLNN5@(0z#l>1U$_cV36uSTjwlN5eR#ywm`&p=$_79_3sKJGF3esTaPFM>MM1
z9Re*>i}DD3K|bR*FQIOAbkf%Cj#penZ4kw`etEmHZ_7s|amKuP_}G+@!}GF@>%7aS
zBq;P<j$AJ4!5ao+sv7J`&HRT%1AHDKgf_P~*W65!g--IL!#SdAyRm*+lkDHW^FuRT
zH1RRz|K@X{d}ZzV)A(Akq7mnpC@{F2hh`G4mD1gQ&>^EW9vce<v?pK6+ZN@ry40#~
zBTNqS{MY1^=2Bii(R{Biuwc&9WB{&6!>`q=s<X+*df@pc#DB}+_Ig?Z<54!5TEwb&
zMA_t-C9Be(pCTU>Ft=--9Y$n&hlbT~WAlh{&I0%DI23WtK`I246;F-Xsl?&kh2dON
z@M_b1kNrvW;_GdX{ngBWl|MZu{@mPsI!RHunY{MZjfUV%TT{L1cQ1FB6I5B;za4)s
zBHhL+<=E4l9=?AuWIy}1O1G15Z?RnPCt7OW1Z)wT!j2xj9hqxH>TD5p^GiSRD0$Xv
zoN#@{TeS)?BlvrOe@sU0o*@lNJ%>XZvPobZ)1(*`L_{p=hzLFJ4vO>L@9*_QJ3q4v
zrsv<Z%bIWz!!C}kK_eh&CelTjyaLkn3zg`6lSzVOE-oJ*MAU=3L0C#eg0x9Q0Txp@
zof!p*!K#gcfTynx1j4C*@=7t%bb@6a{EA22rWN&xsD5994&n!JO4X?l`7!L;IO(XG
zZ=_=TxSv1EKD|9(qieXfa6bY&Sl?`z+m02fOtcul4Ez%+W$H8eEyak`jLGB0)?O!k
zogLxg&xan;gI406BD{`cidx)bAXj}g7PBe6T(6|!>c!pTwz{X1K`WkY!4-qIk~jV~
z)FEFn>l9N0NnmEZ{-m5!c{iFkXxYFEh{0S-V{1V`-5lN!rNO53r1V_H>lGRb35?<X
z)OU`lJomj@GDFoGXyJs#lt6jcLD`~j_~}zN!382aw|TEptp?D|=qOnl*D3bgImc#-
z-nP>Q;8iE3Mwy#l9r`ApLKLZaFV2W1z+lcyJ2nlRK0=Z<CqD$fNi1PHFxb5gFO-}q
zo4iZP)w^lvy3tMLB!uDuWpn?$B%S=k`STq#465g!BYI8VJ%{v2YtG-62)uqbP}5Qb
z4X&>?Zc3`I_QP2qVSFLccAdNHBt_+Q>%Kex#E5hIq!mnzj@6UIbKlLtS8_9Qhx9oh
z^pASJB=97=z&(IRZanDDJ$PyT;3tMDxN+BMa4yv+g|Df+mKKBV)ye*xL@*9!)>pee
z+mVo#k#)NKDYP;E0Xy$cZQ|#&Pl4?-#7}f2j|Y{o+MGJv;O#5vIg^hos2%9EoDW%F
zZzNMp>ww@s`$PsJ^e`5M6S!gQr3=Ya`@*d6DA07v^luDn!ZH(p{ZjIu{nA&1spQXz
z$OxDDbcN*x`zpUV5sn2}1CsjTmZ$v@4<^JvC(_<}m^U|?g_pCxO)GXmLrjou^~i6L
zjdOKF@Xwq4jsUaNc*Vm{Oj4(ERQazJV-55BHSJWn=)S?o3yYL$P3bXi#gO#OzwLQ@
zmc(j}N0YIix=I&bh=RqwQE8derKNd?3evx#Tm~OAwtaCwbiKP|$>^7cq<&JWmQ$&e
z@z%uhWZ$E2rvIcp^gsSBmdb&JP}u-PO`atSQaU%>*AI|U3%U_}uK3a%46)agN7$#!
z$N!b&K3Xg}Kf{wVWlc7V*h)~&&eU1jwxrAe<4h?4qJjks)xta2F;V+tD$3?&9Y1I!
zx;;#xiGnarpPPzDfctekJ`(bb%@7>X5Kg@kY(VDsRZ?1%W1G4zL!%pqikEWAQdU1I
zc<9wBg6yl;hX7a70*$w*?W7!4@Mb#%vZ(${T?6L11**AmptTOvx9)dSIZXQcE(6Aa
zSpxQ%TFRaL8Q40cF3hKMr#$!wnIY~0n`Jxl2e*tdj$7LtZ^iPeGtQ~@_^E|Nlzp78
z%m+`E^u@a}XZ6!3@S`AQYpNb)Z>Mgx3}=w0t*<YRTgn{nJtVon#}8@TNZrN-?ozCV
z0)6-fs3vjze?xY&L!_H^#LAW=x`y_|GZI)mcJTSYUG(NT@g}$Si#kf`EK{Eig@$YJ
zs@Bc{tJLe{WO=2GqqGF^H1FHf`Wf_I+rYRZubsTqJusEl1)ElyeJ;vRwP!X2lhZ+d
z*Npqtz|}srwSO1oisO4VxM=6<fQ5}9=2Tk-nPu%E0Km%qNYT{NKi$$VJTmxrw^sMw
zypHPUud}21g8>5;w&V>(0vp2oPPDkGdA)4(#ddOUWT?I!R|A*)ORe=S4-@dgy4JE8
zKll<np01rfzIcaII&|E2(gL$TKln*t8=B%Jt2w!gNwW{=Y)kc*MXz{{-G$$3i~Z=i
zEB@&n5_^+mR=B_ukH$(bcgwAGt7(DL2+*A#BQkU%Al=G&U{fF%*kg6Jfk22TIJ%3L
z*=6vU4xa!QW&)M4V-Y~fGWxQue@(f<6GC1*;aip*LrfQ5f_Vc;&TU6HE+--BMSxlA
zONo@bOeA|B(8Aw=xs7pb*A-81AU!_wzY}Q<Vfycl$^g48AR0Tun2;_t#DL36ipb%q
za`AI23$?PkptIFlrl;hirN#(q<7M+_0x=(?!e{)5o|gnAv0|z@yRB@pIkv2b6;#Ii
zA&BQPHu!%(mqY4`0f`07OCSt2A|OYCE2TAZ(&1M~)qOt+kLC?b#9*3{3rhGWFBL&g
z*UIK8N1{=l`s#2Puv~Q>?k@c3s(=^_AkN}6nvBQa7nAt!Sv=bPVpiL|C5{nirv0Y3
zYbaAHTrggDHo(<4FNN1M=Vty~_zFwIojBxk)CkxFg@?W4iG7USi6<oDXPnLW&T+0F
zg5MeYxR#OX@&459+Z-7?ENhKSq5PUQKtjYC2PD)!-+Hi{A8ur>t{?Y&W}2IiCR<vc
z6#&ui+rRqtuwm#!jyBhwYcJOp{IO+3{0(dpNZblO(X_eAJiZ$?4FcSieKVkiyGl&J
zHs{qIfGp&^C}T=Hf0eI`(v&n%&!7;+Rv+SV)Rc<)+X)QOSROh%qan+`>DG11=X2Xz
zl=Yqd@n!!yPJBnZM{L{GH>Ys(oBo<BIzGsZ!r5$om72DUmdH#Rc7kp(4n%x)F!mH`
zs3nADK6tKb>T_P~-3GFvH9$~2rdoxpMWIrBi4v{1eDSLbH_9f*xiAuY8@4axh7Q6x
z6oog=JPI>lVtqvn6M4zQLX2co=_`2(b8s;sa^Ljg5RWC}vikUJC-_P)zE9e)!&gx>
zZy70AP{i<(_eqNooxPEpyUN@rgyoX&p0)Gc?x;OJ-&=QR=a$+1t$-q;Gl_K(c6N3l
zk8LQ;q|-f%K|oHgVW0J_-hQhx8OHrF_ptimF4-g8$4X`+_k!2YZ-(D&gz3Z00T9L;
zq6^>X>(riG_+l=d-rse(+crGx!g_c^6(1Zfb$jI+l!=s8lAnNdoCw(DN{>q8de!gB
z*Vt`o2jB5wAWo;<LG?7lee5@(30oxy4;{C;d3CaFol=xGSo>0`C$Pd;#3Cn-D4IK2
ze{bimjW#@xA>={XZomMQU74COrFi<sOUFfKckHeNG55{0Cmo0Pn8~ko1IP4hJ%>S5
zfyK!-DbHhQVx<gA-xhOwQAr8q&x2Ib6&?IGv-L;0Yo0e?ul(eTX6dbTGxzhKMr<z<
zLF)i=$=|UX)F2{^ZF7AU+sPseuJg4No;2n2R)a=GO5bG8=H|5Rw$9kpAkG?}o=@r=
z;9z6N<q-(qhxkjUpsP!~p3hLh5t#^ARYKGyM5wThu2LNJul?bBXh{UUGj&3(m7mUk
znS{0ttu+kWym<4;h|Z>u`{F?iDhkdi9n-57N&8m+{9gx@5k)Vl<M^w(_&#CUIOFU6
z9)3k&U&#B#`?<D}1Rxuvgl4}bGAfaOa~e2w+zIGKFSy&C_Wc_vV34+c&%P^tVcE0s
z4I}rnT#C%?&P7^xJe3yJ1IIi`^;-Fb$Hl7F#~-kKn+0&A(xQ4?Q+Gan5FN}Qx-emy
zry=k7qM3aWKksFUmha#0B*-t@&i$9J>8^UIBKD^W?<5Sox|6PbD8nFmrH&lMoc}A$
zNsB-K7uxd$@>%|8d~j8xUWM1Glob$L(aW7dJ-gpY<DY~2MZs<2OcCf>Wz6UR;aQMT
z(F9%pGA{Cmg$_z=)=ziEG~r{D8%ix*19T+~fBBh_*Wr_Iv`RpHWvGJrNMVoN&A|5i
ze(3l9|5q(6QZqUqevF*(cXPHChw12^Bo|NSHa*bls(jsm6|nu6P-8;~`C00Q8w;VI
z&0^O)hd{*VdXk#Vvt!YrP!}F_50kgg@`0~q{_tho{#!sdGy*W<F|NI~{`+*AxTdk-
zyJfHW#_z^BqMKn%w%(@+1rfG}de+@!jK0QU9c9cRo@7J*&<vW@Xy=kh$kc^NT|*IV
zLNW_B`F)|$`}FvM&=pubt%s&#e^Gu7B7Y!Ea347MALSRt{M010egKMB9H8YdA6$Gh
zHE2is`U9&h?e^Sk=r97wR<VkFnXV)A@!r*VdICyHB$U$74g>wxQovy6F_t>v`|igy
zv`m>B<E1Z7u=*n@;*JxmU*buCc%H9AHBu<z5l<uV?vOu+defLM*-2Y=yLrqznLoCu
zC2W=mEFpopQH86bU@9BRy+O;zF^@Zp+A@To!<uKBV6Rewf>xY_3~xV7wfXxu3?ijD
zM?u=I4x%^Ac5{=On3fL1zi0uF0G>GD8w@=}%t4@u-xs;4VHl2ruG1>vNI9%W+%2)_
zTU0WS_dcFVx^<Blje*p-Uumhp1zNzGE0`0ky(`6*ut-y)OvQ0$ZtQ4EdFrK*%>X&d
ziy`N?xHVAJFGc{d<o?-EVk1f;d&9=sOPyhD%s-Ikf3`MEX1MFU&L*Bl#{O77cW+h-
zXp}<M{Z>+ITOAJ>f92kS<Y9Gz9Mb|GTKy{Hl<d_7ZL~s}F3JSbmy)AuBs3vVHa-Oh
zdKd_vBgc(<`Kp;ttTK5)CWx&a_kt4VXhnOZjxs_73{UwrGt4r?r}A3qHYG++7kDk2
zvcr&<e&e-Sh31sJTj&mKWj<nwUn5@CV-5481n3Z1H-8`U%Da!R@)?L;ZOwfUjwi|I
z3VBJMg7leW^#lA__bVFL=Qn3tVnw{K)-UOvsRx|V)5r#Ld_0%--)zZftwa4nu3TRy
zbhNvE2zD^2XlFgYLHP{+Rum|cXvf}kHa5%?%(sjd2bNy-QWy3KzI-K{qlMwR{rf)|
zK}M1|=2`c{jq$_lU+VtcdodtrJ#eGH?oYk{=#T9JW{vL<m2x!L(ELbIy~^(4V4TKU
zj<#jhSG|sVx(z4ic1LREM}2QttJ3};aTXEn?-60@ndI~8D#nOYi}L5xBK@yZJK2zx
z7?$P?E{KE<{PtDz{G?aqiqVngu#?NE@uTm**{@!(Jk!SdpL}*q6axBoDcYl*9}N0c
zHaWHVMnhf^aWs5}jcx~lywAS;t3c>yL7>a~Rjc}bL@M_^tB4@EDpXCInfn7n@2OQ{
z=%jPNH}&hQFxUATRE+gj-QtHuJK{KFJs<~hTikqyLUikDa2#1Ykr3u41baE<>-)rs
zj0>8VC6LBzTCFwhh+{FRjB7bzT1>-)y2ZZrNmD|=0*AfyhjIi&`5lO~L9=yCA0pCh
zM&urfrBV1VA}z$zEaEf<rL?n^^ElPwUp(ep^M2d_J|Ft`^T)?D`RiD6rD8TtKz>#U
zE0m@Xh~aKW>pCfEvha-RERIo@ykE<miXbslvgLGSa5Sc33tK<~q<YxxJZ2M1bnT*y
zBN~VVkafcF3lG0=RXMicaY0N=?*p_;ZnI?lG_ndDL{dB2|3}zcM^(9XZ=kYikS;-D
z(=8y~n-1v?MM6P91W7?Uqy(fxy1PM2K)R)sZUjU?LK^PcsOOw}f8QAQFUD}Fys_T7
z=6dE6#3@Dsrh@=?%LfS&LUq$yhjzwu$CyvhgE@tt2l48SPz{YiFZGC^qj`)LL&y+d
zY_P+AU4IeMuZ&EWB0k<doO*bPlcVQnp%I8KQ!J4LwSQ~!?UW{<!r0&ATHJO2iA>Zf
z&=Td7v(`}aq{=jPOHP5TSS6d8<vFtHxAt43ZZoyJ!*3I&0PRK<){9M?{mNYMXhO|o
zSXJ_fEn&%1DfNB;zOQs`YUR#%N4Ucy!yBIAK(4vWYa5U*B=Jf1y~Ku_WQ}T`1LZR*
zD9S{M;g?oEd1RbQnw|+I4e!dg6Ch?~H!qbK+spdl_Fa7d@gMk^T^Q6@$-KH(jAE*k
zqu48<omspDInLm@Z#B-8GyMx(Fp5$5=Fe<}G$tp3Cz*ST;zS(z3E1Bgh-pGP@|54p
zs@d)Hd&oIOtb6h9$p-?>hpJE|d)snica^O_x2fSV;@CkAQ|8e(Z@sJoR0K4-b#T)U
znb$WCd<@@B?3MbAVx<Z{@(PO+!q$+$g*<-BE6a5M4eg@o?Z5?U=R!W8wJ0-*R^~sg
z{Yv!(Fe8A+4><V1uCkFD<{soT8_+oWC#8K$$vj3$NvC+v9j}(f#jeRk;zX4!?J+||
z(87cj0*vq-{9Efa;9s7DqZIvJuz~iB+?=k7T$YQ{S2|c8rFdo=x;M5G#@}eCnIQ=x
z#SeFdSzJ!H%F#sTc>*Rm2nxo^VgGKItq2g^hwlSrfn5l5SJFUBi+0|iIG;vdvQVPl
z*v;?^ex=ENH}b_^mJi8__SluO%H;-b*H?&oM{Nlr?|CA11#Q2#^wrl4by;mT$JWz2
zsa<a7QBC0;qHqILnvA%1_!QM4N`_9@RQYIY!V)N8%%`XIRa5rN9oGjeg<kF9eySi`
zr530hd=E;a_@w@Fz`}$y+WEV<>BPqM@DL2t4giYgJ;(*pLmqp&AmKN2AWs*Mc@oNl
zh6|<EDsm>JA`M>vb`|j&l3W_X$6hsvO$(w+rg|emt5@yJ6iC1p^jCI~G#I2Hk~^>$
zyodj<WfAF)4xf-ovL75gOaU22)Ns;xU+=+c?}iRqz}Po%0#{b+;u^@O=y`58{usFW
z>8Ah-!~7=4%OwG_XJX(48$wZmLT#toSFRD2GLf$?71>-alPcuZ@#!)U0zu<dYw-5P
z9Y?L+!X~s_l&0*1g}^cHwMu#?hcC10OT}^xm!BjWfzmI|@d<ws&Ej|6xA-WubAYr%
z@_64lfU*J16rO=YyQj9VeGfq~xo<a=ojq(g9_hQ8Bi$hpI4<b)-ToA0y5Y6NBnldL
zqNknvL|R$qSo6ojk+jTO#d=&RS`8&MCABtrjXRECE?*Bq<3HwMQ;PHy(&b38GzElj
z&J#uJnTb7H5AW-6%T5z_8glcI%ET$sKn`)+98@b#jJoozap4-{cF{U1P`dxDvm<%n
z{^1me_|VOsf7X{COkZO7fnnPa%j`V}3qzuJ+g|AINYbD$K`Xf8pFQc9UkId@Cu>~@
zmWV~{djnP6i~+rPzPy~uXzzA;3y`clawhH4dlNwGxn-C5u?F9o=ZYzM?%>;Fb6Vn{
z9KA<^38WoKuI|0UrJUUNU9@<<M-EXb@~fo{guFS%^kkLrVGGvIkR+Aop>p5z!MLl^
z$*o0ZZ-g6!{P!RoDx{e78!Qfm`S>d3Tc$(^iKV245JymCtml<sNe&LRRH9^bh3J;f
zN?TMJRTRs_ct}9yXzWV0tffmB<oKuxV!&RRY=*S$iSs>E;6FH)`99}k#@{oiR*;P$
z?=&5r+mWqhh&0JAs(kGt7h-d#{*Pi9zR`_P?+wMvZ1jEK|0*jMn<V{!L8Q+FAM7l(
zYJW!1%P2Q=?r3cgNYa6|r=br2#QD%#d#?dDskKTb-~!GbZ-AtsOQlNRT9I6(0DMbX
z;8$zmeFBMN8Oh%y>;SM<i3x#v0Z%FIny2~gFw!UB<WUW@cH0`r%of<NTZ$%X^9c3p
zJ(09<5~Pgc0D^XZly>VmYk+o)J>x13j@QdBIg9)#;DEvSAWj!!?H;7=8gL79K|>5o
zfs1Lnfgfk+)y0A8ku4Ckz?sZ0+_B?G2qd__O&&-|3h5Mniz!WjgScg%e-d^D-8iTI
znjA;Q`YwCh!`?mRXi!(kgJ8wl@TZK*M^+F8p7Ik6p!g1S@<~2n*$}8u)Xk&X)CuB^
zq`3hoLRt-X#Z@NWqZ)P&wX`7$QDp^Gsh+0A(1NFQWBLLP6v#erTo^|&sR%LNPi}oS
zQ145%mn=pPjX}Ek3sA8eE7$w_YalnlJ^fw{;cqGDpMMSi0aa^swh>1bNG<|mC3d5x
zJB`1mhEEj!$_gI%(ggrnflr+TwBTL5+L*97QE;pzmcujR&%A_IS5qwhNp+SF!MlTE
zAer8rZq#Q9gK2K0J-9cEXi>$T$T3+jr)Vs~k+ma9J~GM<7+uL2k6suSv`zHeSbqR$
zj+N`6iIBrIU&6zGFVcqzuV>JdcTATkOXKPG(+QW-0vW^jaS~JI7=Z^oH;=)d><{!Z
zYtPbad7|j?2a);r2b2dAA^Pi17#VLLTC1sDU^sd1!DrzJm9)4Bg$(s*R(Q`g^_RS5
zzCwKp^gl3nnp={C6gZWVgPO?sc3gGX4Yn4EQU~q2dCv~hwazSN^pCmU5dxMJ_;PSr
zC&<)`dLsvASm?oBC*HtLkLdo>9TEZ0TX_Wm31}eA-UL;=Mux^~8*BcR%jm;Nhk{y@
z)y$jb^yagDa@%KzgHb*7-l+|NVC}WL!Re-yWO0Rz!o8bp7hLfhDPUr;eSZO+=ezi%
zKZ!4Y&UxTRE3hli2c26-GFcnyH}DyZAmWICd?5(@K6C!slCWXL5yB6k_T&&`%yTr}
zfRL^?Y7Xnwfz8+V!@8Oh6t7>9NGgY0OKVU|K)cX76l-M1txVll=-~2YIA7Zn#~S)S
zS8WjN3WCmKYp9^u^Yq%Rwp&!raw5V~5EtQQ!T-JtC)~PRBO^Bl9g`V|>(F{;T9<Rq
zw^WK}cU6AhCGbH({r9y7<BaFT-8`Is@6sE33kk%k%L1GKecz=FP>V-_bvZKm&6=p>
z`BM#}b3R4E**LIpiFD&gq=thG_wi<3gb?P8^-vzNLI0#j!rLD2>~EKA;K6ST9rB`R
zGRQz#=|wah3wV<+-@*YztkC}eh!_<N8rgKoz*-eA--sXQBX47n&7cbT@$e54Dw>{Z
zzy$hxp|EY-o|M3iqQTArn8Z0z=A^r)xYPGTqiqZt=D!w8cmEvA>({iRY`mj=|If>W
zy}|8?X2hWob&J{m#y=h}3Vcs$AL>s7>%ljpu{}4xtAp&X)0~qB{SE!c78IfSq7sSx
zPo8YIF?d7Q@AxOp2?wYVV7{eSmVnOV+8M%M=W+?Mf(LvFPu?_DuEPJ8`Vw4g8W~0K
z=mc?n&adOs8k!TH7*@rGj^%&*C}-2&^ACdvlps_eykb2~zgX+-?XtAHYTb<%)5ww5
ze;&h2-+w!)WhCc<-(#1*&j^eykI`0#Qr?RJzXTQ#ebcrKbYHoEDBdM{4+3|qe&X@B
z41K3%pa+$~XFKSYmL({%-59*cnB8E}3JpqOii^wa;)qE|=J$q|NcXX!J+VQWy=fea
zJ<@U<!LNHKidTLvT5P@Ap)1iHtlc1VDgQyTIGB(bcyO)4klb#V)}%$j0fqDac#PvX
zOKM-%{-i>{FQYCH7T=7VZ}S;ai+<JEEL3B4lJf6nhfDRJk5Wu6-r)bd$G^tg&VQ4g
zDp|udUVt^%lLYeWyZn=!PscxWCE#0`r)6>}<xd@aU>bii7zfS4N2(NuyeNRL%3rpa
z8i??J;&$`8a-9Ct15-~*hr|G1UwbH|Ooc`1%YtbO%hiJjsw~^wv_7Ajc$R<Gr9O_2
zp&^4ti+aFwOlLc}`@CiKZG3P4*4!YRe^(?vRsI9^8y@I?wu_tZ3)EI<<6pwt6p<YB
z@pY98#Vrkjg>#CdXL8!v3bC0`w4jf-e0>%1@U#jaX$`kveF_mrLcS}n&PIqA^8$g0
znT5$$9uHS0hgn(3MNas|_3r!aCsnT->$hM0s+;jpt*47FcpR1Z<d??*`~%*R1OW=p
zqF9p3Bg+-)`GV*HNmD2p=AT=@D}UtTJuZtIq>7ZO4L>WKG;w5WgI8{M23|SkWDRDw
zu0&q2cC(OIo+0(5fQ;fYI^<<gtN-o;jiZKB8^g)GA0`GLXxjP0lmFd$5L|Gd4Hw@E
zJEQ63t2#t{y4f)O{`pFe!teX{>41R*80O@g0|GvxlV#4o1^mA?3W~O>{dfJAEi)Yv
zgY`S>15>o5%E*<k_=;ymJRl;mVkD$^{R3B0d8ck)15*OZg&U*;|2*+m#3FUTN`ej1
z8XtqDOAub>t3puZ?sTOBmhQ`tm<|G@Wn3Rp>jxvGMEg^<+v~EZ3Oeu01LRfyS%0O}
zcIG+mJRVNWGZ*1`M0q+BwNqV@(SN?bU?1>tY6#r4Uvwy2ql$mu?mzv-pEuuA^ml(W
z>Qvui2c<EO#3$yy!gz+o{keILL{r&(!KDo3ssuIe1aw|dX&l_j>SFGp(cOa&Wx(29
zp64F52mgDJD8zZ8z2VRfGz3X{Yefex4Me74n%Te&7dH@X;IfK`h{LkQN@m6YmFegm
zBeo%j@;{*#_EeBN@myjv`dz_o<C&u>jZA=wadUxC33!`>VQ~YJAo)B>`V*@6AGGDq
z9lQP$rIp30SOXUKA`i7%6guyZ*YukFfV)Lfb9@wB9m1HX#_iRG?iNpE^=k)TQaM{d
zQ3zZ;D0TAi_N|NJf}Z5&S@}}KU(HJ3N>Tb)2g~D>mN8H{lWFLG0-3S2@lyDxR(kqn
zw{vL7IrGXBaz3LZI`HfmAYkw?o+*O~(k1SA06}lH$GP)CG90Ug4biBjLr2;uo<7Gs
zda|BX)VEs`M*EwMjf;ej2q$9NF%*GbN8$E(PVAq%`aeJ3>Hgj4n?kb2;Wh*I#6B8N
z^)qr^?$7;F+eYJM3<sP3xie-4gGE_+47{gzM)NO%L(|UBZlG#hq)fTio!5HbDF5fO
zL!h4QSdxilK!=#nsO2aUodH+90*xF+mYnXDjQNkR$NbR=+6ELd;XB=3G_;B?EJAvm
zn@5I^`k$L1U(Lz+KUe?Pn1tk?n+VDPCdl9>uz_ic(0ruQ2VXVK4$qNUdFvcB-Z9l-
z0iFD*7CAKD$YrKYcyaoBFBOf%QB^meCJcTrw1ofjUVQ4k;&PYie~35FObWpY0!3ba
zb+d$b=eK;m7`+aobX~<0=Y*#~6S3+XVn*N8U{WN`duHynRc{Fb3S@FpoYhy8f0qE5
zFBAZFghNka4OQsTA&oqm@HOfq5&?tDQ7ilJWr3N{7;ue|Y`Oepm}(o%1OM3YF@3-|
zTmxkBS_0xx^^;zFU}*l#6aiZD$Hgh=&fiVGjyg&W{EKBWpV75_^*~{au@aoFa5aaE
zguv!hXW#j|@P52GG5aMjgN80@-E8!$z7M$J&qep9lCb{oSNi|1B$l6BDp>;9<?S_a
zd)m<kfE((IhRs|yb|(EwIymG1K^+o&y*|eisL~KiYHo<Jq3#|F(JZo>`Ts4Gu6KQa
z#j8QxGdy0)*{{#h)=?Uqe<+fYlA0&{o>#mx#5-xZJ{Naei;ZV~d<F5R5|k*l1T3%;
z{VIkJI{I7iHXAIPccFg{V2KZr!n9=ID+B?O^SMWke@;K4LF}DUK=coi-mvNh=(zh)
zutdJ#UfKynCL}8xF0x3LX)Uk{%b!@?c8VN@KOC-`?IOtke}Btu$Mgkma~HMjuNTkS
z55L==s?QPA2_~wcB|A;(R|6}npqUFGFxD`0HWKu_c-7%rm$(O<M{`JwUVY-|2Lm>O
zX=eCmCKv(pGe>?lPQqM^oJYk3g8Js4+?D%VvFUC|9ZO0&@Q(0K{sY#V{kLZZP2~FG
z;BeOAv+G@iSgWgGZ+?E4=0sRITrL7GouAfQupdmm2x2}I*S`8`YBk?f&ny4i0|CDJ
z(*Kt4VMWDsuukd7Ay0>R(0Cu6b%2$#&BQtmKyV}Kak^P$1}tKv<y$`6H(&tvy-FSl
z@L2beZpt>o5;&&vRP^b7+jZ=3Nm{Z112Pz{yfl*0RX)<A(ZY0_vFk64+Py8)vQg2#
zFbK6bl`SxzCDjh?2j*5lJRQn{^8r8W9wy1XY>+XW&lW);EDO)B0jCrh?G7-O+=N>f
zxF(CZMgcJ{g|=su9<Dqgl7Y)TY1N2(FJ@*hX9_g~8CY%p|8`=q#HOk&G{LVNlzmme
zd)QZQa=1#Mny~cSQU&%xdNIkmGy|Y)4ik>M0S0ax8zW{~)z3y|^_~`J-jgu~3mzVS
zjWVkc@?wkR%@)XiNdjMJ+yg@zxW?Yc?r&|f`_2K+@`5(wG8u|dIr&Rs5m;dmy}?X7
zN0A(i*NhlD=b*~*JMWMg_VA~d1COhR8L3he-lrrt;A@1#UxpC;;gY@o-+Q8nNHtS`
z55r9D^U=F<ma~glwJ%q*q3`6;I_`qR1Q06iKAx{ms|dpNfKPT{x*kD^2aqf~u7vRy
zK2INtPJX`dMcObmn*?bY*{2beIAaHgq{3z>+$k*I`XJ!u|GFu+Fw_5(EmzVb*0LrI
zya7?l2se%^pmu$>+!3tKznK|&FB1f0&kR_AP_IJ(7$$aqb=}N~M-xZ;<aoN(dd{eq
zAlC9?PRI@X{zH?vAHW4z4urtT$JtSMH*s@w-&8l3U8CvBr<f+4$MVFGR+gW@47YJ;
zdW5gsMu3vnD#qUE_CPw|6F9bG@h5z0#d{@G10n^JfYjGqmI`EWv+GxOh{RDyK-F<u
zxhL~fsjk;6A1hzyoxvFg+f=Ey<PtCuC^sJ8t6PM$LFgJ?b~6PX#BzYVZWPQ)Xk`&X
z-)>LWuojBY+<S<R0q0MDbUS(m1`35vNd|sD!o99@K#A_QEc)@@2}F*K5h%UwxY!Y_
zu^vND4efjlxPsB+;!WRWOu_6g6M*M|N6kbEf_bci!n;jUE(@Kzo1ZRpCkAtNIT--i
zO__wJdC^f@yANK-<NvH0()jm`Q}DKyQUbKKDlR0shC89u84Pn$2TlG-Qs+Nk(86nl
zFKk{88go;mF0EPF;a_qSK&Fq&U&si6fw6M&{=}*990u_F)xcN94U$3<T6nL7bNShm
z=)>b?aRr<ON4nn~6pa-0o!$Y*^)pO-1%Y2DD-oIo1_lBO`wt&dPp;NKU?3WzM{D9L
zLgH<nHB#Owqbkjv|6#EnMfQLfR2TUEq{0Y4j{mMp_Cggwwa@eUVVx4B*FzUa7iF8%
z)@5OQhHhI0d4w-zIbLJoD?vxrfRA`K_+;$#jmpH*aMH5&F$tMq!1We`jKl`cZoVXO
zI_^5;=yHj1Wr^0gs*HjYn?b^5A<a3`pa^u&2bD7Y!@OI+xRyoHhq$UMiDp{dXTUMr
zEZA}zn~WzY)peben;hv1kRXP^w0DrRF$IJJMU>yK=POpb0;<XH?fz4~(zl}Ui1oZ}
z=9?1Xf97x5!V`0|4F`l0t7rJ0HMXm{d7JT_Wl2{?(LWdk7~n>dwE^kiFDIF;JB7`U
z8lADG8J`Aff@I(EI$yzbJX+Z&B#(yR7||WHV!aAqKyBHIHw9c79^=+qE#=Yzb-*@d
zbC7=_8%{@0DmuSEud*VB-LVYxVJAbvUmqwasQ1A(7y(H69V#;)ni<5KbgPUDbL2MD
zq__Lc9`V!XPk>wpYT+mbcmZ}HuzJv3JCG1q?Wh9IvM-l=$~vLYJQu_GK#F}bpc6Bp
zXE5-n*vrC;H3KB$rh3~NsTRQFeOHx{wwnW;LfgRk-qJL-Y-j-vGO9#Iz4C>T1M7iN
z2#={sh)sWZu4}&XAYu&ZtXwx=2{;Tb<N3f$2f=L=JnfJB`E@*hnyw=$e=)iPJp|Au
zr4O!Hlx0GuRI|%Y4~gA2hAJ2oDR!BZ8n1h^shY(s-}J((8+;Z2e`*U@6la1lQuR4N
zMybw^UdRN76+8#^-%A@e0PmMOq5?Q}TAc+Xq7pNTn|;CemUq`HSCi=L>EVplb5Sx4
z!(Kh1^qk;a;M1diL+^Gt#WurR^kul@!<-_#rHXU#K|rZ6r_*H)j%u1XeBrK65knQC
zcK~OrjH%+RHn|_covG3=!phQ%Ujus%x3^PQ()5!&mZScjSEm+;(B><+HHXhFXq+Az
zOaa--K+W@OwI#tPY7aoE3r2=~a0ZbchZ|WDD7GIyAPT=+rUSE(l=AWrO~mG^M5vzs
z#A~qeJOIY|j!9bJu<7fKh}0LuoYhmS`_L1aLj2`hh35{0)B@w@@BG*fo)F@`1q+{@
z=t*Yw-1(P-&*G{eFx+>AFOEsww|V*4@4-D_W_xudJQ1pCigmUHm>Vda$Xjw7Enyk0
z?N%Yy{(1xe#vp{h_uuIvzyTu00KP!$(2~^=FnD{%Pv_gXXvbSW$l+PNp~z{^R3q4-
zk2-;OP=UB;TJX(C#PMf#A@~Da65wry^Bv{Qzg-Z!XK+_jI()!J2Zi*?L~RtYKV@cq
zoNM#_+(|^TC@!d~6ts!=O>l8d!>S&zcaT{KD6mPihO{CVYaKAKPJT^f<fdZ3T$CUi
zzIG-0;oFZfY(4fst@%RX@YnUIWnYh3Q(@C1?0*h(ssB05HN9mY7Y-*El#~f2D=bY+
zOoY#Pd<`UcegQ=>Q=W86w+zXnm`>MAuq)9DyfHYT5fZZfm^#NtZI^p{O+E>@3LT&i
z@a^e!uk0W1%*pZ3zoG?Bd;_<!YRACN0n+$lUWObTa5lbS=KGXsZWEVtQB>Lx;;oPl
z#xQx8xIgzoKL?N)$-~XL-lnd71#TbxjR&P2TE#vFy-~aI3CA<bouLJQ8>zn+bqhdi
zc==idQ5t~cRsPP664V=U--ZF;N=ZarY5nEiZJ&`qWX-#n0jj{<a--_Acvhy&+n?Ew
zTRb!M2r$^tffm3gA2&hF1C+G!-T|}2jS)rWB4EAF<G7*5V$@V81cDXC3Jq!sX_1|>
z6ZviLJqmZxtX)XdBPa{uE%c$jI>Cz+90JEmOSTYR>ftD33Fz)$%K=v$i2^tY!r!lz
zl;~h&P$|Ej)IZ6WdcC);ms5WBP|Ht^o6<iCG2sa?Du6eY&3M0;#!2)1DE#!><8iYz
zrvJM%=BL+6rkM?X?>_5SI`JX#GiEcDn=9cXK?s|_TNNg>ju5@*jw0#fjS)Y^I5ta3
zLCaA(A-hFySMulI;B6Ri>Hi{9<9X?3P;Ttw`h2c_4Y-sXeIDA&V$*T7vl>j|Jt#sP
z{q!;$xYV&=3Y?nIKNNn?X#$BxSdHz#rWGW)MmzK<$WMCs!P;_bo_jN%bnaEnuRQeV
zpS$<uGr%z^Q6Y|9K*4WEIcHpIUnDWT=XYg-<GX+Hk5#AqjLBP3Z)!nfIt+S^Llfy4
zkMmy|K!~nwfD`ScapEBkm%DHXaO&i%NqodvX@uP+Cq3el{blW{ZNs#whqXJb)`b4l
zvxBV3*5OL`u}2LlMRu~okYFH)4^{=uWU4_;BQRB5cl}s(-RfEBW#j>ZZ1S-CDzyQ{
zdSTRa1MV|F4D5fdpIYk1`On{~rWX~R%ge>b<w`Kc$^>rgL5n@9DW`bW0`{3KlT4<A
z_`sw>IS4KY?GlDSNA<KVra#!KR+)d+87Dogh_Afw7YjQ9TM<)$Y4W+#kLh=XC_1yS
z+sXVc_lVT#lyTxe_~HHn`YrveUQb_|Tl@FA1i2)j_(+@<?!ZK|<yijEl88k|OVKA<
zEIE5U&S{>1&XE9~jNygAH-CMv<-*>e!LdM$!ZGx<<Go{(jSt2f*Qv&B)Z-uV2jP^@
ze%<1|QPwQY;z$eeksV4)TO-&X<;*?=a09`jLZkBGW(Dg|<ruhM9CWU(@OYl63)$n^
z+M3?r0(96AgBvjHThfCE@ILz(dIN@${;!ifp-)7!rE=`utLHNf9~<u%93Z1^YQ3@H
z5oAWALxNP$OvbS4s+8Q)%T*#J=mydcO<<KVxYC*e%D^xXg<6#sh8=)jkmj8hqy!$F
z3@j#E7t^~lbb_$S5)qf~1Whz$Re(h7e5$~#Ua3(Gh$|_e0tV%lu*j?p?MFWGt+6;y
z%B^;tial&g&$C?3V|IK&pmN7pxHdJ@M^t083@Z;Z)RXaNS8M_?4u63HSBeE3u@l%w
zRcnP95HvVtC;ztG1Z=uFAFqFCOA}mFe~eY&0=eTZ$#;_(GqSU=iXv`u72#q|tiEo#
z#oCgYe5;y5>M{7P(G^bm;KQ7!VVnSP?j!|zx_b}mumP<-L2t!L^KqSBq`l&oZ&`Cz
zon+ipb4=KEOK<LO0BSI&D)NkaN79Jd+pU*K)RUZO+s_+MW^83D7`|fBn!NObfnm$W
zgzn><P>zNK7VXj}wcmpoQ`6%gd+Jk|y;uV-I{cqa^5C3>r+tJ(ac*>BirIJia6BW~
zQ0)xT_-!d!rHwu5f}MYKzxR>&>jf|?qx;B%=n}w3ae9^Sd(gb?Fl6Z1WND9OXco|y
zYA76X0licli|h~rT%uEeIV4*w!u%9AmDuz0$9nX*7~DF`w|k}<kGC_pUk#D&){46Q
zg0d|W_Jt5KOQv~U)l=!awgQ^-lb&~K3d=&|9Z5t=#Ul&IMfw%%zG#Hj)3r7aqQ#Sy
zK8s!_YdCkDf4p(o_51_ct{>9U@WLTRemthfiRiq?beegM7@9tJn8&jze&V9>5*QIl
zhK*TUd^r{#l)BmK|GeBG8gJylPJ7Tc1Guy6Kj-Gqh4G)<)i=8-6NI+wC_|Pa*L>#e
z+m2$6jDyOPyQ(yRxE>BG4&rSs0aH94r^4%lufS&Z40Oy@tszu$6>f!2PuHiKMy&ud
zu&P7mmQI>;!?bf<M-pf-%N?CQrLPHL5#E5RMol!sO&-*4OOIa5u~&N>ZI?Q}a*+|z
zdxBQ;U0r~VygR~Gubv^SIgQoKMgHAi0jT$%qUbEn#xR;qKd!y#Zj%rQ5|{79axdk?
z8*P1COrYSR--{E8?FF%K43kWd;6SS{r373RvL7tJ*Z$Bdv6^VjtR(ka6yGn2)GB5Y
z4tm`mm~}#=`%vNmn-$}vzhcFD$%pD|YW9b9!%n7yuw6PK1|BTUSXw3*kQ6f<R}9a{
z9Q|5hHh2coOzNr^?isOlka-CA@|EIxN1v<|R}TY9CW{7L;?-P5G)t9LSS_exDU_Zk
zCP&bLB+=0H9b;am1omr7K^G5Maleaj3Ssjw)1Aj{R{W1Wj);gX<XjF_Uw=*f2zGAZ
z`G>+C4v?pW`1T9(^^y5kD_9T{V9=<dhL2Q?277AP5rn6ercG#)QkOOgm|obv=}!;U
zq>1$F8PxP~JKg~=FS|ZEx9y5<w%vLb;R98B`}^=i_;}$2FUvg@crNM|B`+=gSCwc6
zWj&e5VBWVBoG2B!FOfRP^0Q2aC|#c~iY;a|d&Ewg`hH>_Bto*M_}j{TzZHc6KPXD0
zC2My=$7dZao}=@oc|8NJUpPxxQUlBoXOEUKkw0m2HzjkG4HA5K*J+66u7@R9eL~0o
z3s}fQyV&*O810-}$d!~CvoSpRzJJ?o(ai`!SqWF(4#`xZ7@?ZV;}CBSWB@btfsk8W
z@dhf8ZE?GONc4DC3b0sfNm25`&W^s1fzUULH~KE+lB_z5*ygccE0DyytS~{)p>~zT
z+X}M2s%daKbLCncCU6G@q|(W^YMuTZxPH=bZiNzB4G&xkVm$vAe3)7NeDtHE&fJmK
zd0`o%DynFz*v_3WH@-(&-+VJ^eSY7X2Yfu$mRMYQZo(DGz+Tf3`qJW+9=d&IhB1L%
z(ILp-o%j4ku_l7WQLjLyk_kd2S)p+m43-|KR6>qSaKaPdT4D4o5|teI>G@3w*FFQ%
zcXGW7^Si8*0?<jQXL=iE8-q%+{4+03;1=25J_alb^q41K)dkTJczpzo%JFF3%N1X3
z<$!`;Z%u$;5$;`2-4f}ToIKwa*#b(oR+YJO8^1<<<=V#XpciY-D5$@q08rQjJB+<n
z2?$au>bwh~U76WE{g!r&pDXVX2xN3A<*)qs+#B^Heh%j8*nGjOoT6dZaq8x`LUcyy
zss&wx|L%<V^@S#0N3Lp`iqAs5^OO-tyZ2lu5h|@IRr3HSTD}l6qlr6zJ2UAkoid|G
z0a@4pGmL0k1mwN(sjn}5NXegPJhQ(##mYC$>5EB?lLh|gS;&~g!=S*Xv8JqKa0cvp
z6j?SM5`QbliG?iUmr6|JIBt%b-?wf?Q!+~)wteOoMKUf##{Z&t=V#StvG=26W19;A
zE&(*QF+Uu!JL;m2S8ky)a=<g2w{l2%`)frz4o<XQd+@y1wMUL=&X`%~Im&TSEg&W*
z9miw0*Mc}$9<M9@m?#ZNQQg<am)+S(#NovJ2$Vp66zOIJ56cD?nra~ob*fzA)$W1>
zufKv!mQ`Kx*>80V?Od~%8)h)c4hM9i)o4c5LQiDro=&C8nZ|hSNpiA%gbev(BJk&Y
z<J<zGeGR}Vb;4Z-oZu#S#XK+S+<s+LZdX$|=*1>Tvp%dVHyA+i;JWguE(aYpK&L{_
zP@6#GO{6oWCM)Un>}_<mJTBA=LAatAZSl0zS1My*dLjK9t(H?gzgmd=3u3m3&7_Fa
zoqd@r&e`3K#~`A#xGUCRR+W%lC%J45o40GJ{`T?I_?q4a?Lw^#;IEAQc7>ARmh=5!
z-Z~C`8(*3@F3*)zWXpal=mk*17$A$DZDbWbu1@V18}&JPIp_4{XXMZDBXI@~A3ZZ(
zk=>0v<BcrN7cgE_BpG8gqF?NWd{v`gw8T5R;l9t!6xsK234e<Dxh;u8V<gy7E{Se`
zLHZmN?SfK`yz4T}S?cqZG8i0j(gw4Objy~T8!BA49rFO_w$7_*c`~yUZJEZDNnl3^
z2g=9%{-6=bl!Vz@a5MrFK868M6Lfr)WAVM=OIEMdXy>m|Y09ROgVu5r&F@_q6^)!d
zyfTfUKmHl1?vve=C8I_@_?F22-!1WjTWS~l(BUI3bBpR-&ArF+M4t^)*pWP@DRZ4z
z?^lK~;vxCGl7#{@X@Ts;K<A<ZOf()l$e2#O{nHv62ap$zO(QTMeI~Ueg(I4+BR-xO
z@Su2Pv)HiM{uKZslO*i6`UVDgBA&u?_}tOPTgq>RrG9^%>S1wl>|AUs-RLDAK~|fl
zc)FjqFK+2j{VVUbTdoTW#ffUF#NC8dI2{fR;)?%gZnf>Hr<ASmpvj{B6BfZ!T&9zk
zQxQlLi7*g;d(5Wtn)2+tY*4F8XR!5|;tv!>bWXbg06`MR9c(i*Os@8KcaV(-U6(y#
zbnhe+&}?(XTdU$=vYD%m1%BKaic@6C1WWP;8EyJq+te1f7*Dx}1ZeXeT+_rno&lWK
zytzWH*wqc%-~8HxaBpy=l)}s5m4NqkH_HzwEIs_w?FB91bg`Z<ivpEeGFV4JL`UvJ
z20P<N*XvE^t?v<-LAERc)64Y(J>_X>2<R{~AS?@rTK$+S`~CFS;xe08*dq=eN-}P9
zwxu(n3L*EjW`L+U0=tFxBYv@7d#wmMGH#);7Xq#4Q;7!tnXDX=OCPUf-AnV2&sOd%
zd<XZb6W#;WZpEV{F1`S=sK?BA3ec#pT1`%3b&b@KD2~|9mNM;i)-00ypt!&%Z>Rp@
z(VQ^wGlx(!>z#j$H>@EaJbShuI9_6Sj4SVtJ%2JITd4U#r3X#tet)YE;$|;nnpXH%
zpSADjICKQc4CsjkgXjWwEvM`6iyMaf-sa3)?}mHnSL9ZbhQD3~7AET2iR~f06v8O#
z#rjv#$NCC1?`mM!6=caREg*+5`eC)#wWmy%UX^(mufg!;i@Amo!;9P5F>Iv!r&8iV
z0MMJRdr3<nVmI7$eSL0=lj3TAntqCcVoFW8cm?#HoD$tdZdUd2VUKnwUVQ_Q1YQ1X
zOJ4(Hqe-)zeZC9rDwU1)J)Az<4GvBmq_LClSE!d?L2`Di<|&}LycoQ%yA7nS4^6y(
zfymEKO;?kziHSsCE(?hBYi?{>r0R5VGRA%h;hk*qzQ%bF54563Iod@!g1rnT%S@7O
z1BvbtFS~8)EwFA0w2of;xg%sIMgQ!;NatcaV?cF%P2FOzhBBg@a{E4J5HWPmbU=rZ
zZSA`VqjHio*e!+wO>BSeAU`D)IR>tkKWk^ccrsZ7pB6v8s$MqQ1k=I&HP(dm(U|v3
z8Qqi~Y?u`&*&hKLco>U+))+SMgW|qp5X-{A*G9jexYC^MB1$|q`CAZt({$>4XB5T=
z%cjj2r*}+tZ|!ux0w@NOhBGH3!b$zNRNk40pH9Q+5}z{90(?5GKca?{EAYxkcvd7+
zq<cL^u+KWaGW=*>b385lpv-JZELG6jJnomD))|-!jqEM8SG{EyDtQh+2Ri>9X>A*@
z2*SZGHoSa0K7fxt$PNE!CYSHSf(6?1$AH9E{bjUtH-gVrfddXk{$}D-NI)Y!v%;}_
zEBYtEU^t%aKgY?0-VUDJKMnecH=B_&4sY{m`*Xan<@fz|Y#sD2p-DDRf&oR18RFAH
zR%~>JU8eJ6d~y6u?CJJxY-muP?qx?ryI#$!kxUsDQXM4!_E<gv_Bk-<KjYj6^nkWE
zO897yQefivt$W}>hQoWp9=^{tX->I9gDu%@otH$#mC8MMzkUp5=OsR@GD53VS&S0+
z-TsjNr~MgWU|)KRnkRCHJbNjd^cl!+Mx8K)1bz`!OF`kH#9r0$%c3EN)|kZFG7e1&
z0pWdJA@JYsO!^THZ1ix}1O`<`br7eR5=K#zX(eKYvG=*akuL-3P49rS(kg0iHuxCc
za5Dvd!aD!}7%Ya3IA1T?5F}ePp8RUuWlBu3tV;ND%EReCr7AvBQcPGG5KADL7^`R2
z@$<8ia6O{3CIS;Dw`~*{yWj^gO_3%)Te=7#+Pr)qrFf0|hy@K1G!371PCpgyY^6nW
zSh$z12%ZCo_G>VI%L4aS=+kN%VN<M_7hPVQsf9X4NjXy)eW?12wKFfWz%KO^*or}b
zl<+QZym8Er$JSvj%~xse7hDn^3=i9n5I0q?d(zXq9+Aa7gBiFl2QzN0#Xsh&!eM)S
zocD^dO<iu3)N~1N_O)7*PgdQ%$7!mns%DidYL($GvMFy0&%G`ioD@4{y1R*ADJ9;Q
zL8cISHB9$;L%^;y&{Un-2#S0p-ulv@_E{vmAmlRxA$$6(jGteAd~q!*`G8lKw)ge|
z9M#PCjC5GV4hFa{Gf{<M*JwF`U11VPd$IT!UaJ*=g`Cr4@}&iuN=d~tX_p;XNT9{x
zcxRb(oyvFLWp}znULGLX@53FZKhP(xBGHrO^esJpTJ^%`lc8QlJrTjz;w@S8Rqqsd
zF7>`Tx(@$+d6Z^Wgf=|l$xp9I=EXaL01Q+X1F%x0<+6wh(X5ApIV?}Y(&og@-U8jE
zc~~y@I`^x1#LZ1c%GaM4zYQJJoyro^HUu&#pK+R+V)d}wHvN2~(T}l@Tj06`*c=2}
zpG-E6vF)$D&(beg^?A*KRg6H+?Snow0S;a4-AV;3_fZcr8gEDbFl{p-&Y@}8{$8Yo
zV}3;^_thN&YsYi-$)`%06Kklyt~aDcu2(HIl=H9hWH1-pKh*F5?OZY`&#OJ-qvr_-
z)Xsa@EolpDP86D&nppQOf=--2=>vZZ+DTX<mWSBc%Y@wm{$G68mmg?ceL3&lOw<Ft
zk5V|x$L-c(vq{nD#eU@Qa<C(tZmR94HkP3a*e-I|ZX094-5c%v;p)?PO+SKVLcJd?
zP}$5!-6&Ij&$vweU;+yk6$L)4WEcpIz-NVl{4sJ0gU;#JMaqk#<h-#H0CNK3#chO3
zgsJ>?LPh2l?wLuf2L9OQL|2-bSNIaRLApqgVBV!#0#+{}>gBh0xO{bs+<OMDY_B3q
zB>Ygex*ryM=3^~k-Q}aZP`gur4#=ddbSZFSiZ8QTL8%^%uWCstrZjz2QrP5kSZw<F
zUgV>ZGK%tSk2ogrcY4LHsr{sX2(AGR&mPnMfsd~4-JV>QeqXJn;v4_SToS=FY~oXm
z`{}+I#Cu7mV7bk8&&*lrLsI>H(Z=DYPm(i%#P=u0@lZ>iM6rTtC)p2jJ-`45P7WXL
z*Vh!Qode1XyT|F8h?vc4Wf(jpM5|aL)$Z292ir6atKm{_k5ge}7EB)$Q>?ZcAqlo1
zUc{*-y_FFFQ2+ublr%iZo$J-w36FDXH^vT#KfowX)qviTkQOeQ2g6`1q9Zmu2hMcK
z0E}w;HANj{Q*ETsrBp<8Olgi{nAcJO2}`#YD}wn1M^V1Q7{gl^J5<bfm@unh>>Bf5
z>mJrS33(6~$J>mc?kb9#v>_dwo$v1t7icEtx&?GCG~trJ1an*@ntm@OVzj4r0=r);
zvJAW-Hg7cD3Py8c%*y*h%e*v=V!b~32541gYh!7AXnS9Ej4RE$N;<GNhtn36L6>QE
z2fI8lK?I?Dj-XprEu$^KJ<%-|lbGWwF2$m3GFavNWP<Lici+>~$mV&MI9;?TM!aMs
zixGNDi3uB<esuqM)kV5C;Jj-bo3JzY8h6~`Lfh-2{|kB>NHJRx*!h^Dlq48BQR8@>
zb<jT`(s{flz4nem-V>=}Em~D<L_HQ1Q698WB}|6C*0?JBK``HV-i7Rg^sT>wRvaek
zxW`#zHP?s7jE238h$7Ot$@ZmlVCsXJ%tnN`XP$%AT_lE%6I&bt>%JKR>tWPllYjsK
zW<TvElm@egP-bn`x5_>!M|@1)$Sj9?D-u6Y3&qb)g+X@eac@9OogYVP4Uw!?)|9M3
z@)fXm*w*{m8DDn(8j55(+i+sKB6R*`Rw|MF9S*Bm>xGKjvlZKxG1sHd4%vFh6a4Q`
zcIv-T+Fn0>QgEGl-Q7rXHm7#k7=CxxP{aaSMaM{D-lP06nho9seFY;Cel|T=CB^+G
zL5f)%tXiQMRg$rulEM|HSf!-vPcLb<5FcD0F9j~KPA=;EjS|mWS8p~fo_tu~;}ra<
z&?fl++(n`N)6Z&i!4g|PnIqsS4e*e6ZgfJuy&(j|0&|e<;$LftSb+N3iHV!Yx;VDI
z-Pp3lQ~IsQ1&u-Jdk*qlJE$5mSx{8#EGJGh4eA*8XAy!s?EMujreT5K`p0Mhu+^DF
z#9DXT?Ba2Iwv<!}(}Rj|2rL=$28g=WPR@4r=`TU02O%$>NRSqI!_Yzbm!#zpnu7#C
zDIxkOPVA4c7psjm>)z*BUy=>iKTz{eLhpWtPhd5w`E#CbofwYb7(SI3`_r0$AB*=O
z!N2a-G;VbA@q<G2$o@ame$!tn^Ia*KzZr%wFIZerbMTPOrG72@Y$I#5+asqS?A`cM
zTElWW$%i!WG&{&H(jz8tzwh;xP4D?xfT42%*wa|hiu8?K*J7(3H#I2eGvRD+UuArB
zh&t^%4Tj(@FT<$5lz<80A2#h@_%(iVJ8Q<uvVK)(Zs0oUkBAtvWB8gZK>0}Q7jInE
zpNkg!%OpK(<kU+cuo=Y<FNU@kP2ErBZG5Z1Cw{T#haUD>0dFP$Gc&e7J}bOIv3q#$
zp@4o)K(yyaH{x&R+}*>%iWLl0eVl#4{(649sQD&sYAZixY6E~k)6s6vZ!C|GjWC^a
z6k)!EVe;9YsYLP5IiM+r+4N^;aBKpXQtbtq2BD0tKp|Fm=GF@+hzH>=9~G8I&|~%l
zO(<M}I!9Nik%M7EbZYpbzVoj6tXCaX##_R^#X}r9vjm!~@^04Tui#>(z%tTqblfy$
z$xUKp6Na83(XwtbLDQEgztADBq#g|)0(%z%IHIEnKv>%1X-_Ox;dJ&K{k3+^c9SCt
zhI;R9{BVg+?OUrD-K(CLXZFOe%thWVamlFpLQu81Fi7qd6Nc|SJl>iJ1u6#OYJuVn
z+jmOtp9RU2mh6hPnPFQHQur`j^GkPpWMJvXUqWXf-@-#8?<LNW|K|6yB8Z%M`_gpl
zb#z3Y{d?PqysT-mC$|uV-{}trIxoN0K2E4hv-<&~Y&WG=L$@5yr#_DShEM4JV~5we
zGo9g9PB0iMci<P7B?k72EDQs$AYu*PJJ~hhP}Kh&wwnQ_4p~>KN4ig<?&{-xk26eX
zG8>=XvhxHb9K`yC>QS`6ss&To%q=Vk%vlEjUIs0m9~(%*>QJtVbT8Yi4E<up&cQdn
zIqQJMmfni&o7LjFA<lr4?D__H=c#M}slSE-FN<9vY6jwg@>#04P(L_5_kY$fdqh9+
ze0KMl3)5H#@71@kD1hiPN53%A{c`8TSmMs1uKq-vD9e0SgxFTvprydp{6}GsilmNX
z@(>YcrP_e`g?o3QX`O|HWSfKdlMP&<_+8y1ID{FJZ;Vwn6PPqSoFP>}As%Y5Db;yw
zR1796KVQo{7DvW!yjVUWM}r!S2$zScy{N#~q@|ujS`RjB8WY`vQO$FwdYL`n{1(>t
zL_IZPuH4g4!=>%Sd4E-*P^X0Kk<&yhJQ83eUp=eo<qv+)0~#H8T_2;d+7R*4d=HG>
zZ4p&UNBr~zkfxN>M2UE~2Z+pWe+gc2J*UX=4SY%JPaWgKhGXA?c~pLQ=qO{~ArL4w
z1Z8ur8_c?xH}fMh$F-(h(QM#nlu_GRXlV(#`ZZ&gcndQ}whE8M9Wd2G2oWo(Qn{K?
z?+_a(;AN9lKQa^u;YG!U{p`&*1F3pqreW%sO%L=I1Rdwvrq(TZMc>sA^QQs+48v7J
zGowoKs^_s&&U2zc8iS>Ag=Gv#aL6CXhi=R1oiirH6&cjA1MZAm&YE3PVadTI#KdAX
zhlkALXc-KAE^k09^&X#I7Zkjk)&$VJi4hiTeMi>mHs@sF#M^kH|DLZNb+G0AIbW@#
z(g|<!{Sb)^iJOmS$6xp?DuMVq*ZlVsBnSotQF~}72onWIn`3f<^8&#U+QCcSzEqKZ
zq@GX;PGeZPrbz(kqS)#{UhyIs#E+3Mk9B%WyHv{Jy=OYxrAP^COAVv?CW~RdQ9nmM
zY-uHfKtP~s^UH1CxZUa0O=#^u;f5t}2}c08{!>p96w7Cs$_8gW)VJEDKE7U%Sm}=7
zkMwbPy3lMld(1Bo)Gnxhmz?byFp3DN;w%>6fUKGvTDM9{pYgZui`>_y)g5FpA8mlE
zS)n#o9+?&LHL$vPZwXuV2AM0QU&B_7mH49+zW|fSn%%B6lE?1qg6UgL<(0xgykOML
zjfSs8=HqaGT)n(dB#2N4(GqbeWqOVvJPaz6tnyyan%HT3_){^2oV_GGdmViqLH|3z
zVzYoqGU=1rL-hE!5DoO0MB`S|&i%FGj+c=_;fJ3AJK}*$Ipm!)cDKtu@V2GWz(X1O
zSY`1!w@c|Q=MJ$($NNI<qDtDIl-(1+t!Fsy5=_3QD%Z5!a19rks8l@V(M$s+Lb!Ga
zFFaCHUh_&wJbiKM7r>F;Wsg7*=Xf69=v|N#l&~#%Y0)*1$Z!?OGKo|~UfLoVaSfBf
z2=C}EzPg22kL8brTLR{3bw+NR^bkRnf<-&o=Au|>J4aMYtV=29H)7N$+^hXRK~$k4
zI%bF>RQ{e(QvyXMmIfwyTE=a2dgJL>^Z;^E;yd)X68dQV4aOnn8^_-U60^`uCf(IZ
z9-&&-&6lZXp-Avzh1Nbp%-8`wMQB2S$fHxhOE6n2D(r-E`r~UH?J}deh}FD<wn@d}
zpBb^)<ZTjY7_vyWOwd^dpqiV%5T)Mj!0LojegP8&y)#{$2$RqaP<?US%6yA&Y0%ic
z$zZQCA2L3QC^pLqF+TWKSX8R<L`vZ2`9R=?Tr^|RxuDw1UK0M7$5)rfR60&YPsFY*
zIH%Yib|x8b0L&G2;5A{N2ehyi#)lG&v7*Z3JY(}=q}-%EhUJm57<97vY^&%HG!_;`
zYS`Lo+Da#CBF}9jN_QwPy<NGx58xXCYZU&OuVKBr=vC(9u7s47#1P*3ARjY|R7|Ed
z<{Ez6Q#FLf!_fzW#S&nwuS31#pynwn`-X<h^e0o1kKO@W#K(s46P=fZdR1xX_?K5<
zS2@7uYqFSP*RSam5Rl&g)a4%8@$Di>=pNZlR{`Oe*;{fYm(`JZfk1Uz(1m>151-EC
z-u<g>;VX%G2nS^hkXXKOqVfK4gVTzG(0LuYsPcMkDYLRHUj^fb5<y-ZT~-vPt+M_F
zV(3^P_hr8gvf2U>h2ek6`3dnc9~|<YDTqiwvGK~WRn`ixdkcBfgsj$zBJv6%+~p$a
zqoBBQXh`_hvqk}exGVzNOWhr?b{1DVPmJ|fcz5_4&8R1z1iniqL7J6I_b6*S+$c8<
zzdyg{Bs|?%A4kex-$BjdaW&(patSflqZ8Ic_#W_j^jW9rx7AkpzDS-weNBbbADz#4
z7ZYO}f-pV5*bS0<22X_cKe@hMZ1rJa(~<KOX#B*5AD8GF5=-AdnjsZ9Uh!G*F-4(^
zn*dp$nH?We#hX(atNl`JVLdRjXHI5|Y5y=$PhQlhTcpEYECP>%nR|W6nL;Q*<BhM9
zei|JmFQ}cDlXEIFhJns9?0ClG5;G=Bj^(G%WQw(XMkxC>2AW=N{bh*bE(oGZYUiX-
zwCda&Q!a+brt;cMcNC9-NvlbekrqATmGMtxR<Y=*B{$+|hc>Qgy_En!VoN)j*<ZG+
z?7=_gB)};SFRG(Fo9e-zx_fJ%pxOFBUn_C%@@_~gJ{#4-!A4^sE)J#|3>|W8Scj5>
zz!rocwZGswlOAN?qtY_z{&Ft;nF(*PRm86~9mEQA2H*`lPaizXFo{%qSDH*9%TJA{
zz$YE?o9!&ESSRhO`NM0FxXjZ19k3m|&!2cP{!n|m+>ES`>ru<WNXu91WwnL3)ww0x
zia|%h^u)vn?adVyQVRe}=j9g{If9H7W%}~(vu0kjd9()99tF^R#(Cg1{3xD<9#}0H
zd$|LHG8sq(ZJGDpFVqMwnI<}>>K4NfGek=adSmV&?%7NLzw55CsRc-U=KMHh#V4!#
zJR7;|j<rgj54pDggb$}=`tHVL$$JeF3~k24mm7XZmn8D#Sh4r?1PJxtEf{-#%kQJ3
zy|eet*vTc2LLrCb`Z!Yq*WQehQ;%Zb^o~3^P>3uR-d$-h4B=I4b6fO5RleX=K)j?w
z^YGc_TQWw){}se9H%z2!^r_eVGxokW{t%sx<EPga0@8(Wm}UG-l6?S4asdRtmY$Iw
zMgV&h&NOkRZCGbt?<-RIVgz-ZQ~g0jO6gbS0tpZJ(Pm`N{T}fT1gxaMIv2(L9+Pd$
zmDa{t&T(iL#!xA6kO3&?MgOU&@?$T4LhE9uPO-B`?tulz&pj8zgr|-#%-j(&#Q9rP
zk!lsfk$swc|7`lJM6=nczOYQFT8_Ermc7^gx#5qTpY=O(CqmQ?5D2Vf;k!PX#o$Cq
zv$*vT{*BW6H>^4Wg_=3?Y@{?-!)H7?|C#X=c{19O>BAWkunTHj0Ri6U*Icn8-xr?W
zO?o*hr8=9WDDK5NR5#^IkGG8Blgu!=C^rJ^65bbT6)XqR>YTS1dc8=%wVs8k@8cpR
z(+(eQ`5yMk!KJmA(XJcSfzCRaylKw8_)DGFqEf06%(A+3YE1q0c92C|zO5<8q+iaf
ztg?ZcsU3e)(MkSHb0EOvx<-k|6f61s*Y>>4tE&6BK!uY9&Id9QQBhHn=f}IcTbO9A
z!h*>`ni}Q!&QS(ao;$PkEceI{meR|xFgWbyvAfHsTXf7|4In8cJ=>Um^7%|x2yZb4
zemEAT$X8Y719((bD94auhd1qU&xd&p>>##k3n5gLxTkgMWL`&t6nv%~@_;3lhW%@1
z8qc~=w@d|YuIL+pk+{Zh5k;81SaS;eo)h#OjxBYIB~m~p){0Zf(J+`eNUF(%<d6c>
zC4{#wBWhp9EGhO%Le4SXe1Elnf(~XU*fvv@$P%?ErUc0^j*ilJt!rht+W88E>H+*>
zRjd=iKke&IJaGNSpoUK#f9l&i-Y!#y$w!^X;d~<38>p34U_h*PK+HaHUKZN!IJI`S
z=m2z+FXzNkg|zV=&oXgDlOl8x{aix{e9p=|;L<fksU@1e4RJ#8Iw+5R-Js!lN6h_|
zH1;m|Z?ul7gT$4|_%kk$_gOoOz$AnDDfvL}c%gRad32{D1>%~NIs_DSs+GrO#d2!=
z)BSxU22*HU!=*3{oyiz!pP{WOJZq4_{UZ;C!05!ds)CC_>AojXh2gxJE^-Js==Kj+
zto#V=#0b^_i^Mn=rxvjZ4>h4}Z?^ea)?^ONTATPrBmWEk`$;2L9v7Z!Lq%5jmQESl
z{#<(nd8rzOVm_xAfU+r~BC^e2irS@+ZC0Y^$G0LT6uq!Ct5sZKTL1d^UZH-Kj<m<^
z{Z9uG#|dCMrW={-H1Brt{<9fLOc882q+`zrg}#~3W@ealZ3v2sCDGM-f*e5h?n6w&
zEX7B`*DI+3G+AVRpO)>7)rNh@68M@_kqVt8h)|F7YXSW)Iqu%0LYxU_n&`lECRC~3
zHGPEJ9!V1*lPhrP(ReR^@C<m>=`|)w&jz*}D>!jW-{9O;VK1{xq<@Volc9#z7E2*`
z|A8bryb(A2vpsj!Rst0l5J%nisj{w+1apx&Q~CU<G=-{<{z+z=L-XzSx4i{8dcfyA
zWD>OwCoJDgJO#tO-;M}k*Rvu|14=ODKlE{8?)?4Q9YXeCMzjuUh>xFUMdh{nipog^
z{dp<1=!6tbjj+;tPf?=49{Fe_pGCA)o(hy9hZE{*`A8a1A~z5`I>8d`>4Qg!R>2^-
zNS6r3MS#?N{;WzXuAD*Bd0P&pkxgVf@~r=GC#JULVcfInjK$o>sLf9jGUSntl}}93
zPds<f$u3}ICeKN3$f|&yD*UH`CpvlbOa>MqI9dK`VWGyF1o&<m@s2eJ@yfFtta}vb
zaH=XRn|1nY9D?rd&KA3fKQ^gQ2VFXn5b<P@t}32*+b3*Lyq|fS_=n3@plts9g@<0T
z4p(x0)3=dRrm>f8xu}dHB2Kb@aYZc8P?<WhR>XPGI?I*S_HsU4kK1l>tP5j5xtd)#
z!azaFt`MGXG}yFw8$DL_BSs^@lsN0T>|mmEA%MkmRZsI|Q&DOHQKiVN>n-)eYrTX5
zy%l5{17M5%<&oUM`p}qX15)LK@^e}}!{Sx!<PT!DpO#VIfOhXp$MoAGgS#z_h=!ko
zZM}W-u`TWh&&!^BpQ)wZr~wV(Ak)`C0{a&=ux+yfJ-|0bDAb2H#`GG9Bb?04X{Q%b
zy@)0WcO|F-JcV_1{JvvDl!2Z_`&2H1(oClK?j~R%QooZ<*BdW4k&+?w5rPm)_*fBd
zS@6E&ioa)q>fo4ragmcpl>_L{?>%C8Uo47g-R}x}{pI=f{KxB&#O=3~eZ(9_#qz=e
z5#3qr(cZ<EPxBo`^-=~3$q`XcfWjXo3|RTUN^5DkJ`1&S>%A{Qpa&`GQ$u^R(&AkG
zLArmU5jw*XD`a~fQEsoR*c(+kUe8A02xDuli*11+I9zmk;9w)oi&smgvLdgKD8-m6
z2ZXGj5Gb9}Mdj+Fo6q+bp;_p(Io6%}^2{7(G_N0&v3~Rf%A~jGyeg@m?fS=mN)#e+
zt-N3${C))tz;Pz3!d&PEMEw#Fmub#;-l0qMplb~xNbFa1Q|eyg?xP6qaCq*N5X<{w
z%VOZrT`i^DwKCEH9HFr5O&Wj{l9k>`ZJe;y(KOS^3eBmpnbG+uJYM$rF;Uh`#J*gZ
zTPSBB2Ygzw1N{>evs0~Tj|Vsp@^}L0<Wd>yXER`GDOkr5ZT`_KIrpuIY<IY63`xHw
z((Kl`L&LmA>ThNtXuEgOdhVp~mzBeXK<fX22A3_!U9DY0;(qJGjr}wya>dmq82ZU;
zezndkxoyYMN|wI#kCP5TMY505Lu70K?yN|Uu*mf0?b@;WA~_5m2Ivv6`+}K>-J4tT
zyzS!55;=WBW8n+`4`Xi`mF3pG@k)1hr+^^c9n$g8p-88obVvviN|%UqcSv`4NymfI
zB^^@Ia29U&d-nLBFXz)9!#xmrxYxYrn)AATSK~46?A6I*N$xr!L4IqU_l7qa0&#+q
z&s!zA*W2ItK8Ra2U+q%eVH#(yD%Jn<?`+?e`=@m-GF}2BXeg&#P|r2t$ONlCDT{1=
z9%Ux(V-2m?uOt4!lSSGrHJ*H(oGrEH+gL6LWkVQnCRrfrp5?yBY%kEDS^DTH1j?K>
zoUWPA<)Q5?&5?IFBACy23-nWbPb$+iLV1-DHMmGqRO!1(qeaR)8{J?(bh({u3|HYJ
z_9WHmRmhTz@=*$T2ZIO%`yr8{9^tXbv5zQ&uhPA7H7YEySx7~|qRtXgEv#H9%ar%+
zvSuSl8)DFMwrktxOX9K!RW!r3Z5CVyD~~BP3G4Tnx1=;obPw#V41fRW<xGv`FPFi*
zvXLPCL>E3s{)9heH^})mR_iwZW2U#$!^}*`jrn8dt%EW<$w#xsOuc~VHPNCQ_KSul
zK);SLkj}#|zDe;C4Il2=s6yLZS91fN@JS%EGfUa%ZN6S?3M`+u;d~4=KRz=qz<&|m
zlR<<u4y0&A1o(OtOQGTi`b?|^Vhlwo`*;C*2pSPOS3*8F0yXwaY2Yf!@-tk_<Jg?x
z6EEkXZyZ^j2V=3mi)AiXu?Ac)9d(g4T9AC6yddFp&>p_+Zw|>lbnNbWwyk@>_vDjX
zL%IqAj)^XP8LV@0aokS-M34z7=6j-7vF}NFo}#zZ`tJ7XswbRNtJf)+z?6U|8Y#mc
zhE^p6_wkS~bsMqSr}6v1IbXq7Eb=G-oRkerg%j*TJ9OQov@#`ydL7<(^C0o*49WyL
z<!ch>Q&UQYgZ;b*Wf>$*WZRvhQ}B`Fcg0W%Lu&c}^58qxFNE^u9bPzBbOp5j?Z@<Z
zk|$*BDVDqsgO|ASn9%YPltF5!gf`Mg4iPfy6(Gp(hD*1^tqzrTf$Te1o{IXNRnS4-
z3Ui_c>A`X3vx)N#pHtd@&5wN-9jIHPH=?d>XA<k1`WfdbrVrMmL|o}o*piIAZ+*hG
zXOyU4bUMM-tzGc1FmtVW1NXt%JujVMvwt&Hqsmp&!0Q%BdNDDCn7P49Gga@gPdo4p
zyiz|^K92bJ=mPhJtVHkD*!T4f?uQO}@jo51?GU<f1nbemYVQCWnNL9SYo5t<)RWvB
zOu7A@o!_eGin-}+5jTC0<q#C;C8>$1qhq_1VBio7s4!itq9{do@?TatZ$dgz-5eT2
z0jnJk!}F64ILQ)|#b^`v2{#(m^+y5TZiUdV{TXn@lvY+BmXF|4Q$P`HK%`H$-m^{$
zGW2AMXuaK&=J_H2;-6ujAGy?jPBDs~StMrECTmJ)jSZ^zCl>7z`M2j_7}>Ko?@18!
z_PfH32kbc9NU1~P$Bq@!ky14Z7@RCVJd@?IX3OChHd6&Lp?D)yOkdW`Hid5auwPFf
z?j37(C$T5$t<?Z5n400ELL*`RkworfA(yR!DxNw3Y`m!#bXn>A05Z?R@p~epzMFr>
zmM@WnJ`kN+xLzl0t2J2cS{CWor8V+_#xzEZAKP|$3e`Ka)r<>==lwio3mW4U5V#z7
zqB_6&z4yZ<0fi|hX;hM`0gu(lj_F!DX$h^w621rfO+5r^MnU;HIVDB;Vnmzu=lP<t
zYF^<4{!<_0geX0;6qNkE_GX8dUtjRf-ke+RciF7kcTej+vk&;W++?X%#g&%4d<1bX
z0EEkqX$h<~6aN{;sTx~d-J;x-asp*U9VWE%H#}bwUok<g12fU#WHKJu0&zwY*#ko6
zv&`B>1$V%DutVPW49sKct1Z~@<RzA`*P_I4rXr$aJlualuEu$Esv0J=)ONt~OnAJl
zuL6At1l%g53N3`2|9J5Mm7sYCM-~P2$P<$jh4SO(BSaR@Rc5gD6V&6?Zb!ciC!>Ul
zcI-W<qAqrw)h-{I^a23L38<ouTG=<bN<JCi$~b}xArpOE5E;so@yKrpOmo!=cv9j&
zn<`1$N_ZvO$T2xNY%%X>n265a0<-7E0<5CIqHPbrsu1geJBUun5$RO#@BO)aEV{*2
zL!adiB!bm`c={RJ(gudro9HoWs7Sxpl@RW`+cR$&t><9xtmQHHSVvKC!+G1}aniPO
zb(1!ZEQUoI(#7u$p=}$JZ+V9<IldPwLE7Iz@fbRLC$+z(3WlTsxVCAQu97NSVw)*y
z-AUI1NZDh@KSxmu^~~@$RxXd?@e>yWFmH)Qd;M;ahK@OP-eVIz=p6}CrX<YHcC=kO
z5U5Xjp&8_-08#qlKrq`I@GJ=32SktlB?@#m!A_V;E}zn>eZ^#%;>gTn#j|-09Stt}
zbbW93A?ZAFa!S2#>w+_U(=?w9rta!)asl(nLy!|Vz+eE*4{shmlB_!ghTnBe|J4z=
z6#h>~;HMU#=@ioCvWT%(pci^R^ZE7cyW^ao#l3V$Iz%K*MOly)dHoH%Q>Y$XpKwV#
zFh@xu=)HK6<kb6Me(Jzc)E6j*+2!$@OQDA1gSg#Y4g$SKAbvQClDIbfP`IQd1Fpwb
zW;U}XSZA$Q*rIO_GoqK_ot@a)7kaY0!p6_)2rv^zwnMNC1r_lOg%$VO82-49AHWJY
z{{U5(r`&0hC|?f^^hDQd@M5zqt`MxoHb=VX8n_bN0t5I1622^eRUP=*61K|aah7ll
z3$PdGVv!v)IX<PO^&TF7(0V5Z7HTzIZ|mc_Y*sFhO>%9zK-@{S-9l2Uxdp-*VQ1T#
z{aov?mf_f?Wt{;?T&Lnh4n+f=(4kKcQA2o?4A7^cI2FAPj<3lw^a7j!JwO@Uj?Mk0
zwZ*s=TOQJqf8M;owOt-<&v$~WKy>|ha%ycD*KAd0Xmj>9kVN}b2Oow|xSgc}i%Klz
zBc~tJr|FXgof1w^ad5eQeZ;!}OCeU?+NI%sfgbZDZ)RMb{XWHg$UZQ(m@v+u2{ig?
zv-e^z4aFqULOEjEF@4ul&j#T?zD@Av9Ifz_%Q=f+Fvu{CAah1*6r*I2-@b<F=P_qo
zG4j3DPLH%k(+&h_7sb^&)gZSqQ|no86xCtT&NI+#;};@Eu5pOirV6=gNDGh$jKP_P
zm{97#&5aBqvTOm^!-GNAsr*Rfy}(SVfiQszj_snCQF@jG2a?Gnb5i2jw)pc=krQQC
z@;$76S(#A$qlIEpo)(NR{bZKXoOgCVDto#`IYP865F`BQ1o|T&=%M!DA5}v_U^A@y
z((!|bw~$*#aNyOm4|(v#Ly1Jp8C=h4c%;BPIfIf~tQicDCHrjWLG(>`s2bte{L6O*
z1h%I&PO23ftU4toogp)zwV`MPZs#)-Y5nFfbSl1=y3<2lU8qJ>;r9{??)9~Hq>BV<
zr3c8CBMAxyjczp9bzlMg?e<uxj5c(<Z#Z4t=t%%jJBRj}nvD~7_grsUQj&qN9by(J
zaf?3iw^84@wF~>Hzuu1qwhLI2kFCn*_Cn~2LOQwJ-o<*_Xs*reUumJgyl~M~Et6Qf
zN#hKL2y8hH-c!$4Sz)!E`)0dn&_8{t`9P^~K>E9zcg)xR6b4?%=0Or?jU@o%&Sfz3
zxHd2;@sKh<N)7;Vns#F*Hy8f@Hr=L}_$Ofx)9~mF<nI`er5m1L!?$H`^;lfJuP;$Z
zfBU|2xp?owisGr%^RCL-Ur)0@jpho}+(MuSVo1}Yz%GVg4H^ZDNR}YZcgG-*Y=r<R
zf4i%^u7Vn>oWv6X)Ek!2pFkO>GZJk1z;j1=FQJ%7?|FP?>LR1xRR~px6&3Lo^Rg<^
z2(KW^-cLy>YstkH{cXd^oKmttk@7i(cVzGoGI8RbKaqLMm1iX*C-tX9^;7(NVct%N
z6;^X-BYf+AdlW>4u_;H1c-XuP$(4ZXGa@E0S*g!YeG$)Qpcv0_NkPZ1#du`bx*esG
zcVb>I4X2HKFX+01|5M4d)6ZQ#Dp1di2|BbkAFf<$x1b1Mos%*Gvz<S1APck0---9q
zj_9Bqc4*H{a)D7o==qd>x+zBGu^RB!Z!3k7MzQ_T_YF`%pelKANWV{tc4{6-hCG(1
z#N0O{!iVN($Zk%fkD-BaI1<0N$1>qkdy4Cdl|IgJ9G;z*<ppgFUuy_nk%h9~S?xyx
zAr^JSN?O#EM2W{ozz*l_YFS#TE*{{i-$cXbNbyLWvU-eXOUhj#+Z9;r_noj107T;?
z=sCF8!?_cd5#`){xfQ2=Ch7Ym>N*N8TNwWE1@r#90jF`CgDT)#NL0B69EAnV5)(Pc
zlaI@|p#&^;zvE-19)0QuI-bJ2$BaZ9uX)bK>tD82uENRVIG{&CGce&ES329IX3zS0
z;h(UzE0JCGRL--nj_CD{j@2le!dJkCWh-A=qq;UW#D=1tQLfhk3_tAAE-+V&Jug>`
zb{>)>{$6Dj#s4)Zb?jV-7P^&8lEZt0)*P=cPGNbF%+E8PqPkh&(_2z)AqRAKE&|@-
zBam|l`VLjSju30___yaY8!kz-V(qUCbQSFzG}<4isPJcDQ(CVlx0s424SLPmBHgye
zXwK}gRmdbp5H7Q}Kd8Jr`(+vIj>WWN@>JTO8!$`@+fko22D!!QO$LSVE{3PST}6?x
zj35M8u4I;<qy<r3r!4{Ro)*A&Queddk7n^Qd4lYrG`VaX=Z&FEfXqk^e7XBQq7`P8
z9(k=C`2Hg|Dy0lH#OPR8{kQxZMehXqon`O!Sw5Q%PnPIq4|UO(&z=Ifj%QTF@YZ<t
zHW<?J&jIp@htj=eHwrxz9ZXqqek>`Lz;wRVq0=tu@FLmRtT7Q=q(0ffcCQ8S-c8%H
zK+>*IdLmA?#Ge~_Z9bwV2Zlye?uv0Q9~$oO=OwZ|c8dmm<N9?Q<}N-D=}vy%snuEx
zeH!CXcxM)IpGgg;DZ^%3GUM4m$oq`hzRY1e6d#Vh+5dF)KT)v<1q?~{$p%3+pw6+P
z{)aj@6h5fmqFMb9<jkC3V8O|VOZ38Yy!^%8(ugwY?Morn1XtP;7u@IA{b<PR@D+%~
z*PSNPWaI(mPK-WVfgS27EV?#xP44U~a?b^9q)kC>FxHm5K`z<{n_W%u3E9f5%8_Ta
z{t||gk9^!&g$c97*zt3ntVoLOg}+|+=TE+xo$J#&-Rre(60w~gReji#qgrj&g?-s?
zShzQz;BL`;wYC%fdQI^`wFG(?zm$J(@7BF3DfX^=O3DffGy?rL>3jl=QjevJ9pG_s
zN)8XHJs(7l)6A{Qp~p%vR!&_J{9{B$%K{?bj6X@<dAGWU$cONr)I&bn(z8Hcqp4OP
zb|*ZfcbBb#a5f7Kq<*2h4Y`t)k9`^yi1oFv<`|fOJY=<A7EqHiJD0j5d{a}k?=RM{
zNQK=>o^Q`*>6z*DH^oMk2@*bkqyT@vY#pIJbUP%Re0_s9ibrnP1vXq=k*oEbU)YT#
zfrm(4;Q2)lhKo<n?Gnh6Z4=1EycvK+Onmy(ba*gss@~Cz+p8c<;s?D(u|{sFr))GO
z_Mvh9J}_&^B^i`LtUxJS79wqVCO{vxM{3Co<p@T?qR5%ZRkDijKJvhZTk&43cqT^|
zq5h8hR<~pf!G9Q3>!ET7x8>#A{nZ|=WFlTkD76<bUdI8EfSqN)@4cnEC4mk3H?UJx
z4phu?PIlr~=`0Bxu&couCFAB|K6kCR2l|ayI~$g*Zo&yCoZ7wuM%Z2>PpMvjPcCd#
z*;?xIM%?73b8>hlh)P=oj*~Q6q#d&+R`MJ1%XPCz|Jj(%O&46-3?6t_zNap5T_?T?
zOK=kAFf=f0NWoY-0aarCTC-W|B_L29bHyMVINX_qKigBAtw&yy-tK@!TWMG<E1fYD
zREwv=0jRmvOSXZ=M)KVlMi=o6ASO^YrCZVJKwBNLM)&<oLN`%vRp}9N2jAx-CXC*L
zBU*~=1TZH)#lSWns*qRBY`cBzFe-V5{d-)FY8<S!?0}TT>!n;~QcY!K1CRl6eYR-6
zHL~sjEj-m4n|rJt+!-0K_yMapXm&!AT`{fl0UJs&L^&Viy25F2F3KEHBB57rZ&EGM
zjvlu>qQyl{dwBT*#oRyai#5AldVcT>k0DQq@Qb@{<LOHcw=N{|tSy+G4kWBe6^yrU
zPJGpW=gamcO-t$DcD=vq&q$3u_5Y64OgC^(T>ie@1xQk^M2xfsRzISRU!`+zeYKiE
z(Fb;ULu_PvXokT>=0f5AQk#+B`Ul#C_5l=nd;!gl$RpGQ03JTRJ|Y+I1hA%g`?zLR
z@!P2z7%@GAP(s$hVxAdvx6tw|=ZBH4GuVn|6a|nhFLWsO`xT=j_KrWGr!_~Au88$Z
zZqVdv0hJoJU*ZCWQ&2Dwu!*Q<%S{4!i}rv*htTheh)^wX$~jrU=#@Ud+n|1*^$^<j
z6~~t1udAHcxD&<CidmbBPawldL{kH_vuPqLnemj$c%84Mw-jjqoO;u+zs_GsE1KEd
zr}_A6P|r~?=gWcb1A7Wrg2@k|bP!~N&ETaFa`w0-q$Z~O9(eq`_)2?{4q99?W@4fC
z6`d#0dWjNFZG*Rn4!gBqEt^ub%sTuL9&m?k1s4pcvO7o>mh1ZVi6iZ3M7<gn48`0{
zHJ=`62}^^Ej6V`jUGMC=LYHTqSSn;^MjH`49&9|Bb4?2)7gzINu_#64Ee1cU+n_=(
z+^*v*J7H69YF6gBNI_)l`GOpnqOa_U+!*1$8^~d~E!pCddULBFrVxscx6{O{N7+_x
zszL`Ef9xYQ@Jm4%wafWv|E53<f7xF9jh=$oSf*k6w{QAjsXl298!IAqw~jXn;*#AY
z7oL$FsPl*L3N3-036i4Lvz^2ls&lFXgr|yJfc5rszQwE$GnIahHRYIyyAY8cjrht~
zAU43ooVPd>23*((hL#__;Vt+^Elzy=-pq4ZWT4cb>2p#9uR*~r;0#>Gc;Q!kr09P}
zvZ1%b$u|r5I2)VPrCr%S{PnnRrgc?r+U|W&41iuUvNA9!_VS?P{M*X|evU6`Y+b8-
z&w|i-Ly)Nl-lw2QsW1g__4+U@?A#%(`3`*kl$nKu(I@Kv;nhLQx#593YQoRmeri-$
zt7xz8O4?teZTD)qq10)lYom$bj2mRxHFXyTh=)A08U3PQWcY4tkZ{IlE1D+MAwJMu
z^gx*h2NPkL<Rk19urPHoW6@(ob5_3_hEH@;{tgC?^_Uv^;kSq|Kc2x|c#wCxRwTvo
z6cgm4_mHAhFd!#R8yQ6FRcxd@bshlmm2rKez8*ehr%#+F&=73?WdV@Xlz)lB9%6j<
zlR+4cgygoq7NqYy8KJM2ADW$^Yf0`VJ$3KWgONX*RG1IbD;4imD@ZmerF{3twBL;g
zb0SJF{~NNg;L7#Cw>aRq7PuPv{G6L;<eq|<nR&Utk{ad|+FoPt%i|cBCU}^x8x)3{
z|7VHK05mDdYU_m*Kzh3n{U7smPjlO0JX?O@AG38g1}p@TacKXXN<DnmCF1q5$-XKH
z&swV4>B2moi=qIZqw^uX%`<--+s>Y>mkmzU_Dii#2SAr7YYQzoqHC8(l`RW@qw=V7
z82nj(W0E}}%vpB^qCl-T?jFa#lu~OA=`HuncjYy1YA#ni1#Bv0i2e@}pHnC%<By)W
z6Mw+4dw**L==x~m!+=W=V7Y>4GnUt&89DjwcH;Y!i5V$Ivubv?U!oUQ5`y_BEdzl^
zBBa+YtB#&0g*=xr)TG#tzy6s^vi)l$ATx{nrXsxm!;dLD)qAGQs0{zvvNT#(=rMJz
zZNTJ(p-<cBCdKP`w&z8kWtZQ0(N8;6*9ax;y8L*v0PGnC3<@fb?3M;Yd8dmkb*qK2
zDtkaO%gw3}*!Iy-<g4_M9$pW1%H&7s1t^*!{3f%j_d8dk$-Y(SWrGxJcUI7P8&*o3
zX-rXsGV9eW6(*V!+WT9F@IKfPo@>X{zFw4?w$%b^`{?A}2(rx<sT^b*Fq_WKt`4rA
zDAvnnw*^N!ixx_)Do`1Ko{zx-%}#mC%~FAQ@vd_nhpDRGBu#l_{CXdzhCRB?)Fc6G
zd#<jGmO&y$hJ+Mu?RYXOuWId>L*}Eh#iosLmk37xd$llszm!QJlFMKwfL;<jmGJ+d
z42$6`4Bm2LNL~fHWs5eYh<(U)FPO5jcrkyg5Fj6h>;?0-at^%(@XjqsVD?8H;wv(X
zzRjRC|Gca{UO{uMhZb;!ge=DaEI;3Yh_xs(^{#`QH04m=&>17Qd`(0?2S6OO(8dLZ
zpY>H0xl^NuQvDE|w)vE0=L-~_dgdD|K~y@SQ;#ne&&1dog+8aHm2SWWVUXn!&U32v
zM*>3bHM3TU^=eQ<PqS3XI7SCC9IYyoKg-qQ0tXWapKzDx^=byu@SXeJ&1L9VkW4Zx
z#-)iJ$@w8gix&$Q_%ZqoU;I>UV-%o)UYu)#$;(vjV`#=rmEPCK(ct|X61NBEALGMb
z2jzC4S|`JHM?X<3(kOX?Lp(h02r_)xjJY_Fuac@uudxhhnAoJAQ##*0ygjl1T}3n0
zevO+)@avY$(7C;F*@YXJ+S%y$evv|bwgdtEp*JPiHYv+-$*c9#=bL$a3I=~}st^6>
zTL~~XZwC0N{{u23bpD;t^v634FVnug41b?-ogSv#Ap0yVPPo;|X;>8v!c36->^k>)
z{Chv&AvLy=DTdBvk4@X`a&rKc&Wak*5UegtH)^2+?2p!WUSGC!`l6mPQXmFCE0O#H
zoclT~O{+mKgaf)Bk<)eq{9fS0<y)xE5vFlH$=?Mb)@CrZ2mK53es)aX);`hXtvql4
z;5PAo&i8MnEb@R^3F%RUZ`>15*bl2Ru|O1^6ps6~=Jz0e1-FL(t;m@9`%)&OLWOxB
zHBhj8??TV#KQwY9W_Onr^vBJehL##AosmshZ9+t$q({D2uSg8Mj~+t{BF`V<kEw<P
zIh%%H6|g|YQm)N(kwv^R&XOc$<Hc&&?d+0p@Ja5-5IwZu5teWvdlt}|?@X%O$5m}i
zjwWDc1J{pw<Wwft$DM9KMr5L@$~e>JTqO-ovjZt%uo6ywes2-ZP=#ILyIBDBRT8G#
zM(?IQr0XjL?kB2%9qESpNXgG{>A9*wEj>1;zBhMBWz}V8Qo$LeoX&Z=7>}hR)i`=~
z4$pSe`#3L9k1RYc<Kl0pw4TxP24sTuqjrwWC9kb?zFt6ho2A^k&R#(o2t`td*Phuc
z0h!KP`49^3Sun36(E2eD?vUh;ZaR!^&-!VquF1BW6swis*ObRrd&e`aj{`v!JVTS{
zBtn6nzqba$P>6r-96RbBY*?WcQ6IKmm(mWwLlM^dKcEp-Ud61KM`Be8AJo%@Wj8>q
zQjG*VZ1kuV;6E`+_KTd)dxlHko*h^`^@lynrjx^T=4+HYE}xy$+nLOd8WGwe?501(
zu=w_wH`Y?F!~^6SC%T+QGgIY)K+ArYb<7vR{Mb9McM^{&59b@W>7QdLGM80?z{op}
zyA?`QtqYW=Smx~jn2?I1Ak&=xX$u&j)#;ns>oX%CW}4r0pXQ&Cad-?|-(-8gi~`V%
zNkGOVV}3REi^}&RhOJdP$~|XeV-Z9FWjMNKD7c({@m=S@d({wC%RQe?jo)%QVGP_2
zq5RB7G$U|z2z=ams=D;q;G*@zg5bBHv54E#sU8Qpc99$9=KOO0$dg4QkVdIfU>zxL
z2j;BbR#$AHDus|9RGN9Bmilz!&cn2ytm`}mJ`8PMkUIx13~x{4F+roLZ*tO{8Y_uN
zI0)-)#<eyCEUkr?v-FKX3!{B+9kHoNbNWlT%jryPD^PDOxtLEY&?ZQ$Wc*}FK}!s|
zBJW|IW3i*vsvq(D^+IKw%~T0aHP*~ouUF5gc-gYskrzz<vwO6jBwQS3C~<ev&y@Hh
zhf>3;*)rd$pF%%$inKz~uj326&JVsgqCSaeKEJ7xGk8|;6(N!lavc9)lf@SMdlkU*
zL8JcwPMRK$K)zq5+GLR?I$)D?y}GJ5<4GloCHO$fa^+#Q1Xgs1)p|A$c@N;L0WA^Z
zVRTP6^Q8RC13ks=KMQtKLJIEKzqf%v;eW3290k@>#fxwU26No6=F;=6XKIyUw>Oqs
zm4&?9pS=na{hYRwX(lJ?l0u{<>@ok0gmK^^enq{AKXHhAxrc5V3#0c7_xd6_%dNid
zwd&Y6=HW;h*l}FjUv%O=TRn<jfQuJiXR|BneK{am;QF)&1Q;>}(7}+0EOX>xh)-aU
z3#d^g=@+jBkq~cS>f8sOybwUN;w*7O?)!+w-W(0WMx!5lxM9*f$CU>k4I=DdS?HQs
za`!wDQ+@mKrm88U(qd&hI%N|qLTrICO_u+M*a8PGb*C75W{syIp}F+xire(e+I&>8
zR-d*?s!lYgosTD<Ytu4L5c6{M7RRXO7l*v1uw3{kEUQ0>I9H~@Esl|<mQpm$d6j7F
zm=Z|wJy~INAyKi|7n?+g@vXu?`5dMp|8m!&1S?)ULw+uRP^G8E*JiL7BNG@qGou<9
z0!#X-*=Oc+*CoNQuyXVuR8ojFcoz%`CxRktsiDiFhQoK9N5(8HP_1H}VLn!!@sE{~
zGKL6?Hu^sXz&SpfQoMpXwAh2U*KOF_I9JfUJ3z_%i!02y@b()@hh0wr$ZC5vz;1;Y
z!^(8QM}Q;CPOY?I_n>LZg&Df+)A24F1C3t7@i86Usjv<(6%vrh@cQFfa{0kT&i^QM
z+~O)1jZJj}X5BaNPdF6bdC3+Heu^?G_r2cduPOIuLpp!;&IbZ?w1dn2*I;yck=nXn
zJBK`o0~w}=zQwBtB7nQGE0u^{Kl>Aj+?aMDNHSl9UF)SKLAcjmTznTD3c?1_u30gl
zd%==)vYM*oxtn6v5d|Alm#znzESjzN$4u2L;Ay9v|K>}-&sFlx0OGOvGG;YeeZbHw
z96<}RoKJRp7vPDh`&HIG3w;!SgSW)({hYy|Qe`>rFtp@0Ze+#`IbTj3>n9$QRA8m!
zVR*K1Lz{mg?ImIW9$(7%Gm^Ai5Lr)_-;!#$IZ`*~!tmz%5AaM$jEY^CkRZS-6;vZ|
zgDfpUlw(U#N=7Nc6S|$8kXU51l3pwmT|<w@k~qVKl7-0YnZ{riETrkd^!?J<1PlPx
z2Pv<}dog!A`GsEEOEO8PzjFQVo~3uPZIwP5q3ot+xtP&Qv&cTi#`L+nvpiw6r8xns
zSKQ*I7=BQvHfpHM0`0ISj_uP!W>B?US{&MvMq@F<I|$bV<=?A4tziDVtK^mvXZ6dF
zpg<l|hOB_B1N$6I$+CCnYICcrc?byiM!^I}CC-saeC0<S=|;RHWxz_BbOS(yeqWR2
zY4x>%mK16&hCf;>TWTUj0yFF|37#5Ke2a{y*{sABy+wrCb4vjVPt=-FEw*Am2JH3a
zQI!GKXLO-8A+t&FF9wk>Hvolp6&D8OAo;E+f*$i!iK90UOuasamNzDzT|Sd;Hj}78
zo)P_3=l_&M;i2Z#|LeD@pA{GY048d9oAgmsUM#}QFBF%3EyO?4d@-bk(HW+J@>Z65
zQ>_*V=>@e6o;=`Q|FlT*#DDgU@w5MT-^kfku=zMOFXrt-Wt2mf<`OV?XGY_B9AP^6
zLm)CEI`4neOF^Ld$$55_JMNwy<i-~cYG5hMfK2RGVN6efN7I&%^`nlnv%6P3RyRj`
zp`TCw)JkMnPU*k;0KITQ&ev{{<`{i~0{dnY%4J_7{8tie9*=Dew$r>Dl%sMq-ygI}
zdAQDRMer|<jd#v=t_X!Fpf%oD2lF}*_JwZg!L)$}#^>z)EG&7ZWZ--ML4!T5w54yd
zwYB$+yL%x|bxbi($08GaKFBy4m-t7@z@vv(X_iQ;>k+R2*s!?;{8Q|#vItHB&l!W?
zBPZ%w)9Mpof<cBkg=!A>szeJj|IT=wlu4nT6AyAREAhJ@LZj7JW0&b>Bb4Mco?K^_
z7fB&xsYF?^VX}ba@tRzoi=j71%W--B9uSiBvx{~g$sKeAAZcbw-9UEU+<(MqE37q+
zLCLqowT(uy(h5lR=C@zm<M9}K`*ScZdsv@O^T@Pkc+>a#w=cLY7)nwsIvPo;Y`=Qy
zVDd`*3Y+P8U57h_*YGAEa+_cW@^p2JYz^LsGluXMh?F87$O(K{d(r7~Uk7Fj{x<LV
z1S&@FJ;e;Zmpnlt5q(4D_L~ySL?dxs!mN-+qiZQeJuh%L;fVMI0rY7bAYZh{Gj%u|
zSK$}Wf6Im6SK!G3cFGltMl^viwp*|7FPIt=B#*AC9*w3StZM!;`Vw#9V5IA)Tt8lE
zF!i}GeTrDACF*q^_&Cv6(KskaJm;UFP$DVa{d@6VZLG7Z&2de(O{*OPbA6$SyRdiS
z?yT)={>M#z^g`%fe8~AS*nm8Y`N`y9erC5&>+Q|evFlA_fEYH9z|>=tdM;vLdjK8G
zD6+0^rTXQ{Nl?`Mlgbk54oHw3s1leBn8GJ1g`9%ZynYMHRHH-B>5|A0b1N-}*-kAj
z=6z?%g@}SJJf^EnBU<l&4Fx$bJgXr)f0d3z`Rg~TLMZYVylzqhs;3z|rrpR4Rn^PX
z_C_ldF(AEw;Q`<tsR#lFv-u5u7rgHz5<0G5xCQEC7c^DiVkJ<D-_;KtF2E+1t-CMx
zmaKj_9%uANKH<WXeOPc*?E#@dej@x=jCa_JT_xa`{%g<SQf;`)R8t3T`>5T==VOIb
zBp25H_X^DjO=`^N7-*_~#DVT;EHo&NSu{|+0nz$C9nMT#X1|peL6{WHGs>=nDf9Jt
zYKpsaWo5McLCN{u4+Pk)f6?+=`t*Xxupu8L7Gx0$9_~t60Z03|VbH_7AQavb2sfB%
z+8oX3eWvl)c=}+i5!APRqVjJ=2=@I`5oW&se=0&izST*}|637+1Wa=8td`gyXY5b0
z<*em3|99v5PFD^{NLNd@g^CTsrPcC%V0&=ZU>HR}a}!{r%2Vs`8-;WbKGSUbSOEtk
z5{Qi+E^6}sDhIxpL2}**FKkQj`DvQ^pnsXok6|IOnRXJey$2SfR-0!PeyQR5HGqVH
z-HP}8e7Pj#^p^tPClUlx1m7~MTt)cKM_MW_)Urz$&7JQ!vZB~}v`UT;&CAL6z%AzN
z6@U#q(k^u3E57P*#1VW!8m70H0i>^E?_uDuaVKtqdFP`~v*muEI0c24Q8ZOC@S$>y
ztiAr^v<UA<KZXwE{=5-9kYL_^xZ?L_yl!3IZvszRQkg&SOAhfl+`X(j&~+Or-PNrF
zs0J|lT!wD}prD`Bm|Vlpu*9UTbVZZbJUQ1|XlU@2s)|pi3N;HsUu$V7>y7p9df$DJ
z#0M<wnG3L$ga}y-!+{QVZ?m5)K|JNf!!ASU`t?-Qu-AQ8CIKpmLv@g*IPyz1Y4Fxb
z#2KCYqWp!nb*HG<8W1)Xf0d9GG$S%oo1ER)Xcs5evFhw2d7xXiY?j;HgBhH5eXKb3
zZNc?|dsD$pMjl-5BWywmRDUNFWPaD(9b4=5*1bNeZCo^%L50MMU^pox<d-^6T?Ex=
zUM=3(i1DOdY8U%tqTn&?O#0}TWMR=tas()1>zoB;lAk`NI=RhNAVfxj>6Cl@EZAg=
zY8jpqEkKj+3LV?xm~#NW)~v5>eny8cEZ{`sinU9m1b18)M|=pGPk64Ly;LJnK`4!0
z2X;>Ct6naVq6&4F_tE9%)Mo-OlYG613}6Ox36EnielP_<HhQZ}8#*qYzn>K7*P|YO
zUky{K5R)aNCA36EH%u9+nf9%+YAz9fO*M;iLTcL<3@+yHOEhf;>U|~M8ZK6qZYt-h
zyrOoTBdM@j6Z~nR=vJ=5?@a>i;T4rsbw|n5Ze14b$1-3HK;mA}%ZAWg^lY)1Xkm$`
zTVsB4F2wi7KXq4H`|Xu$Ka;*bf5<lgo9WBcV{=dI-XRhm`$dsyB2c?i38L7t0YtKp
zi-6#phS-(Byt^CX(5UdC2tiL*s2P<vN7~i-VY<vwKU<Xih@Z9VdcO&?MovMZ2=5tk
z+!IZm1vpD%wz-ZIC%^&5?M*QL-Mg#&vUc8AjqI7P{r<_R*i^g}4tvdP!a=^;M$OkU
z-V?-z*Fdk<Q6!Oma$=W1hR+kzB~f4?sohVm-;iFvod?-?8}5a2{+VySOAWp6dvdB|
z1R}h2m&oa_uBZfNHDKDK$DrVvdVLQ}A|)drW3`Wpe@O~FW^o@xcD;1DoiM-ZIaA4T
zuo-gnkml&DQx)hZZqWD`EZcG_J!yt>Z`NJD%@h((FV`f=*Grt;I&n+^>961Ytx1O*
zFmF?8eeZ6Afcc;l>$bLjRN^D5C4jCoq;t)No#RKpuhhgw`w}1cU?uSS3u+4YVvS9K
z_q&kIc#(QDbsa*h&Q5L>G*P+_V!SrIqzWp!p0KAmgPm&Fn|L8WaQVlB&_Duqs?I0V
zC<YHCgjDALNU1nugN!)3AJ(^$IKh?<{=PUVPF^CoEoOH9o$SLPYU@FW1zIVH#arvn
z;y&xuT{)Cc_8-rA%v{_&)@~5uiE@Y*S!HtvJ7f5quD#Ztkbb9&yIa`HP>&&^djjl?
z51UZ`zp)tFC4DK>+M{@q4;ikKk7dai#)>Lt!`$X)Ry?nXSe&PR3H{zK2XMk@h9_I)
z?W`#kD{qmx=fmO5i2I(8+=_e6z!1yS)TmMiM&fT%2QJD;2>D|iiC$NQ4R#?iyQL%K
zJShkUuS#__7_y5wwa;041+*$T=a^FOC6KSv4@VXU_AK4p+_;lpyN#-IRdLt(XdVEC
zkbC6)BXreg9u2zPNo8&0wceH;(8}uZCy*CW#Y?Z5kh6`5H9y8aGK~<gU}mpZ6SES<
ze|ev~_rnX&F&c`{_@FLIB~J_*tGV9x)r|RH**`Hh#h8UPQ#;j`6D(iNnk%a7*t8r@
zk=ab~g7n*#o&0zsx!!q;@!J20e@G0@xkE|x0FJSvipv(N9Epn`%q^Ua<#pdd)`};I
zewH>O{Q(R+MDtN3e<I#$ZBKFDRXJ)N$g2n<{9Nohdtc!KisW!?!yuMWhHt4+o1wvs
z+966+yZ86Y4-qB8wTIY*zHgLEud(N=K^0b8Xmkz0DqgBva7zxgG>+G$s1nS?T3HP0
zaD0leo&{XgF<|JL8SGPkUegR%k5gb-DRULV(?~4;5{RV2*+MH5B%H%hn5a44C!k1*
z8+n~+6Fios_;oLtj6Dwazlx=4;^xov0lD9nu*!Cmtw>`!Q4j9ZWZ*gD7??tvZFEKU
z8ttD5FCJV4(8I#rzPDn_z^64v%=XpByie?%!HXHi?vbKGJHPQg9$?H-z+jI3Cf;$O
ze;pY7Sm;H}w^c1JY-{Z8E=UQ8J@YmmNLRvNfiWjd5$}iRxn-+kf$Nj-x9&9$_PAjq
z<1l_t*P1yPyip(bKJywjOP)zelE<~17OQ`VT6c>t*%h7w`Ks_!q1hL_hfTR>S+a$m
z8}^g0OtA+0z2fv$_-5;s<OSNrgfvc&68P~QjyqZn^hH)^Ft1P+Y7)2aqKFJAySuX3
z(<nGUAEF69WrueuE5eS~JdK5^dA}THKLs|!h4vjtiF(;Ej-h(l)T@BO%$LDy3*B0s
z(vbrQWva|zUh3Lq{ak@_;pN$aPUq0-JNO2KoX*f2pPp;RH8oiYUneILFKQK<7s)l@
z%RCJu2oz)DM|Vg*9a(hH_r@WI+xcmp%$DXQ%Z)$lz%3|Hww;9{SXU@<j5UrMoc~Ag
z6Ul_RcL7T3ea@jh=-AjcdEvu3ql<)6Dnk-%%l<8%=R^+tIi7zG>O&n^!H(u^bTHJ&
zzbprXwOUjm)sn&KpG^x=R46U!!Qea4KY=wwhnyu7<a)`)<>^pp#Nx|r*Mk4?=N7aa
z-cHqKKJJV~0jQJnv8v6)*{S+6TDg`uo~x-`Q|r~u5i3EM7h8^zo!su8I`z!|ONb0Y
z7I+d*tFH62FJaB3?&NDhl=x-BT~}I1jQ~kJ-7rWd-Tt_1&k_Fp`CP44nef`;gHhzx
z^CL}VDVjdjeLSwUqP5`H;R|na7#ddkhECE723?QWYQu!nU%24h+tQCjMUQ{c0aWTu
zJ06YC6Vu^uWeK<ZNu_hICeyLcx*l<gRP@Sirr)z)ZICfwG(-dU;)mWyGP3tplHgZ0
z{IO%_^ApVi5K?s&J-g9p_yq{X@07ujLD^qynRDW)G?4Zs;kP5FSq)Cr+jR${Z<1f+
zSW=2DE^L|$FGgm3#mOFZ$)5v|e6_7qdxeB)1e>n?t-_gV%USXu?Sy7z-~D^jXLl0P
zq|%r>UEN{xSuA7vbUlMVSqf&_MmCWd$egU6yr(<a99^^K+SeL}?!F>lt`!!!hn_+X
z#Z*d;baBT|?gjBUkH3{u;Ur<3^pkp^;S@{gbchOgR=>mdSBuBUsZRGv>KfgDC8%(^
zzr19()EYroQ(gHEGEOM1UIc;xAU@yH2<~+gQ{Zs1<n=#Z2y_nE5m5Afdc1y+AsGPj
z&Q0$6%^QQqW~j{7nuBn7zbR;SCs_%aMi?!69g2Da^py3%sB@vTLUb4pb;HuV7@ru`
zu*W!PbRxHD!Jy*v&3>Jcb~i6~Y+EDX3Wpi`Tw0uz&rw91S`H`W9^kRuHJn^0M3+ZC
z<PIebj5lNFB0K#`htn#(QD}(Ok6gZkYYl#cpdsm_P`V}a@|?A9ac5^C+2~DmG1B4E
z{CMT*yWG0Fl&kzj#06g>gSRns6^QEV)qzcSP$C1M0-y_x%-IcgNlr_H1R!DQs9*3n
zYU0^Gq>Ixw0^Nalmg(1Z0b5zy`=iLdI#~E6*!ycLJlm;9hi15`D&0vdQIi>ch>oOd
zQarD}tI2Fj`LF_Yy>AdLbou6N#z+;c&ef%j=~`ObJ$KnZ_(uBLi!2PD^JZlNUTutO
zPQg~JPFA#dO+zBcP6;itq+F)3z5m)5@y?)Z@k;ZTs`MbRR=oE{a01aNUC(4y1`VRf
zAKXoPbjxv=-7KlfCD$xJ(O^Fi_<2%Kp|i?RjD)ggS{@TKOf|<2j31)1XiV7gBfmX6
zaMz5_0blsf+ZP_@90)V^DU07KZu3w<_rtq&BsW8}8ydaqfabhjMsWfK7A5q3e842h
zJtRAs>WKn!7t4QExzf7=N#DrmuqIWF<RAh^PT_8igs;Ptu-deeMs8Id^UY00JJ_uE
z@Pf9x;<<mwGre|F__r3oz(NLd<v(j*@@~8x`{~VUx;2(L_#P+D)cV?Z6@F*o`YQU(
zBXvNRUHp}PUeU4tR0w#8p5@Ij`hX0Xo%x2UqVB#VA*NkwV%3@j;T_cq;*g{!?<~C+
zTcB*(f-DF5IHH?ArdTFn;pF8Ns(WJzuqR!b1o=bTV3vVCvU)+LjwFH2DIHHKq*RLU
z{peQwJjL0%H=Lw_3+QfB1$!q_etR=;)IWx{hh2d6Z~+h{S~frycZa1ft2LY7I$Cg`
zt@(1#!ku&ex=IH)#5)-hE3iALORFvcJn1O_P~VxzMRQ&5XPCBy_PgHGkbdGrcGY|m
z9=mdTn;Aq&E3Hr-3tUbtaBgpxgJ4R?P^f|4Q0~pXvau^{ws~|{v^Xhfz{9o`TiJ+i
zuDzLcOPkdL*%poNn=jF<@yW=20Rz2Owq|^4SBCPm3G5+iWwzN87%wFX&PVOZ4YJLU
zBE8M%I2tLtaDq;~zR7<oK7A<|vx>n7mA&)-#F*~=33~x%94-ro*mf-y%LzMJPjEGx
zz4b!8Gojphb0nop((CO3P7IuM^cU6_xC8^48nl%#v<2-MHxH^f96Ia;Wt4zATYkBc
zBRnDh(3;yHU-kJ3dSK3-0f&5urdxT)qV#!_EY4dAT$$UIZfa=wMrF8LfSxn>trocx
z_lHxn(lCG2#&nbp2NJ^KbBe6r>oUK@f-;4&h8`(NL)*`XfGGi&_Z7kn&bXw+mb|K}
z?lAX(<r;f`I9TJ-dsDsdE;s9QD|(q*)Oe3oAFLM}f4A|NJ=hG)liHrcdpawM;YbZB
zW`FOvJsoK}<Z4B{D?)B5u3cHWZ=3}-ib;{k9aF~VeFCN`6)C~Ic5Q8u=9KEAc%|<o
z<$<4y;0T$Xeg1wNnS)Z$)V%b_cXNbdhXfHu4K3@d$i*W(+h3hlH@o}=Wm$_`C*Kj}
zh|uLqU@9=XNf|Y;k=v;oSCbp9aSYv$XQHBc-rheoHN~NgU7>)NG-tZK1=vnZ*N7>U
zWO1;@&SEL6=UTt8nzy^kTZ26<o!yxCZE}<O4XJ$<O8$_B=l$=S+xwtZ3wVM>VI6i*
zrBtrDfu-?BEoU-Al%QgI8w&`gY@-pe$<RIYiAtjD`QPZ#|2-Ax^3gWGE`viZ8v36@
z{?*?l*$^CZe+hq@<+Ek29Axzw-`sIt_s<y{`%WS)kBMJTQ2hJ;Z^!YoIEk7JDl_NG
z+#n{K5oEydnAqhMGMBl-BRXY+oQoNRMmTfI1RMIK#>jNpf6EB>HGW0?w_ac?!KZvy
zieV}~_8JO#%%D+At@a&90zMgJ@i1jya#;+Hfw_Y<@SaqsWowg*dh+Bgz^OknPsX=A
zHMg)B2VBi9Q2Nwoo@YuSSeRpF>wQL%8U_joat!P2(x<&7VFWCB$PU4uVN#5{!*?qI
z1N0}3c<;2*kz=?7ppq8?<1M*pNkSvL1UZa4PUlOKBHc!FiCrv9i^<4r+0S2M!-%qv
zUyr4K(Mc(C)(X%dN3j~eim3jOom3tgH=#s#j5eOwawoHv=Uwvq2&ucTO4VMDkpRub
zuKEC*>;oqYbzFK~<6X1Rr1<l`SLYRLflNgCe6J<6FD^yAFR<UdZ2H1isxODe|0WYJ
zrH+`t5%MJ4Fx6drb8dJYPJSBqmDU$J?o$c}B2g}`pwIBxzxD6tf*?nmGCVbt3DMfk
zQZhpQuGzy#{BJTe9`(=`b4L~!^*k?p*`$?G71Z+b(9pMKS_waYqEMm-dx&qw!7$o}
z`I!5;&`GpdZf#SP<+x8Nkw&(;BeOOS;~~mO^Y3VNdj&DW*in5156!{P+A+XhR3>{P
zm9u}tW>{lNLSaGs$l>GXwMN^F;g^)kdsT_S^<gKlyKg^{koz5!RE_+Oh4ohExz&`}
z2+8MD)JMgH2_K~pcB2osPYOn-O*c4=^S6CmXhmU88@&Tki4=lmgY4<TyJ-JhJf%b5
z>k46)xp2kQ8cm=99+pG^p>hF<MkwSEM_6hYilJQKam%DaAf=MEC`)twbGk{~9KlKI
z-{BiJe3lf@D-;;B{8EOTm@L%Gzb0@=(~B$q=H3MIj`ec@^b`06x}6;SkJqI5CPdf#
zd5W}nt&X@PGe6rs+?pSnRY6!U_q2@XM&#M-i{gsK@fMpT5qgp1nWQu8)mo%Sb|ghF
z>Fn3|2pJI<qa|&>Y*GZNy@^}<dvTWj=k<laxzT_2*CG#MW-0u*lqH_e6?5ag#&!-c
zW6>&<buJJPy+4<g;oGh86<%;bl@pO%M_{lruGpwP^vxfZi`0;bl&w8(hV1AS+S=u4
z%$)MBBmw`?8!FzEt0{|#Aekh{eVhbp;mCUY<4w19ZpQj~eC>O(-P$oarI~^$_8L3s
z8EEO?^^+9;d#Bs8w!06Q95nDbNgh_|r90E()`kXEAs^X{@{d~=^H2<Qvbo|~Pi>f|
z>)M#%7$r(b9OZnkE|n=BqqGQiyjxJF(=C<5^}If{bTO$>v{}lS$x-{sZpqCh%{myD
zN7YovDbMVUyP5v7!7t6Mgc@&-KR->I<(SfJz?foFnYT41d!**8+LzYt*;!R`^&o5F
zy7M^R<msp=;o6*oxNpu{sF9^z9^8>unpZQ<(+wkWkgP(37l{|yoN)19a~;Ep%wNQ7
z7Hal5H3t!1T4Tk&OY*)(gQ!Oh5?Cwt5!x8tt~6uaH6$%Al(TN?k?-xB(HeU&Xuw!~
zV3D)+1So*D42y}s8fzHI3}ftHib$(*kUV3x=obin7m^t!N`P5g>a~Z=s}H4<QejY`
z@cj(wuhhPIdE@;$MECH{`*o$(-}qbbh#&y18qvtNz5iP`&3N12*eEc9dkspEgbzBT
z*-}AeE=Qw@J5)XFci~j^8+D;VsYP@*yaOrupuf!)Dg`Xk@6-%}u*&VxrN~MDAJ413
zI~*;-*R<u1&BAMKZ7&s{@SF~FP374x5^{1yU06@G1Z`cbqT3I-xeh#dY#!#n%{0q7
zFGHoDYpa<-7CqS;{?1Vz20s!wwj+nI%)vI+Ju?hyF)wO7sJ?MYFNugH%nm1SGX@eX
z@#>fk8;J_5k2A*=-mo01D-LKdIIcegK^9HsPx$Ua)#&l0Qyw|yKW8(I$T}&wT6$T>
z`(DpEf)-j}S>UwG{ahN)b^};CP-2}lG$0(3768HqEqX=`eQ9`N&g?TVt(A93!SLKn
z1kYCXDug#B0_CZ!Z`!?AtU`%LzJX?Ah7d4v->VzPv2PS`z4*)m(my7rNZ?qr@)Pt<
zN<Qd0+YvB89gM;_Og5Rg{cU_MV{Pz)d5yq#j1MgeV*jR6!ubN;bT~x9R3%@h=;b3n
zo+vJuSLAC5m;MU;6f9{uza@QUyVkSw2u+9AF<I*sd{*)EJslLhacr?+Fw{kzy5+1=
z##1+jIOP=t`z#)K!+(A|@|5t+6Mk=VV-+aZbMyTrs9_^iRemD#J}y+LYyK*O@P+ik
z_T+_h!9dQdLy!cCow)v|CS69yQvPq0Rv`mQ()U%e;)&FauEvr7lq6m#?tI=C7f?B(
z9gy*-mbEGraf-CdkhO~q0wJY3frIHU10h8TQWk@`YK2@+w6Z=Qv40g%uc*eqllwd<
z6qhCAaa7G@uNe}`!H<-aN-J}Lb=OB0A!8q5bgcW&k3jtc=lv1dUB`>fZj>&(UnqQ?
ze!4|_Yd7r4Een=77M%BVy9rT|6pw`lbE1d3#DrYNF3v{{srKjI`Tu?`E;+}anlWta
z;fv&b-3vjr373uHfG&zw;v3A?Nv8=59QDwhZfMHkLp|7(QU88q$P(=e4fr9IMbv#M
z?=c7d2_FX^Cbb{-qY6udGB%Np&YaH$d-osJk4mkWX;G#Uo+l=L7Th2ax>|E5SNY(Y
zB+?`DMi$bYG+k#PXL0BmtU<=ZsV>+N`vW9{jMwyGIdv!P6g{nHhkcy0eQM-H3r(4n
z+*Z==czK^}>m1C>o&=KGsN{-aQ1b5hbxc*fw6jW2wIgr9^dnrRJ1HryaQ(5a`YQl4
zt&lB|T$&<Jad*PAh45WRH8WIf&`0V`M$kbMLJg=y&xr7Bbx3e*#g>D4H;cd{8qxRG
zt}KLiAHQOQXQ~L2<s)6<@q?<uQZb)!YHb}=R`$*-nhrWuC0nvRgkyVik$va)EkUo!
zElg27V$^lDfUii^+Z;;{-lh|t0ht!6ya7Vwe5dX~&)C}sBETLZC&jM!xTg(oe$?4h
zX%}jIh@`Tz2!5EM{OQ`FbCBQW!W}6x#^%IJHl}tX^Snw`yp<>z$adyuMCSiGkf%VN
zxpB{coY{+#Z!WY6-j9hKv8|~8j&(2n09xkKukzegU^vM2P0V-ZEs+Xl-S|Vd;u*O4
zy!b!QqA_y+@@Hcrtwy@P-;@KnyNmeUhRmze{%^SiE4=LOcSp8_7zbX#zxP8~NuIF3
z`3xXd;NOTZ`?r@{H5qymexa27Zxw=Jgb(FVSgVd5Oi)Owd84$}cJ0?d*8CK1BP3nA
z(r23a>fl86)nsE$^+7s%k;mYZso&L;TgNPA%k(lg()iJ`8rC2TipP44S}0cwrRZgI
zf~0W=3EOehEs0CYH`ZQA@QbTmi^cFysmjPdT<A6TXI7dS^DB_(N-!k_hD}TPlTQe(
zRApHc)>q54;@VEOIA^^BMb*l4;@K1kCNv7q_uVI2RKrj6PsRZdRkiNymYtXVo5ynr
zEx*^JBC`6o9=kO5oWH*WI?K9SceDmD8w_O06g+R3xGEQF<2_+|vvw1GWud6yF|vCY
zX)}{`E4w(MY;2=Z`x&NCQKdTZ<lxul8@)XIxHX<okuC&t=O<w#EXe(;5tq%lc0_|q
z-WN=nqtPJILz(lpNC)G4M?yT$FC042PV;Gq^Nh$NE1zK348mQT`+;Tl70A3Ty4||}
zJP^AL5Qw4i8w$dIUnTH<u#$ggd9W~&%mnbLkNg<&NpNf+ySPorykxr0fut%0F+U4n
z>j#hYbHQuvVn4H5`QK{Clmc%31_EWtliqsgOt|_8sf;>6)3;#vJkG9Erhb@i`=4_h
z-SNbaEzjj4`2FDzGO>f>IfYsj)JZv8(dClN%(6uK?IM4l=q;hY;8PV@+klZB_#^X@
zc=Y#jf5f!`?*j4p-$eu^xZS{gK?W#Xp62Fpb-=?iAnCkvQaHeCySZXX9sjETQktv*
zND8&YU8-;@HhRhIr^GGD7?G7zh2q#6w`i+)-(8(%f@Hzuwm(@je?AU**?((yuPyaE
z2pfD-4KFfpzFR0ore)U>phbKOA<vXW<{fYiXxRG-?nd#`3S9r*X@GGx1RNDMztAf4
zhfZ?pKV|zrog_FY8ALjbe;Wg}Eu7$d&PJCX_@t*VUhPb<u&UNmjvkM5pHt<PiSsm5
zY!-41JpJcTN&M3%9`S7STl{6C{pl0I`hF55T^Cq>eF0r&tzAHD*#KP5(@r$@F_8PM
zf`mz4t`fh5^tGF``snsZxn!HJJL#qsb7iOL{_eU^voBIso&w?x#toC;Z|bs${Z}yJ
zFCq2sNJuJxHC4$5KB-PA*XNnyL?MNle8KZL6{?mmh0v5hwb-dT{qG}QX9rmNG4y|n
zI{1%YFqrZR(blgrRDj2WpX)!5#|OPN!Y82Xp=W~S5_r7zA@1K+8s8Is^-VZrC9#VY
zPHCbBPEjL<)+69NSwZd<58mrYiVrAX{+2e0>c6Ahp1A}(R?1KRdjm5)MFx2aF{KR~
zq>byJlnR{V*|dtx2i|Y-&eguj2L9bikdq8Z%j(r&%v!(m0zl*nRg(FNRK(IBq?~eX
zH^)jZYt%{q*0O)<C<UzHwnX#9!1o&UNEFx)Bz_zDpy_*JGu=Uz%Z-@&{9c#uAyY|r
zuR9{2j@Dm755C#+Jm{-D$PfA3=07CE^%GOzymXBH{z|Nbj3nf%{|H{Olk25jom7cq
z*ykKPcwBrKk`&z2PQ|*v#}ic)_DD$9U7(m4gHr$F;FmrvTF=Yt(KVi5z38*8+tQ~W
z<RO0^z)$2bj6XH#2aDcPIDfi1Ry_Y(M?WF^IsF_|k7L$mVr7j0ur&!FXi&^rRMP!l
zp#@*xFaGzzA81QcQt_8~jbr^I4Tblak~$g#v|ADSl?R#E$d^7aNzWvu;HT%=fvT2w
zj?w>qFi9E!6O^P$uk(E9SJwakKKPrQjaN@(th>D|3M5aOS)<9nyZe7!^bFn@9b!J6
zQ4;#|$xBF5|I?-tjG93tm;WEa-ZCnywF?`S2FZsm;UT2EyBj3L00g8{LK+0=k`|S2
zX^@oe?v{}5lI}h?Zr$%Y&UwG{{c`LddjM<QYu<BS^9lyXe{ZJ0KmMQfBgn3%`~zSb
zq%+kk(7<EGA^OFy?HpF<uJ}!9(TuqtThMq2$|G|t^L1Ab-6NQoI=+ejK`_0|u;`Qw
zL5x~vuK3O6F~FmT9k|mD@2@>ss*>kJt_IJ)vCQv)KyK@(j}8Fmpj3@T%!aRrA7TDm
z1Df!?UC1*+V&u}q4HeYSP)FWO@0~87U}zj|EwpkOzsZ_r`l$R*4Qz_=ingQ2Hhx|g
z!Ee|6-!J1oAJ9Ftx(^Fe<N(#(G^*Q({zQR{3lE#^q>%aE*H9@(GwKOBN1Dl{5ycp8
z8@KXT0kCGF#ikh{`1|WX$*}0q48gmbt#QiS`jI!h*cxa)@`?W7fsS`Y^Zg;j|DT7@
zyM>MKG2QWh{aq0677m%$PWC7@_Ac@Y%mKO5#~kbVhFnYGk`8>W1RfM0?=5x!VMFpy
zD_0!KFYn}<S^NKbNq^oIkO8v)`Jh~8gz1&Bg7{$z1xftkO{@$F(oFj80N8Bjm!-mw
ze{?H{(CzI7?wu7eMK}6?Dj}f$lEWT=OBRKj7y=wk<iI=Pe(_(e2{0I_l#>_uy>DJ`
zrWdD(`CDH=oX_vK+<TgPd@g95WAK^s+a*53aL13S-``X*gWxl4ewKfJj-L3RzlJT0
z|I2Rgx4%8rpVsQX{((^ZIeSuJF>O9#FqF1hG0(O{zv6Q9GduakH^u<R0zQd;XL8iE
ztOu-)5-)Z@pGDCerThm$K!!yK5)%p^2EIh-QCOB})QM}Y#QFdGLuy+FSBR?O-x6_O
zX6fD-I%MWao1IS^BU2~Tbwx>v5c4l!^<PThD<Aq~tlW#FSN?X{#bNy2ZyhsZ{*SMQ
z`|qon6{T0f+9&qrnf;*F*!(zGe!H&lhVT3Ko+!ZXlyWX^aWo0+TI2_$Bko_&#w#fQ
zGLbjXC#VB~PYcgmum>&&=x^~RwKo)t9@gC4e5}UNQbFFEo)-izg=|c+6X8&`LpCjc
z<Vk~dC3VW6ED|Q!SK!Plr5?J{EepS79_mCnW+n%`3ZRQ%aMqAE$WJ|L%hT3rm<CA*
zGn9`v;l1Idp~>Jg^w;=%r$UjX{KfQ1Gt@&Mv!XRzH&;KG)J~@2H<Q$4&93cbH~ip>
zN?Itde*gLKe~Qa<!p_78(3Qad^BZRQ0|0bLM?VgRszQ;=g<GLxTq3|qD*m}*tm=^h
zP5-%4XdvneC9PVfZv7Pwq*UQEgBl<t=YpNrvm#yLd#^96`G%zN0!>NLn`0bfW8=pL
zmHi@_Sy>FFO28GU&oIpi<aEk{MHafFRyZn1d0?|$5G!rS7Q>hVCgfwQk1b`eqMdf;
zo_q#j&sZQoHNnkcFS)=LH~#&ry_*FYOkys&b5*7HwG?9;AN>C>kB#j9vHjbMfR75r
zyG2MQ#o3rxc>j$#gKo=y9yT);s)_0F&?}COl==p<kVyC6kByq^Pg~~i+9$t(%9Qjk
zZuRHh`u86d$UnGA$LN5(zK<6Ya{0D3mId*5&{C}xiK0pLW2T^{b@(dMNLPt9<vTZq
z<UM@y@qTCn4*fZ@5uE%Ah>#Zs>SHO1EWhR(Q$)@e;O^sF%|ISBKP|Rdk;}o-r?@!t
z>ANQD%beGZw-N;?@Y3&H{T_6|MPPa+l1pTWQH)C=f$$q(*2KnfboJ&macWE_h<n&O
zE~5IOU1+$UY6b<wAk5nj^eBdS|GOUgkS1c|KHdEC_joLT7QiFadMKSPjPP|pUdmH0
z6_;O=?mmbo-B70Tpg(s$JmnvNWwJ9-?Ac1p-~ao+>c*dM@bu4&W;n0F&w#ul3Y&cf
z%(>BDb%d*y*_DXrT)fT0vN*hZNb4MSf3jfO!RIp~XwxWJU$ocnN9>G&?YuW94V+>g
z_b2jmLZHYSa6#OFAA!oZc65=0Y^d&6qM)d_Y>&9Sai7Zt(y|f0ekS+vB0%=4<8eQ?
zEu*6tk5!c__jT6d!LfXLJ^1TQxCUqwgP=^2Z3yf#vXVK&3&<rMVlCj(DEV*L|3AAu
zF>y3yl!uuP|4Oy@alNGoGkTxPr8OJ&s$Ox1tfC|o7>c;rHBnsNs%yUK#)<r>32qH+
zj6bc56rnx#?vlb^l<VI!^!GYzvj5Yi9Mt(fO@(4nz-lyZ@_wb}HX{Q_Z_Vea<AEG1
zt|L^<<>388IlBm~{jJ$OHT#F<u@EYmAt2(5&qo)~K(|*2MdrCJRCWQ5NXDrRX2X;o
zbc>aMok0P*!2S)|<+rayDaNU<rCC!kEbcWyz@cr{Uln3m3RcidXqRTEzxX8zf-!pB
z`#+}`I_M&E6XX70E}^c$`%CCwycm$N#i42)8<f&Pmzd1bFHafMa7$caEPtM794}=A
z(kYJX9>V@pwc#cHxq=wfIuhY)%17}2K8$}89I)UIL&boPANXKo;ESg3#bv;eX{B0;
zUJQ%w*Pnx>K6&uzlt39C>uY+eX8T4(b7s!_vUK+Cwx4)Y$Ct&e$J?O=u|FsfP+_fq
zKeeoHF3R?&Q>LTN^Bx3Pxh8va!pGM~tRPKg)Ew-y;4_wKp2QE`=L@eV+J|3;b5Ty8
zZ=AUv<w{<i9?VFqOd}ff1Fq$b%f<ai$>sD*LuES>*z!Tyz(DSHebD)Y4HR_=$Z$C2
zxNRFTGj@p#6=Z(;nh=nX$INj5Z<R~*zH%x5r{K4NqehZZoBTQyu|X?9A?7)YM0$ty
zh0pTwq$ej3`jBSIjrq&V{C;*1`Tx|eV4gJ>M@@2&TVgSk>dk-=VAcU^pttGNS=h<J
z$+-!nAZUGUjesnu0v4j&5vIn*xeilPQ)?x*g8M8QnF@GiIzaQr?EyGM6;oKdx4`ZH
z<$A$R*^^(QIipDn{z0JI8U?I&tLWw1+!uxMuFC72nD9G(f)Oh~9g}{aEW7$C<sd-p
zV5uEtP#zcq-t#^z7Cpd~aST${N4JX_l_w)!fJ_;!OYq<3V0>`^KWtQ8Mfh_*UG2N>
z-B@m_0h_5({{W0f7Q#$dZ-bid-DpjdU9-%aytjEklzNWVLzM!Mf~0P|NJhwSM_u^3
zQRJm&jJ6$$j%_o7Ns=7aR&uFexFHAx%>s;hoVjhvVJchT7&ijwxk}d{uao4&I{2$p
zU4q-$u9Zm~-`jMzqF5v*K{n!h)ATXO_V@3*uRbR1l#&SoWjTjWQ$54)HC+>MPEz5W
zcQ3n?5m^1Cj!f2I+7~Mh(x`bBYlXGUCyOb-m9Io}?$3~6I+#3GY4aK=3O-Z*z-8{y
zwfWSmOWyoo1w4Rz)+^oK^!Yde1pXuXpH}!Iy62`%Gi53=w<&-pXvW4Yi9rE~U!CGI
z+0ehdk&Pq3`-K<t6ZA3lSB|d!jv+Q;8?S;giWZJ+!TxS6Q<30jqi3U4Ols@<o$nK{
z1Kw|Z_kJ2hdI2i(B8UJ2F(pJS?l8y0&~dc&czxN$46W}LjE|gPxH45=nqyfOcbF`g
zVOhH2ZizgWTsGvFc_*JUJRuRpjff2f|FGx(#b+Y6fmG%`P6CymniRGK+#(oEn~{I-
zfyfAC>Lo&*+V{U<kRMiwl?VwQwJJlA2^?kzGs_n)(q9>VAK4|$zZH_yzOG8E@tPw>
z!0DIKt52Rg!*(p^K?j0VRK7i5m@VZ^fB8JHpE8a?APs~e6+*0ux{v(^jt|W89_4SQ
z>@0@U{Z};xSnzK;>Ud;|XbGniN2^am$JdWHfp<g6V;<--IMVHi@6cT_Eo7MaUHgUW
zGb#mg`%fc8Z&hW2c(PYi6QPmG$*&Wv<^FmBd<$g{d&hE~erdKopfi^Ll2xnROs$O>
zk!S~GIi}kW?#KgfVLhc(fo53;6KxgGC~*^wf82|`eW%UR%rVf`aDM;QMhoja^A@T8
zu!qrPSxl%vw_X4cX?lg$Opx~UB6UVaM*3c@ir!sd-lw%!m$q`Qh@E1^r86tpWwM17
zCl2$*h^%DrSpAf7+}p-BQp9~zR|<rwk%roIbxLs1yT?9fvquAhgFO6l+V=@pcRfh2
zM-!Q<2jR(J_}77DbbmMW9k`1lKOxucx1c%zSJ!nkV(+4d($I*2l@R_W@Vjr^Tf^rU
zH3$$?d}e(4VlQ=0Dd**Dz|pEl@A0;?t~-#?0xpR|L>KhP)I5JWXlkCHBegE#5B%c8
z*9jpOTio5#m!aw@x4LeLRAZ!RuAKg=%YM;`y`J}I!MbN}ec{{o4|oj3=nCz$28`no
z%sMq=B_Dxr+u&Yt_j!)rE<A?xM{!t39BgnzzqyK^901Q_^?`ne8}KnbgUFuPOw~w&
z<xT07tDk4?!y{9^IY_V#yaBsRr;loe8t(sUb6mm}fKC+L!yj?IHPDph!4>}MpAi)d
zpP;Tj%252GoEG?+;fbkkTY!~o<gaBBx5cPdu9MjeS32p^-}AU?6~y)_i*$N{!WcVk
z(aSSn$d?8rI`InUO^2cWQp+~s{fTiMo-OEL{o*aHKv+=?sUJ!|GN_$t-q3(pcCeMm
zm=-_7cImoPBtN!kg;)~vSbUve2>s~z{-pP?wKP8g%cbl5yHPi`L^*8Kx%2{f5$U?W
zf^`^UZY-b6_}&oUsgYnkUYRU0{3e)=1~NERVzn1QimfGxm*y>Pa6TR{pkDA=4fG^H
zgp(*>AO0}?hzV*lQx)|A*E_~T^7d-}ZUk^q@;z&Q>b`Bq<GOd%n5I*?VH4_r)hKS_
zGa70V?|3(Fw+)}gsk|}jW|_4JXZ5oJG>#~84uakti~$6*pw`Pin*`Q5%0&&Ac~OBy
zy~leJIF`j-;dGj7d~w??TX|tPm*sO?WkB#R*YyHRP8VJ*`EDgTg6P5BgxY?H&)~vP
zCh21<K>7XVtpGSGU*omDTaK>CN83Rxx5f(<fo+$1)UBPa$!!m;$QbOv2#BJXiW8H)
z1PRVMmD1P`Vf`QdsgVx_lSgjPWx&xJ5DpnG^i@!EV?yEfB+F)3%83(IMCLy5kfpZS
z2Ir2z1itqZy(Ca0VQy>gzm1icuC&VD;?4`)83A8L;j#T?*(tC!V@sd&>qhawNY9Z(
z3JeN~9i_VDru<CRa{^ozl!Is>g;2NC&FmCqXbP3Y3gB2gb1;bPSk$0;+R5a!<7I8I
zRc^l3OLnujC*Bqh;^K3V`_YNHRNY3?r9X^ZqHp8TfKkzSU9<OB{Kst#t7HOgx6RDz
z@a?YJp<Eb4<SxV@jQ7LR1i1q2&V0#^i99w_N}ffR32v|F6An2)KK}M9TRA|o3`CZU
z+$RYA3^r>qqb>Hff26HG!)jZ14E_=bB|=~{s1d4w%nbL@T*=n<`ut#vze!}v*O&H<
znkmRxFasHJdCF3?$^7;-ANe2dwt`6{*DTFe3{dOn@yrniUSUm{1IbwU#fg0VT!!aF
z-gCvz>Y&`Mx+A*T&8*`wJX1Y;xEl=$o{oOTkyL5>&1ki~Db2+9^e=^4PXU?f*-9S2
z{Tf{-C!OF=vl#%(J$3UCN;nT(e-Ga6)<s*QBalr*as5yVv8#35mdQ?Xinr^@N}`7`
zmCEhVN`#cq&@$vhO0sG-14c0tIg;LX5N&BuURr{tY!hCV_Vwj@%9fUd&Bkdx$hSdi
z5e+)I9lZ)N3OSY7w;nsA*l($IyE?UUC%2~ITWJz7P7jGObO7h#Qs94f_wLO4zre){
za2@HU-e!7v(8B(^qy#POaOZ{y<|pcFLl6CAqYl%~jm>u$YHRv&<0~j~zfo&{?7tTm
zxqW(lmA$6^Y;aiLn|6U;VzplsBcc90M=dH=y8FoZcqNFauNjKenm`kh3pxRdW;#BP
zeVbLC;<q@<#f|T2ZEiKVdX1|(Ltw`*eYe@%^hiB`+3Y4baTs8!Mgl4ZLekOtT3E#N
zx{ZMzao{DD<~}mhv-hh=uo}227t2Z+3=LIPNQiGgMP$fEV2Ab$@@0^uEaSM}*i@h3
zy8(v)9d3}HXPpN4RCx$&GvdNGZSlv)$6_j5e7WtnYfEJ^H%@0MSe9Ha2aW#X-nwS8
zmJnmaVH3(Iz!6LN-5tGUkc3#C>vk}kyt7QhB{)lwDqvp>ck4to2qP-#n19F%v5e$m
zodB@=UN*>7XW4qMc;A!KT_ZQK6~1ppv>=74b^-L)5BsoiN#Gc_sSrL<lTrOSM5A2<
zf<z=imKEh2if9@nY)d^Mctc95`4&Yb&%1?bv?!>4-}6!<d7er0tWaWKS^h+u>54+;
zO)l-}l{%;@;h!lyi_L(}D%=wuH*HNQT(#9Nz9#Ib3VC^G_PHV$#eJqEa^8B$E4Rbk
zH%`l`jB30vQvN{j9e+_-oF*tYz4%q5#J~Y1y5&1`YZ1hl9ji<;qm9hFZ8LAR1Y8C9
zwCnbkzPYfwv`G=!dN)-8P4HzaAm=bbPVzX_bRXBfi;0Vqw>9wRui??Hq-Xd=k(m%H
z(0}b7aKx~|k^yn=mGpFLMdu|{`&^ABcmqfIlj`#E@ttxyipuz&r=?q@nplU|5kq8}
znDs+^9nm1nC_b5oEVLu@`7a971JFtGZdtXeR3Z8u#p=ZmvJK4=@f-S}CQTO;ekeG{
zYSTM*4cYNLRu+k6nx0Md!Cf0UMSD+p)zm^H$t}IX70XJ2ZB7T%yb79@tnF|01Bsri
z3cV+T7v>l`Z`(hAD30E)TYa#LzVZ0Gmxiw=^pQ96=U1WmXPdMsBUU%o>cchm+1(!S
zQc1vFA%YCp*N!NBLe9OLsc<CGdEdSYKU?sM|6(Qj2Y1cAx_kxE<kO_c9Y}Fxgn9jD
z_Mw~+$Z<yW);0{f+g(LpYed2%UD-aVqfC#bX8-01APc4aZAb+%R_Q`tw?l{j(s$bn
ztVjcyFyi*3OW9WJVc?gq;#ka*b9jl$_Cd{4{0(d{>Nd>;)e@7)Ou<qS`GgZlG#uwd
zsorZROd<y$W5fRf8Qay(J@BgfX@4*6#Ig1bY!$HRzLNX_DSX7VDI(Df9TKeD%{E9f
zbKNsZS;vO8{k}7nnJ=Idl^g3gaZNpr_}$(5<Nx{ajR@`2tGgHs!u&Ph47!2ZweZ78
za%RhCR7}_XJbb1P8m)5$Pt-!t=pBAMe@PUD*N0Q>ybxtc!OCauZGic(0cgCQ0?Xo)
zMeS)2xY*KyBE`6Mch-0}3@GluUa!cd$^W_`o*OQ>X}twW6=uNazPhIk#wFI5wnFtO
z@SSWGOtz+@y`E^iT{Wvh`ZxrlE+z5kJC7hRGmOosdr&Q5@KW;d0;lZtgc~Z0vEaQ;
zd0m78k%W=)@9sB9-}jD@qU7e>j+iM=<AbkU!K~K9=Kw6={1DukDlAIoQZ8dhPe)Zm
z^fB#~q>hSIyp2q_k&D;b>)#M`x5dkx9$~^T9fJ*hnTbDJKoe}<bo=|dBd3z}UZ6mV
zpCG3uabYjVc~UUEE=JsHQ2cym#uF8vsVjOh;Cq9{FWx0O!`$55JTpRzRFwl7T`bGK
zAnx0Pbr7KnAV#+-C%ldg&dG$FJMOd*FR7eP#U%Qflfbu#p~6xCJ%RFL<p7tF59jH^
zv{aPDLM)ajG}8#7oY%X|y<}0XM3>DACIE}{vvEc45F^B+a7+F2n=bEuAbq25<!Rz8
zOZcJLte@c=!}B6D=2N!C`)+zh7(&XE<pY+)W{fWWePU)xWL&2|s0{rt7lo^3RG3L5
z=TjVuEiMiRu@bYzp7zeolSNsxMN*2fHrUtnqwnr}-FNe)*NhN|QR)#x@{1X2*rqsn
z>W&?2E!0Qk>77f{>|)oKdwP_}a%@A7wl)}GTz%d*P4&nsaj2lLejiTvlo_9YfAwS&
zCiQiwOu4lL0SPl;;M1+yIwWtA7SdGRIqmqckbR8>Zt_z}FFFVK*GXYfXvG{PFQ9<Z
zo%a9^iwo_e<iMHeUL354b@{M-p9JNk_b$BJ0L2E?aZlPYw_HZ;>QPcEc+S4Mo;Qx9
z6?pGx^%BQC@4PiQ^_m{wQM6S&xNCm5TTMGbs+cV+l}N9il|#pR8T)f90Fjg$&lS<0
zPs$SIaWlm7TkWoZv@&D!N=%}GN1e^aK;fzwL4+Cq;&End(R=nCREzl%Nw3>ptuKz`
zt1$VtM^Y=|Q?_3MHbg71kFUuWorgjE%`b+ffg>f`n!s+8Dx6=JHshUQC)@L^3%9n*
zO*{7t@KA`>ON%e-^C%s6g5g2_jzZaIJv3MZ{KU2N;x|S8gDbW)yeB);6G{e^aP{RR
z*E@&!S%$8_1T|VUgB`4=|G)4jA??9+2tYE$K2k=;+~57t|GE1we8fsaho4*Er_s*Q
zOFmKNM5|3lI2(c?7oHRKT~m(nPc&oybPpA)o@4!)3%;<^jxdrc6?lKVN^+hHE+?Em
zZf+wkOibaiiq}#t?q3l-@8cZI$$^e@bu}h-(`kqWONrsbR?AFaKK3YvxKgi9W%_5A
z>eM<BIi52zbjtVz4N+mwBR8p&2<_>oJqW4RTBv{W4hx=W8yLlpXOH10Pl0Vh!D)%h
z^ss~~s@%pMOnNKO_uP=p1m9jCsYvHFQ`+ff0y6J0@a{*P@r3#Uo9p=sGeVq&IX`a;
zKA_b{=n_|fG*JrTPZz@a>OHY5C##&6b$CEXmH!g+j{l2es2=<c&;U$6ewj?MNbwX^
z$jyL4FbZCVj@<LQWGeIeRKel}sfjE}(T34J-_JT%`<u}p_mZp$&r9dg&AOw3vVK&9
zeoUo;;Zlz)9vZyTW<^+eV91G#c&SxkGf(!BD}NNs30UyIAat)P3NFXJY0V=&dEZQ?
zVz)F|3?%Z`4I+wOAM$U0jfQP-6&D&eKd=_~{+#h-*6=l|+7q6(IMB#)n+=IQ_AZ$Q
z7n^b{r-pIsB|2j6l+B~Sy*L(~8o%Ro)Ykia8McFRA5Ap5e&~K6cI1ILn$IvKy}6x=
zp7YuCY84bK^bgIpc_(__M~n;EOLg?*nybcGPd;yyEx<2cI!M(%7XQXwDBeVYOgy#e
z9!7kv_h>)IK<aJ9K;cB;t89nip>|h_pq^5Li-!!F43W>^mf#zB#na8(=bPfWzHR#@
zBdW<HQZ51mK3xw>U-)!f;jX^<#yD9{^X6-WyqquEPSqU7Pu3au_}q@o-1eK1eq6}i
zX*lAXzuzO5ZX@P9@W#m%aNI_WB*J<Q*O|tPM3jao$BQq%BJZ9Oc)8m_N<LtgYC54T
z_RZjt)rY4(z_my+5F^eK{kHWBv7OV=D_*0=<16JYlfI*{SkhvBdA+gb!#lFCA%#en
zx`M92#uE;iwW$K2bOF|poL+RW9?pCycqKAAgPFad6&LHTt0-caE~4>|xLYZtP+B~!
zKLZ!gylO+bLq)a{L=5zTmGb^q6~M_6?=x9;!1znet;VytD^jXosm#r4^>5LX7sbxK
z$;bww$7@yO<qIOx4(dXUY4FPCsg|N)6WGT$*`q!i@JL)k2Z*9)E^}|uqFzV_k57P<
z>5&6Wp|c^pbdR78zil5bP1y)Ft+(%G&mWVPARd|YEf$75dI__MPcJT@Sdu{7H5al<
zmOHS6oYLS|T0Xefm#2n}ziZj?aM>=b%##zYNtBwE;glhOPoq&gh%j!X1l#W&+j!G$
zuHjGlM>Vrcah>m^|3C(2h}qQs3N_6CZe|%_ySZct3~xGGS-)i2vkt0_Nx|`i;&+7c
zDUsKG%0(kgr<XkZy&+MJ{iirQQ^;bc{kWYU%&nT3S$z4c@op!-rCIi=OinpJC=p)_
z@gD89Y}5h}jsjgLZK0Dg^pL}pIq`@^r{R;9fK)*V&BPTVm>)u9ZkFfvlQnRDRg*ko
ze#tV_rCKRlap3&`KU5}^P`q-+4mGpbaRBYq*6i^(bW3tDS@^l3Y&fYr91;ea8J`?4
zbbdjr$WT&6svPC<zT5>+o2gl-(P$o7Red7(G-v?l%*61H!hZ7AyO+Aq(L+#{holK~
zzIv|xWLYp%wQ?CI4#?LqLXh8R@`kG(>Gr;s)LU#pcq2E|q0zqZ4qlgDJl2+KQ22~g
zpF$fInIF35_t8rZHqnd{$I^|Kumo}N`{L#hq34I<m<d%G;xyZyh-Eh2xSoJzLeB#`
zjfT)*Ze~hAGXyL_^yC4u$4NolKqd-Jf@-~_M49slnluiI*loBXa|_OR@p}dJ4t}2*
zLt_7alcx2<;@9T1?55-BlhQtrIodo-OM8&OC@rm&4<4)7VpmF>tCaM4Yc0BN7QksC
zkurBpg2){pDuMgOyc`>w@5>orhWm_G?G@oRp^?$DM_iJJc38q<D3y4Yrw7_Qx3v;H
zeib8A#@0gB<h;VYXIRd=bw|n)emoHIRJOmw4E;9MpQNJ5bQ1lIBO5Ph#u7)MV0(&(
z3LJBmswHxQQ}S7vgW+L{=5C_Kh$77Qs+KX_RzJVI=CsQ2Kfm=&)&s7a3^nq)4)C@s
zFo9%R!EkVngprLW__DdDs13mfI)WyBgc4uvksD+?%UB@8))L(y9IYEdbalAS=-E`t
z^tvJRM6xnJNHD&@%f`?>f4*&VERf|6p&I_{XS3sXz_o@r*o+XYU!_mJt@bb$t<W8i
zzja$~6<k&g7_q#Wzq?5xI)H`5=!4X{n(?S~AK;IX(f}~>3CC5e7w*TM<UlH%s=?k`
zvfAPw@~qqqiyG?(_E&T)XE6<lul;Y5F;`*=OlSp8ZXlMbwBPR7X%vL6_o=oThNMun
zRj@L~Jeq^q?pjV{zmgRuU`&`CTODth^2K=roz_v#qi9)Mg625f<IvcNrh^BG%-#=R
zWrd+elTps)85|xFd$YBHjvJM0@!8_*Vec~EOmW=OWaXH`45lQ-wh?>j&nz(zIcP#{
zz2LJEjH1C}CoqX|Y*A7MH%7SDIX{P8;K}|eP-7T85*-vqC+zC<!<G6<F5p17iCoY%
z3SipgQDuxEAspqDIRI(g2P5<|l(~d$H(c$$T-^xzc3ItNubt;$UKTQ>z7(6ZP_0q!
z5>>Lc+ITzFS@yXVsa$7JCkjJt@F4kRo1mjC5p*C~IHbmTPyeM5*!YLyA8Lu&JH5z0
zi8xyC&$cY8eGygbK!X(~W>9+*KFyz`#&S9NRbCxntcA*JRsK}joq?j(OQI>kf>3sG
zjC~K@aZhre>mX<RbnQivw)$I~A{*)&DGnJxxUii?zvqS|^Y@oI#+&v0aGkY2C|~}}
zbib#N{~$)~tNu}fdiIYE0T%p2cyA*h<fv=ln>})%e_=$Z5NzgscXGu<Jjd$GaT}GZ
zTu!JUazD3|{=-0Hc)T3Dy!O!2AWV7Nye&RLG_9AVmPs#4YGN4|0CZ=&J(k~d`^(Vc
z_H7BQfa8rb6NsTR=nqetc^TT<f!!|umKanoR^C~M=*?BkDm{H!))BKr2`dQ7f)6SS
zn~Gr+$EOnNP>#hNQrOZl0IlYpmUQ`TyC1X@Fb*zj^s)3N<MUeolmh@-CdkWGb#;Cg
zEPSi($81-uie%^gBZC-!c7Xqkw2Vi(fIlXf+x<>>zX{GlzgwtVkRS0tYI~ySW%7E#
z`iThs7YGDC<aHzfxSwK%v%#@`(Uk*kx1n7$TjQL&qEd^R0PE^nDv32Y8OK<@3?zTA
z5#@Pkpn+8?!4JWQVG_bve~0b!a&GNOLhdR3_aA$<o-c}Y`%zC$`p+Up4VZg+qzmHs
z)QD6A^RwBKUba;_&pp>$nl)kK>wJw_(81^e#J#1`eh|w~kaNbO_=&se&3Uz=BebV_
zwh;%rCB#b9OeJgfq9p<3G*Ul<ObyUG#Pd6KU0(Eeb*aW+3=3LUEh1XXRFx1jvO5`|
zHGBA!j1&%w10KrfH0-%OI=B_V&{x)N0^zOah>)3rc~s~Dg#b#A^aWeIPt3U9=2&ht
zhttUB{;b{vBB5ZjB2!O`XyjA<uU}Gp-r~ik6=Lb3v-Vi8f<+&#%kzLHTO^MmMD=kF
zwig+V#5jcKo0gb%{KbKlFQuSxi04QV9ncf4u&@GSw~`2_#W-)-SNRWc9{G2Gw2&9s
zkPD0*G<^5<)SXDxHsZt-VM`uvi<Ym%_>sEz`#$5?uNWG)O|Op5lzcjBu}e9m>N}<q
z8-vPyZJy^hxzK*nR~-=i(FHTu#1`oe#46^zByWR}Vf|75qQ5v!Bje}dUKa-rh{)W&
z3Qr5&IuZr&3t#cv@zsl5v)vqoU|AS=7k!HJyeGm9HsW_?(QG`}M~|Q*b1Ds9XR<B!
zy<l%(ue6@Ei0L~eL_3UOYQAo=Ngp83F->;NO6#rYhRJv<75>Amx&GbhNcU%z7e|j0
zRxCvPgTk@;F!y)z!cGe})8-SedW*)IZOSo>u_=|Ic}`zpg<0hgv{HG)K4~1CPTL@1
z1+Y}FV26yN>XerF!({}LyGm^~?t9-&E_C8*LfM{gyaq1JCI)gI+`OuJF3JyoBFMp<
zTUQQfO2%Mcaq`1MGgu<sY5-KK5mAC8MGC9m!%#IZCV@-~OimoW89@oXuVHn3UWvD<
z%-Dlb`k~><_KK~IH{l6z6^v`Al0ZF%e?b`{f@08Lyx96fg?o8zTXgM&-On{P33}O0
zxhn~|yCy}?)6EbNPcf#AZIrxAMYw#yH(rXvH`~H35S)}j#7jl*0e`&a#F03CGnv|{
zbVJhRkcKQJP*cv*X)gRQyg`hq^uy>;#waIl$5UJz)4{mHajgR54UY}{RLhalFWrj$
zd=?Q&$*|`8zkDAci9fPAawm+^nGKRpe5epAC;HO<EB!)J-;zlv@Kac<le}-ogdP5?
z9n%Rl)%m9#gW<g|W>o)G%y7Mp9%Up{$Y<V9YM(7}yK!zon>*S>!eQ2b_)F+BRorjx
zVUX!RJp9ie9s5w6tx6(Cy$NK>D;ZRRxl_)Axo3hxeThhGZjqDR?Tj<OCU9HgCv_t>
zIKg6CK<C8t`YOyvY4X&w!$)?R4Ou<ZAeLm;Db*W`<&pXmWW1eX1>P`eJS|d&%iIR&
zXpS#S^s}_;mIlPl-5|10yx0p!)I#Lykh9u@A3Vrt2QW_tw(e%Zi$VeYVGCe+kp-dI
znm9{Q%Zi|?{*?v%=bu9!*kZao)~It2rYcW#+scKxk-eGcEa?DcG#T4*3~4=x`zxCE
z;wGzx7*hh5ImbJFBRJ^pUEXs~#;54`g!La?e_a%UNzy``>}>vieG#MUmTPz*V1SOT
zc@Svo<Z^=c-D^W5C$SF)mQ4Zit?FwCF5>c`?S#boDB_XESJ{;Zt@(`}(FJY9+IN_u
zv2qhS6VoDZDTF_?YhfKj1356%h(_&A+#S21<6iIvbx|1<5iRErP%qrBVd$~i9te6@
zs#vzgTfGTVrD~=ehc_hNYa@2S+Zo?MyIVahIgo#}+D{-OzwcG>jgjnGUEz0%U{P*l
zl8yP>o2whyU%UbzA{YXqujlXHJ}X0&l;tk_lHPGpBNTqMjU*3aB*JV47s_>fL$D=?
zZIpa;efD_wG?2`VE~Xi)hg9@^9VO1@p*T}1vDe0O$xKTX1~nIRkQaOgJ}>NY8am0i
z{nO<*#5e1Ay>nL_Xbm!<E-gwS^ZQTHo}z|o=+R|UZ(J_dBFor7Z?ua1qxdcxp(L#f
zbg9<3>qU*$Za{(q+%2^|dKZB`BP4i)p^BV?GvX=nyN>p=L9m0#2t_G0-`^2B>Lkz6
zJi^~%>tLNxB=>-4ov2<F2roc9jKX(U^r<6N)@wrcW%0OM?rGv<eD;+@x?s^pRsH&A
zzH^W$)Q?T2TZMJPM9@eUE9fshHhsg#NZtTmENAm}kE;g3FaCHE+oA(aSV{3cj>Hq}
zd25s(j-as&e(+Fk=5?7hCM1F4ya07vbZgIva9I*FSdL48A+*Qnh;~SBwlnHQ!}D;o
zHy`5z#RN9mSKxFSaWvEEN$S89`*O9tvLafS4yu5%{y<)5TcQtmA8I7Me~@Da3>vP;
zF6EKen1-JCVY^4%9dsIqSoI|=2sP1vP_gCb_xqe!bggRNw@`O+F%1qYvxhGg%_cWy
zt&`p^tb~R=AIkzi9(>h<6x;M&f74ivj98hDq>8Pg#!nbRxZae{;)YVFK8E2-If)-X
zrj6r|rpjUqR+_*>qG-mzh!hQU(HDw(Rq=*m=E!yhqQFKw5jB-|Cv{a2W#}N%mxgIV
zn)>vyV*dIgziOF+R<X8ZRfE1>`ygIrBO*S`?VsQCHF4!!)8w>cU+>M8b$SLyQ|@a{
zJnEBhL2^q1?5JkGwqo~_lb6SPo_Evn)Dt3Wtue2|b2KAy8+7f#@9L=7p>{*ySN4v7
zB<%g2-#eA23seX6pi_91RVB$7o7t`Hgd?MmHQA8wG`KIuuY()*xMZjV)OZ^j^N==Q
zVn2$O+I$U|6E%a%M{km7;_V!DD_@23T*f&j$o`g<!%{48@jX`M&1C!+kCvc#z)Bg+
zw91${G{oDhn(*z7StL@o&{Brl8Cm+CDF2thr_}ks@#uq)-)!s5kE3;L!=0qr`T3ON
z4jaffwI5Xxqm+&P1kQp%VCt|p9Mb2K6)zZPJG&RlN0i829MFCZgtW~{n`vb{#^<V=
zScnbo7ttHXPCL&@GS~&)12)2@=bfQJ7&dtr$yvZM<rh(q2834<AY4skz=ZZWWHgA|
zjH-^HMX?6kdiUL4qioRrf=qF2x{$~rcN^tA$cB~Gs4D|$u<9`?M7)If%~<u)@9bW>
zqE-2nk|5FAQKOeL79oaeVEJ@5g7~HJHzAr7FTcMVw62LDh%M--W!tS!Mt3XO01N9+
z+NeP<d?5G>zQVOs4`J}DKj0}ocQg=x7*M0ZcC4U^6vU;quT6pdgm3VYjTk|5<7@OJ
z)oZL<jcfeDU&O=Lv+;vp?$XgnctQc4XloK#inygE;Y%E+DRvi*vH`H?&BGMA_DTuc
zPAqMk<tHp%xJ5T!KzmBk0i9WRw}5`S5^Z(mm4ih{p6t37r@b6yg&2W}NLXY$6(ltx
z6d%N$1*wXt*M4vM5!N>$#939WjhM`4ag#GjDq1d7<YEKhq**a>szgPwhIoYjW-w_O
zGMvnljD;L-`v>QncQT{q=Tr(|jpt!GtW~CoXwp92GSF)1NGJ(AO6hGIU>rYHQ+JpX
z+^ZY)DT-)BNv^!1nXI-)dj3sY+Ejfx;}N|97{-OdzaKOrcmUQ3<I6eMejnjU@v)+k
zH~U`?MBSLwzt46L$-YD_MV+G!Ae7Q~_nmazLg!g%t|dByO;$HFTWiFaAeWJFnV%+N
z+&rzCYH>mB&`}Y(g@2x-kzAIOoNyDUKL)l<c#AT!m_F>oiTw7@4$iH;WS0TL5c>lG
zCsYi!oiH1LdYjzjTC7mn%J|E0wn~cE9$V52n}ddcW<2<L!C1-b_5g!L__+pNn+|uX
zb#aMzl8}*oq59Urh0UFxKbJ5iFm~VNr)gI4bh(PBrv4Qsr`krkp%TD;P__4lq-B}R
zrn6gKr!?C8#p?&6yn={BqwW#86uMZkJLHTLu%wCshT=vy)W^=l*M~3W8-a`0SCXrl
z4Y`Ni!$`!g?DG%T3y|kaY!5RK;K^$FeMI)0NOA~{4|uPvj)9u0s^2<`b3OF-LY50+
z+1H}|t4VJ@ww0Jr!V~VW;T3xj_h#bEd55gg6D)qp$B|WqtUnz<ug2nC`?J7xp|0zj
zww@@M)sJv@D}fax=&E+$WeV7B{ma4u%)s^ROJpb9&nEk#-m1iVg_n_ABC><(f?*)*
z`m~5_yo)~WT1IvXS!Q~UAYS<GtCNf&(7r)6Mn*?(qU6JFsSY1rE>ZJibn#)%-wihI
z*J)AN_AH;0CX#vU!CmZ3%;y-=>iU*JwD;cMSaYtY3{8mHEK|?cI%5WXM95~ItvxI5
zPfrXy5mFIm(J*X2A_!lBHP8lVQS}oMfsb+WLFqoOE_}@b+shG{>qh)XHd0&?BvN!n
zM&FdeRnZ;=62-|kF*qPg0kXQ_I@jR1c{=^eb;sKFQR3T8r&sYn=L7F85$H7vUpnL{
zL;OeG1sGx|u-$qqq5kQ$vR~2_ufB90Jx@Acf`AYZs!z<8;;tP}?*+5o=l?K5pLUz;
z#-bT2=i~#GDQSsJNK6|s6Se^ik{!r|6WA^&!%b_|_z4}2)tm^kDvn~8!_#<RHJTNs
zQ1OXMp!|)#OGU++{ZFKlwPqx8uKL_g27`WO;1OMpsN2WE!I2pMLT?@GGdu*N95^J4
zGW1ASA;`Hr(=Ig%9>}TyD`5=zl_A7x4HxgTL*PHA=E`Hzr>TWP*JC4R!Qa8#JNb5z
zpqYsu)@gKPWCP%6nV_;_e?yX*_naJD@uzU{gJW;D!1`csF>KwX`W?g10br`nb*J8b
z1AOkDNW6dyIvXJpJuP9zmV8ot|0oPtQaw5nC2DorjB7X{@l>^riCbmqQY`e07sn9P
zP-wY$!DQ0{Zb(v5{b~Fm7m+ci;TQCYAHibjd)0EpwuuA;tf)9Uv>y*CpM=Q4u$7OO
znk`aVjE=jUZ5lRDr^Jo@s5YdvuOI0yOCKN+D_Hb}*5c6^pv!TvDL-(1yO=ILma?~a
zPwH@1u_{5n0Z4kDhg6V{e(m$8>gqoxef(Vse?qqcUfHF6PpsR0)hQ(8$#|$#KtVIE
zgehOp%y|Ty9b^0F1|XJsv>I*1Se7EDHd$_o&AjlGUTyhifMk5<LmGHMx!~UyHnCoK
zquO~sSiCquE%bzE#8^tQVA$0+n0sHq!sGVWuBfk32}LR<ApscMZFa&x4EgNQtm1uU
z6tdm-qI~036_s+o_xuU=&Y$VxGO|&Ot8ivmuwqn9mr8*NZ=wHLX_7`TcN6dSBzuc<
z2sV0^sNs+bXSb8}dTyc0s+?!-H9gg|R~OupCZi&?XqoLUe2@#<kXGBCR=!t3sriw~
zPN;_Tpf`awbFA~?G^!!{x7WE(GnEGC*_h(l4G|ZwAU<3G%D|}yggb%S>{pSr>;yxE
zyHoXn%HQ6oRE*&*lb}Wo?t381J`gApiD0JO)_Y$P_ZABRzl8T^dR@Y!HGk<2RsD3M
zc3c%nHHO9{&%M<auIk&x1s!8bq79)tc)Oa3r%Sc)qjrAV3D3cG2!&DsE{HDH=CfX_
z-A(_upC%UzzKi_GM<(#WsU0fuT=Y;ogMCtZx@uZcNHE+H?lGJI+M9ztv~VnrNc1Vk
z*H|7LZU+5aWrag{KA#r+Q<H|~H+T#~8pHx~5iF7KGgxnloGc#~ER4(ou4>;Jjy}n9
zWYbS&tXwwMhq7i(+)4bFUtFKeZsoJt_(mi38AFfO2D49vUPs8V$@+1^Fvs$41*J4!
zzPKS#n`f6DIiV6w7ehdG=x>c>`nop-n64sF4s%<#>wtk9=J2g;%nlrm4=TmsR&?9-
zufamLma^Y)D0uH@_mR1$FJ-VsJ}S&6{D7ye&8rj@zWt}lg?tq0ZO&V5m#;{guL-wJ
zv;gT-7DTv?idt0$b@>Oqxcq@Ii`~WJ<a2_|m*@3nrb<oP7F{{JC}Ym&mwi9?L~M8D
zd{`OVlSI2wma={7pKpmX(meTmQ@&e}2J3#^r@#Wrh~1=7p;y_#S=vbxJNQMSLS>Zq
zoXM}qeCU8S82lgFV9-1GkKJ7teIcV8Q85bY!B4e5<Ib*LEKe{I&zxcHOn+_+pdJ4H
z!tS^DY2hFGzry2eSAC8!{^p#)8)FtnAGM0tb=2eThnutxQ;%Z~TcjR7>g`Eidc%Y~
zIHF<e=Y^XZwsrku6hKfpXjA)FLuqJvU}Ee~x+*)=2sa!CDtka;-T<pn38%*;)Z^zG
z2ysiVgM-0n$fgvSee^6mD}5TZbsCsdNwgZ(KG!8e5;kGtDXDM-c8{<^EQqE6X++P=
zo<J2TsN)DH6V`J5*4m=|T$jx&<~^Z)OKU1c2X<dD&Xjur3dNElHEio8!DO#?Ye2`1
zNq>kmrAvjcnk8MSVmPl;#-n{?ObOUS1Mh0fm~Xezm0}T*AVB=7nh|bp!00#AqF{~k
zz%waU1jG~<-STSgp9o7h$x(7&0YSWH;dn&b4?APt^N^d2ZGk1eJ*4J<I)UdfUC)Z?
zA@PpXbirVX1*4(QnF<|>a)h+?6OM<UX}@%eL$yJ+K;y%V*B}@N!@N9X-DG>5`Aw)b
zVREwz5#p~!d;yS1bJkc4u@w7AXiH0ym#>JLVU@yh?gCujk*USI0CiGvbgGu}+g(5y
zNse_=%@*IS!|sfx>)o5FhY<15E4V?-{nvFaC2=gGP6Z*iIjpP@%j1)BDxQr7P+sI3
z4X?uesK=YId|r=lfxJ8504~$<r3izUJV1#}srd{$*+wd)waW28njqm!)M;UpS6Dd7
z2c@A#32-4AGbqM$>SnTJ!r*{a9Wtzeh1LPriO}Lr20D*}2jOd<`jw^I{AVQ}DQC7E
z_HnNO0qaO*`Q{Alaf{z}PhC1RSktN`_XX(e$G)kf%6gKJ6}q2D3<HZuiA;Ym@A*i;
z0U1iBdGsondwhB04sd(%77_gLrGmYkxAVW%e8BqAj)`f?ZtHV^(DOk`EYy4W^AA-T
z+Qx{C-hyZF=c>t-`b_?3pFikz5$Tf#lpxGoFvmW)WAk(-ptlHp0@Z~<LPpzYxVwEP
zP27G(u%|248{BcOf=9T9iIy;=ye*7ERRVYxj7rMO4nVi@Ol#rY)9UEZ>1MAaoO73N
zcClg~5OOyNaWbSpn3CilalT8d>`xqWuywUxlSWL-x}@)zgoJ$2pItf8mo=NZUM`{2
z?ZJg#LcCv|yd6y=<buBxa8Hr2@uH)D5-)}sy!ZY{Ql-B!m8v)d`3A_s)z(>(eq1BX
zm)W9W4XCMc&5B7`M0%%I+Vl?NDt@BdwV~CD&+vR<e6>z~UJ)=S_>|oNM=0WS(DP8D
zz`OYlux|<+SfSWn8Y%hpFzCnZ-D{&kl;e7H`i^5>9(*=Iyu%;@&a;1uyCB1wup-zN
zE#uhjcKYpzZ1l#K<u{B3%&5u20gPfy9}O;83Aw#(HzZ+v{z2Jq+nbGd1&;c{h;B7{
z!)>S@q>u#&%AEXuqH-KZTr}a4eP0N~Ttq``1O`H{VfURE>b)CNqEls?rj>y#hkXop
zq$E|-{!=`AASp)+k4D08SoH38dT(raX=Xrc1+41(22sb?pN`ECAF1+U$fSmdze*L0
zZ*L<m5S>Ix{PKypb@R>R;yaU%sjX&$q``jC3)`r|%?r)Gt;UM+N7fx?Sl|8h)X%Mh
z@1>c0iEC2=Pq<dM>Z~g-QM;_%{;}~1@EQeX7&@PK|H#0AvyZP>DG`4*FTDa)M@*X5
z$zg?8`MWC=w*nhPW}G5Rlp`iNS+I1F{VnzR-PpY`3_nS3Y4Jb(g5XgdLpHbIx;xU#
zqOM4$fkxMZ=+xxK$2b;djFk>3$Be$b<tSwJPP;P$s2?+*13SA|IUoxw9N@M3DaB00
zVmk5w?UWMYk{eKI@pz{Dtxc%9jC<542#XaUvf2}scW{;<-dun#Z<CT|;K#Peo=U73
z?>_9O=mL$B7OwO*j$L2O+X%5pl*a-FVXnys^Hr`mIs;Lm;76O1jSKG2AlW!+K$n;L
z5$fc=5#g&ZK0mzQT7(R(E;mJR?$j40m@^4NqZ5aB(5wSzu@i^IBHL|Md_KJev>ZA+
zZ2TrD$rP0~<|#gKM@r08cv<W61=zRcA<GKDg?i~bkQC~%?@yV^s$ZQ39OR=UZ)HUQ
z=}`f`(Q-TIRn`-dDb<P!L+2A_Bbp^Rk@puHXAzFG1m<8Bsb-NUs^t|&@9At?G*<X(
zqV5iePn@J-ODqwBB5mqVzPR>uU!}zbIpd0l3rlU?`Wgk8q4`Ur?G^7E<rRG$wsW!Y
zqksvL*dIAx?u%nUKsyoNkorW!^b3yH<=u}P*<C8q&s%wnMQDmFTo#ABMKt4tXL%8w
zwRjfVp<fYas9vz9ehcQ--VA>fOavlfiQ&}4u}Ib-mTUtKh7sjT><hitGo$a$er5K?
zjd`0<IB^)X;V6A2B(&Y)xShNFRW=A4zDd3%k|5CUekZL^keiUaYeR%%d9KOP#aiQ#
zzOWGpRWbWW@^q2J9a_nIeluORUj4x+P0{Df%ss)3PJ(D@<G5sIiDW6nT#4s2;;u8C
z>@i^wf^2131&oX$G+)e?`{luzH%)Smuec{yNk`Qo{{;Db543lZXk<Ykg8YuflXIQ0
z9oj-F_Zbjr(DEqGR3c>;y9uRC+=21z$+GYVIbCQ?lX_*eABR>C{b(1u#Jtjb^5t5>
zJz7Hctuq_IlG_A^rOtUYD(iD5MI8+3P-pD3l!$dIAhdg~hD1eA=;uHPN2GQ;2|MFX
z8TwRjGEmMfEl4hE?6Ell4C`O%Xm5kOraw81bdJ8MVI>BW^bZSyV9h}ML1T&jeX4SI
zZ?MTgEX5^#dB4es$X9N!VuubjdCSHO==AM*_Vr2hCx{im5Iyj@@D7(t!*JP0n%~Tj
z342;&xxsQwP^&)d#uG(rfIdica*Y(_oe``7cLTrM{BV_VCH~I*rp)m+?XnGNArW;2
zr7|6bQ4Z6Csq4vPM}iqF)|#kd%&j>IJ#|$-9^c3n*!A_dM$>q-DR+Hi_YC~mEHpSa
zZ0}Jy579~?%tP}aUBT@;Lu3CID`HyN|JesK)Xn6XR|aR&AXFD^IVIq>6<vQ>v@YB$
zM@EaS^Ic3vCsGK%?CPcO38wN{ip7U5zX*KhTi>5-xy+x$&cYo70Zlqi{?{eB_T66N
z-5N0V7}FH?w4F;`-!eg<a9YqtuzZ^187|pI9PQP*8Cs#GRp9)C=lBvS5m)>SM>%!S
z#hp)(X1O<nF_fE35c^q`9rlSkwP88dn}-uRzuUdI|7!OJ8FXR(f7-p_uIm4onAjkC
zbe6euUEa1H{|J;_z7<e}`@Lj94*U-aN7P9g_O6)Sh=zoKge(T`hxvXJ(%zU>`M@LM
zO3p!n=!Jepjp7Awe@sbthCEobj*@d@+|Lpqz2{knha4QMJKO%aPZ#;TT^o$$rvw1K
z0xl0ywT885B`@~J0Bgl6D~ytk2sE3XFN}&rL_~|F_>-=LrasqS5ki>Qh|yP#A;YR)
zm^9NrX2h^5OL3t~HK07^Z;L0@W~thTy*%t?l{8j^FUVAkxe%5i?w(<K=KBdwQjKst
zhHjt@pC^F(i><1R%igxk6K~JO#p`>Pf$_#SkkI9ddb56bM;BqW|L_cKm<|oB=^az7
z=1Svi(avfo?0{b>F>25ZmKEJYJIo*zSBTKE2GsqqJ39viFI9gKaM*}+Gg&-g8}s<c
zs2+*|9Z@Ze|K8^>kU@O+S&ihbK|o(P9B$lRM){f<xYL@!;Ww8MenU8$2iy1}PsBIk
zWMEu+vdPPd?DsQA5UM|f`KUyOT!zaV2=UbGmwo3?<P4qB{E}^DCTgxV-sm4=WL%jj
z4UFAPOgMIg62b4qyG40i92$pimvL=vsyW!LMh{cnny&aP3XCiXrb6WmffF%%k766~
zY|(yOJdw%Wfwc$VVf9<SaW3?YXs)7r_LX%u!e@qfSTi<pv9aSoxgf5;0~2guQN5V7
zkHwh273q|HVR%_+Fn3oKo&|X#X5L2qGsJ9b{g;Qv?5PM3)>Q!3kNz+X^e>Tf;TlKa
zS{el^(|*%x*!Oh{^-~tM-e;4p!h@XK6BCGYmItJk&4kuBy0TzHP(F-u_%<OvKCcdb
zxmK<hop&a6j^NUHAAh9-1Dbu=D7vpJ&K0ei4w}BBlq1KQ;}OMzK)W3v2WvfE>7s?}
z5S1!{+Z>Wlq`aVDz}O`ZrLaa8$U&M%ld}oc9Dh4LDY_^qPL)iUhD^Vo9%6F|;(#f$
zEojm^pl(XQx!pzN{K3(~<%YJk?-RNi_pyar4E=PTq*3Z54~hJVJZ=Fi5jPGK(E*sl
z2toBcvJoq3v44B0FYt8jtw=$?6Ao*}d>!tOX#1jCwqfe4T4tbsO{7J;{y9c>o#394
zf0SQ;@D#JbO1vikQibnMWw`6lU-EpdFRnF;L99SNjVdrgHFu-{=~%VkE8^YDig?pU
zg2~dacx+zGZwDQ|-`7L@ESf9yeb2VW0yRDolswtZG3+ORH}}RBI2u<4CGOd9k?xB-
zqphP3MK6=5f0HQ6K)BrO3gXVs)vdpL81uvOwj%7sw_cX6op|nBCp#*vlKSWQFZ2@(
zACE6ws;s$^t-ZT<=WQDH_4XV*avQl<53?lfxB@`DD7)dV!fIBA_M9N**95tKNc;k2
zTce3lBG<O3Ri9<W5rA7ISaaLaOAyaGR_N8A-E^%7X!y1P`b1h^pO_#@<}$}L`qEF7
zTT>}@>8YXKlV4Lfwbd(tvGOz)pWbKSP!u83d&J2}yDS1ljfg0)@2c))(Ged<jsBju
z87qKZVl`TQOyTw(@s`>n;aKL04aP|fF*kQdCh3J2bqfrw?@iTrdd}LNy)L{X(&4BH
zguOVECU|WWTx$I?<gF4UI{^LVKGbmH+blA~RSFG0ohZX!Pn~Z!`Mwp5_^(!Q>tu(g
z<TuFICfQ_et57$m5b;;b?%gH^{fN-ewox*SIzB(rAMxeu_AWy=Vwdao6mN7mgHL}+
z#*oi%_7+r??;lOwGrM>Kn(bI$K?9a|@mp<aGdGuZfeCH#Ej22ni|O@N#}K9u-gRk@
znVFyXm!<ThM$YmHAR5hHo;+g}*4goWlZGx_X)`}COlLvgYSo&9WB!y+LG~D8>5$(U
z76SanAJxTrza{mlQ8pNyu@1v|p5=afZ%X~M#){;*k%|j(!oA0wUV#m*K##xY4>@LP
ze=W0-i$cWe9K#OagAFFde3if%29<-??)v<D8F|n1S3f-bisOZL*vFX>x1p~1Xym3D
z)8#c#f&@`wNJ+wbt$Q#jz3v`nT5u&Z)?>#-HPAqw0p@i{&BKTdw_BF>?2k}8_J<lQ
zelW~R4OZ;)V3umu+*OM=lY1Of=-BsvPyXTk&alY~=6It?_H&H9M}OEucO`tSUYW5o
z+#J)>(aCqvVpY`l;=&VB59-kc9v#w-_!>Qpge}V6f1ihdn!NvsTxrCYcK|@XILO9T
zA|e(0R+GivjFd4n;~VQzHjYWvmE*V_BIz7>T-6VgUzNm)xSzA->DHSxT}|9E@YJTi
zOReOC@j*r#<JKQiN{kQ$oDSn<yxBRkVD6&2!@Ma^!Xn>T3tAV}oBdYItxp{iHajCE
zodxvnLdtiq^a%t!-V8PoHU;tF%ANoC?h|x6EK9zb(j#e66+{Td_0`bE;s^;NA1KDM
z#Om)4+JKpn?2TnNe0Gd>a~<*MN`T@rFp1D8Bau!dO^9`RwV$GEsURYlJIr3XfE^i)
zJm;60qbM2YL1Cux`Zt|tar1_JXRH4YTW=i|Rs4ScN=S-;bR!_$ozfxQ-9ryu5&{y^
zsdR^Q4lUgvokK}?ch^09zVW+v-QQoZSPPh89p2|T&))mBU#)8@$5_<HMow>Yl?Vc?
z)M(}80M*2b$gdDPz6GFzyUrTGHF|9zOLBe?tZ`QbN?mowR%-99VXOfQkXNlqH$wB~
zK7J^d66=|i@C*4@id0!%RfN9XSF;ds!2coMcoGJz#**LaO68@`A9#o=_L^J;YDAgM
z9D0&b?dGd{nZ4&gjqM`*T@P=^XecUyE>;=OMycO~#N9%(KiZ$%p2$T5V{sTdIf9zl
zH22z`dsoa(1>cG1w}fMEo#<r3WmNPT9o+cgwkwH<AQYnnnE~cP^8bl8kwapeP)T7h
zY)0ESqwKwS>w{8%2onUQCPV}&tDR~Y=qh8p4u~wF^?0rs<CV6{Enz_C=B`7ylBDeW
zXT@L+bo@E-({i03lu{zX8$Z&&E_)7k#;7EoeM0R*Z_8jqM2a2^xMLYu(tT}!!|_pl
zac>@Rx;L37paa4O-rV;8igMNWPfoJtl6~=|yVC`}+E+L@^0E&M4zi~q?&w4R<ako_
zjOx0Z#*bbwO<$oVRZ=4TkTEn4sWT)#`uF$V8XW&iIqKEE)CBEYO2{RqZKQ!la_<NO
z>-GZS=UYm3uIT=#fh-^1D7HZH<{YCNvB*#^`IgfNB2xv$9zxJF(39=cL%4d`8A4W~
z33u`NFd0RHiv`}R2$1#FMpEk(o`^2Pv|Ty`I_=Sr4)lPdTVr;U$GvBwSdC_}U{j)#
zJ(E<zyN+QjmEgcO*<oELfEN>d&AKLC`SN%yatcW#J8EldlTtd&sFBcfr`Q}C!Z*Nx
zC)!n_W-C^p;zr^H40cg*b0WKfarS2g{T%-R{C^35eIEWDE7R~lkSqU2Y4Vh)bSEeN
zbd5-a7$$M^A@Ee-F7tfJr+#bYEC6hVY^e1bU;!+4-~QWT$2Wrd%X65G5=v{=py_$x
zV|tt_u$_}mDOLUcV;C11fOVBF`)K~=y@MZM^OgJF`3bjO1beQ{hlmBCs_yL&ptD*^
z*?m`8rz=d)-8UkA3qe>OC-d6<sKJavAYfm%f<ME%*=vN~c4FGCQrf5k4=G>?^_gPT
zGEgV;zJ88<4RI_MFz#5<Dx_%VR{HUd9nn&alPCR0+jten>%Sd{Vo+x{Dk(4xhD15!
zD5Ki(p+-YLtganltVoYBINU-wZF`<j9Ud1hh;RwLfdU*{(X||sz9e%0@3i$vc-5;+
z7!d~KP7;*lcUF{6@!tnT0fM>Om@?-U-&V0UB=?*XXe@0|qnBYUih0?qRP#7TZ#&5t
z{N=KS*&%iWEV+uMqW070M2JuavV8BYMYfH_`b-TOg`@CBmO`na$<`WNi45S^KbzaE
z^t!)x-Xi)>%w;Bzv5i)6s6EgotoM=w)O=y&K3D~WeUfZDq2EX-KE#2Ntx@QjjsSUU
zoBAa>Q?z9T;8M&>(LAEnXY@#E-i{8|jN4%P9~OY3-OEUvqA`^bhBM$!oU+XFDL``L
zE3W~Ejp(F_H@#H0+=o+OEhJzcfNCs8Or5EL!QDYq;!Wohfudu(US8{WY*gp6j8`#G
z#yoYvHfh=G-G;}~rR&AhN~21iKA@e*7_<J~%1<&>RS`JV!%}p-AQp00$E<Lh=EqQ(
z{A0_+dT%}RQ|Oi*8-Q$VI<)iO13*rIoFNBvn(&;w<jE%&4Q`MMG9eDZp$i&^;css^
z`6|jdK=><xEE=Z4<pmkCC$UO5#=Hz_ZuHv2f?~Y%5wpD2Y+!>hW}Ec*vGz|?lXN8U
z_G3#|o`VJ{Ex5j;g*vqrHgaaTKswu6e=FSy6eMYKVZA&w5&^HsCierI10?bVFOfs8
z+()4=MI6$RNeYe<1?d;nG_M%5|H2Ax8ja-_c!<4;UKndD<6XK`{!*e_Dl?o~Z(I-1
ze-6-r?9@9mx<+MkK=Ei-cIB3!T>B2EB3f62BO&|FUGoD)X?V-LKZ)|%6oHXj;Y9-=
zKWZXB;{ueashO6}RO$sE2?M{_oxrB&36t#*@;v#>$bp0b(fF^XQ4=T6Pxo}WE%ej9
zSv6yy)7mU8Wp%foKF5Sz8Nd#3+!NQ6uLJ0%mFcN(k7wA_P<?BK^~OrzL74OF@tAzs
z+MObCcwL4S={<>k1=b;cH-EePK1Dj-_tnZEhh;UrEW#^&bhEgR^5N{q^s0*3tBoC~
zz+!J{G@5t@`+|o?w1;pQt1c*98*UvXRNw0Z5QkU#u)Sm`b9xWw`U&Vghs>a7e;&qj
ziUA9KqWBj~0Tsy#Ja~_MwX8zJ<hL3R$1ky@vzwTGQG><X_GSX*$BT=xZ~}`_Cvyr-
zKP)8_2||sp&eFI0NHqt1dW+)quxOoG4i4`@5ZeiJEp6wc0P>Fib29x;m&xnN!wL^@
zor0uVw1Npac?6>+7GwPY=C(-#Y3?0|i{Kkcy~NG`P_M|_|9eb37qd=T@fnM`ubMN~
z*zcCR25_1#8s&JDSGdTR&k8(Cn>W94lhKy9nL&Y$tw?!c)S+*xeHPv64Rad@PZz+I
z6F}Q7vK|vT1*2h$_EV6hhzC=MgsE#0`zaZtETL41cB+iZZ_-$-miWQL3`ZL2GB$ys
zAA#y??rYGYdI{CUKdYN#@D5h267R16t>j9g;^a#vMfrAa=<-&sfIZ^ANK<v~D^SI~
z$2iIdw1p<wtv?ty>Q&_7!zk2GyOJE~(oFyXAq|6%IuW*d?9!o7ZQj{j<Eh#GL^UF+
zQAxBhh1H2-#+>Ck82^UsT8q(06i?s7pMvyGscE5uhy>SMbaH|{?qGMRx#5}M7gNhp
z`&l1Ri31b`ApJ;oH~X}pmdDX7``j4dr1a{-<%-f$+Pv8PkaUnpaHf1^bs-3G4=7B=
zT4S)msC-`kBH}0!^alT@h;wFRy3wqmcX_x)z!?K6RX;=7L@G`T?J`j79-4}<zE}#V
z-!a0#4G;%nfdGW7;9MlQ2*N}MXlYFq$UXfR^H{2vb?>n}2F{17OMkVW%L1W1k=me%
zq(mYEFZJG5L8mT>4PED~0w1<05y5IKyX3r4r-jP)yM;&$E8qZPoWm%J10GBEM5}u^
zko-jxq7^MY5`VzRqdbjHc#N6^GT<kDmp~E)@~79z0N8Gi)mu+t<0~qaYD_P^+Pmk~
zhuS||DTh;q=uu?CpS2WN;l<#knSdv9j?UH5FW8Xa6&+oXY<NR^_?0>(;j3EDr}hj|
ze$^Qz;gP>=oEr!2QDKLy5@N2U+9)zHq31489ZP4fJ~>36U?;41gMHB?rJ;14b{!3$
z27W?|q&uy}o)A9bFZ-rd#i@*fNBc3|?0*9SiLiRTAW6Rm5@-bAM5S!4a&e5WXkBC<
z0_4H({~hgm`^H~g=liS1FzWYCOd%l^LODI0^;T@eM7NHr`HeS&1}f}X;4A9yc28xK
zNmP9YP)imD2FNV54BgDXtF)<wg*K@I5EKsNHWtj{1^UjFGvGJ3J7btDV6~tNy_K2A
zz+gBPstElgiYzKCjZMfz?kFJHInD3z&2Rky@BY}TEicdwD=snc<SXxA_MZOt<@W*~
zTrkcMRX*2}?+XoJ+4u4*J*7}?bQG%y4yMk4iwvK<ET%9Fc>f970!7>KKihD35$gg!
z`3&1fJUCwNPJA0|Bs%{9LGZFTc>FseWIs#Y41(z)RKysJOXR8qc-j`Nu=dW!%YF|F
z(q;acUj5NW=>y4VVwQe}l^11_lC2K0bJSoD<P+V?Pw})Aqu~inbm4+QZ41chu_h$q
z5k&oFn*DAsW6N%Dq*UyVuykRAmJ@y=1<;b?ieTU*<eI`|8i+tTYhK(%C4IuLinUd;
zvL~@^wciM2hknP-r&aJmRY0Pu9rkMzaIv)HsY%N?w5?7{N1adk^W=M@DB#lO!n(ns
zUGLgQV7W^lP*47eZ;WN&PV`;4G*YV3vGxV-T~tj^-jZ4I)R?!c3WhBJD_cpx*%Q)h
zIFv*bOFC<+V=gz0_~Hv3Y5!laC<c5up8s?$HsQIKIseADXF?otW6<{He)oe_SLi={
z8z9~CO6$ZHf{hE3GYpdTMPZ5{{c&4YBnWji&S|@A71e1c<#Ybcg0`Ng4dD3h0^#H)
z=4F#EKvdY&b?2+~Wp=gBGqU^RIf3_=HOUG10GZ70#st7zg!lOO^vdC?lxq5O3aABT
zkjHSPqm$up092G9s^h{=4^7Y*GI<G=mv-AXzE?_=+plKDR`09aS8q_!qVk-d6_OvL
zsv%COa7C2Gr(R!C&wuaZ`%@#|alOd&OMk+!$VFEE*M}&5;7tx>_i<!Dey%M8WJP4b
zP=!ZKJ6bHX>}eF#-7};r8D#K@h7KQg;TS+O(DuA!loSd0z#M9_HOR)dp!<^JGgoW<
z>)Z?uoo52!o0&H-(%}EZ=E4cNc+bFaGGR(+CouX=8NYQaQO{~%^`m;L6j#FU#Y#OT
zK`1Ze9NZ3`t+27vF`lYFTAb!$MMu9RHokJ4>Qhj!^cC{)qV08<Qq39sa7Hr#Ld#;|
z@Jsok7(;`Jsu#A=A49X0Wp&;!fX)gtv5I{CJkFaJ)|S5|@UnUYkAiagGZCq?xFz6#
zC6r6?)3eomQ7G2}2u+LqDy?!ao%A>y4yZ_fx^b^-c0bNdbNDm=IobkIQ+7i^_uVWy
z1~}6sqBSv)%=>Ba&hxSHWj(b_Ecu{P5sE_v3#0|Jx=8pR4l7hV)w(WbBz#9FK^dHM
zvZiPafe+wXQt-2dc|>HS{4uc?5;%|P+_Own5V%Ih4vJ4#y2k9#2s;jN<J9PO0gH?#
zJiLdD%9yAAr+9r41dfxfs(PE~-AwOeNPK`gSMRk8uymTLq6wMbcsU=c+yz#B==<m`
z^z`OD;HRUp5slfy07gs!ob&JYqR^s@sC5D_SYf$?yAV~vV-kB939MKjw~jiZAO+f%
zo&)`0Ee`D^pd+u@J~16$S(5hPTWY5LfjT^jmxKiQaXa8ealZ?27umc^blUCF4A-a{
zeaR6InFOGKy2@u_g>ZV>!N=p7E1<>)06+~X7>|_|CBr<GLs(Ce@9WR`qJbRw2Egcw
z6#Z$ZnBk-F4NB#+6?qqwy5-li<0I_<F|h&}78CRIV1W!02e0(*>Nw0&$wQsbbyvA3
zbMa(oA2OZY{$i+bI!Zvu+<K{czZ!@h-|}J1Pfw@f6^d~>d5mi;@rFILK$=SfJ;Kuy
zp`B$~!?|bt@sxFOE0Q@KJ%c=PmXu_ZL|sx>n01#k%RENGjqL93Mv5BR!5Te_aau6+
z>Cc{jV2fv2XU)kkR>F>I0{8(;%D*X6F{C4;Q<a<P7QU6iIM@*C3)-v4CsZOdm$s9l
z_f!Khzuv&>`=#9wiDnXy&Y`}GV0r)(Xed_xmU?q{FwY1c-SM0=|0YTb<`n`|eCyBy
z%j0S~zvXJTh@X4So(?kxoy6}5KCrICP2z1z=fIrg#GHRc7!YQCnfL(yY~~#5JBzXd
zL9kmBBHjspPoB0)h{4>k3kS{nx#mJWa1TcX5h-r-d6)zs`j*nG-oJjmF{jZM$XUs=
zc<Ler^Xr_Hd`vMZTFb{XVpl>8HebAE<mQI{OZXO&ORoNSwPWNe(M@S5PvKdn0^j0&
zJX1&%1xh(ic+B{u`kf739}Cm*sO9l|$eM*noQ@VAXnAnq-C%q>qoz8w;S>{1(<v3o
zb!HG2w_M{kWavunn$l(KB$+7ODFXdo2BQO4jbMg0Ko|?uz`cIK|F=GzPlXak+lqE~
za;64HLHkRjEDwD?L$R&uI{As)xgi5M`?4GcmziMgxupydSC<g@IIT8$&JGD~-`YIO
zBDn1ia=1YRwoEoNiRC{R-D<8}rFFA?B+e0)1;|b<x&kt80MZ(gY=p(<Tk&9>Wt(6o
zGw+JLZ1*`()RJ^-H<{Bnhr(=MZ0e`A5HhHN<|06=**@U$4niL69!$GCZ>$^XNfqJi
z6Q-o3EO9Q1z+s0o_lyx<ZA{#Xi3Uamfq?T8IQtl(etg@#iMn(>zq}}cvmPvsG*9TC
z!0Wrf{H0N#e=@x=p%eEcDh!L75U>!T?$h|qRS#iLyXVU$yrbTLAH2BLX5-#Je>+2#
zIx6utu-uwZ)CJ$l;Oa-Ka&P(5_;Y*#=s6i&g@}l_5Znpnu`9dFtI;*`dU^t63U^&j
zUN)6cm9LqdWU)dCF$Gb%!+{4AEuI&|pml@K!T|vr$wEi?mF>?>D(${crer(}mGEv@
zvy7E<Y!K?A_w~Mn_3xi)N#nl)W=4|>uOn%dO-puv{{sdr@oC(LQQ@A_y47R{L2>yM
z;sHEbfrKGNO*feC+}(CPqOwN}P8#{~$PyO7e}h6F;$5Kk6pHG*&Hk=FfSB9CiX`Xe
z%{A4IF88iZz*(h6Y2sl2Z-R$!X6I?)AW4?e6Hf#zEp_5;Tu~(w?=vSmnp(fJ&lcbr
z1T5F`XTm-poPnm<g1k?pcTzV6>F(*yzI604GgVOLYbgvGA|fR7Ix=PvH5rQb^>bAg
zxHmpowhct4k@E8Lnybz^l5!ai(swi5b7w7XJlU<brJI#yw@hPlBa+_*yp_o&W5VoJ
z{jzXwd(Lt7Wo7st_R|re2gWC2DhV;_UmH2oQFBiCZH)hYh7N0J8Q?V3cVB&3m&+>q
zoy~_%^K$CkDoV7r$rj2|g#Ode3;GGmCNfhipOEayP^iEG@2$^+$8k5kRE?&&`O(=Q
zx}7BB)p~@-rt9XJ3`VEJw+2Fqe13DG@hX7%y=n)I!H?*$b4jP?7T9eXTx{I&(Xt4?
zK)QV9EVXtX&<Q(S9C#T&?zkXO7dCF2Q#9{q4IX$rn~(8U+;!I#Ziuh2v_J#D3JEy+
z%_RqP_I#af`{#8~K3Zmz6~<NJ>GmSlAg>KMxV<aP5=uZTXUObl&@8KZC|+@&FO7ux
zV`SBQYaejKZ)qH8AO(OeFSgK_B0#`?L$dtJ5de!yYmAmN4+I@K!03~0#@rfW*eg!l
z{6NJ+s2h_Yn!Qf(FhR#OFc$|`+@(KFp2Hf%3jGac>JC<O<*qZQl#l{C^z<(_gnmZx
z!Ef~oRM*(GzIG!bZh=lTgD>=wMnjnHhY*+o&c{-(BEYQ!G0!A|9;~#&L@aD`G6rqz
zvArALAb~UQekGr8&G-NUElztLX?27#&4=JvQE~SY*x8t`E)86z2Vd<oE+Wp?%w$H{
z^R0z5Z>*^2@OLs@6nnll$eg&@TPGPff0=XBNZn>K1i5t!axxp*2^_q&#>8S0{r}Bz
z@czFgvlT*|FXsgI)NAZ*q07(;UirWW%c)dz(+MVPyg*ld3|dK~Dss9*t5$8WNsjrP
zS30z|%Yr@!c@Pw}6-?SybAa%Ud_@TA*jm)Ac(l+nxQ_A{Ak>Tl?yq9^a@~D=;el==
zl2Bplaeyd!bX!)&xQ>pqpPUdq4^m(dU`i#?-f9Xdx3z{@({no24t&nZQaqu0%%P+c
zgm&ChFz&+m8OLX2C;&F!IwmGq2Wn2bqBzTqd$j4sFB?TI8fB((Hz_G8?o0AzY~@X*
z3luLMlY9(tiz!{efPvs4PRMPGZ!iWO8LIj;Fn$VvevEsywj`-JM-JE+pF1hMnC!7k
zW-BrVyWcRll!B;rF#U|R9&Y_{hJd}5uFls$9KmN9S)iw~TDJc#d$!(yeE2<MuTU>o
zXClut_kB%WBS^QN>RE=UKw>S=?%?21a*%-aB?O%`cZ~&I3%Y_@-FL!xaN`Sjz{7AV
zHR>KS6g{2TDk-#k@8HhKJ8Hu8L~KaCc?bnPs`Q-W`q%h}$@q@kvply60%kPe3lWhb
z@4j3Ytmdx(x6#JW&%C3FVwI|HG*jpd$QTtTLbYFUhap3cruH@S-rD?nWv-*28JViZ
z0`I%sDtQB{QHC<5#-%Uqx`VH#t4t`B3?Suax*H)k=tRF_Q*BXOGzA5_tj8!m>ioLK
zxIz0T?E1a15@63BbO$YVIonvaC8mI<()v>Y{_(&Do19|mC4qtKL~q3g98dyJ?e*k+
zrj6xFw)115MbbqphD+N;-nn{lR@0>6%;)`$T-jOPyZ~t?&uZ8vKsMC$1h#*mt<Kmm
zzHdeZgHCkUPahK5us(JW4X4<C(q8csFW-0t;ZA3kzdtxY7hL#ux0UqP-M||F9KB%K
zuJJ8={cy41?`A)`|Mk&!qE3%zmfLBi+Xd;EQl54=$&$nS&U@3uujO&=j+B(4PKzwk
zZtj;7!ct!P*w@qRSzu4?tuQ~!QJN%AKC~mrw(^D-R`U;Vuei^qq9*v@X!<rk`&Ju0
znX{nf<Q=gGRzip#0(}3IB|tgq84LgxR{9(YzHj|aB|2CLLXFr{OoPNyqbXq<v9B9B
z6A=Ds#o5fp9HmJ%+OM+<JcI;t1_(`;X)~^HbgfaH1-!uk=DVyzD}JH;14tEw$SYAO
zyDM)4FxrNp&C#MS$MU>4_T<HKaMk4R7>g~u@FvW>T7qpU&SkdzZmH<eWE=pqW`J-m
z?y)}2E|PJxo#s#gtThzK5bVz6BP{i<qK#W=MJz7XBUnl-x4o|R>CnkyP9M|?uGKe+
zU`Q9%ikFx=Tt)%l=Ru-`L6a-8mqYMBD_CzNWVBQL&Y*~Kt4|L<zOm=a6I=xcRJ`<!
zb)S;*S&kBSdBe6O!tz|%=i5{v3_N&*7{mIudAY%7b`Ci%=|1yVUfbIIjLZ4C)=Ka+
zD?0Sz7<mIwu7L}+EZTUw#?!zAt?DCnK*QKCMgFi=CLB^Q1~&U6hsS{NCph)EkHV*O
zf2mjmaF-*=Oj^!@L(hfgylGpCLl6d>=Zz(-T-!k0_1Not%d3B0)LLuS4G@>Xt9A(0
z^QH4?a#+`!CHEs-Ci;VNc5S*}JQdBmQPWMgqoJUYAY?LUf{Q1f%Zz{X@67aLL2tB3
zSS?z>8xL<L7iz7=<w-w!!1g^9y+8HJ0_NvN%^-wtJDoT5TWI}WFM55MMWpL4n6ua(
z!9Ni9MFSC39Ijcx*PP0z6`%0Iji&&Auk^<KH-p)&<x?LT=VN}-v|z5UF$#eef}g<D
zIidlXnOtml5GnvwR5P?xU!wmdO6JQe>Y3P8hq}?F&+Z>u@#k7swihmw3+bQMYFHoB
zDnCjL9e;<*q)$uY)vf;p&?f3y-P(I^avtrYuj?MZPI^>`NCf<&tt1fq@3T4GFzzhE
z9oD=5_l)_!L#NC>mvX5g!A4zWNuTYV-Tppd9|4}aj1A)^iPEx@Ia+))7lutvujipT
zTGhC!Ahb~sE}0a3`|B6Z`Z}bqdK|~57gh5{to!SVZ`R2&c4<GM$J+@Qsh0IM2m+|)
zC{{<{?>Yow2ea%`iSbp2Fys6M#^zGMRF)^T7YoB9WYD-eK3I65R8DlKvU;72mJkYx
zsv-6G>7U<~ZCr;g=TxQt{@`Mi-SFCEjfp`{uWFd%mNIq%nB=qXjpLLMYyX<%X1kid
zon)&3V4v{$%~y!vFBym>={oqhSO3oYJ|HqU+iRX&daQb81B|z}2EF9Zm3j@(!^9u?
zXvM9ds|CpOA(gju%<w-_K}Xt;eF%cubSYyQA}_{tsvGcQ2JRF)Yv`x$siRodLnt5o
z!#y2_li4x{rAwM=CxN>~(Wv>BWLKArcpZ3$IWR%}-7~&zbleUZlx}U_<`I!h+ex&W
zCt2UL1N5a9iuXheFh5A(GS)<m<<}U$2ZQmT4p+AjIJrM-=kzgL?iS*nnJ7s-Ivas4
zHK&g&?u&LNXEH%-)#u{lO_5L#+VPavI$Qo*&oW41#@7~)GJkVb9QC)JQh>&xf7a8=
zcv7_#-Z@Bx!0<cS{O&8No*&d>!>|O7p|<!Iy-#n4hGrNbPmPuiVeixndz-OBxchf{
zDgjEr7omD4lU}`q4-@jacDDx5Of;kt)9kR)CVD~k7Zla^e(XcpBVLtYDGiLCJ=yYA
zaxI}7nQN;TD(dQ~APDjEOY-oq%Fm1EY?iKG)uKX;?`x`m3UO;n3~L!S6gI@zu-N)<
zS*|`IKy}?nh3ii<1D{!Q7Kn3aGVR?j3Pup1C}jc<6PeR0{AMI)nU?Kl<cb9HyS^>`
z<X1Guf0RFR^)_ZZl$`y|z4a_gsjn+I8*ZeRngaETN2Q~Nc{5Wj{JV8KQmS?GJVF-~
z>By5<HCghNMe|Cm0bPk_;5hAMe3klE5U8~$>Bm?dQJO)!sg(8Y{7cV{<N#cJ19*mP
zi4IKEq5=Zt?}(5!J`veBeza{5Y$-n|akHKUkq}tnJC8fMu(==N8|b}pH~5iZdVDzI
zewmiHx+S3B8mdKV1G~h^fLtyB@7IzhSyDqgsab*K0W}F^4nFFnHd+yXvwjI^poH==
zC<4LCW5tSh%?7QlSVd&Z>bV30%iy^Ic)VJA<mI^u_YvG@d>S+7wFPBT1>&dhK2X%T
z*S=L28s?93vZIBQg=qIfUsD>+Wqv5BTlW>oO7daub+m@dz<uDilkv{)N%-6y7?82p
zy1C3M^`#~l;BB$h+s_kjZdokUl2Jm@Q~VsC-b=Wu@yn8=#i8O3^HBISR|!oAMH*?~
z=vL-EyUf+d@}|o<e^Xh`dpb3yBn7WOJQCd7AGzyG?1+Cu)RhRNO~AwH)Pz?2U!Gf6
zxs)pe32s`o`7E4AF3VE;>TWbS2jkHyj81U{Ur!&I-(0KI@{>-6_d4-nax72u$1U8D
zz7NU$f!cZIyE#3C^^^I%dr-tEK`oxk#d`DkyJzEk;mYLs9>P+}GYV*O1}9Oc13M;o
zj)LNbcl^d_qkoZ?#h|IDkBl!HgXoC3Ot+4HH*o3`>A!x5h1v<5*+Z>bGX2vvbd<t0
z9y`^ok;xYD(c#d%WtE0UAk&DX(ECWdpbxYNl-cz4?v^5SpRpqVF#BG?J6fucf(=2|
z!1<5$#wjdHRs{8%fN&?yI_F%_nHh}2h^MR62)e!nwE*@m*?C;MU3XXEoZDf|4IeOz
zzHmzqgCtx{V6;)q&gk>v!*%rNaj0^$=YHVtq|ZGZEP9&i;9)+ZTO{BXGnIWeA-iXI
zZ~Ai!c7sgF%T;W|P85{F@wFneCz}aL>@Z3+N)l6s7bh=hJ)oinuqpMOZw-}Wmo{>}
z0BO(Ki@nFQ<?1>eaQMhwM`aS2@%{z~f$<{dVk323>22iB&QP~01`nl`ohh24oyxnK
zMp4A+T5{bVglU6D1i8sMgVH!?s0SVnD&uhYU4<Hz_Kho)yF1)u3PbE)?hjZV?k@pA
z5x(r*TO7wVAjQ(f+@Ba6wdC!drr|<(9B~+@oFFLxKt+Qycxt28c`X<^q^_4+OWJqI
z^fc;l<mL!V`QC#!ZNKpEw=tNLsaqwy_2gPO`|*KFG(4*QkgcB>RK<f=vl4v`i4dk$
z%5dyLR+$ZPpTDo8*e=;0PK|mMz3Zak5U=(4-gO_%GmDJ=<*16Lh>Box)^kf?+W3lG
z`AKkAf_~={+E{{?p=ZATL%GX0FN{OXe&Z9%%q)i1OVRW2Zv|E0*I6vFz0nSJk%um+
z<$y3K74@NzFpD{wC1-mNq7gZqC{`5b^2u{7VLer#z2d2N#!q<h$G=FSPp6=?4LZf6
z)c#wy-$j&y2Z52K3c{ytK{v!ChfRkWtJp0MlH$*w&j%=@RseGLj|+|b(#_0D%M{AZ
zGJbLg{~kh(932MV5g|*o?Y8Jyy$nh~DWJl@he0XkY{aR+IDtD<QNzq~eL5teq1X=B
z&J)O~xA<Z}s&d^{_Y0#ZHGv$OOcD@@&%XNaAF(&Qd)5(t9q5tY+r8(jLzxl=IDPBL
z$Gg8wuAwq5G*LT{o3-!-;EcOUMwicr*3cb_YmP1lQjF$Po01DZ=Tbv?bwfl014dP@
z<G}<8j+d&hZ)}E+HS)*JMZkVf^IDZj_u<bb!6u87i}(AW6ZyX36pkb8Z-ER&zSkak
zkuSSVm$+8~-GB}?-OO15!@qj3s{BMwoZ6Q<{ERS5#yjcV_#V<w6ne$Kqjn{U2PHuz
z2LZTJ-kX0elu*5K?Vnt&yvGxe|I!e4lpAgi#v3Vg56(!eFwt_=4b|cGbejLa8?paP
zEB|IiV&oX`SISB;IXbt0#FXIA*o12+=vW>f$U9={hkK(1q5MqmH@w#*emz?lDq1*n
zicaDk9u&)*kfxU4&tyC22jLjJ?akutbQT#^zGYSy9N{peX@kDpY>1X2SKnJ+LG2g~
z9}Ly+N;OLR$gZ7_lt}02z2j$tfWAHfsC1?!!+8$bXEM*txT4K8py3alQ4Z|Wao&n2
z_+6MWe<RO>93GeIbCuLVK2V&4%dLXwwg#uUGn!%W5QzSjANVu&bH&UT3Uc)Uyzf8T
z6QCa&jTr(aBtqj~J&MNnAhvMO*{^%ddU`bue0+R;K%*vFte~ki4XL?Z4A(~?bpCfD
z7BLL5_R$kCU@=Va_xVPr4dSWH!q2JQHYM|xE{<#1Nt7n`^k#^CS++f?FmMQ`oCwr-
zgO7ke!hM|ZXd*8p<1G~<4XO56BktB}s-;E^sQ<m5%5q8r?X}6s6oI2bc)bDb`Qg7m
z-8zC4$fH1}bezYI(X)qe3+G*^wo4juPihA?ZlX5UUyuU8?1BA$y4)8<l&1@VfpuN|
zh6*fT!MMvmUEi!76c=VrhV;dL6c1Pk3c(SQPOj-_<sUn}(BWKgDbD`Zs92JnJ;32e
zB9;O)LCla+y07w8W;eMu*-6fpe7w}nzxZPp*fQkgi4nYXE*aho_{O5zfryQbfy1zO
z;Pp8ox=Gx=2%!uYnN^IjCRcX|I=bF_EQikB{lBL`7Jvu5CLbrGN&ZMbX5Mtr((92O
zs;!)rrx23AHD5#GQYK(ucL!vPs3K|Ft@^3%WRt%21E+%;0#*|Xxa6`7=)=a|$OW$}
z6?xjoH^q5IV%Ti{qZisqTyZ&?ty?+q#nJLPE%6v)YM_N9QbbSlmzFu0kEcA9M+H$E
z-lIr*dmQw*n5<?X)RiJi&kO^$dHdD*MY&expgsYlP?XUo<&S1R$*Ve)5oIpu9bL%3
z0Zy7dlSQXr28(0cMLoyWt)Skw(I5L+tcc0K!AE{d!O08m4tdWIAiT_>5@OBT-$Pd2
zFLRX&|8^RO8otTOe1PM-_iz3Pijnw;G|<1QWzM!~AhQCuLd_bJbKA{24CoJp33%$U
zzYbT~eFdUuG(YKicKxlPTyp|SJm^5AqNG1z(ePs-fyS7BYr-mw{&JO9dwPcCu@jqn
z=hqKP$_;)WCed~<Rhsd?BH6ute)_Yo-{txH0dE9XsxDrd(I5ftg@D6i4uMKO(4G4E
zH36e;*RjF9d{R?AH=gdY-pG``aq9IJ_WgZvOKUEA`U!B8iY)Q{#^P^DI>7$DeaWqV
zRFHbwc+><tpIq>@<pR(4Xi$A;I*ub8)H<MuvM<h_kGCV##0pRJ5!!R($$7c7aMVZ(
z{Te=l<UsYm5vc!YiT_1?bjGlPmM{w48RC?z6v@j6E>wSx3D*Cm;KwA0ttO<Z{h$3-
zJEpOT>Kwgv+k^dF?KnKHZm`sRf7zksNm4xt?kSev__|3=dio1`<1s(ss|d;ZO2L|L
z)^ebb+2%b(6kMpcOY<~-0$h2?w|No#3(a3y&JcfRyAs<jCQ8=XKsB!zb!$VD593&r
zej`ys$6IffOMhq6Jw5=B<(PVn(ds=egnjkqLz=Q{5W+{6AHaT-<4l@;%ne5i=J~t)
zD=<hGF`1*@OXd+0L+TYeh8{?HE39Whqq!N=6af&)DX4pj9$Q+ho|@{Wb9>o(f}%p*
zPWyRHD+x781D##2)wV7Nd%`m@hyHvz?`Jc0V4N=Z{z?k#W%88T+#`GEy5e(#$?&Ne
zho*j1Y05WU3N6|yd-9Hw0)Xdw_9U+AUvNNzZ$T-tdX2^(>x6Rbls`YyN<N?y|8{P9
zTX3}4KtsI>zvV=F+Ry2A>W`>l*mEJxX|vqI2ibD9<W_okT+3o^e{}W&`h@6!@|YoZ
zd=&KK*dM5VMQRr$asrwBTWlPDtCY{=5|2#LV*<ww{TRCM`)`T3YyhEkJ&9yz6-eG1
zXZgLF!E7SSg%_gE`^|w(Y1ZL}m3y+bxH!Uvr(GvEQ2f-iUs9S&mx|%39hQd6s4Y^q
zs>CatEFxXj@nM=rbGq<LmVzqi9}o}#&TU~@5=7nxBcCe`pzVU8@~T<TVDVf7R~(+f
zTlqk<ys>iouEBH2f{Xhf7AgvF2${z!jA3(AW2ajk_4V_bZhRQm<<iZ{FZSY&5xeD<
z6hcncO1p};KUsga)+Eom9O$|O4mvD`JDNk80;Md)Lbu~Gn%L1}FK(YP7j<R8xWco>
zhYEA?a<}tqH#dUWnF)Y!+s+%MTP(lOjw#=8YWg~jR8p>N?Hv00Jme&kIKG&e(?0-|
z5uiN)%Nt)sWw9Cn3v~Su%&ge2pKR6RF=da+Fj!h7-Sj3Zj3(mAtY}<Ed3f%YrKkm(
zOavS#4q5ra=JFz!aISB~%0$Ef(ivo|{gWK-lvDrNA7XQ3Q#7OQ+b4-~jf=<j#@Swz
zENRy|U`<-<x~FRDTo&H-cyqE6v+VcclksVv2@7ox5LZ=Z<HD_R0=EfChUSyxSXypj
z*}e>3rQnWnS|tfJIoe;`CH-aX8T?`L(2*+OE>N~B0bm2s28T9HOqIfk`^OF`Z&Q4X
zpDo=r5!5bJAf}Q;FXcLVcl!KMv-j26_b4vk-J-pRB=OGqwzOZGn1=-zf-o}^fPeI`
z$nQq%80}{4VUY=!OoFi1IBL8980eK>ni*6__#r-iNV6!%<{X8j*_bbXb7uTV-X|%n
z1I8<I6nH>ske*PVH1~fO)~YfEY`ElNam2Wh`8byyPWzX&O$o7xm{i%_2mYkIQIs!H
z9xJew&l|s5_av%DA2>dJ9FzL63q(7q3R|-E|2UWU4%Cj@=fYpp${)rhoel#l3HKjY
zz@Tt+ELxwp<q7oPR&U5nj-nXCDFA4y6YOLGZdhaV7@@q$zcg88Z4p%vyiYsfZfFY$
z{nmx5`LaEv7NM=9AAUhY|Ekzz)bN=3vUEq-nIfc1(`2~*>>`_mp)N1F8rZ}SH$Q@4
z@|m(o1l`zBcVe~$R#{RI1!+ThN567kO7g~>G6kUJ(v{8MCK~va324XMUUwsm#_K1b
zX%umP@FD5rKVE0dLb|a^hbz3e3A(@9$3c1vT}H&B#!gmZ$A0fGb28r~#lnx#6gkV(
zV;JWB_J*JEFAD&%{6B~5e^m>@`Im61?yh0(O+OExM{A!nBckJT9zt6$kNP{qt(U<+
zKm6xAZbL+<)_&sUs8S#?J&E_Wr(#12;IP1q`E2zaM?{d|a%;7d&#ixK)Ht;Y6_>vC
zq*_VAvcLbVF4EW#pYS6l5Vktxjd{nzF&w1t@D;SWCjdbpi$7KAn~YX(mE$&zQkBT+
z3E$OB?htc{O+JMXtYP@2+f7&HsP@!XV>cpWfNPsRO#n9>dF4ZNyi*lKvm{x@&4$EJ
zqGyk{09!?is>GxI{g;=g6F<A$7y~5K2qIX5?O-DNsys+{B0Y{jf{$4J)8j<z&fSi*
zx>v9>YbXkm&SG^DbEUVX@sNus+5D*}8A`Q1Oc2wzX;t_Sqi+NswcB}l9xnGl+}2S>
z4Ke7Khim?G3kA_X?btqkk2!3tScNOkq;>&6SNBTV41lK?*53W*ik($oE$NFS5x3o|
zPL|SXw8+s8h*>G^#u%<@qmfU~uDHM5)dzMxRcNB3-q(locLRJTsJH1Ii4;K3nTcrK
zXP{f>wNnbQ_Xp|f!Wlr7FTOG<AgFZ%<8s+d(;fncqSH@qZU<A-C`rP1<-$5KKstZL
z%~8l3*q&p#^O9RvYnU|q9s1B&MIQ#mlF&d9zWlm<CW+RD`+-LrC0Z<w7;7e)h=!LA
z?)7}K9;G7T`M|mFuNq4oZu5Ai4^Jwopvmk)Z^`@DVf3>I!P{^v!%FZd|2hQy5L!Js
zj)^8RDVG!Y?aA|a-V*0yukG0C6Cw`Gr-f?;(-ADu`J8dYts3R(Y!0l<vHQmZqx}?-
zemsfEnY%7KfUYOjc)Yq$yrZ|%d*)7EUMqOrn7!jcO9RQbrM|k{A-13sl=FAQ{!)x0
z`(1fJEG~k%O{Uyn3AY|N<#8ZyO>utgaMybr$y=OHI&u2+y{jv5?aCBt-UDFps<RR#
zT60VlAIZz-QNs1(?x@A(FGp2^h+?l%DZ=pTeduVAZo~fj)bWj0*bf(e#UGi!Z3;%_
z;<D)vQHE<KNL8+}&{?Mq>?bp@7R3l9gY`9d*q9YB*1@3tgQQNUmDn^>YTjB(aJfIs
zr(WrV#kCqm#I1V5%~WjC%Zc~~!KdY;T&scxgHd!7wA{K?XDuqVJ@9Z|TJCF_lotO?
zi4@wT{07RtP}-F6S~U_EeSBWczQ{3-2bNK78k)7UXV7R}sY>fKN{glJ0y-P=c1@JU
z`~uG46oH$3p5;J<`^&8*I@1_;lxc=xGylwX^@8>cOq-`)n>A6jszJ1H)D&DfA*Y`f
zcliQ1@u=$xy56=I>%tq;w$Nfg0#zy%DLhXe&%Emm-fhTew&o)GD#F`rk5)38Oum{n
z`uUlet(V?gqE~%iyCzju)Vfyq>kg?dIN_*|SCfe>*6)zdHMwoMT6|_E^vuDireuNV
zjN;y{-+rM(sujcQ6j+QifUY8TvjRi*7RxQ>sS_r5HwkIM*My5Hi-7*FUvubr1y&(M
z0+mn)x)bwC2%DuyB<Im<{l+TMQlY!wDYR?08uC>m+jvB%qf1zUjHIG1xUZ5c3@zkW
zbIVoA^lQoO_|#a`3PcUrFZwCx8+pKEWvHH(S&a5*R$#eNaw=6yl$UB~J~Rp5*9IO)
zH{OQwt3yqyP(UevmZ%6w)$%`(XeC7HIyHMew}TknNjSqCeWJ^{xsq=w`-9W<Q+u^3
zvWW);;I`km_+O7#%o|O|iXC_yTU<OQUB=4X>{=ml167_U;gOe_<$Bk-a5+OBOExJ<
zv(7(_{%<m?$oDNKPAjWVO|HhN?CwDG15ImlmZ6s;3kkNb0wb%e4&bsoMfR8J`otoZ
zFY!V59zKd?=DYTkzYx?4(vk%{u0aaEvK$xcGy{5^6#`7JGlmwqtx<~=a-Gdlly;sF
zbhrMIoci@a9^|6Cv{9itWAk?lf)L(7#9danAP+xr0JAZUs8%3vA=!U%8Jo9Ip<DNp
z-6UlUGn)~WVU>KY;~|_wq?06Z%p^?ZZRwMqL+ea^z{&858yWmt50(H50Y++y-H%;A
zKQm4qeE%$;#hpROW&b<Z*~Mj7tX|&`V+}s>!QzR1sB6Sna>HFTJX0}-yNW8zH)uWk
z;q=jEZ0cnxA14cqzk_Tz+TNG5GuM#OWVyxDwF{Z1i2m|6XZT&$+m?qO({h$X-!Jn(
zu@T{C855g)-OM~=_@C{flcAWDR;wNUcc?fRd4f>w%mvn$ji19H1(<HNzo(h37Q<G{
zN-Be6vS9+L0ll$>K!b8Pxq6KSS{vz_OEtl&(@xVtC90~~>{8%5KlbPuCZ{6~Wz%*l
z7|Bqo>n)8Atf#RCwpsg6Y)**O#!qZg@SG~ueZ<|#eu_58uy@UEq(!rFjJoU-nVb5m
zZ;RiHuOzkGg>%5w)xVlCm4p)Ay}-q)3gI|G=qjJ-jc_xa0v$@qlbFM*4}uwzWszP3
zvz<i@pI7U_*~{q@x+_qS%2z`@{9Ggd`wsx;6W?Uo=5-Pt8y;52_0BtDZBghvh$nz8
zks-hDjL@d_N7gi3@c_XaxD$Be)xt3_j~Hc_ok&sC%*~_D!9Y;QVEWm@s9E{8A1%!+
zXwij=U=#C2a|u{ib%{TQzgGS9!%_USi9LC(V%+>l&O_1Dj8+j+<)vRv&+x^muCag3
zuuxo_+GAtstsVz{ZB|0CNY5Q$6LPIju0O56SSYRBGjTH5z7@XXgb3aBa$3>poBp@O
zt)K_ipbH^UlkUf8GkXFum$iDUW=dn^`=Hb;YO-vetbv@mp&!avc(M7DsuACfW}j72
zv)}1sgu=jJko)pom^B9zIz$&%Un!)Z3Z}@n77_Ww=dAdBgeke{ea9_jG7gcKxIK%+
zLY_u}J6SPT-Q5$O9d{Xm7Tqs9_z`EQp~K@}JRPtm6~FW<(H}`r+sX?u&4KPW<g}(v
zaSqh)LkeptV)f<2e&xu}E3~B+Pb!)O4DMts6xp0b2VaLUKBp~f*NNu)K_O`V(;#l}
z_$XXR@!hnMI-v{?S;k~4YRZXxXv<9u4yW@Kb;7g8&PK5T+OIc(O%0o*+0d28l1bnu
z@CCHEL4-bd9vtG<-?hbQab%J3ECiY;k!|AA4TQ2<Z)~1y7u9B>*-RAuirI4R2tajt
zc*$&tq%K=2ne;n${=j9a8lq!|@+;_CwUEqL&AK(^9s7C@ZaM<z@HyNzbKPGtmr?!{
zcKS0OxW1LuorF*F%KPj-ne^V0d0w0?b}n_5zKu_%JB+!-xYj#26(mKXk{HRutPZoJ
z`)nY8{@s<|FhaB2R?E_r!#6K-mQ1;1OzR_ALW()5^3IbN!VzDn@g7^06I$6Wki2p=
z)Jn_vUhqa9m#90+_!}rSAc9&sZ;<$2Qm;bjz2$0xFJ*@0iBM~OMwL*LdupZg9d<qG
zHQvyLluC}ZEmdb!q_;}sKdnqe?fK&pw2aY}=md_YwtWxNn%{RkxMTg$Z4G_&>kKLd
z6UYT2M*qzgh_UavlaEySYMB{mKL3F2Z$;8jI;GqzEq_VKd!nRKPxV5zF2w)gzYKR#
z|7*A-sMNb)ywa6D|KDD@|1-?SnjyJ<ZV*KC<;tMfQc;Z-yYc<&5VD&CuAu{C$wmLy
zazQx~N#X;uCno);-0^CBle#obQn2D)NB_1D=peDYHm&^g;>R-eC}H6$Rg(DI>bs2~
z&B=zLv*<5ndv=lp^3lcOiwzE6Ey8S!yvszldk-N9i7iX8jCCuXI~gTL{3K6-vvI*_
zYfkX%UZH1d@pvCDjSypnj)7_d-Nx}p<D8AWC7>i)r&))7UJ#l+t6UJ|=EuP+Ee<bv
z1qgUL))RKH;Vrr*oXRwUc^xjWEp^{{-PXfsCV?NnQ&r@!-J9O6hZe8-grYcfxNVt-
zjRJr+9hAkSzr(U)Cl<=5s2by~5`a({P{DFLz=)X(Y<DLCA72q5nsT}gs{bnM?$(1i
zza<%gzAVAUFKMx}(J^ru|7Y1&=m)Hu?dOGhJBND1Xwt2GIX7J3!)Jg&G>gVTB9kzF
z_=i-sQM)g*1oFkdU4UNUFTv6foY0^85q+La5)ecS;~SUGsloq%M0=?qUG@ukxagX2
zTsMHjp}XblP2T_z%9s@&l_e0_Ffs)d`S&E+602bS-Grw$Uh4t&IH|29R4W?7xGb)|
zjXLBapvlrh`Xs0)grM?SpAAORGW)x{%IHG119vE6A?j7zWMC^oEE&tu^kO9aQ}`JO
zNZMxc+G7;gLu%5en;Tfo?-ilC!3T8itjoX_b8^DBEWA}+ZAH%1|G)<5HNja}IoNq*
z(Qx=az&o6-18XTQxnF35G&{zihcjD4EN#Dmcw3Prh6l#sL>?m68sR3GZ0AWyI0Z!`
zlOKbcBhyI4iWKNOSkuuAiHv*$>7!zap7^rQ2iwzTC5ZNlIdI*}Cr2cRAA&l1U+w2-
zRyt-N=oA{Yx20Q{E&`>(SHsc4j!$W9ApT#q0F;&iz2#w5XYx)z<{;ZwX^mvmn#D_9
z@)wPgCKcEX<2-Pu3}9K$L`yD_@8U?5)uRqXlgEl6!3*obf3*#jtbQNuCVVbOy<yng
zLyZDUKArBTL4Rc_rcjr92E;OM_usSv$qba2<rA#8;(H<Fu}Da-VrvjYh73RniNEPd
zO|h)sxIure82ueV=)eIfU>?%Hy|UN<3=eO6QV|Y|16Q<$XH&4L`-jP6%|yyl`piCX
ztojUO7rwwkEXu>#!??IS$yPw2?_Ip$m?PdO(HdnTz{z?N>q*tU+)i@6H&p=h*C@KR
z5IX5xKX)05r^-L;BnbUzZTL0{kDvCt3xdvMh(iu#-78RIYQbBd2VRmm(ES-BP`?eh
z)6tGr9{TC2p|<L7O2_{a6)Oy9Y?E9%1>{OA4MMqW8tLTA?jpjt^wj51lq^jZ>SS#=
zZ%e8yR$!>1!<yDY2eSn#NAiD?!_-mGQeS8kYD2k%-T2~kf7g5@EM3O&zESBX+b*eY
zjVY6(RJ1bLh+%t+2_Ch#`~c=9eMvlAmNqCDQ8QOCMf>H<>SHs2U?AiTQ7F@yD;M&{
zG2JMHK8}ILgcVIkF8X=YZqslu4A#k^j7Jl3HjB$Xs&f}Z!XJB5E&d^1wBjBaJr0Q>
zLJEEX;p-hgFX($}D5I)RTfVh8q1T~`4WrjH&>`5SP&PjRpLA~SY0vcYL(8CkFG%dR
zeLHHe6IVf9N9e^;A&ZP0s_&SJ4B>h)vK>Uh05=j3<{8c~&pMX`YS&~g)=|sYeb!~^
zA!IqRv<dk3>_iFV+4?MF#Y-Z9*|o-N!hhVn#eU?VXjb+AI=fOyWUEY7U-mTb#ZTng
z%D1lkVTvLB!<>dY%(-;mr=EiSpZ<&N*#ACF|At83Am45!rMM2#k?7Rg`6xM#S@i3!
zdkgSH*W4T;tF}y~&?u(M_}pJY4w%b;X*P_j$_8y6Ccs5-)Mu%ogPS!Mao+hB=qksN
zIj<@V<7z^(DIe5qsHv%OQ&+WB`g~`W%F$o@ynT2|nz`|6cN#+<y$!g|;Y^E5vGy_i
zm_PmC_j2PGZ3^=^b3-eQ;+~U@eEtSoa|1KGL-5<JqmAbw<kBFtpOChu`{5lnHzc|p
z5F?73Pl4ig4D1I?)~<i)#9WEf)9PuCfwjPTBE`@<syw$Dp4N^33n%%F$e!oZ{T1_{
zR!Qp^E})h?DZ$b~4Z6}gL;fcM1%~!RVA0zjpq+WxT-?#WT^@KS9%XD`Vh)RU20m9a
zl3c$jtPx3hR<}BcKhkv>a-ytAPVyy5Iof%gwB9h9yCl6M-b(k|RPKkqak(X&0(@)v
zK_`X_kw1QVb(98n_M|caO=!8ke~~TIor0TYMT$u^jZQ!;=4VYIxBjo{^X}!8$=1}n
z%up^Y;cBvF0C^=rjvs>hS}?n5_D^~kqbc`1DV+@zOyTyOHP?m5VL2d?#VAv_2Iry+
zm4!B!{eAh?msbpDr=GDD{?KwYb33qlB!Jl?hXhc?7r{v2#rn+5UB3BK)3_5u$Tg_e
zAX;`ehNU#Ok;Q5P>t?m$Psn4k^(!{}Y!<6xkxane_FrgRWB=-(M!%Qmf3_&*7Vx25
zK%Jvd>(#bii2%95&Cw!{4J>D<7Kf!+*wF80kz%VC|EJolFWfUFov9E45m@`#3KAMn
ztpuLFBaBG~<?h+yYVll0YR!fASx(y7M*kE}%k{qiYN(ZxuZQBh9i8#=mL6o-u%Q20
zD+Wbfe)=xbJ4iQ;6_nb%E>mRke&n4;(Vf!!ay`1B)HnA)p!PNfm`;6Pys5cRU_X(;
zOo0>xz}W~hW-(mm@%@AVL^aK7@P`3Y4A*@|a2S^n0v6a9_swM}wlBI@@E5Kz-JaBm
zqv$-+E73Bx?i@jCC;<EB|K6WUcL?c=FHdWaVKD$$acaSjFy3W+5kk<_85F7zAKMoC
z0nkXXT_b5c$H-4K!<L^?+N9vm8a=5t3v|dwf&)FNbnj!hGUh;Nv!%-ujlte8+5}={
z#ksshMLspaYcv=FDWJ$w1bqN2W^>O>YP1eO>>ZdhE?cj)yL(c9>84(#E?Jn47`-<g
z`8%D9;+-M7+BK8{^jtx<@oU=2e>pw?=W5M@%CABBXp<^(E!|pg2jFl3EJIhh9!3D;
z_ohO=OojB@NbpWgf9f&tTE^<_3r-uFBmtXW)7Rg+#asAqU7r1xMtcaaWdj{S+i{nh
zPV&*6+T|FBkOC}V4Nz)<!K(;j;avcYl*^ZS?rL8wMrQgI_NPV(38;BqkLK;X5WQxD
zF~P^CF<dufaHdg5@xek+;40A*DHByO?o49|6oj%uf`>{uM+0pYO{31WL2F}QE+-H^
zLa}*(;L-Qh*NARk50zp2>v)*<n#Taf!voH1UHSj(c$ki~nl}f>qE*0*2mkLqo&{A1
zI>F^+;KS}z+yknb{@{NWqZl0jonJdQned@E)*<d8MC$zbMr(zg{EG`3ATVLaN|&&P
zQE}47aJ71+{dB2lRm^A_r0}w83y6m6Aq-3qOA5Nu#v+IE-dmq62$b;c#61@|zGjyI
zjG}?^SN$pAva&e(y9+@G)T?$yt`jUUALq?sED56!MSe>RUpA4J3^dwh8c%$~(1V*D
zlHaIw=B2g(h#9tZEo4u?qVsL-2fgb0;rG3lJj)*474Lv9h>1U&Kjj1&?kvpAXb;Y$
zNu7STK9`ed2nX}|Eepbu+Gl#Ks#wlfo5M?5kr;Yz;h2jWlJ!@&CgF^%A#ap7ELY=b
zf{K*}`|$jW)e6}=%ZPY##|TvoZ5^2bT{4vG+tN-iFLQMG{ExjXGIPgcfN#1$QBCk|
z#j0G!*>{?xlK-^x834QF8K39ak2pC^t3Yu{l~&QrfOh(iba&`$kZpnMzI?4o06V~h
zj(>L{D?dU0{qA76jt(~Se^>yd(w7%WG^Qic@?{V)(H=rGfvagXYe8>RYY(cGG~S^9
zhp?{zt8&}AmJl{w0)jMxfOJYpcb7C0(%qfXf`A}McS;Bnf^>sQr*wCh^uM-s?)|^}
zefK#$#&bCPZ1y|XTw~5T<`|!EDbPrw1s!0{bG2(qiHu5<0{F&se9l+$TzNfv2xUdh
zWx#xvE-=G}c>eA(a9cz5xPc$E1Pg>bGOgD3KG3eJ&~-qT5J=-l4_`0|c@`@v9`q;M
zHWcz%`w!gPHvWPzTuLG;6s<R}Mm?K@5gj4)8g}6!7nk~p5b$XxfPF!RsXCozyxmu~
z9YMtX28nk((EIjIA;DU}Fiupg#*#9X1h-v<>khG+D5*axd>O9|Od88V`C5zwiPU=4
z`j~4N4!rR15}LZw*FmQqGC$RFuAZ+ffN5dTjNEWb9>vW*+{g}S#LuY$32(9D3Gez}
zL4jKT^N;NyJ_Us!^Ch*#hJ^39Zw(bzB#cRN(K)RqB!Tams_mE-^~*pJH<mR_Ej)X&
z31(mMyCYllA!crMLNo<d1dQ+Zj%oa_>@5>6<3i-^2fW9u@kUG|x`>oAiZx@;tJjE_
zI+xqqWHi@^GH+VrMvV@&o+TU&F}gABi!7rKR==tclui)8WERf0XI#@LD<C~6c#xUl
zBJ#y>R>{>&vpvJxG@bIT8gAS``inY={QP{C;q2F0L5zp>*gX7VPm7`B2)5scG$nmC
zajPn486W<S($U{lda^LAbzgO{y3@39Rcg*PFrj;dxV|xUR?*??UM0P_LZ*uk{|5Va
z&+G-Z7T$`eHy9*hF{B_~WQonke<viR%T`W-L5{Y;+gk`T8pseTLZ;~+EzwQ@hK%Wg
zZ<Tg|kWA=n-d&Bp)U4tXk%eyz$D5<Co&uu!+ZU&t1E!(~9YWothYRyC>HK4=@3lrh
zzR8{X6wZSVpAGil?{av8jcV?CB6un>K8>nP+upBV72W-OJ*)Yi`rdf)?A0H|y;+6o
z;Trd@D|pg90CFtxdGgmuyR~a1VmHi2uWs<$ncWTtEPE|60M|9A8)HAsk7e^DBo|SE
z&~?TcUx;9wK`;{sZ^*g(+qX<`WXhe}B6*~%b0IK0-giZwZH4+W*8QNEa%fVgA@SX|
zCp;X${CuBJ_ERkPHhqOkAY}dmGITGl8)&J2GCtb$XIww;x^~rm7Oz|slFzv_n3-V{
zpmD^{=cjqr1U6lif!YNM9rcF2=sehdpqNH+4n6fjc@q)qgbB9TtyRnPbYnCH{rvND
z20nP4SctLSUAUVz1g41mr)Hz-#=Ku&y#5-2+Ne;5!eBybYGN`}r1tpye!B7Gd-B+_
z!Mq0tuz_gN8qOL(ADP4aO(q780}AXno>L<AQY*n`*uz1vf3#BD1cc&HTHj+p=z))U
z<@Z(<Ss;_~WU?K5?RJ)blHIn&i;&fg+22n^2PWrn1VK(xtD8Jr%s!a?slemqClE%M
z_AGWdQH@7##j*jfG}GAaGYr$DS+25pe@p7cmy5+~w_pa}b6TA9rS};oO&H_{72JUg
zwF)+id?$;Vv_dAzMV|5;?{I+y5fX=hj{&v?=M_U>(hHqOLu<=o&1IIphZJP|`0&_>
zF!$|Rc^|^oBQEia27ra<I6e4f5?znAS?gu!HRSVq;Fj|`I#ufb-=GG;o-smhw`jhy
zF<A3hr7Ec<{JoyP#>tui;WSh7^2>MX%MY8tYqkgcnzX%vFc>A?rF{&!hYmwCZlV=`
zt_a6h3FRM^7K2cPazk>RK09wc6RVqS^!}WxhwUQBvo&4&QQ2z_Y$8{D-y1FRmTDI2
zgwFY9G5y9rLJbIC8*cgShd+zQJx^eIE>D)sXwq>cdb+t_os;|cENt}sfTpwMlXYcU
zcAps&t#l)g(1sFC5xV{tva!-al^)L-<#d|V{6`wa)p#1@Pso}(i)Un$9WV+$fu7C_
z?XC{QK{Zdu^;k4bx)P%j3n#kUjp){RE+0DeuuTr$q@D*=g+_@Yl}2p{-|HN>TORZt
z@?V+_<yq5tu6-Zfh)Ys)R;nVd^}w+fu#46=K6bHDKf@1`aox*h;z~X{WZN9caN#I|
zf9t)q7kS*@A`;iTR9~e@+1aYN(&DaB$(x*KGl><~ltThk@};iyqd4$@+5>)jK9Jul
zJR)7jO}X5c4LYZ#g-JUHqENL6qKwhzW2L&<%E6D8z=}ID3}7w$^t=csj#i%a?^T-+
zLBp}Qn9+<WQgxlJy@o?b+spU1kDm1~B(>Jib=XzEA%Dn$1bZL>4UsEPI)=Z~)m#*q
zT8V_xb4IgDXk2Liii;9V%X$jx^Zy|rk|9d2c-GB&oF22`^c88vkZ^o<27&)6O|prP
zH*=mx&Ib?`5ZFNqOsP-HVg(O{znEbq0K;To2w=*FhjApj1Oe79uaL{1bn36{*BqHB
zF@RH&bSQZ+iH)33>$LE7ZX;nQPoimb%_ZyDVP)bm;;`Hk_ls)X<1$TOnin?4L*LpM
zqFDLwv1HzT9nzQhG~H*a3fk(MOr(pGTX2#$s^}e;Rjjo*HGJ!$e$?sY`-<9P?n)|?
zE&sYxG`^wek#@g8<Y&Wpk#uV7-Ygi$grZ^$lHtJJ`NL@{^n$sJK7kwYFBNA>T9d4m
zjcbKPa>jGfm+bbsc%UZFr{g`I$X;K~lc3u~y$G~BbagaC!wD{W%Ovr*Z=;)qsBD@n
zZ$N{ZV5U(&u~&TvszF<clK3V{LuIIoX$fkQ@q=<c0kxeeQT9rLp5YVu9pP_5*D4#E
zQ%f=h=LSPU9+y4dmG+UC7SpRji}3zg!q8`?bfW$p>(4DDjRNZs#mK-$JoALI!E}qT
zeqC^#Rj(#mF&SICjEV|+H2*%|-+f0ac>hD(&|Tj`R_9A~3t7qls!6WN!|92I+G0T*
z6}4aVB!5_}(1$vQ6sp*n#{P;j2tv)S@Un(-2;-LFrc*V0n9JpzX9&5%e>4}!{INc<
z97$4-5#{cNhd*lze1qbvX@+BY^Ry<LVyxhjy2(lbg<_07D@4l#^J<{z%&xKO^j*2%
zDt8!1uT)7dmS}Npau~~5m6;)yhAajP=v|;-S~5P7^A_dkVWK_e_IwY9>8y}hJD@1<
zn0YEM05IMb@MJ<3;cVVc4tr2NnBfkZ8CcIgue%|qUR`hBDbJI5l*OtuaHX&`0oT8S
zlHl0+h7`7)Fk)2S@SC>NupHOhEB`ML5Z4*qn7{Vm^STij(P-i6)a5c?(F@CYaMVX#
zlFnzYUiPCiXHti7mf#9U-a=#|1N>Z}j1Es*P3eC=T)~GRs}dun4kh4${g>>8U)mJF
zcwD{xXNLUeJE5WzaT@T9>&eote7a%12~j)Q--~TAhK%4o(q5994M_gX!emc>cPhdL
zSf3tIRw2X2U_wqbhD&vMlYsPE4#gYlKhGAtTB(_8l%Ru>?c(MaV`SaiDdl7$w(Kub
zs)bgXE+j^cM*ZI#lW+KMnS#1F<Om0>x2rl)S55nU9OGHJskOzfJ;r)ep9YyTnP7jL
zs2-*+oNi8YK;T@dh&M)kQ5(=Oc~xba-g`4Oxx<klF?ymqhF*7ehHG(a!sx1t5NmYL
zQ(R0bUGvrYUITZWRvzlL!)i}T>kHyf0}D0^vlXcbOj?Dkv-VC#o8zAvzlW$0R$^fA
zKX5ozdFx<o{+-|v?H(U(uMygy>4S>(!Cc?u$hiGEKn)uFJxinV8e!#wDi%q@U%m1F
zK^05eTb{65EtusWlGM1JiLjjH7AS(4sCYz+V-dVEu<8t)b`d+3^T`x7pwH>o9R2U-
zzYFQ+|4y7RMEC$aC!@t*&q-|&n7#dn=LCVJ!u5}T6rX(?A)C)5K@(dYM6wz@8e%Sa
zYo%4YY?rwe9q{v9i3*dwvFmxIC>0nVqZrRF1fHImA==1G@(k;_#s;_IFpLr9Urt>=
zzx^Q=Y-W2BRJ4|F?BU>*l=@x&7lOZc_Fgp8CtgB?zjch~4XM?1CMX_30i~}3QYLlc
zc=&O{3(EQNs-_p&53_&DGeKof!bw2t#^+xRn>NfjvzoE}FR8noQnBF~GXF)B6VJlP
zAxk6&S67DV?zGIDj}zJ7)NS)O{D#nUK^?I8su|!US~j-W2mWr)LOGD(&;O<&cz@+6
zKTJ#MYi!Ir@cy6vWVLBT9q9)J%Be^k<2gfbYgjV#!p~=SxbE#M9_@5dXPSKRI8a>T
zYE(`qn|}nQEnIKMj!^vxY6z^TLsY!lyeQd_XLaA&e>qRc@V#+Vb=N)8BbI_%Uzdp4
z_(En$;zdF*I&|Ku{z`(8e<6ysq_G4h4J&+Q7vX4zf#M}Pal4UROePcBPrr_z-z}I$
z!R)p_PnL^(pb=ZZMs<x1{|5i}Rr?0Nr+wRfl#pyBhb5Cbk}ekzF&r@i2|07_VsL14
zK{BaX)B}#CF(Km6d~H|a@bI6XshooBY+VD3UULPSuVa39HG?#ka9REX(<v9(pN*+V
z5CgSL*8HvEC1#D?n2?8Cv)=baD$*(GQP?*v$guXJe<RO6<zb-~Cgf!^aP1b|*w~<h
z=DLwW74yr}9RmlM%748La^d`+m1qqfbdlO(xSSuZuzQ|bU26Qgjze*O|D=Y*QE^;Y
zcQO$WDoQ_5H3lZxQYQr_6-QDmB~(3vJj8)Mv{>`6L+XDo0s_ovPbb!|%7Z5?;IZun
z;S)o2JGVf%=wZcP{l0<z`Au;XvzsrEp8vpxhl4MCW}+Rl+PW`01mVT-BKKk)HNySe
z0`Q(S{zjs5eB8hO4F9|k$fyW^k^|biF2@Kp%REKfSBfg-4pzbCttUtGFfSWTgun4_
zXHr|nVp{y%cNL_7o+N>oE*kH@C+Q6qI7yF|iaW~sp&;n}@A8j)%uVb+^k@VNMKc^m
zfx?iMF;A65(7R6G^$el@nX0Hp!Wx0UC+U~G5JhEN)xQM7zrJ?}Y%a|2Ph1*q_29k{
z*nBKi$o_JhDt_>AMD{uwi9=xx6V(I?&;9=u-2Vno*f`KPy;^@yy^I1x)l>Ihz4HGx
zs_^{p`2eN7mT;$0YurNF#0mX(d_DsFzGqK)KFo!dP|!ln?sM^em8?`)TJhhh9>V>H
z$`_h?q;X+=dIYcD^u*Ead*8P@(Si|{)5`tfR^VSx5||Xz56SEdp^c@3_YV|=o(5r5
zaUSwN$03dkT8RWu4~Jxmdi0Ntdj6O9P2!Jk`ZhmAB1h%zX1RC0_UR=#p@ST!wcaDV
zJ?ZN#Y><0|$^HUPVLv9ZKj`vrvHn{mifrTeolm`ShdvK|yg)aYe<Wwf7=%ZOp%v@7
zoyeU`lU_IK5x!(FJ`JsZN+gfX1bImLGvMvFhfMzld<+%{c#3|Y!N33w_{je|;K?A&
zuxNw{nU7{=@=dzlzCw!fby*$E=Gx7z6(ETyB%y_xrR8G$^T{_>QBwXj?*8|mMjE`E
zU!Qy@_dcr$rp3kdJLv@U2+~XwUq%SWWujuN3~x1ZX6KhHNj)Jl&7a%d^h<AjkpJ*0
zdjkBXvty0(!t>Ih!H)48Jqi~egunYwosMQ<G7g5I-E&*2O&=)>*>}qaXLahNvl2Ms
z%bluG&;g78P|)8jD>GgfGx+!a_5a2Dp9~|t#dnW=m(aO)pov}>?Zy59glt5X#1z}Y
z#RS}UB===f0|c|kpvP3Hlk)m+c`A(z#l5s_8~sn95s&xZ-Z>e(|6?1e><5VIITb60
zm)%^~U*L|H89dFOS7QwY5dD0Y(WHMr_SLRuhX4Ly|MRzuW%A#ky+`CpHP&dcCAI})
zEp6nYSV7M>1L1EI*X*0MI^S#MAROi2-o@d;zk!1ROI<xIT41>P8;(cw0T9CDzFI`%
zK?vQJ-uu6AHwY{vq;f;?HHqh=)cpgd_zx}}1|A=tbC&4(Jc+l+DF!ba2MbSulu!?O
zD{7{RjQ0vv>3_1jew~5tpTYnHiP$*)S|_l9Nyvre1ZBt}v=ENx{)$Id=sVvTgG+v;
zM=4S0iIr*>@BEv$?mdEj?()YtWjoOBSk3>P9{tEL5n#OTczM0xsj^UKlzqLP1sbb8
z+Gj(@Oz5KOf>3`fkV607U%3wF-5LL*()@X3{gp$0k4bWEV1*HMK0k$Um|rDG+f$Bg
zv#rZekJMkLU|I;zCy0QC1vFloM($9<;r^+U!XZ$Agf$>FBnto%Km7j<kOVP9{RC?d
zMP2!|6Fv0a5|XPsNl{LEkUagdvLcr7{O8-=_y_RiTF!Kv{V#8ODE!acb|%?mrx8h*
zjuZE`k<2~BU0lkfHf6%G=w!}ZJtw*_4RI@=EwPmAWPsd(#!EonpI{+_{CEY<>2(L&
zq$P9|+O_)MqD6^pV5vg2?y^=oE(vccq@{KkeQ4mpNAnu5t5>kjJrs~Q#($!vZY(l_
zlNA;S24!J9{o#k&mJt_sFX^c_kMquV4hM261}w-(k=m<xMosBaOI7tF0U$EL0~*~j
z?GrCB|MgV2qxb`9bmg{Sv_jCksB~PD*Q&8e{oeVWzfc|XUm%4-#as2jB!Ah0+NS_f
zFp0Bet^$!#6@}`%6xX=$i(ni!e)Ggxq3G|!lV*ShU6sln&ptHh{Qg^t6%7*vLO&Zl
zXGMzF;>KC@3u)G0F8H7~#dXzwv?oFscx8X(u)m3FCZQil<q87N7H5SBR!%Oi1e@BG
zc&}PSG(u4*172?SiQ9fguj`Nf1uyqDF+QLt1$vD9yzYm~Jzyz^bh4{26^u$!fpAQI
zmVUOH%2N*a_CzixfvA~0{+l05(n5^vXk8itCtKsLG8IcHJEFr+;|@r?kCi@qDeWHb
z&hYwCy1HCMY(O;-?-}Fglo$%ncT+~?fd62sq(6~A<h!exE$Nrnzam0#kc?Ts`OYTL
z+pMylir+l^*mqh?f3f(5$-F%jiyhEQgk53#Mcg=(`8}(PmOu+RAF9(+*EkU1wkiUE
zENCQTn0!k*9uUD$3Q>{I<SfxrsXbgV1LGEUo9QPWz{3Ms&IYEIXzwwwBgsigAsS!E
z$;}-EjuV1F&;zZCe?e5boaFH2;|&mWH6JNZDkA;ty5~|NLUGrBQTTTmv)GzhFje9x
zN>CSv#L*yBKwFlJeRFE~yjm!|sVRNd*B^<4*Iw++uZ##t&f`8UD*q&{D>1>{!JEaz
z?J6;V42+I`F;|hxJ0BbVr|CZo21K<aMi?VXGXK;x{!Ee1cMR?l&116~x)OkR_((be
znj@wEM*T*k7nsA~^Q^?7r`#{z&wSf4Eu&eU(xbmq(J<**2w0it6#;kPU_*$X_(nN;
z41htfwS?Gr^RZ$AhUblZG+)4)WdYFo5wgF~2W-z{pa`=ivP-nb6N2kuwZ@Jk$m#*y
z`L;FI`oNN**tcXZSMkCNmYcW?aUxCODHQIT>$ABHFd1{bM^mTkR(uZrSb~6-#A(gr
zyfKu!&O^q4`vCSll*7>k*g}0mfOZ7@MTPI2r2v{_AtUg23oRQS#xC?6O1O_%pixc{
zNqP&T>$Y&N^Xf>b!^?@gU3A&9tRWs)o-EbA9hT>fr1s@VOmjaa@YpWzQa%5c%&F3F
zaiX({?9RMEbZ&Hr-^TXb#l__YJ`A{2`Zd$a<|Q&41RUPWE?N`Y^t!nLUD!gf0dCN9
z5L3qvxCwP9W7?xtV^Tb0##+@@J!_k87C^1+HS*<Elf#O1-D3D-LyC^OgfaX=&7{9^
z`GKiwn6}Ie<(C2DmqY7|Fb7m4vL3bqkv>Wgj<C<zIi>-Ez#kMOZZ`??lD}j*Dh!ag
z6Y9AmjYXjlrQ!F5Bbi5z5&!!#{rBH6AHny2XiE|jc})A>%jySP9`nY`H5i(DZe;9R
z8n!ASqrg(Z{JrSw$`E2T9K)qiX;ctSg=GrkqL;U~`DKd>36s>^MS;be?(&T5T&`tl
zE%kG~8i+>Tm~<wO_rYQZNQsZV?^l8m__&@C@#qy7`Ap+10&XRG7I5}EJg2i4y<bmi
zeAyvY3>CIm0(3B-U{VtH=6*uDMjMIc_g7s>Z=Q^^<}HgiyZd%Z3UUT5RtGLi=yykx
z_sXP^NX^qyd_Yw2P}~ere{CiLn!)Q^g(`&(9z6VD+540YG}4pbX{ZPUw&E0U3|Qzh
zMO*9^-g{gN4tk|&f7$@O13l!&Ye7)gA>~|TCz=)Tcfb=O6_2Nze9ai1P>D_02PBFC
zV%O=+I7C9WFqSxt*u=y_N!BWhQC1{V|J7m>BCN3^-@Zm~CyTW{b}%Ml(v7JP5S;!4
zn{D72R>QYADj^)>c@+{RR}~O}x*Asu&jNseE@cZQKcbA2BK^D_KsNwvt~R(pv+i#g
z?%xWnf?FxnS-(I}H|1|$`u}>m{e>w8&ZiIAe~da6=}=Wig}quM$qGSyp)!_NFMKK*
z4uug?z<1<#17`O<xvD)^dZ9x|-@76q;w?$X&ptCA79$gtMSE5qry5IS$=fDkU%~#7
zlDzzDw)b1mG`8^$uM3{QF>1Md6?`-~P*>)rT4#yNy0!Dp9{$i^4V+?4(^0(ou)6CF
zw(zL06zzoLRO)mT!1<+Lv&4Yk^fl;zbhLjVi$(W8KU!6p!ukIF`YE7Mn)SS)IxXLR
z_y`uP!YHP3(5aUeBkY59F6c_84A_XvgxZfl_4gh1udG-$XOwy!<=82A1~TDdEgJKR
zMrE@;#%g?YKQbx$(yzC<Gwp+kT*$#bdomGm&rlBH;&7h}-pV%4Vk40RFWvXXU0O~q
z_m=O5)WTjC<uc~R3DC*(1LW8jCQZ|Gf&BGUAM&2QyrG<y7U8#4-UPcDwH6Cf3<a>z
znUl!oo0-I6iOnaP`w{WcT5T1uY_gcDX4si1Z!3N70{^eMom_8c%vLp+F(@R-5-?RJ
ztp`l4yey7oxZ|!d7{#%FKIu@fFgJeFT_RcZ50r6(=5;-+$CKwj`yc<M;tLQ&g|J&_
z#izM(rLt<pMP}k<rA{5L$81%Yw|o_08Kn#Yu#3$7Z?8J{lJG+qM||r9xQ<1j`!~G0
zJqoYc-(@V~H374g+=qnU0sp3;D8_6@$7Ry^7?8(<O)z+#E(GG30h0f-6kRw5!0sq%
za|2Y_hzQgGXG>GR)=jY2ZCi5-e55gpJ9xVS%0Wvn71$AO&|B=)SdPD@dscj3wNrjM
z73x=(Lt+`dZr^{@m+fs;`<)?!A2Xz)qGI~Z63O^`b*2lD=syuVrsc~eivx+PFwf6J
zf(MkC`x-`R>cYjeH{3BK5Lg>X^ODe1b(iBC_tfATPB-h=XdTFW(r2ha$*|&0I_HkW
zQ_i)M6vw{I!fj*mkT6njQE!j4XCXk2H<Mb9)k%4RAx)|A=E4#<y3~0fPxkb)>Gtu(
z0w{u^mdEXRNuyqx@*TrF-jRg&%-P6I51Ox!DT%BnKE<#mu>;xI!>06XG(K%0{ileZ
zP(p^u0jnWEmd2ePxv)_I__P++C7KOMT9sx(!1diGa*IWC8*G|C>tA#0CgnL+@&KR+
z#mW|JhB*NiN%Hv*rS(@#O=NudyyNVAllA+pzT8pnX^Uaz9fvFM*tZ~P&1+zU`)+T}
z1i)78`(DeZnppfm*x#RXd%A`9Eu9+jWg2FR%4>N4)<2!a1&KErR6+%{SF7xGAD(+5
zaiqWQ&1GqG3{s5M7-nj~g2yI-f@mZG#f^+V!3WsZpy=wMrth1H1TZ@Jch(U;yz3qh
z|EHe+-5>Z>b5ICp7_Qv&m|y7~SqVU9sdD=fEyTqCDe5V4?Vs)rIM#pKJ5K4hiA`6+
z#Jnk4C!J=^elY9~%aUhbNsMsI^Y9;sB=ML(Gk-}fcag$ax1JSR#eccY<}^NY_D-xS
zp77Nf9+RH1BI7A`#x#OMBmt_&(YKoQykwh0WifyEt&u_zATEv8o6wCnBNxSiLoj$x
z+Q2Jc4&P92VEOGl9N3owxSZ3ocw?XuK1w5)8Uo%|xP~JiV2qq8zTERJ3DSraRhi5X
zf(uYbrP;RJV_(0C<eHVF6tB<RZQ2?u&ZQ=h3lP7X^-VqY(R=fL-V_OQ++hlAHF>@R
z(Tme5TU>G&+!h(}@lsu+tw&>uKm%Q?)`3c`L@Ve$E9olKFver$t-8*8A3z!?05=f4
zazHHoa*`&+T^K}Ievjh}Fw72XJYRj#N;urYerLcRL*<^i$@)MhkCe60NJ##pz_VG;
zsEf0`veUL}J`kz0rKDB@xAJMMnyJ~BXNWK4$cA2iP${$Fzr6VDeTlv~QBKKV>L6!U
zpqL2@d_-R9-xCbk-nQb|EXWR!6y}k$b={qiO7}hB2KTo&;2TmCGiVDntN#Hb&UPJ&
znlC$L3(mQ~xHMmR(m_ZVea-`}j6Izt(}W$z;<TcM@Ye<A2A!rijl2BoG)+_-q_LO-
zTI)i9N;vR-qmuf`-sF7sk^eYHcuF>KTN>`P!w(v6R(+Q1pB2ythHLh6r-|GfjtsPN
zkcG(1G5^fmaxdeiNG!>CmTuunKQr326Y4ame2M|8JIa6x)aDW5Uv&h0kD;YjiF~##
z{p`~1D?ElQumb!UgIcy2sPCRGi4_Al|5)1sF(SOQHM~I097sDFc&OD6hHJ3J@Q(>v
z^cq|o|ExIPAs%XHVcjV=nYlO#R}4DGkK&;6s57>cityVh@2tB8D8U}AbH8~=qO`Hj
z6Sply?{3yD5~oL{>z-qWZAs$3j750St>}Dv4VuXtvZwS*sDKi|4rIN)hO@4RQocrg
z?=%Mn{}&+JzEf3E;r9hf*o@SpPT94NYdr7ikDrDLxz0J)To>Qz=LJO=)gz=9&GY?*
zHf!Z}pdTU*^Z@cHXr3H(H<E#20GL1A0F9X;K$j|>7147)!SzHVpMxsp+sAemLHK|T
z>dj@}BK#g3Ajvmr-GWuJOkj?~MKj~R59C~j6;I9Jgt^mv=U0$KZ>OhqOgTpr09Q9~
z33iW^>^+oOFLa)QxS#XU?SO(#d+ZoBDC@TId=856c|YFWX?t%hM%@Q=xI^~SE&xz~
zcmIaeOOF^JNd9NjD7ua2)KG$*g?~h6fXgQ^iT^WdKuA(K*~oTFkTjHk0f7w?fi_ka
zv=Ws6sh=^?0niN*$#d$I0FWru?N=-GAWUOcup#R|%-?_R;o&GeR)}iUO)MIv$5DxV
zA5W6ka{bY~Z7#&9NCE)e0Kvn*?NFIBZuR!0C+@|GPwXj9c88Ol$7Kgoq_|ivVP5S(
z-C`~C1^4|bXan783gKQSFzU&6xgTpBji@M;56v{_q)*0zY1Oc66vv0nw6||N_)(8}
zVgMsL;}$&h<N=3-R1!;zkpdy85K=|`Oo0+*k!s-^<;`o)6AMBC8=aA7`$Ex#PwdDD
z5SSNg;kUYxL)#8KsNK8(#fc{=L*ISMx`wt)RB@JnF_h8_q#3$7)@p1YG;In2yLrD<
zfoXXV&Kt+W3i(E_xy3b6yPoLtdvQd;_yPx**zduQ_wb_1*I6<c^Bqh9SdQy(Iwoz$
zrI8Va@nI=br?EkZ%bBrwO{AVHjSKBvAW5H;{MdVE6qQ>!q8bpu1~FqZ`AU{z32a~^
zbYk8Tg8NwfE7F0+Fj#u115mk8Dwf)UzIl#29@Kndqw@B~(lRpHKrm8j;co7c5kVPH
z#=;rfqdTqN?;`ab$WQl?4#AMPrFREE9nM%Zx>09@f!M^%CZ;30@f<M!RR+R~amqWO
zHq4du1LB^KLD<XT%S<qIj%A7<1`&*D0umr?LdjX#T5ab!?A;}BM{?)gr~_uxH<ZvA
zGdQ;e9la_G+!Bp)VXS)4um8;9vr{#Wz4~ArCj|unpSRl8UiSO@{s)`Pi48CHQ*2;J
zAoAQn4Rx9@4@FbPA)`fV*+4g-$lmzHi+y`FYOLaCz?b@l*Ixtkn>%3Ucv_4VCyom1
z9|5zPq9aU}PvPiGMwKOH={)&IZbZ+s?R9h(fQM)ee|`qWbD#6vq-3)Tl~Nt<x`p5e
z+r>}1>`H6z($v263-CK={c@M@J@A9n1%L}##Vt(|;L~F3DKo6}%l+<}$g?F}pqHt~
zWruG51{^C-T7Lc+8|>ces0s%G*xfcL{eJj^%KK*wqtCs&+Pw$9CEu+-^H+Vjh7maP
z`8LyLrfzt9^5dv|u>hIi(iMELhSGXJ5ZI9|YdF=KC`ttbe*vyX>h3!*Ed*y`NHFkz
zOlcj*=O|vlr)5Sk81~eTVF%EUbV-hGz}EW8<J|NPr`CYGD3Jae$S1Bekb6cNJYB_1
zy1cRM-8f%%k@Kay*Q#%FW=$?m9s-%od;qDnOIUOfW}{O`0GQVli6rEDwuNzKsH2hs
zvm$h-P0gV<uFxQi12osj_(AK)We{McG{xtj^dQa$ufImzdMy^Bw%w3N>zPhVGf$Ee
z#;<X!Lam_mBlvNREfbC<omyq=!S)?x5{NJW)=j@lu+B1jTs3|PlXrnhM`chcvDlbe
z_FZH0k$a={(!8J``pNK->XPbw^k?33j|19BvuMZp+yHH)t^K~Xy1icQSq6;HO8+Mh
z4QVo1=O}uSTY|UpU6obEZg>t6lehPee`Hu)tR%1BlSG+}xCf=Ozzh9Tf8ijn)S(x%
zqb?A%`oX|m0n)-}{PCnLNwg1;5?sMdEA+cQHOz*99(sU4zGp{LqFWQ6QHgpmyC^18
zptki)GMvXOila`DFO<W1b0ibc5jUyh9#+sj)_hb6RDJ==91$XC3K;0S4QG1B-#$}3
ze*ZqB%KQYzQ~upkBOaW$yV;r_O%mN(ui|M{f^k_f?2Ght&)Ix{HcByMI#Pduw6}X)
zm&>pBZ5bvF){qZCn4?;~&tnPTf(CA68*BGwFAF9I6;&~cni=C067suoMS@W?fmQDJ
zgBTZdVjfB3FZ*n$7^c@3XtI@N1@~&YqDcG0PaZiabv@ZJk;9Nv%<AMhF3Ld8RnXw&
zYT&ZZjbB4v|Hj7DgRZHmU3+7?4tjDmORYPA51$c%@&;giYxCpvvyF)*t-IoeQE*iR
zm8PzoGtOta22%D5-|36oq7P?%F0C!b;@0#QaM<sBe(+vX>1@(8P02l*+`^v-a7EQ0
z_-f=KmLn}Ue_?Ej=q>26lMS6I76OSTU)!cO$Yp!NpeCJ$_sMg+K9NDvHL_I=985LR
z9s6eO2~t^)&$_gSo;w14QW}eh=1lBQlJs-;Ym0we<xm_3t~&D0Rx`Ykf{@|bHPuq+
zgaEV6-FOZf^tJ1yD1p+&ftz93Mz48)#DK$fIGruQKG6qHZV|3MF&ltAk~BHd$-6Yk
zlx*gYhS#7E7FAliN+`BEbW&>#Cq2DzFus#LJ8<+oSJRZCn^v!k&BmL~Y8ohe6$o8Z
zZXaQcQ_=KG;ABh-1id}qVGA?(#z!~&_L$w{*bJ(=Q*40-TyDQW&TueEJk!R9ZRWn0
zCwDn^Pm~+IWxWONR69R3TwfdJdb-W{ouSa<I%2M(LvZAyIDjHyD2r*T^E1Vx>Mp|f
zX$WY91HkV6<)I(*k2Z`sm6^JeC$saHi+avX4C{?Vs!}kB@^4VTYt*<j)Rl?XnG~_s
zOcS#Eu){4rfMDWK%hF_w4#iG?G*!5`dc_30mC9l>wr!S7_6!r7)aOSg#qs9d{PcX`
zFa8><$K4e#$GuCEI7aK*&S4kh2@aq8-soM|)ZHx9PWJDY`CiNLmzlLje7sf{3FRQ;
z=k)I)Y`i($+FLDOm1^vKH*OpKQ0N{q+Nb^n0$IfRmt@bMRN(LKNw5!$vn)DZy4Rtk
zB_l)JZa&#GJD*<6HQo9h>JjXwz-*wyQ?kMDBc7juP6Lo9flZ2gA(SEbXJ-DP4e8B7
z_ty%ij&J7h!g9s4<85f>XMKDZ%c<AwyKi;ACqyPU{wk8Lgvf@f=F;(U@cyZp_8-7>
zCsIMD3Js=~5C4$RI=(z8{NK<pWpiDH)2+<g!l@=@qap%iW80_YQ9$lht=E`EjrwO<
zpZUA24?i|2XGdWGYnv+iwR5V*C+Ql)+9hSaH|G+x^|KCNz7;WOtOrX(DDL4a9tXpk
z%RRez`DE|Gv$zty5*`4P<wxNxuRjd!QoU<W^8CaJ)|}SC1#viI6iGd%;qjr!GYqVN
zxbDp!4~l}`fH{}h#W4X?N&BMd_EG?`vTyK7mOJ>@R-2PYSD~{{WDO%vk+X0Sh@B62
zYvvN$lYre6t+1@!<Ja}iVC!p|wIj6eu;>rvNR4DmU?jXaJ9XWj&MbZ<VYF}yboI6)
zbBc>RfzY|rK}W+x*5y6D_f^}chi4j(2&QZL<#H{FaSc%R5EN%IXkLeyj4^DKB%%i$
zfZcmW$uQ2!9J%81@V-R0WD+mC4-{-q8tBZ7X9f<Rh|hrtUXE9#w^x)6Qs%AK)t_0x
z*gX;mTH6C^+j-a9z%7J1jALe-eqa9(0VnNy+Cco06`|k79jqr6A0Kbs?q_8-Ov5XO
z+hSb9aA#UiExqi-TvyuvF3i^l;}!=y`8erLkae`U$ML!i=!DmAZ|)F(md0$)2z;&x
z!L<CEvVW%>?CS=E$54#%b+!|3U7P{n7(zAXcaPO-g88$2vFCJ@T^TCB#9>rmoH_~S
zBMSE`!06jHIUL=QMXG!-P4T)HpRF9hajib8c?qdLN(?7Xndf=aSN(x)ZpQ)<PrmeJ
z4C?Y#X?f=UC(-tqe0!_sh;AE~O??8svtFCMFKRWAY(Jg)#X0x$pDEs@W)4id{ZtJ#
zeHB{HO17yBD<icnC@nTxv|H+w3qXLBam7AryHpN8bPMFP+ugWhILCgvsb2O}l7rSu
zU{;0(PQl9Ou3NcQqak<hGf4f%Hp>U0{V}bHrfG$ckQPUh=G)6XU&V3EofBGRT`GRG
z{?BAtqoulCBy{yfPs+GlguBtl;2~AuW~o-OyKUpXM>x>ubBZ3k_PiLMPo(7wp`^U=
zmqj0fItwL{{VHQn2UjOh<=}D7H^Lo`YMQbpeyzLVnd&-$UGn21Af7)*(`x`JZkxqp
zIYz?i&3B6cEDydndt7b^4hU|3=S(=fCqEWeF!YS=W<TAn_qCcsFM9@~BhR~28M=m_
zhGxI>`TBI~NG*(|Z7W^7`4Xwu@9G}W#1zhR-Zals?kbX2EWdy+-7#eIuYXS~cN=6=
zCW0kcGFRi-H{Fe|ARM=Tibn#mim@N@)tP?P#XnvN=2$j0F-TeeQ^m~V!{(kpsCs^%
zTALqK#wQwoa)Dn@^8c0#?1kq88VjAehTj%=o%^*48zd#2><D>S>2tQciiGAIn#A7(
zAhC-N8c9L%O;@b3^4z)ECBkcu%1svA!$$HINYz}?wXH&0sqQU1`B7|wiZ35y^fUm4
zQE?eUjHH057|site5$GoVH{_hbZ-r5?ulNb0t)|DyKbQcJa%ZGx{kIOSRu0<7`~V5
zgcM}usw|w5U}AHGFCCW3>zW5PtFHt+*I>xRVj2!4Oklzgj#ZljL$Q{KSGN=UAh3pl
zO8q#F@E9!VX&&K*f_Lw-tzL2Y!|dLJBSR3p3r_{1G51Hi_I-Y$s-Uo$IJ+^KI&&>O
zxw}#N<NfAlTGH9cD~bIb4n-!C{l%qD7lr0zVE_smG~C)QUNJS&oXxtX_XiI0rdGsC
zX}qk^b6dDe`9_e!H*q2a8Q)&{Wp?oBmx%7ivHD$k{sxXB9BF~}tBV~?8%-TUat#RV
z`}laTwgr~wLJp3mHe<-O{Unz`T96M5t{*GW%@)QyV_2uQ^@Z`c0gWCmgjL!wBSbQn
z=BbI9`R=p1eoTl3*{2@7=`5ncVy6+A{!)U=r`k`I(nA*N<}ya3+MCj~N+|){t|J1C
z0;J%ro`d2Koz=cqZsSYw=P7x2ZplCUCVO=IpT!zNKA{7NZy8)#m+?&v$0fIK^X4d>
z6W>bKiY91l7+!nuJfV5NMyO-EH2T9g-TUmDo3;A0XX`1$Gz%V8RelNYWc6NHrF1l1
zQ*Vh(;C-$fc0c`Kf2~bf%tB&0jn@)QjIgs2@Lc2N1Wu=K#_Sk6vyz+z*Q>ys8<B~0
zRR2(qpnwc^-A@~N$*l2vQG2To`e(K~1d<9kcFIliQ(fLLouuiTYOC6w&A6!8il$#C
zzp04w5K*1V6wxVwC@7q5G?451eaMUQC@w$R10a5PM*B+g>ayn?ctGG=d}uPAuS!qp
zygko}m1pf|PPQ$bW%q(H=GnwWL%6-)w*Lvpxeo<%cgz;*FW-)7+qx^+F}0-xG8=WG
z`bT#nxP5GK4ss`2DIU9I=1XselZb;MktVbIVDQ@GTzJyhc#^+Uco`XnkngBcB_@Cv
zg|iH~Apu>z?BRH!23*$G1yL{tDdXDKlbQ%-rn7Yz47U~-qdfxR=Pp-mlxTrL8*W(w
zZ#uHwd?xkb+%Bb4Z`s~Pm|{;`zQ+dJxJDP3peMt1hR20*w%I;syg7(AH@rJ)AH&y~
z=CMh;0UR$0_8ix~##>i!SiryPHp-6V#Ey`AEZgF>b0+K9au@$Td6afqcY7F4B!r<m
zdHMOxVFEn)!->|Vdwj%sC}vY^{d`Mdbu-UuS;~Fic&xcUT_tDLy#NsHL75<+9!R?;
z&87`0FSIbjXKBHOS;$e#&b(*aHPNmMNNP$`78Lq6B<(JAmqTH$Mx3Uguy58_^P%D`
z1peFgNfd&%EbK!;=vUQCLGPQ$I+tU`<0{mPG;0|YBRGTMz!^hbY)cK)U*GeuzlsTG
zXh(HAEEPPfXrbPirBCyuTI(;3Zn;+Pjhg#jv$&QP&m!dqAD%8au&De_EH{G`Z|f+F
zD&i|yvH$9wSEg{s3|HCWk>M}&o1?dXER7gNt~S`rXmNO1mWp4^y;qv%%quQb$t$RV
zck>Dj$xl;-t5fkvhQqXJR>X9kHV7d;en&Z+@2g@Q#W8k4h>-1O<P}G*qyhH^EoYs7
ztR3}Qf6MxkR6HrY2oa|q7eLEKX4|8#scM5Co}k+I=zJAFBg53NOmJt)NN8$DH5Ar9
zPDXMW2xuz7EeJ@R?M~Hp9k&Mrp~z0EIC%-)sQ|_$4=}K-r>a>Hc5b-NrYiP14=@C-
zm`?}4T?4gCW+MWGuJl3!-dP&-C71i<dncyEN1WB-O{+=oBMq{@G0jxh*UKK6r8pql
zVmApz9+P7WPmwA6W1_+)5W7yWfOb6(w2|R5trT?^o>_Rxz{~~?_3(i6cyM!a4HU|;
z;o->sp?8AUFyornhXK2@H5=hj5}z#N+<mq_@O0|t`OPIM>-L2~09V}-+7{5m&+I|5
z-WWmZT#8Scat$HzFsq{|I=K)36mwhCR`}f`<~;K6p1>0cjp=@SD-BU+_w7i{%*zsM
zq(vYU$-UXLnHh2762yV{sB%s`IK=N!nwxN*e5wU%fB`%P%(pNL*)24u8UOpp&XT7?
z#z%#H44n}i(beMufk^x80rn9*!Y@L~dEEB(YZwkX?;H81$Z(FG*1%AFlOXYVXmGWp
z<v%bFq(fqXh;Gj@9AES`5b{faF-=_|2%GX^e!U>|SmlRY5Qh)BxSK(wOU&tQ)6HAW
zb{zb%Pvak&825$YC~ZPf7HAP?m6<!6QCoAKuu7n{DX?^o-yFM;E@7F+cKBjySbg59
zE;oA(HvY7kL*Rpe!Y${zQ&mb^mHzG|%Zr+sQ>F&$Qe^t6P`TM><x<<N?R*Fakc%g}
zcKl<<a*n+*pc&QaW+dE{LAsv;!%}i$vbxs;L<>gS<zkgOMecvxuY)PKv#@izxteg(
z>0L%bTO0y?S>(X1`@WB#(Q`Q>L|lW`g+4bJ$TV27tfkv`60EY-Hw&tM1c@}MKQ}iQ
zmgNwb+pTp(86vLCKV51<Y@r<Jj-eDWo&&oIjtn|>Edn`({YQ~2QI-e+oP3TZ*r`Eu
zAMnJ3AK^_mp~l|7&qvvx!p-)9V@w8I_<L+n8>8!YALQ`HG;TsGz0Q$wE2s6-(#aM#
zhjL%J%Ll064}G@S5@Y<uM}1E^+4Pw3e$<1xwv^xl)Xaxzk1(%RmKuRdL0e=2!dZsX
z0(}El1X0-KM_%q`Hk|rZa@)`vyXRFcdzuJ~rC5~W>$EkZI#KUTE8Jhd<Vx=%@%(CH
z&ezxSQl-w%a%EKg`CHIo>X5(G!k_Bk$Jpt|+xH4z!-I^eyv#NnJAlo^Y~arwT8Pny
zRVwipt)j-!nc5WieEzI7V-*6iIMhz+(>NDURXn<q#2tyyptO8u<e*OmgH7h&iVaD+
zMA2q-d<6dX^Q(Z)|KjR_vs&Wlsguy$N9(E93Bd_=gBlI{C9csABlpwcCH?;G3t2Y9
zIs}1r8;e75r?Au1J&+^!^|Z?%o%>mkv-@&u4#;#Jl4NG~v|qE72QXfpxDPsjs)Vk1
z^2^o*4gc7EgKr0P#e;X+T<$lv%}*SQeNB3RP&6iq8UTM^tJ-X^HOIf)0~pkOJ>RK*
zIj9BqF^b&)7B#)>(_;8L^VDpFV@eeIVtagwsY;s?gpZF~Rja-)cJhxs#6wb`#k`w?
zQ`Q_!BAnD`;f4hB+US`Mv5_LAMU@ezGX&G>zNH~Y_?2}Jwx+;p<{e|<dboa)#97p|
zLe%>Tt)WA~=Ffc@5VxC}=m)Q2;W`&2=vvP|Hol!k2n{70fFKu;4iReiXk6wO!f@BK
zg4xewD%uE513UAMN7Si#yA$?CY<()&?dXCNb^}6)?)l{6el6QuWlg)pM|)SAV#3VD
zjI^`|=xtx%M7cI#138XVpBRj&HbR)6b!aav%~~x!GPr~O(xzT&uDVI5Au_H&BG$=>
zDy`y!C#G++#@L*zn~}i9!`O0eAXT#UWL6{c)Y=b=4IVYIGxbE{<&Wy?CZ<~v5A&u^
zX^*Vr4;WB=LeL(p5)0Nn%}>b^zg6(;vM@xXXij7^k$!+%G<k=mQ%QoD3Mw<qANL{X
zfEV5zQhhB!;+#_t+9yZ{+lZmMji57&+t5rI@W7-mC6~f4JL1z2$&oF&XlqK;R%bAo
zk$roZu*OTTfcho<r}coE!;VK??{-;MzbS15pK9~9`IBdB;DJY22Ln5K^P%S*mPF%o
zxINs`A0vwDdvu=nWAx)+aQIwQd|^~ESg_W@NFWeZ)wU`Z({E@t1#?Wj6wuWdd(D5U
zvFCW$cOSI#6!^z}zy<1c!!6NpH$!~E#<Ih>u4`@8P(Nh2@mV<~d^9&a#4xxh;jQo0
zwU_zGL&m<3QslDDZY|!d+I|TSL#qQ|`HTe%5V7QeNc}gWdiQ+!)tk@a9lc#7j<TKS
zk7(ws*2w7mJ~an}+^V!yZXNyz%k$|Jo^uz+*!%GwLbY0Lj_35`t9B3sR-~AUmFH9a
zBJIm((t&HsKnR7~=;MAwy(E8MWd#3Y%0(6)y#4wZMnRowml+<ay*|o#fmQskpEp*$
zL-p;5h1PMCPh`h1pDz;jQqAk3MDcZMCs0c=lIrP^@w8lF-m_C^X2WY|IK6qfrf@Al
zKJNp+CkJ_FNvcZLeT4f*3!uC>l1-D}gt)k0ZG-RP;fpNN{p~=HseO<^iZpmeUk!ln
zqbc9ukB(HyCFR%UWBo!TgH{AL4E1U=d2`y;hwvf?^}Wa3PVEh68+`2S`PDy;R(!6h
zFh>a`RV`UNhH#Q?XCyCl26WK()Sd`^+mq1|`o?yqqNqZD-RBW4=uM8vem0JeVD$(o
z`VRDKgcJ7UoNM#@n7Yy^*Scpl17!qSpd60Ns}r{3r!2p|=U;!lN8leEDau%17(~RC
z;XcdZxtONx{v%$wkfxvau?av|iVxuf@Df_->F!>C&`8H;SU^JvK8I-VX}=Fn**ZoW
z`XDY-;qA1v-V$sPZGr!NKmGu4DhJ%|vtW7o>KJ^p_h{#^eu>40O1~flFJ1~^9;?2(
zEQ+C@6}J7tf3@GX0=(Bw>!&jY%JO}QkMxY*tQ~hDTUsp(yBBMIGK(>752>rRQTZIm
z!OS-;f=2#wmMDtrrU`^PUhbqYpIs{9gEed*(zE;tOjF(;c)jGRQKyP{z$ITD*cGAy
zsF6+6Zf!kr<Ih!9;cRg@n<ucvhlMihWrH{(Pco_S$Y}frKjQ{ZF5NKnflc2o-oBIV
z$zeBx!>*5)`wqvSJtA9H49ei{m$$y-pqamF8qJQhzCD}sW}0&On8t{$LTC`_QG9?1
zX(scKepFO`V8;(e-N~ofldWG6O4pK*Y33RyNgc7jM>Te7&%K0&_XGnWsZiK{jQyN9
zj35!WgiI8E74=Ved%4#npzGvVTpRuxFf?g>FJiZJVG+rRQ0nlFztXqjK%p|$S+g5Y
zEM^a0D*X>+!h{1>tFln6R}En7$@#fb8*&Ng1%|d)yz;po(wi^%5OQQbXheQl7(|Al
zvd+>=0Se{%PZ6(M_!0H;tRSpjr+j1W$rSHw=M$-E1n)Tk<u<c>XGKJS?c(4!`?xLe
z(+6=C^YhPpSlca+WM_jSP!!lOU$R@k8^lcAkSSsEWnUFtBgEvUp6TyLnTE=ePa2`s
zt>NKP3dUjHo9{r@v$%rE8YEK?eAh(I6Fkx4F|(f^(u{H@<Tlu?W)^x5(|R{l2Wxx3
zV&{27ohm<fswtmovkZi<%z1ZuvRyZyBCP3p$Cmb}oI7!i6oT6H%)mnAI-v9$CyZe(
zbjsYH(gC5LcSk#GURErG7Z>2^g)rc4d{p&D#$=o==x=R~1bi9mqV39Ic>*l`UC>Q^
zz>Kk&^o9MDthnXf0b+ZQB){an$XeWhcus58K!OJuu>#3ftK`0VJ(r)v?MhOB#UW4d
z;-c^a3<|x_y;i=x78J7fj(zEP#`%jNObiNQH#cZg#&D^|-yNZi-v&CSe3M7?a$*gz
zH%>bRcD?tVmN53S;9x%D27KSbKf3dM54m9`8*@W&1fM#iKN=CfPW+P*N~m#wCHz_5
zV73I}(~n?d#(s;@KVwb)l9(@HOrnZ9Rz@=@V%YqFx}xJmPY})s{pB~@5k3^5$VT{Z
ze<&;{N){ZLem>)D;QqJ5sl|dEUDMYP5xk#3g7@yPcE<Qs4kdB)d<grWctSGc0zzbG
zrBZv57C%%k_BA{|TudQRmHL+GgLFJYX|W-?tT@O6vj3h3bfY*tEfS_yK8|qT7e-OP
zKVM+2`|@34F|`seRy~JDKdSBEnJ4(D17Nf?H4~vG1{Hwmu@wC^jbFRYX~Xc;V8mvQ
z6K!c-j%!Ahj*jld7{<pJUl^)Kp1el3Y>Xi;0<tqf&6UPEjU5+&@zm1Q4O=e=tPK>8
zULC!Z%yMxnwr4m!o%8iIs!v8kZkO*wktx3IDiUnDb-Bzq-U#A&o#Jyr(CPPt#yW0o
zzw_+vV|9-u!fS*w`X}+UaR~`hZQ}M!cY#Apl#bg%P-2#6SJTDj<9<oJ90e7HJ3U$G
z4+zrKsTI<!1f<Q)3lEN?<>>E-^l>|y$be3wtX3tQ5o>Cub<u}Tr6_szqL8qu1_!pC
zueU4slzPl-RV*Q-3*mEbmR>SNthB*DUO#*leYLR=344156I|KJ8%WUln)RLK>{U8Z
ze>%b?&5|STn--1nR9wRQ{^=*ZZ;0kjG#o#0-O8ych)E{*X@?0g1XAEX_bPXVh<{+d
zkV9wA3d-ac&O3dS-*ZafEW?!+%fI1u*we*Bx);m9yD^*}74b{~&n4J%qa$HCCwBOi
zjJ{5UgVnw#_J^(^>!~VB)A~n0#B4r2dT|)>QR6OJ=aO3SEfDF6rGr=!o#Y)b84aYD
z%BOu0_ar@Qa%czvr&7~r(2|@+u;6alV@^Rc*ciokcMQ-_w|A~xqxesYfqmkD_})`2
zOXhYP%!XW)PwDm7obbuJ>j5x)sOv9Fm&%PgrPF=9XYPFmoE4@xhMH-*op)ymVdofz
z^`AnRB*ap?OpjUNgrMxm$Dn3RcD+u%s{G1fIo=JJJl3M{UlN$k@MDLcws(TcrtaV#
zLK%shc)$S687d@TK?=EF2y+xb7kt|*p@>*1_;)HTFfsFD@FYW4%}44r)}M155+<G)
z)D#GOM>NPu_1(dv7zczm!a?D>OZABg3lXocDEovj5A@;S<r77jFh^6I7Nu6tAE?W6
zE+BN`mQu0Fbs{&v3r@(eYB_ltWD4hcbE#YQ?Q~}<UoO}as@GyGY9byHdk<xtn2)Cc
zG@zwS9WEhJ9BwWl>!ri1sV~$vEE0K)!y`xqW+NU|V)bW%4#J_vq8{3TFV9|3qI8Gj
zopyPM8>7D@wxJTd8$<iYp$@t`C-ti|6IEMd>kZek?toezF)ntild#cf%lh>Y@l1Hc
zcpw8M$Nb~!>@|LB*rCAtwb`_a`K68f{sm)#Sz57$g|9cJIT#BD%V6#z$vC53oQI5c
zJ7SdPX5^6tCek8B3zotMtlsDx>GS<AN67p)sT9+FvrF}cI#jd<&aAY*FmqyZ%BfXr
za4ie4&fot}|IjS53>Z&|_r=~?E_r?W<gItB_ik)#qfy)F)EVoHqbb`ll46`C<CTnj
zo^S1{zuQHdVW+RbVWFk{s8OU!VZ70SzQCmQRfKOp?|WYG@IAkuKMUeWf@hhTu;{i}
zL?e1KK*Fk7bcQK07qPScpsL}M>-2#wlLGGCJGUR%w#sL(*S_OhG!w{_oebQPdhNFi
zxW9NTBZjI+Jd-Ld5kwf15L-f0o2K6OtgLaJ7}xHJMSZG@_b9Kl*H^PeAUjNx?^KYC
zY)Ma2=EdRVG?Kx-#^+#dRP82{z{ue&c!@RQ$MON)VtGq^q)9TH1P*p->JYFWRy243
z<@ULK2QI;I;dq{#&Wy^-!3cZn8-p4PjWoOKOa?%J>n<e1T}-?nM+*oSRURrNuggX#
zwSgDu?Hnty<-%*>UG|a3A;(ji^GjEbCrsJYOzyNFd7vhpBI`9`6xK7D=ksGBge_0a
zK&}{XS3Giqt@9)yzxDC+YdYeS(>Tsl5B~5KTT7!Fqz(_#`i~}Em&>V$u~62CwdN*O
zTNJ$=-^CQR$#b+A@0~)YKK`@mN@uBrd=ILD$^*_q(~JmnU+pVx;b|beY7()<B;#9f
ztdaTfL?f0Zl`z25zC3n|-z<dKWniLv#`;U0vn5;nLRqr!^B@kIA(X`3svGu>ioHJL
zgVX-BYn9H^rSFa}ztAOkR(ceNWjYTROpqDf^vF`nIjyn8g^kepe0le*VepgM(Y0Dj
zS@xo&1b@#9*#sREwWsg#u_Zci^?K-JPj`7O=J&R1=6ujs@8e26shM#}&JMVr4+i^c
zrLp0kh;Wz9-5FKx6$=N^t>KTD;NtMmZQH35tyMAU+2lG-+jrsSOlR<nE_0u(t}N+o
z6u5q@+)*Da$_|i9NZiQljp(1R=BF#RKdPQ7(moMP;Pg%Iv{9R1+9GvT7h5tPE*jv#
z8&{8hO?6UWbV2BCbHvd8tRo%0RF3yGv-;8`k+H!m7U%1eao$byPNjwXmj`L3BsH@h
zGQ!DFa~B+pozCl?>b{vmCp<=-^@248CMMk3A5T<yHmSbwCY09eye@n<><+wXzMUlM
zG=>Ks_6a2LFw5`geh#xhdpPWY`WpY}$NEr74BL;!_`X1~^AZ-s5~U{ghLYjEtX27s
ztF_(<j3#=;+B*$nldjvmuVKJH{&idk0clO!+^3#1Tdt?HmvKfJHxV$r35eG3jritg
zaDEu}|Ll|gMF`M8C@#ZmRgcvKN14&`V_5z_%Dw`u%595Q>24$?1SzH2bSk2PgpyLy
zNH@|Q(%lV;bcZ0_jnXBElyrB!wZXdgo_Fti-#*_t|1sF>Uu&+p=9+ViG2;~_hYy}?
ze6%jF*_Qi0*}K~n=?7AvIZ19z47&H(dnT1VrpFXhW)L_G21Lh=3WXd4^?T#*8dB>`
zhZaU_B2VLn3w`P~L7xZh4W6Cqtpj1g5l__V1~g1k$ppw?B{LtJR?B%hf_e4r*<sje
zimee_ujAdN0OHt{WU9eo?cM`Qfe~oC^hL&T8us&yw!<)hVbu)H+D?6>Rgpglz!vcq
z{X%9(%@Q|`aw$JP!uu!yAfKY>=n#x(6%uN@+bGYnXo12I&5YA-kl$ja?Vic`2c+q6
za+`*P)@?B-yT*nw&&TnA`)+^giHI3}i&4Bx>9D-j4waApt`5qU(%=(;k+C9hvp_9~
zF}`W$)2=dQ;)aM;lP{#U<#=)<rG5Az5<jshQC>T7_p9|RbOO8f&RaOOMBHo|4KH_o
zWQ5RAHz#~eaal14fXpUddNK;hiXJ-gd;62uRCY{e7^*TGHjtC}r6<2B=;++V7&u%V
zl@>r0C~!2roXbB%BsF*I<<&F-WB6!iB)Ax6d^4Gd1#{vCGnwIMV<86>yB+a^t%xZX
z$69;88imuoG12NIF4yF5=$D<drTu~tUu``P(aYMtaBUynvJ-yG=9IB(UAbOu|B?*2
z!<}izP5Ts2+q@X^Nj#@I&_Z&M=J^*->mQ#RsctgUj@R%}Qcdj^*3Tjgy2vyLd|$~e
zlv44T(MJFNofvcvdNc9y!t-N4=pN+1gJLj6E$DhQr#Io-9uHE^Jhl^I6Z6HHpbVVJ
z%(66bqws!8jxW4x!^c-!t1!z&E{K~f{1eW=<foUc`9L??FGlq#Jl@5YgF#&W{@4SL
zBzX`panAS0apuUK9Y8k04@FG&`>Y!ZORqqSy6#}6B)ioP2z|mhUE~nCCSFs`WoH7s
zJB`GYt9fCI6AUKp>-v2T-9mg0{my~AZJ+QK^NK_srCRo|w#VvpSVcP!HVTj=!=mDw
zC#eF>GWzKmIk>Pd0WxVTFCvb=^PB+5yX_n|i;B5mK8KX`zNfTFcnO7H)qNGI>|oGP
zKyzRrsTd};p3;O%rE2GQYPAmKf~O4$!H$J3rFW8vB0T+r)XX@h-SDcw4-kEn2O}-&
zjkZ<tt_gSUIl^dT<&3k@#?p}>tRAMoQEJ(+z6i+_123D)8C;|DgpgwM<b=(NO|MBl
zAv1@adW)6W;l@|(mmhiDV3Ylpdg8Q8Ir~Lj`mGu8@bF_CmUhRVTfEn}nCJuXK5-KX
zW}PQXs4&gX3}i%h@^kW;^n}&4#ZX+E+D7%y*aRgyn(z5PkQugR`w}Y(-L|;d6Z|Wa
zn5vG8<Zw}<(R9YCH)t+#RY;K`h4VK^ZWkxG47=HW;~zf4jD)Z!%^$1|zTxPfHJ!~c
z*zIpcOAYEdSy9yET1lo>1m9FsO>d8f>6>hW^*%hA+asgP;k8u_foudMJoiE&t#RA}
z9R$}T-Y0=>g?uBp2wJ%!X#|+M%t{{{bUgTn-ZIXrGTC^x6TiYuLL{@JuqGStdb*{A
zdoK^vMU?2OotNLoSn8$wRAdU>AF|(FTu2-N3YPb{E5`K_Of02R^2xv1AecEQQn}~@
zK}S=8{X!GN2d(RkV<SR>gHf`JyC7MXhnQ(?ao&Cba`v>5<bd`A(0p%lUtcKj8k&<5
zk*LtxiM-V{;|#dP)KwJ-%?nEg)OSe+xyueT89lHp%KAE2EPasrm4?>;dFh8id;G?P
z`Ez@pZA&+<o&>CW!tk{8CdED9irYqD4TAZ1IzT)?Y<zh#T7ME?E~ga1nY5bUExLbh
zAyKw4kxxK5>$dsOm@A^4xF1&OJ{dpS@hWNSMa}VUt)UiepgtykTI)bHKH82t&LPeB
zHuQyJnh_UkV_vf-(Vi$q&zy~|?Wl^DJSw07tMcX~*Ka1&bS~9a0LI$gTES_bC*lZr
zgHa6_R+^T4m;!q;A-J}3d9gUOJm`WUw=>9$YGjtPZm@o^eiGp5MQ$95$bg>{`@&c#
ztD_+>=9xO*z%XR37_dJ-1N{DI0DhNuB?tcf_=DLxPov3P*9Yr+o-}UlMF3FArB9Yc
z-lm|HWi97K$PiMrF6w00zw5YAY=gueqxgPGOTu&8s~8_M3Ztf#dQ{fi*)(#@&1<w`
zp1U(;mY#-;;daj>%^J81fb&IemTYw~==LJ0_j;>LcXBJ)ID8dvVF_1?yRodZ?0Hq{
zGe>6OaaK82IOsB%vi6_UE;roEFhNRh{4{%Kzd8bIuZ6^{X7*hIBG3<w^2(~3W-nni
z04&9fJ2K$Nf>bzUCf=2Bxd&2>``?ehztYYzBQvU)DhRV$hxOi<+5J9hk|apLvBX<C
z-Ohs;`e!Ey&$bDFHq$yFJthbi8V_d#8$>v*-7Trxv%<RocBlOl-PS_z)t&rb685RL
zu*X(YX)M=a^L1(~j%&5d+lDNOEKkH=ZdvIv8Vcl!!%*LVsm_YT7LBd6e6(LN5*U^J
zo|Z-`p4xTVh`I2<&ZvU8*&$_j98|ma^5-^;Qb^wVOT&>NU=5A+)&!BF+IF%#l|06@
z^C&jcP8M`_03gaO3>fKKoOqxz$tYU66`uYbV<znTczH6%eb>EVKicG2S@_v(T|g5h
z9+eaT%B$lXLHdE#akITEKOT~F?Mogps+3{j-x_-;KiyP{dZHW@K4+gbdYA9wJK~&U
z#W^=jKPP|<JD%wbBolLeGJ6<uTd~>xIqc#)rEK!ySpLR2v7MuX4>FpPXT6KcRU!l!
z;j_)Yry29LW&$G=RAsK)>5pJGk*eVEN1j|>oNGTHbj%wE6)vY~p))UBi6`nsil|@U
zFkku=x7wjRO-|TCIL-*>@5;NN5`qlG0Prj=kHJL$7Q)tWB>nUmtTYn?+f$NJaA$t|
zM#>wKE>=Fh^o)-oFAUIkC8a-=lJ4-V6*a#|2RXvkbe}%=b!n;jCEdgITNbQlpw6A$
z+1Zn@FaJX;IO9!)LZOt<+15kx8Yc`p^){{183YpJAHC<}JFPaxCqz+$Tc106$l3zi
z;}2Gf#kM_)h-FDajA%3)$U$>-0(d@Nt6ilzJtR!D7&Y6M!W$d(J(%xTmK*zoVugn6
z_>6d6y_oCY+UHsLh7hF?eki|7YWK~3CHse3e!-+0D8Gq=v^f0SkWU!Z<F1$bNxDKP
zHMWhOPb)zG7{WOll%%&8n;(*MzZKt!F$c68V&+E`m#pm=#2j=c1Atkm3UFFsmZo8S
z3E;(R8%W?!ICI?elRz0yxMH)ewi6K2UevDc#ejNflamh@I!npn%&Vac^5i+W)c9|+
z^8nTv?R+p!yJT1k(wNB8WparcHCA|CD(E!?UkeV~oQL?HTQ)f3YwH8>P3MP|J^jEL
zSt*`YlMIZ$v{7`3b`NK@x8OZZ_4ZrmA>BwoU^_Q86f~Z2Vij5-X1T>u2NDpGW$i^@
zCF&(K)*S5H7hP2^$({GY>@oZuE}GsT+o?Vz!T$CLtDgeEMFQf?wrl+K6-c1G4`ONM
z)Adh<W%A74yf(2nFxj`@JXy@Y3EP7nkiB)s+ElF}9pHAOU8DN`lOT(!{Ks6icemde
z_@aLj{J_&2<J`owL9%7XZv8`MF0EI9D52JRw>9Qf2LMIJhx(8|X^^JKfrMs_=$zRE
zSi0tJtfcA5_>o$x4tm&U9h<m2WZBCI3}wZ{mz>pCi;X)iKKh9~pq4>Ih7)idU-UUP
z>B*dpzB*o0xsmV&`9q>;E~sFy0(5AQ{`bdSnG=C?dFZpfwUg0ObL+8vp}mcmihkvJ
zLeZts%gfzQ{G)8v7Jg(#D0W{@1X{C-b2Hj>!+IlJLLeO~=wh`s!$^J4VMR!#kPP1~
z8doo8s`K~U+!EJ##RP69e$&R5aa7;{@)GBYp}<6{nvY=$UMg-=kVC!i5CRe00Z2Jj
zm|bg=vi6}TUt$I<9)Lhw>=y2{m+5Y=er`HNr4;9!7cp<2|3dd|h_J03)_hiR_3rFr
zavnmXi_l(e^csZJ#R?<{S#S2n)_}$X<FoANZ`!QEa7mW{yr_?`GN(pPX|DX{zVb(N
z%|2cJgRvi8Wy|~{a^>psPYDi{=I;0$2B_{n*9=OaJKzEe<C81JZRn<#&A5_G3rZmn
zn*0M-nh;3UJidG)Pw5i%o+jXhSQ$#h?y%AxdAe@&M`Lb?Twrm$Y8M#!NS_zuVXT`~
z)b>hLAp~*|-@c#Hw<=z}!y<I_aJ<?kji)&R#)GlG!amPk5p7Pbm7~eTn|Q{Nl$guZ
zcw;j&tXN3Jq&(`4;B7@m<29xFcH(xBJ(HGN4kfcH5`weOYb-^n39KUf{C&4x@)BgC
z9xaAtJoLrr11&CeOF0j-2KR`@Du^cIO7$F;00U)(0zKHX9A%&eq-%#sQ~~)4cVN~N
zg&(JDf0Q!ioJp;kTUzMcTHOAi0Td_JjEh@u!PE1lI4C}zpbF-0(qfs^!3=_mPjC@X
zc=#(>e)nQsYe^Og70GtsM;=Vx9D)rbnc7b2)JETtOAFZr^WFCSlvBCRza{ZQvO`x3
zvaCzy$H%vABDec=B#yUz$#3(|j2$uDFa7ZBbhf-X*^>Zr=`8O3dYZ@X-7X^Er4|iS
zF5R1pi^F<H3lo<M#zi<+nOO}DC)bI2XFJmq^%Rw_q1=}mQcY9yhZPZt$thLA9;`B>
z0wc!d&Q$t89_H_;Gh-;5xwsgrd>EZc;FFBEd}g6O`&X7ItBw_GFCkW-{UWRGg2Jd|
z5&Yg;2eh#&{?{ZnaYQ@XOL{kC7aj>LwzIafHvGp&Mol~Q<t{>B`C(<xY5Tx7>Led+
zN<spqu}yi<!WQp$LL@TS=Zp-Gbmd-*p5*d+Lq={4vDIwWot{<b>s>;r)-Q8QU`Poo
z?~)Kt)Ob`VC}{`Hd7zbh?jZ9bHe0~SishwFQ0LhL#;WW`eZ#^!gK3l3c}Z#B5TD4?
z$1`h5R#kB;8Dxh1bH?5EWo1Lh{jL{tsRbqzWlvSaN}juLYD&gokF+op!b6Nk?wizg
z?w2*i6EbNJC6TtfDD)!>@&;QiCB0dc^N~#HzUA}bqR4#KBe7d11Y-JvL+o<6tY{H*
zk@thNzE3drB5Qdl(0;_q4T_%zVi+LBP+Or*pK~Wq1U*1RDP!6IX9C)-Mc?=hR(-0K
zi>EGLMKcE9ylNY$HYUjkz3pX-!ms!FdeJWr{;(lV*T;ou6}D*2M}(%xQc>QJm8g5#
zL~#@P-jJ;OU4<r1#AT?)Cg;Wb{RUV~#}8vq-9atuZhnv9+J?UIst&u%Chp5qy$DH8
z3w~Bi-ewG>h^Oz_HJiL@P7n`MnCp(36guBxg)b=*evXNXvOk;wBp%u!k~!C}6E^B(
zSV%{cNd~TGN89SeYfhkztXNhAy^DPCoM52iEHS>#JJc2YRvu6Qw8}A)?<zz$00PtV
zSs$X{_<?@>ZfutRtwUAgv%4W7&rH0E+Jsao&FMk$ey2CGkIeUIBt`IvlJ?-Yixq;{
zqd{%7DceaCDS(@_D)8(bH4Srhx-ZXLs^4$RYqJs=Is>RE{Lvp3sVk2GbLp_}6uu41
z_AcVkD*<9&y3rBJuebWoSFq_QdAy94za!k1-Lwb5yX~Fr-G}-b7tM)yk%;~9h0H^=
zf<KWS_^0`AUeBu@n!Ce8WaVV6$lxK*_lb6AYg&HWE|w`8^iZ(UAr7Xu4<mpEimXA`
z`IziFQo>=l_yGG8PwbZwe}c<j*DYbqwg&pk*Bee-MxScnIk|#O7s_4_yY+9JT{N-E
zptcFi1&BW<iW(|E{)%FxnvpNQ1Y+ipwd%K^$<s#Ebiy#>!<wpT1Qi$F=A%|aEPW5L
zcm)76Geu^3at}--Xd6WG4t|(Mv$w{6*J|VN;{3E-d6^BJ%67u^)4H)@{5k;Pgj-eF
zPEnhIRzTjS`!Hx@pl-97cii-30O1{IF-OmQZ`AVpUGH5bT!{|}6d@33q~p<efXZAG
zT7J8ezr?w(UAy-+xR+r@L6|^Cg*4wX-fi_p{*?UlgEM#n)o&2^+TAIue!XX?n&{Qj
z(Rflyo>SX}++L6^%WorXa-E$^CMVIlJGMLX{Zl=?GvZ5tJm25J$zxJ7ijHIwJ~!51
zP}FwdKWDu;EugKMfH0{){4hoc4%w)ndWC_@Fz>S2G0);u=#?fqZ$2lA(G-sm=XYy%
zfb9Pu0(;0e1|Uk>RB#F6OQ5XOnSJ@$SG!$VGidS>>&3|__iNXw-TqI`#HXlf@-j}B
zJX0#Fjk0-?%4TNL@Tk&&6#Y7c6f4Q;VElur2?sk8IRXZ0><e1ET|eJ0Jj$CZ$nO40
zW+nC`!cIu5Qzz}pX(Us!=O%sn<mtfyOw}Fu^=$!L@gjvJenvC^H0#@rzy~rONy~u6
ztIT~*F@LWwU%j@=0BixNa~=K>iuWNx{1poWg;@2msRrv$d`Ht%2Oc0HY<KM}xZ2rk
z&?=zo1C@zBZvu6~iMcLHU()Z9(FL}<LsN@C62l9m0CUqSuS|Ri99xpv>;<}Tm4Z>p
zO2pGC6nzK2-l(^DW4lYrNz}cKgH2C{HU&lO8|E-;PTqMDy7mZE@y{ScHCp+Nsycte
zzD6S)vu#$(YU__tc>_?>X->X9C9c5GQ2pxNneEO!Qg%|<>(BFGP{3q%Wa`?9C1_{9
zN+SsG2mdsG>72^h;i0TNRQo){(<E)T|M;X91i9ZnG+wLWs5P1X9?yz9$qgV7Z)b9c
zvT_Q7uq!^8?i>8t1KYa5-M?K8Wbb=xEM@1CNL;9l%FB2|T`l~$H<4%X)_P<CljJ?I
zmuHh&%u<BdzKq=f!W)P8Oi6pfP;xJ^%%s>RXPFW0Ys5pEex^>FT9Evcd{KsXNs%fZ
z9I;%AKQdO5fi+?`ji`0DO|$uIVREihT}37(uZbAM2|?siO4=%G+={9ekJ?VsUQJ=Y
z+qTgM+-jn^iHfmx&~kZgZ+WntXCYFyhcn{w?V5!t@-s8TN_*W22LM}M8U9#A)A-46
z$Km4Gf@fF!AQoagRupIuRMdvZm{>%UTtvt}!VeteHp;Sk!2@hNpQfXdOOF}}{?jS$
zH<oYF7aKhh^>q^XM`+Fu>rKx?6)tZ(t`!pvUhr=&*{N`BU_T}eV=bsBwVU5UB-A+V
zKRp!Cl)NueP{ShB!mQVBN4&k1&|Np<iR@n(1dakLAZq0@)dv8)1c=i--^qMQ=M3IE
zpB@mMQpOR~w72-gbe@9#8(lmmmY3SkXqU+Mh-`#F3xUL$=oZoXEcfg{m5c$1k<~%R
zO$#q)jE9D2-PCX04{=DaFg^U{9hy!9v%N3eUsE1C?dYp;?mVi_9?X2Sn7`P9D{T!R
z58io$2z#?_BxZ@_!JRe*-6^x^ZE)5;7tKnK9Mu}dNU2%~sw}s>Alvg8$pn^9HwhZ3
zc;INUlIO!m+C<1{9b$LnMc-@-+|X`q!<--km#V&%;oM#&_&auwCoQ`f2Q$4DNl92n
z!gn9>x^4x`&bEH34V@ya5?m5Io{umsQMz;nEJ6AOno-ykN{wp3nn-%Fzo`A-er-Ov
z=#yBhrV?qzTOq^wHsPu#?AJF1%WSJ^?#Yd*u_{eDO?XJZ^Y{pl1qD!(oa@)#H`~xi
zDedP$;07N<m+RAD?L+(~?ZM4csLiM5XgBmNh)m`z-e$Ya$4Xt4c<U^dwjKYUl5UcL
z?ieHxYq6Ryyk-`?DnjK}2LcSkblEf$Eo}CT7HIFt{{%qDVo^miCAW0KP>jyZyhk4l
zARBQ4C!h~t?rfJ8(+;a;tIw0S^B14%Y6cm+n;dw6Y_t`9Y?mbfEK*s~!AuO3cn)jk
zzB(xZZ(1$rvLrj<A`ye>1)=K?_kW<IJ+onQjA)-yWaDwvf-rdjHp0GAMgg30iFp%o
zM2{rNd@;i4gT!E!p)_Bz_$X$y4U^L_AdVtGJvnNUu+BNI-3w63rHf{RC!m6~1WE-x
zJo&v;U5xzq#@r5??>))1Z+XQedZ%0QKG#!|OGF}Kl@Lh9r4_!pup2J(y8(IxP4t#0
z+t1gyC8C*nJ9irau?n~C5~_Grys5UWRugG2)Nds&F3#3g={yK%d9C=;h_+h;n*Fq9
zLMK0{L|Jd1lUtjNmw~l@^#GZU@pT_Cskd3WUN|Ek<=G_}?!AgBzFlG3IVmj^WX~!h
zs4N(`8Ue23{hkQumLyoGsGDrDqgYeZ1=dXtOCD)8Lf<BQx2Q=%JTZ9b-d}?B@@U@j
zzycr+ATB|(iU8Jtv-qk~K1z%mZ$2zTK34%_!#x^i0#;gixvPn_z6il$ww0R4l)t_!
zuPL4?gnwWz<P1xSd)6zGKcc(ifJiG4NED3qV!2AvPA1zXG7qgH6zm%xdkKV8k?*=q
zlAESDU7iOj(~8Bi>53;NZJ5-bAM};>^XHPP`EbYOe#Avn)09jUI$(LT*hD}U`%L%_
zBe_lYU2qBcvD;sNbZ6nDbj}sr<uTf%s>`^2sz~}dInO|p<bLRgB>(JlH6Ym!(jVIb
zK|sy3Q*X#t$vmc!{C(e50okwSq*`4<7jAFJV7sY(i-hRVYfdzR-OSt2pO9b@c2kK3
znm4`0@a&qa+A~78Ns^E=qIjO)>9tV1&g4xn#DD6IV%TT8!D0GBsQt;9pr%tW?(ZA2
z`w9`PNdKz?($jQ@6OY8sIOu-Cy527jA`@t&ZOw`4+yfi}`&pZRhg^Ry4l+I3H-sG2
zk5i)U1B?V!r>1%4{Ba=Fnyv&^uBE2?Z=A%+v+$yxKUE_MSTtD><SS@GGMa)ZA$qT?
zbxHz2Q7cMdZ|HweSjH0v>de@=c{5L#WhV_+>^$1z2~`O1n}TGs`>HocLVYNVGi?Hp
zY6weEeUSH-k`)k<XkUHjIL=R?LHeC>N&RiPP(o2=`U7uBoeU%d66FrGJp`4ib+V(s
zE1<a2O2!kuCJf|?tq1j(x_Eis!nzZ~Ho>RcN~Nj(O~jCrAj`U~kO<X7t}zYRT?FS>
zb5?AheOYS=w9f0-D<<`HqEp)Am2N6)up${@aDTf|xUBa$7xBw8tlc2aiH^9<dOZ%x
z>Esl8bhOmDMiW=gLI^rB#}@yavF&=TR{w@#A)wn}$69>|AhHN3b{ZNOm@vp9xiRue
zxSJ2=<SAr7owo<I6U&As5H?|69;+UtI4cA#vl6s)J$AHus%UIso|JxGK^f~x4un7|
zNwiE(Cuc5h<(PCf5$hZ*CYT{C-jrtVO!w~c7e|DNaf)3BFjKLt(9aZyo)`JgR_*h2
zCC)KUXMJz^`8jvqje#4<ko_Q~UVz2(4mYVc`8<LE3PB~>ZQj@8zteg|g+2DoPVoL`
zpB5};4L^q?9uU336fQe;`Eaw{%yKyaM!RDuEx4a;H~|IyQCN3YY1Iwq&Uu6b1{Yz7
zbk!o}whzg7XZQYbF9)nDIRyo33Q1a-&T_K1zS}lHyr!*I{;oo&+%6cU#BwRPH%>O5
zHz4%vW3|CWp)*@o?0g&vpKCe|N>b?_rqTrT>RN6f<Ydt$Kcsc1WhJ(6HAtu*fVdnN
z#vQ;<OIA<5_-N~TGCF7?H#5Sx^$=gE(K9Tl+vHpgwX<?F<C^O-dSJc!Wvt4=T2o+1
zSjOt?CEn2d1Z`lu{oCJRQL00`@aNqafp)>{X+cV}ovYik<@N1Z*Z!t`u{P{Ca|<Z~
zn@(Re=wb7(AHyX=?mR{IrLXquJGaKBcuWO8*qojP`f>Y!-_feae^dYH*JfZAQ<`I6
z@*j|Knfhp<-e(tJ#|u(0TYHXA;tuvl(?-p=+FnPRJ)HH1m_9uwUR`X;%(1}ng4_x;
zU0Nk+04c_Acx@~&WIUGd%0>2GG&Ga~#SHpw_F-tqMI<^~j9_*uRWJs`$dW3g=v>ns
z48#dakd3;!x^VoFsq(f%>W$SQfFmAnx3ATFt0~xe6tivYA*Mj2)#(@JEu}e3)Q5I?
zMCW7L7w;e&#R%(-lT*07Vn#>c4B_4?h~&qhRCNGUbest{%<(|){GztLJ5>&-MPx?C
zmjs?g2+ssB#+D=>Oop7yk|EEEu$2M^(kFzYw}?u`MS#wb*|Ps(e?}!#Xu1Y-Q^93U
zm&6r_VcIr~r3m>~0g)`2Za1a&ciVS+h7Pw*Sb+X#B%6#>Xc59_E;=2JVh9A36-qnN
zUiFYy)K-}`c9Y9oypK^|X^;^d<@di!*5ZOxSHPCCji|m%;eaBmcrTCj_?XSRgXOC#
z@^h3L>!bAeofLrHl373)E(S7N7&Wgcj=En~S?WF@+Tz#+`eU(n2htU%FkP+t%P5gk
za-GvAJ?D^vJkK4V1(4!pd2ZvYA3tvnzz@2Co6;m@t{}x1Yka=$`#O!c#`GI+MPoF1
zTZX#Kr<`5Vmcm$OT!b*ro{D9V{6ra+E@&r?3G9VUQ7BYjPXy#@gI$j&2IB*|rs-p^
z-bo%hDdBT&HF&UUPAqqme`@x{l+GGVxjfUKLv9;!{*wL(qs^hhQkrj_BZ7TH2#`YP
zcRiNAh-riPoBgg(ze-4F>jI`U=W>^%o$26F)+!*3#Bde}37j+_Ox6Ja03Qf3d*eq^
zFb@M988b3ZX|IE;lI<S=JhZgfj(RW4esPzx&jL1{Cy3Z;O=lG|d5Q2(ekrR8WS7o#
z`-@!-3tT&r*PtQS&TUv*Cd`RN6)~n|l9oHmKL1dZDZn0c@D;Nh)Lwns-b=hVTm~<s
z*j28d7rFT%aw!TEH{u4e=wo4mC^kLLmEv}`WC_7EvAdnVHT^&mV*6lqvajVS?n1xg
z`8X)f<aGiW80{3zQQer~A{$uhx51lRKqgWbq<xA%)UNL`^E0<!u5<S`N#>g66F^e;
z`<ALf@$KD1-+f7NMfeav{N#fqRG_m9DsI%;y5cYSo=UA^?7<MO!~<ey$MWN~A17w8
z=hEn#qUejFGh@kE%%hmENyf&{LZ#qo)6K^EAOVI!ADW1-bZ1tK?xl*kuxU2IK@%R2
z&_6~wJC47ZITwW|GR2!I;$B@V6&1i7dq=P0JX!HWWxrNXL<pG9ZS|U&%Of>}WP#78
z#&2YN0nt$%s7>l&>Q3dj*PnqjLps7K^1A(D?VE)v{S;jM%MRe>nE+br6ZD?bQW|%B
zDuFJb3xKc(a$O2Ro{!KIY_?YEZ>um2go51IyA}z-)V73lGmS%xS2n})=~~QyRuJs3
zI}fCDBGEY;V%sTfQRQw1LirDRfjO+1)?YJo6Kzin0DtYa);88)Se4bHzk^K2*sZ+`
zhd`FrI~U&{1w0Z6kL@47;Lo%&y_7w5940m&xnU2`rloi9{Wu0F!xuvlQCXim%g9IB
zn?PN4J}Tx_7&vIVRA>0siabf}=fm{)Cs5MPyDycPR`S~daUJ>XbJH<-=0hJ&HBa1z
z5;B$p=IuID<dvuj>o2O`5FV4-j%iLW#ZP?U+7BQ;r`dX-2FPU$hFi?B`i;1w&o-xP
zt;t>!7v@x4#_6YgnX8>^A6UEXy!WN+QrGOF^0~QWZbL_88Vqdho3;Cfp9|;4lIG^N
zGsQ#5KA{wRdM1MYS$DCAXdwNO1a7N(KjBPa-9gS7%POM5WJUg3O?RK=4@zHu;#gW8
z-MJY6n6ca+R-{cVB{#3lX{<Ycy~upIzn1i5;%5DeTB-GO%=HE*Vw$WNgySDnQ@K;D
zCIbwvf$NyGPIN;3r!QFuC|>nEj!iMgyx*2O%_=IeMJ?C6qX_Hx9QM-7YFsezHoL;f
z!D{Z8sblPI6PMU~!&Y%b{Td#}SflCw>o_H9<qlEClQt@5g3q{LwqmYiOBDe5ovI(-
z1K#;5uccoq#zc!t++OAUW-s?hQ{8zREe8h6k=OjRTwUnvHrl&h^&)fkIBD~*T8X(?
zPli+?pJLv+&9(A6St6WWj#=NQ*Y<1WVAw)1V^0J?(qT<G?jkefVdI~DeEC|P-1)2+
zjhLgOr}=n3K|*<7i5f5&?f7$|XJiZ0{BUX8LWaWydcpvAFl&TUuugJ#Upqw2F?Oj`
zHNnH)4D@WH|FXWnv-}~&-=A>-N`mYcU(tPZFhf2^cX8(4!o2EKko^9|PUq5%Mb|(g
ze0TftDd4-dS>HXD&!b_e<5-Hhs1Q96+Hy+-Jp0QBlg2W|^;y1;cBd);Aiul2u;zaL
zsF8_O{lSEhhhwymacM8-i7_kV!o}giWi2Da=NQ2Jd072s({%Bc)hk8S0|%JNT1P!{
z$Jlz-H&j`#r7m_;E<1n#CG(2Nz$^TX2x0YK3|U$2it*yGL<@VrJ%0Y}myEJ!0Jg6y
z<;~Bo6;!ixq@3MxX7@5GFV5!|*24|%4lUG4qN46j#!5CbUQtDaEwJdK?WYTF+(N`3
z@%Zycf7Y%kEMf8|A)#rQ<>6mJgSxV4)i4{Q+g1Bb@;e*5tKrhy>;OB8j)kPGin2Q=
zA}4ki;+C5FsOg)`41!6QdGDc<n3L_OaXxZ#J+CR8Gs&lhSPmZHj~0n2+^VILtUiMh
z(f6MQNAK^N1-M3FtX3+rmm};*x)m*gyx<iXf>LM_ZZnn4jjm$^ne&~3FKn|Rp_1ZV
zCCwlq6IN6sjgAW-!j^@dKY0NH@yE8ZR4uk9ns%fa+Ou&chL^4&KUcDp><CZ%9jf=#
zmNhb$Zm;`&^Oim;zIIQ6+0Dang!R;!FPEK^$tr8l^3LGvFAUKLDyMML$q!Zgn&Ood
zs!)uEn!U4Pkh!9wU5AGon}|h4FZo#ykm(K55Mna1v(IGFOd+=eLI0FBqSq~n6nS52
z2$<C)ON;dOetbzcbAL0Lj?cH-bCV5(*<(xblMPw;V)ZL`snc)}k}ltpJt1>lN@SNA
zB8d=E=n{luMa%^;3klW_;rU2Lyxby$eIM3cp$iLlt_w8<K`k(sb90#A9|jhyqPa4I
z9;$GZx7#e)TpiD=TB}?&HPD*%_Z=e8i_u6ppIv|wMP|^_;v(OB#F40&&U)))-=oGp
zSFAprcq2f`>0P|=yzpgVQPf0-!?*3pYLCa$3@OwjdNa%(u(v+heou&u^}S7|G)2$7
zHa8<&kC+h#)8c3MFKZ7bBtV_$(mUq>2<=2l2r;9heI3>g%z6XaR6?N?_(txERXGK6
z{px3QZ>nwA4Fl&X_Zn)}3JKb<3ed}3$BI^a2>Z&{O8a>4alWd`mkl8oqq`x(>Sf58
zRf*o*>~dg6$<k`y4Md7!8%X2LikgVuQVz{U($miHTS<m&mjjJ4hc7Yr)fh;Y^;vRE
zZfe{2#IhDoIyqf<zs0-rg{2X9<f!cuVA8Vh)JRw+UrI=f##&O=(Tg_3)_{zTPK4wI
zb!SujjO*ps9ukTDQ%{E;WT(2P1qqjY7#l*ac}h%@a$f8HFXa$$69O@Ke^5Y+LJ<)G
z4nmQ@jj((;9!()Ud<WRcjjnX-b@(GS*$)~5m}+G7dSb&k$uPr(1%GYPDn_+8hWBB8
zBj;;_6A}wAchAQ0clMl(=F5xmhK{Pte{(uhKVFrO<9+|FeB8|+vcQ_h!TvW@>-u>}
z69%2cKasD@dyluUN}ule@M0amL(mkvU3uj5SF-Y6&mKb?hW*}`VH`Pc+1zK@0+pK)
zM&HTGF884TA|G#15rUa7RR31%CEtp>b~6LuTh-VVFZEJ@dMvnHQDO4m&<tXNk4JGR
zx4yS>`*GCS($-6O%$NLrABzVrgHVmMj#u<*M^-_Y)Jh02kt$rE-6Top$ECKXleQ-7
zvw@S~SXT|&xeObol2%p1UHYyp|6xa(>3+wU01p$|V<=qKx%rgRf#KsNYjUvAterMR
zJ%C%;y@X2lXf(vp+mRV@z!=Wlc^m>MC|bmkoE^DGpozZXR+--nd$Y!p8Zjh!L+Ro6
zEBX5z00_{SFzV@XFYowehDN7q;A2l2GNY5j!-C|Su!hsxK?YtIJV0O&t_=Y8a}mj5
zJ2KwsYoAV2KCUJ0<*WfdLM`D@o5_@;8_C_8qBh2lO8g^ZK2Q}nF%4KV=_<II^Lik{
z#|HMgYI*`U78WpbNd^bQ4I580+Pmaa5fo%cAQ*bl_nz-YBQzy|zsvb;Q@^kj68EWI
z>llTnYP7TM6XL*_3}6Icg|`SO^Q-`IOVEdE2y+*w1%;)ove77RV{Dat>6<`-e6Czr
zuB77NB(S;;!kozqVQ0Yqq~Air<|J%<o>HfN$}E>*jr7KXe{#QGSzuVzvh#i0s_2+y
zbDPj&tTr53`Zw%;AHL0+=C{P(-qYl@i`vniXq?$CAIH?k+Mb-wSgKW6$E|%F3^owh
z{-%7nn6O;3KVq>6VB=M56_chF4)!LLmZFZildA~)@gN`Ml<#<Uak)3x-$DN&F<jBn
zkb!j(VmB#qt4n@?XP7z9!t68{plaaye_-Y4pl<1T9V7P7c%!|(*J29L9nU??haGUd
zjuwE90g^m?4fnUpq2|Cif^%H`wPVD%#)w-RMsYBrY|u(b0b^DFvkT(^HCK>JNwF8~
zi%hxX<{tuA;I4axN0?SAaCd%jBP%Q>H`Y)LZ6^!;1n`b+r==}KBz4yZmBjiyiNdeA
zkYK|MF~NXTQpH_O7tjXd<J~L5H5~>_D_g&IE9$D+KWIbjyEO>9nlOw`O}<brSRxjd
zEAJenjp-KZSEUbd&CfHu1@;$pD6M;H>w6X<GErT*aD6L(CLXjyz=BaSvL%xd>W_Rj
zcXZfFc-ff6$`eS)`at7j&<#vDyNDxl*RwEE##Md7U6V>RuAtEd!UJf=&R|f9`961?
z(nGFl0lmr#J8oOaWHKWFAfNZ+4SSU^l3$7<Jrys1`N}Tw2$aGEq_A9}-K0x~6;z0%
zF`%q@4zmKEEp>C9^$8n~%J(y%9NLEUg%7_7HbVrE?3Ffb6P9EB@?16EqG?JMq^0E;
z&6Xo9q}@JR!|YbflbQW&#Rh;e+U97O$e+M5#6<e$^^AJn%@0CJUwwaM205q{F?J4X
zqkr^>GkHVUtfZ=Fr+vcOP73G^YsBnpWjs|7PgWl88Jkfx9((MUuIKykcSs49s6Mm?
zSRN|DZp&X@d@y7<JZ4aI_fr{cy7bm$T1*7SpcKADT{fJZ7y0s}?ObvAE&dz@LE*hD
zNDZH+i-jlEV={X=k6E)d(XPzLA!INaHgJ)b!47rqv-Yg;NFSvDih}CdIEbqT6wvPw
zY<W=H%EPp1ZcV_=DoU(Vom!hd-cha<>N&b_1Tp?&9ni$%MfN!!XF~)I{}<z-;*2->
z<NOrn?%NzK4l~Y&Nu=ftH``*zqcJx@!vt&!3R1x{vfi$In*uT;P@2}^FSGuU`hXc-
zcVS^b?h&>;ItxR4P0i`Xr|hM-w|n<V`XvOEv%C+}y;0w3$2(f^tM_iKgCx>U8gcLG
zsq5?D<lEs!8xA|mBz$W>&cAC})i>DrblmblsYfK(+h^O?U;{FoQ4;cN`?bN9@hDI=
zRqSW8>M~Joj_k>WpT!OZ*AIl&E;QfGylM^EU3#b!P7KsD)=K+(kohh`1W$hCdGrh=
z`mes7;fpmH&Hj+Qf|Bo_o5M3ovq9Nsg1y)ZI<vgTe+I9MqS+f?Zq#d|Ki-gWr{;B%
z^}tihU_maTv}k7B$6tv+B17VvJ&u)7@~jbq)5-GX{Mqj1Y|v!U{+Moz8kk4&C^g$@
zp4sUlsY%yb2mWSf%1Hr47;65IiOoBHHs*!(Jf*JT-NO>H@|zKU_B*)Xc%T^EMHfe+
z+6Xk1%v8a|A3|daQ#;H*VSmoayXz8&Y?!Qc4{t~(qjmQ;J)u>c;>U6K2v3ihHYa-J
z^)I)Iq$(}i=K-cb=s#TNld;B(yTvSb=knhTat7@fZV$9Zu~|2W$FXJe7dpkp!aYef
zyy7pO*<pd)@33;C9JS0Rq`VJgZ@)cCwOZ<9$a;AX{0Blt{jK~xCmhm!+X(|&ssgud
zy1+n63?%^jR&*w;Q`trjd=gM=vc&<5$p`dQ<f$A+f7k#RW!qffDBXq~9B)p8RySiM
zp7E>zoX;L1EJ%o`lWtJsjJ-q;`DDA!@bN-13k!V?mF!XaPL21f31~+OY2rev!J<g?
z8hf99CtlI7*-;6gz$Irgrnd?_>o9l3qMoE32{jraqs?iuV44yR#!jYdDV@Q!Akm!c
zt*`o7hkM6jiD=as7kWJ1avw#XYF~NLU{eilAqmGOpb<o7P=FlWOLAELq@$GGd&A+?
z+U-H`WGgF<Pxh6l^e!4m&0rUu`FAzd<Jk3spT)7gf9*Y6(WhCdVpdzeR=rv<&Ca+X
zU{R=5wNL>f>UblJJ90gh8mFK$NH>uE(QWcVUL3z)nMfD_mlNXVYWj?+9Dr~p*H;{}
zmW$ua!a;;CWhkY;eiECu^Y@6KuUpjrO*ET$O!0$@buyl%K4_d{>e^@HA-ZeDEz#%E
z2X8kh|9kW!T#iF$FfLp`HvQs2sM<h%K_QaQ@(_}{@cy~cZkI2Vu+oJ4C-j+BX~6q!
zQ{dq_HX*xS;NI$JKQ9*Q`t1E22bPrY6f7H67M4wZw02SMkE=n9+kL8<@&<&-#E;xk
zIw3#q(Wz9@tB3fT-aU?idqTkgh1<*7NM@dH6!^xgf$kD$qQA5NZb`|ozljD3D_oEG
zEh)^52fa>zkiZl#u1=!V|M!;-!s1>Zz~PzO!7+$U*(HX`dst3B%GHsiPf8zo^0?%)
z!_zC3j}SyCAf#JLBLlqu&o4}Ug7ow5jvC|^h6<&)T7sk?-qUrZpcNVxfK-}$j{@8D
z<&~!|ICa6TA+I|3dE|mUnUPbur#`g2Jwv|%R%D13`o3Oj5#wYY@sxK<c3+{;X2r3~
z!b6%zJ5>5*WMtcY7!w};L_^y6+^Bi4IQ+>RWUS%w7Asi+Hp6LUq2KrPdKu(tWjgEy
zpXJl#AwLYT_%qt;VW(o?{3w>jB|0#dFc_@URL?)%o<t#<a`o;>=~bj$gb(+7O7R#*
zBtSybh5v0=7InKJJMmA|1E`@0-$ZKAM)!iCYpHVsD*T-);G9uqa7ypn!T_)qP>qRx
z>WOefG=>hU<h}sFp?{!VJ53DMdB2*ZoeFPcXqB?DZIm7QzVp>IrOjjvSydIHwiCH-
z9`Qu@JM*<h^Cl>UE90;AtbUF>^#FU!+!I~rbmQwnZnms0=-`t^483TxqwG!+4wRYA
zsM}6kJU(_fv3^-*SKRBPa%KAN#1t}b?duqNQ;RYghGPG^re81K3ZXmt<&cai=~wZJ
zuT>W6j4H>zR9uObFKB9IqZI|80oAL}q$_1Aa?Er#hE8?uwreGs>wnYXxfg2>YfH}5
z#n-Nu`ZUEq2xGc%0^!svXlow~_}i!q1ux6Y!>p%1X1KJBX@DI0)V<aP5l!@Vc;q2+
z==<xaM2uT7+hrM0@@>S;C>zQ2_gqZl51q}F?#bY`_$~dmYX5)+O<&SG3C1OUpw~NX
zWCIQG+Z8OMGi`aPnV_5l@{jc$lbbc#rQd!+V||cBdHR<I;ZTfr(%Q@Z-ks2A8DAt0
z0|v_lSLLoWT1a=Ue9<tG{jQ<?_YFu}Fl}KE)R9wAL1QmfSAO9|VD~zCbkcvj3A{u%
z+C0B(c)C1#?wxc-4IojnL%y$|Lt65~&d#Z4&T_)w1HOrK?k6qDc#beoqiCgmtz6A!
z_%*XUPpLS)I=5*Dyq%$&Feq5Yjp*>27K?IzU4Q8QA^w4PN}a4o@79as`Xy_oX6KZ@
zjv+&AcR-Vn3FiNfRM=L9V$3GsYEQH*RWZ|c=0TjWvClOV2?#IaAwQkm5SBaL;xS#0
zl%^yeUL4|IMtl{w4pG9i@0rmMOiw(B=U~<2mNzA;8EJ4tuNh+juPQ1H-Sv|ph!+ze
z`uyG>@pE%s&*iX01MWzRKY&78Er7y7-n4j8SvrL|0AY1CxWfkd#v`m;F<xQ-sH>1}
zMFA~}1u9=HZzi}s@mX6k5&rN=Y60$BKt{Be%$$ps&>5|FxkqBFtia+W^gzZ+I=hBF
z*dyLUM6d1l+6_NwaLC^#h?dldI6GUAHacWK=Zuz;@#NT3(C)Ojq834<DL^paH6g2n
zdsp%gc={J9(Z)--k}PPU?RmbArzA+mryrNz(=oq{+L^1ww_j*(0bi93J-yk^?Ly;G
z1xDU!H);R=dTN<Tq`kFqS>87r#yM)wpHqU`T*S`K&cq1XNDLf`RH|mP&%mcQX6`>a
zd;oJ?+59VX7KfZ+o}(IZPOor{@4wyD@qB<{Xsx!7j1T$=^20-HT)j%Xe}03E5E8Xx
z{&ppA1h8%{)&#-h%Hn4p#3U)xhWC+8p{^ix5m`<*nOQU+99wYmi%)-H!+*b?oA9T7
zk}ey}%1AW_&dywjk@)SZ^>mO)LtWjuKDKa>;TW0jUfPAmvTe(PY?@yc2%Y}6>)+3K
z1W;;sLDlS&xime`URP!%Sba%mMNEj%d@TWe7CoVk-Y;1^Y8h>~0sH)IMVglyxB!AH
zsqaeCO_L?!<d9oDbafeW|4xCWm&~ja1V&D+wkY}UBQH-BZWg;kFg^4jnB+76X7smS
zf6SQ|4ERn4mecp+ZX&w@Hxz!pO+oMiY{mH(x8{Bi^Isl>C9yl+pzBng$9Ig6Mk3WX
z6EO6gO@O(+#DRx+YxOw&J_-zzg+oV)UDxv(7RNF|4#=fGUGC%-cJ=XX&?WKOh&xWZ
zhE5xvaeS19juco`$3f!=)3w5lgr=ktz)Sr5E9{3hxVsuLD1QC=BEGU=v?B*-Y;?`1
zz$$c>v_{GId(ngMM!UMr1&ALLM-}n?c5eUjkjjabwL;usddd~%?kS-&nS@H06{o<Y
zy{S$`Uh}MQrd;!Waqw@OJjU%<ET_4|8CViFgDh8V(LvX8h}xZE5>ch_icUYq=nc{I
zV-@`OMIMrcQLBc8Cx}&sOIFHoaoT&$+>*O0Of5$&SJPkyJ`;!8Q)*`YWm~_$^7?@i
z$qRpE4MU_i6lb|w7DQj{nbF<2SVJ`rL?qu9=*az!y?BKw6bm8GxuT{G(`mZSpzliI
z3WGN?-yh2JFKc(@k>uFjqw`5RLa`a&FKJ5dGF0XDgz+(}Fcu0`abAVjE9_TOXl8_L
zB=;gEu5|t1#&!L_<uSqqMih=@W1@Wun2^aeC7@_@C<_#gjuh#$vTLdQA-`48I%0%v
z9P`sE<DDD>m#;P%3!H~Bxv`(7F@<q_k%~hevP9taZ@c-qR;kIjuTy8J5mnXhoHN}I
z^FiPB++G|sTEM^`%TrmMkHMCW5Eh`uD*K41_nclaBi!L_tR&4ZA-2B{(X5AXCHd}a
zHlg>j#vOc{kPYijOH|vYOT`48f;{4RIm*WHeorL>PL^pQWKTi1HCT%c`m1y-R|{=T
zNP?6=vliVHU)1sssw)jx4Vb;7f}h1li~<Uy<uRcv6*{Qc<&knPHyK$r{LerWo<9{3
z?WO@G$^TrLArYA6QDIVp`eS2*igBxw{R1Ep<8<onkma7bXanMvZ=bayep~Vmg^ZfI
z(T{wZ5Gkz}{~tr8Dth|C$8t_PUnh6t`25FNdx<!a?Eym^e_Bs>0v}l1#-HLa@3|N(
zy2;pdo|xaYyFT%MhorwA43XViAXtxitxulvZf+Ns8DFY2DsS~J0qML9F1mknV7bV0
zF{8u|x!J?bUQxg}vgWR=V6gdGBM)!H>u%f8f7%){Ts*{#LUC&N0R~U;%ckuja)x2e
z-nVJF3*(3*UXoo$)H2LakL&$XPMm|`spwNp$vc=XcmB~5YT>3JKnmAIPNbI>9)IXr
ziYnOkOi|L)-vhi1H{WXZ{x-s#0O7RSk<TiUHv81wl;DTC=U^k1I2ZBw6uTE0G?~)>
zqs_X(4d8GukoW3-`-WpMMEDAlOGZ;uA!S;n5tXODEGK}QN$u_+e68n);|<x@C@S*%
z&o`!)=||@He7750yDzlwQ}e^#aE~uXF5*>zb!zYsGoTm7($NEgr|U&X?Zc)kyP8(3
zhd$`wwX?LZrd9TTOe+!c+DCous*5iQ11`-zM04|&T75kzL{1apak(l8;C=Ugn!$6W
zh-BuF-&wf7&34EI25V@|SyYy1bp>a+#3aK5@}tt;WPX>cx>M3~hE9p-&p6%SR@mGh
z*hAewHNW8)C-{cy2T-41!TMi)J~h^p)MhdC1s@PloPH%iZ0okgAu}>n=!cN!Ad)}_
zDtC2{xD`d0_f)r=>K~>Ph{`4iKXl#<!5^~TPzHYLdzHz;F4w2dLiSOvhNisEA=9Qj
zfj=8kR3cU8YG+NjXqiHPo(vL<NCjp^&B{@oq-N0j-s`H{-&<H%2acEvyK3(s+EQmm
z&wh#LWad5F$-#^4yq+uRH5>nH`PsWDMk_-_8rr{i@t+64;&w~bOQ9Nm*eic`z#stz
z)o|!xZ(*sbNRT&~o@CoEM%wkVH{8W4hpzf;2h`HtCq%BMGL^T^Ojb$T!0MU={`bLx
z#pB-W0Ey_{d`&5tFuSTh+&qu0Yx%Vys>CT<cK=yyI2^dEKog+>;^6*N?B2Ne_e%eK
z27c^@NN*iO7Lj>5ieUv)VyfQByyda|jv&1UaR7f$eBiH=<n`pZNl+L<-Dz{!70C{0
z`-ZN2nuypq&14p}4J`hrS4pLStC&s;o7x+57_eYJx8VyZ`nDtv7`7;JtmSt?RAVFI
zRxthyGl{h^`k#jBO!Zj?!g1OP2T3r4O6`rNM0G#pDu^zmi3aFJw*CCVG4)?Q|0Yg{
zx3JvRdJ{|pUR{4%MJRyWUcTmkcls0@?j!5O9iM%bJ2y_bkdsDIG`=c7Hmx;}syRL5
zz0?GC-rAq%lV4N31CP_#>p$Q5`+&+ChNH+DzkIzfV<qorT%ilj`9VP&Ku4j8a;yQW
zKih2c7=KJN)a@|^19V2`C>O4N8Bf4Hb#uI8=jTz1V$_7A?ergC<IaHnIt#<7@Akw4
zhon(e;%}3$$z~V67|ka)=DrU+iRY(dGccjOK1;z^Cj@bnT>s;T0M{Y}#>(@JP*J9I
z`4%{vo*kRT1=eKCWnOSGWfezeC`S|@Utbb_?!wR~6tAG8jp&F-fm(X%zlxZ>P{NHS
zocqtSaQ=}-N<C+k(S`{<<7l_E8oRS))!A-ORLRk3SIr8zL&&K7^!VvTnH9k|MiPE`
z!|Tm)_3sr-cd;OO87pI1p4~2-@iK{<@DLC!rjba%CmXVP1AZvmLjT+!cOf%_Jil7*
zGi|tXillW44gy1xtEpxDA5&X{Y+NT#Rkd8K05<KJ3*4*fk1u>Ml;~w?bT88QHoMqh
zap*w42&6ioE{c8tL$CbLrvGcaa31lnmy0Nd7ehSt<t-G6uOzMEA+KDcos(E|AaSWD
zTcm%?VCW5u{J!!-GswR4e9<|0hw?CCAn!Dp{9ohIP+@8~ab?cXT;Iyb^AOW`MHl>q
z;3?-`LwkdviJBcvEu1Wq-%ek4(jj^Oy2$d|R-q5>)zUXS;yr_{;2{at42h|Whp#vz
z##0U=Mf5{Dhz!B*;vwjG``5H$BFhaMq1O%%H8(av<DMZ07^pPv+)ruT|J$)2hV#Sq
zsb8Y~BR`Iw-NWrSuasyj)yl+2B6-6`LgK=Ni*caI!O|>IP08W|met=R-oJd_eu(fK
z4MP@BU0eYHXG{=Rzf~j4jhW3|u80)DuPO@nC&pTcL3ks))*=h=%+!lr(tn064ATkM
z<N!UbbUe<kE@8lVL8sW`e@yW_900obsdAOPFEf~7DXeilt9XEL_=<8@UvUwNMH4om
z__NPGPv%kmS=Ha3Y{&$htE5gaaFDwL7Nut+NS;s51zmSzQ!}+6aG)6Bm~X-U`CEAK
z?nO}Vl}T3}6m_*ipRWP{z5iji9#~_q2YoD?fpGeIU(jxs>jHW3l=-xH+@91+^<x%R
zWIVp#juy{Ry42<G$@%##^9C7UT|4}G=L5R|y6@Q7DUL(kAwEwjkuIxPhd!^E`e&W)
zYFyv%gyureEcyZ$Z4>Aopg@BL*9p@9-Or|Kz{PXmSM3NfE_{xNUy%2idmC6|U(RoF
zNZcq0i#+t`_vXqD$nZ4zhse+UJB^!7!$aT<s!<HLnvx#CBN>WK%Q4J2%QJx#8PI%|
z&EjN#^~d97Nx)fN?Jt|`(eYcr=ugC;CxR|^8$wei@~8JjH1}ECe&26??+-d|ckL6d
z2vjo@O3C<dMn=Q1zjHmpYE(b3DhwIn)on5B(3tW+{&}11hCTE{!sc6=Ub7Isl+14Z
zA!XJ}M)phvyC7&DCkH}Qs5SNXUqzx0?S<|xxbJgMp+AuBdUq}SkLVAyh5MvN(v2Qc
z5suE;xruo2Crh(U{w4SQ&HG+(D9^3~)fXU#0dLAw`k(I$zAvj4dL-1uIAm4jQE%?o
zQJ`Hq_u~)sBt}=?BD|)WiL8I)`RiTXx!sMgPTM|zN2nRekX#8{t|MOyTmIk91UV9S
z+f}Z}RWeihPkYAUigEv*cu9>#y1oVYcV)ec9V=q`2YUr-+9JSvQ8{o$41!aayPn{O
zj$v3vU(V$Rk9c1YZsiq%bU3HlHN#=d;a?B4Bw?DX!z}5xHgu~B7vmy<1_i5R|J^9E
z6kvjg1JJ6AuT^;$1bLi-p!O>lxs-&@PT@}c^W3EehCz(jg!0=9{QZE3?w;98+%a35
zK#U1leRCfKUIToq^Ro#Ox+8l%9>l13ud%kqwSR3m5ZOpkL_l|3XJ0Vu|Ht_|gyar;
zIU-k3{pmf2nR{W0JSK=nI)xDP!gT|FUTCqeU@4_n><erPP5gg6(hOs7z(HKs2j!$b
zm5K5i%YjI&<q{kmLQWBCGTLFH<p?;#@qzgSKg)vG3s!~w<N$iKRat1?zp`lCtMlUP
z|FCF?SgJ%N6a<{FGsqOQlhgeD)xCuj{esA=TnlsPGu4crXB(7Wp24ET{l|s=nQ7r5
zoYx~$_(Oa3?;1SfD@U^*ue><ko}7Y*+<NxUMnU4Pb7d4HTy0m8xE6rt2~3U%{P#<Z
zOa>5aGo^SaRu-A;3+c@z)71^0vL83JKkqER%96^+{xw=zQYEH;9BnT)i${FjN}3ov
zWS#4NqHlc2j2tGUGP+Az9&1R6_u98<ueSb#3C=>BZFt{{;`tqN%|C>8t3RgJpHO}R
z^>8Lq|HBuzft#sWDQp2#m@G2`_SzEew4Gu8l>M>c%gT{XT9VAW?uNhY`G@Z#CS?W3
z$`awfcK<JzmxT`(AHxw!b;=nb%WS~3&P5lPnpyIq1<BBqK5Bf%_fRGRjqTc|!&lq;
zuTA3%E)d+$U=HH>eH#@OajSp+Sfto?0UOME1@-@!bq=@#-Yh@WQ{j6r3r$>B;sHRs
z8wwPM7g{acN28XJ_~}AWPBg;;qRVwyisJvli^xbqeO^w=y)1)D7Z9(L*QSvf%E_?n
zi(P<Q5RE@16yYQ~W$^T{@@1xRb6)2=>c#8VcJDw23lK)ohW0zpaEG#CCX1?3DVGfy
zWEar0gEuGfMwBLg78m{)CWxEFVG^tj>9_>yl8vj>GHO;X|733NUnTBx0&WCyYxIOM
z1-hJBSR_u>umGL!jz`_r*7kw_6d;qkFESE$eEE=*y7+x@pyDC)Rq8fS3>W(XrDU`n
z?|~Pg&_o~6Dxzt=j;dt%p#r1dv+o)`_?4RdMhnJ&&au)i2+|oDDdXb*vDWT}6mDF)
z>Mr`IJo7Dj_E8Gi7)D}fYK;AYMKssL_PpT@85W7t{rhcyJ`11IfD0$<dBW5B0F--K
zaa$9-_M!qqJxTbrsci=up%yj;J}O|s;;$x5c%1?&p&opzDEjBJf&Mb{C+B$=VcDM;
zGmA1e4j5H{{_$TiBJ^42A)G_XjA|dzi#tKY|A)0Vfrh$$|Hq$fDNP8;*dr++F(hl*
z$r7?}WjA(NCp48USt4YzFWJUU_AS{-69z*l`@Zkn{~pi!>G^*CzjMC7-#O3eaZWYO
z`+i^dwY;wDdc88Xf4+g2u1Fnm)1o8#xLnqE-oR^L%Mm=dKzacy`oTdiH?0N5>BP_b
zuLlLeP#fge&BQP}9Br#qh>cZqUG{U7zS870Gw8GyGSZqE#OM6b0*cR*B^s)wx*WE%
zg&e0&OW>bX@&N^~?ckK>Xre!Fy5w1;L0w`x`vg)?>Decg-jFN$-#$x{k7~dvzuEba
zEjR7ZfA?@q!qUR;oVZ<jXCF7=8EUpumz?U~u^D>Uc3#SQN%n!^9#D}mDCs=KlKMZL
zHsj^5DN$!_RNx%mytX`cxO8Lp*0PGI%HgdBuDm3Rk+8n9t&2@id9W@6=hKON^m_)w
z$wYA1#dozg0`^yCvO+cH(CY);G)?@3+QW3)UitH({sK)TArv}wrcz+xmJ4vpj#n!J
z3c~k&E?oh04PeLPRfy!V&tV#n<+IM&+!-iTfAHr0BgK>Nem^+o^k3KK@2~@KuxCkA
z13eGb_%ka?c#@K)4r2`Ob{!-->z4&yctQ<l(Ftxq-LHIBdI{Zz?;3&ssL4U^?&zmA
z<S%=64ft&t<C?*nYIVjB_CU>y=~r6;<`RwsWyR%~VXS)ST>lNJ*TdvtNA6j#Yil&c
zo=qh9^C={O1pFn;6bRL{cXYU96VP(IeEKZ^KYWtf)iWgqAY0m~NOF9RdgUxP;%gl*
zx|gpWD^WFdaUey9(0aJMFb-GdfGhle@V))&LH^KYeryjAAXw`K{+$iW@eZ>AVJ`Ds
z^{clpcSlIVC}%Qc8I*g0#a!B-h*a0oieA!e=i|})!1Ciy&;8G=HFF(!!rh_RuC#6m
zMLZ)~?tAm+#CN+8yg>8Gc&axpsTs=)+z5v<O1@6q+cx^l&@CG5Ymh%9&-`xo%2a`m
zm-1%?3;qNxGJ}ac!t#P^^fm<_=xDgQ-Tzk$4jg1MscnrEzUJOc8@GT+r7MM2{{tr4
zrIv`4ETwuzdHS{RH@~Y1UW!0fJ?8)aL5fe}aSK3<a&gok@&2+tZ`Ny<zBWIMqU{~{
z^7=6(4e-u=+kUG79zq}8WTO1XCQ_0IsWjg<ttwoH>JN{QDT03j5pOyfB~EnE^ULf1
zJh4B1Mlzf!vG14k7w=Vc8<nV`U~REA`x89MX+8x)Q6$mc#u7JvhD+b?Drj|L^9<i_
z8LZvq(c_@{@z>nVKVC{Ef`}zW_^5Nrxm^N9_8I0!I%ytSz5jeZnS?j<+DD2eC@J2R
zNAj3(lfZBNF)i67#qz46&O@&e^@lr<{LWgBc=CO=gH!L`|J>0AfAA&CS2afLpL5Fs
ztvx<tU~0!~C%jKx)>}D{$)uO_Ox(Ae7y*Y7QM+hWiGB3j=;#k(F;ByTquZp95^MIK
zg{5>J-#$uxsW~g}a9KwUl@fT>k+X@A@`0o(F?wuJ49)WDRn2_7*Y&^*9ZE7zIV$?h
zKt&o&NiR8MIMKa`;{y`Q<H)W~3!$p_ohstp^Ta)tiJsjS9SiI60oFtmlE0n$$1ewL
zjykT3$<3s_28^SUA$?LIw1hD1;Oiz>cyF(!u<utEc#NOF7q@JFo7%}=t!ASL?)O<{
zfWYpad{DgvLABiev_bPvc~b1>dvCm1p8ZBZ`j21K2#`HTS~5!>f`;G7gLw>#7ph)3
zPhW{GG7a8ds@EmBD5!7~*+6(&QBMzLnmg(>&Dhr7?rd;TG%`54;qra9Up_8;hX4vN
zR^3-|`A8IJKC;4k0t%v@$Mm!6HTjRW-^UK;O$*COBJ2%<>C6lwov#01efN7v4mqs|
zCo29l>m0=^!;Q&ZZg$-<RXiPUW!A}MOPK{_fj}5?I)RIhjxO|ljfiK_LRaUL|NiC6
z2_C{V!(Dh)-`tfA>vDqhR(nfwTtcGk=xdlW3%^+0$U`z%7z7J<mvfA3-U@;9Enn(4
z<2*Ue5z4?Oc%6HwKkE0x9b-N<l6vUEW9r0C1b>{|D*nrxr3k#o0;_pG=%U<T`!Ho7
zSbn5|z|%`J`aWHqMD}3;AHf^aIK&rW^-`OMEy^Q^WADKlMA>-JSgb+Gxr>jW|1=Wt
z$;i)CnR@GLNfrf^nF<|&c`+7csoi2~EXVu&YQ^3fKZ$J6<3Ye^01ZOhv!laY!;8?R
z!D2ZM)mU{aCaRMMFy{i6b5tZoz{L3bhx?vgVOs27*ZY=wSw}-%_@mT+SSE!eOEZC#
zy<(J-gF@6RjH?k_@rVcu`DlTva}n8N11Yxv$p=W6FXH;yU}XIXl5hHuT;eT$>A^{N
z<vx!dN$|~(|1`vBl0QkUlBTM;)TIewUR1Fj)R5k%Yg4CUcdS0zj>dT)p&jfO`z661
z3+v(jX0e;D)y_V%Z|gquQ?>v7Bb+C=`a~h3%J_ba4U9{+bh}StvMrMpVLt8<G}Ov+
z{Wn1#7}CdkUols&l!19U)w;*5C6qC=<GH0X=9bROe_hvVUbQaL-g68IkB@@kNmCFu
zwRN4xr)|*1bir^Iuu%|Q3<TxDH!pXYR@rr&>%F)B?&Q96>LYckw_+A4S7v^{n>!lM
zwyk^<`XiWV8c-4Kg^d4kXThCwx{x`PnKCTGDivz4Eu1aaI&8?YZ4`BF-E!)8yMHU0
z$lv|8ksK^lMulal#sD4jJ3M3wZ89AulpiSn)5}n-<VkMo$&!Og)FOP9_X@YTSTDnX
zF$=RQjyGRCflcJ&&h!)?xVGx~_J+d6?hY+iTR++jnc=P`-sUmoz<)Q%LDJ7ZOY3ZH
zzNB70aS3A%c>DMEVH3Oa9QNi_q4=*)kj@gWDKmk++LJzAKat6ln<P2Wv|l{2nc*zm
za*gz}UXpfq!nY(J@0Rah-U(|6+T$k5Z+LLfa=(yyC;N*f)*}dDHrj8IY!Q>yTtDeE
zW4HE_CA+@l9-ZmcvE&Cv0~NGl5kqVg;E&#V-C}q2HJ#rS9};hXY99#-Jr;R~*v2rw
zpUO%GNTiNL^RI|qe-676Wh(LycYJb(74IgEel@E&LJBA-zFPW{h=Yf08Ku??s8Ix_
z*8{OirO4(GMt5P$B?H|mx7`1FWlt@W_GW5eX5u+?ztnsq?M;()k8fYh+?u{5)(c!t
z5O^I`A9s1xOc~vQ+YzwCX;!+=TPzlRb^N}!c?<C+?tgDQ>FkaRXQC!Y&d{l6_%vlQ
zIW?k0>BIyyw2t4>_W2B4I#ngv-=h+Vrven`x_JaBD%<mOw1_`Qg*akvy*=OYt1IKA
zKgF+>dh)d=-tG|D01xnaE_rHu@dyH5z)kQ1LTxekRQ5ZJ(ZKQK8P({C@Ba+`zxe)~
zYN>8Abo<VgCG&CAeW^B5egO^xSAhefIQjZYN$b}#S|8-<=|=kb8~d(@9F#nkN%pj-
zGAMZx`RL!4$g8%#^6J)3HZ`52r?UYeOv8AH_Za4tx$7Np48pB0KHQpy3d`OBjEJQj
z_Zn%qwA&Bz>v#mh&ftv$SwsVTzwaQ^sj$!!pY}|G*duIf+LL?s0VVoc7Q+zhFHush
zognIZumQptb*}uzl{vYDjr*b19=_2NjIX`47n0)0;ygZ`aXxTr5x#z~G9I=zo+9(#
zosOb$lGgWQ@?K4aPz`dJF7aa$n*>f{By`@k`6pDq>Z4t{ca<BKhYW&WS{vmy(D`4x
z%cR#tmD~B3%fzC1u32MX_BsL*+RX-)`3B!9OJ1wQ<}+(a!Jv6g*E^zlV;pF2IO=Jd
zG0}1dhu<XpHC(AvWg6zEocqRP!=n`KL1H8R)U~d9%k1MpVR2G#XcwHE8aT|Wt_z&G
zVPe?TuL7E>KcHvv9^{!j(c@Lza(rqs#~+?j4tUDrxDFkLGYu#m&LZ-EU02<tUA`!@
z;m_x+d#lwo!>YYqYAN&D-kOhMT$MG$Cq2di8-H6?M%+)%0<TS*-GyLYh8W`Ri4^k!
z(w^V{jla{S&&EkA-l+j7-?UcY7!D=%s2Tg8KedJyyonzZ+qf-XuLDu)DzNuTx7o{b
z0uJ`zx~}ASu1`!G`&|3qTT3&+>_e{B`0jM2>4b+V*$VddNKx@yN*|PQ%)s8upPm)4
z9@bG=9{IjHCgd=3qsOw-@xSIc6cWfgkn8QcvPmC~udIR)yYdPn)zvdt6d<WQW`?(z
z<pi;GpTj+*H%`u?JC<!8n_t%bm;wI=lU@^p!#8dYsBio|?;BGBPTt#7PwA-|5nS<p
zi~q0Z^|nTmcw9Cm=6k|*+R^aqcbJrScR~)=cDi((O(fjUhcp*f+v!)i*$L;UGPg92
z6kCSfPq@|e-&2GI;`0txL^gRUZ;At&dDSALbXlO8o`-Y<FqIQO^F?f$Axp0`yJu<g
zuVDQy=)}(pp7{CqsP)W03`ZXyViQN@!$M7z(;lP4eo053ohH5$|AyT(p*#Gw4JI9~
z#dC)<HNd<b3)y87Rm`4{L6-Dt(urDyoA!Y@&xyW%BbWbv=pv^ey#|iTuQqxlmDQj0
zhYj15IxK@2#SZB`_!6Q&9G6l<qTWzm^`(i{h!g0xJ3kE+8uigHDlTS>;iEb^dt0Iu
z??vACdVhU?W4z%fj?0IJn<%sEp@V<@{sF{Lfn3gUBeDB>4D$!6UgZx*4-vJmYaS;)
z-PW})e_voEOnb>0Ba;^*JQ6FadyAhxnC54Sd7(7=7jN}U@-u-eJ>%KlI2d~^61Ckj
z-I`|UVz$goWb;Zy0b@ki*4i2lvh4Ytn{S3EK0BT6l*R|>z+lOe{6nWKY7v<~;`I+P
z)K#S6Gm{X*7f!!k(+dVHS4y7`A4sD%ZVL%#v$b_odD`1XS8p1qXzo86Ut|4xyilAj
zyEzwnTEu0`cCg^li}73gAd|$wF3J6p53t-pdW9UH>othG5j75`QW6MaZ#9tiibk@<
zR-Ng6iD(9={VnqW+pN>xY$*BmA8F)R`Rmvd(=fkEu?XCGU!2_UiAMb>zJ}UAko9^(
z1iwsU>iw_pE;)9PqhYo)qnV{yaZj`K(01;W=NQgAB1&OAFq6f*cGbAbEei0<2{TP@
zOB}EO=_xk@i^Mh=r7Du~e_fgTl7ukdIzgE|BkDTI5PvHHwRL}t6a<NC?bayim&ARv
zW@E&axj!9)0V#49iTzF|I{uT?rCo&u2Qv5^;jq`ghoL%r7~0+q*_a{4N?&i@`}Ln<
z50reb5;d8yN~d0|(v)cfQczJkn1BYBPU<Xy3V~gygpmpm=dgZbJ5c_>;tOiByi**M
zeQuEEUlYi{`$;~LAO=YzQfbQMuoM7Cp+};8CtJl+%b<#FMEC<Af6brvpwT~+ymn=O
z*?cwPN@&L)SJRCEQ@a&o*yU;V`^g?~*CR^mx*gKw<O(>{`LO?bzepmR{v6J2mDX_V
zRjtr$EY7s+uzw-4JA*OMM#6SM6-lx=uauu4#z@X1B&5wzv-@FUdRm01%)%M-Ey5)5
zmzyHFLjXZRk*Im~^zP9d>F+LuL#a!L<7!D$%8=fLD{A9Y)Jq3ec}PvPRopTePk37U
zm;SvE-S1f63pt!GyN(XPf1a8m0S26(I*sS0-yfnq9psUdYb8}o0u4}Y=*7Q-$R7nF
zIkH{hvXEFDk2kMk=-x9UPXT=+Pxc{F1?d_tw=p(kyR2y_S%@qnuDmg;^N$p`g!3kE
zuS`e1;_F{O?%A_z;p&^@*2DJ5UJ;vE&CE-QWU$lQ8LiFfi{_0#so0k#X(p8JngCh=
zR_63aFG#)>%aOOZZhYX#uUqwJ5cT-#B>a~d^=>ZzG}`aT!;0SeJ#9at9TUSHPv4GU
zW7nYyx;X!Pj{mox2(Kro=~j(?6ZSzNYU1nz#-`^L6}Q|xJ}eA2F)F4$-xzGR+*;z^
zoWr_`W<Si)?wB@y`lGxPMK_m{UV2{Y*Zb67C20c#p^3{Crl!Fv-fz(<*%cu{2&*8n
zQCt_+HW;7d$%yCR2I*;`qW(wwDAf<D?gq{G@gD<nF;dsVs&a4-a)tj)$NrD^pg^)?
zoU5%oC_tg6`HNWaON5m^cDiHc5n^(>$@7Q(CogeYEANFuXZ4oQP`NC3Mep_8GIpWQ
zj2cpgB~OO0{eEu$$cj1T2(tQogb<lQmPs5!S7ijb7fm+QdhV`~_P%W^=F%LWV^3`v
zTR%;HaZz|6>uHq<@`irxur=lS$!E0duz(k!mVan&V8_Jtd+Ibw28!{04F=4q3+SN0
z;HY0Op!`#0HjJy)kk?GU#a`h7V<+;Ec<n1>-)yStbPA4~eqV}(8}^U{!eBMb-CrS>
zmOs@61iE&cHcjx4@I8?DSKqK>P=BtsM93@Cx>VF&taoB6tz8x*J|RjfHwj=EL*BLB
z*}(lDel`Z*v(1%+OI<8m?od-x<JWB1bO#4RJ7V!s_pK087DC_tadBv}{*=Rk59QpR
zduIR9n%l@v*J?mS7_|h-B`>ear$D-{2;>|dVw(52^$Q=%%wluj<4Yc(J9b*P#Oloq
zLYx?X5G-!M6}No(Mvl0wGagXf%J=Ivph$k^#BC(|S|M^hUnR(~BPFL!f>kboF8)fz
z$c}fm6>+SF4S`tqZ7C>JLg!A@qKI2-uXRt{#n|<G|K%JsSxWbozoi`tIeT;@{-n)p
z6C{9v+Kj$50QG{%rg}y5Y+re$L!!8PhMQfBjr}n2l%2p+p2QMKPwE-_my~n;`Me#z
zdu<C{IcP$`<UHR1uSMRlX&!6~U^z35mGJc$D<gw^02}mIZaMw@tRn$VAtM<DSQn+e
z0I$cuvhVS}F&!BAF+owD82Jov?4cuHRS_&Y;~B+mh5tSU$r$m66QPzpSGXH1V%|8q
zm;D9nDfxiVpFmEG5`uGluXJ-2E6QT_y)hJ0MRX@hjg~teYsvKco&Vb+e5mDfYL%e$
zV!fZm)QUfo_Fg(B?e&<~oxqmwt=w@<>mEIt(i^Nu^yYy3?02~eli@Go8b?wCba7HD
z*x#o8jtSM=UfB@b*HgrD_09;MlffPkEZwUbt@bPyu=<+xRWx6DW9j?QIV@5CJlHOe
zNmq&?>E8PO#P2N41R!UnhFtQy?&;~d21tbc^!&_8St$G^)zu?_b`)#D)X%2iD^+nD
z{%wdSV<r5LNh#1jtpD~+zyBmDOl#$Ac7;xePYU%hn>y-q-r}xvzj<tt)loy{R%3Gz
zSCWL|IVJ&l^@vDD^2dPu_||@pDr1aarPXZtsej$@l{b=lA@SeJ<4J6^J{Ja{1O`{Y
zRX;W{XP(`iRvU(DCm!|JDMoTAGmr~d9{YICFP2@ANTeURO@L45+S8wj>6o5+YhKUM
zyXjEY?~^lPbV-okKD)2N#j=<-3G8|Yl4t<2T#zboM;tcqhsShGv_+Z&E<%}Xom@TG
zAfOd}vuhN~_jJb*Pq!7UX*pJpTwAJIz8Je)MhA#5!@-(t-aun}XXjhM&W;|~)H!vE
z=pGqfm3XOqJhWc1h0#6khW^JJHy5YhNc)Cutyf$NHf7V%xHu<zg6RA+NHS5$E9aYR
z8T6DoL}l7UyBA0%FhyO_Xh!S@yv;lx5h8W=iNF$zo54IbZ~o<6X|pTazr&pda!O7+
zA~;X=P0wj|diROY?z?kcY9l`4aw7<*x(PGd&(n7Y;KN{;x2A6cg3<3%Pi+_?Sn6dG
z@^u6GOwhG4YII>}+6@B1Ae>bhknmaq+&l97TZ`Pa_Zo+X4KPNKrJ8M{-LV(sXSvtg
zKfmKMHCq@c^vebufLd7qgiQ7{dOy>f71^06Q72!D`)CJ8(dv#B(*Xs{2sK|J><=$K
zot|_h+E`(i4-aKLffn%uprIhU&mVD{K>wpHC4|cVI`pf4?bC`~)fb$An5-61CA0Q&
zBp7_SD+npk^)u;W4U#6U)0=+%$d(A8?agm$e!ZAuWJp$_^!D_!G{vTwJNa*7Bey%)
zge)*RjkMjbTF+k=69d#|h&(6v?`K2l?T2a&KLCQM@mm(Z##=QbA{4U=r<#jDWv0{a
zSw>h>9@w(Ry2vI?r5_eafLg9M$-IOR(q~oPoQ3zxBF*tT+lxcfCGR}Jy_T$?A*DeV
zswwiDP+eyk>yxb#8g_gky75&?Jzdd5$P59f%O<ym9Abb*o!;k!OQ40LKh+t_1}<*3
zZem;<18_!x)$3nG5;7C@uWSfKua38L6g+Aq$PwLND0W^f?I-zgC<+*l%kZ+Piynb(
zZBTiMxANPM3qI(V+2aYXNal_POvBm$4LV`ZYxdB3aGpGUIoe^Fp9Tj9*IL=a-gO%v
zq7yj%OhVRU*X9r<E_Si>WT&3P3f0V*|8`Bk|Fk1N_L4`zc{_MkJ=fw%nSx=dsH!aa
zcP@J0j_o(+11^M0#sEC)q8!7I%DLfgi5NfxSI%R3>R*12=JqGhX?bdz{bbqriain)
zuTM^F^P&2F5(g$x4#-=@*f$$?e{aw3%b8r~t##Dzd<xug((u`rROmuSMK+U2s|FMh
zjnoHx>{gV1Vt5kUbj$996i7#nroc-~o}tLPzX>k#<(5t)r;G%d)om+4(^dp{P^GiL
z<=F)^ia}dvQ~gYo4YE28zbIWm7k*6u63`V(mGfwYZt^qO+liN-!@ltNBq$X$cU#7o
z{))*by#$ro!?*iJZQ-dnEuJ4t=k1~^sw9`_-dr>8ZVv&;ahwhTphAuW$ovo(!><XR
zdrK}JK5URc8{}${#r>ZrBB?;E$AI8eTkkI(r_CHJoCIjt>8-0@zQ0kjncH5Bh=}N%
z=fa5;XkbK4<~O5&M4t$^1wMMH<q=;#gVj-6JvYFvUhu*9_~@2-J9Tgq!_?$t8_>mK
zHTFF$MH^J8qXg})<8?;xtKk<8#M8w>tQ;WJL_e=c7H}99EePq<Goc3yO#_GmZj@-Z
zfeK0iuoB2fIW|o4`tAll1*CHg#bUzB1<ku9M2RlbrSImi0u%AM^PYk1KOf~c0zPW_
z?90yck+a`iOq?|leMWi4l@dxui0g6}GF#>|Lt{f^R2GMI^-67L6$@?puI~<)BN$JT
zj(=8{5@Za6l@SJJ8{klU0DMxto6q3s`^kq@Z9LIZ!$}NiD+3PLzG}}SVUtiId=;E5
zsqh?zZEOM7!3hH5p!w=gpr+;u50)x<w3~GIg&7vDi}b|G1LcTGLbi#lyYx8=!xb8#
zH@!+m0UiPiW+mXGVbP(|P;t9>4;+WC-3IkwiVRsxDU8CmiAFncOFcC|RSPpxM?%hI
ztL!yiYNqxl7l4;MIGGI!w3FNFx&b&B@hS%Tf1L}(jF5nCo-d1XM8tLJ(lSr}^R3>;
z16Ogukfvh-YisLU_2tF!M)%#g61`XbX8f#9aJYX)2Op~2Yvqc!CO&A+;)HyVsKxi|
zlWIvEa?a#C$^}oE!vy7*yl~ceq&Tt_#Bv?4uC8w0d;$JYIlBokMDQDBIwZCiuIRX|
zHqmzE<k#9zzN`5H_m~1}vS&NVm@rKA(cJnRcS_m;-`6CFa34A7E>mcL*UUF*X30wQ
z34+zB`r2R1XWFb2X+drbblIPn;5yid;BnpO8=#i765UsH#;1!r3V%gmDcF#vs+l+N
zW#paB<6|C?D}$T<Fwa|FRv*wu*Yerl7b`zM!(chwmp2}CwGyjGk9@lFBcMo#zsHsF
zU-2zmo*);e84(=9WFw$Pm0#qoQCs~8SV;pHk<CwYgXn!RApSK0XvfZa7DoVZG~B!+
zCdM4*X$w3^eGfq)DOTt7rQm1EDMavE0k2qz&0dXMshBfV9Ib%F75dJNo(^X7b|B1Y
z9-FJ4Z&Y(RrwkBO0SR+;vs&0VVCbV=T$NSoh~}Qe!Ska`0cyH%B}K2)rpRtcP`A#f
z2G8Mt%Vp|5ytrNsfpzj4u0QJwdhAP8MPN8!Z19n)`&_TWohLIaUZQ&4HHdB0rWYU%
zHCr04nDU8;j8qnxpE8ZSsu~?aFLX!Te`z@_QpFiS7!~y%$B7cYwcc8-tJfc9w*n0a
z&b`5V!B}Mk`#0y5)uArDCN&P5Z#=pXD^I8nq>O16%~RQbd<X68*}j|@>l87BX1>w)
zCr68Aqn#^#al$TXNi2KsmnqSFs>y@=uf*L$C5}ASM*-81DbH7W9O%R;T68CH@5NoS
zwjQmm*=3Lb`Hb}OtczZE`Vt}m15a6!7=C;_bO5NZq>rt;c`W0c`~jn!HqctS0UZBZ
zMiIAnVSLu#*mZ$0RC!g{kk=a$_=ZE<5e_^k+nzh63Z}mtF!R30&gJ?_?d{AR0&eP8
zwY4*Fcv|{Bzq3<5n=qyBERvl~*gS3Oc1MhnmqPUMZv5pq6Xlh8+$$O`Gr-02=%F>p
z$Wp|$Jb~;?SN{`u<&A)??ffw>3iO$atH%HnI8HPTrTD$N!zcA8KOx_NrjX%-=Gr)(
z#XD8ok{7-ZPMU#{tp4T5L5=b#o7+Y~)haUMk8_PE^34s-xpVn0jBqpit_3ZO){WZH
zO#Rx9AM-?lI1n9dFCPR?vKW#D4N2S3r#sh?&~6D<MV-5LXe=PWm!Zd2M8#LRwe@Cc
zT#+DwXnbdSv5QX=Qpi5v(lOPQrYcZ)D_e!L+FxlT7b4dsq=4FB{Ck>cAZlp+G9HSt
zubVkqxi*3$A074&w@2SYsutH5#&IH{Bq9kwLx_&=@k`BY&9>UxH+=x-ea8!O@k{Tp
zZ&E|=Tt`>VRT=_p3@;le$9CNYFD0c|=W}RKw!b%@KUq1S$NxeI{N_`Wn!2OSe2aoB
zHyZ_u+!pn$zoiA)g|5$(JB_nP$d2GN{A<dR;dvFZC4Bq%00Wph9epTF^8|k~3);TJ
z2a3G;0a`1^G4Qc5;1BSv92D}FHEZV?T*FJCR0ARcEcycE3^s74qEGFm80qxOtWza`
zy{zY4(QR&bRK|}0P=t+stMm2EXhM48xtfLn)708(3-jqI;SeGT@=_r#$d{RQNkVpe
znMM&nQ7mLkHH;0~K9GL_gFY7aSV^LTeydy*^dao)Z~58oIURqsTYMkgo2%dN`VcZ(
z4Uz>sIm$~$d5nuNr~<=%mw^CN79Bw~5CrM!H_RdMvKO$m<P}3~y*&o9p-5e5iOop0
zrwPuEosCWEqr)f<5GR_#S4rPMDt3X`Z<vV<wHvg?#Ms4GWVb^=1pid_Sotqg`@{0u
zYb4X#FZYkca1io{Jo&)|4Oi%%Xbs-BR2XTn+WPd-h&J<)!ObPhVe^N}qLW%1JT5&U
zBG6es&3}b~Aa=(mRqdcOzUGV@{n;<og@puw9$KKJ@?|mD<-8n0rX@Q8tzxKv$Rdk`
zgt7R+4BPHum_*z~escVG8B;}ifM75jL1cR&8IvdY6l-)2oq{|E!M@ZNzcrG$A*&C9
zoB`0NF;hlKEpp2s`W?COoGPqR{;eR5{I10}EkX>}E3;>GBRq&8f|mru0N>{KH1Xs0
z8qakL{%8C^z7QV$@+{U$RS=NSlcMJZFW42xA})Tbf4x<tVSwpo0KS15BIS1-dKT-#
zd_B{2)QC`D+t4+tdgT+@)*xj_%(h7vkh$UpG;8k$U);a#=w(_3bdRglSU%Z_KzgiH
zAO3iG_X)F?IP4$=$zOKipllb=RLcp-r-=1ZYnPmU4*-vjh$~93^kF<5Y!K|sC>B^Z
zLuT;NVVvWqGsxL7%EgC@FYF0P&XF2T+{rx!^M-KcZKLq|h6h9FhdI&nwP<2o@f&|A
z;NQFekEZiFdmhZY8w@UZb{u(6PDH9k0|iULg7}_RAaZ2lpM^GuegRGCGMMgEqqIjD
zk4u`fCWxJrGiEM-G(!HaQ9dh<ZMbB&=Bj>BQq=RyblV-THmH~{@?P5yJ;2w<BNwgg
z4tCrkRtfKC7TF(q#hMP&DDZyL_|tp+>C9Ft>oJR59TdRZ#PH)Y$08#{W?OXFQI+xe
zDKEp9dopZE?qy5%!)>(+<ISw@>$#X2yTP?d*8WGV`1ItNyyy#9v;+I8_tI7+Ims6h
z6c#|y#`yy#Sg%|Lq5qZSSW6gCiv(}nUjq(F$#e)2FL<v^=nkIdb~4s7q0sHeohN6p
z0&r~SIilNkX<}>xjqG?Q-~5P-mWLWbMjW%*>_D%T4+(+7&KfbiJ5PfK_t8BKXiOko
zbKjJd2nf%nMWPjg1a+teFo|qX=vUc2Xf~LhiLzqzMZ}7@=h2ZCb(1mq0>?55eAf8{
z?zxnGY`vPJB$tp-zbG?KLsdc1^E?($zav0IIs6_-NWbqiN-Zg)US#Fi*-fj7pK(z`
zXgYJ@s)9)7o+DI-D_k;e2#Nd7txF}2?`@m$?p2M9HN22XY*D##TZgJTlw=Q(quVVF
zT_Y+o@3?4<N%1o?Hz#(0ZqLd;hwXV9K{iUeujBO9GW&fE^TfUm1pDJQ9OMvHn0_V*
zn@zsjR?4*03kQ1+=(AnmwPY4w1lJnPj%J|z&u$0<0hN6(ydXSNHu?|{xurUTsHGnn
zDzz9t0|cth+^CvK37!CH2KoJ%P5De0Y6ad$=H3jVKL4%IRa*iDs)j<da%Z?~7nh9d
z!jRB($2%o{^Fj9~hEz0ZEkfdk^Z^3a7mA{xd9nM0mKz3C>Op?L6n5s<9PgiIQ*+UH
z6^F}lY;4rQDiFdV6WXG)@+mWe(sQ)8(;;kCbnfvyJetM!-|xiQ4ZnT)`~;4#mb4~<
zqxD{b(Df38t{Gi=34Iwv)Yk;gPu#cyy%+pi6+o8`aLF+haxex!KVu|A9}3(|3TY~}
zqL4(uFrK)ct^_fJbR$3X6~?jtH61^?>lLGo$uy{4&!cIr3oJedQmz4_VB(Mcl({h^
ziY2G2AC^$=0~wg9OZ8_!b^Vc?iOIG10lvJeU(g|jmpnS0j4|c`l5po2^rdKU^YV$}
z-arB~4KTjl@;y4Zw~jsoxob6Zmwr`$5=hrXOegw|oTJxWuLOGXR997QY|vu>spl1Z
zsa~om-M9utW5WH!w{z<#WqQV1XtWJjL59X-mf1asuhZ37OuxhlJMYbA(CT%IuVVEb
z+Ca5IP+XEhe^_<?eAOmUes2zA=G~%rO=2n#7ZnpjJ1_!uBEwo0C?e!Bzs0!r)PNx$
zz<61$KH(sL?7x^#gRLDGJ13-*`h#yVS&>jy3sVPyG+E_~sK_+>?BSv#A};8lPl+#n
z2hZtqrV~(`H+Gu#k5n(<138zoJ$ka=z(x6u`TA5BKY~cV)F!GFg;#%n+I0soh7zo!
zDa%Yk>{B;OUM0wR+qrS+9Rqed;!u8mevLV7?mYCY^N_0O^wm#@C!%4A-n184bTkHD
zT>CYmQ3Oh|)YuSLtXDjl9p<lWX0NCqUfXHC5i4lzOBD7cJBU5Wy=ZQ~td6Fl=iv(9
zOjDn3bCY+CiR$kYlYdux^;`tGs^}ZE0QZzeK-ha%ZH;0q!E}omZl1{Ip75@<uCsGd
z9Hyp<0l?I(<VwprVw>Y(GV~574pHLF%N`HD2cbfm^&FJYMZ65OMR?O1wrT?&&Nl1}
zK)Vi*hfwMmN(q*Yc)?9@*6g=+OmmBE2XA2xxeY6OQ8axwaV8=D`T`S=qjZZcVrzD-
zmRTBPe917sH3D))J*zj`&jGRBLGhqcxS%~b4|J=KNok>G-^ywn)C^~h`r@z@{{DRE
zl)IdeC%2=3>SrLyV{}T$!HW>Ic@v}qUSNR-4FS$`+5>c&yT<&*I4O0kUOxe)K(8)B
zy|5i<H0y9uKkDjFaI6JF%kXauHZ1KvUlz#T3)<NndD}S10Ta;OUvEGyQ=>7GI3PD>
zyVd?U*AlYnFk?FIk}pP7V$lV~lis2qD%x3*V`pCA2;CGE6fBm-yfp4Yumka>G0c!G
zZ{Tk7T>F^x@fx6~qw#LlBL5yPHBzGjPzcH(=8ImDOaLKs`IehKfdAfROWrg$CjlDh
zd{*4r;Wjj(M=i3ZoEZLu(L3bhwsd+otjx48>1&!oBtaF{D9`^qBlO+YC;7|Oa)^ek
z>R#5k?hjP23#_VxYn^F7@82De__Zh|{cI$e-(uvZpN%PJ?<=>u^!-S}$Fxq=$0eF!
zpxgZU40}yQ=}1&&z7wl(la;`Q?1M)F7ctBqw(cAhP}zt?|Etu5Be9I2G<*$b%Gi*9
zwV6k_KU9JREonO<o7eQA6Et>-bUI;_zGr4!Blure`D^EEaZey8F9|Wn-myb1e6JFk
z@VbHTs$Y1(64x9Y<Yu695*+Cn(3phlOn}zEV0V9enOEb({WECeZzdsvN&WsiP-x)8
zAP$|>kb~l)B5A3sdII-B7^3HY_ESq@5px$&f?<+H|BM1QMkBB@uEkTeP;~Du>NNAy
z0yR^geK#BDW;#ovPz`$QGOo0vRxx>1`MqG!*6UOm#Ckz9SuS<6v?l~Bx>8T*JY-i!
z{8GS*4@_s)m-GZ~QJXE#P9z|f-c&7B=Co!vOdAr4ZjETavNh=}2e6^cwj~Wj6%YDg
z%}i#f(X&Hk`S<z7M=NO{c2*8N*wOgB46%uqUOp9zT&UfK29)F0Ca|qmkk@t^>D|lC
zXZ@HOCXO?ZK`dY#+X3P>U#S4vNQ03fhs(0YaBv+1?hRRy-v;O~*@2;rnp`)shBodV
z{UxfsJ?eu>it9S5$qL?f1+TV3H-M6A59)?MZ^G9SIe{F|9&J2;44}3?-J~2ztr^oa
znxpt)sR0tE$-#Q3Utw4ZKgkUK2(!+0x`+qdl~20bWlo!K6x<)IWR-Y@E6CC&p6kuy
zuJPK8NNbOh(t5zV@rA##q<?Ar+AndA^zFxbCurubW%iRhVgNx2sRPp%Ga<5|UARTZ
zGt|-}V(%^p?VgR#lZrA(U0pgp+LjosHI>Wk)A0w<jvXy|JA{^Mb8O+%djNx!Gx=|w
z8~4M%Si#P+ZF>r`_g2qgxO!uRWkEYo$><ufL0gyo06tnbQ|=-!$dqZx5Do$=uKi5P
z0@P+Si+-&mIsULZb}0mlEd9|CLL|t{ck9rqTLh}LrH)B35(k1@l2J^E1wJ}qXET$^
zG&W|Ro#Z%ItS+*vw54%zsO<EIcv%6DLdPGU>QnlS5XwBYJ3s47U?~lOz~^C5Z}H`I
z%{wBC+pjf2L9Z6ZZ%kh|8}HnHO$K}JJoe?fXm7j~FPCM4r-g~Hea#|~R$!w%fYJ0I
zhjto`SAni>o@uT?L5443)bT;Cz%WysKx-P3BPR}UW#95#GXsu_$@gH@g0l@J><Y$g
zyXoORD>ut&QECk+*$`+Ot`tuABOdVXDHL0DNfX}NO7VRU@HZ;RzSUs4Q>qoY#P05h
z4WVQ~vk&$X`E>EM257?7U*8!EMYLq21<hnFXd4Hcl7Fw@LWeMD;cB>yn~f?|Yxg@V
zs^9_2@1i6LHs|^x6PgV>7u-?KIkj$&Bu2|SV)gr{?T7f7|IC^H1OOsPOT-I{&Q%FN
zR9RxWv!WK^tv_U|1EBu(w)aFfHzL<=^|EwbyP9}|Td#z3D|~%z&CDN@nm>Yx$LBpe
z4^P~K&jPeSxOs-VK9>URDJWTN)^-8tcG$gX9z@b8-oBt*z)^lR0qC#avj?)>kKt?I
zlz4--2p-b_TL2PDgqZO&4+YSL;C$+@L(g5h?Py|hC+-bEY_cpiS~D8x>~tIsdziFK
zFm=&Z1bmY)oz%J*>)ZMr;H{>AbYiReiC-45HcbAM{Qj~JaOG<;bqDFK7SIVWb8czk
z1Cs8EFo`45`&A)UYAJVLkW4-H9+_G8Z2)=hI!2CJ0RRBcwYhHPBH8yxQ5u11t+IRx
zl)%%4zp4RPk)eQD3JEm&3=g^3hFY?bKYFs^M-1MWdFl?F;f3=n&&hZ=sYRd6=`Miu
zjGKA}lhE>{=I5)U-1(1MATh(ZTF{MZl$^eiW0k4c_;Blj!6+bkmbR~$ytsco;nG!|
z@t~fuSjCs?gKtejELCV}{*@Q(6I{!pnV7ozEv==Z?ZHDq3Dt>;LS4{)5q_{35K?n3
ziMGG{CQtdrewPre(K4>2zC0C+g2AgjZ}y)3&-cypgIEuw7Y081*>PX5gd$OWo@;v=
z!fS2%`!e|mwe}cEkSnE+pSddU<>gg)I!~;x6D9UuO}S7VKcp4JL!mO>BrPGE>l+-+
zZAjjj(*Vuw2at%tJY%)V%2>l1UIESG7>IaKFA}q9R_x6I?kUS-37z$P-$Nz5(3RTz
zw^q6W4JbgQYh2QQG3Q{M`550i{DOMIXyZTw6tzrDDrld>e!P4xKphzsMU&}%y8#M0
zPfmABdgD^5Qw_}#>B8qnFM;g3+N;=7(Cm<07PYTwyuJ7sYzqP)LdpdLbctkM@H9;B
zFW1EY=KJ@>?w7=_V6!x?deF4YR=7+n0%7$obLi#!Dm;5bj*bPg#(<4Fx%}4S&w=-R
z;j1%U+2wZzUchi<A^NwC0K)ehimCrh^Rzd@=`4}!@`#}Yed1;=J_H5%JHFy-5L5i_
zdwhr&ytwy1D{||;S=5ilfDYfIb!PZC*UWtFy-M?_pU+`HJ{?G2dp7AA#u*s{ZDFw!
zF}L{=>mFGF$=T9-1<i2FG~0Qq6Yp=12=#dKYod8U{qM1_-LHn>ie40n%3>_!234v>
zx5(U?4Zj^0#=C=trhCtw#+eF^rXTmJG`P#?3~|0yCU*-BheYf;M1TFzIHFLAtRe$9
zMD~*%r2_)kyfz30_kG_>hV%%p?2FpH^8iCA&K~a|)Gk}gLd(C^e7EYA$6M&Z@5Ue`
z|CR|(a_$lfs0GeD)&bli1v1pgjbr-+b^#c44C$?gG6>0R*&0AEBK{t|WlJT1_*`bw
zW7=Pyp?a136EF4x1`o_k#PEUPbod)KIg+^vfeY3ED3QQSw->fB&6kMudlJD7tPN<v
zYXPWAxKWK)MIPKZ=@ANyNjJyQs1$H$KKop2%y<my^GS5^ms^$r^k_`I(QV|Xu`&M`
zP~8WU*p_TFL%!(9NGUZudOv2OT#l^4PvW*mapf8+0V8w8qjrQt<srjBJjEIC@)KXA
zrA1?yvPxnuFVZM<>}}4c1Aqs~)KZd>4GIMXMa5SYPk|WNH_&Hu8HOKIqE~Tr*M{~^
zZnE-V0^6UPEq|Rk@+FN`Pgc<|zqphQHtSL!ANo*ORTk3q*0s)Aq(75Ith~@jXYtX7
zfd+_D1@JOZV(L~uxpwM(L6Z*UFE?2ysjDr30Jf+aVw@zKZ(41!p~gSXNC0a%ntY^3
z2(t?nQ9*%vck9Q}NR7S))IEO$(18vN6<h|nXLu{Y4*{e5z2iNyQ(#a;A(}1(*YcWf
z-0RX0ft&8%Z~K##G^~?L98e65gZMoO<mS%ZrxjWQ|EHd_YtD-CAM++U4F?Bu6|Z!#
zV)H<?1-#Tj^O&}s)$!omo)DoCoy-;{2@-gTkhtBA{JM3bn0OKQS6fdbVq>k3dRV$Z
zc6FoFrtcOIx1qYBbx<Ov!w=}0XMvXBDm@+iz^wW_UOQ^4@&^$Z@mLhXxX`O$>iTQF
z_fa_)(8BCm#TLrN{R)e1ZP&r#UnE#{(TlWLX$G1XTfOOGZZz7Vcjh>C!C;R`spE4P
z@C=ba9Es$k6Mc{XKprp&P68`tQ6PMD9PF-lwUP0ZOM=BqGwxeb-q5A>$E-TzO~vYN
zu4nO)7*!-9@#k3D-y!)7uUeInOojVBe-J*MHCs}|u*1Gx;=50A$P?s>DpNiERh-*g
z79~solloPp?C4;<y9m@KZDJl@5vOHCe!;LruO<ndsu-S0{$MzL>@Y(;ub6iTnpzeg
zNNm<jYuvY{&5`WmJbgHd2g@9}Vw<w+8?u*;pKeB3rAp(0QP7|NWL^Ev>f)AU$R;X*
zG{g<YkPO+*{fYf;RF6rtWZNLQ11!h#0{JD=6&t?~FJ8a^vuK1v?e2%KcM9JHa_I1T
zteD_i2ZViJnHcd}j`bx&fr2dkOdjZ;EhZsboft3zGEZR+&hnPqf(~9Jrb|uPqyd0c
zB*NdzF{q1>;oHQ{M+UHoF@R7;g6i+xymWD5IQfO9Pxjv{+dz8_Iv{mhjF?#KJl`7d
z%}O9m>4FIuJlecR^_a@#4avt+cS>#`k@QB;4)<<+>li<e0+NOlVR#(|k+O9!g1MQN
z?vUOAD1ucOf`!|(jEo_7cqq6fdeKm!j^cKpuE2>G5rEC!Ik!a2MF~V0&&x`^+8WV)
zetd){q6f%Fsa6aqIMnQc0tI-&RyxZ#R({Jb=Wby8FC%#QD<$IvRvt$=dXe|I0$~tO
zI`OCFrLolgrXhKmzQ=oaOlTI<m8AUiKHQJJe7L_Y)B^IMG^g?AA^`Zh;NZtsJ|Rl_
zI}Z#J9mU^YK(DUzKl^vfoSNu5z1>55uiJ)uMsFE5Q<b0I(B}}1Q*L~ld8MZ81-6BW
z>uQ4Ib4~OmU2?h8A!=QEsjF>ZiqUL6n){LA!uAM8gY7Sma^e(_9%oDDpJoQ2KF0sN
zl*#8c?7f&}8@s!7Do_9Ox|x^ls25|ZPmLE4S0l5w*-+6KB<+`0FTwnDC!<A)L8Xfr
zPv$FqC`dC_AEz*6Xe2dfBT@u#-)p_AdXM-oWiPAZI+}Rv4jwc5&#NKQ`bRD;i1la#
z7d5~%aDi2aiXp-S2uDtUv9YQgzfzOA-mGsKs?2cp=k(aq7hhgFvlRar44z$3AeZwx
zDk-~=^O9;sURjw2BCj(E#x<glgs1f)k;y(In0i^@Q-xJqVDFNJq+GagV4WtTM4Bq_
zT0Y1Dnyd!y2G@>Q?Z^<4B`Y58ZJLAP68TVI;igIbGmz{*Ccir)3V8+ZhBeWMg=xKO
z<}RTEN!)sduXa-uzG+qxzzI^Y&m$edFo^^y#5-$XfK$_lj!X`qtpRF*3?eNtFis-V
z7@&n&hSVMH<-o3isCIF2GN;_R@1b^E9q5D9+nfY)V{h2aTsBZ@U08JLypC&({*FO0
zi}v1pcUTREGqN<ZBGATlMjM_VWEx#@`WYk@0NT3f;r(?*8BAcjVE)G60~#fZbT<>V
z*#S5}1b=$OKDP3UPoUuBMvhEh?P!(jNmF$8%VctH7&f325!KjQxPCHsEQW+OfB&cg
zWGJ)DpF|l&q=@y~ey{ASHmP#y=h+<(ljvl&QY_;A_)9_wW;eGoY?3&Xad8Z~1FR8j
z$9Cok2itA7v4>1Zl>G1oS3^6_5AJ0Ba9SV{SKbRdx^gAzMnQT%zN#YI{a)^K7ypUv
zsgl)DUN<4Aoe*e?Xv#|w+n~>~E2e%QYaYWD9KPbpD>&ci>H+ekHbQby3Z1<EnR&Y8
zo}Y@BiQpTaX5^P_!R(^mN84*)04FaXBl)u3OJmU1tH0v|FhPfFxSTf+OJYziUxZsw
zH$V{*x)qq#z{rg+u{ToG%_bjPyzMtpcFb1;f#o8E3-nN4d4X;pVNSVUF}@g+RAsAN
z3ul#@0B_B~L?nZ9F=`HY9r7DMJ+KT2d~}oF*T-iikQ@P!x~V<u(Cde4Wvbo1$ZNzv
zdSUkIciH13#p1W@pmawZ+I<^gi}asZNajCWO3G)2-p8c;cnhSfiv|+7z--Cpy@uLQ
zMv->+QE%-)X~Y);pkiv0uw|b%&7w&zy>}>!=tcvE29qifj4Ic_-@D8v7kbXteJ%^A
zqt}6{y|bNoz*~BLsV{<a8Dq4$u3z(Wb5KD2s-3cw(mVM+oqyGBMATM&A0mdg+aIat
zJDJr)5HZh1Z|>gm8Zg@RbegH+BX(#h*(SaYrR@xZdF5m5zeQfI?#SuE%raJO$UzS?
zXtRr5oI8ViaPi*u_IXy9rjz(Yt<V|sa}qW6arxuT1xZQuao9&I5~i|aDbYNse|K$8
ze)5n?Iy8=$SR2xq+=&EJs=CELLA3fpO#m~xJ&*fMHF&p<FRy1xBHmKKvB$}7#scDM
zKwtduTe8z(auJ`=pw+~O{>vqW`T)_?%u<(84-qMZv#Hh4mmL*D>to5aJ{@m(eGUu0
z7z<HfuSih4CnE?(k0eMx=d3;kIQu8CXgSUK0J?Y{D8F&se`oS=s5&$+Bu8pfPoswv
zWmBI5A`WtK7`l8~YsZD167(x-^Xs^0rU`NYur7<A+k+RiuHZp$efx^^uTSfU;m-j?
zF4e7T8UJwR-69pPV1%hRsRdf4;xP`2lB8i^HOMFce!9$pV&3GAhjin8F#SLU(bEAP
zqHjhq&pYs7J8$jIW0NgVdv!A9Y`A{Owc=$5{nK+d-1e(T&tqpuz!+f^&<+(Tuv?M8
zk;vPyA8J;~-hcvjX?zlq8m*G<OYJt)vD%Eu5DIBk0F~Ye0E!lH1fSa!>hY81joD+A
z4PH`h6xrJY9h*Y{&X!_(%t6pIPhN)`8@Fu2_Co<}xPZv82>{RR%-*clG+$R0+k8uh
zo*k{L6IMFL!mQ{FRN>yd=cp#daAaDO$>&%1Y@u0<4Ji4h|G~~KX6h9`AKtv@|71a{
zsLE3>z{kWFDS8L2gVj}xqME+S5#U0+5##rql*W0KaWj4=?T~*=!%8|4Z0awCpRdts
z{n6#aYu|T)_t?ARQIUt6lJ23Yls&87<EFKWWWi_qll{g(lhXH`XUx)CR^tP<XJDK;
zP<y<|g9LsV|H1{p3%3cBrvEuak60ckl=>pFEQ&F@_dIxnMG1ki0-3nU8$oH{%qtc~
zYD8{!+u>il-u6kUUbFG7{?Uwb6&M4S^#th+=z~c6wv>ZGw@xO8=ZNBU{Q_CSha&Dv
z=NRe)CnR<!BF)<(uHfgS0?spbPKRy1dPHMq=8*VtDuDk+z8IFl=`*E8{?)DUZA16(
zSN8TU4ATesb0s`oQ>d8B&LK!cY%js21i={Rna*O_IIkO+=|fn*0h@4h*W7|k;xv?@
zCn@aV#%k#{HG*6adn?0486-64m_&61!f6aPJk!Kj1`#zVaqr4F&#BmkoM!Jk5hlt&
zzjM-`-aZ_-5}NFOv~F=4kcje|n*9ulD2D_#IC9IG-=zMxU7%W<tBZUD{K|bQFeut&
zS66ueh9JVh#Jg@zJ5OrN{Y|Xh^Wd#pX~=f!aSMX23-$%FWxJRi`<{<>`YtYz3$wKw
z!kLqx^Bk2d6@VHyI^=x-(8;;!GOwojV(T=O|FV~4AnB0F|M)&C&e!z^nd=o{w7YTV
zo9U%55eD1zSEJ0zY{w~)psA^#KOQP4cS)j|vS77UAsPC2S`B`B)<Og=8RlX1%71%D
z8Q;t;t;nVW{w4Di5*iloo3AZL2+Ii(@#WR4zK!;8Hj-eP!!N{qR%9^>p&bYNuL?=c
z&WdTWRWC>&5unr2kzFa=n}98G+UPg-<>u1P6F{)t2HjR^L^+0uYGD%0+)^zTHf33n
zJ8s?d^MAMyvR=wsE-vec@y9_e&OF?ZHQag@=54r`E;VmX10l0=U~Z6UWUYgq8W?j1
zbF1%b)`*V9`P`Qb@PoVupu1u5E)$IZ;x#vj9qd&CK>vTGex&MU=?gGV7~60Ku1DW7
z3fjL|v*hG0=-ZeWZL0+A{>k(i5NT;Kn`nK5w9cRulY@E{v;$|sMR@vr2?rTulP^qo
z>Ro*`*sN~{N*8QESFO-0P>T5&;7{R{Y_im|tNG;sTfYwSFRFnuZ)vIDuEse&7gn0b
z0qkS{XMH)=5`c2Z*>6y=?x3IS`1$ep`Y0%X*t|R-hPTD0Hyt#NWfx^$Tsq$Cuajn$
zLu6r|y&y4OU_EDOyp1=iAt7#orCxXtN&1F^<_oGlsK>!-%lk+1V@a+2-J+)c?PnKP
zy3b^ogp`heI<RYc2V|{yW+Hr3@cj>JvGX}I>w4W~HQLm0-RgiXlP-YH1i!S%*2=vO
z=DaRKYVjla3zOfBu6ifT!_$v)iD(zkxPa046joS{m83#92B0OK{WuZ`RDSwPybOZy
zn~?)Lr5mz%Z3;9mKU&a0=LGQ*i^-mQ-V>lF6#?)+eBIvFue501zPKzBVbUdWysL_r
zN+uIafF*a+WJrdU4wE%XdxdR1j~Sh($nXB}rjimJr1^UD*PWM@pue{qq<G2SR^QlP
zN*eZ5_^>b#VZ0V!ke)%jae2M4IZQwfDmIYDeic!fr+NNA%l4!vM9_~oo<&4{_#Ne=
zz#+mV+=gY61s@Vs&bt4kMOZdf$S$dB*s3=hGXFc#had6JJ`cX=29y;642$7VPc?*u
z;m3#Yqd_@)kH~33l?1(BaGHOC{`9t*sqsadgOp&N2w?1Dz1Q(sZELmX`n^1j;ZFcG
zzgyx?laV6!0uOPh1T#!qzt{#C2ywI@FjCScrrT$2sT%_Q)k@r15kc++7J%^%EQm<&
zOZhYO*Imh>4F)(C^vgF#c>$U3SQ2l=Du4nguI)f|IyyUV6lIZw3VbzR24i`O=lOw<
zpVby8HA8Jte)f+N{Avjp9(<f=xUhpCOn4SWG8U%@pf@QFJzqq(h{}-edIFGPzbCe=
zv#w+c8h39eR(Yyd?!^jNN6deKHuuyUIKQm(oP%S)RHor}4w(^R5r%0YFS8w~)&=`6
z%sOHbpqHBgjM1giX<U~73H5NQ`q*93TM_ch7$(+rSJ+8HCGbC!Lx^Aa=dwoMgGF2d
zuPIO0+a_b?*Z2)TyHhnT`WoX)s@x&ZLPm10?eI8_<cXF3G7cT+C)t?#T2eWG=d{hm
zE}JWL+<eeWG@nzj3#p={GvAX+kw16=<wdR&txdiVI~V!%OH<HeTRw}=YHol@bJAX&
zwIHmOc76ps^Dd%*)SlChQDa37p<BK?^}fr9`+1CRPY@_+OuvTv>xc1S`R3HP1#Jgd
z08-;<@{#!k%*zosA5Ovy2BisF<^)n|I%h0XI(Vu&4baJN#zsOHhA7ca=dqb@j(9$Z
zJsUuf6%7%G!8w%?IfGyiWF_Tt4|D}}$2I#22sEv?GU)Wexbn(BEtIxdS9~D@>C5`0
zP-@Iogu_Gop}9&N{VRQ)Zo4X-E?zuVH}cKMqC_s|ry>|Km;ket7a@W)+P#l@GapRB
z?>mk1ZIteVL_B{GuIaSjx6%!!7S9r6z|i*0QHE+F50ttn27u!1bI;<ueOIBXCEGI|
z!UdyJ1ShEp@Bf&ZoJu@n6$UGl08qoeR?uAQ+WCvEJ*h=z?K_8fSe)u)0}UnQU_Y`t
z<2W8_3$0B4C5he9PWpivN|VZuQm=A4@X8#lC`8>reOJfw936{n?P<Sd3-)^^Xgl=L
zy9&+!Tmb&s|G9bJ4)4X5Pk446<hPUT1hl#e&-Lm=gDnwlM4kumEa?CoL?+0hO^f{Z
zM%)`ALalGFX`prU8bK#gb;NU;v47H@%xh!j5y_e1==lSk4V_d-W4Gvfry4mMe!%R#
zeWe0$aqL;m%T!wULB7GjBje+}nEUSqyKFTbmLFVB0A))SehMHWKHlzO3j+X5G{LYA
z<IeX9fR{!B4E3VkRgV<s$#(oaZq|cT{|foAX%sx~j8X2_691b5za`g%6OEGE0onRB
zu>RBm-1TG=t?^r+Ts1tr_1>ZO4UkTx!G(EWzSw{Qv89!ax7uXXF6*`2kg2kYO6Y99
zR9VWj0)nk#@#By*g5xaDo`Fu~lPs%l4iJ`{R(^)}c%28O8l2>|iumrt<@u?pDHQ8#
zpxXIOG+#!G6!n7|z1t#j^l0l946GE*0zaKR9a2l8?|M9w8uolkt1=QQcrDMJMNAb+
z{jCmmm{D^iZIdUaGCQB?Po4u-`6gIOg|b1q{plQCu5R&%R_8K0Xdb5D5TZ2;H*?gj
za84Ip{Y*7A8bsoT-oEZF1_%6bwu41iJgEAu4~MXIP6$kY#}%dM0ZUKk;XC#j?IBEJ
zIBUW6r#DVQK*WCu0a^9o^cs`j^fl5SKZ);e?QGkhd_!N%e)YY6``$*D`b5D-01>3W
z7woUt^r5fyWoyS|2QXgTV%&VS5Ki|WdCqSH36xe(It7aZP`D7cmRuE!B#UUuObn9^
zCuuq91Xx^1s6m)-1InQ<hE>w^mDZWE^hpm+AcD(O__P>|9R8%tEEKp<gRuu?6dmX6
zKw+w9Y7Qtk-a+zcc?>uK{ZWnUyGfhOAB2$N8HvTd$Dq5Dl1Pb85tB4<Y#_#_4ORsT
ze%p4zjHrOB#*&0kV4x5TyXpcGsIl<PW-DwLrdJWMn<~w#_WyDA-SJqj|Npt8j1r+D
zmCTHgk!0OvZ?d=S6=iSk78SDPCL`G+85vofWRF5d2`OX?k^OtUZ`3)b&+qa0KF%Mf
z<D3V%?(213ukpNIujjM#r$`Q{c|G|Y1bx?mcVoLMOHV5P<zY6hA+!7mX4@A^&VPD#
z_Z*)-h5ZfUZhe>c8kp8NK5LV^w-#4YUQy=yF|Hk&YS_=Py8I3vX*%`yo`>VXdy5su
zbAngD@mWDS%ex@-KvWrex4EimMC|IatlBTr)D|_t)7=?(;w@no5m8uNwo?AWY&?SV
z<;4FVUw-7mbKCI#ABoPHn!M!$(wb-6Bi@NGH`NW!`{cYyreaB|{O~k2{Q;?3kY8gR
zf$7F+6)sv9>Z|chva(N-<D2Rp$1Tn%uC}jY>S-n(nvIHWe6By%W71l9(kT7k59!aT
zth`$^tp~kQyKIhCqgz7`LO*JuZew}s4--9)CCv$x=3d1Ux?fg4SatqP4<>O*Af4VS
zx|+Tzlp)SQn{sf)F*x~r{y?=~^_0L}L8&iqIW{c$>^o0y51*b#eYdGNCY#TkfT5AU
zC-pK~!i2O~`ryGJt9R(!rK-bO-y9Cnh+0|jne;q4O>yvGE&Vs+nb0jV89EFEY9i+|
z>hC4dep(yQ6F(Bj5JH%9V<bxJ$oMPf{I{kI(3>)KJMxALvXvdL>9r{`k4^s3&h3OL
z4`yrfd1lt2dW~J$<!S6olCKHmvGYH2=4KNp-|sCr_eUp-1XI%T*$b)C;rh_cQY0)|
zK8Z?ZD;N&AH9tWCJ(n!{yo4jQJlg=~FcaCcx(9&q%Q2{!mc6s;372U}UTW!tq60mm
z3|hCK&Zz{&wi&>Ojxa!oZ{Y_Jk0Ue=?oXq>lq5_<mh*k9uL}I?2sD2X(SAc-gaMAV
z$l&Jge2sj>pI;*{y~lk-aR4g+$468l#m{2Y3XMCZ9AS7XJmsQp<+*YBWy?le<&kd(
z<^FE(M<{!-NX17N$WJ4S$MBH7MJtXO$~unc!W3NTPXZg?)lHb;&mPCOX&T~N2;IKz
z`1t)*5d@mtQgZVf9P0ULd_eGX<1Ox!exCdMdIW59NZrt@=(vz;j>04EOE2vD4nKl1
zRi}4OKCcLlJ`x$wBXM5aS|1bZW)#P7irVvv#xG$8;0;*-A7Q?uJ^IzH{YO1pUqauK
zIYe6`r=bdImX=JmsQYzDsTRfZzdv>e{^~)+=*d$u5l<N_Z>!AnIMm^lEob=@I2;<s
zhO-wi?HU${<w{O9$#q<B7Gt9&Lf#ryfpSl<cp2zK!g8TN)2Vt3jYSD(7c9s!ST_n=
z7`D4sRMcyxg<^eL!@b3vh`S>tlj<&#(oE|fz9-lM^c^r5Ap=<k$qBTfVFEPuD=p1@
zVuy$NaKKKE=;l#CV^J91-hGs@?FaCi9>9m*dOY5#B%~+`2u%H6JrHe&t})iDp4sni
z0aL(t_2`eWNUFek=6|LN7)R!RFi?efBMejs^ThfN1Jy2|PWPRl-}el=?Cadxo*_b9
zlt20w-D>TZZL*}|RC_LJSQBGD2v4LfiLyz|_6<|Vwh{Qlgrz<#g|k_VgY8rBWCRI8
z{iPHGdHNdx$omvBqj%AD>e$ky7A$}BRO3~6fxhbS`aa=LMPMfxY|D^=5gRk;IhG_<
z@S2<GNPU2e@<~Q}c<ci-P*f<@sKnjpeDET0D*i8CjYx3}O)+~@Z%k*Z!PwJ`5K)$F
z^vUw8i7g3^bU1Fm-N1u3At~KZ(G0(<ckk?_X{c{79aP+V`F%DIbkUvw_tilFNBOzk
zisyL@wnOB#FwOQM_)LU+I0u99SxcInOrz>5j;(PPofqC^R;6hiB*CuDSHxT&8l=im
zl_56~s*l45Ka@rx<pzLSt}BCEn`1AC7s$^Dpb+^<sB|x0uYjC2za>t9i@Q~3m`M8-
z7KZuc-`uAffvt*I9`rL>?6<J6VD88Je)944bdEf#kRHoU8FuI-`5*ioN%6$$Ssaej
zKGr-#1(6v91McUgH+ebb=Sii2+Tu4B73z;%;i#vdHBl!;H4w5DIfEK2Cx&$Eq|s<~
z>y!0B```c*m{qajNoifmJcqWKJJ9$j`@zPkmuDK63-1EG0938b!oZWZKiKsw(bfS(
z0^J%wCz!0z;^E;j10}|JnBc)~!S{_N?j4aqV)lLvG<h$GEDgP?-3oiWY!yJEA7Y8a
zQl9>AETyTeW_(ONOfOL(CX%l*M`eEgZ_V-Hv$xOwu1*hMshJKa2<PjUli9<DupCB)
z`|q<UU&^JA0b(}`={L^;uyAH0i4Ivdmn0s}U`<tPe!ZOvlSTP|bmLi5+L+r2l6tws
z+jL>u;sfMxtAxh>BH{nugmj}Bx}R6szk5eu-;j@le^_BC(3Gvw2k0<W7rxPq0Ba(b
zn_)O>^HJta1DC^7lvfOPe<Vo(xmF9}uXB0sjJ!e)o)`g;Z0NwihV=)w^4XbWl^D(o
zXShT{&kp)sN@9+r*dcwJ!y*gx68UcnRG~}JC7|Azg~r@MIrZZ9*0Z7Y4sGGq&t@M&
zRkzJr1N%i6dI32YE6JZVhvNxs?(aeQQi5|Y@x?8V2O8ww4AMr%oN9?aRmG_CrGAXO
z{x<SsLw<NAvt~;S%deMtX{JB4pV=YTaYs19X5&e?x@y@I6^XjtKlh<e;I`~<f5@m{
zi0h*6G~3HZJ#bK<I`J?~A+E?Mp_ECUbc!p`Lcs1f!BXg+-Q}U?)FkoA=91Bp4C0*|
zQ*1m_qw3a<X3)3spsFn@20A`{<E7gRMZIZm$lxyhhz#O|AJ-fjY_}@16-x=)A<{Yl
zL|T<joE0!S=zSjM74KN?b=heN`vq9gz1n4@BAb50j4g0{cZphrL+yoq!^3|pZ|DXR
zA=cksZ~n5V4zo4h%*Oj{R-w%ZPG8+wc#(`vWapyBU?mrbx_(Zv)_%sFy+}1=^IS>A
zyv0)9<!uD<>~Pt+0VnWMTiM)Y#fo{PBkE|v!7iW@_}k9GI!_Vu*=efI#JN8zx^TYy
zuJB8@k>*cYQxSI($5zpI#D9O1>xZweCAS<pSP~x+gPTOpA>dPSs@nt9B4L>w-vYl?
z1q5V$JY+;I(k1-p?1O4%6^f>NK=*(+4MyDATf6b|^BQ%q>$GL$4vAC+1JDJzwcR;t
zGaQAy-BrxlC6VsBSmazFKm1eTpZroy|KOK0{r!0yh`}X~2He#fpiI|<Gj$Xr7X?1(
zu1!*!2iSGc&l*~GPufI#%O}-wwVw#t;kJ8;ySNv9fBH(L7_3p7;|S9LOdtf_md8$?
z?_k|WkTyat;o}*5cb2q$<zaE66c@H8D5*s+YOrZ^%O7ImLfVLyjS7_umZ>7WS?q8G
zW;nMwB)Dr@r(ShJFzsnDZBU%{f#b3C2jkvwsiaia+Z^JTKT4MI{S8IHt?iAkk;H|5
zm;Yv3;v$jMf*>Ek?G~ALxxZRuXg&ew<uB8EpB`Egk3B_WzcP$}gA6u~8aU4uX&-Tj
zZ^7EIz}B|#$r8d@vYn1pQXTJD!u{>fPo|8!wDpVs<<ib&qNhEn5GKoxRf!ZP6-p-d
z!kVeF(FN=MPkKq>e)sFT!xjYai1GE$-#_;Q++FRFjCeDAn>Jn?nahv3S}v7B;~7vx
zh~|14fUWhdQ}Nn967&?;*OV*<ADpU6gnZanih7%jszwNiXuQVs-IJLOSM21V9lCwr
z1kOZ%`@{bsL3mbR|D17oyvq$ob!i<Hy7`GOeLSB(+cma7w9)#T$5W}>bt%-H^K;+<
zhajmNPZjA3Qi$>lPMO3zPx+=K@g#BA`l1L1tD?lQ4RzH#!-c~7$$(i)z7NHTsyobF
zb0I)JH6VD@Fyo#UuJhrqLd?Zq=WXY1K&stVL8r61B7;?z?QSM#;_?*u^!zoic{?YA
zwfOHxY8jcGL&ttvnJ`KUQthK}<E|{yL~@AR-*-qFjT3~vCw5;t6}3^8$Hd#eMddDT
zVH)%Q#j0mmaI@5FhJqXuFK6s^61h?WrlS&%jzv)jlH%C#kdQYQ<NxuHxvy2}$TQwI
z?_gTB0$fR_7dT_ZPg36}lSe1}%6K+k+39>|z+l}I)t0sy{?PQe)UfB%e{sP{v_$H;
zI+tcVPm5o4mA^lcacGJwR9f1iaZzAd8sVGXjm2bVe0DZ1dy(Hm0-sDq0<MEqd!D$x
zLe%Tya2t62(#SJB&gf}!(yK%Swo^=Ohd*=$QlRvmYX48mizia&`*0xmSOoK!(c{BE
zo-&LW?{WczRj-+Ztgq#92tGRgBd_;2`30JlXvgEFN;E(og1$)%Jj@P$Vw4$~opcav
z5k`P!sH3YnlGA(eeX>-pZC{OHwxOlREt^UI$>)uzC&?$WIo21Bhes#JoRKq(_Hk30
zsbExdwQJ;BQR+SXE5syq;qT-sSH!<!fCDNSAM8B#mswdav}Y-80O2e#vBc;`JMa>T
zbgh?1q7^TTT*~0s_OhuQ$RZ67{ukeeapZXW)SQWWS?(_1H4Rp_I|d&Y%v(}p3w}=C
zLLklGDc>;Hj`MqC^iUeyi^SoKaQ#cG`s$LLRi`y<dL9@dh3I~>ihVYKDak$^+wZbX
zikOsxGxy|||8#-d2#V^l9}{I`@_d5zeoCosL#~BpP0ed5|8Re`dOPm#e#wb7X>fl?
zu{?;771+y!>AFDGxH2Chj=^d)gFtj0?=9VJ&pC7u=bJyQ{FjJJXH0yc?3@N?CBFun
zRF6k+OMHdV$+C0uzz!U`%Loa#IBEA5U$W1FEH3p-XuJu&4X0f6gJIeX=-<POoxA=b
zXY(Ffku|~a<WB$eAnp#IO8PHec@GK?L-CTi(SR#WzL@6Sr*~Sj-9k1EhJ8in#qu4b
zen+&5-H7HP{zw<@g*6G>i=5LC!&WIXJ=~oKEk8{d8utrb-pl_b7~M*P;<s05+2Jb0
zaO3PM@Z-PuKXg(k`jZq)7HIK&&3F+-()!Su-pLKq&r7dXl@N7&Jne5}rDW2(&N;+x
z3TWYl&xD410x?kXKAVYJ7L^Exey+bJk}X<^Ju|)!b^5byPbP(4$GPYI!~f;j`J>4(
zY>m(WU*Gw(C+RDFGkt8b+oI3-CWps#ChfZ$zbC`deMo(aBva<k<PQQ>4j)`yr=@(e
zl{FXED=Ds!PdSNQq@#d*s(Vu*DUGj<{UMH(8(Iy>k+>`7`(N%Y9=<+7QF4dNa=Pr5
zaKpMY9<c&n{QJG$i}sZSI4d?%_5E@#fx$bckLvj8CFqneq>_;pqRc1*1V#NB71W;r
zaA(Ogbe2glG|xnjW*psiD)(?s#aH1!c|OA1lvX|_=4F=Nce^rA8Kk5hw4FEWdaIzb
zKyfhQ^3B3uoDhM9yNk2D=nBEhoG}T5-HZVzr+>U29sFIxrEUNIuTj!L=+<Z64rf!;
zT3_!#J)F41cWB_h#E?B0pfc{tJ7a336yy3h;=X}E7qOtuHksg3<lbaWQ|e#&E1C!&
zD(t6TCHC}GmzU3LXIm)NSI+hqH{b9zwz!08i51cUTG_pY$B<-Vm^Bn@c&V+7PlvOV
zSu*Q=(2T_)`yVX=JbZ<NgAT}C<pvvEA*EkesF%n+^}ESbNP~wm-v<}htDM`v*BsbE
zoploN97L+NbDQpkn;-t2+xVjiA#az=D-%#`s}yFc)%|?d^GhG?N_#0sqtLN|)to!K
zg#jH|!Y*t3YdyigiV?iKaJ1##b#_hUF4G^CY3BCKc#h)dM=GZRBSa74U^Y>ggl}{&
zY9~m^u?<DlJz&c9EB$!Krs(*Ls5Xw$fUc+}3kG2;Z$2-QY>s}D_nMec8<isc@P!n0
zGzA{6&{I=NxK8^A-<t2?3mfrlj+1KR;o&r^O5fg`Sa`VLIFi@A)Ho|N|3EKl<ac><
zmAETQLw@zg^zqRTVYy46yD|%)t#-?FR<!1EM#sL8>JqK2BuZA3OZVyS6Rb<9=X@)7
zeMcan5+*6BPRV+u_AVtmF+JT8_3Ik<>F?L%eap(p2~0`~<Qtz}3ilQ)i)*fIG~;PB
z3zwZw94j06+9GJ#w(|I@y~1egaeW3Gq<p<ZnBkAjUMuJ6S{G5X(LJV8XqLb(-pVUk
zaE$_w{}RU`jh#SAr+{)T^Eju_`TgTT48N~+FLKv!EBe=9^Hn?EZQ&E<$X2Mm;(Vq^
z1YbBkr2aSjO?DJ@RyVc5w9s*+_sbQ9R?B75fS_*?EMmFzF2SraG{3%I13#xhSNsK*
ztwva44<?`FYnM#8tv(*V0BJsFQd&g|tNdtGLyje}`NvuBSK_@#!$@~lSd;H>U~kAL
zJVeHRSet^LwiTiQh7J}+gnE#57yf;&mQpKfYW_oH4Yu@R?xO?0uENf7y2li%$1W?S
z&={q3G&L6MZk?&dEUHaMGRSfM{h_}l3~bvgq-o>CC(FM~mu?ZR|52dxgo;|qi?W=I
z&AKt7j99@cl@|`oNp*=7yXGbn#CzFaVf7=3DCfEEqfKPd<S5$<B!8Vn!~BHg^kAZa
zVccPD(=Dev<iBg5ox9;5f)1bj6kK)4Qnr1qKtkkv;aTtOUMk}Vu2d!q&*b0SU;c*)
zExTz7-d|Tb?T;=cP)E1Al)OaQ%rsH{NIj@&XP(D(<EOK|<X!9uKQa~5JLi0MeihmZ
zXWpWB#~mb%gKx0gApbv^m`s}@i*|aQh)ny7GP7^<G;Z^Liqt>)?MHStkBBL$pgUuV
z-@+kKx4K+GYOyimv0m9YYqAm@^Dg>w_2~BEZu@wABZK)3m96#+l#Ts2himB8;bk_I
z&1U~!TYAh>jdiu#HqC#In6wBRSVNP2XB_F<q%%D@hW<$aMy<~L<W<yctA?_j@=fCw
z{S5-_e8tW2w$!5_$gA>hkyVsQX|U#-TWhpD6HDGY#ZT021$J-=WPkS&{1sMw*N%VR
z^Guk?EC)S`Q$!i6eKtA0>UXiq1FT^^EJuj(_%#VTS$2-hZTT8b0j|5YqMNIy^Xn21
zAT~z-#Bb215S91LBBRS^{@#ZFvf?}w@`Z@SZ*Xk8z(nhlm{~A2hBgXzOk<tZ7Mvp=
zC{U}>%~q(JWu5wc6Yg=4h2-#?Wy^l$4J>0YHbfJ2>HArxt+Axv<;UYsWlOS9l0+Zu
zQ*M1mEwXz~ejsLiw;Jn+5A<dG@$z(*3?;kk2PR*CBMT(Jd#Nu?m~-z=;hl4ToG|oT
zlmKZ%<R(J_Bh&l-V72C>kquw=E~Ih~QZF8}+FmzeYC02U+k<^J7j8i^ua3dGfYtI|
zGN|Po$XNL~ecvBz@R2r|G|F|?$DwNprW_YcYFXOw$kBAn1f>>zUVYl9yq+CO5vMzX
zKjnDhbUyQuwj*`_CU)L~I!#nG8_AZEJ|_2RwefWEGT(=F`LY9(k6HZeoKv4){oOA@
zC|9F^oOygnMRR<z>qK2Saf||`JYC?~ru3bRiiQpq71gpn7+{O;VWDcAna1z>>GWg=
z<H4U|H;#!rI?Z0&09;;5DE-2zBAP1Gt3KX}XD;o*4DGD2)FtKozug64?IU#MzD6Eb
zsl1}`M>aNJR{|c-wD51~tDF48if72=kkv5`$Nm*hPH2!6O`K8IOr_%Fm1Z`geN?4z
zuGJ(~B-S7MR_lKB&bArv$N7r=Bz3UyOGk&W=Ck}JK2w#VL8KUMvixS3+cjA23wp;G
z{#eta!!q(msTa*c^!s>UtBWb6H%wBHnAdgQ86J|qr>N7h{lo%zvpiiS?rh~a-R`3&
z%kqas-9V@5e{3)J$A+7v!`=yi^B~={<nAhdvMXWJD~+7{57M{8pttZbVoVPgiT|55
zuacMMNZ;)j6V>bBZSnE?UeYXQzIejJ(W;ug<L<#<1!GMpx&&JAV1npami!AEaRv3u
zbjbuZr(775n0Y_;Z+%(0rH;o>GLS)-(YL^Y$G;hWW;bi^rO0n?eej&6J`heC4OsF?
zvi<s_oQF7P=KsRqAddmR+v?e&)ZMeW<GFy3HXX%MEFV^H(H*)^F<h~AuKg@2b@y7n
zowl0&KTJtahC67CO**fH$q5aUlq${K?qY3MEdn+0_}@rMqimLDV4*)Xo)GL<rq=+s
zW1sGQKke@?f)6`pdSX5J=8x|z;({o`V^la>(xk9UiTvTf0)d6yCF1KlEyt0dJnOX4
zr&X0dWA_t(jSzg3wwnCwu{iV$N~)kcG?!VmZQtlU^mUZW1WU8xUY-#fDwPOx#k(|A
zD!f}RT6eLQ56lKRQ8qXAPU9j!;<7l1dPhdzX!Drnz$S=*w|G}5;AqxeA9&b>42@Q<
z;p~ABwWxW|=g%!GK{e*x=64Heh(5WOSF!!ZK(^%ka07~!o&OWni5+hQQb%Wv84-6e
zyh>^82r{QhD4VXWl8xerE_nQscaZ&>Wlm!LCi!F+(x{C`JbsYig?->ORmq32R8+37
z_nZ41P*a7Wy>-Igk&Bm&gA)m!>>lLbCEsE3q-Lt|`Gy6-@W-|@#~=3iYB4jcer-$;
zD<fh**G6lx3&kcAe>2ZLt*4lgRU}Tgi9d1D2W-aB*}wH{rBfA;|5Phmw15FMEba5w
z$hx5FHkQ5z^_>Or6kn91slk*~aK3U>4jLuqDF5UbC&EZ<TM{cFvNu-Xk6Wou%qMbA
zvaYo$$|7&j0$QMGc?uj7*5_PeW9_YWqa$H$vA-eWDQbUo@CxD|Afk0?3G>G;9(aJV
zA&#Ga)nekY=7F*~aF#e~=R?!YfO}dn>3_v@8Zr?Hf7@uLoU4<OKZuSqoj4tu51}B0
zOgkb!osMGXj{Lf2doWojHq%xWtx@cPKDAwo-H5r8hrK&y-PH5>zm}xAy`*qh5{kIX
zHk3nP@G-03_~X;j`?^}rM3*FRMeg-_j8`u~uzP~(3~@fcVwHyrpN#O?o}QdJ&$}-L
zx>@?4EC!&5;mm;z6^9>twft#BXME33F-#>QkxY%F;XMuEWp8+VuVW~w91EjK1zzrc
z?n?wkxFdf_d0*P&In<BikiUY*GVCVkq8xn*xhz-G=#c2<sk5JzFYej-AvA0iy*!e&
z9)n|LF2U;QqMi7y&&dpbSaSQI503<yoc!~opT~rZr8cH?IzH8WY^LV3xvB<H$VK~w
zgX^P-fV$o9M++0Hw<39~yZ+VD+x}JP4XH-QW|3G&^Y#O>@>-|#0{qnQ_-B>5!b^-e
z!o~5sZl8K|ayPfOz-?Fyisw2+?F`tkK*#+)+}$izJT%~FW42wF``H7^J3sIXse~kw
z>1~p1LD1tc_Ol<;u_!_B8dlS_vBsXn80_CyS3$J1Iz5BYTmF8Sl;e;U>Qmd~<#Q*c
z(Y3Q~so?Q*k*j?1W9q%N;}`F7Am^RJ<KQD9*fS&u^|R6N>m*ubI4R%{Lk5I6hvCTz
zlk>ZK@Y|tEVHSfJsf9)LIA_DIW@!bmUv0HjKGHPRcxTudJ8}Q_uIUqH<nY_9*^PIU
zVvj~ZscD2U4N;7a6T#!R?l#M46w?n&9z8*&Qkb~4GH*a2w`0sb7-A$sHh3bQMhe_2
zwd#Gt6WuJHOSIG&ck=BB=`NV^X#XE~QJeX?%isa(3+MGZ6b9CRj0_+4utQgUJvF?K
ztNF--kX@6c;j4Gk6i)+tOIls4F*&-X6hh?6$I4j8m9x}oTZv7Bsv79l3atkc1YH?9
z$#%c@U=>NhRQ)2S5TJ|=artWs#{Z|_En$$PSg%gKpqtm_@W{@FLr);8Ie);Tz}lcJ
z)7R>sS!AzH(j^M;@Jum-@tm2=pd3CZ{66ML>0X;OGL<jwfsVVk5sk}LZGwZ5KA{>t
za1tefl?jESOS*@oZlO7dw{Igyf=CecZ}+hC)f0M^bl-xY*aU`Yq14!mzN#3tgbD$p
zXkV3oq&T?G?n-fef7by=g04lV)0k(I)JB!q!`+NH5C3@dJG|3fK+JZmKO+;~*6*>^
z@HekUDa*=+&CkEBqI}!*;~X#N<ogRKs4V4cnkHqqO&Rl<kDZ<PGh6+O<muD<2WaEY
z_AE8?UHiU7BFW>TkBN+low-lCdq!{{0+T_^XNTnwnQac2cPKiaG+LQSg~!jPcknlv
z?yuUX6A;S7im*dJ*nOQhRy$rO9@p7;eW<`;uquhv>dh+Bm7sL3N6A<JrKaojz=<yR
z3^2-b7UMTAjFbc@+smpc2md&4j1}k-+Ya-Fue6OVR~(+*uJYM3&kUM~;aXe1aC!Fj
zPRjR_+#Ti>)~AwjE|E?j1tgS*w6xCkDW_OKn7neObz>_`%>VU%xU<4|`~gH6-^U~%
zgV>I=#_C9xL34uZ2V~{XAup4(*x)MrMsMAf5e7}&f*E__uID{uNlF7Mssf)iSDgwo
zSu^H#=zyflLVd0Xc@#^C;kQ`)9g2PhaH6wspJ?0<@)7|GR^@Xe`Cc!@rTU})S+)Np
zgz`}QG7`ecTrm;{bFBOOA?dgv&sipFkVBjySTuNgw(`mpX;vdk67$p;=ZW2-a~MwF
z=$l#tJPQc2uL!TV5=*f!mP?=VDRQo!Drh1N+ShyA)F*3Sm;1ha&{8vpR-c6Qb$G?P
zaD=elg^^Ymzx%d$*dOwoXu7Kej<!sVfvja6<4^e1_NAsO#r_1L4)4NVbuqr}hn}RY
zJs5rses|<rX)}acVfcd|%7+==;x)kIrw+H95!$!T8AzPru{;7*%}rsRuAg^Iet8qJ
zCrFxd;Kolr@jV!P3P@AR2$s!qGd*k3c;_v;rP-6DPQ=;qYiRk`6V_Jy$4<&zaDkdN
zo!RH#>{9;d%fyUj16RwYlR0`?&2^rfjO#m6r7=aE-x$+zn78unk?mtlj0a!=1#+|}
z2@W&mkM<scV|>)e(g59hiArVRN?40mu{L4G^UeMd!ktqQT8gXbZ)o|)!u%IEWGpw|
z@o4o3*($ytSSRScT_C!NvJtLl&<mEnIY`KO7k#FOoZXBt=lSI*(LVwW-Dv`Fh*6bN
z=FiU+<{BB`QmeLHQr=HHE5tjUe>VaNO!<a%1AA?;v_|AAPVb73?CY9xY7kNb1gG<U
zDtF=m{_ezL+TA9E!?^UAdG#W<A3hm3Gx&o?YNa;T=DC@mlH{+*JxOr}b}|5Vve+XC
zZ9<6ebmIAy+ch-oF5|~{eAMsTepo5##)293vgbkTcWiIE`VW=ca~yZ^s>;z-wb@PM
z8DD_a?^SNLfHn&~1KBeSzoIhK663DHkdf|RI!Tnl%0`#O%wx$R{p%4`0?8a~|CTv;
zyDS!TtBc%csTs{;f0m=0v-0!x%ECyO>-1B&VA!?nBjLGeS^FN6%N&4`dkhe<poL(a
zKb;sp0Z(DL8F+?qSH<$ol#5N0X+f-^ZiZ|;yLi)KEdbbR4ZM*15k?VMQiHW%DFZmC
zIg&&JtysVxvbJ0baeOOrVRO;sbW$CSspqffnMEWyd@^vX4;krlhdAWHNWUzk(oK^s
zb@>HNw%PbmJTDUP{>sL|mF*N#I3=Xtbv}G`j^p|le}!=|dig6?-s^EEJ2a2RG*GyW
z$zK&69HK3!N7|+^wUlp3a_*HJe?{SRCzc;YC?Q8jo|oRLmvw+&$MKa>=h=f9u?r?)
z>*A1_&2I^WUGx(q-E~_-XORq}vi^+|geJ5+INs>{#pY6l-M76c&9aF~&TO0}I}T^K
zjY6B^_!vnFLZbHD-0&6V7?sr^tA><vdsO+4R-g9^qee~U+OxtdrYY>fyB>ZY!QszR
z*Kbj(BZz?lQSHKy#nK3NlucW86V$psFHbx{*@(Ia`_rhg(EmdDFx<G(h+8qh9-L$w
z;!-U!4380*=$(#b$bJ74u)B|0^6N+~_nN|`{MWilk7NvT>(o_5vtPTIhQ(TLUQHJk
z<u=+Z%c$LTKr*x#_bPJX8=7IU^#1VI4S)Ca)X%0I_wkF4-->aT<%u!toIMQcJE0{;
z*9nkRdzk)TPUtEwn0~f)x&UdTDpJ1bl*(8lrHylUYzjSsA|R*l`Rl%#WB=GUWVv)R
z1RWic`$S?Hw3b@9`L)s~4U76Bh)0UkiQjzYeowW%6^GGuCmryW{mo796Ve%rW8BLV
zUO=jN$V|3}J(nb-?`0u<(&+10{I1?PXN&DNDeyl2;MOIh5rmza9`1BtIKK;h6tZ{S
zIj1V=$=G*n%5Hh!%bzwaCHp&4z_j=aDG(c6r9H~BF9O+aj>CcJ$6uYLf1(E;r6v?D
ztgrmg!2mqY02LGgQ!AaP9Yl;6ky0+)WYg%)+eS&IT@%p}<J=;#+8DGIlifmqf)cB+
zCM0+NmU~WGs)lrXd8Pq{m~+1Z;f_xQcTE)E#VL6*PDh-EPxd_jtXRnfGws>e1y>b+
zC23^UsPESq+}HC+c_BP#9}XOu94ER4NLjga31^bUGN~3)l`mYx<A3gK$g8=tFd2Qs
zoR3P`Ygz!nI0oi?Wp$@I@4b#~gC@l$e~L7|i6SK39w=Lr>6WV%y9@fAbRPxAA?C)6
z%(9N}6?|$7s)MMFmc0b?jqxUG8*jJs3Q+=+6{)$O`s5byhVq;ZRHHj7KBP6|9x`hz
zi_tr-ve50smXszVCvW27Lj9|b4RuFJ*&8YdQz1y^a}I9P?i}rtRk`){E=X<I>$yTr
ziy|lIp8vpi7+D^DK|d`bpYX41F-Ili%9K;)B=KVL*rV~6yba+V&yF-cU?6##!`E<E
z9ao|T+Jcxexp3am+l_!<w-DEM{7^S<nDX(HL<egOy(|&>8azg9{%XnuWs`F`&1m;`
z)Lh165a7~vcP~d``(K=7_m1bi7X5Z?;%ycJa@|Xg@K2^UPWBJ$Nee3lZ({nHz7u@a
zzm01w@f$(N9>^l{S^NHxhuT9g0nQV-Crc6TtOl}Uw98?ML$UCHQ2Q9op|7m>LfOSU
zqcPHnQ?aDQujhO~MJCE-fG(+y$L!*-t`d4W<FBJ{2oKy6M}F;9ion`~Jy(z09dI?%
zz*X2ojWwa<`RJMk+fEtWtu?+e{_Cyt_f`^WCXL@?<+0by<8#_LUx@gJFrS650neIS
zoW+sL0Lc+~;GN^IzTC!I1H=<g(!1j6IiH!?p$G7>38V{cxsW-K3<BoSrtFfo>-LH0
zk>%kh8)M;8u%N<9KVMYJxSSwWT8LmvUsID#qi(Pme=W9V%v-?kuKkqx=Jvl0jHc5h
zP_M%#=e6*UmGSgG>T7C_pYNYTt<*1??4}8Hr1(?b4DPgZJOCHS(BTOgDl5-~viY2B
z35Q1)ofn7w>~Az2Z$HxXq~v~TPFKO}x4(ccfAnpFASfO_pS}V94qX$8X0$Io&tG%F
zigpHP<y#RqS?S1_@fe;dGTZ>)%&b5nC-8@n;aOw{V~A&JY#jB}=PA*yJH#3K<Z^mh
zh!sxQoU9P#3a8NG95uA?G8Xv*qh+pAHerU~%eXgq{1K2KI#8%Yyk!xM*u@@8skjj!
zMk(PQ99w;2iQ&7GrMF2OuYAV26}r!vAz+AO9@l8@O$uZ9Z}>hr_doDH#os`~38#1b
z$9tk1=b-xYDP%Z*`WWuG`m^E*eIh(LArWp6mIfq(p>@H>o8cdGm_<%@l@x+)p)<o1
z(pUcItPnK-m%8t$E+EWb9y-Q~WXgzo1o&e^m5<Q;%0$lL&g8L2`^1r>%|<|f<F(7k
ztHVDpmEzzPj52*%i2vPR*Cyrp1<F72*F7j#GOM?YI(2OgQPI<CiU-H$@X~{(6IqP>
z41c9vwF&+w?C}LsToIiRP%21;JSplEGT`m_$)%?eX;Rmz+lyG$24iKY{4rh?%KnxG
zB*Ij!$e<NHV&7vIf2+KbT|@4>lvfZf@+L$mMR!<0e?M=6d!Fri9NM@tWo%D)oB7vq
zz5T+s=<>gVG$erZp$TD(%VH5{V?IS*op2}B`U~5Wz!BLOLGostZ#M**$k1U0ZpN7d
zVSgF$1V}kLIr)?~H8)3qw8{NdvS_5b4ZMx!%S*ooh(IWZU`jURk{^A!Vg%s*$PLCG
zu`=<pw7+ze^8@_D;KkwG6)9XwiE{*XQir#STP%>!y`IR>LWVIW2&y<AT<elhgEiI)
zS^XcUm=Zs4Chqj!dJB17tfvsm;&k&Hruklf??y2RXT@*#an8y_nvO=`W`O({NUuaA
z%zzGVKC6piFbTB!=~T!Gw$iajrXk<Q<p`E?3|@Ap3rF5uTy!3Pn|#Yc&E<d#$80X}
zUSRR02Yv{*mt6bE4D)3_gcUOe`E9yFVd{OY#RCd55FuesbmS!f!p}5DCThRj7gt~J
z>nm|cj1{kQ{s^ORFZX_}U<pf1<O0I16t^2)iPH0)A7*mY$2a_o@%Z%wG|Vzq7DZ4t
zPkbHE?SUik4?CGC&~@Q1_wzdzrGyePkPZ`Ga@vvU{QLLP8RCC{Lx1ksh}_2A#<aAe
zA0>{|uC@twx>~<5{9YrvaM%t#)8X2CjvxcEGQ9+{O;NyNxq@s5u=!UUpuO-m9fWLF
z<u1C-OsSoey96wwU*UZA*m<@Ijx_+oozGHc0ZeqRM6R*-!mt@F@Py~(xZFFc?FI9w
zjll2jZKXaZ1>9aEFq0&@cKPfXHn#Af17CFZ#c&$19W`{nni23|#TBSD$~g2(-&&po
zqMmSIyjhKueo1$@r_<FE5KQ4mOzCdVImLrl{V+WMMysQPZ*83jT(W-i{okcope#lO
z=cJiq;w~{5mqCFOP2u#%PJ(Lyth-A&`dp{eQy&&5fsa&F7e*Sr7R<-@Gkp)J4Z`gv
z;sY4mM_&E4J_KYA5#ycdYtt^kM#%%3E7fbSpEDSuY$GKd9eFVPhAff@aa4PPh+%57
z*unGp7Qg}JTS#?B^nS0ktZ{2tvCUv6WQRpgt(A3R9VxRZXR9^wyEtOy(ZSD1nf^G^
zWI5=3T&|PD{exVF7V6!@mbtO4e}-u?O-!VE7{;`4<b!W)+M3VW9s1addFZBQ1vkn4
zxWD#1W#C8+7L;@}k&*7$*ebtSViJ;6P%9~bd3O2tkP}A=&_7;l)=nW+GFFs`u9>|=
zm_sgDT*#heArtL<p&lv&vGe698|EWTma{RT$#t`R4W$1*oMeQWbiK&wB+`7eO2`<3
zl$M3vEtG%eKKT+!ANj|5%p61D;Wu_-YmH0ZT)o=6ue$!~Ze`l5Oc%Y^Hqrvf7)C&d
zB?j0S(+<1Uhnzsj307c|m#sO+6Kx!<Q4ls6Q-f{L+3)pKf;4A*&d<7I?3IF+F;c<C
z38>?DdFP>Q++$Bojm!R_c1*@iXUvDV$OFb#2r&3Z;H4UKG%@p6^TltYd_HXTj!m+z
zGJ!ofF8$Gj<Fac&Q0ZA^UtP%w8nsXyhGNG|Hi_ZC&nqaXgHTTi3JPA))MN&cqln?*
z;Xs*>{P`A5k*pm+gK}zb6h>0V`dZVwac7(9`;!GS`?@&{%GEnMy!SG}Fjq#b2SOm$
z0kevwOFeP=^!vKwUGorIq!o#MTo!&;NNh2P)OOvsFYB!0;qkvJRXzI0F5f)LjYW!#
zS06i2(8UR#Ele}6>{qb+{n?HXf9;kFCs%V!?UWp~f6IQ>rRnKuf`XyohW;G8(?kZi
z?O0c)JBhM=QufI8V9s|-L~^?C#kIImdz7uWjyhcj=nHte-7fnI)Lfw}3u9}Ga>lJA
z1=l}cJ{1EK)%S>{3O-tl0`con1f!j9lPk8<t=`fnS6O%yfZwHkSE2s*a(2ENVjw$%
zn8v%|4ER7UIR6!BcbK2~;~UA!(O1G7t{pO$efDYxFVf+l2>2>*eJ|E<8^eRrk5}O4
z$E)k_PVU9<zca*JsDEfyN8^{mlrN>nOwQBoJT(E>7Pq{TU?&`P#Yw1#qkSaM!Ar4H
zf8IE4*6o$$bfr7s4~v5j{;4f(fkdx=WzfBHX!BfE;3YV6ocH7Edvl&A$_Z2b6WpB#
zui?Db6AohyV?`;cyG3?=S2{X7&B1|$$Ha{Bc6$DpXasVmGr%kuKIprq7_XGBd_ss4
zQUz%sxnbH*N_%}v{e)U6%J#~YE5HrUd_`ZM<E7hdbU{Hu(`!%*QSb#8#ilWhQy_UE
zSOC-dS(4=s8DgJHnb!zz{`?|o?gR7r4-iVXXj@u4<KT;@Y`SyC6O=tX%JNFyJq|mN
zBRG+f^7S@_Qt0}+H^L-MNkv6{BJHzhY;5d|-$z7p^8zg?qFufwEkbJXV5|nq3tpLB
zT=64s4oSw!wa4s9qp_ED&R5s|aNs)T@h51#2lbh1E(STFpIy=f@P{>EBi#*jMt`@n
z{eoX4bp8Ql$&k)CHDo{o^ZWZ(e${fGsN{X!74uE(2r-HJ@t+_Z!I4!BSEvc%4ih_j
zbEOf%DHwx&Wsql|qlL!^41W;-$v!RE*xXn{<Vble+v1ye+V@f&-~q;+b~W*8h=~TV
z!X5?5<pWVy)YQyDVxXhgR!a~B%_Iq4xW6#+U6DjcrV)Y7{!8aAP_}S+&mba9AO&#=
zWI5T{j$OJIaOhMt6}x5w5Fwba%#X}`sh2Yj*6oBFBL_5EUqR<4FFpdRb@giUIw(5X
zR|B2seO)IZJ1LYcXXy}%TG%~5-fO08CW)rq8wB<kvbCN?v#O<)`UPA(#lF|JYsCvp
zEe|K<DcrOtRj0cOGyNX2eH#bb8rNsdK(d<pl(8uCHcUar0EK<CPta&<+ymmAta2(!
z%D{OLNO{=bvzJQZh!qJ_e%jU7cX_;)>>AtJmc|V~W!wPT&H!QQMAAy)AbsU_;cEJ%
zG)3u2l#TBO#4?K&6w0O|Ixzj;2cM3N3C|A~yiy51=R;=*go1M~j1T<jq{sr9FfBQZ
z)I5UH<pvS<Hoc@hM~jtl-9MC@4nhL_+}jO6#}0ElGL&jc!<MHygOu9`P8Z&n#5^jO
z_;f!aq<26_6%J`i(~0~i8qx&EAd|@dkOu-wrF(g@sP?%d>L!hq%LB)gqGxA-C8kNw
zp^}M>B4{A3U@9KSVP?=Ms@q>GGX*`@+$R{Bn96SPsb5zD>hE;zg=J7G86o|U?`0fJ
z?0W^H0Q5JNqrw#=hfZ8%ghBEm(4lSf(yC3|Pn<)|XBBz?c;npk*<`-H+JB0Y%8IJV
z0y@YpmO%GNAUOlyErp`GGGO#{^ZDV>_pLkW(q0cpx)ACu?hmyavnz<u0B|9UzaToF
z&&<M7e<H{g7>;b0vQg!c_cY=LemK86nljS8zwqggaX@zL^!1nvnJF8I0RYf7S^Wie
z{z?TYPy>h^Oi>YOgp7z8h+YgS{4X=ytqI=Cle!hiAuFrM<MPvdLH_H#mq@>p^=?F#
z&_vHlU5|398%(3pUI3cuM)Md0=CZOfY%A$W;cuj(kAUB0)K?>eD)ew>4Hn9Q@`pz0
zN++tnWS0YB<Yp*PV#o|u`D7n?dCIo+wVE{?=8kMFuJ%s<YUvPKR@1tmcRA@Of6YAo
z%rfO{6@nU|M^Utm5qcfiNxeRJBPvzOoG=RrIV`F@?`ZD7`0d2ULS-%-lhDi_I^|F3
z>i1<nAbqqiYqU_TAHxr-Ffr!M(XtSMeL~Eiq6rc+iiwII1L;|T`*)K}$bOQtuxroC
zC)nKf9dCV{_m?rwoA&FtMW%MA6<p1<<4=zNqPt-oWo@85dHxe)tJ0R74eds{W<-AE
zwV&?*vQh|JcOs{@`3oiYC#AmeJ%2X@$<Y2wA)T-`6}J9HS&qWLNYjzzK0gt?rj*I!
z>6zD+n`mcq9pWPAYlqedNVqycl^1b>wYUAdgNG6@`;TiE`fd3VQAQS6wx0wg;Dj+U
z(dze#vmiwIgva0gg$zM=%VQy}yvilr_NxasKt?TW(RpynVH0KBCc0+lHW!lIwxuPj
zV`gS%b^e~Ku_7>FnZvOBlY;2m<(UdnY+lMm=PPSy7{;96-^gBAF6vZrvb4|AJ^AfM
z>0Vn%=pQHXG*|`C0uf*+5LR1HJH{}vurvcXCCiD6&U1P!Z$rS(G_$71(*xmMR}PJW
zaVzk<&>@0z{A;S0q}cW-Y<LwrE>tC0y#K&;!XchjR`%8Y9BS&ZyZq)nrryckE3%pF
z#tkwg6aNaP$bY`0nlr=UxjmQ^1~OX|ATJ;8pELX;5k=G8C72LUh(^kvxRA^=i!i@l
z)`y2$-26;mWQJ@%9g~sc%^rW1eUh)Y2Ag%QRag`_UCd#?I?6zTK7TZbw=EQ*FrI&8
zaq1^Y&I$-)J+<x0vv~Q;?$mS*R$$P(^D)6wN3_Pt$mr-MkTvSRbct<`w7KfL5AzEP
zsSuH$+|+*Ze4L>BHctW^{@%ydOpjiCK9GYWu7`xSN2KEQ=j)bdI<lRIYVfQ-6lKzi
zdTSSc@W<Qx73gxh`o~z+$Iv!5S?u|>eijl`_|Bb*0v=fPwQGF&7Dxqh^{?AD*F=<&
zh=Ay0X9VeJI_UOvYJNH*v+>1`XfoMv<ugD9+Cpi(f9_9>37#_Ua2`-xCajG`4(E#`
zh5dizkYxEU(b!3n<yjwk>CF!LHNEe_Ax1u?OH;CU=0B+287_B-z`bVI)S+|*ZhGB4
z4rw3gUP=%=gt#NE?s{Z4Hz!BI(09cg$rSp%W-FiEw5TY7da)7QsDVySL2<nA>UdOp
zw&=Hk%9j~IECU~%+C|1@24D6MF2KP4$Jy2mHo{BhueFP;!lpi!D60>(!|u(tDGWao
z%dMf|-w%}dgK4Nei2f5IAOs!c&5y5&Q)kxT`|0uRwFPQ0`sNs}9$GBhENI8U^KW0J
zk-31!x8wk`mPhBX3)CR0C=4kP)<?(S*qMAOf&c|9pKLUT4n5E_hDRui_KCzG)%^+V
zhxD8GgX}*O=ctY~fp=TCAX^1acymNNt5aD_7APj30wgfwv<4``JKC&^?JP?1-wKtB
zHEbuj#6d@Q0pZ^(@ZIoqGuC)`QsgTLA9b{7rlQ=`#YRS|s1F6>ck4T~^3Q{7wcba3
z?Nhl%*AIz}klHjCk=DFIt&0QP>>L3`&`oS1@})_ou#}Hu;~m19@heT*{)+4%=Eb!S
z4G;4;dXY8^e^p4GgXC0kZT$ZL6QBgnLB%XLguePx$RzT5tK`9P!J%Kz3%?o>Yoc+s
z-A7|*+zJd#r-60CNnjRG5g-oP`tGc;qDR1@@BRJF%otE2X2YJQI3|u=LP+!o(vJ11
z?`cPL#MJoVnt>>`JW%uatf~sEWe?o~=dAHD9d*p2EuOWc-%-BUv7I8f23x%`?SSMU
z%277e8pi$~j3C9$)z`@|O;1hLU|$x#XJOp>Hd&ni)>Yu9Vg#z_@Zn*Tc5ZgR>;*-F
zckT{s;$g(v_v_M*YNw3O`E4!xiQ=iU^~#&;BJ`Pj0p5tDcMFYL@m0v4BVQ6@-+f5%
zS<*p5cRMGQddg<*d7G{V%S7V&D;SA^xQjqeJug3Nvh4I@_PJ5D@34AUx5oJ>ekp=a
zsZXD(v*x<Hck5VMTLckPYNve26a{%A?^Shbx2_Ml!9$8Pn}C&aI-2KNlY4e!e+<Sn
zDrliwo0OqnLC1>tSCsz81%;O==t^E=vbgz&Y5o0u2y|a4M?L3tOuZCgrgm%H>4Es&
zAEF$`yMOwUlx2okBhUAAWJmD)1a4-x-8z`#0dfd9_G~YGkg6)M?cq#3K*hHA0kNm5
z9$+%&V2+u3jV+4c8k#761_6PR3GV)f8fjP{hy0kXPlcd4Gc7{8IzmM#ThL}wMWlO`
zjrJcp85yB;v2l8&uv3Hk>R<RslG@D)mI-cBat6Zg)gXBSg;s3p<Kz@zrkvq#Z*0u$
zRwCdwx3IY1WJNiy2yq*TcYL=_@Y(nGJ~n9Z;<Q;8zkA?@<td(vG1m#Y`>($~IOBh6
z_6s60fXL>mj+J$%p{_#iHd@#`SpJjbzFMCTgrN&a8LDiQH3U~M>H%jtSWb)!A{f8c
zkJ%~6ZI{NC2_y_08<~80*;e=n)iBkQ)=Hm596z``doWpWtn0;$*mm66poRgDSE_&S
z`niSLG<<#B&fdECYa~x&mJ6E4mNqh||8;n@#?UHjWY?m}va+0RCS!WEoSaQ8W7z~V
zi;#YG^QUU2#vd2@SY?M-|GLH4<IwYPHTV3osl2q(7|x<$N3LJYt{DI7Ipw=qJvi~a
zGi<#&k@7}H{Yd5&!)UgU0zOtQ#fXCN1FY<Ap&d``SkE>@53lR93ajhg*B1+M9OP=e
zx42&MFnzh~j+;S3{X=ly-wUH#{>LK|fML)yYu44JKkjab&HR4!sNe`R&6<4|sv6XH
z$THndt{)wF`|_T|>|VC0+{*+t>Yt>LY^**cmtz&!UsE9+jcR|tmn!f(z^3KRAg{(G
zCRY8y_XlGKhJT`N9Z0fQgT$%uiS<==<hu@T4#a61w*pHEC$NPr=CN)4tQY$V{BYk{
zmnq|K)KO#mgu6D;UltZWujCel|0MC0QF8Ukn20j3?NpjvhDgd+7@Lv9I@$BA$7Yxl
zQPzuh{a`J+rZH3xa-gGX_AVGhV?{vtU3h^Lw~=LL59#c=DyaJqU})B)y8ShkHX@IT
zNcFbg$l6!AGUZH_5hO!EK=6d;oQH$%<a{lKasx<+Oc`j1uFr|DOd4%XsLimJE_TRE
zZn!(Z$WfHv#-!1}n_;eeDRapyuXuI)oVx_QApmO_T6$kB&PUIBPS~wKjdznQPfpn?
zYKlshBIxo~LI<Db-kD0o`DAE|8L<6H_VPfPj;Cu?m-YP1E}MKsoAbRh#94jHWC{&f
zKHM7`VSqnz6RWU_7U{Gd<9;{3@D06?i15Frt~+1LI#k(yR>4KVRF8yA+jMf|>h(XJ
z0yKg!zN`xT*7mut!_+u!oT%NaSjLc6ht9m8!RkXe(g23JPW0`Q#FJ$_4Pk<AB;w8k
zJHB;zcea)?up?!cDO0VIn7t+bmFI~1hyYJ9kMlQ`L`P*z)BKZy4_4RSLFA*pr|uda
zNjd1ip&ol*PEM}q?j)yLtf9Ks>@;x2l9^MEw;CA-zy3@{wVsN)uzn5E1%}#7MD!)X
z8pozx_w7ht4GmS2BY5Fe+NG+snTW;+mbFN&v%P>Z2xTk2=lP*+YXBu`I`x6`zU%I8
zFHX*iUiO&*Q#=!^<Pi6q{_DB&tmnlqxzZ!W=EeGBRa*?X8+ib%4$6BxpE9~9^0y7T
zt&E-p62Oq*YL+OU(QgzzWqvj%3t#(5E^UWO<GpxPR!;6(d;5x?M@_A!MhSH)^4(2c
zE|cV|dwq_90Y;I`;st*Xf&PYg{P8fHc@Dk4Usq@w+uoNQe~T6q^VP$30S!DR-CLZA
z<El0;_8(AW^gN~VM$I_bQ0iFjp3p2?$Jxqdn|&Ns1l=4)Rr-kTd2!j=ci^$7qhMB;
zV|}KZbW^*i;GP68<RS^TonC$om;VaR?Vx4M-PL&rYBytTdh&8V1~hf>c`jNdUSx}C
zg+gppZRQ!BK8By<2uCO=^RCfKnM+)W4O@HxFn<3Di7y7y=8tqL+;0?{L>Wju_8N#=
zU7iEOcpME_7mftyFDYl9bFc2mH_wYOz+ur%^JCQ=#w6yadEh1DxjpSKssc3K4T43(
z*Ls;x3f|Fvw2z{7>QD}k*R>c){<hPCB#%#P4ugB*?0u_)Ay?e<LzwF4dNK#bj%<EV
zIk|@9?3*+^$m7<(laV}pY{Db@FBT(mygd%QAFB*Vk;mrsIR{ZbzArM!V=f&n%2Och
zn|;e0j$JMQAGD;g;vG>8|K|%N?5lM77FCBvC*EhLf|P>j$R_bRLHGIymtWY*g3sEs
z4-biL%7F|&>qyzD{S?}rAUom#!ODA;u$T_^z$NFned#60zA9A?9d~?*Mw$VhHY{?5
znC0p$1OxmE)rmA(v0-%k)^4xACJ64hJMYG-ZJf?R65nL$l@QUZmF#h;oksjk+iw$r
z)Ob;EES8_09a(k|l7Dx4gNv28=jozo31QE<Z}q;gBfSoa$H@Q8<U;5E*_Cdds^oFi
zEV*LKCM<TRQQr6pdY1Wc<)IA1oJ@b{zxhg}ohpdouflLao{A(qUqM|k>&f&QH!X%A
zq?khiIr`~((nK}`!|$~;6^6*70<)dCu%lHgAUvnyV%du)nO^dC<>+3zi+%gl`fq3)
z+8rM(A23EpRr&O3lw4<Lb-1bW%0ot`0D(2z3-z)_+c)nv+mJJ3^NUkL4WqLc<vpx*
zGd>-&L1I-hX$EBkV*jrN=dAvR{m<chk~T$Fi5D^n_Vs(mw49AjpJfjnrmsqFk7vw9
zx~tyeFWS?;Tk~gAMcIOkDm&;zE<X^m={ke3F8eGYTALl9!eui889%%(m8-L>DT83c
z4J5}`E=x}W!92%rZ%?;(+Elj-j?n^@r+G0?B@03h2gIt@nzF7Ns%{s9H$;@=<iZOI
zczP!NC%Ew8ag%>6!vp>7FHkX;C8fPEq>;6y9{dHuNaPpLLIPerdI(oX0~ku<A`#|z
zofmeo?U*io2$hf{)><Z}8+SaF<*|B8*WTGQ2s!lgj3ly@k05iK9IMLQ3;s~p3Fs47
zX<ztlZ9*h__7nV}cA-VE(6LmTSkIh-UT*-2-rwIauS1|lMCmtEv!^9O1<=29Vml$s
zW_0dS3BRLAYl~d?lgp-CZ8uNufjRZxmyGTVUS-eRC6}jaWv*Yuo{p>}%19|CD4liP
ze#0g^g?h76HotH*rU`RR@oZcu+}P+7ks&FZ;03#4{PF%>XCNW&8<F!AmieJlD=;r)
z4XS7NNZAh76`S)^xFy&P()I`E>-wl7vW|3Q@xMKxB7k*>YTn!1lU<n+-Wxx9uxsj{
zKRQ&O75L;F(e4X10Eiz-+n0Xcmsk$8q7jpMFWb`l)vf~)hCmHt9=g+RkdJoJop%`E
zP>fskMSlR|&lM1U%MR|m&%*Z21ks5B@wlcL!1gTR)$ZXLzm&M@8k+Z2pafU|+E>*E
z&@UhZGNC+)8z9o-{P4A9K;lhEJTvUhu?5z7d5y+eA_(3hXCAyCMM6yv5OQ`w%R&;Q
zL&zI%yV~-sWulKJqGx1dK5=h#`8r^wYmr{WU1jfpa1^oh#JVO%cOcFXfWs%{Ako}^
zNem<$zuQmv4gl^IE8_KHRFs`qyfQ)e+>upWCa_~JObtr2>aMbjkB;A9K;hH+8jemQ
zB`SC3t{tBvikf&CH(H_1*m;sGw1AA_yxxky2hl4*jK`qXyZP?P&UpFXA|bTCsN+M2
zR&>3^Xp`*p6-Cf9+wDs@IV_Zu7NUd@M}c#{yy4H5fU;HZj=Fgh)!5K5CrA!57<7k(
z=lqq+*fK!3-L5zP67XWr78<cbew0Ri(K$o;lh4{xSz%trpl)2XYnD9}-qG;73lMZw
zdnh9Bj&QvDG`-fj4)w_WCgPOh%md<0l3a9YHbRv(SYQ=y0`l!_&1c@;E$8Y>dyLE}
z=Le*Mj-B@0dQmYM0KbQzT^OV~2R=P$+UE=a|IgOZ&RjhKodUCa$}cM_gfjsbRAy#?
z`t7D|g@DFZuj#|6Tl)`r&N_-|UE=gx`#9*ml<8+WF^F9rh1vtEQ%6upDc@VFPS*~c
z#N(g3HuTNWHlY7B$f1Z0R-l~7Ei+7da!<VDAJ3NxgxWu7s#FWjsDIex#d~L!Ew?Bt
z1gC9pw1mOX*j997hObA#^zu-(_Pea=cdF7?_nd~*je_UI!GFJ0BgUj>{oQl%)l1xw
z`^)vx0WF{D#>=rHWp7>fT5_Mw#7#tzWuY^+^N0eqJ}!7*9Dl8?t2;s3@e{g6vZvKv
zU4TNhpdS>?-ZzOT!|gyM)ZmR{K|OQH3qe{-9E@1GxSnYaRQY(VELU&kQZJ^kN%7{7
zf)s<x!?F+l2FjW_+NS{){XtYW*A<5dMEETAC8W;`D)pS&1nuk&7GAkxIgNz%&9zx<
ztg^5M=-F_v{j6(1N^MkL+l4O&+>GVkSHV-sr&yJu0;)_;wg?1i*C@U>^9AmfzSndy
z^0v}ynz^9d=jFi4$~xaFFxX(7{&=*q`EjnS`L+3ha<^GdNMhzVUBym8u^9y1$(D&t
z_K5hhTT;lf5*0dy>dXk(@O_p>$8*#ACi|}V_x;5$5FU8m>`+aELi1!p<<^riOMgH9
zWLSTyXP-5=TRWW`Q<qdn3o5dysYZoHmJ^#l#SXY#b84viL<*hxtRDLxhXYtBNkApW
z^oBl_Ny$yqzu-#xWF-N4z+`%V)UmG5*UJZOwPs6%w%m*g*UrsW2e@{;MR48xSbui%
z;*3)(gAP3(yJyZaG0BFj)oy(8TvNW%8P0gtKwpfinzt~5k)1`hz$xOXR4!wtj)3e$
zw{mpKaTfWM(Z-JTr<AHLE%y7DE}F?a6xP|Qba-J?S(W?qc&?l9QlF834b9p!a)!UV
zie1PqgZ|eXkG?}2wy#QGpZ?*&hM0A6?{dq!uotiy{^S@Hv84OZH4QJc4~XsGet-lX
zL7~F$kQ^=Ny-{TK?(#{`Z>PtBBvjMIEmjk9);$3>i%<kUXi_SKRsmHV74^{=cq3Xl
zn5BB{3AJ_#&%=xZjJhAe_0FDW<KW2Y&%GIT()Z_8Y5{wF^Vl&=Ryt1&0GqvPUzbbf
zx8QHun9&kaAC0`|G9ja*GjzyqZ|(z{xF+2N=sYP~69EiX()^Svp1|gd!nTXeCvt`z
zs@*&QsDUmcfQ!yvdFgs>Ry>gSuo`s)vr@b^^&;@nGPvyt<$3!75l({)XRa@`fG=|>
zZEDIc(I*RSpcNUrT9R^}s)*OL?tpbq-s3fo_gfkKS8P+eTfBH_v{UL9;B}%2#9F66
z-C0{X;nx3IAVab6Qs5mpM&iOa-h|yvM%O$=mm0lJ^zg;Ph}mS7s$gVtX40eC=$OVP
zW%;kwMp2FMEN_50m%J!(d^^KwA<EeEc=W8=Q=fG8apxQ5&QswNq-{j0T2zD=;W>ES
z%R2M(=;zJj*M8Qh7v(}F|H6P~A1-A63)UM_PHLsj<XI}#&+Mt`7nr}ck*k0A6P6{e
zyR1t0;Wi4(gEO$1G4j-efQ{IFY}(>b(z5Nn@3wSQ`7x;2X(K{gNbt(eE0=@E%Ud?)
z770*`WVFx$zWu%>h+^V7cn|fVMkpL|`pm;8CrRF4C^|oq+Fv0Mr6_GK^nE~G4Bk{I
zsH~n8XV#EJoizj9G3Zs;Ki*$fXi*bjvj#a(VD%K&fKRqR!+VD$MRU4UP%uJiM4&PJ
zctUeob3ohLBVyzrAEj#E7B6U6%$_ij`=EL#30jXDVU6E96N%fd;PGv~K%xroTvv{c
znr?~xS+=#1Zdb_PKma&bNJZMVy4>~uG4|C_QE1)UR|F{uMMOXlrKCg}qy~{Lr5hxN
z?#=;4Q5pehB&EAsy1P^9mTvgYpkBTA@~+?ahpurg8BXl8^V!e-Ttn{`iI(<a;iaBl
za+LWO)GV{ke#{DWK+?_$5l_FD0Aop80~(HQqx!r`o^9Qo4!1v{Q-r4Yd3K7tpLf-t
zI=QAA!md%LVQ*7*-HCxRG16~;!=hOz|J__&cthTT)B}T)>^n{u9#uTKc|Svo<v=F>
zR@fx|dOK^aUjm@wn7KCJroT;H^_KGavH6)VrZw$elNY$s++0bYOR%eu8nHM(8o7gy
zOB`8d45HhkIO8(A7=luhk@RA`kM}5y%qwLsVk!vgtD`G&WLJM>;ADqy172g(EkN*m
zyp!Vb5?<PgwX8&Vvy!e|h~NajK3?h)El?(PR6AOwdGu&6vQUsFXCBb;+v!(sE<LRG
z0GK!UI0a^v72V7p<G2@}5rp8y=0lLuP0bxD{_L<l5BGAWh~z~3%Rbixo}8ReCyiq5
zg0D6Q1ANPD`FWGz6=mQJ9j;cM9j(B}?d?)w6;6W+=8X{iW+$0&H!wv2xq3vUV&&M$
z6%^)L0es<f&Gz^Vj&tmPA#|9D1AWsPpKvlzjh9LH5qM{3KtY6o!nmQ$9A<Jk82HNK
z*~Cov>iM1JUFKK(J^x5>eIdcl8v0Hx(;;IvCn}n@hlh4Rxcuo96L5_P^F+y!B~>7;
z`+NWB`y)h6ES;pT%u`6-!4IM;Tt0C98Z!A@&b*_p_)eQDPF8h~<IGD9W5DZyqq5Qf
z{ZlA@`Oj%6i6vDhCmY&%b7f;p=^woI%sbx;Y;6rASOip1Fhi8#Ui1@$Pp%MLe~=ct
zcAg{_W_Ya?dLzv}44+9)`26hD#r7CI<AoAue-&J}rTA8RfYb-jtnq~coWQ}sP2HRT
zvs0Ao&=r&OX2+ekJNS^JKDU#CeEas;Iegv?$jQ0xD@=UCQ)JR@y;Ta_;xzdomkQ@Y
zOE5VsT`w9`|2})BuhLS}$bh;YOViEI-{DzNG5ShoP3%?-ws8{bguqr&oK{vakrO`t
zWGcD#+8}(Qfa}SokO3HDpci}Q5y(vQ=6{{I-_Uo;*a+$_7e#)?d|%{&&-<kdmC9sH
zu~Z-Gke!*Dyj~hw{{X_cCn1y;1|{yFDrI%Sm^{#E<uualyEL%cjyBq?oACUqd&-bC
z_1ob`ZY9>}e%X;%AOT3gBYuhnVZG2ztlO2x&%3J6^JE2g+rmzr?kbm4U2tBYyddev
z`uvR{NAO0P&uz1cO)q9>H$*{JceDhII7<X^9V0wVQU*$(vH~F@0=fvLn~j7nqYikq
z>J^IcQQ2TZg#Z4d{cR}np#8NZ1ULB9H};1jg)&ysS2(QZ+c&MX5B7l$S?fGW2%!_!
zO@|xE&sP0E{2rG1u2_0dfitbi((8($gU8swAMGdu5SkZP*@4eGu>i7^u^Al1nNLH$
z3PpEO4-4qtWXvlxo0=rLW78`_iJ!WjB_k_KhC?a&0oc}y+P|PpGJ*#W7^&LfijIah
z+b<&(f<ed%U1w7PZ=rwH!r(Xh7s%Zw)Q>J0Q-I$^{4g=OpQrVJZSTCa;lB2Id+@z#
zmc4V;PaQt;Kb=&;i>BH`Q9sZJs&(KFT#eZ*F|3xNsZ+Okqdl{NueHw^^Z-4OcX*Ok
z+@CJgFMH&8!CX`_WR+xLHDi`$xLXXmAiefmf^E*f@(a=3YlY_<pCiKK2#|9~O-+SY
z9a3ddBnv;K6Yjble0|f(7;-~U7ZY{J7I^X{`N{_?8z56}i+meMJBY=%Qj<N@K8#hL
z?b2VcegoUFm%0stvA1Yv-D5|4Yw)pEA3#k`3;@as)5eO;!tgYldx$dP^U9zc9MO_5
zYOfQrnPv>`0fo0ul`en3nG!yyj_c%m-4&HoS}H(qA4JUU2rYAXG7L~qPU^r<#=%|0
zUIv6&ymq|Q&s*}>#!F>n4?U33xdg7b6&ntSf}AO3hohldZ*T82UHKQhlM89B$^~Ge
zh20=Au3|!Qs^-0r9tzb+w06h+2|Jr|F%c09eMIS%l=)Of3Ih{RX9)(mP;p}W=_@Z*
za8~6L1TpqGEPDj-T7dUUv0S1Yi6$(AZz}t}6Kt$7O_pXGPca|b^I&PR(y6daF7^W1
z1W;G9bZO2fV8rD6R#okW+dvlH<#zw#;kk&=5pO@<Cg5R_&}nUANz!#vShn>X>%|h&
zIhlrIp+gBgbuK!;u-{&RMG=T>)x5e}`h9tM!(s*F6dt{(g%JhiNv|pEoe3K43mas>
zW0w`0=7Y-~C^eF-==UGE!7Qn}62|bpQszX+?TQ!1@puszNJD$X1NT<aPp9QzKwLb6
zpsf3AG@juCgJ8~PfpS}D616!$5*SWR973)G0{tzJ1HkXRb93}XrcqK}GkiW>Z-If#
z3qDcm?*d3@rvM(c9!tb&`xXrittpwYHOgjCDIS4SHm(t5`~-CQbG2l)0YHcPYCAZz
zxf<A2c(L_G?StGLU48vFfTvrG@l?huyha@dg-`xM2>Mc(T6j=>e#~n}A_e@1)v4;K
z%Jt<a`32hy``6!UcbRXo{7J_KI6;`Qziv1(t2kb7XXZL!i>3RZKup7|8VkC?;LoEz
zqGqiwFL5D`wyw?<j>BbCK7Jq8e&Wl@^%r&H2jB`Y6slhT(Zc$-PkCWsNCMoCf_BD;
zTZ7a+^Hrxl5`Qi_J9}+*yRH8Ip|o<mhJWS7Z#Y75h9qs&!4VHNfEYn0LAw7p@eibi
z7~xZu*9NHch}&m-ecCS}!VFyS!figzR^BGZ-V4yV^<$Z`D5?`|{Oc}ULaI|QZkfwW
zRa&3#zL_hZtI|-twLG=xj-2$g9Zq(YFV<9Z*~Ra`*#Q1%eucwT)W%d}CG#$FNw3Ii
zkY(@Z_5fCA?C&?G9hZM}L;q4BMYmDrSe)f0gDN)V=u)S;Yl!NyMx=;EpZjJ?AzZuY
zz`?e?f~#sZ70lsvG*MZ1pPTdE2ak$n{7(KuLX$;z*>_4H&!J_zbmu3h{6Ewbgap>u
zl+;|JRN5OaCmNSlc8mq!f(fTvbG+v}r>vwCUaJ=YlsWqME4Y-Pxt;`@_)1pFSBpfv
zKa!FU$gx|E4z48R&m~_8_wqWJ&wstus{Pt)SsL}?=3w?WOSJ7nPu3=B`DG$N!*Rt9
zl&`St(}{1^;1#KFk^szDq9RRkLFWEz<3E@AT#kf^M#M|JGthJ>QGpKgC^tcMb}ko*
zaiJkTZmbtB55m@0c;ZBNnA{IRt8xiqNsO0Rkn11Gyth7(ob0cJY*L{kG)5wZ%G2Vj
zV!RInUlcD0=smc5G2-Q~D}-o15ur4ACoJFfYe4GQZZN{6nv+`p#_H5!s~6W>6>x^%
zYz@Ni{5SFJ4v~WqZnOgWt9&wYmhZ?xa$^p>>a5kzVoEqQ(6-Z4(D$>N;9aVJ!AOBb
z+SGdpfxWrogJb+>+dO0z$-@PM_uJgv(#269_p7I%UVUWE{qXROee4sG+qf57`X*13
zA{f-Js4fN)RG92d<lFCbjn7Y79c#~gvy7WPOK<^p*)%K^G!y+U!2J4Tj~+G3AtzWF
zL1VLm>}oYoZZfcNfuM3+XbII&GcX>$_#Fsn->=9)9#F%dsTVj;0<HL)f9$*H2Fg5(
zbMTDJ_=sW9)e4u)ymYLJt$>pQlW>#y7RZm2e;W!JW0IbGeaJ%2A>f9HsgM^D6OASB
zT1D7C1wD-!A*6>Vx7$P+vHp;wA3q9GJ^X-|g`(}_kYy4ewcy`9m22f){m_Iw-^Pm<
za*+P?g1n&S9H`58GVpeA6#u`5`o6RGqW`hVz?)!|C!RP~v#l|({DF#n(LgTV3KD6V
zY4#tYHLO0p-jp|Y#L0Bk&+2Sd3!k^-fsw>xw}h*<;;&2KL~M^j14S-N<RX#=gg&y|
zz~R^mQ-|>;_OA>r-2zQg&()pMp7$5u&Am8!D0nd;Md)zGC`b;1s+1>g0?!}I>%o0|
zM8H!lA9OO)G#V5bp2#`Pe@xUn)9z_`;J(sf!20u8%+dQlN0XVk$%q*a_DCJetImE%
zA0rN^@RVdYVk7uge*>)3VlV>q0sp$>-z&C9Rh=Z|zvkZ+x^w*$i>5N-kn42*y)?d+
ze$3s`t~cR4?6=_m`4M<N-<aVEwlAbLyUbjEJAcfMg(M>SRC!e=Yb0NHe84%K6XTQ!
z0qh)TivDOnj*%>)#->=mmD?ZrOy!9K6b1?|H)2)DMp3mis}Kk8K;kc}2trQK3XEf^
zwe@F3Jt>Z+G;aw{|K`mi{!D!g6e;W#&SOU$kuSB6UxKqB5eDyt&*KHrg+jUb3-%EG
z4tOtfGZDSX+G*=hcm!&*h9zv^ZOUT%<2(Wa5Db<cSe>h=Ecf@Fx7)4R=~@oyz5#@j
z&H2yVdJzW~k8>^octS|7AQ}!z1ig^4N}6#nNY+d5eZ#vAfzi6A{33z)CQ-8}LAJ?i
zHV@~#d8OjMVo)Nm^e1k}2()2&@h*SVad49`5kU)3bO9X*;b%IZ3a=*H{kUB%#kh?+
zrQYK++6jF2=m$elmXP&wC!Z-tz~Hf8?oYHy1Ih^}S#+Qk`1iJdeL{Ik!XYs0_j1H?
zdm@~#pf~I2Z0CW)%V)3;A%+_%@Jy^QGiY@SB#S`ZhGSiy0^>sPj=<#aM+REYVnBc1
zcZG?F`}_V~Jw!T6sWPVQYjL1u^^=N!^w0avgEQ4_t-BeD7)H1jEWbkDp7X&GJV<(w
z(?~C&x&h=bD=ZAO-2PYo5NW#S-dQ{RDEqqAu{^K-UG{bE#&3Ri3e3ql=nsD2?S`ZW
zXYu=pH|=OR89@AdjapK1>pd-&Ef`CAS^#A_CQB@X^LUHktcA@gX}e*mZ2QPiB$dm0
z`{l%wAC?NpabU`1+WqK9qJ2q<hX&psOdN;Jh!BiUdh2&Fffjdk2*3NsL;SHz@JJsb
zbV!N<BDZ<bd>o((#f^_`Ido=C9JcDJ+r)RC%D{zi4|I_}yLp_ia}E5M`!0CanQZgF
zI5WTr?BNJoHAK;lNGk*d3$SL<1>>TN>K=Z%Ia1Jh#>XnCUe|S=RMej~6A{(9*B|Gs
zbfA|jiSe_h<nDfR(N)Zgi&=>mXT0_l^Khda>i~)L|2*Ya)Nzc1Tw725te3LRC19{4
zdK#NnbW-%WQ-${ka=N@=uh_*GhdkE8!~oxg4_J5ITM>m{TL*Lv<c}B1m3u!c?b1IT
zJyI#9jn(*umPZw`J2kf5=8I72#akrsu3~pXxy<OR^*hx8!|J=kD~>h4Qg3Alh5n*J
zSVIGWiDVOoTWe7~@KUxOIf4<h(zmJrVBIyeDE`aR2v`9G;}+t?mpGZnPEFqG9Ubme
z!qqV3-TQ;@eLke1@q>gF31Q;d&B%PMez00#AxmNeC!<JzYrA3o=_L-9*;Vwu6HGnO
zKAmB%o2xXBR&cl($!NiTBJaEC2g&|QU&W*5RsA8IGr5DXHA#QLv&?4r7p0pJqF#0e
zd52^Mae$MKSwq6E7=5rBp|S&Bc`S}%Ld(tl*T7Jn-eltY@n(OECj3)A0#SK#OM;Bm
z1;Lk^34oZ_IW4s8b+8`F4)q&fpl9#kp;+U1Z^nFv|3*L}GSpYE?xy#5OiGqlGvTk>
zwri(H@L;&Q={_KVT<+A{J`GEfmN)zyP96e|e}j!#%n_rrmwPY1_&sR6+p<~7`Gqp@
zL*Y76^#huIMi|zZz<XvDdUCeLLE!WZC8I~j`}ipkc~|dnlFLM%{U22>8@mH{nZ~`Y
z;A~$1Zz-k8dXS4Mm;IH>NMVl+-`f}BqKx1?ot^X)|L?DZRG^$B*?Dn&7nkhrYvEk{
zK#v#JyoE%*wT7pAqw8VTe#~3PpH7YPDWO{|kuu4)Ivtwh`h~>-()a=?Yb}gwpC>Sr
z^HhycDs(qiBTK2tZzDe`B}EM0B?#b45!%V!6(Ay*$5{GB|MKTOknkb)G6l_^>F3Sm
zMegt}Ua$XZPM!f+$w6T#^?6h$q;Ld&-1SxPx@s;jryg8c4fEId-8&fL_j>W$V%B>?
z-dIV`jbIr@M2G^$s;Z+5S@{wV<0@roY9xojYL4_D+qtkg31?v(7)ut4Rkps7zi6s_
zP@^X3Bw}34y-er$ISE^ru^gd;xqZjRfEvLjUA{Nj#%Ls0`wgvHk+@a+tq44loW_d#
zzZh7-qJA%h>|@P9v2JrsQyw`TT}yxa02?_ED~ZMymeTo^tW?+q8|P!HI6EBgmMnt{
zp26?DS2$g!yW)R;2MM+*EWuAC)a^u((|95@lznbYm%2Gw_H52EcCVYqXfk2IN$Odp
zmz%yEoxw1(fSo#b7sta$H>#n`I?12n&i<f#HSt#Xsct4I5`q6a%%KlvlglZ&LiLy%
z`EnSaU-%Ccfffz$(iLdAz)N}66sZ2^)%-GCH^qNv<Vi;Q+l$ZSAJqZ3GkMqfu_0Oz
zV0F_pf(x24xA{d>Qy<_?#Pi+%{v)s7ZES#WBi1;Lbfk2?YE{^16z~ozRpyDy)6!a;
zerTPH&iXL(a&%`B=SB8cUR4FS#dq^ROp`~}Igk17OC>Io0ICd+A3o35lPI0`KqsT5
z6oS|Az-)Mi;UGhxB-7lF#?sCjmrashk8BOBoU)`p+je2ibSLN{LCx3{R@{!PAuv*1
z!u@4{;U3e^t&&{%-5#65ER#=1ZQhErKih_PFZ3tTpSv0ofMevYe8h)y$pbik&q)pM
zV8$K#zwaDkj3$0skQ_CBf1tOPQ_b6G5Rg!|;Q{GW<>fqkjMp81A$#Xb^G|}pGw;z_
zTjZa{Px9*lnvh`}Xm6soGaS)TVxhf19i8=F93-rDKq)hq4PVd)avk+^b>^~J#-zM@
zD20Y(wMxDy^Q3+s`sdB-K{W*auUM)<TWe#8`epI!&1WPHyBzCNO*(DB(Tj-LjKX^l
zkHEf>IQSZfJprfS1joy1Wd|}6z^#idQv4>&ZPk$BZyGtkV{+@xnY5p(vy0%fH&riv
zfoZWp9yLBAKVUQ6y|}QjwG5h;BUGcr6^7&bmyfC_<Mr-dzk=w8A|A#50zfINNRhIH
zIBi!004ha}W(FX+UnxLNP2}9PcUe=IM9w%0kE*4lPKB$w^vJ-Jp+^WtTsH?#C~_JJ
z6n?WB{UMjak71@m$!!5596{kp=W8A?#VU|3C0>i9MBdxNcm>NC1_~pW6g)4H)pn0S
zxV_oC{dN}lY%f&P`u8i{r0zB{58WY*z%5)dnS8Du-ZCF{&j)E_v63=gy)HPH!mzCc
zcH`~a8USd&z;$wfG2ex=0Fliz2Yhb;64JV_E%EpY3o-8hxnc-xMkNJKf}UiteM=^P
zs&e9f-SKQo+4+a?KiC@op+mrLs6c@;0^xIW58duNw0fVw*sKSfU{|JAL;zqkS2IbK
zk>b_ukjunuKI2Zz$Zhz-gwlpYzzii>Zrprc493B?!c%Sy4Go`cZw%U!3i*BLNf1(4
z0l3fT`1uy#01W;uTFM{kG%6=G^52DMXyS-MeDwnm2kpr+4ka{wS+aJ1KjYt?Vn7@`
z@2xZ2R%&Tm%q)WdD#+nG&tbIob;S41K1N*ky-XhAw-|hpR&B>E6yLjfHV=2y8ot~d
zl1lyTGdm|&MNjoyvx76;P^2#TNTa`hJIXWf5KkN5@(VAg<1>;x1nOxW&>NkMshq9O
zJyLF$oyzlNK+#thWUArqkR|1`py^({(y;}A+w_;QL7Pi3Kz)@TKcu4z0R^{VSvO`u
zl1apf=noI_tNYzLo?DkBJ8#SLcBHD@P}By*?M%Y|=VfZb63GI8X0U!xvga#OVE%))
z?QiAsx0-ot0`qAnQ>rNSBU_=g_g*H$TC-~odoMP3K2nbL7gg)F|4bK4Rxclf{RzT5
zEmCR*Lj(e`{$VFX+**%2zPUF91RCl+8r7elKSOG`*^lB~DQK+tmPDWg=SF~Q<?6|l
zzOBC5o~fDH*xK5f3nho0t!*o3ti#6Mu#?+%#j$7ztz)`!R<)^P!deesIab9WpY0f1
zevi%0Yu?x*{<C98n`RsP8!*`$!iAt19%jAGZ`8%T&+JAo8hkPGUP~gQ?*wq2f%J8e
zUBZBarKP3T1o;?gfHZ>R)%t3lKAu?)6OAdc+tt>}-8SFwGrWc38V$&5354Puh*1Zb
z{b`*3+p`BW4n*)?D7nlAJ$9xRPFumwdxG#MRnJ&6cI)#9i!S{EaG=VCHTki2K{2mI
zqB1O&LPHd#%UnNZ92&b(jr9)fzO$1@UOStsb>`T4wR$bP6mQ(cCY<XW`r7HnHcDa!
z<W#c*kW&colLd+^s9L-RT}^^{BIzn&niA>dI?U{~2$WG_q%x)*=nnlDG+r&eY<}0v
zB)L3THm~=y`;sZn2s3w))<(=*=w|;`C4YPxMyN#AY3`7^-CXk(pTXOBVm8nKM|5s`
zzl7<fc)%ai!+yUc@F`P%I%{F9jR#q`@ZH{!tjbm$9<pf9H;uQ77-m(8p0-`(qbi47
z{ivra;iBOdB8G1epGNa$7t+CwYT+$fGL1g?eMQ(`*i}4OUXtXBM&2%2j&bJLD(J2q
zi0|4D9*8|*{47HYmW}fNwTS~H%dOto*6>5d`4ZbR-JW)PLjmH7RPp)orU}%9P3@m0
z|37r{g*_E^g)s;7Ee$fMp&{!FSKps(s?t-hRu(Iah{r9o>`Rp9SV=VS^;n6UsFO6*
zSKv0YCg1)NF;{3gxM=c8{woWjw4l4Dgnd{L_)9Wlz!pi*ZMf$I^Ttu~8?h;_r>wO1
zh;LCvXi8XJGmH|vnPEW|u@I;!v0aj<@XMC}{oLT|AwY?;<nt3Rdjv*`5>7_v)VAKg
zUBOb_TE+j;Ar)7PPW=*Un2Rk1DUf87NZ78&nj-5lGt02EPY@%Hxk0@O+Wgj-`a`)^
z4$l>+_9f|p-&u+!OT|t?WvshRV`2DPJ55A{YC;-0?M*dJm{Si_*ft%^g&(Smqr&FV
z1{1^UC-0qoM+PIt*N;v1RzY8STWEkOsC9;@*-Z>ocLtG*&HA!Ue&@C8iUNTD_%$=b
z2sS_$4T?Jwne%R^c-0325W=0;0NL6Miq8hwGBPrxa42>5J>z*$d1K}kJHWr$?xmYb
z*_~<>ak4$u8P7NIypPhVQS7j&lTX2$OfmQ^#J5k_gT?RJHrLGz17d}}%XGUt<T;F6
zNWT@sA6Iz6bcs&`%kJy4ZlUGc2)T4c?}yx%|MTRYlT0{YAI%J7<C;O#M?-c<@7h_!
z8PiW}u`N?!bC@lnO4_RZV(8Wvfi}4Kj^1rZ<cTe&iN#n_($=y}$k{2QCnODxSs^DG
zRQ>aeM;PG+$k=sI&<NO^X*!M_ip6I)kAnj#;p_o!ppa`3e?R1?57=b51=z(yc5(e)
zi`)i$4#J&6O>{9)Q8Hd$-eIwq^lk0!A4ZkWVs5eCp(!$iC3H52(&&4g#f%g+kZSN8
zrwDd2MQKVrxXL2Qk2=VHvtwI0C)q)DgywG-?@#0Gi)PK{CR}A94_hrV(@R)+wf%Gn
zQ_QsL^Rr2}%yq27uf|PJ{?G}3IQ8xjE+k9Kz7t%1G>KjOOi165Y+TPWlhRs5iLNsj
zVeix9qYc6V%avK_k-Q$9ZVz)07N!{LfSxyn=)+lO!Tx75bP6$531?a{qyd%)9k>}U
zKFdgQ2+X&K<7nE}-w<jBl{|#dC7ew2ZBRsNGjrW0JU61R+zGl1y7kW%*25=k$0Vu_
z=Lz@Y1s6y$24&^sMC9e=6OPdrkUUtvKFpO&LmA(*cH)@s?w(@0cUCTHSXG_YBZG;X
zVF1Q?C1A|Geud6OOs0yzA-D(-D0ki$Qy*WJ;SyGc(Qe!IQWej)F}OARrTT^*%9`w{
z%`vVNS#Bbpb+9I%UNZRn$4O1!7Vg3xF<7v+pzeg4CvOB?$A8SlUuKP^X3<a?D;trW
zRxn#j*s428ipwWO6Q$XaJCirJRsFgu-&LZJoN}X5!u8A>3+DMW|FJG&1e0QdFuZ*o
z-d*FG4O-}Wp11oE!D<TumP-Q;13PylWHk=BK`Ph`D78-8#+go!_rn0I!gysf?^L;+
z89t-V+jED++6aKQet%7SJeatP+W;3`>|C?InHc~Ks0W#~>b~{gTm*_Q=CmR~NAES}
z?xcl4st>_Qn4lDoVvZL$IaQNoaKa|bmR5O}iH7$r_N@Zr(p58u-bb7}$Q2TW20bbR
zmNWG?N2-!Gd83#MvDE*==Kj;ou-xujeaG~gg1xT`QNIEg+J5pV`A8h+s@z!6<U4Yf
zd8QF*rOJ$=o<2-U#lAwJvWo9Elrr}ddM1YHxg3N_xmXjpc0?=QL1^%x>zWe4JDdYu
zO}8=1*@Le^!9<2xt`HuVqh_F<5<#=UnQqfvE1wE5T>ESxuw6WcO9>!@iO(qluNj>X
zmg&Jk0&pT<HxWShQF=&F%gKwvWFkL3o^Cm}+GfQcQ?1wSH|3IkVzK4xo3(&Bi2d2%
zL#jj-Bqv!y-BuDzL;JT?0a*qFej&Zq9ZIxBq1fT|<&vz=@>$&F&*Yu^4(?gv*ht>J
zq@NdV0D=^1Qh8=DJ9mhp?q9X*2#{3MK8|8^=z79K*LS6~i%0Q3mK1&Pa$5KCbxOqq
zC;RC_%)GHG7%tWFhC^EyF^9$MAcen@8)pn@cLd@xvJpJw1T(kIhf{I@;mhH2diK*S
zH_%`EE-P$aEZ-n0j6u7x8Nfhw&GrGdz4?U@(cDxMXiK1mD`{STYcEZSM6C>eh&6)J
z+Ov6=5s@c$=%-HYK~hGu9;8KLG8;XJhVkW?sa3eO5wg`^qV8Ryd?4@90kV!_h*y>5
zm^cboMunP&w)9j=H_zNuSN!-TJnQohBlzQ=&{KrNHMrJcW`}E>+<Ou%5#niG&zxaw
zS=-HY01B_aA^)b(M8?xLI{L1gx}U0}QOtHhPm@y?Rj`A<Ec4M5dXnx@Byb21xMKlv
zhZk;r4;mwZy+tzM;%EUu-Xri}D%xYX$CPNxoB<*a^s;`_xV?1!$1Hi+BBEqe0BP8G
z?~c<_dwSdKYayDa*NpVqKi(HQ*(;WQ{v2CPdze}m31u*GHEFK?vP}euKsIhVoE17)
z9?eF5jNJXAjcd7~(V9q6QN7XV%w_uzhwAqeqazJ?i59)H7`=|se&7LrB~0e_Ue>EO
zkFwV2`o0Z1^cQPn1&z_uhpefHw?sTnG52IaBI=uMFnc0mE2-SfZ8r5Su+KGD!qyn0
z8solQ^LrGsa7erd3m}Bp0uAiV!1$aZ0Q5{?9VVwZ=yKAc-?rzn?B+vmLB8iatw9-q
z@4?dZITr&V5I)$3qKkI<A<>gJXO%)`&+W(GW=iLEpy6kP14;>KL&Svbgq$rA&v=Y~
zhCoJi8Umt`^lq!A`tRNR{v<+W{=H{ycG+o*Jey?a-it`XlcD#9vt?GII(?2(GU;1S
zE|DH!#K=k^u5Xx^^ubXmJ2V1)5u#tQjUS%t68dpAWiq~lhQ9C_O+&?z^{m{NI*h?Z
zRCNpes=F@ow!sTW<NUZ=`>DDTSxsmt1%kl&BxuV^EPHao`Bu&T>+3CmMSpFxoFM2M
z1YXH=Mka|T49(UnnZ>OvL#jBlA<egMWO%o~FTC+qP;+aCi^D*xkL{7OgKg5uTAuVV
z_SY&QoE-1@cZut)VB7nnmU_Q8`}>m!b+=jA_0q!@gk%4iTxN^yYyL!8hs!e&ineud
z?C+mu`sY6b{(`mB%D5!^)39rMFkkg@h-I_E+e2igH^6+^I4Y)$koQcCCo^yfOg<VN
z=$WW^IL=p&ro#S^AGn_{dike}HMj~N;N7cKt2%tu<s6*M-y2YZ+K)9@aX25z2t>GK
zt@_L_tNP3_oYh46v;j`KF#<X`6VuYtYK;OPktpJ%-dQ2V4zGFpBHVJLjfjk!bjLOs
zB_xW)G4fV6vst3vyIF4KAxg~zG;u_u#jBU@qQKo<KBIJg(wSrNvn>NrW%6i{Q#MT)
z9&zSdgFnZx|Kk8QZvdL)4iyV<%}1D`#p&JG@5bEc+;4+gYC@fryPxby%WH}BY$;PI
zf6^ePV)Zv_-yVxGHA4`VAbYOGE`X7JGNm@RRlhc88$dp)Ch!EhPmeX<_KA{Nt#Cd?
z%io9BKJvr*V^mTKL}_7?o?3!T`Fz6xYARaV>(B3UI~F*DW~D1_@h(quD3m#h_~Q(&
zWX|s&j1eX2zwE2VIaxRqEj0)#R`y-z=-r2=upk=W^)5ODewT^5UFW5S02Og3>%G2Y
zOQ2PDr_q+=YYCS9sV)PDZN}7$Y5z<KrOnG_J{L_V{CIAMc1J#v>TZVryckXQ^$u)|
z!N(3rhDNkX)q^<LQ9XZkw3->|9?wt5>P~K|I^o;ArP}Db-sf**IF}3xK_YjlzdU;a
zEw($iE;p3WlkYp7=?F{H8GRL`hpicgQV<1+_hczw&3msFO~J;-Hl3*75kY=|mdk_2
zVrO{E`>kO^l{H5bkhUVVBdhp6O}=`bK2So_EpSAvR?R|pBL`4p`c@q7jqOH$*}jF-
zJ%!+Tel1FivUTZFp~m*Dshan%AL5OoC*ovtX-ay2tdW~2m>i?_fH#X_w=?`Q6$&zZ
zdN52i@!fDfo7(Uy&q`b<8nU!YqpwQYglz1lTvX4rc#kMX-$x-CSaMNgrC#4Qh=hZ2
zI@=5!8&HN69~VpOsmoC)Q&AGjFnik4SKOD9tbANyb=)*DxAlEz%bq#Jg(Tn>g7@`7
zK)lj?yirXL{<w8polk7IQUcl{!|tI6O2Ro5H|YiiNStBbGFEdwRIpDajYTn5Zm-AP
zT$F0sdMbpFS8Mjk0<REyu+(Njmgz_pgB^!HFkKdOfO(st46$d=V5@mTdg4x#xdTCj
zl%dT{X7Wu1e~xzk!zjjOqCj`U)m;j|iI=H4G^RqB%oV+6?Vx34!YUTMoL|m&JcZsV
z*z+?H#3^ZeFip|34CoitQ)a}6HbqC$xkUxuW0OB^-`Q&uo69_uvlvTRe;C16?to-q
zI}ov6`K7$2S4|<L^}WAbW8uW=by(<4VV#_L&=(#E+HXeO(&pyozGbQAgVro~<2Ib8
zqgs-g3DGG21qiF3qTe<9QWzEyV8Y(lXhU4CEOFXx%)HU_$uI1!J4S89a|IF`b#NR{
zhJu*>=BfYVrlEo`+Yy7U6wa;L?=pGXPu?}%$_TenDQ&MxSIsy$<>pLg&HYWR`=_p?
zqw7vhPvv1^=Z~89!AV@=u91yJrC;doO$i8?F~Sx|@3@t{b4@>r#P`v5KdT$g9yhz#
zdejilBjlr#O2g8_Z_*<N+9&DGLYz>At_{BhpjYpN)17AQuN$EZ)h99K1mDIJmd()v
zS)~sEOAUgKP?yf6q@?&Z(A+M9546S9O8}9#95qNej1^)Eeyb^=t{ww@{1z2fL=yKY
zLd>)iePy5VlyGs5axFk=%D_Iq7CMy>$!OXMKbYhlnHd`omQ)KO4)jZ0AQ&R*T|3Dl
zRW9zXlO9T+>I^SvA2NEH&XGWpbXl~u1PaBa$pGQ|S1KJOvRx-znIz2x>zGg9L@YC{
zX&+b6z5bXuDjTb$v6@TMS>nl*UX>XVWI}iBptT>}H+M+S*-)Rku)jz*$GA}SPOtp%
zIW2fCB|K7+<?NKJR#Dvfs>G$?u-%uQxmwa_WDgV*JSTZicL34D{>16EFm=b&aqx-J
zc=$2sWX{#mADS3n1@vhbRIb&giA)_gGT=>haNOWo@@h5crQB?~cO1(EkDjA=Srj^>
zf(ZAPxUI}ybD8rib?<6B_+ysUIig3BYVzkK6I;IHAy*8J#|!Pd>`6g+U=xD6trGcY
zi7H*grXh!QY09O0t2Yl_4G>JpN1X4j{z)D2_hG~Sf^|-a+v~k$gUn<?61tdl7F=ax
zE%;5wzTL1hPlh^VrV7s7I30g{V?cl01l3NR2GF2RC|PdBqd(15m#1Nqqz=TLV&2e?
z&%Hj{!Tg$;*gn(qI#D?OqxTh?S|Q`ErKN$rexVpVuO~F+pT)&M9)QqTuRQ?XujMui
zcQ{;k(>1(>+IYj4fM*`6Vp`g8egt|uXh{Wq0su$WahL#O-`&~>5ZkE3*)_}>L7IRD
zq+eKI&0caGCo~M65SrWF)7!Xq^B{1ZdbFCaxTou@YF=-fYA`$YAas8*hw!DmkA4mC
z<H@!%?bJM2GWSP&ewRvq$1bqjw-(2_)l8#8=t=k4+;n|krajD`9TXFt%XE`($iZX1
zG`C5Ti!j1rkY{yny@{+ZK8K&l!=wpR)4UK~neZw;**tVWE;NIt_b-XeRLG1AE#r22
zdxx#E5z9)`<SGfjEf%l!H{t37#n<D!nX1$@5xmYfL8@&s*^D<Q&eM9C;H)EL5I4NL
zyIxvp%fQ#FmFq#bAv~-p_zD_Bua}=jvYFk+=w!bA_z*ObQxQBBk=l2al9QE%C!451
zPtx{1QYI!Dz%@q(+S|h1M&S8i;PUPAX7dJzg;9E5!$_+hb_Mw1lr}j9wK=ynz$H~Q
zHmxxo3HtHE05f2}#@(GmhNiou=YUb?s=7+0V<8u4%*g}H06mYNgLrXvcD5PZBncqV
zI!73sLq*6G(R@|fGi`%~E8O+v`xH9p1!+clV-{AJUI{DJTp(O=vd(ewQoikfp&Y-Y
zZWGi-^p}Q4UBVXR4)H)@Y@hO@FJq7bu?Ws~;KI1VeXL2@&;q-7pF_lvU!&;$G`EoS
zTZ?(Y@-dt8z%=@};zleey9lO-0mq@pWDgNhpP5TW$Aj6Znr%Rz;GW7T^<kD<38e8Z
z-dXeto7qnDbt|h+=T(!<nJbZe_lZ4i->ijD?N{{k#4mFdvV^ufB}&IsB&doOvADT^
z#$O!PCB#L!o#Fi<9-jRItYg<A=N1;)0AVmbwtLT#!u#95xB!-!3F=dM4Tk3dYts#k
z5%xaN0z3;cfYveBxQNYbuEd9!X@$f4pGH6#q;6Co!eIu^ov6FB*dvjZeLoWLCa{$f
z(~txhBY*)<EhW5jt*8rg!j6ImnSGI{>XxgRN@eWUHO#~tJztB{Ogj`2fJ?r7SH{6l
zQ-WGUt@BsK{13|mA-RvBEd`}?tD@1cwA3aP`F?mjz_?a%bEM3hw}fK$iv{-Is^a%M
zxuOE$9qL_XTAvoLzK`GQ_PKP$Royt&A?c=tToT|IY9zIV3VJfpRBTVVsnF<>n=CL@
zZYs>S9`G)B*%<Ji*o6?KiR|ddY!VcdPnqTEG|C_a1>%qF*A~$cJDg<WZf%{*39~e`
z)-BTjQh9_9{0xU_7_ts%Qa{5vS<Ua#sXpBc9vxLaX9WGNY!hSvyuV8i_$(t=0bvKA
z8eP?xZzp@An9wu<m?v~VpOI|?Q3_Elaa(KaGkyVal!DtZ+Td{Rz16$;b^?CZi+5c^
z2)L^w48^jZBs{xaIbi*kRehtUr%Vth=e{CNrtQioIKh=N^@2;G(IQCK)*|lr#nJ{$
zo|2Z>$OKr893%)O`_TuQ@nX5KrI1AC!I8#)_%c62eV8CpL7Dgv@lx`rk9@QU9!4ko
zHa9v*eRz#+<<oO%gDl$H<YByckZ6=2Yw0QxL6|R8_La{>Ws&=!VtKx+yVW}r@=4yP
zXy_XURKYC6dad&C#v7xZ*#hOR6H1!7SSi`o<XtNHvd<$~#GIz6Fhx%mIXO97o6xnf
zVr1rOv9L#Zfc(3q`TN;gVp*B;-h`b7JUfCgXvGT%Hba3DR<)h5Q8@nwl(GPvv+vwW
zpMtnH0tx+g^9`Z9UsoA3gfc%fS$uoOv6@q-16mycUt;#PF_ZC<7a(hwa;;6t9g>G?
zy%PSY$*(O`8+>Qy!~^vH#ZqWtQ6xQe$dCpKy$C_q{-Po+De5qAw*|^}jm2`3cU5VW
zFH3L-%p$;J{4!*8o%38JhQflf)}U~Oke0T5m3jY829}shtCz~M5SsR#!DS{2c3|RW
zF#3iE^tCcd#^Wh(FL)}*D{jpgYMARKWgoR~DFbVn`5`PQ2DO1(XaBmxYKNb1*4Bgq
zi{(tBG!<hE9ZQ2#_xhQ3-fW{1(hCLiUd}_eBU4}Jk{r>h6EoioC3zZROIEr8ZuUnA
z7I+UIK7{w$;8Ybn;}--@leA17M*la=4$PSc0j7COr?xEtfyg?oi*=(dRml3Hdh$9U
zBT-(98{%#ZY}c=+Nc-kXR7@pAp9&t|k%U{Xg0f5_PGG%O9D9^6lP5B~ZcaD9bE~W<
z7l*7xXWBlR^Uoj3vuHz>zngx))%I!lGVzVWMO|EwoNx^P?vs7wBOj9sqO8_{nAd?*
z1{Ju`j`b1~6}1X3daoUr`LI$DI{b~eWMjP*`^sTn#kaH8$EUZYFy3)lnZeVMs)lI^
z;|@`*Mt*$db$hKJs(EBcqRSf3xRb924mm5(jAF56><)*R6qyty>O_(!anhzf+;)`<
zccNeN{n#&4$5BX;E^E}d?9`snq&?y?jJf!c_M^N^e7tnj6YO%h;~ex1;<k`~J_#je
z!A{s-S~3VSeFxFiiJ(7bkjNYLJ`@eM9npFN*{yuibt}whic0N?wPf(4i9X$w$)!B2
zYU`dH8`G+h7e2xA2jyD@H-Wm$!&b7Y5Nfig-kbjfSC`-JN1YsZZ=ikKZE=f+-oGRn
z1EImW<DwXPG6zFw*}xt{?h~v29GE&UDRtB|Fdy?I5^G@h>OriV;h8qeerXOEl=bh2
zPlAiMh1RR1pUDG?EzW!dm2LL6lE$%|Ngn=In7DJvM?R<zfpaQAA)GLv&m32C%MIr6
zo^#}?bu{<NLRcR?ZD7MaLDrJN5uQo^4i>0FI-i770mRFXIU)RK_y|ovAV?0kRrfwU
zR!Btb*d-=1;3QDtRfG$XwsLAAD47_lNp)VkL@SYaBRXO=2SlD{L#|;m8HC(P&Wz+@
zG<llpe5SrVG5=!2kY)SzUfyb(a_K(3E<-`6#ASei2RFLQ{l^CMy>$BJ;uu2o3lMxK
zm7-c`;zHP2Gxz0VsnkD(=r4OPue>XHAs&C+61_dz2A;sk8<&{|>e88a(&eL~@nb#1
zn-NysrpX*@s6(?Ysd#hDpGW%?ofI#iSMCsMl=_4xYF1T3O?|^eIZDN*(eOBYbDaBF
z^yw8377_aw&_6Nszw$GXWfW-YE{*<t+s7GkjJ-P)Qkl)3Y#}XC$JAQGVl$DFF)@^s
z677Lo(qfXMM(M^=U!u7PhrZ!na*>v>lRbl8C->qb@lVwV5@kej24SE+BqYllo0r}N
zwbv`X1~p6?eg&1MXByBK*Cj}v#Xgwv;o?}GH4b->OQm%#qUIo&GZc7mZOt%eak+Xy
zY+oI5T6L%W0&SW2xu)?Rx8-(2?i9-78yh{F6^H~mpy8?<jDKK_Dc98yc?ij&K&3~M
zxGW$L=7z~UY7P_`niCb&Gxj8#jL;xwdPG%t_3@@Ofs-?5>G~&x^L65Lqf)N3V#{+Y
zz5fU*cyM2Ex|f-hb(6alERxN0*o6Cr5Dm&U+NYg_UZi)OoXDs(;A3<`-36;%se#<i
zd}#M~I;R$MRgJC~qIuzIyA9-g9Jkc++biRi_IGRfQ#w_X1s85e+!|UxD437)hrkn6
z7Kg_=@skN->$o%R3AiBjmU(yVGI%oy_NSVd1g1H!CbUGW;^AvA7AH8eK~3SELIfi#
zgXGmW`pur`oWhc(6Zc$-O*WfFNQ*6yyYHBc{F)#BdymozadFxaJNN<jCCx!L0lu-k
z(E*pvpfZH7(Mn3*<VF$)U+zV1%Ec*dM+;HYeCEN4d0-?!8nDP$UU{6-#-T75(K<I0
ze}$9chh!+1yEo}XPuYKxCXRezs(|A&bVzo@{X~6Dk2jSb+|zD0gZ^cJ+<<2Y6HZBO
z68?D2S;sn!{b{{Kq2takyerO?ZTgRUu8{awO!7JIcR_RftEK3e>|)cPIZ;Tzr1^I;
z0HSh3nC)Kp&f@|7f>9qfc5UjOj}9)AnUtgp(XMfxkKEK1z$G>-xFipQ5cvG+fc{-d
zVjP-oX;0fi<pJ=znqN{D2Cv?IYU^FOeD~s#0Hr>GtNMX)NPB7aQ&&aYV~Mln;?$(c
zgq32?1(bd~5=9w}S3gJt|GggOe26?}AXV{f8-u=n6Chj~@3-1(w+z=^zm;_L&V%sY
zc1lmC7(;sGyAin7`4~prmdl?rrEy$LDHmOpE;%N$NQmok!L?d^vyY9ku`ODf6;_oy
zg7ik!*|<Oep~=n8nLTLCZQRmwBD?ra-c7UE&?UZdN122=$<|PQDX9LGjtH;6?vIB;
zqJ*?#=68OaA$Yd!9CC78DoQmn=v+NkxZ`A<r+#&GU<;)1mCp#e5&rw=|AMhnoA)n`
zY&BKSi<GYppQV*8W*U}SXn7k?-8_3UK64&LGn3tnLB%^0Bz-u<6moy)tH6K1CXNL~
z=i|3p?&JQ8^xYI-wE$2B>t)eh`~`yL4wW4*ng*K6+VBS$od?w2#wQG4ILp~N=Wf7m
zV+$wql}DGSw7F{M>G+>A>ehc2FJBIkuh;sqxws_69XlQjQMYXoU*}y5U*{9SLgUGR
zk5jurOWb8iCmg1hXxJGdohwh%f4_#`pGM!-bO(kro_ZrJy}V{y&zMP*7Gl)(2p;6C
z1*xU?LklC4*MmOW-94?NFZP~HdiSH>>>s8Cih-a^{zb)vg8wLdTKp>sP8`Gih%bCg
zLEvuA$!X~eLEUqG+6ObMzsoZ%{UjZP@aee1^WeS`k|wYA>Mpo0^|~xrFXhR=`6wtr
zup`FzhPkB!ucbvS->kX+k<e<ZQg||&V=^j=o=&3oHKe}wrk167G2R&oDYo;T^W!Sz
zHGzb$7ea}FxMOC$H??%(RBRKx%WYRM@jXl{Ol;&D{5Un0mQI(NhFu<{DG*LRm-Il{
z!n{e|-+0V;q&6Ib;$fw!uaK+afF&++@F90uEs`c*<ju#`qNr}S&1Ax&f-lPl2(w-}
zO;{=rLeq)$%+oSd)_qN0tf5Yg9&6&f!@qF`MxL(_2_T2RVv-<Ic(745dT{n-1bOAp
z@b%}L-n`Y!eDBlF;tYBxbPoTxO*R%k^ga=1lkIkpeNleW{v5-;2FQY|SP1en3D}x9
zlBS0ZwuZ*ySP5-~vQ8hED7f>Lo9vHT&0ict3tf$qwWUUE(fAWk&puqXTL1Txg)c#b
zG$7`_<wdnl#|r(+ofJIc^Z=)GX$qDB7-wsXL@+X*Z2+vUqxMTw)oFzw#hI<sN27d#
z|GG)oyfDn@zQk_S&X4byT;AC8GrL?%j(mslb%L49HB)4gdnvj(8l<f){flq<qW0X%
zPCOFdQfE;f97ni5YGV3w?*~{P6kRKXgccg*JAevJKJ?{)_e1`pSaR{tm*{t$tvO5Q
zHx7d$?~P3dWaAAq+bn!&PTsY#yH^Pe9ln!udCu6k4Ev=`&e`q#<s|DUJcmnoYP7I{
zy{kMj)J%TFp71i_tUOT+Ij!qJ*T@Kzq8eZ6jo0Gi#2ik_#*UP=Y-KlrnR50>cM8O$
ze)xDK97y*5lJJ-H3}vtz{r98#<C8EW%(eUiYWDF8`B#NdR9m!2Hv3ic4uyL|%_2Bf
zI$Jvxn&~X1t9^E9b9p#+%<zc+zdQCtwO$OOICUP-uaJXQ@I~-kWqd|+$moD1TBKxy
zt1oV@*Ma7*<|~-q)t@ct19}ITAJxQ7G@97|XugDn-kRnvBj4q)eF$8oPuBLA<p%zO
z-M-mu7E7TJ*~)@UgzNAc-!9lFjm}|dQ<P&p%DAQ3rwXm}RqmMXNEpky8q?qH6~C8@
zh@unj&)~SbEf?E^-GLoFw9JGr-OX^+MUH?OgdnxE9TDr=UE?D(NSj><JYHoNJ*7<J
zDNyLRRZ<`I^d`z<C>!b{1c{%g`}VJm9=_*_A8rBd9Q2K`8|A-h6}T(}Xo=xc8=W76
zNEg(iDVLToG^A#O?9@|(s(t81X76*{{*MHar9x7&3~V(UQ9Ey8vI0lVJ=xAJt*eB7
z#4pnZxZvzFhz72Tax{hG)%6k$Tp+U3>^2=28TlC<SlpQAe)7m?H4z^XWsvzQb@?T7
z1xh$E2nm=iK`X?aVSdpnkJM&1-~Om1X810o=F<OXE5g!OGI{!dZ~4s!C(0tNYt}CA
z$j53Vt?)=ZP-wh1Tni^&=`<Q7TV5|S`QiAZkST-b?tOc~{>S@xMG2lefa8VOgZuSn
zTX=ifvq5=A+nxVdEt&TPN%?9Z{$|nU==4Ae%^bjcZH^Kh|EE*|Bg0-CS4;1(2ibF)
z?$z-w*;s>8CucI4xF$}K$>sqK!2Jk%G7Hdf#SiFnjlUtoey@*02K&p?HhB6$H{y>y
zdZ2tKCPI8IAaWl#MO2Tj`u;mUy2y^i8T?riiBvDeAMEDLi7D%19VzMjko{jM8%&-w
zE|zMCSn~@Lzc^9^-;y-@u!?u`dG~b?c7T(+znkxWPr=B&kuajCiKVS_Kwq@zg-fHw
zw;QlQ67Vzz%|oE1aM6IMz@G-4`tG1bC-4HAY~n*MJ-Y)WKTb*&3aPH(_KbQg=kR05
z>v21^@M^mD-}3+S6J!g;dhP2)ya#YTy-E7_K{8#zV1>e8%>)P?J6}X0DB{f6Y7xT*
zY2j;e-i2~rtc4o=J8^WYLa|LOVQs{q%PExmKhg_UOo%&dBAYzUoDM156_@%9aY!Yn
zTq~3-GW+@{6ZQ3i=D*i}fC#{BItyn6aSU=GHtW#baIRKkBxB><Wcw;Sh%Tcxm&ucW
ztG_zk4zDUMGqo8eS#!o3k*(MysG-!00YTdg(&7E_t@R4-`-=yTdBq2)Ab}H0?sQo!
z0wMW?0C!n@^S5+o&y-NP*E7ai=e)<-a2&-kf3>vzS|vo~rZ5w!<LWX~*0hiOrzQQE
z{l(x%M^AzkG`9u)+1T^H5Jpik@<lvgsy6in$HoUdkkK?CWMj)A3l$gPXpkCMluq?i
zlVTS{qtW%k1u&!aLahGrmo$WgRAx4waBH@H+v=t*>sXQfY+~S8VtiD2ImPwptN)5X
z5)&bk>hOr+5kbVJeB7p7EW;H77vC0|=Stvi!Fg3^Og{Dn!7a_#ZPX)>%Vxd>^aMXn
z=np9fxDHGiWu06cMlH}2h%zsKH+21W;sK^u{3pud+ZUF7(A&06oCVbtW&U@|EUX?K
zY7=f+Spf^L;&=9^5SC$oyUetk-8hdytOQbrw>W{~64s8%rrCNc;F`>X*9AKJUV%g6
zpZ@+cDyDk~R0R4rtHl7IPoHjme{hLt{e{p%-#s+mu=mx?E}+7rexk*;PT4=|iD7!b
zEM+wo-M>{G$qj_k!{noOh=O_JEB&Kz5?}d*mKKTe>Jc1^tlYc&G}C7#qciDTGZ>w>
z!bv(JHVl(0KJ19o78wH;K^tKDGxIeuFMLUu&P||J!?UiLSfFEPSQtIN#Jm8}r@`rd
z+`&&F+t>BV=qC4sqeyDSp15v2R@>$uHuvK0p$tHw=F6CwDJExM)4w*<v0_P)?JgfS
zQZR`Z!vM!v$z%-Zw+~4*$i{w(1z)|5{uo;P+vVti>n&e`F|wxs?fl>a$F)n3tAotE
zZwATo`^G3s-z5Kr{07<ao2$j%n4$mCSR^5cJA$`uck!)A(J{Ig6+WT%ENa_g4(LZ2
zslCu^!J0z8rvEN95~MA~6S5iLvPee`bVF#+LdlPp^6EA?paQJyqNu-WBSP~Rntwf@
zKg9{9p3J=Q!`*Op7W!nD`m)V_3-Yb?bh+GhueE<a3E>h*8P($iIk=OPiJ}7i!ZhIo
zG-N4>BfoF#&|z8^bO0YGs;8ByfyPMp6bn=dqwJsk1)IGIq});sOBvpk4O(<bYn%#5
zYqj%$IuI$MdHREq?BADx!ml~ynLoQ%x<<}T>Jt)a5n{%h9q3&>G3g_uIrQ(1)KdgZ
z-IqW^cBJWKhev=EYI&nzWj%ECyK=D>9!v!B3>rpN+rqj?7eVa}NSqCD9g1yKDU~l;
zPP_LI?fp%ihAIl97=RkJOoIM{lICxH%>#>h4g7OLqD1Y5*%GJqq^6C4uBr6-qrd;h
z&y{s+Avn%>69c%WnmNTp7FR-7`JS+!8*lL)zm*zd|K`SfE8q!cKHhGtyb^(7)bU;%
z{2=5V+@BG6M3!<vK?Q)o6+$WF<PzaH4-|tL!-8=BV{gGjfTAO;<@L>0DU7v&uuR0N
zEVe?8E-bcy)L@jK|6ivHPiZ6oFu+6pBg`!H3tt9$EgZm<!Bu5B1E}+$Yc?#r3@FuD
zD#ih58Z}U!?3*CiPNe~Y)n8c@b)k|VD?qTyY&5E6O}PQ0v8pVJ@Dfpnw4$n>?mKgQ
z>t9Xe%tqBNZp))L3d&Mr{#UIdZWk4f+elVUzos{i3q6WhP_vB(_E?}0WShftuQE|E
z%VgBzYXs=cJW1<fm*jfj4jF?h&asO1sEaYWkS(|oJQ@0&Srnitp;EaO_4lajKm1}y
zIu5(7YLV($@kyfb{usSN3kU$nbjP5#{<(OFC7Si(D$v(g2+9$BQ=cX9MND1=2T`S!
zG)70Ij(aD{;tTVD+&WA61#Z_I@hQfr&EVG?KLgw#F-<_;+cH3++?Nd0ctL7gN6YWl
z=-+ZuAP<|^>-gX_UmTV}dn!zSNKq-au9Q$Lve`O1Yjl_HUse*z0P_(CV7F{~_L*!*
zu&D<#es3uu&;{r-4Chhm)kRs~KuTvOH@*Ah1m~xa1J-;1ynIE|;P^u-$^3yHm@U0+
zl1mPZ-Z#00sDHn3xE@pvfh?>qZ7l*Jcgmh6Hs4v639$r7{>;|U#(!<3UZ^IEC!rR9
zTaJC0FskRf_{No`pzuUC@3&pyLPcRgCLrZkO7$A3DFyr0CKkOj<b0oHra+}UR&eCl
zqCibPBvSu@G6#_n${^W-jR8&O1I0ct4SV5NFZiEty_*_or?ZMl14>Vd)hv3nr=Xx>
zxy&=BGxKCUF>(j*{k4BC@CAxagj>Y=IQ?VxKD)Q6f=py;Pi!UEBl(_Ll)W$mF6bcc
z5E9U#PQWFl5YBhFdamb35Q+g#1%?9d?wNZ-+Ru~o1}8^WifvMW4^D3J^+Lg=C-<gi
zqq9AfcQTYxX>BK=Vm5Z3=lT78HHFX<jHK6p?gNM==rsVD3uTkWv)*Z+REW&y0sW&C
zF+shRgMEVpO1vh#0(h|{^?2(eH8aI??X<3b*``g%>c#vm5-s4f-w4kAS=y$11VY~i
zdcH_guu4{!Lk@@k1PfqCgCucwh{Od($=And>YR03&9?{K!mf*S30Dn<g(+Kpi&4>S
zlBYQ&5PJVFt#JZ}H9G<|aVl<}+>Hpw3h|gtd4Ko0DyO2|Que@~9&W8O7;vG3poTNX
zx^1S><Shi2Kr#Xjk!%e2)6bCx%yeL#2Bd>f9<cll{kE9P4iOv53R{h0#(%=d@3p>!
zF%BJ0-gmz)BIc1Rm2XNDMrlfKR%E5$^sm$Wif+B0JfOGi0Z%0?*s*?!(5ZOIf1rPR
zilx|$;Zu0X<X^rQVz@BmDzzJaAezqxUmyc;Y;Ml9zxc7^w<+NK><#m>g@G(Sqi(!p
zOf#@xAQ%Kd+wzExZf*%(LY$JGoDRv#KAg!>lSppM-Wa%d$hl8(H}>kEvIvR11@Z=t
z`AFr<KJ@G|?9D2^$#L2@$6@zHvP2~cL17JqmWLj|avJIuI#fk3Gl^J+K>y=Gc!P{;
zu)USvCHYTaKE5FCbk^7H@5vYS#QA!eS}R#e{c;f3>;6?}ijzRKUXOWn8a3TMT}C32
z{S<_=oF0?kYl)&PsJk=N)MyG>eyO;BddAQGXXXN`jlhlGxAu;;PP|+1BkCwX`A{jZ
zQN;9l6%upU^vkEF`YouBz<aHKtL1<rxcN%ludy}*=*4-9w>WVv0VjK#r^PBpgBc*!
zc2!-wkqH89{ddmWoJG=ng)AB>qdy8fzug!}JY0JyN6q}pBmeP>0(1}z=4HNo;lpP%
zra$e?kx0Hi!p)y&rA3%y(>5!u^f-mFrL)K(GFb2N{+Jmrd=?O`Qr<H-J9KD+d)B<1
z{zj%;9KaA*0+4~Iq@-WP{-hJ2W;+^mq%MFHoC1*rbbQT=JqZmno6VdX6{G;^VN#@>
z3t;=g6_6off|yibZb#vvU5b3H!P(OUZ{DvD{Yz|uDM{F8FnaB0n^+3JrTSW?82>xi
z0h=rV&#2__$IEhw;A*lVj$X|*FRS8wv-3JYb7D|9uIsQ~hh5<o@GHYLMaG_YfF7x^
zg=*S2_bf^{UUDxnA5ME>&F0i$27_s<MX(yB2iVCqJjbh~*_C?{)*F)()eT8u>ePU#
zzWAJj3EUP$1&@8JnN$}5C8VDRm_6)GZ2W-0wcFqK{|I~Yc&OX<Z~Q7n8!ZyXRw;^5
z82eHwN+HR<M%l)`W}Qk!7?SKe*~d;}pQOmX&)9~_ZZI(nX3WfU&h`B6?|prr`?~Jm
zKfUteRrs8r^Ei(8@m^+ugR3iEI#qRliRN`zqyBcoAjY`LVZU+Y3w>L^SYhcB#MJz0
z$2dVb`_FMPyk1lK;>7rORgoMw8}|RWrotE%YDcc}uSRD7zKtGQoGngE)Rf7~Mo8F8
zO)4#aE`Q+ruo-aJ{0Nr%mtAv&<y<D^{Pxz3ZN=T(CNpT8!gV0uVEolS;kVao_Y;~a
zrF+YU=t?R3_}%yly3yl@&v5nLu{|lWp$$A3Mjd)+l37r22V_g9HI`CqpW<zaRh13q
zEke0E>9J~=97nteaV~Vqck>r{WxLxCqg9r&^=~FM?oW|bc)?A`VRi_=A7-0okg=Zm
zH@0cjk<Y29qIh{}jlX}8FEHZoQZj_pG%)&~xSI{%Y%aq(aXWzJ^=I3=rlhsh4oq)B
z0<+%!m;3#{0Kz2_$O$iB4q8j)-5+zrX}Vc!97T8_Ycy#%w-hEA^$xHwZq+xaWBUND
z;sp~+xJoHVy9ZehgNyYQjpakyig(e$)RhPQKsKA3oqawYm;3dTNsN7W;`FF~*PygQ
z+<eLG+&$eSPwB3}2#dh=gYt79^z?o;${#-P>tcWIJ<hRe2fcnAtAL6b@XCj5LeHFk
zU}^Q`)nP^!yZmJXq6e<el78G8t4>aLsEK<YTS{c}l@Z1QvcR1;U?bz^R!=P;l3fLb
z%C76@UQ`7*0#Gi1dr(Z4IA83=GGezV{MW1E&r9Q7;C^xkTZYMnY45Dt|M6MImsFbo
zD)iu!(L1hBd0598tOJ89g<9WtKc;NR0}`6s=Zb3UZs1YL6c25W6)mm3Nh4!{5wmmB
zjkgYY%EvoE8nIbd((ON90It8m8#bQ#9wRO`JFaEmOfMX9MlAgBVnI&S`=jEu*Wa-6
z2{AEmCTvC?6Goq7*m!=6;OtU6u6%(J`cPD~DB?H=m~V`f(NJEGu$7ATwZR;XzUMM!
z7SyKhGzwA)*oEJP?odJAUJMXIb}<NfLH9W#S0_Gopy@;bzm)-vIbbP#TrTuZdL-Ai
zFQr9Uk0Ex0R0OwFTkvvMpPa<OD??a|z@G=@f4*YhZ?SNAzq9#SDXAegrZcBc59gGx
zWXJ;%TmedHD+4G~^-3Go6c`C;D<BMh{*@TmhUaboYt}I|J?_JYTd&_eo)Q+XUp%iE
z_|P;ZaoRY?(f$L^5+RF`odAa|*i{k>B;<(X#K*xIE_fgd5FZM%bG2&Y1Fc`Tb`$84
z3HNPnhy0`YQZLJJ*kKa;0pobLa8zxJTlO^ZKOWY*O$JO$J2i0qKkk7)E6yRq>n!R9
z*x?<PCA?1%dGln+*@4NScjwe*tNK)8Uy@d-?bsV{S@^u(HPUI6o}g!>j9>d%_@8B(
zd*9a4qj<u~9=QSgjV>|N<U`r_*i<Y?Y!sucR7F2BG}Q2hdcWIB^PRdJ%I9MwIHB13
zrb(jA6s6s(fWN}X)PWeQ?)<W(ld(Qm?aFu4eQDo+9SGSUefFVEF#nR@6bQ2<1^!l+
zp$|$+-P{4zt{;HvQ1!P}DYK>bh`z^!GwNV7f16ZE>N@a>?kX;87FkI3jDjEaEpKb+
z_r7W;Hl59R2qSl1>HLYandd$eg%b&;l+eg+9^0Q^UV8wGSr#PrOuGF8SHPVfEHOnr
zKM_U*LB2%A(U`pDF`#D@{MG+}1ZaYn?b7}7W)~LjfdtiC6R0<Fcox`-49jlJ$`zeH
zd&QRI)JEycytBn}GsS<3QZMs<I7f!W8fI30d0zwCb457@)6pa-4|z+Go&A56=0D%J
ze)7T1p|dJW_Ec<))c<Y-{^w#A%nT7+RYWL;9PF{l`117tY;*q#;NVIEkLNLvo%pe~
z)@ZGn;$mEGwGYeuBJqGUZyvQHsrpwqsRE<o;Qbuf*IU$kt%_tMia<?X|H#h>4qvX8
zo?Hja$I+1Nkf>VH>d5ZEk08gS3{HM|@B~*KS$^{FM8o%T_@)lmm3p377(ng;R_Zv1
zu}RVX=WJXVfJL3rCJ%>~oV7GPuDsbFCW1Z2r!dSmV$bmw2r@cA=m|bw9z=xYf=EQ#
zg8v%i_kaVCw|TU2=cmXMh@9!N95C&V)(Au^{Dj+K`w<9hlfyf8(WgzsUPVdgacPEQ
z5UgZ4W90Ln;jy6Rdw!2iQ-wG42CP(^!@8pj(6YV$PVuSAYfb?m#SOM;=VFtkN3Anc
zeN-qewiP_rtg@brMe&zN=8V`jS_;Y8%u|xiYN0H^EUxHkwy<uP3V@5-pdX|j=N}TD
zgh~DlzV$I3)XU+9ywtx5UjmQM+2Fb;#{Tu8X)gDPbGjU`@hb|#Q!D!cqxU_a?e0Y9
zj5tc)`7+D#y{9KGTI;IaHcow1o^^p@DLNep=)NSYm19zJK|AG-D?!5(ki*;jJajFq
zvC#||zJM&`D~InYSbsef03TbeZ1Ounx&CLciVA)2Xv6ylAPs!wY19Dj(WBoww#r?5
zP}}h>Y!{HhFIuJ+aPysQ^6!CMmfk0pCL$uTaw3+wx-E)NlwsVU`a2uYfzIp54gF{d
z={ZYBt-e?jk`&ZYqCLC*E%^bZzrSa)&XZdFprTEu1+3Q&oU8>ZjM@1VSCXMTATVQR
z8JTRPJk1}+^c;#UA6rruD%&IX1nS!P8Ce|%#_s(%&-y_Lv@`o&e161=Igsx6LkfP$
zoRczj-4s?XB&*uP)_XcRT*~KpTPdKttHh?|w4P6Vl$kyGUcg`^9K0|91q2(Q#z%_#
z{_cg50j+3t55Ct}@Xr_M|M}zH`~3<h%7F;y*m^0Cl191EMG5Z|h0Cm%E3z$+d_#8R
z;l@26=?*pX8GA~4ZY8QE$nUU`LEmdH<=<LGO*11IcU{*so#xw6|3T`xk>N8C19Pjm
zURuUsB#e>M%Qi2Ew-0j#&JmJ9WUoRb((BAUptxW}mu%##QtyIlzhG$rytA4poAJP0
zd^h2ub)M^Vb7k8BSSm3Mn5rvn_N&p<DAev#B|l<QQUoEIN1&{{a{Z;LS*n)agXO&-
zzUi`S)0RO&&tBPBOR~I>(F}eVkU<+$pZ$7E!zSJM<HaY@cWG&2AhM8Q!ms+JO9Fib
zO~DaX7A%9d;>>G$X4XghptJdeh$Zg4jj-M6iwS1d^we!v-Y4p6P)4}{9zz54qUPfa
zi{#7th2P*F=WK_+RCrBY$_C;^iY2)2!g9XR-tX$8OUYoQ$~gxCs(YYzdc1K2;NBTp
zTS?F^SP+MOV+%KHpOewOEp7(l{+G$)&j)kj{&o737H5r@9p%8<=zngszx>T4?YUk>
z8#h;hFc@A(%+(=gI+?-WAgmE9_4v{)jAMh2VTsvACT6zZOyJQ3czi#HmN^&0U>@Jk
z*G_w~o?xk|$}Tsc`fgD`6DS97CBU|lt%kBf`9<ioZ8Vd`;J&dxyAsLmScU;acf*<_
z%kNx`?ub}nop?a=#x|K$^$GZlKCT4XWQv@BI`EIK4!PR<A~Z=;W$lmmY!59#Rs_)_
zq#X<-9SrFMWK<Vj6s{>x+HNOD4_JkqJ(?O0jZ)=+y*yiS=!MKBw@LT*4o2w%(Xw=5
zY>J3w?<NR8=m3><C7)ci1-cM-i%J1YIMo{%|3@aFW=k7v{afSr!49YlWVCWxc*Vr7
zR*BlDfiO%&v+DtuG?TvlBA6X*v1BU*t+~nin}??}O`MU1wK7^Exb|R(U+f*z@xJ#y
zW8Q>fchx(h6ORFx;+tp?c#i-v=YWIsePXL-w0TlqTpC-?-*m@-psVUVh&H|Yx8ogn
zkR=wM1(53q3?}rZRKF|8123UX%7H_eIdWs~2Fp-kg1D&&@b7eOI9n=(ebM;s+ilyo
z+kJfe(cr-ydzXFQ9JBK=1O@H>Nzr{w&wYYA+0emwI+;Aw-9C8ygiAwD#ux=;CC`0k
zI)2P}?0WGZ_Hik?l&B4vz7;><=;Z;l^32NWo?=R%I?VxJi*jGI|CZxKd(f4VB0XJ;
za}G;WFFJEHqT{1g$lsg7;mzgc{iTm`+7uje(;4NDwp9k-#U8es{}`+N9>j8C2=p&R
ziSLJn(@*r~ApvlH`BEmqfiaQUhw0|?xHeupYm{>i@p5lJ#jenx)DQGIOsA`lU>Wlm
zJu-NCsh{Kfn&gXi5S=hgC)!PU5@)N2bl6L7V$S40I|w6h6C|d~$WzVgs->pFhUR{`
zK;o@f`jH_&w}~n)KER1h6JGCSKu}2oj#9mb>qiKl=DmM2)to&cu#aGw{HscW^cRQP
ze^2I&hP>dps?m(&ZIH4u2h`+VVsD*E@6_1VLm^kB2DMoFSW-4?Kwg?~=BZodTgwtd
zC%(DDDW(CRo7Vzn+7ObiQ;p<eC^--a9TJ*I<6USf5Zp0zD=;rASz_@ArfGZ1)Mq-#
zms{TB<0fcwKe<l39Iu9hCT=$5xXL>)<C0S@;)s)zW@F$N$cVcp)veS|xV(2!0urkL
z#g7}gg+`3jx?5hpWOi-28-SWFOff@&R6HR6K8$bN5~lz;(N<Y;)-mPa^1Dnd;PRp2
z^YM$<2+~Y1@czjc?NZ%sKPzl)=eE4xo`$1ozh0?kNIN{tnHF4Yv%knu^5B8F`Kra_
zY@-1eNRy@g6gW!-PpS*d+(WZIj`g(sE!yV@m&_5iP0Gd5%}uF35Gj?u9MJK;OM)J2
zu77T{e(mX>qNtu9*215O7*p2{h6TvBNRLWcOLDtbxjIG1Mun8Q^)j$ZJz4q;av&p@
zEalvv`GVx{-&2k5%}`bS<ykI@=}y9LgGjlBIgtGq`AoG%ra!}&vWm)}HkHMtRiBS;
z-%}EonPw*kWif5k)W4=hM)?d9bg6#j?fd~@20PqV!7eF$oeW1C(9>3;(W<+%>=Zk@
z>h}KC&4u4TLj_y})S`!SG>$V=uXL^mPKkJsBhuz^h(gg*vv$lKsi;1s>)=or#LH^!
z@ZE8DW9}MX2tw^B=wQ1TCOUM@tYB0>L7$q$x%6?G6LRU8jC?zIr}~}@WZ$Ogv&sGV
zBkAhnm&arXk3;SY5I|}O2y?Rlc}*pY1Sa@ym*pN_U2zCx`2?uy<(XMeQ378aCYJck
zXx(@TJt$;k6Pz5yPwis<i3WBK`la>DA`RR+$|KLp=Run;tBRJ`4!vOMzjTqZkf``5
zXPU)i>7FpvqTem-jSeFhQ66*+x0&8RMlm9`j3TA_E@wK!L`!2Dc$eaB;jYURDmZbY
ziv$QneSqM;;M)Oa@KJRBD2wYaRYs#;4Kgz<?oqyh4n{Ee_y}Talt0?w8Ic<QcyCB*
zP>{x5xR8yu8PSRl-1({DvHp#-3j{;xqU7Gxt;2icfI_CNJal%d8u_(QMNbq+7e)I|
zgZ*C|#{k;y;<oXS!e8pv|5wY-)*aZ?<7&Q|dAy%xk!f=u`H!IID%S=6=0lSpfvYVL
zDHUlq$&i*Ae+zI^6+BJS9iQq@oiUyFcQLry@+K_WnYp5#<Z7IE{?!PG#3<j+Y4_3*
zLY}~UN>lXx#E^Uv3J<2garNb=K%JuxA9+>N7f)W<1#m=p_aFYUYPd%H<ozGt6~Huv
z?C$O^?_HR=9nnZxU!7sNAQ?@h0h!spcG(b19$fFM#Qmk{05$j!Pp|YMT%vi$I;Z{4
ztoM52Dan0$P5EoFe5%2LU!7$_5=s~nTt-0Q6{+%ZNv8JJ@lfu_6_|l}$o+zn0q1Fd
ze9v>&06qPz0dHJ0r<vQ0&r5WWAjb+*+*YwplVu`NosDW7^+RS-!lCio3V!HzExEE5
z1rW1Q37SboLh!Mq!y~Vp`H)m0BlaJzFQKea+)fP-y1IY3!8T?)4DQ~2OAqAf-VsL~
z4mU#6sRTilyyo`AaJ6`adP*tEbH`^*vEZ<mi4c-w{n{q47T0%4j-Wv7ZQsQv8B!@V
zbiH>HPNy39`1pKnzjk9^*RgIC!^_7qBB=d0VY<PH(gxI!PD!6_zqTLpZhBU-=2|7x
z{m2)A-=&nkL`5vI6Uk*D@j0yQ_2KS=UdDFlDGfrDYyFZ?0Gdp7DS9x{bowJbh}eTY
ziE0Vz*`kcRqFvD&`|6wendaNdDtllBp3*G4{G@mKmFa$qA!>3sbPC%8*ZN)1g0GKi
z)~OX{KT>`FPH9Vhe!72A02B|@fi?HT+8VmyL5)&JImN3t13F&qUZGGGt?m)nai20X
zGh5uNwHSsnu15*Gu#`#dta-x{lR3xOcFA*7<tT50;Y3+EiF>zKrz|=k&23(*yv<kq
z#c`qPCrNHGBjCyi+cu`x+C;nBH8oV+E2jAe1aacW`V@^(suNT7v%srsd#0+S2_E;<
zn-Irer4uBPj-{r4+kf<*&-a%Vrt7Z{e+xC9XR<*4|GW=(z~V;LD8~jA4x^mb<3S<E
znL>{<t56nmzOxMAKdZ5pjk5fHeag!!P;Sk?bnR@2OEAcTg4UI=J*;6tF{j3+=eXYZ
z?4hg?y|prFC+3+z724+0ZkVaj1}2AMo_2op+Xb*@`~gU}KN6S;^bR%;JyPsW+Y6$|
zC|;vuUvz3Y=agyiA}gY^+kZnb+dot*u97V!55fXly_<S`(;|F7t)U?zN86?N@w{IB
zH=11^6V_#&Cb>37Au<D35yX+vvrA;>6dC7ad2QU?lx4kh(Ry&oGbLG$@;&qX%IrbO
zk<|U;ndjIVUN_s4@K}i--g>xpGPnh)l!C&-Pxo_^pL?Ho&S!=-h7xJM7LhFwXCmlE
z7hH3Chl40P+T9jLD9sY%mPeCei%N{?9#=)B*DD{)7@EOiFyk#{Zo$oRZyHuVrT9J%
z#vLA3yxuyKA|mg@)jp=77d_6u8qxc9^d<?Os<w13@Mz%iyz}ahUPR@Mf5BUWkwR!M
zO*9AF{YdX^!E}*3)e5ogR!UDcW_12!&>fBcAU8ypP49<%4=0T$g^OcY1cFEH=~8hS
zE1BT|bjTZ`Hh<Tn*A8w@dapaDzoMzDuB;YVPa?Iy0z35M-J2u0cDMOeD?~A!h30fk
znDCS+K*){|uf$L{W9fCh!_I5wV~;ux`(I(jjKX$Eg{vOp1H&_&aiUC7+<N6gKT?$p
zq+61Ka_e5}BssvIO2oc#A@ADZzZ{jQ4!NV1W@=MLsMJs|K0MO+si<n}H(bF;kI;vr
zxfz#k;cdo^LwtJ5$DeBOn>cod%<VO#{{4mg=XHI3l5&00my5&_Y<7QT>Azm!;Ok#J
zBkkk6!h2Pw|JZDVZ~1&>mpJkG6R)VbATK($L9=?~a<zX*fIsLKTgj99EGY`kgR9sw
z%jqZ3GbMw%1_tvgsqp?qprT%xGfBHCyxqLD8zIqj+MwZ=A3|*v4;owh{x)(+=~4ON
zo~1gl0npQ8BW)TY2UxpLgC-<e!LBz|9k)_+b}OqX6z1mA>o}#c4Hv_|=CrWi<jOiW
zLrpe-y6Jc?dwcfcLAYx0&2c2to0^x;XY$+OAVK(7u#<9c<<om<LsbHH-H9i+XQaJ*
zyvl2&3dsPL-9PRIQrHwEOo7+JzG)T^--hyf*Py+e8^_|+@2A)9wCv3~UrV)N<!u5M
zE;N~ODH{FR$`Ki$tZT-m<zTkZZhacXrgg%0IzZZ3Q{DNBltpI3rDdqLy!@jjvf5)b
zoI`Xnv|<6}f^%%5k|yGtW$pDVi#kQn87MyffQGi0`?sREfNviK<?42|N{#ivNXDj|
z3F{roEh_cmh96iB<;Km>9J(!KEFI~6aN1Dt@UB%~+FUQJzS)Iv1-pQ?2yl)_OEBm+
zY49mYm|Qa25`P}xE)B5M@=uRvm|?BV(+P*!9gRAb9d>SooNN)=Uf<d(%Q*y4oNFal
za#S^z2|C-mv4``)e0=SP-Mb-FwPd7}!@F)i10Jxp!r7$-rGs92A9e%S#Gz!poAI1i
zj80T`4qeVq{l(i!UtF0m{-Lz3QcuStv!Z)V;>~aO$#ZVon16#1d}0p3nk{QOeGd@a
z8AEnJ9DD;R4`w<~3-IQnwm{dJWR6KguT8Jh0oaDLq%Iro!+sz8RM?){SpD$`2GBU<
zGkmIP3i(Y?KmT;^m0BlQnWe3XdBB69yh2z9K9KouJCFZ+(BG5xiQj*3kyxPeVAgE0
z@4vMwi%iaS#}}`6)psanx`HDSMX_^EN=gETcLpI6e6=RBen|TeC}>%oa4y{szG|{N
z4eY~enVa&tnS)P(Es9KC4*(_&P;Xt=lI+qeCiO+V%1IzW(-}amrBiZuqjMJ~A2pm>
z|Ff+6FZ`@vAvk_}_p>#0g2}rEzPJY<<x^fiY7GVsjbH%iBKKoY@Cat7J;5Id(z|8+
zd7FAoZrqw!3S2ttNPj}gVJc6<LIPHX@^x}R0gqY_)?!(xM;lH~D>9}9dbO9($ri{e
z6L9?W{OrDUw^l#v{QAJ}F!+W!U70j0_!E1(Z<nG!O-Z&6PEQOsGBo#&^9J%+W0SEK
z^EpixHN}O3kRNFB@;!RLJ>M}nh<+wp`eGvHb<x<#SZ4L?v1pf^kRNHwhk8}IIyFl`
zYM&|a!b6El*sJn_@Z;PUr<9Lxe+=1G1nJJR>#Ie|G=fdn0poZUK3)}{6{F(X+V4S!
z^a>yA#h%kp=43rk%l4S`vtrXOky&C=MLB>}-FmUNvv)UCZ_5E>haI<1Q8Ctg^u%H3
z2N?4zq-(P+(ICUV1pIid@~?b(hH63c4b2o=^SNjp45qPWQcBxSc`9+c$%=nJ2R<ER
zYJqye*L(LCg-7vCaK>_<=62tmh27#ZTGzxwOJdGp%mGCgYI!2<KFOfl(@H7cAbg<6
z1)?-axEw?uJ|YfzclcAERoCXEIA|HW4&xO9eEFd}K42a+$r5=1yaqY;C3v^I#I9s$
zkgEcElIsbwtgiL~?&Iz~P;BidqLIm1Uhs&xo%zB*sgaFr@7tF~7=Z8EIGivRB|n>1
z>;CrJ{CGaS38e6m2-<uKUgc{|+ZGF(dwB2F@sVlq1*5GH=uhTCZ;HOCp`YmzOryFb
zd2`bXZW`1OUSy5%)b5=j1aB+4)=1gqvn3MjA@))8zMBi^6^Fb$)_-{ceE99w0~1v=
zJubFlWgcMJ?2XBl$Fyjms+f<-PFJn&_~v^y`F)i5=dS+U`Jm@Vlg@KiwuDxsat5XR
zmm>2=P-e;2KEB2!ftE#{TF#!#CVL!d99H&xMuP`AaJCCzdn<}egw~z)8MR>Mf;hj%
zg*AZ|`&F=qIfwdl^E6&;!>^`q(BCpSEl!omH23G+wQu@WljbkBh3X2R`5V%h*yJ(?
zJ}uz}o7%iBGJ-c8a_}%yPBF<A2==sYd0UNnftA+P*c&0tyPrs(yg_E<xe0D3fsWc0
z0-W9;U#38R1{^Ic7e8-;vI_%9J;vBB?WveW<EZPBa%H~p%l*p!o8>ae;U~Q97Wfiz
z53x6juOHCveeP~}v}XkbKD%z0h@02fBD?h^1>4&ZJe;UtL&J);fIWLr%wmn0xVX=j
zU^Mp=blVXQtBp5@_xD*Qysnp7{<%J^?>^ubWzb>ol(LVB*@%OKV;^v5I`PXUT6tc9
zvx`4<di0uC9ozkFZHsoh3}>b4g%ypEljNWU+!q_ev89n>!<>ce5#`DD{vxH`W|Lpn
zjII`}Ere|&16kRA`^L*J{7OYvnH6KIzwW;B>;|q7W71V?lgSs2Y<vj8I>RTr!7qT4
zsKutTV}%Des#juA^c2m|)YJz2jzp22FCpR+)nlf&%cgqGWcV9?McFupx|@l@+#bMf
z<;rA9Mx4lX7hrWh&zvQoXpf&pwU0YBN_)+$=(Ps=98{ny#8Xvf8`qS`A<*V7p1a?Y
zf$`riX$>crXN4f_Xsbr#f)ndPF1S$vR-T_rGmSKKw}~3XLgs#Raq-QT4XC3~IF`5y
z8f79$#~f(~Rz)~qG>XsWAtG&P@0?fV6^5240ih+Ul%;IlkLjPr&qb!o7?#@1%RV<+
zX+L6!d4!?=I<I~-5$Lx<D9iM#Uhh8w`)u3DD{VBR5?l@XRGrVb-X8l~C0E#s(jZ|2
z9r84QSKKTuqZILcaCUVJ0`6s6E;(^9VECz1eDjr~sV7>-arhEEHkF9&JPt%xgiK4g
z!tc*aJUADCI_xXfIEAd3n49p|4hx62j3a$Je02&rkcEm>Zbi-qh=)HRgY_XObN~(D
z;%~dsc5%-A4RqL#Ze@LaL(_2`9R8w*#l-C1efJFXP@4(+=>ZX$aiNl_jivky1&Oh&
zWVhyw^YR(b^pOKMD81L+oXgWExb0F0J`z=F>&MM;s>&dP^Bk&BV7`AJQ7f&C?F{zw
zyWpk1@ju-@5JRr1u#X{YYJ8xx=}Fm6rOQG1kJdCdlWGgcbM{Fs;A>~1j@BUzzIo`3
z<VoPIcJrc!%*q<dUSm53iUR1KD8mIj=1{6{+4kT)iJDP)=+ik?%y7*o<Cjp<TGK-a
z<ZaDzl1teL!RyF89UMi)Cx3`1zHL^8egBr!NGX}1z_l8n!vzx-U}4egNLmNGf86sQ
z_sXe?UDZe80|rkN5R-`osuH;N<DYuL2E&EW=J2>2e`ZH9u08yMQI*GtNEO_U5YM4k
z-Zt_OXVVBwj1vq52e;=2D<vs+w&+#BFBlI11p6N=PK4mS8E3z12VoZGQ)B+X*AoNX
zo;S4v?oBLwBfZP2AC{$DZ02_({{iA)bjUK3EVVn%JHJA=_FYs7jQ;2QlhLYF$E*A(
zF=Cf9(-Ez^P(+;yhkCb1Uo@(k4yO2p-Xpi&H_+ENHi2eUc=`c&63^x|pL$~hQBu~0
zFY<jBY71Owvk+_3O&K6Lv)lKI>kMQ<n=my6;Hi5A>9Xjbfpu8#KE(;<hT>fiAI|5t
zHXTSr<Oa3i=B<b(Jzwi`8l|MSz+mF&)d2dL{4^gEb4vG%>@uSnoR$bPY*;m&1I8ct
zyG6kh`+26`I9mBTu&6@~;Q5e!ud2Eamk&cf+i0c8h^@6B&KKLl4v6SO&W<{Xs6H|<
zFkyJ{Tw#rqJ9mw$LfNr#ecnw=ze)$8x(_EOEL4r82)he83U5f<pf9Lu-JKEun}FPX
zCn;U8!l$l<tl>@TGK!-E21Sl$6*bnP5Km|T__}&Sh!C^+^9_&7p|R3CB)-*pNT*1#
zEEfQ~%fF51PFIm>Tx$siM~>Lx%zegkzx5Ut!iFYWAeISLiWc79h#C+GtnM6>mga08
zv^By&RS2eSHNH|5kX>uc7~*oK&<1XliD{Z`iBZLu(j&PeOEh8z-3Si;q`oM-Ctml}
z*D2_?5wNy@91<>-)m>waz&TnDL0_4|gIFu-@<Hb#YqY>P=+OkEFIcn78fmn8if4tr
zgcci=c=$N0w(%pcnKZY3RWzE}arm(eR?y{4DcUo*1ywjL@LpL(Rd9Jb!#7XTh={50
z#S97fR@*A+Tbav=^^tK&V;Fu40OW8$3*P<V>7HrEe<`}p(Z_e}ZU)~u$ff<?dtLC^
z>y_It9v#6H)UciM)nT{~_V>9%m1YaKc3A8%7>tP)Yapg&aE>F+cDwl;!d5z$t?2jy
zL>1g&-V;f6R!jr{JHQ+iQ#%4qZ~wl#@ZG!J7sI+$pnrB5wGNP@;wC9O0=bc<fKWz*
ze;thc2%#pxoPywFt{Tm(hn|E?psMyZhjg|t2bglf%5ArnM-ZO|<ZWTkh86b|o33jR
zWh~3v<im&50uv^cK(=@9prg+E#u48BRplt=6_d&@8np{~H)H?=ls?JS2~BltIml_m
zSZCx;3j>fp!oIw|floNi>*43^uS8<@QH=fVLzJRq`r-^&RB@wVOD+U$#noBcYearV
z5v-Q3{%IkB*#*hREWG(`hIcO+WnRVD+^%eT!!<edd>_*<|EZYjT(Q#P$4uL&ksEiV
z#b?^1x7RRt^%Cu(l}SeTu^6<DVaQehv0VUbLRO5Vt)0{|ZN<%l5#Eb$O!7ucG!;Ez
z&!r6~fz%dU=?`zv8y;Xi=}_LLoCda>tmDYIbT5x!8rTDoorg6oY2Mw*h=Z^_hci_h
zXbXyA2z7Y6j2Jjlq)(fHDgHW%tl)8?4Vi%*?<(XL95|gfg?etOn*W{(AiS35^1c(V
zYDDbxO+!-XBQ5p7<R8Cz&jV@6H`cqXc%ikk``5Hn6f}DC;6KA#i%ciCUW6I2?cO-^
z+i38=)=tKqCpN|cEW*4pAKx=T?tVX35dF;t(AG(rYIfZ}U%oyXvZfQVZ*2+e>krkZ
zH;9F>9QSdxj*E{sp{zBO)qv}4G+JSLG=W)drGsDBam+W`mteXetev{Z_j`%=8>-Zm
ztEftPHmB;XCymT?1^j5DWd~O505?o49X({x)YTZ^|MN8`1cxb{{fQ(z+I1t1jOH-I
zmai)58Y10!ALd&3H2IXb4Me(rgYX?H$vO&Xo8KnoJ;h3@A3dzhS$keu#~9=MaXbyB
zTM3L^9ZLGp&->$W=z)<cyS~J2bGp%bu*HK1RwHHpxg%#b<;)v=sB}{A0qyYR$qAO*
zW2X;{ZQs6ekmJ#bZ$IAn-~I|E4~3~mCVbud_1=CnSU2Z}ZrQ-C9NjX=ZMW82IoFP4
zU)sMvd*9eL-<a=MQXwHoXQZ%ntu%RwKY3q!{`;gN6>rSK9^PvrmEM@H!l?~+2`q9z
zOnLx9!o0EyZjxV_rrQ<<O?nejTX6EV_14e+tO4=RzRlswdX1p~y=nmQM~#f#e9!rD
zRm8`cs6{dgcrOuHjeWhHVbje)EwW9;V>e{ye8G&NH}oc%S^88Wq*$8#-Q=y?=`?sG
zbdF-IBZ#@R%h_o(gmSeMM*W`5z8#@FSYAA|WGIUHeh<<jUbNNZUec%G{)gs723VsQ
z78{67l#*b?d99yvIh>uc<LIl_h$n-~nv$4Wf9WHJ^NFbyZB2Axkg;^pgZ3k25&;F&
zwuld#N>sCiu~0J$JTb#>#n}ddPbf_hH7vakqY)pT4;zuIe(&p)&}mVk5Z5Vfft6X<
zFmM0C>MH%5x&IHybtd~iSTpzi&j&vOviAus2T1A$%3q>CZ#QfCU!VP7A7ox&EvOKh
z>~@d{@4^|eJHtWbd%$K|yo;N6$}Ml~&r<V-6`}3>%s<~eb1<90-Z_=zCu!-sGCJV$
z<QN5JS!XFel4f2H*WT)gelPXx_YWj#*ob{iNk<}`B$D;|tPTk$XM>=5dj{O`5pxG)
zwC$F#=ib^D*wqPC&t<$hPM51dMi|YsWh7Jx4IViO+w;B;e&u$x?@q!d_iWRlA`vbd
z25Mpa10+vB7fHNlnU3PBwCe=M8Q@pHSa~p}?6112(COZ4yWe931&@f4T+mb>3h&~)
zBQBL+Ih7WdzA=Oa<GubbPj*=s+^f>i8A}739=W&bC;1;5g9R#a8AKkx9~|zl^eKVd
zH!IWA{o*8Wq3+HaDDyp#dTrn~DY-Q_V#vax=CS%pa5E)F24sR00BdPNW`7tY<2~8z
zVSRGpu(Z_uV<{kGJW~4Cxc9<$vAW@P8|3BbK5NH4x6qAg)3#)9JSOh$J>4{88$_J8
zrYZzi_#i0i6Gs!qXKS}AG!<0bGPN92v(tP%3Cmdvg-NYRclW}eg$;Mxk{_}4i4GXe
zY>UWSYq?qK_;JJL`*h+y^qv;7MyQIkpYN>g$Aa&7%OZGp?t5}k>t%7MwN@78M$lYC
zdJE6g|2`f6dj6k-OZN%ct7c>>(@v$VXSx5+@%h&m{Oe%75C~^>`U+_f?w{r=Fx~7_
zw$}Pcbr0aGP+wpStRs)V|BSv%-|z3G;vHB&fI&R#@-<b09=}EWYEEf`T%J2p?>YM3
zS$ad{eGBeLFT>o{AOK(-n{PB<v1}8;it`Ot+<CEorP{uEv8Ia$<)@E;J|@g&14z-B
z`qn1X0pOC`8Ydt926)oNc<(ld?A~4^v^4Z7!f2+GqWj=|8!<@J&_+wnU9xfGDn?GF
zozFKF<&BT8CNUsS7o^XwhrD9z>yg)KdDgDLwLe{5pd)v$xm&Nq1nLU#lFMw8VGV2b
zqAMu%ob}_1S7BEM7NKB4%{oYpRo5;stMf1_t-iL~1}PjW!-y7F*ali54;QV26=l4q
zfF|B%lB%7i;0Nd>+B^--ZmSf|pQ12L1w689V>#9%@Yh>mWQ?S~XHjo=+sFkB8V=W4
zkSy(UcYs{8UmCnvF_Mp(wWzC_R%lD!pqlNL!J86fR$ogN)-FnFe<t&D<|hOWN{d94
z!k0qJ1cC^vq*0sgT0}Q(b7`nuR7B*+rrAniOgne)@O3~%uFO;%kG6(g0KSpacey+F
z`zhL_yCMh9(-0SIv>p`paBPoVh%{m!j^(NlvbV%{wHrx+Hf-XcVdDvqB8Huk#w^+m
zv+yqxoN<*S8jFxPLw{~NOmC{EMFIhG!1Y<qzi*kp9uw8GtXIcWGdsk05=P2rc5g8K
z{gL|vGMXU`g0g5bO;$WJDpxVL#wA`=%rp91$4yyYTZpPl-CX?d=J_zGy01Mk5k~Bn
zon$@0TiP|z6yQ4^t=>5x?W>L39wLAlEQpM=0Y63J7Raqyn9S%m5<Kgr23F>+>|OO*
zdr`&~ZOPx;24tt6305F#>l|_QHO_<j74`$!jh*cyb*dkgbHNw50}v}7UpxGrUHJpy
zuyE=-LxD|+H3{SM%m)K@q&}*nz*b(N%%Aiv@KK4$$k5hTzHthm1SytGla*?1K$MCF
z;7znr07#e7@U(B&iL{M?7XAuwM6Sa`NS%<2LIO0hiYLw+>*u&PQDLLuc;aV66gqyC
zXTLu79wISDhbP&P)l^4i_ah3pp&ROd6l>btnm?t1AWs&9jep@PaH3S}PAq-2ydumo
zVXxhVHec&Wl4?(pbI(T1X3a#N8B1t`;C{maLDmoOGT&_jQjJFN9yU2WNFq+cU&6m+
zJ~D8jpm78CHTe=fFumqInF}$Sy@jTcVFw?(fhF56ECOGxc#p^NxgXe%HvuXv=SMMf
zSV^JKS?Q;pANn8V*q|4)BL?|7Mxty6OU)HwXNL2yH|&5$Yb@Zh4Pt7&aWE2^oPP^s
zywPsiAU<qYolfX5%2|c623RBAS1I0pPbHHSO7Gf1SskTYIYbTXZNF4~LI>`mrf7S-
zLWl@2nkm;`>(^gT7KbJihvU_s30Ywu{BHl_vAWKrs>FH=0d+~SuG2uibz;BrszO~>
z(-jBV({~+{E1Zu9FYBA>*HQe+3QoGdwr$RnkpX8EycXYy2p<HjG6;q{0S?#Z0fZOT
zGjO{C+?^B{ysU9U^D@ur4|;j-mreQ-ETd4X2kM&45KvX>l8zu!nE=?2-4WhkC-`zB
zxa+8BSVfyU^6d$+zA&gl89=E}d7+DYunW<PsR>@7S#qXKzKZ2JGy{m0y>(IVm#0&<
z84wig*g9p+cMkw+eaooi+R2lMCpkfSHoeOu#W-g|0($SM!S2UvmllR~&E~_PHf@n_
z4Q9+ed!X{$=L%hSKf-2Td-lq=>-h-4_bO0;eweB168A2_qY(cIGP_YYF{dy1HrZ)&
zmkZ<pp{#!1oKXLG5Uq8$|Bh^_*`)Ebj<ml<H8R;YJ&0u7kY1QK)9G;Q{C2ASSdas9
zAWYc*R{OiO6Lmaw)w`9QWbP>dMT}OouU{faLou}{VXjA?BG&~m2cFMvf*lU$hZdl6
zSr#)YzwVzJY##waCoB525D;kRWL`5IU-k5Op%-~`$pXZk4lOr;ZBT1ljRaZ#ngSef
z#twDCh9#R8;NpiOR+B2!S$KHC8p6V4VjKfjU5nGK8$rJb$167ggx#cl`&W#}$Gvne
z*cTR&MU5;$%%Yrk@MhTAiZbah10Y5uS+Dxr;!YjyC)JnVvp4l0?yR}YeP<JxW>e7z
zawQf|{?qvb7ug4MCKg_|=eX+p2GV3((@m=WrrG?($o=fqmWON*pD1846go5bLZ0bm
z@*8P|H;P`dPv`RnJ_a*5hNX!h7Nc2tq4KWxTJh#NwKoTB){Pg{$IU1nKyUKV{TZMR
zv3CW_)9zUs?eoB0O1`HUt+a8ch&8E9k_T38zH5Am2k+_N3-61S#OypvE%jVQsSp5<
z;t0G~mA=<l*9|aBsI13U%JiO4zQXXM_u(IAi!PA>A1y;~*Q~u(cB8Oxp1M8loAzk|
z#B5wGo`#s8t#AH%JLIi;qi)*5`1H-&$irB5#qeLqFvU7j{)64&3z>Q6P%o5@)V<3)
z2vt|vp7gx&kqxU^k&my1w7qZ3;i2deqC76*b;bJ-rSc>#s8I-s#RkFSz07;1+v7;-
zL)2tk5IPb~nWDKM2V7WGUqdl+a_QTE<#5qa_^w`VF6fhch87JlUT0e2fU-oA(kjg`
zHw#EG_c88#B&e1s@0o`!f82_s6+3kEf!)Rhd@!g2IA_<0#Kam}<t^k}si*3KY>tBN
zZ434kg(e7hAEM%8Rr2g$v~q+|j`!`1$eY|S=_8Y2Ck6LP+q&uCA5%iOJsLEsGbGW)
zU-GT|P22qEwet?eydnNM!E_^6<Yvge|7flMeKn2JrQh*av`x382<I{1nLFHPj%=7)
z_oXXSTMnS1k}sDDz*#%y!!I|fC(S9=*b;L*_EN)TZ?&h!f+8z;yK_CY=VzXQ3)gd=
z1{MQjEe))>D4dU}_1IO(sNA(V-mKPWnIAcr>8CQ71N!!-V}g1bPppQvC0~bwl4fHu
zGq@1c9<`c9<A{c^Z7fZIUHZ_xYJ!S=gDaY-eU+zhfdCe99mT7GWaENHhNdGwMt<e7
ztbBmQ`*Z5DUp1m@?GG`rH1lrIhcaj%h&v^WhCq}-ri<|$yA%OrA7`B`DGH#cPhB3+
zS~9c@^4{?T-|xaB@o}HK*iM(u*$&zl$n{~?*^6<PyP5!5HWy);G8iU)*Bj@zkaoj$
z_QKwXRon+`u4r*~cJ?Frx^Zk91Z5%mW7yk-lgfD!7rCfYwMst;dpILnP@&FDxH?DS
z{^D>U!4uTfRZsq?^C4*Uo}oU4x?%5u+nj!oxvxF+RC&+F3+!7*B^po20JVXR?Z`CP
z5W4L_sOl4apflBk*XKPYQSDSI#-pDszH-C7#QESoIqEsx{=Y21sHg9dKaQ<*hk2Qe
zbyxR)D2+SNo(_Q}Lt*o*5a~;>Rky9agiA)REqtZgzu~C_^iU%E{DHr}9CrBapWh|(
z-+!oWnJvjy9r<^o*q`U+w;J|3dHy=Q*xC;Q`w|BAS+J@wEC_=_<gLGjbGWExQ$<$+
z07O(r@bl#@^>zdPs!AE*6ngLB`SM#)W;xEofKi|bxB;yhpp4CrxMX=3rF5<K4qN#y
z$CsEk0`)-Lcs}qOA3b#Z<N=GjJn<}Syj(qwg6bb+&fpg7o$;TlJ$AuQ2}1G=f&p@s
zuFiM4WR*&7cJYYf-&IErtZM;wc-hSbi(u*fx}`HNLwQ=&egQ#%*Boa;W8hgmJl}gP
zj{A9rq4IfRJXd{<iAxIE%x8S<>G#63-#Z}X=M!xzM;pn@#lhYWjqiFtB^k?RY3#<0
z7<mF{CUh0{8d8x9oVtyv-A<}Q?=}Dc8E2$Ets!k4O=Aco^{VdLrX`huzK#tZ*>YXK
zILR-&OkY)%JP`fGXB<r5*1=-MLJ~ascP0*vGZZWAU4^?{e96k3Qi<x-9ZIC!3iX_l
zt<?!xyhhEB!5pl(7%?>|Un83T?PNBwTB1~{C+T97&fJjd(^JZ%yo$Che5z+}#mTh&
zb?PZ7tRpX`@1Zz+e)r{3@e^uuY{I^->0Hv9{_`XDcsBC}DZC<3>P<RR#xw~FeBh|E
zb5^uLVCR_-QlMXcwV2iC2_ztu<fJh|b)Vq)o4u9eZ4h%FmSbF2f~Uu_G}_8$CPj2#
zFt(aynsFkzQP(`X8|TrU1OFPa{-dkzyuox`oaH+h;4N>pax?!^JAIugz?7*FOo`}U
z=IL$8U<^4(5u}npS=M!du%7oXIGk5|XJXw~Ognz@(C$XfNwt?+(f)&4OO7&^O47ml
zonjtybDRcZ74;pH2A3yeDfJxz+iSSNe5<P-G=K{mIA)ktL--=&hgnN5a(j45*>65*
zz3)IiV*EO&OVt{&JWj{&6=J+S*VC^~2y~i&9`LEZNJPavnDBLOoNNO<z`C}=(#MBm
z1sx2P+0lzRtAKwz2I0`Oj?LS5`6C%CKQ)ceF3MumMx<Buw<&fVCEfG+4}6o&*>EQ^
zI_8J^RBE<24NKdUN?O5Dy?Fx%m$3N?fo2%lyv1-lGDtSQB^#`hq0$rYwmJ=1f5tWu
zQThnkff*noyK)U3uisplGa3L+;k_&Dxb89|OsV;dewjtnS4jxRdGDR8_e^5FQdNXi
z3AGg$HS`sMR-<XG$~HgwP$@W6LJAV9AMj1i$iw_<WlE7y1v`lRy_{C($?^e_&Kkli
zgA0UjvXgFV@uZR28uRQAbvKEe-P^;@Oy08vGUZR-G<%c{(A7yXrYZ6k-r1a1#kHxa
zoh)310MPoa&Rl;BkFO9_%hU>W<CV$`%gM!ivX|3ASOw`?)Tgq;uO%_|u3grr^;@@#
z^hE_9O(yQG11=<7!n;{go|sJcx%H0?@g+OfKN?eC*ahusQtVIs{r&YPY;;<kss5x=
zO^>e3W=F((8E($x1_&&BHJVTHlNF%r4n-O8JAYm^{!pc^@!eOc{+vF2BVaR8$}TD&
z-lAM8K1W$lrOPMdTpV42^4E!0t3WbHPJSY}zGzugrkkT~FzsbS1-hf5wk!$FiFfc3
zeZ!~gbO3zYK=XK}LfQC4YRK%ia2CF9S!;x33KSM^8E2X1WMTePka<-seNMLw>T}!n
zXB&@ecXII{IZHP=wc`9|yqIP4WW(<-N7?a%(4PQP4Xa+UWGpW3U&*0<k5tLbm0lh?
z7s~t4x^A$o97TX#C^MYf4O2Pm9>epCe9bWX#)0=;<6f5e+I&1gpI<O|B)z;h!GK(g
zJPE7rt2Z`1E=%_>nTe26EOgh-80($^?CvWS1n`n;!mj6tZ!V9BR@?`qCS*sK#4H_9
zOH<g1csXg4hx6;vsA*D>K?%fETXvQLi?q~iN87Jwk2bov05SM@w@~ARD_Z$eWbze_
zNUoE;@Y7e@lD`Dnx)UFckAWLyKV1dPFHtp*Ti;9psln;epSte0$++Rs{Ju0rpr2ov
z%Wef^j}dJ7Z&NT*VC-Jn-%0^W*9p6y01>Y4k(<5S2ElSFCp<w8_*b*Lw?XEzbbYhi
zKRHh*1sUWZCu`Iv#(~OMx}gapbL1&<;7FG>(qXJJcQt}K$1En1r=7AEclvG@0^zw^
zD*F44>_2rtmCy&5pPv_IJ9Pbz{U@IPap3G9`~IK5PP^{6-n)9L?gH<tl?5s5>sN2z
zzO7y+x+I&e4)2I=l!;RcIN~y@m~7MX^5A^+koHg)QcZ>}IYPYtlBhXe707S9y|Lz;
zQ>L|UStkX<gH}<liYKNL!kB-0i*xGb-M<EiB6psSI45j0m91Dx1N;kWed!LaYke8)
zfo0IOOF}6VTiSjd>K=kA_YqtQ_v1wLBb}6FfoQcUrUz4)oJ-AWeFNDQ-6*UM^17HZ
zh`1GZwQ@{p<%J|x{u$Ulg7GfU=Bt&J^z>^^l8jigEQ(lW@*onRveFs?-XF%wAFCOu
z5VK8c9BqLVIKHNd^S7nM(<h$jBt_rS0i|*EXny6$TLC8=<cnmgP8UD9;1L#sXh<4K
z*fBYOhI{O6YPv=duWE2=lB9Q$@h7LGru-RBshA`%*@N&_D}P$h?;^XtbafdDb6<wd
zPv2QEa9;y0w#zQ8)AY+62<iU5oZCB7?J(3WoDulmTZjF_0#I<yAOlWa>tq-z_*gtq
z`{XcE6qlBw5>yJ+zdc0W(^#Ot(oQ>-aqgNf*R?4wbDJ+};c&{TWR>me_~`oWjNQc=
zuCPchk1viS(rd1l-NZh+W&iz)b-^v0`0eRWeYrJaJY=mh3(h;AH0J>jBGnYIEn|JZ
z0`^6e$?}?^*||w3gzwVexBH1#&mkXXFug@uS!%SG9Zpm5k;GCrP3kr>(3fL{dEQDs
zCEC))Yu$7FcIOhLVq3BGLfXmFmxEkpOH0j*k_tBh7thV#0&^~Q)NVLjJQ4oZ><N$(
zBTxA;+J#77`9aJ~FvqI6B%yh`^ki)MNww^gAqgj|vai;C*_#zst$@V@(7t`PpUAjo
z^F}n?8`D@c*8623|Cg#neW?-{X+R#`HCw>=m<ND~Ea2M9$kLZ>J-?o;<PU{KUIPmm
z<nA|#;23?>y%SO&j4PiM7K-B^J)#52kxUtyl)SiW3c^He8m2@=%_&>T>I;|s!;%J@
zb<~Gyr=<3uy7*1nzW;3TsUROoRnbMIzEpXsZl0j#ul*^t&0%#nFp~<8<wt_h8T83!
z3^!(B8$g?^1pI4ENoZ+LQFN2L7g$O#*q{c!F8Z`q0dfiuC~kj}50be6NusZ4hBr?$
zi2drvF6a6zG!lEI)Ix)~W{WT2yYKl>hx)>;KTE4k`FAawwbQ}1Fxuy)z<KJi$fcPi
zAb+ViNwhREJyp@xSmMw^%qJ|7Elw-%)olCS-QZ4#2kR_#_1m8b5E}?!^r0?gxSg$}
zA!xhuHd#d^&u$1{Sm?%*hrzeg>EKQouensb?{0tJ!5|8$CL=_WAZkgC%nF6kKTQNX
zFHH5IO}nI{D+l^8Ng52C%=<>y+kM|B!J1P2el^MSnQbsIEBf2(!wP;Ob2?_7iRm9v
zX-3P`ybS?D<n>VM&{&M=0CoVT+Lb!ydCGVvM*QP_pKEs`?hPwQ4N<YX!Lp1cw@ks^
zPVmafI2l>P@4<2`n*7tb9-eoyu7?rGtZ&eB)@_kX=}rg4JtuOv#HJ}=8cjlZF7$3z
z+#;vfpL`*F7W(V%q-9Dp%ra8%A>u={!A}wGL^?xKI9I*^E(<|d+ZxA_4Um3%Qrgs?
zVtwm-1EO|)?QXLA(T$vrCtt#wqbyhwt>1t3ijm_-aM?gpKdS~*X0x6SsB9TmoQae=
zg*Rj-08@?CRLh6K8pUo^DK?ZjY61oNFkJCNj?+7FcjSOPXNb{*VF9<8NcV|PB~xvz
zJmzsECCA|cqyp4B0sepgmv%c&PcsADfXea~gBX_-n^^A@m$X!fJ$j~l`)k2NN9uSR
z(7GoPiJXxac>7spC*SGuK;-SVibj^c3EKP)5J&5&9U=yc;cK}ln$8PAU3sMItHB4|
zZL&AM>9U?s{#G`iHRXPamDgz0rlCe{qU433!rXgQo?z=8MRA=>co6!ob;(KN!3Cwf
z?8FOVf_e2{{bR?wgv|wE4#Sf9bo;N6ru<MJt?o#_+qMIhpai4lsFvT-x$K9|YdX|7
z8@`nvkJ62V#_Xk|?7)n8J@!1t-b?ISf8APc4WHH~?EUlx9q{NTc~k<QiIi>90i)$k
zsvzP%M~Ns!VczE*7!Ii_OCMvcCmaCj=z()Ze$4Y_Y?(lcgn;=wSwGx!R;Pag0m==f
z%MC^5A|z!V|7ETA@7e!f3%3Vr5Vtl+!*2id3oe(V!x8rH(LYs3nG)l>h!@D$t*0+a
z-ZfdXPg_$jU0tB$`o;=rDZPFFBe}xPxfnF8Z>c@G2mtd1Fr>dGO>4I?J1<|Afp|=`
z<VdZ1rzK)r#_59=-Iz~D$Ac(w@<O8vu70jtM|Meb^H8X8DYvTDxM6Jc-s@*N=lw?a
zO7FQg@7ngR*10bk<6{<i9#8oHT#giS9H&_!W5SdNTynQY*0IOjZQKsxn%+l3JJngO
zPnmx^Q}xvBhM}&yVrdYr`-<*S&9vio$c`1K`%=O2i9F&CmfWyM>_yWBc^-A~myTTy
zcR6t*y!nIub&E-L*xTS35n;oD>?pKtSYAHIp|v?{<bZpDyf(!eA5kcs{Badu<poSS
z=D(>Syie5$uh>>=CvZ7io*#L3%3H8Z<^nlUSXac!+jz_FLT$pStWIE1WURefz?@C6
zDP_ab7mc&s{Z6g7>Je?L)wp0sD6Yk)aF@fC8+(&eB40%&d@@^VgZRzwXOB7)R@~}+
z*5B2Y2R3Q)2sB1I=Qp37f?gf2ax{qM=rVi;6s$*w%##Lt0UM*#oac-Cw@5m`M-Jl|
z>m-_-O%DjDO>GnJO%*)}GmDjReP5wYDa%k}*s#oFk79!pxh~v!bs2Vu7m%8NiZ#6}
zkon0anKTJ-j>l7#-Dol1Y5~*~EC9R{TboT}aP+iSG;Gn&%tvbur2qb>HvKa{%qm*+
zN7uF8$`j|l{O9rd_i}e7fK|>$%S$o)tKeZ-b}|o>ltun@+~l`(mxtQPI&d`)b-uM+
zMUC<Mj_|pnxVO$1YD$&_#t<JKt4+=Kq$nefJ$#sCw&rm9C^Yw&A1tgyis|0(%F!X#
zG=L;_iZ0(4U*(kSdmPF)kMrzPo&slbH82lxTV{A|yrwJVn&%rV*jh0r*QeI`JWr}<
zUjna+HTV)Xrohkd=2_@+^SErKCWV|Z*`+WQ!lv-OVJR;~394A|^E1CTTT5MmJ!%`Y
z{0Vi4N|IQW87R6oyZJpDZ4t<Yi?ym&pP#1!Ofj=PlACX%w@702L6Z;8gaF9#Thb4&
z+oESIC~e8d`b!DM9pX0(vxjSX#6c@Kt&%S@L)MM2E-tO_<dL?2kaQn9p6=oZpN-LI
z)Jie?kUX5|RWfYP<4zsCbklvtcl1}U@$SV~^|7#GRdKB`723`WhPQCbZbW!*tFDPg
z#&>n+fYfeGVm7QynU~Es-T>%Ttut;wJHBpJRaXOiEdbOlUxSYb`;hv)OE$P489JxW
zJDU<kLjbPfcl-9`s=S$8s$mak$l6HLvY6U^X95_9)~}r##tVn1OTqJZ@^Kw^)_^`F
zd%i82C&56Pc>C@u;a=6NW&IqXizU!pWgUM@gaWt=yLc<1llG6w7Nuq*q2g8QtzAwt
z@9&APyRz)fL_#5jBL25}ZYFIl!Kf?Q%-=z}lt#os>^G9KC9C9`XM{@5nwYv4_F@u#
ztsXDlf&Y3+ujxXZG`;j3H83m2;TKuQ<T3W(voGxbG#Jj6z0lm}CGY-^%{`;#kBmVp
zxM<7IogM<6u%IrOXC8F}UTimryF=W9?M$70kEqN8GoZRmXbPBqC)`0RF6B9YUy#S}
zs3hCiA=s0uG6zt@PPRs!tcpki4kJ$u-Z5(y-ny@^4*$Am<wKZqzF-|X22#dilq+*T
zAV*PP!S2+nM7*D4M~y*zH)vJZCg%_@ym=YUn)NwH?+s85zcFXu+y$h&_c4c#J{DDV
znEK3XwV9E&XVdYCZ@y{-zZx;)$Y&juAfTU12`z+|8QEY6lg^fwL)lqQq9{75jMEwC
zqGDOq62JOvG;4k|8M>#LTdqlInuB%cnK$&j*od^mA#Xg>!6%}<Q@E>abL7d3Ia-8R
z#S}q2R!R}&DkaL<E!*{!HNY98M8vI~_bgYEGL$Ki5NAj0nE0&ypLVV^E~#{lSF>cd
zZOR%Ix6(Gp)C|G2%xV<JlE^h&62(Ggkx5)qQ7Wg*Ez70AwIoRmBt`LBqqevuqLqb$
z8JUTDE{G-|4EMwR-P>HNna{WL@%*0iywCfd|NCtJ_ngR{w(f9kT<nRF7Z&4d81Exn
z@ek#({GV8i;%>LS$6DEFbhdtYWr=5XzWfD9*vguldN+(x)kw7YSrgGEfP1`)BDm$b
zJIdj7d5QD<0fJ+HH27J?6$H={2SPA}Cp+)y`f@7EoVltmSVzFYD4Ltiix*iS`fL9x
zCr(~Z+lc*2Zt2!!t|=v<B!^8-)CTud^p5JGyxd^;$jP@G*=2&|?qCgzQtj!kWf_)n
zfui?6f!EhG=SDj1bsfAB-zMEC%vRCp=H90mB=^=4=Xx*qPS({KendC9u=hSTM*wzE
zd>a-A%ZiE+GY3077?#quq0_+`(KP9~<$$ZzFNK1<`)$jOd4d=RJ(u3FjkqKqTohDp
zm(Rk-S6ok~-AQEkmZO_Zf!HF1hvZce8{sj5k?1qL0zE>6!7K#7^4va+>=91fz~wyD
zht}{}iTU)Ze7NkGO#WSOg{177v>ujFhfpzMQ?s!y!sRqrLCBVSxw?-WABlouMFV$<
zr*7cQym9Q7XlHw}?=vUf%9_3==4j(T{c87kif%F5pfhKV@RG+LMxTySH!`+7!S~Fm
z&R+2lNJ<LoQ=4nU=uu?MJ37eAnUbQ!eRGV+5LM&C1jG@wfsw^P;JX@C-&_RC2&~K6
zFD^qGDFx|Skg-tOwSD5+0ZL>7o7pjTt0nrG56{S?G_O;3&v(dK_9otm2}zJ8f*Z;B
zleJGl@!u?(7%y42G3&rP3_Ho%uIal!m)w6Y#P3?J*jfL^!qqc(3EIu-K>N>)YQJ)?
zcVaUlD{&!lJ!LgM9H`99HQy@WogEMgj=E!ovS)^*??jzFoR80)k4UZQ8`WZuO<YC?
zhuFAqM%Z|(Tk;aaIxwNG2ff-fIVNHyn9f<vcPh)_<Fog0P_;_3hlmD-E)=>{|G2*|
zV+ATvGjdG^bdb=#w~=|*r5s0^Odk;O90^#y!)086lpTbWoz%|quwJE)hpI{L=wNAi
zkrn2R)_TLN%rjAaq2eAR0*-I1FXZ_cLJXCcXE|i{<ECs0@_V!wBZ<;YWo2f}Ai2Iz
z_TBd@sR<Miy=R$u0*XFo8&UW6gmlU&d2QV;??yEq1T0M3E4$^lFBBmlAzvfK4^l3$
z$5w7nq7<J#!9jQqDFJo1O(7M>0~%U7VQM2;YaC^1?LJsV<g9^r#IeCyksK1flQfPm
zm!l^YK{3n>a?XCKvqQ}I9EZFdWoiLy^L%@IYOmR{<Co?KhSdcZ#CSVW*HN<n{#9+j
zjQ01)Y&YimcIB<rLJ9!_t;Ef&Qy26%>kfAW+^9EhhJg$l$kgUa0t~#m6zxe;)caKP
zeRheoV<rL_fSd%ou88z+7EbJrNYbk1S(bD6DmS>YLI;i^s0}wFhP$7z$IJ$-?!of6
zs~T7DtwwU5>L7u!9VDBlH>-xb4(CTbQyr(bj@$iur+Q~P0E#N;0^#KGKbGyP&mUuI
zr0K={HXob}>66}0NmB&nI`=z2E?u5uG#EBzT-8w(=zB%-mPBh~HRSJ6{36Z}rQQmm
z0CC=Nm#;{WRJsN_Q+9afO=K(c?QqQzZfvJUcST2Q^y!kJlJp4ppwCgRt!BV#8}y78
zL6rFLjQrq&V?KW=P(x5b%8<N0gdvN&Y;pCkW2j@3dt0KXf+LNz={a6c3|T)?Y}<XY
zoavRH+6h<BE=bD^)})vpT)c|>&)v+m=LTwt0jc%k-{C2sQKy0xow)w$YK;H;4TMxZ
z=~`E*Z00J$Aq8}s&lQ?Rx7|n5=RLE_X%rLcRq~5ULhEhy1~|*ag4dn*CazIrlDJA0
zhDkv}fxIwWosTu`X&=8Fo)JP{XLxQ~Phn`ByMfIsDHq%G*x@9VD7;p|?|m(?EQ-96
zIt^%i48Sxz_(3mJR%CA!z%2-)5?L47Oy28F<JGdsl^A+3DiyTd8wM@%Gh85uuNP@y
zOs}_WR{ScD@NR)b1=6)h9)VhAG+DDc;ksQ4$bpg!NH%i7nJ;to1QAJ@e3SUFCO|X7
z#)gzz`&2%$lSGvp=cSn|UrtW?Rc6u4X?^t&K7<$m-Coi;$Q-{zap)1X6r6=zoKMy?
zcqJdr^)c>90`i>3p4qR01EN|Al#bfq#Bx+Rt~gC|m!7TSQYD1ils!Bc0g{xutr;Z!
zZs(Q=!05f-)C8cM$|K>FZ_T##sKLDYUMa2J3v+dLFuZauYE_|XH8Hdy6U?#u6GI+1
zX&}bn&_7W0U}4h?rgx7{{q)4l)OA#_ADyxlE1`sT2#u3H$U;?We!8x_31KE5WY%g<
zd2pipv@vqYBO6kO862euMElW)yLmn)*}sk!OxuPIX{-xK)tuVGwIRW{-s!t}mky5Z
zjukB=;p$NODMx`kyRLeR0Is9M5jTKsQr4<gw(vlHl>3Q<>;y6lHgR&4U`;&3J0#X@
z0{A~#x~P9W=DcF43tNI08jtMI(paSN`H>fX;A?nyNFG9)K8N<3%F$$_^J-M*p5$Hz
z?Q$9pc>jcOftm?<LpEK938PBaCtCL_<FiO}3j-Kvy|S%xWADqd3-ei!ysJdTwFqG*
z&~g{ASZmY1Tj|w}aPb|-IxErKdjJ0Vt%n^-Pi`elt0lq>wFfYc4cS_QAJ*-tbjK+r
zTkko)PO}CcC_L`dRalOw3KzOn<cuAE0-29quOiLbb^K!91#0bR%2sM%GnG`k4K<~d
z>;SpR2Z6#I*~<UHj_yoIVFxNN95q$I!2J7a8(^*5WSntGlnI9LWSgv<2k-PU;H6mq
zT=+|OAu{x;j6|t^vCo2V6B;x78`TNV-@#0<-3BW>PV`R&Ip}*mCO$vWW<FbH>)X3U
z*{vgli_@_cw$%r9W2)1~Ce7B5+5?U5G*4BnG1t%KIeQ(EIQ9?^3sew&FtFbLons<;
zgHyVVWC49oiyfNBz@8tXI#$JR?JZE^!C0_Od@=C2B|gZ)I;s7)&H~6qX;_thxhO|(
zd-DFZ72zYq*qS&Ns3WK`)*B37XvESII)wxMx5;L_kjh|S!yXq`70IKNZc4j0W84I-
znu|02zA7dy#0VsfeLicZKqk$j7gT?NFpxi2I}Ui+oQh}`*z;D}BB7^%<GC4~O-jMr
zEdR{Xgoe9e49%%m(%T)L&1o;;tRn_$x>yB)y<co?sF-^G2+!?3m|5C<UUG;sa&ufC
zM9k%H8MP-Q!hs$I^DWXjy5^L$1MVkQ6jANy!7d%Xqc4JFjK)k>ZE?4b(K^h2b%Ima
zf-5i=lK-qf%G1pBisX>@OV5|HIf|H+*j`@)9&yGH#2Z@uWb?SvxzM}BJmJ#`#aX3|
zr@gLj|M|uG_#eUfivkChElmNf@cZTB7bU+WP~83PE}q-i{Hd_NXTf56SEy8NIlEcw
z6M^Kf4{!si{s8V;K6w4v7emfxYo~wFC0l-Y0MA%*@#LR)lfQ1vzr)+PB<PX=Xk$JC
zvV2jG^*_|iXNI_PK&5J#X8Pv8^U$KQKY>+9*ccP1rTZlxx!4=~$i%Op`e->mw_aaC
z^;fsRS5W<bfok;;Bf^p;OQDW-2i-nB^RHg(bKd2@W11=yeo%I#BwH2Q<&u`DE=T*L
Kc4deB&i?_N!RM9$

literal 0
HcmV?d00001

diff --git a/llvm/docs/vplan-transform-pipeline.png b/llvm/docs/vplan-transform-pipeline.png
new file mode 100644
index 0000000000000000000000000000000000000000..3bdd433a7a59f57d1777a3aaf2ccc9e1e1c8f428
GIT binary patch
literal 555223
zcmeFZWmHw&+6KHS3F+=lDIiF9ZWT%CmIe`}ySr0Fq$CCD?(P&2q`SL8r0bhJ=XuXL
z<9p9}zITjsetwLx-L=-*Yt4Dj`?~JyzLp^hauOJ5q-YQb1Vc(vObG%(M}a_)U!gnz
zpQO6It^ps2ynpdRLF&Z|Dg|4sx9`o3ArQ%s7<FU~<?n<kT1wOv5O}5MR=&ZXAxh8R
z!F_WgqejJ+4);kjlprL~e)xp2<ZC`nX!2W$UFOh6Rzm#G4yF3)^WVISu6obbuhs{=
z?)Z;WIX4Do-H;(Cc-p0W$YT&H_0Oy!sHY}$kFwb$=O9Sp-Vb_O0{Z&!%G1-^;F$gP
z7W(SRAW@a-6mMSN?cNo?jArdwf%6u7mee)4vi95^8`2O;AG?YG`+T&neB8-_&h#_3
z1wyUM&`yG;{iyvtT7$YJ9#MN>4FRO=NgUI62yBmIl?15)J?HB<a%prhD?VJ>fcGRl
zuM-D!StZyaErQrEL4<P;IXS%feHO-h+gg$r8tg<a)W|5u9sILfcP0Y2PTzk-VP2N_
zs&Xsi;?gE^_RkdVemCGKK(i@kcuIq7_%v7)<-*o2SkW|#i%!=dlJwWFNj4K0ks-YZ
zFBQ7si(o%7ew-y~k*7Mu9X`ZG*!Ex8Qt^;WDLWn_x%iYyC(3^M&MvL@&ZEx%htjW&
z8r)jFgkt33UFTS(JBKXqjM45<G|m-V>a6GHid3(Hy)`flyBl^V*)o5`x4m-gNQ)37
zr43tvUncfpnAH0kN$^d^*YNXcX040gaVL*48b6M7gQzGci&zx2Af`K}8*9T?hcH;<
z$WOqx;oD(r?I9AJPiU>t9K51?fzpN_--4((J7IRHUHa$Kngl9(jjqIJyPwpt^U1^L
zSKE<te|C>_UMSjx@*TA?TfX~DL+MU}f^3crA%>xuk;bE!d6zU+e<hk{boGT0Nx~jr
zLy#YF^3f>H)A~+mD4^W>OG17Hc3t$jA*Dfn=>=(Yj=_EFb9=Z-Th^)^Z1;?~$LCDx
zf??=mHR2&6N1K=PMq97n(R$01ox8JZs;gv!XOFeNOCk$%j3Pe`@#%mVqOb?Q3nN<;
zsUf1oEcPrBSa~&TWKFrShboUs;V<F8<1g2?C)GytU0(46QnG@mWI|X4)XqoinXLLO
z^ek{&fn=f+!y92A{-(rQZ*z*E%-ndktbomP_;v=})0tFSFcc3d6FuWx(cF_V_igsN
zz&Dz{lgJCjjxRi)txgpN2XCiD-=t#Al%zf_4VZ-6Wp>0BjHGBKg&+ui;5M|FR3xE%
zts}phZ}w9wTqwmnhxR(Cxs{%&?@fwwp>pI1(M!E1{V%%|7?aVRBahm```sTr^u$3#
z{}8;v=exAKx(n~UdzJ3&(HIiz#fv>xbufD^3ON)JVPmsC7bJ2e_x9MgzT)-RFpobR
z`3m1s`#42KDQ(euDgd&8yKSAlg<zzQX};*5<U@7{?@5L2?<<Lo{Haxp*yFMG1}+;x
zKjJYAl>woz<w*&qIigkzW(nCYGR7hbAL@X2*dj*_=6>7Q2$HV1$RVb?IJUq4Pi%qo
z$at#QVBzI(HXJA&HmZc-$2W8Y-(h0G_z5BX61^X7-%ym1<UTv4mhF*R6&m)L6{n;T
z?lD`%3;mjx!(;0I`YWyC<KE}(Kl$w8zxc>KdpKlhk6QPjP~7S3$)?nA8dr)vF}<`K
z;uaM)oql|~mUv>aunwgX`&Yg{x};0d3P|(Yzbx4`g*q5EPD&&NkJvoehp1blP(~gf
zGM<~evo~Qb_)M%6o~vF{&j)$Y{zM5xbn~V6c_B&j20IH^8il~$!vBT;h&8JN&OA<t
zm`*zP5XFAy8$&{y2R7_B!Zuh(h^2n#@ssj2QNN{5>8zCKLS%aBV?K_5a*X;AUG%X|
z658|R4PF^1e{j-s!l<wwg4N_zqgAF=lhr|djL=7&(YR7e%2~>>N>xf<7~>dpJ`bwM
z=5dcDl1ltYAIK>BI<6?Me4x0Hlamvl>zpH&kD|JsSD-kRZ!5R<wO2y*sj8rA)wZ*0
z?l$h&ULL)QResagp?7LrqOZ}OrdIi@y%2oq2D?fVSGQAH$vGNZ&r!`A%hyqK%av2C
zRx!zyS7B8tRuE0|lxkG>OE<`D9bYu<AI}+mH(ZlDR4g??o=K9MkYB4|m%Z-t!}95;
z*ci1mLcM$qv#@c-!fyL;OF5Gfm){q^3xALNX0b@Gs^pOnl&OE^R_$2tnEE01LztZJ
z09Uefa)v;1U3^gtiv+KPQ|`X!99dmzAPI&M1|GT=SApeZjKRT4;LgPs>Tl_j(@mX;
zt8arpu|@~S^@<kKycQt5C7i)L@unrMixtlftoN35+g;jaf)1db-Pqg}oZ;+A1~r`}
zKS~yQCK)Cr%`3(?KPwiO7L}&j#2!hn7<CChw#S3wef=&Sz4CaFL;#(THmEdksCSc-
zn+t<I=8Gkd1@DddTMI%fwKrAWyG{IKN$*oEn}!=Y8wT4t3=2A6g}n|FjK!D9GR|5v
z?JJKxC@d(%EhN<7)^O0;(X=agSZG#wF!j^i-6CsJWk$RF(KLrexy8hcZq<uD?LCgY
zCwnYY8K;(~&#z8SDQHn>g~Wr!RU&J)(zaV{8LV?0oHuiJi1#&CCl}9czjgL}j_Q+0
zp}6?*eT=dDeg4Rt;_Hm(5(E*Zlqb_u0|^`2TkaNI7WO|k`7_8Qnr%y+aB9r0YuBWm
zD0b>vDmzc|6YCQ@3I__i=h}qF8hdXZD<_h4YWE9mJUcu*i(j$nSjosvG-=xK9BghN
zZzyYYbajX<rKBY;cjz~=;<PF{dwU=U4V2Fte4Hw$r>s|GRe3-wYA4#Xz|(BPtffY1
zMzoXJ9NpY;_wmlHir$UzN(@4XO|C%tlQM^Z7+S+}`0S&mg1CZ(f?kQ)VfZ2*wg1PG
zk89Mt{Nm)4*?}WO+mr%K72L15nR(20CNr$-t)-lgHSEndCkc!WUTTO3(~#0UqocQd
zxq8E}lf$SOlo2wdI21D^ZzSH?AI?uE6Pb)(&v~ko<TX27Q-8#BG<+V5c;bJA%8hz}
zhk)CMCm1dM_=j{|y!l7wSC+4+GwHsW%WOf3_$<e2ijIm_WV#mO^@R<l{g3f59<RR=
z5|Hx1bsF|jd2VXBJuu2TD{0AHEgBK67Bk*$bgXu)dw6rsbRi6H^{jSLv(CxY#nma$
zsVg)*<<&xeXumGCR*l^G6DB${qw%ZVp547E1pW|yH{0*ExwW!1PZg_j9LkPX-$x5}
zoG#<qY~T+v=4btuYNMO^^|1lfCE{ZGP@+lJl=kN3Wl2kRTG?XzsDCbU)_XZTS+hh-
zUXP*QCIQa^nt!yI9KNmS=St?cT5s1Z&`dO;EgyMX-Orw?ER=0#c{odPoO4{ZaihDc
zqN4lT(R2BF-7sUhwlmTT^)|CgtWZqJ*3M_bH{#(@KwH3Gm^ka}`b{my=HJf~lr>*Y
z#dS7x5_wXb?0nHfj{U~^UT?d~zMW&4W2lU*&m-=DVS<^J*{i{DH}o<lJSLNAq3BD+
zwtIj~XV=*1*zj1gS=$6xQKd$``Cgnu!cLL;rTTs@N5N~A6Hj5Un-~mFvTYtx3+9>g
z_tDlPg+E#ock&DKn5wX=@y%W8xD9ru7UG-7%sD63s`hpqhfW75*acsk&6=4ylJ0B2
z4jY#?b90PXaBy*I+3ykvs{7$)fA)C~$t}|@^+0CM$;fN+HWr~D!2$UxmKsGm^hu$_
zcJ{;>_ZuF2p2xh7Tmd|lTyES@fn8_kE1VJ97zu|DEW+Z#YVPKX&PzwPc*}T=^qg`c
zmI<S_XDY+FW`#qAd)e)Bse*X+>!X_EQ!5({gB1J+E=^nT$IsheR)?Kf6j=m$I^EnT
z)N8rjJUWw!h!bYvbqe3&bL>3p{hkk}KB&E-UGFTs?Ajf1PF8LCp?S(B%q#!OabQo~
z32ooKN$`wvNq5U5;)(zZ9jbC?aMhh6S3Qt{g+N&)9O_|z#(r=VM(-luWKSTPDO~9M
z_E$mHOzoAVyu#qx#!8A)BX;xotk$pW^;A_>3th_VzJubbrf*F}>YvmRiiw(syvlD1
z`(|Qqb8e9jY!^H{^)8>?wSS20k2Dc|<mr0qvQ0D`$E|14%;WX&%F&9-Ngvi9W4+c4
zPh9ZMRRB_4jR2E*;Eg$GGHMfgoZ3JY%9&5@bLUTBBIaU)kcN5NEOFM>oR|pNokNS%
zQ_%Aef+%u9mW3b!uD=y1gz~=+#*{tx-|&_l(i^cIX=!E*<=nusxtoPg$a{ERdqcEN
z;_mRYqDC7O6rsi%Qg3BtAy2?-6bK?5DFg|;f&&i`II_RL7KdYiApG$;JOmQ-9)kGi
zHge$k{_i7r-0$<ZXM|6I5M=NU7d)KP;Qwc9bd)rN|9Or43Va87ru;%m3Op+t*%}*L
z+P$-~H`~>|1|OhWOKRFdAcT+ZA8=Aibcf*hlkZhD>@{Rx2^d*fuo%3tGBjp!wy?fG
z4n)XV0KBv?wl|=1wlKG}6L1!$`C|(K@cRB^RvM~5HnBGorqPg9pn73tYfQz%!p6cz
zBZ5XnMI~hW=B<E|nE0Q&gLlF-@9gcZ1z1^~oSaykI9aT0O<39a`T1GdI9NG2o`Nl&
z+PPTT8#q6;w4?poLH_4BV#anxw(qU&-&<Ky-5=M$(8|GHn1<&5hyMQexAQc1e*ceO
zvb6j2vcLth-hac&&cep}_hW-yh3-EUP<ZcbY_2Kx-U5sn_ze+uJ{~@yKX&+EU;X1J
z|7ll^f9%T1#>e*0yZ+Nx|L<K@?Tl?-SXqD{wHNuvg#EemKY#h>jzX;W*Zxmq@wb8g
z<5MuwB4|Raf1foGG=E%;`v4+;FD9=7o<W%1{~=_7e;EGu3|=EdCQqQ|RzM&yh?Lkf
z6=%3z57b%Jk=xrRiujo{G#sj;G{yr8s7qg-*JbrTdC|qv%lv|2ggg2H-HRd@20Bs(
zDN>y%#w9N@E~arW;f;Bn4dG;u)9Ic1Q<C}Pg24@sGZDAqBhA6rZkHlES}z$he?Sp%
zsNluCQLy1?V2Htx|LBcE8;T0MLpChUU+|+U{eSDI5JU!SDy;wT)c$d8D)8WVIf%(U
zkN+E&{Ks+AVBX~ay+K69PJ^Z4+S4Td<DCD~Glhq<{qM{$I3?I4m^ZwS^v@7;`2Q$a
z{`u<A5PSX$;rHheV<2#7h)<r;m!XRNw=Nn4AoBkofd9b={F?wo3v3zJvMPLCX`1=E
z$~>26*7^mv+o}1<n7UrB%}Q6U=<Rui{)u5Ln8-I0`Zp&N>OT^B<s)X^{`is`@Q^6a
z>+Wh{JB+nSq5SQS-jra85Eu@%LZzu{<%pbMCSSvhWqqz{zFH=qkWm*px6N_~O&(+Z
z^Z#p>n+5Bz1%o$>=3gW-ICzWc3dKm_(|FV2Z&Ksh>jTO8mnS>flrF0{mF81r1vbtf
zQLufksY_ZPkmc+Y)ytjiPARJv>k6B-9yDDQ)t!uMjcpEPl&5SwqfvN=I3RqcJyva{
zr&?_}SD>08<Wg3*@<DL4#Gt+Gev5xzD}V7u0S!fgN&&60Ljd=GEvF}_@Q1jN_B+eD
zy4)wKhmv0eUwz?{qHq{h+}=p_8lZ?W34!29qWcJ+PQJh(<&xYh>tj(f*7mwN*^bq+
zEDi0G6r+NoaG8!uVi2>zC~SME)jEZ*cFVVS%Lmn%)Qc)Y2POZn>n$z*KE=hm`Z-Yl
zV{BoBh^C`CUwP)8ja19u%2v9bm{uBgqE)(I{2E=*0}THW5*E(=WrY&KIU{aPSDF=|
z-Cix+se$lOl8I#wiTMx(F5x5X&$MR<+z#6+s?DA^8VRj4uv8Hb!%TxWG#?S+cZ!>v
zNJTx}w*_$N@8Wb_rIMYOLu@{=vHV|C83P(X4~9@nhE(a%JXM)Wi<$A-)&;M-#xESk
zp?0p1>HfSRg#wh(LM`>Z>IKidmtlmt&2x@3vKCDj>x>4F@GoUZg_W0sUbh*Zms{DR
z3w^B31u5V}%oSb2?}1nWABEgyHEKJ8ciuYh;d=G)fZ!nGQ#^3>iyvCdiW_5pE#vSa
z2a{@5TNXZk6_3r!xT0>02u36hx2cW`kLfa!n4LyV$+^7dblS`)6P2bttVLN~nBaOf
z_Wz3WpAq5lQ9g_Ds-{TaF5KOuei8Agv-3NoCeA0cT~jbYNdV#f$7G=Rn2zP;wquEE
zC%ddCY;R^ns*#zO_~bYairzJB&(@YRU_E|(KPYqou$U*RqgbNXX;ssvOxZNI*T?de
zKX_(xYV9@zGEHc})GpIn&)Re|qG0PFsOvkuEV9k^Q;(-`njNZ*(inrzmZ1vnmiT?)
zHj902)p%l_UW~T^CR9!K3zs7HItR9o5)7idK_z50U6GroUaY$<{A51U5Sni9oOX3{
zwlJ3dGE8X(q>ZdD6v8(bzxr62eh5xr@XjjS$9kSYd!X5E)vRsbsEGghVQ-LIZ>rZV
ze`Tl0?RtMg2AH`-rA^a(0mIvAlT?pULM`)BOeoLvJK9S7tzm9$2)#FQjkl<NYwaM~
zTk(IDGX81Z;y;2n^nqoFN2naH|7hge2T?L`C9?q{IrjkusdMgKH`J?ukX1K{vslMz
zPSv9EWSoD6&IT7M><Mz<C{&HKt2^yEVlE8gFmFF1OCX}ljt#efvfFgEj|P3nWY^N&
zNW%G6W&LrPg;?4fY`()5gfVY81hFo~gosQS6&}tlm=UlDy?t=QxgLEr8BE#UuRn-=
z(^?@$MF_uki|%#Mc$O!_QcFkpb}LG~@O8|u;0}{>dzBhi0+b}1E)vtj#a1=g^I)bE
zNXgypReq^a_v@H$kRcXOT6*eb6M4*TPbZDA2-baA@+{dN1lb5R*rP*5v?EBlr41-q
zV<3txI1=Pv1st|N|8hr=uVOKnu6Vy4K%^^qBSlze!>IM;!NL4e=31H`rnC)w8)O1P
z{k<auZ>;(Dr1-POAs8O}A9*>$2JYk_4~2}O#^0RIScR+&hJyq^_2&xwS2@!Ly9HNc
z!0Vvl_``+cB0cen_J(UsmBH!ml<pHN^!-Y9<_`tU>;q>?DRj|e`RY4*o)<3@W4wPy
z%ZudxkifGD->2Q#54CX>IF5R*bfPj1E?_mYXzOIX>55$O$z;M<d&R{j<GM+y$<Jml
zuMf~x?qZemepkY?B;y9e&8z)-`IY%rCt@CR;%>ffK7JS`{!?T*;S-gk3SR5QmUIhf
zadW4~jl*85tydtF^KH2E?0)0fJb85kkLsc~4oa}|Sf)%I+Qk~_0+z^mS`IUv$ldjE
zVPBsrQA&Q&M9A}#d`)lh1)FXP*W2q0!w*5m-VG5sI*FY=p^oUpSgLhK-5)M%H&RBr
zR?Ib&lOoAVy_=4@$Q)L>BEks$@>KJ)*v`QMPMRT!!9HN^ii8%|JMQHj$z>=soGI=*
zOz04eX%AgA-|SgEiipGtPe3kqT7`hC_=xbYpI}4QFtsSJ>lJ7^QPx&Sl)6ZW!EAUo
z5$iOR*D^_U?r!1k$}8RWlQjAeMC5t}>B4cw!eopvt|=V`<C)iHGa(i1kKq0MEl^!?
zO~%KY0?KC@to(_bnB$G4;W0cC{Ho=rS>yTDxX%c0*SwCd&u*^{=uZo1ETALm1MEus
zgNJ>wPw^sW@Z^P}R$}3O(8znTVJ!vMtn1bQ5}u5v9($}M9mr(rMeu6h^m7hw0=n^{
z-(m(`pZs=MhIQ*X^L})`9+fs5rH4f#b!gDe68D65#*gjemz>bi0!`(!&7l`puOZ(F
z)1aQ`&-CnvUe<|6kiPVL$&&Id)eIydo|IUN#BKaYcsO0!Vg7E9Uq0ki$UT6I|Es68
z7V?h{-tx3pQM3~4Cz-Z@2_)(MdMjhU7Nh=ptWhK4;qZ*V>L@Y>>9<5~v$tbee27{1
z!?djbqvoh23r0n$yGnFKgFMRzSxKiuK!Uftc`ky-YfB|cen^e&3=&79XhzB1@yK0f
zs9mjS`xW*dAVXe_kD{^&%SA<WJ=s?MdYdVbS~{QU(U`B>Sl5+~CFPtRb{u|>RppCJ
zJytbKftO&Z4dzSIw7lCM&As$Sz+^a2`dR0&)9?@h9^LmI@bYE8%V0OihLdtX$?-JQ
zLPL~xD_X1kbR+R^0|mmb@STt9;3vS0*}jLW1@pc{*B9{4=Z8zW+YztK)#q4|R(v6!
zMenYaML}Yz1DSIE(jOk~!ntqaoT1tl@N?e2h+}iBTmFSI<Gw}Q#7QvU*Jn)Kxa{=E
zyZqKiokU?9&54$E-)G8EuLhb9dP(xZ8cSt23Ur?3dH2)%@A09(`SYtGrW5%0`AwgT
z+?$z)c6+lH)pNh&^gJ2Y%Zkwf;HHuBT55l^@i{vG!Mkv78mcUU{a)zye5Fn<S-_<m
zLWyo!yGDp%)zXQrR-~P46RC%0*FI&EnoQIKxI@2?CK~x0itiZzIt>1Z4Ig*AwB9>3
zF0wQ0y~>Fv$A;6weiff3$Zh;xulA}(KG2nG6~5*+7WthIKtJqZJlFeA(?*fP1;3sp
zoNSMkhiWlL^_#-BK}@S;NJJ8<e@8(a{k=8v^1{4sE6a4LEg)SeJ|5)_jXom$_SHf2
zppf>K=Yrj!@HPO&V*5QqzhD?6)ggG34_GM6P~Lr-$+O!S2#Ia&2Pu}}DWb0P5>mhw
zd7M7;yHNsl?XO)QD0V)%CvyQt7zqg0wLtd@Ip2D`(I}uRg!HH@xNtc4z+%<#Fpov2
zj?d_PB~nyi@Yz5BhrtjnI1STo3Mc!b4HA>2uMe;&&0}r>BrCribI7kTh}xbgF1|fZ
zz4K3Ax1ssS08gZ2AH|^?|M`gO<2!yEPQ4O+pEiyD4G)rDGB$nhJWx%{Za>CdVE*!L
zy4tGP`=to{{?-p)3DZI0vwjJi(%u%%ZvKs=3;?j?Ogc?8R#j7PO##N|0cspV<T9!#
zk;^;pOnYRj@z-D?*75-E2#r-lEdVRl$|wJg@>a*)V&H^&&H5L!f*mdCe5|Y$1S@!9
z*WV?UCK}MoMQLg^6EridNP1DV;{bMH6qS^wej*bMOn$r|1W?yZXc)6?!LZ<0jmdR6
zEL{#?)=AreJk>D>+0strvG{VV@=mO6I=HZLoW9X=mNk+cr#iLohNKC=?SK&>D%9h*
z;)f}rhLzC1o#)+gx{2ZMI=`rQF{s=fbwdY)Zdaq_3;LdL6)`ve(&-N8Ye}fTkPJ4N
z1>BOA)o8V1L~g`cRL9(4xg*5oh^_?CogK~cw**?pOryxUEbgu}r~KI@ikbm+k1zZv
z5uYq*??#99hh6{seCAEw!-;$iVPXF*=|wH*uszz~u2KWgW+aSYkN5p>N8LcCsWvub
zwBE^Tr|K;&2NJwzmllt?%d5brNbnrjJ)+AJn5*k8s_Hrf+R}J)N>m@~p*_mW${)x-
zk{VnRqU_cmG`UWd8Ozxjko*0#-8HM?Pv9?_O<ihbBEG2i2Pv7FRXlptlVesMSADCJ
zasMRE?QEX~f1F9J;Aj1AS=gqq#wR|GgW(m=BoV=z-y_E~98RR<??dj_Y(!j+Soe3v
z?4)$SBa_ukZT9Y&`m{5n96ADXtWg62zIw(BQzIc%y1*AnN70mLi+)&`xfUEjjMzD|
zc6I{QHG@r$|81)GCTD=dJm313Xw@_|LpBzOLH78T#$|W1RCim@xc9S$!&8S9c<%=`
z0ZgTg*oU0++D9gJ*4K>KDBXb#)I6)+NeEqy4RKc=`d<+~ef^DyRkxu_!xtXTMK9{0
zqom9RX6nDCMFnw>1b8;d)Y5eQt8ZYCDyC3=D&=}W8LUFAro!rZ*GrJR`WrApu019V
zf^24o-C$~%lEtz%NbrkC$xibm->3Rfgxya2UwaL-UnB?h;T(VzqgMR7@=5fkmse!I
zYz5Y-VtNkyvo1f5Er;OspYV8IJ2!rRoao%GpAo?`Y}6IbxSmz4TzyXf=54Ik)Ie@J
zQP=aRl%t3ig84v-af;lH1#E-<6&n5tkyK!u;2f&w9FI?RPr^9|B!55RP@6ld-2Mr8
zJ9d6iOlu8fKO<7)a#LduGZ=7$)FZ;h?}eg5VBCTpxkxd8_X#D>d0m;OTVwq-?&=#V
z-s4S>q_@_8B<_GbQCLgZZe0wN2G{z%DlNP48--G6F!B+DI;La<TQoLG8Y@fFxgk?j
z3n&G0ug(sN4ailCJbE079B-eTnVfrj)f{*qWypGe-2`;Dm0iXL#(?~7;xUoWeW~;N
zBlQb&h61m<8>hKKE#3&OR^r8~5<46IrRKF*tt`HGFUM&@f43q-H$esELc#g;bp-?C
z*A5_OY*yYZ-T>mKCDv%JPT=m;ns?L5W@%No^|)TzE&L@IIik)Mk8vL%y1xjqb13pP
z9Q92LQA3c&dV&eAhfg>GlSe>JQ5Tb+4O~g0s8>@o4h<6gZ5J86O|ym))gqDnuW08_
z2xk1=G5DEq56ZHW9Z4fTzk3(ADa_HmH*KSpQ1VeE<Iy8*NY{@q+#@f7Fn>J3*C;i7
zX<9itUZ@p)mUM;I>jPj_J{61FRp?|?`J_Ozylj5DABZ-hzkj|6DM(Aw#oCX%J&|^`
zu*E?^Z15Q}oiv@rKEVXVfmQhr9;9R^-#S~ZOKqpE_Uu$ofgN%!G~{}xgMEwL#V^O-
z=~F*HFJ~GZa!^b**KTctWUfllPy_=(g=Ni$L970kLxEKEW70wV3rtYFZp<dL>XwuL
z3j3@0!bHxyW-XiQX$Mv2*P^=ZCpO~EH>ZiFKhu0TSgkr;0GWuApj6bUcl_b_WV{rV
zRXJd7@T-v;X{T+E<r6W<mWopfyH=QvWXTq6llXXpLfN%|D<O(^&R*GAWcn@Tx)lx8
z^A-TWXA54ey&3qp{{ytF0eNK~#L|YTepb9!=C)gZrRa7qAr&?Jl2)XA^$IAhE;Rb#
zY-1k;_cv$HwX7Odb=>wU2evG5#{u~?DB|*kGZsqEVN}lgmO>K#?6a(G7d3ncjRN4k
z28%rAlL|{0^UkbF;>2l>Wxsp_3Kbb)a(4&}2@yo-ii-*crBKcD6dNxfQ}A{!B|A<_
zJ=LjG@wmO%Fa`BcNMl&ImS`8NJR+vhAuQmmFq1S(HZdz)u|95_%u0D1Q0=o%>LQ%%
zy7o#74`Z1%X;{XcjTJ0rst7iebI|)~WVhGi^jK7mwm}>#i{r<h{fZRWW;p30vs!J`
zCY1bZ5w1t9B_E~_@LC1$@v9!@_ARG;wL&H6MiQ<B-Q{|sl@hd+HvMIaBYSM_t1cu+
zPeJH6U7%D+7WkszqaaFJYnk5*VdlbZm)JhNel4Ib&hLsK+n7zemtvk)&ni9DYntW9
zx>sw`+;IS&)CnDbW}P;7F&0Y$2*PBPS|%GH0duwZMUBlMgjN8DlM@UP`;_W&nqWGV
z{<0I#SFic@(yUjKG~F-N^HR#9?EB+9!2KB!9GPAH&JMa(lGFShqN~g%<Uq-5I)}&p
z!5kFvjHR34`pIck?MI87E~S?S3oMJl+K<5<dj3-Zlv<k|?1r;$2aVf+{Degr&gCQv
zI;nwCn60BHk0dVk&pqIw6%SXywq@3B;H!k;5x(c3{G^;T>2Wp}b2PiC(N_0eKh3L$
zI;g;n21_CwS)BV#&qtxp6_z;oC<S4tk$)&Q|4zkK;6rSvz2%nuX}8{gv|KsnOmp~}
zbri$v?a3?FlNXPlYE^~nNEm5TJ-I7xz8zji@*HS3?u!lS4UTpOg#al_DuR1Pl{ghP
z{Z)a@&S<WYj;@M_a|(+(JSMRZOsz?^%J1gtY~%c5;e=|_&5|FB?uXC4YQz;oYy;Q*
zwEHQjBl+xFhz&y5(?mQj`Qnm)U-w?SU<@K7_ui0*K#9x4f06YKn<>Y!=r~&}H!5bX
zcx!HH+YiR3Ocv-CyPfXl=Bwt%;$?$XxdT)oT{3o8SKFc6hpY`B%U>n(-|5|5#^qpc
zg&eU^T<nz(ifHas4vCX(0hL26G3IV0I^+~Uec{=pQDni|R0Y%qFj}U(7p{i1lRw3=
zaWzvLPp4%PJ5z_FL(d|DN_uQ9*h?c!8nEs)g3j9gw<lg36_V%G%{NaEEUG#Kk9rSM
z!aN2<=>KhoPHW&_9P{(0jt2BLwGLo(DHGa3^IenaLQ^9{q<IKy%_95>pcQWg33etY
z^t=ie1BEO<7Kz+5=O+dBlmz$t<w3XKiQ(klU{VM$9Qz7AdB1y{Ea0HT+H{_-D&UG;
z;u{i$<_%F(<b)zN#)glzeKQ&${+$mrFxUKv^P?34wTDJPFQM{6iH~eAZP<G=ZVuGb
z`?{fDsDI%PIhP6XTRzu_pEp5&xJn2P13*3L*-G=*a5x`z6E~lwc_(+L<Gon?8bqly
zSMNl$ao5XKoC6YK4!D$gfnrb&ikvjf2G10Z_`l@pI?8pL0Fu|kuusS5V3)jk5V=V<
z=JK~7o#X*=0&RgLz3l;+d0L9=Hlsr?W474KN&(Tg{lpF}aL5{+Kf)2Q&^DWae7)5r
zd=2W*UX+*k^Nf=ZoUwr(oWDI&UHogQbqOqYWuWcmS{&&v3tw(#Y+oM_I`!^=yse0U
zjHv=}%RxF)>^IdSItAYU7Rvu=eSCz#>#B3TpR22U7Z5{r6iUFvrFO`^cXGL5hLkRN
z&@hs#J3NM5`X!3#F`zahSVD*1x}f_2DUU0>IDDf+?}~lcXQIx+rsH|p+U-74V`H>)
z)am&{z8TkKnLG|bLs$-1`+$FWJo@Ong>9OxEv!zO6O;vVy4DRdXJ(*Ak7hd6rdB)E
zkSdn_O{E{n?V?Gl(JHtD%4B92^H-UTTUm)o3)CbCF2XLq-q=ulglpKWwH;)BzLJS4
zo|fjc;BlsL!uH_eGR1BGNJ)fa$~aEPFb1ms7LjsUpg&nq-3rK>a%&ls;tbS92ps1i
zw32tQ7jaDL7d)@tJ4aZf%EYmZ%~(_?-wW;gpWJ0hU9lM=TzD~yiQ_z^|EiqNBR|}K
z24R}fkGTT$mh{z|r5@Ty6`<P2GDla8b`_TWA%B-B)o>A4cWjTyi$)~NHmO1TRoM%c
zvE;EpsVi3WZhRwr5oS@^NoWe>tf=?WbAB)&_N#yhSJ|7cRGzulZl7wDzzkFiK!ZRP
zF3c89yd$FuNbRG}yih$Q;_Kfd^1A8`tj0nSFc^i~ZfW}Ccr0bpfh7KPp-@H}zXupf
zOlbxL<v>{AELPP55Hb<26K-&A&`(;ulG_Z>Vy#jnfCYVG2Y?0#cc9sLK`y|$bT-cj
zMc=kiuT1Qp5BmFBlfo}*LH0KV)d(6B63cyZ4j{rj&aBdx{p5;W1KK~MqPItpzGI^5
ztu6D!6gfckezo*PUnD1+4f(c(!b5=tPTPNicy<TiQbSH+L@jZ?fBGQXsp7^|xyeT6
zz)oScuIWp~Sy6rM<y)Myjhq*Za}ox|vIRK_o_fJ17!?R-0h7z~ImEv9F>hV5b0)(!
zuYe?zwJzN6vFX(B(8`UR#AA3C8uH|VRXg%XXcB~?YTg-sZ5_(WRPdhN%kM4&;}7+V
zUZGu}r{?R%?2)gZEWZA5^<4>zIC0U^x;SvbdA(H-eKy;mS>dc_*bGk?rJzqvJDLq^
z<(B*`y^D|Uq}q0^*MT;!1*gHPU6|=S?5w871j!2FHrBMhElqSY&2Kaa>PhT4y?suf
z>YjeO;N>+`AA2uv%O>$D8gP3^c_S}?E%+P>7un}!7H=Zx7Ly;$&&hw``p%dLBzLuB
z_g~-T-jCpsl=?(xuMK5L<Rf4Su?go-UWqj23ju`!EyeE<oZXO`mc^s*Z30jC`$VsH
zIV(XEP%~R9GWd+e`OXj#{x{V}O!TwA)QK515gQ&Ndi!f;>goVd&rkt~u(}Rl@q`?s
za(HVkfO3Q3e5>IGXy!`LIc*Mx2P?j5^Ug~iK&g8m`ckWA&*S>2oA#K2zy~6dvtQax
zncEB$tIR64Z+?ne8Ytnj`uGq0jEJXvt=Mq859kokZS~_fNe`n4X8Q>aZB9YZs_)HI
z%O63ZDMw#<w7h*7v<=z=&VZDXC33!qa02x2H({@HBMItZkO$gq*iIO1hj$<yzKG^E
z8-JDTx}BQ`cyMkrAa#t!T!Dx_X^2p?qoa#yvtm_jjB^w=_f~B`cJgpNck+&!0RsL0
zpwdwGa8cqyVk|~eP=<z(x~PFxr7?42zNO-Z{v?_4>`0lhjKk@aN&Y<x>^kI`G0#wN
zd<$EE1l{w`F4iu#F1xP?=vBTh{FD17?U?)<YiR>)(@CIh^aW@aUj!UXO)W2%!&rSs
zmY<5D0ygXLPUL<OgC45J=0y(BXpj6s0=H7B_*_EE_zWk!kL?cXIBAgU*>bH1I(d@q
zINA?=_;0BS+>d|oD#QEsbU6om@9?0z=pbx=*wL)=He1#-NzP;|VyiYtkx7-O{s_BF
z;55mTfO>GL9HJRzmzce;vgM3inkMBIQS5TrNRHzmY>S6unc63@hOz;WF;;VabrS&<
z6)ES-URadv{y;&{+XdZ+mXTby^gQe$&p}w%H4l4~W_31eH^@DKudPKD*md%ld66_^
z=e4yS%8P7}an&1v#=(eCU1HDDZ6FlH&%qo{&d>4a!9zi)gZpHOLGF&dijek52D+&a
zi8B={Th%}uoQVII_%lr5-#<qik;2E>pzB|>tSN2)&mGsc(_aj5TXILy2a~i@PnWi@
zxJ)>!o@_V3k9OutJc51gKc@^6i!g)i(;vp|%quD#(2c}iaSEZYKY~Z?Kj^|sS0#Em
zr#$uM*U5yQgm}BdCP?z?ppH-vZ71GY<meE6=6xhov=%?Z#$s(j%5LcYL@ndZZ@hIT
zW<;N>=tW(Ou1^MV4=RXL0TY+JS(?5Bv_A0VnX6DLJRAy5W?1~>AsXbOX2769><B%n
z(~Ymcdy!?`Kxo0_VGT5a+e~en67iTVB_xy)!Z~Z=Q;!UFx8ANrJL8$!S0CAlaAR(e
zK{*d|n^YpCfAMiDU%mME<F4s$;dsrbq9>HJYHiF>fGV3h0NK-o-^i=Sxm8(R$05eU
z_pY;HIh1L1&S`-{Q0FcD3Y-kcB@PKeg<dOlL8b64((wD#`sC1!WQLkZGlin7A2|ZT
zzj|o%fCi*EL^Fth+#nMrRKBc3kdm6QUfI*DA@=bQ`5>5$XlYu~#~W!f{4ae1u<ulh
zb3e>xsw{rg>$$M8E;o+vf%%G6?_Gtcw`Vye-}VQ=DN^(YJ<ql+ADeK0_j8@LI!EN$
zvJV3C>#;uKy$4cB)o(M^^2^F9`6t+o!qemhw&*NG;!Tg2x@tWWP$PWed7i^rbg)dj
zSIB<bXKnng^tQCrGcnTY>j{T=@w7Yf=v#*c4VNqlp1@Dj1>{*jf+~O7+MdYfrf|ah
z*b^t6{^-8#VB>s{DCdBNn}GF8bLT+{7U<k3Mw&hw?I~n8@A;yh)U*JW!3j{evm%0-
z${P4^e##x&sFL}SS%imlob;PVvpYsACl}a6JkuX{B~gs#>5`u=5R2zd-g{<Lksryw
z9$#P#oa`G`MnAvZmv<Iwg(F?Wqg8ckN8!|sJig)>XUD?=iK?W9z5Nn&E0toiDUe7V
z$Eio$S>ZzwU6uV3Fwv&l%bn?9)e3f=L$!&tfi11?4i7Dm&W4ZMMb;tZq***&zT?)j
zKt<VE#alH)H!nQ?PB=tt=D%)%zEr{9Kh#x2r#0MT)CJQs@<iJvHF=mMC%DITbFq;s
z5sOFIN^GzVNajTDHJ<}Um|3`3C41gbORGr>u`_U)oFtf}<SL45T-baNJ?`fe$=^&O
z+W(rdZGoPWbc}^tWjQBMmXzbVDsW#^Z3a=gN|LRNrAZ!*I}`i_<Y72I<ETF>-~Bs(
z|IDzy*h|fwAQOq%hs1CtIl<qf>9hA`>Gt}arhy{Do4*x$HCr`kt@jwCwzutg6Rbw)
zjzEeoR9iU+og_WNDW_nCPJK!$sohTfV7^g#B9CuScgv!8w%d*GcP&o$qiOLi@|jE-
zqMgenemPR}t3l#c>9FCh9>EF=U_}{XZN4tsoSzM*sKALd6=IsM>T?qM?6Dj|zuoGG
zY(`6mXB3DYUc$>Q62Ygs@@1`lKx!kX_9bpVJlp)Vc}L)<yl=zHB=bm|LJN|9I3imm
zPINZg{!WY{@HLd#+T}TZIU2OD#>Lt2^ZT_=+WBH0(B_<}rIyt306z#>)JJ92qb4KJ
zGuBm>pvAWCrI%D;K0`IoII5X(5?&Uc{dX;|dy!B2Gs3MPCe*cmyS-IVacI1ttbBc~
z0DlpPav|HebUZ02NjoZ_$Zx@edi0}-JR3=>K0h}c5$$BaX3MAA0_o)Els5~@JGKh0
zd?~FZV?V}aI%@Ehw`uuuB~N~GmxbXwcN8T37VpUQDhsAH=e7GCEBJpf;`(^oI29SR
zq(qkJx58UA6g%Hce)$X#^~2j3-QtpXf{&G&{DMbEW@;CsK+TD+b_)T0rsXzd?Q+*l
z<(W_^D#(h%R>ONW|8Y=K5Rg#5H284*Hb%xmh}N0&(|~!kl@Db9t{<(9M>ZduegPhC
zHi(Q>Th7iVAT$yBmcZr)DDqvn=O>_x_~Pd_Q#`?0asXOFQpp8I>D0Xlh}6u0ZXFqL
z+KSSXWocj&`jU$6{qVOQNBnXaWI}d@n;~zeV}qKlO*`3tla0}*<ry-?Hv@%j&c$gn
zHLJ+gSHuU$E)Dl;h99iO8Fxfuw57W~QlEUWNkhP^{@0{4r_@vJB`p|P1%?(1=Alrc
z3K=7|q(uF)Cl5GpdAhc~&_~y2aYuw-1?@LKp1GXYtb7>3PNyR&c?X*)(vgv7IGlGo
z;D6!k=D3hfxmAuo@6naK;vU=qNk4i+QombLM>-ffaDz+d4%n;V81g~*Be;ju{*~Qp
z=l9A`uG|v~k@YW2?LmSQoW}{t@Njnv4|Qs~xptBNqA><%+wu^(;Z4~<*apD$(ThxT
z*S7Kj0kh-=9fA28t;otR4z3i#XQgP7Zyx0ta<V{T**?TQa2E=EW}>oy9}R=#O93Mm
zr&XKxt8fErWL<HCw|4W@V9xQ`oF$(Cf=M4zOm5+)HEh!>OXWSrr{fm}O9L{;a?Fv=
zPT1H6aIaXiuJED)Vi$b%+yB!CykDKYi&7ELf}%5EG07%MsJ>Cm5P$D18!}VXOM^Z6
zK06uEInm@AHmdR$L@e4l#f_)0i`&S4e;v(vH0K8ix1`3lMNQ~>uyy@w?-^tnbjZat
zt~{@gBq9h%Ms`wau_B>tO|PD4J1+(B_YD@v&n{^EfOc_-1VmM7w6x$EwT$T}!NW}*
zGdC<mPzq01)i-qdEpCF}Bofwo`%4I%X;GgP3H1sIE~PQYjG}q-eamPR3I4Dr_A@&N
zA~JU6@?|2{yk)~t*JU7?<xlMS4W`9;C?#@xLs(5!BD~KbWj{U&>>HXqms9|mRRcTg
zb?3?WMBCFU4Va3}q7U+Vrbm$xwInaF*v-5x#KU^{81dqKT3YzQk0gFh$K(>p5H9^z
z8t6O%JbAst&f`L*?F-WRIbWfSD(6&M)cc;MgQlWG>xCbdh=x+K%5;q`zWH_xCGta&
zng>EV%a>nLph)m#8~OIkwerm0|4sA!Z(UPgJE{b<bL*uDqI!+iabzhuz9qzovkrpc
z)_7f|R6{P7mTnGc>s!?=LmZUdx}Boem(Qk=-=13sL}KE~Q5VYlKpGs+HK>`;pzhcH
z$PeGogz<auqasqRfAZ&phYPL%@@$+F?YjA%pW<V+!_S95LFBa27f{8&f!)UGI40!G
zCB!fJ?Si&W!kfy%UxA7jfcul>Tf{u0vIY{otB&df+HSZcFe?tTS^iiJ+_BPwha<L=
z6l1!;MY(qr-21i=RJ}x(p5}orh2|BqvC*SzI)wFP&@+AWbRmZJS)I`besN}oqajEA
z@EB!3*C|UzY-_iafNG5J!RN>YPj+Ouc0unalFIA$!ekj+$KzzI)Wu3r#Jz(FNH<EL
zZ3d(-mm^7T(5pFk@H=7FV!olaD{nIZ4d^n*+o+0{B98}zeIn1I6K-%$$>wo0sdcEV
zK{>Ot))yyJ9{%d?upP^dpdfQa8GT}gKE=5$f5_SO%6*R&bT7(M+i1|qc>1p#@WZ}H
z`i8xO8}u8t2Fu^I6Mnv!&I7Mv<6E>`SrO6vmL$CYU6XT&d)17kpp=8sdyg=};h7+p
ziOvzdpA7Dt(t}u6_$W2PddAB=3ms8^q2PmF8X6{@GHQW0BX@sW*fdZsWYu$(jFX8f
zZN9JcWx&l|cRdvp_GBj|N-K6RfH?sq7c*2whld*->F!QmeBm3p7o)CA5~3pz$#Ld^
zipaCzZWo2~Mw_|m=2Q#A3MpYPm$|v`B^4wp%D&TB%LgJ9)9|a8$tE}U-SxeSJ5j8+
zkkBoH1dq##$wFnoEeu^Dpa~1Xqpwq;F6GjYnmxVE_>2V4HtDmNrO~$B8Rn2Jm)vkD
z?FvQ1g4ZDM`nCF<#u@0$FyZcS6570M+2#8rldSrQ{jQab0}7WP(XKGi-GH}h156AX
zosellXu+ua=GUMuQ(7eWqf`@vkE!3%x~iJ}oY_q8)#(CHEWV_s6GrThqyfkl<#W2O
zTbawJ*+xrt%>WHDwObrUt3;_jeF{utcE0KjGH_7P(+)C_B>j9~_YmRC&^gG*>@#aI
z6bR}4noeXWi24diSW#*Ue%faRMebek_f8T&8nn8A>NMaOdBh>EbA&p)q^pa2C`7cn
zW_AVYP?@~XzobdCS+HrYG&a-Tk7Ui4I&bG@i)m0D0g`U4Q!Kb&^FoCZeSJkAN<gIh
z>Koj_tMpzQVJ{#x@v9$HPU!Z%W<^6>7S*dgg^u8u{Usn!*<jMCdq)y&I=4Sukz&Nr
z4`#;D*rN!v|5S&cVFNdPJR|{?S{mZn_kH*jn-BDPxkXHgv|1!e29gB@?fT*XCgiON
zqLXs<)Xse>jgPEYqrpBb-L8*-L7>N?V0(Av(YyxWa5J5VENMc7KgUivxUkq7&U{@b
zFn8}>HW^5IA7Ix+J5?m2kq3&RiV|A~_CPJm+L!JYz<J}%UdOe{Q6;dEWG6NypDH38
zFao$KYcIXra=*<{+e6k4DV7iQKs=dz@Y`VG8g#X_v{UIgysddZl=6WIPB8&?A2s}x
z4vyM#^C^zGQX`3>*$o-065x5;Az)H_qsBh8z%U4)r^M#K_8#zM6sf%3l7Or7aqpCQ
zLOX;+*FoEvFgD9<qR5clv8qn`!<^^U-ZGC874|#=Hi~z<GjhTKgk74LXoWJ!sTZh=
zLJP7znv~A;9-a|lezNW;BWKfQtQ*Zh1RK_BMmmWp==mV$4AcOlv0RU<Q~s-g^;LP%
z8$b@z3Kg{)WMTsW25L@V97t+kEUu7AKc>VEltS2l%6WWpM7BVpz2Ns$vrNni^jtM}
zUi!cqHkhCu--Uwt|H4WJ;r~dCjN7MSw5AGO+RiT?xGWow#RrNdtq?E-rMspzb((6k
zxu09Ji*2O1S<;CF;Q!$e7=U7QENZ-gWW@Vn3NYZdn>U<MV^0FB%W<86jsTsKQI2^$
zetW4cHcHD_O{6MD2AE)d-Gab=xH$GmAlh)B5}!W8{#bNKR6)B>$P`o`y79C4TtT;g
z>9ew2$nJ;|z{Z~(fR`Bt>dL4`pHhR^X>%Tihx`2~+~601H&a9HqJS2|etS)yag%H_
zZ%=kkTc*{|Y|s`8P4fEr7!7eZ)x>GmMnc%@);*5py9}uimVV5*eNorc*(2;j8>}68
z;v$(tOQP%_7r|PfwU^0&?R>rzq{FcJ2qWc(#}V$?^G?#D`h5XKL0}L#&_qGJJUf6=
zQnzy6o96@LC>Vo|_mVj<=|Ed&=un^D!pK;z88qpG7mhnT;v&z7z1^1SE|Jbwhaqq!
zzb_tzR86hAfSyzEm&mFaSCfiddDkS+xm>8VtZleZqSC)}kC}(g(p&y($KRiBK$xN}
zRjo9zyU&G@**EOQeB1zWZttm?lu`nyGMFt4Fk6R!|7P)yy<h11SgZgK=f2n>(PD_j
zrqTAq%;tiH0HuZEqQ+(g?E%(h*H{Z5i`qr_L3>Res0&Ij^k>9TIOz=L8(dy*UkWjZ
zN>eSa7}+SOeVPHSqmF%6D7+8A&qq@QJq`8tTe$Uei(8#UdgKXCUul{3nzUcQV$8G&
zdL;$Oc!;#EpAFPK*du1shgGe<bU<IG4e@^U1iJ<_WVnT{C0`7~`#kTD1icM;5*HYi
zN)F@z6E1bd(&k6&B+DtAaW8)U6)l*=6cg&K3m|WWHqLOb7lDQ=-_cAEEbZqRKK4Ne
z#fVfl>plG9p)5A_cb{l+Xp-CYvl~(pUBM%_XxOHT`@s0xsTA&;2hRTfgNr139K~%Y
zm!M-IRTEpbD|&s{o{s-b%K9u}yxZ+J?FT*@!pLt2+^r~y0cP4ZmzO)m#@{$|o#tIO
zG~#Q<$4u%lcqU=HKi>4mvxih%jVjAK?BfaHspP9Yt<c3snH1K$SS3BCS^w)fG$fZ~
zUKy29gFava{(k)0_g+BXOwuDpMm5M;GU9GG<YsZ+uify?60oum>A;t!Vl$s)+i+YH
zG+1czcp<9bFY7;y^&a#fbea-t<y^Wc-OP6#gO(JTyP1G)=^G0qdxzvEd;X6yR1k#A
z=cJMy?+5SU5vYPjHdFV5XLJsXw;^M1!xe+|Vp()Zk>`CoBXFO@et|7E91k!N$J#v9
ztuojA2BS3}h(kjJcZCpC*J-ZMGh(+qIetjU-1FM<6EaB8xdshtJ#3D<lat4oHNICa
zqs=Pt;g#)tl;mee4QmKk1-F!=jOJQkcx)64wW>cb(`#LT@*X!XxfuQJU`pZ6cp+JR
zvIru4KvQ<Skc;iEqsfj{YB%UhHjN^}gU)~7gaXD><dqz71_Rn%=d8aq&@Hs5QH8GV
zS*G0HHy9O0Px##Pvm2}Co2=5supOakU}^{_B-j|dFJo#WiY7ll7a;bR>84LPUK#MY
zzo8%=+{?F!MJc4AEjkfEfcBjXsyp$G(xj{sdN5b=E=0z=1%sx+{Nh131GeO;n}IiH
zbw|Qq@}!=J$e*4f=DS^@NKm@G86c`Qc;kbEtvw~(-r|Fh+&fsHUP?*?#7ZT!s4C4z
z#n|vZ_XFyV)XKNm_LFr#h$|OHC3o{F6xf&#TuF=)jEX)sgPClhC$dFFl#W;cUZL<#
zV<<das>jk7qG=qH2GEGHD0)dir^CVupGk)e6D`yzO{XHRBSbS&Bx2=M^6@tF;2e3x
zh~2}?l-eA}todnhxh!_gkby<>J%`yNToDv3c||)WB>17ha^S#_t;xPNx#yn}ryh@a
z-BA#)tLZ#HAd_3ecB@PbmyVr4$1Ls(j7ntXla$TboRi@3WH$}b_X^WFZO{oQ|E&JC
z#z@Z8o4otyDNv}js+ZX#C#$gLO`m}O2ggc)2bseDqi&<C!*|$XyRy0K6;r!sv2J6>
z&DlLhf4L*&dPl{AC%(XKIvT8MnRHtRbgNDWcTfT0=IbR@O?Cxn$hu-7{seOOcH>;j
zFt-qp8>3FX!l*7UdSfNRCH}vQmnpQxm3S<KbYKDC{k3Mn!$IM^=u>nM#`H~aJ0~Lv
zOi}dj&PzZK7c(gnPM$XnV^>~c_A>he3ga8|x@I#y%<#l6mqfv22g|*G1`NJYrC@Y-
z=P9V-%!QM0G~!|UKu4XpFAYw`Ub^BlEv(t9rhFd(g=(--k4C06fmMFGA=1gc*Cf?(
z`aS6bM$q;j1tD6|^m|z51l&WC8ON%(AHaAh7Cq(EdZ2`Z0y#6(P@Twp!P0Qprf}`&
z*)t2AM85X#Gh1)u-!@C!1X3I22IRPzqaoUg4gyz7$*RBNBE!CywQSh3Js24p?3NS3
zMxB6pkRYAW5NC@>5523#)U-O+e*rV-lA69)9%xr@1JA+hdm|$b`%0>K^2!HgKo3`v
z2JwN-&tO-iv-eoaS+wQkaK@_nWWAA%qB~bPTL3j`<yWK1ayu=GP>s|JZLF_gT&pW2
zZ$JaRYu!A_v-Rr1`+F4-uh1yzbh6`Q$-=qKA9a2v-05A>0qT-N8o-b!hjbX#*>Mbp
z7jY2rr+?#f{~PMPZ1-7Z@{RHPW^nm?D~i>sJnH_=sVKt2BWl%UUBO1(&Uf101g9Q|
z#J{qtxo13^kl;br`1mdPCE$ZQ;2t3D7tCc~aqWOs;ufGDTX=GO&)VO?lb;YAE{TwN
z8gxJLgFz+@R2NEkVwY_I=tzjg!ywCSP=ll8&Pz~gj)2ZuS)eug@x5m#2Q*WP2L%s=
zg!Nk>7_^jnUx9&lT)T)Fl$<FuA4YgArh~r*9`Rn{rTE&$)B{V}_U+BpP7ITJXm5vf
zyv8$DEzL|oh1bRyDSCkaO|!jkSeAG0+mbLs{@VR*1M?As>w~GqJHUoOaA)q}YlM&T
z?M!8DHYlzH+)D8J7@md(3YXr6Lo2>{JSeJXq`D07O!MC5<q9lTBiTu|p<Tz_K)77a
z$WfuJMU-9?K!UGLHL<}sq|jGDG0+A&=>465g{7g44Z`P`z)u)b$>C&-gVLWOJmabz
zq=Vp2`DfHfoC0e;URWQt-9S8hdt*z*=NhK~qKx}n6>`l~*dALSgxA6kGJ#ueq-B<x
z9=2L)vj&6uawR$P^C8~@zdLVUo*AGc^=XY|y-G-rf~Lfj9==kz1(JRdxYJ^Bw7h#b
zQ>O9r1V9Jw3*@!WD~>A>yyKvd$2GbHYF1aB6Wq8+gS-3hA&HRFj7XDP(4M4!Es@#~
z09%mQoGi_l;8~eE078&M5oyYpx%Hnx6&U^k{jVjB^d;CYg4Q4jr~xRWt6W18<%;A*
zF2uCC!Yzl7KkpJ{*tCM`m9YzjWYQgb0O%3<YElU3c87`_c?%O-R>Bc6$0e>ckr1L8
zMdJq>(YPorhj^64`u;OTI`yAz;Lu_B_Y=^|ep$^(Ao#_=?KA^-YXBVkw=<1wau-0P
zzV~a!yjH~Sx=f&cMMb5vM;?d)9L+x~O~*pX<+0HL#FZzg+Fg18oPQbn|4{bc@m#-e
z|1gqWL}eB4tg^}8RCY+RGBUCWAzMcFmc0wvD_i!?EV5;fki9~<kJtCQuKSPszOR0d
z`=3vrM@et5*Lfc2aXi;ay3N4og=Pa_{m}tv4Pq(2aC9TUx(e15oP3pz3ua`!BSkus
zc|S*W98*Y&BxuMkBjcW$(Pe&W=z;;42?_T3J6Oe$>SfCEo3L6Q<6V%xvkBEeyF&p0
zQ%$BTAR{2qv9*I$Bv0OVqTzKv$4=Y&Dc~ob;Tpr&{a#fM^l0}HbvUoO$<Bzu%;ml8
zJo(X$j45LZFqF&_+&xs$N<+xi45r}IEdccI(KoGq``{xAl%C3i+r6?4Fs2M~_El^>
z<QIO~MN|(*m}G3=={y_^X4?-bJ_-j&5G4taW?o>4(W7h24X|6zEKlFCi{>r|u7LQ$
z6T7LypYKb0e|6O)3>go5tic{C7eUr`RO(A6V5;<k7oN{1V@cvZh~^`S<riE6j+gp{
zwtj#T!`OUt@B7HUu+8m2bL)owxKl2-RrmMH|C0q^M2w?%;D$qfdbp8PE8S7Xx`{Y+
z(iFw4Po2&KTUDsH*q<B&?C{(f<%OC9nCEa;+tWffxe~^_G~?>HM(_giIC(phVHLQa
z?0>K3)jzPG(FaLe>86U!*UpdR+viwpl7UUlJX7IwHEI;wMIWIMjaY@Y?mZ0*K1$H|
zIV~Mfrt!dYK=y759UH|KqtD-{F@rgDvnDQkVu!sH6JW#I@mbLFL48D1sX>>=O0-|?
z2X2<<3rb$%yEFJ1Cy6WL*#B%gJ^8LW_1xR1XC{!?j+Ca@p1ZHuX_z(hn3gq8P_O+b
z|I%`hmSx9fjX;adsT#JDXEK)_?rLvb?O)PDp?ENTgqEtOX^{iCod`Go8&<WXl{-GB
z-ml5x7mZyv7QzIU$<E+}P@VN&lqN39&M1-JbrXOuc!y_5Qyo;SRyZN3-GTiF_#!@|
z_)FJD0G#@wc#BM3ZrO$myH!hkN|Z|2spMTpZaJX1i4HZg0FHnp>2A$Lm__DR&BC{b
zgngREPpF;jA~4B>u$|k-=S3e7*MfZY8JpUx7|eoH>g<&9`4f#8PSZ;StCq!~)tjwE
zEz(W#m8_&>bo4|`BCMEJ9Q_ku>Kk39EHKAmr{50R1M>}+(XF&XC7WK@;2ruzmwaRV
zkIDRuo`zHCpW$9&c6yG-Ox+2x1XK2-REgX3PSh??ExcbUYU#!0TaEWiB-34~W)E&X
z>vaKLv%Xt=LiYNW8$RYMSaghkbN9)P$cigBI_lWnH}RRyVL58VCA|}$1sqfM=fIM=
zD+q45Q<L1VdT-Qp_pS(-y^soYo3Esw8Q2D7v&8f}jr(d=oS%!&1L~$RGMX1(_oOR1
zZ4kwl9U^KU{~X{zV8<KczOap#|5;w`rPSpGmvfMhp`L!W$_PKzoBYfn>W*D!*X%&A
ze%mI^)75^8>knR>_G{Vx`$OwKjt>XX6vfNaQj7Z<+VuEyxCnk&b>|h>qUa8<iGu4!
zW4azgcrHes;uRb#k4}kb$X5Tfc;A;u3U>Gc(wk4hjPj?KR_98GOoZl?za5ZM2m6b6
zS!hw?VMbl(nog32bT<6U#el+YyG3N8ckucE3hlK<Ex+#)$yc|2<i4+dTwtHF*)OPg
zJU)Lj32JGI+dG8bJy3v?UE}*8kKWR2vSXw5&K$$R;g8BDT2-An!Ai6*+}t|yzA0M~
z^c&tNgu}7AMf=WeHTyQ{D%94~nb-^{;l76LVg)&D8)9TbGT!;jc4X`=O}fON6jRK9
zdH|19@G6vdZ2m1wd#hjvluP`=z)H|%*&Wl{ch7Pq#j%p<ZW&wr3%y^zD75R7ultDW
z{m4-+>MM%a1Q}Rhs7Oe^9``bB8|Snw4^@|{l8vMP?Kc6T5)7n1Gk{`hTqBnvk(jQj
z<e)hWB~*JkWf06-(Csgdi}gqpLydc2cx2y>h@%>-w{R8KxBrtz6!T=|FGm#2-)ZJy
zc!6VCgB=uEc<$X~1OqV%)lz4EM1%gMH^`VLT?Y49iJJC~7?Lave!g*1=~-s%&LSyl
zJa<`topJKOr2&DibWmtpJe7tx+Rb^ZGk7A!nRe%(`YxZp|E_NkO(JrAZy(rQIuC{b
zT0dAp-IAznL_iM`v*)%re??$UKB#lv%X}u6Rpmb5|9Z|@g%~=i8;nvtJaG%63dEoi
zcCZMs8_u{s@Y87_l=#<3rZ4&%ILDK1qIFH4;5sorrg*#RO5$`=zBVlCFZjHE_fdWY
zamrkMVRdRPR)F7v-a{{Oj|S3?Rgw;)wExZd_+K4i|1BFuvi0`0)$Fv*ldFyjf^PBQ
zh9zg9z$lhouiZoD3Uia|*Mh%RqcPVN>DEu&NW@#+o)^yB;7<ecP|d>r3LxwsRg+pK
zSUDd&Oj#&+x(>kK%db%d%r*WMocv+Kvu{sU@2cl8XSU*HN;G6Ou=2s*>+8gpd~xPC
zJfYJH!YPJ4Y&EhNOgH|@_U3GHv{W(69))m6nB|d`vjIJsnX0IEpFBoL@X$K7$qSkg
zSLnJ@x>?J`(WJmT6|%4}<)3wgU$-uEh3Oks0a{wBb&-1-)^>Fo|J*y0f}VcW;(a>+
zMu{5Qbo9@?@gbKj_!(7oW8bpEo%+IrQ}8uxf0ipN+;M@KI(ZU5gD38Wl@Bk{2=*HR
zn6OQ{Nb1PU(^B3(_DS~`WSI|HuKuORGI_f2G5rGM8dk#>ur3k9t7<~Rna6iY(#yj)
ztw3Muk-f{czOaCdeezfM?kc5P{*2rHxWXlV%hP1ITG*2h-Dzp(T12?Q@!*kq^;V#v
zOlqMbsjkz9oUhx&v8*$eotdMb5aiGF=h8GKzUUvEN~t}f0l8Mt73%e+Sh}^i(tSM@
zn_8Fnk<7Z)?e;s-qA2ps&&2(%o*VgaG*~H!775utYArg5c6s89?Ojc!tG!OacRN+x
zEpTDmW9;aQ&J=aEi_IUvXAEcXWwz!r>KfWMs5=G*j^yVbC;zMSW1<CPLb0;(drJ;>
zzZlMVH6*zpCmj(XgI?}CQ@DWLat;VLMwnh{(Ct0YxI)Rx^4tFRLPxbY7?;`K)S|qO
zs|!GTe*LfgL}K{0?rLi_(8t8(X_kFbp;`^>#dk3h&1Bq@5InOPai=AEGL)?};L?AX
z&MB^fPs&yv<N?qQ(*k-6(<>Lc@>N9NyWINhWKQGMSk>JDIFvTI_f*bJp6K0VS3gnX
z{p7<ycL0{iKeb9kU$mR4EHBDxA8#jwGva=}g*yN8t|1Jb>dP6?@`E4c4Z(E?GV2?l
zgUP<vYb1+V*3v*B>Q)_B06^En;UXQO@9|#?x&US<>Xl0bNsmYPZ6P)CNKe%%%gApx
z4WC4BL?3XLMO7lLH_Za2m*;^4tGe;BVa0u{g6;yjIOgXgc9d4-k@uPFn15&z_cGab
zJJAL78gxf9c2^gLhmhVxpKAZ}8rPWqrO`VUHd&Ri+7RhITMhC{l-c(qy9zuDL+-T*
zoaJVx=B0X^n62!~n1&cf?@Xolb~>!M*|>B1Rraq&5{<4DR!+zz#3<A5HM<95R;8nr
z*8^AF?Q|3Gs`KQ$V7M*q6{#ONHta-qXFY775i|!QQ}BBh3vI(E@wL|7F84c|C48_L
z4QJzMrS=&O+X`$5iEs3p?{+&m1qH<(2rutKG>7QAz@*RK%7A*czrz)&Zy%^&;>}*{
zz~ORmw~@0X{s~8YcR;@OdvzHTOpNXVju&$pQ`BJUY;185WkM?kQ=_D0kMneZ_4U*j
zA#wT1JFipP&M-Hb>6)E*pGukDNbtReb8n&j-hJI?nA(muO6b!aSELFun*>(5n2WTG
zBfYBI?-C=-5lCncHYP{OwP*=ctBb2}JJ7zFyG<_o+79KZG8<tA@qQl-9D?fmWAEKg
zNM(ObHFn5oeU~fgs`k$sJCmJ?ZYSy|pb<&kOZGkDcAD`$t;j)H74K+#1jFYo)yHdp
z_5E*u3UR!QSUx>N6`D@(E{Jq6nD_G=a}r*@+Mme7Y*JV99=z%*9q$#!%<>niN9GeL
zZ)6^ofi&$2OlE4PWpwtxzDd*rGt#eFJv|^usxkWxV5w(Pvs+m=rP>O6X}Hx4#k9Pf
zF3fKRbiL_5iC@$muZ#c#ks|p+m&sq63;JR9HI3bg3E4g!Jz*_EB6Z<0Si(Y85c4rv
zaqFPjj=ACT__X}X%4VEG!xxvfMNGjcoS><B(c#2FbzZQ*Mf1=6fk!A=%w+XHoH0Q$
zH}f0h9s<5p&}})G8CZVLN_5Sv;5B(wO3gf}%R%!)LXl^0kC);MEC;#B#60SFq>c|!
z5&7&<qK<%UWw%~;y#OPBS;w$HHhO`jX~?_6p5C?rGR0p|7e(I)agmVGK3)-gIg<OW
zv8UY!P5Y~1KP&gNXiq2J!JwXK(R2Oz9v?JeHW`lU#VGkqg^@~K#}eX3y1yWmpKxot
zS)uN&I#up_h|(kYMsJfnNP!+tW<8}s{?Ni!$5MUwXi1_8hqBFza3O><hj@Y0{KvhO
zlN5&$&H3F>n-*vK$JF^LVxYY5-ymF%qYKMOqeD-H+GrHQK`*)f^Z><yxShU4j*(3X
zE>P{qaXz^@dWpOVUn;_6Y@{>(fhLgYnlCl=9h!DvOr@bz_B6@UHd@@%MgYD%Y_G!&
zVCG+9KBb|;4xFp!BsB^B!x^wv-hLLtsW+f0RkS-a0Mb`8MF$*Wa*1435h(NBn-bzK
zTcM6`f$ylZfAwyIko^9mXFtG|ZKbNHalG0|pDGqI+omNlxc)5U6p<L_3=EPOIviF5
z@y6*4T-Q*U{RH2~4{fK`9U<h34)RJbPgC!aWMq0I`NnXcd{d8Ti0dBd2jR|Gjf>Rc
zMjO5hQNU3jyH?{48Rx%zdMNB5ze~sYGNPgBy>cJ>*~5L_acmwa{BWZrZOSU%sZCcB
zSh{|NHkj@~a=7yn9Z^K>8ZzfEsB}A4`^NbpvI%WMGa<JucCZ}+WWubFmUUDg-BU^J
z;SLmTcT_)CtwfRqMC<hBg?=;+za=mjgnu;cTAXu>?tIRlnmZ(^&-}HT5!MDvTje(l
z^fHV}GHWfv;(Kmc^8(z9fFzZy*#W9cMwT>OQrI1A?pnDgx*1A8YF+lUES?eu=Ws63
zj<R^CX~r8~p3iz^y_%lqC93+-3XDR0Q3dX;qYx7+>rpbCysSIjO=NZ<gkGLIi&=O6
z_VJpGQ?-uyc|IeCHr`PAw&l7xN%4R2mwI)X&U1vA`=V{3PKy2QJPRAL_qEG+sw25B
zw2e$s-}k3g`hh^;ony^Tc?KX*?3*f#9e{6D<XuYbp=-|Go^;B7T@<wQ#(a3Xms7>O
z(Hb22CL#ApD7JhjoUpDhFw`qa)o)J=tu}C^5^>!LIHTNZe_xbsbuP3~{yxrE7lWvT
z!^>*C5<KQS(`Mc@23DML4?In@)Lew|Fn5mtVszXjar`$ieeEh;l-WpJ@t-Do9*DOH
z7b{RZh$~VI;p+!$qszz}nSgsdY@X^qCaKO0E7Sx%q0BZ{GGa3yYG*tHaL<(vTKxqE
ztzO4fb06`}YLA<v5ARF4e`3Y<LJKj`$6O|<GWgo#QEnWbpybs=8!kMbMVP58E)`B?
zr5X+noR`>rbNBcIh6@S#XVG+k$;{u~>nnYLhgm|C&iyz=aG0`)h*ho?Fi{g#Y9P!q
zB`tvA;O!XHPcBw3PAwOD%IxdEWNqh$xFrOipoq>`?(z_^ND^W^HPnh29Id#qfX2Kl
z-wK%Eb%CQ}A{>HS{2ZTeC6%55W*3<F7n)v|Tz>_fjQMClG8pkN1?_*$hKik?9FW+O
z2WAKxAc+Yd@4GdA@%~V5Q4KLNg(cuxxAes!n}LZW-okDqGdS_C_a5b|>bXSjBB9EG
zXA%kelMpa5S%H^pAAj1ido^qeko=GLE$gpiE=gK7<Bx8`iHYPU@qg}9cDT2^2c9!W
zUy~GjB8(FkwJ01R@1S<gx{iwQ%p=~Md?kqx3!#W2=e=c$RC4#rZvL|K?gX3iI;{ZC
z%n7G55(Rj4L~n<i1+xE}nO48LSiuoG?3;DR{NNE;zCjF`tVYf2ET~<Td%=qw#PXr|
zbXIaSth(|36+DwW;C!9UzzK1pyORcT0j0cWxy1SGWe4m8_E}ySUO_m#uckDn*vC)s
zItAbS)+puhe*tc73xfmvd`Ho+(LB{2^w1#LUB>@#l=b_ULX_#6N8*<PXu~3|lO(=r
z_P%@q4?w1`5y9=JU-z)q&(bVu-?@x2NH~c`ng3XhQ$~sX#z&Ut<y~uu^fz$8ojYLh
zWr!2!5bDL+NFj4x_VsK89qvYOyoa6TeSuG*NyhhmR`-|?!(v>IHt2Xo?4pL(YavzX
zk|DK0A>WJ}*XsgU0{d&j%%79>uJ%*_hT?cT^1}LaHysfXdZhS#HBi?KL`~+)5Xg4W
zVrX+K-%$Qin@vqea2tx+q1Eno9mes=d$L%=(jaZCEB(qSqJE|q`8E%KG2rrIpa)8b
zu3-@hH8=&kLFNvf*=JEOlob4-+^&9B8Bdb+Ak$v&&L{qzHj;XM2S6gTOt#7im0Dzv
zI>7_~>si(QBd5`5dy$M?f76+jBU~qUZ+@n{sp=KmfeUcfWwn~r6Iv4zH+%1k;7y~q
zTd^;!98uEmFiigot1@cy`upb1(dN&>7t4N<A<rWTsO>YGJJ4Ev=|o>N$xq%>xB$*s
zgZx+4!ZULypFcN%v#bS!?8!S4#cv#ot$2?(ViEcX@rggpNB02i_ykyh9~@Gi{Ot#q
zq+HS#RZyM{kGJlQx);2ozXCZek4nZaV}Lc}sO7eL|B@VbDcSZUY+xSI{^r)pq7?99
zv}PctCzo)My>8o^3e^0m9QxjVk$A$zO)8<`5lUX3#o^&T*n~HEeyYLuj-&{jRNIGY
zG+11>8`Q8_yUzU%H_56}b6P-4#;YkSPS+e?)wB*mK>C@D0jYm&s#$s0a#<>O?)2M-
zhxL_0lWwqpfGTI77eu0|Doad4q3S<WzHP%!k9&L_#}dq2wP2$<0cdO(+^i$8JsbSy
zej}1|YL{Pk&H$%07k=2pPu!6T<$IV&{{$So1(M&TP}{(!7b(nEV{bW*cNDI3<~s6`
zWl6?tJVbe#q~^S~>_?Jd!n!W_$S-5i{CPm~MHnIf(5A}fOq2PaLSzwbmnzkxq8R$w
z6q%w*td)D9j@U^Raeb~&rbk{`f3lX}ieJiaNJ~_`mY2`+nknVpw)k~FlU%A9J)bN4
zaCFr<U`#vCQ;prF8oaFbjbA(8ZG=%v7dmm$Ih^9|@!h_a(l;9utH9Oj{0Q4`dw*AH
z1X3<`rr83X1OFeb`>ep7MUS>Q27+ij*rWGv41F&Cmk_$kOXlh$DWdu(eZSGjM)b1o
z$i{OTO7#mKtPV*>xieWkEed1~7DuDHX&GX7|FkjwG6rJ82mOMV2VfuvAcRiWw^#3i
zciaSYOPbwdqeH_nYqzFI3Uw9KlSl_%sS^FOYYtQX*j^9Fv`j42;jbTXcHVmbT!2dI
z0!VBUyHW$Cwu>0}28iX{h~VL0I^0T7vdq!PY11&uMc>4djbGRQ(||-gG_z5%`xt`A
zM&|Uo_~dH=Q|lii{^(s)uD`F1TR-(`Lg*Mxsb^Xl{qmrFdl1e<yH@uMg*wpVu$dMA
z?xgqt<$0RmemE}Odhabvk-5WGb2T3$p!`+*niTe#1vwdNGk`+KcAybjl_;iFP#;Q<
zd<9Ve6zj;PvGUvAk7d)v#l3Y8%ZvU~S+=dJ@fsdlGF<`WQ+K4)_ePjI3kRw&t$1JA
zAVBXtT^x1GR(KSPU$zoRve!;`ZkkvEWHGJ&$!5-%)TLQ>&{}lpRrz<R6Ntvx9g&RB
zuYI$zb~~eo?aO2BPSCx6cdfsFYS12Z<D<(&NN(?a_w$Y^?^D?qbMUZ7fya}_bo`2;
zf~W>Y1OM~J_rKaLw>QxKul`>|vN|%?B#7O{uACw6hyH=v5)hFO2k$t&=n5cYqm*<i
zMiON+qG{qgpO$|S40`04_p`WqV9qD*ri<cD!|(#WPUyg-vLUwU(eQ_d*wzkT)f(NX
zPDdNh>6fKA@iDh;`Fp?7Q}ZJ>Pe$53#GNEB_9IA$Ngk_GYM8@5)(-#zCvv_)RR(18
zUUzBI_&`<^SiSrfM)_K>Lr)x;e{bf$mnl9AjpEJk7NB}7t-pME<Lcy_wI8z2?$TB6
z=+ZzW94Jpl+^$+!`~X%mn(ZcNB$GUowljADv<MPd6S{<lIYMJjOXO+7F56Tu`QuNh
z>p7To-#=rFsRRw7!1~fd)U3q~ja=*`Ah+mDv=-*s178pvF&pg06KDe}sDSzgGv?M*
z4Z^wW5HaMmaw!<$6Nx+u6~M!6as5f^uTQ{&yX{6Y98UC9?Gu`<*0Ut}B{#%hHc-=W
zg1z-N%0;y0K$Zeg@S4svllX9p$LriXZ^~epcK?Hi`1-GYDUzKGIJ#dnunYLd4~W1d
zT6pXwe%?qb=iIvgo9o}LUi768ciY%iTE&k_%{d72J}-sGSY%S+NrjL_@@!cywTq~c
zhv08EoW3$~3z9&2h7a3NUghg*zXhjw#J<@wSp*&?n5k))gEdiTM{5aZY2h<SBkHjj
zDP33OrgqIitP#GnsneW#4EB?)Fiv`d#)r~)jF36Rbpg==TCUApzurT4H>>=RxZ&d5
zwO9j<Bw#Odu8l+~q#!O|S>j0*bfBW2L(;=_AoA<FE~*4L+;!%n>IaCVLSmG32_my9
zej3mc{Q*9Dl)+}q6{ep0@C(4u`#&(;c;7_%qUgB8U%V;K<aVyOCxieU*6+tHIv0Zm
zMwW~v72+QU*>DcJ${|uHc#o};!159NL+_3z0LV$TD;jXPEhBdK2F~o6k&DOp;R{eE
zZ~H~hl7H1sxRUnZw$C<B{+Clcrva)%s%dFTcc&^h(7VzeGvzYZ@yfMA`XA8r8mDZv
zzi5h6&5;l;@MH|NO<)k(7NO51x7Y;OCBwdZ%N{Z$7SeNm=b!t|z!s(UlAT?XlE<9R
zgHr@l2?Y)UtC{b$XYRmNBmgB8IG>`_a`B;8b{Wo7P1#{A7f*+rCCOD*y(`CqZ|)Iw
zT}&++hGefOkBj`LfpTV(>HPYp5fn2N)OcAmCFT0Dzt}EgyfOR(5>bV6YVG?kpd<7E
zLykS0wPa<TWF5RK#LP*GGm@Dg=#Y10yV+{d0W&?NCVQ^<Y)(d45DV>9!q4ql;^`%T
z`K#6HHGA{8u`l0-&2$(RDn1LXjYd+FKH-BlOGCBSz5X%{5ZELDf=bVy_0m%wEdM1q
zqnn%i4bBa_cv&U}GcYKR?6^YDp%$Wn>#+g7v*u)?=pk}6{(4G`6GG0#P<MykX&HnT
z;ft>X!-H+92TA}P-}MvS%oTLpOuom4Bz%lI0+<yzvIBpgWYl)CD>86|lad;7HrUn=
zaW$P@eqk_ZdC>lM<)q&lFt_QtWJqu76>+E(UEwrP_=jy={3>0)R~vrU$*$|)KRc^i
zafVy1znGIg2ojvo=|hp#)&A~xt?F4Q{;b>yoKZxLrtHz;hz$7z6p*jayHIn{7SHi8
z@woVWLv`X7uH*b!Zv`+&sr9xi<7JqsPA{PBFn1UE;$U_BW?K*wQGmAih1-rV&LrYD
znoJA3e3-;-Mw3TEY}kI(>&Ep~1*OM_m1UQ2pS29!PS78;ISu0(>9c95)%(z_u`uz?
z1Tqq71UrFoNcT=A`oI-s;UL#@Xlr8C<hMX;|6ebpj<jkEk&8jBYA`8#zkt;Ip*zux
z{}C-&6V`E-wR-?z+xy@Zg4rF``q1%15OQMI!<TZI^b=Ui;HEl*FNL-yDRiLybpfcU
zDX3=MeCg$5e?jU4U6^d}P1ZJ5$T!mec+h6>U=VG~<9J>U@`vofHJhng6hXDc57U37
zDWxdTBX~v~$JgGZI5fj%dAXUZbK-3_M*6QdchNKc(mX-3F2ND_X8MJu3v59PT<p*t
z>w(i8{EJRjHK^k<e1gG0rW=>y9^hc2u99&YJW{)QV^88gFXDLCIRYE9y-v0vS#kto
zT$q;U5-VPT3h_uxR2T6dQqa^J;By4=!>5xRZ9=xi#b3K?nQ3mjfjJw^Zt)Gib3xqN
zAYzX^_$c1=iXb*gJRQPX=@%53+>@W6N^XYUg9GO($R6?rS6Hb}pN8Kt9RjK=p$ySF
z_PN@gmLZ^mEUE;OT9XawUd>?dfHsCbMQtA}Q_;w0sg?k1U0i_^mV0c&LFulei$~2N
zAB0g?VP$23RO3pB@K)?+@pKqnPGDX*@FPpjI{fMQV%;{@Li<rlYxT}fzsZmSpF%+c
z$f0i-y2Ag{t2Nq(%*kSFAChAckfC(B@70HmAHtq>T48GZGasy53Q4R9u4|TO{f2Fc
z{}k@YBwq?7jX1c>N^m#pgX7lpjonux1qs~p?oc4ryN3AC<@>7j_J4h!Fh1bZO5SB9
z%26v!Jk-x*Wr%%<1*P8j)#s(p3c}P=7E{%>26j)u*nB&_h8I^&y{cJ)7`uZX645lY
zOmR10=z2@A1(HP<U<@TkdpOjZunpANNT2?PWeeCw+}w6wTDQHMR45cd8sFujjBJk{
zS=5t84|fr(32w$Syr@(-MoOe{F*%N8LVKYhEbS6tunfZ&@&E^Yq1C&EX%MEx{r9`k
zcfbDTKazD>l;?*EINeun-PJ!!UaxL5<%Elh?g(j-gNyL<<s?U$eNd~fgNiW1#s7-*
z2M!ZCRg?`>GV_A;EoN9cgcL{D$CnE-!xU15HMP&cj3rhoh-e<>PxN<!Gq>?~e?ni+
zEqvn-fkrQSupp)pT}6yj?4t<?fN7hYNtf_>L^QAYa;ZB;-~t%%x;9RQyyAH4`vbSy
zVygco{Gw~~YC#t=8>IW*GtxVJ?z;c$PWKiPk)vf*q56^CM83UR4@%@jw#Bq(-SJJF
zDHzab6tCasyo}+Sl!VbB2Zv<Wi49C$-=Q?HEpl9sQSkC~y=WzSv35=7hqESe>zLbn
z1zy`)X`~SY%A<df;`AZM)yRI|gjJx$^iJ{KwIO0d+W*vkGM2jl!*P&2=&ZZ-NnDS$
zmlwkeOfgng82<Gi9{Nl5b0*kn{n&!mY#KS$AL;ntBPvNc`gp>$=k{L$N<HX9_e4Dg
zL{EPQ`Fx*KkKd;3n2lk5f<mo-hf|^sD(dHso0@B&eV?r{g)?P3;$3_tNdBi+zV}*K
zEN?+Rf%cRiNM`7tZh;hR?K1w&fc2{On_6^F8`0#QzPkWR-F855zsZr9a==~Uc(Cvh
zis(m8bR{D4j3yjB*E@Nc${fIKOaG5k{2m%M9=4z`noJAz+rJ+yhVv?xjn?3&V*9EH
zbkQ3HU0H`moPePc9l;no2=vnn1Ab(n{@fJ#0F!#@De6ge&DYUC+2Z0O{$P1<yW?k_
z*NY5u+PvU5JAVOLCENt-r6)S2PLb6RhnP|n4&x31Sg6~K^uimqdDv;vK%T4#UGOxp
ztTgM7;NT@=A@xFgC+&;Z6UR<UssK<Zjuj>b((6oMUUemU|0nq<ViX|TUr*jw$Zoq`
zQ@snP(sKR1TU-N(-Um`xqny%g;lmc+4heIGaq_F6ACN{`owr6AlDJI5*giGhv%X7a
zxQ{BSytscP2Od<36z=OJj-^jsmU+_S-!7bwbe?vm69W29i^Rj+#PopyMJ&f}4+_={
zILRSE<#mxt1PAV%yC9!!C<K~4ev<4IJiH|Y$2(5y>Ve`+HWa4QG5e4jrI3G7lwDq*
z7scl&eimqQ?clFBiLh(`Vgp-8^b8_x`V+imaG$M7IunF&>ww@%v|h0T%^0!uTi|j<
z)fF&GA-XX7WNa5s1@izcP>()v2aC`r0u3)}g4eTN7|gu-_v;}^^O0Moc)E?<`{H0Y
zsyE$n2pL3Mku*Mi7JhJhxPxQJtoI0XUSXm@M%#lpibwQMOL`;FUKSxAvX)d(v(py|
zhC~KUNZ3@Xo+0F0QfyA(UqfZuX~tAW{AY*?{$LJiy`!g>LL|%}3-R{<yi07aCa^^2
z=tnPbArW@13pWWgiy(~viSxTc%i607YR%#fJ4KlK?4dhR(e}krC%y;WKq|CnAokNi
z+A}Jg`^A!jGftaP?~p+8&x-$jD1Al_B?S}`O)DK1Q+><xdoG>c4dVjwl)@=Ub(Q}-
z0)R+gqIKoxpjEQV7;`^$<uJUEfRs9zU2E-EpGN&<^(%ch+_{t5s^0V^3gLPS#N5c{
z*dfLZ>d(paqzw>A!TMMYvO8D<w*a5GIm+3Ew$(Uyz@B^J2Z_q!_m4{To%Z;5{uNb;
z!tVCF<ZJcoN3M#`(SYJ4HkA|Vpk>0J(=5KnG0jK9T)V_(?m;LaiR}d_or)moGt-NN
zKpJdxdpuGlqLZXC>L3pIB~Ojc+%OG@q6;LwhMeg`dT$k#r@qehaJeO6c(bE%0E2gg
zij(I;+2KM0R4mB+JC+or)JoriqImj3+5K|s<pLDphY>`E-1v7_z8)ICG55tNG^y9h
z2R*_sAh+R%772$Sz<`ET0wg~BGl)zJfB(kdyTX<v-K$7DUUCbjexG9;4vNmR+nX6f
zHO6Itcf38uX^!I_!@-<(S!XrlbO~my4EQ-vQ_TUU9OablsbuE2$r;dlOb54c{@+y+
zGSDLp>C3|H&WfI91;tLQGzn>HkaxYP#o&L0rGHGi4#RK@J1!hqzkpnKddSk>BqUw>
zx;%c}VOUjNGu?vv@`G@WYbp&kp8Ac1#ab|}?)5WL<9`Dbwa2yo8{hT=+Joen{_6q#
z&&y3CWppyCAN}W*DOc|a+y3vtVhFInB!Z1M*#vpVjAtVDzt3PW$kKg215|P8(5Jgi
zL2bvVFF&eLW^rw@h7s)l0s0cB>lIHm=!x<<Y78EDd6N9jjd9fBWUsV`?nvjwwmzA`
zJy}m7dz?#j$q~`v<?y=)jTULmuK0B|+ONOTyL3g|nO4Uq{KElG`>{GEIH_QS!OPpM
z1Gv)qH?RKzZOE*iJqi~hxre&$7KF@J|A;3XyX;I~;d_@vHXb^Q>aZmgkrTwM(sN`F
zK8UV^bT)wnD4}nQaY5b*_X6CW7bO%PJ|?>$qLgn9xJFBJER8%5Do8Ya0SN0psqo_y
zvM*g3FsKdA<wL4{N|`7Tk35F(=pPVN?t&7(t`pioPhS~2FdXi-*K|J(3wKT47ySaj
zXV51?L{6#8lYIuKq>B~d^!lIeA2#5#bSWhL6XN(=v_2CY%_=K~JDDY=GXMYf4ooL&
z`z>N3*VK9vIP@Q!d>A`La!`**55}>SYbZ$<<x$%Zf!4V!?&1v%{;C4+_@g%5T+=0l
z<7IncZvO`0bgj6U(9zY-62Xsj-+aU+_oM#ukAf{R$hZA3kM*>$b)D?Ww>$o~h|3t(
zrM3pMf^%MR@Wx?56YlyJT@k)Q8|mPe^u;IABu=0zcP^_zMJ1|*zPW3tGv_5W%~dlp
zf!CR;?$8fV`o!pf|NRLmiw5fB&+WO^!bglX9e(yp-<gZ+27`{q*HAnyZEH>AyAA@e
z>|UOP#n&KOZGYqyHS^gt2&)j^1F~IuenR*KJC7F{gBuxZ=f+COpKUezD^9wEJ=C4l
za94`*HIozvwf?_$F*-SR5bl?}Qr~)2=<z#t;>mf;0f)gSav12jNBw%=NwwU11gJL6
zvg{-U`R2R`4VDaA_F)iFDE>GkD*XA+K86D{6@tUGBG+y<+3c7n<XZzgOn7C>xS^%m
zcEKKM1ztBcNd9WV``T$!Y5x!2Crz95?a=7jmUg*~!C^PxlC$gTFxed!yxnjy5p(``
zzNkxA`NezvAzdSQP25X-ZiQi0GJ*<o>SP-fP(-0Az=@~`dw_M5A@33~oI)cPgSK(3
zA|Z@=kLYx2sLq0fKgUF_YrreW83ka@KQH(JYz@OIAxeh-kh4@W1g(*lAbm#a4uRSm
zFEoO8bm&r207aJGSh+6{e@;0~MfV{BeCfJFlQiQ4apLLFf)Q7%nB8I7slXGc8<u?t
zid@=LFAVfA&g^Xdi1GjWWemhIvFKDfhyE@WZcT~2`r;>q?j#=wfRJ0#2!f471oo=T
zEE*Po@ta<P;)Q07Ap6@LkCMG2Yug^GG&pPJ?9LipV_rDLL*n6%w_|UT76WSf+=pav
zwukuvSR2dtsnD#PVei!Sf8Vw&1BEE#V~atg8<*s~3MmuIqP&~@FWn9={gYRFvdaJD
z4;Eq~ve1?L@>@pz@)>XfbqzTKJTjToH)jR;)z8~K*nqoaQqB7X!&%J}#u#Jh;A`Pv
zph;Ht2aOc}C^{}9(Y?P@bhnj|nl%R=RvHQ+Wh{Q)GdN|{m2XGH!%1{rwMsq8&ygJM
zA|Nr-p>tOJ4Z{nRV(7y+FBhb)2f0QRffwYtd3m_hSkRwLqor0;to%1}7$N<a+;pm7
ztDxBi(>0Y$-k3z)euVdw(+*xMAeotDK4`(PVxCMxgUPab^6WV^^E!meJl75f<E|-L
z?qz;IE97xZXZ#oK?J41igdT<^l$2h-6&G~w(>w#kDA~bllvx%+hK-Y~V#`9(71}?;
z><G0?0ksX%GJn2~`CgFfzj!*;cSP1wwgO8;-z!t+y7my#^L=celGft9`@j;l!Mu7D
zlrDWZWv67T<6SvR?f|{J#K=7B<{m_6E*E-8i0D*a$Dy;MULPcV-1>h|#WE2h3N^02
zPd3ff?W!GrKIz?n+W{zlK&o_d+-{#gnUVIPV4RU!hq|%k<CT41MN^O`k(CAJl|oy}
zw2&&mO}MA8op7V~!GLrM2Xi|UEm1&F?4BFsp#A{)QW}@ykFPN1=QduPZ7F|$y7Izh
zl^&6pX|?y=%!Nd-Zn9J(p&O_1oDHhRSwF{Cy)lt22_zriCyl<I2lQyaOFM%mN?`pk
z((+zgx)xgv36th<r+|JoZTJOnR^4PGQXRpRbAWcKBc|c0mlpFcY};?-y}dZM@aX)>
zi9yW+v0xQ2J+&)5zjJ<7=yy7D4-8)C=d*3wD{aZXHS!clKZvAL37H^9^IXJbY;uD1
znhC=ib{)7vW+5WO;OVA>;5+{1S~zKXK(UvJ6nIw13H-e;*a^BS^40{0ut_S82QqL0
zl+NU@^|mkIw~Eo!e3P<Zd-X58+-@@FTp-}?D}Khag(f#YBvRkk(Dp6jW+ngNbz=s;
z$-7DT0IX|#NtNUah6jNAjy>w5FafcOIaJ1eMB4{KuU_>c^rH$Ou^bhOu&&<9>E~Y{
zSlLG|vbz7V=-YmEF-NKq@<Pbqf{y!G)5_mpr*4gxW!D0xYu6Dd7$^)N_GXT$<^Uz_
z;|kyfjYUlCYyd65JE|1*zJ}A5C{`9z^WRs(7z?S+je#I02XZ*x;dRcRbLbjI?{H^_
z+)<m#w3uH=eb9ZUjH32F73Rnyc_pX<%^^ruKf5pt;$^hvJaj2CM^hoNI|oU^etw00
z6$%*D=qv439UskzI}U{U5W7CZyD-U644hCd$b1>6#A-2D4`}Nj*JYrEx3oaJ6X)W8
zr)DO&mdY<CWrKh7I7I!D@5;x0u{MQ^l$!9pcvt9&Sbr}QK%(L1?GR5|6Y}P-$N^Lj
zKX0>AHtU2rNeE}vuSBo=A=S!&ER27uPPRpS1P73Fx)}0)oo+J=F+efPttIc16OWET
zJRVcs*8VjJuGH_n^YS(Knd7pvu-ySLvk8(9pPB{FQKt&++;Rrw!X-Cshb22iPZo%{
z9(fPhm)?#Loz8g0sW`xSiia%2!*>PEP<Sg*lIdqOqslV?*&aD#28M1cAlGwgXQLXy
z>wIs)>dqg{?ZRs~Na*VII;!J5akx0-2SlPL7SC&3{ihnuC`4SjfOrowT7%T<ZTp(Y
z)RwMXD)>Q%yN^oeQA6=A@a$UyED<SQ5PB_>mwNf(-he7^|D8fY(IFpkCu->fiPGe3
zZ1gHoH)z@<4admd3WdM0=_KP1=q4l{@@c9)3s)_tN6){YjEv8D8ycCPiARxtts}({
zZv*6G_aCJ?U^IoGJ+g*7(n==@>q>&YaBdQQemxxq5~W%<7a|Qu0^@~OMn@>UkdWkK
zzjtLO1C`6#yl$Zq5pk-b_^T4Lz+4mSM}*0Xir|n%e#+VMX_K=5HRVC;MDY2Vm37^Y
zRKDOhAJ3n4as9GMXGkZaN{B&b4_xO^N7K0Z-%Pt^oXGtsdih5SyfT&quM4~1!h7+i
zQ`B<GYv#FIf1B-8svvz%q?3o>w&0Jvs{lcT+2lGCaRkQ)h|tg7_QZd|HDtdzC2&h4
zm<t5Ku-smb^epk!dAk91a<if`*x=D!mmbzCmc&C}{m%ezS(N<>*2$nT(6MR0thm9`
zLSXf7=UUBV3NO}{qC;0(HprdlZEyY{8^vFY)0Fz~79KRrXug~3u>&6e0z**Pc>P}9
z>ZpSU69oa}b=m#vH>pGRWLsYIv2jpgww+PZ6LBz`WrzHwzdV=m5uZIztuU{TsUqs*
zXeRIr{L2iaRG4P>h-}s$lKqiy$&`K1-C^*J(6$m-<r2VgDl-4@n1M}q`oh?#i-)Y{
z_ZWq&-@zA6s~=foJZ5_Lh+-Ybb{^nkK4GWcyC#m@&-44R)f7N>(H6~%wE9!Dq>r^!
zVjdgU@!u#bTaHC&JSo*9r8KaX2&FL~q;QpIQNCt~n}|Wfr1tx=#qS`VAOi_bCSLE4
zo(9c=7k0~W;}@}vjuQ=Egl3#mkDMK|2Ub#3>V@nErhEu1r$uow&eNLL_^l2w(VZzA
z*1o-`x>>MLG^M^?VCG(*qVyhxQi?uOd;jOc3!5swD}CzWi%~8K@zg(SUGG~-sO^ru
zci#<+bv8mMEC1jZrDJpA`VlGZ-IqrB8}Ctij??*JJvm2`7guI_aWPIlyr&|#y&BrG
z#+6x`?jWgv(`Ww8??RdL(-2R>{SA~z1OG(RjIpc4UKdH0%0>q|_AR{%cVbNMgd|o8
zaYI=bQyfuLFyl#j7|5o6Fun0MP<N4kow-63TzEY2bZfm;qVvPT!O;-7lr|bOa69A(
zv%`2@2q%qc6?)dFygG(wn|%y#S22vP^JKC3K{%|?;N<gyjnkM`5GyCPZ&t5=%XD8=
z;64yN-Y}W4rNdcN30Kf!V^OLhy3~foDNgXe{L**((u8!S?)6uDcY2Bz7j8YS|1uQQ
zaaros)wc2WAm@r6s~Y8JoJ9%KGTxq%l>51FG*hXpjQf+xYp*&yH%>5mu!5V!?|x!G
z1lseApVO090fFO&s0W_icTAI#4^}AQGOKZDb04nkFqj`rPF!&uoP%DDcW|3!b^b=r
zSCWQ441AU+oNWD+8yMPOFsywgC>AN4U99pRYU=ym=w1B!UZs%MRKJGNbQ9RD^skW%
zNVqGl(!20v_DzBl@{`G#^P7&PLB~sGjmwC^f{gUw6JX`b;NjwuXDNO3qd;vaOCr}9
zBu%s{Isop42xoD<ZYUwA<Ls89Z7b0%_$0f9y9~{HN#dwAgL6yWIB-0F8DcuV4&70>
zI6sr$T4?yT*BBea?G*YYN9Y1EPQ3Z)F#4!x?E#YfJPre@AuBsBhB0`aN@E$*oZ;w|
za>-_-zHxd0h_WRJ>Ql#hj#-gFq(nGmdVhIHUfIRe%`lp-uo%tJq|KIRmy;4x2Vh-g
zu};v34kz=qN*VW(aoKzSTdxDiVK(b4XxMuU@_!j6jfNphX}L>pMu*>=uN{}`<{x~@
zYz5JUVJ4^XV0#}-$5T!O&%Hd9WNCNOzp5?VeH^cs`8^Q=Ib(<a3J$^bR#y9Ckd&;o
zZq#Bd42Vf6cPw5Kz0CxGD@&<-mrP3H(`Sx%Uz%rJ3=u@3h<d(TYq{tmI@mLY;gB!e
zE%>3_uZqa29p7;e@<H=dBmkGHh0qa8v5A7+n9FjEBcqXU@appc<hFxCHCkMZz((r&
zD=g*T9!Rb4nfb$2A|nW=6R5ripM~7)M~c%i3n|NqwbL2FstM!jHq3V9X4{q(PP(JM
z`&`0+^&JU00x8IIN`osY73I<kkJ|){)+$^geE}frp>W2l9%jl^&Wn76p6~2qJXjkn
zyxrsmbF)O*pD^`HXIJ2wC)CU;FYUqKoW?B6wE&kwNecVPf)PimLZJ|dPIG*IAj7I+
z)}$*UIC3hm{H-S&uJ=0JFq~8`Ry7n)J%rysJoPv`Nq}}v=<xM4#`CyXD*n2v4;L`7
zS!axv3=*9DEQ6Q}vQ<u&Lv!RJ%Oee|qfzTii0DZGl7!&VIcUT2+za>EvD4OP7+31Z
z?sFs33*+@Nok%(@`jg*ZTW^NYS;m91X#5qEy=c8C9U(<Pfsc9p0$d&N_+T=}O&V^9
z=zg~HOeap)WMl?i^NI`GEgth{YWcwxsR)MlIN?6M4Y@24$Kb+)*?qMn6mHk6pP9p9
z!3z>-=$J&}|Ihz;br74zW1`oOf4b5*$i1fcAZa+1Z_<ji-d3WX7_&Swk9Sdy%hX4}
zYARzLk8)9VNy@s?%;%03LF}-&x42rARTUL6X4JSvRKR{)q~+Y_tbMNB(1ec^_G^76
z^{aWI*FN8pA=(}lh6_M`V%48RRWvt@L4x7m82B=XnG!!ryd&EhDJ4VnP3i?QoEp!l
z4AOck<F9UXRM}SZ(v5r`Nt)iJo8(^1GOGAK4ZdPE7wQ9VOs{(%E|R9#peR!=K)vLd
z&|~jIDe3YcV?YA88Qj*0C{7u2klg45?yE<<u?!O8l-TC9&E-D4Q$t%L$h|a?Glde9
zd5+$m!p<|MG@(|l{pn|i9Tpi5g<Qx@3_Mph_W7ED!nazKt&kcx)x3iEa1M@c<6>y$
zsuyMdYMj%$BavSBs6xM%A<?|$gNInx(8o}Z{+qxm?~dPhE}Yb~D6A+Lg?m<|6F_Xy
z;&Pr&?(pRV4J}D($Hn6}MAlDoxPaq`;Ca@tlL+60$4mr@=yBq(L$?b9j%iD+Xb!Ey
zCHEWO2eNpVso-E;^pUVXhi5QyHjQN4tMD+yN#}n_Q?-hoj`d+sSzY1H_rF=bRQjuO
zA@x$v(GgslWAmP?r!&jBFi?IlBRG<3rx#9jJ#O2_*a{&(+R)C5h8~p`_yO#c0I>^w
zM^&Huc*}K<!E`!wreMYOma8{|&>Tnpc@&jMsHk|xHLOs7?v=9q;)aQ{E5vVDAdRBr
zJDnWX|Hr~Y78}ai%WqXpE%kG}W?T&~a}<`SgV-B0ZM`xJuRR%J$)87>7#w9*a^B*z
zrQ?bS#=Lmm^?uP(om3!A*kSm1!}{%dmja%2>g;csOI*@l+-AT>e#<#3hgJy{qAeXo
zp84Hq(bmO?R_Ug~=fJ`H6Yf=2MdcS+I~r#7vbr)EGrfoO8>N{#y0Aitp0Nx;p`8*g
z*V^H3g@4L!oUbM@(YT5!HYTZQbLH79Zow#=*<P*z;V^C*n?=T`SL%Da_9oJ)nd+sF
z)vWDZc{wvzV#`}W$7xYfm~$4{v6uTsav)B5Qo|@l^{G{>mIkrdEfGgqcFM@f=nloO
zRFyKa%u=((!@Mh*A58NUdWOagt;<}f_s_tzm~1Zg_vXkzINhKCP|ZFWE5c{*JE}d(
zK2c;J+;t{)WWkmtlBrNIV<EM0^;lM3$i((~c6zV=e%7=!@&|;^`GN%dOIBT|6uG)c
zKG0SbD{2x9Ms!7DDoRsK2|!?xV`k|W+R&g|s}9|fnNN?BGaU|uvCMvvJ10x@4~o<8
z=|o8>h3$JR2p=!L593aB%v4UN4UMNcQtlhPx*I*)wGpkQ4M?>apJrKIYTVLlLzuA9
z$aj}&I$psjepRj)6W3}(cMStwSaC7T_*Pk}XT3`+$;*`neZ=$Joq^Th8V^rD3+8Ho
zJ(3lvo%cEV?gy>7EXZFs*9xuNras@sUN;<#kCqw<)6MQSM+VM}8S0aitWi#T$`e<a
z^@Wo!T1V_XE=hFHHK>c>G!%U7EaP)fxnK2SqWLiSL%$hz{q)N(^(DvU4N=D5>|TtW
z<Y&v$7$x?pvg98)$;#MgO0!E-tO`bT&2I)bjLf`o300(U5gH8EZGSC0^UKlsVCTna
zy~oAcL9iMHa(_%)kxS@9V&2zt->M_0sZx+2w77$HxvbH)w$7qb$9{p8#Y$_d3WHUB
zo)}pXvG-=hD{DOIE}QI=-J8=lVl{~W(PNeC5pXMh0LS&RL%kHMQf$#_`@}~+wj6bD
z@mM~Fb&6;sN?M6@+kORErN3s@_ssaOb3(N$L<R|?G{0pxS;?0?H~%LKz*~IVM1%6o
zh{x1N$GQ6>tx}@>BJN&4<gsq>l0{})3GJ3UOH*bU@d$%GBXI3XXcQ{_iUp$3D&SwM
zO@$TCRVjjt(VW)2yH!C0UE+7~x*}?{ag9Vwbe&47Rsea4j|HrPG109a#~->qArx!0
zvaDH(jr_EfW<relyWe%mn726ioH7*?y-~?zzxqF&sE|a#KuN~};B}<+yYK8|!g0e!
zQufHWmAjYs7W*E0i`y`ATR^4VnNpx45dFNUl4*jz0#J`qgLecyJbra)7qQ?Qjk>o?
zj$aIr7V>7w5<SscB_H{|YgQ?dN{}X)iky7LLchDt=p|5TF6OZ-cSa=1ul?hx6J25@
zzZ7cs#Dyz2HT1!P@K2WfS$3BNZx?zT(}fI(Z)DnCendL}=h#AWQk2Iz>?~cs2_fk+
zLOLAGxJFT{FXYzygkWs=9(pb%OZ3f76RZ`ttn`K+`!Gb)t<uw4`RKtlE5T95za;d)
z097g!kCn=-;O%+7oZI;=2N%O-lakx?i3ra9pB+GpUON@?$Ml-5l^#5)V^iOs*`~vx
zJ2k!734a@MMt(TYZAIwy^^pJg>4c5efY|i;<lnmzsIf1aB$(w>2dn3OheAq}29s@;
z(|Zx9*^Apn^}akq=_{=<YR?%R9!H{PsR^CmUaddKu1}RG#IVxx&=uHCdyT>hKsWFt
z`(J*UA~P)z{{wP<&A9b*YPW5PwcrC;jGSMg_-iq!bfXNVuvg1om|h)%eDue4+4UbX
zceL85Fn_ot7{!{^O7#w}P!eOdh_#ODx?1sQK5Rtpb<GG2k?p@68jVUn6@&YS{3_XI
z1Pb{8cYPf=B};^JLEuuV{f-KAUn8mFh+u0kf<Z#BsAl;!<bYdPtJa)ZFJhvLycL7n
z!)k83jxc+GJeQbWDiWE*_WIveUcjnr7$?T0a^L@6AQn|RdIY^SKU@j$;^RqhhY{Qk
zl$f{jJ|j$!>97>~)>~?Q$*A-lA&~gZ`dE}OA~7UGz?zKhSe%|+GZ7`m{M*<HoaN%<
zRN<Otn!8SpVUU~mcs>vl{lY0=FF}sWdSbnX4yWAeIE&FWHtYAl9_DinEQ<Umz}HXC
z=Km&FtO0A?{Te-LK2*?jb$Ti;Mt;F(xgn970^v4UBGUppm8pT+{<K5CsE&uqB$z*}
zA+2qRI<U6k93nJTtiv(Awg+rh`cu{i^>`d4y73U`@tG33WjXB1d#*4<7)!ig7u;XX
zfab@2{)7S-V=hf%%eC+xE(T!vN<}+~N8LkoIHsD2{J?qZ<4_kahIt8uL<y4IRqXwF
zX9H`H?*IK<_Y+$|q!fRc5pRABpuV8(S$l9#DDOnqF>>9-hP24d255IlZZkvq$Qt^M
zivj(86&GZy&0eyMV1!=QrwrIMhM*T)c8N=GVe)NkJ(%bQMaUE?lBA9j$=fsDc?7jU
z_~mg8{&V&JdAgTlm=_pH|M!I&;N{&RZg`_^T1IX%o%zqxvx_vibSvrQoA08Ev)7?l
z6@gu%%~roZc=6(wgIku;9T4&CJ++Gb&*E)APIle<1j`rw87wb_-2{-<=M5+i`o2B-
zGDELZUVaM~<9nLERmmyP_A$K_2knrFrZP-U0aN>c^2xJFXzL4<zy3Ts8A>~k?iR6f
z|7<<ekXyOsPWtXlY%${D)0k#R&5CiM4u(?O$VmkX>Sxokq$4Ly(CMby+d9^i<6<xu
zmOtHGnUqo@`Zn<KPzF?eX_-vH!lWZxZ5x%7Y&$E7m|nkXrN;yWOrL*$+sdcq?HOyR
zbr_q8t0_@>-O&vMeaVQ1U|>iw&jE=h?T4W`;+GTbs!!I3nCPK<pa|qg#4Otql{?oJ
zu8u*YNZkHvYM^xEFCEUixBg)B=)R)k{{>dT{m{Fxbr+fs?PBa3%17;BNlxy{Ki_1L
zuK8C*EjK0L)@=@>;9(lolAO5r62Q=|{g!$w8{{3-lVE~mgKY)u;@v!{dFn+<$LrJe
zU+h5Hp{5jckP7cYh;Q7Zzy5e|`pKXt<7;nE!Nb*|?9EUEsjyqC9%5}LO((sNy*+=L
zI00z=1`#q-H&;?0fYg#7Nv&Huc|gm6^J~ST`7)`!uE+6GrGL@HB`X|0r|sDvn{+s1
z?pVkR;4D^(cniwtJQ}u{D&$hFA;<&cg~z@y@a2Ot7adNH5AVk`UaRK~m(%n%T>2->
z^9!G*oXQZD2yL0&YCC2=vHoNHm<S1O+Jjb&?hoB3twoR&+CqwRs>aTZ8^MV=c@Be8
zl&MbhxY8*ZhhnCJupj6r@NhAnZ-ROteh;Ffu&7Nb6%s^TD~AF?9c76E&R=bg6ee-O
z4WGP0$+LZHt!Cc+4zitqMi7c7{{6)L{HdbGcxHX-WWY*%$%4`I{bOv}kbl*iM1Cxa
z;_X4+%;4H+_x2m=FFod_dC`z!GYy}EZQ|ZKfE0mv!M&fT?Y~L9a*ISogxCXDVO=(m
zy|FU(rlFiK`dP^@C=2eul`!Cc!jy-A0SiT#dVi8VE^%sy)?jRu9Z5u9fw`w>+hU#X
z<k?ri<AvVuDNnfc!JWcf<AjZNaUEY6+PS;a>-|g(B4!#oZc8JtutWwIgW_Rgn`ZG|
zD6cqdkAo?vSL;yu$6z4#SOHm9_r}6B3;grn;bMaoc(h;pXf}f?(6bJAiH?NTMZ|D1
z-mKJ{SN#|K76L!hC+30@?+n@J2NYCJv*;!efue9Ga2PDDK!w~eQ-78IwGg08Rj2Qq
z8r>`4Q;2ie<wFT%G5v{*&jg}lf=b`_oYfk6EDe9Yf#UzWp%uKosw=ST3+)y$tvBK3
zy`H5yC2vn1;W3Yf`Tn>=7dBkvEVb=PI%Z`5SADt}?$yDpSDz~gNvpH@Ta4L=`Kd;`
zt_n9A8a-2JrdUZgf}T^JFAJCsphNbQDPyG!k<2FrZ}BWjT#SpO+XvZVXN*Uwm{h=w
zo8)H2GP(OP%hQ|-5o68|vzK495_w<KB4-Z@CPnc1%9rZxol)^B(cZ;zI{gXP&6~ir
zkopjk?h0@CwmVa1m6F*70GLPQEYH0?J7Rdt|8Bj#NV0fr(V7K^@#pIUp%JL`&IZt;
z3cY7g=w2F2Vw4bPT?Qg-1yC>>g!R=Hwp2$vh*i>Y<txHE(CvNM10VOt3qR`-_;FUC
zyi~Fs^uRp%%?wDJ)nsbE4$I}y8AdolsyA}!<Bl6n)|gGe;FECcE^H9VA9A=Dx|T5I
z&xJ84Rxa7V9##80eRNo;d=4M=;&uZ7OG0qRq3Mv|6lk4(nOX-pw-e%G9PUI&lSiPw
zRh~OXGp=Vo4*W+;gLmsvsOBHg>AJHWh3@BC*!A(Kzarhvz{rKJW`apVyhoBmsr=eE
zpwpt9EfwN9nl%kWd6?#dFuh*((&0R-jYMr<G-mKl*|cg?{6Ve|3-izgg;{^lfQ^xC
z+goYxrVI8jIa^ZJwY($3Q2UdS3!y}vu;hQKv@1d6Jb*CExdE50FQU$^{h*ntR8`Y#
zuYn4`HvJ9Bpy-G9aN0{b0zD1990Pbcryqma)~0~3B2N}DSO0LOfmF+~G-yvIgAV6c
zsYON}5c|WMPTib+v8dUUAs+ojUs7Ee=lvbhw@6eAWNHs5KrTF!LHlGe-GwYt#o8-a
zks78*^zHMARF(4q*+&uT+i;yyf=5{Eyh^c=*_OHU-J0W+@7%Zs(Af9<c9Zn>OxwK(
zwP=TPWF5#P{^!zpN9m2YSHY#wYryxLE72HQjGe|-zVVkNE8sBAv5+}~xPX3eIYRQt
z80Tm_jReX=IP>_gD!aV5_~eOt9QGC?;Xk$vx&ca+AoDqYRq)(|22Hp=aI-ZVWeTX9
zaFl2<;5aRnfkj0Kv}`U6b6HAS4bV@yQSwAV6^qC8lH()BWNOSnFrL2$#cv(ptT>g+
zdA9{|=k(d&U22%Y$5q#S9)juB{dupynfGEP$y&5(X9#MK?r!g%FYo7CI0cg-1^l$b
zyHC`=ZwM;UG^c4x5`($<g1<1f!o&}$SD;=RXitH4KcELaTAkDAG#9rleXzs;ed6!=
z!Y_Z{($d5Bs{rnxI@Gw##rywa@2{h>+PXJjSVBTl5ott0K~a$oNrO;Qq#LBWyF@{x
z1e8ueK)Sn>l$Hi5rCUh>fp>1>=y^QP?-}D8-yiQg#`#BY_r1OM+AHRoYtCz4ms<g}
zn;OXltpmNW-tl9Ji(T~}Aq~$Lj?n<tG<Lg^rDR_NT>thsnu+J@u&K-^?{_dBT8snf
zm+2Op)pFKEgifF(Kb*?f)LOo7!cv;+$1X}`>SWG1gi{HLLIyk{FX?k~fO$c>JGz+B
z>iuDBPxTw0ibtDa@I|+%vfu7I?aMjawmUn(IUM0-NWFsiT=@>o3#^jJ(CEtrX8Kss
z{jZ}zn_yR?HURk~W(KzQAl4d=vm^BSr5a%>8A80pF?yIVnNO7y5ZlB5x<)otGLkeC
z65K1Y@%-SMDBgO($qdk3%h_5q0$f+^UJn#BZIA-cGetLD<Z;@VTWHX!5`f5aRta&D
z)Gqo8+l?x-;#*WAr+Rv}9<n%ZnQULEQCjjF=h<+H&~sQW8es9GsM*J%DCxS6!@W!9
z4E3@)@v?<7YZ?b@BuM5gaW7~6997}cDW&_tU7OBLslKv?<RS)1MGIpDm9|a$?trbJ
z<GXKP2Hflo3lXN<cPOk4CY>beyD^>{#-tO7QxMJ_AnAp`Emh45DY2RNz<rg@d3tdl
z_Brd(ngktX!<WxT8cz*t$~zA?Yj<JvA~c59z;rehMmj+yHOw;rc#QapJ;;$qKapFc
z9s7KKU%+gn^xmGPb2iR!4TzCm?b|94J?NunF`oQnoL$!Fs}i&W#dRZbSR1Ou{db1O
z4-dBQHNA?Dq^T>MWKUd%aFk7)`+?&uc=-~Q@f=9FlSv{Z-i;s=e7Q2Ev}s`q(<vlB
zxj!c6FH<xDl~S?bE{zNUkkv{I_mj3@D@(h17D%C$S7kdc-dWcms_n*wxPVmlH0A7I
zw_7~hp7|Hx4`7jy;UVDLcxtP(Fw3+|C|^9BF#+kO*w2$q^yvzcpu@)vB73o!<Q6{5
z8SQKm>3!u7`?K<IXZE4XNW4H86k~P6lp4#IOtwl~s4si5d8r3o(9jq(c5QKp`T(Uc
z1!G~2iczcFn;LdQx>+S8b|XLraL0PE0};%4>xRCZbU-bzm{~w+Y7^VH%DbP%4Csq(
zPW3Rp4Rop-VUtBZ^b#$GFF$}F^ILZBmi<lg;QsKUgM<N{d~!gRx|XO9LW;PY%8z;g
ze2b{QGi+^652!n2)B^+7c$POuoSN<=mFAreaEyO$m8kmP9V-Sq+_uj(N1nRlVx5!~
zB=3(xa&?*I3+cjU_2gyPD+l4J(qx7)MpGC`!NOE8YXB(p8aX~jC{nVc;xXg`bR!xJ
zLgvTyjCdWxBP%P*P>FwU_bYO6ngLn_$a=j%33{mZ<iyj|i&1dM8W#0(-d3#|)b=A=
zHAg&Kl5p=}Ve<Jfz{=V9vpKM>J(ODqz)Cdi8{;eSYz}AeZ{L2$)T<E3Vdk{GvOH|8
zbg8KK?n=Bq(`5pGwt7p2^8uOdg@!#N0{hemtlFQAkQqsbZ!|`rd2*b5im{{zS!p(K
zrmnw|OdG#IKF*U>2_Klq--p!qcss`|rxdAQdMlJ75=1Rrqp`WZyl=ZS_->9{zDDC*
z@Ox1f&v%{WY;hdB`|0jH+0QX9JI{|o7A?M;yYt$iA@AHo>|U~E5*)I;oyS6___ntn
z(6N-y^6%qo3m)+_K!Mx@6c?Cm_@A+Munm5yG$N+#GXgI7Ncl@%-dfQ&(R--_M>Z<6
zH%aET8uxQ*4xwVjm`;etsK}H<t#jNdd)TaEd1Sp*L8|8v@Fk`1=BGFBiyhbm$vUaB
zgpitU@c7U@DnIC!)sCjZq_~pe(U|5o4lEMh!cMk~uCqp87xeG?tbOU*^=ZsL5R*bN
za}}Rd2yu9V-7KSkWiTpt;(i;1(oB<O%AMP5w<?4}a7K7W!0R=h@<d%J@_O=o9pj%8
zDc`Pno<Bh&lmJe*+N~dn7Izeh1!d5Y9K~1woimC618oo9NAKKG8uiX61l%uroyk%D
zH1!pQDXE{cUS%d03Oe~cL5lddJ#GQKmxuSO-(fTiI|QSkpoie}vI^1csB?^mMDbSC
zpSbfZWVBGGsQ)<`p#g^H(Nl`G5@YY^-1mD{RU|C51Z(95bBGN-`kt?GI<!>^xXeV>
zb_dJ&Wxa+9AUXas2lVrG4Af9gpevV{aI^dw&ri63E=p+dL|dEw_#Zi>pa1{+7YW$N
zoe!>Sll*z7r~E1CpXYbNb^dE3e*C(p)3K*C(o&|ykB*>X(Rlp*4{2J6+2LW+{+Z7D
zdsB{h-b95Uo3_rGKVRsGXC}0>`F6<WfByP^Jj_Gz-BD3O1}AijuYWYeLy!Uo98(Y|
zoB8u)|K64aU8GwgUrY)9*=o<3lh7^PtpQcS|7~o3c1t4>-1)iXA>Z{s?>u%6x+U>R
zwAatc_YZIVtx=wo@J7wY!{uoI^If5zZ^9c9n$}bP`Lcf>1P_mZBiN`Mx7)n_<2v%s
zr_u0T*}Hu0|JH<`ox>0THy&lQz$f{6<=6BwV1xNco^^BZ^#A%519`YJs%UD^{LecR
zZXq2X{9x+ppB?}AXC|~mw-~IS)x`YsPET#<7U`w6*}4BU&@|VeE%^oaCH|=ck+mV}
z(RlWszWduXk09t5eg^-47-qt>c=&F!K=01q{>MLzLf0@%|3bSRE1N&>6s$vL?x$Pz
z_-FrN68yd<1VgdlPP(HFX8&+!*2S(@vh>yvL5KokE_(iYsEXqLoPX)@KQ!m>&nbKY
zZ;^F3gc9@51^)Lte{amMpA}3*mZi;S>ra2a+^@U-k55@=VYy^Hqigc~KW_Rzep}iA
z2Kt!VS4!UhA73B?>DP<r-dg_i3;xemomBQf_q<LbfEn~ZZ~ez7bX`QI$89mI|Gn1#
z`2mj*Yu?q+`TC#i$Ui>v?*seq6!>=v{A&gNwF3Wc1pjUX|M~*|`U3yDga0pf2Mb@+
zI(hlOtD&o#bZ46x-3z^B*mfn!UnyTCUpHq_r)|4ulIfZhrVhT_#RIGI`tVTyvi*{`
zVQN~8I>l5`CPM?l6zgwt@3(I8w@38ZOBFA<ijc3*`@IbtU8UAJrO5L-KaiQWUn)1|
zQobgM&Q=xcP<}kGe%pQ%or+yRmH*Jj)csa|_gBnBW}?L=@qPjG<&OEf=I@&#=2y?D
z*wu3B%)~Jr<d3^*wQas{(vS>SG&0rM4G=G0O3rKKwUp1-b*46JD@O~xyf;YJ_;vR6
z{nizy0OmV3(2_Y*|42U8gZy<nwwVO_Nv700E}sLL?Kj}7Cn^*L?T_Auxo#@!s8{k1
z<wwVwwHZ~w6PJ!RiaTBy%CC9u5aa&pODj(ex%D*0=kT6$JeSSe#!Bc_wrF=M1-Q6|
z@<*q63qJe8kH~kJn~yv*8CssMELUB<-#T`7c0KYmol4ltkJ!@Nh3hX?3}=(AWwr|o
zkWV*9dqAljKIz|rPrl`;u|(EqzI)JuH?U#LkZyS0@t2z|ake&qrQ*Ry;mFv~j)nl1
zYm8>khPW-0+Tb^ki^QBQOEIM~db9PapO2<F4@bgdth>wLlM?dj&qw=~eZ+xAu_pc7
zsDFFv-#PXFx7GK5gcN=(?m#$~Pd#o;|6ie|KSO}u16OI`NPs9r@7n)-)IXo=v8EUV
z9|#75gv8$iWdH4Xze09D!<0^7o}`%!i~lJ$`1csVU-s*7Z590gP#1_^zWS+bLe77L
z!GT=%x5(jRM!`|mY|{1b!I3Tt7*saZ;M^O3d(j^QAaSNPAU}oSyxobPJ#K*QDRhX>
z?|+=nE0y`^&#RC@DMjt&n|#77@IB~+M-L-aM~r>#;Xe#z*Gbe7hSr>jek#(+6$N1o
zDV<|KLsebr@H6X-N(*s2L-~3?qNWCtC<}4zN@h#!-*3}}CO^~q^jDh%X>NK@=$_#c
z{}Gk+ynf6g=~9h$6Ni9GzyFU0v|kY1XQ;8N`_F4>zv9~o;t;rE!BzhiPke}FafSii
zN9;%6NQ<H@i0lwwEB=qhiR?V5_~Y9DXfTe@-x@3}(2_Hl*WMY%{v&85XnhR1Hj?|F
z*Ni%>baK8gK?Wr1j_VcvYv|6Ps$v~`t##r@f1bSSu`A=&5NPUFoJsa$u%cv<Yt!C;
zT(b_Na1Qv^TY{0WP<2TwZ~nEiu#Qh$m&U>m{V|Uco}%^!tufk}tQY9E|9E;=%(rWt
ze_V?y7#TGFK9nsGXJ)9IIQMh7Jp>;gA$f4!BI4&Xd2}+t?U_cNPIUZb;~z-}$U-33
z%C!FTT5rsBqx$#Wl0Fa9jSPe7mu0$+UP^fKK^@(XDa4R`WHagO6RP)LO_W%FJe@5S
zxz>XB$F-SDoLk>bj3*1kQd_P%`2IDE3D4F(N+6^>|I^_)b2RPrw8GNhjDBm>kEbVa
zf4gS($2E5Svb=%sz4Z(+MBcCO{FN8*Bt*G_{%GPV_K#6I_HDZUJB6|Sc?u^S?!Vr1
ztqXkJ6LI1)*t6ls+&`zLB3KYVxiQMi$S@1ufqY$8NJERzj{!S^%8doa0rQ^x&oPy_
z3^wlTlb~|_-xdj>V8UIGa{ZT>tG`*!@4HCCJd%#tJv(*e=U1LQ*cM5dl3gahGO)GM
z+jLkzzehn|I43w-px$kQ`O9Gb__RO=GdDJ22>0ib=-L3|X#BUIPdPr<&*`8*AAlFv
zc=E3qgM$9?nDn+kW8<Iszpe?ys5_wKIsdc_f_3oLnOaXCQ2leg-5>4x*7mQkTtCTb
zQT}Y(4?iFUwPvT?yzNTGl6JZ6q8`Fe1z>AMR;@P=#Y<%BpF=@Y$5PGVK?=|vt$YCZ
zb)E1b0u(=MWzqJtL4Pf%f7o_Us)HSV<8$Q5-)<JHMx~FjctFN-|8XRMR4#I`Ic*qN
zcL`yVHCbM4TyK<2FdoRB0c0#&Ms~ST11hUt+f`r+h;#&1RQUea*!*SGes~4Hzq_&q
zJi4DrCy?;>tAx!rCjf%O3I%3P6Xk`k5208^mf014m_tfnzcA9egPXC@+1w@;G-4vR
z!w#9z0c&kp0OW#Su6zb?1-S$b!e8u~%KF!ym3!P^yt+yWs!iKZ+`07O;Bsy)s&MFW
zz)~ERy91uO3~16cOynDQu@rG)Y`VqtKpPD|<}f`Yn+0g}uTR*Vr}>ATL$2K-flXoV
zc<tvO8wt|W()6AP{06CXtybxwZ$5-lIu1}6%3ie*=I};@I<$*n7;9t6xGWOhHS$c?
z0a(Q-kZ)^nO@=rp9stD!pv5ciKPgdI8*=B0A;|ub^XPz5kO3F*#rm5+=Hw@$IRIAA
zL;2v4+um}(dFKx3C%1yKNratf9n`fVudS5Ujerb;FcAhRp(ta5SW~n~zSxA(*0{U7
zH5448OX{QkeX{*EPo!<Y6hEOxq5R=te487#C%RJouK>)%Q{@`e7qlzs#}aeJVw*BO
zr4yO}@+!`2OSv~DIyz&xdoF<%v}aG`1XJ#iUUVezt#`YBJZuEvst<Yq@k?C4di>{z
ze?sq9wU)339B-tW-B1Gs@!}!C{frP{N^9Usj=CV5Qm-5qd7J_1XjQKt7X6Rm5OjeC
ze~ivU|NVvN2x)|9P?Wtchd={!-1jqf_E+6cJJ5kz)Kz3rp{AC6e25^H=47S3yyM+=
z-y%e2LZPFdL8u_m#GwS}O`;ICI~zXnzjkK?9-CCfPvtljtOxaA;&?2^{g+@)&Xz8I
zg;{hrCy4l@6wK?7&$&f@x<h|kT1QZaz)U{PqBs5>|KNE9UI0}y%D^TB9c7tM(D^tP
zIl_r(4Ujc-3A4AKVeEY02}=ppZI8Q~1Ll#Ct&Hselyg}@jF+K`$vu&eHM8{N6FsOo
zpH6c8?pnZj-orxX;S3b8vLk|Z4PP#ejR4MY#sdxiDjoLyM5Rny*c>z0W(<C9=f6xT
zly5piyYe}81Ac#p3BX=>5Zn$8xD8NypERjtn7ca0fp$=xtq07v>>$9Q)?O%29W3{t
zb|E9jtTBK>|CGAJhhtZkJq4W^N@05cZX_5;5S%?yT8>79-#RO?>jZ@`DZs%)hfR0%
z(wr3?*2%&NZj;}P@oz0`M*PS&2BqV_kNiW}@d;CUD0@QAW3_?AfgZHT_7Lcv^xZ)E
zd>!ZkqJdc`62OUh-1z);Rc!zYwtt0~H28`W>UF~q)cy`oQ?^0TmKKnWKm9;maPdbA
z3~qTE=T!Amf;M~E>EWX!s+u54SvXQshoLNjShNdw0?~iYZ^6GAz+qwdzSvSf<G0>C
zDGAqRnAVc9QQO_8hNy+Yuu+Fr$bQZa&lgyL7dnqsPX>5w1y+C|M?uwE3o@$xen!^~
zK=Pv<>d}p>a9K~Oz%OZs3yA!eTrjeiikdf*|JHkgwbwn+QE-wm;vb=VnjCu!e^4sn
z{OC$Lq6iQUXag9wK^UxzW?~lrmg@n}pD=|9&@Z;?0p5It1;Jmv->XHc@xzq#BAYiy
zfa6gwHP-G)Ob{8NG)$kzXy~ZuJUJ9sg@g~cpHCQy|JW8iy-DWYCHV{iAFc#a6c50~
z_Q5=zcIaIBl6|DOqaCOqbr1sqH4`VT0V^H_;N3BU7hE@lAbfS|=dAFQM<7?vZ-?sq
zw&5mt9v$Wp(uDGqt57Nxhp;oW1DWR>$K!5c8IYt7`Zl~!eYkp11t{ZhJQ2IeJ#2M6
zi<vrh<F57v{mRCXfDVoZ=C4SgNsRyzo>o&;c0jJ)MPnQdRCieg;po-#sZ#M~2($mV
zp&PA&s^u-<dOVHP8F!C^^v<k$+aYj)`yaSMvCj<PJUI}mh<Dx^95-O0^Bn+j0Y>%#
zl*?F|b3N<Kg^inn&-)T4pPp%i-Zg{=a!#WxF>DfQ8fJGTK(i?z+<B#}i5K9tGvOL;
z*lL@rZrj(keJgl+E@otEOY&{@*6tvRpw>WRDkqMh!`r`no@8vKg8C2P$zk)Z&i4rW
zEazMM7&m;5VFOxDvkmKfz><-geo4i{+Eip$-;kofKq&m6fTjAgTF`hi6bp3QZbYY;
z0i~Vw{CbMixfNhM8UhA~wquN$nj4M1NI~=%Pz>9@M`UCi0hk*=eHslF4>{3#Gho=E
zg8zg~=`@$9SO55UFpl8~Pd6s1h^uET)Uz1p?8`Xr`;u=st1XTedu{O~*lr_mO9ano
zwx0}i+A~m_ak+1_D9{2Ri;4phh1FS`D+yY_K(7aM)-T`gI%{<$2${LcFFsN}{T1-H
zqfU|iej8GY=fx=64b;qlqQf>wb}_D;FU!YpUnC3Q>|RgC8NgM#VL1Lq{Su(S&4!M)
zC*cJYs=us*f-22Gu8&v+K-Lotg1<xiuF1Cgri3;?UVSv{4OPzOuxT-(NAoJ^z!u~%
zHY(~eM2xELE$%Ca7n4B&k&JfS^<%iZ_MfzZuntD4DhYqa0}_s-vwd~~xj;Q20?8hv
zs=}`T#5x7yQ0earuj&{OSOZh9dqaAWdiFLMf@Xj6x$?r}BetCnPO|xY^PsyHP?bZc
z2+;n3)HkzZBn#!JHsMou!1PruW(qKifl8iMU{|+)z^TrDi=2uZO7JRaN$Pc0L)o(J
zz#2bFXYa3mejjDFut(C`=5T{ElJ|M4T}nQ4`B>VxSvTQ0%z6KHXj1rPvV*(i^IS-k
zI<1sjfNBktl4UPFy@CHqsR^{ePnP?ym~UFX4G8Me4Ut4j#*K^Rs!a`JRzxQ`W<g;u
zpvV{41-PLtakO~-r~^y^2Rc~=PI`th8U<#CkLCGP8k`L^H44OcJ})c+@ddXPT1#-F
z3URA<)eh_*F<`5?GqQ@6r*>^ALM(`id1euS`~a=*nLxUQhaDM2wtHZi;`@p-g2J~|
z58v`)VC=tV7IN=p{1O8t*@9?|9{qeTWn8DWm%&=iiRLHBk=>>}B@%L;cHn|9w)`S&
zjmwmV4yVib&Ap*2r2F0YiEajdtNlAnRfQ!HtL|ba>^2;}6}ACai7n{tuDVRqBA%K)
z%!+^!yG{G2bT@g{?su)pJ%T!O97MF27H^lf<eV60;$ru>Ubz4iBNimtD%A*)Z(y2%
z_;@V9xkrQVOV!+RZ4$eiPa6<ODYk{BnxjvL7wj_dAz*2|#(j#X()M%wuEw1PLFp>l
zCe8_a{Jr|wAX0-sQbpj8iVdW?W<_n7xSpmy`D{v6AIzLxvSnETNY-#6ERnQt6Q0?k
zg{@;ym=HSg9fImq?t#md{N}SOd{8)l*eff=9b&OGXHHoan7NIzQSB4RBp|S8PF6Jr
z{9;y)4PU2Q*UGs&)_q%j%jxsRJ!aeLKey!(b^?!(bQfut2%gag*1vmVWyj`-YBt}-
z3)LaIipg41T~jwFryrEQr|{r^&HFY%YDHke11&S2*Bx|CqQFb8+(>qaHXQ?xL^vK@
zz;W&Vwr{dfYqJF9IuKFrNCvJ?$&V8^i@Kgz0BSo^U{NhkP!bQJMyC_1RfQSsxmXF}
zByBKy^2}=S{&aY?jE%aVg)0n^+9g+omwcDzLM4D`)fS<B8FyagD*!({%Kfln>s8<|
zqu?e{f=T~;J1t(xNa|as)y*3ISMu>Z6l|hsZnVD4LZR)zL1Wsh5kC`cgHT!-m}8Zz
zF9+XnM#mzbIv-APXf@fgoJV6&z3`bl0}TWBY6zh$pYJwSuapP9;k5p!`TbAjt7N3S
z0T-}n2rnjVo^f?@=}q9n;R6)COu_RM<5el;RjiCvgxNOH7Im3Q(0OPWs%h^m2CQFq
zZs1u+OB$iqz_<*BQ)UF{LOr&XY$2@5Ou#y!Wprvv;3pSAft+|x+?$QJaU6W|m%;J=
z=InKAW@u9to7<=iUwh8~LBOC(9lOxpD?!M8b_tgIOkNW;>%t1Bfw@jdrVO*#Y+rfo
z*3wg(2yW{XZa6{)&>h3fen5?mY8U}W0f-OY0xR^@gjKab-CD&U@e%4k+PKVck4}tG
zj<LZ$jX~F{-SXvEPF-<<8&L|>URz@u;Wb~+1<6%ZouqR4^x7;Za<tr*b+-fn?q%;o
zlJAsY(u!%a4X9cNI8#ZJ<Id)vo@VSD%K|978h`ps5#8-5SxN4fdM>?KRWpjNqj^qb
zRe|&bZx{mlcRPTK7IZZbZ5WNxfp;d(kMc*0w{X)3!l7t!aZ5Gb*1Af43EnJj&4OZ^
zCv8iSqF*Iqe9gaAeFBQCSe#th47$oKd-ZR+*Uj;O^6b(<9JX?JU_76*U4ywt8ud(}
zeO~S!kJfu_<xt3dnXIAS-~R3!e-S{*zi}*z>THA5kj`&RT3gIr=E*t_4lD}oFXhTr
zITLoJ(lyMp)68LWTfH6$HnX8^b(pC26v}u^$;63r)|v3BuyXZYf)po0GDusZ#p=b6
za<80dYc%k{tvLx3WpsOC{Ji5PO)fYfeI|EGs$hHOf(pI-Gf!ef%r>}@D)M%_?k$>6
zh)z!?D+r?IQWI?$<{^bTQ=bRhET9zhnbVuJD|cK^_j%Pqk>1Q4k!~n15aZko@Ehm9
z8aRGf)^J3syJeO2F@z?MW(UpKT`@Q-yIe+Z^i+7#_oGz&r3-kSXoqynXDq6AH{6$t
zXqFQ;mBc>bjY!@jKrMG&CcF4HVN=kskhIh^pyrBNdHn)Aje$U+z84gkov~1n;lnb>
z!I7zoliQ}D!Rw3xqI4#MaP5Er>P|YLGZu&QLLTK5vx^?232rAH#zd)DA%GXH#dl$0
z_{CN)8Rw}NPU~bmHfgIoNHM+QxT~dJ|017d5Ur70l^cUEN+54Sy_3R5`2xW7V|S|W
ztu6wch)=U?H%}ZtS(VdJ+V<Mjr_Hu^?wT|>74e79Ch*k?FPALaTviaHw4F*k=WD+c
zrebDk;&i5Hlzs;13qAoiiSTA4hjD*qbA)gCfig7>C;_-O?9!ci<&wX;2yURlr5oH$
zOtUF$Np3esd}1o1SW>vDwWEMn;RzYq(O0D>3oLe|WmkTYtUXo3g@E?8)(4Hcy%N0h
zd^D_+upJKsrwZ(-w@I?xWr{KmZxbS^+-MBuearebeN@+kJP7j~mOqqMf418{)vNP7
zb$*Z9m=Vq}0*7Baz>nK!I1%IwR0qc7ZSS914q_CD6DmS=BkLXfPg}@NX;-*sRnsXN
zLve3x*W^$tmgjl)JDlCvk0i#XEP=bhtF!!}`Lz&~@1S^8{G5J3#I4=C&zW_{MWG>K
z<MA`4I%_^IQ<p1MGCN@fKw)tu-i8ZIm1{|c*>Ej?x{O=QviCU_k#kssCaF9^G>dfV
z67tXKgLj>RX=+_7cstlH=SePzRL}a0x154FZ$|ZY&ohPIqFV^Xg1`pPJTQytu5Yeh
z>jXKu5M!6Q8|o(BrEiY`?YTA7OI0+>!H()n+%y1Xc-=bL>p+4aLLJ|Z8|8A5&r*&n
zc_%YJ^H86;c^Y`*@GqZ-16W~Z;6SHvOd^la1Sk>VsHx6|$9=rMea7T4;uI4#d5hof
zam#ouzU)Np=*hFiP%@gJ$lO)`JXo9f<Z1Di=wFl=!yFi;guTTFmkAGLI6vo$6Fx3a
zqjqYzruo9=<>Fkxxf(x3P<XJby*m`Jb%#(9ioo*U3l2qXyb^M28d`$p?mM?zifzBw
z6^R>d1%@y=lMI;?=*iBX+svrtjhDgN5;rY1NCN{zIRQ{smap1XEw9Hf@|QItQE`5E
znEdZiH(OK%27rneb~-utM+%h#JmyudK@i1x;@bQ%fJ6J8`iu0ShVw!+?OR31haobg
zRG>B*gjAHZr@My7a+pT-09SC-vTiI*ZX>zz7c+t7W&sF3vF~Ne(_u-wXB;f89k??K
z?Kf}71dF+7M#r0a+h(L0`_W;2ad-Ep<Yrk`C|-w{82<pN<NV8TAOZ@a?U;3ixSp?S
zYDezJCm<73LUsb(Zp2bVWU)etVEDZvrOwReVOxW(??N!$oN`r3Y$@NQJU!MNpLeeD
zz;K;s$>jrm*%^sR(f!ARow?KlY?htOMBA`Py0n?EHXTaVt_4A5%g+<3Cn1vXisO2r
z|A{Sy=92(QnXmp7cCaUPck`HNh~R|Hd+hGHCfHFhh2sK=%bTI_RTnvW(Mn;lTTM3M
zlCw2~3=XYLB=GN!;GX8M)TW*v8y+2la%M$l{-uDCO6IlX-drW46w}JjjWfW<mqF_O
z2Ae*@?@D=^w(2HDnUE$DtWbB|luO!UkdkHZu1$3+ezFZ`I!?I9pn=O3nCyh(7*trt
zD40RX&>EQh9nD7%>0@m|f*YlYCtJo1xR-!)a5`y3Cxk+cc=FAd&GK-cXkA*aQX3ip
zuO@Jb(<g0BdyPz71)tD*C+*1q`Dvq6ZzN184(uguP$t<1b%*lI8k-aWS@4QQY`aLH
zXD#{McGXJ8iuK?p2<OO&_p{6c$x{Q@ycIuNpuCDHJMHLeg-gJQ$a45~R)Dc_rz)7f
zoP1t;RJ8M`FnI)-nnLd<Agzr6t+*UL4k*D2;gx3PH+X16u__s2*6pSDiWCuoi-tPo
zY;8(&!n9~)a;AODlaVM3#_^cbX<h9SjI%HYc5PxVzC~<hrjX>L<K_B0#YhO&8bUv0
zh92fnFK&PjlEVc+^BmTBcU_r0J{n?^1=^Jd@?W5snz8X#)ecZD)4HoX%Si$jjTuCf
z>$EPNVJ#HV4T-+7Z1}Rksm$58o5$8=xwry_@@Y!W%S9Rb84m7nw2ZFVUp6P6EIinP
ziZ}&-zrB>oK-)$#{RnWKIBkfPSD$(B>%Do1y|S7C`iRkT3To^&WcvYWx_9~JoH&N=
zhc3K(xee$3nlfOkzP;+`DX4{x<du4nE$Bm3QugH*G<5%s86smp;7P%)nn@fO-58eN
zq``9va8sCJo-1OUf5{vwx47A{U>qqXpSgLa>U_$8?;5Ej2z6US`QF29<wgJQ_kwf0
z(?mY_R7T169&0k}IWHAtw#yZE(5obQfgxZ`CbAmgoC?@*iFSxZ{3EAW=5+N)8#D;#
zRFr{h5PGG<3j64WYSO@N24<GZqN^uDUgH$9r|$lK-aI+Yx)BNdY`D-@-HJeB6hH@j
zFOz9bl^KF1+uI)y5guj%^J8ZP_tR*OwrhRMv=){hQ96A&Z1W7tok;E9e3+r8BT`SA
z#|kwq$v2<nWI7Q(jTT5Gkm^qT&>RpLwbpB9Rn)qaD|99KnpIwv_7DU-=}m&W*$hpl
z5t4%rC<9;RpSA|wAnQ0_+w0y)BXQ%y5>49dYM5$NZm3=`KRS8{iG=eu+_4OJ6UJd#
z3IwQ@Zas6MT1jya8L?>t>8yGwgTihN_7o?hRMUzbWp5RG@n{`kHz&W1b{I8XIT_L@
znb^7zs$b<;cd&Ty2(OzaLz}OAV$2cZLT^r4rW&d)^mMnBQ~?<!Pj|vqt05p>sn+IJ
z^D!kr9UiTsBjrM(K1qC3qDcZs`TFCL6l&R>>a{3iRNDX~*X()#`QKtq+ph9BBJvS=
z@_rGwoH3X6cH00xS8~%{i~8dty-T5V?|HfvnQY#DJ16Q~2|{^Kjh=iw#v0-~*IF+X
z_*Yc+bK5$1C!Dv5=CK5c5aRc}%Xf4dx~Aza^bJ91%P6d8oS|Eh!Da{|x#RDg)ORc!
z=Y=L9-C?!C`HW7D3uL37uLGND{|->iOwFwlo5zNj5SA10)Ntl(Yix^!@LUh?@x{56
zE7+dDK_lqe;YWL*(15y`6t#X!kNE-kb7cKNFS0KK4_hDijU|X{&v&hA?*|?>2^@St
zoQt*HVT3GJB?c)VPTZT9c!sH{xa!6GsZQn6ry7kmHY);DWz7CXeMkIVHm58vJv+Dd
zPn2CPd&#;LP`1gn)<uR8yjZQ&9J+09`>506T{DZ0tENjFe7EV7C6fBq%QJ>qr;%$d
zfl6c*&A?nEM4Yfp6)2<Boz2Y1i55ia1!gV2hcG<lAA6q9_9yhnBJ`j{vlRbHPs?@(
ze5c2X5@Z?3etetovC*ZK0f&YPnv4L&sSEpw+lo_$>%@7VAga%3;})An(!YOY#`uPT
z<43otH6AYwAQ^kq8wsk4tMPXGDzR-7RH+;m;i-i_W(sk7a+WdWCQ0Y8(lmM{m`!`L
z13Id7m=2kwbT*6D#izwyRkzOb6f&;nCYEQ+tSuKq5F6r#q{Xc(Y!W=d^iOxFOE?aK
z$fxG3*;|L{un^V;cD(n!nGmW1s$Lulws{|laMkx04*mN!*<n;VUGxqR(~>V4u0&RF
z+Lmb&;@?-682e(+^9ej!3FnyU7hV%P<QUZSda?<mwgriMHjsdlZ_}W1w}-8EZmsX_
zQ7HS4dI&rXD!XP16rV-#>b&tXlsMA)X{Ba!lp3$eXZ47%M}$+CT?{LcASAtJm)TQh
zJ^MtrYB}t1VBAWbs%*EWfAax3<z=AzkWuYWHqMeO*iV~$K89@+c41sYU6#{i2z>Ep
z<NBw;bz{w3zon+KVZ20T%m0+#$j42=@I>r54=@mRO?Qj;mOlpC^}x_Exf`eb6K%uR
zjD3_laVk1e2FNE@Zp7?Nzxonv5~XTXAUR+4*4HW5+bnbU19|z1uq60ZDT??&IKsq(
z$e*y@Zzj}-h+qei6z_3IzCjgrC*LBg;tM!4Z%fouwlBWHkU)KbyLQ<tZ0n_4{VaW*
zlj<FJd5ez)*I}OEQKDIYT}8iiNcz)z06zljcmm^FKnx6FHB0ao^xnO5w8IJV3KU`|
zuwu+uwcuRDw#N3kR!P4-O81nSjSMZ`6SUc2?{Q=c!il+pjUnuW#~?AVJa6ZP;dnN6
z9#RIH1HIixpg=oOnnJC_?%bufI6~3BeEdc=FH6v9r9yHPftirhF+{qdbdA$$y;=0l
zjU9#(3OhpsdFJe|?TCkxEF`<busDF!ZU?aK9*aE$Eo^pLkwDy7`nNY7qcVZ$LFzim
z^XN=rv##DYTF!NUHm7v5)qtl{sNm6Iu9{A`m%TGgW|p6@aZHawLVCA*rp3&K<J480
zR=2NDYQ&wG=6p*<FBQkScK@(;e)we(@2e8+ayD{S5Z@C)8s=1Kb)z5MR*nzEd7?@$
zYN%`E6n5WUnN4%zT|M5chJaM+VUD&&;V^bpIos1#(LAC)#HNqYyaMy3)-m}*#15EZ
zv{Vs^xxS3>;JBiZp~8-<I*qurV)4~}hJ3f(1zZkxW~+N_?*l)cKim0lov8<L*_ZPn
zG@1w<M7MJd3FnwYQZUAxTj+@8WK$4lhygj5n){s%;MibOJHd)%8Wf#0_OCBWb-#Ze
zob0zC_Hx^GcY}G2?p^oZ0!fIb=E0XHfiU4az(L(8_;TBgsaFdk+?=}<7dnMXl2(Ln
zIEp*08>lHQwh7fiM$#N9`A&H2FOp#U4JA_i*A5>CMlpDRFWp|`1iHVv05*9t{poGk
z<88n#0zy-Z&ABJr(3;ZV`tm219Oi}ibOE1D_3&avs5!Lh$a?JKATJp~EU)-lG)IiM
zXWDRT22?LHNoT9xIVJa3t&w-|uD3lg&USnMDqDZ2J!!Hq+XBMSM<wi<?&LfdK0$0e
z1GHqdi=|_2${PK~_r)-(#t_yZIzRGg<3uEe{%+72TyUe}Gkol&@yErSvLfaAuZM^R
zu<<gTZdk-1?lR7HT4pyWg(!8v#>rD`k)N?1;z$b|+t#}HJ$oZTlW-Z^NQIH@bzp(t
zuXK~`DZm!R0#%tcVpxH4*KagQBnZDgG<C*@<6i$!gqSIbq{GV_p{Ae&PkZIQqfyzr
z#`awSgQwXJ0b>)#C_s3X%lhrP@Q3XoJVJs_B3GTf?vx+3JKSjW9K*sTC+7p+%7fOP
zoCkK}R7|5u(Kqn9>1^V-h^F5fXUFmw-C&Z4z@k8i)y!WQp$c}<s6DRWY}gMWuhd19
zNob|^w#RmXVxdcZ<VetM!lzdX+9AI-RdI!i?D?0Z^*KDXs=QVY!BJ#LMZTe}%5qVM
zY36`q1&_~7Vu^S9nJnt;TNYq=?>0x|Cu>dTAHq6hw<$k=2NKP(p6dp#5)_{tz2J<y
z5p{By^bkm&SiLX_w&a!8HJt9Tzz!^D_!ZLX`fT$wCXaa}B`#KjGmEMqGy19NywF5~
z7d{I@csRimDw%dLnutfqUdCYn$!1aS&sbluqy^T5-Wr@NCKH;eh=TJ~Kp0I|KLP6;
zBu&cToa>5nE;Fx&U$J3TIrTig!5c&M8;VvBO4=dU)+mvQ>-@#9BV)iPiToC(nqM(c
zPVSDv+*(?eA3ZQeP3vEOfHAK$u(34MBOc9h+2B;t0OL!26l;(z!01Twz3k+g<0LIx
zGuX9b6S?LSIA_OZD8Jb6gjWtqJ;FKdi?d%K{bLcTS(r^N7uL5YeV|^F*i->z<fQZ5
zeWIPm^?c8TS!~N(wz;;eNgc2aLF%gWXHs0$igPxF1mBw!1iTxE5Sjz{e2fiKBpNOs
zhK;Sf56*Ypf4V?NS+ks3HWMZ6q`V$I6&OnGe}^DFIL?!C#ay(rYz9#SQ<W!DDy(e9
zbWI4}sFx>yshiYvlFi`3Hc*2Rvz6&pmhkUO6hdaPLU`;(bTufv4T$O}1tja^(DEG_
z$n-jnhxoEW#~zG+Ppi_z9#P|Evmhb3TlkCs<5%RRi<DVVAtXQsU*fCdQ@_LR6)uOT
zV`z6N?&zdU;N54tb!91hOP2hKNOlnr#J}9}TG$?rwe6*m+JB-N#OvCp1YSASvYCq6
zUGBAd1`#uFuC-?h8D>ST+gcEN$r*|pDZCK>a;Au_F-K`8IaD$|GRNg1+g!TR`R2D2
zOjQ@S)*kk)_4QWv>zD&4){?EU{X+Mh2i9;z;aV%aBw|Lxy>^dXByLH$ARz5{w`RNj
zp3JD{VsWJo-rngxIVW4rd7~2?qStRt*%kS1r3tmBy40K#qOmzH7WgWn#zo0B%knZO
z;f;YaoM|@(WU3^`th3G(eBZkXVDFAD<&PZFFG;}YW}w~JyGI`gn$B!nh1t*ao_b$<
zx!D@3)|1&{q%_Z-QOFl?8d&L6CCT^iU6Q4;4;i5-#}{)-Q&e}-#fla1^POxA$QVpv
zqx4hF;=Np^9l0=x>&)a^5EwQM5sk%0|BZC%-O=M^AJWNMbl$vA*`SeTSD03r|8VAW
zwe$N4fz8(`{@?uSEjUAR)Hy*csG&#l+T|@vQ3BNG8Z2_F-uP4t6_R3tnNbB>S?Zi7
zt=1BXu7XaMHpz+h?FV9Bv%_dR<D_D{8Ib~ulYSQoMu}^T!>0EV2PR)=-x?sYxh!@9
z-NtGz7oz$zW-@N#W5)AJj5alN!F@(G$>CQ-$Y*@A@0>t4>L<2!y%(ZAqD#l!5bV8^
zxH#aeEE!@VQ$V%q;B{b3W=tl!W<AAGNtR5nD6;su+fQj}dEU(+b`uaz8}j;_8cxTp
zllAY3oG`divg0H0^^A>c$cVJE+jBX2?AQUlkFZSCl;f$R>8u`JNf~JJYI)&O8E3u0
zdtCqNSR>{9=QmXcD%DriIYrfCcZ21Yh;r~w<dF4qUyyBi20<w0OBf7Yrx>L#GPukW
z{l#S~$j@B+D8WS)=VXoyryiK<FRdB}CuVo8UpN-MuG*%!9+yc}hVQ7XXXs{Lws43L
zpx#G2$VEq)C$0_7&LU#hCg3?z6K#r+yv51nv4J>)NSrwFyt+ERll3&DQl6MRrk<MS
ztbTU_oz1Qe!_i7PIg(oTo|QMp7Gp`kZD3=!o?hRpieopHAI!!a>)li#nJmSBW{4{-
z@Z{^)^IMG^0`F1#1c;h6{KySybJIRRL>@Pw>Jrd!S~`w-akyMGf2tEolMpaor!C?{
zgfqIlfbMzbhWEpc_=cMj2A9WYb1^*2`5Z%DnJ3k**?y@~zaM<-TduF$Px(~Ea<&&b
z%7UI-USp|8X@c85%j{M*NjRXvM3U>>qyg@)PAV!M8;dEi51!*gO_bK+5@oHSjcBvs
z>UWK2IybBt@~j}yieWY$Z2?yQ(F#!6@Kq|I9V&1>y&@2ZIHOJ;jR3tOH<ERWQI6a7
zh)fWuF&>ryn%11^@@PTwqWHd}w0!(r-4q^u)yCZg{I&G+G*UbE5Z$?!Vfrfkz`Gn`
zVj*T7YLY}6l~cZJET7aNI*QQ7zuX7zWKa7s$J9bs^Z3?7;~_&9xhNr`ikBq8UCYnR
zLkFJE?v3!Kojx8))<0?GC%YBQbq*opqPd8%aL+P<=EboG^YvK2^g7t=;6Rj?*P4pW
zFYtZmE?r2^=vMV9R+cZ%ztZpq0^w9;{^LERU#~oDk6@;?Q4_*O1xY@ejYi@A8^D=q
zaQ2anA*EB=?!A2~`;okR!fyT3lN<9JPV3?nV!C{6$`1tZ(e5F#6fLG}Q*RURrN%iK
zH8{OEfsPR3KG|t)ZE~h6S{0IVT@RZT7;v-lgS8zTKVH8DUe$Z_&Bl7my1ZMB?Ep(4
z+vVdo9WLy$O_m^JFyBag>$^$H208f)(dZ~yr=8aK_Jg8nuDX2rG(up#wKPP$M+sEJ
zmJ{|ST9bjiYoa1VmIc(Jgb8t{PT7lRJRw*R$?H_>XMXtYWZMZR+YlOQ+YZ7#Bo5@M
zF&g!$1d`s*TGVseG4Py5GhwF#u6qF$=2l2s4B!tx>i!b0?sUOMz{JV?*c}#wDRj@Y
zPLKZS3fsj~3!y1X@P?2EQoWd3$^;ywWM$*Kf)QL*80gDhjn!K;6uR%KU7hI{Jh_G5
zr%~tdks3_8)tGWv1H-djH@f#FZ#WVA`_$I+>hCtlrc9=t-f`1oWj8Bi(RXLC{&vV!
zX2vjbXIRtq#{M;&pu!?3fs81zJ8ti0#0xXc`@uAbTC2Y0WgZ)<Vd<eg7nAb&l*ILg
zo6*tw5F)?R>s1SR&QoRk$*gSJ4{AZ`gUFE!;6ReETq>=O?YV<Pb~BrG9v8fgtDqa)
zb>m&MgO>bQVcKR45>{V4BjmyJh~(Pko%mguhFQKI7FfS3SJp^|w=@oS6%1Sj>9-d8
z7`sH{8^<f`&BuXicK24cGo;y=vi9JB5*Q=kM^+ddZtrFUM3xf5?ap1ol4HyZhnYm^
zh|1guz9%f#C)0VYi;o=!zXk<j?@sPjedHy*7MA9mvYsLP6;70NLJ3=x-DTGL^h=3!
z!#MBGN1J>c0go(keAB6&eQWljmMVlU53xr{TzRVaZ}A(rBbdRiY1<e5&mV*VF|?=U
zWaBOBX!Vk7AbmhpHAZRUFV+NW?&gYWJ+9m4b#sl)*UWIV2vYRgpMq#v{}4n{uLtD2
zkp`c~Mi}bZ-<!;!dxHE%&eq&1Ye0j|O755o=X~AKaCXb_6O)ECJ3VhxfhxuGbJMrl
zzVW$BPmxZv`W*XhF~aD1+`<KnZ?gb73-?x9w1W~I7A2h36T>4n&o@GBF@9kcG~1XQ
z2idL;gNn@Ub<9{}2+<aG?f3ik@nOvog}yQS^eF2SkgDQGFG(5$rE@gBG#@Br^pDCI
zD=mQpdoaEm>s&xH0!uW%+fXWehtL5|vW#0fyY06~)J~FRK!nizK1(Zvi!b)yc;L}s
ze~Dz~qW1YY)uE@8WinwJLo(ZKonh9JiKd%|uVm^#5Iw%x01l7%YDN8tGeBqAUA_$o
z?yj9pu_D98mGa6hrwLb|!QSzL6(NLy5SvfZbR>R@i(WEDkw=)X9}N4(g+iGs?j5Rx
zttH*{in^RLgAmxAENSU3g-ELcWsXAi*VHjFIGp?H0H9koMgfAUAv6>2_*`=dUbn{+
z#ncT(uco^_`%*Bw{(kBW4wL^#yX)3a6Q^F0Ol5!88*d;;>$F!|B_-z;(gaaCEgyMi
z)0wu&4shS64b%z&;~dD5yB)|pe^^O`-WW(lQ5vMgc~jw%#8Ygmym{_Y)R8sVZcL-t
zg){+yme0H3P%w|?8q&zny5mkT@c`%N3Y4BBW!QKvo42v^t{vO66aD9PLEJBmM>t#3
zuBBvN6CSN^Ju9RMFk|ehVP5U}$KtIr6dod@q>z|2nBv&<U6i?D`@kcc)2Qo8Nf6Hh
z!!~vf2z<OKXg=%5wcD$(?_UGBvP3`5yANt%?1_hH4k0ap>*^FS?zjw@TLShjVTqD)
zw~F<O@%@gf;NlUh)-ox_ZYzb>v_#mHvq8eSORXAWK4*1oUp|*VrsgBr+o@H?iH+LJ
z*fQxkQ14}nE4`h`6|P+IGBYPHw*SyynEm5+^<2?$nGuIl2(wf*xE@MHvAxocMIdmi
zE@REfzQ?roMo(d*%HZi2CMpc_(?^EsKB&#rs;fMU{q+z~7u<+^N)lhD+~LY&r(;*4
zCf|Kb>_pwA8>+p07-`OW2Wrnm$b^x#O!H+VctdP%w|*B`r+eA?kqA~Tj}NC%P!m%l
zDDzCVpZFrMfI=1!Lz4?Vfk~~<i!M>!NB@c7H2=uEu|xnBifP;sP%u~Rwf&Ulnj5(?
zP*Dd*vBkLq5j-a<1tM((s$4>H)U`{h*P{81PcPRT`zV0WSdC=tcOmw@I07)W4!{sn
z?N<@88+<g~BwCxK!@30E#Fgs#gZXg?fm0T}eX?U42{5Lbj}ZV7pePSSwq<QMChCnb
zul9tuUGReiNmXdZ+Sa{SfpjuMJ+f+l!u?a(X$q5H6oG=;N6>M9{S%?nzoZR?P}L+t
zzEpmLJLBeo-_1Z~QNMEHK=`KEGo^5)-geZgxzqj2MPv6NuGe{eTZJByW4vIy5?kkF
zpT7yT9WFFXY~=*`6F4cvGF5*$YooE5@)}|-r=2SMPC-V)evYVtjax9@NTn)f+bw{-
zA8V98cF+J=ARngZ3hDIaL>sS0Y}SmK1d;cqxIlJpsWWuitg<od6LXgqa})Z$C11fu
zUX?G3XQ;Isy!g+srklMS-wfhidz{3FIn+2{?zSG?Tt9?}Y!WZ30H)?f1~DjwXGe_N
zX}p;+1}~;hL4$9Di`|gjBtBHU6^_*u7~XhV*zbmQ>LG|nTwHns_;7Vr#V5?tw+?G=
zKE8MFthM*kE0HakES-0cy;3>Tp0pqwnp_hym4fyB-m{zD(Kl~KvgqDMBaXy*<X?2S
zSg<-eThdgt#oam7Y2C5z(!Xe{?z~#G6*abK-Jzr(ziNNE!|}vBu61cnu>rxZN}|s(
z#>u#7qXiB3tO*s1O(OF&Uq2{Scgygjzu^@zFdwtEq%r7lg##p`e5{Z7_b|t=JD<5s
zwtSP!A<p}e;5`)KcW#?_R6O^dw~MbwI}f%Vo21Ewjit1am<Oxis!tV_&}>91J);YN
zh#Db@utGYU;B@?SztE8i(NO@@HwqklCP*@P_d+<D!_*(7wI3O6AzZ~|Dwzy9$1JEV
zm=qsuY0u)s7BJ%Wk9JzvVyqjqz73_vE3rm7HRG8lFeZ>b(LoaA<>BlCI_@vhTNLxO
zANizK;HtUrQ=&(4n5Jiv_znu;^W+ZrjAL3{!Vr+FHG8w=!aF+>M4o_=hGOJ_{N?$O
z{i%hpWb+rn$G=~VNk%!45w6mgUM&~+9P)w20>l|J;uxa|UK;nNB~snVbn$)8|M?A?
zRc7HZgcsL?ygxShU0Xv^Yb`v%ToBYlGqpanXeb^~)M5~~xP}E#G>u3+426~5%tc5y
zOGlFUzM7zs&eass0p-eJq?jSg)3s=KiT@+PMLD4049agCt8&iBk2YKp2+!mIz>Q!u
ziEGIU0nfsL$s4Zs>)u0m55Qryoywc_k!^rX$_tZ<)0{k5pUV8yM0iuFzrA6z6JG5g
zqqv8bL!Nd6>MTaTqPG6I%$Qd=K-m2}_0nievF&1i9It#;|M3bc^!Cxbs4Cmjfl41}
zJh*F&#=O~<QZ&s|$GK;o3hm#lS#k`Sra3~JTIWN+=p)Z={%G>?l`mxTuk%!h0nhtk
zE~jdhw{y3XG!Uf;>Dw3G9xFrI>D*Xp)vXV@KjEGm`kG5x59b2%doStTV66(n*_dxx
znh$3Y;-Rw#{u`5U`q?OKr9z<>x2FOmx$XE&=~%gKMic3XnlLFQib&!aCh3xj?3>w)
z3fU%eV>oA`PgdaJ*GutjKJ#?UNSDx}{3a|t2_U7lXJ~a6OJfN*7FR4vw7_V!nC$Ou
zZ%^g5)ULNpw}vyWSGZe09I`~KIH_n`8g#=OSEirSYD!dO%*KYW_T$a+t`c(%5rO3x
zh<-8aZUKKQ=1fs|9VG33P90<hbr}gdqtUCAu|0f&^@{oXl=?BVP>ci(4ogM7y^e5o
z#tUfdSdcMtyzV+@Ag)Q8!!q&Ou5_#NYBx4f?n}#5$yi)2Qq5^PE~>eMnA~t<d&1gu
zjq(W&{E}h+Y9A%Ti^P=ZFw}f%ob6e@S=9~JkaJ>uu3Oi|CrUr~z!``=TfU0oS`lIu
zP;nLC9M}|jDcif+f24Sl-)@OC1QpFL>=UbWUAzNK5-Ux9#@FWTY%ou*NvG<j$kO&6
zVHd=^pN*j@{!%6H-L9FBHO<VU2%OR;>_!f7$9y~;L$7EyU0RBnzM=ye;g(Frce1)^
z4`cm6Vf2Kew=-s5rUjv3Jffsf%W)QE88^!LtN>+19;&{y1qIRhR-q#pkIJP}#K}5K
zU$l2My}0FPGgRi~aevxP<5QIQx6b@D;L*Vlg^#Lk=5X**C18ACH()g2>x>3Mq`TKY
zxCvP1XT7VeZLT)2tKM)?s@S3g=9h0Pq+i2I*ta-=wQh2^sV){D^MiE<_89j$e=m>s
zgJbQ^ed%&5MtlQXqYjL6=~H4QB9vSfwK4YPi+LJ4A5P0zxZBOyz4x%T0nwvg96T)r
zuY`#)p-WU$=r(@46ZUj&1cObl-P}O6`~p?*K#s=ghyiu{&Z_vm``6EZ>oaLI_-mpY
zDv3QTUcANZ{&i6h555ybC-2(##jKkK5ENe`39eVB<36HZv$4c?DoEhkNQUp5xoW9+
z{uaz02BZ=!uRN1yl6fPD{cHzOh1ZdVM`ubdCP<vS{mi0}*gfsxXa||Mru<=MQb%5G
z6(7sms{`JfdoXhrUSaNlC+_~LTt7wY)&&!NL>o^!=kCkf4GRynVy^g8UF)~Vc|4q{
z#Hs@cukuVir>@htK;OG(W;|cl?_H&%HJW=^0;P$sX@Hll|A5eu9pM1-msxwjT8wcp
z_{dF~N#3@PY)9qM$9?^xPM(D|{!g`NU^3JfA9K9AUS!l)sH>>^xp`Lb=lgqH^~R$}
z*)`fs{!l?0SPIo&4<hZs0UeHgwDbtwbzDa>g~dgL>?;22+Z$V!Ud((m(~8UYh&4@g
zI>Y7gQ57F4kc3@Q(!Q4_{ra8ySK-%`dtHe)T9yzouBkg=UPlJ!XS6KLELQWqEn`{r
z1M}PE&)O4C=;!bAwOB8Bg5kMG-o>Y4L$Q<IlHWbypLVR1Z#!o4T7Qd=K8Fl?Nl(0-
z##xcJ_con@4|YAu;`WrD8EtNdT~^t(x}7}Qj5d2O7KJy?a<~;FoAssbwLgOtyA>GO
zl8&C&LPty+uG0Vsu(#b6szsM(BGRUcS-Vpdv0k-Lh#3oADqtrv^>zJxf;Z{jBghFv
zRUCk*3-hP6p2i4f9qnaruZLGgpbjB#?=r^<t=;P{a42-qa8Ng^5g4UJpZJoshv!3u
zy5zcX?KH1LF4JiX=a0Vhg**E#_bF$O|Lk96y*jW>_)grsbCB>ty7}8a_LtG|AQ#QX
zuyOx5PhFYOJ=*x%kAz8n)GS4dI38-khHIb9ez}4DUVjmF#qy9FBlh9wEkT9qi=)dl
z9?!jPP$Wqdvrf8S(Y8$i)W%$XeRn~(P=Z7>$FKmIHeSHw6d;lV_WQj)RILQzp2SNk
z0CYPk0_&;X-+Iqwgfk{|#NFy+XGSi{^265=d#qyH^f}I$C{s)-#hF$|GjdJWDhMPm
ztZ;mUwf=Vh(xy%u7fZ_bt@yD58UpIdGkwodle3ZBWI9^jXTN<Uqdg*M5!H@&u+%b%
zJEyi1S9bx`?i9n_<i|R0Auw6y7T-L=v6bMMHT--a>^W&s?Xp9;FP`G%VP-3BbIe+W
z3mt?U8*2Bp?9&9*!RD>E?p~S*Rz~*!SPYxaQ}Z;R+{v9l5dAZIbmCROPB-C;Kn_7q
z)6uOn!DjN+ozvV__cdKbJ<!g@T*$JrxhDJiL#?^OCB_<CaS}|Fj4FpK{FWbYqA||X
zGlKjjPh(#?rbw)p;8WS$DqODJbol|1(;#lXptuj%W>NVP)YeE&bM?$>z(Fy5;%7vu
zs2kpU_HNzAny?mMyPQzZe#GBO=KzEa=yCFYIVmWQFt9{W=wA^z_ut5pLCnrK3xoVM
zbsul;puUIip8QMLz%@0gZ`I<X3F$m0mLtO3+g=^ev%QAJ3&k44ykuo(7{Ru^=b|5G
z0{hZ<yir%iJAn#anp6Hw0R7TXK~qxIO-gj>?NZChkFR)otG?)Tc9!C7z7z5ockM#a
zxc}X%Ef^jVo*I13*m8t+A2Pd)HJG{AJj~QXMJX`xDd%2s!ZS4Kh;Eg_jNn5?La<j@
z$<11H>2|^~sIvRC5gE_?YMZQ%`9lO^BlFo4oQp*<Jr9i}OO$!4b@H%UKy|gydl*S=
z+85U=YTJHEcq-t2knGOsGjt9?D{(G1uyT!lWP%Le?KF;*XfKaV{?<_XAWyZ7gYm;X
z_r`qPR@}BaB0R<WDcTKBsMUEeJ)yw1CA7&Xem~85%Wrn9e;-^JsX~gxbuD^@jOwe`
z++Pchl?#wH!8ky4;fMp5GMyJ*>xAlq-+e+{W17077TQe$<*$BQZQn*oFau>Y-XSr5
zaBpPJj<A;CmPuc9n}=zcoPc27*?oYh5!pXiq&kQ)&VFYWy=+Iwpjv8iSHatglNY+{
zg>rlUGEG;Zp{#vLs8f~|p#uoyd`xGI#7BK}jF=^tMV_S&&hErt4idbM#8;&8UyBrW
z{kHi$Eh6_o5f}FNxwnqpW{t$+xI}V;_;80l4lUo7W9mLH5p|MPuV5hbIX<cZroVg0
zJnm;GzSvpoBU}Co(9r6grOeEA*pbOG38;I|NO8sDqblu@X$G(cQDTBKBZ`SP7k7DU
zbzER$JGyTi48et(IuKjn9%qT>G|z7Hxw*84mbB@huGst9Nad^u=|YmaT&-jz;5>_!
z2xg4R%ie;!XQatbXmh&pTyXKin&wMKHH}PtR|~Gbqg<483pbs>t;_5`-C6okeQAX}
z<>3A(nwx`OpyL+UOaJz0&P77*?hkYeW_}$)34qUv<6U8~$Wk$A;%|Bsdb<~8(y*ui
z6q1J*zS@x%uhCnHb7Fd4?YOoQJ96T~8{U}cZ{74`u{=GANB8pS;KN9->v6v?p>Ow0
zq#`3YOuzbs;wvS3mp9aH4RJZHC7;zVr89X&rH5nGyVFb?M0(Zx7?I|NscDIyX|~*J
zFq`GMveyU8mvnWezv5AF3?}i;a>F)(JIUT#BKUzoR85*2YpON8Q?bM>Hv6%n6AKl3
zm$}L71IVm5wMcEC@AjCUmc&GHqZhPx2}b$0u3u$%lCv9;QgE7y+we?86Ev#A3Bhl1
zy+*CV&bwo-xr9Yw3alPpThUjx@IDyX9spuRoYa2|@=O>Yu$SU>WOxLSD}qUaByLUT
zW}?f9wdZM`mdE3{t)6MLtOS>fh7nnJSU0vD-fzSCo~ycIgoMnJ5Mp~cAI+pHm+i;h
z0f<;uF;RK|U_-nT%XQE|nlEiy)rL|@w|9CYAoqTDkskNGoECTzdZ6gFDLbk+QAUnC
zGg|pIyN(M>)qd1Av1N@>fN?VY-Ye}&D9THmc~ROqDnF`P1xhU62C~X#{_eDc7704z
z>9GMB%%v6MJ|DFUHXlH4_3^0rbZav+vMXY}zyU8v^kS0#CtdXpwO{TxjgPShYV!xw
ziTW7tV}Bpf$QQ_P{y)UMcQ}^+8$NDjWJLBRM97vcBBQd(-m>>z$%>4uM^Thf8Oe^a
z$CI68i^`sn5t3cs>u$aKjPK|B&+j;X$I&069?x^%ulpM3d0yvL6WN*ng#{0@YYUds
zS+N_6>FO>)%o@GE^tQ;c4wf{{XKJkaSiUGiQlqiV7r^tPlZza)GQ%+Td;#~S+aYIj
z@`LTi#Dy$?<zBPCn>Gy%P>?u5xHtRD&I&iImlqss9(s_XM-FG0zYBn80KJKz&9yR!
zs%O(oONHC|vXm8GybL6dtv9T2q8$+E{1VV7kIsx?=4nf${29gNbnP)}E#U0%HskK4
ze44Ip)P8!oVH3;bJKY&4y8{@eUA0%vbL>Kva}<euyKn>#Qveyo_rZJiP~G)iS!;;a
zeTU*yuxfaY=u5<cU^(|g8~<qwZC&UtCGl~(61~@s<ZkZ&tP@IbL8c?svmGZh4cjDQ
z9KVs-n0|&arro==Hh7^>%BKif-(rlOVd!b#)-C+_f%%JYUx@8PtwqY7bZtye4QaER
zGhy85J{BkX^762dR*arn@0f06l!VC!A}zg3)yq5W`2#`P5^3fP2v)?Syl@bD#v&*!
zarnNd<JGNXJ`+O@4&fq)q5Nn2D~A`pNW)Er9KtvGJyFqLAai>^;ZjHVKY>^1A!gHS
z3~6<TUAG7SeD}vGz<(MFd&x|<N5!YSlEX0O_x(3!1>@XGJ5LhTc~3z-Aq3<VlV|c)
z>^D;gxux!a@6-5knN~O%^l+PSF-Hj3oV6}3bJR4taF^Hq0A$-=y;bDQ)@|R_`ESM|
z0~wv@7u-7%_#hKdOKwHym#m(F5zUoOX&Q3hj8dh|&KrTpi=5lXwx;NK?jf5gLY3T=
z;~4}ejkRihoXhJ+%1kOd(>m>5J7C9tnVMY(7>G8HjSF7!<5DARX_9d&y0)Tx@YJ)N
z(U%64;rRKo&>P%&CdA;gfZp#kRqvIEZPu2r#K!ED@%#KRAK&x!mxp__JqxnGkGq^7
zF`B>w9fGM;=jq#}17cWj?1I}iNn-PwdZzw4>#*06YeD(#EJi%0hV**4#FMu}ao-rv
z290(u1$dR@f8x7RDdeg@qgZ$~W_J?+m^^24x;+5;=;73x<xOxU^XTao?|cVxmP14x
zmaYk!`p;3$z+R{=>&OWaklP4?RGEf{xy?F%vN8+}WrU8w^muxcOhg^Y_f$DutMkwc
zJ<Sf)klj_>J;cbGDed9zEqPmH#PeEV+PQL!YG1v)Ttx?xKpr#2=h60Yylshw8FB)B
zjAj$vvSLC0fzz=!Y_;=k9X?(u&@DPwDX7DUBaqNqtS-+l%^V*oHD8!tYCmvN!ygVG
zGT3~leJmFsKRunIR2yb~=^ou9AK#iND;2L7eucT{d~Su)DK{oWxOhHVyIR&r;6Ge}
zvPDy<?E5i`wSq%wCZ!(;(O<_eFZ45A$JSZ>4YV8<`@jBqmnqlx`20=dI5P?!-LC7Z
z*sNL)<uF;}tXhgqYQsf>sQPzKibP#V<|e4Nc9}f%@D_ds`bM-Dy9zd!pzo^joQ&Nz
zU_8m&9cSfRfsbIHWj=TA+b)ax*+Z{&80^EhpS1p}mYN!U60^GY1}eh&q=G#W|15Lj
z)ejZU=Oj<#+h;RLa8xg)qIgzs0scg|%NZ?xeWw%^T!mkK^e&17GjW(-4}H9Us_nXK
z8X&ednJ+Qkhux(*ZOA^``7E(Aj1q-G@bizA3}$tHvE69ZGYK&it|}J4y#Plx=3Nx-
z^gQ8aHf%#QhUK5~DujvIVta+<-N!xjFg;zucWb@AQl^wxtn#HFZ`qi8ePsT_B2)IG
zq&&06pk=XJA{UPLM~^H&5%CcT&Qjr=Pg*=VANh8Tq|0Ev7TXZ5Z*EMZ5`u&LVL6JG
zr4)A@ro#)$?CkA$Os}t>pOaB(*&goCxp>pKSHZtTq>aksB~gz|zr38v6LF0?na9_6
zH|94P0*nV2Qp6@{ADfK;RD|jd3R1OU$ly><6W8h(n!{dm=L`uoo||AyAN&L$4q>%W
zpm}P)!C)E>?idRr5l)7?1rzN_^M)8(_x#=>BQZhye)gf$sv3}f(@@={p>xxRtc-?O
z17igJQ%cY6`kELSYfbC?p4oveZ<&r0-c3h(JA?aI!qSK~?H<iJ_2Ow5K6zognKtvT
z&`VQbX0A!<DnJn#0fqPnRQB(mKlM&cngPm!=X2Ltt`5diikY&3H*UZ2s}3tgbqhCa
zPc}wE1>4sDxthb*KhC69!=^j!)GUeBsvS%LOcPdzJ#eKvku!io5su^JvdK%Hj)5)=
zXFN=z!8ocdG2UaY*g9T(|A_3x8s9aSgpIUdU_|F#Su}3MqSs`kIc-@mkN^48As(82
zZLo8`R9fh-ZsQj+%8(TbvoPFxuMGemPUJUhkYnmkX+z(-lpjLbyJHm7DEc3D@L#Q{
zM?NkSY|tE<5f)bCb1*k=r)Ojvi@(Q3BsHUA88A_H*&7dYS)76*o=GNuESk%8ysixb
zHl21jX9J+Y4cEO6W>oJ*^krO|iAIs&y1x+A@g?<xMlE@)O+Cxv`LVD4WcszjOS7-;
z$c%ac5EN=>+q-jh8)K|<lS@I8wWYH+4$nub9ta%EZM>neJT0~?=MNS0gtdP1dLVfa
z@NKvq-w}U+OpNr#qFJcD1WO&=G)+c^bSQH#rA~J|h?l~zgZc4%zWY+y)z$C)UmfvC
zlFG1Xr`bOQ3_0?zRP-A|w)=tvyWf%dXOkvjip?nHqb*CMJPp~8$f<zGmpf|xHcuv<
z2PLCI@oLGNv%re!!zQpfSN(s(4}V=v7CeWC-769J*VcC_G9S!h5ThSz7h{zK#-G#^
z6mzMbqcT(L7>kaDeY_5@(Zn9!juS$kcOT@FV0nmoTTetBV`duR{1k8`Ce>-Y&cJ}=
zxO+k}XD1t#Y3O>uW6X%j_2(M4sCXZwU~^BV)i{*jj&M&x)sD~s0j^b{wkqw|G1mB8
ze2)8jhDVb){sC~H4eD5s7tR2!jOSecG3-I;XrFc}+(G|{QOxE$J>1Hx;{_s0!N>|l
z|L=fJo%*CO4T6q^Pi}0!JEBy4#(7=G@Hh&b#dEX5(9qiCEVCV-E5;$QJyy^|Qvk3F
z&*oX6aY>YPO__xKztM}VDPAW@@hweAEfjOOwZ`+~Iz8^kINQ6Mqy8V4d{FK&9Xit{
z;OnxMY;VQ!XB#v=*eIh}L#_90y4y|p3egn_yLrw(UgN|d#&@onZE|MAD}UWLlmXqx
z@+MOGk}e1APmjjS^|1vyiRnwZ+F?bbIT4feNQlKjnuKR81oNA3Twtvkr(4~XA$fCf
zTzUUZ25?Z^4BI^!9t5Qj1a-DEZ53Q*{phihG^=*CG;u-g2cu#Z$^=8*#X&2VAoVTV
zxC2P!OC4BZp*aDlGq#kpc7g7jJiU?(cccmx2tONiQ02<xYn5{$S=y4mO&-l{3n68G
ze-##;RfbY-HcR>wF^KnGagCumU3e8cL7)?XT-f;Am>9fP8bIX2uv=QHk+QJucFAs}
z^pQQzvG$O?txxolwzyvC!@{!wdTuQAxGs-8V{?-(#1S_XweN2+7C+^?I+}}3ShaYr
z$nXV&<@C0xwQ+clRMFbrLx0Zcq5wa3!rw=hB}WwXDcJ1&e@T<<D16M_$VW6X*St;`
zyzrlOP@4uEX=uEi{%7zn3z5sDe3{&!D<vLx>Kc~+#*qon4ShGdE5C9Bk{)W0PmT)L
zB+iWrOIE1z?E=9`e(^4^s>jpu>Di;79>XxzIliw`YDt1fc8J8D5aB6<>~Qt<-P3Qa
za~U9(^Um^Ny|4MYI>rN5P~6$%H&L}hQGnUq;?~K>uimRewe`I}cuU-GZt~pugRV-L
z84vgEtkF~*72AZW*LUNOoLilw)016fVIg4<f7mtfD^bkJXt4tfez{|4A&cw1*8IWl
zCdlKBr=pr~iae(V_WVAltN|H}FCB-{iL5`d{mLw4NwvUh$DPP}H(*Mx6%v8tl0IE3
z;iA4pZF4}4=YqOcjhDUq#;mIHG}v4n-%G9Yx=ZG#%898=8$`Wx32hrDqk@OI4#fXR
z7Q=$DZ8wDn9)WV2_oD#ULgVVogT+NbvK)+o;l?Wy-2I*huy8uu%i(j&1V}{ANrw1M
zAD+lo2-)?gi|B(YMKSs~$EDmLYe7D80=L!F=u!j9Uj>1pLI?>~pPTj5e~KQ|A^zut
zvSqQ{Wv$Buu{46`I*PpS7@lM*oq&Q1>sT-C&l`3b1rJq>i)P0e$qL@oAL$*rg)#(g
zK@?+EiOyb5Y+SprwV476Qbin&@0BVvw6BwgZyF~`m&R**lgfq7b*@aqY$kzIKp8m+
zvn;RS3kMEl#i|uGBTtE3C>EAFv3MIf;A#%6dL;>*F)G%0p-}(Cz)DnB7m{@hloU}E
zw9rwA8u-p>2Ph0P0@S5pCO_jESs&1K>yV*$bFJAoV1AhldcT?{9!D2le|rJoSHF%}
zob-g&;adOkxR7Qt2L92^(~C<iEWUOETl!^IuM`vgHl~76XPn<hL9hzwr%615BYw6M
z>to^VUmy4OMYwve=3e#xb%ytkPli2Qma14bxQsvBnSAlfr(q3g*wB8?<=KNrLV((H
z!^%_Ib&!)-=P}rv{E?=kQZN>j`!G*__&8E#^*e8eeql%NYI)5lRrEcJ%bSIw=^|*E
zg%@+0#b1Cx4~wU#;2gIUKEx0;Ueh}9;&cq{K;k=Jq0x`xDXF_$sQ+lhn*Wmgt$@-n
z4Bd;z=*_!GW?UF%2%avpgakcz88bxMoml{1GF)!fQ17jc)mG9#HMDjp39+iqz3)8r
z?)=bsCJ7J4q|-*aQ1Wwrej9xF$?PgjYK1PYpUh)8uBi*8)SI3gRx9^IOb8-~Xs7LM
zY9#R=vYj@3E|BgvshJonVAspC4jyY9SzLD$iLla`Il@l7{#BWT@^Z-Gh4QxcbgKTW
z50L{DhyPh#la83`!Tn?$cxu^U$s`F9A8&z(hrc)%vfI1S8k%5_t1pCbe_X0i3QpuY
zVJpIQvVQgF951SdY-W9Ii*w4dq}O^S5FZ4CLO8nL)pf<8#j1kI&KKR7dHl%$)X35#
z0<2?2du9!^V^HH0oyUuSpdX#9_TU!qrbou_B!;_;)kFiLFXlz|@vIMNlvs8RfCphi
zUy#xFP|DFel%IL`K<r59MIt8y*N-Nq5ulK1nDt92Fv!~nM3CTx0tN#Oa4sA$`j}L;
z<Th~tTpxkmg-TCX(5$`jY5l<I8+hf3x+tUZ!O}%rJj^efF#JOWc7dc2Ros20ia$Yj
z)iNqG3C(ak53_4L3JwiflXc&(kBpMQ`N)EjNHZY(^=w1QpM4fn7XQL`L91gz$#Scs
z)o)IRJ>%f3c-NYVKZ08WS^%tC;^pyrBLsh#cZ4`P4O$U$==aD*98yAm*6eEd$P+Dx
zJkbV1zh7GJ+VRN)t{QhXII>E{8(`<gr<A~S8nA?vNB8G@GF>I<Ck;$eP)h*bH^T;A
zuC@ebeq`gkfXPNZvY%5_&neO(XrVj3HCy#gQp7v`!+OvaVC~*!JW9#U4iTLGJr`5k
zgq>?0D<dFvPYC&{<mw<e<d)0T1$s?10;c4-5VG}Yxb_aiFf@hvIt8~Z+vB5=SuT&c
zPKga%6Z1<`AF;7FOEINXk&*4yR)5WJFtNf?N{;E1jZc3eiOWKgU@%{Y$La1!AoN<o
zpPB<YdL%H!asfEs(|m5Hg~=HO`Ey37xD*1mpg9_2pgO!8)BRD*H=FK+$F^N`zn*)R
zt5{y7Q8WTEoPVnBP$z|uV>im)XqK`4Ig+X;;Go`?+oMpZkkzC)|GUvdW&MbmpYOmG
z7Jf7f+2sLGXtP}v6v&T=JB~3*TG4SdckBVidaEF3YiWoaA$#UenKjg*AF~+HmWB{e
z@h^hIp}mSJG^Wm;rGBqElX+S}sMG>=^UX2yN|+p;22euh?z3xK4qD|lYI8EtC)G1$
zO`f|>m_9fJSy1jvA)lL3V&?AmCW7|Wo@ObgADec*n6m%Ou^BwaH~~)MH1A+KgGL2|
z4Cg9T5LclN3~l@Lbpwh;82l8&k__wHm)Ra=1Ajk;0J3ME;&(1VZwB>>rulZXpF)PT
zzam`YRv6H5a$p$F{|r`DsrTDfv7?tq17PjuEkLoO0Q!vvr=I(SSWlNeJhP0feoZj0
z3`mRC7&nNdEF~q6j3Hoz+*qa4yUSe}GE!UxJ3YbQEIp3a$)A){aJzl<sQw5(iE>Ry
z&jWbWQe)Aom01`*ClPxlu%(GY+U8=vT-zg<-3etkBtNnLyj`K$#~+5tU3M=M_#JyO
zk9V};?67drALFZ4`g5O1;lCYkqT(ih7tz%6;7`o=K)(N&-l;4u-udLcSe^MPc1^&Z
zv<meFJ;izulp;;qzhC5B#MCw*BCVkMJ!Rez4}MGLJMlyH7K$z_UPr@B#WB!w^iFS!
zO&ldJqWy2M9^#_~v*3c{dgADMon5QjYylTZ4)c-VNi^o>P0BA*q~$hm=saJ1_q#^>
z7D`3(+K+GZw3@wWPC-Gs8`SKuF7Vc@;d*uRZcFE#4zk0#<+9gZ+fw7(Zluo`WHcuL
z+wIy8JYl^*C*N550ApO^CL#41Mv_T6jEWH%>=MPaD+fy#7Q|x%flfZs#uTsYHXSmM
z)cO%daWxiJ=VB{7gOY^XQtN+S!;ZH|@F|MCm(KRjBfyTNLiQ#rtv^q|yBkQNBW8Q2
z)P9%a&TmvAoAbv76oU(R`XXe+jx#vZ-lQF87;TkRSe_t(=1IKtzS@ubfQrNKm1m(R
zK3RM2_kEGWN<@B<%6ggG@B0@j@*{VKA-VIv=Y-d{A?K74V@l#%J$PgVzlQ<dLzDZo
z$?s^|gnXc;u;$#v-)|EocYzFkPZAx;nTS92&XFI{kp=aCjgCk{M}h$f);k-ncVx%$
zl_4*=rTJ0+3Z>Zs@E)>1PnO#Mxk~J)%gEu&i3p2h{e3l6Fd=uI7sI!rXEeHd>GwxR
zWk5x8%VqR;5ytX?6i#vl?FFB3oF+23a15N|zvIlQOYj~}NsBwt{SM-gA1ILg@N~-D
z^3Q3$9P;B9Vf{aE5nLYUh}(>;p>FUVj<!ep{<=_dN6Fzm96d(z;LnpIhjr>aM8nGt
z;v+PFZV?3wlD6U9{{MB0xF$|ijXhYvc_MgTBg9ifhHYFPiybvX0OzayVhDx@9cH(T
z=liJg#-BIl9SVaHr)=tB*~Ncu6e$4+$fyW(sV@AvBM>)~4ZOnIt8M?zmvDwc?qDor
zs58}_KyE2B7SKGDKzMvs-E|!Vzna#Dz<sQN)PIE@*-_QVDXgwDp#STh@x$-D+hEy#
z_s;`_lB*%B+N+<W3U+h;*-@R~=&f?M>?^ars0ErFP=y_ZF>|74{+shEpSqTtKv>0F
zT(iWjm2-iz+!g6A;5!r%7FIr2%5(w&Unc_{1~tqAJngv0pP3@%Tbu$epW@)+QL)4+
zr2v;H7e{>i(m<$X`9G!GpBgR%=LYhSMiI^Y{lhf4FsXO&O8%U*-^mk?j7GvE`v;XQ
z69nD>t>At$7p>7AWm<nN8lvSp5%bDQW_2vEnx%rt-RH`i((Iu~SnWy`RtD*FbK(pj
zmlQysA>E>T%jS*WPv>9n=%3%pVu@T+!Jw|60p;JW_@BR`B5=SXQd_Ie&s_^jflZxk
z6*Bjay{rK4`Tz(+^2~9VZygZw9E+aY6SGV**P@`wlPaBry`F?LeNQ~|3}58y*I#?l
z@j$G^RSc>5kpf~uL@A#DygLkx1Gk16#8yBrh*Z~rcriCD#{SAjzn}R(0N&3pqB7wC
zPTRII{B!>PyxYINuvd8q)Y4-(Uyq@Bg|VAr(j$aaM9ZyEtDZ(!A><?=y-{>)3^if1
z0M=ShH-R~Zi@XJ-^~W&gC;!Tx3@r761p)j-R}`Byni>P~6S{cmGbeELK?6I?BogwN
z?*Zu2)fD=dTl~*O{(a@A*`dN04_){9{n-E4&ldXSgyv&Kf_11uV}O!1fU<@25sVC$
zD@lW0u6^U>pcCnlS&FT<D6z|n^`D&^ge9s1GK|L&i|;<zV~);klEA4YgfuJkndzh-
zfw?Vclg{{&I|+`YVoHe!{K3Ef>lf^ia9wXVC+7UupZ@t~HcKcnbuA4u=vX|doNVSf
zn=tvS#313pPTP9k6j%2v|K)5Ia~V*|P$TwY=90&I7od5>8Q+|WWWEJF6He|E1+28Q
zxFifTBZGh!;)~y(&Bhq}=o3M6#wkS{6ecWG%l*%SzXv^fFFy%R3Z&q_jL#cJ@V{^P
zIvK><>veu&|HOztZ~C7v<mgDz4r?X#U+-6e!u-zod&u{<z(Fz^rg=}+FI!RXfPVKD
z&=^OkKK#TzvVp@Gh3XHGOR~^ybkub?AEAqjD9uV2lV?uv%Fwy-^kVAZyvidWBXtrn
z8N51>tJYck5_1xsN9}Q8*Kuig07`boJ->X^x{?Xy-=>-^`~QBI2k>x`o)w<|x3m9m
ze^s#n0BM#CaV53`Yh9}%W2kiO#%f-S0I+7|&7L)BC61rw(Rm817RL*=u@7XY3ftZS
z?LB_8fVBN0Ro~C!&7x5be((YAU`?5GpdUb&@iNLv0zZjc=K`-ujdoh_x7~QELqI~O
zK}A6gLNA~HuVTwpoQ%M)2J0X-P#n((1Kj)S_`v(d)fFr9AX3t$ws<0a`9^3$u}H8Z
zk1<n0sdpR@jB6EcB&9n|?=WlC&8-iNt39I6$52Is=cm2itI7@ACzfRlIB1-1OsLTs
z^R7IJf8_xU)>Qy3ufDp?@s5CY|F4?qKhM-Dd89xPByz&}-?vBz-&`k{rntFE$_dSf
zQR}-#g=o6$JJO_lS4$0A^R%*B5Xx`wTUUdi0SQ*>n9JLHTR68toUm1ILHM6B;-BxZ
zEFk4eH?8B>r2iGd%S15?oxZ~)RW!hCj{EjB5j)$B-%rDk3l9|7)U}N&i6<VzY}+(=
zw;YEcm(Yrr&LHJ;=?9MS{sqs+yS#Bv!sPBwYkca2clEZpz3rb=`ls6a{l%;~G<O{@
z!+MJU$3H^J`J8ZjBRU)fm)`6{4hWB8z_0igqSY8Tg3u=D{9Xje;uUm2VK~q&w_s$<
z;NCOKsivFfLEU+Ec=ZmPf?EK$w>>7L;9>!xHS5w)fvqGXS{~DR3|gTb@EC4^_``LV
z-;8f^l$2s=xP;$)#QEmSFuWKLKpEW2n1Md76-cG4>j)GHlL_%AGc6{$GGxv$LvRa$
zJ!A~EOH54$3k?RyY#Y&h#{fJR%qsn$04Po?^oP!pzDcL9w2fj;1I0>Z7POlKlConh
z&?zjxV_bktTwS$V^GOlpN?ivq@|ZsY%F;#N8`oXyx9*AqcFmTs{}|rNV>pnjA4j&}
zMO^OE#-KQnIs!?DGThAz@VwPZbXySC%-+7(<<qR%dQHC8&G=<8jhpc<tR(0%5d|Y_
zVfX140_#374k!vCA$v&wQJHcNEGx|k(;<ouCgvDbx||69b`UTA>FW1yhx7YgHE$W8
z)hdHC)cMW%H)`MDsh|Wfu=t(L$3=MCWDt4C{an>j1mr;+ET?D1Yx2aij{kFzVu+0q
zfR5+_c$7}=!@%{M!BGGby0*6vh!@V1<y^bx<)@c9$`jtEJdTG8aSNirEimDDaM<Dw
ztngJt@ai7-#AW>AuL3Yt{vqHPkF8(d#oBD3<k5|S-ZKh7He02kR9$6|(X@ig76&4h
z`AR(TE@dBlb5h?{i4=2bWJo`T7%*=v0A1Tvz-q3-yr=t{{r;+bC&_<#AY0C3d}sfB
z9`|1<sQ>ym`=U^=K)>+*LtdB&%b9QM3Bln@PegV^vztR}<WjlgW{4MwCQp%X<uw6A
zZRwvK7W&L}LUw4y2FdQR;Ev0Cx99M>8Qp>7>)x^6$L9%^caaPt^copvt73Mh6EF`E
z>H#TXmvldRY~;E9oefiVs4;5Wpjn;EQ>ME`-`{cJfu1BC4U|gtY$XpqAnqT$wtyaF
z9Iz?HYR;>QXwl^dEO}Jw(GAktU6Gcjo1urtM+{&f3jJ^3COLDe-?K!-d^_nh@wUwN
zJGQ04r!%i+R!hA01Q1X_B`hF7cdlAwUJU&FEukD#Fp^ywSUia4%n*2;sICy<6|VU<
zPH`~dOzr;Tnqr-TyR3|4u`n=p(Wt>2dxwPnOc#m-$9*Wr0dHWZ@zgK1P7)nblWvL5
ziyKb7(0ZG0V7(0v<LJ5$#%#85R^rj!^z_L!mqi+fLeu*9aYJdos+^7<YA8pWdVc+5
zjYj{dmqO?|yojx?-AqgJy!>BCtiLp#Kec|^#2u8AG7%?&$Rriqw=*eh)7+J~nhych
z9UWj!4+}9qJsvW(i5Ohor=!`1XhRRMWaSgt=Q6Y;SO6q?CKU;%A;X%C+gXf&L{`_-
z_C1uxRw(|=G@{|#bmIsiQT2s91HDggi-@Z~gzi(#ZNZH5l>HOYapYfLuH|&q22N2K
zN5e!{%7TKjjZEnXmbws#0=J6TvPiIcbBA<4<$02ZkQ4?S#{0p7#wx8L4rjV?>L+O)
zx5}8(yQlE67ih(CTFGa-nIlG%<qaxFX<tHhb5*+a+Ug+`8AR2V(Pd0$8)7eJeOHxg
zw2pd6y+AQ2D~igYLtiiUTI|P7>j0F=Qu5@qFs-_qhb=R9=H@}&;o3sc=;_y+j5Y~w
zJqiPf4@NCIQMD59YUjk6MeInoP?NS)45z}d(u&qUoMselKJ%pSI<Ndn?i>x?8!58P
zJL07!46Z+ktk5rSF*ps=s{WnE{nuy6-7zFMY5k?!wX-zcJ~P9R=va&r^$*~hXQD#i
zS|-R)@vwcgf}q2c3lk|#NsQW@x+lGZeR5*LFk%Br%Yy4LadjxUwKeHXjwBPzY(e?q
zmlA3rDUOexJk=FXAb(id-rP}d^X5Ji)zf(vXxL^nNhH5E97%5kth<vkz?%e-O9+{V
zXR3UhXl#spaad3P{I*A(xBLvkHovtlxwl|{=KBi5V)A#Z<^?#g42WvxYoD3{C|WpI
zBeU4su%%mIBoMK06lP@igz&StqL$oqz3^el(UC(=o%i?cq(0Y|91UuuKVHPcIrlpA
zY@_?Zjyt6xLZjQ4WTuJaE<4`pIUhEk9<@*xM)dn&hkEtN-O>=}%Jw2V{z0|q@?X5_
zAJH;gUAgZ5A1qPb>obCiy)kR$AFG~77}0!RYC`Nb!*|mC=TqBYAU3(R`Evol(eWGb
znYlhgBJ1?fiMAtuZ}Z(#)`S5^`|xtkOS%1-gtNM1k1rWPTNQS%m2U|6%=tHtyv9|U
z?<)PspWO+0!4}ran#_RHnRMz-seBM=3?$(DwLbcXFPPzl<*g&4+g(^;5<I@->uUY>
zxWM`F2E1-&bK5ogp7K}0kJ18^o_s<`&w=YfEnhjmA0{wj3QR~sVg9klW;-q=!74I&
zgS>AT*)h;IRn?}h8lCjuf<MxRC})mRyL|+Ovc~&u;qp{VTU?V~32Pgl2L@&vXTx5J
zkf(aAzUr~>4*R91=r_f#{wC&s8}BEH@w+Y3B<1?RF24SxCu|_)uVU?8GD4op1Qhu|
zd;bNeL$@c&*AE|R@bgJLdfnu%=NV+L>!*3YHJFChNmtvr9mWlHz${8H^@-}8ljrE%
z+gB4wu%=;#I^IOt{e&+IsfW&?eqX=28p3P)q<5%-hOZU+X-k8gsplvuk(S~yPEg!Z
zg5=Xv@feMO9I{6tzDm0q%I(w?G1rVN3E*I3>BYLCk^oqqUlC5~vKG=ROoL&DM+?Wy
zG^n_#38{G9C^+KqZyLJ~liiDrF><LsUk6T1&mL!og>dN>>Mu)<liNQ>ND&I<3Ymbf
zQP9{A!)qc0_71$lik1y6lya0HQ*dwLoWWYNtkW2kf8x1)xNdrk{>&w57*mONI&zeP
z^mif3Hok_HY2h(voG9h%(rBg3@Z29pL&T^35{Z9eb+-YlD>rxrR6wa8zzuJDW~{uN
zp^KOXnuZL}W>qEk#CxH;=7}fHV|_}jJ%wL=<=p?JnVVk#-v}b7B>=6%y+jBu^~4kO
z#w~5*?8R2!_NpUTZt!ZO>X$c6ySDT+bbHjTnfNNqRzW8X?Mze{;B4;b>7rkSIO;wE
zOWdDn()|l?TtWcHL3;9vsNeN4iz7l7Fa{%F^BKNf1o!{~+R*$M%r5c&upsj?X@bd1
zTe_mW)v&}}aJMI`<q&4TnZ3357!J%dOxf7fBO<|%0O}VKLwq9j*xDU=G4TqBYQ{bQ
zF=}@XHotq~4AZJ}Hp@>!s(J4tNoQY+Dkwc#H+!M5Vq|gvicZ4nn!K3^7$sl|{GtT{
z*kj5P`7ua^B-|~o;2i@5YII0e^at^yN(W+NURTUbzXU_5#EK_^Nl>A<1NZMR*Ug-Q
zM<CD@8QMTOnPN`6sUX&yz>OAEUcEt9dxQiFv=1>RGwftQr{M;Hk!`%k0iU^MwbMHx
zR$m2DQHMcNY7?)-j%32%z$1~+3TkQRYrmS|%^s+(RTNMTZl+mgTLbQ2n&cRQ90`PL
z-vk#Xtw|YN%X2g>qR#KNiD-osVOtAF=)2F=EWW(>mu%17|3PboBD9vxo!jf_QESm~
z1qBynZg`$zz?>ryQ013;cLc<tmGi!Kn$_@$$iZ&>=!G*gQ^o|!6%R4kHUJ#L<#=-b
zI{fU-uBxWxRMPtby^@EsPPD;OH>vM?YU?aaVuVb1<zsM8zz}@XHdU9ZU^|qk4_qbL
z05F?IpzAcWpVY6SFZ*QZ^9fw}tJci_G6NKvCyFj&fBGy>v~A<}Ok$|l00Q3H-v!e1
zfy-JsY9%=yvtM10Q5NQ?iQHFUcTf#hDt9_QgExo^oS8mF0GIv})i&RTtmZ*#|0<A6
z#{qVt|EhCF;h4;9-tf0BV>*vho2DQf3iU7Y$d*qli$xTeS0YB`<x<-(q=~yf28yoh
z02v2ZhpBj4@PxXTTqu1Z%MD2)N-(TLWS{LkSYkGTQrhU0R#8mXohZ!mBGsHXdTy7<
znUE~jr6q|=(@&;g5NWJBjw*O{ZApFt+2Lb74o%dxjj2FZM%<*`?DYoD%pqp`j-0L-
z#cHe|^PmET!AD1l=|ny!vBoXx7<4_?DbS_;u{*;R`T$~k^QUhDU@}>1<57`hy3x(Y
z&cL1tdvg+hC9ZxQipX>gUWF=GPZvE=uijLeJXG<6|10g^`xdI(f!_McF{i@yNx``9
zS7r}7whvJhY6wH8&^>osD4uMHgOM~2wybII`lt6PE-u@2P`<qq#Nag{V=<rX@r2b9
zDwu|jSr^p^uP+z<Ni4o_r)vf7??BU<)2bi2XeqtT?r3TNx!MvUFH^<Np&R%P*buI?
zG$(K8uGjHBUMtLYxUtC=l$#x<{!VEr>4y?62e&l#sEhf(q9LM~#T1X`o3(#KY@ulg
z#qhB$4LJpB7u!0QYm!d@iojP&DEAn%_E@1wt!`={*z9BR>=>~w!z|tlMxNZY50yQg
z4ouJ9AAp976SNp}QNCs>S)$P85c)p%PmfJD_2wzy1m4@n2;F(O3c6<gDDdx?U|gw^
z1zs)BxVo&`qfQ31U;;E^1{B=d9ZW0YW;I?<QMlvrRf;~OZWIrz(6_$!ld5|Gj{2F;
ze$Xm@3Qo;@^%E%iuI_HkW%%(-!xbW)%`tjOWt|4jgILmItOh*wZM9)?@bjBqDehwJ
z?s1Ir*$Po>4zE87`&>ogM#W2iC2V9o?<J`mR5x7ODeO~MidVSIpN^vl_j{!;!*da5
z#xL_?Bn@jij`X$x88BMp$ZjF;Me(bYpOEEmkZ_P4ad)#0gEF3A{$Qine?(ZWT@W!*
zf(L-gtjtU4a-d|du8r3x4kz0kd6lB~*k!pRg!0RWN>|q)GfO69zgZ~bo$NW^$8RoH
zUr*Ve+Na~0v6SUvRDZ+x%Bx`ZiLvtS-iKCic={-oQ8y=!_Nxwzx|{#PXyy<23Dm5~
zARSP_8{j=IefuW@A!kUgd5yFqLooeM_4RWwiRfkWi5=O=UyA=9m2Dc`<Uz+Pw$wcq
z2w*2Y(=O)~f&kxNH_=NO84shKzV_C@wmRmEF_N?6C7YI6+-5t-vi#Pk<F9{R*wEK8
zt30s>1uk>SSz4a@C0$(&E4)x>d9UdPSMCJ@wd*%h!v+>W3?twjb?sGP?F`H!G=80p
z^T5Mg6rawUSUX3@bnU&>dPfIYEzidrUiqJkq<-B$xNb1Qci?lW)bl>r%~lX?ui{Cj
zX96{vcRA-4KhNM8pt{cKfP?tVwy!ehyMbB`@w~>>EveW}SdjSAcoAsU0A_@0yJFXn
z(qU^0O-vjB3PxL6E2<EZC%R0+DqO}KCTrO;K7+@9?iYP_OViX2aRH6J;580MQ`OVi
zja1?h_978!%sF3tg(fcAe=)tVxhwEO2?p-$OG2NoVKkz4&!78CZ764B2h>$wc~>v(
zr-$9Y>fe4<Y+&P_=pkoBvc7%V&JnQb7ZrM8xWt{L!=OK~|Kvtr6zNXt2eRB3D>pD3
zxY1Gi#dlYY)P;zuy&?vp#qp8Ov9kVcraBZZy58?lJl(kSk@7FNt#ZTAfDz<~@+Q?1
z(s<~+Lx0o1)WkIB@D(?Wms>l}9L-{4JCfo+f(6radc>Is5us2lT{t@gvt`z_D&dxi
z=}*-cJoK>I55LxlADdAe1S&|G)?<OxS<1Z`s_i&{*Xcc|DY9TfCt`H#rB;0H>ALRR
z?|~jc6zQw4K+xshwQzZ*oY0e}I!i6D$Nu_olf2fd_?)=`kRaIcu2MeB_&OvETrre9
z^IUAnRmfGoaA;v9w6=d60KfM8H77r1+d~jaI+Rs^PxEA5I*w?^KpDBJa^l3Zd(2$^
z{}#iJV7hQT4QXco6T=vO$af;rf5x!8ZN12I*EwDf*urB_KXnFZM<bID2sO1twT{L<
z76qtND^lg2qswY9K2(RX8ZN{au`yfUI9<{k(XCq!2XTV7@;I2D#Bi$8c|-%+?LS=<
zg@}8gOg6aJplmOGI@=wQUIT>24cd<d6=r}q{J<B25yap2%e*x{D-H=BX(dLL>MGbS
z%4vsK_Vj0*N|jr?;i7P5_XRG$4=nRpG1RKR=B@l3QsiGXa=1uWg|dB5pOU>cU$&s{
zWPSf<O%=AFb%>&P<1_bI^ctyq&(~kuYJoK*C;6%-u~_3sZ;D)|2Kwg9>*F+fKtoMz
zCy2}*hZ5>&S0v6L0<m$gB}(DrFNTx)CG5)x*mg6(toJ5*-?l4-71@255|6XU&YBJ6
zMq7I5wmc`W6A3w=y!aTYTaXrP<S>YnrX5PQX8u>Pj{Vuu`c-Zgy!CEVP3MXQiUw7d
zZ9YVcD2le*tWURM)y=<ozzO`8ZEe_-cej7g#5_O~pNYHJ02`lL;A51myqs!|ygB+#
zsbkC$Ywfhw<8TSFm%a^rJ8ue5coVc9mG{xhDHEVz6uKK<jWyqp!h?Xa9gGDy@43?{
zTfkt3O%}F|c?O-5p}y)r$_WA=>Xc*p@{-L*7t!V>*Ww`<QGre=rj=RJOU1m~_SmY>
zioM-ys9nNGWnMEmcYknrTeuiiZq{{Q^rCq~C`&ruk%k!n!5p+2uKOcosPHOWwKEH&
z%-27MN}MfUsLTYUb8Z@Zq{#dG`nwV&T474O1zJ4Y`2fhtEig{7Vj#WH?{8opL{1_5
z{lK&QQotU#;c@5d7C9tzewLW$pjjxL0x6^52jY5temgx$Hp0a^p6olhI{<0o_yvO-
z)ikW^>wFC0nIR_Q2|1+DRpB(lGx`XbFDIUNXXQ39Yas7rW#mq7#4Ow|v;+k3JW@jX
zbM9EG$4c<LCz6?}9I9~sfHQ6cv=Au_w3L=PBuJ*u1cq+2VTpVIArq!)|EV#nU@$XC
zy$`@<@@_-M+295V+d<uBPwXJVt1c#AgJ*6@UVt&E?qpB}xqKyWM;HVkM;zLRo*Y%O
z)&rG+^>Ctf<4;iDg9h_jT_0QgUov<wyeL5m4qHXWM6#b~8HOmTTm&KWXVieqf0LVe
z1<0Ac;!bX2d*PNBdL^dsC0**)m6RDRd`OWsDX<arJ9&m<A#Mp_KmMyb$E-OlbM(eK
zm!g+hSmK<ne{kF~lf>t$1yFVleK2AoM=4|plS4FxM&9~#M`(Mah86_cVPJ$FL^1KK
z#AVthu{@`Td`M9dnw?)0K<PyxJm&BQ#<WVZ7}C4Pq{4UN?up21SB%w%VRRxgAR&#E
z7HHJ(*SM7!uM8`-`Jc5Mh#*){j^7502}CgXNw`FHY2>{AH)x}C4)O2ewmfQlo%6DU
zm67DR^@SNf1xakbHbt?Lr0%k3(*5oOsCdiwHPAs?swg+>%WjXTgatd(*3cV1x0~WJ
z+Ya3C67CUSReq2?^R$d^an^o55qCAh%SC%UMsZQxY4yNztwe-EEIaJx8U6-Hr{At=
z(63mlwqLQ9=zrF8Cm4F_PU+~A7ZkiTOukl3*8#SS`<_Ary{hydI4Qr+pj)L5-D<#U
zxv~nLp$3yL?TB*$*iS@w^*^`O$*>%kwdtaPR-NwshE}Hf0@4Q>kofH1jwkJ`NsQX-
zv+*_49tO7uA>W5cW`90sBo<QQ<tLVnEpP1o^!-bbZkeZ{B5&owH+h$C)clwxVHprX
z*;Be>z2aB@EfYsZ8u|^e53*wS)NBvSRtEWh!KxHcpKwW<AX>sKKmKf$RKQ^itcNE9
zl6XvQqIhD%ydOc!5HIN!VJxt5mq$r*1Lsh}14<O@Ee}z(36GB~ldV)Q00oU3L>1Ho
z@c6WL3`GRCcWGevzCwp$6IUBH79_=*<=ziNC!GD3x2z5HfP%Xbcm|cFQJeZfrt3z%
z?lJF^(S-zzR#7W%DR6QeQLHWxc}nhE>zA(fjd1RuV?9GHzZgk_r-U?O_4qxG>}WFA
z103sH?{P0xGh!cJy2|#F*XD(4!@AHlis~7)yb2?+soEJ(VSgQrtbn%pA<*yfW!@8&
z(UKWw?0V5Co;stkekN*OeRE_^*vaS@{(2_iFajw+`}p_uKS7gzVI`wZO9H=xkxbg#
zz`2z{AeJq*eqz;A{FR9F`@zciSV?@j2f9W@S>5s&lBKI#Qq%RIn^f`@o33fwKeu{0
zr%?HcXBW?uG<M`ezZ;`uO(-b_u1{^QiP7YoZC45x#zEfq1|@Zu3#O!AS2+nSJa4F#
zI6EDH?#D_P#QJfU77Ih2&d5Ka88C=-lG?iXyA;YU!`)$t+Ph>OSmr#)`Rjh_+992W
zmooC$-`c)#atKj;`w`KPlIcGZB+EKNW80i(Obuyvj?2up<46|uYXQ|EzGYMhWs?qT
zM@Sf$!8mA1;S+%=n$c69c~~OpZg*bvn@RvUESYeyuL%t?;Jhs{D_I@|xgg`M^kRm>
zQP%~gHq`5&#tHuufE98A4}{A=70-OOcsXE1vG8DNShg)^H~2A9pE)A+*~vH9cMf;o
zf>Yy)@+#D}u4q!R@fv3E`*D6ZqWa3D*u|6p#>uVmbmGU#M!j=m!q5Ps5s&$nnl<7P
z(^(-7CE`BIDQ0CtB%3>bfw%@70ODT2|9&w%F#yCTGQ5}H$YV%J+<a%ib}92&7Ijco
z^@Q--7cQm}4uf3SN;}aw>E*mWA1)|`v_Y*vGooBjBAvT?aCuAS%`VwiNk85|-mxD~
zQ2%-CcYbB($Z})w->Ew!{|>$|r~g@=)cZb(q&5B1E5DHJu0bRZFCtOvLF!uKrD8%{
z*%|?`6b&C~+A*cBaC$d=%E-3krV!$e^J?Kt8JAvhTJbd`1T@U`%A;fWF5rcNdc}{&
ztCiQNq6?9$toc>9GYzZV7wMCXs>&DOw-_u3O+q8PvN6~^UCNXl>yzDXMS!JS=`NI0
zuym#ad9HZ!8F7{^O+itF&4bT*b=Gd-3vfQP(2{$9=`lUlGw`KR3kvuF5u{|evk>-b
z?|RsJ3E#nH=J!icA5v{Io*o9ZE%?2>afER$GF1=WllJg3+N+DU_l#7tmpdu`a?(jm
z-1i9QKVLjKJ5-=2<!7*bc}yN-vlp}sxW2rh6wB<BKG)Sb@Ir;GJK3F3%;A>67R?tk
zzODL@P~&f{nQAdTMdSs2%#Onv$ghd7D1Ho>Jizq%+ORIIwvZToo!K);afS0S{rn@B
zAEmyEDDmvWSA?zi9HuP;@>G8ZIF@t~G@ksRqvT?;bB^gRNLNVVM^hk-96Fmiv9i2)
z-6BJKVn%ZT6DaDcb`j-@(N~D7tEa?bS@mFnk@W7U5;d@i!*7G3FWbGVfQQLnlgEGq
zJ)89b;A7)Qodtmt6djdPt7`ucDC0SQ$%x?F69Jbn!S~1}bf}%)hA<)6=@s(cdhTop
zd$99Q=gkie)vV}4^-HuCH8MM<EFq~-+~r1Ex{b-RjyU_YK{-NPc)5sP**Dni+9HSY
zcd)6HR{Ksr2l4p_&QWe>n$p)JAs&?91cY@`LFA_;WJj(}>jQiJl~Ggm6#twG%{Rng
zc;YsgI>v}2B7Tm;%zrMK#D=C^+k?~&Mj)RA0wF~>RO#n>u5snQ8YPVAN8jT&V{-L9
zHx>+~mE@k&mYG-ykxte0$z^HlA!|n9zRl4O&%*diic6$q0Ymqx0k83t*fq261nD*5
zPls)mBtd5EGxe=PJL8YR)~#y@<LvX@R<++j>Ib1idJg+KX$v!V>b}Cwx>vuZ$yv5U
zkqX5UnG$%@mU$W~6u~B%M`+X@OvMww*@0|V>BN@Q4l3*SS8Qk$ilHv@->44G>V6=m
zVIZb^!FS<V@gmFQdFq>C$UJadkx}I}p7K_6(MhO^@#2$3ZC10?he5JI%uBw~e-r%n
zs-Ra%Rs-XhaChM2@V?R>0AHEKiyb|Z`m)>5wj=o+3D(z&DF$`s(?%5{pjfg&R`S08
zHdWqjf?J{>{r$q++N<lfG$%eLcdn>MR=5L!KzL8xn<N;L$7|AZczKO|i6F3WkXXVO
zE*YfCug09<2Q9|6!f{xu+B8yXp|I=?U9=j3(y}R!S=~e>z{S@KgFXci#uKiG3#<@v
zAKMh|5^ch5+?9u5njpKE7n$DLyB4K0>umuQe?w!y<VfLB^HNY6&w$4N%xc}HI=0jc
z=ZOZB<kn03fcrC8RswIMpE<dlM+j~q@nZ?HC1#D|T1;O`ZF`uk1rc;^HdMhUC1m>$
zNcA`Z@{=@v36bvI;lgA=k%opZ;%oeKYGurs-kLM}H|{O*Xru#L>qGsix=p(OzVeKi
zV9<j?!xx>NnKD?Jgu0D>mo}@jKh>&BWyo);x3n8kt#Jzfc`6G;P|YU=C=7K8;<hda
z{ATmyY7in%t9h~5q9T*-mlKgn1umk~yg!Z1n!O5Q&elg*g0!)_te+X$xDhP#?k0GU
zI-&IysLg3^1C^lRZvpiMlyt8g52xaXEWI&trQaT|BAHm}?5gytV|wVt6HvF*xer@5
zMrg%zYZn_!2sFC1-?Wm#M_1W^)+2xZzCGB}9djzLU;FAw^EnTT=-%>nZBH}JQR^1@
zhtLaj?)7E(5mz@AIuI&|`kJi_3vgYz2@6nY0R8ezdNfd-iTSWzJq?QJo_q)5eTQCT
z`V28Vz-i#C`+QD$f7#n2GMwWgB=`;C{PKG*J-?m3Ks+LXuC|?AP*Wcykj?a-%1V&!
zBrONdVkuu>Qqg>Y*~c{cMlg(dLUYCRFHo>rv!ip)M#PU0Us6w8<39&_FzS?HZsqO3
z2OV|&OVmDNFV!#P5qbt5oWU2qr~D0isaCrW6-5$f(RB@=a>-4s((&mf^EM1DkB6E|
z-@lomM|_6J5=;T*)r4k1Zs?rErPVZW=8M{b9T*~NXvo&_PhvbgnzDK11@k7I3Hs=c
z{BF^X-$HT+wT+lK&8tipagX1#kY=_yA0(b+v{D`q*e-8x1xyE~)-=1`lJs6mY5T5w
zt=AojzYNL!txwcmwY1;CBEfItVSS3;()b6LGrDK}pTFT?zqrAI{rmKQ5so~uo91OV
zO>y_x+0@@z_aW|O4X9*!x7xzj$m0oqcRXOGr$=mjGS1?L>AM#_%X*i`dj#-rOu43D
zWPJnhboVVG8v#veBr)Qx=PCgw2MPx(H0r})VGVHUZIHztIk!RAD(zoX1|q@iuCw<2
z+2}f5T1o7CibU0Acf!yx#r4=Kwgm(vG|&n4msO+~wSy>#=XKlB>fOzS?bFnfB;|&V
zs4zJIHIls9o<XW;@HSd$7c^DiD@-aeqNH9U+`iSKn+_z^V8uQdfowiEda?<}QM*uI
z>s>eCM-P;)>NbulFEczk3M|{!mp+I*yUeFVKc6MW0oJ@n!Z6-av&>3$sJ$TdD?IKt
zQLkI6%?gWVKw9N7NCpRovNwj})N}_SV{;xu?dy+CFft3DX*;w#N*r8|O1T?`)c4n6
z0;t#f9om>-#-%6Hvn$cd(_MVIdh`4uOUN!egSIv%PoEh`1v7jerMs<;zHro=B;@I;
z-M2IuE;2$q9uSX{i35W<7ENg4ogYl480TrSPWWW?NOF=R$nWL_k)vxb0Qk7JbTbQs
z0qy<rP<B}2^-R5aT+Ti?--^+_zQ=#;x(@s=fs$HYQ_mpPzfy+W1$&kpx@J`lW}&}r
zw@~?Xlg!qGHu(;BRvw@E*C}`>I)pm(uAqqj0)+^@zy_V{^(stkbDB{GnQMdY2}@Y<
zxbg)i5u&r_cJ&E@pqaOUl}(Er1Llghu~%OZ#^cBb*P&G|Q6ucx0LD@4Dm@+Lv~Dy9
zrxgB|)iI;U;gCnZfO8Pj(A@L~Ss34$41r4nHvzbNz!UslG$dK2x$y3-vM>f5bFLgh
z5Z;LZ*kg{SA;NVOuq&&?URamJXF2AB-+QD^V=4{1oux3s2<CpU+d?hQfhVWT%HX0j
zq@PJx>tSl`DdsxvMCVnLdX>^_te6t!%Nm{}ta|vp`L6QgQ-5}N2uHfJ$A%qQ2v55g
zds8@#uPU9i2^tM9qdLEJbcKaZ;MlE?ZhS=j=kq1;&DBYtH|<I+q-7~M!z8tMX-~?%
zf^%iQxks<Mwnn;#Ux%I6So@aAFqhIu?!{_nnyN#BWih`LBz1CR_foo~&aiXOn*3PZ
zadifrd~E^-kjP2ee*)g3g6*u&cg-W>RLw6%KrjWMb;HK!e!ARAtG^&y`(jKI^ZUnG
zIM+6Mu;N*1|F_Cx5JLo`*&u!WcBk8k3=tXwL|goM*9s~CzopGuUyw|MZiTgjBH^%X
zx9ue>W?)nP*4OPjaI2n3;v=Ni>>QymcA{llto!HTBIB6+(&=x72IaA^#+??B!Ki|-
zEDC&f7$g`L*hJ|Eu}8_FlC?zBRG6&p_qM_A=GK=qfkQuTM&2f}a0VuZ>Oo>Y1IW=C
zaIrr#1JkR5h&w5EylPFzP|V7jWyYq|Z6rK}y3zHne7@0RCSGP8&Y6bk{nGh4la1*v
z(B_UGZu&pqHs|}eEff81X@11uMHmJ{kj@Om2p64uaRB1s%@41P+({+DLOfy@wF{Ss
z=5ItV8&b9$>&|p5Lm%A`P>H-!&W-bir@T|%{9tDtl9$w}ouX;ig}y&PvrK}E>0;$(
zxJ)n&=VRcHrIwcab7Cset1KY);xfdp)vn1RD03TpnpLj$agvP4{c#pf52)RKnX83T
zyv4xdyonb{?!NmlbVlL-rD`Kh!0o)pUt;n?I%z9>J~v>sT6y<CD3<kJ?onq%?V9Xd
z1$GWEtgBq@hKtj)E<sXO_a@Z`8j9ZJa`6}(Grd-g=NoWO>K3SY^-1-f3D9{q>cL%+
zqXl=^=z>)Nv8+HCgx`2hwzcTRbTbTtXr*bD;&^WGTN!BE1WK{P19<)3tX=+*H@?u}
zmroyn_1m3CJtGjX`}aCXq_2|LGM5N!cH$D#iEoT-8ZEfF4BZ53eDCfZ@N1|s@dRbz
zljRNYf|>nbCh!SZSs_;&34An$OxNl$+YEmjb2DDzs)P~ZB0!~aeV9VRUIJGihon|<
zuaBs*wV-JE&CWs((iAYpG!wRT2T*r-<HntEmxc(&7t6-po1Flp4!AeUlz5Zind^Wt
zF$JHoegRqIMlWR08L<vniC{@_>aya8GX-(dOO01LqGEsF5U>r45k+HmOk+;Y8h$IC
zr$PPnEBjqd{kvjKj+p%lHVhhw!kJpI#ZH_Or=dk`9o{e8mxMIbduynRoLIo8zlaoK
z$WWDD^^>p=UqvvdnWc3t)TKWUZ_Zp9udq8)rMGC~mQWZ`_2;0L&d2NFR+9L$J(&|J
zlHLx|Lyl{KFeE5FV;8vkk*NAHSkaC_3)^2Il|gSVtq<~ObdcfUT;fFakGHww$O6&f
zeZm-se{DZ{R{RA)*7$QAmHvT29<+(aVk-g{82DOM2QL%~)18XprABa|1O6s-_gQTS
zw-soyi7JrFNJlspz&84Sutzr^2`KE#mi>;0%_l=*Ds>p*w%!Ew2<<J}G=7~a=q|mJ
zvL5Kw_~NhAmt+QiGtkMI@_|G4%B{3@jpXArBvYR+uS()#H@`EnQhPl5>xt;cz+=0K
z8%FLVN%;SYU7{#<Vx$X1Y_a=GPmBYoe=zV7z|lOxep(L;oemgcrh}?3u9CKGb*z>+
zS@6zbwU{n;94GYt7kMB<3*23&TQZUfNknz4vUH#bw=zIBkGscohJ8RDV+)=rquR3I
zkqX4{;zA@{I;$SY0|gWVfo^N|QSfIbIxwtsVO-jL^^F#4O}z_9S`E2^jo^A4n?8_g
zX!i<lx|i~S-@;q1NEg%SHH`F`Jkf4ss#A9Y1`U+QbY&&+{hpl%m%VC~3+46a`LMg7
z!O|u!{SZGL@w-5;-M5N-=x?nnfsf27ck&@tg%o0AuAM(1J$2nTJkLJ7gqWKYNb(@c
zU&ia|FiB7P1m&ze(?5q6g-bWdlJk<qy+1wsERMw6>&JBXXn%VFH2&=i^0$8Nm%GbF
zY$(5lafxV9Q)4)wUgDoAsvH-BAzSJ_|DtXbA;=;v5y3&>9^;up=zgj`K8`t8i5+oX
z-+(d@pBzxjoTMT6M+81a=wfmUETU_O{C@x~%SM8gw+$<o1)5gxqQKkj)8-yb0*UFh
z5fAuo%OE_JJM`R+6Z{6&9XDm0G>^qg1Mcr8cJ|8_Xjv?bq2GQVWO&KH%V6RrgKNB)
zC}}3H?Cvc2$^tS}PQKUGYAYMt>HP=>f<RigdEq*q<#++Tqj>J3&q+W=?zl>Lk$#Rh
z93;U4vDcRF(k*yFz3c5fDuhTcSP(vI0#GVOkl0f!29+@JcQf8|=u^W%!Mfc%&j_i1
zqHasqK0XKPO=8bxjk9NF--VTb<FDw>+l%qY7Sl`p;T*QB2j^e4_7CXzFC(EJ;irR_
zI1Nz*Hh$aU_f*DAfBAgO@G3OBZIFKB02FCGymZC3!&(AAL>{`+H_8359>J+(+?^z=
z5LyjdeOe(h7bqA6^|-$S8_Y0Q!#@g#^{CcOM6fcD0m25o<W?_z%`FQk$U=85mM;UP
zrjf4e4;aj`t{9xN3dP`-!MzF2Z7t<x4P|b6(_m1EwU4XzQj4SG5snjh23-{aG*7n;
zgvlnN;`|V<cr)IT8C!%G_vdAXp@BL|qpT^W13{`|fzG{&XPS@BDV3Pkcb%!a_1U<I
z7MG_U*{k;`mIHy`wN_Zn*BqfSe3NWATxgIrwhMl>YsgOO`1Oi=?VdsHKZKR26kJFp
zv%4<t%E%_W!R>niMCR8^2zOS!76#t%XxSi!F}tI4=yy8Q4ug49MkI_lb+ZTa$;iHj
zZ6%}FiMoCn^{UmrtjN>%zAwUvKrEEXd-3<oQh$PcJ44|T?LZ74-ezz|GRu2;bAT%1
zzgF{)L-DM+?4_ROHv3|;`vQ33GVZL1U2VaiflKNGrUZ^%1z^~=^f6NhFCjXx9YtUL
zPYEq&W6v`Do%`Q4j{mA&K^WhTZ#db44TvonmqA%JTEY~OXzqYNT`TB+x`v@sZGm6M
zcgAHjMS#@4gCo07)BGE(T2fGHKcZYC@|g^oitm5eeCbZx2$pjquEA9Z>+u1?9*9$0
z`FC@~fZrWvkXYloZyGzBI01a2Qji!+(Njl8j^0sKVDu!)d3hJbgBt_vL#kwAn-SSC
zpCai0x@s;aak0*lP-00YlkiW23Y#KTD;NeK!bMCefbu6t#6|{ZX~4F6c(BCFinO2X
zqz8$XE(`H;R}a>gHP~c9<squis(1a&{*mr}KW&vTepcV)s4`mK2Z*B2gFO^5!e}5A
z^xye*CG%C!9-lMrDZyAopM>K+caI|Gj?(LMbgAvd?vEenO*2(n#-rkIh!Yd_Pid%R
zEg~j!k)o``k7c&aWx3NyrPp;8!C|BYEENhbPGpIsQPl@*e^pb+HWoBfiPTrF8702d
zkvtAmb`0qJjF8ocOmpI9U|NdmZb0nB=Es0qE(0>#yACGNdGhs2m};3hX=-7lL1KED
zI^~Iqt$IMx0s~!QJkIp<0rkyn12QDk=NXjeO5i=8_G(AwtbnBCc6|T?qbbt<WxF3N
z!A&%m>T~T8PxZQ4KAO<o9P$zgX3q&0MjR|!qwQ%if}n(_SV09A@e~xlSo_!d8vtas
zf}a3+txskb80#BDd$?-XE*ey_ThF{Krpl??p7O~{*6*`9GRqY}mf_j8dEp-4N_6Fk
z?D9xpu^)Lql|ILb46UG~GE+)`{^x1$GfD9h(e&_|=yCilGyvgk=pd_{hDNcC#$PSr
ze^xoaGBLXWc1^PBiWGiFBT+}@zbJB4Fo}UUXzyJ?hjCQpkjooONx_zit&Uvc{@#f!
zY-ne0Mi`_9!RSNh-m}Zsdh3#;@csJEKLbToXA_@Ieu&N(AZSeDy7HEJ1Ktx2f!MDV
zYrR;3+58VAkzxNIBvG2uhqSb^JunC$WLgAC^zFSJfATksxVxYJ50a4=`n*@pP~`TV
zHxz!PO=2PwJpUxf|4e&7eija<{iqlctSN)bzCT<ZfBlctr$6j=97SU2p^P@~NH=}{
zl?aj4VrIFRprl%8Jbaln`~ln3aRIj~Y5&tM9(boeRiI~Jumo!ps4jpKy8-W@2Qdtw
zl66mrs>9^W0}=%+ns79j3S=4PNU%B&pz|f^;`arqpjt&38ZZZk4xOoF?-N?Zvdkkk
z0TU|=0^xc`s-;58x*#AUYc#;$7F96E^GVrk7G3~ZsDai}g2Ah#?f=8sd&g7x|Nr9|
z8LzC2BpJyplATRaWR^-2N;s5JX0}7a2pP#HDajs@%_~Co$X;ddvJQ^J?{OLRdcEJh
zKeyZW_fO+;T<5x;&*x*`ANR-c2f*bQAZ2fI251XQvY#PYHEXVu7Z@aBWMSNS0Jh)m
z`;9r)_*D9N)XBI$Gbo)!5|)loP-Ho$Np)PBdwz>oCtE>>{_u6oV$Hna@3axrIxiBK
zeZ8(OF-nP7TJbNG=6_O1Z(BTc`|hG9%RX_5d+^KkMn@;3K{+2ey+KZMtKA^w8sFg7
z&)Y5s2v#z`!6exqd>6zra=*wz2!MCaGT*WndL)p>;SLLQH4NE(xNyT5>OMTDS*Q_4
zXh_nXL!F6fV<JSjJUQ;LH--y*s(EqpW!Q(RK8b742d1+EP}OIUl*NV}U~&gO*`4tc
z@2(^Vi&}xqx=!cy1l}nId>{`?S-}-NNG4J2+fF83F&1e<+O#x^>?AbuyyQR=zL=GQ
z4zNo}>P#AvX>Nw&Bv&EkGP|XpLdI+}=D<-MGyJRjC9#<3aw$66w3X*S?f)N~0@hjK
zc%hL5uYFr_Lvf}%N3uh%REX^0()XjsX2(cgRma?kdUFREy!PJQMA{6nwxS5H<ek+Y
z--yUOWY;H-BaLyfSWmVgQ%$f3gEG@&FIEJzs5gK<xgoE3sa54OB(2WQ>U#(dDN!Xh
zL2j)#<)<_<<>k)#?{inbUP^DgFR_!D>3x|jP5op55i?*HW}Ld3ezTzK;olDp=s7iF
z&C40e)6tNJWB`)aW>_Og&p;bfVnZ3w$Gip)>pT><)QCxtaVx})!@z;XJVO<K;9=a~
z;=}>+T@|mP734G2htlgz-a5?od~H*YX$+ZV-S;#I)cA?sBF9CO;b%f4H~(HP@$4tI
zExLv<7sk3krY44bJQvA_7ui-$o1fV{dS@znJ3%m$faj+CAmJa$kv|B0Wg0>#pvU|?
zo5iJzC4vZw@3vG!4jO%f(k^A^SKLOg|G<SW?2zU0yNUkDK_Br}AUGWj-0xN|&ga)&
ztel#CPge?2r%~=5$d_D1Kspq94F>Kw<9Lnh9|ktp-U~%HCR5-8nn%p>G11Y5&(MQ@
zX7y>eB7WH=cxhvBYZ{PpMNQ<Uv+i>*NXZa;WPSr}Iu@|ses)OeATb)Um%Y%#c&qFE
za3N*U38cA~;<IUz`gkowRWS&dCl9<h=(KlRH{0%iLF!(^Fnv{MC_M21;t4@cE9;w`
zXtWB_55zB0P5(PHWcCW3Ys$`bDEj7a@8!QuKw=nDGq~$dJjil1khXnCF>bhfx+iY`
z7S#=cXlLv65Ko(!=5@8<JScVP-wRC@N+%C-gW3YONP3j^ljL&_8o$yMc8U*Mdq3P8
z=hgyA)nXzHJEb>u`{gr|d@s4Kt3PSdbRn*U>dHsoD)qy2P2mV{|L5n&&V7F`S>Ip^
z`XNoqLcvuc=E65tO;B;ol8+xv#@RK2yF@m#?;G<0{c#U*-tpLW*CS1Xs|+s)WM041
za65mA;FU0t__28ZiNx<0j{sF9(s7c%N7qNrLQy*IcdWup^%$D=I4gNUCDeAAx=s{$
zCY=H0N${JwLI6M7`LiZF?vYJX;_(K{{gCU>2C5!i$UvN#)C)QP@q5q(=7UJBCEL8e
zcmdiA5D!1$?Bl<_SbI_<T@0GUJv6j{tvUfpiVqrm7;XKyfE{B6X|~LI2nanV%p4QA
z1%Mj30_<u{2VqqEzGK?b4wc>fz}svAi~t(Wp~DCOKLK(M;MXX=IB~TSBH@`!SvI-u
z;M!*0!_-1b<#8WJ#X&x8nAZB)-v)jg*yj$k0Eeh^Xzd3mkOzD;x`_-{sbI8tA*tvM
z>Y`dtDmUmPz3Ma1b5H}Uf~HAVtB){Jw-U&fT-r9+F2&u984O})B79_f_CDNj4}|C(
z>Ig_IHv#A*YJvQs#*Ku7kdsHK@g7;($20;jKkqtbem<(x7b*KH0wyd?r6~)tKZXz-
z1gt^^%?AcQe?9mz$Pd&AX?qTMhtFs~6;EFDuxYYo63^TACc&XJMB3-iRnYvyy}{!E
z@qSzRNn!6A`lu4r!xY6Pwz*`YlL>EmTYn-K^M>BQ^^ODQ(u0m)Sbr<9$8mQQ*j=oL
zl&1D~bJC&g`BIS{63A0hK{Ds!4SIlE86L^gh!9lNeG7%KfFx*46J}xnZ!3wf$&KJy
z`fb9aM(kE)^o?XN{WDeiq9f4Y^T(#zQpw%>4&?bx`?wCBv3eIxh-4L3kh0ePXRDu(
z+)upB8O7rDsT7K*KBs4Y6rsR)e+OenY5T7B1Tk95alsp8T5D;d<x@uYgQ^%Q$IE~!
zmv;e!_+WInq1zi^5#6SzyXm&y8Xc|5+J23Inri;|zhY49Y@&}Z*q7r%mYYhowsv)+
z+YU$x!5QOn>(4CgRt%vKWPlX-#m>XJ^_s7Avysn}7etmI9kXUd(GOQab3X%?zB)iy
z+%#Q*_9&fikV>}<`wGK%hGEXm7w1_Y>v4~*rU6$El}j<?N8Kk&xJgL76TJ81iIA}Q
z`Z6hpvKjh(zHXR#%qOVJpW|~^!lgT6d0m>aGrr^szU(K$xc}nDOXX%OX>b^*VnErW
zOi=uiMM6bej<I$QKp-r>C^=v9%(+Rj8jO?fh<&(xM+daI-J29q&_N=|<X*C93h9{)
zcU($Rae&sP#Z1Ls8qwIQdR8j?j~{09dWo2=sE?iv6v=r!di0(N6^XP7;1<QJ!y_2Z
z1}0B7_F5M4jo-Yx40D|;ZWLG>C*Fnn3@Rd#N(HMg`;>2Na<g<i2aBrX%g;^FPl>IQ
zX>=2-<-MJh4@laVz@(arvm{#4g+dc~8kgxf#Gt8=IT)3`P<o~e*r^seZo^mJJ#XR+
z$z#{))1*hklt;24Q%LLN(xo0!X*dfupme!6pjf-&u*6*8D#N=>>=`$!s}=KnhGB!c
zm2&E2r@cJZW}7_s;ghg>Ab(KFdvGZ#$R&{MUbymNALhbOJF&c<-Is-SuO(`EU|+AU
zllQKgpUfSD@2jZ^I&M7C{3;vTFrUg?>R0;#GynJ(h3A0wUcSEE1(RMJ_qv`nezw&Z
zitjJ{_)E_$(*A_>H?{+TXcT~990fRO2vml!VWhnu`|W#cp#$?IF_Z0c^WvYM(TX`J
zs*eDf71YS2lo)!SlKw^N%|MAeG%s5N(_x{Qqs$=SPa`4@OEK!5geo1qP*?Yd(WF49
zSr9MU7{v;!mX$#0y|dCb>3zl^ln4p4SY+7AOQcYn(@^qV%*#Xhi1&t(tApN}%srMk
z7;9TvCH&TH3p-c`C-DswC&Z20#{jHVU+JBv@e(}no1hP0YIO`yiF$eHoge(V46$oi
zjD3%?r(rwUE&3R{@)AUis~Y+{hm7Yms-Ff1?IzK(7ht|?f#2j*+4|RvN_M(cx(9g?
z3i}0=$s@QHqaGa)@S$rgji+4u4zud{n>IEGnKF9SJUwiu+Icd)8`lMH|NW>O$7qpz
zX$+-?!fP(2h+30Bd1)TH_fH9(^X?x+zsPwlLHJSjeSq88iG#>6`x7@j2+_WC`ll%H
z2&s?@RNM*s3<&8ChP>bTuUtGE{DN-=W0rqYSmgHaP|2~z;P<gqby{_OM(OSsnZD(+
z$Ua}sYJ*G707<C>uUpWo(QarBbl}K`PW=!{nbr8uY>(MVy10@J%1V(|O54+AoRr~7
zZ&5u9K%P?zXlE<>-XRC3p){|V<*$j#G&+!j4c8&7l(NtR2E;gqhs70KuROa5bGPyU
zb*H*DdF!E&))P}fzt6R>yY#(|v%Vq~3U`t~qZ^v=)Kn%%96SdmPAhoPVX{@xzVAV`
z>i2Wd7bY~W?FFb%37WFV=B)eQ&K4CUyNC%%8oAL=EF)SNF*5f$QfqI%l(_f(F(<o%
zvYQgeKL7LxIO-xc!`wToM9nVeDd#ES^>#-vKstO54f(|@I8geZLl#kPA6k&)zB?_8
z#Q%_u<N{NC5oIdwp4wy*<n!_C$hMTqjQj3vi#oC`)0a<e{}8#Ox3UMnr=jPY8@t=2
zl(sE<O#jFI*-2>qgMGKRF#&lPMIXT(L1srb{U6VYM@Y?HPA^Y7MHaay>^@XOo7|<A
z<)SjWchB#>k?mh_6N9qbjB$J-^lEU6giDtXB4RH`i#HdTz<r5m=gk!kJZF7AG56}h
zuG;XB^CVnl<Cc$+*V;W(tOvKx4>_D4m4@9<Jy(aDe4iFGAI^C*mO%QPRa5xliL=|U
z&Q^}QM9cAIXRV)`;!zV)_e%@^I<ZR6;ly&h6Egj4R|m15MRZUh9_XNChV5s<BW&e?
zo4DYfSh(c1-xP*h_?}l*-2UoI$oWoCVA$PRKG@lWqNksaZL2)^gOnO}me7t!O@4PB
z6p+O<5W9t}!||O@$w}--gk&%i5fVR6xJ3*1&c?F(!aiLJ<@VavqH+oQm8g_#G0kUo
zw^5E&9^d6)XnbQl&)zzaY_9_mvJNJDo1KJg_A(=~*)}bNN>toOIOw(?P~jl#lP|UL
z&cZ6??jyzbnPWckbWdlqE5Wg$Zf6kQzq=09$B=dC_dwR++3u$%X%D+-+%IC0<$TEE
zyl~+De8iwPwJ|-?HGI^jUZ54kaWD^XLl+U?`Jk)0!uCSM-@==dKEOG=Z|5O{k+*Jj
zdbF~$O|UM?bVHQJ%CihTk9WUklo~GlthJeE_`P!Qqlf!HVwRjTpX@#JuNx%Ov&yqB
z>(%W4b%g`D1Bd6B@ZQndhyQm8m;ByVk+5o>fQM4yXmH#;X**knynDO?K08S$No3ov
zn<dl3&64NOp8EA6C4af<(h9X0F>zRkf82rliidO4*(&b9|9V7H<PpCyDJ<@N8eK{9
z<7sxK?SAsz<FdQ%dq4m4e>Y%He`wui+&#d*9&ztXD4->l6(b#J%{ufGl^KK1rzsrC
z6V4c8TaFCZz7tgFU!Q{Z4olP0NX#ppiqvKby0QKq*QQdhUHM_@o?(vBfcr+V+kq{`
zoqbcn5A!zGFEkpkbB&q)aLp*Dv?ILpZqSGI8ywjtFv6Znr^FU_Z<Lbl<RKqk?0KU}
zZ8lk6Djg{+*Y4#Hiw1iq_t&Z+X5+m+=!5r^c>DhRo~&8;k%Y~qo$K1Yb8LK~2C^Sz
zv&i4x-g&*y>vA;YrEw=0z80|^c<Wd1ZsWQj=Mz;i`Nw-H`NA1jo+Rx-|JP>h-0qzm
zfe@qIO9I6_3Asc)${cjkI?dsoA1TnE&Y^J;IA_$c$QCEXm2LfC=^T&GpY9{Kp9C}z
zUQ7+gf9}Pu!0#u?9Z*lKzQh(o?A@MhfJa}s`$<g`^oJCMv0`??VYeVF8D5(^XQcnQ
zt#1yp%Et>gM(;`Hf4=Pxdeg606Pp;=n6RsmMBBZG`N#6&S~jh8`n8o|@(jNt|M^?U
zW>6_*!Wxmz9gX~Nnc7|mI^O1wFKSLDdD}9vUrw>#Z@~R>=aX`02~+LZSo~=s+;VEN
zACsEVHf-PUKmUmT2*O0$8PfB|uLtY5fiL;&bS>U-#VL_SgD~oy<!9BriT+xr@D`^x
zf_v(x%m3k*j@|wF_S!z*hbZ`FQenJ5%gTF&_?FLX&U)<G-IgL{Lsw<JdDm4(p6+K;
zDWlfeOsnx#WJ?M2PSwAwU#HlSN~KfOT6`_TJx%wxVH#a+aa0o5G)*X9U2cV&7>m$!
zq5sbEqYeL>h&$eJxejM((%51bBY(l*!*nD@g3WF!Z7X%&?ud80g=z5QkqB*?pmjCH
zBYCuDxj{-Hd29`$bY@uvN?v2r(;2ny<Lb1I%Nk>YE%SG+g{%K~be=2W;Wz`zR@t-f
znbQ98V31r1m3unCMsP}mfKe@2?pWQIDz>=N6WO;4Yv`>qG|0F|9)4llUMkI6d^NR%
z^0$&cx$_*^Rr-&)$ar<R7rv~qEhb&frtbXH_G<!#Dan;fD4XPeF8V(-C?`5ymVVMb
z*6~EPql9(4BJqh*OnlAO!1b+>t#{RzcFZpNE?u)bN^<s8>+Jo5YW04FsX{q3svi}M
z#<VU4#vlFTd->3TuFTRq_vz1v1{_U$DfL7QPs^#Q8(a%5XMz|oe!P)1vxhJ4o)fur
zxw=L+X?LDhyXQ)2Jq1`?o+`Gerq=;G%d?f2wnAz7DV5$6lSTad;GoZfapAr4Y+>hR
z*%800(vFmT+p+kMQ?2xPpF6jTqS%X)AlD0iwz7gZgvQhkMp&oS<mV*3NZQ-15&Eu=
zel_VRU)!0GvedBf2XSkd)k0Ha6`iTX$qH1_2kjp+zl|W6q<8x;v5$Q(F>gN=x)NNQ
z=O0hKlK<24CsQFNY2t0RsNhsR-pNw>i2KPkNg1IZ3Zjqh{JQU<2`Rb6Ek`!;$nmmz
z_D1&d64a*KYr<bEF86>C-Z<d+hn>@>c^;(ci+5A<uIv6Ut$mrdoaM{+>EeTd#GSaI
zp&?i0o;JJZHTv|URxKN5Y#BGXvuCC^jCuA{EhA@hqJPbRJGB|hHy><<KD*#zGP$II
zK;YOwPI%-%;fZepZO)3Rzr>o6v%F;d$=U<qou?zEf&+jTBS*7$0MJ(uV|m4v#(&49
z*oUV63ofi?wpsSR%CC_Z^KMV4v0Q4DX46s=&-%MAi+*=!eYrGK@|5D*6r45$iyQ6k
zDaMM73P<phs`ShU2H!f|PGs(7)D^I+*uS0tm*>NZRHr|3K@)gsNv>$3toIx)54luu
z!DvtNzu0r;{@Qlt+FH$f?;j7kP3bFaN%#E@HU7ZI8za(M0ekxT%kgHMQcx)>(q=kN
zg&@rMfFd)~vFz_f9{3!6(PP-x)Pi3pu5@*H`+M!aZoDVl7bdWewr*c|uLx)H?xa~d
z&E9eC(!?tz>Cu!AxUjQJ=!-On7tXAfGmGx@vF(wr{LD^V?1HZ0YI+iSn|^o8LltgK
zquHDd%+=|F*-twp)YwESX^wF;U@jQHt#fuVU!=aYx9RUy!7)BDegD+<I>u9jGy1Ml
ze>C#few;xZb&9J~d)py=0C9Ea-+ayMy&vS}PU7G=e8Pl+gx-*U=gZ&F)zq_}me`06
zNvCz(`u0Qc!Mxfu?o!{Ws&_2ChTNBlcc1Y}FgUO;+M{kBKrYC=_zR5d8>S}X-4)-5
zzyH7wMa>GMqkDRY?iAvtGDV;7{hX#i;eInwfB4o*hq;3hPOd-P_M}VBk#IfwRO@2+
zkWGd@=R)(XAmamGb(JMt{+J67qG(gx%>6i&rFZ2_lLumss~0t_nYJ%n<(2c4qjQjX
zxpzuPXptCb|AR}1cemrc91=xHi2LW^nrOOLns0@ywJF5PRL$gj)@AOgm@miSAl6if
z`Q{FCtZuN#925N3I)r;LuRIK^z*+SkRzdkP<=!eFvA~Pgkel1fB1Z>tl4ey*)0^L`
zKnklMf0K4E;ySS%g{iAYS)`=1otvX+sd(&G=Q|Gv?RKoh#)m=uygkiGO?bqLO>jm&
z=n3D3?wvFm2ItV4d=~6{p!2;R2&-1OO>kX@v_cB^jv_S@Za&f<+3_FV--M%hf^y#}
zmc66s%?pbWl+5kDD??Y27|ew!Mzk00PCxnaUS{<Hl#k2$#eZyuAfW^$Rrmatp}hJi
z-i3Oz3hZ!SNFMg}89rY<vAn87Z)EnOnIc#Qvw8WR9e3K}IjbDiZ&Rs21L2mbdg!i(
zMgwaP$Uv!jh{d!;XA?73qJWhzgVMvh&H8zJ^4dzcz_BT9pIalw5?1=vtWfquN#m24
zgT3U7!=}66`A!O=suLL+1#HL#^m`icy`~DI5A3cq9^qYXcwv!8!>9Lt;rezM)+5|a
z@lheMPJZ)HheS};1#cJ0I(J#!y*!8`KBE<f;F1B`hr8JP*$|YXz6?-SM(eiOy3;0|
zJ8E}_@#zV>9b4@o7YX8j(PqBIe&;I$K@U-p{I&uui0nnuyxH3#CnPS-$Y?X{jW4Ka
zY4T}Lp11n>jfRcp)be2UwcW>5N|)PORLGNVFLS-8?bGKFU-PAE6Fb}!8HRoRF<rLN
zDa&8$W#QiA`hy6($?7!SP1Z4kX^=Xa`g`pO|AO<!N-nwg2p6{%;V?V`qUm0HF)_=i
z8nr|Z;b!(?VWYotXLtoSf`;<cO3#n<6MJH+vrsRCDn67tYJnLXp;xK{`*iZvFZ-l<
z6aqBAVI{tu7Z1GzCr|LZ&E4Jikz>7zzwwIqYQN&0-G`1RL@vmT<n}pJh!7Cy+ATYT
zi(MUw(AvOc(d{1BVE)rT&Ibj!ZuA!1e_L}sguCW41HM@SCY8)09e?jg%k62wi>x5}
zq|e3o?}{a>=0W(1RJOt(zh9rZeC+uX=O#`Ig#ahc@I^<VoN^S6Cnjf4?1T9-9o>Tn
z>Q(XN>u>~$VQzG}FK5lDl|GfABBjjseD&W88>$DdneBi_{`(8q!1P{jy(Iig`@bW^
z!}SQ7W_j9&+B<i8DXtWDZAD95|ACfg>yX&OneRkz`_Ll?+TZh8k7%()(oW<TJH4u@
zPr|Ka^4a0@e8i~6@R0vH(!0C34{eR8TXomQZTp94*u;w-vY~I@7n$<zErEwzF}`#3
z@$7!8Et}%4%$mfLI9Oivi9-nUEs*|R0$c(sRD;tENl!815nevtJWt8i{{$<BDGT&b
zE$(w?*^`OARo?k_?$;2B7*~$B|32sc{MYl#=yUVbW^MgtKn&nr&pclim_~9Xxf~GA
zrx3J>uvx1@7_SZ+bg3jgJ0fzC_vEo0&nsqS_q$10pI^9D=c>3;@^@LY_PnpugXk@p
zyj@Y-Ra(2>^nbcQkj(VQAG`O}K!Yl|x2EhTB{N(0!&2l{nvd~a*O!8zOF$*<#yXcz
zTv&*Pj1Ck+!{S7Lkiw}_ij-TVU^~9qRCe`Vm|tJ_;LP!sS{CoLf4<<JJsYJ4g>!Qu
zde4E`+ne3P*~-)WkS6GL+_DumlFlS^h}ne2Yd;SCcRm6c%l<?mdFJuTpu4+Hf%Tgl
z{b!T_>B|^}IY4&rB>>!I1PPi(2^L@e`#_CQN2fg^gT_KxFbK#a=(irVg9_)MK|1xL
zAADwLTtlcZ93e3ty}ab!oLKI|$&>kN>kXngvaS9hQvZ5DG>ocY)#QAr?)UGN-FrJ8
zI@3<A5qgJJvCUo$1!ksiu#K<qfn_d8<X*?aFWbAwr#sf2_X<$m3%)au6jm2`deqt*
z;q%Z5T)z$VfmWbgNLq!-NX7t<hu3kVZm#&$36t(L7~M7$JUwbW@CXv2+QD$pe+$yZ
zWfRH*;y=u54o-At#Mw@FG;~|)gh7^<DciDN0;O9@3G}`nk=c_xE6`)pKH=b}C4W%{
z8E9qQb^ED!8WWI&Bjre)3~$O$%(UGPalj|I{7h1R#LvN6H0O`8=YI>*ZwW{KHAypJ
z{hZ;evyXexc<HHyKJG17b8<Q*El)XUJ{g5($Y6joh<6IRZ@P4w=QJn~A4)QQKxqQZ
z{B|uN0_IPlW~6QcR3iWY{~%^O5GXl!rU3d9rF-97j2djMzkGK1r`F{b0PdWS2q982
z+u!aoQWF##G6(Rppb$6cSz)yqZ+roz2T409j4&qn5WgtBqUc$U=B!9=o7>b|TUcX@
zBdk9$;!gNyY4t|bRF{u9-Y>QAul<I&H1M^eJ-HJ8>d>X2&Wv~O>iE7^CGyA*O}q{a
z13;+Q)3?Ae1nt71(X+~ZP-G-P()ew;fMfe2+G{?XN{{^n6_e5Le*9qopss_F+(KSM
z{KY^1HjLx6TQ=q*K@h8@Iedw8e_!snRYr&|nk!UsgE?6j3WI?buMSwoW_eURJ)R!#
zKUoxbu-h$sOGH(mmOyB3@wZL<rDM0x@E0UiA2B4(_vd#?yb|9u$tE56(`sYM-Fsu)
zes9mpfs>R|Vo2J#0A>M>!2H%P1JO0hGlY0dxr2pv(_Ovq$Mv9UH#4qWM+!OJ^BQX$
zBrrPcdQYCkv~nWp-B}=m97V}uBT!<b_{;&e88a>kN*RNz2A?D`-l`=7|5IZEF*d%j
z=Xma))Cg$3%jl9*)RPjRswA8}8rjM(XUg56CX5s|V*`lD2Lw+$egxGJ$)Q(hTiYF5
zNj`d!N-056yjQ_fPJG#2BHzH%V{Z5GuOxZO#MzGd1Jdj?ijovxoPDxCK8JXM7vHU}
zK1o5pq{fP}_g{)&&z5pWArJX_yz+lM<V7?=`oZqvGWWLXI6grv$Ihavc$NcU;%LhS
zuMjz?$Xo#IuEW_35mHHIDy6gh-Vhf*b0fL9J+joe=J~N#{>1?O9sz2!dY*Hhaumu=
zV&(-IHKm+*%~osXo;8cF!S0doJ<>mS9i5HnD^Vx)y$tt%JZ3WOVOu{XY}}d{ZZZgR
zC2R9{?!7~nk@?{ep7?j~<I(pb#BJW%0&Ga947jyN09Ag!BV-*-h&EaxBgL<scp6WL
z20YzGK#CXvcmYlRbr26g1$K>Y)P&YbtwpcSSp|7nEm&=Ny8B5jGs64tn(u!<ZafU1
zAqgdoZTq9)@{qf@|4AQzs}tkaXW5@8gE9sd2VV}QK}#=STMBEg=~{3^Sr#TEl&vz*
zQBXe(?D{W(^2<_l8Y*2U*Nt#ca^$fI0A51*vBsDvN2hf#su9V~=V%BcMJ-N9J1x~j
zzXtwqL}#`XZq09PYJmRvmb(Sq5kVy@J#{*=BhSwn^?iKM=+s5bbY%<VPZ_ZJLRAQH
zI?Or<gVJK7xN3=g*KJ}{2`396DLA4m@**beChkE~!%OJH{sNddAxnhldOUI==3H-z
z>$LvT4FGt}n786JBOUJ@I2lB6p!PYMe=;Wx#wcW0Gdp{ij_V?(9|jVdlp4n30F77s
zu!|8!!7)0*pzYbO#q&M<QRdk`T9yC;T>vzL<n)?ME|l<Kh;OWqkSmZWKX|Jl{FYNa
zn%hb3@u^V=cA@%cUNoB*5#IdK6s+c$#1cxMxM;e`<d_PmXdzjL6#;~QT_|3^NdyLV
zsFr~ZQ?|uHk8dy(G37RaUV+x-6!z#!QEYEN7w}ZnZS+B(bxMLP7Yqa`(Gy;h=9ZVE
zl=$|DW&{BlN|!>n4En)MB~Lj3&`7%`$mRfvYt075CLrvLfRns*oZsjU5E>-9Z*4{x
z5zRWxQsVS7%2<})$f~Rt6*P}RH8>3Hf4w-&lD;iyKXV<`Kkv1o11KNqA^^~{txJjo
zsW2`|;+l(aQ7a7>*aDeG%QCujF=Waz&a)a-GK|Z=(lI1gav&A;wGVq!5>bY7`$?~l
z-FSFgj*N<+a-YEe_=}dBz*+y6a&yXaC+?`ykCGb#D(aDQTX(y~uXf%cn!5pvo!Nhf
ziQ9}_haRHr`hClu84c`=*G+!7+*$@uRdzki8lO<+pdhH1(fWNe#pA*R0cm9$qx@mY
zhBe{?1Gk8*bfMC2`u3Dic<(^8<5m^FTM*{xio*nqj?Emu9^&%cET7@sTz6@MXs<$g
zTY9fs=bgo6pV)dbnLG+$tW_F!1iwdmHf8b)%(|g-Dg^?r<JMnI<g88g@}6Q{VPZq<
zdc(fEb?nC#-dD5hU5|5Ji5Oq?J5|Wtn#Gx1F6b36U2T$K%8N{w4`Z-pN>;}fs(N}P
zoxthjVCb;@>@C+(F?M0g048#7j3^Ow+jw{n0G@$;0qz^aEU_n6pgpNR3tG+I8VB1-
zVf#*kpPX0!j4^DJjNU1YQ=q&MjClXoQ5XFj6Mqqa<km^`%Ua<$<;!h~i~9M?Hmrw!
zcHXq{w~1E1m|con(M9RG2Q2Do!+NFyjXocjXv{^ojrg)pf34|_%48_ZWWv!pQ^(F=
zq&=<ExiSb$xM9l7UwSPaaNV+uPyi>Fb_4Ypwo>BUk{6Na-8|ek&bx-f<x5Wtx|VQp
ze=WJG$!P1G+*9tD$w}mmXXtU575OV(!Vt<_Qxhn+*h$p(#S_v*N)$sXLVKUs{oud<
ziXS?Rl=WO6^b~t>$Y-#-+;$ylt034PmFL&=#=Fxpmfa#EX{@LkHHJ&|GP3h{&>ly}
zRKLF5X)vCqeIPt(L3hk&P26~qgJj~C(<1Fc^zer0#84-ifcOSuFzc;Ywz=eW=u~QY
z2z02@v9rjUAD8Tom1xERs3NbPCRf_-ahmDnM--9y(<DwX8Qkf*NDyXJh3eyw=K;3P
zt09DVi|yFx^#+=jMg6iugrj-<dK!OC{QwMOetIF`jx_pncEl?-s}g(Mn~PMy<eTz_
zl<qNar0T7Fp&1->jQ+e*1bD0KK-&5erQ78T+ZybV=%Z#L>YsT+8e2beIE)kXNbj7z
zXZhgPrb7}1PW&DK&KrNO3r$--jmRn&)cEEQQ1_BGzHD(UoG|rlm<oT1a1zt)&Nmh=
zV-cvo#2+sM`cwBYyEt7iYrJbxt?-;8jYD3Sp`T~^$(hJI-JP~=L++U6*m&<PnWr69
zB`B8;c}z@U$rW7PKerIgUIp@9c_;XLP##s_wn(=N6Ri&O8_fJ$!WF#nHZ`xcF<J%;
zyLapz0Q5zFxO&LIoU4B&oZgH0FVUVeGB4S1yptX+2oy$IMrZ-$YUmca<=$HHrrS$y
zF{W}pdF?hSe&omxP*Y<m9XSSZOIrwOQJ$jfiF}#Iz0z79oJXNsO?3DHc-@Xu?Rdb(
z+h>f-g3@4moEmN0s{Cp^C6o|v8SGKy)APjEMND2Y1~ASkb~raSK6}O9)z?k!kgLRc
ztn*w%zMHfL4>h6apo`5t=s6N8pq+mAI^c^lQ#UC-!YW!Z-a2x)S9(T|M*44)1EdmJ
zo5h>Z{zEBs!QNA@%nc`X6*uQlRwr&>?tvF8$Q6Cxa~U?Qv-?mOY>gy9z9vKYMn3T*
zMF+QqvF0ERsQGu*23px;fSR_wQwuZAOD?}$n14Zy>dcS_K`IQSc-Vu?<++!Vmgj5B
zj1gDa)Y-D2EEH>T_GH6hqw{0qMJHA+e|UAd$%{|mN^(|%-ND{nO)KIL`6cg|$j=nj
z2PL<!v2zmL#jQRLa=i18$}y|L5EK<8t!C+p1NSnS9&&HCkr+*!{VN|y*_MxB4YvB7
zeNu0HFXKO^+YpjlB}F>Wc4U#(T)=C~7&x*Q=H0h+BgL#vfgZ{T=lbTyz&^!;&56UQ
z*3*<p_hxI?)1-{Xdr>Ty>qCHf(|Pv80&u6)K`d59b|VbXqP`3dPeOCk&(IIrwpf7o
z%v^240t~#0Y<<PNjV@$^6~SI2`00r0F0-VV5uj)b<;(9AOiHyp0}e!G{oA3Urjiyi
zqA-N$v|AVg1XZlvIp{R4WT#XDTJ#v)m`|5s?6cX_r>@B4q40{ovs=zHI{KQ$^?{dY
z7(2lSq2CMWL(^B6#o01Q*uHCj|HfcG!=Rp~wMvTI03oLYD3xoV^vZJpojuGm(}g)K
z)8;~?Oc72o<T5O|n@ATgs%c%#iD`X7R|5DY0il|ZQzvDn2LochrCz|@qUITaIZ^k0
z-G5fmx5CV00fO=vi*A<oFJzCxplM#bsJ%rmFlV@$a?EFC0hGQEL-9<v9n9;jg9nTC
z;ZMs4(g;EDz{MA8j!Tb%kHZVjMieS)&{vu2`N8Ao&lCw?%P5+YK$RS3*<5RK{~}O*
zbU69qFa}IZ(?)%Yzhh)cG{DOurtepWeIB*%wV<E6M@?ACp8RksUwcW)L=>-OQ#9vJ
zO^2=(9}Kg25{|cOA5@QpLC!)Q>o&Bx;p|KTwWnGU=8uCVGD79DZ3di<a)@^(4wJTo
zI)_iaE29w}+b~oNWPy<!T7C6l!QwABZ%!tZEiO64Ik%iSo#SmJ8iAf~h2Fh-0VIep
zcLvG_>-096O)Gb-?ii|Dztcgw8qkALlQbbU>?R%f;xJci^y6?q2(X2mQEX79iWo$J
zUt->E6rxsHZ~Bc#&~ECAfY7-b;7N*^T`#)m-ig9|LwGrnjxViK6JE+xR3Yr6Gt3Z_
zk30CybwMmS3OS%KzzvI!A}o;tKEm5BUN1HFm77+OLCMr$@z=4zcSY;i`I9RhiGwV3
zSSJ-7mb^FZV>frb{pUm=Vy(?8xd*qMZgerePwlnSd~f&;|5F=2Z$CVwe6Z5w@|_U-
zgHgNEIqHjKfBi}ldzrV+{|I?B-5%KVMk&i}z69eGJG24IuVccRY*n;eJ8bf!AgP$4
zYf86c(@i^$I;LbIWV50C=HM_wZfeH35f;xt2*BwVZRrR-xCA61T$^qa{m=y;Hg1q7
z^_>vOM95UXAA{z!_)7-!*#V0P`QcbxUf>`W=01&yUyFOrY+tOOBy+I8Bph8F)Rbfa
zjwu>@zmDCI*0y-i?vcfGUqQ~e%lgtn=7Ky{75{AG4DDKMM}|g>1HVMk;|>~IihegB
zwfRU{8!a?Lx>UB-U?nwbXAGRxug04fE2RP6Xq#CIqq{VEZAJHo5DxW<hOiIuq?}wA
z#+^=?VIW*O?0;K`s=t*%B|Gq|1t;lQKgUBcKF5nvo%Q0r&+9Lt%Gpf_nQbZ$UKrVk
zCVbHpD__puejRB$#M*`_AG+W8Mpkyv6%zyIAtA#;q*kLb4;E!<y<zBKQnXw1AfDT`
z8>kgBjvE8)PU0317z<>l=S49IIY|?C1$7@~QB9(^s1M=S6$aLcd6fe#pe3J9c4a?p
z9Xq2A=TG*_xH+2_vo`%i6<c%D(&)*g;NoC6_bVr!o@w;5Ff%+8Q2JVZJz&Sh@i^?w
zMFeT}>K6~IUhH~3xdNIA{Am}RFA*Lb*xml^zi1_iR@uD6PuQzoVi8c>@sXde97d`U
zht&&bNwqwQ|LRC*j8h46Ghucd392u@#?<$oe=&5iS#+A!0Am=7&vZk^c_@Q|R#any
zLi)4Rn|?u!Pxf(~i{jk!w;($rZv2U;eY#w2ap^&wl)TjVM{ZJliSVBK>>=?yh{|FC
z(wc{v5!0AhnV~!)_1?>~A>^xcHNrUN7zjC$tts&_2*?_opB0gtsZllGJ1unbVY(i%
zLs14FHiyF7f#2k_ck_Z%vE%zvEY&oq^d{$L5#pb0o^kpc<#RYo&w*z|yI%jHOLl%t
zdGi{CA!mDm%3rw2E<CC0X&6R@s-~06#GVTL)d8DW=rVZ}TV0>{jmI4t4jzSR-^1xx
z^N*3PmsT5bk(g2kb1D_kJlI@Hk`=FJcA9uKcb8An(Fp8x<4Qaso{*Nev5CojH_9dm
znl1;<bluO0>K2Q3TK0-~d@Q$^Ii-xI-!DSo_|jpEmk~cJ1alr4>6_#|YhJL&{!L9N
zEM8qt@y+tdGI));CSgyN?0Uz!vCbBYLNvBk2^hl9MeG^>Pqvna=;G2Fx|rBR^(4N^
z<G*7cT4n;5gI7K>Jb1#ecTuxj0I7Z9qHKF<lJ-;5roO=D&g4?|^g@D2%1hGjxoejQ
z^j6x$F032PWG<|f&fv}8P}cM;2dnGcB#ulADE5k7mU7EsR8Xdx=*`a<caeziw9Ln)
zd-?SHah}fw&grCRy#cpDLqER0r^d0x%%inE5{sVBE%N#HF_#9~qFGPdPH1A@=o&yT
z)=*ogNWn1Z-^I`-OyW9pVmJ{!nV$J;&E%-&VvNH;w+*wjRcWcWq&(~38g6A;&KhG4
zvl5=@gBRrS*{~~QYSu>Ira@$`%_ug(YQD}b15*4SV5n)*2DB5Vf}wkyr!w6<avUU(
z$EY)H8oVp$<ZTFTt7LR;;rh{AT}WItz1*XW%p=M4)0HR1w`Y1>jA}APCd*4$$lgsz
z-QxlveSZN(_ke#bxX5#=HoxEE)p%Q+hV*%N=Dr->iBcdM98)`7=TMqG)9K#pHK(=|
zTYl3@Szq3wdG^iSb8ljabq(KLz7sXgmgln*rQAJ2(8qF6QpBGr;JE#^<EnI-EObKU
zq=d9`jJmG(h2N11w-ku)yZb`l$$o_|@!5B_!Rca?d0Zt7Qkt4~tHv+C9psT9=`2Bd
z9bR>Y<3^St@OToIPb{(m-7v0^_2ZD~ff4KdTGxn=^e#UtnZEV&jmk@>0RLdt`(LkW
zI|h&*iN+#)SYKjX7HS=PKY$zL9010BGX!%Odj#n#oRl+hoop+co!#Ebc#AFY_N6Vs
z#^uj1Pt(-*`%zr)82<DOI&o%YDkEj2*N1c+T+D(I$LL(si%LTDjLx}JMKjsW_iEIs
zni-h?cA0hU&asrj&g7u0jkzj65>NLo?=LALp1oJS&fyHwh|C0N3xf$i12hMYXh5DK
zN7$@Ve{&O4COK>0<f~avh^EnyYLU?!lO7}^zxzr^)`)+-h<DI@zkGmvFOHfJT%y~q
z;Mv5sO*h4Ik`sOnnVB6MTq$>^vC+_c?;c0cONDc|QrmOmM2KMwac^tYgRJ#uP7}YD
znsoqCl==qW=?!+a1WjEjm&Q^?B51|?k7V2<l4xP+hvR7_|3vM)yMh?s>F&hd*rm4x
z_PD!WQ3ZDNIhV?M)dvlU8ZJVnri#qXPJc{!3S35h_q95M=mUZTHpak$Xy4J#Qh&=-
zKx6HY5Y=FJ-y0Qr0gXO~4lkLBwT6(7(nSq&*8~V2NcY-SNqyB_>9f&56)|}8n>^3X
zVP5$LZd@2N$xA3wh>uV)h*?JG%wcXrmZr7928HT`0>lVHs@gG2ImR<+7~5l!@dery
zxH<94buifbKongI>S=z)X(p?>QVQLx6FBX~z#cbwvS@R(?xmk{O{TWrX1d#yDL*th
zrH9K`uH3Dd-&(Df9T6*AgXlkfYbC$T4(<*_$se9EOZVv03B7)d#7K>WgHMM8B-Ihd
zK-5cyk8|QLb3ToGDgU7IH+JqeiLx*PsQb#@#-d#}v$siVQMCZZ3q~)0lxc^aF;!0=
zVXJ|@HOGASm1UG6q>>twk;H>I=&hLL8OrsXanT0t0hs~zac^bLOzlzN7^4!Zdzv%1
zc^c^ok0W;$@R}d>J|2*%eQt0WQw34+WT=4GhA~X$Xe@L)O)zH+wTMPhT#1;&+)V+Y
zW8M+3&H|P`yc=br{Tyj7nkyJ*PMU#nMklnA)EiQxD(M7jB_f)vihn){ujER)SxRXV
z_NZK@W$<|QQJZI4Rcd1W?8p=}*n(F;dunKsMTL`|($DpWcYB7A&LA<0pYoLT{HfI(
z@E?UvKDFtC$t$d!YsYQ)8qY7jf|Qyd^oU{J(z=Rny*1vP#oWtY6O&!EDjO_ivr<{d
zT?*pQPyBjY_p+t>9#W16L0s&QQJ~qWK9kTlm$fD?W?cQ&S*5>MqTYu#M9?Z(ipi9_
z+1%D`w2E{%Scr!5)*Byi8C!V+V~m}4n^@|+tHeZc_x~o%hTuyD(gJUQ%sA-AVF)=1
z>Kka<W1Q+~0ArUh?Hp<hiR1{p0{d<-mrM^+Eb?B$=pk){=*(GU2?uQw%VgzEFIh9;
zBz%Ef^JsgrMIU(BMj+meqS#NMA-+SKgwyg}0SyfO;)r>XQezRt+s$E?tqF6noMScf
zZWu|FbULM(22~gMmI}l^y?&0?MY&dk12p#{-z9j(HSiv`&r<RV1yaFh3FNJVz}z?j
zOqKSHaky8Zr1<?s4liE^NSs*AX@eA34e7Bpq74+7{_mfG!6ni)S$ulkCPi6zg*pt<
zHc$AWFC$;i)@winuR6v&{smNb%s+g+rheAXd0+A8ZfLbp=uP+IbT_5;E`j{-nUMRh
zrY}(w*4L9nptshs_4PN8MTXKD`#?%$+0^<OXYC_CVE!p6hM_tOOKln`YGPq@{PU_=
z8vE#o61n;r+S`i3MH*WTo$grNA)J`O)+~Qj@Dld*<-?VB6z0RhZ}v7&DFwlw%!>Ey
z)Wbb4$k7n~XBL!-nLw3;VeZ_<(7C<r=oJGp1Pm1i>5)e~VT2#D>`PW}`mQ>Uy)4^!
zV;t*i@zrPz@>B#FE$k!@-wWrE5{4~f_1Ek8%a)F2EU&!~BCwJkRSovC5-+<0t!!>h
zSz|n7uc{+;@W30dB9p+lMkLvdhl&tHotqZ(%8;&x72?b-9u)a%6s1h{Bf7J`ood>#
zfeb%(F+#qB(M2XC^?Py~*=J0<SuE{*x0T81Ox0G8q-Sw4@ad!Z$4;8CI!>%1iOh9q
zd3+p(xo1^s!O2jw-aEKy5|MbQsURW!rhT{ZD+|cY#=cjV`dJCymh;!oCE;8>#LP*u
z7piX5&ya|Z3&4-38p>Uh>cOblG|6DTK)5;|+BPjndb!aur2Km~${@dVdOi=CfzSa&
zJ~0cAU7A-4rw?GwsRMX%YXRBN>|a?3)3bSnY|f6Xt9i<WzUK?!Hkr+;W)7MrFM1n1
z-v$oAX20xh>&bMUcjSd7-WZarI`ny!RI=xOJLN-2(xjNVl9+k-8qYI46ROjL)>*#`
zu?kcW&K-Qtz1PWdl>_nK`eST+hX<*WwFbA==eyL5W>0Lnl6J4B7~n2~noMu*2uI@#
ziF{4;*J(H4F!%5n)R5tu{74Zt?`jqo9V8B6WrYI717F1$`EoLe7tpWUFMetU;@0QK
z1EpGdf~|<qz=mxAy-ChVF0WW(T^5uDlu6J}jhk5S14oSy!*izVrPn1}WxPBjkm1hO
zEsFHw?Cb1Uz!Y@rbaEw4)3nc4u{+lwv!gPIO6hHg(9rp&@i~x%<zf0|=#SHQ!gA*R
zty_%s6La<TEGtdLji<hEiu2kMp+nRql!4qXuY9pv@{5R9`R%7noDw14&b6`fa;W{y
z-FUXi+8MFD*P&3DZ#gTY(5{9$f{>l%^pjP-gb3TX<C6*1!yz>B2<Z3w0d<@4cK7t`
z<0fW!k?N8b?-UvS3LZ=nFF=a3f?y7so*h})oWX#6Yg3#V+Sk&(e}PY4h{Uuv2c@8y
z0$kHZ=_d_Je0mJ>(KQ+Fky({(qW>@QcwElY<6H&tZ*_~nBL5n_&9z&>z48j=_otmx
zt3_bct@F=BSIRbvFw{C9D%}n6^QKrYF|Br|>Ifx?NO&6%Uz?U&*#fu6(Bu%+Vj>E)
z1;&ySs^oSZ2b8Ir<<lkPt`HZF>D~o;JBGv=b#CuSP+dRxG$27)s%5l-q70)@q3VLw
z9f3Ao?<-cC$JFw=K<VNWS5C#-sr{At5*9}1e85q&jL$3jq!7QI(H(BlF9lw!de*!C
z20~gM;StPIUauOozE#zG+@_~=k)6$L8v)zpurfXL6H?lpI{YJEjtRx#To__zY6njb
zPE}LY0(zWdDHNWKfLJO*{s$D7Tum~I^gc^HkjWX;*Es9x5wWa*nA9TW1Rt4gJE>no
zjB`!&(U{}lm6xmQhpK=zu)Gyg*0OV$SqD3V9z6|ILoIHQ6`(Spxgnf9gp{?w6=`ha
z0xq87J{ZK-c<eDJ$r<8EN*Ep(3M^2@`aNzqrulp+XiK+-sd3oZ1@lD}s?#~i2pnSs
zCdJTxu;X`PG6^Hayd7~*AG|W6ow1&8^I;7$o4&b;^(_&Z#9XK<9k|@_)Pm=O_#S38
zQ63y|;U#K|9iS;|#y&0C#QRgkZ7EBPM8D--v*+-w6Sb{QG=mRm_DWQ>f!p5UYe>y3
zCUD4GnCCulIi<K-a*OC=qyScy&*XH<2e6P&^19jSHf%lCjEJrihan-3G0}k2m#B(&
z=+XW`8_TS!L$Sl)1d8-xx`6$wMXQ6%bcJs`eQ?v9DAYMXL<2)9wGwU9_%l>Q<6wS$
z+t6Yu)I8_JErAIGJ$_n47ZWPtWld1m)2aE)A%t~EJmd3OfhqF$ui<dTE;fV<yyhMk
z61wnkN<`yLQ+scc220ZlK#4BM3<BqXRCVM?L`^l3o_n*skY=AvIw)wm<(n{x<VU-#
z@fTVZtjs&<LL7h&7LMLA$tB^lqJAbd+AuWVys%R6A)~Jj*qm4%Ml+btow}<QeMTTN
zRnPs{ya`oNl3R%C_vA7l0*=KMCdd|bk2l91ca9J?yWHr;Cu`YvrbuZF2Tja5s%$T;
zbc)<W?D%{1od=(k!f*5o){;WD#wLz!v723aR5}nbBPbJnuqyInXF}fHF)K6SM`kkV
z&Og!Dz7&XrY@7X^km}yR9!(DTe+<75n>M~Q`MuLf@$sM`^r{8h(rD<e#$DVC<XB0!
zqwfdcTOE(1VCL?!Z3C-Ig~{)?k3vNwLk=SHOU6i>WA8wkU}-DOq$zfm$0{}~26NP@
zPo9`*L|k3llbX=7Wciwg&Z5GU(m1pLT^}=>i@o!Sns5_0@9z91lB@25L=yx=-^Ax$
zJ!Sq&$`VS1KSRF~K6$ql_}<I>18%(slqwrb(z5nNPdR(f$Dg0*$2p!Yt^Qj7g{u2P
z$c;Xg((u3@1Gmxd>QvWZ4j=Y145V@zwfhK7&(hWG$K+WhFV)r`3c^Ti6no`CZK`>=
zJ_N@^FC^ss0rK3VxyLSgdN>xAY6ZGwhTQP%dE1*&Fm|L#ph}f0vni#_uYQIxcJUF1
zST}@wBu62i%lr~3kguDZC)6l0d3z)U5JR!n_O3oIzvAenOUn9DbLLRDy<+jMeUWfE
z!!o1&-dN#;?%v9J947HY$79v|neC5ae;nH@1P0R*aIClbs%u<MYCH5jXfJQZO7vGC
zQZ)Ln)Ckf*Fqk;#%)EL1hg?Ni%&-m5-}9kJo?l35F~1CFF|aZ<bPE$~m6<DZd$0rn
zCOI~}a=+;YMce+T#xlelB;e?L)>;9;RwmLmjggXQlUtz<WgQZGjkfe#c=`j0V{MLD
z#Bp-1GDYExT&3?r?y}UlzZwG}x=jF3%#2Hac6EXLRVplm<CG~3yOg*4I9-Pk$4dB#
zcp>mBKBiO)eu?`xGa95xbmN|t{`@>fI&{$e38Xwsv|l=ECqbFxyo4K6a=z4mc_m~6
zTnNYK1s-scn7W(r^WDfLu?Ug**)L6{ABZU5M)hFaBWE>z-_|5>d2zSPmX72E`USIQ
zE_dka7R{GBkI@M)UMwX<15(Ju!RL)YNPq0iGBk=i2EeD{m*=wUPDOO8O9GS9QKv>g
zH&)xkaw}hgObe868^k*=U0@nb*473-<CbXa?#`I-W&b>~O$UQ{QnpRYS2sPkIAfNa
zH4LA{{R+7MU6b`V4CxQ*4P(`B^yl{7I=|<_Lf%{MrT_p>z0*D#`Z@XV{XPpd0cf<?
z4eF&Yr*s3Bk70-@Y0b5|wE9$}uWQB`&Ffp1u8lRX_JWB>gX~8kW!3@*$xWn8w@{ei
zb?#I8RpLr4A7p~ua>)CD)%?}CY1>=>7dT-%X%u9_pjITU){4%#-8nSS34~_R6&U&P
z7If3PumvW;3ymWZ=pBCtgv27Cp)kif2q>kf3S8}_Bd-kUFD!?+-c%bHzDU)aCk17d
zxw_0o;N5RpheM&C*#L)TBKZ!Gkc1UJ)hm+Wk;aa57<|7N4TkuzD-hYmP9wF1a~a<#
z-B-Pmiqk<9<@%FUBfgtOtpss;JzDnReBuH~mDu4tAQp(elfX{$D5iSgB|n%@V_7x#
zA43)5vsu$^m>C_xB|~`W2|E<ml^JjuUyZlexWYg><`iz_!w|RR&dtPWNw+|l7L%`L
z>6h)=0s66&p3&MsBE4cszI&tf{v0B0u}@ao&aeCYmaDz`>rRHt`i=+xqgBJF|N1%W
zS+XBb1vgP_7<bOo6_0-Z?7a{XQ9remX6Cthg{TckI<1gx-UO7v2dG!C=IO+Rr`PW5
zR)KQ})wtGHC&)Bi!GX$~mz>ZeE;FHgpBG3D3ybdQ))4Q!1DSQ16vxFe@y;CJrqIbj
zHRYu&7Za2B0q<I;^ysOQrPgStPn!mU&Zie)uq4yNjdzz`!z`~C2ngsd;9D96?pgy~
z#Ym_uK&Y(0Jo%u3DFJXytdlT00fO=Uogwp5g|dx==r~|yjGeC0XAm?}s_0y*GuRX?
zI&yEOE9+V_?*Skl%Fm8lNEZlg;%p?ulg0oUsZ{4HuPLz})7R1VW(i-A3T=pHJTVuC
ze0fHk-B=Lh3WUd<mU@=Qxg<YBr9p2n`;98qDyD2}b`mO1XG46m?<j^mLSPg3fP54V
zZ7^)`^g1x4NLPb7biM|~LYu}raoX<H1@tlEpZKrMEUY-U6XMMV&WKLVM+5YPH&bmz
z{LAJ*md1}VtdqvAV#ogzIP*O5@Vn(Zcj_6pJ6`?;quh;E<n;GIa#=s`D3}BScFQ9T
z$d7ZvE0SZw!sX)^hP)_!8B9-()pi&XtwPcE6etHeA-S(w*R&B{(r%~eG@HC4F;iCr
z>*}mEmctNz65u%0Le~wT635O-D(|+E6E_-~BZeSw>uCyRp>9jc70tSBJfqT)kO1K`
za|ATY8<@tK4QP{E5OFS(eYWa4{r78*J@s}xQih6SxX%lyFPaeVA@ljRq`+{zp+=`t
zO4#=FaJ+nVfkj#y0WXg#c8xQ=Gn6pzw*F%zS95Qa;JnG{G%2V%8k+FeoB_bAs38#<
zYU%-fyG+aUO&DqHY%|b=z&S<{HVYmxC=xnC3a_&MH0u$TF`n53*V`sM&jH*jnN(a-
zLB<&d71Y(w{@b-eL;8lL5Qc|ysYS9A;#)rh-9L&wcI#L%(-niFW3=^DI<5e)Xwj~>
z8HM%hJs=|zg`pM7k*;BK=nHy~*<==*ed+?UV<G|&*6NUA(wjp9^1K<k5Z31cHdb4Z
zU+7P9l$~yx1>6CTEDX_8*9oCzzw<eKP$gYat$yEPy~sq<DimnB5T+2ovKC#$EnT{!
zFf48=Kzh=U+BLd=y?Y+j#`9zN>2?+e68EUZ1N_CT#7FOYKu$~Eoyeivn7-*$Ni5`Y
zL~NwNZ=?&UJa!|0eHw*9Ja?1y{t`9Wl<JzG<1r2EIif&@f|8yBAO9<eo$61qfrsa|
znP4Vhs4AcK%?FQBNF8|eZImUoke1>YN`1JIJx>YfQl&;`==knYOUzsybil14h;b<_
zsPib*4VmS3t%x{zTtUZX=e^Fu*8K9b4li4pIOXLiBxg>e7J-PVlD+NB^c*GP_id=#
zSTV}F70;AyZ5jsC^hJd%J0b}D<&}uG@+MAiDcK&KQG!3hY?TiVB_ERN_WG^4{;l=)
z{)c`iklCqNp+P<1cKth3gNF^~T*9K8s8>}Qz#1h+7aCY^m992Qz3Cyl;Sk`naC>2-
zR$SvHMWnnO)Knkq0`=#^(DyNd2o`U-RfQiCQWX%H4{omH#|m;zmOHEvSX^jM0Sf0>
z3EL|66NF*1z`;*|x?Gc7^Pz%_HxJ<!DR=VfN$O=3HXV{wd6eSRbRp)|vD8uou@<`^
zqSwN}{dePV{yYS580r27vB8`7_3UwA@{5d#;;<c2R>S0CeN>ah-9jWIDPpE&gESM%
z5~{7r6`i+;o;a(AghJudNi{gcI9OO})~|2@QYvk&-4|vy?aE$S6XCg@a9;^oS_Cjr
zoI(VYQI&Kcr{cbFb+%{=5=ptL)Gsq$^s32TfHZ{69F!57#K!?}acJ_UJrjpJIDZ-d
z&1-eWA1*&Y@Y}X;qr3j;AfO*r?PChgr1++yP^!eS=lShz!Y@D=@+-kd30&1NEbQ;R
zh#h1Ee!6x7(NK-od4vft1t1w=j_ZX&@j~Fex{db$17eL=6D%n~oz(3CL5*35#@8%}
zTYfZ>{Z{OiS#eHN#Sj7_ile6eAQQ1{Zlw5Q$z!pCW0NByQ`(E{4>hFMh*F|-_VF$f
z+BON?J~>9Z66UyI@84tM&OZS3)^$=Zq-TU1OI=9u5x|_5{W+O-cKLECvwlpdNtiGB
zf<TV(bttpHvg&tZ6p1*`WIpIJ|A32RVhsb`5;e74$Q(ms6Uv`AZv?|*IoS>sM%4b@
z_BoSI==ft}6JZinie?Fw2-f)VHif4!v*@u%Z+DW9risaZl%6aU@0u4)sh>yqf3h(z
zeX#|0G%R9`)tZbxk~QAvfX!I_PvKAX#kV4K7}?^qGSU+H0dWUt(0S5}{mhL|#rNuo
zf52JFZ64<S`O<WaQ6cHJ!atm~KY=o9{n5-9z&-18RkiM&_2pp_c7lOO<7BP57tE*H
zUS||eolqgRo+`h9(R%lqlVsunOi1~F0IpPM$Ttp2Hz5%dPtxcCC>Y;a@)?3I45h~U
zeJg7a-C9aRX}NKz4a(5n8r&?xrkeiiSL4{`^??F2G(Q$5s~D}1!I96ky_GY#fK2>`
zK@C|)MDB*>%ZPlzT>gYO>a7)1@<Or`<rrPd0*>5tYj^~ay9qwdJnYv#J-d--Zsre_
zjmXotJyD|Mn1XAm*KbdTv;d6y$lXxa*y;45Mm;runR_rk>4i*@C1!mr$|}~e6$`nh
zac2q~gDgNzgzP3}=p}CT>hJoX_AT{TAFlH%p<iN+haojdSwe(a28&nOn$$S`4Cz{0
zi5}E%nT`CB2}s(JG=w8Old^8<C**>{FozwSXPPYY0ddTm8vwNXx=<1cT!yziGj8wx
z5C9*kkfIY8a;}AvhnZP$NQr_PRc5b<%j*4mK<5?4IY`JSVQPTVXmMwm8BQpMeE38?
z<61y3WOi)a!WK9*M^l74aO+minw}m5GzzQIDiIr7hi*SQ_`lCXl^^<stYM_sp<gMm
zf1}3t0hC3Lf|}vJk9$@@hM$z!7k7iR`H|?VY)E|gGSHp?)bhc_D^3CgT2*B1b~nt4
zo}*SKiG)73=QQ!qF1!Mk<ZS3Q@nx_*X`)#aj@Muqj8`cYp;ym3@cj6J-Z#rp=v+f;
z#zYw98(~u)Ev|KlU;pNHMN%FMA}SRCSXPp0(ueS}0(^_QggJzj>fV8ddUC^HNd>Il
za5=k<H^XZ5B=)*SwC5K!{HMqO7Ex$pQhgSYl!jWbs^8H3(WF>#s<Xc1n8}#8s%k_f
zkpRIbA$|wvB?t6M*(pwOcCC&QS>?D7{Pz8J+GbzhzktagvWqquvL_)f*SX|gR_WMa
za*a@clqTYEYn-@6k@EeY(XIVl>sUGLn`TA-C5HzH)02R?i9k)|JOX25XQjQNw;U?_
zLn27M2x~B3H}2=MkorFx#yF9ldBP^M-4?`ux3tNzzQLb$^4<7OvGv&<>r?ak_&?gF
zTDeX`6Rv;JYQA&dPd0<%VWgq$cXImoze1Hgpz*Q7W{=P5pY02<+e*oD!)giv6c2Cw
zh@@k4n7e(d{^Dd+i}~(}SXD0H;92gZQJfOpFbl!9xf1*GVn3HWZu;|g{a5$lznVJG
zwxkocDo9N-Zynz<k$vc0^L=d-+vU6KvrZl~9gsJw+*-?bXs`~U?Sd{hp636bU2Z4u
zPw25(6k_w;Hx37N4<jvxdlj_fnWLp_Sp9GRA8qd)Pvsx}k4Lh~3T4)bM2L{AaD*Zm
z$;gPxDtpUF4lOgM%<Lk2geWU5$<D}Lk-hf_zw5?P@Adio9^dcd`ya<~->-FDuWLP@
zS9e!t7p?OT;r~QT_#uc1s+h|=py|I%84wc-!siVOkF5LO-MEgG*Ib!2z`-%&&&PHj
zmMI`xw3@tom^%FS+yC7E|A{jJ0D}qmS1As(j^5ZP<6XPZ|DX6w#kQsV`KG<H#uDTI
zc?$jxfBoxGlS}p>KX7Iwyr)c3QoBEg<o#;v7LU{}{I3UEir$suH524duVVlTdL@Nz
ztVFu~11{|WtAc)oFhPVIo!>fjDyNZCeXiE~-%hpa2?UPXc}MovuR0<bZHM_x(=Iiv
ziPH3MSe0B!>3bn>`=Rmfn)?E>Wrv2}@n)p6D^g@Ef7g4z+JYMx^5QuB4%Y*m=bX1r
z>wb2$C;CUhJ;L8Wgx_)ye-@yqpPLoiDabNby<WXvy-=K3NKe<&Tt!2fd&VuS{JDsf
zYby?<FHd{3Rm*UR^?iDTx$%M9%|}T?Y<s%|B*p<9T*i*B#+?NZO%GS)Od_+b-{Vz4
z)@U3V6nCJp6$tva;zN!6_FACcBOv3F8hKv}uK||xlI*I9CGS2zRDN{vP2+E%7V0hj
zCKZ6n#zOj4Hrs9t3)ew_*qfPYL%l-5y71`d(@IxIaNzw@P^24FruSldP`43^C+n==
z`$LH!&M~LPwk03KNsE-01t>oq_fU{2VCuKJj==cu0r|+U#T=ZRJYmn8)?ks|U{g)a
zyW1||tkM5-c5U(VTR}yjJP`eKNhr@INfnnUiPu7+?2Csiw)iAakXYG;!C^Lm&QTHK
zt<Oa+k#5Ui%K6Re7*WN%^n6-+Tm7uhMaNlcBN>WfaFxe_+?Xx+zjeeRG41{c3a`a4
zui24T(|=Ou&9nKcy3Gqg9Ip<~%3WX@-&&1Q;5sUe>-p^*pp$^J?fy&_4*i^>g1lQ)
z+#+w8aDRz)2#}t98kEKDrwSA1#KM8rjm-BB!2;8~7K`WK`+RNpvY1)9cL2wL;0#!G
z&P+*@t+~b$gPNeh!*b#@9VUemcm-01jCuBNPnQ%@{Yksh@O@uaJNBi^KIr0wHQl(M
zv831kjf$nRWTCqjhqC8*hiKym;vDr^_wVWeLz1X#>4$K@*>^nyf`d2WK3bA_pBv+O
zwzV@Lxq)mT?%-QO<O2*RaD>HH?DnYKruD!hU_t5!3^_@0%mBfgjB@7_zvTh3OZy=P
z^Xwi)Y@#w1@IBzFWdcv)&9XOmS<wvFiM_e{|B7hhg+-qR%R@;X)l8eL%1xZ2#BE6&
z6I$#U{5whqhn+7@z18HUcD|VvPImLUb@-db_~~^H%cnP*9$UORnv8?UbIQVwUKp0o
z*o~}rtPjF9@}p+39;baU2q|C^iiZ@(;y}B(2<(Y}aE+K^7uRX^9u#kjwTdW)Ie@Zt
z+^Kgk!akvVmEvKjx2u)*<l`TX$Ik)3g)~-ld%I(wAp92bUH#gdAEc;(0^l6K&BM68
z_mmu0;Ovv`kk?TrS#M^QdQP<;x}7eUD1O$Q8>`7fv%C}@$UlaoqIg1VFCu4H8dH@#
zr~DGHvZGlxnvpgu=UzmE+g-J@FBzgp+ZkTjP~y^*S&*ir?RcxpunD$$jSz~R5U<A3
zLF~e3<OBu4$XPy(o8GNWR>T*#@+{t9%d?1N8S+V|`Q-7RDzdyUq`3E+@#Pa|*2hPO
zO~{Gm$*<(8^k2Gc`Pu2!X*SiLw@M^^>@h~%_blkAMFh9Uaa0vHLdJ2g|JF2ODd8F*
z{q}qZ`ma(A^fzkN#kH@ucfunE2JX)Jxi!SGoxE6<yg(}6G0OTe{i4rB!Jlaky+2!U
zFmPF#lXnk|1K&)o!G}@Ze2sps%a-f8Q??0h5erawA=1=;rscbgzroI~AOtx5T|e8l
zc0~r+mEoH_I3+;Yat$HCxptNlrxQsx32?~M3EDHfWEKiI-hJ~N23jIi_FU>;X^8+m
zU(9>pn_-DtW$Me0D)!l6&dzn}V?;5d3ng$lJ=KrhKOsJ=&&IYrlwFu1xTDVHKR>#)
zN+bw3RUET)&d%RIksv&Nzp0`y{Cg2<BGFjY32~=i!E%$AXT{QWx%qolu_C;Kyb2^Z
z;SVo8;OaXksT_aZ_tzKY!b|<*g)^ZqWQ>T_DvC{v<_I-$Y)ZJxY8SpN5F^DBKT_NI
ziyA^jmFi<Tz_KGQGQ?{2#e*3yWL%h-{T<+PKHlu(wkn@Hv%{3~3n3%P9t~$<cB<Cb
zP$*jUDDO9W9-N>CudYX4P5dvfX5>J7mJ;II3U8kll!_oihGAOpZ0`<YwSuRvFZ8`)
zn2ps`N&JMj=u?lpZiJ!hDT}}awbm*2WKEgEt#y+t0fw$)@{=4`3-=?yESI}b;{3E+
z$LSoGLUILFQm$)l#V~ya0Zt0Hw;5qy0TY3&S9l-pFG-?EvdQ6U&7R+O9?hlLQ51H4
z&=*LditB-lr^{8P|L#+n@9F;TOl(^Vm85}bbC5mVt{C6mYh30b@r`z_yOku=r|Oiv
z3r>~(z5$V+;t}qfASYDyj&{Z!fz7dCf}oi4MD`*Zi@y%-_nu&1S&2L9hU*bv+n>np
zpZyS!xJ_;q%w3mOELX@Z+qSsB^#(7NuzGJ{+s|&FBDd&er|(Ax{wPuTZT$enAly@G
zXm(qY->2cvp91$|9x!^3ImF2&cA9lvQ>0s37r=hrD-&{uE&G=kmeyBB<Am0RcFtKD
zeKXBx%_H-f4rVgkv-bT}tD>`7`XRTx)+{BAzdPW<(4P;~Ny<M<2yt!>ZCszuDTxS`
zSbi50ZgjqTv~hL12_0`KjdcM!NpHEemn>vDKBwBQ0O2H+;=Y@2-?nyfUv}_c7ig}k
zpKbd*(5?FPXz6cr=8siH-zOdtvi|is@vP|6%AGy_$Fef=)UT=u^t`gUz;IDud;3rb
zL10&re19<!*U<AfG}}V;*N1!u;w1FZKUVRNckjagNqMv5LVUB8Y~J$cl|=T?7i5uJ
z3I=1kzmDGCSi<PZZp%|SQ`~7Uxdydy@`XEi$eU7t`yInMV$OrJy8beBV9VvGOGyRx
zF<H@xXQgG|j&2Q-Q|1~-W%6BWE#aKAJdgf;;-&f?1bBB3{yR|#Oe2BzW^uaQ;}<fb
zldP(V?zd{ITen`GB(p^;$-PM{xvt}_SuXf*_B1LGtUNS13FF_!jQbVCPkQPmPxibN
ziw-lYKCkfBBX|Ew+%A42;Ko8%+A0^l{-=X78ax4xbT~Sqzd9yK$R)xV&-!-Xzitp^
z5*h>CIih1M^(CQlj8D1io<IG1FzG7hU<KROpjD*t%%aK{W4zs#Gs-=ONA(rnrCc{+
z!j)nEDFPno6@)bR?qS<KfBASPr=oHKs78EGO>)$O05;^Y_-lKgGCy7)8!lITOtOT*
zrx7}*yi6-GZ5GHB9o2tb<vL(g!ccRRVQBR+uQ6v*2*(dmI0e9Q62=brrUYN|k=i2r
z_BIfu0r6Sw6WNYUE6vY>S<EzmRI*_G^$ZdAXLoq>St>>RrR$jTAI_td8JXc5|BB#O
znMcI?4)LzPoY22b<32Hy)7Y!%A5LqtG0_ehxSjg*L~eb?5elpeKc5U1{p|sN{u+1z
zc1E?;;uHhMesAs=Q^&fj-JiwPFwy@5O2lt+u+J!Rc=QjI1SJAWVp`!_E>jaau6rg8
zaO`G?Jl-d@p43XB`?)1ma*;4z@6r7X_jdN`_c{cUf`$+Av3!n&;iH*goj%1~hyt&K
z6BrFSD8zmXrVj^vLdBQ7<)>g~5JT#aQw=Ua@t4^<ZbDcoCN`RT3iO4!wrG4YEO6%Q
z&>9)K!#W0y1eh#QH!fpVxbP?u_Qb%d2Ude!b|1--ailMt3V?&I7*WpHB(#p@L`>HT
z%faWkAz>I0+Cqj{n;jzjT)3`%GhMei(oxx^0phL~AQjgiMK56Z^$A;aAH*t8S+uFS
zEDRk%kc|1=kn-o|LZ<GIn5tAd@m^W}pJa|~%Tp2y*rO>)Id9dF4=@+NJc(6x4H@0x
zt@Jqo#^c{f)u!iB+kD_M2N14<eTPq9gaR_wn5t(#<!OLgBa3=>f%)rTX#T5l$bIgT
z)@xJsemhl4IBLW+;C%)fdpTVze1A_3w@tLipm_4W14rnlcdz1)oMwYeJDgwUfmTF}
z_+zF_IPX808>zM|_T8qJ-a|5$+ZXU$fMf`vm|(xulNBc|K7WSa(cHLfm*PIi`8yiN
z)G$qbZ?|%-c!j8;#w%d0GBo@Yr))2hq1o2^7*4PQlPW|bD>rSDBpF2eywZGo45xkI
zXqFKBNCzW=jK~eDdefA4-6j}U3G@*6MMHf^WZ$VvrAqi&LdI4)#V&$5L@mNatR4tu
zrX**;3;h!<E6F>IA}NUXY&vl#1Ng4MfjJBQds1LPKR-@BF)e$5u+8y?5^k}MZyBo}
zw>Z-=o_jiSh-JjXT6aIcfUu^rAY6*!DLtMo2QHT3ZJ%0|UKptQZ6Z78%@3LoYBIq&
z{$?ES9x;tHI4VAIa?H4Ykl&nH1a4+o9^Ksd*Mm>GqQl5nVVGs`=cX{8NleQC+jE@Q
zsUht3>fd{Hc@QfP&4%!Sc^6mwzCdh<!69WzX~c0M-#UQABD!!>VIZO5_uZ94)c7b2
z$ShHaf3SSdwB&I6vv!+p1DaUvC43s(nyE{>`3`l2fknNu`ty!nWJDqEj>nfc+)S{W
zGfBtIgcXyNLk#oqzbyi+_gjqh$;L)m>EVp$aZdhEh^eT(y;B*y`GJ~0=oj^dCS4Ak
z@tTDJUnUph!GsGK(Xu1w@v6`Ec@#YGXlo*2p}VR6UrodS<F+-CTOuG?f1gO{Mvl`l
zYXS9*!y9e+K7nbis*YF=P8l((xF=`pXUuqM$Jxf`CpB+TFU$JMMJ&Y2?$wq%u3*Sj
zIJUYLGnww%DP}iWb*A!6ZI!^}tc$<vf@{qxun3q9tcnxiQ+n(oAmzb}eoQ*i#xFpO
z-i1$DPKPf{Y*?i#aB1^b_yQ_W8dfPt+T+wi_+=L!F*>%Cs%M$QS%I{@|5btsF&cHA
z;6@Zn=`fziSp@_5^$<Mq2UUf;h<Q$JKnU^@<lks6;-5K8TjgM+vH3~hB?4>8@}!@}
z*|;yDQA7x*A<@)LQg{iB#RH8wS~`1aCWJQctARo-{2l=1Hk&W{Z~=J{6@KCw2EEZW
zvk#RV-pU>(wM)>?4fnssR6VT%Io~5%&u%};!wZek)i}0i`wP1Wo}fJrS3KDj%yR^p
z6?{w14F8u(NYT%}Rsm=8VR~2%K1}Ro0%38~mn*}YPgzGEy3t|>vqpa3UXZOHpPb!A
zgo(eFaCzr%D!WLgq_yyL)m{?pyi_%4S8a6f-pff@<?HGj&cPyA;_YRzPkvU7?!x|9
z^`_xsRn6^}%w%Dc@u68ARDY8s7stY~&@>mfMpK{pFe=r%?N&AG_RDRbddw@Ege(?l
z$~1hBzMJ9dv_aiG8hv%6B+qxO>$=`Rfthc4O*Koq(6eSjY`U3DUK&kn?vI#IqZL*(
zs*vFJLKTllo76vMf(m>Nn<Mw_{`D(6b42|lhPhOnAldD?GwVGe=w+AQ%Q`}mMQNxI
znr{90wo3y|7vfb7u)9a;yf%L9v3XfVvb9cu;`pwPK4}|Q4V##-KN~ijlLheb@V)oL
zdR|g(^xApm?-wc(_n-og?frZbw>=){66&6Z*>3vygxedoc|4C0_&kf+FnfJaBrV5g
z-_5mJ%ME&dhnFN<OJ2^0C$4yHIiF>=i%RRve9UEH+m7SR|J=RJla0EAyi>yLv)s-*
z|28m;=fzmv6WDXbG9yu(&)JD6H~$Y-u$&$f6(KY5j(ukKHkQoS?)$;?|9D2=F=TO{
z-oFvHGuOZE02+@?r6fbHva%`}?@I*9h-MxE?o8Q+dKf8DFX<W%5Mv$o+U1+M{L9iI
zk3;)Vl}e#LAGp{1A9LM>Pk%O6&AE@x8oW2lGTvD02N$2++7y;u1T$EV*P8v!LVhnw
z8(7-E|LjA*5!@2~zYP_n=sX<zEk*E;zilrL9zj)ewqgyH$jlAw4<r1Fuec+GN+xi$
zT>c_|t%hsh^ng7c{(roKloAix`PJRvr^T%UZrO1ni26(tl!2;W=fRGo=SbFdw#%-s
zOB*(jVyjcD$ji-S^Q<UaU&$MW^4;!utu8Y<)^#M?DX~2I>OjG2Oz7(`y44e3U0ye$
zRQom<&b4X!9nE=nGWQ;`_n2$=D|YcD_lq>`Y?ZEyiw}02#jzaIo{Q1M-e8fv5T<E^
zea<H96MUMtPet>_U65p%3*oi(Yn<L=`s(rK)pti)gv!e#D@%ZkxUgJfR_g+8$bVZc
zG~N-=P<UE$FL5gDZ$}T0I9?bd-NrEJA?9b(htXn8Vp!!3KbHMV2P^qbs0L5L@6tQI
zG4}24=8r-j>AR+~K#5H`|4fsMYAsDY(KGGd_^7Oepc1~qyNU~3Bioz(kF_)+0V&8<
z_*nY)zu<<9!IzG`u8hs|f@LevvGu&(RLk<N=vem63duzLqb+`>TPGXi#}=DN)+iw*
zBk9B?c!9}Fv`f*o)NMjZu82(Vzog@zst5PhZ5;{=W!-4RQcZO9G`*mT?owQNWT<ST
zlm|{HpcjbMOu6fq%&`URY!k7IA33n}tl3VI+W(^HkpDr+B0|ry_x;KLo~~Tan>ZG-
z0`}9dem2OKxT_g<{`xd>dGGcWfLS7_Xr>k!V8EKD|7dPvu+u-deB<^FhveUP&0ijk
zp*ad$P0B+mjT3~wO%bl;T^Jc<@|L_BI+iZh?4=z1leb>tZmCQ7uiJc+`t*`#rises
z@u%r8?f&~+wD~&J|Gtm8Y;{?Q<*$Uw4zJf|%Dn74@>~gjN7p=2Ku7Iv>pr6TLh;wC
zoAbMJ-ptRLp_+mWG1TsfBD@bnFX8W8Yal3Th%Qh4lti)hp_~X<67d~RPNM!*W-+;_
zQVOG6uP!e0mrJ+#ME-({?!zl9$ys{r<oRc!8e=XHsocIgd<m**U!2!7)bqNrcx#+?
zxrYJE)^MY_GPPbx;N#?+dF2%Ug+CWKP#yn_t8p;h{Pr0;j;N?0o&AKrui?LZ{Q$vI
z9QTQk<F!Ou1)k-=@fmdH<gf3h)=QtoS>Bb<>pk3$o5B-?Sa%_8--9y8q4p9z!w;=2
zI?czNxV`6Ua!go)l6L=Ik3dG`dL{Fj+m_~XO<0b8E_X}v-`~NVSP=BUJY)>c3PL@;
zQzhq_59)<XvJH%WZ#Z-?+1-9gu2H1t1*6OXWwK0Z5}b~7mX6gR!ctE@9zU8j#)x&w
zJlXekkoxZJg^SV0>KDL-ksAUNW>ec>_OX}dm^|3)F)WR&DOw5z=TGX2?7;64%y{mS
z4EnjpXac8w(4tgu+4iSv6l7c2W|Z?amkrD;cSbT%Mo=bfKedQ0mYsy@uELnB`+OEi
zp+U2}<E;XT!)y@>mU9&C9kFtUaaSD8QN;R-Z!+ZB{3I|AvHt4ro3!D+fJRjkB$M5y
z!f^qBC0JR06E{9I9y?+Q9uQ`34dLXqU0`JSeBq&8__AO5V(tCV%eZrhUZ=cigO#iF
z{Ww`{=Ubj?;$yCJP>;QpZFtEo$o$ybCP{NU-=k}|8&O4tc*;#pMdOL{<|k-EYPd4J
z4#p&JuF|IV=ca-@Ab0O`e%z|lqdf*+^q;U5%@(&zlJlDAy%^%oHq)j4vV-)Gy;+ti
z{blGfV!$HUcCY*H#3+mYJ_=rCM|n|XapW!MVR2aGC2@17p+OcWDA@4O*5bTJ7N_U4
z$q4QXXp{nC8Y{d>+P8gr(1etT4elFhX<pe(J3O5Y()0eJeSR;A$>3zh$M>wP2XQ*#
zp+_{K{dX=F7c$cCdRXtlM`=2AYR`TZ*}F<ykid2n?l!l3%qEUgjw&aSH97emtF^Tz
z=a4md5gPOLk2OJZquiy>Id|4XYI9BO3SNm1U&m^Xe{goY9aa9L_CqD~g4p^JN|3nG
zl-IUiy4T=at=*b#?c+ep+VjirISLo=SvY?EMvgnFo)_R*o(0)eTgsLH(Jpd)`rMX_
zw7BUI_dkMU@K55y{<s^2!-L?q#D%WsFR9yYU(2`@8($gHOvk&vY>)jKx`!qZr-acL
z_6y&+C-++8h;)uj*&<DAmT>$T@7Cctw;-8)^1}>y^6xJs;#NGK51g3#+tYNSTjTFJ
z4B{KRqZ&coLFXn}l^c;sJC7gO+!3=VCG++!RDF`>$-QQo<)0*2MhA#OpYFx&+Xxkh
zzH<B*;l5Q(;oIhtj}L|~psls|?HY!cIl`Hqd=oN%Dith&o0^;noQ#Co5kAK)4dkJ?
ziwN%?PAU++K?k9X3x3qEaSE-R3I2o4D!t*p#kc(|gT=h6ZX4GQ?7i3;q~lpxejoW4
zMSM1ekRTw)RRHA6_#tS*#}0Ia#;0Amma0t~la4@lZknGjd!eUh2m<NsQI4b^@GB+x
zC_^sJ_U18Ixn<9?JH<RJp9V+p)SmzP>@NT6eGC5KdLi6-7?y)QwAU3Z+di#RV2YML
z^di9h9W)RJ0@9R~#>0I?k8FGY3AchQ@i{{*^#sY_RM{SuRCjXTL;^6b`7!(1E&$p!
zY*7RMI&_Sk0gjE<%yganV|6UzDUGR`$^+FAgXE6T0$CMkc7+r_Dfa*z4~qbcsw2-h
zXWLJ7v?@|#&?w-olPT~g;m#IF?BLGs@tT@342rLuy~IBdJtlEI!u-SB_Wk97E<oJe
zP7S~5?ThF+*a?*P@9o1~25<<_;2P6Bx_=S(5r5b&SP`9H_Saq8`dFGClMKWL;EkZr
zpoMLN(Euhe8gF~8HPQeO=K}lWBEWs}LusHEP&$XSC2AWn5oG7F2osO>sY3(r;~HsU
z5z~MZF0^66BWRRDGw<>Ikew;pRm_rf)Y*A>#x3X!{?}3ZZRdDP(Bj#Vb8FX0w{5PM
zO2dJn*!LWFu0g2zkb}EVm<A{Df!Iy)$xhte{$tfwcgh=4B}`xo@J~kd@Y^2Qg)yft
z5AgC>G>#Cm3V~Ch7wtSiZ{?`{*@m*b&9)bn^|(2V&!8XR5l#Ya`BuCLV`jv${Zc(y
zJS^i@kIR1v!KF>#aLI7+WzURZ|I3faYDXQs^WiivhaClO>FT{<>4tN){kA2YQVuR{
zG`Atn3V=fo?dyDFIgT3Q^mkw{GV<tM^%#qBY8fRrRM`88aTeZaXqIGVE%FwQSsqD0
zESyyuJLx`#Q*r>j4Ij7ZH<5;QK6jW1bEk-fd6T;&Chb)wA>U!ljRe^hcMjFuc;0qJ
zH+!%0=f^Gihp}bFAFcQXo50BUK`r+Xq6|5)gt&7{SuPH?#=s#3mu*w7pBh}18Im*k
zh^TD|>aFt-XQ;RQN_P_XiTp9JeZKCFB(IJBLQZw7g*EK`<Y8Yox69V_$fyH|2}FqJ
zMw+-%YJuh!%9#+vhCMW<ybv<SxCHP4%>dvpTm}-`7i}+H2+8rabdE3)EKGlpx^~$2
zo<Y>a!DIs^Qex6~7b%x#N@DlV)cd1F&$zy1yj+IULpy_m*OlxC6|X;SXNTJgRNy(p
z8u35i8@IFim{aVrh4T;gbcRdPo?U#!LOV+IxZk&z#ZI4s2f1s?sW2!a%-Q>Mi2+k8
zImU|1?>1H?hrg#sWI|<q@loK$pZ=L3jZcJ0OW#k9zj_-vHnJc190=7COJlPJG_lRK
zdpVErb`r;b%rPn8Gb;My#NvMJ=R($&R#FHTiQ0vKO9olno3H40E+d$bDhUlIH-C=;
zmgTdTmB2bT_~Txiy3bYyB}R_#@{vW>r#$yoLp<RBCe{GVdO|%#4I!U@<2+3HUJ7NF
zde5s=LJtJd0BfBF?!MkYyhm)7Nn&}Dfoxi_R6;*acK*=B$d9OQ#Felbddr8~Y246v
zj%)}4c@hs8Fz=<ii?15aP19R(Bi+|U09oKgzZS6y@Qklr#T#_$Bji&EXCDmn9CQ@s
zD(G9Si4rpc6_R~>`iAS`BJfFm8mm3tad%@a0lHP!=ia)!s-9=A)xJx25&_3(<y%Hw
z793Fb&q~!4GF!7+0{I*dV_-ZCD8c7zJOJh?3t;BjdyvOIrIm0Hq1J=C+3x6WKt*sM
zj5FNYx#j6k<>Ifry5V@(pHV1p@G`{zmjO#W?mE2I*m9MgZU2Q}PNT=I$%O`A$d^Y_
zGb0T<QZvp+i8^$+z^L-w04QWBsuzDTod5Z(_4jz_Grl0o(E#+AYS3PsIdgCRnL={>
zHq6KkAkDGVXF!MnEka7F=tzuutNNxsQXFgoJ}^6=e>~1_P`0lXcnbZ#Px~TQRCR<<
zzJ7|NkvNd!JO2#U9DB16s7{18xr&tX&OkZ#r^AnuzhxUKd|P}<ZSM|>%kfEx8R*^w
zsAhJIHqu&ftxi^XbwhR-_*>JDL^VJuH&B<jV}Ub73#dJhGoCiLPB9+Veq((-;{J>%
zAk;p6D+A_p6r^dMKpvuX?dJn8G+Z}7p-TLzG=;dDy?>r%r|E`8{X1nS?x*_sEVvGM
z1zd~TjFI~a5O8m+=dF5xj^A{HS&b+J5GrkHLE#yglDpI-E71+_Si7u&<gXHV3@41Q
z0){TnvD44vQBCc2dKKNN9BA45Zh2Kd*KkT#pO5{ix$Xhob3FfX$RK!QG*P{m@a8VE
zSiDmngmt?Z(7XTf3r6OYN_sGxmoecx7eBAK>%xGhG+<>CqkG{Hp$-7I69#pRIWyku
z(R87yd}s5cZ~q{v6lT7+VC0vdZuM^MJ3nEU&FU(!mW_^nq1&TVCfppFNMbN%4l-3e
zF$wIuul;(h)tC+XEgU8~L?*=N3s;`Y`f5u**!Rlsy^We=mmfewwKese6MkCz&5Liy
zvq$w?m<mAidF``(-`u-z_mSzZLQik5xI8ddauLW?<yhmS*`(r-%kWA>)Ef0j10BXS
zqJ8>`-D7S6a+Z<>XjHE_R08nZEQB50j4`~<;Sk@Ad+N_+yzi~d1yl<&5{y*7eaKgp
zAML(!tCFr)r%bB!YaaCEp4S{&^Lr6s#3bLbQug*}Zidqt2BET5OEG^s>0fnuWm;wF
zsgnX0nUh}u#>Vq9#NX~w6CfVz+*!Z$$InkbC?Wdb$<J3d^&O9h=^8x|p8jh<>9Y5(
z(zBC{0?X6mWhYrd2S5jCLmz6Dvuhz-nl(a))*|9(MS<`%4Uqm{ib8G9z0!WoA0zeB
zJ*_$1Bzm~;ywd(Mk>dl!H~e3e1`V6IwhWd?P=C^*s4y6%m6($H=az16Hd^&mV35)>
zzaQ`JZ~waS+ed;Uj6pa};Z0nRJjH=35tB=m-lD`zcYjTXw33NsjGLYz!XTJ&0B^Z=
zxJQ!e<e9&!kysrIGou|wO##e8wE9JcF_E>xmurr#0i1UH2;3w2x^>!N(JG#ykk;_-
zTTkj>=67FLO$_j@s(a1LeO!W`7NW+CCXvQO&g~>0#ONIESyz0_7zo;<;2%y+mf{6m
zYGva>iRAd$bCW<9AK+XI5H+2T56FDq=3(t3CGHd$IaWMw%?xdimGa_lGGvhoPl+oL
z`eaLaQf$>VRr{+gWu1FeEIIy2SRn0*(bek8quiE`pYjoY!LW04HNtZ#WoxYtly6`Y
zi>`k5Pknk%)cT%(=T}KUvfR0o3~Vd0Vbo{cfTh^s4gJY$=&IGEm$qCzsB7-$n`#$M
z65-uBR{q1}==`z;!+(kvisIC+{xcp`p<ZqxKK~(m$QF?L5C#k#w&Z9TaQWVvX2(K~
z546ciui@n0GrwXWLJT=*)Dd>CSjMQk>&tPYu<VqLT@i_5MPK#`by!Oet)-|X-xwyJ
zYbBA~_?4rcteQ|)GB0M`d8do91Yopb^HG!4t8>GB7RJc}GQ;%=26^dPIWL2WDfGW=
z9Lsp~!zb*{V58JENH>?Xm)X&Lqo&NJZfp^YmVU%P(U^QWq)^k#)=Ey62;)u7?(&dM
zD1%*|$)WKQU$dTbr+tH>&8klt<yS5&HNgXNW`<V9v$q_ARM-gz*Qo9N?J1YQHc1C?
zFk9oh%ks?bxiPWpf)qp;-SzPtf19JEc9eA{_#Yz=?)!Juge;Hh#ogR;J|~FvS!QH8
z|4n~wA7eQqGNwC<Z{n`awiJq*R5)Q1E(B*+gq#?41wtIF0E^T+%o1*Ic+<U&DnqnP
zi&mlch&I5T(-6!~i&*|c5n>unwepp*oZ9Crm--Al-<QC3r2*Vp=|GjJjqpS_X7qtr
zJWLS#M3k-&=v5jo|8&f~{k54%@+Ef_I)F1Z!96ny7|?uxC-tU2ANQI_aE8wDi%m_R
z>Jt<!r1d-b^!zanO3iw%>94PUe2`SoCXr@UZ3US3On(N$=A=uk#*7w=WsFu!Z2)%?
zwTu9@SJ`}y8Wcu!Q=Wy6g7Q%g)%Zt%<YaX<aqqO~VubtCqM0N@L5YcgQzLDtIG7nw
zryr#U8(?rrPj@#?_f~La=8iO_n$Ys-w&)#f8ic!xvS|?-<j}U|*C@XhF{z16ax5qT
zuC<(mwbQ^`B--aEV7~SsN(E~Q0!Ymu=9rd{s5C@EpK@JLuX)kMux`<|rNFF2^xKd;
z6IRM;;vKY=x41L>EV<zekYDfDs#g4BN13FN{){$R!J&&@z}U|Lh$P$E)Yld!ySuf4
z=b}B6+S_UWxp*&+2Bl1wCl8_n>j0*gYSUMyvAQsR!a;-B_Kh>JtV=sH+m;{E7_5nU
zp?CS^#p$k^dI`@F=h-=eCs#(FP#LYX9vVQ!))JhHDZ2Jw*Bjocz#9zLq*TqNmiA+L
zRsRQ7h~_klU1RQj;7L}H<9Nhu5s;Hh`z|*Lcaw||#}YP6bvLEtNVNoXn8q^FBfN^G
z?zvWiZQXC}5;MIS4iI$;)k|FiN`XA7H<vY)(JHL2R=2_M0d<1O7lLjAA2lcD9Dwc5
zIknYBFk92$c4EL{ir5^NfR&2Z01!S9es^k1Ir3Ua*G|#=qlVxBkiHe>dpE^NKMY$G
z1|pq1^MZVa<&+9hB9S(?Gfj6f-}>bSuz7;-O^zZqlq!RSR&}c>*mq86lytwOCUW2Q
znjk+spb~qo(Uf?S%%u&CE?WH6=G7z(){kIIatls4mh~2csnVuhblb*|noFIT07_kM
zMcGyy$Jl*qpk-Me`E+C4C$mNf+Y7yr_Bn7|I*&m;^t?@uzfv;w0Lb-P%T~HxTmM*D
zJYN5`4=4?ZBXb&#RY_d~D=u#F+hShyg<i6h^NF=GnIru@WMEn9g{;cW>Fr60cFzNO
z%YDcE;7H8so(sFHsG?qMi}BwKl;K3M^p`wI|5M9unl^!1gi0<aJ5;i2yxrsv&9FJc
z$ufv{0Z4zVSTcK6cf;L&%a-YVcjpfwjN^7@tD<+;f0YXQ+EWtq$0O<iHfgt~(;-vC
z46z?#QcjACA!0V-)1{0Ex-rJRCy3~(!{P?8nL0;kAz<Na8%nDPxu@m^c)<Z$@04RC
zN>&s?xJd-(oF=9L<Gb4C1{@E3lX2tF%4ERoMOEZFOJ=JLWgYe23_x0Ti=T??U|n^p
zAKA5^-k$w#;+Y|rQaqw7X5Z7M1sTuWw!Rrr{`5d&T&(TSin#gtXcre*Lpd;O9S)HE
z=<jES=IYqXuavv3D_AqJ?6$u^<t*Hy!Oj;A0}zsD<k0lGQj%8%pu9Vq6GHVE><%%D
zInH%TlPwn;Jn|zFAx>gd3VRyVu<pZ^G1YlqIjHV3N92r52ITu^wa!bne`F$<3oo60
zD|05o@A{*F|5D9??F>D>3?*sO|6zY@+Z;#JFb(JLF_M12`LtcX=RnxHCli4LI5|w!
z%1#`M56b)*zl<xBWjT$t^qy{rTJ$~0d#H)0UMd!d1ui)NkJ;qH3#!gmN!9ZRI$MXm
z%KA3=0+!7F2p!s`HJOFfVQ5&k+LrwBtco~$CeYB9thnHxZLyqh8}JwEnreCn;9Y}-
zObZ|=5YmqT^4-&x5x_;SjW{NmS6gaUrN9dv&yS&928;8V1Ms=(9K33eO?c3cy$*hY
zMlh%P*)QK;n`!D(caC@U(v7+RulY)wI8aH(jU&OP)KvBppGMk```UBJ!z~G~<BfYe
zl(F1ZSLifZ>Nw&?ML&0s{Goh{dUyK>K|k3v6TycS(n`&7+2~`c*@j>1UhNSf?f_(S
z+<erkY0S!E1>=t>h99{Of87v|!<pCr2Oa0aP(BcPJz0qA)8}XZgRu|V(2dNVnbwly
z%JQTBZb`*v;d&`FnJG9%(H45(O3t-?)rKxH3U4x@H_KAnAs`L9WWmZzz^R@>qizCZ
zoh9xUf;gzuO`t%`G73e6CwT39RB5pkSpYSkuzKPit4FVAOB%X?8>`m>tddcs@Wle4
zw$&*}eJg}mGFcf7N@QW+YLB081$xvEVv(crySwUA)Kg1cVI6L_voS~lLGA8UrvZK-
zFV@#|slR{uz-Ze`MMizGs~fQ^1rvH2ZAm~fhH+qRUmS05c6zL4Pg&PF!TN<cD8T+K
zNyCeg_l#hL2B~bg)B--Ys=cwof(WnpeE(Ofm35M5nFH{ip<ATa=ZlwJNWF!Iy!Us_
z+_r&G&~il5reYcrkb36MBQRk;f|)-z`wxd{Q&XW`ZhG)aF5&O{Up13U5e=+1AK&io
zdy~P3^!eH64A7vTCOo65$xZ1-$xP?;p=HZGJ%s8!KiL6jN@V=mU{jd)vwUG3MyaG-
zd6gszhi0yEw-xx{vTBF1iw^bR3{Rha-S=wW)7PC~1iM*$P+}eel8wx+s&Ik60IvG$
z*TJGANq?Bp&K~3TqfkE#@2(OvJj5H8?sHV-8=F42zDo}=GII|7*z+rG$9J0)7il_N
zm?m`CPg4Il-=F&NsSss2&tUu$)U=0t+4;N0kR3WBoYnajNLW4_4ZF4svv$n`Bhl>Z
z>I^TB=o>19juxpvR-^&0_xL+T;DXB~+(&oKqN}LY>G74xk8oJV#k$Vq>MCbXJ&+L*
zyVHjXtTtg@Gp9hzKu`QRt!mxuWU5o8tD8#wpNCyeoNP*#NZ&E;-|q_CM(zKO<~3+J
zWXy3T<Fvl#LpPE`!zgPI+JQ&vp*&ZM9J57yD42v#nsQ?wRD!2)v%93&Y7Bu0H(t4M
z%MJo7B@0cdnvYvcbl=)uV&B~e-g}B-f6avERR?vosB#-11F<Q2)qm6m_-%-~c-hJE
z58wUu3PLWerS7bQGjN>@8ZKG3!~6-lc<cELdq3RA%(Hw+7?vYU9zv-kDn;EJc)>1k
zTRIH<42Rnv!5ey}vQaS_|5Rz_wBiy0262zqRu}VufN(<#*s`bQsd%xp5HX8%f6lN#
z2%)c`R$0)~5{vWUsD8W0zs_hMUV0(*UmiZeEe~FkB~SAI^`zD~b9Japd6-J+e|Tb2
z68YGk5*V1y2fXQj3CA_%@Z0s}pQzXe!&n3F;Ak<YjAp$SpFwCm10BE2N={?XwVHMm
zLC==_fC)=|@Iv17RVWf+7%kr%nR3ULO^cR;o+c{!o~b`sBgKj^nO=*La<i)hrUS;t
zyC4uHlAa@be#%BOVSUK)3mV+(T3BbZM^UloR<U>PC$K6r%J&oCS-ePmVkl(zS?U_-
zW|G=HL>M<0jc+)YEcI}#vUAmwEKM-8SNl6%J~DjA9f~@%yD4hy?d927JKYY(b==3<
z&oOQ*{Qj~P6)cwy7XO#xL^Yj)IBj_9*V(SULNwge-H${s-<1RR=H~Y_t$<T%D$E4?
z`Scb{IUuNr9eb%9Goj-vac5;|>^U{_Wp~hO&bRx(d!4P8raqyQH&@-H)6Mur3Y`WG
z4p){0p%qL}RJ2DA+3*<38iY|>%xPjyiJdd!XTj{o6E+3&!Ng+M#g&mBluD_)Yvayj
zO&YWKZn`5=E&bCA4UfZ%78fvxa+b`gL8ghpPsg0yaKCQI^GDWdI}5cP{Y72J_cDH<
zc_+{4Ha*)k2~)?ZqMC}<UL$99+x7D-Y!v%S+I`u*A%&_N()+ed3e5m4C#u3jv0ey?
zu%&1!7Gil<{dx=9&4F3v#=iDWdkq%jd^bMAcO}OL2=Lwlg=<Gq>~sK=h$>If)Y*Kc
z=^vjzkL69i6x?j_QHX7KHcoVcGvkw~TbUa`jdZh2*M4pL#3r*$PK*4Eb<Ck^68vnp
z<(aT?A6hQs=5Hb*ad6dV-~7fRwhaD~Tyw_ND3;;Q!hFBOl3j`qWucuSm%jUQf==2r
zL`w^Y(`53jx_QJL#~zv9P@<&F=_}xUr*dR1nN>9GGKC=wM>R$5c)m3>cdIXUbrNDe
z;0!G0Km7!1tbJYQN7xW@kcgTK{cvmpQY@7y17qA*zE*EQ3^W1wf-FTf;-ObZws|Xm
zegzQOCV<&y*~ZBCU}XkcjuvV$Wa{K+L<G==i@W9_1QY4?&t>;6L+_)GT%j9G1TupC
z1ewv1MX{+OQfSMf>2vwzkxWORaa?SKBMXqZKLsHaljkF6Dw((0i_el`)Rap-XC)iP
zqY%V#1lSawe58=<NaiV<(LZ)E3^mY*&qpqj4OxPucL#yQA_Nw!zycv$ZOy&)G-Bc0
z+*-<{4*Sg_a0rz62bv;LC@>H0M<ao3dIo}y1;e+o(6gZ&A?wu`g|-7oyHiDKk|Qc9
zMRMJNiyI5`EE{5c@z;4wX2-%S(c+y6D>;cKNiw)pr3X5i5(+?+Pufv$<q^>QNEy+^
zf;qCCQ%AZV&z*xyF7>PKPP&o|DY;~%@miI0$UAtE>bMPhs^v>0D%-%oTI<2!IZ@&y
zK8yG`Ck;wwn@>~{21euWfc;Wyn8moW$d*zik%YOB8obvYfhDIIuMAVQF^6oG!F%9#
zS3;V^D`j<tWiatv8^xI^kFcm(Bgl7x=_OxS!jPXj{gz?u%ih<fO}0_{awlN5Wc*w0
z1VF{0Cs(1}UIjFq86cPPFK!xI8;rT@IP1y6dd@Epav6;$3Lw7n6Ud0|8bthM7$(f_
zLTn+}pRsRaP36Uuvc&Qy=Cc}11fcSGD^BWh=)TD^hr4i47aK#454XddNODz#P_4<Q
z!)6jA_SWDl7OLL>w+I6L5ONs!-c$}ZEf?^uXY=y%$K9<g(zT&{bTnw-)>>6_?Rso%
z%<@WW%)ui~F^^|FrLrgR+6}P@`nc3heK3+mE3oxr+{%~tKyIg0Rtj>cxUw_UF{Jq2
z`{7yKzwifc<}BybOxic>us0IC1~)ugkGl_s^kna=tp*cH&~TCA@+iqJus!6<g(a{F
zk50uZe<#5=ee}*@RAYL=Gm~)HbdH%<W}+qY++=~k9hNv)kmrfC!o9$BL|uK^U<h1%
zk=7Z=Kvb4uc6ZvR$zfS!Wt0u-s7WIOlfD&Rmh2KtC5Hz}LOlQ5E0JFCUwmBKJ^?o%
zJB9dMV1o=2u#NfXp%Yjw2eBCo4Ht*AV6Ga0->t+yw*m}t=aB$){+ST4NG?!1KfhPv
z)VL(6QFoaTT2c3;K7pmjx~Q{<f!D`76-3e<t9VxVP#raKBqiKLQD+-PK9P((`j&i|
zot^%6(^F2C?o44~g69{yt9}^CtlT+pX8o#;e|Z1}Pvv-r+Fgl?@#O)dyIIu=Z5=x`
zVD>1eP!UbrZ?IW5h)4VhsqwmahGr*&hDZDcia5&u+>DmZ{3WGo)VK$)tj~H*p$pBu
zOJBUlx();@K4>ZIAGKUoDjz1nZ+kpppZQ=_39<B&rk#pZUYsNQ*0~sU5~4U9Upzq)
z>iClI@)>N!t^?t$E$A?@uXUqfU~mG8!M5_t<HNjgl{hv`vtX4NO=;H4Ssx>+v-2g~
zrcY}ng_QV62s>MgfrQl3LeO}TLAP)Z6;;$b25qtH;B{fjy0siA>((24rI-&#W|`C;
za$Q`8ZXBXomzsg}<txOTI(|VAvcWRvR>LtNUhaA@otk^4m9GK0>7{E;;K5yO#Jnf*
z7UoHkeh>V7@el-45@;8`8g;?bGnycloP+H`khSk)kpVg&8nf!KSCJ7EhM0%`C3jI`
zg7j5r0L}GSVkODS59<vM$s<$MC=z7jdfpQq4gy!aE145&7J{HhW4R)cEHtAyf^g%~
zCZu?k0{I!zEOX+@Lvm(LgwF{Q(`RFE9>?P86C9j4otWw;dio7_zTM@uWsdy8h|sn7
zr+IJL@#18o9)pNe)&)PREdip0@>|m>PrN?^l4x_Xea~*mx9oL%)%u=D0%u~7z9jq*
z?SCppu$JCS&BV1=v$wWP)@RUsV$T(^GP=$gLW9^C0}Uk8makyenr#%qM-b3z1%cXe
zm#zoE9`0>wz(i1_&fQl>ss4qIVs#`eYWBs*#VFCHbOXPt7Zvl9_gi~A(qB*+)LGE8
zKPWAz=?(>o<~E2M=6BdCRGn7R88qEjdgl(^3+u|rT|EQz9~~<r%>xr8{IP}YdaP#6
z8Lxk&zqSP@xZ7<U$VGEYR~e*j^u-m0XGsddkvf%94Xi{5bFRW=e<DYrII|OB+RX9q
z+CZ3wU-Q+l$65I8_RP#U=9xPb)ePygLrnvj@776+ALa?x94I;MqHh(0od(KqtpOQw
zeMUv-DRCw@$|i5BNhp9K&Y^e_w8N^~I3dL$rZ!8McuB!%b7sDum&(mM#kD|w*JJj=
z8%d+&R)a@BlxoFv{OqXd2ISmw{9ZiF^gmX0oGfk}dU~Su@7#HOJN%OmglWH&<S~od
z%e_foaD|DjwvBctrj>j#@Xss_K{6;g-V)SwdCoJWjJs>2MFuI_BL%8b7x5Oar{lsM
z{r-fvB`O$ehUhn~C~%y0$@;&AYJ80%V8QVIvF}gPL64p_`1+l0?FHYK4K61{HU&i)
z)h|WcF=f|Infu*;LH23()6J7)U6Nm|DQ~vtrPsEoQyb^ILaS44%*v@km-e#vUFRT#
z^vZ|k_<nUsUS@(S%c<{O#jeF4vm1rG$L0<R@-#hd?d{eEp3cmYmtK){H-;wzrOCYu
zGV>cij`nm(YQ#XdJ?{~_GC%t7U|k$)(FBt!4Kiz=U3J=%V*ycyna}A%#rFBzA8UPO
z@K7C;LaD3pEF*+yt#pA*D;xNZzoUz9MgSpHjdroi>&$9!cBot^1SbOa4Sd3@jJ!wH
zbWSj(tP`uWw@o72s}x8GX`LBIA?HheYM&%TDaY5JGlcp5a)f{+raZQ6SxSj^Menef
zqq(7j7;~jmtVwP3C5w!=1J9D?>*h0nUwa;Gj&n`TergVsC)sbw_%}yByEeR9_f5Rn
zqg!!4M|{lgZcnrG7asF{&#X>;<s#a5&DZ@l#cvnT9hSO(5F^opA>7=$TS<tutMxZ)
z_Y<o0(i6TO6<pyb`rLk`Y_2kT#GJ^TOZAQ1)UWe-2U-i+-*A7l^?#Kc7S-3OCAq%T
zcVl06=N#BVyb-4@Y#Wpefv@z2->)dcj!`h}c)_*K&Wv7wfJJQuk`ylUxKb6+2~F_C
z-a;4AeqiEUXetKItbQO#UHEzHgD`P9t)_18Ktxo<XS&|PTmE6x1W2c9`&plZygX%(
zIuy-5jn!Fn>-3cc>OfxS`M0E#(2HcatGJN&Zf&$U(nK~KT1<yEP$4v1HzEm<-u5Lf
z&*)BG^9XxN{CuX*2+|g^esM0>1PvN_fqru@w{P&Red`Gjaw*{)FvRC8T&M(EG<NVS
zDGf?bl)XpjAO{=npLPM)qaPS*XJ0$*2OGaCU{tBT<Qa3_o#EI|kj2@o=E;<6UP?*L
znEB~UU9k#Dy<l)vU2?|f!+@BrDq39Uv}Fg!KrY^JQ)+0W`^s=hLCPKj8OzM5%-za2
z;EJ|wyma2#cHR3~)aXx$p9VvJn@V8L=3H?%g}!EL;?-B6k)&FNu1MBJ0fbARk9_~7
zpKb3lHoz+TB;3@}r=oN{4hZ%ZTsf0EB2m|Bn~pa)GqN9Kr*{!#Cw|oWy8Fbt-Up1I
z=+X*rS;+_*FuBB=w`O;le_Ql!)#^SJk@3*x4VGF-SPDXYBdLQ&kyeY)78^<eQoBAz
zs$Cmx(7DEhhcvFfp@fG8%U&KRpvT{_!>u%>Vb|BDT-Pm>9((uk++-y6sfjh6;eH4V
zM<eF5uYI&=o{ZNz+4sd<K*b3HL^9Y7G8hXT!Equ@(tb|^(mNBQT`Dw%SgBXSGh^1I
zq$FqOb0m4srbr(CxyQ?}b9dfX7KL{>>nxrNDc^GVNw-}~SIQRK{Vdgl2=BN3whR9i
z2^4UdOZ={ncHS)DdX^~K?)%c0hLh>c#-Mh%mF^;8sF8{#1epwLB1KxCy<#|l)P9iU
ztFt-^nhO^8vUijvFTs85)5>?IB1yk;woR&ZOxUuPmCMd)U>X{;%0bK`(l8{$H1=@+
zmDmTP={hF0L+WG$4vIkdnZ@ld%QdRoh<BEm)bp$(GeK@<X30v0W0v(wlWWrTB8Kx1
z_vAE`oPoTQ>Kp5FSBdzz)?xh`kwK<Q=RL2)LBMDsvU-@}RmyS&5C$GKPP{9`8|~$x
z@e49^+3{E?#ArQzbg_&59W#MFG>Mp_bU=ICx7B55LDmthyagLbEcl3$voTa9Q{}yQ
zBpM`q1X3CHZph^qdpSs8q1-H--^dWGL~Lt#@b_A`wJ{3P%{_3YQia=|m+t83p)AD%
zS}V9pE!0&KP~RC`U*2r{G(3=m(4~>V`6^&R-`EcoXkoWGwrxB#YB%yPMY75cd0xh0
zZ$~v#@$iW77=0!lWe^Uuk`&k#OgxDRR!LT6yTsZ9ck0oOi%tWR;9ZS%ycNh&)Dt5f
zLfM<lA?&PBId@q2W%)(s)v4Z(<C^_n_h219){lh6vNHzpTy4`tXm3M7@zjTg2H%@_
zlHr&0<4tT~N9xCO+y?!T!o*1X`GL`>hZEN%!F00HQ2PaLY?jmGZYb7xUg61q+N#Su
zhC6On`KaM%`5uHGRGF*p>PA%HBQFH&I=1*obn??)$+fws#UD#1`ZL*Cg^$oO5!iFP
z)ISg<8RA)-_7r_LRO!koe+DuVgwBxYxMN3A_;_Od?sqMdufU5uYL_H$M1&_kECY!s
z6Pn{U9LpY4TT`C&u%>+U;t9S=<VbDwo?U#GcRZ15C^Zf;-dkq9z25H$<_PfgPW~kq
z^e7!8C2)eo^*zsNv87U??O+!AeIvRh2Ro26AYu?WVr~$SsZukbht36)DQ%3<02OoM
z0SIpvPs!2^q|JT4<5&yT?NxBTt{9(pN(Jx4X3_?%<ib1V&?g>DN;S0zEbI9K2Z-w!
z(0}}qBnn44MaN|Q*Yc&GQg?Zw|LaY~aQ@F02kH65LUhW?TlvenNahdPQ&@E&5!L*Y
z0Yo9K-z2`=iMZO8zP`4q1B;$#<T$$6`B8qXv#^=R3WB3&7aJT}-q^fo=qPa++|Q+#
zq;jNk%x}_c6*l9jy}liTr;%Suy8hj@rsl4Qlk8MijGY`Sh0h;tOms?vx{<(yXHbO^
zp06NO0`ZSZVg4&+99>X_V_9;}&+9xqeDRUZqb$GOUYW&D>)a<wRxK-q{ae<=p{hy=
zn=opqG)i1t3)Xi>0YfrWFx|X5S%kC=&@X)NMa14%BYftHXo=hj;V4K+IEC2k%3kFU
zkXFjOa|h&Xf58`|(@gkN6>d9*dzr<@f_kCiliLpeerM>S6(&S%U$LFTkFsx{+8}wU
zk@FFV`KcKm=RJ1IbcEJ&y?HfjQK>ddS$?cRNK3|tC6XW47ewwD)-<{0J#=5FEni?T
z4A&o`1Hc4-W%=NVmVu}M*4C#F#J9~YOk*RM;mL1#iuA7>+%mPwJqd-SO5F@(aEA$S
z4RRj4#AJ`|^5(>8Q$rfircd%al50*keF-fJmXC;bShN&ZjD;A;jeQlePCEK;tmEid
zN9(9}tJ>HW*f~M%8gjuRK>|J{;^&2hqXE;y2kd#uNYH+Lm^<2-XziXfZ@v(v)M2QW
z(}#c<3hYs8gEN++7Y1h_PB3u$PytuT!d0l~b$#e2uJ~S<?2*kyN(05{zV+9;`#6q3
zm0=Jh2QNS<;<NDSatI+GDbqTe?Ydt!V;?tT)KD^cna%rw1UJ8r9?<5h#og#dVd*iK
zf(OwvvFvU)b%A%i<Lz^HSz)PghKF?Q;f_oWV=GP#!q`VP@%#G>*^oM&`Ee!A2G-=M
z))-O)jwDeJ3#qGG13P8vj)=y(YMZ-_8;suvN>0`II34j$HJO|Z0{H#0_tO+k^Of5n
zMdcLEDLIPs(`;5oIABR&IDu+!jwEfFii+mpygd#^;RkU~X=MX<_;+D{AU^_n3{SX*
z!fg&3AbW6+`S4PH#6cKQwaD1c^yBs`ZX0HcHr_9Hzjj@XPFm{fot6WQ!A5$xrx=F9
z$!KLeFSZ?Xs-T2gR-+J446fjE7lwu%kmScp(EBL1FHy{yO)yCy+?Q&H^=C95L~rY#
zaCsE1#A;>JK9hI7kGINw@6Y>7UNHUrup(9f(saEVP}&**<|u-)l)pB7VCRCS5r&AC
zMo&l(39d*K0MroUlJ?+xmbj8*lJc0BP}^r`WeM>lR&(%9_&>`kxs`im-*1@1?{~@|
zMWw`SBRikq5kH!ersw?Q=$CWhGf$AdYe>w$ANo_Bq_)gi_Sx$8_f@@i#jGX2BLB)D
zn<D9Y%&cqf-->`SY!I%%T-82CxD&0j4M!kB{3->w{Xt|*2cM4Q<#;`x_4b125yLq<
zVSei0k!<A%Y@(mSX#xl7{-f@A>qoG7{0JI;TRbRNf~B$ALqh+7*6ZI3$8~0pm=NyL
z-o}XiF2t=5*>%qIR-yeqCj9FJ7eaYX&~mB;LaE^c^&^L&OxY0MZNX$_M8FB>;^TG#
zS<OI4<dhb&8I<DA*B@)X`6E&+Eq8LWSi1k%CMW?2*MCB_WAKOjdn>_S?FUBnCy)l^
z6S9)lhg#$U>Yg0QPR+pS9sJDB`T~Byj|Eo!+zFwl|2ZAoi@R9gGqJ|+CXOe?Kv75q
zG>`3LtN{-(LCTAeA6#mE2@<G0_9B6Oc<)==xD${4W2w<A$T3~}?nm`+qu3nKvOK)!
zSA1>pk-$^}=t9dn3`JmSF)))?NaxT?%i&L^!LPXhp6mW+6L3ecLv@ngS%+i-OfFlg
z$=x;{=H0)`vj14VaHNj^qOQXL4!ZQmxVC;YjDa0%H3{`RG1`>Mj<gK~4rMLV?`g-0
z4!dOqajIW{a;Fnq8fi%QUH;tTLnjb~!Ih9;Pl0Q4B1fhm^QM5*3Kf*w6Jek-SOM(5
z=Whs%_R*azb$?6{{JC-zG6X><bWZENE}emLj;~O(LZ=Q@FCvJ?!h08(nX?_owe|H_
zyrKL~9!$p06sSAG?%8|b%`Fys8IRiaecI+P|Md*l#{fWS_}zo5e;h>I!75jPEK6A7
zFxZB@tuHPfJJ*kAsLGS75_cBqjz$k%FKM_mxF?nl@aS?t;a6%A)K%GxsvfyRtZVy~
zZKm<E@-kr&R9N3?Ou`O5GchgaBEmpvoww+)A$t<FGSs_0e+C6BdN7~k;3PRzLhB7x
z=au2aPW$Ir66#}ej}M7=i~#pG7E-`(Z?S}*Ep;1c2UlcPFlKqd<gN<6^B>#w=a2t?
zq!;7+l5xZ1!}}L^GlHinS{QZDSyoUcK}zQrlm<VAWQ~#YMWIgnr%0W;mgRY`MmFo_
zt4`h%p8Dz9g04<W2%e<NZMM3R+4_**7&H<Mgwz*d*KpE{fG5zF;*F7rh!l#R9A7jG
zDsZlVAvP2+4YuY}Xg*P`$iDyhrRVp+ra1>^Kr+7|`r=fZStiVp{w#Xa&*B&|DREz*
z#%;wP8go<J&m!U;_xT#fpR@k+OK1k<J|R1B-0cyfMj*AsW<5APrLq15dY&jEHrCx^
z$+_&YguN~<yN;a3fDu^%a~RU7@b)T2BAwvoSn6we-;08TDbhZ122u+Jofh3CSD|i?
z18!@Z*&nh>B0p&^eZglpZ_iUC7afY_p6|{5`D_3g-0U91rGFb7Y8JT^oXu+d{!QKy
zxxz$J(k-RS^|HJd_cK3{?W7FYlH*c65FiRM%Pjsbbf7Z3AIglekgt$&U+N{&0w4U8
zX`@^K6gs{A8KTqgv73CVNz<SYYSbz~vAHcDMFo-&*fvO^yU#<7_0)APy`ncrmMGK-
z+7o_-DDQaXWs?A;b}@ADD7f03;6*<LM2Jb&-Ls1mokoylQ{tbPhZ6WF3OsK=r6(p{
zu!mxCMe468{6C!InBThmFZ+g}L6jxw2|{TcsPG>mSMO0;YUSsJ{?>yDI8z4r4N@Lf
zp>7f;s3t@-<+rFGQL*Q~fSKl=6ldelAZTu2<Zxh$urmroj(5?R>LL0ZC4`VMsB0rA
zO6MmZA->EF3Xe49&@&+jh{9(2>#RfEMs8z$4YC-IL=SLZEl`6ZmH~~1chtNkOV1;Y
zJ&__HJt#nquTP|W1v!KNchkAA?@e?3(3!eHf4-6EsP{(w17a8ix%bX}t;1&MD<0&P
zgCINERlV0oE$zysftn~T7l#j!`y79|hlMOW$~J?NBZ;8?vs4d+c%DLqarxB;Q01Ti
zR;$ruSFxqum3~MD1VKi+;4?%5p8%rcE10JpXF%=!DMEv?#A2V(lhwxwonPU1{KlIb
zk|SAXll1W8n}#3F6}azdS`v(I{M(`3g)h$r33d6;5%p0NORDiVzxEsn5LGf<%mbX+
zY%m(amWB|zf^flgCnl@n`4GgXRs7;JH{Py^6h$njoU-h=;=aCYqOW)5Y}DS7=1g9I
z8w5jFt~RlFUcI6#a|w5bTC(>;D`Ad;)Pe^95UhZXFr%lNFH?v*O>jfDFWU*N`NFvP
z3*^^W1dOY)Mhk;;=n$x#I7O5zfIH$L{DT<s6>y)AhfcI*)KGg9UJZqK?c32AP*!bg
zw_LBLB26@(30YYRa9w`scZVXX5@da>0kywa8R|%o<~Stk%~5>^xvvDOblwgIE5u?J
zU@O?6RLAkQY@5DwTF}p?Dn?Sj%YD@XV*d(7mN!C1Ef*g$JBFp7XbLs8fc?qKJ8(kV
zP%Qt`7|Qql@DEgM8mLgG?y=1r_Z`|BL*vQL;G0!IbKZdgH)yvA6`YB0AdL$O^aAW<
zf1NbN>*fTP`H>k2*?k4r%~60Xgg|{)|Lx8`f6R>TQI*`Fav+Qn>i{TH<E!^ZZ==L+
z#xz_|Yp%`nt6=%P%V7Gn9s|QS@7p)ZEkieu6VXn=!Pl7neWQnE5pJD)z4PRxldFX$
zA1h~4;x`5XwN2DLe&3Nr%+7d>v4iie2wN=SJhto?(!?1q0U;8H^aow0Emz~^sPf(6
z-s%T;(Wya-N-+K~zu_Zj4$$8GUB{{6rQ&8T5t?jrfLSng;C`9IL3_b{&9P-;(8Vi<
znedz+-E}eX+h2yqWys2i_R*oBi}bwl7ypANwACQrV9uzRI&F953b@naj{yvzlxN{P
z$ql4#Lgf%Gm~<4f?m<U<78#6WMcO-9s7xL>Co^}bK%r>H_h2KXe$=u3BuG9=*LD20
zw;EaciyAV(4!zxXSVFh))e-KbmwM2@=PjhNeu8L{rydB)G_!^Nb(PR~%24bufd+rL
zX;6lp3sgtTC|8&Nf3*E~Jl6gHK8{CbBBPX5lARHfkX00AWrVC`C86xiNy8|!k}WBl
ztn5)_WJ~r+vQB$%-}_Teb*<O$bG!X+@4v2`*Lgn2W85G2bsUF@^ih1Q$oMFdSQ2J=
z#Z<O*d$7^20jK=Ve@eBT{d}}GtFdx>gAI&r!egjn-4!0mqmz_e*i6>H&hikvr7O>a
zWCgK5*E{9)8-Ik*TNO}c$W+#58Jkn|wgXJcX^Mu8XV2da5I~p-Xc~HOcc)Sk_ADbA
zm-e1-h{8r@??9_1FBRrjmMA0^z^(}3f?_4EvgB1-(hDm*EM5~G$Kb>-As=md>pSX=
zC9imC12&Lg&BL?;bnX=P^g|kpqU*=)2e4SEY>ENyI%!!IAeMAdvU2Wc?avAd1=%}Q
zcImHkg$PDB`areRDSp$-{7q2TjVj{(5yvoG083z^BwyefCcIvNdY<#+Hi^X|htkcZ
za72u{kH0tk1eMxj$XY~)r&fM*#Qp>ot*PP9K2QxhVY$$JrK-3QHZ)fiow59iG(7?c
zwVPMdDbRF@m2eoN9#6#fez>3HytwGrioW1@!W;5L^L9kdeF5FhU4=I`HbfuecNE%q
ztf=ZEh1FU$SZ|yM(2k_dPaOW?bZ~T1ZsvTcIZ%{jRrZ`sIWv0-5;!>td{38MQIWx+
zr8o<Gx?<~|9<IY^V>g|LNb#5muciS1Ua-_lSIpThtp@S+Tl5zm^IzlbehpB&M{&t8
zqYtO~D}(@m`S=0MPpZY?CrlR2%8`?+C%l2M!hQ<@zM5&nF~53s04g&d?5^A4y=K85
zCB)Vsyj$in`ua(iTx+!}^rUBkCs@YCl|Mx80Mb#|wYLq^wDMa@s-Uk`Uv+*GCp{iJ
zMKf0(P^Q(d%!(cr6qZH_#mEz8cu-&7hBoI|yfF=8X#e-MhV0HeJ)1bsvWJ8eLr@m{
z(voUzY}fl;<ObrU0%p5|MHLj~1={Xo*S_S^zDtu{rRgM6KL|TF0O`qnAFUp`z3jHl
z(hlwKDeMnwTWTceM~6ti6$^cPh%2)C9j4IES5<hjEA*@0;6rIHc^exJ#Al3yrl1eX
zN^#`@Lik7~2HJe4&VCQOi*xnG(Xn;CnS>AFjqcl#$FyE{=O=DCKkuHeuid&89+I8E
zm!TL#m>jItnJNi>MPyzw5UBi-+Xe69K4d5Yd_8`rM&9ovC{D%+b<g)vu(>GCY*a-u
zXEH5uef>yWBX<sqe?_%imj#-UbR3tbCK;_3LBdAk`Xx7hN;?ngA%(EBGOZozcEtT4
zTv>c-8!R_70_|0<FR2s`6+i8uKss}-FU#2f!sfzPq(!P(d#v`wY2#^LH&8n3dSKCe
zRL5RoZFzdS-pA*3e3XdO+FS^yCNtD^a$#04f$+*b|58x!iM49ZcPa-rj~Oz%b4II!
zfh5>{7wNVku404Ep-eiS*3m7{wF!V+mqW65Lq2#mu_T0guOa+_j;O@4+jRYrc70<n
ze-7cGs_D=<UjYF61!{tN>4kn7N^h^WWawX^60%MhHDBxt@WR@L>efj_;;6XZps&aQ
z>4ofGYYFB8XO=!ZSg7(jhek_k^5szgJ^DEfdUVs%lbaBT-74EhCE{o;UO!<tCcgnq
zS~L7ehq%5_*b4g<#^(;8BG%J`_^KluLzEjdtAGfd489QIY`6mQOoBCK$x+m=9Ty)X
zcv6-AHPHtiCGCqYmLPL1q_MOJ?#{Ip$wRuA$blzCmMKtl8n53JeK#;89xvpltvOPD
zS#qAw6a(Tzv#Gu0m=MC@KzGBNXSZwPHUf?QW7k1!pasdXp;&qh<;Xsqd&ovqAV_Rj
zipZ5*N~gZCtyVy7I?y4Ui7&t?(1`=ljhrBWYkInoT8Jfdj~0&gJWU`gr0y4%Ag%yt
zf@CN_+k$`U_)>D-rLl2R-~(g`l+E*PJQ$!$z2I~K^>HWSpvVv4xaCIT6$y7+bi9`!
zxo~Fa?7^=y*<a;@0YE6M-J!;Rak@P{+T}JXy^OJyU>hWtUI4j6p_J3|)DiwD9msWt
zk34$_K!Dd9AS;Y*r`k?8bHZVW%tFBOtDHa$a`e?+JNp2<dGDx;0}m&J2wD@amy4Q7
z*sa)oLi`6fZ4UKh=+IFj2?7%+XDe&lPW6~fpDi`4r4-=i<pn4C`jdk!qbAsn{@{EG
zX-wq+U?g@?)mUD-VxTvX##&AQ9|dJ=Zz1#Z5|PFW2EH*dEQUw~2O);hQS`d3O=`Sw
z#eub`86laCl^^~~)QS3vie3gQyY(oKq@a)kfT3o#Tnhhqsx$lI&`+0W&W|rF*#w{?
z%F-lkwT1ZJ{2vJ-`bWr~{p4HTv>Q!$Be@VB`PM%&Rs+8DXMaZ>8&FQ0o3E}{Ny?h!
zzaFPmKS1R#Ae?HUet74ovj?YZ2(TA+RWZWj$=~nuesbHIKcU-?G@12)eCX@kyM0CZ
zJK*R1si?Hd9}@EW<lk2F_DbnK`&hhr-X_u%RFMoozY+?qH=qd6)S*knA|G*wCQ6tX
zQMjU6uj18FJLS+@-&bh52|~js3(zi#1eT--uIOj0a=G0Qmr6oF0@jBp>Vb!d`+`SI
zJ;a!0UC)V+lCHwlKBcWGf_0MPIV1(Zg=D<@Y6>T^keA?qkevwXlc3E-h1fftnQu8X
zG8q#JB9|e-P{S0WHTbgX=Am4rr_yl`oRXsHn~+{bC|~zbP4*Rci+<LFjzse)isrA3
ztu>zs$M3zFPPYG^Ryakz89s<32SRw0^1S0oB*M@DQZVXx_Z6?9k4u-hiWIMsXBo;y
z($VkG@!H2?z`H{5f7L@}<{u)-PD^=VyS<S@W8QS<9HfVaicFzqBsT|Vn_2w6cr!@6
z|7;Z7whp@9E{%@@kw`Bg=86N3C-{AqsNaBB*yO#xs}W%5Z{dtI5801T@*JE8q__0(
zp%doQThq_cHH3Z2xDn39NOR)HLh91wX6KaqP!6jFL9?kF*Y~6S=!6TP_0$qNl%UJu
zC1jgAiWi|v%9(`HTtd<SHdRZho11`|!iOdbF3Z!BQFmTXKjP9ZFeqHNhMsuuq$u#A
zem1NY%{K%?=Ul!CODxIRap@lqB^BN^QE;|OR1kJ408T8DvE7<Qv3Y}TV#+SfGCtcj
zYWAH_yBsrr(0Jn&nMmg#;U{K6SYJgjIe8$W7<rSe$&0s}it|ER)ELQH{&3+xB5uYj
zB4DA|&CMb7ph3Ayi47mntbv}(lUh^zXWPUBOI;TX1Eq<v6DR0n9u7A~b0gqYBEa!(
zulzV9Ku6^bP2@fKS+Y?V1T${}kMTq&ME?{5rC^F;A5R3nwx1o0Wx+knd9#(X3&I++
zD|iB@7?Hj)rzdXl_Rw);*(8B4DAUgm;5w*Ym&~rRJCVmn+J;$_INn8NC6~nCyx80W
z9iS|a0I86-3`I8R0X~t5&eObhuZlGN>tmq@P-Fe7e9HJbU9ew0A=1hl;XGF(&s*DW
zZB459#1}bPB9o)Y?}KX&Eg1=IYN<Dw@Q>Tagu8X<J?Wg5gC`<)aC~^<D+LaMK5wRw
zATW`SnImH`etIyf;!7uk1<C)}VxAm3C7G0SAZpu5&4*)?J@09cGgH8UUyvmv&~I%L
zowEVxhzz9vv@Sz?o|jtx*!iGx4dX~A3Kn^5pI%d-9rzg^?P~eeF$@)TH$a*dQku_C
zp{UC&y0Y1l04rApuuTTg_mo#Du6|lQ%L$YafEYch!TH+ST#$f6x1^_vwQcMmAu$Kh
zivvg44bbt>RS_#RKb{O}#@KlB^Cv0jaJsThPC-0oMOqCBAT#0AQST%2Ue<Ann*wIk
zg9nGtt}R37l3aIS3}9GF$ZgIyEl2RSuXAgDs8f??kRxclbINku%@e=SxuANe)}z)^
zc*l%8l{j>sxHlhkZW&{x>FJPSF-da8v0V?C2fB+%DGoittW=0mxu7G<@;d47^w2qE
z@v=OG_PZ8OhMc8Ia#*~xq=Efnb32PS@bN5QI-c}E7%fAm1$d1~?JvS0g?l$9r7$r0
zq#eV_sE=+#gxHSh)kQJuqkc<JK(_Y%M2oY-a^t#|s5uR-;2{vx=R?>nt-lK5hB(w$
zu0S;7hlZr5a?#XVq+R{~_lvnY?2wLvN^ow;_)Lc5;$6F^`E*F<JA!AktmCPw1}SJx
zXpB)9r3@XVkC{Hi-=JnBe}1-t;wP9D+n_mUi}(U3`sBOCY0s@l>9YrE&dDM=+poR>
zx`iF$+Z=^bM1ar$=A8$v{E7i*Jc6}ltZ(2W$2m6^F<RRHGgo-Y&WpB9(0)tz;L*@^
z#~r&XZuf2)cBJrQAeOp}#2EsS-DJHYt`%Nnme6lxl#EUhB48Z15735Qg7*+SUi)Mf
z3fK~af+RIM0h)N@&KP(-$o8ga`6xl(%<s+jEnyI0ZBbRzHl#^M%V22`Bg6DG%6U#E
ze<IUGxESf$vi$jCxHZJhQz#-KEdrt6Kw9sEWk-|bag+M1gK<2l(QJ6*&u1VrQuC5i
z(<3P$gdK<4`lz7!otZns#hkqS`3*)>&!8E!n(hwu<y+y8eJcYn5tlSOvYLEC8g@T$
zkbrohD5g$iFH0#dfsj_?>KzH)k1@IVGGKCh;>M;=f9pK%@=W@x=oXJ~BGoBL*JD59
z^4^i6oTl=xJSOQi9zYJ;FK}E$JtrHN!2&q9C$xPCWp6{Nz6LqF%5N1TJO)b8Qk(0V
zBuD_JRYS;*<iXPiSBkkkJl#9c`OswNqG-ux3kmj`)Z9MmtXwE|bgn6xjZLjW-Psy#
zoa8AE)E+J@cu-qbzuTT>W`$H006JD>cotf(*DDSQoFXD~*+$FYK*!sBq&jKaXMl*G
zLM(5JZmDR>fz?&HnALrOZ<Yi@Rvz;?MN|v;uVn_klwKTp6DN1!kkDhZ<Kn}Vm`xdX
z1jel8ZvHWjUjssq(78#dRR2*?^kRlFYnH9X>{|LP#9u2~gqE5s3r9^BqX$+S9aM*s
z@qxmpUJZLc-oXImK&JDg-Y)>43a9}fj;KnyPH*ZO?8fz{J~`sgG9WvkX#!4;roySa
zx5O-mq~Uzl9fA@<0JLqbd~pL*Xtac-t1v;fX}kP8`N(m<2hctl8_&Q<jqsr%$AG4G
zJ}1~fIpIJghq2JQk0oxN5PN*#yUm>9n+@P3Ij>y%!W8L32OAh0PjFtsE_NG=9;P&<
zNRXUqR8QSmXqT?c5oj8k2yXm2pcx5KNl^S~_!YQ%R$;2|jhwV$u>1sVwe(5#5jY|e
z&{jujIgwFy6}k?G@_zUc*&I6tIDE)|3b5q9G2<o7rs8O^?<YK4qABO5G*s1Gk`l+%
z;KEXb4k5Q2I55rVc};1vlg~5c@g9X?fC7^3VVUXKtxyLzLE^4j<Rx_4V6qDb&LOCb
z4s`7VW+hh(!6&ssS(2J@Ygr?D+LB%U$1j9`j`nfBbC8gb?FxO1^i8`PVh9olfcu30
zstUO0fg-184VWoBFRaA$pNs?VDYkILsT}c*sdzp`aFLxF_511Q<4!TxEdj_yybpYy
zd{H3(y>f6Xaw8_0`c`sa0Y&S59R=l%Vy9e)hKPbZ%OHO?H6@)+p;GSxvb9rMR;T75
z7gWvP{z_SPi}<u!T6W}h*vP{MQrg1Sqx5jXJ(%bFCLxaXl$!tQ+OmMW&h}$X^L(jR
z9&x)_wGkqgJI*_h0kyR8GfhPfR#fNj#{e@}^~lg`5<337tpNaq9r{u{${0czVX)QE
zK2F**)+#JDTucY9{~D?EqH*KG$(ny-hn6OTELvRk#WTO@;R5Cm7OkHlURZk=03gn#
z{}wHh9ID-_aENC~XFVJk0&$Jl4zx&G<F7-NdhMNcr(sBqvVzFZS|R}YT)&8PjeYhx
zVwH?|FR_(v2)@_?;XLZ$uwie~uF!u3@puj3sa*;iM{+&I0sd(Si!%0dy?laJerAG%
zV^ItXCxS~i=NB<~4gMrxl-Q&Jz38$KXg%Iq6B$K~5`#Raxe!OJ-$|GeYhP7mJ-^e@
z&U4-fI>@{iKrbo?07HQ8@byTs)ws@|8~p(_B-qeH`7*b^&$upCejk{r9O8MpRCGU~
z$mYjP4t$L?U8mhn{6(ihZ<elj@fBW~I6TzxAK<1Rzwojq#G9j0+c;=|CNNwDJ8^3w
zw6*4|AjWWFEC?LyxG&sY%WM9|XOzKz=FU4NJ?@UrM))UI{y^~2n^Gs(g)!-+tK4!B
zUz{=ifu(y>DdB$GaQFeMFYY|Ye(0jX{XcuIblszKwwGNgPWmGHTe@zk3-R_dXar+R
znuOTcd68K@;+QJSoUlVyc_H+gKSf#A*#;6btRKAQ4^XrA)?@5X`iv3KXn|4uVpBoi
zdNF{4N^HZHUON2%Rv|UO8cY+XqRrIe<>_>Vt*|d$oGfxVKZ{@#+Cw_uifOutK-+YQ
zD`Z>@6Q?59A8J4QSdiWeddpN@MgVasDX5eIL1e2^$SilZ4@1JttGm*K#+pvhGA>;#
z>y(?{ns3n+K(M)Nnlg+QQ+9F%W<OZSc*^J3UD}!7U$EGw*_(a&xYI-iN8<=&PICQ(
z##z_D0m|rM7tIx%eiz_|305qW9F~dH+UQjl`-=2j3-bWMjeBDgwpi8s5&3#!m;ZvE
z$Hn7W&R)x0z;VB?A@BzPl`$3!$0Zi;(T75#ID5d;Y8>K(jDbhGk7$TqCO;xkQo#?=
zvh;4GSEIPQaJ<p0J3{o9S;za2?Khr{cs|8D^{gH-&&l_{-;XD=a&+f+TDqs(A0`c;
zPpEvYR~pJ6eh<YIvhx;CK7-RV2`B;B;!N3c5|*UL&fmY)+S4e6PHD<Nodw6HCKN*E
za(oSMi2{R|ZZEXDRZa9q+_^Q`nN`)!988Dv+##;~;UPa6I0v|xm9L;1a=@Jvt^865
z28?I@3`?N;n(N$SDF1xE_oK((^!RpKV)U3^1J)XcyE1)&T(j}<ts6tL{Rgbe`mz5Q
z2+V>+u*=c<CG`Kr*0jr(U?8)<x9Ty?!E>VxN53a7OXU?a!%>g}FEEl*lk)?qg{eY{
zw#)_%LDgFpVoAJSn+wu>aF#tyx!VgcDU{8(*#St|c_*b)B@WOx0j1nEJN0F}-j7?a
z^!*4U6aem%3O;~+a`zZyTJYM`Tybk45ni$l-lg+=f2N_Y%}3m$I1P}=dD#Xj?4QY{
z5Tlh9__Z-87&rCK*hEHHPOIh=-u=d|yB!9d7Faf21VmtgpN=*C_=&Dqoc;n&>!K|m
zE!ykzb{*pCm2qf#tXJAR?_w$U+L(M(b_ef`I=oq=uf-O3^6aPe-z(%VEuYo+#D~|G
zi5RR-?zG(e3Pk_@Pw|U5esl~i@2}kd)-Oi3x>AXh5F^FYb&4yFX<T`x59tbCt6^D$
zB=Pex!fOC4je`+w0fb#bLFI}XQbYru20wPb%O$s;Rv^&Ll1z(|)ys8$8@J_;2f%()
zuWOh^;&e7B@uf!vH?5nB>3vi`y6#iQMo?m{etTnOwgo8ZayyP)tF7>+=(RSGtSzvc
zo~k`39Ot|ArU=d@W~fdc`EizByTE$(+t&(rto;)XSB`9i>(Sy2m8?%$`SObzcsaOz
zCIkuYKtNO#<N#WY9!gkNUjqZ7E4unY72VNLv-oFNrLTIsdo$rt9`Yi!!(;Izx#k8H
z5%ed4%jllL2RIs7=Ac&Q`#>=eDl6BZPBaAVzX8=Yg~MT{L)%-^+p%53kZoNEfR2#!
zGV29aBl1*WN}Qp<bldiAu3!XI4{ysKn)zl${yS8jar{Kc;?;>O_P#En;AG|NN=_PZ
zM??e!$|(?cbqJdI7l&5fa2L&Zs3hO04>6JNAuTQyaE7xwc76fRD(Q(rZC|GPETHZb
zp8k0F>Z|Zpbg?j2NSF}K`7KM0DO>}x!%~u@Dw2{Fv8=4I>fHse2jx{&;&fA0QZ;*%
zbR~LBH?!Mt-)0Yb9Jtfl#qP_>FzAs<;K6|XQuSQt$$kfX7G|s9(7L+%9)scN5h|A2
znwpyFqVkIT5jplFPEY_cEOW<t(Pn&ew!L&qwT|Md*5?xN44c<LV*FV}DH}2MS<+|a
z(0{8{U^_X^p3L)earMEG+pisQ{0gtd$#GBXH=d3DZ0EYMTw6$LicxgJ!Uzc=>Ol#|
zv3U8CPV$vo^G_Wk-Cwcqzm?h_JVO=xFP6TU`8=(zRCwHlrn!WFe<mm2opzh$se6=l
z`|c&fF>12D3=A#R@2*~(#ujr~vu+o?Ztvl~<!6S+uW59vFxgDvDAF*KEq~MPbrk(A
zyd-=x+b&C^_jX-YErm7P^U>R~nos&K^EHW@^^U7T@NEkIsjI4eTJ%|yAXArjAUV$?
zJLhW}t#$PAL8_poh@z-bvIX@u>xWv~ZM=>PcN)uIRro}V_kdXe$}l5`wosF|qWl+^
z8yB@YxGPF?$L#W5<JUzb#M5yZ*0iq_UVKWl7Lmt-=Y@6K{TTh<|AVrx*e4b+bBgM}
z$7OfSG(L+*;=R-o`s~&;;L0+)-=V^d9w7zcw8Ed0)wk2`wuxDUjttra3nXdXUgAoG
zVA$#+hf;xAt6r6noe+C1{|>}6{6J->w%)>tG<PEhVJh?^8)vhtc$7gtBrL54m1}sF
zRFT^X-x-K=rVePuQz9!i)yK!@sm@I|=vv{NXY>?bWYKTaIIryC&Mc_%?eI4o6hs%L
zEo@=&UN}A)rPBxI4X{lio_#Scm;31gZaEO6D95-i3v;0_6}fg!8*EQI6yr6SqP*PN
zn@dQ4c$TnidH?GGVcn+h;wd|HYUBLnga3Q|REW3n-2ddcwI5%BuHn#+<}JMo&0<0X
zy7<??(sK45(=i3-?*{ayaJsCCpL<|gNoA~#cyT#k)2}W<(cq)waHw$#<oU&}k05=7
z?7?gj;uJ&?%1yQ+P<d(SO#%7>9MO0Hf@<hj`PocPYauoAMW`m;1Xvw`)`*4cJdN>@
zHUu7aryneodYD&0M9IpaZ}$2TUJ3fnqmBY=K=`0p8^Wz{?(1UO1Azuc9a`uwGU<*=
zX<Y4ta|zw(ww09wB&KSSHMqs6H=!#2{MbyI0t&&R1cj6HOsAZ-DG4CNhCEhw_lwCo
z<7k(cIhu1?fCV%*h5nGVOItrqY**)(7!<F$La{<4m_^|OQUFZq&p!jsLb7hD>&u(M
z%Z8%2+hKuDEiddV#`F5POl$U3U6zc`+5F!ZD%R~7aUR7VPfk3pqoKe^M7d)`u@JpF
zL=Ec}%Egu0a$<7pbCnirruE}(+YgeLW1xO!+4G<ZX&pqmKW<MAaSb>00c+{g3RB1{
zIpnCKg@I%6^TDf>eD5{0OW->-(6yqq7$!W`UCW$U7Ukb13rE&lDBiVvFzbH!f@&YP
z(fbdtBz+S2`N<~}fw~WxVN_*Xp1dlo^OcDUfHn@s+jWeBL%o%~w%u|_cZ<=!f>(@`
zJT?2^;H#^!%(R?W+23qCYJivKD<dy$J}AP7wV4|S7RK96c_FLx_U8ku(Dp>Z9XvqM
zay{|G7o0E~QbC(MHyeEm^3Zke?a6x*f7Njm-}y6!8d12r>`e0PZ)3CjCl6(UD%V`C
zfaLhu8By<iqwbFI;@k|%U85O$`u@bNKo{4k-jaIek3ZP5%uKfkMf>n?asKO&uYxdG
zK=cWx-@2~56Y*;{*odr4<>SgV`moz<1L0^++4$rnjIq=7j@WWbN5=Rr#$67z>o$6N
z*kRdx+AL*Hs1_s5hCCTPHi*?p#!%eB{@2GKs8ES{U>cYeCGf00`liS`DP>8W;yV5P
z$-CE}55i^<<!Bi^6a7wu;Z);c4gP*(;pBfmFI+*6E~)g;9jyPjlg7Zxy5!9hlWut1
zB{vPWaDLAN>JCBn6r1Hn@s?ThwAH4L@t(flUtZ-C@+x^&cXpJd|2an$;xRIsfufDC
zo)A2D5U?%38hEsUtl18uj_37GyqB55_`>PfX`SF4?V-<BQa2jryQsGcL`W9JDhCg3
z<YwUhJ^@kpaPyYxZakg)rPC%U?+z8?-#W;@zX<>GLUf9Kr38+o1YYT7ZP~KkE_dOR
z8CpDDXIoo(P;>VsV#09hqGjD?-KA^Ix2mG~uNn-JZcpl2P@Va<P2g`c+h0lf$N8Ii
zWT1;rljqB?1wtwagUis)bQUz>{v4B?g-cHfTE!fcy(;f7kM}-qJkF^UU$T8NUX*>V
z`Nvjwr^oJ4eLHo(`YnUD>rc0d`hMRFb~hJAiM>n}wzu?%_GV1{+kyL&8aYv|za&V-
z`4N>#;!#?RTKu2)jv>HVxujtq*|9FOU7%ag;ihS0F=iiCO;&oX-zOK}+j&cAjrZQ}
zsziMvoaon#tKuB<7Z?)_9BYi@rdYa>CQ?q}ht4s!XJ~!8`;>3F%5sbouk0J~M?O%*
zfjK`c!D4w}S1u1UK9IXgyb{EyPF6^}i#>nTu91<$FP0lM6H^`aWW8G*bdszht2bu1
z3_CFI|A_2Pukuj5oo_4ayjb6-8isV@v)!EAypKA_yU}kh4h%UQyf=?yV7EKKtU_36
z{yu5f1WzI#pjs$Edvakdg6}fMc36UFS4>Y&2hr}7_^7#~-^{F?YxuAj<`XfSBK^5x
zOEiq}%&ct`;{bQX?&N+t+QfoSYook2<o>MRyXd}O_1cl|xh!AF2Ay4=kM$%u34i$3
zyPx*N?nPsn%NDP<IzC+bWgY@@K<iH<IAcF%#uQ_*H%L`t=^9NivgPUf+OL$Hb+GU%
zoYd)+a<z3^!+PR#cJX?k|KqUGyCtU{^wC}3m5{BFDcf*Z2-t!}@uH+TtZ6wtS(36S
zyhQ8uhsw>h1V#CLTKsOU)((uAy^jXG;q58y24dvtVkAKek~Pq8_$<aApo(x;PO@73
z7A|L?7+7O1BH_gA-rsOoaBi*ca=$D&m0NiA;}@Z|O4z&aI7gAtEg3gr#=u!<m^Knw
zC}|`bov-!!>f!Rm>#tKXMMyq$`dmMb8D(wc8K(6b4(&Vx4)P2r`QgKR&%h4yE-~zS
zapahJWPo}6gs!`tA+o?5*l^U?wq?lOJJ+`^Iv}3|BZN*bfNHzqOKTJC#Y|`rChDp*
z<oSf}1^dL(?R-)8l$-D0=+V@9EA6g9MImgh6EEa-t={c?Aaz>YL%i9RlVwk~S0!Vg
z7>+6r>$j7;hwB7xp6G#eBHx;&1RANX=;L*Fn-2#rPaV}*m3seMxx5ZB>w<&qfs{`S
zxssWTp7DA+UGjLrX^?okK$JaBGeBRDM$(%X^AuN+r%<x6O8fN`?C=zUwjA+~FqTmz
zA9)Hvwj;aRlj3zdPqE*({@iFVj!J767w>dG!`Ems4WUyP^zZ&!*D)Qq$Z0vF75JeF
zHVTpLhwILk;+a1`+qyizkj`49PxDA8hIsc8NSR=0V_Rvmf7wk`1WZP?_p?Zhl|Ydn
zhPfc0u$HIVoxPR=_g$X9WJ!Pa`HoXrC_`^Lv8ecUEPjbw!}AQf&yI9vigGL2?mh`4
z&fRv3&UJON%|38aV}hASi=7(%rh8-74;N_h=$tsx)iiD2u~zdQvox-%!qQNhbZDU2
zSsGr*(ui#sQrUYPMPzxZ=&<i<oTHAx(;U``Kaz~>6vsYhr3@=uTTKBU4tq6&=5>aC
zSH?`okA5t~25Llaf2}W8Q7ji;@i)biVJ8?`Vw1gG+uM?=r!EM%vhZ$wpzF`+oYd%O
z`RwSTqAoT4Mh`vt=_Y0sYQs=n&7Bss+Oa0FWMED16a-Q1c_J4nVW<L7iI|tU%>o<W
zUi(HJSd%bD)b^3i`I(;0vwF7{+Zjxc`le{IXX-twu2f|RZwq4~b}s#4vj3(LnmKSX
zT<U`v&$OmpQHt4cZx543KWA;W#K>Z=toQPXVY`&?rN1UmCD45}*noI5D=w6L+UQnk
zNqYjD(!2RJ9n-30BeQv(s)GyM{X8AI(FYU@OQaq42OG4j$EB@u^(KZ&dy2?fiYHG{
zVB2Qju5vpnf35~A*C$1w|Lyf?pVhU5g2{~3spA7Hm{r1$tdh1<*}u#PDg}2T<owoP
zFh7m(p3lR?T6Rc+sbegHch|7R9RbV1sh~sS+t;?i(LD<!LmOA){HSKCF)GTBj&x+|
z7>u7{h!)ZJqX`y!LAkkjUGGwv&Dnd}v>ba^{<KIYv9}AQNxfW-T4HP`=y2k!oJ<gJ
zqTNwA2D^Uy)Xt!*vVX7%_p)9+5xA+qZzq4wPQhr`BSGEA<(MCeXsnu$Ch2(3+oP{)
zGbv)els-H8ajUx{NFzoiO@!~7BF3_jcAPBBVF~<S_rIipiV-}x`$!d+cg`}^x3-g=
zaj7zg^JL>}4(TU}b@E~?$+rWq;&JNSx*72<aAaEQe7hlMHhYvc$Fy-}`Izyq`(XAz
zG6Yn}j@(yaw7Ux796JHWC)MlU-&ABpDHSi>>$wTuCd2yH#W;D-izZgIzkcAOc5D7-
zAh*uZP#z_NZySRm3Jn}$$oK0Mo)u#(?(WtGU-KUh8YvqhM4dPl!?U{<+{LdGwVDmO
zInvx%vpD^dGv`bFR~$#53A-R^-?ZV^*oL1ijUw=WjmCE(IK?T8xp_y9%(uGFGH41s
za+CEVwLmGAE<LrIZg$*zDa;ehkkU((?SG_-x!FGK<KP+9{pAyaha52q<aZv7`bnaU
z=p=2;Pbp=|SL46kz1nTY_hwnucDg<skr}6WZZ+dGtPSnMxRAf%1kCw=RRvr^@=1I7
z9dm>E6k}{)F|o!dP~3T6<)^mUD>**Npw!$^1TyTMGI6#pfj5r`>+shH6Wnd<n`KWn
zjFMDTW}G!2Z*nA8U_K*8*4`BDUH{yEThFzMddqibM#Ya1stb>2)D@OU_SZ*l1*cDH
z)^D66bLs!^^GQhl{C3+jVI9p90ne>^(;s)|Z!#g~p<vr5MZ4(7+r=}x>*vq40iz^-
z(H#d18zcDr`_cd3{)J+I@s1%73RM2<!2G}c8ksb>`|W=Y+#kQMzBmvIbRsW~pL5PL
z4aNP(jPKnz=^9MNc+g1R@45fSzeulv6MY*Ig~u$Iy`TTOB`YC(;c9HSpv-^Yh%|v6
ziF2+7IsWmOf4xr9O?Z|h%D1Bb<0`Co;VM1_PJ)8}@g@2;@O&!@*6&={Ggp6`JoxkB
zNkttUE-0$br3l^e5HAqAaN3CJzrXJQiq`=w^o9Q(bpBdp2wh@gFw%Pb&SfnTx}_Uz
zJVMEFY`aWY#;ltp<{-&Z%kuSI{*vK;FB{Yuh@=UWLkj+3B7RRc{Yz{lIh@sS{t|+S
zv>K_mVmh$GM8HU~f61i#2ILBuPwCoA0J{Ti?R<RmKPD4n!DbKBOVCGcUjFx>^w-4D
z8=x8h@wCQ>6$$ASbCv@Uo$>zY>c-NQ+k@U%VaF72+<4jXPJv`)C0If0r#kW@OUm>2
zVz$$ik1zcH9hOO+f?+^8;{+*?fzta2?Ptsf7=!l(omOrKh|HsSe8va%SN5Tj$c~-)
zRvJ}RuwtSAJ01gk3It@KsmFf)8cMGS@c#qR<kXl`@Xv7}B|@yNj=;CH5#8mNi*O(z
zq0dA50cRiarf~;HaYeQ-Cn3{=l_s)Z*-$&cv@xZo12c<0TJENe4AZydfBDsi5$EZV
z&M+-z3Q<X<1BVP%|BzspMWW84zKU}fG{Jd<W4$PfAs~%B>Ue<s6Egz(fGh|^1iu>g
zkaa)(En!a%5kW$QP{h}rxRCWSvI{cQ1X(b<fc_wsE5U(3B<vf&%YD@D)bZH(T0LAe
zSp1v<*#Jm#lVw3J%YYobgkZ&~?u`0gKat%>jX4i~E_;LP&y<}Vdi5&o#-MR?c}%8;
zwEB|!36HrleYM@2vR>VZ5N_Nn&w2|dWwi|bs4|Ydqquz01PrqT4*2e|&o!y;Sy}0G
zV}7CoKk>=65adHXKm{T0oIybt#a<3(4)J_1V0F^%ni?N@BseTM+&yzI5u9O5N`g5c
zaX22p$1Tj@F3t>B<0brALf73h*+GH%i8B0Ta6+x&0P;ah95N$`Q^N*(GjajE7uG`;
zY%w8r>|rF@GXK0gArZK|B2w=0_(QU_egyQSl5jQptNAZJo)ZvQ!niYo;ASP*ufOEG
zgzQXKD#ZT1OBTGo`<mkP3NRgCC<1yhV>^nBM*6i3;>MR@ARW(*vfh~vhvrPa|EuY^
zRyNIseL;kt`vyb?D0)~AxYxg=9{iA>pK)~8)*VKI-p}2+5md+r;x$Otf>*oVdpGjg
zhwX4+=8|HR`-I=F4VC)kUHh2Xe?T%14n+39hN}K5gvJ3fFjTS^^DU~**$FR6{<2x9
zK^WbH^Xo6NcE*mC4H>GWLP3qap*n;N)qa0%%uuoJgj<@Jp|U`=#uLmexEIMJ!WBIT
zAoP%A%jNAD9<J2c-NlMU@N!2i`YnxiK43Z}?jc@d&ClYq`_u$OY~Ut4U}p+r-a8WB
z`@0R>hwq(AF#amu?(@h{#axw6#KFShC4^v0hIM(eW7xbZ3UiML<Q~SQmN$Ogg9+{-
zV<W)w%gKNn4V^^pF_3W@bC0(>_c*<G5AiuR?3<m{fM^Cxx|&R}3SwneAe0)q7QBpi
zZ=N|0dv{D&d68X`a!HU5`2eK^raI=ZQ>4}J6bcfn|2@Gl<9Kv;mn6xGM>E8Ht%`P5
zGPF@5g9jQoUA7p=E!Ri$zCJwQz|Q`Ltx*k&l0B2;r}fSUSFgi{SDLVCzyzJ7)d2hQ
zu)MB+0`pu(I~!hmcf%*}V!8OZ%fq;RkxqmJTMJ?K*w50%+FZy8GJzF~3Vp_k+MR;8
zOpu;3Srjnd885}>NP5aJ*AkN&a2F>)tRjv|5@y+`6U{uyoGI`(RdDO3He?`P>X=O`
z4VNy~mSm?6S34Cbv?z5qTrJgyE<Nlg1Hb9C{flqpiPv|X!3CyamO<(@zKn+P^+WUC
zM>cjts3h|Dk{t6bY_laURfTV-hjUz#9r{1~aOtKDtS}9(-gN~#&!Z*JrSYVzGN40z
zol7MRCI{)bQ#}b$T1aYB)o*im{C+KP#K_RcVz0GFop62Tdxyw`a(4m1hB}Mn+|I6{
zZXhwm&boO^-!x*fo#i}xVcYC;-dp+okhBO>Qwuc0_n9clrJ5=Ik@oYmeL=)-!9j!Q
zUN^^FZwSLRV=oTJ{vy#cH8`eOX^-iys>88EKIJTrF_f5Rg8c@oWpBSBXL6DbL`SN{
zZ2WuM8WseTVr|cAsf|M5ru;c?tS6!2Yv{<W(A}%h2e^yMiBsSX7cua<6**v|9LTYD
zpJ;bGVBUv`5%vM0&AIr!y|(wNJKrRuyt5Y~Nk-F_Vb0%GLdd0**lQb9N(ZXW&nNr-
zGb}=e*UKBuZJsgKFz?;*``hiM*qaPa1ag(0Qy?DIlUH=hG{jiwq$BU^zT)&zhOVgv
zzOYLEaQE(xsYb>FGgLb>cN<BLr(C*2vNvmgxzlqm$bK{P47f=yiG3Tweg2|a#wzUB
zL#VuQhjjQn5A*c_vOwa~$r&eU|J6Z8&A<snuw(cCcJtjo4`Iz`-K(;)W5jPw8_9^o
zgkhvfWiJ<ngK$KhDrqY(XuA{s^~rzz7{x#i)9ob`th|>I`SbPmp}dG%P1#%K{TZBW
z>x*AQk|t&ArorUxK$~l}Jy;Gu*aPKEr9L=Y=omG<dD&SYW3X-%Xr@`W-bTl58tx!q
zr_g414PSD6<Iw(76w?QT>WcM$owH?{;@h+)@2DH>Agt$gacnp=&3`wn^Cm=Rrs2y%
z3=p6uUz7|UyW4)0&4OO3`HvikBy~>Lb>;`|v|CyCGpkoLrP|V~^^9%*_4(<;km<?N
zy>sN>r{{2ZnNF{qWto}L{T~gTk)3VC3`BiLG|oy5b^FU&mSOWH^dFqqEsMmo;@*9Q
z<9XT1`>dE`+#{KiuhTc>*sG1YRw8A{^USL3%+9S}-*a~UTqr0n2mnZ%{a$PQZTao~
zsVIq!AHmkLdDCjypjgwg%w)_uFqGTNncucb84G*rGS9CiclCyN^ttz-mAO{jopR|u
zo?MaB^6~lNb>|ru)_+~!;*t;#c-^UL;`qy&-CISh<fu#B6l2a0-Ul~h`OI%I@os7a
zzUA|nyu`Nqg9MHkLTvn~(=AIY1B8wz#MtIOmcKyXCjQsa<bkC`e>qZ08RIGab-~da
zVeeFF7H&P$fdMWS^G=DNNxASAV>7bEgI%Qj@~*~x)(fq@>#gxdl9`0?#>+hu|CfCw
zsR)M+f$agg-TnI4TkV4onE*2V#HZ`UE4rnI6<9rjhY$a{kynB^`i+!dflQova2@fG
z4E7(V%%9UkssWw`LpT4~gMYjB?w_|$--k@<6`kn()nCbVNMU2beepc(Fxsa1bAc~=
zc~4+6>$I4p_ulyLtP!tQV4}#-q+y<adBHt1$aAzKxv;tsG+3{{+>(4J$}rbV@$Tu1
z5?W1+!*c+)F8a!$G47V1YGdf^555~{_CVa}cid_Tsqy|ZVl4?JyCKQ$HN`(*YdIO6
zzU=U8k{MXQ1P>KF#QV2gMa4z1{hwpQ5A;<#%1JfY--x?yynPAnUJjXj=dB<&st=fp
z=3%cdQwom1TOf3yDW!1uP8+y*`jgqUfG&L5MearQquXYGmVfCrAsWg$@@6;64umQX
zy`}a=&%YQS%t)UsJMrZc`W^JTq^~DuY;`BO_^#$JK71m%G%?;!E=RJ`Xu~jHwH~rp
zP&<4%22Q5vzDQA-Ti*==+wsDdROTt%@|F&AGv9Waq&iN>zZ;8-dlNz7Z4j$$Z0Cc+
zo-i4on|vdF7kUUFxVu4M=!fw{9*{#k_I>e&mYHxAi)F=hOJR|S7Ylye(c8P8+oK~*
zEQAYrOq)FY>%rB=3h_{JYs=F)@n}-%$fCaH5uaAfr9Lx4+LLFId1R+l1|=m%g6DUt
zthzAEP`nL_OwakLF!@^XKq4e@j$hUM!JHz{s?rPW{WR|)8I+jTEQ~SHtBQZw$q*mw
z)fd|txhY2-mQ>(y*{pq7%;eX8MK4@Mb+hBf3&usvv}J%ptimBu<FNC=^Y`%nOg*a+
zLYN&M@B*qChM%SMb}a%c){YN1biV7j%^T_UubXlup^S5DM~n)8>OSVg7&JNTVTE_4
z_Tq7d0^2_tshM~mcx^Ae&#jnUW4#xglyssyDJbqZ=8BYvxB8Yv@DTC=ikuP-q=&-M
z%DdG9ESyYo@K#guSTUtaCVC`W97!wC&Ajfl;BXVl@wwi8W-E`Zl9toT`#AM6rGG};
z1rzZ<D$nzhHyRe9@?35l(1hUvD9F=3@rYZ==1SRzagIdH8R>HgLXpAID-Amt<(N3I
zqpV|7BSRR=hT=eCKq3wwOpQkdi7Rvjs_K`172Bt0r<1|%%=9;4ipyS)-Cd*6%LV;$
z{Ud9|I&{iCY5qy&tnK=@!;@*(F?r?;@aI)hL~0#&K6oC0Oh;>H1pD4}gurzC{9M(5
z8R>^RaeCr|OtYSwLn$NA9&NRb^KgbS%#9l!r%}fgr2-G1pVb3barJCHIhDh^>ZDcC
z?n_#9@#r_CR2LJeyA{<F4`X&42aN7hVgGl(cF6ln?l@Qsm}~{Os;}SyaIInw?>^CU
zKO{_9w28P^sLRswox_62w6NND1Z+FHI5KDVDM>|8%<Dc2RC<=%Uy8cV>Bm(U+BsPe
zS(!yZiO`zhjrqND#kcai>%j|2U_6)->G~Bk{ykUvUk{*l&-8`w#vE~=5OERba0wUh
z66H>F1}gS&LPA0SV1#R+dpYoY>5J29Urgd7Z#*Bld4vVgT10^73E}dHI^qV<!le5o
zCGJOUKszW$W@BS9x})gE5VVz`nWrc9QlHf>URNJEVN~4lIPqF9=Jl??3Lfk*&EAcC
zLSWXoNGN~XGG{1&Rfjyr)bC)YBgvK72N~Z`o<i<T7SvX+t6QjCH8uRjn+czXgw#Gd
z8kTqH40Ov~HfoHDv`+=7Oxf{S)=VHc{}Lf+HBrf>ktu20?_%c^N|jrN=sN(MI=grd
zfoKDlOpT5avlt5cXKMlF9AU+TwB`fTdrr&N71{?6ttUE~bxJ{H1o}6#izDjmKqHzH
z*o&@CAXG9`pl}PsQJmrIpV+!nJgXDSH_^wPf~$MnRxJ+cU(<paJHPxMU2*Q7-Z-Y9
zmj@B=;oReIihm7=3gT0e6Q2G(02hz}2!8YevuZ!>48ZDx43Y0A14Rxk+o}s`8_0?w
z6BZ+l8+O)&0eD!d$0s;ygODa#0IxHF&@UO?9oi9Z9MpuPKMv|dF@OtMfC1oxT8d6_
zS5%|TJHRXPn)l>s*@ZwOsWt*;ySlFHviQvsrnU=hbIT2Yg-qQB(V$wXy0zZFeN{sv
zR9kJAN@~e6ry*iZ03s<z0X^i<;9BmZZ0pK{nX=i^Yx6aGRnUs*P<nu5eEf<phySt%
z_o1n1VDQ%OO;V-?fv>Yk$6l1fj#w<Z6ZqXi#FAXj*WxPuJ*l<ne^^k2&ij}iAtE?j
zD9%NQEgua1y*2#pTBi^;7%Q8UVHwiuv_fgpm1a{!hXER@J5Gbb2asb-{M0h~bajUU
zzAjR@{aYOHm&pO*=*P_vlh@F%#(S^hOyMhLf-&Vo46`^`X5U;{ycde<&7(~#q5fPH
zQazlL_U$!S#rHK<6?f9pRp%tNl6KPrD0QSfTWkJiH~D)WjNkpcXLkBRc3x<J8!{DJ
z=P<`Ps%0nt<?8P_*;nPWv0!fDnMxzMHOm*si7w8~qC_zqsTPnB*I4l00b2p*V5Jf=
zi3QJ)NmQc10R&C3eP*RUQ)V<8<g;a??oC-o{}@`v!AhM1hKWFcJa+PntquhPeO*XL
zt%T7+&B-wJfCxmPW30c_%`#)H-?W`Va9fulXWTLOr&znUGk=?t<}uSB@FJBCM`Pvh
z5xuVC*<FtG^04?mC+6PSJ+GtS*tO4m|GU*DCJ0QNDuW&_#>C|zCJtAe_MU%{#E`pa
zAD(Ur+uZOp(uIa%pq#l%dLEsU#U7bK!rmt74(<Lmu<JOE966H4+ns9>61_3R;*C|E
zM$DrQPEMgbgT%m<^fTl**-OwWuLBUzg&(N89uSQ-6&!x0KVbvV%NK>yX%P$OOxrjR
zCI{^XK`T(`tUj^5a>eNzU>hmy0F!PJAS1R6ZIL0syWag7VSR9z%xRzaDP2cv7cU+D
zqbQtrUMkRTYNn5}vP($prfQPw{PK$%84N_I^WgqH5IX!jX-Ww_PkZenF$FN;34aul
z-Nm(#JcO2`k1gi7dyRZx-|`@D>4M9{YOcv5VrCMuQ2sAYG^S*%x3SW3P}KYjjig)3
zz;-r-^k+smjwhyFM2r-G&)RdiNQf8-4BD?T5^Rhfx#3afe`V#t6hcT142?}Xvn|)S
zRo4!7(YP21fp{OALyTsw`5XXXRwuseq}J?9FPifywuH8J1mN9S<d9X&Mb0;=a8z*o
z^7K27Z7qXVZkU^MI@s`hPeWK!_g)lZkWhCKCnHSQy?n1ElR*Y<UB(zY^<E;PLMH}o
zv8nKVb9=u3Mgk}Q7fCMHD{)42)2I9{Tr&9Pyko4qa4ls>75$+1@(ZX>oJLSfHXw!?
zmPeD{r*}b~f)KlNk&+Nw%<(g)gt(?Cbdz=hmGeY!-h6}b&&M=%6t>v1$Yw{lCAp|N
z<6<gPr5YSRn#Y?Oh07S(&q<S=?og)CTF)fqL?!4tUB2eqw(Gh5juylZLW00cf~8q<
z_XHKECr7ej<h|mjFmH^7!$9A3Ujdv(djSLV<v=|V1`+icpyUgHZqNF{O|g_n;NHL2
zU!q_3rVnfDxw1T2F`qS%OiN9#q}kc)fZ8tbi-|p0UjFQRG(MF(KLa3l2y2TVfcD7K
z!9aVH>}&t=447(wH!_qXY)KnmZ#qAI?8%9Dz?de?6Yu#9lnw4TI|8%jX&~i)4w^;W
z8XP<W8zH3|Zzn((+<P9;Ng4uimSBYVor}WY9qc7<AyDKft5^jT<MBBlVq*4tTlMwh
zl*F`$cCLA^WO#5@FX&OphMi^0e>JPU{stOeZ3T;omam*}G2k{1OVlkb8FgS>M~D->
zT@}9qT&)IVJ9$<0EE@SG#7{TntpN4LWP$ZC-~eUS3vRsAQ1l&gKv0}#ZH?>0n4oR>
z1!!oALt*Tt(R5un5b&@8PEEe(>{ja<Ffjx`H}N^1bbf<y`hkrL=0KzZY5)=WC*Mf|
zj9OAmp+=_B0-&!oQtj)ofH{H0L_Q^p<Mgb?HOdZ<go*<_1@B`mR$aj3T^Av!j7XLt
z1^`%2n0$>Eb)Z7_UNId(j|64R0`{a9r1YTsn$vqNi9zBPa8xNARO*LEkX*k5?Y`b_
z;S2roV_)Na3yk=n%*WQO(+e-bKL-q8U4`gURO!lRZt`sssP1V2Gf3cjCBDi1n;Uqc
z3w-h0e&^$1PQCk7#RRpj>iNAp<hY!Q(W^|H#UIvFlYLkS?)1VL{D2APPS0q*Vc0@e
zV*kmzuOh&DpL%myYGzD0?67^{s$ZgN;Ju{vO^x}5V1A-rwEDjW=+GYf(qupP!T;)<
z-s}4(Am7hPl4Ve)GOZR(7AQaq&I#422v=UuHj0tvWHs*EgQIeqEAe8RCY4s_JeyJx
zMnYa0{1ThckKJt`tzi+0)klS1FWp`jQxh>#d~b+1QqVT$K{atN5-gJUT7&TPh1!F(
zX)2Wm=RU{_jm15!YVB_}{gz}HTBxsSRQBMlW5j82BxumNrJJcQtYw#J5E`)Q9)7)6
zK-Y>-Hak|qsoS*;aA1;b3n{5bfO4?Y7?$5m+l(aY((T$FZrDPNpY<$@EMyt}8tF8C
zE|`9>Ya67i4q5kHA`z?zJ7*SV+TV&hIRey89qioPhg9OT%$PgN_)3DR&g`F0y>i^4
zg4c6~GnhU=E7Bf81f!iU1SjZ+N|F~A$#BhbqVM@PgnwLLauw3Y8(g0<4Kkd_Toe4d
zxyZK)gg5ce41;NJM>-6LmMp&ez?Pq~Z9ZyTW}!sEcMO>E-oG{^$I<$%0|3ynDn1$V
zmsE;TOZ`WlB??dFm3P6r?EluC_ds9GX`!D{BpoCgYaJKAseL_O%lJn937t{N3E5Ta
zbnrduZJs()%33OQ>96LLGZN&HByanzypYS)S*N$*CA>z37FuY!q+RF&j59ZYLyC7Q
zB#ra6`PXUN(rK4s=IW-AdkWj3W%wul4e{{{<#5p(rO5uX>Wu3_#=k>C{tAOY4L(KL
z#;+VYd|@l9I@+oU2L*Bu>ci$8X%Z1SL{L-r16?RIV00F`zF9?*92Nn}iWZ@F%mhUX
zE!U!K#;#i7-@n#<YbtY6(+KpeUXmP?U%o<Mb=11HC$vA8B%K(fB=QZX%%eW&(7lfm
zI<p1R=(2?hnUTXJbKZ;uBN_Z;4Eep&y%!8WwtTFUV0q>V1pda*riFuQm6!%-iCLF^
zTcUCmH3&3!mB@)y0SZffSbo(6ZBsxD$3V-ggS?4}B&~MxW8^q>k(-fHsqevC=o^Y7
z#14BEAjg3osW%xEu$c0)=mmNp6Z~eD?$^^od1CE2DDZoGHB>7d<bs-TNJ>Ajv`+))
zOIpEMKxHV_QC!qQZ&IH9K0N5lE$>A8EHNo8NmxR>w`6P0w0#U`b@`fDZi9^e$xi%0
z=XRJG9IRsI0mno8|9einP@|tA*4em!<^LzrWZ>BO6^3n;M{k<<hW~&BIS#O-oG=Bz
zYr|}K$vnI%qF2ERBwTYuTzO^fy!;Ynrfwrb#pO=##t=0b5gkBka2D@(;rjZNiZ2CV
zuB&aS58~~j?c{-F^QxvORkF+alFigh>BrU9N2dT_sUdMe>&W)|GxrC|vM0X?NZk79
zftyxtboFpE3;hu9*sW<<!jpHM0cByRT};AiCx2a*)E>}lk#cqC(-nf6giBa{G)!*-
zO{hy{ff7-iBd&u-83;$cx+xc9DEGXH5wCO!z1E6@LInE4_>ARa)`B0M{eA^KD=eSE
zM1b%U1MIrU9|&!idxDvPYC6&M#AYr>K4;K$uWV+Ju7|r0h2*q<X|}iM$^WsU2M)RY
zKaTGPJB)AS*M32v8J>$9-wS|@?FOi|Xr*^%85iQ2l2Gwh<fIT09p%clo=GFYbAcl@
z1(?2Arh2AA=n)Ax(@zfxUyL_@tZ~F&2d`2Sp*KY+*U$=XYszm6{m|NpizKr;MmmJp
zZ?5>uw>kkCqNWgQzo{d@1yg}=NS!6hK6VF+L)P1mRxdCIq35*Bf|+`Oes!THwdrH&
zo~XqL*X6=~4R8#TRBkLUCu_Fb0cmfJKS{MYFqd|@SeJrqWZn|ZxqIrBE;_EOhV7yS
zGr)hK31nfp3*I;YI?V>JFd&bww9l+h5D05Iu2)WL&B>4==V`=~t97}i?G4M$K<%tP
zR)ZC`GSl6^)z$@^cBz?}nUbY<oL#|!zZgd_KckwtQPGvd3u9kTPsVRL0cn}W&1o!r
z+FOWvz4ywZ(?zLPavUXvQQ#QPTu7u?dwB4f-ZPWUlz90yjU}|5cDl^jHAcTynQ4fQ
ze}=FOUR#$97L9UQ>TJEeWcdyU1$0^Ji4PKMgv*~C$_3t%Ajyfl8#>m_Qf+FwE{5&l
z5j6{pPhkxWxvr%tv^23WB7Slk3jg?>#fi?ZEe5#h55w1|3LD#jTxcm1IiId_)2UBK
zWRs*@HrTA;1X*tm=E|*C2#%F9xf=4NiUFNLq$-nW$Qb$k1sReC8SO=3wyzJ8Yzkd-
z*O>@L`BAF}XFlznuJr$0cl5V;#GgPxYlSTEgwlTj%yuq-VyzNy?B2jX@Ig%ErHcT7
zV@K@euHlccu*l&wVYTa&zPX!;>Zi-MaW@}FNC+I--q@gHHlcW3Xxl82qEjU&M+&rF
zS8t2kXQqf^N#J)1$AfT9xoQZ>jII0$=NXi#Q)U{-EPVtwG2L^*=8SXZ(|PUbrgWnQ
z8T_-rMbeh#Y^h5pJsFW!z`hxyOjlP5`HK;*Hl8Zl;H6^-$4GJ!SI_-&;+M=;jNRXz
z+p(GwJ)<F^VAPmd05td5VLby83wc7-{hd)mMIY3nGvUvv8CODFA+MYGpM>LKt*L`U
zxtOBs|0W|$s2HZ=CCQZhhw{}vP;xf~IhVvj_3Lt^>Hwi#1fXQkyq5KSyQK6e^@74d
z(>b~E1%J1Va)H;96u3p?2Y6uuMm`C0Y8Ra4QSa|OTH~;}%*J1%4|MTSBN07S+#nTd
zpDu${W7%N0c@M4377<+eCQCNJLC*5NeCftd?yR&^)eo{|Rhi*BDnEJzBZ2It#H^k1
z_{aj#l&`)db`*Ubf}E*1sHoh_rJL^>F9{4DJ4W%JAX3ttV*^)Cx*k6D&vo$EH9~*?
zAg!DKw)n?$V$Zq`m7%(V!{`HOix9qG=Bc>j9hI~A?yZG-#?&jxRrWv(8uUcI$PJ8X
z)TDwO=?t~>`quh%;dl!U&WOV(y;bokSId^dcfz?WBa7bn4JJTO;2nR0F&Es;-faO;
zU+t-P1pZ+Ww89iZR9Wz)6xa_;Y7v!oRada$CHpsntB&#CeQp|!==n|ZSeF72zi{b#
zmVry+bUq`&C<Rek-r#|D9&3qSWwZj9DYo=@oZXy)`LZLR7GLK$uUfX7!%3VvEt;rs
zv+1EnuU&s{Zqae&Xq`YIDToV#rHUeg(0)Kd(yA<T!R1uG$fC%iH!Y&>J!o<`@n=LW
z>|dL=`UY!u7b#Q#dM&)Zh}PW}nsitg|5j^rkgj_A-lg#(H2`}Kt}w|kEuR2l5cK&q
zNkW$cQkG5u%Xu(|`694e)8^d(L+RYO2{zLDMg4-x6MEa%A<etT?-{(JGyk$l_a?_n
z{b18cC?2^=4n0x%ALle+Lcj3bA1vS6kE?0xYuN9=M6dzR`vtAFK=QMpW_qf-q2JhM
z-x9#tCwdne)**ICZ1n@f1i2Sgc0S;{DLfw8EW9r4|8^cpCTdq&49p-L)x1Jhkd#qL
zV#(3bJ`R-1rebpHGwdN+?eFAh)Er!}J^FjcZK4s2{-qzl=O*ZO7szN%c8x%|RhOp#
z$5RThP!=xD7tWWcd6a={q`}*go7s2t7Dt=Q3&0`cq?3*kN>$JJgpeRwbR#sVT~gDw
z(cp=@BGi{l2p5R+ayXx{Uvi45$w}?r8nZ+~Cgd=O|Ghubw~xPHFcK_s)Vj%>)Oei2
zsC3v$G1^!&*gZn|9S&A$!A6_u#9L9hxY*cFxn^**0Zr%}XY_@%w#026!FX8(VwtIM
zXMZaX@o@@`v)<+CG2S)n10%t=A<Vx!39bp1OYr4yOKJU+uE6XicWcY#=Oe8DflU^#
zxb05jt^Q=@6oXhC`G0Oraqy_F`=R<2)(#!+*&I+`(!T5APLePQOo9y~nYCPn=c{UN
z0$svEt#&xRwAZH~%6_o~CKb^Vf6II635m@&WgQ)z#JZK)ACtWBUPA5~(>PNR)@2U@
z`k(Wgd=z|JIPaQS;!VLKWS6l(g03-ZSO(%pbw@R;k=7Cyr^d{hP3wJGB$aaVLec`G
zbPbC#`y0mImy^t2cnJ&Jsx)REJZC@S%^Iux*-9F1)v;i%pzfC9o#Q-usZ`Bj(k}kR
zRQC4Py3<R!UL@qPqdVmTgOt18YU{)q&-{Aj^P5UftDA+kW-)7vuMomZE4Ab#3Y>*w
zi3)yAjsESc<3i~;ST126x(*|bE(g4=T3D{0&ovH%>2PP>KUzB9Fb3SMsn%P-TA7yR
z1wie*tA@OSs<pP8=F8Ven@l}eb+ekztv&fCj(j(0Kq)<gI`7JNzyJIO=8I)KNb3#C
zqY#Hil6FNMckR0<Sc++nC<PjuTPap865HU599&ofa+Juzn_}Cmji+;%;2RX=O5NdM
z^aC~mXjwp%x9%Uw1}2QFRWc&xmifR6(x6Q-KZeJQb_FV``;$MGTTvx0>jhurB8B+%
z3zaA+fx>|_K`gt!&k@bpKD&W8GZIc8qYydTo~~PIg=FiKMZ(UIGM2c`1xM#CDSH~Y
z3HI}|eHJtc@>1J4zbw-bPx8|LwnrJ9ia#9sYOJJ7;$0ikikEISZX(eyOUT>k?*??C
zecXi;yzM3VV3?E-F7-KCpkq8qGJ$|1YQ^qH9baQMB7SBHsf;xJxOG}bCP)W_HeAX@
z2XjOyD~=fSK9DDEvRVOOYxsf-0?D>w$_28vO4VUV>!qtOkC7$XVdezeMa_yfP4$0y
z7nxpI?=#B=uZX%1{qJz^st8^Kujj}n-hp#{!Y^k7!|3nBz8|qUj%aF{lHL&|+FV~w
zZ++Q{s?}p6SOBl;+X7HlsNB6)&>sTy?L%E|jUvmkg>FD|cUpuh1`;nXmSl>v)`5cf
zdZO&zLWJi=O><5o4~Q==_=hNXl!aTAm;kp^byi=i)#`}YcBsPqqzkws!wyBQ6#*cn
z;_&jCSqN7=wIq*347d-dhAYu^0}&hKIA;^f1py~-g&J2&hvaWxzUct4Cw%4RDi~!z
zLttOXZ#Si-Mi)w%35oyVF&66bhD2~c5j~HQG0nti8o5kF5mNhdw8~e=K9pq)E;^j<
z5Q1@C?`t>pP%7mb-1uLk8hdO%Bf(5L-TyF8zvw1V`U5>7oA?LL{eQH5by!vJ)~#$p
zLJ>tp1!aSjq=Ja_#-gRALsCRSQkrdp0TR-o64EH$s3-_XcPj$YNK5TI7qHRaIiBx(
zo_p{42T^eCwcho{oMVnT#^cWh6@m0)Dw`;Mn{S6&C_2WeMPVX*REDY^JCbNAoht%3
zwd={0T4VT^f+){*=Qgp#J8vCqX2JMjoY%!^1<@qBp$=h-j#H0ARE6wOy7VV>a99kF
z0cM(6Ct(Ka8N2I*irtdgB{05YYM!vwd59HNIj73H+g{iY`@!WEJxc}MU<=p+I~7u^
ztp(@Jk4r?MK}1nS+BOnK3)CPl_S>D9C&K#pkXu07o@Uihr~G!xb$)cP^NjP1v8(o5
zIpe<h!XAjS(#e!dRL*OsUAy7km#k>nfA-#(b;=u-?H~v{`k0_}?6R(Y<ixl3rS0&k
zqkM<w(!l$B|Km<kDMSL$_!;qY2h}_DOD4q?GdzZ{othO|1Rm3u<*9yvs)L^Ul@v0R
z1PPDuia;<CR<#fSfV}0G)_o;u5;+oGs=81K+B3Y`|L%_N@Cypidbt^yoLOWSMMUrh
zQI?xQG<(t`dq4LGeiW4GXBGeDo$WkpIy|Bp);8fXj02Y;jb-cb-o#$PXJ~XQ08^7p
zZ{`x2YU?!8W|!wkhwZ_apnYBp^aE<;uUwx|Nvq(j^jPkJBrv9ph<GXLGTn#t-32Ac
zz-VkgJZNet9yx`OFz5K9&6Y!vLBJaN609#(`<nhwA|;5(1n@U})*h(fW()hpAp{va
z^RtiVy?9On3+Xr<WzLZm-mFnSj*uqZX{Uv#t|=L!j-XEE*Q}(my*(lh86NST0``7y
z?XEX4&Qh}=GJ_C9H|K{B($%LDI!88vNU7TAiez2L5S=DX<s;6XN?5P=;N;<16p3%6
zCMxQDIX0KL6U?BUvO%6uDQZ5x6)7>~F9?HVmYrB|bAf~4JQ?Z{4G(2AcaQo?yWl;T
zNKmY3c0R-*Lx$ohrCWVEia4G}N*3Iuc~S_1U)^eYJ8`~rf{l=q1*8Bvtfhz3;{+Mf
z>^)?tuMp{%O4g2mu~4SCFq9yJ>{MQs7KwR2(@B6hq0QYzUh(ymQ|Xx>%jdGO0|xSc
zz$X~<a-hD(HPpF9jhJnObUs2P*FnDVnX}PfvD6*dG8AF_GL3u6BixxB<AY~MFX4Ep
zA1qmOG(V@P@Cg){6YHEdx_Bn_XYawU*(?$mKw7MG>zPO^LAYsr@P-+s*pdN)6i)PH
zt%LY&d-bo{5F!FZ%1-wS=$WuXktsLW!SqM{i;V)@SWVeg$kZPdr|=C__|gOr`DvJu
z*vbMtqubSTp(`o^uoq_GsUkbuQxmRGan-eKG-^u*abhToIEwd#xOjklNO3n<ei@RS
zc_e+z4|Gy498FeFe-2T&E?EDDyr~7v#7Bo?EQ0y&^nToBnj4&K_=$;B)hB$Ts?^)0
zzdl0)DxKBiPfX`Bpq#A)u=&;W*)V6Y${hq{p6ucg0Oj`EIoY}j_8kuJV*sP2xa1=N
zhZrncjn{9`gNoTqtG0E=3Y5{A8YCCUE96FRB2m232RRie3L|C(qphhHG@{_lpc5_V
zdMEUh0+k;~=d~x>_a3HLxE~k_PpEQw!M_PGQdZc~6>J+k1jvpUl8kg!gIvZlC|z_^
zkC<lF7ETsabI*X{iO=D2YaU@Uus>o!T-xn|A|QF0fs-|_HIDm7smbLw*e|O<T1Mp&
z^Yi&x%YClnmO|;#gXktWa83Yd`u5Rv&d0O@wR*Lu`P`Ol5NqghL(r})5t(mRS+G^B
z4Phm%gi^sZb#s5K?;yjS9c&jVy(Un_WK#*$2Zq9loN&4DGhfoK49~kVo(b?69&;*f
zi|zZh+@t@{+oEnd@Ri5J;oeXcptNrmnfkfPvJ-MCwgfO9J_D@p)F@}zc8N?i83Y)T
zuXs@6`TcwMBjhLtnPy8K7wuR_(WzNn_~9ERzg(v*_v&f4KoBa`EEwVfb43{(hcI^r
z?wpLHjM>8>f!X;jys-O`onIkkWt&7_D2rW%2a;Wm8w&mC=??AHyDZQYCrk4SEaS!*
z5kaXRLPvLr8kUo~ZU~yg8J`>6^rbmPQ{~K!cJvS#vE!SVJG~P}d266B6GQmDbm<#K
zLk@ES2=-_#FHD5%%(<0<z}_I#WU_NN^02!EuIHcf0pum4G2&`F3WewpRw*VL_dd_r
z@V?6Pdqj<`6<~)XOn{)lIJo8T+W3!toP9vPN(oYcSBn**CQ=?y3ot-BwmP6BmHb4H
z0$e@<Aj0biZ-j73`60^#CRX|Kvy*uP{*R)jhNKS!iKR1<KD--*j)^pgLNg#COAr0B
zbs&pA2uW4yIYIJUwNdpIoPl!Aqq*?@o*J&Pi}&ZJd#|)uyz+2YZ5h_CT+ZN%JyU@v
zE}uo1NDuU0KG}SS#``nm$n^e2NGjHs;23jg?}Mdx+kr7#@gby(Ma;Kg0@S3wwSCfZ
z_+Ur&6IKt}z^ZqBz<4qwa?v?NxCf{4B9!k=7hwzHGgRHJ+367~(HXSBM}|r(_(Hvh
zJqT8cP0%UA0S@H5jB2-)iQJ%^G3#N|9l5%f%&bb+9P4ZoOEVFP$h0+k!ECvL!Dg{j
zPVTt~eIY4R()XBVi}~dEWM^M@v3UKuDY~$SzEfaua4>2ml!!omt-x=R&LLd-md%6_
zNwBu*4!6#=n#fw+m5>Am4O0X6PIvzaIAsIexmhIMInmD&5wxAkiu^3J+RJ}Mo)Hq%
zS!43PwVHq(oU?N~)qj21Bi^0sHj{^xwkBRtKxuEGr%ptoOiirEnbr5j)DS$^+D5c&
zbZdSE<bnq^TQy4$pSB7W{#E$&Cu}oIlV(wR{p>5SgRVT6KbD{nEtckP&!C$E%Q3Gi
zJYkQq$fX$#<T~!YP~k6IxqcKN<5&wEtyLaF@>;+I>5LWBU@96twQrBp#MXe{n?|Bl
zfX;xHSrnU!Nd-k?d{Oler^3F)5<TjWb&ksF&C{8!b-+M&*8z(gc6#M!HX}Mr+NrYn
zy>9|6#xv;OoJS57OdzNbiEA~CD2>{8o)h_<Z(<feBTAwpe6Lx^UVIYz@hBm}ODa{$
zEjIQCuqVvnJu(FXo8SFVdxqYy!XvrMt&>;J>vTY2OeI}URJK|$0bpje&IHg`8loJo
z9nu)`Mm=Xa6-)a#@lKIS0&HD^6Jm+;BmD-$v3K5oq24jBhMb2cQ8)b5B8Mhd7uJPd
z-HGO#&t|y-&hK5>ePpZ={mwe+*l&r$K?QE0+MYk!dQXn96wqiBb4ILP0f2C#UEf7|
z51U1N?fw`B1c@mzfTJCQT8Ps%>|3LSerPR9!%AJL4da)6)Ddv3P19_&h%)7j!uO17
zR<k5N2~5`#pe;<{{<m<TI>$L{?3*<_1NA*Xp}Avasq(;ANOzDG%D6e}po6NN=zUB}
zLY11RMt5F!IDt&A!{d|6z(;FA{wvA65*Xakb~p^jbRsCAF67+lF>7I$oraW8EU1fF
zNPn($sO9R@+PlB#R7|OO_xuohYx_{qL(5I92CHM_HFn&PwXOZCxZ%c9#IkJ^RIdF~
zx{C@;0(3(enx9$saEaUb&`T=hb<sRf%AK$#iCfNolQN?_Ixvl+y~;#tKh@hK$a4A=
z4y;qOq3FEuI1)}57LuL9PFfiTSdel1A~Q%4inb{u*r@?56{j=RjqiQ|JM2Q7qq#m%
zqPnz7-0}fFbrl^%>)sfLs(jGXu@w91{&!Io3hHK{`pnVt3)IszAl9BuNrp+#npr0{
zCUS!@F$R#7OgG>&b@nzJ`$Gbx2{>iK@G1Xc0XVkQ3+*v8T)Exf4)YY$DM=nRfvmYE
zyAE+3bKO<E;yRiIYUarh3<oVV9BO&jlPFD5;0aJV=#~o3pKJv|(-FawtKd>1+pKQB
zGxKEuqz5CY!~n|LMVX}fx@C~M4bl>M6TvOf4`*vimTzAG`J6W`q<6`xdg>P-A&-9R
zFEpY;J!cp4cqPeY&&GU!<V$#7b$k%Yp0BFv5|VV>#t;l{YHfY$z@*!1XUy0W;dGH*
z8O$C_ySCR$xtB0Wo|vsG75%L(p|$dpNT_D?&rl8hq|7UEotA#=Tj>)#L)Hs}Fg^F{
z)m`ez!L2Zw@E6y4UQ>TI(N&at@mlI(POhQapN@}?xlK!)B78eB2sJTGYnrYF533ns
z%W3N8X?tp51$3kt#8*1%bOy6}D~`F!A&yh02p^ijx%vE+ub38h(e$|v7+4(>-1GzS
zN>EAG8MA)}fnOPxaAJPw@}MEf*RE<IKH%6uX+n0~0bswqd=I<zS`XVjm@I$e(7WsV
zb6^pU-H03cNTVOvq#)jt?E$0^UH}9`g&dMX5{^m1Uq3SKt^~@{IC}yG;_n*5x>r>z
z+Wcet8>5fAR2Ml-N#?78v?!7rr)^HMS`BQ&33kIMIrLE48bVB{0!ZuQuKt4o{R}{1
zu={v#at`YG(rG2j*W1=V69=HtLy1J6Kb9Bf6Bbx>SiFD(g7@lsqBfR^0gkl1_t~}i
zF}*{VzbhwS;Tj&j;8`^No?>PsA-W1UIprdjm|CGRPC<~p*EQikSc9Iq(3j6{!zmZa
zdy!IfPO&2FXy#Du{0zX;)sRpd1Hij?O(Yv;pR%Ct8{^OWIv`9H%^j)IU;Rd7xQ~h5
zvTZpVN5iAX$&$#@K<O|OPAT}^uiUK4Vx`D=cHeMVcl@xWzmET-k|_x85<&_Wy1LQ}
zCxuQCwoaG~KmPPs`o}Zi$_T1I^0<tME?(%$5|?}bgbo7Ix9|q@s+ei?+~?H<A*b+v
zl;$LQ-&t9LQ7n>Hj?wG!p;PA&tw})3@7a$uGXI>Cc6QFs3=vJAzqPg!PdwLme9ehl
zaHMFquj;!?x|3)fsP1)y&xvGPO0cd8XoxqlfG9>a(8oUR%2S9aJQ9}R@=&@_x_nX}
zx2&$-(R>fGuFsIMgcoSls3=~*?UugG;(Q4Z;paM8eP2{0ZI^S$JlBUk%hN5FLmNE>
zjQB^KN<0=_<(DJV`_}G-yo7Ls^iIZ8tfj`=W_(-YO+-&5O;oaQlPQ@28YIqN#tr^R
zWYERHwu@W5itl?2jM^X+J3|N=$*BCQW@VLLyk_-VL658Ug;ubCmxx!0j#9C(uSe$2
zv0M5_y^;vEzw}HKP6co-L00FTZ*(h8Vy&RK9!{jE1KzWhL)`(xKOx-z3;=YnPR*$9
z&L6JK2Y-~!r|H_!`Xvz#Wh=lgS(?)M^4`48X@Yaq<VEp1)I?cn(g7^Y`#AmQM^D)s
z6j4512(O7;7=$z-;4N=--K)B%Lzi)gJ(Lg2zSwZYXSG}~JV88`W2o~Cs9v6OTFz|)
zUC8>|fCPHJ^?7!f&y<P8U)Wcr2O!X6?#pv~<uTIoo%NO;DoN@EKz1DP1naVk)DLc7
z7Mj-G*<x`{+SUh<ken&dF<g-Ik;xIAFNSCmF<WbHS+K2(63vuR&Tv}2Is~_`kafbr
zJ>Ep#r-3M}xo!wjtjHxDvrci_u6beo(v@PCwc56+^>cDcrDE)NB{V9^&Twju;yal0
zkiDoa>eUvZIgjYHgcl9TKePG~BQHtm_(cYf>FsK@3=eF7<mq*<tc3dsWmh{jUU7;9
zvoAY+>Mri6a&Q6G?OU-@0Dm+g-?6&#zzxQV)2_m^&j^X*+)Z~l#l1zsBHp32uAnN2
zmfB5fdMY?9oz@+#CLddc7Gk>w`M%&qH@VUWgvR6rtoKo;fQoxb;|mXn%VRAq3WOZO
z^JNQ6NaEC!1bWE%Y`T)v)Ac$B!icakYR1=PZC$~alw-oR;;2)sXw;-Dm|6lEWIAYJ
zUi*c~7iwbO*6<s_()%)M&~~MxI%9}&QfPu=UTY|;im^y5ylson*rRP89uz~LNuNvS
zy_w_t76Iyr<F0fNTAC?Yz*8=8g^O6b3qagE;RGB^)9yME^ADZCOSBL#y$3;kBd>zK
zjim=@tT!I}u<LBP{bU+B9mSCO!ROAX2PRQV{TddU%lWoZ33htow6)S>HdYD()=}vG
zZvLC&_JhSaBc-k^BOiICInxSO5AwRR$lvzrOT)JAIVN&;`d31fug5Xv1mHS}4=*c^
z@%(rTrC#?beW5s;;qPrEy80K}NSU?w<djoIGyJX)*;K$Pv)c2GaL-mGa_iHM&jidX
zM(QGj&pogG5WxRxOo1VAnI9I9`iMT29p5_oF7P8Sp6ARftMD9m{`Yo?FYC^Uk)c4I
z^7F`QRdsc${@SQ|D;!T(N6MVj!B4^RdH$=y7SN-AT#+{tA3n}*U*-f*GRMS#9F+zH
z8gYb=R6HphDc`OGH-+6&fh)ZlNq9d!J!(NO?r>>mz?gx{Z0vo^yz|iy`BtEkns)w2
zzeW08(qS<Pz#V#15gqhpz+%lq3+D@;KGSK30)`=&1NA@!XyH!2wyW)8xvO8!wNT*O
zY7EsbrMa8cMjZZ;H)KxwoP`>ZYeo24L()Xg^Mzd*1sSh}`wu$yIH=1SS%KbWE&IeT
zgp*kC^*-&s&uF2tE<s{xs``N+W*2sTPY11z{6g4<5#j%1-Rot9d)m9QqHCIj+)`op
zG!1SEjUhv+>pBEkl%iqjbB9gGOvpaDXKaGxu@3OjtessDqJ1ANcZE2|I2y5V@s1@=
z$Ie|p3jtHL2$u2YuHjN*7e9M4l<-7WKsJ=Q3*|eQNMk*&@C!0@^xo+ELe0P_f44D~
zv$QUuuhhNP2+puxv0xoj5=y#|SwSG2AJeBk7cuduuIQK5;@YHu^c|K-ikTTpLNb)|
zijj86A?{|gdjs{+_(hLdhg3po{eC&a{jGm$p(Et?SbA4pbn1^rF~3X5uJG<EzPmRt
z7TLWWT$?Rl^_EP##LHX<O5}?WiFb|QQE`f}$^;bV4JU25P5rV%-c705K0c=kNPs)y
zgq&sBb%mXsy_|`m5(tefvd=kBlz&a0JLJO6LWYt1h7A$+e{}e<)uQ)dPTjZ)^1`<u
ziK1T<bIrUv?@h?N0q?xVC0>Pxl`Hi_=2YGV2qe?+YksWlk@OL&fDp9lxLxnO1u=`#
zIj!vL%sc69ybwmI;Xc~$;Qn|pfRh2dvlIP|_9yM*YuA(ecKdbka>!kRP;pO>Z)$3m
z^|$yHoT+B-`T~E$vj*}CIe#yAQht82Qwz4;blDTjQ}NP_djq7HSw73u1YF$r(=vKL
z-&cwYUkk;_ElKqYT%LMa&ga>w;#*wOV3ZmrXtlQF9x+gYLY-6E?lfd`22BI@UsOBr
zt%AzXX)i;N`2OSTu1m%4YV8M9@tr-F(2&h4T|57VgfQo0XJ!Omn&CUra*$>_{$PVA
zj4?(i!gA{MRNEBcZ_rz||0m^3XM9hF4bs?k+_%88xr;`xPH_taiaSAdZ)%+_>xG6Z
zeV5^cW-VTJ(H1$`)dd$e+oI9DNTY2$$2Gh>B%10bJcG}zoed&eAIY{sqX(tVGoup}
z>1MAtn`NMT;Ng2GT%zY}E`3v-C#^Ow#f2B1e7XMnw(iRd^OY!Xfir?%ax-W^Gr?e;
zyB&vDc*Df}K*;C~4kuXg&EcRr@!+ipLDTrLd5gN&xjehM!tvbyodhK-c9=&E+;i5C
zg&xIk@!2=VXmg;b;Oe*8l<>-AR2a-%@xZ-X!i@+jc5mOq<!rO&F7?+Q-_CB{W1!ED
zzO|1%O>@>a7kSJ!NBoaVl0o;c;_b-V6We5SK{B|(i1v1TyBV}q?ZiTN)>PItA6(8k
z%deezij8zzq661Rr+et1c#tL^P%2&6HCB{Jv(>5fo7?Sk1-fBhPH0`;o<Xk-k+#K;
zF{;1shDw`&gzQt!*tN5p@E+VmPc${H@E@xg->*Wd>I+a`%RsFXzTxekABj#w?b`R)
zqOQSe^WB0ZXQB9Sd6R$q^6P^zqDl86@BG_XZw~kk;zLY_y3$^K-5ksBOl3}<mv0~n
zF!%|u{hgvHOfpqpVab0Z^lyA3Hjn(I)XCq$W~IE>)cPMnp})rqv@5h=LA*ZbNV$z=
z{*zY!=752p!HvSF_i6MDz!ohCQXgH9h0JJa;H??w_BaM0F7`kqH-`Oe1~Qt@8CO3T
z1AWd=^U7D+%DpgbSTxFQai`KtyvE$0*T*0do)4uY>v!RoBYvT8-Xq8s+#xb*R63n0
zL=4sx3<y<a*D(>qVJY&>2|!N+5EGgx;PgH=d6bU<)-g1VbtUA1MbJLh8@o+GP8sHT
z^4{q4+f(k3!8AsMGjj|Bw;L?~92ihsJqV9(z9?=MtrP>r%o$)n>OO_M__2u!1=1fE
z42U*|gTrS|Flbu?hb;BZ;baB<Lr5=K4Lq!B04#<UqPnYpn>w~2*n_6wU-)aYK790n
zL(^D6r#15yBbRZ-%*L7aGIk=F^N1WaV!mJw)P`Et)d%9M5<pZ4oL#@!dx~G8AA<fM
zq^syx;5A{?MYKW5EOp<SUV`D~pIsWim%#S?25pUbM65=1lJ3AiX6r^vHuB{gyUob{
z0D8rtp5=-?e7^{~HoATGQRP7_``xK@C2~xTtJ)vB3{;W2JW@U}0kQPP7w315G`xDy
z4Y^$-Xp^h|0;5|Qh<}iCFKs?914jPD9PjWUVFDx`_ehs6ot-a?{=E<UF%Ph$&{jtE
zqBiElAMfsuSwuw$3(K%x`t%ziqJ>Tio=~8`Ak*N1ez8loj?Fvb8K}H>jYaSQz!I1c
z<--1qvtI`X-4++QOIK}&ll((iC1RKgLIWO$!(zWVcB&JfwEgjUfkp5$W_r|)KR)=+
zQM`g$>!Uf5JW^a;pT^jg>ue3pw->ojLigDSVjcS&n(%=aT8VUHKz#;SfV#%;5?@Yz
zHiud4%}N2${fYb}re0|(HgBsTrU8~1q#QA|nOgGwZEHePF~D;(HXlCyhiden@6Zrk
z?+F#**MmLvFC;^Vuw1=e8jmo0DVkHY)e$c)6BCmK%~2IXpb}Z~B!mkBF=Y^`y(d=!
z?~;wrw8kcNR=fWDZ6v%vY(2p0W1l={2oedzIb7?SiOlN#Kq)K-{N?EKo1<;%Jeg*7
z&!JeTDz-At7YczpbkT@TUqw9qkRGrZu%do~lzJ8lpuOTV^lwa<q#u?~Yln`HHkLXE
z;1~wJdMYFXIh=SGG#;pV+et+|;)Z80+k-O6mcL!*+Do^UVlVv4$0-e!eEkDQg3ZyG
z1In`7>ltl}oWyCe*Qow|1NFi{-|~4)&Vmzxm-N6q37VObqJOZwwai>R)5@q=j)J%d
z6ZdWv=@NxDynWYEa?oWJ0Jr#%QcvMB<gCw>UR`3Q5$)sm?!QlK<p+>oS9W^lG*qny
zp@!=RK%XSIMvr&zzvcmn(Q{;*I_>@93Rm=Zv`Q1$rp5uZg4j8mrQ~jPYd#*lz5hr1
zn^$ttVxmS$^1X@)+SbKtaKYf5BC*|u{bg80`!0d2;u4X}ISjH~`Yal!+}0*}%1;|4
z;gpX316KRLACA?hQe2C=r4s`p&ViM!yB8SwNCSP!kWRkka<A{Pgg>>R2TEtxT-u#(
zum&>-M%?EI0Y1(NRxz8iS}7>Lw0liOatI+a&NM}@tIj=Lkw}2*g3c#jdLF1=U>!Sh
zEW>~3;&$#i#wj!<5P9>{;ThmWtytDxcZVhgq>tzsnkviIUFbH;eyG!N{ooEf529+<
zTMp#kdbyZy$*(X44PjH3YQ@hj>i(&7C=O~0YsMOzJ(`#s$mt@l-k|u$=>kS6D_#LF
zD_6wVO*(XKuiTD7y_8biyLbNuLB99cJ#0REoiChvX*=p2^}u_UY9R9ADw#`Fq_59|
zLyD?I-i-J8yckJk&R!sUd-vYGNYA$Z9KOOHzRbd=%pSgIkK)#mw4}5&7xk~r7V(}<
z`2E8xooMN0P2tx=OI`Axn7L=t#hCq-qipWYkM#?-MDncEoM=4eakK02gJ%1KNLn}t
zo|K)srxhqqDp0X~F}=MDLiGb;%=^hlV6C-){|#O8A?VePav42QQKaDw4vj`jM*zI`
zRpX2>L(x6w)O-ney<`v<8(Te~e~1OsQMc)LNzj$2S|WIo05z0IXr-M*NObA(p%;iH
z-vztJSs@8icrAc*vWVB<q>#t+Ou^>ly)R8PKma8LC#e_Z`AlfPQ>c9YSP8p`LhTHd
zFt@jMdT&)UU0Jl}a+g<27FJW^#<zE?#JvU+p-0TpHWkpl9-xRwZ#|T4!4C^Y3vA#u
zwRW>cZXJVIAmv=t+()IGG+j6`m>J8?Y)wkJIMksM*VCmVbcq)=!Z8iy33;S$-t!KD
zSXE1}SVxDH0b=K1Y&GY*s^)2zxn7D}vvvpGtQHt^*T}B|o|rNTz)QAOk5E$UEd>@f
zb`jH>L%Zq4ihi72z2F7CBkJ+#7qJvW_m{z@;|IWLSHLlR{Ozw_zl5Of4SjLq9p=x3
z<-<-l0CGkXy1$p6*H|jA<mKa^0~KRX5#SKAcZ-D1hows%6>=PJ;BCQv8U8jibAoXw
zamNtZE~L&KX^LW$3nN>2*bUyApI^UuQe#AqeN1(7>eo<lpE)1$hxmc;zBb9klhdO|
z+Fh7sx4OvDRH_86k`hM+J2%^|y|^%5Uh?mL3oT*f2*gNzKA&gV+USg9c8ndLBI$u8
zS$teAb+{WhTpQ?U=~#RG0QU63ThO-zLIVN<HR0i3irbMhP!#ANjx9q&i+rGl3G9!Y
zCLhNd`JXJ;&@<*FvccSHVD{zXlUKs6k%1l+X`*l9NtggqTq8g_F9F5v=%CTD!OF#4
zFh4W9|9&Oy+=Jb;@kj^6X|+H$S7?c-Cfd8-5@;vtV4tLOhsH<T(g61N>rgmt7%5`(
zb4Im{$_CfkPdb&%2L^aXe2oYmS1nEV_3!CZZGB+Lo}G`E0wa>}npdK_<b7r{hNe>S
z+JOxbG5#G<x1-#sj8!tWC1marYtTCj>|M2T{k$u28C$y+ZTmFK=#|~EMAFzdWsvcv
zA>gp+Dr{7-fE@LST^_0hA_I-uBAie25F>n?-!~+U8X8)AX$c_8SN)bf_s(6Eewq$#
z84nA-0?+=_S5}&{2XppW=Y1v7b?h+tdP5~QKfh1pOU&tT8Ulnw+xY!+z=Wv?9IptE
zcVgh1@Q^nlpINOcjKK#@ump3cnfX{Zd<dIG{klQ1U`c-1Y9cFY-;A5WaFKDsvb%fW
z_<H<Xu^TFhU$VC%R5U3n#rO6}nXJ8=0e=Xrtn%*+c;!YaOxeNyJLvG+8@hMMub?^0
zmMaUr`<=S5O>tT~1s8GmN*Z$<YMo;l{XU@*cTur%89|FBsFNO}Bxe(9Tm6Rf#I5wb
zJYS)1cNruR#zkgjSM7#tvwe%(e==hFp%104Z+$_aFn@Nq4p9y`z=L@&hCNEes~h8}
zGV)$M^1w{1EN*n*zwj!B4U$wB6^GYLZ<S;@1Z}=buV_M|T`NHSg|tMg<2YhLiL}@)
zylJY1<isAVaXU5Ri=59vXPuit0<&wyqa6sl^g1{=DapGpk0dxib$$#=Yu+-$5PH%P
zjb;bddNwe8km<a>=K1~Odad*4%kwMCY#nRzChqO=%HbAUyBvm!NQ0jEYT&iwo2%~>
zF)7u<-)j-=<;7LDQ{oJ<@b<2!;WhuviTd4C&Km9n|JnEXxO5%Hwbksc2cbt1fgWul
zA>~;Na4^_^E&|gd2}C;<NSXGbPeQ$E4N&x)x)Ys<U74IG1P`guH!UR$J~dVBi^(wM
zhE9;O<h_6xOKF1d9RXr-hcwg9s$#)|^TMDVM6IRdbOyxa^RB(2MoqTnqZj`h59iJ*
z{p6|RByNr)Zg;fc@G~FG5sLkLs7?yB_1V>lMlGD>;E<)!yY=F-_?o=~IPf5$7P}g+
zzH52kJ~FD;>giF?GqgXgE%g)d%|>AhZUHyl`A@Xg*8qpKx=1av$cQP^xdnCDu(LM}
zYtG#0&O5bge-})3`RI21gx~+jslb9de7rvIPP5r<J15Uo9T$fT<gi*R5tDKBRMDi*
zNo*d{+IlTiM93a{h`ICOMwSg*DvHQ(#_zYBm;uGKgx?>#F4aK;_wPLyrVB08-zYcL
z!p?H@RhB7sYiFV0DT+b*_a?u>ouooi)>-kncLex~faE3*JQYrpQ)AtSHek2v%u3XA
z8DWmR_05}1cNn;^tsu&AVBKW>?q#+smB~8@>_U^%3S^*VJfkMy;5SP+lX#78!PFiD
z!qZdr2sffe?f(8W<vP}gl{~9AbDy&R8!yvLjENr?X@~18KEFJ9bXY`ruapra3AZrE
zFFnq1z2<SR1WFAHH+h|E0#G&S3-Fho?9SV_QI>rry26g*q@_7PC+g_vaQt<Fkf??X
z>X^Xsu7^6T{b*tWyMRDjT7~5>RHj-&P8oH-)om)?MRk(KvN6^hv`b9qr~33Eb+fmW
zRQd!=J$rE9x(3nsMd^*BgR9Q(GVYOGsd_2Tm!cLyo{mHheR}4?(!`WKU8>jZxVAz+
z(%U;ge)jP`%S~NObo>qi)G6o6EtO2zA~T4W(3SB9n^+V4os=R&Y{rUlsVApGYUE)r
zykB7%tp%U0_Y0g<{+r(}6h1%*bsG+#XP-qv?H~N4OdT5lBA1Des^o$xjzHT&4x{@+
zQ=p~mJT;44Jro}2A&|F#O;hPtpWEycIRlA?D`7jZFYRh9AdazblxF>$)w1d^(s&Vk
zQ@uG2r@(9Ma~QaIyGK+$Z6iQEv%Ch*%)m%LHu+#g1ws|Z%!xH%yWXvCCmTomnWHS1
zrF%RnynII2XWC*r7B4V6JN7TmKQlY3An~z1en+V6RuYSeLH4fedDl2M_pZxujGRsM
z+x8JbD=K^Kxh)gsw;90gK!tK4$H>pc!U-|~SxNozZk+EYndr;5n~}xY!9bPMpzKii
zV5n^cniw#H!^2(XKOJjS2%u5XfChMXUC{m9N-fNobKF-%8QS+BNj(E*rO=b37HeIP
z-nhdi%&V}Mt+VO~TG8I4!2(Vm`&Lb}5%`@(K&iC@={zEV=K{nv+Mt6|=pX{r$1cMg
zJjHX3)6^tMYRLzKdY~VTODFFn`x8PU-b?-M?;3x$2{|p!o@jh289D18Il4Hn^mZk@
zYB@tb?o1n=D~2Y`5F>xj)^KTawiuFAp{W8M$o)<mv|$`>kUQc5B5Vc{VX@rqvlP~0
za44zG$LW`#xeg6PNV22MISo91d4kx0l4udY5kN^)xEPmzG(`VvU3Im}<;zcX%e!xh
z1B$cKK8~OhE?<eq`a06X$=D(LGirD-keuJ)4oFE=ZEenMzyY+P>I>($?~qU>O4Rir
zv-xJ^6bEzR@<kHT{n_boEQbkL>osfWfW*%caSu18y?T2qIk9QEG7~&Dle6Dg<LX$1
zhna=5n-mw11V6T`JI-3-)z=pF(-lL8e?)XXDn&G4OxkjDKUDRC^>M(fmKhITVKL+o
znxh{)xwM6(K{W5CG;}l#u9LbVmC!xWjcb{b<x8a{>!7MUiIoXrHwPBOb*M@7*xB7#
zg`heO`jR`$t%Ns2q6UGEGuNA?Xa%eu3k?FSR{vV*)p(Y=p7TY7!{{67h;+vyK$nu{
zPZW_{*?X|0wTw2pO%J?A1A5_!kbW=N>zisj^X(F9s7YippB(19gmjlGoT|v})hfXy
zgp`CO2!?R&CpQN`3i@D#fsWb$dF%}{05W?DASNF-a-^zc!}6uFhMtD)E!eE@IG5)a
zLEq+n(k1ta!*tSG|Bi}^?L}kCSP3Qg@N<Enk7D{_=+uLUB->%A`Ei>T$w}SAo<**D
zNw#5*&t0B4lyt^h$0pJUBPU@P#jc}5S>@D(HD6Yt6F`0GlYQ<a5abXu=_Hi|m?1>8
zr!701#ke~UL#ENp)4`)(qGQ>arF8Y`Ri$UI)o16t;6zZAJnEpT>Qb}=3!$q%!n1IH
znt!zfcp*sK1AQAWpm5)ziIm-&0gqdOEefc15+IRtS9cB&3((hy+(-U4L&2y06qXWH
z{C&t(PhxjL#Bk+8qJJT>HzA3iun}MnN(xpPeC-lw=zmanb|lMja)h=`X6zt5&Jj9-
z(sa=~$E9WQb7dnYMZT1=!iKVBjJ`&KhhX|$2};;z*QgpW_=&dzyPvGtibePAmtTgi
zvvHqW)2$vvSYL4z^HwElJdYL@)`kRRw>Urgtt7PmhH*o4j%7~)Kd)d>ANb5p@BYc9
zmG!X7Zsp@LqWpvu*i|)Ws~6(ketsc31}x#rP(Njlin~RCg6J+Efr?+x)-+K7qv+l1
zte1q;H5mr)P)BVm87@kHwa^E>kA<6z!hk{KK+v>T%^u8Xzb23)NEW+j!yQl@8G(kq
ztVf=pI3fgDQ?~7(j4+q&@E*^lJD|9+{BiCfSCsk%CC9x|JXjgc#sl<H9cqy)^Y8N;
zbKqo--JT-&Xb_1u6eVDoRH;OQ{{x)wMII7Pyt6KVeOSn;|Kn_IYI5YP)J60cVv=2l
zVsN18!O>6+2jn%F=Dej35#!<e*7Z{WC4u*<*30p4A0M@(Y70#C6y{#gv$9Gn!L7R@
zPM+-60MUerIKKrjFagpyq=PKz*F_vuhA#wh2t9QA;0mSbY_K!rzg8dfBXBy|*}n`V
zw~V7GGmI)I`OO<Im_<4~2To|`J`-@gK2A-i6qX94jIU#@phz-$dcX{|6C5gct^xka
z`|c2Q6tw_}Y90pgLJ*gS4Ul@T&;4jn`ply7fe5n-I%JgLp#@sGR;dv2uy;pSzGwOI
z3LuMT(&NYd1W^v*vV?k$lT#(4jj;iw<o2x$e?zQeP!NB=#<+3gCCdIBXb61?4*U?f
zC=`=zH!%Fv|9SDZjUAg?F;SU0RKKWnri!Dwxd}e=p7kUS*d{u4u-$|JXZ#doM2DT7
zoxjqK9k;gir9m5vf+cngbhv8{e*eBdop<sl40~F*Rlx}>!2}dbdp~S6_EolW!eIZI
z?U=g46QZ7|d=a2Py|X{wzqx(Lo8{2gyVRHK0m?Y~YQH9AqnL0H<;A_VM^`(Roqkp^
z|HsLKMJ5YcEcntN5c&3s+ky2WkFbBxoKtEyQ0l7gvgo^!;xV9*3_S|0=35d~Sic_Q
zffI67c6I0k;m;gMVMl<K|5RNL5TrJsx1~V1ngn?T1Z%MIeg)Y2ZuCPNXlCvl{?ti!
z?_3zKi4T+=VqrV~$UYht9269XH2E!qjsV+i`Q2OEMG&=?IDabx3jS;GEUiectrbvk
z$%Eu~9vbc1%^^m)iAoS>%NFU$?~+ZBnO)y_4gd0{HvW2m4daC&{$^*sy<Pl%RQQhc
zHRh2@9N#5j0gox}k^Ib9s4o73l$HTC6z?*78VkX<?ohl_t|hP@3qWZKiP=JF$Ekae
zhCjsKn^%G4q(GUZ6@=A@{tOUvqNdaP#34#q{y@KOKqGw`Qx>5Jh4~(+@d!aJy(Lzf
zy6|abBqSx*LGh;m8nWl3ajUGn$u%pmK{tTMy*<b(B!gU15<m;y1&)*5i3s`$@CVPc
z;lTPrC+i;DYOc}VVpp=vmos2&+5|u5q`rGK>POjq%qQ~i8n_{D_b#7|#38xqTFb^H
z&HwwU@V(Wa+fyOwyKU3#__M~DHo{K9uqPN>;v=W8QBm$YDKH34=8DL=tP0`)SV;dK
z4#<_-X%u10fZC_K%#S8Q`v&hkY%X#TkvvKgU}dck<7Q*aSfuVrR88~;m@65m9rstR
zj%tG;avsl63lZ6^Y_L=p>=tNJpEbS>tw_~GI|JkB<DY=0-|oeaj~^V4E-EUr5l7lg
zz<}gqvXvqa>fOGxFLRrXV(r4u-)qcFkanGkn3ewPcKE+8@PlWEr<60PBZM=4%9?kg
zkz{4Fm{o>D4T~M;zcJNaoBb$OAl(U~)_=^Vt(q5>{2&?=b9<*(_1}tyP%*<0Yeh0g
zE_=BSB<ONmu2zXgZ`oB*O($SJI5y_j735mBMmGB7_RN0%YlZKBe~8L7M%|g`f!fvi
z)EuT#nd{$?jcRj+EUEW5uf$z36VA0(h?*-3%5}=M6HENt>LFFlK*BhY9Xfbdx9y;1
zb3LIr>nF}6_Uss@yh|RJNFyDtes(wjAM>nJzwVP6#VI5zKey{DaZw5r?-caKGdH%z
zYHR%5&;Uv9A;Gfzhw2!Hc=Sv#Ns%#suUqqIMyfVv@y*opB+^^=+%JpOGm49k)D1i6
zuP374S<uNGZT!~-&#ni6pAS8u)*q#}jeLFUi^RdRvsV*;otP925!BHVt@z{3Jn(WB
z7nmBe3g7tS#$ShPNdv-O`aP8Lf0el3f!#scY|T86OPHs^at?o*?BpzVdtt2Cu{Ee=
z(Fl$?T#OD?PvBBzlP{L)EP8*UZ6)j2@t*Wk!k3~}G{65bKK-0!olPD0wgwc%1RI?f
z$zPZl@Mud?UOZK9rw@r;lUdxW9PTl>TQd_AcB-aoPkt~1K4WE<DqF6~8XjnTr`wZW
zI)1=>l5No~-7Zb*o5#JMk=?VEf6T;~EQt1Hx0SEmSmE_e-T%AtHPJcv@FLpy&YgvA
z#g<Q8^)3#D=+l<igaxYf6d#cNBpoehQ^&e}7nfoNs1lo6Cfm5%PZZl&eFTr!?1Yu2
zKmpc=7IiZVp|kiX5vu|BU^L>lS(V11iAbqP%TG2);a_~&P9(v&BN0P}pC%kg>}YlN
z_I20hi;dp7<&MAl#YV=|je;5QOb}&O8)N;E@8Tazdf!XN@~j^&eB72Vp-b;P;s1E~
zjr@H^2Ylu$M3x~Ylh6>$vvKjm>@fQGDMfm=4^<fiwTojJAlkYeDLcT2dUx9xSf=`F
z*ZnGNPI*))|60*y_>B4djYVGki^W_g3S0E6J;bF}Wn0?Yd9g8EK;sv~B5z_P+4wTs
zfwO}}Bsqgb-|g{N6^5&t3&h%Q*y^HZVGJsZP9;>}XD6nEXbJuLhE)Wf<ut;9B;ap$
zzqOnvX>{h1QX9)(2|;7?_e?VpI5F66&ctcGG7-U;mB=@D2xoGoc=XvOy1%=-e$!>B
z9D8-kJ^6>*4GtIJQQ@}}dB7w4F5w+s6|V#GB_sa{Kk6_p<kJ$0e-4g1+;rOM!CKOS
z35K57%wtd7KfI98;=R0V{4!hRU8$7LB*CjK3zpkV2JWJrKV@J>yrH#3E%%maE?@N+
zw@s|ty}Zn(cG)DKGRH(|0Kb1qnZcz;L<;6M3|`Uak@4`u#)of@M?Ro5ue<L;db)Y{
z<}Ao~5XkTB${Chr?K#$W?F-6$LCU;sXvyRRUS52V`t=q4(NoGoc^0cKLu+bU%gU>^
ziR{lP$6l#w1}zA+Ki{@lM*B?yvfk)^cZ44K01OE!721#BjvUSQtc@jw(NI2w_}grZ
zcB2#m=iUkB&Z*GnI4XR$B!G-EWW_7X<?`y=K>UgFmh80n#Ziw0<J&~vETb>`iyfg&
zSG_*lWa1vEm0g~8-_r1MARddT*=yh)C}<?Fu<?<NS767>I;thK4azWS0v6%FOU7Yq
zP^fNg%*?Ao9gg%2*(1RrEd?JNVfKIF*vFxO_b+(JqlmK^DG!HTb|9Fa`FXJ+@#GVk
zlpE}_aZSr?K~~;66kQhr+3<CfP^L2QwYn5uvXQUEf?*KKTqwgHPkEz~D%e3!#2gt&
zzBMf9ei*eP+A{l0`8jFSrPHcxsczcKGsNZt>b-~YZImP2^%;@P>*q!?GGk>lz2{qf
z6)4%RI~Sz-`YaDVP`|Nuk~|A99lQ!RP^s`rV#~XlEdcJo#BhKKzd4`@VIr^*?u_K#
zI->!wZIwLbTon}hG-rZDN*Cw1wA+WoW+1q+F*r^Q2@D4r2e%Ft-gD@Lg640ki4)Gs
z9~QiwI*5LJ@vL;|b3gIZPFa`tD3svN*cTqlvEq%zUp<~YL#(R#Swi2&pqnF`^^x5y
zy4zc-$btSlXa(m)x~*A(VngPogsghk`2uOtOEPL>LA6#8qCWW52ye$z{F%qEtGECX
znQdcNDRF{XV=tixQu0DF^1`x1#b*k&MwUZMFNTTQ;#@69YUM0P>~-nmt~C6*-Y!(=
zhd-o4VMAcEtB8u)xYO7t$eZE|w8zg*s%Z_FEg~fGc(Y2<2n4yh&lzc%ERb=VXfi0w
z;Y49yuN#L{zv~I>;;c1E^ET5Hp2=ClbuNu%PuCP^Pns-DJ||z4S|LwqD_+5!TS-wL
zQ}_R3!DgKOZaiGwf}(W2pj7-GZZoy@V<H&i_$)JIAq_v!5?BatlL?CB%(T|aT*Rr>
zzR|NwrCm$b@Kj@-RrQKEq8y}k#ZOl^QOiBi^mt;c#1)&e7c14{>G^Xw-NH(8%B9RK
zJC%_-`T4^mMPf49CO^lnYU|uJ+L{HG_VCU_#Rz|hZ>-fLq#+95mOioWCh_OqK4+s4
z*nYTN2cImVHa84eSM4#Qj#nfyk@qJ{M;e5suj6-qR*;Ts^uO6igL`XBIHP9XneH1w
z!hX@bJ)ShI8h?5iAh@aN>uYSAyO0&6WO<+c@Xfzqc7~iD4aexVE8(y@!U;nSEj$Q1
zh{))i)8?xI=wQv<`ngvwiec=tpuR#EJ)J>R8Fg&x&7F&_R!cRaz7b2P(6<YZc_g-x
zI)l<377C`Q%}>VCpfTv_^HMvvVh{vP0i4b!CLd3?Bf7M4d?(3iyS7@J5R>a1=Lb7i
ze)^8L_yU$KaY_PF%6b|3u3IvlMU44sD%#V;yp|J(SNFc!iQjp*qbw>(*pSX*jDN?*
ztL$YoCUU>MF|n#-5wik|9L3IeT=nYhAoA<n?MFFvVz8Cm#roRDyV`*|z>~qMu<Uq$
zuH{i<Ys-PYV_N+RSUFwSWxLaV@u*>vfDdItE8~@@(SCa&IC#GIgB&Y<V<sSSL;LA|
zbO|e)iaqWkD=)0Q=RoC*#H(vtBSo;6^Z)=>veo*zQAegrvYebSM=taI{@?fkgas`1
z6jssao3VJuf`7)kP)z-Wf|{_rZ|-`5)b1|Xc`d0Ry8zFl*SAUwuTd)tQrcIfX-l(E
zgko9iV*WJ&gzbM^|M&GO72&YuweKG}v-vvzb+m~T%S7a3S|wI+*Uvmp;`d8x&0Zgs
z0{R8r7*FBodTCW_04e+MIIg-X2OL~A$xF2yLR{V}uw;larWuQ|<m60`rLU~0Eb(k!
zUV0ztL^`W+ahQ0h`dIM#qjmB{t(6s~*@E=`Hu14w+~Uv_j-&l*&z$wzAkJ{jGRV==
zKgZf{rgaL(H=nURng4N@7$jz9*&5`3{A7E|D3W72xJyNn=DwRnKM@+jFU5Z!5v<QX
z$h=;D&*%R8aBM6;B4wBh7)shF_!Wyk6Y$sNJqN$=_>ig4@8TUkT=;!|pW&0xn*PoB
z6Yuv=w!ZVb0k6iNOUC`*e)9Wbeyy<ZW9O$E|D%a&J^TWZ%hLa7qI!!#F;3>dL~n7|
zn^7o_8Mb`YIOYAHKQ#~`J=mgbx1C9T-_Q1UvC)_ZMo*QXd@>hZC*HO&(e+B`5Dz8>
zdXR*Me!R*C_|APH8h0VGn&3Z=v>sGJ`s27Aey{ZHT@`<K7&Ni1ogIIF(X48|tx2jz
zJ}<js85bso?PN^L)d%K%CGIXi2ZOT#Ou{Tg<0}W-^J4pUiI7J!cuJf4pI#|4ReXt`
zPSi`l!ZYRI<jf)@uAS-=AFbk%k38aG=tZ`xvZ{)FIFxm#+j57Q(%{+OYwTaYE3<$X
zTXZnq<6q~^AM=qk7So_YWNT|Xz7USVzC?SyEdORsOiT<GeWVZYww?ca;T_mO7%@L(
zEAH)B?cWA$2TEe!L$E`AIWaUs;4Py>NkFvkVBUAG#g}!_6H$`CIXyd2aR&@0G__7u
zZ4Q6GFx<790MqG>OaVXDw{96t1-s$FhgV-dgBOEwCXR?=q)NUVb&N{(0W=-3WtOSQ
zb}Jn1`z-2<544ecm*FVu%*?{mpUa$JsxDbc{hcm}-XO4Dz3y7#+9RuPTX!7E_ok&{
zPH(tx9@~v-Ve}h;e6Bd00KRP4m~IRYY5We0y5DaWjVw*lU5>xET|<QNwTtd8{z;A#
zBTTug6|KS>rmnnXOv{K*sUnNs<#Ri*7vS7weyN(n+y4B^lT7^oaR-u5+*EtxrWfjC
zDA1sd?uR|y3`k&-4esy16Z6N4iH+Qdf%a=pZFr$par#wm_^pOI3j&|!8)Sw)tf~uc
zH>#~FLbqnRUlfuS*y)UH_@`kmk->{r`XFzHe-NxKk`@R~pT#3ejmk(`@WCh5X~W>t
z<lw`_jps1cx_sfk^&HGpV;k<9I$0555%d$r>LWF~t5}Xn?f|}&DW!lUSb}MD8=kJ{
zS4e{O`fGgV_JS5f5>C2X5%~Dh`~(unUj4ZBK5WVIHjGLG9|Z+~{pw!0nm1bzn4y~3
z4dBhyr1X)`5YCy~Xu?1nA)gWQBL3Cy&p?v62(JiyykyRbd<MhzXHcBo{EYA*aM(mg
zLE-+^Ig`4IoN>np{FXifx5Xw}5u~?qC>aOBfqj_2+kV@1j_m;?C_#k2cndZ(*9H+R
zrQK!2##hk_`)y<94zk2SrYdy-Xl0BtCLDJxJG#%}vz$hC_}As9T%sFjP~{=ys5X5)
zvK8||+2%6P?$}w;30oT^rW0n)jFr@z`PvO{+7qZ}8=oP~5VsE;7~vt=q7*x3-*6Qk
zHQSn-7#GMZWa@6lsc#I<VP?oHF6|bG+(tgD7$A@wEE~YXZc?%^w!OPKavE}i-o|R}
zoWLbFNF{lAsb;6mt4mg;G7M^XcBnz9oyMqE+kp!X;!;cEyr-WkBqutg&|2|6s7CUG
zX<N~H0sZBEaZ<G3Lrj{ZMcJg;`-}Q1yi%^lYh6a?cVG=s3@41&Lb+D8@^Jjdda0%F
zi(AJ^*3wtVW2gJ9%&phgOK%@-jG9nhKQm^XoHc7^Ug4|UyS(NZ|5UCMA>4Itjq}D1
zPlcpr@8|@yHcK*?yI#MBhT9Z|7?nG=pA#BU^5Bgfp!s@eD}&<IMHUbry2;YcLi&$b
zOG99F*kxG$3yePt)@YHNJbKZ5fy_iGIsmRgw-I`k1f20;PF<;AUa2H!spOEnM{t(E
zsZCOka&L~_sItJ;iASV_a0w-!oY{PRRLI!x-}y(tn+9CM?=t7{wM*mI8>~t(YRk-2
zeR}KO^FrSiuNYv=#w6Fi@T@^7?-fm4qe4{ksfln^L?(;NhzNW$<zmWa+TvO=Q(39T
zaf_i>*6PaVm6AxPx>Tj@@Z>yFADH1WkM~b(#*bxcUONcXkMRCJj$Xo=Wb?U0h(E9}
zNH#$fYOI}`-)!Y&e@o%Bh@jU7VgkE*6{|rM)AYoykz|^<)Z}XQ;ON?8B7MdD0R{QP
z&#3j$0j*I#`M2$4Nu^RmL${i`h!AkKY!AcHjh!3$qRa%5{fXN{yjx3`>d*!P_nP<W
zgl1__^T?60L;_arr$a_sXSVK5Dh*Q!A^6Tl{{#Ra{I&VB<AO0PDWXjl<v;%2EU_Hf
zswt^D@H+#-Oq;%VSOt#)!0J~+jT9}UT<Ig<b%_0_bJ_tKzaVy{w9TjuGei!sb$Ws(
z7Bvv78_w5;jVh#wo$YKYZo}hozhe!cpTVB}>5gn0eqgb&2&F+;FmbzC41>O3)<9Ee
z<{|b`w60CCRq~DG?rwTIKb4*~uDF|<cNrUpc~`q8fzn^C9*vr}CpbgoL)2(Hq1h~`
zhd&IqW-#16H54N&%as!9@K1=*go8AN^LoS|ai^DRF}8U?FHtl2t;P74V2@vVRoXn3
zxSKd0^ssf1<FGWbaEka?yrhchlvSdQZ&rGlOHUz=C#%jS+e5UY=zh)aXQsF|+T$P6
zyO}f>uwOU#dNYFA1F_j^K5p4j5dkJqQL|$faaFBW3y?GjwpyeG>V8~KW)C$e*hs>a
zTMCr<$BMH|&qETf9C>~*m`p!r7f!QKDs0Wnd^nyrqUHqt5q=LTe7B?zlNIN?m+Lo?
zYJ2v;IAdeT=^DR~Qywz!>DEZM7i}+;UA|h91w6Grg_M3T`jqhHRN9p-0KL*4D|5w5
zBJzAJ%h9;_E7OtA>HHGJaUC<c8WO1Ei=}PPheWEj5A{E9hYGfTBGgH#)0=eI?eWKl
zNhr4Z1-HbmR7knQ^<^*HbS=eY&6$|Dq^v0rOHH5l+01PG-k^re3Aru@)vQ>3aP;O2
zDJ%T7M{}x>huMR2n+NlcYm>smqX-T$g>8}s|1mO1=tn?(c3o?v{hc>|XLn5AjO9o*
zP3#kWW=BfA*Jn(u7a7$xaYdo=nsUN9u}CKO(>cbvv40600aHM7MU;@uY1)6>!}f5y
zvDdrbZI0JBe4?GJp|$id%lV?=;0Zi%{F8W;o;&hZW^<O^>f8A8x9YL^ul8}s;_<=l
zt86R|H*Tm0)a`sY^dC|_FQR?!wq<EwTAw`JBb7W%^xE$9X554I+5$RNQ$1Gi&oW_`
zF5xlIf7~?bT^Km*A(kWf^Zd`Z2?JL`bIrBHzH#Dbc)lt2v}-KjruYQ9Oq+n3uHGot
z-llD-Ep$=cd_?wVwbF(?A^vsa7dQbuEcyO{cQ4HU<?>s9rJ}+DI%g)zu2$vK)6YB*
ztH7dVa_&v_GkpJ-^!|*JeG|9hnny<49qyCI<yt@Y2%#^e%m3^3{`cyUsx>-t@wRGq
zl&jH~-4E>O{YpGqigyWaU9CU0?3sJqq|+zLAJu(~P+H2e3&^#6;fUY%|M7StkiV84
z*GT%~1pVXZSUIW;F+)+L5XwCn$^VXEs+%P>YC`WmKK_?_f|)(mc=(mkPO=nTCO~Rh
zjzuj6gh_m5|J$Q|-Vgh9Xjn6TGX2Z%gfiP~7Gx%8cP{???$v(o`{pwB(;|R55P;ji
z5=KG)tYpb-oOkOd8-c;^Fa76{?B*=n>AyAr5)h%#im4^`Ur(P%C~!3jejFec+8?EW
zPnJ|4V=5Sw6Owz<bu(04zUlyg2?dv$M9+|wmNV)0o4Lx?1jja#XDo209va--tEVGj
z9+uGVJMy<KTPjjG?I_AmI(Jr19*;e<*3I(<CZkbE5WOwWdH8VE5Dcg~W;={snE<ZM
z`%b^d6hM5{1;sj~>Be8_r~}$h0@d0UD@q?sIXT)pY?3ZzCz(H-z&o78JgH9m|C*0+
zQhoTYd3-F_e;KEe6gcQcHf!;LNB>8({!vFO=0sBCSPUqmAdClusRPNY3lMJXL5rS)
zhlfY;7*I<afXo9H3^XTzrwa6hHbh}>Q<Zn;lZ!Fuh-;d>YF#CG(=ThkRt$4_mgRg;
z#p-&KhHHFlc=Gnn5DH~Xz5vPSl8rDb)*A-YoY&8u@E_}abM(<4Hz<Qkp?eD3MD6li
z_1Nv?Xv3qwd0cF&ZL#Y{YUNn+y$JRDO1fSubiPUI_H|zqH?H)1eY%t$(fvV4rK)@s
z6cjBWtInlgBElXCQPD#NgWfu$RMDE1eEL>;C@StU6wEYyQcC}+B%)^kUu(m}G$TdN
z?aJ1Yyw?tb(kG-7+aUf1km+&sa3_AAA7F;~r}y(pPmYzHd711L+?R@E$}Rx#Y1@R1
z{b0@<;Iip7PZ$BcPOgT>z@tb6f*5n-TEsN)OMXD#5KlvSa(amt$b<4cy9^jvIn@vT
zE7v<a;xJM!Ki$m7t_QE~XI_C04@MFN?mMZhDXRFqivZ~5`}^O$8rfWH<AC#eh5lYJ
zej!K!X6OQb|E{D%@`5MXeU^niEZMaE-8jV;r{@{XXUT`%fd2q56Z*8{p^G`_+IcEM
zB0bj${`twC!VCJORBRIGP^gXLVLsaM<GP7p@Y?7_^Yv@PMXGT~^9lP-p0qouG)c|)
zy;=gfweQfrxp`}MA;vLa{C#9Gu|Ue=EvU4SVZxroD~Sa1MiXXT<Lr&h*3$Xe5i6yJ
z_fL;0Z17sVk}=R&8W@y~`_%pZ{W)k5%zw_Ma0z&7Kt)L=+#_-_DGd6Ilz}Mat{T?2
z=RW1pB&3NqDHP~`SD-N|%XYBpwLK+7TLf5@sbH<h2^|~Xd0gypYEf}v<gCiG(cyU|
z_T{NT{)~C$;DW6n4gC=5pgDVe%c>O11{=`Jvwy%wHb_P1cNmU7zmY&rd*tv7P=?Wx
z_T-oc%hHzGRx=+ZAkrZ!0a;B@8WD@oR#c3E-a8BE>gA5B4tA?KRZ&Sq1Ugk0p}Q<m
zCE@-Z!Xk1RHhl}DXm?<PU5EY!XhKN@6S5cLt6%pZrcy@H7Xbe}OL6)JYvf9bU6gdB
zLqgU!OQe{;S|m18Fs0Y#o7(oi8E-@>K@IOCYcJk(G>{4~R3b96-<$MPB$25+$GDaF
z7;li5N&4z?hF5j3JKnbI6Yw}8PrJ!5Dc}DPxc9`+<M{&o2r5Jpib4ei1%V9|T|-TA
zyF?9$a&+TVa~|~`MC>~K-Da`(SMCJKWSLAbj(j0^YkU~pCdr(EdhTUeM)G)QD_HXy
zhOri59oqW*TU-!b{Bi%Jv5#Of$5l%;ZWtfK#v%0_G9qK?=P4EGIr-VK)qMf%CP6nc
zr9(xPwbly|3O<@jC590?G|oJPXtz=h3y8nyC80%8V`6Gi*RU+sI>Ej%)zUQIqDd$A
z(TXz%s&w!h(ru))DfQ~&mTR~T9BpOK?p@eEWU>2Tv{Y~3#TWi7ZIG2EzoWD>=d7BL
zQI=il;-r6+uq=D~NQ44Va>|jjk!246MSkqqh|$J4YuS#Li1ei`=9XT<vV;3~hf)<K
z@tj1(p(9&wg^CHTDshCXb923Wy@GA?DJRqK=r)iFFspq`Ok3*;bAO{90kzb!5Gx#;
zvlh9s)>GJ6=xM*fVfu#f)q@|x_`*Fj(FbZpO}b^rsl+*Z-mIM!YDphEp1u<40|bbB
zylwb{5Dj1jdE>Q)E1OSF2dmh|nBjK-9dj2!P|ytcT#Q!-S;o|5={alhQagbFyx6VO
z_*Ojvn5bpPu_-B>**7|rC%SUmK+5nyZy!iOf<WKlb);Ds1dp|nlAS=F7<Bc>qBG5#
zSlmJF=j$4f#1nz7ECfQZpjgYN2B#<V)zt%$e-6<h{5ocAX7-><t4IkL{{EE0<0pml
z!6s0v)T7v4$IT4ntav4e-{b)CQV%+_AFGwuhKsu9oO|KtPG?b(7Y)`$j?aK}W)9|B
zS8=l3`4?cLQP$Io5?h@<FAu_ZD+?2u)lqh}f?w^Pd|mAGOlOvTehoUp*a2Uz-rt8<
zCW1v;$hXf=D=b=58ld-tZq|@FY=I6)t#5qFzjQ%wc1;Awgd=bxN1<hx5(h1&TBd0T
zT@G|58=={M|Dbwj!3keXC{gEJK!LA~*!*yMoI3aa|JUM+;6d+ZW2^FO*F1iTz|n#b
z-=;C3jm%w7tswFOR4)r!ODwz2JUJ?S1##L0f$1MwX>6~+A!8Z*Cf^HO3(o>Zy$bN9
z8)5WaMc0=Gt^-3@so`~7q_EQ{L<bLmCCqGh$0cO~$a_slqf)ZeIcI1;esx5^YFsM^
z+)2X(>Xrq<`>!AsjgCy#7mX}u4M8K*u>)wP=Cu)mE+7{W6i&N_Benam3T{&D)T$?;
zoqoE+{ad;)Xb#kaB;!1I((SJ!r!R%}30?wf?;EBm*7<|aFZf&vI=o58ee*i2@mPq#
zUg`}0AEvjJz-f|>a+i&8Ma@-e<D0Wz1d!$$tB+EW(ABU>8&0!}p2vAhvr028#vB`_
z4)7bTm{;0YF!+c)WB%jGdeW`sKm80a9{SxR@|!V1%niWA1AimUJFqp>uq<n6?OgC4
z>~G*$__%OFA&uVBLLh)UVyHX0_*dii0FEJ3iQ$9K!akvP9<^6oipN2>AxpQc-M}x9
zWIVQ>y{y3+30*y3X?q@3h3vrQ_PTJ;(HVHOi1wP^Cah5*LD-RPFNaK*A%5uO7jJkB
z1oole<-S`3Pc9^@IHQP6LAJ|--7E2fxhqV+W>A&Vg;?%#$JZ<{nfU>|R1P&bXdP@W
zc6F*K`sOV`N_A3VKbU{zBErL=tTIe*Ls`k@#@ZAt+S24>&zexNMSGEv7kwi<QJ7>W
z$`irye^`6(c&h*Rf800<l`SLVgfhyAWOK@<Y-yN9qEJ>w#;KH<5R$#JiIQxU?3KMj
zviBZY-|H#r_4#~X@6Ydd`~5zDbW=K<^L&hJ+^_p}zbw++^T#Te1Kz34X=ESid-d(c
z{vurlOORL6x;K`b81P*MT>j8K!-9)yz5tYi<{Px*s6B(vh9GSeXMD7W<$K;lChFhx
z5nSr{a;aZIlXtf#nGN{uDfPUY%yzBniJ5lL79=|TzGVp(svzpRIK~iI=s(!0hK7bd
zMrn$F=`C~lOvGLlG5J36rn|eQ6l=@E;-bl67y-i&#*k~}%2^6MSfIV;qa<`Ng^-N)
z>}|ERj!k9VA}XSjd_yp{#mBh<QF?@VuNL=fp$={I<v^}kiyRzY?x?S4@TzIXq-4#T
zy7T*lGK1T>?Hd;Pm+PK=zY{IeyIYQT)pf405<KxI$(%)9asG`eBjH(s0)S{#=WXcr
z9SV95sm-h%Pr%<Z{V$9mtx)E_En^42>B9_br1b+CR#tnYf6aD#c-B*6ibc&5*G4lp
zKVQ_kC;R5JnbIBC`baSySeHZn#;!1^ZydtQ)A!%4fJFky{yWr1kIG#(nx!Endn=G0
z2|Bam$o&<@7(x91!=QY&>s`{X)mi4Nr;Ds=zQYu-6U~<%5m^cY$ys?w0>()rP3}`m
zlbMAoQXsDnrgcPBoM~SRz!a{M69}n#v-ecOu#YB?y1mOx(|;B0I?P78$sRgNg!hI*
zfCSpxZmw}20qa5RuXs6R`P2}353v31>IFM)g8?oW^gi`@m^J4MmKs{(jEprL`W+x(
z&X6fkq~$Yo2jVr8SXR0!8K0^rpDL%q63rIO#%-Nrc*lEp8WewoEni^i+$e`A92sv4
zf!cZft25Hb&>w$g=wUevDwB3&O~)4~kfxr`s?45BLpZp+sD_6M=G$VUuSa`Nxd?oR
zwjM*`wfoa_7ZvxA=zroh|L@v*1}bw+pPfo*=n$cFTYvQ4&=-kC*c1_YAP`Z{(kgbc
zE3AN_2)fXm>ALpRMr^(#64U_3;V8F*VT;eXxDH(?B3v0`P$eCJF2Ifxp}KJ(&PGF>
zYrqSNt#fwBREH#(ZY02fnp<p?Li*?Sg3CT0h=|3Ii4T%0E6<4>lEFXeSvcaOybGVh
zxM2|P&Ylz!E8X}$i!|YhO@oAKQ2w`)+aS=tcosZw+?OZ%^@4S|-XC(HZ!9vpDvhXU
z4fJ~K2CxUazg{|q2+eMEnpL8X!-Qi4q<=rsB1-gLgUIMuxK%oPHdRmEtBVoB>NEzM
zJ(Sn#D38K;Dsqx5oR=e{QGutL+K4yrxM6J~eWG7VhEGMvoAVKDeq1^HhV5Ufy3s(Z
z8wkS2(Cr7Q?z2eUZ&g|be;-7}z9&z1d*!TyWbljpqff#U<J}4s1t&C^{emSUse;Ra
zqH**iw9=k6Z{;ZLMw)~X7c3=MrgHj2P*4zMNp%SvF#G50!<$vd-cKLCQ^|=LsAP6)
zg;q2ko0}hcZdo2?7P8dUT3PNe-~~F8Odk3mA<XiUJL%54P<{9$ts!)u*{_yvF4WGo
zP)?!lB(2qNjiekJ89$WR<=yea&fPuVtY1815Y{j+JRfEm&9xk?3p#Pb;ue!DbTX>T
zLZ4#Vc*}P;ukWa+SLjF3G|>qqYBTYr7t50~cL^Oh?@bwwuic5vnp>wb`xX!R&;iXd
ze<+wLHJk6-k3Xo+TmYuZ{E!<h-@#Y-b+{uqHtr&242C4-QTWTGhpMMZ3p?%C2EM>|
z{=5LaW)o9c^#T~cn;mZq<~AxBmbYjnp@|{3Y8qUxQgS_X==q3q8-FinOx4ZL`I&#&
z9iw&C<{oHSiH7)X>VIlAm8eHg^unr%&o~CyUBSguAB6!wAGAj0+k>l0U^qzf<nYMI
z$%hh?oFwX_4X@6`m*+$H<IA;}<+hb36DH6oE|6LWCRXXCR|m~03_A5WjPXrqwJR@_
zGu{ULAuclyA_FxDmnaI}P#-3<n|VZVh(dH0q;dR38=X=y7e-}2gu9d+JiW}oH#==P
zyCJb%$6)*&dGtRI`g<k*@y~%(k+Nrx61UY}C{C{`UfOsS|Bn^>$LscgOLZi^spODr
zykV_Cpfh&`iRv?vIcVUK`OfA7GO27sFN1@rYZ@E<MOT}oN$tctI5B0v2p&}Ce3jB+
z7O^&N6-+nqP#r6Zb9%&p<}+(fpm32GaL11mPd8ZMRUd&qA5X5It2nrmisb6kxluRB
zMWHKHc`5DoMW52=v}af*HbItvFV+UKS$ey^sjhdFqFHOpA|o4R8uhq~eKSYVhV3Yl
zo#>N(A{GDhbBBtkjdwh~!u#+1b|jg2j#pd(3`4P7W)*(TuL+PG`~?bk@}wKms|~%S
z@9x}ze6&5wz>2xD`)Yx03IxPAX|gRRcBe=)(@eu5K&HHYi-ktan3I=YcUETGgSck0
z+xB9#J^7>;p6JM{n^Qj>#4fwLL*8gb0|7?l#lqt5baujI=nk`A2IwhZ*_L2Kxc&Ju
z3Kzh$FFZH4;)>JRxjQRwRT7gmfrylG($&(bEa)EU`TZ@BOqzZV?Ny5HK{Qi8W$fXX
z-G<&gd-YRf=kt24eOw~a8Cxz32qC{jyANEu6GTPGr_g+J5g;x-W12$f|1puE`yy%I
zV+K+-Ad+*#Gz)I~GX`r<mpT%>C_;u_?TPg#Hf_KXv(laX+i>w0+d%#%3Hf!|!N(ZN
zKM}xxxvFKOc)z6&EQf3-D+czo4=8p;9g2#?3s2hE6eud&Izyu0mZZvDxVE#|xPxRJ
zEBC*t%;R9em85`Pt8CX912Maa1c=nDc;4OZ;?IRKpOVk>q}|<zI&iQ^yB_2{Aw)`?
z-N<a%NVn~E*=Iz>B@i-CK+nwulbh6##%K-K1mC1SPKsSPban>495$fa=pU_|MS_G~
z{{qlK^8;d-|G~Nci#YB6ZNLV*-Od{?1gaGQe^o2k&c`^oW6WH;`gQv@l{QX<URhk~
z;Qoe#g>X~HG~&M8H?uPa^;tXN;hR3^@KDV^iSR%R0FgyDekAXGPT&KKcJd;=##!UD
z{5l<a$NZFn>pHH3edM@T7NAi&8p-?CEdT6sUXqb84a>iNZmc&YfF6TW`TTn!=YQQY
zLX@mwnO4e7Y(s|2w_nuL8EHt`3L3Ou@lh}YS03qx{PCKIj6)Fu!IN!S2K^1m8j0R3
zQ(ZUV5~S!C7u0Docz=KiffF^SGT6Oj?jB@#!Gw$FFR}6p@~T^_b>h`=(1!i%*JSfS
zS#q?YKn^7eTfm)w%PW#+JGt8IvxRhQ{xRi$LS<-?J&4e549WIsiu(?6KQ>v@H5{;o
zqFjUD!`Q6%7LmIF$N3o8|Lzi(LOsp%=dqA3zu9fA&bCLAf@L;$5ZKTG4FYp7&AP1e
z#xQ|Ol^qzyX9r<3a#^7(G<AJ(v`5zA5oQh_g^aV4*m=0bo4g7_3t2@A-VzOoU1w}E
zVSdL7C+IZ12n#pqa)s99oQMh$LN*??!W3Jh>K)amr>d^=ZcXOpM>r(D{MY&)S`Z#C
zp&~5pzir3fKQS)|c*2frOEu*m)NBZ_zl(qwST>?pIJlG?YVVN=G!WvQ7WG{(03j4X
ztG-^I7U*7df%dw+y8@m@W~;_kT_mXk<W9^MhN|t`k#1KQkh3A&*8l=hu|R$6`3g{S
z5@;?X8XP}}@UmrL{0J@gmA9A?Ga}g@5c)yHUuYOZe%VcRW*5MCyQxLXg7K3`0=#lY
zjBrgUMoxNsP$)9|{UZDLdK6+Z|J%LzeL3)vA_gDf*?q|5zZr;!Jomc9G}@eCD`zf3
zkz!XyBM>uxYz*#u2E@;-meV5#X2D2+GM`87(+kJjV2lqU>~i079tT%51|;XYKb~>L
zse{H!TD!$H*ybb{fSFz{*h}HP>W_mflrpw-T>9y*VH||P2dN-C_f?Xo7U_Y04<h~m
z(KJhLL0pJT3JadoJT*nhYX6j!ITKRGPEP2N^dY82y1YSf4^d<)5dXpA=CkB4u-T;4
z{clg~4&ZH7%@0!iKhem)_|`qoJ8<qGPc(J2wsOQyYHLM(z28zoy&nvZZ7ayz$(x?e
z>sLvh0l~ZFyr6W6$rSv|(W?+_nACp)Y<>D#$yg!5Wc7hqKyzdx#cq^)ejKHaw9=Q_
zn0lAIm=XR>+0NBi&ui+sDw$^iv^ZOS?s)fzC$1rq&**}e4XN;edon-t%ir_jXOFmd
zK`RX8bb>i3vv`qkhtXerK_O+*v*;<c0>$eyH|4Y<Aq$co1NG|DI>jUOFi_J{s~E~7
zq~IJ+#X4oM(?0<<3$pDyfQ?)jm09umXK-@sQrX6_1OIcdu^ol~kfZ9E!~guM-Ty_8
zFnk=uUp+=(b7$`O$5JVjs?U7>Ou=NFW#+kq(4=l(R}vQ9nrHbP!iM)|ExL{OiOCSf
z?_>}P%^ZE@CIs!NR;_|{=V8dyjVR5{{_FENgp;9juwL`O1{o>Q1DiXWFh_>h{Pg1D
zqBO*L4>c+VA;viZ-Qq&2+u!2lD>F8V78`s-M*)=abyg{wXYY61yzh1I3<P7ic*yfI
z{Xai1IkWwGVeCb=if6kq>wi8aWemx$MED${HvArSwQ;?<ROjJqoduFkV=%AEyFmRH
zWKTqT59hx=CmHEz59PjoRNECurm@hYEF2;vR47V7_=>nr^I%=pcbd}U=h)fVmq5eo
zgz<!5FOYHSGzq*z{p~>f?Nt@m1rHC8SeTaltd6(4QZy^w5s;ou%VdycG+M2~@~Lug
zq-D1_HcD>XHC~Q=^ypDj^!8Gc8u(MjtQJT%C>C8|ru%b~$V~Zb)%Ppd&QtEkY1H#f
z58p5EIOmmm-zM!yf92>dHuI0IPPjqxF+=Q<6IW)V;JsV>F)Sv36q9Js|2~6epGor%
zAA$G{g)0MNKHq&1W{Q7=A@SxcUW#?5O%J<$@XGZ4UAsvPzyKrk?rV9`gQR;(zRl<a
z40F(iTiup@Q(h0W)NAVfqTWGca_}<SBZ^U)6Z?^zQES?A?JP1t2pODW(N}a8>12kS
zH5119$ps4>4Apl^fq9ne6yw<x4Z=SUhq+SqhwvI?Labs2LMM=X<)z1<u709GM(RLq
zDR=Pe^v%hqHhSPSQm^HvMjp*y1c*_NX#(_{UQ&;d1z)zE1A+SqNRxR)wLr`VVI*%j
zzyR<9Xu+Sp`R=Af@%hV`XOSx)7I1rQ?#D`YSN0>G_xFBW5SxAzZrKd6k0r#S41gHr
zSia;H^>q?-nF4vG$a64*6DG#CNj~pgheU4)gv#Vkc5Q&HJmMKZ$j(Bne(9T2;A0mC
zt4iNSNWq!kbs9Zme3MnWeGQ@#^<hLk5sukam|~O<&wp~#44)nl3TTF=)@~=}Py-ky
z7rLU87ez4Wal8?U{-RYei`_3tPKD-P8`PB#_V}^;Q2YOfn)8W_03DvdoH(o7xsy@%
z;<T<9GUq&Y*TD<s(r!*r3_1&f3>pTXdkw8D&*nvm4mg2>!19)?U1;Z>P6I?ebCjx)
zPiQ6@w1je8PnI}b?)`}<tL!XF?NH`LaL&Qhr!iPrTW)8-;cI(4qkX4A{Gkmj$}OGX
z3Glx!jUGwX6c%r-^oW9uWlYNlBVh;gVTDIdy#DC2VUO5C%lnjs^AK0mI*H2k<^&L2
zd{F}!#}_OrV0?;ozzcKvx$UqOuDE)~uEe2K+p#;1T0S-s`_VGmhTw|xt(ndi8{|8X
z?n52L;%VUXe~T3ow2&s2p9qm{M)7;^Vo_cGg7hg&OwJ<pN!x5ZRD_#j?>b6@oTJb^
zKjjzXw$0FoZ;W?<qHBBJ-)#((97uK++|m~y-UD4j3FnnouqX7c{O8kcF7<&t-<Q&I
zD@!WgT6=&(o^G_4y6t>SsysyYaV*Z~tcj9!)QHA6fM0c%FWr-O6e>j#zH)_G)9m7s
zP<FVOeEpWn*6hx<Rr4cBiUJM~{_AB$qv5NrN%hE=Z7nMxL{qN*>~1OS)A%z`kj%h6
z*YqK)((i{Iut6dRNz7T&{h)l#569r;`Ynz>GV*IOyKOv#Qr#q1PitGR=T7c)TqUM<
z8a-c6C`<?npDG~5)#Q$g|1wf<Vf-jaRq(6gd<64vVtH8A*oGK{*D8fTZ|Ctr(smf$
zmI%<Cyml@G9&>ivzdA}!PKSPP6d7dzC`uK;{>#wqubppQJ;1P-_6szryv>z|u%Wnk
z70zHzkg+0bfiSVWIr2dth+;6p>?}~wi=KK9_C^mTkbUs4S3zzGD2IqO{e;PIt>s>n
z@ing=YoQ1aGfV2icnr?_)6HzJN{~tlQM7eAx%`;-sZ2sC!8q{n&qwLL0Qz(R84ma1
zZr9+Ir!pvU^8ccf*%r=>H0jMvIQ?(vWcq*5aWP)*81CpMBAc`1QM1v-_V)J9UD|lY
zX#Wa8&Ui*K`zKbNFu!}2n%HA7iuo$i<B;s@=!DHT+W{d=4N!Fo;n7Ql>c1IOrDSCT
zx6Oa)KsG<OTxSR9^cR=Yr`b5z$2+O7{0NX|N`d*A$siW?WBok@wi&wl8arFFZl~UA
z>x#xBgu(P5a9jl5$+`FK7f}0w%e`(PbDQ*cdXtC<6to&g{E5kco63wb87phC*t=c#
zs6m}hNpt3~RCmSQmifz}$`4rW^Zk->a7j^v1cVOv6jPCi{UMj3iw8XkiLH6ma+FkG
z(Q1-|PyW&SZgEQ9*XinNjgHmbkKdPT9oS{f?=t3*`$~mS;8||-{2?*|eU$VOp41cD
z{WK>KG$%Q#W60aqzqR{Yc%yI9V2zP>+QYIFyZ3#r$GNn>JLnv~g&nHLmj2SOeU-6l
zGU$NaeHC&YfV%*SE^4kk`*d+almhZjXRqv$&VZ-N0t9lgTT<On;_UOo5j2J8#G@0P
zvjwVL1i|J?-)!EBa2kOCb<f{c1RPud?wK8T>EOsZ{wr`#M)hBT`*wN71AfU3{vTZL
zYNF^03E89p?KkGoI0!%Q=icA9vDF^+p6+%UYv-Gm+s!n4o1yR&DQEA#3c2spNVB=e
zg3bNC|Hhy(P;#b<`4k6r?cFVXL4nkq&uk#;wqS`$7Li7dMH4ouhY&q*dLVDrwKTW#
zD)+0D&I2~Rw*~idUI&CmzuPu}EE`%D$-VyeEaKo{X`puX4}LF}4Qk3tyM>7N%QwSW
zsyb!SbnruL(gC9I^A@Pz3ztxjxz70MT%=r3$)0J(@FQ&)yRSm_IvF7*a=nc9+2g?j
z?~wxD3FpYsOM59V_9f`6Ej;f+)-C7(E2f@~OARzt;I7xtY=MtGYudoD@e1KXLr22i
zyS5mY{qMdfa0bv=&^bwi@(@x&CMG7vYqfsu?cTU|Le?IxDYc2;RsJe#w6`-=IIs_N
zd{wr)WpVU(*ad}E@;7e#NjfG6VP^lMi)<V@h=l=5@fplLL*EIJ*&!&I0SJ0v+H$;B
zyxb84WD7cZsri!7_XIT#suxU67f}!QceGMkXX1^~b#s1|{m9`g4cvqg{EMr*sU-h-
zIBgecO@95Mo5J~nUhr52#{Gk%0!)jdW0(Hi1Xgzp4ywR*JkH|~O{&I;MeRnbo~Bso
zW#A_5gp@Ji5x%@sW;MHI(fDAtvN;q)BD{D9^0J@z-`Mj^a2)%9F}Ktn+Wi}tSHQy6
z;>3*nol0^I>EO9`m^E_$L$e{Y>fLf*Wo{C4kCv~WG>S{Z5ST6n0Hsf#M2K9V8k^_e
zrC3r1iC-pwpKoQpA~k*xv{GY<1nLDxBF^kqBI?k+`xT;PkL6qTs{+W$-nB1fitaeL
z?9|PB|Ag-;A#bZlsqyzaMdV0ll~c4-k3?f_M4Dudx&L;s<%TuD_4d*$$Hl@pn?TKH
z3r0Ec+KfuA4^eVo%}J;m)midF25UJjjTs5WLq326#x8X#3j_f5yg|gIWxhhL0L%z1
zKjD?x^&jBsH8WN>=bTG5nar?Mb6~q)yf{n$=cDf1!%Y_8y%@+nvfsN#IC!o!13vtv
z{$$gC^+x7246AOOl0?_86Ypfbqh?kcgGx#T;j8eU^_Lb*zB4%t;3B$3CKAP;8(A>}
zdB>AMFU~~`SV!Qt0pRFGCJ0IHY}%NZn#yZCf2GzL3p0)KQu>Lk5T{|^V!d-|%Pr)K
z)UkiX$|R`)-b})m-tVs3|MdIoE)N!B?Ka<L7Zo1L;gNe-7@)c8uC65#ilLAZfRxyd
z+e!nfFEzztK-Dh1u->~_{0^EziOoyH%kzMhQdXFbah13aVb*b*d7@aEK#mJR6WXs6
z6}c{$F<C`gNo;(0U>aSxDl*zPlfQa+XL)0K<@M^do#lu^ol)^drwCh1d1RWSBglIv
z!lUK@Ugrp$#$oWyjnkD)%qR_ioCA0%c_VeScx3K4C^QZ(AqMBqU)jiU52FHap9v%W
z@Bg#^A~IGyd{;8gD^-Py9jfFa4rPVDy)Td7gEBxB)P}6Z4a5nDX!Udj4-1m1C2m0l
zw&J1huU^OqD>g;Maqu{$F)+AcW|jC?L&pOt+&^!;|LcE$ocl^R_*h{*+NT8n<<tG+
zMdjpJT)Fu|!`G|s`cn0@gj1_LtU`%|wlMdv<|XIP*NcekI3m9mB=lfL*AYavBi}P3
zaQ*3P@2gZ!_HKn+&7os_LJSVz5wY_frp23Ai7_;~iRNo@e)<3Zqn$GKq~aQsh!MKx
z#n3sbCudnh`4L8f4XuwxKM)351A8+F&ap?HCO?eg*P+128VB4S30OcaMP|XMkC@qq
zE~}LPZDaNyoh%8(N9Nh$VRYqRc5nAZ7zXk&sY`4buU6gr?y?4E_aGa@?vBNndm%fY
z+gEqaiztDbW_+i36(z9ZS>#yofG+<({5DT~JX*r?Pb_?>f1Z;44})wLx`AkELnYxJ
z(d_x>;+n?&Dzp67no$@_IzlQAHH?bHvKHs%(=BQyP)f^rxNiN-3L@WOFt4GIiYt8e
zrZxfgLXN}yjmcm0mCPr|2!-w5QF3ahzB~)51ITvrz+99N2+C`KG$<=^{PdN#W#e38
zOsJX=?mNq-LQzs~B`_O7%D#D3A1F0uO)*$arkXO&IWLYc=54807n|}K9`xQnsyb~7
z^Jf>pnih-<wfF@AqHYFk!{<Tmp7exu5h5+sJ;2C<G@(J!Sr*A^0a;z*%2^;jdYS{0
zeez;?V$kV*A!0KklX_I!@!GZ5tyu;Jug{v@&;uG^Ub}u}YKJeKfbr_gm|r|G4bSPM
zNc-1RTv}C5+i(SjKzRGzV8J=$JifpE?Q`G{ev_K2iMBNEbv@AFxAQk|k_?9RFs_cs
z4AmFS1D@5D$qeiCNmX}uA51_Xi_XKz8f%J;2VTCyUlb{@|JSwsz477{O$G!X4c!l;
z@vCt7|MM63=)+4}qG+A^<r43Zr{T;(3-}e7vau0n63AX^1Bz%*EPjc6)j9X$ONOiO
zXS*_vCxtZ^D1+ElNoa>>?n>k-1h8Ns0b#iZx|dBe5lcV~V;*bI2ye?eLw0XLuo)y~
z`$t{Yemu)8fQ<Qr)%SsZ-pqmtE*-J(^Y<7_Kb|>F2tz!+xAmPedfA5$+zEZHpJBrH
z<8)}c8X%i){46;uwX?3;>Oya_`qpKAo~S|SDIpf71fF>dIR4wlEkHN$U`d_P5)u-r
zrX;j{Wo`X1zV=q`B!~waf^2?fPE&c|Y{@YWgOZ0@jYLtkaJHC{TE28)ZQwaMLfo5Y
z1E;qiN6U&=yWVMqk=f1$tEi&>|8eQa`uSfN8+3jZx+{4&o%DZLUL5`xp8RuJS)0@Z
zKBfj{)L%chuEVAtMrnCT3MX4{f`lLgAN$9HrW+OYoXc21?_`%(<%7rlI&k|-9v$+P
z$6|oAf!B40Yz;UQj|vIPn=zQ+@3T<MPZk+3YvmXOdW&zB`AlzqrsL3A@@mWgu%Ubk
zCRqi^nsTGhSzK#vZ1FDL{&ffGXD4^u0_8~i<+KBMz+SVt!uoRHKq)9(mPKqc&x@?|
z>ufztekgYg5F2VWR;ogY)z)>f>HPLzma76}O%mr>W=v=WCZGPtBZc9r#(2AZdiFLc
z@U4K(5F;JtzzbYsoPYa*Hsx<JjVp~atX2}449Zm?vdrcuTZ7w+4_C|c@@iLV67e(v
zF%@BL@x33P$yIGgUB+N^6yufb8y=^`A`W7U&XN%h&(a$|kzq5LHx?k{fKhyyV!R=?
zbp2w{xKM93d@KO<G8PPm{))w_GV!l6qVS6(na$M}!9pAa+=`+T#Bjf7O!j3NlqN@!
z*9#{Hm6e{KDE!`t!NC51G5XJU28W-ll3#70iRQ|P-2d%1{~AP9JP9dk#jL@rY_kUF
z9E)Nwt(>Z59Lcrt5oYhL0m_#EL@E{0Zch;t1K{ONU3Gyb34WM^wBEn7%?@&hL14EO
zff6ViZK2ZiAa}ea$={b=uqK!$;^T10^&tQfNE&hm0Y(S`Vn8*3c8S2AZU#YrIuP7!
z1qO_=Uj0X)H4e3?){j=O(&FQS&Ootd+&rh|8)36jPxa1Fz`DM2A{DFQ?mJm_&9MPs
zqiPn0YZG<ztwC4sF+y`3X^iHTQ`80JVc~ojxsm`-$c@7s5e2a30TAdR_BgU=Kmk;7
z6JSX4r>hzX3gl<JXI1)u$d~RKdD!1`MGDdu|G>J_k<aw?e*k_)J#6+2;%p@#GfWwO
zn5kKW<87b24${p-z?yp5_0H7i3|;^b@s$uC0TD|m%sZ;l*6HAL(9`ocX>=Gh1R9J-
z>OVAB9KwnNPow?ym;g*dI)UJ?OEcfal(!(hss%foGj!0@7kHdVD8#PZ(dFRrw*9*U
z@!JIu1Lp#G3mJ2z#HE!y&?+cL5vS|NPQ>o7Kup1*r#+OTTEt%B^0>-buei3#<3Cv1
z2(q$Atk1cHeSP0b^8M!%k6eR=>j#{C=cN=s-sNs*FYRD|9?k6Zs8~Ix5$r5JZU^V6
z{S6ogha_IkD=_${2O}r<RIjbznHcsvhvGqBjiR(|sL|oesrS6wSz1~u1Gdu7R4ewu
zPmjy)WEI~ctt>d^F<S<J<&+4`7LWd@<J`b`op6}1RGC-OvAQZZ-rLs45pSAStP}lX
z!I!0Y((%yI(^cOVRbW8qX`3CufAN8b``*;;wbfxf-NyEEMfS*{s|KTF6X(k;Hz74c
z0SkG!D-!~Jd%@A|a!OT)_9G)ppr|aiEO^70;9<{Mvu3uprj1NGTQHGet)0}BWS(vN
zNSdz2&T7dF(coI~m(IW|JON^o&i~0PktH4;IA8Wq`G2@sWFeT<2eMqPtC{>C{HywF
zi)yPbIpSq6p<|LVC&yP;yvFq3%KOqD*U`Q2p?TJ-PaGNL9cg<n0m{z&uA6HEr)!nV
zkiqjchg1?06RQylg&(W1RH+1E@+e9|umr43!`nXRS&8wsw9udxZ&0At0z%Rd6(YKR
z<1lxGZGvZ%lGV%lNONec2%$KkBs|{xd`xizNP5-<^(hB&Z(ckO)!=)-)Mtq%xG}|=
zYXlN@2F1iLxv#gUEWiLLr>I|9)J5Wi$XHNhwy2y~|2XT}xI6biuwS@qBK8L${`SqW
z<jWH=`h*Ze+@&6O8l!*WTme|EOtJ0l&cX*-v|mtH^Q{wuN1~}tsAb1i$<8{cy0%yc
zocXa6;vgjcxmV>`@i*h~@x^2Bj01505j<lVC=F-BZrYxG^7}}aAA&%C(BuT|!G8_(
zF*nEup5>HZ%VK4mCb06>vK;wdn<SUA;=kT!GW13qdj-U#G{1dyU?Z$u0|ED7m?<Y}
zhr~^(m}zRQGy2p+({CZ#JCl@FyS=qBeY$Z2lw(gBJe>DPJ6(DNA$xG9>15eg7>nay
zu|dM}piydsPreE6K;;`)eQwb1ei`(fN>K|R3nuWo_pg;rgqa(9x^q2tRV_^YF%~Kg
z9+%%W>mR^_s^udS!@;frwKw#3h_OuJciVnw)>o7tMB+#5Prm?#JK?T^6)w`!t(EW~
zc@!Qw5rfMkg^rHmGz(qyf}-1O&Xo~;!H&{Zo$G6%&@K5l+Qm*$z{2U>XnnS~aE!>p
zSt`<!{rkdQAcKYL<PPE*UiaA4D3{tf5s=DVEH5~JGNAvKLuf$w0_uhdI!386Q7M>0
zB34p28FXS{IL9|_#E$1c?La5$q&^J2$%jf$<fdtqqnu^_@KN?@8cwwscPPMrvD0vM
zJ+_<a0J~`Dut>xxDyccvlfOm8tP}cBkZ|~!5=%ct8wN8<J4F-&>!i-vs*`s467VA@
z-PLafg43wQ;vb9<7o#voBmCYJC_?E?Ig<)ndl|ovy_2Lb_^4PuS^ZP#_A;+J=^+%M
zjBL!cQvuI+mNN@nKU6)LEULW`R@*0FZ~OE9EL~w2NQ>;}E1nnKi%pH?TKAk}xd^gc
z53#gNe_wG7Z&U54kn$(YpP!*bzEkMoDgS}7E@eWnaUp7P=o3p?$pR#G!$jpZ7<8}V
zbZk?bC;H0OYt)~Rg6_3TZ&V_=V0qpIxdDf^>Qq+IV8Rz!E!+(V^B$xQ<zdVGg!(uD
zIW`*Z%f4Bu1U%iy`LA4oo(J%D309LGVhURNN-#Ybiz%yK^hxI5Zip1)j>jofftuYg
z^i$;qHAu8#1`xLWvLrd@ljl~@wWJYy!Q*XxpX-K2KA<&vcCE>`fSkl{^cl4dcYFv=
z+WW&8-CzYw*-5nxhJY3S9?n&ldLb5rzLADnA1(12|AiZUj`N2*M+i)!_HQVLh1}5X
zr9#@*E60B)z)_Ip&6Qs``DfAGF-P$plD{M%qTA2Z5`=MA<g+Q=9gKTca07lt7LRUj
z-DFX&blN{-d?qzih{TZAL+#iC@a!sjQutUHE?;Gq6gkq`vb`+w?FmAkN_3CzKd{Ty
zIm~p|_W_9ql>e(U^7_o#Z`!NM;A0_(M9k$7AHr!yWcL;2y~k2EVcggK)RfpN;3G!7
zXCx9NbAXZNL#b(*IX*74Tv?y=8RE{6CF>P_fb>eU_1F`G&UdE%Fr!8uq3hB~xr+A@
zY+@44UE_UZx_gHuHim!7zL6Z$J6hZ8(98VQy2P<d=<Z_RxyDC(J{V0-_*P2kiS`ZO
z-z$K93u{E^^X$UkyNde^#{eb5af5<WO^vNsX+%3GN5RXR<!jCBwD$@D8Y~{l-U-yT
zMa0z9oLa(*T(3@Q#Jupd6LPsN<>b{h5gx>lVfdZ&>rH+K(Ot)n?IaVnHP3Q!%CeK<
z7!fwG8UL<(P@rxLGqP3YN<P_rDlnUUvAYzKUXlQbp1NFF<Bon7F(L1}{VyENn4d&Y
z!k*gL2VARQ?a@5weRgtMVnKx0tIoM)HzOxIg^#TcVy_h$eMOlWPbeZtb{n86yFn)^
z{};eyQOwdWj!2!qg^sDf1M2eq3Ys(pep<rhSOtg?<K@n}{IZ2n7r~lR*YL4G6VXQ4
z$w`q0Z0rZ{x<lzje3>M!K}I>$k$cL(&j=_HJYmrLb1GWKo8oq#!O+lT5qqiO%c^u!
z<R%~U--y4`AjG@z)Iz^`<m&zjW5a_J=Ji&|m;5(OLly!6g3-Jh;a~GOKsMlmJ}8?h
z<DKB6CDEmu<f&zTS|js=`WkQGDZLh=-{%+2g?DhpYxAAbP~%sYNjJU@hMO1PN~FHg
z5|we^QlFAJ&eRpvq?$0i+-IY|R8r-e^xM1B5*kxuk#Xw7Tj*=51Ww(X$R)R31G)TK
zfOjSn9<#<Wp%7}k<I*?z$7$tkQHKxUIW4%p{p`l3HgArO`Kh9;90Wh4fz0rvlb|}C
zd|-LoXlHxN8iL>JPYJejITVTTGAt?0tW#FCOiMZV-JU$SdfMjaeK)Gw^%0Z)xAn^1
z;)nK7Dy$5;Y@37pve%glzY(*~6lE|ujEQ*f(jx1|0Jn4~rs_UheZ;20y9|Y`hOno&
z?TR}hGk#C^&Ta3Bm_Agin&G-B%bhp$q^Jr?+~eY~(m&Q26wSnQESj{3>Nbi@b4QK^
z@cOYF$HikpQFLKqV9Wyy5FRNelYPZ!^vHIiwK|3F5Gh{l=`VDI;+u0{;^c7cX(b<i
z)O`@yXf|M<sQZyU4(lHEz7*y+n63bx@e(3y(Q$<U7;~inHOmNdFmpi=kBgd#EkHxW
z_0~J?j5nswm$^Q7k(x-3AMzDwczc+jRr)hTglrzfbp8*IpZ7x{gG`~@i-_}hm<CGp
zuweg>h+&E06w=ociw=kvt@ydk5wcicTVfO*`rh&Az5(;Xz)rKIQ{3E(Kc1WbMCyt-
z)!s@Xo<JD2#!_`YU|ON;HH)9KXX_+KTogXx-l93M$3&rA!#Hx*@B_H-47$8u7m@_?
zcaz!wZrTSbk)0E>B38ey8eg>f4L>DoY(P^i3t&sN=RzUuyN;w<?_1VHMk9WLmpEtD
zcN(ZFHmMMtU556%iG*o|vFdG@yb=h7?`|FvXHsp7#^=~FXg^?Kc3D#I+|HeYv58r|
z&|jodeD45WY~O*CDiLG+hWA9PhZR`p3AHkHuL2>C^jSH0)&wBC$9XRVT!2|zCg6B@
zWH%tas+HQ=3`#Tu=lz64p)y5|56aJ3wPoigB>?r{2Q;PWWP+v_W>uYqn%<$&;nA6M
z*Q9z1Plx2sf0Xhbez=%p)j4v)B5EHK#jucpfgwyQr`<b$Mnr~avZJIP`}Y^JQWD$}
zLTfb{Moudg-MjZQMDEWVZjs6B+e7m(apZ)wR(Snhb%ILY!gWq|58RZ}M^=@=|E@Cg
zDeErbSrOAIayp}xhZA0paj;}~UdHGc_JTBuJjOnUpGW9`_Yt{NOmph;Mf<7F9~u;-
zrsx=bq6Vp*he@T#bXG_dM-sMJfMXm00E5WaABRyFj&SAJaJh~)L?#-3dem+M@W%<r
z%B-X4I1$$@%C1)$ctTmU1qMm~G#+nGRO#kzPF80{rnzZuLe`-S6pskodu)A&QB&C|
zXUYL|mXToYJa3W#*pPjob1J!-zyDSplQ00Fb^Ste`aSKAp<nrhm{#SpBdqjSj`S2I
z)W6%0JQPrPQ~nf2!L+P<mg+QwE1w5r$^HV<Wyi~C&Xah11`H1S8I=jL9H}R*_!ZB;
z2VK7yLurjWhN=5TZRAwV)O$&W_a2kli0{2g*R3g@)8?YLW5(j){p9&Mfvrzp9eU?Z
z1})^nBHjbks|H}Kne;F&zm7DWM!SL{O<F2-f*quHn?s&00$T<%($L5xPffRSPx9tc
zH#hd_U4RoXP>eT~$R^!kQ?y#zWp4H1?SU9MBH-gV-n$&)0Pex0`|RRXDJmeKqoKtk
zH;8=JDQ+7EDkNgCU*rbi{=bAKDA~K=LBz3(x&S~bf0O8z%tZ=s5feW;MT0LI1Et8i
zUT|cm^w|aB#-olVXf*&#)g7}RJLSzS+%^oI)EibGZYD9OYKF!-v1*An!esU;CQ1EE
zBvX`*i%Xklh<z)!)N`s1+zm205%wp?$d?5xU>T6W{M%<^O31;0_BSmW{_){248aO;
zcoPWpOums2@WUq#^^B9$-rFuBIm4aIi>z3W*)Z%eV0bZgGJjRU8w{S7jhEqKx-4pk
zyS9BMAn~T&RW@`CilQ1518lCVRAchb88`udUev3DT(%kkIsmfB6ou0W#Qd=Z+8E+x
zsI>;$O>U|VZ37u50a`dRO`mAq;MGcgk#10`&!v+&^&Xj<9RW?V6od|?Us~^!eZ2oO
z0gSRrI9y{@2(pJ0;=RUf9;<$v-GyvSwq487Lm}8Q7_z~QQ1Asl;~yls^7O!`KHP}U
z(&WqhF>=Rhx=9+v7k$&1#SCU)mFFOYRBDVG*%^6u;p_npWU5^+uvqOWk-6c+aXviO
z%`zIk7}j?y7BdS`OZ~Mn6)W>RK6aKxGyXRuW{bv{KRBKbGu?-yUMQeCnNKvEpSk>d
zecjpNbc<)dAo}aTUwA;sWO9D6hC}^)!n8C?Wz=9z!)2D6{#ES;vLCF-y5^HaV!I;v
z@=blJ`leYn9=xvQFfotxEoM1b?`3yzCEQtVgj(2eX2jOC_<9Wwy;Q=|4UoMz%n_92
zl198FhKaH%%i`i&RrH6M5tL1Q(e3T(A{stoM}ijqtP39X-y>2ZF1PA_m1(__aZfQI
zZXCijZz`eLRE=Pn<}}ZjDFIh2oiHTZSA~>~C$?A$>k7Go@i*c-GhB;uBwXHB-Tz)Z
zT5O(EH6#Xj%_Xy43q`yD`x5+Xd9twmVsK9u3{XQxDxWHa%di00sV7&;v!`Igp&}Rp
z6KV@zC*xv(mNE=D_Jd(&=c()kfW@jEM<YjTI%I_Lak)#^N5W;P+aG@cYMBi63PN-l
zxXY;?_e6UY;$7?EwEOP#M8@nFBSp0v#1}6nRRau_2yGn;K+Y9$%%`sA#>X8_XQFAv
zf5d3Zy}g{}=H)~{cm%q^LjlIV4o%dwNw>iw#>wHsPuVRzVnxmZbf?z9Ag)nyQC3#z
z%#VY_2CRM%i(kD%d3VSfK#5u?R7XmBCeUJ6q-D(WR-={z&#<&Gt^Ti^2V*TO3txOD
zzM9bCi1cKZ)TvVY*Hyo|SW0(p`{Mgj$d+-e?3bvrPpXyC(!NaLv99xC(e}F^UQ*kd
zj21I$l&Vk|z+L1CkBYoptsM%1Ub}#PSGzg8bwg0!VnZkQg4^gVdON2Mv`{q>gWKl(
z0w4({VJ1LqPXM~Q$~KICJt4KV6cqG<)oCI>%65jP0qCZoaUZxWqM<9@wM}n5T*xZK
zdYEL#E%9zwPMp%DzEPPwZpa1K+g;##WP;adc=b<@kB=kp=;ZpRq{@SNL|`-ls;ni8
z1&{RUl1!xtbVouoXn^?acA^z?TC((yjJ@bUEGGNLS7WG9X}|{>xD`VFh@N9ibgDbI
z_8RkYAlZ}!=Gg7)k)zKU#dN1xPaJ?@-O%oECBz)*5v>asYFxN`uZh+njo%f<PRVw-
zZf~w%o#>$}un*O<56h+#Hx|E5^Qw@Z@G$+^s_JicP7uKYTqP5oQn&et{>Ip^g9n#C
z@YzpYLBe*V>CU`0*~rnTsbyKZGXAHUD$4bE)?pNskW8!1-ex~;knY%olr0b#R@DH&
z%J{L^G9h!~+CMSnIjIVdCj^O0CD}91C|<<)-Q|roUa9z6%)Z_>&5~BS^1pFjZkRAU
z@KTuUsT*&Q`^MirH+m!B1oN%!m~?&I{RnRJWQ@03>RS42u99CovmgD3OF~JL6VBY!
zySLe?D2Mxoxb;%xVCkx6eAsaG*XM^MDh08CDP1F85@Af>vP0&gmeeoSKrtj$YHa|2
z97;~JS1ufCeOK0-Z<DAew#Xww^_h<Fb}(oelsi4wlX|@YV-yE(P*r0v8_0a&jM3z0
z?o&_T-qj+l^~xcqt5pQjEP<$Q0)I3BR!HtneRg2ke1Im(VUXw><+T6V<Uwbs_E`y;
ztIY%SE@$CD(HoFiy3lO8=265?)--x53pxsn&)k?hcKzNq^ms9^Lo9tPLRLJOuyr=V
zk?=`Jgx%~$dl$=wZ&KwZ;e3Hq4e8PRt&5!_=hS~a-p_5n5kYRdL~Xpe7rTAJ;2i%)
znZffpVx@;|7wlN-H%#kSYrZiD=(uv`j7g9SLc7)PY-BLlou2UQIHyL^<5CLSvCh$A
zPX5uvrrsgdcT(Qp=_>_#RCqlZoKshOcXZkKXNVzE4vNC$&=QVaNC_x|(8?+@lgaEy
zt8fhH5jV%nlaBY~Wob$TV%;x4{NX_}R4|$8lU$yWRGenL4z^WKp9&KErqN^?cVHsQ
zw1!KcJ+Jyh#~;tY)ohzI^h?vn`!{P@%b`t>A^Vxa0lY6yj|4$zEH}lj;Q{~b)|U`U
zrg;X;beV|;t`5m)#}T@GXiqj?wfw1g;$oFDJ}#pi*hFrB+~L|#Qi0SjvUGHGap_1I
z>`Pd=hCe1Rpu+Hj*J@EmJ;nOwf!Um*DsILRtIiMo3pY^J8@>*G&L?^8ZXMY3br^8~
zK<OIOzI5&d6Y5Cp!F5q<A76k)By;7kIdnd}P=;P_s9{k#Bbu9kkuIRVbgd1|{5v2d
zb3@g1P>R%ZuwJ&H``#JyoO*pbdu-yQGPf&-&v&JY#lCkl`7F~+%h4Oc@o2yE$x_j5
zJZ8Xddae?l7tm|la%lnZxt{9NtC}hzvbXB_Yf_$l27+z8<hTn2yUlNnqzhC2BqhZx
zFtzmk%W^y2Pf@s21<or|0sd0kS70{Z`E{TYm;l4R>l|5A3A7RN0=qd5YSR5P-T7$?
z`CKcLigPf}F#sUKD(DT2TtF%wmW9*j*co{=K%^!g`mqrgWew9*9qCVOLXU6Aqcyl;
zH0oUvz3-;@O_HA;5S16CLijU!L%pO-;v7H$FAzM(F2EKTdH`=F{yJdTh+u;4ROh!K
zWOCgwfL|}6W#D=r>y-VE8FK{$^%m9Ehq0S)7eAazm46Bekfu^7kA=zjyHi@phBVKL
zJCfC<TNj@=1#;+QMwsdan+P_m@iGB6ypk?`%bQG1y35jPU3F!$Q1<f01>PjK`$Xl7
zeFFDW_^tnhjK2{x<l-y@06?)$7%jgWc_4QS`-G*8)Z|&{scVh6g+j+<730M}jP!i5
z@H+qd&QSPa>IG!GX-$#`)gwy6Q=?E67MCcEiY%Dtn1@^WOtv$uv)D3{6{2|o;c#Bf
zzY$l}u?h&=AuQ@d4Et<HrXER3?$*8*!1d;qmJ{6Em%#nA*M{=aEHk~h9t)vnT+$50
z_^K<j{k%|?O<O*!-Q75ih?=WEJ|y^3jbJPIl+LwAjT1{(o<dzgAw?rm=E%XN_f8V`
z-QXQ42vIlIJ6R{LOAxoFYS;2xHf5(-m(ZIfhkD2mdk|9x!i2G!GnEXi=BML~Ag9j(
z(rm4OM-476Uh@ojK*6Zv<IX(2Ax6!R?2f{S-tv=Vjn0kbCu_(40bA_nLsjg+_ZXsd
zU^?_CM5K6*BlXgRr|W*GXN0T#(C~)EQ%|m}lmOtl?HVh{fLetyt9WhKWTUFzv&)1$
zq|Ij%M8Bv)UF+v*{`hS5FrHks?M2?766PKcME(#>%9*hXcCrP>NNEU&o%}8453;mG
zFY&O_{7t-m@6g^SyDx_@g07Q2#sBlc&}5aU)V!l2t8_ceKGU&{F0w<F9hbOVRk81%
z)iCbeR|(uj!DcUfnQOX?C0<<es`MN=_D*X~FE5}2h+bxJHcKBdTAI}Mf4hrnixXGv
zTLUx?QN&K5LR*NWv_SvtLqihZ&YX&*h`axC>X1)ZP6Ltk;2jCtBXH>UElv>y1l2=>
ztFqp~i_EcHGOjzxRXM7glWKeQ1qEBYBfMm0HV+&g=S%E(KC}(1%Dm(ydA))Z*A0&`
z>S&7N8@Kh)Jn!`N_R<K7{dXRG>|WBiTs%Xw<dQjShxhmW_I%0S!F~c)fhb4qFy-F`
zMN1LAvam@Sm3=PJ=0;}DFxNIMM|3yUjZtauGgn`Txph6jChD*i+gr<!s!^fWbU)SE
zuZBcAJ|W{U&`H=R+>U<yJzU_3$&$*xcS1`NQSnAQqM!5?KXUCCcoSMRLam=Tw0FP$
zEXKa0<3TWr(Av*2|M;IlZ;a-m^qg7V6Xxazq(4Q_;>VSC&nCJH(n_qr%n_-yurD%4
z=?TnX<Ulayw}F-_?7!uCanql(QmulS;>2&qiQQx|bUit}7NL-Pc^m(VXuE^Y!@7Sg
z-ap<I`D^)cWZUkk(^C9hM&M61Q4aAOW7qijWG7~%gk|C9o$X<e>D*n&z~C~koug*=
zb9bgzW+@y?!wK1wHw5C?vWaAJbslz2Q5&dEIr2OG-Yg71g&}XaZ(2>ne|z7(Uv?PY
zoeY0}W0BW9VJ%B}`(sS)x7XrSb=TC<)JNf<ALVf-?A?bK4yd`O!47vSJpbkX!mh-f
z1Z1r>Ir$$b?F(!0TGZp)nP<|t$0Y0|j;zPm3R8E5ll;DoFSt-ksAZpins0?VgbY-4
zEy~7=#90~(=octdolbhBKAFf0)lUA{=s;SJZ2QIKEa^R57YEN#`uB-h+9{Ej&;EU8
z6fm0qi7m!P1y4+2o=<!FAL|`OF*f&Jp<ZB%d3`uTu6pwNj`jY5Ou2yR5B=4*bm&(}
z>*`5Rv`~tX@wg}*c4K(cG+7R>Uy3VdigHJEe{;}e*YPGqV(P#D@WWr(401%5XI{!?
zJXHM09sIQ&$R$O~AzV80D+T{3^DErLGr8lvwAOU-LS7w{(bnAnq7Fx@7kikSf;!%4
zDVs)aV&hPKbsDoSOOA=3sz?`^0v=x5b)Sfhiw+*rkt4mKvaS!m$Os1r6J3tv+}&%;
zXEMJ2*$81w!q>JR^63X$f)046M>3T#CGG`$|2zc9Pk7!$h<EdYd}sgu5B4Ej(=Ypg
z3W=hFF;YkU$GLOXX=bgh_l`k|Ld-*nfSLHs;CLE0H2+q$7TcEyO|P>16=WkzVl0ho
zd2}qA%w%1~BG!85dj}AG&lu$$N@KR+4)-NsBW&eK6@>)fqkRkZUn~kZ({co~9n558
z-K&M}eo$)c`#2Vt5;D%fnXM<(l(Hoy9fiKSdl!I<^7g8y!uQ1@qNyI9h3A~Los)Vh
zJg7jAvg(Rjt;OvC)A$NpWE)3%s_at!l*1YbTg5dK>c6v?vypMuIcpj}pZAvCw>W$E
zVE;p69#KLd_lf4kkAG})xjW{&jchla33n-9W>!_S&z4g4-iJ=$!7BO578DOl?PU2`
zw|grhmm7VR@O{2=!Z+D(^|xTp!@l@FNTvSPmh=a|4j@W?kk)d*Tg!qjKeHVS3+;@=
zQN8b*sdjA)JM5)SIc{`&Cg}GWtKz|3iRwBb=d!mDy-2h<6Y>Q2ABJ&A47(SR^KI`(
z<okEJ>MX=QxLI#4-+&o*3TC^-B=z#dNWpMha{Mw|%ugZ)s;g$crbmnYkDKdfkT&j8
z0<lAEF&j-ML%mwhJd>j0p#9E3`PHUO+h%KY(frU3|Nf==?X~srrX=1e`~KRqn;%(W
zYkRqg@b?`cta6nOo@}%1FO~Tna>ogLmRhOWj^hh+W1xyN=mTALlu75GkuVwqa-t$C
z!~)r7Xo-uTj+gWgm_{E6HTxdL8M3#82#@002j7{Y8pxBXpwa30K{!WhrSBAa4i;5r
zDK%L|a_3kTK5&d1UXZ>zbvgZO?zT%K<BY*ks}oQKcuzl6nzuxsMW4X4t^iF)fIvC6
zp-K0g8rBfcL)o(xAcfwqEWsph2g!853S>IBj7!%$uSz!#eylZy^7IMCkYdv*!f$_y
z)G2aMqfX5bUi@9=V`GD7r{Lkz{QK-@j=$iNdO1#7Gb%pr#|LQGjvYq5He985_$Zf_
z0zW^$?oTct!%uYw7kb+xgTUW|{Dflz0gWy?+Vxio&<+s(s)|HgrL^pA6aOV>nB;`2
z$vnsLQ&B`v#Fy6Ejw3R+<%d@7+o{kVGp~rDe$TU`fzWo>N(v-2=er8eW?Qevp0=M1
zzIZn+ui@-#&dWT`Mf~<Y>4I|4CQW@dx8tkYbHkzz<a8X$-8(kb@)!yHeI(|(_gLY%
z42PAG{846d3DkxDTrY}1GdC)$I-_G&r#?IE`=lx>z`z%r2^=JU8|48{@X`0m3CI6l
zv;;z|);i1B(x;K~{Fy$Vf_=QrPYe&rY<#zy(BE2yZ0HfX6&*+hsf1U)*9+q}iP#3o
zgy_i%T9r{ES}6D9()b-`ro*nNdbXa6!Ylgx8?WM0EGp9KVjixnVLR4CLG2q*hQN@4
zHm}Xo3zl^PrjN<k-=*p*zG~cfG@!VGwQ@T$OfLNz`xggg(T}P_Wv(_t6@g9!B(DlZ
zPF$VEy3;+bShjp*Cq>m*_rqZC_VJMONl71Vie|a=c1AASV3vT`i>mcw-){u6*i(Ip
z_qEyWRJkM%zW_V+pX1qAt^(RGtE7EYG3$mezWLg*_*;_Po28St=zh9y$l6;F8}Rli
z1zEUJsXjD1rkeKY%Kln<9)z_VG@_=(-(O3@u{`U|Up6v?W{3ZHAB`uN)Q9A@-#7xi
zj}*}?Z&b>JJYNQ?!wjJz`bD0y#{1jqUGFM6PB0Ne77=>j$U{kG`$Vb&e+Cf^Z|X(K
zOItuA3<1#wv9ojNff#gP6^dSut$BoJmTt57qVZkk*&LbLOTk;}uIb7eIf4dX&rNeS
z57Nia-G0eLwC`_*xUpLm@dMzij+U54g}Y1npB&!iu#zoXHG8`(D~-GFpgENQviIf@
zZ+W_#PZ_hoUE$9-Y4ewaoXz%v*Z`H{^J}|BhB%CJvgq?c8lSsYe`;UQXQ1%Z!N&r@
zPhTJUs~ISLbf92)01rrmfj~JNafnK3orBCX6iT$sVf+X1NQhayF2>xw6u`vn*f`)W
z(zRLN;-?ZK@=5%gG#4DwJAQ#-8By-(a(G7y!-9h9w_)z#-WJ~_8Y{60n=enQz@5Od
zt4s6G!kNBg7~X%XY;oVHdI}-CV<AgR`upk7JrDp?o#Ut9cZblncLI5sV9Cov{ELsX
zb*lK-<NbwMMPAYskAyyPgv0(KB;+IvK&NIjxaR;`jlkmJQfUW`43oN7XH>eM1F=~T
z>Mn-tKS!QpF%6-3_}GYd@g@wEYKfz+{ay)<&J8bC{cbFMu)8mg<#-lCyb;f#KID|E
zWq7?L^9<F;%L#9~ZhE_3Qr`D==m$i!ytsGj0RFreP=CrgZ*p^tn|eil{gZYyQO_nz
zqs^bE(?Ooj&V-g`@9E4Tgpl#`_TJxe4Eifk!e^TE+Sk1yw^_aLr(pUs5K}}?xb}%&
z#5!SVx~GG)1WFpi(204!d7cUG?dYi|@GM>5_#239nNMiMVpq}Zy6icwA0Av~8GiG5
zo1?UyHMldi(yu~7X2On&Gcd}a+>vMRQ#cYLuD@0$;43p*D0`JwH}i1r+0W;)ycWfO
zZygS5FcN1Y-ofB_#lhdZ>8Sw8o~I{tDek`lZAh-*!TX5bmRe*{a+bhg(XT2r&oMOM
zSSYfCJtoM9EILD0B_ejMLIg`9U0*$B>PTum@tCEy(mC{mhUM<*k<bXRxc05C@#0nk
zi}}aYDB(S95|>a$9(|9m1;=jPwBnBYItlwtE8A&`l*?Se)-iW64A)6;B-y_ZUKij(
z#HCaH+^b}-0STat<_N)`RmQ>N!AqHwZ6ex|xu!q+`a{r_T~0#;TfiI&xV<NkQj@6N
zL=}><BR2t>q_1=UC2JVE!+(f*s$GLoVIn`CP$qzUK)b*Pt9~B9X@Zd=6G)QY&Xv@_
z$7&=hDFTZ&-Sn(w?-ddz@!QK=fV1sVamZak-?RCH#dUU<PXshoP|lu6i1!-`If#X~
zPzGWxlwB_-enx;|=;E?GBE1M|9ftKOpqQhPW2}g<3&&n>LW2f$Yb+!`y$*FhALn}}
zf@H5mY~BRrn$#XO1HCFBNZ#%FQ14Wt6K0?f_$V~omt8xi#~fJ+s@W!>W4Zv>T3+H{
z;SSt$C15yD8;5K_1=6I#mrkWba;I6DHyBh`zui#FxF*;ItnBpbc@3{?;akUn41RAy
zp3*B3>t6G?CM%~o4W2d!ROG{80R7k65Wm|M<2p&ugMff#KF?f7v@;PkE|)&-H%ss6
zEP(P~gcK80?1JNA0Y%0Y$Qhq)*JpoWcH2A-Wg(;*3J;$<bpC3JZbDMh_2>;~1gU{K
z%$;nXP9ka&%G)M~CtvG22{ST;Z^cz@l7ABMqEqz%YnpK}^_{jVp0?__4o;Tf*Pbua
zPG}@6xMmz^+)E$;)4<=5*}lrXFv@b1cJ6Whh-dPsyD>>h3DobqKc5S+9Y3}oxyY~~
zToN6>Gka01E*#{0KhXOjFc{kbWi)jOp{QMq!n?jc!jU32{@F`_*)s0dtpH|I2Igr*
zQIjV<F~WsjKv*$R63!QVzP!<%e*h(8YhcDWwfB^|l|F>x0>5z;new;<E9)0+RqD_O
z)7`^0hb|@OEPf%CIt{!B>z7l~lgZZ0NU;*SK!O|u$UcI)foyQ=G@oG^j^j^1p#*@1
z7eGVMo8D^8qeI{Ml4bGoW%b8C_l!W7!n)6K)FX{Ty9+eyWK2$WH-UDSvb~m9FK=>O
zNoGH&v`*-J4c>v?hqaoZ%UT3B*MDs0)bhNe1x{j5oBX{CM{c%`v4|Z%dH~P7Dkl0X
z5SZn)-fVpaF2E27S_A@leNB75UWfHHu%0g?j7X7RWAQGNX8lMT{6i?<l-Nx@x|5kN
z2YNl0?6Q6et;y$)2ttrmM0s~E*6ogE$@+-rr8d5SF4O2>D3SbX*Zy)+@F>EvgyN6u
ze1`maGVI8ftV#(F2P;RxE;s5CKgHSCV4*j!(RUV9rwL(~xWaTe7lra12m)@H)N9Kn
zC@2Pe9#J|K;z-S9?n^{$s!!13%l_yBOZ)ew*sUv3M0FjX36|1~DAPA$9d42A0NX{K
zY%FipQ7403*2*?f5gd;a`i0kV!x<;DYpTEUvL27@OySzNyZ4b!N!f-KyFFdc#AI!W
z`8Aj3uiO>vI2!OhT*iL$c#ezoNa9w4hM^l(VcS!SDP*|YUJT-;2=?hl=CSx)hmA{!
zB5yKqC2<j>d8thG7Yo9{i$!Vt@QL|A>e;%getCPhZZ4hjLQ*CMnlJco)zJbRt=xK1
zE4u`&k9zcnOgrswda4ioskVPj@`^w~DJMdg2?{H!Z*+K36Jcl#LfYX2+Loo$A9@CB
z9+xpb=ry8K==`aJk1H{}4F%J@cc1&*ww857K-!P{t3#<b5VLdFeg(^iGiPAkxyDLi
zBf0{41?`bWIb)zL8a<2GC$%kc{dA6O3@F*Y^d?lVS)VxYku_1SwXu3Qb$Oml9xSZ!
zqc4^@hLDmplT_?{+2hQTDo%l9^O5Vh;F%XdaUtK=CT_VVwh#Y8F+qfVw;4ag;cpvT
z8?$RdCB<IHFwp`xjGL?U3E_SarBqw#bJ3-#@N|H+rjgrAgGt;yH;d&US~_&*G{b!R
zb@dMk^swMq1`43yNZ(UG``Ci&Me;sx7^8&HIW=nQjdy)BS_A^>7ZH;+>HA@#J7$Iw
z@n@p6U)x@rI=%}Rzhc~`(`UjNb^Wd3dF7N6&?xCTi4>f?xpw4;Q$Lo1vWvR(liQD@
zmKZ0h(Gghpm5jzFQ`uu6mB5CyWPn2vz7(f@%buRWJNd$cx$$9yTqM3w#Twhj@<#dy
zsx|&I(s-1ucaK2AWd}MhNE)oaE;eIg)M76zfDss~LLaAJ_pZPUK!GAp<vH)Q)`%*G
z<G`bJr*Aq9irrjVeC*1m0^)OJ_zMG%1E@dTX-z&3OzmP5hroJyPbaLW@$J~C10aXW
zkaQ;_Ynp{%t4<s=bX9Vpej~prj_8fIcTe<ZPoV_Knj3$u&C69s4yeX_xLUd5?;M)7
zE;m%$Q&daADTG8fxlC{Cc^AY14)zQLBvbuS9IW>XXW%moGu1Xl86H!5a(4g3r>MY*
zPgTZQ+|BT@pa~D@+-VjVnz{c9#|yir^fwp#zDkkeP#zt!+G#Z{(Jm`tKA)kUH5{ZW
z82|++73g=~QH0Z<2!tYjhdIOlL)lwEWwk}^!V(fnhY|`1D4|kPk}rY?N{4`?0!lXs
zNWG$jbVx`lNOw1gh=hVjNeN1~wDg@@@f^>+-@X6%$8dN&in!ms*IsMRXFkEEU6zxb
zm$-fie4VP*YvhRW4Knk1Ui)xa&t)q1fe{dZ5zt-vwXx{sJ+PLh<iq}P;u`@rvQ2f8
z_rSyawWq!dIIX>=w2TQfkVvTBABi@q(A<8vzTl%tjfWjF4rV`5V9BQJu`|m8(;d%C
zZ*ll;g04>6hOdi!GLFyu0t_r|Cx`sunae<TwUFw_sgmk9dA+ftCt?>w305qk%d`Sk
zk)Tp^jv9y+7;%U|P6nsXRr%nd0b&ND1cH2)JtTw${`m*F`rtdP>*$!n&gAh@EmP%{
zG~SzUV9raC5Q4LK7{FN<dNHq}@#fGW$gA!kav*qd3XOI^0a6kJqDMssw*h=)B&6dq
z`CK6aX#VYfq)hP+PK+<@QZ3bE4?&bx+*}Qg>RsG=V7(p1jVkE+h)}F6`?h^gTE}+V
z9s(1hGHZryADBl)Vry+uwYT>`tafUn2B_{H8==%{ru9IDwF60u@WpoBAdCa>k?Cyh
z=hnzllU=_;&w)^(UhrGasdZh2vHM#EA>ME(&&?KI(Vc$m-JvMX0?e`^v6Dbm4uhki
z`Gum5=zd&sQiP2?eqK`eidn9d&_VyBIq0V4Dmwnh7aq{+^Q>6_+Em_1eh{-v^L75k
zqfXg(iC-S2eb2h`;7@(pND9NnU?SrqH08-T5-z66#~A-n38I{-E6l7=dwE88AHIn@
zuou~R^206ssRYr~T?&_vcL#mn*0`);BDx0_&Xgv4E|w+wJd1nM+e5{9`-hP*n(J5B
zC!cz<?xp-;!$Jcb%e;c$T#RH@ZY&PEYAM=Y0hJ<M#ro%4qVwtL>9}wnpUuC44Q7UO
z1I>vcqa&<N)jQST)@IievcfPn_g;a@@EJmJ5ZUF$AqHtr-u};^-ixF0dCMieEqS`t
zMZI6MJHa~<C(y1UTi?)`OzA*xBS^)y!5MASn<V6z^*}d&)9EXe9PwgmdN4bmOa17-
z@>|Gc8c2va#q8S2Q*E7J4WiJUei^vlq7mOsFwW7|5P#wP-Jb=s`B-Cdebi-boWRor
z8galncj+sApUXq>L6R0|AKe&+*kc17RQMG;{Zc?`6P+LI<nj^0*xx&jYk$Y{3qK((
zKS3TH*UeotsA-;B8V33om2l$_)>*o4E|_l64zudGKN4ee?|+y+`c-1I@~ZQ6lWtsM
z5>C_dO`q+_=`(vqH`dndbwp?nT2MDzy_RhE52mQF(5N4i8k{=x9+m*SUHqkFT-C(+
z@@S`SIrN=K)b-65EsXzptVc>))KcrQ!_3J5!vBaL(X2)clpd_}2YL03hVOj#SemBA
z&mjii4e`>;I*_+;qJ6Gx$m(QYKA~I;*Ex%Q5X|<ekH%ZU<M;5jgflI{>#F5&uL8Np
zFTPTT$`)eQv`b*W<J=I>koa0xga!{xr@0q&)GCLZDQ!MZA`-cw3+v1GFA-n^5+~Fj
z)1%oC&rI*6gUFAwgDx<(-kQ;QOo+YE0Ru^TuCD}*V!|a!Cf$|B3?zdgG`FO9G`SM5
zEU?|#0;5#iKn;vhm)kHgHW0xUOnT@pU9r<<0M4Iz52*4f>-L8}d|*4s;|I5+6GKxV
zpK|OK_<F0cuYa0~m?V$i8WcNC{IIF!Wq$-R8%bf1XnJJLe08_=Ve+z0($<M><%M<?
zbVAaCi3j_Kq^m_8EhwFq2K=eRv$-D+t$Rp#*R2%`Y`vN#+e?GWdd04_%dF)|A_`N6
z4kNExmSy%$z5iKMhFWb!#k*rfxOr`VDoFeHYeLWPgq{$O+@KoIZ_(u6lkM3nfUt#&
zu!E1Q6sVuywb?`D2^#xQ-(I$Gug-rd>032u+SrLQ_>?S>R|NxXd#TqC+=HVZdP9i}
zdK>+wh#p4{fngi(fH#qBs;|CVtOY+sH^BBeL(>iMJS4p_4hwJIbdo<2-TYJ?jcx#0
zb&9K#iGEkAB;~ICoP@#DhuAmW!EJF*U9rCu-1W}Z1vg4N$2{1HYqAf4mLY!c5Rp-5
z2hsTcbZJm(2K}B*uSVzbZ(p)Co{l+ow6x%<<?D%B&@L(lg*^6lRmIzikL2uzWq$z6
z-iPfpftzviQYFSpb?8|!VYn5cH+X;o6^5c8znif<8U28p;>Q(|0z#}3<QBUf_Dyz)
zIEyW7|60h5EncH_s%zWYgjoMs*neq!Wl+?tHZ!*f?V0|8QJ`;UAV615@0`wynb41t
zIsaP}EBbwllVqUmE9f6ANV#rStO7D5WC__m=cLaK$ofiuFr8EZ6Bw3VC_$p(^AF4J
zUcUpe1JFO(Q$}CXQUS`=WLx(W<xf!vV$|vhhi_AS&Js|JrzYG7APa!%EmnHlZxq(d
zh%N3eFeL6N+<tf`mb~jXp3x-@yka719wR?hci{XW^i_~n&GuwQP+1cxtRUP_gnK5W
z(nN*lr(EHb6&o?5s}uj&Lh6s)sJoFhs2zB@i}0^DhoS<>s#M*!Mjo~shKNq3GF^9|
z6QUOet3B1ZzTAt@hx#Tw&)!nno*%8R2V)n;p-!%w>*0xsOC`X_j|bA)JNS%hdesiI
z-zks;n83Ti5n`YHR%jTJzNR}U?`fWFg@FjYrYB1<(96;X!K;}WTG>J%#lZ#`iP*Nv
z-Xa1Wj|q3%u7aLPDp*M0BS6P5Ae;j81*0eaRNf~brNuCw202>xz|l|Pg$xP$v-)+8
zU1p@!`6YJs=J7kC;(w1ot&uB`j2~~H8HO&?`00SaUWox(=W}hfc>{egHPd?m^9zpi
zJ)|S;CNYQIiagc_ZW^Lwf59LB$1l;E$cU!z$?Oq}2Elb0Z1L7(SyjHXF0Ta7F&C{V
zLTf9Eua={!j95FcUCRDkq&LMYe{0~-pC!>|@W~Ytu5`;gZFO0G*0}0Dw~fEbNF*J?
zW2ns9rp9~1{z~8eBOkO7`4xjZ=cj0&CXExb5tt^xJL-J6zx5hgSC6+3SL1k18Bd<(
zaQOVK?_DMZh5u8uS6)(H%2@^ReFv>-r(7VjZ_ReUHzxT3!q*mXi*uXzJ@teJPXwBl
zS2vdmPgLH8q`f<Qr{jP_b_@FC6q;4g&rKl8%{dV5ZL9*Yadp;&(bUuwVcD?hR+Y?O
z$xnK;FNeKFAB6T2Qzg{x8GA?MQIylnODwt`sw<DA?aX$i_yVM&10X36aQ@OwcGhQ=
z-4#mE2epWTs(Uk1RUr)Y(ZZ&T64*Cylg;kCC`l_hL~rlzDZEc6cy&~E{A5l(WSPCY
zc@tC}PB<Apm}R713DhsjJr6y$$5KL?oY#x8ZT_0w{aYaad1Mp#FrZz<zC!#jyh|d3
zhk3YjGgb5I^2-B#TxdXnXj_Vxaj$^=ZZ_E#h{-OvK7Po1E}pR}@k4%k`Or7OvHDt<
zTthC*%+4COMcQWVP~!nQ`j)0xw-BkRL6;^>y?GF(yfaJZ;A&DsjhDgQjCkca5#7l+
z(fP*bG(4W{Voh*$8L8G#J2x+b5N{_Co7Ji~B`geeevQ{_noozhNR$kZZJY|D{RZ|9
z(bY>;7w>7z#lC_Fq`n8~*8@%|KMM5gLoqlg%;ic3k@N^(x6w3E_(8ZC-USX7pg?o8
zd49=uqwl4jupQIPwD^}V3<5iHFT`LVTDf;?d3Is@#>ApMNRj@aJJe_cPRuSM!exxE
z@K^up8UKg7$PahXE{=8ouVEiE2f9u_fa3C^=t23Q|H8<Wu5_LbXz+O)W|T5PomXc1
zL{*9yOmm*D+<W)z`hFl7pWHd|&dJ+^5F1V>CfEau<DS-r(D2<vxaK|n0?=7MbqCj|
zj-PvstriSF;NX|_C2DSAX(KUXuz>=r4eH5P776=ipduq_&5#O0(?R8-tcV4qp|;H@
zDPR{7d<e~s|7y9nD<jpF`+*6O&?U4K8MnumxF#SL-$q78f-ZVt!@MLF6XCa{2%qB7
zflE!+@@;UbQFAXRJ8Ee0Q4Q?@sEWJeq6}xww*AxP{?mW`SzfRSAR~9ayE}iaLYXs|
z`*q}OI*pHNm^vv29{yy0w0tu&7V&0kfsUe4L60(TW8Jz|O)E{W)>XkBJad$1OsBF+
zEPNj=*J{wYOpppwZ|s9p0SVuU0xTR%#v5iJLt!nm3`jsLy`sj0%1Q$IW1|4gs7DMW
z1tOkbl3e&$KSdlahMvVjAMQ<yuY&8ypuF+=Oqbn&ar9Nw{rF#<>khPG(O|0I3VSr@
zF_|LTx3AxPrn8d!F#Wd|%fEx}Gh{8nr!i`9o4#T4F#qWa{?8l1CI!eg#T|>Z6aPZn
zBxsbU{XMrAowMuKWu&cN&c5{SxtOP%;&Vx-LJ1gU39a*NhluZZAf%XpjUqd8s+<wD
zX<*C0o%<q3x4KXE!Fv>nG6bYC=c}El-+jh=3DTOVaJz!{XcJL9BwSAn6h?T8EtiPY
zqnf=@7owrE3vl%k3{Q6dGM{dqPsa`(1JR2d8@$^I!YKkj@HZL2YQ;(U_?D&*yk;gn
z8DW>SikhgW5W5M$uN8!ZN%sjZV&&T)(p#u#bg%EYc+6#{Px#V%ghIdH4sGl8{&phB
zZOT@+N-s8KNcK4nCehI)CA`^{c_~u^o>-?YX)=HgrL;N<I&ZD1j)ZwQQRg-$z@*3N
zSueStt5l2Zr^1YH<ZrxDcWlEBMjuhc<Pb0!S|={)O8=QI5aCPBUq7~tVO!x$fd4nV
z_0N(+rkJQ_$oT!;srcEy^kz@a*-R3IUAta$v8L<h1ZtBqqE`4;;xMgMg~WQ3l`x<v
z0I(VXZ`<j1R5cAHkg%IuLExm6p>*z+^1yAPK50aEjzHxKkLqQBcxMboy?OV<;k~>I
z0OO{@2vicS0H`j4;>=+W5@u|(00;7C6aW>}03Fy8MRoaO2}_3=U}0>WNv6ZTM{V~O
zg$kll;~pi-D#>f+ee-O`X&`L49<r`XyPK)x&Ym!%uz{NA84cynr$biN@`-M%vgv}t
zz^0W6a{MK`+CNgPg#jXug5GLsB3mN7O@{X+%&d@&H+>HA!CtdF;dh1_!##bjy7%IY
zu=;53A4M1@sb6!zv9DNUCKA-_f8dk<+)L!8Ml)YXe!pY;=70Bjq2jvu%U0;bDW8z{
z4<<mQd1dFlueUO0aLX)g-@?NUO^EGCfhDNECbK-@U3;2qeR!A#GMzl97hEiZkIH~7
zs-f!`)L8aFwAy;tN}-%Mrh8V`5{dp$*RCes6|<X=(GUk%Los-#65?%i_C-K;r@WyG
z*2NmtGV}LBhU!jVzS9G)7a2D;bgFCn-GQr+Cx)a4F)ek^Uw!EUW=Af0%2grTSJb~g
zmW0OCZv6xBGA~xdd2u(lDvQ4qdE5<{nyqLG93rGiE6{OO4T=EVOge-uUX0iWZ_>N{
zK=<m2Grd8`4$rFejmghM>i)RmKmB=2YG-NvQ9e-&zJ16Rdr|bQB3x}TeW<!gn5en*
zurb{s0BU`HsDJG<Pwv}M`-4QL|F>Ba(q2v!6BHcVt~s`+Ns>ZuuBjn5^jAY;Nez<*
z=q}$}TNo1Aw4B0TGB`Wb0JoktEP@>u*H`A*3}W4sM;8O;?M)A61{LE4yM8y655Kyk
zXD=c4B47dH&@?>^5xDRiWYn?Hz!}Dc!P3lkWzf6(5xjL*(083Zui*(#Dk!TSj7Ypz
zc)#g@lq@hSK{QGV#zTDOQ2`ny(icFVXQ#;~P~YjF_rQefD`jKRWu;`)7yw#MeC>IF
zCVt1Jy#NJrH_bRA9rsY*1jAf9r%RhVb0f|y_*D703Xg&^R#XtDs1=3p-(TCWXUDCA
zXWTj;3D>et$kqv$h0!CAq3^nY9bhPqALFC>ROC!AK0bcs|87<Er%}HN?Y>+%=j3+R
zf8B~0iU{@|mBF1vKMos8XBY=__i73YUt)T?sBW+hKdm(Rb&LeemJkP?JgawLh;w%H
zDhO%YK1*NOqe!Z)Aw&xNg)3^ALYk%n9tK%7cr&N*VACt-*4&brr<lFv3m{qbPPY^U
zE56MO*rc8#y$2m1tg4Xn%7oZ3B0qkbb3X@EN|zUg)hpd$jF#sF;eRU@5m8a6zQ7<S
zdkKOGCbjyfTtNR*J!j5X4X?gNMtL~cPj~h%gKrPH)0s`@$ICBqC}GrhGUMefn}ko!
zfb(I2QpQQN8bFr2Lr*rtR7~9;LRB^+EO2}@?nCVIxiF-mlX0W|vBv1N<<+FGEtZsX
z?tj8Lj_xAzALcNEf*9G7n7xW%yZ=C4HyE)%yRGd(=ecJa{mMdGT6zo)*v*CMl5T~Q
z8QdrnKO+Vuc%!{GY0s=rEW<W=yhdTEZZ0dki0a%I%Y6cD@Lh|UjI_6eJ!2Ug7~=`X
zVhC80bEmXzsn1c-7z+(UyDH5LjCa%z;=`WmmmK1%PG-X-wPI%&bO(r7y3U>Gj35`N
zYB4lub@NS<GB>~x0|^GvQh28iP6m#Hwf%eQ)M7m`_kd)LhLqgxRT#a#4#;Kiebn1e
zz603i&Ly1VgMuKb2M;(~Q32kU&CGiC1kves6f`|+$N+bIAlmg6-t!eGxLx}Qw}vCx
z-=dCf3x>N>TZYlY9rZJ${;>*o^q7bF%}jT_GoE5vxMBhSzP<m0;glEli8GH4to}ox
zOcg|DPwikiZy2ej2?;$hz(mr=_CRQIvLhTZ#{-mrQ@ZOmn3dCWpE$S|+4LH*YogfE
zb>stt??fg*zEOK)1dGeio)Ch2l=M}NG{qziX)S$t_c)sIBou^ZQ!i~S1j4xOnX*?|
zMBXkmKTYu}OKsgvprmSwSKtUZ%^9`6?tr+e#Dn_k9z}9-PjBE!jtZ36{<>->&&L|5
z5SysDjo7S*D+;OJ8L{Y3r)rougJpQmRo|(hSy1$XkpcBRmfW9XZ@d#q^riY+Sv`f{
zb>#%8ZYxP8t${_<)BEj!6ET3#2Ko5ihH&DP7>=q_2z~^SYvQ-|#|Y{jw_yaC<A~^i
z&ug?TgDrMT(E*|oi??aSf1`WfKM6*MDmBiwEQb&SLUJ|*=lSVz{5NbTgJHz1Fx5f0
z_K+#V4i;ZiJTHBuy`UMz6rvqd@ZpvVLtJcT$NW6LMyBgK&%TbHK<u$7<5UkP!Ac|)
z`mc7t2QF_ytO`Yn8YjrY?0U~4l!^KNPO#_Vl9sRoSuf552zS=kb6dG?etF}{SI{ma
z-5uhaq28&G5jYHzJ|~{6>dov`Y9T$#nnzn!$RyJkfKlZQN*&qQc(<Yhckwe0V$C*<
zFFFCJe|_!giF+a%ek(Z!?}+;;Dq%jBQ;9oGN_@fKU`n5lMUaWr>T70SH6LpL>+chr
zK?09T^7gFl%Daw4<t=UcnEhEONXU$E5H`4s45qsF4)LRXi~mnbVDjKqldJDWintj5
zt!UA1=Sccc$=B-$*huMwJocPtL44ICss3~r+@JiemX4Z{2QlsxzNfo5HDLuWsX|X@
zEF2AxU)!y6ri}LK73gqm)CrbByrnDpU6(RkT|(!1{UU6v4bEbr0JjbH7Q5H~Xgp~Q
zFLK@DYBws`r0dl^g`8-xFW3p@tS|sIP1I*c8$A;dkXk_Z{MI5jV8oZ^VLZ%YqDY@5
zoOTsRZ}H`u!ONwO_ka+kR9O9Qv=VLkE^zB{x&_@7q1`shdVB!!tdpwjNS7YVYpNoZ
zf|pOUUYyy?`Hf-`>OOX}7{eMIXUA``>CHrDHHN<NRCp#szI3iKOSM~i)wI)jI8tCF
z=J0vIKO?Lg4mS?KTRK2Zwge+bOKQ~&Wr}kIXd};ZAT`FkOUFZsP&;xiI;@E&V6S`O
z>L}IgjsSz7FNS%+*O?)d1v}i`I)R161@GUQ3#Eqr<&pGL|C_(dFSIj@SN1)fmyG{g
zy}*tIMvnENr25o=HaH*&egsnF9<UX7Dbo?hMDQidK|)`E_2`tiszNL`8J@|A$>Xc)
zVg?E{#4+Swb$f3`vVL1XJzRcPKpGSA&#M^xc^)hgXl8mcRU6_fma@RlvA>{`3kNfA
zXF#3`%co$U77IfoTKc7kh5MtWW^b=--IDox<>kv$%!gjeybm`fqq$fnZ}9QkySkyK
z!x9V#6fwa!KjcgOid<NN6%==9$)<zn0o-;rrNZG)l=DJ<${FI3Ble8N8!-SG^DCvv
z#rnC=yFuSe4wiin)b<ZY5BK#OoQZ9F`RK@BxdvZW-5hbIvE49a5pr6HNjX^X2-*ts
zT7k*R=@lyiOw8sXnFgw>XS?hdr&;zz2S#&S#p;u{Y(BTM+fQ24z}aP_3Y7yIm84gj
zb8CDrL1fQUK(hgp+7FJjtqE&O{!w+eQ-<qWMrVIsI=bPhkKvD`I#eY(VwvoM!4OfF
z1Irm$-GL;3w|NGnW4wW9PCiaN;|WLfEwCENRo#^o-T_mlcdRGcVM}5iDz`pusKoFw
zPM|#N`nW{HQa%jd?2v93F)V_4PlwaePCRr@?Lww*U<7!N09#5?q+C*v7}M$OhDCCE
z+bHPBI699xZNu=TiB(_Qo*Rc4zA6>*Xe*RjWw3!7J0MXcu^XbIFLsd+>^A}%hZ8E`
z$EqSg1ECgiE{NC+h~(7Q^Eljp$lWG?`}so7Et90S0{wcA&ybIE6;e+{Ma5?_0icBe
zAJpV8Ke~4hCM8l>2XdtXK9RVAbZ-bmi5SfQA68@@`vDCWMi=>vJy{|m<pZxmMt*fp
zyxpzk5KYb(C&EPVzPMARoCtWX+z}1KEDg2I+C~7(rHqXrmrCzloeQL=7Kq%<c#j7B
z;eMt7c(}O$gC(ca5SUI<qOKI2dc#@Ks>GOsn#PBq6n0F6WJKxtD;{vdvv9JIPNsO)
z_jtEJJ#en-{!58TS>s7S4huw#pF~A$o(o-A?lVeC>MWX)bt*64qrbBB<57RT3RyG-
zmtM^0Y$Ltb!Kc{tBvNCxwRLg=Wk}by>Iy19KQq>Q`s^76d%O!JujkMS3bMkbG~&e@
zoZR}Paa%r=2&;pP$;gbP<uc~SEBCc@Ki(3q5pT_{o5jU!z0t5Yb>`D(0#Hty$)mqw
zxeD}4tR0qn4fN0Ndu<;^I4)AQeu;2d9AzuA_H9YDoEl^-nzs&1pat1|OHE0L5%Gek
zVR<9`^gb%_564eN4rtrt_9Ik$M?1Zr*TJCu%uq$L&JhUs8&|*s8l!mRim}E_)-<bb
zp3YT-P$m_GbApnc8L3DkO}(F&s-tz<d1J<_QdB00=mOcr1uVWO!NBi6+gaKW>?rOA
zcn)^*q}oa8`*9H)=S4+FRH$xydn4U_nWgkOGzlNZc%?*7g=D|(qyd<m6q)yPm@lR*
zF=xDLL-H+hH@N98H96adB1m*}GrkpOR!qLndFT&{`XrvuJ?{tga;~FIe}_jFw1f9V
zZurr-XJ7<vW^34d;_Ke%Pj1`0BcSI^)1E;JCm`h|Z|eQwi}@f5=hHZ>YntD)ch_k>
zu#(^0o~%L{rAU6E^HFEkO`U6<61twMLLO#U8rbNWCNcW$W)n25dkjiafvZLMyG>N%
z@WP_^m~0G^TEw%Z4s<RR9pKu3<fi|yxGj*VDH^-4oYV4bg?(;xEBo?ZJL|tlXe&a6
zEy}yJ_KPQura-8$Coc1={wp83A%*Y^Za%$A7e?~w$Y48E9`pBQ8rxUem%3&Z%U@}0
z<{Ns}vwV*L>kggJthfe#H}%oubivOMz~H0MJcw<1k=cwTk;bjqs3_PA-X%JyFU{c`
z?te56p?-+9!}l!i`@fH7uK-|4aGYWOYcF#9MfS|(x-oh@?TE@vf^~E)?@-LImUz?$
zqCf&y;VV-c%s6+|>kr+sA&gW>5wl*7LTeA*62*6;Ps5{Jp?6<d&&JnMPig8O2Qp?d
z#2>2UP9WuP;AKqq7#4=&C8NJE7Nc_ro%VI(nc_>Aj{Y8+icDtnp?Og!r_U?W!g)1A
zAil~3m<RQHT_K>^Z;SeK6{sb$BXgRw97QhImz)wbYF`f?u;y3H*&F=-_ZC9PPHHhJ
z#mQ1fx5x#>@cUd4At4>Tk$Ph2kH1Sbb+y;yIziSyxGqC99yZalOSBINj_aKQ`AoFX
z#}q3SW6#JNg1RB-+>^NY|5Qg_80UG8u3yFt*tg6FG(7+6yFvwka^+j37GwBT=A`1$
zx-$B6H^e6eX9r&78^>*a;3Z<!VHB5LJ5n2>%*ejfU+k7K`1a`jL31^V@k#0^{p2`f
z13&$@HIqO8`1f!i(N^f3!a33^wGaUaT_hP$^$&W(@cd{+yjP35J4N(sr*Kg`x(hYY
zF>e*+?w6fs_bYLinfRhg)G1J17%0l9?MXRMCt>uy`2%_AmIdiRwXV^jf<5)?8@UmM
z8WUq;VOYB#MVZ!4oZ)(C?^^T~D~lv^Me<o#pQuD6b8-8Tz$7#Tdwp7l`~>!B&~*Ic
zInn7)wWxFC@`x83{}v;o>GhB5a#BIFC!#6XUT~=7<jB~raa>lHWv+cD6H628n1eep
zmgHD0mG+gh$Xu0(X4*Ew=qvpsciW{9Z_886Mt=s6q2J-HcwKRf>$mI%WajV;-m>%H
z{<%pEp{IY*-ryPNwVdcYdc<F_!}mvCEUXpxbBiyvww1Y>8=m>$rEGO6S~QEg`g)(j
zX)ST>2^iqmHrz?+v8LEPV$q}3SpH*uv64_{-usXB1^UB7OQ*p}#}T6u`boYU`O|Ou
z0#3O6c^aua@bYBV?0opGR1HV;nzBWqa#+9LC0IpG@hEn%{5xSESeg9e*zZq(B0*}(
z3%h%dqj)+~T3a?NZb{u4y|19y*Bzch;K@OzwHnqkmw%-eQ)(d4?5J>z1j8{Lvr_V|
zEvwhfU4}|;tAFWZ-p&k7IDRx5hKrM|hl>j#*#76@*pZ8?TK({wt{U!#TwF<v-0^=e
zF7nUCWrlvbdbE2zK^4_&N4R^OYmZHDH8Onk%dX&tuM#DNh60%uu%MFyQp_bU|9Kci
zWZ>9*LG=2ZgWp$__18E?d%d<n4Uq}vp#xUBm_b9YT=~bszrQu?UoAkG#e-CfU%$aD
z0*jl>13&iP5km_T!n#&s|G*v6^z{AFinw&N-%X}dv5*h(KlJCVVikG;e4h_zF2JVZ
zu#x$Xo0bvg<h*}8zf!7Mdy8Xf^XEdwQE^as)#N;v+fAvNIQ6mEfR^w)MDzT<2cb8S
zrS{qEipH;xBcp;Wwb@3Ve?A>U==`6hmUj~8?9ry!NOH9KefZp5!>aW~dj6`x*8LQ}
zpBXJLtPO*saF4#fO4Ki5`tW6y8aE^Cl^I!75YhfM5gOi~L40&;5ys2ZAr{o+e{K$N
zK5HNRhw}+=Hy)1ipF0f)9@n3}C+2(_t1TW_<mwzG)m}kI7w#PX#O<$(xV&3;p{=f|
z^ESJ3CbLIt)U#7N>OW6kQ-1kWEj1|omC7?2r)%LPMZ*@YbrXY2wVj7<^dAiqW2a*$
zzY1UJ{NUCx&j+y-k2bfqJ47Gc@7RimM(8Z$%1MKqT?rH|%*@PV(Dwsnf*w+<9z$z*
z5A4~xy4lJiv1J(e@tbyr+vwY_rI!~b3-c{uz7SnZ>Z((32><3qh5sq+7=GxlA5WsZ
zsBlZp5Jg&juTP{jMCT@LE54`bQe-&Qm)IqKVgcjm>wbUr(a$IxvSc(0r_o568HIWD
zgQOe#J8Z_zGlp^(O&1<=fVl#du+t?vFncp<W2T>W6Xqsy2CTgRvI$1<sc#{F?<HeH
zC7+s+Vg0a4e*%v@nwI5QEYB`o_J%bE?N@bEUi_0%TYtO&MrWyh{T<|=3q@o1OwZ1{
zVr|f^>!wAWxa5rarTX(jp=N1uw2A&B=|j8%+<&j^pG!nxQp<VaU{VuAzWe9%hoU}^
z`T5efeSaCkot<1bS_q1QZ!kZR03h=5+yYPxyJ{aj1dHK|8E$fN$VF3M^RhShF$iqG
zW;ap3yt%EqX{+GZs!lZ{Lf)eMsgAGco6WX~!H~wLDV5|uH|Gv!uMp-(-yQxLVNcAT
zH77$IE|pl`oeeur^3lArCERfFf4p3x^C&C~Um}9I>wmrx3Nx6AgjhV%mON#=Cz!BG
z4soN^toINFxb67`=FH4YB!kVAIrM_-2PwEw9GLOi-A_YeQ^LG<sy3$lQJFrQL8r&P
zo+@Ja-&*MA@_}rtUq2W`u-i(p2kRz=WDE{z#)dGmL?L=p#Jlm%(HH#wvcG=*d<*X7
z?D>|xzc0N|(kqV2@WCa+92PJqK8{QY=XxRhmm3m~f%ZrPt@Fw7@NfVtElYr22SoX6
z5PB~^yR|sv3zpsE0Fz(1iiLbf5*4^~$!F0|g+m+obkWUN?gADAGGEV(#gQ1w9$j+{
z_QVA=ZQ_RR^xR5=HT|x_TTL<sjQ{zE4ITy!o}dxK7lOb49G7IKT7mCcRLiNQgq@o$
z5#KMhC{f(Q45H!t9CU0rTUJi42?pu@zznGfAUx#j*YO5S>lJ{WK5`<U(0mVbbU7Xz
z<Q6eFVE4plb>`mY%ebnZq@DYnl?lyyKD3zR)8s9N>=0VdzWU=9F*L8w8WsP!d>?$B
ze%y?$WOrQ7^B7m2e><og|9jE=wQhgkVxn`%av;EtllXTzFpnL>bnhF~XA^&oE+Uq3
zL@^CKab_xj@LePrV`dY!h4r{&!Dp`CYsnD-Jhz7s4`~ObtOtsY1K`HfxQ@KjC>eaC
zHiDL6vq>X`=atWcvG4h#*3O|egclx_SX2HOF=S7aPTvguXF-QCCVTK$9?~dP`MYCz
zBw=0hd?S{~{r5un&#%WwFnLLAT2KYlfB!ZS1AH7@0?fEEg=FBCLyl~9H6CR3y@6d#
zo~Ific5#D`Khq8n=Ubj%JRKepK^W`&>@;UEG&*rciD(#he*02r=<^etbbz9MR;lpb
zbu~x_17PtzFl`Efrh(*Q%=ck2rTDT~f32zNdQ8R9a#1l+<R{<<h0tH^0h7M0oY|Se
z5c?zrs~13GG~a+I00Inh=`S7jFgr_u4MBPxjCKkjGL&gysWp;~9CB2GVSYPvud?fI
zXQF78zTmLi+Q*FBxis)Oo`NT@WZPhwB`u6#0|4c%I!#$p4@|324G_0EEU9==d9?g&
zxi$BS3FAFUGTO_~=P@BZ7W&$AFJbXA0siORKH|%bs=jZzD004^z|hi|>`Y3U_r55@
zxD-+93<j8m0899lpRXlkReputktiC_ovxT(&N{_{F-;OW<R#RJJz+Z8)Z*x*G4@j-
zTeI!|_5#Z^C}U5wu-CPC38Bs=6Di?lVD(_k9dq;&`n1bJ=-#hg&-36Mecg@5JS?-6
z%Bdxw8mGg6>e84KAX)E4tCuw5#%fo^fxoUdD{{Pz3e}k&h*1J$X%pCn5@iE_z_Y2H
z`>{9d(yOjvdgcCyA6mf_756{V$j{x5TiuUhb94j2pR{mOH{5ow)eR8JMOp)PQMQc`
zPH#f&GM?xna9{Y0rS3knCq5(OAQMqyR-9-#kQlG?4N!D5gt^Dr@fmLJ9eHX?Y~8wS
zI<f728J{_?4+&0dSsyCnHCh;=9Fj)_AGt?wNmQOXv#d#D?YniokEZxRXP=6S%KFA*
zp1K~y2z|}~`@KAMm>IZ}hH+nL%TS#Md6inQKTl+Dq-{=~cH0k4ZJ2VT1=77L;?xPS
zSv1eY;(0KXd-pnry$AMTQi0XnWw>?!+nxODe^CrV4Yb%4=Y{^>f5<-Juj^qMDHQl{
zmvV=?wh9HSkl{fT6Bf=GKuiKRHR7y40mGMRp)(>h)C<tx*>aqU&p=qG08G{H_(nPp
zNdpX^<#;+9Ag~3eREr*<d-Aqu&Sh%L6ZJq|c7~)(d7dp)G>7iG2PK<IlkP&dIAqp3
zKt73CN?fM{(6^*OH5bVH{SfkN0}B#vP~(jOWq|+HsO$V?gtj65Wh8|Fg9~<WNnRKh
zM{?-Wh`O$G8P$mZ<qny>*-m|+)^8205Od!dgWZz5@jygn36MEIz_Xh4RhlDECiq4j
zC)V{Wb$vf%@|1rLLtP?aRU)d`z0ym*oz>fTR`cYCM_(8pGm_yEVrT_GIOX9LN)7?p
z=McCIB(P}N4wN$<0W}Xcq;&ABrx93fXxA)Q<?@_&l0jLu4^}_qe0J)xnOPVQ144T;
z4d+H-D$9>W382FXEvk|iu<nl>kC!X)PmlR?+WmEm{eC+b72z-oqsJ=y`%fC!uw66d
zyS0k6%BMyypxUm_3*MAvkixzo7vlx6@Gz?`|H{@}Z#y3Dcja<C25`l6RdK|^@Bmd#
zz6JU>%gv-OCsqS2bU3jvxOtKM8IfXa3K+15yi~aJ)FQ3OJ`7k_EEzj|FMe3$Q<;`n
z43HIRbw-nag^feHFUKsUG%lhr@}t&wNa6C1ziMCgPH)p`u7~YoPVI>lFYM3PP*ZBL
z+PU%n+*gww3Z*rx`9u@5ju#`8f_2?OiLj(F(7-Zz33MO5^N8&mNlLr3!yG?*4Hz*(
zQlK<ygkH#j6^NsxN`oje80PD`G-X_&l2p<V^HiL*hEbtXmKs|mB))((c4=Lr8q2}>
z-u9Zz&6}Tt)WOY>_Kc`&kG`$68&sf$;D67wu<EC5hsHdvBOxZsu!B#LMKF9-a^S$Y
zY^nX!M!m!2tv=hx@IN)^e`-tkO)o6GPpAeA?}iJ1|B2uelwv6c^S4!Lv;0%?=0wa%
zYh<cWLo#zgAQ|M8y~zOaKPlqaOxnR3Vj<S~{9<&b;J^eYkTQmWKI{iaLiOVvRentW
zH*dgQ>!m3TyLuLO<x3R5zSang3>4`sMyk}sEk|pULC}bZ@V->org)6Mf!TimxbITJ
zfq!G_!xSO)0l%_~Ro#jAgqTb219k4gU){HKVdiyRz$infk_Q=R@LDant5gjOLTns;
z#e=O?POT#B%Xdn+CRh~WKPh4-in$AM=u{-PgJ>(4R-p!*?@5|DSS&MvfON5i1^-DB
z>M@VbY~^s(*NL?T-b0)leX(rkF>py`T8JFth>hCKM=}ydF#?iGgHs~0uSpreL^?`3
zVa0+W^s+X;;UWdzW*~pb9)yu>Y~(%|Ip`}(oZWx=Ith5S^uTdx^$I-KWZ(#zoP%e4
z-@zO#FMpcwO+6hAo-^59!3%gLG{@DdTB|Oe?zOb(R@Ygb>@@20XFdOasjL{#Eo4d$
z?>^?_`OhJOY<(gHa+0GPefbuZt-i})1@m5mT=|<L`nGeMUInG!V27($DO}C$o(uW`
zFy!kIfZ?MoWo2YOMHZrshkve&2l416g2X~WRr*-eEcm0n-UqsEUh0lL!1!gelf}6_
zBwVkpN2*GV2NILKpL0K98wK+>%QeC?f}bM$D3#Ld?`(>K%4ZNPtFvFHrdEW{DH?0O
z%<R^+oF|O}5-Z=Nax-Sx>~qq`8fPil)e7X7TfOdj2!wwVVt{0$8g_e*X-Xcy<7YEc
zrzC|%PrG!zI5VwDHO)Hz_DZLS;T>_AF*xn-_>-Ogew)6x*@g2$9rjsarw>ZE5qsP$
zg@lQG^&Cy^w{u~Rud6cKv+49P_04ZpyST!HT<>~TTpdpiz0-zDN2%EtN7R&!zP;OP
z+RQCwesLdN-xl9x(qE_XUuOt%IRE}%#tf_)36wjn)Q!JtK@_GCdiONez?#<EpUIRq
z-%1?Myk;h0#DtU%S}0?TBfrA%=R>0C5T5Vn6as8Sh?>tn@#Fx>28L=&&-MVDMKMtd
zC|W^ApHI(5aQ45xbsCA+OJlB6a0u!l#I!s4cvto?4d;+HhR65>;sXB+j5J9RiB-f0
zN><f00y1*)m;8@rUqL}8VYy&`U0ZB&(+A{W^R{gjsQOwIwS#8b>rm`#3jf`>O2{i*
z7?rZZ965=saq1TCp1r|_|0L9FxPe*VWUK5d5QpA@b>$BjX-o9UuFOS#J3oPQLAXCh
z>k8~jpZzvVzw(kDpLKyTU03k~#o9#hqMb|M%O>2Hh`N`gWA%du*qFwTQ&X8+<kcF_
z!B!ngn=!$k1q}*nkSs8QwkY!MC5WHHG(HvpDrX65)EHDRZz$fDFq53c7OEwts9lX6
z?wQ#)_p;uNlOp(gWBSjpy;PCCi;d}x)8D@>mDu3f>l1tCc<#`Ovhdij?a@hs$w$Jf
zTgBTWyW*j3yEJ?iAs<=`$r>0l*zEnZ%}`Z1N4xLsSoF;d=3dfeK*D$lt%rvIbL-Sk
zsO6J^=pp^}w4O5t+-$zar9>IXGhk9vM#KwW3f|PR=^emm=^I!aqQ2I+1XUb@67MHW
z`vVKA(?R0%6W%4lFpC9t?XpY3bfRW0Gv_dFkhf^z%^cJ^l6NK@6SFW!9)r|Lw8h-v
z_P#hY=i24Ytu_Yl1(59<6@!-%v1YsS{T)~DsoGw65pY5EUEPINqQa{<0d;Y8-PKXb
zj*RrGm6NNL4U<M61)+S%=-$8FA2QoZ$jvV!FW;-5C$!Csj$u-oyq!F0GjC;@*)~?n
zXR{LM=*F5#NMFjik>0;J{!ioayMK5?Ha2)g^PZgI(b^AYf==Q+AwDg%y#I7fB;1%I
zby#+kg?S#$9~%b76y?e}0^ukOL?z>3)+wCa$jJl_A4US5Id}tu0mwa=k0)Nop@?!l
zeM!?G^||~ZGy?pMbt2HDHiA6j9&;=x_C0v&&e^O<HKI`b`x==l?{X<OzgpLA;~<9A
zVFtiH3hg$^)kzd_v6%{VDQ<=F&nIyH9QoA1PHfCP#)?@PfVt&vK9ZWs)UdwUXWPX*
zNo?H3dxv_oZ*}feU>T=YbetQ}&l%}4m2gLq*YIhA8A%kd&`J(9C^Y5{Ax2)2@wf+Y
zmmMXwqXQMm7|^^VEiZ0S*JO1T;$O|Z!s&M6i=n+?lw<I^>~3Cp=b?h=u3+=uOEDBx
zfL*ak5JlA;@%#Ms8b?;Ftz;w4zo5u5e5t^OT6=Vj28hdwnQV5vj&_olTqG6qdDTCF
z$h&J<n2v7v_SRP!rpvrZPCHQ47((=xbgi9PU3?<FUo}XV5fU}E?BuB@P_EP5eW=Jx
zfNj+63BB5I+}-)pSOL&i(X7D7k%V-Pz=HJm-N~r$5fKrE4znuY`AZCTE$<;1%`x_Y
zo{ucUckI>Y*dUKwelci2#qgum@`AD&&QD3L8K2AAbh|~I2Zr>gJ7Vs8qiG)A>92T1
zS};S6yk}n6XKpYI5@I(|8G!pRPCpEa$6uJ|asO(4Q@v4f+^DDgea9^l2y>jfgg7Wv
z@<b&iHtb@%3N{IJm$i8QMZlcN#|FFO@XsGDG7@8K69y7vtmdhIOBqxXUejA*<A>Sw
zPA4NvrA6jB#wO^F-gXhfU*Q@6Cbegm8Ca3XWTP8iT@OGC{QN2`N(fpyfxO7!2pf|M
zcrc`!lbMjj2jl@WW~9tOW{FQmM4@7IhQ<`~xD+b{-u8{=>x;`JiVXN|JzzaYTit}m
z;CZkoO{b`+q7v>Qv;bw=3LLbCz=uhcrMiG_fy$c>D)0F!#RL2gMFX0a32$Rn4wOz+
z+4|#@J^XMiFU#zs!S>09W&50Q<jl7@gEkTPm8CKp#l@Ys2-2Q$!D0z!lIPf2qZqvO
z_eR!J=8F5{t@><VhDyKPVWrq7x)EzcOVpO*{ioh$c!6i*P9dN2uI%Ved7TF|M)Tgw
z&_7=Rb$3GmN(5Ydrr)E!vFqvfA7Ay(-}`av2IgJn0@n>QUbCK-3}uCHo{)+YSkSIo
zd|zJY;fP~i^6XW|IW?p9sE?tqdr@Ox+T18fENT=<p7XvN)sV+|dlCsMgsHi}tN8m2
zTp+|JpvzGRBWl=~>B0gwOJdmPB(xVZ#aW<p;y{F*!Nl5Rm^N+`l_4z4x^V7b>}!f+
z?8Ytj?;Yk{gBOq8e^?qEaKTh0CxnrYPT@fu+m%nM34IBZZD-7{@nskDaLDueQ~6nA
zLhAQBLV;l~K($g}7vA|O2Q^Fgw0UgtJFa!uD<ve@@073_ou%>{Dt<r=rzVzcR(hCI
z6uWwxOufYb&xj{n+?+W%w;u3;Sb$v?47(Ne?9E74G#uBn&GFD=@5tX6_1L|f>LJWx
zT$yhS^X6YIVrxExgYKuL0VenoOt=L-4@7{I>I)sftN{DKbLt!sIQ(I9{RFHZ_><>e
z(>HQaMW4TQ3c?PDE(Q$eO<Fyv0syHyS6z?#fN~GSAU~(+=e4!wdNRF%VmfHd*Q$}H
z;}0Fm3nNoP3Jx93`hC<6TXRvR-DD%Y(LX@gJ?B&Gv`CmH#*NC;F3&sAi`+82cI~lM
z=%wZBE{h}80x{oX`1|lFSknXbEUmuXQg#*3rA~hB<NLU~|HikMBroo?@bl_uvHyAZ
zQ0ru4JFKqngD<ljRiOfyFILyC;~XIte`=&U$K+*KLZ^JqyZyE=-01`QS3a`brYfKh
zrg>2M?WX-hp#9(cw5Up}GaT?q=;6{CC2E9_!8Qs<DQ4S)t!))*yqGps988SpbOLNS
zmtOqsGM8RL%(M~!-pl0YzxDy|>&0p&;q9_*h&nL2xeH3A!8FHw$-OpIL*y3d{s~yE
zLCskTx!rQA(o8P~EgrDHWly%E`~;@L^hyJXmPRq7VMZ}TEyJeePN7!?PEDPuKJXVg
zwR5^vOfbQ+PGxgv#Qv#|Tx=#*SPwmJZ!SNbIkCtOuQDLEWwQ;1w(cE(mdb=@8h7VV
za)ifdVryEtT;RA)Ue9vnsY{@bO5>k=q@KdRU_O`XTDt(kW3wWh6$i88dZ+o^^1;4C
zc${%X$he|CLAV@{1l~meUDE`&8sJW3<>z8fPF_slA+h1SGykP9#qyE+^Asz4p6BBi
zxghl~d0q(zukIOvP#zRyQI<+REEu_0bt&1$3ES=53ReR!f_^^D>XtpF>9(3Q-XH`3
z<UeOqy&%8AHcq$*Jy^k|A>9lDC*Heb1JXX@4hfg+t3E6`nUiagKQaGWXpYsNI7^RJ
zBrpEfrj}Tf`2Dkg;FND&7<nO$Z)11lI*y(=$+fd%gqU;M!hho_=wq<^%957X;73{*
z5RO;hK)YunyBL~;l#2TvcF%o|l=V)rT>`{rnUtuh7+i2#@MAfm`)IKzO1=QJTlEZo
z=PFbwd%%xM3RASqdJkcNfd!l)@vOT58Pdu|qP;Q|rJt;_VqpO8(oEui2nr3ZMyEue
zB1>AfU3uj`5|a$2HJ~L-i+~O@Y-8Dim@s4Zlpo59<i{lqg{kM{egL>+)lWF|e~9gs
z?W0>-7$^pt9T7ljzmvXYz<=V8n!SM9?a#d^TyQF8tH4o*S9biaJ4NP-jgYq1mA1jI
zwzETdRxK5s>^?<;Oblzq!wV^ao0v<LgX_V-*2J|l3>Rl%NodEPM^^wt`5T<p#LStA
z($shcVCOHrQ)xf_9aPd+8kguQw^5cEzJnFHNli<1)2r>7%WW;kpb2pB4@@8C8tag0
z1F(k|X&zEnCc_*DU^Fs&(SNieuu1}Dbw>$@$YVYNoBV<t-b|$DR^x;ZW~B>biVm<<
zE?eIVrOlb(FGeNgC=#C)qA;<`cX+i%&Z6*Sn2OXdVk6g$BRq}_)3Hs7rtKI;EKh6a
zy@L_BGKTWRTA(Q(kYBmJKwe?*Q)OF}<tT0}f6;u%Mld};r11@dnq#OXSpVw$LDJAB
z=NmSZOM=e~$sZMfIB>jBu|h%n=Swiie#kP=)!~_K@hB~c_fef9#Bt&*`Q1H_0SNpa
z@oSIFQz(*xT<f3$aYZ;a7~%@;CR*+?KPS{Edefe?HcZin^h1EC9TW{Tx8q!4>z)(?
z0kdD>H3kKKf=XHz2t&t%tdr6YbEXWTx4r!gfJD{-(63h6dsSB2Bt%*zxd-xx7md<V
zB#ChLUrQJc05S3@K!$lHDX_@7JCd2uek2*mU(+r1O7etOZj9|7YQ4W6!AsY7BaDVP
zy*M|E|7Ll`y`~3^6c=|a=P0mD&;0-v@;A@JwFCW!>odxT^>|A(XHuQ%*X~b>-vOhu
zsEqwud-fYZ@cI31oId_@J)YUDlA9p3k&JN`Qp7?7@p48wIC0kPsW-P+DLcy@I37-O
z5OjqE+JOY;3ibD@Ho|nYS5694#$#DRV-M`?ima$=ph&BC>G_DE2`umE;4E6M0x<AM
zO-)S(ECc}iXbmm+#!a9pn?9e;(1G`Jr_Lmkr*3fR@QRZUiT%zFtE3S1%WEzC*T%Fi
z|Jmrh4DoH;C%(MA`}PR7W>A9J0bV3U_rKLCs)pRpm-F!*BHXNDGttlO`o}zy41YC@
zekkC!Awij{vxnQAo|s~6#jtckyEjL4f$_u;m$z3L$WMag{;rcd=x$hjt+|>(ZVi%z
zrM9f32F`SsJ6Es1)|j)agwE>j-p`f1<V960Y7@wXBc%^SH^Al(R7z^5e9jOmffB2c
zL77!(NMi|D0{wWMSEoDkt$$=`7QPA-zFe+A6+3$ly-t?lf%|EADlPP^KGrSi)N-Ge
z=@8!y<y2PT1j__mFB}bmI=LABY}$vnYwwY9gB$)c3bw~@eTn0Q4nXKMi-WZ^Uk%6j
zU63WtKzaaMZhZ2x$8I>=>ZsQ+MWpB>r)<`BKp-8mJY20wLT}dXt=>X%dh=BS&w!oG
zH}!Ih)Tjz5Kkfp27#cCDtFQ_M!(v|Y$~rFz7h~iN{@`16O-)jME7`;G0QC~TG-?fB
zYGsIfcTqj-35fH%nTdHrm2ohOT16go#7FhE`ES`t#<ahl#6g38>pPr84wQFAo(5Da
z&v4f7-K+BdATu)s1zBYO1450Gb%~vO(?&YO{q?4`5-5V$Nqh9}j{01@hzk#>zn=l`
zdxqtpU7ot=V0o}KJ5$!KT_WPbTr;D1hrQdo<jL3viEm(&4ULRUosQZHAAYe3dSny<
zw~m69GzpUqH)C>uA*Os}p+@eb9E_X<Yzu-2Qh%r)`kuU!yjj-8tzdEf&$2@OL<&%`
zz8wFfg7X@<DjWYl>xUWqU9rB!**L`<E(7s1d>ycWKKJ-gejLgCD2P#R3#{oSt<v%L
zA-M4xP&BM&sqSHQm~T)Q!$3A|l{uMzV)wGDrNZYP95>akfQ)~cQe|0>`<x{&US_Jh
zsOCd42pR{u?VdE|AvZ9v;eGua2_3eG`>GzoRpBrj&XzC4-OHa8=m|(tP_&To7yjIA
zBF6x)OgG0@S)(F(HEyfbVFdjcvBHaeL_E{Yjq!&4;$iQv9WoB!rWa3Eg$XT9@7oSp
zmd8#_02pG{7SkCOC!{qR=>fqJWu{o$hp((SxSpq5QJ%;W=`0-sVmzN?H(J?eFfjjh
zTH1@b7;(=!Qcg=X0kPQc{khr?o;}=<gb(^zb_@#AJD~Eq)ZI=q3y1oTHYQ7_=hO?T
zmJ9;${YGxuYdJZ#)AWt7G4&B-r@)kM>AE=>8@c;Dx9m+4Klu4pO&>smZnY#HGp^_M
zE8nl@LfcbjEc$cIJ8hi|&**HP)dJU~st<daBN+7GQWZD}jMBJM7Tww_FE8Qr_bO=h
z3%R>bJg{<Df8=ek?aA?CBib_kK;`!U3&rZ-HP1As&bt20SSA10Obb=v$beIykwW%V
z+h3F3G5DXdVkH@C1voAVxC;V2{4U|Hbi}Sp4PvfVry>&ALDIRE9Rfx3#$aHcLhDvs
z=Z1IfbXvhW4PFMhKA)pK+B7A|!p(lvWGZWDKU1NDZ!!!8z8x03NM{!FxqXVx@q4<Z
zYvw%63V7kq3UDwXD84!>G<2+(F8ibP+=Tj&NKtFYNM^U3Wp`B!xazcu2BazIZN$BQ
z9<Ash4k+ZO3C~SrkVUkYMC%au9Vr5A?SzJ+-Y#xb@0=}sIT}W@OO>gS;AHh8ViXeO
zl3bO+@X8XzlP@e@MMvkC$}k*e^9-VrAtYUIQA2<+W+%hSmghB0F(aHNT1c5bLfs<2
zlINkB8`d<_MQOdE#xiRTjB0_lQ=Q;B%$M2CUMo&bi2Ya_*BEM_D(XgFA2gMV^IU*9
zmKpa4(zjUA<m2R?u870gcr9!-C*((LLNC;=raufMR#;|iyEWxhT`Jpt_ggn{HNT&T
z^i-$chPOqgq#LpFzXL~uJ}B=#h-A=DNE}TfyE3tq*N^KsU{29`k4yhMo<v#9AYG!`
z2LCR-+feNp+VPex!=9iNi&Mh3-=CO^2dqTO-!I~BTYGOmuz-ZFECR48eJ6Sv&iJjH
z%dr4RnP6eCJ%l<aFO8}b-a)hTHzR+TMN?4_(2MEZQc&Rk{OkcZS?HPST1HJHv8^{b
z(TLMj-#H(wQTMF`4Oh~igOLv$y}c&1>f9=Cmsx1#Z?;k)3W4yZi9KqI7OHCDfe#b&
zQoRrs;yfbQm94><zge;NDrwaJi5jlaQi-;>TA}z^!Boq(8JACao{XI{Ro`<((u+4<
zk5UfW?TyRpU*4HFoe+ePb-~<qX<9s;M~kI;i%FpS*j0IFca|CFB?n7-h4#Mn`tT3L
zHA~(qtmx)E$6C#6iMhf%8dcMUYuzY@3O;3NNS|npc<*rX>2SOgjA6IWZUKDL@md8T
z&>ybHrbA43+BzE>+Z`ANxK9;!lec+yN5p(TZ(lt!$Q(a<dY}|--nm|%-papEaJf&C
zd&ARfKPC8RR*b5dJ=TAtXWO~kWiwhx5uQ?0GLg~FmUpG=zYHS;eDWg3bnhR!QK#A|
z#_{6bP5xt6F5Ok{-j`t&FBku+!Q{q}%^op%Q4bh2dJ6IeLvVu-w4Wk?sJK7?k6u9)
zrDqC=eGrVHAGPW}hI3s)kE6f(p>>9;fv!X#tpL&7x*QDXw_)n>l7Fxw;L<+Qk3s{a
z1nMQ#J&+%+fLGz{zTl(HOAwgtPitL6YPJ@_HUf(P(Osh~Wq4Xgum*~Zy@fu~!!*F=
z^f=5QenOqJQDgQtwqpj26CVpH_e6G3Izun@1J29J$7wKv{p1q!<Pb+8LFg%@tDZol
zuMoqNTCO38M75+%G-kseeW`INmR+K&0K31J3M4)}J6uiSik1d%K`ENLb-i)!6lMl3
zU9wda3c7&6lE!Q}MwA2usCZmu7;><e;xQ=l%ycJfe9>6ux#O>7X^7`jWP*IV2ijgJ
z_XXcx`rYZ=V+~-!Y~vyaBhtFe_eF^`{1#M3wd7{d%O+dB;>ikgMM4c>n4A)L{bitV
zkdavgUnE8r>8X&aOuRO#uFzH?sXqE8D@fP^jnGpLa55$7!SUEK8G&r9PoQIv%p>4i
z`SI2)@yEDP5c4Ht%V|5<-rr@Zzo|boa(6^Tl+Pr<V`-f);atlx#UwHJA0RlSIxio?
zg;A9E0k1zzo|g5kb_(lCxp{vF-h+}GedEAi=Xv(r_W257GoQ()V{@lYFtx+l40PSV
z+_jQu))mwCy9Ge8BWi@qAaZhvR8+YcH_frg&0`0~!~sywd=zEdXEsnwb|6pQpYK~K
zN_$>vDHxC72nHlsLM{AiMp_*I)Bz<3j&SEj@ZyD|*1-E1AvOWUFBZ(7dE`%K5@LPq
z$&1Gwwm;KIuDXoX(nfx?$wKg0l53bn+t(9N)`NnSa*nvfbUgxD?H62xS(<SYX}1<X
zyS9PUF2M6(`@O0Kbrj*joRnuqx5VXfcYI$^>Y1{Bi=766^JAf|KGlUQB4$0T84(^q
zyl<%R5Xa4(`xhLU&|#xVH%5rOZqTdK%Iphr6Odwsl?V>3Gy7mWFq4pC#`0hhU&12G
zqsomL<Sr(+ajlo7c#d|~q`z)v5Z_TY&YK)lE|C~)Ze>08saYn>u$JhYQiaXkjEIk6
z!Vj%f1B^s3(0+$HKABek!i5VjzF%Pv?ZUyl{)_@m<ya-G2xICV3M|=&(WkJ`aBiK+
zuibp{h`Y>#yUjr$7+Rx?V%rnsL+ehYyd(uohL=0>u+X@4&d0jbG7n!L3ON>^M#7Xd
zK258R$lcu$Q7$!8M<UkwJE2R?pzW?=A$g0*?RC5SkpaABp*)L$1jB1x^y%2o`Z28H
z%tp5B1u*nCN`l)>GU54AkBr(x-6;E(`myRC4$CVv>^NOqH!YZs9_deJ!2Xk*sbv3x
z{tZZw{=n#pp`zn8*C-3=a`{B91ayS|1yU?#5)Mqe)k(t;Q6C5~zs9eBe9q`?fQ8g(
z?W6L}OLnX7b#B{kX@vnFK;x~%@~y3P8%DI(VRKx3#@jl{`x$BTtt9Yz<l_aE#a;UL
z*WI&gituF=>_u(Hj)7v&h#9{bW&`hAr_#Vmh!HIQq^WPy4e$!M{j>?Bz9KEY*ux%_
zo&%#CWPtX>o<Tuj=`8m>DO58BI`Ta)5ce>Tw_!Z=v`XN+p729BSIdHG<lWny7FP;M
zPDi03Ve#7nMrwsqT<Nqn4sig=+g@+z&z0mLxtGdjj;B@Z|ILfjTR|fcmDpv$U;~@f
zoCxM@I#n0A*Cj}+R8~sxd*jXnXnm$Ikc%X=lOOggGaJgGRDcvqt)5tr=Z<oe0oY0b
zf`1PDjwJE?(XV`-2W~BdrZu_^r_|k~bzWNyRU1}0j)t&}M-J%bj`Q4^Ti;NY-V%tP
z46_t8E#~;saidKbpeg4KBpZ@BdNdyRChW)cjs9SDM?lV)HttSdzrohC^G*-t%9kH(
z2)kW*cAjV<!0>$)T-G2`n0hK$4t62Ck&QSMA@z)x$3b-eGE;s9qknv7FEj8=36OOF
z@KS8I>-A3@1Q{1^sj~h>%>n~p--&|QxC_kuth79!Z%rd0DnZ7P#{H1}6uAUmwPm@g
zmYY{ztH*6^l=Nn)cQ=R7=OaU03Qmk~*;WdlkWfJFynvuy)r4nR?$G_aUe$Ygcu}cb
z&}8mDW-fG0rde;Z^xA&9%f5gqu3WXF_lMF#p3W~D$`gg3-^*;gowZm_w9Fqr-e$kY
z^jd%@=OK8X-2r%hWZg2nT_-OPuM|-FbZuV*Tv~v6H%A5!%)=d3Jsfu<G(3!%O>Y{O
zUQiRjqcV-;exgCWTCYMaO=a4Y!v9D+t$3$i2(rsn^&Gub8eB%&0lZAJLqvROcEj!K
z>lahhRL>S!t;<F9734e`839Nx>8`PMsjQfJg|H*J)~5|NEQ~G2&y&#y+NIsPX;wVl
zh*;4^w9Q%5{8KV|)NW)xS=^_;gQt8bSKdgTByanFvGyI{RQK=y9LEY}L}lbyDMUk7
z;gllE$lfEVY#G_(AgPE_$yT<=-dRORN%p3)XLk1g{#f-q-|z2#UH@_QT+gNBbk6&8
zzwdp&?)y$~;f4&WRSt{lSO$aM+p!M`y&1_`;|J_~xBcqZD#YV05v&0%Vj10xXSZwE
zzNe9Vok{bBzYErscsK{)W3Suwqdz{UIVyYO?wiKS1EPl10N{B7)nMgg*(S|<W_VOn
z4)({o<g_0)X(U6i&frF$Rg@(w?19WBl3Yi2okhZ!qGBgz=<%nq)lfaubmT0w$kITo
zluvly2K1ATrXV25=Ur)^XZiIDAbG`NRlq209ewBL$D@$7)a`|>UvmtMO&08GPx-|M
zd^~~g#teBmqvK;^F(*$_L5H(vI5wQhF1Fy#=gX)`bcY`ySN7dt?P*JuJY0&yw2V-w
z>ws2!p4->!P;&dt**BLnv{gb|G`uO*H-`^ZAQ%h}dJl;Y`W8AZrC{jj>Z!n>TFZCS
zlCBnuVQojw^Y>z+1)P6^x=bGl(MTajTY=Ac=t2+1>QE&-?YFl&XT7dSI)~r&!a_wy
zHV@Z6FUZ1z4<qgx?Bb0Z<Ez6px><)m)ZDrcgTo~bjV@oVOw7jwrm@!#aF4x>6ee-8
zM%2bPiLw#Xufr@gFaDIScFZ3*Y(m~(6&X)ani1_7PIxU6`)*a?2*X-O$qiA)nju_3
z(~(D4q+6yDAWsD$zB>`>4n8h|9bXTfIfyEB-^bpcE|>G`m`B)9YP*8dy)vf8wBMqx
zHieEd@0lkTm;&GU+7+L+AgVi@`#6Swk`(cA1O01eOdu(P?6j-77?B*HWl>OhikH%W
zpMk`er4r^!AG$1f67W8WT3LSCFht3Zl^&K3lOM#hQ@VHZE{r{W4@+GAtxiM&zVAXe
zCEZoJ?w5-`3>GJP0${e8Xy%QYA1XcvNufdXlHjNLhp`wpl78MjU5lx9?m^GKFCit_
zXOjEy6VgOu?z3yj{AE*i>&pKP%-ZLGnuf;Y+;?SbquHOX3UHG09@;zp`{VS7N+O_+
z_hTUEg{l`ERU|mBJk;^~TK1Ax2*nfx$+SPeWVn#R{puO|J+`lu3<n4SZfdUvo2pc#
z_O=hzs%=BzG&hU`v%>}>RN~v{&zSUq3=7tvC@yAg?n3qt*3CTGv4EEjlVO!pHJbx_
zV7LZsl;QKUijD*{=CI(o{095cCQ?0DBOA?p?rVYa1$RafoSyZLTgI8k>wC?k_L3LQ
z!>@U~eN^?8dpFF!Fg&$jRr6t<2bMPLZXgJnUW-B9DHuJT+>83b8zWvv($8btj@Odh
zIV6wAoSXCTmET|GyuXoj9%xoc0N7DUe0{hLe%|~o<CQ)Hk}lcu<?K7c-!pDo)!HAP
zK&>CwU$)oGQ4Y`HD`dypk*=r_^zKbZLWS24%GMs?#+OpGR&j+-M&BDF|ESCd3lmMJ
z0nor^Gsx}p2@0tDV70GN(1$#sPzp56(G;B<Z*R0GRxty(ISXf1ZQ}qE0zg%#&=X(W
z7(;o2OY}R80mK1-_fQB@WP_B%q3_sp)?pu^0PP=p)_(FLK<xdkV$T6hD?-nhA{h1<
zQW%p<P(k9w`<S#koX@88d62N)pxfUiiETG?+bUX7A}I9RC))P!m`UXK_bC%?+3HQQ
zT@5R+SaQN)@OGg442WRe);@wI^)>J~NYYkNDS?Y18^3;x6F1EAtL#+HS>a$i=U&TK
z7BxbE(=hI$OjOTc?Z0S!r>3UnH`uk+g*D*$Gv+dGFs^<YQgy$-+<J}yazHc^XSrY8
zy5Co&mAoCQrQUGajU)-6=%B8b?=bzEg{(;eCbvrK$OTgop#Tt5Int=+m&7kC0Y%Xa
zr<_wq=6?3CaV%9rrsz7nlq(E_9BvgFT_C5D3s>MQS+NQ@H$!qnPMMf^bxgprhrD0{
zA<>OmiaB)?iatf^Up*B6I|LvWO?o@o;=dr1`x<cG<Md9eiLa)d#?;<CxiN6PlGBc5
zuN@YK-mP6x>w*sGMB+P%#H+lOgGI5bF)2fGAxcIT&xKzsUm(heCJ<F?8*{oK_xKKv
z-yM<VXmWyV0KjT%@Ep~H)XnN>CFH5hKiM?O3NC#yTohM9%7ISMHSNhY25EJ{fl`>M
z@@_^xLZtbbWJLee^P3PVNrpniSIe#2u>r7IdSmU#njlWCc)NK)sKa*vK0>Eveh9W2
zyA0kL$Z_H#z*o`$@8YVBhU&nh6bO;xMZn>RGO$Z*oRBqo(|Pn%irJfWC@(3wLe@cE
z$raT6>E~H)YfiG=ufJ!dz(f&~(LoX=F}-=1TUHgH4dkEID4DZ|(VX$;UNfsQ9?jC&
z7yol3!7$`gm&GE}lN8eH8DGa~s-Ewrz5Z1+g}-96$FPj9I4L9lyGBMUNgw`thqpX&
zsF=~lPs^wr`aLRp^|XX0Z^$xPJ=<H4_2^}@A_(+{TINxk*zUn&AzDite&p9L5j&-3
zw$Agk3)N+usv8-;Z{}lfKZ>?a1+|-7coL$X#r{wcS{?AuqRpM$H{@fwu~_}9q5RUd
zA&s4VK-jmW^74WXzxw!RU*kN!bXm+Eu0BeD1B>Ro7o462vs9R=|H9A_P!OgtP5;b*
zM#lTa!Eq`!$(Id!g|^Z~FyVx!Hh5750+t+b<U?nwS895lA5WK$C^St!E$FuJ=Qc%!
zE$$gO`St*#{ixV3tZo+qzvHCtZNXjLa1)L@Z2JdUv}p>6aHs)~B5|ej8mF}K2~2U0
z`?M}90RD(_3=Jp(4^R;F-Ccl8M$o`)SND-t9LH((pk)NoM@p96tXg!+YA%L3Je0<K
zBH03fTg)|m#Ex<xPJ&Mv?7uk)g9Vh_0}hKmgkt3ZScyde<D+~b^%?`20f*n;??aA?
zOsNBg8@&Rve<b8qG@yH7Ysm7(BrIw61Uqq115mESc8aA`cNY`C^r+`sy(%IZCBi2^
zc%m#2py22RH~Rz&N-RKc$ghE06)m90U@)D?fc7h460s$LwAhKtE7*=^i2L~pNFe`m
z4;pz4p|l6pC1#k$`Uvz03$%H~&%vzpmX$Xa7AGKmq9_Ge)b2|@8-V<|SBPIu0^q~W
zwrt}dI7M}vSy%oyog2TOPotoR!UDi7V9QF4-_h~ulII-i5q<DIcs_1eI$Ew9qfak!
z=qr3FVQ4qE4j_!`Q?&&9zl61N%b`Cxd|+4mcu2@~Ia_*gj|*^tkX4@t_iZPTM#4{R
z1mMJ<uDPVvU(6cLkv1Gvc#Qy{6ZZ;5-#=6B5H|cs9`spM0d~UZxOY{lO{`aelJwsv
z_8T$}1lsQ$s6KfR!zejo=s`#-71S|;dR8A~l!lyo3E6d{LYg=3_q<7U=CdjJy!LOF
z_`@N67^LB)UO)ZVpVzRcy=?Usj0$z{u2nU?pCX0^ELqmg)*-NkupYJA<=AX|M9`Ar
zpsWbBFD<IrU6BZ-k>Y$h=hZ37%NgJ!S&&qJ9EGTIl%d0GVh%xGmAbiBAGxC^C-C*r
z_v=pGu$<;0WS(9=d=QH(gS_827^hr4IE{@sNKq|mdp1EXw(1F+6o#R<ajeOB#$n*O
z8qXtS@$sE*yH<^lglyXB`nVDnvHEj|%rW(acBW9yJX$~M8XuHdaqxW|AP&hP7`|6{
z@&oUKR2RTUACaId@|G8!-w#`8g5sy$$1|#Iy$%&+9t)sBof@j`vK+Wk7hDka_<!^Y
z7%zfXu$P`(;ND+vDKkLtQ9(_vs_U=@wig-<Jtl~H<Ig`6%eG}oaSoeZRgOn6^4tk=
zsH`GXmAz)Aq9p2@baX(W>?aF<p-U0NyiW257Sa9BR$%Zn1>CNF84pb7PP1&$gJNax
zx$9U3Sj1C0vdUo<it{+gT`j$VEPO7F{|ULX7I~MvBBpW`EQr{A_FF5b$P#jNe+z!W
zB}_4I0};wva|2{DxW{66Y3&fnWme$<J-N9wKvZJw;^X5-r&0j}L-b6Sx*Nv7E&Jo&
z+tZ$+c1}h{PVOV56Q?1a=v)DS8#F?j-cL2)Hi$F*akKw{)dR2F7yJN5{)1+F{(6fC
z0VzK-O1bnyiNT60ey?c!k25R$e%r-v+)Iw=5>1n>=B$10yUkgY{^;emN~yWR55kF1
z--KSXn9p|G_ohZ!_rQR!PNiuY8k%V+Bpbkxn0(ldft?_Y!<B(Ijp*xQCfN4A?}pbJ
z)aS2Ggo@a4ff9<V1*;aH@Wj7!QqFH*?8c9~lU+%cX#0QzrDl`Rqg2d7M}S`gVKz4m
zblPWq77MPIzr!@+IR{E80<qoj%J0hW`&wCuL#ieS#Eam=c7(&mn8MjbglZ^35$+lO
zc=%KgBpELae(v#Ep6Pih=32O}3$v-E(z`D}FlAR<0_9vYOCAK~bo28|9OQWo0epJC
zwVeE<46xTpk@Ru8zlir&m{Y+E-dM$CTjs65#q`j5tenK;3`gLZU{~X@ULt|383DJ*
zw?CnR3pcimH-9*LsymKd|0ZWouA#i|$!{_MJ6d^Bs-#!>&7tYA!s<srMI8h^Bmv{X
zFbOZC&N)&;0Qnv)*d0Jo<r5m-!t|k7-+>l=V>vA5wkm{-arE#*X)i7LiJ`97e~Ug(
zu;%&{dVXN=<;WE6$%jvZ)*8nketF$Hr||kBX?OaiPc&>yj(mrOtneazct=tLmJD$o
z%fc#xd$UI$h0e|w^m08u^|zP#kDq<X;n6SpoHhL$)$e-*)oM#fzi+PlhY=gjTyN=5
zL!~bV(!nZtZmdY^O%*|@>-bp;Lum{>>2H3@!~_1K*TbYS*U_toNG_%rq-?g?bdg$3
z+9#YB|L>F!^!PvusMGHR6@!23`oQft4^4sIRU!a_yBcOI>wi}oZ7b*at0^+$LbUdk
z=!q-X_|FTX9i>FS?|M8Ok{*TDM4B9QzVr!``;RH?4<kOd<<&M;cw#bC_{#BU#4zs$
zaDV^Mr;I4-(du-azkgrH8u;c!z2lL<m}u9cip%;-w|gML?7Q9In1RJw?7c14ly|3W
ziKc8w$0OD-quy`<hki$D6%;K<BXb$!55UksNvqG2;1B4c<X)~!&D}ZOYG${>@lQ#=
zu^<1f=;?>-wL7xwO9hS974=!y|GF$(BAY_|F{_{GUIx}UrA>xZIUM<t(a!KQ(H4gH
zHzvA7vhb~-!qj<YW94TDKOf&}h%Qw31$c4BiHF8xUNi!v@u>r!FF^zGAF#Ckr7}{R
zQko9^N}s=31=q$_mUC;||CsX@-6-u5psT$^-QI1?@TLsKKEnDJdEAs=;k(|;rC(zO
zu=Wk(5YKH*9xjKu#R8CSNXu^=*Evbowj8oyn-S<K{_LK`e(vlHKIcW&m3*s8tMLz)
zLgy>+5lJk^WIbZ1BtwUvSUmfRBICphRadL!j>hCK&;Bq8_9&^<#37kNp0MpNdVc|g
z+@Sok%x;eOGKPrc*ZoXxFZB(BXp67h24|xV67w)qu=D0zIWN4;kU~+|0y^yME9i#*
zZBu%cprv?(Pi4Gf_I_r)e_Ug`NU`fHhC?r|q7KQy#~*t9j1Y<AJw0d!-8i#Z_M1g0
z9BCIZ*xyF9dr_QJeaqF-G3B}28XOx6xN3}S+shru0TR^(;+2w|+UK;{Tl@J`Q9UlF
zY?QQYT6|nlL2jEx>(~?$Pu2e>jlSVk*0Hd{XmoJ#g1)gK2W_j=D+cL=EIX%_3xcP3
zLeQteu~5pt2BQFCx}dIe2&xhMTkX6aY$QSkllB)F7*8H#xs7i3!Q*<wzGXe}seMgQ
z+RT^ce`QQ>vpfba7c2UA-!zFI9CK!H|FA6TsnQj~yT|bazM}V9PV+vZ-hN!GJ-F9r
zaGE+E4+WDgAD<#_Np>g_F1vJZjP1Y!KX;Lc)W*)>X}0&&5bg7jAbn#1DXW0K+~e1e
zvBD9KAu<tB@AHygIj=4-m4V#pPg(#Abo4dZ5z=c&W8O7!?{RtlM|^mAd&{LSo>%@$
zg?m4Mqh=E6{N#-u*+BJTkHs<ekejVBwq+g6e|AHM7lF_t|53`JG;Ht(wD8Y}5L_}P
z)Xh3rmg=JM#WadAlJkY4wd?-EZ?+;@33+_BY#*M&VfNzyGW!z@<RxhCu^MUMfPsn3
zNc$8d7$wBu6EkGCM5k=pUj;RO;E$Sn3H)h1SWdb9%0(7L(SDE4sM$UVau_PqDr@y&
z=a6`8?RV^B&jsU@>rcNDIIH0ix1ik3@#H3oT^c=9G<RBeF>IvOU+|V{@i^WRK=Jej
z@0IF9+7ys<x#IJtz|6>M>H4uV2zSW-#T`E)vE`nGPhP(Cg}s9z>Q^H~Xkql}$(n<3
z(Y`F(^gl?isH9fi;_bD=9PmrX(l+h<)z=Nwqwo638~gqeu5;+$5+Rm#1$(&Ja6>pn
z3fF{LT5ckOgmrWsBYJ&_LCBs_$elVrTJF_5+l3i(f~QB8P=QWVmkVMeYl8JOA37x~
zg%2bdC(}*WDhaPGrEO$-yKi0pYafs0(IY`Ft})^3$_y(5A;-u9HCAV?=C6Kwsv??|
z<iFnD%uS5RL*4j7apUV7zqUPi!esc45Ahw-R6pVy@g)>!TmN!jOuP0G{{v{{@*)2?
zMs4PM7W@%)Eh7_T-PP$gsC}JLkoEZp*t>T^fV(7yI`6Ni&_~El$+oR`PM!+a?Z#;F
zvG3pY#VUjWpYTH)zs+@%|8eRVOz8=m1O*;H_LVLE+Wh~kBV~M^C``(W*48evrZIa*
zrJH!dFTJ-F)$504;4-Xfm03n5-kjy;h>WGZFkuJ!4UYD`K7DwYqaJe59s=o{oSYH@
zMV6hp6v%?Nxw}Fak6U(RA5%?H5G;2Tf+jsYgFy6k|7~5=^0B#@In(Y*U(ojVz*X_m
zfDDE1oNQNy{w+6q7_-Ccc_hKICj~_`kJRl1G?Wu)e!J6Oa{2feh^r|CZG!=mw_ilo
zN~E)7Ou2X9jSAVKoGV8xzCYoihpboi!LLJ_LOe|%OO%jd(Jg{>GC<cIDVJr+k-8Vh
z8Nl-nV7>J4WucI91N+=mZ!fzc67#5%Bns<At)wo;47#{Z<~^*{4?(JaUES$)Qn?0|
zUv}8v1O3_qs(#l_TsC9fma5-Jz=-v_{22)X&)UZd)YoQ5xzi=bk`o?Aed>;$X*0u+
zz2lGHR(?1VN(F{U>fmD^JZj3OBd8L4zpEgkj}MR<G1Cc(^9lS43;3TBeIBD3B{9c|
zP?I{pAy5AeP@m9x`%#B6cpV_~bk`~CVO6Gfz7;;wMea{feT26e7x9=FW&PXh^O~yM
zpi9WghwtGfVta_a6q?WHzGAJl-qmlfpCbiK5o<HS?MFEX>Q_HH`u?BWyIjVJ??%;6
z6Y#YfatxvN1kXDO+Gm!pe!9G)ebKl_CbL7#z4%%;@i}RjzOhRQc#g%?Q#@*HZ-502
zhF_mbNMee?gHDG|)>q&{LZ|K?jk7UHeSq3y<=`MpO5)%1F#e9AmNe!zg0H04Xfgs*
zdwSF(vwT-i0pE<pBr;#PyP2#7wgNlB^dEL&k6%!lGHH9WIqEo$i^=2ZsIBl4E@UU>
z3%`Js6Bmt^lkA?(DLkW38t77f?6{4^*?h4*-EDw(pNjJMtdX)Jp2HFtdVRCoEYJhm
ziXe@b4nc_0qdhQ{0orsidL=Hm+cV=P1vwOOkYD==z3L15Pj3!n$+&_=C=NZ)OQ#m^
ze-L(c-M^RURkD|v#HAMlyXB3G+#tfQ?fTm+wuyKGX?wz+-}gsz2YSdOdYLmanr+|+
z?&*B<l{H`OY&wPXbAcRM^_3?Gw%-~X0=?jU6&8KJ=!_0V3!Gz8C$9d2Z;Jz=J8Gp`
z9&ans*<<<)hxN2eix*HOLIrO^&`GF&@ly`BrZM-pt(bSxaj>%w``|%CyCgJ=`PmMA
zjgo?B=Fab51`?8+XR6@Geb#u6D>E76{RA<h`~M@P0c6AFhqQfc*O9=*(b}|gmALSG
zP;O}A77siw7^qTAZLRHQbWM=z#x-%d+A9L_Mv9#6Bl0Rf&tiJ7M!#4Nxlxc7zP{S(
zuJOKfi$ShQV2ipGWkm-F9g~CBb<&z{45%f%_;mUbfFAWm`#1V$gY|;zl4UnW?be?;
zMWE}L2-bzs;ld6}90T=n5g-M|w}$u<8rv5%jV!yCENJF+m5GFG+51kQ%IQh+Lwf<*
zQ*BN#X#v<*WS2&fuN!i6zB(xD=)c4)b1*Y)K=G+7qv0wI;w-xFd%5F+0>_?uf!7*&
zs5U5W+kwQQ5vL4Qud#~^g;qBXd}x1q{^6r!5H^L{0np&bixzr_nbkiXL-YnNkg{)|
z5leaUfzi2~TVH8|DFjnVuA06jx2@BtRFFhbaRjOo;2Y1uV%~b#eiX9?<2a;Xd?NxP
zhqlW*fGmoJJk~Ndq=SW==8nR~iH96lUi3jOVEy4Q2EIH-*fzz4jAqfZt}TB+!-=4%
z&3a3ikepxtf|tHFAV6jFIDdcQUaPk?6w^!kd?)1DNqYdAJmGkBYFU*Rs*{k0`?iM6
zJpl}<A3&p!{e#uv!lz+mw+8^y4JNw^@*PgP`do$pUJ}~*{Z1a!&QBNH3)#F6&>!l{
zCiQrD;_}xh$S%Kzc>qWy0i@0wTnD6mv<#_@wgT|#$<x>5Oe2@eM$pvODA8c8JVkPw
zgfjr);&&nua-BFN%HFQHEna`?38caSgH9xfJiq+)aup5G^)WN6Kp^_Z1EktvoZYSl
z^v-9mk1lo1Yn96mAXEEHP}@WyFOcxNr1!_n7pCtfWMeEiJkr=IY~Q^Dx*lEhebz7u
z<J2MY{QgM!c5gA*_HzzK7>`~*J*M?VQAsHS66bRxjZ(e*lNvsyRu?XykR1;I^<&IT
z>ZyB>Q3mkz`%rS<d+Td*r`@X1NngqjzSxK+rt$bSfD!xv5AaN~17Q1*Q8eU?LX=P!
zx&$Nq{e<klLao3bI$jw7`FslA!Mu31bR9~{`p6gp{@Z~DFZ)8I$(fJ@Fp>a4W@_Kc
z?oB_)k4&<Fg6)M|v(bJ#273_st~7FY&;XV}p4X{iP*Fj9MwQ~&$LgB3scwBJ?aaZ(
zzmC&Rmos;~*&WNiC5iuSF3@AnGUiW=;up;@q!lzsY}I%>D-C&j9~QO9N0d2~Wzj~Q
z>Oq!(6dG|P)RzNrSSX;Mgq0RC^GH5d{+=M3E)rShvOKs5)g7%0r$;ckX^vvx^sTzN
z$JtM+Q_-0t8;}~!lm31ja!70dI!Ns0>0@%f#G0`9HB4~oVOanl7}4t_NH<JOKcpUC
z!|oKwZQo~^9e~By9MBRK0H5|-=4M5W@%sBK791oM56V9v`D>V4MD$c!p3RVzh}}9y
z0wd=~<&8VIU%SxuSRl_;3DkE^43dJ~GPWasywe?1wf#!jc`#ai(dGI||M?aa1j4v7
zp_8<mY8~rKvTDx)5x?Dc<{W}LraYwB$a)zg3gCA~XV_#TIR-OS9>NL`dINcRXp3NB
zn0AdgJQY$!`+{Zw%77AT^$Q__mYbprx$|^c!vh&ml10keV)Cm2-ncU0C(6Y0_ug1$
z9@6{xzISjLCNW(H2@KR?^0b2{p`$&3;4N)`3*CE2Blw$rP{@PvZc%PFUR}Jf^8r6C
z^N3uyFARrqs5yqgm!hmrvY17hwUz32o3(dc)_?tQ5y>O6q9dDk4dU(_Vk6*J4Rw!&
zw|taS9@xAo5A91SWTAIXTP>t-cK)76jhv+X4NC{=VS+%+JivyyA{~B^ZqGx4UKvkV
z^HQ$k2er{YC_R3Jsnb4CSVPtWEK4te3K3!tsD-}20|3ZaC~(wkkkLSf9dn1RpSwGL
z5b&MxmWl^{A}iH@0?6_`On|a0b^ZY<ZbJ~JN!B1efreV+^}7}TY_5(HFQtm00yNCM
zV*%M4AfU~H^9hi5y}C7cqpM9cWHOk6SnPKn0bfCpKB&Io`d3~>B(p<bt6<nkx^d%?
zYKpzqSsqO+?@rySGmC|3Hb<dm)3+G!xv^5Z;nn5YH~H}~)1&F1RX=}yuiZFclrqrb
zrKamhc>VGZYXHk8Dko7x$EFUem~@#i+WG+D<Vs7SsI-*F$i!yd{=cd~&}<}Bzuoch
z@HRGh%Ckjx2HTBy6MO6*SgM-tRS{v8mAICU+0znDgkO&OsH;%%@cJ5q692cvjLJ5e
zAaxqr^M=Z=Jvme_UV2D))|2pj;B_(#s^RVUhU;LPk-g<j%c=b0&i1vE8j;geD{?FE
zvhqGmu6*W}rqVrVMe*PNc#qHvxmNi4_HthxD$AIO{7K&je@Ld!SiNtgidcKR@y6~}
ziRHlAp_F$-FwF1UWfzg5aGM5^I@g7EPDNea!=)zuFg=<=dl3@b@`<KoFtT4)e+{b2
z{F$&i&l_2zvs$*ikO#hUWOIunVA5Ej95b(z2GIH^Q2r4&V1_pDfc><zOD*z4Q?m_m
zS7>t8G&MEH%ynRY8)>I6Zx@WLc>={Owa!SUDOxpcs-(n3U)A||&>epJ_wQGJ!z>yQ
z6Cft?s^qw~(?}d{jGUhTC~Oc;_O|1NiF0KWQj4?j85}9I3F2$VtRlc1UxKxikGz+m
z_v)_x+OXJ5pWj)hKSxn4v>%=;uv|nNhzUN9SlFTkLXZFUr@F>8TK_|^s>Ov1c=%{a
z?4SQ(7*V`JWsBG<=-adu3n9UH;GsyG-jt=^Q4C966I>d;(=g+>Zq(0O?~CHaVyl>j
z-hQPRffF_ZH#6zXMNGmfcY1NRce&knJJpw<me?;cYzpCg#=GAkz=xLrXsgFF259DH
zgZoT$S_@&w=AOAndaJCe{I}{D&7tO&G0-0(zCImL2uu77+j%|8&&Gr8|2XsX7R<be
zg_3@aU4H;A4Bb+L@dY={3ItBWNx)U`Plfe9GzHlEZPp^lOEB2r%7tcdxlQ}ZaHNhk
zWXK-ux^uIiy~m^0#&21lrzUm$MrbN=fSppO3A%NfO5HcwGJ8M6(405otKau!yas0{
z5v*p0QE7<0_?q(B@pOrlL_ll7W2u@6I}XAUlPkCF4`fG!?{h@W@DT3ffi@S0k$kDO
z0K=u=h8T&p+UiSxi2LWK=qG3r%G>M?cOL<HH)*`}KLvYy6nfO6Ds=P3dCZA9gCw1O
z8x;QH8k#dn?uQCU_+MkKJ<2TIW*LCsuB1}Rii)oR>aPjyG7)>BSfoKb`TqTTZ;{*{
zcr`TeK<O_i?I**oo{SU#&S*@ME+S8JJg-Yp&uz&lW!BBeC=MQnuI;-Uz+FXPzLQ}X
zGI2!mG2<bvy`j>RM=c}9gD5;(Wb3|O^zd*FFM)-GF+fQ^M|;v*EwhY!NFO20<KucU
zC23a=v$GCrJ&sxeecKdM^|4fgfQY;vhOYg5qa4<2mi6TwhEj0(LP`n*cA2w#Wnd|@
z;!s_LVWkw~9q)Bm`26uj$azdQYQkBjYPECYS8&~nj}gOqlDz+*62g-Rz5Lm+9$2>W
zEUkaAFdnXuj1r?@f41c!LhwXl9Glb?t??hFuM)bIic6QuO8vCa^w%)L*l6DSry#7K
z08n1$<8ikdk-!ZEm`&>ndm>5Y+XHBSe^#eF4@Ktd91HBr?O8fiT2VKchJo&j>NV$y
znPN$4j#Rwxx~Z9c`xRK%j}KF`iu}bCf}cIW!ylVt?tLQOd`EQl^Jx+2Pcu|OjH3>$
zrpy=^z6!_(gk-d|w2m4^Mn(V}bJQF%KaU1~`H_VTICgo^Hh)-4F4FO*zu*nFLl=Vx
zANg6axfg@zf8&WK@mA8W?T%%WTw-3yI~|)~vR7e7FGTyFWY`w$Q*1z_o7(MM|0E%n
zh>(P`8p9hVxk)6AzWSorI_ITU=XJG)_pJ!uMpF?AaA3<&sL9d|HyxMhO3ln0(tR4>
zNh;7?q?Cnr#TiLfU*3dJz*Prz04({siW%AqX1cKCGDaLbdbA$q2N*@H#ngy2YYi<!
zTQ~`2x*XDT?4X%&r}lW>@Ov^ozh)*;r|deRpMGuDwl`aK?3;<1&B!%10rq_%VdJ)F
zX;MxBq4&i`QqFJ;Lov(I_F-!GmCoK|T6cgEWZF1G0ev2}tUKb9HFG_|VFk!+yQM^a
z3v94yk9*#X6#a@^_224kM*`_#dN5AX^A^lL+hL~7pX?4`u+fuuQyw53rmznsvCd=H
z`PE&5hVOZaL(LZBM8CyHEqJeiFEb1wOUm?cs?7>?Jn9%0+j525wP?_vNnWNYg07`h
z*zU}dOG>7TOj3g_cn4}teM7FINmgM>Tq|s*4Q%<2Y^YhT5?`l!v(o3aZxpPN|8c2X
zDHBKJBN=tNVn<-bYi#N203WV1{Rt_AGV$GaZREA~T?WPpJ!$Bp3yZi!VV_^-wLM(i
z+yo{;{@52JW~bFfei3bWg={4Ae`nn|Wli;$e%~PgC?C)?^rVs9tp6YYNs|sxrm!>P
z2*NH{Q4;b(&CxqAH+<r?`CL!+?21lVWZeezC8?c#UA)+H_IuKF%PZ&mOn2)6<?*Ij
zwl4829u77GNg>PGW_X`5V)V<A#Ysu3e2s^I5Ap%iWj7MXTYc2uw!=8V7*>L2gi=sp
zet`zi1w@yPS(Mvp*sI#=zCIh}ZCahC%Wnz0kr9J14+zwYSua@wT~TKbvr*)K-pIeX
zp%ywP_5b>ytDi-`I|m1?3eGL)G_1sIOTbeC^f>7pNW6Ux8<af55M@Vu;6T-<aq0Di
ziyNVehG-I4MI{4)Y4S+@XmL0mt|?aPuwyLnG&#lLbLvq4mDOo~t_!|&;g5&hCMi;c
zX!=X~i%%LNhnA+CK1wu%xzF;k9r&ji-nu57BsH+(lfL%j|B~Hqkln;0CezKkYw{|y
zNVwv>_TF9Mbd^lnM&<S;b>rcmtw7e)z=o*n<F{MW^wMC^qAIA|hXYGr7@dY+Il)ll
z>z81vK#H8mG0e*&s<rL_Jv@`Sb|eA%0jfPwKN@d_na9KA%!*q^E@CwJC}d3>;uU2-
zUQ;Vwer{1b-=f|fO@#(ndIH2fA4l6)4ek^Ac~HL@5Qixmnvk!qD?Ne9gIr#wp>N6Z
z!o(Z+hNTv<AHHmJy&HuI89|iQ0ytORr%udM?#hi*&&1k<LhrlBIejv35pa+&eL)y}
z&GV0q#F`W4FQ%j(oZsT<Y7t=K%(v1)n14t)#+^{+%9S&}JqVN))L&A+l2}<N8Q9Ru
ztPm6rJ(QHvbcL^UytAE0^ZqfQYhW&(__H|K=QgzVLKo;A;x>%GxT}QriBLbKtUiV7
zK$DDvN3;Cv!#<ii1o<3nCnNv-Zl}3mG4Ou|7qWjD00p#KZ_`5HA&qCh2nLv2Ac_oH
zCa%{~oD6z^hYQwql7@X((qq4-VB_@Kj{^(@&nIE>tM|aLq}fGZ@Mta_08OTs1Q?Z!
z0iVc>J`Y$It^ofm30A9!$wr>jfIeHluh$Rb#wicZ+)C<nTdz7b63{SCw8u<%g=xbu
zy~}wS{eMHLQo`jTPCtv%?z@N1zg$}y3SC$pJJc?~a0w50ZhznX%yVXQ6bW69d?WQm
zt6$wmZZ+f=ik|Ft_v&X=5J%;ZQl^V`G;Zt@?hkCMn|QF^(!L4+wnqaTUjhBj^<8>h
z0OcV%UHb0bxq{In#}PQyXyYxzTWv7eVFdO<y}1(xo4{gnd)=JVbmPbvUBBNuFzR9k
z3IaiV<#w37q5t}U>nilC5a*Y5++Lnfb5Dg`m&XeahSAZ|)^xmup=d8!53=!A+$}MP
zkB-K|!mxB7*wV+n47#Z~*Zd)8Y?a9=FDwBWx$9+<Jg*J8C{cOMbKRvWB{l_%DVEoA
z@9**!C3{)%tw8dnEsNkS32LyOpO*OH?i&z2*?-@|j)%c%?pvmu#C!<r6W@=b)849Y
zwKbx`jI@B4zM@tYUKF(SF3$9s%a;h`r@WHUw146wK2n-Dlhg3hHSThpyD&I$PqF3%
z2>2W0{My;2a3ns=wuU3P;+9HS9-il3CXtF6d65zJG&TnwHvM>j37Wvry%bx}fVhao
zjoGLT5Y5-d&==ofueVt8hDD<oK-N_z5l*8?PB6C7ur?>827I-N4QTUgh5#gZMuDfk
zT=MEsUf##eXNRM`_zm8Bw^^@#hANUFj_QRa+rBnmf-`Y|ic3N~MUyC(nFol$0#7c(
zupQQv8EQaRM-C{3{ot`@P@#3@_w?jYRet*PX*D_MKK3Wc{`_wfa`x_IkHVn0ZTwop
z!=~24jfZyC13g)LZyUdTI<XjO97)u?V~cT4G<`i;JxALqw=|nS2RJE)HCcK)a*Qzp
zO48-h{dAxr$-y3xr25_EfB3|iBm%|6CqfQr%G)mZZbxgi8EzD%+R-rzt8j763p&iH
z8!x#acHlNUu7ilA%>aC=u?i+xy4wZdCDcaF9e$Q#vjTH18M$@eIb<L>IRGJ6LSO-q
zKHjkkm@zn{eob63S=oAM`u6uJx5cw-FpJ?auTvvT;B=O6maq2H`bFlDPDKUd3TpE;
zN50<V!uEmjZ-!n$#)FIoQZLQS99wu{x^H^BNlj#v`+HK(ikaZ!TJeWPovQ1=f$=4v
zi#3>keN5%o1?w;uxGhqAf^X&(@&R|xmqSZ_fRToldRQQ3H5X2xJJujzg;}xE$vB%#
z#3V7g%&b&=Ms1H2Z|?WooR6uuEe5@u$4ZP()=l{>#=UVF7?->}Q)8EsL+IbEM~6b6
z(OkfU5so_V$Cun!J=ikXMoE%^w<r~2`tC&A;n~0@H)40MW35-B%*N+N^&WtStQyUW
zk3#(NNZ>1Pf?~iA+d@W|y}y!Ay=py4Ne53>JCk#<vvC>nf6T#JXF36QXX%YU=5kmT
z5-7@$NLIOajp~ccOF(EULe5tS2JRXzajT}(S3hApW6_nbu50_zlOWP<**FI>8=(WD
zICs#T{>NMI9#O(a-DMoKpQ+(gBhzhz5!H4uQ_2LgA*A<LV3fkJ2f^OU)sOVjJ-k5X
zPZ)T=1<O>`S%nNF(Q6K~1cwWNeK9c2Nlf+Tc|y9fjJv4}9F?3*j{Q%ZW@;*)s8~Sb
ziVj5FVmaMW@~wC8MnZ5mJv9uY6KPO~T-4ZUa6WWx(+7>qjMklXW|H{VZ){{<?f3b<
zew1dYo3gVam*rVI@wQp^mWC@hkwaItr7S$s@bFP3JU9M$R?0MekD^W3Z-7U2lW^2#
z^9szgGVIVZs_-vApK(2Cwo)XQc9!FAgFnvY<h4)-X|pGX(3$6vD<C8!>{05s2S2Ip
zn7Y&*XHC~+Dtu%^D3fjd`3yU=kCM#!kOrOul{W@$$boiunT4#PCH)S*I-c9c`fBpI
znr@Q#+K4<70Uyw@ePU6vG}vz@3b43P0HlFZi2;psyE>3%8F*bcwFj1(zJ-x8rJ8R1
zF^P%QXLG^tl+2=z2MXVMKa=J*|EA1?C`%85feWr$`LLeP4xD-A+B`pScgxA@(jTuu
zMAMUqqPzGFTmtW9x{Dd0v>hj&DFkNgM@;=ZyQ(z}5$8J(FL0|`XEXIF$bJb;WH&Ye
zKGgg8Dk%A8OK|eN-~A?JAbjsh@wC+$tZMOHWUAPW^Ni3}6SvUo?j8ZHsy!~N0zAI`
zAkmw~@#6uZUy{aQ1Zx354G#qJ&lCNByE4<QD?5u^ndat|xgRtun&P8)+augAaZHa*
zbXY_xIJGXEN=q{O1<DQ#qmFvQ1oW3;3@lrOH~8F_u3XSv^%SLbgE@m>*iwIFHHPL<
zk-@5Rp)@6zWZD#oWthtv116GB<xKCXu`WxdCMAxuM5o^bWW!3=F$k;^bCde<@SRca
z(43rXmzkD(UHc;h_q=3siN`b)?&nAY<k}=zGj$cxm@QCcoFv?@%<WezG@kz9+-U|}
zbZ*VtlVn#hEP*O0jxTXay3ZaGyd4SVeKA<vV0E9-2ee3IVy~!yR%xUe#kO*J&sVl~
z?RRJ3>CF(L+8NxfnRRNs-6-O+k_Vq>gZFECND?JrUHw?ghVkdyS<T4s(bVXCahblv
z)_W(~UgSQNe8)q(e>rM7_Dx*-*WbZU9%0!?n$BROV`D3v3te1pZ9pnV66RL4Lhl;E
z*I2MZPO!E<8+=oAi!^V#;kkilO4*Q<>u2p9`0?X9B$v#WOE*@;duRHuk}<BemafME
z`IH6*25KrvYmuF(hrz-1epkpF%YZonI8c9BPKR^^e->5AyhivInn$}Ge)=b;q+C@w
zqk1z&;N>)qnCKjZsH)A+&tNi#2+dM-BXr9ed5gwh!$h<cRU8T3!IRJm3ofn2&&MGW
z7zVTx;}MF&jtv-Z@48rq-?03AeITTo;=|gtBXvbAxev@Gj%MxKR>LGgplc0Gjz6Z|
zqO2ZJ(nX_sGefaS+r%Sf2hKNNuEyfwK9iY^9!2>b4qOtrzK^6oXH!eBr?(xpzTY%(
zYkhC;muHdZzHEB==gT7bnKznN;lyf4kfJFyGp4tp>AtbjOEU@;GzDlOkgL9;d%Gd_
z!J_hS$TW61H@F)c8(Y(+*%T2{t4l2+F1@Cs$RQ+Xc`Ux}zywCg1j$c;P37Xh^W&n9
zbqz;}{7hP|XYK72JX}_t$Q2ll)C4KKRA^gBgv1-^{XCeSQ4DNa2>FSzexviWbl!J9
zpR97gp^*5et>k!{=p3|Rjpfupo-O%#>obGJU(g`t@3WUF8O9tetL6zfPFEh7hfw$$
z6d@N=C61u{d99k(6svmH?`I6ljubSpJ^7uMPg-D5Al~7vWR|UxVb1S#*KgZB)8GO|
zeIY=8|K8>c*ku%&B-Zkh?5-6oB?K#2+8LE4h2%Ai5P7~`;bYiJ((koN-+}Zt3;V$A
z4ZHHa@{a>c!f3VSNKuaL2$tNE9|vjIBp8dh3{%kgp@$FVaPm!15t#hQteX<JejchT
zP0-Td3c1Z?NPNvz9?+|f%#Si3fec3zq}r&@@-Un)L!n>Sh)lt{5H;vg`1RNw3{+2n
zI`R7Q<(o|~2s6jBR~J4v*q@xrJV-f4qUmNY-MIclMkSqAHA)g(ul@-`$fIhsAIT9G
z939NZR_Qie+lP+o=XMVqe0NYg^1;*eE@q=2U8jp>%}yWxys#bigD2I+d@r?9NOy}C
z$)u1R^Y<p-p$s0r9)1bQj1dbBKKc}&LhCj3K~c=C`G>=lAnua^r|e0+D|7Q7*v^nC
z8gsip68?H6P5b+1My|gYD~peE+=qrt+_ChHoQAvNi{hU1+;Q$jnKO1E3!f-z*RI9a
zi3fF73Ir9JG3+Rnqi|OTwfqe?Q%_jTdGJ`!d+Bz3Lwqy}xvM}Q6q>U79acw@N`KDd
za~_#97BfYdFaD{EgYrnH+kq>^K1*5`x#I3{%G3`24i17$s^f9+B6S7DIitRy%=3bL
z#1NgsJdh$4XDql9Y;;QSD)>9L)fKrJU%a_Gw`C7^g3QW&|6t3{wk?AFg3N5+g+D2i
z%kqedvw2O2BDrxfIyZ*ogcDXW6z9+Cfz1TYsbQVZV?5Hi>8usl<fUo4HvAZA$QoMO
z9|)|t#0w`X(ZdgSZ`s4>6x`&P=JDat<$%MS$D^1PBFo25bTb<Tvap4H6ktm++rLep
z3p_v{Uxok)sZBi_E#rZQ{}#P_U${BMp%djaVe!CzM4Oy`h!;bABjr%r2^2{M@+fK7
zN@oqSn5-W)nd_&Qd-uxB>XGp#pz+Z&ia_XhW_RU8Itj;TyTn?(p1MYM4mladrgt-+
z@wuk6_`qCeKz-l#F|1X<&}o}%)PXXm#9{(~F4Zv~2bFiyNjO<*@M>g#7#g7|+iY>R
zQpi2Y=G`QNI4B+{V|=0cm1dh+rv+@}Xro|N7<SJzeIXsNFW5{5sX7RdP-4pBJg_fF
z`AxZD@K<J%-a}=XpK{M$e3|C{;?naZ#%po9sk-l!j0qh{XYs8sw=Pob&_KTtoxj+{
zFW$1vL1f6;JSF=mcX$Y=K=MmM<7nsuuZx&g!iGbaJiZZFuW|LQQdYBW1`dYa(>*q2
zMh%av&|y3-I!^1YZ_iO{-=_wa<3@~Ja)EGr$^k3!ki#x-n(U(b*w${m;~!Y=vzB?!
zP5IIJ>n{_V{w;_5uLNQ}9%vnYyi-NH++%&ggZY#7>qEk%==Zz|jfhsKT%a5L4K&12
zpbXwEW+zbWb64atKHTLuwW{P~sca?80rXQtb(&@QBZ-BxLxf~>!!VuG@AzpiJQT&z
z^ePR-C>u|E^;}JQ=lwDGgk<-M*!>nGztas3?_ACH4Z^K`fD_#~`0@+ePL@xYt!$r>
zaO1OZ5|qJ#%V!Yg@h-L#b_3GtdtnpA%J=IP(`dJ}ah#1hI$wNZ{@T^`pt%vHq%nn^
zNSK1Ne)h6)FSp{<lz*uVvuJqX-nj3yBxA+594dvacdlsHMW{{(Z}<?=&DkAtVe62)
zspIG1^u)FidTxq$dLB<z01{t&daM-A2cC`PbtaCw8ms6ua@}}6wol~QHkGj-NNgF{
zDM~cBj*|EEz7~udavS-?R&ICn{O<Sd(jK1v5yBU*KlUl?@WnP^8fbnoT<V(@w&Azy
z$et6x9^X3kDUSMg5w_#%Nh5>h&gmH3vDTAMvFVg>Aqe$u=$1+DJ^1T>Q#rnAY4BlY
z-c;StrJn}-cAjh75^eppDF~i=9$j<uU*AkGhjHArJuu408~GQ3K4q$Fv0m#h3O~Fu
zAB=F9nd%<8>A-r2K3RF|nW7`nU0-9I!;j4ibIvdDL{C+c#S)@lC-}VH(HDPz1*Q{u
z<XpP%4u8K2#A%@RWk#f+fi;F230trdd&X_Un>H5DOx^iD)&PFlnOc*#S9V0KjURW@
zJnp>O6`jQ-$FoY7PsQ9)2A!JzBJLl8V``D_%q045_2)Zhl%t~+Oq!EMo2{=TX`D?m
zW?1aF%4N>-JYVfjW3t*cnxw?oQ5E&^$g5iHj!JnHosWzAE#}F_@Alp_FEr?>?Gq>I
z_rboY-bY`!-rG~-$1SSCP1dz?&l_7unxF7cO5mJES6E@f%Z{eVn<i<WtOZjOLsg$i
z_n1Z~m*kGBEW~8HJ<q;-&-r+;D&G^gRaMO;mFC*q^B2IoA{}>(%wDMu&9f}WS9%*X
zSgLBHW3t0v-RMG!XGE*zu0uaO(Izre0WW$=-o$G@+m)ZcZ#(cgs^!Y<4{4t@99<aq
zZwG_OClwr1=#UwGm*6~c+;051c~7_9Z@w>IYz9&&tD~_SGY@)N66ly;mKUcRqpg*r
zuNTj)9cZx6imkLPXzk>Esc-Z+k|*IFmTeDt{?(4@mvejdr)@_-)@lc1)uJ@FPoR*6
z?VN}!Igmf4zmQz;w)yvscVt~fV#S3`|E}q7-V}xmsrS5>{2ut{lHj}BM3+iCF-cY5
zRm5eS<(I#Hvl+;yyp6$UxKu_RL(ESpR8(V*&mdcdBD^X7GxEAvjX$<ek`)7{XN@HG
z>{E}4FFlWJ^q$>(oVuqkDT+;>d+T)lQQ)2^PJM64B`(0xnrBz?Q?n7r;x5wuhk5`-
zXa+?WOsAIbPl;WgA}Pg&StzC%G%OHb5wON1-IQB+WBL=cn7eN1d}6kXTQ2kQs{2?W
z^thW<RBv4G`EPZjMIjAU0as>qpL|li|MQ%%%+|NjD@Df^G`YGPCm6nwG`qT}&I3b`
zqPmAB4M*Od7yU0M!bfMSf~ia~r{7EdN0rVYKE*Ojvpb`&XU}zaOm>UT;9pS*hL+tJ
ziEzJMJLo;pC#;Lkl-Gab))4(BE<UVvvh7prWwQ8!Yhv+N^h!C`(xXjcaUJIOI<_el
z>ETbF-mcmpzmQ0u`c2IAQCRcjxmGtm*}}rntlYb%iaT!8pL>I!a4GwNXxw4<y#Cxa
zB+9aKc~A6dCk<W8alT-dC?D#z=|WT+<2%0|O^`3d;@o9SUkuD8SzD9b(2jOIdw|`_
zzsYE-a4Oj)m5lngqUO@`?9tBT#et521`)w!srCSs7z*`>89T)Rr=-`PWozH)KG`E^
zH)R|qsZAMZ))UX`*hn#}BD}g)zrZ<BTZv24nK?Lk3feKPm}@Gq({{X*ee_265{HH6
zsK~P5owWF;{(BS}E|~m~G&1Xu%$(BEv|z~@?e`?eeVeS*8lycIPqsH-Mzc8kt(N}(
za5-{iB@ccch)OY~+e`B2e!@|c3r<5c$({t`Ri>Ld4g~RtLT(|~<-QoHu8`^Hif_;G
z4K)U$x&#Gc1y;5Q7afYXn6nvo?S9fO7gjTYD`1^)T$pB;NpK(c_erPwCw(nJOgJNL
zdYZYmj9WvpsDa9)${1&AVZw{N@#-K|QRQ_nsKuR_QGUUXzK`3~^nxgZ!-q^)Bs{}L
zW`{g$|F&7txu`DZ7FECHEvC2;0oFsKm7Q@rJqoX4htavCoDgnO*pxa>>@9Zj&(;sk
zb|{>icW&56*7e~fw`m@Q$(EW@IDkVP(BERJK#{5A?YA`UG<KL9Pb`9-e<N0Fl6P~m
zY4Cn#mvQY!v7p?}f@_KE@}s$FXP;7(pZ9mYI?lW$Z{gHdF?wI3#$h!6{cJNip|9iP
z_MQPNkz(aWI|dnfFa<6ANl{a7N!KkZWy&G4V@mz^-6*4qeWX`b>*jx2GaYY~g$P#6
zO*?e);SGt7pe??>ZH9X~sJOTm7x6eqWI<-*Hcz@aBBULLBWXcb&Mdv+3D!J)wk}Rq
zux64kPh6eL{5exq#kDQHO?eXsVaL~~;G3dc;8_ozev-dPZohJ8Zi}2iy_euHk}H|s
zN!fbk_Z~`2U+sm>BImZ<HYVA!3;bM#TrD{3$wv*JGe9XfXx-n|xcojIgd}r^dCi>_
zmhr8&EW^pixpx+8_8EoDicg31nTxxqTWP#~wqNj04Cy%Mywy;LW7=HrqXO&o@a<e-
z7^{A7dE0IOtnRgnRkgSoyRgihagAW-ub*=+mGnB_lzVY}+tgsyq0~igT6?E!>&O*g
zWPPul03UGsp#l}bx1@F?(mpcbaP_)w{nNFhYajJKob`6CcYK04Q<%U`P+7stx(dz0
z!V#ypJXl+kK{+NQb*^>2=v#{<^)O<kCM16kgPGFd;_6nfW8C_JC0>OL-DS5(XN(s^
z#+_-c$@$zHY{dPVKh}@h9Dis&2!sB&PvWfvfr3>ogOvZ~*|6cjymt<M{*(TUrG!Ai
zUufhpWBSdIv7?R=&YIn(46R0cx7aM5)Ry5W#c!{s6!R}qk(O+vR_@BPXRfu<w-MFS
z?w3gwQ7P8680iU{h>bVzGCiG?eNXv^Zlx?J6$^IH*H4qx3^EuVL|>kXw_wO+a;|Z`
zq^++<)0rYG_}Tf;D)H8LEK~a`n;-oVzwlJHJ7_%y@fh@2+OuDqeW(Qs=wjadW?ZUu
zq_?<by-&9`?#z`qmEOzTrO8h1cdbS5xIZhP`zT<OYj$W`N4!4<5t>k^f6R9LAy?)$
z(f-R{Tbnm)hvR@0>|`*Y^?9{imGqi>REm<T<EEe0y|0D$vd5<H=K=mNkC1+Svtp~O
ziC9pQpft0%s!g$WV)@#&K~1X8&j70j0LuYfqm<v}wz`+ZhcBz&#D_B^+CyYM8o{4?
z-pp|S=WX8w4;2Yfmo~}P3A}Yd3hM<~ICmU>68=CuJ+!pee9+$IFz3W%QR6e0oQ9$r
zoEHoBRQ!lnc)h7gF}A2l(^%2EXUpM1%heMjseHPa<1b$n>heCb1~RBtY?ER15`4Uq
z!3{PLe+D>=NsLNSVOvYxqiwT>30#4+!+;xujnC$Jg!v#4t`j`T>AS<fO+nZacdDjc
zhda83#rl(7LxMbE>njLW5gV6IHg`^u9z1S*n;nmvej!WNb#=+j{h8*iMwyK6%@`2p
z2Y{czJJ;{Rv(~`Zay)A4J(W9utLDX)Zoy&%AsP9}|876e<_nO*g+SQCP44jaeNrag
z!|J+I^;?APE07q?v_J8ZkocW#2EV(a6_d=?XH?VTEWBdN1rzpCtIx&Hb@6o{8v5lc
z;^^xB?E9lbo#X^rjt~YbgTW9`354==bPF7$Ih1FDHeELC;lmx7<wU3=EnyS7`9c8?
zP=z;-cgSa#DhXIIcdG2!2a3I9N~Ut_G~hPTZF%~bR<s#A&Nc0_tE%Oy%JT5{CcF3A
zp<f}TVIjDk@WD{vU~ubgSY=morjdg3at=i=1r(rBBxsPS{p?vIOKlc~ELz)MJg$qn
zZ~bUqM^M&gq)~kZ34Al#g^hQtUG!em(()yRtlWlQoN-SF%WwY`q9Sb7pImnMmb7af
zsaR2Fb3<E9?3r56G4BF)y{Q2D*I&}jNsJc2@|__W&mDVf0#iA~!cR&7<Xt9FN67;V
z=;53(-lAa`)!0PL?DjI97ZL{^*a*0(WEw3^Zi6s7n&&(B7l7Q~Zt}re1@3P*Gl7`v
zL`$5@8td(?fZmk%9|1k`wAQL+t~s{9QbEu>zy1oEZg@84Rr^8vY1*O`#kZSn0^X-q
zfZyoR2y9@y%}_;IIab(a=m>fbnvmoo0YPr(T`hQR_*)foD?lAg+h{70v(m$5#^IE!
zxY}B%9`DpPF}IK;ReStE#TKtg!C6<0P3v}}H3+R-R#_7#?l#olkMq3Xe%6_O=H~~S
z#iZ)>Te90gw+ASJj=Bs(j<98@81qkFG;U`sZLk^|__=mXBIjT(1;-aRN>~Pk>$dNd
z6?p0q2ZdPmXxM~w|47Obtp4lq#iu){psqe|d6_}DZ*8C0<-{r!XlsRAh~BuW_VZy|
z%jJuN+xJ|38eW}ADHqn+lsaw1vpk^n+-BFaJjD_ipogX?xHKwoLfz)QW!zF4VX)E`
zECX9Y(l!+5YC*)Y@-F-97ws!c4(?3~=Dy|<(pk)KG3PNN53$nH(kbzTdoFt6y$-2y
zgz^xiK`IR(p)k(|igZnA*_=>IoOWjCwxbXV4z&z*ZRRtqF9HP_jW5d)>_}^MKDNY_
zswqB<fBa;XN#18MV~CI1XI=JGZ@c93gnNsxjJ{JVtC5P83J7}fb>Go_#&=5~%XQ<l
zL1$pcIHa^pC?Oz@_DRmTF1c^ZuFiA(YRvM=o0}8`|E7rt-~eAFcOqWgDs^Aa>C%p{
ztW7*;wKU=xW@-o@+mRIv0cvSMLZkFiobHEJ7PjXN`807xHmkKYTlh~5Ghv4XABAgm
zWee?c)Y^7R^4^(X=1-?ut(Wdjg$PUMLNa?E(<k^4oBZXJ6PAnV6A%Q)vmd^gjNzWd
z?saUF2@fYrg~948&8hE#hpIVoF3Yb{cCxh_bdj^~=04FPc->7i0QiIzJ#O<w8+XKC
z+D_B<vr_U|4Qfqac`zHZAg6s33AS$I;W{>pITZDnaTFq8^5El*DFvCc?Anu^keLn=
zd4YufWh}dP=oMvk0`sR7!x%d&a+$4cy0VPewp+nnpAl@^L{L7o@rgNBn2%_ekzVYH
zJ-eC9G$ycC-`1D^@nwf0dmSj{|0U@!@8XzMHIrg+)RNk|Mf_a9@$S=lc#u)%>y5T`
z>Rta9)?hQV0%fO7J&S!o7Q2-;b!4-4_#eLtc`joA&&U3JRnvv&?C1PHjFO`3-Q%Ry
zu4F|`|AB7B9=<Q_^>!iHS6OBsBPLaEL{Dy8nl*T~Qh$M!{#CF3gMgbyg5)&<W<7jH
zAYmuZhUA8@;1HJ!uBh=qh0!5UIk%%Ud<W1NC=+@m;oVVM!P!;^$ddJN?>=|n04ixS
ztHxt_ZtqSucZX?qWeCt*=$Y78!n-9BJRCpavL4q2e^cqq0h+i6!AO~I`)~||d*5B_
z7%cOzowi{P2#W8ajQVhH>q1{qk<cTFr3;PTj7PUU!d*_-rfuLXq#^rVXly+H5Icuj
zXpkjoZSVS<s|C8tH{K!O=B<C1eoep1Fcv@9T>Ej97E*fe|IIkKmD(hE?$NuML`^=@
zum2y}&aJ==7fuHXEFmm7=ubW&@j3SVD}L)_W#!S*j+iQ$+CR*IhhiY?unqDNEx9A?
z_V#))e_`qVB)_jlTfPCVo`gLBscOD57TLS2!vB>9t0qHe`5HucM;qYbve8?>AH5ux
zRQ{WKBBz(f0=K#EwFhB$gqQ}GMd(Xr9eU?y+1IbxK0u&o!0oSSw$gMLT3r|+q5p3}
zZb@+?LQUIQ2;z5tp97A3u<fwU`Wv3LtMv9y|4>swM}JsCv4|;oNUJVuNcvI2mKZ!h
z(OESnD>c8Vq*I(-P;3;KY}Vv5KlVI3anHvLkMP$=CRG&Yxp-dedB|=#nrqN`EIrLD
zMoDlC_SJ!cFyU56wxiFTbhFHn3JZ^)x1JJ*h}(?PS6P;AT1n99*>NeZ<oM?UDzgHq
zM>~I6@4IzWZ%Efd^S|ch9(_y9@Ax-=ZjzV;F-4n&pdFXx9ZlPJ663z->j|P8D#q_G
z?J~Uw)(*$w{-SU~O|;iqw@7k%+!#RuRomklR27FryK4#4?m{ZA#DDCUUg#88xHwAV
zSi194SkIkez16Igb7Fh<;KFGW)%jdE?#*?7PL5V0JC~!6+Rh{VA@tTy4sZije-s`G
zhLW~z!qzxV?_w}@!X^rn<75EGHaC#awN(+e%!~Dux-3UD5F4%nxEk!i!|tG}B8sO@
zjDskGg||)}?qVWVbtK+b%Ot7ND~9n|OojFeE~3r&ag~<RSy4|0<&){}VHWG2p8gL*
zfXFn)9C)xH{x$0^(_rh!R>x7s0BMz<ohXW3l-kvQc&KK=yUfQX9ChbiYD1<2=LUN>
z&lH6QL2#l7$`x17d_r(}`=Qf~QDbT{QQ~}GoRl@i+#7Q%cYOSRYG-d{u+xD8jQ6*~
zKtyu{;^WS%sfp$^TQ(}a4Hi2zrFP-)_W6F3jyw4A_nj(znw(H1SJ;7+vuDmnPvj~x
zAHX)6>)Gcn&>r8)`{AQU)i6IKX&hvohL{^N3+9VPK6Z4twW6n%nz_CHPxB}-q~uFk
zW3f3=8<U*6T&~mq6qT^@2ArVs*SzqhG#MJ+89p*4UKsn5l#}^WJuJr#hd(4E8en|~
z|6co@tBizOk5a%xne<G|wSjQYs*+|FPuDG{g|p>V?($Jky84T&6I1aPBXekDw(Xz>
zC#d_DGo-{@ksv~?`rD}cJsfsDweE{wH)J$_7yd28(PLr_<#{S_vdYG|TiuO~A0mIh
z;;}j2c5|%HoNwb)K9A{@uGaLkQ3|w-3JvE>1D+CWsXy>XTQ<~b*-zM{oV_Gih&@QN
z7#U)k6JE6VnUPj&D}V*CClz=Z^7NsNn^n<jBgDNauoD*CQE1U(gfoD+-|D1DTbK)J
z5M=(#J~R{NcFBrdR|bAsy6ZF7ba^B5!~l$y9n|CwHVexOBGO^AYcHDA9Wae!*4D51
zloqSfTfm9od)@*Y%u=6y(!O7n+5EKmm>#Ff+hDFwUD@wOX_Ks!7Uyn+Nv24|ez6=r
zI3g?2A)PrHx`oXT{9k;%1z6Nu+x86zQqrX$Fe05wN`ne0sdR^QhcwcnfKpN-4N7<S
zAfSMBcPP!!H3-bix5n+h@8@~H_xlcyy*UoHTW6U6TI;&5b)LV|74!1<$d{1wLORj>
zVfmq%dpzuJ)!($rbI1z|BK#5+zp1bg=5g8H5wc|wa=a__GV=?MI2a{7O?5QV>YkIO
zM^<mD*p6zA+QIc`>k^dU3NuzQj&-$o-Tw|)qIf`QYF>|V`Oht^X*`&pPg^_`{?9|d
z8X+U~icIhyht75Sz^a;6jsJ?!vzlZsGe+;D$}*fqm8_LkR8PRO<RmrED5UMT8o8^D
zFm!33D0{iGUK?LChy?TD5#hxTZ2?0#xZE!37e?Iye|H`*hFQkCAv~wJeWC$(%axh6
zF2$i5XDvk-`%d8CzjplGW7sHNq+%eXY2-ks_zCg%M3B)19TS0)l55&Os~8&$yhxus
z!u-4XK=Tsv=hZu;Q3|Se<aR&p+hB>U`1ieWfQ=yejpVwC*<Tk^M?xwITp?<Lq0zKC
zzBzEmE6&fiN|Hx(;ADY&%N*X80#}?Uz_2Zo7y9RcInr*Ir`v3UT4XlU`+XTQ4fYyy
z^2}_G$%m?%PxXgg=6)BxF)Ljs%fI^Xw5yH4B`~3!XUONzH&$c83`unSV;aG~FF5vj
z@T+sC&DZ(GZ;IiexBWiXyr?w6_xZa=Ewc?bRSRBmK*?>_bVlzgoF|8Ih%oNO&=yN}
zI5lthjPBhX8>=Rce987ifH4kq-T$sS4~orcgKbH}S<fdIn7;oE^_(v4H)}fr0>_7q
zYi9r)cqHdv`_Uh*$nW7FX?M{tOC}mUc32_m-}+BYcgh+I@)y#tTE<&KrDq<!eIgyO
zVIuD&j{^wsS&@+hAUBFT_tI<yEOu`+%muB+KSASJ7iB9pE@*5<o{MdXqIS!WnJ4?9
zh`#^)*v;zbT~caR?r-(-Wbn2girCI<Y)#|?Ny49xdz^dN)30zJM}4E!uFPIY>K30$
zBe)2P(&Op}AKF&ZU|Xe%LWJ1gpH-|HdIq6k|FZ1QH$*wXMiea!E}Z_>dU)f_=tskS
zTS<Lv0I|j3u9j)=2-hPpt$%f|VI|skHBO-hFjF|mBQ}BeV+@4U2!VjD{+-xTAc+Vi
ztZ|&vCXqaeB^KL+P*GCa023?Pc6oBt$0u@LS<!g|58U^s|H_EmOP&niL)cfCJZ(5S
z`iU&4)MqZK12aNXc;cUHxk=rQoxj%#^E8{nPpaA_oB2vP!pxY1D;60iEHa7KrmY@v
zx!m~oeW`%=WvOBB!u|KBg7)}`RaXAz2#<Y>_4`0>I9%s5SUz;{V8j;>E^zJ*gtr6H
zda>l$^j)(#0jvH4tn1|bdZ6<fCIB=TMe8(I`Hn+i0@FvE-Ly;f6Jv)S;^FKFOh_3Z
zKKSR&TH8-+{8`s@VI!XTE1!|jay^GXqdt78QoTspe_oA4QxFa=+Dy}=HZb?eH~!?%
zTxiO^gx0J@+K#(;c8_LmHuCvDb9XSVFhWZ&oXrORRPPWH^!;k6{5s{o6>qRFSr3Sb
zP|f5{6ohXy1!mPC(f$|7VDbGHjaEN+v@I~|KL%XscRW*8dBClD0px(Mbhd`GLxJJ>
zmw`MhmIsv$@Z+FGvOx9ild4<ikP<C*vDxGleL6~6L3XWf#wyjW7J#`*fVWY*kL|7n
z))$`7V^|r{#z1&b?cAf<v*B|56WOy-L&wbJQ1{PVQS;fyPaEmYU*PhMVsYcA?0?OV
z!M>C((r=L`+;B1RQ(@2?zoP+HVqeOpbibH2gZH1`v_mu3R8UF$_kmtTD{C6(o6_k1
ztg{SsC&orJVEdm=ER`F*&Z5#dsepl+R#Jx3WW_|jEHW}u)7Urzg67N4#J9sf09p4Q
z4jv^9jgCPpa2)CbaAxf*{%zi2;1i9yd7r72s6@B&VW$6C3cSevGOsrGdRYxnVA_Ki
z)XG+nHdjCE*2iPs*KZ+@CItZnZhDa;aQN6U|LkXXVvS1s6}SUVv=PMrT&Y!#i>hLt
zCFAbLraUm02Ww)0EEEEJ_OFR0g=*6$L2$Wh9Mk{rOmnbfpT5IAFkvRC1XXHofXZ_q
z``e^-ppT%2xtQF4j-YnX`?g#DyGMiSD&x;O{SH%qUI^{>cVfSYJO%aeq+d`_oa)DW
zAZ`AER4aHpvw-~btNO?@pn5>~UAmv1o?d$BeTTb1xL4{iEXPbf0@#&80axBefZ24h
zghJAYXWSbkg;Z2dfN|JFNeXPb^$zeiDFV~k$T#2gMWDuR+RS-JUT1svwF8LCt}>-m
z@A<zki0_mBt-d|GpgYG=(sOB7qER|#`_FtA6Ps2(Yc61PIaM7OpW9c%cQ1L*yUL*}
za~PbG${Ld}<BC!pRpEbUqoNS-(;yaP%K!5dh}}mwL6=nRzyF>f7+6~btDQ8m)=i8-
zgvHX8i+5=!bc(o@bh!4nC*XKk*KO|N%ma<27%<mZ&?f<^TM1gq2z2Rv;!CR(b!^CT
zBUlDNf6m13drh0Y-F#7+4|KfYOA#Ir)WUv1KG-D0rNlLlvGE)n%1n?)$xR?7{Kc)w
zI<(K&p;5|OQBlzyJTvw{rEZ+{1D%cp!a&u^p$x-?EmB^CbwFLG%as264JzQz=Fyq)
zXV;r$=dSxlYg0SADt+Hg#_o>dKkM8^7nAO(=~AD@{;>z#IV&aRbJ^JN7odN%vq`>~
zW~@8S?h*c5U_z471JW;(tA>BRafB|#B^<p~|8sX-Q2XauSvTO7$?D0p8eim<fe5G#
zpDmd+xCxMf_NBtn;Kw*cf9@szn4qYPDTs|RmXR|WGSj$*?En?dX-CUF0bXU|^d7B;
zBp7lZvEkyG$X%!Smjy8I(n-_=tWv_-J?)0FvcZw>-dX0F#kxIdm(tj+p5U9YF7mH?
zBe@3DvF5eUZ#tybCccs8NE9T|_U*<QYV~sdBYgVv(GEYr@=Id@au@F<SYQJ3WlfEh
zo=f^0UVQNXK1v;*LROUWJ+1q<PHSWWk40Gx^Tt1ytlj9x!gb~4_TN(>NTw9~=;ynv
z>O5SM`+@t$Ww`E^r9DT5Zmid^k9zMfymSo52nK#6E%7uXuZ0Oh?1~WrV(BXvArLQ|
ztqRQ10T*ML+dzH&q=%#v3lodZpzuC;`+Mn)Y2@te2|NFnw}#uWTu2{-Pp!Wx|H$XR
z!m>XtEU;a|DYm}WD;mM^7g200UpS_4lSSNnshhF)59e<E89b7{@f-Ry6!Q1r0Cffe
z-(_DfFv;Ps4C4Rg3M3T+wNdtaaN3z9VdIUbl&NwF1OJcYh@{pm0QsK6p(rB-dJoNq
zwZMI=%a-o0{?Pa)7~4%xD}sl)#?*-d=je_MxJ?v3WO`cZTesv_x1ddX@x;ZYP}1@I
z@aN3w_)p4t%73by*kmkW+TlO5X%<2|dasZguo^gg8ku8($EVR^pz%L5iT`{8p<w?W
zhV<|JbFBk5w;oLN3+7%E-P@v#%ZFY!>cz&x?zLc*ro+PC=`UHx2ToVCuhcUK2)ZwN
z)dJ8?{H~*pdyj(JziTtenC9ZVOyyi|^Ax+>4T@?Q8u#0&PJAsB)IPL8;U79mc&3Sz
zSAFCZ*F<zfDjXl0xDKU8%X$BKK^AXi>Sae`q1cS?(DFwyj(VIb#G*j`si^qB-Ohg%
z070M*y(MEA|DX4w3N{%MOO9*U7Z#4yp8JbNXyZMA9T=3Gp8@^@I&x?q5RtgSb%DcZ
zr8AK4&H~pZTEg#liB6lP2KiM$YW*0bf0fqn0+&)kLPCKFG@xjU<FpqS39Jm8z<i-L
zcxL1@mKWvKKhohog$}OwL~luU`oOkiHIzYbP-gp`8~DevfUU^e`0O)ctuGk{)ZL@N
zb=XqKlb@<z@3ia4*4BoD*8;q@mE3cSz#hT~%v*YuFT&ObPXo8j0G;1*O68Sey`v{^
zcI>zOn%Z50LTMh5W+vM$zW1E5q}jJTtJ?>#lgA)?JQA?Kc2(Nm02c7J5uqoM*Vd>U
zK*xll4Y5)5;*+WM=EaUd?W*Ft%zn<g`qfrZ;G9r#Zp%ZV8Ajmvh}5b+HUObo4vl#!
zz)|8mWesX73qHRk)cH!xqTlE&cuEn@J85>a+Q3>q4wxHu6RHn>VjH=S$W@{v@L<3z
z5zcct0G#MF9Pl|nbJ>^tS?}bQ(Ymzbufy1jBtF?&5I?Je559uKM@n|~b6ffB<9GC{
zp!FkyK1Zw5r?!`3M?H62-HLUzwbzZh>oU>W_gO#grYQHkGm)3T)_+>2VSij&H~V8Q
z1ECUuG^&IS=%qZonE8~LbX)TbuV#yt$giJ1#=mwuo?0y#7|ip7xX}+yumg1iaD)i~
z@c;Ck49{6xP<a?Tq@>)YesIEDcx;TxwuxoHD5<J1C7qS#@c(+u&|5>5M!SFYnj}FY
z2eIL`|NZN?KT#S&^|5R@cZ)DF@F|7+`T8+yxuxG?g8m+GW-8NgEuEW`#F07zdUI?i
z;F36v0}K7ZFELLA!~Ao*Q}ALK8itC?nu_!h?)-0V+#+unSPh(#IGZ-dPn7`Y;ix_w
z=+;$r_Zd;=3i>cAYHE4I?(H4`W&c#swiE($p|=G!mHwPRU_~K>pTk<LGkjQG0>N1_
zd-g$6gNF~aHD(A6RZ$iflf$_gzye?bW+1bUre4A_fxuOe8yDFSy<qDo5OB#m4f!Ru
z%;wBrw<ET$fCQE!`6dq-wJRo8<*U@+AYd8`zJYa($rGUv4|8D96+wDCEP7DJl&z~6
z4L_WPfg~jUUsx98lWN8;U$NrAi6jT$O}Z<d1JJfnGKJf5Zh>X-uV$OYfW1k>iEC!u
zc?Rd=F#u=v(!xb;M?Z(du#?V^ALfQ|{Jf{u0V0F;{$<u`h&NMNsNU>lM23c|tLrMj
z!NfFRyw6hc`<<AW7&qq(ZB^Zm-)!R-Cb}@Jz2d%dSo?S$eXrC$Yw6op_<L)Wn1bCi
z!=)Bz<MoE?+q<+Cep;6zibYf5WOtc>0o$f4YKLkDz%RBPmuwL#{n>R$&<s)RH5z7d
zB#Dl<ag<wM?$iw(Wcle_?hMfo4#wB5|6j2!|Ds%a677Q+k>~0$_Cf#0ulDBaApv-U
zRjTjzo{q+W&rd9S<)I4U-qW2p0I0}A#N34K08S91Ja52v4Lt8gU6MS_`U~LNIuHRf
zDV|Vc%N54N;%hxo?_d?6hb(8vXI4NzQFF(d0%KljrOF3!=BXM0r&PX<$IO`Pck$}S
zF((gC6q*!827i5;)mfS(4Op$=eiwm1cE6)|>WMWA#T>f#+oQBeE2A$CIkbX;=xXn_
z*)fk{V(r-;#?hF(x^I>##G|{B_o=M>%?(mr00mZgdz))%93xm@@WI^|^hfsbdu^9{
z;dG5T-(fI7JyDr1sZ~)4)<5LB0%n&YSH=<8bVzgtKq;J0*r$2j{qmUm9&1xn$;UUV
z)k0h)=T$<@WeZtn?4FqR`K`3`!7eEKPFDum4F##)ZlPF@sdiw!Hd>LCc@=ha<hGAo
z{h-7fp?mLcrt4PVl5CL7g|?`V(TuhCaBH#u*_<Rs;NAQGeh~j(U-5>7VmD0A!}7De
zHAb}m_hL?I`|}ay07S^Wi%irNTN_GY_83GFXzEYJdhkXCiusC(AHX1~YM5BbMxH5r
z@sfHdu(d}{KOvrc&L&z40C!b@8=vQpygCT)oqOUX8jpzu9H&nnya&{&e2M&2rzVc<
zfF?q85Zh<qzu~BNq!%WZIB0hlxm64YxQuV>j;sb)*beT|Xo%6z=}{wH;O6@bI;3FZ
z#+wjRwXJgdCQ!RxE6t$~1_Xo;3Y<@^m$t5REr8O$Ewy^N^p=3t<8|Qr1T>=p?d~6A
zKADe)I9A3;U)5R7L&o5I4$$v#r`EbdPh1=?h8kF_dHEYjhFM1Evfwo6b>vCDwAN*d
znn>p!A#xvu<lBhahV4rLD+$ss+y6~|lUJ=D26FvHyU_ptFVYWxC`Ni{icxR{L5;)6
zLr38y!&3da1V8=?{RYcE``MC|PeP7GR!*~^;}h~I|Mh`9F)69w^K|QcO+mqQDjzSe
zW;Xb@+gL}@?~V>EDk{pNb{?9BLZ0S)JMGrhJh{EKVJ6}Iys$Fm#rf|MsmJPF6K)32
zyQCOea9=hxX>sWw8sD$`KxL;n=?Lz+o(lJC%ZohUw3@$Oe75g6tvz4SS~Rm6I3ta$
zZA8A9vi;I*wHxT^<D-%4RsJKK)3LEiJ#n$hpvotCU(B|#MKLj1ryyq(yt<N>{<Y-d
zl73^DR$ZGitnUX1Z5J!`DEF|RvN`=!tKaNdIv*ch`=d3$uvU>vyDY))qBPn1k_9Mp
zui=5y;y`t83+VWRSilBUV5qSFT9VheMYrr23eY0lQ$AW?_>=^Llq&JHYL>;@JY2{G
zpNcPVSdt%opqZyA_5jXgHjBBX5-4KV8Kfo@0q7K=BfvdugDm#|fMD^3GvFtRa8FD2
zf{;N(<Tyw*sCAs1tml>*ixP}=;l`zRk7ZT<NNU33vKR$f7Jbb1vA2&8dfVtmS`*t)
zpQMU}HgYHssZpk(0!T{V_i7Cl0?}ZNNGZUhL+p7$$GRdY%zZ%q_t{avaq3dnk;8Wn
z#U#HMzQ9M>5rlcI1`H^^SvOubsz=d@@Fs0wi3z%I4x{-^0Bl$c{+TQ9v400VrK|}v
z{w89*9x}-=?}TT7gPDOgX^|i05Q1q*wHMKMznP6p!m42qE6fKM4A_5ld9rnPN>(3q
zwg&E6&f8*7wt(YU8SOUSZOShw)+skWoIwv%PBx@bdQQw(Gk#X!_l{-BhMx49Ao;U*
zbGTBH2^Yh8S(O2faIcjlC*>?-8aNhn-*M#gE<ZYxde1ub)PxrU(xmHXH?;`*j>K4~
z)c|b@Na+Tnw5{4Du-_GfpVqv1`T38%h*Me+v-wKfVX=&>x$VOk_^WyS7$3*eRXzqB
zCTI5%FT1Iv6dac0Xy@Z|UjX3A9u$L>=qA<n(tFL58K`=FFOw7De+!(I$rV2c`C69g
zp`vq$BqI&bqPllT<#OEysWHAFktz4x1gL{1XZ!hP{6>+~L!X5D#C4jlm#UzLL9e$x
z{bz;M7nVFWhh?4IlU|=1lD?bYy#0yaQjr@9Op%LA7w2bH9q5qeIj@oz=7X=7t84H3
zOnc$s5H}iak4~Z5p{XpATPTsKKpZrbCioN1PTSBPhcL?ms4dZShx8+)xkg<k7pw^b
zvym+4MZP)8v+~kWodN&$X^>$>CJLX=_YoKq{P))H-<hSw7nxHjk1}VYck5dITf!%W
z_Omq+ib{$L`{hph`V%o_T|6{2G(J$c?oty|Dj%nt|0cnF%`{A0#{#j{l6;!V<$_o*
zaZZOBS&R`utF=$7tBqYomW&o^wozqEajCUsGADt#sLWHz<1+GQh_E4-YxRgo@&tgy
z))zT85>1$+l$w)@16JG>%u~-UhI|C}?r~|Eb?;sb=rM|wtECU<Xr2}qyJ)~Pto>m>
zjHH_)-KL;rIm&C3nPn}$nPRZqs_%$8duWux<+u-GdsBm^mDw(BcEUmhy}NN!S40aM
zHM>8jfp5<j#s?h5XBIqaCbpZ(PM>vJoNT^R*;z00vV<`iK6vI<q}U>TLYI7(4DM>?
zbt}nIgm%$LK}%~+{>!Dgu}xrman*ca@duCOx^&8h=&?j61QFv?<Ln7b6kXPh&at@z
zYwAERS8tv0jgC{DW_TtJ%_|hbjmI^e4^{6T(f0d-jt9BT#iL*Igpiuo{VU!Y9j^o{
z9;Bid>&|A^IT~ZM8m|_eUgRN2&u{UhmUveCtJtA|!s+?oTK$4&>ePMay5P^ZU_s95
zwkz=-Ck(<4AHjyRQr-F+%}9u{c~(kMPnAbNT2PlA^d2PLZiD_HCeg|~D9EEfV7GSa
zMfZ8)C7mOEY`pk@0++6WH>`zD#mPO`jy^osQdWZySrdq@5Rb^6j+!PlDmfhw5!$AO
z+yf!+jrn!mXSOKzdPt%ojYM+Vj?z7m+1E6N<UG}br*Ch&g*(dxr*8n-MSeWs;piK2
z8DSU!h(R&Eb4>>bq20hSdYtTd0kEO4*A%L;bXyRqxzr`V>T0#0UgSIE0KIET?j=BK
zP=wf`UG?VCqVi4_k>{PK>q>_ADm<R45g$~_ywyyH$BS6dpX@rz38G_m6p0-ok|w%O
zK-A+fOR^#PoTkw=Y=GQCzBa|XbnC_P*f~dttuVTFk@+qM<C0?)au1sUBXcCZ($Ra9
zpdcLg4A<|s0CDe?n+IReWdvKs6FluIMQTCQeMFW;x0T~973v4(!7S99zmmB>eh<OT
z=?%FPf^{p$gVupIWN7kwwW-2S8g@sPD3`Bq_HY@N1h?#LJzqfbohA9vYg8YP85d7?
zco~pQ$>$yd^&Vi}i5CI}o?Gr$pRZk&9Sye#lh9bQ6}OKnXyJt(GfA@FZC;|LAE>Sc
zl4;*%3|oXE%I*K8P|+Gfvek^Nr}CZNWfu6qJ=<Pp;GC%2UEecnWI?YqMg-%6_?Yjm
zoCfbHs>I!YJINo_n$&=FpOjunbq34B(K^*>vynFQ8_=N$)&+SK%zl$hx9O($(o}=z
zssdn&roQ2}hh1gBI(5Jr{IL$~fYFtz!~iPyPVVw*&S2q)z+sK(`CDUaM`!aoNqFE~
zSu%Id2vb8&pK$GNqYLIcZFAK5ON8ab<&apLUTnK{we%R!D(2iOt2TYsU3u;$1RQPP
zQY}RFeT?Lp?n4C)y65d;(zq<Jdv^2+xxMnZ6&@j@V4MmEcTKQW$8=?$VF>$7{1`|O
zTCt;legxQRV%-L8zilo+=Ua+yp3uj^UE*nWJe36v4(=I>A&rUhI?UuaB1|kfIl09i
zT+e|nHW)L#;HO|oIV3o*V#>?F=c0c0`Xb8${vA-+P@3MbX$$(as}GKS89|IH)Xz+V
zxz8+Rp3wp%1dxU%JWB(A*x70z?1Ui`(2SsV-Lxi7`{Dw1S-)(mFy5it#wOhr!S~=d
zWmGIvHuHivBKahsDxij2?39qbGAr{u|3m>cJI*I#3S#^41F!vp&9y`^n__AUs~ZRc
z;?)DQ*Ksxh?sk~Bgzc0BXTk|4d1Y;1isYn4wUbE((iJjI^svd!^*#x}lQIx+TXYZ<
z!D#{@g+*_ZM3)b1ye|7+@2IDsHlN~qfIAqVG`SNC=wYa{ic1oxg>>XxYeQ5>lEkMp
z#9Ub*3hBE-1gCWW@DFg~ccR`etbH}tTxx0^gkS4yFLX7;VZBwIC-r}ro&R*}Z^#O^
zHR|8NG^WO28ub&}M)MR1r6NyNl*k2Y1m)UE7l+K8L`}%E01i<cJgXx{x_3y*!&qnO
z&F|xWV<5IjlzB@`FPLQ_eTtYFvPj>_!NfRpR!{Z1R@ShGtW>KmQekypujke=en!PW
z3T>moH?=fYKsA(h>9|<}$WK403D<evB32l^*1V^}aXvD$dVO(0h<@NKjFJ%EaNtKG
z<JSjMVPb^PY>qwr0a>+jRR;l}W`?lFhNj=d1%n@ewd__M5j!%lp2)Ju#huo;$h93W
zcn%K!>YR7M8n6;awe4uN-{cS%(6ht9M;(*iBzb3P9+9ETP+Sus0BCEbYD_As!n9!v
z#k28G15b-heAGCi%Bz8}pKl*l4vij)9x%Aq9<L^K%b;%XDZtFBIf&Y<gNL;hz*D~K
z9s(S4D_dPR|1YbZh)eJ+`Ez&6J5;cZPbGld(GAHIX?;GhCze5xzG%?O$fG0qA^?|u
zhW<kU591xWiu@kwC?0s;$@rpAXyEmI8B=Q-Q+nwi5jcdpGKxj*q%SK}-O<eEvaN!5
z2%cup_2j(6A6vx;b}u)(6n6-u$@wpOv_!ZSOkgb2F>?#^mdk8fOVZYtt?jkNUOpqb
zJG_KYS+p%cwY7a@SDAl68yU#*rvRBH9A}k*G@XL5ZlBy6SN;A|w==oJU4|s#fF~k_
z_OU#8|0+ZBfK_BQnU15K&;P1H&|?Lvf+&jpFF5MY%=#G=t5V9t|I)76v3{oF@Xco6
zBNW?3;0UVf@B%u4x*TfyHkb+z%WJ}QQ5}ev@=&exof^DAPz+3h5A{jA*uX9<W430$
z?OM&F82b68u_*dT!UaX+eEwwXeRV%KXscN<(*D*+OO&&Dd30a>YSc~BPMiia=euD}
z&2&d1l)kBllQmaFso{H_RnySi;}DkH(bpno9l9ddf9_+>2yL+EOx>Ew#Q7kIyyW|}
zh{|k=!^u<5DC*V7_g0%YMP5GuVqHAyp0BF!4IFC|OX4314x|XMWgoi>N{gjS2e#EM
z`ocDIL}r6LluJ2IUj|2O<g2O_B=KQl^`<WpeHBCM=btG@ZI6;-V&O-ntOM>*S{69G
zuTRzuUVzzwQOOmcBagBqkI$!!hTw4@Fnt0@JfYtn34A;}6QG4KS9@N6A<UT^HTq1W
z>iKOXs0P0kS%P^lVEAO*qYafLTg<p_piwo#!Ts^>ryVgvkYVdy`Q7&BHEclq+Vco`
zmtp8AHa;h0J7pyZLII5~zVQW+87Y5c#l<u9SxdyH|4qT!%`^RkRG8_ppwV9(Uk<2v
z19BB~hdZ=%jHufDnr>e?lunvjtj4t8M<va|Po$U7Or-a4zPloP*kG>5<_IRb977X(
zW;+~vNjpCw3ILEq!)=TC(UYVN0NZ&$ljW)Alq={DfNmW1G(0u!s)R{%mdQpf$qTsL
zdR1M7yfx21aEA+|TgWCEHMvJ<MO~+zlg#iC1k+$Bn?qL#nsvK9(<8^0`Kk&as<zUX
za2a1gF1a(nND+Yg1Xx&D7KMG`d%Njs`Ksv_!TEK3elv)yTVLWG#>>tJ55cvslIW5C
zk#MObx^CC4TTlbDbic~APgXli4?P)1geylXO6dDfz}Ta7;u`%nImEQvIye@@2L{w;
z8PnYvJQ`E6B9kN`q^z@tQ^31Q(cE5u#jbO3?^98koUC*7<C_+;$qkyhY%j|OSd89T
zXN?b0bu|F<V+(?Y;AiIW?`Oy=vI~crDjOyl2xr`e2I2LZEE#A`7h=xssyBdT^*@q{
ze<thWxj~>5kO?H(lx~lr93DIU>M9wxiK2!juRU)@I!ta1!5Zt#q}pOqEk{s8r`Dhp
z_rJC4v7d`FpSb*nq;rkwMed9Z83u}hKSpuytne5qHNLWBB@9#zws_O`6hZ+|4{@=V
zOFk9R3)6Jm|9-xTyf=rNfDo->G}@`QGmi`?(m}<-rB03H<_xuN7)<Eb#V95lRAl3N
zY2iA}H5E6zzSK(Gh3+s0mO;~eQKd#Ec_NExdH8%P<7~F>ALh@ygG8~9lp$T|H|Oe|
z%Y_|hd&F?%On~I$M2uFCZO2BD3SydGT$=mXo%yaocSbw?(r>n=^wq*$X`k^AuK+UW
zxF(u;a*It$kBx~g?k7mMgo_EJ`1^lxAS6BCYu;}O!h>O_b4mi5tQZJlH`m%JJVWP0
zf(<nf0CTGH)8)s&Rj%b%zTQHG7=}gC;E^jg5eHLn#i^yOO+7%6wg=G0pyaH7u{J{r
zIT%r3WO3g_Q=iJvV>r+EDWCAogW83r4C9A?BEg(!0$_$yYr?@yUoh&H0IT-YVm){i
zT^vLMSVd))Kbl96P^d|enSAq-`h9LkJMk5nF}*}}c}bpGVxH~MPy=fZsVZ49oXjT;
ztg~w6!JV$a!r9ADh~vFkRWKe1jbY&8$=nC(7-Kx<rH%9SnuTN#@gk3V+a<`MixF0F
zI{{3pa{vU9qhPwPeh9yLN_xNLn0)W&icVD!c5_VC^RK@Y<|4`|(xKoY)&h<>VgWY*
z2sIsmN6OJ#92c`UaLI_eVF~vX-u~#(6QhPEk3D5e|BN19OU-#LidC6_yM?NDaYFYS
zXcF;<C__&$0r8ENJ>_1y`n8|B<2MQ5G`^TO3n$+t%{1ykTm%u|_&dcC%juKV5J#dR
z)r&T&pf+op*wIA~E@r|X9>O!*5{;}t5%1tfJ%0$mk!--{DqTI&Zl_57#1#I7`Zj4o
zNjg~U%OQl@;}J_(13b@;V;>RY&xY+R(W1_~rCYL6z<mzx<S`ep^3;mV&K2foZ0qdc
z)J(T#w#IOKhXQ6x-c_H&m<sK>Ji&ny74H*U_ds}V6XN3k1bP=ZW5P5<R)fCoWz+gw
zLXE~yQOLlPz>;lTyM3XHK&w2IeTpCZfS^cG`#TD@ywl~c7s%%1-d!?C>{Fw<D48@F
zGqdcACN@WU|MBOuPN{oE?sxR4)7B8(Fa+c77w0h>H!5KZ1;e$6I(+(e#hM;xOS)`L
zlW(+y9|!2g2JiNKsQ++T;d1}kpJmhHj~d7j&{2~*O(4AKUCpUpw@reL0wNI#yytB4
zuuT^LBrSr{MHQ6OsiG0V%Ed$)IC#2mzW~hUec<gQ|LpNt4IU9YsdpDyxCV+FQ&SNC
zF%DWbn(E`!%4nX+tu8URzCz(@5CPO9<fw{62F>(8%W%saw85XO+^*;(CKM~JIWEmj
zHF?YBr&#P!6vOIATOb)Gm_A<TXuxCCB&aQfzJ=_C9tN|WX)Xc+75Cqj5hQuKB(EEu
z2MTW!Gz`uC{+gf<c7n*Fcc<T{>LtZqOe{2KZOr#mMO1=oCy_{(u%1RxkX8#|q*>-(
z|EzeXY3WCMism389G}ARM+)oo_AT#RIF!O)T8B2W49`kx`JcxlH#GO;ZEsccrQ;wx
zj#xz&FoGrF^JB1cXK9x_MIju#9WdNlL|B8?RKt-Ku&cP!K?lg8;HOAT>V2`5e@*<`
z9K~YkHN<OtiWatsZcj>)YCz*G*9a(5U7%*Z?|Wji-@E5poR?O6d52DAJJq=jzr-5e
zcP47979GP1>vFm)FrL7@Ji9mHNH=2fmAIFFH`j8cFiO{yetS2AC*30V&UAqswP_ru
zqhe6dg20@OKJ_`EET+YBKKNFqLO5ALHp1<FH{IeczV7m89M{jOXx&)*^VF-l`!Cm(
z(LI7?b9cElO0^DdDYj2Wm{5OKZ7(H7IwQYcf%tHw+#Z6ZmbvrhCmx<IS{~1Yu$LS<
zdGN=b^j8@J%ukc(f=hv6EnsSnwN47LWLVES=0Zv~C6M;#0r!15j2aX3w<<!xHukb(
z8prbo7lqF16F@U|i@YNBBVwLJyPMvxWT!X(EUEaS-P2hbwq~;FAE~o8TlIw~)4EP`
zlRC!?bJ4KNU$oVXzF2!pvG2!^+85v45EZY&Qp<*8>;uMQis$<f2iDR~Aob$Aq?Ud9
zWT{F1=^nnx8Mc-|eu+ct4=#tc4izpfJ>zvTj3(PJJUR^!tol;|8s+7r*58VR+!g*!
zX1QF&(MxChMBmIc_OnIHLg+uzzjEKUTG7xtd()wtKHdMlh_|>Mhu=X0#VN^mdl2ik
z;ed_agd$xX*@C8flk2-hLoH8rBWS3H)^uRNg<q~`L-Du8K&@a2PtiG?pKkL8!(p(3
zVS@Ps%QJ?0Fz04+*4I=o$O&PhYGi!2t1$fnew4?;!pXs*ZTqw^HK{1#va<H+n7WYT
zsItl@A+2gwm5C0Ie#eojF!<+b?XneTI(Sud9gEXK7946c?kUt9NHiTdA8WVg{C+@(
zM?WHYhez*}xC#sI76`a*dvuZ)(<`4=#S*nzV_FrjaD%U?@t(4MM_KPHVt~FK^^$Mq
zk)C4DhsUS2Y`ZnT3p5i-cyi=s*~6;BEPA1m*-XsYH3c=<M5QK$nvtV<qYoz=kVE_Z
zX(<B78SvJ9kNsw860gC4!G7mK(U*%UOUG%Tpy%jI0Y#op+`i6pN^H|xf=M_-E-MgC
zaTq-tOEM+4%eQ2cfJUa{7aMOnu<dIA$$c0o8S$A9auoR~v6cC&eF7k6=Jf<T75xN#
zXNOOoa^Pk0yv!$D@YI!E2AIVxG@+NR+kM1D_QlwTXacNf)K9KsO3;D7i$yqNfx`tV
zd&SSgNh2qVk>nwLv3YmFF{i0H2NLuH4M8rs{<f5Vjy5EOCfxI8^CA1r^!Ae{hHoSc
zDBO0{Z9VTiYrZv-LNZ-$OV8yzu=PSMemYGS!hjQufG=exRaCVVV%#Lk(j`FC)xLm;
z5?#IL@3f`#uK;*V8Pw&qtZ%8XOH$rHk?^Z{6UhQe0bsz{PhWs`F|vhtZ6oK$MrNSu
zvkyV!PJ%w0*>5)^xylYcm`kp-D(?jIw*x|33p@KGnLBif)FYokH2VqA{!m4}EtM$w
z-B}bU$PpcavE6k+|06-aqK`TOa**1^T9LN|5M~DI{SVk3vYfxG<Bd>dshLiN=MYRf
zJab~-k)%!tgd#8OL+HvOfl;(VSWu70vJm5ws(U7vTaPD*D@>*dTf+kkt<>UmWo1Z7
zv9U4#*O%Z1vSGj(+Axt3vYULi)^0OHg?zU@%AuX{6?#o!Z~ruK>;g)!x*&ZeWlw=Z
zRL6;W7>JBfua9v!(LeBT=r^NlMmd=;-hR1_<e63#|7e0syb!&6_4eNJwy7b>_kPc#
zC1Tdmua@%k-J&gGX9f0{`9%Bo4oFaU@WkC2q&jZ594r}17zR-)5<g_v?<Fc?URnN$
z=h7;X(LL1ryaUB0i)1`^n~yoB3Ldol%AZUuawkUAeRJ~WoDXk4F|W7#Z@f?i+zca*
z_3^|(Yt;oO;_~R<?#tgfjE%OcH1l?x`U|;=8{F5tH}qY!+Y5`^_j9-o85bS3Ev1Ys
zkz5+SA~8yI45$ZOC)@|9@dujL5w5p-p>oD243a1d!u&ygCYMB}kRZr2ay&ul^_$y2
z<{ZETO#B>z6~CCmMU_46)741Am6$~#8Xa-IeBm_H64qvNOz-@%<xRlVZ#MnQzM(73
z9g&Ol(llfzW1%MfUbyFNku1K~p>{^udHnS}L-z@op!%;4EDCa>d~1@8@_F;!=;=#l
z-<|S5z4YE}Xu!UG7jmwIs(vp`ssgS^zB6*ztMecsMt+(4HO>qEwkF7P!?;Akw171t
zA-0l$1Jl@-D7WU1N4o2oLeeGKTLKwbU;0)%lR|TDzoZSm0l9Asu!BE1ElVz6C*OpK
zt&s3Z$~+!rBETxQWzT7XT!gli<8(2N@i#`-H$p_e^4kKqY$4Eb)o}+um?zQwYS`-Q
z0cop{0gKrI`&AlE0_}C;+czJ`>2{#g9j>|71+nABcHnDo)OIm_eCtITgr(~0nKA50
zy>1m6BEx-Tett2gCyTK`&~axy5!{9HnT`m*q8EE_BnZ;dzm%}T8WNgr9XP0PNR4{0
zLx#^Ca;QUhH<~Yqa&Y9>nr6A;r<LU}ei;5V!c02c-vI*=zhrEdGQer}GxH9(I#iH5
z!xg<{aruZ?^mT={4UqcG+CpMeuIc9T7&gSJibZ&p-m?JGX~l?}LzwIr51*1l>A@uB
zBIv~$xU($}U|bHh)^#Aj8j?7^5qvNDl<mp7@S2kDfC4S$M>f^8C%P2&vOW@>4*Woh
z)C3Amtfv(+Bo8IHN6n*NwPWHIr2CtWEY6qS2m^3mj%|>Yp4D>e0XU{N0Z{ZT=Kzy>
zXacC1yj_J!25&Xrwei`h?A`<s>l3oYgnh@>nFGhK?*Ns+2YvC-apE<s9q!>m4Ymo~
zgE(cv$yevQ7;@ODlsaJNJ6Zb4Q>7xhU66|>em4vUmV>7CR=mWZlm+4?8IW7g<PX@c
z+frE)Pmn*;@m$IpbSkkGv7a5>!%?hgrxz~+1(~KlX%_-qu+61;!Q`z@Qb1N5-TgS@
zxjpQ2mzX7=@+0k^(_vWl_J+8WR9QDoDV~#7WPiBpYW@CB(b8x@CE;6g|I;){o{`E0
z>c`GM4+V1apGHC;cz9S~z5M<1OcUE;S5LExIgpGDBfm^3c~H|U(_don-J_p4U%!!^
zQ!?g}BKxddw$Ib2yq3zdGXs!u0akafpJ$w1?g!q>cWOOYqKqZ}M4-28sdFB<X>Jv0
z4t*E1VpDOY+!3hEhcy2}KsC|ax;<e(ELD4!qc4$f`gYj&ax#}-?T7e}YrH6Guls5X
zGX28OZU@a`?A>>2-uLPl^!R0Tbi!t3!JW@L9y(F`Ll$g|kr6VI85X?zBk{yy1ss%l
zx>-CpJhB<;GLs%SPdN+t@5Bg9nEL+qLtMXD9aGyJQ<M5ab@dgCk&wRhq0y`t!Y$zT
zs{#X|a{O-24<wxAnAGOf;RS37Z$hk|=VSSM#Bq$aE%PB_6rMipHer~s#QFRkS1|7p
z@+Dq89<r16gM$Z&<B`{T&Y*Okw1Fyxx4~tHR06D$uM85Ki1%4Eg3N-NG$4zt6law&
z_O?vRn_72W$JY~BJV1l4#rN2~ft>(L*H7jtN75H%BR)J{Nc2Z>VvB8JEFfAZ);-{k
z3<>&vjkiYw09={q1FsI<!5ryX=yjrgiy@LK;v-5q^+E^+6H$+Tp4>57odoVQbv9Z0
zQ$=EC>l|Tg*Rny>&`oSyKI_ljR(B*@LfeO=QOcJGN<rS(imi~(wok{2u^{2WHlJCN
zLS(#kq5)UWkPOdb=5b?60mKDt3>xLe4!vVkq^%%Dt2p%j0=Vqpk!gP~#47u_Q*w+P
zhTM{H@$$gMtR_AOCWmK2CZ>b!S0SQ{tRSK*9}sBAk`lm-FtCoAj`a64Z3jGoypT>_
z7CX;cgnRN3voBklXt!~2mlYQlL^kwBDGnp&p(_>^xuECu&=)eg6@wv(UhJ1|a2}rm
zi6Ap6dWM9{-IW^Q&0t!3olsf-paeHqjN1$cGxj7^#JP++=Z<r(eAH%fc^g5?(v5vq
zoHd=7{<tja?D21K2uS%_4jw{ip|)A$rgQi4p<(FwK(RN{4EC5m=?HE@N);%skTYPe
z)3Pe%e=!U~9a~+dLznc=gWjLkC4u{hFm$LJ{d<>e3``s%4VeN)A-e?MO#APhOF&=m
z3&zUVnu&t&NaPIBYds6t2*~z*f$mV{ndv>8{nG4bm|<8vXqltBHfFXIce~BeYOh^4
zPWp8g+^apkZh6O?Ua;?6Q^fWr30r_Tz{#4GOzUqP*Hcyya+!a0H0%HACE$579<tP%
zJ`)?WEA_$)Z$~0b$V?Yh4)323hs88UY%UY-(zNUZ<is%6=Cg<i&rMB6d{^71`CsKj
zI3uQ>-}jirRc>;uNR6p#Hu~wBh+lOD9lX*}DZCB}^+7SjK05R~SIJN$GF8+g^JpKn
z)?a8KYyVNBJFyT|FvL-QQ>n-|8t?NR)6Zf)4r}w2KcS5WED0!!M&}UdLz#;u-ZN<i
zxCgk7WpxNzm&RpOep_XhQKVJWl*@RG!+JgG!A9oVRT<-V*EYVc^Y<FI7q_Ff`21BB
z2OpF6XkyER!{#d~(r5B&XE|{M!gCE?TZV}jR;Y!|vj>vyS#Uk4v+W&aIhh1`zWII9
zEAI{UEnVEUTI_x)_~^K|h?u#)X1y+6x|J5^)Rlv$H&2kb!suR;>kZlUGi`GD-QfCq
z+1Z_$r0_}1iF7HYh3SeXd*G%hnUj0}VR!1i5m{WuCv3O9S}r~ks+S{YJ{yD!qh#)q
zJ5o~aWNl;w23s8|z<HF#sL8P1h=T<KXQ)`qoEKa4^v1Bwm_7wPxyE0g1S(O;GgEjl
z5WvEAelV?qOD}1@+XEWV0Do{=YC%L7D6xN43Pup(QRI4R$wDx$<C@;5WWJ}1CDMD^
zgA4n(-*)NL@MPSDlM35K-Hcy2;i*i<?;Qc-&hMnUNkU%cL2`3w`MrEZ4vUb@aFEOk
zew-vLkZ9s8uZMsH0IV~#^D!v%kypoRvw($Mdip8u+2^l=4C{y(xqG<TgY%9Us{*>(
zDm4~4A?+Zd&lcd$83fUXk&){+D&q%9xc=n+8iXacOuVzxZzR~^VERYTervcEM?Unn
zV#1D{L)Mjh?Zf)e<cpQ~-6<W1pvdR4g6a%KhDeM1oQvOb2Am;cpWn3}_({BdMak?i
zC!pw1c864(ba;y7-BtafZ?QVou<Kz{ITYE%qyf8v%w&R~ATb+{&dKpy1S?*7Xr*Vm
zu%mvKD!Z_t{bc{UM{}$rjLgSC!IkLsu%zs$eG?yV+bl)ET2<!j3E245Edtr;6)4!h
z%gTP$RQz6Rwo9(kVgDMaq_Oc1e!79VGQOce5A0ke?9@px*e$403G2z(Gty|5pGJ)p
z?_>nI&c11pw(m=@C#Amr9ZfIJ8Af<~I~#*sTlcFf!8HAyPeJW5r(PhUs)ZY<Ni~}F
zyvgnieY|g80OpFu+?0k^{U3jZbu#P>TJ7Dec=0VV=$Fa^6c{ej9n$4izzRSZPLt<?
zvqyRkBrC2*7JjhzSmBbO2NPQ=MYV>xDd#!F`6(_qaYw^1PQT2}j!6p~;{T81IF=S1
z$0@Y))eTw$h4y3XP)*L5#EX=yoO5TKS0g(*hSY}A64TysD=M8I>5zJA-sQ*ZbNi(4
zP4>9)+KS1L&wGzzTo_*kFik&}Z4qUHgg}=WvprWl>DJD1zo|TOKDtaEOKdUD>#H>Q
zaMdT_vP~=&tNP-u!@!J;;@Se&p+pD;$1CK>)e^n9=jicqW`2r*z*~)tpR_kC$w6xp
z%yc&hf0Y*bu~KKCwXsCpuYx4>#!yfC9>{=nu};PsyabcgF|L-{L%_9ZXT0uJz#aVp
zgFr#iJLJYX9<lv|ly$gPrp_WyQJSfcF}3$+|L+3ZO>B6#ACi7$gs`*cFdA+qP`tkX
zO#Z^;6{1{oOWpm1Oks2@KSTa4z3T(Mmp3(+i=43i?(vz;F-L2T>~wdqWn#()BG!^?
z-T4A7P!jY{EQ1*!!3j*Ugp5B%uPkW<Kd3O_gi}FGx;}ul2|fAsj?Y&wCob6idaO9D
zpiNdNh-hO&26t+^5|Zg=z5ZsWp<W8X==+d|v)kr>ISLd?mKiw{Bp)91{L)ucl$Dvn
zeK=v@+)7pPi-1W0Z{Sd|hzfgHL56|<sXXK~=oyug3yE^^9!nHYf<e{;bI!*0A3Ny8
z(DWq5YxjF=KGP}5Vl<Nn?N||%41`=UDw(!Yn-3>|U8Lf}J0t3vb^=MfurPdX3PzxM
zHx|}a3=^92W}%?MmN88OQZ%oKcQkx?;9hcuu=_~nDb8bl9tb%xov<?QHK2UN+l&`K
z9oI;#wPMe<0qVsBdUak~qyvwz)J%Z?x9bHrsXo)y@w_1wihkG;@RE6iMi%z-TZ0?N
zCpo-n(7YQ#K_p1?tHhu5Z+_jm*ZJ@QScWz@;=dkAQBv1~@aqRL6(Jat6lRuY5(Zh`
z#b1Ehy2y$C%_R_Jk$Xyow7dBMt$V_fgiHhxL5==_IPT;oVLZr>RYxHatq!^yDa{1l
z_+|N^Q^D*zfn2PlRi=ijWWC4dZ=~MAGOf#BQK6YUt&9(*-Z=_A4d4zY=NiPs_Tz!%
z&vEb5fRg(3(a2ZL@d^a8-6Qo!!86l0OfSvcKn-2IK<+NJ<KjAFnY?9Mxp1Ir^2|<z
z%Bhp+b>fF?&CptPGjeTOIi&Y@9kWwK-#1LdHS{A9*LK)Y6E7bVzMUN-r%!c)<vuoX
z=7-TZ#7ETCu>7||P6-MsiQ)grYuPW#&~-hk%<_=}XbxP4X6|Tk1w<w)z3zEsxqNh2
zI3fRmM@#21*iT*uCUy_&-WO;JO;r%Re<UFJ<lOmr<qfR2q>n+4sH`j*V)<cy8svR2
zf8kEx@`Ky{34MhtUd!untqrWhRv|Hz*KWD}R>qB=<KG%^f-mo&rvc$17<C(alV7;f
z{jKN*zdrhr%6NO>&JBLGk3p&3Y6Si9bRteU$<!^%e63gfUmqsK^QosQ6RG!yM-IIE
zWGLOjO%`wY_;vhHZ~yN`SL+Poi4`k6`<G8I-zP5*wLbDndj|b!Dx>%C!UQk&<YwSR
zu}<z}DM|??*-E8V@a*dYj<LRDk|pJdg>>=gUF=Og*2@n4<+I(nf&O1ZTG(zs<(_>q
zG5^@GSggbMCxqv$2hj0{gD{Z_yB(C+#+F~?;w8xVIItD*p2^>1nIOP2Qz}F!=X&ff
zE=YXCLt(ajn+I2ZzvA*^v+t4nfo)gPKJ%lJ8GtbB)!2}wyN^CKu9FSDP8KeG2x3+%
zum}iC^>fG~-F>-fCigonn5&%(W`NgF1LP^TGPfjL3jitHQ8YAXX+Ri#0wnEzBOth|
znv;UW2LW<{PRroVi(-N;=3!-mC9C8yzpy9?WCB9JdGJ(HVjp}k1qlGyazs4cRbMc5
zc_UcN9?~6sS0@E?TyZ=_O#KQvydG;5^0GX-s!1bHIr|-NQ;$-pB_jLLTi>4@5+4mJ
zlwMQoehXWheEjrWEIZ4T)4pqL@6ciQEAuZi?icVd^gFjY1fJVRr+N+gAX4mAh?aRO
zGuRh?F#GBdlynj_6Q^-q9NcM<%%f(PoCh*1v5#UR1DU1YuCvaSmrr9*<%YO-p1r^S
z_|#yQZ&QnvEATQ>Qso609#n+Lv<&6jWSO<XfhxhL_i*pUYb+4dYEAeQ*T=HbDO4yx
zqk{koOb3%^iQ4-GfGq&yH{Ik?1+S3h&yKK+oD3DPJuRB%V`vl#Fjy+p1Pf(1hU#<o
zU0$T2&5uEHC_3ksIMCDQ$4k;|V7wX-GBy)GTE56;E~2uoRPAg|_Fl|2-3<|XpJWqR
z%90JiD86ji(66|RZCrdL5g_slo=eT1_sRY_J)>v#;~*o7n&o;`Z~^6B@z`*=@cx^S
zcV}~)B>Pol`?KVc+|#`sl+Q`;jr80P>>P_wZSD*hDPDr;yTqe=+O0Y%#gkLk>VU7p
zDK<(5|7~ew@$q^t*eUkZXESrx$J4*1iQTN_?(y2`=OQ3K-Z)BluJqO<nItzU(tjYW
zDG+27m$!Au9~kE>O(7V3(yx3p&z&QCzfCV2RCZ!xKg0XUekILFflDU;)L(}NnL1|l
z5b-9o@dpYB!o)z~N>w|WCgz_0;2bJ`^WM#pS81XYhb@Hi(#m1j`|g~BGP9J}{5LUQ
zF*g_PN7a~gUVlY(Q?+b8mO+m*l0>(wK@@obbN0QMvBq0%9!M3Yi?O+em+<nN-tYse
zA{(hq{F#VTx2kUchAr&1?Vy)$oQU6I_hZse*0R&c8OEIj$AxBxgpq_m$3$;ywq$fL
zYy}b!5xsZwfJlBfX*{CZd_Sg^>$#Dxq~%%O?{x!cJD!V{G3J-7-2dWw@xvLKcYkFs
z5`qKKV(upG38Rs<4Tv(c{}o0yC2Qpa?gO!Y$cAy?1+jx%icFb}z1do>;2R7ovo*Gl
z&<)mOH*oYhHT-<j9*qg%6|x%-4*=1W;V&o1Dry*5tGhaGt==Jpxt8izq;>$Y_2*D+
z5eG}b#dO;KAEsX!l6Gn-Ju}^<Qk%Iz3C;AD=SRrB$!cpg0}9SEFnNu9??~?ZYX<+?
z>)ree-yg+raCejoyol3;sSNB88M3dGq~(h~8}tI4%eb<f%AcdZK!Q5Vit8)te8IPI
zwuP=iR)bfglC$%=)Jbi?L=gM=g4OOs2?+sq#jO2VFYttrhx;B9crLaR67fLJvKPH6
zMcq?!8qm)yS5-v^;jhn0-X-5&=#ceO<xS(oi%H%hz`7apECbU=T}gs~-R#}yVZU4x
z@^usXh&`*dzC5L(%X*<1>olG58{_$|)fu60oYx&a3WTURA9H16kc_TuR)eC~G=2B0
zJad)Tzoi(y>f^wOdFnPEgZ~0Dw|`Vb@Lm%sfdp-6>H3gu6nGs~a^RWL!$B&M@5HZ0
z>oWUR?02~KX$NI06&;NB>2AN*lE=_DU9&m6VnGWPrp&U08JEEAm_V$*P|<<KeVY0?
z(Eg6dQDrv+;f^8ou88bYc08Gs&md!Yamx^x$$}S>V@b6!7#_GlB!M)~IK?Id#1mal
z_#BMsS;JYhd{sdlk|Tdk(p+k*WDK4JQx)BMC!_M;z?~9m?873(!pqK-+AJ0x5`=5M
z=d;bAFn+;s>49o^zZqtHR`a+!Z|;J9WGc1)0%y)5hsW{1g{}-M1Om%dZcwuIkB-#v
zGB?T8*f>&SFAvgVS60T<!V#oOE&_@7NF!u_^|8%YE<*3_10Jc7>k`zR?#%>EI*iM&
z-jd16>%2x?FQ8wg<bFj87Uth2;I2rWo@GdXkgG*4GRu=4>4W>ZkqN<vxHR!uAFmZd
zxQp%2*IflvL<Z)%G7_G*i#%7;IWSdxk<O;yctl4l>BRkV`&FmZMO%L8yRrJSA@>KO
zqPVh<bg~B*o4H$Jbq;!!4}GLmZm=laqgp+Nrt9lrj7Z2jTv%7@cS&?VPUU%{#p6u+
z;@dANpY++Z^y@bg73y)u4i!`5UJ2h(k?|AO`mXfh3@30aaPedNM{N=X>oDq@CIRhq
zk^z*L{AO1mY&~gE3g#ddQy-0+LablqlYME@j<;!xsS_Eq?^TU63;(^+@PznoEFnne
zxT5W3*4K4#o+zR+vktBZQU*cMnsVVHV+0zcHq#Z^Rm^te4WN3HhrE^X!O&Jy!JM(W
zIzPl@!A{Sx!$C*dyaV<jQGc-?VOSX9(vUkT>iMelpB}ir=;Foep#Z1SmRZgN?&;9i
z<O3h1xRXLCofW|Wy#;zwfn0*q)GjWI>nUG=p}xlZ@Yln>Hv&76;Ev?ukodsI&w?jZ
zx>7M;UX#^1kwjg9x{s^`maF!54dj?I$8IVA5-IuF$=RkN^9J|+e8MSgilXG#TPmaB
zA?MapZM4IjsU(&<4{`u33!~s3#!)yod>xdeUR@7@e_?oDw)YKJfVGW*PoQ`+k01M<
zJR<>@V3sjEj;tOo;wfQ|Da4WLn;T{VW>(h=0?O+Z*WTLJPAl_8OH0&$vtJXMUkA<b
z-3y$aSdb(iK7Hq!g?tU+8pg}|8|*p2O>xJtmPYkW2nKEE?OGrq^4kr;h&0mAi7Ev~
zqhcR@ns!5@QhlKggj(b0-_^C#=NA8C`5)tr7$bS<gSq_ba7XwUadVg<c)+F;f>Bc0
z$$42uFv*}Bn($htkisHR<HHvSMwnqqU#%&kh>F17kXxjLosi$V#+rE>^YC^g!TKOZ
z-KoeV(Z)eDInR_yprn~2jZ%g7LgHzC=^(H>MSy7?U7V&NPOxGkryTn;Q~yYq5Dql`
zfUc-P{bsPqRLrF-2v{uei&-jPr`6u5Yz&)-@pCL|KBN<w-827rk!|A#c5T0r9LTb4
zoL@#O$z40Tw7Zg|lK~F}%bRJGi`W(>6`r=9V~em(v)(i<H2&`u6+bbi9;4+vk5WE`
z)AzI{`<VwiES9EGUc`8ZYA$8xzi#jn%n!8K!^e&@%vQ9HQ#86GG0QBlP~W9>HuNQO
zh}f1ds?q~L^FBBFVhDZleD%<X<gh%=TZ%JAzTf@y$-Nc+(7?Q*>E;mbhJc-$e!|eZ
z9uGX$&dJl`c3ixTp!Y+~HsZlG;3#V0rJBCmc8g!Gb5aBPD(C^$?lhBaJ+IO46k-ny
z>t(L0+iGsnb{I+QANS)vDU`P$X$>+zslr4`yw-1+cT9!e+u5#m`8AN6Nv?kF8GaK(
zI#&mg<?z>v;y%&C^%1qWGx~vhQFw+?xwmKz-tupnPES~-c^_});~_(!_YGjI@PAnV
zi3QQwmvMV6WKQglsOl=FqDn(De>_&4@JSCfxj|%lxIXAE?nzy-)?iz*qsec94ZU+{
zc>}BQfNApxe(ssIWU%@(SCn7ki2FDBO&q5CJ%xEex6#Uh#TSQia@aV(e}0Zv%>ku1
z?vwH+WitY-p0^jg;3zigdTF+H3@$KONteyY$eE*oHhNTv#1GxdV*NjaeRWio+ZHde
zk%mo)^rjJz5Ts+%-Hp;9ph!1JcY}mTcS?zXbSNnxp@bmaE!}V7x#!$--*{uZzZial
z*nVrRx#pa|m>j*A?Vl3?_d2kOtP5<R_k;z;=v1G7GHdA)%+TCj@V#7~N9*uD0J20H
z;}QPY7BlX|mD8mU6-P-FP(*aV`mqwpe0_PUlYe3V3CN0=6^^ZluaWO>Ik1Cnh!~Zx
z4I}$*X#m2-bgon+QW1jb6d)lIhjAgR!gQ-mQIgmUrlYjAWq|xh35?f7_x~0<dk;Py
z^2CdaoppRMvzx}hgPE1kv#Z9u4<AXS{a%xt-?#4oi?!z=Vp9}Dh`>{gL&Mw*4d?`u
z=p{j%&{;!F&y{0EK}_t>y~k$a0cc%!^Qbj-JNqA2pa?#)8YmSIMY90S^U;omXY<k3
z$We+@%^n{bpF_(zVzP}9O^Fk6;dP<E#ku|I4Rj|zqW2~gCQSf3pVSeHf$jP46*W0`
z*k=h&MxkT}_B%mC;v@SQ)dqMQMy%jb6CjgIGC?=$78n%7j6a9raT~WIe6Y6Y07C1&
z7jCG}2)dtKuqPUI=F4xt#j8ACgwxVf&Rdpx)=e|7{`oTcn2C+}hfGN$#YV`hOA^WN
zq(4XA{!CgwB|6~_AGC&m4BcIODYdptJplu$&LCVJ#0aU+Rf`s^QJol#V8OE4n1F2c
z5s1T>Ogo*X*Pyvp`eUaz@K|YQ`XlD7QZ*e&%)qR}usI^WltpGV<JZVv75Tv&IV5>f
zS(b+=h~pN_#x&&juWkZ5U%I!D3|V3lYl$J8;=L?#Uj{PMg4Mw^FFh%-_CT6bq3>zX
zk7>`_c3q6{1~+U}6r61u-^b#=Zs{vGs#0Q@9!>rmN`j%8BO%p@^anL1Ca<9>8%(fe
zoX^62_OHaR2snJ&^X?QZ*3(heN2eFq{#R$FREGoJOo2Ed1_mVrc9hqI3r6U-%rg_t
zqN6Qz|9kvyQZldY+-`1s)aq@=6QXCAx!jbMOm<&&?IKq3V~rfRTb}dnDW}<{3(PWR
zd`QXzpXjqB0_1I3MMHq{3NBNjB$;ek>r*8H%6D)hpO%CVf*-^=Dg~c55QOLz{oeXI
zEtS9FhAAn1&Kbi_?uwaPqkDhUg>y&#GN5gWIREw`gogUSjNFJX-DhbJp$^~sdy)=w
z4g1jjnm219LWU#7Qc8$2>yVSy&g6c^{U5tLEW=DJ3185bJ0r6*o0YuJEP`7L2+|Jb
z+x=wU6DHLn<{qiZP^`p<EU7{Lp1jsALX8(RL_8ohw;Ee?0Xen(F7Y#aM1PxSg#Prl
zx=9o2Zvzm}Mkl|R==3&JUEm^NpdCsP)ne8-H|LY;4)66xnD9(#fkj23d7LO!>n9o0
ziV&d1?f*o3TNaPugUmqXlGykrFj7e3s61>B9!1>^|Fq?oYI`V=ikz(Zk+2Zv=M8KQ
zFfstiw^25St-ReApfCQeAcH{ow0Y1&r6XuWI1Oh)b|q-9z;~xs(?qn!jF36NK-;V?
zU+YW|E<e4mQ$>J`b}#~2vIc|NAPGilEAoc8N^d!+pI4hdM4P|L@ix=1iuN#-rs<|6
zG;h7V@uKCCG8ED{u=$9dS>j&zym7hnf=TO?X|`&oJ%6|Kc-!i0qu*(pp~W2mL>g8O
zvj;8^4}(7%pwK@hf>b?!7_o9;paw-V1DErj+K1-=rfZ5)TEgYiUlcA<RE4&C5^ZT~
zWJi%1vJf8E7wQ>JQh{7XP}cZ8<yJjHqCj<8!A+O4p1!0QoR->F9ItrVueeL&gyH5Q
z#!B1EOF!3a?A7bU^)036*8#5{k0jUqmzDt|fW{rGG0YUy3iNOk;b=AponpPZdZX3+
zL7=t6=gYm>T9|K~_*xHNtNkR?!>f7@cT)1Kt<|ozp7T$viIQ!8^SJI^?pFJJh8IRZ
zPTG)htQ1=+5Q=c-LdlUbHqkv3kgM6Ukaw4lncGginWFhzd@8?RYdr&fpeh;!%v#<L
zE|at#ktQDrq69rTy7ByDDPyvV0w@Hn*a)IB%?C6i;9Jq$6HCDd&C%n?P6^S9e0jaO
zKKUmkQdpHbKMo&+1DuoUN0(D$=!jhY?Nmq%r3<`=B=eqm$OOxlN7QF$Qr{}VnfUkI
z^@3~U{VZDVD*FvSt)oP@D)V05pi+&lSDT?}r%O4a`9E{VOGE^%OG2FAK0^6w(0CJi
zvd?t7@u8pdz@AdY2V*9A7iqErvjbg?2g!ese9R9FKp0v95>JhwMO`|gWn}aMa4YmT
zUI830f{7l4HW<O`2(UI#n`?cYA?`pf2H)$}Uo)vuCYNeqt@68B&x_T6zv7kpOeL2P
zGPGdX1As>YD;!<p&_G?JMS-Cb^#b|o8V_q2ssXU^aA5H!l#)aEaaR4*-&&9RFBXx}
z4nAI;<X%SYLI(~xtQF9Er9T$D0h|5Xxv>&uKTGhjSq9be4QS<+&OUH{re**=aA04e
z{z(kZ$z>>iPQ%@C+!S31B1pvV15AD*9Vb|iiCgR)sp{KRpE|TX9FD#YNXrbm?lC%5
zAMWi6x_ZfB;-UE0Hei!@;cwKPY=0Ik*=dcX7)o7(zP}yGWT@tkzAFzw5%@yptpN(k
zOIzKa6G-@iVLMGUs+(AQH*d-*AXT>QcBGF?YFcArS4|o7KnJNyq1%pl8%m_G?}g@J
zk6UZ3!@L~@>F~V2rT=JB?GdGv856qMqY{*xTqePFNBer4DRH1yeUg{h)*y&B<9mCJ
zh%x}+X+0wAwZXKaM1^B?<JVYU1`P0+#D7Az<e3KFHOVYo^ScUqfr+H!%BT%03#UqG
zTLxbuN@0E<1-eaz@U=zaDIkktj}#6ySKY`c)~48PyHq4qYX5+eAae|F{bl}xeUgFT
zm9@gZGDpO_%(2rzKFw)7y^$8jOqs&T6dvin%$a3d^mu6PP{nn7M$BndL`h}PC&JBs
zX2E<spD%Gibg)e#SFD;7$+kTHt@0IFI`3zG;U&83-gy@i>ZG4crtU7mhJ(_aCm6R%
zXLh@M5~5}f7s}bF8?qm^-1%D`OXGEIB^?V%o7SDS0mN=3nAkP@aEL!v-b8unrH)eu
z4Ib^=;Ukl;Za4yGnL?i1y3@(`(gl`y7VbyhktVSHf3FOTwLi7Z3B(B~put*4;$e%5
z-(Bv8J01mw!Rbf^eVS?JrZzhvIpvON!CJz*_RbUr+9=2lJ|YGxpXtbYD_=SqD37gZ
zMx{(t>SU3QYL%z-M5=*|(^|B)7h}Fa^#nJ)&_N^K&=4Hp3>xMKVTXqso)F&-(W+kT
zBjUELjhn*DXdJ<%**%?TvL}VDuPch)eSi5FapZcTfPqg@v*R3ByG(||8cJ0q<p{Yu
zq2p!`-Hz2y$OJYXSRqc2C6Hr|TDAFH7ki!VYJ#s}KTL#4@Am*vmpmDdO*Q}qziBXq
zfp&XRK*MWLIeY`8c!_9_`uM8sjE5S9->OWy5-nt)8VdS=<U+5)>1qes3E$b`HI1JT
ze17oL#V-!&F;>_&7~|bERR};xtFM-J7_<qaW#IQV$BJ+}5{enBf(HEek+#lNg%CK)
z8juFk0M)&u#vHe7p)P|8o}0>Tt*)8N`J+j<=bhxfd-!^9sAYilcY@EPT;w~X4x;wE
zRWrVT%A@-MSiIRmYDjmf%D)jy!4Z%TE=$WQFpiWAdVsEG!{o<&k8rok+n{oHLWnAE
zH*L56-mjMT5chE|Fx-_?*t3>27UpdPx}lqd`?=ZHV}(b6GQFu0u@Rt%T7ZAVMRrbc
z3}`rs*@_4#Sqadn4G^aE^(MdA(rJ}yh8N#|i5N8oDpFn=B^f#mF@#sSDiS#mGw|f}
zd(pzD2Lv!kr}{4NKq_42VCM`$)xN%)ZO<;#j6u1+7Ay2SOve@o3Ayd(;q2rC$c~h&
zCfnnPm_j^aM$%LgWY0zGR+L_l-`$SWkYP=sr(6b(Ca)7qy%KFFU0=r+2dJ>(v_Eda
z9_cnhRs-PNU4sdsiOfH}r7VB~4Hz2IR`PAe%BYR7sD?4>0rJW1+n^ng;<LcdVqM;S
zFV(_i((~gp%n?Vq$!-?b=1jWZ+6oBKdcvaB_qy%U99Q~FykYfbmbLD`+~!Z;YCndX
zFr~k13)c+(B19-7hA`oOI5rWYfw)1{R@V8u9T~US<wHzQg^_Mkaz04jL$W7xQMPo-
zUx5+nm3cJJ%EuhCjs=Q2Q2YTeHYEEhr@)}eRlY<gH1E^dbsZRjuJVcU9Pk@R_4%Pj
z)eLGWOxKQMc9LWIMfZZ7pHdav3;tZoJjrlRr$p;tafJ9Tj_kH5{7&^sZw$nB^l*l~
z<7f-wucf?=3cNbZ^to|)<V$OzJj=_%A5R<SPcdkQck^aHOFrRt`B-%J=L}nq7zORM
zO1Dw`YF2mjX;JIMX%6X=vO~fia)H^t=GHHiZa0@lV=LT`vH4tnmGDy-#efPEDvf}d
zN*#D_1kr{<)4`lSkdp7gvTpi1(cq}X-dM}&tOD7&L;${MA9@d;{)bIj*NhCS^sq3L
zcpW<?sg%>vJg;&+7rLfVk=+_#Hf@1Ct{Txj+>1B(a<G(kis%aE*8HR`LJ_UC;`F=o
zc)Uy*mGyeW$Z8vV+eJbMR+Df=?CQux+h<GYUFBHZgYTo~Z@|s^cte*G>sB9MoeX6l
z6^7xTkJSn3U4y+$gOEUbpQIu|u(%#&en6$plY7I^-%tBPx{4da!jFhWTT<6Gs;D)0
z8FAwy=)4-@=?3%mT^fp~Q^uu<<p!*{#VNKf?XXmjA<3@OKLi+|7UUeX3GrnWRmTz3
zRshiO=y6|5@fu11VWOb@oI_<!5JE2I?rQ<eiwC`??Ko@LGz1)%K+_ZXdb;XCWTT0}
zeV_T9OMAJ60Ed89j4xrYPo`wzzkX|j`$!nE$ca5wy@z;!eo7iap!(PkNmWR)&On&B
zHj)z(Kw{tZtjb}j8*QtrN#8fO--H~AMYr@zLpxv4ivCfcBK}wU2aKvXQ-BA7ZZ0oe
zDfwYE?f2He+U!fu=9i+_cj@M!b6@bWTIsG@qlwoDi{y%T<O-rLrD&klA(cN9IvNs~
z%MHL*b6`u>g36ZpWiqFzc+0CYAdXQnyMD(Vp+JLa!P-B#VP7A$2^n6E2i%O>Q~~Vi
z?7Cw5$bXIF$9;Xcd8ny{34YyVtfc^<_YxkHpg6TICM!R6h11&53riLXQIFr|n<IIl
zINi<%f|y&cTFl5Eo3)t#Dtb%wIbN3><&Tx=!<4i^xX1neev=McE7kPx_`}LC1nipG
zTuB0R^OO0VR=<H4S{bNKEdQlY{1xZ~Ok(ejYp5J=2B;TK6{bLc!%axq9F*`-5DB8m
z4^!se6hBR*;I6A$yci$TK$Zn1oO+<RsXmbnL15W9^S=Pk$UDIE^cVfX*C$04oW5>a
z`MS+p9c!|44gutbjpih?hwZOxJ!pMD5Z(|4F5BY>2RO{PqTjM+GyCg=IryyBJ^voR
zjd44);W2va`uYk4%J_;;7o;y-kK%{SL#r_<Lo9Bdb_gZr*w}Ez&+UzPvyRM)YE#M)
zKpgKQde1jTi^9mi(mfWkI^uyL!qczCtpw}>h0^Nb>Ll0&h#xnrg$l;gvndU~P=x3b
zW@~8{ZjG`HjUUl&ht&?hMZcLL*}j%IebZ^M;foV6#e2gclRkIjz3FTsQ<P%=V1g9t
zrhesoUgUfkhKCmfMkb@1qyA~X)VPJoCsof?_VwWHOad<UWDpEm*MLofhNfC_g_snF
znIw;L2gUYkL-rNFeE81bUbr~*ZX*G6D^GJa55RGY6COX`_PcTyri<5pXMFXOOU{Ok
zMs*YesU4cUD>gF$(hcHw9eebc1Wf&6Q;G_@B@YfjvIu{wY7?SfknHX@;qkkV5xr^4
zD)q_B`*wTYIXL$>L-x>J=Nf^p$a=cd&_TcWCzFH4FNO^qxP&n|e&r}l>>Z@OCIMTa
zG<o;8J1u^qTj>x%z|oY34Wu;%iKQZ9knyZ^g44ijY)uu>2pg1``ozHxM0CQzxZ|&O
zOO2Di4CxpCmJ(|EC`OGE@lYRR!R<IEe`cH~sq-bWVqQ{&Qw)G%yemn8GK3H$`(<r}
zVQ-<!R{@qHnb#$8cYzrxQ1au$Nd^qP5}khqW}>^myqTde?GdQK8^-T~dN70-K^B?(
z#6>zQSmJB$23J=0l+)_wBvZwHY)z6c;Yvto?#j);^)T#)<;|u*t({7SfMM@g=9t_2
zL#nmQ#AvU=1dX+$vTycpvO#^%!5jBp6nou`i^5?UPo5K&Ma<dX$b|dsiT!x)J#DVr
zf~LKelwTeb^%2FW$*zI!j^*kyPU_!B2G`F&`TY`+OIm@*1P+$;4OYAvzgo<Pj17AG
zG**0-(~{xMTU+}&+L{yD+m3cKBtUuHyTewhkm^HZlK*p5(|Y3;t1vJ|pb_rjW8Axu
znashMUvt~>c+RE8Ys|KJP@tCLyGP?8)h=Um9LS~ob{m|ycP6X5!sE<b0Vjn$ps`I#
zvoZbpM%a;N<YW*ud>%bUs;b2qSgRt($h}t{vnYixtMKvVL?K`ms`bQBXr)(>YqyEb
z<Kg{9?dyd?<Wm?oZwjx?>+PsLmmyyI3}06N6wkD?RV|5qolRO*c?Xkyl0SK-$LM&9
z@X+3j09v{-5XZ9I|3lXN@&2ioi=P$l??V8KFMSCXo;1Orb1nbyasKp1a1WBykWd2K
z2bC7kEI-Rtk<nW~C3!vg?up^E$+2b+C(j9Y)W{<sc#Q_?W6B}YC**a;zQBCq8#eDu
zwyUf-^|o<HMrcIM03jjw{-x9a5U`0=c~Wf|n)M(uS;MlDLCN;U-nUC#69sYmtKTd_
zwzJ8uji2XU)1FA<>bsYnO2r&Lc_wZ&EtljpJy9eC)yQHE9IgoNn<Yif>$*(>2zyU$
zKYbZ!GVVYEsb>-~j#rh_C@}OPoD9wd>BX-r$}K}5BfEasFX{(;(gfS$2cU<6T0!9<
z1+oiMkX;ngjL=7J6JP8x<jSbj+_5MB|AopF|KhCJZJJ#U=&a7nw{7Pm`7k2lhc8Z6
zyKS}oT>s^@n^!;QGAHKnjQN}$)+~G3c{?h^xQj!%)_1h!U$9hLwnUL)7JU0W%6E8c
zzaN2ALEvBxpYukA@5H?^c}^<%p@x9z4e}EXs;jmPl%ozAs=_D@%e}Ut>-aoa+!)%k
z!KaVRSWH6;w_aQ@^|M``%T;q_S&el^moIRszj7!?edCHe|0#ZC^y*9KP|hs;ks-Wb
zF*|rzCgod&vU&v>$s?ud30&{!I9FsQPsF5%!22CW7gVPxtg!dOJ<(Cg$>>X!4+*es
zDZXX(>lCjK`u>8dUT68PAAD~+nSSYf3%qUOscm`wY~i4@q{Pg~9R`zgLJTCScyI-#
z8#|~={&o6c5>QzkQ!Tm96c8E{K9@1M6p+>B%IYUJF?tw4@BzANdi&I_+Vlf8$O9Sf
zEG{EvY$aLjR{E2^;~>MbULvBm-ElqSZ~8Q$eEaRlL}HZYu8&}AZo4!5;^+t)Rr}xq
zU2bPXgUw6uF9J!xgcji@D7RZN7(W<%_=fh2`Tz)cp8#+wQ1S*C{tbV~jzt0rZxNTB
zX{{Ds938(o3Dr0bHxWN|L_$6r?Oo7?YiC15Jz#4%lV|}9L(c}i3t$<FE_tRah@_bM
zSn8PGz#V8TxTESjLTda6nLgiH&4A0Co~g}>M8KH|8#s=mBD1kxc6GKroNaQ`aewQu
z&--dRk<x3Mvr`$Z5VY73Snc0~)af`vYynwN3hM!fANI{Yn{-IFok=LtA*pi>6PgvB
z<2NN8DV^r}2!;}M{V-RN)caYsFNYPoREGW)T5E6tsrUJu@S_fv%^k;0?w|c818=dH
zN2Gb4J6uxW6b&hIUPs6*wUMfSFX0=?H{fcExm(^VvO~{tm-%d|Gs}znUKJtIpLU~D
zF85-EaH;M|U9@#eokDj!nNWn7iBKOk@!h;5{}yCR#^-08--B0?n1sX>+xI2nWz@M1
zW6ST&t)q5{zP0f$(qcx<4FT{NYI{mw!iM6fI+X7`5-abOn$eT#E5*E{9huv@EzIe%
zEz-JL=ui0;r~WQJ8@mU;Eox}DCx}=meQ|g-S$;4fy_}_X`6%z|u|N|Q?M|956F0>7
zn)|pZZVfY+{yO%+zFy%F7O{W!w&3#YrEVKFHmNs@)>KCQ#Ht70ZH5;XF>#0lqEhMw
zpYY-KhcIZLG~gsjkkrNY@cicKI60V<Hq4(y^UFU1dxf|p6No6rP>7$Y=08r*`_GpB
zf;OKTVOgehce6hklM+d!-^D5XIeqxJ&QYs0>e`c$N~V@pmhYow;-rlXg1Pl}SOJRm
z*RAoAyub+eSG1Oa<gM-wSffQcflD461t~P;6?ulxN|RR<f_6Iqe0K&H``Dd8t*X;j
zp8Mq{*x2poUsJj6e90^`-_hg%yD<g4ZQrv6)Q^Fa;0F{evtGNxsE|g&R`5t9@IZTC
z;3M!RA>4vgM<TjDWh(ue&d&uulQ|zu<7Giv14?*5?TTk|XA}?=-0gJZ6U=244+`zu
zSNPgcI3-h-6(V&jkOczLEnu{`W}>mA1aX=5)Pc-j6_{Nk%c;7z+tmQG+n^tnMk<r9
zvL!-sAYTZ$*j+=qaIhNOKs|Z|&R4O`RE_^Kv5~jhj(C;pj=syI9&#rv&^~&kTF<5m
zdSbuDauA!Of=~qMSb;;<L)-Cky*Duen@mrXJI|eU2_fA0{OgOUv6gL0`(JxL7|}IB
zW$Qm$y5_Lx(Ed071jSXH6k2*2pUW)yyD%?<?C$mlfoA%TRayiebOP=)miUKNZ&xe~
z2iY?noW1U8-T*7~+lYf+nwzAf+ikL2Y*Oz0j(Y5T=A<${+IN>_T#UQR{NL87M0s48
z2NfO{v2myAT9StoLL4#t^Um`S>csk&g3<03e{e(4evB2^^g{3U)RUCY9qPM&rY1!y
zcTZcnTzSkkn}*iy+t#Xf%!Zp|@Ye6H=a23lSJdv&n&c84HF+eOU7+~F=4sc@uByUK
zIR=N9F2Bx*wE+rX;auO!#5{Pcr|>_IjixIH@<SFRLe*B7VJd=?4QK~P>tEjrV!Dp@
z3d)RJUL8E8!^G<i#<d`B5!bZEy&Uwq!g}t0Wdd?j8fsNS6hx>hJlO*@=4e=j@0Y0c
zvHtZE<GdmLdvfEi{99%p`Qj@@*V@_)vsUwm!yDfVd$F42;fZyYyBRT13|x%fdWsz^
zGayyXw0fWaHfTHp(4IPQOju&W#J>zc!9@G9AIS{<tr1ZVa}1@Q0yLevC4mUXos0D9
z$OSMFpfFI%QS4hN%54?v)4C|Q;v5m83FX9SgToRHanDCMA`%7)51_D2{+5aa;`3M2
zwK<?!&#s>gzXEeVRF=A<^RB#gKSdZ-{e%NMk3<@=a`n4z%AG?=bq!>+iVPtUdNO56
zMMM=FfZ*e_=tH>T6HvPkB%GswbGrMH9RrDy@Y7n~Ty$|y)5%sh7uB#&{I)o3_DAMa
zaK*Fe{{mB>c?VC1At=xKJ-s;i-af<jz~$XdEM|~sEcR|i(Nk1^4b&w>hPCb^+P0L<
zbM9M7dQVhF?IP<5)*XxJ{f19TW)~Sz3yokA;|a>fYFTzJW#3w%T-#Evy!GjzLOVkf
z5D~S>eV}-FcqYQ-_HA!f?QWx-Kj8z9f8hVm<Kdx`tsf-{^|)mH@<>S@JkH}t##R%{
zRC~)ADAF;FTqWs^?z@`%t#Y!^UQ@`4X_?qkcT7$M5wrBOf)7#D(+v)^vjXNE?+rf+
zBr-bkd2se~Hw+Cu@jw?CvNtOrnWAZlM`?LrJ*Tm&N{RbcmeA}MnS!e*bB#K?^gi)b
z%baAY=nWaS6}%poM4pJn^d%juHAlfhAm?Q|LI<<KoDc5^2sf*^cn21Dg;7rVR-`1+
zi^;u-Bc5jH*>}*YmJ-hfm!dO~ov>bsI(?=J^Qdjpj&y;1k~)MC`q<}uh4=!Lrs$G!
zK%Ltl>!R=+R84o(ac=Q!Q>9$C!yU%j@QH{q{hwoW+E4@}RJVwjuHTHgMfA)CJ?b~R
zTK$HXbPR;V+f_w!HbXG`qY)VbwACgWg=AK0-9Gk+_KWi|6;Y)g3gR}Rm%6K=S|@;f
zqL(IL7l5Fg3pfx*U%7@=0i&>djWFi(r!E1Z8yk6?g{wLaLW2FYuDO$`zVA(Lc8!3+
z`ATXrqN@VCy@4qL6KI%pFv!w)UPVlIS1_X?lmWMoIO(VdVr|#jwQk(tPZR)xY1y2O
zn#xz%_NMFswkAb`HoRBI0o%8%Py0J#?ggXoj(u9oQ~;b%YNl@Zq8bz~!h~*IAMM`#
z;|{lOSUp(kA#g<+0YXmFaB}AZ5Ci8J+*g;T*b@xU@_$ZqHGNAe`%jYp9~E#f5`<iN
zR<~5OH8)pV%@ENj>A5ujZg+IrdR>Qv)%Q&;ANBZ3gNY5|Bhw&X`YV?Ggv6?ztq1?J
z6E=KDZ!tD}vA1}$O(a59zsP?r`5B%`KuCys+OVfIKTaR&wR}7`Um^?kn=oYG#FWsN
zwS%WW#_#Oi@G!pNH{<r-x38Yoj~M?r(mPm5+1(Xhg*smMSD$@SKTcOA2xcR8cIlOw
z3sW;}$wCBf*6u~`WRJcst-2hRkW3vZ00?!HYu&)s2>p|Q_C=O-J6zJk;)*|PJHJo~
zC!6{zuR_tfE<ae-W&V@8l42NBHpmK-ZAtjzjD?lyA^|B8*CHd6etNcA{(-~V^Gu1_
zo1QSM@c*i${h81-E0o3^GKC%fgFDET!{TDlj?5R`3<+OO1K9QD18*onp9Lf$53o^h
zU}D5t{mG9MszQ7LQBV!c;zf^WFhNWujez7yAK3ctwNWYK1fl-kJG*20@IhO1f}nLT
zM6E~Zq#!bP17%8tthy2bT{+=yuu<;ay$f9H+fr?gvphFTf=(PLc>IaHnT(INCSpY{
zx4E7s(|qy2`UO1OE3p!>jdLh>2xazTNaP;?s9qxi38cM|#kP!goY%m?yx8kEXlTpi
zhth)WGuh};Cg7GX>RRJ>d<DRzRN%-$R>8K*;VRK^G^(I=JGy*jZ^A?guZBbrsh$9+
zNw5R7uazg?BV5xqKke|W;|cTOcHcKH5EAh^v3W7q+EOZFh=OR|{dRq_I&ZpqzLz#O
zich#UaCY;~ZYELsKD!|+1`;0AiWMaD-2LPy<gV100w97|wj%wH#!?AQYdfBD^$aw@
zFG*$nZ&-Vl?06YO64?fPuf7hfeUyE8W9!!@q+Q#A^X#mv9yw^Cb3V>-s?1<bLZSl3
zJn`_a3kr<eZty0#$taNg)NH8HreWa5$l`u~pZ8^hpgS+q6%bZjoje+vYimyXS?3#8
zXFK24mNEO8_rR{p*T=OLlST;$#~dL0txGto<4;VLGoFv}uhJc~5-zdF*4h4)4-CaA
zVDBh#GJ8E4^<ahQCOR-||91ZkXnJ>_eaq{iV0_`7>J!BiOd_3fw{>N^|JsuTg16ij
zw2ToNIw8KqUR(^7a?fr!3%}a6UNcw&1ZeXKY!L(LQ+Rv973ce%Da#U%?|O|0sAggN
zxZ%}qv%n+j(iRG%Rp6jjoi6V?4IK#S5-1|0iYFJ4vK9^V3s3oOC#_%Y%mv9<kA`R*
z&1zIX*T|eZY(VKUF$e_T7iIcGwn&7lZ<Ckyy~`((fEbNWA|YGf4z}OPX}6JtJ1$hk
z*e}s0ytjaEgr&}b?^2^9|AA-gEm`w1YRr|dk9F%@IQA(6x$@yjtR-eJ@5v|siAT-$
zzZsQIP1CUxGCg+rJ)<5qhod2p1j9r5CAGGuR6Ks5Ncz67DFO%PA5E|%O_hv9F6MKo
zZrat#y|Cx}lA>Ps!igZGSpEZZO9Y*`ype$Ii9#4I&x2Hz95V|k)hd$)jy;UJlDELY
z-U`49YHnDvC;Z)Gl&%k60s<oNtalVf+spE~u&xf<J*Z6!6KY7lb2cm0sjP5i@q6O^
zTBXj$ghu`zXiF;<0}w_N=;i<^>KbDrAY(3qWm0b!koM`fw-E|bfVI9q7`t%j=O|C<
zdk!HH8>rZ(MC^heZ+_QkqJhgH?|`XBG~Z$>i~wBz5Cv#zor9i9BJ^`U!J#SbYBJw;
zK~wwK9it9sGo_yRPr?L4vw~!^R_(Ine$0Yiw>K$Xcgt^?GKuHW+Qz`XW$?RLX1LsM
zqi|_}WrxkPXW`B_i|ZNB@C9mwP89dfhm+wsH*2nhGc7Tu{aGs0f0io$b@aa$p2DDI
zDP)U7eUPqfE-LY);5PJ90;Bu(podcC^;-?pv1L47$-y+i4RD7P$-HSViS+Aw@N|&~
z;&{&=`FyE+-=s^oD<TwfpXmX~^ioKXFlQhjCcU&Wp^g;{DyQuWGiOJu)n(vU)UWr{
z>@Zz+rpf(W(wwfxBI-r|is<df#zFZ@HYfuA>31SB?>av7(~tX_VYtuYpM1dUGyHTr
zdi%lQDr5M^exvh7ajnP5Qj0^de`gb#aYducH`v8hBzqh)4Vw8qq#a8>s#Y6I!FraF
zJr<jX_>BKe-FYCTywopv2a^}QHPe=!+SXYn=!(Ijp$%8p>x_r{Bk9kZt->-j)c?h#
zpoPw#rpfbvbFI1hsx$2K^xbE4q|`8k)6+W|;#cW$(poy^r~Cx$;zR!_)!<NBN~EU~
z1n)fq*UQ$zea-=I%1yiv*+^d=1qr9fmg%sdiXd^oH$WSq78$m!n8g|ReAq%cgFtiS
z!Z=dm-G1c*0lN!(wVD76$r_zA%XLRK=(;RrNU7I5v3&i@{D?JXh%ZWfY{JNQXL$s%
z4PPlz-?c;AVgmse=Y1(rsVt|Fd*gx_uA{F435H8P{L;2H0^x`R(qFBWdrKy2_$m}z
zNhV5KNcQ(x_@)q=>!{bjuRKBNSZ(VU`N$O4eg_PqX7a4&=2CR1+QPCHYAVi*MOj&|
z!$M^*ad6n^hBBN?4iKGw2gTs^1&AVBIy23K0+`dmGh)tYl5uI6L`m9h!&gJ}#gDI(
z>Q501W$4-n7T#~$t&fG)?Q5UX1i+<X_CvHWF04QMM1FR?9<fwN<ohfC`lBO)BhbC6
zdqb0}1!1)v+_Js19Dz~bg_XeUH3c%O)8G6z5D=mwdYllW3O`IkM@#$}SS<JSMliP6
zF+-8hj9=#O-w3;g>b8Ygec`$~WgSTt9oxHFj)M4`(tt$!tf_nCm)Fkw7wU>ok!|x`
zEUv9Hs*wbX0jtEtaR>OBPG=kbO{(A~hk$O7Z_3zJ(+%%wUS6y#mZAwS;`WOO5B0%<
z8L1yPHOUn!97W0_XX(F~WW->+cnv_oSF)pLSr)0YMQDj8U?LJQGhJ^a3KHWy)Hp8t
zvtkAZ$)BP4$>NjJeTq-64TCEf(^rFqW8aLR|5lq`3q|8VM_gWb2pl`!99YKSjP2nN
z45Jr$m<%tOy99VUuVrBnF^?LU$!H9cdX>z18V0LY%?tIOe03TgHPa~alv~7ujFS3u
zQNc=IF}1v9yj=V!`SKU1*=P3uo`h;>u@|}8me)4F&ok*GadxkFpD~a&!u;YOV+R$z
z{QvOa!zJLE*Wq2-FO(l4C7sX+Y^(p`^lOVwW=orPivRHkZ@9~xZN!WP$p$Vr`GMG*
z{=)KA-Ls@;p)fcg&hu1~Vc(!8BDaqrxk1t!MR;c=K@N+N9n3AY=4jSZwdR1I4x=)-
zLEpqq9zPGdrN3fd@J-F@I%?|r>eOj%IC;445IoWy|M+Icnh_EgY2|R%M#S>C$QH|K
z^TqEy`#jHe^kt_9XTx5$l)^mp4(S4UdX=$ZGIv}4<o(l>T~gi{`4z#Ru$^U;=)Alo
zc$HZXPNT&<Tp!H)@ej9{ta>h%S%$bqN}+SErOe)RQdQP0^m$4u?M1R$%oeXDB+sI!
z?C=5lALj)XY&79?C>qv7J4J_b`M3j}gu~&R%N?0K>8Sjx(ev6j_3-|#^v?rtk=h4&
zEo8(HXTR06zt`)!PbdlTcb1v|OAs~u`}>XLTq`gR`j?$fm}K_ChGfw!9!c!m?{?5f
z4Sk@H34jaG!00-hn^_!{|5gvd7ol;MzKc~!?=%A2ZHdBgTHdb5RpdU1dnX8ckMWmK
zcNqyumGI1*bmX=WdQC>BxYoHc@Z>o$l1oPknP9Sf3Nz8Xub(qA%Or{*=6W(BGGS2#
z-8o}to0BF>(Cz1?KK}qOa<A`*W{O^uT80b0eP;@GH?M%1fz^0#JdUWI%M0gcS42$2
zSIZ*2zhvha!!(LQ(Cg`F(3a_tx<auF0&#?TJ+d1}V}6YVqb`<Dsf-YCWgS&7;4M8=
zGReL`6ji~>;-V%aiNdpTkH(_5%kG225=ST#PY&Q5Rh)3%X6cYPUIY&qwBHIB<0inD
z=d^8zLU0siEo*P6sDl$hX%#p}1lUq=GrfgT--QP&&Y<^}85W`+cxCWkb57j7xqs|K
zpyBW_S4l>kuqYWU))$Y+U;HPeKog7wei89A{u3!2(0V_JI!wwB_Q@bLp>Q|7D;7Uq
zu7C9CKkpO`=~GyH;aqs~>|QhS;FqnGmqQr9-IM~*%8@B3%N(TUeV4f`cl(N-j}fR0
zIjduhCM`UX4{k>6*ZS)}aw~tPGxfEt=G0SE`@(ni_!cnp4)fJ<5n_{e-uc#DaW%T^
z8{aadWV2WBe}epcQws5~6`Oziyyx4a>q-3_!EetB$cefoFLpNfisuG-$4)4I(hlpr
zS4!WBq`~Jbg)!jGhNDwNFIc+A_kq(SFaBi)tExX*N+$?JZo2zc0lFmFWQNOlS0k?r
zZly>}N&|b(A|;D7OGdd}kC;l0+FAu~O`wf|n5E6-s!v`VJ<xBoAqhzhgT=L?aku%u
z2fewhIWJu3>!UCr<@@{SBZ|TB{kxh7C}vq6Q*vf0GU`A6_5a<uZ$japm?ar;8A{#M
zIj5K1uE~69^>2|!MBAYL`wizRx$k`mUS0N7$6#Uh2~L{+K6&-kalVb>wq{^X=b$Iq
zqQn@rewa;HNN}ohN@)7y>pj+P5(TyYRPoQm5$5vr;J?+_K5YNq59Me&FGz0_YA6Vo
z-|4m``K1(kE1yK)i6eN%(j=&s+Aq0sYEQVyyrZP~2}kV<>9g3U^)Ir+Lcd3b-N&V!
z?i^Oa6N3dp`BopKXoNc5JC>l}xL=hykf$~GJ~z#_HQ4xknuL~Fqdihx8GV6D>amR<
z4kj|)RUXrN_91+xVg&*`ER1OS_AkIqtp-iLN%?%BQ|fF@`g$!xv&~TWd$@y2m3ALX
zZywU>SkBfj{^wYa^cpn_QkJQ?S><b9lqf)DjUd0>fs@}2Td77-g;Rpi>bN#Ha}cTo
z;_;TKUn?-~+M_WQV!IL&$~|f-pPwJ4c1<D<OpmN4csPTP>unILoN|&UQo${`$(z~z
z2(1#Qx~w^%?MQT&=kZN*^^#Wv?=AO>TG83QpEN6n<O7u#TD)C&{fcL5A69#wIL~79
z?q5AenG?kZE&wHx^HmBIFZJu3bcgOkf04XYi2v8!T}5qA&ay(~o%85Rb@uT=9zYI3
zT8@WEjb6B95buQlxA6ylsGra!LFoJObaKcGUfLJ@iE=S8p8HZ~*VpV>(Dxo`v$5en
zUiU>HkVUHdFBAR!V>PJ$kBP!dDd3u0_PGK81b>ZYL?+896{l2gb=qy5OZ6qr5Kr|@
z`9sO}<?)SKB%~``ZG@JBTlmE$+3L9eWsbXU9RVBVt>l<|!1pa^Q$NcN=FQA$1DRyR
zk?2BJA@Qy2ZFjWQQJ7k38FS`i700)BAIr}dLWAr#kIGAa;^{zp(ILv#BMF|Tt<y$z
zX|(>Rb1aoD(zrrLah;utkFTr!NBJdceJFN_TxkPhQZQomJ|CezmV!7dTZ?9EXPX@C
zJN{qKJB;>whUs;j>%W=IHqD%cnGacZr}|W4l*z0qD&00+^p~vzM}IZT%HhxaO0bT8
zz0+A4JJx6Gl6`7WSs4h9)qwWLdsD+U$7GV2Z`YXs72t29h68H*?gzOSk9ItWagrX@
z?exmpH2%3tTDTky1K^{0#Rg40*PzK+>yQ^m{|2X(n+c7M03=k5fCgEXkH?o!rgr%9
zlXi1KtVWe!JF!!EYv$fIA=HwykKymXvP_7S^a5{Z?p?99AkFq;X#IZuZ+f@Giqf$~
zHfj<Se@$w8`GI6-rOL9|RS)6RwPI9b3-lj7WtA=Rr=T|RW3+wo?Ld8DWToo{UKr-b
zmPC3J>HA}}b-&Fq_Lly-ZFw!r@N0?SQmw|+I-4st{5oVm-E;Vwez;^z$E23dPz=_D
z=eIhDeELfUD3DV=oV5php+NsOG}UZPV9?i(d|N&TeKz1;6%-nc$J4Cr%1c8d61Vx?
z#xs|2fpvk9*2W$Ak>qDo^}*-DzKJyHl**i)A-vzjCB0$aUydC~KJ3)%d?-D+e@{Ow
z3^)JXpyv&f6rXUqX%Mzuc5v7kymX>)BV0Y`Ly$&P)dF@c6pu>wN!sz1T7<nP-xoBp
zUvNyk1Aom*r7?m6G)wfN2PR!)R@C4Vyx`e7&m&wovX20t%KtviB65jgS|Ou6-Vpx3
zK7q}TLz#!28{lJZd%9IN3kvtjBM`A@E>Z<G=l8AAM;~<Urx^WEuYBn_z=lLQy(lLL
zG1lAqin?8d@>0*Kaq_Np`A-ju2G2@#wz`TaRUD*x)$c62q!ves@6P33OFS0=Bq0b6
zpm%E^4$&;reslpqC;$-|=CT<yeha}SOK`q78U@mBI-<MnL1b%Po)4WTKJNJiKgt?9
z&^a4JmCO>BtM$zBETZxi>-$d+1%|fFf#h)x1mOF-B@`ZR{<Mpzi(@2`dogQQ-Ka-r
z3JS_%umv<GWr{=L6QrV{ox*IXHM~=gRbQK#npD8&Py*T~-1Nu}a`ex%P~k^+@4ZqZ
zhYSzAW*4L!Gq7MR|6Pk0#Vehook=9%9T(Gscg(!=6dHy^vWE5NwZr-DUc0q$4Uwou
z!{dL}{kjy}2>cq_|NffNk<z_r^&qZ)E$K1Y6K}AjG5=aE>9SVdXXe05yC_r_JsXM7
zMHh2k)Y35##dVbZWva$RlR3Z3g46!6vNvey)7Z~c_lzpNzWjaW$n8*-sTN5ZbuW2C
zMF?<*zb<t2;7#@;$Hx$h7{#>*8}!7wy<Tu<B#$|cJbzg6Bd3=APYn?IFn|$t7emZ}
z5zF^i6@yp2a;w#+L636t)hx{EdH_ra@SnyGBRSb1hE}~MGXL7(a5U7k$J|B^G6;!m
zM~d8-;YpR3m-hjeU{MU61)b(!*%85=@Y|VFZNu@zhHIE!|2FXMa`$(Zgv$nOxa%yZ
zL;Ulpt9mHCz5%{!37}m1^DqK+5g_1TeU`gxo7aHM*84=~B{{{6i;<Mdo5UG7%$GMe
zSt9&IkhXOzn3E7n=<6xkdX?}mcJ(&ewo_tWw$2WMmnWa=%77#$>LKdU&O+i5#=taJ
z%dSkqkhIfgd+1mYE{9UOFeL0b@Nwheb73A?@C{NKv@A@-y8T=^Y~6<*YIgl$UnBpu
z#j(*@Xs0OQO%7Ma!EzkVRTEJ2O$771c=KF^k_@cueCkM89Y1%<eIbt%06(GScz%>o
z>)REg*}lo^ey=o;^jgs4<@&Mn%J&v4a2Vp@_NziemPWsVYSL&UujfCPI)+&ubTYTc
zTg|cNIR@kWRS$F<S?O2Z8~hY>{mVPh{XTm+5_ziMdC{}fn(~7)1g#4F)FPgGlNn(*
z%F7okyyJVxPsh)D#h|i+5cu=$oBL`dk54qxTtr5U%66>rS^f^JXh<SRT3ZAzZ+AkK
z=3-t+#-_Kw9AQ;$e1mJTnlWg5yRvt1yfMd{qwxC7Dn`}LM;slW@J|so5BPNjI!3Rb
zEII-|Q7P<mgAZZg<eHNq{b^qObUENM625$8jUJ;QQxFU>s?nrkN1Cxzp}@;>Y}c+k
zX<WVyYxNWn^m+3IGk%%i@0%At9HEt+M<lZN{8-W=Fk<D$^BlVUi}&e?-Ft7$cdT1n
zrYQMc$1`e;@~)c)ByqLuB&QD7F4~GGzPDHp+~+A>Yh_Tncl())VWw~*v$k?KJ``0P
zCdS}*OMAvrtd(`TpM1l3jVlqeCbjp_Fz7(1LXpHXj^7245gbBTnLG101&k`6>E!|w
z1D!pg?-_3bVR2v3xL=J1Nn^ETgc3H2+`I2~>wRNpfoR36occI;Rrs_#7QwPl!+MCr
z;a*gPrbT)ce^B}ZE#chRY@?hdr(6k-SE>AwwUtLCMD~=nFG5^)O;s&ugG$H<uL9BB
z`AjcPIerWIJda5LA50~dvvTe;|ILu~&t$ew>ok?3{9FYeoG=9pZ=a%$Mi8+yJ0GAS
zSqH<LfLUsu?z#J&Z;jK~rt^yV-+d(iTFGo{!CV5=NtGQpL)4+27b*j_y639N)^0Wv
z!}V*AZyt70dA)jEP145Lj}%P&^P}9eZd!XV)BKB+G0>R^`fta~Z2~HkQ@4?uoKZVq
z3#r((eCc1N2AT~h46J?)3Zl^A!FE0RU0b|J95^&#jC17;nq5M#i<}nLE<ljBn~|<P
znCLu|$N9VDfkT-DkceL5XFN_X{(%>qa_xL?Z`Da>;aojQ;zwqQxQNA$KzidsgfxVb
zqHRAv<7)}B9a?;>QmXu6<`X`b^sc3Nv|HK0g&h2bct6#%&~-J8``}gei8>1EBd9}~
zJ@ap^IfOKkjOD=mt%HdZo7HiH&UArH;%~8-QPutpH}y&6MMETv{*=H&2Ihs=P;vTX
z9N5>b<vlv}DWGa66>`6_nK{EfZvHJljin^><m{|EUpkCa5qO*)M>*z<j$HUJ2es%j
zrs#A><MK%(d@6VyqQ0MUvlfLy3H|jz-O!E|CC}mtk-cTrX^jw-hG^rfUiUMCp-4Je
z^g+>;fuy`}0?J&lgqv>``#<4%!_C&BgCo~7f%-=Y7&?Mus=Th^Z0vPu{d(X&tx?OD
z|EVnJ!3r_uh7_HfjGdm7=hp9Xm_43gr|QRd+q)4I@tkF!1^1TP!(3P#lX<}}2rb8d
z>^>vL`!9~SHo)A6JOJR_O$CV5<ax@KVlto%hV~+VFIJ<q4+od9=rI_X3uXMn=|qA6
z;yI?RDIPmDz>cN(uO%9*KQ3-zJnW!?{K|qKjnEDWUWhv?)cmVIhKyKFkGrSrJWlj1
z{05DxKxBLKu&<UW{9#P;Y39RL#6G*5>6aBb8p<N!){H)F!MG-0N}3iiLN$8g`&P5&
zS%u<(2f!|lvgmP`O6kPC#cw9^VYhU5Omw;7yO&oKnyOLn6^!vKed{fU>*^fuea@oY
zUtTTGBbOXj4;IH^7AXE$VhD8K`*1}SEe4X0haQX(yPo+Kw$Bu0<yend?q_2H%COZ3
zok|vQ{p|F#Y*$==BT@T#+*koxay1nuY+UlCZPm1dFh_yRfwHU7R^L!8MT7lkarQ68
zPe*zqxdA~eD@xSOkuyEaVX4sMW^1|t^P^yW2B{v3(C}&r`b1xq%TJ!ha!ZvP>Q5uZ
zO3l`>a0Xu{7K{dLWom)fwJo1d1|?(bX)fH}2pjyXr1i6^$B>+LPum_{*|BOKjkWr|
zbyBMsnS^T*-ed+8v9Sfsm=+H&kCg>Nl$!5gU;yZuLR^6yRcPh>a7_b{BCWeZu%A0C
zU)&i{28g$F_K-^eVN(I1MKi4+bTBYB1wM1GqxHKZsq%CHTELCwAI7u4Snl*!O`Dg8
zw)ZW64HwUoX%jx2RIas08|hEUHIK9U@6nY-LyEpp7&k?0Q(>ma^L!3#wtq4btL9o6
zq4sz|F11lao{wW{%CD;=O?IU}u^^xs$<QD`G+2v)D%FX{?bq1R+0jP-9%mVra+3HL
zG?{%_aqGK=XVQKp%~}F@ozL-Z9zMrPW?gD?-I)oV&T`A_q3^z)IJA><+>F}btr?Sx
zg2gGJrB4-NAFmJTH?qFK%flIZz``^smty2Xf8A=E;ncU=s~Y1wZ?c|jf$K{3EqBS~
z-V@F;XLc#eX#4kXnw~21--zi{CX(jArjp0Ch$d^o5iyL&;$J4XYAI-njp~j!JDV!S
z9lb?BxjkP>Lq<Ey;-9$&KfdyTqNe3mtE`Su$$MR_JL)}MLoSCV>aTFG-{$Xb7b6Aa
zW)$Ht9YaXcRL1(^guCD6YhHV=eX*UgwLe0XzvD6@vVXTFKUG^4({Ln3L1DUl?IVS-
z!cR3Kzio~2h3^)cenfO#Ss<ARa-&4UH<}28iXs7XKDff2$r&K20wbneCTt-vK9YC3
z+6<M5aGwi+e;_F$@V}Y`UgQRs9~S_h$d{Q8@Hb1^VQR0Q)6}h$LNi3)6pP!tr6EXi
zIM&Q}AJI|VQ`Y>ucDu_O5iDA{x&`iO<r*v&&V@a9qnrNiR!d$IKK%Ju?L}OFg!Zt_
zYzSLZdDCJW6B3rlo5Pp5-fVM-%Ckv2BaJR?J<UIC*4i4rW*0x`J;uCc72lCKb9*s)
zHZFn6@)X$UBES*kB*iqLgN29ZE<;tL2k*kxio71`J-K<IS*eUGyboJhW6>%PJ9k^8
zrSB?XA#W+Jk`I8z>7mUW`rgWbIj0Vj%{YTPO5%bZb9sQ>6(&8yIcj!M>W^pUd`1w5
z9cU~T1j8AbK9CV^lu9BfXHb=#wiF_w6-0<6L%VYt+ebyih*(D@7FdCpU?s+bd$y8k
zY467k@_JZajH)Q*^>-{cj$L5m|4d@S&{rbcR}MCa*IPg=_G*{5bZo{)!pA`Q=qvr7
z9>YJ&nd?MFTk26o5l;2iT)3@X_z{7bllAMw?aMc|lZGcaAl-5$WT5IX>aUx+tB+(k
zkxWqh!wDEiSt^_H`N{xNs17(kcUkV;HMD|jKHm-uePmlak1=EvbpLQfwk7|c^zzj3
z$xP0e;+wt^ypAD6c0-na#1Xs%!oQCzj2{gymwhZef~_*os?Ba%(AwI^pXL6v4=iW%
z1j9KEw88Sqz4U2)1s-ldd>;LNM<~_EempHjBWe%EEe7m?>Qpn^nmm8S+`H(ph|V@K
ziU3h~RgU#%^cxJ?4d3b>1%Z7kqM}{2)p3;1slF5C=b05PIcMpO4>VM>wSZ`ZBpMC^
zlJ(5`NI62)n_EGB-c^)Yhqa6&piOqr%;!nt(rc8E5;hL#Vlb@<3**pV%=O0Dbz;_x
z?Pu5t{kTCW-%ePJep4E|Ng2dOFfvO+6E{G#;rsaJh(ld;cHB=r1DTdp?Qo8B@ZAVd
z{g?C&^W2`aY%pB7erI;M=h%w?aomtn&C`}FiVE}5$o%wnePTGbNZ}u^=eRPoaiK6G
zNTFog#RMw;dvAVE&%%P`(F>-jQwE^82D$QvqbngLf4tS@^FLt4!#|Sil_Z@}U@(-M
zsRej~Ni^+Xs&_FE*c!Hi$v&!J-PVDd#PQ<iKlDBL4;s}*A+=N9os|05m?N}|lm9l>
z8|Jnl%NlH|!;bNXw%z((^W5wlt49eX(ET?nmy^gI@jY)^+L=i|wD>i>+nLiGc<%Hg
zKT-1e^aT?=%t9t_emvSj*+7I~+Lif)--wu<(cl`P(F~LtiVmAmTPqc;SG<G2Iveg8
z$MEGnJM-<lJ-mc?hHTPz`kK={AVyKLnz84|W_?3@C1by7`&`+e_PpM%ZP95nKvwWD
z5eRR}C9+?x4UGf)c*~M#j}R}By`db2^toHzqB?O58}w*yophAwH+oWsVk)xA<tcB3
zcb6*&63dWx%`3ND@@DZH@q47+v5bnSy0oiT?ThWZ34{hlOlV93fmu$Dzgh)O@7ZCz
zG3yNc@oJWO!EbA%rzGWdYni2y?+)30pkGrjxs}p5CEUy%#wh%sdE#j{va0Fa_-Q|P
zyEM={&pnJE%w+zj_YU8svG2F1@!qfLtHYKY<9l_f{q)82?v=YmbptZ`7H5Z}op~z3
zRI8ue=CpJ>=tvIKs(^BDU=2nhpfzE?9c}aVB?Z35B=kOi4ibo}paaOJXWmxl%Zn%M
znecCuwHw7wS8!_+57C#Z3JZwuiOL==EU#}Q{HrO}Co>;LDJf$>(lV>mMj$*DgSIv3
z9H*#HRe)?r;*B#u{?Sr7BbVG?A(u}drQhImjmr6)NJqUx8HZ&BG1MY~`6hf-O74<0
zeEGMpb6G%}wD`UX!>wXorV=m$^$`)i`W9HbM)^IxWa6#wLI#(mZ8g6OgCWUPae@Xh
z#FUo9XGXWwsIoN;J(8$sOXARBXojErI-D_`<fla@Es@7~O}CDRW_3s4YqXV;nxS0T
zm3m7|29kFwqaxXOR|U5@WE>dXs_KE=f|IDpQEhPyH53fNxhaYB=S7FX6dM7D!)rVa
zTp8M&x8iCq!PqwEu4rCQH!UIS7<fesnn0#o8((qYb>TdHZ~Bm=Z+UHz!auZ%xJb0^
zsjy;U7T;BqEt5!Zp2+oAzh>Ff=q7&1^fkL}gzS8&BuIL1BV*kEQ1a=-A7UJco=rex
zJ?3*~ZUV-@o>mwG-?j1gPkGs&y|uh+kKA6Zi8}XEhZeUnYn6Tj1ot~1z1(0-;!&Wu
zP}3y}E;LICEL2Jx6+@-Szo{DN$nw3BoAbRDehqxXj;;=e30y^)BFJAKjrh+Jm?nL3
z*&hF)XllVd`uj}y>m9@Wj?!0Mc0Sw7?B@0bc$lgKXt5ZOG&F%hRR3D6Byg0W1tZx5
zGGCwQ0T1#tHI0!Lz$wnZb7^a01WXW((>uA+Q8$_0H{i9ceIzeI|6-y-36PeHfN(b!
zY(7a~c2x-YhVMkqmkEyR9arSRMK4pF2to1ZmiT{P1#**=E<_`EDN-_vMYv&x^Fr>v
z5C63bHv$oUrDh&O)!x(QW)ToxWv{2;B*<;$Ik&R}4lh(xFBOYjUwEVFiUtfXggrjv
z^I}uV>Pr`RBpcJYaFBriRW$zOco3~IpzY_Af?H1L5X?pT!#+C^sr6UU?M`|9wR*u=
z)LN$=jvCI})k9#Isza2fu{Ma^@lZk>G(KQ{xGJ&AncLXZOx&hlCFYMdex_*qo@U6)
z@Ch-cS9wUv$wIKFqFt!?n9ccvNbekm6y81B5>kFwOO-3eq8sqQnh={eQ4F=Kf^Fe*
znvAH_G_D;Q5x)<&erB1!E3GpyD!O9$EfD*L;ac>~1(nFq?OcO{e9la|uY(G8$}?Az
zYcMB886L9{eYY!1^|OGfg|~{c9rkP{Rmwq6onekF`ah5xe^h%aF7&(YeH4V7FTz_$
zxoz4qa>L*L4_R*=7v<Wuff54@B}0qU&?z7x-2+IcgtW9EjfAuyAR<Ugr?h~8G*Z&t
zB3+WwNW-~jzj40zod5hi%IMy+p8H<wT32iu8PYNxKL)ame=(0m(V{b@tLx7ZHm}zK
zj6y;Xw89E~PbC~%VIP<X(x6zfc(PVIxSnhxe+`&8k)DsL?=spQ%aEdr)7=^Ek6j%e
zBxC^a<e%@s!~$G18Tce{aK;q_hHCt_Bk*5!Uk}nHvS6s4;a(Uo$rGv<n_qjT4D6SD
z!=rrs9+QAZ#qaVYI|v<XEaPYBD82o3*=J;E4KQWi1U3D2Eu7|QnUU}Vpu{*`_gsGV
zh%Sxt-Mzc3#k)<AUY-Y7&2Qsy@ny5kTLV$uDOxswid|*m8mWXf1|E2TO4nqc3HESh
z(D)6PpD%-Mc5AK``A^;d#{(oe_~qy3bH8;X`d$AQ#H<wqdrztl^$d#vUqv_+C7$dK
zWFyy21#aP~gyN7wmD25}N)<Q8<J|zt6XZo$s2{g^oje+)VYV@8@F)D%fYK={qCbgk
zC-(<k46(o)?X#bG3g<VfsBW8So<e;vw3gxCjZZt5VEy~3V2HhE?`S`!?2Wlzk_XbG
zDpv8D+;3_DqmgiVXUaT%vFXSx<sftI7WZ!GlhQ;C_6I`ocX<D|h?pjC%Ke)g`2FXM
zw?hUlY6Dp)(p+4GJsuGIbplqKL0IcoFhJcnZg)?w+lmyc?Y}@Py}BXTc9~oc!cay*
z5tpfsM2zoSe|Udn0bFoO4VTaLA1F0^diB|_ZE+WO?2ZKOW6U|yJ16mQ1KwK4=yIFf
zLt9~gfUEMku-Fls_Q;#}_nqfa*dM0GgJpw7MVGj6DX#BO5eq8Ko4DLJO<CNd+j67h
zOmhDL)q6-G2TecR_)?a<BQ~sS6uHWxV?7AQ@n_4piJlgkw_I+$3<<GPkbgdof-J_9
z&Cxf^YH6Fsogg2qq_q7tA(=WLc9LE+10_-7L{A}b31gE1k_(lXaP0~X)F*c!WWX|w
zDO}?A+(Kj{&23@`?JG{3&mD+}hjA#wU+R%SvY<R4@Y!tZ!AVsF#1rno3SqPG(;iSe
zt9~)8&Bv#F{9~Yi*nE)I%Umzm3xY#{L<Ut;_}Mac*$MWrNuE^X%8CU?8@Hn1U;whQ
zlb9T1G(5%#A~m<oySVV1dBM)&nf?E##{OW}!l=XNeSlw!Il@$el1VcMBWO+vWrvZ~
zVygJ5@)y3W_@AJj>!#JVf{CV*^`kwkBXfQ}-3F?q93YAdr9lQq2+Ii*fE)eiQ?SBP
zc0bD6{4AL{Xm}a*E+eBfUd4cx2{)SaWr48Uw&V9L=KbwYoy#t_(_ah~#LMc&B3;I=
zslF3*3dbXEBW80v%l<)PhnG#w9T4F(rN87b$M?y?|5FDmUKDq?YenXxWp7m*!v$}s
z(te~hY%fi0@xB#(C+yMqd_Yc*@P(2>BnQ-z0FB&qB5Fvy;9;DWK?!DV`0M7X$W}F<
zH?@J^&%R@MIb(z+@$txXbl_uJ+{7GRv@Hq!T``U(@7h9a(+T&=GtYha|GC(3-<Wnu
z=HGufbYiAL^u+m1o4#b8ufK+kXR-Tg`Hmm{H(QrV!8?)3$|2zZnE&s38@s-VA~k>E
zd_lDOb9&ohdBh)^1EHmtk}uJezgF}yky*Ule3JGrkVmQvj)GfK;fpcNENWmLpk|XJ
zsryTdPn3&%TA{ZAiBqsJKLaK%H`_KyR<gps+vY4GAd_&4zW=DbF<35=?9l0o%tUC{
z?pJjY{YrNw4J}u0M+K{-zz0usq^m#jDt>qHxxZ_t3|yr5K)&TCvODc;cm+tUdohwF
z-ISDl6!bp&wdVbLnnn?SAchDrrhdmhZSwlzJlM*Gl9iY=i-2aFbLBye(_$It{PPXW
zT`mpaULW27Bs>4;Dd|_3WMEH((Gu*5J<@C>qwpmO#oO}A{tqGoarm10bYmlTpQd6@
zIj4U9oPiZDA5cXB-4!U<q=9%ezb}CU3k7AAMpVx#He`B5$aA+nRNxGhe1c*U{RUD$
zZVovZ{7TJnr6USYTM79Z+4{aJyu$4%I}<UO&3~{c|N5RcE9amQ6tKwi64OXT-}i@O
z%t6Z`rjW;(!3FJOqKJdxkO1auzU9nh?Oa)Mw+|*wlz3g2bnC}&g!%4_UaLC2a2Zov
zxQ-MyI8Mt<?oMmyF7SboO<J@XOytCx!g}x^Zx0`j-wB_aZCFkQTeCR52?1)yG1*eT
z<D-%$xu%rN`DNGx(tj*f)(_u7PZ6B1Xcw|VE-73ET1`#<%|jPL>pG=LSV*&$=xW+$
zmKX=F&L<y`ek?8~CbW8Oi?aP&FKOh%&s@fYl<wkg$@SN;E+ZF{o~8R^8oS7;JKa8I
zb6ikQ;x3RD(~Go3L94%{-~54!P@R=)k+xr;^EnhzBe-3PG?Dr*2QtV(CXn3Y%RY1Q
zYon9M%jaC9sSLRVHm*u#;KXwo9>br#=N4W(Rb_f{SLMkRsH-|3(9BFhFL#>YH8>Hb
z#LuCrNT{iZ9Rc{;DxWo<(9x0L6ETG{;*ciL@#lQ}Eg>7wH<Gp0;(wLqHzcu<;kNJB
z;kUnVHdX9-wqQTpQgv2;l~MIpaF+Pbuls2%M$+&k!k%}*$RM=M;u0f{PLMH)%t6B#
zf^tWbbLvZN97p2CAx-}}X8K0KL~jec-E~0Tloi_@s5f11G4~ndY?#3QGzVutm-E;5
zd}D1{zKnh~C68GkkyhA|R5GvCCh*|T2LcO?&cSx{i;Ipd={G8gEcc{+^p6R0xm&Jd
z)r6KIETW=fP@yOn$ZA-pA@o!h1@DtP$PC?{E_Wi+cJZS61T0D+i<UqM7su6nq7^4s
zk#hwI?IWO4v1zE53{d<~xb+JrtKBx#ih&EEihU!n%VY48=CS)(VSWNuIY^yL@lSo)
zir=Eoe?XZg^80B-+{4-R%1R*@{OU5&a5k9GS9;AW!8c)*_I#tMdYBtbDX9({xrkvw
zmRWTS2YV9*cN3D&#Rd-Z9^Hb%5h*83P)2GrauY8)zw5(p!qs3I<{^P11g+l*AyVf`
zD7!bBi-nAn4d%uJ2lSIikiIFp7st=fvyv(*{~m3&oV9b<XsfZOMOWGtU(_c{UyH4m
zRt>qExD`CQuOTS6nNWl)6@hD4PnQs>wCzz(?J}looN#h5=h{I1QIhB)*J~~M#Akd*
zzQF1R72DG^d4;3$?)-23kF!j&LGB#Gi?s6nmftB|Zl|}J88#)Ani68;AGFlKjxk=&
z3=I12V3}&5Tas?Dgou?tt4L)OgP~wJVh*m29ECi8M1PO57^8%ASnpfe{#uv<TI37n
zurbM@k7OWZ!X|+tvM=ntiKKi)g{_(9;F`u%W@$_5UJDNC*jCXpgBSL*Gt+Nm23KjH
zQ8zPN?((^=wG3G?U_Ipo!Djdu+n(V#w8Z`&zb}V*DVyFKP6EDP|7r!`<@6eWNJL#n
zaW5PFb$Gr}hG+AiwdrHcp7kN!SC!_t<yuT7nCdR_<T>f~!N(yS&^2DP<fky(kt|eT
zD`KNlVJ2;ZIa+$X=-h$N?>Hw^z)25!1korE3kWR5ksu@TMh8>V6sMIU4cU4JG<X;G
zLU5<mRXqNFjpO_SP5eZQ<<l(d!4$QHz4^u>#zy<MW<J)?J}8q?9gSP1wzS#q&##b9
zZ0MiV5PKBdN)1OIjPq;+GSHQM|Gl+0b<4^-4k*q8=Ir!t+5oYNdtyWbDFXegFPDIb
zO}7tXO?i91J5!Z#(DGaM0_8Lq`ZEAfM|6NM4~t>Q92F~LtO_*EUBuAC@1Ymsu3I;C
znVLh8b!JGR9>Cd`Dn1Sua)Xmj#MA*wJZPfdnNlVhgw%m3S3)KWIPO;4j^>0@6oBX#
za9t?8{8fSBLc4xeHI78gjT;V}4mgR1@<6N9|C~e17}i6NmQ-0e9Pk25@QCHED;;gs
z%Bwa*IsRlL*QlK-+T-d0B~9R0$V;keV>df@r_6<u@h;V=_usBHsE!FZi<7C|U4QJM
zwR(EX`ipl$c~D!P&+4Z4cC~!c(kOtGT`J$PuGxX^ZfhZGIsF^%>%~FN<d;@8_(d#G
z#)oLm3_^~0xI3TiZXQr&c+a>wZyrc*|E$NQ770(1Ub0C%$*-Q(eVJ_7|Gv?$LWNU|
z?-9mEBa=mEUI4Ci+oG}Z?-btEy=SqfELYqG`;wpw&tr>Q%nGhRbl)X7q9o=udo+Hv
zp9V8)hjAW}vKr**_)QkyMW@rv>m1Xh;kU3a*e;T^&G~EposNnkTDM+C3=xo_xHF%0
zY9mVYD@4k*{3QSdH-(w9=B2?w{fR+aRYvbSUP*3(A!3jN)(^FmmtF#|tXA+kI09Ye
zTRitotn6qKZ{OH1G<mzXVUP<MAj8J9q~f##88k2hiGq-wBx~e_$R7gpi<W@9+k02Q
zm-vMW_=o0p94i2&QVHG<5;m<Ly}<(hUT8gdOiCaq;67+HBSaWkVeb1wXOwRMc|gI`
z?=+#!C0FBKU~FK1XC%2cV3fM^@`N}CAg%Poya<IZ*5cCLH7tRXgATARX4z@k9VG9=
zHHQpi{q_jk2!<=Sn4gAnL)V~aR{CEzdHrii5G|#QKDCA#YAfD@nb7^!+^hCuMwZ8D
zy|4bBMAxZ2G>NmgF%pd0X>rL&P5cCpv4oOv=niPQha}}8$16I9AIVO$H7^M_l7MD2
z10=d9$4C%QZa%D?lK}?u!3Ex4YsH$mm~x_bKJ064KLT!#X_51Pq77Lzy+fktM@&eN
zK2*n6p5%|O;4!oRLVV;q&e1Mrn_+!DL8hYw6>3R)XXx*5bkBc0E+wEAQPDFUPrk>F
zj;mA`bwM9siCrfYUKHMA8gczpt=}}6Lu1NxK|dfAhb+IyN#*yVO5N~LX6$FT{d$9s
zMQ(<V^gjH+2oei!O}4pB!o<J7Nu^n_G|Bwj;|I9WMva?)CJ4Ay;}Wub&>C=`n{s~p
zxh?#%7~>nkjZ1>vTFGDjacmC@-LlJSToIRQx*ObEpM=Bqm{S|SGjqzBS-C%~d}a6L
z{WUU*zqD4nP8JxCMC9Vn1Ypku42y}!s)?VHY~)8a?wWf%V|u{j4F7VuC+bJRq@oLn
zy^%AJaanEr!Q{|;6yLDiOl2O55QEKD!<*iNuj=1q6K{;5ll&>%(rUD2zF%GKNmctx
zzBg!eO}2f%dm<};7&lmozOJEB)!A!Wy6|^q8J!w_^Y6pb9|KjtFaRLEV!6R`w;=Q_
z3V8)`w#f$0R_jE!7oGt<rPp?HgsJ~UD3DjKSD4M*Hr{VJ?vH1a1xako51R<gzvf0u
zq}<+|nHEs<@mlqfYy*8+#{)@!F^?TG!za_+tORtgB{9`<AJIo>z=YbJ2!V87Rym0)
z>ep)kEZc#L8G<UR3jDx619)jGkHx6*)x)P>o=q+bfD?NprwqI+ENRYYl7fpe2j8={
zfCn=DfarU8h#naGS*|Mkxft}DNQod?4}y;H!=FXrWVHWfLhLV+rLoLS$iZiZGW;1+
zD}s%_^<Nut+?)*~oSbPgRB^D;6yLRi9tNrul7)_S4>lCk0(?nj<X?k(gO~(ud+Lia
ztPmusH)!2^60apA!)z`n0_1ca3!MhIebpegFW+-<a8dpp*d2f5Q2R9mL}^Cf##M4w
z`yih9xt2U?%#2V*D5_W^1+Q74)f|~CKW9{K=9<%NL+IYFM>IpGMdakUe7n)dmPc!*
zX@wk-P#w8jR=Ohv{qo9NukNGE6#~z}N6Ib-ZHznI9%}&#5;yz_+@=}jnI4k5ydG)n
zty})+Se?F8j>d;Q>h__CqfiFu8aCwFV<)xRlM3S(E0{f4#DAFAlxs@)9nPU1dO91V
z0v63o?)UND^mT?Y`?;;Vo2jN!B(mJBmf++Oj%7cW_boK?wI#aFWw7a>2xmv0-<SPG
zH=d@M5<2B&WQC6eAC2N#bdAi_yPcLL@)!_4`AcpU(e>w3EJ9C}j@{?Em%0~Wt#hFl
z5me!c>%Uh=M`&mVR3mlVcb~#@s4AkHWt%+&Ml+m)+M7^92k3&JjI%RU7>%B)EpV@P
zT7WE%E3|pH1LYG1YD=(`_1h|>(aTgRMGH?&2K;<B5aj}(iN7nB=_(KrL~0_>Kx$@1
za4~k`CKzH1;xb@pCt&_@^#X70`yGRJq@4nAr%BT$Xx}@Kh@nsl2}TLIRh7QAT?I}_
zWZxX;f3fifny$$@52%BI?K7x>R2(wS!b*6!=fI=x9|HT}sU}$Oq%+eQ)mmcXBEm+1
z34-*2WfpcH{q)z_9k6w_O$-T^+NRU^Dm*F|jS>BN+&8D*540nx4aonvaRgCIW7$<m
zVR#AL+IoDtw5P!#EsP0U8a@N#A%*bN0L94&Kx&CZR9M?+o|klM$f$aVfaK8lO4;IJ
z2GK4cbe{>zss5+~o!FLlcMLz;x|NJDX$}vi(7KaI)cMqJMg@7s?<8Hj;LAt7yqeF@
zoV`~5VPwOUD|?yY?QEry8fi%ud+xeJlQ(@po(pP@*N1#rbQXqUSL%Z=><w@4MXjj0
znR|n<u;`~7oFoPT??XewgFDZ%{}M{@Q=HXQh`;^aepK0Tx3@nZwq2z%Z@w4g)L(Oy
zNb2n}pr3NA`P*_Et-EA{GaDYW4Rvq+`>Y}7AuiX?kcZ`Gr|)YQ^A+P*W~p|pv*yb_
zQFgp9`L*XbHTWRAmfxQ7XX7L*5uwM<<_cGLfc7%gmSkV^EaF4ZRG_?>Bk!C@LtQ`N
zwJImcgUVaW8-rS9H{uzW{TJB30*pmG^EqBl|BYdLvZ~rG0XhGkChyd8e6RWPmp$qq
z9F8K`li%c4!Wa`poQ$JKb|$Stu?pEvCXCN;O471ZnU$=1|4W&UB}J7up4?N@>DK*0
zK*?<KY;a4i+R8;Adl?t)D<P{1UDH5W)UtO(pPfid;fvIzDRK#=Y9D#$3}=iWBY8N;
zFVTR0;xR`9&e?uo-WVDI29{$XS%|DHpbXZ_fVzu3JHIDM07$_#+IfI?bpjec5j*NO
zK8U_wXhjP%bW*lXf`yF`2Mu=F0`SPylr9kQrFb^y7OnkiuFl0B*yw^kAGAIPl$JY;
zFXtO38BX>sAqeNeY*<bah`omF<C{bFiPWn?IV6#gJWnkkN{s6{lw#rF0$jQe?kq2=
z8}7&IRk!m~wkuc~ykDxukk~)FcJv0@Dmne#sZ<O@!!)KF`!Fv;7!f4)e$bo-?8$Q>
zOIrge_!H?Y0!r35G7j=krLxg@GrqCJcv47ZNUO^oU4u2VV36k@)RQ56L`xY?4=fQL
z;{j<fnGLnt=Gu6BuF`w=kb)otF}~;@W|HdA<769DTsZ%nI+w1I?Lz~VlhJ3y?URnr
zJjec@dME;Ya4)nRsr~AWJ}fysr`w4hjk!s3zbfw+;MAVp!*TGstx7;&rVG9sUb5MY
z<7Lwhn<^Ep+E1iYl!kM|N3IrL7ir%3)L2KYi=)s$;IWq=cV=`hS^_6Izga-Xtn16l
zUf0*>j=vm+Qa2J^EUfLE=D2dv__%J+QOTt_e&?%_G<!Qz2wErp?fS#Z&a<k^R*L~1
z59&7k&*L#km@{ED_#DLO|9p;Pf~#Q#^5Z-I=W)j6ji3$w$*j+ugNoiwpU&3sh4h@W
zfA7QiH7Mh8WMx`Nna}GS`Lpglmz|C4b}#to!jC$o58^0nbpP3oS~8$1B*B52^#b!N
zr7{2e%`m&K{fgiKNq`l0D{<R++@txb{}R7Tu1+yX%9<2nX5Ip95|M7EU1?Y$vo=E^
zQnFK9`Kcf!%+y~vCmX51eYL;v=ODWYX|S+S$25uf7pCuXy7F3u%Dw+#?Faxvm*Ug-
z?TCPJ(xl=Jv}_*O(&Wb(;Lte>Jcqnq!Kd?L)^GB9G+AoEmj_|nY39%?%Zonuz6MpV
zexMaYUV8CXO`lhbHpn*7NUw6ah?^ndxmLg!@0&citiP(uO1~t_;VBeFtclfbAA{Bn
zktD1|lS_w8uNYwWLB2a%%RPG)i-{3LCA-A?SV;Hx?$TbobY;(CPg`gxKos$U7S0xA
zqkuHKC-a3Om7z?ie9H;_-P^mLjw37EV$|Crqp!aym$be&{C54a-mdCtWXZ8R>rErD
z?(klHMM3(>1z2~~L7zPme3`R3=+<BNsH$4fF-bFsqE@N?p+0zAv+_TWlWv0oBeO`|
z#+lzOh}FKQ<WqkAug@uopA9CppJ@`FAIrs37BCzeY<T!&%J1-Ega5jyfS*G)dsh8z
zv7)&&Tma_fyQxvE`|W`ysxG!!=L1eJr8jR?AK?2~LJc)nux$MMPRhbdH=W~OWbE}6
z&=<{!xU<{IH<???g)!Vv7Hb5z9O9GL*QLo3CzGZV8;maZT*jS-fwY$|DK?sS{seA#
zcs8NVyirqSh6)5H_+R@%`@jKENr&Q@>199_nMGCM%sQH{Zjli0``;PcoA8NKJDJWE
zk+dS5>>;HB#M}-Dzsl(}jxlqN9SJffJ+&hwXlee94;^qoTSe}`K9FAiDM$l~o;(MB
zL7L#o-2~(9d)6Hr%YtAj0)K2?>w)_L<ZF7cz)C5w24v`CO8E%I2m*g7jRsf)*S+-z
zW9~S}-c7F*&i*HX7)ZOsmQN^1gh7XA3k>Ts1W0t$j)7aa==NfZWb>eje<<KfBeJAv
zbM9W@(fo?th)8U_;;D-P=dc4JYqHQ;?xm*ApnTTnA_P4g2Q30281M}!%r16(Hd1G%
ztgTu^YBANxyLu$*`2LxZ$q2o!N3(SjthV9x=7c}A=z29<6N}u$iABb!>F*ehB=s5J
z39WGA4f3_yddq+9u|<9hJL*%WG$sK8_8F?^m!}&K+v0wRc`q}twr1K-$>e`n9JdZm
zyNQcv(2|2a@qu(z%C9OissDCafw_eU;`w6f8*Y4P+{<}6TU#P_6Gd82hxg%K`&Hm%
z%H@#ei+EcC%cO7pMGs$eX$Frw%=JAi+4-40kB0Np(s{Rr^PWA9%Ubx!un_j`)bp>?
z@0!&tZw?!HzM92Y-mTx>Y{a`3j1+zx{&_XkK)cEt^y>C6<-EIn&l`f0a9<p>-Y^uS
z!H#ubv1&i;lG950{e8+6wue^#;r9fsNo65Ob1of5>*l-<qijy`bl<<`|6Zx31bZR?
zDK_ad;m3qT`yg8z&L^y(wWUKt&JI}t5#lA6Y~teSNcSi(ce_@a_0p2vSZF%3La)I#
zQU^%#^=142`f9?$@7aNIa0@Ji^L{%V76WWz6xVGJIO@p|Na9%#fkKU?8wn=88{>x3
zOax0Kb{25|nX93Y2Hz*{1Bz`ATL4%7u9hEsdu-8CL1-9-Ac6H{@$NQhNbFdJ^bVYB
z@n|gZLd#orAo7bM#RiycY=sc3X?Qr71%=USJZM4K|Egw7cN-FR?_(=~i9v3_BD~sV
zRsQn`xs#-j^7B&vl*cb00N2u9^IBkH{A%~-a^4N0S5>f<G`|SAk+(n&V5IJkUc(AA
z)E2$z{dnqEW{^5TyRjd=vqMLaou56VWpYVe#)W|9bOPuxqW~k9i(^sO&4_hhR_3Zl
zOk%tRV^coT1R3~`q!(tyDaEr&+hkHd7d8@f2t(2s@hN$4KcKD_X@zDUS_b{yo2N|j
zd=x-lkpAjHItb6*Ccx>@w@0G#-p9X3!Qb(xZP-+hHQf*gtxnB{d#X?q?DE=gTTOmQ
zJgc%+9dVZRCS}*ja=*nVlNa^eJK05LRY96wyLVC86gKNLxZ7QVuL=3h6yFu_+G##6
z4OrgbLAxtbb+^mQX?oC^k&Y-gPU}FA-8;fX<32vlj%89Vd|eeQ+2LvRz&cajbHR`0
zWvEJeqB6Q^H>g|sJ8SdKC#(Yl8f~EtVR#>+K6escNZLQ7I4_j5EthNl?J&Uk9*GR~
zr=DdV`C8XnZku7GlQ%!ROQbtlJa_QolbjyLOK#-gJN^6t9#ain>Rvb0&ewV-t^5!r
z=jL3g%M|r?)Az>XWH=+>#4}7FI-J^^uPMlLh|pusfscd7kympta|$n&_9L~v!sRsd
zlNUypsx>_E6{=HKYOPEFrF>a3>3zZEv6j++JD;96S9hN*K75{LQL7%}@hLyJxo?xw
z;gDQ$GpW~o6R4|~qyg*kfUKM{1_rgKk4C$Ne@~S;**UA1aHsfxg|P3WiXJ#TI(?C&
z^zb6h1YZ4#MOF;UYbgdngFt!5`$e|N;(GB}&Q|*IQl6EPH)@a<X#Dk{LdI@`0z=gx
ze%+MG7cD!^cA~+<9?T_)+H2rDDHuxUt7U<Eh4|mN?|$tjibqokcAf=0Z@Wq&8ugIi
z*5hGGbs!dbZ@-J&y^9~?z30{^`7Ql#<d5E1rIRi^iQ-qS6YU>xg33OnD-C<J*#h{{
z;~=M(`e&C_lCk`LGfVqa&=QpI3|8mLtEu=`jT*6ErKfm57&L;eXhHO)c%C=cF&XE}
zmm&FxyQ4<6Y_TvYBnfw-P*bN^r)Z>pnTp@%#Mb9>t6&S1BmTnzJ}|E-f&u@jOyJUB
zhIp#;hc}fhL*$?gN94qpGsItSPU@L{2A-Yq@GP1rBqIP|0Fk<kWlcz!p)a0|_}Nsz
z6!t?~8!C?p%^;hBEFX;z=Nq)1rp@OoktZM#Zw%qz@-*)Wp*kt69mtJ-pQXbCBtKG4
zO0>QQs2*TZu!PDH?tJgvS=w8BzhUt+hyZbdCpo1Wm9VSj1n*Z~Iy?fV2rukD)&pOR
z^+2BX2VG7vVA0X(w=iG_u#^+`&t~%=&8WhT^)^k|&31DnyV`lVy>Q)~nFK{Ck+Ud{
zkj~%K-0AsCWYfRB4<gOpvtnVAc(os)aX|SlV63j30gK#AF4AW3z=^ZwvomKYHgjw=
z3K<k$k8uzi3jEdBr2~3$o@299qe@u63Q|cw^b2>e=D>NMsuF)xAkgt!!4hjqbH0e6
z%GYep60^pN`~cGtD+pA0)kSMvFP=mH?AMmMZMGP@sI~#*Pi>h_Wd?}%+^!n8dGb&u
zLA}t<-hVs0Hn28k*+cA#+*X&bhC4J8_e`i}@Qgkv7#HUtJZwS7BG3#|OO7kT-5s_w
zr4N?EVF*lu#WBC%pg7>|4p`VwTK)J9{UEl;g5}IgZ$uutf_jhYoe?_vtD@F+%(&d*
z0C5<9p1PXtcq+mXu6|fwaZ(O<)x<#?L1-K#orc|TUkx`9nqRH0+}LdIlxu%%-$U0K
zk8b@oZAyDsX}bRk8zU%BK1&`e2ea8bx|2;RqWAiedX&J4@MW5UyP3L{x;o~+Jw{^G
zVXVG*nxWQ+$#Q!3)5UWZe2ltI2lLY~yTj>^vB6U8^v>682)!0!v7<Lv;a!=-E#JH(
zp#bK2D<w|?fj3n>@5C9#L(-LkHGKGkD`r3wlSYqJ1+*1gzar+eObO<$s4o7pVC`vY
z{)b6MdI{iKIr?Y{K4PGPsOJ5z{)iNXpbb_J1;U9#>iFAaoa$ZI-FdrZS+R`3g6CxE
z4FOR+hRP*KtP|7q85E1aMXL`+$=IJepwP^Qao^YGV!7mNSkWnaZzwXDibVA_8>fRe
z{u1DZsDEcfUTTq2;2$-4CfaoLLm4L4ea2ZjsgMiaa#QSv9bOIC`9TLkf-6c0#DXv$
z9b8mOk~b&XwwloKv&~Tke#wRR$ROzng3l9PY+xz{qv%r`(rQ9PM;L+L2B-Gul&K@g
ztC}cKr<FAP>_YYe%$8;unaMa{fX&$i2gE#x1fkitysiIa=S2iUGt(>IIvu@P5eE5F
zZN99BGLJC(`VNVv&>@!=qkxW+vyw{+$?!eu{iH;f1Bc=MA?e*0OWpjb)%GTRG}P9n
zt4RkU$Y><pDhG<bT=kPwn87PVZ<`C*+AlK~&%0Z6_iyG>!EQnwWW0nI93NX&KEsdn
z{>rx|R@P$Xbg4;eX~GC7=)ne4_~#5OMz9r+aLV+$Q-O+q`iEex$}agEwT{;X{C%Y=
z=|Ne~;*RK`YZQ<r_I(3GVw%_T^YC-@X9A)DS5b})r@EeH8I4>u7;#U77`E*P7d-!_
zV_po&PYBXF$ff>4Xz;|+SAGt_96}+9!M=ST@l9*)wEfW%8t;;p#-o0=4TMf_cyzdf
zK|tlLtMgry-!cZf_gL?CMF|-T_Y-#lxggQKJ<{opmT<X;c+zkRn9w&m&G8_F+?$7W
zA@>fKOmZGZKJ0gM#`^vm*9nJ^6Sq^D_upL?Fmr`Uq&u?{8tHfNihS?K!7%)b|Hpli
z7J-ZTY|$z^lu;Je!vHlKb~i-mj6C)FJFu&&TQ>8pw<9(L9)kz9L`b8ohGvmM%J|!I
z0(^XYbV{l_-;o~cABDM)K}V^?yfg&k9@0U=Ks&u!J((?wmGbyuP4F)Yn>1<A<Y=DW
zqm?LyEc1%ifca1AC$Nye5EZ_62?&Q<z>mybsr><GDee_E8H&pv9CEH*%k{V27zJ*k
zXv-bbGhRO?cjdIkk^at1O7SC#;7y)W$np}=#Cp69Lxlb8=&h4K+GU1PE;C^Uo#2K)
z!y|Bf-Cc{Qs^ki%o~v_qLl2S(#bLv}v&rzbe%si6#x?}aAIoncYzb}!5qAl|7q@At
zd{w=p3T&89p&oBox!IwR!SS`WSt6bv7aZ@+*k~(p!Lm#b@i9#UR+oYA-j))e3w=ii
zb}#Q{et@Ks6{46k!4vTQ5D)!<y^59xmFx_-lN|e0qg_OaXFn~XM`a1bNfN)DAF@K1
zYVV2Z{h$YJYN*D$HB*r;A2mw;x%%da#}A=>gc<M{JIFyZ|2r@&aIGasipaFw*2MW#
zsH77q^B$x7ME1zP5oQgbziCWTH>PO=XYsD&m0D;cy7s%?%<n&VJ%D9grTD9R6Q|X`
zU9mFNExa$kvHM6e?8PlOC<5_+B+|snZZnJSMXup%Y6bE_ED#NJsIcAfY{&n|3Ci*Z
zQTjkPR`Q!=4Mz%)Gd5E%2UvUX?XeAD2V}&(q6e5n#nO^`+oHtQ1zGqvmHRB9TJK`_
zZ;v68_e%!|U|&RexphxVVr%)hSHo!hKl$r7|1xPKf}*j7w?5jvSF>f7qrGkIe^4RO
z_?mCjR5bkdHq>^1A8~aabnfr(;v6{tpko%Lk7`cc{gbH@tw<{!?)=hz58XY3dX((T
z?7+I_ULD(RK1q?He`}OuIKcl=f}v{_c1t>|5WDM@<S)qv=5!y9O{K}uI9uEMLw^=i
zb7r)2NSW8TzS4Q5`<z3~oga6RS}&zg-$3`C(8D?Q;z`4k;g_bBuB|9J;&<fDX<$9)
zjg6}B6iPY`e|F9~x=kT1+zz4xNQ2(9LtA~=LgH0|V}{WQ#LOz`SHKRc<mxEc4K%%_
zkTwRK%liO7D~M&fsEt<z87;<z&)X)7w6T>?Q^>fCp1|BA<yfZ+32{$;O?<Q0OX0VR
zRkLPQ0dO(q2x!WJNf&J)L2O<sfd^Un$n@aSUzk7)vO@)O-a5uxH%EaNu#Z=Tkx{V@
zTUc#M=H%|;aEmlL)S~EVNn+eFFrw!E9m(zmI|l<tx=48I7-vXc2pDIjmfpWYMA72V
z;BeNq2baXaW3-_<PU;h^M}GO+{^9{n1N)iU3?KbnQBd`%yX*V&0&XwDXuQ`5+r2Rt
zdttvt0e!8|Y4JN@`!8gE4{$_3WO(k+MI{A>vnuG2(1A|1`<7O4C5spVn9<*r0kvP%
zr+2w?)Sh#MZCL1i_yy@*!<4>ALhx%6dKWHm|3sF@r~a5hH%V-V_7w^#yd4Fgk)p<j
ziFi^1G+Xp(RFiK@pHVj+8xiY&62x98bF?y7N^3&~K|{HXG)m96OUeh>+3z(iy<`h_
z{!7ho8xxmcK!Bz(_Ct74OhF_H69F3B{HagLlYmI~NeQ#=R$tayvh`m35wdr?>EV4)
z$E9)U2b^u889gcwcrC+^B+QPk55YU$`YTwEUw``X7KpmkMZ41YA?&^xBZ4J+NZR6e
zJps(=gI%5OC+`qj!DpXmd|Z}>gR%7X%5=J)<<h9iI_l^WlbaIo=X7yIHEIOJzl5H$
zT{9{y1-Bud8d>=b+meerdNZBzFwsFR@q-@tE>iH?@{lg+;m5EGw;$;#K_c;!Z_mel
z5{r%{u!t@gKDH2Dl&r<I;Kbc7z5F8gbt0-=QIv4!)|}}=dyV^Lt1rq-?MzzJ9X{>f
z`%jDYXS^SlzC}SF_l>%=*2?Gd(*3)}2!m)*1RW2FC($(ANr;sGg(ivw7|05J2teUB
ztcib%M_v4QoN!i2)aKZF+E6A;8VXr+Nwup%$rumW9<?3YZe1y6K977OyS3P*jqz%(
zP3a&SA!T8?22i1!B$>0!u&dM0zig)S_+P1du0d%sR=zvT<I>Xp6?$lH9|-2{7Ywbq
zmjh}D567<|yxOmMd42wi3K5~QmiQghj}{9Gaqk{lSPCpd(Y(=u<Oz-pub!Y%2-Idu
z&|L~4yg1ZVOqf3wLoIcrQI12*a6;+?+Q!%*NoL89-^1tb(ebKQu)hYQ!8qWae=#6j
z2GoOGNFh8hzX<RXXxi=%D*Ob7tPdpS?B#?LmsDfgWqo`^UM@7vIEd_dfs{A+CVEh@
zZ3h1JeT$JgF7tMeiiyP{;ya0N*fRD2OlKOPdvA6_ru2ZdfGy=H_%4Oh^!r>cPu46?
z2M36Mv>tFKEp~qBtcrZsq?hQ)hQ4^2A2|D)4Wq$Eh`@k@6XfQ|+-?f{;+3#j@`RgL
zS}ru{8oJbY08mVpv}k_mM-_fof47_RyhTE=3P4a*@3C>TSbV_F7c*{Fkh=n$V+)X8
z4j?bi{a6jUY;)oU$UmY@U@R-->lsQHr8X(;0b53~;2{u^u7URR>$cTnNe-LGM(EIQ
zscYRH1D_tbkE;qM%WVeZa%ExQk_j~-y}04F<y#=kDhJrk?pPYpT}qjz@Rq={cDjjW
zgl++6<&^dAIi9!=;!GcnDFp2MU;8Vx?w~ZqeF4@ZIrgkCn~wT8XqJ7h_X;(#WhRba
zsIfwuj)z1iYF--J3}~cm%~Y{OQHxG3CljXCf5hq}Rn>5|8_fw->%z;12Kwb-n{-tZ
z;aM8VS0?QH-&_*@&CJYlsCa$mC}}uW^j2zKG-7D)f!Mawg+!nv01GJz16AhVC}E?#
z3$?rM9==R~6?15_966D31{%{@cC0O7l&wTu)6$|NZZ)0me`B5qGZEI8LCx#GZ-AHA
zllg3)S|ClT8@(u@vW}g>3B*+S(dPUTurHcD{Mhj+^>YTw#RdQl>9u)1P&o$m-HAvc
zAlw-DLcvSDMj{J4#R21-8CCe#FxkVrf=utOo1xz_E-x{;)>u`u-=>btBk;d)BXYw>
ziwFg9&T}%j#Ia4=mhw)b(IbMQP#gYvNYnGLhfCb&{g({eAi&mL!jR3yvEgT2T@vrA
za1HzzTQZ!C2yz5Ls~Ev!3L59$SjV@zPxs&VrIS#oof~JW$$|~5wId3g<Ftf@t?kr+
zIFTXTB&NEObf=L+2v#f5paE8>5lAOI0gO2x>qlm-C_su`5oW^-|Jh<YN+0)`bH4rA
z9oPqo`7**SF;QPZGwp+6NMd=sfZ8CRto{M$+Zoc`#}w~f0jn-XeLj4mnqb#f;3!|@
z<?=Vjf*A5=NaD&NeZct>sX5>LjQH7{EGVfwH4(alH_jQR-l$Vjbk1wFOQKE-VhsoF
zFgj~TQXDJ}-BJziSa+5RpGb+|kZQB~S0E^S0`M`6)3S6>Fh2NQozsv@!P0_J5)^^A
zvyk{XoI?i#R!03Bd#5WUUnQA{Zm#f{5m>8Jp?WPfymG87_!7Ox@JoohJB*3?w*}<E
zQA)`u-TgrB3-OJ<0q-B!jv96K!2LiJFyu6Cpc)Uvc1o+_r%}STBBj0(=njE>!tPqD
z>-VC~ElX#{C(<aoJtaCtGO&Sr4A8eN=k$~~Jka*B0~{Fl=Sq+(S-7>eMKg@2f!S)H
zLQKNx@|U-ldyb$S_uzXB;Zf~JTMSg_pAN&$dQSdgyWUu)zAKcFwOqofWdMhW%Q@@*
z@D_W2kcy%N$ya#fHFm&(k$U=u1j5yF?Ye%^&Ph(T*yvfV@Ug@6p2Z<Yi<#JcJ^tXN
zkZgynF~(hU?ajr3<L(<AG!fbjm$b#@AZgZU2Ir}7h|@isjI}c-qKEgqX-JJ^0WV7h
z!sYM^b>3z6Pu>JNyew#XhI6ESrSRJLfTDjwP7gtP!SXSH=z_%}l|m*=eu8x=(g$(x
zU$wd0($sk8Gw;FDKauWa?4=w(WB?f!mrSwRo?C>g$-jSMWX~&48T$~*$b^<;?87$i
zY^+C<+B}eupZm>C;`{*p!SRvMkZekCEpchy=OBxuioPv%u9&Vr%eExag?KnUmAoef
z;3bhOONhUfh3TST-_mXNTvk2)v@7YCi;W$X`hiex8L=4PxVMnb<v8=sRP9)<-J_jg
zutT?dl|?NtdB3?E8=M9NbQ-~?LlgB0tT(<H@!u6UvM+jd{zC>pMf0_tJ)z*hY?3;;
zdh5$$nha(~j3D~X@G=W^RCAmxTuh=bsidD#!GhM62KlnI@)Cxm6<0NStHM-*zFpeV
zph{dH$iA4V{9KkJ15IsP*#v`p`~gA8d@8@4%(Ss5H8OHqy9ThS_$_;21jCt|Ab3#x
z5RqpFfdDCYwRs1o)gO{65$^Bj&jUOfmfup^wufL7`VO+{S7bdf9L<pv=qJ9!Sl$CC
zrbZ~H**-a!j0HIBd)}<8v)V{i+_eUwD~;eOkpSJjTn$)%Q0IV_rg&$K2b~`FT@d3|
z6C$IjArxq(zl71o$?9bUh7=^;?DfnWgxG+vnElg(0SM==%uK5(L3>2q9)QZ2SY6`T
zwB-#(LV4&`n582u#~2Bn1s;9$0x448@WD9Qh920y`vW)P7N0x^zl;@db#b9at$p54
zxv(9Vw1*5Sxx6Ti6zhPq*LPexNGUejMeRc0aDs(K<EuYn?_z}{rM3ucFeEY9pi(F|
zU^Doa{4~!+qTOJKB#>6%1<8CTX7<Ww9bH%}#q>#hsIXqC{zw1oLj{{lX`E=G=%Hl3
zZZ<bCJB{=4n0-R+v%^3w4W)Ey8l_kUYz%p1U=c7$7jCKuH+4CQ<o=$8iT@yk)bkwa
zv|uRf(R}=#QgDb5vmlEaTMKrO6p_%i4>|tzY5j(#Cku|Z9tOxi$({5oKRtmx%yC*|
z<FTlwmtuHWI|+n$OkV8fNiw5+)-NlB>LnI|^NmB!1)X=B0APFmK(u}=TD6g~b4a=5
z>37-`oT<8KbmAPsN})Rvd~{Hq=b%H!@UA+6<e~+JEr=1*a~GP$a5enP^J4eb&Fw7p
zJo%M!Vo_DTQ!g!g$UjiRevPFdXAnFF3s%=4i?!VvFy;T%G5=gEh=<*XLoMS-J{%?)
z@aVX$J9n$g3vto#YJ^vyx+zt;KuAY(p)O^8Vv2Cf6d+$LSL%xdr99m80U}kRze>tp
zbf1~dL6Xm373Mm97mmtr5}sV?FM!8P!&oyM4X%pE-a&({Wd7qFBvr|P(pGTlkd8-I
z{OYCB-_gEvNKl&WQmjpcS`qEsV)-I*vzzh17zi?OJ{+|B>>)KQmGmu&up!z<&qI2T
zewdT`@6FY_gM#xpKV*cH02<(HCKB2&VvsSL!4$cQUn!cw&XSOcT*5Atfyy^vaFFXy
z;*PfMD+j`dYXG4hLW$*cjvoDkm$}Mt25VuJDFRiW1!+{z-CCUbrq<(j^w412iF{Sc
zP>PToUL}1C4HYDL05hon;=5k4{DE_gpEUo~hb6IPu#A(*TGvJtq|?qdcvVBYnB?j;
ziaTbcwSmZg3mE!O_KsA5g-*52Fbzl?R0T&lAx%UKc^=%k9h-&KO2Sx>lS;@L-j*Am
zfA;eyag$o7a#1N9g+Td0--@V-UfRoKyyqiBkzTopNKrd5xFQT$6JTl#4qViR#H(bj
znfWM<xMAbmke!6#Qvy@D11Swn0s_#{lXJiF(m&I`GmodnK@&QbHDL^QtY0VdSq>#n
z<Ma~WnX9jg`#{#V9|Z!h2HDWc;h-7x7#pRR#z6xc7~OZm(m0Z%zhaw6<lrgOeS?{V
zgY_(OEX*Bhx)AyVj->y10aU9{hpd{hw&&I6s~1cg<g4}nfhTo}IE=B~dm8V?uT1VD
z^D%(<Lhxh2_|+4q$R@_S&yGZucY~C9-v_9y_im@_wOL*a^Li*abgX04rFRqy<P;^x
z7vVM|x@?))p?QpG^XHxIpRZlp8+%+V6>}AVbsk^*j)mwnrPpu!DZ$kY8A5F1F;By3
zhk~ZT157n8dTUaE9*}$-Vfq{poss^wv#sCS3!R0Wu<9Mf8OggRo3Vfn<aACk+TU)#
z{kVfts6PDD(}B7I4TMaR%FyZbhk4(2oO|*`PY9BT`D~gMR#{Z4`M^AL#EKSrE~k2l
zfv9+*w<or+iE*@vuzD6OwL(~DE{SW4IdYzhFk?U=18Ddm9ZE^zY?40Nh~OZLNS)+&
zAFyofQHSa{Do!}qp1%iSQF5r@K2e1Alm<&Jw}&Qpcx(T>17y;CBxUNm-<6Dw9+<a%
zuKv`nR+1G{w%soW<RhfVngGWO{Xz}8;K^TJrQ12#4A;ziVCBH1zbFPMd=oY0^}ipa
zh<UqC@E%%smB3Myzf{EIn6oa?M46G^NAszu^f~7-A?RfGdWq6SZtSkP+x{v71P5`1
z;zOyrWRZ^GKvMQl_8nB?eO(H^;_f2%MwAy%BCl{D3JH1^$Ok>vZ*_WOk1bNFFLPnz
ziSOj!S+eXM8`l7>_WoGtkiK*{Z2h_Rr77kHw%})2V2orFe{$a|5C)J-E&A$*Ks3_m
zd`M^&ufU<odeHXP_f~bEp#}6fuV6JF!bHAi;XpnMd@-tz{in}+PmvG$AI{cQD!W)y
zX?V|$7CRf%tW+qH4Dg1NfBOMnLx2E#j(w#Ul3nHef*pK#=oQhv9|m@i@_M;o2$z2<
zq*e_$7dCS)VOPH1KzOu${RfXn38DlC(G}I&^gUG}aa)lPkmOz}=23iK_gFf?Gp=BQ
zp<%6l>Eh3MitW^wHt&eFz8Suco4<W#V~Xs@^9A}}N||-Q$x2fB4f-d7qJ6E4W9jeE
z7)k8(-Jh*m9y3(dr}gr!su2ef8uJRT2h7pUuAX2zKJGqIY32V{y=wjDKJvU+y+u!M
zuotx-c5S*H=R8H(4e^|@9XiJ*rhlc#`F26>YoFqGc#JD_Zpe|GyCKHZry{W9Bigmx
zvad36Gru*bpOB~81b5oO;PUXJG5HC!0c*s8czw{%w>ry}w^R%-5#r*=2iZ5|Ye_G?
zP%vtP%B&k&9{r!+piCNIU3D6GLaC`vFu-k)lQ-rgFz=&45<M1~x?XOT7gg92Ae%mA
z$DUIcN9>(2&l&I#{PgDlxEVT8P$U*L$MO|k(Mru$yr9wpO^(@r{(d!OYsNer^23e$
zb`0BFbiw<*0*xLkbWepAGcCXe7GTk~JnfI%`kmTf_|Tw*VRB#5i|~9cE_&;N{5&)a
zj2EV`K$alajU!Xz8nrXi9AXX_sXJ}!2Kjx<48WQs1TED7JqsY#q6f)xhF^5clUd(>
zAVvSaBI!Q^8Ag>kWPo$`t#q@O$A`4rBoTBp%e-7_^1eChbFiZE23S*y;FsSW^uLw4
zn@Oez_8NA$f(3Nq04OGt>DM)dn+xOo+0K4Vs%MnVd6OC}-|!%0Hv0*Rw5K-w$woJE
z&CA(DkypD<Kr(91N@t{kPTTPS7#Z{pxbQ32XaU2f9ssdm=p6-{JT<{ZUp`~cc@Lc&
z1orU3?e($xB}l=*4t|#0hO0!X(T-e1Xpyi!t1c|?W6XMT2(sBq?4Lh#iDJFq&Z=8_
zAD>3-6B5>9H~sEw#%2@;_?R%rmqbqU&j@gKB*Sd~LN(VT-zbFe9oeTP&EixGHloE7
zqEi#oBWN@=?z4rquCHFyFDJLj`qxs9a&rJ?bgCAV%5%;IEb#Z74v~OcYh=<=L~r%)
zXukU-hu47AqF-%G2-wL#O*wh+ofJ$128Zz+f~}4L6Mx5YK!XGl9y|F)3Xq#8g^77l
zQ_PHMb^r3o9I&GfYZ)+ItARe#%$5(_A;`7o)@h#u%)%HX&i1;lXoLjGyl1{I9;p{p
z2f{}BW|PaV+OTLxkWq|rnjLVAAYiC9))jR<r^Xrp*SO@&saXx)xZGj?tM4uAY(ih}
z`gefL{qRaOB|Fq%i;*}=h}Q#V-5!uQT@jD*VMN*DVX{bh8ZkLa$GvOgw-ryn9*6~e
z`1m+S*Yq(1sKHYLj_jM&CQK)l*;*%pH%!?#sd4y0a+fxc_OJhx>WgIuKKk<vqkl>o
zTsXq=zruZd!k1pV`iXb<9Sd3`Yc~P<okj1GKnf1w<=PtlI!12l&_8kbhYa#9od0=l
zF6Z6#Xp_3FJG15XwYb4UWETXwt+PFoi3&~GZ|OYLtWc`oBy(r)eP5B-x4kyCtWRy5
z5_yBm<j;mCA#u%Rsw00sqhOO;Lto;zqkTLuGN}0BmUe}9>_)$eHf$0k@DUrWcpELv
zpug&VImEpCm`wK<XJ+$pH^GMBg4dfN(daMye9NdLb13xuz0?MO|CV1>{PKJ~>Py`q
zJk~PiGq7>B|AZzZK%0!Q?)XE7Z}g9Zn!Um8x4nmQyD}$=9hGkvXfNn~MBW*L)khe^
zcZZ{nU8@iG&<>b2N=r{YjVN+*m5E5L=#p|+Do}oXMTB0I!*E@e%Hg>Aq?Y}$=OB<<
zkXVvOkFGZ8RlE(_?W<8Px~Jhzl%QJo;efZ~z45G1<vCqeCYnp(5Ds+adC0BkKw?ZZ
z)e3DB6^)45-5;Cf&wf*jK>_-knRkEx{BAKuyuORx+|P`-eXbmGE-NaNtRQ6)Z$;Cn
zQ#NxTk@J8$*d|b1pqX!UzpWj!EdeQ$GKPGRy!BdF&9dx`?PKTST)qWMAEgl{lW5eh
zBS8nR-(ccz#%rr(TSSW|B14^?0Z#tS$+$Ws@~T%J5Mc<!fu3ZjVo4eW2QB45tOkHY
z$eg>9!wwnl-)N|}yKu9K3Uk@u^;wG#!f(yQ^-be;iJk>kqL*PohO&6j(D^qlfbdAs
zTeT8h&yJy!kX18uv#<B>aeQFC?;Z2#3JV&q>!*ZZ8+Gr!<W2$CNlJu%rZoSX;WW`{
z%ylStCkznLEPqr==HqbufjyrhDGx>=VFT2fvJyG`oA&-3xHA#ZZadwxF4~qx+1Ny4
zYFT1A*waAN)j;!^G@bn602c=8$;bRfdx-n}JUGfe<&4LNQx2%q1*ThMSeCN|yS9y-
zG)jvF5BGGqg&0E&FI!Zr;$--*LvQ}@gZEnx^<0<=pU8NzdHq#^-ZOBh?s%#<`j`xE
z+RLkSa6g{Y-KpGUB0LlS02l4HYJrZUo=J^8S)@ZKWQ7vFcFe!buywFE^+>%_|B+ey
zj|g{R!^^iUtYSNX^l&`$z@y9S<Ezu8i`BB_GFQ(H!$*QFPJ4F<Q-BK%6*T%R6gl<d
z*fn*^wInALxs^xUy-<7xd*T8CFtWc$?NEK`diX3+qDZ(p@AaFb2$r|w<fdOe5HNqU
zm35`k`PTKZI;s!yk9r*<S2ff$PFrnU$JpBj7<XC#*S#+`?m_9{XvF+a)rjsWV<Hd*
z+T`7*mHY@F7?2P@E!1dx-g~OvaB5@1!IBaEH<QnXFE;GA>0ML9M1d!*$BxHd%l5rw
zdF7v%0~Q@`llg`P51Fq=VBEKQ2g(6WJmhjgzHVDqIbhO>!GJiL^%qO?XVCUx{}YN3
zV*y3VRix){A`faz%V;x_h=n`HBSchIGY(J~18cbHeC}!j@j&Yceq4$~Sz3w*u(*Sa
zq_R$J{~p*<P`uksU)*cz)s6Rj`EH=kP7NnRTpQ@VD>KAJRan^lpB*T@J}zF<Pio60
zcEm(hkkrgq6^h4jVwQ+tDMpREvw*ve!yZUzr&oe@wiGt^y%ZDgb<*vH=0>$**t<k;
zX=#*&_9X!oVlji(%>^*@^K<O+l!Mii{S1KLR@>=BNg+T~s4!XlwY5&;Rys?lzr4^!
z6;*rTTk>$O9;PKDvZRM046#MO%lAg4*{)k<`+yCwtz0ZC4r_9Q+wfC2EqZ&HWk;3T
zy|?1RY2UB4`X?Sy8o-BMt<4LS>8BM`QC|?O39FQ(ZSZxjo4yr4gq&`Tck?l^zDiHb
zP`k6}xWE0;g(jmV<(b6yTiyVZOV^vpThG0)CsuLSc0j7T<)2uDZvKU>P#EoE!|D5N
zA`SK5y1xD%tX~>Cj~Np3fZHAp6mNGib4Vg}-kE}9?j23G!dnR?<)nujc0AoP7-~P#
zX{yG4M#t*nRw%JDzsfM4+>{c`PaVsi&3PyMhtcjXTKw-P*XBQzkE5&vwFDRE(6jP>
zT;q~*Azq*DGU-bmXaj4RnWL#_MbDdI;SDM)Y0eZYh&Z`zcDq2G^55X`9knn=R8r-?
zryF=lSSff{<-fKj<FTT$Jl5V%ez<NUTjFk6aozdOaBe6BoT80O$i(MrX*Ul2@&Y}$
zHD4H|nS6c0gm0;La`EvX2aiTw$_pj6`W;&DWe%q#*MsCLAn=c2h4#=Rvo-Rh;25Uy
zPoJpk=|<f<R4&TPx9qOLFuYT9BD%Lc_d4RS--8TpVPvs7AzLcn*IQOAz3-%av~$Q9
z+7-g~-F(Yg(3!3-SW$B0!9i8}^VRcg9WN^U9JZGcw{tY{3{51mC<}fnk?j=Us3bi1
zjG|dk^s7Ki?#ZALm~gHx=@F3Y+u3yj8V7Psu5eT*)yNs#YUV?D%q0$5x~OM9(E6*I
zYLS96OyIZZ!Xw~bfJ{-9{;}U~Mf<9snw&k~ID=~qW%~7|b*(pUE5*Ld#V4$Q+w1yW
zSlM%q<?;NsPl0k(Hp56iy&cPmk^RpWx7QaSnzO3VgOM+q&e>Nur-WgD{zc*e+SS{}
z7FUPgy*?6sk5DQLazFS=ioF#Nw$*xv=D-7^!GM#;81jc(Z4ZeYJ1AH)L}9u9B&(q0
zJt2Hn<AC9&pMg$6B}Av7T3Pu=rolnOd`KDdB6Bz9?kWY}bgD?ZJ4Cuu)MWnSLR#O#
z#?krpV8qT2!3@}&Drkev_u%e`SIc)lyJdO_!~F1T^AV=bwUcY{*Gz@ZTwm^hlPAae
z@CdHCr5L!TAo~hr27GS$&sm~Y*i&|oJ-bqs&u#MWo_%|vbr(kSyV$q(d9uiy;HgI-
zm4UV8whUJ$pBV5nV2f<EkPRE#dYb(-3O%yXpu+xAd@lOCY1OYf$yq~G)F;??$MV_^
zq6EI8mzkCS4_kj3Rpt6deZw?}q<}OmKqM9EmbFNwLAnt^lrCu&3+WOi1f-?Al#~XM
zM!G?|7u~G)WdHB`8Sfa+^Udwpdpp$YI<MnA=A6HY%IuvsDja~~%>1bdr#{({PFPc?
zd18F|#P?`dIBAb@xqMo>UA0E_kq%1!_i5Zt1+y8=5FA2CC(@!wm7WmL(qEV^d&+np
z<*{xG%%Y&{m9q6PIGt0wR95Qd{MF)hRS`U>YHza63TJ#ke|q&~UFJLk@zJ#;pQ%_l
z#i@(>l&&^^hM-JEJkxM7+XP-w3fEa%$nv~j7?ZYlOsurws1mmHCL4ue)MR)@TL3kH
zrL~gYIXO2=HXZb1#Oz3a00V1J{1cm9xiyy8*dN-`P^NH{oQ-Z-rl8HlaGIp;2}?e=
z2V~)Pf9mLi7f8L8G*YGC1vlf(CpPPX;FH(<;mw*6npD&xTkIkFQz*d!kyHk1zRADD
zi{AXEe2=%=O7yD|7y=KW&mSGhC8haka^iP?g!p;_5daJE2nr>~`m)Fj8q3jP{_NO;
z3cD9UQryh{&Za4Wp!~9kJ!WSzAz#*YmG&C50E$$dyb^DKHz(s|#{7riz`@9ZnE5kl
zz#uiGBK26q+OI-MaE3{B@>Ax+Hr$1T;t1k)*1w}guGGCnW2ZKEGe?iIP$eqmy?BrX
z_ahu2v+rR-!$xPvw=OiQtGw*b9T{$mq7~Nk@~VG@g<-SccU2NQ4!kC4ktXr%q|T_3
z_#Gsm_~e0_M)6rzr3Cf4dbd>9;%?4|0fA`MxqwzR1Pa|VVy76waS<CaU@bOv42mv`
z-eT6Tkl7MmfNRObAA%4P{Kk`U!&I-kCR53yK7LiP2Iepw5)Yt&-m{p9BQ*l%cbFFl
zUASq}@LLg_Y_xV!E<=FjrP1?f19)yCzvq(w;m4yh49`E>S=`*Yh}!s}b9cGUT?Dsl
z$2UKBqjzV#_iE}>S!|(Bm{RFMEpP4F@g=$x(eiY~jsMt?FRCeZVP^>_<U63yk8j@J
zrV)K^eu;At%a@}#Vre(!kthDbkf8gaEe!4O4=Vp~ai(*vKV3npbirSTe9D$>K~daD
z+X^gOfW4EYA(+-^u7X9ymn{6XkYjiFaC1b?SoBCN{k^S)l4Xv#f{O+_%MuiGP{h#f
z%~gr~zw^MQG4}=Y9QHJ6os~~?#Wyw-L;n9dn#p@TTWw;Z@@XcvV@>nm=aFd|7iB<?
z{QAB!re+WIacX~Le0OG4Z`4C{8Q33(SWC{&j3O@>%~{mG2{|me9GzABHk8oqjaDdZ
z-;2aLJ32F+U!WS#2#{)U+m<IgZ!9|v7*>{F-!~$2C1bm#o`OHMN`p;L3@qkYqtq)s
zK|<K3KUm7#|7%$v&FQlf2-n1$I9n|~1C^fp7=J#sgqpYK^W8@jNw(FX)4e8g!AO|8
zl0f=9&t+Sg?_d<gHSIc<@(_O`tOI)=R%^Sr3Bi?z(#?K9Ns7yei*Oq(P1E)k792Wi
z-P}sctiC(cF4Ec+zLqzr!JI&t`W#?WurK=lncEBJ+SqlUuP}7SG#*^l;|&%oM`bBc
zu+}<jby(ic-msan`Q)%yys4pHnB?O}y&IWl))RkrFW{s$GfR!^lk9Q2#x>&}>KQ0G
z?xwT1p4BEnE?G_BwJ+4O*OP?2kMVDtP<@O9_to{ypELF@U5)QX9v7--X3bO%=W@Qu
z?!P@iJ}{_x3?nuBjk<kX8Hdq*k1b_!Nls}oc^SsRplM!wb-EuRiUI~*D-ik;V5Ec9
zA(}<c-t0E>dGAvVB<JaUp!)6>+cl>4O3L4R0iG@z<wHj7rIE{)t`&}P2zg=p@E$fg
z404YId@CQ-;_+btdGv)oHhRem;t1`#Hy;1(U>7lI>hMz^Hrtz&jgJi?3%5)3CYpCy
zQ3xXwxcyvgyzbSCr9<}<nF`0I>=njiT;HOWy>AR_Mh1w+#p18CFp?k6j^MtKk2RKJ
z29Z<8l9x|psRgN$;#MxPH_Rd1;0*jEO34mirVsGTi#Rho|5fvLFfFtmIAX|K#Gg<#
zhs%|-zsLWi214}pA9iVql4JMN)7kMvhGe5!yd(qr1YsWZZ~qhz-BIuse2$`)$7t+Y
zCk?@MWpgM~CdvL1C^$K!Eb<~J;O|!MQ=5P6=gt>d6j*z;%S3lbMgE>7{<oY6SO_UA
zpVL3t!KT{n(gq1Txq?D+>vA=LumFE~X2u|Oy<Z>fk%8tu7z<aTD&Q8;_b&9`CXX@<
zI??HV_6c;x0!21M%4O&~cskY8a}+7gD_8@C661D)S%pWh5Le&_@#emdzg@rl2<W3g
zwOS+LrwCC<1s?3V0-K&EqRP>UM{p$KYIFEE1AkS#aDL%f;a}7TJw6rE-O2N;2ZzW;
zk&Q4XS;+><D!4*ruk|Ym{=cNV=ZBRIC}ugnT!<sKccT|u%d&Pyt&8<7SjsCV^zEr)
z)9WuS_X{snXINg3o`lQXK3G_yG;&0Y#ECV}^{xwF1z1EAV-+*yNxxVD0<~6U;SHmo
zo+o73dz@php-a!)kM&KSEwn?EB(dcOv#wQTnUr9v|AqL-@v7z7<>0_Amv-TccKtPK
zU370_4*l};r?N%|#=}=lk-ekTBkqvTfY!!F;t>poq#N97xh-qb%Fj`k6wLKYsS7*I
z7e7FlK0O^!M_et<8uHX;Z~)$*)QCtDK1R}}>70iXTlpV`r+WmoA^Nyv@%t+qS)}HA
zsEH4C2^p(I_fI^Y1u`D9Or8)V_rFK^^<RD1ogM~mi`{6I!H=R_-0yuiHZAi(&V(m-
z_Vx}R=PJ;!d-<h=<)1lz;5N+sNLR#42DKHCpB5l>eIt{UDQQE3t5NZ~bK{GW-p8TG
zy0Df#x#cdVCaM(dOfIaz+gokYIU+W`aFiUS2_VIE5El~$P=v=9pU}yg-w-nhBr>eY
zMCAg{k|qePgz3ROHVUu}U8g+SLA-g$&A9A4cKTDK&Ov{(WkkegFm35sLd7d}S45GH
zO3z0q^(iC<Ohyy8YAcsgyVWDsE1_&-ht-)@=Z+o9cZ=V@#=pGYi5zKD?aC$LFmsS&
zufSrO=pFzD2K2$*CgP<d=^MPe;r9?9<MP4doGd73isZrJPZp^`_J0PKqlscm!aSr;
z=(iuS{*S>3#%dZ8tS?xIX<^c_P15Iy@F}w0QLZmaXyOzD3eRO%>+*<WA|IE1Q*a$U
zX#Me6h-Dh}7RXqyh!F1j&@B+<cj=M@Q!;D$?lWm-fc<0A6PY+ByC0=h*fBqV!Tthf
z*5Z#YF!@uRT3xPUamtL|u!y@ixKZH5vNt|A$F-mM{#PDTRS(nrd;50leM#T{_qjHE
zk@*6nlWei<-u~Lal6!gcP1nPoDyWC)`=)&qBdu={D0DE=si+V#TOqCXKHI}Xmvqqs
zUv2BTdO^U;=-oU7iZ51B!pZi8cw+|MDQ*w+ox+G5rEzu~4@md2Ghk(9#;P46eLwBr
zK-$hEo6cO4HIX|>BP5+8rrg-*b2#MKNXe!YOB)DYiZx9qpBr!@)~u9GI?-TN#z{~q
z#4FDw^oK9?gw*||H@CPSi+UM&{j%LwlVsy>4!??((m`^H{`BXJi{&d<k={XAU4~YH
zrs6GuqB$e{xZ;b?uw8+AO?y~mk15A&tsiGMBS>0=nt1OD4vzL?Nj#0?(j71diY=un
z7Z<~Is=UwV;!>&Z93bZPzH<Y`G_JcpjQIlL2V@ak)Jlt)_tp`f#gXq!P<}oM42Rt;
z^)G^@8+)n6KXBh!;2Bm2GA`5-pCQ5@KB1Ys1%0L48dQYGkQnA|VlskcsXd#pS`-#~
zvhhtieo}p&UT*Y$ucJxHnN;@HPrEs3_tSc|{HuJ4x2AGWSd&2Bfce>F-TH=<Wl#RE
z=zhOIVL~Bds+t8)+}-jYrxLFsYI!}JQ)R)ub*k<5tE$*{(pRvSSHc6gH>NRZLdSFc
zR#`7&o>(}rwmrwZ3_+`2AXL}6Is^B5GZfIcbtbyJEr=MbtuEEL^!)LWAbr#Nu{A1y
z_8BUpEooD#>ipY{ZizuWs8-%CuMY5US10v;AjDrZu`u&9K3oU|{K+E#NVVhi^75H?
zugA)8=9ZFhzGTST#pb@Vj3J*P>(g`!utbWu1W(DU1U^2O8#S@bM}wIB?0D9|EGd<D
z;Qo+T?5$$g?b5w}qI&Z!zSK=^9&zte=uivM>6=17`#v*rq)}SZd^OT0kHLjq^1WaO
zq2HJO{-80cRW61J5I&%38YDws`3lIfHCJ~$*92uxEF|J?Z1IMZ-39ypdsm+;fJyuC
z0wZRV4HqM?hE;4qP@PO9fAAt`$K3(TYL$jxdq4pP3ci|XEd0Z?&sjKk@V|K_Ck_ST
zg>y@LVxZ%lOZ?vdsRA@$3g7dkkjK4bzr_mym?mcP9c?&Ht~pngbWA`CsL1#pSX<+;
zVE^!>BNJ*yb0YniAgw=<xBb;qgSIj>Ugwn;Tk+738o<!uBlusc=loa6<8HJ&@KRe*
z!@%GRJWO+>I#_<g@@N9t_9M(!kMBHgp!8SNtqH(4EL9fF&+o=%{}61dB!0qC*a12Y
z@%Q0cr{&4TSwdA(yi4hKi$MST;4#b$eig!0pD(&AB3av%SZb$e-a*uGzq=jy-A3*a
zU6Ne`CM}-ixH?1lShObjWbJ32@+`Et+y`<o1zhc2Jon4dqFMd7PFH$HVP7!$=pVAi
zf_DyQY8lp22FY088W6d>HWx(!+v151SkkCsn@~wws$8(C1uJ@RF=O`-UGNvNGCUUE
zYQ7a4ObYDpRz&gV4rXwbnqm8Vb2RC^c$Rl5oKV3KfZu=XB;Th0as;4%O39(sAvqpv
z@P5IG^0q&3=CGEO4c4SAHx|*$ikaUmGW^L7@VEB?&MU0UTWFh^=VoC`^i?lR^$4UU
zT_>&Y+tsg>gp#vUOqfbcEF`C1B#9sK=tf&^rFjP9zQz-C!T*lH*egkjDL5MFNcj$k
zHslIsJNJ3i|HP{O#8Nvk(nAxB+Sd}E^WwUHviGJG6*247@!e)pSjoF6|DR4HSO9Jz
zA}%bQxN2<b!w}xjN8V%7e3aNBF47$VSDN&k+Y_|@TMC|lOeJUQGl%}^wAj=ra{P1S
z&U{U4GCk@e-A;{<RT3dVc)?G@+OE@QJi-Mv_M5Kk*)Z`j%Ahbv$mQVf#oM2#LTzRW
ztU_HWmiBY-Pab6`4?i?#kO&r^a0lirEs!i;zx=($NE{<%H@z~dutJcwo;-%UG%J%6
zpD2S2g)s8j32n96$buIyUj9)VFmcTY&Y7(bq-sUQ^8l=;Ztw)^4_kW2bb6^AVRL~0
z%>CaxCyX|?R`h~1w)611In<vV)!eW0&1R}2(Rmz}5#Jnsu|jzEtN0GHl)r!G`fr?V
z=g9<D(MRauCK`i92@AgK$@Wx*K~4W=tCPP-s$xSqH>~Ib<NUjG=5`j_ihnF0ZtCB`
z-lRF+jk5~?2=ZFaGDu5}4&w!w<?bR9;HQW2vQ43*K2%a=oxS5E(_J`OR_!riCjx@m
zS_4(D)_hi^Sq3GIx%nuzf5zkw4Yz9l7JMvLD|{CZ3PGEHyQGYk30Em+@R_lz@eGV~
zWtjK;b<joY3NDy750~7H?iTk&Wlhc=R(F=Ui*g-;`?bh6d(Auf+#Mayqg}H!7l_Ox
zk;$++h)F=+B_M-Ma1bs126QvB;^fz}Q!3!i3bO`z(D)#@T>&Gjeba@NB|@mubxni}
zxV%b@8^q$xYl-n6BZV@9wo{a&p`z&FkJsuM1*}QH`eTTDEsV_ohL(B}MIV4`F14f3
ztnJ`I-~pT{`3l5ky3Bf~*i8pt*zr0dv>2X-?0fs{<B~S*AN)!mi37XQlwkI>TfRa_
z#!QF%s=T3wnJxx7c2`q3pBt-EMIWW#5mb{qDjr+pO;SAr_thzvL~jQZD&FvrjgYn?
zpruWY=BXQeZ=tzaQDy)KL+qft8JuXi(V63+oAS@J+Q)I-F*clSZVMj61NX09U?Y+u
zbAlkyD1Y$)b6DAPGJkj!&6u6OYEAkbIva3RxA<JSvrz!Dp$TkWt{kk6WkKh4`1dD@
z)V8i-bYW&7+Ai>Zl%L^3D&@?H5#aIhVe&sb4*sb&h+#OqW|1#<!X#6Or}n<NqQJ6m
zte^!e2oqs?N7!t!mLZ&~F02{*tJ{;~V&()xESay>G)kklJlIL~>-xC^!^Gu#gH*c?
z+zTE@PpH3w7kV@?L>4svDVWirHpD7A!gd7xzlsv^N-%MI$kxyK0EpvBNO*&(U{|Cz
zjLXi>+NOwI2yTt=h$!05(HKmt8L1jPR5Jg@%|$rmU**NyJ9FkEt}sz9GxnKc@CB-#
zal5qZL_5U3%s#SeNA|;j)coy@R3SYv1`SSRe{Q1)T=@zAxlE|9N<J?7^43;9zFV6K
z5smJ*)X;7CrsdbaeeO;%Rd~1$^<JC@J$bZoo7ImZIz{qsC27=)-GJoURt`sSq@dki
zv(yx(|GQT7r|i0=2JfE82`YGJ>1e<I^OWB_9_#576)$8f?j~qqs?jTC5%0n5B4)bT
z+;4spgNr_1sY6D%A(eJfw71i`+Id3UOS^2!Q1r}suMaJl?LKRieett6Mh<?S9syO+
zLUssofWXfG$29wv0_!-ZUt12LCFafW>dQtUhpgO%s1_CMrP-4D!6TC3Pe|~h{)Zq$
zq(ITqFlo+K-+nI5u#v5~I-eUh)Fs=!XbvqldYk?Cq8mpk&D+XpZ1!AKqm{WeD$=GK
zALHsyL;ixdXiR}sa3cWbS%X?frV`WUv?n(SnP3w*0=y_H<$5-M!TR%F8eSW_G7N2%
zsaw)zP5D3Vdt7r6m_O<_vLf>DyiEPZAg+BpZBr(P{LY&!?4X($l76Q$+yURg6wmqT
zR9(@(vIBe<Rri0Lr8U>a2FG85E2k<oo=c+q3fNK!Ys!Lz@Y@W3G5lW_Yu~UeUMEA4
zcAQdoE^);q*gDIxH&uv#Z2=mB4=iK917jqHhTOBXv&c%ELQx%$JC-5LlV?KFJbFFV
zJuOxc7>APq87o$ZMg3V4h${o-+JguGqdhQbH@3Ia9Sl4-38+EnCD?<joGgH0XM#z}
zGdW13vu55aLZ?#D7H_g$k1lZCpvHce!+Ws>J%Dc)m<A+H`elJpJ&woJq#x6!H5**r
zxLEJ^VGb%e?mlhU;bX0RoA41P%D#VnfSb-Coeu~CuDs1P<5<c_BEu2jvVU>E{yo@t
z;W`jXdlDd|6)GSj{}x};zusOc6Tll#dL)cJm=+euQf23Erp5tS0=8B~T<GRhlSTt)
zX~2SLSx*m_>mH3aOA=#+e_s)-4Wys%##a>~X8+49{PHpCZ@#&VLHAocmGWZ3x@^3*
zTDzWzL*YaNfs3LypQR4HNop||u&kQ4KcoI0YAHTbR{FY%yfo~`Nms!(T92gseaX|Y
zu9w2mf28wGX=Vr{49ave>TWM9a1l(wxajP+00bfD^|!Y6`gco8N0$vQF-<Y~1C`FX
zyHxbw5w-6(SEn`|J3>2XCvI^wysHac!CoYxI}0=+z3vtT6}M=Ju;zduDdmgyH+pcY
zw_y)hH{yAD%sU27bkD1<j~E;L2IG`dJ%0_|`A6Q8BzEicrG6`&UVZO<bR9k9HdlZ2
zX>jg+>A>R_;oM(GC0~YeUsvvt6xyzYyfU-|BzJx&Y|+`hU0x_-f7R&m_RaR%NUip#
z$Jy?Q^@;eY#dJGT>M(!C(l-=v$%;4W%?xa9k&DDObT%>+ShG4V-nR;rlF2{9;3s%{
zA^I73$tVy2&t?$YqFyqOU2rRi|0)lBvUz!o?0Cfnb(~w5Jb)PADrx!W3wMh-S#fH1
zULF~Pgt>Ap-@y(lR+Y1$cz&k8To_?mpuEYaaW!FCV&bUqx~KK#>&#92*lwV%$=WUv
zp{vd9&=-~ED<1OarTnuF$rk<F2gQx-Ba<tkS^g4ue;t+nKL&1HFmQ|Lf`NN?$CLBE
z^^Z48;no--XvegEwGHGZ<Z}uGG;i;|<^Pvbs)2|6*gZWLwsjmjk4}EMcFa8efH{TY
z*N1;DuXcSpLc3iCxt~VtaT^(BI6AWMRQ+CQ$GyXYeZgy~yb?+Oy63@$f$ojRwIBW3
zd=mICnJo0CLo7!<?gd;$(Fp!?m!L*TF4xe}mZ&un3kFqPTLO1R8P?U!bzip4|9qS7
zg=@gU6~~ux{8>Ah|82Cw;;b!kz*>1<I>jY;&I$kmm-m}La|ZxRZ61h`Rz0Bf|Dp)W
zE#sJB)rWw;;(rT>jM6!d=&;*Jl2aV4>s)1X77Pu%2^M8^h@xOTwk0_pH~uOwRt5=P
zXUqTVj;lj~<;qywZ2MY{)dO<PLQ9Sfd8Hdk@f*VU(ZQ{o`d~@3vvhwv>O++@C40^4
zr?yK6a7wE4jbdRVoPprDu;OrhO}b^16nUsH@>ZD`2|<?f`n)~H;aQ(p04ul6%n%0W
zE;MZ8?k!Pr3FY{?0E*Y^RQMI@;oQcB<Xo+{Hf2gCsi&m%JY0B*;a^R#(er9TZ-jl$
z9q$0dhW-}7_woIbaEaiF>IHu@F1m*pm6Ts&-fsELI>#F}zUk%~!n~g$sf<@rOM%P!
zx%PBC@hm&C8E9~WGfOl~>OZBzu199%YRpt~9`6;YVURzhiPu|2J#A<Vxr?{qokXvb
zv_9CmOWxcJE^w9zBc!yFuFVhn)r5bu1rIV{Nt0u_QUunse9Mcq3gxlUqi7*kMb_9Y
zz<}4;t*}iK3Q(Q;4YyjBnP>*bBpzK0l{reAj@2Q2^(^+3I#e0jVYdQ?A*V=NzvRQ(
za-S)_pqzM0Ct4j_xUXI!^4f6?<JbPKoplo_AvIs%yyunOy4ci0h`jf^YOfRG;)9<&
z)hIwkCab=<x!Lv<VWQNld!6I8h@JLypV-Ya!ojazwDy**a_u35dMmRu2w%~^L3g3+
z$!xvfdqzt9&sE_Azt|@6oQdP}a_8^fB<1fnKDge54(USKZZb>X{`cE#zWw}trd8$>
z2tjSN_4MF!vx{Kw&>5SAbn1K1mG+bi34&e2H^#mvN7_He(-%HMRdh)7T9ZdI-?)$F
zC{f;JE%WrilWr|&T9t9KZh~#xxg|C95seEH_$4jh_`hGtG-$#tr$^hvSNjpW5ELEV
z#%8%oMPhuP<+R>jVM16&?^VG<#&e3NZ_Fb;2vsPF`ND0xiCt+&%hXoYB(>i?YQyg$
z_EFP6nLE2?aq)!QKGxabGC;lOHd?H;i2nYPj7{XG{v>(`?7j=Cmb|pa{;)Al9b_rX
z#H-7pdy-_EC#VA?@fS37P!1gs0g<^AkA1#FGI-pgIq|dqO;!EJ@Up3GYcIKOb8m%a
z{rbsTiv05BzjCDiVM6rf7B?Qqs^C6||J1&<1rYyI%DwMDg%za-th3yiTvmr}xO7O^
zsb;%--SmUE)$jIV57G6oe1RDmZ+_w+TAaZvej)5^YpnOjF|Fnwa8P8PgU!`2pC9V}
zAXWt)hD><}G3=mk2wZ%M*FFHKXIC+U5)2Jv5)f#jr&J}doeg1V;~SZ?pS2YqVx#wA
z?gu*gD)S(R*FprbmxMfALghe2+TW?#cIt}>{KoBMUSPa7hM1`h+zmzm{p(?sM~uH|
zpW+9E_g)>F%sk^iJKdgDJ`8MpQa2PI2)8!<KJ!z31zZ;{G+7>(y})2o#nb)hNk|ry
zT#eQ0BIA5Jra*kndf?v?&i%hRIrKj*V?*o9acMo0!xV1Y$;*dVZ!-55{(VcgR~5Km
zG43Jw&Vlzv7e%sHI^ZmunJxYggI534iobW!`hu2!^rSy&yAV09l^hlG`WCe_C1dK_
zxeouPYDrx?Za2-YUkxBU9K;V|?gw&DgQd8N_vs#jJyxb(1)hID@rLsCMJ6ZrIgx%%
z4|lBVzp&1GiVtI00>0At<Zb>xPdL-o1%=!aY%$gSM*!64FhqZo(XGTODZO5<2F`9L
z?K$$OTe3|?6KgYUntJg0Rq+?vvDIs7l-1y{B%a=r^JHCRkLk84Qob+AkgM_d)nZpq
zcz$ov4GY5_tfeq>X843f+7vxi)<dHAHnp`fj>6LyIBZ}XlnzT*<6SD>;;+{kcQOIL
z_35n@W4$DDNFK<SJ;9ZufFE*<t?gU|LKoU5`KqF;<kP<^b0%40+^>><>Af4|((zu)
zdmk<J`m9XJOWMW1kitg%{ZAq{`l`1MX*F<<lq3PCtP&c33Hz_>kqg1K3a)<wl~Ze^
zB$+{sU&6nIc4r#N{;)jcyPi_Z^xyo}to@F*R_}}8jJ=1$5?CAbSu%^SyFOd}ErwvD
z`xLxGcc>~Z3{6JA5e6)lyE305Ja`j8wYY-%B2C|i^YkELG6p;Ve~uK?%M5F1-}NT+
zbN14FW5^WOc|%0?ECz>&Zqb4H8R0~{hvSO!IQ#0)(xivD582iR)60_d@ej;aKwrZB
zo#tl^9M@8}hoQgUqvN2v2LWfp8r3I$H2B*Luh(L-!So@)iy-|CB@qw~no6ywn`#$c
z3-cNA`xmz46FQw=*_X>V4Nt^Hclfr^1q3*_0bwWS*FP8A4@v4vZ0u<of^5|QPH~K2
z8ic(nC!0UX94XBMrWC$%1U^7*klpXrbteANGyuV~du|kfi-_NWL3Q!INyX_PKX$qP
z8Vx2MpZ*a`0|GrEX30*fm&T#e{){1KdN<z-%C|r9f|QKvyPSWSZ>U;#IUAiC+w9Yc
z%C!l<&cY0Mw~shK7z^7aUpe2me}8qltwbr*p#FnwcZp@T=M?mt_}#9)l@!bquMzHd
z7PUfk%h60W6HQbNjdQjXwHlLbHTe+k7hJ!Xy2w&q{z~%Sk+l^6|8ZpHfg{T^njJrT
zYwBorCiH!$c_(%c`u&gxc9&Z1oOkPv{{t0%BaE@B5=wK%8t?_mZ6FlAu{~U&xN7f*
zpY{nT?hn8AsVJSdip{&?3{4)LUx>~7;;+hKnT_I)D~Y>qK!eUD^V5;nj0|XZ{XHZE
zpF#r2v5*Y-IE7}s<nt{*7NhJPKzBf4ItT%GR_~N<ci{8XtvonkAiz0*kmF+N`URNj
zRG%${X_27Kdn~wvG<czT!Pz_8)v-sYt5z`5q%Yt~nLpH!Rs^wkM{-Gs$Sr95@#ukk
zCIbu<5vkW5p`lIuZ7m>=U+}=4qH&@L@hjNWsgnZ0uYa_7HCGIabU&9sT^~8!ulx54
z3AVh_$YZq2R7)(y3bgf1nJ|q+X?V@7^L-dQaHsi6H<M%sF0wRYK-r%#^n?dM!KNwx
zpY&ZaiCh|O;p}{#nAvAx*;^`)16(mg2{j~ce!T4zd6mnWCU%g382-J=%^m4&E`N{D
zjlrVq17^%-ZL+1%>=m_0f_ezWQJ&jipIXedhB^wEf`ODVCU=rCt4?ef(#^a4;bkng
zsHlf(cnAb8?9rV;a0WjgGG8zD-=8kLHZ}|&wEC{_=`o%pZffi^2KGm@-Rm_MByl<&
zET1q@%nB5lr4t`mQ;e0FTpCE(&V#XwmPoC8$h&<lZ!W<~;^|xc9{0DE0sc%pfdeYn
zC9Sn|Tx)8EIa>W9Yx|omc8bDTmGIZL{rnZr*E_f1M{6psoV0Jous-QJ{5na9AN8Fw
z2d|T{a*a82))joKfGys+{3)gyO%!c9<HU*ZI^zn=41){Hn<q@1d9Jz1+MY<hjiU>M
z3S75F@;+VKk-EZRFRz>YG1ki2f`_LCZCahXXC6;J#P!DP;f6GRYikmT#@<Y45Y_x+
z7ku&qv^mu8{+zL%Hst0C2}`0+bQ-G^o_SptRGcVfr(3<gkE!;yp0HueIL%Ozu#BAx
z(YVL&5t+GM&@%6PrN35JrNPpaKV|15m>mZM3q0l3Z!bT$$$7{e2rb3?VbOd#wc78N
zO1@pCCF(8;u}~S9rfS+XXZvL&95lJ>UXF{<7&M4Nt~V6#2J;C1_&8h>T|0(x)f{R(
z?t6UjTU8todfv>l32ctNMBoa5tXr#0rVgd4a>P{scji1P@z=PHlDH8M4q|jj9Fq@)
zZ2SAVWseopq0SOFf$^e8++Zj=NtGXZYUQQ}&{ikd4|FC59p-{5u;xPoplhXY_n*@U
zo7ebXudberoVOu@5I`5?oiX@ysl>#xUG_e{5G#T@IU^HKMJ`6VEu`7NSv8mRATDCG
z{YPb#bgQ;oS$EV%GnEuKf(`9Rsn%|$_uXS65BZVXi)9kDv(L$ni&}f1cf{ni$GB$K
z6Qy#wu@UiBl=AR-`96UBX;_XH-vDY))wY-SUqY|Ccf}C(FG!?fYz{=-fU;&%mDK^&
zeLToGou+Pra8_vyyy_bR=)Zl_ro1W2xy=VrlVPX3;^nJ^y6-TF1>lzlY+eUBl$eZ9
zudYJAO<7SILAfpvRdV(`oNJHWzBawqK`<-O%e`W&3P5{B<I12vLRyc}4WxZ|0gIvU
zpHFe2KsEVo`=4q;qx1iE9dy?K$?`9omIfq#FfD3|@PK+(KzX=*_xkWGB2T~2-1pe|
z%l}^|`~i!405;Y3QNt%^Cw$G4wAOI%PhAUs_VLQCN1r9m`u@EJW#HTxiI-ysMC2HX
zB%z{+Cr)kW6y%r?lF&0Q>da?%D&M@fv~{!XRAr9{4Yxz7$**K0uSl)Uzf)wd{q8m+
zTf{d|DpHpY2Nlq5*|KR1d$?Z>5@@C4WPvnHTEt471+vI$#IH~i8_NCN3N<69X)xME
z3U1!HDY6?LO$lbma^h|9+D}!=d6N5^k9oYq?_DEb^-dB~)ytw&w8tc%nww*kx$Pj8
zt=P^oZN84^{74m{(QSEIYfnw#5<BlK)NI}^4@x1Vt8?)dI97WSmYW*aS&C|hsT5~k
z@=T}{P{E6hNSnW=qhsO>#k9)s&7#bgC2*;8K@KGDE_EvgWaQ#7xr+|{a(B@GFdC~?
zpjm+o5zYhP0(R_v_I|I*j1G6UUQN+uSF=hMRk6#HbY6>kO`6dtnFQy@y<&B`y>>4;
zXS17(bhJYp6jEG|Z|TiBs~_l(SvnIqzgH(1t@b#0t*g<oIXLs;Zn>}f)EHOZ6>-M!
zd;+Hg)kJ)f1IoK(DcMG5{NGP){-!R~PPmwJ*ip6k9qqPdztEIwCMq?4;?6l~hs%N2
zVxd=jTM}U<{2JG~!6GF3n5H)Xn?qRKmzK1fw>740)$@-Xdp?TL#~wVYS%pyx_bffj
z;K)|na4Xk|jqt@e^t2&zIil35;VXCGVdSp-9Qkiwz>+9@S)oFCj<&2SPhd~0SCp$y
z^k5%7p<w$FJ_bJD1qqgaOEt$UW(ToG&uC<DtCeWWH~G(1%gn+&hZGYW%Z^&|G)h1I
zIc5rbf~95-BtxuHs`iPonp4WpT|C_Kjhl*02-tx#vTxrwvOM5vt!xAD9qol<_l6XR
zdI~WkfaTFjt<#}m_RpYf#{ne6p{GkCn$wwlcZM-t=y|>UNb-};LB9o>AW0n;ldZ1z
zpENAREkjG|a`{z~kKfPspnu>8GPiw>Z`~2<Ls#Ri`Avv73G&_r?ef@-IMaHk0o?q?
z&~7q{c6Z~%;S?bRsU@QPl59rt{Yqs{=88uvKjXpA-|b8)gtP(~gU#1%B1N88D1IQ$
zk12j{wEsgC<<!oC&0TJ-xksf?sUCdPZ(oMrpj4DngZgs!eN*(F9@;G$+Az&4x&3<%
z(=-p7>MWj()kuD39#Gk~WH$mrI<HhFQfi-<|9!BA-flRfxYPvjE=RSeaCKkXg|?vZ
zp>Gb2_Lkz4&VleoIuTcYF{541P^_Py6_UbyG;*1GAIc%feeM;k|0m$ANZzr9_l;jA
zeTKw*!OLE}U?=6td62~(6ENvD?%0@yrG77A(Jw{mc#c-K*>C@y`~zTm_&<|S3_6zL
z!O=YG@TgPq$vO4oouW=y!J;TsjU^d_Igaj2%-uA^)6;M2g6R>%0JRvf%sr51$p|gS
zLMKB29-Eauz`*_YzgrQ&xb`h;I{SFJy6=7Ri!tsi)C63W$U^0FzcqzR;<dC`sSYKS
z7(S6h7rLd2$W`(DXt|XEX?eJFA8OKGh=^U+=B{!=E~0-W{jjpqCdvw5Nmp|v#z@2_
zo+fhrUlWiQL+<q}`NnG~%nTXRDim$sK%*PUln)2i2Ygo~j?;kXy0uOpNx%jYiOJ;T
z-t_rKhR6mug0Wzyk(yv2d)FPq^m(k7HtJhXb%PtuzwDQu(+FKA;3jEl;1_*o%r+Mt
z)?A}vJc7qRB=x>viOPcS_q)@D40BAK=lUzIf$ogRTI}T(2EI#NdE}$)%~bdKOs?qz
zpQv9Jr;K|r$j@t2N}a`|H+qQ^!M5Lx;gP*NjGG|UbZ?mr8FTYdeiMyvZ2{nx?joK1
zkKn-%EBuNk@>RbHS03`C?l;`3a|=>n`8zaxaH6sP=a-@Us?6Tl;E)j2^dUEoM585f
zLW=)LcViM<410FD9>A+orr@wstKh4u8Wk?jMKoX_lK=X|en;)Cbwz|C0BHXp@vzy_
z;_Sxwt^V_)eAG|%D1D=`3e1Kim*zLE_h$)`|75;ze@fvbBaNvn={?_SpzysI3ogch
zMW^Z^>mKxgvJw{^I1-Anx#rcRP6Y0}!PRy%LLiGKafe&F+M{Nwo?o4QDsUk<&WX=d
zJHw@s71$$qg^T%=K2I%uZ?swhPhiQlg>Z^vRSxLDD4F%qXo=d{j)&(etp~iIYN7aS
zo_{|=C&Eje&hO{qtIMOZX3Q%!C8a83GOw!%$`cTwd;DsH##Fidul<+3o|Vl@Jg1uo
zI_)^)hquMtt{cwf9??7u*40l;u0?le8bSsnwz)xz7Gr8>=F3kvQr@~4?yJa~K|_ak
zEF}p;COw+(bY&|Ya<oaM1LNSdoKF(sBxS>8{a!}=;MP}s!%<S1C!3iiCwGNPNm>Wp
zARiXI7f$BSB#%57#AlLRteOV<T-Kitin_>0Y3@OH`BZ<iG+UKSdpu}$pT)8z#;AYh
zbDax|l$AK#>|t}hX)qn5e=H?6_2LGwdq)dc%-NDj*7*%<f<f(h%g_i;!dlHV)({QR
zXEpr6pSXz2ldV=~*UVh$jSN?cI$F>c6Y{$qy@aK*Ll7}zLHFbK=BaFdS45(tcl$Fj
zShlW_Cu<|J?LGtZB{~ua=q(s2roBUAb+%J{j!W2-0O58cvHbWf{)Y|YCkAvs=ZVj1
zO$*SLhqB|N)v5_kjSE$`Ge)efWmNZ6ZxAbkFr_}TP>0*kOx>=l!33iDv;rT<%XpT4
zFba>;4;kCT2<i^9BAxobWI_wM-tpo+0Pj6ilLeZ`lNxokBp~sFTPI(=$;bNSF$$B0
zaSV3dloa7jH2wP}*d+m7xECBufCm4J#Eh<LOdTS%Bf0$n%bFJrzTa`HYH_WljDxYD
zjH`FV)G}|J#`)*}T+{9hB6t{bifCGPQ9Oc3@2><{{|K`~4>mC`y@lt6q`yAHIGp7-
z?`|YKY)_jMXa2Lwgz*{@6S5d#{onff9}FVbe5NLLAnP_QTEK?3wU__GSBz0FSBcjl
zk0MiheMMWAQ5E6~A>*^&&&&Csv+eIA^-E>Dz(vfgC*^r()?0(aq1c%v-BDJ<pSIMt
z(dO*WET*~Cf#CIhysf7$yKy+v<t_M|C8Sr<PQX#}qIU{Z*L>Om8uYq)#eUwt%txSk
z!LnMiB(7_pp<j#@>cAu`WCUpK_n-YbowmsY5D)o4J`yTX%~jv*`P1hY=9jNV{ayXj
z{4U?Q{gd)67H#ZX8miT>pb~S<KkWXVw52wP$kHQ!44*`rKl7P%Y@i;>EChQ3olE;{
zxV$3LxZV{3NUxscziRB~@Z*cLf_G<rdQU4@wyg{TtJgX#gMLe^DcJX<k1ryp>~Um)
zoE@9Y%Q}+f$Th|i{lC8@KrsiwOi@|g^mmjMXG#aiw@L=ngV5m_`@y$}f8EV4NRng7
zB8~AR1C~JV#=)l!Px+(ibq|mK%L4FkzrHwHisT<yEGD(YXD8*m4I^Vi>$f11^g2d6
zgk;E6`{b@C?zydW1~yVTW?U%M+m8w#wm?Hr<DyMOvrfP1hsY^TcAxoOd(;Bqf0tR=
zdvyi}<GHlQX3p0*B6d$VPLPTnl`-zmv@X{u&}1d%*mV0T_FX%8!c_RMNBG=RB={}W
zd$lK&ViQuzsLz}FdC}kUo%>?E=3t|4_MgNC!*GbpWNGjJg2eRU^(8;2KTD{VMoL9R
zqI=ZmwOV?75!yAfKB!%d8&J;p7Vz((IRyI9vDNs?mO<b(J0`m3`#jpiJ4mscR&U-n
zDr$<DfBYfT{8oS&>}v5s`S}kvVQwW5&(+Bw#8MUZ6kdv6==@Ugm{YvO@H4YBz*G=M
zJomk9nXD)^4=M=d`do4SZp0*5HQ_HWHC{|nl580&=c`F!Bzmpk%2OnURzg~cPJ)+V
zpM{EyMU+6?fl!JIQJU6DAh0hb@?_{uq;K9h)#4)Rt#0e|T%<xjrhX8$l5d)sR+U>;
z>tcIDZkDdT02&OyvWjPPUs~$+)ALLAA)`fTX<;{#RR>9nH!6dP0KL|)$<mINmZeTc
z*LpJA*HpeySZdo)TE#pyZNl)9oV3obMznlR{PaYM06y(85_U`1%^bhBb-LTCB1Dah
z`ARZ9>qzz72g@Lv=y$QCl`y>}1Zsk9UG>AV_o0>LYg9@eKHo+iAkokzZ)}&{GrCa4
zbTds8w5b0{m?0ey-u-OoQ@Zeim(tr6{@<kCahcHY=ZQT|&*bjLlsDoSPYkR5lLg<u
z5jTB(-u_9dE$=?%8lHpsT`?!%;zD9#aWEuqZF$KzghHmUm%q*~sJ8Yz;u0;zZlP{R
z508IYY;iU8>R-JY`ZydNDgHct>L(FgVEE|Cz5hE0LD3cb3HB8kaUDlF(BhGue7+O8
z4TCS2rPrl=9*Q~C-j+rc!QPX~dCn@8XuN5vN%8_=C~q}ShHn>PkSCaUp$I721CIY4
zbaf;zhWBsQ@-dx#cBlUY<s8d1L4z>>d?cj&;}0uUI5z&U1DCi6+y2k}NpvFicuJkM
z7e=T~D_Xdi-%YF|c*5}3n!0$#aw1J{{n0Va06{IcK!^kqwBNB3LXCF~QmR8gY=rfy
zH<VW1ZqH_CKKQ_DDWDEKge>34&wJxC-wq0u6zGHjaLvQJ5bP@vdN)oGBfxQGuA@CZ
zAAIlfu<>l(+hF}~!6^Rzt7nvSrir()y|3~Da4_t~GJ8$x&kt&5I$0KSKdCkx{*I!}
zBVbKTxCBowZXeEXwTU-#^`?X-fgJRc$`h=df!-t=YeujytrK4?{$-CGt@VEdJA3MJ
zHHibP<Gs0=(Qn}d4`9R?6KzM<ki`kO>u=k$PD?*v=gDMh`JL+@iP>$~@y$cMS}KNt
zhg!x}p5+nIirignS3ZPW*)I`+wz+QXHl~dPI5DhasXrDto0CEg^C_MAiN{p!uFC?S
zy>F;(gdc|^58eXl!;jy|x3EB-I?crc{v7X0V_lyB>-gl08Vrpzqf?V=;+~icpHf@`
z;giW;F*nEg0x==c0WZzZkEJ7+dKC$KtFe<_o1MS2x8mTmTmyGnBeJEpwYcb3x!Lbb
z#NPewftu}I1rxL0C5GJ!QNSw1yc)|o0qhQT)5Gr;^HKtc|41@fT*&$<-n?2YVl_=l
z_=)fzG1h*Ymz3BmvgS;mxAbzA*qadRG7HgSVbOmc_LTp;iY}q|K;ZcC5-&+Do*Fd$
zM(7LMhnt4XZwyzLUnpkiYZeA-UO#jwRr^t>ZGJK+_03r^j$0oe{3~Q~+w%o#dEXyL
zxtcGR=@&ipA~~N&vfLl2ciWQfai65|N^Q{;OmZvv9Cmd-Hacc-hHmb!V+04vcvj#p
z;gd@Yl6s5;416N4*HSDx+NAtM)kJsq;IWGQ6G|&c`2$J3$r;wQyk!A)O#%9czIAxr
ziT;>&Xgy2vP~(T(1np9`*RIY7V3nUpIE1tEqPj4ORSF-@hcwqUKW51Q04eChXjk*L
zE;R-Y``;J8%>n$}k=2W7al4&&0zAyc20!p=QvRZy?oH8p-fTX%^1g=f5Zg^xYuMiK
z<;V1jSY3#9UGd&_rJVE-^!>dQ$HN`9zn4K0@Bt0?a_!q|$YLk=y34&^&y>bN8xnEs
zi#y%QNPclhhB)Z-9O<v7_vh)sF}*mMG2P!^Y61UUdax-FvtSCbt?E?wPs(vN%A%vW
z7l|g4tG}iI!@->KLqC6u*%BSK0jD$$LWS^9e^_>(Ht`vN&f|nyE!bZ>1(`UXF88Kl
zULB)aZ>YXIp?CHer2ic#Gom=+neTH~Gx&cx7GYS(dzoHIVS=1pxF5;$mY5V=!*jz&
zc%x835oSi@6gLO1@|ve?WoUAW7<A6;G*k3M>W$pc0v0r)PBJDXUDa7g%igR*`gf<6
zXa2_L#>r|jFkzC#-sB+{HTyVQr*KaE$Fz}u8KeBE@Vb@%cl!Jx1*eZMFUgBT&!D+0
zyBjY(`CyC$``)5#|M2EPBxSOn@(V={#gCZBnGFL~GTLSDV~8vqG8WwRlJDKHt#w|9
zm8DvV8g=o0y&}YVvA&M4<>6kR<*C6!V16mJV!YJL(|p{0+^cSPAj7gu4^Kq2%z0FL
zHy(Ok`+<|I)c$XFnCQ^gs}A0iF!HOHQr$#=`ttD!97WIBe9qLlq$D7;NH+g+ANfYY
z@`k6QWz8n<#_NOYdr4}T2j<Vvqsze*!I5<slooH2Z&#pqi7&d&=(iqCo==HubV+Sk
zdpGzGUb4Lh=qLGEAjf7%Y=!L3M-9%a4E5PJW>QL=jEcOv1V=^N#d>O98FoT&f3r9)
zu{<%xS)x$3z(R@5L<`OGb0_6s8lV{p?Xo*9NBuIpKF^yI{S>P-Jc1ki>_Px_je{}q
z4>2}KO^i`kki^Js`t^`4f%%rw*n=00+ix0Etr1%>1`@79il|bbDQBcOgHTFZ^k6^=
z)WrpJKF`D0QM8wkpQLuD@)WscQ^mY5)sI|U3`+MRCa|5&=JokuUGcg7`ej<`X)kfn
zt0&H8w>e9Q(JEwy>Ay19Eo2qRY2tDErd0{C+&V)SWyqOS(@$wozAK1Pg*?6%m@XF)
zh8WDft9|y8^@OkbgX<~Zo|pFbCDvA_a}w<*zK)Rj#3pJr@fwQmD1?9eKv->jbcOmH
zkCy7I6gM5(W1L05VRZ_9-MjhoDaB`J#}#3@-dUjTMJkQ`Sq3}T@l+S<XNSQg7o(uM
z_|mE!Mu>rf!E_i_!+_AmqI6CL0=KDz;4SYe%%4N#^uVdIFyXm~p2dBe&4t>bf}J-U
zxfnvoWkz27gxnCaJfFi;Xaa(6W5vC)Pk;&iB9j4YYh0s2ok93_uoP=bTYh~Com8rT
zntho_OhjR<s#sW6p~E8-maBM;^Qz*jCeI@)5#Z8~pW%MB@n}?J1G8Of28GbfQQRc(
z8&}9dt%A?Wz47y2-F&o_f($PE;vHz0prn^kgxu$<$(WhLcbK*BzE42?XYD-$Skr60
zF~%()0xBM==rRo|@n^YZGcPq)e(@_TD0RrCN8m@8qs^bAZj9=c(Hm#69<nYbmNJ}`
zIsbNGYs-B9qcBCq<97H-l0@T1*k+}Mrv%{<;Q_?#<CmgMpMy4TT~ac0FjB;f;H?Bw
zA{d9gTiE_dLcj$HNeRv+JS_amtOP3jq3jk$88&00Ata7}Rz8v)_P;XVJGYH9B|2ZO
zVdP+;uBu!eg6&7$%vZVuotU#!^P${hmqBuEa)JgT&4jbNfn(kja})93iN<^P&fi$6
ze3Pc%esh>;5!WLjXLb`zSPm|NZg^|l#}H_qIHxZ~<-%Ipl3LCwJ@6i^6>uYa|8PVW
z_7>mXk6=UeeY1F_a4#=rp6Mx+KB*5!v#;zUZso0iNOdK1)jx{7Ys3c3ta{;N%Nz^Y
zIO>2FZ|KiS&8$yVo8<fJl?wy`Gm-etGkzaFPQ7Xmjkd#hg3HCl$mV^!n$~t=slJ?i
zwA%o&lGKAwTZW@2pAI1;tm$jUaRlDciA0(hw6cHBhV+xzFM>2YQ;%;pIy{N8#ZOMf
zJ*X-K74oBxi9yWD=kZ72T>u<IGG0bee7qCL5aG!r_x$YvFzlOfoQ!Oq(`_94{*`$o
zSt|Q`UzlXW$&NVjc=pl92Ggc5o8<Cioe|9o-{pEo4BGP2vxHYe55V~bDr)G7*Jg#A
zuPO4Sthd=eZ2rZ55fb;22G`dyG-hy&Kvn1W;C0)|@-CHC#sGh(VvRnj#_wa;`d7D&
z?*8RynJZd%8zv9~rb6iEE#DgxXfN^am81C4-XBRjYY7c6>d8gp3QGtu++LY&2rI0S
zvaib#rooF8l<R`F#09b%-Q#X1OiOuAsYbRA<@x*a-sJ-P;888rA(xH>MsvpPq7`Q@
zA#8_C^W$Mbw%<n_CLD~C*F9(7b0Ya8S%YVuG$Wvl0wh!sVEpx~QSgh_BeS^ndv8TG
z0&S^p6J>&v8l(ulAMGY+0{D`ipsyS+pngCe9RAqip3C7$l0jHRYr9e&hr9A1Fnd-=
zc`xCdRD}*MrUI8VxS+@)%S=%RO}cu3EFC-Y_xfA<dYU&;DFBe&r)MVa$*Z%Y#j6JO
zlAV;evY>?S(mG$i>M_YFa6odgf2URulXKmYTB^0YZ!q$&Z9?)d)(0`=CO(q#;^MHc
zyd-AOC;F|~T!=q&4;)Od4{6WG+4Y5mI~f-qKa<I8*@yXa8xe~a3q&XRO*6TZ(+S;9
zpIgiey`KVAFXS8nY+l#_b1o6){zQw~vwiIP>{>ITnuY{=>^K-S;f>eIo0L|b_fona
zf_KhV0n+dbFZ9gcwpq6APA`5S3nl-tL2`u&C3cazx^|gVW1FVzfU-e^>q!^Sf1b<q
z-IN%SWXACRpRFd|UB`ta>Y9kyfN9wujvAo&?D<6YG<8WA!kX@#o1bNNP7(9BQ65Tw
zb$E#%`oG#SKIq-np%V)ExSbq0#&uve*jlp?W-_Y|!$FPWb}GKZA};TgagX;4A5K{o
zlZ#BsU)+k6-Au2gixDEbH07QGe*(Q^VYEW;@syej9t?NTM=&p<i;kKzV97@}(%PKh
zCC#qA&ZB4&x&4T1BpZ4HvRW&2i<>z@_>$2+&Hs)-^9@c}A*MW8tb^(6kRRdGHT(XG
zJ+8<+BA^8G8TpAa_>u9~`ksE0vW5&1nZPmOdEOxoPVSxs(pJ_QSh4oBkKL87`DtwU
z65+9rC2>K`(+|=kP%Gc6B0TrzFb?7?D28?xBSWOIx`H0yIto$@RE6-)c6td9JTH=r
zi>!8#DkRi3a>9PgtNIZXDnWt$DXxi@K47L$o2$r#?doiwm{{^S=G9)bP}+f7iU@3T
zm3M?;1LjXOG1j`8bJ!OXCSy@U7~8;Va{Z^bO>{lQqBf><0eC9k9GbjW-;aAS5resT
zJB919?W{}FoA<GH6KPEJ)a1pt){`@3Rjcku>E#TcnhvR~$wSjrUNTmZ{lV^lFe=g3
zqNBc?3X2BD3XU9|yO#Pm49?S}Ia_t*dCWLm1ko-(RaBLL)|)&xe)pvf@^y!M2auss
zJHr#-E`eKCMv$q;MWp*(-g`e@_-;?q&%~Y~-Ql|%Fd%LOOx&{^a<<+6X_xS~N~K@x
zR)h!)5MgT+1H*>vf3sUb6Bq&DD+)?aL0iZj9kf~Ua(ex3J{wUQXg|GQv}aj;3u-K4
z3ywTUp5=UKOL&(j7{#TQrx?$DZx#8-ldmg-2d-%)emvBUY>soT&;p6s_C~)Oocxxi
zzLj>epUT3<MOIG)zzsma91}gHEp9vT>8e;XH9uno&$fC|x9Us^N69u_xH^NDFo<}9
zjonn54AKh|#5^f`R347x+Z+pdc_Pmj{{AR!6cF&6+T$fzJPqp*5Y8<k+pUXdfysRk
zV{i7kaApv;Ek*d-uU8ZG=_img%<zq5NCzD{dL*?k4QUFLAsN^8I3oY#FO@dz@o_ie
zV8AZ!zBqNx{}yxBHN$@$D#fuAfz|>6tn8szwT|X8zbyo?I7#EmgzM{5_6Eh&IpQ~=
z?wbId7E$$Jofm@^iG?_mx_&kqtH9u{(s8$TM3;t$|MyyB2spjT2gPd4kZ^;&AW|9h
zjx1X@0rD0~d~Ma=kyj1faxXRi?<F66m<6$rQ_ItzhM$?ywJa1>b#ZnY^fvLIVk;G_
zr*RarQ(fV|urpzTvSobXqYv4gskz%_iE>r^Pb`6&u*767T!HoC1R0y(1$?>Ss5gNp
z1h|*w=(k=xiH|P!5_rj&t${l<?y^yu#phPMWWEh2k#imLp;#t3hU+!0*}1r0bpa@n
zWIvle5(1qC<_<iiuiP49zQt_I$Kwy9H#4qp!G|^yd}zBrH+_hUlJo=5TcLJgi$UY_
zd6GEPnxO;wrnb|D|Ag(DvL@$gi1mf*&^9RrS6gQOc9)oaNaoWAn=C(Uryc-ny=CHO
z1XNdm#O#{v*DVYa@e>;IRQ#8W=sYh?E9X+lQO&{MOBbRa;XBp2M@P;_(|UKgX8{8E
z)vmC}>7<3Eul>R;pWC!J+3@`3-A0eQ3p{0#=L4<Q5XVn)Up<^bDtibBR|LnNMoE4*
z4gR?30wf*`gjoOd;Ri-2usi(TWGFY9BP%I<&7=LrbzVL1!3ZWUfmRx^$7qRRdJql<
zBcFlMPRBcI2$F0P2V*+JjA0zt+eSFIy$}Rd?ZSa^3XfRg-2dI0i9@RMQSQCRjc!k)
zzEu|ddKV%u_rc&*9i40VQPq%$0%KV5rgdB~%13|kb|;=H!J^6BP4h8cwg5(z`ce(a
z0P<sXYYP_r0~s=gzeE{7SU4j_4#v!xMv9mT0TR{fQ_vTnG;)XOQT0TzVr~z#(Jovi
z;3B}D9u0t@+6kCn_aB3RTQDd9r=oFwQbh=qptFw1Rk9M4>mg8w(Neid=>tCUu{Iqj
zc#^0$s(iP8xg^*cZ1eyLaU2eM_Ev~9Jp|jHTxUHlXUU`c@({q6Uzz?~DGllQcMr%<
z;j7~($RFWM%{nDu1=P<pITUw}9JIczbLpI@`TtS%)d5Y1UE44QjBaTdT?z<@fW(kS
zx>353E@@$ODJ>}7proP#(jcj{V$dO71Eja_#^-t8@B99CY~X(9zR$VN71^d7Uv<0)
zn)8Ety=7JlMpc;oGEgF&bQ3s(z$vs7G)BaAl~JHCr#3e^6Bo;+2U4XZR=K&FKU$x-
zZtFKl!|03%mMq)N0`1Gx$eeRPKt+ArPlAexy8XSZIvH3~ZQ1~Dw|K6us|H7%;)P1Q
z3*2fF%J?d+^K^2-(A(HiThaC)js@C_=)@D(o@lH!sx~9A%Y6dh{3N0s#C;!d#S7u=
zDdrZ?-e81$u8fF-Jp7EDT&ZT^&UHzy?DJ#sEcm>BwT442BL+*$_<z8C8`$<4{zCsp
zY@<cdff^SommKX%`+54OZ2G8XM=;>VSYq?hTrUv*qgTm>m9C#WeaEdiwHR}|pF1-!
zq4P$+aHo-UL3FItTP^(0Qe&ylv3BnHc4zCkTRr?MQs}ZYDd*GNn<Fxzfup~)UI2uv
zL>e+&KvB0gn1dbkK+_#y5{TGvyvS6C*BbF0mcit7ZkxbI7mwo#4^p1WM#SlA&h(nI
zEVQPWf)QhE@e;2Tr<ZLC0attqX4^&JNu=B24tz7BC(0g(CT2Grz{$rjT@G`#Eg?70
zbnzg94S8=Fq&}B(rlZ_;=_}rgN#(x$gWvY%&se=K$M9CKe71C;i0|H9-kUBTtJ;mm
zD)IQP$afBK;o!fnAa#Thbn?euiyqKN@xnORJM!=!UixX+XuHPy7QzFbiy)fKm$cx0
zlrrc0Zj6dnO{zoXpul@KIKTr*PR|^rVeG1pf9cF#mFyY^A<Zg@p)3d-WH#HtREC}K
zQ?f^s1NyJ5V~BAE5QrLq75U)@%n7^e-{SF+#B9_N5mB8Q28dgwX79IXRC2QJH^uR`
zSKB&X-d9S*_Nf>X>w52;Sy}pFl#S)_*LQv;5b+^IA@lc54~5y7xR-Tz|IC;=29iJr
zOcu6%MwG=HR?~9}kqG)9AKc#TvJs9k61-S>CaKc92%6g5{Mcm6XZp9jBz`(G-1(#S
zbPQ1_l*D|IEwAKf6e<zIyR$#R6+c)YzGI)veiTh>0R?e*4&~e*N>deyjp8{v<_39s
zsI+B^s=4lH+7tks?u^h?(I?_;E89m<<qb6b{p0Td1ekyM_%FPWi0L6sbg3G{jsD;(
z8EL-d{cpKcJssYw%T^#y^A*=qzy>zvYxDRW6C)7Ub3A<#<(Ti4c?8r1gexqApJUUz
z&I&-_i=E`}PbW1XM}slDxtYOa0gA6Oq?9B#6WN*i1FNoXQrXhwi;e6JbaYA~kNd)I
z+lfV?z8NJ6_%AZmr(Qbx&C{V@@m$!Af$z^2Vh$?hznes_x!1V#SYHeGZ!)Fcf||Ba
zlKm4Ro|@tmF|QsQhA@V?EJsN^$fkulL|8|lMd0_1CIdmX6Y@e@yJ2vly{E%lVOeKa
z13m5%xE))so8G3v8|<KP2CSiqx&jOfV2jg}WcqGNggW_8+R~kk^_Zq1`qO0$d${`P
zZ!MWg^@H1LC;y}SK+fC`pF63>Ko;<iv|669bhghnb4l{84zy1Oo9dnaE6$+0@Rk25
zO|aq<gz6mycknU5rfT$gYS{>NciHRxwv1M9TRkhQ>ygPmCS+-OFtlD-CLbHfe??X{
zhQxyetnXsWMrR%0cQq%s>po|FILYC4RS|GBFD-Rf_6*oSgyRy)E-n`Cj_zG&IqQ$^
zv--W7s;ClQeciiQ=1e&m#y&?4ZV4>F64Z4*rxSOT00UNM#-G=6n(z6HxnApS$?@gh
z5(%w;J^8ChAAm{|>FvqWh(eAa@@(rB2`1vTVkI(pr46KKKeInp-gdsz7#^gwzpTGt
z{g{ja));wrEE17{wzvAye!#<DP?yN?vFCv;^~JgN)w}`lFGxhXfYdo*AjY_ZA)?_Y
z2L3cWz>z@fTe^ke>X6~WwB@wD1O{(j5-J0maXanH0Y;Q^85G^VQ_TrwVShT%*2@JF
zUJ+8QFYH0g9gZ{}MEuVa2FBBaI>R^OER`}h0H=Y;Axq~A3)GjR;^f5_P(ydb$W0Vv
zx-jb(qgN-fbTD2ri2Rj8{_mIyF3Ut~)2@RYae`V*{h4;lh-ihjPgIJ=caQRmBkZZb
zCnS+#7W82)GNai>yijDu{cvg>=kay>`<7@srot>Am6<QwcXQ}nOpr8-)BzlCu(@l*
z0ZsG)0KtV`{_cJJCR^%E2M76X^vUm^GW!QG23R3Jwlv=?x2SWvC*8v{VH~8<x;!H6
z+au=sUnXVz&c;7Cz%P6~rEYcU9eO_u<fx;|Y53Te1ZJEC_C}K&8GISo>F@JHjfD)>
z*xHB}btc#=nQZ(ZADaxs(^{i}c?DrCQ-si29s&;{FlL|CJuWm_a57l;j0M_$fev=S
z1tlej9wRiP*0gy?r1?8~s<&VbQX#Fw{7ZTGuW_nNiL#U0AW~dsa;!M>T#IngPJW>4
zz3VX{X}UJq-v^oepEa!|Z7$lPDU8R+3P9#g*LBM0n|+M-TgYG0qqUcvzeyabmQXZ9
zJ>}1GH0Vu}S-g5F_&xr8){iC3`4y}5(SkyZvB?q;(;ND@0PZXz;rv6iT`4}cono=n
z0ypw0r>7?gG?&scj2L0^bU*ywpfn2<;+Fl%xrWg7ZFhiWei1R0>7kuw2j4e5%GDGB
zF@okqgn9>j*t}Bl6=*+KC?KZC><BVDhxT*+u`8|rK-q!u;k=J=$1CP(<?2jvh-d|)
zuCiY&z!CXS{yX1eAT-)oy!s@Cp1Met{|hnEA+Od$pA3&;F&aDfgdZnj4fupZqr<oF
z9V-sdj>_m8=<2*E{!hdEZ~F#8QeVx{;vhB5wwT;0hlRtH(vevCZxdTMOMPD--%)Ld
zny`ZxlFiz0I$e>K5ZO>_F@{+<*QCI!{W*-W${=ntk3S_$RD5v4E(oD?fG;#QFjlfx
zbTT$WcxjxLjeZ+u?8;eTCDGf`!-!SLARNjucXi4vy1tCcR;Ld%BEyGc^fl_7_HAFy
zBMNQtvn6f0)A9RI;M(g1z3TS&*iztOWA8f)>^cbhnS7FpL|CBpz4NrKxpUnqt>9<O
ze}C9#V1NZgeEp%nXz!8!vQbTf<SKCYy*^82xdU6YU6l2f=`GW@P=-6Ml~YGr_jbfi
zL5XT=%Sb)Kwy@|@7z3Es$&Br<+D9KrWigjN>1(~f3~Z4{dLjL2#%M{7IqoAF*{WeS
zM=FIarPhP6e7+iYjckc=7r5sw2AK5Sc%@*QoIE-68zh`YOY)`Hy<5ezydtk3EgAjA
zXnP=IB#uYHh4(o8JcOf-x;@~C7H^mRwzQ2zHHl}rA)N@UXLc-fhvC0R(kLt{30(gc
zIe$udb$V#Fkb(cYM*J6|^V<%G&bq9o$0#Yw53-$m<MN&H(K><g4djpI;gS^DNFIAy
zK@EK-%m&%s^P#1NF?pwX`WohwW}*C^8~07<pH01w(38AG^pSd%BC&?op6lql_H_vf
z3?ahTZ3ipPLA~!d!4b>J`mzU-W%4+ugdIO5Zkp6%X{q2KlU79Au%ih>mcv#SRSH2;
zTTG)zf$TmVJ<OylnZVdy88A*3x)TMXB;SoAIti@bb`M=Im?VIAT=UBWxzh!lfS}9C
z{o|p!Z+vGUD7YYYi`Fs(TXW3jmBJO?aVBMO^_F&{HOi$38D4YgE)9#K_{~%=JHEv_
zly)9Xa}<fdkLAz!!QZ>W9O)i=q5NzqW*?-{k%8>G^7ehq`IZoTT7%lD$F%4^?kTM;
zM_(i1&`B_PrEwigHG+ebEP_|Aq7PVWOa5qdPTweUfiEzJDWkUXJs}~--;7yoDMcUw
z!1#9&Fuip9;Jp`EPk)M{frvO3?6w$w=6`h<INl4FTN?OEhc8ZEA2ek6#Kq77!|MgA
z{e?BLE2)V`atDak6<)Ug$ZFS8wyVh-PfRhFdkYWp=l+T(yJahU*g7ey4HO(F8N~E}
z9X^MXwkD;n4Mj#g2gThi8vfL{8hz$?3Q|{lg*&6Uq@rWKd-CRoD`B|CdD|2N!)*cr
z<Qsah5{kz|Qq!<DPCvau6D_mhMklUg?r)q%RtvXpmt@bKN@9+3o;_XA*J()q&V~AK
ziXqspQ$a?kEP@KFSRU&o?b)AVI%PHX`g_KN=Q^g(`qMEdi=t1?=RZ+pq7N_~Zl}o>
zX_zY_tA182Orf#KIgc#meG1<7i;<Ay8Brw~Tm_zT>i=DTMF31}|EI>N62);-@>4vr
zG}Q?mz}KjZlVbUxh3)rk<hL_RjCuJ}$$IvvI=Aqw;c)$jA32Ld!|#41EHZ9<2-gn`
zaO@p+AKkcyw$&a`d>0-;_8Lh`zaag_nT##oZe8r4k%PBU`3cF-@Q@4^uUdzrT2ZS{
zD-llE=3eL+I}#`CycNdFz;e^!ruw9GbrQemn=nO`w%a%*3G@aX{kA+@!w7^iuAZ<8
zlaoM|8P9rRijZLNCP>C$nPUl*(sTnocl~z;Y;NaE;51L_1Q+%x*K}^m_up;a7vE(5
zi9BiqVTIP&cRkuC6fQ3IZmF}z(>J<Ild4t~MBXO%T;VAWQ*A4VZ;j)L{Qzd%KE`$V
z&ZKYjB~-Wbp4Z&h`19wo1jj{<C&}+urJQfmeqBdREdBxKIlyaQ*3JPtB4LN4_0H9I
zK^{){CCE;h7syVK=~(1qd^AR`>`Lpfe9Ci90u3reb{?iLy$bsa*vO%HjUMxJNzW*K
z;ZvZ~s3pe#z7Sk!9za*i4BpU9Vz2d3LwD=DV;6wZ=p@nK-h`o;3Ln)x^lKfxs{WlL
zN$tuaOimYv2O*ONku<Al+vNH71?w!K>Le+0t`6b$;g`2}>`$aBLGtLO0LgsWPAsVn
zys)+)1O_2Q8keKV5~#;3-i)^)Lgg3mO~3iJUwZ$DykW>4V8nb&leh(&d+P0z@9%sn
z%7WUvB$P6Cz%|~tMDo06%V|g1@-dp+<WlzR7s9I6`7f}X?D+{0!5g18X}?7c@U(|r
zUEKvKLNUAsg_FC3k>SiDY1)k>T2*8Z+ED<E9@1&wG(v>mV)jg5yhFYo*$1&@4gSJb
z+(##Z?4WR=e3c2vZvo_E-*|}_v&~hSE;*h{>ou1yaZ^vy{tYeJoizX43SNeVPCt}k
zfo7XBp$Lq<2M+-$-%DUw9wAS|Hc$SD;7xFgnAX|jD9)q^s)LNu>O~jI1ijFyTJuN8
zSr0zA{)7NBjX^$+#b>HeNuBez!=B6JYsVG%S0C`P?ZHBw1lkn#<wpX}>~Qy~f((o)
zeu#l!sKaGl2X1V@G*JA>Ynj`L;JhSJAW~CtV1944hYkkCt86~M_2?%8o_`kBSxpvV
z8Ji$ox>5%jQujsE+|q+uDBm@M-aCZ<wd1Y#pMSb;*SzM<OQ<+SSd-|MOfD(89&#LU
zlLtvTcfFf3fZ2Ud#FD|H*~3=z<0})5Kp>S13k4pJf2QpThqOWM_9GDO3Y}q=1!95T
z?TjVFs@J^(vYH(}D)TO`P>*^gc0<-eU01ko7{=Ochl{)n#|mTvd5+ceLm;Rsd@h-b
zVSTn(uiKlUR!M0HFinIqG|3EJsB5c~Y~6%PVYvf7F1<G|ppX)GQpFDnTmT=!%DJ=2
z_-Zj_sR8%%qYZl0KGo6o&q;kT`ypyStDFj9<v*$+Do+Qwvgi^3Du=^I5Q96RTX>Rc
zO5)3Y1c1VHL-7fVYcO~JLRN_csz<zTwzq3vDQeW>yTO$CK+1af0IrFedX;L8<!>i7
zH%zpAk~8sp;TD%xk61|E_XAYzF5hY+&1#Ul>u2QV&Gq?ub>eu@5Y2ANguxrSEKqMX
z*-*=1X_y2a<Y0Z?Ny5vj(p4eW=~of?U%C7PFi>3OJqYF$A%~@7r1?WF0bfO#cqbT`
zB_^Fm=lIe4%loSh?u8T1Pda`Pc1uEr^Nq)MuEvA(B@M8`xfgKtY?^(Qv7LL7YTFLy
z8wF~yu-td!HcbWkw@Y0+9!O=Z+UNwpJ5*cdnEDb}jp)2{(A3x3C@`w)JGQ}Af6sRA
z5s6O*0KQL}kL=-NHE+1fxKHbvvZL6JIU(N&lERjH39!i^qb2y*X)dR)qExB}Gk7hS
zK9tPmkF)i>ie(9couBOOc^1JW&-cRsCuA|#XXJLskH~O#zlXn;M*({4gL;NRU#N6}
zm2lGyJa9_V*~7ze5)aR}h^$QlD6s=VP&8fxr!?IZs7c+5T^cdEakm5guRLi{j!xUZ
ziSj;~)lS5RK+j*Q+7{73)98`T0rvyP*<&klYo=edM~|28&Aeyao{QueKaB{J&u&#)
z$CFAdNVuK3kp`xlCqDOfs65>30g}};5rL`_{1VyuWQO=PBhUAKj)K$E%;cYuSl^sk
z7Ex9Ck={ztX>Knwp8oKFhl9hV7#}<5c>s7vrxEauvX_1}7lSAO?a*2PMC(rz3$!U(
zJ4ppNz2&~q@^<`HFy39G{FPzE;p@DfK8>IO-3QlMNa<K#k$kB1G4rh_JrM<ogZJW9
z$sq39GOmG_*P=*gGB4G<qB{h$_Z<&JwJw{_rq_4Zq0)Q>IH6sZz}ngq*kO654oyb#
z5zdTgI*FJ+FW|)#?0|LNP$brS|Iu=7Uv$4fM@Tb@f(1&MuFZHnEci)-j9hLHSZ>mV
zVy=Q_8Q`tUd%t$P1DuB*ls^Mki~8kqvKYDW(FopTfyx<x4BWt$SDXDcp@}8F=7r~F
z;td%n3DiPQv#RV5a$ES2buR1E!c{XoW+p(FI|duq%V97Z6mK-b;U83Rc?aK4ECTVG
zW5JkH^Z0;sBJ4J?D)bikuAoQ*d2k$n5x}6?dqVpfPtb&3s{+Q`S!fCdi407#!{^?7
zPFJQA(R-0hv0os5KCZ%w-R7Yo=DRsXuzqzLKPc?}(9VTS1=rTEu!COxizyTLbX95R
z2~M~|q{*VU^g@c(!&}C+W}H8%w572__uHhd`kwPp^BeFAb4zUm_uw7%QaMV1d<L05
z5*M8_vCU3_ATo`;$af6a=1p{W-OY@Css>-v-3#}E&*#X9jzYEPFDUgWkxu|p9Y6}0
zmE+$<M2HbrWdS5Obcz#IB?|t>$m7c2$a7Y8TqsT8Co^R1?(SSqDgWQC0v+q7@;LWD
zb)tdk^&9oV@>l7q0Fo6SyUX#N^DVdB1OhI9hTeKn{uf=nxFXy$kQ`QuM8sVJ>I8(@
zVYmnc(<65HoFpPn$>>F(yY0UXtsHT%8D;N%ZI%2xoq87nEE=SIO(z!}Bc|tf?mal_
zdwRz0LHWu8JUlU%0DbDQ+tNw#A7kus`#6ht?c_|Mj+b{GuswLu+jKZy6$c2odJGmB
z`<7wm4pTMKjn;nUa@^SDoAOo@hd-p3&-z6e?znH?{?3(WD0MSiI^~C1k_k_P`>d42
z4Vl}XPN7CFr|Uh*!Y;hc>YuIgATEROA%#_k><68Zo$w>LuyQat{IO<i!R^|Pa-!+5
zVO80axQk_0i|otd*oC4``sj*hAG5LPO{`cq+GprV)F?@mpB9rug9G>pw7fXY>%b<q
z6Z^fg)9Ym$t~U%*_jjJG(jesiykC6&j@V$O^h#ht)TF$DFqOhoHw}sCj6~c!f*0<>
zW=L%_d%x23fkYub^K$1AsW&98n5RI0nO5a|bK|iFXTg2&TKXe+eWo<b^Inpn$U|8Q
zF(**tFHU7(x$XyS&RU**+LA1!Wtnt0Tn0}*{R{*;BkRsiY$MyrxF4tYRn75G6>ZWa
z>+)o=L_#gFKVR~h<2w>*K4PtZdy};$56zoE#4zzI4#-`PIYZqkPFvF+;V(Wp!NaEo
zHqT=RD?X&+K`2lq`^jtPw)kpxs;W$3`4|pm;@D5SEyMeqCFDUr46$dw>i&4=nEZ>^
z_4fy6pMycN0P;#pGNYSdyruKDz4T4PZ%$vT_gYq#Sb`nF!`Ud#&?JGh(n|ABg$FNp
zn&od)T^o8->ES`XzG7IvMBRI`Z6$s-r3$E)5q!PB6skpHgt)d@xxnam07qmJjC6oa
zOBco>fMVXc@XnEZP(#t`WSb{Qd+1&@Hj_6aG8-7AwYQ<uStw5@HX$GQ2;<yT3UBWK
zV10WYsfh4(Nkbsa2S=>GsUF=hvHxK}j3W}I3cIH5M^ljFAgS8}Cj72nb?0B<7u~=C
zVnN=ohl~(k2K4E{^4EHhXkRfWn&!Bx2w^t*UXv4S^^I7XZCW;Z`cEb!9^{#B^JC!9
z<`u|asL;QD1rjHMcidyj!G@gr1|Q&0P+<ec+Q?QKN!t(s4z^T7uq2B{ixgE1wlwEN
z74Vjma2Q<wGESkZURkSUE`4s<;=FXO-NGtAn-OrV{jgDjW?-2(7N9T8&t+RE9Ynu}
zEs8tLxXaI+3*q1$odl2rZZcgGojl=j{Gi;r=&7c{vg~(EVl+mePUCRfT$R<SFUhnB
z_0jD&VJ!2LY<l~2ZcQt&+saPv!!XYi>;Z&HAm8kHkI!%G;Lq&KTdeNzeLR#NaeNgi
z!6xfxfr6jG%j*YJ%*)!!*8%1{3{%v)U;X#yYHF{5y$i{`iC9M25YG4B!q#(4jCz@I
zV&ns#4`Km0fC(`>)VBO6CjGTtwDn8H?DD|CC*w$chNgw+FAc!e1ShbeJ`x2FkLl3e
zlKX<N`pU49+I2}p2T(U8eg>!v7H>Ci3JmWVS3;52Y@*QXGnLr0a@_(?i&XUCZkWHL
zdWE1}srY5u4X7S1GRq<WTRnr<8ZZ2HWw>QYX%>mGMJUbokLJmD-<KW|tJ)>StSc`%
zWaNSycZR_1ox1GWPVlo^aubo^Dy2NOCNY8JaDP4OrA+Ncz~XwC=T0JNCj0&w;mr>4
zw+`82O^(wl+WKZT39T65mFqTupIK%*bYz!-!HMQvOw_`BGEWNHwQ`1s4EcNJWEROR
zooo@c>>$);cTvUYWuvZf7v*CvnWHD=MmrDpul(n8ih;EvDOifzqUgEk!W0|{-lvsI
zr6Ab-hk`}0C+Z{4D_c8!H{RCs8!8*U0RCW4?b`eYozFm*a7w@0dukg<uBwQhY4#}s
z>zVRF5KDpI$aVxJLX~UgbVcB@=8*7?pUJ=ie%X`*EZ)htuqM1pixK8)WH<0CD`MzE
z4ltrY+b-u9kbfX#_?EL(_51sq8GkRt8tSb;YS&oJh2bJGD$rT0m6xg~JVe&nE&!~i
zmEaUYkR_SIa4kyu4`8mhi$l26!Q!kDF<Bts9F!C*9F7M$TVNl$P7Z+gQ>3irn?{|s
z2w{yq@^i|L!^dO(!TOLZIs)-jDV%hpcydqeAWvbs*4+AaS$6NLt{3pI8ADo@ZhuBn
z@2VZ#y1CMb#NxXGRNA4#k&_M#IubODYR}$*AfT~EzFL2Q6t5o4{>oZX1RcySDrJfY
zq85c<3&A{{U0Y->zP>NBfNHSu(~+J8=q=xs?QLGk1reV*A3q*tOMYpsCFRGZTcPz5
zKcyu`>UXNx*2Kclo7#xHQhi#EdUPxs@NvFb|L<!lu3l#ai)V<R10YbU2;6RF3xl&n
z+zUFQ6Roxeqq8@ZFOEkXjWc51`6ccz?m^VQbumEn${%mCc33u_w`|o$Iu-<rQ({P&
zr8<u`CS_)=$RK9f0Enl9Em#kGwEk@=saSbQe3RWFxAGE8G+9-f8=;Q7cTUWGsmL>Z
zkKU-(q$=U+^un1>E?~f_X>B)HgJ#FxS)0*g;5lHK2vc11mpIZ|%~Pk}1VDyQwm$Cu
zXux^idv!iy{^|bDu}kh`D?oFqt(ZH8FYt-eW0-)<nITBa)OK?x=lL^m&xfz21&aud
z(($o$ur)+IRMTfx4Nwdc(V_I)EU)h;Y`IINz_Wed>{F3SqN0LXsCq`#7;xYZ@3d=m
zW`5$tx1AvRHM6Y5LekjsH_ivQ3;ZqK<gSmDzAu;?ohV0nC`;Moelh%H<G{hr`OY!L
z1K432awk_OBxe`dvKQO$H1p3HxeH)6^spvkUVBguX8IjneCzTH0+V*&iAD%Q4dd7>
z8@Sko#9HzW0|CFSim~1N6bRW69Z#>q+i&{%Q}$aPl=M;GJOg_-k57th%N3e+q0!ew
z1fbP6sy>U<>fHPNvCe<_S?hrL2vxG&V#D_palrTBno_6C8wpy~4+OaO>lMT^K9l8X
zT(tIs%1qAJyk{{ul!{;FzGcHWRxm)KaIHt^8PXc7Vx{8NP*r^jOcgKXsk!De9Z7zv
z>xHr}lO4l7S11UESusZeg$F)`_+d4kZJ|}wPnr$-9p6h5<*R(ux(0MBx&z1yC&(|a
zlm+_`@aNaGy$Pa<k~ne@QFdQ7vlffj{9HI&%!`uYCL6{0IGJWYFCf{3Excx#7IsD^
zyaUM{ba>R2RUDBtP(miiK@bJ`9yYG~<AHFn>4IZJy(bU|VjRjQ^843v624@>r7peX
z%D{#}s$x5%my0i?FUr?jq^nawl|nHsqx<wNvQ;9T&g?8Ye(+23H&`{)FvuAUTdMGD
z>$^U$Gn(%r44W?<^9fe?@^SJY59UTTX4OrA&?cpLoHgu{LYcu-84v5IUG64Gpukx#
zq#^~vGNit?49$9>1YxP_9fvP;>+>CQ>AYNbg68ve2V8KlQQpG97(A(kz7hk|PBE9?
z3+94Ba0!szO4zmLbh0+*?G<}hGo*Y?Ctuv+mQu5r@gR~1=rC02noh;84$vBiNxwSB
z+VrSz>Hk(KR!v&G8bNo?%7DY}8&}QwZNpUuqxNwU`f^<EF}U>Nu1OpNTxUmD%JZX4
zF(g1Al<EXeDGYf;8ygTv42`3WCcnhNgH%pwTf}s)bgbR0_XRPx2n*;G-!5-YAJ}VZ
zse26nl|Y!swavf9bK|PY8!7JsBBUbVL1<8VA6Tz09#2~uLqCMn8sENCPA>A)ysO}a
zj)2<xoAci>%SQal5dkX~Z6-gZomf@QL%xjedSqOJflJ)>u?AagAM2)kr#N|>!c*Y7
zrK^i^oYh`CRFb|zgH1g0DnPYS2Hi6*aRLP9P!YxbSl}dyvxAZDN&}?ON~0PEkPXzw
zHmzsSr>-YEf#`HrV@_H-7Yz(>1=XbZ=3V?OMm<ZABt2gVhvl(`ym{EUYR|mD@1l!4
z@|V=+No1BEZFJN#;VCvSjr&T=69GQsTT1hr&bsL$oA0MB$-+F&?k#aA5K-NgmF19G
zS$=9h^~P9xw(&BBDR{d9gjKugYghs^y8wN{q2jQK!*ur5kIs_}e@6URDaUXkeMV&Q
zr&}K$-{A2`b0-CUfgjg;ao#G<esH@-xtS9UiNcF*R56f-{QBszXMdmL^TBl8Jl4SD
z6r5RCkf~}t5@e(2@N~C%ZR%^h7st%(oHwcMr9k3zH1+45bow|pnSJ<!29Ud0^_4qs
zL5JA9B4xaXgG`Rn=h7Dy^8T^mS3leM-mG@31X`YH6Dg2sO==YSZfF;7Ty)Vly8r6d
zosgzt;rubw?8!W~Old<8U_2n(#lnCZCM3XsMN76h|84!D$BQP-D?Ab_$smofY2<_1
zCXJVG*KfQPy)g!xK^)dO)Vvz2&}P+F6-=7Ko8&9n8a-u|_Qc8$I(M$^k_fn}A+ifd
zn@Rf9p|q{`3*~k>+ckkxJzhi>;biMMrK{2f?ry3yfY)}uS0hU*UfCj)!7MY&nXdVD
zyuDXSgR-Dvkfw(v(cSJ|KwLG0h&+x}?T{zz+g<q2WFh;H3@H`76Q$^DARBC}&%>S|
z?lgvTwj>I+3=F)Z0s`Fn1OM$G1Z{X4qMFwzO_Ad1@yD=?ly2?C?k=jtrDn9)uD3)#
zk5*@M)umUFYAaLjOpCq1QPW)8UO>x>QPFElH5t0jGwl<Xj<P`6SnU`daH!xCBLn^o
z*${C)yQT<)1$#<_(2#fc^}&Pkrc7|XZmk6Xy9QpDLfZ^l2<5_x6LJnR%26?tT{;)=
zOMVx?rK+1kjn~udb8BA-((aK0hfxbd*UdLdl4Tz&YZ4}vE@%&}A4FZ7M=)dcHv;>b
z8)ksnmrK(76bDJAC4t=zj8U=79f&R~jwTj;uQSYInFDJp{I|f`JC~m)npZDrh<D{Z
z63f~cqFwRg+8)3Qp7IoiNY?HlIN8Gv_&c@Vl6zCLtIbiXJJoc~Z9Ee3QUj1Zag4H-
zFT&1A2X;>|qeWWGeA{VhsVaQT_(8W=+?#RzHcc?R8W=^sw#tOwBM+F$L|B3f=GKeL
z8@C<;5fsG7^eN`bU*g<JjC?=J^C~M~^{(p=RlbSXXP7@Z9(X8JkM-Z}pTdqKOe0ba
zFED^jUQD`RrtdiX(a$R`yiQ5^{4SWuloOW9kLm#7FsM)=cn86gn#?N40&Yyv>p^V7
zuk=a>#GAN@q(32n(vc*EP8S^geQi|p_*bRC&bsBq+_S%;OsY>ZZZC946X|+*nCgA)
zNMN(54!8+e1x7(Ox&F4HSAi@(lIECu`11rOGgN?wQ?gapd9pZqIV(Zp_hqElCAjrb
z6kmQA06gpZKMew$dMX5N({cIPG8ES`1Ua{Jq%_^Y5k>)=tt(Sc0((EsKPA#$mTw03
zv3&}5WU)Z`phl0E^4$FTt~gyypQaApyN2aW-aUEN682IMm#zY!ZoG72PAOINAS+z@
zv@_DQ9n+f_=5Qo5oOz3AREsY8&~l!KXb~$<)c>G8S29n^uZoy34Ww36K&bMc5@z}f
zobKW@-R??b3_6Z@R@v&|ZZ@J0dqn<(z5+P%CKaygL4J&DKDjnYKwS-WZ_9_T_EWB&
zz!8jJYfh~caanQByWeZyO8$J;Xx1|9s*JkKoZ&{VrGOR*t?l3P9QOzSQcSp|$tNTU
zeaHNLRH^0P?7gbf5R(3r*S?eA;f`rwnmjiCn_<s8E#t3_E)UHztu=VG#&C-9oTxjy
zj^PL{Qh$LK*1M!f(HY+mgeMytN;(nmYd+wYKusbz{%${^y2tz}OoP4^t(hYhJt{}m
z(pNGmJJRc?Brrw#C4OjIY%&U5w7<FAzQ41*#PNEEFpB<3F)-nA@kHcD(L&Nd{Iz)-
zTwpB#$89R-uD>=R*vixK389(q`9&F*x7qw)#KHEs3+Q3l=~pG3Lx@hdftps-OH-u~
z)r%v2?W*d>JuBHoelt{iqqw*2R{ymYz4RQmH=u;#5c)uh!o8Z8b?pp+o5o$r+R8$p
zhJHRr5VrFUYvCZ7?hs3CW~>h%U!<)|RZ;vC#b=R&>DmnPu=kn~mxv(T{ZF3bu1s*@
zgnmmf){A@e!pAe$?8v$bRgnmUJR&?}Wn)47UZe9Qo-A0Ne&Q9`kwa!^-OjTscmW(n
zC-JUBcMn^0xJR+T@Kjqzi)9B`V$rm(y@Q0P{Y>&^b%>#^Vw|^NoRW}#D48{~eQzUJ
z?xO<ua399vfED{b1cDx~+N_EMXCJFS7e=+;IqtnWTr`Kr=W2p&=;n?Ht$mgwE)`up
zMp27d{NTIZn)>J&-yg;3C%+V^2ZYOvpC-Nl9M&$Lm|tMxC<(#jcflSA-*)>Yc1@RY
z{36W{I*A`|k>LUnd~M5sW8<qx1`#E$pGi0!L$-kD>b*+WuT|E3)D{PzuTBpO`TMoF
zVh<FXHKQu`-rX+6EMD4X1|9!3N%6mA5vCE19bpXY73Se@oMb6R#PCg8p)Uhjgp4!*
zZ&UviHLR5FN3+*NGpXa6MM5u4xBJ{0zVKn$C_cG^oaItIDA5|fv*mL60U)Ro;>`y-
zt{-=-q&--8KW&kC@he?+8Iacty_Wlwo51qOK%(+&nw`1+)1S3Fg7hU@ou4a}0?8MG
z66F=s_^A2|8gJ}$=2m{u`Yo1GYf{9X)E`;rhZqubV?nryyU!8Jc(jgJCCVjm2|X3O
z&j;gVgDmF<Fg=%|tid5;36Bgw-DN0y>FKlg$hv2b(MGUo{c|dwW9~TrBVuT=ogzJK
z9sBf=X`lw0`H#HSJqDP`vLk$bD$IK;?VA-bw7NFnoPk$Gg*2KFyV<08@_XWqXA)Pv
zn_v@v8G{Gu`fbNxP;G*}E_SsIYaNGeytUDO5q7}4pKcuOQrAYR9o#`K12J=NabSc+
z4Z?*;KF&VLDH?T+Ac1a^#1$B4hm^<H9!{GC3RIm*;2KG~Ie_lOxDvR}CF&j2e4HZ$
z0oYA^?_On_5WN0hF94Ce{qLY4>xTymLvmhbrJFD3+1q{b0Sks;0+E;RNT8Rf4PPn|
zO5xfHn*oy;3*L2n=qFk(Js{U&4C<e6;9!8YQrf^fSB$pza;{2x0WDScq1x9W0Js=S
zwNOgm{(CzVQ+AQ2plks}D`r)<xi?3v5gFvZq_q#Up-ZV28X}x>P3;03X)HGK;l%I&
zvJ}bdB%kJle?>`(gKbY2UhyF$25;fQXPt|t6#k)E>O?X7ns6g|?IW?4g^&U_tu$26
zGb}@7R3@F~w~H(uuO53yt=v4J)#%8+6kw;KKq+Z`G2$<#I@wscN`mB}wF$kM269eP
zRagj)YVYxb2_12=JDaKE^+LNM7f|wJ-f>&B2HX~N{h_nEAyX8J$YVos0d_5Z99XE?
zDHl5Ng}+&gK1um+X};Y}_GXN|IpmT!^v-_06fMy`l6Nn&Kb&>Y7NI5M<9BZDyQ%sz
z39VxpX<)_YBf7blqLpVn`bBr!lO5y9o<L*jo7d4d^hoX2FrO#8R7m=fiv%4GwqX4d
zmVo|RT>~oH>||I9ZvMU+Pn;Zv>f+GWtr#Z-8l}yIT7Saa0T1|Cg93i`ZJ+Y5L#0(}
zHNW0lgK2YUE3GO{b&RdAM}_!=&Wer<f_&=Ctzu9O3$v@3n^Op=7aDnRH=uv-#g>8M
zM+5u+{thq3Dn@HSeHv(;dt?e2_q9KaJ=w(jBic#mf>(k>0Jh!TYA-xyrxj}G(EIHA
zrK6*xCK=b4Bg&iuhy^V`X1@}Y`^j8+5y(1WM?J7iyMdn{uoRE5kdSb~K?Y`%u!aH3
zjEr-t$}Ljs<%$+y*)TgYKLq}GGPlAR{WqfRqjB7-t99dn6BQx3zMD+EP7Y=AY*Bx(
zF41eOo|CuQLMcT?&bjPo?gN4xwY=5M7JO;Ej!Ne={laJ;ET5J3<qzm9Kozz0jXvBE
z#OvSeKQJ!-wH&*HD$~l7GWBL&H;ys7>LF|UFj;6i$-T*S&p$;51`<=WVrQRjJ@lUR
zzC5T6&h8lpk_e+IUyDuGyX5s!>zTbD?}YJTSqFlYkT<as6}jH(l{xa>ZU$i2_oE*B
zZXpws5w&6GGv&X4C>0gU5`uNNV}M>r>Y#$`;@r}+3s=~t7Y1l8`Te+*sS{a0b3ED!
zP73dz>4iurSf;Gpj)-)FciK~@Gqs?9y1-Mkstw03O$oDF8Xlz|Tz;f`_4HG&@6f8_
znjarMWX6UMnT`sym)==6?#EKhJ7>iSd(;jVOv)6E2M;@*h89{reA{8{UN`${;WU<*
z8N#9B&;ic#*)}wndPY<c?_PpHZ8%8A?&H>VW?;;-@t&`|u$rF>-IO&bQ6@2TWP~*>
zkKg1Cn>(W|eDLmP3zpn7d~A(RM_mW{t6)fukCC7;1e?qh3lmL~k9pQ_DAC?|(*=gF
z4Ps))AH=?hdh5KTPZQ(U6u6sCf+zCo(c@5VRPjx1VjMcHK=NKy)tjKm1CtOLwAcqM
zDgCs{Ks16&fux>cwFi*1F)>a6OBwvbj9>gUaoy?~j>BD+8s-Hybob%7Cn~^ECQ)?#
zH5oRk<jvf{mG4-ee%g9T#Akr?J{oMP@aNQ-5-AZncMPb9c(46RJm*7C=tWRch&g<l
z=v?5`P!^_#UL!B3^#qw9KyCPRh;tvXce?$i%^gv?1h>iK?kf+0071p@f8~91>_jGX
zZ|#XH=&aopsFPa*h@k_*R=5CzTkL1`nGp`ZxTM>hVy?)TH;*PM8%LU!={p8)jlW`<
zNjrXUS%wZ8irp&sac=xNvxh&Sq$3w~=kfOlpj8;jedx|O+I(c1-r;WkmjR~Jp?bHE
zrK9Gs^tGDt5beytI^j_A@rJn{V1&PXjtqaW#~c=oT3uwBvUK_Vp=KP@pfrx{s$Jq3
z52zvr^pq*O_JOVnU34(3hll=UIx-tn_62iuSjqleb)}wXQ~0EUZT^=AyZx5$y%?Hy
zveTb8OMglH_M^eJNV`BbA5TNOyZC6nn;6>Sw==YPKykT^Zqk$P&0c(Ubi`iS)6{R7
zSJir(T=>#7VYn91Fdr9a2LDzdmF%tQq~17e18JHjVL~^}PU&ajDi(lIiGEBBG4xO9
zk?zWA)-6lY>-|_!ol%Vx1MJcj^`v{pl;LvTEp(Ahy7FaDp*+qw`3d&O<`3tE4@5QM
z(XRv(etiZ;dUwC@E~Z=J9<#6p?z5Ti8l=-xq^8Ob)w~?5O5D9Ak_m^Eia^Mov*3I?
zknfbAk}Z2wOWKs)B{Ye23dAgHzwt{giTgq%=>0`CD`tM8@y?L&4*{adQmi+>Bxi*K
zZ8%ZKyZ8-!W|j5cu_o0fw~tp5weABF;TgLG6?Q!Z(yU`@RE=!K>5FQCIa<E+Z~YQ|
zx;D-s)SrdS+tI$1xJ>nbQq97Au51NsIa|@GC)f|yLP|OtP4^6*cdZ!W5jmfDeAG24
z{6mw6mfy`EIG*%4U0&WYd?PIf+vsBX6v-Z~dZ)wHB3m3)$rL&GhWLi^fN}3->a)}7
zY71uNWQH2|&_C!3tOt?b<mzJ3IQR`y5Sso&P)Q^>r;kC_t@3>vFq^7Epxv#ou`2!o
z6QSX>!kh`*>w$F-7&@VOMe!ibkLfynxN#d+Yad+K9WD-7=J3)@n<|4AhX%seB$o!D
zvt0SvH3)U0gxBBCK3M3hdx?<JW_ZdgSjyd9@TksBo`LyiaGAMNf5T2zMJNCO_IC_@
zy*EHKmWJ%jRM6IP9`b~gn=;QAcwKV`twCx#e=s0~_W+lU?W(;5LOTFE*`o$bh@tF7
zsf5@16JCoO3qvQ}7fSOdM{Nba*>F$tn}KsJ3ZSPcvefi=Az+wfkx2<Sb;wlSm@=I1
z^HaBvqE=J&!Yx7i|B&)C$t5`#%11ktpw%q4I!a$(s;e;L9_zBk`zEPQ$0f0;&VMC+
zXk3-jf<Ag`g+X4`w(F*?vzGCr2g@-|mj2qWYJDQM<l0oF03Dw8IoS1;Jy4sIahkuD
zw153&irG*~pwpEm)5PWxVFOM@&wo})4DQL@aWRO!eUfZs?zcO%$&i9R9F?n+&uc;E
zFZQN4SSQ6=I04$WycPAsHs-Jpw-e=Pndy`2Na3Se>-x<c55IU*T<*o^6p0gK$&_z=
zTnG*0zBK;Cp59O<CF4nG*`Sl1;wbl{VlSXTxI)8ThPC#6Y0<qWZ5qZvHWxo$?B&j_
z)oW^@#l*;TxQ_O@KilF-7HC&!`7(Ona@O`e_LzSYU#sna)$U_tf^8e`1HvAxb_NtY
zE;Ypju!Mkpe8;~N7fP$t0@^0Ue{B<xs^`<;?Mti6yEbqAGYvz_hikPKD`<4z2jakn
zfT)8^5%55fL)j@*yRLv8X;SN}14FX|aYQ`fVxy2bR=tmHKpr@Kaj}q@l{$gIEcsRH
zgj)3ne=Pk)&gar7)zunVlFSI+u|0FtA5HZK4fnyqWyW_>bAKgW`;~h&fbiFk3^#!f
z(z|eCc8OMCE9D8yq`bu)ZsI|1FQ%%?!<)%@O(_b0=kCky{DKOvUjBbjN0x;*LB_4+
zJEwL%SSUIKkIR};6J{>B@JWr7vSVaBGnvFgKZRWe!ub$|p`{mEj9W#gQaRJ-JnRLv
zX+B<G?uZbaJho0<pN~i9t<O6%-)p-+RjecBB2~9_R6Ner+NiNz%bO_@vK?$s*FZ@#
zfB6P5jc4vHv8*H;6ZvgA&5ArI(~s>4q$C+}qIBavia?+^gqfaUZ7!-|2lXro-gtP2
zW3QR<g`j2m8|vtH)V#%o85pmkDw~SuLz3|gZqa?8&FT)zGakPF(DF=wH@=#9JRu}X
zZ$Im-VmjhsWust#n)S`C(Lx)m#i*G0F7^dsx-Yrj+IIGF;<0;jB%9NbpT=N_pZE(3
zCT*W5<ZfownGp1-B$ZlFH<AA>zF9?#bDHpf&k9PY>-G8zt|pk^t3b}{^zW2l!9hyj
zAzqNoi1^_>&2=rzzKld%1sz}e{M3hhCIpv6tfLh@JuHLYkZ`l;Vhd!$Lz=t)p->Ts
z7yo@OzAPDbT#|$<B$GO#MEGGImaG;$WdJTiSe@=K3cLOg7S%(M0zINn`Uww!f~2y6
z;U=F2uxbOAu-z*ql+D)gl>B^B0nZN83R3>ZQb;XSjZP7STF4?dG2`w<P?Ox%R$YgN
zX>}AGtoOdkCJ6afSg2RWGPfYxo04P1Dz~7t$VxA}hS}95M20UW5`|*k+K($hW6UvV
zBD)1;797-&aCkppPMKHg`@fx-=2bj5RbPxUctF}c5WE-3HT;6VjJig1X+6W!_Dceq
zvCDSjms2LK2mNAZ&-vZ&O5Q=X$7(A?Xc5em%&p8LA;v<-<f7*k;TB&{)d-&i_?teT
zXYEnpuGL?gp;><pmnR>vgw5P`A{TR;FsY+3+ii@O3?~9vesouzk^5(~1zXsU-XQeH
zi@hW2p5MqZ95x_5u_9qc6sn)aJh*i;KLMFZdp2~lbK$YA(hnH;)QC4pFETvlBt%ue
zTYArhy}IL!7m0Ude4^Z<e9o<De4C`*?1*dTZct+|883qGLdjH*0dN03NOnw$>b)ix
za>D9qu{TG)7+d1D#H#<tk;eG?+ta;?gDe#$ULeZ;f1d!bZ79-@B}KS@3+qldb5lTq
zW?h`PQn#%oXc04%L|+erjW2N$k`%$=fFix_-v&X{j8>MGC^l&w)5uCDDb}Oum}#{t
zxDFP%1U<>^tVhb(E9OU)CC?*3oTI4kkuM7Uxw3Sy$g<<<J|NEdh>MJn;9yhEf$;BB
zu%$x&EwElxk+wg-JRHSG36gsNI3^&KQtCS6S)!F6QL(Ye{PUcsQ(&iO%loYuxIrWd
zR2%|<0)jrc=JMR7aI)a=7x>1#cs8f*xX-|V9>N(Xa$<Ju{O*aI1fJh!``GG8#riyF
z1^Ssv{CIQBx%c@|Wr&{`!*~Zu!z+{GN_4(8^|)>gx3*wr0t1YX2sWokbWrP$c{xh%
z(^;d2KPOsGU7g_bxZ`nsM~#N-y#|qo3JVzbPb69{<CIy1Kexw4$~J0yIBfg7!Bto-
zQgqkf(Y|uvYT=5Ufqp%@|B?x08rf}XlBDC=q_Uz`mpaSYPZak1f0j6i*c2!A<o;ag
z<T@P1?lv#Ld)e8t3^5IB?m$0${}?yaW25HJo8ai9ymOGNgN$Vj%@EQ?CTu9t=Rp+`
z@QysQh<~Mt3`7=gf4b@1lvpv@vEDJ2iuHLIoBAFT<^c}%Edm`VW-@UD(;bUNEm({p
zDqBA1OWI}OQ)ao;o=+Rv#QRif^U#_SaNUuJ;n2M|$fs#@<yOQd$}dMLI%Tl-a125a
zYh-#!82rE1NrJI_*PQfulKMTls}6sjP8R9e(~h0f4w_`nU&QrPmPwO*`bXN*oOVLj
zO$VVUxhg-np6>I--SRiTZeu{Lh*es$4y0q@wphL`!kX3I5f1&Mv3fh47)O;p5fVsh
z(n2ir^S_VnJ@WrtMqieq_k?K*UB7yAziYvEMa#BospBF6qQLicx>C0%=|wA48f<2{
zM}d$-z4Oo49GQv(5V}eOLRW`5QyYK+#kkI5AJ`@-PW+~6^v}KLU;9hp*Q)hK59vLP
zJpNEHLEydyUa6#aXFPF$xi*&$?HhO<kWH$B#IO$yE$dZaU&ff?h))2Rx{8G%ZsEfg
zgskWDa*!`g(yf9#5A;3vUZ#86z3{(QqXJa<0YeWKi%LiPM|l8wH=-Ha#5;|Lt4&XM
zH>bW@hJ&^Sets7gsW)5bzsKK2j%PD2HxzosalsBlM}&25e>VNV?=6@15YzZ=qoz=s
zb*if8LZk|Ea24D4!ZfcXY0u)XdRikq<vwgCN^WFScI)Q*%8h&H@V0a>@jJ^r(l~pq
z7Z#pkNJfQ}DD@c<kOJ;+s{1_RiE;5{Y5}|DP6ahF2pUZxKwsi6AU_WVAWN8VUxq;=
zBG}J+kXlOM-`wob_HO9Bu({^3!<MaY#}cYf*Y*W#{_vHrja*^%$y_z!??t4seT;Z?
z(B(b>&Y$S(^G9*XYg_@F|4tvK9E(O9%BxR9gYMCWsOL5}l3#<fu`Dy?Sf@NBB(9tQ
zYek4bE)G_fZpq=en&H;7Xk$z}V9yA8x0J=KyuO3Bc0in5RlNr*M)rTZ+A|{TzwA1t
zFb4N~$lU^0D7oPW|2+$^N=|zI@8r96eb1?ath|Gvk9kp}l-h)js35X%Ipe5*#}k5_
z?wLguJ}Hxv@&gNqVB3)%kP&fZnL+A6ysD%1U>et0y|qGmawY)(dy7fleA+0_S*WVW
zCwwDf|82Z8<J<y7hUz=;X!R(RtFjHS4rK|EtypJvw})N;g<;rn3fSC)SBQE!fUqG*
z;*CX70r0rUD=fsNV+)1~hD<RULt39`2AFHTC|{Tp<yjo?XjQAI{c2iMM9$y2|58c1
z?|_QCP;2&6_A&nmZyV`Nnkx<nay%i9dEH<odVCUbl>D|lYz2FA1crY6eCRSUv_rk`
zxWaF<w&$lSpOo$M+SQqp`B)y&B7NHB8+LUJYD8dvj!1q}7X!9yssj1mC%V{oNc{PQ
zTE%c3+SJLgXFOaV5;x{vepftq*tkifv}-Po%uy;t-2Tq`Bjd>Wy#Xt$5*rFP%wKCF
zL+4%8AL_3RhO0SuvyWZB+oR90bcr5>!HwRcbDc{aUtq^N7`o45fJYON@sm&815c=&
z_xm1sbeFCb_$P8^hQPof)my73m49iuN*w->w!|7iW#-TrpY^|SAe;n{70qyB+@}FV
zRy)!m=}Gx8PEG*b?<Nl*!nL$HM-2-?(URLLT+^k*d~t4iOT2lM*Y+~<oqyTMg%l20
z$c6OoU+?<7IN%p0O-1V2!ln&vFtRDh7Tr_NKQoa1-)oD0di*eIa_B2@H^PGDy@#Zq
zp6o#MUh$MYy_aH)WRV%G>?2T!Ax474(yVv4m#Z0I#0+Avz9f1ozGcARhyo^~So8);
z(Q@ch1$qD-hR?Cp2>WgR6QS~(espIZ<NNkRRSBpw_pt`Z<SxfpVoGtr(b{?YE0$mY
zdq_3onA4vci$y<4(^u6OCAmyK*h@9On}V%$WCL7EoD);(3bxoufV};guTuZym&>X9
z@46H_Y|it$#bLdVT>~q_N~hKp+aLC{O8Mwx9_;f>5qLd=uDvY|^QRA**OT(=lSRxI
zs0}fCv}N+;NqLY}Wr3qTBm$A8l~)L&E!OW~RY<zoc$$eV^4$*ou91B%(z4py_(`z_
zDR4h;n(%W_Ri(|PRkW-5&`vcIkKq^ky<1%+hhJ|>lyhs5pZPu$iy^#mNq%{}GFF-W
zGJ?QS+$TCglH+hx3DV#x@)jM4Ho0xR9F^OI@ol^GUY{yzdsX{qM1vL*tr=`D)p|K3
zaMNPRP7%QZ{p@c@S_koj?8-mRCqv}pTptP7Vn-Nc@Hti><>`T2xM36R?-uc&!8vAE
z4?z3!jo&bw8b9s3aCn_3q0O*)*7dxDVue^+Ky9Aswc`J47XhDS5;lP5ogB0O0lt2j
z6YKDyidg?J!TeY4${9{QU8xfh5eSC%{gU~Q*VGnz)Mr{tVviyeF}@y<Rh`Ti*?dkg
zgZgbwA;IouWW|;XkW&_b>rV99;hF-7M%w5GJj={}B*6CQ)ie3+hX%=u0a8-$NEQ_5
z+YO934Cw7Qrb4Ac=SgkgRwZoC*I90rzZe#_VYbDVvUmX}81FEw!4`1hbE3+61@oj7
zvPPKSqcbO_JFTS-l3u=BD67IuE;}eO()cJJC7W?7VpNanj($;Gy-Rt-b&Hy3B??X7
zFye7o_j}d6Na5m%n8=Cp1BqhvVdKW62*9Q0>tLeb<Y|{*pu9W?`)X*saZve*c;1h+
zJJ)|Mnt+PuwbVP<`twDnd>3`8udI>n3eK`f?a5#I=`o200_{?t7wuk87{rHkS^g<7
zr8>IBvqiQz5PMRPKW$D1ikWoIeGp=f>64B-=@84sq;qS>+?@-VZ1z&m$ha=@A6aFt
zB;m>d5h^r6&MNc!l>ehnM;a(#`7gR0L{BbNV4=}`-=qreUS(eCjw}lP_`~#}+Z24A
zBGCx;e-0Mq@E6M0-7E#<t=tf-=*u*Zzz4_Yy$Y&=FxKgi>I?L<f3QCxIRvCKL<25s
z4h2Yz#}4|t?TiAMp0UwHw689KDg}op*E3?!F$0|G(kC-M^&o&e>D9|~uqmOtA!Ol-
z%H;)4#y1~*Abvbu`&FqPIO{kiTVMv3SwBx!XseX*oS65Bj!NTz#vrl?h%fevfU_z;
zhLEN6TeFX&zDm5EF!}m)T|tF5y}i?y4)nQ;vYnm~Y()<Gyn}k-sAzHd+>5kR#Xov_
z)z0~Ph2Z@KH}3Anl=(9$zv_3&LGG2v1dQ|k-+LZCqErQjPI2g*bxb_EWWr+7AKyY)
zYVAzSuo%O4USK+g*$%w&^8^|Ru&}$Y1G5daEFT2zfRyN68Brs!my|^m9>1(ve327W
z#roCvP5B_u_vu-8^%rz<B+I-;wZbkNUiXrjyN9*hX?F{cW!1;Exv^s4`JL70-XoF8
z^_EU?4Vqx*>_nOin8a3cbF|<`xmgDK4oAk)aBC*|GQ>8=LbcqVCoMB*Ul(Xt*Zi-T
z*YRD4agJ*6zZC!8<=<Ps%nSQC`-@gfuBJ=`i*1CuF)DH&L_z{4P-rF5s;DFdXcQKw
z%OqORZoV9y2X0>XX$dP*_5NWG*Hp|*a)~&y+;#=p?!*GT(~~`e&A{Cg!@nw$vCURF
zK0gQ@ubW8f|5C_aMi?^?fKEdCy5d<fL{!qNe|$g)VXst)-3&p~btcgEJTBEB=hXqg
z*|~3;EI#VSwfa%IQoTco5#)u<0Ck@k$+bL+m|moJMO6J1^n6X>NyKj%F0a^2!X3W8
z>H^&)iOc2|(4IP@GMdomTh{C>QK~~2&=HDS;3+4w?9lU!|M|z+C{`rcOn_ehf-q!y
z9n|SNPwwWHGZqmmtZsO4!fuvzbtRjKeDB@Ef_N#vyNhwUk8o%<CV&$TY3%of(||6>
zR8{bo@#Q$|1fwHiXzp>pM&a;|_DAk_|A)1=j;eC|qK0XXbc3`YEsZoBkVAKik|Gi!
zA)SZr5<x`i1|<X(q*D-3q$H$~?ykc*@5URy`+Z})|Gj@W9OL2o4DPen-g~XN=9=?@
zLv6pyYALPX#~6>nlIy^X!t^l~d<d$pcvXX_M>=ng;9k#6_lN|!hq`tz?$Qd;1|q(?
z?pd+BZy-+ZnV$p5hbvL;<(K${6Zue=_C4dbpGtC7TOY6M(L_T_#*{dI-k52+C(drL
zB?1gMW2eb74^};-UL@8-kRtM(nA&s+(S2G{<sphr(OTz+cZmA2;qB10)wv(-p}F#*
zI8jpHTkmmBCqwa=%mbtObl7eSVjM8{dbzqVX1jkDCE{cu{v0r`Vgxq$^%_H&tGv=g
z$?i)~DqBCBKIojNeo6p#Ip6J2f2?23?0VPjL7xOU=|2ZSmIeDdI0xXQ;zw(1AFgU{
zHE1~qDKVRzW)b>f{D9xTkHl!Bw*F_GDfmojdO3j<7T%L-%{L;!ir+8UCAV7v54UJ1
zMPDnpu!M)ixt9ww7%;HfKlCs@Z5aBiL(Q+5MOHA6V)vbGnGeC&I0qvj!Er2{{>byy
zA_UO&58Gerc`MBZpz~hqc5t!J=f<YqngpWnqPGEiBp4hk=}no;OVJ>goT7*h4hp_U
z79#frBGOEmV}9z=P??3EgPJC)2IRmlH$%qt(V1XL52W0bqWX`{Z@qpyu4N^#R=OWt
z@GJ?-&udJA1_Uv5!jAO%@tK~Sp?`mMj6Yv}<1Qmu4jUxf_61!`#J~|^lE~de6?NV+
zWUW*D{nD?!C|2!PnoP`fdm{xlkA=vE4@SMXz$dZ2kgxVGpbTBl4m2mk#My`^nw&b@
zZaeDO<0a<U<n?glUVhR%w^TH(`dZpU82})&&UP{~?^W;d2#9-3-_tVDd|zPyl+37b
z01C>}W+WqxFs*|)G0{lx3MntcSEHb!<@sENg}2VF-6>%Oip(Y!Da+WFY<=h%!IyE?
zZ=RT6w4l&FU17-ayLcf=we673JZ(F0AUoUzH|V?lCZKUBe~B^Z%1>~%7)89(Lle)$
zWyEnycj{bL_rqx;+hJzPtG^S3&yi@`Q36$)-jqwMO#V0DGhXRr7}303zi4$KjRT-I
zQwXXXt@nPpifWJo5jLl%xST7Zg4+;4IKNXW-rJ`HY4{@|_%l`sF(;0V*=bh~uNzmZ
zd{tXAZbT0kuJQA9C1%z?2-?UGHHANC(*afR;~Br7+mIhZCG~}Ef1T>fyiP1;Br0TB
zSw1T_J*s8BRv8T(=wo&>97apfL5j#qt#D`AQ{lJz?=XN?2s=*K52-u|;<yX6>(<$w
zIf$SL7n?0P>xHVoE8oJ}xtijqa-+bR<ZLZJ@pV_%!#byHJ)^e>ptS>WTbaLN{q6$H
z4m+&9V?=O;nwT-;szBGwszfE7nS=wwc2TP5O@&}m@fgP`roV8f{%znQ`wo({KOMlT
zyfNN?<bYwp`*IeC5&>|QA1Kf1&__41i>K0D)yma|pk!D1(?6lr9K{3BbojXyQYH#_
z)99V9*$mud&}1f7P}qCTEPc4N#%bh2mM<I7;^pUGTPFnfFa?ZC(SOX9fV>sE!j|!y
zDcKz0hB%o=$)(cL>gRC}yLiFG(%RAhWF9M+eZObhAU)TEjJ@(<$@}7gLTg!;xl^{)
z_I_q(G@EHGIIPWb`8&_-Z*tkP(I2nL(os=223bwu(dDhSUu5Eo>r;xq{?3G-Db$>T
zo)W0CM?4PFGa$D&W2>i?HtVVE=mqFg`TK9P`^>eXZOUTbPc&+7<(FRy@~Hoy+#Pu1
z@8}7U(_&4E_<~)Sh$+575u|1}tu$xz*dR&e%o(5d2Eq88O>`~!19|2eNNr5+mI#x@
zo4?VBM3SgwMY$&2-$RO|AO5`TG39ko7)fBxXW(`!D-Tl7xu=G9Mi&@>-U|dITRwXy
zzntH;SKR-Au`%`z7X_w)aIX~?xr*I^5ss#87EFr<a(4Mvw^e07s|gzbuiUl$zvY9Q
zNVnz^aq-SO0P&^P*15O|H17mF9TBt2QJ~_Tjo6IkZ-D?MAu%7RTR^LZ7L1mV!R8+w
zk;n}?b$0l9&pnXhf3MVs6|BLU<FUc0;1`6MWCWqKyki{%6?+x}m}kRHG8gw`U0=<{
zl?m2rb^PWv$`VS`I&y}gbdLBHQ3gl5!}IIKU*r`};#NLeU`&TxdJ9o#iJVQ`o-$p-
z4*sq9N8Xz{E4$!5Ln<{x#x|%YwkB)nB)ztagumOaJ(|Dgp#B9rLDXYD;<5O-Lw$2;
zqE=Y)^W0u)gbniRML)-Bwmr)4MhsQAz7y9Eu2pv-zN?9@wLVa+GDvxraG$H_VI?0c
znmBcM+~Ww+-<PN7dhmr#G_YLz<o;pq2_A{FR4(~Hnq<%M&<{j)KHf#kR5IBQJ1b1G
z`c+Ike>!z^_6t(@r>3}0B?n2O4z9Hw4kp82E&QoAk_aFvP{#Rct_7LN=;Fjy3fEy5
zZ~SXE2`@%1G@tx@`{ltqgHjBjQyl^FzI4|*yCDoJK2y7c{o8;#nge#IfK5!}C)FF-
z6sOkVpfKnJ6BGL%X<Kv-!*%nO+CkQ^&NDW7T{V7O7Wj*Og>Ney2RI}7O7R3UNw*Z{
zXZ?2X{&yX7=zLE&O$3{U-t(Y|!m0nj>+GaDk{&PFB_41VROhC?<oakk$T(Wsh_P8R
zdgJKWTF8`!F*5o2tuJ|SWgQ1O{|6CNIW4qb2J@oSBCa2Yw9fM%YV$+pxNBaeR7Q*^
zDw3uWur1$<6Mg`HuivAYLJzDt%uY2nMt!^$JxsyPVE_vXq}R@tq{O4<3wej1N;B{&
z!icbqa@;*KQ_9n1Ih}!Rh9Zh>JIq_vRYQ7PWZ6SU@`J-6h54ZS%^nsDR?f3Vw{Kbc
zo>Q5cLa3v(;@I?9%|~2dFTZaEG2M}}%7K|<{;`^nxhk`;5f0|%y?v<f7>j-B;tSQm
z$M_sBxAYc`_9#l5mP@B1ka~VI4PCS%rZQd!WP5)Vi9X;!g=9qWN4(39{k0@cIR8o(
zx601c{K)9>zv36F!E&$v^uj38lq7F~V?9I;c`s)z7tI(>A;|9k3W>Ps%{}!!%9jgp
z2dOaJtFI`HbI8icBZ!%Z6e5Wf&4N419(P6%$`!M5TrhHTfV<!By6HJ>m?{hUpPI7V
z@-gRu3TU+2z(vjq2$JM}1Nk5!CN7bP2W$>tN%tGvOSxjzlC02g(oE!nR|>eTXf*Z~
zbj!8L-2&E7f><%Hx^^$mhn|g|Hq%#A6DEy5#z7x0^;=#pO)?jsc@{|W5xPe_k7nvR
zHdf`vu`OI0<RQK#GPKTop_~UV81wH~#T|SZXFb$93>o&dz9YeYv{f5h^u85QgH*eo
zMtjMz{<OR(*za3Sp8o1{*VXs}fChxNb?s{_Og>-!#+%IE)L@k-p|$()vcq{3KH~9R
zP3PK)fuWy@3^*|)Mtr$t>-2=D;OWf6YRw*xJ4|-g;k;wT3U<TySphI)9UDuouwrqh
zO=P=X-yi!k;em<MR%yWl?4lyQY%R=}?2OgO{hi;4UIokQr$`3Bfe!*FAC&hVIsC9s
zWzuOh3~Cj97P09!v({{yOpcX&BY0%*C{r@6dp!Tx0>l4ApluG?t;}!m@5wF1tn-Oo
z?(i2H_uc49vAORU_gyUfYk@HP-{8S-VJyU-tWmbHO^E)_#SLD{GmFltE8C>(e1dfr
zp!!!ElOPyq7J-3?v-d14*Ccx|#JmzUwYCV%k(d?20ReIh!d4g{MDiL6Z{fmMPvLz?
z1#<sNIbhnIO~7MZL6D*^zXbB>e*XQ;aLQisg=x1zLk`%(?yTEB*7pIg3`~}Erhq9P
z?d5B3K7b7YwM<E2I7AKuLhpgs);$;|Hu3SMWuMO<Na`^u@F(k02!Fmjx{#p8g>`IC
z&r4eW(V6LNyY4s-9!kR_W5#iE?|Yxfd(<NXD-kDyIQgSQ7>b|csM6($M4zb;$}W|g
zLWIIeviqk68zNrdWJvh~>Y^gweM7()a!4nJ!j^F!$k58fjBQ~08p)irOH$8NzV>vy
zqP!>kDF@+4(@D)fkcb8s_P=U8Rafj*4_R_iG5AZ$K4F?dJXNFfn?YQ7#R0Fr%T}()
zO+$4idRWVRqMQ6mb|aCGeaVE0R_;4Ie4DHF&r5mKF)6_YW5c+SSUb?;&GE>_2m;Rs
zQyOS3mkYS{tQ}WevB_WvRv2sXn0{|}u7oqX!ejjVU`<7g<)1~nt`hno2KLx4v^$eA
zY2RL_dXx3iu;%`$A3%Lo2buV@%q`6We@15k4#XII*n>gu=im+Pt)aW(dwGo{>41vt
zT2m;dbKY*Gh|vFf64_9&9~|95Sl9L^fE=_l#a?&``uj{RU{2u&Qsn7Rf+C<Y2xv|>
zMk`HfaIUf3w_^D$t}3sEZ4-bQIj=<vJrFj+d1r(z!~8Nblo||lCP?jwH{Osw`l0Zn
z6%N-<&m;x2=7a!#y}|6#_Yp1LH(cz@<5t_Rx<7rgBt<Nx89yd^i+5D?u$~SjKZSKT
zeb~b3G=6^?ufDGeqae&~J?d-|hGIWGa^4g2#x$pEx7Zs$>@X=&dOIv-C|+E$78?5T
zQ#z(`=>(9Ee|etv@q9eTJb)%b(gSy|;YF+0a=Jor07(mKLAgFafYISiG>LDsP`mOQ
z2!?8OTTtND{OHOz&ABypm6ur^`fFzM&7XIz`^<2*JeN_EjYM*#(G`BB!!RtELCUEP
z3FTH^6gK_@diy;}Y_3P=iZPEc8}lX<+q4WzJ-adT9xds^<gqCzK7{HO?!pIezZ`h{
zawZ_aYp!_@od4;+A3qUN{hqmRJz(&AkpeEF0FxWY$SA-ku$z>JCK{bu7X*^yw`UI}
zofPjDd5Y=XYW%2zvX;&WXu4N5tSkGZTM$|6G#10RIihTspwRirqbU+C@x9Gj{%5!K
zmdo97T=oj)Bv}Z(x9pJXI6IWFDd(+B97G9&<=<1Ih!t%q=v8N)D(A|1fH(0_-q$B_
zXa8y}>IDbQDxz^*E~aqZFt+5S4C+<=`v##P#E3k7G@Ns2E>jUQ=X0wzE!}|gx;VuP
z=91>v1Y^bYwD&!(i_0*|y+dwA8)g?U?KkxM>&Yd>mfG=zb>a|^;MNqNnE-I|dQ~S;
zK5K$_$keB2>dXCUiPHXCvOi*4J9Fc5=mU<1j=;f!m2VHsdyW<&nTCP;x*cdHrM!Ks
zJ`taz7)xdF>5iRh3j&-$qxG)V59*vUnQrby2g{+6+`z*qW%;qfmDdm%_H>X=@u>Ud
zf;|}zfVqQ%2TGsjihF3x>V!$UXw>U?GBU|TNO%Ze(Te1DCEnPl<VW2`q7?*cmJFx$
zE5po;JbW0Y#TCzNX)Df(!n7=N*bu^BSihaw?o)qjeUd(fe3;#r(a_?3((`%CMN9BY
zAku<FD>G3;+JTG}@>m*!+GA4*c%8ZOKFyfHtO`ZrEFCG*E_Nknhh^mdx|?Es!Qj!l
zBrXeeq_QnOmuwjv{d854@LZ#Hlo-n&iy@myRJV4-pJC3{;a8xsvP99`(1E&(f8fNO
zZmmNP|LoFdmu3vhuLDb&AZHLs@RmHg|Ib_Um(%9Y5LG5InJb%TviiSH8YTM#uhK2x
z<9ueNou9YkMm)0lB%Vyg-4t;`c(ZCY0U(&UN*h0KgmFH&Cy<oU>+!}ieQT326WLPA
z?R$Tr72vcH!Sd$wJu4i7Q3OZ|2C5z#!Sk}A!-9+dCK`n}=vypk=gG!n<bP%gu8A};
zSizE-(Ni+yx+1OC+1^A$7jvw`Y!65{%^*OMBbr?+gT`xkQ2LPIcx{N4Pc}6wPl!Zr
zMqBp$_VHG_7b_EuWD}ScazB~0eON5udt|o(20|%|F|RI9jit~m_bZyz;%-=hRJj01
zwFz1of!TdXBJhS?fEfG=EjjGZpc)dhHNAJVI=<k;W<<it4Ay2Fteqzovh>Mf8$MN!
z!OCd-skn#Md##a*@1fH}iw-<JM^X|YHs@9~Fb<<)d*hAM7Xm}of348`|GO0e+{1g@
zzcTx+t2&*2+E{6130h0WGca|#{d}`fVWI8a(1nL_GrrB<k!`L{?BUN8o0n)ScXHbG
zZ{D9s$xVb_g+<KySs>1r8;!ecj~=DiCs)KYmZ#+`5s2x&$)(zjA6)BJxx%s4-%C5`
zX~y*$6xf&4y#FY#{w&Ts?g&b}`xKXm*)ezn<_&SJB8aCIj+hnV{XyQ&Yab%#%B)jJ
zHRLl$Za8(fK~#fJwmx*NF)h#Iw&K6R;EqLG$0muy2~5zm6sTKg%Hv{)7Om(zs!3e}
z1|#Tyfdye`o}@w7jb!a}*fp&E=ef#Fbm>?Rcv{Kg_XQJKrmwFYv?GgEfyZ?pNr^d-
zTLJT!1*>)!NAd;m?T_Q`0@_Zxw2-aZ5v?Pjg*XBlmB|N!R<`JCz(X*0XEpP)JVI<4
zaC~(dNnX270m*~a5x~@g92aq44-qR7OxQE01KyWV*ljZsi2S^OsJUdBg$jo9Gjny3
z69eQcqJ)=G%sOX~ApLPL3mB<h9D~bCnewT6p!9NLanxsvXra{`LA|^mbmOyZ^zgmW
ztCH&vIM09P=P$MZeL{zF-OC9cMXaqzQ|x{8z_DIVS+>~PIH1}Udd0N=B@yxXm2`+)
z1M81kd-4X4HD3SM0MhWr3vI^y5!yFA$P-wE+Jwiejl^A@5m|L{@_RD5sOg`Y+9&}b
z#)^t}#lB_hkvNf=492ErnLQpJ@42<EXaxIsj-{fi3Mu>^g+S}b?BQB)*UOk++65@r
zMu88ONyAnMpC}81oo26qx*5qOK_c?!hgDWb$6|Bh=;xGJYV_z%>9(?J2D@u(tnK~v
zb-3ib*$m~ZT>st_pMY;na><xWmI1o}*;#m98T?93*7_<gnU{lq^%kVy7RFkv)b2XM
za3-@v?U|@!c{_Y8UEC<X<ep*PL25?pa}5Niar-&X!>mhJi9lYQw0<49H+Se<fTVs!
zpq_fCTbcrBSzZT8uMXC<MBZ$&`DcU^wv%Jy($2)n*@DyD2J#up)TeaR#7skQx$95z
zXOHqmi{O*u$If|#aFX)Nv2*f8gNRRr5?d)G;tT97G|rJXJc(bAMqDN);daiys2`3*
zBS8NyeW<b91@sEkK_e^o+uQTNWMUH_usBwk=={)MYBBW^fZom*S6(|_;haEmIX@T^
zS1*>T>78rX9h2%XK<2;Ud$thS?8ar$(j-J7zEi){WGY}+V%8}741id%!ZNr_{sPB4
zGjI9o=P{;$M$W+PelXqcYttm)zNT|8y;NnU!Na~ghB9m^1g&v;dfK2mCV^Q;lC;e)
zU8~7Y?qkEkY9e;u$~09L`-c#@Tg)I#^jAh_1}d@B8_|)JS%afb+l8dFP%vY&&J4+K
z{r8$qYQz}MIKD3xN(#`E68&)Vh+;Jre3W8*=0(KP(dsr;JtXV?@cFN32jimbhbgsi
zCF~hZYf2YQkxU}HuKN3ug2TvxlhZv+;=P(LvIIAwzusZxgsmckuop*fu1TdS1Y!p}
z-m@Gx^C~PplfNlqFg+zGx=?us<+ud;bB*q~&mNxJ)V!)^Tu_=w_~|)aFCAE&PO%}z
z`hu6{C#jRa>2o{AQVUkgMN05Y;e$J+qcZhweY;ScfiTyovy$4?Ffw%eR{sTDBJk#!
zw6nZ%i=%{BtYlryo_E}8FJqdy8VcFZbOOT)c9+Ysx6NA-V9{XqzQrMhdmBw9`R{|J
zng~5fS58g!Hq)H;(Qy}yHkloyBj@sH@49z9C{TA<%SOWN#&I;xyo`7UwOy0{)tv2>
z#QJo|$B06x9C7f=0jJ|t;xAY%A#$IXbxsBHM73ergj7N%C+6-A1=nS=67vs0*OuI>
z>P#WM2kdG_y=s64QvAwbwstHPFZZY$S~LpuMA>Kv+H`UNPbuaH5ZJtT5kY;_p@l6N
zd2U#8tak0fWXYW5S6#yyOKRg|v7>l&nP!v%I`$nfw2X%0bp7Hjj5gy1=&~(QkEiJL
z&P3rmp#+6Y_DnYLy>Nt?->bVEhinwZ9N-3OT<NsljBaYLN^n_PN>(^L!?oIoy!h7p
zquLbv*FF#A+Y-my?!O*v#e{B@75z|Ir5Pvy|A^~Y{SyBZ?mB#$$J^4wH99m>zu(wW
zYv$~RL4t&b+Tgk`H<S1vHFc0awNbWllQ-ZK$XQO;4C;N%e5ZReNvHi@xCdHT=2IM^
zJe&Tsb>wPSW2<gvXB|dp-aE=NM_9G-7;%6%a|LE)Zgfs^mH2IytNH{YCC2BumN#E)
zQ6h$Zn=+$iN9IQ6;TV3#@%5DEfFNnfs0($f;#5x5E?N7&el5IIp56?y{n$!VK35go
z;MsZ~lc$1Nry62Vp*re{Q`zu<?LvGdp5<ck>zcdjzX?V)WpL+t{^*N<yg?_PO083W
zsXUaC@}^1*0K5t!<b2G}p~Y;v`8AUQL0+Fa%AWt54+AKS7URBZz;ZDx*22RR-o<wv
z2}-POOtg|(9A)PGh&zE-OhfN(WQ}#Wf&1xSf=0@1jMcbXQ%ie$toIv3=4w8ziY#^d
z))#^M8vEY$NBi*NzTwy~MP|H&$NbOW{pW<cPBn2}p&#xmcECxI#Kh_yP&X+I-)upR
z{5cx=g0dEhIQ{+icMww8FB0`=g4jd&wk9W#&)qh7v@#jt%@JL&1wy)?lu`QBc<&<2
zP34*CNa{T%bR32YO5H+NCJBlJ?arjUj~I@-?A}NmK4XlR_(JFLqA!o8S&P~=(6E4<
zxRSA1V(1pC`dz*ZF~t%apUDO_qVVc719o$$;sw*`4cmgf9{dw@Y89LWX!*msynf%)
z{Vx^-ergkDRr7STzWoAX_vqzw!bruDzIm8?9lZC92H1R;P~y5?T@aaQz@q)@#;+zr
z&uYrtCmEXUQOcgF8;O#w7nU^B13zUJ#JKxBil`x|WZ@=kLPGAENBQ?Z!i#8o<pEQl
zWqxqRGSu3SZ}jRn6_y%VutFpG-A17ci21hq(Pq<s*BmSONZxXgHTwGs_Sd}MHOf&d
zQ0i3f{Sdj_TWDHl-~i=X7)RN>ZY=1nqZf1-u=Cw~wld0gcIUgv*5id%L1tQkoK6-5
z$Coa9u0TI43!KC34E+A$HpYODh8eYRd(`<Bs`#VRw~Pk!*|W`B4v{Jv-m@pS2pL2*
z6qi>yDl%7!Nxrf|3WPCc_#2|W%M=sNJr%eu^#$G6CF`!As0?B)e|7;^DvbI>I-!jG
zw}l6}vZ-7<FX_ki(w7y&rXzR8zd635?fs2)ZoJ{OTZ<lq1rHCZg|5TbMUaTORO4Y)
zj;uA!giO-(x7>c_zUMZ+p40d1R%2KpDa;smPgyqvUhGzqzN>k3_cs@Vjp1*yCYavG
zzv2^Q@<B76-~F;6POmLL{w&RmHsU8#PZbAOZPdM-m6j$2WoTXo`F+e?3E1DCAN-Z$
zLOM_VqJNI#&8hgHA)IqMFSfiE=7x1eSvEnW#Lex5#l5XIn(o!zrloIHz2TQ<KXaBT
z1$`O)H{Nu8R$j0K?Ivyakd9435i}nz$6&L%&qtsO>)lj5ZiA0P8?bm9UQ2j9`&Db0
zA*m>j`J$0vkYf_)37)R!Do@@IS3FbhfU0pIUyQJu_2oF^o)9bXAKsY6_hVF^yKzvt
z$8Xk#f%dsd4aI+fgpW}g>-CdA@Lj5`TjR3;B17CX#e9ge25Mva>&+p)@;ZNgDme7a
zpuVO<Ez(ctCc&0;w#n5X_r~N~<I%Qf^2azcY_)U|aw@ca?z};!`C_R<+{2}E43SGV
zZ`F<3elqI;sCb)rM(&h%yUdB=?1t(~WE$zxAbGx$E*Lp%-mi<BdP8)3i)O}cxB2L6
zano`gNFs+JwT$v9ZjX+12k7VhRT6$CQTzkr<hU~jU9#=G)x}t(+<9AgZ!eCF4e_3b
z1#(*f?Te`LL1XEc?Xww!uv~{~A&{v4yEpWd(SEP*KB>w@tDK1GW)@QQ<+#rfNKko?
zC;19O@5L|_ysDVJdRa#nfO`JdrS%ss93uCXwF*v`NA8>;o1my4CEFw=*<7#pc*n$4
zCCyHzhkhomf<feUV(^?zXbmPZUcDUo^q6~NtT9niwtG|;3UMGldSQ6x+<{UMcKH?l
z+_ZJv$UqnmVaLGw;LPG?J4$Z}yP~`yY#4=~kAh10ij0+YXGnS>6un}$(LUL{))&i=
z@W{T&z~(%gl=Z3(1NM+a#S6=G(uUgiq_nvHva94y18K`(r(7FDNkz*OG$5#`{rz?7
z)?<NBR$2u|@8zm9w20izIcoBqkK{V(%w8E5Qbv8ZE)d5|zS$ml>Rk{>jE9*MG#)aI
zhegpD8q!*e8lFOJj#p|E$J3CCA<On7dvap=w^cz``o^yKH4H#~eE{*j9st^ZG%P$m
z9aprKYuwi`#@}f#5r%F2?WuPrShUh#{r3Iy;AzSQwz{?coc(#RCey_ohu@1b>6e1=
z3}YIfmumlSi9R{H#9_58%zQYCOm<gRT*+Dw439ycm2H3D6)0BRWvTKhufg1ifW{*4
z7vTI84<PfE!Z6$vZ>-rdkdfy1B=-IwGjsv%7>N}&TFL5;_KY4Y6!*}URn#ca4`VFK
zKY)^a&2V11!X?OA9w^0+*~<<?89X@GUc(<o7G6<5xg7jD8Od0`E6eD!y>Cq1O?`TY
zrkE)IvpO*m(J~noRy8iVdtSzc1W}`f=A#Ky<}WK411H(iHh7x;!4_zAGVSX$TE)V7
z0dn_q3EpnBtubpQ-6Tu?eS6I!|MI4JfuwvX6vGNTX`U%Kp%z*ks)`!H)C_w{AQ?g3
zrznVYUmuPBzS+hvgt5f`<?Aw(zZi*$qeNc}!^h~3lEXK@xsZe3!PC|ETwm(&r-FLt
z2q?w>4sp(S$f%>rs1=6xK8umG1-Jo}J8PJ3p6eL7kDrwU4%xtEg(P{&^~y$B9KOD5
z{3#`gQf_=kKJ+F#>gQD-Q+ztw$sG5rFVZ6K+h)`!p#|4XC6=}M&VvMA*sr?k#&74b
z&ur6H>s`j)I-bqo+Ap&Qlzd0Is%!nV5W#veXtR>l^E?f!2h!DdR+1|o7Sa3lOTm}9
zNm74Bqqv1g8^}oG*0~MXra&^Pb~lRqu(|dDLtf_T?Ww|+{H2tP(>V*-TW@4i>{#d2
zN6f$mb2sLExss)9uH^1LXm_1L@Vj?NdKYv3C)CzM(Ps0rj>U(vlwnLG)Ec8%J(BCp
zLZ=#$?{x0;?6n23mIP`Abg%rNT-;`6oTDgqB8w{{zmat~eHtA>Es>G4H^)L_j>o*v
zs+be{1=IEID{{!?UE0ir@S<918^;xYUU;D<YIC%dHFGXEbiDrfEH-MnKQIrB)p^^W
z?_)J62F4_Tvh;%W$4576hdHFF;>Ng9;&&r78#~ay8GWa1F}2XlxfJ|@5ckMSH^B2o
zgnD@%yRu~V%xCu~v*qQA_xVlJQ>@4M$YG>ARLB=w`vKYDg$BN6qu)A7V;|2|rFc*F
z^X0ywwH~J82aN~HQvSQ0-G&c8%s&!X^_>!-g}UY)^lRfVC#f>^9bnk{Am^u#HS=N+
zFC;Wxwa6dMYFcrTU2|Cf?9CJ>EUlJIyVj9T!HMbwJL%%!jErwAOnoss8&7yXUDjGI
z1zBl0P2Y8%2;M>+D#Li7Y!}=3qC{uu*_B~OsbVCG4@smnZz!Ol$)Tyj<PH9W9y1Rq
z+K@w90DazA*0(5^%5YtI^T#7lSWN_3`Rli7rrniq%MbGxm4#ZqXmn8F2vZ;MVHi5}
zWJe~Xi^Z(Pa(Jki{%<p@;9?QAj!voyZQK_{Y93+mH$*Bj4a*DJ2c{5t+w2LtZ%?O7
z*7k%Xwum>C3)MP|ac{Uy+SD?{#&eP)<2&zDBVX)Q)+=x#$xVyBCR#gC6iD0WNZZc<
zg`n%sXn1qCc_qKcV()e_Pz%hEG=E=?mZ!XpL2=6jJNQT$=pf^WJUVlc)rsq!-tq}b
z5PVC+fcAWR&u^_}P?mUk;7w*1K(z7$KR#)DFQ@D`ISu)>ueTsr^EyzSi!bMHVPHR=
zM(5(fGN|a%@DkIzj~=SIqL*`>;UgnVLY=70{ajfcd(^{w1|D7T7S;#-)JQxT2O3%p
zPJ5QJeS7;}Z8`EsOVu{6wzo*_W8^-yKrlX#V)byQ4`r?mhU=D+S#&_QKK*j44Y>5Q
zIWp;pWEs0Q&Lh2bEzDN-;3GF^IXEjCgi^e9#@e8VjqZG2+uEIxt34`TJ91R{`tw{@
z()6uk*Gb!9^aFy$NSXWE#hkAr`QB*MZZ#jB%VjL)#$Nl9{0SBv7D$kErk@b$PUGv!
zB5r}EPNMs<MFns2&yK5I0}T+I5#z7zCqyM9<Fz6qvSVYGPxtTdJ?7$%VfoMYWJ?UX
zpz@((!fjbr?%jGAMb%37;y%Vux+s(zJ*v7;EnC0GwrfP?Vc(CL&fOwyXm(Gz@fkm(
z^qW^p!T-OHn9hh`l7OFLtoJXP@r!kMHW_#i+$oyVi@~?k{EVEGMAC1&Zk#RpF=&W|
z35i(l0TH4Ncb?(G7#_%DUJPRB8S}PTWtLmun$GlvA+99~wE~x~n{zGp*cr-Xef=nm
zuu}gu9VxYQ;e+E1IT)1|Q4x(*=hHC~@MMFVSBwRJ6H7m?YT=_pRBWAIxvvK^3qA+#
z|DJfHzM^{(?h(SJG_m*KZddoiQln-`k{97GL;v@rL9IW&w?nd>g4NNxoOW@s+~A!#
zLsW0a*Z({*D_mB3Y?WylLn_*2MCiJ2RAyL71be*j`#(=AhM%)mS7`}!L+H$FBIum!
zvl7I&Tw49D5m!FlcnxK6I8@VrJ@GnzBoB4-e0#!{P^)4vVLl@5mlI@)^Ff$IIjS$&
zhi*mEcYMkgyjYCoF9D?2TKDx~G8w*7Q*fi;5eL6%dT{N`y9V&WrJ1R|AAqAK7jSWu
z>opfpx5xb=881Pu(Kg+C>~>QIIJbSTV)kksqrd%$v24dqszz6apR9Ku%j+Pl*oVYk
zcIn)`+XYkq=dyw*qaBX2Gen;}`&KEcFY@E^z&>wKIU>z?_`@U!4twcu7s8~!ezHbA
z*Af}*IN6<0I{ISRggIvG*%bDlY}=c3BG&7(<dRih^>Y5E8B%s=0DWkk?L~6*VSaKO
zm->dgxQE~2WYCZH)&UgflD9o0JhXY~dt*kexOtRbqII(Php)pS;Wxobr;>q{elzBa
zjJlJ-s3F)9F<y3yZ&vMiLqvwu<UZS<2r*PJZAx`8+B|2icJcl5`QJs<d-IIF8##hK
zwo3ia5}A~*CiZr$|7T$<wMNKagd6@RK%RWeg0kQj|0jr=B{j8SR7N-c7kdBKU#MI4
z^ZLam=m8P0CRLX9%u%)q6z^<=xNRVkzp;|HGAD&YGCaPdkJcz_v%ji;c{6KN{KU||
zTI$hgqlLxA<*p&UOL@iXppZ<QAF6wyW>$Ax{Sr6tt3t4rGF=3I4z3idT=i|J1H>EL
zflEEvU<za8HMoJQ{n@3?_{EX3`=k0t7j({Nn7L=eOu>%+qz10Gc{>^+@fRx6el$RR
z-{HdIt)6!n>T(XX<`Q^c$mUvH#Ih|8{BA%#l@H^u#>C`^WN<v4*f4N+*8(|b-6ES#
z2t-8kt#JiCRUjI(1k?*}=#|I9`XtHQ_``hn&wP_%+xz3>3_oN+Xw6@oq34aM>y1$J
z{^tTokoy(^`*xns-D^_XdtEqJQKzc0J$$kGcmz?>+umo5ogVe9&@2j+b|rmILmZ7B
ztS<?1V_wLK+)MuV=%V1ocn$y5k_c%uRs8y;HL9peceCiOE-f2*_H}5<bMrXJYz_#Q
zP`hnVh-r~e%vX24)*p|R@OVd2taZ3ho1AZryx{79lOySNC2t|13#c0?{uGpPqY*Ey
zutUm&+uHGO=h-*WQ)W6AN7q@txjsOI$QiNd0KGoni)cXu*W%2HKEW@d=*;g9N*l!T
z?1hVl>i5a>2Q!b8`0Y4I-n4I-Y(;8@^hJ78@xRBQkG}aTUD+NxupqGAVk-NnwXKST
zR-}jQ#BYgk(WI#g&*d>`0u(4U77=ZW+`w7N5Kef60Dvm~@!D9dsN+OVf_k73K&t#n
z!+2L85)!wf{X(+RdeM@~7DQ9f1otrd#k~<(NWC<*pJ}-JY_`b+_wct-rFkYO3E;X7
zznmIB46V-jaBHUkp&N)nnL+e5e{srfP$qQ)@YL1;l|l2B<}iK~nkPZq&rkC9V`Y@r
z+7In^H_|9D&g#duc>&tpnK-}Q4(p%A*Y1y)aLiyT?DzOe<)}X+G2c3-0c#3{@O=4i
znLKn2twSRGQlGN(Lm9ltdS9LEK0k`JVAcPCy}tLphlc+YQ;z2OgeqEW&Yb|^JBEw@
zIXFs4CU)Iztx{o9xIyUqQ_Ji218Jp)`FRalCgv5LrG6pYMkRKdyzL&H8g5gy>mp9W
zSlP~tTF}^+@DR-Xjh7dOu}6Dyy~|KMRMH2z&Umr~i|0@><kLL%#k&lE1@!^)CELlY
z;m7Cg@5>dN3AGiW7j=$Y(ps55w6mY1l}UPyuqql3)Es;kZ}{`WMpmID*8K-eFAc|?
zCsdj*k8)9>f;V+yH>Uftw7bO_loZRPwP>34W676h9A|R0#wx%LI2N+!^)x9YV0z2=
zs@Z>oXg4EV%k4RKpL?iMbl+xbsP4eXzI2PEvg%bxtCO{P;|s8rnQ)Ax_0~Qu5YX<>
zEJ^~Qo+6HkVN&8`659fIPp9@aVPH?`yEdep@&4eg1my8jY|c^y3B$xK1TCWX_lQ>o
zKDa=nZ|PK+N=$u<hbqyA?4$DpuPT=rl;#m&^`#}wB~Uh|y?+pHmJ7qf56+xu`veSM
zxhU&Aw;x=9S%)!;z)EK|O3E5NK!UR0U+gw7ha>iJ$I2g$<Hr`;k=zGk-3;FI!LH%4
z!E*Gs*_d26W?+O`giug;N6U7dbeeEaibK`X)Kdv382@U|E4z1_M%H4pqLcpb2;h8x
zWl0~zP<0|J>LbH6t59qA{CV@@aBsx9uB%n*xUbO7U!jep42N*l@2}^64AN<0by9B~
zvH!dIAsC90Xr8>#_qanZ0_utUrL~!}rB!Mz{V~GHH2Qv@>)z;i_0`bAW0?|k!`Evc
ztb}nYLEJXl=zg`$x%&m{pJun0S`#Ywvqje@Bz&xz7g@-@fzoh`Yw~182=6@kdT#8b
zT+={-!I;p28nyH;xo=sxaHe0XN9p@$V9DU4+FcI@ulo0rruwzfCcA@qU`BUTG9H1G
z;h8))ovcmLD}Y60H~o62Dbv`xv!CDTs!ukEw^PLRc=ChwdZDqoE3QQgZz<y=ck*o;
z<C*;F!|@?sInNIB_xa1qu1mZ<5V?nhNC3<1)?z!6nA7Q*SMp8{4I_inT2>na`hryy
zB?Er0J(V}REf3mY`rKz{BPSVdYm2dGomlTvn)}-sy$}q~mo$%Nyyhfc_dBmJC&lxJ
zOldQhHN(&Dwiz8v=$RL^K`1^2qY>Gy45VWvS8-*-krH>Ge&H#O{=wI{Cwf@n2+onY
zFEpHaU?Rdtv>|FA<_CH^O?kZ{4d7f@uWa>>ohG0XB-!IP2hVRr1U>j!X`%hA&Uw83
zo<}4xlU67WVRc^@(XGp!M*emh#oO^D;z3^V9cu5b4#Wn}i#!O`zb4fGIYd&wx;viQ
z%ezluBBZ;R!pUz$J+P4eO1o17XE9U#m+^@eeD7Y+oIc}FEXtRIk9b#TZbSR%P&xWI
zhUf0cs&aR@e&7ap(OV0J(OS0yVbn1kR^!AnwP*X>Z0KK%2`jjXRp%qokZK+2wh!!}
z$wK?~#ToLqFI(Y@b#QBR@P=;6<m{unk8jr<JX9l*rgtr2ocHNkoBS{+ks(Qhhr)4o
zV{muuiNXx)fIjlZ`wp$1%~Zd9iR{VN6KY0%1o)^&>lzDpEOH^?bHWYMFC=-l#Aff?
z0ldgSZ@Dj#S%;gXemBdHA{UDzR=aY(f?FMZ$MGrBOd)`OSNW}|vr@IZL-_bQpH=qL
z7ZE$O6e;D8jtQYUMZ;pgR;CicA}LfBv2Ko5Tz^2jM8EVSC_GH)OG+L+ylVJp?M0%X
z#77+Cr8q~av)~VLv@SjzONHw8KRl|r8yj%aPhLLb50U%1v42bVbIgMAV)tPb(u@(b
z<9zPra1vlu>9+wvWJREXg%?~U4WvK8%|H#jK<{Yxyo&Fmi4=P_^*fW+s0AqgLlkJV
zd8S@6VWQUn)nYJUtCFuQj4q_0NPvaF_hQ@aI{{<=ex&uESe4@TI>OgWf#8G8K`-lQ
zt^%%2$pPql>x15*-OlvqaFh5HLmJ3v@2v()HE-2lbtEzY^n?91?x}r_Z<)d^x|5^7
zJ$8II$|AYzlc(0GWF0BH-_8J=Yo_>57dMDmrxqgOv`W*!F0}URh<GR^0Y*1sS=XVF
z#YDNCddGYHW+HDZ0vf*h>hhCTLiZEV`W}bwSc`A7tF^y5qayw-sp=t?vXg6$vtz3c
zd6{J#{FjmGu-(}<Un_!~-;=S5tV<x!LiBzbVrZF#2GG^tA+D0Cj}LL{)I>0Kmo{d6
z+7nY5dmbAN$4ClCtt3!&z_~q<FR$V5a2aG!|NfbEq8?#)X(w6_T)HI0tS*0mXl3+D
zCA3oA%GN>TJH>5rgW@Db1FYZyrTK}KZ}#&+=zVYS7zTE+5){98slK%9t1Wfv(LbcW
zT3Rnla2^<G7BWrD-YJBIP}x$TX_^uFozGUl-D`#SUt9#IkP+L&(=|>obYKqa;$KlB
z6yhF=B*eY0g%V?}!-GOli2Zm>$YJ*6Oww#_0Pe<csc=u{cim;ZhBNs!S$5LzS7+Az
zTGBVe<UTb+%FGi>mg4sCtKz?o4Q8~(i>iN(Td*~I{6WzQ*icg0D8x8myP0XTe+G6)
zU^bGJfct<4`!0!G^Mo##c%MD`{qp`Ti9;9&kEc{3B=wsWjcXU1tVRi^HI%)$tepqT
z{p6mVB&zyYdmi)260l{SaHjb~jiv{e>fR5EdY=43aAsC0e|gkTtY|BH&tUhporqwM
zzn{R+!Q_zS|B^xoMt7*(cU4&V&H{C&ba8z0pSnj)%h35=er=4bto5q0BC%%h(_S3M
zexrHHSZCF5DriCF7g@%<D-4x^nZ}l91!(_VQ#l5#RjV&x-+abPA8_%FVl4%kh>>qk
zyqc}D=|pfglWO(nKSkG?xr-T<_HjjYLz>+lNqNRZ&Y@YrrC%J**tl022B)m67i{J3
zv95iNU`n)2!9%IkwLTtP8b%ge*{EGgWlMMf$(844UOB9;b9=8nwmzStuZ(`<1!>4H
zLWQ1Z(G*yq&6K3BBzplFn`y4tFcaMJ>yjbNL^r~u`{`WJUs<30^ZhENND>80I~_PH
zym$=xc*J2(9}4sReqxwl7FWo9ESyC;{w34M+xrHRh+-H?6WJ@eBuc#w$IA~%fR1@p
ze0i0SsGG`dnBki0n)sPGG3x9pajujp@dI;A%wuMZ$rBd4$|o?}c!E}vGda<Z>fUd<
zR&EC8fqj42o)SXPZN&^^@Tj|P`Sjk3EZJvhqy#Qj5(cHoniR7itZ^BDYpVzxiKuT{
zX9SZANJ?&CHPY(2Uw9XTfgJ|N@+)T=>Kh!s*gFhx0fl8BK(La4clxubbSR$1OPVB!
zrm?0i`Xm}w)qnlSW606BZo72EM?cNzmnser=K7$eTl{1LQBvZ{6?agB^DyuRaR|#w
zL1|wl3PAWF)=k{8g#|Hh*8ZlFr+8vnn!XuHUdzFcK9Oee5NXL;qgi8P<1a9O{KEdp
zH&BF-^!0@VXwB^S2jUEcD$FN4PE@_PF!#YOf7qYV0LzQw;0x_Pjx5DTX`VR`joxZp
zV8$Xv(#h%QfXQ8yJ`ESm)|1CpMr2Fc$C+M684bXCL*GO;$nugOPs<Dga!2|$sIVw8
zI6gY68k@N)_DbX~b+3Wjy0Aw{reZj=y<?l(A?RbH=Cma#wUt4qiB;l!+(_{A>8X&P
z8fy49ENP>I;#e+AId|{lA<^Xuu6!Ua4__cxiCEhM3*=(z*bSR`6PL7GWdlmaN{jbg
zmTAq`@!RIp7Sb;i5@pqDo(%r{u=s@b&(fkqVH^i|3|7_tR6_&u0E$c_zLvyh%AsUd
zcLn|r*j6m17Ok=}XY-+>6$~%$5TuK{vUsc~`+O05#5`{S0~x>v5(~88pQmJ2$72FM
zD5Jbx3nKNeh<b2*7miSVg*oz!yL*<D>9Em$^~sNJ@n-p}OZvxI9`xfL1IZU^kjCY0
z&Q=Gng}X%~A2O8dk8-7pL%AI^@ZKRDJ5T1FU_GYYYqta8*6npjuRLd;yAy1c<n8?b
z$q4r$220^8bXltIa(*ml`c&+f{s#g>A+B7joUYyAcS#k`Nq0kw(f5*Y+mOCQ6PWZz
z_WasyQD9Y$Cqu>;?Uzc_u^g4gZVj~IN5ZYi7vn^`QX#0)wR7!|*13xKpT&&Ag;&d!
zSI3*b(jH`XWJs<IrlLnlH$3v^?GNmdF~!P|j8i2w9&UI~MuJt1hfeZ_F>oGzs?l6#
z?AvH>cvg;NmfOpgor_zw-e2`jBR1SAJ;+y?G*#-Pl1PWmmp6jSx;I^@Rz;)us_PdC
zO0;frzkwG7;?m<%5_6r@ccyFUR1iUsvSe}LbdY9r=5Lhta63by`*>7Hu<{~AxRjNJ
zRUAq{Pq(0Pw@Lk0QJ`lP6}w;t-W4TJU@oe$2x?6;b>MEbSR{PC?rT5Pqu!V?CY*^4
zl7DOFm)3lA7zaEUe&4Lduu;LqM9)C0xKw5efPp`?Ub0~FHDBgrA{2?dbmr$Hm%d(k
zwtrUMda|nGM&)Zt_G}p?5mb6r%0OcOCjexcz0l}7Ior%6?$Q;ncn$ovP_AXt<cjjQ
z_W{za;|;N#ZMN#fdGOZJu(jizo*EgJ5IuTesLvM#m>&~e03)+D+{M=2SGU+bgVC(>
z(6Na*8IC37x|6=;_`z>OwT{Gak~jScbLMJUEB`$81=ENAM!84DhBUoV|F*dbNrPhy
zGtKlj8T$^qs9+w%Pbl`5rHHHd`SVN^gq<S|)BRQ!SZveEI!ha~H^zSbvfuGejeb&j
z&P4Ai?x8*!jgN}5veZD+&!FiH->VEFMRN^Y7#qFCVU%whR%`rOa$ZdRHy|-Hr=AqQ
zO<d3WTAxgGRcY0!G*H^8uuVVz5iZkG-S~*jZR3Jkw)p$51TCQ}cCcPZo1n5-emf?v
zTd|wduwaXIitDcQ8cqKz6b!Sn`PHZ0i51SUMY7p&OR;sGM>84=4%|cP9Sr>?Om)I{
zeO#1EeoSpY%&KJh(~^iQtsDP}kj)D)Pd;kJNQ43gak1a-=eM-Ke;-VI484WZ;Bxk$
zM5ZNb=d+i1z`?q$Ruke0I|zZ?Et*-U%lW9(0VtE|(eg;TPbP+PCLkYQZr3qwH}1B?
zylw{I{eu?&&le9t+4pR&MJ9>Wxg8u+nM5*2I@eSo1BT6?<@um?NN3l40|S*}B=?<4
z91KcsgB+kUzlYf@^{T*I<#2Vd_}ZU|Z5voKmRSGnD_xlZ$b&s-h>uTRQ=V(R_aE_R
z3}D_#fFiv`%Nz>!I@B(~=skY`U`W<lwqq=_WRw|HnwzCsZ*<Dd1Ao<%Je^s<p56-Y
zlx8Np0=cG@>a}9)y+%Q{v8N+|Smt5=WOgY*O6w-kYu9K*+uD36u@StD(E|?t?~cIy
z)e!J4GM^W10;ITipMpe96A7aL&aa^U>Q3(UPtIK(eeRNo@3mpx2*5l53GWaPy39j-
z4ZPO9n@X>v8Vxyo*c#YO#O}dkSXMB#Dv!$mJ5y-I<st|m8$V4oHsAlwWny(*zki9R
z>nR1Cyl5Q4M=bR`+R!TznZ9>7ZzluARY5D@$n`O8gFL$Lhy9xKnpgNO8eA}cuy~cb
zduMf@^uCoptaTtZhu5lUOb8_8l>vDbK%c10eeKyygW*T(QDNuVrh3_;{$9@0Dn^*{
zF+TJ+Ho15Bd)$r@AsHjt6v`ROf!fcu-vXi~H5P4|@03%r+<)e+){|`JA^yASwi-Zt
ztBKL2o#Pm!UOj*QeAM)@=<aWCAYA^W$Dj3*FD*m2;a1R0(Dx-p*7*q4xH&(qZCI@&
z;#}fRY?lofniW0Op!})UxJOi`5WSI|<d8rj6WG%l93B4@?&?W3(Hkp^=-HhU1$zQL
z(Ds8z{$X8!@Q^6-*N0R)Db&Jc*~OQ>T}H3@x9BK)W8vVGr~ZRCO3m{Afq{a5L-y)!
zU&*Kf#$?f**};0J^ygc=yNJ|xZG5QIxoBgFMd%hrbZg95l;`5*lV!8X`U+89!@$<o
z{@JaLAAHVpW$%}+G3Shj(W3hS`Y@7SW~_vT?p7RP+TP|8M^2RWxpu$(kNKzeZ}&5A
z$|!!@`av9yDysM-><d8++EXLwB#uuXB#x<vp*RsejtkIN{1&ew=G=&ve-7$Cqc@YU
z^R0E0=AU!h4FDWkgF@<+#EN2p{*-F0;5hmDPY<07*)?W^#WUr-psN(@+UfURzNUHn
zpz8dzto~P$Zo*6Gk=Tz-cY(EyCatFP&0;JkOm6wK{Ev&%K_NsqW|@Q0V(=@9(zymg
zE=!RF+Cp)$oSaa3<5joCZx&F+J!-_&ey=@s1H-xz*(wgj^?UsPde1M5;sl2Ejzh`4
zl<xEf%QPU<c3gsY@GD*+as#95n*mn!wt`8&6)gTcQ7i3zLZWvaHdPq`CzB$Ndr2fu
zCe~7S<-!8iawF|1oH0M!j}%E~@*3s)ucX^v_ZslO;T~*KWU0>pXRjE$*wiJ>Qqb6X
zHr>laqVSkAks2UJ_^PV(;nwLAZciSuiFq9S{^|~<Bzzi*vwXRm0Uls3-Z#?d<xDH%
zTbo9(#n-qc#81D>)z6<V-`D;*9S6tiqK)!hZRkX%#;o@ep(G$oyewtYyrxJ)p0R7-
zJz@?z0NAEx<4zy{nAzvGyjZ*G`rp6S7z581%HrF_nbj;O1gtu+HU@tly%jrDUXzAY
zbOOJ;8wCgfbfk(hUBpm~Jp^MZt-+(SjyS@qErycEfH4LCxpdS;DI{u9ncg+57>}R@
zaiGqh(L0i_OiU~I2rD>>k!{UQVD>W@3R@q9qT=-Q_e(C}s7s4`nFi$YabV^vjde^=
zJ4jcNJEY~s0P`&q34x=V63+`*wm63Wuz$eNP!1SkmU{6fjjAb7V1BaN2A_~gHse~D
zs!T5&DiRKf)2nE?aPS2)01A4-oOr(+tdEZbOc2~dTULMC#4y$*d$++qZI?LN>~zei
zJ>C0%_+`N?Ff^P*0itr?sb<&@j0#G?@S#!0HSvrYBRJy~tb8|C<+t7RpsNM-S3WGE
zXFKV}M|7D>TLpL>`BDEAz(77!21B5vv!X24g6pCIkT`OpNb0X%JzSGBk5dy=!axuk
zoG*eo0?kL5PmSrHM6b-9S7h$k*_&}_o_Wb530*!24s+5v$;sn}R(xORdfOTpGs=&s
zabL{Nwb}4||Ij{zPUlB^&bvd2{gujWJ^Jm|!*GUmFrU7vb{BurW@JC4(WY-Jr;<8Q
zU5v2VgvlVcf%fG+UIRRGEFo<4_tR%>QI{pACZA|LySD?m!)7oeOyR7&=T@c8-BiO*
z7xNJ`J=6r7Q{u8iMCg)lJzsn~YV7V(J{L^jr+Ib3fJH;yKIi<a!Nks3jBcA{_NdW!
zUpFvr2|pqp4-56ajfrP@w)?ll`<QZUYp$O*ZKF8W_9%2rN)x@}*tfZm$G;-0L7uyu
zK@li8d$Z&H>?kC60XxO?PSgO9!7X(AzCaA;>H~VGB|u1*tI~jI21I;}!?+ewcOR%k
z=!MxP8EB|JW_AHLA5TYJX?6^h9|waO(ZMOFu1g28yz2{Gu(-i((>#pYcvngG<OW$X
zIX8#O9W~70c|jsZ>DN2WX_ppZz)|4EwFPU#E3ZRtix&T|R}Ar~CjfiXZbcl#W|EMG
z<n~|^OOvQhh}M8j>NN_4Yp#2UO=~cq8~pimAEz7-^~c~g;MAg^ExnKR(P`#^ar&qt
z90t`IZHXC?LmMY-gcPuV0)B0>rdX(8!iL(MkXsQ9H4_|f|4O57y3xW|gC8W1?J^QF
z!*7}NqE|<Kvm<<gfWS`3h_;qR^0;EJ(?T{@)gZKE<hBvu<5|vOElm-Z-o#R(G+W#M
zi=EWHcrFGr3RF6J(3*r5O@`G$>b>=$XOu`xlJfMtiB@5wNtZ7UztL-VhpjVc4_NLT
zbS!>_VhEgtEj;0Fbh0uBCR23?W2)Y0vkwD4@3bQ1AF(Ltm2*^pXs(B3bW5r4jNQQ?
z3gq*C0o_10O0$^d36lkDyzyJ1wtS!951G3p-r0EnCS&gT?d<<;02ph~j5}2v$Hj((
zZDgjMCZOLAGX@8|*7rgMu)f9(F~jmlMo!&~nG4;SpTVqHI#pF4C0a(98=OH++EMKv
zPq23zGq0oYDd>6+BR^+O#4C#tWj|dLV}gDk*RwF*VnGmqaQvArAsgS5eMP&Y?ll+4
z;rzrPVS=hn!Yg2H%W3t)(VJk*bzAeGD$XwDa!$20A?#oRx%uQBYJEC$-c-WpO@s9|
zug^&JA9nOWerKbH;*3O~L`#APc0>YI&Jj5>c0T(RJ^1<Gy`^q*OzL1H?3-MvYztS@
znDbYz3X&BDwnOc3*F}i)U#MK-Z%Q_dqcary(I(ye>&G>@C_`YMtSs=gM$2sW$qQSD
z*KX_kYS?G~7QW|(z9*koDU1$YF8tXKFebmb`WU<J>9|JA`-O~;NAdaUnFo6k#+pC$
zc?q?bZC*Qz)JggZ1r!WsN{J;fKYT-gCHJX=`?vf#$h5B0KZEiIcS9D+?Cr2;fJO@#
zqK{uYor4oUIqr}@XiiH~)&@f{K}TVSBg3Wo93kznl<Cf2o9$7|l^7XB;`3-1;6EmR
z5-zAx#?!GGa_v6k@N<A8vLH4XZHi#3EjN;}jQFmN7qBfG1!Hs<m(J#2Ibqd9a`RwE
z7&YpJWjGCUZi%~q<C|j?pB7?CP3cyg9F{)QWe}gL$MtT)<79>5>347OrU$o_BbQh{
zOoqzgw>UFU2%=*4>E=q!y#EhnZy8l}*R2oJ-Hp^1q(QnHK^moz*px^r(k0#99g+gl
z2vS?RI|Y?a>F#<L_c`Y|_Zj2=<^6)l5Crybt~KX1uTaXO|K-@?z#tnAw}0UN%Zs%Y
z6_IW^nO)!G7vbJzz}24-k%Qf>m+n=pzkm`R4JA2sCt2ICdWRi00A)ca;|yVElX?S4
z_Tj9mSV$|7IJAEl#1sV1sci>B)kun=Y@g!(<}W`*C!j}r3x_a$X?UNTayar8)*yeC
z#l7Uaj`tsBBbFpwF#IgL@jZ{s&O@IsV)6?jB=r|d1W<!KY?49^Q9L~9*$CoFLulWD
z&%G2PD)x%uxPU|aBUBX>yo@P8`ayW4l5_`UPgkPgllR!jUUxG!N8}ggy;r}DuIpM^
zv`WWAmoLWln?*8(r*j&4iofhc=X10kV~A-4W(wzLxXr%?T&oi490I%MTS%pP^Uc09
zSHw6&3=Rq<Tj!58LTKFS2L=|Q6&DJZfx`mS0Gl}#2DMS{<C0@h!<p5fsNZFRMz8*1
zY%4slZXH?lE!k`pzmwY%iWG^k4^eWF%z<5}+gMJ%^JWG#mI4Zd_n#Sl-z^>Hdt9mK
zEpCPs725B<vB;9GY`OmG90<v2J)KbE1)Vd3d!5JGw)Oh$(-*;A<&dzX=bhRayqKTB
zfUy<HGs2laW&K4r3gkyfXtEZWP45B^s7YUVM6_TyllVT#91t-9=8NGfTWw00C!8_#
zRIYf%294b1hRr|q&}mA)<Gy7*1m+*TA~a>$(G8`QEx#iIP)mEE_T9wVndE}^m#fST
z1j5=&z0V2szT5tirV@4<Yky6;%BQ`IOK5-@+pnTv&Zqg;ECqd+)&xU-Yxe0=WnftN
z^><FAV_Psb1=>pzJJ=D*U0Z4F(x(_OgFF%c<VMML52!{O&A5R{QG$sgKKGx5L+fSu
zkOP@_t>Iq4os#cHBX&?~mkPwlZYqEvagc5vDQO_E4#G<5mVCt@h6NUjTbw%38ONdo
z7K%p(r+~Jwtj?h-4JC8D#ovEkp$aj8gpAY#H9*GyJMvH-l3U0!)}4(T^?0cc2ZxQf
zE1sl25<D>f*Mq(}e3UM3UxNEb384hPhvYHNQ>$pix12`mIwn5H&$lE*JgnLcq!Di$
zjHB4AJmIx8{#tU5z8d+qnOHY%g8p?1w$^V^a&KNAHsjK9>VM>{Z-o&cTFzbxkMcQ-
zn#9S4W-b6!Ap7AVO^i#ziO{ZZYy?xjcbWnNsKUXIi)`9~RceFuf@j!BzC0U4?LsJ6
z{nf0=p)qjT3d2H7+2}Gz@R)uHc8tFdoLC1iZQsc8W=~2Y0RbmBv7cfy0pEvCpG^X-
z%9Ij|r@id|VdSwP*tpj%9=tnT9F)mWtjzt?4@D~Z!GwP(<gEOGFX+mN>pKC{;Dh+x
zu0~Tt>o1{59i}T7zq#(xhDP;=c*6A^^hcENk!hf>aIzSlcfaGN)|DS){pGS(7nV7N
zgHVOFl=a5VzC_~TI-z3Us5VpBX4bja%pS>4J*3SKRx_~&`>K&vGxA-&$*fHf4)aHQ
zRv(QT!>g>G*+8s5YE!>lY0|YqIZtUV=V#3$s!XafIVAaGlHntEmNM=y9zUqw>bSms
zt#pHX%AK!-tnT!{#hVAq{VL2Ib+hPB^;?4!Dm#XsDd7=HS^%DbF%<#aLJ*5xYSv8*
zQa}B02*a%TFeJWM8HU}Dh!P9}&0(>Pj<}1CWFQbiJ5x$g^h1sUia*2;KBggFw?5oR
zWQmO+!_&Og(~vyviAKgG;k@F3=-%A}Tcc1q`xopWU@9lI52Bshdf0*a<J5^4502{w
zI5A#C9|2ocEjJ{E6TPPmf7)uy2Uq&U^aKXx!?2B|oPHv7xDhWL(YLAt!)S|a_uwk)
zW|G)DlLCKFnb*cnhvRZI%&P=hDBQn65-7R#Yu}SkAPb~ei4>aji+Hywc5tl@5C~Nd
zfO{Wz)0j!a>@q^>ifK{6KWgl+*!iug@i?F#YvO>yX1Tw-X>ww&vs;Aj9h*>d+yYGF
zHInmWG0G9Rqg3P<LZX==f6dCqj>lf$c9VbqTifyRplRoM(|*$qEbrI`@|ZoM*Mm1(
zf&WrS@Sc;~JA?oIUm<9DE`aEu^PEncuJMNi2Cq{6;9P1!lf|f#5p1ELU**{c>|~P0
zlhY8SX|BC{6!v;#_S|oFIp5T!>bxdZHzngGNI^#1b$v44?<(`TFLGbT=Xs=R3Xv4V
zSMoa<+TY0zu?z5PD7?SB30ZKggYq|Wjs%&Qq#84#4B6EYo#P-d)tht%^Mbh3x(JHH
z-VY`qwlyk~ZpV!I{qslxuFXk=7iyw(5oie12+p%ruz5!)=g7$dwv5tZctkR|nNIMQ
z1a3GlKREHL?JLNLXpO=&V(I=PiU}w58@35c9%Z=BcEx%f5#?^w7jeQ=@LVcq7VmXt
zlfIqSnou{@>K5IDEAn1z<Y0*-3C9aPgUYeEzQHhKYV-%iOC9*+q(5Jlkkp-S{`)D7
zH?;DZcnG(-6^d#7W!6rT;1l_!*cNHF`(lg2o$+_x#?;mZ&9uccC>|3Zn%f`R<;dy;
zI1fhHZP6!6*8{{09>2e4#0L+S>SaH94K0IQd8V&&pTtEcPRVS$;_=%Ha0%Rr=lQoz
z4i(-7)#J7nK08TTxOZi%#1cPDhZogEc;R}T*OVKxHCPE7_yZBkz>-lQMN-7^!@Q)I
zjk;bwYwdd*?UNGpc=fFHC$Q%+yAZ)$2`Qo+CCBIn8DhGiN@g>;10xD{M1)va^@KAm
zL3hi|z7n|cY0Iqb6s@@M;Do53xwEhK#uR~q+A_|JphA5&lBoceGU`?kEc%HUR$sb_
zc1gG{{@qW${AL*P6Kc)m#>7|jO3A6)Q{Ug~cVL{UA($bO;ir~%;J#D(w2Okrz=X-V
zWaB3VkM&Q~BNg!`(0GRN0{#fAI2DM4Ir&}z`H|rO1UTJPrD@ULsF4@5X1E+rP)G<F
zRZlzmMbj)HzT)t!M72unvuAB{=u*8IA5x{zBfTFJ@HB=`Ldelh|BiP)K@>Hg3CKb+
zHr7gxnyv$c-+|vIGQCAV^yyOm`u8t~TphfU;b2(ke<qs$DD(DhON+sqwxRaU;l%Rw
z^9_Z>q9WA{OG<veE>_D&Yo<wZX`h>%Qz-=tY2F@PDw<JSlb%MsBHm`OS=3M!lHK!A
zOXrGn#Fc_WLEt1pOE9|BAK%dlE|kxN#5STOE?Z2_Hw@2+{Ql)m908o1vKlw(<evY}
zVNQgCK=HuPfxzVRN#%5qQO4)!hPsRwNFx$yk%)@+ZTV|BR@L;ccGD;b%tDG{bxCal
zSHM0Fj-@cgXVsTsbCf+igXl@p>WLwwV0Gz2_R^>00dZL1tJftveOl$mo6i7A?pdD_
zdO@$pnJ(Z$;QO$y12s{dckIV0YuhSTlG0p?2OCC6Cw?1hLi0gGfTZaQIkrKG*#X30
z1l3ea;2rM7qc7_{D#Jc+-ys|qJz0T`-D{no_*{0YAw2st&)>ZSO=wm(d!N>f!a{XJ
z{{s8?)TEU)u=~pz*M5}0Qz70_WXTI%nt-i<t<i=ryy$H|0>paC(x=IMcDPofI}3J%
zD&$K$M|FEcXJ_Fvk1a3;AVp2CW1#Y)N61Qbf}Lb9hn#=X&#PnYoUCCw2U7GRA;%3j
z3nW^JzOQQjz~(`*H5uTc@t~tB42XN)_glB*PvV3r1TOxTE{4;7jJPZocdk>LZAKW`
zIigb%j`cGNa<iK8^B(}yoz&7Xlm$MG)tUgpDPIA0oC2&+RAVeoGZee-oq<O)eQmMn
zWhUT+Tv-TE=s!x2wk^})d(gn;_V62BH7!gR+cu1|{<k2Z-rC%|JOv!tp!H(JCg+x+
ziT_;!M4QO~U~&ZUP~)J3gS}03L|pu)k)+IvXZgFfq;3#B64^L59&mKrmdKBQr?1+3
zz14tf+pbW2Vavz$wdNnr9+OB+y{e9bzvKDNoQ`wbp7_q6m~Y)??i6|ZjdBGH6hIS#
zC6hzDh}+yc7$bd4%%=Pfmrn22S~t849U%_wtv@){FZ|-ZFgMh9^zkt=hE$o2e`wx&
z$G|G#GTm-a&Z}hq1TFDhriPF!uyH`9AJ2WciC0JY1P7^wM$8q%M^KM6`#Dfae~0)A
zab#b7oi;avZ)w$8NGc{c_^wl(exl>&TF%x=_;6%OYR*cKRJwGV#6EiRmVZj;xz8J4
znq4Z+zQjT3I<4w|#D`312$!i*cGGMF>+VkshGq`y16p-@zl-djm&;uKsSdKFIOu}C
z=NbPQC0e9^aQpr`<!`G`DF3}|Z8j-MN4<2~fza%LUCB$hMWuaH)p_rwDK3H$EF4GC
zQ@%jcTIU^-rY({XjUrP@`sGDmpW@e!zL>{$bK^=TU+<c#h5CfuP$xCZ{8~eVSUtQo
zng7{cWIpev$``ygGJ`@kl1(~`J5-5ZT=g!6v8}uAya*4|DC)%*s|aO9OSDD;FZPRz
z6P6~e(xbuW7w1cJU#M80b^-Nl+y`pm$mF})p<cyB10jZ5jm9`NtNW#VzS<oflWR>D
z?^^jg>iyil#}E5U=<&<Y6&TL9+3FOo$VJq)cJ(!W49**fOZy2ls192{^+VM7pnkex
zP504ia7<ioGc_iwC{Nl-;~$PTKQ|?T@1=kd!Eo`TxOEWKd>~IFmxT|-g2iXAzaVey
z81K>iOE&wQz+e>c8A`nGuA#gx6m1oV{RP0y<<a9s&$<vKp~-uoqCxKcr&`_(Kfz4s
zz#t0olk|?R0}v7jXC0M$hy3ON$Gfq3J==5tY3yixKQJv-;)IPI1Ad+cnHqJo?xU-u
zp6oqfi9^bMrz=-Qbb=%ou{tMCyi{)U@Hz@a?X~U!SU*yi7@4D`l*n9dm_%R!6FASu
zs?FrUdLiC5xZg$Ti0D@Uf7tgD^R>P>Coui)<gBA$BYoI-|J1H|k6($A%{L2DMHiS1
z3~jPLG;J|Z4N=xxQ{_F!zd-u*n!@MhMrH(M2_%*Up^=fW?g@28RLsz-o(N4^Cm%q3
z&}1QmSV)iCk`6BpMb4SFA49x~fzP!^dT&daqRjK7nGILKufnhbDGp#>qjmB-bnr)w
zf0RQmDP3~xa2fQ@q5ss<KcblbEcC$NVzq#Mg}}<fvSzs=-}e)w_4&8$ZKI!O-2J{}
z1mhH%4<6m6_0uKe@Y<@;3%<!h8p&G`EHGpZp&(C6wz{QoroJ4f-8?Y7D7J}wN1Vhk
zV|3|5=6p*+@LS!<&hfX}(~H*3Ov<JxQkqJ=MsAv5)A2^ST%LD%O+b~*Q}p3uwf#0h
z{!;r>YN?!4$VVcdr6(PM>}+~|{mO6dgCgLOw4H5#zp$ApE2K!<tzJnnLTE#3N@VFG
z_dp1LIX0R<eMjcg=fM3j+(&?CU<}H`*t><`>`=V3Ilk>3Ib~;>I|hu_ZD|YmepxEq
zn2AndAU}QW4jhrv#vZ_aGGc!zO!;ivR;Ca|Hk0l%mg<YU-Pi5S`z;~jp9Mxml#b94
ztRmWE?M|$9U@KawzC&k=+f|8?k)^=IVY*l?plZAMzErzBo%SXmu$(^553X+-f}hgv
zyN;oae&^?krNB`9P}!PFX)O<Wk~a4aw0tnz_54cU#%YVj0`q4uZ9U>mN4ygc-em%I
zk1-xRffH>SrwM}d{@gE(_OIu_9qpt;jT#o4X)))jU8XC=>5Z-!8yRHy0q{>dSkn)!
zr4$O@e0z7gRrJ|IZ_5~R2#8?%&Agf5h(G<QZ|B~ttC<bX>4v0D4u<7>fZXJPQCgTj
z^m)srsRvj;h#H>imxWU@fKwVcFHVA1xl^;eVd-eySqHVWDjrL-Vk@?&Bie+pvWf4z
z);|wWT2z}oEMd@(s$+Dy*EMvZ@%cX^Bof-&06$-GF*1(xtw}l5q(5J^SUonW_Gt<=
z{T<cGN;@p23oh}&n6c6;YGxyvw3xvwNPl}ss`B)DY;pYg;|=g%GVuS|RXzT#4+p8O
zq*>K|`&7Dl#NF>nfL))ip2PfaXQ$QI8&l<%veojE%zl@<WCG7_S(lBrdUEwbugvO6
zmOpb*zv+<}e0gv4!^4xiXrF&$N=s*EkerZep?ZVnU}?uq;KVUJL4)+K$~!s0&wuTK
zZ#naY-MO`x^_2l<u6=XZxIU-cn{OZ3v77f?^uF)PmE*$=_2+!E5)fQRf9nuX@f}z$
zB~Y>r;${>IRST<ti7+NGEP2n>88|c{`#_fH{sJ4Nu)P0cqZFNN<iC2-`HtBoy>j(N
z`Td`RZKmGSl>KYRoz1b={E=U)9i1&g1ncQ7W$MaT9hKKIk0zA(3kV31CW!s^VI6-|
zMTKw^Hr@DUhiy95xoL55sZyXG%v7#(_ER7(>MQWlRP4>_lZ2D8F1yY#zd7GSq7|Og
zTmt}joi;lMI|@SE7r7Yf(?{EhK=fzVpb06aB8PgNO$!Eu2c{v0ZM}sY0$svat$mGO
z?8{*3nhQi=A|LtParbAdJAczlri>LR5F(U72V-E)i!Cr>YZi5|B?j&lNk!?PZSZ-x
zt2cL>0H|OcOQmb9>{`DJB7E6(3>8VO0YGj}ft}Zuv9v$~;$vH)JX=Sr?2m!~wnIo>
zN8O|*UjY_0@W^C?HAM2^wTTg5{N!v0Z?*BIAb;>IC9;cMRu8{jFRA9)6$KS3*#4%C
z1>1E_|87bXkOTkv!e=)xs1=yje-XWi68?X$)!*zn1zaH)<bX~W33S~7T2!c1PIx9C
z0&Ap#p;g9yu9g6ls&O&w7}4Qqy{$?4;K6B!z^@m-2gx}z&V<?i;<gz<pc5n8DP-Y9
z>AqUR-3(}a&e?fnmQwS$UZ2iOq#hXWX->^gzpxgSXy>ej{_q&6deX4`do-0Rfb)U{
zi&+^FzP8ZNHHy;%my*q8?stv35`^EU{m1!lB|}UD+}qsYa)U;7!<0tH%?W_#X1DrB
z8O#PDoamp*){5;4!*|*k#MrgoC4+A7y(QxXwL{-i;A^J6cf1Y^;tL2>1=}P3{;B5x
z%SQ{S;Lypmj!}s?aWV;U$a%pZ&JlF4Pave_abeF)x2B1eoIj9!X(|oZ<I9PGmC)Ks
zm;RHFP4-{1f&V5Ifl^dN`d#<_#eB2jn!~nou)wWxEMB=Grs{n8E%YxVjy;SvxtM0A
zaw^ku-Fd<Qn{oW6?$L&9v!ut};L&hQZk(h22-splM22UMyuN(@uSzNO{{K@cN_UcN
za%14D5X%0auR_J%iqyLqGOWJKYt(xoB_hBW63kVkN{;$bHv<`uLM82XHGXBv#3@;#
zUGleN4z|I<Ma}Ffvj(iy>gheYS#QsigT=mXoOg&WMk~2(VVw(#1E>Ad_WaL6Cv~m}
z3%8D?4NHu`EMV~+9uQ9j?spf9)oev$T*(NBt3O^R_ETmL9cgI&UQaP35w8e}e&gcF
zSDAH*oMCE%m@*WmcHqA3LtOgK)^i{1S9SroZgDTO5Tt++w>gZ0pzR0CkSzEx%lFnH
zDv4%2XltE*AK>`)1As5?JYfE4si?gG#x(ju*bmQ(@5f?~(C4NZLTZu#OWU&|vIH$t
z*om~FIVzXgM55LyAV-{NML$pJKhkzb<Mb0cJpfo<wUc|@S>nz+M0i#^DTM1;l@x5G
zGFno0y~N|{YN?1MQm-MlKc|};8+uK!!IuRUX}I1C=<Nf8Ywdk7W%X}P_F4Yx#ImX2
zlRU<?Bl4&X83;GauT$qfZ(h$zwy_|t7Ce_o`vXWKtQ-jov?m}Ey)fhNb=nztC+Ps}
zXhX|7hf-X;=S1@NOB*%^GGBA{edm82T%WqzZziqTTp33<BN|X4M8VvXD)~ODw;hgf
z-c0#J3^0&n9-T>!!ZuAMl|?74Hv@BYBv+VV&)TwJo#w^}Y*i=H6*Zz7-p;_A>Igjj
zT|l!{6#W_fTv)FWWRQi4a`59-IsHO=t}Y<BFq$beIWX7*&%AD~?z~$!1O0Z4g6PIz
zk{?V)fnT8c4$$jOVG)TsSQSFi7a(vk`f<Yy{WKPEbv5})g37GML-7oWi?h=fVoW^x
z_Tp1gf7dM;g^#W|Jx0~ITgx_Aj2I&$BMB_q#MxA}ON4p7RD7yEIFrkmn!aJbf%@j4
z)ko3Uz@eXBO?F*KJgKFaO9ZfC%wU*rI)nk!pJsLbtEt<<*j1NG=Y?rQk=*<N7in82
zBw*j&q0DNwr{GSWCY~SiUh=nZExZ-t1WEz6JVjp>|I)|pHe&e75vN}gzm|Wiklj!^
zZmWB`)PL3pe0aB-`}5H0I`j~i9Cf&T&7xU`a!|$!{U)PU!pi5|GvMv*ix7>AOdgV^
z)y2~i4};UjUi!Y0Z?Bl#{OH*(6Mypi#@GvBH7@MK=E5wPXm{6UBTN3aX_#>+XA;@i
z?^f?)%P)SK%Lr=vbt!;Tml5C8|5xzO84A(C%Yw-}2X{F{=SkNA?umJ~&5g|-rMr7)
z6J=x{vFt4uv*w9=Uh&`4mx=rLuC^x|)thMrz99`umqj$YoFofQcW2>Uuk{Yd9Tu7v
z-=_Xq?fi=iTW2l3dH==m5Af|4e0l{J2GP3(%?=k{=3@rQ7EsR|^Jf}?vy~pN6POQE
zBbta3x1YgDF;wnV34-AH=q(@=juGL5n(!Aof;PmT{2yI0^D?FLo_f?M3Zfqfcw#ay
z>ym_AHT<ok0uO7bM#HAZXrnum#xdH5tvQshGfHzCL2Rk`d-Wh3A>5c+z)u0Mz0|wc
zt@|`_6t~NBs8s`hL5t-(;6U}3FwoHlFb%UV_pW0`z?IyB<fBO96u&lr$&__I5`yAQ
zXF&7;*@5J{btNL*YE#)=5cj93_Z*8;+3%m2b@qIC9<je6<2<2P$c;tB$sho!my%Ev
z7P##H1kJ$vYQ0($JNpcL#Z;ZWSLBtCYr>w=t({#3LueFdj|ki|cAvwgc7;Jjt*38{
zWd5|d2JKvlkz<jo6EAGB^)#nE6HD>073<gc6=K@o3R`$;q8dBmNFv1FDE?m$SM2|N
z+K3z|+P)n`&ETrRgnH$Tzt+vJVY%WjkTUvd%w!&s-bY=`{dKqseAui-UzS+$3L2NF
zU0nN=?!7<Q;WKsM#{1#Be&%hjSq4pML+6HRv&ZcuismF(0-K>bY?I+4lZiZ<E8)rH
zm<C}utzrwjo@7wTd`2B_Ci>-2CB2?+W0_nGEq@@IOvU3KIbGB_Tr2QbyY$_=XAUTx
z!W~gDN;t!-H2x(tkIrF(4#c*-l@Q*T&$cB0^?@2zoUf(n(sWs7oC)BQ3Z^*rIA*B}
z9>(4G{%CBDP?4jfFk0WPUIs0)75v$U<&S;;>grK-i-FMXD>>RIv(6H%b2?QGTp<EK
zVVD>Hj~75up_T33^;wt#axIYXc*-?^L|nkZ;pI9?cQaR9(HafYAv&8#`y>NG9KVr5
z0sPHpJ3EG%LCqjN26j*|CI(?V!u(L3qKlP0QDqD@clSE2S3x@R%FLB1PrEur(8d71
z|F<yH<+oKAtDeWQ4D|wtoE1CMe1&`uG|4lVa_(+t4jyZ9a<I1yqT}W%h8sZ|XNT7D
zPPa$B8UPOc+prp)51ix1%3ykp6X+_Dp>Gl4O}WEU23|f5+(|K7PTB&(L@n)eZfjaw
z*%G+Q1;xP>j-r7iR?jey-NOi#ngag^)MJa@6d@~R2q`zQvM8nh3=lFH$YfywzT0%d
zk0Ap2Vi$87sDw>`dPjP9Jjjualh7HepbecpUD7*dX@(Nr!hV;Ls;<4MGKS}RI<Fyg
zd)T=Jq6AMnzM%HP!RijJ&j6$S6BQdbxS0NevHpBuQ_D9@N8`LBXm6PcDO6|nB3$+^
zM*O;%RJFUabW9y+T|}pWb-_4bV{8Tyx^way!^Yx^UmaUF2HJd~A3vKE{%G$l_=;Wm
zOFqS=5br9|h31@o=Ra>{f$CG*_`ewV68Tz-gerRkK(b>M0Jl+`z@S1qBy`LIvlgp+
z|A*g-pJBiB3YCKP+4M9<!wjWonb<pqbdh{pQPi9YO=L$Ah9!MRqD<ToR%Oo5T$`n~
zaEc8Fg|w4jRPq5Ib4G&-(44R-1;;Q^#Zni|vOnvze|rVoY3VjqnPR|C3TCn6@OJ@N
zXUYjFjocKT#XA8OMYm`aXm|(OOXhc=6aNb!iNlT(k6B4yad7c#PKQM$O`jMKkYja^
zWQj}yv3g|c)DMuhmJ5(Q5v$;DBf=4fbFnEc{?bQf0T!&H0W#>1U#<c_DYNKO#!HPf
z(6Q&!^U99q>JrCx$7cB_v}1lZlI5ZH=X~Bjb+6t$p{}Kj|6lILe}DoY56oV+Yq>|U
zo|syn%4O+hHMx!(Y+Z&-W~FIdbYPyY5N%8~9#xpp7+4nMn=%Al56izr4|Wcuph=-j
z?=2BYeY<gXo83E1h^2Igg;kG2nj+}t4{7GK?w!It%ZST0Z^ZbXAK^M@O#brgKc0Yv
z-Uq@rcujD>qCZoHr+v2Vp!lb>|6R$-4_BMqMHNbn+R|)FvzNW&i)?RI=c(v>OL$`5
zP=VhhumFjI3$^Y1(2{m|ghGYlV~d-^z%wEfOy>rlrdx*PC4S(@Fes`>!S63>;ly#?
z0fgwt(Yd6Ibl+HmzeIC8dfFcv6Ckf5$<1Y&Ag9uQY?|V@RJs8MY{D@!Aq{r3Qqy%E
zE3vQju8pQylC}m|D+=;sOgk_G;2V6=@BLR7;|n9K{+8F5;MQca(W1R2K}hlLxHvps
z?!A5M_~B5$b8>>0U)}&!4WTPfCeVb8I8`PbFp6LVfwWmE90z7DfHJ6ul)(hloDQL`
zY*q0ZPqlT)r4fRxARiyiN{L{pinNfI0Xb##ttbe>tu7P-ACy7gm=Cs9`fDBqtHK*W
zw;J&%2!aa`a*Cd3kRKdtA&WBH4r}s*vj_!_v7?wu4vz0v#0TcV;Ss7st)kZrSirXA
zv1{F6iliZX^NcW-Xr|K0&hSA7>BRkHV;o<Oa+w%&Wj>Oe2li3ToqJr;`0rRQi1FjZ
zjps7!dHq@33-*x~7t42yzYcb}5|L~YA048<jg2(GeRVd0mPfl^(oi3chVUy;GT$*&
z4+mQ!fBjqNI~)w4v#pha3tZDN@|qUT21H+iNRrK=wEP}mDpSguua4(!GgWM|wIV)L
zdaiHZ-UcF$RV^~@gNd<eZvZz-60Aydxmmh0DZ9?$5mE3q`b^?2_dgBu&EUyjNG&H_
zPtU<Qa%Q4NqI1pI0%yk8T0s#Vl5j5zVV=q624i(LGx9Y;8}A@P8NsQrRX<?~yeZHi
z7oe>{u2<7jdkO`^W}u!?BY+JvcJkuwm}Hw~S0>)e&{VT_v?qbr0M<F`K+jU)Tc?rm
zhNU<6(ih;V>4GV<Pxen(eEot)1Q^F?IE28f#EB6i^<u1w1SQdp#Gi{bQpGsO-(4f+
zxgpwUwBDfs#tXwqU*H@}x`!0J1Ag>LwSg)s6jl%}FMgzoYdwPV0>j_}P`7?h2B5w?
z0=ve#l_9uD=gvsZM@Z57XXK(+qr;!~_2U+w=^5NxK}LA;_>YlN!_!K;maCrh$^20>
z@$l*(*D#Gc%9BrI_&Em|2xh0H6-@(?1U^lWv1@2vW<p2IyK{EnY>57fb$8aoC8Ik0
zf!_4_2WEx;jD!Cz0F}pwPxJhOrn#l9ktyum@)J|XIM=aT1%Zw09<jgc>@oW36tC5D
z9CIRx<Q^LWM<f+@VqqX^Y>+73&Q1G^^;WKykra)W1@}c!Gr83C8rEqU1JfSI!(OFB
z(pRVHq?E~Vw@OPmYW{n6=&cyQV$KORr<OU_Uvb9r$L3%k1~RzpjG4grCWtJQ3Q9B*
zjou@7+>>iCcrK5}#6<Ghrn-q3Q-6978(casTwU6fF!LlcT8z@Mst~_%ZgR<jHNo6w
zCMTU09|hx>)-3dUmAn@cwie>nYi8DE89inhzppK8czR0M<06j^&h_o0j!>=gAG%@$
z?q?aS{AHhf_fwLH`e(p1A2c`k;|((1Q$zL!50^U=vO57Nw{KxglR!+cVWJ*VxL9r0
z8+hF`&%-=05CT^@k){qce3?23M!-fuyHjmcW(auWo~E0m4qY&B8@PDf5+!o%UUsge
z<`Dd;F^|xaJ6QN}qKQg@rhApPlvs4zzG~{wtVN9=Ma7eukwF18B^gN}=!WPg5Q&79
zGSQ$Yc8dD0?;qx^r{Q74W--S`pl8$;hfecw(<BtJ86u_X5|>o&yj@W@5N9}+-7uT@
zT=+-{e!0CZyu_`>*7H{QAjXwSf%)_EfXC5~26+FAuFtF&Y#$F?YkS#ZC4uqpvWF!g
z>>YDT`%4z>1+e=O;^qL~UsG7cXZL&1ApAeZKaWvLD?vu4zUt}Ky}0~)4E2gNl?Q&u
zzx3np?`QqCd~PRtOabost;dK#xYq?d$zwLXGP`66IqY6@bOcfFVa-W}F>ncsX3FZg
z!@XTcWYJHR^EO86tf#|8Dgc$B+l}~sO!(uXg@dOqvGoT$XYuw;IR0ofc8OF8!4T<g
z*l*9bP=4OJ=%^@0GsY|@^#bvPZgwvuH}gArFJl@8+enH|N`ALnw9i0wR}H8=e+Ghu
z=r=S?uKQO$W~nS^EgWm9)_5G+T3}EZu9Z?l-51$jUUAWwuKpZ5K8z&KbOdnQU2fAg
zrJDs`DUPTToGU>4ZI$sb#Zgz>QIEbrS-cr`^?{|3LiQlq&rTF1?KE2_qF<XwpZqPT
zHQX2*$PP|*mR8ZIQvxwf?%j8D{@#dhAay$%MWx%(wKPOaZR3{}g5mF9{dr+NcHi7L
zCxcDtzoX$A0z=DFDi-2lD@hHaO;#FTdYmXP8O>gbQ`_`q62*`*-U_s4p+CI3jQQYm
z>X!BH0d1=^J6rYJLd>!u`9leJ=Eb{~77Dv$&>|O}&wO0GGmSYBvHcj_UX;QAYXdK1
zb@BaHX9;T{saaE8*beS*f7zcV?fP`xA7I|0+q2=Eg9)it&(&Ohzdb*ZdFgsx!qGyu
zGtuZ+Cf$gEf$HOX)_-flVWT2e;A!G0dA@Q6i5B0*T5IPXY&e1nK^G3a?Y1BN8)m2r
z?%u4X1t^`LwSwhOx3DvelRv0G*n>7dDkpJA)%Nt#brpN2mqokG(u9yQ;2EX(PmJeQ
zfl8I5c@q1{=!_x-Cuf)hGwx?S1ztqYG&SPM(O{rU|Kec_MA8a7aPOqzkubt&h1QW*
zAzaS@Z5Sq8?ED<4Nt`Aem!~;QVb}Rd9OPk-L%qwCnp{aMU5qHV6_ub*Y>LWq61&Dg
zQFSLQNk@+_BGgax%jC)S#<V2ddMBifDE5=2?<-AL5Q{s+Rt?yie90{Ccd;*U!j7Uz
zwmHJ*f|>%>zH4HUO3p1Oxt5zLy@kNl5QU0;c}DxyO4$urjVI!mkv0`I>;$GHhcyFD
zAzc>-v76druym^l(RXja#PRL;RhBT{tE=G}^&`|#9tji#cD>rFX!1Kc{O{KdF7wVa
zv*6M!hXGbxsOX})9mLnuVoHu~&an~q-MZ0~5T*jFNM-%6=$I{<`3>BixVud|)41IG
zep{?jkvlgZn2koq4!`9XWGB0=beeH01V4)zkfCEtM&d2mx;`-rF3OOrIa=VQ2Xr)Z
z(wEWMehA<;mLzkwZ5PsH0XH1M2Jv!P=juF?DXi(!TQOjxSbge_nj>%MRrdMQaafYY
zM1ew_JIDbvzQi{t%iHm7mwo-AVL7<Z!>NT+U=j=Iaf!V%wfN~|xL}itcTTs;SgJ$R
zvKNqM?}7E<v#H*1Dr0CSl|RlEQyF*lfbW7NFyLk{!)pN21p;bPc_Nsne-ejTlq}CF
z6z?a$^y){unDh%1)z=1_G|pJu7i=vyuMKAOw&QGA{tc)kI#I+n7S*<k)=GW)+~h~<
zkcv~o_8<f!@ToRr9I_hvHkm05%OB9bkLs;=;5zN-Xsx+UGVu8%FerYXHnZ@p_%4N2
zNBn2nB7QGeS(vWwrmrrlSap@G?&OM()zCd^EBo=<&ilMPF1~0NWe$D)aMA7*bYW<}
zzdV=AIk0C81Y0*-MeDo{Lu7ilccw-(SDM;?tDbWgA5ll|%$PfB!hnY+@);n3y2*a5
znK+%Q3o%?z0{ezXK2N57E&#Kf-W;?fW@s#u=}z?(1Y<6u-cg<Idw%<CztED^kDc5C
zyrUyu<jh!ndc{WJJ^RX7q>5_U%Ie8rY1kw(slgJ}JOC&{qmW=9$2wVE3BEhLt%*wk
zHd*S{vsJmII36z}$)sQtEyVM6Hae#9b;K4Zk3Q+9$BE>Hw?L*&58dK=F3y2(3z1a*
zc~o=AQqBwFnjS?TAk<|giA{`{!4&@>Px_HoPfB_^1Z?;LebY*ou_I0Axad*mKB9$5
zSQx6nCrq~O4FI^VuBRM8fuS1cI2>`PnAFAk%kxdVA>%8qu1*w$ZPB4;L(YWg36+nd
zRE=i#r(*^1+^2x=%FP^VIzeVKQoatL?!xu})E(nuPoSB>C%lF6GMs3DK@V27l_u^b
zT>aFaTvnNyk>8~nbW(&9TXAyKT={=Nx-A46G#$=^2qR$Q`!F{fk<Mj?lI6MI2o?M*
z^fxCUR0oT}+s~Sw3W#J_7Q&qn_x{u~GHmKAJUWJP-QMEUTk05nItrR*aMyySi+Q?e
zCO2F@&S!3@I2_qg^qQXQ)8RiFI!>$n;EphEpmDGO_-enH#{pM_8lSK-R6#%c8ibz6
z0sDw>Qk)ax$TnRQjPL!1FtSnCqYe~=|2yc~vPmloPqh!(+rR4m!KeK3@hwH+1?=y)
zCysBO0Ebg<X|8%)c#<Prm9-)<xij=h78Udlsw;t)2s>Ed@sPkn*E4~4$tu-Do@@l#
z#UWFBChmy_d;f;C9`4C!U~a#i<B-2GG8WK6GMCSZYkFK8jp1x>&@N;J8jr2@6eFBz
z3q<&V;o-9hRl()J1|f|7dh92>uN|!!WC<w_JgyI`{Ib)Pd;o7t8-{cEb!*$Zyfam9
zb8QXpfHo&zdLCG1BrSN~yytj-e|yyx><i$7z+tsthXG&?7S~{(dH^&$e2O?XD`4DK
z2(m>zmo>#wV@z5qgItqx5OD-0Q89<jSM8kHo%}p^t5HoBl1WQICPejka8e<dT)JM8
zYa4J@0ksriw%jR<dQ{K$GNaZIWl(_#XQVy`#1D1w4Mcz!quL}gCCHV%YNHV<9iC12
zN_0G3XSEUOJ<fZHIAffcO+|Jv-_!9cPD|NOBKsl!CZzK5x*TWo4zfIFU$zRO0+z)c
zFQ+&-<~OtN<HG-C_KM`UA|(k7S_2$Ort<`ye*pWeZKCd=Nl{_1aN}SRBT}lx<Imn-
z)bi!VFE#{7@9&vz1(rHBk$u<QsM0@^{-kzE(z*1xv}M2Ls5fNlJrlu}sVlmqDK{&S
zf54>jIk#(|Iokqq`fDL9k{H{izX+^lo5PiNRo;4RJ&W>&_Xi>eExl&?*B`f~D{Ae>
zyIH|`l#5ooj*AL%O<tN0+LNxRj@Iir`TUekG3ENP`<c&^L`>iVCFo8`wp+-=#tyHy
zq-a=8Z0*CV5Tu63svKYN#z!h*SLWg!>S^iu`I@@D-qsoJ2AIzL<3Ih3+(?XO9US}^
zbSrqjYw+G;`)46*o$%S)4?OQ!o$lOL#>Oi|Tsqv+N1n;WsRQ>%FD9i+(;cRkz6}+T
zyGr+18B98AJRdg;zlH$R62w3fycHFNcM!M^TzR?lzkV?N@3v|(+GM!JAy4z$7HoB|
z+7NMdU!n;Mg|8J%YS?3%LfT3-Cs->F<vzVIh8rtXB3;qqj?&EoG^=s&x=I0#NfU95
zBrPI*0csd4M8}T}nf6mEk^*$BzrWw4bMkY?6y2)iC5n_RW~dmXHMM|0D|u)*#t&}b
zk>-w$g0DBG?WQ#s_>v6PB<A<}OOPvdjy_0@mrTXsQASk5MS7BO={3k+k5|cG8c}5s
zyDWtzU=D7ouF{oW@}``uMYq+cE)hZ9_tfqQ<Y>MIsQz)xkaP)<fn|S(6pRQXMD~*>
zPLx4%TquzwH%aZmYa@HlWI5*RU_yjsMpetm4pU^EF3||_b^J>3tnMid7K+E|Un!rv
zGo7#91!bg^e~!%{z5{MWfekM3ol#00KX^yrf?XBV1HBA*Cdp8OnZ7*LK76M5!w-%j
z`iBH=$4c;3wlK=YPqxIs6Lc;-rZM7IuD1F-%p#iwhZ>(O(8goLO@zRvo(J0xtQULJ
z6X-KUIE-2{jp0g=U%I5y&=ZnzfYUG}S#N5N{3WWG=lQ7<*9Fd#Q;)D<)C&Q;0~NDn
z7#SLwY~gPAiS*cE`%q*f|AFX+tludOFZUq<HZB&STM@!RHg>)&-k;8ALqIo$797|g
z==$#UD8C+VFb-#8Un$b2`j#!EO(cgo-@z;TVRRu0PLI!JPpjUYnvc1V*DLRBF)x{-
zd_Nw&bl)L;TDM*H6}Rjc8D27i_@@(+MDzvmu@jIIUhDna9rw}soBbc73l<Goo(Dpk
z;vDsNBb_G|Ugw_Y?mzV|RaZon^sc;hGDph~q6BTEm+8_>fcEnF6*%7rkrp=z(EGLV
z+FZi{bGj$n-$Oc<^|ZHB7k<-;iBK0l{O2~+u;Ot#3mWB|Y=L96dZmrtpys)F<9Je%
zMVO`p&HAs7lUy^?3|xpK8m%N!C>%ZnK?YKew{*o+{+%_6!87<tuJ3Y9-gSbMASAC9
z-|dpIPs`^oxASi&cjqCS8u!VK-Ro+OYwL(xk)SD@UwCwTp})KF@yliIbGuRO{qW(F
zj8N7qKhF6O59q{VjvTNuW`CqTg;$1CYA9YX5B3SkKy#%WIK(gV$xqS{c-k%K{A&`E
z!}oyzo)%96LGHV_dIdS}>wzS83jywWuX9xCRE^X-shqIAhYu;Pj4h0Wj9w5Cm)$Y`
z=s%<QvKGs4b+_QeWRS{V%iyYBj)kFFMlNX(J+HcV#@Y_e<IP;Vgh!+B4fTN&PHa8I
zPF}e%>W?ohxUrfqu!xx~v=yqOtauXjQ;q<K?}k^6_lB@X55wD*=KPfN>YXYPX36sO
z$)fU0$RrfWeAYwq8H&&}xp4{JGY`D?l1M1+G|xzM>73%B<^0w1s>Pf1>-z73dS1>9
zn8~`u7UcwvflT01!IdX64&lXQ^qnFpGNNd{a9GFUob~{T=SPEawXHDpy@cTVm1%-q
zj0K+vwUQJZi2{xL8h*1lEx^?hb)fFD#>3Rfd`d|y^i_&DK^R3zZ>FNm(@30eJ3tC4
zR>^|?Iao<i*i@2{9>YhR%|?KUAET`NI%tjsP4efvq15H=<C3E6c$<pGo}^7>Nk6z}
z%+9RaL&;#*KLGT!=HTK(kd_EFfLI4uZkA!%G|o<1W<x-%B8AX)v3wpkTP^~PG2%Ko
z6AjWE$EZRGTsN8o0^~ZxzXP;Ag@7OTj(R-Qe4Gdg4sYLlX$2cX>|cw*CKcPMw`2vc
zh&=KUh&l!UU1WJEoE&wb!4-z_67=-~Hupd(fP8_EW^$MZFHI(Y{~C&h0g--`|F(2j
zRp3T+XuF*Z3(&0%u;_zUJcmFTEDfSeqc5O6&^AKEP~B8IqUR7DXvA2oh+EJh3*H;E
zRYhcY<fon4<s*04k)%C*;AmU(*X;fN=VP2Od^VT(lV>$6rOyyM_Kz{)@iKtq|1Dc~
z=eE#2SUp&b&;aArDzPX_3yVy?qx(_!jh=5j@(NS?nsj7~rB0+yF6S&k{*hSf4x;cD
z5}PLil|)Fsh#vzwENV~Mzzc9tvnKIjPOuROXjb2ZFNuVnDC+{Z;5YRj3}y5y{ozE@
zP3(SzkKkkTb$Yf0sQYQacU49iL4}#oB;COQI)2U#v3uPxT<LpYHmj6rQGPSl#JFsZ
z)#brj;O`HYh?w)t7`g9<h6w;^phsR~Te&Rh@e%dxlUJdVhG!6m0gZdaitCqa0hvE8
zDiaIlK&jJPr6EsXSF_{I3Nbxaxi1<XJZ)?T+To|fj*NtB4H0!8x|WS?DfJeEH%9m9
z+HgPdb?ar%iR0djeDC)>P)WRyNPYIFK0w2QC9smjiazuBN=M;2iNXlJZ1>0b+j_X*
z6!=i^hZ95;EwPmo$L~S-gRmUY^yJw+PiE8pugoTHb0>N{|J`nK;6}zPirSU{2OQ$X
z?rI&j{jo~m<NVRH-!rEweAJ|ktw3x0y`gfwN*t<;*08)>(=$O*BUpbKD@jfLm7?A@
z{$j!F6@!nk(B|t^1W8<mFElz|&hZW<9W)WEXKL{FTN=v~p(>gZ#bs=AHy;yM34Qvi
z^*Nb*((E6SCTO&qUAEXPG8bNiRU~y}lzB#-C<zvu`g-5*I_~+%dmA+=Q<Gmyn6dx*
z+Hdnz_Mc+^?V-tLf=`otw2BQ;>|`mBeG>iBq2@Y}&kOHG*-1<i%^t=IU*8y?&Cl9W
z<haaT411Y#IVrgLi|uJ8Nq*I5b6``}^A$GDkn9hKHgUKDhv1Y4ul)e%3TE;PVc)wO
zAWEJJb5eHz(C*PaIb<Cz_lLW_25RBRn$T$u`L);}fDP-t!Pn8X2nu?m^qh_b>8)?*
z0?7L^>VSPi4>g%3ESiD0Xf&c6psYHz3)00vC`K)<2R2mS)4O0D)=Z&NBlf1uXueLD
z=^iYI?yTc*b_j<o@Un3-0W%$TW#@<Q@P|E-iBcuI<`&^zVX3Mo*GtU3dK{FWrn(QB
z7Op-uP?wTa8)~z#Vf)3DXcLXD7|&ABB$_3zUeIH5ACcu&()YxtTjT*{h+Q5@Qao5L
z4RHA~93xJKp1g;SkBn3e#n2#^!M&0f{CwpuQ?o5t`I^a@r4BhO>xTt<rr51$sL;*#
zmDk*XHMzvc(o)809%^v8lry^U&>R9S1FY?DU@mJJsDvWJXQt@aTH;SdvC?c>GvVvQ
zD+Yky&G&eFqe%s+ww-w_#SxICXFq2%tRl-~y&}R+G?Lq3<^D8uT8D)RZOH%!Gprx2
z;UPC?AXXYU_cKqJ1wI7cr*j-beI9DwZhpDGV*0f+JbEAc-aBu&`oAtjP%0qJ$#*l~
zX|^yiJAZj<{^O4)=NOLL9SK2a@A-5}3Z8XpjF(Qfp?Fhx=@27lW(tqJabRN8R;*<B
zO0TVr^kl6y?&0M<#g)%KZm{L892u)ni4tjs25JQd4EgfNrw*b#p%pzpvFNR%IzN`U
zK85Tp)0Arx4*p*Vl5y5H0{xoPflK=8n16u$<!ZZE!g61p(DhccO21k{Qs(h^FHOI2
zgD#HXH^Dz5t5PTEoBDiWG&b7b=LJPNy{eV)?fLC{43lUHv{j-H@{;tcX5Gv!8aP~-
zX-61F8cNk=I=(90hDk)TR*CCssR}igvY<L8)hMc?F{k0FoBGo~mtT5oiBY4(ch?e+
zYNRp2#Kz5o=NC!WX1)}`9-AS#fclg{8xEg*4O6!`gxEdM@)@Ir&ufBI{`YzYYmT~4
zdIhWAL5ozWnlm8r35HLj`|erLj6;jG5-0*qHwPJ8b4~V6YX+b5{9G?mPBYaBD}?xl
z3v!5q9u%Rp_y`4Wj02>Tz2J_ikE*9;%K>xyV9U?rAI>L=QgAZgT*W?|`XY-`%3wk&
z%v7uXK6y3U<Xrkla<OOq^epcgoXzc_Hk{IlcBq`x5<tMXptP{$zvbeYI&MpF_}gHh
z<1>3>UQGnhWw0YDm2{_1rqn-M)+zuFcQ>-a9bN%H0*PmIpMugo$Nh|cLjme2$0t@o
zDzpr!nhiSJg*joK%Gshq@4Q&hm}JN=ZKE2Gg6xO6q!iSHG<(m>s%f<-Za7^031nv2
z&pFIKC_eQ)A<1q%5+}_}h;ae;2-@yk<D^@Z!}l1H-ZzL6d3sO(?(S;g`OZKx4Oh?w
zZL)w8!ev^92<c;w!MAr&^FPB+cYjT;`=M_#y18~-n+3gPL2FC+1#>^44L$|$DkXik
zC<zTt_dAee)h?@w0G@Dg`^38YqY-f3VL&I=ufUj<Sh~-nB9#Lo4th=uD%y<mVNyAI
z8gV?1qSg&4Z(3#6_A*B`bl%humUxRRo_m~plQ7NrZ0X4MzfB_5W0Oeb!o0p|>9~C=
zP?a~J0q=thADQTt?!c*|+@b{!rD-p@Ib4lN(m>^iTa$=;Ali^1m$E<3%q0JHh2Q<s
zAB%7D&7ZFP2eUJotHHNdh(-Hr6^w)Z*U%bU^38+krU$Oe)~&XZZv}!)W#1A(F{GmL
z+Wl0nXz|FI5``%9wwo!ux8IqRjp3C&lQPM27$aP{#2O!j*3Dkc<1Uuc4>BK@-+!!g
zJ)oZAG#_DLn5^%zrx3|ozWzmye^oJ&SwU6p%b?e(zalS^SAkD^Y*-kr=9mYsQtMye
z73U&jNGG$_tmug1+=Y;CiES4KdfTR#p?$7N!Y_xzP<Y2N76{~i<2y&;#T~E=lTYDe
zTITV==%Z`twx2wuUDVnp<70d>Cd2BcL}?{A#V0M(Qp#)bT8=Fz?B`*!;6!w>O<?FJ
zof0!ZSCk|yUt+f}pi(<cOwv-;qZwChD(?0DSsvZ@6iZkfDB~<$L@X`z4JF6zqvG*{
z#dr)zp6i9~lZ8r>`6+wiCtB_FCu`_3AY0}8JOGbb?>n@upmi5rek6DcAU+BfG-D=`
zp%Zc!L9oBci+Db9GCGaG9U@?y2_E2nz?-0hJX8ST^A-j|RWt@|C~=d^o^@G({+3&1
zRqZt(4P90sYC{)255!&sX6HX&Htr^P`+@cLz~>=>!VWgB`0|H8oHxLG(L?p!o^FtW
zJ*0Wr=8Ge^pXV0>Z5R?|pG>155?AitzQ|YjbT9h=OtUXSS-UQ!?nuzSN}rcc&GwDo
zZeebpLXmEy)^I6IvVzl9cgk*}AgqWt^A?cS)<MI{eS^w+8Wsq8Op299E2gb@l~P>Y
zCNIEZLh_7s77TlT#DtbZGZ)h^gMvf(SIJjSf}`a!PZao*Zg_t>?q@Uy3)goTRoN5D
zt!B4ClJPS!4=UAx+K=zBA?G0D^XO<GnFfa;-jDYB`U?DPtdA~fM=w`uoovrq&3bVh
zKaZsI4Gx4iY!vy!p@b;E$VIyN@WbPbb|b)Z1rRC)pl?33-npERlc@DGn)O7IPXJv;
z^D`(iZr@d39NnG^uIQof99Sl~2$$26U)x4Cp@LIF4PL?2<g1zI$RFAyoMWTIj2|-m
z_xnaYbBR)L7H{z8^N~e+4>LPxh&FTKEQ2g9HkEbod`Y>PdfX9-ke1y0$ybg#0<Jaj
zKIVOYuy+&r=s{SjRhnF>E>{}|qeh|zAXFf@vCj6`O7-f#UPT89p&)<a__i-=06e}}
z8%E0{k>N4Xp(j{=ir6wE?t`s2QG>DzLq~lpM$8J0kF`I+jafipst>7heEq+*AA}ET
zKS+S_?hA_QJmjvtEZAY2?L+wRYjox))qWNeb9fa?POGSWjgfiSK}!m=hIm5b;t06m
z7C1LY5dHY;oiIh|&zbPA;^?EHl;a;ecOxH5VU#Q`e@*k)W#d2fC}b#N3hFWVU-d|p
zzafK|Zi)P_dfWiR9Ww9ulYiBtb7VRpwq+jizv?lar<jgpOvdJ4^*F_4t;(Adrv0ya
zv>M5Y)|Dh={*u{5r&k}Jpt?(2o~W-=8)}bktf%)`LLTYJ=Ry7XP*u@q1x!^#JQSQ%
zGI}sGLy;7j;frJRRz&xvG@@=+t8o23+AyK*PT~qQ97v)0_y*R|sU$B%LAHj~Cve!j
z*k9YUJf;7##Lz7f+!xV6TywCn8<AvV^y6x|R$!)L|8jQtr-uds5sEBk^s8HDAEArz
zF!M!4Dc3{+iOlnjjng!Z8lMXq-U^e#hJ3vqu`enV%%9&o3uTYe*zn(+?<e3+ee0jK
zUqzBZL!n)g{_i%{AcI)7ZQB?e3fKxltgM5&t*CE2N3*M>X|4Svqq&jrLbJ>nczPrt
zs=ya`3XfN^DX7qk36Tv76CC{!<J;ynCWZQoCs6h^qx<6XP#y#a9&tITTLgI9tK)Tq
zSD27oMjqfp-;cLuLQL|OEE66$vk3snb;&bDhgVKe$hc%9;!X6AU;>1)kH!ZYLXtg_
z$*3-=0)2nlu#)w`X%T>O9jv9><rhNV+24VDf&P{_93QlIBurzHElcfmzn~tRB#BbS
z<)oh2Y5wu>@{#UWw!IAvA&N;X=4<GmkpT?M1McW%Irm@G7f-7Y`mrvM_d5X}|A|oF
zpgqs`1L;HCsM+R@h7B~lV<x)k>OCH{f=}-sx?8=Q`Zl!g&4*Obkq!-d8pliE(9PQb
z_k98M&bjy`$5M*l5weL7+023-FXAt4j^vw>y6~fPqA!k=>!d-gHMt%<2Z(-6y6g|{
z9BRZ@)2^x7@?8OvvHox)h=PktyYG-Q|Kwbf?|Alr3!T|@-Z^DjdB63+dv(5jL>_{G
z(CBzj_63TIAV?>)X+3??yh0hm<4=qcfa+&c`9nL0$T&z~!L8FRw`QWhsiS%FjYVVs
z^kFo73_`;zd+`&NSDkb}A9*vco)Qxw<v<oIjU+r(4<#>t>ca;*G$;GK1*9+L2dZJ8
z4f0HVDKXk}?@=E=eeH<Ion@os-G3%T&mUL6XsU-_)e4dP?}Ui+I3eORCu%$LW;Tic
zDEz~0qH5@NeCb6HnP%Hml~h~t>uPE7J#x{9bKCVPFY=A4l4D;Zd6I{oZIPnZkE()u
z!*$jcm7UKnV!%cJmnv+PwWz(_q4r}<g^(M*kJ*{g;>@-Z!?A>xWH^zQs^Q6lJnH&X
z9apBX8>Vkp*ZnQ}jQajT_K$&*8{M9kZ1K#dOwSIsq=(1p6$|MwV0k?SYAvoQJuuWw
z$4GgKlurYbJq>XE{FP#vXZF?%?aj|E6rDKW-PC`3B{6_2@tiX-JE>GB&1G3#gZ`3c
zGppP_HbA9$*J{@4H1-a{sS)|p{T!CBRn6FS^bH=hrE4)+VBmA3tgUG5@5P%}e`d-Z
zbM8~;b@eEJOXKL6+FoH0cYGFXIB>3#5%(dvtCfekf%IKhJnEr>D_s04yuMmeskHCZ
zHlH#2r)}&hNLr;b2b2(;p86#ZK6{+P*QotE0G=LcGx8(w*-!52YOx2pIPU4Y8Qzx)
zGk<<wr*H|-<N3PM#Kx4v9ovJ2?i#oe>uYE$UbFc6e<jWXhygt5rh3r~6Vv`ZFiq4x
zX|n<A*(mOUZ_=^;Z2(`k7L|SDY;xoUI8xf&+r%fX9w~~!;0`;@o{andSbOWJEc57J
z6kaI-1q1|?E-67kB_*Uox&;9N0Ricj6r@xdL}}^nkX95VM`@6dR6@Gz?#CG?e&?KZ
z&${dWHA{vW-}iaGvG=F8E8L$QKwO1d7}aP@X1*(rB2}STiK3{-MXB@$plmqiFuPio
zsL(@O#qSlRcT;p+P$E!ODR^4;uuOtS7zfFh<MbD*SN+yjZi>Z{AYnYJrYAEQVchOe
zam{_hR6uUr`!2E|r!XJ3N6-z%JbQ;HY;w&M2W3V;_~S6`^!|zF8`K-+buI;$M|mmX
zQ4JGE%i+WGS4Jtr5PaenBniW*+(hJq4CFfnMiT{Zj6Ox{&>tQyiV78z@0`Dwz{CFI
zTa)!@y{h-iCb5f0{Dv2=-lNZtRS(Mg)?+6{Pa6T9R~oLtb)r%x>@o|;3+mqOh*K4M
zj&<01T88Z`$$xV4t$2?@$sn59frZ~Wh=!T8f7n2^?c~bF{$`(YSgiW){ZaeH#QU=m
zoYEm^M6y^Yn->DoJxhHui+MNH0@irn(=RWs1Suz^A7po+n8_?Lz9w9XV9Prz9A{*V
zuhjwhf^_+(dRKjl?lHi%8{IT?Z1a`<HC9$yRoPGLrAwR@r{~Pg$wr-*gAE9dNw^*H
zdfD~go4yhV>Kk#qS@=$X(y#@MW5jM2?b3_D@>@FlnuBbuq8z7UJs-OhOWswu?nZV$
zv2@`fUPFLzjbgn!d77eP4+rw-Dy;@0pDYeUdteXnUc@Zbm@zq*a`7MmRl6Rlv8S|s
zdZb(Xu<{v&ko1Ekf(Cp^S~gU|4n0fXAhGl2mwdum_Re=~!J4!`V^~z*>5LUa2;J7_
zO-<YF(Ewrg=DdvPDbu-4`9DEpz@3aAS6%#gV<BEPQ?Tk-gp^ftgy#8BOvPKTCq5L8
z>L_%=&*HMx??5)j#y9R19lO==PEVj9?k?Rcf3(G^$D=T)IRCA{`$kCOa}i{5=wQum
z52AT1mm`jg;;LWsZ<enmMMS_uZwnr;l1`j8GMkj^P=cmRn8$c!t;ZD8&6FP+Mojqz
z@oykoiqBQg9{m99{d3AUt6I^>dj5Wy(I?{AB=GK|S}E$sYvOgIu#vcKc`!cW7EU6;
z00S8l9FgTi^)UZBSM|-)Vpps>H5ZKCux+n%6SaxQ{7dgPNq0tZFW<B#baUI-L^B>}
zAzl3>&Qfw4kOidS<hkF9V<qo@NHu9d0drlxXi9SKwfb-ioe;TKhWY50IjSE&j#=KG
zH}`41Df)R{=T8g{*0`7;ZS$t8;If)tZ)x}Ej$9B%I4mwwcyx+LRN$%`T$pPjR~ipB
zd-zpCz<H6>se5stBm3?ZnA{V6_24}InFt@Llh_a3XWDR=SxIp>XpE;ury?7VdN&qp
ztM#_?)Ee}hb9M9M`g3&^7Fn~4ypNyYXP;*;->csH<GSnZoSg2F4+n3&P72g8Gd`Pq
z%(|M*S>L21a0c!GcYn06{xn0m(TROa*17g#=cpuXg=z%eNs0P}=3(jXF=(BBpt8g*
zxZORLJ;!XQP)4JeV@M)7Wp{?Ddqpk$%<Jbves>nV_a<#lJ8o*S_zG=Mjh)7>EYedw
z`r^vg@@S!Gyv%AyX{5otdU<ef;+Cf=KzknJOrBdZZ^3G0p?|1Fws^QUIT5iup1tlx
zt}J?f`z>g~C5L?Xt<=nkPF@DoZLJ*@*3u<WIh`}^7U1}J-9ewyJ?9a%$Y{`~9!`ly
z(gCX)F4K9_yK~<jYR%bE#n?X<OMV`<G%bt9Ic=-r-%QeygynXfL{nR0UMFMMI&Pkn
zHN6eHq3Oom&aw=PxU3soovq~=FSE)kgcW?qxwJSGLdn=UKZM0!TZiPYm;CxU5b2>O
zAN#Q~Mz4{R!K^bT-r-?G(c=#rYNfZCa&>b3t5~VsPJ_bNMxy)O=^yNjAWh0O-DnC<
z=f@`n26b8!9)~swH@!UiX?|4Fogu=3I7lvk9-CVf^w}k}A-z{va2PLO;ge65@xPM2
zI|$V#0!T5^sgS1>LJJ{xA^zofhF}#LTV$;%=Be2;L2=WC2d6yqSs17kubqTMJO7{R
zE;WmJ`LiSDDV0%;L$K}*A2n=i)p5l6H^;E5&$bdPS1k2Hd*jAh!i%C;xs6skO5sm}
zQJ>))aY7(4OQ`!sG(Z{;Vtg8qw$D*h4mO@3*afZTz{l>mJqg)S6d*cf?@ee9zX|gF
z>T;ujQ@D)c!6QH7E>0A2w0;X^41?_bY>r4UsC|y?Yavs||Lx6v2<FkRQGpC$)1fe#
z+}5Jy;EZU0F{#R&O8Xh<w&hqFuNF)mk1Y%BWIpt8oF1<y`dUT$BSY%%YZsEOg15A2
z)~3-v1X#2zb)Rvz7gc_J^QvBXaM@X6=5`A6MB|0FO#vhAoe)Z}rxppI34IN}wn5BZ
zVe$E$)-d|Ly~oC~+!$+JZRL|(QwibbKHdQcCx(sXegmtto85GT>z^{H2q}Y#gyOX}
z&pR#(b_kg7@<^ZE2|B&d+B8<U8aicfH1n~$%zf#>NR{Tk(^xfI^`koL$x;KCJoC>G
zztr&}scx&`0{+lSq(R5~u{YXBD_SRCsc$@}R<3pQY|cCrq~z{0^_fB>$-yoHE95-o
zTNR692bV3;R&j!US=oJ=;F&xr0ZQsW+{?;K?3Er8EC%+uJaV&LxjC|56vp5;I_tyw
zv|03-6#3)l=I;a8*Q-@ajZ5i`UviYts$9Z<uy;i$H$%Wqt|eUkLejS?($Dqj%tiWw
z*IBc^*3#?h73@!awtTDed0I`Q_`?9Pt(F$|`!x1@`nAp-`RuCcH7Bt#Q3u$ZR44NB
zY^-J-ahm7lbms<QV>d|IShAH)*)%o0@0Dh(**2tzBGj+6-J-A%`2_!o0-S9d5Q{=7
zjGG9dlQ!dn!kgU&<lYBQ9v491nsDNIpQ(cw-Dh*lLgXtuq$Am<>9<6lY#R#CZmSR4
z1(Yk`m(rMiTQ*vBII@oLrC!fI`c{N9DaCgFq%0DBchXkDC)yRbfCt^lcRrvkorxUU
z`>Wnc-gDuev(o&7%K;*q<blAt<Y(9VtD2xiVLto&u6hY~6`2pGTKzN8OT|uVaT_=P
z+&73X9+`NsqYrrv0<{|*>?;*~U%=m-z-#^@Mm5lzy;Y5nT+qn^V;#W~;z4z;V<4%o
z48E9sNTsIVctV14RDZ4>{zC__70YZHn>7mxam+tCTEYLMLFik?)qbLxm-As|c@3hs
zKY^g#n2Djt&GjvWbLZ-oXBl8gy#RB?m}L`A%$bWEI;kH?Llf?^Tr{#Np<FoUjf+~-
z+M#i_JOu-bTmG0O3!#x{68hk@%nQmNU7_sBO1LN;LQ_meS9aQS^}dcOZrcOe&*}X%
zMcg-LaWcvB@#U|pWR;@Y*Vojhs(o_QB`0kpLb9xeinNFwyPpHUB!NXhZN1BDKP}nf
z6)~m)S&n|y0Z|VLTXEO4P-Z(8=sYJsvKM2Sg{DO8Jf=!E+^-s8$j6Y@T0A{vbXi*H
z?xZ^Ij7oaFxbVibm{0VOp=v+mW*hxqq>~uN7j*RG@Gr*kJP%$dDUG+BBj}m8MF8@Y
zjp}wE##7#u{bGgqlcL18uDnmx&A*3JY*tL&4i<Yw`$vJLx<IZYS>>fct;!^MazP&1
zPbCM>Utc&CtCQc(dv@_I4uOz@*`3bj^`Hl7xnzy0wXe8`M7v$tumU{0nT4&m{pHNe
zjfExGx!SIrL|}KMOq`=|J8kiw(0*9BMtt0|p8X-Lv=2%ldkmfxDV40kE9+EuK3xo9
z!Rbm7oz!8b;(L@>O7bcSZ#v@_HxPLCclFMdpyX_Q*h|8y1yg+^sU&x-cM-6cb`918
zZo%qZ<6^y2)Hm_~CtHiLszTqj<qGO>L4cjli|iRm6!oaF%W5^dZo*Kv?uhR9$^3;E
zxEb#g#wE$)XjwuF=`DN=Qf`b4M{{3jp3`i#l;8CHQDh)wa@+tGq#kQx<t#Xn>mi|U
z7CeTo&-)7R^6kR`Y~bG0dSghR0TN*YcoiU~j0rrZp*nbM=q3?w#dWIR{aFKTl|&Wg
zr}(3>`9_RG<Cq}moR=@MM#W<q{MC8LpbCXzgH<m2^g2L`9@@ul7k*^XqUo1l_gn*I
z-H-9Q>Xz;Vg8N5cgu24_)$R%JU4Y<NdWAQivG>Z0T?AjjL#FO+u)FfgbJlPrq2I%q
z+D`|h5<Ye>thS#-tMxzYH<jGMY2t8v3|L*t^^J#j2zU2E?n)0QcaUug=zrqC(x?oP
zF(x;Nw!UfTm*A1J>R^QC#z&nP?)4h?o{x-N`M+Cq4llZ2-Hhfm5bP|qiBZ><7jS;s
zfAQwtL!3?88)n!WlQGj$Cn?`*#o5|@TAz!bm_OPp7TRvOC5+~?@q=Wp|2XFOY&1oD
zq9+6I;CoF%zlz&vqCcH2ChrM4FS9@hQdP+5$Cb#5B>vfe@EhM`$kq19`R26ZtUk1N
z4PIz}WI1Z((jNQVgIw&iSA_MY$EBcY^tNge-eA!=0zUGeWC2@sr)zX&dkZOZj>q42
zHn|!jY{o2Fc<SSi_=a#yN5fJT?<yDZ-K@M6r!ps}V;eZ2niv~O##QtPK3ZawqgX41
zaXdc)|Mmt19JB*6nW7iHVEyN6rTTld3Me~UbM4DB<_Cyg;;JMbsq1WdR?F%KeFl=I
zUsSt1R;_%`=h)dKiE=?HqE~29s1AgKojO-tH3ua*Fwy&Ee2_TE>f9r!wl@INy>#CR
z&`=C{99mrLKw9_(B&8EmtE0gf93(t8qt|t7?9yVm4DUgqohe%6!fkG4YmgCLNQ-8*
zxJEyv;kE~S=PaZ%Y+s(3Z85lR=Ny%(>gzp40bV@sYm?YhW>b(;&O3~{Z0<{jrSozZ
z4?2UKu;UbnzUfL5(w-8zlzdi9&=&HyCAGo8Qz;A)<3#&fSi#G*f4m=U;1d%^3nkQQ
z)=|k?!+S|we|Haa>s&a=@ZF{<rbda+E&12(SDq24lE2`MKKX%GqWL`qIOWZQSB8nV
z@GxH?#@u%sw`SL1Pm%Q#;I8%9gXQsUXxg5CrG$H`TBiJ$bSb<y<Y=`Uc(tm87J1T-
z?YQ(Nt`vCtVa5bPY>M$jW?DUrdX@f$?LdB2>?XH)zbn?WC%tD_YHf#0#1{t(twiqH
zO@0$9AuU-5qV~v9Ju2OByF>Kf@??h(JDg|p+Z0b+t2FIEZ*r-2^QFJL2xv_?QNo_g
zWXd~n*HCT?eX0V^R6<S>%y(XsGt@m@kUn|MO--4F8Feai!myuGJJ&Sby)ylwBUfF7
z%FQ{UBlo@F)p?V5G_BT1OR_<jbFbm86Mt31zIaLG30gw;WPG|u&`);U&Yepwr*VAf
zcf@9LS!9o2nWKI_IHV=v?BzTgZ#7P@M>cTuB#t$0_+X{ODf`s!@wD=O-(~dE?3QEf
zRctudnGJhhUFD6nUW2uHIQg@S2~{gj+vm;FS+PC)KzZcpUyZ*Q4-w0=Uab?AIK{IL
zyRnNKPCu-@u+_V6nX}(WFKkpCvB`A1ql#f<NzJ(F_MUI@v59P#ioO&=^MM90=0{()
z0;JZ@P)R@NO}@IlG(<-DypZsYI2vvxxRA0C3N4R%6ZbY>@EWb=UAhry5O;xRxDfFp
z327ktUh;1yu1712=uUVnU43(HxhwpNH%M15DXfS)Hvl>!jAmR1V;05l&~J!uJ@x;T
ze9M!*qu?1VRvp9z#h$Rl&hg#z(~_3?m1MVc#RQ%Y@;`@2z1$nF@CyZ`<Hz8L^(!+X
z%pc3h)I`X|sr7FL&SNXtD&L0A*L1z+Q+^@1$Wq78hl{en563_%?{Vddc95JJn4YA*
zjn{VwQGJ9nT9@kL#zhdA63HNgrD^OfPOnl?^U$<=BLzKq-))CCgAbMKy`yOBB;ffs
zc>hcHQDTS_AmH4FTW>=BHPw&g^N=ILP-EYF<3oEId8F)fl7HdU^oQ-m>_-pMrutRI
zn@`pKYb8^~h;npF(<!}m@GJ)l60X~Pm0YCO(Jl)m)pSN{Fx&PyF5odWYdtqcqaCfX
zc{<{}eB@<u#>~&|oP@(5$AG{!o0SXdbain*xP)GC-bp#!GTi#G9^#|pLW?KhABZu>
z^|+Yd?o-gAZTJbxL6F|{!~KO1WwPUx-6^LgsZ#t&6laKrMd4l!DpS1Md$PiLm-sJ)
zXxIm1&wAVke?;OuDwk6^-n)=XyUbyJ>DhCB=_dMB=lN^N{x}*rDlLQq#=4?fI%TFI
zP7wGB<4jD2_0znC5xE2ty3R&W;I<CBRv&i{*~7|qODR!+CJq9k&<dhc+~*TYpI%$V
zKiK-gu#o8ZLLcn&F30sMg!~}G##d5Gzb?D7W=eRqdHSd|gcx&zbowmq(j>?}93ZM)
zrtqgGp-z=`0^Q*g>+_j_8;9b>8+zBSTVE{vX%c>m8Ekzv6>RpgQtE`h^imNZmW0uQ
zp+JM@t<n14^1}{rL*Z*R^L~ZYE-Gc`+PS#9vk*1$OqY!D7;b+G_QTWTRiBlES|E*s
z!qOyyUU@ywOeF-F5i_PD9oHH&k@9e(R~<eje?$lRy04~+^FKY0sDuYY#G-lcmUMji
zmA?zLU$=#v1Z`KYkm9RcbFup3oa}oCp}E+S8<S=3-t$pZ*s-1TE|I6%IxMtO?LHie
zN6PhlCo=?d{khq`QekABM>Gp$f=W~(SZg88$Z>+p;7hZ=xELO?Ctm8ZGOCuZbthM=
z@-3yEm~-V<4OId`fs0FaQ>=NXfgEKw2QzU|_uVN<sE<4XZ#c2wPD#*eNQ$1E+Z-1@
z|5?@($4Z@UERjN`H%3Q$_CfT!8&8|pLqzAe_IGK#2~T9k!%qu;4u6X}?iC$s&l|`$
zg*fuE+o$Qeg?N(*NAWm}XZ;ip(>VGDqcdzC@clkM9iF456xKd{an6g7MYrV&uaMQk
z{8ZTyTdm^N`|mYk4c*o)yu|N&+pn{m-NU6*DKnKb`OyGHOe4lRFcckt(LV@Y05veT
zR_E0~e>YN$lCQw7f080IFfI6TNL0Yx1FnWyXeT0}mU*>$2qmQvzS~V-rpFQ4;&D4q
zp-F5PTCF!3+<;1}lfAh9LbZQ0xZE=Ux^{_|p?1%B3>UMDpdQ9v5)9sPA+3L(16S$x
zz)R2HSpZ!$_l+pd#|?j@`1bux_)aC~DOff;phvJSXnwLx!@kFgbKT;P<}U0*UZZAT
z9Mn;fLEW?a?I1%o-eDOeXX%xMX$*L#t}L=*J+uUIeCJ|xjll-T1IAkSt<N%;yNdK_
z;DgwJ<5d=bOw8RZ-l_3ns4_U<BySR<F0PrzbqAB_3LC7%5<#TSjZ|;HG@BtAazQK7
zYdm#zp0;ist9|eTnIR~!?4PPytXb{C*o}5DE6zdL$3b$j^XQuFYhmHd_2peWxA~co
z-iQgDjDr#0hR<`$0KU@xvqZ&`>Clv@_MB8ma)9#O&%ams3dRZDVbjjDuIG(Hx`{9J
z&&3~X(7dm{&%7?WyR52|)M#=h`@E0U$&otDjQ*6{dD0))Wzs|e%UxC-!)OFYt95F+
z_V)h9!4nhRJMp~knF_H{UosUGa@8`oWnwSP2gP3SxT;zZpm&j-S@%G@UWfaQ5t_oi
zJ=He-@|bA@Muk2pMupAL?=&5u)0VkD<m#Gu{VLQVA{F|5#zWWF98%d(S(F4?l-J&I
zK3~S>c-wx`UrK$`SLDcew(HXbVKgbH^MLByQ~JDtY=)Pi<O;lHnhe)OfdF*m^1%{f
zkT?j(jgI7st@1k5nAhlkCmh~THq^SCaO9;&sAi?=S=lquaENaynz26aK{!%pUM;se
zQ77ZFI%;#`vNFMBSJ^0W>arNVb()u1LUmAm$LH(BTbtL`12n4DEUajv1^$$<SJhC+
z_ISqlF@}&`$^XyGm<I=4<SwrTZ*9y=9Q>DWG~cZ<(l4sP5RxdTcC`IkNavhC){WDn
zx(g{>96BY<nK9wJCM*)ob^em1JL|~>xq5v0`Jy`0NmT6%Dkp{ogJDKV!n)|kh@VsK
z-X!U?0nJBWvViW_>SDFbWMKg<BrHru;U(ChpA9<7DoWMPk&z;7n>hz1pO)t`!N->H
z4%!?O!>Jpn!`3NjG+athw3lC<<uZ8N-#+~6hSgV6$4LJ!P!;Gu?9ZVq!QIO~$-&ON
zNZxISMTl15>&bm|($n$jm~`g-dr05kz?^$!b5O)dq`UcIqQqFj@^gA%3*IUjcqm;N
z-sf+(b(nUK2{HG{N&DiW<gjAKiT=8G{#lJ1)adm~_MixR{MMnPoL%#6%(uQ+EhPh1
zyB}M((D1RQyd>7AJ+8(v=^ngDbt|1d5K);t`;c-Kv%LR(;<rEOl$+Qusy>1;Ojj`O
zTVX<e*^d}k^LsQ_m%3`NMW1tiH*&4n>OOKu$3Hld+2q6%eJ&Po!<P%2VzQcmQLw>Z
zBl~kh@RH}X<F*(-<)AXJgT`~K8$13c)iV3ep0P63cl%xi?UvMCZ}hBo_~G)CCsEMu
z$C~%~dO`AMEuL?$-i=kS4orNzc3$#ys8sVadl{&0{{0<INl=X35BgmkxDlos8SWEL
zI%cuLtoYKXz+F23Jr9%@lsIc6i)c?iKa-)&tA{b-z1~8}>tc3(lC)lg!I`uXBJAH3
zVF$PSU~DaDyrKNgS0=J)*j%3C_us$jKYwq=d6WjBY*NFN;;$R&_gTF`?3a^K#IWhe
zOvz3|n4?{soCp$+9oZ881AsV3eteZSLazdgj>{%Ux;kr0%6ZA<aL{!ER|fL6Z`&NP
zAvsJsp4;q)(A8$bNF8q}B6o0IR_H?v9npRlyjEH?Sny|jja{Va#oT+_ZRXN>E~B&I
znC_f7I7`y@5>1th_R`Ix{r*7z`E<Yj@JRyJ%BS~HxBjzMGKf(}#(YUGR!s}I7fwO$
zx$NBt>H3zFjSWdXHnT-_KcnCd=zHYdM&;VxusdOuO(BQxcY3rST4vPZ-+F1k{{)ey
zybmJSWhF}U_vPZ~Vnlrs_w5~=ooc`L1-wt59?Qh$a_}-##GBEv-oOWBe1Vp79AfhE
z;{W->Vm?J9#lXe)IIsAZ1>jeasFI4a7Br3u6m7yT2K7k6-Rd6di|e4k8G{E5AzxRJ
zFtFDQCm%1{4qHlJkf0R>;<+Eam%uv>ghgLK?Oqp5jbY6~PKR2j1Lf#r{!*<6wD35e
zdCjunlERy1(HQIgaKUeuq1oJLgWD|p|NIHvkx#gRujlfgPZ;q9IgR?>+Yeu<w~Kl3
z^fB(s*dCmMdkuQPewlHlfh=GHu9x8-UnL>L?5@CcFvD^d(p+)q*ZiTx8%2)VPopKY
zk=8bFjuqbQ;&mZq!x>h&b{k_FqN{V~M;5g;|NO^CVfu(HoCwC3S(n=^$sS<j(5k$2
zO|iOyk`wdoJz?~1=low+!0#^u>T))mS+XybLH}Opw3UQ4%a;cTSQbUI$-=dceZ3Iz
z!CefI<1O_cUu(;6okD<f$MQ(|iqj){C@UhNm`KP1`j=;Yyv}tYi@iz}14ymr4S4#B
z<ga0#7x3Gg#E)b!{Q~Gr&d^6ZK}*hct2)3GR$sR^qeUeIpvn0sRJ_(#pc99c8EE>Z
zR*^nJ8}@99c6*s2?BD<WH}De{gI{+?W2^hcR{!3evS;8KL}0y+`O1of!tl7Q8P$Pt
zqh1A)95scKEHa?RD+4gIZp0rG4@t^(c@^18w8n;GJ=ti`5yKkoCR1xIj_Ie5YI`7w
z28-3#L1mbbR<5LEg@5QZI#R5Q3py{F7Gs!qv#Mvo`ug?pRFRFxCd~1l&+^YH48J3L
z75QL#hb<$&KkBc4Jxd0$Cs)<3cluNiwFNJaErG<ULTV~Xfin^`X~Ua7Q-0X9FnFW9
zFGLy{Sco8C<I5oAj~DSeKvuU5JcVDIh}Mj-nU}OztI5}#Ex8bo!f5!|*P>mbt>{pP
zfnrI5&ZbZ^mD!9{9WMqwNBClUjJ;*$^>fo(|NAG5oU@6@b`8Or`MqX-{bnGxpMrV5
zzZm!F_6v)ATvXk0=gZesIFycE+&rFDKRsQ@fpXbe9O#rE1!kqL1LmAqXJ1i#NefQx
z@T4kwq{R?MT-$0VtEVT*h}J~N0HQLH-Ysb-3;<0-2z|39L#FwDY4CKl1bZzdav?))
ztJ~mFSpXT3(&5xskvPgF>9FQSk&jgetoQ0`AlZqg400&<0c0}kes$yXE14M00v*+B
zeC$Z)dedgqGz#1qypBbNO^rs~;LnL5W!Gv?6mT>JU7S{QO;JJttqLUTSweRA^@siK
z%U8cNpY<#={d6IoFSMq4L&LuStqErBbQlLwF*C`>T=U}8tB_?@N@VfeeHxpLfF<)r
zH^7=Z*-csCRA3cJFNGjSu`samrT2Nk@3az658*RnZam9Z&obF=+rYn_YGJQQh_m<X
z9rEt5=p*Ei-7Hu4sglr7pUw_po57pL2nh~Ru)R8wnig_RVqsAFY@X;%K(1*&AWQE9
zUvM7->OXJkUw>GmC;WncLUg7B&CHiaj3O0Z1&ifOjNdz^Gx3i}qtG3XPs`*<I5}`b
zmgBh$MJjA38zV?#x|ZvDAv*M><_G<mLRW}IuRExN%o>B^Jthr`<_c9a-v~DPn)!vo
zVtKrxljFuV&Ek8vGURdTDvcIx9%aX}P<udMAR$BLtkHE+!2R(Bl~uJ&23%@sFlghc
z^b1Og(OSps{H_Er#nId*GjLM<I>XX3J|esY&B4d<!B<Ns`o(V-*@pH2Eg4%~XV)&G
zd<%(>5>VkgZm%ecabCIe%GB<T(~0~r0TLu-G`HMGVffa`@_Pq+F@EkBF&O$??0PW}
z2)&}3!C-rjon?vxh_yCYIlf>EEMpFUWA@vxzkBG422M^+jZkdqf8ec-5Vor6A&L8J
z+G!L~&mcNtQYw01o<dfDqM1UO^Z~<*+o0aO-_aeLA{Y35q_sD+zTtFlx%cj4)~43@
zf1WM+e;+k^hnSj~FEz&23O~YL|NUyj=p<856(jFw9+8QZJ!wx~LC#t$45O=5aX*Qw
zY$rM89mveKUsh}ra7=>RDpKo9jCP@(CM3tu2oVr5IVn^>8ebOHd3w51J-J-Lh7)|@
z`$~mj2ISa&%x?i~<|>$s*YfW#+yb?!1GtpNENwBg#*Hg0^lKd^=qIliB;JE@+Q?D8
z7bnCkdbB3t7pTFW!xOL`XNSZq>8cjP3yZz{sD|z#70H8F*nS|Q{R9!@%<{BhwJsVe
zQs{o>p>R;OYSK_&Mo@etG(>+ph=7JOkAwE~aPoBd0KkUooG9O<^P#DQY!n!zGP58D
z(FeeSxOylo`25RAZyXZ#t6xs>PTf~W=y=K+K$a0A_Llyd(F+(ts^#?#PsTFq@bNeG
zFkN_X8QiG9a4V+!n0t-r%0~F`Q`1YRgD$>a?<VITypzI*#~o3LL-V6k=gmc_^*>&K
zgX~{NTm%aOMuw6v{YGhI1u)jwFZvY9*V=pw_}4FrVQf(Pg^7}}-uvkDQd=-&d2}8*
zou#K>f5-PJ33cB)6N4#cT7_!gR>5G=DtH-dR`l@4Bnt*VFU)5o^HW3f*LzF0tzv{T
zp_K%UHv+#|PKw;Lzxm2v3|6rjkOpdLw=u2uYKmwxT$hJcq2}t$*#;DkR^)@4#hMGG
z@Dg+sR!=N;yJYgpiKhaB&SUZsU$yvr(T$JM{M-HNbD&!Z)!l{^&RF2(+6op5u2~Io
zfbG06i8S25nN(Y!u2<x+VEkif&VasdtwUxEr-2@nN@Vtv%cGT@EM7;jNwkRX@lyC*
zFq7hZ^{wgM_-SbU*UwEH<Bt0hx^GvFelGZo@a55{^fw)c$t+zSLJ9&VAS=;&&aV>=
z?=Fup@NZtdSFdvdMJp=VdGQ><+*hC@r4^syqKG1{J;5=>LLnXVDj9<Ah>E`Rt!;Tz
zA7S#Hw&sGue;#+RSR&5aGQ|~@0_neA7_Bo=s=wIc&vWL#fAXsWjEKk4=hw_IMN9DL
zKT9vfw=nm8xz3jmO%8@fJ}Q+7nsbgQC`3HX`w&)Ikm0o*wHZdn^(Y3#A?1(w2~}lU
z^z4+xl6cI%%CmeoZ}vrZVd^pmA(+c0qKG1&y>HkM$LF_x4~+?fFv6(iR{N7?Fz*nh
znff{_zP@=!r3%9Iky{C$T6lQwEZU6maVf_io}?Im^x#HcQI3NZuMM{Zv#zf07l<%6
zv228qURtD%n4j>i?Ac6f*evLyLB!C(qQg#t<;%B&(dxmH11OkE5C0548Ohpx`>oHP
z&oW!^!dy{pA;zgvBXHjFKnh?vO%RpKOMRCSGlwqjHpzY|X%0+3h$QF$pJvAW51}rH
zFr=j&ssk=)VEtD7W*{Oj0zu;3r7yETyUeEq!{LP4KUpy|{3;T>(R($7@A7i51(ns0
z4Wv3=l2#0Kyae^^qty||Wcs!F*S7M-_Zlu-G64aS|K=xzM}z1?<s6$VEl$<+0D)fj
zSaE<LDI6O+`<)#PBvb#O7sK@Hnmo`D^?Y!0-Y)Q;lP-_0c{e$?iXoBUuLIC80C~0V
zB1+N!bt2-jg3?PuSClC*K5w}xp*V@L?3YZd?~-4m*Wsm&ippRR1?nYxA6T-pBIU$a
zh(mE14Q*cZfD*JI<GRhL2IRY65juuD07n?B6&I-Q7$ZgY0-e&^$n_jWyZ$B|@+*ef
zs)|)4(axSt#;r$R5r502hGDq+uBDEXokqJj3S&cT-mt;rMtWn5vba&(3n~Z5e(HeO
zLbA_UNbUYbflID8`IdnhTur=;4%%4^k*v5V$YqObIyq2CWxS8sPAco{jUGxvaBC|f
zKL;WfI4K-2rW;0Vr5PDvvKTdA{(zuc(L04a?Y)TUlzj}LTw&7nKGX+=MM$ZJ$~@Cv
zWVq&W*;nR8o@Tyu*QesEn_nn@$+v<vN_g@|*TdWM4Zb|~L+@K4Fhg%<q!St_ZP>Dk
zOxHcZqe_$4J=_76W;_nbH55=kc1jcr?<N{NL1TS4aU_rjF{9&=pi+ZjLnyh!RsGxn
zt&}xpx>?Fx@IU+YHs~KW&OE8V^kjIRv}`&4?`}a>8__=wCC$|T2Ec}qa`CA@(Z3H|
zznov>;HKyWR`qe8PbNyFBUOXI=~+HUlNCnBk2YWt?iAO~UlO|Iyr>}nMf;sgv)Ss(
zn!10KSPgOJYUD;iZ9bj>qd7St<oLS%k>mSweC>avNvt@jwC2+q4Ls4@2WZl8u;`Bd
zPe2~cVsKN%eAi&$oGg@|g+&ZSIoMXGP(hV-3c4&?6pW5hBnJbhjI#j5ZE^Mkr3;KM
zZsIW2w>0T&841Vdpxbmpy%g(gIB2fLQGVr(eCJp34868v`GkIM!8{o=*?U@NSi<>9
z1q+UJv<uwfB2$B+L!GpPaScUO@kr;ZVZB=^B}9+A4BhhBB6Oo$!-E8(Yt*Pjv(gwY
z71yJ^b%#M+bAi_%wrG9bk)hx&m)mb6H+cmgit7T6*H)yM916XI9l}x~q3H9>=Si}}
ziMb_~u^M!ef3RyC`8j(pKGXA2R#Lc{rKlR!*q_z7Ppfo=CMLS~&;2LwzxqoX`H*5<
zJB2Rut6S!V`n{{)w@F*~_;-P+(FO~oHsO0mMR-jcXZ+Ihj=X+woJRh6I{*LkwngNB
zavF~Ld@4@iEOU>A3mrBvlv|&Gy2NCBfI*mWl?jc|Q|wx`>^M{mQFPLGEm_X{ZaudH
zyG|_$N=anjG(p(Cj5Ta0JjEZGNtgltu#Y^R6;+=&4&hQ<c78JnmhMl0ZeQdN&=yA<
zoOylS@;<vF#*MA0PSs-Q@-{#ue{d6C<DOP%Bmxt6LY;rVe}fCenZ7=&pB>wtQ7|kd
zxh5M}f&&NjrOEI+#N+sWHOFjMfvKPvg9mQ{vCds0m%>m;z;HHkCu~lHK0@rr8vqrb
zVwlgZ%VXf9NEd^*dQ_S1p{A{Duc0m~cqV5$y3Z1OlNolRB<Ytj`8|f6C=aU9Uo^I4
zsxQ)NBGHd1Gfi{f`y3?H>>4?=0IDP^eCx~Gg)wfpJxWMHL+E$g4-MhU4lxnTxi>rl
zG%H8Vw8bB3tK0)Q4*wL}AuZY$Qbe=jyufe!d)M6r1fwL)DB$mnN(*hmFA44U#A+!v
zs@bMlKF<*Aqj6*xhNBhKBA$wZ8K@U43#+Ov`=lRq<KD%n5XSPAwRI<{ftq^`Mu_pQ
zhy}l$4rRCII4`^=rd!EYr52LnFIGQ+z1$K-xZZOLhhi);a-qb$7=9d3LN68C4p@P8
zCLTVT!hyKfNpu>m9fSbl3X9J&lVtceKOqodss8E^c5B7L0e$rx?ap1}cL8UKlF<uz
zR!OOu5u#B@D9&Fj=seZ4Sfe^WX{3h`wngxsgl;!unbeII`UVFQZGAgyea?-nG0(t!
z;UwY&7ZLf)2HC$gFQtg)C7|K_zP1r35eq*8PVo1MoE~lZ5ko%OOI3^-m`;!h1+hy5
zLUo8^Q^l|o=(t>Y_N!0-g6VgEk*esF?4*3WT*B1U^Z&+-%hVJ3*RvPXfUcjAl=>p%
zM*LHR4*xn4%Rh!{*jRJ*k|x+Sw!i&Q#KFvi?FYD^0t9JVm#0EP2>FI!5qK-!AZ9UU
znotyg$<Yw9B$G<+y9TTTa=hJn44?&yUJzAKrJbwxq1EJg1}MEcm%{#hZDp6AUn%k}
z-_4a=@M%WJChshs>x?H`B4{mA;H;HIP>kJ#9RKF2mL~|uQ86YH2HC>|4#UPlBbr8$
z-d}6%T}CNBIbou^<*tnpFW-Uq(%BewrP6P~S+r{)is*n)ETbW>zY&Vh^lK#cC;JB!
z@l@bNHR@5x@JkdtBWypF>`V3nD#8vpj5yL%+I0ih-#rz(p;<5JSaF&+#6{$OD12Jx
z)Z1OLsmit@1Fp061^N%2uyaC1Ok8a?xlG{28DU%i?j|d2lfM2O|9gm(n1zWg4iC7M
zkV>&#@wdur)a8_Y!$Ic%zqiyU6WZTScXez@(sVkL-m1~nj5|^Q=d1fdlEaXJ>j>UM
zxHq6`=i&iP+K-8bx=%sr&2aT_7`%1jX0$MUG*TwTrd`AVjC?FS{jq=_FgrtSv^u<3
zFBOihP%Lr=UGz=<rGoNj4ia!RW6Gn}10+c?l!R8U`?&_Xy@TirgTWP0n%_om8Rr6~
zOdo`d-kN7NQ_;$x;%nZpz;yox=ifo^tzCVj?yrFP+VXK66gIdny4NxNU{{ZCS!({A
zY-*~9&k0700rpd<HU1cZv6ZaNl@_1V{k)0|n<iLro5xx+<f2VM7USov8<F<gg+Be;
zh5qk1l;)Aos@p<Ss(K}O`RlUbM}2ygW}I0qakRdtYP9v@y2enka%M`gi+;z=YzID@
zd`^`m0^}yDQ;`;cweWZyIE?_Kq0(2gkya7Ld+h`!Mco(Hh4CI=wr<%@2-guLeqz&T
zul0d~%4&btkZBJKBXJ>df*6&TU|qmKAs@Zgs@df2B>>v`a}S<hx)Ig=ny(XPe>^z3
zW}et|;Vj~kjJcZ6Xnqc!S3HMaMf)S9;<?;wxrU1#sFgUU(rEYXc7l*=5h#j!i(Xq~
zK#h*FwSmABl7chCGlxR}TXxgLsIF6m`Qc<-|NmMN)e!oIi2kqG%Xgk@Mvr9uPreY<
zJ$nCSNWI#?iQ|iXLt=SDwtT8~kCGD7+)2ncpJi?N*rS1}&n<@H8t(3u$4tb5P=b=G
zp~8}&0PY>}YCCuYw6u$mWHOjuYR06O>Yzza1LbGUff8dHHD-E`gxqA&&wkI_{Rcg@
zZ1!CPRfbL8em0uo&6?UFdVbDHCXDPwxTvD_pmUewmeyft)1^z?_a$XNX%vaf+{w8w
zIW?R|JGCg?z=JWkecus6n`%b$$`ti&2HuOy%6S%5iH8`}xdbKy`>O&+)Xd_TkW5zJ
z7S*}rB=H3dVMpF=0g{Rbv*wGhHmwYDU)VG$KQU``BK_&({poC*oX((zc=I+6;+~)R
zDbaATb~G>DTHhu%hS~!_r#H!Ac8ywJJY`9-rx*M7De~p(=~nmKZ>ZkMS$LxYmHC}X
z@-~WBe74m5S!|O0`5&L<`#&-=>=BR&o2d(CtCyy-0`FZK;AX-ZvxhYwg#c>TKhBlm
z>Qf>;b9=bW0(~xs(m_^#hJVNz4zj1ncmy?D%g=8Ka?5Uihwf3S#%Dq2S@w^8?j(`-
z>(&H<ucs6v46V+-I@6nc*o58ca?0IReCRu4R}7HapRiAaC=a+MPNo#ynR|Z!D2PXl
z0Ch0<;y<Jk?RgBIZnajq@X&tI37>E3PlGv*`~SL>LpWhB?7cSRaE3WU(KJ3spJ!P;
zrDDEbg{v!Et~Y^K1{rup+=PhTk@g=3-wv<9B<hE%@1m1lTwM1c41T~Qq9@wJqmyw8
zo!m1fE%Qd+^I<9S^ic@Y-vlUtLZ*D|GFylT;demxTe^PI%45VKqmdx+x%Ejil!=+3
z4{{l+f5F)KL`+L7t~6k&qGTzl&QZC9Z%qlkw*2<>ku?uOv_{~xDql%A<oWvw17_ip
z<R*-x6U#07l?$+<Drn03JI>b^=~Z^PweNvk6tz@B`-G5Rry<R$C-Ul0N8BCa{aI;k
zt&fa6E01I3FGnv9C!a>jCM=!=KNrw69n%Bx^94T?l(fP)Q(aI!agDMg<L^pwZO!*2
z5m?;)M`nBmG`b?0k-8<b#U1(nn@Ymsq}pcvIjSi3?NU^aA%EMi;0CJ$EnhPB>>FOt
z=xH*)7<0V1IAdMZ5f#i#2~8IBUSn!#f_*jSy_XF2*Cg@QUZE+YX`wYXpSzbnz5c@2
z(H+`8o7IO}qT<?ap@=eYdhxQ?l+ZA=BXclRj8yZF^NVr&lBrXKES|oylLW_Yr+*df
zOhSAEy)vKf*?RBDSZ@;BypZ}El*%}R5~j&~&uRbScfXvD^wBwMB@zFw;hKLXJW;vE
z%g*#I&(+c!3e#@%VQ60@PIw|#djA8y9Q7<!0OV~+U*yzhc{u(hBaJ0BPT>yasqoIQ
z3F-qHB;qr-L&eOzTEF>1ZiXfzIZHX^QT7`tD2ZrKTNS)<7k44xG4isl6R~Y6b=M)}
z=0`&?@`r+<5o#+eiV55hHk?HJj}#k_I;hwE=H3+mzM(;>ULwK5grp(_$oK3HHlh*p
zKyl-(!+~7ap-Qt-5E1N;HoF;eCAkEa$o=zhZ6+C9A;Ltji<G{^_lN_6SR<fbmxFb8
zXH@sm2fDDD6tQ-bb~}t$NvvP!WS$!28*DVTZ`B#-Qbe~SHG+*|p2?Vby6pbwNpQn&
zgDe6`HJ`BHxb^bWX3~H<*N^=n5DJe0jZfFwKC^^DDk;RPXVZVX0a3=EWsLz8H6{Jx
z3K9*a#TSK(i(34w_@F1zvPWyDb^4juNCoK5)8kIg+y&?^ODT!1P_L3lKF{yDFs4Er
z2e;13NO_z3w9@U*=(t_?G6^fxdvT=gDLfZ6bgDdjr*DvyPy~y0(5Z;WMtC$Wh!$SW
zm-@K9_J3n1oTmKUPPa<>a$ef_L+VMo^5YQne_{6tbY~Lg<;%J&TYi4KyWRWj)=UFg
zO}M;<8bj<ius6>B@siu5z*P3rW|52h)9OyJt`$c_t*>0Cm-n?F2;ZR+BgN3Sfr8;{
z%^22pox_~Gb_&b`CLq14cl}`#f*0mL*Ry=CcM8mn(?HXYtk>y-u05I!H)=*v%I3Aa
zMY9~*29?2=bcRMab?r4@**zDv9xl;;UQErbe~e@r&}k}$jK=G2yH?i1-+g8YW_)ZG
zy&dj#<9-RM*!&d(*KtzUF$)tU8h^{@bk~PY^ReA_xFj^eQ9RNW>$-8MRqC~6d_nf9
zgx9&f8L2lIL}8MO;gY<wb+5aS&OG~#sgs5HM7zWQvpeuzdKW=VB>YpPf3!Y-WhPvd
zdRXi1DxD^+-GIzAK;!2tE&G@?s;qS#MbxC;MX*&p)kye5(-pkobebGE%QPva0Jk_+
zir+5m4brgZae{j%2)R<!2+q)Lr;_MFogrz!erDTWq?}uMJCS6s8azPxmmQzqSV*-;
z3{Z>MA<#PnfLOc-xsxOoKd}asZ$NoWsD{}U{5Ee7{`tMxImcg2@-n*#L}Wgo+?F)@
z_#GrX$ViWyI(p6pOFjUXH**(MepCE@Zn2Wi2}<^18#)?t<y(9;d3hJaWn=l4J+xBl
zvzm2_OME7@MELh=!+FKMzeOXh;X696k91Z0SK8c5eU_sSW&UP)W;@Z~*lCM#kXK~=
z^M;~5HS^`VxViGd;KY{GmgT<E@81QQV#5@1<X6`7@3oD)@G%nbbr{Lt7XY8r>)<j_
z3I752ka6}vUXxW1NEWI)3GFW=4HoFm<Xv8>lPV*{5F^>Zn*H!9$V1rfhJuE;5ls8!
zZfm9l88|h}pNh9Cu)ru#RDl8Pc&*pX`S*;NqFeVkRKoFlnA0q9QJ|GA&o9yodp;ui
z5u3;g6w^gjpVSCT27``aa6`Toe#AzB!#&<TD#g0XPg3&T=0Db8f2k7g>ygQ=P|kg^
zqH6+$g<fA*?7l`6Q4f>0L)=|V^zBO|DEBdngY6|U{va5TG^{(&n)Zxh%bg(b_G+#D
zRI{JoHt<}-!6)`Tej9$1<j#xzXWe_7wXi?;WJ7_uSs#iG@EM5Yq%8pM8+7W@yu}cZ
zD*TPIGZY*If?MzKxM~&|((bi9UV38y&gKfszT1$szHQ?Q(b$2&;{3d{02lOjp;h-e
z?P7;@kWXA5#+o!(^Xj5V<@mN7zK$;<n$@?!pS&V&PMmml!|JNIqnNjMH=TMLT~$n)
zVysP3SlOMwUQ%r=wEHYVwXsxHZ<hb$@nRzV=tsDoZN2-a!~E5|rqf1P7BRlgIowrp
z-Q#i_xgM$fXbZd_(zK}oW~yN;jaI7u&vG=qwMRf>!uU|2KQ^cp;dUUN>qR?3hYBWo
zEo6PIOe3Hn1gaQk6F+CMltB{r>Js#9<3mZ=M=hIVL4sxqZzp|FW#v8-@h4y_O|u(_
znD*OQvlH)|=f|oF%tlDe0;gBCc|;F6ftmO;B40U2>s*V$;U^t)WVRr{kVi}V&}*_s
z?RHeIt@*6Iq?}bcHRG}>0`m@ek6^0OH=Ds^zM@M2sHbb$gLfprgDe7)w?gzZE6Z(b
zzF|=aI^`s}NcRXVB;wQxKmw)Y*?T<?iPix$4rObr*;s8ne5A^RuAR(l4^^H#9OcUe
z0Lq@-;ns5l3@F3AH`#@>a+)0kuZ2ku7$u1Kf2c+tv>kFiIvA%{nAqh~*!`*~M3ec&
z8w`!v;Uu~CFGcSYbCK$9{>fIMGo<38zJS}%^P;xCePE6&z0eAoYWiEnm)FGmO(G1Q
zIbP~%%d@b8?+QZ2*qQKNcTQlmu^mP&vjy@cTkUt$@U~q}iknUz_9VJIWk}C(R8`o1
zK^b|TO0c~vffxD8QJ=xh;2jAmfI{V%<|>t@s~ur{X}BoB)ky2!vV~zKJFmzm2k&+j
zn7XS<VGjvR_DS}eyhJ<*2r5dJyTJL2VRw;+g1g^!zR;j<aQPb?%`;st9|jZx5Hh3Z
z5V$R=SVLeDF*YqA>Bhhbc)(DPTP1xH?I0PQ9o?mqc-jtcu72PpGYT_ItUHeWmh@il
zx8i;J)oplA|L=DBf8TGkW)-APS&4Yue{s@(70`J+5tecEi<DF!4#vnkjPp8-qvkVb
z8M7nhg^B{mCL4UfyS5_O6MP}XYYE${TCNJ)51=~!Y?EmiS*g-505_W>k<Eo|THo_X
zQ<O84lYU!PPNt|dhM%VL4znAf%DV)sZdpp`2vvuU|BxV%stB1EM>{NH^wNF}!&2xn
zI>`KHjP5~9m0hJceH^wQw4t2?^Kic7QNE8l1Rkz-RI>b{b>+7kXC{IYGkbed%TdX-
zatB9Bwch@N&yExg9K8$KA;;sUu+IEa)y(hPs(VR%>XlW|S>OtKwu2mzmR|z)oQrI%
zEJF{K+aGTXDtZN0DQ?S6t1i7M2F2i00C6l+80;p_mbITkg*mcKsG2$U^4J7lQ_rbV
zB0cpM4*unvT{p7rXFh(Yyr@a&3_rXbh}&p$ER4g#Jy~7kXsCI%^#)eGY_r14@)vwX
z9fL(Tp=t})n;c92fzMC&VA`Db@1v)CTPTq6J6F?uMD7g63Rmy;{*}oDw!8XdX2E~4
z+g~)X{Ckqxm4|NFPka^ni|1x}%+nnIT)5`ff%Fgg-3oOI&^du;p7?xS5#?o+3f<^b
z#qtrz1N}u<#eF73lr!nII#isZ`N3JU^RRLIiHLesA#G~hg2cn;lQ*tZ($lDMp%ZPg
z0h}UxweantUdUI7Q!3OOBS<q`^*|E$)Sy7$-O()clkRt}>c#-2uuNgesFk#7Poe<b
z_Ap_o3*kinYbn_VJUFF^z-M9N_Q}iQO)}?g8A^M}R(`m+@pGLGH9~Lz6+uLA+LyPT
z2;ZFu?siiIakxLxt2$S(YTfBZ**mRqtpAah_^)@JE4X`{MCQZNS=VqbDqDljGHJ`A
zq>JV&)2iKwVS?G=?Iu2NbDY(3A+1Xb-GGH++}8!WqBa@W^l=K7yhPl3Bc5O8+p$?H
zbp2Voa4@*Jv$}gLHI)>jf6O1R7%q%kMU_T2i=R6OZpv~GD2t~8z0TdH8jeF`uDM-v
zu`)?p*9kN(X`rx?wwW2flGV4T|1!6iLU`;+4>Lcp>Ec_wzPJ86e40JUftFk~THaaz
z<K_JQM_N`M6~@EN*n8yvgN-GD_B7TsGCkc?I<nxxNtpMhb>#ETBT)P7;IO30fsB>o
z;nji4IG8CE?15)DN0)YS%<RXPOrm{I8O4s2o6F4JrFdwszGjZ8$rABs{H?AWa!_|A
z<~n>Qp71&KQq9j5Vfx8Y8OMfr7ueMcT{_^4Uj?rAqUJ?Hrim~%fnX0ZyMgixFXQ!s
zDx-*oApKNDNnkD}__65`7z#IlE$e(2h$nS5E;Ono6OQq8Q+jQ$sq9&H)%=p8$L6R_
zjk5TpO0Po~)%-mdK?)+KY6ZZPAJ?fl)UmWoaK7d?dCC&fqa?&2UlT7E%`8y9ya!|w
zR|J!}3j5m!$|WL|Zy%Px(AZ*x(dl{Ua`g1_qiY++=5F+e^gU<V?~3-bF2rkw@Ux#f
zIxaAUB$S3;9OxBJu>*@m7~9#sn1FKMPr*bC`f~t95Nl>duFtgL*)@BBxtuX)?2QaT
z%dr_|JN-|Z%9~bZ>#ss^H1fC{dfXt7V4Ml$5RD(l77Tmz=?MdxDT=XTgX{aaYB-wY
zU(_jNtXrPE@mlw7S0;v+qJV)*j-F@(0p79sx-7&7_uS`5YoT^WZf8F)r^S9bq}_|3
zNhEiQt6(>inef^*7;?v7u^Ze%ME*5BuNT+F`rkxV>q%ZRr%yDqldeMbq{5`UvZ%8L
zFz1R3W{#GQP2m^d*1HHMgVaRo(*eZ)%_C!U&o)Nue1k-`7>ERHlONUAB#Z!Bany3z
z5YWUGr;1=#?>tjFFyIqdBa%FKP--!Bl#23;=KF^rdEsSHFa-;Oz?C&jtejU$@av*z
z1Ji1lV$&GFAwQmo<)z%}!<Os$AL`#<RSv9KSvriUbj#EwW6{=s>6@UMW$tXIc&K~p
zJD0(b*ZfN(JiG3m4B_mh#*YHFOgnW2HuPzxC(T~k>E-z%1&xUIsYIla;TZbDqMIR%
z8t!xtCkI`x={(c^;ze%|6eByESKRtD$0XO^H~$2j0TS|iE^{B`TTRYteo`rVgaifW
zY?fV<ojrh_<L$?Go7*7|{T!cU-enSI>aTQFz7HJIE9~=>FaIEqh(>T($X&%oXZjpi
zf4I6xD~~$FwyF$SCRob)QAaAw`-J0BvcM}I!Ue~Wjg$u)lk=lSwYCE(4$8I4x)kk&
z*nV(^2aQ%(5)ZwFh$rf~fCwGch&Ik*Y-Um0JK`z%c_mY?B*i@m?UM5~#rZMvkHsB#
z=ff}E=%|bX(_njb;m{LbHoQGJcfu>!4&HuL$(+ft(QM({0^@h>s2j1)CImO$hW%!T
zGf`wWzjgVwyXB5`*KDsha>CTX39~U;d-EsPzuMBjz^;E3W=^r_jYhL!zi)}(!n50K
zXI6qRPMgQ})0}QiIZoJn3uxwl=S=Q^9^uV>KFBB5J_eResxZ7a7F;H^ql{6q0ki$t
z$|;1hch>{3soiLu5TY=TIgi#34$r5LYUeI$(}<s~<i8{cxT(o~%=!%paj~mQ*mrKE
z9YJTI6Xp>beAiy0;a0%G?&|6#QEEtpJi~-uskZcRDSo)d-sC%&IPu0oO;9uqDYfKh
z0t$tG)-Yha)2kOR{PDEk-$wuN1@~V3Jo!`th2+b3<#C>h_&lmudiN+fc2^$FCYw(u
zo43CLBn9|eRh$Kh)k(P4=RKC6{4ri){JzFiC;<S5*iWvCK#l=e^BCrhJ`q!NeX5~G
zs6wD;Uwp<2E{2zmtZ9HY^b>1-ong!ab{J~htFy6mLz1`7Zu^o7&OGCtPL7^z<l*Y>
zjm|xx2s8Km!=23BUn50yq3u@spSr$(uHU~;cwZhBa_ITa?N7h>uSP93qC@NnvA@sM
z%BhrWXgupPLqgo{SJ(v9%jua=yV5O=A`(39V>lUqy$uNKMhw?!(;vdxmM8Z|oPjI~
z6w})+*C#*?fNUJ!lF9~-XA-gzA#ezOUkD&p!tpbmmr*gSe5h#woIn~3qoDoi++U#E
zm-&4SOx|tdgCSBCZr(r`od=Wh**l5++!Zh@G=5np1j?+yUu4)vfFkSco&jMp)1Rw;
z1u%btz>||h+kC4-T&{~S`_du@3HVOLrg{7arvxLsTs-CF-PupkZ`eeTNw{e+dEiEt
z>$!a8d%RQoVdhN93E}~)!d99}`!e`EN?4a%3E~c;sWZDa8)0(+MT?A(ZFKxF(vt6c
z0C3~M;P%8M2>zFOCx86XgV-Be8z<-Cq6Uwqr0^GvSwvcie?nguFlhVyz14jlnR*95
zGN`E*3=+@oePzJ<pBv-9uZWfTj8vJS-jBflfWy8xBs{=sk=3s8f{gT-Oyrm|k}3Mv
zIj=^7Ub({wxOFhtro?~^V9K+mg$gyF#dz|DNo5(EROl3YTCm<zGEI8BSRrC${9lfm
zU(!IGtF8OYbG~IKAWSk~ZK>5dm;nv*<T2b->C_a<8}F#mdmh?rBd{FwS|HTCsH#wa
z__b?N<xG|J&ryZI-}cE5o7!T>5!`F(9cN#tMIv&im>jbO1NXVu4TO^T;YN=zjoUl4
z!>*O*+2PjXU21s++&`HzSwXZM&UKMHvo&=`+9Y3kV^yz&E{{$l_gh2{PQQR&#+cnH
zX~C)P>`LQbIx}B399d8yE$iyre<0m}<u_(&Zrb&d+W|(_jOgZ(JnXs7Zz74eC?}_p
zFj%w!21MD(#@f0oF*=;&LznGbDtpRU)B}GhhpS)NdVB~aov$|~oz^e6_)O0ZI8Xz#
z90cVo-vsH@R#LG+%a!y{a(~BEQecP4?Vpx+)N8ML=V7B$+;|vM{R&~%=6jN?78tyt
zyC3;sL6o?@HvSGbPnaHwmx;1B=!|8*(njrBa6D9HbC=cNZH4U^^(~KfSn8{HKPu9L
z8ZhyObjCyC({G|R@}>iPzrN*f-1ol)7-B2K3=kutBGbG4OBC=AiJWRjZ@1V-k6pEH
zcsPpLk=4)+7o}t>AswNDcq*ZI<OPMm3uvHJ@x9OtvWEiGDbkL!v~Gae^cM9&JcFt-
zmdh(5`iN_&TZTjeQ!1wq*v-Qp^Z$YTPiV)9a_mxNS6z>_OU;AYJE;J<DA`EzkpK+7
zUZz=rB|M8Zl{#-09L08b<gX5Priwq&cb|}iR_6J;oLGX5Te5qf@KyXjB(&o${*cYO
z^ZEhPnPNE4Z*;s;n)CP90xzj8@I)&Dh@^D`?#fYkj4bmgTXv_PUv)(e6=SDvr9mMq
z+{fL4{&ESpUW^V@&l+6I3fW3YiMr2-{rsiSq?tHR^fPo=Pu=S$)J+|oKCqrX1zqC*
z`AND5Ay$p^cX$44)i{rHeUZ)K-Ej-A*Y54eO1vkXy0<ZK(Gh=7;hmkRZ6oI@Fflwm
zqOr+B^1+0GBX=RynJ83A^Iu0ZGUvICaY1|vjmdL|lJx8SMH~7irSqKTXNhFZF~lw|
zi>gDA!71B>@Lkgxo<CU(Nm_uu6^RE=kx&-Tk0sv;@HRfzOSOze^o0=#qxrsUGVi6X
zOF4e-E9TQmBr`k?eYp-Yy-A3QL}31<iRq5DrjcU+JVt~F$pAI@&a)oT)5m_hPj%2c
zQXp1_XIRhKY9NmZnJpHzb7KG^q(j72O_qmCQ3PgMz8gV(>`!8Q*@;-LVXzI537FTP
zu6Yi;|0LVslS}eFoWSun{QSGT0{O@<)uk9%alkS{LcT@$^547!EfXP&mfGx5xRiX%
zZc0xvO^9LxqAih3ji3b@qkr@}B2fgX*9yH7h)HZ6z>3)k&P*C^T_{+<v*iw|kPK6~
z<F3y0yyiVIfO+W1-3t+$#!0kGk$n?=Jht-x@b=#EShw#VFc&h)3`NNmk*%z37m*Pe
z6(OUt_m=Hy%1nx67D86pBcp8-vR5H{WM*E^@iFSY@9*!g=k<D?Kk~}BKA-bEkK;V{
z@jkj)1oY~brxy9m`KYT22^){{YGtM4R4Q&_hEqv*<U|1G1ypu$SG_0`#C)HFI#5S5
zg^$lM8Zg~Jq7q8l?~&M<6UsvkAU6oUg7b%Xt_)^eIpw=?Qw);tQ=UjipXSp$sI&QV
zFcnKV?3mQm%Vc`8#*jd2*@mn{fOy}$VkT2?-wIyngO(EGq8<>P`z6XStg9(#jm#^j
zVP5nTsl^1pElzHob48gHlqjTbtZ!d94VwzdV`E4NsGystRKqEX*3?b=u7{J3t|Rq%
z|9j4RVt^KnG$Uc=#DqW+>=7XV6Nq-0Qe(l1een}`japguU^(Kugq#cqA=KZYmqa|E
zk`<a^Ris@BA68@5aNwPof*r2)QkB87EiD}BNo&c}6;xCUhdzSC^Ypa)0kJuoDy>Cn
z7*N4qznD2hU%@#)!s^Rb&x(TN()w|h24|Am?7O_1swTA!P^l7t;X*%5yM_1jz`Tf3
zuXdH#kHp&)G*>(+$@&+DAdhwc>jwqSvTF5RNZ?Kj!B}quM6M&U#}vPlpXo&-<EjlP
zsbfRz%GAm`2=9PzSOc{OEGql;WbtYbCMjmT4`*E*Z&gH~J`_VG^s=vttwQ)K*{tE2
z+dD8chGBYlUTdQg*bO&Vh-xz~+{GV_&HN46TE`Ev4|81NHLB>nnf|)g1JXjF81tp~
zZgUT|TCXI>6PZa5BjpO1KiRkc9zPT^R2?A41^hGN0ty`S{RviO>8yZp=vKoLZc^m!
z-1QIZx4d!$5()W7;z#d5b!b#E?OgeD?aODaY(>m3(gdeJ(VMx3KYOjaGU#32>9LXr
zNN3K+r-w~S4+x1K|1CWRA(2mY8v6tc_O&?pK#pBp7%@Cfk6hxNDeFc%LS2}=ZR*FZ
z0Fj9Y=u4FqxP|8s**VTOTh^VX<Nwf8Zk{O>YCxVNHZnw@<=3g!oTP8Jt!pVg;XVuX
zuz_h|!9N>f$|fXwJ_jN-9^XzjTD<ALS@&(i*8+*I`$BydJ>w~3e|sZ%207ICx@WH`
zL7e|nk@V+_MEE+JV$O@mv}itPVEv~3<SeQOVzT4erAkPe7rO+Q&h&av?U>O}>^Ov)
z{32oAK4kB!9Dy%c_e+&heB}oZ)oA%g&&TKg+Ya&XMwE@m_?}c6`AEC-KiJ$qR_Ic)
zZ>lj`y2Q|=D|pZ;k_vm^T$MF`At2x*Jqsptfvx72P;!<I5inJxD54{|658puonEM|
zcl>6~#xfX$7aw)gZXb~(|F2#X=r;?+JeX%W^eWXKt<KmwOQTC;nI+kJ?faV65p3^G
zDh{pW0B97(JXW&jw}Wv1=OBzw7~Y&!8_e`8BWIbDo`Z0I08OvLgHgnI8gEOFKp+>i
zlFpIq7+jWn2t(;QV0;ZyvE>AomUy13H`GoGKt&7o%q?(l>JUh+Y-+kZ!51D>8bm~b
zMTvRsf=!Y`6K+t2nCm`ks1yGLZy7F6-lObEz!T?$o=W$T<ENouW?=<P@1RHHA;P|~
zpT2X}GC(~X_nbTx?R*`~XTMu=?B*agJH0+$s%J`KnP{D$iTEU7>_J_5^9T`s$ffYh
zhF;UcI$m&jvB#T#3xyBO%rhT5c2Y6j9BZJzQ5OxMX#XO>KffWPLL@)>_t9tn6j$iG
zRKNX}X9%taPS_0ou1jGJNW6X)!}$Wu_2w3dNH8xKdilzY%+q&1=HK7L0(knH;l>oN
zm5uD1kNvD@3!Zn`0UT69Tq0Gs{8f^bW*6ez`SKd4;=1Dc%F5hp{o+S+FE++Tuf{pM
zQ+hRdU6{PULcmNpGgx2~UmcL7u=s=s`<QeJ8su8Q&Zxg=eR2W9PUeNW>H65L!fq9S
z3*#~SsNDtd=BBsFMP5t1w7g#kl@isEq?t-UW>Zm2ojRM}K27etbaTIuEw8|sGPTc=
zetuG0^$mreY6}vz(u9vrIy7s97kd4c>4@cINm5^w`D{oyp_KR(55N5KbiS<_|9o$O
zTO+P#VC!esXn&_!jO}nrl;I;tYL~d**XwYrgRD9yl+}=_=EN-v8{oD!30h5<kyL0V
z)cd)`)QClLV=%L_4JvP@zzs!ASVA+`$*iJUC^I{BQ*x{dx3waJRF^?2;3UU`OI#R?
zJu)-m4FE>jGTxMtCC(3xb7@hQz26mv86l!LGwWqrx@zS9^Je&=#wBFAR2V$`|9+Xc
z;*W<F=fmR*!$Xmv{gE$J$FTqw+~tOuBllSktq$tGcDu3QozhWRJZvdFQX&lyg>voL
z2HfJM{QFTU2TvKXGTn@+jx4{Gx3Op${-$Fe^C$^Q`QVtzImLm|XM?FNc=+VB#<FM@
z)TezUknU}!5Or~1hKkEs1p+#VxE%ZK!e}CH)y6UYjr)taVtbco7{avB;wLbv)SyOA
zbNKZ2J4o}NFs0daw5a2Y1BXvx484CTl$|Yi7S-{u0<FT0w5dhpYv-(@2!))<iw^@l
z*rm&t@0LbNmxaDZlGwwlMm0dJaKsRYz)i~eC_b^-Iv)pcNMGc!LT|wP@)%dbi3=D4
z0wE0A!ejb+xaEyop}EZeb7pXkSaCWuCh@{*iRj*``D2m^U4({{6oZ6p=)LtLW@w_l
z?dKdrch2<TC2V}^gDLNGbFEo=T=T=<>kAL^d<;K9CS<|mv{9+RSTN^TYWcI%+V9sk
zV{H$Ocf0R2+JZn)^ny@sG;7ph>N8X<apy;x|B&AC@{G@=`}il)<{*fDX9`@;7D!Ms
zS_-|^Bp}^d>!=J2oi`fG1thFmFq`ee40GNcXnY}fQY~Im7b(*yfrQvp<0v$9L_vE|
zw=#<+8C7%41XKZSjH~-Sf)sjxhZSgU0n=LA9+igoJmPha<1gmKZz(I;3>mm}vDK#w
zfa%l)ixaeza2Z1bqOF>lgoOfucFx;*6$Vj<L>qvEsi=YEFe?ll2<Of&mV>VR!I*n-
zByG`E(mFvdYbm%99X=Y5wxjw6-~A(Haf_AAeG)&?25$hn{W<tV4gBu-yMf`ywts@H
zCZ$4Cm9Tuj_m`n3E*Y=iP*@g~S>b)_`lQ;tY52ki%^hBUbfS6bRCqaG&8Ga2J^kv~
zuU^YXZjWUivt`K#dBykRPPW*-#+9_|g@=E6P#-hl7h6t}tZEGL$RhKDoem$!Q^4{U
z@LIO{O0f=>+~qU|f_caB>h&1A(sY~L8g}S?M@0);N1ik){|ONhDb_{%{IBN(+-G{d
zb#*pw0-_yao+I-Xwk^YpAHb2e(YM4X#708YUIqN(X{WFL;ZPdeJt7T#UAhn?i9L4L
zODza9*~miBFm-$hWw!8i{&98j5w09Z&rrNqUT)bNn+^(pt}=F*ta(}0u4~?8`v#{|
zqcLb<g@)TK+t(TSa_8VFwc*=#hb+tV9h|1emU72QmY0savlP9xed{q=gyM?_y9QkR
z$tKV#O=1fMATorxBkfYmzf+oya8HJA-d>2`(wciyDw-=x9RBV@!;_^9?F7?E2D<gt
zJa3BE`|$|_=@6keMKRjA3h7O{X`6WaaVVyTjw8MZRf$)`A$AGMAIVT8$?AiH*I4wY
z80UIlman&kv+S>+5H<YVmoHU!|43?~`6_KS!ye^->-ratHqmDu-w{j~@lk}=^&EOd
z_AZ@<BUWhhd1lu=mfP{PmMvUQb#D`XHvFK9dMCHTKT5a!8;>@R{^)&&UpE>QEUIT>
zxp>Zq&x;z`o#v^(G!)|0SeVxKdA(gEK6rJ_Z2P`qWKjr2+g0y3tKND=!OSxK@zGUX
z%U)TKutFwe=T2tJxUcy@%yAfEP)ks%O-QAb;lJ<8GERT4b#k=CH@o8#P0_NZ!C^Wt
zHJhL(Lz26sQhJ3qos=BO({vX(_J=I69j!~9l3ahb!;Q`Np>E4As>UxGo}o%l+?FB<
zy2@PvH3hF!V}rNFvk(5l6V0YpdABXw8U1hmC8y8In{7xkI?o)WaTBtgPEw&a>b22#
zr)Bsp@6hAe^>zZo=wk}PTW|jqOlya}rT#AYc+UyW)dqWC*8PY0GQ<7AVH}3?z{sIc
ziALus2;V&Eu|FeN{RR<UT+R8eOa9br6Gaw1?!%{97GAD1@!~3q3&LvowMw~-^~KLS
z$?e=#tO-f8w8}ZLdD<ODqz6X)_NnOu>mDN#ktF{LN}R5L{L9PmDf4)byL^A~)jJeM
zt656e-MvjyeVcU(=k*$?>?q1Pv}$s5%SBl&{PVJAO07U>q0bgi?zF%0aa+NPoAF{7
zj4e;mC{mG})VO?^4f=u}1J0S}1q4AWm{C{-8m><)j5bB7o$0$&>N3u9o0u{Zs*rEC
zATZv!1HS7u-32c{fC?i)qkpQuB<_zaBi|TrH4~|^o^0d_olYuk<(jJPr2na4wMV}S
zpx<!RY4w0@`rQQj(HMP{ME{{53pU@EI=@brMI!DIv*vAr&xWioE;i)t>Qx;qg^$A8
z)vHpTRu$$qalYXXKFp`6EdR<c4CB&#<(U6%@LAHi2SR%MS1elk%$C#r2oXg}a)%zd
zSSFySS^&(cc~q2iv?1msv=u#s*r-_9O$9t`{{F=cwVAFwZ9eMNQEFESQTd}iijL_z
zou5;Zi`92mpY|A5ucGYCwDWx}JH{W=Wl?i@@3JX9#XPGdsXfD(S(aw5M#bQl7j5VJ
z%!_|=$1af9Pjod8m%0~V^TxV(J#+Hw#SeX^s~1yBd@kAb6?fbDWv&{C%M>;0>pbvY
zR-F+#3hUB9K>=VC(ykB+VGE6M*Ft{;gO0Ccm?m1x*(}Xj=?)%>lWNTKX|-Y0uy3=w
z9nI?R2fdcHK0?{kNls;tS^R9yKUX@oYnePef#0pD%%v7jgsd5zf+8eI&L8x!+1(%}
z(HNWWM@{R-j(@kHJy4bT#+JiIa&;^x(M4y}4vpvzTC}PquYYA>qNGaKp_|FAKRTCp
zvAai6=KNz{-onD4lkK<f1?Kljb;|by@*7#0#d}JLk}FbS646+s*T^5AU^uFBBLz`x
zFr@t;P#^@%jIKOFq$d<QN}NmJ4h=&?E*FPA4XZ!qJ(2hnZ!T`Br02ch0G-G8s|44+
z@t(8kQFN`NpP1;pcb$cLmwS?x{FgNY$zl{0kjn(30w1N!s~ZoJbw|x{6yO;VK5#k0
z(=r)Wg0qO7mL9zku6{1Sd+AyAtCTLqu5;U$FmWD59`nGaAgtQ$wtTk>rJv7qo8-*7
zg%T0T4KZbP<@Gdgwxi}fW=8@K2w-2R=@G4u&9-)0KUGFw31B5@Q`y#hw+WH*Q3z({
zTsB0c4k>+eG{M@{*mbY?kp+cwPyM@7oF|X5G{PO}sPiH2{JH#{V{aSku#L6a*pu7K
zQHq6aJmA(kM)GGH5wMNR9||XVTW05bnJP8{jh6(%8HVJ|)TmbNa;wz^Uzsz`ov#C-
zikPb^oQZ-2`8FqWb=AXp-RQnH<?d74Uv|qa*mG!pqAB-%qpF)#x8ex*KylB+Y!@On
z9nw4_X6U26b9B>(s2|knJR05>oBkL=qWT0yuXy&{1$*a#n0<>O3^Z{f4!u2!B?Xhq
z9EGlvP2Z?@EM&ANDnIOn4twt|ZHUOI&>HB&HM_sY!?uylDKs}J<RracF4Ib$wv(CV
zcCjsB_%xqtFHhe<ue))(I@g1x?odGH6%#YYD_T-L{x5aQ<xHkk!i6*HCnqMDtwTJ7
z{gT^t&Pzm$ua#afUFK%0C*!4PK}6+Os#8X-ZLK+0_17z(g>J6t<y0s=$)2>nJ&~|R
z-eAqz`nB@SswK;dJe1H?^<>BW2R{~Km0H_b8qQE#z9}K4urP{yDL(oh7q^;CN>R+i
znDmm~cV$HC_Oj-xcZ6|SUpNa$-+{3n_g4fC`BK$8HUJ)Kgs#~wmty0!Y0gH6$NHK@
zL;RCF%bLDRrOH^_zGCOgyL=ds4Ps<0lHC0sA_V<0O8A=@g5~$+u7@Akqp=dgs-|?6
zmMZps!3?%D!4nsvbrrkhw;2?@jLL|7_O)S_W_65ofy3KpSU)~L%v)C_SCF%?ES4>|
zm2*{DNYr>rn8m0vT85W%*UC#2-4&lqsxkccS7r}Ii}Dz~sy(KzvYwV}^*#HZt>{Uu
z9Yet%<4I`2oRXU9p@l3O`w{5(u=B;pyWL0OVQKcyDLj-(3zbZjMHx_@VU>va8mGJS
z+ki5EQ?r+=HZHCXc&yJx`P8}<OMI-Hg_L)dj{{>Kdq8ldf=Pdy!?^OcCI_0W66v|q
zcijl6rZ4-cp1~>Y(@i*IaD>m6?IK&c&n4xeQyuR`f@v&w7l2j+25Y(0Ey=xogmOeA
zR7w`_=CAC^Ie$zI(Qg)xGnSJV*fj<rR_GP&(p_)1V-ch6P-c<-IRgS%eD`OKHLX9~
z7F?Yx^TTRK<Ar)&3zDX#?HgD1Qy#t{==CL*b3~Bce1~w4_?lnFD7g!+-}tiio!B+t
zM?L-pF}5t9VM!fY!#5#voDP(`ucu-IF{{Z%KC>-m@vtulNvJpiLn8j19xbrHMzP#L
z;%%j(cPw#~w3clwy<*8bRzM;jkv1Pq^xBK+$Eou4-Ud<bG?mt^(p*N-Y+v2VSQR{t
zs48y1n6UD%I{87N!c)!UTg$vhgy4?&P^Px%R5i))kzaJyAs~Fu$C!9#=JP~T-|Nv!
z;s&=rztg!|&hq=P;{5&-Z>FPh>B`$y17cke?r&Z`Q1~6xC8BIkBPK9A(I?wKCa^1C
z6>}{aH)``o=7)Y31B&$c_Z5bhuZ>19;uAWN#lF3sHv5a1F=4VSAr34vwnU~xL>~PR
z6SS_HWO3oBMUj=%FxSe-(S`G*+cvqCE36stwIpA@#*q;fRDamq*M8vOAL+%xb4Y8)
z<asEGzgdxDDiPk``E1t6?cmp@?tN-xUX36``-|VM<rTgbRF0#?CKpOYXh};F?<{_g
z31C;WJfNfBnZzPH+%_VTFZs(nrZbuokEo(AyM%jQs^%W9t^c6S{*t$E>;pWMoIl<Z
zJS>W+J(D&X_k<y2V`zfna~^x(SUo-uiTq{bsJvUVHa}hu-f$Fcu{nG2gu`8>p1z<;
zee%P<$B%?7<SEcMQR|B6n5?40>3*(wetR_0wNEHyA1+h45@)R)-SO?4OgrO()vY^=
z-*W4v#!4%meeyLZGAzIEw<8-l!u?kxOr#c)f;O$hmnW^u*H|u>X6KN<@|!p#l=IW;
zAo=IE#w~Gf%!eg;#+X^1XEx}T`~sx~HF-T7yITa^hmKKc_#FC%YxM3QB)-a?+vXja
zo2-Mbv29w6<WdjaJBA&D${+KJXfZ}~>qo={SJhu5^ElD(v${%YR5dOx5}T}rTS5bE
z8`pYVXj)|)&RBA`RByb~$BkwA;dLae{S5oQcHeP5Z0INDNF#H-Fy*LXF0Q&_hH~b(
z-Q5!;A1_02q&V{EN}RrNpN>$sY{z(V2J5DTOm|7gOsBPBL0-=04RKs|oWGi)@igt*
zmJJ?UdTG*Yx5I`TKCZ;qr{yzOUxSF0Csg&aUB4mk`>SVS*+_mSuxD4#2!kjLR4qL}
zoOH6Dbs@E-?Xd)9;D0s=-zLk$Iv&#sg#m^C!5t@*2Zs-;FZ#q1AM>nsmyy^ZK|r#9
zsQ=pq1{MR#9Ez>rVKt|VJD)kBgt=`dbHsq@=xZCpTgQT72=Jc3{zqVF&>XB<-$G;Z
zfBz264#w8U<iujNZHelD%9417!RE<}Pnot%>J;+qE=xlXU+{cYhkbOF|26OwqC~8n
z&i-F-L0^Z*$!B7Uca70MqGbbqK759S2-zp*0FlTT8r*ZGQ38mN1bioOsM~puysGgC
zOL#hepsI(8$-55fGBG>if3J7vrPz5A(A>v#oVIPMzq{l}0<*;QBb)X1W=aoPG<Vjs
zH>)St9S6tMjGZp+)w<UYid}sW<X@m|((VPVHs)sJH_w_Z1s3;;t^Au5dkm?Jh)Pn{
z=-u!4$aP5z2&iJ^%)9+Qak}Pc!otoJag9W^GjM%_;QFQ(ef+dbzsFV2p0F%b(nbrW
zh@UcUm#2;s^WTxg|Jfq;F{0O6ZjHX#GfcMs04us5;v=IiL;c%7wqi@nBhBx8yJlXl
zCjZR#M2iW*LaSEtS5Fr52KC$Q-(;2=R0)@kXyYykpAw{QJM_j#?bx$0DM9RNU~#;!
znhpKBYoF)Rv8|mWOq54yyGloTd}IEqBiK^8kmpvQYsC5@$67mU{WL2s$Ix&y&+5VW
zqP=e<&cv5KfG37K_qP{FmJB~3X8IjXv`6f{?Ysu{m&o#;Bl~y(j;xc8;mnTp1}Aw!
z1zje!R&|z=ZDRH2;s5(l7(DE6I25XZ+@Z#QA4(gUiF>sZdhF?l(%j!Grf~r~wahWy
z>|_;j=AT#VpNqj92cD#Tp`iEgS7Mb&kSE#&hU$hA=Iru62<j2qYP+x&3f@!^k3W9z
z-{0>1kCp}G`f<JF|FC7i9gN$3p3#^8KQGdsx7jAiI3Pu4AM?<z0^MHbT?i}jTnb*?
zS+hJskZL;8GvKoQb3YH&(Pd{NBjR_DL4z89OGBI5j1Ms@<u!q1DiT70DIcYdKRktP
zJxL_fEG0qw|M7buT1d*RaqN#s2zZTW+eC7RqG;D-6$KIZUM!q!j{$cCLm~==^bl{Z
z^yy4ozC7}_`|}aoVXqB8nXR$1fiBA|18)AxwO0_8Zt41j*likb@%%Wr`}qc<efY^Q
z{QCLi-Iwin+9f48C^A`R212Uc_*IHw(zLFX=KQWG#1@0G66{6lgxfhs5-2o4{b4Sw
z18U<rPwc^Ta5~#jUduDQaT2btT%~;(yidEn3icXhsysV|Jc;bkAf@Tun7dL0J+HNr
z&q7R^24d2ogC#sDOZr5gOBF3&3cU+=y&pW(FWN*O+9s!=?45hz072Bj_r`R8&H#NM
zjP{YL?^Gy~$6ASiiYPD_z?y6*waITiPG6U6GMuKF3ytC}rYaY}Lb3xBafI8?rq29)
zBOC<{oT$osC3g640JsNo`DEvVY_Az6@A#y6*x~&U81iLUxU;?E2Zz8s`=wA%@W(u3
zqZtfJq>+6@4&e@JjxRi@iw#rw_JUrR!@ez`2(G}}uBDKv-?#m477I2X9TI(i(R&p<
zj!WQfu1??=j^wfJ8oE>gsj!5bkw@G^wmH@RygwfPp?w*DnO2MttZQP5`*6Z-sFZjG
z|1JU*1G(Y*c&I}7<ospA-EC3nB|=3#wAhmkEM6lZbQ<{{a~jdISTw9hR-j-Y%yyc*
z27mVmAF*N9Idn&ATDOHqG9vOX(MwS7ECN<Sd%Wt9tU{tA#6a85B|gq0Ipu}1Rk`I%
zjX`U`CeUhGl*anjasu%D?)_86A1HDG-M_{Uik<2C4X0{O8Tn%%jc9uRn3M@b<>%k|
z;U>kR$boMXIbb;oZ|;hd_GqY{*o(!{lERT|URy}p-nTNG7&@J;&Ux2E!b7Fd!;xzR
zJz_?Y3{!J_<3ZIQdr183!!}<}C;gb7k9%M%eLNIm7(|3fmGW(3@<-4>RhK{BDuqSK
z;-fAYV^Twz5RAq7wgBo}zyKxd<&A>Xe*|K48ByQRm)3XMV}P4v&C-G9C5|0CR44{L
z>2U4!UqSrvtUv!Pr$GmZ6V$S41nLq#l}fyta<&3ubKu(OkiO<k;1z?l$f6{si<8Dw
zQKc}POynSk!*CTg-uqtIQOl;oiCFy0moG1$juRW)q?DdZzCi$#;n0J#{p#YO{!xUV
znsit7piRO5($XH3<lDA0Xb}JFdiuQT9*bZQ4oN%uY8bS<2ceZj5MCbfF#gX|K)DO>
zxake2xDZ=DrqJVXI7G3!gp5$OS@LQhnW!vYW8;($r1`Ed;}PH2f!BusN41Eqjoivg
zAY0#ieFNhDU0suu*nYhn9EOsT?EK+BukQl)FKz1MRk-CFL{@^x>wm}O9)fVl3Cwz$
z{6pz3<H_6a`3R}fxbatbP}gR$T?Fq{!YS|x<>2he)Gj;7+3aT@BHR@!O2x2<JBLn%
z0f)og-3OPoMWa@d6?r@I6Ad!_3KFinEFN&xs8OF9S*oV+6^xyMQEA!XeW!Bdv~K7J
z&|6Yt66;aEc_hQ7C*V9<z8{7^;DtZ1D1CeKdQ4OR;qLPu3IIcLayuM@6+@iv0qD;Y
zZ!UST@5#8tqbP6cOB9#q<?*l&QJ?CO_Z@ds2I>3>VI4%IcK$s019EGZt|UI(=7sln
zKa5Z*1tHh-iO~Ps%ifE+yS=v&KI(ZzJbZmbba0_|pT?nro<c5x1}*<dpzC3*QQwi+
ziLTaLVU8y%L4RzVOWrjgu`Z;LZ_)8s$lu=Y16sIR;h$>H{<&JFR3Mp6F>L37{_?vU
zpwew&@yC1x*`A3USIct!d3g}}X>T!4w>LOH@}3S6Eqb6EGKG&b{L=#27Gs!vV3PoC
z4~H{<*u(|#bX@B0Nt{Kqp{|)z8!!7mzC?2DAU@$EUQCZ**GOIKajJ`0?`=_|x<9f9
zxs4WHTEzdgz%b}*2#qo^9!LMs$Oc)eO}+`(U*Cz|Z`0_zlrm!W%4-|v#lI_B?i=1s
z)7A1kw5G>5qvQ~j?m}#y5t_tlT|V9U|Cl$hyT2-GcN0_+U@p*?u?*d#R(gS{K#l_a
zAf-|8&slz@CV7Sk=oc5y^^)!}>^9f_{qoQa;b=pGo4$Li-!}izW~dS~<WQc{u@?nx
zA0mADs8MB)EH>bptkbSL@xN#NKM|Q75@-?K+J0bir07l#2U&ysd9>H|1GQKG(q`{%
zyN||nBO=C&d~o&OR_S4~-wiCMT)Txlyn2fdOOqG<v3Ai^XOOF1e~hZAWytqmpW4+o
zKt!wwnEDT==10c=MNzCgnMo6JsUGl{9s6VD1Z4UP2Ko07UW7UK&_KxQmD39s_<#1k
zv(r*UCjDXYy}!a-C+yv3rt5)Ei2t~c*I5(uNXF*3h?zhBj4#<uyZ&oI+baem02aF;
z@RRgkyMe`u!pA>^92Hi9^vvVx(c$NRKm39ddG`A${I5LBGk><SdlB~zLMa2>IZIxT
z_#=Bi!6e!dO<aC}xIL+>FIm*i&|2nVSHb(YkP5%|GgUSMLa8AN6K68F(yh>1ITZn$
zR`bo?i;AkP74agou9uBEvU((b6GLWos$`9B{d>RNYa=J-M<>rm`?Fn-W;U4WDEO8W
zS@A5PAfsXO*+hqMhWu0l9ZN=(4lYygx+f<ey>3PG1`n0)T2l2RW{3%AZ+(@#0<y3l
zmCrh4;M*$&C;w0nzq=%z*YnuXKg-_c+wCujKTzG)7JWu>+`hH|;QfbC5ZYucpz6AQ
zXdy2~DQ1Qxt5ASxD?>3xl@g^Mh#4o+yCTyS&G*XumNoLaqCdW*0=Te0&0mnR8~+UH
zwK%E(L%s7^Sv5#{w=*M{WFm{vbEP+n#r}1Jc9w&!1;S}~IA{M|#zT7SKC@vQ#io4e
zQ7aUIoAB+mNhREF6hK}cV}7t1gXNDuxVlCzNPvxgfH<%(j$<SQ7)PoMwps4uMSnLt
z$-0oTT?^q~N5Byc1Z7J)!TOKYw3FI2%O>|2hc^-jjs@iGy~_Y&PV;mx*vPb0HnK&f
z1QF`$;YeQfZF67a3@YDw@^f6qA7gWmo8eH~(&0s}wN1OeH^NsmY|W(+)s5!QbP1LP
z!cx?1cfHmh?8ebug#9$pE~(XTEgj1|W)6X`*&G^d-{m;E{m9eQN*IZvy5sNK#kmh7
zX~jJWm4e7aF6H^18<W@wcPmZMlke|ce|Q-CGE2?43JKbram2s;z=P))A#x(<PbjEN
zXgU{z!IsHKHk>~#AFl{Fj~@P_u{!>4CWb6DJ1^1D!ip9-t0&N^IyN=#`XRfo$2>>n
zOYNs)MK}%?6n+r2m6O7I@;?%m<fCGhP->CPOL5V}Yk9wWMcjAHGRzh1<?|W038g*X
z$6h;UVET(}?!G_{u|5$=b0`m9Q1Z_H9vh-Vr>}O$r{oY1|K*QCeoRm6z}@nlzxjxG
zM2G3<i~ibsP-FxtsKxE<$)|y~i4hhVAJ$xp!s;K)uM{l|D=;RXSi&!#uMZo4oNlR|
zL#11$WzxX4P+Ge7Mel}T!&u{*mF0ak1EUVdR^OC^=D&zOM8mr<2Bc#hLT2fszQTg7
zm&T`rCJiUQoyph@KZeLN3W><5$qQY{`tHdRq8}jCg-K3d&9v%2m;C7`3vZ83>KlHI
zk1|N2Zb%?J#W_e+>|H8SDL^;l$WJ)B*(v+(%MXV;ZYD=n9(ZkMGdR-y%Y3W_3VX`(
zo%gf9oYp<1J9h9<@A82VjNBgYR<XfZ+<mv3<^M(7Y(G3EM%$FT=ZS$BuZ5${O(^eK
z5_yLo71J1%<=cPA%=VvoF;a@B*Yt*7-jLbls}cG%b8+K>-NcxxLyT{vrH1VBy7^Uh
zr9mV!Cf9i0Ne?{niWaq3ywlb5-xzjmaivtuv+fdQTV{plKd&FTt2KCRSJ*=H86>?f
zG4}4hK>-{@4ca4M3+)OYPFipnCgf&Ct|WIbH(0p8Ofu~>ZT!NL5KOf?Q81y4|2|$(
zd}q_O5OX5m&Os^$=8w~FaWQs^{K-LHKyr{NG4lGGK9fH9$kG^h-GoF>?X2WN=ZH?}
zK$W;pB3x|meD;|55+rmKQce_8?z|unN(&M##4qfml;GBCv4eBPvXeggPG%^fae<k2
z{#OKx^UFID_b=_RVcT8*eIzb2ddWlh-zE4e{VJ<JUUPLU`dZ44Se1$`HkOK!IoS_)
zQxUQNr?qmpwQ)Kj*v+m5{JjzFL+fv{_Pb{9h5j_b%cdP^`QE>Cf+TDZ)Zt*K`Ey_-
zO5K$t6AR|yw#0WbNVc;^x+eYkf3ijx80@{5G`d$7qn<#|;ALB!udMcDd0A7ePFT4x
zpS4c+V)c$z!)~CIEl1hYeI8e~amAo|YCWBWcUdd|sY@7(qITc8fS6-o4$b2^{>kiP
zjN_2hKAFUi$y|?@NyA8HO}Su48lfvtOlyIQC0CLyCpVfNE=Lo#^h(znNmxa(aKiho
zbtpx5->*gXu{<v3@v<goUH&DHYx;i#8Xh(osryh-o<F*M(40u!$EUkZNq^#BM5M_}
zQ1=l^+qLk1<I+F{>-^^t7RD1*yX8l{L_(s810yW@%eA-@_D1~kHFF!|kDjt*LA}Cb
zQyQR)_KTMeh^q^fCOa75EOCZ(I_bSihEw{RtXN^4J(u?;EF(}+%rR<*-osR|j%_!8
zM~~qXUPW1Mu%sE-hZERQr|XQ|n$wMQiK-r6uZQX&blDCW12X(qyVFK3StM{GD+E`p
z6+R!Jjy}!jUV7q~+Ku%zt&zj-`rYAAUU6RCi7#PU2(NcHhIp-~1w4Ay$=RtL{A~X&
z3t&`%Yd`G;&G89ufe}dZCrJg1^w&q*8W9TjXqYrdu&C*k(7m1PKJEmW&jhKVz9#w|
ziLFcn^$|)5LC(bWyb1gQKP*5pMJx`lCa^)V&jw4%d1`Z5P)Y{z@J?h7BS!S2y)ndj
zJuP<RO5oc!yxWf5j#>8z7Q~h=<gu;O{x^P%y+x$8r0~#^o!O-f5<|D5g+vEOaG=FQ
z28DJD^3a^XFAt{t&d#8*(X)YX9kM-s&^Dw+yjs@0%k6Aq3$!YJ>9A`?Se>Lwb+dzN
zhXU>wfw3XB3QtS+0*s&&mNkRPg7kNsgeoI{Jbb@H%YT)$$^PVV;F&PGuAvPZ{MluF
zozu?#I(K$L=*j!XNHV+(9<6hJ8Q_`!I#kknV%*|cx8f5%z@=*m&2&-w?zwyE4bgnl
zy~TxLRz`9oP53?YN1TcFu1yqNo10G#x$pP}{$XGXNOMf>aucZEIclZ4L$Yf6K}V(Z
zlCKFe-*GKthj#4M{=uQW(?~~2`LRMp!R-XQZ(89eDG%w2@QbQF3QbP&xpc^sUJ)qc
zs(G>4>=+h`@Xteglqpn0dT2QzcT=eZbF{m}(T4d`ir%#&QICE|?;6I5S|EzK*cx1S
zIFcg<7G|2nc4K$lh_`0B%97TUpisy}b@e84pMH4!ET)US$GWrW6WUeYPh19V;YN}9
zzTXinbH7e7DW6|Y<E!Nfg^N82H>v>&a}V4EaF1cY{ET|_iYppA;WxHcDx(1ChA8%(
z9lQ`djXG^Hx-7|C{P1u!ODB7MxM-o&2gyTwscby#bs`HddhRbCJFk{M2fDtRmgvl0
zSpkXsRW!LDhjpK_$>0#n(yNXAXBE#Lb9C`Mr?9MJ%#ZLmct<Xe7iHB0$1(=4XVEcu
zWP*j|s~AxbCiD17j^t4Lax#-{MJc9HfwUJYGM(NL9lF_Q0OP!1Oobu9(n>V>ifkWM
zJ85&Hn*HSf{T{(`woeg%<j>C1;6+Lx8Rg{uQK?XlVcgPa$FYY~_}mU!OOXli-rT7I
zMDEK&di>3|ZA}e{3Nrjl&Jh3uOLXA~v-i((FGA*w3|2TqmH5ehB&xBZnr_gD$fy_w
zaJ3#2>8VyzVedm>3Q--jgYyFpLV>+GklktO4f|kl$^}BbLK{v>>EDU@c1%=MPhE8o
z6al2F+nPTH+*7cN5A4q(EIL|LXij=8nrDXz{^9bVyAWr+%-dsE+d>4;1*MET+qI8L
zejnu-!*2OyUC3U0Z1E%z>~b^IN)b~Y?7H6Jf4@jRUR(>3X82#aLNK=oEfm?)PDLzm
z0D9jOfK18zOrRi!F^0PHNElLTKH~x1<t@<0U+*TSz1-(+)CF_LrYE60L;-NLIbd>W
z6rlPTB2!EOx26;+yAv8@v8@AeIKH7Tqa^LMD9euQIAl7Yu2+{l`X$)nH~v|{xo2De
zh5T7D@b4WGTHpC8rj*DNmjEROTl<jmT(jX?kJgQ_yzQV89#)7fUnEY1SlaCR<g1MY
z-#~`vp1tH-Upj*@AmkL;(}fN$@c6B@CJe*qj|IslsLfT7c;^S*E%BS0U1=u%RiP3y
z9LPjyrlV9pSGR$n0e`}xSx%20f(cqJlBqd((^rk?-TsQC-E{$Ld{hK3|NWS5*HlKB
zMS~B!FGL{}SEQ6gzoqkZ`9-yL)Nke++YF8e5&6zhtMChzi-)e1C=F{m+bogchNcwI
zq~L``Xaqj+z6aN@<k#mTBhb_o0aI$K5fml_ni@fD4fp9+A=-xBbAo=L07kl89{oa9
z3eiEluU!l+ug9jK@x=A=VA3(qvFFg$l~?a*Mpdox`VlQSn97{W@kk)_9}I-3L!}fD
zD(O6K$DjaW_jgsSiA+b9C@W>_#ttg{R3#O!-J;oV^Y+*0To`o_ea7DKfirs<2boP_
zml-k+2WBQ;qqPt7lEcG#qijnb9Np4zc-6s2A1%WETU3!MpbWgQ;wWRTnM#l*NO-tf
z#+nU92qRcH5%57FJDXb{^B}W;G?Bh0AQnn{Z@&t_kjXkZhRB%h-%J<2RRB}zbr;zu
zc$lFG1!yx*D8S{=Qh7A`pUoiOYH@=3#T68H|F1(iM{;yqk4cpY9C4PWXYkQ7i+D0m
zYEw2{IREjWdNqZ_6uMi?e15pb3Bd-3evn#sO{Z)@EX-A^hL!;Ymhd%!T>s4>a<PfD
zB$Jum5?43%t021rX+<|=!$tZw)#F#QtM`v?6Iz)nktzC))-VGLi^dEd7Mf9NVQU;P
zb!lGm&AtOyk=AIavaOvTshhrm6h>kRh-;xqgv--tW~5{^{veZ-%=CVUWaQv;diZvi
zaB^B8vAqAGTs%;&hWldtpPD*MA{F)=ulxn?%gKgMaJu$@g#F1Gu7bYS_IS1zte&NZ
zrmK?h33zH~nyK~8>z^L)V$FkTUafA5nI=-a>bEgugr!GWT4tSa#85dYnV(Z#tFZN@
zD+m)yWTE)EWU+XKgzT{dCJ}llU&yA<jKKYl6l@gzP+phU4`q<+v3+hm4SC3P?x-*s
zk$1zKh7dk-{GK}g)pGP@dcKGD|Lg*T)<mS(D^ZWXsa!BW3cD3ea7Q-O^~+U<!wKZI
zd6#73=gW~f>n^if3jtp@*H@xoUK4FfE)?<B0cP2-*9^3Q@yKlbPMSUK_W7c5YMfiH
zc|diAR~Eq$o7b|-c7+Qh8}m_>kC8Fi5fjjR8uk6deP|E*MYgT#SVKiw@)ly98^fU?
zkt44`g&HZKj6HQOS@t3>!e~n-JjZYWY0CJ)1M!Q-lrQCq8i~F5D1er)=yhpTi#`d!
zhD|9jZwKM=dg;n14I(|NLbgHl?8V-F+kLJ8{t(-6se%Umu=cw8z}ePDHKXW-k9%Tn
z+cMk!0zTt^G68rf1+pni@o#o#nsG2PhuULAed~)>MG@4Z=6Wt!g}wKj%sP2KivE!5
zS7_!00s;PMyrk=mq!5uuAhp{&n?fO%gtUghcY3DDOs|eXAI+`#PH2W2fdSI#juiMP
zVI{fgVrT|IfW2#ynV@r~7akiueU*Ey%)R>f{c2>=6(C_G*_4jIY<YE=TNm2>>YoXp
zmtLO|gyF;HNNqAQ1MsTur=OhwVQ5q)nCScg8C_5`;M>zRTyO=uxy6;c<hy0=C!jSv
zJ-HF~<`9%AK=8zQ{98@=8JkWGfUxB++x&*~Nr1xWYy760x2Q<aPS49_^O-I#JYB!Y
z?s@zu4E~^*ZZjCm1%9hxkFmL)@N#++rYG=6e1dLY%OW;$=tI%Bd2_eV@Tq`lU|)NT
zsN;26oD2s$2Gb+Zm9&mv<}Mz~je_oc)lfkEtDh;GDLvl^1NQ3OvgybxjtW_5u-AYh
zWUbtswioPK)fVY>I9nE*pSh@5H;qUt^KUJMH828~#k%8}uDjKnF&kn)K7Ino(GT<{
zmSMxzQL8X0=33FTRk%;c#X1J`>(!qzbs{}BtZQB^fCsR?BA1iudZn(gxymIjJsW9?
zkAK&78U|lLP4vq=%Oe!WBt>tezdMc$#)$iAgj<V}TsfL|!0dI)YdMKC<SF%n|A$I?
zK>ND>;;-BIuKwZ}rbG%2;xvLb{0WTsWR?FH9x<OsxTZpa7E78R?(Yamu5<;B$6n;l
zL6G_-&RYZEenG9EGOpt^lOE4#YGHe59c!4R)`z#vOwb;{^C}oHgA9kXU}DNmO%JiA
zh<2TW=)pHati$)6B%v9dr`59F*{=CBU}f_Pwp654t~<Jm#LY>Dfk^ZEycNx&$-Wj1
zSc~F=n!#ti3(vhcTGI4>f@`|^{adZ8m8w*kt|0F90b@30RFK2#g9}yBJqji(VL*V%
zpYD8fBQKA%w_>s7{Mpe-U-g!CC;Ha#?#2&<2d=OxhUTj_zjjXhoFQ#r>ANXs)aqij
zGW`8&oe%%a5=dXDn_YV~dbF>2@Ya6soK*l$^Jszg$AXIS-VTR$ucZl7OV80Sgp7B@
zb$VvmpH?{=RFb#;>JyNS*Dm*5k{g-qjjDJycwrI-@x7_fwD#5QS<gS9mg-hVc#b}{
za$Rr6we{M7v(!Nzt;x55?QSatf*{^I?~JPPT~}yS);Ih4Mx#S8FG_btJT@-DE&*_n
z+zic@#V4V;ya4dIN22Qc&2K5l8Fbiw@~p@0Q<jJlPxv8nVAklgAMVA>{FcIBLgM|Q
zYl|WA?=W5?yIl(J{USM^qAeqLxXt~RgpghH@oAhciT9QhA<5Hd6px@*iRn4MqWZMO
zNCt)0ZmWze)a-G!%A#)N`f_i8z-4nxufxL^bjB_LNUXluqIzAhytSp`5y4z5<#=v2
z*Ru%)&3O3JH$i9=m1JBRbDbvXgjTaFQ3t^<S*!18s!znj-^~2hbhYBnN3^o5zuSo=
zF%zHcSFdsnp)s~Oq2SpZ40?E7KQ{h6xS^il`2wE^rV3&|`S$11i*hCIm$epm6x>*b
zHZ|r6T{2Rt=H*|2(m7V~xZD>Sv4{;P0XTRr83t-a5wu4B5S)ZYb26sj@!n2A3bT!l
z?vfjba`R0%V;}GA+20{--F^j>fZo`KvzSQ-wtF85G9bufb|9{*zW7>nSI)fj`GE?b
zzSPxhYk&WJv2WLB$|}C}Nsv-;T+|mB_Oi+r*Dbt}@4T=T|D-{>cIlQ^lHq?2`yby@
zZHhm9&%f}(h-2f2#yPlhdj!!S=F%i?w?EPw{p)ejc>U9)1GUuX!3|m6g$wUFMj96$
z{}7hIU=)dY6)IdA(1X2|p=lXRqp`Mz`*AX!&+)f=kjla?zMBk_GBY!*`L#v0bOY*;
z48O=dNq38E2Lw7a{$_oMWBK>~x=Z#yuAa0|oR?tA-RcJqsUo2OnY+d_-J27aS<vdX
z{Nl@p;D?6ec=&O2#?ZIWRTSf_e_OVw^Lo%*S9TkJ40JhXfx%h$S=b)1J!3nHCthV(
z#V$-5;lxM9K2_kL+gfUrV^P@dW5G~*YCM^%dXHUMmauu_(T?cvI+T)gRT>@DOB*Q%
zs7ThNYF6aCv+K@G%5~ylL1Kyyj7$wdr^ahFh>o|<9xCZSFB<uNApM~&$HYF@+y4|-
zf6p$Bpdf=_sU82>p??;QX|ccll+8j+x!^Zek`NRu*@!#r^~q&03NTH)PSh(!T^GmQ
zTz;d;h<^}`hVF$X6A|~BtEUC|ObUYxPUEAT0CLRy!kMJkwSbK*&~r_@-iFc%Z0vi=
zD$fK+p_Lav<t_@8P%;R}+m$c1*B8RW8zZ3q51^0Kddoi5s~m@eia;Q=0sS%=+T5TQ
ztgIFp4h|+|%*iafF=@3UUp~2<h=49r6T5Qt1nGG&IFMfCyH!<R<VAWv%iJDqqKT$!
z5+A~0rqs&8MsCx=p27oc)i+NmdgGI4%U9yB!azO&x2ewhEd=@#AU98>Cir@qC1hW9
zZV(1jX5&*Hzt@NQVzzGWMg0o^A+kEejC(5P(q$S|1MT9`T@+>4Jr&m3g|XzwfQlLf
zxV4B)6?1zkw~o)LhAk=}yDSR9NwU)^UNOyq@Ith-kEdBtp>BQV#b{fD)9+H|0rtMO
z3QHihK3hCQ0g$TBlIyu=5GCyaGX$l|8V(Q(xQuD6#~<R=8y3@!As4(Dxj5O;6!8g`
z%7s9vzJ&%~WKiALTnT_4pI&droiB+xOCcr?GXgwTvKc3F(4s2eT5`xoT2SzoK_$R$
zbLthfjRdfUI-HDnaPVZetyEJZ>Wr$tKe+kQ&e{NGFx+Eb*<f?u;53#3aS?7liqWM;
zjE{Mc?GLsZpC(cvEr$jdy5n8>5vMx%@agwpqGsdVZTsJrm=EZh$C*am&1)s(=&a;s
zcj)Re#SOF*4hBT0V_+hdnLhON#m{$J=R7csV1p77=ZkwKB7bc(<bMwj;Ggp~PDcIX
zu7kG4y0l|$ed#1y9C1flEotW6Wfgxn86xnVx=Jz&Mm4uGxrA?5fxWNS>_fM9G?_oM
z_k$;$!0jpkTZ`Mwur3E!0d0s7`P5Z_!_*F;1EZl>P|-DKd^)|d>s;#58u6YMQR%*J
z`x?-R2_-}oE8!KvrV6@k719v$D!)cH@jb_PiMz=>ZG7(7@92zKkFKON^?1n=?M8O0
zcg~yFdpzS++ig|j#J>r=!>d?9z=_T$d@F}xs{?1U*2|>_4xq#*+o17xrm@}dj4YXl
zVz_(u0!&eB6E$+@fms9sSmH=h$!gtm7XXg|f*om$9XA2Dh36pQpfxgT@4R`i13&xN
z`!iY2eW2iU*)=+R0!?ozqnb)!b5fSBXU)+L^V}gdxUfAefS(1I1W6E7r+6>HcL%ir
zIooH}7U{lqq2Ir99mVF|`qT5}0QA1k@bHws%tMAFivzZ=x4Bqr7nxqZh34FnQ{VnC
z*IW7@s7SacWbl%%DZd-#s^mYt>hqj$UQBjRhhk0Iu!(M|GpWI+|C&cU6L^9oPqp{|
z&DLR5h}<M~O*-;6nYV^cD)g_$;7%_K+{WCU#H5%}akyRTB6D^q;laKZBpJ)Z6L-oA
zeLKm6z@<yQ$PA5-u`U60n~|x!^4TbHF1uxpw<YrdwWr~OH<TAkG0E$iTZf)H+^;64
z{LLJ+(C?E?QL*0P6t>3X&7yc6$&b>1y3}#TQv5g!*yw#!LcS@9z}O1*baB6IEgB2H
zApWC-4`k6JK#LHR&>gdlOUEdlbB_edz8uPrxV!mEka$e>EW7`^yn%AZUdJyr(1|wD
z&j!OgpO00~$)3+<AXNQgb6PgYsYUsu$Au0bP?XP129i#OTi=MWEewao*UPzz&)wOm
zZ4Zre;GQ)SYIP@d-fu8k_!^F*6dy}ejTC=WS@;2_xP*5_l{*1Gd37ajbGF_tE+qr-
z8zC<6s9t&{kd*7Sxt#?}&@I>Rxe24VDO{Svf60Km&=h=t-Cx0l{?_#<tp>CMz+-fq
zj`iLz6jMV7<mnDwHV<=vLawHiJSy5;6P%s+s8tCiU=qhW)^&a1MipW%b(f^7lkq+D
za-)#`O#Kh9l8g>=tZ9eGcS5KENZ(mFH;vsteB|q>GD_-k$5`A5JH<f{oG<Q%>s$AX
ztioK0r%yHMO!FE<BEU%sbZcx*fT_*_Zk_23(BB<Xjl_v0rt>4lgrPj)gqgg;=_JEi
zz(h1Q&x(uG&M}-W31*<a7|CZa(KTv=z#GiGe!=8LrYrq>kl=Ju$UX6G)QQ<H<)h2J
zFcs)(e$vA{#Cxjv{bRqm;B1B-f^m}{5$ah0(M)=tD`;f;n>Wm<S<bM#Z&081)J=M3
zkr53rMZfF$8tfB%mTPR*vz~_Yo5I*R-}?gV$2^9qk(qgdJ)hhd>r4joUbretl-<=u
z#-MjIyn3kM#|&=KL$MKPJ<o3tlt5|cN3?+<%r+yQZm8Gsfv@{`2Dn!(fI_;S3K%f+
zU{%$Cq~*;iUw($b!d=UoSL*;bA3>im$N?9}tL#Q4gMqf8@2crKW1}+EM%tB4i>&s|
z`i{n$RZh&7Y-JDYFZ#$1+*pi{Rk*z$0xhS^1s6(3BK5mxHKUoNxx1fhWW|~SXv0GS
zO3CKzb-kC29}_Esper{Ej0pm_lTs&bn=`g%L}v}Um2F^Pvl~P3Q86uKW>>vs0XbE3
zX;S6ng_6nA3TCT{?r;simHp%L#o!e_So^6bMD>C6N3(ba(;5PV>e2Ld;K)*oK<QGu
zzUlH*S4l4Sz?--W)LkPyxD6R^)#1JbiJG;Of!__0bg7(mfvZFRLgfKz?R3`Dwu;(;
z_a`sye@fjq@+u>{*1^Jj0W4Y3JcF4hPDBXma^|@~<K?`u(UiiWrG+kCp)}(kA0ITD
zg>)b$@swa5VKv0GX2cYGa!*G4<BtlqIG_4*udqXyiq$67$yRYpPzI0;<Co7jv(-<L
z#xK8hyV<ee%)1thphj)_k)Dhzw9fa@d^G|X7ZEqYq(ma(3ikte?XxaE7lj@st8pGc
zQOoPGYji6y6fQn{n=jTBMjHGeR1saDFJB`fvFOq4G1c?sA!(xuP?3l#G!J>f*%g{r
zBz1l{(d17h_Z`~AX$G1=lbw)+ygr)#BGX~HnS%uX(r~W2#Co>2%FXHq{W`YUeX~yI
zFC+}(@}~W(zZHRE|0uwerdHWH_{oLqjjB0uoy!m2-&<<0v8Ly;4%GL$U<&G$LL<j%
zt)nme5^ez_ooDXZ%OQ`D#cOVhvjZ}xnx=592rPUAGHC9|#MRcX)z1BK?)?lClC9%4
z;)tUucs*s*yOLZmH{BG)Ava<z&b?}?IadL0Cg1FmYATm{1*l8Qe&^giXWN*@g~;*O
zBPHjuL!bdXe{ixA!UXX=QK06djYHn9j?F4BI{~)Zu#|d2hl^nm(vw^ljEaf7Yu-aR
zb>-UR^PaCRRC)IfFkBzcBjkOV_tX^v``s(nC_>4a#jf5}u}Q$R_?0XeN9)rnHlAms
z_C=%#K|WZL!cqGDzI`%o;2y*GcIgh6_qp@mRsk!lc}2W&Gn(RBWWRoPso5<qp3zPD
z>E60{ubF5;zftkES03JATFtatXF`zTOUsHRui{FH?BvTl!w|___c6hVRlS7z>cX!3
zQau19=$AIRuTMQ|dD7gxdGwX<&-R~!60b)0N#)FhO-Owisd!;`BoExMH+B{J(!V|)
zl$!lo`m3mNlh>u}b>Rp^Qzbk23$sRRKH5A*-`ALRWWUnxY*kShNlG=B?B30Y%%PB4
zcl|)f--A2J*<MX}3cOkT^Ee!>+v#^Lk)#J2SEry@K38=x@#J~IgP2C=So2f(O_8LG
z^&6t==a)PIn$=_Ge2+~*O%EZ-&3QrD7X`IB54{|Z#Ne&o%9&j8!y*BtL2t4W^Op98
zb%&bXq?X1I&vg#IY1PX0kC7{wO3!!F6GXdiZ*mDmv-FFt)RL`*sWw00O0<|O`lnGX
zTPpf@$_t)!B-XLxJOsEgDg8JkyJ>%5rn=ab$?bLhnn80Tw^_2C`e=~%5wk_73;_Lq
zKKYm-q+5VBVzjPZIy=s&U_o*D5jSQK!n2JRj#>4LHy`;Kw$_c`S$=2wsez4u6~xNM
zE3WptlAi#yQ>OVy1gUKPCmp*iEsD@>z-Q%D(KF?XV5%}aXctKGF`I9G^5({upHs0}
zaZQmtx<AB5!E3737ty0EII!T_>}H-B7ezMioq`qba_x#US}tV>Zznr~Ha>TDDOTE|
zIZ>l&YGZ2}&?e#+ge{bA-;Et=wC)5Ky*AU-Om}X0yeOO-Qx%+P?E_;<d{e)5rY^}7
z91J9-I+et%4}5MPBI!KcJn#;T+V(le*w~KO_$af_L2qR_J!M=O!FY_5ohI6>Z&>p_
z|Iw{^VPflqv^`q_=S1OYOGPcp$>p+H0BCClR4>Pnu>FDrPgAhZ%-B!X)!(^PxDo_%
z@2Td{*}gnq`cfo>{Zq1uA2SF+SQqnaIRK&e#jVPWbE0U=sILCHNi+kQ9MOJ@^)K~$
zRKJg!X``AppD{$Bz>D9?TWB6V%{ry;$=rRR9e0okfcV=<PsSQsI;44VCWk!pVd~>^
z2M1l%BCdJ4XrGIf7&%x16py|o;+b<Q`1==j35$N173<O|noSD-6{aF_W&-L33hB=K
z@Y;A#^0>ya$V|}g^3TU|gnuZ$lv6n4+OBQ0N}{Vj|H`R9rr-|ukZ2M=`mNCX?!zq!
z0lO}48r-!O%?aFUlb^T`GnuCk#5(1qK0qk%hNWVZ9DQZ<fficy+^7t$E6Te6tw%cy
zoHKFfa|*K>KVewx@cc>6K}men@&2&fEoLtWUPye5?f{6mj-su}v>8Us5Rx~uz01($
z3gOyZ-;H02E9uWWT;*3!B?u!3I%AT;&l|79gb3!FjlW`KG3HX$6v>^Isc~|B5zYfJ
zzheHX;+YL`;d`W108VZn61pZVU{u7)fx(!MJOy}#X3L6o1Pcu?Aa&z!Tr;COEasFv
z8vFW)o`|n3ZD@K0@<G7T9V^vI#Ye4mF|-*rH}z%?^vT;0gB(%+v7kL44RYkR+X>g#
zb0P|2^?)(C@s{~WH~m-YU&6*{)tD1gJ^)(<h0|h;{^N#Z<Wvp`lZG841x80sWS-A<
zVcc-2<xIW8*h?j(|9IastCQZ9-T?f_ypq*D%~a`<(pyx=SJo+baSZo0lHX_skW)g^
zh)9I&dOO#q?ftBq^mMB0DN;zNo=W?a+IvLi##VDP&ekYIodq-Sc#q=cpSbdr4N1=K
ztVgzAx(&ILZMk@?tZP=iU;YWupc!1D&Oo3xd_b`_d^^;TTuRZVzoKx@<JzABR>*SS
zuNsfIMcJR{Joko!S!~KTT+Q?M!1uh=2i`a=?hV<D6ky$ba=Gy9>Q@&#y4F=AXwBdS
zw%B4dGqLW1k%Pl8g7|0zYhRq4=J)-zADLTgZnS)lL0PNbA~?S9rC@NcQyni73#fzn
z9h@(df~P>CUl$J#7=JveJZUqw1$b2AG4UOc`3qPb$~R9KxEF0%<;+if)WQEPL_d9%
zt)~Q&vU?~R@r6B>zUjm@ROtxb_D7JRqKC{DHN+O`1rav^%s0ALPG!~mll8|1SxPp3
zh56C8mPq6^xiIFgL^pOv`;;On#A@*9hXAK+tTi?4spglyvSe^MuZwe-FGu@RoMy@t
z=<(&2*_gWxsqL?`8fFO$CEBWr`Spe4ANgQ>i>X;zA-7eE>q!d+L9^hen!|$k8^%`W
z!DBTq{E%&-3z#{E^&~6N7gfLTc0ugzlM7$%R4ZNUxKe&_Ur;*^(JnIv6_ZE~JQf_t
z_#0-0s1n%aCl|t^NrlnIIMXVBe5X%zIspvtgd?<PzXbbpne7X$ub83}EX{x{GnZG9
zMW0Aq^$29?@)v}X(z=_TR}T2rAu~ZC7O-@*#X2TbJ>JLc(v8!yyzxG(BKb2#fb|<u
zAFRU7KFE@n#jR&lFx6Klv-OKo41oFs%d*R6#Mh;`^FCybm8T7zOz>uTd9fUULG&H?
zD((RuX<0<~<PI><uTR3uGGL|71NdrSv~HE}z<5_)>SkG5WrT0gR^zyrL9-JwG9~)c
zC+E)k;_H>n6YFy@*of8aqb?Gl*sO>)7%gZ-U}WPL`i@won9Z0%5Po?<VtwQ7VjiX(
zMvYlj@~@12J)3{ws7te~TbyHK*4X4riNK8sL)`hTx#!h_soNgpiNdfKTE{etLS!Wb
zxD;x90yd)^zg^b(_S<_g@2kiM0M_MO>$g9buyUTpGl7)qA$WqvBPu!z<Ht?nYtHI`
z#xj?%_y8(e8rLwT5HDi8?BP58WJ;*9(03n%qWW{)ruEEqYK#ihvTZkT>y@~SU%EcH
z&6PUPXL>`=*fCWLepH*dfUU~pqd|ou`5`7KM!Bd5>(Yx|cBSL&E7`Z&8xEl23o8mI
z)lQkodWb&oTdgyyg;9B*2o=Q3<SMP0nhS3n587f@!&Of5todEotUS3Ng+*yvj{dY{
zj;_KR?|hRqWxQpdh%<-uzuyzKWX!Rihb0h$+4cUELZOUj_%WOMcE!m<EBsBj1m*+V
z>5sqhM5hkyiBQ_pbH4_|L(fb+>-m2;d+&Iv`~MA`(;=f|l@U21lp<92DzhZ1jO@(p
zdF*khNFpO6I||t=*(V9vdu4^pV`s15^K{?$=kD|UeSZJ^9*_QL=y1;a^?Hu$c|ETy
z<@!1Rha1Oswm^nX3y;1nRmLc1^!gH!i_O|VGc#J$dPu0mNpi<+cnd6<FLrfwZ-Pc-
zU#I)BmVFCqOq+Lf9$yrEEAu`)sM`Dy6*4QcAl&DuP2rRSMO2(K;EGdXHqb(?ruKC-
z_%+Hb3AleGq_GVZ@UAxP>bNreg0z-P=tQ+g;e6EWjBc36e$LO5Hb+aeq}s^9$-PzE
z4X1^{9#y){w<#;4_3KW62ox#Fi#k&`AuU}xKXsM&?XJ&H3*Qk`^3TF{{BRz2Qn!Es
zb%KKufznv9;}dV@((cNcfI~co;HTitTvCZVC4Wih)#fsQd3JWkbfKH2b>Ur{Zq@B$
z;2(N1OHX$p5fWIp=bNig*}`4dHkn;|uu<%&mDuH&lxx@QC{+jrMI;=X?(fJ4JqnrI
zGcz$FnSrBW7Q}?{@1&O6D)Cr)Bq_;qWpHc3EAbmoBQquZN7slJ1f;1KZ$0E7Ia`?B
zCy3#y+Rk{8+(%aC{}x}rs<=x_I^5P|U*w!|b@HZ8ED39)b^1<U#FpXL;nvpiv5a(V
zts>2IWXMxZNk7Thm(yZhR~3+qn%->HR#Z~ThAD3`God8TA$m;7VrBemLKB|mQDq!1
z?uVVvtzL}f!4>;;&K{HHz;-y?88|IcP}TY|N#MR&xu`ow*5HnDNoY|vvtxDXhJ~os
z($&7Vg~Dc9cfQNqd#|*@e(CEmgu&sRSM$O`r?NMP!lL9M_Yf|meCXoTh;PX<^dml&
zr!27|vzum<C?UJA966K|wopH)&5i9nIJ|AO`cr%E?0;rA=|}9cj}(QZm)m8R4+%O&
z{Z2@q=A~+?i&FB2pSFmsWB!H1G2`=6)YT-6V*H=jCWy68+B~RnyXLWFYjLo*9m67U
z-KllAaS{0ohR@TWDH$>J0-e)Nak=yWWhs|ZSmY<S5f({+30D^Rwl*{PO(H4^-G4Qu
zS<B>#_b#WNTXn^Slc=Ok6S=Mv`-NZ-GUb7;Hnm&gHumwE0==ebw?;WZueI1wqs6=4
zEJIB&?6!WIT>_<L?(Y3tqK)bg9vH8pC8G+xV0`4>+Ci}(9bZ!N-CFd{!{IHMJR}Hs
zR|YejudZ|TB18<@jZ}g>(j}XbR)<e|S7&M}<G_$9dG#9wrR}N<RIrx>))dplhBjAV
z-Z1!$b?w9M(Bp!>b?1)ZjuBx#b*QRadKCdza8HeiVV!nrZYYV2`T50R1awqG*Wz&Q
zE$~A8yYQdQA755P?=)EcbEs}@ZPUk8_$H`ta;~^?Z!7(dlVKwC!%LOLm7cwtrkW<t
zP<Ptn*1VP~rYxGKxhHUkr)f_gn;QyQd$Uy5x}pjlv`Xs7fh(^DDldYZFi1yh$&T2x
z95aOGW*#^kxpoWSFXq<i32szZJB1{QU0W!w_r91bZno5njJeyuU)bF9Y4K{G_C#q3
z7k(O-VZKL^rCT}3U&uRf$T{1d>}Rv9dE{~YJ6b}-oM!Cm*w9SXXk?b<+s@V&bCv-N
zVVcCF<07-p5D5{;z^o9>j|qo;_Y00U7rRKl|7Xk#{m=!dbEus7E;xnmCh@4^r@xrN
zZVYQR7h&bKnYdf~$$xyUmlNu_9+G*@(pJ81y(B=7A!amF!sLk%gc|Ot9ltF=^lY5%
z=UEQ7<Xe9!>*DRlr^SXzzIP0M`?7SELVyI=62b|leW9ZJaxR<hRgX3wT!2KNfA$)f
zIrXTlpPMPY#f(b&`AR(eNh^Ijz|8Yt0<kO>ioVc!_rP!QlKUZ8T8mE~&Z$>7x?2OF
ziKEgQ<|KZ(CxH$6HKPkuNz)LwL{FB1@Axs(yZH2>TQHp|ZfoUkMt`)iLLjT4tiAQJ
zMq5y{rMORb-pfdIi=%nz2VXtY1VlvWP|WIuH{Zth6fAwU(Hgxq+yp}u+Bq>!-f992
zxFXX=_=l^@>a7<Fq*nrxu4s8k^hd!Pi-m&s<6;$O-&i5YJbn`-QJ0*5aVeG^`P&mB
zX1edA7P=fy8>(V%C~N^X-{aY!o2_?}b3{bEHaY?BY_1gU`wkom@o!fG&MKY$DlCbc
z2k;G7+6vZq5rP5Bcuraz%~Jx5AB9MR!yRxLEq2u7IlAuH=cfuozCpDnCvT?A^3BeI
zLHT#_<vE@a+?RoI#yYRx{XOBerN2XVT;dI5&)m;v-2dr&Rhg4KWjupaJtuY3Lw>iz
zN&O-!lXT~N=*U}LyP5y~I()-3uKBymS>YviHqy?$eX%J67z_y%r?y21!6>^g;n%i^
zp^!Inez?2@p1Z;=A1w5|Y7+xXW%E3@M45#p-g&(Q{T}d=HD7OjM?tCLe4MylQ*fvW
z%wjEC;zj!^o65!mxyi~lpOVwLwVn{2+Xw&RdEq<qshgQB11Z?xGvb4S@uQ`TB$K-1
z)G!kG#_8`>hnwz=IXcv&cd#0z;ySw<ffjS%Ekz*lTBXja^LTF-r+6F#g(Gnw&a|O8
z9!}V1MVGSZ$q4zJy(YAhq^RE@`A$r1?KS;BTUt^}c~#T-19$s{ik$fP;U;iXe&Hqg
z+}#HYxT}TOE)2|Or}F!ND=^16TQFbf^{kxk_{9e0a0@N@f;Qb>hq?pPGfI++9SiGm
z*&oRSMTdd>Wc`@(!vUUGb9EJ+^X^cvq3RhihfuwrZn5XA?psP!67$+{6WBeiO2~sh
z<WO3FwSaURQ<2{HrbT$@P%@BT$ZWucD2M`C0FY|g-f4j3xCbbvAG%7uU|k!L(Y&kn
zH>9D`Dk8NTu=AjGM|Px}_xDp&2^CSX8TQt1`D1QDfDlG1C(?Q>C&c@O$Zj{}>YaMr
z*?367GRs?FnQ)j-f*UqVd0FeIg4aMa@&ap9o`vrvQIH=c9Q?tJlt$o@tI79DkdS5L
zV|y6LfZ;1F2Q=%E$hgr*Keb;w_<6FJf1j3(xF)zRo9ZQ>HX$a&v*P&Em~F9ykL12I
zTTVaZV~cHqf<H>}?Yzm8-DY%qBcPu=U=HFul{bsy1+irl6H#~B#I;zPs!%+};Ls3s
zpVvf=*cIZZ@z5u`w;S&sDsiI9v*Pxk*5G)bQdp?@YaZw*tZ;ukI;QUc*S1+9M>f$0
zfs~n#E5wr#@r{RQtZelecU=BzzhMYbg5={)_5nakYlFlXqw74eLG;`LCd~~vm=}dQ
z&i7s_jJNoS*SiF^lyO)z)uwTOo<WS2=~qV?x-Nr}fe5IQrFklrYw(<)E=}EFq45Bv
zFuds@2<5&R=zR?kum<3ErOj2^OUGkmB)A8vO-<yE<F)@hRNb7#cV<%TVftc>p4$bM
z!nyYo?k0L?x$zb((H)TjY0<6W{2p7&_*cJl>(V__K7rhT7}DB-Ulmn&kWwV4?J^pt
zafh+`rEaC$^QuJQCU{IdyST!T!kEn-BuN0nfKa${lMDEk#w;}CfMm7(b*%fw>Ne|%
zC6)m#-krLzeu)c=DYcw(!`JE-8mOaa2Yyn{Kp|aZZ8dVb<?TW+ST9J*&&}O{*^M`z
zyB6bEy*s>QG}E3Ovv6Gv+JfQtwymWwFRpdWbG~*1bm+PhiWUzgT_}vK#K2z3elk?v
zWWzkx4BTB~7H}uKz74N=Q&n%=gaNs%Wa8~H?I}l^`)4`E?kK64ndof<dW}>A{tm71
zL{+rU<F=}#m$;3eKq=zF*WNbvR?{<uLinK<E1p+ZIJ^5{!PDyK{NYNRF^ty3f#}yg
z&|EQhxCB#^7tJ4Yy1|dZw7e0?(*ShWROotR)(*DnB?DthSqpV$0taUgCF8#$l>%bU
zHVzzHH#gCtrl|;3+yw;mcG1YxL;TR3?{icP`(Q%%G_MhG1K$1|zsRAX-h7O_l_Qs=
zn>Kv{ZT)(CL{9P^x_=Gz*kJ$u+JVQ02cS5sCIN@#TXAn%b?(}@zc0M+@{c@d`Yl*{
zcgkoh{%Z!s^&D>YIRkc@f2LF@B81`|>r8E*3}fz+p38}t;5hD6J1m)gmGF40hV8D4
zR+-gL0a|^fTW8DZgPn3W#T3{~+g+0t?KP{X^*mmPSEX-G-;POr3>=xG62MlN1hBkG
ztbDW)9A0qB3cMA|zQcT?qDeJNaY>%9LC~2(e1N&hZ&7j6g7ZCGAu;u^xcWvY{pj`q
zwmLufSj@d+jBG&#&%H5aAsgz|_ko8ZwCr;BXwgI{TB=dEK(yUrED#+{R%mt^Q_l7v
zng9c0#cm?s{7Zx1cLX~Pes0w)>Plpo?HM@c(c&esaCwG;m`sds42x$lf43~WJ!J-l
z={KCpx@qv@^L#0*Cao>-6U48Up%>mLYT*KVosI{Plp7r-!3$5XBE6rsq>a!15_lk~
zwSQ_~si&<s^^&3`tbYXo7?pxHx*vGOYBL!_2(W9snMzKfnW%(1C;Ij#U;!>7b}XQP
zkEZ+(E`K&x@fc-0B<lh1gEf}jL!MxNuiizVS7Qj97&WpFy)!<1uvgAMg1nmU0){{_
zo?hIHOxc9*7h9L1|Dh5Ph|seaRcIe>?bt{|<9c=|i;SE#`rMK}7Y=-8aPtC@2cr=&
zo6F@A<-}3QOWynQdKSn;xwAkttQ%~#J9V6*54DnH9De&+{&x@aiHk@QA?k|A2T8$$
zrBKV!2)$o2d<uU_`_{Nqu5v+hY~s~_)_~KCp`(3B6Gr$K6&{R-8}1WLNtaR({|Dy&
zoyN6~h6%l%{AQy$b;9RB@@4M}B~NjfjbH4d`u^+qc@I8jV?-jpx97i7NQ!*FZqXL2
z%ng9DWVPc{oB*4(`zmg<+IvQ<rC}<;0Xn8pmFYLZ{RwM$3+wjjn9xb_X=1BrW$DOS
z7n_(?+2qyMYXXFbgsc{&<FY@)<8x!AvEfU2K0}yCVJ=Nk_Ig)kG&U+sXVkH=u=~6_
zpMWK;$MG)GOQNxFOKwkJ9G#D)_>~}r|Dbl?uN77R#Yk-W4TZ_o)?urXVX?=}Ty>EQ
ziXY`fs`rL>n-%gCJd~zOR9uZDN1=SCnG@SA9-arh!GXo($#DC2jHU$qZ)_&k;kJ(t
zr&a!G4h`;Q#>Hv-UikC0s>z0rZ>#fa##ScKB|LNHOs(U-OG|H_`BhOSIm5K!U--Pt
zN@*~(`%Brr@}MKhW<_GG&y7r2)xcCA!Gk=Um9txxF|A{m*|pcX(9ch*TWYn1KObmt
z)|vi9O7bi2&C*(PP}!1V;`|}M_tqB{vByqdn$08DMhd=HxwV2xgUzf{ynCzH0?bU;
zTSb@5RNL^(opZ}f0!^XsjHh**vk<+26~17cjvk=DNaJA{gR*kAZ$h&5Da==uY_Cnb
zcgDt4UZly>nP8$xdK5#kb#+nxj@k|9gr_;CCFCu~ZWB#!#*XgQtud7unDnf+)HmlC
z#SE~Sr>d7<U($g}<0oK}7lcBt=PK@$c}nIKdq4eN%5fPxH<S^6)eK8858`nb=kRT7
z_x$z2VF+fYRxhT$TfFxdbrg@Cs089`54Zb)ZZc=O+5W0M(wux4kj*8&e=lI;cOt-g
zQAoA+yv>U%X5)v;o3T^{>bz@`YM(1P;~m5B<6PlfGm(uZSix9G1mhh=0QdxE$z!CY
z<XbH}*|BWl6K&iBDY5S#6BEiy%ja}=4Bj^7<YRsjH;l=Bcm{@D)zPh2Fpu{r?wb>1
z05Mj|0d%}ti>pb3dA1mel$eU1d#2~OzCIE$`38n>UmblnjZ8&ODlUyw-e`+R=8Uo@
zCZtIkV>s7ST8g9ExFfbA;|lZj_%FhGa9E<1&10^30iC0vWS~&Pai(1wC~&dOuG?yJ
zeYw0lUK7`bjiYpMR8DL0vHKn(<nmRUAGKXg+E7WxsVR&8ns^MYcJyX5N?(M?ZO*M~
zw>d!?HJXmyUd`-ZA->&0%Wb&NTjEyQu<yQuCw=@kWEV5FdzsMgstLso*qAQFq(fOD
z93$@j(d_h~qeo<6tH`aLjfMFi{LEbo=ohA@y>)PwE&^xSqIYDj1*&Q-B}$#C=-Feu
zBIU0~-!Aae*v>nr*KuvUzgKQkhKm<5eARkw@*>F4hlR4qi^N{lEf&vCjA;#nV=F>~
zh2;AqW<a5>t7F=U)~#B0t?!x)WN>kRC*rG>>Q<ay05!jDOM6`0BBYNcaR2V<8{AzD
z7|-_w)=jU?vQbO5ox^N^jtAJrECymJ);EIGcZ8R8uf*$WnJSVzTm;MQ4u-U$7YBzp
zz`jdNt2#})3nU>3iycZjrnL$B`G_#V$Ow}X8S8V*8I5-DM)J692%l~fTMGh?L=!&e
z2N4XO={a2bL}!(I9OlCbcW=Zi<z+@=UGIo?E3!Zp+VTZyCY08Y?0?3*jcgj%{?S}z
zxO*V&6i-kq&bL*3?V$mNtk}Uqz)<QVi2#Ywi^+jtx9<XN3l9HTkmV|cfEyMbcJ{A;
z`x?Jp+&Qj&;va}HAQO+oh^#Y*?!4T)HCd(0YM};+yL|*a37Y^jo1A@>94}iW&`^D_
z&dbN9^*uhL_{NJuvwD93$c19VGVi@)SkZ$)V3?wXoJs`rw=Wo+#0z=NWJM=uN*8^6
zKN0`#sew}0-kdVx)Td()qb3tPn0}OL!7MA@wK-t2phrz>JAv_iiu%TLB>ruTTw{CJ
z2u+xgKQnsy_7>!GO+1J_YUm|e+?7ziwhbi3RdnxKll<oFQXU$b8OY^$7V_$5^`W?E
z%+kkRz3lqdaXM5|eUafkaCwb0Dgf%<xBh&>_)>ukFWqlm-W_Zv?}<t0NH*4DSO?mo
zGS?DzE0Wf6KyIk)8qL1k1daTHvZwN(HX-6+?_%)lLDGAp>f|~_6K`Vd*nd49AVlB`
zJTw2!2Hu^R4~f}cFTCE*nOby4{=NosfXKK2a{V=WW4FC+7%=jCXs|=~JwMeP!^Rje
zZT)@M_$~9?7vzVX`1z4E=0VueFx$*&E<2F?Tqy&&c1iyGcYIyF?@d2_P()Kk97tSz
z=NCh<&Ra~#*NcqN*~~4~P%*}(S)kq%Yk%@Ha7Y*vq33v20QMi~%UK637jbW`l3)>l
zEnG~`$0sKAHdyAqR@xcNGSeOG7_li``1pO&xq3H#^z5kTj%YGrCUh>Po?=QqSg<>S
zU8V8Cvpe2o*`^oI({VY7mPB9(l@)>KtMJ6*B^<Iz!+&vVt{AD<#m|~8+d`mV`}a~0
ziXWC>WZrXq`VW-}H4a7Mje*BYf%jeI>DnS{D&J4k4nBx!Dbx72_^HZrfWQ3S7vOGE
zx`;B~A3GB-0r1$U*_eTRMlS`gNZEV$=MQ~+g4IMG)sk#p+R3rt#KFW-&>;)BxJtzH
zYEW{ShuCMt7sp<^nqpplY9a0|vm+=;kvw_*L3vv6VXmk@*pd%2FV>)cX#yDV6X=|c
zM#<7w7?r|@HF?650$$_p>iZu*%^@;p6D=$MttaMu+&M0(G3deQHc5){sSs}^fL3nW
z{lGR;Ca=~L?97OwGy$-02C9uQuCE5GT~to)8@L!n#`kRT;Vv*Z#V?hZc2VG5cdZ0c
zg4)8Y++pn`Ah=3<@f>iZO?{$8ovllGwRZP1n**BDh$>2EO+R>j<gfyFio>>%80@gh
zzV}{iJnl(AL!@|MV4%_M$N;X!Q-dj<>g=HrMg3ucd$(c}ufI5#s?ylWv-{@K^|Nb|
zi5){GQ?2NzX^G8|X$M_ilc{FS(P>fV*2=a`2j#k#W(95H;T~sV?2dd1V>{>L%EXkk
zySc<7G!Kobqp7FR)m<W?TQ_RyEEHrh1Uo||HhM9a4Se72LNnc^-dZB_hG$vaU~g+h
z753!rWgVU$X^T!YnD#{xuCZtNj>57T>YEeQEOe<HVDEGyq6*Kv0%(D(Hs6adbKlT8
z8(7Cr!*)oK4~xyuX?&AB^Z6Ev!1NG`@0(x^<(XMyZ3z2muYjwj0Q{)+U(!g#hHIXU
z?0Y2G1oS@Leb1F(B8y<S($+#^;KL_IOG3|%I1}!<$dGqhNe;m-^*??Q6Otj1QHzBU
zkiPoomkW(Ty=27t{!LP(wpwtUOBFTAUI~A-U|disxjdav1-$`ufc9<ToAgn^^5r!C
zAOJ@+cA{?T^<ufxJO`M|IA8qEL^}NvE^Y(UxtL3o(O{N>RjebCj!(oB0$+3~S0E8`
zqKsF}P_`1SG53U94^EGvh|^M&FbGUis*E1PnTnRo8X#`nJL9!wkfD+PC>=#&b@^I-
zFcA%6AI3KbPAfJk_CmV}xe5NM37CkQ4)o{)NPepP*9Yt({R{`CkN$fo6O!M)I*~a@
zZ0vJ#rzNvBM<f3_-NPqjzl5HbNMT2!P>CUCx*)8#9<Sbfdwo!P3~%2AuI5j|c7;2i
zQrEeJl_3b%mA{tvoF9u1bml%niagiec>{aJS*2Z-pGPOGjZGkD;-c_F1o+f<$Y!`i
zy_r5gwTk-14DLl0tBu?{IX9-<t*19!eY8ZXU3f7B=+79TP7-Nn%Dg3eM;O{^);^PS
zt)mh>t1pjalR{zQOZR;D-3dB(TY6BZYI9U!p(>x`X#8AF4iY}i^opIi%ts@YE$6ev
zDO#7(Q^00sXNY3W%^ka_N_jPjBH#uqscl%jSUQ}e$oRKG{CHz~yn`uE9PzEJaT^cb
zmDs-iYAr@%Ifq}=BuQd86>#{L#Df^Au*tSlTzUrggy=L;3gATZ`+QTy4Cg7aE>Ya-
z%HSAhVolLj$V#|}rmc6imLoe7^dngci=tYNhH&FEtj|x4BK#k}E|aLB^k0pIA{Bi8
z`Gt*@5;;i5Ju_xaGjdqmd_v4RNILCK8dxlvGYZ>v2-63-`g+2{cTmfKB$35k4f1)g
zkY84*hrJnQOV8)ue2OH9S+uucd<P0YqRc>dBB5^mldfwYV!aTALZ{;)o4Dm`NS{&e
z^fLHS^At~he-b7R17CilhF9&yqoi>zcoShhQNyo>vMXT7t%Enx{&0pu5Wns;Ba(6l
z%=qYrrH9j}e|6nC;V^P-kw|!|DeBJH)`>J$HvI<=gih$V&A#(BZEUs$^rXxdt1$2M
z#Gz+<`fz&a*k{7k-KXIEs4(04?#4G5mY9B~KyDDENehWH5C$s)b1i3tg{^hg<>zbl
zIqv#5OQod9ynDpx=F3L|IP(yhdS-x$^_!RWdlFoaL!BLg_jCt5+lP}soE^h{igk7)
z8NL$VFUe-t1hJuBSZJ7mkb9{&S5NP<gf3yLH^c?<a|2*oClhRZo|_UYdU8oE&Byt|
zNcJ)IS3Jlth;BU5UHL4~{?g#C5OpxBMMI3PUZg!CuUx&UA+p+lllaWH5B*@*=u-5A
zVes)4_9}UUoIr0%1DOlQZ0G1?P5}*!y7>g3x>RZO>G2uC659)HO#Z?=wRvC_-$BGh
z>4bf+@=;hSPruo}BTZxeY7}#>gYAcVX~8@$1e^gJmj!Er-Q=WL+UeMkCo33dUD+`K
zq_vL|Vj*aC2>!pq(SPF7E2&;~DU4Jvlk>)#f6fvdDgtEKP(FEUjV<Ff-^2-^fA-+{
zkO<DlY+LZjzL`vt?tS3$<FW*7+oc5LO@N&w50noE8XhL($T8aE{>E(B<TIHm?%6fZ
z9}MOXz87|}d*;^&zB0dtse{D&ZyU763D>=wSb?<I8SfMnEA4E4xpN}m^{tlRDX$5m
z)_HV-HG**WLNP3ay$0FLCo)GT>RxuKb6j=7o+r!P8ZTWTM-!68?fn=k0f+0pEL&}u
zAH*y=ny}Axr1)bsN6x79;>pF&b@TKwRh<`~a48dRe1a)_Q!9zWGzX=Wn^5WNOIp|N
zGbNnpUxw|1>vkLAu!?(QlrI%1_sLp74yDdoC9mK7eO&q_aeYjCNOH<eAG-uG*X98+
zBE*&lcKV4ha)7Y5imet(2^6G}%XAa2$`~KcGb8MnNmToZC5pYGIm$38v$^&Qa-shu
z1CY9t!$`3{C6uE(PkC8kS>b==5J)VV;RHkoO^2ZYi$7xk<;-;c>h9Lg1x(>R>$hX#
z+mqo4&U<Xwm{M4Q+_^9#7iae*e9a=a(nRwm^X48BVY~eG!)vhe6ATPXkq>DE(3Okw
zL_c#l%)L3`FFpoP1_ePv`2?eeNuZP@b};Re)Q`XmsC~2<wSMmZWetks8nm<&D+Jrc
zFQ4gKhvcv<!&*gV$+>&xwJ{+ZCmZZe1UH@*w7$Z*ChlJ-q-H0GP>Z(0D;B_9Q|^9k
zy(2Zr6^mXa37}WE2d4Yj!FXi8?J41V*mf<uAzn91f?quD&el?gb!1+)#uXQ++vnN>
zT}w!XroY@}E<e?O1nmhsaCDQ;PFs@<*L+AbZj1<pQ2lhgnbTpqwa4lklfRAx)a_-L
zPd_`Z3@$P@<-+2v!$`y@{|zA0oJmVw21s<S?HedW+_0-7mC%CqL2=L5hHMn_S+$2?
zK^2oC);4j8(x=~Ge`noShqIZmjaTN*)JYRNmJLjsg1g($+MqPSX4FN^M+2Hf;{m|D
zyuqOy7x3MHuqFr3_+OoBIbAl*ag6j_UVLhn6hUieSyxf^jh%OHX>$-T+=JLKwCt}Y
z*BWTq8#%)NIt)_OGy&ELu?#gn?9b@U2PX;2`=3&;J^AA*{wKu4Z%`Kyb9C8C8*YIg
z2AXC5>(tnx#%BASbeS#$WCy@~{RT1E>ilHwa~giIyTo_Gi$=$e9^T?9l@W28GqMZ1
z#=lL9d`0n72S$x@k?nSH#mfvYGL3=fc3N(c7tF4oEq=eluLbeYN@&UNxiF*yT%FJB
zt{-_?<!CD%dX;pQvknFArHN8w_%(JaHn}IA6GmXjSa#zwQ$~KLq#9T5!{QHB55;Ae
zrwHoxvdmc?e}ZKu{E)z40LDbznx*#dba`<{f@kT;g+fGyb7ILG%rFFM?fyqR!F-{)
z=UM9yDI1EJ31CMy-%WPN!cu%xCNv~<znU9#*dfoN2!2~NVQkjG%ZqfWem(uNmbYr_
z?U2jF%er@7VrX2n(wi%aqvr`8ozAmqPs03K9M-yUKV4suJZ%LA0-EE^bWkfg(Gp0j
z5F``qM6=0LWI#N<Vly&yH{EX)%BhBsX=J%Y^pRSryxX|<KFqwRM;YK=c2I!|5gv=F
zaUc#2)WTXHfkpab!*a39n#NCHAG*0i!r+VYbt1OJZ5OEL8t&Z@HxBM>6SNvk&D{y{
zg0a#4<x--LdGcvvoRqew>4Hen_dYL@_vfdA?P^*3c;_wN;`@c}o7NaEqQ^ssL2yIP
zZ@0odGdNWs#csbm_PO120%=I~LZN|+GpC;2yQHo-_u@qhnxt&gg%#5!#oy<p;0%Uj
zWS^GUCH5#0V#LEG>VF@oacKf?=C&c@yAvJck<$Nlo;oN4#zM+FXjwwUG^kj(CypM6
zovQG)R?$0PW&hrkl$Fwl=7L(9py=98&nCqGIf~kMH(!&TYq7)-Fv6O=sVj(|T(=^#
z$M3f_-JF3sB1`*Vz%)@YMee09t&QRl|2cA=Pf~g2o~eZEHB<wO{R?WXK%;b;_9)p6
zXfwlL0WNwk`q91k?2LHCRj_@npS;&S=VzOA4Ix8JX<#`}@HL=-X<`4_JvSJA8+E*u
zjda|2f%k`2+m?h)J6wrJ?uuni#^`MV1KP_b&Ge-(IeB?&lns3fL;n^q%H$$l1R=Yx
zfXM1(TcTtH=gha2iTa;!EOLJD+g~M@@Xq7LxDt8x)qXrpsU7Y7IB)o)_FInt@i8pR
z72Htiz0rjfgD=mLFT#6c4|3EFs`=4<BDB5p-L04Dd2E&^+t62lw(9Yo{ec3;*+O;Y
zk)di7ft9#ZeefmDzsfp{841#5^Bz6rv5Q2A&q91B|7{rl&Cmp-wby+R7>yY|-ar0X
z%AAH|xqi5L5ke@D)AW9s)BMggqRc2mC&w&A@PkPzUlO?9tQbH*kJ?Mre;7(X+S(04
zQyMMK?k7cXoIw<wh1Xrk$$Xk<ncdKlBL5RYYrFcp`kP~N%QPe?X0`h)5+0w1z&BOy
z+9(@Sxcje}vxy$7&ut!-AZ^(p3l>xrlxs1vNiZN0z@qJRF26iT+thIzWeZhUBYuW`
z9#87Rn_J$B+eUB~PY7eaR8v7+rZ(koSg!69UkySnoyj+j<{Cx1Yy7ChPtUm*V+F)u
zL}w2g7_li#Z9kMkn2#?rJ(BgpS01kx!7?}kz#x1f_|mdUo<ti}yR7v?68d1oS?Mc;
zZRct)OCOOznB6HfY4Ir3KX(|+EJ}tZ<RiZTpNEQ&ggPD=+~3~JRyx;TY!Rh98g}lM
zSh!&tF^R%kQc4Shi{TeivE^4YT=MxcZuxvnC^M{&{3ldjL5hv+4{&t%(dne8pf5Up
zm*#zQq3h}iBE+mn0gmpEhx1ROe<j6Fl_Z6Vk@K@U`$r+KaFGy}0~@#2sK1{I863Ol
z4sk*OP}cPay9<gxo&=uk_<;)Q`B1$*b(#-JBu#|iqRD)|37wl>Wd#I@4Rc3R^{j5l
zr6Kdm-5T-YP|s#H+vpx=;<mvP#|^AjUicI)!>$BMc^?}%&*HI+SsbhO*tXBmEPM~F
z8Ik*wu$z=EO6kukrkq#)1>`P%og6sJXiEnBORoWkBz<yJKA6ewCiPt=HPa=aS}sx^
zuYZ&l&yvS_04aqicvY9s^XdyG;$!qEsLR$GsXHoVNRVS1$0_8nJV)rgZ?BD9Y<pZn
zDrQKjzxDK<val?iPY#f9=~VakH^ZLCCY)vGBUg#*6#i-;e<T2jx_B9l<m>$W%oHS$
z`u7ps4U*7YP>~~5PBE#29M9SP!gnq!<8mto@&OzuA!+fgiWTyF667_Tk&8bYU$14>
z*t4{qsT&9PVw$y90=c(0{3&Kd&m21=rg6=BD4m?_EfsWT{fsiy_-Q>vFvbTO!Sj8&
zeTmyZ#68z8!16+hAkjE9;K9Y?E$J_4T}w>vZ)#efVwKWryYAPL^LH!n!#cLIXj6J{
z5K!Bv8P5sT*)XEt`_Jm^pWkC-DB;0VMQ9%V<5)=9Q!j5#So4!2o0kJ=&$B|13a+ZT
z7H9jzqYAWvn#_l7IqJlbLW=TbVS;gqunuWCynQm$!BEyFfDlh?{~Y*}f#I5EKMKzP
zNDmw|Ul%W2^O}*vhJ%?osoHR^c4@%~TIi@@+EY-y5Z5RD9CngNTjPUzZceU3NYCuq
zZjBP_c>M=EP&pBCvp3#{qQ2`ZGre=rRT&ch6D%7}uR;n$ObV&60*Ew6oE$;uadaTI
z|BE8i1f>O+W<hFI;ZaKlryntg5b=#*9GQQLxiO@TJnOAqrrIT)Uq{>}Y{z_X{rNXW
z9}||p*aI{x`PVlO)hZ9y43%VGiBnq_znJ#H6q=oqv{qCqfCxiTM{M|Vn%OWUo~*!8
z;j^<v<QZPH1Q^Gy&kXQ!{{7jdvJ!?kOQI>$?`MQ{tBq!S^7A<yQ$>xn1)Tr41OBf%
zNew|uQhW2&_FviXuP;)K@~LNW4u61pRhGA}75rc&fBYj=9+`xi$sr7mgSgK8^~GWF
z1f$vCB(7WPRgkpzPRNG+{5up&nNyN1FSfmxX*~H_C5hrGBEj(HU#0#ZSNq?79rFo)
z0(j^(N&Y+mH>vSaI|8RpeT}gkq3`6BRdC$j8z=LMTb<{#`7XYywH_{0?90(v7fRXI
zczHV0LEx|NpGrwsrFG9NufIDQelEkB`sTl}r&sn!R!AaTW8!7BBra<|bpAPm|NUbH
z90gQrTsqYF&&-m_fgw5g6#Ir*DOe5cFxA0l`yDxw_3>R00x<$Eb*PcS!{0B5Y39H>
zTbP6p%-;buEVs7`g=hZ$HVPDoyA`|VX>HGlkRjtrFGv1=lp2bV!(2LzH2*}*GGtMK
zl8f)vj)tccgOzv$bOCH{>zbqaZiqN9GTlAH!SVY&C?F6XB^;kod8DLg+ik!hKr&n8
z$;$eJ)G1rPyZG08b(I9KrDhEiClymXstPT^aUCe5RYQE-rstQu(jqgRs}2^O%bfXn
z1fC%JU)%k7E4%ozF-f#Cr+TEshz#jL#YgzZgZfVdhwQ5aKY27&0l^=iD<<Nc6ldL&
zV>;{z42<Hw!6GpmNItx05_h|DcERx6UvJWm9X>NAgM#!_4%;vM?@lyK4b_q!kB}I9
zvL;>^CPhrs5HDI%WIg;4FSmmSF(NKJxEXkon**+5WVTjGBX9K{D?WNx0H37*TH#~X
z$6vI4K<ew|NwaC^GCk7+BEUqLJhg&2!ex45c!_qowZ2?ZN+DGReq@3pHSB+W*o(-V
z-R}Jni<Y6K6^eh}s#GWbls<mud;Gr=ln|i^9|zJkN4s<qHj*MhlLpUJzaZ=|6@eQl
zy!Fv?zzDQI3JmKh2C6*EEeDGfflB!ONzwDyf1NHPN=(*<^-q>Y{)ShoNg8BGRsB~5
zx-_s}ll#J|O=SM!XzL+RB>+95WDs{NViH9+3yRacl43`4Q3gCY)%6~U5h6tfRdHPB
zlEU~%$IAct*{72j6_2{6_|_`@{q|lB!GBX&TrrscGaT@T5qJ+ZDwJJ}{e2=)+|&)`
zOTAgWqu}S6Z82n_CEl{H`*!iy{!Ml*^z<M{0<Zlz%d~sYIm|KE%P!0Mt{`T{#<LHZ
z@+1j)-WP%+q_r=gAzxwrW*_#9U}W26Q<SJ;ZDlcp)#&%<O>&fdU^;;c?f>oHr~}&h
zd&2@IPcHrwo%mznC1S)8S&9E%6V<19>@G;s`d#39emwp?q55A=0!|hM%-AN7a8O{^
zBsD5_wose5VO`)E7}(fXB}Li`(ZskUQ<K=rqD_T@q?U&Z{?P(xAwl2=m5|3}b$$V%
zB^5Tx?@pV_QRmSlNnw?r^qs$h(o#N0sWCTn{P!C;YxD$vVE#YeKpe~0pjSly1VIyc
z4OF4L&-&k05nL3h50nb^EWVxjcQPK`vHTshWu)|L1E}yTG75n77(=Gux1UtNYdW%W
zVH9cPP??olg%7=HOcXx6H8TA^O&*FXjeD+S&b8Wh_xB?ZqTy0a4ZxBX+5V`B3=^~-
z;=)^>1H$p`!-LFJkL^{}YA;XxBK4KWowZ@7=%4__D|J2lD&UI*g>;P6mF{TH4AQaq
z5a;T9SW%Xx10kdimrDNI?hG9*GFmRmd(UY}WCD2q*Ax5y^M;Y^?SK#BMLoW#!iLOs
z34mqtT;z=S@%Ny`eE)W;>p9WyBQIp_C+#>^8^5yFhX{=tUL>uJ!S6(#2ii#DyKZm`
zQcP2hp9EG}N$mtH6t$~x3g&A>*IJ;*oIxNQnA#Lk4<Wb7+zQ?fi<AnkV6CJR7F^oW
zW6jl`^x$pmcOe0+Fo4nvhAwP4its-kQZrIWGLYdc`QI4|0y=z3_<;U*F9MMkbN}qC
zU`gaV!$j4u3H{3WJ4d3R57236u3mLscfT}tk&+=Yfa*EttDcc;&Ft#uMd|SL<{d)g
zw&R{x3~Efyoc-(RQFA@`5Rs)-qP+TZx9t=ixZuK3jskH^*WO=Hwl)FB$WpNjRLU@h
zf^e^H!s*i><@fHSYH<3X(Dt;oMFg#3w0f>}^HI=CSqK4APnPi`7X#<$xPR}>BM$8L
z-DVQae`2-KHOO%M?nn*Ef1(=!X(}1UaqleGXHWCqte7~bUd<{t5CE{#toEwhgCC^8
za~D1OO9`Js;*fE1gYggDX4=o<^%H-Y9<)gwI^GO>H-T>%@B_>mdKDDqGj?HC>OJe0
zU&^M%2OQKc1%;Jf2?~q8tupVgoZQ~^`5oDrNX3A=$PYo})^orZUPBY6H6F(U=MDv!
zMOe*Ka(`=H-3kq!Dvbr7ACz-FnVe6_PIfLqY?TM}@m*q~edBq&F|shoR{RVSXl?&W
zCW^+jSGQpmJpf{%ct;ch-V;&y1prk*WA<6*utT$;>J|nyqf#!X%VSo`2lq48TCS2e
z>VCA;obiH{mfxK%fIkWVQ@>M+;kO^d5)Kp?zW`xOUSDP=AQLm*Tyj#)(H73oF11gO
zgBtPMv*V|Kc>}F_{iIWWlQn=Sac&>%AF3Gu^#o5B^$$y`gkABBIFPSXeJevc@X;#P
zi)CHEBmiv0pLELEBlvE@%}j-oW{1kvd13Gze#oqwb5|`3LHG-=2o?(c%xuNYD4y(1
zLbSj4OnoTZP{!A>7DEKdA9wfIv9rlo*{EBD3n5Zh@1iJ4y1bu<c`k12-E#e71o!+b
zoQFq3g{S^b77bExQ;bmBr*8j?Y&@1lF?*BTv`xfs9?lMxI>?w1PKW<&ifV#gDr0D+
zseT$if}7BMF6r^Z+T%Wj3`|+Dyv_=Z(Mut35fiZPdfFT>Dwt(qD_48r+c8Bs)aRvM
zr6o^%Y{qbkexH}1`gX52reXe)Ofb%=s$PRSfDMBm$iv8(@dB0d8lASS^vvKAa+D9R
zA*fOSJolUVkG@{-tNRAqglwZ8YcHja`7Hm9Z>0iGfjFxNrz0peasm_=e*#fI98iaE
zz<#*2xyPeIR~rcuZnOamLkK2Zhd;<Hm7dG5{jv=e+)kv>8c`h1qB}iVejU^?==7x`
z#!dx-7@+O1O)|<r{PJDdmntoh=H0qrW^Op|6+Aw+w@hnq<gKjt>79<kV~4Z7*|9uE
zE<h}PhX*^{H^KpByAJZB8RSg3&7qt^)3zLkx*Qr|`$=0R;*WR@7c5W8Ns>t@C-J*$
zc=<X<13yFA3)FAjNQ|DHU^!FV|2f;TF5HI=Qz0ws=7Dp$b)cF^n0wW^if+2!az9e>
ztG8E#$YIv<Niw)`XZMT}-R1h3IsOP#(WJIsv;x;p{WE=yxeXDd`3zz2oqr#7#`V7+
zHS~VbEk}ZdYzahMmXn0vOhbxQyoqAO_;kFkKG?a17ycMN-x|==C@&OvoJmMUPxch|
z7#^$Uw;uYG6~ykkn=s~V@!qci;yC*<L_E)r3RPW@uPU`C+j;(uczL4bn@gj5bs;P)
zpNMhRAFYQ)Zn@#Omg`x1douFF$~7bWm(Dp%HFta^d}fAK4Ol>o6OinVWihfx@apLT
zkjxdOgVuaC>GlS&n0e)&woIF3{lyt@KXMe0VXz>1k|l~O=h)kP7A1w!`e<1uH4YL>
z>)afUEUY20PqQ0Hk!N@Wjkt_JB4FhTk6~%8PPf^}l%L7p@q%&2<V1)LQ^IXwzB71v
z1zP6luz67)eA&!98UP=|ywj<=;x4LkVtqPt5>;Pf94q`BCl)K{Xz9CT?~HX*v-WvS
z#vLv%vDy@&tLT>cy{F{|CZFW~t{i@F?hXmmEM6A=H}XQ+yq5Au+1y}>zl|Ftd0zk4
zatbFAvs*V*s(%JbysxD&5L0^{<gsq!A9nq2J{eM_9RT|r(IEY+%EWLZCpjozNMUCf
z)b4z3EIr0aI@ubpg`YW&D?CKZ#eb;Qif2q8$=E6Hfd%UNcN2SDn$Oq<2GFtRRg;PY
zvS$}$DQxNJp;CIui4k|-AYtl$>GOK-$7SX#`VU+Js9U&ezN|2n6Gi)huS(D2tq88O
zQ0&4yL#&=hv40qA<Ii3(50ov@48XKaU6lp%wFsFyDz8&w2x<uvy>>EuLiCoYNtB{5
zOXG=V>N5SF0eq<RWim7dTj9EfkAP6mG_`V2Jl%&?vmI+v-M3@`Oeh&?Pw6j--Uk^=
z^midrs+^L<5Io`7s`7wA_S02*PKf%i02iAr*8&z&T>u61*|~H8sjD)jxWYKuDQyr$
zuNR#5v8CO9-(^NDr#*N_@;`2eM*a0rzV|=050PxM|3?1rN8UgUbwDHamuBU^FpX3}
zNSdAQ9s{EPeoX)iK55Q8Grr04swJl53Gq!9+nR0Nh<r4LfJqKY8=6~>HS%e=AA-z|
ziZhwr!{Jq!@>6HMZ^0+hAtVlg8L4z9XL{e94B5<nyq0vo)aLug?1@0UR%Qm!ud#72
zr|~Iw`9DMei?XMVaV+PW=%GyFSqh+)=!)spikIEUb^aYgX$U=R`a!|?T@mEHCeSlB
zf4)`@lLS9VJ>DN7j+^2|*U!c3p!BOqLu)|r(&_GJIKV^{LmGtDaNwtmc7!{JVVTHa
zWr_rs^a~aoVp5P4W3W_F0xN0SUWfb6c#NV5iZ{Bur?^y55vLq@vm~m`dp@Yd?IUM@
z{IY%mbtfsBlOHjM_<~+IaM+M74cG-j9?|uq0SG+H5d`%5+7>2!3SeZJn)}Qq;>AR3
zyy#CDxwShRV&@KFV{xKxb<FQX<Kr|<XZFvFq_#WG<Y<hK@G#6cCSyz!I$W)WKHY_U
zc!@IP7~_=&R$}P{E$dQQ5P&3dKsO0b<pS1SQ2XN%FuTfb9Kzs|-Dadh#5w3vv=EUb
zMhXglWI1Q_5%dTAtmV?g^HKqHSk2<msrIxxEI`RpZ@w$WiP@s&vX4}lE{SUzs=hm!
zz1$`?UCUQj)*RxPjIFRpRbCi;<iVo~q1S@$quY5XH%Y?(ZE8Aw38v*A^9A9Mzc0H~
zFTRlMJ>$KR@Na463v?G2Uw%#0W#Bt92wKO@)9atRLqeF~GEVxU>$2MR6Pgohca5OH
zRM17h`|l$wocW61abc4SdbK!Qt|fvOMI7#ErG~SsAP12{L;h*7jM|w{-E+G`g<V%d
zA3cb3U(eZL8r@I%U2?^d=oh=J-8_tT#@-d}j7R4^6h{)s*$!4V+(upYKJf4yd4$dr
zY@8h|&OFdLT4TzlPj|&5WT4W$gvwa>`dA*w0DtB|$VFeXdQ(Mq2qe#-^!9w!cQ<`q
zRiR$GVP}gi5e@OT7Ky_u9zv;*ENVMmAJ@{)Yyj#}zRW}$i^HdH8pdHs>BjHu#+}%N
zEA2YGJ)-a;o_y@rQjs45Z75Z3cg1(qngj;h0XK7jVdzQZopN!4<n9r4YEw982-Ul#
zh<1n-?Zw-Z-S_UihOSK!h%`4kws^?SD7S9qRl|9{1=Zh;*_Sku7%a2%Q1bNEPJA8t
z=9<+7kbDz(dtIJH>@p8!A^i`K{IMJI#C~dH0{|mwYk-w`)RxDjt&Md_djcONB-T<a
z%>^kupu;AR?x~S|F+PkKU3J^WP5L4@?!=61Dc8_!<=u91*EsucOos})HRR6<`+r~~
z4Xnt{75}6v^QSeI1Mx>~$G2HI5A3SwjI%tC5=vrx?%oFEfT;X<^bs{xCKxaX2ma2-
za@fU-Q=F8S-$;Viq~`<e2}XG#>}Tm07-ZNRf{_fD5)q<`dn?wrOI`)9r5G`!Ko-``
zzOPg8Q6M|UQx@$p?-7#EPRWU&({)MqfMO-NGcQnliZPUU*O@kJOcH>GTYSpE`{?|V
zd<eU1kwsJFLV?v#`%ZDSot%el2SObtD<dBV?kOn^s?keQ$ChC}tZ8lNPlyYmWbY?J
z$ej_YxsfIAx@yFB;q3&V;Frp|@IsRO?i-eU{MMg-5Hm=6dCW-EohqmNM8t_9Ku*Hu
z2D-nMdWwGxd&1O%ElaOj7l_E&L5q|;WizarMh-l6ixhrGPoNb%`R8bV##K1ZstO-^
z%!`iRLE#5hO^ek13j|h9<Ul~73E&DMFld0I_wv>mMbrWEW;|P`^;?Hh(Vm=hEWdtd
zWS=!Q5+X9X^W$9BfzIK$pLbFEoHc)t2Ndoi2&WUy4~ky<3li}`79b(|7t~%Z4F4A?
zqCEOLDUb5L20agfxaV{iFj8CQPg|e+eI1Oq@rA@JqNZ|1GyoLsPlMQ5i2ljU`x?Uq
z5b0I5vRVDT@tutNVr^iuhH>X(*=Fq1!&tb0dkA@;xR}oz=814Iy#{>_b`3rUnA{@d
zW0R7h$@hJI_vJF^EVzl3)6e4LAHZr?dDb6@cpqqH&rDN(xSzoa=&0SNHN9?_*-XxK
zq$CHx_f&-WgOmIl5QE1fB}Oo$bd!Xuz-?Q==MPNzd0dO##v0ySVa&)Fc^RPTl&V!0
zwZkvwaR5g3&AY;!$rz}Ua;j}?=wt$N!y!Hn4NoV)#oZXJwTH-xH;I1g_uNNj$qZI`
zx(}bnw?@9N#rYPYDSC~U)sIt)bjqwhe`dm{|5R|EU>25a*Z0WDL23&E(+I2OnhQBF
zk?}v+!wA=*!YH$)l3;9EcJDMjV=`p*RoD=2`}9jZ!LE)W*$aF3BEI8q;;o*UIOtK=
z6V9od(@MWYh!(qCmkJ|&U(-_)L44B8CTX%-YW25-W(MNFG0}Wb5IUBlSy;F5(V}Ab
z8R&AIdU%z7AG&~-RRw%%jbF`cVg+q_cUOK!&3RH8LR)^{ObL6On`UAS@sUU^G-nIf
z%_Kw)58~b-&Zlq`CZJBru#-c*_dVax|K7)~uf|OoNW`u9rZ2#H>UAWa6_UIe$#)a|
z(Mp&0d>1yR7x!J}+Trx4g}4B?f2Rq{r3luxAAJp!)t))rA3aorz0K>}aA?NbDL7qn
zeYWc%e_wJi5b_0$x`p5Ykm$Hzn<`iX8mT{FIDC~yCxlxqi?ZU#OP_5KWR$dTtdu^a
zd3??!KF`YYcv<K8k~y2c^3QeguZpPlOm$wojLS<_rjTg5gXJ$LlHEbtF62Px%j0)m
z<toe$HXhzJ%9Of4e93v*Wf4<XU{s3(%be4MqnV&htA+<p8$1xkrvr=5Obi8QH_1p;
zeB0KoFm&f}<X!2hTim2}@kd~gH6{=Ky&$hb0nS{E2?c`W-+u`tuq!=O2W&?M39Njs
zNu5qQ-cD?Gq=?Yw^i0WhniB7f#+){~MbE~BrdWAePc;PLPu@K}6>f9loEyUFdr^1e
z&!??Rx!!?oy4%9)W{uB>!$v)l<E~%%*_?SkZAQv?uv(If({TZo)Vd<ps!RLx9DWw1
z!gacNBNk^Yq{A)ma*F}qJtTPmb)|69efC|H_8MLYHL^&VuJN%(TP~9EjXgE`CUxLE
z)JhsUc=ef?s_)OK-DkEVXkNW%O1TpBT#H`NT3drzc@MaIIST`Y_1W~|y26qcV?QHC
z&s#&Di(HNHxp6P^iFu=IV<FGat*q|>_SEFsQFp7U4a``)_wwKArjeh!-X4+8I=*`M
zge5G8u=KVo=O)!sJX8^tDcw*>RS3l{>_TsOY2Y=RPEtkVT|MuFOEFe-H@lNFdTftK
zzNpinmlvZkf32pK@gu4cqvyxWxV;w)15$05vpjrok-f(3Th?1a+D-MXeRtnwo;<6c
zN;ajYRVvpJEwkI#q889OOzXvfxWv977gY~Nzp-!c{ywf2tX@HV>G93wlWIBQh{q8w
z>=)3w>;8*BhG(y-V?_l6olQHZXG>M!Va@iB%4~dViKI+yK!57#d6ZgSnb;#HJUG*f
zu1ov6#f9<tX?SL>$}Hh{g?BTd1FO5R`B6d#>ir6s#2d`wwr#W3VoN6DL4@8y;vC7x
z)NJ+K1dE3bIyYtXzon@ioQODK46kvHmj}Hxl6_v<X^g+w>iGuKdW;|S4Y%H0Bw}iE
zr<7XAZA$vP{d6ny#<0rsd(3O%U^5XZ5<ydKD3T_@ZIFF#3Dz5?W7HRv^c6#vZkW>%
zdL}I1mzdZRtbFpFU}VACfu@;s%lbH<3WBg@Nl$gv{`9C(jC*I@^=pw48(&s_Z2fga
z|IdjXl(HWr@K%^SL0UxkU;jjCiDEuf>2=TFWpQzse7Q>(t${?CugksoQVXM9kMGWK
z=FCAREB0Js-_wVewC17EvMHUKmd$p|$T%rl8F;)rO^vFjn{2IW-2)cYBP*nt3k9yD
z2&UPE9_$iv9vx4D?5z1Yd5{+d{bLo!>oXN<U@M|nyL3;M4}Fxsc(D8$V8GZ1p1>)4
z$j*v^4FXM3fY`=XLn$9I7Oz(2dNQE3)0CNyUt_p}eZB`9KF97?RW>Ui5$L<Mc##=R
z&QB5mc~o6zc8CSV(QF8Y-wL(;!tDTeOM8zd=A)Z%rwFB%p_gN@mM<R9w$YUHH6ucx
zL+^lN_|<ayFX>-qN%(@ab{AGC+lD~K9U~zcRMvjVBWJK_wX6z6tb=fvU|ZA!QB|4y
z{_6XY?k>({Ep*GRw_922`g4q!=>)ZIp5n~r@EcyJeCX(VRf3JMDx5vD(&TzM|2_C9
zmXbCGv$X`9c4vV;;n4hIJ66Z`*jbjizWoDn*cOfy8S6h&<f)9nz%rMl?LpEy{Vbq4
zA2OJ0ia<(2z8qFGiXne!mjkD{7}|!fo&E|$2(Q_fJ9hU)$u#Fw%Y%eB=+-Vde1d@z
zb5~5$wbmXUr>U$riBbfH3+(3V<)_TH4!Sb#akfxepLSYqXls5l`1>%0bA%9kU85eH
zQgRVY_VyYl7`k(l!g?b1qC=Y!V_PZ{cY#8yX~0B{UGm?d@4pVuH?t#qRH2gA|L4dd
z^PtbXp_$t@e29(Z?!T}Z+OK>OPQk&OBTPIMb2)0+Z<fqwrdFG$y;@JpRT7cWf(XWU
zlUBy9JrG1+c2c-=3L^!^=~so`BT0b`VAAsDWGdOPbRGSXT`J}AP_k=5H}hF5D#dn-
zp)VZ$`I)O}p5Adf$Rek-E|b2^y}zSo1f4dIFB7<H-A+x}uXV`p$4lWf*{DeR)!Wy#
zNa=ztJCHcsU8p{!D#oy<tmvU)wPB7q3pBwCodo_KobCnnpL<hCDcK$XaH_%#Q#<sK
z9n#nX#GM9V&`mpK`M4+psN<Nqyv?RH#wF#FZNy0Fp24eQtoay5ohx>`bc(_ZuQz-#
ziyOKng7qRPp9+UwSq%Wc6c;k((7T9SEGm!iJc@zg08!37aEJwT#doNdIMdL80QQCP
zM3Uo<P2Tsv7Z)ytO20giA?D76zEciVr>h=xwTn8^eu-bIq4M@Pn%B30wxSCyCq+u^
z%&^^Z_>sD#_FmizbXMUmRvBKuKS$S&#u@p<fAL*ZGPOCKdt~q6OWUBWR(<)fRl*Bl
z!kDM@Kx@QWjXZY==4JRl&+?ByzIt;F%W`D<3&nrs!a*s5)A$_-9m%%+6BVW8b^GM_
zhh5IL0i~SIcmD^OFead%(ptf9c>+j^SyaIER6v=PEe@h=2HfCe@u{QFm8Gq@A(ZXQ
zTt9Sf&mN2dha=ZOcJIQ24>R0mhOh6sFFWKZKanB`R)8|JU=MJzn0FZ(p(AJ76M+J2
zfk!77^Qv7|9xcFkeG@K4khS21mp4~Uy~li%f~j3)5u`;@@fH+}NF?m3Y|=tsY445=
zoCJ0+i7>`+lQ;_z@}PF%2W*Af7jp-d!pUltga$JkRO%l>`+WuX$!C}=u9i$A{|c17
z3F~83;ao>D=e{tc14)B1Zo!}p;JJ>x57olZ{%DBKXBCRgOL@mJb>=q_GNpl4NY;{!
zO0oHa<E@+YdJaA`y3eW4iOpY5t9MnwNCEAtCa&xUa%Lmvx?YE=1D32p<L5dkYxO!?
zVb=n@@B%sjb!>w&zjo8LxmpSu&>D2m(Cz2Koe`zNwyxj|U#pAe!X#!kQCQAZSw4&G
z#fHfKJaBy8ExIhA^m%@P97k84)-5Oki{Q4IZw2`}B#7&myoRFAm7iO)z^GrDTz?tx
zlgE_IKO6|r!KRBaBP~`>u7rjl;pgc++;wU(z2S1_5?sGeK8uiT5_+<M>Xvx_?ay4C
z*76mi!%S%q5?feW`>lgx3y;TjH4TrPCZ?fYi6L2LezGxb=^aR3Yqa6hGs^}!q-wiP
zwfUvs2))`ffaYA?+TE5%1pI+*v|Pz;wi|~;SW~P?H(lxcoa_ZFLsN@KB|`3_@&Kv9
zdIqbZY(f1FnKSW}PB-t@)c`4iQA21$PF^c1p*1SQI^St_2BrY>C!M-}h;^d@R!KjG
z7neLCaaNJ>DQIE{>Mvh;R%GfW*ZI4g`>&YxJ3peVEKyb>H7q<^6DR-AX*arN8*guE
z=U~))y42z2XjCd>lH;eZ9dig<w$IBG?X2;yu~=V$M(By~+6br`CS3f1wQwdln9Iy9
z0)mprl%L-m)XKLPncghof=@0De>!&^HFm#b7&D0LK4PjZBrN>X@19^>di>(R#2N2B
zJ92UdX4@fu8Zjm#+%~qUrL+o$Jd83#Q$9L`d0l>$OxlK7koByTbG8?hey~&eFiX*n
zck2pC8$P|`ZU(~shOt3t2}m;=7Y9Kp<gF;~^y6lEA;&M<gG?wdzO?+rFL+vgZtU&r
zTl#blwt5VBCDY-i-JM#7$$^=zZsF$VPtP>IGNE#O2pT@wNIX(PiSQi_ARS83Vax5J
zAcAmM^p$E@;RPGh=mN!T9+ATvh1UGAT;bY#DBI;R>EVo#3~|a!{gN-*gFpKnb>A?a
z>&}>(n)`S}`o%qF7*%y8c&sYkatm^W(wC|0wDH)0oCw=k1z@W=u6X$%_MQ+<<+$aO
zBJ@r~663?O8@gXbqt<Bo6*y0)54H}EXa!Abq;+X<KGw6JYG&AA{yZ-}w!v_W)4~D9
zI<nFGCg(6%;7u)vOgkFR(4N5L=wX?T?SZXQl7atKhlW4|p`Rguv_;@tVFoq|{A_mO
zhj8_@tJxTgR~xF2)L9;JDJgGp$SgX(I}}t}#qPQJ|B&_;U{SVP*eDD`DlH&Ni~~pt
z2uMhclqd>_N=Ye5N+ThSeo{(_D3S_TAkrOU5=tu4p>&s&)LAc~?)~lmT>rVwzTA8;
z^TreFS-tLs9%Uj%dG!Rek$-7aM@)i$(~zHd1kFKJRjxHhfq#5I_us6xxb??`@XyQl
zpUMUK==pM~wy#MzSOr7CZJa@1Po>Mtl8!IUzYNAJD2%o9c2;+n0W4R44vFVw4suXy
zzY!;ughWBzCQ%JYmpmrf_3&ogJAtt*>A|-r+ALqUW*b-f4Dq>GSib(b=m}1^bNjX5
z+-V()Mt&vLHkI$cBZ^x1(qk8d`%HL+u##a)sQjmvGvj(0G-=9QkS0eX-n`bImK@sp
zg?pGF7qBn3s-pirWE{wRNntA`T7^)(%shg9+3Xg?)%V<X*t&vE%rb(Dnx*f1yoI`{
zTp_166If(H0+^Yj(%va|R;=AfoQsF%K_YjiH<h84s`nfX)r$^5t$l2o%40NJ@W{Yj
zqk8_OYf-gcR0~7kAF6{i?)aL;dL-*h$R#$ie<lL%Ltg&7W3sjW!dC&)+uWcc$#9V&
z_Na=Tt;i2!-}UR1{CoS^T?c~9)2vbisyKQQ?845w55EMj0CVbH%4n-TyXl0d&EHna
zxZjiX>Dp02!eIDwqq6>qH&*KQOl0U-()|s`-`5ZI72G5kY9I3Jel$_)KwQ2^r+r{j
zBy5otjsIH2G)e31uqvoT({kkj7n$&kA=Q(!&MPz<v5F!T=q$C!;yR;Y*X-2ZwXZJ(
zK8FJwtQV!T!^G`E=V5hG>?bK#182UQj`42q>quto+~S6>sN<K6f(U_-rj}*?610B|
z%`K{m5EG9Oqa<H<SL*M3_+OWCz9%WEbJ--ae<R|Ul+;0tg78pjMe`x;TM`>@{FecV
ztSPf(jL#N=&X>d9&II7(gydYS$O{>!>#jmpQtgj2YSOE(TV|eVB-uGula+`pBMH7W
zt>9H|t>6UJgN~a=RHx7x{9f_N!c=x1Yd;Q|2$NPa)Zd-U`telC0Llz9ZL^>{h%-UL
z-l+Ab3hSaFS|!ZISdY2%Y}?HT5$~U;K-Ytqi!RyN^at}BkVSo@CC}!WcEF<Jn4jr7
zh$4X%KAp^oM-5S$h9>uY`t>Bj_B&@3P+um9FPQuejxl{mKsOkNQm<^TPv98$`iE-7
z0^FrXR{C59zxhgLwY%+olSyeede>dD<V2$x+ADLPph25NQe>~!XC4^$zDcSZLpScu
zLz&Wn?#6IkZ}vr#E(!MRu~$;K{ofZntOq8$igTsno(ZEpW(*z@v5<GW?maqIJ+n(h
zzhq3y8di)?x8qK5jX38n^@JXi`f$l5M$3CJH!d&9c+sN^hRI=_?tL<?zbah1f@u=v
z5Ak1;%&HBl51FQRi}t;B1kx)-X?JzX3^7>{!F@P>i=$l1gzcY|M4h-pKEi=!G-PFv
zYBhA&O7ZsI*@>IQzThv(8n~+hqMLe{=8J-`Ft{iwosT->yV!d<COq8hij@tn5lp+3
zb$0ixKBS*9WWIv)CrFZ(qyFFA2%##@yOf+~6&2*!nj|F;|JP;Y+5UpHhrUfxm;J;4
z-rL8TiR>Zkfq+C)E=Ng~qS@@%huMPXsaQhm9wVX41q6(JDD~Irhj9L?Oz&$+Ivw0B
zV{yKB3i#MHXZb6#ZggZyKOFimYyxJefN3)n^x|x9N8nl5z$M%21>rLYd>MRs)VYk5
z=m1uL*ZkR5F9eN~%a${}=<Nn&=i7SXA@R0`$<hPp>jp_+cc*z9h0Vp>I%i~JLFU<6
z*SeU%f%I5t_^HwZsTP;?e}bQ}WauXD4GRDd3_tqD(<^y}<<8MJ%ll^|*+frcXud|$
zB^kP&%mZME&TM(`)E(<@o@r-Kt)vNkc(XTb4>%%^a<?|j79(x@NL~u5#$naZ4+tzR
z@%0qD#A9!|<Pb{kSzLMBJ+$Y+u=}$w?jUj?Fl_I^gk*TUlja{8o&>-pftA)*mxdp<
z3ew;XS0=o?BaQ&~YgIw%)sQY}v!aX%Roi4O4|hr!GuRvZzp)6b`%JpgZFmeKX(4qN
zM-<M#{yF5iWpuyp?79pYS1~RC&^0qjI`$^hV6oU$2Am0UQ1{Z=EN;-}c}t42VWm{7
z3n7NW-Y{XNWS+*kiE|jdg^*o?QB-|pn|z1le`+zi?yS@ZN{1{h%Pax;ob2vA!~YOs
zM>}xOtfqVij!aD~Gw+!Pw}8W$4}23oP^^<4Y}zu1*{s<3WfOYupSXTeG==VD4@;T$
zx$qdh95Qs42y|s|_+-IE#v<b`Jj5WBUHe>0%=#-&V-d*V^`UDW;Ab~nwP7Md5SlZs
z>bwILBczg9=Dp`vAP>SueXKO3M9RL=S%!Lufb=P*fvmYrM{B{UPokA}RO-Df<Xz-G
z!;r1#PqPqmmtnl$h}uw@^nH8{IXOeA9iUwqekr+6YvZLtj=u48gtkZse5C~rmhCxd
zrba>Dp9!Db#h3<&4f(M$fhT5CH}H)!;Gzx*{91w0y{b!;y@c3V3#?W-77bH+x6f%C
zgn_{?rSUyMjg(qMRpQ`?WJ~u@`t<?#XPxfcmwhMZ50{aVP_gW<UbJq_WL=~}e3z{A
z$H~!13PM%hP{WNN21p#Reo>`%=eTW$m|912Y4nMaV8gaMzHRsGx~N{y-=QHhu{i9E
z_v%ciEDH3<m*RX!g(4!^D=cxU*KsN%H+p~LupVB}aS^JEpt;;_ZOD^bXTO!<bG=Wl
zLNS(TDC5N3(X)3dQ<m@e>d$?U?zRo<oTcnYs5{I<(<d4elSCvgZNY9e{9lKD=g$bN
zINtwsr2mibJ8QH5{83W~J8)&T$?N<G8C+NI@;J#)BMAjqw;8x@e!kRMU~97o5Zz_a
z_B8A|#EcuY1mExRQO`KhVW)9V0Ob!T(an)J%gZATI*C!xa4fu-N|LyW11j~Z@{hRs
z<G_au9KgfiddtO_ryk_!DFi^Vqvjqr>C1NpNxf1m5kD9pQ)#H9!g3rxR_t>HlM)WT
z`5SpHw+7$k+D6;R{m8|$AoYG#E<-Z2at%EGc(4fe7YTu1N7S@2v*-s!RT=mg<GBn4
zm>?O*E#qBZsN1{8ih(lee#@qiJk^0j+b)7Gl`E6d^*nrMn-5MeNq*w^^0~l2H2TZy
zxH^f0^|Lh-vo)v%)3il^576%+au7gj&z`L#CDa{=P+NgStCz)6zMwy0_J6b!H%gjZ
zloH~bXlpW*5w&c@sKw35zI!e7ymv#EKgj)!rJD*F8d}g;j0Uz&=~<HYEVVtZVTn2c
zrcjhDf)ku*8n!Y3d~C8^OvoPxB^CnG$G&^kF<ZoczeN2$dZ;g-DDh~I%-pp9xpsn{
zQM+l+&%`<S)x%M8S{VGW0G;P${f<5V`_wm*h<qgx$#d8c{>V;Z^`FXXR4YkG*~rGp
zo*c^l9=p<h3T^k>&e`ki+8GjoAlpb?$L>7CC4+_;winU?px><Y)6j)b1LV-7%rm^R
zhYO|ER-`D~=cNs+)G#xE+Ecm+jBo@bEI3OKn$nViS&r3Y9s0Fa-wze@A?tPvu}P-!
z@`y8_K{|005~K8@1TPhQ&OhU6I16U_f}!V1QNbQGFxu&d^}P0FKSWMrGz;yz@!l&V
zDd7VB(PaL9aVbw@ZNN@EwmL-;bNi8CpJlM2@JsDi2a0-!O1v6or(B#9-%3Al#v;e}
zdE?jaAy1)ZP7Ap(kCM9{bc@d*li;bwuY1p+Jf;cU<WrWu4j+56E-UILG8wyU#=8Ol
zfFA^Ir6^({)#t?tGaI+Qk9MfGi}*?J2zg&0sEIX<DfrGB=;3#sZQ>fqQ9oI<p+DPI
z^INz+pwE0U^66e^Rwubg=o|^j1ecoou~1#rM|3Dx7AK$&m4I0r$wCOCylk=0Bo_^j
zX0qRHz}!Uw+^;krD_p*we0Kj5R53ofXH)7n?RsY1OyW*|FPbBVu<X;Ux?5cT6~TyG
z)3-lgqX%X3qsW&HU5Ux?zmM@Q!$;Id#)DY~Gl2pd5hs5KG`983>(`P$s4mIRCAP>+
zvB2(btglLc^Jn19w(*mK445}ZU#)*SYfgrS4y^m1+b$OTGUN`$B}3|i<8+cCMi?Li
z9*(Q(_~U^;T`@N;%>PVBwvSiqeYVhIi_`3cggx7YD32%_1(^;`6M+fNGOu}QW;P~C
z>hFTs3txojL7Xvb<}eaiBe7_34~sNn1TDTEw0Ng5Ws*^IqGAe5r<-)wi`%^!pm=Rr
zz}@$0q=uPSpTOke$|m$>vqm~Hxq^qz2K*o`7=`yx8}6_?b9oZ7ZQa)IIMI_Ovv}D<
z;hS3?DVsa)ps<PG?F5&J(gXV!ih~(Ho7(lBrSSM}vI(j2;4ZyYfMbzx=)>7Ulvbtb
zaX(~t8m{`^$QoCGTu+xkZ^=n$Le%oOOtKxaR7TjQ;x=0{4LftMrLf=UyGTKQGI;_D
z+D7<CQ#Jr4_mrDcVJJ@o)ge9bT0fqy`DRrTL4+NK@^a00Zn1i~=2k|%C(D2&Zs$x~
zhS28?$fEbH(0;5yOo_S$RjpS1I+=OgEyCR5%k%r&Eyz4pX5N-|P0EH#5e%V5#Vdoe
z<@b>-w_Uj7Wo190_MiLumpOq$i$fS3|AX3l?R&`I^np9xj=T3}%6H_|x1&?0tQRnT
zD=)m2l_W34LkzRRHXt`x3V)7>Ur1$MNGH=EY~gZ?h9^D~HJhjdBB2NxR|(W1(J8ET
zC(_)vhMF|VeW^~6#!vKO#&wf4T@HNk7$DS!T6dLeUqAbv3gxgU5;*$hS=Hx{^#uUn
zGQ^h$EsvBwP&8ZwPnbcc#S10FB0wb2pMr#=BQ;C{x_$bY#q!`_%N}@Vz1bu)T}?!)
z=-mEMiY5Cq0y+>jdgmvzBdGg0PW8D^CLwXt#x7Ty{ZNCz)eX>ntP2nE!AG?cA<?`g
zvwTsXlK%t(f?jeyE$zgnkx80iM%zKFmBdkmPb6E<S$d71ezrok8r4zYxT5P;Iz3};
zy>z!AR-~V_Y80}}B)z*<KVI$$TDjReTxtlV)Lt1gY^QW{%!0=a-Wj-o>3(d&lHn{q
zjKrrmHWQNYEJMD{<IFDGrMG&dV%3BAFnyv5z%qrI{Hr;m{HD3ujx+!a0SZdqz5qOh
zeLdYWODUs?;P^4NvZ_iV6<Z?r2c>oPM@joiMeZQ>V$ZU{e;I;*{kdZ=uF?jpyv_bE
zzkRjl!6K97RrVX>i(#v4VsucJ{VXpKFrSwTM!{IjHoKW}N<}zNT9nlAtY}mL<VWb1
zi>UfIhy>~qqmY8=3-%EO%vKX9!oqIOJ3mi^!Q|45I}gP7-5dzzJc|X8or|ZD^oyb;
zq5wu`NhlJ=1m>&i9(pAU`x(K=|GMKHfK3l{E<w_%UXJW9%e928xMMcvFNo4<Pu60;
z{bc5%nQ;yhEA#}G%1C8b8rr5K_wjV?kQ1G^QHhJyB=oOn(NQXJ!5L!YGAG}ZeyQ`P
zm^L`B8F}J(2QDpU$~AoWu@YsN=kYL4x<&H&Syk4Iqofo$FDt>z$k3&BpD)d}x1xK)
z+Of<B(x$}7us176od&8OPrLW&K2+nZSi;wIy$%CZuBJ5f<_Mg^o<gq_A2L=b@E$OJ
zemeW|JCaurkofg-P35*3R3FOIkRwntDqm#6+wMLAw_^ub7dEGcxoL2+fbaYb>2s$h
zSGgyM2&`NdcY`~#-y<uJ5d-Ib*F97aRK<cJ?<I}xK1_jL_PSa?nd8%e`9Dc*1#z0=
z->LQ14*eHv$Jn&vrOLZD+d5}5i*kdMFn&S;NJ91pc2qqh*Xih)YfRcV#Yrb*2uS1H
zeF^m|yo!qtbdz(cvO4J71DeAIb-KN0K4mHUL!@LBwa^bU5ViP{6M{1V6)DEY7!u6<
z-HDkcIFJq2C2VEMs`UWGA)<ve$mTe-3U!Ey7HpD+vF{+$Z8WyWgr=s+K(4+VjcS`E
zY$%d48N68e3*yYC^-szHJFvoItv0XsH6-PwQDsoXNxMzq7Vz1<?nB^iT*G-xEc72w
zL8HK{=#nhXa76#>TLLDBJ1*AziL%c4+|g*kCP#PQYuzO+U1GL?A2n-x>0N1eZ#qbc
zM354eL{eXeFE^}A74*80Rmrke$(~w8KXVBpUT0Z4I*b=o@=2dOnA8a=LtWtWPDP}J
zGzC2ZPeIws91!Alx8(E5iLl!(<C8zk=0AOY&M}fH4IwC~Rec<kXyMgb45jkN$UL&o
z-;_0JB@LnyZopMSg&ouBAKWzfB|hiaJ`?ld>lKiHt0^l<E<*iL{rp24Unn_>xp6mO
zrri1C9p*=eSi-@q&s%vKXV;9_P$`yG{b0c6XBw7{{UGLG4_~`v++~|Qs=2fz-Lw9X
zi1U}XX4%WbcAIqFj1PW+KA-z=X87#~#&)U>>~C7Rl1EtR!Tn=pgX>SWLh3Eb__=Ad
zORW3<E}&Aygwj6mP*0uZs88q-HXi6NO_)|@PDi7jXrJFZ$OcmJ&Or%57hCyah&v7f
zR%fA-Jc4NqfU<f&CYY@6+nlTa_;>_C5`i}2j3utpyq5|x_VH^AxOhU;&IvXJa@~}q
zLI6b0P~n{Q^)u(f6d&6Z(x;wJe-dbyexGv?WIeTEoCcDe4z`>#k2RphH-l!=nE&Mz
zh}*01|0s}njg!z0NH3a06B6qOpyhQ0)IJMGXF&R14mgJ!ugd@iFlr9cLjptQa{tj(
z2)_r?r>VICqM#bbT_;8&`v;D2@zLh1De!ayESZwjU|?n)_o8DU<(997Gi<o_=;tV^
zhPRNGGx@LpG_4%pO`lKY(4M#&IKbuLTw@+?-W34RQf%_wSb4@1=%sT|Tbe$|NtZgU
z%6%{EaWR)k*C+8A%CD-~O4ebiLY;E<z{H{@?yYD=&p(*xENq~I90sZ@5D6%Jm@?4i
z+;ATWeZYs7&8fkklU~cc-k>yjrhV=@t0D($SiW@&9alwT3SLd>b*z10{e9;+o^yVE
zmgxMsvOeYoirP_4D17Fry{w^eyMg`=`B;b;BprA`mx#yLtgq`q!vw{V<0HY|U2YR?
zkLTxXzCezuWj25{Eh~QID-3JltCCy+SPz9IJk=sOqt9c=KYH9CG_VReng+Rj&J_?}
zdAw>pXr^;Xj1EaeiEgVcibgQiRu?A~TaPB??dR3n&>FtND06sjelvbL&FR|g%c)nr
zlX5;&vF_Yl<YFZ!Q$PC7W7XL6=piSOBd2vm^jlO66OzN{zH~({-IeK{$ko~r=LkrU
zqv-K`SPF1qOJ^GWJT^eVQmjEba-Dlp-*{}y2J9=jleMv;&*Zqz^Ou&^zf)i)N;IJg
z*f62GUCcjp{$Wx-`>*>HCninO!cW4;(RZRPHD;!?!xkP7@pG>=dhMB4Eu6mccyFvr
zfFpj1^yjHy72;lY6P-mVuZ%iH*=50}w|IV)?df&)U7eIzPwH65e)|!dgf1iB_lmRm
z)OwgxZOydE<>1x|t{gKZU6<P@e_r}-^kI3vYnp2&jE-#io47~qzFsk;%;@*Oh?=b=
zqMBQ}vOD>esc1rXff@Gy4y>BV9_0Cg_dZsne!jn+qnQ;gtf5ms7Q;YK%_7-XOd&OA
za`B|ICe=ZGfNGOnO<}nxmMR(u)^N#CF(f2>KD96pQmFXp$<pY)LjvKg*7u1>sq#{*
z_)>k+<M~cnsj(TD-DgEdyJIt^eM|!!-%$J*n)xi+P!wj0b)~_@Mc<4pKJ~~cYO%`}
z<&jmAA+=?dv#Y+2@2smiHfWa08+pfqEhFB#S2hJIR5LvU7@qHW&5CY3JUH=+73w;p
zCXrOsBkUd_OhTF8+l2!$>2KeO1GtWSt@COj05Li{d2`*!(3*&0n}z(2?16LjpkS5P
z);@=a;JFjs-{u+uOjOC!cxs|0;-qJ_C+efv_{VN6P%lBGSmskm+4C<pOM7R;;K2tp
zpeb{_@}82L0mhw*OS4lGg>P5bdy3h_V{(?deuXZ(ObD$G$YlQWiN91Q-L14;F^(x=
zK9ajQHHflk5`%A$S}LY*kD#>0e6S^$wkupZj+A*?e6%RCiHowBs$oTIbuEg%%ha<N
z@=A=5F8;C|7W7M^OvQJJo*x?U>2gnMVBF!goDV2x0x8t@|5w;2=Q-s*nRn~d73EAe
z)@emWJ{}O%Y0NZ~4i)7VL>xg4uE)RyCJ^2cjB}H7&AFaGmm&^*KGhPzZRK(Nse;1{
zq<>tRd(M!b50MzD0?Ho*`OVL`%JnaI9(~pHkw3HOZ4?_5xHi7@W!R)14Wps@oG2?T
zAd3Y-jhlRwY}=Og=B){oaOs0*=AAop7v}9U=q~~uWw#Cm(~&(-;1aNX`%j0r!g>C_
z+Y8FC^k3d(krELo@9@S7ylS#%Ii!A$zd(QPg`uceesiX&Tz<37!-oo!EPbOt?RgCh
zA$_M&F#mi2S0^N7isip7gSg^`447`0J)rm}^UdKqVnZn%_c3Q^enRl<_|s!^Qr?WO
z07eiAg}<G9ana{kKF7_pvqhTeGl6NxYL}$VQ*i_hejN$Q@Ah|gK6Z_5{8~?O*?SZ)
z)^zoqdjL3AOn_R6mxlV`7v|yprk=7i<@=Ws!mO7V)L3aEj@fp6J*EMuta*^Dsy|oc
zZ7$t1d@#`MFxZT~l5eFYihXE8H9WVe!ydc(^~N1Ug0_bxU(eq^Xt$XaICs~I{)C>0
zVRJhLAU^zD6-oM#>d7cosM?qt<W|tiLOs3q`G#Cs$9+B9oR+v8P<cVV+IkOM4@gvY
zdN$d#8#~50+hT}^65}5@8mauYyzCE34jSXe@V^<u)-P;m{cy)hsa5eM&l)e%OCOeu
z1fpagIP>?GoHpAe8TLH)p3mRUWvWXm+8J8BA!SM+5rg%!4Whr~@C-LKPC`^6#K|rO
z6eZDp7V_uPpQ%A+?2#txW7g;RLqCD%a}WYR^n)1!cMnR$jJGUKbE(DTrC!9MnMKTk
zxaxfz!RkReZ%i~jVF3~l7p!f+s4hJzc24~Tpp2$xxs=~nhyu{7V*))vn)P*aP|z9V
zgcE=AljGNGmKH#V@S)8QDE^vhcuA-d#AS@j#}AO0B!Kk5ab$Ka>kHVEdf|oEWA{N_
zB+-}BkYgHvTA&7w+}#H{8-7e1hLX4tEyzL8d;}7FD~q(%vLA966<des@1*G;>R}nY
z(X$N}s?g@0f1~($H^Ts&3#PUUAdgZ3N8}01ha*<^Efu40mT}d45l#)c_OF5CU!2=f
zrV>EY1x9BCkf>Cc73~Y{vkwq&-ABS%&@E!1pB}DJeQ@s)u+N)UAEU8FtQ2rk?m_M3
z$f#l@6*m;XU3qBxoMG)Mgv&}O(-Zb?CG3+>9ti4qd;-Ck;w4_(Bkj7UmcG*|9b(H*
zZi&Q5t`bj}fxZ%B$#ozC6bD_vpUN_p`Qpy@`7(fSA*FWZGBOg@f#`Eo-<u)X_|5!P
zG?zjIM6$*j;%J9m-a8wDFZa_CuDwx+-oj4*l~Cn$AMIk-6vPyuqY^*jydID+?sI;H
z__v=o7CU*hcG@ZVMRpE9(=Rs$9n73FpQYvH>XZ15BaR=5<?NACo&=(gFTrteoS(~G
zIBMi%8~c=h+$GJo{78vxGK({QIL2wBwaN^vtS3rBH(1?OKm{{Z=R2Lj0cu<R`&c1C
zhzg&3!s{H3_nGy2k$`fE^P_y3z&ildX$)RdJhVzYvOR~EuPI3Xprd%eG%E70__=Rb
zNSRaQuX~BoqQiH1X+FIw5VecYx@+5X8k_Ldf_mtI`uA$1NM=K22Q!TSK8|EGDhE=;
zHPaEHzh2H3(npTS%+x*pW!4)$EiSYy38<l5M2T3^c~;8I{s1-40oLK5@Rrym*N8w}
zqp!ck@EecS%oY{Te)SJpR&xNJ_a&2d_&<3gjM9I@V9~gtMt>=4R=vP}*mNc-bhspT
zW+2e*$hm9-B@P;#w?jd~mDPUU_<1M*vuYi}zgY+8QVf887C&_?B_zE@#@uH)#_;uY
z+Y?td8p#(b2aqbBVsL_Z<vwQZV?4G0xXW`H=v)0*zzDM5&ZJ*l;<_Zx1Zp7fcW(Lh
zpZAdR-tQ7DSNhyp#9qltcNAUy#ORsVe~cyedsZ*tCRME(MLN6;G(o?mDvW>#=2#%d
zfos_g_E+rHcd-DzzqPA#yrWZ!8u9r4d5Y<DG|%HbqGd|7c2|?vAb)IJfw)kt36PN(
z6Zvyf%Jjb*L8jXiX|K|j7KrTnzv@hA?`}RgYsqno3tFj$&<@_X1)G5(^Tm=|%HL9O
zwQ%)h&N7s&tvb+mi`q#2jUbI?`mfLqTLq?i$+W_%6Z;6V1KLkDCw?qaF~&@3mKy9o
z=N}Hx+0SIajaxC=LdA>9)=MSq4{Hbp<x%BPi>!Bk7?w%C&&ziy96RmmR`Fmgd-S;c
z>D!K{lndNRW(11<sWT4xTw&Nf*pO`(v_B~xfqtlWyqpe}%a60T^_si|XYAF3B}{jE
zgJv$BGyiAbL-uo+2)&PQZu|wyP>(2I%)@NCk~>vzs?tznl-eDUqxhGKb^_hJ5SSb~
z#q^@5hPlH&Da7kW(tJ736{v1C`CuVw8<()2{X|xKY*Am2t-e9G{b0H#DWxTcHlO^B
zpO(o24k~P-=4aaYChTIrNB@H3FiKTZ70&LaW*!_!Gz|?(2FCHB;=K;<D08llYp?t~
zf6R501bq15-62kYf07<S{CNQ-4#^n+t(bSmIQh)x;kRSUBj+3aSW%$i9bW2Y>T-n;
zA~&$xVxdqa3Us3}nPDG(VratFVh)($c)>F0NNx7TMy7!Tl8z^_FPEYCFbx8l0WI_8
z3VA4hFqU*R1Eoxs!oi6fm-4n=sB}Wa5EIaSBH$L36^|$+V15v!$YjigAAM8LdIF_A
z4`;A*VYo+V9iIRC_0#DG_0v9_C_*wO%um0OS|O7qA}=`-TGIVjF9*{7{8XV9>e%b^
zet3UkH&icm!blYRW?>A0leoT^?=#=uIYE0&3b{-D2Od!}Nu|d+NA{*$IL+RxU(%8l
zHyl1$BP{r$nlYWy&D$8f@NNP8MvN=wPgcEq4*3JPi9e2xiXskx5NUi-3%^3*hYwZA
z0&qYHUBAVU9gbI6{s|7V@wA5Subn?nwxy~SWohm1{2^rL8%?rb{I&BhwsxN4tn#C9
z2Nln?ASmZno;lekiI#MDq*GV*vH@JLdoGBuTubs^)Kk+3j0ZPt0jDODZgB}n9w%@Z
zjq$G!*WUk>sFZ~jzl8>9>IdKzyVXri*95G+7#u-q5GDxYuP`Ncl$?AwyU=N0NvV~}
zbrx{T(4|z8QXwhVGQ`3cg`hMMpbL1@bzV~qR;ll?tFHtvLmy`sG|twPHR5i9$_l*U
z%L}{>7stBCh;u<$j)qNq2xW3RQ()b~#|)s!_*wZN6H|~>h~ccMhrlYcHY?H!a=Qk)
zk)1OUI)RGs{=UB#+RORp=6|uHc2KCAKL%(WQjl96api~=<|RQo=$WTLfgpZD_-qEs
zG)7&`8IvW`M1QzWb@9@S?SBk1F2}xCuD^M{$gGAgw21UcGUNnyi&EPuNPdMPjrG{7
z+`fH{x54Z_hrlWLKgTvhJY=Ksp#gwAM}ey}3oA^u>~&L@%Q8t+4ddqs-t_xD+!I|U
z)6};J5rj|qlO{Hiw;_VI$JO1BEDc0fSo@njSQ;`SEG?S(pOzQrPqKOcJm>w4L}FDI
zP^gSZ6Yl|`Gwby`-x75RFUCsE9e=P?Ba4M@pS@}eZ7vDyT<?lf`8xGmgA63A<NAgb
zm3-Fl(v;ItI64HiEturySpdbU-jA{<)ah)KoiJR~%7527X4PXQ^2?6mb2tx8wIzqi
z%3oxILGcPNouMJuR@|}lc-5dTJ(~5n_+PLUbZ0PzzGSr*aRa{^l+<g6_PV?=iA=J9
zL1ni=4#XH$6%-uagF?Vs$az8YR{{2h+)NmukKxBKR9iMdde6I43{=gCd{~3pQ>Tfx
zqm>gCOM4jjIKgb7kz7oXk?b)g^|OqR_vTl-hjws9a2ukJT+%in(p_Nd5A*fm@8dog
z04?9n77CgE<JHFSk-dI?mcm`!<ekGrAcxGgHh1-`xf^`{02C1d-+(T|px@?(RrIwN
zPTSGgR$^{getjO<M>hKrfrmI6{IN#1LtH6p8Wg-}aBav(6HAjJDgKYg5d*%I!;d+B
z1cCv)P~Ak{9ln5;euE&rveQQs8)fw$xl}M>y3b~X=AX-Mq!YS+!^BDVc-S|8ti<2S
z&#h&^o+SxX9;fXfkK}z2%yQSDtlz*S>ci9VH6CW+X9AZNoW}>P1H(i&Rx`-o1rMX%
z#!~KiYj-pN9Tn*}TOrXw(xk^JJ+^<vofCmKN#)e6sHypfqsxa$A*IlZcdqHxn9{q8
zLpw6UW9ZGJzc@wamW*iK$PwP<6p9F^_@11j_ScplLN~7jV!DIJQ5M%qzXg@`epz8N
z-F}Kzh+h&8lXm>xnY)uSFICBhS^3qEpYM)3^XjhsrDhMGea2dm4`OPs%`?y!TbHON
zVyD(`tC}>?cb&o~&`j?F?tG-j*Ap_yk5pKq^=BS*5ZkHfM82D)r#ji4y(QXc7Dhxk
z{*-8PGWqX&=y{k)xZw6qIc`)qKn~SN9KJ^T*Hzs9E*poQYpF#(tv&JSNSSYWb!NVu
z8}l}NLIf6sYPFv~-#Nw9cl(wChIs?ijfYEmH5Q~tW#nrovQJ$v7(5W1$1*wTK=yHq
zB}W~v8DIz&^(@od1z+Ke_WLz&Szdu1URcUx2&gC~QWifltN!45={8?J_Yr^NkGBzX
zlN{m8H_5ZycURj3S?x8N9HPHg8-p5MQ`8LW!gdb_4e<5too{W&&a=59CE^IKjej%(
z{X??%;+MzR`}$H$UO$to503TBRwhh8Q@nP(0cuyhn*D7T>ak4S;~Gx$OxQgnp)Bp=
zNilzb<gRc;6h2qI_Xd=RUk`GZc_XKL7d0L{-}nD-)EkPLDV|E!E{?Z@8TxyQRm`Se
z*Do=vZLx_6It2A<UYG&ANmh?mDMoF2>GVXf_4g0!IgZN7*PiJbeOs_`7JJ=u_1-N>
zlgN%H>vsy9d1roWsx++rb9a>mwWSSCaP~22S73A2lq)X;mnVT9Oo67eq06}Yr!-sS
z4vylptZ2rxz{8ynn{NSuScybTko+MKGf97!%|p)}JR${k8K};i1!i53tq!bx1NFGb
zreUs<12V_(W1AF~r?p}Zycg&_)Ltc4_}=`qE*7l5v~tdz>iqHS@@2wEYu1}h@8g|E
zMfW`=N%?SLXD?~);F@Z&OjJJ~mUe`!d^hCkI%pnxCg9VyvW9kjdw7TZ5qK$?Es@RE
zZN|KFcSJp+^bI#%?fr{@645ztsZShH2{t(<RX8)7V9?TXm$Dx@i5OzQ%crS2@phGN
z`fQ=!#m3p5GfNQ+LnmX8-5&iYKBA85yW5@qt&TxY)X=Q8<MRQ+Z`&J#L%q;QR-%l2
zWT?zn;L^KL+w;k+OA2pky2bp3NkadEB~x|W;{A?qugUGKqM0NDN5pcJAEC_cfBs^J
zA{czg`1DVuei2)eKfvD3FQT}9nfW9DTx^jqheMMx)2T?TvclwTlRm(KqBfl8#2_#!
znOb3=B##qIN>n<Z{iaJO%l|n%{-Y$V<7jn(3RC25x8jCl2}M)-9CPb?BNyhC!dP8T
zzu8r8F@&Q2@heJvVhk^Q?yCkr8j|iAe)vw1`9)7@o1w@l#oT9I2Ny@n|8I*OsgTtV
zJZHxK*Xq-vo10F33hw+s9du!Q#AVp5Ye-1Ryf?G{*pgV;t~9h#!>vbu{UsHdL)q=Q
z{H_!mdb_w+<C!$n0DZh`UQ#dn=@^}rh%RS?@0BzhO*lJkd!CZnxJcw=Z_rxy=zuzr
zzli8A6H7hne?BL8A0M~V4cSN$N~M|1x=4u}s?B^2vp0SBraP*gMDp(fU5jJN5Z?dq
zfBt8%z$*^>Tjc-6U!2g*EvG)o(dbeZkFR2N9qW%|xa$qX1^;G^IMk?GvXOD*JW0_K
zHGjOuG``!li+omH6E!Nc^jYUjFq(}?s(&q6?ClrUO|@XOcnC|^;kn`ZdktO2tI8X4
z&s`5lx;PaQrb=xTRtKkGH(AbC^>c4pZ9Y%Y=VCrd`9vpvli#N$eBayKL>7CuC<!-#
z{+h(8@g|AuVq<u1Hhb0ioKUlOQejf+8=n%-vpVyh-0T`NPP|vyG$3=4(~XcThu_PI
zJAI8ynIc;Iho7j%Qz<>Cfld1rtzaTH2B|Xcxw6l+DxjALbKdzM>w?hXnL~cYKbD!2
z&YWmC{g*`iK$e<&FUFh4+K|KPJH|6(;n(~jth|rO-TqzR;#5)p+|E#gxhyy82XY)a
zHy*hi=;89mptI<*F5$;kc-g-;<m8tm?CWwTKYzX=X<`?;i<6g->{2$t`!R`&L@Upl
z-S2AE0xF5m6!U9ZbE!Z3^OuwCg#ygr0yA&Hy}wvn94_XUV(g>C#&X|0FY26qmG=wu
zy@QNqi`BnTXR6&%gK<G(?2olwFQ%845NA~U0xdw0HKU8>Xd?P8naL90M6F{%hsV3|
z6^c*QWV@u}|D3FEDW1Joo|j58N8%;+VNC)K#@JQ*UiyEIk24|)IVLriO;i4?pdVT!
z9#?S%t8-T8QY(|=YYN{BMnBPRbPn9RPZ`A#NRt|@B^S=m8LujtyCD0;dgHTP1nvh!
z8&YqfBtoJqm0Ip5U1R=z`hKY4f^{XlGH($I1y8IWt5D9(qXE{Vz~ip4qlW(12xAev
zwKE?YYX2Tt)FWK63iilJPidtmL~T(Uai_*^@gL@1aL;kvzMYm%Q~qGfczS%ys^{=c
z6T2x~<XN?{FSCN=5q|RiZwHmFU1(o)>2vB3ZvOluoBGc7jhu)yHV@74Rs+}NbY9s(
zQ5f76m1@-Q3H-+|7(qlD?ona<d+-qj=$CGc#Un<WQ??rlW^YVr?KGwvLUyEfR1&V|
z322I;(Y7PK-#%yrdu<l@9)AkNO{R`HX_p^|qmSgXRjw7AVagy8isjvzHfEh1hc5xS
z){y%(lW<-+Y%^iQTRn<R3lqUf^v3*;1lpxi_{0p+O3XayF@K|h5*nix?CD-~c1pww
zQtAG@^{apWZ~0tvI@!xKGL(IvzbalR!%Nw0CDiAII{-jp7*TDflFxWAOnXJr9a9du
zKvR>*A43<ri`L#8ikCmBy!eW#Jzs^Ks6uNqCe&n@;Px#9g7JH`-e15Re~(p?E*XcX
z-NW(s=BnaV`hJiqy}X{MTw-wLeo0zG(8PKR7vU~#V#tBI1f0I(@1@@-wd$ds4iPxO
z$jS8up8NhDp0{0zB3f=m$d{LEi1%>Rv#eh_s+_Gauu?6REFQpxGF`FNiSq-M*QCCx
zquZ5#rtlFB_9Z<}3etGf@Lux6Y_GFyz~)Z(3tSnQCHJ1@ue6)!llje+v(xV#uWy8t
zg>AB|E+y%&Y%UvfZ?a0K#&5J#R&Ho+E~c}xECsC<D-QG(+}qToQhbU`{!5@A<^N_~
zgb4&29rM#dPMm*}5Acn?Qy38RJZRsCMiM$^buRe<-TDnzJa)N<$bDS!0qn8TkPFvd
ztOa}*V8J*H`w>Ju{LfAdIgAKg#y9ue!v8u^+ke&|*4_Z?!Gb$~c#<lHnf&Vrws|JA
zk1BeEsGw-m=XUik4{Q|`x9RJ)!`pH{Lj_|Y7YXi$m&ROLPt0d}6bTB=Qg6I2XO*Eo
z|J!Z3y~$?e2n7aTHs}724c-M9)$#+O$1ALQ4f{u#i@-(E^0ca^)|f=fC}VBfW;w8N
zF+tZ159h2OjFcBo=(57(Xg%ER|AOWAOCY4UPcK!Ts2WRxb$>8eeNa!zcamODczK{y
zASeFJE}ov%N&FOA+M_l)Z=6GVs~troU6!x-@xkHYG<t@Qzx$p1O`Q?=&0IHj6Rmw#
zL{-MFm-U=zXo>HqKEa$I=L-xVq-LOZ|8bd-Mrz^x`DsmgXPKWr_wtbO-M*L1!xI$X
z7Zel_M5vTQrwo0nvZRg6Z+5FUxHi-ah~{?ZZu%tVm!`x2d<;v!P%XG}=|HFVfg4%}
zwwn=>Vf=7bw9noB$mxnSQjJP~6uM6f(sRzFRDU~$?et-6FFs|~Y985rsqPF>=Te<`
zvd7{SBX2c*O|R)U&!zQSb9EhMAM{!uN4&UcH{K!eePOzDRcl!Yc{t7l{aNSO)t}=B
za&NVon^b0gG7B{uEy(;dwOS)+@lDD++1F$K9{-bERCGG0Rz(Q;m2^R^$4^xUZz{cH
zxe`D`)%(E0xF&ehtwn&I#i6h|>~3(<CuVps0gB8l8TS*TOHyXuclPNiGE77k?q{z`
zA!^xk^r^z1nZboyS7Ra!2-le`RH(RGqnl}P1UrovPv7<Ph?iTWi~Q!jL|H3vZ1U;j
zt?ScNOd6;nr5ZxLR&QI07$3U-XjNTttLt*ez92UAC%;x~O*M=n4Tt$<pK|Hm%K!PT
zD0I~LDXX1PAh(>u^)_alK%K3Zuy+-=cQZe+H5kkY>X4{t-CMUsV$zYl61&Y$`)Iaa
zr$b6D4RfUMkI`{|aNNFVDO-#!c%yT~)oR12<VD2o-DxhL!4Nbg%F(AB-+tk;x*q{y
zZ#Ny^t=%gyG`jwe?w(D?U&k(Jh@WjdY0J3XK04sAi8;CM(QY(}40n#zsYbq}jbC%r
zS4Sm=L*Q9{8t5maZ$$c9xFv2|V<Ku$-b3z%cB(ehrdBd^>xE5|&1>Z@=MPctHng0H
z#StXRdLLG~zP;Ep3DB#$qLk|b+3poM%nYh!JXCBww3^mX$>}A!LP~6FC6f1%4cId1
ztyNU7ox{fIL<f*?zX%3JI10l{0ymQ-ZtpwEG@F(w8pTcHXVEZHdDrga|MvlyqHAL$
za>-rxN#&%gb{Cjy`Dfg*IzYL#A=L!_n*P;eLQ0I~7Uq->`EuOcNW}Q+&i1j_YLWJI
z{yh77ka*tULjbI6G>{%bi7_*j-{;7pN&E?9JXJPrT}qEsta;YtKI?pi-CdW{xyZ7p
z%}+j_+}@z~95_Fm%;|w`B0%O(^PRnR@io)P8PxU8he@v3{J7w+mYAr7T;Hos;@i|5
zOg1nayjEghw(fRVxX*cB^pcVVl?iVJ!KS9RSLu`b_XRwM!Z`eJ<a?B;0~QATTIs}p
z%mMkB!N}N^VPy0(gT0>!+Dh~Qu6gHxq@M8`PKj>P65jrMln#T#7&aS_?rs-Z34t-1
zYl&*?8tNid`_lPhs3Rv*a!QX&vc^i?SE`Lo$|+v;$*DVoZXaA+H@kmKl-4ZtR_oKf
zN4RA~{XO!8Qo@(a$zS{S_B?p&yLFOK95fv|Un2Cz!$&&GMtgtpw%HWSE4w6A4ll9K
z4{es0F0N0s+mWotB^g%6Ce>j9isafQ?wsp&73@a>xjaxZ>|6~ZC7jn%l-o`c5GIjt
z&s8TW_8cpkrcGmat~S|1n?@w~Z*@^(zx$+Lg`3mv!~|@OgGTiM(B^2=(9W3ww$(_y
zjO7_|i}=NXa2og^huB^2bioMw;N{go;!9+hJ~jW56Im53Gh=E}eE$%jJ}{Eu=g%Pw
zsNmP!^i_gu-2$f+cbSaVVfe1rl6SistiKC64JJa+D)Z*<(xVOigaQJ5NMFeB>;>l~
z*bB4JjdN@TzPefPc&@{m&?un2yz}$d{v2eU2WAeW?|pFPlmfh?j2JcJwGYnVtD{D~
zLfvyuj~pQ=u*&UI*rqTzyA(5CO7h!04-Org486hFOz?X#&Z4|%qYqmO?vBTwwr_>!
zdLc#U*LE!t6xsVDjXBDDUO<BZuk|{K9NSLuDvzd{CMl$4+L;D=`eOEi02lSZ!`OFH
z@&PP7E?E~v=q?um%;{VGNaX`NkLE;1g3i1+hZ8$*<mrahzdyCAF}}6@A_^dl?iAe>
zL+z59>P;ZEG{RVLc5knKG8$IDd-ARL^@8X-1SrQn0|Nuq#wBhKCMK@w<+!}y(8S!J
z$vD>6Lb2Df>v`nB@bIVS61~x7Wo1RLPpQCb@#ldpo^0|x*xhkuz1&4w`-&A|){&zJ
z1>tF1tCV&gR=nAx*Xr2TcB>#;bguB;$DI)npm_K7g(mO&p(VDnMQAO!;cK_}weuEg
zHE^^daOp8elr88Q(~^_%RQrmaUy?W}mC}2IU&hprgv9LX)qlcAKNA|)#SK?iQNc4b
zp&5`2J1s_YqV!Yn?~STNqh=;Xg@moQ_rh-zO-3`*EBjdG_Z9L$nbk>4|L7muS(_U2
z8L#N$p1;T8cPtcF>@+wrFQ$p7KxNUTe1SC6cN2t+R5oZD)i7W`;bagNeD!qSE&(LJ
z2LhVV`=N4ShhUTT&|v!J4=OGEmXp0?Gs+S6cEk+3Y$}Ni-db|d-*)U8FEb&Ujxi`+
z8PjtwRu*H#J9o$r*M-mk7KfQC-CF;x2jhF;!^NKxnEdUNkvu?5M#Hn>iKE<okG~1L
zM^ku;K3{2jDE>dUW@ocIyDYy>0aP}{<wZZ;<1C5<{#T%Cb|nb@+iR%4a|<DQjSEDA
zhOhs*gWEG)<Hq^<ZBCINLk`Q<tJZk&OeZ$VguYVjd~jMF+0Ay-xZ{^jtH9&PIYmMV
za!L1vKD-A64e{fxt@?U)KyqjMF*E>gD&t9_6lIOw+6c`naIT{F%QC6#z65m`UP4%U
z`2y3<OSA-m4U7e4pG`Qz1<W}c{Ubryubn95!zWN*Xu@&#_LMmv@#<XFJrEAmr*cgY
z4*vGOr)nWm{lzu8tK0l2f*f9E_SP_!a+fY}$I<&!?^9jP^xwJX1RWx%+zQ<n&+NR8
z_($y4>*^4Ciy%avfIpi`zWapTcVQxEk+0s{rqb?yg*$*eoWx9U_u(*H#^v$S9-7VZ
zG4DGZa2<|<IOGxXc&#9fCit4nA5TIyGr=6Rq0F_-WPht%G&qb8y!}TZmV0+GHA{$>
z19O_aH?3rMKiZ~lzxP`Y9l^kXfy(TI@Y9}U61e>??e37t9eR*f{dT;Rzqfdor2bs#
ze}T-s4vU{4e9`#y45$&0fa#woHT}A~J%8>Hxd<HTOnGkJo<Dd0>$fPuK5>LbI1^$~
zv^b0(EE;;|>A9=_IWoAdS`}t1GFQ;d^RKV|tcmJBxSs;3&ukGCS8J6B{4o?X7@GH`
zDhdByJ9;=Y)$dDB$Nd@azkb79MMdV4347vdA0k2rbr^IEyxsm5<$o_611y}W{SnIF
z!s=fSBY=g|%_XyJPwDYGKQ}k0<;V;}gr{=BQ<LqO?*IF#d!fBK=BB;i+5fkvekB_b
zK$UnMvp2F9#Jxah2oKzjtc^0IWS8*%&}243lM}DfF8}MREkcPnin^KuCPU8jv*r_6
z2AkjkZ3+2{%1(Tth+QG>)~n&_5jo)m`blj_bPXf<>$A-I>dfo<ZTfL_GZjy>;pJGA
zRE~7Ktj6bQmOUu4BC=bHY5!nbiLoW@wY`OcZKd{?T30-MHru&!TGxCgkB!TAD$f-J
zPjp=I5f7bc!{n@6M82|FT)%b3L!g2dgTqS{288^U9=oH)@iIZ<_h>hQN8!KcwoU0f
zf6DEveu330yt`t>uT>hWdiS<D-ZzYz|MdOsZQ(iOza|rD{2(}EQ{8NVY1K#Up<mj2
zST>%cH-5{q-Iu{5CE+Uvhpb)6{?|8KLyL~U>CH3F{bE%Plc_?J<aI6;Y|_1?c(A>r
z7zfnxIQ&?HGGjS+&XtY|Ju*~lkcrMi0)LZUdFy{qAQn+&hq~1C5uNy7Qzt;3*!SGX
zICa!mDQpD$M3gjZ=Mww|`QPgPgt1;cYayIX$Mcc2|2lOvsxROx-oZ47|HEOzD<+$B
zM1EWUoI%JPxA)h@Be0oUFXGg}n3>Zr>zdU&gNc8*(DF1tCye(H9o_$+cskg@WTm!Z
zp8q)!vS@vOieqO@aK@2`hvSO;GPFM7-ff>$)i~U9oY{Bj!<u~_kqL3luI;HtBP{d-
zefec>39XTNPw6oc+IE6}v~Kk+&Tb<E5&>DY2xf%PKYFJ!{I3-pWdx?)d?b;uRn!>z
zDb`f<`8hF&Cd-NuZY`9*AFO0A6X#4f*?C9NQd~rgoO79tbZS0gEgr&Q?zS213OpLs
zG{g`&-|khrEwC{3Kq}R@N>va3dpQWv%|dwj=(PH^2=Sh)uVW{@KR2A@W5YDh%0=Fo
z3^D#xUaI7?dd_V8K&0(>-N6Wnkckd~9MAM>Hqk!E1}fURT(?5i6N#!DY`Hz9)7;Va
z--ae@N~lKtRJ%94P0ZX`vra^P&=~-xj3e;RBOj4he3W)(pyPA=y|monwF<#|w&Gh3
z1MVopp0$CX=sJcblM*MeIdo`iUI}Ww@ru#TJfby76KNed{fWK;cXT7yI;J|WRN3`m
zLvykKC>?vR`QZ-Bd6J|VpV!zSoSA3H0<4pte6>B{nQ$Pijbf&+=v|_lDFl0JXnIv~
za%V{}>nJco$3BbJEoLnqmh58_Ey`B8Vt%}B^O=USwy$80YbH6>a?Rc`38GcK?R$78
zQCH9SoydQuL?zX;mdDW4pijxCYSzAxUVVX{Gt0>wYjNM82h<B<aU6!AfU9-32vqLv
z!+Dx8G5qwP?K6<4KpU|LyEX6D(EojyQ`Nz2(ArWzw`hmv4BVjQ)+<u8_%Kwdqu078
zF~eRh%vn#~*zApE<ly<mItrR$E3+N#>np{`H6luN7*cn~*?x;ge5uB!t(;r$oz3{D
zT4ujt4ix(IOJ@he-GhdyLDaW1PNBS1+tl`Np{vCSDi3c!n(od$WEr4!zbL&Pd!_Sf
zl4^Q-zjHHTh?@Mql_^SEN*a?LWzzo#`B%Hc%^88JDfP3YBi;*7swYl-C^agnr@+tC
zf@L<g_m3H>n!QE00=U`jRYDtqtj6~$9Qe&fQ%}LNG^6XT?(Ui}V(d+6Uk$Jg>{Je`
z5SB}6F%I%tyjA`5!|Wh8)6R15JEA#H7D>Mv`qlQ5i&5R|`u<1_wh$rvQv=(h;A%Qr
zba<B?L_iNI8PBK5E&GDEcpdKl+F&S1?))h2jITdyj`?vD%P*&%z_oMqLBSbw+t-9I
zr#KcGGc$i?TZ2v>5mcf2y#+ML7VN#naerIcO-3Uc^a(G)ySp40eHtc@x?cKRU}vbQ
zClJl{5NSI?6B|Dpp4e*F?~8Q}4<aGoWxzOVVlm0&epzf_;)S2r&1Lv<OkA2)niDu+
z^U!3erRN=Fe-A);iw4ep;$_^S0brCEQ#-D0p9!MRXbB)RK=Rammj>duXyEFJeA$fh
z>R64S=$a5)QH!kVuE&QpcwFVew)JN}93Amz`B1jr&b6PxhUU+412;FY^4<rejZ4Bw
zwsd?KH;T-_-Fc>}B7BqATI$30?Vtbv-~9Y4_0TTg#DyR>{0IELT|wnfzh%}61Qjq)
z(~Qa~a2DP!45gA>8#od9k7@b5!|u7);%^-Aili#6q8jOhV;U!`<a+Iltsxz+?ACg4
zIw{tK?ha#F7a4}!w-oE`*DixCOK99h!uMNKM&pnTak%#!ySpJ5kqvPqt^ej#cjMa^
z?1K7?cvGL>slwolheHX0h%${lJ%Y{uSAtT#yO?VP{-qw4-cER7w&srfwpAtD{UCk&
zGPstPi431>QA=60Gbln(F))1+5o0^P3j&lCJ=m^!=bR7iZb{JAmYh40E4%m_%iY7$
zY5HsZGYie7pp@;mr|RG==p<L_+gjsSimKi0-`C&EOxb?5VS@kopI3v|BiAB?n1^kf
z8t`B^(kx$6CE7W91pZ>=V9-5EF5bH{d$h`y+58gIuKk?1xt}%CiJ@F}XfI>zYQU0X
z*v^A-HDoNUmr@t<ds^!|1x4miAW6&SQ^!@eZC4$OzCWIt5X5$ky+RnWZ4~_9yEph$
z3s~)#q<$&bhu#-e?%6DrVw<afM+-}Y!J&$ai;?C*NE_x`3Tw7|8egTn%tQ?(oi1I|
z8ECv`5PqA!lAU9&MzN>r%XQA}bCNFVfVe-lRBX(Lt#gu1@L_usXxzaDK>6;m+9e!J
z#9_h;veWcp4|bU#e#@f1#IUdP(OspI@<UpOKXohgim#bd`<7jAE5DtJ5W3?Y@FZ-t
z)PpL97{J1-{k1SY2;j%}08m|gN~Ai-Pl(t>Jb8*k$z`(O7KKtziIUkDt-_R1wREqn
zrTl_d3vw|ZIp9>&#;NUi5C~8&_95HI7pKv_z5QkZAPQq0n5>9)b`29j<)6H_`E($0
z>-PL$k{A4ZdDIN$$!c>ghO3mvl739ie3w_!_L}gH-!cgO(9rFJA3-O6e4x;#RlNc#
zAr(eBF*H2>pl=?RA2PhUF4Nj@J?0^t-1h_DF}v^DBLA%)5#q3F%_?>O{3a*AJ?N_u
z+W6QPa=)+kqm^8tvcEk_eS=+XIC5-3c|)Yb;9TXE>w4O8Z*uxqi=jFgsFSAx4?x;!
z0FB|uFjLJDUJW_A>^eQ}B2d7iwK4aga$oIn6FfnaO*V$c-9MK!`0K*2b&B9SC8}og
zA$SSV`;=s$L(qWA<k8P_aEsjk{viT>cK<u2X=1<K+p3YEJ;Y#jL;jdYWU08-PNheT
zY&-|;)PTdiO2o!7lIB=lH~>CDv1@Ib>X5ovBnY@1$MuRF9t&J5ID<4#XqAe6Xa!Xk
zS}(NHiXOQFFwQ4H;@pP}o|0Y8KoDxh74*nyc$5?rG>NU*4mD3+r(b<A*b}glq9nIo
zxO!CU9m=1Qrf=i-hVSqgSY&3zG`!ud`RKP+@%wOZUBt`CbIE9+IT$N-u}diLJxiR^
zGk6{?4$s(F*ff4=lwJ?KOB_%_6g8aWYlO6e^6>Boj%Qmg<6_42xmHq8P#`__=Ab&1
zxJO)%vbwf5itpmavKInUI?<8OulM1+`b%PUZ(QnZQuphq!ixjPoUcvWE%m2T5TaaA
zy9a9PRBiphvje>>_sePS`S*9o1!N((eo<w0@5a)CX@f&KJ{4Ph1*eO*#d~1I01*tS
zFsX*oCRSVd*RKL1pqiasrp>{QJS&dl{5SJCJxK_UEJ2C<eZUwy1)$-BP+}JaImm`a
zzTIVx2_G0Gp@A9F4{H%=_?rf$9*c4Z<(Qs#Ul>^X-g7_RDX=<<x-9&`92~RgIV#-z
zzifEC1TuvoZB2yR{F&MB-^gy^^)y1iN;j3iJg`5R5CyRy%~bpb3E2oS($yD$Y@$5o
zhZsD{&ol@FIzaKzBby2Hr&t`GuMCP$pV)u2;1!@@#Rx<D1%z(C-$`B8!4aXsA!w_&
z5L<ogU{_d#IjH-|2mf=kw|0Dk28>tsC4c?@@f}=%qd_~p3^JIFRE-r_>QFB|Y1ZU9
zRiz3<9*A@C!$av5Kjdara)eLCJG>?To(DDm)mG0Ev`3bqHYybwFGf9hpfjRtbz0Q|
zqK-~y^IoCJIQfR&t(|mb7<Di7{GdRLeOK4}xa`$bm6``1Au6qmsL|_R>ojv0SX4zZ
z!-ZX`{a*{|CDhn=AyRZ>(x?uM6QfCi9E-qJM}%+>Au9toa8iKb$l~$DO<iuqj2WO8
z#capCO5ON+W7rKzE2ro&ec{rIetz;(?|fRG7$`@6lH11{H*WMfwNoiZ!%<$l_B@=p
zK~}$^-%@t>Q2E)z($3VqO2+Pf@|)>zakV~Ae|bt8si$$C#~z$zj9n92G6mj_JLLZZ
zO=fv0kk$1e=;cuPmH!EWt3-0)7k#X{kWF!e7@BWa;CkkV(eMy40U6U%MP0=9A)npN
zDup70iNR?Y>~>%LU#A>L16m^455tuD|2-g05foF!VXgGw)qHsm)q&Ncu%Fi%s%vT@
z0XwO)3aPcw1F+x`f=;9)0y<T2LOE3ofD2qu31i`XtjQB!i1f?@?D4<`+2y{Q?}okL
z_c;LVX}|iR^jz9$2C2n1(=;I2?@7=y2<e;$4Y%ZDpc6w4R9^i!!WXYGZUpG$4bBiz
z;8lwe$PTyZ0Ty>5O!7yNgQpE7`2&v&&Ct*1GYABH{txxz`R4%F#R?i1=0TZkSIor3
z#LUX^ARp+<tEH()BbR^TCJE98v-Bb4`v=aqlRI96g1gRw=ECI%&{E9j*C==F@#`;<
zcDj(K&wP@1HD{j$4%aZ%TUog_EWerkI<55;L_E4rP7tvj1jv*7DPhNJ?LGl&KU?N1
zAk_iP9dC|h6E<A5``$taNR@o*m*tBbhJfQB2&wh=VU8h-tq_37T7^<TuH!ZVEw3l<
zg%(F<5D-$9NkRj){+Ay*@@=kufBTQ);1Ry_c6N5liI$bk?Ca~jPzF)td&IEhLXY>7
z;R+Pyh44MC?uT+j#`UH->f<9AqyP;Nj}&5rj0N{F3aF0wu(<d6YrQs-tva?KSGn?(
z8Mm=20?0?j6FHwU3|sRTq^EB*a-8#B3X>YzUrKG$+fi~o{d2sOXjkBZJ>@4KGE3oP
z65^602hSw2JHP0R)JjtE!@ZbB5&Vk@_=Tf?8edp`<0pq>Po{1p9U=Nj_=~{%fByM?
zjOc2Pf>Yh+h@Y|j8|TaAe6rUJ!dRWE51C*4$&PfoWEuPNOv4g-p!#xdR9q#|3WV}Y
zCXt)36rt9G&&7aVC7g}*rEc~WgQYinN2n0A>qDaMJ^2D>KQ}Z`$Kr2Bkh2A)pAprS
z%=28bYnc0q_uX9Wb<opqfSyseT_@VU$M%*n$PZo5&B;*!B)A}tv!d?OE=U8hGmW^B
zH>B(?2IBx#bVtu;ppy0BiN?p!<U_hXyr&&HmI7L);~{z_8lB?9IQ+TyP*w8dxW@SP
zF2^dV`W9AE-2X$~TgO$kZf(P`SQ09TfFKI8XoFBdIu;6&0xGDq(ujmKk_$vi35$@D
zGDt-_O;S==G$`FjN`GS_u=hFpJm2@e@AJHWeSe(ugS`;uyyrc~J;pVzab2Pra~H7}
z%wXKbw=P8av`^|M_cKkH*4)M9%0bW~-xo>P^#YC#Q6q0UmIETW3mZC4-44vu^HkTm
z4O>2-NrkIhSC5@4!@mXVLb2+AoH=;g>Bf&V;!9x!<~)ePSfnZWDraHkI}clL1{UAn
z$grR8kq~g2>WaW4_7Ug6g_1my;8A`@^77-i;vg?^M_08o$#S_uYUALXwO?x$y5OGj
z4aKg(3wPgIZgi_lHJ3<EFKA-p$aEy}imp*)l%$6iu5>&!D_dUw=|n>GUp>a8!~-)@
zKitTq|34pi14-Znk;b^o0ovlb#6Ek1e))RXdwEe{o%T^=5%=|gt%)qf<+A=@vA{Kh
z63`1{=D&i|xE*G}7|{h2$SgGyF!2|}r9_0bKmA+Vx<tdM=Q<&3=;6`}RmtU~O1`xH
zaI|>7B|>Mwkrx_S_2J~3${dOeYY;69(_5v+s6pTB+=3ZQlGa+>BES)!W_!1vo{!DQ
zq|je8V`I{JG6&D_MbEa1Jr>6m{Ge=l|53r)Wj^RfhcGiR3zo0Zc-GRB2u&e2k<|dI
zQhi#v*AgPl*}f;p2C+;BEC_YiW*y|?*Ks~H9F;+Qr0?d_A31{j5AtK<%nXVNQ6{ZH
zp^?m(sh?*F#y}pTXS#H-W#fSgo`7Ke_R?Oe22*bXw<+Tl5m}&j{8AQ*cVCm*^PgWE
z<fZZ$Au{&Jyr&7s^uL}SyWcwR{cd^6b`i7m>8K~AODd9<N$gSf@=DVy>6mk#C!EW5
zKV#h!qT)CKitY6V#EeRMda<nC9U@5f0`f`*#O~S(cQP|U;14lB6Q6rdK!~`ziieC7
z@>5=~ectqCf5*c})dNv^F*Tq;r++-o#V0=^k{rqBe1l!sBXA!PgC}!Pzlj-~_S|&M
zmvr>5`TB)Up9`$YY(%KUs%n!kpi+?F1}SSS{Y5XY2Mi)jl2#@Gq5O5ipL#-a?hJ&<
z1-iHvu?9tckQTh%)+e9sSi67kO@1NOQz7^c^dbMqgXQBk!rf?6`TAJTHS+7AyCVQf
zfI}jUBqRrpaScr_G*;c2hD$&5fx*#Z?y83iNN7#0I}6qJ{buR3cxr5NH1t$vaZ|@d
zv-Bk+P`PS?qUl~;bFB$6sX3^J46o<z{oXqV>d@73n`@@0PE6&Hc~J82J72rTfyLhX
z{^}sVsyN9vWoiSHdH={9&}~J`U6r~E9Sn!;#C{JvR#gNY-EN97TBZ2TaTXGW!B|5#
zsjwFPe5O-hUULw{SGJZJZCLF<E&uW?ldeL|@O|a7=x#0ZFCQBPmWS!eir(EnrguhT
z;FC6ga<aLFpSG8m%axL~No&u?VB}H*LQrfb7rf3bXcjr9BLU{ehu?U8&yo@BNTn<Z
zh1`K-`2L}9>wKV?DFVLBtl2dne{a-Rj+;e}6^h$ZLlK4_ePXQ(M^k1*u9`5ed44`9
z=S<9Gwv1#T@BHwlF%Hkx;OS;J$AWKLqx4dkPvF@I=}yF{CMZ{*gkW=2)sxEdWv7=T
zBRa$9Lt;4C241`H@CZ9--NZ!X#rcuwEsf(<qd>bZ=4`RPx)&A%T8!Y>y9_7(svEh0
z^V+WpM9n4~Z0_aZ4gNTI;P7c|9TbjYaaIxZ%mW~*btg%7YC_cuYv1<~(nuL1fnr)l
zH%k+18wL=xdff%eFnk~bp_yl;E1ZQ4oRjBeg0x3)F*p;1f`puK6N^&<5t=3BVw|Le
zoBsUWBVBhU-c~5aNxCP2h*ntp$IhW|zh}WnltwXxlUb7x1!L#b{O5hf*+E)q!lo;Z
zfN-7hm_Q}HeGUjwpUKwsk205PU5zixL?(6xth(ARagw(-#>ZT*F%H+gdi<4~kC4E~
z`OFt(c6Z6wC0a@;+o#sn+|F1}(3M#0)qutm{RX4Icm|;&V!IHN<S8_7t(7J_p5#en
zz%5#-?`g_PN1oNtD?!fDRGtW{sQ^RfgD?D!J?iSx5lfAXlofbo0<&<AM~>bhwLf<J
za@k|oujWgETzB#7*WX;OF>(oKI63pbQtAg{D+TT`?f-u}#*ugwFpuNYWd60Hki;-c
z{IWNv_#~VAQ2u7Ly^>AGk3>mR6#86a4MdNrECHgP#O!+@NH<tH2Vw`Jv(X^$8QkCM
z)Hh_=7wlGONZe2~)M8gBxjGR_Upy^bfo5|a%FCcG5cC;h9De@c<6`ME_wn?+`?VNX
zJm1AE=@<6=ULr;&3RyJ`<ovHhVP-P(oJ-Cz4mFGsF|F9FwwOsn@wmv<^=}x{Qx?u8
zkc+8-Q@D8j8KS3*%MQc$s8wcatx;%(WzLIH1CjVQk26lGzoO(L#01-Y^RlkDyI3x3
z1_PYIYeVU?%ySnU@pj7b(v|2S;o0ZqRQTenB@k3u%cdfv+kdQDpyc+(q~bjwpzVuE
z9K3GKQ3x1C@a+3CPOS@<mWv$5hw|^c)OftjxKmC@a<7()Aov|EuO0*62kMN$(do*2
zWYi3BP4PYyw%@FCKA~R=*TGw<dd{lsT2wEZ*w-_hMfEhl`)97+LKSF$T&B{LsYHvi
z>0B^!DYQiSo&y;fhbI=^r^GiMU-g6h0mHIBIQp$;$k~_hd2A+WcT3G}9q_d!3gaPo
ze*cu0MLfsw{Di0%2_9>guZ@KbZHzaX<{#)H**)xkzdwNPYkyqY-(331F3x&m^Fi-H
z4>@E%G?QS<|6^YFx2!MJh6!3yioH^f(G9(>@i$UamEz>6O0|P2C~c0)yMEY1Crn&0
zM$kLbdyUbEsX*uMXtM9@hzO_<2Gr8tg#l-sIoW5MAgUu=*C<^ZRuR+z|1xVYzxDnD
zIp)nAFW${-y^OXE)VNnt3TDZHT%3iIcWo~bq7Y?gck*ukVbLMWwP3p*=YI0~!b6<c
zcu-?v=4bWkKT>Geb79(rksKAEBXB+5ak3+JxkZzRIe~7@(X-yJ3xzHWPdvAy1%Sx)
z*RS0rCufL|aLw)KJEE=zDjN~QE(--PA{8ODDb|rL29iAw+Na<MICNxw17Egrc1Ywg
zrz%CH(72x`m{rhFpk`*9=fQrb?Gi8fPyeUAlh;q8(8pXS&d8O_Px}#X=Iiz<^7E&{
ztr8IN+bPuArP*n2Zs?@NP3qOVt|;GFQP%P-9!nS<%?PSk2;582(C_j3)PlU-=E4Yn
z7s(XSZo1&sK?10!Lhmp0-X8@ao5L@*67kYe$Qd@X1Ws)eI{&RCru5g2uV<J}JFn%+
z)GyzNvb3J5oBPt-Q*qfn3BZ53*nW?vs>i67Js$i$m5`J)_)J<b@JMOGy<2!HzyEVB
z<&Bq!35)@;Pv>gkP2#8;FrArv4~Oh!(6F(Z);>D)ZPkGIzIKs8&pU38;~#pQIuT!9
z#8dv!<HyvJI%;Z|A@W6oo{i~eC2Ej`QqJYBoEv=^d-siUjA)M9-AiP6J=gSCxEv_>
z6oW;_j<<u~-qV3XL7z@z5Yv0~^r<lur1_gX;0pJogsI?WPXez=E|FMO@uV(dHETjk
zDE7_rQ!N3r2?xO{;v4$8_V<lxIV+i7p}A)s;B#tWM0>_(<V!KT-Vq@yhN*ps7<NQn
zz1XaBoTP58mq(*(e{Mu1<!3Zmo8MH)lHQpQ6pN+!uOO$xb~i39SuW&Qu!l+Vc*j;+
zA~DIve^^4tlwY`8#jf*rbKC|S$H<pEuRxI$DZ%8l#Kj<$fRy!!SSme+2b>}vmBc+D
z=lAg{1|>hoODnD^W!3>a2PEc0Bj`CNn!#P$@C+de{@{K3<Gh={E2JhqY1M(q^3Cb#
z>8~KY$~j#;7pz<RfSy0;ydTA)hxk$W{h%Fv!?eZq^^(2kGsu$lH=vB|l#L+Rha`6s
z?s*Rps$9O!&3q=U3c!R;aUB2BVXXV4ypb-@KEGQ6pWq~$(mn3MrSTd%Oag=ivbh`V
z;q}8Ts)`Q8Ws<Hbh22kf8Y!gC;6m2koH~}YjeV3$0!};fHzv_9vY7+MzeK`z_%n(j
z2o_rCI(e<Lj!1UQ$<}(wh`+(1oyjMmn?LLbH$XBsdh&Th3B+w1aX*IA*Bu{GOtcdA
zgauJVw-CN>{Yi^aOTEgy4ChH2R{~KUAOhyo6&(#2Z@>>aH0`k)kfTqmUq5O^h@$3H
zdBUSx_*wMZTpKu>oe<6H)v{M}p6)40?~WJ;EgyL}Pbb8IP?B)bp}`Zho+j-|1O=Q8
zK%VUfZavC$yfytYcnsku8VOMuk%98xy?j9qB*-PP&{Tz7{4_srTZJmf{hZ-H*|w^8
zv;#j?x@xtWMInPH9JNw{a)&t03@a^;*encyHDhk}t<Zas-5+~GoLGSfbi^qB195AD
zv{V?5_v+nwQ6ZsgkkFWLKG)<ybYvDcHmuA}2N5@2uK_&(W^n1qcI?-B$%4iDI0%?k
zXdMwS!yu*xjuQAIGB)kT;q4V?36zN5UjIpB=6t_$^gxm6pJcH+B>7}ZX1(0ONpgU#
z32w7SwD0ocF7qNhw1)+#KY8Z~gsk(oJ{hUe1E60T|E-r6$_9<)N9fVZ5A-eTkFj0O
zOIPx}*t9AXm=-<i@jzV_H$s4`VDi+lDj1}d+kkJ1rV5p&z>iQacI3kZJQGo@4oRSP
zP-%-;kMzpQg*!1-<y!ONi~~&d@MN>3tbeuQ`k{f5>u$bqInVhS6yY_RrZL8*4-aAD
zt<CXyIodU#CCI!9dm4F&fX66+=gp)zND#;02Fb3TJi$dFY+q_->E3#dU!eP|EnApz
z_OuL<Oqy(O2a|}H_tRq~rSnZnjD&=r?vp<Tg#=mCebaIzq+O7$2KS-xk3CesN3yZ>
z#(J(V;DhS!hK^7r5#X%L)?0sFbzf)PfG0_1Y5R80A`a@SLeOVoU%LXi>Wmi2(s=J-
zi(dd061SOQ?&aJ&$86q~^#UlU+6u82mP$r(4-XCe)RZ@uv>8!g(T%mGyT0yQ`CH0_
z<4*mP0rfITCRu^U1US<Df;%uV4g;jdcK*uZ?~Gd=2An-9AbWqK_K`w=1|HDg=zpO|
z=l{{`QLdWI+~JbrwrX~!y6Gi%Od4hiU0RQg3-4*7hlMm|0u#&Vvq3Qzd)qR7-u7!k
zN(}vlQ!CHhb?Ho(K?8E|0e`J_x~D#3yFxGKn3i+Y4vb1Su2~f~qO;dZKkub%&^*FI
zqNqJgW14Pl94C}HGhMNPCPfVYew6!rdwa8_B1lYCRn-YT{Y&)r1Vt1^PTKHyph;ni
zNcwlbqg6Z#I$DOb{-CN7R7VqrEHokcVUa#Bq6rQ`^ClUO-#<uPrP_}k!PtH~i<gJj
zau9MjN^(knQ&?s!lwOFo<uCQTN^wM&ettZBNP1GXwYq|S>^+SLzTbKs?5ZA=h?YEE
z8lSs#4@HPNc=$HbJljG)rMFs^p)Rk*{(L_Qa5sjrT8C(-n^TE~#$5H^h%lH);1950
zY1VwEZ&|diI^e|`<B+rXRp;cpn=f)IL{)I{#H2c|lQ{`Kg<DJdj7yV6j^U1s_{+kf
z018TZBS9Yom9mALwqr2m6t)iz!tesS;SPL`Ags*QwCl{GSET1JgPg=UpQs$hFn$J^
zIO>A9Gy)u$<0(LZVhzg8OC&<Sw*-(RH>9G&dF=Jef%`|Tva+9ig|(w?dvE^OGBoWW
zXiE5Rl!98W&t<^R?gf2&Pk;lrC?x<CsU45!aDKnVV*A5l>RWHV{6KnHA(N8E8~wNt
zmJTH^ESmWF4mgYrMOv!#MtWnxTWbc?QXmZb@%`2M<CBJ-Hs24PCgN>eUOoj{pjF@P
zpABB?EGuTeyl#W+ze{uhxAm+h-v(ca7a%4Dv!hG)YMyt(l_@AA#9iJQI^L5{gfmhB
z?4l+ROhEx0;*c2V7=<S$MCtJbymT6Gy@;5atbPWFtmf%8JqY?m2PJ&45El_0wScAh
z{JCzhQTY+cVGxrAm05eP<@4LUrMmVP0s(>TXeXo0I-xjxVS$*%^B3uS`&we<E2~k5
zU;pF#f|s5X_(#Z19~=(2t!ateG$w`+g#@)?kR)^b2GaYypyl#@iMqXk@C!7_?!-_@
z_>^be{FHPpQ}0g9__GtD*gk*&!vQaD;kJ13K3wl^)tLyf*8M1mQF)+xD&a`KLc9pb
zX2)v4?UBCmwY9Z#FJ<a6l(473#5WWqsD-P4clK^@z)IRC6-<t-SAY9P&a~I}oNa?N
z$rtxcmYU8YCu?U)7bDnb@Ym1cbyz{MEx|!}XayGZ8KRF^cjbr)<!^|gvOcIZM4Xt<
zx<}kX=)^c2wbiibC%^<EucjH}1^0GUK+O~?`F9@ccjbC-HzxQd9`hij6ow6G3@Wyc
zM?s;99lpp$Zn{i+Wwo5@CAOuIMdE9rY0s^xs-wgIm-e7iFWqbB`0R|<o^jKF+xpf!
zj{_7{o@AD;RA;n<nv4ty<;%P9f+A95U<6at7S0G-=YX+xOKu{SHe!|q7}XiOA!U(Y
zpQ$IgAY~Bn|AF}B&6Unpp9+?qSj4Tj$+KbOq5YAtlm>6>;vtaUmlOHv&uRpqwqsQ1
zS1{y8x?YPf41+~3lx}Xgx=PU>h0G#Ea^=<9xc+8f>d44y!r~G6M<WUDN<`FYgWz&U
zOTJfO=usLydZ;#%6;9S--LvEbi0J;ry3y?}aJNOIp*mzOdx|xYwq*xXF)g`zgqTx?
zd)om*RC{~-59xJ~%G!rl^3ehR$In>LRi0#<#7ATXqaq*HQDU_Z=rRv^3LaiM7bL#`
zFM~`D8Kr;u4eFg3Ncs*TS^3l|b#Jp8{h-@6sy1|FH%4$EM177f*^ZsKT15lwn#icS
zcWbv?m)XL)j?@)uxn?@T@L*MkA>&J|C=nX*DF@fdw-zbZO|8<oD_4K3^Hfn#HsFTy
zx*ebE8pj-Z0iKqS_g;X*9eKaQK+(BV(_jy&{n&Sf6y->RVeUGt`KnRKLm3qMy?<&(
z;hP5S<$ikfCLsCp!*xSW^Gkyt_whb~E@WrN#cEMe><zFrM><m#8jIi*k_@1bV-L7U
zR6_EMpoHEunA4!=)VONReMI?%d(o@GtWOlw<cn{gfm1|vYr1y2ZpBJ^UJq3XzLgK?
z0WJVoLO*3ee{<!U3uFqDhF?aPpe(Q#j`(uY2xMlzc=VTPd<B!XWbkxowQB1d&n(Mb
zYL^eg?wNCSY4@A*onFxxc*@_ko1d{`F({vlI}J`u<rc$>kJn+F&%y|;;-vGD7P~lb
zBnJ3Ze<x?KA)_lsw?u1ZQzs9ZWZ5sjL6-iL>Y?`aiaxtjkq)<U_Qa%S7eN)Dm*m**
z3NA{BLr*r>r#;`EG7m;nxorWfXj=vrNl`x0zDJCh1`v;QnJO5bD5RmGk%xSkF`KoG
zgjLWg{|B)PoINa;W#FNVU~jvhFUv8*fc+{r15sSMVdFS-$rM54ky51<yWf4Ttn-nX
z7gj|jm<xJY@GMQ~@}OnbNZ3Bsm$UzffP6;Ll;~;}F^LQXL*4-yOEWcvfm$LEwtNI;
zkTQcEc}E#BS%7%V+$MpX*XElwyB>fYd{B>C2o<+VjyW*OeSH-J^||=k%%@;_+V7*l
z>)^5#^hZt$*gsZzYapm~r<Upl)9It4hVn+Ir9+NM$PiH*aIX|4ml?KRjwT?qtT-UG
zcH<B;0w{ccb?N;4x6LoQ94gVrO=DeB+*hDa$&=S*8&oo#^TIQ4emXp7WIE&;*7vEO
z!#yvLp>fl)&V;z-mG-#0FG#jcj3$-2k@bfY2!|(H6QPGm4<%3id2y1?1LA9c+>U(r
zaZ{s#q((i(V!6!&z!|};))VE6rkWl(C7^s%5QscX-Z+*=dEHy@Q6UkULpcHiQe|%~
z@&N3Z!At?-)o$K+6hzc$81Lx;2D+*SNs}?ejzy2Ohejl%BVMMAzZnV;;VD~?xelWE
z>ZDB}`TIs={Qb(j?-r%dWYN^rOptUK`~Eg)4GdvE;@?v12nq;eGn4sgM-p&*AmMrp
z@CQwGjG31gtqmBL1yKYC>jWP8ou0iP+9+JJG)O<)2m~;L-;cyFpT92;1=r*Np#Lmx
z?qbXatfE-#0YnX3fS}x>>f5)cpzB`^O)M+Z#&$F`g^i~<-yK8}!G;E*H94Lw0H6S$
zKr{{M&(YZBPt}S1y2{T5zjZjIAu!%ZCQ>d#D~>T(JWLGI0qS*K@CJyE(Sb)=lG)K)
z^Q6(!yE8A^zDZ#gRM|9($TFG0HBGL23Fs={o(mmVio5P>&LlPYzIlS{$nkkVpxRb%
z(=dl#Q5dKW>E@q2<Jqhy+Kzy!>nm<Fje00F%jve=63UwknL5SHU3U8OB%9ddGr4~J
zdkYQM<{n=k9qPP(gRYZ;$!!;74UKd!A8(g$E*M@l?Yq{QYU(US9iE=$X~~o9{uNCc
z5q@j)tvIyIeHlRAa7~Uld%<&a$<udsZMojlaiHL5;HkZXjlD|zS-oMZPARH!$KL3d
zm0D~-A$fnx$w>rgL|2ZBiFKYW>zpnnqqK?MgsAx0Jww|i%7l1bwE8<Ezm4d3=xClZ
ze(vSp0Hy&t$ICdnocka8&dTXEqbX@br{M*nc&%h4X;B0SIWsjwpRg%3L^E6#7=Rm5
zX+do7I+T-f32LAQNDLT;-60w>^B?Y<)8U`PRu;eLxf2{^KiSbT7Y%v7-|XBR<8Xv@
zPwetV08=w_=BiI+-m6tc`__rUS$Qtov_AJ;`#7XFwAB_lC9mK(%BpF6hy?gHV;9|2
z?L4bs3p3A|>Z7N+(ed|6LF-gb)1=iml)ijQtLIDw12tmEz*@FQMD71GN<UKq^!emA
zV#J*8!|~W3)w@68K)OyrQ<%P!Bv}69*WPxdRLoDy3?w*Mx29>nt!|pFO`~FlnbS&^
z^FTi!i*f!LkYPO5dz@OnztY;<;T%Up)mhI=lP7X27P7m?BW2>qwh-29x^D9m=74?6
zt|zr{^|;ayXzLe#8CNQu-$*Y%YQ2rI%3ojt0;|W92Ocu?AOegu%heLxz39=kc_((p
zyx|zVLpqb<FC1FAGd+>4xJ8w;D(lcQ>^U44)88|7lQUqTsH~)W9er5?)6Gvhau!To
zw<48QM-aX9Q4hO<(nwD@JufF-AQu^Ya^*AnNToChqtS5l!`+%(YaqBZd2`_LrgN%2
zAVJ3EUWAUc?y1Y*TQRX-NzQ**vo%BaXyia)H<<D)L*cuZu{kf$6W)0PVm+nh*+C>J
z{6SB(*LY2W8M4zT+1`6r#LQwQstxl@o@<wC%WcjO%^5?T_Cu4nPGmTGrgQd`2A_3T
z@bx;;aV{6Ii(tLH(Z*G`A9Z&Tyz4kQ)CNMlYoFD9pMk!qD70!Itg&+lv+|uNULSgH
zUH$#_6@+f}L#R~9vO_(icveva_!EpSIq9OVKKP%4K3-4p`)Lmx9dUmRddm6HU@xJn
zoaOVBy;x9sUxIa2?>pllRtei-=nE|rzXS>6c=b%Z?z#+z*}E3&NJiVy+eJ%VOc{Ig
z!(<yIzSF`f#t|6OytIVLfLL}6Um}?x<wA&_No~YIkH=+o88^HyA;4QxiK|niCN~S+
zy2SThxlGyKXeJ4zdv#`K+<x0c@)-w?OuGZp^wGKvzcHRmV+yP?xz(t}g32!sTJu(;
zHhM-3ysycao^XMPEW4_ZCToiIU(-%-xk#y$FL5`nMak1NXF95nvawy9B^<g;ay<Ec
z-j!^*K~xR(ZtD|H+VlMSQmf4cI^thfZ?W>uBh8-L|AVo?HBtFfjF!W@sh!tzWa=U1
zQM3TSev47;k2G3K9-ddFInma?h4nllrEHMSM!ZoN9NTWjF3gIacAU6gFp>}?L-Ny=
z8`A4H2+FVLXP7eHr_1>HXng|FfJ3}naSdoT%J@*TkdV>=tYiGVo!)4Eq|ui+yTYzJ
zf!Kyl2pbRe!Vk<I#_N{cqH60<^k+HlOaJzK=+k5a;Zy(*pnhZ|56rUe;i(ug@kF0N
zkM-S*mu041LbHQf9<iP?_fYx_fX=*yJ-vf)wmbvUUU$ynNlru~fBdZs)wlg*350xl
zL6Hm6^GnH~MyjVN-VCy{OcZ-h96vr&n@A$iR$j|M#jiRb&1h!bQ*F(8NC;bA{7WA&
z2|t^xFx~499wvBPn^z~_h&X-IJJI7zpgjX0zpFenoQwci3buC)q^vy2*)-0-q`VTD
z8Mn?w0njJ|!^Cd-wv$04Gu>D(Y(S&k&D~_+2Fimm>Cov5Z+7|R7)n1c{WuBN$UI9^
zm;JSj9d&5Zfr>`qcc3~W<K?AS27QfLkh-i<;QX?=I1Gh`FNxdmU}WBfitF<aN9pvg
z7Ym7u_c@TC^LIjQR|X*=I$>M>o40I@Mf9+Kua;-?MWOaD71WscNbUD#%s^%d5_@&7
zw`rrC=s+ddom=SFSSIEZqO_is-dSH6&vc?m_34mW8DW)!FhU`Z&&%3GiWueKf)e}D
z%Z-|p!1tr!*SGh=mU*myw02?=cebgWo9_e9!IQvOIW%ebmX_Ss|4=m5c-M#_R8(+y
z&{FvOOL8`@C!o-2vjS?;t3f?+&a^RM34pVoG1bbdZp5t|VT<SIr^%^(F2t*oQ5w*z
z6?haAKRjMb0;^6P6ZYa2dn^GV_jyxddQp3`bJFr)F-Nv^b6u?bQT*T2wECY2yYa&Z
zWCDAqa@qb$Edj-B=iqVr-$+gV-6SiuWC8dCj&CPtKKT8e^@KHWkN;r~P0<qyH9h*s
z8D6EB_K>}uf=Rasx<oG^VY-(#2uxD$fwufRhT}HQ-QW>?&Sk4N+MKSPcg`iDj6{gY
z@;szFmS#Uc2QbfFb*xx!Rq^of7=dQ={v6P(t(;Apg+tTf?njF`;<fxtL)}aux4i&k
zNIoOtNV>uvAmY*R5oMm|pk&p_pii5swCjH;^wnL0zd_W9xYZ9zo<n@rmQGI$Cjl9e
zKr)?s&?RBIR)93t9Bw&V_UJI-T{;$ljwNnHTq9QJ2`T<h(Oxvwz1s50Q2Nu8!cw-w
zP{%;0tJAMGel5RRfDje-(ht)2!WkE!U@*@;TWhF`5o4@y7zSc3#Zu<^0}=um7G}DS
zaw4!~^%hYCxFnsg`!kk`@3(lS=Pd5wpOJVEEN8rhMI68N(t*glz6#wGjoe6c79$kP
zIk2s<=XQmhlbIhYrT*fa?-sEKEXi``A|p>wmBu>g>JASL-B(sqF#`%8b4hTg0qPb)
z@RA=9T@kVZx7Qa?>gOM#S6M~6tAJEER-*OT)E$l1mDY1Ta$Vi&QtHu?E8+`Ey2@VI
zSFf1LzPDUDkU6Y!aahvuzb3X}304XCNh7QsuK(3r`$=i$9OaZ^++%GvFBz~izJY8m
zv~qm??JGA!gmlUpTHPdcD6@z!=_<uTw)>+UO(EPzaK`5zf|Ddz2kS%nLUY#8OXEIM
zE4a8lffLZTm(QB`JQ6e+TB0P~@BU67y808yH97;@<LSj=7BR~o&==()rDi*IN=X@W
zu1zV^-1{U;_R3u`F){59Xz`eMm6|dA2A;>q$p+WV&Hyeeep1A2!tvl;>+i~V6ScgN
z&as@hx59huX>riNl!tdI86?_&v>!q+SWWO^1BP}@WWixq#8Zb_jV9LN{A$KxI}lHT
z$go*T&GwT=3nRi76Zj3-EE)whKx<iC+q;hVCMxMOI)tX^cXV|0Om7C51Nf8WBSl@C
zdmDO!rbtI+O*dZonfx?wm&n#BrpbW9#YmSo?gd+wT^96CMbw83m8`kS@B8Ox2ws6<
zM5Ji+jpdKpDeEVE@`h=q+UIKs+jLPDe0~leyKTkkd^>`k+6c-7z(uY;<G2{*Pf=>p
z?umnlv$(nU2|TP(y3?Bvj!;n87rk4qJ>a=m#CWR29QfPdCQ!2j%<rHh*m8Y@zdi1u
zwk<I5?-w?cimfHUwR)tVU8p{d1NR0bg_*oMs{L`na(o=0nb+JUh`5<71Jl%;sYxh&
z{AdMhZ?qMY5$jDjE~q&A(P(exa#PQ9O92G9d-;_+DiwbVS|Ha#fcs0X<!_&C1vaB6
z1U_i~F-PK@Glx#9hl0DihW!Ns3&gl63Gt%ICPXAO$c4Hv2oPmtYO4LRGXGBh01y|m
zm)hH;;4&8LL$VLfq!YxglXV|~tNO{p|Iery$_GDr`GV4if4_h~;WhLUYCl!&t4{iX
zulpH<)wUW$)b<=+C$6Ma4pcoxl0>A}W)7ms!159K;b1fb+e97=S`df&u&UeB2=cUw
zt9VWNThpt4<K7gUr(}1+Kyu>W7hOIQe)2?O@{NCdvbB*aj}sD;nS7tzZ)*sM7~58w
zV)J3oms?*8hGS1nLl5t5-5^LrqR?Hs^0F{S<hICZzmJSk-an}JpZA8XMlf-D?mhot
zfB%8ZFOZ#<Kb>f0${{cW8SFDah}5f}aUZ>A2T3F|f_AxqmB*by&MRxMlRjI^s=zco
z^1Snc{+rL+M9~oGm~W+Hrh6JDp(a%H@F;}`Tm$%*an&RCA90{IU(@#1kUzP-)Itv6
zmLzD0IhG(rmF8Om-;#P_2wm#PVp}o%Hy6A1``3Y?Avg;)5IFQ5ypzh7DZCvTFyWH*
z3&`c}mza=Axf5W=CdI7&-`ryghLp!M5JP06wyc}BGapF&h|VBD4uW>UVN07QUk8(~
zWqW7Z;~HXM8k(7h?>{`f6Nr+8@q3#8@X8@unx4HJ^Wi~Zi~uq2Cn<s#A?+Keha93M
zVG?7syC{$KAxwc)5)ewRQWplmY{O?+lYj0=j4=Z1?Ftb6d(guqqE~7Kzcps6>po+)
zR5qv_Bv62nKn2vx7{@5(Sn&e2#|ZTsUKOq<D$vG7YnHnW`AMyIcifPi`xLisRdM-=
z<8{|*Z_kzQ|3`8nTf2Ds|3>MMp>u1!rfU=Hz5xd+Z8N{@v%Khr#UX=0Gq`eKeI3IS
zEF!i#6ps2Zx0I$^)Jk%8>Dr-=JZoMsT|cgV0)^fi7V3qa*|QbyyJ^>J@%0<_JcJS7
z5J1e4G~+L+6Sb|u&f5~Qe;&?1)kyw+ZRT6orlR+g4p8{d)`W*(%nPh@2NHsK6kmBk
z4p0eP{Xrbbu}~N`V7IqQF{y*G26$do@ZUzveu7KmDTFS$_0UN3A%c+gVz(pkc7Q35
zL%B_l)~}x1o)eeH!X=}#Dg*z19Xk=iiyP+T+i4Vf^Jg5USt{nCDjQ=hk)*4tb}XxI
zaAeuDjLgcElc2r;;GVV!bg(2ya>3)GwQ}HcL3e`ezD6%E$my$~A2A5H^Ij5?8&^&i
z5k0IkB1G2ABTp6>UfEJHN`TK39mVwFP3!F!dH-5IFuH)=@w>v?+glrp94LFAhwgw1
zpj^cPmss$ZWL&Z)GRW86y_9O1smXzjPjfBui#3DrpU=d4h>(eN!rIGA@FrMKg1-PA
z!o)eCr9P{19Kbf{aVtqnOIKaHe+1)&o!#@UI+6fa-&NCs)TgS$Kk>tfq&^@6gDw~J
zj{FPZUVwILa!<sm?GwTPIY9O+Z;x}sFZznK7QS{V9@)8O1uAGQD1`ES`Q5S@70J>r
z!gEMa;gbqgRlS|mg_9^rN`DGUXXwiqfssRSlt!kWZm{^Q{A;b8H}>EFP&`?A!x1r7
z7{zDm+w-l$qrwi6jhL2tEEaKZ)o7zSYXI@xglg=6t_zxK3Ni%_tt}m{Z+$9zVp@+`
zpxrou=7K=$^y1E0rV>CoJW7FfhB$@fT#ltMHBtpdE6-Mb(1t_=zRU5q_k^WN-LdZI
z(PiP?sQO<<Bbnh|M4uEuAm3hOGDPBw%qdLp&-Y$fK19zfB_*2+(517wNjkDf?=oEV
zU{xnHY%i2Mp#q3Wual3oq=v&VQdo@_ArW*wjs++iAfu0!;<#u<mTMjpJsA>WGXFa;
z>Hpq^osms)(?Bll$IHNp9)mv@_Jy3X;XX3B1JT7gZ!`z5_qc|qsJ4gqRw5iUwg{~k
z(uYq{2xJWbvJ%>7TreQ;7>u6|X~@*}BkA=sZteS?Tg<Xj_oA(_qDAn}nzRFxN9WgC
z3n9Hf5J=Q<;<S&O#52f@g-ujvDrr?(9q*lcab5GL=+zWu`qzND3IOT&b}Q6&0J#N0
zlS5!F(V+*v)iKtLL^ANFCnIrnCd+^+Aplk@vpKX#p_g?+{FDr&H%RXPqknF=K3v3Z
zNM2eR7bDd8u+U+=nv63V1BpGArC$fHUh(x(poz`YLnDz*N=mBHpeh8rg8tVy>ftUK
zcpA>F|G7re;A0tE3*qrwlO-LRM#CHAwz+`@53wiFi=5a#XpU0FoGsTfsf!*ZJ)G5)
zh!;J+-YCa}3`_=4xH)Ot6BJU&Om`PKIXC2MAIp3XBV(1&zUBl<)7xvbkpkwA#AnKh
z5L>Ut4-2B#d$ghiEyoUXStpNc5wQrIX6Z?WF-DjZq_cL!yA4O%spLxlAKOkMBj88_
zSfo?Bz(Y2I{}Q{LMpVJ*6zvAYu-1$RhtHhj0j8qOjdFl82a+DEybF=F9&tx{wV!-e
z5<|Xp;8%KK24#y@0$nN7)`f9=&7o^L%)l9&ul+=bR#cBacM(SY>@z=R=*#MfeEPri
z#;ugfe<4CxBzjUxMyq!%Le8Q}soUZRaFB^rfkEqyJagDlKk`0ylJOi}xw--fJi?C9
z38qzP!cB|D>yTtsWFx$4KEN6q^S;yNm{ER&bC@Rt0$&%<4uG`(fJv(KRl;OHG8Y5f
zIizp*k!WcW71XB2o1R$rk<y}3lUjh_lh*fZ-N={~eu5OZ?ubi?zT{eiVd#5}nQ4od
z6X%HbA*Ra?pWUyS>LOJAPlS8Gt>jL<%t;hGyV~nq`FgUcYrVgMDlK&XFEusU`$5_1
zXIg5G&`<F&ZrQSLoz-<PF})840=!DeH=y_XJ~hp!Nz8G=Y+BBm!F|3FC=10A)*&I1
z6&-k?>N3~>tr_E5y=t3)amWL0UN@wO9h`!%I!^a+!yd>xP!T1K(FLAbws)O6<@m)7
zN8d!jT<cyQqzQ;P!(NixXnP2}6@Yc$-)d2`U-Wejhqh;c)?{~)(9nq^BQsE$)9l^b
zIEPFyYU!6awScvcdL4hgIIB>X&G{GhRFuar;ofi%yso&ubn><?>+IKpsfx=_$Ka6k
zU!=lN;y&z(i%=pmovy@@;f32N5P~5#y{J9AQ}W=HDM8RdIX~$AAKf29^rI6%>Ek;a
zW4zq5$ZH*-F4d)Z&BFS-EX(t7ZP&DeGco1B(i-U;0-GbNywNJoxF^t+>0AvvD%$4`
zEfQPk49ao*eKh1g$rzJe?HR<bsT%CK*zN}Btj#y8+QM{*Jj@Y2RX1CQxjA|0r!tI+
zlCDFG?J3eRhsO6HbPSM~SqbUONeH&~138Y+EmnV{u0SC%uXf7f#X_29b`pSKj(6{v
zQjRpg4B3B7yp2arNFekF9Do#g#?VJchC@VvZae^;_V8EQ17KDx`r6w2*gQiXyC@dJ
zR}Ue4Kx%V21pGU-kqK3ifzb0G`MJA*4Rks8c6()-Ei|P3+!+Hdi6J8zU7nlk!QT--
zOt*&%<4fPN5|#>Ka(lMTFeHHZ(RC%Lkzk7BI%M-V+239OWx`tmGzDf)TqOiIkMq5G
zzWP=@d1YbwctXklWP^8dQ7Vim*dDTkqjs+=@B9AA64bf~2;H;S=SQ2_p@nC4B`t4F
z-=Rfg_Gb`phl@kMwdd+{Xr5PwnM@)kcnJEd1YNkaq?zXf3lC?ll%V(f$+{~<JjJK;
z><ui*q#Q4#mro(f)m4F((8&28c|UotXT_`C6(@uCqCNT;4a9VqNO&||w%PMb(9>Oe
z6PYUN1>)6<Tk5aWq&I+TtqAR0#tM7Sk%kw+0u~jgfRHMT2oKCWUr#S?)gn4I9}9?_
zmIB}QWHrJ#CZp!KN3yA1aSR%-%VnFZQWp5Pl0ftOh$ACEB9PUbS+b~wcxoaQCp(W?
zYCxrLX2CVXewMJjjWL9Yj2vrby#JH@y?TD1*aNqwkXVutUWt$+k-SVTng9nBU%%e9
z-dC4cizJXZh=4?;qrH|$CkKw~DoFv4HD_pM*}<KWlM^>!P!<4?DLB&-T1eTZTff2!
zwaD?nxY2PM0ZJDglEm>e6nIYgEA#v!z=7Ft-9h1ldwmC>rx6N-tB;rJD`V2qxWL#i
z7~)B_iXN0r{g!KsgxNo6fvGu%cl`~3>je{s+fqP1`q9pq_8DkbN}uizS&p5dO|Q5l
z$7nok2IJ2o`$_eJ>M#dgdP(*lrlgRWsa6$5nbW;$zC56_)OU2b9sLWTLTb$Zk5k_2
z1+B4}ovjdOrq_?kUafn-c%8LR450!)w<=Ts_TA8{m!)4efmIp~$b(-mI+G~h0eg2K
zVR%$~qU2ee=Z5Aje1;{HBs97Qi>6Dnm%0VVzQ1M_-3N>jmESgy4~=xr&dtv5HD=o8
zWvSpDb%6nbFoa;}ZV-9>`)we%OJ*~rbcQHB2n~w7ae>`|h0KRnR3$Vdp<7Y$0~tJr
zn89FP*bgh5EnW=6^}e$gILruebyhdHmzS4+xV|&3KX&SM!(B)n6Zi$M*K8w0896ma
zaD8M9cOS`zzw8<>=|Imum>xzp=%n==&vLV;E0a?HK?u?Q-;#GjwE9*d_VnA!+kdKm
z;7o$1<ljOzP%6FjNz9WIC;#G}+E7EO-KGBM@etH?2vdIZmx;ZcA?E=~O~I<L&lD&R
zDyVgit8_!xghNzVx7U6;_xl?SC6DDny4BFq7o`C91LuRBoTY(qwKc&yLI1s3SdK+z
z;@rYW<9V3YH)mSvcm;{;i1rL?_#+BR|5(c(FdZ5W^N$y<Hri>&v~+4|<=#F4cwr9w
zyKZ2-Va^kY7;wYxql@8AP^MouO>(cf>Bq<cApLu2&q=LLWY6p;Z6z%EieyquZZo*2
zpO4^fkO_#NADH*+9xpa{Q|Xsih>Sx5p8E5-YV$ww)N2TyI_9WGzk`&NS0JshK|{Vb
zyGnT&HDyBRNIDkF4OH579bKq8&(a%m=T89$sS_bNX@gK*FA$O6-TXvc-x_UOeIh{`
z-3lWZYctB|R*y88fiGr7(#04=vxD!>PZ`(Vj<y_&@R2?Ti(f|$TQlahic)_BSc+Xp
z2(P%8gzd?|eiX2u9{W&;zPi2;DPorr4a9)O0y;hC7scT9xLBHV5IT`d)6gt?Pf?*G
zhgbFCvQHt&lr;^9ALQK2*jnz37}BPw#GHf;UvRrKH(l)*73ECw_S{9ut)SY8!_89;
zQda{JL5VLyaVh@*rMxl$Y@BJCjOW{h8AA3z+qR~{NPS0F98_u}P$Ex@Rckw^+__rh
zR7QB4ieIj3m>i&59+O2BCi;kQKZ!{vx{I!f+K*t6`TBQ<5Tyd(sU3>sjopt33cWb*
zM_a9>rq<`2>JI$DPehyhjO`T~ynKPY_*L?96F=!IOGih?e94XsFu<z@kYaM>OUGeJ
zw|pJgf{xFCuyK;%^G(jRQdpN6V2~)n@+D>$O!0!04a_dwU^;aYiyhyDR5g&;(iwT{
zkfF;;V4@hU{r<&g_;r6{ZZ;yWef3(<`_W;&9BVzgNDX1kR|&gO1(;+&nxRUn_!r4L
zE3pg0gc?DT^k~20ot;mxU-!Zg3cag^`Qe>1*b5tt;?ATpUlm%a?R`iSDblch7=UFg
z2FOXhb(^43bLfpG(2<b|(wt@o>=J}ocErlJxmfTKJ;ny+$E!n`&+pYeDT<BPDX>i-
zn(Wscs*5=V)a`fKD<`m6(+jWEUU=)~B?!iX?`^@x%J}5%U&N$f3@-<1mSG&71#2Vp
zDFJlV;sf5H98;jfL%^O@rV7V-@a+c-Vk*v{n^<g`(pkRoIF@~<s}HPog|h#I3NpoU
zqSRF2&q%=E(Qm#h5Ljx9GvDt@JN_%0_=wxa=vr8?Q&D&!@+O*KnD?P14?!3i>CDT7
z%qw5ba)J97G}jP0sg8>IiyJE!HWO%3=-EIPTZWKmX{;$l<+pDR3?VuFcml%II-sG(
z>db9UbdyBTXe@$H>3(`*vOq#gN}|J*ly?XZDIYgr3pi+WaXe$i>04n&)_v8_Ss6FU
zQ@KnC57At@bP!F5lciC+^hlnHUH(4q^>pm9qKUEBV^d@r-7V{>!}Nt80j=#3ZN6gP
zdu1ua^T`zz5%%P{LKy!WfGPb4Q+20Mk<1O>*f!3JeJJ6kCJAy|3SP}+&{OipQs}#>
z{D&1%^g`nv6fy`P?g9V0I9uQHU5&p)5_U|rP{?``6(+=E-}_Yy&4<Obpb0I<5NmEa
z9UENt7|@9nf$3-j446gHyhubv2vGFNi}3t1bewOtd+<W9j^l3E?-L$q9E|K%ftE#o
zPyx>-@co+r6mZopl%qV(#bwWC0QKv!cX#?P6uAeO{LCB%bj9fR%nx=)k+D~aGIHep
zDjp#iN(q(u9Do9wfQ7UA(y$^pmX$r)p;dFXPRu?(xZ_t8A*w`=*4)P5^?Xl>meH5}
zOO_(V&(JuA(%;eloG6EW5E+WzLBwoFh;17P+IfGB(y)xpZ}WT2mWC!fc=#rq52jYS
zi&-3$&&Im>*=8O@zmJ5c@(g$bO^pWb2I~O3j9;fK1Mb(+Wk^gXT96aNjB_n88ljm&
z!kLpQ4w=i~W7S&HV+tvd?IThzNtfBzwUV3_oxIB-ei#AgGDuJkL5Y!ZzKKB@{;{;4
z#!+b5J_961xX`RDMNGQfs?wStvG24Tx6JG2h~lCYbBD7Y(8%yMjat@FNt~h!OSs3$
z%k73nA^5t3?GeJ}C*2>a_J%1?%Ew8_?doy=_2>=u$W!{a-1s9}_s=~sL)B2{P4v=#
z;F<&PnS{K``^i(C*MqQ8dlVmI;xD$m+yqbSQODGUfDHg}9z%)KHvZETgsvD=%kc*Z
z-WzT}Y56ngnTw`a0|=kh^=xi#Om7YVK#~()VuA>;L`u3B8rl_7qd8P#xj@G?tve-%
zr|>x6U7wPVh8$8-`W{fb_*=fO1WN8H7<Ufl>Bx+HhM}~K;J8%`#cUJ!o{pSs90&pN
zx6T?M{anQrev?@Vwzd9f#a~!8H{=waR8A$yM*0MEbI)$@+j-QOP+CGNbkoN=>3<x=
zTfJAz6O@ELReN@_Ycppr=j^xVJ9qyaV*`xK0LexP$0LxocikONf19dUYk~Tp=Q?Se
zSvK==A1mXB$wSBZFGdN3f#rLf_Auz(M_oUC>-+IhfD(1B0M!5i2Bk*yeVJwuA{a=^
zuCt|4fb9TbWbly^&)d#<0L$cJ-&kutE2*;_Lgpc$2LIZFum${LN<P%0pCU3J#-MAJ
z?mmJ5`LlxoaTyt`q%b?$(j#a3=4$#$Utix~2QIiz+*JwL;G*vJhBCKPvqQ2|owjb=
zWGIni1kr!#$L%Z&OufMyVt2xf^q(t+bn}gh%irb&Fs!%^8_BuSd2-D*atCX`A31?a
zzDU{NXfbR#`I#{W3STxr8Xt;}N)+;JH~?c3p^3RJz;<V0{4HeNb2~FA?zLuiirZ4R
zQxjAhu2+EJs$_o*|4^vMIdMOu=y#r%{P0=a7P-~o?LLXS^P!NQea#50eJLi5eas1C
zA#99al!0hbTD;g{amFG#JI4b545~LhXeu(feuC392_e$kPAIF9p?=|wLG~i}9<EJL
zb{LW4bD8_r`}hsQXsSGZsByBG1-8#)KYs?BN3PVRf~@<L3wqtPgEQHsF%?8!&3|a<
ze{0I)9vs9-V+?ur{3ZGM*P8`j;U3VLc_dVxIFK)7hj$F4D+-{oa_FtoLN^^(dxj=-
z|5+;OBIQ^JHPHam=5M))lNH5|6LC;w8FoWQm{qk}9~2!7OL(AV&dK-D%kgK3iqyvM
zL;>p_9e`^Xb&boWAbKVw5|4PiZAPLq%**85?#ge}iwO_sM7Y{;s9lFy1B1+vx+GY!
zxWY1<A0OqLWZz!|9_dr4t9s6y!`eR&eli227K+aWEfZ2%GNBXw7?|J<Z+QwS)xFCB
z(BOt?tKegTvv+~8SYw$NnRGJT9M<o_=PG7D1_SGs&!iVaZ=AzSqBL#=>WJ$l**Ywx
zxH)H6J|g>X?lWu)Ds00f$xG2?>SBzr{6Nto)ss%VwOVLz>f><8B`ZEP0{s@-kr#d)
z<hnrjPCD<JaFDRabJJY}M1UsU?JvkNIS)me4uD)&DCl1)mE2uOf;6V7%+s^ob-IMn
zcynW^tft!u2Gr?_0Es<N2@nrW(Mi6{Dng**20yrv1vn`(m?fiMk|INff;=rOS0i((
zNBVaFXRM7nOuOL=IRwm@b2cH#V;D@_=eg1xHvqWO0|-@srqZlyX5GM<nXbT*s0d6j
z898nL7(JK4Lycji#VPSr==ZuEtxy(<MF9#tU+I7!Pr%E`NeUxJ2CwIGkRc7adlce9
z7(m92Z>9E;Cdu~o@yira*h;<EVB_n0z?Eh9cy*CMAJ!w7gy7}wX!7o{M$d}%#(ve=
zyW2(D$qTpN_ZO7H>u18gGIl>NG%2mDt{|G*(E=FR*a~<@_EbN@#D8m1=>&&zr}sKy
z!%{HU6bx_Viu!M4LUm(xnyE-M*@f|KylR}}<?Gi+H1)(LEkxXHJ>E1U5EmqQO>Z<#
zxrW0Eb`YOU^;ZyEs*mKa$bRFwF&Ags?SOZ&TI!eq#1+vML+FjaPO?a|lbR9L3HLwn
z*Mt^;cxQi(wBc=_=;}S!48dlhh9iJO<yXLDw=58L2cK_km>!D3QtsaMUpE=a4EJxM
zL!z~-MEkElbn~EagE;;AXV((#R5Dc5@$!3EVzMV^rQ@NA^3LYRIXTl!fH0L1CI$dX
zv8K>dKkot~j7cEd6l?rb$m}TK7zXW-oUEl#)kLvv#du&A1X%t$1&x6dNY@=oAKs=H
zl7$kNyvz5{>0lsxuzhb<p!XQ=aWKl}4xDmIda)HFY<IXD?C>vVHUVM0qa7V2%|SX6
z2Jb32t6cCCp97L@?V)HT<ZXNqoCS#xI=Ss07K@5H98ZSQYrZEYL+jC(b9Na27!x|j
zOt2Hr|GHfxe<ZToGMnx8!;pCX*XQ?S;%(1{1@+Lq*6Qo&^EK(bS5{zc$c-|ma)v~W
zg|Rx(uWam1>Ht(N!8{ypEiXIAuS^-fgdy`vV3r;!dh^*0_)#pXzEVK0LS|J%fpzih
z_!V_^8U?oe&5c#rxLN&<QGgnsmTfF#hP}jF@SV4R$E9RaAIlAmrecJYl@L=;$F0Ri
z!eE1a?{9S3+e>f^!Q>DXES|y<mTRd<Gka}$PBvX&ndPa1oN|NvNsXpVA2tKG`9`^y
z?^0np;^M}PpH!Ri^RpHlQfo6lB@<__fYt({Ab`vmLU)uJZvY7JGoV0Yi9tOHdh<5i
zpA?ab<EI1*N$ilvNX&B?J-$xI<P(v}&E!*)O*vPXW_xJ+XsOVs&qHcEi{=%FV$NTh
zpPPUEM+p^on-b21qZuq<4EXnz{j<iD#RU3K_E^U%$W6FqSKf5*);2m6P<-b_W#M#p
zUN$X`fd=#;REz|0kTM{IHX|rK!zfmPh_7J&*~`diWndkEPPxS%SL^nyn?4|X_wiEZ
zF+13r!JPBc0MfjJG}%alq_hMYsZH<AuOMwUkcr{tyZi~nA7s)6Ua%-%fqpA8lNC0e
zK*m}V&Ksh$gp85jym>Q1#W#1z8@ui!b2l>V{7IJw17|$%Rhk}1QvDf-R#2~huc}{U
z5UlKt-A|movjEtLJ?Ctu=}q)Hwr6B1QEa5>7qw^p<tW@vE#Ikn@m8-DrL>IS#h1AI
zEahVIzFTrMIVFiCczhn?OFymp*<<NeIC<JR0^BSq1IXirQ4BYVBEy1>bwgDE9*9-}
zh0%nrIe1J@H7{@984znNU~lMG2)n>2Ue``V-zaDF>2k;<VH(AU5>qp3^hdZhtp6R-
zopPs~mh?l(%kHZt+m~0a;e{r=MtgO4Dw^~ObQIoEv88R7M>qz$z97C$6oxCe`xupZ
z>zB@(Yve3SoG#X;;HU6Js}CCLVOs-|)=<}mg^^7h;^Bu=g3n=0eYD}nQg9i(r%$2#
z(VQ%h3LHJoJy8yaMi=9kUXS<Mtrx}Coj@<iy|@TAF_xm|LkxGw3KV;7JfHZ`#a!7+
z+A9SiPklETXe+t1>|XjPH_Cx<u7&Lj$6nu)FEu+|?fWIyV_%qFW|TB~wu?JAotnNy
zRYpG&K#_B30KIAGr96rgGY+y|sD$H+2SL5l@V)?1Rzp9C3qf-XJCRXXC_3^D8TSFx
zCJ}XZENI^2qKtkwN2>h5!#UYjZrjcyU4@igOko&G9bp@`-L<`jqwI2Bxx1mhy%lIl
zLouOy=)$h-mYS%8eLnHn2$7x4@}9icD(fcC)ri|6MY>5Es@6eFF9X3JqTR|R{3fh#
z5?3gsGW@1>Phew<EF#V2;)Pc2Er}~oew+Z-3c_zf=DHELsY{EXa}fsr(+mt$eyPs^
z4f;ETa~kGLa<WnOvyQ)qE{0VtxmX{#r9?UvB2P(r^04qO|G6(4OFAXtP8)ys&;6X^
z#d>2$c$qnOUt}~}If*o(wB6zD8zy~7uq3yk`{s1}So8gQSEZy|+}0r=S6l1R>MEuU
zORkRcW3csqa7+S05;fnfzn_7gHZtJ-&frVn<>?v<SYKb%BoCdrXYx|OWEacqN<rkA
zBWhb0H$%amyBnEbP{Fn}eI;(bEoN_yL>clC2CAK!jWn5n3GF-kuF-7!;$|)(D{J$~
z#P#zY7RJq}w*KZK*;2;G`TMoOqsJHr(l0egPY#l255?6HGeX|t8SgHbtbGR^=O>FV
zC?U@;z}95T2AA>wQc!WdZZvf&paz|D7Pc$A;D`1z?E2x&XkySyv6B1Lwk?ZZ(80yc
zhlf)C_1LN4v3tdf?3CR1E^Iw^jgi~mD$-s`aWwrt(8F;x4A4=3uJzAF`;o=_jzTcG
zSP9o^5Y2@iga9%FTLVBFJ1;LUpJMq(-+c&`VD>Y$Y9&w>K!95e_$<%%3W!M$JYi;e
zI$Nx2vuqEkLB!-Ur$|iRy}k{9G!8@LZMZ87E4lYxUr%#I_RCv;7VX_RFElPK|3Nf*
zCv;PK!Lsi0s%72|7Z`O^7t^v+!ACRgY@GZwUgiyc&asgVSGC0V$SaRhoAP)b7DS0z
z6j1PLJ(=vx3onv936%gBO#}3|idgU1mM<=_B(4P9)_pEQz??6|B8{>8==*H<K@V;{
z=z_YCZNH;RLU{9S`&jG{=gF-T1?x*XBdz9p`ZC$}MH+=|ZNu8Ej;UVFf$wruPQvOr
z$Tr*viL46cq*`@ifBlCgirY_h8l>uql#|w<26QHoxijhTzMIJYJm}qX#Cw&M3hhm?
zH+a`r;5Qqkx2tMgUG%L#)p6w!BnS;%wq@D56t8+D2=Sk8*@;>sXe0<d@@-#ikYuUo
zdDJyj@tTF2B;dAocrh3I_8P{!AtA+{k*z7|%Iw{g{Rfxt8qL~Dhm(F-KS+j>CfQT5
z>y_k&tj0@KQ<a=Qf70k7<S&|bCqxcz?{aLpDWN3kEPce-b_I{I-+Dk*AFW*HTUabf
zIF=d*ysAJBF6PPKuHZK6TcJOLlb=5a8Qb$xXFa2#$5o2SrmkBrp1i7=jrz{-kN=1q
zZ<;@bm3_A_AriR+M`@8=h5K?L<Psh}3*5Pcq^(Q%@$SQ7ey6O~g7P=BFZ;vHL;TZL
zC39w<Z2NNI22cwJs79Sj<P#~XYHkHs_Xq^IE+t<TWK2G1<0<2f6-D_N?w(4Lq{vS7
z%T7_;-l+(XmsS`Zz4Jz4`VwKMF8eFBY(E@IMZ&3&wW8bY2G7UjQ76q01#lhV<(t+B
zJ3F_YoxDxNe5V`#<0gw=9B)&?bNS@r?{$KY<tY4PrMK3>Uyn;~`<4iBq5I$h_y;};
zF#U-*y#O4|#6%G7Zd^?k2#%Vz%p&C5OZVzDT)@c(?_^}#ja<!`!s>*Qd!#R&5s>a1
z4`Wf{@Talj{_tt<_7D40zq))SX*qyp#pxO~`f`kb^mO)%FP=_Ux;b?R7p3Y}D{DAh
zt@qjNBSR4y*zMYfm8))@I4&#S|JZ(D1(6en{92gZZs_r*tV~hSl6JncfBR2WdJvW#
zpPN2n+F3HZ5hGhjLG@ZM)83>gH;8Lh+33qKOR7oGT49=V60WJ`ocZ>0t77Gd+YCL!
z+&cpVL@&r}wqDK9E1c^p)ie4T|FbM7x$|+;G01zI%d?K`MC9Eopd`Hx56kO=2*Ykv
z(ZbyxPde1GPkP%c2_ggorSNcB52wgoucch&p3B)d1**eb5EqTDlR5UUt@u=?JY{)~
z_rI-u>Yck!W$;ClBpmmbL~^r8T``$J$G{05fy|;n&$RBL7IAh>eeRm~(!&ca%3=84
zp&<OsmIE;X>JZzF(!jJNeYvO;v$sSS$1RfG!o*|n`06iG%ODi1(cD<3b;s0`@ld=X
zci62x990<^@()*7a+YXMwAHhb2ynZf!`?X+xT!1I%|_D2{;p+>+xX3CTmuKGQGbc4
zudc?)f0)kUPVXJzv~`=K{(4I8`R;Ct-}5_x%k0iyu?7?BfEwy=W38hT8B+{<lk$m`
zdV8_Zr_}N_j9Ra-*y*3G7(CS!`1OFLH&*t%8q>~d?w-$>P!j~sb0ZWM|L14OPYpzP
zNfH*UYp`(FpHWp2R8=+Exn9pbT>Q3md5J-fiE`8DmadEFhygn0_DjTJd)NH+G=V-(
zh2|TxZP-Zz{`zR^t4ey@3b%4Xc?-+8V4;_3Of7ud=MDDD-iS-6XCAkxh+=6p^IjRG
zrsPt=rujdDCJAu8=rKWRk+jT8>2BFKwB<V#(ZNG)1quVJOEN<tl<isXi*w$32Erdn
zB9%S!k6S^aCl%U_IAuHK`A5o*VaHXs)S97mWho@jCpDfiQNtBKaZH5VVC%g<OqeQ<
zyfM8G&n6Np)3oOhJ@bONKzca()9x9&9gbl6|LYfIYu)6Zdj+#KjEwx0!QJ-@_%4QJ
zE4$SZ^$GW9(^LxMy2WH$d1*EI%Y@I|_@aD}QN-(nC30sEV~PIJC?r5pz@0HLaP7vY
zzpi8JEBfQ{iDWbr*Ve9BScgTq2C8RTc#Qe0r*mIUekMOS1}}sOoA=tuFH$Z~zsaMQ
zS;DSe@oS}~k1Tf+&{2>&CU|9FA#eHT*|k6OiP$}4*S{=WAJXuY(f{^%BT-7WsCLcj
zpnJFO@6HgH+0Bf}FH*kR$}c_Vve&Q8WlI}%YK47El@(0q+GxBR!FRz6x-tbc0{>L-
zx7C4<7!DAqY=_-{K0?4F6*(z;$sF5eek7-QHNo9sY4?6E6Do{hnGUx)nYMeF{`GWE
zUWMNpxR50HAAjASpR2UR{Z`4ccm6iJuWl4<OchbTrXN~~{qh)1`6&9&e;1E*5?)<b
zz4agd+|D1yanbotqm}&dTxEUH#~`_9qulHn$)*1P!)2my2<-v}pmgflN~zPX`Qc?8
z@~_q+$9cI@{Iq-~Hw){EGn{ZrYp#i%FG_lNyQ|;*aTh}l6wE~gW>9{--|zpS?W@D8
zO51(~HwxwkBvg<sASEHG^hQZ30TpSLlv0t94hsb7mXZ{uC8SYNQd+tjNs-QTFUpy3
z=6$bouIqgNjEdvlYdv|#FSb2YFkG<mpAIY5U;F1&K;4-LQrO)#-&+G`9*HxIP?zy#
zD_*$@wiy#|g^Gxc?OwDXIe<^H8q@>SbZS7Lc0m_G(J}aykbEexvW+iX%-)xStRzal
zjrE4)BjdXh2`+fd5F6HV9e%juZN;I9Ea75NF7!XIKP-j@j{EB3Dzn~u-4<+iJ+`MI
zzTfA3=)_w9rq4}wsevYfZeO&Rrzo=I>W`(?mpbiN%k&1=Y?Z4*j1IH~NLrcRK}{fx
z1WALC5z2a5q?KnP-N82{*eG7Rb4Xw@SCD8e<c%5Ew%+2uROmU>p#;IPhFdSNGjjOI
z57ubPnuXO2!SGg*Ak(B#+?*CC{N>A+6Qa&Sg*LoDPcDBu?L)YqiyZ^GM~1A#v8B5;
z6cW6o$ZK#wWVFGMNq~Dcjpb2i8qMz2bBz?@pUUWOJ1P4=+m7C!AbR%yMD`+n_kvJG
z!e=i^wM}CrV840}O(}*HAr~A%8PKx0t|=Z`%C~NnkK!m$KwuVi7quNIepNR#+%+Rc
zHLxNRlX<#+$J82L7v+4U7|V5hcVf!PAzZBtmKM3PGo6?#TT%Pv+vAZe+~CHWh<8_l
zP|`wQ{J7+emu8TnW<X*L*|Oxaq2k{3mK!6HmmUu$d3(cyAs`&Zo2Tmbd}=Th@Omda
z$3@TY#>JPAi1fp^-GM&*@fEfATO#<H*e`GrF(COjd=10zAG#8|s~YnGUUE7_<#y(F
z$J)7?7;?+sDFiILH#Z8p{>NQ7e&^LO?ffIjNF*O>)82WV5Y$!9m-}K4`nv^(!$u0(
zc^ON>w(Tkf#05`9kp%XvPi)BcEfFgVgO*wUXpcaL(LvrN<FNRnop&j*Lf(aD_gy#$
zV)?eeBMA8p5*QmOj(Ar-i`|RQ(YVNfJ;(acZ3p$<H<4JSvi8$nP=qFcyv~)CRCard
z?{mg(Er*jk%b}9lyOIId<IFWZO-t~_)qw*6$sXIpbawRnz-yRj>pMh!^>)4kDH}EX
z*&YzaBi~WZO8$N4J8*iCvXK}0pYJd}fg^sLfs=)w+6l|F^Bn<)rS|Yh7!-069dV|X
z-IjpCBqI|a5cy>94nB({`)Cd3&|O}TGyVg|K`PGdjW$J2D49uP>bKs5Rn-#q+yBmc
zoa^P@nh*B?$fN(4=7Xh|XX`s&2ORp}T32R=OTDN|MsbGd;$U9^8FQ=n5^HvranGdc
zDRyh^UW;|xS-Visbk3Z8wC4(o5k=N+w7Zzn?mk2#Ygc=B?H(d@r4)0j-_P{a|LWFz
zXp&45i9PG?!0o*3f^}-8*hACIC?Q|v5Fuo-J=fkxVd{0h+!&<Ty%?M^sL@!GryOP2
z?KY|N1u(q}aa4A<mES(om_#vrFP(1k+-+h&8Ya)0q|YG%OE3~O{{9g&x&01*3EI46
zk?&~yF{r-#9XP^xqp?E?tUC{Z#Py|zGK-lkD|m&+8Amvzkq-<-nV-N3FAC<Qmu2Zq
zLw=%1@RUcy$+ouo5UuLo;O%|U5Gh6^s`Kgfy<B3@6)N>Z<<Ew>cK7Gb`Kd@)r+_lQ
zJLhyglL09{W6<p=XZ4Sl!uAgSKc9ghO?WMGc~y3gv%fTt;p6bLuyF!DF6%EH4JTO?
zAIbatwD>Mh>GLj2>|GO4qI%&HeQ~IyLqLCIK_!&4#Hdr_k0mGgItWnMbARyB)hK(q
zT@MmFbn9z?N_wgob)NAgs{>2>4T03u{KJl`ZLAqaFP7O;A@by@(es%+{k`^pPja!V
z@v@XoV`V>8Ra2S2C2~SI5EyKI)ZhPuhVHf#c4sDcZ;!n{a@>eAbe4>X`7VFZ=lKh;
z=qfQ^VskLGj^_!n+`iAyZ%d}asmICOlS05WS^Sq59q|P`R;&xQ7Z%JvJ`eqX1S3)s
z@GRgTOJI9EVM>`^({ElJXEK{PNypwB+@G+XkjO}}<Gxu(FlvB%UWjbjG^xD7?Q0DB
z=@jkfw%bIg&(>WA+`D)3-qh$s!ql``$@0?thpCCdcIi>Sm?z<VoA2sLQL;VM`*4Sd
z|CU|ZS)^Mj!U*~jDg1qEw}uH{)CuMMmtyzcYTJ3m-m}+=pGmI1|Na)I>^1I+56f=O
zdp*fXSDZMQmSen!dYtrz@sB3G!`_(NN*DgQaPedR=YIeDgOsZ{pPHAwTe6k&TH{#r
zR+YGI?_PAU(+%3$NdrHI8UONZ3uj6m<zrBGN>RU8v!{OeXSOxqRb+}UXYPd&dw;dH
zFEeCH?3is!m%mov+%VK@%YC?GOZgOgdnB-**4a8W<XdPxUN93fB!}+A68&@7qdH-y
z`@i;v!}Ra3u=7Xf6L^L^=&_aKKHth>KR%I8*XR&)SQpW0yz2AasG8Wj<)v{CCPUj#
z^3B-Ht&cZEt0vmdDrPF_SiQ0Cy0NA`$&)%fR<UAu?}cReX0o=<V1S%2Yd<uaU?nmH
zIXO_?i@%zlsy7%WKq)wq({goLFWc;wyaS2Pn&EK5J4gwsLuHg9H_XC?Z7OCMTpX8m
z`sIlU`V5K2eh#1g{(X@{jbe1&+eHzk99UjMbg_HE(4RMgJ`5yEXmO^!knMlGuP^>Y
zPaEyQBOKwU$Img6=cw?nvh5b8UJp9y#w!(%wbt(J{glJkIS3BePOxS4C=4e&aotG%
z5$n<*)6wE?ch?u?5T+Vh$S_Wgb#Xd;2%1lSo?`oV&Rj^j?11^PJ(<40FHv5)2U~@^
zkOcbiMqj0{=4!HKn^=H#e2u}4{doH8_9vFB;82=GZPo5_`tBrh8uWMR9}h0hk5TM-
z;d>6=mO<q6Zk_KhkAPu80*7bePxgwOf7|)k8hi@-EXfY%#PAf$YwBE@I^FN7Los_h
zQJ2`@@yGJMbx~DwX+e=`_YpBYA(iZb<wE7;=iL&m8Q*NxoYde}LBGDTSBr4is`uA6
zCYWw7zYGy4l(X^2mtlMEo_hxb+!!Eypg>w+P44x$bCNpQCcb%=gi|zLkUC>(wW(5R
z_g3^h+$1a+J1|fEijS7<{kbA*C7oIO#DJq2O^`%+Ey&nTo8Xw96~SPQkL@wLml%=C
zy0Cm?-uC5EegMvpj6NnaXbAn~33iU-Plsp3RAz?n@ZWt$bI@2#pO3oxe1^GVYmN89
z_RyivIO}PMaXm~GUmrB7DfK}@W+@l!Y+dx9KE|P~$ndj|_)cUfj-30hwQv5%?Y1?e
zm%+DKcSEBf<w?PKMDXp`i<+PA^JaRLr{~ZLPkAYc&4qDEIU!D>$ZW2W)D7+{tTkw0
zTz<G$%MU=(H0Zu&^mTK-y)<}f=e4{~Be7%ktHYmEwjbjfa{W}AmPPEoH`Wva1{G6{
z`!(C=I`FxWw>OK)2*l5ygmSH%y=bhW=3VFQ{syD=Io7h9J7>vTEDnJC6@p$zo+oO7
zdbiz!!t#t-qM*ZuMvz!sevLunc0_&l7}y(PLWqO#QQQCe|DoI0(QJ%qSmg$T)dKs>
zFMWfy&{iAJc+Rr(4!x<^sz;)p`GOBG_w4q1#_y{sCJO$44Bn0<Cog}TpDPnRwfU(-
z+;6{7fr`$U=ro7%u7~bzRA@NHtkSzw3b!i3-6Q{7CjEDsjqM)T?azTRdxKDf6!sgf
z_8;51X3LRKY#A?hd2dQ?Bk?0S_raZv%h?+V=X;(VdOt}>=4!rXbT>$TT5r&)^s@0z
zZQ>RVrBv@y@j0R4rWc~ag`pokdep8UJq@QG5$yWj=VG@K2}zeP+s}elg|%?mcFf??
z?rn`uLz-_#YWj9gF3iAIkDKUOeYUv0k3$TENpqGV^O@xn`h__k@OBnt15MqOna_f<
z4VCQ(sZUESNn3VJK0Luzvi$$;3AS51b3Drr-<8|8gg}e&<ZibXB9&_8_%14vdkqz<
zOFadQ9Lqw}lMeR@w)$HR2XW^2QY9P$q^p7<IJSMUV-1kOpNxy?+Z&`3WbiLsda+Yh
z14v=-MtQOH37(D>7Ezl*CPl{DF40IHfp=Oqk$e-~9L+nYDfT*6V*k%|U~c8fLSyyx
zh<lRSVD^tqi><7&Wb;BV{@!<BwkD#~`N_^i+(jnB?w7$Hb{hQ_nTX2opLQn#wbkMB
z`K;G5dDcfyqd@Q*T)pPLi<Wz1poh;j&eSj|GMzoT1#|F3U=DEJ(_FZ;$L{QXvej|F
zWb|$)m3$AEMftm`ii*9}DRgK?#M|rR^<7u@Hz$zSuOg-E*n;_Tk*+AbugbjpygXky
z-}p8OWS0`XZ(}@tU!1X#WV$iC)a9ON2cQA7Uth_}k}AHpAqqs`T6RBj4fs&!Qi}}5
zh3~++PQZH6i=DD~kn$mj*@w_xl!rO11;s<W)f2^aKX7a}{KH>5kDT0&IB$0h31%QJ
zLvI+BpWJ)mPe^z6LJh_4a{NqFvhL&9!X%R&exDS|_4&^r>iYJKdc4RAfr4gSivmis
z8HhvD3Vw)=rd9#SXLY2o79i#VGVSb~i$R^3<MB-I(vOrXKq4k1961X$*5sx^CpFXD
zuR=qw?-!{&KSpWD_ChxvG~^oqo6d333ZN()V2{W#UF*q<xweV%(PVqwSuC*V7cP8x
z@Wa8*oaJ1etqEB=xo@X;2Dc500Vy8s<Gyradkelf0mT;opjYC13-%rwa(k|*N0K{d
zZn*)HOXdsQNl#Xn9gJRtPQ*twX08spweR+)jR?yXorTx>jz?3bNpmJ^6$|+?%mA>h
z3c;!d>PFOs>p%g6z`JVLSPDJb_93C**SFKG`oZn*PMdxdIRm)drb7qS-vQ`n;Vn6W
z)&jNKgPpj?^bdzU_9}NyXmj@r*+FSdK1f=yKfC=#XYeTY0v&vDBxU`CeQ;v?fj^(e
zx?mrtQnKuUd0j~0jcyn6p4vHebc(jR=`m@oXCA*RcU!dK_n2bK&?`dLHh?vBrO>Xr
zfY*K=2FP~=ZPohY+^{XEbOLyjF<ny`WTHiXx|-^sobM74p<YsQ@QU09nso-uo!F^7
z-73<1by3a}t%MKg=SJD&Qi^#5WCp|fx(^Xgp;eZg7cGUe!<Id)lBd=C3Ry~60BGU*
z{mss<+XtC1-0Wi`V@emdAGQqxZ^NT5R<lcfM&n<`Fo$Qlr1g^T?rx_;NCy589cWwI
zNJV_Fv%X7&ct4kj6M-`YF~fZqCo@3qOXr!7Q6nfrziyzAGlP$6+TuBps9<U}B2=E$
zG%IK!en`E4PyaK(Ku59wre6+#RcYIo{;3Q&fcO^0$+?$%l2Nyl+f>cWh7xjK`(<A1
zofYyY;w#6%xxz7ew{B-Hpx6+D_qjppd<@~ez1AqwXt?unF-aBEu&M0>xZeb>hu-B@
z+Fd#6q|VHwuJULGCZ?C>3toal00)sqD38ECNASF>Ze!`XDL|yPo?!QTs`T6qg=3Wo
z3#;zKczw)^rX-cdhtWCRHt#cmh@g29IB(WK^`0CWtwwZD!J$vXoIOTUTLL6CpmH}{
z^=Kx9T4UPm{!;+iuF?90zwXQ#8C3-=#@y#e?Yo1!moR%jM9LJM=iZ3y{uTE5p{H;B
zZa#0QXWknLXT?Jkjo)lv{CHH5gO?izA#;L2uR|#$GMW=~inp>8BJ-6iRZ*YMq9ZYB
zAR>9MyC7G8fLo>4lYh#)czyD|C&ytjR(~Mjm8Y}*z#~F+#$_;yluRDkLNWJ#dNL^s
z!gYR#FGAIwU<f+Dq$ra3JvICF*r)PG*Xf16@6dSvc)zMX3jv&cDb)A;b|PW-Xx)AA
zeYm<%V~sL4o!E<42P2STNH_P0$DeSq0MD~fcCb!!s)NjtcD6ow3n?6U*q=uPQ~3+n
zrwAk~#@%PD$AGX+#}!o|zt=!!B}hgUz{+$QuX@_eX3;+Tt0i$v8(U&T<2$Fb&*`^3
zgOAWpC8ysj&+eVAZA`G<vo%#}cHNZrcE<J)uMv+f`<6$rbd5*aMV4Q@pLf5?ISZH(
z0iej(p}ztUw{k>}9)4ohZ9O7|P?%u65LIk9%R@QL=M}RR<E|oSvFHLo!tH5R5%*>e
z&~(jDxsK3U23=`9y+I0J!U34N#}N-EL}&-$Pa=Mc?{0`y-s@Nd;z6xF!o#np@0Orv
zv8@*)6N_+49Z>@l<}LjbU`_e(6aYn}J~|>5=se!=T<UOO6Y~avk45>$qmdefiYTR^
zprE5=&*umP#R(gLK(0#i+KL_G1SOZuWN>D^ka#mWlo{BBLd%212saxjl`PLxU%DgE
zWxyOF0K~F~b892Dk<+(=DqSK8a0AXOlO}6*#0#%ZgSj)D#NgW;UG5ce<z7KSO#(=!
zMz>I}#8eQqTdSwKpV0={bj^I56gFv>63~cZ00`E@CSLs-3dD0+JG|vHC~ULXm}Eta
zPP?eugQHD!dco~axL_C@)G`&Ps--}KKRO+noeLcdpMh}(a>UKL15u?B&mPZe=`peQ
zYP=MF@qPrNSz?(h%E@Q;yePF)9^uVS0I6!>Cn(NqLc7YCt*jPtGC{;&mVOWf%`d8X
zCI5Oq?YFX;njY68)>Fn+1bm3_G|#a1w-rvICq{YJXOqicljWLJv{}&N4^G65l>S+`
zyD`juytxiChYOekt~hs{$p7;%m;my({`wVE394panHMATzMHi_1iBWd{f1-YM$hJ|
zV>7UQ?~ss^ZWPVt<bsA^IY;<|WB~f9f}t~8x`r}h_4*F3=Bx40gUdMMxc0EhJi;nj
zS~=7+HBd?*O=cE*)grVeuNi_9f$_$p@DT>FtR^FMZ>}W6ZEh2|3C2S~Gatbzr5Si%
zfiuBKAYcLRuB!1gA1uKCg3r?eXb5yk!V9ZTa5AeQJUs2qCA>zQHk;MS9F|76OK~=`
zE>1Cr8q@jW)@6uZXbWpxpbS=Ozqk-)-fiA%)uhXJEhsUf)n-0V{n2;sW<<!`qKpl3
z(9MHkXYq?wjMom;Ic@~p)L%aWZ#7@~+4ey{(en;8+2=dW8)XR=FNP0FhowrTq{}E?
z{k+~JJAjyUUR}zMdd2Q`;3kMB_%)N+vJ?!%a}2D3gX%KJkN1;q;d53pUe*m#^eS}e
zXDyZ7mchE^7M%#g*uzNR4HK;f>V5^ihVZ#t3T%Glx4S?)rx`Xg@tI0mf?IHK=j!?v
zDyHz;cqG~VC?3;voSxUjP7^5RT9x~7(20~>U3fy)Q$@VLbRRqR2>PGDpe2u>%+Nv<
zRX)czmg0sM-TvD;7}t)H)1F57aD#UGpO#QKlTK5C6zIt}$QaX4#yS9_ns(h|8BuGl
zmc*+!^eY0E*H=)O)oQT$-q`5Z97lkwheC}5ze!S+N5KUPw^wq7A3c3CjcPB*QL;XA
zDM5%pRUU)rWTvtAC(nK21B+rmn*eT+Uc}I${($TfL)EI382uJDI`E{_8_`uFKurha
zWPWhG*S-lVnz^Dc)<nzdOc3H!T*cu;iUckKTmT}n5mKFElc2P?-k?qSpaduIWIByX
zBV|XV%}>TEk5He?DbW!X1p-_MV;*M85~B_rqX?d{XSO^jX!ToX<LL-^O}MsdR0>2I
zx;W9<8niXYtZ0J1SmDzVpHUl~?>w^m>WU{drEqQJORSu-MKT8Wb$OC)D*u;l<xP%1
zh_P$^^5V&Vnw^~~^QL64d;7#n;*%zuei?5d&CLN~l&^*456_hfI(sJD=meD<!yin2
z#EZl~94pQR(H7_>(&2+H2t)my2@_2qoK)DtNoJXQUC*5B>5>nb;z^NS5ciahTOral
z=W<4<s&h<6@LK$(a#{|{x}OJVS8xHgh%z@n{Hi!ZhR^0R5L_Y~tkpp!Q<qm%ClaU#
z+;Q5mp_r(OQ$R1X$Q*$+ngx8u;MlT6jr?qGt10!eqP#St7FHt=i}RtMu|0RZV9ji~
zTX_8{nq1Jvydhz#EOG`6GK1@w%2Ce9jSebVYIej*su2)e#>LpuuGm+(nXKe=ofm->
zUHR)GXUD=h5vS>{4=nM#RCh!FZKYyL*-O42{`Ga>iiD8)x&L~wxp+E1shyi*Xf-jY
z(K?=kat84qwRRj;{xdzW$5JSjl%WtD4rBwLW1|RF&W&T`gjG(h*?h&iADt|df@b7!
zYWR<N#yPlCvQ=|TO$5Pt<fejxPfU`~n4rR}b8{-Cj<zFWpC&<xgeS=Wd**GPY)$_X
z3cjM;#;q5|K}Ib-^GaGojIxW9P<@XHvX7F1{#kAa8zc6~0%CJ@Z^Sy4QN_h6q70ZM
z6!}KcpY|UYK<5g9!fv`@bJEH5)0}!9F=4D8l>E)}e4?qcDnuxog^7FjPT!@UcWqW_
zZ+R($uA}cLlLt~!`rN%uzGxSrI~rORDlKL5RQ7WU4ZrK;=D}3Owdcnvz{bb%K$77_
zAz||3IMtakV=A@Bz}pWL$`TGyVj{qOL1`I04=km!n^Y%%M8QQZHQq9y1$UpPy+t2z
z@jigQ*I?@=K$1PySWC&87hQEm`m(xGQ9wETIXwg1GM#Yn9tl(h5sEK1z){q8Z~YB0
zheZK0O1C}_&Kiba#F4=dAzKY%AkNnU3ot=1LW>5SXG>#nW#=3KVH0buBv7refNNA0
zQBTbRCDEnmz>Q;~<CSg7>m5@8pu&I>>3N|u^Gg!lYis2|^`GuHjP_lO_NmVY4Nk5(
zpD9jTi};rV@-L@m@mK6?=|cWPmU-+W!XxI==(?KB^j`xDa}!JSjYbkHtz)1wX4XWQ
zJl!D8_KJlxq(!c>e)3mp`x`3&aCq8p%mx)d4)p5L3aSO~gR%`koqPz&GM&UGQBHO(
zBm?ioZN6P(4R}Nu<8>$P`xg=W=;)+om1I@!ad0G0h_N$2tKw2ZEH6OFa?1M0u}}12
z4_w9{&fRLicgdl_^QY-Z{J0{-`@erEP-XsJ1x3mJ_c;*fQA$f(pWzeFm?sIq)B%5#
zh~Qj%H>~WkER(zq#MS1ZQ2R*%h1V~RoK(C^H90Y*`=I?^f(k;NwdYzsSI=&)mVv^N
z{p==xrk+s96DXLm77||T@Rq#oalc5rNH;+z;@aIz=j#$zTvMfk*5%NTfzL(bcFMG@
zc`$xy4A@c&0#-SE<{Ea<uF6T)lT^i;e(wGBl!+pa2M4NtW_&9F=H^hVSdTlx>}i=b
zwa^*Lle=iW+0|l;t6e6ajzfT3id%g6KOL0b?Cjp`NBh2T3Snvg>#lcouJ&>kR-f)t
zNSiFFzF%C+P8x~101{zDbRa5_60`p&5KpE3>*W^G{VdHbR_~rIdSabH-)TgN=m8r4
z2o#d-NH$Hi_(gzQUkUDhMAo4NEe@LF`Q~Fp$)6o4xeTr~@O+KTp9*6Z=|#>&uQ!~D
z6#5{U>1nTNKYyb(`zt;EsO!=8PnT9c*soejX#6_N*YBb{qRUpigv<|mQ|B=yK@7;c
zM~tLQ)=jFiS~eVO`(@&x;W_OOA@DRkbL#3hdQ*dhsaC$OVd8W6nFjTr!Sck~_yY@N
z2sUzV`esl|9td#p{X{%^iKbGesDVC`=?FUF0ciy-4=oBI5MB`)4wxf0QCf37o#=6f
zu<PQAYCQpNq}{CBfDQAlG|ZSa1(-pd6Hg=u(f*KB>AB9!>+c*p3dPZNwz$NOYbQ<v
z*N}N8|1aC)?{jmKc>l>pw2;8Se$oFLTso1LuaS@PmFLIYejgFi;}EIMEF@Br%xMjb
zQ;P7Lh&5HvgK?0jh>-!EAD9`oQ_v1}1jcfHp!>Sw<;coH?XqL}8*qlGL0Ti5r<HjO
zP8(7Aey^$V1sl*TYd5J60WXqzBm!+%pa9#hz-&RwtNxfeAV;@GHv<#VS5e=N5{KGy
zrQe-RpaWI}I$tZpnv&#AfxL`z&&#XmCE(dm$Tr84<cyOQPl4K*`9u%{j+lhMYb4RD
zI<#P*P`y23*_smi*e7X!zVdct1F{6|7GoaGEfB=|*IaC;vw!8dQqVGOG9g65J+1G8
zWr|@yswl^<$dWTn=H4)O!LF^w(CNoD#DfEJ<wI78Y&Ea0#wl7cRE_pn!fT~i4Q0!A
z3gUr==k%G%o?jm?4O6&k|JwjNGrQri>IAAw3ONmaIsI?{$6nBnaU~z+Y08i3#Yq@k
z<AwsmV;EiOV2a1R&Ym?1R$VtTo(us4Wtd^0x&AR+h35W9vgf94ZNKqe9_1CDsg$eS
zwlr@F4Zkt$d&&WAdV4ebHTJ?lGFEl+e522VsD4>+)`)C)XxQ}j&Y0xB`g>4MiC7Jw
z7r~CVJbeMBN`^QT0h=ETv?~<>mYBJ~^JKn|5TQ04n>Rs?ZM+D=H?ItF&8!#)V-2@q
z?hS4QtfTKM<EZ(M=kU1<o61!|+bHH}s6b437cpgrQ3q&9jlY#Xsx;|8bwmH%IK&)?
zFXoLz#kklrK%O)YvjwtWgMEK(GDJE3JyaGayb1~m{M%zI<kI~-1<*asVXDm#&JjR3
zhb}FYkn|LB+1Hc*d-XrA_YlJU0S@`-;b*&Tq5qMYL#-!qr^d>*(49psEN9UZNMi)K
zK(HLh=!e#H-;H={$~NvH1s>SfEo{TbTF)b1kA+E`@tpa=a*GRjIQtKjgEqpXZa9Qg
z?%<(WuXzznrN_Dp^9#ca^uR#nt~f`U`=|rZ|BK)!-)Qm6p-@L5hKT&Y5n!GqZE*)Y
z@ET$yj^}^0NmOI<8}nv~=giTV0}8&M<NCN3=mb<FLhy(&PGP0T6lTls3_ko<|5rtj
zTF5U4d53d)C|p}!K5ZhD4kYdMV(b;DSxC@YG@DOysy;CY3$7V!0QKTTwVV{~eAeKE
zMQ|hur;Yz$<7QHEGZ=-#P8m9Rcej+QDW^*(<S9F1m<T-CV7W#qtBc_kz{X^`OLTrB
z0}<5#dznv~o;B(q>pKfqju;uuFJnIajw%vfcmj%+RYZ%nG~DK^T#$xk2tuDtf3Fx(
z3@+tY*CKw_fOOp}h;{in?tDfeS{rLnnPF@RF%+?<LbP`^?10S7@t_xL%liZVdZpZz
zkY=JE$gtjs%#r3oD0_^9Xo{_QzSKKHKXM`snXwx^{M-xCjeBw&fj@tE*kB+e%vh{n
zl;HgQ5=K5AsuwU9>F&X_L6QRFyhDRcv3`<Pi;Fi@*slzBy?^BUbut#FzQeFA@6JQV
zD$3I;f2xFkeg_1S2xCIu1k>R2HCdnkTZ-;PE@^cnY`tL7@9E+_zuzro=BBUgXEJs9
zVuLQYQHk_|SPruS-NwB|QyTm*>){Yj?Ew8x<}X?652V)W3_!_`{@LrA-;C2#TB$-6
z!j^m><fW7{4l*ULy14uBapQky#u!_Zwr4){nqMMB!M*1@qYmMyo(8!4++1@#)}9U0
zx&nPgzdY-QUXqJML8yBy0(@-)-?69)SvIUhPrBQ4S76e=GB+l6WvNdH>4tkziIhC-
z4Sohimw?=(E4CM`X9t7Zuh#rzAcS}_K$qwQvmdITH}CSj+D6cvk;%YTo=Mk#Kcdq9
z=xsS665^ihpa2qj_tjiMzPm0`W)COs6JZp)i!(>Ue+|vw&u_s;HgA)C#aVro{(0i-
zr+q125Tce?FQ)appiP(LcNN<`6{Et~do6kr$Cvp%aCXyVbRapy$Wg;gu~)lV061iP
z%w(aKMDPE8(f##L*L99SuXgI2QmS$R|DmZi4l50Z)%8P0E+tSQ@tMtXmq%ZSIU8G$
z+Cer35kx3O-_076E{NxQM@tbM1)C`~AIOmB@LC3tGNO&#@Yq^&81EyBkqn3x-b0@x
zrOa(=WZ|cTi_^*asOyt;v*}9j;E)pRs!617j)Qz`vS_K_y{gmG#RYd9>^?jN*Ps4g
zS=AO-Wcd9nn#bWBegOzkG3Kx=h;p)slz5c*&RV}6Lu_Mg`EJTVXo6fbIED0Y%U7#J
zEv&OdTIk!^N=#_+X9{iOOQ|O@q;5O+hO^l>q<NM5ZK&WI&W5nF(bp_$B=o|VuzP3k
z*ROtY14XqvnE!feUi(Q7Annd|yVU8^-IRFaJhd7@Jw>;wzoMK$EFe|AB^>44Un~L&
zDp{FI78B+_ZYT$|H+!L+A+`y8N2x1|ehjh~S1rOC)+T4+fX}lqVz$<CfO_Ul*o~Dx
z0>Lmxh40iSP;-=Sn$Er~;ZaB55;Pokx5^jOTM*<9+XDPs83I5BhCKq(%<&@mX#$s@
zw+4P3(4g8{Ehy*Uk5YF77+X9nAAJ9lM3itJK)Mj^rc<9dhBptWQ+2Ct{iUSik?Sjy
z)WKxHy8W|>|NKr;rT3*uw^-!8>B3hm6?m*RJ^9tr3D)-y8&)mfcRw*P;*i-=L)!N7
zGf1u^aTDMIM|S!^{Fe`i7oX<`VG!r|D!|>Tsn54i3B_3xxGSt*76p55-wdwSxY=-5
z>?p*vk9R{opIx3hCrFy}gGmPuf;GIFA54hS0l93;Hi&iQSo0@1Y;!X?=|r#yFWNRo
znl)DxTs6gPdF!jei&)eIytG<0=r>ndilf~<TM*MYVCCzi=EL;pg81ix**!g(g!}B_
z%hD5f#nFhYe%m;BE-*m?nG@vq-41d9MD!v*SZ%HKw4k1hixY&FCEY8qF8O|DUZf31
z%y9?7sXarQSv6CV%Ty_xlvSO9MKv?c{WU!5RA=sjnfWlB*g;SzxRl2*d~Y7~j)ZM<
zsrBK*I?bwBJei=lM76A81WhH6(-&*fCn2BlER7qhp1!5k{tU8F#E&Z$5>+3n@KmTw
z7;&2xkEGyf-2A!fdh*<t?&#vv`wt$=3{nao1NB+ZjsDD<@*9yggJYbo`Q6*Ml7P)8
zCSgmL?ctvn<l>uz&>{mu>CwB1nn+my6m%|5+9N*O<yqOvw84q$xf!Fj(M96)oGw@$
z4RhWE&2cWGvBg!}qJhqf$tSGmnah}ZNMQlp1Jqo^M%u^XAt@{TXl#6s?SVTbd!hl(
zw-CI_H=dx~HWC9Pl>ViNoA-JF_#Y8y>FYzgqy$XR7Rlg!<XARYh}<~$qM7h!kUUvi
z#y4a=9pw;-f~_<Q+Qh9H?YV2<ZTlf;J=9Gey;SSCK?9XU>yYCYL5B|Utti|?y!fm-
zAHG1s2!S?DKG0Jplk&$zNgb&bT20bu7En-MNvI2H>^}P}WR?!U05TIZP?c_%uz@o8
z5I9h$1u2Cj0^JyV!#e8O)%?z8#%#r0bk`FV&o%h)Wj}hnzA`VyKTsxb(>1>$k$|YI
zH`RGD3W@XrBioa9A5s<f`6381Hi+^t-?4Cp2nsyu^lRV|!`t_icrgC8sjr_LUq$)|
zBJAMDS;^sp>Al|g)B_A<TF;ZQX--J0reFM6O<`UqG7B4~O${0s^r51@AC!`xQ<L!#
z8yLGM3%eIIb!pC7&^1nbRad-#Ll>1@^o=xNuiyI%8vS40)3z-@u%pnQ$L~59vuAMw
zP=_T9gY3)>3KN9-Z*CF0W#c<EXWvN9+f0pY-o5B!GTxapRNjsuMnMxr2cqBXB+q?_
z6gSc<D6H`cc`1h`M|8A6ve;6|;`&oWE&-yC`UG1e`e`Vow~G;3j5f=)5!1<}Rn>sv
zJ)#{3`#`VqH6}{_r!YwAPQ;Hk&ff=HUR&OPke8C^x`9y!f>ZMKV|7=K2iz}Orpp^$
zen*2b$hc|_5^Mnji8J-{D@e^P1j<o`h9E*8T|r<P!`b@G;dQ+NPv(b0Inc(iYWl54
zSSD@fQi5nWpR%o&iu@@IEf5KMGy9hX+3c30&kdGeKDLKe;*h-Kil0Wn#*`toxQfFI
zDRkON8cx2IlFESLv4vvUQ%W|ka_;w>(HL5OH!Xa>*hIW)*(4O)-zmT+yZ&pZZL)&a
zVE?bFk-{$8je?m~{0mdhm<&wfd#FMhX+@%3Qn-KYt^F}X#6myM?b42RR|tZtjn_Kc
zzQ^6xc?pOO-O7L{%AK~JH{%v$;~Ym<cJbJ4jbzW&<>SH%I%U(8&0U9e7m>NSj<kX^
zS~QEjMjwEO1o)B`_@2lGD{oKwyW@UUkWHvVlB}#@5wHdtb=e*2*RRt;W@6>xdHcRc
zNbYc`5h78oTz$$1RDQNPOrUscX)V}8<0a5gAJWxLe(_QH*%zXGrsIaPrLL3vsMInA
z2yh`g3><Boh_Q&b5sVp{&x$5OAwrUH<bBl061pRt&6!_#)>J$-PP1_o^j)aLBATc)
zJng%*y#$f&UHbb$NLvFDi}-X_hqFQiB7P^KGaPi`Ih2XH7?G3O^PQLA!YxE>e{g81
zZCoETd4QvT^;QmP%JoEF3X3KMA7zc^p6{)X8!dkMeft5P+2gkgVW*cvWH1Q5v&%fR
z_(-R1u8@(>Vzf@AshHVM(+e`G*5Vx1GAQRWfCuljhMz)fGgo+*2mL0^62BD{UdjGG
z+GDrgG0iiE{6bag4+h!aoKVgfTxadQxx_)3-JJXIj%}+GVc#MUp!uPgKQ16n2m}bj
z^I{i&b!`iOrCVv<AuUXjsMxr^w%PG@E~2}?h()6LKse@tY3C&*D@xA4`}Gi_o<BG?
zy-@ivB$V5z5^^ef(3Q!^F`w#G0XY>zHu})SQ552N<}Z1mj^5JfpvI&JJaOYE?!-3^
z_P=$;gBcZEoGe#vQcXHhh6`;hX-i1-j;^#SPnIT^H4q*)Zz#o`Y%PM2xAm^iat0kf
zmq8t-fiMs1lAe57G<J3e%_MMwo<VY3=h=iLS(7xQ#I4+JpU(4MKKzqmKN9VNeM-lw
z;A}aKr_IFC;_n8`XW-CUFw_v5Nc%!$S?&m{k=hMa9}f4-{(3?@BH2V*{LzhHG^(R{
z$hExj%gb>zX@m>%m@TjFN$51Xg(*kZqY+#KXmDjNF3-1U(YPH8)NLDA46k^O!dWkD
zp?x(PJ7Nxf5(zTtbw0gn^7_U1qp=kX2hTi?8kHo`Y(#7XwpwYwq5P;{Uw_8T@|~eL
z(a8%H>vM;{ulA-ni7S`St<ul{vduO)Z5OeK#t%nA!z=n%pKNy>C7(DsR8}58@Yh%w
zz28k6;-NrrL=I{NN95i3%6Y;AC)Jilqn4X3QY60PSXjUPef3=N<r=}mcgtw;gNq7_
z<sm)2O+efzGQWtcH?@`9d^5MS^dIXfc7!s0z8>EKf92R`D5M&KB4c%y9W3@FsC)7)
zu**+o%e*j!0%LJ){Fh|yP0-Te8M5#C*=RA$_}i1u;?BZETcDIBgF=p!Yg#pNv$TH;
z3F64;Hk)3?MSwcDusZFyX_P2!oIARPbCBY8$l$&*q5H+_*+wc|@N}0dL*@D-bh8Si
zxYDZqCRqGF_O{BCRTHnV_%(Sy*fikR)<U|FDR3OeH(40izdC&}TgK&iObRp+ROudm
z$r92??a-$Gt>akb(b=mi)IJPB8LeeddBGTiX$UHiXccN)-#O&h*%Y8@$*RDk>NxiU
zzIGvhEdI6nV!d`S?LmpQ&z~<>c^rd(uDS`X*dwhHWYq?TB3jZwA7%v-^^w*xT0U9<
z@wr>3sqvhJF)4_App3=*SYw$&ElA=;m<guyhnBkfFS<u~W%F7#ER92TEpgnfd1PV3
zfJhyP;;u&FWE$o^S&-+p(%o9wxRq~d(lT2=ptJsd!(=h<V!bWyZwJD1FXhu5quq_P
z__rz?aMPJ4xe1(2#Pb50LuNk}DEE4BZs!MBu~I-Bl`4pp5?Y`ow$?~2_W^Q|61&jP
zd$=<U*cQpZl*DnP-aXHiw$Afh&aMS7n7id0UA3FJa@6@<kxNrWR~C|w+MsYq>_D$?
z2o!uRZkx7eN<~2CTkf|Ep#g<8Q?5{QC+(?nXyH#Xyq;j}TJeWc_~tr6JqWjZcbMjX
zPZ<DM!kk^+0t^o;sPtx3*?c!2Z<d{7ocZN97EEbPT{y<f7C(@k3Gp?@<;!-tqh`?{
zjw%CgB)WZ{pLy03o3De6Acjr>TeLPStx{-Xq2uq&S##+MELnlX8&v{={CW-a9>ow(
zvl}@?zM>Mj4TTR7b)8<*EU6xoM4h6K-|sNU&Bq~Pe74X591Aj$+N@=kpjN7;Ab1!>
zUVj&>_2?2kr3jWX+D|He#%{$PI#2=-Gu20*)0CZkp>Elx>p364m{|Z#Y(!HTf!A??
zbutn^j6nz6Z2?IyqG<?eL(Mat5jRKACN%##h;2s8JGB_y<uj3pSQENk;^eAk%<Jcm
ziT{#guq>+djt1X=lJDKK2rV#ln_!ZSeg>!_Ik3qvO?3~iAAkGs9nIrXIdEQeBR8^N
zL9K$o(n~2iv|lwffGFHpn83q)lOC}f)Ye#oc9fO`musX9hx@3!c|#As#jD6Un__^%
z{dzyC>xaS6h<l_3S!g;QR}A6eiT3+;b3&4+e%?)U{t{vv;6`n82HItQAWJSGPEtP}
zP4i~d*2@#u*I)(IV-fXBPipW3LHjLX=m;2RS5s(ijdIs7zF8Urn?bkIJ8LUQt#5fH
zr^loL`9;{35rRPv92$HSAwI?aU7im97R?NY;lYlQ)@2q5Io`5Sz1joP;hq2>-M)0*
z9qb_f)5}|-pb~_FJQ}pA8LP{i2{DoiZ$O|}<+Uet&u<BWj{v*8AzN04Wl8g~Pwj*~
zMmEs;`YNE(Vf5(pJkpT>?IgZCA*14)zApL_3kD&r%D>H5BoE#Z6gR&bh>04i9zBAn
zN3+Z0c})e*jzgTw5_+wV8@ddS1Ee!G`EMGHIGV(?aDi10JI(ROCUVw=)|IxjcRm>%
zlzn@FhOV43A*}yK>7#W}DHIi)Bd?&7#n#_S0xVocwC7}dz1rfBn#GFv23mXr4G0VJ
zJ3%zk({K~q=PThxnFi6wD5*<7b;V;{HNqPTB8tD68PnV5P;wK}C>(UmvA^5W_2A3V
zS%`zmQp~CoI7P6h`<vhN)RzR~$(|Z*VthC<C%D+DL?;uBk)U`eS{;%2-SZST)9gmD
z>KHR?fIg8f>+4|$mz9fJn+pfobywDX*a4C0iv0uc!|vMx?_!;=Y)2GmLj=5QzihB4
zfq{-fz`K87gqRasCr1cJ*Fr|EtP8dXCT4^j6ax!5(^hNR(s=;D$CMy{&Jn6hzLtTn
zZZy+B(E_glA9}O06tBpZn(}tWTxO+vkwxn{#p|+=6Z_9q3<z(oeVCi<P_7YLohVmo
zi7}BrCxWf|Nj#E8WA#po1lk$Cwo|zhptVOK=s#{`4Z4@wAL;N$m&Y4_63e!K;#ZKE
z7hU~kDh;!$Yt;0X(7JskL_gwW{`--F=4Ajsh!QbEr+!$4)!_+*^4#@=XxI)1Hs%t9
z<2qG}fPTF)w+g9`s4qqIWWA9&AWFpe(j{8H7XNG<o#!+3h~on{90{MneX=%EJi4|n
zPM%3`Xp{}8<ue!qP&mm~dZ@}gDNmmapiv3W(-^o(9LRN+HUJfL{pS&jlk2Zj9ap>P
zwtjgM%(1>vP7bhNQOtu~O&Kg^ACj*gh$=a|UHib#0s_#RB0H`<Wo7(pKvVvUav=K9
z7IIc5#MCy|TG1*_VDYzO_7m3|gGdxddg25Ud!87(o}7>*Z%xH#{&ryExWEi{8CqbI
z7Rs8|aRR@qITtc&;Pkx-;)y1y$R*X_S%-LiL8V-j?^NW2&jg60sO{#chfugCF+t{s
zOrbu(fJBJ7DfuB>X#D<`I4(H!;(gZk`eMh;jdKwUn@~Qe1lJ{uywGxNl6tP<mUesp
z%kvZMN&xhv1d&~~<pHWqhR{+#Cml?@E>+C&Kn~H8h2mTV=u<L)xT=Mzxe6a-WIAvQ
zq*#0<Yq_LS@Hm}WC4+YM9l>d+MZ97g&ZE$A!DiyD#4dnz-W9-@PT4p>qusr90zo*L
z8H<g37B#<DmY-V`@gwAFqh?DTwEYgJ=pZ!N-Pbft+A|sRj2c=1g!78+Rh+1w<w7JF
zh#(+O*7z8UFq@x=3mF|Ao&`}fvSDC)6V2YoGd2yj(!?||dNQG070IsSd4MC5AR<%0
z7@wvnvELffZ9(au^OP4&!wNM96l-Y$q_kD}2gYMZ5~N6V!=dIu;`SuQOK3~Vjhd88
z<qGk6$V>xg&mt0~<>ly=qCuWy%|cbKv0fxM2UKb<NyT@It-YHs`@gk?Uxo~KN?XQy
zI(cJywbcDHhQVT3RGiqLE=q(}MTEc6=!Np6M-dFVVsl_oKY%>Q?4uvwx%0-r47*0$
zEWAFagm?>0959s7o2*lBmy&QW=qV~RIA}5HUu4{t;Tm+tkKUD^+d>pawjaQP&ze#+
z)VSEO#HYEwF_|@>j>%rFGm7@3Q$j?Xqr*Rb;ktME8eB&-lq1~r2=LkGQT<cD3+t@$
zjq1d#T+JXxOwAYiLa0y6&x(uS4YMDe*}eYf-_qz5@5w;jOdh-wf-<fL&95FA6~=C;
zO|qqos7NuNOdELEVv=M|M$TjRarL=*%J(_jTJ!b?#yge9a#`3*X20_uHE%^nD(B#C
zx~U;2x2JLsVtB`0>`LPdcm4sW<sr&_*)ln@twk}f+=n8Ma_h%AS)%TD<z?BM!ln62
zAgfEWk3EKl4ovYXr?4qs=cEPz7(LVaWLse?j-)Q3|M+{$S0koDQ;(}3oj!f_kT>R9
z!&B~EAT9PA0qxqvd;4$PK?imyXjmyYov((b?L8Q_L>8W`^ZKpb%srPGp$jMrH=1x3
z+HYu_G+dC1g2#dSLS$>!{kDvAv*y!VI_sT!Q^lJOxkG_!6S6C!5A8V~t3)!HDwl2v
z4miQ>>?EsjmXj9$IzCOnUb4AyD3fwjW0S1b_P#c#D{lS#pK9OUeYoZ>l{1G$fA3-?
zA0U!hUILBNf`6V0bp10N)@|wuaiSZc1QlyF-JKGHb3Ix0MoRARDa6eP(mDw%AGkKv
z;8rlU4cXEVaGnLq#5IR6t*8Ez^8Ad16AH&sJ&|NL_9z#^;S<sA;U4nXEi60BkdQBh
zSXD(#PZYAeW$8S9@Laf385rKb-KrKpYWCTO+^K-?@c{^<Dq^MsWmRtoK51r8sVzad
z&j+euRnWt^tIWBjn0!qRW~P#WJjYfX)T~wt^~27(?@r_%${Wlg;s5d_NdM!k5OoXG
zLtJ-AWLuB-5)ya}ksbOibo2_K8(}t;I}(_<@S%oJ8o(&w9FtXYTOZsFi1EiA{<_le
zh5kdLt$T8ce@f=%x8dtJmip_s{=KALjz#`{CR#_{Zv)EIh$BZV2){ChH~BLv4I*~S
zP?V0l9&vf=3*!)2x4C8oX!m=|xc-xB?}S~3M`tKL%e}W={x$+wRsvv)N-OiTjMzXO
zM>0~#UQ`0FZW&ITazvI9)cTuKG|mG;-D3?fuZrJ1PWlD%m>~)^7-fX?+Ik+sjjuL}
zr*~G~U%sZ74i@a6V(uP)V|Ogr;6)2`ZcY1PA-w>lPwb(?hjpR)BSiUZTb-~IzTklp
z<0{|Z`GkL&x_1cOlu3;DFGIN@Y-@$<__aXTVXZ4du{}1^)sjW@^6L{_(r48aa~JA*
zSk<P!W<4GEo&0t#ckLyu$_MF8hG((Qd>h_n-d8RDy;(rdSray0-B!D38`vUnCV{g`
zUx@SfT7kU&nUPiN+P_Wdby(H?JG|R9qTYPr;`Z0?cV#&K@=zIwVhi(q?0=GcdU?x9
z&P8mG6~~~5Q?%-y`|p}(SOIwH(ikz7zkL4g9zu54?UEzfQNqb9GPP&4Z>kENzV!_2
zK!9LGIKaZ0Q6(eRzAv?SBM<SfSbheIW=>d7{NQQ3)%>1mvudlN^{(lhR~CO;B)bEC
z!V3bRuLCs-{{UWZ4FN_Ck9D(h@ZiMwocbtHgpvO2{zHe7^~8qpxr|${!M#d4|CG7P
zm*o6SMClnZ>yECHNk5D@0hj?-BbQ$Q%4}wc_Bxn80CQ1=CaPz7rEB`)m{pO(BIh3C
z_pi_9ErM_~u4zAD`s+`(X^+Tbd*dO0FG7PC*$>VOY=}hn1Dr^fj-2~J&>X1>x9Vo}
zqnBS`{PgYttj5n`w?Mrl2}BDpMz1}I_tKz0!#j&G^X*KmQ?`|%cTcn3C5<8?{%D*h
zS)sVGdQA9c?CsP|o@TwLSZatH={I2SUIoC9Zb!DsoA)8d5z(6?PsKpaRugBvre)fp
zSgQE<e17D=qAft>6aM!nr=BSNJH_0Y8U4{XR0_>7=@|XR>mmVe{(4S24-Lu}B0Tpc
zuxN?XuZ_rnOUL75yo>;_m^jjJT%rg05y>c>lYW?)xXqN6@YJ4lnSUKd9dDTUlTGAd
zhyJ<Ck+IfsE{RhTyBaVrTtCv`J`vaWQCLnd=_oW}pxmZ2)B7pH&*;ay0K}c?)J0Qf
zL<1INs%*2zKR3z?vp}<oIWvM}6?Qv@2+kFK)EvK7@#;F=)~Nh@=aEAa=HZCp5xHGT
z-k&Mj`dtRL*SCT?u4*-M)K!k!)?u;dTe}AuUmhI{fC&m{138b}Ous9ZJDJ_Fb*91z
zn>%Da=XB*5uPE4)C|gl-I>OaIlYsXK7SRxH+#+$Vj_;<}MB7=8=$Osy>#37Gv>(68
z7rL2#!fV^r>4U+;9>Ikkmy;;r_U`0RgXgR^rK<TO`}OA>#3<n{T~Ue8mRYGC?`IRO
zygnmbvnhve%lhtk+KBTqrW=Jb5}TU$lj$UnaqtI&;vR5$oLwy5?E2vO%pTS?ngE`S
z6Ib{AOZB<j&7p~I*4fMYn<j~x{I~TUysFJU+Wn36FNZ`w2^6I@*(<jjLEG#mb-xGO
z7s>yaca^gx;ELWN^W)TkmO=-Gne6Cxvp$rMS-`s2!d+!xPHitQB3bbLjDBhds8_Sk
z1at(0UiT{f85`2I`{*TFz_|?)k*O2E?IWQb1NnDNmt~WJL<J)K3xpT{FUR<+&m`Xl
zIdqUGHap2{7K`-9Oxq+m0W1>$UT;#6JxnTtbp^6zqG`?{R;3w_fAve|OLX1LGrzam
z4qzP`i8I!*h>s(3PXXXG0Fi%(med5Fq7UZU2R+$EK7dl)c@6%>O{OmZnO`+sOfRe|
z0e6nrxTU5|HTVFypBaH3C`e2%mlnpqy@Z3;lmwUTRMu!YvCR~MoB^-Lau>YAxC<Wk
zS)S(g(0RVs^wx*+%0Vi-G=nvRurVWO>`K{==|^38acm)L^x+&IS2m(>zDmAw=2T;P
zWMPgD5Jh6)#kShV8cp&8mi5tM3lD)1d2sreLLg!u3N%8I+l;TY1g-SnLC+O|rdx)M
zkNiTE@*&Fah!o30)bPWj345T8b$`>bFVoF|)S{HI%m_SCvb9RDwg#WX-P*?vehwq!
zR*niXowo7vyb$9xS`9UAcl=<#E**wu|MX|o?Zp8)9~>i9hxhMdN48Ko&gJaalHZ&Y
zVVj`t=zDT}7xv)&n*GA<c=&O<KrYX5vmf&zw^wt#o^&`A8-_ge4C(cMGXnZ7Np<&N
zX4rGuz;lc;?|l~ZEPCt_Lj`%>Vm!Yix2KJKagw0WsuHhVsKsvEb>+C=plw)P?US(r
z6OD&oNtyxRuz$S%Xnai859qJGoqyImS^d5Xaz)(ni<BRe0ZOW(Tj`$;9rJ`v1Uvkj
zT~@*n;ju)le<8D+XuVSi{&j(4bySZEYb-`rk+QZR0D$^_kgDEH1tRJv#h$u#?eOWI
z;$mIC?_}>Tntug*up~&bX|HdBhq8Vb&|d<j4DpjQ>ee6*O8)fZpxQa<K#$c+Hk05t
zk^!sp#@p^05~$XN-T=#zk6v#g;)#pr!Rg|;vWh_)ZS*au(7#+Jr^AW5M)~GE0$I*y
z%|gx_X1^ooyf)6%UsTDHvc8_&(BY_}U3|Q7RI4@LDwP|mT2-`I(PYkc_O^)O;zvKN
zZ0ic$x`|KxSk$QF)u~)pP~eRnxYSCjL6H!%{?>1SW)~f-NVUHctmpkNef8fa5=6Ob
zI}xI)Q!W2;s<Y$I#JZAuGwdE&*5^vr6;?v|8%<V=*%bS*o8wd#@5-~eH#%;;Cd75~
z#6;2eWEm;1GOpD;3qDr7Ks+a_Z=oNsZVFJjQdhzVi@w9ql*kJnM6FhAtUUcK*$5^O
zY`LK4zTbVcu^tfjtgwL(GZPoxfvz3E8*<7){`>Pq5>=#6C$K)(h}b<{?pV7yO58l}
zHjYqS@|O_{U!&B*W{qQ^xpNKDLjZ^)Y;+cW>#ASqnf9PrMZZmcVA0q$?Ey6CD;+mi
zwd(2f%J>SQzPg|^H1D>YE(4xH*<NpAE|>B_G49b_-)lMmW@Vp5jCCXr@%dg!^7EvA
zBVVQk#gwG{a=?ZRzI&hYGItFk1nL!DcZ;UOjqE)|rj06^bD?*`fwz2z|9MQn#R|gc
z7&L0j6iv|P#kVf#6Ov!A)+`3MvqWe?-#$tC>59a&Iv>)z;Z0J7O|KZQ;4w9y7+Z|k
ztT5S>)fr^m=uj-=fnuzpeW92NRkm2-_w=2s3v(Kz;VPfs?5EfcuUr|hE?ChsJUirn
z*tQ>W*U*ECe*}YCq39RKhK8cwT8<bK<%HvUOf9;t;PdwF5rT3Cv&SmP8U$M2gpcHy
zcKuW#14@i#-E?bujJ_7g6al=oGQbboVRDE?d5%%31zX{qF9J&>tQr;n9c!NpHli_s
zhgmK_nH`ug+MPGjY4MTb3x~~|fmP%!eaop%M!?Ax*a6K$!3r9aaT=_OFMdT`t#14k
z2ic>@>#lj&9vQ^Kp1^>!(n#XeyN===IP!Z_(!WCBhZf`$G~hZb_r|w)W92;*Y{lOx
zr?OgriF~F#;sOMQ%%%Pm2wI}O;%r}lF|<6mc)xhu#=v>V)`X+wM*)ZXS)}B#l=3VP
z;#!;frvakz(C))lm*W#!y@HBm#{wWkJZ^cJdO#i0jW2YNNa&ueLH>1UJDC!j6xT`l
z?T0ohU0;Rx8{J?wap%h6M@=Ivo5Zadih}`->p5{YNXI^I#l}5D$k<)+^{&=D2vdWf
z!_nkY-t8$uE5KdvDqy@Ve1Kf@5Ac@GmdD-Rs|=)p*2PsVj82}(bc-06emkyf*Ki9;
zlC9p?HWA0C^}{>>VkzQ~-m6&#=g~w>{nSfgJ-AV|y_nH?n*7Vh$uE>>#8JM6_MD}a
zihgEe)^?F>bQFbjfFngy<R2VAwFQp!M_hO!Y%=#Rfd!?p@QqE#*@-FNMA<G*Ma!$V
zJUyV_pgr0+0%74~64P8aINc!*prcCNR}x?t+Z`?!MWkAmm_pdLuCoH56voV8ekv9b
zti%z3Qd1%BKT|?@cz6p2xH00dhzjW}OrXp%8BcTJsoX%BR}gIS=P9${n}9M$%$Xug
zevj0W+1nJ-ks}xJ!cTy|uU}uCp>H2&pS$HlN6FattBh*#gbJXZf#y|yhWj%AOQQrE
zN}tP-s;S}Mt^nkTuZ?E^)0D%J_ghlJ!=Iw_HUNn8!Q^s>QpXVD*8%48&q%ZQ(<M$1
zUG7kqaKWeieh0h+%o=4cwjDSn%=ZHdiDxJ;5AbryAy!QaWs*v8l22rJK~yF9)x7k;
zWLm08hSjuY{iO-OO-(b9(ns250JDcaV*Lu)<yx&4fbI4n4BQ-7I2L0b7|gnIj<Iu}
z;VuIE8J2^T=H5tPX{m<y3c=wMFT)FkGfh~%y0|`J_}PQ=9*38M=@*lx8xlRYE3wY+
zUbuzvf7>kO*`VeaHb$bh*8n9*Eb5L<I^Gf8G<<6(A_GCbPAc6i=O21*|3H+aPV0g7
zoxkG`(Yyx^RA<tTKGV?Qx0^Y;dn1@}9&nj@WiDtE5e9wdh;_4>!iTT)Dm3_k2_o!=
zxxK=@$qNW7mSOGCK__7=u1E1PkE>UOATx}xmz6NLvWCiYGlKu<$*cC8N<Kuwv}9`3
z`7vEL;?h)e44}9Rbt8UGsddv*+!}V9ikL-u!1q-~HrX3LwJB+mow1t{sWNY7%Z#*U
z%F~TFaDNi?AnJr{*5$$-rjdS&z-q9<$t1YI*&k3fp6r5uB6D!6WY$K8^Rmbrdl`Dm
zDq5VSDSbXWHM&oqKh#30*S?CcMJJ;5zDajM+lN}<?CFM-9)Yywp!GPQQx^TbERP5V
z^1YA)xa6z}F^du3q?%r)b6tNkcv;%lmgOF^zeqVOxN6**_Nn3bQ5Isp`{Ji({4ZAE
z1%4AX2$`{dRa{r$ao|o!JrnL?eQoMD@hx}?Uf^-z2-X)`%A@-Av~joM{TzDrYiLiS
z&Q#H&pG5@f`kuxWcDFgBmx&{9?Kg<NmD6L9mV&>To!B!evPfqmf4lvIq^8J7NumDi
zMl0R@Th}A^dtOl^+~ux{5yQnz@{04<Ui<je*2T^Ce|K@$lAkrZX0$o(xXR(9rN6v=
zmhib?foky8ypy!CudVPHB?)G<V#D-}upSH*0_*v#Tm|Pg^ZM#y5R@KIw3C{A1+9T0
zXhm4Lj~U?b{d5|Onu5(%PhT<K*mi}U^3H8Y*x$ULu=Wea#B*kkglbzW7LQG^LH0J+
zL}r4*p%i=rS=5veAJb=O*-f&peh(fz#tWqSE?F?QA~orWlB!)t8rK)uw2C?xIKmsc
zEc!Sdm#ee}p~>(-dAdET%Hic2G=w@W-f-G<*{s)Og0{vgxlelw?d5WyeuZgFSW&i#
zL~2rTJu7QOAUY6jA7x^NB6ARe73wneB^?A5`=g+BJ#+s9EF&^}mqonJ2!?&VJPntS
z?_!UmRv7)nCUk-=ZAbBm9h8v1HlV~JWcez;vgCC3tB@6;&n)I?NCaMRBkdEj53-2e
zSDeA8>f%5;e~Ku?JodTc+5pv5N}gSdhE;A=A16R$5MF<7B-<<JTbsfJsqRBuhnx5m
z2j}`W4yu@l-?knn`hVElh&BuL@za0mE9y87_-2g3dr7}B6vZ3J@Iy>Hw1XwXEjL;=
zH?pX!;#amO-4|0ACr!2tfCzfTq{jB>aD5LXV-g{J+}R7qrt6YP?!6d<D7q2|<y|%d
zA?z`Dk|62OrC|yfJ8PgHb`C{_fNOgtK(Z!Hd1A=Q0kJj8%Fo#@)g^$}rE-Fk!^vRe
zCqg~a7`YfjS&Vpk4c{A|8ir6}IC1W7ME_JEc=m+@UZ_M)?t$`3br^K*EM1;`xYwMd
z(g2r?etKR0Nx@GGmON|4n~R$|trvb<dtSJA84+<1G@(lT?eN%4l-cc=%@Xv=uIE`!
z{)~`<qU0cC5*9&U^u;~M&w^=Qdj3jcLd2{r_TgdL3o+?&?87q9m?tb;0GnsN_r5mM
zO^JTc-Xt@nY@Z74C7S$wTfE#G?=Y#WWtvrd6>Gi>aMi8#+52s;xnKn>mMBX+1TZTn
z&EtJ&Br3tmt(SQ=;ydnqyL;H|(@m13rK>sx7tdA}-#*l=x3aEqh&8?ST}gK9DDQ=x
za){1bXg!>4zYs>c{eyHwK`5W(-0nO&uaWk?h*~psOxbN69y<KM@p>!m4I{wrt$v|C
zDmPc$E^kV<ysW)xlJp(lc?N|57dj}{ugIr>caK@?oflGou^h^5l=A~1C8zlZ!Bons
ztS8n<ep)UiQ6f(U5za`~%gaNVSJnt6DEd-b<TeqaNInHuj1&r1Yj{5I+n54NMMsFG
zI?Yy^EbJ9+nHv$)aPYD-cNYgN7wTDELiYIEL(p}!T09zB@7cdX>!^SIpuMW&C<E%v
zY5<GcO~6C2uK3&PH;5jPZE)oQ{*f3M0gsTX720I+Qwhby$b><Ua_T1_)k#iKmUNMG
z#KfF?DrB;O$eo<M^8Bg)OpRgxCn&s)r7$xvcq39rTs2%iG$cxo4>~0O2K|j}ZThu}
zO*e{ZKr8w~a?##(V|)sbA^8;d=mZF_BsVb!R<Ma+H>nKZdQz-79!e1<h5N=YdkoS@
zS|9O5e3n|f=Z7dPGs>loFGY6x)`|+{wK=N2Ye6ROU1|=)_?K*kEZ&{D!q<5Mlh;;v
zbyw__5DMn5<zK}AR8Ewgs9e+!?YfKY!%=fujhVx{9N%G?s}@@kNf{}K_LF(hz5IV@
z`^&JX*02p2RzL(rIz&<$DG`fq=}?pq0VM>ayFsuB1px(V1O!n+LOKjeX;8Y9Zlr7A
zy%sR&-jDnFzT<e`=Z}Xo%*<Nrj_bbaJkLR)x0*Ga$lOUa>}bIPWT+W`JWKfpE(Rc#
zgF(-hwa2Ew@Joo}qm`z>$*y0}P*;ChWgLV$!w}{tt`xZ=k!ge?C<G!1T?y?v!a!8Q
z4*^W?uq+>fxE_Wwq-}r{OqT8SzKFBzKiagd9LO2Vwn66p3Zz8x>kw8C&zSZoC;`Kd
z+tsG3jo}=gdB|i8VeN&~%ZU%Q0|L(w2?+QUpEOT~3UTcH10E;JM4ClRT2@={A7kbJ
z#GY?EEbjt=*f5kvvt{x%p|v)LeP-p1(O2KVqIjDV0^sU>&(I4M0D)igAhwr`cPa!Q
zc(LfyScnJNL5F-T)RB8713@BdF^8shC>*4HD;{c2d?&hu^ukY&3yd50nubDQzh-{8
zR`LR0+%b&$M&j|a9ovc&C5&f&#r@v|Nd;6=*AEK~yH=eJwvtI|d>9!@E_+`5CS)pX
zfp^1IAgbE-%8!_rqCmaEzh<~_bpngk!?jF=NR+G<8v0Lli3_;z$aDjp!=vJLAXR$S
zZFM2hu;TOsQ5umeK4Pn_a|W+P-pPXGs+5d6P8=AoQ=p}sS`5IR&e&#(o?~(iW1&AC
za(xU_povP=-td8JD^Qk^Igkc`j%h&Pop_pW@0Dq6PR5Rda&QPV=_#E($!r2h{FoTF
zIcV$ZtZrs{pvgkROP{M&&La01+DgK7UdaT}E@wr5icCYIVSxameiSj5eFfordB-9M
z?GgeF<8d!_v4B&aFVo=5oXXm{puf#@sOtuUZJpS*{Ozb0G0Vo;{$4Lh)9V*E5=nJ#
zXI;H#k@e0FDe@pu3{<RUg3c7!Ts*h?hCY(HKF=!uJ0U+ONyxjOb9Y7v{all5*;_4w
zN;H#tw-?HP@@M)hdUWi@rOn_faNUHY@EPC>U}t0V-MIdV1PdD3C<{S`@@GbM=gH?l
z@NlpM!bj!rGj7gcqE(#Bq3UK-;VT+y<bO(hU@S2x44loXQfy-uAy5rIq%1JbWNi1f
zPwhGV9QfS|D>J=}vEhNV@r*n-oO&IuK#khh$RW{j2!YuSBMk>;Lq||`n?SA+3>Cvm
z!c~yv83ynS;gjbkei*P}k*r3^LZHhDH$2EH#vp35)(R}4K$evjmjcI3(GoLXVta<)
z*&mF!tKSPvTUp$guq>*bLoaMj80bV|b@g+$%nNOQEng3+Nu${OKZznkTcwJ6hP_Y<
zA|z`y_DqIfH4m0!c>~)DseoQ>HrN^FGtk$*v4oH$0TIoS=K&7V&cBZfhZTuoC=N>L
zXZoH4A>jefpMI?C8X*;335tZ)$~g$75>#WpQbOGn>7UIJnDh8_AJ?%Hy1TK%aQjuE
z`>zn7SHsYB77Pio2Ugs<C6MZKyZ;!@E{d*^G|0C%EqFm{Af}$EUoC?E9Z(4&nZb&|
z%Hk|&LeGQ1sWb@)*DgD?2g})G-G{WW6qkaxKH-=p@%9O<O)uVGaSTWG70l(<>-bPx
zY~x2ctB$<my5x?JX1Jnchy2y0^G8{CiLL%eag06=VsN%N?Km@ZHAfUy^XAnnIoz!l
zler}0ymi~x+fi)uuA=)VFH#wTqEz^@-BIKL5N?l8)m$1G(X6+`<3w~}HQb+QWqoZ0
zO%+XLKNxX@wpIrnQ${5Yd8qdwtzob_i?kV@&|AZ?`vviZ!%6{6GR@js%<=O@v+}H`
zijOG3J)}Z0qD`Tn23joCgi+vmUjfK_@pYv${A2O=kaU#|9drQ++lXG)l_4?#MR5DE
zrjMQuQyrIqBxBrd3INseNLjw$qFMb-lw%WeKV!T;)Zq7Xz!c$A+ZcR%eYqvyN^iv)
zq1C~o6Y0?jJe%luL}2~_9|X^C2y7Pdf}lln7{Vu~S|pL5z^IsH&~lkJP+)?s8**4&
zY`1O>)dbsqJVcQ+c=k1)mUoVf-&F!x21-#?!%B@Wv?udW1VTauJyr`o`B}_G2<+!l
zIfndxB{&tSJG_tm_7Ew^-4){neqvR;KV%MT<PUDChCQ#ZlG}r*h@C*{BJ0+CsJ<Lq
zonR5Gb-iuch5Kt10s)$bwz{>nr!XSG)lE^Ma{*{h=sdLQrjLd8#*7c#BM5_4%O4<D
z-VgHa^YFT8$sM#LZNIXo6x0Ua-mx;!`qaY^OpJw~#gN|eo@{j1YsU7}ZG?pj$wFv9
z7a7-+xz(h+nTi0~Sxr0Mz0@8?l#GL86K2U3KR<R<MlumMT!x?LrEm*|>Xx$*m_t~q
z^?a^aH`6o2vT&F@6>^*@v54LwXetX&Mn8Bd`N)-68Dw(3{HktnwgPO7Tk;{EO#pP*
z<-|JMU0AKVkB})Gft=|Ad&K&+E^KIvB6_k0nQ31Zhl^_B;@%+rs-m88V?6DrFbS6Y
zGt$D<qroj3#9l{YTE=u4qQWVz*ZXsl)Ic{4=irb%+6CRORK~9&4FOx{2k_7(Gw&(F
z0Rp4WX60z+J%5(|=~0YcI@~h<Aj>0t>Q}XMa|W10cj|jTU)V{V=l%uNG<}MiS76Ce
zVuE)CLq!hh#666ClL{s~mgyl7?ns|FYeJ~D6z8>7Xg^~fVzl*wff*+n*q;(&a%0Bf
zI1K@)N1K-77dHTYOUKP*&Nd{;VFpm}IFe)bJKV<|mD|+LGxsyugaMZaW8PcT00bj0
zgHJB>4F8NQ1Ss?n)N@y;w_X?o21CT++Qys{|Any25vt4fZ<_-*U&^t}VDlw19v`bY
z(XhDOZB+|q><8y1cP|JFhON9dqO?Z<BEIl#*m5p{y#dR+sg)n*zzD$PFW(M9mu(nQ
z4T3_OKKbhwv84R`m>#|^vrC65n}d_YD3}6_&!GDF0!;s+CB{Yzm1O_wn8f|WZ6_-f
z)3P(`FB<Iol-`TD(dVutUE<qbXN^d?-bgoE{^Wfn_frL249GxdqlO$0FCnMVG!CuB
zn=7yUQxeM^H~T@P`eiTRn>e5*HJ3IEh!{jT70JP`-?yK3*3C|mOI|bZe@3qgymZZW
z*2&=7*FZFI67dAp50zc8==w&BD|U3M>pjO$5dok_I&?8q<l`2?{y^^0U;CVc6-mRs
zj^L9%L3R^h<hFoN<(Hm@21{b!^W&i_k}|ZKWzZFlLnN*SOnXTZ4~{I4Sajz}msqCY
zD_M3xDlJCZZ_(b1>niXmAhb!^Tj1dbBj;tUCtoDqOWvP~0Ni<j*vW>3(NgeT=7A-v
zXcEZYxLkFh&&0lct$s|F@_JRFCzl;`e13t0ctWG3smLB=wK-wocZ$RYHl21`o|34>
z*UK{RJk#B&1t<wG%5!&~<<z(VSmi=ozw$j)qv13G!{9pPR=Lp2df!JuY*Qs?{NU*P
z{|^j0J+`m6ra}DRbQs=?@oj<elhg(doDXe*hU`cT@TsXYO&i5NqjlAsM98Ps3zF@M
zQyKuLT>F@f@O>W~#LmmbfYkR`;f;ymTKMLnqSeaVeI{naqjs|Yg#0tMTS`EAkQ2l`
zwsaW9^@)jC6Pohprwn2fXf^v%tL<y@6jN-SL}M<*RAH`A3A^TMs~aN?=@IyI-or~5
zfsCcpm@bs_re2~Q{*PG}@y|#GZFI@Ex=A3`V_B=w6QP|O5Mgoo)RO1O0vWb5F>9E$
zW~i>PWV`a508}5)h4^kDgm7wQaC#8cPTC-C{f#1!)WPdX0d=b|x$AS!=q^~^xam^u
ziqv~Nz9m5F_wtI}GBg>)qLlQ->ox=&?w8X6w+gM^drJuOD_o)IU{HN?>`TS`+A-1d
zwWeECDV`~ZJ^GzWxb&%7&zo9OEqLKq9sh4`BEsXC-c+lXAJB!NF*N(s9{=VHm<#Zh
zEXH;^USgQdY~&-zzF^?+Yl|fYHqwacTpJ|zB{nFuNBoA=p#b78n&0K37kcm$>w=Vf
z|NM!!kYliSwy718>&suram?F>Mv0@(+sWW?SiA^_x6Vrx#9Z5*G~Fl>zS~R(kTPqm
zM;}B|A10dwpx!e)u9+zqb@@;lL`p?Xo_Aiwpgs$Jhra2B2VWn?>H$tG9jb=T4GMx?
zagJT4-Ww6DQxpQqAUTg&8`X@>$@kS~^mv@da)iT&gNIjV65I7-<!9NgF0syWQMq!d
zB`f}#$^nn6`*g12*ygKn-3j*0qqooc`Bw8LUL@@@hGV&GI^zpnqtJrl%lXI~M6R^W
zBdt*OhDgv1is!L*8xUHi0c02}Hj2{XlbUX*%~{Wc7}4&4IYFuN+H}itcQhBklAtgf
zu#z3xoFUSDbQE3h{b!na=am?5wV0o%XTce(Kgx>#Y9HXmeU{E-`y+*cS$>@&&N;uv
z#|at+@-z=Y#kK@$N;CY5aUwd961eEZe~oWs{6_+hanV$EZ4%gnh6Eu%jh%6sH+15+
zAAv*xQ7f;o{k4HjB+CKE!Ls><(!go6Xn_79q{@n;M!YI9;ozQYDzdt$bV8X~Liyqk
ztj40I9Sv+K+gAt;i`u_`(NL!!a?lVKsD5k@4>(BV3rX(0Di2nk*^$S(&&w4;BXkU8
z3ITKx4q=-dG#CzcE(5KxuN>)9q()ZPx=|^_t_#r|`)w@jRL!h}lqtwUokF-tjCEE&
zAweP-y5+R<Dg)sc`kW=pP|Mm{g|@HY#BSuagA5J%sdnhFUIX;E@Z@KT-1InTpP&1V
zkdgAztQkSQOJ87G0qp8fLI5{$X#4>C+x8Ql<|+qs<{%Y<^yUe@MZoCEIVCTeDJqx(
zWf7!`>o?&Nkyczi?t%e~#;qSmlRuXuo?ebCG)*AvBAx9C)qE-+!=bm+FH9H;uG5pY
zj))b1!8#crciUQ>%vEC&N*m&eqh5{YEu1Vu0Yei&dGbS`b~@BGt72_kW7he*L7Lv!
zW6Y;&ryk(+{p2HR2E>pTZvyppfLydqxTLXtlpF)D4%tPk5Yr*J;YITcj0PQ9;C|7;
z5E6y<+|sLsG#(dl4X33{LBj<HOYrUqYto{YRNK47dW+1rq(jNDoqT&b?Jr$)ARztX
zP)!Tl2<NfZt=xnE!ruK8iB}<(SNS1T1zYr5L3=>1@xwgh%TMTU3Shg%;?p+z1mY1+
z&Q?KS61N1Y&PugXZS}ZVJ+Kh3t6Nxmss#bW)&PBhL|BWiO01S<;2+1L^mPgVy^^5&
zh!&x;Y6GcigY-7F1}4o4dpwWGlJTCI&I@gaJmkyAt?8C7LVYry98g(i?_+IQOqr~q
z2vNW%cd`w#I-~Hp2LQ6&2&^N#Ue1@vs5{oWFc_ow^R7_{Q^Iu=AQK<w-^t%ZYRm6E
zM+NW!BcNbG5@G)hx3@nvU<GfjIKngh@Dtkjl>?OP!qqNDvYUi-?w9_{RK`0S+@+IS
zefjur^n!nyQ||(U_+~vHLHs+9K1Fh^{wj9%+FV}Zd4uH*MIz=Fm5#n@A<F)ho_qRN
z1(QSpk!3?;1${4Jj%Nto@VTRaS%E?g)n&Q?2YTN6%IqvVor2F@+SbSRHzQK2-Ikx^
z-n_)~ZU(%r7nzblmk|m$X#aY0HQhMMQMVjABYp}zgk|SC;C)zVNlN=PBbz3ItXjKx
z`~Crh6~$jY-MHBC<^`1>L2Jh&nQq2rSVa(V!AGxaLWaZvx&kWQ@hn2iH-~w`W+)~N
zFTH;!;50+s%3Fs+m8GuzI+~^Wz85(Imzizj>&uBpzLS_fV3{!Kk$DqtQE45N(EmgH
z%fTYu*@I{11Ox3(J=8pjzsmVvN0iv@ssp0W<_swR7el$rPij*wHf|esV^Pcc@TZKG
zUy07xTWo#MQK6!zGkufBsk_=*$BzXEDM+0HdoMBi4PvrMF!kAOnR>H@RZR~l`EF}M
zi9GNXP5EHyOk$wGR|N5B46KWcx$<thB$zD9(Iznv;W(DU4~_AUhkko6xs_|$%9-A7
z1<qnr4CzZoj0!iJ>$;(*G75ek*}z@%Y)`EIR&f_#fFjA4$$c7r0L@?a<SK+Co7|su
zA@$d<1ylw(jqI;uWhW2gQ=51|S`X+XAA+sx^X&3un9itfYpjRrfXQ@EA>Z7^w;Gvw
zfCDlI3P<*r=bIar?qMIxMxqqA$KZo%0>55jjPvC(_q7*6!HPa&P;@X5Q8u9(Hz5iB
zsllU~ISdw0zMxk!sM`I+%&kR1AvlZ5H<oDWAI@TYO?qiz&AR+w%lqze-xHY3TdTLe
zmhk>gN$p3UyC+ixbJGv<VDeIwQ`QkF-C5Rb&zIYdZ46IY4z{l80R5jcKhRxN+P9I1
z+S8A)4=*EIUXBi0w%+K++VdncmcKB>=8fr}F46!FfBW7aWMk(Ll+NM{@c58|wn!V)
z>nnu5i<CG^)FvAgL8vYcQ93*HSL6iUidD5LkHUIze*&)`BEl;z89IWg8kw<IGj3=p
zV)%8l2*mlzrNb%BgcOlqG#@eq(EdD7Qg-u%ss{T@cKrGVndFV*z$;Cd-p*eIy_0IT
zc<D$Dz6C;pjZy%0=nJ&gLdSjulU>r(ap}v!RaXJAG2PK}{Cpx)fo|$5Cwb+$p<&)P
z=g9fpp2_-!Y=w(DJ+67?1?9rOyT_CW4?Xr-jFf*k&A-oity8M6yk%$G>%L5?q>Hrr
zXG&EOEOvf4g-#mEdeeTEn#etR@!W!S9`C5X`3>TCF}pB}-#@bZKhkofuecvO`ruz3
zbsWPSu@xDq6VZh3hv<5lvWFH1ruQ{b>IhAmq?0zv?=?!7H!iuw{73gJNWszHqk5#a
zC%Lg74evH%dUF!KZc`na(x{`yU`DAduPiYbZ1|1|qP4pWlRtQVmujHoN|^Nwy+%L3
zz6KI#YZgW{f4yHyMi{xOvxivGJy!pEUM-F$(|589IX)_0omYcbGWojG2Y!pZ=o6pz
zIwL%66YM&DFG$oYx+dx5$L>EnOn0&<yQ!3ih}6@`V;@cbwc7Vyijf~3Ykk=0CYfYj
zIS}w-O)uCg_Gn)Nx+j#4);p1#!qiQAF3ZV+#AT`OC(W_3Pq-gPSM=KlEYKs23>CfZ
z!5QH}vcx_7AtfvFfah>oe?ysnFOsKZ7@IFSIU-YKRkE+?1rJ(1cCX=?AzS;AUSrgO
zR*#fsZJNRhZrJv9FX5phTiY70F={Dw_)s^%DAax4c}GeXJw?76An@m0$+m;&xC{6j
zcw`%;Pq=^E>V^*vc(Ezy@x7m+xUn*I`D3@gN5ER)(l=J=30eww7Ip8M>U@9BHO>M&
zM64FMLc9sqiD)+4c}eh8vFWVV&IYe!@FMdeo%O)2agp74y3}}G_BANC?tEeX2HcDg
z!2#|a>1KKgXj!g#pgweBKb4Yzq*A0`)}cvn=%cWZRMvHlC<~6O6#T}yvTJWXibOJ<
z9{0RNj6<%!2iO^>XbQ#@=q2LkzgdkPoaG^vm#MJ5CM76wzMoFIIPYg)poLBTf$F^!
zrne#qsMEGyzrV0s7pAv_jKfbBgTs4!PmEp#NqVYK#nxWhTi{}l^u#a?>3cezw<PGh
zX=E~l|H%LBLODY}-UYtOO{t$rRFC&W+G<!1E{rxOa+AQy7z%B>Uq)a>s3pK`TbR%l
zIlUqyJ^ZZ3=ADw%6OX-z29h4=TN`d0?%Ew9L=8^J)P#Fqa8C9*)XzeG3#ZlaB1P=>
z_`;iRtAz|SR!=N*&EM2i6ia*R(Uo#Cqkop)8}-J|9x80Q8)Hj~p0b90X3e!^*Zgi2
zh@Lw2);Z{&MT!11^oZO52_3d}?H7|f8@vV+24p@yxKL_;M3|v>?iumELYp@=F?_+}
zNd!k;H6E#{ot!<Z68;PO@|uvw#PC;Wi7h6UrU9&AWhY;g5V73xZXyQ0hteF^a=Yz@
zw_clk*B3mIm&Jb>EgCj217*QYQ?m@xoh@SwX||fq<l#^IYlHOYoTRG1cq+OTJX;s3
zBc8j(-RXFOvt=`_BTe0%d45<mc<HDChYI`Nt^0DTrf1dsy5RfGl%g~(?1g9p)66ly
zbo0w$Utet`CfElIYj#v6NbTUT;KDVmFu3!n?+bSnw7jViEQ1VHSnVFs$U&?t+#d+U
z*;!vn?EQX??apl6Exb)NDT{IH4w*0Kx~yRX2ZPy2a^s80-TYyVgLfpC)b&`NmUqQ)
zcx35Vumn;MM7w14(($)H$DQrCEx}=p7CL(>hjc7mF|+30HGE7=U@j!gr}nMI(8~j&
zSi`px?Y`_;M(CW|Nc0twv#68q{@h45Zf!S3LrSJff;CdglI`qMGczq0oDDLwh+T6z
zshaFl^HN88(M^K{OF!nUxYgIs8)cv-lJdHC_$Rc-Qeo2&bkkF|p|i<lN1#nFldEx8
zTyzM*GVp(Uabq9L0II#7IPx4kXoe(^T8A%`U8=p-y-ouK7OpF0cibJ_9DO-s7_#+?
zYTj5$o#*Pk&?4skUiHL|VesxDX$n4w|C;B@23xQ|wf~rt#rXU$ozcK!3?#FCD>_vD
zvC`TC8nZ$E>do6G9qYyWydv`&$fB`u%b?y}G>AcoNFI{dUo>ZtMHA_rif$apR^C}O
zTHM|0uPsBcw%D(%6jY$j-n!Y!!=SD1Z?3~QzabDzLnZBK$;)`my!%4e-LTFawWOQj
zHGRD!9*t$;@7qhFQkW)-jUw!tTs$s>qPD6x%l1X@{juuem=wn1nClJ8qXJRux(Ux@
z<zm`bS~hfFeSNW#ot)@Z=n;c*xO(>tne^9dXXG5ONiDx+$=()cc#ePm%fUw#afI(?
zIhRqz?vZl%1qz16YuxpV?BVe2-L&05Ee{6n8u9}e5iC%-Nq&zrx~G%XgR!na?e)tq
z>Iu#M(+iP8--M94JvZ6_Gr-i42D7F<;kPpL46|@3t7+EoeRht9He&BP@g|2HttcYa
z9@ng7z8JjyB~IRM3pT6d2fMT8!ql95C*uK(Pbebs-9t;P4Z1a+E-BR<+W(_86YLRb
ztomDfKT2-~t-11gYQq8PA_ohnpEK0EMzFF*rLdwwNn);tfI_5!z-mSTf7QdoHR$<7
z*TV1?+jq;XZ;vcE<)5J?+}shf6L&qzY$v>}xeRv7HuqNHzMKsGFk+`{ubkRzBK77(
z44cQX7gl-SPee+2dovsppi)D92}AoiS2%87!quFAAcuFHCEN4zowwX?y12npN}?#n
zwlYw#_jWH{dCV<=i{9;^aV@+cj8Wi;yx~nv2<GANiC~4@(+~~|f2q5o&&Y834x{gw
zlNe64akDh{(v>twgiBimSEcwqOMfP!cW3u-q1VH5v>?)2EvxC_WuOWls=a0x^Xtga
zWT|n9GTJ`x?2_HBk_HO2^k$L%B>MrrzZP;yJ51|6-Ffggg0SYH_G8b_7b!*&%pxp3
zL2ZUj#ug#-r18~mR{d$x-D~v7X7q6arBxE?<tfE%3CBhkZ};+*x;5+(jDOcXYDAGE
zA+x?ve{Uqj=y{Q{(+euM-=0JTkM(Nd$BjBSE@y3S%jRIL3F;i4%kj6521B>zvr|z4
z8eh-te){Q)4}CfeRG#Eod`^1$AN(m91Z)qGU}PP!o~3+CI*eYx`_E0Ufb8s$hd=kF
zhrRVFU|=izu%drEbO$h?o%T6Cd9z04J*iXQ2-)>2OHKtm>(xtAln!ngUMZE22*#0r
z-nnv0tLw3ps(gj#1xDhr*I3InSR<@IiP!nEBBoYEM^@5*=<Pn`6{efhZ>!JDt`mD)
zP+51<J>*umsY~S1U%U|%?lQj}OS7R{oFAdQJ@Vcl)Ww2y%_F9!O;0*6t!Jgh^~L{q
zh#g<z1Gv)vtsuPfou7mStDlYvztVI5CEa-D=)c_GKVFiCBaT0Fqs_l(YkxVyTHrtq
z(yzA%lm6pF-X+K)=8235_}zH@$M^RfpsR@AcuH<!zmx9YPbZ78JEW4o5<#zz-7oIX
zAo8-t6xpvuE`=BW<3sdsk(-?u865hLn?-&u9-#wcyJHiG#)tngBD*Gt@Gaa+mi%S3
zr||E4-Fdvz2m%cIe|#71iANy1O38V6qx*_a`{wQUcPd#_Gj8w$lV-~7F9Rh)->Z+T
zD%1XZ80itHdkB`H(|<hv&KL=MGX`ChIHjJ()?XjX1w0Z<H*V%)c|)2|FVLj-I`nt4
z<dkhR<na723nNMGE|$0b#FafwwBI-U$Nvmpi9vq-)e8TpERbLirAP+|`C*9TVkqm2
zB|bc61mZMjZck|f8=;1}t|%+IU*L})*d33jF$hA`xY%ZwXW)PQeP=wM-u6B(L(47@
za#tC8-YpY86dK1(^Us@QmRklLkM}zWN&ejY@tTAE4Uxm_)fj&b3wM5b90^2wKYc?V
zf`7Z57W=?s2GwLm%9YJ?mp{m1SgSwG>@Ib?ka+mw#S+H5Vt26cq1+zZKXT|d;2(LG
zUx@L^JU+a;^lC1`T%E~~yLj?1ljWU;#3@?MiqTAPuTRs&#@H~j+A7Bm?k+lg^5duZ
z3dV9Q?&W9w0kHIj)}EEe8jt69@7gdLRw-pWQ#9dUo>xpAXYNsUnhA?CF|2u6^$%90
z-m(s7T*a7Z@=o3r`vfRS7RBnlo_6H!ZIHWXJzlwY_d>|sXDVI%o0hk?Idb>oX0O!Z
z@G>?^&7kc7xl7qlF`-19)2?EXC%o?v)gUKbE5$(oz%_>Lxv^sOg75@&4UU6pyO+U1
z4kN64zoFFLI7sx0fgiQE<>+~KS`i{?x)(dIfCtLNL^#QY@tF20A&($D1}AvBl?l%@
zDa}dj40!eix<&^)YTJB^!ZXMl-X+L2tRf?i?jLv|h*!gOuoKNVV7u$nvFDBFyt)m~
zp-CTh8SA(({ViQbsRQ07a1Ch^d3V7)zwRv7y`dgThNi{B&Wsx%j|n{$x`=htsIRZw
zuQ|)%fZs^$JcW+9`H?=E^++=Gb8r$`kRH55Ap0VWhDeN_9PU{EvV0m|o?l@W1^OBp
z$Thn2Zcy%AqYp_oTu`ci@cSA^;2I%!r_iEq{H#0IAVpt;nb36S8vbE1aE(a#wdVW8
z{ec!g9jDOZ)P=}xS>|zQ>{zL%8n73KuG!gTq4$LtG2#zl(H8GrR7`otH@{tZ-Xjme
zGs@v&Jz@ATc2F)0i4Q6m;MX|i{hoh3&wF7R&8^r&1}>CScNoWRY)AzwYd$%(5Z&G;
z238IHf8wXTdp4VXbjRBZ)sGU<1|P>e>ComSqMM6%leAL7z$M^<2dc#OYr2Ewx5n|)
zD43+-+r_rmc5d7SAuG{6cYz`ReYP{51dmc~Ad3S%wn;D=<7VpSt;P!Vumf|&R(F=w
zS#~m*&i|Wb<%jHYSh~blK&SlwW?3akeV~*U<DlYmOhbAMn3t~L!UGes!L-mTwNSlN
zirklir>{ofZdusFwmVyG9sy6en>%EnJ$*u1DM%X-UpAmaZ!4pnK((l2cR$k`VF>u&
zRR)XuJX;_Y2lhV>Da`Xzmhod9EK0J6_EykS17v12@7eP2ys?IW>O}Gc+V`=rObjqH
zey_6#&@+?0Gc!8o_be2ik*3<umHnqP_MC+M+U@?ghNi*9ucW4&)E>2|NqEsZ_SL7A
zrcLg-PqMG>zRx_vu~;f56`o$Y(RXxtDyn^@w0PTjny@pcwWMxi#)6?Z&wr}JT$f5Z
zFB6oFuN&5W_28-f7CuRWtdU!LYh?Ezih;*5`h8m6hfPW7^O1A!=3(t%!D9Bm`xy<J
zuTj_;wW*FRQ|-JE*%*Vmw=uHyVGP#@_-M9jw+dtUcZYXg=jJST@G;SVv<SxRpKm>Q
zlvrxsr*=Ob;8?e}o;#l>MFCA*MMV4AnQ|;Jq?yYjyG$LnS77#{&DFWl$q+)>dq^q{
z^VBu;##KKG=fUDRy|R9{u=6TgR*<!oQwcq8Jyn*1N>PbICmOg8(kR`mv=2Ts>(3cc
zU`6WFQzVhW@*QK%b2j3R%EbT3-+~(r3@9B@cS&}fBIKWJB!n@NleCH4w<7pBNIdc!
zn}N$U;H-$Wlnnk&v~?KzNA$v^LE$!<Ay(GsHewlIYlQv*^fn$Ug(J^}4&wlFV(t8q
z_Xs@J@t177&27k=u$PBjKVCD}B`*Q*=c*Z7i(@?Hx%iwyjDLo05zSa45UA$XtV^e+
zM|NfF(f7{?=ag3#VC*gU{RblSWq?%+bg}ys&N!P0J|E}MAD_I!@lNj?jOQ!a99<Tk
zc*8rB$>;a}%*J_>%y)AxS&r2C($LD%gZ!L*MB?vj{52Z<hvVsOPJCLlHc!Ay&sHqr
zNT>bJ=sjyzc=DZDcNvp6e`5U-R4@J>IMkd3yKDQJw{Tx67V#E;U%$rM`^b0cq8`R8
zD;*Uo_Ko<#O69)hTI|^lPh2qO_fi@}ofZN@Gkx3%M$1#A>>bS6PWdkVIkQI7=VnL7
zzpPuDK9Y^uPBchDDYJg5pn530W6?JcEj9+<K1H-{d8vUtHF0$F7bG<zxJqZQy?HE~
zYCCdGJ;~mMNGX{^?Q+r$=HPiqXz=yF+GvN(|E}iXqbUvpI(l!fFcp#r=kOZG1)Z<^
ziLCuq080T_RN#DMJod%t&v(%a2mb!>=(pLMA(%7gW2oL%IVE&;dX19?leJ2#dl-aZ
zhD9o)k_|-n3Jz=Bs-KvSQ>*F?4J6jTjq&wtzjH-gs1?#laS=@3Mrwnd0ongvnmboD
zOha6<?z7Ry_YK1DJAnTUJ^t`x)Zf0KQ%G1VE{u9G_jX22NxKxeO2>s$Vvmb2<Lyj|
zcL`ZjGdBM8sf7OGbJ`UR@qSKfTSjG0)m3E#`<Rx0PM#{_uI1jk$o1!x{xP{k-UitI
zxrI!50nPaS0V2c^3;w<=IH&hQZ!vw0bK3s7Q;aR|+@^3{`vVChcsa8M_dSt+jwmIv
z8y=p;O4(&s{>K9%pX6uTt*_`8za!K0bLYQmi2Bs=RSj*L5{_Oq<Fws8#^BH;&GVLd
zB#0#9EK^?gv;7R|AB$3q9uo=ZK3w|o?9X?9uUz`WSo$>l11n3(j6YEf3L=f(0VBX+
zf_$Jx)f-z!>a2}b`JHB!r@>eWRF^USF=)HPLvMgwp(FW6#eZDE>=3@FM$%w<^G><Q
z&ndtp1Tqb~)1Rav7K({vC^J142hln5zh35mHy+Zq|M|rI*1w0qkZF3|FFwR0DJ}o(
zvHMBBZI1bRYqcu>8-eM2OYNz|PGNxOLgvBX$=oASA|>L6<Iie__p3VpSdwB75ufzd
zu}|awoIUi1pg48)HMebp32eA~JQ9M`%0+rZkMezL-=QOp7LKO0aM_fP!6fUB=7f~p
zGKL#Pi?K)F|F(jE{4(4eG1e7dxm^bgeUbgO@w5?RWVB5!jx^Qn#%Ujx!|v_@H6eZ<
zjXKaZI%&qH0jc0nI{#><ckYojZtX7?*hh%Gx^57g&rkdQe_akU8oFB}VZIv>K*rLZ
z$-BWT$h|H#$hXdg**3!7ism%mD$f6V2AAxF2>&UjmA5P{qe{VN$b1O+k&=%PMM(_T
z-Vmt@VaVukw=DY)7|eFD%uhrxlxGrfht6<8+@$?tea|I^^-K-n6b)|Q?i)4NVYIXJ
z#xL6ZWxsptlOf^RAl+%kf*Io=+&=YaZGmItrk?6^l$=pOgz9j+nC!;G{X0;^7*R=1
z5#okLJojsKXFEE(73EvcMu_|{FIiEH!#=k^<qSP;#CxutA#xUL_20j-Kgp_65WJ=N
z#`Ev+<-yyAPj>2oYDQ5O?;kQ4mor>RNGkUev2Zqz`{^<9ZaX3i&F@Wp5Ne;%%?#@^
z7qkvBWXADidj|KnEoRo{L5U_HX$GewxgjKSZ->%fKwQ@+jzgz*-k86B7t6cE(vusV
zqbKy01lRTFhXl60Z94Q|$EO@I&zauxmAyyeBP>!RXZU6GkHT)m$)@-K*%hzDaRHuj
zlPj(^ik*&LSE(=g(UNpa4R;XiSq|H4kp916p_Q?tmJNAQupNn2BUjM#<eh{Cl|Ob7
zM|9zj8gYs}o<u6;T|42s=vl8h_U~#vy^S}(uxb&8P=wA#=L^8phU@qm6-z!*_vZkr
z{*3#o_2mmONM|U*Ut+uYrm*DxAy+(-)K>1BP6~!~g^>2eGFKU#FfS67XOz7Drs_Ob
z+EsMqsRitZm&q^CrtF4^A4Fj3{6Vhg-GSWkZ#a?I7Q(7fme;`2ji2wEJ2q{FbcOY)
z7@{P0y28BgV*%ihl|t9~mg0tGUx^kX%f0d5V`KTn@|TAa(76=`K+l4dBG;uI0j}b@
z#~K5Gjtd6}M5hiv&=UhkbV_j@9tN%VohQ-_v_`hu@~-N|NA0S871zRxH4XVX?Y*uV
zvat&g=CS)Eo!DF9HIztHpUhrJh`iywi4@knx??%euoA4FBm<A)S4-H?nWjP>c<%MC
zi`wQ2;UweR-m;zYTjZh<79?9U(CZC;;yW?uFENBpf17D5tuRE30ARVS00#&LB{8lH
z6kqyjT#R4uad0ubT(1Ga5%53^W+0+(f{lv4V(lAJCs5}MrE8`#xEL7g^MnN@)yY4a
z=K+%Zp!8P)Z;P*Tsf2s02I%KD5`WXrd3y}%MrEBwmrRhu$`^@(F5m4yhea;Cp}df`
zITcDeX%h7j-zt(dIx~W5$8tP<x<hGpxU0zFEwDAnZh>~h!;FSEzySQyjGGv1YZ2jm
z-s!S0P5}0YdFa(IuC3cVshaaZHk0BWKTFTa`PN4tyIZrEwV@VlGd-R`(rtfKO>}R=
z7)HT{>1@-F=GlGY0Tw855#n<EJ|f)_YK`y*&-cvO(`tx|US<Busa)Qv_Rr2jRxXnM
z1GsAJ6oB#*Q<0S+J%0M;U1*II8XnakB@3Y^j5~u2?E_YvXiyGat~ucqcC&%331mS@
z=c2;A*ysDsd1|-h17BcJ3+vOe$SkEeF0>^Qb42nGG1FCAyQYZ{@kwy7bABH@pv!fV
z?H`FGq=b}Xe9xj&o`J&iPL&DQ`0VLlmJsOxdK-Fhn3xB`>6?n3W+fbGZ7PTy84xMy
z9l$SaPq(k6Kpzafid>)^KcCk4@DMW8%*aqPh!3a)ytY$PCK<O4_u_pkHdJW|5#_3t
zN_CexT(jDNLOP~{{K1*(J%0@Rc(4gk*8}W!51<bkcd-uYk*n|Lb#O?DVJS;6rJ=td
zv{QsS`1DxbBS$j+HN7otsD^IpIR*E2qw;gWhK3N&DF73g#C8vn5RGlk$@vb0AUVKx
z2|dsS@nwY8-B{;5Cxohc9KrN+DHJg7*QH`O)WnB;f~4f#LIZkGB4yHszp$Kmr};Wz
zWF7CHr8&@j_Tv?FbCO{?tO%`c`{;|iD<Tk#e<0TE@9_>sS{||qwD&9_{q{~0Ckn>9
zx?37ww7Y*`R7Fap5TRCsWR$756!dt<Bdh=j4ix}_ld+!xr-OjevK^<dJL#OS`{(Ue
zoONoEWOx)2jIo*QVj#^TI+o8YJemDti%nP~cI$eUQaP^-dc3BP0WR(e$k`iUg#46_
zRDI9eg@b7ovU5)fj=V)5D49FNFbl@_rnMNQ@0?w@@ZzkL;nu}%3zytwAQ}$`Eoru&
zWT3A;dPP<BW_UB}2k1~9Zz=O|Ltsf`B>hg)UCbbO{1g(=9V%XWplevM9{mK`gF~Q6
zY==j!9U;MTI?z4{kb}!eZz1WE)r^>T&?P<@w)QiCXUb*T+#R?W{n|F5EAvZDFryI=
zlv(mmTlWFD0^s`*9v-mvLI5_V0*$RxVSYx*iV<fVX+Sb1H*<6kdevFoGH7b-bk5(X
z3pi?~>>R&BgQyyal1y-3@@QYa2;v_-wg8X~LC~8zf#=nLgs*whx+@8|m>mUQkArvx
zOF6Vn=bq}BhHgfaLnkP5j2j&%TOt6z*cBH0oQV9}r@}yhUieqdAS{vqNJKOMI*XmG
z?w2N70ees^v_4jOnr@~pL5r>tNJuW1v=LhQh>eZ@0s@F|Aa;`$B~nWO)Jj4et8vR@
zHyE*;F&ELrCdpUO*t;xHn?2PSsTL3Y(zaEU#vQy#?YW@G8gaoYwk}3c<2ESGP+hRJ
z_pe&g%{G+lmM9GrnwPcFAMCa&e~xs01Cqs}J=`D58}I>uaa2mWoOmh0A{W%|Hnv$%
z+opgaNXYo5C_`;?-=nOk5vb0FL7~FYseKz$1w_j>w#8A~$f=O`JkpeEZ1h5yoT%OU
zw(OWaNT<pJn?=txq_@(bUFQejPe(P^%^Rk#MdH1d^Y(1*qLZpxTImjnbwKt|EQg9$
zdPj2q3&uS^S<De~m(O&jD?4|$IwAedh^3O#^k}-6oeoQ1YD~=A*NGQDac9$rtnT-}
z7E)r0uUsmMJ+IP0-wK(r+@Y4#k1N&JPB1y4_=!b<1L%y$eMXh!)*_4V>Dm`+?Z=*I
zt^$lIaN>@@OKlb5rDsMV_!yICV)D}gUa6gY&*|HGtn*J?72#yHcSZuEu5IH!<5T89
z$Wfb-30Rb{@ryTJKOP^lSeokOo8VpA4r+O&ppDS=8wpTTykXrXY(qH0Y^fPh{jJb=
z-=t%a2)wlw_9EA30dzFpxefiSfu{XnKD9y1=c!Tbx7kind+~0C?tY<8(PRN7*14wM
z={4DQ+Om2kU+3lW0iUD#0yCFoXpBHC%qi9bdh}x+9izGpQYPHV0+NmOOjN5?;dXdH
z4dyx$Va(Zz+Yd)b!<`KjKyi)<#marlQe^N0KAdo@uw%k>fZJK=McG%`R}l2HGu0Lt
zE!ib+Od8J@j>ZNYuVvw{5CCL9=HxL^R%5O0{-#E;rX-eRzLIOOduxG5*uNAysd?!d
zndCHq`XOL-+rR1?2rJ&C0To-`@7k~A$r{(jw&h%PkY4kuAUe8Vnk<x+cK|s0Z4gn!
zj5T1&xfv3gK-*{<9_`jXyBNPgTuK3A$E}Xi^LJc}n8h;_1RmWGR#v65)6+Yf74z1n
z(t${HuaCZy8YVoZNF*vVz%B)~*L*wAf$cvg^LA5WSW?Q31QXqIFE&#Nj&^P7`C&Wx
zC}Q;6UywHy8hEa38Ugq6aM8z@Lp%{Ta$^oL@{=#*X_8<;ceZ&%$}O4c;uBor10b-T
z>j@U__0q*dP$aw!;w`ND)Q?zk5R6;H%!9sqk`G+MhZmkrOLRw*0Xi@%*T6bSG&w3I
zUVG%rW&<~G={DHPFqa~i3fK3aU7f2}VPA>-6lrjYWUc9zBYer4=ttPF9Pb3En^!Rr
z1~F&x0Ro!+dWj{9p0IBm1^ldm&UMhodmScUuOjbQc~xfF`qwimUxPHQUU%O;h{^fn
zizdst?}6(t)snCdE*Ne%C8!sH>Vd9t3wRBRzN}$O3*>i*&sxR$TA{!v3heOf<0tTF
zZ$N=9>j7}w%WKjx?M%2z12_Us#vVXlA?#oZD7^R=L=r-i_sGxw7zt6}_H(9wl6qCi
zC5Ap8_rQ8Dt-V#k3RdN&N9`}Qx%`><<?l<mJpYM(cpAg|)Q3(fd*KtVX4jKHI{v+#
z^&u@v#Ph4-sF5GVZ>~pK7VL%#Dl7!9%xYF|meUPld#LTb+?Uz9wg^TwiLsz7*)UzF
z_<XWR8la+p&y!Xy#Jk^nYdOFCP@a~v&)xp5my?UQv(3psV8J}+jPUL);x76r0zLhE
z=zylw%y+c@meTl0SuG_pMX!3Ua17)P$1KBGB!RT^<j*pMsD>=yQOZwIasV?&`Z};i
z(Z91FztVUsk=cI?2ggsbt5fj0gztJBFv-|xWYT<<Oi@(aQDc`>;hiW6WU{H_t(G!(
zTPlL!B%zH=m%fK1j0g)+rO;JUtc_%QDTfy<q79IKIKHyUIW-B@P|7EjR-0XY0wsOK
zj%;rsG)?^*psf0NJMo#^Ew`l4f@5WB?J0xK-~2fKtHD1>6#LBP;fO^{Lc`;~cftW)
zLIu3JM`wOAg<X7!XCr~PZ$Q@wF_n~$&aic$Zs`vM9q)kSF%1*{;)jKGgeR;leR%`u
zQG{xX-T@b?g5WGZjlefoKx3!FdDW?6Ng%&Nw7LY2G{qYkLDor<DLC#;o^$JT-2vXU
zC4iihyH2bEc2s|y<R?ntz&gmH0v)1bDQmPftoC_SG7;k?O*r)4yDpC1O21Lqq9(Pn
zA{drI#drI0N*XtWA9XE(W$P8r(|Bp^_5ifK4yPRFRrM8uH`?t$!r7M-<XDpdfc**&
z`|=C@?yWif5Hb=v#DPuj1_&dfm#F>-pN5krI39Z3)d9vd#kw%rIwF|@;Y>X@i9OdB
zCL92yGcKwwyMt?snVAAu?FX0en<^e5cKi+yi(P{gwOHq@2)F><5)S=#jl$jzlS`)N
z4H0}PO`ct4z_0e`)e^8V{%8YcVasZ(CLFSeFnU5yaYmWITmZ<>oRl=sG`KE0`6JXb
z>|U2nyBYD5zzQOf@z4#}%Xb9Ja_i5I8-P>rqq1k%1Q!FC2D3g5UI}$VR6-{KE%n|&
zV{irBp*+OLX`8H*qQYeq693St((k}@jOOG^7yxy^2BdTA6lq4L4TM`SfQ|{4>-S8(
z{t3WMy=(`bDhUE(QyR}<iS}_Err<8-(fn~IAR4;cNAjX$=)t}m@86(x4=WEG3xS+j
zkniSG`QK{?ZD)58B|EPSZSkmGY^-HIF5fdL#<93y)hDD<eRVo|RxuJMo+bn3nfcxD
zUOcO<+2P(f(}54(WlJW#U8Y`eIK=AY8f5j`d?vvvTc6-4mMl<Hml11HUVhmM;r~FY
zf}rywWdvVr?ld6)7$3=i&V{Zox`LktjU@V)a8Y;21xz1}0EyIknA7t2z;7>GL0y|L
ztHg1nADE`bxZC_#V~^X!A!?6mx)hc`RHzWjAH9I%(T?BrBPI?ZvDpawjtY?@Y#Msl
zBp38HW>d1UI--5y`(<(oRu2JpRS9p$gtoxBm!W}q2*pJI`qIS{Z-2A`ktOGJ&vdlR
z*ao=kW?Y~>;!eS#MuMfDscV4(MEz1JIgQTA9t|GO7PVjJ)>z!v%eObmKinsVcus-M
zH;g_$E}+R2pcRhBsHQyYk)`dFIi?_R(bjZxa#U1(K1_AyU<M_|S<pOZuy1j;pLo2a
z6G|<SvW42H){#l8{u*IBZH2$p*Y4(n)9+?JINS7>dD-9Zm86&-C`g1vkI&y@|1TS|
z^WmpYc&($yRcRKPew+_)N+#+$3+rt>X&pT3E|n`PL-}2Ea7Iq9zO8l!ewS#;?pZ)d
zFIwc!VI3?F%xz6kS^4$~7w08#eAX@btu`d7C=v9|)J^&xe%b0z#xd1Q5IyZVyS}&$
z?9syZ7bbyPbuqoab33DWX8svMg~k{*3-MXXCiqvFg6o3S$3ZQ{I1Y%CWrQBifVSX@
z%kpKul(M-Mh?wjvd?quB8|?6OR9EKKV~msz;KnV=Z8bQSuRu~p8~DYSrW_aQlkIFi
zj54?<9a3Tg#RT=6F6lBdnK*H0fMq09+9yM&9T>!Z{Rr0&DkI8OS@orcTPqblA+747
z;{|MuWCE9Rw^`4MBU%76lL#%VGY!IrKrJaaFC)NO4o7VTu6&5?L=o`V2~{@}G24?K
zF?QpeWx`>abGnSOBv?o0=x*bMNs@O}A<jD*&VS<E_OZnD*LXh!@ofU9<^Ht&eM-gZ
zc$<c21_FKR8|A+f<nwC<0qYi^pI&@>KS2TqQW%%^rF0d+o3lNIt-VTdC3<=0oe~?6
zS)hhU{=wJBlzdg`GRBKvwti$S0v%_8k&gg=W1MI|S0Wz}*O|U_{WSS~7}&9#ms{^2
zDz|MYLb$QjbY;YFT(6hTEP!X?`V~=2CYo&bK=NIUBB1v3xTq0{>3|^OOO9Ss=nByO
zEcbhPaco{oQKoqd=WiR3X<3@e%+Hy^@TmjO9cJV7W4?yDsRWf`Cu+u0gd-ys1ypEj
zA5d^E0^F35OX^K`CwZ)Dh#xKnV7)vQ+Y?mk31Sd+w|>;wH3s(<A_3~qcqf#(8L{dZ
zDVI3?>x}rN6K+2e{hsmYiOltE5@XE(waAD02AlWFsO?ks{^#%b=437?`-owz2!c2B
zjJ|0&<Ms4N$3pB~VHylyp5Ly>KLZhZY~m*eJs<x#%>Un?*mFzu1HESm4m*(wak?6R
ze-RXVPt+CsMw3TfAc4uWKA3k6!jmS?>^8`38f5H(xTDURUNPwhsxCl?lb_;)6sL+X
zfVDWT-UdY>OG;Q|(;w1QFbHCDKihC-CCD8GmydN;a0w8J?DiBJxRVWeO)K)xPid`R
zU1&-L9xtLs1;Nv^ojKLLMhMfe=#LvRCvai_sx|7y38XwtDE*jOpcS=31%3pgMd54X
zvJ?$@va^5#U71Rmp4HIp7XBvh1Rhg_Rp%Uv2Lbc)p5X;r9!EOQKF``8a%}4rUgfqY
z2wt|rYICn$o8ADl`lFdUuLQ)!f+$67?|XV~Hb}RxG2@Jx91$OuY@cLf+Ac5MIN+2Z
zPjzw2c08U`{3*i~EOE{P<logMX{_J`Uw<-<2k(8%sw*q=0?ag|+Ocu`88qLW^gVQt
z%Ew-uRP|{fy+hdo_8CK4;}E%mFELEpF0pQW!_g6Z<u0@JE+GpIQp)R6el^p-M&t-(
z6Uq~3r*(n7iM8g_idh4@-$ejXYX5s)(MUe;o0SLXtmjTr{IB*AIVZ~u2I^Q#4j-ZI
z3nBPm;C_G>d}Al^mFbymJ(}ZG{NwT4KylR5-l(DHL8kRxVDK^D{Cj2?7ZT!xZmJl1
z)e`4l?nXahWYh>4QXrZe>6Tfv`CJQh>`&58L<XHZtu+ZjRO999$y7{_^+lU!QRmFm
zdmN=%IXQu=vVEy(av5@O{DGtA?rw{bU@fMS012T@j~zl<pC}x8dVsY|S?HIPjPN-e
zz04X7C3(N-vvL(&o(z-sM-va%9_AH~2hoF!Q5um)b7qVEogch0btYqA;|1Rc+dB4W
zg=lsOfclQ#lps?Xr>_e8cd}qGFb6USWb5V4ImRkGt_c{GpBFEsl}qMh9qe@~S!kMG
zX9}qKl4<AM52mlVzj*~wx}PQ~C=t!##x<t)WB%`*bL0&6>fMI~<LJ3Km^Z;xg60DF
zudmI~VCuvz(6TvehWc#Iou6DLi=np$wQ#y7{#d}@1`iq%m~6q3(%2q0=3SdAPl&J<
zB~Al>R(*Sz=C4AaGLS*RhO0``dnxh2T?P9hsHM+;_bbcUc;BuYm8oc73}k21Cbiz2
zJwU*oQ7!ls%NwZDnojAJl}vgjHX6a^Jlx`5JOgZ9IzbdM$ARRExZSpSx@Fkp9z6@r
zw{-3H_O-E4N(NRK38e|&Xdsv#G~1r|4qB--Z_486Rns(iCn3nnLL5G`Uq34YJu0+i
znZ)UXQ{qhUWRyWq=H+(6-Xb0dFvHs)puY@?4+W;_>Hw!Rs}jAf(*O2B_#1uU@iCFF
zZt*vVAAXtUlgW^__YnhP8-kj*Ak(~<vEF=7=qQJ{*x_WEeCD=!fO0ZaAJzT^N3I??
zu5S{R7vs%odP+c-v-ojeZy8t~o9=5*aWNnlWGk$@IN)qC1gv{HUXa<dio4f<=%G2p
zS(Y7oP9I<G(CuaKb9c+5;JKYP1!~9_25Ky3xwnh9l)Dosd~)eZc`}E`Ql6~3_}p!4
zP6=TWWDcQ!2U*bmQ3)ThlkS<R=uI8GV%?BKs2|kvb2i|Q*mD0sMNZj}po1cH1DITY
z8Lxj1k$bgRrcEE_A&yll!pRNIFJ5z0H3apEBv^pQUB$d(oX3K*N?AfpDG70dRCoOg
z-p@8%>x<)q$0?@sl^#5&Cmbi@_X5HCiw%+#2!Hv-_L}+N1tuI{AhhLt)Q`&=Jz$!m
zBIX-?hJQD^0Kx!7E&^*K4#j<IUTgEiVOv(+ImSVWcu+_Dmf5todkj0>&W(mjXDOJ^
zugLw~F&MJNn>%@!&VHZ4_wQcA0p0;YY*&>zS~ks$OWa-^)GIykbf+{jsPmQ6CT$^u
zB!E3MLB9+MD%K^n1Svmv3%Pa@tb|!hT47DmFmKCN@vp&ZPfq+Ad<Uu*ZJ>b7_*N?z
zv#2_VZnguI^fKK=ET;jY{5-J_a-JbYw5~Lul05FJn9qt+@LsxoT#X<PSl>-rArCTB
z)Ia8dD5eIw0w7y){YJt{dTp}`p(gNCWICGalvV|+y@{nqcy($%Yl_PgzBv7Hljl6S
zrzgU}dND{7m*<2NaCkdCYyY@yP#yRlN#1~TMx*V9SJv(N6EZZ!SerUjqV1VYwPS9@
z8a{WIaowv<4rjBm6pT~||Cfcw62)FU{g7bjpDW`pufX(BluQb%zB4BiAr1QV(iyf7
zWbg57-Xdb=2MfX75spVmSBpVD267fd7lXi{Z-@|_dmO!(F#@7>3)e)~H$m8v^i2w8
zR6gXFW*x!5REO4a_vswS`a258Xl3a?$aHO9&I%)e=mdq(x&@vezc~wKxoV4cpsD`x
z+6f5ps?tf`_BMB)J;~?_a~Yj!5DnBWfp7dkPgFwAUb`o3VSav|S%dU8WOhuXlo5eB
z2U=Il6WzgTX;|Kft4MyV9=5*VOcrRCOum-!5hGuM4#8g7{u@}leJdSd(l#F)Slayl
zv9Ep)3Vn|LU4q-I)`9=E1@}MN&uiU$7`B)K-KWDtydv~GW3b=vQt?~#m-Vg;IE%Im
zx*jO)CLsV?U7}7bh1oi~(pjj7bQnR^;JH;Xep$QtWUw0KsTQ6G^ks6(^k;cN`nqhw
zv==h)muOqSmaD&>fuzK&X6ERaqj|mqNF=^LD|ga%UU00}!pm|J6k?jZS3xS&-#p};
z?NCj@D3V(cm_W3iY$gldx0}VV*iM1lZ1Wf>1lI(r&b8kdd`-b29uF0P!0aOHCh^0S
z!hwD4qOPavu)N2duL)+RC`v_bgJ`u~v^)&x+l->GmzUxEe{A+j<&AGFRKycJAC%?0
z0bf)cebokScQzY-K|_VWf2~17b-cNccytZ_|Lo><<z?8-E&aXa9&{WzBIbb`!(!sB
zIIjm`bJ7gjI#?Ie^&_rG3oGG$ub!ErxD2d=7MXpfqPC#rwlzE{1m_WnR(*?+=qMdG
ziHFieTe{g3I?=u?U|#8B%58jbltN$v<kP>GO-za{&n-1+WPa>NXdMdgNQez3;bICL
z5SbcSMlwS(>RCXrU|M-%k)|*RG(P)8D3nxvd&v0_!B2s}AFMXw2G`=ki1;gkEtd7*
zCd^WBWjd4_-169a%7BPYU}8+-77I>+&I*zzopC_~5xPoTat3ecGvf#WA3|U@LM+r{
zU_aZ}ds&7;zfvCd5zX+@IOeSt=gi1=g#3{0YjD!(U4cXmg;sA72p1B+q098pD%9!k
zE``LX%FP?C^LB043MHN`-?0NjW6hrMHT*V~svW!o-)FxqJ?K3j*m!Q=P28*!ixkQI
z>9bl&|BO+6e=+}$1EuEv@qvKso+qseBEMi(63l0cR#%Ik&_33MU5&ZyIvV3deNV>=
zM1=~RWO&q)AHN!AKti<<GX(*}%vzT@OsXq{9@qEym2s~mF$F3(!nb3_P~!swHFAd^
zSas*s>%NCn<XhQb2Ha^o$Xz*l#58|w{FU)u#0Ak72y_LJ7hB<sZn0lu3;wzypt?!D
z(e}U@vpfhq8+R32$kZRc(M{@7LM!ZgF_3IB72m4l*Tcj#?t)7Fp{<RT*|<#QG*GG)
zg!GLJjj-jANa84@-J)mZ2#fB0QY36TzXh!efeV`Do}~_BFCSG2g#%|tA2J!tI6dwQ
z^?~z{00=7F25NrIW<T1Zd-0HH1|pWgg;v&Rru7T7G(1}6X?w@_+^;PSdZUuA_{mn7
zjC<$)+z^FCn2W-8g8~aHbq*{@L5RNF0IHTEXZPi0{yGoDfuJI((0e%LT%^{|C6fCD
zX*{TJ;N)brq}t(rDx|wf1u9HUZ?17v-_+1S>OpA`5f$qO08{(}?87qZ*MW!VTCPd6
z`*3Rny)}p%X~O~JXCC(I-hwueFlOqkZ@+=8LL0=QGi;eaP^aO@HM=C50rqJvkgBj1
z2$Wb~qQ-3kg_k?rH=;qo!T}_v3TNKhkGHgOWR?)Sbf{hthvjgimNk?<e%fD|rL)YV
z_!{U!wHw?o)ph=HNTwJ>OiX)$@H69Af&=)U7U^0!RhNws*|O@U;&K@kn2}s?JVG-A
zt6t;Jw58BVseoe!Vw8kBl|{g3bK~$sr^&YZ7F40qgBbeL)&n2h>svmqS;~>E=Oga$
zC`Tp-%-14Rvh-*&S;@voE{MR{;y&yk4H)M|c8=yyk#pkP_@8YW&90Vf!q!>$=wdzX
zwPCvU>(0Fl6%DHFrHi^KWu3V6(K;6!i{Ix0y*ggg3X_O_<>78$f}Q<wh5_4uYD4vN
z3A(rP0MyF~CF3vg%|79{?feH~=xI6z>b6T<=Uj4McL%W^{a9YO2ltiS`x`Do-B%1(
z@K-GO3)=9)$n8d*H|juj^D-}H_VjQ7caiLQt3Gp{N5-JG$$I;#mox9oB=E=$Ew-vS
zJ!?hb(}~<DTN|O`#q@opU+HfPIV*XfdwB(ZN?<l9$E4s<o~g)ErT``|JWw;ywVJug
zeVeeXG>s25(Vqf0d!4REf)p<Ay_#vG7+$9_d>ZZ1`VVHQ>gkaUBT-0)3_^xufPx4B
zA!|GXz8<fN@KZP~{NzqN1x3!<%gPkorHfa1RTygWMf&_9lNCVudggi^-b6T<*CNY+
z_bqs<;b5}bFINafh}G$A3?qb;2n93c2QHS6lH8zFC3C~%6{3F(Burzk>2#qJ2dLQy
zb@56g1>*(7ddU&xF^Fp%%v;Rd!6y)CljeTC0tM+;npyh6ZzbCYA)rBsv})m?tPrvg
zgZpDPj&(^+Osn#tl%g$i45X3btb4&L3CzVwzUSPG>tiN|fo$v<B991M?(FL;U~dxL
zhwUq&Qo!<v#_`4Ed7U*lM6N0NNR>?&d6jN)Xk?IvUM0EnNWeu2(NU1_QnTz1=hPw=
zXrDVTY!y00th?x53e;s4J@5PbL`<6Kc`@6jhifRGP=+=!0?%2{4X$;kdEiug*p=M=
znW?7+k!Ay@&gmJX_8|q|0&dk@!r(Pvu(?X;-&k{ug-GmDAT>L%6o!D4B@0=6$!!(+
zFiuLXy4ZG90m`ct14`A~jSpC3f;ROib!V5~jKY3%a|?}4$*Z4D+P=4B)y0H$@eQKj
z+6M)(roy)`UEVx;!4#u9B?2x%U^ah#{OsC#-|YEux3aR86oVtK+iW!0sRGz}sUb$?
z=ysn}jK!ccOJAEUce}I9527k=@KOHRdG5>g;jnm?$9{8aA*<Lnn?t)0EOQl#FLwCk
zdAyf$2|EX=sV%K)vm#(&)q?p55iq04?J04w>vwP<EXFrK=k$3+hxNuBB<)D;s^{GA
ztvk3Ckzm!)R7;Kq*AzZ#90caS5GdB%ryz0tesJB=?~s5s3hbP+iQbP-0ylLt<sd@)
z^wzz6qR<56J_~7q(ib?*%S3=8P}N7;x!e_x=6L44#8MW!6c12D*<y5YanY5&(H<p;
zQUC~}GzqT0QZ<cJ^ca6sifF~Dk&a~`>+~&bEidKP^|?#F30Y!d|6rJ>hu9%9v?=ev
zo(O^THh1J*;+Wi*-6hE;nVPs5ML|HfrhV3?X4WwF1d%*x^Zk%H(r@6J^h1e!CX$rQ
z(rwWU(3D2d4$}PA&X_;5pk`RFATG0PbkfaBAUS>|sO8qvHwHJOPH8QpK}~=#-DXqE
zq@LdGHAH~HACiBI9f{E=gGhbl&@IN@jvbt#_yefKZ#s+d-MU+^K&o=WbBnR==7ex@
zZ~kz{^-{(n{L?D8TLe=kb>>MTc0{Fd3XBxCoow3#7xX}rD%;>uA!csvvl+N=pFT<t
zkb7&%v(OGFAJ%(i0dndlW32064y`E^XEs-0d1Q-6oH3$41UzIsL84m)209E!UBm6q
zDKpuK&)R0<Qe|N?(#iV#Kka>aJe6zPHX>7l3`r5mm?9}crXrM~L78c1&Xjqcl_|rD
zl36IViO9@CMaIk_V@fP7tc)w;`p#RH+Rxt4`#$gQ`@Y}r`{Ubx>}|7#>%Q*mKF8xY
z4_T<yovTERKX!Rp1<?zq=GO!q6v8pnGcbzjQl+(x02E#5-SY#Jt5Wz;Tz)Vx^5lF0
z85-;3Ivb6QSzT)z4(7qtB!Jl83TUY>sI4y>2YzP^gf)gqTbzp~pBKc>@ThvXr_^WO
zuenIqW*pU?t|-|uh&}C&CqP(G57knv(lcs%e_!bd`Ti{RRz97sFKb6X2P)GYKmR{{
z5Ln5UfdKTckPfzOYo_3y1N5zaR|m=$*<P^>kPb`l8{6fI(Kqk943{Vk`6YSmt&LEo
zdwD;pnPe%06*5W-eVB3F4K_X-s+Hvjy&-pbSejn`0g}#=x)SKRPI@7OkB{h=IaW%5
z6@S^Z3e&6~vS=mdf`v1hJORDH#^S*#pYt&I@&(xA{)IHNX11ndBK03n1fad9TBr7*
zuwYmwP6sNVdD+Z1wcB-gOc2-29jkW}nUH$I1q=1+ikBc()wYPZeu-p*Aec#aCo9O{
za4UDWCyG<ysDqiO;vmd1eBERBzLV~#$g{q98K25@?@@m@dOwrt7Mr+HD#Mj~R@lcu
z2X9$4uYqGml^2Y|K*d{e^4ptsgcdiG*m|H_9+B@htm~}W6HWI=9Qk&|weSAZX7y2r
z9xCsrp2C5x++~|TFA<mN<hmgm`Ksxg3Iu^?n$9QV7kUhF&%{8_&u~HIO~8|>u!uX~
zTaYGO6Bo&yMHBj<H(BR$ZP5!4bB*Sw`g=~OpUrFT<X~NJlG_RAo1*EAA7?>sJ%OT3
z%4(lm&WKyI`OP(m+I5begbsYSrbmJI{O~PEr969fvdgUAqL)4zO=aQ*NXulO6a&J_
zo0a+yO)p#gw*<(0^l^g*)~PtRPdz04+r(6QhDqy2bha{_i4f9d#R)r}WV?-6&v&2F
zA3LC*H20j1$SmOS_1#T6%{a0xgp+7<uY%3J;!+x5({KA!+-1%DgNggoxTVuBkxkpL
zKhc%N4ZTojcm@NnACZ1dbI~=-Hx79u*PX1sL&@d98ml^P0EA)mk+EknN3LB^q4&oO
z4lYE@#EDLnKH?$~4xxOU$%MH|`=+O6$HSIqod~Y&<gG5HBO1>*OuxJG<;?5Bjd!Ys
zC$+Z6t0!Aq#4*UzE8bsuKOG1XnVwuC#TmNXpywV9J{%DRHdzPJk>kmeLq>NCH_+do
zjOL@s_(OS8KIPjDdgikR-;C(D8-vAroWkWiuA`8W^H|-jjn@-XVGs$${19n=&NI)S
zD+f~xnI@(e=@V?@x#)#MG;ZF{9tCScRa@+JrBvc7m-;A8B=2X;|HxC(lUx0!B|?6a
z=DXY_l`WY+*<CbkG8YCFc9&T<FD4m-><EeNH{XvyZruQJ{a`+Bn0fd9w!vGE@T()s
zLn(LpO$q51`}arH233oO2?=HriD=5y@HwI37k<8vTp2Lx9Qr%^h<pbsa%~W)_xE%%
zCn|R{-bKvwbIL(c%+$rf2Y<BweCOy>S-PQ>fVR6<=;dBh?1QfrM9`u(oS}xs%Riy%
ze$@yQ@zF)Ob3}};pD!(3S+x+|`%TiGUjZ#FiOsFeI$)@>OP^)&C)illhK76uEkk8p
z-<=g{U1Xr$vVm;}RPueMRG*k^)`g#+NOcb&pMIr`C&`H{8mba-Fchm~?PoSI+Dhff
zC7}NQCH%j87n-im6cLOH@uEdF%v5g{!iO*X5(=1LNCnP^WyDW@oopzO8<h4OE#KnJ
z!|WrW#{B8~_GdRu(rsTd&&$rfzQTV}qH!$qi?yjHMEeDVYY!iKa~6#|*aSJ|u&2Mk
z&6{9IUv}Ujtnyv{MTL|N6lx6qMZWZNq%NrZu%gs?*WLDF&)(&C6ZgkXZ$3@ueDUiy
z^nx@BIWb<X8cAX+TWj1p{XbR2g^uwOA4n?3mn4QUOvm}kf00=0wfwwAR67^auf<yY
zP&R$?HQx^*F!>f&!?+9~&^0Tj1#t~=1d;3!2P3}tR7mHN+1+t2mH}?9yV3Y^YomXj
zL3jt5K{!(4pt1R6Y$B71PfoptpMdG3Ro^rCZkf(zkZiYuiT;W7r=f9%OqHZ`Y(b#D
zpaZ$=yD0aynXVVgLadH$a?VDUUC$0d-cb;1^d1_%21u$TzKlsnkh51{==^g7bQ#Pt
zl|wYNwO`)wLn_WE{&+*K4Yog9zjgqk2!@I77YPNn!v@HFOh5-bfGn{A%4g@SlOTOR
z14H;{VM=iAfcoe=SPG(gby$=8Ox5j?vk=gQrYBEd#j_<NGE_np-_N=VKkfl#$U7~X
z0w6A^2$)@Nm^h+p&d%6!o7Ahd2k%|1JUq96{x_=XB*0OlOAV8|O1y2v4f?rX9NQ3n
zjc=I74^>3qRDp6Ydwl^+oPT~bv&C=Hzl?=53Wk}6fY$+(!hOl<l5<C{{mF<|fjRqJ
zNUD1V)bH{^aI@s=TZ;M9fGSMc5TnTm_I|<Wu?%B!M<JMxfg$aD&T_AiNyHBjtjRIv
zEZ^jdpo?GvN*>O_s>ZG`d=--6_L>mLQ6;-;b@_A(o~?qW!7c=XbqOH$UtO<#Ojbn(
znRI>_)%*L!_$4AUa(D$rc!`4-HlH9OjhTYy_EIdTo5G-r*k@yhFbl#q+d-E%eO7dQ
z!Q_6BlJ>d?wr*!=d9BOQPVNT~$wugZ9zq5=<JF{!DgdY20drRBeBC3{>=B7?Onds*
zKnAt%M?RMsP-3xHch1|a<URFQf~q>?r&kb#X7&qSFp8Vu0xMN+SH>MkweK){p^Gd!
z&jVQeR_q7kK!8A~coWuKJlI6UIeyOxi9@%L$_8=$P~+f$=b6k=*nDfQ+poVNKN<U0
zf9+KDmlej}jWGWMflAc&q3J9g<iIiNl{8YgL_nFf0#t_5T=HEK0D!u8KdT{NZ27PK
z8KaA8WBZQ&79@Z~BnSXpqE|YuA^+$EsF+rOK5_B~&>c#p>;mbIX!17*QYi!4y(0>S
zxd`0poccK`W~lc>B4iF)q1ku=GPhb)|7zxGvn(XlZT)9Hu`irGs<zrx{@<_i*EL0v
zlKrgLc$`bI=Fj&a=w}bo=7}<sr!kNtRT9X}FJiiitv+(LO0QU*b>d5mBZC$NrrI0k
z{3OD1cDt?4)SZ(7CGp+XN3LW9cjX!yBK?v5&kVC4iR4YhTvbECTaVg{MJyq>_7?aU
z&{)1m0GTz$Kk<+_HbQM<6s{{42m|^yWfMI7m)|T^*+IMDO0I!adj>@@KlI=!XCk4%
z{ldK237Hp;8C*M@Debi!NPp-J7TPY{@h9`V*OtHkS&|D`!u`XMXF3GMZd8-B8wr1X
z<cp-ARs9+Nd%NU)<Vy2e&cLjmB}#vK+aDsIn$2&L(#Z2t9e-IuV)yE{CTt6Ggg4Os
zH1Vu$kp6?DNCaBYeGGVTF(Ce~WMGhtRyHbtF4Lm@s0uQoeXccWTbgFJ)*&hP`KXD`
z_g6Lao*G4npGUN1Eda`BgoV-(AGDYpvT23j%gkqABS5(!PuX0iBN|Z%CK6Qu3h&JI
z^{B&MQ<^EM0A7~qHCYIu0$@?evu#%+s2K&0**{nG@5zx-NS5TTlRd5f59Zu26;T_U
z7m#21`*V+&GY&1XSWnI?756+6l4Y(L*RfB7wD(nH&`Q@==}*zBTyi%6b#D*V<-WSr
z{^4~V$Z-qMlcz*LiM$kwKvVDC@jh(wx!Z=*Gr{m(0QSb@hxkCys^5kTw@0^FKZU0^
zO?@w)c76uh4eZtxF!mne_B<ft<(`N!4u#%(BVgVSd?asOLiGN?@JV}=^PD&TOn!A2
zO$?ua{fq{lVYu|#sLu-F`T*?~EAuZG<Tp<Pgkl5Rh%JZu+tUxvMVw%pOTJQr{j}N4
z8w<{Uu~N{)RHM$2kV^#^MOr}K+vr`Nl7)@Vy9$pX)1BG2l{mHGK^;I3+m)lQ3)B7?
z`ijh254QV#8BxZ)9D=IVbM048HU894Mhcx(tFvLyR9D`6OovhMVV5OvW6o{sz<$W$
zRXpgHp^>e18Jrb2d~P&QaJVW6P(-*0K6FwN12`|40vYSXp=(vpH?@C5&ATDV0@#zC
zY(~J2nT2WF4M0_imY{s00nmSojtph<Vd!4Qf<{4Os|?|&(Yt%p+IL(g#Xz6oMit0G
z-Y#s*DiU{3O0TY@DAcQP%`=MLHPA!4ii&>s0YMIEHXxelh&C4+wqh?-s0CThxV}Vd
zGtljlz@)a7_6y+O!^V7q6gU=&lJBG^!D5_iV3Ks1F6eQ4zo7!<BQS#CMrWbVCRtH$
zDM-DM<&>)~C+FhU{#p5fTt(J)=iA-?7MqSBc{Z9qKSKP%q+ezXAH}D11l_>Y9j9!3
zL{2Iq=uYUW@*$9gheL@3H(VMg5`cdJmiV32dC1aP7aT{xUCw8lO<%W-g_5$zvqTn*
z4U_@i8?Z~ITf4!G_-wcdZqq$j5Aewsv`jA*o6zq_bHQpQBaML}xACPR0;ou^E(ihc
z|I*+P+CvwaOm3t*|K|J1+ZP~1+l$YeA5eJ{c*<s=1f&1439@Y(Ynh3R=(8)QsL2*Z
zlaK%hJJamzq$sY>2Q4K_ZLu$ibmM*r$7gF-0cYmv1iLtS`W^6<jz*K;&rR%KcS=%b
zwd?Akh~^s60i@39#$5+Qn+p?r@;<-F1#O=v-KeVooc?_99Cm65Etwy0a-aWmcUaHC
zA~OsgwLCPvP?_uH&+4lFg;nIFbBOA?^sffM@32Ncy$Dk2wom8&6XZoT1<rjp<56Ai
zSQ)Fafy0$n%1-{WFLVlzI*2*ek(fHT{CE^lE1>c0#tW^yH<!(z`|+?;w`#nw^YzGx
zG6T7q^pde9*tU^t!iI&OP(zc#v#p7_fDO?Z%UKh~v!G`Hqx&k9k)IyF55VHD(*NY4
zdQ?6D!Hdpub-|Y)vh}j0uHB)76w}^fu+nX5K0Rr>q)K;OowmMMV2+W}a4K|hQKu3l
zIoRsjjOlXoZLsedSZ0&dfdHk4M}0#|m!Mxfc{&9izna2hsBuKYQQ{jCrrBZTyUnDV
zmKkB|z%6_E2NTws%{~^6_t6jk>OF29#-AYEQsHXKQ=VG?EfUlhw+|!r8aiC1t1e#_
z(ZkJaQE@_4dO7-kGLWCl5iG-S)^V8XyTVSK$*p_UDvjkMz&f>-Px@*J?~mh1eZVZJ
zhv`+Upm^Z}<g%JGcMlpyGef^5!L{6B&>JefHG=@wsfLEcx3XL<0fgoU8zniFWy#qI
zBqM7Wx8HfW7E4Vg@xy>Yh%*zo&RR&Ra-gWTtOeEYNa8hm658y6crS1*TJR1p%y_(I
zP97BLC(0rnxu~G&!ZGWZ{p4b7SrK=~4Je{UNO<2onlC-4-JMCqz?4)_W*F-m*b)1-
zmDuo8-M{wk(xc73?MK0ga2aA75x_2?lOCO-LjP<XI6moe<d(R!OG%<ZCT@<xJ)GhB
zZRj@Lo8r&^<N1nc_gIXtGF=p+^FD61oj<I!AC1<#Bm(h&G^9XvLxVui97Tc>P<VM`
zkSPKP*cc!qw&qPE@~NQJ-jOHh{(8sv*w3|dgen-ASA1`u@Y&|UuO{H$t^m)_qCM@7
zp3{eGH@62!I?sW#trxCiJ#1Q2Xyi5eM&D(Sdn{rdm1T_Z^CD6_XG0KmnLn?;I09{C
z1I#{@^$GQpbPi@-AAy05R25;Z5HtwcKajHB-rEE{cR|2PCuJK}88ox4Jpl@vJv8b2
zO&rg+E&-O0L5e-s0y;V!Kvb+7gXl1Xg$i2IDtqQ)D1`$EwP!-)w(nx)gQkp2%uXSA
z;W=pTvluv5RDyogEVO+-<k^6pV_MG~WQH!;>ZXSzoIgx~YOJ!UrXBX#MZ)^pls(UD
z1w_f&qm;u`rQ<F@7xkK>c_s%ZoRE4fG=Ad1D>XqYRnVj{CQs9HH9k|fx2bvbk!)=l
zr&L{8%KY6F`FGAP!`Is^SKiMKec5OYZdHt6hD{pTycEOM9C$y*?@6jRbkw!~OORYJ
zX*g-FWWzW2n_T5P%OYnh*B=}(#g{VlSLWJEBy*$^ECEGhwFaU%5#>p(31Y2wb<XJW
zRY)>eO@_Si`iA55Yk+rbfUv|E2=k4F>p+;XotR0_1P(^Y!`+0cdpRN?p#xg+CEXKk
zj{{8T53vT7e1Ai$-*gwxzDDkM+r;bhS)>Py{l@k^op(^(HKjcllIuvpgJrr1sZ|gA
zC}TSo1Q%L06*XO@08i@%%44P7?GH@p%1!8xW%H=UYnjScVC~ZT+0a(oTO=24>384n
zQ=j)9eAqcdPAWA$KheoH?$~c#?3R2L(l$1eQ=)@?EoH3BEtfX?xk^1A>6!;H9#3)m
zGq5@itU15Y0vMf<KJIo}$?f9O!b*sntR}4^QKvhe()Z7r7Ev!rTskg1YIeCZ*5l5c
zwEhxb<Y+yyoT&8S%Rr>)h&{@%%4s8|53ZzL0fAFvKz~!SJd#_;hsJn`Q?svW+-bYC
zbiGmi;e`f@9zBO}w0^%GD1e6+@ehqIl9NWWa+uyIH~RoYkGs<?27sC@Z~ig!<q2fh
z5CPv~Nfi<?YhgrpmaD;Ub`Cz5dDsJ`{tTeUEZVF?BP&$;PPL`IhkrG+4Q2;JWb!MA
z?A8Hbv9pl7Vm;0*83jTA9cH%NW%59bC^Qmy=3^)nUkBfTHZc+m)pFU%XETK?kdq!e
zy%~qHIi)do*+QH)iwFuWC&=H&aON~E2L?jY|FIuD)M4nnXs7_Dtxa?#q^Z6L6U{KM
zyf^PDlqMSBp@FH;5dC5JcP)kn$Bn!VYGTsV`y;m$OvqOdQ7(*)YmoDM8z1b*GUENY
z>iu4~xhr^U3iQKDGr6$oVZ*kS<Q3gUR$|H-b&zy9dBSdt?`LwBaS5_W#?Sed#Q2;e
zi~L90(JVn;-k;O0NysQTs+mnFd@*@psr|`Ifh_3$X7mhhU4PUJ=hr#?Kr11fteu*y
zAu)Ydfgu!5a{p`(hzB`Ew*4gfz_V`Suw$D%n2cyo<)RmLkQf#qox7T4a20bmyW7Jf
z&&&oR?R3G+fEIu&SP8LX?v<8T?SMdI=sjp-@xgqI!q{{(NDcw*z^};Wq%lB#52_7e
zyKe4QkLFc>xH#1x1otDiVm34pP!)?3r><jNqe~Oe1JEgtxCwxVf|vUdRM3?I^UI3*
z#S~J*AP&`l*gbGk&vo~B=mVIjH(cq)Q|OD|Ob*`j>K)R@hFmGX>5(FNu-k=ml^N7)
zl1S3P>fHN>^G8S@8v(IxA|w(WKo(TYxP(1C&0Bf%ZhgOIxvc=EP!7k=G{5xJ!Ke4>
z)dVBrtSBC;Hkro2EcpXJnf}52Q0e+rKZi`em&}vS`08=u##057W7wYv()ru;&&Z+r
zrUqIGZ*}FpSF61aSUOzb7=$E+av9P0La<npmiBT03TDGXQN^SbY{=A!8B^Faq@@bw
zF!zdEh?MB*>2Epe6nwxz9dP%iHJ9X!W_7&n4eeT~EJa?b<~R}Fc}9v>niWyZCkOOH
zNGBF^Lnv&brL8|fwp=W>gr2M9Q8V}8<dvZ&(=ni|#72`N>RpB*WIATRO|$=K659ZA
zucPpAOgJp>{xaTfz-}dFJx!f@pYR(u%^%U~(%MheQ*07e0d;Yu+}YYAE{>Jy*@_P*
z+q9F+k_mfQg-n~5u3YX;cfzyOR<Wd&x1vdXti;0l2|3^O%RGAAeZd<w6j>E0o!p##
zxo4~TRlWC907rs9R2(-8VO&V_+>g$&zbHUqzoWAZ!Q!04SS|aRDTBDek6d&~8*pLp
zj91TQ_97BzV4}TxEC8D7IorT{KQ?w;XB>b_yLN^Sm)NCaor@p9S+8StxCe~-o)^!I
z2B9Z+1OTnh*;x<}X9NBwPq5oRIjEZ-0G6zv;2OcMDq8~b(39VwN>d}iK9rY?12#Xq
zK@60{E`nAO;@w~p!Rh-Pt3z&;Abp%h&4}uG54`^$KahOd5!<bbNQkzd=7Tn+Ycv)M
zvC%QJM7z7#1-@o-((w0A*zVGzkCtCSdkIlun(4F3tpeZs6;ZQ_<yA{W-j^5aG-gOx
zjvWBz%wk#4L3gDMe~WCsxnqP9cFk_&lGf?V#*OPG_ErLzJJ2_$S@!HKm9gr4{yvYm
znGA3sP2OBf3;$_6Y8at5$FqC$JR=)!MJi2|&;Mv^P}f<c^TtdiS8VN#uDA0VX!RLm
zfH6+vmEncr+_3q_p<v<$#`}uRZ5R^JD6>8j=~<U|^y0~p8?&PgOGULhdw>K1(o6^z
zBm2l=5$G%{0RVc+L?&PeplVczplSv{H`81kBC~CA^(H(+z61>K24^`(t-ZXP_ry`{
z{2Bu`0XdGoKSA~LvlxpJbkEYp4<@f_-Dx^s^^or}r*Wpb3{?IUG5p)wO&r~Rd>^s@
z`T)2VjKC6TS72_qD1?s5)sxG11)SFtke*-6xgyRXBk}QEO6sNOSGen<8BemvH{HpE
ze7gx$KldAqp3<aHE72Iu$Qo?*UQ|L&>@NMkVi!i~QKmg5DTG8}&J-EU#;#rh)P%%o
zPmpJhcs)i#=(EgK1}$cW%I)m|Lt%{VxlF67h*YY^3!M*wtd?ycbV@lhY~OUu4(Dx)
zJARV~%A`ni=$IX>U(#|=3D}>t1jywb2W{sYA!b(sRLJCqyzI%d(q=%a7DViKA`qM-
z_QL>}T~e2Cf*4@L0?;sT$9%xfR&U#Vx{<N#ET9$(-nX|v^!NaZQh{oqL@g1)>ukRI
z&8ZhiB63NBD-9Zc54R~qi1(luL=G}9E&5FN)3{r=tte7>RZa}B7fp1HX>Ar1fjs~q
znvo;z=YKxtH=({l7ag>DDv{z!5%kwSGB$73Sq?$R?>TK?YQN~|iy2e;)8kc>MkxGy
z*|Sd*nfU;Ui<E5VcYQ}Rc=ZCnEAXG(=LyoY5eqYWJ0KHN*DM*^A6b*&6+QqnH$b$C
zdp@0-0?_&e3F`j7sv{E7v9Y`$yPGi&T%5c$5Q>Y2N4Gc62C<2dy??~c_hA^0UlD#H
zDaWx)eE<QB3FMS;I^OCr@OZHbN}Yx3bD+q0AAtlLnDPWsNW|AvfyF0%I&duicG^mj
zzrlg|^~-W8wi5UtYl%Yfy_^2r*dH2)DkT>RewskPM^GNC`Gs78jtqh(5~UC!)?#Iq
zz=D1wJA^4Z6(5qg6u7dKtIu&FCvqc=k^O_4N(H8@07QQB7Mu9KzCz!oMhv+h)E7>l
zlpTKZ%;<r%g)HS<EdSMa1~@0pRiu6jA*OL`XB;4E8(;qM))?ZP$K{o|y2UMaym*_z
zvIfBD+VoK0^hbt9$>YxA%$A`7n*lr!=gUF@<+Ai5P<M&}z?o#S#wXDqgk+CtRKHF6
zdujinKMZjNDfL~z&Q-i0D*fOFROptRQPbppi2ns=HRX(1K*}0g54n**z<FwMH(8GZ
zc>3Lw`OTv*W$`G^2NCv^t}C}tOd<otO!IyJr_I_ivuPdgAy@p|@Iq!ip)Xj=W1#qz
zs2MrUVYyHFO@a)nWWlGBV4}q=@DOW`eD2)~1_iY08^MH6@qH;ub-@z=C&8A`7%L<M
z<w^VO&O6|343-zCEOUX18HbUzQKkPLzv0Bui}7f2n5l*B$&CfnY_Ih#(8=U`{n}te
zINhnhW_0uc$89K#Uj#DXR2rgRZH2&qBgE3x_xynBs3~*j>2Ni^4FzhbX})SYs#kIe
zH2I8f7wF$2+~WztEMXRQsUt;U2#@YCPxUxL?YIAAB?Pu6#g747@R%zGj=WOd&Dyir
zQ-=^-WWHsOU^%OZ_>oHhD7ddKANl}x`>PcUKrQ*dVeYa5D1_EDGR&!60`jEfMUOfl
ziNYQw7k^nu63)2<^+>J;W$N0ZufZ@CuFORH3$72c^-8Zqh>tz;+}1Hgc3A|RH?ZVT
zFxW=nL1Pu-@aqP9KpBsz?5fB5ea>Oex$u(~9W_IapswE7n`bm-QXB`dzy94FEH`aw
z&H^t&E%EjnYUhNUvTsM+03kYQk{bt=)!<$8k%<-b<iNc@3Q7B`u;sjb+62fJ+|CPD
zt=fBhK>A7mh%SzGg0m~SM1P>=^p_oO;PY6&R_sOQ@F2?VMh*}UDD)qc0y7~SNdU1#
z%^ZlC%lXd*?Ro%6xyE|bnBdIHQy3v92mQ;>-#K$jk6wCQq?EhXptLJ=P*OfwgkaeD
zfN|gaI%*s5+r2N}zob3yKpZUf6>-LD&7?oQ`KD}yDPhYp6j1@}p-4};W!q*rMRNgc
zg`D!(usTm~@?5^s*SkyoIhId%GR5J`5PNY?7VJ7uMvTE%BXHTiN=QlR(IRELcnA=D
zWN18JbE)bkU6*JZZUt9*90<Q{B(ttg3pO(d6jpx{V(^EZYLakD6F)+D6(gY8V(<YF
zGI3D#L{EyQ2Yirpo;2Usc<FZ86f3%9$F|a;86UuLja<vgl~5vpdbNYq+y%Peh%6AT
z9?in`UhaAWboj3@$E&bs2)KB&DTLKJgYGymI1Mmm;&X5u^a4Vq_q=YM2h`$KK)&xP
z#{jwDoI1nHqfG|P$^7@o{SZgaoAOBV4GzJ_c`N~XN<RA7o2yXGH3lZ%h=PHGb}DK~
zrhECii{J2gScSLWN|8qP`@r?fM_MG=R%%4q8d-_kctRNoTJ?|h{TwXli)duz9Q{qL
zqMjFQ^pcVmaV)I3AQH5aMo|}K4shDfulrT+y2ma<zx3U-?#Nzybq;q9Jj`?%g+k>;
z>Kedq2|+>g6&~#{t|mTH!=OMZjdcWWV-t)5GPCW81E^l`uFsKT3cOT5o@a&7K!|8J
zxeRRj7^v0t<q1~21S;ZlNQPa79fMY2j#aloL~?sk)$+Hjh!1km`IMWHI4y1eZMdom
zAr_&sSm&VLdS6}UlsZ>PJRJHo%3*A(2V3vKsXc||ZE5d5K`-ocFZ3qLJ$o+Do&omv
z34oR<enb-IWZ6$zEQP6dVqVLR{MWM|p%q%yDZiGh-a*04m{ZGK;gIr|ol-ve_%YnC
zS+Ip=HfAOf)Q%;F88#mH==IlUv_D-PCtQdAM}a8AvLc1!pBBSAi!6A@eK*fB-EHvo
zq*byfH{m@$5RWL9MpQ-vj|jLWI=iiY<Y@{FySNmMJwJmLHj!u6XLm`;f}qH=d2R=@
zxg^~|Fz3AF5CgQq{0~I`n9aqb3gROZ0NEEgKraP>KJVb(8K}7qm8y;sJ0oqF{NVZk
zo2Bbt4X5ZU!~za%M~AXBh1`}WGM^y2^Xrau<<jqO0HJ2FtUhkH^bc7HN6d{}omZye
zn5*PK^Dxb&ZN;poxQ(&YF;F_{(gByUy0l1q)?(-Kyf2XL>!t7m(KuzYWyT=hWLM7N
zQp}PoH&xg3tBOE|=YYJT=jLN&%6v=CH_&D0Kso`rMTUiwP&POOoYa>_pDGRNka7i>
zqRIS`xO2P2VcPHES&45&NOb|sjtdfQmzZM;fj?poFuf)O)B#DuGoyYAW<3rVS>g!2
zLB{hSlDmS0Ex=beF|eZ!3l@`2o#D2X@u6b!(AYht-@Q;bS5m4!ll-_5Ro~GNagARe
z^Ht@ZJ*%aadL1V%HI4DvVUgEY(>6!rQE}Xsg*x5t;K;_GH<KLcadYTV^ljZOh<6NQ
zeoH|(qSO!`N0KwoywpY3qa8hG_*0dV%iqSl=A<3ANBjd_uo_)k^g2Aj^b&XoGZ1jH
z6c(f3SM}pFtwmqKbN%xv`%_))dNQfx)eyjPbNs0$;g_BWNEiB8^*5Ks2I<M)+>u_H
z6p0hHj$egL_)6CGV@Of9z&3@T)-Q2epEA-zRl-%DGFed#{==|yezjmHlqzBpfn1E_
z-ST^1jR!<=q=wJzca`>8Uyb^ZrVw^EfN+ZREV_aR7{AAI)wyna><{4KMO4P~U1x4C
zL6mgX<#n!MRV9XG4WZ*pb^)xE6Jl}C%7qao!jC$c(IF4qPNBZsGTZPc0Jgs_YBRtG
z?uRxh?48#9{Lj#TXj?3BHw(Pee0bw8rP*aR?DNiq7EdFh=d7cbQlF|!rb49@(86g=
zS$<B-L*C#x0q1PxJlxXPOkW>ZX=WPIp+n53k%5~Vc{QIgB;GVjWWOn286kBeBSEH}
zTZp0)1{_zHlamfIJ)aP+c3IFv&3K8dU%5@zKwH91#Z^`e(Y6u{HId31M0h{E6c`c<
z^wp|eL(U2F4P*!k#i&z(S)WS?J%8UwfAlO;Ho_b{enZw6^=QuBWf=;5jbJU#G{Gz(
zow4S)!;l9r9A!YuF;H?+_EK^~&oNmB#Mr_68>J;omS&G5WAEBuua59v_eT_XzJf$M
z3fOCbVZ5u$FgXX<9(e@C38!|_bpXa*a3$!BK%BNyJR2~quPSx@ea%Mp9q3Xw`hd$&
zzt$!n1Vc(B{={_do8NEqF~2r1`3@`7=@kmpk?=wF^;jYKp&3F|n-_2P&U>;?=cTr7
zJHG`mB<`Xt2L_B2`5I08mUa^d9{|Jl#t6tsZsy!1ev{1423uBGW4$KP#|CvH&ghR8
z__(oOgRaeGUw&SSD=<+&eP^58>@uU@cSph+K5S@V(DKOTv_Q{=51J9Lu?Ix0qKqP?
zt^?SHx#t!{Kp=IDLLF9YB&-b+N*_k*i93v`z-YTjjQ-pg)#X4Eul;_zS|Y@}evAY?
zt_g<fb(i4&80cF?CBIc<_hPg!Jb)2SS2x5Al)kC}C(%(dv+h@)2S!R0<UwzB@j;j#
zR+l`eiR&x82>i@LVuOy+5SX%0-{I`u%YQ|F9iSy)?;h@YgO^0ZgtW)*FlaSg`^ifv
z)LmXSrd5p!Nd$@#2PBK(dbOT8H0C#bd8k}fqiq?qC)XKia*t&1ez~!wDoUNiUE17I
z*|gR_NJv=ZmF49$06^eM_RxY}X5U|Dq|`+C^-Ud}S$-{8^!uDZXcEB)vP;<++{`{!
z%2nFaGMR(XN#m+#R?2dKtDWeS+6g`PGm$3SYb2dzeAYd)U|3Z0*SEvlPFi+1p9rX7
za|+tQ@>#nT>62cEl7%Cy$jiRv$ycqc{f@8C)K+&C?KpU*_e2A2^2*L)-q}K6l66D=
zWF042KHi_3qg8VIvD4d~Vp_nXxDH5^7Z7Xremx&~kZn#2DRb*zH$&!=Jo@@}N@6tX
z9D-=?cT&v{$aNo^Ug<_k!d`Imod@e#95lVKVD5^V*(T4eh^X%g9<aKWYcSYj2S&s!
zu7<`E>H!hUHnh;7dbR(g`6tV&k-6Pfq7e5kLtlN+8LL$c5kb?Amw2yx5d02amWjIq
zmXL?l(%qo11=Dcm+H05ox1trM6E63b4^A(v=it`*C_k;yZTlP1kz;E@qo41)jkdth
zKeq+Iz$;T1?C0tziS@W<8ChQ<&58XyEH&swnXQoeogyuT<Hq)xFqQ!uv24I>Ab>qY
z_u2%W;u9W=kdT{%p`|{uh{zhKogN`?mmErv8_0)wWPr|&7-Tx8qoI;7Yo5fyA;cs8
zI!}s^cG)oy+Gu((n3&;Ky7UXV)>^KEj9Lr}Wr#L5WUvhG{&J=EUyb2DJkAuuFtDoF
z-?17fnemOxR2MZLu0(fbYhRtWp#PS;k}T<BgE0fEOX<18*;n>4E$bD-j1_R49)zk<
z%hrKB&J?LA+f%<y0RHXFQiVb+C|62fK#UcC`x|0tPpGG!wMH4_#yevbUkG-BTA!`q
z6auw0zAIZn?$>;pRtLewadF&+y4tCDH;v<BwFmmI8oxaeIhDORNqfuMY*=sDOwe#!
zwUnj(m#^CVUyf)BiFL({#pgq2-`B%Ao1VJcn$ePxBI;k&%t22<I_foTmyJNj%<$04
z2fTxqq1ivn4^$(T?Z@Y~lYjFa8xEwN#P8g<5Jz_FgPc+0L%r5mc>_sxdBZrm`s3hP
zPE7i@>w%t>LF)V=PAP*u?V806eTPrv6xWw=+vsZlqx_ug4Fq=9{3P`dF`?Ng5pVq~
zDv#`iEt7>~gv*@Iy58uZ<$LF*a*$J|N`+fMTETY`gek9Z+^9UUhJDK!R6IX~ujo;D
zx^v-|?DWRMxBf+j3{I96B!9p4o4;(R-;TX)<vUYW?ka`k5Ah0PEl{~BGstI=AwoVR
zLNuypN&Nhoe>5?1s^-oQ;UZ+@-#LD<kR3YvyMFbS!=*yRUrEdMog``u{N>BwplE&S
zom0>mpj#oAX5l<Rc;5?})v~`K@uK$o-PKp<8`$G9Y`BAA$y;Oc#O4_Pt?&4GAsigY
zof|#dw&UO2IY-htJIhQ(g<GO{={@kMQi_EgUFqG}$|zarAU;vrvv%YU9OThlpwDOK
ziqeD+;&?@ItC(S_YDFP*)Geh>s8eBx<wlY2&@UZ>zudLoSH2bdZB_GN1o;S=$&rwG
zSA`4>1sp>=m!AdUvohJ%-f=9DEsy$k7uTxo`r&}Cw;T_#(oa}kyiNX(mqerFOYTSJ
zZ!GqgC2SoWUs<Kp>jJ1w_E4!$muOqubw;s!Oz%q42MzrAsZc1ZL*Cp2>auzn6A$tt
zpHh3_^o8<QU-kYweTi!1p!QNc{weJAw=a#NAXCIMihmvZ!8<!Q&Utlvn3XPb)t0>U
zr)u&G?3^9WFY8B+eZh@)t|)lBp7JhreDLf;--=RGrrU2!S=hP|nl`+O22uHrTn#R@
zTAzOZKOr}x_CW0Ma8K;>O9$LKK5+8t?XRMaXFWb;2v*>o&_(La%~MiKU3|{@!WP+f
z*G{q4kY-O#gXi-$II*S`*S87o|NGzHcHrakb=3LsAC~XH;u)Ud%C|vP%@TD^{C<!H
zbunAmp_Rf-XPRz>xaB6)pdYU{K7GaL)#`Z5;g=+BF$KnBY;j?Ha-#y0e-^GiyX0Qt
zuT=22d)!9>;j7cA)OiowO5xB@zkgXRE}Fbxt@|K~8yD6rygh4t`@%i*O^LbJ%+ETs
zM)!5vKI^%hp_E*;xP7L9bmU<te#b?|Rm-9UDO!VIsZHd*Yz}oBzPV7KWohFV#*g7i
z)^@!t&P_aw9Nfsh(_eQc3J<Uxg^ug<a@UXM9**j$>)Y0LU((Q5rTZ~*4fP}%^pRDQ
z#5^`7Y;}=5>$yXGa3$^*j5Jkdn3wjrja$hXmY5KX4PQ%lJmAi@TUY3prgRRlbQ(Vh
zw7PAKe>X-Ui*XbdXWClK6KF!lTrqGaYU8o+A;k#hW4hLsh4dDZS{WL(HIvS3{Q0XQ
zC?><u>}4I}FSBU%{Yq9sF`J88A)jv``Dz)Wu@a^f9<4BsF4B5l(=yk(0EZdpm8+U<
z>VDD8lE8-!omPC?J$MH+1vVb?sB)-K+2i{h#x{PTjZgu)^)9Gw@n)3rpsxR9-=o1z
zSq<5D2v=y3yUDcw!~>Q4)q|X;5_oZ$)Y}gD(0{DBV%&5KWo@ajKERRh^~$ZIn%yHs
zG}Xk}n)&0abH;eJxiU*!WMO|rd^s03%Op+|Q&AbuKVN3+joa`2^9wG?l8p&8a4r{S
z#GTDGy-`)Um;cE9%(-P9E1Y`!=dr>-7L1`i?uL)o5!so1PVU{J%Fayk@O0W=x`qnK
zN*MJ$Os`JW+aHRq@`<wn1tt2Aw}vhTNSinip1qvPJ2oj$Jwq(cnZu>~J1dxPs<~0q
zVtDO5zeFEDz(ln4eyoh20#aEkFo`ENi~lu@sXmO+<G!M%=od>`i@igfyaxPN-t2HS
zCTfJu(@YK=If3Oz;S>|TnAU4P(H@q$U`us6;_i!pfGrU&Sq%Fg@}Kv>xB?CYix0K<
zR@A_uiW~?Y^IgQYG|OS+K*S!=v|89zuS($Q_-^GeZtQ?JCMvfnz3??jP89K39UAqu
z#$NHr_8;=dnUAFDd*8bLav}Yf;yX7~YDygo#SsTV;&&7grno3{G;>S$l@XyyF`ed@
zBx*|UU<O`Q8SNv>qM$C35HiiME*m42g~PhUZ|2h?os01`NqJP`%PZe~jK5bu)6%xI
zTjoM+)<Met-)uuYIec~8?FZ(Y6klNoih>*7ZuXOq9oGet*>7|=zbJ&DMR7K#@A|W2
z5Tq)tRfWc!KJ8RF`BV1KPj63ep0;s&Ij2KisFl{kk7h|(@bu}Ovg&-?EWCLOIiivN
zX~&1O7e6<<1o~E=8sA(XK6@m7p2%KD1~L&Jh%ZRVmYJl~MmIadEnZd^wLMAUR-)Cx
zBT=yumP8s_5jCXk*FE|3zaN|H^5q8!1+zMqF<_rR(A?9n@h#tW^YiQFk&AZyV7|%8
z+Xayp>u<{9RIX4Ni{jmRfVf;mD2+Xteq&TVd_9Hh(w7W$2-&s$VH(*2R|;y@r<#A_
z8ltdd<u1Y5gv-VY1LzP^0n<=S^tOEKG`LX$q1NB5witlMPhkt2D{KljrRS;O>p71<
z+A0^&1s{SksPOGxqHhxC-#BP&y0*Gr@zt+C;MZeb9J%r|U+Fo`SZ1P9NPv83EEE6r
zTR8R0b6oyuwfOZ>CJ}W`z9aPRr&ik50+Us4NOKCNu`?w6?Wyh_TGJZ4<|A+TK56}}
zjE8gF(jez@na6k~yTdp^-)DO5<5CdIysh{8fVsD5u$8Vr{#Q-Zdjb49#yj6@<I?W^
zAGx&tu8;X<#M`3+x954e4G8u;lp-29Q^Z_1W6x*(v=hMy8;0z6+1Gt=Vi>mtsS;02
zG;(5GKIW+=9ozH~B!10|=5FaO<uZiG4fa|pj%7HQp7;h`sJJ;`-5rWHexZ$kQmz|Z
zS0jp!(JA>s#q9~n1AU@9J5T#vLHnB^yI1RPi-#vBlf+9slk(!`=l+V?qGlkVG+yN%
z{dLSwQo~2Y|B^d`qAZ}us5Z|X1f(6@vcn?!9jugFW{4Q2c5uN0e}5}{Jb2HU*r;DR
ziYLyeYFXf(PWO$W!;k&89<C$ssh5QtH#he;?_UkiKh+P4G&ka_A)A1B;CQV6-Jh=M
z*X8_IKiR;QmOOv?U#efjPHk<(zx=eRbSAj({jZ1Ozj*n-yc_x7YE>A0BgWMIOR(dA
zvV4fE|3L0lFGYGOaU*_h(tq|-2@umhJ$)nb@2>G57X6nEIicc9?zf}-#!2S??v=kz
z%U^#I1G_Syq4keRA_>`R20pLg`a3w@o}6LRunIQ|dnc?5y#>B?^qco7J$aw|ar2Md
zCyjZeG^NIkc>qc#pP8hqu-?PduH(_9d5fdNOT*qNR&DRnjZ5}UkyT2gqgA|`E`66!
zZn?ag_-0=DTPH8(Ok)H&6Et})_jt0k*_zTXs60@W39INhR$6vSH`vkrWNDd9#gW;G
zwT2HraWVI^RyJ~N9?jyqeW-KaxHa*m*TIhS8WgLTplErx4b&}HmtWKeHbrOkr@m6^
zf8FpQX(6V*=kWb3k3K0*#blJ;;0$wEc5<}dAcz$5Uz46GlU%2BTkopDT7)<5RA_M%
zhcET!n;nt4pVdEl6J@#<owX8DDm_=nt6BUiB<!1Nbe3de&6kIU47mxK8l$s947r)7
zL|3^^6nWS?S;kf2MGmUvOta4UjLlI-XI*?zLyqxS#;4&XT0<){Wq6m9*AojEz0`fK
z&r7X|x8vu>GoHW;iqp6@gB8Yjh%D}$Z}r=Zc);QB`4o17z<gh*aaDUpAx?(3l5Y;)
z^&_o0g~>2x$>DR@2_H3`Q24X%<&G+_E(GMWwUN)3AMQ+ve)zX9CtQ!27m^+BtYZ4x
zLS7*+4nbCWOaXZTQ4TV-4YD!5lkU5S0#a1QAz<*9s-iqEkF_FSF21q0W3|jC>)FWK
z7?8DfT!B9ihu&X@6ay~uzukolAAnu1zknD2bs;IpO4X1pv*1Twz(=Gx-r)f|_qg5R
z2Ju1SxCj=hnQ&(mcI7&<wh&}(7W}YjX2{wSk+nrXgiZUO?7|vt8-CsvcI00ck_{_$
zKrT!9BJ8)Vg7W4v_)fypMH8O9ZOevgb3(GRF2n`9@@nIbA!}2<h-{hwazks7wG}YJ
zpNrG}<6W3A^sUVF5r7@}>q5+sl`<e(rUR$WN8kd{DLn?WbIE+3bsy1c)YC&>XwK)*
z(%(<hzZ^C=3;*#h5Wrck>B5fubs;jyN)wSS>$rlv;4&f|k63ORu*ZI9)5oic=2Z3}
z)pvzB0&T<JPt(61wikT=@h&Vgr{X4tk>~KQ3;FA_0>^c88JEn-6kyDR4O3sc`0>r>
z6F~A~v@w6)kPbX(-ST!cS;u3sGIZkR-iCf3o3cu!cCY@zdmo?n_cSxP;qI<|rH{*U
zh*!!=P~ADz@=cwrj^G_rKZy!qwf6Z4Fs}JtU^WZ;eW`Lu5+8T-!164Yms{9QG_LHx
zUt4roP3nhY*Ts2ru&CJr_G8heLy2FK2gmZ6=6qIDWSEymlSd<v)VE5n5?|u3&mg{j
zPf=knI$^-Be>anOc0(n0f6J3apn=8)TP>Xm`4Yc|ck{prpYxG1WqyqP_T`7qqc0P7
z(Q$vCVzQ7iwpDG?Gn9Hd{6)z9qVc;g4XjQ$C+jY+VexMSGLa>z^^*+hGCq05YH3xH
zGRLk|K77tJ15?~_Uw-5!a_3trrYWBE$qfp26eCzxtP^S_yfHps!b8}vT6Rrwbj&_o
zbAshf#>!Xr5XO~NUThEPvN1a{9hUiBaFx|?n9@6iJqOim8JWlVq;V>2*vm)kan)n5
z@)?Qsh+~{asz-T5*z7Zuc-IaneIis9oy%RCZIyo-M>y4>67CUl@*`dhH%_QJc-V8j
zlbv4s41ZjDJb?)xs<F}}{l%;NyT_`SAmfse>gT%mB3^grhT9pGR@cntE}*EF58kiQ
z_`-#99`jMf^i7BHR^fbTc|w~z@G8V?>EDMIskbTC+2hKWZP&PMZAjoBWd$|)oU=Fl
F{}+=J91;Kk

literal 0
HcmV?d00001

-- 
GitLab


From 7ce8d2e5fb9cc5f6f9de214584170a8cf01b98de Mon Sep 17 00:00:00 2001
From: David Truby <david.truby@arm.com>
Date: Thu, 16 May 2024 13:47:48 +0100
Subject: [PATCH 1005/1206] [clang][flang][windows] Prefer user-provided
 library paths (-L) (#90758)

Currently the paths to compiler-rt and the Flang runtimes from the LLVM
build/install directory are preferred over any user-provided library
paths. This means a user can't override compiler-rt or the Flang
runtimes with custom versions.

This patch changes the link order to prefer library paths specified with
-L over the LLVM paths. This matches the behaviour of clang and flang on
Linux.
---
 clang/lib/Driver/ToolChains/MSVC.cpp  | 8 ++++----
 clang/test/Driver/flang/msvc-link.f90 | 5 +++++
 2 files changed, 9 insertions(+), 4 deletions(-)
 create mode 100644 clang/test/Driver/flang/msvc-link.f90

diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp
index fbf2f45b5438..b7021d4b996d 100644
--- a/clang/lib/Driver/ToolChains/MSVC.cpp
+++ b/clang/lib/Driver/ToolChains/MSVC.cpp
@@ -134,6 +134,10 @@ void visualstudio::Linker::ConstructJob(Compilation &C, const JobAction &JA,
           Args.MakeArgString(std::string("-libpath:") + WindowsSdkLibPath));
   }
 
+  if (!C.getDriver().IsCLMode() && Args.hasArg(options::OPT_L))
+    for (const auto &LibPath : Args.getAllArgValues(options::OPT_L))
+      CmdArgs.push_back(Args.MakeArgString("-libpath:" + LibPath));
+
   if (C.getDriver().IsFlangMode()) {
     addFortranRuntimeLibraryPath(TC, Args, CmdArgs);
     addFortranRuntimeLibs(TC, Args, CmdArgs);
@@ -154,10 +158,6 @@ void visualstudio::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   if (TC.getVFS().exists(CRTPath))
     CmdArgs.push_back(Args.MakeArgString("-libpath:" + CRTPath));
 
-  if (!C.getDriver().IsCLMode() && Args.hasArg(options::OPT_L))
-    for (const auto &LibPath : Args.getAllArgValues(options::OPT_L))
-      CmdArgs.push_back(Args.MakeArgString("-libpath:" + LibPath));
-
   CmdArgs.push_back("-nologo");
 
   if (Args.hasArg(options::OPT_g_Group, options::OPT__SLASH_Z7))
diff --git a/clang/test/Driver/flang/msvc-link.f90 b/clang/test/Driver/flang/msvc-link.f90
new file mode 100644
index 000000000000..536da2599431
--- /dev/null
+++ b/clang/test/Driver/flang/msvc-link.f90
@@ -0,0 +1,5 @@
+! RUN: %clang --driver-mode=flang -target x86_64-pc-windows-msvc -### %s -Ltest 2>&1 | FileCheck %s
+!
+! Test that user provided paths come before the Flang runtimes
+! CHECK: "-libpath:test"
+! CHECK: "-libpath:{{.*(\\|/)}}lib"
-- 
GitLab


From 7a66e4209b0b4cc0dc871a54c4f07a4b0054b5f7 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Thu, 16 May 2024 07:49:01 -0500
Subject: [PATCH 1006/1206] [flang][OpenMP] Remove unnecessary `Fortran::`
 qualification, NFC (#92298)

The `Fortran::` namespace is redundant for all parts of the code in this
PR, except for names of functions in their definitions.
---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  201 ++--
 flang/lib/Lower/OpenMP/ClauseProcessor.h      |  113 +-
 flang/lib/Lower/OpenMP/Clauses.cpp            |    7 +-
 flang/lib/Lower/OpenMP/Clauses.h              |    7 +-
 .../lib/Lower/OpenMP/DataSharingProcessor.cpp |  148 ++-
 flang/lib/Lower/OpenMP/DataSharingProcessor.h |   66 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             | 1057 ++++++++---------
 flang/lib/Lower/OpenMP/ReductionProcessor.cpp |   23 +-
 flang/lib/Lower/OpenMP/ReductionProcessor.h   |   13 +-
 flang/lib/Lower/OpenMP/Utils.cpp              |  100 +-
 flang/lib/Lower/OpenMP/Utils.h                |   24 +-
 11 files changed, 790 insertions(+), 969 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 0ea87314d571..40d66460a698 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -62,13 +62,13 @@ getSimdModifier(const omp::clause::Schedule &clause) {
 }
 
 static void
-genAllocateClause(Fortran::lower::AbstractConverter &converter,
+genAllocateClause(lower::AbstractConverter &converter,
                   const omp::clause::Allocate &clause,
                   llvm::SmallVectorImpl<mlir::Value> &allocatorOperands,
                   llvm::SmallVectorImpl<mlir::Value> &allocateOperands) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::Location currentLocation = converter.getCurrentLocation();
-  Fortran::lower::StatementContext stmtCtx;
+  lower::StatementContext stmtCtx;
 
   auto &objects = std::get<omp::ObjectList>(clause.t);
 
@@ -144,7 +144,7 @@ genDependKindAttr(fir::FirOpBuilder &firOpBuilder,
 }
 
 static mlir::Value
-getIfClauseOperand(Fortran::lower::AbstractConverter &converter,
+getIfClauseOperand(lower::AbstractConverter &converter,
                    const omp::clause::If &clause,
                    omp::clause::If::DirectiveNameModifier directiveName,
                    mlir::Location clauseLocation) {
@@ -154,7 +154,7 @@ getIfClauseOperand(Fortran::lower::AbstractConverter &converter,
   if (directive && directive.value() != directiveName)
     return nullptr;
 
-  Fortran::lower::StatementContext stmtCtx;
+  lower::StatementContext stmtCtx;
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::Value ifVal = fir::getBase(
       converter.genExprValue(std::get<omp::SomeExpr>(clause.t), stmtCtx));
@@ -163,12 +163,11 @@ getIfClauseOperand(Fortran::lower::AbstractConverter &converter,
 }
 
 static void addUseDeviceClause(
-    Fortran::lower::AbstractConverter &converter,
-    const omp::ObjectList &objects,
+    lower::AbstractConverter &converter, const omp::ObjectList &objects,
     llvm::SmallVectorImpl<mlir::Value> &operands,
     llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &useDeviceSyms) {
+    llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceSyms) {
   genObjectList(objects, converter, operands);
   for (mlir::Value &operand : operands) {
     checkMapType(operand.getLoc(), operand.getType());
@@ -179,7 +178,7 @@ static void addUseDeviceClause(
     useDeviceSyms.push_back(object.id());
 }
 
-static void convertLoopBounds(Fortran::lower::AbstractConverter &converter,
+static void convertLoopBounds(lower::AbstractConverter &converter,
                               mlir::Location loc,
                               mlir::omp::CollapseClauseOps &result,
                               std::size_t loopVarTypeSize) {
@@ -202,45 +201,43 @@ static void convertLoopBounds(Fortran::lower::AbstractConverter &converter,
 //===----------------------------------------------------------------------===//
 
 bool ClauseProcessor::processCollapse(
-    mlir::Location currentLocation, Fortran::lower::pft::Evaluation &eval,
+    mlir::Location currentLocation, lower::pft::Evaluation &eval,
     mlir::omp::CollapseClauseOps &result,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &iv) const {
+    llvm::SmallVectorImpl<const semantics::Symbol *> &iv) const {
   bool found = false;
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
   // Collect the loops to collapse.
-  Fortran::lower::pft::Evaluation *doConstructEval =
-      &eval.getFirstNestedEvaluation();
-  if (doConstructEval->getIf<Fortran::parser::DoConstruct>()
-          ->IsDoConcurrent()) {
+  lower::pft::Evaluation *doConstructEval = &eval.getFirstNestedEvaluation();
+  if (doConstructEval->getIf<parser::DoConstruct>()->IsDoConcurrent()) {
     TODO(currentLocation, "Do Concurrent in Worksharing loop construct");
   }
 
   std::int64_t collapseValue = 1l;
   if (auto *clause = findUniqueClause<omp::clause::Collapse>()) {
-    collapseValue = Fortran::evaluate::ToInt64(clause->v).value();
+    collapseValue = evaluate::ToInt64(clause->v).value();
     found = true;
   }
 
   std::size_t loopVarTypeSize = 0;
   do {
-    Fortran::lower::pft::Evaluation *doLoop =
+    lower::pft::Evaluation *doLoop =
         &doConstructEval->getFirstNestedEvaluation();
-    auto *doStmt = doLoop->getIf<Fortran::parser::NonLabelDoStmt>();
+    auto *doStmt = doLoop->getIf<parser::NonLabelDoStmt>();
     assert(doStmt && "Expected do loop to be in the nested evaluation");
     const auto &loopControl =
-        std::get<std::optional<Fortran::parser::LoopControl>>(doStmt->t);
-    const Fortran::parser::LoopControl::Bounds *bounds =
-        std::get_if<Fortran::parser::LoopControl::Bounds>(&loopControl->u);
+        std::get<std::optional<parser::LoopControl>>(doStmt->t);
+    const parser::LoopControl::Bounds *bounds =
+        std::get_if<parser::LoopControl::Bounds>(&loopControl->u);
     assert(bounds && "Expected bounds for worksharing do loop");
-    Fortran::lower::StatementContext stmtCtx;
-    result.loopLBVar.push_back(fir::getBase(converter.genExprValue(
-        *Fortran::semantics::GetExpr(bounds->lower), stmtCtx)));
-    result.loopUBVar.push_back(fir::getBase(converter.genExprValue(
-        *Fortran::semantics::GetExpr(bounds->upper), stmtCtx)));
+    lower::StatementContext stmtCtx;
+    result.loopLBVar.push_back(fir::getBase(
+        converter.genExprValue(*semantics::GetExpr(bounds->lower), stmtCtx)));
+    result.loopUBVar.push_back(fir::getBase(
+        converter.genExprValue(*semantics::GetExpr(bounds->upper), stmtCtx)));
     if (bounds->step) {
-      result.loopStepVar.push_back(fir::getBase(converter.genExprValue(
-          *Fortran::semantics::GetExpr(bounds->step), stmtCtx)));
+      result.loopStepVar.push_back(fir::getBase(
+          converter.genExprValue(*semantics::GetExpr(bounds->step), stmtCtx)));
     } else { // If `step` is not present, assume it as `1`.
       result.loopStepVar.push_back(firOpBuilder.createIntegerConstant(
           currentLocation, firOpBuilder.getIntegerType(32), 1));
@@ -280,9 +277,9 @@ bool ClauseProcessor::processDefault() const {
   return false;
 }
 
-bool ClauseProcessor::processDevice(Fortran::lower::StatementContext &stmtCtx,
+bool ClauseProcessor::processDevice(lower::StatementContext &stmtCtx,
                                     mlir::omp::DeviceClauseOps &result) const {
-  const Fortran::parser::CharBlock *source = nullptr;
+  const parser::CharBlock *source = nullptr;
   if (auto *clause = findUniqueClause<omp::clause::Device>(&source)) {
     mlir::Location clauseLocation = converter.genLocation(*source);
     if (auto deviceModifier =
@@ -320,9 +317,9 @@ bool ClauseProcessor::processDeviceType(
   return false;
 }
 
-bool ClauseProcessor::processFinal(Fortran::lower::StatementContext &stmtCtx,
+bool ClauseProcessor::processFinal(lower::StatementContext &stmtCtx,
                                    mlir::omp::FinalClauseOps &result) const {
-  const Fortran::parser::CharBlock *source = nullptr;
+  const parser::CharBlock *source = nullptr;
   if (auto *clause = findUniqueClause<omp::clause::Final>(&source)) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
     mlir::Location clauseLocation = converter.genLocation(*source);
@@ -339,7 +336,7 @@ bool ClauseProcessor::processFinal(Fortran::lower::StatementContext &stmtCtx,
 bool ClauseProcessor::processHint(mlir::omp::HintClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Hint>()) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-    int64_t hintValue = *Fortran::evaluate::ToInt64(clause->v);
+    int64_t hintValue = *evaluate::ToInt64(clause->v);
     result.hintAttr = firOpBuilder.getI64IntegerAttr(hintValue);
     return true;
   }
@@ -356,7 +353,7 @@ bool ClauseProcessor::processNowait(mlir::omp::NowaitClauseOps &result) const {
 }
 
 bool ClauseProcessor::processNumTeams(
-    Fortran::lower::StatementContext &stmtCtx,
+    lower::StatementContext &stmtCtx,
     mlir::omp::NumTeamsClauseOps &result) const {
   // TODO Get lower and upper bounds for num_teams when parser is updated to
   // accept both.
@@ -371,7 +368,7 @@ bool ClauseProcessor::processNumTeams(
 }
 
 bool ClauseProcessor::processNumThreads(
-    Fortran::lower::StatementContext &stmtCtx,
+    lower::StatementContext &stmtCtx,
     mlir::omp::NumThreadsClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::NumThreads>()) {
     // OMPIRBuilder expects `NUM_THREADS` clause as a `Value`.
@@ -388,7 +385,7 @@ bool ClauseProcessor::processOrdered(
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
     int64_t orderedClauseValue = 0l;
     if (clause->v.has_value())
-      orderedClauseValue = *Fortran::evaluate::ToInt64(*clause->v);
+      orderedClauseValue = *evaluate::ToInt64(*clause->v);
     result.orderedAttr = firOpBuilder.getI64IntegerAttr(orderedClauseValue);
     return true;
   }
@@ -396,7 +393,7 @@ bool ClauseProcessor::processOrdered(
 }
 
 bool ClauseProcessor::processPriority(
-    Fortran::lower::StatementContext &stmtCtx,
+    lower::StatementContext &stmtCtx,
     mlir::omp::PriorityClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Priority>()) {
     result.priorityVar =
@@ -420,8 +417,7 @@ bool ClauseProcessor::processSafelen(
     mlir::omp::SafelenClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Safelen>()) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-    const std::optional<std::int64_t> safelenVal =
-        Fortran::evaluate::ToInt64(clause->v);
+    const std::optional<std::int64_t> safelenVal = evaluate::ToInt64(clause->v);
     result.safelenAttr = firOpBuilder.getI64IntegerAttr(*safelenVal);
     return true;
   }
@@ -429,7 +425,7 @@ bool ClauseProcessor::processSafelen(
 }
 
 bool ClauseProcessor::processSchedule(
-    Fortran::lower::StatementContext &stmtCtx,
+    lower::StatementContext &stmtCtx,
     mlir::omp::ScheduleClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Schedule>()) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
@@ -479,8 +475,7 @@ bool ClauseProcessor::processSimdlen(
     mlir::omp::SimdlenClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Simdlen>()) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-    const std::optional<std::int64_t> simdlenVal =
-        Fortran::evaluate::ToInt64(clause->v);
+    const std::optional<std::int64_t> simdlenVal = evaluate::ToInt64(clause->v);
     result.simdlenAttr = firOpBuilder.getI64IntegerAttr(*simdlenVal);
     return true;
   }
@@ -488,7 +483,7 @@ bool ClauseProcessor::processSimdlen(
 }
 
 bool ClauseProcessor::processThreadLimit(
-    Fortran::lower::StatementContext &stmtCtx,
+    lower::StatementContext &stmtCtx,
     mlir::omp::ThreadLimitClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::ThreadLimit>()) {
     result.threadLimitVar =
@@ -509,8 +504,7 @@ bool ClauseProcessor::processUntied(mlir::omp::UntiedClauseOps &result) const {
 bool ClauseProcessor::processAllocate(
     mlir::omp::AllocateClauseOps &result) const {
   return findRepeatableClause<omp::clause::Allocate>(
-      [&](const omp::clause::Allocate &clause,
-          const Fortran::parser::CharBlock &) {
+      [&](const omp::clause::Allocate &clause, const parser::CharBlock &) {
         genAllocateClause(converter, clause, result.allocatorVars,
                           result.allocateVars);
       });
@@ -521,30 +515,28 @@ bool ClauseProcessor::processCopyin() const {
   mlir::OpBuilder::InsertPoint insPt = firOpBuilder.saveInsertionPoint();
   firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
   auto checkAndCopyHostAssociateVar =
-      [&](Fortran::semantics::Symbol *sym,
+      [&](semantics::Symbol *sym,
           mlir::OpBuilder::InsertPoint *copyAssignIP = nullptr) {
-        assert(sym->has<Fortran::semantics::HostAssocDetails>() &&
+        assert(sym->has<semantics::HostAssocDetails>() &&
                "No host-association found");
         if (converter.isPresentShallowLookup(*sym))
           converter.copyHostAssociateVar(*sym, copyAssignIP);
       };
   bool hasCopyin = findRepeatableClause<omp::clause::Copyin>(
-      [&](const omp::clause::Copyin &clause,
-          const Fortran::parser::CharBlock &) {
+      [&](const omp::clause::Copyin &clause, const parser::CharBlock &) {
         for (const omp::Object &object : clause.v) {
-          Fortran::semantics::Symbol *sym = object.id();
+          semantics::Symbol *sym = object.id();
           assert(sym && "Expecting symbol");
           if (const auto *commonDetails =
-                  sym->detailsIf<Fortran::semantics::CommonBlockDetails>()) {
+                  sym->detailsIf<semantics::CommonBlockDetails>()) {
             for (const auto &mem : commonDetails->objects())
               checkAndCopyHostAssociateVar(&*mem, &insPt);
             break;
           }
-          if (Fortran::semantics::IsAllocatableOrObjectPointer(
-                  &sym->GetUltimate()))
+          if (semantics::IsAllocatableOrObjectPointer(&sym->GetUltimate()))
             TODO(converter.getCurrentLocation(),
                  "pointer or allocatable variables in Copyin clause");
-          assert(sym->has<Fortran::semantics::HostAssocDetails>() &&
+          assert(sym->has<semantics::HostAssocDetails>() &&
                  "No host-association found");
           checkAndCopyHostAssociateVar(sym);
         }
@@ -616,7 +608,7 @@ void TypeInfo::typeScan(mlir::Type ty) {
 // Create a function that performs a copy between two variables, compatible
 // with their types and attributes.
 static mlir::func::FuncOp
-createCopyFunc(mlir::Location loc, Fortran::lower::AbstractConverter &converter,
+createCopyFunc(mlir::Location loc, lower::AbstractConverter &converter,
                mlir::Type varType, fir::FortranVariableFlagsEnum varAttrs) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   mlir::ModuleOp module = builder.getModule();
@@ -671,7 +663,7 @@ createCopyFunc(mlir::Location loc, Fortran::lower::AbstractConverter &converter,
 bool ClauseProcessor::processCopyprivate(
     mlir::Location currentLocation,
     mlir::omp::CopyprivateClauseOps &result) const {
-  auto addCopyPrivateVar = [&](Fortran::semantics::Symbol *sym) {
+  auto addCopyPrivateVar = [&](semantics::Symbol *sym) {
     mlir::Value symVal = converter.getSymbolAddress(*sym);
     auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>();
     if (!declOp)
@@ -704,12 +696,11 @@ bool ClauseProcessor::processCopyprivate(
   };
 
   bool hasCopyPrivate = findRepeatableClause<clause::Copyprivate>(
-      [&](const clause::Copyprivate &clause,
-          const Fortran::parser::CharBlock &) {
+      [&](const clause::Copyprivate &clause, const parser::CharBlock &) {
         for (const Object &object : clause.v) {
-          Fortran::semantics::Symbol *sym = object.id();
+          semantics::Symbol *sym = object.id();
           if (const auto *commonDetails =
-                  sym->detailsIf<Fortran::semantics::CommonBlockDetails>()) {
+                  sym->detailsIf<semantics::CommonBlockDetails>()) {
             for (const auto &mem : commonDetails->objects())
               addCopyPrivateVar(&*mem);
             break;
@@ -725,8 +716,7 @@ bool ClauseProcessor::processDepend(mlir::omp::DependClauseOps &result) const {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
   return findRepeatableClause<omp::clause::Depend>(
-      [&](const omp::clause::Depend &clause,
-          const Fortran::parser::CharBlock &) {
+      [&](const omp::clause::Depend &clause, const parser::CharBlock &) {
         using Depend = omp::clause::Depend;
         assert(std::holds_alternative<Depend::WithLocators>(clause.u) &&
                "Only the modern form is handled at the moment");
@@ -741,15 +731,15 @@ bool ClauseProcessor::processDepend(mlir::omp::DependClauseOps &result) const {
         for (const omp::Object &object : objects) {
           assert(object.ref() && "Expecting designator");
 
-          if (Fortran::evaluate::ExtractSubstring(*object.ref())) {
+          if (evaluate::ExtractSubstring(*object.ref())) {
             TODO(converter.getCurrentLocation(),
                  "substring not supported for task depend");
-          } else if (Fortran::evaluate::IsArrayElement(*object.ref())) {
+          } else if (evaluate::IsArrayElement(*object.ref())) {
             TODO(converter.getCurrentLocation(),
                  "array sections not supported for task depend");
           }
 
-          Fortran::semantics::Symbol *sym = object.id();
+          semantics::Symbol *sym = object.id();
           const mlir::Value variable = converter.getSymbolAddress(*sym);
           result.dependVars.push_back(variable);
         }
@@ -760,11 +750,10 @@ bool ClauseProcessor::processHasDeviceAddr(
     mlir::omp::HasDeviceAddrClauseOps &result,
     llvm::SmallVectorImpl<mlir::Type> &isDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &isDeviceLocs,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &isDeviceSymbols)
-    const {
+    llvm::SmallVectorImpl<const semantics::Symbol *> &isDeviceSymbols) const {
   return findRepeatableClause<omp::clause::HasDeviceAddr>(
       [&](const omp::clause::HasDeviceAddr &devAddrClause,
-          const Fortran::parser::CharBlock &) {
+          const parser::CharBlock &) {
         addUseDeviceClause(converter, devAddrClause.v, result.hasDeviceAddrVars,
                            isDeviceTypes, isDeviceLocs, isDeviceSymbols);
       });
@@ -774,19 +763,18 @@ bool ClauseProcessor::processIf(
     omp::clause::If::DirectiveNameModifier directiveName,
     mlir::omp::IfClauseOps &result) const {
   bool found = false;
-  findRepeatableClause<omp::clause::If>(
-      [&](const omp::clause::If &clause,
-          const Fortran::parser::CharBlock &source) {
-        mlir::Location clauseLocation = converter.genLocation(source);
-        mlir::Value operand = getIfClauseOperand(converter, clause,
-                                                 directiveName, clauseLocation);
-        // Assume that, at most, a single 'if' clause will be applicable to the
-        // given directive.
-        if (operand) {
-          result.ifVar = operand;
-          found = true;
-        }
-      });
+  findRepeatableClause<omp::clause::If>([&](const omp::clause::If &clause,
+                                            const parser::CharBlock &source) {
+    mlir::Location clauseLocation = converter.genLocation(source);
+    mlir::Value operand =
+        getIfClauseOperand(converter, clause, directiveName, clauseLocation);
+    // Assume that, at most, a single 'if' clause will be applicable to the
+    // given directive.
+    if (operand) {
+      result.ifVar = operand;
+      found = true;
+    }
+  });
   return found;
 }
 
@@ -794,11 +782,10 @@ bool ClauseProcessor::processIsDevicePtr(
     mlir::omp::IsDevicePtrClauseOps &result,
     llvm::SmallVectorImpl<mlir::Type> &isDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &isDeviceLocs,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &isDeviceSymbols)
-    const {
+    llvm::SmallVectorImpl<const semantics::Symbol *> &isDeviceSymbols) const {
   return findRepeatableClause<omp::clause::IsDevicePtr>(
       [&](const omp::clause::IsDevicePtr &devPtrClause,
-          const Fortran::parser::CharBlock &) {
+          const parser::CharBlock &) {
         addUseDeviceClause(converter, devPtrClause.v, result.isDevicePtrVars,
                            isDeviceTypes, isDeviceLocs, isDeviceSymbols);
       });
@@ -807,7 +794,7 @@ bool ClauseProcessor::processIsDevicePtr(
 bool ClauseProcessor::processLink(
     llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const {
   return findRepeatableClause<omp::clause::Link>(
-      [&](const omp::clause::Link &clause, const Fortran::parser::CharBlock &) {
+      [&](const omp::clause::Link &clause, const parser::CharBlock &) {
         // Case: declare target link(var1, var2)...
         gatherFuncAndVarSyms(
             clause.v, mlir::omp::DeclareTargetCaptureClause::link, result);
@@ -815,25 +802,24 @@ bool ClauseProcessor::processLink(
 }
 
 bool ClauseProcessor::processMap(
-    mlir::Location currentLocation, Fortran::lower::StatementContext &stmtCtx,
+    mlir::Location currentLocation, lower::StatementContext &stmtCtx,
     mlir::omp::MapClauseOps &result,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *mapSyms,
+    llvm::SmallVectorImpl<const semantics::Symbol *> *mapSyms,
     llvm::SmallVectorImpl<mlir::Location> *mapSymLocs,
     llvm::SmallVectorImpl<mlir::Type> *mapSymTypes) const {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   // We always require tracking of symbols, even if the caller does not,
   // so we create an optionally used local set of symbols when the mapSyms
   // argument is not present.
-  llvm::SmallVector<const Fortran::semantics::Symbol *> localMapSyms;
-  llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *ptrMapSyms =
+  llvm::SmallVector<const semantics::Symbol *> localMapSyms;
+  llvm::SmallVectorImpl<const semantics::Symbol *> *ptrMapSyms =
       mapSyms ? mapSyms : &localMapSyms;
-  std::map<const Fortran::semantics::Symbol *,
+  std::map<const semantics::Symbol *,
            llvm::SmallVector<OmpMapMemberIndicesData>>
       parentMemberIndices;
 
   bool clauseFound = findRepeatableClause<omp::clause::Map>(
-      [&](const omp::clause::Map &clause,
-          const Fortran::parser::CharBlock &source) {
+      [&](const omp::clause::Map &clause, const parser::CharBlock &source) {
         using Map = omp::clause::Map;
         mlir::Location clauseLocation = converter.genLocation(source);
         const auto &mapType = std::get<std::optional<Map::MapType>>(clause.t);
@@ -881,9 +867,9 @@ bool ClauseProcessor::processMap(
           llvm::SmallVector<mlir::Value> bounds;
           std::stringstream asFortran;
 
-          Fortran::lower::AddrAndBoundsInfo info =
-              Fortran::lower::gatherDataOperandAddrAndBounds<
-                  mlir::omp::MapBoundsOp, mlir::omp::MapBoundsType>(
+          lower::AddrAndBoundsInfo info =
+              lower::gatherDataOperandAddrAndBounds<mlir::omp::MapBoundsOp,
+                                                    mlir::omp::MapBoundsType>(
                   converter, firOpBuilder, semaCtx, stmtCtx, *object.id(),
                   object.ref(), clauseLocation, asFortran, bounds,
                   treatIndexAsSection);
@@ -928,18 +914,16 @@ bool ClauseProcessor::processMap(
 bool ClauseProcessor::processReduction(
     mlir::Location currentLocation, mlir::omp::ReductionClauseOps &result,
     llvm::SmallVectorImpl<mlir::Type> *outReductionTypes,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *outReductionSyms)
-    const {
+    llvm::SmallVectorImpl<const semantics::Symbol *> *outReductionSyms) const {
   return findRepeatableClause<omp::clause::Reduction>(
-      [&](const omp::clause::Reduction &clause,
-          const Fortran::parser::CharBlock &) {
+      [&](const omp::clause::Reduction &clause, const parser::CharBlock &) {
         // Use local lists of reductions to prevent variables from other
         // already-processed reduction clauses from impacting this reduction.
         // For example, the whole `reductionVars` array is queried to decide
         // whether to do the reduction byref.
         llvm::SmallVector<mlir::Value> reductionVars;
         llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
-        llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSyms;
+        llvm::SmallVector<const semantics::Symbol *> reductionSyms;
         ReductionProcessor rp;
         rp.addDeclareReduction(currentLocation, converter, clause,
                                reductionVars, reductionDeclSymbols,
@@ -965,7 +949,7 @@ bool ClauseProcessor::processReduction(
 bool ClauseProcessor::processSectionsReduction(
     mlir::Location currentLocation, mlir::omp::ReductionClauseOps &) const {
   return findRepeatableClause<omp::clause::Reduction>(
-      [&](const omp::clause::Reduction &, const Fortran::parser::CharBlock &) {
+      [&](const omp::clause::Reduction &, const parser::CharBlock &) {
         TODO(currentLocation, "OMPC_Reduction");
       });
 }
@@ -973,7 +957,7 @@ bool ClauseProcessor::processSectionsReduction(
 bool ClauseProcessor::processTo(
     llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const {
   return findRepeatableClause<omp::clause::To>(
-      [&](const omp::clause::To &clause, const Fortran::parser::CharBlock &) {
+      [&](const omp::clause::To &clause, const parser::CharBlock &) {
         // Case: declare target to(func, var1, var2)...
         gatherFuncAndVarSyms(std::get<ObjectList>(clause.t),
                              mlir::omp::DeclareTargetCaptureClause::to, result);
@@ -983,8 +967,7 @@ bool ClauseProcessor::processTo(
 bool ClauseProcessor::processEnter(
     llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const {
   return findRepeatableClause<omp::clause::Enter>(
-      [&](const omp::clause::Enter &clause,
-          const Fortran::parser::CharBlock &) {
+      [&](const omp::clause::Enter &clause, const parser::CharBlock &) {
         // Case: declare target enter(func, var1, var2)...
         gatherFuncAndVarSyms(
             clause.v, mlir::omp::DeclareTargetCaptureClause::enter, result);
@@ -995,11 +978,9 @@ bool ClauseProcessor::processUseDeviceAddr(
     mlir::omp::UseDeviceClauseOps &result,
     llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &useDeviceSyms)
-    const {
+    llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceSyms) const {
   return findRepeatableClause<omp::clause::UseDeviceAddr>(
-      [&](const omp::clause::UseDeviceAddr &clause,
-          const Fortran::parser::CharBlock &) {
+      [&](const omp::clause::UseDeviceAddr &clause, const parser::CharBlock &) {
         addUseDeviceClause(converter, clause.v, result.useDeviceAddrVars,
                            useDeviceTypes, useDeviceLocs, useDeviceSyms);
       });
@@ -1009,11 +990,9 @@ bool ClauseProcessor::processUseDevicePtr(
     mlir::omp::UseDeviceClauseOps &result,
     llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &useDeviceSyms)
-    const {
+    llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceSyms) const {
   return findRepeatableClause<omp::clause::UseDevicePtr>(
-      [&](const omp::clause::UseDevicePtr &clause,
-          const Fortran::parser::CharBlock &) {
+      [&](const omp::clause::UseDevicePtr &clause, const parser::CharBlock &) {
         addUseDeviceClause(converter, clause.v, result.useDevicePtrVars,
                            useDeviceTypes, useDeviceLocs, useDeviceSyms);
       });
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 54ffcd0c964b..7e7a291b94ae 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -47,44 +47,43 @@ namespace omp {
 /// construct.
 class ClauseProcessor {
 public:
-  ClauseProcessor(Fortran::lower::AbstractConverter &converter,
-                  Fortran::semantics::SemanticsContext &semaCtx,
+  ClauseProcessor(lower::AbstractConverter &converter,
+                  semantics::SemanticsContext &semaCtx,
                   const List<Clause> &clauses)
       : converter(converter), semaCtx(semaCtx), clauses(clauses) {}
 
   // 'Unique' clauses: They can appear at most once in the clause list.
-  bool processCollapse(
-      mlir::Location currentLocation, Fortran::lower::pft::Evaluation &eval,
-      mlir::omp::CollapseClauseOps &result,
-      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &iv) const;
+  bool
+  processCollapse(mlir::Location currentLocation, lower::pft::Evaluation &eval,
+                  mlir::omp::CollapseClauseOps &result,
+                  llvm::SmallVectorImpl<const semantics::Symbol *> &iv) const;
   bool processDefault() const;
-  bool processDevice(Fortran::lower::StatementContext &stmtCtx,
+  bool processDevice(lower::StatementContext &stmtCtx,
                      mlir::omp::DeviceClauseOps &result) const;
   bool processDeviceType(mlir::omp::DeviceTypeClauseOps &result) const;
-  bool processFinal(Fortran::lower::StatementContext &stmtCtx,
+  bool processFinal(lower::StatementContext &stmtCtx,
                     mlir::omp::FinalClauseOps &result) const;
-  bool
-  processHasDeviceAddr(mlir::omp::HasDeviceAddrClauseOps &result,
-                       llvm::SmallVectorImpl<mlir::Type> &isDeviceTypes,
-                       llvm::SmallVectorImpl<mlir::Location> &isDeviceLocs,
-                       llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-                           &isDeviceSymbols) const;
+  bool processHasDeviceAddr(
+      mlir::omp::HasDeviceAddrClauseOps &result,
+      llvm::SmallVectorImpl<mlir::Type> &isDeviceTypes,
+      llvm::SmallVectorImpl<mlir::Location> &isDeviceLocs,
+      llvm::SmallVectorImpl<const semantics::Symbol *> &isDeviceSymbols) const;
   bool processHint(mlir::omp::HintClauseOps &result) const;
   bool processMergeable(mlir::omp::MergeableClauseOps &result) const;
   bool processNowait(mlir::omp::NowaitClauseOps &result) const;
-  bool processNumTeams(Fortran::lower::StatementContext &stmtCtx,
+  bool processNumTeams(lower::StatementContext &stmtCtx,
                        mlir::omp::NumTeamsClauseOps &result) const;
-  bool processNumThreads(Fortran::lower::StatementContext &stmtCtx,
+  bool processNumThreads(lower::StatementContext &stmtCtx,
                          mlir::omp::NumThreadsClauseOps &result) const;
   bool processOrdered(mlir::omp::OrderedClauseOps &result) const;
-  bool processPriority(Fortran::lower::StatementContext &stmtCtx,
+  bool processPriority(lower::StatementContext &stmtCtx,
                        mlir::omp::PriorityClauseOps &result) const;
   bool processProcBind(mlir::omp::ProcBindClauseOps &result) const;
   bool processSafelen(mlir::omp::SafelenClauseOps &result) const;
-  bool processSchedule(Fortran::lower::StatementContext &stmtCtx,
+  bool processSchedule(lower::StatementContext &stmtCtx,
                        mlir::omp::ScheduleClauseOps &result) const;
   bool processSimdlen(mlir::omp::SimdlenClauseOps &result) const;
-  bool processThreadLimit(Fortran::lower::StatementContext &stmtCtx,
+  bool processThreadLimit(lower::StatementContext &stmtCtx,
                           mlir::omp::ThreadLimitClauseOps &result) const;
   bool processUntied(mlir::omp::UntiedClauseOps &result) const;
 
@@ -98,12 +97,11 @@ public:
   processEnter(llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const;
   bool processIf(omp::clause::If::DirectiveNameModifier directiveName,
                  mlir::omp::IfClauseOps &result) const;
-  bool
-  processIsDevicePtr(mlir::omp::IsDevicePtrClauseOps &result,
-                     llvm::SmallVectorImpl<mlir::Type> &isDeviceTypes,
-                     llvm::SmallVectorImpl<mlir::Location> &isDeviceLocs,
-                     llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-                         &isDeviceSymbols) const;
+  bool processIsDevicePtr(
+      mlir::omp::IsDevicePtrClauseOps &result,
+      llvm::SmallVectorImpl<mlir::Type> &isDeviceTypes,
+      llvm::SmallVectorImpl<mlir::Location> &isDeviceLocs,
+      llvm::SmallVectorImpl<const semantics::Symbol *> &isDeviceSymbols) const;
   bool
   processLink(llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const;
 
@@ -113,35 +111,32 @@ public:
   // They may be used later on to create the block_arguments for some of the
   // target directives that require it.
   bool processMap(
-      mlir::Location currentLocation, Fortran::lower::StatementContext &stmtCtx,
+      mlir::Location currentLocation, lower::StatementContext &stmtCtx,
       mlir::omp::MapClauseOps &result,
-      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *mapSyms =
-          nullptr,
+      llvm::SmallVectorImpl<const semantics::Symbol *> *mapSyms = nullptr,
       llvm::SmallVectorImpl<mlir::Location> *mapSymLocs = nullptr,
       llvm::SmallVectorImpl<mlir::Type> *mapSymTypes = nullptr) const;
   bool processReduction(
       mlir::Location currentLocation, mlir::omp::ReductionClauseOps &result,
       llvm::SmallVectorImpl<mlir::Type> *reductionTypes = nullptr,
-      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *reductionSyms =
+      llvm::SmallVectorImpl<const semantics::Symbol *> *reductionSyms =
           nullptr) const;
   bool processSectionsReduction(mlir::Location currentLocation,
                                 mlir::omp::ReductionClauseOps &result) const;
   bool processTo(llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const;
-  bool
-  processUseDeviceAddr(mlir::omp::UseDeviceClauseOps &result,
-                       llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
-                       llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
-                       llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-                           &useDeviceSyms) const;
-  bool
-  processUseDevicePtr(mlir::omp::UseDeviceClauseOps &result,
-                      llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
-                      llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
-                      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-                          &useDeviceSyms) const;
+  bool processUseDeviceAddr(
+      mlir::omp::UseDeviceClauseOps &result,
+      llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
+      llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
+      llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceSyms) const;
+  bool processUseDevicePtr(
+      mlir::omp::UseDeviceClauseOps &result,
+      llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
+      llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
+      llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceSyms) const;
 
   template <typename T>
-  bool processMotionClauses(Fortran::lower::StatementContext &stmtCtx,
+  bool processMotionClauses(lower::StatementContext &stmtCtx,
                             mlir::omp::MapClauseOps &result);
 
   // Call this method for these clauses that should be supported but are not
@@ -162,36 +157,34 @@ private:
   /// `nullptr` if not present. If more than one instance is expected, use
   /// `findRepeatableClause` instead.
   template <typename T>
-  const T *
-  findUniqueClause(const Fortran::parser::CharBlock **source = nullptr) const;
+  const T *findUniqueClause(const parser::CharBlock **source = nullptr) const;
 
   /// Call `callbackFn` for each occurrence of the given clause. Return `true`
   /// if at least one instance was found.
   template <typename T>
   bool findRepeatableClause(
-      std::function<void(const T &, const Fortran::parser::CharBlock &source)>
+      std::function<void(const T &, const parser::CharBlock &source)>
           callbackFn) const;
 
   /// Set the `result` to a new `mlir::UnitAttr` if the clause is present.
   template <typename T>
   bool markClauseOccurrence(mlir::UnitAttr &result) const;
 
-  Fortran::lower::AbstractConverter &converter;
-  Fortran::semantics::SemanticsContext &semaCtx;
+  lower::AbstractConverter &converter;
+  semantics::SemanticsContext &semaCtx;
   List<Clause> clauses;
 };
 
 template <typename T>
-bool ClauseProcessor::processMotionClauses(
-    Fortran::lower::StatementContext &stmtCtx,
-    mlir::omp::MapClauseOps &result) {
-  std::map<const Fortran::semantics::Symbol *,
+bool ClauseProcessor::processMotionClauses(lower::StatementContext &stmtCtx,
+                                           mlir::omp::MapClauseOps &result) {
+  std::map<const semantics::Symbol *,
            llvm::SmallVector<OmpMapMemberIndicesData>>
       parentMemberIndices;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> mapSymbols;
+  llvm::SmallVector<const semantics::Symbol *> mapSymbols;
 
   bool clauseFound = findRepeatableClause<T>(
-      [&](const T &clause, const Fortran::parser::CharBlock &source) {
+      [&](const T &clause, const parser::CharBlock &source) {
         mlir::Location clauseLocation = converter.genLocation(source);
         fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
@@ -209,9 +202,9 @@ bool ClauseProcessor::processMotionClauses(
           llvm::SmallVector<mlir::Value> bounds;
           std::stringstream asFortran;
 
-          Fortran::lower::AddrAndBoundsInfo info =
-              Fortran::lower::gatherDataOperandAddrAndBounds<
-                  mlir::omp::MapBoundsOp, mlir::omp::MapBoundsType>(
+          lower::AddrAndBoundsInfo info =
+              lower::gatherDataOperandAddrAndBounds<mlir::omp::MapBoundsOp,
+                                                    mlir::omp::MapBoundsType>(
                   converter, firOpBuilder, semaCtx, stmtCtx, *object.id(),
                   object.ref(), clauseLocation, asFortran, bounds,
                   treatIndexAsSection);
@@ -277,8 +270,8 @@ ClauseProcessor::findClause(ClauseIterator begin, ClauseIterator end) {
 }
 
 template <typename T>
-const T *ClauseProcessor::findUniqueClause(
-    const Fortran::parser::CharBlock **source) const {
+const T *
+ClauseProcessor::findUniqueClause(const parser::CharBlock **source) const {
   ClauseIterator it = findClause<T>(clauses.begin(), clauses.end());
   if (it != clauses.end()) {
     if (source)
@@ -290,8 +283,8 @@ const T *ClauseProcessor::findUniqueClause(
 
 template <typename T>
 bool ClauseProcessor::findRepeatableClause(
-    std::function<void(const T &, const Fortran::parser::CharBlock &source)>
-        callbackFn) const {
+    std::function<void(const T &, const parser::CharBlock &source)> callbackFn)
+    const {
   bool found = false;
   ClauseIterator nextIt, endIt = clauses.end();
   for (ClauseIterator it = clauses.begin(); it != endIt; it = nextIt) {
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 87370c92964a..1e3fea7c4cdc 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -150,9 +150,8 @@ Object makeObject(const parser::OmpObject &object,
   return makeObject(std::get<parser::Designator>(object.u), semaCtx);
 }
 
-std::optional<Object>
-getBaseObject(const Object &object,
-              Fortran::semantics::SemanticsContext &semaCtx) {
+std::optional<Object> getBaseObject(const Object &object,
+                                    semantics::SemanticsContext &semaCtx) {
   // If it's just the symbol, then there is no base.
   if (!object.id())
     return std::nullopt;
@@ -1211,7 +1210,7 @@ UsesAllocators make(const parser::OmpClause::UsesAllocators &inp,
 // Write: empty
 } // namespace clause
 
-Clause makeClause(const Fortran::parser::OmpClause &cls,
+Clause makeClause(const parser::OmpClause &cls,
                   semantics::SemanticsContext &semaCtx) {
   return std::visit(
       [&](auto &&s) {
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index ca610c652896..5391b134e979 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -107,9 +107,8 @@ std::optional<ResultTy> maybeApply(FuncTy &&func,
   return std::move(func(*arg));
 }
 
-std::optional<Object>
-getBaseObject(const Object &object,
-              Fortran::semantics::SemanticsContext &semaCtx);
+std::optional<Object> getBaseObject(const Object &object,
+                                    semantics::SemanticsContext &semaCtx);
 
 namespace clause {
 using DefinedOperator = tomp::type::DefinedOperatorT<IdTy, ExprTy>;
@@ -262,7 +261,7 @@ Clause makeClause(llvm::omp::Clause id, Specific &&specific,
   return Clause(typename Clause::BaseT{id, specific}, source);
 }
 
-Clause makeClause(const Fortran::parser::OmpClause &cls,
+Clause makeClause(const parser::OmpClause &cls,
                   semantics::SemanticsContext &semaCtx);
 
 List<Clause> makeClauses(const parser::OmpClauseList &clauses,
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 82d8d8dd98ea..84e745f28f97 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -25,7 +25,7 @@ namespace omp {
 
 void DataSharingProcessor::processStep1(
     mlir::omp::PrivateClauseOps *clauseOps,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
+    llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
   collectSymbolsForPrivatization();
   collectDefaultSymbols();
   collectImplicitSymbols();
@@ -57,14 +57,14 @@ void DataSharingProcessor::processStep2(mlir::Operation *op, bool isLoop) {
 }
 
 void DataSharingProcessor::insertDeallocs() {
-  for (const Fortran::semantics::Symbol *sym : privatizedSymbols)
-    if (Fortran::semantics::IsAllocatable(sym->GetUltimate())) {
+  for (const semantics::Symbol *sym : privatizedSymbols)
+    if (semantics::IsAllocatable(sym->GetUltimate())) {
       if (!useDelayedPrivatization) {
         converter.createHostAssociateVarCloneDealloc(*sym);
         return;
       }
 
-      Fortran::lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
+      lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
       assert(hsb && "Host symbol box not found");
       mlir::Type symType = hsb.getAddr().getType();
       mlir::Location symLoc = hsb.getAddr().getLoc();
@@ -91,10 +91,10 @@ void DataSharingProcessor::insertDeallocs() {
     }
 }
 
-void DataSharingProcessor::cloneSymbol(const Fortran::semantics::Symbol *sym) {
+void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) {
   // Privatization for symbols which are pre-determined (like loop index
   // variables) happen separately, for everything else privatize here.
-  if (sym->test(Fortran::semantics::Symbol::Flag::OmpPreDetermined))
+  if (sym->test(semantics::Symbol::Flag::OmpPreDetermined))
     return;
   bool success = converter.createHostAssociateVarClone(*sym);
   (void)success;
@@ -102,22 +102,21 @@ void DataSharingProcessor::cloneSymbol(const Fortran::semantics::Symbol *sym) {
 }
 
 void DataSharingProcessor::copyFirstPrivateSymbol(
-    const Fortran::semantics::Symbol *sym,
-    mlir::OpBuilder::InsertPoint *copyAssignIP) {
-  if (sym->test(Fortran::semantics::Symbol::Flag::OmpFirstPrivate))
+    const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *copyAssignIP) {
+  if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate))
     converter.copyHostAssociateVar(*sym, copyAssignIP);
 }
 
 void DataSharingProcessor::copyLastPrivateSymbol(
-    const Fortran::semantics::Symbol *sym,
+    const semantics::Symbol *sym,
     [[maybe_unused]] mlir::OpBuilder::InsertPoint *lastPrivIP) {
-  if (sym->test(Fortran::semantics::Symbol::Flag::OmpLastPrivate))
+  if (sym->test(semantics::Symbol::Flag::OmpLastPrivate))
     converter.copyHostAssociateVar(*sym, lastPrivIP);
 }
 
 void DataSharingProcessor::collectOmpObjectListSymbol(
     const omp::ObjectList &objects,
-    llvm::SetVector<const Fortran::semantics::Symbol *> &symbolSet) {
+    llvm::SetVector<const semantics::Symbol *> &symbolSet) {
   for (const omp::Object &object : objects)
     symbolSet.insert(object.id());
 }
@@ -150,9 +149,9 @@ bool DataSharingProcessor::needBarrier() {
   // initialization of firstprivate variables and post-update of lastprivate
   // variables.
   // Emit implicit barrier for linear clause. Maybe on somewhere else.
-  for (const Fortran::semantics::Symbol *sym : privatizedSymbols) {
-    if (sym->test(Fortran::semantics::Symbol::Flag::OmpFirstPrivate) &&
-        sym->test(Fortran::semantics::Symbol::Flag::OmpLastPrivate))
+  for (const semantics::Symbol *sym : privatizedSymbols) {
+    if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) &&
+        sym->test(semantics::Symbol::Flag::OmpLastPrivate))
       return true;
   }
   return false;
@@ -238,26 +237,26 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
   }
 }
 
-static const Fortran::parser::CharBlock *
-getSource(const Fortran::semantics::SemanticsContext &semaCtx,
-          const Fortran::lower::pft::Evaluation &eval) {
-  const Fortran::parser::CharBlock *source = nullptr;
+static const parser::CharBlock *
+getSource(const semantics::SemanticsContext &semaCtx,
+          const lower::pft::Evaluation &eval) {
+  const parser::CharBlock *source = nullptr;
 
-  auto ompConsVisit = [&](const Fortran::parser::OpenMPConstruct &x) {
-    std::visit(Fortran::common::visitors{
-                   [&](const Fortran::parser::OpenMPSectionsConstruct &x) {
+  auto ompConsVisit = [&](const parser::OpenMPConstruct &x) {
+    std::visit(common::visitors{
+                   [&](const parser::OpenMPSectionsConstruct &x) {
                      source = &std::get<0>(x.t).source;
                    },
-                   [&](const Fortran::parser::OpenMPLoopConstruct &x) {
+                   [&](const parser::OpenMPLoopConstruct &x) {
                      source = &std::get<0>(x.t).source;
                    },
-                   [&](const Fortran::parser::OpenMPBlockConstruct &x) {
+                   [&](const parser::OpenMPBlockConstruct &x) {
                      source = &std::get<0>(x.t).source;
                    },
-                   [&](const Fortran::parser::OpenMPCriticalConstruct &x) {
+                   [&](const parser::OpenMPCriticalConstruct &x) {
                      source = &std::get<0>(x.t).source;
                    },
-                   [&](const Fortran::parser::OpenMPAtomicConstruct &x) {
+                   [&](const parser::OpenMPAtomicConstruct &x) {
                      std::visit([&](const auto &x) { source = &x.source; },
                                 x.u);
                    },
@@ -266,14 +265,10 @@ getSource(const Fortran::semantics::SemanticsContext &semaCtx,
                x.u);
   };
 
-  eval.visit(Fortran::common::visitors{
-      [&](const Fortran::parser::OpenMPConstruct &x) { ompConsVisit(x); },
-      [&](const Fortran::parser::OpenMPDeclarativeConstruct &x) {
-        source = &x.source;
-      },
-      [&](const Fortran::parser::OmpEndLoopDirective &x) {
-        source = &x.source;
-      },
+  eval.visit(common::visitors{
+      [&](const parser::OpenMPConstruct &x) { ompConsVisit(x); },
+      [&](const parser::OpenMPDeclarativeConstruct &x) { source = &x.source; },
+      [&](const parser::OmpEndLoopDirective &x) { source = &x.source; },
       [&](const auto &x) {},
   });
 
@@ -281,12 +276,9 @@ getSource(const Fortran::semantics::SemanticsContext &semaCtx,
 }
 
 void DataSharingProcessor::collectSymbolsInNestedRegions(
-    Fortran::lower::pft::Evaluation &eval,
-    Fortran::semantics::Symbol::Flag flag,
-    llvm::SetVector<const Fortran::semantics::Symbol *>
-        &symbolsInNestedRegions) {
-  for (Fortran::lower::pft::Evaluation &nestedEval :
-       eval.getNestedEvaluations()) {
+    lower::pft::Evaluation &eval, semantics::Symbol::Flag flag,
+    llvm::SetVector<const semantics::Symbol *> &symbolsInNestedRegions) {
+  for (lower::pft::Evaluation &nestedEval : eval.getNestedEvaluations()) {
     if (nestedEval.hasNestedEvaluations()) {
       if (nestedEval.isConstruct())
         // Recursively look for OpenMP constructs within `nestedEval`'s region
@@ -307,46 +299,45 @@ void DataSharingProcessor::collectSymbolsInNestedRegions(
 // Later, in current context, all symbols in the set
 // `defaultSymbols` - `symbolsInNestedRegions` will be privatized.
 void DataSharingProcessor::collectSymbols(
-    Fortran::semantics::Symbol::Flag flag,
-    llvm::SetVector<const Fortran::semantics::Symbol *> &symbols) {
+    semantics::Symbol::Flag flag,
+    llvm::SetVector<const semantics::Symbol *> &symbols) {
   // Collect all scopes associated with 'eval'.
-  llvm::SetVector<const Fortran::semantics::Scope *> clauseScopes;
-  std::function<void(const Fortran::semantics::Scope *)> collectScopes =
-      [&](const Fortran::semantics::Scope *scope) {
+  llvm::SetVector<const semantics::Scope *> clauseScopes;
+  std::function<void(const semantics::Scope *)> collectScopes =
+      [&](const semantics::Scope *scope) {
         clauseScopes.insert(scope);
-        for (const Fortran::semantics::Scope &child : scope->children())
+        for (const semantics::Scope &child : scope->children())
           collectScopes(&child);
       };
-  const Fortran::parser::CharBlock *source =
+  const parser::CharBlock *source =
       clauses.empty() ? getSource(semaCtx, eval) : &clauses.front().source;
-  const Fortran::semantics::Scope *curScope = nullptr;
+  const semantics::Scope *curScope = nullptr;
   if (source && !source->empty()) {
     curScope = &semaCtx.FindScope(*source);
     collectScopes(curScope);
   }
   // Collect all symbols referenced in the evaluation being processed,
   // that matches 'flag'.
-  llvm::SetVector<const Fortran::semantics::Symbol *> allSymbols;
+  llvm::SetVector<const semantics::Symbol *> allSymbols;
   converter.collectSymbolSet(eval, allSymbols, flag,
                              /*collectSymbols=*/true,
                              /*collectHostAssociatedSymbols=*/true);
-  llvm::SetVector<const Fortran::semantics::Symbol *> symbolsInNestedRegions;
+  llvm::SetVector<const semantics::Symbol *> symbolsInNestedRegions;
   collectSymbolsInNestedRegions(eval, flag, symbolsInNestedRegions);
   // Filter-out symbols that must not be privatized.
-  bool collectImplicit = flag == Fortran::semantics::Symbol::Flag::OmpImplicit;
-  auto isPrivatizable = [](const Fortran::semantics::Symbol &sym) -> bool {
-    return !Fortran::semantics::IsProcedure(sym) &&
-           !sym.GetUltimate().has<Fortran::semantics::DerivedTypeDetails>() &&
-           !sym.GetUltimate().has<Fortran::semantics::NamelistDetails>() &&
-           !Fortran::semantics::IsImpliedDoIndex(sym.GetUltimate());
+  bool collectImplicit = flag == semantics::Symbol::Flag::OmpImplicit;
+  auto isPrivatizable = [](const semantics::Symbol &sym) -> bool {
+    return !semantics::IsProcedure(sym) &&
+           !sym.GetUltimate().has<semantics::DerivedTypeDetails>() &&
+           !sym.GetUltimate().has<semantics::NamelistDetails>() &&
+           !semantics::IsImpliedDoIndex(sym.GetUltimate());
   };
   for (const auto *sym : allSymbols) {
     assert(curScope && "couldn't find current scope");
     if (isPrivatizable(*sym) && !symbolsInNestedRegions.contains(sym) &&
         !privatizedSymbols.contains(sym) &&
-        !sym->test(Fortran::semantics::Symbol::Flag::OmpPreDetermined) &&
-        (collectImplicit ||
-         !sym->test(Fortran::semantics::Symbol::Flag::OmpImplicit)) &&
+        !sym->test(semantics::Symbol::Flag::OmpPreDetermined) &&
+        (collectImplicit || !sym->test(semantics::Symbol::Flag::OmpImplicit)) &&
         clauseScopes.contains(&sym->owner()))
       symbols.insert(sym);
   }
@@ -358,10 +349,9 @@ void DataSharingProcessor::collectDefaultSymbols() {
     if (const auto *defaultClause =
             std::get_if<omp::clause::Default>(&clause.u)) {
       if (defaultClause->v == DataSharingAttribute::Private)
-        collectSymbols(Fortran::semantics::Symbol::Flag::OmpPrivate,
-                       defaultSymbols);
+        collectSymbols(semantics::Symbol::Flag::OmpPrivate, defaultSymbols);
       else if (defaultClause->v == DataSharingAttribute::Firstprivate)
-        collectSymbols(Fortran::semantics::Symbol::Flag::OmpFirstPrivate,
+        collectSymbols(semantics::Symbol::Flag::OmpFirstPrivate,
                        defaultSymbols);
     }
   }
@@ -370,16 +360,15 @@ void DataSharingProcessor::collectDefaultSymbols() {
 void DataSharingProcessor::collectImplicitSymbols() {
   // There will be no implicit symbols when a default clause is present.
   if (defaultSymbols.empty())
-    collectSymbols(Fortran::semantics::Symbol::Flag::OmpImplicit,
-                   implicitSymbols);
+    collectSymbols(semantics::Symbol::Flag::OmpImplicit, implicitSymbols);
 }
 
 void DataSharingProcessor::privatize(
     mlir::omp::PrivateClauseOps *clauseOps,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
-  for (const Fortran::semantics::Symbol *sym : privatizedSymbols) {
+    llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
+  for (const semantics::Symbol *sym : privatizedSymbols) {
     if (const auto *commonDet =
-            sym->detailsIf<Fortran::semantics::CommonBlockDetails>()) {
+            sym->detailsIf<semantics::CommonBlockDetails>()) {
       for (const auto &mem : commonDet->objects())
         doPrivatize(&*mem, clauseOps, privateSyms);
     } else
@@ -389,9 +378,9 @@ void DataSharingProcessor::privatize(
 
 void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
   insertLastPrivateCompare(op);
-  for (const Fortran::semantics::Symbol *sym : privatizedSymbols)
+  for (const semantics::Symbol *sym : privatizedSymbols)
     if (const auto *commonDet =
-            sym->detailsIf<Fortran::semantics::CommonBlockDetails>()) {
+            sym->detailsIf<semantics::CommonBlockDetails>()) {
       for (const auto &mem : commonDet->objects()) {
         copyLastPrivateSymbol(&*mem, &lastPrivIP);
       }
@@ -402,36 +391,34 @@ void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
 
 void DataSharingProcessor::defaultPrivatize(
     mlir::omp::PrivateClauseOps *clauseOps,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
-  for (const Fortran::semantics::Symbol *sym : defaultSymbols)
+    llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
+  for (const semantics::Symbol *sym : defaultSymbols)
     doPrivatize(sym, clauseOps, privateSyms);
 }
 
 void DataSharingProcessor::implicitPrivatize(
     mlir::omp::PrivateClauseOps *clauseOps,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
-  for (const Fortran::semantics::Symbol *sym : implicitSymbols)
+    llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
+  for (const semantics::Symbol *sym : implicitSymbols)
     doPrivatize(sym, clauseOps, privateSyms);
 }
 
 void DataSharingProcessor::doPrivatize(
-    const Fortran::semantics::Symbol *sym,
-    mlir::omp::PrivateClauseOps *clauseOps,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
+    const semantics::Symbol *sym, mlir::omp::PrivateClauseOps *clauseOps,
+    llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
   if (!useDelayedPrivatization) {
     cloneSymbol(sym);
     copyFirstPrivateSymbol(sym);
     return;
   }
 
-  Fortran::lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
+  lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
   assert(hsb && "Host symbol box not found");
 
   mlir::Type symType = hsb.getAddr().getType();
   mlir::Location symLoc = hsb.getAddr().getLoc();
   std::string privatizerName = sym->name().ToString() + ".privatizer";
-  bool isFirstPrivate =
-      sym->test(Fortran::semantics::Symbol::Flag::OmpFirstPrivate);
+  bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
 
   mlir::omp::PrivateClauseOp privatizerOp = [&]() {
     auto moduleOp = firOpBuilder.getModule();
@@ -468,8 +455,7 @@ void DataSharingProcessor::doPrivatize(
           hlfir::translateToExtendedValue(
               symLoc, firOpBuilder, hlfir::Entity{allocRegion.getArgument(0)},
               /*contiguousHint=*/
-              Fortran::evaluate::IsSimplyContiguous(
-                  *sym, converter.getFoldingContext()))
+              evaluate::IsSimplyContiguous(*sym, converter.getFoldingContext()))
               .first;
 
       symTable->addSymbol(*sym, localExV);
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index ec6848f7bba3..111266eeb784 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -37,65 +37,61 @@ private:
   mlir::OpBuilder::InsertPoint insPt;
   mlir::Value loopIV;
   // Symbols in private, firstprivate, and/or lastprivate clauses.
-  llvm::SetVector<const Fortran::semantics::Symbol *> privatizedSymbols;
-  llvm::SetVector<const Fortran::semantics::Symbol *> defaultSymbols;
-  llvm::SetVector<const Fortran::semantics::Symbol *> implicitSymbols;
-  llvm::DenseMap<const Fortran::semantics::Symbol *, mlir::omp::PrivateClauseOp>
+  llvm::SetVector<const semantics::Symbol *> privatizedSymbols;
+  llvm::SetVector<const semantics::Symbol *> defaultSymbols;
+  llvm::SetVector<const semantics::Symbol *> implicitSymbols;
+  llvm::DenseMap<const semantics::Symbol *, mlir::omp::PrivateClauseOp>
       symToPrivatizer;
-  Fortran::lower::AbstractConverter &converter;
-  Fortran::semantics::SemanticsContext &semaCtx;
+  lower::AbstractConverter &converter;
+  semantics::SemanticsContext &semaCtx;
   fir::FirOpBuilder &firOpBuilder;
   omp::List<omp::Clause> clauses;
-  Fortran::lower::pft::Evaluation &eval;
+  lower::pft::Evaluation &eval;
   bool useDelayedPrivatization;
-  Fortran::lower::SymMap *symTable;
+  lower::SymMap *symTable;
 
   bool needBarrier();
-  void
-  collectSymbols(Fortran::semantics::Symbol::Flag flag,
-                 llvm::SetVector<const Fortran::semantics::Symbol *> &symbols);
+  void collectSymbols(semantics::Symbol::Flag flag,
+                      llvm::SetVector<const semantics::Symbol *> &symbols);
   void collectSymbolsInNestedRegions(
-      Fortran::lower::pft::Evaluation &eval,
-      Fortran::semantics::Symbol::Flag flag,
-      llvm::SetVector<const Fortran::semantics::Symbol *>
-          &symbolsInNestedRegions);
+      lower::pft::Evaluation &eval, semantics::Symbol::Flag flag,
+      llvm::SetVector<const semantics::Symbol *> &symbolsInNestedRegions);
   void collectOmpObjectListSymbol(
       const omp::ObjectList &objects,
-      llvm::SetVector<const Fortran::semantics::Symbol *> &symbolSet);
+      llvm::SetVector<const semantics::Symbol *> &symbolSet);
   void collectSymbolsForPrivatization();
   void insertBarrier();
   void collectDefaultSymbols();
   void collectImplicitSymbols();
-  void privatize(
-      mlir::omp::PrivateClauseOps *clauseOps,
-      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms);
+  void privatize(mlir::omp::PrivateClauseOps *clauseOps,
+                 llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms);
   void defaultPrivatize(
       mlir::omp::PrivateClauseOps *clauseOps,
-      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms);
+      llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms);
   void implicitPrivatize(
       mlir::omp::PrivateClauseOps *clauseOps,
-      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms);
-  void doPrivatize(
-      const Fortran::semantics::Symbol *sym,
-      mlir::omp::PrivateClauseOps *clauseOps,
-      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms);
+      llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms);
+  void
+  doPrivatize(const semantics::Symbol *sym,
+              mlir::omp::PrivateClauseOps *clauseOps,
+              llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms);
   void copyLastPrivatize(mlir::Operation *op);
   void insertLastPrivateCompare(mlir::Operation *op);
-  void cloneSymbol(const Fortran::semantics::Symbol *sym);
+  void cloneSymbol(const semantics::Symbol *sym);
   void
-  copyFirstPrivateSymbol(const Fortran::semantics::Symbol *sym,
+  copyFirstPrivateSymbol(const semantics::Symbol *sym,
                          mlir::OpBuilder::InsertPoint *copyAssignIP = nullptr);
-  void copyLastPrivateSymbol(const Fortran::semantics::Symbol *sym,
+  void copyLastPrivateSymbol(const semantics::Symbol *sym,
                              mlir::OpBuilder::InsertPoint *lastPrivIP);
   void insertDeallocs();
 
 public:
-  DataSharingProcessor(Fortran::lower::AbstractConverter &converter,
-                       Fortran::semantics::SemanticsContext &semaCtx,
+  DataSharingProcessor(lower::AbstractConverter &converter,
+                       semantics::SemanticsContext &semaCtx,
                        const List<Clause> &clauses,
-                       Fortran::lower::pft::Evaluation &eval,
+                       lower::pft::Evaluation &eval,
                        bool useDelayedPrivatization = false,
-                       Fortran::lower::SymMap *symTable = nullptr)
+                       lower::SymMap *symTable = nullptr)
       : hasLastPrivateOp(false), converter(converter), semaCtx(semaCtx),
         firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval),
         useDelayedPrivatization(useDelayedPrivatization), symTable(symTable) {}
@@ -111,9 +107,9 @@ public:
   // Step2 performs the copying for lastprivates and requires knowledge of the
   // MLIR operation to insert the last private update. Step2 adds
   // dealocation code as well.
-  void processStep1(mlir::omp::PrivateClauseOps *clauseOps = nullptr,
-                    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-                        *privateSyms = nullptr);
+  void processStep1(
+      mlir::omp::PrivateClauseOps *clauseOps = nullptr,
+      llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms = nullptr);
   void processStep2(mlir::Operation *op, bool isLoop);
 
   void setLoopIV(mlir::Value iv) {
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index f05cf1f5120f..af27c59eb993 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -45,45 +45,45 @@ using namespace Fortran::lower::omp;
 // Code generation helper functions
 //===----------------------------------------------------------------------===//
 
-static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
-                           Fortran::lower::SymMap &symTable,
-                           Fortran::semantics::SemanticsContext &semaCtx,
-                           Fortran::lower::pft::Evaluation &eval,
-                           mlir::Location loc, const ConstructQueue &queue,
+static void genOMPDispatch(lower::AbstractConverter &converter,
+                           lower::SymMap &symTable,
+                           semantics::SemanticsContext &semaCtx,
+                           lower::pft::Evaluation &eval, mlir::Location loc,
+                           const ConstructQueue &queue,
                            ConstructQueue::iterator item);
 
-static Fortran::lower::pft::Evaluation *
-getCollapsedLoopEval(Fortran::lower::pft::Evaluation &eval, int collapseValue) {
+static lower::pft::Evaluation *
+getCollapsedLoopEval(lower::pft::Evaluation &eval, int collapseValue) {
   // Return the Evaluation of the innermost collapsed loop, or the current one
   // if there was no COLLAPSE.
   if (collapseValue == 0)
     return &eval;
 
-  Fortran::lower::pft::Evaluation *curEval = &eval.getFirstNestedEvaluation();
+  lower::pft::Evaluation *curEval = &eval.getFirstNestedEvaluation();
   for (int i = 1; i < collapseValue; i++) {
     // The nested evaluations should be DoConstructs (i.e. they should form
     // a loop nest). Each DoConstruct is a tuple <NonLabelDoStmt, Block,
     // EndDoStmt>.
-    assert(curEval->isA<Fortran::parser::DoConstruct>());
+    assert(curEval->isA<parser::DoConstruct>());
     curEval = &*std::next(curEval->getNestedEvaluations().begin());
   }
   return curEval;
 }
 
-static void genNestedEvaluations(Fortran::lower::AbstractConverter &converter,
-                                 Fortran::lower::pft::Evaluation &eval,
+static void genNestedEvaluations(lower::AbstractConverter &converter,
+                                 lower::pft::Evaluation &eval,
                                  int collapseValue = 0) {
-  Fortran::lower::pft::Evaluation *curEval =
-      getCollapsedLoopEval(eval, collapseValue);
+  lower::pft::Evaluation *curEval = getCollapsedLoopEval(eval, collapseValue);
 
-  for (Fortran::lower::pft::Evaluation &e : curEval->getNestedEvaluations())
+  for (lower::pft::Evaluation &e : curEval->getNestedEvaluations())
     converter.genEval(e);
 }
 
-static fir::GlobalOp globalInitialization(
-    Fortran::lower::AbstractConverter &converter,
-    fir::FirOpBuilder &firOpBuilder, const Fortran::semantics::Symbol &sym,
-    const Fortran::lower::pft::Variable &var, mlir::Location currentLocation) {
+static fir::GlobalOp globalInitialization(lower::AbstractConverter &converter,
+                                          fir::FirOpBuilder &firOpBuilder,
+                                          const semantics::Symbol &sym,
+                                          const lower::pft::Variable &var,
+                                          mlir::Location currentLocation) {
   mlir::Type ty = converter.genType(sym);
   std::string globalName = converter.mangleName(sym);
   mlir::StringAttr linkage = firOpBuilder.createInternalLinkage();
@@ -91,9 +91,9 @@ static fir::GlobalOp globalInitialization(
       firOpBuilder.createGlobal(currentLocation, ty, globalName, linkage);
 
   // Create default initialization for non-character scalar.
-  if (Fortran::semantics::IsAllocatableOrObjectPointer(&sym)) {
+  if (semantics::IsAllocatableOrObjectPointer(&sym)) {
     mlir::Type baseAddrType = mlir::dyn_cast<fir::BoxType>(ty).getEleTy();
-    Fortran::lower::createGlobalInitialization(
+    lower::createGlobalInitialization(
         firOpBuilder, global, [&](fir::FirOpBuilder &b) {
           mlir::Value nullAddr =
               b.createNullConstant(currentLocation, baseAddrType);
@@ -102,7 +102,7 @@ static fir::GlobalOp globalInitialization(
           b.create<fir::HasValueOp>(currentLocation, box);
         });
   } else {
-    Fortran::lower::createGlobalInitialization(
+    lower::createGlobalInitialization(
         firOpBuilder, global, [&](fir::FirOpBuilder &b) {
           mlir::Value undef = b.create<fir::UndefOp>(currentLocation, ty);
           b.create<fir::HasValueOp>(currentLocation, undef);
@@ -126,20 +126,19 @@ static fir::ExtendedValue getExtendedValue(fir::ExtendedValue base,
 }
 
 #ifndef NDEBUG
-static bool isThreadPrivate(Fortran::lower::SymbolRef sym) {
-  if (const auto *details =
-          sym->detailsIf<Fortran::semantics::CommonBlockDetails>()) {
+static bool isThreadPrivate(lower::SymbolRef sym) {
+  if (const auto *details = sym->detailsIf<semantics::CommonBlockDetails>()) {
     for (const auto &obj : details->objects())
-      if (!obj->test(Fortran::semantics::Symbol::Flag::OmpThreadprivate))
+      if (!obj->test(semantics::Symbol::Flag::OmpThreadprivate))
         return false;
     return true;
   }
-  return sym->test(Fortran::semantics::Symbol::Flag::OmpThreadprivate);
+  return sym->test(semantics::Symbol::Flag::OmpThreadprivate);
 }
 #endif
 
-static void threadPrivatizeVars(Fortran::lower::AbstractConverter &converter,
-                                Fortran::lower::pft::Evaluation &eval) {
+static void threadPrivatizeVars(lower::AbstractConverter &converter,
+                                lower::pft::Evaluation &eval) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::Location currentLocation = converter.getCurrentLocation();
   mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
@@ -152,7 +151,7 @@ static void threadPrivatizeVars(Fortran::lower::AbstractConverter &converter,
   // non-threadprivate variable. This can happen, for instance, with a common
   // block, declared in a separate module, used by a parent procedure and
   // privatized in its child procedure.
-  auto genThreadprivateOp = [&](Fortran::lower::SymbolRef sym) -> mlir::Value {
+  auto genThreadprivateOp = [&](lower::SymbolRef sym) -> mlir::Value {
     assert(isThreadPrivate(sym));
     mlir::Value symValue = converter.getSymbolAddress(sym);
     mlir::Operation *op = symValue.getDefiningOp();
@@ -164,28 +163,28 @@ static void threadPrivatizeVars(Fortran::lower::AbstractConverter &converter,
         currentLocation, symValue.getType(), symValue);
   };
 
-  llvm::SetVector<const Fortran::semantics::Symbol *> threadprivateSyms;
+  llvm::SetVector<const semantics::Symbol *> threadprivateSyms;
   converter.collectSymbolSet(eval, threadprivateSyms,
-                             Fortran::semantics::Symbol::Flag::OmpThreadprivate,
+                             semantics::Symbol::Flag::OmpThreadprivate,
                              /*collectSymbols=*/true,
                              /*collectHostAssociatedSymbols=*/true);
-  std::set<Fortran::semantics::SourceName> threadprivateSymNames;
+  std::set<semantics::SourceName> threadprivateSymNames;
 
   // For a COMMON block, the ThreadprivateOp is generated for itself instead of
   // its members, so only bind the value of the new copied ThreadprivateOp
   // inside the parallel region to the common block symbol only once for
   // multiple members in one COMMON block.
-  llvm::SetVector<const Fortran::semantics::Symbol *> commonSyms;
+  llvm::SetVector<const semantics::Symbol *> commonSyms;
   for (std::size_t i = 0; i < threadprivateSyms.size(); i++) {
-    const Fortran::semantics::Symbol *sym = threadprivateSyms[i];
+    const semantics::Symbol *sym = threadprivateSyms[i];
     mlir::Value symThreadprivateValue;
     // The variable may be used more than once, and each reference has one
     // symbol with the same name. Only do once for references of one variable.
     if (threadprivateSymNames.find(sym->name()) != threadprivateSymNames.end())
       continue;
     threadprivateSymNames.insert(sym->name());
-    if (const Fortran::semantics::Symbol *common =
-            Fortran::semantics::FindCommonBlockContaining(sym->GetUltimate())) {
+    if (const semantics::Symbol *common =
+            semantics::FindCommonBlockContaining(sym->GetUltimate())) {
       mlir::Value commonThreadprivateValue;
       if (commonSyms.contains(common)) {
         commonThreadprivateValue = converter.getSymbolAddress(*common);
@@ -194,7 +193,7 @@ static void threadPrivatizeVars(Fortran::lower::AbstractConverter &converter,
         converter.bindSymbol(*common, commonThreadprivateValue);
         commonSyms.insert(common);
       }
-      symThreadprivateValue = Fortran::lower::genCommonBlockMember(
+      symThreadprivateValue = lower::genCommonBlockMember(
           converter, currentLocation, *sym, commonThreadprivateValue);
     } else {
       symThreadprivateValue = genThreadprivateOp(*sym);
@@ -208,9 +207,9 @@ static void threadPrivatizeVars(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::Operation *
-createAndSetPrivatizedLoopVar(Fortran::lower::AbstractConverter &converter,
+createAndSetPrivatizedLoopVar(lower::AbstractConverter &converter,
                               mlir::Location loc, mlir::Value indexVal,
-                              const Fortran::semantics::Symbol *sym) {
+                              const semantics::Symbol *sym) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::OpBuilder::InsertPoint insPt = firOpBuilder.saveInsertionPoint();
   firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
@@ -250,8 +249,7 @@ static void promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
     mlir::omp::UseDeviceClauseOps &clauseOps,
     llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-        &useDeviceSymbols) {
+    llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceSymbols) {
   auto moveElementToBack = [](size_t idx, auto &vector) {
     auto *iter = std::next(vector.begin(), idx);
     vector.push_back(*iter);
@@ -285,23 +283,20 @@ static void promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
 /// Extract the list of function and variable symbols affected by the given
 /// 'declare target' directive and return the intended device type for them.
 static void getDeclareTargetInfo(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
+    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+    lower::pft::Evaluation &eval,
+    const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
     mlir::omp::DeclareTargetClauseOps &clauseOps,
     llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause) {
-  const auto &spec = std::get<Fortran::parser::OmpDeclareTargetSpecifier>(
-      declareTargetConstruct.t);
-  if (const auto *objectList{
-          Fortran::parser::Unwrap<Fortran::parser::OmpObjectList>(spec.u)}) {
+  const auto &spec =
+      std::get<parser::OmpDeclareTargetSpecifier>(declareTargetConstruct.t);
+  if (const auto *objectList{parser::Unwrap<parser::OmpObjectList>(spec.u)}) {
     ObjectList objects{makeObjects(*objectList, semaCtx)};
     // Case: declare target(func, var1, var2)
     gatherFuncAndVarSyms(objects, mlir::omp::DeclareTargetCaptureClause::to,
                          symbolAndClause);
   } else if (const auto *clauseList{
-                 Fortran::parser::Unwrap<Fortran::parser::OmpClauseList>(
-                     spec.u)}) {
+                 parser::Unwrap<parser::OmpClauseList>(spec.u)}) {
     List<Clause> clauses = makeClauses(*clauseList, semaCtx);
     if (clauses.empty()) {
       // Case: declare target, implicit capture of function
@@ -322,11 +317,10 @@ static void getDeclareTargetInfo(
 }
 
 static void collectDeferredDeclareTargets(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
-    llvm::SmallVectorImpl<Fortran::lower::OMPDeferredDeclareTargetInfo>
+    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+    lower::pft::Evaluation &eval,
+    const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
+    llvm::SmallVectorImpl<lower::OMPDeferredDeclareTargetInfo>
         &deferredDeclareTarget) {
   mlir::omp::DeclareTargetClauseOps clauseOps;
   llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
@@ -337,8 +331,8 @@ static void collectDeferredDeclareTargets(
   mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
 
   for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
-    mlir::Operation *op = mod.lookupSymbol(converter.mangleName(
-        std::get<const Fortran::semantics::Symbol &>(symClause)));
+    mlir::Operation *op = mod.lookupSymbol(
+        converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
 
     if (!op) {
       deferredDeclareTarget.push_back({std::get<0>(symClause),
@@ -350,11 +344,9 @@ static void collectDeferredDeclareTargets(
 
 static std::optional<mlir::omp::DeclareTargetDeviceType>
 getDeclareTargetFunctionDevice(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::OpenMPDeclareTargetConstruct
-        &declareTargetConstruct) {
+    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+    lower::pft::Evaluation &eval,
+    const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) {
   mlir::omp::DeclareTargetClauseOps clauseOps;
   llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
   getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
@@ -364,8 +356,8 @@ getDeclareTargetFunctionDevice(
   // directive is a function or subroutine
   mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
   for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
-    mlir::Operation *op = mod.lookupSymbol(converter.mangleName(
-        std::get<const Fortran::semantics::Symbol &>(symClause)));
+    mlir::Operation *op = mod.lookupSymbol(
+        converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
 
     if (mlir::isa_and_nonnull<mlir::func::FuncOp>(op))
       return clauseOps.deviceType;
@@ -392,16 +384,15 @@ getDeclareTargetFunctionDevice(
 ///                           entry block arguments.
 /// \param [in] wrapperArgs - entry block arguments of parent loop wrappers.
 static void
-genLoopVars(mlir::Operation *op, Fortran::lower::AbstractConverter &converter,
-            mlir::Location &loc,
-            llvm::ArrayRef<const Fortran::semantics::Symbol *> args,
-            llvm::ArrayRef<const Fortran::semantics::Symbol *> wrapperSyms = {},
+genLoopVars(mlir::Operation *op, lower::AbstractConverter &converter,
+            mlir::Location &loc, llvm::ArrayRef<const semantics::Symbol *> args,
+            llvm::ArrayRef<const semantics::Symbol *> wrapperSyms = {},
             llvm::ArrayRef<mlir::BlockArgument> wrapperArgs = {}) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   auto &region = op->getRegion(0);
 
   std::size_t loopVarTypeSize = 0;
-  for (const Fortran::semantics::Symbol *arg : args)
+  for (const semantics::Symbol *arg : args)
     loopVarTypeSize = std::max(loopVarTypeSize, arg->GetUltimate().size());
   mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
   llvm::SmallVector<mlir::Type> tiv(args.size(), loopVarType);
@@ -424,11 +415,11 @@ genLoopVars(mlir::Operation *op, Fortran::lower::AbstractConverter &converter,
   firOpBuilder.setInsertionPointAfter(storeOp);
 }
 
-static void genReductionVars(
-    mlir::Operation *op, Fortran::lower::AbstractConverter &converter,
-    mlir::Location &loc,
-    llvm::ArrayRef<const Fortran::semantics::Symbol *> reductionArgs,
-    llvm::ArrayRef<mlir::Type> reductionTypes) {
+static void
+genReductionVars(mlir::Operation *op, lower::AbstractConverter &converter,
+                 mlir::Location &loc,
+                 llvm::ArrayRef<const semantics::Symbol *> reductionArgs,
+                 llvm::ArrayRef<mlir::Type> reductionTypes) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   llvm::SmallVector<mlir::Location> blockArgLocs(reductionArgs.size(), loc);
 
@@ -443,8 +434,7 @@ static void genReductionVars(
 }
 
 static void
-markDeclareTarget(mlir::Operation *op,
-                  Fortran::lower::AbstractConverter &converter,
+markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter,
                   mlir::omp::DeclareTargetCaptureClause captureClause,
                   mlir::omp::DeclareTargetDeviceType deviceType) {
   // TODO: Add support for program local variables with declare target applied
@@ -477,14 +467,13 @@ struct OpWithBodyGenInfo {
   /// which the code is being generated and returns the arguments of the op's
   /// region.
   using GenOMPRegionEntryCBFn =
-      std::function<llvm::SmallVector<const Fortran::semantics::Symbol *>(
+      std::function<llvm::SmallVector<const semantics::Symbol *>(
           mlir::Operation *)>;
 
-  OpWithBodyGenInfo(Fortran::lower::AbstractConverter &converter,
-                    Fortran::lower::SymMap &symTable,
-                    Fortran::semantics::SemanticsContext &semaCtx,
-                    mlir::Location loc, Fortran::lower::pft::Evaluation &eval,
-                    llvm::omp::Directive dir)
+  OpWithBodyGenInfo(lower::AbstractConverter &converter,
+                    lower::SymMap &symTable,
+                    semantics::SemanticsContext &semaCtx, mlir::Location loc,
+                    lower::pft::Evaluation &eval, llvm::omp::Directive dir)
       : converter(converter), symTable(symTable), semaCtx(semaCtx), loc(loc),
         eval(eval), dir(dir) {}
 
@@ -503,9 +492,9 @@ struct OpWithBodyGenInfo {
     return *this;
   }
 
-  OpWithBodyGenInfo &setReductions(
-      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *value1,
-      llvm::SmallVectorImpl<mlir::Type> *value2) {
+  OpWithBodyGenInfo &
+  setReductions(llvm::SmallVectorImpl<const semantics::Symbol *> *value1,
+                llvm::SmallVectorImpl<mlir::Type> *value2) {
     reductionSymbols = value1;
     reductionTypes = value2;
     return *this;
@@ -517,15 +506,15 @@ struct OpWithBodyGenInfo {
   }
 
   /// [inout] converter to use for the clauses.
-  Fortran::lower::AbstractConverter &converter;
+  lower::AbstractConverter &converter;
   /// [in] Symbol table
-  Fortran::lower::SymMap &symTable;
+  lower::SymMap &symTable;
   /// [in] Semantics context
-  Fortran::semantics::SemanticsContext &semaCtx;
+  semantics::SemanticsContext &semaCtx;
   /// [in] location in source code.
   mlir::Location loc;
   /// [in] current PFT node/evaluation.
-  Fortran::lower::pft::Evaluation &eval;
+  lower::pft::Evaluation &eval;
   /// [in] leaf directive for which to generate the op body.
   llvm::omp::Directive dir;
   /// [in] is this an outer operation - prevents privatization.
@@ -535,8 +524,7 @@ struct OpWithBodyGenInfo {
   /// [in] if provided, processes the construct's data-sharing attributes.
   DataSharingProcessor *dsp = nullptr;
   /// [in] if provided, list of reduction symbols
-  llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *reductionSymbols =
-      nullptr;
+  llvm::SmallVectorImpl<const semantics::Symbol *> *reductionSymbols = nullptr;
   /// [in] if provided, list of reduction types
   llvm::SmallVectorImpl<mlir::Type> *reductionTypes = nullptr;
   /// [in] if provided, emits the op's region entry. Otherwise, an emtpy block
@@ -565,8 +553,7 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
   // argument. Also update the symbol's address with the mlir argument value.
   // e.g. For loops the argument is the induction variable. And all further
   // uses of the induction variable should use this mlir value.
-  auto regionArgs =
-      [&]() -> llvm::SmallVector<const Fortran::semantics::Symbol *> {
+  auto regionArgs = [&]() -> llvm::SmallVector<const semantics::Symbol *> {
     if (info.genRegionEntryCB != nullptr) {
       return info.genRegionEntryCB(&op);
     }
@@ -580,8 +567,7 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
   // If it is an unstructured region and is not the outer region of a combined
   // construct, create empty blocks for all evaluations.
   if (info.eval.lowerAsUnstructured() && !info.outerCombined)
-    Fortran::lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp,
-                                            mlir::omp::YieldOp>(
+    lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(
         firOpBuilder, info.eval.getNestedEvaluations());
 
   // Start with privatization, so that the lowering of the nested
@@ -617,8 +603,7 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
     // is delayed past this point. Insert a temporary terminator here, then
     // delete it.
     firOpBuilder.setInsertionPointToEnd(&op.getRegion(0).back());
-    auto *temp =
-        Fortran::lower::genOpenMPTerminator(firOpBuilder, &op, info.loc);
+    auto *temp = lower::genOpenMPTerminator(firOpBuilder, &op, info.loc);
     firOpBuilder.setInsertionPointAfter(marker);
     genNestedEvaluations(info.converter, info.eval);
     temp->erase();
@@ -662,8 +647,7 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
 
   if (auto *exitBlock = getUniqueExit(op.getRegion(0))) {
     firOpBuilder.setInsertionPointToEnd(exitBlock);
-    auto *term =
-        Fortran::lower::genOpenMPTerminator(firOpBuilder, &op, info.loc);
+    auto *term = lower::genOpenMPTerminator(firOpBuilder, &op, info.loc);
     // Only insert lastprivate code when there actually is an exit block.
     // Such a block may not exist if the nested code produced an infinite
     // loop (this may not make sense in production code, but a user could
@@ -699,13 +683,11 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
 }
 
 static void genBodyOfTargetDataOp(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::lower::SymMap &symTable,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, mlir::omp::TargetDataOp &dataOp,
-    llvm::ArrayRef<mlir::Type> useDeviceTypes,
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    mlir::omp::TargetDataOp &dataOp, llvm::ArrayRef<mlir::Type> useDeviceTypes,
     llvm::ArrayRef<mlir::Location> useDeviceLocs,
-    llvm::ArrayRef<const Fortran::semantics::Symbol *> useDeviceSymbols,
+    llvm::ArrayRef<const semantics::Symbol *> useDeviceSymbols,
     const mlir::Location &currentLocation, const ConstructQueue &queue,
     ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
@@ -721,7 +703,7 @@ static void genBodyOfTargetDataOp(
         converter.bindSymbol(*argSymbol, arg);
       } else {
         // Avoid capture of a reference to a structured binding.
-        const Fortran::semantics::Symbol *sym = argSymbol;
+        const semantics::Symbol *sym = argSymbol;
         extVal.match(
             [&](const fir::MutableBoxValue &mbv) {
               converter.bindSymbol(
@@ -751,8 +733,7 @@ static void genBodyOfTargetDataOp(
   // Create blocks for unstructured regions. This has to be done since
   // blocks are initially allocated with the function as the parent region.
   if (eval.lowerAsUnstructured()) {
-    Fortran::lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp,
-                                            mlir::omp::YieldOp>(
+    lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(
         firOpBuilder, eval.getNestedEvaluations());
   }
 
@@ -772,12 +753,10 @@ static void genBodyOfTargetDataOp(
 // This functions creates a block for the body of the targetOp's region. It adds
 // all the symbols present in mapSymbols as block arguments to this block.
 static void
-genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
-                  Fortran::lower::SymMap &symTable,
-                  Fortran::semantics::SemanticsContext &semaCtx,
-                  Fortran::lower::pft::Evaluation &eval,
-                  mlir::omp::TargetOp &targetOp,
-                  llvm::ArrayRef<const Fortran::semantics::Symbol *> mapSyms,
+genBodyOfTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                  semantics::SemanticsContext &semaCtx,
+                  lower::pft::Evaluation &eval, mlir::omp::TargetOp &targetOp,
+                  llvm::ArrayRef<const semantics::Symbol *> mapSyms,
                   llvm::ArrayRef<mlir::Location> mapSymLocs,
                   llvm::ArrayRef<mlir::Type> mapSymTypes,
                   const mlir::Location &currentLocation,
@@ -812,7 +791,7 @@ genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
   for (auto [argIndex, argSymbol] : llvm::enumerate(mapSyms)) {
     const mlir::BlockArgument &arg = region.getArgument(argIndex);
     // Avoid capture of a reference to a structured binding.
-    const Fortran::semantics::Symbol *sym = argSymbol;
+    const semantics::Symbol *sym = argSymbol;
     // Structure component symbols don't have bindings.
     if (sym->owner().IsDerivedType())
       continue;
@@ -914,8 +893,7 @@ genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
   // Create blocks for unstructured regions. This has to be done since
   // blocks are initially allocated with the function as the parent region.
   if (eval.lowerAsUnstructured()) {
-    Fortran::lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp,
-                                            mlir::omp::YieldOp>(
+    lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(
         firOpBuilder, eval.getNestedEvaluations());
   }
 
@@ -946,20 +924,20 @@ static OpTy genOpWithBody(const OpWithBodyGenInfo &info,
 // Code generation functions for clauses
 //===----------------------------------------------------------------------===//
 
-static void
-genCriticalDeclareClauses(Fortran::lower::AbstractConverter &converter,
-                          Fortran::semantics::SemanticsContext &semaCtx,
-                          const List<Clause> &clauses, mlir::Location loc,
-                          mlir::omp::CriticalClauseOps &clauseOps,
-                          llvm::StringRef name) {
+static void genCriticalDeclareClauses(lower::AbstractConverter &converter,
+                                      semantics::SemanticsContext &semaCtx,
+                                      const List<Clause> &clauses,
+                                      mlir::Location loc,
+                                      mlir::omp::CriticalClauseOps &clauseOps,
+                                      llvm::StringRef name) {
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processHint(clauseOps);
   clauseOps.nameAttr =
       mlir::StringAttr::get(converter.getFirOpBuilder().getContext(), name);
 }
 
-static void genFlushClauses(Fortran::lower::AbstractConverter &converter,
-                            Fortran::semantics::SemanticsContext &semaCtx,
+static void genFlushClauses(lower::AbstractConverter &converter,
+                            semantics::SemanticsContext &semaCtx,
                             const ObjectList &objects,
                             const List<Clause> &clauses, mlir::Location loc,
                             llvm::SmallVectorImpl<mlir::Value> &operandRange) {
@@ -970,20 +948,20 @@ static void genFlushClauses(Fortran::lower::AbstractConverter &converter,
     TODO(converter.getCurrentLocation(), "Handle OmpMemoryOrderClause");
 }
 
-static void genLoopNestClauses(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, const List<Clause> &clauses,
-    mlir::Location loc, mlir::omp::LoopNestClauseOps &clauseOps,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &iv) {
+static void
+genLoopNestClauses(lower::AbstractConverter &converter,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval, const List<Clause> &clauses,
+                   mlir::Location loc, mlir::omp::LoopNestClauseOps &clauseOps,
+                   llvm::SmallVectorImpl<const semantics::Symbol *> &iv) {
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processCollapse(loc, eval, clauseOps, iv);
   clauseOps.loopInclusiveAttr = converter.getFirOpBuilder().getUnitAttr();
 }
 
 static void
-genOrderedRegionClauses(Fortran::lower::AbstractConverter &converter,
-                        Fortran::semantics::SemanticsContext &semaCtx,
+genOrderedRegionClauses(lower::AbstractConverter &converter,
+                        semantics::SemanticsContext &semaCtx,
                         const List<Clause> &clauses, mlir::Location loc,
                         mlir::omp::OrderedRegionClauseOps &clauseOps) {
   ClauseProcessor cp(converter, semaCtx, clauses);
@@ -991,13 +969,12 @@ genOrderedRegionClauses(Fortran::lower::AbstractConverter &converter,
 }
 
 static void genParallelClauses(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::StatementContext &stmtCtx, const List<Clause> &clauses,
+    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+    lower::StatementContext &stmtCtx, const List<Clause> &clauses,
     mlir::Location loc, bool processReduction,
     mlir::omp::ParallelClauseOps &clauseOps,
     llvm::SmallVectorImpl<mlir::Type> &reductionTypes,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &reductionSyms) {
+    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processAllocate(clauseOps);
   cp.processDefault();
@@ -1012,8 +989,8 @@ static void genParallelClauses(
   }
 }
 
-static void genSectionsClauses(Fortran::lower::AbstractConverter &converter,
-                               Fortran::semantics::SemanticsContext &semaCtx,
+static void genSectionsClauses(lower::AbstractConverter &converter,
+                               semantics::SemanticsContext &semaCtx,
                                const List<Clause> &clauses, mlir::Location loc,
                                mlir::omp::SectionsClauseOps &clauseOps) {
   ClauseProcessor cp(converter, semaCtx, clauses);
@@ -1023,8 +1000,8 @@ static void genSectionsClauses(Fortran::lower::AbstractConverter &converter,
   // TODO Support delayed privatization.
 }
 
-static void genSimdClauses(Fortran::lower::AbstractConverter &converter,
-                           Fortran::semantics::SemanticsContext &semaCtx,
+static void genSimdClauses(lower::AbstractConverter &converter,
+                           semantics::SemanticsContext &semaCtx,
                            const List<Clause> &clauses, mlir::Location loc,
                            mlir::omp::SimdClauseOps &clauseOps) {
   ClauseProcessor cp(converter, semaCtx, clauses);
@@ -1039,8 +1016,8 @@ static void genSimdClauses(Fortran::lower::AbstractConverter &converter,
       loc, llvm::omp::Directive::OMPD_simd);
 }
 
-static void genSingleClauses(Fortran::lower::AbstractConverter &converter,
-                             Fortran::semantics::SemanticsContext &semaCtx,
+static void genSingleClauses(lower::AbstractConverter &converter,
+                             semantics::SemanticsContext &semaCtx,
                              const List<Clause> &clauses, mlir::Location loc,
                              mlir::omp::SingleClauseOps &clauseOps) {
   ClauseProcessor cp(converter, semaCtx, clauses);
@@ -1051,18 +1028,17 @@ static void genSingleClauses(Fortran::lower::AbstractConverter &converter,
 }
 
 static void genTargetClauses(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::StatementContext &stmtCtx, const List<Clause> &clauses,
+    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+    lower::StatementContext &stmtCtx, const List<Clause> &clauses,
     mlir::Location loc, bool processHostOnlyClauses, bool processReduction,
     mlir::omp::TargetClauseOps &clauseOps,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &mapSyms,
+    llvm::SmallVectorImpl<const semantics::Symbol *> &mapSyms,
     llvm::SmallVectorImpl<mlir::Location> &mapLocs,
     llvm::SmallVectorImpl<mlir::Type> &mapTypes,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &deviceAddrSyms,
+    llvm::SmallVectorImpl<const semantics::Symbol *> &deviceAddrSyms,
     llvm::SmallVectorImpl<mlir::Location> &deviceAddrLocs,
     llvm::SmallVectorImpl<mlir::Type> &deviceAddrTypes,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &devicePtrSyms,
+    llvm::SmallVectorImpl<const semantics::Symbol *> &devicePtrSyms,
     llvm::SmallVectorImpl<mlir::Location> &devicePtrLocs,
     llvm::SmallVectorImpl<mlir::Type> &devicePtrTypes) {
   ClauseProcessor cp(converter, semaCtx, clauses);
@@ -1087,13 +1063,12 @@ static void genTargetClauses(
 }
 
 static void genTargetDataClauses(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::StatementContext &stmtCtx, const List<Clause> &clauses,
+    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+    lower::StatementContext &stmtCtx, const List<Clause> &clauses,
     mlir::Location loc, mlir::omp::TargetDataClauseOps &clauseOps,
     llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &useDeviceSyms) {
+    llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceSyms) {
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processDevice(stmtCtx, clauseOps);
   cp.processIf(llvm::omp::Directive::OMPD_target_data, clauseOps);
@@ -1119,9 +1094,8 @@ static void genTargetDataClauses(
 }
 
 static void genTargetEnterExitUpdateDataClauses(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::StatementContext &stmtCtx, const List<Clause> &clauses,
+    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+    lower::StatementContext &stmtCtx, const List<Clause> &clauses,
     mlir::Location loc, llvm::omp::Directive directive,
     mlir::omp::TargetEnterExitUpdateDataClauseOps &clauseOps) {
   ClauseProcessor cp(converter, semaCtx, clauses);
@@ -1138,9 +1112,9 @@ static void genTargetEnterExitUpdateDataClauses(
   }
 }
 
-static void genTaskClauses(Fortran::lower::AbstractConverter &converter,
-                           Fortran::semantics::SemanticsContext &semaCtx,
-                           Fortran::lower::StatementContext &stmtCtx,
+static void genTaskClauses(lower::AbstractConverter &converter,
+                           semantics::SemanticsContext &semaCtx,
+                           lower::StatementContext &stmtCtx,
                            const List<Clause> &clauses, mlir::Location loc,
                            mlir::omp::TaskClauseOps &clauseOps) {
   ClauseProcessor cp(converter, semaCtx, clauses);
@@ -1158,8 +1132,8 @@ static void genTaskClauses(Fortran::lower::AbstractConverter &converter,
       loc, llvm::omp::Directive::OMPD_task);
 }
 
-static void genTaskgroupClauses(Fortran::lower::AbstractConverter &converter,
-                                Fortran::semantics::SemanticsContext &semaCtx,
+static void genTaskgroupClauses(lower::AbstractConverter &converter,
+                                semantics::SemanticsContext &semaCtx,
                                 const List<Clause> &clauses, mlir::Location loc,
                                 mlir::omp::TaskgroupClauseOps &clauseOps) {
   ClauseProcessor cp(converter, semaCtx, clauses);
@@ -1168,8 +1142,8 @@ static void genTaskgroupClauses(Fortran::lower::AbstractConverter &converter,
                                         llvm::omp::Directive::OMPD_taskgroup);
 }
 
-static void genTaskwaitClauses(Fortran::lower::AbstractConverter &converter,
-                               Fortran::semantics::SemanticsContext &semaCtx,
+static void genTaskwaitClauses(lower::AbstractConverter &converter,
+                               semantics::SemanticsContext &semaCtx,
                                const List<Clause> &clauses, mlir::Location loc,
                                mlir::omp::TaskwaitClauseOps &clauseOps) {
   ClauseProcessor cp(converter, semaCtx, clauses);
@@ -1177,9 +1151,9 @@ static void genTaskwaitClauses(Fortran::lower::AbstractConverter &converter,
       loc, llvm::omp::Directive::OMPD_taskwait);
 }
 
-static void genTeamsClauses(Fortran::lower::AbstractConverter &converter,
-                            Fortran::semantics::SemanticsContext &semaCtx,
-                            Fortran::lower::StatementContext &stmtCtx,
+static void genTeamsClauses(lower::AbstractConverter &converter,
+                            semantics::SemanticsContext &semaCtx,
+                            lower::StatementContext &stmtCtx,
                             const List<Clause> &clauses, mlir::Location loc,
                             mlir::omp::TeamsClauseOps &clauseOps) {
   ClauseProcessor cp(converter, semaCtx, clauses);
@@ -1194,12 +1168,11 @@ static void genTeamsClauses(Fortran::lower::AbstractConverter &converter,
 }
 
 static void genWsloopClauses(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::StatementContext &stmtCtx, const List<Clause> &clauses,
+    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+    lower::StatementContext &stmtCtx, const List<Clause> &clauses,
     mlir::Location loc, mlir::omp::WsloopClauseOps &clauseOps,
     llvm::SmallVectorImpl<mlir::Type> &reductionTypes,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &reductionSyms) {
+    llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processNowait(clauseOps);
@@ -1220,21 +1193,19 @@ static void genWsloopClauses(
 //===----------------------------------------------------------------------===//
 
 static mlir::omp::BarrierOp
-genBarrierOp(Fortran::lower::AbstractConverter &converter,
-             Fortran::lower::SymMap &symTable,
-             Fortran::semantics::SemanticsContext &semaCtx,
-             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-             const ConstructQueue &queue, ConstructQueue::iterator item) {
+genBarrierOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+             semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+             mlir::Location loc, const ConstructQueue &queue,
+             ConstructQueue::iterator item) {
   return converter.getFirOpBuilder().create<mlir::omp::BarrierOp>(loc);
 }
 
 static mlir::omp::CriticalOp
-genCriticalOp(Fortran::lower::AbstractConverter &converter,
-              Fortran::lower::SymMap &symTable,
-              Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+genCriticalOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+              semantics::SemanticsContext &semaCtx,
+              lower::pft::Evaluation &eval, mlir::Location loc,
               const ConstructQueue &queue, ConstructQueue::iterator item,
-              const std::optional<Fortran::parser::Name> &name) {
+              const std::optional<parser::Name> &name) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::FlatSymbolRefAttr nameAttr;
 
@@ -1261,22 +1232,19 @@ genCriticalOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::DistributeOp
-genDistributeOp(Fortran::lower::AbstractConverter &converter,
-                Fortran::lower::SymMap &symTable,
-                Fortran::semantics::SemanticsContext &semaCtx,
-                Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+genDistributeOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                semantics::SemanticsContext &semaCtx,
+                lower::pft::Evaluation &eval, mlir::Location loc,
                 const ConstructQueue &queue, ConstructQueue::iterator item) {
   TODO(loc, "Distribute construct");
   return nullptr;
 }
 
 static mlir::omp::FlushOp
-genFlushOp(Fortran::lower::AbstractConverter &converter,
-           Fortran::lower::SymMap &symTable,
-           Fortran::semantics::SemanticsContext &semaCtx,
-           Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-           const ObjectList &objects, const ConstructQueue &queue,
-           ConstructQueue::iterator item) {
+genFlushOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+           semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+           mlir::Location loc, const ObjectList &objects,
+           const ConstructQueue &queue, ConstructQueue::iterator item) {
   llvm::SmallVector<mlir::Value> operandRange;
   genFlushClauses(converter, semaCtx, objects, item->clauses, loc,
                   operandRange);
@@ -1286,11 +1254,10 @@ genFlushOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::MasterOp
-genMasterOp(Fortran::lower::AbstractConverter &converter,
-            Fortran::lower::SymMap &symTable,
-            Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const ConstructQueue &queue, ConstructQueue::iterator item) {
+genMasterOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+            mlir::Location loc, const ConstructQueue &queue,
+            ConstructQueue::iterator item) {
   return genOpWithBody<mlir::omp::MasterOp>(
       OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
                         llvm::omp::Directive::OMPD_master),
@@ -1298,20 +1265,18 @@ genMasterOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::OrderedOp
-genOrderedOp(Fortran::lower::AbstractConverter &converter,
-             Fortran::lower::SymMap &symTable,
-             Fortran::semantics::SemanticsContext &semaCtx,
-             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-             const ConstructQueue &queue, ConstructQueue::iterator item) {
+genOrderedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+             semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+             mlir::Location loc, const ConstructQueue &queue,
+             ConstructQueue::iterator item) {
   TODO(loc, "OMPD_ordered");
   return nullptr;
 }
 
 static mlir::omp::OrderedRegionOp
-genOrderedRegionOp(Fortran::lower::AbstractConverter &converter,
-                   Fortran::lower::SymMap &symTable,
-                   Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+genOrderedRegionOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval, mlir::Location loc,
                    const ConstructQueue &queue, ConstructQueue::iterator item) {
   mlir::omp::OrderedRegionClauseOps clauseOps;
   genOrderedRegionClauses(converter, semaCtx, item->clauses, loc, clauseOps);
@@ -1323,18 +1288,17 @@ genOrderedRegionOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::ParallelOp
-genParallelOp(Fortran::lower::AbstractConverter &converter,
-              Fortran::lower::SymMap &symTable,
-              Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+              semantics::SemanticsContext &semaCtx,
+              lower::pft::Evaluation &eval, mlir::Location loc,
               const ConstructQueue &queue, ConstructQueue::iterator item,
               bool outerCombined = false) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  Fortran::lower::StatementContext stmtCtx;
+  lower::StatementContext stmtCtx;
   mlir::omp::ParallelClauseOps clauseOps;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> privateSyms;
+  llvm::SmallVector<const semantics::Symbol *> privateSyms;
   llvm::SmallVector<mlir::Type> reductionTypes;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSyms;
+  llvm::SmallVector<const semantics::Symbol *> reductionSyms;
   genParallelClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
                      /*processReduction=*/!outerCombined, clauseOps,
                      reductionTypes, reductionSyms);
@@ -1385,15 +1349,14 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
     firOpBuilder.createBlock(&region, /*insertPt=*/{}, privateVarTypes,
                              privateVarLocs);
 
-    llvm::SmallVector<const Fortran::semantics::Symbol *> allSymbols =
-        reductionSyms;
+    llvm::SmallVector<const semantics::Symbol *> allSymbols = reductionSyms;
     allSymbols.append(privateSyms);
     for (auto [arg, prv] : llvm::zip_equal(allSymbols, region.getArguments())) {
       fir::ExtendedValue hostExV = converter.getSymbolExtendedValue(*arg);
       converter.bindSymbol(*arg, hlfir::translateToExtendedValue(
                                      loc, firOpBuilder, hlfir::Entity{prv},
                                      /*contiguousHint=*/
-                                     Fortran::evaluate::IsSimplyContiguous(
+                                     evaluate::IsSimplyContiguous(
                                          *arg, converter.getFoldingContext()))
                                      .first);
     }
@@ -1406,11 +1369,10 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::SectionOp
-genSectionOp(Fortran::lower::AbstractConverter &converter,
-             Fortran::lower::SymMap &symTable,
-             Fortran::semantics::SemanticsContext &semaCtx,
-             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-             const ConstructQueue &queue, ConstructQueue::iterator item) {
+genSectionOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+             semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+             mlir::Location loc, const ConstructQueue &queue,
+             ConstructQueue::iterator item) {
   // Currently only private/firstprivate clause is handled, and
   // all privatization is done within `omp.section` operations.
   return genOpWithBody<mlir::omp::SectionOp>(
@@ -1421,10 +1383,9 @@ genSectionOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::SectionsOp
-genSectionsOp(Fortran::lower::AbstractConverter &converter,
-              Fortran::lower::SymMap &symTable,
-              Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+              semantics::SemanticsContext &semaCtx,
+              lower::pft::Evaluation &eval, mlir::Location loc,
               const ConstructQueue &queue, ConstructQueue::iterator item) {
   mlir::omp::SectionsClauseOps clauseOps;
   genSectionsClauses(converter, semaCtx, item->clauses, loc, clauseOps);
@@ -1478,7 +1439,7 @@ genSectionsOp(Fortran::lower::AbstractConverter &converter,
       mlir::OpBuilder::InsertPoint insp = builder.saveInsertionPoint();
       const auto &objList = std::get<ObjectList>(lastp->t);
       for (const Object &object : objList) {
-        Fortran::semantics::Symbol *sym = object.id();
+        semantics::Symbol *sym = object.id();
         converter.copyHostAssociateVar(*sym, &insp);
       }
     }
@@ -1498,19 +1459,18 @@ genSectionsOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::SimdOp
-genSimdOp(Fortran::lower::AbstractConverter &converter,
-          Fortran::lower::SymMap &symTable,
-          Fortran::semantics::SemanticsContext &semaCtx,
-          Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-          const ConstructQueue &queue, ConstructQueue::iterator item) {
+genSimdOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+          semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+          mlir::Location loc, const ConstructQueue &queue,
+          ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval);
   dsp.processStep1();
 
-  Fortran::lower::StatementContext stmtCtx;
+  lower::StatementContext stmtCtx;
   mlir::omp::LoopNestClauseOps loopClauseOps;
   mlir::omp::SimdClauseOps simdClauseOps;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> iv;
+  llvm::SmallVector<const semantics::Symbol *> iv;
   genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
                      loopClauseOps, iv);
   genSimdClauses(converter, semaCtx, item->clauses, loc, simdClauseOps);
@@ -1521,7 +1481,7 @@ genSimdOp(Fortran::lower::AbstractConverter &converter,
   // TODO: Add reduction-related arguments to the wrapper's entry block.
   firOpBuilder.createBlock(&simdOp.getRegion());
   firOpBuilder.setInsertionPoint(
-      Fortran::lower::genOpenMPTerminator(firOpBuilder, simdOp, loc));
+      lower::genOpenMPTerminator(firOpBuilder, simdOp, loc));
 
   // Create nested omp.loop_nest and fill body with loop contents.
   auto loopOp = firOpBuilder.create<mlir::omp::LoopNestOp>(loc, loopClauseOps);
@@ -1546,11 +1506,10 @@ genSimdOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::SingleOp
-genSingleOp(Fortran::lower::AbstractConverter &converter,
-            Fortran::lower::SymMap &symTable,
-            Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const ConstructQueue &queue, ConstructQueue::iterator item) {
+genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+            mlir::Location loc, const ConstructQueue &queue,
+            ConstructQueue::iterator item) {
   mlir::omp::SingleClauseOps clauseOps;
   genSingleClauses(converter, semaCtx, item->clauses, loc, clauseOps);
 
@@ -1562,21 +1521,19 @@ genSingleOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::TargetOp
-genTargetOp(Fortran::lower::AbstractConverter &converter,
-            Fortran::lower::SymMap &symTable,
-            Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const ConstructQueue &queue, ConstructQueue::iterator item,
-            bool outerCombined = false) {
+genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+            mlir::Location loc, const ConstructQueue &queue,
+            ConstructQueue::iterator item, bool outerCombined = false) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  Fortran::lower::StatementContext stmtCtx;
+  lower::StatementContext stmtCtx;
 
   bool processHostOnlyClauses =
       !llvm::cast<mlir::omp::OffloadModuleInterface>(*converter.getModuleOp())
            .getIsTargetDevice();
 
   mlir::omp::TargetClauseOps clauseOps;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> mapSyms, devicePtrSyms,
+  llvm::SmallVector<const semantics::Symbol *> mapSyms, devicePtrSyms,
       deviceAddrSyms;
   llvm::SmallVector<mlir::Location> mapLocs, devicePtrLocs, deviceAddrLocs;
   llvm::SmallVector<mlir::Type> mapTypes, devicePtrTypes, deviceAddrTypes;
@@ -1590,12 +1547,12 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
   // The following code follows the implicit data-mapping rules to map all the
   // symbols used inside the region that have not been explicitly mapped using
   // the map clause.
-  auto captureImplicitMap = [&](const Fortran::semantics::Symbol &sym) {
+  auto captureImplicitMap = [&](const semantics::Symbol &sym) {
     if (llvm::find(mapSyms, &sym) == mapSyms.end()) {
       mlir::Value baseOp = converter.getSymbolAddress(sym);
       if (!baseOp)
-        if (const auto *details = sym.template detailsIf<
-                                  Fortran::semantics::HostAssocDetails>()) {
+        if (const auto *details =
+                sym.template detailsIf<semantics::HostAssocDetails>()) {
           baseOp = converter.getSymbolAddress(details->symbol());
           converter.copySymbolBinding(details->symbol(), sym);
         }
@@ -1606,21 +1563,20 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
         fir::ExtendedValue dataExv = converter.getSymbolExtendedValue(sym);
         name << sym.name().ToString();
 
-        Fortran::lower::AddrAndBoundsInfo info = getDataOperandBaseAddr(
+        lower::AddrAndBoundsInfo info = getDataOperandBaseAddr(
             converter, firOpBuilder, sym, converter.getCurrentLocation());
         if (mlir::isa<fir::BaseBoxType>(
                 fir::unwrapRefType(info.addr.getType())))
-          bounds =
-              Fortran::lower::genBoundsOpsFromBox<mlir::omp::MapBoundsOp,
-                                                  mlir::omp::MapBoundsType>(
-                  firOpBuilder, converter.getCurrentLocation(), converter,
-                  dataExv, info);
+          bounds = lower::genBoundsOpsFromBox<mlir::omp::MapBoundsOp,
+                                              mlir::omp::MapBoundsType>(
+              firOpBuilder, converter.getCurrentLocation(), converter, dataExv,
+              info);
         if (mlir::isa<fir::SequenceType>(
                 fir::unwrapRefType(info.addr.getType()))) {
           bool dataExvIsAssumedSize =
-              Fortran::semantics::IsAssumedSizeArray(sym.GetUltimate());
-          bounds = Fortran::lower::genBaseBoundsOps<mlir::omp::MapBoundsOp,
-                                                    mlir::omp::MapBoundsType>(
+              semantics::IsAssumedSizeArray(sym.GetUltimate());
+          bounds = lower::genBaseBoundsOps<mlir::omp::MapBoundsOp,
+                                           mlir::omp::MapBoundsType>(
               firOpBuilder, converter.getCurrentLocation(), converter, dataExv,
               dataExvIsAssumedSize);
         }
@@ -1671,7 +1627,7 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
       }
     }
   };
-  Fortran::lower::pft::visitAllSymbols(eval, captureImplicitMap);
+  lower::pft::visitAllSymbols(eval, captureImplicitMap);
 
   auto targetOp = firOpBuilder.create<mlir::omp::TargetOp>(loc, clauseOps);
   genBodyOfTargetOp(converter, symTable, semaCtx, eval, targetOp, mapSyms,
@@ -1680,16 +1636,15 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::TargetDataOp
-genTargetDataOp(Fortran::lower::AbstractConverter &converter,
-                Fortran::lower::SymMap &symTable,
-                Fortran::semantics::SemanticsContext &semaCtx,
-                Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+genTargetDataOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                semantics::SemanticsContext &semaCtx,
+                lower::pft::Evaluation &eval, mlir::Location loc,
                 const ConstructQueue &queue, ConstructQueue::iterator item) {
-  Fortran::lower::StatementContext stmtCtx;
+  lower::StatementContext stmtCtx;
   mlir::omp::TargetDataClauseOps clauseOps;
   llvm::SmallVector<mlir::Type> useDeviceTypes;
   llvm::SmallVector<mlir::Location> useDeviceLocs;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> useDeviceSyms;
+  llvm::SmallVector<const semantics::Symbol *> useDeviceSyms;
   genTargetDataClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
                        clauseOps, useDeviceTypes, useDeviceLocs, useDeviceSyms);
 
@@ -1703,14 +1658,14 @@ genTargetDataOp(Fortran::lower::AbstractConverter &converter,
 }
 
 template <typename OpTy>
-static OpTy
-genTargetEnterExitUpdateDataOp(Fortran::lower::AbstractConverter &converter,
-                               Fortran::lower::SymMap &symTable,
-                               Fortran::semantics::SemanticsContext &semaCtx,
-                               mlir::Location loc, const ConstructQueue &queue,
-                               ConstructQueue::iterator item) {
+static OpTy genTargetEnterExitUpdateDataOp(lower::AbstractConverter &converter,
+                                           lower::SymMap &symTable,
+                                           semantics::SemanticsContext &semaCtx,
+                                           mlir::Location loc,
+                                           const ConstructQueue &queue,
+                                           ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  Fortran::lower::StatementContext stmtCtx;
+  lower::StatementContext stmtCtx;
 
   // GCC 9.3.0 emits a (probably) bogus warning about an unused variable.
   [[maybe_unused]] llvm::omp::Directive directive;
@@ -1732,12 +1687,11 @@ genTargetEnterExitUpdateDataOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::TaskOp
-genTaskOp(Fortran::lower::AbstractConverter &converter,
-          Fortran::lower::SymMap &symTable,
-          Fortran::semantics::SemanticsContext &semaCtx,
-          Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-          const ConstructQueue &queue, ConstructQueue::iterator item) {
-  Fortran::lower::StatementContext stmtCtx;
+genTaskOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+          semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+          mlir::Location loc, const ConstructQueue &queue,
+          ConstructQueue::iterator item) {
+  lower::StatementContext stmtCtx;
   mlir::omp::TaskClauseOps clauseOps;
   genTaskClauses(converter, semaCtx, stmtCtx, item->clauses, loc, clauseOps);
 
@@ -1749,10 +1703,9 @@ genTaskOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::TaskgroupOp
-genTaskgroupOp(Fortran::lower::AbstractConverter &converter,
-               Fortran::lower::SymMap &symTable,
-               Fortran::semantics::SemanticsContext &semaCtx,
-               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+genTaskgroupOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+               semantics::SemanticsContext &semaCtx,
+               lower::pft::Evaluation &eval, mlir::Location loc,
                const ConstructQueue &queue, ConstructQueue::iterator item) {
   mlir::omp::TaskgroupClauseOps clauseOps;
   genTaskgroupClauses(converter, semaCtx, item->clauses, loc, clauseOps);
@@ -1765,19 +1718,17 @@ genTaskgroupOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::TaskloopOp
-genTaskloopOp(Fortran::lower::AbstractConverter &converter,
-              Fortran::lower::SymMap &symTable,
-              Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+genTaskloopOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+              semantics::SemanticsContext &semaCtx,
+              lower::pft::Evaluation &eval, mlir::Location loc,
               const ConstructQueue &queue, ConstructQueue::iterator item) {
   TODO(loc, "Taskloop construct");
 }
 
 static mlir::omp::TaskwaitOp
-genTaskwaitOp(Fortran::lower::AbstractConverter &converter,
-              Fortran::lower::SymMap &symTable,
-              Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+genTaskwaitOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+              semantics::SemanticsContext &semaCtx,
+              lower::pft::Evaluation &eval, mlir::Location loc,
               const ConstructQueue &queue, ConstructQueue::iterator item) {
   mlir::omp::TaskwaitClauseOps clauseOps;
   genTaskwaitClauses(converter, semaCtx, item->clauses, loc, clauseOps);
@@ -1786,22 +1737,19 @@ genTaskwaitOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::TaskyieldOp
-genTaskyieldOp(Fortran::lower::AbstractConverter &converter,
-               Fortran::lower::SymMap &symTable,
-               Fortran::semantics::SemanticsContext &semaCtx,
-               Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+genTaskyieldOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+               semantics::SemanticsContext &semaCtx,
+               lower::pft::Evaluation &eval, mlir::Location loc,
                const ConstructQueue &queue, ConstructQueue::iterator item) {
   return converter.getFirOpBuilder().create<mlir::omp::TaskyieldOp>(loc);
 }
 
 static mlir::omp::TeamsOp
-genTeamsOp(Fortran::lower::AbstractConverter &converter,
-           Fortran::lower::SymMap &symTable,
-           Fortran::semantics::SemanticsContext &semaCtx,
-           Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-           const ConstructQueue &queue, ConstructQueue::iterator item,
-           bool outerCombined = false) {
-  Fortran::lower::StatementContext stmtCtx;
+genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+           semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+           mlir::Location loc, const ConstructQueue &queue,
+           ConstructQueue::iterator item, bool outerCombined = false) {
+  lower::StatementContext stmtCtx;
   mlir::omp::TeamsClauseOps clauseOps;
   genTeamsClauses(converter, semaCtx, stmtCtx, item->clauses, loc, clauseOps);
 
@@ -1814,21 +1762,20 @@ genTeamsOp(Fortran::lower::AbstractConverter &converter,
 }
 
 static mlir::omp::WsloopOp
-genWsloopOp(Fortran::lower::AbstractConverter &converter,
-            Fortran::lower::SymMap &symTable,
-            Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-            const ConstructQueue &queue, ConstructQueue::iterator item) {
+genWsloopOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
+            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+            mlir::Location loc, const ConstructQueue &queue,
+            ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval);
   dsp.processStep1();
 
-  Fortran::lower::StatementContext stmtCtx;
+  lower::StatementContext stmtCtx;
   mlir::omp::LoopNestClauseOps loopClauseOps;
   mlir::omp::WsloopClauseOps wsClauseOps;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> iv;
+  llvm::SmallVector<const semantics::Symbol *> iv;
   llvm::SmallVector<mlir::Type> reductionTypes;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSyms;
+  llvm::SmallVector<const semantics::Symbol *> reductionSyms;
   genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
                      loopClauseOps, iv);
   genWsloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc, wsClauseOps,
@@ -1841,7 +1788,7 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
   mlir::Block *wsloopEntryBlock = firOpBuilder.createBlock(
       &wsloopOp.getRegion(), {}, reductionTypes, reductionLocs);
   firOpBuilder.setInsertionPoint(
-      Fortran::lower::genOpenMPTerminator(firOpBuilder, wsloopOp, loc));
+      lower::genOpenMPTerminator(firOpBuilder, wsloopOp, loc));
 
   // Create nested omp.loop_nest and fill body with loop contents.
   auto loopOp = firOpBuilder.create<mlir::omp::LoopNestOp>(loc, loopClauseOps);
@@ -1871,38 +1818,36 @@ genWsloopOp(Fortran::lower::AbstractConverter &converter,
 //===----------------------------------------------------------------------===//
 
 static void genCompositeDistributeParallelDo(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::lower::SymMap &symTable,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-    const ConstructQueue &queue, ConstructQueue::iterator item) {
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    mlir::Location loc, const ConstructQueue &queue,
+    ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE PARALLEL DO");
 }
 
 static void genCompositeDistributeParallelDoSimd(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::lower::SymMap &symTable,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
-    const ConstructQueue &queue, ConstructQueue::iterator item) {
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    mlir::Location loc, const ConstructQueue &queue,
+    ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE PARALLEL DO SIMD");
 }
 
-static void
-genCompositeDistributeSimd(Fortran::lower::AbstractConverter &converter,
-                           Fortran::lower::SymMap &symTable,
-                           Fortran::semantics::SemanticsContext &semaCtx,
-                           Fortran::lower::pft::Evaluation &eval,
-                           mlir::Location loc, const ConstructQueue &queue,
-                           ConstructQueue::iterator item) {
+static void genCompositeDistributeSimd(lower::AbstractConverter &converter,
+                                       lower::SymMap &symTable,
+                                       semantics::SemanticsContext &semaCtx,
+                                       lower::pft::Evaluation &eval,
+                                       mlir::Location loc,
+                                       const ConstructQueue &queue,
+                                       ConstructQueue::iterator item) {
   TODO(loc, "Composite DISTRIBUTE SIMD");
 }
 
-static void genCompositeDoSimd(Fortran::lower::AbstractConverter &converter,
-                               Fortran::lower::SymMap &symTable,
-                               Fortran::semantics::SemanticsContext &semaCtx,
-                               Fortran::lower::pft::Evaluation &eval,
-                               mlir::Location loc, const ConstructQueue &queue,
+static void genCompositeDoSimd(lower::AbstractConverter &converter,
+                               lower::SymMap &symTable,
+                               semantics::SemanticsContext &semaCtx,
+                               lower::pft::Evaluation &eval, mlir::Location loc,
+                               const ConstructQueue &queue,
                                ConstructQueue::iterator item) {
   ClauseProcessor cp(converter, semaCtx, item->clauses);
   cp.processTODO<clause::Aligned, clause::Allocate, clause::Linear,
@@ -1919,13 +1864,13 @@ static void genCompositeDoSimd(Fortran::lower::AbstractConverter &converter,
   genWsloopOp(converter, symTable, semaCtx, eval, loc, queue, item);
 }
 
-static void
-genCompositeTaskloopSimd(Fortran::lower::AbstractConverter &converter,
-                         Fortran::lower::SymMap &symTable,
-                         Fortran::semantics::SemanticsContext &semaCtx,
-                         Fortran::lower::pft::Evaluation &eval,
-                         mlir::Location loc, const ConstructQueue &queue,
-                         ConstructQueue::iterator item) {
+static void genCompositeTaskloopSimd(lower::AbstractConverter &converter,
+                                     lower::SymMap &symTable,
+                                     semantics::SemanticsContext &semaCtx,
+                                     lower::pft::Evaluation &eval,
+                                     mlir::Location loc,
+                                     const ConstructQueue &queue,
+                                     ConstructQueue::iterator item) {
   TODO(loc, "Composite TASKLOOP SIMD");
 }
 
@@ -1933,11 +1878,11 @@ genCompositeTaskloopSimd(Fortran::lower::AbstractConverter &converter,
 // Dispatch
 //===----------------------------------------------------------------------===//
 
-static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
-                           Fortran::lower::SymMap &symTable,
-                           Fortran::semantics::SemanticsContext &semaCtx,
-                           Fortran::lower::pft::Evaluation &eval,
-                           mlir::Location loc, const ConstructQueue &queue,
+static void genOMPDispatch(lower::AbstractConverter &converter,
+                           lower::SymMap &symTable,
+                           semantics::SemanticsContext &semaCtx,
+                           lower::pft::Evaluation &eval, mlir::Location loc,
+                           const ConstructQueue &queue,
                            ConstructQueue::iterator item) {
   assert(item != queue.end());
 
@@ -2058,38 +2003,30 @@ static void genOMPDispatch(Fortran::lower::AbstractConverter &converter,
 //===----------------------------------------------------------------------===//
 
 static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPDeclarativeAllocate &declarativeAllocate) {
+genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+       semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+       const parser::OpenMPDeclarativeAllocate &declarativeAllocate) {
   TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
 }
 
-static void genOMP(Fortran::lower::AbstractConverter &converter,
-                   Fortran::lower::SymMap &symTable,
-                   Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval,
-                   const Fortran::parser::OpenMPDeclareReductionConstruct
-                       &declareReductionConstruct) {
+static void genOMP(
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    const parser::OpenMPDeclareReductionConstruct &declareReductionConstruct) {
   TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
 }
 
-static void genOMP(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::lower::SymMap &symTable,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::OpenMPDeclareSimdConstruct &declareSimdConstruct) {
+static void
+genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+       semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+       const parser::OpenMPDeclareSimdConstruct &declareSimdConstruct) {
   TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
 }
 
-static void genOMP(Fortran::lower::AbstractConverter &converter,
-                   Fortran::lower::SymMap &symTable,
-                   Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval,
-                   const Fortran::parser::OpenMPDeclareTargetConstruct
-                       &declareTargetConstruct) {
+static void
+genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+       semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+       const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) {
   mlir::omp::DeclareTargetClauseOps clauseOps;
   llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
   mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
@@ -2097,8 +2034,8 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
                        clauseOps, symbolAndClause);
 
   for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
-    mlir::Operation *op = mod.lookupSymbol(converter.mangleName(
-        std::get<const Fortran::semantics::Symbol &>(symClause)));
+    mlir::Operation *op = mod.lookupSymbol(
+        converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
 
     // Some symbols are deferred until later in the module, these are handled
     // upon finalization of the module for OpenMP inside of Bridge, so we simply
@@ -2113,33 +2050,28 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
   }
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPRequiresConstruct &requiresConstruct) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPRequiresConstruct &requiresConstruct) {
   // Requires directives are gathered and processed in semantics and
   // then combined in the lowering bridge before triggering codegen
   // just once. Hence, there is no need to lower each individual
   // occurrence here.
 }
 
-static void genOMP(Fortran::lower::AbstractConverter &converter,
-                   Fortran::lower::SymMap &symTable,
-                   Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval,
-                   const Fortran::parser::OpenMPThreadprivate &threadprivate) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPThreadprivate &threadprivate) {
   // The directive is lowered when instantiating the variable to
   // support the case of threadprivate variable declared in module.
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPDeclarativeConstruct &ompDeclConstruct) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPDeclarativeConstruct &ompDeclConstruct) {
   std::visit(
       [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
       ompDeclConstruct.u);
@@ -2149,18 +2081,14 @@ genOMP(Fortran::lower::AbstractConverter &converter,
 // OpenMPStandaloneConstruct visitors
 //===----------------------------------------------------------------------===//
 
-static void genOMP(Fortran::lower::AbstractConverter &converter,
-                   Fortran::lower::SymMap &symTable,
-                   Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval,
-                   const Fortran::parser::OpenMPSimpleStandaloneConstruct
-                       &simpleStandaloneConstruct) {
-  const auto &directive =
-      std::get<Fortran::parser::OmpSimpleStandaloneDirective>(
-          simpleStandaloneConstruct.t);
+static void genOMP(
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    const parser::OpenMPSimpleStandaloneConstruct &simpleStandaloneConstruct) {
+  const auto &directive = std::get<parser::OmpSimpleStandaloneDirective>(
+      simpleStandaloneConstruct.t);
   List<Clause> clauses = makeClauses(
-      std::get<Fortran::parser::OmpClauseList>(simpleStandaloneConstruct.t),
-      semaCtx);
+      std::get<parser::OmpClauseList>(simpleStandaloneConstruct.t), semaCtx);
   mlir::Location currentLocation = converter.genLocation(directive.source);
 
   ConstructQueue queue{
@@ -2177,17 +2105,15 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
   }
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPFlushConstruct &flushConstruct) {
-  const auto &verbatim = std::get<Fortran::parser::Verbatim>(flushConstruct.t);
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPFlushConstruct &flushConstruct) {
+  const auto &verbatim = std::get<parser::Verbatim>(flushConstruct.t);
   const auto &objectList =
-      std::get<std::optional<Fortran::parser::OmpObjectList>>(flushConstruct.t);
+      std::get<std::optional<parser::OmpObjectList>>(flushConstruct.t);
   const auto &clauseList =
-      std::get<std::optional<std::list<Fortran::parser::OmpMemoryOrderClause>>>(
+      std::get<std::optional<std::list<parser::OmpMemoryOrderClause>>>(
           flushConstruct.t);
   ObjectList objects =
       objectList ? makeObjects(*objectList, semaCtx) : ObjectList{};
@@ -2204,30 +2130,25 @@ genOMP(Fortran::lower::AbstractConverter &converter,
              queue, queue.begin());
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPCancelConstruct &cancelConstruct) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPCancelConstruct &cancelConstruct) {
   TODO(converter.getCurrentLocation(), "OpenMPCancelConstruct");
 }
 
-static void genOMP(Fortran::lower::AbstractConverter &converter,
-                   Fortran::lower::SymMap &symTable,
-                   Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval,
-                   const Fortran::parser::OpenMPCancellationPointConstruct
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPCancellationPointConstruct
                        &cancellationPointConstruct) {
   TODO(converter.getCurrentLocation(), "OpenMPCancelConstruct");
 }
 
 static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPStandaloneConstruct &standaloneConstruct) {
+genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+       semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+       const parser::OpenMPStandaloneConstruct &standaloneConstruct) {
   std::visit(
       [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
       standaloneConstruct.u);
@@ -2237,79 +2158,68 @@ genOMP(Fortran::lower::AbstractConverter &converter,
 // OpenMPConstruct visitors
 //===----------------------------------------------------------------------===//
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPAllocatorsConstruct &allocsConstruct) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPAllocatorsConstruct &allocsConstruct) {
   TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPAtomicConstruct &atomicConstruct) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPAtomicConstruct &atomicConstruct) {
   std::visit(
-      Fortran::common::visitors{
-          [&](const Fortran::parser::OmpAtomicRead &atomicRead) {
+      common::visitors{
+          [&](const parser::OmpAtomicRead &atomicRead) {
             mlir::Location loc = converter.genLocation(atomicRead.source);
-            Fortran::lower::genOmpAccAtomicRead<
-                Fortran::parser::OmpAtomicRead,
-                Fortran::parser::OmpAtomicClauseList>(converter, atomicRead,
-                                                      loc);
+            lower::genOmpAccAtomicRead<parser::OmpAtomicRead,
+                                       parser::OmpAtomicClauseList>(
+                converter, atomicRead, loc);
           },
-          [&](const Fortran::parser::OmpAtomicWrite &atomicWrite) {
+          [&](const parser::OmpAtomicWrite &atomicWrite) {
             mlir::Location loc = converter.genLocation(atomicWrite.source);
-            Fortran::lower::genOmpAccAtomicWrite<
-                Fortran::parser::OmpAtomicWrite,
-                Fortran::parser::OmpAtomicClauseList>(converter, atomicWrite,
-                                                      loc);
+            lower::genOmpAccAtomicWrite<parser::OmpAtomicWrite,
+                                        parser::OmpAtomicClauseList>(
+                converter, atomicWrite, loc);
           },
-          [&](const Fortran::parser::OmpAtomic &atomicConstruct) {
+          [&](const parser::OmpAtomic &atomicConstruct) {
             mlir::Location loc = converter.genLocation(atomicConstruct.source);
-            Fortran::lower::genOmpAtomic<Fortran::parser::OmpAtomic,
-                                         Fortran::parser::OmpAtomicClauseList>(
+            lower::genOmpAtomic<parser::OmpAtomic, parser::OmpAtomicClauseList>(
                 converter, atomicConstruct, loc);
           },
-          [&](const Fortran::parser::OmpAtomicUpdate &atomicUpdate) {
+          [&](const parser::OmpAtomicUpdate &atomicUpdate) {
             mlir::Location loc = converter.genLocation(atomicUpdate.source);
-            Fortran::lower::genOmpAccAtomicUpdate<
-                Fortran::parser::OmpAtomicUpdate,
-                Fortran::parser::OmpAtomicClauseList>(converter, atomicUpdate,
-                                                      loc);
+            lower::genOmpAccAtomicUpdate<parser::OmpAtomicUpdate,
+                                         parser::OmpAtomicClauseList>(
+                converter, atomicUpdate, loc);
           },
-          [&](const Fortran::parser::OmpAtomicCapture &atomicCapture) {
+          [&](const parser::OmpAtomicCapture &atomicCapture) {
             mlir::Location loc = converter.genLocation(atomicCapture.source);
-            Fortran::lower::genOmpAccAtomicCapture<
-                Fortran::parser::OmpAtomicCapture,
-                Fortran::parser::OmpAtomicClauseList>(converter, atomicCapture,
-                                                      loc);
+            lower::genOmpAccAtomicCapture<parser::OmpAtomicCapture,
+                                          parser::OmpAtomicClauseList>(
+                converter, atomicCapture, loc);
           },
       },
       atomicConstruct.u);
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPBlockConstruct &blockConstruct) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPBlockConstruct &blockConstruct) {
   const auto &beginBlockDirective =
-      std::get<Fortran::parser::OmpBeginBlockDirective>(blockConstruct.t);
+      std::get<parser::OmpBeginBlockDirective>(blockConstruct.t);
   const auto &endBlockDirective =
-      std::get<Fortran::parser::OmpEndBlockDirective>(blockConstruct.t);
+      std::get<parser::OmpEndBlockDirective>(blockConstruct.t);
   mlir::Location currentLocation =
       converter.genLocation(beginBlockDirective.source);
   const auto origDirective =
-      std::get<Fortran::parser::OmpBlockDirective>(beginBlockDirective.t).v;
+      std::get<parser::OmpBlockDirective>(beginBlockDirective.t).v;
   List<Clause> clauses = makeClauses(
-      std::get<Fortran::parser::OmpClauseList>(beginBlockDirective.t), semaCtx);
+      std::get<parser::OmpClauseList>(beginBlockDirective.t), semaCtx);
   clauses.append(makeClauses(
-      std::get<Fortran::parser::OmpClauseList>(endBlockDirective.t), semaCtx));
+      std::get<parser::OmpClauseList>(endBlockDirective.t), semaCtx));
 
   assert(llvm::omp::blockConstructSet.test(origDirective) &&
          "Expected block construct");
@@ -2356,51 +2266,44 @@ genOMP(Fortran::lower::AbstractConverter &converter,
                  queue.begin());
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPCriticalConstruct &criticalConstruct) {
-  const auto &cd =
-      std::get<Fortran::parser::OmpCriticalDirective>(criticalConstruct.t);
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPCriticalConstruct &criticalConstruct) {
+  const auto &cd = std::get<parser::OmpCriticalDirective>(criticalConstruct.t);
   List<Clause> clauses =
-      makeClauses(std::get<Fortran::parser::OmpClauseList>(cd.t), semaCtx);
+      makeClauses(std::get<parser::OmpClauseList>(cd.t), semaCtx);
 
   ConstructQueue queue{buildConstructQueue(
       converter.getFirOpBuilder().getModule(), semaCtx, eval, cd.source,
       llvm::omp::Directive::OMPD_critical, clauses)};
 
-  const auto &name = std::get<std::optional<Fortran::parser::Name>>(cd.t);
+  const auto &name = std::get<std::optional<parser::Name>>(cd.t);
   mlir::Location currentLocation = converter.getCurrentLocation();
   genCriticalOp(converter, symTable, semaCtx, eval, currentLocation, queue,
                 queue.begin(), name);
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPExecutableAllocate &execAllocConstruct) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPExecutableAllocate &execAllocConstruct) {
   TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
 }
 
-static void genOMP(Fortran::lower::AbstractConverter &converter,
-                   Fortran::lower::SymMap &symTable,
-                   Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval,
-                   const Fortran::parser::OpenMPLoopConstruct &loopConstruct) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPLoopConstruct &loopConstruct) {
   const auto &beginLoopDirective =
-      std::get<Fortran::parser::OmpBeginLoopDirective>(loopConstruct.t);
+      std::get<parser::OmpBeginLoopDirective>(loopConstruct.t);
   List<Clause> clauses = makeClauses(
-      std::get<Fortran::parser::OmpClauseList>(beginLoopDirective.t), semaCtx);
+      std::get<parser::OmpClauseList>(beginLoopDirective.t), semaCtx);
   if (auto &endLoopDirective =
-          std::get<std::optional<Fortran::parser::OmpEndLoopDirective>>(
+          std::get<std::optional<parser::OmpEndLoopDirective>>(
               loopConstruct.t)) {
     clauses.append(makeClauses(
-        std::get<Fortran::parser::OmpClauseList>(endLoopDirective->t),
-        semaCtx));
+        std::get<parser::OmpClauseList>(endLoopDirective->t), semaCtx));
   }
 
   mlir::Location currentLocation =
@@ -2417,12 +2320,10 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
                  queue.begin());
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPSectionConstruct &sectionConstruct) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPSectionConstruct &sectionConstruct) {
   mlir::Location loc = converter.getCurrentLocation();
   ConstructQueue queue{buildConstructQueue(
       converter.getFirOpBuilder().getModule(), semaCtx, eval,
@@ -2430,22 +2331,18 @@ genOMP(Fortran::lower::AbstractConverter &converter,
   genOMPDispatch(converter, symTable, semaCtx, eval, loc, queue, queue.begin());
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPSectionsConstruct &sectionsConstruct) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPSectionsConstruct &sectionsConstruct) {
   const auto &beginSectionsDirective =
-      std::get<Fortran::parser::OmpBeginSectionsDirective>(sectionsConstruct.t);
+      std::get<parser::OmpBeginSectionsDirective>(sectionsConstruct.t);
   List<Clause> clauses = makeClauses(
-      std::get<Fortran::parser::OmpClauseList>(beginSectionsDirective.t),
-      semaCtx);
+      std::get<parser::OmpClauseList>(beginSectionsDirective.t), semaCtx);
   const auto &endSectionsDirective =
-      std::get<Fortran::parser::OmpEndSectionsDirective>(sectionsConstruct.t);
+      std::get<parser::OmpEndSectionsDirective>(sectionsConstruct.t);
   clauses.append(makeClauses(
-      std::get<Fortran::parser::OmpClauseList>(endSectionsDirective.t),
-      semaCtx));
+      std::get<parser::OmpClauseList>(endSectionsDirective.t), semaCtx));
   mlir::Location currentLocation = converter.getCurrentLocation();
 
   llvm::omp::Directive directive =
@@ -2459,11 +2356,10 @@ genOMP(Fortran::lower::AbstractConverter &converter,
                  queue.begin());
 }
 
-static void genOMP(Fortran::lower::AbstractConverter &converter,
-                   Fortran::lower::SymMap &symTable,
-                   Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval,
-                   const Fortran::parser::OpenMPConstruct &ompConstruct) {
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPConstruct &ompConstruct) {
   std::visit(
       [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
       ompConstruct.u);
@@ -2482,62 +2378,57 @@ mlir::Operation *Fortran::lower::genOpenMPTerminator(fir::FirOpBuilder &builder,
   return builder.create<mlir::omp::TerminatorOp>(loc);
 }
 
-void Fortran::lower::genOpenMPConstruct(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::lower::SymMap &symTable,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::OpenMPConstruct &omp) {
+void Fortran::lower::genOpenMPConstruct(lower::AbstractConverter &converter,
+                                        lower::SymMap &symTable,
+                                        semantics::SemanticsContext &semaCtx,
+                                        lower::pft::Evaluation &eval,
+                                        const parser::OpenMPConstruct &omp) {
   symTable.pushScope();
   genOMP(converter, symTable, semaCtx, eval, omp);
   symTable.popScope();
 }
 
 void Fortran::lower::genOpenMPDeclarativeConstruct(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::lower::SymMap &symTable,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::OpenMPDeclarativeConstruct &omp) {
+    lower::AbstractConverter &converter, lower::SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+    const parser::OpenMPDeclarativeConstruct &omp) {
   genOMP(converter, symTable, semaCtx, eval, omp);
   genNestedEvaluations(converter, eval);
 }
 
 void Fortran::lower::genOpenMPSymbolProperties(
-    Fortran::lower::AbstractConverter &converter,
-    const Fortran::lower::pft::Variable &var) {
+    lower::AbstractConverter &converter, const lower::pft::Variable &var) {
   assert(var.hasSymbol() && "Expecting Symbol");
-  const Fortran::semantics::Symbol &sym = var.getSymbol();
+  const semantics::Symbol &sym = var.getSymbol();
 
-  if (sym.test(Fortran::semantics::Symbol::Flag::OmpThreadprivate))
-    Fortran::lower::genThreadprivateOp(converter, var);
+  if (sym.test(semantics::Symbol::Flag::OmpThreadprivate))
+    lower::genThreadprivateOp(converter, var);
 
-  if (sym.test(Fortran::semantics::Symbol::Flag::OmpDeclareTarget))
-    Fortran::lower::genDeclareTargetIntGlobal(converter, var);
+  if (sym.test(semantics::Symbol::Flag::OmpDeclareTarget))
+    lower::genDeclareTargetIntGlobal(converter, var);
 }
 
-int64_t Fortran::lower::getCollapseValue(
-    const Fortran::parser::OmpClauseList &clauseList) {
-  for (const Fortran::parser::OmpClause &clause : clauseList.v) {
+int64_t
+Fortran::lower::getCollapseValue(const parser::OmpClauseList &clauseList) {
+  for (const parser::OmpClause &clause : clauseList.v) {
     if (const auto &collapseClause =
-            std::get_if<Fortran::parser::OmpClause::Collapse>(&clause.u)) {
-      const auto *expr = Fortran::semantics::GetExpr(collapseClause->v);
-      return Fortran::evaluate::ToInt64(*expr).value();
+            std::get_if<parser::OmpClause::Collapse>(&clause.u)) {
+      const auto *expr = semantics::GetExpr(collapseClause->v);
+      return evaluate::ToInt64(*expr).value();
     }
   }
   return 1;
 }
 
-void Fortran::lower::genThreadprivateOp(
-    Fortran::lower::AbstractConverter &converter,
-    const Fortran::lower::pft::Variable &var) {
+void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
+                                        const lower::pft::Variable &var) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::Location currentLocation = converter.getCurrentLocation();
 
-  const Fortran::semantics::Symbol &sym = var.getSymbol();
+  const semantics::Symbol &sym = var.getSymbol();
   mlir::Value symThreadprivateValue;
-  if (const Fortran::semantics::Symbol *common =
-          Fortran::semantics::FindCommonBlockContaining(sym.GetUltimate())) {
+  if (const semantics::Symbol *common =
+          semantics::FindCommonBlockContaining(sym.GetUltimate())) {
     mlir::Value commonValue = converter.getSymbolAddress(*common);
     if (mlir::isa<mlir::omp::ThreadprivateOp>(commonValue.getDefiningOp())) {
       // Generate ThreadprivateOp for a common block instead of its members and
@@ -2600,8 +2491,7 @@ void Fortran::lower::genThreadprivateOp(
 // that have the implicit SAVE attribute, to simplifiy LLVM-IR and MLIR
 // generation.
 void Fortran::lower::genDeclareTargetIntGlobal(
-    Fortran::lower::AbstractConverter &converter,
-    const Fortran::lower::pft::Variable &var) {
+    lower::AbstractConverter &converter, const lower::pft::Variable &var) {
   if (!var.isGlobal()) {
     // A non-global variable which can be in a declare target directive must
     // be a variable in the main program, and it has the implicit SAVE
@@ -2613,48 +2503,42 @@ void Fortran::lower::genDeclareTargetIntGlobal(
 }
 
 bool Fortran::lower::isOpenMPTargetConstruct(
-    const Fortran::parser::OpenMPConstruct &omp) {
+    const parser::OpenMPConstruct &omp) {
   llvm::omp::Directive dir = llvm::omp::Directive::OMPD_unknown;
-  if (const auto *block =
-          std::get_if<Fortran::parser::OpenMPBlockConstruct>(&omp.u)) {
-    const auto &begin =
-        std::get<Fortran::parser::OmpBeginBlockDirective>(block->t);
-    dir = std::get<Fortran::parser::OmpBlockDirective>(begin.t).v;
+  if (const auto *block = std::get_if<parser::OpenMPBlockConstruct>(&omp.u)) {
+    const auto &begin = std::get<parser::OmpBeginBlockDirective>(block->t);
+    dir = std::get<parser::OmpBlockDirective>(begin.t).v;
   } else if (const auto *loop =
-                 std::get_if<Fortran::parser::OpenMPLoopConstruct>(&omp.u)) {
-    const auto &begin =
-        std::get<Fortran::parser::OmpBeginLoopDirective>(loop->t);
-    dir = std::get<Fortran::parser::OmpLoopDirective>(begin.t).v;
+                 std::get_if<parser::OpenMPLoopConstruct>(&omp.u)) {
+    const auto &begin = std::get<parser::OmpBeginLoopDirective>(loop->t);
+    dir = std::get<parser::OmpLoopDirective>(begin.t).v;
   }
   return llvm::omp::allTargetSet.test(dir);
 }
 
 void Fortran::lower::gatherOpenMPDeferredDeclareTargets(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::OpenMPDeclarativeConstruct &ompDecl,
+    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+    lower::pft::Evaluation &eval,
+    const parser::OpenMPDeclarativeConstruct &ompDecl,
     llvm::SmallVectorImpl<OMPDeferredDeclareTargetInfo>
         &deferredDeclareTarget) {
-  std::visit(
-      Fortran::common::visitors{
-          [&](const Fortran::parser::OpenMPDeclareTargetConstruct &ompReq) {
-            collectDeferredDeclareTargets(converter, semaCtx, eval, ompReq,
-                                          deferredDeclareTarget);
-          },
-          [&](const auto &) {},
-      },
-      ompDecl.u);
+  std::visit(common::visitors{
+                 [&](const parser::OpenMPDeclareTargetConstruct &ompReq) {
+                   collectDeferredDeclareTargets(converter, semaCtx, eval,
+                                                 ompReq, deferredDeclareTarget);
+                 },
+                 [&](const auto &) {},
+             },
+             ompDecl.u);
 }
 
 bool Fortran::lower::isOpenMPDeviceDeclareTarget(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::OpenMPDeclarativeConstruct &ompDecl) {
+    lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
+    lower::pft::Evaluation &eval,
+    const parser::OpenMPDeclarativeConstruct &ompDecl) {
   return std::visit(
-      Fortran::common::visitors{
-          [&](const Fortran::parser::OpenMPDeclareTargetConstruct &ompReq) {
+      common::visitors{
+          [&](const parser::OpenMPDeclareTargetConstruct &ompReq) {
             mlir::omp::DeclareTargetDeviceType targetType =
                 getDeclareTargetFunctionDevice(converter, semaCtx, eval, ompReq)
                     .value_or(mlir::omp::DeclareTargetDeviceType::host);
@@ -2701,20 +2585,19 @@ bool Fortran::lower::markOpenMPDeferredDeclareTargetFunctions(
   return deviceCodeFound;
 }
 
-void Fortran::lower::genOpenMPRequires(
-    mlir::Operation *mod, const Fortran::semantics::Symbol *symbol) {
+void Fortran::lower::genOpenMPRequires(mlir::Operation *mod,
+                                       const semantics::Symbol *symbol) {
   using MlirRequires = mlir::omp::ClauseRequires;
-  using SemaRequires = Fortran::semantics::WithOmpDeclarative::RequiresFlag;
+  using SemaRequires = semantics::WithOmpDeclarative::RequiresFlag;
 
   if (auto offloadMod =
           llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(mod)) {
-    Fortran::semantics::WithOmpDeclarative::RequiresFlags semaFlags;
+    semantics::WithOmpDeclarative::RequiresFlags semaFlags;
     if (symbol) {
-      Fortran::common::visit(
+      common::visit(
           [&](const auto &details) {
-            if constexpr (std::is_base_of_v<
-                              Fortran::semantics::WithOmpDeclarative,
-                              std::decay_t<decltype(details)>>) {
+            if constexpr (std::is_base_of_v<semantics::WithOmpDeclarative,
+                                            std::decay_t<decltype(details)>>) {
               if (details.has_ompRequires())
                 semaFlags = *details.ompRequires();
             }
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index b3f08eb81c79..be6dcd6f5569 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -72,8 +72,8 @@ ReductionProcessor::ReductionIdentifier ReductionProcessor::getReductionType(
 
 bool ReductionProcessor::supportedIntrinsicProcReduction(
     const omp::clause::ProcedureDesignator &pd) {
-  Fortran::semantics::Symbol *sym = pd.v.id();
-  if (!sym->GetUltimate().attrs().test(Fortran::semantics::Attr::INTRINSIC))
+  semantics::Symbol *sym = pd.v.id();
+  if (!sym->GetUltimate().attrs().test(semantics::Attr::INTRINSIC))
     return false;
   auto redType = llvm::StringSwitch<bool>(getRealName(sym).ToString())
                      .Case("max", true)
@@ -180,7 +180,7 @@ ReductionProcessor::getReductionInitValue(mlir::Location loc, mlir::Type type,
   case ReductionIdentifier::NEQV:
     if (auto cplxTy = mlir::dyn_cast<fir::ComplexType>(type)) {
       mlir::Type realTy =
-          Fortran::lower::convertReal(builder.getContext(), cplxTy.getFKind());
+          lower::convertReal(builder.getContext(), cplxTy.getFKind());
       mlir::Value initRe = builder.createRealConstant(
           loc, realTy, getOperationIdentity(redId, loc));
       mlir::Value initIm = builder.createRealConstant(loc, realTy, 0);
@@ -680,13 +680,11 @@ bool ReductionProcessor::doReductionByRef(
 }
 
 void ReductionProcessor::addDeclareReduction(
-    mlir::Location currentLocation,
-    Fortran::lower::AbstractConverter &converter,
+    mlir::Location currentLocation, lower::AbstractConverter &converter,
     const omp::clause::Reduction &reduction,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-        *reductionSymbols) {
+    llvm::SmallVectorImpl<const semantics::Symbol *> *reductionSymbols) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
   if (std::get<std::optional<omp::clause::Reduction::ReductionModifier>>(
@@ -716,7 +714,7 @@ void ReductionProcessor::addDeclareReduction(
   // should happen byref
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   for (const Object &object : objectList) {
-    const Fortran::semantics::Symbol *symbol = object.id();
+    const semantics::Symbol *symbol = object.id();
     if (reductionSymbols)
       reductionSymbols->push_back(symbol);
     mlir::Value symVal = converter.getSymbolAddress(*symbol);
@@ -731,8 +729,7 @@ void ReductionProcessor::addDeclareReduction(
     // information needed to iterate over the array
     if (mlir::isa<fir::SequenceType>(eleType)) {
       // For Host associated symbols, use `SymbolBox` instead
-      Fortran::lower::SymbolBox symBox =
-          converter.lookupOneLevelUpSymbol(*symbol);
+      lower::SymbolBox symBox = converter.lookupOneLevelUpSymbol(*symbol);
       hlfir::Entity entity{symBox.getAddr()};
       entity = genVariableBox(currentLocation, builder, entity);
       mlir::Value box = entity.getBase();
@@ -828,12 +825,12 @@ void ReductionProcessor::addDeclareReduction(
   }
 }
 
-const Fortran::semantics::SourceName
-ReductionProcessor::getRealName(const Fortran::semantics::Symbol *symbol) {
+const semantics::SourceName
+ReductionProcessor::getRealName(const semantics::Symbol *symbol) {
   return symbol->GetUltimate().name();
 }
 
-const Fortran::semantics::SourceName
+const semantics::SourceName
 ReductionProcessor::getRealName(const omp::clause::ProcedureDesignator &pd) {
   return getRealName(pd.v.id());
 }
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.h b/flang/lib/Lower/OpenMP/ReductionProcessor.h
index 8b116a4c5204..2852abad0036 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.h
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.h
@@ -67,10 +67,10 @@ public:
   static bool
   supportedIntrinsicProcReduction(const omp::clause::ProcedureDesignator &pd);
 
-  static const Fortran::semantics::SourceName
-  getRealName(const Fortran::semantics::Symbol *symbol);
+  static const semantics::SourceName
+  getRealName(const semantics::Symbol *symbol);
 
-  static const Fortran::semantics::SourceName
+  static const semantics::SourceName
   getRealName(const omp::clause::ProcedureDesignator &pd);
 
   static bool
@@ -124,13 +124,12 @@ public:
   /// Creates a reduction declaration and associates it with an OpenMP block
   /// directive.
   static void addDeclareReduction(
-      mlir::Location currentLocation,
-      Fortran::lower::AbstractConverter &converter,
+      mlir::Location currentLocation, lower::AbstractConverter &converter,
       const omp::clause::Reduction &reduction,
       llvm::SmallVectorImpl<mlir::Value> &reductionVars,
       llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-          *reductionSymbols = nullptr);
+      llvm::SmallVectorImpl<const semantics::Symbol *> *reductionSymbols =
+          nullptr);
 };
 
 template <typename FloatOp, typename IntegerOp>
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index cb1d1a5a7f3d..4d665e6dd34c 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -52,22 +52,22 @@ int64_t getCollapseValue(const List<Clause> &clauses) {
 }
 
 void genObjectList(const ObjectList &objects,
-                   Fortran::lower::AbstractConverter &converter,
+                   lower::AbstractConverter &converter,
                    llvm::SmallVectorImpl<mlir::Value> &operands) {
   for (const Object &object : objects) {
-    const Fortran::semantics::Symbol *sym = object.id();
+    const semantics::Symbol *sym = object.id();
     assert(sym && "Expected Symbol");
     if (mlir::Value variable = converter.getSymbolAddress(*sym)) {
       operands.push_back(variable);
     } else if (const auto *details =
-                   sym->detailsIf<Fortran::semantics::HostAssocDetails>()) {
+                   sym->detailsIf<semantics::HostAssocDetails>()) {
       operands.push_back(converter.getSymbolAddress(details->symbol()));
       converter.copySymbolBinding(details->symbol(), *sym);
     }
   }
 }
 
-mlir::Type getLoopVarType(Fortran::lower::AbstractConverter &converter,
+mlir::Type getLoopVarType(lower::AbstractConverter &converter,
                           std::size_t loopVarTypeSize) {
   // OpenMP runtime requires 32-bit or 64-bit loop variables.
   loopVarTypeSize = loopVarTypeSize * 8;
@@ -85,24 +85,21 @@ mlir::Type getLoopVarType(Fortran::lower::AbstractConverter &converter,
   return converter.getFirOpBuilder().getIntegerType(loopVarTypeSize);
 }
 
-Fortran::semantics::Symbol *
-getIterationVariableSymbol(const Fortran::lower::pft::Evaluation &eval) {
-  return eval.visit(Fortran::common::visitors{
-      [&](const Fortran::parser::DoConstruct &doLoop) {
+semantics::Symbol *
+getIterationVariableSymbol(const lower::pft::Evaluation &eval) {
+  return eval.visit(common::visitors{
+      [&](const parser::DoConstruct &doLoop) {
         if (const auto &maybeCtrl = doLoop.GetLoopControl()) {
-          using LoopControl = Fortran::parser::LoopControl;
+          using LoopControl = parser::LoopControl;
           if (auto *bounds = std::get_if<LoopControl::Bounds>(&maybeCtrl->u)) {
-            static_assert(
-                std::is_same_v<decltype(bounds->name),
-                               Fortran::parser::Scalar<Fortran::parser::Name>>);
+            static_assert(std::is_same_v<decltype(bounds->name),
+                                         parser::Scalar<parser::Name>>);
             return bounds->name.thing.symbol;
           }
         }
-        return static_cast<Fortran::semantics::Symbol *>(nullptr);
-      },
-      [](auto &&) {
-        return static_cast<Fortran::semantics::Symbol *>(nullptr);
+        return static_cast<semantics::Symbol *>(nullptr);
       },
+      [](auto &&) { return static_cast<semantics::Symbol *>(nullptr); },
   });
 }
 
@@ -139,12 +136,11 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
 }
 
 static int
-getComponentPlacementInParent(const Fortran::semantics::Symbol *componentSym) {
-  const auto *derived =
-      componentSym->owner()
-          .derivedTypeSpec()
-          ->typeSymbol()
-          .detailsIf<Fortran::semantics::DerivedTypeDetails>();
+getComponentPlacementInParent(const semantics::Symbol *componentSym) {
+  const auto *derived = componentSym->owner()
+                            .derivedTypeSpec()
+                            ->typeSymbol()
+                            .detailsIf<semantics::DerivedTypeDetails>();
   assert(derived &&
          "expected derived type details when processing component symbol");
   for (auto [placement, name] : llvm::enumerate(derived->componentNames()))
@@ -155,7 +151,7 @@ getComponentPlacementInParent(const Fortran::semantics::Symbol *componentSym) {
 
 static std::optional<Object>
 getComponentObject(std::optional<Object> object,
-                   Fortran::semantics::SemanticsContext &semaCtx) {
+                   semantics::SemanticsContext &semaCtx) {
   if (!object)
     return std::nullopt;
 
@@ -176,7 +172,7 @@ getComponentObject(std::optional<Object> object,
 static void
 generateMemberPlacementIndices(const Object &object,
                                llvm::SmallVectorImpl<int> &indices,
-                               Fortran::semantics::SemanticsContext &semaCtx) {
+                               semantics::SemanticsContext &semaCtx) {
   auto compObj = getComponentObject(object, semaCtx);
   while (compObj) {
     indices.push_back(getComponentPlacementInParent(compObj->id()));
@@ -189,16 +185,14 @@ generateMemberPlacementIndices(const Object &object,
 
 void addChildIndexAndMapToParent(
     const omp::Object &object,
-    std::map<const Fortran::semantics::Symbol *,
+    std::map<const semantics::Symbol *,
              llvm::SmallVector<OmpMapMemberIndicesData>> &parentMemberIndices,
-    mlir::omp::MapInfoOp &mapOp,
-    Fortran::semantics::SemanticsContext &semaCtx) {
-  std::optional<Fortran::evaluate::DataRef> dataRef =
-      ExtractDataRef(object.designator);
+    mlir::omp::MapInfoOp &mapOp, semantics::SemanticsContext &semaCtx) {
+  std::optional<evaluate::DataRef> dataRef = ExtractDataRef(object.designator);
   assert(dataRef.has_value() &&
          "DataRef could not be extracted during mapping of derived type "
          "cannot proceed");
-  const Fortran::semantics::Symbol *parentSym = &dataRef->GetFirstSymbol();
+  const semantics::Symbol *parentSym = &dataRef->GetFirstSymbol();
   assert(parentSym && "Could not find parent symbol during lower of "
                       "a component member in OpenMP map clause");
   llvm::SmallVector<int> indices;
@@ -236,14 +230,14 @@ static mlir::DenseIntElementsAttr createDenseElementsAttrFromIndices(
   llvm::SmallVector<int64_t> shape;
   calculateShapeAndFillIndices(shape, memberPlacementData);
 
-  llvm::SmallVector<int> indicesFlattened = std::accumulate(
-      memberPlacementData.begin(), memberPlacementData.end(),
-      llvm::SmallVector<int>(),
-      [](llvm::SmallVector<int> &x, OmpMapMemberIndicesData y) {
-        x.insert(x.end(), y.memberPlacementIndices.begin(),
-                 y.memberPlacementIndices.end());
-        return x;
-      });
+  llvm::SmallVector<int> indicesFlattened =
+      std::accumulate(memberPlacementData.begin(), memberPlacementData.end(),
+                      llvm::SmallVector<int>(),
+                      [](llvm::SmallVector<int> &x, OmpMapMemberIndicesData y) {
+                        x.insert(x.end(), y.memberPlacementIndices.begin(),
+                                 y.memberPlacementIndices.end());
+                        return x;
+                      });
 
   return mlir::DenseIntElementsAttr::get(
       mlir::VectorType::get(shape,
@@ -252,11 +246,11 @@ static mlir::DenseIntElementsAttr createDenseElementsAttrFromIndices(
 }
 
 void insertChildMapInfoIntoParent(
-    Fortran::lower::AbstractConverter &converter,
-    std::map<const Fortran::semantics::Symbol *,
+    lower::AbstractConverter &converter,
+    std::map<const semantics::Symbol *,
              llvm::SmallVector<OmpMapMemberIndicesData>> &parentMemberIndices,
     llvm::SmallVectorImpl<mlir::Value> &mapOperands,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &mapSyms,
+    llvm::SmallVectorImpl<const semantics::Symbol *> &mapSyms,
     llvm::SmallVectorImpl<mlir::Type> *mapSymTypes,
     llvm::SmallVectorImpl<mlir::Location> *mapSymLocs) {
   for (auto indices : parentMemberIndices) {
@@ -323,30 +317,28 @@ void insertChildMapInfoIntoParent(
   }
 }
 
-Fortran::semantics::Symbol *
-getOmpObjectSymbol(const Fortran::parser::OmpObject &ompObject) {
-  Fortran::semantics::Symbol *sym = nullptr;
+semantics::Symbol *getOmpObjectSymbol(const parser::OmpObject &ompObject) {
+  semantics::Symbol *sym = nullptr;
   std::visit(
-      Fortran::common::visitors{
-          [&](const Fortran::parser::Designator &designator) {
+      common::visitors{
+          [&](const parser::Designator &designator) {
             if (auto *arrayEle =
-                    Fortran::parser::Unwrap<Fortran::parser::ArrayElement>(
-                        designator)) {
+                    parser::Unwrap<parser::ArrayElement>(designator)) {
               // Use getLastName to retrieve the arrays symbol, this will
               // provide the farthest right symbol (the last) in a designator,
               // i.e. providing something like the following:
               // "dtype1%dtype2%array[2:10]", will result in "array"
               sym = GetLastName(arrayEle->base).symbol;
-            } else if (auto *structComp = Fortran::parser::Unwrap<
-                           Fortran::parser::StructureComponent>(designator)) {
-              sym = structComp->component.symbol;
-            } else if (const Fortran::parser::Name *name =
-                           Fortran::semantics::getDesignatorNameIfDataRef(
+            } else if (auto *structComp =
+                           parser::Unwrap<parser::StructureComponent>(
                                designator)) {
+              sym = structComp->component.symbol;
+            } else if (const parser::Name *name =
+                           semantics::getDesignatorNameIfDataRef(designator)) {
               sym = name->symbol;
             }
           },
-          [&](const Fortran::parser::Name &name) { sym = name.symbol; }},
+          [&](const parser::Name &name) { sym = name.symbol; }},
       ompObject.u);
   return sym;
 }
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 345ce55620ee..d20f9187640f 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -42,8 +42,7 @@ class AbstractConverter;
 namespace omp {
 
 using DeclareTargetCapturePair =
-    std::pair<mlir::omp::DeclareTargetCaptureClause,
-              const Fortran::semantics::Symbol &>;
+    std::pair<mlir::omp::DeclareTargetCaptureClause, const semantics::Symbol &>;
 
 // A small helper structure for keeping track of a component members MapInfoOp
 // and index data when lowering OpenMP map clauses. Keeps track of the
@@ -69,24 +68,24 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
 
 void addChildIndexAndMapToParent(
     const omp::Object &object,
-    std::map<const Fortran::semantics::Symbol *,
+    std::map<const semantics::Symbol *,
              llvm::SmallVector<OmpMapMemberIndicesData>> &parentMemberIndices,
-    mlir::omp::MapInfoOp &mapOp, Fortran::semantics::SemanticsContext &semaCtx);
+    mlir::omp::MapInfoOp &mapOp, semantics::SemanticsContext &semaCtx);
 
 void insertChildMapInfoIntoParent(
-    Fortran::lower::AbstractConverter &converter,
-    std::map<const Fortran::semantics::Symbol *,
+    lower::AbstractConverter &converter,
+    std::map<const semantics::Symbol *,
              llvm::SmallVector<OmpMapMemberIndicesData>> &parentMemberIndices,
     llvm::SmallVectorImpl<mlir::Value> &mapOperands,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &mapSyms,
+    llvm::SmallVectorImpl<const semantics::Symbol *> &mapSyms,
     llvm::SmallVectorImpl<mlir::Type> *mapSymTypes,
     llvm::SmallVectorImpl<mlir::Location> *mapSymLocs);
 
-mlir::Type getLoopVarType(Fortran::lower::AbstractConverter &converter,
+mlir::Type getLoopVarType(lower::AbstractConverter &converter,
                           std::size_t loopVarTypeSize);
 
-Fortran::semantics::Symbol *
-getIterationVariableSymbol(const Fortran::lower::pft::Evaluation &eval);
+semantics::Symbol *
+getIterationVariableSymbol(const lower::pft::Evaluation &eval);
 
 void gatherFuncAndVarSyms(
     const ObjectList &objects, mlir::omp::DeclareTargetCaptureClause clause,
@@ -94,11 +93,10 @@ void gatherFuncAndVarSyms(
 
 int64_t getCollapseValue(const List<Clause> &clauses);
 
-Fortran::semantics::Symbol *
-getOmpObjectSymbol(const Fortran::parser::OmpObject &ompObject);
+semantics::Symbol *getOmpObjectSymbol(const parser::OmpObject &ompObject);
 
 void genObjectList(const ObjectList &objects,
-                   Fortran::lower::AbstractConverter &converter,
+                   lower::AbstractConverter &converter,
                    llvm::SmallVectorImpl<mlir::Value> &operands);
 
 } // namespace omp
-- 
GitLab


From 2bc9af96567eeda1effdf0cb7662511d896fb386 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Thu, 16 May 2024 16:49:56 +0400
Subject: [PATCH 1007/1206] [lldb][Windows] Disable the
 TestGdbRemoteLibrariesSvr4Support test for Windows host (#92341)

Windows does not allow quotes in file names. So it is impossible to
build `libsvr4lib_b".so` on Windows.
---
 .../libraries-svr4/TestGdbRemoteLibrariesSvr4Support.py         | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lldb/test/API/tools/lldb-server/libraries-svr4/TestGdbRemoteLibrariesSvr4Support.py b/lldb/test/API/tools/lldb-server/libraries-svr4/TestGdbRemoteLibrariesSvr4Support.py
index 846adade3440..02c9d318525f 100644
--- a/lldb/test/API/tools/lldb-server/libraries-svr4/TestGdbRemoteLibrariesSvr4Support.py
+++ b/lldb/test/API/tools/lldb-server/libraries-svr4/TestGdbRemoteLibrariesSvr4Support.py
@@ -4,6 +4,8 @@ from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 
 
+# Windows does not allow quotes in file names.
+@skipIf(hostoslist=["windows"])
 class TestGdbRemoteLibrariesSvr4Support(gdbremote_testcase.GdbRemoteTestCaseBase):
     FEATURE_NAME = "qXfer:libraries-svr4:read"
 
-- 
GitLab


From 588ce34ba626a369a416b883342af48c69e70925 Mon Sep 17 00:00:00 2001
From: zibi2 <62662650+zibi2@users.noreply.github.com>
Date: Thu, 16 May 2024 08:51:56 -0400
Subject: [PATCH 1008/1206] [libc++][z/OS] Fixup two linear_congruential_engine
 tests (#92261)

---
 .../std/numerics/rand/rand.eng/rand.eng.lcong/assign.pass.cpp  | 3 +++
 .../std/numerics/rand/rand.eng/rand.eng.lcong/copy.pass.cpp    | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/assign.pass.cpp b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/assign.pass.cpp
index 73829071bd95..63a1a8adf4e3 100644
--- a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/assign.pass.cpp
+++ b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/assign.pass.cpp
@@ -83,7 +83,10 @@ int main(int, char**)
     test<unsigned int>();
     test_ext<unsigned int>();
     test<unsigned long>();
+    // This isn't implemented on platforms without __int128
+#ifndef _LIBCPP_HAS_NO_INT128
     test_ext<unsigned long>();
+#endif
     test<unsigned long long>();
     // This isn't implemented on platforms without __int128
 #ifndef _LIBCPP_HAS_NO_INT128
diff --git a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/copy.pass.cpp b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/copy.pass.cpp
index 8387a1763714..c45f45d0f20a 100644
--- a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/copy.pass.cpp
+++ b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/copy.pass.cpp
@@ -82,7 +82,10 @@ int main(int, char**)
     test<unsigned int>();
     test_ext<unsigned int>();
     test<unsigned long>();
+    // This isn't implemented on platforms without __int128
+#ifndef _LIBCPP_HAS_NO_INT128
     test_ext<unsigned long>();
+#endif
     test<unsigned long long>();
     // This isn't implemented on platforms without __int128
 #ifndef _LIBCPP_HAS_NO_INT128
-- 
GitLab


From e948da1021b207dbcee9ed484ccb3409c2295561 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Thu, 16 May 2024 20:51:51 +0800
Subject: [PATCH 1009/1206] [Transforms] Fix -Wunused-variable in
 DemoteRegToStack.cpp (NFC)

llvm-project/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp:58:21:
error: unused variable 'BB' [-Werror,-Wunused-variable]
        BasicBlock *BB = SplitCriticalEdge(II, i);
                    ^
1 error generated.
---
 llvm/lib/Transforms/Utils/DemoteRegToStack.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp b/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
index 653ea8bbb72b..7995b7540803 100644
--- a/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
+++ b/llvm/lib/Transforms/Utils/DemoteRegToStack.cpp
@@ -55,7 +55,7 @@ AllocaInst *llvm::DemoteRegToStack(Instruction &I, bool VolatileLoads,
       auto *Succ = CBI->getSuccessor(i);
       if (!Succ->getSinglePredecessor()) {
         assert(isCriticalEdge(II, i) && "Expected a critical edge!");
-        BasicBlock *BB = SplitCriticalEdge(II, i);
+        [[maybe_unused]] BasicBlock *BB = SplitCriticalEdge(II, i);
         assert(BB && "Unable to split critical edge.");
       }
     }
-- 
GitLab


From 80fac30a09ce0fbd2047cc210ec0a42cfa95b79d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 16 May 2024 13:51:27 +0100
Subject: [PATCH 1010/1206] [X86] rot32.ll - remove old shld check prefixes

This was missed in 8dbd745b09c9f65fefc2ffac14e8f7f288766861
---
 llvm/test/CodeGen/X86/rot32.ll | 2 --
 1 file changed, 2 deletions(-)

diff --git a/llvm/test/CodeGen/X86/rot32.ll b/llvm/test/CodeGen/X86/rot32.ll
index d8edd31c3c06..114d558954f4 100644
--- a/llvm/test/CodeGen/X86/rot32.ll
+++ b/llvm/test/CodeGen/X86/rot32.ll
@@ -285,8 +285,6 @@ define i32 @xunp(ptr %p) nounwind readnone {
 ; BMI264-NEXT:    rorxl $7, (%rdi), %eax
 ; BMI264-NEXT:    retq
 entry:
-; shld-label: xunp:
-; shld: shldl $25
 	%x = load i32, ptr %p
 	%a = lshr i32 %x, 7
 	%b = shl i32 %x, 25
-- 
GitLab


From 44eded31e0bd5739391298497affe3412e4091aa Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Thu, 16 May 2024 16:55:06 +0400
Subject: [PATCH 1011/1206] [lldb] Move TestBase.runCmd() to the Base class
 (#92252)

runCmd() is called from Base.getCPUInfo() but implemented only in
TestBase(Base). Usually it works if TestBase is used. But call
getCPUInfo() from a class based on Base will cause something like
```
File "E:\projects\llvm-nino\lldb\llvm-project\lldb\packages\Python\lldbsuite\test\lldbtest.py", line 1256, in getCPUInfo
  self.runCmd('platform get-file "/proc/cpuinfo" ' + cpuinfo_path)
AttributeError: 'TestGdbRemoteExpeditedRegisters' object has no attribute 'runCmd'
```
BTW, TestBase.setUp() called runCmd() before applying
LLDB_MAX_LAUNCH_COUNT and LLDB_TIME_WAIT_NEXT_LAUNCH.

This patch fixes the test TestGdbRemoteExpeditedRegisters in case of
Windows host and Linux target.
---
 .../Python/lldbsuite/test/lldbtest.py         | 130 +++++++++---------
 1 file changed, 65 insertions(+), 65 deletions(-)

diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index 5fd686c143e9..1ad8ab6e6e46 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -531,6 +531,14 @@ class Base(unittest.TestCase):
     # Keep track of the old current working directory.
     oldcwd = None
 
+    # Maximum allowed attempts when launching the inferior process.
+    # Can be overridden by the LLDB_MAX_LAUNCH_COUNT environment variable.
+    maxLaunchCount = 1
+
+    # Time to wait before the next launching attempt in second(s).
+    # Can be overridden by the LLDB_TIME_WAIT_NEXT_LAUNCH environment variable.
+    timeWaitNextLaunch = 1.0
+
     @staticmethod
     def compute_mydir(test_file):
         """Subclasses should call this function to correctly calculate the
@@ -796,6 +804,12 @@ class Base(unittest.TestCase):
         # import traceback
         # traceback.print_stack()
 
+        if "LLDB_MAX_LAUNCH_COUNT" in os.environ:
+            self.maxLaunchCount = int(os.environ["LLDB_MAX_LAUNCH_COUNT"])
+
+        if "LLDB_TIME_WAIT_NEXT_LAUNCH" in os.environ:
+            self.timeWaitNextLaunch = float(os.environ["LLDB_TIME_WAIT_NEXT_LAUNCH"])
+
         if "LIBCXX_PATH" in os.environ:
             self.libcxxPath = os.environ["LIBCXX_PATH"]
         else:
@@ -937,6 +951,57 @@ class Base(unittest.TestCase):
         self.subprocesses.append(proc)
         return proc
 
+    def runCmd(self, cmd, msg=None, check=True, trace=False, inHistory=False):
+        """
+        Ask the command interpreter to handle the command and then check its
+        return status.
+        """
+        # Fail fast if 'cmd' is not meaningful.
+        if cmd is None:
+            raise Exception("Bad 'cmd' parameter encountered")
+
+        trace = True if traceAlways else trace
+
+        if cmd.startswith("target create "):
+            cmd = cmd.replace("target create ", "file ")
+
+        running = cmd.startswith("run") or cmd.startswith("process launch")
+
+        for i in range(self.maxLaunchCount if running else 1):
+            with recording(self, trace) as sbuf:
+                print("runCmd:", cmd, file=sbuf)
+                if not check:
+                    print("check of return status not required", file=sbuf)
+
+            self.ci.HandleCommand(cmd, self.res, inHistory)
+
+            with recording(self, trace) as sbuf:
+                if self.res.Succeeded():
+                    print("output:", self.res.GetOutput(), file=sbuf)
+                else:
+                    print("runCmd failed!", file=sbuf)
+                    print(self.res.GetError(), file=sbuf)
+
+            if self.res.Succeeded():
+                break
+            elif running:
+                # For process launch, wait some time before possible next try.
+                time.sleep(self.timeWaitNextLaunch)
+                with recording(self, trace) as sbuf:
+                    print("Command '" + cmd + "' failed!", file=sbuf)
+
+        if check:
+            output = ""
+            if self.res.GetOutput():
+                output += "\nCommand output:\n" + self.res.GetOutput()
+            if self.res.GetError():
+                output += "\nError output:\n" + self.res.GetError()
+            if msg:
+                msg += output
+            if cmd:
+                cmd += output
+            self.assertTrue(self.res.Succeeded(), msg if (msg) else CMD_MSG(cmd))
+
     def HideStdout(self):
         """Hide output to stdout from the user.
 
@@ -1764,14 +1829,6 @@ class TestBase(Base, metaclass=LLDBTestCaseFactory):
     # test multiple times with various debug info types.
     NO_DEBUG_INFO_TESTCASE = False
 
-    # Maximum allowed attempts when launching the inferior process.
-    # Can be overridden by the LLDB_MAX_LAUNCH_COUNT environment variable.
-    maxLaunchCount = 1
-
-    # Time to wait before the next launching attempt in second(s).
-    # Can be overridden by the LLDB_TIME_WAIT_NEXT_LAUNCH environment variable.
-    timeWaitNextLaunch = 1.0
-
     def generateSource(self, source):
         template = source + ".template"
         temp = os.path.join(self.getSourceDir(), template)
@@ -1812,12 +1869,6 @@ class TestBase(Base, metaclass=LLDBTestCaseFactory):
         for s in self.setUpCommands():
             self.runCmd(s)
 
-        if "LLDB_MAX_LAUNCH_COUNT" in os.environ:
-            self.maxLaunchCount = int(os.environ["LLDB_MAX_LAUNCH_COUNT"])
-
-        if "LLDB_TIME_WAIT_NEXT_LAUNCH" in os.environ:
-            self.timeWaitNextLaunch = float(os.environ["LLDB_TIME_WAIT_NEXT_LAUNCH"])
-
         # We want our debugger to be synchronous.
         self.dbg.SetAsync(False)
 
@@ -1983,57 +2034,6 @@ class TestBase(Base, metaclass=LLDBTestCaseFactory):
             if matched:
                 self.runCmd("thread select %s" % matched.group(1))
 
-    def runCmd(self, cmd, msg=None, check=True, trace=False, inHistory=False):
-        """
-        Ask the command interpreter to handle the command and then check its
-        return status.
-        """
-        # Fail fast if 'cmd' is not meaningful.
-        if cmd is None:
-            raise Exception("Bad 'cmd' parameter encountered")
-
-        trace = True if traceAlways else trace
-
-        if cmd.startswith("target create "):
-            cmd = cmd.replace("target create ", "file ")
-
-        running = cmd.startswith("run") or cmd.startswith("process launch")
-
-        for i in range(self.maxLaunchCount if running else 1):
-            with recording(self, trace) as sbuf:
-                print("runCmd:", cmd, file=sbuf)
-                if not check:
-                    print("check of return status not required", file=sbuf)
-
-            self.ci.HandleCommand(cmd, self.res, inHistory)
-
-            with recording(self, trace) as sbuf:
-                if self.res.Succeeded():
-                    print("output:", self.res.GetOutput(), file=sbuf)
-                else:
-                    print("runCmd failed!", file=sbuf)
-                    print(self.res.GetError(), file=sbuf)
-
-            if self.res.Succeeded():
-                break
-            elif running:
-                # For process launch, wait some time before possible next try.
-                time.sleep(self.timeWaitNextLaunch)
-                with recording(self, trace) as sbuf:
-                    print("Command '" + cmd + "' failed!", file=sbuf)
-
-        if check:
-            output = ""
-            if self.res.GetOutput():
-                output += "\nCommand output:\n" + self.res.GetOutput()
-            if self.res.GetError():
-                output += "\nError output:\n" + self.res.GetError()
-            if msg:
-                msg += output
-            if cmd:
-                cmd += output
-            self.assertTrue(self.res.Succeeded(), msg if (msg) else CMD_MSG(cmd))
-
     def match(
         self, str, patterns, msg=None, trace=False, error=False, matching=True, exe=True
     ):
-- 
GitLab


From f7392f40f3f6d5d4fc4ec11a982cf5379ab3c5d1 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi@arm.com>
Date: Thu, 16 May 2024 13:56:02 +0100
Subject: [PATCH 1012/1206] [AArch64] Add intrinsics for bfloat16
 min/max/minnm/maxnm (#90105)

According to specifications in
[ARM-software/acle/pull/309](https://github.com/ARM-software/acle/pull/309)
Add following intrinsics:

```
// svmax single,multi
svbfloat16x2_t svmax_single_bf16_x2(svbfloat16x2_t zdn, svbfloat16_t zm)
svbfloat16x4_t svmax_single_bf16_x4(svbfloat16x4_t zdn, svbfloat16_t zm)
svbfloat16x2_t svmax_bf16_x2(svbfloat16x2_t zdn, svbfloat16x2_t zm)
svbfloat16x4_t svmax_bf16_x4(svbfloat16x4_t zdn, svbfloat16x4_t zm)
```

```
// svmin single,multi
svbfloat16x2_t svmin_single_bf16_x2(svbfloat16x2_t zdn, svbfloat16_t zm)
svbfloat16x4_t svmin_single_bf16_x4(svbfloat16x4_t zdn, svbfloat16_t zm)
svbfloat16x2_t svmin_bf16_x2(svbfloat16x2_t zdn, svbfloat16x2_t zm)
svbfloat16x4_t svmin_bf16_x4(svbfloat16x4_t zdn, svbfloat16x4_t zm)
```

```
// svmaxnm single,multi
svbfloat16x2_t svmaxnm_single_bf16_x2(svbfloat16x2_t zdn, svbfloat16_t zm)
svbfloat16x4_t svmaxnm_single_bf16_x4(svbfloat16x4_t zdn, svbfloat16_t zm)
svbfloat16x2_t svmaxnm_bf16_x2(svbfloat16x2_t zdn, svbfloat16x2_t zm)
svbfloat16x4_t svmaxnm_bf16_x4(svbfloat16x4_t zdn, svbfloat16x4_t zm)
```

```
// svminnm single,multi
svbfloat16x2_t svminnm_single_bf16_x2(svbfloat16x2_t zdn, svbfloat16_t zm)
svbfloat16x4_t svminnm_single_bf16_x4(svbfloat16x4_t zdn, svbfloat16_t zm)
svbfloat16x2_t svminnm_bf16_x2(svbfloat16x2_t zdn, svbfloat16x2_t zm)
svbfloat16x4_t svminnm_bf16_x4(svbfloat16x4_t zdn, svbfloat16x4_t zm)
```
- Variations other than bfloat16 are already supported.
---
 clang/include/clang/Basic/arm_sve.td          |  10 +-
 .../aarch64-sme2-intrinsics/acle_sme2_max.c   | 150 +++++++++++++++++-
 .../aarch64-sme2-intrinsics/acle_sme2_maxnm.c | 150 +++++++++++++++++-
 .../aarch64-sme2-intrinsics/acle_sme2_min.c   | 150 +++++++++++++++++-
 .../aarch64-sme2-intrinsics/acle_sme2_minnm.c | 150 +++++++++++++++++-
 .../Target/AArch64/AArch64ISelDAGToDAG.cpp    |  74 +++++----
 .../CodeGen/AArch64/sme2-intrinsics-max.ll    | 126 ++++++++++++++-
 .../CodeGen/AArch64/sme2-intrinsics-min.ll    | 127 ++++++++++++++-
 8 files changed, 875 insertions(+), 62 deletions(-)

diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index 15340ebb62b3..a9ea71cd0777 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -2095,7 +2095,7 @@ def SVFCLAMP_BF   : SInst<"svclamp[_{d}]", "dddd", "b", MergeNone, "aarch64_sve_
 multiclass MinMaxIntr<string i, string zm, string mul, string t> {
   def SVS # NAME : SInst<"sv" # i # "[" # zm # "_{d}_" # mul # "]", t, "csil",     MergeNone, "aarch64_sve_s" # i # zm # "_" # mul, [IsStreaming], []>;
   def SVU # NAME : SInst<"sv" # i # "[" # zm # "_{d}_" # mul # "]", t, "UcUsUiUl", MergeNone, "aarch64_sve_u" # i # zm # "_" # mul, [IsStreaming], []>;
-  def SVF # NAME : SInst<"sv" # i # "[" # zm # "_{d}_" # mul # "]", t, "hfd",      MergeNone, "aarch64_sve_f" # i # zm # "_" # mul, [IsStreaming], []>;
+  def SVF # NAME : SInst<"sv" # i # "[" # zm # "_{d}_" # mul # "]", t, "bhfd",      MergeNone, "aarch64_sve_f" # i # zm # "_" # mul, [IsStreaming], []>;
 }
 
 let TargetGuard = "sme2" in {
@@ -2113,11 +2113,11 @@ let TargetGuard = "sme2" in {
 }
 
 multiclass SInstMinMaxByVector<string name> {
-  def NAME # _SINGLE_X2 : SInst<"sv" # name # "nm[_single_{d}_x2]", "22d", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_single_x2", [IsStreaming], []>;
-  def NAME # _SINGLE_X4 : SInst<"sv" # name # "nm[_single_{d}_x4]", "44d", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_single_x4", [IsStreaming], []>;
+  def NAME # _SINGLE_X2 : SInst<"sv" # name # "nm[_single_{d}_x2]", "22d", "bhfd", MergeNone, "aarch64_sve_f" # name # "nm_single_x2", [IsStreaming], []>;
+  def NAME # _SINGLE_X4 : SInst<"sv" # name # "nm[_single_{d}_x4]", "44d", "bhfd", MergeNone, "aarch64_sve_f" # name # "nm_single_x4", [IsStreaming], []>;
 
-  def NAME # _X2 : SInst<"sv" # name # "nm[_{d}_x2]", "222", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_x2", [IsStreaming], []>;
-  def NAME # _X4 : SInst<"sv" # name # "nm[_{d}_x4]", "444", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_x4", [IsStreaming], []>;
+  def NAME # _X2 : SInst<"sv" # name # "nm[_{d}_x2]", "222", "bhfd", MergeNone, "aarch64_sve_f" # name # "nm_x2", [IsStreaming], []>;
+  def NAME # _X4 : SInst<"sv" # name # "nm[_{d}_x4]", "444", "bhfd", MergeNone, "aarch64_sve_f" # name # "nm_x4", [IsStreaming], []>;
 }
 
 let TargetGuard = "sme2" in {
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_max.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_max.c
index 8e696f1d1fa4..2b9343a2c6b6 100644
--- a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_max.c
+++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_max.c
@@ -1,9 +1,9 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 // REQUIRES: aarch64-registered-target
 #include <arm_sme.h>
 
@@ -224,6 +224,32 @@ svuint64x2_t test_svmax_single_u64_x2(svuint64x2_t zdn, svuint64_t zm) __arm_str
   return SVE_ACLE_FUNC(svmax,_single_u64_x2)(zdn, zm);
 }
 
+// CHECK-LABEL: @test_svmax_single_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmax.single.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 0
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP3]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 1
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP6]]
+//
+// CPP-CHECK-LABEL: @_Z25test_svmax_single_bf16_x214svbfloat16x2_tu14__SVBfloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmax.single.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 0
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP3]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 1
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP6]]
+//
+svbfloat16x2_t test_svmax_single_bf16_x2(svbfloat16x2_t zdn, svbfloat16_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmax,_single_bf16_x2)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmax_single_f16_x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZDN:%.*]], i64 0)
@@ -608,6 +634,44 @@ svuint64x4_t test_svmax_single_u64_x4(svuint64x4_t zdn, svuint64_t zm) __arm_str
   return SVE_ACLE_FUNC(svmax,_single_u64_x4)(zdn, zm);
 }
 
+// CHECK-LABEL: @test_svmax_single_bf16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmax.single.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 2
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[TMP9]], i64 16)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 3
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP12]]
+//
+// CPP-CHECK-LABEL: @_Z25test_svmax_single_bf16_x414svbfloat16x4_tu14__SVBfloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmax.single.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 2
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[TMP9]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 3
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP12]]
+//
+svbfloat16x4_t test_svmax_single_bf16_x4(svbfloat16x4_t zdn, svbfloat16_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmax,_single_bf16_x4)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmax_single_f16_x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZDN:%.*]], i64 0)
@@ -964,6 +1028,36 @@ svuint64x2_t test_svmax_u64_x2(svuint64x2_t zdn, svuint64x2_t zm) __arm_streamin
   return SVE_ACLE_FUNC(svmax,_u64_x2)(zdn, zm);
 }
 
+// CHECK-LABEL: @test_svmax_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmax.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svmax_bf16_x214svbfloat16x2_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmax.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP8]]
+//
+svbfloat16x2_t test_svmax_bf16_x2(svbfloat16x2_t zdn, svbfloat16x2_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmax,_bf16_x2)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmax_f16_x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZDN:%.*]], i64 0)
@@ -1424,6 +1518,52 @@ svuint64x4_t test_svmax_u64_x4(svuint64x4_t zdn, svuint64x4_t zm) __arm_streamin
   return SVE_ACLE_FUNC(svmax,_u64_x4)(zdn, zm);
 }
 
+// CHECK-LABEL: @test_svmax_bf16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmax.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 0
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP9]], i64 0)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 1
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 8)
+// CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 2
+// CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP12]], <vscale x 8 x bfloat> [[TMP13]], i64 16)
+// CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 3
+// CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP14]], <vscale x 8 x bfloat> [[TMP15]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP16]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svmax_bf16_x414svbfloat16x4_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmax.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 0
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP9]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 1
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 2
+// CPP-CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP12]], <vscale x 8 x bfloat> [[TMP13]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 3
+// CPP-CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP14]], <vscale x 8 x bfloat> [[TMP15]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP16]]
+//
+svbfloat16x4_t test_svmax_bf16_x4(svbfloat16x4_t zdn, svbfloat16x4_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmax,_bf16_x4)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmax_f16_x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZDN:%.*]], i64 0)
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_maxnm.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_maxnm.c
index a956235034e1..ac2e20cf98b8 100644
--- a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_maxnm.c
+++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_maxnm.c
@@ -1,11 +1,11 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: aarch64-registered-target
 
-// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sme.h>
 
 #ifdef SVE_OVERLOADED_FORMS
@@ -17,6 +17,32 @@
 
 // Single, x2
 
+// CHECK-LABEL: @test_svmaxnm_single_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmaxnm.single.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 0
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP3]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 1
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP6]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svmaxnm_single_bf16_x214svbfloat16x2_tu14__SVBfloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmaxnm.single.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 0
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP3]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 1
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP6]]
+//
+svbfloat16x2_t test_svmaxnm_single_bf16_x2(svbfloat16x2_t zdn, svbfloat16_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmaxnm,_single_bf16_x2)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmaxnm_single_f16_x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZDN:%.*]], i64 0)
@@ -97,6 +123,44 @@ svfloat64x2_t test_svmaxnm_single_f64_x2(svfloat64x2_t zdn, svfloat64_t zm) __ar
 
 // Single, x4
 
+// CHECK-LABEL: @test_svmaxnm_single_bf16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmaxnm.single.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 2
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[TMP9]], i64 16)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 3
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP12]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svmaxnm_single_bf16_x414svbfloat16x4_tu14__SVBfloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmaxnm.single.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 2
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[TMP9]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 3
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP12]]
+//
+svbfloat16x4_t test_svmaxnm_single_bf16_x4(svbfloat16x4_t zdn, svbfloat16_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmaxnm,_single_bf16_x4)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmaxnm_single_f16_x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZDN:%.*]], i64 0)
@@ -213,6 +277,36 @@ svfloat64x4_t test_svmaxnm_single_f64_x4(svfloat64x4_t zdn, svfloat64_t zm) __ar
 
 // Multi, x2
 
+// CHECK-LABEL: @test_svmaxnm_multi_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmaxnm.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z26test_svmaxnm_multi_bf16_x214svbfloat16x2_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmaxnm.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP8]]
+//
+svbfloat16x2_t test_svmaxnm_multi_bf16_x2(svbfloat16x2_t zdn, svbfloat16x2_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmaxnm,_bf16_x2)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmaxnm_multi_f16_x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZDN:%.*]], i64 0)
@@ -305,6 +399,52 @@ svfloat64x2_t test_svmaxnm_multi_f64_x2(svfloat64x2_t zdn, svfloat64x2_t zm) __a
 
 // Multi, x4
 
+// CHECK-LABEL: @test_svmaxnm_multi_bf16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmaxnm.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 0
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP9]], i64 0)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 1
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 8)
+// CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 2
+// CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP12]], <vscale x 8 x bfloat> [[TMP13]], i64 16)
+// CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 3
+// CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP14]], <vscale x 8 x bfloat> [[TMP15]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP16]]
+//
+// CPP-CHECK-LABEL: @_Z26test_svmaxnm_multi_bf16_x414svbfloat16x4_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmaxnm.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 0
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP9]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 1
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 2
+// CPP-CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP12]], <vscale x 8 x bfloat> [[TMP13]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 3
+// CPP-CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP14]], <vscale x 8 x bfloat> [[TMP15]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP16]]
+//
+svbfloat16x4_t test_svmaxnm_multi_bf16_x4(svbfloat16x4_t zdn, svbfloat16x4_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmaxnm,_bf16_x4)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmaxnm_multi_f16_x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZDN:%.*]], i64 0)
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_min.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_min.c
index 860e6237bb9c..dda0837cf1f3 100644
--- a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_min.c
+++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_min.c
@@ -1,9 +1,9 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 // REQUIRES: aarch64-registered-target
 #include <arm_sme.h>
 
@@ -224,6 +224,32 @@ svuint64x2_t test_svmin_single_u64_x2(svuint64x2_t zdn, svuint64_t zm) __arm_str
   return SVE_ACLE_FUNC(svmin,_single_u64_x2)(zdn, zm);
 }
 
+// CHECK-LABEL: @test_svmin_single_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmin.single.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 0
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP3]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 1
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP6]]
+//
+// CPP-CHECK-LABEL: @_Z25test_svmin_single_bf16_x214svbfloat16x2_tu14__SVBfloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmin.single.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 0
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP3]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 1
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP6]]
+//
+svbfloat16x2_t test_svmin_single_bf16_x2(svbfloat16x2_t zdn, svbfloat16_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmin,_single_bf16_x2)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmin_single_f16_x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZDN:%.*]], i64 0)
@@ -608,6 +634,44 @@ svuint64x4_t test_svmin_single_u64_x4(svuint64x4_t zdn, svuint64_t zm) __arm_str
   return SVE_ACLE_FUNC(svmin,_single_u64_x4)(zdn, zm);
 }
 
+// CHECK-LABEL: @test_svmin_single_bf16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmin.single.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 2
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[TMP9]], i64 16)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 3
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP12]]
+//
+// CPP-CHECK-LABEL: @_Z25test_svmin_single_bf16_x414svbfloat16x4_tu14__SVBfloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmin.single.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 2
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[TMP9]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 3
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP12]]
+//
+svbfloat16x4_t test_svmin_single_bf16_x4(svbfloat16x4_t zdn, svbfloat16_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmin,_single_bf16_x4)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmin_single_f16_x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZDN:%.*]], i64 0)
@@ -964,6 +1028,36 @@ svuint64x2_t test_svmin_u64_x2(svuint64x2_t zdn, svuint64x2_t zm) __arm_streamin
   return SVE_ACLE_FUNC(svmin,_u64_x2)(zdn, zm);
 }
 
+// CHECK-LABEL: @test_svmin_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmin.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svmin_bf16_x214svbfloat16x2_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmin.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP8]]
+//
+svbfloat16x2_t test_svmin_bf16_x2(svbfloat16x2_t zdn, svbfloat16x2_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmin,_bf16_x2)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmin_f16_x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZDN:%.*]], i64 0)
@@ -1424,6 +1518,52 @@ svuint64x4_t test_svmin_u64_x4(svuint64x4_t zdn, svuint64x4_t zm) __arm_streamin
   return SVE_ACLE_FUNC(svmin,_u64_x4)(zdn, zm);
 }
 
+// CHECK-LABEL: @test_svmin_bf16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmin.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 0
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP9]], i64 0)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 1
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 8)
+// CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 2
+// CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP12]], <vscale x 8 x bfloat> [[TMP13]], i64 16)
+// CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 3
+// CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP14]], <vscale x 8 x bfloat> [[TMP15]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP16]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svmin_bf16_x414svbfloat16x4_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmin.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 0
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP9]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 1
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 2
+// CPP-CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP12]], <vscale x 8 x bfloat> [[TMP13]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 3
+// CPP-CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP14]], <vscale x 8 x bfloat> [[TMP15]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP16]]
+//
+svbfloat16x4_t test_svmin_bf16_x4(svbfloat16x4_t zdn, svbfloat16x4_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svmin,_bf16_x4)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svmin_f16_x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZDN:%.*]], i64 0)
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_minnm.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_minnm.c
index e34b9d98eff3..23378ffdf85d 100644
--- a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_minnm.c
+++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_minnm.c
@@ -1,11 +1,11 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: aarch64-registered-target
 
-// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sme.h>
 
 #ifdef SVE_OVERLOADED_FORMS
@@ -17,6 +17,32 @@
 
 // Single, x2
 
+// CHECK-LABEL: @test_svminnm_single_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fminnm.single.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 0
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP3]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 1
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP6]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svminnm_single_bf16_x214svbfloat16x2_tu14__SVBfloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fminnm.single.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 0
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP3]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP2]], 1
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP6]]
+//
+svbfloat16x2_t test_svminnm_single_bf16_x2(svbfloat16x2_t zdn, svbfloat16_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svminnm,_single_bf16_x2)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svminnm_single_f16_x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZDN:%.*]], i64 0)
@@ -97,6 +123,44 @@ svfloat64x2_t test_svminnm_single_f64_x2(svfloat64x2_t zdn, svfloat64_t zm) __ar
 
 // Single, x4
 
+// CHECK-LABEL: @test_svminnm_single_bf16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fminnm.single.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 2
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[TMP9]], i64 16)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 3
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP12]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svminnm_single_bf16_x414svbfloat16x4_tu14__SVBfloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fminnm.single.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 2
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[TMP9]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 3
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP12]]
+//
+svbfloat16x4_t test_svminnm_single_bf16_x4(svbfloat16x4_t zdn, svbfloat16_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svminnm,_single_bf16_x4)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svminnm_single_f16_x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZDN:%.*]], i64 0)
@@ -213,6 +277,36 @@ svfloat64x4_t test_svminnm_single_f64_x4(svfloat64x4_t zdn, svfloat64_t zm) __ar
 
 // Multi, x2
 
+// CHECK-LABEL: @test_svminnm_multi_bf16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fminnm.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z26test_svminnm_multi_bf16_x214svbfloat16x2_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZM]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fminnm.x2.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> poison, <vscale x 8 x bfloat> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP8]]
+//
+svbfloat16x2_t test_svminnm_multi_bf16_x2(svbfloat16x2_t zdn, svbfloat16x2_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svminnm,_bf16_x2)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svminnm_multi_f16_x2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZDN:%.*]], i64 0)
@@ -305,6 +399,52 @@ svfloat64x2_t test_svminnm_multi_f64_x2(svfloat64x2_t zdn, svfloat64x2_t zm) __a
 
 // Multi, x4
 
+// CHECK-LABEL: @test_svminnm_multi_bf16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fminnm.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 0
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP9]], i64 0)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 1
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 8)
+// CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 2
+// CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP12]], <vscale x 8 x bfloat> [[TMP13]], i64 16)
+// CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 3
+// CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP14]], <vscale x 8 x bfloat> [[TMP15]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP16]]
+//
+// CPP-CHECK-LABEL: @_Z26test_svminnm_multi_bf16_x414svbfloat16x4_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZDN]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZM]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fminnm.x4.nxv8bf16(<vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]])
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 0
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> poison, <vscale x 8 x bfloat> [[TMP9]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 1
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP10]], <vscale x 8 x bfloat> [[TMP11]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 2
+// CPP-CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP12]], <vscale x 8 x bfloat> [[TMP13]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[TMP8]], 3
+// CPP-CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP14]], <vscale x 8 x bfloat> [[TMP15]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP16]]
+//
+svbfloat16x4_t test_svminnm_multi_bf16_x4(svbfloat16x4_t zdn, svbfloat16x4_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svminnm,_bf16_x4)(zdn, zm);
+}
+
 // CHECK-LABEL: @test_svminnm_multi_f16_x4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZDN:%.*]], i64 0)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 80272213dd38..25f2e4d7c4de 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -1675,6 +1675,7 @@ static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef<unsigned> Opcodes) {
     return 0;
 
   EVT EltVT = VT.getVectorElementType();
+  unsigned Key = VT.getVectorMinNumElements();
   switch (Kind) {
   case SelectTypeKind::AnyType:
     break;
@@ -1688,14 +1689,17 @@ static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef<unsigned> Opcodes) {
       return 0;
     break;
   case SelectTypeKind::FP:
-    if (EltVT != MVT::f16 && EltVT != MVT::f32 && EltVT != MVT::f64)
+    if (EltVT == MVT::bf16)
+      Key = 16;
+    else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
+             EltVT != MVT::f64)
       return 0;
     break;
   }
 
   unsigned Offset;
-  switch (VT.getVectorMinNumElements()) {
-  case 16: // 8-bit
+  switch (Key) {
+  case 16: // 8-bit or bf16
     Offset = 0;
     break;
   case 8: // 16-bit
@@ -5482,8 +5486,8 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     case Intrinsic::aarch64_sve_fmax_single_x2:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_S,
-               AArch64::FMAX_VG2_2ZZ_D}))
+              {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
+               AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
         SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
       return;
     case Intrinsic::aarch64_sve_smax_single_x4:
@@ -5503,8 +5507,8 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     case Intrinsic::aarch64_sve_fmax_single_x4:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_S,
-               AArch64::FMAX_VG4_4ZZ_D}))
+              {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
+               AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
         SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
       return;
     case Intrinsic::aarch64_sve_smin_single_x2:
@@ -5524,8 +5528,8 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     case Intrinsic::aarch64_sve_fmin_single_x2:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_S,
-               AArch64::FMIN_VG2_2ZZ_D}))
+              {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
+               AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
         SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
       return;
     case Intrinsic::aarch64_sve_smin_single_x4:
@@ -5545,8 +5549,8 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     case Intrinsic::aarch64_sve_fmin_single_x4:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_S,
-               AArch64::FMIN_VG4_4ZZ_D}))
+              {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
+               AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
         SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
       return;
     case Intrinsic::aarch64_sve_smax_x2:
@@ -5566,8 +5570,8 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     case Intrinsic::aarch64_sve_fmax_x2:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_S,
-               AArch64::FMAX_VG2_2Z2Z_D}))
+              {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
+               AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
         SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
       return;
     case Intrinsic::aarch64_sve_smax_x4:
@@ -5587,8 +5591,8 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     case Intrinsic::aarch64_sve_fmax_x4:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMAX_VG4_4Z4Z_H, AArch64::FMAX_VG4_4Z4Z_S,
-               AArch64::FMAX_VG4_4Z4Z_D}))
+              {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
+               AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
         SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
       return;
     case Intrinsic::aarch64_sve_smin_x2:
@@ -5608,8 +5612,8 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     case Intrinsic::aarch64_sve_fmin_x2:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_S,
-               AArch64::FMIN_VG2_2Z2Z_D}))
+              {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
+               AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
         SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
       return;
     case Intrinsic::aarch64_sve_smin_x4:
@@ -5629,64 +5633,64 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
     case Intrinsic::aarch64_sve_fmin_x4:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMIN_VG4_4Z4Z_H, AArch64::FMIN_VG4_4Z4Z_S,
-               AArch64::FMIN_VG4_4Z4Z_D}))
+              {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
+               AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
         SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
       return;
     case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_S,
-               AArch64::FMAXNM_VG2_2ZZ_D}))
+              {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
+               AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
         SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
       return;
     case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_S,
-               AArch64::FMAXNM_VG4_4ZZ_D}))
+              {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
+               AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
         SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
       return;
     case Intrinsic::aarch64_sve_fminnm_single_x2:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_S,
-               AArch64::FMINNM_VG2_2ZZ_D}))
+              {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
+               AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
         SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
       return;
     case Intrinsic::aarch64_sve_fminnm_single_x4:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_S,
-               AArch64::FMINNM_VG4_4ZZ_D}))
+              {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
+               AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
         SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
       return;
     case Intrinsic::aarch64_sve_fmaxnm_x2:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_S,
-               AArch64::FMAXNM_VG2_2Z2Z_D}))
+              {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
+               AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
         SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
       return;
     case Intrinsic::aarch64_sve_fmaxnm_x4:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMAXNM_VG4_4Z4Z_H, AArch64::FMAXNM_VG4_4Z4Z_S,
-               AArch64::FMAXNM_VG4_4Z4Z_D}))
+              {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
+               AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
         SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
       return;
     case Intrinsic::aarch64_sve_fminnm_x2:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_S,
-               AArch64::FMINNM_VG2_2Z2Z_D}))
+              {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
+               AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
         SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
       return;
     case Intrinsic::aarch64_sve_fminnm_x4:
       if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
               Node->getValueType(0),
-              {0, AArch64::FMINNM_VG4_4Z4Z_H, AArch64::FMINNM_VG4_4Z4Z_S,
-               AArch64::FMINNM_VG4_4Z4Z_D}))
+              {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
+               AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
         SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
       return;
     case Intrinsic::aarch64_sve_fcvtzs_x2:
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll
index 92e8877927ea..79db677853cb 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+b16b16 -verify-machineinstrs < %s | FileCheck %s
 
 ; SMAX (Single, x2)
 
@@ -109,6 +109,19 @@ define { <vscale x 2 x i64>, <vscale x 2 x i64> } @multi_vec_max_single_x2_u64(<
   ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res
 }
 
+; BFMAX (Single, x2)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @multi_vec_max_single_x2_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: multi_vec_max_single_x2_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    bfmax { z0.h, z1.h }, { z0.h, z1.h }, z2.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmax.single.x2.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
+
 ; FMAX (Single, x2)
 
 define { <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_max_single_x2_f16(<vscale x 8 x half> %unused, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zm) {
@@ -298,6 +311,21 @@ define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2
   ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res
 }
 
+; BFMAX (Single, x4)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @multi_vec_max_single_x4_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: multi_vec_max_single_x4_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    bfmax { z0.h - z3.h }, { z0.h - z3.h }, z4.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @llvm.aarch64.sve.fmax.single.x4.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } %res
+}
+
 ; FMAX (SINGLE, x4)
 
 define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_max_single_x4_f16(<vscale x 8 x half> %unused, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x 8 x half> %zdn4, <vscale x 8 x half> %zm) {
@@ -478,6 +506,21 @@ define { <vscale x 2 x i64>, <vscale x 2 x i64> } @multi_vec_max_multi_x2_u64(<v
   ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res
 }
 
+; BFMAX (Multi, x2)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @multi_vec_max_x2_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2) {
+; CHECK-LABEL: multi_vec_max_x2_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT:    bfmax { z0.h, z1.h }, { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmax.x2.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
+
 ; FMAX (Multi, x2)
 
 define { <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_max_multi_x2_f16(<vscale x 8 x half> %unused, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2) {
@@ -729,6 +772,25 @@ define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2
   ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res
 }
 
+; BFMAX (Multi, x4)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @multi_vec_max_x4_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3, <vscale x 8 x bfloat> %zm4) {
+; CHECK-LABEL: multi_vec_max_x4_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    bfmax { z0.h - z3.h }, { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @llvm.aarch64.sve.fmax.x4.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3, <vscale x 8 x bfloat> %zm4)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } %res
+}
+
 ; FMAX (Multi, x4)
 
 define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_max_multi_x4_f16(<vscale x 8 x half> %unused, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x 8 x half> %zdn4,
@@ -806,6 +868,19 @@ define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <v
   ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res
 }
 
+; BFMAXNM (Single, x2)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @multi_vec_maxnm_single_x2_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: multi_vec_maxnm_single_x2_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    bfmaxnm { z0.h, z1.h }, { z0.h, z1.h }, z2.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmaxnm.single.x2.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
+
 ; FMAXNM (Single, x2)
 
 define { <vscale x 8 x half>, <vscale x 8 x half> }  @multi_vec_maxnm_single_x2_f16(<vscale x 8 x half> %dummy, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zm) {
@@ -847,6 +922,21 @@ define { <vscale x 2 x double>, <vscale x 2 x double> }  @multi_vec_maxnm_single
   ret { <vscale x 2 x double>, <vscale x 2 x double> } %res
 }
 
+; BFMAXNM (Single, x4)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @multi_vec_maxnm_single_x4_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: multi_vec_maxnm_single_x4_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    bfmaxnm { z0.h - z3.h }, { z0.h - z3.h }, z4.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @llvm.aarch64.sve.fmaxnm.single.x4.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } %res
+}
+
 ; FMAXNM (Single, x4)
 
 define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_maxnm_single_x4_f16(<vscale x 8 x half> %dummy, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x 8 x half> %zdn4, <vscale x 8 x half> %zm) {
@@ -903,6 +993,21 @@ define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <v
   ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res
 }
 
+; BFMAXNM (Multi, x2)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @multi_vec_maxnm_x2_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2) {
+; CHECK-LABEL: multi_vec_maxnm_x2_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT:    bfmaxnm { z0.h, z1.h }, { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmaxnm.x2.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
+
 ; FMAXNM (Multi, x2)
 
 define { <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_maxnm_x2_f16(<vscale x 8 x half> %dummy, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2) {
@@ -950,6 +1055,25 @@ define { <vscale x 2 x double>, <vscale x 2 x double> } @multi_vec_maxnm_x2_f64(
   ret { <vscale x 2 x double>, <vscale x 2 x double> } %res
 }
 
+; BFMAXNM (Multi, x4)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @multi_vec_maxnm_x4_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3, <vscale x 8 x bfloat> %zm4) {
+; CHECK-LABEL: multi_vec_maxnm_x4_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    bfmaxnm { z0.h - z3.h }, { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @llvm.aarch64.sve.fmaxnm.x4.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3, <vscale x 8 x bfloat> %zm4)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } %res
+}
+
 ; FMAXNM (Multi, x4)
 
 define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_maxnm_x4_f16(<vscale x 8 x half> %dummy, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x 8 x half> %zdn4, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2, <vscale x 8 x half> %zm3, <vscale x 8 x half> %zm4) {
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll
index 363f9ba5d353..e5c36d42fb13 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+b16b16 -verify-machineinstrs < %s | FileCheck %s
 
 ; SMIN (Single, x2)
 
@@ -109,6 +109,19 @@ define { <vscale x 2 x i64>, <vscale x 2 x i64> } @multi_vec_min_single_x2_u64(<
   ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res
 }
 
+; BFMIN (Single, x2)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @multi_vec_min_single_x2_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: multi_vec_min_single_x2_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    bfmin { z0.h, z1.h }, { z0.h, z1.h }, z2.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmin.single.x2.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
+
 ; FMIN (Single, x2)
 
 define { <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_min_single_x2_f16(<vscale x 8 x half> %unused, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zm) {
@@ -298,6 +311,21 @@ define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2
   ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res
 }
 
+; BFMIN (Single, x4)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @multi_vec_min_single_x4_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: multi_vec_min_single_x4_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    bfmin { z0.h - z3.h }, { z0.h - z3.h }, z4.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @llvm.aarch64.sve.fmin.single.x4.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } %res
+}
+
 ; FMIN (SINGLE, x4)
 
 define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_min_single_x4_f16(<vscale x 8 x half> %unused, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x 8 x half> %zdn4, <vscale x 8 x half> %zm) {
@@ -478,6 +506,21 @@ define { <vscale x 2 x i64>, <vscale x 2 x i64> } @multi_vec_min_multi_x2_u64(<v
   ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res
 }
 
+; BFMIN (Multi, x2)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @multi_vec_min_x2_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2) {
+; CHECK-LABEL: multi_vec_min_x2_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT:    bfmin { z0.h, z1.h }, { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fmin.x2.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
+
 ; FMIN (Multi, x2)
 
 define { <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_min_multi_x2_f16(<vscale x 8 x half> %unused, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2) {
@@ -729,6 +772,26 @@ define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2
   ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res
 }
 
+
+; BFMIN (Multi, x4)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @multi_vec_min_x4_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3, <vscale x 8 x bfloat> %zm4) {
+; CHECK-LABEL: multi_vec_min_x4_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    bfmin { z0.h - z3.h }, { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @llvm.aarch64.sve.fmin.x4.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3, <vscale x 8 x bfloat> %zm4)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } %res
+}
+
 ; FMIN (Multi, x4)
 
 define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_min_multi_x4_f16(<vscale x 8 x half> %unused, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x 8 x half> %zdn4,
@@ -806,6 +869,19 @@ define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <v
   ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res
 }
 
+; BFMINNM (Single, x2)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @multi_vec_minnm_single_x2_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: multi_vec_minnm_single_x2_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    bfminnm { z0.h, z1.h }, { z0.h, z1.h }, z2.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fminnm.single.x2.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
+
 ; FMINNM (Single, x2)
 
 define { <vscale x 8 x half>, <vscale x 8 x half> }  @multi_vec_minnm_single_x2_f16(<vscale x 8 x half> %dummy, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zm) {
@@ -847,6 +923,21 @@ define { <vscale x 2 x double>, <vscale x 2 x double> }  @multi_vec_minnm_single
   ret { <vscale x 2 x double>, <vscale x 2 x double> } %res
 }
 
+; BFMINNM (Single, x4)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @multi_vec_minnm_single_x4_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: multi_vec_minnm_single_x4_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    bfminnm { z0.h - z3.h }, { z0.h - z3.h }, z4.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @llvm.aarch64.sve.fminnm.single.x4.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } %res
+}
+
 ; FMINNM (Single, x4)
 
 define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_minnm_single_x4_f16(<vscale x 8 x half> %dummy, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x 8 x half> %zdn4, <vscale x 8 x half> %zm) {
@@ -903,6 +994,21 @@ define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <v
   ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res
 }
 
+; BFMINNM (Multi, x2)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @multi_vec_minnm_x2_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2) {
+; CHECK-LABEL: multi_vec_minnm_x2_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT:    bfminnm { z0.h, z1.h }, { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.fminnm.x2.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res
+}
+
 ; FMINNM (Multi, x2)
 
 define { <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_minnm_x2_f16(<vscale x 8 x half> %dummy, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2) {
@@ -950,6 +1056,25 @@ define { <vscale x 2 x double>, <vscale x 2 x double> } @multi_vec_minnm_x2_f64(
   ret { <vscale x 2 x double>, <vscale x 2 x double> } %res
 }
 
+; BFMINNM (Multi, x4)
+
+define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @multi_vec_minnm_x4_bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3, <vscale x 8 x bfloat> %zm4) {
+; CHECK-LABEL: multi_vec_minnm_x4_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7
+; CHECK-NEXT:    bfminnm { z0.h - z3.h }, { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } @llvm.aarch64.sve.fminnm.x4.nxv8bf16(<vscale x 8 x bfloat> %zdn1, <vscale x 8 x bfloat> %zdn2, <vscale x 8 x bfloat> %zdn3, <vscale x 8 x bfloat> %zdn4, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3, <vscale x 8 x bfloat> %zm4)
+  ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>  } %res
+}
+
 ; FMINNM (Multi, x4)
 
 define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_minnm_x4_f16(<vscale x 8 x half> %dummy, <vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x 8 x half> %zdn4, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2, <vscale x 8 x half> %zm3, <vscale x 8 x half> %zm4) {
-- 
GitLab


From af57ad6536c7df19e6df7217d9d976067fdaf882 Mon Sep 17 00:00:00 2001
From: zibi2 <62662650+zibi2@users.noreply.github.com>
Date: Thu, 16 May 2024 08:58:44 -0400
Subject: [PATCH 1013/1206] [libc++][z/OS] Correct a definition of
 __native_vector_size (#91995)

Fix `std/ranges/range.adaptors/range.lazy.split/general.pass.cpp` which
started failing on z/OS after this
[commit](https://github.com/llvm/llvm-project/commit/985c1a44f8d49e0af).

This test case is passing on other platforms such as AIX. This is
because the `__ALTIVEC__` macro is defined and `__mismatch` under
`_LIBCPP_VECTORIZE_ALGORITHMS` guard is compiled out. However, on z/OS
`_LIBCPP_VECTORIZE_ALGORITHMS` is defined. Analyzing the algorithm of
`__mismatch` shows that the culprit is the definition of
`__native_vector_size` which was defined wrongly as 1. This PR corrects
the definition of `__native_vector_size` and fixes the affected test.
---
 libcxx/include/__algorithm/simd_utils.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/include/__algorithm/simd_utils.h b/libcxx/include/__algorithm/simd_utils.h
index 71d65e8f4afb..aa4336a2214c 100644
--- a/libcxx/include/__algorithm/simd_utils.h
+++ b/libcxx/include/__algorithm/simd_utils.h
@@ -74,7 +74,7 @@ using __get_as_integer_type_t = typename __get_as_integer_type_impl<sizeof(_Tp)>
 // This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance
 // in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms
 // as far as benchmarks are concerned.
-#  if defined(__AVX__)
+#  if defined(__AVX__) || defined(__MVS__)
 template <class _Tp>
 inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp);
 #  elif defined(__SSE__) || defined(__ARM_NEON__)
-- 
GitLab


From 4a5dffc67499640c71357d8f3f49edc97af5482f Mon Sep 17 00:00:00 2001
From: Jacek Caban <jacek@codeweavers.com>
Date: Wed, 15 May 2024 11:57:05 +0200
Subject: [PATCH 1014/1206] [CodeGen][ARM64EC][NFC] Add ARM64EC alias symbols
 test. (#92100)

---
 llvm/test/CodeGen/AArch64/arm64ec-symbols.ll | 27 ++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/arm64ec-symbols.ll

diff --git a/llvm/test/CodeGen/AArch64/arm64ec-symbols.ll b/llvm/test/CodeGen/AArch64/arm64ec-symbols.ll
new file mode 100644
index 000000000000..389c36974bed
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64ec-symbols.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple=arm64ec-pc-windows-msvc < %s | FileCheck %s
+; RUN: llc -mtriple=arm64ec-pc-windows-msvc -filetype=obj -o %t.o < %s
+; RUN: llvm-objdump -t %t.o | FileCheck --check-prefix=SYM %s
+
+declare void @func() nounwind;
+
+define void @caller() nounwind {
+  call void @func()
+  ret void
+}
+
+; CHECK:      .weak_anti_dep  caller
+; CHECK-NEXT: .set caller, "#caller"@WEAKREF
+
+; CHECK:      .weak_anti_dep  func
+; CHECK-NEXT: .set func, "#func"@WEAKREF
+; CHECK-NEXT: .weak_anti_dep  "#func"
+; CHECK-NEXT: .set "#func", "#func$exit_thunk"@WEAKREF
+
+; SYM:       [ 8](sec  4)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #caller
+; SYM:       [21](sec  7)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #func$exit_thunk
+; SYM:       [33](sec  0)(fl 0x00)(ty   0)(scl  69) (nx 1) 0x00000000 caller
+; SYM-NEXT:  AUX indx 8 srch 4
+; SYM-NEXT:  [35](sec  0)(fl 0x00)(ty   0)(scl  69) (nx 1) 0x00000000 #func
+; SYM-NEXT:  AUX indx 21 srch 4
+; SYM:       [39](sec  0)(fl 0x00)(ty   0)(scl  69) (nx 1) 0x00000000 func
+; SYM-NEXT:  AUX indx 35 srch 4
-- 
GitLab


From 54e52aa5ebe68de122a3fe6064e0abef97f6b8e0 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 16 May 2024 14:44:00 +0100
Subject: [PATCH 1015/1206] [X86] Reduce znver3/4 LoopMicroOpBufferSize to
 practical loop unrolling values (#91340)

The znver3/4 scheduler models have previously associated the LoopMicroOpBufferSize with the maximum size of their op caches, and when this led to quadratic complexity issues this were reduced to a value of 512 uops, based mainly on compilation time and not its effectiveness on runtime performance.

From a runtime performance POV, a large LoopMicroOpBufferSize leads to a higher number of loop unrolls, meaning the cpu has to rely on the frontend decode rate (4 ins/cy max) for much longer to fill the op cache before looping begins and we make use of the faster op cache rate (8/9 ops/cy).

This patch proposes we instead cap the size of the LoopMicroOpBufferSize based off the maximum rate from the op cache (znver3 = 8op/cy, znver4 = 9op/cy) and the branch misprediction penalty from the opcache (~12cy) as a estimate of the useful number of ops we can unroll a loop by before mispredictions are likely to cause stalls. This isn't a perfect metric, but does try to be closer to the spirit of how we use LoopMicroOpBufferSize in the compiler vs the size of a similar naming buffer in the cpu.
---
 llvm/lib/Target/X86/X86ScheduleZnver3.td      |  11 +-
 llvm/lib/Target/X86/X86ScheduleZnver4.td      |  16 +-
 llvm/test/Transforms/LoopUnroll/X86/znver3.ll | 982 +-----------------
 3 files changed, 44 insertions(+), 965 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ScheduleZnver3.td b/llvm/lib/Target/X86/X86ScheduleZnver3.td
index 2e87d5262818..cbf1de840879 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver3.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver3.td
@@ -33,13 +33,10 @@ def Znver3Model : SchedMachineModel {
   // The op cache is organized as an associative cache with 64 sets and 8 ways.
   // At each set-way intersection is an entry containing up to 8 macro ops.
   // The maximum capacity of the op cache is 4K ops.
-  // Agner, 22.5 µop cache
-  // The size of the µop cache is big enough for holding most critical loops.
-  // FIXME: PR50584: MachineScheduler/PostRAScheduler have quadradic complexity,
-  //        with large values here the compilation of certain loops
-  //        ends up taking way too long.
-  // let LoopMicroOpBufferSize = 4096;
-  let LoopMicroOpBufferSize = 512;
+  // Assuming a maximum dispatch of 8 ops/cy and a mispredict cost of 12cy from
+  // the op-cache, we limit the loop buffer to 8*12 = 96 to avoid loop unrolling
+  // leading to excessive filling of the op-cache from frontend.
+  let LoopMicroOpBufferSize = 96;
   // AMD SOG 19h, 2.6.2 L1 Data Cache
   // The L1 data cache has a 4- or 5- cycle integer load-to-use latency.
   // AMD SOG 19h, 2.12 L1 Data Cache
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver4.td b/llvm/lib/Target/X86/X86ScheduleZnver4.td
index dac4d8422582..7107dbc63e27 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver4.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver4.td
@@ -28,17 +28,11 @@ def Znver4Model : SchedMachineModel {
   // AMD SOG 19h, 2.9.1 Op Cache
   // The op cache is organized as an associative cache with 64 sets and 8 ways.
   // At each set-way intersection is an entry containing up to 8 macro ops.
-  // The maximum capacity of the op cache is 4K ops.
-  // Agner, 22.5 µop cache
-  // The size of the µop cache is big enough for holding most critical loops.
-  // FIXME: PR50584: MachineScheduler/PostRAScheduler have quadradic complexity,
-  //        with large values here the compilation of certain loops
-  //        ends up taking way too long.
-  // Ideally for znver4, we should have 6.75K. However we don't add that
-  // considerting the impact compile time and prefer using default values 
-  // instead.
-  // Retaining minimal value to influence unrolling as we did for znver3.
-  let LoopMicroOpBufferSize = 512;
+  // The maximum capacity of the op cache is 6.75K ops.
+  // Assuming a maximum dispatch of 9 ops/cy and a mispredict cost of 12cy from
+  // the op-cache, we limit the loop buffer to 9*12 = 108 to avoid loop
+  // unrolling leading to excessive filling of the op-cache from frontend.
+  let LoopMicroOpBufferSize = 108;
   // AMD SOG 19h, 2.6.2 L1 Data Cache
   // The L1 data cache has a 4- or 5- cycle integer load-to-use latency.
   // AMD SOG 19h, 2.12 L1 Data Cache
diff --git a/llvm/test/Transforms/LoopUnroll/X86/znver3.ll b/llvm/test/Transforms/LoopUnroll/X86/znver3.ll
index 30389062a096..b1f1d7d814e6 100644
--- a/llvm/test/Transforms/LoopUnroll/X86/znver3.ll
+++ b/llvm/test/Transforms/LoopUnroll/X86/znver3.ll
@@ -73,456 +73,8 @@ define i32 @test(ptr %ary) "target-cpu"="znver3" {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_14:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 15
 ; CHECK-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_14]]
 ; CHECK-NEXT:    [[VAL_15:%.*]] = load i32, ptr [[ARRAYIDX_15]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_15:%.*]] = add nsw i32 [[VAL_15]], [[SUM_NEXT_14]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_15:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 16
-; CHECK-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_15]]
-; CHECK-NEXT:    [[VAL_16:%.*]] = load i32, ptr [[ARRAYIDX_16]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_16:%.*]] = add nsw i32 [[VAL_16]], [[SUM_NEXT_15]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_16:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 17
-; CHECK-NEXT:    [[ARRAYIDX_17:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_16]]
-; CHECK-NEXT:    [[VAL_17:%.*]] = load i32, ptr [[ARRAYIDX_17]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_17:%.*]] = add nsw i32 [[VAL_17]], [[SUM_NEXT_16]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_17:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 18
-; CHECK-NEXT:    [[ARRAYIDX_18:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_17]]
-; CHECK-NEXT:    [[VAL_18:%.*]] = load i32, ptr [[ARRAYIDX_18]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_18:%.*]] = add nsw i32 [[VAL_18]], [[SUM_NEXT_17]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_18:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 19
-; CHECK-NEXT:    [[ARRAYIDX_19:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_18]]
-; CHECK-NEXT:    [[VAL_19:%.*]] = load i32, ptr [[ARRAYIDX_19]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_19:%.*]] = add nsw i32 [[VAL_19]], [[SUM_NEXT_18]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_19:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 20
-; CHECK-NEXT:    [[ARRAYIDX_20:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_19]]
-; CHECK-NEXT:    [[VAL_20:%.*]] = load i32, ptr [[ARRAYIDX_20]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_20:%.*]] = add nsw i32 [[VAL_20]], [[SUM_NEXT_19]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_20:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 21
-; CHECK-NEXT:    [[ARRAYIDX_21:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_20]]
-; CHECK-NEXT:    [[VAL_21:%.*]] = load i32, ptr [[ARRAYIDX_21]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_21:%.*]] = add nsw i32 [[VAL_21]], [[SUM_NEXT_20]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_21:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 22
-; CHECK-NEXT:    [[ARRAYIDX_22:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_21]]
-; CHECK-NEXT:    [[VAL_22:%.*]] = load i32, ptr [[ARRAYIDX_22]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_22:%.*]] = add nsw i32 [[VAL_22]], [[SUM_NEXT_21]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_22:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 23
-; CHECK-NEXT:    [[ARRAYIDX_23:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_22]]
-; CHECK-NEXT:    [[VAL_23:%.*]] = load i32, ptr [[ARRAYIDX_23]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_23:%.*]] = add nsw i32 [[VAL_23]], [[SUM_NEXT_22]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_23:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 24
-; CHECK-NEXT:    [[ARRAYIDX_24:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_23]]
-; CHECK-NEXT:    [[VAL_24:%.*]] = load i32, ptr [[ARRAYIDX_24]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_24:%.*]] = add nsw i32 [[VAL_24]], [[SUM_NEXT_23]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_24:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 25
-; CHECK-NEXT:    [[ARRAYIDX_25:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_24]]
-; CHECK-NEXT:    [[VAL_25:%.*]] = load i32, ptr [[ARRAYIDX_25]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_25:%.*]] = add nsw i32 [[VAL_25]], [[SUM_NEXT_24]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_25:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 26
-; CHECK-NEXT:    [[ARRAYIDX_26:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_25]]
-; CHECK-NEXT:    [[VAL_26:%.*]] = load i32, ptr [[ARRAYIDX_26]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_26:%.*]] = add nsw i32 [[VAL_26]], [[SUM_NEXT_25]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_26:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 27
-; CHECK-NEXT:    [[ARRAYIDX_27:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_26]]
-; CHECK-NEXT:    [[VAL_27:%.*]] = load i32, ptr [[ARRAYIDX_27]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_27:%.*]] = add nsw i32 [[VAL_27]], [[SUM_NEXT_26]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_27:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 28
-; CHECK-NEXT:    [[ARRAYIDX_28:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_27]]
-; CHECK-NEXT:    [[VAL_28:%.*]] = load i32, ptr [[ARRAYIDX_28]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_28:%.*]] = add nsw i32 [[VAL_28]], [[SUM_NEXT_27]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_28:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 29
-; CHECK-NEXT:    [[ARRAYIDX_29:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_28]]
-; CHECK-NEXT:    [[VAL_29:%.*]] = load i32, ptr [[ARRAYIDX_29]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_29:%.*]] = add nsw i32 [[VAL_29]], [[SUM_NEXT_28]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_29:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 30
-; CHECK-NEXT:    [[ARRAYIDX_30:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_29]]
-; CHECK-NEXT:    [[VAL_30:%.*]] = load i32, ptr [[ARRAYIDX_30]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_30:%.*]] = add nsw i32 [[VAL_30]], [[SUM_NEXT_29]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_30:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 31
-; CHECK-NEXT:    [[ARRAYIDX_31:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_30]]
-; CHECK-NEXT:    [[VAL_31:%.*]] = load i32, ptr [[ARRAYIDX_31]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_31:%.*]] = add nsw i32 [[VAL_31]], [[SUM_NEXT_30]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_31:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 32
-; CHECK-NEXT:    [[ARRAYIDX_32:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_31]]
-; CHECK-NEXT:    [[VAL_32:%.*]] = load i32, ptr [[ARRAYIDX_32]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_32:%.*]] = add nsw i32 [[VAL_32]], [[SUM_NEXT_31]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_32:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 33
-; CHECK-NEXT:    [[ARRAYIDX_33:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_32]]
-; CHECK-NEXT:    [[VAL_33:%.*]] = load i32, ptr [[ARRAYIDX_33]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_33:%.*]] = add nsw i32 [[VAL_33]], [[SUM_NEXT_32]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_33:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 34
-; CHECK-NEXT:    [[ARRAYIDX_34:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_33]]
-; CHECK-NEXT:    [[VAL_34:%.*]] = load i32, ptr [[ARRAYIDX_34]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_34:%.*]] = add nsw i32 [[VAL_34]], [[SUM_NEXT_33]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_34:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 35
-; CHECK-NEXT:    [[ARRAYIDX_35:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_34]]
-; CHECK-NEXT:    [[VAL_35:%.*]] = load i32, ptr [[ARRAYIDX_35]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_35:%.*]] = add nsw i32 [[VAL_35]], [[SUM_NEXT_34]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_35:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 36
-; CHECK-NEXT:    [[ARRAYIDX_36:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_35]]
-; CHECK-NEXT:    [[VAL_36:%.*]] = load i32, ptr [[ARRAYIDX_36]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_36:%.*]] = add nsw i32 [[VAL_36]], [[SUM_NEXT_35]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_36:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 37
-; CHECK-NEXT:    [[ARRAYIDX_37:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_36]]
-; CHECK-NEXT:    [[VAL_37:%.*]] = load i32, ptr [[ARRAYIDX_37]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_37:%.*]] = add nsw i32 [[VAL_37]], [[SUM_NEXT_36]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_37:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 38
-; CHECK-NEXT:    [[ARRAYIDX_38:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_37]]
-; CHECK-NEXT:    [[VAL_38:%.*]] = load i32, ptr [[ARRAYIDX_38]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_38:%.*]] = add nsw i32 [[VAL_38]], [[SUM_NEXT_37]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_38:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 39
-; CHECK-NEXT:    [[ARRAYIDX_39:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_38]]
-; CHECK-NEXT:    [[VAL_39:%.*]] = load i32, ptr [[ARRAYIDX_39]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_39:%.*]] = add nsw i32 [[VAL_39]], [[SUM_NEXT_38]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_39:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 40
-; CHECK-NEXT:    [[ARRAYIDX_40:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_39]]
-; CHECK-NEXT:    [[VAL_40:%.*]] = load i32, ptr [[ARRAYIDX_40]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_40:%.*]] = add nsw i32 [[VAL_40]], [[SUM_NEXT_39]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_40:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 41
-; CHECK-NEXT:    [[ARRAYIDX_41:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_40]]
-; CHECK-NEXT:    [[VAL_41:%.*]] = load i32, ptr [[ARRAYIDX_41]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_41:%.*]] = add nsw i32 [[VAL_41]], [[SUM_NEXT_40]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_41:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 42
-; CHECK-NEXT:    [[ARRAYIDX_42:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_41]]
-; CHECK-NEXT:    [[VAL_42:%.*]] = load i32, ptr [[ARRAYIDX_42]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_42:%.*]] = add nsw i32 [[VAL_42]], [[SUM_NEXT_41]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_42:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 43
-; CHECK-NEXT:    [[ARRAYIDX_43:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_42]]
-; CHECK-NEXT:    [[VAL_43:%.*]] = load i32, ptr [[ARRAYIDX_43]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_43:%.*]] = add nsw i32 [[VAL_43]], [[SUM_NEXT_42]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_43:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 44
-; CHECK-NEXT:    [[ARRAYIDX_44:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_43]]
-; CHECK-NEXT:    [[VAL_44:%.*]] = load i32, ptr [[ARRAYIDX_44]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_44:%.*]] = add nsw i32 [[VAL_44]], [[SUM_NEXT_43]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_44:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 45
-; CHECK-NEXT:    [[ARRAYIDX_45:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_44]]
-; CHECK-NEXT:    [[VAL_45:%.*]] = load i32, ptr [[ARRAYIDX_45]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_45:%.*]] = add nsw i32 [[VAL_45]], [[SUM_NEXT_44]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_45:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 46
-; CHECK-NEXT:    [[ARRAYIDX_46:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_45]]
-; CHECK-NEXT:    [[VAL_46:%.*]] = load i32, ptr [[ARRAYIDX_46]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_46:%.*]] = add nsw i32 [[VAL_46]], [[SUM_NEXT_45]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_46:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 47
-; CHECK-NEXT:    [[ARRAYIDX_47:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_46]]
-; CHECK-NEXT:    [[VAL_47:%.*]] = load i32, ptr [[ARRAYIDX_47]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_47:%.*]] = add nsw i32 [[VAL_47]], [[SUM_NEXT_46]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_47:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 48
-; CHECK-NEXT:    [[ARRAYIDX_48:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_47]]
-; CHECK-NEXT:    [[VAL_48:%.*]] = load i32, ptr [[ARRAYIDX_48]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_48:%.*]] = add nsw i32 [[VAL_48]], [[SUM_NEXT_47]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_48:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 49
-; CHECK-NEXT:    [[ARRAYIDX_49:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_48]]
-; CHECK-NEXT:    [[VAL_49:%.*]] = load i32, ptr [[ARRAYIDX_49]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_49:%.*]] = add nsw i32 [[VAL_49]], [[SUM_NEXT_48]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_49:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 50
-; CHECK-NEXT:    [[ARRAYIDX_50:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_49]]
-; CHECK-NEXT:    [[VAL_50:%.*]] = load i32, ptr [[ARRAYIDX_50]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_50:%.*]] = add nsw i32 [[VAL_50]], [[SUM_NEXT_49]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_50:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 51
-; CHECK-NEXT:    [[ARRAYIDX_51:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_50]]
-; CHECK-NEXT:    [[VAL_51:%.*]] = load i32, ptr [[ARRAYIDX_51]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_51:%.*]] = add nsw i32 [[VAL_51]], [[SUM_NEXT_50]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_51:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 52
-; CHECK-NEXT:    [[ARRAYIDX_52:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_51]]
-; CHECK-NEXT:    [[VAL_52:%.*]] = load i32, ptr [[ARRAYIDX_52]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_52:%.*]] = add nsw i32 [[VAL_52]], [[SUM_NEXT_51]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_52:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 53
-; CHECK-NEXT:    [[ARRAYIDX_53:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_52]]
-; CHECK-NEXT:    [[VAL_53:%.*]] = load i32, ptr [[ARRAYIDX_53]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_53:%.*]] = add nsw i32 [[VAL_53]], [[SUM_NEXT_52]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_53:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 54
-; CHECK-NEXT:    [[ARRAYIDX_54:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_53]]
-; CHECK-NEXT:    [[VAL_54:%.*]] = load i32, ptr [[ARRAYIDX_54]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_54:%.*]] = add nsw i32 [[VAL_54]], [[SUM_NEXT_53]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_54:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 55
-; CHECK-NEXT:    [[ARRAYIDX_55:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_54]]
-; CHECK-NEXT:    [[VAL_55:%.*]] = load i32, ptr [[ARRAYIDX_55]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_55:%.*]] = add nsw i32 [[VAL_55]], [[SUM_NEXT_54]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_55:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 56
-; CHECK-NEXT:    [[ARRAYIDX_56:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_55]]
-; CHECK-NEXT:    [[VAL_56:%.*]] = load i32, ptr [[ARRAYIDX_56]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_56:%.*]] = add nsw i32 [[VAL_56]], [[SUM_NEXT_55]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_56:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 57
-; CHECK-NEXT:    [[ARRAYIDX_57:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_56]]
-; CHECK-NEXT:    [[VAL_57:%.*]] = load i32, ptr [[ARRAYIDX_57]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_57:%.*]] = add nsw i32 [[VAL_57]], [[SUM_NEXT_56]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_57:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 58
-; CHECK-NEXT:    [[ARRAYIDX_58:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_57]]
-; CHECK-NEXT:    [[VAL_58:%.*]] = load i32, ptr [[ARRAYIDX_58]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_58:%.*]] = add nsw i32 [[VAL_58]], [[SUM_NEXT_57]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_58:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 59
-; CHECK-NEXT:    [[ARRAYIDX_59:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_58]]
-; CHECK-NEXT:    [[VAL_59:%.*]] = load i32, ptr [[ARRAYIDX_59]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_59:%.*]] = add nsw i32 [[VAL_59]], [[SUM_NEXT_58]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_59:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 60
-; CHECK-NEXT:    [[ARRAYIDX_60:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_59]]
-; CHECK-NEXT:    [[VAL_60:%.*]] = load i32, ptr [[ARRAYIDX_60]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_60:%.*]] = add nsw i32 [[VAL_60]], [[SUM_NEXT_59]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_60:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 61
-; CHECK-NEXT:    [[ARRAYIDX_61:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_60]]
-; CHECK-NEXT:    [[VAL_61:%.*]] = load i32, ptr [[ARRAYIDX_61]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_61:%.*]] = add nsw i32 [[VAL_61]], [[SUM_NEXT_60]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_61:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 62
-; CHECK-NEXT:    [[ARRAYIDX_62:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_61]]
-; CHECK-NEXT:    [[VAL_62:%.*]] = load i32, ptr [[ARRAYIDX_62]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_62:%.*]] = add nsw i32 [[VAL_62]], [[SUM_NEXT_61]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_62:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 63
-; CHECK-NEXT:    [[ARRAYIDX_63:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_62]]
-; CHECK-NEXT:    [[VAL_63:%.*]] = load i32, ptr [[ARRAYIDX_63]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_63:%.*]] = add nsw i32 [[VAL_63]], [[SUM_NEXT_62]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_63:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 64
-; CHECK-NEXT:    [[ARRAYIDX_64:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_63]]
-; CHECK-NEXT:    [[VAL_64:%.*]] = load i32, ptr [[ARRAYIDX_64]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_64:%.*]] = add nsw i32 [[VAL_64]], [[SUM_NEXT_63]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_64:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 65
-; CHECK-NEXT:    [[ARRAYIDX_65:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_64]]
-; CHECK-NEXT:    [[VAL_65:%.*]] = load i32, ptr [[ARRAYIDX_65]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_65:%.*]] = add nsw i32 [[VAL_65]], [[SUM_NEXT_64]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_65:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 66
-; CHECK-NEXT:    [[ARRAYIDX_66:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_65]]
-; CHECK-NEXT:    [[VAL_66:%.*]] = load i32, ptr [[ARRAYIDX_66]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_66:%.*]] = add nsw i32 [[VAL_66]], [[SUM_NEXT_65]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_66:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 67
-; CHECK-NEXT:    [[ARRAYIDX_67:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_66]]
-; CHECK-NEXT:    [[VAL_67:%.*]] = load i32, ptr [[ARRAYIDX_67]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_67:%.*]] = add nsw i32 [[VAL_67]], [[SUM_NEXT_66]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_67:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 68
-; CHECK-NEXT:    [[ARRAYIDX_68:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_67]]
-; CHECK-NEXT:    [[VAL_68:%.*]] = load i32, ptr [[ARRAYIDX_68]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_68:%.*]] = add nsw i32 [[VAL_68]], [[SUM_NEXT_67]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_68:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 69
-; CHECK-NEXT:    [[ARRAYIDX_69:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_68]]
-; CHECK-NEXT:    [[VAL_69:%.*]] = load i32, ptr [[ARRAYIDX_69]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_69:%.*]] = add nsw i32 [[VAL_69]], [[SUM_NEXT_68]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_69:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 70
-; CHECK-NEXT:    [[ARRAYIDX_70:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_69]]
-; CHECK-NEXT:    [[VAL_70:%.*]] = load i32, ptr [[ARRAYIDX_70]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_70:%.*]] = add nsw i32 [[VAL_70]], [[SUM_NEXT_69]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_70:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 71
-; CHECK-NEXT:    [[ARRAYIDX_71:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_70]]
-; CHECK-NEXT:    [[VAL_71:%.*]] = load i32, ptr [[ARRAYIDX_71]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_71:%.*]] = add nsw i32 [[VAL_71]], [[SUM_NEXT_70]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_71:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 72
-; CHECK-NEXT:    [[ARRAYIDX_72:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_71]]
-; CHECK-NEXT:    [[VAL_72:%.*]] = load i32, ptr [[ARRAYIDX_72]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_72:%.*]] = add nsw i32 [[VAL_72]], [[SUM_NEXT_71]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_72:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 73
-; CHECK-NEXT:    [[ARRAYIDX_73:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_72]]
-; CHECK-NEXT:    [[VAL_73:%.*]] = load i32, ptr [[ARRAYIDX_73]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_73:%.*]] = add nsw i32 [[VAL_73]], [[SUM_NEXT_72]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_73:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 74
-; CHECK-NEXT:    [[ARRAYIDX_74:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_73]]
-; CHECK-NEXT:    [[VAL_74:%.*]] = load i32, ptr [[ARRAYIDX_74]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_74:%.*]] = add nsw i32 [[VAL_74]], [[SUM_NEXT_73]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_74:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 75
-; CHECK-NEXT:    [[ARRAYIDX_75:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_74]]
-; CHECK-NEXT:    [[VAL_75:%.*]] = load i32, ptr [[ARRAYIDX_75]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_75:%.*]] = add nsw i32 [[VAL_75]], [[SUM_NEXT_74]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_75:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 76
-; CHECK-NEXT:    [[ARRAYIDX_76:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_75]]
-; CHECK-NEXT:    [[VAL_76:%.*]] = load i32, ptr [[ARRAYIDX_76]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_76:%.*]] = add nsw i32 [[VAL_76]], [[SUM_NEXT_75]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_76:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 77
-; CHECK-NEXT:    [[ARRAYIDX_77:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_76]]
-; CHECK-NEXT:    [[VAL_77:%.*]] = load i32, ptr [[ARRAYIDX_77]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_77:%.*]] = add nsw i32 [[VAL_77]], [[SUM_NEXT_76]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_77:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 78
-; CHECK-NEXT:    [[ARRAYIDX_78:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_77]]
-; CHECK-NEXT:    [[VAL_78:%.*]] = load i32, ptr [[ARRAYIDX_78]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_78:%.*]] = add nsw i32 [[VAL_78]], [[SUM_NEXT_77]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_78:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 79
-; CHECK-NEXT:    [[ARRAYIDX_79:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_78]]
-; CHECK-NEXT:    [[VAL_79:%.*]] = load i32, ptr [[ARRAYIDX_79]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_79:%.*]] = add nsw i32 [[VAL_79]], [[SUM_NEXT_78]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_79:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 80
-; CHECK-NEXT:    [[ARRAYIDX_80:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_79]]
-; CHECK-NEXT:    [[VAL_80:%.*]] = load i32, ptr [[ARRAYIDX_80]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_80:%.*]] = add nsw i32 [[VAL_80]], [[SUM_NEXT_79]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_80:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 81
-; CHECK-NEXT:    [[ARRAYIDX_81:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_80]]
-; CHECK-NEXT:    [[VAL_81:%.*]] = load i32, ptr [[ARRAYIDX_81]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_81:%.*]] = add nsw i32 [[VAL_81]], [[SUM_NEXT_80]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_81:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 82
-; CHECK-NEXT:    [[ARRAYIDX_82:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_81]]
-; CHECK-NEXT:    [[VAL_82:%.*]] = load i32, ptr [[ARRAYIDX_82]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_82:%.*]] = add nsw i32 [[VAL_82]], [[SUM_NEXT_81]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_82:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 83
-; CHECK-NEXT:    [[ARRAYIDX_83:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_82]]
-; CHECK-NEXT:    [[VAL_83:%.*]] = load i32, ptr [[ARRAYIDX_83]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_83:%.*]] = add nsw i32 [[VAL_83]], [[SUM_NEXT_82]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_83:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 84
-; CHECK-NEXT:    [[ARRAYIDX_84:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_83]]
-; CHECK-NEXT:    [[VAL_84:%.*]] = load i32, ptr [[ARRAYIDX_84]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_84:%.*]] = add nsw i32 [[VAL_84]], [[SUM_NEXT_83]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_84:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 85
-; CHECK-NEXT:    [[ARRAYIDX_85:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_84]]
-; CHECK-NEXT:    [[VAL_85:%.*]] = load i32, ptr [[ARRAYIDX_85]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_85:%.*]] = add nsw i32 [[VAL_85]], [[SUM_NEXT_84]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_85:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 86
-; CHECK-NEXT:    [[ARRAYIDX_86:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_85]]
-; CHECK-NEXT:    [[VAL_86:%.*]] = load i32, ptr [[ARRAYIDX_86]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_86:%.*]] = add nsw i32 [[VAL_86]], [[SUM_NEXT_85]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_86:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 87
-; CHECK-NEXT:    [[ARRAYIDX_87:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_86]]
-; CHECK-NEXT:    [[VAL_87:%.*]] = load i32, ptr [[ARRAYIDX_87]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_87:%.*]] = add nsw i32 [[VAL_87]], [[SUM_NEXT_86]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_87:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 88
-; CHECK-NEXT:    [[ARRAYIDX_88:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_87]]
-; CHECK-NEXT:    [[VAL_88:%.*]] = load i32, ptr [[ARRAYIDX_88]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_88:%.*]] = add nsw i32 [[VAL_88]], [[SUM_NEXT_87]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_88:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 89
-; CHECK-NEXT:    [[ARRAYIDX_89:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_88]]
-; CHECK-NEXT:    [[VAL_89:%.*]] = load i32, ptr [[ARRAYIDX_89]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_89:%.*]] = add nsw i32 [[VAL_89]], [[SUM_NEXT_88]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_89:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 90
-; CHECK-NEXT:    [[ARRAYIDX_90:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_89]]
-; CHECK-NEXT:    [[VAL_90:%.*]] = load i32, ptr [[ARRAYIDX_90]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_90:%.*]] = add nsw i32 [[VAL_90]], [[SUM_NEXT_89]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_90:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 91
-; CHECK-NEXT:    [[ARRAYIDX_91:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_90]]
-; CHECK-NEXT:    [[VAL_91:%.*]] = load i32, ptr [[ARRAYIDX_91]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_91:%.*]] = add nsw i32 [[VAL_91]], [[SUM_NEXT_90]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_91:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 92
-; CHECK-NEXT:    [[ARRAYIDX_92:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_91]]
-; CHECK-NEXT:    [[VAL_92:%.*]] = load i32, ptr [[ARRAYIDX_92]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_92:%.*]] = add nsw i32 [[VAL_92]], [[SUM_NEXT_91]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_92:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 93
-; CHECK-NEXT:    [[ARRAYIDX_93:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_92]]
-; CHECK-NEXT:    [[VAL_93:%.*]] = load i32, ptr [[ARRAYIDX_93]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_93:%.*]] = add nsw i32 [[VAL_93]], [[SUM_NEXT_92]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_93:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 94
-; CHECK-NEXT:    [[ARRAYIDX_94:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_93]]
-; CHECK-NEXT:    [[VAL_94:%.*]] = load i32, ptr [[ARRAYIDX_94]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_94:%.*]] = add nsw i32 [[VAL_94]], [[SUM_NEXT_93]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_94:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 95
-; CHECK-NEXT:    [[ARRAYIDX_95:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_94]]
-; CHECK-NEXT:    [[VAL_95:%.*]] = load i32, ptr [[ARRAYIDX_95]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_95:%.*]] = add nsw i32 [[VAL_95]], [[SUM_NEXT_94]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_95:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 96
-; CHECK-NEXT:    [[ARRAYIDX_96:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_95]]
-; CHECK-NEXT:    [[VAL_96:%.*]] = load i32, ptr [[ARRAYIDX_96]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_96:%.*]] = add nsw i32 [[VAL_96]], [[SUM_NEXT_95]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_96:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 97
-; CHECK-NEXT:    [[ARRAYIDX_97:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_96]]
-; CHECK-NEXT:    [[VAL_97:%.*]] = load i32, ptr [[ARRAYIDX_97]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_97:%.*]] = add nsw i32 [[VAL_97]], [[SUM_NEXT_96]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_97:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 98
-; CHECK-NEXT:    [[ARRAYIDX_98:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_97]]
-; CHECK-NEXT:    [[VAL_98:%.*]] = load i32, ptr [[ARRAYIDX_98]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_98:%.*]] = add nsw i32 [[VAL_98]], [[SUM_NEXT_97]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_98:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 99
-; CHECK-NEXT:    [[ARRAYIDX_99:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_98]]
-; CHECK-NEXT:    [[VAL_99:%.*]] = load i32, ptr [[ARRAYIDX_99]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_99:%.*]] = add nsw i32 [[VAL_99]], [[SUM_NEXT_98]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_99:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 100
-; CHECK-NEXT:    [[ARRAYIDX_100:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_99]]
-; CHECK-NEXT:    [[VAL_100:%.*]] = load i32, ptr [[ARRAYIDX_100]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_100:%.*]] = add nsw i32 [[VAL_100]], [[SUM_NEXT_99]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_100:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 101
-; CHECK-NEXT:    [[ARRAYIDX_101:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_100]]
-; CHECK-NEXT:    [[VAL_101:%.*]] = load i32, ptr [[ARRAYIDX_101]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_101:%.*]] = add nsw i32 [[VAL_101]], [[SUM_NEXT_100]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_101:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 102
-; CHECK-NEXT:    [[ARRAYIDX_102:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_101]]
-; CHECK-NEXT:    [[VAL_102:%.*]] = load i32, ptr [[ARRAYIDX_102]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_102:%.*]] = add nsw i32 [[VAL_102]], [[SUM_NEXT_101]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_102:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 103
-; CHECK-NEXT:    [[ARRAYIDX_103:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_102]]
-; CHECK-NEXT:    [[VAL_103:%.*]] = load i32, ptr [[ARRAYIDX_103]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_103:%.*]] = add nsw i32 [[VAL_103]], [[SUM_NEXT_102]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_103:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 104
-; CHECK-NEXT:    [[ARRAYIDX_104:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_103]]
-; CHECK-NEXT:    [[VAL_104:%.*]] = load i32, ptr [[ARRAYIDX_104]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_104:%.*]] = add nsw i32 [[VAL_104]], [[SUM_NEXT_103]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_104:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 105
-; CHECK-NEXT:    [[ARRAYIDX_105:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_104]]
-; CHECK-NEXT:    [[VAL_105:%.*]] = load i32, ptr [[ARRAYIDX_105]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_105:%.*]] = add nsw i32 [[VAL_105]], [[SUM_NEXT_104]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_105:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 106
-; CHECK-NEXT:    [[ARRAYIDX_106:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_105]]
-; CHECK-NEXT:    [[VAL_106:%.*]] = load i32, ptr [[ARRAYIDX_106]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_106:%.*]] = add nsw i32 [[VAL_106]], [[SUM_NEXT_105]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_106:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 107
-; CHECK-NEXT:    [[ARRAYIDX_107:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_106]]
-; CHECK-NEXT:    [[VAL_107:%.*]] = load i32, ptr [[ARRAYIDX_107]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_107:%.*]] = add nsw i32 [[VAL_107]], [[SUM_NEXT_106]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_107:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 108
-; CHECK-NEXT:    [[ARRAYIDX_108:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_107]]
-; CHECK-NEXT:    [[VAL_108:%.*]] = load i32, ptr [[ARRAYIDX_108]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_108:%.*]] = add nsw i32 [[VAL_108]], [[SUM_NEXT_107]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_108:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 109
-; CHECK-NEXT:    [[ARRAYIDX_109:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_108]]
-; CHECK-NEXT:    [[VAL_109:%.*]] = load i32, ptr [[ARRAYIDX_109]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_109:%.*]] = add nsw i32 [[VAL_109]], [[SUM_NEXT_108]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_109:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 110
-; CHECK-NEXT:    [[ARRAYIDX_110:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_109]]
-; CHECK-NEXT:    [[VAL_110:%.*]] = load i32, ptr [[ARRAYIDX_110]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_110:%.*]] = add nsw i32 [[VAL_110]], [[SUM_NEXT_109]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_110:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 111
-; CHECK-NEXT:    [[ARRAYIDX_111:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_110]]
-; CHECK-NEXT:    [[VAL_111:%.*]] = load i32, ptr [[ARRAYIDX_111]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_111:%.*]] = add nsw i32 [[VAL_111]], [[SUM_NEXT_110]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_111:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 112
-; CHECK-NEXT:    [[ARRAYIDX_112:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_111]]
-; CHECK-NEXT:    [[VAL_112:%.*]] = load i32, ptr [[ARRAYIDX_112]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_112:%.*]] = add nsw i32 [[VAL_112]], [[SUM_NEXT_111]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_112:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 113
-; CHECK-NEXT:    [[ARRAYIDX_113:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_112]]
-; CHECK-NEXT:    [[VAL_113:%.*]] = load i32, ptr [[ARRAYIDX_113]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_113:%.*]] = add nsw i32 [[VAL_113]], [[SUM_NEXT_112]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_113:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 114
-; CHECK-NEXT:    [[ARRAYIDX_114:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_113]]
-; CHECK-NEXT:    [[VAL_114:%.*]] = load i32, ptr [[ARRAYIDX_114]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_114:%.*]] = add nsw i32 [[VAL_114]], [[SUM_NEXT_113]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_114:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 115
-; CHECK-NEXT:    [[ARRAYIDX_115:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_114]]
-; CHECK-NEXT:    [[VAL_115:%.*]] = load i32, ptr [[ARRAYIDX_115]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_115:%.*]] = add nsw i32 [[VAL_115]], [[SUM_NEXT_114]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_115:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 116
-; CHECK-NEXT:    [[ARRAYIDX_116:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_115]]
-; CHECK-NEXT:    [[VAL_116:%.*]] = load i32, ptr [[ARRAYIDX_116]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_116:%.*]] = add nsw i32 [[VAL_116]], [[SUM_NEXT_115]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_116:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 117
-; CHECK-NEXT:    [[ARRAYIDX_117:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_116]]
-; CHECK-NEXT:    [[VAL_117:%.*]] = load i32, ptr [[ARRAYIDX_117]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_117:%.*]] = add nsw i32 [[VAL_117]], [[SUM_NEXT_116]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_117:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 118
-; CHECK-NEXT:    [[ARRAYIDX_118:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_117]]
-; CHECK-NEXT:    [[VAL_118:%.*]] = load i32, ptr [[ARRAYIDX_118]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_118:%.*]] = add nsw i32 [[VAL_118]], [[SUM_NEXT_117]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_118:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 119
-; CHECK-NEXT:    [[ARRAYIDX_119:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_118]]
-; CHECK-NEXT:    [[VAL_119:%.*]] = load i32, ptr [[ARRAYIDX_119]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_119:%.*]] = add nsw i32 [[VAL_119]], [[SUM_NEXT_118]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_119:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 120
-; CHECK-NEXT:    [[ARRAYIDX_120:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_119]]
-; CHECK-NEXT:    [[VAL_120:%.*]] = load i32, ptr [[ARRAYIDX_120]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_120:%.*]] = add nsw i32 [[VAL_120]], [[SUM_NEXT_119]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_120:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 121
-; CHECK-NEXT:    [[ARRAYIDX_121:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_120]]
-; CHECK-NEXT:    [[VAL_121:%.*]] = load i32, ptr [[ARRAYIDX_121]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_121:%.*]] = add nsw i32 [[VAL_121]], [[SUM_NEXT_120]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_121:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 122
-; CHECK-NEXT:    [[ARRAYIDX_122:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_121]]
-; CHECK-NEXT:    [[VAL_122:%.*]] = load i32, ptr [[ARRAYIDX_122]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_122:%.*]] = add nsw i32 [[VAL_122]], [[SUM_NEXT_121]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_122:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 123
-; CHECK-NEXT:    [[ARRAYIDX_123:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_122]]
-; CHECK-NEXT:    [[VAL_123:%.*]] = load i32, ptr [[ARRAYIDX_123]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_123:%.*]] = add nsw i32 [[VAL_123]], [[SUM_NEXT_122]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_123:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 124
-; CHECK-NEXT:    [[ARRAYIDX_124:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_123]]
-; CHECK-NEXT:    [[VAL_124:%.*]] = load i32, ptr [[ARRAYIDX_124]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_124:%.*]] = add nsw i32 [[VAL_124]], [[SUM_NEXT_123]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_124:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 125
-; CHECK-NEXT:    [[ARRAYIDX_125:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_124]]
-; CHECK-NEXT:    [[VAL_125:%.*]] = load i32, ptr [[ARRAYIDX_125]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_125:%.*]] = add nsw i32 [[VAL_125]], [[SUM_NEXT_124]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_125:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 126
-; CHECK-NEXT:    [[ARRAYIDX_126:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_125]]
-; CHECK-NEXT:    [[VAL_126:%.*]] = load i32, ptr [[ARRAYIDX_126]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_126:%.*]] = add nsw i32 [[VAL_126]], [[SUM_NEXT_125]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_126:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 127
-; CHECK-NEXT:    [[ARRAYIDX_127:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_126]]
-; CHECK-NEXT:    [[VAL_127:%.*]] = load i32, ptr [[ARRAYIDX_127]], align 4
-; CHECK-NEXT:    [[SUM_NEXT_127]] = add nsw i32 [[VAL_127]], [[SUM_NEXT_126]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_127]] = add nuw nsw i64 [[INDVARS_IV]], 128
+; CHECK-NEXT:    [[SUM_NEXT_127]] = add nsw i32 [[VAL_15]], [[SUM_NEXT_14]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_127]] = add nuw nsw i64 [[INDVARS_IV]], 16
 ; CHECK-NEXT:    [[EXITCOND_NOT_127:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_127]], 8192
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT_127]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.cond.cleanup:
@@ -550,532 +102,68 @@ define i32 @test2(ptr %ary, i64 %n) "target-cpu"="znver3" {
 ; CHECK-LABEL: define i32 @test2(
 ; CHECK-SAME: ptr [[ARY:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -1
-; CHECK-NEXT:    [[XTRAITER:%.*]] = and i64 [[N]], 7
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 7
-; CHECK-NEXT:    br i1 [[TMP1]], label [[FOR_COND_CLEANUP_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]]
-; CHECK:       entry.new:
-; CHECK-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[INDVARS_IV_NEXT_7:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[ENTRY_NEW]] ], [ [[SUM_NEXT_7:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[NITER:%.*]] = phi i64 [ 0, [[ENTRY_NEW]] ], [ [[NITER_NEXT_7:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[DUMMY1:%.*]] = mul i32 [[VAL]], [[VAL]]
-; CHECK-NEXT:    [[DUMMY2:%.*]] = mul i32 [[DUMMY1]], [[DUMMY1]]
-; CHECK-NEXT:    [[DUMMY3:%.*]] = mul i32 [[DUMMY2]], [[DUMMY2]]
-; CHECK-NEXT:    [[DUMMY4:%.*]] = mul i32 [[DUMMY3]], [[DUMMY3]]
+; CHECK-NEXT:    [[INDVARS_IV_EPIL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_EPIL:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_NEXT_EPIL:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_EPIL]]
+; CHECK-NEXT:    [[VAL_EPIL:%.*]] = load i32, ptr [[ARRAYIDX_EPIL]], align 4
+; CHECK-NEXT:    [[DUMMY21:%.*]] = mul i32 [[VAL_EPIL]], [[VAL_EPIL]]
+; CHECK-NEXT:    [[DUMMY27:%.*]] = mul i32 [[DUMMY21]], [[DUMMY21]]
+; CHECK-NEXT:    [[DUMMY28:%.*]] = mul i32 [[DUMMY27]], [[DUMMY27]]
+; CHECK-NEXT:    [[DUMMY4:%.*]] = mul i32 [[DUMMY28]], [[DUMMY28]]
 ; CHECK-NEXT:    [[DUMMY5:%.*]] = mul i32 [[DUMMY4]], [[DUMMY4]]
 ; CHECK-NEXT:    [[DUMMY6:%.*]] = mul i32 [[DUMMY5]], [[DUMMY5]]
 ; CHECK-NEXT:    [[DUMMY7:%.*]] = mul i32 [[DUMMY6]], [[DUMMY6]]
 ; CHECK-NEXT:    [[DUMMY8:%.*]] = mul i32 [[DUMMY7]], [[DUMMY7]]
 ; CHECK-NEXT:    [[DUMMY9:%.*]] = mul i32 [[DUMMY8]], [[DUMMY8]]
 ; CHECK-NEXT:    [[DUMMY10:%.*]] = mul i32 [[DUMMY9]], [[DUMMY9]]
-; CHECK-NEXT:    [[DUMMY11:%.*]] = mul i32 [[DUMMY10]], [[DUMMY10]]
-; CHECK-NEXT:    [[DUMMY12:%.*]] = mul i32 [[DUMMY11]], [[DUMMY11]]
-; CHECK-NEXT:    [[DUMMY13:%.*]] = mul i32 [[DUMMY12]], [[DUMMY12]]
-; CHECK-NEXT:    [[DUMMY14:%.*]] = mul i32 [[DUMMY13]], [[DUMMY13]]
-; CHECK-NEXT:    [[DUMMY15:%.*]] = mul i32 [[DUMMY14]], [[DUMMY14]]
+; CHECK-NEXT:    [[DUMMY29:%.*]] = mul i32 [[DUMMY10]], [[DUMMY10]]
+; CHECK-NEXT:    [[DUMMY30:%.*]] = mul i32 [[DUMMY29]], [[DUMMY29]]
+; CHECK-NEXT:    [[DUMMY39:%.*]] = mul i32 [[DUMMY30]], [[DUMMY30]]
+; CHECK-NEXT:    [[DUMMY40:%.*]] = mul i32 [[DUMMY39]], [[DUMMY39]]
+; CHECK-NEXT:    [[DUMMY15:%.*]] = mul i32 [[DUMMY40]], [[DUMMY40]]
 ; CHECK-NEXT:    [[DUMMY16:%.*]] = mul i32 [[DUMMY15]], [[DUMMY15]]
 ; CHECK-NEXT:    [[DUMMY17:%.*]] = mul i32 [[DUMMY16]], [[DUMMY16]]
 ; CHECK-NEXT:    [[DUMMY18:%.*]] = mul i32 [[DUMMY17]], [[DUMMY17]]
 ; CHECK-NEXT:    [[DUMMY19:%.*]] = mul i32 [[DUMMY18]], [[DUMMY18]]
 ; CHECK-NEXT:    [[DUMMY20:%.*]] = mul i32 [[DUMMY19]], [[DUMMY19]]
-; CHECK-NEXT:    [[DUMMY21:%.*]] = mul i32 [[DUMMY20]], [[DUMMY20]]
-; CHECK-NEXT:    [[DUMMY22:%.*]] = mul i32 [[DUMMY21]], [[DUMMY21]]
+; CHECK-NEXT:    [[VAL:%.*]] = mul i32 [[DUMMY20]], [[DUMMY20]]
+; CHECK-NEXT:    [[DUMMY1:%.*]] = mul i32 [[VAL]], [[VAL]]
+; CHECK-NEXT:    [[DUMMY2:%.*]] = mul i32 [[DUMMY1]], [[DUMMY1]]
+; CHECK-NEXT:    [[DUMMY3:%.*]] = mul i32 [[DUMMY2]], [[DUMMY2]]
+; CHECK-NEXT:    [[DUMMY41:%.*]] = mul i32 [[DUMMY3]], [[DUMMY3]]
+; CHECK-NEXT:    [[DUMMY26:%.*]] = mul i32 [[DUMMY41]], [[DUMMY41]]
+; CHECK-NEXT:    [[DUMMY11:%.*]] = mul i32 [[DUMMY26]], [[DUMMY26]]
+; CHECK-NEXT:    [[DUMMY12:%.*]] = mul i32 [[DUMMY11]], [[DUMMY11]]
+; CHECK-NEXT:    [[DUMMY13:%.*]] = mul i32 [[DUMMY12]], [[DUMMY12]]
+; CHECK-NEXT:    [[DUMMY14:%.*]] = mul i32 [[DUMMY13]], [[DUMMY13]]
+; CHECK-NEXT:    [[DUMMY31:%.*]] = mul i32 [[DUMMY14]], [[DUMMY14]]
+; CHECK-NEXT:    [[DUMMY32:%.*]] = mul i32 [[DUMMY31]], [[DUMMY31]]
+; CHECK-NEXT:    [[DUMMY22:%.*]] = mul i32 [[DUMMY32]], [[DUMMY32]]
 ; CHECK-NEXT:    [[DUMMY23:%.*]] = mul i32 [[DUMMY22]], [[DUMMY22]]
 ; CHECK-NEXT:    [[DUMMY24:%.*]] = mul i32 [[DUMMY23]], [[DUMMY23]]
 ; CHECK-NEXT:    [[DUMMY25:%.*]] = mul i32 [[DUMMY24]], [[DUMMY24]]
-; CHECK-NEXT:    [[DUMMY26:%.*]] = mul i32 [[DUMMY25]], [[DUMMY25]]
-; CHECK-NEXT:    [[DUMMY27:%.*]] = mul i32 [[DUMMY26]], [[DUMMY26]]
-; CHECK-NEXT:    [[DUMMY28:%.*]] = mul i32 [[DUMMY27]], [[DUMMY27]]
-; CHECK-NEXT:    [[DUMMY29:%.*]] = mul i32 [[DUMMY28]], [[DUMMY28]]
-; CHECK-NEXT:    [[DUMMY30:%.*]] = mul i32 [[DUMMY29]], [[DUMMY29]]
-; CHECK-NEXT:    [[DUMMY31:%.*]] = mul i32 [[DUMMY30]], [[DUMMY30]]
-; CHECK-NEXT:    [[DUMMY32:%.*]] = mul i32 [[DUMMY31]], [[DUMMY31]]
-; CHECK-NEXT:    [[DUMMY33:%.*]] = mul i32 [[DUMMY32]], [[DUMMY32]]
+; CHECK-NEXT:    [[DUMMY37:%.*]] = mul i32 [[DUMMY25]], [[DUMMY25]]
+; CHECK-NEXT:    [[DUMMY38:%.*]] = mul i32 [[DUMMY37]], [[DUMMY37]]
+; CHECK-NEXT:    [[DUMMY33:%.*]] = mul i32 [[DUMMY38]], [[DUMMY38]]
 ; CHECK-NEXT:    [[DUMMY34:%.*]] = mul i32 [[DUMMY33]], [[DUMMY33]]
 ; CHECK-NEXT:    [[DUMMY35:%.*]] = mul i32 [[DUMMY34]], [[DUMMY34]]
 ; CHECK-NEXT:    [[DUMMY36:%.*]] = mul i32 [[DUMMY35]], [[DUMMY35]]
-; CHECK-NEXT:    [[DUMMY37:%.*]] = mul i32 [[DUMMY36]], [[DUMMY36]]
-; CHECK-NEXT:    [[DUMMY38:%.*]] = mul i32 [[DUMMY37]], [[DUMMY37]]
-; CHECK-NEXT:    [[DUMMY39:%.*]] = mul i32 [[DUMMY38]], [[DUMMY38]]
-; CHECK-NEXT:    [[DUMMY40:%.*]] = mul i32 [[DUMMY39]], [[DUMMY39]]
-; CHECK-NEXT:    [[DUMMY41:%.*]] = mul i32 [[DUMMY40]], [[DUMMY40]]
-; CHECK-NEXT:    [[DUMMY42:%.*]] = mul i32 [[DUMMY41]], [[DUMMY41]]
-; CHECK-NEXT:    [[DUMMY43:%.*]] = mul i32 [[DUMMY42]], [[DUMMY42]]
+; CHECK-NEXT:    [[DUMMY43:%.*]] = mul i32 [[DUMMY36]], [[DUMMY36]]
 ; CHECK-NEXT:    [[DUMMY44:%.*]] = mul i32 [[DUMMY43]], [[DUMMY43]]
 ; CHECK-NEXT:    [[DUMMY45:%.*]] = mul i32 [[DUMMY44]], [[DUMMY44]]
 ; CHECK-NEXT:    [[DUMMY46:%.*]] = mul i32 [[DUMMY45]], [[DUMMY45]]
 ; CHECK-NEXT:    [[DUMMY47:%.*]] = mul i32 [[DUMMY46]], [[DUMMY46]]
 ; CHECK-NEXT:    [[DUMMY48:%.*]] = mul i32 [[DUMMY47]], [[DUMMY47]]
 ; CHECK-NEXT:    [[DUMMY49:%.*]] = mul i32 [[DUMMY48]], [[DUMMY48]]
-; CHECK-NEXT:    [[DUMMY50:%.*]] = mul i32 [[DUMMY49]], [[DUMMY49]]
-; CHECK-NEXT:    [[SUM_NEXT:%.*]] = add nsw i32 [[DUMMY50]], [[SUM]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT]]
-; CHECK-NEXT:    [[VAL_1:%.*]] = load i32, ptr [[ARRAYIDX_1]], align 4
-; CHECK-NEXT:    [[DUMMY1_1:%.*]] = mul i32 [[VAL_1]], [[VAL_1]]
-; CHECK-NEXT:    [[DUMMY2_1:%.*]] = mul i32 [[DUMMY1_1]], [[DUMMY1_1]]
-; CHECK-NEXT:    [[DUMMY3_1:%.*]] = mul i32 [[DUMMY2_1]], [[DUMMY2_1]]
-; CHECK-NEXT:    [[DUMMY4_1:%.*]] = mul i32 [[DUMMY3_1]], [[DUMMY3_1]]
-; CHECK-NEXT:    [[DUMMY5_1:%.*]] = mul i32 [[DUMMY4_1]], [[DUMMY4_1]]
-; CHECK-NEXT:    [[DUMMY6_1:%.*]] = mul i32 [[DUMMY5_1]], [[DUMMY5_1]]
-; CHECK-NEXT:    [[DUMMY7_1:%.*]] = mul i32 [[DUMMY6_1]], [[DUMMY6_1]]
-; CHECK-NEXT:    [[DUMMY8_1:%.*]] = mul i32 [[DUMMY7_1]], [[DUMMY7_1]]
-; CHECK-NEXT:    [[DUMMY9_1:%.*]] = mul i32 [[DUMMY8_1]], [[DUMMY8_1]]
-; CHECK-NEXT:    [[DUMMY10_1:%.*]] = mul i32 [[DUMMY9_1]], [[DUMMY9_1]]
-; CHECK-NEXT:    [[DUMMY11_1:%.*]] = mul i32 [[DUMMY10_1]], [[DUMMY10_1]]
-; CHECK-NEXT:    [[DUMMY12_1:%.*]] = mul i32 [[DUMMY11_1]], [[DUMMY11_1]]
-; CHECK-NEXT:    [[DUMMY13_1:%.*]] = mul i32 [[DUMMY12_1]], [[DUMMY12_1]]
-; CHECK-NEXT:    [[DUMMY14_1:%.*]] = mul i32 [[DUMMY13_1]], [[DUMMY13_1]]
-; CHECK-NEXT:    [[DUMMY15_1:%.*]] = mul i32 [[DUMMY14_1]], [[DUMMY14_1]]
-; CHECK-NEXT:    [[DUMMY16_1:%.*]] = mul i32 [[DUMMY15_1]], [[DUMMY15_1]]
-; CHECK-NEXT:    [[DUMMY17_1:%.*]] = mul i32 [[DUMMY16_1]], [[DUMMY16_1]]
-; CHECK-NEXT:    [[DUMMY18_1:%.*]] = mul i32 [[DUMMY17_1]], [[DUMMY17_1]]
-; CHECK-NEXT:    [[DUMMY19_1:%.*]] = mul i32 [[DUMMY18_1]], [[DUMMY18_1]]
-; CHECK-NEXT:    [[DUMMY20_1:%.*]] = mul i32 [[DUMMY19_1]], [[DUMMY19_1]]
-; CHECK-NEXT:    [[DUMMY21_1:%.*]] = mul i32 [[DUMMY20_1]], [[DUMMY20_1]]
-; CHECK-NEXT:    [[DUMMY22_1:%.*]] = mul i32 [[DUMMY21_1]], [[DUMMY21_1]]
-; CHECK-NEXT:    [[DUMMY23_1:%.*]] = mul i32 [[DUMMY22_1]], [[DUMMY22_1]]
-; CHECK-NEXT:    [[DUMMY24_1:%.*]] = mul i32 [[DUMMY23_1]], [[DUMMY23_1]]
-; CHECK-NEXT:    [[DUMMY25_1:%.*]] = mul i32 [[DUMMY24_1]], [[DUMMY24_1]]
-; CHECK-NEXT:    [[DUMMY26_1:%.*]] = mul i32 [[DUMMY25_1]], [[DUMMY25_1]]
-; CHECK-NEXT:    [[DUMMY27_1:%.*]] = mul i32 [[DUMMY26_1]], [[DUMMY26_1]]
-; CHECK-NEXT:    [[DUMMY28_1:%.*]] = mul i32 [[DUMMY27_1]], [[DUMMY27_1]]
-; CHECK-NEXT:    [[DUMMY29_1:%.*]] = mul i32 [[DUMMY28_1]], [[DUMMY28_1]]
-; CHECK-NEXT:    [[DUMMY30_1:%.*]] = mul i32 [[DUMMY29_1]], [[DUMMY29_1]]
-; CHECK-NEXT:    [[DUMMY31_1:%.*]] = mul i32 [[DUMMY30_1]], [[DUMMY30_1]]
-; CHECK-NEXT:    [[DUMMY32_1:%.*]] = mul i32 [[DUMMY31_1]], [[DUMMY31_1]]
-; CHECK-NEXT:    [[DUMMY33_1:%.*]] = mul i32 [[DUMMY32_1]], [[DUMMY32_1]]
-; CHECK-NEXT:    [[DUMMY34_1:%.*]] = mul i32 [[DUMMY33_1]], [[DUMMY33_1]]
-; CHECK-NEXT:    [[DUMMY35_1:%.*]] = mul i32 [[DUMMY34_1]], [[DUMMY34_1]]
-; CHECK-NEXT:    [[DUMMY36_1:%.*]] = mul i32 [[DUMMY35_1]], [[DUMMY35_1]]
-; CHECK-NEXT:    [[DUMMY37_1:%.*]] = mul i32 [[DUMMY36_1]], [[DUMMY36_1]]
-; CHECK-NEXT:    [[DUMMY38_1:%.*]] = mul i32 [[DUMMY37_1]], [[DUMMY37_1]]
-; CHECK-NEXT:    [[DUMMY39_1:%.*]] = mul i32 [[DUMMY38_1]], [[DUMMY38_1]]
-; CHECK-NEXT:    [[DUMMY40_1:%.*]] = mul i32 [[DUMMY39_1]], [[DUMMY39_1]]
-; CHECK-NEXT:    [[DUMMY41_1:%.*]] = mul i32 [[DUMMY40_1]], [[DUMMY40_1]]
-; CHECK-NEXT:    [[DUMMY42_1:%.*]] = mul i32 [[DUMMY41_1]], [[DUMMY41_1]]
-; CHECK-NEXT:    [[DUMMY43_1:%.*]] = mul i32 [[DUMMY42_1]], [[DUMMY42_1]]
-; CHECK-NEXT:    [[DUMMY44_1:%.*]] = mul i32 [[DUMMY43_1]], [[DUMMY43_1]]
-; CHECK-NEXT:    [[DUMMY45_1:%.*]] = mul i32 [[DUMMY44_1]], [[DUMMY44_1]]
-; CHECK-NEXT:    [[DUMMY46_1:%.*]] = mul i32 [[DUMMY45_1]], [[DUMMY45_1]]
-; CHECK-NEXT:    [[DUMMY47_1:%.*]] = mul i32 [[DUMMY46_1]], [[DUMMY46_1]]
-; CHECK-NEXT:    [[DUMMY48_1:%.*]] = mul i32 [[DUMMY47_1]], [[DUMMY47_1]]
-; CHECK-NEXT:    [[DUMMY49_1:%.*]] = mul i32 [[DUMMY48_1]], [[DUMMY48_1]]
-; CHECK-NEXT:    [[DUMMY50_1:%.*]] = mul i32 [[DUMMY49_1]], [[DUMMY49_1]]
-; CHECK-NEXT:    [[SUM_NEXT_1:%.*]] = add nsw i32 [[DUMMY50_1]], [[SUM_NEXT]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
-; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_1]]
-; CHECK-NEXT:    [[VAL_2:%.*]] = load i32, ptr [[ARRAYIDX_2]], align 4
-; CHECK-NEXT:    [[DUMMY1_2:%.*]] = mul i32 [[VAL_2]], [[VAL_2]]
-; CHECK-NEXT:    [[DUMMY2_2:%.*]] = mul i32 [[DUMMY1_2]], [[DUMMY1_2]]
-; CHECK-NEXT:    [[DUMMY3_2:%.*]] = mul i32 [[DUMMY2_2]], [[DUMMY2_2]]
-; CHECK-NEXT:    [[DUMMY4_2:%.*]] = mul i32 [[DUMMY3_2]], [[DUMMY3_2]]
-; CHECK-NEXT:    [[DUMMY5_2:%.*]] = mul i32 [[DUMMY4_2]], [[DUMMY4_2]]
-; CHECK-NEXT:    [[DUMMY6_2:%.*]] = mul i32 [[DUMMY5_2]], [[DUMMY5_2]]
-; CHECK-NEXT:    [[DUMMY7_2:%.*]] = mul i32 [[DUMMY6_2]], [[DUMMY6_2]]
-; CHECK-NEXT:    [[DUMMY8_2:%.*]] = mul i32 [[DUMMY7_2]], [[DUMMY7_2]]
-; CHECK-NEXT:    [[DUMMY9_2:%.*]] = mul i32 [[DUMMY8_2]], [[DUMMY8_2]]
-; CHECK-NEXT:    [[DUMMY10_2:%.*]] = mul i32 [[DUMMY9_2]], [[DUMMY9_2]]
-; CHECK-NEXT:    [[DUMMY11_2:%.*]] = mul i32 [[DUMMY10_2]], [[DUMMY10_2]]
-; CHECK-NEXT:    [[DUMMY12_2:%.*]] = mul i32 [[DUMMY11_2]], [[DUMMY11_2]]
-; CHECK-NEXT:    [[DUMMY13_2:%.*]] = mul i32 [[DUMMY12_2]], [[DUMMY12_2]]
-; CHECK-NEXT:    [[DUMMY14_2:%.*]] = mul i32 [[DUMMY13_2]], [[DUMMY13_2]]
-; CHECK-NEXT:    [[DUMMY15_2:%.*]] = mul i32 [[DUMMY14_2]], [[DUMMY14_2]]
-; CHECK-NEXT:    [[DUMMY16_2:%.*]] = mul i32 [[DUMMY15_2]], [[DUMMY15_2]]
-; CHECK-NEXT:    [[DUMMY17_2:%.*]] = mul i32 [[DUMMY16_2]], [[DUMMY16_2]]
-; CHECK-NEXT:    [[DUMMY18_2:%.*]] = mul i32 [[DUMMY17_2]], [[DUMMY17_2]]
-; CHECK-NEXT:    [[DUMMY19_2:%.*]] = mul i32 [[DUMMY18_2]], [[DUMMY18_2]]
-; CHECK-NEXT:    [[DUMMY20_2:%.*]] = mul i32 [[DUMMY19_2]], [[DUMMY19_2]]
-; CHECK-NEXT:    [[DUMMY21_2:%.*]] = mul i32 [[DUMMY20_2]], [[DUMMY20_2]]
-; CHECK-NEXT:    [[DUMMY22_2:%.*]] = mul i32 [[DUMMY21_2]], [[DUMMY21_2]]
-; CHECK-NEXT:    [[DUMMY23_2:%.*]] = mul i32 [[DUMMY22_2]], [[DUMMY22_2]]
-; CHECK-NEXT:    [[DUMMY24_2:%.*]] = mul i32 [[DUMMY23_2]], [[DUMMY23_2]]
-; CHECK-NEXT:    [[DUMMY25_2:%.*]] = mul i32 [[DUMMY24_2]], [[DUMMY24_2]]
-; CHECK-NEXT:    [[DUMMY26_2:%.*]] = mul i32 [[DUMMY25_2]], [[DUMMY25_2]]
-; CHECK-NEXT:    [[DUMMY27_2:%.*]] = mul i32 [[DUMMY26_2]], [[DUMMY26_2]]
-; CHECK-NEXT:    [[DUMMY28_2:%.*]] = mul i32 [[DUMMY27_2]], [[DUMMY27_2]]
-; CHECK-NEXT:    [[DUMMY29_2:%.*]] = mul i32 [[DUMMY28_2]], [[DUMMY28_2]]
-; CHECK-NEXT:    [[DUMMY30_2:%.*]] = mul i32 [[DUMMY29_2]], [[DUMMY29_2]]
-; CHECK-NEXT:    [[DUMMY31_2:%.*]] = mul i32 [[DUMMY30_2]], [[DUMMY30_2]]
-; CHECK-NEXT:    [[DUMMY32_2:%.*]] = mul i32 [[DUMMY31_2]], [[DUMMY31_2]]
-; CHECK-NEXT:    [[DUMMY33_2:%.*]] = mul i32 [[DUMMY32_2]], [[DUMMY32_2]]
-; CHECK-NEXT:    [[DUMMY34_2:%.*]] = mul i32 [[DUMMY33_2]], [[DUMMY33_2]]
-; CHECK-NEXT:    [[DUMMY35_2:%.*]] = mul i32 [[DUMMY34_2]], [[DUMMY34_2]]
-; CHECK-NEXT:    [[DUMMY36_2:%.*]] = mul i32 [[DUMMY35_2]], [[DUMMY35_2]]
-; CHECK-NEXT:    [[DUMMY37_2:%.*]] = mul i32 [[DUMMY36_2]], [[DUMMY36_2]]
-; CHECK-NEXT:    [[DUMMY38_2:%.*]] = mul i32 [[DUMMY37_2]], [[DUMMY37_2]]
-; CHECK-NEXT:    [[DUMMY39_2:%.*]] = mul i32 [[DUMMY38_2]], [[DUMMY38_2]]
-; CHECK-NEXT:    [[DUMMY40_2:%.*]] = mul i32 [[DUMMY39_2]], [[DUMMY39_2]]
-; CHECK-NEXT:    [[DUMMY41_2:%.*]] = mul i32 [[DUMMY40_2]], [[DUMMY40_2]]
-; CHECK-NEXT:    [[DUMMY42_2:%.*]] = mul i32 [[DUMMY41_2]], [[DUMMY41_2]]
-; CHECK-NEXT:    [[DUMMY43_2:%.*]] = mul i32 [[DUMMY42_2]], [[DUMMY42_2]]
-; CHECK-NEXT:    [[DUMMY44_2:%.*]] = mul i32 [[DUMMY43_2]], [[DUMMY43_2]]
-; CHECK-NEXT:    [[DUMMY45_2:%.*]] = mul i32 [[DUMMY44_2]], [[DUMMY44_2]]
-; CHECK-NEXT:    [[DUMMY46_2:%.*]] = mul i32 [[DUMMY45_2]], [[DUMMY45_2]]
-; CHECK-NEXT:    [[DUMMY47_2:%.*]] = mul i32 [[DUMMY46_2]], [[DUMMY46_2]]
-; CHECK-NEXT:    [[DUMMY48_2:%.*]] = mul i32 [[DUMMY47_2]], [[DUMMY47_2]]
-; CHECK-NEXT:    [[DUMMY49_2:%.*]] = mul i32 [[DUMMY48_2]], [[DUMMY48_2]]
-; CHECK-NEXT:    [[DUMMY50_2:%.*]] = mul i32 [[DUMMY49_2]], [[DUMMY49_2]]
-; CHECK-NEXT:    [[SUM_NEXT_2:%.*]] = add nsw i32 [[DUMMY50_2]], [[SUM_NEXT_1]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
-; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_2]]
-; CHECK-NEXT:    [[VAL_3:%.*]] = load i32, ptr [[ARRAYIDX_3]], align 4
-; CHECK-NEXT:    [[DUMMY1_3:%.*]] = mul i32 [[VAL_3]], [[VAL_3]]
-; CHECK-NEXT:    [[DUMMY2_3:%.*]] = mul i32 [[DUMMY1_3]], [[DUMMY1_3]]
-; CHECK-NEXT:    [[DUMMY3_3:%.*]] = mul i32 [[DUMMY2_3]], [[DUMMY2_3]]
-; CHECK-NEXT:    [[DUMMY4_3:%.*]] = mul i32 [[DUMMY3_3]], [[DUMMY3_3]]
-; CHECK-NEXT:    [[DUMMY5_3:%.*]] = mul i32 [[DUMMY4_3]], [[DUMMY4_3]]
-; CHECK-NEXT:    [[DUMMY6_3:%.*]] = mul i32 [[DUMMY5_3]], [[DUMMY5_3]]
-; CHECK-NEXT:    [[DUMMY7_3:%.*]] = mul i32 [[DUMMY6_3]], [[DUMMY6_3]]
-; CHECK-NEXT:    [[DUMMY8_3:%.*]] = mul i32 [[DUMMY7_3]], [[DUMMY7_3]]
-; CHECK-NEXT:    [[DUMMY9_3:%.*]] = mul i32 [[DUMMY8_3]], [[DUMMY8_3]]
-; CHECK-NEXT:    [[DUMMY10_3:%.*]] = mul i32 [[DUMMY9_3]], [[DUMMY9_3]]
-; CHECK-NEXT:    [[DUMMY11_3:%.*]] = mul i32 [[DUMMY10_3]], [[DUMMY10_3]]
-; CHECK-NEXT:    [[DUMMY12_3:%.*]] = mul i32 [[DUMMY11_3]], [[DUMMY11_3]]
-; CHECK-NEXT:    [[DUMMY13_3:%.*]] = mul i32 [[DUMMY12_3]], [[DUMMY12_3]]
-; CHECK-NEXT:    [[DUMMY14_3:%.*]] = mul i32 [[DUMMY13_3]], [[DUMMY13_3]]
-; CHECK-NEXT:    [[DUMMY15_3:%.*]] = mul i32 [[DUMMY14_3]], [[DUMMY14_3]]
-; CHECK-NEXT:    [[DUMMY16_3:%.*]] = mul i32 [[DUMMY15_3]], [[DUMMY15_3]]
-; CHECK-NEXT:    [[DUMMY17_3:%.*]] = mul i32 [[DUMMY16_3]], [[DUMMY16_3]]
-; CHECK-NEXT:    [[DUMMY18_3:%.*]] = mul i32 [[DUMMY17_3]], [[DUMMY17_3]]
-; CHECK-NEXT:    [[DUMMY19_3:%.*]] = mul i32 [[DUMMY18_3]], [[DUMMY18_3]]
-; CHECK-NEXT:    [[DUMMY20_3:%.*]] = mul i32 [[DUMMY19_3]], [[DUMMY19_3]]
-; CHECK-NEXT:    [[DUMMY21_3:%.*]] = mul i32 [[DUMMY20_3]], [[DUMMY20_3]]
-; CHECK-NEXT:    [[DUMMY22_3:%.*]] = mul i32 [[DUMMY21_3]], [[DUMMY21_3]]
-; CHECK-NEXT:    [[DUMMY23_3:%.*]] = mul i32 [[DUMMY22_3]], [[DUMMY22_3]]
-; CHECK-NEXT:    [[DUMMY24_3:%.*]] = mul i32 [[DUMMY23_3]], [[DUMMY23_3]]
-; CHECK-NEXT:    [[DUMMY25_3:%.*]] = mul i32 [[DUMMY24_3]], [[DUMMY24_3]]
-; CHECK-NEXT:    [[DUMMY26_3:%.*]] = mul i32 [[DUMMY25_3]], [[DUMMY25_3]]
-; CHECK-NEXT:    [[DUMMY27_3:%.*]] = mul i32 [[DUMMY26_3]], [[DUMMY26_3]]
-; CHECK-NEXT:    [[DUMMY28_3:%.*]] = mul i32 [[DUMMY27_3]], [[DUMMY27_3]]
-; CHECK-NEXT:    [[DUMMY29_3:%.*]] = mul i32 [[DUMMY28_3]], [[DUMMY28_3]]
-; CHECK-NEXT:    [[DUMMY30_3:%.*]] = mul i32 [[DUMMY29_3]], [[DUMMY29_3]]
-; CHECK-NEXT:    [[DUMMY31_3:%.*]] = mul i32 [[DUMMY30_3]], [[DUMMY30_3]]
-; CHECK-NEXT:    [[DUMMY32_3:%.*]] = mul i32 [[DUMMY31_3]], [[DUMMY31_3]]
-; CHECK-NEXT:    [[DUMMY33_3:%.*]] = mul i32 [[DUMMY32_3]], [[DUMMY32_3]]
-; CHECK-NEXT:    [[DUMMY34_3:%.*]] = mul i32 [[DUMMY33_3]], [[DUMMY33_3]]
-; CHECK-NEXT:    [[DUMMY35_3:%.*]] = mul i32 [[DUMMY34_3]], [[DUMMY34_3]]
-; CHECK-NEXT:    [[DUMMY36_3:%.*]] = mul i32 [[DUMMY35_3]], [[DUMMY35_3]]
-; CHECK-NEXT:    [[DUMMY37_3:%.*]] = mul i32 [[DUMMY36_3]], [[DUMMY36_3]]
-; CHECK-NEXT:    [[DUMMY38_3:%.*]] = mul i32 [[DUMMY37_3]], [[DUMMY37_3]]
-; CHECK-NEXT:    [[DUMMY39_3:%.*]] = mul i32 [[DUMMY38_3]], [[DUMMY38_3]]
-; CHECK-NEXT:    [[DUMMY40_3:%.*]] = mul i32 [[DUMMY39_3]], [[DUMMY39_3]]
-; CHECK-NEXT:    [[DUMMY41_3:%.*]] = mul i32 [[DUMMY40_3]], [[DUMMY40_3]]
-; CHECK-NEXT:    [[DUMMY42_3:%.*]] = mul i32 [[DUMMY41_3]], [[DUMMY41_3]]
-; CHECK-NEXT:    [[DUMMY43_3:%.*]] = mul i32 [[DUMMY42_3]], [[DUMMY42_3]]
-; CHECK-NEXT:    [[DUMMY44_3:%.*]] = mul i32 [[DUMMY43_3]], [[DUMMY43_3]]
-; CHECK-NEXT:    [[DUMMY45_3:%.*]] = mul i32 [[DUMMY44_3]], [[DUMMY44_3]]
-; CHECK-NEXT:    [[DUMMY46_3:%.*]] = mul i32 [[DUMMY45_3]], [[DUMMY45_3]]
-; CHECK-NEXT:    [[DUMMY47_3:%.*]] = mul i32 [[DUMMY46_3]], [[DUMMY46_3]]
-; CHECK-NEXT:    [[DUMMY48_3:%.*]] = mul i32 [[DUMMY47_3]], [[DUMMY47_3]]
-; CHECK-NEXT:    [[DUMMY49_3:%.*]] = mul i32 [[DUMMY48_3]], [[DUMMY48_3]]
-; CHECK-NEXT:    [[DUMMY50_3:%.*]] = mul i32 [[DUMMY49_3]], [[DUMMY49_3]]
-; CHECK-NEXT:    [[SUM_NEXT_3:%.*]] = add nsw i32 [[DUMMY50_3]], [[SUM_NEXT_2]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_3:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
-; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_3]]
-; CHECK-NEXT:    [[VAL_4:%.*]] = load i32, ptr [[ARRAYIDX_4]], align 4
-; CHECK-NEXT:    [[DUMMY1_4:%.*]] = mul i32 [[VAL_4]], [[VAL_4]]
-; CHECK-NEXT:    [[DUMMY2_4:%.*]] = mul i32 [[DUMMY1_4]], [[DUMMY1_4]]
-; CHECK-NEXT:    [[DUMMY3_4:%.*]] = mul i32 [[DUMMY2_4]], [[DUMMY2_4]]
-; CHECK-NEXT:    [[DUMMY4_4:%.*]] = mul i32 [[DUMMY3_4]], [[DUMMY3_4]]
-; CHECK-NEXT:    [[DUMMY5_4:%.*]] = mul i32 [[DUMMY4_4]], [[DUMMY4_4]]
-; CHECK-NEXT:    [[DUMMY6_4:%.*]] = mul i32 [[DUMMY5_4]], [[DUMMY5_4]]
-; CHECK-NEXT:    [[DUMMY7_4:%.*]] = mul i32 [[DUMMY6_4]], [[DUMMY6_4]]
-; CHECK-NEXT:    [[DUMMY8_4:%.*]] = mul i32 [[DUMMY7_4]], [[DUMMY7_4]]
-; CHECK-NEXT:    [[DUMMY9_4:%.*]] = mul i32 [[DUMMY8_4]], [[DUMMY8_4]]
-; CHECK-NEXT:    [[DUMMY10_4:%.*]] = mul i32 [[DUMMY9_4]], [[DUMMY9_4]]
-; CHECK-NEXT:    [[DUMMY11_4:%.*]] = mul i32 [[DUMMY10_4]], [[DUMMY10_4]]
-; CHECK-NEXT:    [[DUMMY12_4:%.*]] = mul i32 [[DUMMY11_4]], [[DUMMY11_4]]
-; CHECK-NEXT:    [[DUMMY13_4:%.*]] = mul i32 [[DUMMY12_4]], [[DUMMY12_4]]
-; CHECK-NEXT:    [[DUMMY14_4:%.*]] = mul i32 [[DUMMY13_4]], [[DUMMY13_4]]
-; CHECK-NEXT:    [[DUMMY15_4:%.*]] = mul i32 [[DUMMY14_4]], [[DUMMY14_4]]
-; CHECK-NEXT:    [[DUMMY16_4:%.*]] = mul i32 [[DUMMY15_4]], [[DUMMY15_4]]
-; CHECK-NEXT:    [[DUMMY17_4:%.*]] = mul i32 [[DUMMY16_4]], [[DUMMY16_4]]
-; CHECK-NEXT:    [[DUMMY18_4:%.*]] = mul i32 [[DUMMY17_4]], [[DUMMY17_4]]
-; CHECK-NEXT:    [[DUMMY19_4:%.*]] = mul i32 [[DUMMY18_4]], [[DUMMY18_4]]
-; CHECK-NEXT:    [[DUMMY20_4:%.*]] = mul i32 [[DUMMY19_4]], [[DUMMY19_4]]
-; CHECK-NEXT:    [[DUMMY21_4:%.*]] = mul i32 [[DUMMY20_4]], [[DUMMY20_4]]
-; CHECK-NEXT:    [[DUMMY22_4:%.*]] = mul i32 [[DUMMY21_4]], [[DUMMY21_4]]
-; CHECK-NEXT:    [[DUMMY23_4:%.*]] = mul i32 [[DUMMY22_4]], [[DUMMY22_4]]
-; CHECK-NEXT:    [[DUMMY24_4:%.*]] = mul i32 [[DUMMY23_4]], [[DUMMY23_4]]
-; CHECK-NEXT:    [[DUMMY25_4:%.*]] = mul i32 [[DUMMY24_4]], [[DUMMY24_4]]
-; CHECK-NEXT:    [[DUMMY26_4:%.*]] = mul i32 [[DUMMY25_4]], [[DUMMY25_4]]
-; CHECK-NEXT:    [[DUMMY27_4:%.*]] = mul i32 [[DUMMY26_4]], [[DUMMY26_4]]
-; CHECK-NEXT:    [[DUMMY28_4:%.*]] = mul i32 [[DUMMY27_4]], [[DUMMY27_4]]
-; CHECK-NEXT:    [[DUMMY29_4:%.*]] = mul i32 [[DUMMY28_4]], [[DUMMY28_4]]
-; CHECK-NEXT:    [[DUMMY30_4:%.*]] = mul i32 [[DUMMY29_4]], [[DUMMY29_4]]
-; CHECK-NEXT:    [[DUMMY31_4:%.*]] = mul i32 [[DUMMY30_4]], [[DUMMY30_4]]
-; CHECK-NEXT:    [[DUMMY32_4:%.*]] = mul i32 [[DUMMY31_4]], [[DUMMY31_4]]
-; CHECK-NEXT:    [[DUMMY33_4:%.*]] = mul i32 [[DUMMY32_4]], [[DUMMY32_4]]
-; CHECK-NEXT:    [[DUMMY34_4:%.*]] = mul i32 [[DUMMY33_4]], [[DUMMY33_4]]
-; CHECK-NEXT:    [[DUMMY35_4:%.*]] = mul i32 [[DUMMY34_4]], [[DUMMY34_4]]
-; CHECK-NEXT:    [[DUMMY36_4:%.*]] = mul i32 [[DUMMY35_4]], [[DUMMY35_4]]
-; CHECK-NEXT:    [[DUMMY37_4:%.*]] = mul i32 [[DUMMY36_4]], [[DUMMY36_4]]
-; CHECK-NEXT:    [[DUMMY38_4:%.*]] = mul i32 [[DUMMY37_4]], [[DUMMY37_4]]
-; CHECK-NEXT:    [[DUMMY39_4:%.*]] = mul i32 [[DUMMY38_4]], [[DUMMY38_4]]
-; CHECK-NEXT:    [[DUMMY40_4:%.*]] = mul i32 [[DUMMY39_4]], [[DUMMY39_4]]
-; CHECK-NEXT:    [[DUMMY41_4:%.*]] = mul i32 [[DUMMY40_4]], [[DUMMY40_4]]
-; CHECK-NEXT:    [[DUMMY42_4:%.*]] = mul i32 [[DUMMY41_4]], [[DUMMY41_4]]
-; CHECK-NEXT:    [[DUMMY43_4:%.*]] = mul i32 [[DUMMY42_4]], [[DUMMY42_4]]
-; CHECK-NEXT:    [[DUMMY44_4:%.*]] = mul i32 [[DUMMY43_4]], [[DUMMY43_4]]
-; CHECK-NEXT:    [[DUMMY45_4:%.*]] = mul i32 [[DUMMY44_4]], [[DUMMY44_4]]
-; CHECK-NEXT:    [[DUMMY46_4:%.*]] = mul i32 [[DUMMY45_4]], [[DUMMY45_4]]
-; CHECK-NEXT:    [[DUMMY47_4:%.*]] = mul i32 [[DUMMY46_4]], [[DUMMY46_4]]
-; CHECK-NEXT:    [[DUMMY48_4:%.*]] = mul i32 [[DUMMY47_4]], [[DUMMY47_4]]
-; CHECK-NEXT:    [[DUMMY49_4:%.*]] = mul i32 [[DUMMY48_4]], [[DUMMY48_4]]
-; CHECK-NEXT:    [[DUMMY50_4:%.*]] = mul i32 [[DUMMY49_4]], [[DUMMY49_4]]
-; CHECK-NEXT:    [[SUM_NEXT_4:%.*]] = add nsw i32 [[DUMMY50_4]], [[SUM_NEXT_3]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_4:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 5
-; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_4]]
-; CHECK-NEXT:    [[VAL_5:%.*]] = load i32, ptr [[ARRAYIDX_5]], align 4
-; CHECK-NEXT:    [[DUMMY1_5:%.*]] = mul i32 [[VAL_5]], [[VAL_5]]
-; CHECK-NEXT:    [[DUMMY2_5:%.*]] = mul i32 [[DUMMY1_5]], [[DUMMY1_5]]
-; CHECK-NEXT:    [[DUMMY3_5:%.*]] = mul i32 [[DUMMY2_5]], [[DUMMY2_5]]
-; CHECK-NEXT:    [[DUMMY4_5:%.*]] = mul i32 [[DUMMY3_5]], [[DUMMY3_5]]
-; CHECK-NEXT:    [[DUMMY5_5:%.*]] = mul i32 [[DUMMY4_5]], [[DUMMY4_5]]
-; CHECK-NEXT:    [[DUMMY6_5:%.*]] = mul i32 [[DUMMY5_5]], [[DUMMY5_5]]
-; CHECK-NEXT:    [[DUMMY7_5:%.*]] = mul i32 [[DUMMY6_5]], [[DUMMY6_5]]
-; CHECK-NEXT:    [[DUMMY8_5:%.*]] = mul i32 [[DUMMY7_5]], [[DUMMY7_5]]
-; CHECK-NEXT:    [[DUMMY9_5:%.*]] = mul i32 [[DUMMY8_5]], [[DUMMY8_5]]
-; CHECK-NEXT:    [[DUMMY10_5:%.*]] = mul i32 [[DUMMY9_5]], [[DUMMY9_5]]
-; CHECK-NEXT:    [[DUMMY11_5:%.*]] = mul i32 [[DUMMY10_5]], [[DUMMY10_5]]
-; CHECK-NEXT:    [[DUMMY12_5:%.*]] = mul i32 [[DUMMY11_5]], [[DUMMY11_5]]
-; CHECK-NEXT:    [[DUMMY13_5:%.*]] = mul i32 [[DUMMY12_5]], [[DUMMY12_5]]
-; CHECK-NEXT:    [[DUMMY14_5:%.*]] = mul i32 [[DUMMY13_5]], [[DUMMY13_5]]
-; CHECK-NEXT:    [[DUMMY15_5:%.*]] = mul i32 [[DUMMY14_5]], [[DUMMY14_5]]
-; CHECK-NEXT:    [[DUMMY16_5:%.*]] = mul i32 [[DUMMY15_5]], [[DUMMY15_5]]
-; CHECK-NEXT:    [[DUMMY17_5:%.*]] = mul i32 [[DUMMY16_5]], [[DUMMY16_5]]
-; CHECK-NEXT:    [[DUMMY18_5:%.*]] = mul i32 [[DUMMY17_5]], [[DUMMY17_5]]
-; CHECK-NEXT:    [[DUMMY19_5:%.*]] = mul i32 [[DUMMY18_5]], [[DUMMY18_5]]
-; CHECK-NEXT:    [[DUMMY20_5:%.*]] = mul i32 [[DUMMY19_5]], [[DUMMY19_5]]
-; CHECK-NEXT:    [[DUMMY21_5:%.*]] = mul i32 [[DUMMY20_5]], [[DUMMY20_5]]
-; CHECK-NEXT:    [[DUMMY22_5:%.*]] = mul i32 [[DUMMY21_5]], [[DUMMY21_5]]
-; CHECK-NEXT:    [[DUMMY23_5:%.*]] = mul i32 [[DUMMY22_5]], [[DUMMY22_5]]
-; CHECK-NEXT:    [[DUMMY24_5:%.*]] = mul i32 [[DUMMY23_5]], [[DUMMY23_5]]
-; CHECK-NEXT:    [[DUMMY25_5:%.*]] = mul i32 [[DUMMY24_5]], [[DUMMY24_5]]
-; CHECK-NEXT:    [[DUMMY26_5:%.*]] = mul i32 [[DUMMY25_5]], [[DUMMY25_5]]
-; CHECK-NEXT:    [[DUMMY27_5:%.*]] = mul i32 [[DUMMY26_5]], [[DUMMY26_5]]
-; CHECK-NEXT:    [[DUMMY28_5:%.*]] = mul i32 [[DUMMY27_5]], [[DUMMY27_5]]
-; CHECK-NEXT:    [[DUMMY29_5:%.*]] = mul i32 [[DUMMY28_5]], [[DUMMY28_5]]
-; CHECK-NEXT:    [[DUMMY30_5:%.*]] = mul i32 [[DUMMY29_5]], [[DUMMY29_5]]
-; CHECK-NEXT:    [[DUMMY31_5:%.*]] = mul i32 [[DUMMY30_5]], [[DUMMY30_5]]
-; CHECK-NEXT:    [[DUMMY32_5:%.*]] = mul i32 [[DUMMY31_5]], [[DUMMY31_5]]
-; CHECK-NEXT:    [[DUMMY33_5:%.*]] = mul i32 [[DUMMY32_5]], [[DUMMY32_5]]
-; CHECK-NEXT:    [[DUMMY34_5:%.*]] = mul i32 [[DUMMY33_5]], [[DUMMY33_5]]
-; CHECK-NEXT:    [[DUMMY35_5:%.*]] = mul i32 [[DUMMY34_5]], [[DUMMY34_5]]
-; CHECK-NEXT:    [[DUMMY36_5:%.*]] = mul i32 [[DUMMY35_5]], [[DUMMY35_5]]
-; CHECK-NEXT:    [[DUMMY37_5:%.*]] = mul i32 [[DUMMY36_5]], [[DUMMY36_5]]
-; CHECK-NEXT:    [[DUMMY38_5:%.*]] = mul i32 [[DUMMY37_5]], [[DUMMY37_5]]
-; CHECK-NEXT:    [[DUMMY39_5:%.*]] = mul i32 [[DUMMY38_5]], [[DUMMY38_5]]
-; CHECK-NEXT:    [[DUMMY40_5:%.*]] = mul i32 [[DUMMY39_5]], [[DUMMY39_5]]
-; CHECK-NEXT:    [[DUMMY41_5:%.*]] = mul i32 [[DUMMY40_5]], [[DUMMY40_5]]
-; CHECK-NEXT:    [[DUMMY42_5:%.*]] = mul i32 [[DUMMY41_5]], [[DUMMY41_5]]
-; CHECK-NEXT:    [[DUMMY43_5:%.*]] = mul i32 [[DUMMY42_5]], [[DUMMY42_5]]
-; CHECK-NEXT:    [[DUMMY44_5:%.*]] = mul i32 [[DUMMY43_5]], [[DUMMY43_5]]
-; CHECK-NEXT:    [[DUMMY45_5:%.*]] = mul i32 [[DUMMY44_5]], [[DUMMY44_5]]
-; CHECK-NEXT:    [[DUMMY46_5:%.*]] = mul i32 [[DUMMY45_5]], [[DUMMY45_5]]
-; CHECK-NEXT:    [[DUMMY47_5:%.*]] = mul i32 [[DUMMY46_5]], [[DUMMY46_5]]
-; CHECK-NEXT:    [[DUMMY48_5:%.*]] = mul i32 [[DUMMY47_5]], [[DUMMY47_5]]
-; CHECK-NEXT:    [[DUMMY49_5:%.*]] = mul i32 [[DUMMY48_5]], [[DUMMY48_5]]
-; CHECK-NEXT:    [[DUMMY50_5:%.*]] = mul i32 [[DUMMY49_5]], [[DUMMY49_5]]
-; CHECK-NEXT:    [[SUM_NEXT_5:%.*]] = add nsw i32 [[DUMMY50_5]], [[SUM_NEXT_4]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_5:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 6
-; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_5]]
-; CHECK-NEXT:    [[VAL_6:%.*]] = load i32, ptr [[ARRAYIDX_6]], align 4
-; CHECK-NEXT:    [[DUMMY1_6:%.*]] = mul i32 [[VAL_6]], [[VAL_6]]
-; CHECK-NEXT:    [[DUMMY2_6:%.*]] = mul i32 [[DUMMY1_6]], [[DUMMY1_6]]
-; CHECK-NEXT:    [[DUMMY3_6:%.*]] = mul i32 [[DUMMY2_6]], [[DUMMY2_6]]
-; CHECK-NEXT:    [[DUMMY4_6:%.*]] = mul i32 [[DUMMY3_6]], [[DUMMY3_6]]
-; CHECK-NEXT:    [[DUMMY5_6:%.*]] = mul i32 [[DUMMY4_6]], [[DUMMY4_6]]
-; CHECK-NEXT:    [[DUMMY6_6:%.*]] = mul i32 [[DUMMY5_6]], [[DUMMY5_6]]
-; CHECK-NEXT:    [[DUMMY7_6:%.*]] = mul i32 [[DUMMY6_6]], [[DUMMY6_6]]
-; CHECK-NEXT:    [[DUMMY8_6:%.*]] = mul i32 [[DUMMY7_6]], [[DUMMY7_6]]
-; CHECK-NEXT:    [[DUMMY9_6:%.*]] = mul i32 [[DUMMY8_6]], [[DUMMY8_6]]
-; CHECK-NEXT:    [[DUMMY10_6:%.*]] = mul i32 [[DUMMY9_6]], [[DUMMY9_6]]
-; CHECK-NEXT:    [[DUMMY11_6:%.*]] = mul i32 [[DUMMY10_6]], [[DUMMY10_6]]
-; CHECK-NEXT:    [[DUMMY12_6:%.*]] = mul i32 [[DUMMY11_6]], [[DUMMY11_6]]
-; CHECK-NEXT:    [[DUMMY13_6:%.*]] = mul i32 [[DUMMY12_6]], [[DUMMY12_6]]
-; CHECK-NEXT:    [[DUMMY14_6:%.*]] = mul i32 [[DUMMY13_6]], [[DUMMY13_6]]
-; CHECK-NEXT:    [[DUMMY15_6:%.*]] = mul i32 [[DUMMY14_6]], [[DUMMY14_6]]
-; CHECK-NEXT:    [[DUMMY16_6:%.*]] = mul i32 [[DUMMY15_6]], [[DUMMY15_6]]
-; CHECK-NEXT:    [[DUMMY17_6:%.*]] = mul i32 [[DUMMY16_6]], [[DUMMY16_6]]
-; CHECK-NEXT:    [[DUMMY18_6:%.*]] = mul i32 [[DUMMY17_6]], [[DUMMY17_6]]
-; CHECK-NEXT:    [[DUMMY19_6:%.*]] = mul i32 [[DUMMY18_6]], [[DUMMY18_6]]
-; CHECK-NEXT:    [[DUMMY20_6:%.*]] = mul i32 [[DUMMY19_6]], [[DUMMY19_6]]
-; CHECK-NEXT:    [[DUMMY21_6:%.*]] = mul i32 [[DUMMY20_6]], [[DUMMY20_6]]
-; CHECK-NEXT:    [[DUMMY22_6:%.*]] = mul i32 [[DUMMY21_6]], [[DUMMY21_6]]
-; CHECK-NEXT:    [[DUMMY23_6:%.*]] = mul i32 [[DUMMY22_6]], [[DUMMY22_6]]
-; CHECK-NEXT:    [[DUMMY24_6:%.*]] = mul i32 [[DUMMY23_6]], [[DUMMY23_6]]
-; CHECK-NEXT:    [[DUMMY25_6:%.*]] = mul i32 [[DUMMY24_6]], [[DUMMY24_6]]
-; CHECK-NEXT:    [[DUMMY26_6:%.*]] = mul i32 [[DUMMY25_6]], [[DUMMY25_6]]
-; CHECK-NEXT:    [[DUMMY27_6:%.*]] = mul i32 [[DUMMY26_6]], [[DUMMY26_6]]
-; CHECK-NEXT:    [[DUMMY28_6:%.*]] = mul i32 [[DUMMY27_6]], [[DUMMY27_6]]
-; CHECK-NEXT:    [[DUMMY29_6:%.*]] = mul i32 [[DUMMY28_6]], [[DUMMY28_6]]
-; CHECK-NEXT:    [[DUMMY30_6:%.*]] = mul i32 [[DUMMY29_6]], [[DUMMY29_6]]
-; CHECK-NEXT:    [[DUMMY31_6:%.*]] = mul i32 [[DUMMY30_6]], [[DUMMY30_6]]
-; CHECK-NEXT:    [[DUMMY32_6:%.*]] = mul i32 [[DUMMY31_6]], [[DUMMY31_6]]
-; CHECK-NEXT:    [[DUMMY33_6:%.*]] = mul i32 [[DUMMY32_6]], [[DUMMY32_6]]
-; CHECK-NEXT:    [[DUMMY34_6:%.*]] = mul i32 [[DUMMY33_6]], [[DUMMY33_6]]
-; CHECK-NEXT:    [[DUMMY35_6:%.*]] = mul i32 [[DUMMY34_6]], [[DUMMY34_6]]
-; CHECK-NEXT:    [[DUMMY36_6:%.*]] = mul i32 [[DUMMY35_6]], [[DUMMY35_6]]
-; CHECK-NEXT:    [[DUMMY37_6:%.*]] = mul i32 [[DUMMY36_6]], [[DUMMY36_6]]
-; CHECK-NEXT:    [[DUMMY38_6:%.*]] = mul i32 [[DUMMY37_6]], [[DUMMY37_6]]
-; CHECK-NEXT:    [[DUMMY39_6:%.*]] = mul i32 [[DUMMY38_6]], [[DUMMY38_6]]
-; CHECK-NEXT:    [[DUMMY40_6:%.*]] = mul i32 [[DUMMY39_6]], [[DUMMY39_6]]
-; CHECK-NEXT:    [[DUMMY41_6:%.*]] = mul i32 [[DUMMY40_6]], [[DUMMY40_6]]
-; CHECK-NEXT:    [[DUMMY42_6:%.*]] = mul i32 [[DUMMY41_6]], [[DUMMY41_6]]
-; CHECK-NEXT:    [[DUMMY43_6:%.*]] = mul i32 [[DUMMY42_6]], [[DUMMY42_6]]
-; CHECK-NEXT:    [[DUMMY44_6:%.*]] = mul i32 [[DUMMY43_6]], [[DUMMY43_6]]
-; CHECK-NEXT:    [[DUMMY45_6:%.*]] = mul i32 [[DUMMY44_6]], [[DUMMY44_6]]
-; CHECK-NEXT:    [[DUMMY46_6:%.*]] = mul i32 [[DUMMY45_6]], [[DUMMY45_6]]
-; CHECK-NEXT:    [[DUMMY47_6:%.*]] = mul i32 [[DUMMY46_6]], [[DUMMY46_6]]
-; CHECK-NEXT:    [[DUMMY48_6:%.*]] = mul i32 [[DUMMY47_6]], [[DUMMY47_6]]
-; CHECK-NEXT:    [[DUMMY49_6:%.*]] = mul i32 [[DUMMY48_6]], [[DUMMY48_6]]
-; CHECK-NEXT:    [[DUMMY50_6:%.*]] = mul i32 [[DUMMY49_6]], [[DUMMY49_6]]
-; CHECK-NEXT:    [[SUM_NEXT_6:%.*]] = add nsw i32 [[DUMMY50_6]], [[SUM_NEXT_5]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_6:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 7
-; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_NEXT_6]]
-; CHECK-NEXT:    [[VAL_7:%.*]] = load i32, ptr [[ARRAYIDX_7]], align 4
-; CHECK-NEXT:    [[DUMMY1_7:%.*]] = mul i32 [[VAL_7]], [[VAL_7]]
-; CHECK-NEXT:    [[DUMMY2_7:%.*]] = mul i32 [[DUMMY1_7]], [[DUMMY1_7]]
-; CHECK-NEXT:    [[DUMMY3_7:%.*]] = mul i32 [[DUMMY2_7]], [[DUMMY2_7]]
-; CHECK-NEXT:    [[DUMMY4_7:%.*]] = mul i32 [[DUMMY3_7]], [[DUMMY3_7]]
-; CHECK-NEXT:    [[DUMMY5_7:%.*]] = mul i32 [[DUMMY4_7]], [[DUMMY4_7]]
-; CHECK-NEXT:    [[DUMMY6_7:%.*]] = mul i32 [[DUMMY5_7]], [[DUMMY5_7]]
-; CHECK-NEXT:    [[DUMMY7_7:%.*]] = mul i32 [[DUMMY6_7]], [[DUMMY6_7]]
-; CHECK-NEXT:    [[DUMMY8_7:%.*]] = mul i32 [[DUMMY7_7]], [[DUMMY7_7]]
-; CHECK-NEXT:    [[DUMMY9_7:%.*]] = mul i32 [[DUMMY8_7]], [[DUMMY8_7]]
-; CHECK-NEXT:    [[DUMMY10_7:%.*]] = mul i32 [[DUMMY9_7]], [[DUMMY9_7]]
-; CHECK-NEXT:    [[DUMMY11_7:%.*]] = mul i32 [[DUMMY10_7]], [[DUMMY10_7]]
-; CHECK-NEXT:    [[DUMMY12_7:%.*]] = mul i32 [[DUMMY11_7]], [[DUMMY11_7]]
-; CHECK-NEXT:    [[DUMMY13_7:%.*]] = mul i32 [[DUMMY12_7]], [[DUMMY12_7]]
-; CHECK-NEXT:    [[DUMMY14_7:%.*]] = mul i32 [[DUMMY13_7]], [[DUMMY13_7]]
-; CHECK-NEXT:    [[DUMMY15_7:%.*]] = mul i32 [[DUMMY14_7]], [[DUMMY14_7]]
-; CHECK-NEXT:    [[DUMMY16_7:%.*]] = mul i32 [[DUMMY15_7]], [[DUMMY15_7]]
-; CHECK-NEXT:    [[DUMMY17_7:%.*]] = mul i32 [[DUMMY16_7]], [[DUMMY16_7]]
-; CHECK-NEXT:    [[DUMMY18_7:%.*]] = mul i32 [[DUMMY17_7]], [[DUMMY17_7]]
-; CHECK-NEXT:    [[DUMMY19_7:%.*]] = mul i32 [[DUMMY18_7]], [[DUMMY18_7]]
-; CHECK-NEXT:    [[DUMMY20_7:%.*]] = mul i32 [[DUMMY19_7]], [[DUMMY19_7]]
-; CHECK-NEXT:    [[DUMMY21_7:%.*]] = mul i32 [[DUMMY20_7]], [[DUMMY20_7]]
-; CHECK-NEXT:    [[DUMMY22_7:%.*]] = mul i32 [[DUMMY21_7]], [[DUMMY21_7]]
-; CHECK-NEXT:    [[DUMMY23_7:%.*]] = mul i32 [[DUMMY22_7]], [[DUMMY22_7]]
-; CHECK-NEXT:    [[DUMMY24_7:%.*]] = mul i32 [[DUMMY23_7]], [[DUMMY23_7]]
-; CHECK-NEXT:    [[DUMMY25_7:%.*]] = mul i32 [[DUMMY24_7]], [[DUMMY24_7]]
-; CHECK-NEXT:    [[DUMMY26_7:%.*]] = mul i32 [[DUMMY25_7]], [[DUMMY25_7]]
-; CHECK-NEXT:    [[DUMMY27_7:%.*]] = mul i32 [[DUMMY26_7]], [[DUMMY26_7]]
-; CHECK-NEXT:    [[DUMMY28_7:%.*]] = mul i32 [[DUMMY27_7]], [[DUMMY27_7]]
-; CHECK-NEXT:    [[DUMMY29_7:%.*]] = mul i32 [[DUMMY28_7]], [[DUMMY28_7]]
-; CHECK-NEXT:    [[DUMMY30_7:%.*]] = mul i32 [[DUMMY29_7]], [[DUMMY29_7]]
-; CHECK-NEXT:    [[DUMMY31_7:%.*]] = mul i32 [[DUMMY30_7]], [[DUMMY30_7]]
-; CHECK-NEXT:    [[DUMMY32_7:%.*]] = mul i32 [[DUMMY31_7]], [[DUMMY31_7]]
-; CHECK-NEXT:    [[DUMMY33_7:%.*]] = mul i32 [[DUMMY32_7]], [[DUMMY32_7]]
-; CHECK-NEXT:    [[DUMMY34_7:%.*]] = mul i32 [[DUMMY33_7]], [[DUMMY33_7]]
-; CHECK-NEXT:    [[DUMMY35_7:%.*]] = mul i32 [[DUMMY34_7]], [[DUMMY34_7]]
-; CHECK-NEXT:    [[DUMMY36_7:%.*]] = mul i32 [[DUMMY35_7]], [[DUMMY35_7]]
-; CHECK-NEXT:    [[DUMMY37_7:%.*]] = mul i32 [[DUMMY36_7]], [[DUMMY36_7]]
-; CHECK-NEXT:    [[DUMMY38_7:%.*]] = mul i32 [[DUMMY37_7]], [[DUMMY37_7]]
-; CHECK-NEXT:    [[DUMMY39_7:%.*]] = mul i32 [[DUMMY38_7]], [[DUMMY38_7]]
-; CHECK-NEXT:    [[DUMMY40_7:%.*]] = mul i32 [[DUMMY39_7]], [[DUMMY39_7]]
-; CHECK-NEXT:    [[DUMMY41_7:%.*]] = mul i32 [[DUMMY40_7]], [[DUMMY40_7]]
-; CHECK-NEXT:    [[DUMMY42_7:%.*]] = mul i32 [[DUMMY41_7]], [[DUMMY41_7]]
-; CHECK-NEXT:    [[DUMMY43_7:%.*]] = mul i32 [[DUMMY42_7]], [[DUMMY42_7]]
-; CHECK-NEXT:    [[DUMMY44_7:%.*]] = mul i32 [[DUMMY43_7]], [[DUMMY43_7]]
-; CHECK-NEXT:    [[DUMMY45_7:%.*]] = mul i32 [[DUMMY44_7]], [[DUMMY44_7]]
-; CHECK-NEXT:    [[DUMMY46_7:%.*]] = mul i32 [[DUMMY45_7]], [[DUMMY45_7]]
-; CHECK-NEXT:    [[DUMMY47_7:%.*]] = mul i32 [[DUMMY46_7]], [[DUMMY46_7]]
-; CHECK-NEXT:    [[DUMMY48_7:%.*]] = mul i32 [[DUMMY47_7]], [[DUMMY47_7]]
-; CHECK-NEXT:    [[DUMMY49_7:%.*]] = mul i32 [[DUMMY48_7]], [[DUMMY48_7]]
-; CHECK-NEXT:    [[DUMMY50_7:%.*]] = mul i32 [[DUMMY49_7]], [[DUMMY49_7]]
-; CHECK-NEXT:    [[SUM_NEXT_7]] = add nsw i32 [[DUMMY50_7]], [[SUM_NEXT_6]]
-; CHECK-NEXT:    [[INDVARS_IV_NEXT_7]] = add nuw nsw i64 [[INDVARS_IV]], 8
-; CHECK-NEXT:    [[NITER_NEXT_7]] = add i64 [[NITER]], 8
-; CHECK-NEXT:    [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]]
-; CHECK-NEXT:    br i1 [[NITER_NCMP_7]], label [[FOR_COND_CLEANUP_UNR_LCSSA_LOOPEXIT:%.*]], label [[FOR_BODY]]
-; CHECK:       for.cond.cleanup.unr-lcssa.loopexit:
-; CHECK-NEXT:    [[SUM_NEXT_LCSSA_PH_PH:%.*]] = phi i32 [ [[SUM_NEXT_7]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_7]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_UNR_PH:%.*]] = phi i32 [ [[SUM_NEXT_7]], [[FOR_BODY]] ]
-; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_UNR_LCSSA]]
-; CHECK:       for.cond.cleanup.unr-lcssa:
-; CHECK-NEXT:    [[SUM_NEXT_LCSSA_PH:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[SUM_NEXT_LCSSA_PH_PH]], [[FOR_COND_CLEANUP_UNR_LCSSA_LOOPEXIT]] ]
-; CHECK-NEXT:    [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_UNR_PH]], [[FOR_COND_CLEANUP_UNR_LCSSA_LOOPEXIT]] ]
-; CHECK-NEXT:    [[SUM_UNR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_UNR_PH]], [[FOR_COND_CLEANUP_UNR_LCSSA_LOOPEXIT]] ]
-; CHECK-NEXT:    [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
-; CHECK-NEXT:    br i1 [[LCMP_MOD]], label [[FOR_BODY_EPIL_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK:       for.body.epil.preheader:
-; CHECK-NEXT:    br label [[FOR_BODY_EPIL:%.*]]
-; CHECK:       for.body.epil:
-; CHECK-NEXT:    [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_UNR]], [[FOR_BODY_EPIL_PREHEADER]] ], [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL]] ]
-; CHECK-NEXT:    [[SUM_EPIL:%.*]] = phi i32 [ [[SUM_UNR]], [[FOR_BODY_EPIL_PREHEADER]] ], [ [[SUM_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL]] ]
-; CHECK-NEXT:    [[EPIL_ITER:%.*]] = phi i64 [ 0, [[FOR_BODY_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[FOR_BODY_EPIL]] ]
-; CHECK-NEXT:    [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[ARY]], i64 [[INDVARS_IV_EPIL]]
-; CHECK-NEXT:    [[VAL_EPIL:%.*]] = load i32, ptr [[ARRAYIDX_EPIL]], align 4
-; CHECK-NEXT:    [[DUMMY1_EPIL:%.*]] = mul i32 [[VAL_EPIL]], [[VAL_EPIL]]
-; CHECK-NEXT:    [[DUMMY2_EPIL:%.*]] = mul i32 [[DUMMY1_EPIL]], [[DUMMY1_EPIL]]
-; CHECK-NEXT:    [[DUMMY3_EPIL:%.*]] = mul i32 [[DUMMY2_EPIL]], [[DUMMY2_EPIL]]
-; CHECK-NEXT:    [[DUMMY4_EPIL:%.*]] = mul i32 [[DUMMY3_EPIL]], [[DUMMY3_EPIL]]
-; CHECK-NEXT:    [[DUMMY5_EPIL:%.*]] = mul i32 [[DUMMY4_EPIL]], [[DUMMY4_EPIL]]
-; CHECK-NEXT:    [[DUMMY6_EPIL:%.*]] = mul i32 [[DUMMY5_EPIL]], [[DUMMY5_EPIL]]
-; CHECK-NEXT:    [[DUMMY7_EPIL:%.*]] = mul i32 [[DUMMY6_EPIL]], [[DUMMY6_EPIL]]
-; CHECK-NEXT:    [[DUMMY8_EPIL:%.*]] = mul i32 [[DUMMY7_EPIL]], [[DUMMY7_EPIL]]
-; CHECK-NEXT:    [[DUMMY9_EPIL:%.*]] = mul i32 [[DUMMY8_EPIL]], [[DUMMY8_EPIL]]
-; CHECK-NEXT:    [[DUMMY10_EPIL:%.*]] = mul i32 [[DUMMY9_EPIL]], [[DUMMY9_EPIL]]
-; CHECK-NEXT:    [[DUMMY11_EPIL:%.*]] = mul i32 [[DUMMY10_EPIL]], [[DUMMY10_EPIL]]
-; CHECK-NEXT:    [[DUMMY12_EPIL:%.*]] = mul i32 [[DUMMY11_EPIL]], [[DUMMY11_EPIL]]
-; CHECK-NEXT:    [[DUMMY13_EPIL:%.*]] = mul i32 [[DUMMY12_EPIL]], [[DUMMY12_EPIL]]
-; CHECK-NEXT:    [[DUMMY14_EPIL:%.*]] = mul i32 [[DUMMY13_EPIL]], [[DUMMY13_EPIL]]
-; CHECK-NEXT:    [[DUMMY15_EPIL:%.*]] = mul i32 [[DUMMY14_EPIL]], [[DUMMY14_EPIL]]
-; CHECK-NEXT:    [[DUMMY16_EPIL:%.*]] = mul i32 [[DUMMY15_EPIL]], [[DUMMY15_EPIL]]
-; CHECK-NEXT:    [[DUMMY17_EPIL:%.*]] = mul i32 [[DUMMY16_EPIL]], [[DUMMY16_EPIL]]
-; CHECK-NEXT:    [[DUMMY18_EPIL:%.*]] = mul i32 [[DUMMY17_EPIL]], [[DUMMY17_EPIL]]
-; CHECK-NEXT:    [[DUMMY19_EPIL:%.*]] = mul i32 [[DUMMY18_EPIL]], [[DUMMY18_EPIL]]
-; CHECK-NEXT:    [[DUMMY20_EPIL:%.*]] = mul i32 [[DUMMY19_EPIL]], [[DUMMY19_EPIL]]
-; CHECK-NEXT:    [[DUMMY21_EPIL:%.*]] = mul i32 [[DUMMY20_EPIL]], [[DUMMY20_EPIL]]
-; CHECK-NEXT:    [[DUMMY22_EPIL:%.*]] = mul i32 [[DUMMY21_EPIL]], [[DUMMY21_EPIL]]
-; CHECK-NEXT:    [[DUMMY23_EPIL:%.*]] = mul i32 [[DUMMY22_EPIL]], [[DUMMY22_EPIL]]
-; CHECK-NEXT:    [[DUMMY24_EPIL:%.*]] = mul i32 [[DUMMY23_EPIL]], [[DUMMY23_EPIL]]
-; CHECK-NEXT:    [[DUMMY25_EPIL:%.*]] = mul i32 [[DUMMY24_EPIL]], [[DUMMY24_EPIL]]
-; CHECK-NEXT:    [[DUMMY26_EPIL:%.*]] = mul i32 [[DUMMY25_EPIL]], [[DUMMY25_EPIL]]
-; CHECK-NEXT:    [[DUMMY27_EPIL:%.*]] = mul i32 [[DUMMY26_EPIL]], [[DUMMY26_EPIL]]
-; CHECK-NEXT:    [[DUMMY28_EPIL:%.*]] = mul i32 [[DUMMY27_EPIL]], [[DUMMY27_EPIL]]
-; CHECK-NEXT:    [[DUMMY29_EPIL:%.*]] = mul i32 [[DUMMY28_EPIL]], [[DUMMY28_EPIL]]
-; CHECK-NEXT:    [[DUMMY30_EPIL:%.*]] = mul i32 [[DUMMY29_EPIL]], [[DUMMY29_EPIL]]
-; CHECK-NEXT:    [[DUMMY31_EPIL:%.*]] = mul i32 [[DUMMY30_EPIL]], [[DUMMY30_EPIL]]
-; CHECK-NEXT:    [[DUMMY32_EPIL:%.*]] = mul i32 [[DUMMY31_EPIL]], [[DUMMY31_EPIL]]
-; CHECK-NEXT:    [[DUMMY33_EPIL:%.*]] = mul i32 [[DUMMY32_EPIL]], [[DUMMY32_EPIL]]
-; CHECK-NEXT:    [[DUMMY34_EPIL:%.*]] = mul i32 [[DUMMY33_EPIL]], [[DUMMY33_EPIL]]
-; CHECK-NEXT:    [[DUMMY35_EPIL:%.*]] = mul i32 [[DUMMY34_EPIL]], [[DUMMY34_EPIL]]
-; CHECK-NEXT:    [[DUMMY36_EPIL:%.*]] = mul i32 [[DUMMY35_EPIL]], [[DUMMY35_EPIL]]
-; CHECK-NEXT:    [[DUMMY37_EPIL:%.*]] = mul i32 [[DUMMY36_EPIL]], [[DUMMY36_EPIL]]
-; CHECK-NEXT:    [[DUMMY38_EPIL:%.*]] = mul i32 [[DUMMY37_EPIL]], [[DUMMY37_EPIL]]
-; CHECK-NEXT:    [[DUMMY39_EPIL:%.*]] = mul i32 [[DUMMY38_EPIL]], [[DUMMY38_EPIL]]
-; CHECK-NEXT:    [[DUMMY40_EPIL:%.*]] = mul i32 [[DUMMY39_EPIL]], [[DUMMY39_EPIL]]
-; CHECK-NEXT:    [[DUMMY41_EPIL:%.*]] = mul i32 [[DUMMY40_EPIL]], [[DUMMY40_EPIL]]
-; CHECK-NEXT:    [[DUMMY42_EPIL:%.*]] = mul i32 [[DUMMY41_EPIL]], [[DUMMY41_EPIL]]
-; CHECK-NEXT:    [[DUMMY43_EPIL:%.*]] = mul i32 [[DUMMY42_EPIL]], [[DUMMY42_EPIL]]
-; CHECK-NEXT:    [[DUMMY44_EPIL:%.*]] = mul i32 [[DUMMY43_EPIL]], [[DUMMY43_EPIL]]
-; CHECK-NEXT:    [[DUMMY45_EPIL:%.*]] = mul i32 [[DUMMY44_EPIL]], [[DUMMY44_EPIL]]
-; CHECK-NEXT:    [[DUMMY46_EPIL:%.*]] = mul i32 [[DUMMY45_EPIL]], [[DUMMY45_EPIL]]
-; CHECK-NEXT:    [[DUMMY47_EPIL:%.*]] = mul i32 [[DUMMY46_EPIL]], [[DUMMY46_EPIL]]
-; CHECK-NEXT:    [[DUMMY48_EPIL:%.*]] = mul i32 [[DUMMY47_EPIL]], [[DUMMY47_EPIL]]
-; CHECK-NEXT:    [[DUMMY49_EPIL:%.*]] = mul i32 [[DUMMY48_EPIL]], [[DUMMY48_EPIL]]
-; CHECK-NEXT:    [[DUMMY50_EPIL:%.*]] = mul i32 [[DUMMY49_EPIL]], [[DUMMY49_EPIL]]
+; CHECK-NEXT:    [[DUMMY50_EPIL:%.*]] = mul i32 [[DUMMY49]], [[DUMMY49]]
 ; CHECK-NEXT:    [[SUM_NEXT_EPIL]] = add nsw i32 [[DUMMY50_EPIL]], [[SUM_EPIL]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_EPIL]] = add nuw nsw i64 [[INDVARS_IV_EPIL]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT_EPIL:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_EPIL]], [[N]]
-; CHECK-NEXT:    [[EPIL_ITER_NEXT]] = add i64 [[EPIL_ITER]], 1
-; CHECK-NEXT:    [[EPIL_ITER_CMP:%.*]] = icmp ne i64 [[EPIL_ITER_NEXT]], [[XTRAITER]]
-; CHECK-NEXT:    br i1 [[EPIL_ITER_CMP]], label [[FOR_BODY_EPIL]], label [[FOR_COND_CLEANUP_EPILOG_LCSSA:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK:       for.cond.cleanup.epilog-lcssa:
-; CHECK-NEXT:    [[SUM_NEXT_LCSSA_PH1:%.*]] = phi i32 [ [[SUM_NEXT_EPIL]], [[FOR_BODY_EPIL]] ]
-; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT_EPIL]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT_LCSSA_PH]], [[FOR_COND_CLEANUP_UNR_LCSSA]] ], [ [[SUM_NEXT_LCSSA_PH1]], [[FOR_COND_CLEANUP_EPILOG_LCSSA]] ]
+; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT_EPIL]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_NEXT_LCSSA]]
 ;
 entry:
-- 
GitLab


From 93c02b7dc3bd07d3d62b56cb3299288901205f3a Mon Sep 17 00:00:00 2001
From: Jacek Caban <jacek@codeweavers.com>
Date: Thu, 16 May 2024 15:47:39 +0200
Subject: [PATCH 1016/1206] [CodeGen][ARM64EC] Use MCSymbolRefExpr::VK_None for
 function aliases. (#92100)

---
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 2 +-
 llvm/test/CodeGen/AArch64/arm64ec-symbols.ll  | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 3ce766fc173c..bdc3fc630a4e 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -1170,7 +1170,7 @@ void AArch64AsmPrinter::emitFunctionEntryLabel() {
     auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
       OutStreamer->emitSymbolAttribute(Src, MCSA_WeakAntiDep);
       OutStreamer->emitAssignment(
-          Src, MCSymbolRefExpr::create(Dst, MCSymbolRefExpr::VK_WEAKREF,
+          Src, MCSymbolRefExpr::create(Dst, MCSymbolRefExpr::VK_None,
                                        MMI->getContext()));
     };
 
diff --git a/llvm/test/CodeGen/AArch64/arm64ec-symbols.ll b/llvm/test/CodeGen/AArch64/arm64ec-symbols.ll
index 389c36974bed..b79dd7d61dd6 100644
--- a/llvm/test/CodeGen/AArch64/arm64ec-symbols.ll
+++ b/llvm/test/CodeGen/AArch64/arm64ec-symbols.ll
@@ -10,12 +10,12 @@ define void @caller() nounwind {
 }
 
 ; CHECK:      .weak_anti_dep  caller
-; CHECK-NEXT: .set caller, "#caller"@WEAKREF
+; CHECK-NEXT: .set caller, "#caller"{{$}}
 
 ; CHECK:      .weak_anti_dep  func
-; CHECK-NEXT: .set func, "#func"@WEAKREF
+; CHECK-NEXT: .set func, "#func"{{$}}
 ; CHECK-NEXT: .weak_anti_dep  "#func"
-; CHECK-NEXT: .set "#func", "#func$exit_thunk"@WEAKREF
+; CHECK-NEXT: .set "#func", "#func$exit_thunk"{{$}}
 
 ; SYM:       [ 8](sec  4)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #caller
 ; SYM:       [21](sec  7)(fl 0x00)(ty  20)(scl   2) (nx 0) 0x00000000 #func$exit_thunk
-- 
GitLab


From c675a58edec6d1a876a0d0e7d261f74764855b38 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Thu, 16 May 2024 09:48:42 -0400
Subject: [PATCH 1017/1206] [TableGen][SubtargetEmitter] Early exit from loop
 in FindWriteResources and FindReadAdvance (#92202)

This gives us a 30% speed improvement in our downstream.
---
 llvm/utils/TableGen/SubtargetEmitter.cpp | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index 9e32d2de19b2..323470940fec 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -902,14 +902,19 @@ SubtargetEmitter::FindWriteResources(const CodeGenSchedRW &SchedWrite,
   for (Record *WR : ProcModel.WriteResDefs) {
     if (!WR->isSubClassOf("WriteRes"))
       continue;
-    if (AliasDef == WR->getValueAsDef("WriteType") ||
-        SchedWrite.TheDef == WR->getValueAsDef("WriteType")) {
+    Record *WRDef = WR->getValueAsDef("WriteType");
+    if (AliasDef == WRDef || SchedWrite.TheDef == WRDef) {
       if (ResDef) {
         PrintFatalError(WR->getLoc(), "Resources are defined for both "
                                       "SchedWrite and its alias on processor " +
                                           ProcModel.ModelName);
       }
       ResDef = WR;
+      // If there is no AliasDef and we find a match, we can early exit since
+      // there is no need to verify whether there are resources defined for both
+      // SchedWrite and its alias.
+      if (!AliasDef)
+        break;
     }
   }
   // TODO: If ProcModel has a base model (previous generation processor),
@@ -956,14 +961,19 @@ Record *SubtargetEmitter::FindReadAdvance(const CodeGenSchedRW &SchedRead,
   for (Record *RA : ProcModel.ReadAdvanceDefs) {
     if (!RA->isSubClassOf("ReadAdvance"))
       continue;
-    if (AliasDef == RA->getValueAsDef("ReadType") ||
-        SchedRead.TheDef == RA->getValueAsDef("ReadType")) {
+    Record *RADef = RA->getValueAsDef("ReadType");
+    if (AliasDef == RADef || SchedRead.TheDef == RADef) {
       if (ResDef) {
         PrintFatalError(RA->getLoc(), "Resources are defined for both "
                                       "SchedRead and its alias on processor " +
                                           ProcModel.ModelName);
       }
       ResDef = RA;
+      // If there is no AliasDef and we find a match, we can early exit since
+      // there is no need to verify whether there are resources defined for both
+      // SchedRead and its alias.
+      if (!AliasDef)
+        break;
     }
   }
   // TODO: If ProcModel has a base model (previous generation processor),
-- 
GitLab


From 6c7ec6e1e6646fb334064bda6f301fdb52390d9b Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Thu, 16 May 2024 15:26:11 +0100
Subject: [PATCH 1018/1206] [mlir][ArmSME] Remove empty line in test (NFC)
 (#92404)

---
 .../ArmSME/tile-allocation-spills-with-mixed-tile-types.mlir     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mlir/test/Dialect/ArmSME/tile-allocation-spills-with-mixed-tile-types.mlir b/mlir/test/Dialect/ArmSME/tile-allocation-spills-with-mixed-tile-types.mlir
index 27757e29c1e2..752e6b61b083 100644
--- a/mlir/test/Dialect/ArmSME/tile-allocation-spills-with-mixed-tile-types.mlir
+++ b/mlir/test/Dialect/ArmSME/tile-allocation-spills-with-mixed-tile-types.mlir
@@ -1,4 +1,3 @@
-
 // RUN: mlir-opt %s -test-arm-sme-tile-allocation -split-input-file | FileCheck %s
 
 // CHECK-LABEL: @always_spill_larger_or_equal_tile_type
-- 
GitLab


From 74a87548e5b62881108e6cd1fd63b45580fc3097 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Thu, 16 May 2024 15:27:59 +0100
Subject: [PATCH 1019/1206] [flang][MLIR][OpenMP] make reduction by-ref toggled
 per variable (#92244)

Fixes #88935

Toggling reduction by-ref broke when multiple reduction clauses were
used. Decisions made for the by-ref status for later clauses could then
invalidate decisions for earlier clauses. For example,

```
reduction(+:scalar,scalar2) reduction(+:array)
```

The first clause would choose by value reduction and generate by-value
reduction regions, but then after this the second clause would force
by-ref to support the array argument. But by the time the second clause
is processed, the first clause has already had the wrong kind of
reduction regions generated.

This is solved by toggling whether a variable should be reduced by
reference per variable. In the above example, this allows only `array`
to be reduced by ref.
---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  12 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |   6 -
 flang/lib/Lower/OpenMP/ReductionProcessor.cpp |  29 ++--
 flang/lib/Lower/OpenMP/ReductionProcessor.h   |   4 +-
 .../test/Fir/omp-reduction-embox-codegen.fir  |   2 +-
 .../Lower/OpenMP/default-clause-byref.f90     |   2 +-
 .../delayed-privatization-reduction-byref.f90 |   2 +-
 .../OpenMP/parallel-reduction-add-byref.f90   |   6 +-
 .../parallel-reduction-allocatable-array.f90  |   2 +-
 .../OpenMP/parallel-reduction-array-lb.f90    |   2 +-
 .../Lower/OpenMP/parallel-reduction-array.f90 |   2 +-
 .../OpenMP/parallel-reduction-array2.f90      |   2 +-
 .../Lower/OpenMP/parallel-reduction-byref.f90 |   2 +-
 .../test/Lower/OpenMP/parallel-reduction3.f90 |   2 +-
 .../parallel-wsloop-reduction-byref.f90       |   2 +-
 .../OpenMP/wsloop-reduction-add-byref.f90     |  14 +-
 .../wsloop-reduction-add-hlfir-byref.f90      |   2 +-
 .../OpenMP/wsloop-reduction-allocatable.f90   |   2 +-
 .../wsloop-reduction-array-assumed-shape.f90  |   2 +-
 .../Lower/OpenMP/wsloop-reduction-array.f90   |   2 +-
 .../Lower/OpenMP/wsloop-reduction-array2.f90  |   2 +-
 .../OpenMP/wsloop-reduction-iand-byref.f90    |   2 +-
 .../OpenMP/wsloop-reduction-ieor-byref.f90    |   2 +-
 .../OpenMP/wsloop-reduction-ior-byref.f90     |   2 +-
 .../wsloop-reduction-logical-and-byref.f90    |   6 +-
 .../wsloop-reduction-logical-eqv-byref.f90    |   6 +-
 .../wsloop-reduction-logical-neqv-byref.f90   |   6 +-
 .../wsloop-reduction-logical-or-byref.f90     |   6 +-
 .../OpenMP/wsloop-reduction-max-2-byref.f90   |   2 +-
 .../OpenMP/wsloop-reduction-max-byref.f90     |   6 +-
 .../wsloop-reduction-max-hlfir-byref.f90      |   2 +-
 .../OpenMP/wsloop-reduction-min-byref.f90     |   6 +-
 .../OpenMP/wsloop-reduction-mul-byref.f90     |  14 +-
 .../wsloop-reduction-multiple-clauses.f90     | 164 ++++++++++++++++++
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       |   2 +-
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     |  19 +-
 .../Frontend/OpenMPIRBuilderTest.cpp          |  12 +-
 .../Dialect/OpenMP/OpenMPClauseOperands.h     |   2 +-
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  59 +++----
 .../Conversion/SCFToOpenMP/SCFToOpenMP.cpp    |   7 +
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 126 +++++++++-----
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |  43 +++--
 mlir/test/Dialect/OpenMP/invalid.mlir         |  19 ++
 mlir/test/Dialect/OpenMP/ops.mlir             |  38 ++++
 .../openmp-parallel-reduction-cleanup.mlir    |   2 +-
 .../Target/LLVMIR/openmp-reduction-byref.mlir |   2 +-
 .../LLVMIR/openmp-reduction-init-arg.mlir     |   2 +-
 .../openmp-wsloop-reduction-cleanup.mlir      |   2 +-
 48 files changed, 464 insertions(+), 196 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 40d66460a698..875599098b3d 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -917,20 +917,18 @@ bool ClauseProcessor::processReduction(
     llvm::SmallVectorImpl<const semantics::Symbol *> *outReductionSyms) const {
   return findRepeatableClause<omp::clause::Reduction>(
       [&](const omp::clause::Reduction &clause, const parser::CharBlock &) {
-        // Use local lists of reductions to prevent variables from other
-        // already-processed reduction clauses from impacting this reduction.
-        // For example, the whole `reductionVars` array is queried to decide
-        // whether to do the reduction byref.
         llvm::SmallVector<mlir::Value> reductionVars;
+        llvm::SmallVector<bool> reduceVarByRef;
         llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
         llvm::SmallVector<const semantics::Symbol *> reductionSyms;
         ReductionProcessor rp;
-        rp.addDeclareReduction(currentLocation, converter, clause,
-                               reductionVars, reductionDeclSymbols,
-                               outReductionSyms ? &reductionSyms : nullptr);
+        rp.addDeclareReduction(
+            currentLocation, converter, clause, reductionVars, reduceVarByRef,
+            reductionDeclSymbols, outReductionSyms ? &reductionSyms : nullptr);
 
         // Copy local lists into the output.
         llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
+        llvm::copy(reduceVarByRef, std::back_inserter(result.reduceVarByRef));
         llvm::copy(reductionDeclSymbols,
                    std::back_inserter(result.reductionDeclSymbols));
 
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index af27c59eb993..4ae1fe6d9902 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -984,8 +984,6 @@ static void genParallelClauses(
 
   if (processReduction) {
     cp.processReduction(loc, clauseOps, &reductionTypes, &reductionSyms);
-    if (ReductionProcessor::doReductionByRef(clauseOps.reductionVars))
-      clauseOps.reductionByRefAttr = converter.getFirOpBuilder().getUnitAttr();
   }
 }
 
@@ -1173,7 +1171,6 @@ static void genWsloopClauses(
     mlir::Location loc, mlir::omp::WsloopClauseOps &clauseOps,
     llvm::SmallVectorImpl<mlir::Type> &reductionTypes,
     llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processNowait(clauseOps);
   cp.processOrdered(clauseOps);
@@ -1181,9 +1178,6 @@ static void genWsloopClauses(
   cp.processSchedule(stmtCtx, clauseOps);
   // TODO Support delayed privatization.
 
-  if (ReductionProcessor::doReductionByRef(clauseOps.reductionVars))
-    clauseOps.reductionByRefAttr = firOpBuilder.getUnitAttr();
-
   cp.processTODO<clause::Allocate, clause::Linear, clause::Order>(
       loc, llvm::omp::Directive::OMPD_do);
 }
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index be6dcd6f5569..1a63e316ef06 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -657,25 +657,17 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction(
   return decl;
 }
 
-// TODO: By-ref vs by-val reductions are currently toggled for the whole
-//       operation (possibly effecting multiple reduction variables).
-//       This could cause a problem with openmp target reductions because
-//       by-ref trivial types may not be supported.
-bool ReductionProcessor::doReductionByRef(
-    const llvm::SmallVectorImpl<mlir::Value> &reductionVars) {
-  if (reductionVars.empty())
-    return false;
+static bool doReductionByRef(mlir::Value reductionVar) {
   if (forceByrefReduction)
     return true;
 
-  for (mlir::Value reductionVar : reductionVars) {
-    if (auto declare =
-            mlir::dyn_cast<hlfir::DeclareOp>(reductionVar.getDefiningOp()))
-      reductionVar = declare.getMemref();
+  if (auto declare =
+          mlir::dyn_cast<hlfir::DeclareOp>(reductionVar.getDefiningOp()))
+    reductionVar = declare.getMemref();
+
+  if (!fir::isa_trivial(fir::unwrapRefType(reductionVar.getType())))
+    return true;
 
-    if (!fir::isa_trivial(fir::unwrapRefType(reductionVar.getType())))
-      return true;
-  }
   return false;
 }
 
@@ -683,6 +675,7 @@ void ReductionProcessor::addDeclareReduction(
     mlir::Location currentLocation, lower::AbstractConverter &converter,
     const omp::clause::Reduction &reduction,
     llvm::SmallVectorImpl<mlir::Value> &reductionVars,
+    llvm::SmallVectorImpl<bool> &reduceVarByRef,
     llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
     llvm::SmallVectorImpl<const semantics::Symbol *> *reductionSymbols) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
@@ -761,8 +754,8 @@ void ReductionProcessor::addDeclareReduction(
            "reduction input var is a reference");
 
     reductionVars.push_back(symVal);
+    reduceVarByRef.push_back(doReductionByRef(symVal));
   }
-  const bool isByRef = doReductionByRef(reductionVars);
 
   if (const auto &redDefinedOp =
           std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
@@ -784,7 +777,7 @@ void ReductionProcessor::addDeclareReduction(
       break;
     }
 
-    for (mlir::Value symVal : reductionVars) {
+    for (auto [symVal, isByRef] : llvm::zip(reductionVars, reduceVarByRef)) {
       auto redType = mlir::cast<fir::ReferenceType>(symVal.getType());
       const auto &kindMap = firOpBuilder.getKindMap();
       if (mlir::isa<fir::LogicalType>(redType.getEleTy()))
@@ -808,7 +801,7 @@ void ReductionProcessor::addDeclareReduction(
             *reductionIntrinsic)) {
       ReductionProcessor::ReductionIdentifier redId =
           ReductionProcessor::getReductionType(*reductionIntrinsic);
-      for (mlir::Value symVal : reductionVars) {
+      for (auto [symVal, isByRef] : llvm::zip(reductionVars, reduceVarByRef)) {
         auto redType = mlir::cast<fir::ReferenceType>(symVal.getType());
         if (!redType.getEleTy().isIntOrIndexOrFloat())
           TODO(currentLocation,
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.h b/flang/lib/Lower/OpenMP/ReductionProcessor.h
index 2852abad0036..0ed5782e5da1 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.h
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.h
@@ -73,9 +73,6 @@ public:
   static const semantics::SourceName
   getRealName(const omp::clause::ProcedureDesignator &pd);
 
-  static bool
-  doReductionByRef(const llvm::SmallVectorImpl<mlir::Value> &reductionVars);
-
   static std::string getReductionName(llvm::StringRef name,
                                       const fir::KindMapping &kindMap,
                                       mlir::Type ty, bool isByRef);
@@ -127,6 +124,7 @@ public:
       mlir::Location currentLocation, lower::AbstractConverter &converter,
       const omp::clause::Reduction &reduction,
       llvm::SmallVectorImpl<mlir::Value> &reductionVars,
+      llvm::SmallVectorImpl<bool> &reduceVarByRef,
       llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
       llvm::SmallVectorImpl<const semantics::Symbol *> *reductionSymbols =
           nullptr);
diff --git a/flang/test/Fir/omp-reduction-embox-codegen.fir b/flang/test/Fir/omp-reduction-embox-codegen.fir
index 7602012ebc5c..1645e1a407ad 100644
--- a/flang/test/Fir/omp-reduction-embox-codegen.fir
+++ b/flang/test/Fir/omp-reduction-embox-codegen.fir
@@ -25,7 +25,7 @@ omp.declare_reduction @test_reduction : !fir.ref<!fir.box<i32>> init {
 
 func.func @_QQmain() attributes {fir.bindc_name = "reduce"} {
   %4 = fir.alloca !fir.box<i32>
-  omp.parallel byref reduction(@test_reduction %4 -> %arg0 : !fir.ref<!fir.box<i32>>) {
+  omp.parallel reduction(byref @test_reduction %4 -> %arg0 : !fir.ref<!fir.box<i32>>) {
     omp.terminator
   }
   return
diff --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90
index 7cc2bc2e0c71..7893c4d7d573 100644
--- a/flang/test/Lower/OpenMP/default-clause-byref.f90
+++ b/flang/test/Lower/OpenMP/default-clause-byref.f90
@@ -351,7 +351,7 @@ subroutine skipped_default_clause_checks()
        type(it)::iii
 
 !CHECK: omp.parallel {
-!CHECK: omp.wsloop byref reduction(@min_byref_i32 %[[VAL_Z_DECLARE]]#0 -> %[[PRV:.+]] : !fir.ref<i32>) {
+!CHECK: omp.wsloop reduction(byref @min_byref_i32 %[[VAL_Z_DECLARE]]#0 -> %[[PRV:.+]] : !fir.ref<i32>) {
 !CHECK-NEXT: omp.loop_nest (%[[ARG:.*]]) {{.*}} {
 !CHECK: omp.yield
 !CHECK: }
diff --git a/flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90 b/flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90
index 49d1142ea4b6..72e91680a431 100644
--- a/flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90
+++ b/flang/test/Lower/OpenMP/delayed-privatization-reduction-byref.f90
@@ -26,5 +26,5 @@ end subroutine
 
 ! CHECK-LABEL: _QPred_and_delayed_private
 ! CHECK: omp.parallel
-! CHECK-SAME: reduction(@[[REDUCTION_SYM]] %{{.*}} -> %arg0 : !fir.ref<i32>)
+! CHECK-SAME: reduction(byref @[[REDUCTION_SYM]] %{{.*}} -> %arg0 : !fir.ref<i32>)
 ! CHECK-SAME: private(@[[PRIVATIZER_SYM]] %{{.*}} -> %arg1 : !fir.ref<i32>) {
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-add-byref.f90 b/flang/test/Lower/OpenMP/parallel-reduction-add-byref.f90
index 2a1d26407b27..7347d9324fea 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-add-byref.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-add-byref.f90
@@ -40,7 +40,7 @@
 !CHECK:  %[[I_DECL:.*]]:2 = hlfir.declare %[[IREF]] {uniq_name = "_QFsimple_int_addEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:  %[[I_START:.*]] = arith.constant 0 : i32
 !CHECK:  hlfir.assign %[[I_START]] to %[[I_DECL]]#0 : i32, !fir.ref<i32>
-!CHECK:  omp.parallel byref reduction(@[[RED_I32_NAME]] %[[I_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<i32>) {
+!CHECK:  omp.parallel reduction(byref @[[RED_I32_NAME]] %[[I_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<i32>) {
 !CHECK:    %[[P_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[LPRV:.+]] = fir.load %[[P_DECL]]#0 : !fir.ref<i32>
 !CHECK:    %[[I_INCR:.*]] = arith.constant 1 : i32
@@ -65,7 +65,7 @@ end subroutine
 !CHECK:  %[[R_DECL:.*]]:2 = hlfir.declare %[[RREF]] {uniq_name = "_QFsimple_real_addEr"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:  %[[R_START:.*]] = arith.constant 0.000000e+00 : f32
 !CHECK:  hlfir.assign %[[R_START]] to %[[R_DECL]]#0 : f32, !fir.ref<f32>
-!CHECK:  omp.parallel byref reduction(@[[RED_F32_NAME]] %[[R_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<f32>) {
+!CHECK:  omp.parallel reduction(byref @[[RED_F32_NAME]] %[[R_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<f32>) {
 !CHECK:    %[[P_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:    %[[LPRV:.+]] = fir.load %[[P_DECL]]#0 : !fir.ref<f32>
 !CHECK:    %[[R_INCR:.*]] = arith.constant 1.500000e+00 : f32
@@ -94,7 +94,7 @@ end subroutine
 !CHECK:  hlfir.assign %[[R_START]] to %[[R_DECL]]#0 : f32, !fir.ref<f32>
 !CHECK:  %[[I_START:.*]] = arith.constant 0 : i32
 !CHECK:  hlfir.assign %[[I_START]] to %[[I_DECL]]#0 : i32, !fir.ref<i32>
-!CHECK:  omp.parallel byref reduction(@[[RED_I32_NAME]] %[[I_DECL]]#0 -> %[[IPRV:.+]] : !fir.ref<i32>, @[[RED_F32_NAME]] %[[R_DECL]]#0 -> %[[RPRV:.+]] : !fir.ref<f32>) {
+!CHECK:  omp.parallel reduction(byref @[[RED_I32_NAME]] %[[I_DECL]]#0 -> %[[IPRV:.+]] : !fir.ref<i32>, byref @[[RED_F32_NAME]] %[[R_DECL]]#0 -> %[[RPRV:.+]] : !fir.ref<f32>) {
 !CHECK:    %[[IP_DECL:.+]]:2 = hlfir.declare %[[IPRV]] {{.*}} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[RP_DECL:.+]]:2 = hlfir.declare %[[RPRV]] {{.*}} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:    %[[R_INCR:.*]] = arith.constant 1.500000e+00 : f32
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
index 28216ef91c3a..f6d3b0b73f73 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
@@ -95,7 +95,7 @@ end program
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_box_heap_Uxi32 %[[VAL_3]]#0 -> %[[VAL_17:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_heap_Uxi32 %[[VAL_3]]#0 -> %[[VAL_17:.*]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_18:.*]]) : i32 = (%[[VAL_14]]) to (%[[VAL_15]]) inclusive step (%[[VAL_16]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_17]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEr"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:                 fir.store %[[VAL_18]] to %[[VAL_13]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
index 8202e6d89715..b44fe4c1f4cc 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array-lb.f90
@@ -77,7 +77,7 @@ end program
 ! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_5]]) {uniq_name = "_QFEi"} : (!fir.ref<!fir.array<3x2xi32>>, !fir.shapeshift<2>) -> (!fir.box<!fir.array<3x2xi32>>, !fir.ref<!fir.array<3x2xi32>>)
 ! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.array<3x2xi32>>
 ! CHECK:           fir.store %[[VAL_6]]#0 to %[[VAL_7]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>
-! CHECK:           omp.parallel byref reduction(@add_reduction_byref_box_3x2xi32 %[[VAL_7]] -> %[[VAL_8:.*]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>) {
+! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3x2xi32 %[[VAL_7]] -> %[[VAL_8:.*]] : !fir.ref<!fir.box<!fir.array<3x2xi32>>>) {
 ! CHECK:             %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3x2xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3x2xi32>>>, !fir.ref<!fir.box<!fir.array<3x2xi32>>>)
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 3 : i32
 ! CHECK:             %[[VAL_11:.*]] = fir.load %[[VAL_9]]#0 : !fir.ref<!fir.box<!fir.array<3x2xi32>>>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
index 34f4ee0a9eb3..60b21c9b1ebb 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90
@@ -70,7 +70,7 @@ end program
 ! CHECK:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
 ! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
 ! CHECK:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.parallel byref reduction(@add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
+! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
 ! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
index aa14092554ed..5d4c86d1d76e 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90
@@ -69,7 +69,7 @@ end program
 ! CHECK:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
 ! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.array<3xi32>>
 ! CHECK:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.array<3xi32>>>
-! CHECK:           omp.parallel byref reduction(@add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
+! CHECK:           omp.parallel reduction(byref @add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref<!fir.box<!fir.array<3xi32>>>) {
 ! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<!fir.box<!fir.array<3xi32>>>) -> (!fir.ref<!fir.box<!fir.array<3xi32>>>, !fir.ref<!fir.box<!fir.array<3xi32>>>)
 ! CHECK:             %[[VAL_8:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<!fir.box<!fir.array<3xi32>>>
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : index
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-byref.f90 b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
index fdcdf0c0b8d9..5685e2c584ac 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-byref.f90
@@ -21,7 +21,7 @@
 !CHECK:    %[[RED_ACCUM_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[RED_ACCUM_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[C0:[_a-z0-9]+]] = arith.constant 0 : i32
 !CHECK:    hlfir.assign %[[C0]] to %[[RED_ACCUM_DECL]]#0 : i32, !fir.ref<i32>
-!CHECK:    omp.parallel byref reduction(@[[REDUCTION_DECLARE]] %[[RED_ACCUM_DECL]]#0 -> %[[PRIVATE_RED:[a-z0-9]+]] : !fir.ref<i32>) {
+!CHECK:    omp.parallel reduction(byref @[[REDUCTION_DECLARE]] %[[RED_ACCUM_DECL]]#0 -> %[[PRIVATE_RED:[a-z0-9]+]] : !fir.ref<i32>) {
 !CHECK:      %[[PRIVATE_DECL:[_a-z0-9]+]]:2 = hlfir.declare %[[PRIVATE_RED]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:      %[[C1:[_a-z0-9]+]] = arith.constant 1 : i32
 !CHECK:      hlfir.assign %[[C1]] to %[[PRIVATE_DECL]]#0 : i32, !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/parallel-reduction3.f90 b/flang/test/Lower/OpenMP/parallel-reduction3.f90
index 17d805c0d142..47b743a558b4 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction3.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction3.f90
@@ -74,7 +74,7 @@
 ! CHECK:             %[[VAL_18:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_19:.*]] = fir.alloca !fir.box<!fir.array<?xi32>>
 ! CHECK:             fir.store %[[VAL_12]]#0 to %[[VAL_19]] : !fir.ref<!fir.box<!fir.array<?xi32>>>
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_box_Uxi32 %[[VAL_19]] -> %[[VAL_20:.*]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_Uxi32 %[[VAL_19]] -> %[[VAL_20:.*]] : !fir.ref<!fir.box<!fir.array<?xi32>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_21:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) {
 ! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFsEc"} : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> (!fir.ref<!fir.box<!fir.array<?xi32>>>, !fir.ref<!fir.box<!fir.array<?xi32>>>)
 ! CHECK:                 fir.store %[[VAL_21]] to %[[VAL_15]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90
index 66c80c31917b..32caac39778d 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90
@@ -4,7 +4,7 @@
 ! RUN: flang-new -fc1 -fopenmp -mmlir --force-byref-reduction -emit-hlfir %s -o - | FileCheck %s
 
 ! CHECK: omp.parallel {
-! CHECK: omp.wsloop byref reduction(@add_reduction_byref_i32
+! CHECK: omp.wsloop reduction(byref @add_reduction_byref_i32
 subroutine sb
   integer :: x
   x = 0
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
index c9d03435d9e1..cc4d055b3ea1 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
@@ -82,7 +82,7 @@
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
@@ -125,7 +125,7 @@ end subroutine
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
@@ -169,7 +169,7 @@ end subroutine
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
@@ -211,7 +211,7 @@ end subroutine
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
@@ -262,7 +262,7 @@ end subroutine
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref<i32>, @add_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref<i32>, @add_reduction_byref_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref<i32>, byref @add_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref<i32>, byref @add_reduction_byref_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -326,7 +326,7 @@ end subroutine
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref<f32>, @add_reduction_byref_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref<f32>, @add_reduction_byref_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref<f32>, byref @add_reduction_byref_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref<f32>, byref @add_reduction_byref_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
@@ -397,7 +397,7 @@ end subroutine
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_18:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref<i32>, @add_reduction_byref_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref<i64>, @add_reduction_byref_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref<f32>, @add_reduction_byref_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<f64>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref<i32>, byref @add_reduction_byref_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref<i64>, byref @add_reduction_byref_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref<f32>, byref @add_reduction_byref_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<f64>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) {
 ! CHECK:                 %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90
index 6a09fece80ae..ec9abba41db0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90
@@ -31,7 +31,7 @@
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
index 8310aeb23df0..ccf186cf9908 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
@@ -80,7 +80,7 @@ end program
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_box_heap_i32 %[[VAL_5]]#0 -> %[[VAL_14:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_heap_i32 %[[VAL_5]]#0 -> %[[VAL_14:.*]] : !fir.ref<!fir.box<!fir.heap<i32>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_15:.*]]) : i32 = (%[[VAL_11]]) to (%[[VAL_12]]) inclusive step (%[[VAL_13]]) {
 ! CHECK:                 %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_14]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEr"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
 ! CHECK:                 fir.store %[[VAL_15]] to %[[VAL_10]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
index 197800486c39..be1e3ec0cb9d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
@@ -84,7 +84,7 @@ end program
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_9:.*]] = fir.alloca !fir.box<!fir.array<?xf64>>
 ! CHECK:             fir.store %[[VAL_3]]#1 to %[[VAL_9]] : !fir.ref<!fir.box<!fir.array<?xf64>>>
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_box_Uxf64 %[[VAL_9]] -> %[[VAL_10:.*]] : !fir.ref<!fir.box<!fir.array<?xf64>>>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_Uxf64 %[[VAL_9]] -> %[[VAL_10:.*]] : !fir.ref<!fir.box<!fir.array<?xf64>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_6]]) to (%[[VAL_7]]) inclusive step (%[[VAL_8]]) {
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {fortran_attrs = {{.*}}, uniq_name = "_QFFreduceEr"} : (!fir.ref<!fir.box<!fir.array<?xf64>>>) -> (!fir.ref<!fir.box<!fir.array<?xf64>>>, !fir.ref<!fir.box<!fir.array<?xf64>>>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_5]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
index 7aecf870cf80..add422f5d956 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
@@ -79,7 +79,7 @@ end program
 ! CHECK:             %[[VAL_11:.*]] = fir.embox %[[VAL_5]]#0(%[[VAL_4]]) : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xi32>>
 ! CHECK:             %[[VAL_12:.*]] = fir.alloca !fir.box<!fir.array<2xi32>>
 ! CHECK:             fir.store %[[VAL_11]] to %[[VAL_12]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_box_2xi32 %[[VAL_12]] -> %[[VAL_13:.*]] : !fir.ref<!fir.box<!fir.array<2xi32>>>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_2xi32 %[[VAL_12]] -> %[[VAL_13:.*]] : !fir.ref<!fir.box<!fir.array<2xi32>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) {
 ! CHECK:                 %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_13]] {uniq_name = "_QFEr"} : (!fir.ref<!fir.box<!fir.array<2xi32>>>) -> (!fir.ref<!fir.box<!fir.array<2xi32>>>, !fir.ref<!fir.box<!fir.array<2xi32>>>)
 ! CHECK:                 fir.store %[[VAL_14]] to %[[VAL_7]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
index d1d8a2c59944..e988567fc337 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
@@ -79,7 +79,7 @@ end program
 ! CHECK:             %[[VAL_11:.*]] = fir.embox %[[VAL_5]]#0(%[[VAL_4]]) : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xi32>>
 ! CHECK:             %[[VAL_12:.*]] = fir.alloca !fir.box<!fir.array<2xi32>>
 ! CHECK:             fir.store %[[VAL_11]] to %[[VAL_12]] : !fir.ref<!fir.box<!fir.array<2xi32>>>
-! CHECK:             omp.wsloop byref reduction(@add_reduction_byref_box_2xi32 %[[VAL_12]] -> %[[VAL_13:.*]] : !fir.ref<!fir.box<!fir.array<2xi32>>>) {
+! CHECK:             omp.wsloop reduction(byref @add_reduction_byref_box_2xi32 %[[VAL_12]] -> %[[VAL_13:.*]] : !fir.ref<!fir.box<!fir.array<2xi32>>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) {
 ! CHECK:                 %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_13]] {uniq_name = "_QFEr"} : (!fir.ref<!fir.box<!fir.array<2xi32>>>) -> (!fir.ref<!fir.box<!fir.array<2xi32>>>, !fir.ref<!fir.box<!fir.array<2xi32>>>)
 ! CHECK:                 fir.store %[[VAL_14]] to %[[VAL_7]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
index df07a9065331..bcc71415f1dd 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
@@ -35,7 +35,7 @@
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@iand_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @iand_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iandEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
index 50cec61b602b..6825800b2cd8 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
@@ -28,7 +28,7 @@
 !CHECK: omp.parallel
 !CHECK: %[[I_REF:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
 !CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFreduction_ieorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: omp.wsloop byref reduction(@ieor_byref_i32 %[[X_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<i32>)
+!CHECK: omp.wsloop reduction(byref @ieor_byref_i32 %[[X_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<i32>)
 !CHECK-NEXT: omp.loop_nest
 !CHECK: %[[PRV_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: fir.store %{{.*}} to %[[I_DECL]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
index d847bba89782..00f4552917aa 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
@@ -33,7 +33,7 @@
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@ior_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @ior_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iorEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
index 69789e4c751e..331232a08e62 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
@@ -92,7 +92,7 @@ end subroutine simple_reduction
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
@@ -151,7 +151,7 @@ end subroutine
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref<!fir.logical<4>>, @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref<!fir.logical<4>>, @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref<!fir.logical<4>>, byref @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref<!fir.logical<4>>, byref @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
index 54175994ecd8..d1cb7b05f0af 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
@@ -91,7 +91,7 @@ end subroutine
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
@@ -150,7 +150,7 @@ end subroutine
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref<!fir.logical<4>>, @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref<!fir.logical<4>>, @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref<!fir.logical<4>>, byref @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref<!fir.logical<4>>, byref @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
index c0a82476c7b1..64c056c9c6bd 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
@@ -92,7 +92,7 @@ end subroutine
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
@@ -153,7 +153,7 @@ end subroutine
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref<!fir.logical<4>>, @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref<!fir.logical<4>>, @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref<!fir.logical<4>>, byref @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref<!fir.logical<4>>, byref @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
index 0af9e0d5c9fd..1e5f91de801c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
@@ -41,7 +41,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
@@ -90,7 +90,7 @@ end subroutine
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 fir.store %[[VAL_16]] to %[[VAL_11]]#1 : !fir.ref<i32>
@@ -149,7 +149,7 @@ end subroutine
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref<!fir.logical<4>>, @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref<!fir.logical<4>>, @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop reduction(byref @or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]] : !fir.ref<!fir.logical<4>>, byref @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]] : !fir.ref<!fir.logical<4>>, byref @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:                 %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_24]] {uniq_name = "_QFmultiple_reductionsEy"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90
index 5358806cdcde..28c70899e6cc 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90
@@ -1,7 +1,7 @@
 ! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s
 
-! CHECK: omp.wsloop byref reduction(@max_byref_i32
+! CHECK: omp.wsloop reduction(byref @max_byref_i32
 ! CHECK: arith.cmpi sgt
 ! CHECK: arith.select
 
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
index 11d039f9226c..399afa106556 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
@@ -46,7 +46,7 @@
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@max_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @max_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
@@ -77,7 +77,7 @@
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@max_byref_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop reduction(byref @max_byref_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
@@ -98,7 +98,7 @@
 ! CHECK:             %[[VAL_32:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_33:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_34:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@max_byref_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop reduction(byref @max_byref_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) {
 ! CHECK:                 %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90
index a352cb195c25..2f3aee9f1300 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90
@@ -33,7 +33,7 @@
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@max_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @max_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
index d168b2a89295..ccab017a1b0c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
@@ -46,7 +46,7 @@
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@min_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @min_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
@@ -77,7 +77,7 @@
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@min_byref_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop reduction(byref @min_byref_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) {
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref<i32>
@@ -100,7 +100,7 @@
 ! CHECK:             %[[VAL_32:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_33:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_34:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@min_byref_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop reduction(byref @min_byref_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) {
 ! CHECK:                 %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
index 7c538cdd470f..762d9dd4f254 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
@@ -85,7 +85,7 @@
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
@@ -123,7 +123,7 @@ end subroutine
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
@@ -162,7 +162,7 @@ end subroutine
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
@@ -200,7 +200,7 @@ end subroutine
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) {
 ! CHECK:                 %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref<i32>
@@ -247,7 +247,7 @@ end subroutine
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref<i32>, @multiply_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref<i32>, @multiply_reduction_byref_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref<i32>, byref @multiply_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref<i32>, byref @multiply_reduction_byref_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -307,7 +307,7 @@ end subroutine
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref<f32>, @multiply_reduction_byref_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref<f32>, @multiply_reduction_byref_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>) {
+! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref<f32>, byref @multiply_reduction_byref_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref<f32>, byref @multiply_reduction_byref_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref<f32>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
 ! CHECK:                 %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 ! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
@@ -374,7 +374,7 @@ end subroutine
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_18:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop byref reduction(@multiply_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref<i32>, @multiply_reduction_byref_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref<i64>, @multiply_reduction_byref_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref<f32>, @multiply_reduction_byref_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<f64>) {
+! CHECK:             omp.wsloop reduction(byref @multiply_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref<i32>, byref @multiply_reduction_byref_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref<i64>, byref @multiply_reduction_byref_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref<f32>, byref @multiply_reduction_byref_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref<f64>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) {
 ! CHECK:                 %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
new file mode 100644
index 000000000000..ed462b58bbf8
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
@@ -0,0 +1,164 @@
+! RUN: bbc -emit-hlfir -fopenmp -o - %s | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+
+program main
+  implicit none
+
+  integer:: i
+  REAL(8) :: scalar
+  REAL(8) :: array(3,3)
+
+  scalar = 0
+  array = 0
+
+  !$omp parallel do reduction(+:scalar) reduction(+:array)
+  do i=1,10
+    scalar = i
+    array(1, 1) = i + 1
+    array(2, 2) = i + 2
+    array(3, 3) = i + 3
+  enddo
+
+  print *,scalar
+  print *,array
+
+endprogram
+
+! CHECK-LABEL:   omp.declare_reduction @add_reduction_byref_box_3x3xf64 : !fir.ref<!fir.box<!fir.array<3x3xf64>>> init {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3x3xf64>>>):
+! CHECK:           %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f64
+! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
+! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.box<!fir.array<3x3xf64>>
+! CHECK:           %[[VAL_4:.*]] = arith.constant 3 : index
+! CHECK:           %[[VAL_5:.*]] = arith.constant 3 : index
+! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_4]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
+! CHECK:           %[[VAL_7:.*]] = fir.allocmem !fir.array<3x3xf64> {bindc_name = ".tmp", uniq_name = ""}
+! CHECK:           %[[VAL_8:.*]] = arith.constant true
+! CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_6]]) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<3x3xf64>>, !fir.shape<2>) -> (!fir.heap<!fir.array<3x3xf64>>, !fir.heap<!fir.array<3x3xf64>>)
+! CHECK:           %[[VAL_10:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_10]] : (!fir.box<!fir.array<3x3xf64>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_12:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_13:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_12]] : (!fir.box<!fir.array<3x3xf64>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_14:.*]] = fir.shape_shift %[[VAL_11]]#0, %[[VAL_11]]#1, %[[VAL_13]]#0, %[[VAL_13]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+! CHECK:           %[[VAL_15:.*]] = fir.embox %[[VAL_9]]#0(%[[VAL_14]]) : (!fir.heap<!fir.array<3x3xf64>>, !fir.shapeshift<2>) -> !fir.box<!fir.array<3x3xf64>>
+! CHECK:           hlfir.assign %[[VAL_1]] to %[[VAL_15]] : f64, !fir.box<!fir.array<3x3xf64>>
+! CHECK:           fir.store %[[VAL_15]] to %[[VAL_3]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
+! CHECK:           omp.yield(%[[VAL_3]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>)
+
+! CHECK-LABEL:   } combiner {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3x3xf64>>>, %[[VAL_1:.*]]: !fir.ref<!fir.box<!fir.array<3x3xf64>>>):
+! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
+! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_4]] : (!fir.box<!fir.array<3x3xf64>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_6:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_6]] : (!fir.box<!fir.array<3x3xf64>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_8:.*]] = fir.shape_shift %[[VAL_5]]#0, %[[VAL_5]]#1, %[[VAL_7]]#0, %[[VAL_7]]#1 : (index, index, index, index) -> !fir.shapeshift<2>
+! CHECK:           %[[VAL_9:.*]] = arith.constant 1 : index
+! CHECK:           fir.do_loop %[[VAL_10:.*]] = %[[VAL_9]] to %[[VAL_7]]#1 step %[[VAL_9]] unordered {
+! CHECK:             fir.do_loop %[[VAL_11:.*]] = %[[VAL_9]] to %[[VAL_5]]#1 step %[[VAL_9]] unordered {
+! CHECK:               %[[VAL_12:.*]] = fir.array_coor %[[VAL_2]](%[[VAL_8]]) %[[VAL_11]], %[[VAL_10]] : (!fir.box<!fir.array<3x3xf64>>, !fir.shapeshift<2>, index, index) -> !fir.ref<f64>
+! CHECK:               %[[VAL_13:.*]] = fir.array_coor %[[VAL_3]](%[[VAL_8]]) %[[VAL_11]], %[[VAL_10]] : (!fir.box<!fir.array<3x3xf64>>, !fir.shapeshift<2>, index, index) -> !fir.ref<f64>
+! CHECK:               %[[VAL_14:.*]] = fir.load %[[VAL_12]] : !fir.ref<f64>
+! CHECK:               %[[VAL_15:.*]] = fir.load %[[VAL_13]] : !fir.ref<f64>
+! CHECK:               %[[VAL_16:.*]] = arith.addf %[[VAL_14]], %[[VAL_15]] fastmath<contract> : f64
+! CHECK:               fir.store %[[VAL_16]] to %[[VAL_12]] : !fir.ref<f64>
+! CHECK:             }
+! CHECK:           }
+! CHECK:           omp.yield(%[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>)
+
+! CHECK-LABEL:   }  cleanup {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.array<3x3xf64>>>):
+! CHECK:           %[[VAL_1:.*]] = fir.load %[[VAL_0]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
+! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<3x3xf64>>) -> !fir.ref<!fir.array<3x3xf64>>
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3x3xf64>>) -> i64
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i64
+! CHECK:           %[[VAL_5:.*]] = arith.cmpi ne, %[[VAL_3]], %[[VAL_4]] : i64
+! CHECK:           fir.if %[[VAL_5]] {
+! CHECK:             %[[VAL_6:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<3x3xf64>>) -> !fir.heap<!fir.array<3x3xf64>>
+! CHECK:             fir.freemem %[[VAL_6]] : !fir.heap<!fir.array<3x3xf64>>
+! CHECK:           }
+! CHECK:           omp.yield
+! CHECK:         }
+
+! CHECK-LABEL:   omp.declare_reduction @add_reduction_f64 : f64 init {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: f64):
+! CHECK:           %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f64
+! CHECK:           omp.yield(%[[VAL_1]] : f64)
+
+! CHECK-LABEL:   } combiner {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: f64, %[[VAL_1:.*]]: f64):
+! CHECK:           %[[VAL_2:.*]] = arith.addf %[[VAL_0]], %[[VAL_1]] fastmath<contract> : f64
+! CHECK:           omp.yield(%[[VAL_2]] : f64)
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK:           %[[VAL_0:.*]] = fir.address_of(@_QFEarray) : !fir.ref<!fir.array<3x3xf64>>
+! CHECK:           %[[VAL_1:.*]] = arith.constant 3 : index
+! CHECK:           %[[VAL_2:.*]] = arith.constant 3 : index
+! CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_1]], %[[VAL_2]] : (index, index) -> !fir.shape<2>
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_3]]) {uniq_name = "_QFEarray"} : (!fir.ref<!fir.array<3x3xf64>>, !fir.shape<2>) -> (!fir.ref<!fir.array<3x3xf64>>, !fir.ref<!fir.array<3x3xf64>>)
+! CHECK:           %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_7:.*]] = fir.alloca f64 {bindc_name = "scalar", uniq_name = "_QFEscalar"}
+! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFEscalar"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
+! CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f64
+! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_8]]#0 : f64, !fir.ref<f64>
+! CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f64
+! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_4]]#0 : f64, !fir.ref<!fir.array<3x3xf64>>
+! CHECK:           omp.parallel {
+! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
+! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
+! CHECK:             %[[VAL_15:.*]] = arith.constant 1 : i32
+! CHECK:             %[[VAL_16:.*]] = fir.embox %[[VAL_4]]#0(%[[VAL_3]]) : (!fir.ref<!fir.array<3x3xf64>>, !fir.shape<2>) -> !fir.box<!fir.array<3x3xf64>>
+! CHECK:             %[[VAL_17:.*]] = fir.alloca !fir.box<!fir.array<3x3xf64>>
+! CHECK:             fir.store %[[VAL_16]] to %[[VAL_17]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
+! CHECK:             omp.wsloop reduction(@add_reduction_f64 %[[VAL_8]]#0 -> %[[VAL_18:.*]] : !fir.ref<f64>, byref @add_reduction_byref_box_3x3xf64 %[[VAL_17]] -> %[[VAL_19:.*]] : !fir.ref<!fir.box<!fir.array<3x3xf64>>>) {
+! CHECK:               omp.loop_nest (%[[VAL_20:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) {
+! CHECK:                 %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFEscalar"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
+! CHECK:                 %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFEarray"} : (!fir.ref<!fir.box<!fir.array<3x3xf64>>>) -> (!fir.ref<!fir.box<!fir.array<3x3xf64>>>, !fir.ref<!fir.box<!fir.array<3x3xf64>>>)
+! CHECK:                 fir.store %[[VAL_20]] to %[[VAL_12]]#1 : !fir.ref<i32>
+! CHECK:                 %[[VAL_23:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<i32>
+! CHECK:                 %[[VAL_24:.*]] = fir.convert %[[VAL_23]] : (i32) -> f64
+! CHECK:                 hlfir.assign %[[VAL_24]] to %[[VAL_21]]#0 : f64, !fir.ref<f64>
+! CHECK:                 %[[VAL_25:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<i32>
+! CHECK:                 %[[VAL_26:.*]] = arith.constant 1 : i32
+! CHECK:                 %[[VAL_27:.*]] = arith.addi %[[VAL_25]], %[[VAL_26]] : i32
+! CHECK:                 %[[VAL_28:.*]] = fir.convert %[[VAL_27]] : (i32) -> f64
+! CHECK:                 %[[VAL_29:.*]] = fir.load %[[VAL_22]]#0 : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
+! CHECK:                 %[[VAL_30:.*]] = arith.constant 1 : index
+! CHECK:                 %[[VAL_31:.*]] = arith.constant 1 : index
+! CHECK:                 %[[VAL_32:.*]] = hlfir.designate %[[VAL_29]] (%[[VAL_30]], %[[VAL_31]])  : (!fir.box<!fir.array<3x3xf64>>, index, index) -> !fir.ref<f64>
+! CHECK:                 hlfir.assign %[[VAL_28]] to %[[VAL_32]] : f64, !fir.ref<f64>
+! CHECK:                 %[[VAL_33:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<i32>
+! CHECK:                 %[[VAL_34:.*]] = arith.constant 2 : i32
+! CHECK:                 %[[VAL_35:.*]] = arith.addi %[[VAL_33]], %[[VAL_34]] : i32
+! CHECK:                 %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (i32) -> f64
+! CHECK:                 %[[VAL_37:.*]] = fir.load %[[VAL_22]]#0 : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
+! CHECK:                 %[[VAL_38:.*]] = arith.constant 2 : index
+! CHECK:                 %[[VAL_39:.*]] = arith.constant 2 : index
+! CHECK:                 %[[VAL_40:.*]] = hlfir.designate %[[VAL_37]] (%[[VAL_38]], %[[VAL_39]])  : (!fir.box<!fir.array<3x3xf64>>, index, index) -> !fir.ref<f64>
+! CHECK:                 hlfir.assign %[[VAL_36]] to %[[VAL_40]] : f64, !fir.ref<f64>
+! CHECK:                 %[[VAL_41:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<i32>
+! CHECK:                 %[[VAL_42:.*]] = arith.constant 3 : i32
+! CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_41]], %[[VAL_42]] : i32
+! CHECK:                 %[[VAL_44:.*]] = fir.convert %[[VAL_43]] : (i32) -> f64
+! CHECK:                 %[[VAL_45:.*]] = fir.load %[[VAL_22]]#0 : !fir.ref<!fir.box<!fir.array<3x3xf64>>>
+! CHECK:                 %[[VAL_46:.*]] = arith.constant 3 : index
+! CHECK:                 %[[VAL_47:.*]] = arith.constant 3 : index
+! CHECK:                 %[[VAL_48:.*]] = hlfir.designate %[[VAL_45]] (%[[VAL_46]], %[[VAL_47]])  : (!fir.box<!fir.array<3x3xf64>>, index, index) -> !fir.ref<f64>
+! CHECK:                 hlfir.assign %[[VAL_44]] to %[[VAL_48]] : f64, !fir.ref<f64>
+! CHECK:                 omp.yield
+! CHECK:               }
+! CHECK:               omp.terminator
+! CHECK:             }
+! CHECK:             omp.terminator
+! CHECK:           }
+
+! CHECK-LABEL:   fir.global internal @_QFEarray : !fir.array<3x3xf64> {
+! CHECK:           %[[VAL_0:.*]] = fir.zero_bits !fir.array<3x3xf64>
+! CHECK:           fir.has_value %[[VAL_0]] : !fir.array<3x3xf64>
+! CHECK:         }
+! CHECK:         func.func private @_FortranAioBeginExternalListOutput(i32, !fir.ref<i8>, i32) -> !fir.ref<i8> attributes {fir.io, fir.runtime}
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index c9ee0c25194c..e54ec4f2b1d7 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1344,7 +1344,7 @@ public:
   InsertPointTy createReductions(const LocationDescription &Loc,
                                  InsertPointTy AllocaIP,
                                  ArrayRef<ReductionInfo> ReductionInfos,
-                                 bool IsNoWait = false, bool IsByRef = false);
+                                 ArrayRef<bool> IsByRef, bool IsNoWait = false);
 
   ///}
 
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 391a4947877a..cb4de9c8876d 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -2121,9 +2121,12 @@ Function *getFreshReductionFunc(Module &M) {
                           ".omp.reduction.func", &M);
 }
 
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
-    const LocationDescription &Loc, InsertPointTy AllocaIP,
-    ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait, bool IsByRef) {
+OpenMPIRBuilder::InsertPointTy
+OpenMPIRBuilder::createReductions(const LocationDescription &Loc,
+                                  InsertPointTy AllocaIP,
+                                  ArrayRef<ReductionInfo> ReductionInfos,
+                                  ArrayRef<bool> IsByRef, bool IsNoWait) {
+  assert(ReductionInfos.size() == IsByRef.size());
   for (const ReductionInfo &RI : ReductionInfos) {
     (void)RI;
     assert(RI.Variable && "expected non-null variable");
@@ -2213,7 +2216,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
     // We have one less load for by-ref case because that load is now inside of
     // the reduction region
     Value *RedValue = nullptr;
-    if (!IsByRef) {
+    if (!IsByRef[En.index()]) {
       RedValue = Builder.CreateLoad(ValueType, RI.Variable,
                                     "red.value." + Twine(En.index()));
     }
@@ -2221,7 +2224,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
         Builder.CreateLoad(ValueType, RI.PrivateVariable,
                            "red.private.value." + Twine(En.index()));
     Value *Reduced;
-    if (IsByRef) {
+    if (IsByRef[En.index()]) {
       Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RI.Variable,
                                         PrivateRedValue, Reduced));
     } else {
@@ -2231,7 +2234,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
     if (!Builder.GetInsertBlock())
       return InsertPointTy();
     // for by-ref case, the load is inside of the reduction region
-    if (!IsByRef)
+    if (!IsByRef[En.index()])
       Builder.CreateStore(Reduced, RI.Variable);
   }
   Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(
@@ -2244,7 +2247,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
   // function. There are no loads/stores here because they will be happening
   // inside the atomic elementwise reduction.
   Builder.SetInsertPoint(AtomicRedBlock);
-  if (CanGenerateAtomic && !IsByRef) {
+  if (CanGenerateAtomic && llvm::none_of(IsByRef, [](bool P) { return P; })) {
     for (const ReductionInfo &RI : ReductionInfos) {
       Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType,
                                               RI.Variable, RI.PrivateVariable));
@@ -2283,7 +2286,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
     if (!Builder.GetInsertBlock())
       return InsertPointTy();
     // store is inside of the reduction region when using by-ref
-    if (!IsByRef)
+    if (!IsByRef[En.index()])
       Builder.CreateStore(Reduced, LHSPtr);
   }
   Builder.CreateRetVoid();
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 8344bca08404..3ed3034f489c 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -4982,8 +4982,10 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
   OpenMPIRBuilder::ReductionInfo ReductionInfos[] = {
       {SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction},
       {XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}};
+  bool ReduceVariableByRef[] = {false, false};
 
-  OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos);
+  OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos,
+                              ReduceVariableByRef);
 
   Builder.restoreIP(AfterIP);
   Builder.CreateRetVoid();
@@ -5230,12 +5232,16 @@ TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
       /* NumThreads */ nullptr, OMP_PROC_BIND_default,
       /* IsCancellable */ false);
 
+  bool ReduceVariableByRef[] = {false};
+
   OMPBuilder.createReductions(
       FirstBodyIP, FirstBodyAllocaIP,
-      {{SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction}});
+      {{SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction}},
+      ReduceVariableByRef);
   OMPBuilder.createReductions(
       SecondBodyIP, SecondBodyAllocaIP,
-      {{XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}});
+      {{XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}},
+      ReduceVariableByRef);
 
   Builder.restoreIP(AfterIP);
   Builder.CreateRetVoid();
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
index 244cee1dd635..de7959db489e 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
@@ -177,8 +177,8 @@ struct ProcBindClauseOps {
 
 struct ReductionClauseOps {
   llvm::SmallVector<Value> reductionVars;
+  llvm::SmallVector<bool> reduceVarByRef;
   llvm::SmallVector<Attribute> reductionDeclSymbols;
-  UnitAttr reductionByRefAttr;
 };
 
 struct SafelenClauseOps {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index f248be1639fe..29c287cad06e 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -272,22 +272,21 @@ def ParallelOp : OpenMP_Op<"parallel", [
     that specify the memory allocator to be used to obtain storage for private values.
 
     Reductions can be performed in a parallel construct by specifying reduction
-    accumulator variables in `reduction_vars` and symbols referring to reduction
-    declarations in the `reductions` attribute. Each reduction is identified
-    by the accumulator it uses and accumulators must not be repeated in the same
-    reduction. The `omp.reduction` operation accepts the accumulator and a
-    partial value which is considered to be produced by the thread for the
-    given reduction. If multiple values are produced for the same accumulator,
-    i.e. there are multiple `omp.reduction`s, the last value is taken. The
-    reduction declaration specifies how to combine the values from each thread
-    into the final value, which is available in the accumulator after all the
-    threads complete.
+    accumulator variables in `reduction_vars`, symbols referring to reduction
+    declarations in the `reductions` attribute, and whether the reduction
+    variable should be passed into the reduction region by value or by reference
+    in `reduction_vars_byref`. Each reduction is identified by the accumulator
+    it uses and accumulators must not be repeated in the same reduction. The
+    `omp.reduction` operation accepts the accumulator and a partial value which
+    is considered to be produced by the thread for the given reduction. If
+    multiple values are produced for the same accumulator, i.e. there are
+    multiple `omp.reduction`s, the last value is taken. The reduction
+    declaration specifies how to combine the values from each thread into the
+    final value, which is available in the accumulator after all the threads
+    complete.
 
     The optional $proc_bind_val attribute controls the thread affinity for the execution
     of the parallel region.
-
-    The optional byref attribute controls whether reduction arguments are passed by
-    reference or by value.
   }];
 
   let arguments = (ins Optional<I1>:$if_expr_var,
@@ -295,11 +294,11 @@ def ParallelOp : OpenMP_Op<"parallel", [
              Variadic<AnyType>:$allocate_vars,
              Variadic<AnyType>:$allocators_vars,
              Variadic<OpenMP_PointerLikeType>:$reduction_vars,
+             OptionalAttr<DenseBoolArrayAttr>:$reduction_vars_byref,
              OptionalAttr<SymbolRefArrayAttr>:$reductions,
              OptionalAttr<ProcBindKindAttr>:$proc_bind_val,
              Variadic<AnyType>:$private_vars,
-             OptionalAttr<SymbolRefArrayAttr>:$privatizers,
-             UnitAttr:$byref);
+             OptionalAttr<SymbolRefArrayAttr>:$privatizers);
 
   let regions = (region AnyRegion:$region);
 
@@ -321,10 +320,9 @@ def ParallelOp : OpenMP_Op<"parallel", [
                 $allocators_vars, type($allocators_vars)
               ) `)`
           | `proc_bind` `(` custom<ClauseAttr>($proc_bind_val) `)`
-          | `byref` $byref
     ) custom<ParallelRegion>($region, $reduction_vars, type($reduction_vars),
-                             $reductions, $private_vars, type($private_vars),
-                             $privatizers) attr-dict
+                             $reduction_vars_byref, $reductions, $private_vars,
+                             type($private_vars), $privatizers) attr-dict
   }];
   let hasVerifier = 1;
 }
@@ -649,14 +647,15 @@ def WsloopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments,
     elements.
 
     Reductions can be performed in a worksharing-loop by specifying reduction
-    accumulator variables in `reduction_vars` and symbols referring to reduction
-    declarations in the `reductions` attribute. Each reduction is identified
-    by the accumulator it uses and accumulators must not be repeated in the same
-    reduction. A private variable corresponding to the accumulator is used in
-    place of the accumulator inside the body of the worksharing-loop. The
-    reduction declaration specifies how to combine the values from each
-    iteration into the final value, which is available in the accumulator after
-    the loop completes.
+    accumulator variables in `reduction_vars`, symbols referring to reduction
+    declarations in the `reductions` attribute, and whether the reduction
+    variable should be passed by reference or value in `reduction_vars_byref`.
+    Each reduction is identified by the accumulator it uses and accumulators
+    must not be repeated in the same reduction. A private variable corresponding
+    to the accumulator is used in place of the accumulator inside the body of
+    the worksharing-loop. The reduction declaration specifies how to combine the
+    values from each iteration into the final value, which is available in the
+    accumulator after the loop completes.
 
     The optional `schedule_val` attribute specifies the loop schedule for this
     loop, determining how the loop is distributed across the parallel threads.
@@ -677,21 +676,18 @@ def WsloopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments,
     The optional `order` attribute specifies which order the iterations of the
     associate loops are executed in. Currently the only option for this
     attribute is "concurrent".
-
-    The optional `byref` attribute indicates that reduction arguments should be
-    passed by reference.
   }];
 
   let arguments = (ins Variadic<AnyType>:$linear_vars,
              Variadic<I32>:$linear_step_vars,
              Variadic<OpenMP_PointerLikeType>:$reduction_vars,
+             OptionalAttr<DenseBoolArrayAttr>:$reduction_vars_byref,
              OptionalAttr<SymbolRefArrayAttr>:$reductions,
              OptionalAttr<ScheduleKindAttr>:$schedule_val,
              Optional<AnyType>:$schedule_chunk_var,
              OptionalAttr<ScheduleModifierAttr>:$schedule_modifier,
              UnitAttr:$simd_modifier,
              UnitAttr:$nowait,
-             UnitAttr:$byref,
              ConfinedAttr<OptionalAttr<I64Attr>, [IntMinValue<0>]>:$ordered_val,
              OptionalAttr<OrderKindAttr>:$order_val);
 
@@ -716,11 +712,10 @@ def WsloopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments,
                 $schedule_val, $schedule_modifier, $simd_modifier,
                 $schedule_chunk_var, type($schedule_chunk_var)) `)`
           |`nowait` $nowait
-          |`byref` $byref
           |`ordered` `(` $ordered_val `)`
           |`order` `(` custom<ClauseAttr>($order_val) `)`
     ) custom<Wsloop>($region, $reduction_vars, type($reduction_vars),
-                     $reductions) attr-dict
+                     $reduction_vars_byref, $reductions) attr-dict
   }];
   let hasVerifier = 1;
 }
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index d6f85451ee5d..60c62131752f 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -449,6 +449,7 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
         /* allocate_vars = */ llvm::SmallVector<Value>{},
         /* allocators_vars = */ llvm::SmallVector<Value>{},
         /* reduction_vars = */ llvm::SmallVector<Value>{},
+        /* reduction_vars_isbyref = */ DenseBoolArrayAttr{},
         /* reductions = */ ArrayAttr{},
         /* proc_bind_val = */ omp::ClauseProcBindKindAttr{},
         /* private_vars = */ ValueRange(),
@@ -467,6 +468,12 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
           wsloopOp.setReductionsAttr(
               ArrayAttr::get(rewriter.getContext(), reductionDeclSymbols));
           wsloopOp.getReductionVarsMutable().append(reductionVariables);
+          llvm::SmallVector<bool> byRefVec;
+          // false because these reductions always reduce scalars and so do
+          // not need to pass by reference
+          byRefVec.resize(reductionVariables.size(), false);
+          wsloopOp.setReductionVarsByref(
+              DenseBoolArrayAttr::get(rewriter.getContext(), byRefVec));
         }
         rewriter.create<omp::TerminatorOp>(loc); // omp.parallel terminator.
 
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 61073af2aa4d..24a6d5b5d684 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -15,11 +15,13 @@
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/OpenACCMPCommon/Interfaces/AtomicInterfaces.h"
 #include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/Interfaces/FoldInterfaces.h"
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLForwardCompat.h"
@@ -435,25 +437,30 @@ static void printScheduleClause(OpAsmPrinter &p, Operation *op,
 // Parser, printer and verifier for ReductionVarList
 //===----------------------------------------------------------------------===//
 
-ParseResult parseClauseWithRegionArgs(
+static ParseResult parseClauseWithRegionArgs(
     OpAsmParser &parser, Region &region,
     SmallVectorImpl<OpAsmParser::UnresolvedOperand> &operands,
-    SmallVectorImpl<Type> &types, ArrayAttr &symbols,
+    SmallVectorImpl<Type> &types, DenseBoolArrayAttr &isByRef,
+    ArrayAttr &symbols,
     SmallVectorImpl<OpAsmParser::Argument> &regionPrivateArgs) {
   SmallVector<SymbolRefAttr> reductionVec;
+  SmallVector<bool> isByRefVec;
   unsigned regionArgOffset = regionPrivateArgs.size();
 
   if (failed(
           parser.parseCommaSeparatedList(OpAsmParser::Delimiter::Paren, [&]() {
+            ParseResult optionalByref = parser.parseOptionalKeyword("byref");
             if (parser.parseAttribute(reductionVec.emplace_back()) ||
                 parser.parseOperand(operands.emplace_back()) ||
                 parser.parseArrow() ||
                 parser.parseArgument(regionPrivateArgs.emplace_back()) ||
                 parser.parseColonType(types.emplace_back()))
               return failure();
+            isByRefVec.push_back(optionalByref.succeeded());
             return success();
           })))
     return failure();
+  isByRef = DenseBoolArrayAttr::get(parser.getContext(), isByRefVec);
 
   auto *argsBegin = regionPrivateArgs.begin();
   MutableArrayRef argsSubrange(argsBegin + regionArgOffset,
@@ -469,15 +476,18 @@ ParseResult parseClauseWithRegionArgs(
 static void printClauseWithRegionArgs(OpAsmPrinter &p, Operation *op,
                                       ValueRange argsSubrange,
                                       StringRef clauseName, ValueRange operands,
-                                      TypeRange types, ArrayAttr symbols) {
+                                      TypeRange types, DenseBoolArrayAttr byRef,
+                                      ArrayAttr symbols) {
   if (!clauseName.empty())
     p << clauseName << "(";
 
-  llvm::interleaveComma(
-      llvm::zip_equal(symbols, operands, argsSubrange, types), p, [&p](auto t) {
-        auto [sym, op, arg, type] = t;
-        p << sym << " " << op << " -> " << arg << " : " << type;
-      });
+  llvm::interleaveComma(llvm::zip_equal(symbols, operands, argsSubrange, types,
+                                        byRef.asArrayRef()),
+                        p, [&p](auto t) {
+                          auto [sym, op, arg, type, isByRef] = t;
+                          p << (isByRef ? "byref " : "") << sym << " " << op
+                            << " -> " << arg << " : " << type;
+                        });
 
   if (!clauseName.empty())
     p << ") ";
@@ -486,7 +496,8 @@ static void printClauseWithRegionArgs(OpAsmPrinter &p, Operation *op,
 static ParseResult parseParallelRegion(
     OpAsmParser &parser, Region &region,
     SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVarOperands,
-    SmallVectorImpl<Type> &reductionVarTypes, ArrayAttr &reductionSymbols,
+    SmallVectorImpl<Type> &reductionVarTypes,
+    DenseBoolArrayAttr &reductionByRef, ArrayAttr &reductionSymbols,
     llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVarOperands,
     llvm::SmallVectorImpl<Type> &privateVarsTypes,
     ArrayAttr &privatizerSymbols) {
@@ -494,16 +505,23 @@ static ParseResult parseParallelRegion(
 
   if (succeeded(parser.parseOptionalKeyword("reduction"))) {
     if (failed(parseClauseWithRegionArgs(parser, region, reductionVarOperands,
-                                         reductionVarTypes, reductionSymbols,
-                                         regionPrivateArgs)))
+                                         reductionVarTypes, reductionByRef,
+                                         reductionSymbols, regionPrivateArgs)))
       return failure();
   }
 
   if (succeeded(parser.parseOptionalKeyword("private"))) {
+    auto privateByRef = DenseBoolArrayAttr::get(parser.getContext(), {});
     if (failed(parseClauseWithRegionArgs(parser, region, privateVarOperands,
-                                         privateVarsTypes, privatizerSymbols,
-                                         regionPrivateArgs)))
+                                         privateVarsTypes, privateByRef,
+                                         privatizerSymbols, regionPrivateArgs)))
       return failure();
+    if (llvm::any_of(privateByRef.asArrayRef(),
+                     [](bool byref) { return byref; })) {
+      parser.emitError(parser.getCurrentLocation(),
+                       "private clause cannot have byref attributes");
+      return failure();
+    }
   }
 
   return parser.parseRegion(region, regionPrivateArgs);
@@ -512,6 +530,7 @@ static ParseResult parseParallelRegion(
 static void printParallelRegion(OpAsmPrinter &p, Operation *op, Region &region,
                                 ValueRange reductionVarOperands,
                                 TypeRange reductionVarTypes,
+                                DenseBoolArrayAttr reductionVarIsByRef,
                                 ArrayAttr reductionSymbols,
                                 ValueRange privateVarOperands,
                                 TypeRange privateVarTypes,
@@ -522,7 +541,7 @@ static void printParallelRegion(OpAsmPrinter &p, Operation *op, Region &region,
                                  argsBegin + reductionVarTypes.size());
     printClauseWithRegionArgs(p, op, argsSubrange, "reduction",
                               reductionVarOperands, reductionVarTypes,
-                              reductionSymbols);
+                              reductionVarIsByRef, reductionSymbols);
   }
 
   if (privatizerSymbols) {
@@ -530,8 +549,13 @@ static void printParallelRegion(OpAsmPrinter &p, Operation *op, Region &region,
     MutableArrayRef argsSubrange(argsBegin + reductionVarOperands.size(),
                                  argsBegin + reductionVarOperands.size() +
                                      privateVarTypes.size());
+    mlir::SmallVector<bool> isByRefVec;
+    isByRefVec.resize(privateVarTypes.size(), false);
+    DenseBoolArrayAttr isByRef =
+        DenseBoolArrayAttr::get(op->getContext(), isByRefVec);
+
     printClauseWithRegionArgs(p, op, argsSubrange, "private",
-                              privateVarOperands, privateVarTypes,
+                              privateVarOperands, privateVarTypes, isByRef,
                               privatizerSymbols);
   }
 
@@ -575,14 +599,22 @@ static void printReductionVarList(OpAsmPrinter &p, Operation *op,
 }
 
 /// Verifies Reduction Clause
-static LogicalResult verifyReductionVarList(Operation *op,
-                                            std::optional<ArrayAttr> reductions,
-                                            OperandRange reductionVars) {
+static LogicalResult
+verifyReductionVarList(Operation *op, std::optional<ArrayAttr> reductions,
+                       OperandRange reductionVars,
+                       std::optional<ArrayRef<bool>> byRef = std::nullopt) {
   if (!reductionVars.empty()) {
     if (!reductions || reductions->size() != reductionVars.size())
       return op->emitOpError()
              << "expected as many reduction symbol references "
                 "as reduction variables";
+    if (mlir::isa<omp::WsloopOp, omp::ParallelOp>(op))
+      assert(byRef);
+    else
+      assert(!byRef); // TODO: support byref reductions on other operations
+    if (byRef && byRef->size() != reductionVars.size())
+      return op->emitError() << "expected as many reduction variable by "
+                                "reference attributes as reduction variables";
   } else {
     if (reductions)
       return op->emitOpError() << "unexpected reduction symbol references";
@@ -1163,9 +1195,14 @@ static void printPrivateList(OpAsmPrinter &p, Operation *op,
   MutableArrayRef argsSubrange(argsBegin + targetOp.getMapOperands().size(),
                                argsBegin + targetOp.getMapOperands().size() +
                                    privateVarTypes.size());
+  mlir::SmallVector<bool> isByRefVec;
+  isByRefVec.resize(privateVarTypes.size(), false);
+  DenseBoolArrayAttr isByRef =
+      DenseBoolArrayAttr::get(op->getContext(), isByRefVec);
+
   printClauseWithRegionArgs(
       p, op, argsSubrange, /*clauseName=*/llvm::StringRef{}, privateVarOperands,
-      privateVarTypes, privatizerSymbols);
+      privateVarTypes, isByRef, privatizerSymbols);
 }
 
 static void printCaptureType(OpAsmPrinter &p, Operation *op,
@@ -1402,21 +1439,24 @@ void ParallelOp::build(OpBuilder &builder, OperationState &state,
   ParallelOp::build(
       builder, state, /*if_expr_var=*/nullptr, /*num_threads_var=*/nullptr,
       /*allocate_vars=*/ValueRange(), /*allocators_vars=*/ValueRange(),
-      /*reduction_vars=*/ValueRange(), /*reductions=*/nullptr,
+      /*reduction_vars=*/ValueRange(), /*reduction_vars_byref=*/nullptr,
+      /*reductions=*/nullptr,
       /*proc_bind_val=*/nullptr, /*private_vars=*/ValueRange(),
-      /*privatizers=*/nullptr, /*byref=*/false);
+      /*privatizers=*/nullptr);
   state.addAttributes(attributes);
 }
 
 void ParallelOp::build(OpBuilder &builder, OperationState &state,
                        const ParallelClauseOps &clauses) {
   MLIRContext *ctx = builder.getContext();
-  ParallelOp::build(
-      builder, state, clauses.ifVar, clauses.numThreadsVar,
-      clauses.allocateVars, clauses.allocatorVars, clauses.reductionVars,
-      makeArrayAttr(ctx, clauses.reductionDeclSymbols),
-      clauses.procBindKindAttr, clauses.privateVars,
-      makeArrayAttr(ctx, clauses.privatizers), clauses.reductionByRefAttr);
+
+  ParallelOp::build(builder, state, clauses.ifVar, clauses.numThreadsVar,
+                    clauses.allocateVars, clauses.allocatorVars,
+                    clauses.reductionVars,
+                    DenseBoolArrayAttr::get(ctx, clauses.reduceVarByRef),
+                    makeArrayAttr(ctx, clauses.reductionDeclSymbols),
+                    clauses.procBindKindAttr, clauses.privateVars,
+                    makeArrayAttr(ctx, clauses.privatizers));
 }
 
 template <typename OpType>
@@ -1488,7 +1528,8 @@ LogicalResult ParallelOp::verify() {
   if (failed(verifyPrivateVarList(*this)))
     return failure();
 
-  return verifyReductionVarList(*this, getReductions(), getReductionVars());
+  return verifyReductionVarList(*this, getReductions(), getReductionVars(),
+                                getReductionVarsByref());
 }
 
 //===----------------------------------------------------------------------===//
@@ -1609,13 +1650,13 @@ ParseResult
 parseWsloop(OpAsmParser &parser, Region &region,
             SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionOperands,
             SmallVectorImpl<Type> &reductionTypes,
-            ArrayAttr &reductionSymbols) {
+            DenseBoolArrayAttr &reductionByRef, ArrayAttr &reductionSymbols) {
   // Parse an optional reduction clause
   llvm::SmallVector<OpAsmParser::Argument> privates;
   if (succeeded(parser.parseOptionalKeyword("reduction"))) {
     if (failed(parseClauseWithRegionArgs(parser, region, reductionOperands,
-                                         reductionTypes, reductionSymbols,
-                                         privates)))
+                                         reductionTypes, reductionByRef,
+                                         reductionSymbols, privates)))
       return failure();
   }
   return parser.parseRegion(region, privates);
@@ -1623,11 +1664,11 @@ parseWsloop(OpAsmParser &parser, Region &region,
 
 void printWsloop(OpAsmPrinter &p, Operation *op, Region &region,
                  ValueRange reductionOperands, TypeRange reductionTypes,
-                 ArrayAttr reductionSymbols) {
+                 DenseBoolArrayAttr isByRef, ArrayAttr reductionSymbols) {
   if (reductionSymbols) {
     auto reductionArgs = region.front().getArguments();
     printClauseWithRegionArgs(p, op, reductionArgs, "reduction",
-                              reductionOperands, reductionTypes,
+                              reductionOperands, reductionTypes, isByRef,
                               reductionSymbols);
   }
   p.printRegion(region, /*printEntryBlockArgs=*/false);
@@ -1637,9 +1678,10 @@ void WsloopOp::build(OpBuilder &builder, OperationState &state,
                      ArrayRef<NamedAttribute> attributes) {
   build(builder, state, /*linear_vars=*/ValueRange(),
         /*linear_step_vars=*/ValueRange(), /*reduction_vars=*/ValueRange(),
+        /*reduction_vars_byref=*/nullptr,
         /*reductions=*/nullptr, /*schedule_val=*/nullptr,
         /*schedule_chunk_var=*/nullptr, /*schedule_modifier=*/nullptr,
-        /*simd_modifier=*/false, /*nowait=*/false, /*byref=*/false,
+        /*simd_modifier=*/false, /*nowait=*/false,
         /*ordered_val=*/nullptr, /*order_val=*/nullptr);
   state.addAttributes(attributes);
 }
@@ -1649,12 +1691,13 @@ void WsloopOp::build(OpBuilder &builder, OperationState &state,
   MLIRContext *ctx = builder.getContext();
   // TODO: Store clauses in op: allocateVars, allocatorVars, privateVars,
   // privatizers.
-  WsloopOp::build(
-      builder, state, clauses.linearVars, clauses.linearStepVars,
-      clauses.reductionVars, makeArrayAttr(ctx, clauses.reductionDeclSymbols),
-      clauses.scheduleValAttr, clauses.scheduleChunkVar,
-      clauses.scheduleModAttr, clauses.scheduleSimdAttr, clauses.nowaitAttr,
-      clauses.reductionByRefAttr, clauses.orderedAttr, clauses.orderAttr);
+  WsloopOp::build(builder, state, clauses.linearVars, clauses.linearStepVars,
+                  clauses.reductionVars,
+                  DenseBoolArrayAttr::get(ctx, clauses.reduceVarByRef),
+                  makeArrayAttr(ctx, clauses.reductionDeclSymbols),
+                  clauses.scheduleValAttr, clauses.scheduleChunkVar,
+                  clauses.scheduleModAttr, clauses.scheduleSimdAttr,
+                  clauses.nowaitAttr, clauses.orderedAttr, clauses.orderAttr);
 }
 
 LogicalResult WsloopOp::verify() {
@@ -1668,7 +1711,8 @@ LogicalResult WsloopOp::verify() {
       return emitError() << "only supported nested wrapper is 'omp.simd'";
   }
 
-  return verifyReductionVarList(*this, getReductions(), getReductionVars());
+  return verifyReductionVarList(*this, getReductions(), getReductionVars(),
+                                getReductionVarsByref());
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index ed9fb44cf08e..8576be67f04a 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -22,6 +22,7 @@
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
 #include "mlir/Transforms/RegionUtils.h"
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
@@ -813,13 +814,16 @@ static void allocByValReductionVars(
     llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
     SmallVectorImpl<omp::DeclareReductionOp> &reductionDecls,
     SmallVectorImpl<llvm::Value *> &privateReductionVariables,
-    DenseMap<Value, llvm::Value *> &reductionVariableMap) {
+    DenseMap<Value, llvm::Value *> &reductionVariableMap,
+    llvm::ArrayRef<bool> isByRefs) {
   llvm::IRBuilderBase::InsertPointGuard guard(builder);
   builder.restoreIP(allocaIP);
   auto args =
       loop.getRegion().getArguments().take_back(loop.getNumReductionVars());
 
   for (std::size_t i = 0; i < loop.getNumReductionVars(); ++i) {
+    if (isByRefs[i])
+      continue;
     llvm::Value *var = builder.CreateAlloca(
         moduleTranslation.convertType(reductionDecls[i].getType()));
     moduleTranslation.mapValue(args[i], var);
@@ -919,13 +923,21 @@ inlineOmpRegionCleanup(llvm::SmallVectorImpl<Region *> &cleanupRegions,
   return success();
 }
 
+static ArrayRef<bool> getIsByRef(std::optional<ArrayRef<bool>> attr) {
+  if (!attr)
+    return {};
+  return *attr;
+}
+
 /// Converts an OpenMP workshare loop into LLVM IR using OpenMPIRBuilder.
 static LogicalResult
 convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   auto wsloopOp = cast<omp::WsloopOp>(opInst);
   auto loopOp = cast<omp::LoopNestOp>(wsloopOp.getWrappedLoop());
-  const bool isByRef = wsloopOp.getByref();
+
+  llvm::ArrayRef<bool> isByRef = getIsByRef(wsloopOp.getReductionVarsByref());
+  assert(isByRef.size() == wsloopOp.getNumReductionVars());
 
   // TODO: this should be in the op verifier instead.
   if (loopOp.getLowerBound().empty())
@@ -952,11 +964,9 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
 
   SmallVector<llvm::Value *> privateReductionVariables;
   DenseMap<Value, llvm::Value *> reductionVariableMap;
-  if (!isByRef) {
-    allocByValReductionVars(wsloopOp, builder, moduleTranslation, allocaIP,
-                            reductionDecls, privateReductionVariables,
-                            reductionVariableMap);
-  }
+  allocByValReductionVars(wsloopOp, builder, moduleTranslation, allocaIP,
+                          reductionDecls, privateReductionVariables,
+                          reductionVariableMap, isByRef);
 
   // Before the loop, store the initial values of reductions into reduction
   // variables. Although this could be done after allocas, we don't want to mess
@@ -974,7 +984,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       return failure();
     assert(phis.size() == 1 && "expected one value to be yielded from the "
                                "reduction neutral element declaration region");
-    if (isByRef) {
+    if (isByRef[i]) {
       // Allocate reduction variable (which is a pointer to the real reduction
       // variable allocated in the inlined region)
       llvm::Value *var = builder.CreateAlloca(
@@ -1108,7 +1118,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   builder.SetInsertPoint(tempTerminator);
   llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
       ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos,
-                                   wsloopOp.getNowait(), isByRef);
+                                   isByRef, wsloopOp.getNowait());
   if (!contInsertPoint.getBlock())
     return wsloopOp->emitOpError() << "failed to convert reductions";
   auto nextInsertionPoint =
@@ -1170,7 +1180,8 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
                    LLVM::ModuleTranslation &moduleTranslation) {
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
   OmpParallelOpConversionManager raii(opInst);
-  const bool isByRef = opInst.getByref();
+  ArrayRef<bool> isByRef = getIsByRef(opInst.getReductionVarsByref());
+  assert(isByRef.size() == opInst.getNumReductionVars());
 
   // TODO: support error propagation in OpenMPIRBuilder and use it instead of
   // relying on captured variables.
@@ -1185,11 +1196,9 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
   auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
     // Allocate reduction vars
     DenseMap<Value, llvm::Value *> reductionVariableMap;
-    if (!isByRef) {
-      allocByValReductionVars(opInst, builder, moduleTranslation, allocaIP,
-                              reductionDecls, privateReductionVariables,
-                              reductionVariableMap);
-    }
+    allocByValReductionVars(opInst, builder, moduleTranslation, allocaIP,
+                            reductionDecls, privateReductionVariables,
+                            reductionVariableMap, isByRef);
 
     // Initialize reduction vars
     builder.restoreIP(allocaIP);
@@ -1210,7 +1219,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
              "reduction neutral element declaration region");
       builder.restoreIP(allocaIP);
 
-      if (isByRef) {
+      if (isByRef[i]) {
         // Allocate reduction variable (which is a pointer to the real reduciton
         // variable allocated in the inlined region)
         llvm::Value *var = builder.CreateAlloca(
@@ -1269,7 +1278,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
 
       llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
           ompBuilder->createReductions(builder.saveIP(), allocaIP,
-                                       reductionInfos, false, isByRef);
+                                       reductionInfos, isByRef, false);
       if (!contInsertPoint.getBlock()) {
         bodyGenStatus = opInst->emitOpError() << "failed to convert reductions";
         return;
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index aee79264313e..db016fe8e7ba 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -2272,3 +2272,22 @@ func.func @undefined_privatizer(%arg0: !llvm.ptr) {
     }) : (!llvm.ptr) -> ()
   return
 }
+
+// -----
+
+omp.private {type = private} @var1.privatizer : !llvm.ptr alloc {
+^bb0(%arg0: !llvm.ptr):
+  omp.yield(%arg0 : !llvm.ptr)
+} copy {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+  omp.yield(%arg0 : !llvm.ptr)
+}
+
+func.func @byref_in_private(%arg0: index) {
+  // expected-error @below {{private clause cannot have byref attributes}}
+  omp.parallel private(byref @var1.privatizer %arg0 -> %arg2 : index) {
+    omp.terminator
+  }
+
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index fc6d1c316531..0d5fd9383a92 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -869,6 +869,28 @@ func.func @wsloop_reduction(%lb : index, %ub : index, %step : index) {
   return
 }
 
+// CHECK-LABEL: func @wsloop_reduction_byref
+func.func @wsloop_reduction_byref(%lb : index, %ub : index, %step : index) {
+  %c1 = arith.constant 1 : i32
+  %0 = llvm.alloca %c1 x i32 : (i32) -> !llvm.ptr
+  // CHECK: reduction(byref @add_f32 %{{.+}} -> %[[PRV:.+]] : !llvm.ptr)
+  omp.wsloop reduction(byref @add_f32 %0 -> %prv : !llvm.ptr) {
+    omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
+      // CHECK: %[[CST:.+]] = arith.constant 2.0{{.*}} : f32
+      %cst = arith.constant 2.0 : f32
+      // CHECK: %[[LPRV:.+]] = llvm.load %[[PRV]] : !llvm.ptr -> f32
+      %lprv = llvm.load %prv : !llvm.ptr -> f32
+      // CHECK: %[[RES:.+]] = llvm.fadd %[[LPRV]], %[[CST]] : f32
+      %res = llvm.fadd %lprv, %cst: f32
+      // CHECK: llvm.store %[[RES]], %[[PRV]] :  f32, !llvm.ptr
+      llvm.store %res, %prv :  f32, !llvm.ptr
+      omp.yield
+    }
+    omp.terminator
+  }
+  return
+}
+
 // CHECK-LABEL: func @parallel_reduction
 func.func @parallel_reduction() {
   %c1 = arith.constant 1 : i32
@@ -885,6 +907,22 @@ func.func @parallel_reduction() {
   return
 }
 
+// CHECK-LABEL: func @parallel_reduction_byref
+func.func @parallel_reduction_byref() {
+  %c1 = arith.constant 1 : i32
+  %0 = llvm.alloca %c1 x i32 : (i32) -> !llvm.ptr
+  // CHECK: omp.parallel reduction(byref @add_f32 {{.+}} -> {{.+}} : !llvm.ptr)
+  omp.parallel reduction(byref @add_f32 %0 -> %prv : !llvm.ptr) {
+    %1 = arith.constant 2.0 : f32
+    %2 = llvm.load %prv : !llvm.ptr -> f32
+    // CHECK: llvm.fadd %{{.*}}, %{{.*}} : f32
+    %3 = llvm.fadd %1, %2 : f32
+    llvm.store %3, %prv : f32, !llvm.ptr
+    omp.terminator
+  }
+  return
+}
+
 // CHECK: func @parallel_wsloop_reduction
 func.func @parallel_wsloop_reduction(%lb : index, %ub : index, %step : index) {
   %c1 = arith.constant 1 : i32
diff --git a/mlir/test/Target/LLVMIR/openmp-parallel-reduction-cleanup.mlir b/mlir/test/Target/LLVMIR/openmp-parallel-reduction-cleanup.mlir
index b7f71f438e56..84a487cb8c98 100644
--- a/mlir/test/Target/LLVMIR/openmp-parallel-reduction-cleanup.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-parallel-reduction-cleanup.mlir
@@ -27,7 +27,7 @@
     %0 = llvm.mlir.constant(-1 : i32) : i32
     %1 = llvm.mlir.addressof @i : !llvm.ptr
     %2 = llvm.mlir.addressof @j : !llvm.ptr
-    omp.parallel byref reduction(@add_reduction_i_32 %1 -> %arg0 : !llvm.ptr, @add_reduction_i_32 %2 -> %arg1 : !llvm.ptr) {
+    omp.parallel reduction(byref @add_reduction_i_32 %1 -> %arg0 : !llvm.ptr, byref @add_reduction_i_32 %2 -> %arg1 : !llvm.ptr) {
       llvm.store %0, %arg0 : i32, !llvm.ptr
       llvm.store %0, %arg1 : i32, !llvm.ptr
       omp.terminator
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-byref.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-byref.mlir
index e720969e82c1..ef1284547a88 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-byref.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-byref.mlir
@@ -20,7 +20,7 @@
   llvm.func @main()  {
     %0 = llvm.mlir.constant(-1 : i32) : i32
     %1 = llvm.mlir.addressof @i : !llvm.ptr
-    omp.parallel byref reduction(@add_reduction_i_32 %1 -> %arg0 : !llvm.ptr) {
+    omp.parallel reduction(byref @add_reduction_i_32 %1 -> %arg0 : !llvm.ptr) {
       llvm.store %0, %arg0 : i32, !llvm.ptr
       omp.terminator
     }
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-init-arg.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-init-arg.mlir
index 5dd31c425566..f4b77cbf413d 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-init-arg.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-init-arg.mlir
@@ -24,7 +24,7 @@ module {
   %87 = llvm.alloca %86 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> : (i64) -> !llvm.ptr
 // test multiple reduction variables to ensure they don't intefere with eachother
 // when inlining the reduction init region multiple times
-    omp.parallel byref reduction(@add_reduction_byref_box_Uxf64 %84 -> %arg3 : !llvm.ptr, @add_reduction_byref_box_Uxf64 %87 -> %arg4 : !llvm.ptr) {
+    omp.parallel reduction(byref @add_reduction_byref_box_Uxf64 %84 -> %arg3 : !llvm.ptr, byref @add_reduction_byref_box_Uxf64 %87 -> %arg4 : !llvm.ptr) {
       omp.terminator
     }
     llvm.return
diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir
index 7a1a31830ce9..0b28294dc1ce 100644
--- a/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir
@@ -30,7 +30,7 @@
     %loop_ub = llvm.mlir.constant(9 : i32) : i32
     %loop_lb = llvm.mlir.constant(0 : i32) : i32
     %loop_step = llvm.mlir.constant(1 : i32) : i32 
-    omp.wsloop byref reduction(@add_reduction_i_32 %1 -> %arg0 : !llvm.ptr, @add_reduction_i_32 %2 -> %arg1 : !llvm.ptr) {
+    omp.wsloop reduction(byref @add_reduction_i_32 %1 -> %arg0 : !llvm.ptr, byref @add_reduction_i_32 %2 -> %arg1 : !llvm.ptr) {
       omp.loop_nest (%loop_cnt) : i32 = (%loop_lb) to (%loop_ub) inclusive step (%loop_step) {
         llvm.store %0, %arg0 : i32, !llvm.ptr
         llvm.store %0, %arg1 : i32, !llvm.ptr
-- 
GitLab


From d94582eea410a04f9f84e39a54276a8418aa2dbb Mon Sep 17 00:00:00 2001
From: NimishMishra <42909663+NimishMishra@users.noreply.github.com>
Date: Thu, 16 May 2024 07:28:40 -0700
Subject: [PATCH 1020/1206] [flang][OpenMP] Add test for checking overloaded
 operator in atomic update (#88471)

Atomic update expression does not allow overloaded user-defined
operators. This PR adds a test case for the same; the semantic check is
already existent.
---
 .../OpenMP/atomic-update-overloaded-ops.f90   | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 flang/test/Semantics/OpenMP/atomic-update-overloaded-ops.f90

diff --git a/flang/test/Semantics/OpenMP/atomic-update-overloaded-ops.f90 b/flang/test/Semantics/OpenMP/atomic-update-overloaded-ops.f90
new file mode 100644
index 000000000000..21a9b87d2634
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/atomic-update-overloaded-ops.f90
@@ -0,0 +1,31 @@
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+
+module new_operator
+    implicit none
+
+    interface operator(.MYOPERATOR.)
+       module procedure myprocedure
+    end interface
+contains
+    pure integer function myprocedure(param1, param2)
+        integer, intent(in) :: param1, param2
+        myprocedure = param1 + param2
+    end function
+end module
+
+program sample
+    use new_operator
+    implicit none
+    integer :: x, y 
+
+    !$omp atomic update
+        x = x / y
+     
+    !$omp atomic update
+    !ERROR: Invalid or missing operator in atomic update statement
+        x = x .MYOPERATOR. y
+
+    !$omp atomic
+    !ERROR: Invalid or missing operator in atomic update statement
+        x = x .MYOPERATOR. y
+end program
-- 
GitLab


From f579dcf816b5626724e9eae5feea594008b5c863 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Thu, 16 May 2024 18:34:22 +0400
Subject: [PATCH 1021/1206] [lldb] Fixed an invalid error message in the DAP
 disconnect response (#92345)

The `disconnect` response contains the `error` message with invalid
characters (a junk data). To reproduce this issue it is enough to run
the `TestDAP_commands` test on Windows host and Linux target. The test
will fail to run ELF file on Windows and dap_server will be disconnected
unexpectedly.

Note dap_server hangs if read_packet() cannot decode JSON with invalid
characters. read_packet() must return None in this case instead of an
exception. But dap_server does not require any fix after this patch.
---
 lldb/tools/lldb-dap/lldb-dap.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp
index 96da458be21d..170fa88f1e8b 100644
--- a/lldb/tools/lldb-dap/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/lldb-dap.cpp
@@ -977,7 +977,7 @@ void request_disconnect(const llvm::json::Object &request) {
     g_dap.debugger.SetAsync(false);
     lldb::SBError error = terminateDebuggee ? process.Kill() : process.Detach();
     if (!error.Success())
-      response.try_emplace("error", error.GetCString());
+      EmplaceSafeString(response, "error", error.GetCString());
     g_dap.debugger.SetAsync(true);
     break;
   }
-- 
GitLab


From d665d51c9296fc0b57945bb67e06040e26cd03c5 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Thu, 16 May 2024 18:34:58 +0400
Subject: [PATCH 1022/1206] [lldb] Fixed the DAP tests in case of a remote
 target (#92398)

These tests failed in case of Windows host and Linux target, because
dap_server tried to run ELF file on Windows.
---
 lldb/test/API/tools/lldb-dap/commands/TestDAP_commands.py       | 2 ++
 .../API/tools/lldb-dap/startDebugging/TestDAP_startDebugging.py | 1 +
 2 files changed, 3 insertions(+)

diff --git a/lldb/test/API/tools/lldb-dap/commands/TestDAP_commands.py b/lldb/test/API/tools/lldb-dap/commands/TestDAP_commands.py
index 226b9385fe71..bfdf9ef2897b 100644
--- a/lldb/test/API/tools/lldb-dap/commands/TestDAP_commands.py
+++ b/lldb/test/API/tools/lldb-dap/commands/TestDAP_commands.py
@@ -7,6 +7,7 @@ from lldbsuite.test.decorators import *
 
 
 class TestDAP_commands(lldbdap_testcase.DAPTestCaseBase):
+    @skipIfRemote
     def test_command_directive_quiet_on_success(self):
         program = self.getBuildArtifact("a.out")
         command_quiet = (
@@ -60,6 +61,7 @@ class TestDAP_commands(lldbdap_testcase.DAPTestCaseBase):
     def test_command_directive_abort_on_error_pre_run_commands(self):
         self.do_test_abort_on_error(use_pre_run_commands=True)
 
+    @skipIfRemote
     def test_command_directive_abort_on_error_post_run_commands(self):
         self.do_test_abort_on_error(use_post_run_commands=True)
 
diff --git a/lldb/test/API/tools/lldb-dap/startDebugging/TestDAP_startDebugging.py b/lldb/test/API/tools/lldb-dap/startDebugging/TestDAP_startDebugging.py
index fd48e69cae5e..7700c65f862d 100644
--- a/lldb/test/API/tools/lldb-dap/startDebugging/TestDAP_startDebugging.py
+++ b/lldb/test/API/tools/lldb-dap/startDebugging/TestDAP_startDebugging.py
@@ -11,6 +11,7 @@ import lldbdap_testcase
 
 
 class TestDAP_startDebugging(lldbdap_testcase.DAPTestCaseBase):
+    @skipIfRemote
     def test_startDebugging(self):
         """
         Tests the "startDebugging" reverse request. It makes sure that the IDE can
-- 
GitLab


From d1f96d4cfe25770827b5f819b6a9de6fef142c9c Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Thu, 16 May 2024 17:51:59 +0300
Subject: [PATCH 1023/1206] [clang][NFC] Claim support for P3106R1 since Clang
 17

Test for this paper were added in https://github.com/llvm/llvm-project/pull/91435
---
 clang/www/cxx_status.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index 6e2fd745e666..a11bf9a06f9f 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -172,7 +172,7 @@ C++23, informally referred to as C++26.</p>
  <tr>
   <td>Clarifying rules for brace elision in aggregate initialization</td>
   <td><a href="https://wg21.link/P3106R1">P3106R1</a> (<a href="#dr">DR</a>)</td>
-  <td class="none" align="center">No</td>
+  <td class="full" align="center">Clang 17</td>
  </tr>
  <tr>
   <td>Attributes for Structured Bindings</td>
-- 
GitLab


From 5ac34358181b21135851979c1c949632be5a9d32 Mon Sep 17 00:00:00 2001
From: Dana Jansens <danakj@chromium.org>
Date: Thu, 16 May 2024 11:01:08 -0400
Subject: [PATCH 1024/1206] Respect the [[clang::unsafe_buffer_usage]]
 attribute for constructors (#91777)

The -Wunsafe-buffer-usage warning should fire on any call to a function
annotated with [[clang::unsafe_buffer_usage]], however it omitted calls
to constructors, since the expression is a CXXConstructExpr which does
not subclass CallExpr. Thus the matcher on callExpr() does not find
these expressions.

Add a new WarningGadget that matches cxxConstructExpr that are calling a
CXXConstructDecl annotated by [[clang::unsafe_buffer_usage]] and fires
the warning. The new UnsafeBufferUsageCtorAttrGadget gadget explicitly
avoids matching against the std::span(ptr, size) constructor because
that is handled by SpanTwoParamConstructorGadget and we never want two
gadgets to match the same thing (and this is guarded by asserts).

The gadgets themselves do not report the warnings, instead each gadget's
Stmt is passed to the UnsafeBufferUsageHandler (implemented by
UnsafeBufferUsageReporter). The Reporter is previously hardcoded that a
CXXConstructExpr statement must be a match for std::span(ptr, size), but
that is no longer the case. We want the Reporter to generate different
warnings (in the -Wunsafe-buffer-usage-in-container subgroup) for the
span contructor. And we will want it to report more warnings for other
std-container-specific gadgets in the future. To handle this we allow
the gadget to control if the warning is general (it calls
handleUnsafeBufferUsage()) or is a std-container-specific warning (it
calls handleUnsafeOperationInContainer()).

Then the WarningGadget grows a virtual method to dispatch to the
appropriate path in the UnsafeBufferUsageHandler. By doing so, we no
longer need getBaseStmt in the Gadget interface. The only use of it for
FixableGadgets was to get the SourceLocation, so we make an explicit
virtual method for that on Gadget. Then the handleUnsafeOperation()
dispatcher can be a virtual method that is only in WarningGadget.

The SpanTwoParamConstructorGadget gadget dispatches to
handleUnsafeOperationInContainer() while the other WarningGadgets all
dispatch to the original handleUnsafeBufferUsage().

Tests are added for annotated constructors, conversion operattors, call
operators, fold expressions, and regular methods.

Issue #80482
---
 .../Analysis/Analyses/UnsafeBufferUsage.h     |   5 +
 .../Analyses/UnsafeBufferUsageGadgets.def     |   1 +
 clang/lib/Analysis/UnsafeBufferUsage.cpp      | 166 ++++++++++++------
 clang/lib/Sema/AnalysisBasedWarnings.cpp      |  25 ++-
 ...warn-unsafe-buffer-usage-function-attr.cpp |  38 ++++
 5 files changed, 178 insertions(+), 57 deletions(-)

diff --git a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsage.h b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsage.h
index 5d16dcc824c5..228b4ae1e3e1 100644
--- a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsage.h
+++ b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsage.h
@@ -106,6 +106,11 @@ public:
   virtual void handleUnsafeOperation(const Stmt *Operation,
                                      bool IsRelatedToDecl, ASTContext &Ctx) = 0;
 
+  /// Invoked when an unsafe operation with a std container is found.
+  virtual void handleUnsafeOperationInContainer(const Stmt *Operation,
+                                                bool IsRelatedToDecl,
+                                                ASTContext &Ctx) = 0;
+
   /// Invoked when a fix is suggested against a variable. This function groups
   /// all variables that must be fixed together (i.e their types must be changed
   /// to the same target type to prevent type mismatches) into a single fixit.
diff --git a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def
index 3273c642eed5..242ad763ba62 100644
--- a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def
+++ b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def
@@ -36,6 +36,7 @@ WARNING_GADGET(Decrement)
 WARNING_GADGET(ArraySubscript)
 WARNING_GADGET(PointerArithmetic)
 WARNING_GADGET(UnsafeBufferUsageAttr)
+WARNING_GADGET(UnsafeBufferUsageCtorAttr)
 WARNING_GADGET(DataInvocation)
 WARNING_CONTAINER_GADGET(SpanTwoParamConstructor) // Uses of `std::span(arg0, arg1)`
 FIXABLE_GADGET(ULCArraySubscript)          // `DRE[any]` in an Unspecified Lvalue Context
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index c42e70d5b95a..866222380974 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -492,7 +492,9 @@ public:
 #endif
 
   virtual bool isWarningGadget() const = 0;
-  virtual const Stmt *getBaseStmt() const = 0;
+  // TODO remove this method from WarningGadget interface. It's only used for
+  // debug prints in FixableGadget.
+  virtual SourceLocation getSourceLoc() const = 0;
 
   /// Returns the list of pointer-type variables on which this gadget performs
   /// its operation. Typically, there's only one variable. This isn't a list
@@ -513,6 +515,10 @@ public:
 
   static bool classof(const Gadget *G) { return G->isWarningGadget(); }
   bool isWarningGadget() const final { return true; }
+
+  virtual void handleUnsafeOperation(UnsafeBufferUsageHandler &Handler,
+                                     bool IsRelatedToDecl,
+                                     ASTContext &Ctx) const = 0;
 };
 
 /// Fixable gadgets correspond to code patterns that aren't always unsafe but
@@ -572,7 +578,12 @@ public:
             .bind(OpTag));
   }
 
-  const UnaryOperator *getBaseStmt() const override { return Op; }
+  void handleUnsafeOperation(UnsafeBufferUsageHandler &Handler,
+                             bool IsRelatedToDecl,
+                             ASTContext &Ctx) const override {
+    Handler.handleUnsafeOperation(Op, IsRelatedToDecl, Ctx);
+  }
+  SourceLocation getSourceLoc() const override { return Op->getBeginLoc(); }
 
   DeclUseList getClaimedVarUseSites() const override {
     SmallVector<const DeclRefExpr *, 2> Uses;
@@ -607,7 +618,12 @@ public:
             .bind(OpTag));
   }
 
-  const UnaryOperator *getBaseStmt() const override { return Op; }
+  void handleUnsafeOperation(UnsafeBufferUsageHandler &Handler,
+                             bool IsRelatedToDecl,
+                             ASTContext &Ctx) const override {
+    Handler.handleUnsafeOperation(Op, IsRelatedToDecl, Ctx);
+  }
+  SourceLocation getSourceLoc() const override { return Op->getBeginLoc(); }
 
   DeclUseList getClaimedVarUseSites() const override {
     if (const auto *DRE =
@@ -648,7 +664,12 @@ public:
     // clang-format on
   }
 
-  const ArraySubscriptExpr *getBaseStmt() const override { return ASE; }
+  void handleUnsafeOperation(UnsafeBufferUsageHandler &Handler,
+                             bool IsRelatedToDecl,
+                             ASTContext &Ctx) const override {
+    Handler.handleUnsafeOperation(ASE, IsRelatedToDecl, Ctx);
+  }
+  SourceLocation getSourceLoc() const override { return ASE->getBeginLoc(); }
 
   DeclUseList getClaimedVarUseSites() const override {
     if (const auto *DRE =
@@ -696,7 +717,12 @@ public:
                     .bind(PointerArithmeticTag));
   }
 
-  const Stmt *getBaseStmt() const override { return PA; }
+  void handleUnsafeOperation(UnsafeBufferUsageHandler &Handler,
+                             bool IsRelatedToDecl,
+                             ASTContext &Ctx) const override {
+    Handler.handleUnsafeOperation(PA, IsRelatedToDecl, Ctx);
+  }
+  SourceLocation getSourceLoc() const override { return PA->getBeginLoc(); }
 
   DeclUseList getClaimedVarUseSites() const override {
     if (const auto *DRE = dyn_cast<DeclRefExpr>(Ptr->IgnoreParenImpCasts())) {
@@ -734,7 +760,12 @@ public:
                     .bind(SpanTwoParamConstructorTag));
   }
 
-  const Stmt *getBaseStmt() const override { return Ctor; }
+  void handleUnsafeOperation(UnsafeBufferUsageHandler &Handler,
+                             bool IsRelatedToDecl,
+                             ASTContext &Ctx) const override {
+    Handler.handleUnsafeOperationInContainer(Ctor, IsRelatedToDecl, Ctx);
+  }
+  SourceLocation getSourceLoc() const override { return Ctor->getBeginLoc(); }
 
   DeclUseList getClaimedVarUseSites() const override {
     // If the constructor call is of the form `std::span{var, n}`, `var` is
@@ -780,11 +811,8 @@ public:
 
   virtual std::optional<FixItList>
   getFixits(const FixitStrategy &S) const override;
-
-  virtual const Stmt *getBaseStmt() const override {
-    // FIXME: This needs to be the entire DeclStmt, assuming that this method
-    // makes sense at all on a FixableGadget.
-    return PtrInitRHS;
+  SourceLocation getSourceLoc() const override {
+    return PtrInitRHS->getBeginLoc();
   }
 
   virtual DeclUseList getClaimedVarUseSites() const override {
@@ -833,12 +861,7 @@ public:
 
   virtual std::optional<FixItList>
   getFixits(const FixitStrategy &S) const override;
-
-  virtual const Stmt *getBaseStmt() const override {
-    // FIXME: This should be the binary operator, assuming that this method
-    // makes sense at all on a FixableGadget.
-    return PtrLHS;
-  }
+  SourceLocation getSourceLoc() const override { return PtrLHS->getBeginLoc(); }
 
   virtual DeclUseList getClaimedVarUseSites() const override {
     return DeclUseList{PtrLHS, PtrRHS};
@@ -888,12 +911,7 @@ public:
 
   virtual std::optional<FixItList>
   getFixits(const FixitStrategy &S) const override;
-
-  virtual const Stmt *getBaseStmt() const override {
-    // FIXME: This should be the binary operator, assuming that this method
-    // makes sense at all on a FixableGadget.
-    return PtrLHS;
-  }
+  SourceLocation getSourceLoc() const override { return PtrLHS->getBeginLoc(); }
 
   virtual DeclUseList getClaimedVarUseSites() const override {
     return DeclUseList{PtrLHS, PtrRHS};
@@ -921,10 +939,53 @@ public:
   }
 
   static Matcher matcher() {
-    return stmt(callExpr(callee(functionDecl(hasAttr(attr::UnsafeBufferUsage))))
-                    .bind(OpTag));
+    auto HasUnsafeFnDecl =
+        callee(functionDecl(hasAttr(attr::UnsafeBufferUsage)));
+    return stmt(callExpr(HasUnsafeFnDecl).bind(OpTag));
+  }
+
+  void handleUnsafeOperation(UnsafeBufferUsageHandler &Handler,
+                             bool IsRelatedToDecl,
+                             ASTContext &Ctx) const override {
+    Handler.handleUnsafeOperation(Op, IsRelatedToDecl, Ctx);
   }
-  const Stmt *getBaseStmt() const override { return Op; }
+  SourceLocation getSourceLoc() const override { return Op->getBeginLoc(); }
+
+  DeclUseList getClaimedVarUseSites() const override { return {}; }
+};
+
+/// A call of a constructor that performs unchecked buffer operations
+/// over one of its pointer parameters, or constructs a class object that will
+/// perform buffer operations that depend on the correctness of the parameters.
+class UnsafeBufferUsageCtorAttrGadget : public WarningGadget {
+  constexpr static const char *const OpTag = "cxx_construct_expr";
+  const CXXConstructExpr *Op;
+
+public:
+  UnsafeBufferUsageCtorAttrGadget(const MatchFinder::MatchResult &Result)
+      : WarningGadget(Kind::UnsafeBufferUsageCtorAttr),
+        Op(Result.Nodes.getNodeAs<CXXConstructExpr>(OpTag)) {}
+
+  static bool classof(const Gadget *G) {
+    return G->getKind() == Kind::UnsafeBufferUsageCtorAttr;
+  }
+
+  static Matcher matcher() {
+    auto HasUnsafeCtorDecl =
+        hasDeclaration(cxxConstructorDecl(hasAttr(attr::UnsafeBufferUsage)));
+    // std::span(ptr, size) ctor is handled by SpanTwoParamConstructorGadget.
+    auto HasTwoParamSpanCtorDecl = SpanTwoParamConstructorGadget::matcher();
+    return stmt(
+        cxxConstructExpr(HasUnsafeCtorDecl, unless(HasTwoParamSpanCtorDecl))
+            .bind(OpTag));
+  }
+
+  void handleUnsafeOperation(UnsafeBufferUsageHandler &Handler,
+                             bool IsRelatedToDecl,
+                             ASTContext &Ctx) const override {
+    Handler.handleUnsafeOperation(Op, IsRelatedToDecl, Ctx);
+  }
+  SourceLocation getSourceLoc() const override { return Op->getBeginLoc(); }
 
   DeclUseList getClaimedVarUseSites() const override { return {}; }
 };
@@ -953,7 +1014,13 @@ public:
         explicitCastExpr(anyOf(has(callExpr), has(parenExpr(has(callExpr)))))
             .bind(OpTag));
   }
-  const Stmt *getBaseStmt() const override { return Op; }
+
+  void handleUnsafeOperation(UnsafeBufferUsageHandler &Handler,
+                             bool IsRelatedToDecl,
+                             ASTContext &Ctx) const override {
+    Handler.handleUnsafeOperation(Op, IsRelatedToDecl, Ctx);
+  }
+  SourceLocation getSourceLoc() const override { return Op->getBeginLoc(); }
 
   DeclUseList getClaimedVarUseSites() const override { return {}; }
 };
@@ -990,8 +1057,7 @@ public:
 
   virtual std::optional<FixItList>
   getFixits(const FixitStrategy &S) const override;
-
-  virtual const Stmt *getBaseStmt() const override { return Node; }
+  SourceLocation getSourceLoc() const override { return Node->getBeginLoc(); }
 
   virtual DeclUseList getClaimedVarUseSites() const override {
     if (const auto *DRE =
@@ -1031,8 +1097,7 @@ public:
 
   virtual std::optional<FixItList>
   getFixits(const FixitStrategy &S) const override;
-
-  virtual const Stmt *getBaseStmt() const override { return Node; }
+  SourceLocation getSourceLoc() const override { return Node->getBeginLoc(); }
 
   virtual DeclUseList getClaimedVarUseSites() const override { return {Node}; }
 };
@@ -1070,10 +1135,9 @@ public:
     return {BaseDeclRefExpr};
   }
 
-  virtual const Stmt *getBaseStmt() const final { return Op; }
-
   virtual std::optional<FixItList>
   getFixits(const FixitStrategy &S) const override;
+  SourceLocation getSourceLoc() const override { return Op->getBeginLoc(); }
 };
 
 // Represents expressions of the form `&DRE[any]` in the Unspecified Pointer
@@ -1108,8 +1172,7 @@ public:
 
   virtual std::optional<FixItList>
   getFixits(const FixitStrategy &) const override;
-
-  virtual const Stmt *getBaseStmt() const override { return Node; }
+  SourceLocation getSourceLoc() const override { return Node->getBeginLoc(); }
 
   virtual DeclUseList getClaimedVarUseSites() const override {
     const auto *ArraySubst = cast<ArraySubscriptExpr>(Node->getSubExpr());
@@ -1218,8 +1281,7 @@ public:
 
   virtual std::optional<FixItList>
   getFixits(const FixitStrategy &S) const override;
-
-  virtual const Stmt *getBaseStmt() const override { return Node; }
+  SourceLocation getSourceLoc() const override { return Node->getBeginLoc(); }
 
   virtual DeclUseList getClaimedVarUseSites() const override {
     return {dyn_cast<DeclRefExpr>(Node->getSubExpr())};
@@ -1264,8 +1326,7 @@ public:
 
   virtual std::optional<FixItList>
   getFixits(const FixitStrategy &S) const override;
-
-  virtual const Stmt *getBaseStmt() const override { return Node; }
+  SourceLocation getSourceLoc() const override { return Node->getBeginLoc(); }
 
   virtual DeclUseList getClaimedVarUseSites() const override {
     return {dyn_cast<DeclRefExpr>(Node->getLHS())};
@@ -1315,9 +1376,9 @@ public:
 
   virtual std::optional<FixItList>
   getFixits(const FixitStrategy &s) const final;
-
-  // TODO remove this method from FixableGadget interface
-  virtual const Stmt *getBaseStmt() const final { return nullptr; }
+  SourceLocation getSourceLoc() const override {
+    return DerefOp->getBeginLoc();
+  }
 
   virtual DeclUseList getClaimedVarUseSites() const final {
     return {BaseDeclRefExpr};
@@ -2070,7 +2131,7 @@ UUCAddAssignGadget::getFixits(const FixitStrategy &S) const {
     if (S.lookup(VD) == FixitStrategy::Kind::Span) {
       FixItList Fixes;
 
-      const Stmt *AddAssignNode = getBaseStmt();
+      const Stmt *AddAssignNode = Node;
       StringRef varName = VD->getName();
       const ASTContext &Ctx = VD->getASTContext();
 
@@ -2112,7 +2173,6 @@ UPCPreIncrementGadget::getFixits(const FixitStrategy &S) const {
     if (S.lookup(VD) == FixitStrategy::Kind::Span) {
       FixItList Fixes;
       std::stringstream SS;
-      const Stmt *PreIncNode = getBaseStmt();
       StringRef varName = VD->getName();
       const ASTContext &Ctx = VD->getASTContext();
 
@@ -2120,12 +2180,12 @@ UPCPreIncrementGadget::getFixits(const FixitStrategy &S) const {
       SS << "(" << varName.data() << " = " << varName.data()
          << ".subspan(1)).data()";
       std::optional<SourceLocation> PreIncLocation =
-          getEndCharLoc(PreIncNode, Ctx.getSourceManager(), Ctx.getLangOpts());
+          getEndCharLoc(Node, Ctx.getSourceManager(), Ctx.getLangOpts());
       if (!PreIncLocation)
         return std::nullopt;
 
       Fixes.push_back(FixItHint::CreateReplacement(
-          SourceRange(PreIncNode->getBeginLoc(), *PreIncLocation), SS.str()));
+          SourceRange(Node->getBeginLoc(), *PreIncLocation), SS.str()));
       return Fixes;
     }
   }
@@ -2856,7 +2916,7 @@ getFixIts(FixableGadgetSets &FixablesForAllVars, const FixitStrategy &S,
       }
 #ifndef NDEBUG
       Handler.addDebugNoteForVar(
-          VD, F->getBaseStmt()->getBeginLoc(),
+          VD, F->getSourceLoc(),
           ("gadget '" + F->getDebugName() + "' refused to produce a fix")
               .str());
 #endif
@@ -3008,8 +3068,8 @@ void clang::checkUnsafeBufferUsage(const Decl *D,
     // every problematic operation and consider it done. No need to deal
     // with fixable gadgets, no need to group operations by variable.
     for (const auto &G : WarningGadgets) {
-      Handler.handleUnsafeOperation(G->getBaseStmt(), /*IsRelatedToDecl=*/false,
-                                    D->getASTContext());
+      G->handleUnsafeOperation(Handler, /*IsRelatedToDecl=*/false,
+                               D->getASTContext());
     }
 
     // This return guarantees that most of the machine doesn't run when
@@ -3251,8 +3311,8 @@ void clang::checkUnsafeBufferUsage(const Decl *D,
                   Tracker, Handler, VarGrpMgr);
 
   for (const auto &G : UnsafeOps.noVar) {
-    Handler.handleUnsafeOperation(G->getBaseStmt(), /*IsRelatedToDecl=*/false,
-                                  D->getASTContext());
+    G->handleUnsafeOperation(Handler, /*IsRelatedToDecl=*/false,
+                             D->getASTContext());
   }
 
   for (const auto &[VD, WarningGadgets] : UnsafeOps.byVar) {
@@ -3263,8 +3323,8 @@ void clang::checkUnsafeBufferUsage(const Decl *D,
                                           : FixItList{},
                                       D, NaiveStrategy);
     for (const auto &G : WarningGadgets) {
-      Handler.handleUnsafeOperation(G->getBaseStmt(), /*IsRelatedToDecl=*/true,
-                                    D->getASTContext());
+      G->handleUnsafeOperation(Handler, /*IsRelatedToDecl=*/true,
+                               D->getASTContext());
     }
   }
 }
diff --git a/clang/lib/Sema/AnalysisBasedWarnings.cpp b/clang/lib/Sema/AnalysisBasedWarnings.cpp
index 6992ba9ad9a7..b9d0b59ef1db 100644
--- a/clang/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/clang/lib/Sema/AnalysisBasedWarnings.cpp
@@ -2256,11 +2256,8 @@ public:
         Range = UO->getSubExpr()->getSourceRange();
         MsgParam = 1;
       }
-    } else if (const auto *CtorExpr = dyn_cast<CXXConstructExpr>(Operation)) {
-      S.Diag(CtorExpr->getLocation(),
-             diag::warn_unsafe_buffer_usage_in_container);
     } else {
-      if (isa<CallExpr>(Operation)) {
+      if (isa<CallExpr>(Operation) || isa<CXXConstructExpr>(Operation)) {
         // note_unsafe_buffer_operation doesn't have this mode yet.
         assert(!IsRelatedToDecl && "Not implemented yet!");
         MsgParam = 3;
@@ -2295,6 +2292,26 @@ public:
     }
   }
 
+  void handleUnsafeOperationInContainer(const Stmt *Operation,
+                                        bool IsRelatedToDecl,
+                                        ASTContext &Ctx) override {
+    SourceLocation Loc;
+    SourceRange Range;
+    unsigned MsgParam = 0;
+
+    // This function only handles SpanTwoParamConstructorGadget so far, which
+    // always gives a CXXConstructExpr.
+    const auto *CtorExpr = cast<CXXConstructExpr>(Operation);
+    Loc = CtorExpr->getLocation();
+
+    S.Diag(Loc, diag::warn_unsafe_buffer_usage_in_container);
+    if (IsRelatedToDecl) {
+      assert(!SuggestSuggestions &&
+             "Variables blamed for unsafe buffer usage without suggestions!");
+      S.Diag(Loc, diag::note_unsafe_buffer_operation) << MsgParam << Range;
+    }
+  }
+
   void handleUnsafeVariableGroup(const VarDecl *Variable,
                                  const VariableGroupsManager &VarGrpMgr,
                                  FixItList &&Fixes, const Decl *D,
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp
index 7df01c46438c..bfc34b55c1f6 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-function-attr.cpp
@@ -85,3 +85,41 @@ void testInheritance() {
     BC->func();  // expected-warning{{function introduces unsafe buffer manipulation}}
     BC->func1();
 }
+
+class UnsafeMembers {
+public:
+    UnsafeMembers() {}
+
+    [[clang::unsafe_buffer_usage]]
+    UnsafeMembers(int) {}
+
+    [[clang::unsafe_buffer_usage]]
+    explicit operator int() { return 0; }
+
+    [[clang::unsafe_buffer_usage]]
+    void Method() {}
+
+    [[clang::unsafe_buffer_usage]]
+    void operator()() {}
+
+    [[clang::unsafe_buffer_usage]]
+    int operator+(UnsafeMembers) { return 0; }
+};
+
+template <class... Vs>
+int testFoldExpression(Vs&&... v) {
+    return (... + v);  // expected-warning{{function introduces unsafe buffer manipulation}}
+}
+
+// https://github.com/llvm/llvm-project/issues/80482
+void testClassMembers() {
+    UnsafeMembers(3);  // expected-warning{{function introduces unsafe buffer manipulation}}
+
+    (void)static_cast<int>(UnsafeMembers());  // expected-warning{{function introduces unsafe buffer manipulation}}
+
+    UnsafeMembers().Method();  // expected-warning{{function introduces unsafe buffer manipulation}}
+
+    UnsafeMembers()();  // expected-warning{{function introduces unsafe buffer manipulation}}
+
+    testFoldExpression(UnsafeMembers(), UnsafeMembers());
+}
-- 
GitLab


From ee407e17a3a4986bab49272665abc9973d78691d Mon Sep 17 00:00:00 2001
From: Renaud Kauffmann <50748669+Renaud-K@users.noreply.github.com>
Date: Thu, 16 May 2024 08:49:44 -0700
Subject: [PATCH 1025/1206] [flang] AliasAnalysis: More formally define and
 distinguish between data and non-data  (#91020)

This PR is an implementation for changes proposed in
https://discourse.llvm.org/t/rfc-distinguish-between-data-and-non-data-in-fir-alias-analysis/78759

Test updates were made when the query was on the wrong reference. So, it
is my hope that this will clear ambiguity on the nature of the queries
from here on.
There are also some TODOs that were addressed.

It also partly implements what
https://github.com/llvm/llvm-project/pull/87723 is attempting to
accomplish. At least, on a point-to-point query between references, the
distinction is made. To apply it to TBAA, would be another PR.

Note that, the changes were minimal in the TBAA code to retain the
current results.
---
 .../flang/Optimizer/Analysis/AliasAnalysis.h  |  95 ++++++++++-
 .../lib/Optimizer/Analysis/AliasAnalysis.cpp  | 157 +++++++-----------
 .../lib/Optimizer/Transforms/AddAliasTags.cpp |  17 +-
 .../AliasAnalysis/alias-analysis-2.fir        |  37 +++--
 .../AliasAnalysis/alias-analysis-3.fir        |   4 +-
 .../AliasAnalysis/alias-analysis-9.fir        |  59 +++++++
 6 files changed, 248 insertions(+), 121 deletions(-)
 create mode 100644 flang/test/Analysis/AliasAnalysis/alias-analysis-9.fir

diff --git a/flang/include/flang/Optimizer/Analysis/AliasAnalysis.h b/flang/include/flang/Optimizer/Analysis/AliasAnalysis.h
index dfcafe88fee1..40fd1705115a 100644
--- a/flang/include/flang/Optimizer/Analysis/AliasAnalysis.h
+++ b/flang/include/flang/Optimizer/Analysis/AliasAnalysis.h
@@ -36,9 +36,6 @@ struct AliasAnalysis {
              /// Represents memory allocated outside of a function
              /// and passed to the function via host association tuple.
              HostAssoc,
-             /// Represents direct memory access whose source cannot be further
-             /// determined
-             Direct,
              /// Represents memory allocated by unknown means and
              /// with the memory address defined by a memory reading
              /// operation (e.g. fir::LoadOp).
@@ -50,12 +47,85 @@ struct AliasAnalysis {
   /// Attributes of the memory source object.
   ENUM_CLASS(Attribute, Target, Pointer, IntentIn);
 
+  // See
+  // https://discourse.llvm.org/t/rfc-distinguish-between-data-and-non-data-in-fir-alias-analysis/78759/1
+  //
+  // It is possible, while following the source of a memory reference through
+  // the use-def chain, to arrive at the same origin, even though the starting
+  // points were known to not alias.
+  //
+  // clang-format off
+  // Example:
+  //  ------------------- test.f90 --------------------
+  //  module top
+  //    real, pointer :: a(:)
+  //  end module
+  //  
+  //  subroutine test()
+  //    use top
+  //    a(1) = 1
+  //  end subroutine
+  //  -------------------------------------------------
+  // 
+  //  flang-new -fc1 -emit-fir test.f90 -o test.fir
+  //
+  //  ------------------- test.fir --------------------
+  //  fir.global @_QMtopEa : !fir.box<!fir.ptr<!fir.array<?xf32>>> 
+  //  
+  //  func.func @_QPtest() {
+  //    %c1 = arith.constant 1 : index
+  //    %cst = arith.constant 1.000000e+00 : f32
+  //    %0 = fir.address_of(@_QMtopEa) : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+  //    %1 = fir.declare %0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMtopEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+  //    %2 = fir.load %1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+  //    ...
+  //    %5 = fir.array_coor %2 %c1 : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, !fir.shift<1>, index) -> !fir.ref<f32>
+  //    fir.store %cst to %5 : !fir.ref<f32>
+  //    return
+  //  }
+  //  -------------------------------------------------
+  //
+  // With high level operations, such as fir.array_coor, it is possible to
+  // reach into the data wrapped by the box (the descriptor). Therefore when
+  // asking about the memory source of %5, we are really asking about the
+  // source of the data of box %2.
+  //
+  // When asking about the source of %0 which is the address of the box, we
+  // reach the same source as in the first case: the global @_QMtopEa. Yet one
+  // source refers to the data while the other refers to the address of the box
+  // itself.
+  //
+  // To distinguish between the two, the isData flag has been added, whereby
+  // data is defined as any memory reference that is not a box reference.
+  // Additionally, because it is relied on in HLFIR lowering, we allow querying
+  // on a box SSA value, which is interpreted as querying on its data.
+  //
+  // So in the above example, !fir.ref<f32> and !fir.box<!fir.ptr<!fir.array<?xf32>>> is data, 
+  // while !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>> is not data.
+
+  // This also applies to function arguments. In the example below, %arg0
+  // is data, %arg1 is not data but a load of %arg1 is.
+  // 
+  // func.func @_QFPtest2(%arg0: !fir.ref<f32>, %arg1: !fir.ref<!fir.box<!fir.ptr<f32>>> )  {
+  //    %0 = fir.load %arg1 : !fir.ref<!fir.box<!fir.ptr<f32>>>
+  //    ... }
+  //
+  // clang-format on
+
   struct Source {
     using SourceUnion = llvm::PointerUnion<mlir::SymbolRefAttr, mlir::Value>;
     using Attributes = Fortran::common::EnumSet<Attribute, Attribute_enumSize>;
 
-    /// Source definition of a value.
-    SourceUnion u;
+    struct SourceOrigin {
+      /// Source definition of a value.
+      SourceUnion u;
+
+      /// Whether the source was reached following data or box reference
+      bool isData{false};
+    };
+
+    SourceOrigin origin;
+
     /// Kind of the memory source.
     SourceKind kind;
     /// Value type of the source definition.
@@ -77,6 +147,12 @@ struct AliasAnalysis {
     /// attribute.
     bool isRecordWithPointerComponent() const;
 
+    bool isDummyArgument() const;
+    bool isData() const;
+    bool isBoxData() const;
+
+    mlir::Type getType() const;
+
     /// Return true, if `ty` is a reference type to a boxed
     /// POINTER object or a raw fir::PointerType.
     static bool isPointerReference(mlir::Type ty);
@@ -95,6 +171,15 @@ struct AliasAnalysis {
   Source getSource(mlir::Value);
 };
 
+inline bool operator==(const AliasAnalysis::Source::SourceOrigin &lhs,
+                       const AliasAnalysis::Source::SourceOrigin &rhs) {
+  return lhs.u == rhs.u && lhs.isData == rhs.isData;
+}
+inline bool operator!=(const AliasAnalysis::Source::SourceOrigin &lhs,
+                       const AliasAnalysis::Source::SourceOrigin &rhs) {
+  return !(lhs == rhs);
+}
+
 inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
                                      const AliasAnalysis::Source &op) {
   op.print(os);
diff --git a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
index ed1101dc5e8d..9d0d706a85c5 100644
--- a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
+++ b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
@@ -33,11 +33,9 @@ static bool isDummyArgument(mlir::Value v) {
   if (!blockArg)
     return false;
 
-  mlir::Block *owner = blockArg.getOwner();
-  if (!owner->isEntryBlock() ||
-      !mlir::isa<mlir::FunctionOpInterface>(owner->getParentOp()))
-    return false;
-  return true;
+  auto *owner{blockArg.getOwner()};
+  return owner->isEntryBlock() &&
+         mlir::isa<mlir::FunctionOpInterface>(owner->getParentOp());
 }
 
 /// Temporary function to skip through all the no op operations
@@ -58,12 +56,17 @@ static mlir::Value getOriginalDef(mlir::Value v) {
 namespace fir {
 
 void AliasAnalysis::Source::print(llvm::raw_ostream &os) const {
-  if (auto v = llvm::dyn_cast<mlir::Value>(u))
+  if (auto v = llvm::dyn_cast<mlir::Value>(origin.u))
     os << v;
-  else if (auto gbl = llvm::dyn_cast<mlir::SymbolRefAttr>(u))
+  else if (auto gbl = llvm::dyn_cast<mlir::SymbolRefAttr>(origin.u))
     os << gbl;
   os << " SourceKind: " << EnumToString(kind);
   os << " Type: " << valueType << " ";
+  if (origin.isData) {
+    os << " following data ";
+  } else {
+    os << " following box reference ";
+  }
   attributes.Dump(os, EnumToString);
 }
 
@@ -80,6 +83,19 @@ bool AliasAnalysis::Source::isTargetOrPointer() const {
          attributes.test(Attribute::Target);
 }
 
+bool AliasAnalysis::Source::isDummyArgument() const {
+  if (auto v = origin.u.dyn_cast<mlir::Value>()) {
+    return ::isDummyArgument(v);
+  }
+  return false;
+}
+
+bool AliasAnalysis::Source::isData() const { return origin.isData; }
+bool AliasAnalysis::Source::isBoxData() const {
+  return mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(valueType)) &&
+         origin.isData;
+}
+
 bool AliasAnalysis::Source::isRecordWithPointerComponent() const {
   auto eleTy = fir::dyn_cast_ptrEleTy(valueType);
   if (!eleTy)
@@ -92,70 +108,35 @@ AliasResult AliasAnalysis::alias(Value lhs, Value rhs) {
   auto lhsSrc = getSource(lhs);
   auto rhsSrc = getSource(rhs);
   bool approximateSource = lhsSrc.approximateSource || rhsSrc.approximateSource;
-  LLVM_DEBUG(llvm::dbgs() << "AliasAnalysis::alias\n";
+  LLVM_DEBUG(llvm::dbgs() << "\nAliasAnalysis::alias\n";
              llvm::dbgs() << "  lhs: " << lhs << "\n";
              llvm::dbgs() << "  lhsSrc: " << lhsSrc << "\n";
              llvm::dbgs() << "  rhs: " << rhs << "\n";
-             llvm::dbgs() << "  rhsSrc: " << rhsSrc << "\n";
-             llvm::dbgs() << "\n";);
+             llvm::dbgs() << "  rhsSrc: " << rhsSrc << "\n";);
 
   // Indirect case currently not handled. Conservatively assume
   // it aliases with everything
-  if (lhsSrc.kind > SourceKind::Direct || rhsSrc.kind > SourceKind::Direct) {
+  if (lhsSrc.kind >= SourceKind::Indirect ||
+      rhsSrc.kind >= SourceKind::Indirect) {
     return AliasResult::MayAlias;
   }
 
-  // SourceKind::Direct is set for the addresses wrapped in a global boxes.
-  // ie: fir.global @_QMpointersEp : !fir.box<!fir.ptr<f32>>
-  // Though nothing is known about them, they would only alias with targets or
-  // pointers
-  bool directSourceToNonTargetOrPointer = false;
-  if (lhsSrc.u != rhsSrc.u || lhsSrc.kind != rhsSrc.kind) {
-    if ((lhsSrc.kind == SourceKind::Direct && !rhsSrc.isTargetOrPointer()) ||
-        (rhsSrc.kind == SourceKind::Direct && !lhsSrc.isTargetOrPointer()))
-      directSourceToNonTargetOrPointer = true;
-  }
-
-  if (lhsSrc.kind == SourceKind::Direct ||
-      rhsSrc.kind == SourceKind::Direct) {
-    if (!directSourceToNonTargetOrPointer)
-      return AliasResult::MayAlias;
-  }
-
   if (lhsSrc.kind == rhsSrc.kind) {
-    if (lhsSrc.u == rhsSrc.u) {
+    if (lhsSrc.origin == rhsSrc.origin) {
+      LLVM_DEBUG(llvm::dbgs()
+                 << "  aliasing because same source kind and origin\n");
       if (approximateSource)
         return AliasResult::MayAlias;
       return AliasResult::MustAlias;
     }
 
     // Two host associated accesses may overlap due to an equivalence.
-    if (lhsSrc.kind == SourceKind::HostAssoc)
+    if (lhsSrc.kind == SourceKind::HostAssoc) {
+      LLVM_DEBUG(llvm::dbgs() << "  aliasing because of host association\n");
       return AliasResult::MayAlias;
-
-    // Allocate and global memory address cannot physically alias
-    if (lhsSrc.kind == SourceKind::Allocate ||
-        lhsSrc.kind == SourceKind::Global)
-      return AliasResult::NoAlias;
-
-    // Dummy TARGET/POINTER arguments may alias.
-    if (lhsSrc.isTargetOrPointer() && rhsSrc.isTargetOrPointer())
-      return AliasResult::MayAlias;
-
-    // Box for POINTER component inside an object of a derived type
-    // may alias box of a POINTER object, as well as boxes for POINTER
-    // components inside two objects of derived types may alias.
-    if ((lhsSrc.isRecordWithPointerComponent() && rhsSrc.isTargetOrPointer()) ||
-        (rhsSrc.isRecordWithPointerComponent() && lhsSrc.isTargetOrPointer()) ||
-        (lhsSrc.isRecordWithPointerComponent() &&
-         rhsSrc.isRecordWithPointerComponent()))
-      return AliasResult::MayAlias;
-
-    return AliasResult::NoAlias;
+    }
   }
 
-  assert(lhsSrc.kind != rhsSrc.kind && "memory source kinds must be different");
-
   Source *src1, *src2;
   if (lhsSrc.kind < rhsSrc.kind) {
     src1 = &lhsSrc;
@@ -186,8 +167,11 @@ AliasResult AliasAnalysis::alias(Value lhs, Value rhs) {
   }
 
   // Dummy TARGET/POINTER argument may alias with a global TARGET/POINTER.
-  if (src1->isTargetOrPointer() && src2->isTargetOrPointer())
+  if (src1->isTargetOrPointer() && src2->isTargetOrPointer() &&
+      src1->isData() == src2->isData()) {
+    LLVM_DEBUG(llvm::dbgs() << "  aliasing because of target or pointer\n");
     return AliasResult::MayAlias;
+  }
 
   // Box for POINTER component inside an object of a derived type
   // may alias box of a POINTER object, as well as boxes for POINTER
@@ -195,8 +179,10 @@ AliasResult AliasAnalysis::alias(Value lhs, Value rhs) {
   if ((src1->isRecordWithPointerComponent() && src2->isTargetOrPointer()) ||
       (src2->isRecordWithPointerComponent() && src1->isTargetOrPointer()) ||
       (src1->isRecordWithPointerComponent() &&
-       src2->isRecordWithPointerComponent()))
+       src2->isRecordWithPointerComponent())) {
+    LLVM_DEBUG(llvm::dbgs() << "  aliasing because of pointer components\n");
     return AliasResult::MayAlias;
+  }
 
   return AliasResult::NoAlias;
 }
@@ -262,7 +248,10 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v) {
   mlir::Type ty;
   bool breakFromLoop{false};
   bool approximateSource{false};
-  bool followBoxAddr{mlir::isa<fir::BaseBoxType>(v.getType())};
+  bool followBoxData{mlir::isa<fir::BaseBoxType>(v.getType())};
+  bool isBoxRef{fir::isa_ref_type(v.getType()) &&
+                mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(v.getType()))};
+  bool followingData = !isBoxRef;
   mlir::SymbolRefAttr global;
   Source::Attributes attributes;
   while (defOp && !breakFromLoop) {
@@ -282,34 +271,32 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v) {
           v = op->getOperand(0);
           defOp = v.getDefiningOp();
           if (mlir::isa<fir::BaseBoxType>(v.getType()))
-            followBoxAddr = true;
+            followBoxData = true;
         })
         .Case<fir::ArrayCoorOp, fir::CoordinateOp>([&](auto op) {
           v = op->getOperand(0);
           defOp = v.getDefiningOp();
           if (mlir::isa<fir::BaseBoxType>(v.getType()))
-            followBoxAddr = true;
+            followBoxData = true;
           approximateSource = true;
         })
         .Case<fir::EmboxOp, fir::ReboxOp>([&](auto op) {
-          if (followBoxAddr) {
+          if (followBoxData) {
             v = op->getOperand(0);
             defOp = v.getDefiningOp();
           } else
             breakFromLoop = true;
         })
         .Case<fir::LoadOp>([&](auto op) {
-          if (followBoxAddr && mlir::isa<fir::BaseBoxType>(op.getType())) {
-            // For now, support the load of an argument or fir.address_of
-            // TODO: generalize to all operations (in particular fir.alloca and
-            // fir.allocmem)
-            auto def = getOriginalDef(op.getMemref());
-            if (isDummyArgument(def) ||
-                def.template getDefiningOp<fir::AddrOfOp>()) {
-              v = def;
-              defOp = v.getDefiningOp();
-              return;
-            }
+          // If the load is from a leaf source, return the leaf. Do not track
+          // through indirections otherwise.
+          // TODO: Add support to fir.alloca and fir.allocmem
+          auto def = getOriginalDef(op.getMemref());
+          if (isDummyArgument(def) ||
+              def.template getDefiningOp<fir::AddrOfOp>()) {
+            v = def;
+            defOp = v.getDefiningOp();
+            return;
           }
           // No further tracking for addresses loaded from memory for now.
           type = SourceKind::Indirect;
@@ -318,24 +305,7 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v) {
         .Case<fir::AddrOfOp>([&](auto op) {
           // Address of a global scope object.
           ty = v.getType();
-
-          // When the global is a
-          // fir.global @_QMpointersEp : !fir.box<!fir.ptr<f32>>
-          //   or
-          // fir.global @_QMpointersEp : !fir.box<!fir.heap<f32>>
-          //
-          // and when following through the wrapped address, capture
-          // the fact that there is nothing known about it. Therefore setting
-          // the source to Direct.
-          //
-          // When not following the wrapped address, then consider the address
-          // of the box, which has nothing to do with the wrapped address and
-          // lies in the global memory space.
-          if (followBoxAddr &&
-              mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(ty)))
-            type = SourceKind::Direct;
-          else
-            type = SourceKind::Global;
+          type = SourceKind::Global;
 
           auto globalOpName = mlir::OperationName(
               fir::GlobalOp::getOperationName(), defOp->getContext());
@@ -343,11 +313,10 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v) {
                   v, fir::GlobalOp::getTargetAttrName(globalOpName)))
             attributes.set(Attribute::Target);
 
-          // TODO: Take followBoxAddr into account when setting the pointer
+          // TODO: Take followBoxData into account when setting the pointer
           // attribute
           if (Source::isPointerReference(ty))
             attributes.set(Attribute::Pointer);
-
           global = llvm::cast<fir::AddrOfOp>(op).getSymbol();
           breakFromLoop = true;
         })
@@ -393,7 +362,7 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v) {
           // MustAlias after going through a designate operation
           approximateSource = true;
           if (mlir::isa<fir::BaseBoxType>(v.getType()))
-            followBoxAddr = true;
+            followBoxData = true;
         })
         .Default([&](auto op) {
           defOp = nullptr;
@@ -412,10 +381,10 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v) {
         attributes.set(Attribute::Pointer);
     }
 
-  if (type == SourceKind::Global || type == SourceKind::Direct)
-    return {global, type, ty, attributes, approximateSource};
-
-  return {v, type, ty, attributes, approximateSource};
+  if (type == SourceKind::Global) {
+    return {{global, followingData}, type, ty, attributes, approximateSource};
+  }
+  return {{v, followingData}, type, ty, attributes, approximateSource};
 }
 
 } // namespace fir
diff --git a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
index 3642a812096d..caece9384004 100644
--- a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
+++ b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
@@ -145,7 +145,7 @@ void AddAliasTagsPass::runOnAliasInterface(fir::FirAliasTagOpInterface op,
       source.kind == fir::AliasAnalysis::SourceKind::Argument) {
     LLVM_DEBUG(llvm::dbgs().indent(2)
                << "Found reference to dummy argument at " << *op << "\n");
-    std::string name = getFuncArgName(source.u.get<mlir::Value>());
+    std::string name = getFuncArgName(source.origin.u.get<mlir::Value>());
     if (!name.empty())
       tag = state.getFuncTree(func).dummyArgDataTree.getTag(name);
     else
@@ -155,8 +155,9 @@ void AddAliasTagsPass::runOnAliasInterface(fir::FirAliasTagOpInterface op,
 
     // TBAA for global variables
   } else if (enableGlobals &&
-             source.kind == fir::AliasAnalysis::SourceKind::Global) {
-    mlir::SymbolRefAttr glbl = source.u.get<mlir::SymbolRefAttr>();
+             source.kind == fir::AliasAnalysis::SourceKind::Global &&
+             !source.isBoxData()) {
+    mlir::SymbolRefAttr glbl = source.origin.u.get<mlir::SymbolRefAttr>();
     const char *name = glbl.getRootReference().data();
     LLVM_DEBUG(llvm::dbgs().indent(2) << "Found reference to global " << name
                                       << " at " << *op << "\n");
@@ -164,9 +165,10 @@ void AddAliasTagsPass::runOnAliasInterface(fir::FirAliasTagOpInterface op,
 
     // TBAA for SourceKind::Direct
   } else if (enableDirect &&
-             source.kind == fir::AliasAnalysis::SourceKind::Direct) {
-    if (source.u.is<mlir::SymbolRefAttr>()) {
-      mlir::SymbolRefAttr glbl = source.u.get<mlir::SymbolRefAttr>();
+             source.kind == fir::AliasAnalysis::SourceKind::Global &&
+             source.isBoxData()) {
+    if (source.origin.u.is<mlir::SymbolRefAttr>()) {
+      mlir::SymbolRefAttr glbl = source.origin.u.get<mlir::SymbolRefAttr>();
       const char *name = glbl.getRootReference().data();
       LLVM_DEBUG(llvm::dbgs().indent(2) << "Found reference to direct " << name
                                         << " at " << *op << "\n");
@@ -182,7 +184,8 @@ void AddAliasTagsPass::runOnAliasInterface(fir::FirAliasTagOpInterface op,
   } else if (enableLocalAllocs &&
              source.kind == fir::AliasAnalysis::SourceKind::Allocate) {
     std::optional<llvm::StringRef> name;
-    mlir::Operation *sourceOp = source.u.get<mlir::Value>().getDefiningOp();
+    mlir::Operation *sourceOp =
+        source.origin.u.get<mlir::Value>().getDefiningOp();
     if (auto alloc = mlir::dyn_cast_or_null<fir::AllocaOp>(sourceOp))
       name = alloc.getUniqName();
     else if (auto alloc = mlir::dyn_cast_or_null<fir::AllocMemOp>(sourceOp))
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-2.fir b/flang/test/Analysis/AliasAnalysis/alias-analysis-2.fir
index 31459ef21d94..d03348efd2a6 100644
--- a/flang/test/Analysis/AliasAnalysis/alias-analysis-2.fir
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-2.fir
@@ -37,14 +37,14 @@
 
 // arg2 is a reference to a pointer. Modifying arg2 could
 // modify a target with a pointer component
-// CHECK-DAG: func.region0#0 <-> func.region0#2: MayAlias
-// CHECK-DAG: func.region0#1 <-> func.region0#2: MayAlias
+// CHECK-DAG: arg2.load#0 <-> func.region0#0: MayAlias
+// CHECK-DAG: arg2.load#0 <-> func.region0#1: MayAlias
 
 // However, the address wrapped by arg2, can alias with any target or
 // pointer arguments
 // CHECK-DAG: arg2.addr#0 <-> func.region0#0: MayAlias
 // CHECK-DAG: arg2.addr#0 <-> func.region0#1: MayAlias
-// CHECK-DAG: arg2.addr#0 <-> func.region0#2: MustAlias
+// CHECK-DAG: arg2.load#0 <-> arg2.addr#0: MustAlias
 // CHECK-DAG: boxp1.addr#0 <-> arg2.addr#0: MayAlias
 
 func.func @_QFPtest(%arg0: !fir.ref<f32> {fir.bindc_name = "v1", fir.target}, %arg1: !fir.ref<f32> {fir.bindc_name = "v2", fir.target}, %arg2: !fir.ref<!fir.box<!fir.ptr<f32>>> ) attributes {test.ptr = "func"} {
@@ -80,7 +80,7 @@ func.func @_QFPtest(%arg0: !fir.ref<f32> {fir.bindc_name = "v1", fir.target}, %a
   %14 = fir.box_addr %13 : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
   fir.store %14 to %4 : !fir.ref<!fir.ptr<f32>>
 
-  %15 = fir.load %arg2 : !fir.ref<!fir.box<!fir.ptr<f32>>>
+  %15 = fir.load %arg2 {test.ptr = "arg2.load"} : !fir.ref<!fir.box<!fir.ptr<f32>>>
   %16 = fir.box_addr %15 {test.ptr = "arg2.addr"} : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
   return
 }
@@ -99,10 +99,12 @@ func.func @_QFPtest(%arg0: !fir.ref<f32> {fir.bindc_name = "v1", fir.target}, %a
 // CHECK-DAG: func.region0#1 <-> func.region0#2: MayAlias
 
 // They can also modify targets that have pointer components
-// CHECK-DAG: func.region0#0 <-> func.region0#1: MayAlias
-// CHECK-DAG: func.region0#0 <-> func.region0#2: MayAlias
+// CHECK-DAG: arg1.load#0 <-> func.region0#0: MayAlias
+// CHECK-DAG: arg2.load#0 <-> func.region0#0: MayAlias
 
 func.func @_QFPtest2(%arg0: !fir.ref<f32> {fir.bindc_name = "v1", fir.target}, %arg1: !fir.ref<!fir.box<!fir.ptr<f32>>>, %arg2: !fir.ref<!fir.box<!fir.ptr<f32>>> ) attributes {test.ptr = "func"} {
+  %0 = fir.load %arg1 {test.ptr = "arg1.load"} : !fir.ref<!fir.box<!fir.ptr<f32>>>
+  %1 = fir.load %arg2 {test.ptr = "arg2.load"} : !fir.ref<!fir.box<!fir.ptr<f32>>>
   return
 }
 
@@ -141,14 +143,14 @@ func.func @_QFPtest2(%arg0: !fir.ref<f32> {fir.bindc_name = "v1", fir.target}, %
 // alias with the wrapped scalar _QFEvar2. We meant box_addr of _QMpointersEp 
 // CHECK-DAG: p#0 <-> box.addr#0: NoAlias
 
-// TODO: Still need to handle more gracefully the difference between !fir.ref<!fir.box<>> and !fir.box<> 
-// CHECK-DAG: box.addr#0 <-> func.region0#0: MayAlias
+// Handling gracefully the difference between !fir.ref<!fir.box<>> and !fir.box<> 
+// CHECK-DAG: box.addr#0 <-> func.region0#0: NoAlias
 
 // var2, although it is a target, cannot alias with p
 // A modification of p would only make them point to a new target but not modify it
 // CHECK-DAG: var2#0 <-> p#0: NoAlias
 // It can alias with p1, if p1 is a pointer component
-// CHECK-DAG: var2#0 <-> func.region0#0: MayAlias
+// CHECK-DAG: arg0.load#0 <-> var2#0: MayAlias
 // It is the same as box.addr
 // CHECK-DAG: var2#0 <-> box.addr#0: MustAlias
 
@@ -173,6 +175,7 @@ fir.global internal @_QFEvar2 target : f32 {
 }
 
 func.func @_QFPtest3(%arg0: !fir.ref<!fir.box<!fir.ptr<f32>>> {fir.bindc_name = "p1"}, %arg1: !fir.ref<f32>) attributes {test.ptr = "func"} {
+  %3 = fir.load %arg0 {test.ptr = "arg0.load"}: !fir.ref<!fir.box<!fir.ptr<f32>>>
   %4 = fir.address_of(@_QFEvar2) {test.ptr = "var2"} : !fir.ref<f32>
   %5 = fir.address_of(@_QMpointersEp) {test.ptr = "p"} : !fir.ref<!fir.box<!fir.ptr<f32>>>
   %6 = fir.embox %4 : (!fir.ref<f32>) -> !fir.box<!fir.ptr<f32>>
@@ -214,10 +217,17 @@ func.func @_QFPtest3(%arg0: !fir.ref<!fir.box<!fir.ptr<f32>>> {fir.bindc_name =
 // CHECK-DAG: var2_hlfir#1 <-> p_hlfir#0: NoAlias
 // CHECK-DAG: var2_hlfir#1 <-> p_hlfir#1: NoAlias
 
-// CHECK-DAG: var2#0 <-> func.region0#0: MayAlias
-// CHECK-DAG: var2_fir#0 <-> func.region0#0: MayAlias
-// CHECK-DAG: var2_hlfir#0 <-> func.region0#0: MayAlias
-// CHECK-DAG: var2_hlfir#1 <-> func.region0#0: MayAlias
+// The data cannot alias with the box references
+// CHECK-DAG: var2#0 <-> func.region0#0: NoAlias
+// CHECK-DAG: var2_fir#0 <-> func.region0#0: NoAlias
+// CHECK-DAG: var2_hlfir#0 <-> func.region0#0: NoAlias
+// CHECK-DAG: var2_hlfir#1 <-> func.region0#0: NoAlias
+
+// But it can alias with the box's own data
+// CHECK-DAG: arg0.load#0 <-> var2#0: MayAlias
+// CHECK-DAG: arg0.load#0 <-> var2_fir#0: MayAlias
+// CHECK-DAG: arg0.load#0 <-> var2_hlfir#0: MayAlias
+// CHECK-DAG: arg0.load#0 <-> var2_hlfir#1: MayAlias
 
 // CHECK-DAG: var2#0 <-> box.addr#0: MustAlias
 // CHECK-DAG: var2#0 <-> box.addr_fir#0: MustAlias
@@ -255,6 +265,7 @@ fir.global internal @_QFEvar2 target : f32 {
 }
 
 func.func @_QFPtest4(%arg0: !fir.ref<!fir.box<!fir.ptr<f32>>> {fir.bindc_name = "p1"}, %arg1: !fir.ref<f32>) attributes {test.ptr = "func"} {
+  %3 = fir.load %arg0 {test.ptr = "arg0.load"} : !fir.ref<!fir.box<!fir.ptr<f32>>>
   %4 = fir.address_of(@_QFEvar2) {test.ptr = "var2"} : !fir.ref<f32>
   %fir_decl_var2 = fir.declare %4 {uniq_name = "var2_fir", test.ptr = "var2_fir"}: (!fir.ref<f32>) -> !fir.ref<f32>
   %hlfir_decl_var2:2 = hlfir.declare %4 {uniq_name = "var2_hlfir", test.ptr = "var2_hlfir"}: (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-3.fir b/flang/test/Analysis/AliasAnalysis/alias-analysis-3.fir
index d7bd970b5190..91829a461dc7 100644
--- a/flang/test/Analysis/AliasAnalysis/alias-analysis-3.fir
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-3.fir
@@ -23,8 +23,8 @@
 // FIXME: a's box cannot alias with raw reference to f32 (x), so MayAlias below must be NoAlias:
 // CHECK: a#0 <-> func.region0#1: MayAlias
 
-// FIXME: pointer_dummy's box cannot alias with raw reference to f32 (x), so MayAlias below must be NoAlias:
-// CHECK: func.region0#0 <-> func.region0#1: MayAlias
+// pointer_dummy's box cannot alias with raw reference to f32 (x)
+// CHECK: func.region0#0 <-> func.region0#1: NoAlias
 
 fir.global @_QMmEa : !fir.type<_QMmTt{pointer_component:!fir.box<!fir.ptr<f32>>}> {
   %0 = fir.undefined !fir.type<_QMmTt{pointer_component:!fir.box<!fir.ptr<f32>>}>
diff --git a/flang/test/Analysis/AliasAnalysis/alias-analysis-9.fir b/flang/test/Analysis/AliasAnalysis/alias-analysis-9.fir
new file mode 100644
index 000000000000..df24a6d32aa5
--- /dev/null
+++ b/flang/test/Analysis/AliasAnalysis/alias-analysis-9.fir
@@ -0,0 +1,59 @@
+// RUN: fir-opt %s -pass-pipeline='builtin.module(func.func(test-fir-alias-analysis))' 2>&1 | FileCheck %s
+
+
+// module m
+// type t
+//  type(t), pointer :: next
+//  integer :: i
+// end type
+// contains
+// subroutine foo(x, y)
+//   type(t) :: x, y
+//   integer :: i1, i2
+//   i1 = x%next%i
+//   x = y
+//   i2 = x%next%i
+// end subroutine
+// end module
+
+// CHECK-LABEL: Testing : "_QMmPfoo"
+// TODO: x and y are non pointer, non target argument and therefore do not alias.
+// CHECK-DAG: x#0 <-> y#0: MayAlias
+
+// TODO: y is not a pointer object and therefore does not alias with the x%next component. 
+//       Also assigning x to y would not modify x.next
+// CHECK-DAG: y#0 <-> xnext1#0: MayAlias
+// CHECK-DAG: y#0 <-> xnext2#0: MayAlias
+
+// We need to catch the fact that assigning y to x will modify xnext. 
+// The only side-effect between the 2 loads of x.next is the assignment to x, 
+// therefore x needs to alias with x.next to prevent the loads from being merged.
+// CHECK-DAG: x#0 <-> xnext1#0: MayAlias
+// CHECK-DAG: x#0 <-> xnext2#0: MayAlias
+
+// TODO: xnext1#0 <-> xnext2#0 are the same and therefore MustAlias but 
+//       we are currently not comparing operands involved in offset computations
+// CHECK-DAG: xnext1#0 <-> xnext2#0: MayAlias
+
+func.func @_QMmPfoo(%arg0: !fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>> {fir.bindc_name = "x"}, %arg1: !fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>> {fir.bindc_name = "y"}) {
+  %0 = fir.alloca i32 {bindc_name = "i1", uniq_name = "_QMmFfooEi1"}
+  %1:2 = hlfir.declare %0 {uniq_name = "_QMmFfooEi1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2 = fir.alloca i32 {bindc_name = "i2", uniq_name = "_QMmFfooEi2"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QMmFfooEi2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:2 = hlfir.declare %arg0 {uniq_name = "_QMmFfooEx", test.ptr = "x"} : (!fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>) -> (!fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>, !fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>)
+  %5:2 = hlfir.declare %arg1 {uniq_name = "_QMmFfooEy", test.ptr = "y"} : (!fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>) -> (!fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>, !fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>)
+  %6 = hlfir.designate %4#0{"next"}   {fortran_attrs = #fir.var_attrs<pointer>, test.ptr = "xnext1"} : (!fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>>>
+  %7 = fir.load %6 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>>>
+  %8 = fir.box_addr %7 : (!fir.box<!fir.ptr<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>>) -> !fir.ptr<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>
+  %9 = hlfir.designate %8{"i"} : (!fir.ptr<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>) -> !fir.ref<i32>
+  %10 = fir.load %9 : !fir.ref<i32>
+  hlfir.assign %10 to %1#0 : i32, !fir.ref<i32>
+  hlfir.assign %5#0 to %4#0 : !fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>, !fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>
+  %11 = hlfir.designate %4#0{"next"}   {fortran_attrs = #fir.var_attrs<pointer>, test.ptr = "xnext2"} : (!fir.ref<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>>>
+  %12 = fir.load %11 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>>>
+  %13 = fir.box_addr %12 : (!fir.box<!fir.ptr<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>>) -> !fir.ptr<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>
+  %14 = hlfir.designate %13{"i"} : (!fir.ptr<!fir.type<_QMmTt{next:!fir.box<!fir.ptr<!fir.type<_QMmTt>>>,i:i32}>>) -> !fir.ref<i32>
+  %15 = fir.load %14 : !fir.ref<i32>
+  hlfir.assign %15 to %3#0 : i32, !fir.ref<i32>
+  return
+}
-- 
GitLab


From cdb41e416adcd49a783f0d3d28d1e3fafb6f5429 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 16 May 2024 17:54:26 +0200
Subject: [PATCH 1026/1206] PlaceSafepoints: Fix using default constructed
 TargetLibraryInfo (#92411)

---
 llvm/include/llvm/Analysis/TargetLibraryInfo.h  | 4 ++++
 llvm/lib/Analysis/TargetLibraryInfo.cpp         | 4 ++++
 llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp  | 2 ++
 llvm/test/Transforms/PlaceSafepoints/libcall.ll | 7 +++++--
 4 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
index 46f31f918e7b..f5da222d11f5 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
@@ -633,6 +633,10 @@ public:
   explicit TargetLibraryInfoWrapperPass(const Triple &T);
   explicit TargetLibraryInfoWrapperPass(const TargetLibraryInfoImpl &TLI);
 
+  // FIXME: This should be removed when PlaceSafepoints is fixed to not create a
+  // PassManager inside a pass.
+  explicit TargetLibraryInfoWrapperPass(const TargetLibraryInfo &TLI);
+
   TargetLibraryInfo &getTLI(const Function &F) {
     FunctionAnalysisManager DummyFAM;
     TLI = TLA.run(F, DummyFAM);
diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index c62d9daa13ef..684b0759f315 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -1383,6 +1383,10 @@ TargetLibraryInfoWrapperPass::TargetLibraryInfoWrapperPass(
   initializeTargetLibraryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
 }
 
+TargetLibraryInfoWrapperPass::TargetLibraryInfoWrapperPass(
+    const TargetLibraryInfo &TLIOther)
+    : TargetLibraryInfoWrapperPass(*TLIOther.Impl) {}
+
 AnalysisKey TargetLibraryAnalysis::Key;
 
 // Register the basic pass.
diff --git a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
index 77d155d7e78e..dcdea6e7b62a 100644
--- a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
+++ b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
@@ -288,6 +288,8 @@ bool PlaceSafepointsPass::runImpl(Function &F, const TargetLibraryInfo &TLI) {
     // with for the moment.
     legacy::FunctionPassManager FPM(F.getParent());
     bool CanAssumeCallSafepoints = enableCallSafepoints(F);
+
+    FPM.add(new TargetLibraryInfoWrapperPass(TLI));
     auto *PBS = new PlaceBackedgeSafepointsLegacyPass(CanAssumeCallSafepoints);
     FPM.add(PBS);
     FPM.run(F);
diff --git a/llvm/test/Transforms/PlaceSafepoints/libcall.ll b/llvm/test/Transforms/PlaceSafepoints/libcall.ll
index 89090ba1b243..6e26f924a5ba 100644
--- a/llvm/test/Transforms/PlaceSafepoints/libcall.ll
+++ b/llvm/test/Transforms/PlaceSafepoints/libcall.ll
@@ -1,4 +1,6 @@
-; RUN: opt -S -passes=place-safepoints < %s | FileCheck %s
+; RUN: opt -S -passes=place-safepoints < %s | FileCheck -check-prefixes=CHECK,WITHLDEXPF %s
+; RUN: opt -S -passes=place-safepoints -disable-builtin=ldexp < %s | FileCheck %s
+
 
 ; Libcalls will not contain a safepoint poll, so check that we insert
 ; a safepoint in a loop containing a libcall.
@@ -17,7 +19,8 @@ loop:
 ; CHECK-NEXT: %x_loop = phi double [ %x, %entry ], [ %x_exp, %loop ]
 ; CHECK-NEXT: %x_exp = call double @ldexp(double %x_loop, i32 5)
 ; CHECK-NEXT: %done = fcmp ogt double %x_exp, 1.5
-; CHECK-NEXT: call void @do_safepoint
+; WITHLDEXPF-NEXT: call void @do_safepoint
+; CHECK-NEXT: br
   %x_loop = phi double [ %x, %entry ], [ %x_exp, %loop ]
   %x_exp = call double @ldexp(double %x_loop, i32 5) nounwind readnone
   %done = fcmp ogt double %x_exp, 1.5
-- 
GitLab


From 309a881dccb82bf1f101cf138bee3e7400968ed8 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 16 May 2024 16:54:38 +0100
Subject: [PATCH 1027/1206] [VPlan] Address remaining comments for #85689.

Address comments missed when landing
https://github.com/llvm/llvm-project/pull/85689.
---
 llvm/docs/VectorizationPlan.rst | 4 ++--
 llvm/docs/conf.py               | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/docs/VectorizationPlan.rst b/llvm/docs/VectorizationPlan.rst
index 6496263cf8d9..73e9e6098175 100644
--- a/llvm/docs/VectorizationPlan.rst
+++ b/llvm/docs/VectorizationPlan.rst
@@ -289,5 +289,5 @@ References
 .. [9] "Extending LoopVectorizer: OpenMP4.5 SIMD and Outer Loop
     Auto-Vectorization", Intel Vectorizer Team, LLVM Developers' Meeting 2016.
 
-.. [10] "VPlan: Status Update and Future Roadmap", Florian Hahn, LLVM
-         Developers' Meeting 2023, https://www.youtube.com/watch?v=SzGP4PgMuLE
+.. [10] "VPlan: Status Update and Future Roadmap", Ayal Zaks and Florian Hahn,
+    LLVM Developers' Meeting 2023, https://www.youtube.com/watch?v=SzGP4PgMuLE
diff --git a/llvm/docs/conf.py b/llvm/docs/conf.py
index 270f5b33e985..7f2ed5309606 100644
--- a/llvm/docs/conf.py
+++ b/llvm/docs/conf.py
@@ -222,6 +222,8 @@ latex_documents = [
 # If false, no module index is generated.
 # latex_domain_indices = True
 
+# If true, figures, tables and code-blocks are automatically numbered if they
+# have a caption. 
 numfig = True
 
 # -- Options for manual page output --------------------------------------------
-- 
GitLab


From 117d755b1b84c7d379ea5c3d93f8c2ab9bfcde82 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Thu, 16 May 2024 17:04:30 +0100
Subject: [PATCH 1028/1206] [DAG] SimplifyDemandedBits - use ComputeKnownBits
 instead of getValidShiftAmountConstant to check for constant shift amounts.
 (#92412)

This allows us to handle cases where the constant has already been type legalized behind a bitcast

Despite calling ComputeKnownBits I'm not seeing any notable change in compile time.
---
 .../CodeGen/SelectionDAG/TargetLowering.cpp   | 39 ++++++++-------
 llvm/test/CodeGen/AArch64/sadd_sat_vec.ll     | 10 ++--
 llvm/test/CodeGen/AArch64/ssub_sat_vec.ll     | 10 ++--
 llvm/test/CodeGen/PowerPC/pr44183.ll          |  9 +---
 .../CodeGen/RISCV/rvv/bitreverse-sdnode.ll    | 49 ++++++++-----------
 ...vector_splat-const-shift-of-constmasked.ll |  3 +-
 6 files changed, 52 insertions(+), 68 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 38583de03d9c..3ec6b9b79507 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -1732,9 +1732,9 @@ bool TargetLowering::SimplifyDemandedBits(
     SDValue Op1 = Op.getOperand(1);
     EVT ShiftVT = Op1.getValueType();
 
-    if (const APInt *SA =
-            TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
-      unsigned ShAmt = SA->getZExtValue();
+    KnownBits KnownSA = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
+    if (KnownSA.isConstant() && KnownSA.getConstant().ult(BitWidth)) {
+      unsigned ShAmt = KnownSA.getConstant().getZExtValue();
       if (ShAmt == 0)
         return TLO.CombineTo(Op, Op0);
 
@@ -1744,9 +1744,10 @@ bool TargetLowering::SimplifyDemandedBits(
       // TODO - support non-uniform vector amounts.
       if (Op0.getOpcode() == ISD::SRL) {
         if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
-          if (const APInt *SA2 =
-                  TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
-            unsigned C1 = SA2->getZExtValue();
+          KnownBits InnerSA = TLO.DAG.computeKnownBits(Op0.getOperand(1),
+                                                       DemandedElts, Depth + 1);
+          if (InnerSA.isConstant() && InnerSA.getConstant().ult(BitWidth)) {
+            unsigned C1 = InnerSA.getConstant().getZExtValue();
             unsigned Opc = ISD::SHL;
             int Diff = ShAmt - C1;
             if (Diff < 0) {
@@ -1912,9 +1913,9 @@ bool TargetLowering::SimplifyDemandedBits(
                                         DemandedElts, Depth + 1))
       return TLO.CombineTo(Op, AVG);
 
-    if (const APInt *SA =
-            TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
-      unsigned ShAmt = SA->getZExtValue();
+    KnownBits KnownSA = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
+    if (KnownSA.isConstant() && KnownSA.getConstant().ult(BitWidth)) {
+      unsigned ShAmt = KnownSA.getConstant().getZExtValue();
       if (ShAmt == 0)
         return TLO.CombineTo(Op, Op0);
 
@@ -1924,9 +1925,10 @@ bool TargetLowering::SimplifyDemandedBits(
       // TODO - support non-uniform vector amounts.
       if (Op0.getOpcode() == ISD::SHL) {
         if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
-          if (const APInt *SA2 =
-                  TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
-            unsigned C1 = SA2->getZExtValue();
+          KnownBits InnerSA = TLO.DAG.computeKnownBits(Op0.getOperand(1),
+                                                       DemandedElts, Depth + 1);
+          if (InnerSA.isConstant() && InnerSA.getConstant().ult(BitWidth)) {
+            unsigned C1 = InnerSA.getConstant().getZExtValue();
             unsigned Opc = ISD::SRL;
             int Diff = ShAmt - C1;
             if (Diff < 0) {
@@ -2018,24 +2020,25 @@ bool TargetLowering::SimplifyDemandedBits(
                                         DemandedElts, Depth + 1))
       return TLO.CombineTo(Op, AVG);
 
-    if (const APInt *SA =
-            TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
-      unsigned ShAmt = SA->getZExtValue();
+    KnownBits KnownSA = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
+    if (KnownSA.isConstant() && KnownSA.getConstant().ult(BitWidth)) {
+      unsigned ShAmt = KnownSA.getConstant().getZExtValue();
       if (ShAmt == 0)
         return TLO.CombineTo(Op, Op0);
 
       // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target
       // supports sext_inreg.
       if (Op0.getOpcode() == ISD::SHL) {
-        if (const APInt *InnerSA =
-                TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
+        KnownBits InnerSA = TLO.DAG.computeKnownBits(Op0.getOperand(1),
+                                                     DemandedElts, Depth + 1);
+        if (InnerSA.isConstant() && InnerSA.getConstant().ult(BitWidth)) {
           unsigned LowBits = BitWidth - ShAmt;
           EVT ExtVT = EVT::getIntegerVT(*TLO.DAG.getContext(), LowBits);
           if (VT.isVector())
             ExtVT = EVT::getVectorVT(*TLO.DAG.getContext(), ExtVT,
                                      VT.getVectorElementCount());
 
-          if (*InnerSA == ShAmt) {
+          if (InnerSA.getConstant() == ShAmt) {
             if (!TLO.LegalOperations() ||
                 getOperationAction(ISD::SIGN_EXTEND_INREG, ExtVT) == Legal)
               return TLO.CombineTo(
diff --git a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
index 8a0e7661883f..49ed98976674 100644
--- a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
@@ -149,12 +149,12 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind {
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    ldr s0, [x0]
 ; CHECK-SD-NEXT:    ldr s1, [x1]
-; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
 ; CHECK-SD-NEXT:    shl v1.4h, v1.4h, #8
 ; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-SD-NEXT:    sqadd v0.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    ushr v0.4h, v0.4h, #8
 ; CHECK-SD-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
 ; CHECK-SD-NEXT:    str s0, [x2]
 ; CHECK-SD-NEXT:    ret
@@ -364,10 +364,6 @@ define void @v1i16(ptr %px, ptr %py, ptr %pz) nounwind {
 define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-LABEL: v16i4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    shl v0.16b, v0.16b, #4
-; CHECK-NEXT:    shl v1.16b, v1.16b, #4
-; CHECK-NEXT:    sshr v0.16b, v0.16b, #4
-; CHECK-NEXT:    sshr v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #4
 ; CHECK-NEXT:    sqadd v0.16b, v0.16b, v1.16b
diff --git a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
index a8c1276eadc4..f6b58023da1f 100644
--- a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
@@ -150,12 +150,12 @@ define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind {
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    ldr s0, [x0]
 ; CHECK-SD-NEXT:    ldr s1, [x1]
-; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
 ; CHECK-SD-NEXT:    shl v1.4h, v1.4h, #8
 ; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-SD-NEXT:    sqsub v0.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    ushr v0.4h, v0.4h, #8
 ; CHECK-SD-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
 ; CHECK-SD-NEXT:    str s0, [x2]
 ; CHECK-SD-NEXT:    ret
@@ -365,10 +365,6 @@ define void @v1i16(ptr %px, ptr %py, ptr %pz) nounwind {
 define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-LABEL: v16i4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    shl v0.16b, v0.16b, #4
-; CHECK-NEXT:    shl v1.16b, v1.16b, #4
-; CHECK-NEXT:    sshr v0.16b, v0.16b, #4
-; CHECK-NEXT:    sshr v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #4
 ; CHECK-NEXT:    sqsub v0.16b, v0.16b, v1.16b
diff --git a/llvm/test/CodeGen/PowerPC/pr44183.ll b/llvm/test/CodeGen/PowerPC/pr44183.ll
index e3dca1380903..4d2c81c35b7f 100644
--- a/llvm/test/CodeGen/PowerPC/pr44183.ll
+++ b/llvm/test/CodeGen/PowerPC/pr44183.ll
@@ -22,13 +22,8 @@ define void @_ZN1m1nEv(ptr %this) local_unnamed_addr nounwind align 2 {
 ; CHECK-NEXT:    rlwimi r4, r3, 0, 0, 0
 ; CHECK-NEXT:    bl _ZN1llsE1d
 ; CHECK-NEXT:    nop
-; CHECK-NEXT:    ld r3, 16(r30)
-; CHECK-NEXT:    ld r4, 8(r30)
-; CHECK-NEXT:    rldicl r4, r4, 60, 4
-; CHECK-NEXT:    sldi r3, r3, 60
-; CHECK-NEXT:    or r3, r3, r4
-; CHECK-NEXT:    sldi r3, r3, 31
-; CHECK-NEXT:    rlwinm r4, r3, 0, 0, 0
+; CHECK-NEXT:    ld r3, 8(r30)
+; CHECK-NEXT:    rlwinm r4, r3, 27, 0, 0
 ; CHECK-NEXT:    bl _ZN1llsE1d
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 48
diff --git a/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll
index 19ae26d24242..94e945f80320 100644
--- a/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll
@@ -8,10 +8,9 @@ define <vscale x 1 x i8> @bitreverse_nxv1i8(<vscale x 1 x i8> %va) {
 ; CHECK-LABEL: bitreverse_nxv1i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
-; CHECK-NEXT:    vsrl.vi v9, v8, 4
-; CHECK-NEXT:    vand.vi v8, v8, 15
-; CHECK-NEXT:    vsll.vi v8, v8, 4
-; CHECK-NEXT:    vor.vv v8, v9, v8
+; CHECK-NEXT:    vsll.vi v9, v8, 4
+; CHECK-NEXT:    vsrl.vi v8, v8, 4
+; CHECK-NEXT:    vor.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 2
 ; CHECK-NEXT:    li a0, 51
 ; CHECK-NEXT:    vand.vx v9, v9, a0
@@ -40,10 +39,9 @@ define <vscale x 2 x i8> @bitreverse_nxv2i8(<vscale x 2 x i8> %va) {
 ; CHECK-LABEL: bitreverse_nxv2i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
-; CHECK-NEXT:    vsrl.vi v9, v8, 4
-; CHECK-NEXT:    vand.vi v8, v8, 15
-; CHECK-NEXT:    vsll.vi v8, v8, 4
-; CHECK-NEXT:    vor.vv v8, v9, v8
+; CHECK-NEXT:    vsll.vi v9, v8, 4
+; CHECK-NEXT:    vsrl.vi v8, v8, 4
+; CHECK-NEXT:    vor.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 2
 ; CHECK-NEXT:    li a0, 51
 ; CHECK-NEXT:    vand.vx v9, v9, a0
@@ -72,10 +70,9 @@ define <vscale x 4 x i8> @bitreverse_nxv4i8(<vscale x 4 x i8> %va) {
 ; CHECK-LABEL: bitreverse_nxv4i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vsrl.vi v9, v8, 4
-; CHECK-NEXT:    vand.vi v8, v8, 15
-; CHECK-NEXT:    vsll.vi v8, v8, 4
-; CHECK-NEXT:    vor.vv v8, v9, v8
+; CHECK-NEXT:    vsll.vi v9, v8, 4
+; CHECK-NEXT:    vsrl.vi v8, v8, 4
+; CHECK-NEXT:    vor.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 2
 ; CHECK-NEXT:    li a0, 51
 ; CHECK-NEXT:    vand.vx v9, v9, a0
@@ -104,10 +101,9 @@ define <vscale x 8 x i8> @bitreverse_nxv8i8(<vscale x 8 x i8> %va) {
 ; CHECK-LABEL: bitreverse_nxv8i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vsrl.vi v9, v8, 4
-; CHECK-NEXT:    vand.vi v8, v8, 15
-; CHECK-NEXT:    vsll.vi v8, v8, 4
-; CHECK-NEXT:    vor.vv v8, v9, v8
+; CHECK-NEXT:    vsll.vi v9, v8, 4
+; CHECK-NEXT:    vsrl.vi v8, v8, 4
+; CHECK-NEXT:    vor.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 2
 ; CHECK-NEXT:    li a0, 51
 ; CHECK-NEXT:    vand.vx v9, v9, a0
@@ -136,10 +132,9 @@ define <vscale x 16 x i8> @bitreverse_nxv16i8(<vscale x 16 x i8> %va) {
 ; CHECK-LABEL: bitreverse_nxv16i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vsrl.vi v10, v8, 4
-; CHECK-NEXT:    vand.vi v8, v8, 15
-; CHECK-NEXT:    vsll.vi v8, v8, 4
-; CHECK-NEXT:    vor.vv v8, v10, v8
+; CHECK-NEXT:    vsll.vi v10, v8, 4
+; CHECK-NEXT:    vsrl.vi v8, v8, 4
+; CHECK-NEXT:    vor.vv v8, v8, v10
 ; CHECK-NEXT:    vsrl.vi v10, v8, 2
 ; CHECK-NEXT:    li a0, 51
 ; CHECK-NEXT:    vand.vx v10, v10, a0
@@ -168,10 +163,9 @@ define <vscale x 32 x i8> @bitreverse_nxv32i8(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: bitreverse_nxv32i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
-; CHECK-NEXT:    vsrl.vi v12, v8, 4
-; CHECK-NEXT:    vand.vi v8, v8, 15
-; CHECK-NEXT:    vsll.vi v8, v8, 4
-; CHECK-NEXT:    vor.vv v8, v12, v8
+; CHECK-NEXT:    vsll.vi v12, v8, 4
+; CHECK-NEXT:    vsrl.vi v8, v8, 4
+; CHECK-NEXT:    vor.vv v8, v8, v12
 ; CHECK-NEXT:    vsrl.vi v12, v8, 2
 ; CHECK-NEXT:    li a0, 51
 ; CHECK-NEXT:    vand.vx v12, v12, a0
@@ -200,10 +194,9 @@ define <vscale x 64 x i8> @bitreverse_nxv64i8(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: bitreverse_nxv64i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
-; CHECK-NEXT:    vsrl.vi v16, v8, 4
-; CHECK-NEXT:    vand.vi v8, v8, 15
-; CHECK-NEXT:    vsll.vi v8, v8, 4
-; CHECK-NEXT:    vor.vv v8, v16, v8
+; CHECK-NEXT:    vsll.vi v16, v8, 4
+; CHECK-NEXT:    vsrl.vi v8, v8, 4
+; CHECK-NEXT:    vor.vv v8, v8, v16
 ; CHECK-NEXT:    vsrl.vi v16, v8, 2
 ; CHECK-NEXT:    li a0, 51
 ; CHECK-NEXT:    vand.vx v16, v16, a0
diff --git a/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll b/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll
index c3e9a2b6841a..b39e2bb620d6 100644
--- a/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll
+++ b/llvm/test/CodeGen/X86/vector_splat-const-shift-of-constmasked.ll
@@ -3176,7 +3176,8 @@ define <2 x i64> @test_128_i64_x_2_18446744065119617024_mask_ashr_32(<2 x i64> %
 ;
 ; X86-AVX2-LABEL: test_128_i64_x_2_18446744065119617024_mask_ashr_32:
 ; X86-AVX2:       # %bb.0:
-; X86-AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4294967294,4294967294,4294967294,4294967294]
+; X86-AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
 ; X86-AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; X86-AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
-- 
GitLab


From 3abd3d6e597cba5161f37fa0478382fc93a8c9fd Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 16 May 2024 11:13:50 -0500
Subject: [PATCH 1029/1206] [Libomptarget] Remove requires information from
 plugin (#80345)

Summary:
Currently this is only used for the zero-copy handling. However, this
can easily be moved into `libomptarget` so that we do not need to bother
setting the requires flags in the plugin. The advantage here is that we
no longer need to do this for every device redundently. Additionally,
these requires flags are specifically OpenMP related, so they should
live in `libomptarget`.
---
 .../common/include/PluginInterface.h              | 15 +--------------
 .../common/src/PluginInterface.cpp                | 10 ----------
 offload/src/device.cpp                            | 12 +++---------
 3 files changed, 4 insertions(+), 33 deletions(-)

diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index c396099ac625..d8d71e3e65a4 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -958,8 +958,7 @@ struct GenericPluginTy {
 
   /// Construct a plugin instance.
   GenericPluginTy(Triple::ArchType TA)
-      : RequiresFlags(OMP_REQ_UNDEFINED), GlobalHandler(nullptr), JIT(TA),
-        RPCServer(nullptr) {}
+      : GlobalHandler(nullptr), JIT(TA), RPCServer(nullptr) {}
 
   virtual ~GenericPluginTy() {}
 
@@ -1028,12 +1027,6 @@ struct GenericPluginTy {
     return *RPCServer;
   }
 
-  /// Get the OpenMP requires flags set for this plugin.
-  int64_t getRequiresFlags() const { return RequiresFlags; }
-
-  /// Set the OpenMP requires flags for this plugin.
-  void setRequiresFlag(int64_t Flags) { RequiresFlags = Flags; }
-
   /// Initialize a device within the plugin.
   Error initDevice(int32_t DeviceId);
 
@@ -1074,9 +1067,6 @@ public:
   /// Return the number of devices this plugin can support.
   int32_t number_of_devices();
 
-  /// Initializes the OpenMP register requires information.
-  int64_t init_requires(int64_t RequiresFlags);
-
   /// Returns non-zero if the data can be exchanged between the two devices.
   int32_t is_data_exchangable(int32_t SrcDeviceId, int32_t DstDeviceId);
 
@@ -1203,9 +1193,6 @@ private:
   /// device was not initialized yet.
   llvm::SmallVector<GenericDeviceTy *> Devices;
 
-  /// OpenMP requires flags.
-  int64_t RequiresFlags;
-
   /// Pointer to the global handler for this plugin.
   GenericGlobalHandlerTy *GlobalHandler;
 
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 737a8b2a4064..253acacc3a9d 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -1618,11 +1618,6 @@ int32_t GenericPluginTy::init_device(int32_t DeviceId) {
 
 int32_t GenericPluginTy::number_of_devices() { return getNumDevices(); }
 
-int64_t GenericPluginTy::init_requires(int64_t RequiresFlags) {
-  setRequiresFlag(RequiresFlags);
-  return OFFLOAD_SUCCESS;
-}
-
 int32_t GenericPluginTy::is_data_exchangable(int32_t SrcDeviceId,
                                              int32_t DstDeviceId) {
   return isDataExchangable(SrcDeviceId, DstDeviceId);
@@ -1964,11 +1959,6 @@ int32_t GenericPluginTy::set_device_offset(int32_t DeviceIdOffset) {
 }
 
 int32_t GenericPluginTy::use_auto_zero_copy(int32_t DeviceId) {
-  // Automatic zero-copy only applies to programs that did
-  // not request unified_shared_memory and are deployed on an
-  // APU with XNACK enabled.
-  if (getRequiresFlags() & OMP_REQ_UNIFIED_SHARED_MEMORY)
-    return false;
   return getDevice(DeviceId).useAutoZeroCopy();
 }
 
diff --git a/offload/src/device.cpp b/offload/src/device.cpp
index 749b4c567f8e..943c77827873 100644
--- a/offload/src/device.cpp
+++ b/offload/src/device.cpp
@@ -77,15 +77,7 @@ DeviceTy::~DeviceTy() {
 }
 
 llvm::Error DeviceTy::init() {
-  // Make call to init_requires if it exists for this plugin.
-  int32_t Ret = 0;
-  Ret = RTL->init_requires(PM->getRequirements());
-  if (Ret != OFFLOAD_SUCCESS)
-    return llvm::createStringError(
-        llvm::inconvertibleErrorCode(),
-        "Failed to initialize requirements for device %d\n", DeviceID);
-
-  Ret = RTL->init_device(RTLDeviceID);
+  int32_t Ret = RTL->init_device(RTLDeviceID);
   if (Ret != OFFLOAD_SUCCESS)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "Failed to initialize device %d\n",
@@ -285,5 +277,7 @@ void DeviceTy::dumpOffloadEntries() {
 }
 
 bool DeviceTy::useAutoZeroCopy() {
+  if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY)
+    return false;
   return RTL->use_auto_zero_copy(RTLDeviceID);
 }
-- 
GitLab


From b28766eb3f20354d1d7a34ea83b9d915c3715032 Mon Sep 17 00:00:00 2001
From: Eli Friedman <efriedma@quicinc.com>
Date: Thu, 16 May 2024 09:15:17 -0700
Subject: [PATCH 1030/1206] [Arm64EC] Correctly handle sret in entry thunks.
 (#92326)

I accidentally left out the code to transfer sret attributes to entry
thunks, so values weren't being passed in the right registers, and the
sret pointer wasn't returned in the correct register.

Fixes #90229
---
 .../AArch64/AArch64Arm64ECCallLowering.cpp    |  9 ++++-
 .../CodeGen/AArch64/arm64ec-entry-thunks.ll   | 36 +++++++++++--------
 2 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
index dddc181b0314..0ec15d34cd4a 100644
--- a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
@@ -512,7 +512,14 @@ Function *AArch64Arm64ECCallLowering::buildEntryThunk(Function *F) {
   // Call the function passed to the thunk.
   Value *Callee = Thunk->getArg(0);
   Callee = IRB.CreateBitCast(Callee, PtrTy);
-  Value *Call = IRB.CreateCall(Arm64Ty, Callee, Args);
+  CallInst *Call = IRB.CreateCall(Arm64Ty, Callee, Args);
+
+  auto SRetAttr = F->getAttributes().getParamAttr(0, Attribute::StructRet);
+  auto InRegAttr = F->getAttributes().getParamAttr(0, Attribute::InReg);
+  if (SRetAttr.isValid() && !InRegAttr.isValid()) {
+    Thunk->addParamAttr(1, SRetAttr);
+    Call->addParamAttr(0, SRetAttr);
+  }
 
   Value *RetVal = Call;
   if (TransformDirectToSRet) {
diff --git a/llvm/test/CodeGen/AArch64/arm64ec-entry-thunks.ll b/llvm/test/CodeGen/AArch64/arm64ec-entry-thunks.ll
index c00c9bfe127e..e9556b9d5cbe 100644
--- a/llvm/test/CodeGen/AArch64/arm64ec-entry-thunks.ll
+++ b/llvm/test/CodeGen/AArch64/arm64ec-entry-thunks.ll
@@ -222,12 +222,12 @@ define i8 @matches_has_sret() nounwind {
 }
 
 %TSRet = type { i64, i64 }
-define void @has_aligned_sret(ptr align 32 sret(%TSRet)) nounwind {
-; CHECK-LABEL:    .def    $ientry_thunk$cdecl$m16$v;
-; CHECK:          .section        .wowthk$aa,"xr",discard,$ientry_thunk$cdecl$m16$v
+define void @has_aligned_sret(ptr align 32 sret(%TSRet), i32) nounwind {
+; CHECK-LABEL:    .def    $ientry_thunk$cdecl$m16$i8;
+; CHECK:          .section        .wowthk$aa,"xr",discard,$ientry_thunk$cdecl$m16$i8
 ; CHECK:          // %bb.0:
-; CHECK-NEXT:     stp     q6, q7, [sp, #-176]!            // 32-byte Folded Spill
-; CHECK-NEXT:     .seh_save_any_reg_px    q6, 176
+; CHECK-NEXT:     stp     q6, q7, [sp, #-192]!            // 32-byte Folded Spill
+; CHECK-NEXT:     .seh_save_any_reg_px    q6, 192
 ; CHECK-NEXT:     stp     q8, q9, [sp, #32]               // 32-byte Folded Spill
 ; CHECK-NEXT:     .seh_save_any_reg_p     q8, 32
 ; CHECK-NEXT:     stp     q10, q11, [sp, #64]             // 32-byte Folded Spill
@@ -236,17 +236,25 @@ define void @has_aligned_sret(ptr align 32 sret(%TSRet)) nounwind {
 ; CHECK-NEXT:     .seh_save_any_reg_p     q12, 96
 ; CHECK-NEXT:     stp     q14, q15, [sp, #128]            // 32-byte Folded Spill
 ; CHECK-NEXT:     .seh_save_any_reg_p     q14, 128
-; CHECK-NEXT:     stp     x29, x30, [sp, #160]            // 16-byte Folded Spill
-; CHECK-NEXT:     .seh_save_fplr  160
-; CHECK-NEXT:     add     x29, sp, #160
-; CHECK-NEXT:     .seh_add_fp     160
+; CHECK-NEXT:     str     x19, [sp, #160]                 // 8-byte Folded Spill
+; CHECK-NEXT:     .seh_save_reg   x19, 160
+; CHECK-NEXT:     stp     x29, x30, [sp, #168]            // 16-byte Folded Spill
+; CHECK-NEXT:     .seh_save_fplr  168
+; CHECK-NEXT:     add     x29, sp, #168
+; CHECK-NEXT:     .seh_add_fp     168
 ; CHECK-NEXT:     .seh_endprologue
+; CHECK-NEXT:     mov     x19, x0
+; CHECK-NEXT:     mov     x8, x0
+; CHECK-NEXT:     mov     x0, x1
 ; CHECK-NEXT:     blr     x9
 ; CHECK-NEXT:     adrp    x8, __os_arm64x_dispatch_ret
 ; CHECK-NEXT:     ldr     x0, [x8, :lo12:__os_arm64x_dispatch_ret]
+; CHECK-NEXT:     mov     x8, x19
 ; CHECK-NEXT:     .seh_startepilogue
-; CHECK-NEXT:     ldp     x29, x30, [sp, #160]            // 16-byte Folded Reload
-; CHECK-NEXT:     .seh_save_fplr  160
+; CHECK-NEXT:     ldp     x29, x30, [sp, #168]            // 16-byte Folded Reload
+; CHECK-NEXT:     .seh_save_fplr  168
+; CHECK-NEXT:     ldr     x19, [sp, #160]                 // 8-byte Folded Reload
+; CHECK-NEXT:     .seh_save_reg   x19, 160
 ; CHECK-NEXT:     ldp     q14, q15, [sp, #128]            // 32-byte Folded Reload
 ; CHECK-NEXT:     .seh_save_any_reg_p     q14, 128
 ; CHECK-NEXT:     ldp     q12, q13, [sp, #96]             // 32-byte Folded Reload
@@ -255,8 +263,8 @@ define void @has_aligned_sret(ptr align 32 sret(%TSRet)) nounwind {
 ; CHECK-NEXT:     .seh_save_any_reg_p     q10, 64
 ; CHECK-NEXT:     ldp     q8, q9, [sp, #32]               // 32-byte Folded Reload
 ; CHECK-NEXT:     .seh_save_any_reg_p     q8, 32
-; CHECK-NEXT:     ldp     q6, q7, [sp], #176              // 32-byte Folded Reload
-; CHECK-NEXT:     .seh_save_any_reg_px    q6, 176
+; CHECK-NEXT:     ldp     q6, q7, [sp], #192              // 32-byte Folded Reload
+; CHECK-NEXT:     .seh_save_any_reg_px    q6, 192
 ; CHECK-NEXT:     .seh_endepilogue
 ; CHECK-NEXT:     br      x0
 ; CHECK-NEXT:     .seh_endfunclet
@@ -457,7 +465,7 @@ define %T2 @simple_struct(%T1 %0, %T2 %1, %T3, %T4) nounwind {
 ; CHECK-NEXT:     .symidx $ientry_thunk$cdecl$i8$v
 ; CHECK-NEXT:     .word   1
 ; CHECK-NEXT:     .symidx "#has_aligned_sret"
-; CHECK-NEXT:     .symidx $ientry_thunk$cdecl$m16$v
+; CHECK-NEXT:     .symidx $ientry_thunk$cdecl$m16$i8
 ; CHECK-NEXT:     .word   1
 ; CHECK-NEXT:     .symidx "#small_array"
 ; CHECK-NEXT:     .symidx $ientry_thunk$cdecl$m2$m2F8
-- 
GitLab


From b82fd5d75cff87f7480a24d4a16ea8400cc35c86 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve@amd.com>
Date: Thu, 16 May 2024 18:15:55 +0200
Subject: [PATCH 1031/1206] [GlobalISel] Initialize variables in
 IndexedLoadStoreMatchInfo

---
 llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index ecaece8b6834..87cff154b4eb 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -56,9 +56,9 @@ struct IndexedLoadStoreMatchInfo {
   Register Addr;
   Register Base;
   Register Offset;
-  bool RematOffset; // True if Offset is a constant that needs to be
-                    // rematerialized before the new load/store.
-  bool IsPre;
+  bool RematOffset = false; // True if Offset is a constant that needs to be
+                            // rematerialized before the new load/store.
+  bool IsPre = false;
 };
 
 struct PtrAddChain {
-- 
GitLab


From e8692b88bec1d43325804d75166d9483d6c17fc7 Mon Sep 17 00:00:00 2001
From: Tim Besard <tim.besard@gmail.com>
Date: Thu, 16 May 2024 18:18:37 +0200
Subject: [PATCH 1032/1206] [NewPM] Add pass options for InternalizePass to
 preserve GVs (reland) (#92383)

Reland of https://github.com/llvm/llvm-project/pull/91334, which broke
the gcc7 buildbot and was reverted in
https://github.com/llvm/llvm-project/pull/92321.
Work around the failure by being explicit about returning an `Expected`.

cc @joker-eph
---
 llvm/lib/Passes/PassBuilder.cpp           | 18 ++++++++++++++++++
 llvm/lib/Passes/PassRegistry.def          | 15 ++++++++++++++-
 llvm/test/Transforms/Internalize/lists.ll |  5 +++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index e4131706aba0..734ca4d5deec 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -1142,6 +1142,24 @@ Expected<GlobalMergeOptions> parseGlobalMergeOptions(StringRef Params) {
   return Result;
 }
 
+Expected<SmallVector<std::string, 0>> parseInternalizeGVs(StringRef Params) {
+  SmallVector<std::string, 1> PreservedGVs;
+  while (!Params.empty()) {
+    StringRef ParamName;
+    std::tie(ParamName, Params) = Params.split(';');
+
+    if (ParamName.consume_front("preserve-gv=")) {
+      PreservedGVs.push_back(ParamName.str());
+    } else {
+      return make_error<StringError>(
+          formatv("invalid Internalize pass parameter '{0}' ", ParamName).str(),
+          inconvertibleErrorCode());
+    }
+  }
+
+  return Expected<SmallVector<std::string, 0>>(std::move(PreservedGVs));
+}
+
 } // namespace
 
 /// Tests whether a pass name starts with a valid prefix for a default pipeline
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index e5ce6cb7da64..50682ca4970f 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -78,7 +78,6 @@ MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass())
 MODULE_PASS("instrorderfile", InstrOrderFilePass())
 MODULE_PASS("instrprof", InstrProfilingLoweringPass())
 MODULE_PASS("ctx-instr-lower", PGOCtxProfLoweringPass())
-MODULE_PASS("internalize", InternalizePass())
 MODULE_PASS("invalidate<all>", InvalidateAllAnalysesPass())
 MODULE_PASS("iroutliner", IROutlinerPass())
 MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass())
@@ -175,6 +174,20 @@ MODULE_PASS_WITH_PARAMS(
     "hwasan", "HWAddressSanitizerPass",
     [](HWAddressSanitizerOptions Opts) { return HWAddressSanitizerPass(Opts); },
     parseHWASanPassOptions, "kernel;recover")
+MODULE_PASS_WITH_PARAMS(
+    "internalize", "InternalizePass",
+    [](SmallVector<std::string, 0> PreservedGVs) {
+      if (PreservedGVs.empty())
+        return InternalizePass();
+      auto MustPreserveGV = [=](const GlobalValue &GV) {
+        for (auto &PreservedGV : PreservedGVs)
+          if (GV.getName() == PreservedGV)
+            return true;
+        return false;
+      };
+      return InternalizePass(MustPreserveGV);
+    },
+    parseInternalizeGVs, "preserve-gv=GV")
 MODULE_PASS_WITH_PARAMS(
     "ipsccp", "IPSCCPPass", [](IPSCCPOptions Opts) { return IPSCCPPass(Opts); },
     parseIPSCCPOptions, "no-func-spec;func-spec")
diff --git a/llvm/test/Transforms/Internalize/lists.ll b/llvm/test/Transforms/Internalize/lists.ll
index df408f906b78..83dad03d75ea 100644
--- a/llvm/test/Transforms/Internalize/lists.ll
+++ b/llvm/test/Transforms/Internalize/lists.ll
@@ -13,6 +13,11 @@
 ; -file and -list options should be merged, the apifile contains foo and j
 ; RUN: opt < %s -passes=internalize -internalize-public-api-list bar -internalize-public-api-file %S/apifile -S | FileCheck --check-prefix=FOO_J_AND_BAR %s
 
+; specifying through pass builder option
+; RUN: opt < %s -passes='internalize<preserve-gv=foo;preserve-gv=j>' -S | FileCheck --check-prefix=FOO_AND_J %s
+; RUN: opt < %s -passes='internalize<preserve-gv=foo;preserve-gv=bar>' -S | FileCheck --check-prefix=FOO_AND_BAR %s
+; RUN: opt < %s -passes='internalize<preserve-gv=foo;preserve-gv=j;preserve-gv=bar>' -S | FileCheck --check-prefix=FOO_J_AND_BAR %s
+
 ; ALL: @i = internal global
 ; FOO_AND_J: @i = internal global
 ; FOO_AND_BAR: @i = internal global
-- 
GitLab


From ce67fcf15f4ffac00a715cf724bc72e37f063064 Mon Sep 17 00:00:00 2001
From: SunilKuravinakop <98882378+SunilKuravinakop@users.noreply.github.com>
Date: Thu, 16 May 2024 21:55:53 +0530
Subject: [PATCH 1033/1206] Avoid unevaluated implicit private (#92055)

For every variable used under `#pragma omp task` directive
(`DeclRefExpr`) an ImplicitPrivateVariable is created in the AST, if
`private` or `shared` clauses are not present. If the variable has the
property of `non_odr_use_unevaluated` e.g. for statements which use
`sizeof( i )` `i` will have `non_odr_use_unevaluated` . In such cases
CodeGen was asserting by avoiding emitting of LLVM IR for such
variables. To prevent this assertion this checkin avoids adding the
ImplicitPrivateVariable for variables with `non_odr_use_unevaluated`.

---------

Authored-by: Sunil Kuravinakop <kuravina@pe28vega.us.cray.com>
---
 clang/lib/Sema/SemaOpenMP.cpp        |  3 +-
 clang/test/OpenMP/task_ast_print.cpp | 41 ++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 7d00cf6fb5b6..6110e5229b07 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -3757,7 +3757,8 @@ public:
   void VisitDeclRefExpr(DeclRefExpr *E) {
     if (TryCaptureCXXThisMembers || E->isTypeDependent() ||
         E->isValueDependent() || E->containsUnexpandedParameterPack() ||
-        E->isInstantiationDependent())
+        E->isInstantiationDependent() ||
+        E->isNonOdrUse() == clang::NOUR_Unevaluated)
       return;
     if (auto *VD = dyn_cast<VarDecl>(E->getDecl())) {
       // Check the datasharing rules for the expressions in the clauses.
diff --git a/clang/test/OpenMP/task_ast_print.cpp b/clang/test/OpenMP/task_ast_print.cpp
index 12923e6ab424..2a6b8908a1e2 100644
--- a/clang/test/OpenMP/task_ast_print.cpp
+++ b/clang/test/OpenMP/task_ast_print.cpp
@@ -5,6 +5,7 @@
 // RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -std=c++11 -include-pch %t -verify -Wno-vla %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -ast-dump  %s | FileCheck %s --check-prefix=DUMP
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -208,4 +209,44 @@ int main(int argc, char **argv) {
 extern template int S<int>::TS;
 extern template long S<long>::TS;
 
+// DUMP-LABEL:  FunctionDecl {{.*}} implicit_firstprivate
+void
+implicit_firstprivate() {
+
+#pragma omp parallel num_threads(1)
+  {
+    int i = 0;
+    // DUMP: OMPTaskDirective 
+    // DUMP-NEXT: OMPFirstprivateClause
+    // DUMP-NOT: DeclRefExpr {{.+}} 'i' {{.+}} non_odr_use_unevaluated
+    // DUMP: DeclRefExpr {{.+}} 'i' 'int' refers_to_enclosing_variable_or_capture
+    // DUMP: CapturedStmt
+    // DUMP: BinaryOperator {{.+}} 'int' lvalue '='
+    // DUMP-NEXT: DeclRefExpr {{.+}} 'j' 'int'
+    // DUMP: DeclRefExpr {{.+}} 'i' {{.+}} non_odr_use_unevaluated
+    #pragma omp task
+    {
+	int j = sizeof(i);
+	j = i;
+    }
+  }
+}
+
+// DUMP-LABEL:  FunctionDecl {{.*}} no_implicit_firstprivate
+void
+no_implicit_firstprivate() {
+
+#pragma omp parallel num_threads(1)
+  {
+    int i = 0;
+    // DUMP: OMPTaskDirective
+    // DUMP-NEXT: CapturedStmt
+    // DUMP: DeclRefExpr {{.+}} 'i' {{.+}} non_odr_use_unevaluated refers_to_enclosing_variable_or_capture
+    #pragma omp task
+    {
+	int j = sizeof(i);
+    }
+  }
+}
+
 #endif
-- 
GitLab


From 033fa81480a7e8d0a6cf2f72b52232275145d105 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 16 May 2024 11:47:57 -0500
Subject: [PATCH 1034/1206] [Offload][NFC] Remove unused files following static
 plugins

Summary:
Forgot to remove these when I landed the initial patch, they are no
longer used.
---
 offload/include/Shared/PluginAPI.h   | 232 ---------------------------
 offload/include/Shared/PluginAPI.inc |  51 ------
 2 files changed, 283 deletions(-)
 delete mode 100644 offload/include/Shared/PluginAPI.h
 delete mode 100644 offload/include/Shared/PluginAPI.inc

diff --git a/offload/include/Shared/PluginAPI.h b/offload/include/Shared/PluginAPI.h
deleted file mode 100644
index ecf669c774f1..000000000000
--- a/offload/include/Shared/PluginAPI.h
+++ /dev/null
@@ -1,232 +0,0 @@
-//===-- Shared/PluginAPI.h - Target independent plugin API ------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines an interface between target independent OpenMP offload
-// runtime library libomptarget and target dependent plugin.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef OMPTARGET_SHARED_PLUGIN_API_H
-#define OMPTARGET_SHARED_PLUGIN_API_H
-
-#include <cstddef>
-#include <cstdint>
-
-#include "Shared/APITypes.h"
-
-extern "C" {
-
-// First method called on the plugin
-int32_t __tgt_rtl_init_plugin();
-
-// Return the number of available devices of the type supported by the
-// target RTL.
-int32_t __tgt_rtl_number_of_devices(void);
-
-// Return an integer different from zero if the provided device image can be
-// supported by the runtime. The functionality is similar to comparing the
-// result of __tgt__rtl__load__binary to NULL. However, this is meant to be a
-// lightweight query to determine if the RTL is suitable for an image without
-// having to load the library, which can be expensive.
-int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image);
-
-// Return an integer other than zero if the data can be exchaned from SrcDevId
-// to DstDevId. If it is data exchangable, the device plugin should provide
-// function to move data from source device to destination device directly.
-int32_t __tgt_rtl_is_data_exchangable(int32_t SrcDevId, int32_t DstDevId);
-
-// Initialize the requires flags for the device.
-int64_t __tgt_rtl_init_requires(int64_t RequiresFlags);
-
-// Initialize the specified device. In case of success return 0; otherwise
-// return an error code.
-int32_t __tgt_rtl_init_device(int32_t ID);
-
-// Pass an executable image section described by image to the specified
-// device and prepare an address table of target entities. In case of error,
-// return NULL. Otherwise, return a pointer to the built address table.
-// Individual entries in the table may also be NULL, when the corresponding
-// offload region is not supported on the target device.
-int32_t __tgt_rtl_load_binary(int32_t ID, __tgt_device_image *Image,
-                              __tgt_device_binary *Binary);
-
-// Look up the device address of the named symbol in the given binary. Returns
-// non-zero on failure.
-int32_t __tgt_rtl_get_global(__tgt_device_binary Binary, uint64_t Size,
-                             const char *Name, void **DevicePtr);
-
-// Look up the device address of the named kernel in the given binary. Returns
-// non-zero on failure.
-int32_t __tgt_rtl_get_function(__tgt_device_binary Binary, const char *Name,
-                               void **DevicePtr);
-
-// Allocate data on the particular target device, of the specified size.
-// HostPtr is a address of the host data the allocated target data
-// will be associated with (HostPtr may be NULL if it is not known at
-// allocation time, like for example it would be for target data that
-// is allocated by omp_target_alloc() API). Return address of the
-// allocated data on the target that will be used by libomptarget.so to
-// initialize the target data mapping structures. These addresses are
-// used to generate a table of target variables to pass to
-// __tgt_rtl_run_region(). The __tgt_rtl_data_alloc() returns NULL in
-// case an error occurred on the target device. Kind dictates what allocator
-// to use (e.g. shared, host, device).
-void *__tgt_rtl_data_alloc(int32_t ID, int64_t Size, void *HostPtr,
-                           int32_t Kind);
-
-// Pass the data content to the target device using the target address. In case
-// of success, return zero. Otherwise, return an error code.
-int32_t __tgt_rtl_data_submit(int32_t ID, void *TargetPtr, void *HostPtr,
-                              int64_t Size);
-
-int32_t __tgt_rtl_data_submit_async(int32_t ID, void *TargetPtr, void *HostPtr,
-                                    int64_t Size, __tgt_async_info *AsyncInfo);
-
-// Retrieve the data content from the target device using its address. In case
-// of success, return zero. Otherwise, return an error code.
-int32_t __tgt_rtl_data_retrieve(int32_t ID, void *HostPtr, void *TargetPtr,
-                                int64_t Size);
-
-// Asynchronous version of __tgt_rtl_data_retrieve
-int32_t __tgt_rtl_data_retrieve_async(int32_t ID, void *HostPtr,
-                                      void *TargetPtr, int64_t Size,
-                                      __tgt_async_info *AsyncInfo);
-
-// Copy the data content from one target device to another target device using
-// its address. This operation does not need to copy data back to host and then
-// from host to another device. In case of success, return zero. Otherwise,
-// return an error code.
-int32_t __tgt_rtl_data_exchange(int32_t SrcID, void *SrcPtr, int32_t DstID,
-                                void *DstPtr, int64_t Size);
-
-// Asynchronous version of __tgt_rtl_data_exchange
-int32_t __tgt_rtl_data_exchange_async(int32_t SrcID, void *SrcPtr,
-                                      int32_t DesID, void *DstPtr, int64_t Size,
-                                      __tgt_async_info *AsyncInfo);
-
-// De-allocate the data referenced by target ptr on the device. In case of
-// success, return zero. Otherwise, return an error code. Kind dictates what
-// allocator to use (e.g. shared, host, device).
-int32_t __tgt_rtl_data_delete(int32_t ID, void *TargetPtr, int32_t Kind);
-
-// Transfer control to the offloaded entry Entry on the target device.
-// Args and Offsets are arrays of NumArgs size of target addresses and
-// offsets. An offset should be added to the target address before passing it
-// to the outlined function on device side. If AsyncInfo is nullptr, it is
-// synchronous; otherwise it is asynchronous. However, AsyncInfo may be
-// ignored on some platforms, like x86_64. In that case, it is synchronous. In
-// case of success, return zero. Otherwise, return an error code.
-int32_t __tgt_rtl_run_target_region(int32_t ID, void *Entry, void **Args,
-                                    ptrdiff_t *Offsets, int32_t NumArgs);
-
-// Asynchronous version of __tgt_rtl_run_target_region
-int32_t __tgt_rtl_run_target_region_async(int32_t ID, void *Entry, void **Args,
-                                          ptrdiff_t *Offsets, int32_t NumArgs,
-                                          __tgt_async_info *AsyncInfo);
-
-// Similar to __tgt_rtl_run_target_region, but additionally specify the
-// number of teams to be created and a number of threads in each team. If
-// AsyncInfo is nullptr, it is synchronous; otherwise it is asynchronous.
-// However, AsyncInfo may be ignored on some platforms, like x86_64. In that
-// case, it is synchronous.
-int32_t __tgt_rtl_run_target_team_region(int32_t ID, void *Entry, void **Args,
-                                         ptrdiff_t *Offsets, int32_t NumArgs,
-                                         int32_t NumTeams, int32_t ThreadLimit,
-                                         uint64_t LoopTripcount);
-
-// Asynchronous version of __tgt_rtl_run_target_team_region
-int32_t __tgt_rtl_run_target_team_region_async(
-    int32_t ID, void *Entry, void **Args, ptrdiff_t *Offsets, int32_t NumArgs,
-    int32_t NumTeams, int32_t ThreadLimit, uint64_t LoopTripcount,
-    __tgt_async_info *AsyncInfo);
-
-// Device synchronization. In case of success, return zero. Otherwise, return an
-// error code.
-int32_t __tgt_rtl_synchronize(int32_t ID, __tgt_async_info *AsyncInfo);
-
-// Queries for the completion of asynchronous operations. Instead of blocking
-// the calling thread as __tgt_rtl_synchronize, the progress of the operations
-// stored in AsyncInfo->Queue is queried in a non-blocking manner, partially
-// advancing their execution. If all operations are completed, AsyncInfo->Queue
-// is set to nullptr. If there are still pending operations, AsyncInfo->Queue is
-// kept as a valid queue. In any case of success (i.e., successful query
-// with/without completing all operations), return zero. Otherwise, return an
-// error code.
-int32_t __tgt_rtl_query_async(int32_t ID, __tgt_async_info *AsyncInfo);
-
-// Set plugin's internal information flag externally.
-void __tgt_rtl_set_info_flag(uint32_t);
-
-// Print the device information
-void __tgt_rtl_print_device_info(int32_t ID);
-
-// Event related interfaces. It is expected to use the interfaces in the
-// following way:
-// 1) Create an event on the target device (__tgt_rtl_create_event).
-// 2) Record the event based on the status of \p AsyncInfo->Queue at the moment
-// of function call to __tgt_rtl_record_event. An event becomes "meaningful"
-// once it is recorded, such that others can depend on it.
-// 3) Call __tgt_rtl_wait_event to set dependence on the event. Whether the
-// operation is blocking or non-blocking depends on the target. It is expected
-// to be non-blocking, just set dependence and return.
-// 4) Call __tgt_rtl_sync_event to sync the event. It is expected to block the
-// thread calling the function.
-// 5) Destroy the event (__tgt_rtl_destroy_event).
-// {
-int32_t __tgt_rtl_create_event(int32_t ID, void **Event);
-
-int32_t __tgt_rtl_record_event(int32_t ID, void *Event,
-                               __tgt_async_info *AsyncInfo);
-
-int32_t __tgt_rtl_wait_event(int32_t ID, void *Event,
-                             __tgt_async_info *AsyncInfo);
-
-int32_t __tgt_rtl_sync_event(int32_t ID, void *Event);
-
-int32_t __tgt_rtl_destroy_event(int32_t ID, void *Event);
-// }
-
-int32_t __tgt_rtl_init_async_info(int32_t ID, __tgt_async_info **AsyncInfoPtr);
-int32_t __tgt_rtl_init_device_info(int32_t ID, __tgt_device_info *DeviceInfoPtr,
-                                   const char **ErrStr);
-
-// lock/pin host memory
-int32_t __tgt_rtl_data_lock(int32_t ID, void *HstPtr, int64_t Size,
-                            void **LockedPtr);
-
-// unlock/unpin host memory
-int32_t __tgt_rtl_data_unlock(int32_t ID, void *HstPtr);
-
-// Notify the plugin about a new mapping starting at the host address \p HstPtr
-// and \p Size bytes. The plugin may lock/pin that buffer to achieve optimal
-// memory transfers involving that buffer.
-int32_t __tgt_rtl_data_notify_mapped(int32_t ID, void *HstPtr, int64_t Size);
-
-// Notify the plugin about an existing mapping being unmapped, starting at the
-// host address \p HstPtr and \p Size bytes.
-int32_t __tgt_rtl_data_notify_unmapped(int32_t ID, void *HstPtr);
-
-// Set the global device identifier offset, such that the plugin may determine a
-// unique device number.
-int32_t __tgt_rtl_set_device_offset(int32_t DeviceIdOffset);
-
-int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
-                                void **TgtArgs, ptrdiff_t *TgtOffsets,
-                                KernelArgsTy *KernelArgs,
-                                __tgt_async_info *AsyncInfoPtr);
-
-int32_t __tgt_rtl_initialize_record_replay(int32_t DeviceId, int64_t MemorySize,
-                                           void *VAddr, bool isRecord,
-                                           bool SaveOutput,
-                                           uint64_t &ReqPtrArgOffset);
-
-// Returns true if the device \p DeviceId suggests to use auto zero-copy.
-int32_t __tgt_rtl_use_auto_zero_copy(int32_t DeviceId);
-}
-
-#endif // OMPTARGET_SHARED_PLUGIN_API_H
diff --git a/offload/include/Shared/PluginAPI.inc b/offload/include/Shared/PluginAPI.inc
deleted file mode 100644
index e445da6852f7..000000000000
--- a/offload/include/Shared/PluginAPI.inc
+++ /dev/null
@@ -1,51 +0,0 @@
-//===-- Shared/PluginAPI.inc - Target independent plugin API ----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the names of the interface functions between target
-// independent offload runtime library and target dependent plugins.
-//
-//===----------------------------------------------------------------------===//
-
-// No include guards!
-
-PLUGIN_API_HANDLE(init_plugin);
-PLUGIN_API_HANDLE(is_valid_binary);
-PLUGIN_API_HANDLE(is_data_exchangable);
-PLUGIN_API_HANDLE(number_of_devices);
-PLUGIN_API_HANDLE(init_device);
-PLUGIN_API_HANDLE(load_binary);
-PLUGIN_API_HANDLE(get_global);
-PLUGIN_API_HANDLE(get_function);
-PLUGIN_API_HANDLE(data_alloc);
-PLUGIN_API_HANDLE(data_submit);
-PLUGIN_API_HANDLE(data_submit_async);
-PLUGIN_API_HANDLE(data_retrieve);
-PLUGIN_API_HANDLE(data_retrieve_async);
-PLUGIN_API_HANDLE(data_exchange);
-PLUGIN_API_HANDLE(data_exchange_async);
-PLUGIN_API_HANDLE(data_delete);
-PLUGIN_API_HANDLE(launch_kernel);
-PLUGIN_API_HANDLE(init_requires);
-PLUGIN_API_HANDLE(synchronize);
-PLUGIN_API_HANDLE(query_async);
-PLUGIN_API_HANDLE(set_info_flag);
-PLUGIN_API_HANDLE(print_device_info);
-PLUGIN_API_HANDLE(create_event);
-PLUGIN_API_HANDLE(record_event);
-PLUGIN_API_HANDLE(wait_event);
-PLUGIN_API_HANDLE(sync_event);
-PLUGIN_API_HANDLE(destroy_event);
-PLUGIN_API_HANDLE(init_async_info);
-PLUGIN_API_HANDLE(init_device_info);
-PLUGIN_API_HANDLE(data_lock);
-PLUGIN_API_HANDLE(data_unlock);
-PLUGIN_API_HANDLE(data_notify_mapped);
-PLUGIN_API_HANDLE(data_notify_unmapped);
-PLUGIN_API_HANDLE(set_device_offset);
-PLUGIN_API_HANDLE(initialize_record_replay);
-PLUGIN_API_HANDLE(use_auto_zero_copy);
-- 
GitLab


From 224116ab9f4d97acb0cca76dc63c1d3d4a253078 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Thu, 16 May 2024 09:34:24 -0700
Subject: [PATCH 1035/1206] Revert "[Clang][CWG1815] Support lifetime extension
 of temporary created by aggregate initialization using a default member
 initializer (#87933)"

This reverts commit 17daa204feadf9c28fc13b7daa69c3cbe865b238.

Multiple examples on the PR
https://github.com/llvm/llvm-project/pull/87933

show regressions, so reverting until they can be fixed in the followup.
---
 .../clang/Basic/DiagnosticSemaKinds.td        |  6 ++++
 clang/lib/Sema/SemaExpr.cpp                   | 31 +++++------------
 clang/lib/Sema/SemaInit.cpp                   | 19 ++++++++++-
 clang/test/AST/ast-dump-default-init-json.cpp |  6 ++--
 clang/test/AST/ast-dump-default-init.cpp      |  2 +-
 .../Analysis/lifetime-extended-regions.cpp    |  9 +++--
 clang/test/CXX/drs/cwg16xx.cpp                |  2 ++
 clang/test/CXX/drs/cwg18xx.cpp                | 25 +++++---------
 clang/test/CXX/special/class.temporary/p6.cpp | 34 -------------------
 clang/test/SemaCXX/constexpr-default-arg.cpp  |  4 +--
 clang/test/SemaCXX/eval-crashes.cpp           |  6 ++--
 clang/www/cxx_dr_status.html                  |  2 +-
 12 files changed, 57 insertions(+), 89 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index e648b503ac03..3be9d34b36b0 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -10037,6 +10037,12 @@ def warn_new_dangling_initializer_list : Warning<
   "the allocated initializer list}0 "
   "will be destroyed at the end of the full-expression">,
   InGroup<DanglingInitializerList>;
+def warn_unsupported_lifetime_extension : Warning<
+  "lifetime extension of "
+  "%select{temporary|backing array of initializer list}0 created "
+  "by aggregate initialization using a default member initializer "
+  "is not yet supported; lifetime of %select{temporary|backing array}0 "
+  "will end at the end of the full-expression">, InGroup<Dangling>;
 
 // For non-floating point, expressions of the form x == x or x != x
 // should result in a warning, since these always evaluate to a constant.
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 151cda5d68b0..274e1fb18353 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -5618,9 +5618,10 @@ ExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc,
         Res = Immediate.TransformInitializer(Param->getInit(),
                                              /*NotCopy=*/false);
       });
-      if (Res.isUsable())
-        Res = ConvertParamDefaultArgument(Param, Res.get(),
-                                          Res.get()->getBeginLoc());
+      if (Res.isInvalid())
+        return ExprError();
+      Res = ConvertParamDefaultArgument(Param, Res.get(),
+                                        Res.get()->getBeginLoc());
       if (Res.isInvalid())
         return ExprError();
       Init = Res.get();
@@ -5656,7 +5657,7 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) {
   Expr *Init = nullptr;
 
   bool NestedDefaultChecking = isCheckingDefaultArgumentOrInitializer();
-  bool InLifetimeExtendingContext = isInLifetimeExtendingContext();
+
   EnterExpressionEvaluationContext EvalContext(
       *this, ExpressionEvaluationContext::PotentiallyEvaluated, Field);
 
@@ -5691,35 +5692,19 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) {
   ImmediateCallVisitor V(getASTContext());
   if (!NestedDefaultChecking)
     V.TraverseDecl(Field);
-
-  // CWG1815
-  // Support lifetime extension of temporary created by aggregate
-  // initialization using a default member initializer. We should always rebuild
-  // the initializer if it contains any temporaries (if the initializer
-  // expression is an ExprWithCleanups). Then make sure the normal lifetime
-  // extension code recurses into the default initializer and does lifetime
-  // extension when warranted.
-  bool ContainsAnyTemporaries =
-      isa_and_present<ExprWithCleanups>(Field->getInClassInitializer());
-  if (V.HasImmediateCalls || InLifetimeExtendingContext ||
-      ContainsAnyTemporaries) {
+  if (V.HasImmediateCalls) {
     ExprEvalContexts.back().DelayedDefaultInitializationContext = {Loc, Field,
                                                                    CurContext};
     ExprEvalContexts.back().IsCurrentlyCheckingDefaultArgumentOrInitializer =
         NestedDefaultChecking;
-    // Pass down lifetime extending flag, and collect temporaries in
-    // CreateMaterializeTemporaryExpr when we rewrite the call argument.
-    keepInLifetimeExtendingContext();
+
     EnsureImmediateInvocationInDefaultArgs Immediate(*this);
     ExprResult Res;
-
-    // Rebuild CXXDefaultInitExpr might cause diagnostics.
-    SFINAETrap Trap(*this);
     runWithSufficientStackSpace(Loc, [&] {
       Res = Immediate.TransformInitializer(Field->getInClassInitializer(),
                                            /*CXXDirectInit=*/false);
     });
-    if (Res.isUsable())
+    if (!Res.isInvalid())
       Res = ConvertMemberDefaultInitExpression(Field, Res.get(), Loc);
     if (Res.isInvalid()) {
       Field->setInvalidDecl();
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 708286e192f9..2177972f3af2 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -8066,6 +8066,11 @@ static void visitLocalsRetainedByInitializer(IndirectLocalPath &Path,
 enum PathLifetimeKind {
   /// Lifetime-extend along this path.
   Extend,
+  /// We should lifetime-extend, but we don't because (due to technical
+  /// limitations) we can't. This happens for default member initializers,
+  /// which we don't clone for every use, so we don't have a unique
+  /// MaterializeTemporaryExpr to update.
+  ShouldExtend,
   /// Do not lifetime extend along this path.
   NoExtend
 };
@@ -8077,7 +8082,7 @@ shouldLifetimeExtendThroughPath(const IndirectLocalPath &Path) {
   PathLifetimeKind Kind = PathLifetimeKind::Extend;
   for (auto Elem : Path) {
     if (Elem.Kind == IndirectLocalPathEntry::DefaultInit)
-      Kind = PathLifetimeKind::Extend;
+      Kind = PathLifetimeKind::ShouldExtend;
     else if (Elem.Kind != IndirectLocalPathEntry::LambdaCaptureInit)
       return PathLifetimeKind::NoExtend;
   }
@@ -8197,6 +8202,18 @@ void Sema::checkInitializerLifetime(const InitializedEntity &Entity,
                               ExtendingEntity->allocateManglingNumber());
         // Also visit the temporaries lifetime-extended by this initializer.
         return true;
+
+      case PathLifetimeKind::ShouldExtend:
+        // We're supposed to lifetime-extend the temporary along this path (per
+        // the resolution of DR1815), but we don't support that yet.
+        //
+        // FIXME: Properly handle this situation. Perhaps the easiest approach
+        // would be to clone the initializer expression on each use that would
+        // lifetime extend its temporaries.
+        Diag(DiagLoc, diag::warn_unsupported_lifetime_extension)
+            << RK << DiagRange;
+        break;
+
       case PathLifetimeKind::NoExtend:
         // If the path goes through the initialization of a variable or field,
         // it can't possibly reach a temporary created in this full-expression.
diff --git a/clang/test/AST/ast-dump-default-init-json.cpp b/clang/test/AST/ast-dump-default-init-json.cpp
index f4949a9c9eed..1058b4e3ea4d 100644
--- a/clang/test/AST/ast-dump-default-init-json.cpp
+++ b/clang/test/AST/ast-dump-default-init-json.cpp
@@ -789,10 +789,10 @@ void test() {
 // CHECK-NEXT:                  "valueCategory": "lvalue",
 // CHECK-NEXT:                  "extendingDecl": {
 // CHECK-NEXT:                   "id": "0x{{.*}}",
-// CHECK-NEXT:                   "kind": "VarDecl",
-// CHECK-NEXT:                   "name": "b",
+// CHECK-NEXT:                   "kind": "FieldDecl",
+// CHECK-NEXT:                   "name": "a",
 // CHECK-NEXT:                   "type": {
-// CHECK-NEXT:                    "qualType": "B"
+// CHECK-NEXT:                    "qualType": "const A &"
 // CHECK-NEXT:                   }
 // CHECK-NEXT:                  },
 // CHECK-NEXT:                  "storageDuration": "automatic",
diff --git a/clang/test/AST/ast-dump-default-init.cpp b/clang/test/AST/ast-dump-default-init.cpp
index 26864fbf1542..15b29f04bf21 100644
--- a/clang/test/AST/ast-dump-default-init.cpp
+++ b/clang/test/AST/ast-dump-default-init.cpp
@@ -13,7 +13,7 @@ void test() {
 }
 // CHECK: -CXXDefaultInitExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue has rewritten init
 // CHECK-NEXT:  `-ExprWithCleanups 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue
-// CHECK-NEXT:    `-MaterializeTemporaryExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue extended by Var 0x{{[^ ]*}} 'b' 'B'
+// CHECK-NEXT:    `-MaterializeTemporaryExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue extended by Field 0x{{[^ ]*}} 'a' 'const A &'
 // CHECK-NEXT:      `-ImplicitCastExpr 0x{{[^ ]*}} <{{.*}}> 'const A' <NoOp>
 // CHECK-NEXT:        `-CXXFunctionalCastExpr 0x{{[^ ]*}} <{{.*}}> 'A' functional cast to A <NoOp>
 // CHECK-NEXT:          `-InitListExpr 0x{{[^ ]*}} <{{.*}}> 'A'
diff --git a/clang/test/Analysis/lifetime-extended-regions.cpp b/clang/test/Analysis/lifetime-extended-regions.cpp
index 4458ad294af7..4e98bd4b0403 100644
--- a/clang/test/Analysis/lifetime-extended-regions.cpp
+++ b/clang/test/Analysis/lifetime-extended-regions.cpp
@@ -120,11 +120,10 @@ void aggregateWithReferences() {
   clang_analyzer_dump(viaReference);    // expected-warning-re {{&lifetime_extended_object{RefAggregate, viaReference, S{{[0-9]+}}} }}
   clang_analyzer_dump(viaReference.rx); // expected-warning-re {{&lifetime_extended_object{int, viaReference, S{{[0-9]+}}} }}
   clang_analyzer_dump(viaReference.ry); // expected-warning-re {{&lifetime_extended_object{Composite, viaReference, S{{[0-9]+}}} }}
-  
-  // FIXME: clang currently support extending lifetime of object bound to reference members of aggregates,
-  // that are created from default member initializer. But CFG and ExprEngine need to be updated to address this change.
-  // The following expect warning: {{&lifetime_extended_object{Composite, defaultInitExtended, S{{[0-9]+}}} }}
-  RefAggregate defaultInitExtended{i};
+
+  // clang does not currently implement extending lifetime of object bound to reference members of aggregates,
+  // that are created from default member initializer (see `warn_unsupported_lifetime_extension` from `-Wdangling`)
+  RefAggregate defaultInitExtended{i}; // clang-bug does not extend `Composite`
   clang_analyzer_dump(defaultInitExtended.ry); // expected-warning {{Unknown }}
 }
 
diff --git a/clang/test/CXX/drs/cwg16xx.cpp b/clang/test/CXX/drs/cwg16xx.cpp
index 82ef871939d2..cf6b45ceabf2 100644
--- a/clang/test/CXX/drs/cwg16xx.cpp
+++ b/clang/test/CXX/drs/cwg16xx.cpp
@@ -483,6 +483,8 @@ namespace cwg1696 { // cwg1696: 7
     const A &a = A(); // #cwg1696-D1-a
   };
   D1 d1 = {}; // #cwg1696-d1
+  // since-cxx14-warning@-1 {{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported; lifetime of temporary will end at the end of the full-expression}}
+  //   since-cxx14-note@#cwg1696-D1-a {{initializing field 'a' with default member initializer}}
 
   struct D2 {
     const A &a = A(); // #cwg1696-D2-a
diff --git a/clang/test/CXX/drs/cwg18xx.cpp b/clang/test/CXX/drs/cwg18xx.cpp
index 89adc2838490..35615076a628 100644
--- a/clang/test/CXX/drs/cwg18xx.cpp
+++ b/clang/test/CXX/drs/cwg18xx.cpp
@@ -56,7 +56,7 @@ namespace cwg1804 { // cwg1804: 2.7
 template <typename, typename>
 struct A {
   void f1();
-  
+
   template <typename V>
   void f2(V);
 
@@ -73,7 +73,7 @@ struct A {
 template <typename U>
 struct A<int, U> {
   void f1();
-  
+
   template <typename V>
   void f2(V);
 
@@ -97,7 +97,7 @@ class D {
 template <typename U>
 struct A<double, U> {
   void f1();
-  
+
   template <typename V>
   void f2(V);
 
@@ -206,28 +206,19 @@ namespace cwg1814 { // cwg1814: yes
 #endif
 }
 
-namespace cwg1815 { // cwg1815: 19
+namespace cwg1815 { // cwg1815: no
 #if __cplusplus >= 201402L
-  struct A { int &&r = 0; };
+  // FIXME: needs codegen test
+  struct A { int &&r = 0; }; // #cwg1815-A
   A a = {};
+  // since-cxx14-warning@-1 {{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported; lifetime of temporary will end at the end of the full-expression}} FIXME
+  //   since-cxx14-note@#cwg1815-A {{initializing field 'r' with default member initializer}}
 
   struct B { int &&r = 0; }; // #cwg1815-B
   // since-cxx14-error@-1 {{reference member 'r' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
   //   since-cxx14-note@#cwg1815-B {{initializing field 'r' with default member initializer}}
   //   since-cxx14-note@#cwg1815-b {{in implicit default constructor for 'cwg1815::B' first required here}}
   B b; // #cwg1815-b
-
-#if __cplusplus >= 201703L
-  struct C { const int &r = 0; };
-  constexpr C c = {}; // OK, since cwg1815
-  static_assert(c.r == 0);
-
-  constexpr int f() {
-    A a = {}; // OK, since cwg1815
-    return a.r;
-  }
-  static_assert(f() == 0);
-#endif
 #endif
 }
 
diff --git a/clang/test/CXX/special/class.temporary/p6.cpp b/clang/test/CXX/special/class.temporary/p6.cpp
index a6d2adfd1fd2..5554363cc69a 100644
--- a/clang/test/CXX/special/class.temporary/p6.cpp
+++ b/clang/test/CXX/special/class.temporary/p6.cpp
@@ -269,40 +269,6 @@ void init_capture_init_list() {
   // CHECK: }
 }
 
-void check_dr1815() { // dr1815: yes
-#if __cplusplus >= 201402L
-
-  struct A {
-    int &&r = 0;
-    ~A() {}
-  };
-
-  struct B {
-    A &&a = A{};
-    ~B() {}
-  };
-  B a = {};
-  
-  // CHECK: call {{.*}}block_scope_begin_function
-  extern void block_scope_begin_function();
-  extern void block_scope_end_function();
-  block_scope_begin_function();
-  {
-    // CHECK: call void @_ZZ12check_dr1815vEN1BD1Ev
-    // CHECK: call void @_ZZ12check_dr1815vEN1AD1Ev
-    B b = {};
-  }
-  // CHECK: call {{.*}}block_scope_end_function
-  block_scope_end_function();
-
-  // CHECK: call {{.*}}some_other_function
-  extern void some_other_function();
-  some_other_function();
-  // CHECK: call void @_ZZ12check_dr1815vEN1BD1Ev
-  // CHECK: call void @_ZZ12check_dr1815vEN1AD1Ev
-#endif
-}
-
 namespace P2718R0 {
 namespace basic {
 template <typename E> using T2 = std::list<E>;
diff --git a/clang/test/SemaCXX/constexpr-default-arg.cpp b/clang/test/SemaCXX/constexpr-default-arg.cpp
index 901123bfb359..ec9b2927880b 100644
--- a/clang/test/SemaCXX/constexpr-default-arg.cpp
+++ b/clang/test/SemaCXX/constexpr-default-arg.cpp
@@ -32,8 +32,8 @@ void test_default_arg2() {
 }
 
 // Check that multiple CXXDefaultInitExprs don't cause an assertion failure.
-struct A { int &&r = 0; };
+struct A { int &&r = 0; }; // expected-note 2{{default member initializer}}
 struct B { A x, y; };
-B b = {}; // expected-no-diagnostics
+B b = {}; // expected-warning 2{{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported}}
 
 }
diff --git a/clang/test/SemaCXX/eval-crashes.cpp b/clang/test/SemaCXX/eval-crashes.cpp
index a06f60f71e9c..017df977b26b 100644
--- a/clang/test/SemaCXX/eval-crashes.cpp
+++ b/clang/test/SemaCXX/eval-crashes.cpp
@@ -25,9 +25,11 @@ namespace pr33140_0b {
 }
 
 namespace pr33140_2 {
-  struct A { int &&r = 0; };
+  // FIXME: The declaration of 'b' below should lifetime-extend two int
+  // temporaries.
+  struct A { int &&r = 0; }; // expected-note 2{{initializing field 'r' with default member initializer}}
   struct B { A x, y; };
-  B b = {};
+  B b = {}; // expected-warning 2{{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported}}
 }
 
 namespace pr33140_3 {
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index abf5d4ae4676..2ed79fa21db2 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -10698,7 +10698,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1815.html">1815</a></td>
     <td>CD4</td>
     <td>Lifetime extension in aggregate initialization</td>
-    <td class="unreleased" align="center">Clang 19</td>
+    <td class="none" align="center">No</td>
   </tr>
   <tr id="1816">
     <td><a href="https://cplusplus.github.io/CWG/issues/1816.html">1816</a></td>
-- 
GitLab


From bd6c358d01f6ebc3851996e2c29c47b08e992525 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 16 May 2024 09:53:47 -0700
Subject: [PATCH 1036/1206] [lldb-dap] Update repository in package.json

Use https://github.com/llvm/vscode-lldb instead of the monorepo, for
consistency with the other two extensions (mlir, clangd).
---
 lldb/tools/lldb-dap/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/tools/lldb-dap/package.json b/lldb/tools/lldb-dap/package.json
index aeb24445551c..45fcb83cf81d 100644
--- a/lldb/tools/lldb-dap/package.json
+++ b/lldb/tools/lldb-dap/package.json
@@ -8,7 +8,7 @@
   "license": "Apache 2.0 License with LLVM exceptions",
   "repository": {
     "type": "git",
-    "url": "https://github.com/llvm/llvm-project.git"
+    "url": "https://github.com/llvm/vscode-lldb.git"
   },
   "bugs": {
     "url": "https://github.com/llvm/llvm-project/issues"
-- 
GitLab


From f03430f5e37e8eb64878dc538b05210adea2d80f Mon Sep 17 00:00:00 2001
From: Xiaoyang Liu <siujoeng.lau@gmail.com>
Date: Thu, 16 May 2024 13:25:04 -0400
Subject: [PATCH 1037/1206] [libc++] LWG3672: `common_iterator::operator->()`
 should return by value (#87899)

## Abstract

This pull request implements LWG3672: `common_iterator::operator->()`
should return by value. The current implementation specifies that this
function should return the underlying pointer by reference (`T*
const&`), but it would be more intuitive to return it by value (`T*`).

## Reference

- [Draft C++ Standard:
[common.iter.access]](https://eel.is/c++draft/common.iter.access)
- [LWG3672](https://cplusplus.github.io/LWG/issue3672)
---
 libcxx/docs/Status/Cxx23Issues.csv             |  2 +-
 libcxx/include/__iterator/common_iterator.h    |  2 +-
 .../iterators.common/arrow.pass.cpp            | 18 +++++++++---------
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index 04f6e23375ac..d421feef8db9 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -165,7 +165,7 @@
 "`3659 <https://wg21.link/LWG3659>`__","Consider ``ATOMIC_FLAG_INIT`` undeprecation","July 2022","|Complete|","15.0"
 "`3670 <https://wg21.link/LWG3670>`__","``Cpp17InputIterators`` don't have integer-class difference types","July 2022","","","|ranges|"
 "`3671 <https://wg21.link/LWG3671>`__","``atomic_fetch_xor`` missing from ``stdatomic.h``","July 2022","",""
-"`3672 <https://wg21.link/LWG3672>`__","``common_iterator::operator->()`` should return by value","July 2022","","","|ranges|"
+"`3672 <https://wg21.link/LWG3672>`__","``common_iterator::operator->()`` should return by value","July 2022","|Complete|","19.0","|ranges|"
 "`3683 <https://wg21.link/LWG3683>`__","``operator==`` for ``polymorphic_allocator`` cannot deduce template argument in common cases","July 2022","",""
 "`3687 <https://wg21.link/LWG3687>`__","``expected<cv void, E>`` move constructor should move","July 2022","|Complete|","16.0"
 "`3692 <https://wg21.link/LWG3692>`__","``zip_view::iterator``'s ``operator<=>`` is overconstrained","July 2022","","","|ranges| |spaceship|"
diff --git a/libcxx/include/__iterator/common_iterator.h b/libcxx/include/__iterator/common_iterator.h
index 7b3f4610d531..199de2cc7337 100644
--- a/libcxx/include/__iterator/common_iterator.h
+++ b/libcxx/include/__iterator/common_iterator.h
@@ -124,7 +124,7 @@ public:
   }
 
   template <class _I2 = _Iter>
-  _LIBCPP_HIDE_FROM_ABI decltype(auto) operator->() const
+  _LIBCPP_HIDE_FROM_ABI auto operator->() const
     requires indirectly_readable<const _I2> && (requires(const _I2& __i) {
                __i.operator->();
              } || is_reference_v<iter_reference_t<_I2>> || constructible_from<iter_value_t<_I2>, iter_reference_t<_I2>>)
diff --git a/libcxx/test/std/iterators/predef.iterators/iterators.common/arrow.pass.cpp b/libcxx/test/std/iterators/predef.iterators/iterators.common/arrow.pass.cpp
index ec992eb98b4b..ee7628b3e4db 100644
--- a/libcxx/test/std/iterators/predef.iterators/iterators.common/arrow.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/iterators.common/arrow.pass.cpp
@@ -8,7 +8,7 @@
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
-// decltype(auto) operator->() const
+// auto operator->() const
 //   requires see below;
 
 #include <iterator>
@@ -28,11 +28,11 @@ void test() {
       using Common = std::common_iterator<Iterator, sentinel_wrapper<Iterator>>;
 
       Common common(iter);
-      std::same_as<Iterator> auto result = common.operator->();
+      std::same_as<Iterator> decltype(auto) result = common.operator->();
       assert(base(result) == buffer);
 
       Common const ccommon(iter);
-      std::same_as<Iterator> auto cresult = ccommon.operator->();
+      std::same_as<Iterator> decltype(auto) cresult = ccommon.operator->();
       assert(base(cresult) == buffer);
     };
 
@@ -48,11 +48,11 @@ void test() {
       using Common = std::common_iterator<Iterator, sentinel_type<int*>>;
 
       Common common(iter);
-      std::same_as<int*> auto result = common.operator->();
+      std::same_as<int*> decltype(auto) result = common.operator->();
       assert(result == buffer);
 
       Common const ccommon(iter);
-      std::same_as<int*> auto cresult = ccommon.operator->();
+      std::same_as<int*> decltype(auto) cresult = ccommon.operator->();
       assert(cresult == buffer);
     };
 
@@ -72,14 +72,14 @@ void test() {
       using Common = std::common_iterator<Iterator, sentinel_type<int*>>;
 
       Common common(iter);
-      auto proxy = common.operator->();
-      std::same_as<int const*> auto result = proxy.operator->();
+      auto proxy                                     = common.operator->();
+      std::same_as<int const*> decltype(auto) result = proxy.operator->();
       assert(result != buffer); // we copied to a temporary proxy
       assert(*result == *buffer);
 
       Common const ccommon(iter);
-      auto cproxy = ccommon.operator->();
-      std::same_as<int const*> auto cresult = cproxy.operator->();
+      auto cproxy                                     = ccommon.operator->();
+      std::same_as<int const*> decltype(auto) cresult = cproxy.operator->();
       assert(cresult != buffer); // we copied to a temporary proxy
       assert(*cresult == *buffer);
     };
-- 
GitLab


From c7ae8c6639370ccbc583dca019bbb78761ce423d Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Thu, 16 May 2024 21:30:31 +0400
Subject: [PATCH 1038/1206] [lldb] Fixed the DAP tests in case of a remote
 target (#92416)

These tests are based on dap_server which runs locally. These tests
failed in case of Windows host and Linux target.
---
 lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py | 1 +
 lldb/test/API/tools/lldb-dap/exception/TestDAP_exception.py     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py b/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
index 2b3ec656c107..3250a5093cac 100644
--- a/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
+++ b/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
@@ -19,6 +19,7 @@ class TestDAP_completions(lldbdap_testcase.DAPTestCaseBase):
             self.assertNotIn(not_expected_item, actual_list)
 
     @skipIfWindows
+    @skipIfRemote
     @skipIf(compiler="clang", compiler_version=["<", "17.0"])
     def test_completions(self):
         """
diff --git a/lldb/test/API/tools/lldb-dap/exception/TestDAP_exception.py b/lldb/test/API/tools/lldb-dap/exception/TestDAP_exception.py
index 8c2c0154ba65..58a67d816436 100644
--- a/lldb/test/API/tools/lldb-dap/exception/TestDAP_exception.py
+++ b/lldb/test/API/tools/lldb-dap/exception/TestDAP_exception.py
@@ -9,6 +9,7 @@ import lldbdap_testcase
 
 
 class TestDAP_exception(lldbdap_testcase.DAPTestCaseBase):
+    @skipIfRemote
     @skipIfWindows
     def test_stopped_description(self):
         """
-- 
GitLab


From 0dc80e4b2640f80a5b421e6e636d2a7243150917 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 16 May 2024 10:35:45 -0700
Subject: [PATCH 1039/1206] [memprof] Group MemProf data structures into a
 struct (NFC) (#92360)

This patch groups the three Memprof data structures into a struct
named IndexedMemProfData and teaches InstrProfWriter to use it.  This
way, we can pass IndexedMemProfData to writeMemProf and its helpers
instead of individual data structures.

As a follow-up, we can use the new struct in MemProfReader also.  That
in turn allows loadInput in llvm-profdata to move the MemProf data
into the writer context, saving a few seconds for a large MemProf
profile.
---
 .../llvm/ProfileData/InstrProfWriter.h        | 14 +--
 llvm/include/llvm/ProfileData/MemProf.h       | 14 +++
 llvm/lib/ProfileData/InstrProfWriter.cpp      | 86 ++++++++-----------
 3 files changed, 50 insertions(+), 64 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index 08db8fa6e7ef..1714f3b6cf3a 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -51,18 +51,8 @@ private:
   SmallVector<TemporalProfTraceTy> TemporalProfTraces;
   std::mt19937 RNG;
 
-  // A map to hold memprof data per function. The lower 64 bits obtained from
-  // the md5 hash of the function name is used to index into the map.
-  llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
-      MemProfRecordData;
-  // A map to hold frame id to frame mappings. The mappings are used to
-  // convert IndexedMemProfRecord to MemProfRecords with frame information
-  // inline.
-  llvm::MapVector<memprof::FrameId, memprof::Frame> MemProfFrameData;
-
-  // A map to hold call stack id to call stacks.
-  llvm::MapVector<memprof::CallStackId, llvm::SmallVector<memprof::FrameId>>
-      MemProfCallStackData;
+  // The MemProf data.
+  memprof::IndexedMemProfData MemProfData;
 
   // List of binary ids.
   std::vector<llvm::object::BuildID> BinaryIds;
diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index 60bff76d0e46..66a99f16cdb6 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -833,6 +833,20 @@ template <typename MapTy> struct CallStackIdConverter {
   }
 };
 
+struct IndexedMemProfData {
+  // A map to hold memprof data per function. The lower 64 bits obtained from
+  // the md5 hash of the function name is used to index into the map.
+  llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord> RecordData;
+
+  // A map to hold frame id to frame mappings. The mappings are used to
+  // convert IndexedMemProfRecord to MemProfRecords with frame information
+  // inline.
+  llvm::MapVector<FrameId, Frame> FrameData;
+
+  // A map to hold call stack id to call stacks.
+  llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> CallStackData;
+};
+
 // Verify that each CallStackId is computed with hashCallStack.  This function
 // is intended to help transition from CallStack to CSId in
 // IndexedAllocationInfo.
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index c941b9d89df3..aeafd429e8ba 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -273,7 +273,7 @@ void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
 
 void InstrProfWriter::addMemProfRecord(
     const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) {
-  auto [Iter, Inserted] = MemProfRecordData.insert({Id, Record});
+  auto [Iter, Inserted] = MemProfData.RecordData.insert({Id, Record});
   // If we inserted a new record then we are done.
   if (Inserted) {
     return;
@@ -285,7 +285,7 @@ void InstrProfWriter::addMemProfRecord(
 bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id,
                                       const memprof::Frame &Frame,
                                       function_ref<void(Error)> Warn) {
-  auto [Iter, Inserted] = MemProfFrameData.insert({Id, Frame});
+  auto [Iter, Inserted] = MemProfData.FrameData.insert({Id, Frame});
   // If a mapping already exists for the current frame id and it does not
   // match the new mapping provided then reset the existing contents and bail
   // out. We don't support the merging of memprof data whose Frame -> Id
@@ -302,7 +302,7 @@ bool InstrProfWriter::addMemProfCallStack(
     const memprof::CallStackId CSId,
     const llvm::SmallVector<memprof::FrameId> &CallStack,
     function_ref<void(Error)> Warn) {
-  auto [Iter, Inserted] = MemProfCallStackData.insert({CSId, CallStack});
+  auto [Iter, Inserted] = MemProfData.CallStackData.insert({CSId, CallStack});
   // If a mapping already exists for the current call stack id and it does not
   // match the new mapping provided then reset the existing contents and bail
   // out. We don't support the merging of memprof data whose CallStack -> Id
@@ -389,22 +389,22 @@ void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,
   addTemporalProfileTraces(IPW.TemporalProfTraces,
                            IPW.TemporalProfTraceStreamSize);
 
-  MemProfFrameData.reserve(IPW.MemProfFrameData.size());
-  for (auto &[FrameId, Frame] : IPW.MemProfFrameData) {
+  MemProfData.FrameData.reserve(IPW.MemProfData.FrameData.size());
+  for (auto &[FrameId, Frame] : IPW.MemProfData.FrameData) {
     // If we weren't able to add the frame mappings then it doesn't make sense
     // to try to merge the records from this profile.
     if (!addMemProfFrame(FrameId, Frame, Warn))
       return;
   }
 
-  MemProfCallStackData.reserve(IPW.MemProfCallStackData.size());
-  for (auto &[CSId, CallStack] : IPW.MemProfCallStackData) {
+  MemProfData.CallStackData.reserve(IPW.MemProfData.CallStackData.size());
+  for (auto &[CSId, CallStack] : IPW.MemProfData.CallStackData) {
     if (!addMemProfCallStack(CSId, CallStack, Warn))
       return;
   }
 
-  MemProfRecordData.reserve(IPW.MemProfRecordData.size());
-  for (auto &[GUID, Record] : IPW.MemProfRecordData) {
+  MemProfData.RecordData.reserve(IPW.MemProfData.RecordData.size());
+  for (auto &[GUID, Record] : IPW.MemProfData.RecordData) {
     addMemProfRecord(GUID, Record);
   }
 }
@@ -499,11 +499,8 @@ static uint64_t writeMemProfCallStacks(
   return CallStackTableGenerator.Emit(OS.OS);
 }
 
-static Error writeMemProfV0(
-    ProfOStream &OS,
-    llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
-        &MemProfRecordData,
-    llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) {
+static Error writeMemProfV0(ProfOStream &OS,
+                            memprof::IndexedMemProfData &MemProfData) {
   uint64_t HeaderUpdatePos = OS.tell();
   OS.write(0ULL); // Reserve space for the memprof record table offset.
   OS.write(0ULL); // Reserve space for the memprof frame payload offset.
@@ -512,11 +509,11 @@ static Error writeMemProfV0(
   auto Schema = memprof::getFullSchema();
   writeMemProfSchema(OS, Schema);
 
-  uint64_t RecordTableOffset =
-      writeMemProfRecords(OS, MemProfRecordData, &Schema, memprof::Version0);
+  uint64_t RecordTableOffset = writeMemProfRecords(OS, MemProfData.RecordData,
+                                                   &Schema, memprof::Version0);
 
   uint64_t FramePayloadOffset = OS.tell();
-  uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfFrameData);
+  uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfData.FrameData);
 
   uint64_t Header[] = {RecordTableOffset, FramePayloadOffset, FrameTableOffset};
   OS.patch({{HeaderUpdatePos, Header, std::size(Header)}});
@@ -524,11 +521,8 @@ static Error writeMemProfV0(
   return Error::success();
 }
 
-static Error writeMemProfV1(
-    ProfOStream &OS,
-    llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
-        &MemProfRecordData,
-    llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) {
+static Error writeMemProfV1(ProfOStream &OS,
+                            memprof::IndexedMemProfData &MemProfData) {
   OS.write(memprof::Version1);
   uint64_t HeaderUpdatePos = OS.tell();
   OS.write(0ULL); // Reserve space for the memprof record table offset.
@@ -538,11 +532,11 @@ static Error writeMemProfV1(
   auto Schema = memprof::getFullSchema();
   writeMemProfSchema(OS, Schema);
 
-  uint64_t RecordTableOffset =
-      writeMemProfRecords(OS, MemProfRecordData, &Schema, memprof::Version1);
+  uint64_t RecordTableOffset = writeMemProfRecords(OS, MemProfData.RecordData,
+                                                   &Schema, memprof::Version1);
 
   uint64_t FramePayloadOffset = OS.tell();
-  uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfFrameData);
+  uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfData.FrameData);
 
   uint64_t Header[] = {RecordTableOffset, FramePayloadOffset, FrameTableOffset};
   OS.patch({{HeaderUpdatePos, Header, std::size(Header)}});
@@ -550,14 +544,9 @@ static Error writeMemProfV1(
   return Error::success();
 }
 
-static Error writeMemProfV2(
-    ProfOStream &OS,
-    llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
-        &MemProfRecordData,
-    llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData,
-    llvm::MapVector<memprof::CallStackId, llvm::SmallVector<memprof::FrameId>>
-        &MemProfCallStackData,
-    bool MemProfFullSchema) {
+static Error writeMemProfV2(ProfOStream &OS,
+                            memprof::IndexedMemProfData &MemProfData,
+                            bool MemProfFullSchema) {
   OS.write(memprof::Version2);
   uint64_t HeaderUpdatePos = OS.tell();
   OS.write(0ULL); // Reserve space for the memprof record table offset.
@@ -571,15 +560,15 @@ static Error writeMemProfV2(
     Schema = memprof::getFullSchema();
   writeMemProfSchema(OS, Schema);
 
-  uint64_t RecordTableOffset =
-      writeMemProfRecords(OS, MemProfRecordData, &Schema, memprof::Version2);
+  uint64_t RecordTableOffset = writeMemProfRecords(OS, MemProfData.RecordData,
+                                                   &Schema, memprof::Version2);
 
   uint64_t FramePayloadOffset = OS.tell();
-  uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfFrameData);
+  uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfData.FrameData);
 
   uint64_t CallStackPayloadOffset = OS.tell();
   uint64_t CallStackTableOffset =
-      writeMemProfCallStacks(OS, MemProfCallStackData);
+      writeMemProfCallStacks(OS, MemProfData.CallStackData);
 
   uint64_t Header[] = {
       RecordTableOffset,      FramePayloadOffset,   FrameTableOffset,
@@ -603,23 +592,17 @@ static Error writeMemProfV2(
 // uint64_t Schema entry N - 1
 // OnDiskChainedHashTable MemProfRecordData
 // OnDiskChainedHashTable MemProfFrameData
-static Error writeMemProf(
-    ProfOStream &OS,
-    llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
-        &MemProfRecordData,
-    llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData,
-    llvm::MapVector<memprof::CallStackId, llvm::SmallVector<memprof::FrameId>>
-        &MemProfCallStackData,
-    memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema) {
-
+static Error writeMemProf(ProfOStream &OS,
+                          memprof::IndexedMemProfData &MemProfData,
+                          memprof::IndexedVersion MemProfVersionRequested,
+                          bool MemProfFullSchema) {
   switch (MemProfVersionRequested) {
   case memprof::Version0:
-    return writeMemProfV0(OS, MemProfRecordData, MemProfFrameData);
+    return writeMemProfV0(OS, MemProfData);
   case memprof::Version1:
-    return writeMemProfV1(OS, MemProfRecordData, MemProfFrameData);
+    return writeMemProfV1(OS, MemProfData);
   case memprof::Version2:
-    return writeMemProfV2(OS, MemProfRecordData, MemProfFrameData,
-                          MemProfCallStackData, MemProfFullSchema);
+    return writeMemProfV2(OS, MemProfData, MemProfFullSchema);
   }
 
   return make_error<InstrProfError>(
@@ -737,8 +720,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   uint64_t MemProfSectionStart = 0;
   if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
     MemProfSectionStart = OS.tell();
-    if (auto E = writeMemProf(OS, MemProfRecordData, MemProfFrameData,
-                              MemProfCallStackData, MemProfVersionRequested,
+    if (auto E = writeMemProf(OS, MemProfData, MemProfVersionRequested,
                               MemProfFullSchema))
       return E;
   }
-- 
GitLab


From 525bd66fc29fd056ef18118dfff3c9cc05fdd5e3 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 16 May 2024 10:41:06 -0700
Subject: [PATCH 1040/1206] [lldb-dap] Separate user and developer
 documentation (#92428)

The README.md is what users see when they look for the extension in the
Marketplace [1]. Right now, it's a mix of developer documentation (for
us) and user documentation. This commit moves the developer docs into
`docs` and the lldb website and refocuses the README on using the
extension.

[1] https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.lldb-dap
---
 lldb/docs/index.rst            |   1 +
 lldb/docs/resources/lldbdap.md |  97 ++++++++++++++++++++
 lldb/tools/lldb-dap/README.md  | 156 +++++----------------------------
 3 files changed, 119 insertions(+), 135 deletions(-)
 create mode 100644 lldb/docs/resources/lldbdap.md

diff --git a/lldb/docs/index.rst b/lldb/docs/index.rst
index 7a27f6914fa8..1e7d69002dd3 100644
--- a/lldb/docs/index.rst
+++ b/lldb/docs/index.rst
@@ -161,6 +161,7 @@ interesting areas to contribute to lldb.
    resources/lldbplatformpackets
    resources/caveats
    resources/projects
+   resources/lldbdap
    Public C++ API <https://lldb.llvm.org/cpp_reference/namespacelldb.html>
    Private C++ API <https://lldb.llvm.org/cpp_reference/index.html>
 
diff --git a/lldb/docs/resources/lldbdap.md b/lldb/docs/resources/lldbdap.md
new file mode 100644
index 000000000000..2d345e26b7e5
--- /dev/null
+++ b/lldb/docs/resources/lldbdap.md
@@ -0,0 +1,97 @@
+# LLDB-DAP
+
+The `lldb-dap` tool (formerly `lldb-vscode`) is a command line tool that
+implements the [Debug Adapter
+Protocol](https://microsoft.github.io/debug-adapter-protocol/). It can be
+installed as an extension for Visual Studio Code and other IDEs supporting DAP.
+The protocol is easy to run remotely and also can allow other tools and IDEs to
+get a full featured debugger with a well defined protocol.
+
+## Local Installation for Visual Studio Code
+
+Installing the plug-in is very straightforward and involves just a few steps.
+
+### Pre-requisites
+
+- Install a modern version of node (e.g. `v20.0.0`).
+- On VS Code, execute the command `Install 'code' command in PATH`. You need to
+  do it only once. This enables the command `code` in the PATH.
+
+### Packaging and installation
+
+```bash
+cd /path/to/lldb/tools/lldb-dap
+npm install
+npm run package # This also compiles the extension.
+npm run vscode-install
+```
+
+On VS Code, set the setting `lldb-dap.executable-path` to the path of your local
+build of `lldb-dap`.
+
+And then you are ready!
+
+### Updating the extension
+
+*Note: It's not necessary to update the extension if there has been changes
+to  `lldb-dap`. The extension needs to be updated only if the TypesScript code
+has changed.*
+
+Updating the extension is pretty much the same process as installing it from
+scratch. However, VS Code expects the version number of the upgraded extension
+to be greater than the previous one, otherwise the installation step might have
+no effect.
+
+```bash
+# Bump version in package.json
+cd /path/to/lldb/tools/lldb-dap
+npm install
+npm run package
+npm run vscode-install
+```
+
+Another way upgrade without bumping the extension version is to first uninstall
+the extension, then reload VS Code, and then install it again. This is
+an unfortunate limitation of the editor.
+
+```bash
+cd /path/to/lldb/tools/lldb-dap
+npm run vscode-uninstall
+# Then reload VS Code: reopen the IDE or execute the `Developer: Reload Window`
+# command.
+npm run package
+npm run vscode-install
+```
+
+### Deploying for Visual Studio Code
+
+The easiest way to deploy the extension for execution on other machines requires
+copying `lldb-dap` and its dependencies into a`./bin` subfolder and then create a
+standalone VSIX package.
+
+```bash
+cd /path/to/lldb/tools/lldb-dap
+mkdir -p ./bin
+cp /path/to/a/built/lldb-dap ./bin/
+cp /path/to/a/built/liblldb.so ./bin/
+npm run package
+```
+
+This will produce the file `./out/lldb-dap.vsix` that can be distributed. In
+this type of installation, users don't need to manually set the path to
+`lldb-dap`. The extension will automatically look for it in the `./bin`
+subfolder.
+
+*Note: It's not possible to use symlinks to `lldb-dap`, as the packaging tool
+forcefully performs a deep copy of all symlinks.*
+
+*Note: It's possible to use this kind flow for local installations, but it's
+not recommended because updating `lldb-dap` requires rebuilding the extension.*
+
+## Formatting the Typescript code
+
+This is also very simple, just run:
+
+```bash
+npm run format
+```
diff --git a/lldb/tools/lldb-dap/README.md b/lldb/tools/lldb-dap/README.md
index 16ce4672be71..8ecbaf7ce981 100644
--- a/lldb/tools/lldb-dap/README.md
+++ b/lldb/tools/lldb-dap/README.md
@@ -1,133 +1,19 @@
+# LLDB DAP
 
-# Table of Contents
-
-- [Table of Contents](#table-of-contents)
-- [Introduction](#introduction)
-- [Local Installation for Visual Studio Code](#local-installation-for-visual-studio-code)
-  - [Pre-requisites](#pre-requisites)
-  - [Packaging and installation](#packaging-and-installation)
-  - [Updating the extension](#updating-the-extension)
-  - [Deploying for Visual Studio Code](#deploying-for-visual-studio-code)
-- [Formatting the Typescript code](#formatting-the-typescript-code)
-- [Configurations](#configurations)
-  - [Launch Configuration Settings](#launch-configuration-settings)
-  - [Attaching Settings](#attaching-settings)
-  - [Example configurations](#example-configurations)
-    - [Launching](#launching)
-    - [Attach using PID](#attach-using-pid)
-    - [Attach by Name](#attach-by-name)
-    - [Loading a Core File](#loading-a-core-file)
-    - [Connect to a Debug Server on the Current Machine](#connect-to-a-debug-server-on-the-current-machine)
-    - [Connect to a Debug Server on Another Machine](#connect-to-a-debug-server-on-another-machine)
-- [Custom debugger commands](#custom-debugger-commands)
-  - [startDebugging](#startdebugging)
-  - [repl-mode](#repl-mode)
-
-# Introduction
-
-The `lldb-dap` tool (formerly `lldb-vscode`) creates a command line tool that
-implements the [Debug Adapter
-Protocol](https://microsoft.github.io/debug-adapter-protocol/). It can be
-installed as an extension for Visual Studio Code and other IDEs supporting DAP.
+## `lldb-dap` Configurations
+
+The extension requires the `lldb-dap` (formerly `lldb-vscode`) binary. It is a
+command line tool that implements the [Debug Adapter
+Protocol](https://microsoft.github.io/debug-adapter-protocol/). It is used to power the Visual Studio Code extension but can also be used with other IDEs and editors that support DAP.
 The protocol is easy to run remotely and also can allow other tools and IDEs to
 get a full featured debugger with a well defined protocol.
 
-# Local Installation for Visual Studio Code
-
-Installing the plug-in is very straightforward and involves just a few steps.
-
-## Pre-requisites
-
-- Install a modern version of node (e.g. `v20.0.0`).
-- On VS Code, execute the command `Install 'code' command in PATH`. You need to
-  do it only once. This enables the command `code` in the PATH.
-
-## Packaging and installation
-
-```bash
-cd /path/to/lldb/tools/lldb-dap
-npm install
-npm run package # This also compiles the extension.
-npm run vscode-install
-```
-
-On VS Code, set the setting `lldb-dap.executable-path` to the path of your local
-build of `lldb-dap`.
-
-And then you are ready!
-
-## Updating the extension
-
-*Note: It's not necessary to update the extension if there has been changes
-to  `lldb-dap`. The extension needs to be updated only if the TypesScript code
-has changed.*
-
-Updating the extension is pretty much the same process as installing it from
-scratch. However, VS Code expects the version number of the upgraded extension
-to be greater than the previous one, otherwise the installation step might have
-no effect.
-
-```bash
-# Bump version in package.json
-cd /path/to/lldb/tools/lldb-dap
-npm install
-npm run package
-npm run vscode-install
-```
-
-Another way upgrade without bumping the extension version is to first uninstall
-the extension, then reload VS Code, and then install it again. This is
-an unfortunate limitation of the editor.
-
-```bash
-cd /path/to/lldb/tools/lldb-dap
-npm run vscode-uninstall
-# Then reload VS Code: reopen the IDE or execute the `Developer: Reload Window`
-# command.
-npm run package
-npm run vscode-install
-```
-
-## Deploying for Visual Studio Code
-
-The easiest way to deploy the extension for execution on other machines requires
-copying `lldb-dap` and its dependencies into a`./bin` subfolder and then create a
-standalone VSIX package.
-
-```bash
-cd /path/to/lldb/tools/lldb-dap
-mkdir -p ./bin
-cp /path/to/a/built/lldb-dap ./bin/
-cp /path/to/a/built/liblldb.so ./bin/
-npm run package
-```
-
-This will produce the file `./out/lldb-dap.vsix` that can be distributed. In
-this type of installation, users don't need to manually set the path to
-`lldb-dap`. The extension will automatically look for it in the `./bin`
-subfolder.
-
-*Note: It's not possible to use symlinks to `lldb-dap`, as the packaging tool
-forcefully performs a deep copy of all symlinks.*
-
-*Note: It's possible to use this kind flow for local installations, but it's
-not recommended because updating `lldb-dap` requires rebuilding the extension.*
-
-# Formatting the Typescript code
-
-This is also very simple, just run:
-
-```bash
-npm run format
-```
-
-# Configurations
+## Launching & Attaching Configuration
 
-Launching to attaching require you to create a [launch configuration](https://code.visualstudio.com/Docs/editor/debugging#_launch-configurations). This file
-defines arguments that get passed to `lldb-dap` and the configuration settings
-control how the launch or attach happens.
+Launching to attaching require you to create a [launch configuration](https://code.visualstudio.com/Docs/editor/debugging#_launch-configurations).
+This file defines arguments that get passed to `lldb-dap` and the configuration settings control how the launch or attach happens.
 
-## Launch Configuration Settings
+### Launch Configuration Settings
 
 When you launch a program with Visual Studio Code you will need to create a [launch.json](https://code.visualstudio.com/Docs/editor/debugging#_launch-configurations)
 file that defines how your program will be run. The JSON configuration file can contain the following `lldb-dap` specific launch key/value pairs:
@@ -151,7 +37,7 @@ file that defines how your program will be run. The JSON configuration file can
 |**sourceMap**      |[string[2]]| | Specify an array of path re-mappings. Each element in the array must be a two element array containing a source and destination pathname.
 |**debuggerRoot**   | string| |Specify a working directory to use when launching lldb-dap. If the debug information in your executable contains relative paths, this option can be used so that `lldb-dap` can find source files and object files that have relative paths.
 
-## Attaching Settings
+### Attaching Settings
 
 When attaching to a process using LLDB you can attach in a few ways
 
@@ -177,9 +63,9 @@ The JSON configuration file can contain the following `lldb-dap` specific launch
 |**terminateCommands** |[string]| | LLDB commands executed when the debugging session ends. Commands and command output will be sent to the debugger console when they are executed.
 |**attachCommands** |[string]| | LLDB commands that will be executed after **preRunCommands** which take place of the code that normally does the attach. The commands can create a new target and attach or launch it however desired. This allows custom launch and attach configurations. Core files can use `target create --core /path/to/core` to attach to core files.
 
-## Example configurations
+### Example configurations
 
-### Launching
+#### Launching
 
 This will launch `/tmp/a.out` with arguments `one`, `two`, and `three` and
 adds `FOO=1` and `bar` to the environment:
@@ -195,7 +81,7 @@ adds `FOO=1` and `bar` to the environment:
 }
 ```
 
-### Attach using PID
+#### Attach using PID
 
 This will attach to a process `a.out` whose process ID is 123:
 
@@ -209,7 +95,7 @@ This will attach to a process `a.out` whose process ID is 123:
 }
 ```
 
-### Attach by Name
+#### Attach by Name
 
 This will attach to an existing process whose base
 name matches `a.out`. All we have to do is leave the `pid` value out of the
@@ -240,7 +126,7 @@ to be launched you can add the "waitFor" key value pair:
 This will work as long as the architecture, vendor and OS supports waiting
 for processes. Currently MacOS is the only platform that supports this.
 
-### Loading a Core File
+#### Loading a Core File
 
 This loads the coredump file `/cores/123.core` associated with the program
 `/tmp/a.out`:
@@ -255,7 +141,7 @@ This loads the coredump file `/cores/123.core` associated with the program
 }
 ```
 
-### Connect to a Debug Server on the Current Machine
+#### Connect to a Debug Server on the Current Machine
 
 This connects to a debug server (e.g. `lldb-server`, `gdbserver`) on
 the current machine, that is debugging the program `/tmp/a.out` and listening
@@ -271,7 +157,7 @@ locally on port `2345`.
 }
 ```
 
-### Connect to a Debug Server on Another Machine
+#### Connect to a Debug Server on Another Machine
 
 This connects to a debug server running on another machine with hostname
 `hostnmame`. Which is debugging the program `/tmp/a.out` and listening on
@@ -287,12 +173,12 @@ port `5678` of that other machine.
 }
 ```
 
-# Custom debugger commands
+## Custom debugger commands
 
 The `lldb-dap` tool includes additional custom commands to support the Debug
 Adapter Protocol features.
 
-## startDebugging
+### startDebugging
 
 Using the command `lldb-dap startDebugging` it is possible to trigger a
 reverse request to the client requesting a child debug session with the
@@ -317,7 +203,7 @@ This will launch a server and then request a child debug session for a client.
 }
 ```
 
-## repl-mode
+### repl-mode
 
 Inspect or adjust the behavior of lldb-dap repl evaluation requests. The
 supported modes are `variable`, `command` and `auto`.
-- 
GitLab


From 80a0a067fb8cc3122f53083fd2fe20f9d5dfc5bf Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Sat, 30 Mar 2024 21:38:23 -0500
Subject: [PATCH 1041/1206] [ValueTracking] Add tests for computing knownbits
 from `(icmp upred (add/sub nuw X, Y), C)`; NFC

---
 .../test/Transforms/InstCombine/known-bits.ll | 276 ++++++++++++++++++
 1 file changed, 276 insertions(+)

diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index cb404c0b7f88..c00bdb787095 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1698,5 +1698,281 @@ define i32 @test_none(float nofpclass(all) %x) {
   ret i32 %and
 }
 
+define i8 @test_icmp_add(i8 %n, i8 %n2, i8 %other) {
+; CHECK-LABEL: @test_icmp_add(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_ADD:%.*]] = add nuw i8 [[N:%.*]], [[N2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[N_ADD]], 32
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
+;
+entry:
+  %n_add = add nuw i8 %n, %n2
+  %cmp = icmp ult i8 %n_add, 32
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %r = and i8 %n, 32
+  ret i8 %r
+
+if.else:
+  ret i8 %other
+}
+
+define i8 @test_icmp_add_fail_nsw(i8 %n, i8 %n2, i8 %other) {
+; CHECK-LABEL: @test_icmp_add_fail_nsw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_ADD:%.*]] = add nsw i8 [[N:%.*]], [[N2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[N_ADD]], 32
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
+;
+entry:
+  %n_add = add nsw i8 %n, %n2
+  %cmp = icmp ult i8 %n_add, 32
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %r = and i8 %n, 32
+  ret i8 %r
+
+if.else:
+  ret i8 %other
+}
+
+define i8 @test_icmp_add2(i8 %n, i8 %n2, i8 %other) {
+; CHECK-LABEL: @test_icmp_add2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_ADD:%.*]] = add nuw nsw i8 [[N:%.*]], [[N2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[N_ADD]], 14
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+;
+entry:
+  %n_add = add nsw nuw i8 %n, %n2
+  %cmp = icmp uge i8 %n_add, 15
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  ret i8 %other
+if.else:
+  %r = and i8 %n, 32
+  ret i8 %r
+
+}
+
+define i8 @test_icmp_add_fail_bad_range(i8 %n, i8 %n2, i8 %other) {
+; CHECK-LABEL: @test_icmp_add_fail_bad_range(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_ADD:%.*]] = add nuw i8 [[N:%.*]], [[N2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[N_ADD]], 33
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
+;
+entry:
+  %n_add = add nuw i8 %n, %n2
+  %cmp = icmp ule i8 %n_add, 32
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %r = and i8 %n, 32
+  ret i8 %r
+
+if.else:
+  ret i8 %other
+}
+
+define i8 @test_icmp_add_fail_bad_pred(i8 %n, i8 %n2, i8 %other) {
+; CHECK-LABEL: @test_icmp_add_fail_bad_pred(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_ADD:%.*]] = add nuw i8 [[N:%.*]], [[N2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[N_ADD]], 32
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
+;
+entry:
+  %n_add = add nuw i8 %n, %n2
+  %cmp = icmp ugt i8 %n_add, 32
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %r = and i8 %n, 32
+  ret i8 %r
+
+if.else:
+  ret i8 %other
+}
+
+define i8 @test_icmp_sub(i8 %n, i8 %n2, i8 %other) {
+; CHECK-LABEL: @test_icmp_sub(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_SUB:%.*]] = sub nuw i8 [[N:%.*]], [[N2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[N_SUB]], -33
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
+;
+entry:
+  %n_sub = sub nuw i8 %n, %n2
+  %cmp = icmp ugt i8 %n_sub, 223
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %r = and i8 %n, 32
+  ret i8 %r
+
+if.else:
+  ret i8 %other
+}
+
+define i8 @test_icmp_sub_fail_wrong_arg(i8 %n, i8 %n2, i8 %other) {
+; CHECK-LABEL: @test_icmp_sub_fail_wrong_arg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_SUB:%.*]] = sub nuw i8 [[N:%.*]], [[N2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[N_SUB]], -33
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[N2]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
+;
+entry:
+  %n_sub = sub nuw i8 %n, %n2
+  %cmp = icmp ugt i8 %n_sub, 223
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %r = and i8 %n2, 32
+  ret i8 %r
+
+if.else:
+  ret i8 %other
+}
+
+define i8 @test_icmp_sub2(i8 %n, i8 %n2, i8 %other) {
+; CHECK-LABEL: @test_icmp_sub2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_SUB:%.*]] = sub nuw i8 [[N:%.*]], [[N2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[N_SUB]], -31
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+;
+entry:
+  %n_sub = sub nuw i8 %n, %n2
+  %cmp = icmp ule i8 %n_sub, 224
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  ret i8 %other
+if.else:
+  %r = and i8 %n, 32
+  ret i8 %r
+
+}
+
+define i8 @test_icmp_sub2_fail_nsw(i8 %n, i8 %n2, i8 %other) {
+; CHECK-LABEL: @test_icmp_sub2_fail_nsw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_SUB:%.*]] = sub nsw i8 [[N:%.*]], [[N2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[N_SUB]], -31
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+;
+entry:
+  %n_sub = sub nsw i8 %n, %n2
+  %cmp = icmp ule i8 %n_sub, 224
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  ret i8 %other
+if.else:
+  %r = and i8 %n, 32
+  ret i8 %r
+
+}
+
+
+define i8 @test_icmp_sub_fail_bad_range(i8 %n, i8 %n2, i8 %other) {
+; CHECK-LABEL: @test_icmp_sub_fail_bad_range(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_SUB:%.*]] = sub nuw i8 [[N:%.*]], [[N2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[N_SUB]], -34
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
+;
+entry:
+  %n_sub = sub nuw i8 %n, %n2
+  %cmp = icmp uge i8 %n_sub, 223
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %r = and i8 %n, 32
+  ret i8 %r
+
+if.else:
+  ret i8 %other
+}
+
+define i8 @test_icmp_sub_fail_bad_pred(i8 %n, i8 %n2, i8 %other) {
+; CHECK-LABEL: @test_icmp_sub_fail_bad_pred(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N_SUB:%.*]] = sub nuw i8 [[N:%.*]], [[N2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[N_SUB]], 31
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+; CHECK:       if.else:
+; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
+;
+entry:
+  %n_sub = sub nuw i8 %n, %n2
+  %cmp = icmp sge i8 %n_sub, 32
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %r = and i8 %n, 32
+  ret i8 %r
+
+if.else:
+  ret i8 %other
+}
+
 declare void @use(i1)
 declare void @sink(i8)
-- 
GitLab


From 05347f8c2fde0cde2fd12f1ec3611eeaf666ece0 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Sat, 30 Mar 2024 21:39:36 -0500
Subject: [PATCH 1042/1206] [ValueTracking] Compute knownbits from `(icmp upred
 (add/sub nuw X, Y), C)`

`(icmp ule/ult (add nuw X, Y), C)` implies both `(icmp ule/ult X, C)` and
`(icmp ule/ult Y, C)`. We can use this to deduce leading zeros in `X`/`Y`.

`(icmp uge/ugt (sub nuw X, Y), C)` implies `(icmp uge/uge X, C)` . We
can use this to deduce leading ones in `X`.

Proofs: https://alive2.llvm.org/ce/z/sc5k22

Closes #87180
---
 llvm/lib/Analysis/ValueTracking.cpp           | 47 ++++++++++++-------
 .../test/Transforms/InstCombine/known-bits.ll | 12 ++---
 2 files changed, 33 insertions(+), 26 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index e07d14ccbb85..e8c5f9b3dc25 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -706,17 +706,22 @@ static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
           LHSRange = LHSRange.sub(*Offset);
         Known = Known.unionWith(LHSRange.toKnownBits());
       }
-      // X & Y u> C -> X u> C && Y u> C
-      if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) &&
-          match(LHS, m_c_And(m_V, m_Value()))) {
-        Known.One.setHighBits(
-            (*C + (Pred == ICmpInst::ICMP_UGT)).countLeadingOnes());
+      if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
+        // X & Y u> C     -> X u> C && Y u> C
+        // X nuw- Y u> C  -> X u> C
+        if (match(LHS, m_c_And(m_V, m_Value())) ||
+            match(LHS, m_NUWSub(m_V, m_Value())))
+          Known.One.setHighBits(
+              (*C + (Pred == ICmpInst::ICMP_UGT)).countLeadingOnes());
       }
-      // X | Y u< C -> X u< C && Y u< C
-      if ((Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) &&
-          match(LHS, m_c_Or(m_V, m_Value()))) {
-        Known.Zero.setHighBits(
-            (*C - (Pred == ICmpInst::ICMP_ULT)).countLeadingZeros());
+      if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
+        // X | Y u< C    -> X u< C && Y u< C
+        // X nuw+ Y u< C -> X u< C && Y u< C
+        if (match(LHS, m_c_Or(m_V, m_Value())) ||
+            match(LHS, m_c_NUWAdd(m_V, m_Value()))) {
+          Known.Zero.setHighBits(
+              (*C - (Pred == ICmpInst::ICMP_ULT)).countLeadingZeros());
+        }
       }
     }
     break;
@@ -9576,14 +9581,20 @@ void llvm::findValuesAffectedByCondition(
           if (match(A, m_AddLike(m_Value(X), m_ConstantInt())))
             AddAffected(X);
 
-          Value *Y;
-          // X & Y u> C -> X >u C && Y >u C
-          // X | Y u< C -> X u< C && Y u< C
-          if (ICmpInst::isUnsigned(Pred) &&
-              (match(A, m_And(m_Value(X), m_Value(Y))) ||
-               match(A, m_Or(m_Value(X), m_Value(Y))))) {
-            AddAffected(X);
-            AddAffected(Y);
+          if (ICmpInst::isUnsigned(Pred)) {
+            Value *Y;
+            // X & Y u> C    -> X >u C && Y >u C
+            // X | Y u< C    -> X u< C && Y u< C
+            // X nuw+ Y u< C -> X u< C && Y u< C
+            if (match(A, m_And(m_Value(X), m_Value(Y))) ||
+                match(A, m_Or(m_Value(X), m_Value(Y))) ||
+                match(A, m_NUWAdd(m_Value(X), m_Value(Y)))) {
+              AddAffected(X);
+              AddAffected(Y);
+            }
+            // X nuw- Y u> C -> X u> C
+            if (match(A, m_NUWSub(m_Value(X), m_Value())))
+              AddAffected(X);
           }
         }
 
diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index c00bdb787095..82cd24027e4e 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1705,8 +1705,7 @@ define i8 @test_icmp_add(i8 %n, i8 %n2, i8 %other) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[N_ADD]], 32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 0
 ; CHECK:       if.else:
 ; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
 ;
@@ -1757,8 +1756,7 @@ define i8 @test_icmp_add2(i8 %n, i8 %n2, i8 %other) {
 ; CHECK:       if.then:
 ; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
 ; CHECK:       if.else:
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 0
 ;
 entry:
   %n_add = add nsw nuw i8 %n, %n2
@@ -1830,8 +1828,7 @@ define i8 @test_icmp_sub(i8 %n, i8 %n2, i8 %other) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[N_SUB]], -33
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 32
 ; CHECK:       if.else:
 ; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
 ;
@@ -1882,8 +1879,7 @@ define i8 @test_icmp_sub2(i8 %n, i8 %n2, i8 %other) {
 ; CHECK:       if.then:
 ; CHECK-NEXT:    ret i8 [[OTHER:%.*]]
 ; CHECK:       if.else:
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[N]], 32
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 32
 ;
 entry:
   %n_sub = sub nuw i8 %n, %n2
-- 
GitLab


From ebbf484e1c62a312d88c7f6482af11a58d40f52d Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Sun, 7 Apr 2024 14:01:15 -0500
Subject: [PATCH 1043/1206] [InstCombine] Add tests for folding `(icmp pred
 (trunc nuw/nsw X), C)`; NFC

---
 .../test/Transforms/InstCombine/icmp-trunc.ll | 192 ++++++++++++++++++
 1 file changed, 192 insertions(+)

diff --git a/llvm/test/Transforms/InstCombine/icmp-trunc.ll b/llvm/test/Transforms/InstCombine/icmp-trunc.ll
index b2de9dddb219..c49c6a36823b 100644
--- a/llvm/test/Transforms/InstCombine/icmp-trunc.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-trunc.ll
@@ -555,3 +555,195 @@ define i1 @shl1_trunc_sgt4(i32 %a) {
   %r = icmp sgt i16 %t, 4
   ret i1 %r
 }
+
+define i1 @eq_nuw(i32 %x) {
+; DL64-LABEL: @eq_nuw(
+; DL64-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 255
+; DL64-NEXT:    [[R:%.*]] = icmp eq i32 [[TMP1]], 123
+; DL64-NEXT:    ret i1 [[R]]
+;
+; DL8-LABEL: @eq_nuw(
+; DL8-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; DL8-NEXT:    [[R:%.*]] = icmp eq i8 [[T]], 123
+; DL8-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nuw i32 %x to i8
+  %r = icmp eq i8 %t, 123
+  ret i1 %r
+}
+
+define i1 @ult_nuw(i32 %x) {
+; CHECK-LABEL: @ult_nuw(
+; CHECK-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T]], 45
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nuw i32 %x to i8
+  %r = icmp ult i8 %t, 45
+  ret i1 %r
+}
+
+define i1 @ule_nuw(i32 %x) {
+; CHECK-LABEL: @ule_nuw(
+; CHECK-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T]], 46
+; CHECK-NEXT:    call void @use(i8 [[T]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nuw i32 %x to i8
+  %r = icmp ule i8 %t, 45
+  call void @use(i8 %t)
+  ret i1 %r
+}
+
+define i1 @ugt_nuw(i32 %x) {
+; CHECK-LABEL: @ugt_nuw(
+; CHECK-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T]], 12
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nuw i32 %x to i8
+  %r = icmp ugt i8 %t, 12
+  ret i1 %r
+}
+
+define i1 @uge_nuw(i32 %x) {
+; CHECK-LABEL: @uge_nuw(
+; CHECK-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T]], 98
+; CHECK-NEXT:    call void @use(i8 [[T]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nuw i32 %x to i8
+  %r = icmp uge i8 %t, 99
+  call void @use(i8 %t)
+  ret i1 %r
+}
+
+define i1 @uge_nuw_i48(i48 %x) {
+; CHECK-LABEL: @uge_nuw_i48(
+; CHECK-NEXT:    [[T:%.*]] = trunc nuw i48 [[X:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T]], 98
+; CHECK-NEXT:    call void @use(i8 [[T]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nuw i48 %x to i8
+  %r = icmp uge i8 %t, 99
+  call void @use(i8 %t)
+  ret i1 %r
+}
+
+define i1 @sgt_nuw_fail(i32 %x) {
+; CHECK-LABEL: @sgt_nuw_fail(
+; CHECK-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T]], 12
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nuw i32 %x to i8
+  %r = icmp sgt i8 %t, 12
+  ret i1 %r
+}
+
+define i1 @ne_nsw(i32 %x) {
+; DL64-LABEL: @ne_nsw(
+; DL64-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 255
+; DL64-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP1]], 133
+; DL64-NEXT:    ret i1 [[R]]
+;
+; DL8-LABEL: @ne_nsw(
+; DL8-NEXT:    [[T:%.*]] = trunc nsw i32 [[X:%.*]] to i8
+; DL8-NEXT:    [[R:%.*]] = icmp ne i8 [[T]], -123
+; DL8-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nsw i32 %x to i8
+  %r = icmp ne i8 %t, -123
+  ret i1 %r
+}
+
+define i1 @slt_nsw(i32 %x) {
+; CHECK-LABEL: @slt_nsw(
+; CHECK-NEXT:    [[T:%.*]] = trunc nsw i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T]], 45
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nsw i32 %x to i8
+  %r = icmp slt i8 %t, 45
+  ret i1 %r
+}
+
+define i1 @sle_nsw(i16 %x) {
+; CHECK-LABEL: @sle_nsw(
+; CHECK-NEXT:    [[T:%.*]] = trunc nsw i16 [[X:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T]], 46
+; CHECK-NEXT:    call void @use(i8 [[T]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nsw i16 %x to i8
+  %r = icmp sle i8 %t, 45
+  call void @use(i8 %t)
+  ret i1 %r
+}
+
+define i1 @sgt_nsw(i32 %x) {
+; CHECK-LABEL: @sgt_nsw(
+; CHECK-NEXT:    [[T:%.*]] = trunc nsw i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T]], -12
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nsw i32 %x to i8
+  %r = icmp sgt i8 %t, -12
+  ret i1 %r
+}
+
+define i1 @sge_nsw(i64 %x) {
+; CHECK-LABEL: @sge_nsw(
+; CHECK-NEXT:    [[T:%.*]] = trunc nsw i64 [[X:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T]], 98
+; CHECK-NEXT:    call void @use(i8 [[T]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nsw i64 %x to i8
+  %r = icmp sge i8 %t, 99
+  call void @use(i8 %t)
+  ret i1 %r
+}
+
+
+
+define i1 @sge_nsw_i48(i48 %x) {
+; CHECK-LABEL: @sge_nsw_i48(
+; CHECK-NEXT:    [[T:%.*]] = trunc nsw i48 [[X:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T]], 98
+; CHECK-NEXT:    call void @use(i8 [[T]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t = trunc nsw i48 %x to i8
+  %r = icmp sge i8 %t, 99
+  call void @use(i8 %t)
+  ret i1 %r
+}
+
+
+define <2 x i1> @uge_nsw(<2 x i32> %x) {
+; CHECK-LABEL: @uge_nsw(
+; CHECK-NEXT:    [[T:%.*]] = trunc nsw <2 x i32> [[X:%.*]] to <2 x i8>
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt <2 x i8> [[T]], <i8 -46, i8 -46>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t = trunc nsw <2 x i32> %x to <2 x i8>
+  %r = icmp uge <2 x i8> %t, <i8 -45, i8 -45>
+  ret <2 x i1> %r
+}
+
+
+define <2 x i1> @uge_nsw_non_splat(<2 x i32> %x) {
+; CHECK-LABEL: @uge_nsw_non_splat(
+; CHECK-NEXT:    [[T:%.*]] = trunc nsw <2 x i32> [[X:%.*]] to <2 x i8>
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt <2 x i8> [[T]], <i8 44, i8 45>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t = trunc nsw <2 x i32> %x to <2 x i8>
+  %r = icmp uge <2 x i8> %t, <i8 45, i8 46>
+  ret <2 x i1> %r
+}
+
-- 
GitLab


From 23f1047daac9702876ca99b53e8fe649fca569fa Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Sun, 7 Apr 2024 14:01:29 -0500
Subject: [PATCH 1044/1206] [InstCombine] Fold `(icmp pred (trunc nuw/nsw X),
 C)` -> `(icmp pred X, (zext/sext C))`

This is valid as long as the sign of the wrap flag doesn't differ from
the sign of the `pred`.

Proofs: https://alive2.llvm.org/ce/z/35NsrR

NB: The online Alive2 hasn't been updated with `trunc nuw/nsw`
support, so the proofs must be reproduced locally.

Closes #87935
---
 .../InstCombine/InstCombineCompares.cpp       |  17 ++-
 .../test/Transforms/InstCombine/icmp-trunc.ll | 118 ++++++++++++------
 2 files changed, 91 insertions(+), 44 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 9883d02c87a3..542a1c82b127 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -1409,6 +1409,19 @@ Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp,
                                                      const APInt &C) {
   ICmpInst::Predicate Pred = Cmp.getPredicate();
   Value *X = Trunc->getOperand(0);
+  Type *SrcTy = X->getType();
+  unsigned DstBits = Trunc->getType()->getScalarSizeInBits(),
+           SrcBits = SrcTy->getScalarSizeInBits();
+
+  // Match (icmp pred (trunc nuw/nsw X), C)
+  // Which we can convert to (icmp pred X, (sext/zext C))
+  if (shouldChangeType(Trunc->getType(), SrcTy)) {
+    if (Trunc->hasNoSignedWrap())
+      return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, C.sext(SrcBits)));
+    if (!Cmp.isSigned() && Trunc->hasNoUnsignedWrap())
+      return new ICmpInst(Pred, X, ConstantInt::get(SrcTy, C.zext(SrcBits)));
+  }
+
   if (C.isOne() && C.getBitWidth() > 1) {
     // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1
     Value *V = nullptr;
@@ -1417,10 +1430,6 @@ Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp,
                           ConstantInt::get(V->getType(), 1));
   }
 
-  Type *SrcTy = X->getType();
-  unsigned DstBits = Trunc->getType()->getScalarSizeInBits(),
-           SrcBits = SrcTy->getScalarSizeInBits();
-
   // TODO: Handle any shifted constant by subtracting trailing zeros.
   // TODO: Handle non-equality predicates.
   Value *Y;
diff --git a/llvm/test/Transforms/InstCombine/icmp-trunc.ll b/llvm/test/Transforms/InstCombine/icmp-trunc.ll
index c49c6a36823b..4de64d85bbf2 100644
--- a/llvm/test/Transforms/InstCombine/icmp-trunc.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-trunc.ll
@@ -558,8 +558,7 @@ define i1 @shl1_trunc_sgt4(i32 %a) {
 
 define i1 @eq_nuw(i32 %x) {
 ; DL64-LABEL: @eq_nuw(
-; DL64-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 255
-; DL64-NEXT:    [[R:%.*]] = icmp eq i32 [[TMP1]], 123
+; DL64-NEXT:    [[R:%.*]] = icmp eq i32 [[X:%.*]], 123
 ; DL64-NEXT:    ret i1 [[R]]
 ;
 ; DL8-LABEL: @eq_nuw(
@@ -573,10 +572,14 @@ define i1 @eq_nuw(i32 %x) {
 }
 
 define i1 @ult_nuw(i32 %x) {
-; CHECK-LABEL: @ult_nuw(
-; CHECK-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
-; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T]], 45
-; CHECK-NEXT:    ret i1 [[R]]
+; DL64-LABEL: @ult_nuw(
+; DL64-NEXT:    [[R:%.*]] = icmp ult i32 [[X:%.*]], 45
+; DL64-NEXT:    ret i1 [[R]]
+;
+; DL8-LABEL: @ult_nuw(
+; DL8-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; DL8-NEXT:    [[R:%.*]] = icmp ult i8 [[T]], 45
+; DL8-NEXT:    ret i1 [[R]]
 ;
   %t = trunc nuw i32 %x to i8
   %r = icmp ult i8 %t, 45
@@ -584,11 +587,17 @@ define i1 @ult_nuw(i32 %x) {
 }
 
 define i1 @ule_nuw(i32 %x) {
-; CHECK-LABEL: @ule_nuw(
-; CHECK-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
-; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T]], 46
-; CHECK-NEXT:    call void @use(i8 [[T]])
-; CHECK-NEXT:    ret i1 [[R]]
+; DL64-LABEL: @ule_nuw(
+; DL64-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; DL64-NEXT:    [[R:%.*]] = icmp ult i32 [[X]], 46
+; DL64-NEXT:    call void @use(i8 [[T]])
+; DL64-NEXT:    ret i1 [[R]]
+;
+; DL8-LABEL: @ule_nuw(
+; DL8-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; DL8-NEXT:    [[R:%.*]] = icmp ult i8 [[T]], 46
+; DL8-NEXT:    call void @use(i8 [[T]])
+; DL8-NEXT:    ret i1 [[R]]
 ;
   %t = trunc nuw i32 %x to i8
   %r = icmp ule i8 %t, 45
@@ -597,10 +606,14 @@ define i1 @ule_nuw(i32 %x) {
 }
 
 define i1 @ugt_nuw(i32 %x) {
-; CHECK-LABEL: @ugt_nuw(
-; CHECK-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T]], 12
-; CHECK-NEXT:    ret i1 [[R]]
+; DL64-LABEL: @ugt_nuw(
+; DL64-NEXT:    [[R:%.*]] = icmp ugt i32 [[X:%.*]], 12
+; DL64-NEXT:    ret i1 [[R]]
+;
+; DL8-LABEL: @ugt_nuw(
+; DL8-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; DL8-NEXT:    [[R:%.*]] = icmp ugt i8 [[T]], 12
+; DL8-NEXT:    ret i1 [[R]]
 ;
   %t = trunc nuw i32 %x to i8
   %r = icmp ugt i8 %t, 12
@@ -608,11 +621,17 @@ define i1 @ugt_nuw(i32 %x) {
 }
 
 define i1 @uge_nuw(i32 %x) {
-; CHECK-LABEL: @uge_nuw(
-; CHECK-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T]], 98
-; CHECK-NEXT:    call void @use(i8 [[T]])
-; CHECK-NEXT:    ret i1 [[R]]
+; DL64-LABEL: @uge_nuw(
+; DL64-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; DL64-NEXT:    [[R:%.*]] = icmp ugt i32 [[X]], 98
+; DL64-NEXT:    call void @use(i8 [[T]])
+; DL64-NEXT:    ret i1 [[R]]
+;
+; DL8-LABEL: @uge_nuw(
+; DL8-NEXT:    [[T:%.*]] = trunc nuw i32 [[X:%.*]] to i8
+; DL8-NEXT:    [[R:%.*]] = icmp ugt i8 [[T]], 98
+; DL8-NEXT:    call void @use(i8 [[T]])
+; DL8-NEXT:    ret i1 [[R]]
 ;
   %t = trunc nuw i32 %x to i8
   %r = icmp uge i8 %t, 99
@@ -646,8 +665,7 @@ define i1 @sgt_nuw_fail(i32 %x) {
 
 define i1 @ne_nsw(i32 %x) {
 ; DL64-LABEL: @ne_nsw(
-; DL64-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 255
-; DL64-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP1]], 133
+; DL64-NEXT:    [[R:%.*]] = icmp ne i32 [[X:%.*]], -123
 ; DL64-NEXT:    ret i1 [[R]]
 ;
 ; DL8-LABEL: @ne_nsw(
@@ -661,10 +679,14 @@ define i1 @ne_nsw(i32 %x) {
 }
 
 define i1 @slt_nsw(i32 %x) {
-; CHECK-LABEL: @slt_nsw(
-; CHECK-NEXT:    [[T:%.*]] = trunc nsw i32 [[X:%.*]] to i8
-; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T]], 45
-; CHECK-NEXT:    ret i1 [[R]]
+; DL64-LABEL: @slt_nsw(
+; DL64-NEXT:    [[R:%.*]] = icmp slt i32 [[X:%.*]], 45
+; DL64-NEXT:    ret i1 [[R]]
+;
+; DL8-LABEL: @slt_nsw(
+; DL8-NEXT:    [[T:%.*]] = trunc nsw i32 [[X:%.*]] to i8
+; DL8-NEXT:    [[R:%.*]] = icmp slt i8 [[T]], 45
+; DL8-NEXT:    ret i1 [[R]]
 ;
   %t = trunc nsw i32 %x to i8
   %r = icmp slt i8 %t, 45
@@ -672,11 +694,17 @@ define i1 @slt_nsw(i32 %x) {
 }
 
 define i1 @sle_nsw(i16 %x) {
-; CHECK-LABEL: @sle_nsw(
-; CHECK-NEXT:    [[T:%.*]] = trunc nsw i16 [[X:%.*]] to i8
-; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[T]], 46
-; CHECK-NEXT:    call void @use(i8 [[T]])
-; CHECK-NEXT:    ret i1 [[R]]
+; DL64-LABEL: @sle_nsw(
+; DL64-NEXT:    [[T:%.*]] = trunc nsw i16 [[X:%.*]] to i8
+; DL64-NEXT:    [[R:%.*]] = icmp slt i16 [[X]], 46
+; DL64-NEXT:    call void @use(i8 [[T]])
+; DL64-NEXT:    ret i1 [[R]]
+;
+; DL8-LABEL: @sle_nsw(
+; DL8-NEXT:    [[T:%.*]] = trunc nsw i16 [[X:%.*]] to i8
+; DL8-NEXT:    [[R:%.*]] = icmp slt i8 [[T]], 46
+; DL8-NEXT:    call void @use(i8 [[T]])
+; DL8-NEXT:    ret i1 [[R]]
 ;
   %t = trunc nsw i16 %x to i8
   %r = icmp sle i8 %t, 45
@@ -685,10 +713,14 @@ define i1 @sle_nsw(i16 %x) {
 }
 
 define i1 @sgt_nsw(i32 %x) {
-; CHECK-LABEL: @sgt_nsw(
-; CHECK-NEXT:    [[T:%.*]] = trunc nsw i32 [[X:%.*]] to i8
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T]], -12
-; CHECK-NEXT:    ret i1 [[R]]
+; DL64-LABEL: @sgt_nsw(
+; DL64-NEXT:    [[R:%.*]] = icmp sgt i32 [[X:%.*]], -12
+; DL64-NEXT:    ret i1 [[R]]
+;
+; DL8-LABEL: @sgt_nsw(
+; DL8-NEXT:    [[T:%.*]] = trunc nsw i32 [[X:%.*]] to i8
+; DL8-NEXT:    [[R:%.*]] = icmp sgt i8 [[T]], -12
+; DL8-NEXT:    ret i1 [[R]]
 ;
   %t = trunc nsw i32 %x to i8
   %r = icmp sgt i8 %t, -12
@@ -696,11 +728,17 @@ define i1 @sgt_nsw(i32 %x) {
 }
 
 define i1 @sge_nsw(i64 %x) {
-; CHECK-LABEL: @sge_nsw(
-; CHECK-NEXT:    [[T:%.*]] = trunc nsw i64 [[X:%.*]] to i8
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[T]], 98
-; CHECK-NEXT:    call void @use(i8 [[T]])
-; CHECK-NEXT:    ret i1 [[R]]
+; DL64-LABEL: @sge_nsw(
+; DL64-NEXT:    [[T:%.*]] = trunc nsw i64 [[X:%.*]] to i8
+; DL64-NEXT:    [[R:%.*]] = icmp sgt i64 [[X]], 98
+; DL64-NEXT:    call void @use(i8 [[T]])
+; DL64-NEXT:    ret i1 [[R]]
+;
+; DL8-LABEL: @sge_nsw(
+; DL8-NEXT:    [[T:%.*]] = trunc nsw i64 [[X:%.*]] to i8
+; DL8-NEXT:    [[R:%.*]] = icmp sgt i8 [[T]], 98
+; DL8-NEXT:    call void @use(i8 [[T]])
+; DL8-NEXT:    ret i1 [[R]]
 ;
   %t = trunc nsw i64 %x to i8
   %r = icmp sge i8 %t, 99
-- 
GitLab


From da928c6d6e0e2f619fdaef71cea8747d9aef188d Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Thu, 16 May 2024 14:10:46 -0400
Subject: [PATCH 1045/1206] [Clang][Sema] ASTContext::getUnconstrainedType
 propagates dependence (#92425)

When the argument passed to `ASTContext::getUnconstrainedType` is an
unconstrained `AutoType`, will return the argument unchanged. However,
when called with a constrained `AutoType`, an unconstrained,
non-dependent `AutoType` will be returned even if the argument was
dependent. Consider the following:
```
template<typename T>
concept C = sizeof(T) == sizeof(int);

template<auto N>
struct A;

template<C auto N>
struct A<N>; // error: class template partial specialization is not more specialized than the primary template
```
When comparing the template parameters for equivalence,
`ASTContext::getUnconstrainedType` is used to remove the constraints per
[temp.over.link] p6 sentence 2. For the template
parameter `N` of the class template, it returns a dependent `AutoType`.
For the template parameter `N` of the class template partial
specialization, it returns a non-dependent `AutoType`. We subsequently
compare the adjusted types and find they are not equivalent, thus we
consider the partial specialization to not be more specialized than the
primary template per [temp.func.order] p6.2.2.

This patch changes `ASTContext::getUnconstrainedType` such that the
dependence of a constrained `AutoType` will propagate to the returned
unconstrained `AutoType`. This causes the above example to be correctly
accepted, fixing #77377.
---
 clang/docs/ReleaseNotes.rst                         |  1 +
 clang/lib/AST/ASTContext.cpp                        |  3 ++-
 .../temp/temp.decls/temp.fct/temp.func.order/p2.cpp | 13 +++++++++++++
 3 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/CXX/temp/temp.decls/temp.fct/temp.func.order/p2.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index be4cded27632..2d2928e41862 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -723,6 +723,7 @@ Bug Fixes to C++ Support
 - Clang now ignores template parameters only used within the exception specification of candidate function
   templates during partial ordering when deducing template arguments from a function declaration or when
   taking the address of a function template.
+- Fix a bug with checking constrained non-type template parameters for equivalence. Fixes (#GH77377).
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 4475f399a120..8fc2bb8c401c 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -5910,7 +5910,8 @@ QualType ASTContext::getUnconstrainedType(QualType T) const {
   if (auto *AT = CanonT->getAs<AutoType>()) {
     if (!AT->isConstrained())
       return T;
-    return getQualifiedType(getAutoType(QualType(), AT->getKeyword(), false,
+    return getQualifiedType(getAutoType(QualType(), AT->getKeyword(),
+                                        AT->isDependentType(),
                                         AT->containsUnexpandedParameterPack()),
                             T.getQualifiers());
   }
diff --git a/clang/test/CXX/temp/temp.decls/temp.fct/temp.func.order/p2.cpp b/clang/test/CXX/temp/temp.decls/temp.fct/temp.func.order/p2.cpp
new file mode 100644
index 000000000000..df1bbd5fe812
--- /dev/null
+++ b/clang/test/CXX/temp/temp.decls/temp.fct/temp.func.order/p2.cpp
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++20 %s
+// expected-no-diagnostics
+
+namespace GH77377 {
+  template<typename T>
+  concept C = sizeof(T) == sizeof(int);
+
+  template<auto N>
+  struct A;
+
+  template<C auto N>
+  struct A<N>;
+}
-- 
GitLab


From 9a7f54ba4878c4e66c99430b52d8c014ab7b2a03 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 16 May 2024 11:53:15 -0700
Subject: [PATCH 1046/1206] [clang-fuzzer-dictionary] Skip EmptySpellingName
 after #89358

The dictionary entry `=""` is invalid.
---
 clang/tools/clang-fuzzer/dictionary/dictionary.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/clang/tools/clang-fuzzer/dictionary/dictionary.c b/clang/tools/clang-fuzzer/dictionary/dictionary.c
index 90490477f70c..de1defb236ac 100644
--- a/clang/tools/clang-fuzzer/dictionary/dictionary.c
+++ b/clang/tools/clang-fuzzer/dictionary/dictionary.c
@@ -21,6 +21,8 @@
 
 static void emit(const char *Name, const char *Spelling) {
   static char Hex[] = "0123456789abcdef";
+  // Skip EmptySpellingName for IsDeducible.
+  if (!Name[0]) return;
 
   printf("%s=\"", Name);
   unsigned char C;
-- 
GitLab


From a383b3cca3383128123b33bc73ffc0199ec573e4 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Thu, 16 May 2024 20:57:58 +0200
Subject: [PATCH 1047/1206] [clang] CTAD alias: Emit a more descriptive
 diagnostic message when is_deducible  constraint is evaluated to false.
 (#92389)

Fixes https://github.com/llvm/llvm-project/issues/92225
---
 clang/include/clang/Basic/DiagnosticSemaKinds.td | 2 ++
 clang/lib/Sema/SemaConcept.cpp                   | 7 +++++++
 clang/test/SemaCXX/cxx20-ctad-type-alias.cpp     | 7 +++----
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 3be9d34b36b0..7e487b70b03d 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -3003,6 +3003,8 @@ def note_single_arg_concept_specialization_constraint_evaluated_to_false : Note<
   "%select{and|because}0 %1 does not satisfy %2">;
 def note_atomic_constraint_evaluated_to_false_elaborated : Note<
   "%select{and|because}0 '%1' (%2 %3 %4) evaluated to false">;
+def note_is_deducible_constraint_evaluated_to_false : Note<
+  "cannot deduce template arguments for %0 from %1">;
 def err_constrained_virtual_method : Error<
   "virtual function cannot have a requires clause">;
 def err_trailing_requires_clause_on_deduction_guide : Error<
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 7bfec4e11f7a..202dd86c67f6 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -1162,6 +1162,13 @@ static void diagnoseWellFormedUnsatisfiedConstraintExpr(Sema &S,
         break;
       }
     return;
+  } else if (auto *TTE = dyn_cast<TypeTraitExpr>(SubstExpr);
+             TTE && TTE->getTrait() == clang::TypeTrait::BTT_IsDeducible) {
+    assert(TTE->getNumArgs() == 2);
+    S.Diag(SubstExpr->getSourceRange().getBegin(),
+           diag::note_is_deducible_constraint_evaluated_to_false)
+        << TTE->getArg(0)->getType() << TTE->getArg(1)->getType();
+    return;
   }
 
   S.Diag(SubstExpr->getSourceRange().getBegin(),
diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
index 7c186dc379c7..21b2b8bff500 100644
--- a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
+++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
@@ -111,7 +111,7 @@ struct Foo {
 template <typename X, int Y>
 using Bar = Foo<X, sizeof(X)>; // expected-note {{candidate template ignored: couldn't infer template argument 'X'}} \
                                // expected-note {{candidate template ignored: constraints not satisfied [with X = int]}} \
-                               // expected-note {{because '__is_deducible}}
+                               // expected-note {{cannot deduce template arguments for 'Bar' from 'Foo<int, 4UL>'}}
 
 
 Bar s = {{1}}; // expected-error {{no viable constructor or deduction guide }}
@@ -138,7 +138,7 @@ template<class T> struct Foo { T c; };
 template<class X, class Y=A>
 using AFoo = Foo<Y>; // expected-note {{candidate template ignored: could not match 'Foo<type-parameter-0-0>' against 'int'}} \
                     // expected-note {{candidate template ignored: constraints not satisfied [with Y = int]}} \
-                    // expected-note {{because '__is_deducible(AFoo, Foo<int>)' evaluated to false}} \
+                    // expected-note {{cannot deduce template arguments for 'AFoo' from 'Foo<int>'}} \
                     // expected-note {{candidate function template not viable: requires 0 arguments, but 1 was provided}}
 
 AFoo s = {1}; // expected-error {{no viable constructor or deduction guide for deduction of template arguments of 'AFoo'}}
@@ -196,10 +196,9 @@ template <class T> struct Foo { Foo(T); };
 
 template<class V> using AFoo = Foo<V *>;
 template<typename> concept False = false;
-// FIXME: emit a more descriptive diagnostic for "__is_deducible" constraint failure.
 template<False W>
 using BFoo = AFoo<W>; // expected-note {{candidate template ignored: constraints not satisfied [with V = int]}} \
-                      // expected-note {{because '__is_deducible(BFoo, Foo<int *>)' evaluated to false}} \
+                      // expected-note {{cannot deduce template arguments for 'BFoo' from 'Foo<int *>'}} \
                       // expected-note {{candidate template ignored: could not match 'Foo<type-parameter-0-0 *>' against 'int *'}}
 int i = 0;
 AFoo a1(&i); // OK, deduce Foo<int *>
-- 
GitLab


From 476f7f65f9f17fab7e78f395b83dcb7b0bbd5215 Mon Sep 17 00:00:00 2001
From: weltschildkroete <65237298+weltschildkroete@users.noreply.github.com>
Date: Thu, 16 May 2024 21:15:25 +0200
Subject: [PATCH 1048/1206] [clang][Sema] Emit more specific diagnostic for
 auto in lambda before C++14 (#46059) (#68540)

Namely, we specify that `auto` in a lambda parameter is a C++14
extension in the error message, which now reads:

`'auto' not allowed in lambda parameter before C++14`

This does not change the behavior for `decltype(auto)` and `__auto_type`
though.

---------

Co-authored-by: cor3ntin <corentinjabot@gmail.com>
---
 clang/docs/ReleaseNotes.rst                      |  3 +++
 clang/include/clang/Basic/DiagnosticSemaKinds.td |  4 +++-
 clang/lib/Sema/SemaType.cpp                      | 10 +++++++---
 clang/test/SemaCXX/auto-cxx0x.cpp                |  4 ++--
 4 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 2d2928e41862..65d191b6161a 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -494,6 +494,9 @@ Improvements to Clang's diagnostics
 Improvements to Clang's time-trace
 ----------------------------------
 
+- Clang now specifies that using ``auto`` in a lambda parameter is a C++14 extension when
+  appropriate. (`#46059: <https://github.com/llvm/llvm-project/issues/46059>`_).
+
 Bug Fixes in This Version
 -------------------------
 - Clang's ``-Wundefined-func-template`` no longer warns on pure virtual
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 7e487b70b03d..9372c862a36c 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -2433,7 +2433,9 @@ def err_auto_not_allowed : Error<
   "|in template parameter|in friend declaration|in function prototype that is "
   "not a function declaration|in requires expression parameter"
   "|in array declaration"
-  "|in declaration of conversion function template}1">;
+  "|in declaration of conversion function template"
+  "|in lambda parameter before C++14}1">;
+
 def err_dependent_deduced_tst : Error<
   "typename specifier refers to "
   "%select{class template|function template|variable template|alias template|"
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index eb67546d048a..9c002d56b5bd 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -3257,9 +3257,13 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
         Info = &SemaRef.InventedParameterInfos.back();
       } else {
         // In C++14, generic lambdas allow 'auto' in their parameters.
-        if (!SemaRef.getLangOpts().CPlusPlus14 || !Auto ||
-            Auto->getKeyword() != AutoTypeKeyword::Auto) {
-          Error = 16;
+        if (!SemaRef.getLangOpts().CPlusPlus14 && Auto &&
+            Auto->getKeyword() == AutoTypeKeyword::Auto) {
+          Error = 25; // auto not allowed in lambda parameter (before C++14)
+          break;
+        } else if (!Auto || Auto->getKeyword() != AutoTypeKeyword::Auto) {
+          Error = 16; // __auto_type or decltype(auto) not allowed in lambda
+                      // parameter
           break;
         }
         Info = SemaRef.getCurLambda();
diff --git a/clang/test/SemaCXX/auto-cxx0x.cpp b/clang/test/SemaCXX/auto-cxx0x.cpp
index b4da3f9330c1..07687b606679 100644
--- a/clang/test/SemaCXX/auto-cxx0x.cpp
+++ b/clang/test/SemaCXX/auto-cxx0x.cpp
@@ -12,7 +12,7 @@ thread_local auto x; // expected-error {{requires an initializer}}
 void g() {
   [](auto){}(0);
 #if __cplusplus == 201103L
-  // expected-error@-2 {{'auto' not allowed in lambda parameter}}
+  // expected-error@-2 {{'auto' not allowed in lambda parameter before C++14}}
 #endif
 }
 
@@ -20,6 +20,6 @@ void rdar47689465() {
   int x = 0;
   [](auto __attribute__((noderef)) *){}(&x);
 #if __cplusplus == 201103L
-  // expected-error@-2 {{'auto' not allowed in lambda parameter}}
+  // expected-error@-2 {{'auto' not allowed in lambda parameter before C++14}}
 #endif
 }
-- 
GitLab


From 174cdeced0fe0da07a03d0d118bd70b93badfbb1 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Thu, 16 May 2024 12:17:22 -0700
Subject: [PATCH 1049/1206] [nfc] Clarify when the various PGO instrumentation
 passes run (#92330)

The code seems easier to read if it's centered on what the user wants rather than combinations of whatever internal variables.
---
 llvm/include/llvm/Passes/PassBuilder.h   |  1 +
 llvm/lib/Passes/PassBuilderPipelines.cpp | 57 ++++++++++++++----------
 2 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index 9b4da1c466f5..8de0f024a4b1 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -723,6 +723,7 @@ private:
                          bool AtomicCounterUpdate, std::string ProfileFile,
                          std::string ProfileRemappingFile,
                          IntrusiveRefCntPtr<vfs::FileSystem> FS);
+  void addPostPGOLoopRotation(ModulePassManager &MPM, OptimizationLevel Level);
 
   // Extension Point callbacks
   SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2>
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 72f273972f2f..682bdefbd524 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -803,6 +803,20 @@ void PassBuilder::addPreInlinerPasses(ModulePassManager &MPM,
   MPM.addPass(GlobalDCEPass());
 }
 
+void PassBuilder::addPostPGOLoopRotation(ModulePassManager &MPM,
+                                         OptimizationLevel Level) {
+  if (EnablePostPGOLoopRotation) {
+    // Disable header duplication in loop rotation at -Oz.
+    MPM.addPass(createModuleToFunctionPassAdaptor(
+        createFunctionToLoopPassAdaptor(
+            LoopRotatePass(EnableLoopHeaderDuplication ||
+                           Level != OptimizationLevel::Oz),
+            /*UseMemorySSA=*/false,
+            /*UseBlockFrequencyInfo=*/false),
+        PTO.EagerlyInvalidateAnalyses));
+  }
+}
+
 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
                                     OptimizationLevel Level, bool RunProfileGen,
                                     bool IsCS, bool AtomicCounterUpdate,
@@ -824,17 +838,7 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
   // Perform PGO instrumentation.
   MPM.addPass(PGOInstrumentationGen(IsCS));
 
-  if (EnablePostPGOLoopRotation) {
-    // Disable header duplication in loop rotation at -Oz.
-    MPM.addPass(createModuleToFunctionPassAdaptor(
-        createFunctionToLoopPassAdaptor(
-            LoopRotatePass(EnableLoopHeaderDuplication ||
-                           Level != OptimizationLevel::Oz),
-            /*UseMemorySSA=*/false,
-            /*UseBlockFrequencyInfo=*/false),
-        PTO.EagerlyInvalidateAnalyses));
-  }
-
+  addPostPGOLoopRotation(MPM, Level);
   if (PGOCtxProfLoweringPass::isContextualIRPGOEnabled()) {
     MPM.addPass(PGOCtxProfLoweringPass());
     return;
@@ -1145,29 +1149,34 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM),
                                                 PTO.EagerlyInvalidateAnalyses));
 
-  // Invoke the pre-inliner passes for instrumentation PGO or MemProf.
-  if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
-      (PGOOpt->Action == PGOOptions::IRInstr ||
-       PGOOpt->Action == PGOOptions::IRUse || !PGOOpt->MemoryProfile.empty()))
+  // We already asserted this happens in non-FullLTOPostLink earlier.
+  const bool IsPreLink = Phase != ThinOrFullLTOPhase::ThinLTOPostLink;
+  const bool IsPGOPreLink = PGOOpt && IsPreLink;
+  const bool IsPGOInstrGen =
+      IsPGOPreLink && PGOOpt->Action == PGOOptions::IRInstr;
+  const bool IsPGOInstrUse =
+      IsPGOPreLink && PGOOpt->Action == PGOOptions::IRUse;
+  const bool IsMemprofUse = IsPGOPreLink && !PGOOpt->MemoryProfile.empty();
+
+  if (IsPGOInstrGen || IsPGOInstrUse || IsMemprofUse)
     addPreInlinerPasses(MPM, Level, Phase);
 
   // Add all the requested passes for instrumentation PGO, if requested.
-  if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
-      (PGOOpt->Action == PGOOptions::IRInstr ||
-       PGOOpt->Action == PGOOptions::IRUse)) {
+  if (IsPGOInstrGen || IsPGOInstrUse) {
     addPGOInstrPasses(MPM, Level,
-                      /*RunProfileGen=*/PGOOpt->Action == PGOOptions::IRInstr,
+                      /*RunProfileGen=*/IsPGOInstrGen,
                       /*IsCS=*/false, PGOOpt->AtomicCounterUpdate,
                       PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile,
                       PGOOpt->FS);
-    MPM.addPass(PGOIndirectCallPromotion(false, false));
   }
-  if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
-      PGOOpt->CSAction == PGOOptions::CSIRInstr)
+
+  if (IsPGOInstrGen || IsPGOInstrUse)
+    MPM.addPass(PGOIndirectCallPromotion(false, false));
+
+  if (IsPGOPreLink && PGOOpt->CSAction == PGOOptions::CSIRInstr)
     MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
 
-  if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
-      !PGOOpt->MemoryProfile.empty())
+  if (IsMemprofUse)
     MPM.addPass(MemProfUsePass(PGOOpt->MemoryProfile, PGOOpt->FS));
 
   // Synthesize function entry counts for non-PGO compilation.
-- 
GitLab


From b5f6cc9eb4f661816ef264ad41cb703004380636 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Thu, 16 May 2024 12:31:37 -0700
Subject: [PATCH 1050/1206] Unittest for Error forwarding, follow-up to PR
 #92208 (#92314)

Added a unittest demonstrating Error -> Expected<T> forwarding.
---
 llvm/unittests/Support/ErrorTest.cpp | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/llvm/unittests/Support/ErrorTest.cpp b/llvm/unittests/Support/ErrorTest.cpp
index 1229282cf0de..5d866a67c0ea 100644
--- a/llvm/unittests/Support/ErrorTest.cpp
+++ b/llvm/unittests/Support/ErrorTest.cpp
@@ -13,6 +13,7 @@
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Testing/Support/Error.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest-spi.h"
 #include "gtest/gtest.h"
 #include <memory>
@@ -1158,4 +1159,21 @@ TEST(Error, BadAllocFatalErrorHandlersInteraction) {
   remove_bad_alloc_error_handler();
 }
 
+TEST(Error, ForwardToExpected) {
+  auto ErrorReturningFct = [](bool Fail) {
+    return Fail ? make_error<StringError>(llvm::errc::invalid_argument,
+                                          "Some Error")
+                : Error::success();
+  };
+  auto ExpectedReturningFct = [&](bool Fail) -> Expected<int> {
+    auto Err = ErrorReturningFct(Fail);
+    if (Err)
+      return Err;
+    return 42;
+  };
+  std::optional<int> MaybeV;
+  EXPECT_THAT_ERROR(ExpectedReturningFct(true).moveInto(MaybeV), Failed());
+  EXPECT_THAT_ERROR(ExpectedReturningFct(false).moveInto(MaybeV), Succeeded());
+  EXPECT_EQ(*MaybeV, 42);
+}
 } // namespace
-- 
GitLab


From 9144553207052a868efc5a8ce61a0afbb0eaf236 Mon Sep 17 00:00:00 2001
From: Evgenii Stepanov <eugenis@google.com>
Date: Thu, 16 May 2024 12:34:13 -0700
Subject: [PATCH 1051/1206] Revert "[RISCV] Remove unneeded casts from int64_t
 to uint64_t in RISCVMatInt.cpp. NFC"

LLVM is built with C++17, where left shift of any negative value is still UB.
Detected with UBSan on the buildbot.

This reverts commit 0647d1035cb208195e002b38089b82004b6f7b92.
---
 .../Target/RISCV/MCTargetDesc/RISCVMatInt.cpp   | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
index fca3362f9a8b..0a857eb96935 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp
@@ -115,29 +115,30 @@ static void generateInstSeqImpl(int64_t Val, const MCSubtargetInfo &STI,
     Val >>= ShiftAmount;
 
     // If the remaining bits don't fit in 12 bits, we might be able to reduce
-    // the shift amount in order to use LUI which will zero the lower 12 bits.
+    // the // shift amount in order to use LUI which will zero the lower 12
+    // bits.
     if (ShiftAmount > 12 && !isInt<12>(Val)) {
-      if (isInt<32>(Val << 12)) {
+      if (isInt<32>((uint64_t)Val << 12)) {
         // Reduce the shift amount and add zeros to the LSBs so it will match
         // LUI.
         ShiftAmount -= 12;
-        Val = Val << 12;
-      } else if (isUInt<32>(Val << 12) &&
+        Val = (uint64_t)Val << 12;
+      } else if (isUInt<32>((uint64_t)Val << 12) &&
                  STI.hasFeature(RISCV::FeatureStdExtZba)) {
         // Reduce the shift amount and add zeros to the LSBs so it will match
         // LUI, then shift left with SLLI.UW to clear the upper 32 set bits.
         ShiftAmount -= 12;
-        Val = SignExtend64<32>(Val << 12);
+        Val = ((uint64_t)Val << 12) | (0xffffffffull << 32);
         Unsigned = true;
       }
     }
 
     // Try to use SLLI_UW for Val when it is uint32 but not int32.
-    if (isUInt<32>(Val) && !isInt<32>(Val) &&
+    if (isUInt<32>((uint64_t)Val) && !isInt<32>((uint64_t)Val) &&
         STI.hasFeature(RISCV::FeatureStdExtZba)) {
-      // Use LUI+ADDI(W) or LUI to compose, then clear the upper 32 bits with
+      // Use LUI+ADDI or LUI to compose, then clear the upper 32 bits with
       // SLLI_UW.
-      Val = SignExtend64<32>(Val);
+      Val = ((uint64_t)Val) | (0xffffffffull << 32);
       Unsigned = true;
     }
   }
-- 
GitLab


From f42f57b52dd279e6ae19270d063aeb8d59e3f11c Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 16 May 2024 14:58:46 -0500
Subject: [PATCH 1052/1206] [Libomptarget] Rework Record & Replay to be a
 plugin member (#88928) (#89097)

Summary:
Previously, the R&R support was global state initialized by a global
constructor. This is bad because it prevents us from adequately
constraining the lifetime of the library. Additionally, we want to
minimize the amount of global state floating around.

This patch moves the R&R support into a plugin member like everything
else. This means there will be multiple copies of the R&R implementation
floating around, but this was already the case given the fact that we
currently handle everything with dynamic libraries.
---
 .../common/include/PluginInterface.h          | 13 ++++++-
 .../common/src/PluginInterface.cpp            | 34 +++++++++++++------
 2 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index d8d71e3e65a4..83f6e8d76fec 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -54,6 +54,7 @@ namespace plugin {
 struct GenericPluginTy;
 struct GenericKernelTy;
 struct GenericDeviceTy;
+struct RecordReplayTy;
 
 /// Class that wraps the __tgt_async_info to simply its usage. In case the
 /// object is constructed without a valid __tgt_async_info, the object will use
@@ -958,7 +959,8 @@ struct GenericPluginTy {
 
   /// Construct a plugin instance.
   GenericPluginTy(Triple::ArchType TA)
-      : GlobalHandler(nullptr), JIT(TA), RPCServer(nullptr) {}
+      : GlobalHandler(nullptr), JIT(TA), RPCServer(nullptr),
+        RecordReplay(nullptr) {}
 
   virtual ~GenericPluginTy() {}
 
@@ -1027,6 +1029,12 @@ struct GenericPluginTy {
     return *RPCServer;
   }
 
+  /// Get a reference to the record and replay interface for the plugin.
+  RecordReplayTy &getRecordReplay() {
+    assert(RecordReplay && "RR interface not initialized");
+    return *RecordReplay;
+  }
+
   /// Initialize a device within the plugin.
   Error initDevice(int32_t DeviceId);
 
@@ -1204,6 +1212,9 @@ private:
 
   /// The interface between the plugin and the GPU for host services.
   RPCServerTy *RPCServer;
+
+  /// The interface between the plugin and the GPU for host services.
+  RecordReplayTy *RecordReplay;
 };
 
 namespace Plugin {
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 253acacc3a9d..550ebc9c28b2 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -39,6 +39,7 @@ using namespace target;
 using namespace plugin;
 
 // TODO: Fix any thread safety issues for multi-threaded kernel recording.
+namespace llvm::omp::target::plugin {
 struct RecordReplayTy {
 
   // Describes the state of the record replay mechanism.
@@ -358,8 +359,7 @@ public:
     }
   }
 };
-
-static RecordReplayTy RecordReplay;
+} // namespace llvm::omp::target::plugin
 
 // Extract the mapping of host function pointers to device function pointers
 // from the entry table. Functions marked as 'indirect' in OpenMP will have
@@ -470,7 +470,7 @@ GenericKernelTy::getKernelLaunchEnvironment(
   // Ctor/Dtor have no arguments, replaying uses the original kernel launch
   // environment. Older versions of the compiler do not generate a kernel
   // launch environment.
-  if (RecordReplay.isReplaying() ||
+  if (GenericDevice.Plugin.getRecordReplay().isReplaying() ||
       Version < OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR)
     return nullptr;
 
@@ -559,6 +559,7 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
 
   // Record the kernel description after we modified the argument count and num
   // blocks/threads.
+  RecordReplayTy &RecordReplay = GenericDevice.Plugin.getRecordReplay();
   if (RecordReplay.isRecording()) {
     RecordReplay.saveImage(getName(), getImage());
     RecordReplay.saveKernelInput(getName(), getImage());
@@ -833,6 +834,7 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
     delete MemoryManager;
   MemoryManager = nullptr;
 
+  RecordReplayTy &RecordReplay = Plugin.getRecordReplay();
   if (RecordReplay.isRecordingOrReplaying())
     RecordReplay.deinit();
 
@@ -886,7 +888,8 @@ GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
     return std::move(Err);
 
   // Setup the global device memory pool if needed.
-  if (!RecordReplay.isReplaying() && shouldSetupDeviceMemoryPool()) {
+  if (!Plugin.getRecordReplay().isReplaying() &&
+      shouldSetupDeviceMemoryPool()) {
     uint64_t HeapSize;
     auto SizeOrErr = getDeviceHeapSize(HeapSize);
     if (SizeOrErr) {
@@ -1301,8 +1304,8 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
                                             TargetAllocTy Kind) {
   void *Alloc = nullptr;
 
-  if (RecordReplay.isRecordingOrReplaying())
-    return RecordReplay.alloc(Size);
+  if (Plugin.getRecordReplay().isRecordingOrReplaying())
+    return Plugin.getRecordReplay().alloc(Size);
 
   switch (Kind) {
   case TARGET_ALLOC_DEFAULT:
@@ -1338,7 +1341,7 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
 
 Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) {
   // Free is a noop when recording or replaying.
-  if (RecordReplay.isRecordingOrReplaying())
+  if (Plugin.getRecordReplay().isRecordingOrReplaying())
     return Plugin::success();
 
   int Res;
@@ -1405,7 +1408,8 @@ Error GenericDeviceTy::launchKernel(void *EntryPtr, void **ArgPtrs,
                                     KernelArgsTy &KernelArgs,
                                     __tgt_async_info *AsyncInfo) {
   AsyncInfoWrapperTy AsyncInfoWrapper(
-      *this, RecordReplay.isRecordingOrReplaying() ? nullptr : AsyncInfo);
+      *this,
+      Plugin.getRecordReplay().isRecordingOrReplaying() ? nullptr : AsyncInfo);
 
   GenericKernelTy &GenericKernel =
       *reinterpret_cast<GenericKernelTy *>(EntryPtr);
@@ -1416,6 +1420,7 @@ Error GenericDeviceTy::launchKernel(void *EntryPtr, void **ArgPtrs,
   // 'finalize' here to guarantee next record-replay actions are in-sync
   AsyncInfoWrapper.finalize(Err);
 
+  RecordReplayTy &RecordReplay = Plugin.getRecordReplay();
   if (RecordReplay.isRecordingOrReplaying() &&
       RecordReplay.isSaveOutputEnabled())
     RecordReplay.saveKernelOutputInfo(GenericKernel.getName());
@@ -1503,6 +1508,9 @@ Error GenericPluginTy::init() {
   RPCServer = new RPCServerTy(*this);
   assert(RPCServer && "Invalid RPC server");
 
+  RecordReplay = new RecordReplayTy();
+  assert(RecordReplay && "Invalid RR interface");
+
   return Plugin::success();
 }
 
@@ -1523,6 +1531,9 @@ Error GenericPluginTy::deinit() {
   if (RPCServer)
     delete RPCServer;
 
+  if (RecordReplay)
+    delete RecordReplay;
+
   // Perform last deinitializations on the plugin.
   return deinitImpl();
 }
@@ -1633,12 +1644,12 @@ int32_t GenericPluginTy::initialize_record_replay(int32_t DeviceId,
       isRecord ? RecordReplayTy::RRStatusTy::RRRecording
                : RecordReplayTy::RRStatusTy::RRReplaying;
 
-  if (auto Err = RecordReplay.init(&Device, MemorySize, VAddr, Status,
-                                   SaveOutput, ReqPtrArgOffset)) {
+  if (auto Err = RecordReplay->init(&Device, MemorySize, VAddr, Status,
+                                    SaveOutput, ReqPtrArgOffset)) {
     REPORT("WARNING RR did not intialize RR-properly with %lu bytes"
            "(Error: %s)\n",
            MemorySize, toString(std::move(Err)).data());
-    RecordReplay.setStatus(RecordReplayTy::RRStatusTy::RRDeactivated);
+    RecordReplay->setStatus(RecordReplayTy::RRStatusTy::RRDeactivated);
 
     if (!isRecord) {
       return OFFLOAD_FAIL;
@@ -1982,6 +1993,7 @@ int32_t GenericPluginTy::get_global(__tgt_device_binary Binary, uint64_t Size,
   assert(DevicePtr && "Invalid device global's address");
 
   // Save the loaded globals if we are recording.
+  RecordReplayTy &RecordReplay = Device.Plugin.getRecordReplay();
   if (RecordReplay.isRecording())
     RecordReplay.addEntry(Name, Size, *DevicePtr);
 
-- 
GitLab


From 9d0a8eda0baca51ce61fd5ce7c59864b2b57dcf8 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 16 May 2024 15:13:24 -0500
Subject: [PATCH 1053/1206] [libc][NFC] Replace address space with constant GPU
 utility

Summary:
This was written before we had the address space helpers. Using
numerical address spaces is bad so this should be removed.
---
 libc/src/time/gpu/time_utils.cpp | 4 ++--
 libc/src/time/gpu/time_utils.h   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/libc/src/time/gpu/time_utils.cpp b/libc/src/time/gpu/time_utils.cpp
index 300a72b10236..53a1b2121f7c 100644
--- a/libc/src/time/gpu/time_utils.cpp
+++ b/libc/src/time/gpu/time_utils.cpp
@@ -15,8 +15,8 @@ namespace LIBC_NAMESPACE {
 // insufficient.
 // TODO: Once we have another use-case for this we should put it in a common
 // device environment struct.
-extern "C" [[gnu::visibility("protected")]] uint64_t
-    [[clang::address_space(4)]] __llvm_libc_clock_freq = clock_freq;
+extern "C" [[gnu::visibility("protected")]] gpu::Constant<uint64_t>
+    __llvm_libc_clock_freq = clock_freq;
 #endif
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/time/gpu/time_utils.h b/libc/src/time/gpu/time_utils.h
index 3f1fd11c1791..e7e0e526a532 100644
--- a/libc/src/time/gpu/time_utils.h
+++ b/libc/src/time/gpu/time_utils.h
@@ -23,8 +23,8 @@ constexpr uint64_t clock_freq = 100000000UL;
 
 // We provide an externally visible symbol such that the runtime can set
 // this to the correct value.
-extern "C" [[gnu::visibility("protected")]] uint64_t
-    [[clang::address_space(4)]] __llvm_libc_clock_freq;
+extern "C" [[gnu::visibility("protected")]] gpu::Constant<uint64_t>
+    __llvm_libc_clock_freq;
 #define GPU_CLOCKS_PER_SEC static_cast<clock_t>(__llvm_libc_clock_freq)
 
 #elif defined(LIBC_TARGET_ARCH_IS_NVPTX)
-- 
GitLab


From 61565abcdc4d864a8c7aa7ad23ed441a3762b77d Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 16 May 2024 15:24:23 -0500
Subject: [PATCH 1054/1206] [libc] Fix constant variable initializer

---
 libc/src/time/gpu/time_utils.cpp | 4 ++--
 libc/src/time/gpu/time_utils.h   | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/libc/src/time/gpu/time_utils.cpp b/libc/src/time/gpu/time_utils.cpp
index 53a1b2121f7c..67fe5b4861ac 100644
--- a/libc/src/time/gpu/time_utils.cpp
+++ b/libc/src/time/gpu/time_utils.cpp
@@ -15,8 +15,8 @@ namespace LIBC_NAMESPACE {
 // insufficient.
 // TODO: Once we have another use-case for this we should put it in a common
 // device environment struct.
-extern "C" [[gnu::visibility("protected")]] gpu::Constant<uint64_t>
-    __llvm_libc_clock_freq = clock_freq;
+extern "C" [[gnu::visibility("protected")]] uint64_t __llvm_libc_clock_freq =
+    clock_freq;
 #endif
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/time/gpu/time_utils.h b/libc/src/time/gpu/time_utils.h
index e7e0e526a532..da713886b643 100644
--- a/libc/src/time/gpu/time_utils.h
+++ b/libc/src/time/gpu/time_utils.h
@@ -23,8 +23,7 @@ constexpr uint64_t clock_freq = 100000000UL;
 
 // We provide an externally visible symbol such that the runtime can set
 // this to the correct value.
-extern "C" [[gnu::visibility("protected")]] gpu::Constant<uint64_t>
-    __llvm_libc_clock_freq;
+extern "C" [[gnu::visibility("protected")]] uint64_t __llvm_libc_clock_freq;
 #define GPU_CLOCKS_PER_SEC static_cast<clock_t>(__llvm_libc_clock_freq)
 
 #elif defined(LIBC_TARGET_ARCH_IS_NVPTX)
-- 
GitLab


From 479f4a7b68a8b1eb8da7fc3f28638045eaa7dc25 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 16 May 2024 13:26:13 -0700
Subject: [PATCH 1055/1206] [memprof] Update comments for writeMemProf and its
 helpers (#92446)

This patch adds comments for writeMemProf{V0,V1,V2} in a
version-specific manner.  The mostly repetitive nature of the comments
is somewhat unfortunate but intentional to make it easy to retire
older versions.

Without this patch, the comment just before writeMemProf documents the
Version1 format, which is very confusing.
---
 llvm/lib/ProfileData/InstrProfWriter.cpp | 52 ++++++++++++++++++------
 1 file changed, 39 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index aeafd429e8ba..b5b13550b057 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -499,6 +499,17 @@ static uint64_t writeMemProfCallStacks(
   return CallStackTableGenerator.Emit(OS.OS);
 }
 
+// Write out MemProf Version0 as follows:
+// uint64_t RecordTableOffset = RecordTableGenerator.Emit
+// uint64_t FramePayloadOffset = Offset for the frame payload
+// uint64_t FrameTableOffset = FrameTableGenerator.Emit
+// uint64_t Num schema entries
+// uint64_t Schema entry 0
+// uint64_t Schema entry 1
+// ....
+// uint64_t Schema entry N - 1
+// OnDiskChainedHashTable MemProfRecordData
+// OnDiskChainedHashTable MemProfFrameData
 static Error writeMemProfV0(ProfOStream &OS,
                             memprof::IndexedMemProfData &MemProfData) {
   uint64_t HeaderUpdatePos = OS.tell();
@@ -521,6 +532,18 @@ static Error writeMemProfV0(ProfOStream &OS,
   return Error::success();
 }
 
+// Write out MemProf Version1 as follows:
+// uint64_t Version (NEW in V1)
+// uint64_t RecordTableOffset = RecordTableGenerator.Emit
+// uint64_t FramePayloadOffset = Offset for the frame payload
+// uint64_t FrameTableOffset = FrameTableGenerator.Emit
+// uint64_t Num schema entries
+// uint64_t Schema entry 0
+// uint64_t Schema entry 1
+// ....
+// uint64_t Schema entry N - 1
+// OnDiskChainedHashTable MemProfRecordData
+// OnDiskChainedHashTable MemProfFrameData
 static Error writeMemProfV1(ProfOStream &OS,
                             memprof::IndexedMemProfData &MemProfData) {
   OS.write(memprof::Version1);
@@ -544,6 +567,21 @@ static Error writeMemProfV1(ProfOStream &OS,
   return Error::success();
 }
 
+// Write out MemProf Version2 as follows:
+// uint64_t Version
+// uint64_t RecordTableOffset = RecordTableGenerator.Emit
+// uint64_t FramePayloadOffset = Offset for the frame payload
+// uint64_t FrameTableOffset = FrameTableGenerator.Emit
+// uint64_t CallStackPayloadOffset = Offset for the call stack payload (NEW V2)
+// uint64_t CallStackTableOffset = CallStackTableGenerator.Emit (NEW in V2)
+// uint64_t Num schema entries
+// uint64_t Schema entry 0
+// uint64_t Schema entry 1
+// ....
+// uint64_t Schema entry N - 1
+// OnDiskChainedHashTable MemProfRecordData
+// OnDiskChainedHashTable MemProfFrameData
+// OnDiskChainedHashTable MemProfCallStackData (NEW in V2)
 static Error writeMemProfV2(ProfOStream &OS,
                             memprof::IndexedMemProfData &MemProfData,
                             bool MemProfFullSchema) {
@@ -579,19 +617,7 @@ static Error writeMemProfV2(ProfOStream &OS,
   return Error::success();
 }
 
-// The MemProf profile data includes a simple schema
-// with the format described below followed by the hashtable:
-// uint64_t Version
-// uint64_t RecordTableOffset = RecordTableGenerator.Emit
-// uint64_t FramePayloadOffset = Stream offset before emitting the frame table
-// uint64_t FrameTableOffset = FrameTableGenerator.Emit
-// uint64_t Num schema entries
-// uint64_t Schema entry 0
-// uint64_t Schema entry 1
-// ....
-// uint64_t Schema entry N - 1
-// OnDiskChainedHashTable MemProfRecordData
-// OnDiskChainedHashTable MemProfFrameData
+// Write out the MemProf data in a requested version.
 static Error writeMemProf(ProfOStream &OS,
                           memprof::IndexedMemProfData &MemProfData,
                           memprof::IndexedVersion MemProfVersionRequested,
-- 
GitLab


From f05c068429a86ef2187eae094354c73e8339baa2 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellis.sparky.hoag@gmail.com>
Date: Thu, 16 May 2024 15:40:02 -0500
Subject: [PATCH 1056/1206] [InstrProf] Remove unused argv in llvm-profdata.cpp
 (#92435)

https://github.com/llvm/llvm-project/pull/71328 refactored
`llvm-profdata.cpp` to use subcommands (which is super nice), but left
many unused `argv` variables. This opts to use `ProgName` where
necessary, and removes `argv` otherwise.
---
 llvm/tools/llvm-profdata/llvm-profdata.cpp | 24 +++++++++++-----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index ec046ebfab13..4126b55576dd 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -1654,7 +1654,7 @@ static void parseInputFilenamesFile(MemoryBuffer *Buffer,
   }
 }
 
-static int merge_main(int argc, const char *argv[]) {
+static int merge_main(StringRef ProgName) {
   WeightedFileVector WeightedInputs;
   for (StringRef Filename : InputFilenames)
     addWeightedInput(WeightedInputs, {std::string(Filename), 1});
@@ -1667,8 +1667,7 @@ static int merge_main(int argc, const char *argv[]) {
   parseInputFilenamesFile(Buffer.get(), WeightedInputs);
 
   if (WeightedInputs.empty())
-    exitWithError("no input files specified. See " +
-                  sys::path::filename(argv[0]) + " " + argv[1] + " -help");
+    exitWithError("no input files specified. See " + ProgName + " merge -help");
 
   if (DumpInputFileList) {
     for (auto &WF : WeightedInputs)
@@ -2654,7 +2653,7 @@ void overlapSampleProfile(const std::string &BaseFilename,
   OverlapAggr.dumpFuncSimilarity(OS);
 }
 
-static int overlap_main(int argc, const char *argv[]) {
+static int overlap_main() {
   std::error_code EC;
   raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
   if (EC)
@@ -3231,15 +3230,16 @@ static int showDebugInfoCorrelation(const std::string &Filename,
   return 0;
 }
 
-static int show_main(int argc, const char *argv[]) {
+static int show_main(StringRef ProgName) {
   if (Filename.empty() && DebugInfoFilename.empty())
     exitWithError(
         "the positional argument '<profdata-file>' is required unless '--" +
         DebugInfoFilename.ArgStr + "' is provided");
 
   if (Filename == OutputFilename) {
-    errs() << sys::path::filename(argv[0]) << " " << argv[1]
-           << ": Input file name cannot be the same as the output file name!\n";
+    errs() << ProgName
+           << " show: Input file name cannot be the same as the output file "
+              "name!\n";
     return 1;
   }
   if (JsonFormat)
@@ -3263,7 +3263,7 @@ static int show_main(int argc, const char *argv[]) {
   return showMemProfProfile(SFormat, OS);
 }
 
-static int order_main(int argc, const char *argv[]) {
+static int order_main() {
   std::error_code EC;
   raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
   if (EC)
@@ -3314,16 +3314,16 @@ int llvm_profdata_main(int argc, char **argvNonConst,
   cl::ParseCommandLineOptions(argc, argv, "LLVM profile data\n");
 
   if (ShowSubcommand)
-    return show_main(argc, argv);
+    return show_main(ProgName);
 
   if (OrderSubcommand)
-    return order_main(argc, argv);
+    return order_main();
 
   if (OverlapSubcommand)
-    return overlap_main(argc, argv);
+    return overlap_main();
 
   if (MergeSubcommand)
-    return merge_main(argc, argv);
+    return merge_main(ProgName);
 
   errs() << ProgName
          << ": Unknown command. Run llvm-profdata --help for usage.\n";
-- 
GitLab


From 1ede503910d3133ea1af8c084fee96ca98c66464 Mon Sep 17 00:00:00 2001
From: Hugo Trachino <hugo.trachino@huawei.com>
Date: Thu, 16 May 2024 21:40:56 +0100
Subject: [PATCH 1057/1206] [MLIR][Vector] Implement
 TransferReadOfExtractSliceOp as MaskableOpRewritePattern (#91960)

Split of https://github.com/llvm/llvm-project/pull/90835
Adds support for `TransferReadOfExtractSliceOpFolder` when the
`TransferReadOp` is inside a `MaskOp`.
---
 .../Dialect/Tensor/Transforms/CMakeLists.txt  |  1 +
 .../Tensor/Transforms/FoldTensorSubsetOps.cpp | 29 ++++++++++++-------
 ...nsor-subset-ops-into-vector-transfers.mlir | 15 ++++++++++
 3 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt
index c6ef6ed86e0d..0aabdaf667b9 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt
@@ -39,5 +39,6 @@ add_mlir_dialect_library(MLIRTensorTransforms
   MLIRTilingInterface
   MLIRTransforms
   MLIRVectorDialect
+  MLIRVectorUtils
   MLIRValueBoundsOpInterface
 )
diff --git a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
index 3b8d3708bb73..5396531922aa 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/FoldTensorSubsetOps.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Tensor/Transforms/Transforms.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
@@ -48,12 +49,14 @@ static Value getTensorOperand(tensor::InsertSliceOp op) {
 namespace {
 /// Merge extract_slice operation with load/transferRead operation.
 class TransferReadOfExtractSliceOpFolder final
-    : public OpRewritePattern<vector::TransferReadOp> {
+    : public vector::MaskableOpRewritePattern<vector::TransferReadOp> {
 public:
-  using OpRewritePattern<vector::TransferReadOp>::OpRewritePattern;
+  using MaskableOpRewritePattern::MaskableOpRewritePattern;
 
-  LogicalResult matchAndRewrite(vector::TransferReadOp readOp,
-                                PatternRewriter &rewriter) const override;
+  FailureOr<mlir::Value>
+  matchAndRewriteMaskableOp(vector::TransferReadOp readOp,
+                            vector::MaskingOpInterface maskOp,
+                            PatternRewriter &rewriter) const override;
 };
 
 /// Merge insert_slice operation with store/transferWriteOp operation.
@@ -84,8 +87,10 @@ static LogicalResult preconditionsFoldExtractOrInsertWithTransferOp(
   return success();
 }
 
-LogicalResult TransferReadOfExtractSliceOpFolder::matchAndRewrite(
-    vector::TransferReadOp readOp, PatternRewriter &rewriter) const {
+FailureOr<mlir::Value>
+TransferReadOfExtractSliceOpFolder::matchAndRewriteMaskableOp(
+    vector::TransferReadOp readOp, vector::MaskingOpInterface maskOp,
+    PatternRewriter &rewriter) const {
   auto extractSliceOp =
       getTensorOperand(readOp).getDefiningOp<tensor::ExtractSliceOp>();
   if (!extractSliceOp)
@@ -95,7 +100,7 @@ LogicalResult TransferReadOfExtractSliceOpFolder::matchAndRewrite(
       preconditionsFoldExtractOrInsertWithTransferOp(rewriter, readOp,
                                                      extractSliceOp);
   if (failed(preconditionResult))
-    return preconditionResult;
+    return rewriter.notifyMatchFailure(readOp, "Failed preconditions");
 
   SmallVector<Value> indices(readOp.getIndices().begin(),
                              readOp.getIndices().end());
@@ -105,15 +110,17 @@ LogicalResult TransferReadOfExtractSliceOpFolder::matchAndRewrite(
       extractSliceOp.getMixedStrides(), extractSliceOp.getDroppedDims(),
       indices, sourceIndices);
 
-  rewriter.replaceOpWithNewOp<vector::TransferReadOp>(
-      readOp, readOp.getVectorType(), extractSliceOp.getSource(), sourceIndices,
+  Operation *newOp = rewriter.create<vector::TransferReadOp>(
+      readOp.getLoc(), readOp.getVectorType(), extractSliceOp.getSource(),
+      sourceIndices,
       AffineMapAttr::get(expandDimsToRank(
           readOp.getPermutationMap(), extractSliceOp.getSourceType().getRank(),
           extractSliceOp.getDroppedDims())),
       readOp.getPadding(),
       /*mask=*/Value(), readOp.getInBoundsAttr());
-
-  return success();
+  if (maskOp)
+    newOp = mlir::vector::maskOperation(rewriter, newOp, maskOp.getMask());
+  return newOp->getResults()[0];
 }
 
 LogicalResult InsertSliceOfTransferWriteOpFolder::matchAndRewrite(
diff --git a/mlir/test/Dialect/Tensor/fold-tensor-subset-ops-into-vector-transfers.mlir b/mlir/test/Dialect/Tensor/fold-tensor-subset-ops-into-vector-transfers.mlir
index 6213db3956f9..e562cf8efc35 100644
--- a/mlir/test/Dialect/Tensor/fold-tensor-subset-ops-into-vector-transfers.mlir
+++ b/mlir/test/Dialect/Tensor/fold-tensor-subset-ops-into-vector-transfers.mlir
@@ -75,6 +75,21 @@ func.func @transfer_read_of_extract_slice_non_leading_rank_reduction(%t : tensor
   return %1 : vector<5x6xf32>
 }
 
+// CHECK-LABEL: func @masked_transfer_read_of_extract_slice
+//  CHECK-SAME:     %[[t:.*]]: tensor<?x?xf32>, %[[s1:.*]]: index, %[[s2:.*]]: index
+//   CHECK-DAG: %[[m:.*]] = vector.create_mask{{.*}} : vector<5x6xi1>
+//   CHECK-DAG: %[[a:.*]] = affine.apply {{.*}}[[s1]]
+//       CHECK: vector.mask %[[m]] { vector.transfer_read %[[t]]{{.*}}: tensor<?x?xf32>, vector<5x6xf32> } : vector<5x6xi1> -> vector<5x6xf32>
+func.func @masked_transfer_read_of_extract_slice(%t : tensor<?x?xf32>, %s1 : index, %s2 : index) -> vector<5x6xf32> {
+  %c3 = arith.constant 3 : index
+  %c4 = arith.constant 4 : index
+  %cst = arith.constant 0.0 : f32
+  %0 = tensor.extract_slice %t[5, %s1] [10, %s2] [1, 1] : tensor<?x?xf32> to tensor<10x?xf32>
+  %mask = vector.create_mask %c3, %c4 : vector<5x6xi1>
+  %1 = vector.mask %mask {vector.transfer_read %0[%c3, %c4], %cst {in_bounds = [true, true]} : tensor<10x?xf32>, vector<5x6xf32>} : vector<5x6xi1> -> vector<5x6xf32>
+  return %1 : vector<5x6xf32>
+}
+
 // CHECK-LABEL: func @insert_slice_of_transfer_write(
 //  CHECK-SAME:     %[[t1:.*]]: tensor<?x12xf32>, %[[v:.*]]: vector<5x6xf32>, %[[s:.*]]: index
 //       CHECK:   %[[c3:.*]] = arith.constant 3 : index
-- 
GitLab


From d311a62e2f1ae34a0329c3812f2969cedac93561 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Thu, 16 May 2024 13:41:36 -0700
Subject: [PATCH 1058/1206] [ctx_profile] Decouple ctx  instrumentation from
 PGOOpt (#92445)

We currently don't support passing files and don't need frontend involvement either.
---
 llvm/lib/Passes/PassBuilderPipelines.cpp | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 682bdefbd524..1892e16a0652 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -839,10 +839,6 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
   MPM.addPass(PGOInstrumentationGen(IsCS));
 
   addPostPGOLoopRotation(MPM, Level);
-  if (PGOCtxProfLoweringPass::isContextualIRPGOEnabled()) {
-    MPM.addPass(PGOCtxProfLoweringPass());
-    return;
-  }
   // Add the profile lowering pass.
   InstrProfOptions Options;
   if (!ProfileFile.empty())
@@ -1157,8 +1153,17 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   const bool IsPGOInstrUse =
       IsPGOPreLink && PGOOpt->Action == PGOOptions::IRUse;
   const bool IsMemprofUse = IsPGOPreLink && !PGOOpt->MemoryProfile.empty();
-
-  if (IsPGOInstrGen || IsPGOInstrUse || IsMemprofUse)
+  // We don't want to mix pgo ctx gen and pgo gen; we also don't currently
+  // enable ctx profiling from the frontend.
+  assert(
+      !(IsPGOInstrGen && PGOCtxProfLoweringPass::isContextualIRPGOEnabled()) &&
+      "Enabling both instrumented FDO and contextual instrumentation is not "
+      "supported.");
+  // Enable contextual profiling instrumentation.
+  const bool IsCtxProfGen = !IsPGOInstrGen && IsPreLink &&
+                            PGOCtxProfLoweringPass::isContextualIRPGOEnabled();
+
+  if (IsPGOInstrGen || IsPGOInstrUse || IsMemprofUse || IsCtxProfGen)
     addPreInlinerPasses(MPM, Level, Phase);
 
   // Add all the requested passes for instrumentation PGO, if requested.
@@ -1168,9 +1173,13 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
                       /*IsCS=*/false, PGOOpt->AtomicCounterUpdate,
                       PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile,
                       PGOOpt->FS);
+  } else if (IsCtxProfGen) {
+    MPM.addPass(PGOInstrumentationGen(false));
+    addPostPGOLoopRotation(MPM, Level);
+    MPM.addPass(PGOCtxProfLoweringPass());
   }
 
-  if (IsPGOInstrGen || IsPGOInstrUse)
+  if (IsPGOInstrGen || IsPGOInstrUse || IsCtxProfGen)
     MPM.addPass(PGOIndirectCallPromotion(false, false));
 
   if (IsPGOPreLink && PGOOpt->CSAction == PGOOptions::CSIRInstr)
-- 
GitLab


From 6d2219acabc1882508d695e23d444d3e1cae4ff3 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 16 May 2024 16:06:20 -0500
Subject: [PATCH 1059/1206] [Libomptarget] Pass '-Werror=global-constructors'
 to the libomptarget build (#88531)

Summary:
A runtime library should not have global constructors. Everything is now
expected to go through the init methods. This patch ensures that global
constructors will not accidentally be introduced.
---
 offload/CMakeLists.txt     | 5 +++++
 offload/src/CMakeLists.txt | 4 ++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/offload/CMakeLists.txt b/offload/CMakeLists.txt
index 1d8cab240924..e2a4c2597636 100644
--- a/offload/CMakeLists.txt
+++ b/offload/CMakeLists.txt
@@ -136,6 +136,8 @@ include(LibomptargetGetDependencies)
 # Set up testing infrastructure.
 include(OpenMPTesting)
 
+check_cxx_compiler_flag(-Werror=global-constructors OFFLOAD_HAVE_WERROR_CTOR)
+
 # LLVM source tree is required at build time for libomptarget
 if (NOT LIBOMPTARGET_LLVM_INCLUDE_DIRS)
   message(FATAL_ERROR "Missing definition for LIBOMPTARGET_LLVM_INCLUDE_DIRS")
@@ -207,6 +209,9 @@ set(offload_compile_flags -fno-exceptions)
 if(NOT LLVM_ENABLE_RTTI)
   set(offload_compile_flags ${offload_compile_flags} -fno-rtti)
 endif()
+if(OFFLOAD_HAVE_WERROR_CTOR)
+  list(APPEND offload_compile_flags -Werror=global-constructors)
+endif()
 
 # TODO: Consider enabling LTO by default if supported.
 # https://cmake.org/cmake/help/latest/module/CheckIPOSupported.html can be used
diff --git a/offload/src/CMakeLists.txt b/offload/src/CMakeLists.txt
index b474f29ea0be..c5338130297f 100644
--- a/offload/src/CMakeLists.txt
+++ b/offload/src/CMakeLists.txt
@@ -69,8 +69,8 @@ foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD)
   target_link_libraries(omptarget PRIVATE omptarget.rtl.${plugin})
 endforeach()
 
-target_compile_options(omptarget PUBLIC ${offload_compile_flags})
-target_link_options(omptarget PUBLIC ${offload_link_flags})
+target_compile_options(omptarget PRIVATE ${offload_compile_flags})
+target_link_options(omptarget PRIVATE ${offload_link_flags})
 
 # libomptarget.so needs to be aware of where the plugins live as they
 # are now separated in the build directory.
-- 
GitLab


From 47d80ec1802d70082c8fd32b4396c98db2c4dba2 Mon Sep 17 00:00:00 2001
From: Jacob Lalonde <jjjlalonde@gmail.com>
Date: Thu, 16 May 2024 14:17:19 -0700
Subject: [PATCH 1060/1206] [LLDB/Coredump] Only take the Pthread from stack
 start to the stackpointer + red_zone (#92002)

Currently in Core dumps, the entire pthread is copied, including the
unused space beyond the stack pointer. This causes large amounts of core
dump inflation when the number of threads is high, but the stack usage
is low. Such as when an application is using a thread pool.

This change will optimize for these situations in addition to generally
improving the core dump performance for all of lldb.
---
 .../Minidump/MinidumpFileBuilder.cpp          |  12 ++
 lldb/source/Target/Process.cpp                | 109 ++++++++++--------
 .../TestProcessSaveCoreMinidump.py            |  65 +++++++++--
 3 files changed, 131 insertions(+), 55 deletions(-)

diff --git a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
index 601f11d51d42..7231433619ff 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
+++ b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
@@ -14,6 +14,7 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleList.h"
 #include "lldb/Core/Section.h"
+#include "lldb/Target/ABI.h"
 #include "lldb/Target/MemoryRegionInfo.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
@@ -491,6 +492,17 @@ findStackHelper(const lldb::ProcessSP &process_sp, uint64_t rsp) {
         std::errc::not_supported,
         "unable to load stack segment of the process");
 
+  // This is a duplicate of the logic in
+  // Process::SaveOffRegionsWithStackPointers but ultimately, we need to only
+  // save up from the start of the stack down to the stack pointer
+  const addr_t range_end = range_info.GetRange().GetRangeEnd();
+  const addr_t red_zone = process_sp->GetABI()->GetRedZoneSize();
+  const addr_t stack_head = rsp - red_zone;
+  if (stack_head > range_info.GetRange().GetRangeEnd()) {
+    range_info.GetRange().SetRangeBase(stack_head);
+    range_info.GetRange().SetByteSize(range_end - stack_head);
+  }
+
   const addr_t addr = range_info.GetRange().GetRangeBase();
   const addr_t size = range_info.GetRange().GetByteSize();
 
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index 25afade9a827..216d2f21abfe 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -3857,8 +3857,8 @@ thread_result_t Process::RunPrivateStateThread(bool is_secondary_thread) {
         // case we should tell it to stop doing that.  Normally, we don't NEED
         // to do that because we will next close the communication to the stub
         // and that will get it to shut down.  But there are remote debugging
-        // cases where relying on that side-effect causes the shutdown to be 
-        // flakey, so we should send a positive signal to interrupt the wait. 
+        // cases where relying on that side-effect causes the shutdown to be
+        // flakey, so we should send a positive signal to interrupt the wait.
         Status error = HaltPrivate();
         BroadcastEvent(eBroadcastBitInterrupt, nullptr);
       } else if (StateIsRunningState(m_last_broadcast_state)) {
@@ -6335,30 +6335,65 @@ static void AddRegion(const MemoryRegionInfo &region, bool try_dirty_pages,
   ranges.push_back(CreateCoreFileMemoryRange(region));
 }
 
+static void SaveOffRegionsWithStackPointers(
+    Process &process, const MemoryRegionInfos &regions,
+    Process::CoreFileMemoryRanges &ranges, std::set<addr_t> &stack_ends) {
+  const bool try_dirty_pages = true;
+
+  // Before we take any dump, we want to save off the used portions of the
+  // stacks and mark those memory regions as saved. This prevents us from saving
+  // the unused portion of the stack below the stack pointer. Saving space on
+  // the dump.
+  for (lldb::ThreadSP thread_sp : process.GetThreadList().Threads()) {
+    if (!thread_sp)
+      continue;
+    StackFrameSP frame_sp = thread_sp->GetStackFrameAtIndex(0);
+    if (!frame_sp)
+      continue;
+    RegisterContextSP reg_ctx_sp = frame_sp->GetRegisterContext();
+    if (!reg_ctx_sp)
+      continue;
+    const addr_t sp = reg_ctx_sp->GetSP();
+    const size_t red_zone = process.GetABI()->GetRedZoneSize();
+    lldb_private::MemoryRegionInfo sp_region;
+    if (process.GetMemoryRegionInfo(sp, sp_region).Success()) {
+      const size_t stack_head = (sp - red_zone);
+      const size_t stack_size = sp_region.GetRange().GetRangeEnd() - stack_head;
+      sp_region.GetRange().SetRangeBase(stack_head);
+      sp_region.GetRange().SetByteSize(stack_size);
+      stack_ends.insert(sp_region.GetRange().GetRangeEnd());
+      AddRegion(sp_region, try_dirty_pages, ranges);
+    }
+  }
+}
+
 // Save all memory regions that are not empty or have at least some permissions
 // for a full core file style.
 static void GetCoreFileSaveRangesFull(Process &process,
                                       const MemoryRegionInfos &regions,
-                                      Process::CoreFileMemoryRanges &ranges) {
+                                      Process::CoreFileMemoryRanges &ranges,
+                                      std::set<addr_t> &stack_ends) {
 
   // Don't add only dirty pages, add full regions.
 const bool try_dirty_pages = false;
   for (const auto &region : regions)
-    AddRegion(region, try_dirty_pages, ranges);
+    if (stack_ends.count(region.GetRange().GetRangeEnd()) == 0)
+      AddRegion(region, try_dirty_pages, ranges);
 }
 
 // Save only the dirty pages to the core file. Make sure the process has at
 // least some dirty pages, as some OS versions don't support reporting what
 // pages are dirty within an memory region. If no memory regions have dirty
 // page information fall back to saving out all ranges with write permissions.
-static void
-GetCoreFileSaveRangesDirtyOnly(Process &process,
-                               const MemoryRegionInfos &regions,
-                               Process::CoreFileMemoryRanges &ranges) {
+static void GetCoreFileSaveRangesDirtyOnly(
+    Process &process, const MemoryRegionInfos &regions,
+    Process::CoreFileMemoryRanges &ranges, std::set<addr_t> &stack_ends) {
+
   // Iterate over the regions and find all dirty pages.
   bool have_dirty_page_info = false;
   for (const auto &region : regions) {
-    if (AddDirtyPages(region, ranges))
+    if (stack_ends.count(region.GetRange().GetRangeEnd()) == 0 &&
+        AddDirtyPages(region, ranges))
       have_dirty_page_info = true;
   }
 
@@ -6367,7 +6402,8 @@ GetCoreFileSaveRangesDirtyOnly(Process &process,
     // plug-in so fall back to any region with write access permissions.
     const bool try_dirty_pages = false;
     for (const auto &region : regions)
-      if (region.GetWritable() == MemoryRegionInfo::eYes)
+      if (stack_ends.count(region.GetRange().GetRangeEnd()) == 0 &&
+          region.GetWritable() == MemoryRegionInfo::eYes)
         AddRegion(region, try_dirty_pages, ranges);
   }
 }
@@ -6380,43 +6416,18 @@ GetCoreFileSaveRangesDirtyOnly(Process &process,
 // dirty regions as this will make the core file smaller. If the process
 // doesn't support dirty regions, then it will fall back to adding the full
 // stack region.
-static void
-GetCoreFileSaveRangesStackOnly(Process &process,
-                               const MemoryRegionInfos &regions,
-                               Process::CoreFileMemoryRanges &ranges) {
+static void GetCoreFileSaveRangesStackOnly(
+    Process &process, const MemoryRegionInfos &regions,
+    Process::CoreFileMemoryRanges &ranges, std::set<addr_t> &stack_ends) {
+  const bool try_dirty_pages = true;
   // Some platforms support annotating the region information that tell us that
   // it comes from a thread stack. So look for those regions first.
 
-  // Keep track of which stack regions we have added
-  std::set<addr_t> stack_bases;
-
-  const bool try_dirty_pages = true;
   for (const auto &region : regions) {
-    if (region.IsStackMemory() == MemoryRegionInfo::eYes) {
-      stack_bases.insert(region.GetRange().GetRangeBase());
+    // Save all the stack memory ranges not associated with a stack pointer.
+    if (stack_ends.count(region.GetRange().GetRangeEnd()) == 0 &&
+        region.IsStackMemory() == MemoryRegionInfo::eYes)
       AddRegion(region, try_dirty_pages, ranges);
-    }
-  }
-
-  // Also check with our threads and get the regions for their stack pointers
-  // and add those regions if not already added above.
-  for (lldb::ThreadSP thread_sp : process.GetThreadList().Threads()) {
-    if (!thread_sp)
-      continue;
-    StackFrameSP frame_sp = thread_sp->GetStackFrameAtIndex(0);
-    if (!frame_sp)
-      continue;
-    RegisterContextSP reg_ctx_sp = frame_sp->GetRegisterContext();
-    if (!reg_ctx_sp)
-      continue;
-    const addr_t sp = reg_ctx_sp->GetSP();
-    lldb_private::MemoryRegionInfo sp_region;
-    if (process.GetMemoryRegionInfo(sp, sp_region).Success()) {
-      // Only add this region if not already added above. If our stack pointer
-      // is pointing off in the weeds, we will want this range.
-      if (stack_bases.count(sp_region.GetRange().GetRangeBase()) == 0)
-        AddRegion(sp_region, try_dirty_pages, ranges);
-    }
   }
 }
 
@@ -6428,23 +6439,27 @@ Status Process::CalculateCoreFileSaveRanges(lldb::SaveCoreStyle core_style,
     return err;
   if (regions.empty())
     return Status("failed to get any valid memory regions from the process");
+  if (core_style == eSaveCoreUnspecified)
+    return Status("callers must set the core_style to something other than "
+                  "eSaveCoreUnspecified");
+
+  std::set<addr_t> stack_ends;
+  SaveOffRegionsWithStackPointers(*this, regions, ranges, stack_ends);
 
   switch (core_style) {
   case eSaveCoreUnspecified:
-    err = Status("callers must set the core_style to something other than "
-                 "eSaveCoreUnspecified");
     break;
 
   case eSaveCoreFull:
-    GetCoreFileSaveRangesFull(*this, regions, ranges);
+    GetCoreFileSaveRangesFull(*this, regions, ranges, stack_ends);
     break;
 
   case eSaveCoreDirtyOnly:
-    GetCoreFileSaveRangesDirtyOnly(*this, regions, ranges);
+    GetCoreFileSaveRangesDirtyOnly(*this, regions, ranges, stack_ends);
     break;
 
   case eSaveCoreStackOnly:
-    GetCoreFileSaveRangesStackOnly(*this, regions, ranges);
+    GetCoreFileSaveRangesStackOnly(*this, regions, ranges, stack_ends);
     break;
   }
 
diff --git a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
index 9fe5e8914298..1e171e726fb6 100644
--- a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
+++ b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
@@ -2,7 +2,6 @@
 Test saving a mini dump.
 """
 
-
 import os
 import lldb
 from lldbsuite.test.decorators import *
@@ -12,7 +11,12 @@ from lldbsuite.test import lldbutil
 
 class ProcessSaveCoreMinidumpTestCase(TestBase):
     def verify_core_file(
-        self, core_path, expected_pid, expected_modules, expected_threads
+        self,
+        core_path,
+        expected_pid,
+        expected_modules,
+        expected_threads,
+        stacks_to_sps_map,
     ):
         # To verify, we'll launch with the mini dump
         target = self.dbg.CreateTarget(None)
@@ -36,17 +40,36 @@ class ProcessSaveCoreMinidumpTestCase(TestBase):
             self.assertEqual(module_file_name, expected_file_name)
             self.assertEqual(module.GetUUIDString(), expected.GetUUIDString())
 
+        red_zone = process.GetTarget().GetStackRedZoneSize()
         for thread_idx in range(process.GetNumThreads()):
             thread = process.GetThreadAtIndex(thread_idx)
             self.assertTrue(thread.IsValid())
             thread_id = thread.GetThreadID()
             self.assertIn(thread_id, expected_threads)
+            frame = thread.GetFrameAtIndex(0)
+            sp_region = lldb.SBMemoryRegionInfo()
+            sp = frame.GetSP()
+            err = process.GetMemoryRegionInfo(sp, sp_region)
+            self.assertTrue(err.Success(), err.GetCString())
+            error = lldb.SBError()
+            # Ensure thread_id is in the saved map
+            self.assertIn(thread_id, stacks_to_sps_map)
+            # Ensure the SP is correct
+            self.assertEqual(stacks_to_sps_map[thread_id], sp)
+            # Try to read at the end of the stack red zone and succeed
+            process.ReadMemory(sp - red_zone, 1, error)
+            self.assertTrue(error.Success(), error.GetCString())
+            # Try to read just past the red zone and fail
+            process.ReadMemory(sp - red_zone - 1, 1, error)
+            self.assertTrue(error.Fail(), "No failure when reading past the red zone")
+
         self.dbg.DeleteTarget(target)
 
     @skipUnlessArch("x86_64")
     @skipUnlessPlatform(["linux"])
     def test_save_linux_mini_dump(self):
         """Test that we can save a Linux mini dump."""
+
         self.build()
         exe = self.getBuildArtifact("a.out")
         core_stack = self.getBuildArtifact("core.stack.dmp")
@@ -69,30 +92,44 @@ class ProcessSaveCoreMinidumpTestCase(TestBase):
             expected_modules = target.modules
             expected_number_of_threads = process.GetNumThreads()
             expected_threads = []
+            stacks_to_sp_map = {}
 
             for thread_idx in range(process.GetNumThreads()):
                 thread = process.GetThreadAtIndex(thread_idx)
                 thread_id = thread.GetThreadID()
                 expected_threads.append(thread_id)
+                stacks_to_sp_map[thread_id] = thread.GetFrameAtIndex(0).GetSP()
 
             # save core and, kill process and verify corefile existence
             base_command = "process save-core --plugin-name=minidump "
             self.runCmd(base_command + " --style=stack '%s'" % (core_stack))
             self.assertTrue(os.path.isfile(core_stack))
             self.verify_core_file(
-                core_stack, expected_pid, expected_modules, expected_threads
+                core_stack,
+                expected_pid,
+                expected_modules,
+                expected_threads,
+                stacks_to_sp_map,
             )
 
             self.runCmd(base_command + " --style=modified-memory '%s'" % (core_dirty))
             self.assertTrue(os.path.isfile(core_dirty))
             self.verify_core_file(
-                core_dirty, expected_pid, expected_modules, expected_threads
+                core_dirty,
+                expected_pid,
+                expected_modules,
+                expected_threads,
+                stacks_to_sp_map,
             )
 
             self.runCmd(base_command + " --style=full '%s'" % (core_full))
             self.assertTrue(os.path.isfile(core_full))
             self.verify_core_file(
-                core_full, expected_pid, expected_modules, expected_threads
+                core_full,
+                expected_pid,
+                expected_modules,
+                expected_threads,
+                stacks_to_sp_map,
             )
 
             # validate saving via SBProcess
@@ -100,14 +137,22 @@ class ProcessSaveCoreMinidumpTestCase(TestBase):
             self.assertTrue(error.Success())
             self.assertTrue(os.path.isfile(core_sb_stack))
             self.verify_core_file(
-                core_sb_stack, expected_pid, expected_modules, expected_threads
+                core_sb_stack,
+                expected_pid,
+                expected_modules,
+                expected_threads,
+                stacks_to_sp_map,
             )
 
             error = process.SaveCore(core_sb_dirty, "minidump", lldb.eSaveCoreDirtyOnly)
             self.assertTrue(error.Success())
             self.assertTrue(os.path.isfile(core_sb_dirty))
             self.verify_core_file(
-                core_sb_dirty, expected_pid, expected_modules, expected_threads
+                core_sb_dirty,
+                expected_pid,
+                expected_modules,
+                expected_threads,
+                stacks_to_sp_map,
             )
 
             # Minidump can now save full core files, but they will be huge and
@@ -116,7 +161,11 @@ class ProcessSaveCoreMinidumpTestCase(TestBase):
             self.assertTrue(error.Success())
             self.assertTrue(os.path.isfile(core_sb_full))
             self.verify_core_file(
-                core_sb_full, expected_pid, expected_modules, expected_threads
+                core_sb_full,
+                expected_pid,
+                expected_modules,
+                expected_threads,
+                stacks_to_sp_map,
             )
 
             self.assertSuccess(process.Kill())
-- 
GitLab


From e5e562361555fc96c768b1dd3dd99f403f500838 Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Thu, 16 May 2024 14:33:39 -0700
Subject: [PATCH 1061/1206] [bazel] Port #91960 (#92458)

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 62ac56f8b46e..f7495a202669 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -2156,7 +2156,7 @@ cc_library(
         ":ArmSMEOpInterfacesIncGen",
         ":ArmSMEOpsIncGen",
         ":BytecodeOpInterface",
-	":FunctionInterfaces",
+        ":FunctionInterfaces",
         ":IR",
         ":LLVMDialect",
         ":MemRefDialect",
@@ -2173,7 +2173,7 @@ cc_library(
     hdrs = glob(["include/mlir/Dialect/ArmSME/Transforms/*.h"]),
     includes = ["include"],
     deps = [
-	":Analysis",
+        ":Analysis",
         ":ArithUtils",
         ":ArmSMEDialect",
         ":ArmSMETransformsPassIncGen",
@@ -2181,7 +2181,7 @@ cc_library(
         ":DialectUtils",
         ":FuncDialect",
         ":FuncTransforms",
-	":FunctionInterfaces",
+        ":FunctionInterfaces",
         ":IR",
         ":LLVMCommonConversion",
         ":LLVMDialect",
@@ -2220,7 +2220,7 @@ cc_library(
         ":ControlFlowDialect",
         ":ConversionPassIncGen",
         ":FuncDialect",
-	":FunctionInterfaces",
+        ":FunctionInterfaces",
         ":LLVMCommonConversion",
         ":LLVMDialect",
         ":MemRefDialect",
@@ -7487,6 +7487,7 @@ cc_library(
         ":TransformUtils",
         ":ValueBoundsOpInterface",
         ":VectorDialect",
+        ":VectorUtils",
         "//llvm:Support",
     ],
 )
-- 
GitLab


From 997eae3673dcd635b60ac63031f01f5dded92f32 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 16 May 2024 14:41:19 -0700
Subject: [PATCH 1062/1206] [AsmPrinter] Increase upper bound for size in
 global structs

This is part of the fixes to address #57353

https://reviews.llvm.org/D133845

Pull Request: https://github.com/llvm/llvm-project/pull/92334
---
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 2 +-
 llvm/test/CodeGen/X86/big-array-init.ll    | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/X86/big-array-init.ll

diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 869670d43a17..d50cdc4323ec 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -3501,7 +3501,7 @@ static void emitGlobalConstantStruct(const DataLayout &DL,
                                      const Constant *BaseCV, uint64_t Offset,
                                      AsmPrinter::AliasMapTy *AliasList) {
   // Print the fields in successive locations. Pad to align if needed!
-  unsigned Size = DL.getTypeAllocSize(CS->getType());
+  uint64_t Size = DL.getTypeAllocSize(CS->getType());
   const StructLayout *Layout = DL.getStructLayout(CS->getType());
   uint64_t SizeSoFar = 0;
   for (unsigned I = 0, E = CS->getNumOperands(); I != E; ++I) {
diff --git a/llvm/test/CodeGen/X86/big-array-init.ll b/llvm/test/CodeGen/X86/big-array-init.ll
new file mode 100644
index 000000000000..6b05f901ebea
--- /dev/null
+++ b/llvm/test/CodeGen/X86/big-array-init.ll
@@ -0,0 +1,7 @@
+; RUN: llc < %s -mtriple=x86_64 | FileCheck %s
+
+; CHECK-LABEL: bad_int:
+; CHECK-NEXT: .long 1
+; CHECK-NEXT: .zero 4294967292
+; CHECK-NEXT: .size   bad_int, 4294967296
+@bad_int = global <{ i32, [1073741823 x i32] }> <{ i32 1, [1073741823 x i32] zeroinitializer }>, align 16
-- 
GitLab


From c86a53d75995d65601d5745b2fceb2d50a970645 Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Thu, 16 May 2024 18:44:41 -0300
Subject: [PATCH 1063/1206] [clang] Implement provisional wording for CWG2398
 regarding packs (#90820)

This solves some ambuguity introduced in P0522 regarding how
template template parameters are partially ordered, and should reduce
the negative impact of enabling `-frelaxed-template-template-args`
by default.

When performing template argument deduction, a template template
parameter
containing no packs should be more specialized than one that does.

Given the following example:
```C++
template<class T2> struct A;
template<template<class ...T3s> class TT1, class T4> struct A<TT1<T4>>; // #1
template<template<class    T5 > class TT2, class T6> struct A<TT2<T6>>; // #2

template<class T1> struct B;
template struct A<B<char>>;
```

Prior to P0522, candidate `#2` would be more specialized.
After P0522, neither is more specialized, so this becomes ambiguous.
With this change, `#2` becomes more specialized again,
maintaining compatibility with pre-P0522 implementations.

The problem is that in P0522, candidates are at least as specialized
when matching packs to fixed-size lists both ways, whereas before,
a fixed-size list is more specialized.

This patch keeps the original behavior when checking template arguments
outside deduction, but restores this aspect of pre-P0522 matching
during deduction.

---

Since this changes provisional implementation of CWG2398 which has
not been released yet, and already contains a changelog entry,
we don't provide a changelog entry here.
---
 clang/include/clang/Sema/Sema.h          |  5 +-
 clang/lib/Sema/SemaTemplate.cpp          | 10 ++--
 clang/lib/Sema/SemaTemplateDeduction.cpp | 67 +++++++++++++++++++-----
 clang/test/SemaTemplate/cwg2398.cpp      | 15 ++----
 4 files changed, 65 insertions(+), 32 deletions(-)

diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 66d5e2d4a4ad..873592a2d430 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -9133,7 +9133,7 @@ public:
                                    CheckTemplateArgumentKind CTAK);
   bool CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param,
                                      TemplateParameterList *Params,
-                                     TemplateArgumentLoc &Arg);
+                                     TemplateArgumentLoc &Arg, bool IsDeduced);
 
   void NoteTemplateLocation(const NamedDecl &Decl,
                             std::optional<SourceRange> ParamRange = {});
@@ -9612,7 +9612,8 @@ public:
                                     sema::TemplateDeductionInfo &Info);
 
   bool isTemplateTemplateParameterAtLeastAsSpecializedAs(
-      TemplateParameterList *PParam, TemplateDecl *AArg, SourceLocation Loc);
+      TemplateParameterList *PParam, TemplateDecl *AArg, SourceLocation Loc,
+      bool IsDeduced);
 
   void MarkUsedTemplateParameters(const Expr *E, bool OnlyDeduced,
                                   unsigned Depth, llvm::SmallBitVector &Used);
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index a5350ceb59cb..1b3f1e8bf80f 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -6554,7 +6554,8 @@ bool Sema::CheckTemplateArgument(
 
   case TemplateArgument::Template:
   case TemplateArgument::TemplateExpansion:
-    if (CheckTemplateTemplateArgument(TempParm, Params, Arg))
+    if (CheckTemplateTemplateArgument(TempParm, Params, Arg,
+                                      /*IsDeduced=*/CTAK != CTAK_Specified))
       return true;
 
     SugaredConverted.push_back(Arg.getArgument());
@@ -8472,7 +8473,8 @@ static void DiagnoseTemplateParameterListArityMismatch(
 /// It returns true if an error occurred, and false otherwise.
 bool Sema::CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param,
                                          TemplateParameterList *Params,
-                                         TemplateArgumentLoc &Arg) {
+                                         TemplateArgumentLoc &Arg,
+                                         bool IsDeduced) {
   TemplateName Name = Arg.getArgument().getAsTemplateOrTemplatePattern();
   TemplateDecl *Template = Name.getAsTemplateDecl();
   if (!Template) {
@@ -8524,8 +8526,8 @@ bool Sema::CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param,
         !Template->hasAssociatedConstraints())
       return false;
 
-    if (isTemplateTemplateParameterAtLeastAsSpecializedAs(Params, Template,
-                                                          Arg.getLocation())) {
+    if (isTemplateTemplateParameterAtLeastAsSpecializedAs(
+            Params, Template, Arg.getLocation(), IsDeduced)) {
       // P2113
       // C++20[temp.func.order]p2
       //   [...] If both deductions succeed, the partial ordering selects the
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 4d89fcfc9b11..d0b780c28eae 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -139,13 +139,15 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
     SmallVectorImpl<DeducedTemplateArgument> &Deduced, unsigned TDF,
     bool PartialOrdering = false, bool DeducedFromArrayBound = false);
 
+enum class PackFold { ParameterToArgument, ArgumentToParameter };
 static TemplateDeductionResult
 DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
                         ArrayRef<TemplateArgument> Ps,
                         ArrayRef<TemplateArgument> As,
                         TemplateDeductionInfo &Info,
                         SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-                        bool NumberOfArgumentsMustMatch);
+                        bool NumberOfArgumentsMustMatch,
+                        PackFold PackFold = PackFold::ParameterToArgument);
 
 static void MarkUsedTemplateParameters(ASTContext &Ctx,
                                        const TemplateArgument &TemplateArg,
@@ -2550,7 +2552,9 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
                         ArrayRef<TemplateArgument> As,
                         TemplateDeductionInfo &Info,
                         SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-                        bool NumberOfArgumentsMustMatch) {
+                        bool NumberOfArgumentsMustMatch, PackFold PackFold) {
+  if (PackFold == PackFold::ArgumentToParameter)
+    std::swap(Ps, As);
   // C++0x [temp.deduct.type]p9:
   //   If the template argument list of P contains a pack expansion that is not
   //   the last template argument, the entire template argument list is a
@@ -2581,8 +2585,11 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
         return TemplateDeductionResult::MiscellaneousDeductionFailure;
 
       // Perform deduction for this Pi/Ai pair.
-      if (auto Result = DeduceTemplateArguments(S, TemplateParams, P,
-                                                As[ArgIdx], Info, Deduced);
+      TemplateArgument Pi = P, Ai = As[ArgIdx];
+      if (PackFold == PackFold::ArgumentToParameter)
+        std::swap(Pi, Ai);
+      if (auto Result =
+              DeduceTemplateArguments(S, TemplateParams, Pi, Ai, Info, Deduced);
           Result != TemplateDeductionResult::Success)
         return Result;
 
@@ -2609,9 +2616,12 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
     for (; hasTemplateArgumentForDeduction(As, ArgIdx) &&
            PackScope.hasNextElement();
          ++ArgIdx) {
+      TemplateArgument Pi = Pattern, Ai = As[ArgIdx];
+      if (PackFold == PackFold::ArgumentToParameter)
+        std::swap(Pi, Ai);
       // Deduce template arguments from the pattern.
-      if (auto Result = DeduceTemplateArguments(S, TemplateParams, Pattern,
-                                                As[ArgIdx], Info, Deduced);
+      if (auto Result =
+              DeduceTemplateArguments(S, TemplateParams, Pi, Ai, Info, Deduced);
           Result != TemplateDeductionResult::Success)
         return Result;
 
@@ -6299,7 +6309,8 @@ bool Sema::isMoreSpecializedThanPrimary(
 }
 
 bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
-     TemplateParameterList *P, TemplateDecl *AArg, SourceLocation Loc) {
+    TemplateParameterList *P, TemplateDecl *AArg, SourceLocation Loc,
+    bool IsDeduced) {
   // C++1z [temp.arg.template]p4: (DR 150)
   //   A template template-parameter P is at least as specialized as a
   //   template template-argument A if, given the following rewrite to two
@@ -6309,11 +6320,10 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
   // equivalent partial ordering by performing deduction directly on
   // the template parameter lists of the template template parameters.
   //
-  //   Given an invented class template X with the template parameter list of
-  //   A (including default arguments):
-  TemplateName X = Context.getCanonicalTemplateName(TemplateName(AArg));
   TemplateParameterList *A = AArg->getTemplateParameters();
 
+  //   Given an invented class template X with the template parameter list of
+  //   A (including default arguments):
   //    - Each function template has a single function parameter whose type is
   //      a specialization of X with template arguments corresponding to the
   //      template parameters from the respective function template
@@ -6356,14 +6366,43 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
       return false;
   }
 
-  QualType AType = Context.getCanonicalTemplateSpecializationType(X, AArgs);
-  QualType PType = Context.getCanonicalTemplateSpecializationType(X, PArgs);
+  // Determine whether P1 is at least as specialized as P2.
+  TemplateDeductionInfo Info(Loc, A->getDepth());
+  SmallVector<DeducedTemplateArgument, 4> Deduced;
+  Deduced.resize(A->size());
 
   //   ... the function template corresponding to P is at least as specialized
   //   as the function template corresponding to A according to the partial
   //   ordering rules for function templates.
-  TemplateDeductionInfo Info(Loc, A->getDepth());
-  return isAtLeastAsSpecializedAs(*this, PType, AType, AArg, Info);
+
+  // Provisional resolution for CWG2398: Regarding temp.arg.template]p4, when
+  // applying the partial ordering rules for function templates on
+  // the rewritten template template parameters:
+  //   - In a deduced context, the matching of packs versus fixed-size needs to
+  //   be inverted between Ps and As. On non-deduced context, matching needs to
+  //   happen both ways, according to [temp.arg.template]p3, but this is
+  //   currently implemented as a special case elsewhere.
+  if (::DeduceTemplateArguments(*this, A, AArgs, PArgs, Info, Deduced,
+                                /*NumberOfArgumentsMustMatch=*/false,
+                                IsDeduced ? PackFold::ArgumentToParameter
+                                          : PackFold::ParameterToArgument) !=
+      TemplateDeductionResult::Success)
+    return false;
+
+  SmallVector<TemplateArgument, 4> DeducedArgs(Deduced.begin(), Deduced.end());
+  Sema::InstantiatingTemplate Inst(*this, Info.getLocation(), AArg, DeducedArgs,
+                                   Info);
+  if (Inst.isInvalid())
+    return false;
+
+  bool AtLeastAsSpecialized;
+  runWithSufficientStackSpace(Info.getLocation(), [&] {
+    AtLeastAsSpecialized =
+        ::FinishTemplateArgumentDeduction(
+            *this, AArg, /*IsPartialOrdering=*/true, PArgs, Deduced, Info) ==
+        TemplateDeductionResult::Success;
+  });
+  return AtLeastAsSpecialized;
 }
 
 namespace {
diff --git a/clang/test/SemaTemplate/cwg2398.cpp b/clang/test/SemaTemplate/cwg2398.cpp
index d163354b2e5f..31686c4bc980 100644
--- a/clang/test/SemaTemplate/cwg2398.cpp
+++ b/clang/test/SemaTemplate/cwg2398.cpp
@@ -62,25 +62,19 @@ namespace templ {
 namespace type_pack1 {
   template<class T2> struct A;
   template<template<class ...T3s> class TT1, class T4> struct A<TT1<T4>>   ;
-  // new-note@-1 {{partial specialization matches}}
   template<template<class    T5 > class TT2, class T6> struct A<TT2<T6>> {};
-  // new-note@-1 {{partial specialization matches}}
 
   template<class T1> struct B;
   template struct A<B<char>>;
-  // new-error@-1 {{ambiguous partial specialization}}
 } // namespace type_pack1
 
 namespace type_pack2 {
   template<class T2> struct A;
   template<template<class ...T3s> class TT1, class ...T4> struct A<TT1<T4...>>   ;
-  // new-note@-1 {{partial specialization matches}}
   template<template<class    T5 > class TT2, class ...T6> struct A<TT2<T6...>> {};
-  // new-note@-1 {{partial specialization matches}}
 
   template<class T1> struct B;
   template struct A<B<char>>;
-  // new-error@-1 {{ambiguous partial specialization}}
 } // namespace type_pack2
 
 namespace type_pack3 {
@@ -140,13 +134,10 @@ namespace ttp_defaults {
 
 namespace ttp_only {
   template <template <class...    > class TT1> struct A      { static constexpr int V = 0; };
-  // new-note@-1 2{{template is declared here}}
   template <template <class       > class TT2> struct A<TT2> { static constexpr int V = 1; };
-  // new-error@-1 {{not more specialized than the primary template}}
-  // new-note@-2 {{partial specialization matches}}
+  // new-note@-1 {{partial specialization matches}}
   template <template <class, class> class TT3> struct A<TT3> { static constexpr int V = 2; };
-  // new-error@-1 {{not more specialized than the primary template}}
-  // new-note@-2 {{partial specialization matches}}
+  // new-note@-1 {{partial specialization matches}}
 
   template <class ...          > struct B;
   template <class              > struct C;
@@ -193,5 +184,5 @@ namespace consistency {
 
     template struct A<B<int>, B<int>, B<int>>;
     // new-error@-1 {{ambiguous partial specializations}}
-  } // namespace t1
+  } // namespace t2
 } // namespace consistency
-- 
GitLab


From c79690040acf5bb3d857558b0878db47f7f23dc3 Mon Sep 17 00:00:00 2001
From: DianQK <dianqk@dianqk.net>
Date: Fri, 17 May 2024 05:51:49 +0800
Subject: [PATCH 1064/1206] [GlobalOpt] Don't replace aliasee with alias that
 has weak linkage (#91483)

Fixes #91312.

Don't perform the transform if the alias may be replaced at link time.
---
 llvm/lib/Transforms/IPO/GlobalOpt.cpp        |  3 ++
 llvm/test/Transforms/GlobalOpt/alias-weak.ll | 57 ++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 llvm/test/Transforms/GlobalOpt/alias-weak.ll

diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index c8c835115a99..b7c6d25657bc 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -2224,6 +2224,9 @@ static bool mayHaveOtherReferences(GlobalValue &GV, const LLVMUsed &U) {
 
 static bool hasUsesToReplace(GlobalAlias &GA, const LLVMUsed &U,
                              bool &RenameTarget) {
+  if (GA.isWeakForLinker())
+    return false;
+
   RenameTarget = false;
   bool Ret = false;
   if (hasUseOtherThanLLVMUsed(GA, U))
diff --git a/llvm/test/Transforms/GlobalOpt/alias-weak.ll b/llvm/test/Transforms/GlobalOpt/alias-weak.ll
new file mode 100644
index 000000000000..aec2a56313b1
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/alias-weak.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 4
+; RUN: opt < %s -passes=globalopt -S | FileCheck %s
+
+@f1_alias = linkonce_odr hidden alias void (), ptr @f1
+@f2_alias = linkonce_odr hidden alias void (), ptr @f2
+
+define void @foo() {
+  call void @f1_alias()
+  ret void
+}
+
+define void @bar() {
+  call void @f1()
+  ret void
+}
+
+define void @baz() {
+  call void @f2_alias()
+  ret void
+}
+
+; We cannot use `f1_alias` to replace `f1` because they are both in use
+; and `f1_alias` could be replaced at link time.
+define internal void @f1() {
+  ret void
+}
+
+; FIXME: We can use `f2_alias` to replace `f2` because `b2` is not in use.
+define internal void @f2() {
+  ret void
+}
+;.
+; CHECK: @f1_alias = linkonce_odr hidden alias void (), ptr @f1
+; CHECK: @f2_alias = linkonce_odr hidden alias void (), ptr @f2
+;.
+; CHECK-LABEL: define void @foo() local_unnamed_addr {
+; CHECK-NEXT:    call void @f1_alias()
+; CHECK-NEXT:    ret void
+;
+;
+; CHECK-LABEL: define void @bar() local_unnamed_addr {
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    ret void
+;
+;
+; CHECK-LABEL: define void @baz() local_unnamed_addr {
+; CHECK-NEXT:    call void @f2_alias()
+; CHECK-NEXT:    ret void
+;
+;
+; CHECK-LABEL: define internal void @f1() {
+; CHECK-NEXT:    ret void
+;
+;
+; CHECK-LABEL: define internal void @f2() {
+; CHECK-NEXT:    ret void
+;
-- 
GitLab


From e578314c049bb9ae6dc3983db5cf27513e29517b Mon Sep 17 00:00:00 2001
From: Ryan Thomas Lynch <rlynch34@gatech.edu>
Date: Thu, 16 May 2024 15:20:44 -0700
Subject: [PATCH 1065/1206] [MLIR Attr] add ArrayMaxCount attribute constraint
 (#92453)

this is the dual of ArrayMinCount. I saw that I needed it but it didn't
exist yet
---
 mlir/docs/DefiningDialects/Operations.md      | 48 +++++++++++++++++++
 mlir/include/mlir/IR/CommonAttrConstraints.td |  4 ++
 2 files changed, 52 insertions(+)

diff --git a/mlir/docs/DefiningDialects/Operations.md b/mlir/docs/DefiningDialects/Operations.md
index 706abce55ce9..01fadef5c3db 100644
--- a/mlir/docs/DefiningDialects/Operations.md
+++ b/mlir/docs/DefiningDialects/Operations.md
@@ -291,8 +291,27 @@ Right now, the following primitive constraints are supported:
     equal to `N`
 *   `IntMaxValue<N>`: Specifying an integer attribute to be less than or equal
     to `N`
+*   `IntNEQValue<N>`: Specifying an integer attribute to be not equal
+    to `N`
+*   `IntPositive`: Specifying an integer attribute whose value is positive
+*   `IntNonNegative`: Specifying an integer attribute whose value is
+    non-negative
 *   `ArrayMinCount<N>`: Specifying an array attribute to have at least `N`
     elements
+*   `ArrayMaxCount<N>`: Specifying an array attribute to have at most `N`
+    elements
+*   `ArrayCount<N>`: Specifying an array attribute to have exactly `N`
+    elements
+*   `DenseArrayCount<N>`: Specifying a dense array attribute to have
+    exactly `N` elements
+*   `DenseArrayStrictlyPositive<arrayType>`: Specifying a dense array attribute
+    of type `arrayType` to have all positive elements
+*   `DenseArrayStrictlyNonNegative<arrayType>`: Specifying a dense array attribute
+    of type `arrayType` to have all non-negative elements
+*   `DenseArraySorted<arrayType>`: Specifying a dense array attribute
+    of type `arrayType` to have elements in non-decreasing order
+*   `DenseArrayStrictlySorted<arrayType>`: Specifying a dense array attribute
+    of type `arrayType` to have elements in increasing order
 *   `IntArrayNthElemEq<I, N>`: Specifying an integer array attribute's `I`-th
     element to be equal to `N`
 *   `IntArrayNthElemMinValue<I, N>`: Specifying an integer array attribute's
@@ -301,9 +320,36 @@ Right now, the following primitive constraints are supported:
     `I`-th element to be less than or equal to `N`
 *   `IntArrayNthElemInRange<I, M, N>`: Specifying an integer array attribute's
     `I`-th element to be greater than or equal to `M` and less than or equal to `N`
+*   `IsNullAttr`: Specifying an optional attribute which must be empty
 
 TODO: Design and implement more primitive constraints
 
+#### Combining constraints
+
+`AllAttrOf` is provided to allow combination of multiple constraints which
+must all hold.
+
+For example:
+```tablegen
+def OpAllAttrConstraint1 : TEST_Op<"all_attr_constraint_of1"> {
+  let arguments = (ins I64ArrayAttr:$attr);
+  let results = (outs I32);
+}
+def OpAllAttrConstraint2 : TEST_Op<"all_attr_constraint_of2"> {
+  let arguments = (ins I64ArrayAttr:$attr);
+  let results = (outs I32);
+}
+def Constraint0 : AttrConstraint<
+    CPred<"::llvm::cast<::mlir::IntegerAttr>(::llvm::cast<ArrayAttr>($_self)[0]).getInt() == 0">,
+    "[0] == 0">;
+def Constraint1 : AttrConstraint<
+    CPred<"::llvm::cast<::mlir::IntegerAttr>(::llvm::cast<ArrayAttr>($_self)[1]).getInt() == 1">,
+    "[1] == 1">;
+def : Pat<(OpAllAttrConstraint1
+            AllAttrOf<[Constraint0, Constraint1]>:$attr),
+          (OpAllAttrConstraint2 $attr)>;
+```
+
 ### Operation regions
 
 The regions of an operation are specified inside of the `dag`-typed `regions`,
@@ -1398,6 +1444,8 @@ optionality, default values, etc.:
 *   `OptionalAttr`: specifies an attribute as [optional](#optional-attributes).
 *   `ConfinedAttr`: adapts an attribute with
     [further constraints](#confining-attributes).
+*   `AllAttrOf`: adapts an attribute with
+    [multiple constraints](#combining-constraints).
 
 ### Enum attributes
 
diff --git a/mlir/include/mlir/IR/CommonAttrConstraints.td b/mlir/include/mlir/IR/CommonAttrConstraints.td
index 0d69bb0717a5..d99bde1f87ef 100644
--- a/mlir/include/mlir/IR/CommonAttrConstraints.td
+++ b/mlir/include/mlir/IR/CommonAttrConstraints.td
@@ -773,6 +773,10 @@ def IntPositive : AttrConstraint<
     CPred<"::llvm::cast<::mlir::IntegerAttr>($_self).getValue().isStrictlyPositive()">,
     "whose value is positive">;
 
+class ArrayMaxCount<int n> : AttrConstraint<
+    CPred<"::llvm::cast<::mlir::ArrayAttr>($_self).size() <= " # n>,
+    "with at most " # n # " elements">;
+
 class ArrayMinCount<int n> : AttrConstraint<
     CPred<"::llvm::cast<::mlir::ArrayAttr>($_self).size() >= " # n>,
     "with at least " # n # " elements">;
-- 
GitLab


From 84abe0a6d4face73c15ed6344be74ae231f18718 Mon Sep 17 00:00:00 2001
From: Dan Liew <delcypher@gmail.com>
Date: Thu, 16 May 2024 15:24:11 -0700
Subject: [PATCH 1066/1206] [Bounds-Safety] Reserve slot in SanitizerHandler
 enum for Bounds-Safety (#91032)

Due to how `CodeGenFunction::EmitTrapCheck` is implemented
`SanitizerHandler` with numeric value 0x19 needs to be reserved because
`-fbounds-safety` generates trap instructions with that value embedded
in the trap instructions for x86_64 and arm64 just like for UBSan traps.

** x86_64 **

```
ud1l   0x19(%eax), %eax
```

** arm64 **

```
brk    #0x5519
```

To avoid upstream Clang and AppleClang diverging their ABIs for
`-fbounds-safety` the slot is being reserved in this patch.

`SanitizerHandler::BoundsSafety` currently has no uses in the code but
uses will be introduced when the CodeGen side of `-fbounds-safety`'s
implementation is upstreamed.

rdar://126884014

Co-authored-by: Dan Liew <dan@su-root.co.uk>
---
 clang/lib/CodeGen/CodeGenFunction.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 362f4a5fe72a..5f3ee7eb943f 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -137,7 +137,8 @@ enum TypeEvaluationKind {
   SANITIZER_CHECK(SubOverflow, sub_overflow, 0)                                \
   SANITIZER_CHECK(TypeMismatch, type_mismatch, 1)                              \
   SANITIZER_CHECK(AlignmentAssumption, alignment_assumption, 0)                \
-  SANITIZER_CHECK(VLABoundNotPositive, vla_bound_not_positive, 0)
+  SANITIZER_CHECK(VLABoundNotPositive, vla_bound_not_positive, 0)              \
+  SANITIZER_CHECK(BoundsSafety, bounds_safety, 0)
 
 enum SanitizerHandler {
 #define SANITIZER_CHECK(Enum, Name, Version) Enum,
-- 
GitLab


From d6f9278ae9e587d2d23a9940a2364aaafba74735 Mon Sep 17 00:00:00 2001
From: James Y Knight <jyknight@google.com>
Date: Thu, 16 May 2024 18:24:23 -0400
Subject: [PATCH 1067/1206] [X86] Use plain load/store instead of cmpxchg16b
 for atomics with AVX (#74275)

In late 2021, both Intel and AMD finally documented that every
AVX-capable CPU has always been guaranteed to execute aligned 16-byte
loads/stores atomically, and further, guaranteed that all future CPUs
with AVX will do so as well.

Therefore, we may use normal SSE 128-bit load/store instructions to
implement atomics, if AVX is enabled.

Per AMD64 Architecture Programmer's manual, 7.3.2 Access Atomicity:

> Processors that report [AVX] extend the atomicity for cacheable,
> naturally-aligned single loads or stores from a quadword to a double
> quadword.

Per Intel's SDM:

> Processors that enumerate support for Intel(R) AVX guarantee that the
> 16-byte memory operations performed by the following instructions will
> always be carried out atomically:
> - MOVAPD, MOVAPS, and MOVDQA.
> - VMOVAPD, VMOVAPS, and VMOVDQA when encoded with VEX.128.
> - VMOVAPD, VMOVAPS, VMOVDQA32, and VMOVDQA64 when encoded with
>   EVEX.128 and k0 (masking disabled).

This was also confirmed to be true for Zhaoxin CPUs with AVX, in
https://gcc.gnu.org/PR104688
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 100 +++++---
 .../CodeGen/X86/atomic-non-integer-fp128.ll   |  31 +--
 llvm/test/CodeGen/X86/atomic-non-integer.ll   |   1 -
 llvm/test/CodeGen/X86/atomic-unaligned.ll     |  33 +++
 llvm/test/CodeGen/X86/atomic-unordered.ll     |  83 ++-----
 llvm/test/CodeGen/X86/atomic128.ll            | 224 +++++++++++-------
 llvm/test/CodeGen/X86/cmpxchg-i128-i1.ll      |   8 +-
 7 files changed, 259 insertions(+), 221 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/atomic-unaligned.ll

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e7c70e3872ad..cd252c54887a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -544,6 +544,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   if (!Subtarget.is64Bit())
     setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom);
 
+  if (Subtarget.is64Bit() && Subtarget.hasAVX()) {
+    // All CPUs supporting AVX will atomically load/store aligned 128-bit
+    // values, so we can emit [V]MOVAPS/[V]MOVDQA.
+    setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
+    setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
+  }
+
   if (Subtarget.canUseCMPXCHG16B())
     setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
 
@@ -30415,32 +30422,40 @@ TargetLoweringBase::AtomicExpansionKind
 X86TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
   Type *MemType = SI->getValueOperand()->getType();
 
-  bool NoImplicitFloatOps =
-      SI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat);
-  if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
-      !Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
-      (Subtarget.hasSSE1() || Subtarget.hasX87()))
-    return AtomicExpansionKind::None;
+  if (!SI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat) &&
+      !Subtarget.useSoftFloat()) {
+    if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
+        (Subtarget.hasSSE1() || Subtarget.hasX87()))
+      return AtomicExpansionKind::None;
+
+    if (MemType->getPrimitiveSizeInBits() == 128 && Subtarget.is64Bit() &&
+        Subtarget.hasAVX())
+      return AtomicExpansionKind::None;
+  }
 
   return needsCmpXchgNb(MemType) ? AtomicExpansionKind::Expand
                                  : AtomicExpansionKind::None;
 }
 
 // Note: this turns large loads into lock cmpxchg8b/16b.
-// TODO: In 32-bit mode, use MOVLPS when SSE1 is available?
 TargetLowering::AtomicExpansionKind
 X86TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
   Type *MemType = LI->getType();
 
-  // If this a 64 bit atomic load on a 32-bit target and SSE2 is enabled, we
-  // can use movq to do the load. If we have X87 we can load into an 80-bit
-  // X87 register and store it to a stack temporary.
-  bool NoImplicitFloatOps =
-      LI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat);
-  if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
-      !Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
-      (Subtarget.hasSSE1() || Subtarget.hasX87()))
-    return AtomicExpansionKind::None;
+  if (!LI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat) &&
+      !Subtarget.useSoftFloat()) {
+    // If this a 64 bit atomic load on a 32-bit target and SSE2 is enabled, we
+    // can use movq to do the load. If we have X87 we can load into an 80-bit
+    // X87 register and store it to a stack temporary.
+    if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
+        (Subtarget.hasSSE1() || Subtarget.hasX87()))
+      return AtomicExpansionKind::None;
+
+    // If this is a 128-bit load with AVX, 128-bit SSE loads/stores are atomic.
+    if (MemType->getPrimitiveSizeInBits() == 128 && Subtarget.is64Bit() &&
+        Subtarget.hasAVX())
+      return AtomicExpansionKind::None;
+  }
 
   return needsCmpXchgNb(MemType) ? AtomicExpansionKind::CmpXChg
                                  : AtomicExpansionKind::None;
@@ -31683,14 +31698,21 @@ static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
   if (!IsSeqCst && IsTypeLegal)
     return Op;
 
-  if (VT == MVT::i64 && !IsTypeLegal) {
+  if (!IsTypeLegal && !Subtarget.useSoftFloat() &&
+      !DAG.getMachineFunction().getFunction().hasFnAttribute(
+          Attribute::NoImplicitFloat)) {
+    SDValue Chain;
+    // For illegal i128 atomic_store, when AVX is enabled, we can simply emit a
+    // vector store.
+    if (VT == MVT::i128 && Subtarget.is64Bit() && Subtarget.hasAVX()) {
+      SDValue VecVal = DAG.getBitcast(MVT::v2i64, Node->getVal());
+      Chain = DAG.getStore(Node->getChain(), dl, VecVal, Node->getBasePtr(),
+                           Node->getMemOperand());
+    }
+
     // For illegal i64 atomic_stores, we can try to use MOVQ or MOVLPS if SSE
     // is enabled.
-    bool NoImplicitFloatOps =
-        DAG.getMachineFunction().getFunction().hasFnAttribute(
-            Attribute::NoImplicitFloat);
-    if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) {
-      SDValue Chain;
+    if (VT == MVT::i64) {
       if (Subtarget.hasSSE1()) {
         SDValue SclToVec =
             DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Node->getVal());
@@ -31722,15 +31744,15 @@ static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
             DAG.getMemIntrinsicNode(X86ISD::FIST, dl, DAG.getVTList(MVT::Other),
                                     StoreOps, MVT::i64, Node->getMemOperand());
       }
+    }
 
-      if (Chain) {
-        // If this is a sequentially consistent store, also emit an appropriate
-        // barrier.
-        if (IsSeqCst)
-          Chain = emitLockedStackOp(DAG, Subtarget, Chain, dl);
+    if (Chain) {
+      // If this is a sequentially consistent store, also emit an appropriate
+      // barrier.
+      if (IsSeqCst)
+        Chain = emitLockedStackOp(DAG, Subtarget, Chain, dl);
 
-        return Chain;
-      }
+      return Chain;
     }
   }
 
@@ -33303,12 +33325,30 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     return;
   }
   case ISD::ATOMIC_LOAD: {
-    assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");
+    assert(
+        (N->getValueType(0) == MVT::i64 || N->getValueType(0) == MVT::i128) &&
+        "Unexpected VT!");
     bool NoImplicitFloatOps =
         DAG.getMachineFunction().getFunction().hasFnAttribute(
             Attribute::NoImplicitFloat);
     if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) {
       auto *Node = cast<AtomicSDNode>(N);
+
+      if (N->getValueType(0) == MVT::i128) {
+        if (Subtarget.is64Bit() && Subtarget.hasAVX()) {
+          SDValue Ld = DAG.getLoad(MVT::v2i64, dl, Node->getChain(),
+                                   Node->getBasePtr(), Node->getMemOperand());
+          SDValue ResL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
+                                     DAG.getIntPtrConstant(0, dl));
+          SDValue ResH = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
+                                     DAG.getIntPtrConstant(1, dl));
+          Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, N->getValueType(0),
+                                        {ResL, ResH}));
+          Results.push_back(Ld.getValue(1));
+          return;
+        }
+        break;
+      }
       if (Subtarget.hasSSE1()) {
         // Use a VZEXT_LOAD which will be selected as MOVQ or XORPS+MOVLPS.
         // Then extract the lower 64-bits.
diff --git a/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll b/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll
index 9555c45086d6..5960787fe30d 100644
--- a/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll
+++ b/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll
@@ -28,22 +28,7 @@ define void @store_fp128(ptr %fptr, fp128 %v) {
 ;
 ; X64-AVX-LABEL: store_fp128:
 ; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    pushq %rbx
-; X64-AVX-NEXT:    .cfi_def_cfa_offset 16
-; X64-AVX-NEXT:    .cfi_offset %rbx, -16
-; X64-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-AVX-NEXT:    movq -{{[0-9]+}}(%rsp), %rbx
-; X64-AVX-NEXT:    movq -{{[0-9]+}}(%rsp), %rcx
-; X64-AVX-NEXT:    movq (%rdi), %rax
-; X64-AVX-NEXT:    movq 8(%rdi), %rdx
-; X64-AVX-NEXT:    .p2align 4, 0x90
-; X64-AVX-NEXT:  .LBB0_1: # %atomicrmw.start
-; X64-AVX-NEXT:    # =>This Inner Loop Header: Depth=1
-; X64-AVX-NEXT:    lock cmpxchg16b (%rdi)
-; X64-AVX-NEXT:    jne .LBB0_1
-; X64-AVX-NEXT:  # %bb.2: # %atomicrmw.end
-; X64-AVX-NEXT:    popq %rbx
-; X64-AVX-NEXT:    .cfi_def_cfa_offset 8
+; X64-AVX-NEXT:    vmovaps %xmm0, (%rdi)
 ; X64-AVX-NEXT:    retq
   store atomic fp128 %v, ptr %fptr unordered, align 16
   ret void
@@ -69,19 +54,9 @@ define fp128 @load_fp128(ptr %fptr) {
 ;
 ; X64-AVX-LABEL: load_fp128:
 ; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    pushq %rbx
-; X64-AVX-NEXT:    .cfi_def_cfa_offset 16
-; X64-AVX-NEXT:    .cfi_offset %rbx, -16
-; X64-AVX-NEXT:    xorl %eax, %eax
-; X64-AVX-NEXT:    xorl %edx, %edx
-; X64-AVX-NEXT:    xorl %ecx, %ecx
-; X64-AVX-NEXT:    xorl %ebx, %ebx
-; X64-AVX-NEXT:    lock cmpxchg16b (%rdi)
-; X64-AVX-NEXT:    movq %rdx, -{{[0-9]+}}(%rsp)
-; X64-AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT:    vmovaps (%rdi), %xmm0
+; X64-AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; X64-AVX-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
-; X64-AVX-NEXT:    popq %rbx
-; X64-AVX-NEXT:    .cfi_def_cfa_offset 8
 ; X64-AVX-NEXT:    retq
   %v = load atomic fp128, ptr %fptr unordered, align 16
   ret fp128 %v
diff --git a/llvm/test/CodeGen/X86/atomic-non-integer.ll b/llvm/test/CodeGen/X86/atomic-non-integer.ll
index 9e6f584d8311..9995e7d3a4d3 100644
--- a/llvm/test/CodeGen/X86/atomic-non-integer.ll
+++ b/llvm/test/CodeGen/X86/atomic-non-integer.ll
@@ -131,7 +131,6 @@ define void @store_double(ptr %fptr, double %v) {
   ret void
 }
 
-
 define half @load_half(ptr %fptr) {
 ; X86-SSE1-LABEL: load_half:
 ; X86-SSE1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/atomic-unaligned.ll b/llvm/test/CodeGen/X86/atomic-unaligned.ll
new file mode 100644
index 000000000000..f02041cc5fc8
--- /dev/null
+++ b/llvm/test/CodeGen/X86/atomic-unaligned.ll
@@ -0,0 +1,33 @@
+; RUN: llc -mtriple=x86_64 < %s | FileCheck %s
+
+; Quick test to ensure that atomics which are not naturally-aligned
+; emit unsized libcalls, and aren't emitted as native instructions or
+; sized libcalls.
+define void @test_i32(ptr %a) nounwind {
+; CHECK-LABEL: test_i32:
+; CHECK: callq __atomic_load
+; CHECK: callq __atomic_store
+; CHECK: callq __atomic_exchange
+; CHECK: callq __atomic_compare_exchange
+; CHECK: callq __atomic_compare_exchange
+  %t0 = load atomic i32, ptr %a seq_cst, align 2
+  store atomic i32 1, ptr %a seq_cst, align 2
+  %t1 = atomicrmw xchg ptr %a, i32 1 seq_cst, align 2
+  %t3 = atomicrmw add ptr %a, i32 2 seq_cst, align 2
+  %t2 = cmpxchg ptr %a, i32 0, i32 1 seq_cst seq_cst, align 2
+  ret void
+}
+
+define void @test_i128(ptr %a) nounwind {
+; CHECK-LABEL: test_i128:
+; CHECK: callq __atomic_load
+; CHECK: callq __atomic_store
+; CHECK: callq __atomic_exchange
+; CHECK: callq __atomic_compare_exchange
+  %t0 = load atomic i128, ptr %a seq_cst, align 8
+  store atomic i128 1, ptr %a seq_cst, align 8
+  %t1 = atomicrmw xchg ptr %a, i128 1 seq_cst, align 8
+  %t2 = atomicrmw add ptr %a, i128 2 seq_cst, align 8
+  %t3 = cmpxchg ptr %a, i128 0, i128 1 seq_cst seq_cst, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/atomic-unordered.ll b/llvm/test/CodeGen/X86/atomic-unordered.ll
index df123be53474..3fb994cdb751 100644
--- a/llvm/test/CodeGen/X86/atomic-unordered.ll
+++ b/llvm/test/CodeGen/X86/atomic-unordered.ll
@@ -228,34 +228,12 @@ define void @widen_broadcast_unaligned(ptr %p0, i32 %v) {
 }
 
 define i128 @load_i128(ptr %ptr) {
-; CHECK-O0-LABEL: load_i128:
-; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    pushq %rbx
-; CHECK-O0-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-O0-NEXT:    .cfi_offset %rbx, -16
-; CHECK-O0-NEXT:    xorl %eax, %eax
-; CHECK-O0-NEXT:    movl %eax, %ebx
-; CHECK-O0-NEXT:    movq %rbx, %rax
-; CHECK-O0-NEXT:    movq %rbx, %rdx
-; CHECK-O0-NEXT:    movq %rbx, %rcx
-; CHECK-O0-NEXT:    lock cmpxchg16b (%rdi)
-; CHECK-O0-NEXT:    popq %rbx
-; CHECK-O0-NEXT:    .cfi_def_cfa_offset 8
-; CHECK-O0-NEXT:    retq
-;
-; CHECK-O3-LABEL: load_i128:
-; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    pushq %rbx
-; CHECK-O3-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-O3-NEXT:    .cfi_offset %rbx, -16
-; CHECK-O3-NEXT:    xorl %eax, %eax
-; CHECK-O3-NEXT:    xorl %edx, %edx
-; CHECK-O3-NEXT:    xorl %ecx, %ecx
-; CHECK-O3-NEXT:    xorl %ebx, %ebx
-; CHECK-O3-NEXT:    lock cmpxchg16b (%rdi)
-; CHECK-O3-NEXT:    popq %rbx
-; CHECK-O3-NEXT:    .cfi_def_cfa_offset 8
-; CHECK-O3-NEXT:    retq
+; CHECK-LABEL: load_i128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rdi), %xmm0
+; CHECK-NEXT:    vmovq %xmm0, %rax
+; CHECK-NEXT:    vpextrq $1, %xmm0, %rdx
+; CHECK-NEXT:    retq
   %v = load atomic i128, ptr %ptr unordered, align 16
   ret i128 %v
 }
@@ -263,51 +241,18 @@ define i128 @load_i128(ptr %ptr) {
 define void @store_i128(ptr %ptr, i128 %v) {
 ; CHECK-O0-LABEL: store_i128:
 ; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    pushq %rbx
-; CHECK-O0-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-O0-NEXT:    .cfi_offset %rbx, -16
-; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    movq 8(%rdi), %rdx
-; CHECK-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-O0-NEXT:    jmp .LBB16_1
-; CHECK-O0-NEXT:  .LBB16_1: # %atomicrmw.start
-; CHECK-O0-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
-; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
-; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-O0-NEXT:    lock cmpxchg16b (%rsi)
-; CHECK-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-O0-NEXT:    jne .LBB16_1
-; CHECK-O0-NEXT:    jmp .LBB16_2
-; CHECK-O0-NEXT:  .LBB16_2: # %atomicrmw.end
-; CHECK-O0-NEXT:    popq %rbx
-; CHECK-O0-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-O0-NEXT:    vmovq %rsi, %xmm0
+; CHECK-O0-NEXT:    vmovq %rdx, %xmm1
+; CHECK-O0-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-O0-NEXT:    vmovdqa %xmm0, (%rdi)
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-LABEL: store_i128:
 ; CHECK-O3:       # %bb.0:
-; CHECK-O3-NEXT:    pushq %rbx
-; CHECK-O3-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-O3-NEXT:    .cfi_offset %rbx, -16
-; CHECK-O3-NEXT:    movq %rdx, %rcx
-; CHECK-O3-NEXT:    movq %rsi, %rbx
-; CHECK-O3-NEXT:    movq (%rdi), %rax
-; CHECK-O3-NEXT:    movq 8(%rdi), %rdx
-; CHECK-O3-NEXT:    .p2align 4, 0x90
-; CHECK-O3-NEXT:  .LBB16_1: # %atomicrmw.start
-; CHECK-O3-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-O3-NEXT:    lock cmpxchg16b (%rdi)
-; CHECK-O3-NEXT:    jne .LBB16_1
-; CHECK-O3-NEXT:  # %bb.2: # %atomicrmw.end
-; CHECK-O3-NEXT:    popq %rbx
-; CHECK-O3-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-O3-NEXT:    vmovq %rdx, %xmm0
+; CHECK-O3-NEXT:    vmovq %rsi, %xmm1
+; CHECK-O3-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-O3-NEXT:    vmovdqa %xmm0, (%rdi)
 ; CHECK-O3-NEXT:    retq
   store atomic i128 %v, ptr %ptr unordered, align 16
   ret void
diff --git a/llvm/test/CodeGen/X86/atomic128.ll b/llvm/test/CodeGen/X86/atomic128.ll
index 1f7c2254bc79..949ee9e276a4 100644
--- a/llvm/test/CodeGen/X86/atomic128.ll
+++ b/llvm/test/CodeGen/X86/atomic128.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16 | FileCheck %s --check-prefixes=CHECK,CHECK-NOAVX
+; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16,avx | FileCheck %s --check-prefixes=CHECK,CHECK-AVX
 
 ; Codegen of i128 without cx16 is tested in atomic-nocx16.ll
 
@@ -28,20 +29,28 @@ define i128 @val_compare_and_swap(ptr %p, i128 %oldval, i128 %newval) {
 @cmpxchg16b_global = external dso_local global { i128, i128 }, align 16
 
 ;; Make sure we retain the offset of the global variable.
-define void @cmpxchg16b_global_with_offset() nounwind {
-; CHECK-LABEL: cmpxchg16b_global_with_offset:
-; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    pushq %rbx
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    xorl %ebx, %ebx
-; CHECK-NEXT:    lock cmpxchg16b _cmpxchg16b_global+16(%rip)
-; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    retq
+define i128 @load_global_with_offset() nounwind {
+; CHECK-NOAVX-LABEL: load_global_with_offset:
+; CHECK-NOAVX:       ## %bb.0: ## %entry
+; CHECK-NOAVX-NEXT:    pushq %rbx
+; CHECK-NOAVX-NEXT:    xorl %eax, %eax
+; CHECK-NOAVX-NEXT:    xorl %edx, %edx
+; CHECK-NOAVX-NEXT:    xorl %ecx, %ecx
+; CHECK-NOAVX-NEXT:    xorl %ebx, %ebx
+; CHECK-NOAVX-NEXT:    lock cmpxchg16b _cmpxchg16b_global+16(%rip)
+; CHECK-NOAVX-NEXT:    popq %rbx
+; CHECK-NOAVX-NEXT:    retq
+;
+; CHECK-AVX-LABEL: load_global_with_offset:
+; CHECK-AVX:       ## %bb.0: ## %entry
+; CHECK-AVX-NEXT:    vmovdqa _cmpxchg16b_global+16(%rip), %xmm0
+; CHECK-AVX-NEXT:    vmovq %xmm0, %rax
+; CHECK-AVX-NEXT:    vpextrq $1, %xmm0, %rdx
+; CHECK-AVX-NEXT:    retq
+;
 entry:
   %0 = load atomic i128, ptr getelementptr inbounds ({i128, i128}, ptr @cmpxchg16b_global, i64 0, i32 1) acquire, align 16
-  ret void
+  ret i128 %0
 }
 
 define void @fetch_and_nand(ptr %p, i128 %bits) {
@@ -283,101 +292,140 @@ define void @fetch_and_umax(ptr %p, i128 %bits) {
 }
 
 define i128 @atomic_load_seq_cst(ptr %p) {
-; CHECK-LABEL: atomic_load_seq_cst:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    pushq %rbx
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset %rbx, -16
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    xorl %ebx, %ebx
-; CHECK-NEXT:    lock cmpxchg16b (%rdi)
-; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    retq
+; CHECK-NOAVX-LABEL: atomic_load_seq_cst:
+; CHECK-NOAVX:       ## %bb.0:
+; CHECK-NOAVX-NEXT:    pushq %rbx
+; CHECK-NOAVX-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOAVX-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NOAVX-NEXT:    xorl %eax, %eax
+; CHECK-NOAVX-NEXT:    xorl %edx, %edx
+; CHECK-NOAVX-NEXT:    xorl %ecx, %ecx
+; CHECK-NOAVX-NEXT:    xorl %ebx, %ebx
+; CHECK-NOAVX-NEXT:    lock cmpxchg16b (%rdi)
+; CHECK-NOAVX-NEXT:    popq %rbx
+; CHECK-NOAVX-NEXT:    retq
+;
+; CHECK-AVX-LABEL: atomic_load_seq_cst:
+; CHECK-AVX:       ## %bb.0:
+; CHECK-AVX-NEXT:    vmovdqa (%rdi), %xmm0
+; CHECK-AVX-NEXT:    vmovq %xmm0, %rax
+; CHECK-AVX-NEXT:    vpextrq $1, %xmm0, %rdx
+; CHECK-AVX-NEXT:    retq
    %r = load atomic i128, ptr %p seq_cst, align 16
    ret i128 %r
 }
 
 define i128 @atomic_load_relaxed(ptr %p) {
-; CHECK-LABEL: atomic_load_relaxed:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    pushq %rbx
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset %rbx, -16
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    xorl %ebx, %ebx
-; CHECK-NEXT:    lock cmpxchg16b (%rdi)
-; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    retq
+; CHECK-NOAVX-LABEL: atomic_load_relaxed:
+; CHECK-NOAVX:       ## %bb.0:
+; CHECK-NOAVX-NEXT:    pushq %rbx
+; CHECK-NOAVX-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOAVX-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NOAVX-NEXT:    xorl %eax, %eax
+; CHECK-NOAVX-NEXT:    xorl %edx, %edx
+; CHECK-NOAVX-NEXT:    xorl %ecx, %ecx
+; CHECK-NOAVX-NEXT:    xorl %ebx, %ebx
+; CHECK-NOAVX-NEXT:    lock cmpxchg16b (%rdi)
+; CHECK-NOAVX-NEXT:    popq %rbx
+; CHECK-NOAVX-NEXT:    retq
+;
+; CHECK-AVX-LABEL: atomic_load_relaxed:
+; CHECK-AVX:       ## %bb.0:
+; CHECK-AVX-NEXT:    vmovdqa (%rdi), %xmm0
+; CHECK-AVX-NEXT:    vmovq %xmm0, %rax
+; CHECK-AVX-NEXT:    vpextrq $1, %xmm0, %rdx
+; CHECK-AVX-NEXT:    retq
    %r = load atomic i128, ptr %p monotonic, align 16
    ret i128 %r
 }
 
 define void @atomic_store_seq_cst(ptr %p, i128 %in) {
-; CHECK-LABEL: atomic_store_seq_cst:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    pushq %rbx
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset %rbx, -16
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  LBB12_1: ## %atomicrmw.start
-; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    lock cmpxchg16b (%rdi)
-; CHECK-NEXT:    jne LBB12_1
-; CHECK-NEXT:  ## %bb.2: ## %atomicrmw.end
-; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    retq
+; CHECK-NOAVX-LABEL: atomic_store_seq_cst:
+; CHECK-NOAVX:       ## %bb.0:
+; CHECK-NOAVX-NEXT:    pushq %rbx
+; CHECK-NOAVX-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOAVX-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NOAVX-NEXT:    movq %rdx, %rcx
+; CHECK-NOAVX-NEXT:    movq %rsi, %rbx
+; CHECK-NOAVX-NEXT:    movq (%rdi), %rax
+; CHECK-NOAVX-NEXT:    movq 8(%rdi), %rdx
+; CHECK-NOAVX-NEXT:    .p2align 4, 0x90
+; CHECK-NOAVX-NEXT:  LBB12_1: ## %atomicrmw.start
+; CHECK-NOAVX-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NOAVX-NEXT:    lock cmpxchg16b (%rdi)
+; CHECK-NOAVX-NEXT:    jne LBB12_1
+; CHECK-NOAVX-NEXT:  ## %bb.2: ## %atomicrmw.end
+; CHECK-NOAVX-NEXT:    popq %rbx
+; CHECK-NOAVX-NEXT:    retq
+;
+; CHECK-AVX-LABEL: atomic_store_seq_cst:
+; CHECK-AVX:       ## %bb.0:
+; CHECK-AVX-NEXT:    vmovq %rdx, %xmm0
+; CHECK-AVX-NEXT:    vmovq %rsi, %xmm1
+; CHECK-AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-AVX-NEXT:    vmovdqa %xmm0, (%rdi)
+; CHECK-AVX-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
+; CHECK-AVX-NEXT:    retq
    store atomic i128 %in, ptr %p seq_cst, align 16
    ret void
 }
 
 define void @atomic_store_release(ptr %p, i128 %in) {
-; CHECK-LABEL: atomic_store_release:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    pushq %rbx
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset %rbx, -16
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  LBB13_1: ## %atomicrmw.start
-; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    lock cmpxchg16b (%rdi)
-; CHECK-NEXT:    jne LBB13_1
-; CHECK-NEXT:  ## %bb.2: ## %atomicrmw.end
-; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    retq
+; CHECK-NOAVX-LABEL: atomic_store_release:
+; CHECK-NOAVX:       ## %bb.0:
+; CHECK-NOAVX-NEXT:    pushq %rbx
+; CHECK-NOAVX-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOAVX-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NOAVX-NEXT:    movq %rdx, %rcx
+; CHECK-NOAVX-NEXT:    movq %rsi, %rbx
+; CHECK-NOAVX-NEXT:    movq (%rdi), %rax
+; CHECK-NOAVX-NEXT:    movq 8(%rdi), %rdx
+; CHECK-NOAVX-NEXT:    .p2align 4, 0x90
+; CHECK-NOAVX-NEXT:  LBB13_1: ## %atomicrmw.start
+; CHECK-NOAVX-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NOAVX-NEXT:    lock cmpxchg16b (%rdi)
+; CHECK-NOAVX-NEXT:    jne LBB13_1
+; CHECK-NOAVX-NEXT:  ## %bb.2: ## %atomicrmw.end
+; CHECK-NOAVX-NEXT:    popq %rbx
+; CHECK-NOAVX-NEXT:    retq
+;
+; CHECK-AVX-LABEL: atomic_store_release:
+; CHECK-AVX:       ## %bb.0:
+; CHECK-AVX-NEXT:    vmovq %rdx, %xmm0
+; CHECK-AVX-NEXT:    vmovq %rsi, %xmm1
+; CHECK-AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-AVX-NEXT:    vmovdqa %xmm0, (%rdi)
+; CHECK-AVX-NEXT:    retq
    store atomic i128 %in, ptr %p release, align 16
    ret void
 }
 
 define void @atomic_store_relaxed(ptr %p, i128 %in) {
-; CHECK-LABEL: atomic_store_relaxed:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    pushq %rbx
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset %rbx, -16
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    movq 8(%rdi), %rdx
-; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  LBB14_1: ## %atomicrmw.start
-; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    lock cmpxchg16b (%rdi)
-; CHECK-NEXT:    jne LBB14_1
-; CHECK-NEXT:  ## %bb.2: ## %atomicrmw.end
-; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    retq
+; CHECK-NOAVX-LABEL: atomic_store_relaxed:
+; CHECK-NOAVX:       ## %bb.0:
+; CHECK-NOAVX-NEXT:    pushq %rbx
+; CHECK-NOAVX-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOAVX-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NOAVX-NEXT:    movq %rdx, %rcx
+; CHECK-NOAVX-NEXT:    movq %rsi, %rbx
+; CHECK-NOAVX-NEXT:    movq (%rdi), %rax
+; CHECK-NOAVX-NEXT:    movq 8(%rdi), %rdx
+; CHECK-NOAVX-NEXT:    .p2align 4, 0x90
+; CHECK-NOAVX-NEXT:  LBB14_1: ## %atomicrmw.start
+; CHECK-NOAVX-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NOAVX-NEXT:    lock cmpxchg16b (%rdi)
+; CHECK-NOAVX-NEXT:    jne LBB14_1
+; CHECK-NOAVX-NEXT:  ## %bb.2: ## %atomicrmw.end
+; CHECK-NOAVX-NEXT:    popq %rbx
+; CHECK-NOAVX-NEXT:    retq
+;
+; CHECK-AVX-LABEL: atomic_store_relaxed:
+; CHECK-AVX:       ## %bb.0:
+; CHECK-AVX-NEXT:    vmovq %rdx, %xmm0
+; CHECK-AVX-NEXT:    vmovq %rsi, %xmm1
+; CHECK-AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-AVX-NEXT:    vmovdqa %xmm0, (%rdi)
+; CHECK-AVX-NEXT:    retq
    store atomic i128 %in, ptr %p unordered, align 16
    ret void
 }
diff --git a/llvm/test/CodeGen/X86/cmpxchg-i128-i1.ll b/llvm/test/CodeGen/X86/cmpxchg-i128-i1.ll
index 57594870a54a..3fb561d00f97 100644
--- a/llvm/test/CodeGen/X86/cmpxchg-i128-i1.ll
+++ b/llvm/test/CodeGen/X86/cmpxchg-i128-i1.ll
@@ -110,11 +110,9 @@ define i128 @cmpxchg_use_eflags_and_val(ptr %addr, i128 %offset) {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset %rbx, -16
 ; CHECK-NEXT:    movq %rdx, %r8
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    xorl %ebx, %ebx
-; CHECK-NEXT:    lock cmpxchg16b (%rdi)
+; CHECK-NEXT:    vmovdqa (%rdi), %xmm0
+; CHECK-NEXT:    vpextrq $1, %xmm0, %rdx
+; CHECK-NEXT:    vmovq %xmm0, %rax
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB4_1: # %loop
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-- 
GitLab


From f88760f3118a106be847a8d5296931a8e44c72f8 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 16 May 2024 15:37:46 -0700
Subject: [PATCH 1068/1206] [TableGen] Remove unnecessary StringRef constructor
 call from LessRecord(). NFC

Record::getName already returns a StringRef. When this code was
originally written getName returned const std::string &.
---
 llvm/include/llvm/TableGen/Record.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h
index c1b352e974ee..c83f5d0e99fd 100644
--- a/llvm/include/llvm/TableGen/Record.h
+++ b/llvm/include/llvm/TableGen/Record.h
@@ -2098,7 +2098,7 @@ private:
 /// Sorting predicate to sort record pointers by name.
 struct LessRecord {
   bool operator()(const Record *Rec1, const Record *Rec2) const {
-    return StringRef(Rec1->getName()).compare_numeric(Rec2->getName()) < 0;
+    return Rec1->getName().compare_numeric(Rec2->getName()) < 0;
   }
 };
 
-- 
GitLab


From 2a4ee605b03bc69c296d0388239d9c837e787995 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 16 May 2024 15:42:07 -0700
Subject: [PATCH 1069/1206] [TableGen] Use LessRecord() for the sort in
 getAllDerivedDefinitions instead of re-implementing it. NFC

---
 llvm/lib/TableGen/Record.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/llvm/lib/TableGen/Record.cpp b/llvm/lib/TableGen/Record.cpp
index 2b3e8a0c7f84..f9acc68ce5c7 100644
--- a/llvm/lib/TableGen/Record.cpp
+++ b/llvm/lib/TableGen/Record.cpp
@@ -3251,9 +3251,7 @@ std::vector<Record *> RecordKeeper::getAllDerivedDefinitions(
       Defs.push_back(OneDef.second.get());
   }
 
-  llvm::sort(Defs, [](Record *LHS, Record *RHS) {
-    return LHS->getName().compare_numeric(RHS->getName()) < 0;
-  });
+  llvm::sort(Defs, LessRecord());
 
   return Defs;
 }
-- 
GitLab


From f210152e5fbcec1c50ff5ccc1f6680ab2c39b46f Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Thu, 16 May 2024 19:53:42 -0300
Subject: [PATCH 1070/1206] [clang] constrain
 `SemaCXX/cxx20-ctad-type-alias.cpp` target triple

The test expectations are otherwise affected by the underlying type of size_t.
---
 clang/test/SemaCXX/cxx20-ctad-type-alias.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
index 21b2b8bff500..70e81cd79fd6 100644
--- a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
+++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -Wno-c++11-narrowing -Wno-literal-conversion -std=c++20 -verify %s
+// RUN: %clang_cc1 -fsyntax-only -triple x86_64-unknown-linux -Wno-c++11-narrowing -Wno-literal-conversion -std=c++20 -verify %s
 
 namespace test1 {
 template <typename T>
-- 
GitLab


From fe4d5f0d0e457a0a7dec2c7dc87996706b30a25e Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Thu, 16 May 2024 19:55:31 -0300
Subject: [PATCH 1071/1206] [clang] NFC: stray space cleanup

---
 clang/test/SemaCXX/cxx20-ctad-type-alias.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
index 70e81cd79fd6..285532e3d80d 100644
--- a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
+++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
@@ -74,7 +74,7 @@ struct Foo {
 template <typename T>
 using AF = Foo<T, 1>;
 
-AF b{0}; 
+AF b{0};
 }  // namespace test6
 
 namespace test7 {
@@ -86,8 +86,8 @@ struct Foo {
 template <typename U>
 using AF1 = Foo<U>;
 template <typename K>
-using AF2 = AF1<K>;  
-AF2 b = 1;  
+using AF2 = AF1<K>;
+AF2 b = 1;
 }  // namespace test7
 
 namespace test8 {
@@ -149,7 +149,7 @@ namespace test12 {
 template<typename X>
 struct Foo {
   template<typename K>
-  struct Bar { 
+  struct Bar {
     Bar(K);
   };
 
-- 
GitLab


From fa9b1be45088dce1e4b602d451f118128b94237b Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Thu, 16 May 2024 16:16:19 -0700
Subject: [PATCH 1072/1206] [ThinLTO]Mark referencers of local ifunc not
 eligible for import (#92431)

If an ifunc has local linkage, do not add it into ref edges and mark its
referencer (a function or global variable) not eligible for import. An
ifunc doesn't have summary and ThinLTO cannot promote it. Importing the
referencer may cause linkage errors.

To reference a similar fix, https://reviews.llvm.org/D158961 marks
callers of local ifunc not eligible for import to fix
https://github.com/llvm/llvm-project/issues/58740
---
 llvm/lib/Analysis/ModuleSummaryAnalysis.cpp | 42 +++++++++++------
 llvm/test/ThinLTO/X86/ref-ifunc.ll          | 52 +++++++++++++++++++++
 2 files changed, 81 insertions(+), 13 deletions(-)
 create mode 100644 llvm/test/ThinLTO/X86/ref-ifunc.ll

diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index c3d15afe6d9c..c863936a5188 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -95,9 +95,12 @@ extern cl::opt<unsigned> MaxNumVTableAnnotations;
 // global vars at all. When importing function we aren't interested if any
 // instruction in it takes an address of any basic block, because instruction
 // can only take an address of basic block located in the same function.
+// Set `RefLocalLinkageIFunc` to true if the analyzed value references a
+// local-linkage ifunc.
 static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
                          SetVector<ValueInfo, std::vector<ValueInfo>> &RefEdges,
-                         SmallPtrSet<const User *, 8> &Visited) {
+                         SmallPtrSet<const User *, 8> &Visited,
+                         bool &RefLocalLinkageIFunc) {
   bool HasBlockAddress = false;
   SmallVector<const User *, 32> Worklist;
   if (Visited.insert(CurUser).second)
@@ -119,8 +122,18 @@ static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
         // We have a reference to a global value. This should be added to
         // the reference set unless it is a callee. Callees are handled
         // specially by WriteFunction and are added to a separate list.
-        if (!(CB && CB->isCallee(&OI)))
+        if (!(CB && CB->isCallee(&OI))) {
+          // If an ifunc has local linkage, do not add it into ref edges, and
+          // sets `RefLocalLinkageIFunc` to true. The referencer is not eligible
+          // for import. An ifunc doesn't have summary and ThinLTO cannot
+          // promote it; importing the referencer may cause linkage errors.
+          if (auto *GI = dyn_cast_if_present<GlobalIFunc>(GV);
+              GI && GI->hasLocalLinkage()) {
+            RefLocalLinkageIFunc = true;
+            continue;
+          }
           RefEdges.insert(Index.getOrInsertValueInfo(GV));
+        }
         continue;
       }
       if (Visited.insert(Operand).second)
@@ -313,7 +326,8 @@ static void computeFunctionSummary(
 
   // Add personality function, prefix data and prologue data to function's ref
   // list.
-  findRefEdges(Index, &F, RefEdges, Visited);
+  bool HasLocalIFuncCallOrRef = false;
+  findRefEdges(Index, &F, RefEdges, Visited, HasLocalIFuncCallOrRef);
   std::vector<const Instruction *> NonVolatileLoads;
   std::vector<const Instruction *> NonVolatileStores;
 
@@ -326,7 +340,6 @@ static void computeFunctionSummary(
 
   bool HasInlineAsmMaybeReferencingInternal = false;
   bool HasIndirBranchToBlockAddress = false;
-  bool HasIFuncCall = false;
   bool HasUnknownCall = false;
   bool MayThrow = false;
   for (const BasicBlock &BB : F) {
@@ -372,11 +385,11 @@ static void computeFunctionSummary(
             // of calling it we should add GV to RefEdges directly.
             RefEdges.insert(Index.getOrInsertValueInfo(GV));
           else if (auto *U = dyn_cast<User>(Stored))
-            findRefEdges(Index, U, RefEdges, Visited);
+            findRefEdges(Index, U, RefEdges, Visited, HasLocalIFuncCallOrRef);
           continue;
         }
       }
-      findRefEdges(Index, &I, RefEdges, Visited);
+      findRefEdges(Index, &I, RefEdges, Visited, HasLocalIFuncCallOrRef);
       const auto *CB = dyn_cast<CallBase>(&I);
       if (!CB) {
         if (I.mayThrow())
@@ -450,7 +463,7 @@ static void computeFunctionSummary(
         // Non-local ifunc is not cloned and does not have the issue.
         if (auto *GI = dyn_cast_if_present<GlobalIFunc>(CalledValue))
           if (GI->hasLocalLinkage())
-            HasIFuncCall = true;
+            HasLocalIFuncCallOrRef = true;
         // Skip inline assembly calls.
         if (CI && CI->isInlineAsm())
           continue;
@@ -555,7 +568,7 @@ static void computeFunctionSummary(
                            SmallPtrSet<const User *, 8> &Cache) {
       for (const auto *I : Instrs) {
         Cache.erase(I);
-        findRefEdges(Index, I, Edges, Cache);
+        findRefEdges(Index, I, Edges, Cache, HasLocalIFuncCallOrRef);
       }
     };
 
@@ -631,9 +644,9 @@ static void computeFunctionSummary(
 #endif
 
   bool NonRenamableLocal = isNonRenamableLocal(F);
-  bool NotEligibleForImport = NonRenamableLocal ||
-                              HasInlineAsmMaybeReferencingInternal ||
-                              HasIndirBranchToBlockAddress || HasIFuncCall;
+  bool NotEligibleForImport =
+      NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
+      HasIndirBranchToBlockAddress || HasLocalIFuncCallOrRef;
   GlobalValueSummary::GVFlags Flags(
       F.getLinkage(), F.getVisibility(), NotEligibleForImport,
       /* Live = */ false, F.isDSOLocal(), F.canBeOmittedFromSymbolTable(),
@@ -787,7 +800,10 @@ static void computeVariableSummary(ModuleSummaryIndex &Index,
                                    SmallVectorImpl<MDNode *> &Types) {
   SetVector<ValueInfo, std::vector<ValueInfo>> RefEdges;
   SmallPtrSet<const User *, 8> Visited;
-  bool HasBlockAddress = findRefEdges(Index, &V, RefEdges, Visited);
+  bool RefLocalIFunc = false;
+  bool HasBlockAddress =
+      findRefEdges(Index, &V, RefEdges, Visited, RefLocalIFunc);
+  const bool NotEligibleForImport = (HasBlockAddress || RefLocalIFunc);
   bool NonRenamableLocal = isNonRenamableLocal(V);
   GlobalValueSummary::GVFlags Flags(
       V.getLinkage(), V.getVisibility(), NonRenamableLocal,
@@ -821,7 +837,7 @@ static void computeVariableSummary(ModuleSummaryIndex &Index,
                                                          RefEdges.takeVector());
   if (NonRenamableLocal)
     CantBePromoted.insert(V.getGUID());
-  if (HasBlockAddress)
+  if (NotEligibleForImport)
     GVarSummary->setNotEligibleToImport();
   if (!VTableFuncs.empty())
     GVarSummary->setVTableFuncs(VTableFuncs);
diff --git a/llvm/test/ThinLTO/X86/ref-ifunc.ll b/llvm/test/ThinLTO/X86/ref-ifunc.ll
new file mode 100644
index 000000000000..a43c080fbf3a
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/ref-ifunc.ll
@@ -0,0 +1,52 @@
+; RUN: opt -module-summary %s -o %t.bc
+
+; RUN: llvm-dis %t.bc -o - | FileCheck %s
+
+; Tests that var and caller are not eligible to import and they don't have refs to ifunc 'callee'
+
+; CHECK: gv: (name: "var", summaries: (variable: ({{.*}}, flags: ({{.*}}notEligibleToImport: 1
+; CHECK-NOT: refs
+; CHECK-SAME: guid = 7919382516565939378
+
+; CHECK: gv: (name: "caller", summaries: (function: ({{.*}}, flags: ({{.*}}notEligibleToImport: 1
+; CHECK-NOT: refs
+; CHECK-SAME: guid = 16677772384402303968
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@__cpu_model = external global { i32, i32, i32, [1 x i32] }
+
+@callee = internal ifunc void(), ptr @callee.resolver
+
+@var = constant { [1 x ptr] } { [1 x ptr] [ptr @callee]}
+
+define void @dispatch(ptr %func) {
+    tail call void %func()
+    ret void
+}
+
+define void @caller() {
+  tail call void @dispatch(ptr @callee)
+  ret void
+}
+
+define internal ptr @callee.resolver() {
+resolver_entry:
+  tail call void @__cpu_indicator_init()
+  %0 = load i32, ptr getelementptr inbounds ({ i32, i32, i32, [1 x i32] }, ptr @__cpu_model, i64 0, i32 3, i64 0)
+  %1 = and i32 %0, 1024
+  %.not = icmp eq i32 %1, 0
+  %func_sel = select i1 %.not, ptr @callee.default.1, ptr @callee.avx2.0
+  ret ptr %func_sel
+}
+
+define internal void @callee.default.1(i32 %a) {
+  ret void
+}
+
+define internal void @callee.avx2.0(i32 %a) {
+  ret void
+}
+
+declare void @__cpu_indicator_init()
-- 
GitLab


From 96d2db4ba9016b4512e7cd27d7dfda209d07ce62 Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Fri, 17 May 2024 09:12:12 +0800
Subject: [PATCH 1073/1206] [LoongArch] Pre-commit test for lib call auguments
 extension

Reviewed By: SixWeining

Pull Request: https://github.com/llvm/llvm-project/pull/92374
---
 llvm/test/CodeGen/LoongArch/libcall-extend.ll | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 llvm/test/CodeGen/LoongArch/libcall-extend.ll

diff --git a/llvm/test/CodeGen/LoongArch/libcall-extend.ll b/llvm/test/CodeGen/LoongArch/libcall-extend.ll
new file mode 100644
index 000000000000..957a15141802
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/libcall-extend.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc --mtriple=loongarch64 --target-abi=lp64s --mattr=-f < %s | FileCheck %s
+
+define signext i32 @convert_float_to_i32(i32 %tmp, float %a) nounwind {
+; CHECK-LABEL: convert_float_to_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.d $sp, $sp, -16
+; CHECK-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; CHECK-NEXT:    bstrpick.d $a0, $a1, 31, 0
+; CHECK-NEXT:    bl %plt(__fixsfsi)
+; CHECK-NEXT:    addi.w $a0, $a0, 0
+; CHECK-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 16
+; CHECK-NEXT:    ret
+  %1 = fptosi float %a to i32
+  ret i32 %1
+}
+
+define signext i32 @convert_double_to_i32(i32 %tmp, double %a) nounwind {
+; CHECK-LABEL: convert_double_to_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.d $sp, $sp, -16
+; CHECK-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; CHECK-NEXT:    move $a0, $a1
+; CHECK-NEXT:    bl %plt(__fixdfsi)
+; CHECK-NEXT:    addi.w $a0, $a0, 0
+; CHECK-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 16
+; CHECK-NEXT:    ret
+  %1 = fptosi double %a to i32
+  ret i32 %1
+}
+
+define signext i32 @convert_fp128_to_i32(i32 %tmp, fp128 %a) nounwind {
+; CHECK-LABEL: convert_fp128_to_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.d $sp, $sp, -16
+; CHECK-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; CHECK-NEXT:    move $a0, $a1
+; CHECK-NEXT:    move $a1, $a2
+; CHECK-NEXT:    bl %plt(__fixtfsi)
+; CHECK-NEXT:    addi.w $a0, $a0, 0
+; CHECK-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 16
+; CHECK-NEXT:    ret
+  %1 = fptosi fp128 %a to i32
+  ret i32 %1
+}
-- 
GitLab


From 5a204a5f0a19417c1c0bd71834dee7ba129e6776 Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Fri, 17 May 2024 09:13:05 +0800
Subject: [PATCH 1074/1206] [LoongArch] Use sign extend for i32 arguments in
 makeLibCall on LA64

The 32 bits arguments and returns on LA64 are always sign extended to
i64. So we should be taking this into account around libcalls.

Reviewed By: heiher, SixWeining

Pull Request: https://github.com/llvm/llvm-project/pull/92375
---
 .../Target/LoongArch/LoongArchISelLowering.cpp   |  8 ++++++++
 .../lib/Target/LoongArch/LoongArchISelLowering.h |  1 +
 .../test/CodeGen/LoongArch/calling-conv-lp64s.ll |  2 +-
 llvm/test/CodeGen/LoongArch/libcall-extend.ll    |  3 ---
 llvm/test/CodeGen/LoongArch/sextw-removal.ll     | 16 ++++++++--------
 llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll    |  2 +-
 6 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 6740563b90b4..6a0db9c27def 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -4983,3 +4983,11 @@ ISD::NodeType LoongArchTargetLowering::getExtendForAtomicCmpSwapArg() const {
   // TODO: LAMCAS will use amcas{_DB,}.[bhwd] which does not require extension.
   return ISD::SIGN_EXTEND;
 }
+
+bool LoongArchTargetLowering::shouldSignExtendTypeInLibCall(
+    EVT Type, bool IsSigned) const {
+  if (Subtarget.is64Bit() && Type == MVT::i32)
+    return true;
+
+  return IsSigned;
+}
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 31b4d6519563..a344eadee866 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -239,6 +239,7 @@ public:
     return false;
   }
   bool shouldConsiderGEPOffsetSplit() const override { return true; }
+  bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override;
 
 private:
   /// Target-specific function used to lower LoongArch calling conventions.
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll b/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll
index f57743511496..184ba7363e1b 100644
--- a/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll
@@ -10,7 +10,7 @@ define i64 @callee_float_in_regs(i64 %a, float %b) nounwind {
 ; CHECK-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
 ; CHECK-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
 ; CHECK-NEXT:    move $fp, $a0
-; CHECK-NEXT:    bstrpick.d $a0, $a1, 31, 0
+; CHECK-NEXT:    addi.w $a0, $a1, 0
 ; CHECK-NEXT:    bl %plt(__fixsfdi)
 ; CHECK-NEXT:    add.d $a0, $fp, $a0
 ; CHECK-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/LoongArch/libcall-extend.ll b/llvm/test/CodeGen/LoongArch/libcall-extend.ll
index 957a15141802..3cc8a6645f36 100644
--- a/llvm/test/CodeGen/LoongArch/libcall-extend.ll
+++ b/llvm/test/CodeGen/LoongArch/libcall-extend.ll
@@ -8,7 +8,6 @@ define signext i32 @convert_float_to_i32(i32 %tmp, float %a) nounwind {
 ; CHECK-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
 ; CHECK-NEXT:    bstrpick.d $a0, $a1, 31, 0
 ; CHECK-NEXT:    bl %plt(__fixsfsi)
-; CHECK-NEXT:    addi.w $a0, $a0, 0
 ; CHECK-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; CHECK-NEXT:    addi.d $sp, $sp, 16
 ; CHECK-NEXT:    ret
@@ -23,7 +22,6 @@ define signext i32 @convert_double_to_i32(i32 %tmp, double %a) nounwind {
 ; CHECK-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
 ; CHECK-NEXT:    move $a0, $a1
 ; CHECK-NEXT:    bl %plt(__fixdfsi)
-; CHECK-NEXT:    addi.w $a0, $a0, 0
 ; CHECK-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; CHECK-NEXT:    addi.d $sp, $sp, 16
 ; CHECK-NEXT:    ret
@@ -39,7 +37,6 @@ define signext i32 @convert_fp128_to_i32(i32 %tmp, fp128 %a) nounwind {
 ; CHECK-NEXT:    move $a0, $a1
 ; CHECK-NEXT:    move $a1, $a2
 ; CHECK-NEXT:    bl %plt(__fixtfsi)
-; CHECK-NEXT:    addi.w $a0, $a0, 0
 ; CHECK-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; CHECK-NEXT:    addi.d $sp, $sp, 16
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/LoongArch/sextw-removal.ll b/llvm/test/CodeGen/LoongArch/sextw-removal.ll
index 7708873e264d..c9f3ed5237aa 100644
--- a/llvm/test/CodeGen/LoongArch/sextw-removal.ll
+++ b/llvm/test/CodeGen/LoongArch/sextw-removal.ll
@@ -257,11 +257,11 @@ define void @test6(i32 signext %arg, i32 signext %arg1) nounwind {
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    addi.w $a0, $fp, 0
 ; CHECK-NEXT:    bl %plt(baz)
-; CHECK-NEXT:    bstrpick.d $s0, $a0, 31, 0
-; CHECK-NEXT:    move $a0, $s0
+; CHECK-NEXT:    move $s0, $a0
+; CHECK-NEXT:    bstrpick.d $a0, $a0, 31, 0
 ; CHECK-NEXT:    bl %plt(__fixsfsi)
 ; CHECK-NEXT:    move $fp, $a0
-; CHECK-NEXT:    move $a0, $s0
+; CHECK-NEXT:    addi.w $a0, $s0, 0
 ; CHECK-NEXT:    move $a1, $zero
 ; CHECK-NEXT:    bl %plt(__nesf2)
 ; CHECK-NEXT:    bnez $a0, .LBB5_1
@@ -284,11 +284,11 @@ define void @test6(i32 signext %arg, i32 signext %arg1) nounwind {
 ; NORMV-NEXT:    # =>This Inner Loop Header: Depth=1
 ; NORMV-NEXT:    addi.w $a0, $fp, 0
 ; NORMV-NEXT:    bl %plt(baz)
-; NORMV-NEXT:    bstrpick.d $s0, $a0, 31, 0
-; NORMV-NEXT:    move $a0, $s0
+; NORMV-NEXT:    move $s0, $a0
+; NORMV-NEXT:    bstrpick.d $a0, $a0, 31, 0
 ; NORMV-NEXT:    bl %plt(__fixsfsi)
 ; NORMV-NEXT:    move $fp, $a0
-; NORMV-NEXT:    move $a0, $s0
+; NORMV-NEXT:    addi.w $a0, $s0, 0
 ; NORMV-NEXT:    move $a1, $zero
 ; NORMV-NEXT:    bl %plt(__nesf2)
 ; NORMV-NEXT:    bnez $a0, .LBB5_1
@@ -562,7 +562,7 @@ define void @test10(i32 signext %arg, i32 signext %arg1) nounwind {
 ; CHECK-NEXT:    addi.w $a0, $fp, 0
 ; CHECK-NEXT:    bl %plt(baz)
 ; CHECK-NEXT:    move $fp, $a0
-; CHECK-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; CHECK-NEXT:    addi.w $a0, $a0, 0
 ; CHECK-NEXT:    move $a1, $zero
 ; CHECK-NEXT:    bl %plt(__nesf2)
 ; CHECK-NEXT:    bnez $a0, .LBB9_1
@@ -584,7 +584,7 @@ define void @test10(i32 signext %arg, i32 signext %arg1) nounwind {
 ; NORMV-NEXT:    addi.w $a0, $fp, 0
 ; NORMV-NEXT:    bl %plt(baz)
 ; NORMV-NEXT:    move $fp, $a0
-; NORMV-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; NORMV-NEXT:    addi.w $a0, $a0, 0
 ; NORMV-NEXT:    move $a1, $zero
 ; NORMV-NEXT:    bl %plt(__nesf2)
 ; NORMV-NEXT:    bnez $a0, .LBB9_1
diff --git a/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll b/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll
index e6df401f0fef..c23fc5a73ae0 100644
--- a/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll
+++ b/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll
@@ -145,7 +145,7 @@ define i64 @fptosi_i64_float(float %X) nounwind {
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.d $sp, $sp, -16
 ; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
-; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 0
+; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    bl %plt(__fixsfdi)
 ; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; LA64-NEXT:    addi.d $sp, $sp, 16
-- 
GitLab


From bf1d4172335689f62e4f7368446f0026c595330b Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Fri, 17 May 2024 09:13:51 +0800
Subject: [PATCH 1075/1206] [LoongArch] Suppress the unnecessary extensions for
 arguments in makeLibCall

Reviewed By: SixWeining, heiher

Pull Request: https://github.com/llvm/llvm-project/pull/92376
---
 llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp | 9 +++++++++
 llvm/lib/Target/LoongArch/LoongArchISelLowering.h   | 1 +
 llvm/lib/Target/LoongArch/LoongArchSubtarget.h      | 4 ++++
 llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll   | 2 +-
 llvm/test/CodeGen/LoongArch/libcall-extend.ll       | 2 +-
 llvm/test/CodeGen/LoongArch/sextw-removal.ll        | 8 ++------
 llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll       | 2 --
 7 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 6a0db9c27def..fe2c613b1b30 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -4991,3 +4991,12 @@ bool LoongArchTargetLowering::shouldSignExtendTypeInLibCall(
 
   return IsSigned;
 }
+
+bool LoongArchTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
+  // Return false to suppress the unnecessary extensions if the LibCall
+  // arguments or return value is a float narrower than GRLEN on a soft FP ABI.
+  if (Subtarget.isSoftFPABI() && (Type.isFloatingPoint() && !Type.isVector() &&
+                                  Type.getSizeInBits() < Subtarget.getGRLen()))
+    return false;
+  return true;
+}
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index a344eadee866..de3f45172e25 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -240,6 +240,7 @@ public:
   }
   bool shouldConsiderGEPOffsetSplit() const override { return true; }
   bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override;
+  bool shouldExtendTypeInLibCall(EVT Type) const override;
 
 private:
   /// Target-specific function used to lower LoongArch calling conventions.
diff --git a/llvm/lib/Target/LoongArch/LoongArchSubtarget.h b/llvm/lib/Target/LoongArch/LoongArchSubtarget.h
index bfdfdcf36d66..b87ea6e2ec32 100644
--- a/llvm/lib/Target/LoongArch/LoongArchSubtarget.h
+++ b/llvm/lib/Target/LoongArch/LoongArchSubtarget.h
@@ -91,6 +91,10 @@ public:
   MVT getGRLenVT() const { return GRLenVT; }
   unsigned getGRLen() const { return GRLen; }
   LoongArchABI::ABI getTargetABI() const { return TargetABI; }
+  bool isSoftFPABI() const {
+    return TargetABI == LoongArchABI::ABI_LP64S ||
+           TargetABI == LoongArchABI::ABI_ILP32S;
+  }
   bool isXRaySupported() const override { return is64Bit(); }
   Align getPrefFunctionAlignment() const { return PrefFunctionAlignment; }
   Align getPrefLoopAlignment() const { return PrefLoopAlignment; }
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll b/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll
index 184ba7363e1b..8158c387f7b6 100644
--- a/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll
@@ -10,7 +10,7 @@ define i64 @callee_float_in_regs(i64 %a, float %b) nounwind {
 ; CHECK-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
 ; CHECK-NEXT:    st.d $fp, $sp, 0 # 8-byte Folded Spill
 ; CHECK-NEXT:    move $fp, $a0
-; CHECK-NEXT:    addi.w $a0, $a1, 0
+; CHECK-NEXT:    move $a0, $a1
 ; CHECK-NEXT:    bl %plt(__fixsfdi)
 ; CHECK-NEXT:    add.d $a0, $fp, $a0
 ; CHECK-NEXT:    ld.d $fp, $sp, 0 # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/LoongArch/libcall-extend.ll b/llvm/test/CodeGen/LoongArch/libcall-extend.ll
index 3cc8a6645f36..7ca69811a7d5 100644
--- a/llvm/test/CodeGen/LoongArch/libcall-extend.ll
+++ b/llvm/test/CodeGen/LoongArch/libcall-extend.ll
@@ -6,7 +6,7 @@ define signext i32 @convert_float_to_i32(i32 %tmp, float %a) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi.d $sp, $sp, -16
 ; CHECK-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
-; CHECK-NEXT:    bstrpick.d $a0, $a1, 31, 0
+; CHECK-NEXT:    move $a0, $a1
 ; CHECK-NEXT:    bl %plt(__fixsfsi)
 ; CHECK-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; CHECK-NEXT:    addi.d $sp, $sp, 16
diff --git a/llvm/test/CodeGen/LoongArch/sextw-removal.ll b/llvm/test/CodeGen/LoongArch/sextw-removal.ll
index c9f3ed5237aa..d95a5bd298f5 100644
--- a/llvm/test/CodeGen/LoongArch/sextw-removal.ll
+++ b/llvm/test/CodeGen/LoongArch/sextw-removal.ll
@@ -258,10 +258,9 @@ define void @test6(i32 signext %arg, i32 signext %arg1) nounwind {
 ; CHECK-NEXT:    addi.w $a0, $fp, 0
 ; CHECK-NEXT:    bl %plt(baz)
 ; CHECK-NEXT:    move $s0, $a0
-; CHECK-NEXT:    bstrpick.d $a0, $a0, 31, 0
 ; CHECK-NEXT:    bl %plt(__fixsfsi)
 ; CHECK-NEXT:    move $fp, $a0
-; CHECK-NEXT:    addi.w $a0, $s0, 0
+; CHECK-NEXT:    move $a0, $s0
 ; CHECK-NEXT:    move $a1, $zero
 ; CHECK-NEXT:    bl %plt(__nesf2)
 ; CHECK-NEXT:    bnez $a0, .LBB5_1
@@ -285,10 +284,9 @@ define void @test6(i32 signext %arg, i32 signext %arg1) nounwind {
 ; NORMV-NEXT:    addi.w $a0, $fp, 0
 ; NORMV-NEXT:    bl %plt(baz)
 ; NORMV-NEXT:    move $s0, $a0
-; NORMV-NEXT:    bstrpick.d $a0, $a0, 31, 0
 ; NORMV-NEXT:    bl %plt(__fixsfsi)
 ; NORMV-NEXT:    move $fp, $a0
-; NORMV-NEXT:    addi.w $a0, $s0, 0
+; NORMV-NEXT:    move $a0, $s0
 ; NORMV-NEXT:    move $a1, $zero
 ; NORMV-NEXT:    bl %plt(__nesf2)
 ; NORMV-NEXT:    bnez $a0, .LBB5_1
@@ -562,7 +560,6 @@ define void @test10(i32 signext %arg, i32 signext %arg1) nounwind {
 ; CHECK-NEXT:    addi.w $a0, $fp, 0
 ; CHECK-NEXT:    bl %plt(baz)
 ; CHECK-NEXT:    move $fp, $a0
-; CHECK-NEXT:    addi.w $a0, $a0, 0
 ; CHECK-NEXT:    move $a1, $zero
 ; CHECK-NEXT:    bl %plt(__nesf2)
 ; CHECK-NEXT:    bnez $a0, .LBB9_1
@@ -584,7 +581,6 @@ define void @test10(i32 signext %arg, i32 signext %arg1) nounwind {
 ; NORMV-NEXT:    addi.w $a0, $fp, 0
 ; NORMV-NEXT:    bl %plt(baz)
 ; NORMV-NEXT:    move $fp, $a0
-; NORMV-NEXT:    addi.w $a0, $a0, 0
 ; NORMV-NEXT:    move $a1, $zero
 ; NORMV-NEXT:    bl %plt(__nesf2)
 ; NORMV-NEXT:    bnez $a0, .LBB9_1
diff --git a/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll b/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll
index c23fc5a73ae0..4eb34bfa09ac 100644
--- a/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll
+++ b/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll
@@ -69,7 +69,6 @@ define i32 @fptosi_i32_float(float %X) nounwind {
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.d $sp, $sp, -16
 ; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
-; LA64-NEXT:    bstrpick.d $a0, $a0, 31, 0
 ; LA64-NEXT:    bl %plt(__fixsfsi)
 ; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; LA64-NEXT:    addi.d $sp, $sp, 16
@@ -145,7 +144,6 @@ define i64 @fptosi_i64_float(float %X) nounwind {
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.d $sp, $sp, -16
 ; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
-; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    bl %plt(__fixsfdi)
 ; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
 ; LA64-NEXT:    addi.d $sp, $sp, 16
-- 
GitLab


From 8fe39e64c0ef0a1aefce3c1187c5822343caeedd Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Thu, 16 May 2024 19:23:47 -0700
Subject: [PATCH 1076/1206] [clang-format] Don't always break before << between
 string literals (#92214)

Instead, leave the line wrapping as is.

Fixes #43887.
Fixes #44363.
---
 clang/lib/Format/TokenAnnotator.cpp   |  7 +++++--
 clang/unittests/Format/FormatTest.cpp | 11 +++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index d0aa0838423e..7c4c76a91f2c 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -5601,10 +5601,13 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
     return true;
   if (Left.IsUnterminatedLiteral)
     return true;
-  if (Right.is(tok::lessless) && AfterRight && Left.is(tok::string_literal) &&
+
+  if (BeforeLeft && BeforeLeft->is(tok::lessless) &&
+      Left.is(tok::string_literal) && Right.is(tok::lessless) && AfterRight &&
       AfterRight->is(tok::string_literal)) {
-    return true;
+    return Right.NewlinesBefore > 0;
   }
+
   if (Right.is(TT_RequiresClause)) {
     switch (Style.RequiresClausePosition) {
     case FormatStyle::RCPS_OwnLine:
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index e6f8e4a06515..6f57f10e12e8 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -10539,6 +10539,17 @@ TEST_F(FormatTest, KeepStringLabelValuePairsOnALine) {
       "                  bbbbbbbbbbbbbbbbbbbbbbb);");
 }
 
+TEST_F(FormatTest, WrapBeforeInsertionOperatorbetweenStringLiterals) {
+  verifyFormat("QStringList() << \"foo\" << \"bar\";");
+
+  verifyNoChange("QStringList() << \"foo\"\n"
+                 "              << \"bar\";");
+
+  verifyFormat("log_error(log, \"foo\" << \"bar\");",
+               "log_error(log, \"foo\"\n"
+               "                   << \"bar\");");
+}
+
 TEST_F(FormatTest, UnderstandsEquals) {
   verifyFormat(
       "aaaaaaaaaaaaaaaaa =\n"
-- 
GitLab


From 88d351e2e62d2ff291f3e6dea6b7e425f683285b Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Thu, 16 May 2024 19:24:35 -0700
Subject: [PATCH 1077/1206] [clang-format] Fix a regression in annotating
 struct braces (#92352)

Fixes #92350.
---
 clang/lib/Format/UnwrappedLineParser.cpp      | 12 +++++++-----
 clang/unittests/Format/TokenAnnotatorTest.cpp |  5 +++++
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 2236a49e4b76..b15a87327240 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -4008,8 +4008,6 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
   };
 
   if (FormatTok->isOneOf(tok::colon, tok::less)) {
-    if (FormatTok->is(tok::colon))
-      IsDerived = true;
     int AngleNestingLevel = 0;
     do {
       if (FormatTok->is(tok::less))
@@ -4017,9 +4015,13 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
       else if (FormatTok->is(tok::greater))
         --AngleNestingLevel;
 
-      if (AngleNestingLevel == 0 && FormatTok->is(tok::l_paren) &&
-          IsNonMacroIdentifier(FormatTok->Previous)) {
-        break;
+      if (AngleNestingLevel == 0) {
+        if (FormatTok->is(tok::colon)) {
+          IsDerived = true;
+        } else if (FormatTok->is(tok::l_paren) &&
+                   IsNonMacroIdentifier(FormatTok->Previous)) {
+          break;
+        }
       }
       if (FormatTok->is(tok::l_brace)) {
         if (AngleNestingLevel == 0 && IsListInitialization())
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 51b475d37977..aadfa6dc0165 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -2903,6 +2903,11 @@ TEST_F(TokenAnnotatorTest, BraceKind) {
   EXPECT_BRACE_KIND(Tokens[5], BK_Block);
   EXPECT_BRACE_KIND(Tokens[6], BK_Block);
 
+  Tokens = annotate("struct Foo<int> : Base {};");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_BRACE_KIND(Tokens[7], BK_Block);
+  EXPECT_BRACE_KIND(Tokens[8], BK_Block);
+
   Tokens = annotate("struct Foo final {};");
   ASSERT_EQ(Tokens.size(), 7u) << Tokens;
   EXPECT_BRACE_KIND(Tokens[3], BK_Block);
-- 
GitLab


From ebf283162f5a0e7e9392c3a825e060856eee0991 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 16 May 2024 19:31:43 -0700
Subject: [PATCH 1078/1206] [lldb] Include SBLanguages in the SWIG bindings
 (#92470)

---
 lldb/bindings/CMakeLists.txt  | 2 ++
 lldb/bindings/headers.swig    | 1 +
 lldb/bindings/interfaces.swig | 1 +
 3 files changed, 4 insertions(+)

diff --git a/lldb/bindings/CMakeLists.txt b/lldb/bindings/CMakeLists.txt
index 296eae1ae77f..bec694e43bd7 100644
--- a/lldb/bindings/CMakeLists.txt
+++ b/lldb/bindings/CMakeLists.txt
@@ -3,6 +3,7 @@ file(GLOB_RECURSE SWIG_SOURCES *.swig)
 file(GLOB SWIG_HEADERS
   ${LLDB_SOURCE_DIR}/include/lldb/API/*.h
   ${LLDB_SOURCE_DIR}/include/lldb/*.h
+  ${LLDB_BINARY_DIR}/include/lldb/API/SBLanguages.h
 )
 file(GLOB SWIG_PRIVATE_HEADERS
   ${LLDB_SOURCE_DIR}/include/lldb/lldb-private*.h
@@ -30,6 +31,7 @@ set(SWIG_COMMON_FLAGS
   -w361,362,509
   -features autodoc
   -I${LLDB_SOURCE_DIR}/include
+  -I${LLDB_BINARY_DIR}/include
   -I${CMAKE_CURRENT_SOURCE_DIR}
   ${DARWIN_EXTRAS}
 )
diff --git a/lldb/bindings/headers.swig b/lldb/bindings/headers.swig
index e8d0cda28814..ffdc3c31ec88 100644
--- a/lldb/bindings/headers.swig
+++ b/lldb/bindings/headers.swig
@@ -36,6 +36,7 @@
 #include "lldb/API/SBHostOS.h"
 #include "lldb/API/SBInstruction.h"
 #include "lldb/API/SBInstructionList.h"
+#include "lldb/API/SBLanguages.h"
 #include "lldb/API/SBLanguageRuntime.h"
 #include "lldb/API/SBLaunchInfo.h"
 #include "lldb/API/SBLineEntry.h"
diff --git a/lldb/bindings/interfaces.swig b/lldb/bindings/interfaces.swig
index a31a0b4af1eb..2a29a8dd7ef0 100644
--- a/lldb/bindings/interfaces.swig
+++ b/lldb/bindings/interfaces.swig
@@ -114,6 +114,7 @@
 %include "lldb/API/SBHostOS.h"
 %include "lldb/API/SBInstruction.h"
 %include "lldb/API/SBInstructionList.h"
+%include "lldb/API/SBLanguages.h"
 %include "lldb/API/SBLanguageRuntime.h"
 %include "lldb/API/SBLaunchInfo.h"
 %include "lldb/API/SBLineEntry.h"
-- 
GitLab


From 552927840319572a649cdec2d8bf2e688a5df490 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 16 May 2024 19:58:58 -0700
Subject: [PATCH 1079/1206] Revert "[lldb] Include SBLanguages in the SWIG
 bindings" (#92490)

Reverts llvm/llvm-project#92470
---
 lldb/bindings/CMakeLists.txt  | 2 --
 lldb/bindings/headers.swig    | 1 -
 lldb/bindings/interfaces.swig | 1 -
 3 files changed, 4 deletions(-)

diff --git a/lldb/bindings/CMakeLists.txt b/lldb/bindings/CMakeLists.txt
index bec694e43bd7..296eae1ae77f 100644
--- a/lldb/bindings/CMakeLists.txt
+++ b/lldb/bindings/CMakeLists.txt
@@ -3,7 +3,6 @@ file(GLOB_RECURSE SWIG_SOURCES *.swig)
 file(GLOB SWIG_HEADERS
   ${LLDB_SOURCE_DIR}/include/lldb/API/*.h
   ${LLDB_SOURCE_DIR}/include/lldb/*.h
-  ${LLDB_BINARY_DIR}/include/lldb/API/SBLanguages.h
 )
 file(GLOB SWIG_PRIVATE_HEADERS
   ${LLDB_SOURCE_DIR}/include/lldb/lldb-private*.h
@@ -31,7 +30,6 @@ set(SWIG_COMMON_FLAGS
   -w361,362,509
   -features autodoc
   -I${LLDB_SOURCE_DIR}/include
-  -I${LLDB_BINARY_DIR}/include
   -I${CMAKE_CURRENT_SOURCE_DIR}
   ${DARWIN_EXTRAS}
 )
diff --git a/lldb/bindings/headers.swig b/lldb/bindings/headers.swig
index ffdc3c31ec88..e8d0cda28814 100644
--- a/lldb/bindings/headers.swig
+++ b/lldb/bindings/headers.swig
@@ -36,7 +36,6 @@
 #include "lldb/API/SBHostOS.h"
 #include "lldb/API/SBInstruction.h"
 #include "lldb/API/SBInstructionList.h"
-#include "lldb/API/SBLanguages.h"
 #include "lldb/API/SBLanguageRuntime.h"
 #include "lldb/API/SBLaunchInfo.h"
 #include "lldb/API/SBLineEntry.h"
diff --git a/lldb/bindings/interfaces.swig b/lldb/bindings/interfaces.swig
index 2a29a8dd7ef0..a31a0b4af1eb 100644
--- a/lldb/bindings/interfaces.swig
+++ b/lldb/bindings/interfaces.swig
@@ -114,7 +114,6 @@
 %include "lldb/API/SBHostOS.h"
 %include "lldb/API/SBInstruction.h"
 %include "lldb/API/SBInstructionList.h"
-%include "lldb/API/SBLanguages.h"
 %include "lldb/API/SBLanguageRuntime.h"
 %include "lldb/API/SBLaunchInfo.h"
 %include "lldb/API/SBLineEntry.h"
-- 
GitLab


From 9f15aa009c36d2c108f0f2d09c2e9b283ebc4453 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Thu, 16 May 2024 20:02:51 -0700
Subject: [PATCH 1080/1206] [BOLT][NFC] Rename DataAggregator::BranchInfo to
 TakenBranchInfo

Align the name to its counterpart `FTInfo` which avoids name aliasing
with llvm::bolt::BranchInfo and allows to drop namespace specifier.

Test Plan: NFC

Reviewers: maksfb, rafaelauler, ayermolo, dcci

Reviewed By: dcci

Pull Request: https://github.com/llvm/llvm-project/pull/92017
---
 bolt/include/bolt/Profile/DataAggregator.h |  4 ++--
 bolt/lib/Profile/DataAggregator.cpp        | 12 ++++++------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h
index f2fa59bcaa1a..c158a9bb3e3f 100644
--- a/bolt/include/bolt/Profile/DataAggregator.h
+++ b/bolt/include/bolt/Profile/DataAggregator.h
@@ -122,14 +122,14 @@ private:
     uint64_t ExternCount{0};
   };
 
-  struct BranchInfo {
+  struct TakenBranchInfo {
     uint64_t TakenCount{0};
     uint64_t MispredCount{0};
   };
 
   /// Intermediate storage for profile data. We save the results of parsing
   /// and use them later for processing and assigning profile.
-  std::unordered_map<Trace, BranchInfo, TraceHash> BranchLBRs;
+  std::unordered_map<Trace, TakenBranchInfo, TraceHash> BranchLBRs;
   std::unordered_map<Trace, FTInfo, TraceHash> FallthroughLBRs;
   std::vector<AggregatedLBREntry> AggregatedLBRs;
   std::unordered_map<uint64_t, uint64_t> BasicSamples;
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 167899ccba12..e06debcee741 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -1464,7 +1464,7 @@ uint64_t DataAggregator::parseLBRSample(const PerfBranchSample &Sample,
     uint64_t To = getBinaryFunctionContainingAddress(LBR.To) ? LBR.To : 0;
     if (!From && !To)
       continue;
-    BranchInfo &Info = BranchLBRs[Trace(From, To)];
+    TakenBranchInfo &Info = BranchLBRs[Trace(From, To)];
     ++Info.TakenCount;
     Info.MispredCount += LBR.Mispred;
   }
@@ -1609,7 +1609,7 @@ void DataAggregator::processBranchEvents() {
 
   for (const auto &AggrLBR : BranchLBRs) {
     const Trace &Loc = AggrLBR.first;
-    const BranchInfo &Info = AggrLBR.second;
+    const TakenBranchInfo &Info = AggrLBR.second;
     doBranch(Loc.From, Loc.To, Info.TakenCount, Info.MispredCount);
   }
 }
@@ -2253,13 +2253,13 @@ DataAggregator::writeAggregatedFile(StringRef OutputFilename) const {
   } else {
     for (const auto &KV : NamesToBranches) {
       const FuncBranchData &FBD = KV.second;
-      for (const llvm::bolt::BranchInfo &BI : FBD.Data) {
+      for (const BranchInfo &BI : FBD.Data) {
         writeLocation(BI.From);
         writeLocation(BI.To);
         OutFile << BI.Mispreds << " " << BI.Branches << "\n";
         ++BranchValues;
       }
-      for (const llvm::bolt::BranchInfo &BI : FBD.EntryData) {
+      for (const BranchInfo &BI : FBD.EntryData) {
         // Do not output if source is a known symbol, since this was already
         // accounted for in the source function
         if (BI.From.IsSymbol)
@@ -2366,7 +2366,7 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
         return std::pair(BlockIt->first, BlockIt->second.getBBIndex());
       };
 
-      for (const llvm::bolt::BranchInfo &BI : Branches.Data) {
+      for (const BranchInfo &BI : Branches.Data) {
         using namespace yaml::bolt;
         const auto &[BlockOffset, BlockIndex] = getBlock(BI.From.Offset);
         BinaryBasicBlockProfile &YamlBB = YamlBF.Blocks[BlockIndex];
@@ -2388,7 +2388,7 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
         }
       }
       // Set entry counts, similar to DataReader::readProfile.
-      for (const llvm::bolt::BranchInfo &BI : Branches.EntryData) {
+      for (const BranchInfo &BI : Branches.EntryData) {
         if (!BlockMap.isInputBlock(BI.To.Offset)) {
           if (opts::Verbosity >= 1)
             errs() << "BOLT-WARNING: Unexpected EntryData in " << FuncName
-- 
GitLab


From c33922666ce219fd6cb3341c3394f72050599552 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 16 May 2024 20:47:12 -0700
Subject: [PATCH 1081/1206] [lldb] Use operator==(StringRef, StringRef) instead
 of StringRef::equals (NFC) (#92476)

Note that StringRef::equals has been deprecated in favor of
operator==(StringRef, StringRef).
---
 lldb/source/Commands/CommandObjectThread.cpp  |   4 +-
 lldb/source/Core/FormatEntity.cpp             |  12 +-
 lldb/source/Expression/IRExecutionUnit.cpp    |  34 ++---
 .../ExpressionParser/Clang/IRForTarget.cpp    |   8 +-
 lldb/source/Plugins/Language/ObjC/Cocoa.cpp   |   2 +-
 .../Plugins/ObjectFile/ELF/ObjectFileELF.cpp  |   2 +-
 .../GDBRemoteCommunicationClient.cpp          | 133 +++++++++---------
 .../GDBRemoteCommunicationServerCommon.cpp    |  22 +--
 .../GDBRemoteCommunicationServerPlatform.cpp  |   4 +-
 .../Process/gdb-remote/ProcessGDBRemote.cpp   |  26 ++--
 .../Plugins/SymbolFile/PDB/PDBASTParser.cpp   |   2 +-
 .../Plugins/SymbolFile/PDB/SymbolFilePDB.cpp  |   4 +-
 .../TypeSystem/Clang/TypeSystemClang.cpp      |  17 ++-
 lldb/source/Target/PathMappingList.cpp        |   4 +-
 14 files changed, 136 insertions(+), 138 deletions(-)

diff --git a/lldb/source/Commands/CommandObjectThread.cpp b/lldb/source/Commands/CommandObjectThread.cpp
index 3dbbfd4f9d34..4397ee14ea07 100644
--- a/lldb/source/Commands/CommandObjectThread.cpp
+++ b/lldb/source/Commands/CommandObjectThread.cpp
@@ -151,14 +151,14 @@ public:
 
     for (size_t idx = 0; idx < num_entries; idx++) {
       llvm::StringRef arg_string = copy_args[idx].ref();
-      if (arg_string.equals("-c") || count_opt.starts_with(arg_string)) {
+      if (arg_string == "-c" || count_opt.starts_with(arg_string)) {
         idx++;
         if (idx == num_entries)
           return std::nullopt;
         count_idx = idx;
         if (copy_args[idx].ref().getAsInteger(0, count_val))
           return std::nullopt;
-      } else if (arg_string.equals("-s") || start_opt.starts_with(arg_string)) {
+      } else if (arg_string == "-s" || start_opt.starts_with(arg_string)) {
         idx++;
         if (idx == num_entries)
           return std::nullopt;
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index 07978d388296..1c3a4cb1062f 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -1921,7 +1921,7 @@ static Status ParseEntry(const llvm::StringRef &format_str,
   const size_t n = parent->num_children;
   for (size_t i = 0; i < n; ++i) {
     const Definition *entry_def = parent->children + i;
-    if (key.equals(entry_def->name) || entry_def->name[0] == '*') {
+    if (key == entry_def->name || entry_def->name[0] == '*') {
       llvm::StringRef value;
       if (sep_char)
         value =
@@ -2002,7 +2002,7 @@ static const Definition *FindEntry(const llvm::StringRef &format_str,
   const size_t n = parent->num_children;
   for (size_t i = 0; i < n; ++i) {
     const Definition *entry_def = parent->children + i;
-    if (p.first.equals(entry_def->name) || entry_def->name[0] == '*') {
+    if (p.first == entry_def->name || entry_def->name[0] == '*') {
       if (p.second.empty()) {
         if (format_str.back() == '.')
           remainder = format_str.drop_front(format_str.size() - 1);
@@ -2351,13 +2351,13 @@ Status FormatEntity::ExtractVariableInfo(llvm::StringRef &format_str,
 bool FormatEntity::FormatFileSpec(const FileSpec &file_spec, Stream &s,
                                   llvm::StringRef variable_name,
                                   llvm::StringRef variable_format) {
-  if (variable_name.empty() || variable_name.equals(".fullpath")) {
+  if (variable_name.empty() || variable_name == ".fullpath") {
     file_spec.Dump(s.AsRawOstream());
     return true;
-  } else if (variable_name.equals(".basename")) {
+  } else if (variable_name == ".basename") {
     s.PutCString(file_spec.GetFilename().GetStringRef());
     return true;
-  } else if (variable_name.equals(".dirname")) {
+  } else if (variable_name == ".dirname") {
     s.PutCString(file_spec.GetFilename().GetStringRef());
     return true;
   }
@@ -2440,7 +2440,7 @@ void FormatEntity::AutoComplete(CompletionRequest &request) {
       // "${thread.id" <TAB>
       request.AddCompletion(MakeMatch(str, "}"));
     }
-  } else if (remainder.equals(".")) {
+  } else if (remainder == ".") {
     // "${thread." <TAB>
     StringList new_matches;
     AddMatches(entry_def, str, llvm::StringRef(), new_matches);
diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp
index 07df8c52a2a4..f22070442362 100644
--- a/lldb/source/Expression/IRExecutionUnit.cpp
+++ b/lldb/source/Expression/IRExecutionUnit.cpp
@@ -534,63 +534,63 @@ lldb::SectionType IRExecutionUnit::GetSectionTypeFromSectionName(
   }
 
   if (!name.empty()) {
-    if (name.equals("__text") || name.equals(".text"))
+    if (name == "__text" || name == ".text")
       sect_type = lldb::eSectionTypeCode;
-    else if (name.equals("__data") || name.equals(".data"))
+    else if (name == "__data" || name == ".data")
       sect_type = lldb::eSectionTypeCode;
     else if (name.starts_with("__debug_") || name.starts_with(".debug_")) {
       const uint32_t name_idx = name[0] == '_' ? 8 : 7;
       llvm::StringRef dwarf_name(name.substr(name_idx));
       switch (dwarf_name[0]) {
       case 'a':
-        if (dwarf_name.equals("abbrev"))
+        if (dwarf_name == "abbrev")
           sect_type = lldb::eSectionTypeDWARFDebugAbbrev;
-        else if (dwarf_name.equals("aranges"))
+        else if (dwarf_name == "aranges")
           sect_type = lldb::eSectionTypeDWARFDebugAranges;
-        else if (dwarf_name.equals("addr"))
+        else if (dwarf_name == "addr")
           sect_type = lldb::eSectionTypeDWARFDebugAddr;
         break;
 
       case 'f':
-        if (dwarf_name.equals("frame"))
+        if (dwarf_name == "frame")
           sect_type = lldb::eSectionTypeDWARFDebugFrame;
         break;
 
       case 'i':
-        if (dwarf_name.equals("info"))
+        if (dwarf_name == "info")
           sect_type = lldb::eSectionTypeDWARFDebugInfo;
         break;
 
       case 'l':
-        if (dwarf_name.equals("line"))
+        if (dwarf_name == "line")
           sect_type = lldb::eSectionTypeDWARFDebugLine;
-        else if (dwarf_name.equals("loc"))
+        else if (dwarf_name == "loc")
           sect_type = lldb::eSectionTypeDWARFDebugLoc;
-        else if (dwarf_name.equals("loclists"))
+        else if (dwarf_name == "loclists")
           sect_type = lldb::eSectionTypeDWARFDebugLocLists;
         break;
 
       case 'm':
-        if (dwarf_name.equals("macinfo"))
+        if (dwarf_name == "macinfo")
           sect_type = lldb::eSectionTypeDWARFDebugMacInfo;
         break;
 
       case 'p':
-        if (dwarf_name.equals("pubnames"))
+        if (dwarf_name == "pubnames")
           sect_type = lldb::eSectionTypeDWARFDebugPubNames;
-        else if (dwarf_name.equals("pubtypes"))
+        else if (dwarf_name == "pubtypes")
           sect_type = lldb::eSectionTypeDWARFDebugPubTypes;
         break;
 
       case 's':
-        if (dwarf_name.equals("str"))
+        if (dwarf_name == "str")
           sect_type = lldb::eSectionTypeDWARFDebugStr;
-        else if (dwarf_name.equals("str_offsets"))
+        else if (dwarf_name == "str_offsets")
           sect_type = lldb::eSectionTypeDWARFDebugStrOffsets;
         break;
 
       case 'r':
-        if (dwarf_name.equals("ranges"))
+        if (dwarf_name == "ranges")
           sect_type = lldb::eSectionTypeDWARFDebugRanges;
         break;
 
@@ -599,7 +599,7 @@ lldb::SectionType IRExecutionUnit::GetSectionTypeFromSectionName(
       }
     } else if (name.starts_with("__apple_") || name.starts_with(".apple_"))
       sect_type = lldb::eSectionTypeInvalid;
-    else if (name.equals("__objc_imageinfo"))
+    else if (name == "__objc_imageinfo")
       sect_type = lldb::eSectionTypeOther;
   }
   return sect_type;
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
index 597873af8b2a..cc9bd14c6194 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
@@ -1449,7 +1449,7 @@ bool IRForTarget::ReplaceVariables(Function &llvm_function) {
 
   Argument *argument = &*iter;
 
-  if (argument->getName().equals("this")) {
+  if (argument->getName() == "this") {
     ++iter;
 
     if (iter == llvm_function.arg_end()) {
@@ -1461,7 +1461,7 @@ bool IRForTarget::ReplaceVariables(Function &llvm_function) {
     }
 
     argument = &*iter;
-  } else if (argument->getName().equals("self")) {
+  } else if (argument->getName() == "self") {
     ++iter;
 
     if (iter == llvm_function.arg_end()) {
@@ -1472,7 +1472,7 @@ bool IRForTarget::ReplaceVariables(Function &llvm_function) {
       return false;
     }
 
-    if (!iter->getName().equals("_cmd")) {
+    if (iter->getName() != "_cmd") {
       m_error_stream.Format("Internal error [IRForTarget]: Wrapper takes '{0}' "
                             "after 'self' argument (should take '_cmd')",
                             iter->getName());
@@ -1493,7 +1493,7 @@ bool IRForTarget::ReplaceVariables(Function &llvm_function) {
     argument = &*iter;
   }
 
-  if (!argument->getName().equals("$__lldb_arg")) {
+  if (argument->getName() != "$__lldb_arg") {
     m_error_stream.Format("Internal error [IRForTarget]: Wrapper takes an "
                           "argument named '{0}' instead of the struct pointer",
                           argument->getName());
diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
index 96166657ceeb..341923108e32 100644
--- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
+++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
@@ -802,7 +802,7 @@ bool lldb_private::formatters::NSURLSummaryProvider(
 
   llvm::StringRef class_name = descriptor->GetClassName().GetStringRef();
 
-  if (!class_name.equals("NSURL"))
+  if (class_name != "NSURL")
     return false;
 
   uint64_t offset_text = ptr_size + ptr_size +
diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
index d88f2d083019..5c6b475044be 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
@@ -3557,7 +3557,7 @@ ObjectFile::Strata ObjectFileELF::CalculateStrata() {
           // decrease by one
           llvm::StringRef loader_name(buffer, read_size - 1);
           llvm::StringRef freebsd_kernel_loader_name("/red/herring");
-          if (loader_name.equals(freebsd_kernel_loader_name))
+          if (loader_name == freebsd_kernel_loader_name)
             return eStrataKernel;
         }
       }
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
index db9fb37a9a3c..74e392249a94 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
@@ -1052,10 +1052,10 @@ bool GDBRemoteCommunicationClient::GetGDBServerVersion() {
         llvm::StringRef name, value;
         bool success = false;
         while (response.GetNameColonValue(name, value)) {
-          if (name.equals("name")) {
+          if (name == "name") {
             success = true;
             m_gdb_server_name = std::string(value);
-          } else if (name.equals("version")) {
+          } else if (name == "version") {
             llvm::StringRef major, minor;
             std::tie(major, minor) = value.split('.');
             if (!major.getAsInteger(0, m_gdb_server_version))
@@ -1192,12 +1192,12 @@ bool GDBRemoteCommunicationClient::GetDefaultThreadId(lldb::tid_t &tid) {
 
 static void ParseOSType(llvm::StringRef value, std::string &os_name,
                         std::string &environment) {
-  if (value.equals("iossimulator") || value.equals("tvossimulator") ||
-      value.equals("watchossimulator") || value.equals("xrossimulator") ||
-      value.equals("visionossimulator")) {
+  if (value == "iossimulator" || value == "tvossimulator" ||
+      value == "watchossimulator" || value == "xrossimulator" ||
+      value == "visionossimulator") {
     environment = "simulator";
     os_name = value.drop_back(environment.size()).str();
-  } else if (value.equals("maccatalyst")) {
+  } else if (value == "maccatalyst") {
     os_name = "ios";
     environment = "macabi";
   } else {
@@ -1230,44 +1230,44 @@ bool GDBRemoteCommunicationClient::GetHostInfo(bool force) {
         ByteOrder byte_order = eByteOrderInvalid;
         uint32_t num_keys_decoded = 0;
         while (response.GetNameColonValue(name, value)) {
-          if (name.equals("cputype")) {
+          if (name == "cputype") {
             // exception type in big endian hex
             if (!value.getAsInteger(0, cpu))
               ++num_keys_decoded;
-          } else if (name.equals("cpusubtype")) {
+          } else if (name == "cpusubtype") {
             // exception count in big endian hex
             if (!value.getAsInteger(0, sub))
               ++num_keys_decoded;
-          } else if (name.equals("arch")) {
+          } else if (name == "arch") {
             arch_name = std::string(value);
             ++num_keys_decoded;
-          } else if (name.equals("triple")) {
+          } else if (name == "triple") {
             StringExtractor extractor(value);
             extractor.GetHexByteString(triple);
             ++num_keys_decoded;
-          } else if (name.equals("distribution_id")) {
+          } else if (name == "distribution_id") {
             StringExtractor extractor(value);
             extractor.GetHexByteString(m_host_distribution_id);
             ++num_keys_decoded;
-          } else if (name.equals("os_build")) {
+          } else if (name == "os_build") {
             StringExtractor extractor(value);
             extractor.GetHexByteString(m_os_build);
             ++num_keys_decoded;
-          } else if (name.equals("hostname")) {
+          } else if (name == "hostname") {
             StringExtractor extractor(value);
             extractor.GetHexByteString(m_hostname);
             ++num_keys_decoded;
-          } else if (name.equals("os_kernel")) {
+          } else if (name == "os_kernel") {
             StringExtractor extractor(value);
             extractor.GetHexByteString(m_os_kernel);
             ++num_keys_decoded;
-          } else if (name.equals("ostype")) {
+          } else if (name == "ostype") {
             ParseOSType(value, os_name, environment);
             ++num_keys_decoded;
-          } else if (name.equals("vendor")) {
+          } else if (name == "vendor") {
             vendor_name = std::string(value);
             ++num_keys_decoded;
-          } else if (name.equals("endian")) {
+          } else if (name == "endian") {
             byte_order = llvm::StringSwitch<lldb::ByteOrder>(value)
                              .Case("little", eByteOrderLittle)
                              .Case("big", eByteOrderBig)
@@ -1275,30 +1275,30 @@ bool GDBRemoteCommunicationClient::GetHostInfo(bool force) {
                              .Default(eByteOrderInvalid);
             if (byte_order != eByteOrderInvalid)
               ++num_keys_decoded;
-          } else if (name.equals("ptrsize")) {
+          } else if (name == "ptrsize") {
             if (!value.getAsInteger(0, pointer_byte_size))
               ++num_keys_decoded;
-          } else if (name.equals("addressing_bits")) {
+          } else if (name == "addressing_bits") {
             if (!value.getAsInteger(0, m_low_mem_addressing_bits)) {
               ++num_keys_decoded;
             }
-          } else if (name.equals("high_mem_addressing_bits")) {
+          } else if (name == "high_mem_addressing_bits") {
             if (!value.getAsInteger(0, m_high_mem_addressing_bits))
               ++num_keys_decoded;
-          } else if (name.equals("low_mem_addressing_bits")) {
+          } else if (name == "low_mem_addressing_bits") {
             if (!value.getAsInteger(0, m_low_mem_addressing_bits))
               ++num_keys_decoded;
-          } else if (name.equals("os_version") ||
-                     name.equals("version")) // Older debugserver binaries used
-                                             // the "version" key instead of
-                                             // "os_version"...
+          } else if (name == "os_version" ||
+                     name == "version") // Older debugserver binaries used
+                                        // the "version" key instead of
+                                        // "os_version"...
           {
             if (!m_os_version.tryParse(value))
               ++num_keys_decoded;
-          } else if (name.equals("maccatalyst_version")) {
+          } else if (name == "maccatalyst_version") {
             if (!m_maccatalyst_version.tryParse(value))
               ++num_keys_decoded;
-          } else if (name.equals("watchpoint_exceptions_received")) {
+          } else if (name == "watchpoint_exceptions_received") {
             m_watchpoints_trigger_after_instruction =
                 llvm::StringSwitch<LazyBool>(value)
                     .Case("before", eLazyBoolNo)
@@ -1306,14 +1306,14 @@ bool GDBRemoteCommunicationClient::GetHostInfo(bool force) {
                     .Default(eLazyBoolCalculate);
             if (m_watchpoints_trigger_after_instruction != eLazyBoolCalculate)
               ++num_keys_decoded;
-          } else if (name.equals("default_packet_timeout")) {
+          } else if (name == "default_packet_timeout") {
             uint32_t timeout_seconds;
             if (!value.getAsInteger(0, timeout_seconds)) {
               m_default_packet_timeout = seconds(timeout_seconds);
               SetPacketTimeout(m_default_packet_timeout);
               ++num_keys_decoded;
             }
-          } else if (name.equals("vm-page-size")) {
+          } else if (name == "vm-page-size") {
             int page_size;
             if (!value.getAsInteger(0, page_size)) {
               m_target_vm_page_size = page_size;
@@ -1568,10 +1568,10 @@ Status GDBRemoteCommunicationClient::GetMemoryRegionInfo(
       bool success = true;
       bool saw_permissions = false;
       while (success && response.GetNameColonValue(name, value)) {
-        if (name.equals("start")) {
+        if (name == "start") {
           if (!value.getAsInteger(16, addr_value))
             region_info.GetRange().SetRangeBase(addr_value);
-        } else if (name.equals("size")) {
+        } else if (name == "size") {
           if (!value.getAsInteger(16, addr_value)) {
             region_info.GetRange().SetByteSize(addr_value);
             if (region_info.GetRange().GetRangeEnd() <
@@ -1580,8 +1580,7 @@ Status GDBRemoteCommunicationClient::GetMemoryRegionInfo(
               region_info.GetRange().SetRangeEnd(LLDB_INVALID_ADDRESS);
             }
           }
-        } else if (name.equals("permissions") &&
-                   region_info.GetRange().IsValid()) {
+        } else if (name == "permissions" && region_info.GetRange().IsValid()) {
           saw_permissions = true;
           if (region_info.GetRange().Contains(addr)) {
             if (value.contains('r'))
@@ -1608,12 +1607,12 @@ Status GDBRemoteCommunicationClient::GetMemoryRegionInfo(
             region_info.SetExecutable(MemoryRegionInfo::eNo);
             region_info.SetMapped(MemoryRegionInfo::eNo);
           }
-        } else if (name.equals("name")) {
+        } else if (name == "name") {
           StringExtractorGDBRemote name_extractor(value);
           std::string name;
           name_extractor.GetHexByteString(name);
           region_info.SetName(name.c_str());
-        } else if (name.equals("flags")) {
+        } else if (name == "flags") {
           region_info.SetMemoryTagged(MemoryRegionInfo::eNo);
 
           llvm::StringRef flags = value;
@@ -1629,7 +1628,7 @@ Status GDBRemoteCommunicationClient::GetMemoryRegionInfo(
               }
             }
           }
-        } else if (name.equals("type")) {
+        } else if (name == "type") {
           std::string comma_sep_str = value.str();
           size_t comma_pos;
           while ((comma_pos = comma_sep_str.find(',')) != std::string::npos) {
@@ -1642,13 +1641,13 @@ Status GDBRemoteCommunicationClient::GetMemoryRegionInfo(
           if (comma_sep_str == "stack") {
             region_info.SetIsStackMemory(MemoryRegionInfo::eYes);
           }
-        } else if (name.equals("error")) {
+        } else if (name == "error") {
           StringExtractorGDBRemote error_extractor(value);
           std::string error_string;
           // Now convert the HEX bytes into a string value
           error_extractor.GetHexByteString(error_string);
           error.SetErrorString(error_string.c_str());
-        } else if (name.equals("dirty-pages")) {
+        } else if (name == "dirty-pages") {
           std::vector<addr_t> dirty_page_list;
           for (llvm::StringRef x : llvm::split(value, ',')) {
             addr_t page;
@@ -1825,7 +1824,7 @@ std::optional<uint32_t> GDBRemoteCommunicationClient::GetWatchpointSlotCount() {
       llvm::StringRef name;
       llvm::StringRef value;
       while (response.GetNameColonValue(name, value)) {
-        if (name.equals("num")) {
+        if (name == "num") {
           value.getAsInteger(0, m_num_supported_hardware_watchpoints);
           num = m_num_supported_hardware_watchpoints;
         }
@@ -2006,43 +2005,43 @@ bool GDBRemoteCommunicationClient::DecodeProcessInfoResponse(
     std::string os_type;
 
     while (response.GetNameColonValue(name, value)) {
-      if (name.equals("pid")) {
+      if (name == "pid") {
         lldb::pid_t pid = LLDB_INVALID_PROCESS_ID;
         value.getAsInteger(0, pid);
         process_info.SetProcessID(pid);
-      } else if (name.equals("ppid")) {
+      } else if (name == "ppid") {
         lldb::pid_t pid = LLDB_INVALID_PROCESS_ID;
         value.getAsInteger(0, pid);
         process_info.SetParentProcessID(pid);
-      } else if (name.equals("uid")) {
+      } else if (name == "uid") {
         uint32_t uid = UINT32_MAX;
         value.getAsInteger(0, uid);
         process_info.SetUserID(uid);
-      } else if (name.equals("euid")) {
+      } else if (name == "euid") {
         uint32_t uid = UINT32_MAX;
         value.getAsInteger(0, uid);
         process_info.SetEffectiveUserID(uid);
-      } else if (name.equals("gid")) {
+      } else if (name == "gid") {
         uint32_t gid = UINT32_MAX;
         value.getAsInteger(0, gid);
         process_info.SetGroupID(gid);
-      } else if (name.equals("egid")) {
+      } else if (name == "egid") {
         uint32_t gid = UINT32_MAX;
         value.getAsInteger(0, gid);
         process_info.SetEffectiveGroupID(gid);
-      } else if (name.equals("triple")) {
+      } else if (name == "triple") {
         StringExtractor extractor(value);
         std::string triple;
         extractor.GetHexByteString(triple);
         process_info.GetArchitecture().SetTriple(triple.c_str());
-      } else if (name.equals("name")) {
+      } else if (name == "name") {
         StringExtractor extractor(value);
         // The process name from ASCII hex bytes since we can't control the
         // characters in a process name
         std::string name;
         extractor.GetHexByteString(name);
         process_info.GetExecutableFile().SetFile(name, FileSpec::Style::native);
-      } else if (name.equals("args")) {
+      } else if (name == "args") {
         llvm::StringRef encoded_args(value), hex_arg;
 
         bool is_arg0 = true;
@@ -2062,13 +2061,13 @@ bool GDBRemoteCommunicationClient::DecodeProcessInfoResponse(
             process_info.GetArguments().AppendArgument(arg);
           is_arg0 = false;
         }
-      } else if (name.equals("cputype")) {
+      } else if (name == "cputype") {
         value.getAsInteger(0, cpu);
-      } else if (name.equals("cpusubtype")) {
+      } else if (name == "cpusubtype") {
         value.getAsInteger(0, sub);
-      } else if (name.equals("vendor")) {
+      } else if (name == "vendor") {
         vendor = std::string(value);
-      } else if (name.equals("ostype")) {
+      } else if (name == "ostype") {
         os_type = std::string(value);
       }
     }
@@ -2144,10 +2143,10 @@ bool GDBRemoteCommunicationClient::GetCurrentProcessInfo(bool allow_lazy) {
       uint32_t num_keys_decoded = 0;
       lldb::pid_t pid = LLDB_INVALID_PROCESS_ID;
       while (response.GetNameColonValue(name, value)) {
-        if (name.equals("cputype")) {
+        if (name == "cputype") {
           if (!value.getAsInteger(16, cpu))
             ++num_keys_decoded;
-        } else if (name.equals("cpusubtype")) {
+        } else if (name == "cpusubtype") {
           if (!value.getAsInteger(16, sub)) {
             ++num_keys_decoded;
             // Workaround for pre-2024 Apple debugserver, which always
@@ -2162,17 +2161,17 @@ bool GDBRemoteCommunicationClient::GetCurrentProcessInfo(bool allow_lazy) {
                   sub = 0;
             }
           }
-        } else if (name.equals("triple")) {
+        } else if (name == "triple") {
           StringExtractor extractor(value);
           extractor.GetHexByteString(triple);
           ++num_keys_decoded;
-        } else if (name.equals("ostype")) {
+        } else if (name == "ostype") {
           ParseOSType(value, os_name, environment);
           ++num_keys_decoded;
-        } else if (name.equals("vendor")) {
+        } else if (name == "vendor") {
           vendor_name = std::string(value);
           ++num_keys_decoded;
-        } else if (name.equals("endian")) {
+        } else if (name == "endian") {
           byte_order = llvm::StringSwitch<lldb::ByteOrder>(value)
                            .Case("little", eByteOrderLittle)
                            .Case("big", eByteOrderBig)
@@ -2180,19 +2179,19 @@ bool GDBRemoteCommunicationClient::GetCurrentProcessInfo(bool allow_lazy) {
                            .Default(eByteOrderInvalid);
           if (byte_order != eByteOrderInvalid)
             ++num_keys_decoded;
-        } else if (name.equals("ptrsize")) {
+        } else if (name == "ptrsize") {
           if (!value.getAsInteger(16, pointer_byte_size))
             ++num_keys_decoded;
-        } else if (name.equals("pid")) {
+        } else if (name == "pid") {
           if (!value.getAsInteger(16, pid))
             ++num_keys_decoded;
-        } else if (name.equals("elf_abi")) {
+        } else if (name == "elf_abi") {
           elf_abi = std::string(value);
           ++num_keys_decoded;
-        } else if (name.equals("main-binary-uuid")) {
+        } else if (name == "main-binary-uuid") {
           m_process_standalone_uuid.SetFromStringRef(value);
           ++num_keys_decoded;
-        } else if (name.equals("main-binary-slide")) {
+        } else if (name == "main-binary-slide") {
           StringExtractor extractor(value);
           m_process_standalone_value =
               extractor.GetU64(LLDB_INVALID_ADDRESS, 16);
@@ -2200,7 +2199,7 @@ bool GDBRemoteCommunicationClient::GetCurrentProcessInfo(bool allow_lazy) {
             m_process_standalone_value_is_offset = true;
             ++num_keys_decoded;
           }
-        } else if (name.equals("main-binary-address")) {
+        } else if (name == "main-binary-address") {
           StringExtractor extractor(value);
           m_process_standalone_value =
               extractor.GetU64(LLDB_INVALID_ADDRESS, 16);
@@ -2208,7 +2207,7 @@ bool GDBRemoteCommunicationClient::GetCurrentProcessInfo(bool allow_lazy) {
             m_process_standalone_value_is_offset = false;
             ++num_keys_decoded;
           }
-        } else if (name.equals("binary-addresses")) {
+        } else if (name == "binary-addresses") {
           m_binary_addresses.clear();
           ++num_keys_decoded;
           for (llvm::StringRef x : llvm::split(value, ',')) {
@@ -2647,9 +2646,9 @@ bool GDBRemoteCommunicationClient::LaunchGDBServer(
     llvm::StringRef name;
     llvm::StringRef value;
     while (response.GetNameColonValue(name, value)) {
-      if (name.equals("port"))
+      if (name == "port")
         value.getAsInteger(0, port);
-      else if (name.equals("pid"))
+      else if (name == "pid")
         value.getAsInteger(0, pid);
       else if (name.compare("socket_name") == 0) {
         StringExtractor extractor(value);
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
index b4fb5b68dd41..f9d37490e16a 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
@@ -340,13 +340,13 @@ GDBRemoteCommunicationServerCommon::Handle_qfProcessInfo(
     llvm::StringRef value;
     while (packet.GetNameColonValue(key, value)) {
       bool success = true;
-      if (key.equals("name")) {
+      if (key == "name") {
         StringExtractor extractor(value);
         std::string file;
         extractor.GetHexByteString(file);
         match_info.GetProcessInfo().GetExecutableFile().SetFile(
             file, FileSpec::Style::native);
-      } else if (key.equals("name_match")) {
+      } else if (key == "name_match") {
         NameMatch name_match = llvm::StringSwitch<NameMatch>(value)
                                    .Case("equals", NameMatch::Equals)
                                    .Case("starts_with", NameMatch::StartsWith)
@@ -357,40 +357,40 @@ GDBRemoteCommunicationServerCommon::Handle_qfProcessInfo(
         match_info.SetNameMatchType(name_match);
         if (name_match == NameMatch::Ignore)
           return SendErrorResponse(2);
-      } else if (key.equals("pid")) {
+      } else if (key == "pid") {
         lldb::pid_t pid = LLDB_INVALID_PROCESS_ID;
         if (value.getAsInteger(0, pid))
           return SendErrorResponse(2);
         match_info.GetProcessInfo().SetProcessID(pid);
-      } else if (key.equals("parent_pid")) {
+      } else if (key == "parent_pid") {
         lldb::pid_t pid = LLDB_INVALID_PROCESS_ID;
         if (value.getAsInteger(0, pid))
           return SendErrorResponse(2);
         match_info.GetProcessInfo().SetParentProcessID(pid);
-      } else if (key.equals("uid")) {
+      } else if (key == "uid") {
         uint32_t uid = UINT32_MAX;
         if (value.getAsInteger(0, uid))
           return SendErrorResponse(2);
         match_info.GetProcessInfo().SetUserID(uid);
-      } else if (key.equals("gid")) {
+      } else if (key == "gid") {
         uint32_t gid = UINT32_MAX;
         if (value.getAsInteger(0, gid))
           return SendErrorResponse(2);
         match_info.GetProcessInfo().SetGroupID(gid);
-      } else if (key.equals("euid")) {
+      } else if (key == "euid") {
         uint32_t uid = UINT32_MAX;
         if (value.getAsInteger(0, uid))
           return SendErrorResponse(2);
         match_info.GetProcessInfo().SetEffectiveUserID(uid);
-      } else if (key.equals("egid")) {
+      } else if (key == "egid") {
         uint32_t gid = UINT32_MAX;
         if (value.getAsInteger(0, gid))
           return SendErrorResponse(2);
         match_info.GetProcessInfo().SetEffectiveGroupID(gid);
-      } else if (key.equals("all_users")) {
+      } else if (key == "all_users") {
         match_info.SetMatchAllUsers(
             OptionArgParser::ToBoolean(value, false, &success));
-      } else if (key.equals("triple")) {
+      } else if (key == "triple") {
         match_info.GetProcessInfo().GetArchitecture() =
             HostInfo::GetAugmentedArchSpec(value);
       } else {
@@ -472,7 +472,7 @@ GDBRemoteCommunicationServerCommon::Handle_qSpeedTest(
   llvm::StringRef key;
   llvm::StringRef value;
   bool success = packet.GetNameColonValue(key, value);
-  if (success && key.equals("response_size")) {
+  if (success && key == "response_size") {
     uint32_t response_size = 0;
     if (!value.getAsInteger(0, response_size)) {
       if (response_size == 0)
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
index 391abdae2752..5285ec1d3db4 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
@@ -239,9 +239,9 @@ GDBRemoteCommunicationServerPlatform::Handle_qLaunchGDBServer(
   llvm::StringRef value;
   std::optional<uint16_t> port;
   while (packet.GetNameColonValue(name, value)) {
-    if (name.equals("host"))
+    if (name == "host")
       hostname = std::string(value);
-    else if (name.equals("port")) {
+    else if (name == "port") {
       // Make the Optional valid so we can use its value
       port = 0;
       value.getAsInteger(0, *port);
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 871683a60568..a5a731981299 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -448,20 +448,20 @@ void ProcessGDBRemote::BuildDynamicRegisterInfo(bool force) {
         DynamicRegisterInfo::Register reg_info;
 
         while (response.GetNameColonValue(name, value)) {
-          if (name.equals("name")) {
+          if (name == "name") {
             reg_info.name.SetString(value);
-          } else if (name.equals("alt-name")) {
+          } else if (name == "alt-name") {
             reg_info.alt_name.SetString(value);
-          } else if (name.equals("bitsize")) {
+          } else if (name == "bitsize") {
             if (!value.getAsInteger(0, reg_info.byte_size))
               reg_info.byte_size /= CHAR_BIT;
-          } else if (name.equals("offset")) {
+          } else if (name == "offset") {
             value.getAsInteger(0, reg_info.byte_offset);
-          } else if (name.equals("encoding")) {
+          } else if (name == "encoding") {
             const Encoding encoding = Args::StringToEncoding(value);
             if (encoding != eEncodingInvalid)
               reg_info.encoding = encoding;
-          } else if (name.equals("format")) {
+          } else if (name == "format") {
             if (!OptionArgParser::ToFormat(value.str().c_str(), reg_info.format, nullptr)
                     .Success())
               reg_info.format =
@@ -480,17 +480,17 @@ void ProcessGDBRemote::BuildDynamicRegisterInfo(bool force) {
                       .Case("vector-uint64", eFormatVectorOfUInt64)
                       .Case("vector-uint128", eFormatVectorOfUInt128)
                       .Default(eFormatInvalid);
-          } else if (name.equals("set")) {
+          } else if (name == "set") {
             reg_info.set_name.SetString(value);
-          } else if (name.equals("gcc") || name.equals("ehframe")) {
+          } else if (name == "gcc" || name == "ehframe") {
             value.getAsInteger(0, reg_info.regnum_ehframe);
-          } else if (name.equals("dwarf")) {
+          } else if (name == "dwarf") {
             value.getAsInteger(0, reg_info.regnum_dwarf);
-          } else if (name.equals("generic")) {
+          } else if (name == "generic") {
             reg_info.regnum_generic = Args::StringToGenericRegister(value);
-          } else if (name.equals("container-regs")) {
+          } else if (name == "container-regs") {
             SplitCommaSeparatedRegisterNumberString(value, reg_info.value_regs, 16);
-          } else if (name.equals("invalidate-regs")) {
+          } else if (name == "invalidate-regs") {
             SplitCommaSeparatedRegisterNumberString(value, reg_info.invalidate_regs, 16);
           }
         }
@@ -5082,7 +5082,7 @@ std::string ProcessGDBRemote::HarmonizeThreadIdsForProfileData(
       llvm::StringRef usec_name, usec_value;
       uint32_t input_file_pos = profileDataExtractor.GetFilePos();
       if (profileDataExtractor.GetNameColonValue(usec_name, usec_value)) {
-        if (usec_name.equals("thread_used_usec")) {
+        if (usec_name == "thread_used_usec") {
           has_used_usec = true;
           usec_value.getAsInteger(0, curr_used_usec);
         } else {
diff --git a/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp b/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
index e915bff9e4a4..d656ca3facf7 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
+++ b/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
@@ -1139,7 +1139,7 @@ PDBASTParser::FindNamespaceDecl(const clang::DeclContext *parent,
   assert(set);
 
   for (clang::NamespaceDecl *namespace_decl : *set)
-    if (namespace_decl->getName().equals(name))
+    if (namespace_decl->getName() == name)
       return namespace_decl;
 
   for (clang::NamespaceDecl *namespace_decl : *set)
diff --git a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
index b26beecc6d12..9a282acae91f 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
+++ b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
@@ -1141,8 +1141,8 @@ void SymbolFilePDB::FindGlobalVariables(
     sc.module_sp = m_objfile_sp->GetModule();
     lldbassert(sc.module_sp.get());
 
-    if (!name.GetStringRef().equals(
-            MSVCUndecoratedNameParser::DropScope(pdb_data->getName())))
+    if (name.GetStringRef() !=
+        MSVCUndecoratedNameParser::DropScope(pdb_data->getName()))
       continue;
 
     sc.comp_unit = ParseCompileUnitForUID(GetCompilandId(*pdb_data)).get();
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 17a9c675fbba..582d9eac3e1d 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -6692,7 +6692,7 @@ size_t TypeSystemClang::GetIndexOfChildMemberWithName(
               return child_indexes.size();
             child_indexes.pop_back();
 
-          } else if (field_name.equals(name)) {
+          } else if (field_name == name) {
             // We have to add on the number of base classes to this index!
             child_indexes.push_back(
                 child_idx + TypeSystemClang::GetNumBaseClasses(
@@ -6779,7 +6779,7 @@ size_t TypeSystemClang::GetIndexOfChildMemberWithName(
                  ivar_pos != ivar_end; ++ivar_pos, ++child_idx) {
               const clang::ObjCIvarDecl *ivar_decl = *ivar_pos;
 
-              if (ivar_decl->getName().equals(name_sref)) {
+              if (ivar_decl->getName() == name_sref) {
                 if ((!omit_empty_base_classes && superclass_interface_decl) ||
                     (omit_empty_base_classes &&
                      ObjCDeclHasIVars(superclass_interface_decl, true)))
@@ -6948,7 +6948,7 @@ TypeSystemClang::GetIndexOfChildWithName(lldb::opaque_compiler_type_t type,
         for (field = record_decl->field_begin(),
             field_end = record_decl->field_end();
              field != field_end; ++field, ++child_idx) {
-          if (field->getName().equals(name))
+          if (field->getName() == name)
             return child_idx;
         }
       }
@@ -6975,7 +6975,7 @@ TypeSystemClang::GetIndexOfChildWithName(lldb::opaque_compiler_type_t type,
                  ivar_pos != ivar_end; ++ivar_pos, ++child_idx) {
               const clang::ObjCIvarDecl *ivar_decl = *ivar_pos;
 
-              if (ivar_decl->getName().equals(name)) {
+              if (ivar_decl->getName() == name) {
                 if ((!omit_empty_base_classes && superclass_interface_decl) ||
                     (omit_empty_base_classes &&
                      ObjCDeclHasIVars(superclass_interface_decl, true)))
@@ -6986,7 +6986,7 @@ TypeSystemClang::GetIndexOfChildWithName(lldb::opaque_compiler_type_t type,
             }
 
             if (superclass_interface_decl) {
-              if (superclass_interface_decl->getName().equals(name))
+              if (superclass_interface_decl->getName() == name)
                 return 0;
             }
           }
@@ -9231,15 +9231,14 @@ std::vector<CompilerDecl> TypeSystemClang::DeclContextFindDeclByName(
               if (clang::NamedDecl *nd =
                       llvm::dyn_cast<clang::NamedDecl>(target)) {
                 IdentifierInfo *ii = nd->getIdentifier();
-                if (ii != nullptr &&
-                    ii->getName().equals(name.AsCString(nullptr)))
+                if (ii != nullptr && ii->getName() == name.AsCString(nullptr))
                   found_decls.push_back(GetCompilerDecl(nd));
               }
             }
           } else if (clang::NamedDecl *nd =
                          llvm::dyn_cast<clang::NamedDecl>(child)) {
             IdentifierInfo *ii = nd->getIdentifier();
-            if (ii != nullptr && ii->getName().equals(name.AsCString(nullptr)))
+            if (ii != nullptr && ii->getName() == name.AsCString(nullptr))
               found_decls.push_back(GetCompilerDecl(nd));
           }
         }
@@ -9349,7 +9348,7 @@ uint32_t TypeSystemClang::CountDeclLevels(clang::DeclContext *frame_decl_ctx,
                 // Check names.
                 IdentifierInfo *ii = nd->getIdentifier();
                 if (ii == nullptr ||
-                    !ii->getName().equals(child_name->AsCString(nullptr)))
+                    ii->getName() != child_name->AsCString(nullptr))
                   continue;
                 // Check types, if one was provided.
                 if (child_type) {
diff --git a/lldb/source/Target/PathMappingList.cpp b/lldb/source/Target/PathMappingList.cpp
index c369018122a5..9c283b0146fe 100644
--- a/lldb/source/Target/PathMappingList.cpp
+++ b/lldb/source/Target/PathMappingList.cpp
@@ -86,8 +86,8 @@ bool PathMappingList::AppendUnique(llvm::StringRef path,
   auto normalized_replacement = NormalizePath(replacement);
   std::lock_guard<std::recursive_mutex> lock(m_mutex);
   for (const auto &pair : m_pairs) {
-    if (pair.first.GetStringRef().equals(normalized_path) &&
-        pair.second.GetStringRef().equals(normalized_replacement))
+    if (pair.first.GetStringRef() == normalized_path &&
+        pair.second.GetStringRef() == normalized_replacement)
       return false;
   }
   Append(path, replacement, notify);
-- 
GitLab


From 5d8354c009d5625fa140be47de1f91275f4698d3 Mon Sep 17 00:00:00 2001
From: harishch4 <harishcse44@gmail.com>
Date: Fri, 17 May 2024 09:20:52 +0530
Subject: [PATCH 1082/1206] [Flang][OpenMP]Missing convert to lhsType in atomic
 write (#92346)

Fixes test.f90 in #83144.

This issue is observed only when a boolean constant is assigned to a
logical variable. In non-openmp flow, a conversion op is inserted before
assigning it to a logical variable. This patch will insert a fir.convert
operation when the types are not the same, before generating the atomic
write operation.

I've proposed another patch(#85059 ) which removes checks at MLIR level
and looks like it's too permissive. I'm planning to abandon this patch
and address it here.
---
 flang/lib/Lower/DirectivesCommon.h       |  3 +++
 flang/test/Lower/OpenMP/atomic-write.f90 | 14 ++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/flang/lib/Lower/DirectivesCommon.h b/flang/lib/Lower/DirectivesCommon.h
index 42bd3868196b..48b090f6d2db 100644
--- a/flang/lib/Lower/DirectivesCommon.h
+++ b/flang/lib/Lower/DirectivesCommon.h
@@ -180,6 +180,9 @@ static inline void genOmpAccAtomicWriteStatement(
   // Generate `atomic.write` operation for atomic assignment statements
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
+  mlir::Type varType = fir::unwrapRefType(lhsAddr.getType());
+  rhsExpr = firOpBuilder.createConvert(loc, varType, rhsExpr);
+
   if constexpr (std::is_same<AtomicListT,
                              Fortran::parser::OmpAtomicClauseList>()) {
     // If no hint clause is specified, the effect is as if
diff --git a/flang/test/Lower/OpenMP/atomic-write.f90 b/flang/test/Lower/OpenMP/atomic-write.f90
index 85955af64bbe..e0fa802976d9 100644
--- a/flang/test/Lower/OpenMP/atomic-write.f90
+++ b/flang/test/Lower/OpenMP/atomic-write.f90
@@ -73,3 +73,17 @@ subroutine atomic_write_typed_assign
   !$omp atomic write
   r2 = 0
 end subroutine
+
+!CHECK-LABEL: func.func @_QPatomic_write_logical()
+!CHECK:    %[[L_REF:.*]] = fir.alloca !fir.logical<4> {bindc_name = "l", uniq_name = "_QFatomic_write_logicalEl"}
+!CHECK:    %[[L_DECL:.*]]:2 = hlfir.declare %[[L_REF]] {uniq_name = "_QFatomic_write_logicalEl"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+!CHECK:    %true = arith.constant true
+!CHECK:    %[[CVT:.*]] = fir.convert %true : (i1) -> !fir.logical<4>
+!CHECK:    omp.atomic.write %[[L_DECL]]#1 = %[[CVT]] : !fir.ref<!fir.logical<4>>, !fir.logical<4>
+
+subroutine atomic_write_logical
+  logical :: l
+  !$omp atomic write
+      l = .true.
+  !$omp end atomic
+end
-- 
GitLab


From d395b56a52e9809ec3ea1139f5b30698c9f4e247 Mon Sep 17 00:00:00 2001
From: Pavel Samolysov <samolisov@gmail.com>
Date: Fri, 17 May 2024 07:46:43 +0300
Subject: [PATCH 1083/1206] [JITLink][AArch64] Implement
 R_AARCH64_LDR_PREL_LO19 (#82172)

This relocation is used for the 32-bit aligned 21-bit immediate in LDR
Literal instructions.
---
 .../llvm/ExecutionEngine/JITLink/aarch64.h    | 22 ++++++++++++++-----
 .../ExecutionEngine/JITLink/ELF_aarch64.cpp   | 12 ++++++++++
 .../JITLink/AArch64/ELF_relocations.s         | 11 ++++++++++
 3 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
index 40b9339eb531..c09398f98447 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
@@ -171,7 +171,14 @@ enum EdgeKind_aarch64 : Edge::Kind {
   ///
   /// Fixup expression:
   ///
-  ///   Fixup <- (Target - Fixup) >> 2 : int19
+  ///   Fixup <- (Target - Fixup + Addend) >> 2 : int19
+  ///
+  /// Notes:
+  ///   The '19' in the name refers to the number operand bits and follows the
+  /// naming convention used by the corresponding ELF relocation.
+  /// Since the low two bits must be zero (because of the 32-bit alignment of
+  /// the target) the operand is effectively a signed 21-bit number.
+  ///
   ///
   /// Errors:
   ///   - The result of the unshifted part of the fixup expression must be
@@ -377,6 +384,11 @@ inline bool isADR(uint32_t Instr) {
   return (Instr & ADRMask) == 0x10000000;
 }
 
+inline bool isLDRLiteral(uint32_t Instr) {
+  constexpr uint32_t LDRLitMask = 0x3b000000;
+  return (Instr & LDRLitMask) == 0x18000000;
+}
+
 // Returns the amount the address operand of LD/ST (imm12)
 // should be shifted right by.
 //
@@ -494,16 +506,14 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
   }
   case LDRLiteral19: {
     assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned");
-    assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend");
     uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
-    assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal");
-    int64_t Delta = E.getTarget().getAddress() - FixupAddress;
+    assert(isLDRLiteral(RawInstr) && "RawInstr is not an LDR Literal");
+    int64_t Delta = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
     if (Delta & 0x3)
       return make_error<JITLinkError>("LDR literal target is not 32-bit "
                                       "aligned");
-    if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1))
+    if (!isInt<21>(Delta))
       return makeTargetOutOfRangeError(G, B, E);
-
     uint32_t EncodedImm = ((static_cast<uint32_t>(Delta) >> 2) & 0x7ffff) << 5;
     uint32_t FixedInstr = RawInstr | EncodedImm;
     *(ulittle32_t *)FixupPtr = FixedInstr;
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
index 6b03bb3c90b2..9ce8aecb717c 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
@@ -49,6 +49,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
 private:
   enum ELFAArch64RelocationKind : Edge::Kind {
     ELFCall26 = Edge::FirstRelocation,
+    ELFLdrLo19,
     ELFAdrLo21,
     ELFAdrPage21,
     ELFAddAbs12,
@@ -82,6 +83,8 @@ private:
     case ELF::R_AARCH64_CALL26:
     case ELF::R_AARCH64_JUMP26:
       return ELFCall26;
+    case ELF::R_AARCH64_LD_PREL_LO19:
+      return ELFLdrLo19;
     case ELF::R_AARCH64_ADR_PREL_LO21:
       return ELFAdrLo21;
     case ELF::R_AARCH64_ADR_PREL_PG_HI21:
@@ -191,6 +194,15 @@ private:
       Kind = aarch64::Branch26PCRel;
       break;
     }
+    case ELFLdrLo19: {
+      uint32_t Instr = *(const ulittle32_t *)FixupContent;
+      if (!aarch64::isLDRLiteral(Instr))
+        return make_error<JITLinkError>(
+            "R_AARCH64_LDR_PREL_LO19 target is not an LDR Literal instruction");
+
+      Kind = aarch64::LDRLiteral19;
+      break;
+    }
     case ELFAdrLo21: {
       uint32_t Instr = *(const ulittle32_t *)FixupContent;
       if (!aarch64::isADR(Instr))
diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s
index fccef479c2c3..75e367bee80a 100644
--- a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s
+++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s
@@ -51,6 +51,17 @@ test_adr_prel_lo21:
 ## to test this bit better
 adr_data = test_adr_prel_lo21 + 0xe46f2
 
+# Check R_AARCH64_LD_PREL_LO19 relocation of a local symbol
+#
+# jitlink-check: decode_operand(test_ldr_prel_lo19 + 0, 1)[19:0] = \
+# jitlink-check:     (ldr_data - test_ldr_prel_lo19 + 0x4)[21:2]
+        .globl  test_ldr_prel_lo19, ldr_data
+        .p2align  2
+test_ldr_prel_lo19:
+        ldr	x0, ldr_data + 0x4
+        .size test_ldr_prel_lo19, .-test_ldr_prel_lo19
+ldr_data = test_ldr_prel_lo19 + 4
+
 # Check R_AARCH64_ADR_PREL_PG_HI21 / R_AARCH64_ADD_ABS_LO12_NC relocation of a local symbol
 #
 # For the ADR_PREL_PG_HI21/ADRP instruction we have the 21-bit delta to the 4k page
-- 
GitLab


From aaa8a8000009890b79effb0d0f3c6f989a3d5563 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 16 May 2024 21:53:54 -0700
Subject: [PATCH 1084/1206] [CodeGen] Use operator==(StringRef, StringRef)
 (NFC)

The LHS and RHS are of SmallString and StringRef, respectively.  We
can safely use operator==(StringRef, SringRef) with one implicit
conversion from SmallString to StringRef.
---
 llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
index 6eef5d2c50a2..fa5464026516 100644
--- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
@@ -170,7 +170,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
           return false;
         // Return a match if debug-info-filename is not specified. Otherwise,
         // check for equality.
-        return DIFilename.empty() || It->second.equals(DIFilename);
+        return DIFilename.empty() || It->second == DIFilename;
       });
       if (!FunctionFound) {
         // Skip the following profile by setting the profile iterator (FI) to
@@ -317,7 +317,7 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
           return false;
         // Return a match if debug-info-filename is not specified. Otherwise,
         // check for equality.
-        return DIFilename.empty() || It->second.equals(DIFilename);
+        return DIFilename.empty() || It->second == DIFilename;
       });
       if (!FunctionFound) {
         // Skip the following profile by setting the profile iterator (FI) to
-- 
GitLab


From a26fbf36a78a703be2da0744131a8d6ecbdb7c67 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 16 May 2024 22:04:03 -0700
Subject: [PATCH 1085/1206] [Sema] Use SmallString::empty (NFC)

---
 clang/lib/Sema/SemaDeclAttr.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 777171f4f15f..3760a4c917ad 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -3645,7 +3645,7 @@ bool Sema::checkTargetClonesAttrString(
         llvm::sort(CurFeatures);
         SmallString<64> Res;
         for (auto &CurFeat : CurFeatures) {
-          if (!Res.equals(""))
+          if (!Res.empty())
             Res.append("+");
           Res.append(CurFeat);
         }
-- 
GitLab


From f4066fa2dd21c65bf0e24a479634c9a2d276cf8e Mon Sep 17 00:00:00 2001
From: Serge Pavlov <sepavloff@gmail.com>
Date: Fri, 17 May 2024 12:06:34 +0700
Subject: [PATCH 1086/1206] [clang] Use constant rounding mode for floating
 literals (#90877)

Conversion of floating-point literal to binary representation must be
made using constant rounding mode, which can be changed using pragma
FENV_ROUND. For example, the literal "0.1F" should be representes by
either 0.099999994 or 0.100000001 depending on the rounding direction.
---
 clang/include/clang/Lex/LiteralSupport.h |  10 +--
 clang/lib/Lex/LiteralSupport.cpp         |   6 +-
 clang/lib/Sema/SemaExpr.cpp              |   5 +-
 clang/test/AST/const-fpfeatures.c        |   6 ++
 clang/test/AST/const-fpfeatures.cpp      | 105 +++++++++++++++++++++++
 5 files changed, 122 insertions(+), 10 deletions(-)

diff --git a/clang/include/clang/Lex/LiteralSupport.h b/clang/include/clang/Lex/LiteralSupport.h
index 2ed42d1c5f9a..705021fcfa5b 100644
--- a/clang/include/clang/Lex/LiteralSupport.h
+++ b/clang/include/clang/Lex/LiteralSupport.h
@@ -118,12 +118,10 @@ public:
   /// bits of the result and return true.  Otherwise, return false.
   bool GetIntegerValue(llvm::APInt &Val);
 
-  /// GetFloatValue - Convert this numeric literal to a floating value, using
-  /// the specified APFloat fltSemantics (specifying float, double, etc).
-  /// The optional bool isExact (passed-by-reference) has its value
-  /// set to true if the returned APFloat can represent the number in the
-  /// literal exactly, and false otherwise.
-  llvm::APFloat::opStatus GetFloatValue(llvm::APFloat &Result);
+  /// Convert this numeric literal to a floating value, using the specified
+  /// APFloat fltSemantics (specifying float, double, etc) and rounding mode.
+  llvm::APFloat::opStatus GetFloatValue(llvm::APFloat &Result,
+                                        llvm::RoundingMode RM);
 
   /// GetFixedPointValue - Convert this numeric literal value into a
   /// scaled integer that represents this value. Returns true if an overflow
diff --git a/clang/lib/Lex/LiteralSupport.cpp b/clang/lib/Lex/LiteralSupport.cpp
index 9c0cbea5052c..3df0391bdda7 100644
--- a/clang/lib/Lex/LiteralSupport.cpp
+++ b/clang/lib/Lex/LiteralSupport.cpp
@@ -1520,7 +1520,8 @@ bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
 }
 
 llvm::APFloat::opStatus
-NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) {
+NumericLiteralParser::GetFloatValue(llvm::APFloat &Result,
+                                    llvm::RoundingMode RM) {
   using llvm::APFloat;
 
   unsigned n = std::min(SuffixBegin - ThisTokBegin, ThisTokEnd - ThisTokBegin);
@@ -1534,8 +1535,7 @@ NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) {
     Str = Buffer;
   }
 
-  auto StatusOrErr =
-      Result.convertFromString(Str, APFloat::rmNearestTiesToEven);
+  auto StatusOrErr = Result.convertFromString(Str, RM);
   assert(StatusOrErr && "Invalid floating point representation");
   return !errorToBool(StatusOrErr.takeError()) ? *StatusOrErr
                                                : APFloat::opInvalidOp;
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 274e1fb18353..e0aae6333e1a 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -3699,7 +3699,10 @@ static Expr *BuildFloatingLiteral(Sema &S, NumericLiteralParser &Literal,
   using llvm::APFloat;
   APFloat Val(Format);
 
-  APFloat::opStatus result = Literal.GetFloatValue(Val);
+  llvm::RoundingMode RM = S.CurFPFeatures.getRoundingMode();
+  if (RM == llvm::RoundingMode::Dynamic)
+    RM = llvm::RoundingMode::NearestTiesToEven;
+  APFloat::opStatus result = Literal.GetFloatValue(Val, RM);
 
   // Overflow is always an error, but underflow is only an error if
   // we underflowed to zero (APFloat reports denormals as underflow).
diff --git a/clang/test/AST/const-fpfeatures.c b/clang/test/AST/const-fpfeatures.c
index 083350fdc8ce..8dc3221b0638 100644
--- a/clang/test/AST/const-fpfeatures.c
+++ b/clang/test/AST/const-fpfeatures.c
@@ -19,6 +19,9 @@ float FI1u = 0xFFFFFFFFU;
 float _Complex C1u = C0;
 // CHECK: @C1u = {{.*}} { float, float } { float 0x3FF0000020000000, float 0x3FF0000020000000 }
 
+float FLu = 0.1F;
+// CHECK: @FLu = {{.*}} float 0x3FB99999A0000000
+
 
 #pragma STDC FENV_ROUND FE_DOWNWARD
 
@@ -35,3 +38,6 @@ float FI1d = 0xFFFFFFFFU;
 
 float _Complex C1d = C0;
 // CHECK: @C1d = {{.*}} { float, float } { float 1.000000e+00, float 1.000000e+00 }
+
+float FLd = 0.1F;
+// CHECK: @FLd = {{.*}} float 0x3FB9999980000000
diff --git a/clang/test/AST/const-fpfeatures.cpp b/clang/test/AST/const-fpfeatures.cpp
index 95eb613df7f0..5e903c8c0e87 100644
--- a/clang/test/AST/const-fpfeatures.cpp
+++ b/clang/test/AST/const-fpfeatures.cpp
@@ -79,3 +79,108 @@ float V7 = []() -> float {
   0x0.000001p0F);
 }();
 // CHECK: @V7 = {{.*}} float 1.000000e+00
+
+#pragma STDC FENV_ROUND FE_DYNAMIC
+
+template<float V> struct L {
+  constexpr L() : value(V) {}
+  float value;
+};
+
+#pragma STDC FENV_ROUND FE_DOWNWARD
+L<0.1F> val_d;
+// CHECK: @val_d = {{.*}} { float 0x3FB9999980000000 }
+
+#pragma STDC FENV_ROUND FE_UPWARD
+L<0.1F> val_u;
+// CHECK: @val_u = {{.*}} { float 0x3FB99999A0000000 }
+
+
+// Check literals in macros.
+
+#pragma STDC FENV_ROUND FE_DOWNWARD
+#define CONSTANT_0_1 0.1F
+
+#pragma STDC FENV_ROUND FE_UPWARD
+float C1_ru = CONSTANT_0_1;
+// CHECK: @C1_ru = {{.*}} float 0x3FB99999A0000000
+
+#pragma STDC FENV_ROUND FE_DOWNWARD
+float C1_rd = CONSTANT_0_1;
+// CHECK: @C1_rd = {{.*}} float 0x3FB9999980000000
+
+#pragma STDC FENV_ROUND FE_DOWNWARD
+#define PRAGMA(x) _Pragma(#x)
+#define CONSTANT_0_1_RM(v, rm) ([](){ PRAGMA(STDC FENV_ROUND rm); return v; }())
+
+#pragma STDC FENV_ROUND FE_UPWARD
+float C2_rd = CONSTANT_0_1_RM(0.1F, FE_DOWNWARD);
+float C2_ru = CONSTANT_0_1_RM(0.1F, FE_UPWARD);
+// CHECK: @C2_rd = {{.*}} float 0x3FB9999980000000
+// CHECK: @C2_ru = {{.*}} float 0x3FB99999A0000000
+
+#pragma STDC FENV_ROUND FE_DOWNWARD
+float C3_rd = CONSTANT_0_1_RM(0.1F, FE_DOWNWARD);
+float C3_ru = CONSTANT_0_1_RM(0.1F, FE_UPWARD);
+// CHECK: @C3_rd = {{.*}} float 0x3FB9999980000000
+// CHECK: @C3_ru = {{.*}} float 0x3FB99999A0000000
+
+// Check literals in template instantiations.
+
+#pragma STDC FENV_ROUND FE_DYNAMIC
+
+template<typename T, T C>
+constexpr T foo() {
+  return C;
+}
+
+#pragma STDC FENV_ROUND FE_DOWNWARD
+float var_d = foo<float, 0.1F>();
+// CHECK: @var_d = {{.*}} float 0x3FB9999980000000
+
+#pragma STDC FENV_ROUND FE_UPWARD
+float var_u = foo<float, 0.1F>();
+// CHECK: @var_u = {{.*}} float 0x3FB99999A0000000
+
+#pragma STDC FENV_ROUND FE_DYNAMIC
+
+template<typename T, T f> void foo2() {
+  T Val = f;
+}
+
+void func_01() {
+  #pragma STDC FENV_ROUND FE_DOWNWARD
+  foo2<float, 0.1f>();
+}
+
+void func_02() {
+  #pragma STDC FENV_ROUND FE_UPWARD
+  foo2<float, 0.1f>();
+}
+
+// CHECK-LABEL: define {{.*}} void @_Z4foo2IfTnT_Lf3dccccccEEvv()
+// CHECK:         store float 0x3FB9999980000000, ptr
+
+// CHECK-LABEL: define {{.*}} void @_Z4foo2IfTnT_Lf3dcccccdEEvv()
+// CHECK:         store float 0x3FB99999A0000000, ptr
+
+
+#pragma STDC FENV_ROUND FE_DOWNWARD
+template <int C>
+float tfunc_01() {
+  return 0.1F;  // Must be 0x3FB9999980000000 in all instantiations.
+}
+template float tfunc_01<0>();
+// CHECK-LABEL: define {{.*}} float @_Z8tfunc_01ILi0EEfv()
+// CHECK:         ret float 0x3FB9999980000000
+
+#pragma STDC FENV_ROUND FE_UPWARD
+template float tfunc_01<1>();
+// CHECK-LABEL: define {{.*}} float @_Z8tfunc_01ILi1EEfv()
+// CHECK:         ret float 0x3FB9999980000000
+
+template<> float tfunc_01<2>() {
+  return 0.1F;
+}
+// CHECK-LABEL: define {{.*}} float @_Z8tfunc_01ILi2EEfv()
+// CHECK:         ret float 0x3FB99999A0000000
-- 
GitLab


From 14030d71693b046784ff19cc157074e4db23e74f Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 16 May 2024 22:18:45 -0700
Subject: [PATCH 1087/1206] [llvm] Drop explicit conversions of string literals
 to StringRef (NFC)

We routinely rely on implicit conversions of string literals to
StringRef so that we can use operator==(StringRef, StringRef).
---
 llvm/lib/Frontend/OpenMP/OMPContext.cpp   | 2 +-
 llvm/lib/Object/MachOObjectFile.cpp       | 4 ++--
 llvm/lib/Target/Mips/MipsISelLowering.cpp | 3 +--
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPContext.cpp b/llvm/lib/Frontend/OpenMP/OMPContext.cpp
index 37936d6000c8..ad794d8345cf 100644
--- a/llvm/lib/Frontend/OpenMP/OMPContext.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPContext.cpp
@@ -63,7 +63,7 @@ OMPContext::OMPContext(bool IsDeviceCompilation, Triple TargetTriple) {
   if (TraitSelector::TraitSelectorEnum == TraitSelector::device_arch) {        \
     if (TargetTriple.getArch() == TargetTriple.getArchTypeForLLVMName(Str))    \
       ActiveTraits.set(unsigned(TraitProperty::Enum));                         \
-    if (StringRef(Str) == StringRef("x86_64") &&                               \
+    if (StringRef(Str) == "x86_64" &&                                          \
         TargetTriple.getArch() == Triple::x86_64)                              \
       ActiveTraits.set(unsigned(TraitProperty::Enum));                         \
   }
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index ef390ceca218..863bc1219e7a 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -3256,13 +3256,13 @@ MachOAbstractFixupEntry::MachOAbstractFixupEntry(Error *E,
   for (const auto &Command : O->load_commands()) {
     if (Command.C.cmd == MachO::LC_SEGMENT) {
       MachO::segment_command SLC = O->getSegmentLoadCommand(Command);
-      if (StringRef(SLC.segname) == StringRef("__TEXT")) {
+      if (StringRef(SLC.segname) == "__TEXT") {
         TextAddress = SLC.vmaddr;
         break;
       }
     } else if (Command.C.cmd == MachO::LC_SEGMENT_64) {
       MachO::segment_command_64 SLC_64 = O->getSegment64LoadCommand(Command);
-      if (StringRef(SLC_64.segname) == StringRef("__TEXT")) {
+      if (StringRef(SLC_64.segname) == "__TEXT") {
         TextAddress = SLC_64.vmaddr;
         break;
       }
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 8f7c47370ee5..459164fa7a29 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -3234,8 +3234,7 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   // Note: The check on the calling convention below must match
   //       MipsABIInfo::GetCalleeAllocdArgSizeInBytes().
-  bool MemcpyInByVal = ES &&
-                       StringRef(ES->getSymbol()) == StringRef("memcpy") &&
+  bool MemcpyInByVal = ES && StringRef(ES->getSymbol()) == "memcpy" &&
                        CallConv != CallingConv::Fast &&
                        Chain.getOpcode() == ISD::CALLSEQ_START;
 
-- 
GitLab


From f71749c5ef8667e3fc23820e8e94864653ea9ac9 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 16 May 2024 22:32:06 -0700
Subject: [PATCH 1088/1206] [clang] Drop explicit conversions of string
 literals to StringRef (NFC)

We routinely rely on implicit conversions of string literals to
StringRef so that we can use operator==(StringRef, StringRef).

The LHS here are all known to be of StringRef.
---
 clang/lib/Analysis/ThreadSafetyCommon.cpp | 2 +-
 clang/lib/Sema/SemaDeclAttr.cpp           | 2 +-
 clang/lib/Serialization/ASTWriter.cpp     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Analysis/ThreadSafetyCommon.cpp b/clang/lib/Analysis/ThreadSafetyCommon.cpp
index 33f1f466df24..a3b378c42df3 100644
--- a/clang/lib/Analysis/ThreadSafetyCommon.cpp
+++ b/clang/lib/Analysis/ThreadSafetyCommon.cpp
@@ -177,7 +177,7 @@ CapabilityExpr SExprBuilder::translateAttrExpr(const Expr *AttrExp,
     return CapabilityExpr();
 
   if (const auto* SLit = dyn_cast<StringLiteral>(AttrExp)) {
-    if (SLit->getString() == StringRef("*"))
+    if (SLit->getString() == "*")
       // The "*" expr is a universal lock, which essentially turns off
       // checks until it is removed from the lockset.
       return CapabilityExpr(new (Arena) til::Wildcard(), StringRef("wildcard"),
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 3760a4c917ad..30776ff537fb 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -640,7 +640,7 @@ static void checkAttrArgsAreCapabilityObjs(Sema &S, Decl *D,
 
     if (const auto *StrLit = dyn_cast<StringLiteral>(ArgExp)) {
       if (StrLit->getLength() == 0 ||
-          (StrLit->isOrdinary() && StrLit->getString() == StringRef("*"))) {
+          (StrLit->isOrdinary() && StrLit->getString() == "*")) {
         // Pass empty strings to the analyzer without warnings.
         // Treat "*" as the universal lock.
         Args.push_back(ArgExp);
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 129bc337c892..dd5d65b2eb20 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -1460,7 +1460,7 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
         (!PP.getHeaderSearchInfo()
               .getHeaderSearchOpts()
               .ModuleMapFileHomeIsCwd ||
-         WritingModule->Directory->getName() != StringRef("."))) {
+         WritingModule->Directory->getName() != ".")) {
       // Module directory.
       auto Abbrev = std::make_shared<BitCodeAbbrev>();
       Abbrev->Add(BitCodeAbbrevOp(MODULE_DIRECTORY));
-- 
GitLab


From 9bffe790499e99a4110a33988100ba45835e905e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Fri, 17 May 2024 07:32:18 +0200
Subject: [PATCH 1089/1206] [GlobalIsel] Speedup select to integer min/max
 (#92378)

https://github.com/llvm/llvm-project/issues/92309
---
 .../llvm/CodeGen/GlobalISel/CombinerHelper.h  |   6 +-
 .../include/llvm/Target/GlobalISel/Combine.td |   9 +-
 .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 100 +++++++-----------
 .../AArch64/GlobalISel/combine-select.mir     |  28 +++++
 4 files changed, 78 insertions(+), 65 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 87cff154b4eb..a9a33c7617d7 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -822,6 +822,9 @@ public:
   // Given a binop \p MI, commute operands 1 and 2.
   void applyCommuteBinOpOperands(MachineInstr &MI);
 
+  /// Combine select to integer min/max.
+  bool matchSelectIMinMax(const MachineOperand &MO, BuildFnTy &MatchInfo);
+
   /// Combine selects.
   bool matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo);
 
@@ -962,9 +965,6 @@ private:
 
   bool tryFoldSelectOfConstants(GSelect *Select, BuildFnTy &MatchInfo);
 
-  /// Try to fold (icmp X, Y) ? X : Y -> integer minmax.
-  bool tryFoldSelectToIntMinMax(GSelect *Select, BuildFnTy &MatchInfo);
-
   bool isOneOrOneSplat(Register Src, bool AllowUndefs);
   bool isZeroOrZeroSplat(Register Src, bool AllowUndefs);
   bool isConstantSplatVector(Register Src, int64_t SplatValue,
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index d61a5759d5a9..a1d39d910bc8 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1307,6 +1307,13 @@ def select_to_minmax: GICombineRule<
          [{ return Helper.matchSimplifySelectToMinMax(*${root}, ${info}); }]),
   (apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
 
+def select_to_iminmax: GICombineRule<
+  (defs root:$root, build_fn_matchinfo:$info),
+  (match (G_ICMP $tst, $tst1, $x, $y),
+         (G_SELECT $root, $tst, $x, $y),
+         [{ return Helper.matchSelectIMinMax(${root}, ${info}); }]),
+  (apply [{ Helper.applyBuildFnMO(${root}, ${info}); }])>;
+
 def match_selects : GICombineRule<
   (defs root:$root, build_fn_matchinfo:$matchinfo),
   (match (wip_match_opcode G_SELECT):$root,
@@ -1670,7 +1677,7 @@ def phi_combines : GICombineGroup<[extend_through_phis]>;
 def bitreverse_shift : GICombineGroup<[bitreverse_shl, bitreverse_lshr]>;
 
 def select_combines : GICombineGroup<[select_undef_cmp, select_constant_cmp,
-                                      match_selects]>;
+                                      select_to_iminmax, match_selects]>;
 
 def trivial_combines : GICombineGroup<[copy_prop, mul_to_shl, add_p2i_to_ptradd,
                                        mul_by_neg_one, idempotent_prop]>;
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 809f13abcadd..a87a26d72b45 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -6749,10 +6749,12 @@ bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select,
   return false;
 }
 
-bool CombinerHelper::tryFoldSelectToIntMinMax(GSelect *Select,
-                                              BuildFnTy &MatchInfo) {
+bool CombinerHelper::matchSelectIMinMax(const MachineOperand &MO,
+                                        BuildFnTy &MatchInfo) {
+  GSelect *Select = cast<GSelect>(MRI.getVRegDef(MO.getReg()));
+  GICmp *Cmp = cast<GICmp>(MRI.getVRegDef(Select->getCondReg()));
+
   Register DstReg = Select->getReg(0);
-  Register Cond = Select->getCondReg();
   Register True = Select->getTrueReg();
   Register False = Select->getFalseReg();
   LLT DstTy = MRI.getType(DstReg);
@@ -6760,11 +6762,6 @@ bool CombinerHelper::tryFoldSelectToIntMinMax(GSelect *Select,
   if (DstTy.isPointer())
     return false;
 
-  // We need an G_ICMP on the condition register.
-  GICmp *Cmp = getOpcodeDef<GICmp>(Cond, MRI);
-  if (!Cmp)
-    return false;
-
   // We want to fold the icmp and replace the select.
   if (!MRI.hasOneNonDBGUse(Cmp->getReg(0)))
     return false;
@@ -6775,62 +6772,46 @@ bool CombinerHelper::tryFoldSelectToIntMinMax(GSelect *Select,
   if (CmpInst::isEquality(Pred))
     return false;
 
-  Register CmpLHS = Cmp->getLHSReg();
-  Register CmpRHS = Cmp->getRHSReg();
-
-  // We can swap CmpLHS and CmpRHS for higher hitrate.
-  if (True == CmpRHS && False == CmpLHS) {
-    std::swap(CmpLHS, CmpRHS);
-    Pred = CmpInst::getSwappedPredicate(Pred);
-  }
+  [[maybe_unused]] Register CmpLHS = Cmp->getLHSReg();
+  [[maybe_unused]] Register CmpRHS = Cmp->getRHSReg();
 
   // (icmp X, Y) ? X : Y -> integer minmax.
   // see matchSelectPattern in ValueTracking.
   // Legality between G_SELECT and integer minmax can differ.
-  if (True == CmpLHS && False == CmpRHS) {
-    switch (Pred) {
-    case ICmpInst::ICMP_UGT:
-    case ICmpInst::ICMP_UGE: {
-      if (!isLegalOrBeforeLegalizer({TargetOpcode::G_UMAX, DstTy}))
-        return false;
-      MatchInfo = [=](MachineIRBuilder &B) {
-        B.buildUMax(DstReg, True, False);
-      };
-      return true;
-    }
-    case ICmpInst::ICMP_SGT:
-    case ICmpInst::ICMP_SGE: {
-      if (!isLegalOrBeforeLegalizer({TargetOpcode::G_SMAX, DstTy}))
-        return false;
-      MatchInfo = [=](MachineIRBuilder &B) {
-        B.buildSMax(DstReg, True, False);
-      };
-      return true;
-    }
-    case ICmpInst::ICMP_ULT:
-    case ICmpInst::ICMP_ULE: {
-      if (!isLegalOrBeforeLegalizer({TargetOpcode::G_UMIN, DstTy}))
-        return false;
-      MatchInfo = [=](MachineIRBuilder &B) {
-        B.buildUMin(DstReg, True, False);
-      };
-      return true;
-    }
-    case ICmpInst::ICMP_SLT:
-    case ICmpInst::ICMP_SLE: {
-      if (!isLegalOrBeforeLegalizer({TargetOpcode::G_SMIN, DstTy}))
-        return false;
-      MatchInfo = [=](MachineIRBuilder &B) {
-        B.buildSMin(DstReg, True, False);
-      };
-      return true;
-    }
-    default:
+  assert(True == CmpLHS && False == CmpRHS && "unexpected MIR pattern");
+
+  switch (Pred) {
+  case ICmpInst::ICMP_UGT:
+  case ICmpInst::ICMP_UGE: {
+    if (!isLegalOrBeforeLegalizer({TargetOpcode::G_UMAX, DstTy}))
       return false;
-    }
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildUMax(DstReg, True, False); };
+    return true;
+  }
+  case ICmpInst::ICMP_SGT:
+  case ICmpInst::ICMP_SGE: {
+    if (!isLegalOrBeforeLegalizer({TargetOpcode::G_SMAX, DstTy}))
+      return false;
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildSMax(DstReg, True, False); };
+    return true;
+  }
+  case ICmpInst::ICMP_ULT:
+  case ICmpInst::ICMP_ULE: {
+    if (!isLegalOrBeforeLegalizer({TargetOpcode::G_UMIN, DstTy}))
+      return false;
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildUMin(DstReg, True, False); };
+    return true;
+  }
+  case ICmpInst::ICMP_SLT:
+  case ICmpInst::ICMP_SLE: {
+    if (!isLegalOrBeforeLegalizer({TargetOpcode::G_SMIN, DstTy}))
+      return false;
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildSMin(DstReg, True, False); };
+    return true;
+  }
+  default:
+    return false;
   }
-
-  return false;
 }
 
 bool CombinerHelper::matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo) {
@@ -6842,9 +6823,6 @@ bool CombinerHelper::matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo) {
   if (tryFoldBoolSelectToLogic(Select, MatchInfo))
     return true;
 
-  if (tryFoldSelectToIntMinMax(Select, MatchInfo))
-    return true;
-
   return false;
 }
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
index 2bf7e84a379b..8d9ad8d7caca 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
@@ -896,3 +896,31 @@ body:             |
     RET_ReallyLR
 
 ...
+---
+# test failed select icmp_slef,t_t_f --> smin(t,f)
+name:            select_icmp_sle_f_t_t_f_smin_t_f
+body:             |
+  bb.1:
+    liveins: $x0, $x1, $x2
+    ; CHECK-LABEL: name: select_icmp_sle_f_t_t_f_smin_t_f
+    ; CHECK: liveins: $x0, $x1, $x2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK-NEXT: %t1:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: %f1:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: %t:_(<4 x s32>) = G_BUILD_VECTOR %t1(s32), %t1(s32), %t1(s32), %t1(s32)
+    ; CHECK-NEXT: %f:_(<4 x s32>) = G_BUILD_VECTOR %f1(s32), %f1(s32), %f1(s32), %f1(s32)
+    ; CHECK-NEXT: %c:_(<4 x s32>) = G_ICMP intpred(sle), %f(<4 x s32>), %t
+    ; CHECK-NEXT: %sel:_(<4 x s32>) = exact G_SELECT %c(<4 x s32>), %t, %f
+    ; CHECK-NEXT: $q0 = COPY %sel(<4 x s32>)
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %t1:_(s32) = G_TRUNC %0
+    %f1:_(s32) = G_TRUNC %1
+    %t:_(<4 x s32>) = G_BUILD_VECTOR %t1, %t1, %t1, %t1
+    %f:_(<4 x s32>) = G_BUILD_VECTOR %f1, %f1, %f1, %f1
+    %c:_(<4 x s32>) = G_ICMP intpred(sle), %f(<4 x s32>), %t(<4 x s32>)
+    %sel:_(<4 x s32>) = exact G_SELECT %c, %t, %f
+    $q0 = COPY %sel(<4 x s32>)
+...
-- 
GitLab


From bc9823cf60bf91cc8b45248c4205cd2c67b2a3d5 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Fri, 17 May 2024 13:37:20 +0800
Subject: [PATCH 1090/1206] [X86][BF16] Change MVT to EVT in combineFP_EXTEND

Fixes: #92471
---
 llvm/lib/Target/X86/X86ISelLowering.cpp |  4 +-
 llvm/test/CodeGen/X86/bfloat.ll         | 53 +++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index cd252c54887a..5d0846453685 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -56802,12 +56802,12 @@ static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
 
     assert(!IsStrict && "Strict FP doesn't support BF16");
     if (VT.getVectorElementType() == MVT::f64) {
-      MVT TmpVT = VT.getSimpleVT().changeVectorElementType(MVT::f32);
+      EVT TmpVT = VT.changeVectorElementType(MVT::f32);
       return DAG.getNode(ISD::FP_EXTEND, dl, VT,
                          DAG.getNode(ISD::FP_EXTEND, dl, TmpVT, Src));
     }
     assert(VT.getVectorElementType() == MVT::f32 && "Unexpected fpext");
-    MVT NVT = SrcVT.getSimpleVT().changeVectorElementType(MVT::i32);
+    EVT NVT = SrcVT.changeVectorElementType(MVT::i32);
     Src = DAG.getBitcast(SrcVT.changeTypeToInteger(), Src);
     Src = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Src);
     Src = DAG.getNode(ISD::SHL, dl, NVT, Src, DAG.getConstant(16, dl, NVT));
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index b3e04590075f..8b5ca57df27e 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -2464,3 +2464,56 @@ define float @trunc_ext(float %a) nounwind {
   %c = fpext bfloat %b to float
   ret float %c
 }
+
+define void @PR92471(ptr %0, ptr %1) nounwind {
+; X86-LABEL: PR92471:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vpinsrd $1, 4(%ecx), %xmm0, %xmm0
+; X86-NEXT:    vpinsrd $2, 8(%ecx), %xmm0, %xmm0
+; X86-NEXT:    vpinsrw $6, 12(%ecx), %xmm0, %xmm0
+; X86-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-NEXT:    vpslld $16, %ymm0, %ymm0
+; X86-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; X86-NEXT:    vpextrd $2, %xmm1, 24(%eax)
+; X86-NEXT:    vpextrd $1, %xmm1, 20(%eax)
+; X86-NEXT:    vmovd %xmm1, 16(%eax)
+; X86-NEXT:    vmovdqu %xmm0, (%eax)
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+;
+; SSE2-LABEL: PR92471:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    pinsrw $2, 12(%rdi), %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    movdqu %xmm2, (%rsi)
+; SSE2-NEXT:    movq %xmm3, 16(%rsi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; SSE2-NEXT:    movd %xmm0, 24(%rsi)
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: PR92471:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    vpinsrd $2, 8(%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $6, 12(%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX-NEXT:    vpslld $16, %ymm0, %ymm0
+; AVX-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vpextrd $2, %xmm1, 24(%rsi)
+; AVX-NEXT:    vmovq %xmm1, 16(%rsi)
+; AVX-NEXT:    vmovdqu %xmm0, (%rsi)
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %3 = load <7 x bfloat>, ptr %0, align 2
+  %4 = fpext <7 x bfloat> %3 to <7 x float>
+  store <7 x float> %4, ptr %1, align 4
+  ret void
+}
-- 
GitLab


From f43deca2538b717f5669ce8116cc4c040bde87c8 Mon Sep 17 00:00:00 2001
From: Johannes Reifferscheid <jreiffers@google.com>
Date: Fri, 17 May 2024 08:44:07 +0200
Subject: [PATCH 1091/1206] Fix Tan inaccuracies on extreme complex inputs.
 (#92443)

Specifically, those with small/large absolute values. This ports
https://github.com/openxla/xla/pull/10525 and was verified with XLA's
test suite.
---
 .../ComplexToStandard/ComplexToStandard.cpp   |  43 ++-
 .../convert-to-standard.mlir                  | 335 ++++--------------
 2 files changed, 98 insertions(+), 280 deletions(-)

diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index c2a83f90bcbe..edca8eaecbff 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -956,27 +956,12 @@ struct SignOpConversion : public OpConversionPattern<complex::SignOp> {
   }
 };
 
-struct TanOpConversion : public OpConversionPattern<complex::TanOp> {
-  using OpConversionPattern<complex::TanOp>::OpConversionPattern;
+template <typename Op>
+struct TanTanhOpConversion : public OpConversionPattern<Op> {
+  using OpConversionPattern<Op>::OpConversionPattern;
 
   LogicalResult
-  matchAndRewrite(complex::TanOp op, OpAdaptor adaptor,
-                  ConversionPatternRewriter &rewriter) const override {
-    auto loc = op.getLoc();
-    arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
-
-    Value cos = rewriter.create<complex::CosOp>(loc, adaptor.getComplex(), fmf);
-    Value sin = rewriter.create<complex::SinOp>(loc, adaptor.getComplex(), fmf);
-    rewriter.replaceOpWithNewOp<complex::DivOp>(op, sin, cos, fmf);
-    return success();
-  }
-};
-
-struct TanhOpConversion : public OpConversionPattern<complex::TanhOp> {
-  using OpConversionPattern<complex::TanhOp>::OpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(complex::TanhOp op, OpAdaptor adaptor,
+  matchAndRewrite(Op op, Op::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     auto loc = op.getLoc();
@@ -989,14 +974,20 @@ struct TanhOpConversion : public OpConversionPattern<complex::TanhOp> {
         b.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
     Value imag =
         b.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
+    Value negOne = b.create<arith::ConstantOp>(
+        elementType, b.getFloatAttr(elementType, -1.0));
+
+    if constexpr (std::is_same_v<Op, complex::TanOp>) {
+      // tan(x+yi) = -i*tanh(-y + xi)
+      std::swap(real, imag);
+      real = b.create<arith::MulFOp>(real, negOne, fmf);
+    }
 
     auto cst = [&](APFloat v) {
       return b.create<arith::ConstantOp>(elementType,
                                          b.getFloatAttr(elementType, v));
     };
     Value inf = cst(APFloat::getInf(floatSemantics));
-    Value negOne = b.create<arith::ConstantOp>(
-        elementType, b.getFloatAttr(elementType, -1.0));
     Value four = b.create<arith::ConstantOp>(elementType,
                                              b.getFloatAttr(elementType, 4.0));
     Value twoReal = b.create<arith::AddFOp>(real, real, fmf);
@@ -1054,6 +1045,12 @@ struct TanhOpConversion : public OpConversionPattern<complex::TanhOp> {
           b.create<arith::SelectOp>(resultImagIsZero, zero, resultImag);
     }
 
+    if constexpr (std::is_same_v<Op, complex::TanOp>) {
+      // tan(x+yi) = -i*tanh(-y + xi)
+      std::swap(resultReal, resultImag);
+      resultImag = b.create<arith::MulFOp>(resultImag, negOne, fmf);
+    }
+
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
     return success();
@@ -1327,8 +1324,8 @@ void mlir::populateComplexToStandardConversionPatterns(
       SignOpConversion,
       SinOpConversion,
       SqrtOpConversion,
-      TanOpConversion,
-      TanhOpConversion,
+      TanTanhOpConversion<complex::TanOp>,
+      TanTanhOpConversion<complex::TanhOp>,
       PowOpConversion,
       RsqrtOpConversion
   >(patterns.getContext());
diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
index 827ae940165c..6dafe29e2e5f 100644
--- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
+++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
@@ -538,135 +538,46 @@ func.func @complex_tan(%arg: complex<f32>) -> complex<f32> {
   %tan = complex.tan %arg: complex<f32>
   return %tan : complex<f32>
 }
-// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]]
-// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]]
-// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32
-// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] : f32
-// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]]
-// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]]
-// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] : f32
-// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] : f32
-// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_REXP]], %[[HALF_EXP]]
-// CHECK-DAG: %[[COS_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[COS]]
-// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_REXP]], %[[HALF_EXP]]
-// CHECK-DAG: %[[COS_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[SIN]]
-// CHECK-DAG: %[[COS_COMP:.*]] = complex.create %[[COS_REAL]], %[[COS_IMAG]] : complex<f32>
 
-// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]]
-// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]]
-// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32
-// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] : f32
-// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]]
-// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]]
-// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] : f32
-// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] : f32
-// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_EXP]], %[[HALF_REXP]]
-// CHECK-DAG: %[[SIN_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[SIN]]
-// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_EXP]], %[[HALF_REXP]]
-// CHECK-DAG: %[[SIN_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[COS]]
-// CHECK-DAG: %[[SIN_COMP:.*]] = complex.create %[[SIN_REAL]], %[[SIN_IMAG]] : complex<f32>
-
-// CHECK: %[[LHS_REAL:.*]] = complex.re %[[SIN_COMP]] : complex<f32>
-// CHECK: %[[LHS_IMAG:.*]] = complex.im %[[SIN_COMP]] : complex<f32>
-// CHECK: %[[RHS_REAL:.*]] = complex.re %[[COS_COMP]] : complex<f32>
-// CHECK: %[[RHS_IMAG:.*]] = complex.im %[[COS_COMP]] : complex<f32>
-
-// CHECK: %[[RHS_REAL_IMAG_RATIO:.*]] = arith.divf %[[RHS_REAL]], %[[RHS_IMAG]] : f32
-// CHECK: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = arith.mulf %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]] : f32
-// CHECK: %[[RHS_REAL_IMAG_DENOM:.*]] = arith.addf %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]] : f32
-// CHECK: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = arith.mulf %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]] : f32
-// CHECK: %[[REAL_NUMERATOR_1:.*]] = arith.addf %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]] : f32
-// CHECK: %[[RESULT_REAL_1:.*]] = arith.divf %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]] : f32
-// CHECK: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = arith.mulf %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]] : f32
-// CHECK: %[[IMAG_NUMERATOR_1:.*]] = arith.subf %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]] : f32
-// CHECK: %[[RESULT_IMAG_1:.*]] = arith.divf %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]] : f32
-
-// CHECK: %[[RHS_IMAG_REAL_RATIO:.*]] = arith.divf %[[RHS_IMAG]], %[[RHS_REAL]] : f32
-// CHECK: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = arith.mulf %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]] : f32
-// CHECK: %[[RHS_IMAG_REAL_DENOM:.*]] = arith.addf %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]] : f32
-// CHECK: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = arith.mulf %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]] : f32
-// CHECK: %[[REAL_NUMERATOR_2:.*]] = arith.addf %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]] : f32
-// CHECK: %[[RESULT_REAL_2:.*]] = arith.divf %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]] : f32
-// CHECK: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = arith.mulf %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]] : f32
-// CHECK: %[[IMAG_NUMERATOR_2:.*]] = arith.subf %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]] : f32
-// CHECK: %[[RESULT_IMAG_2:.*]] = arith.divf %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]] : f32
-
-// Case 1. Zero denominator, numerator contains at most one NaN value.
-// CHECK: %[[ZERO:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: %[[RHS_REAL_ABS:.*]] = math.absf %[[RHS_REAL]] : f32
-// CHECK: %[[RHS_REAL_ABS_IS_ZERO:.*]] = arith.cmpf oeq, %[[RHS_REAL_ABS]], %[[ZERO]] : f32
-// CHECK: %[[RHS_IMAG_ABS:.*]] = math.absf %[[RHS_IMAG]] : f32
-// CHECK: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = arith.cmpf oeq, %[[RHS_IMAG_ABS]], %[[ZERO]] : f32
-// CHECK: %[[LHS_REAL_IS_NOT_NAN:.*]] = arith.cmpf ord, %[[LHS_REAL]], %[[ZERO]] : f32
-// CHECK: %[[LHS_IMAG_IS_NOT_NAN:.*]] = arith.cmpf ord, %[[LHS_IMAG]], %[[ZERO]] : f32
-// CHECK: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = arith.ori %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]] : i1
-// CHECK: %[[RHS_IS_ZERO:.*]] = arith.andi %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]] : i1
-// CHECK: %[[RESULT_IS_INFINITY:.*]] = arith.andi %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]] : i1
+// CHECK: %[[IMAG:.*]] = complex.re %[[ARG]] : complex<f32>
+// CHECK: %[[V0:.*]] = complex.im %[[ARG]] : complex<f32>
+// CHECK: %[[NEG_ONE:.*]] = arith.constant -1.000000e+00 : f32
+// CHECK: %[[REAL:.*]] = arith.mulf %[[V0]], %[[NEG_ONE]] : f32
 // CHECK: %[[INF:.*]] = arith.constant 0x7F800000 : f32
-// CHECK: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = math.copysign %[[INF]], %[[RHS_REAL]] : f32
-// CHECK: %[[INFINITY_RESULT_REAL:.*]] = arith.mulf %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]] : f32
-// CHECK: %[[INFINITY_RESULT_IMAG:.*]] = arith.mulf %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]] : f32
-
-// Case 2. Infinite numerator, finite denominator.
-// CHECK: %[[RHS_REAL_FINITE:.*]] = arith.cmpf one, %[[RHS_REAL_ABS]], %[[INF]] : f32
-// CHECK: %[[RHS_IMAG_FINITE:.*]] = arith.cmpf one, %[[RHS_IMAG_ABS]], %[[INF]] : f32
-// CHECK: %[[RHS_IS_FINITE:.*]] = arith.andi %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]] : i1
-// CHECK: %[[LHS_REAL_ABS:.*]] = math.absf %[[LHS_REAL]] : f32
-// CHECK: %[[LHS_REAL_INFINITE:.*]] = arith.cmpf oeq, %[[LHS_REAL_ABS]], %[[INF]] : f32
-// CHECK: %[[LHS_IMAG_ABS:.*]] = math.absf %[[LHS_IMAG]] : f32
-// CHECK: %[[LHS_IMAG_INFINITE:.*]] = arith.cmpf oeq, %[[LHS_IMAG_ABS]], %[[INF]] : f32
-// CHECK: %[[LHS_IS_INFINITE:.*]] = arith.ori %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]] : i1
-// CHECK: %[[INF_NUM_FINITE_DENOM:.*]] = arith.andi %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]] : i1
-// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
-// CHECK: %[[LHS_REAL_IS_INF:.*]] = arith.select %[[LHS_REAL_INFINITE]], %[[ONE]], %[[ZERO]] : f32
-// CHECK: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = math.copysign %[[LHS_REAL_IS_INF]], %[[LHS_REAL]] : f32
-// CHECK: %[[LHS_IMAG_IS_INF:.*]] = arith.select %[[LHS_IMAG_INFINITE]], %[[ONE]], %[[ZERO]] : f32
-// CHECK: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = math.copysign %[[LHS_IMAG_IS_INF]], %[[LHS_IMAG]] : f32
-// CHECK: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = arith.mulf %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]] : f32
-// CHECK: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = arith.mulf %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]] : f32
-// CHECK: %[[INF_MULTIPLICATOR_1:.*]] = arith.addf %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]] : f32
-// CHECK: %[[RESULT_REAL_3:.*]] = arith.mulf %[[INF]], %[[INF_MULTIPLICATOR_1]] : f32
-// CHECK: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = arith.mulf %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]] : f32
-// CHECK: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = arith.mulf %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]] : f32
-// CHECK: %[[INF_MULTIPLICATOR_2:.*]] = arith.subf %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]] : f32
-// CHECK: %[[RESULT_IMAG_3:.*]] = arith.mulf %[[INF]], %[[INF_MULTIPLICATOR_2]] : f32
-
-// Case 3. Finite numerator, infinite denominator.
-// CHECK: %[[LHS_REAL_FINITE:.*]] = arith.cmpf one, %[[LHS_REAL_ABS]], %[[INF]] : f32
-// CHECK: %[[LHS_IMAG_FINITE:.*]] = arith.cmpf one, %[[LHS_IMAG_ABS]], %[[INF]] : f32
-// CHECK: %[[LHS_IS_FINITE:.*]] = arith.andi %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]] : i1
-// CHECK: %[[RHS_REAL_INFINITE:.*]] = arith.cmpf oeq, %[[RHS_REAL_ABS]], %[[INF]] : f32
-// CHECK: %[[RHS_IMAG_INFINITE:.*]] = arith.cmpf oeq, %[[RHS_IMAG_ABS]], %[[INF]] : f32
-// CHECK: %[[RHS_IS_INFINITE:.*]] = arith.ori %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]] : i1
-// CHECK: %[[FINITE_NUM_INFINITE_DENOM:.*]] = arith.andi %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]] : i1
-// CHECK: %[[RHS_REAL_IS_INF:.*]] = arith.select %[[RHS_REAL_INFINITE]], %[[ONE]], %[[ZERO]] : f32
-// CHECK: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = math.copysign %[[RHS_REAL_IS_INF]], %[[RHS_REAL]] : f32
-// CHECK: %[[RHS_IMAG_IS_INF:.*]] = arith.select %[[RHS_IMAG_INFINITE]], %[[ONE]], %[[ZERO]] : f32
-// CHECK: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = math.copysign %[[RHS_IMAG_IS_INF]], %[[RHS_IMAG]] : f32
-// CHECK: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = arith.mulf %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]] : f32
-// CHECK: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = arith.mulf %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]] : f32
-// CHECK: %[[ZERO_MULTIPLICATOR_1:.*]] = arith.addf %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]] : f32
-// CHECK: %[[RESULT_REAL_4:.*]] = arith.mulf %[[ZERO]], %[[ZERO_MULTIPLICATOR_1]] : f32
-// CHECK: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = arith.mulf %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]] : f32
-// CHECK: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = arith.mulf %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]] : f32
-// CHECK: %[[ZERO_MULTIPLICATOR_2:.*]] = arith.subf %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]] : f32
-// CHECK: %[[RESULT_IMAG_4:.*]] = arith.mulf %[[ZERO]], %[[ZERO_MULTIPLICATOR_2]] : f32
-
-// CHECK: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = arith.cmpf olt, %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]] : f32
-// CHECK: %[[RESULT_REAL:.*]] = arith.select %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], %[[RESULT_REAL_1]], %[[RESULT_REAL_2]] : f32
-// CHECK: %[[RESULT_IMAG:.*]] = arith.select %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], %[[RESULT_IMAG_1]], %[[RESULT_IMAG_2]] : f32
-// CHECK: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = arith.select %[[FINITE_NUM_INFINITE_DENOM]], %[[RESULT_REAL_4]], %[[RESULT_REAL]] : f32
-// CHECK: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = arith.select %[[FINITE_NUM_INFINITE_DENOM]], %[[RESULT_IMAG_4]], %[[RESULT_IMAG]] : f32
-// CHECK: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = arith.select %[[INF_NUM_FINITE_DENOM]], %[[RESULT_REAL_3]], %[[RESULT_REAL_SPECIAL_CASE_3]] : f32
-// CHECK: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = arith.select %[[INF_NUM_FINITE_DENOM]], %[[RESULT_IMAG_3]], %[[RESULT_IMAG_SPECIAL_CASE_3]] : f32
-// CHECK: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = arith.select %[[RESULT_IS_INFINITY]], %[[INFINITY_RESULT_REAL]], %[[RESULT_REAL_SPECIAL_CASE_2]] : f32
-// CHECK: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = arith.select %[[RESULT_IS_INFINITY]], %[[INFINITY_RESULT_IMAG]], %[[RESULT_IMAG_SPECIAL_CASE_2]] : f32
-// CHECK: %[[RESULT_REAL_IS_NAN:.*]] = arith.cmpf uno, %[[RESULT_REAL]], %[[ZERO]] : f32
-// CHECK: %[[RESULT_IMAG_IS_NAN:.*]] = arith.cmpf uno, %[[RESULT_IMAG]], %[[ZERO]] : f32
-// CHECK: %[[RESULT_IS_NAN:.*]] = arith.andi %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]] : i1
-// CHECK: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = arith.select %[[RESULT_IS_NAN]], %[[RESULT_REAL_SPECIAL_CASE_1]], %[[RESULT_REAL]] : f32
-// CHECK: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = arith.select %[[RESULT_IS_NAN]], %[[RESULT_IMAG_SPECIAL_CASE_1]], %[[RESULT_IMAG]] : f32
-// CHECK: %[[RESULT:.*]] = complex.create %[[RESULT_REAL_WITH_SPECIAL_CASES]], %[[RESULT_IMAG_WITH_SPECIAL_CASES]] : complex<f32>
+// CHECK: %[[FOUR:.*]] = arith.constant 4.000000e+00 : f32
+// CHECK: %[[TWO_REAL:.*]] = arith.addf %[[REAL]], %[[REAL]] : f32
+// CHECK: %[[NEG_TWO_REAL:.*]] = arith.mulf %[[NEG_ONE]], %[[TWO_REAL]] : f32
+// CHECK: %[[EXPM1:.*]] = math.expm1 %[[TWO_REAL]] : f32
+// CHECK: %[[EXPM1_2:.*]] = math.expm1 %[[NEG_TWO_REAL]] : f32
+// CHECK: %[[REAL_NUM:.*]] = arith.subf %[[EXPM1]], %[[EXPM1_2]] : f32
+// CHECK: %[[COS:.*]] = math.cos %[[IMAG]] : f32
+// CHECK: %[[COS_SQ:.*]] = arith.mulf %[[COS]], %[[COS]] : f32
+// CHECK: %[[FOUR_COS_SQ:.*]] = arith.mulf %[[COS_SQ]], %[[FOUR]] : f32
+// CHECK: %[[SIN:.*]] = math.sin %[[IMAG]] : f32
+// CHECK: %[[MUL:.*]] = arith.mulf %[[COS]], %[[SIN]] : f32
+// CHECK: %[[IMAG_NUM:.*]] = arith.mulf %[[FOUR]], %[[MUL]] : f32
+// CHECK: %[[ADD:.*]] = arith.addf %[[EXPM1]], %[[EXPM1_2]] : f32
+// CHECK: %[[DENOM:.*]] = arith.addf %[[ADD]], %[[FOUR_COS_SQ]] : f32
+// CHECK: %[[IS_INF:.*]] = arith.cmpf oeq, %[[ADD]], %[[INF]] : f32
+// CHECK: %[[LIMIT:.*]] = math.copysign %[[NEG_ONE]], %[[REAL]] : f32
+// CHECK: %[[RESULT_REAL:.*]] = arith.divf %[[REAL_NUM]], %[[DENOM]] : f32
+// CHECK: %[[RESULT_REAL2:.*]] = arith.select %[[IS_INF]], %[[LIMIT]], %[[RESULT_REAL]] : f32
+// CHECK: %[[RESULT_IMAG:.*]] = arith.divf %[[IMAG_NUM]], %[[DENOM]] : f32
+// CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] : f32
+// CHECK: %[[ZERO:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[NAN:.*]] = arith.constant 0x7FC00000 : f32
+// CHECK: %[[ABS_REAL_INF:.*]] = arith.cmpf oeq, %[[ABS_REAL]], %[[INF]] : f32
+// CHECK: %[[IMAG_ZERO:.*]] = arith.cmpf oeq, %[[IMAG]], %[[ZERO]] : f32
+// CHECK: %true = arith.constant true
+// CHECK: %[[ABS_REAL_NOT_INF:.*]] = arith.xori %[[ABS_REAL_INF]], %true : i1
+// CHECK: %[[IMAG_IS_NAN:.*]] = arith.cmpf uno, %[[IMAG_NUM]], %[[IMAG_NUM]] : f32
+// CHECK: %[[REAL_IS_NAN:.*]] = arith.andi %[[IMAG_IS_NAN]], %[[ABS_REAL_NOT_INF]] : i1
+// CHECK: %[[AND:.*]] = arith.andi %[[ABS_REAL_INF]], %[[IMAG_IS_NAN]] : i1
+// CHECK: %[[IMAG_IS_NAN2:.*]] = arith.ori %[[IMAG_ZERO]], %[[AND]] : i1
+// CHECK: %[[RESULT_REAL3:.*]] = arith.select %[[REAL_IS_NAN]], %[[NAN]], %[[RESULT_REAL2]] : f32
+// CHECK: %[[RESULT_REAL:.*]] = arith.select %[[IMAG_IS_NAN2]], %[[ZERO]], %[[RESULT_IMAG]] : f32
+// CHECK: %[[RESULT_IMAG:.*]] = arith.mulf %[[RESULT_REAL3]], %[[NEG_ONE]]
+// CHECK: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex<f32>
 // CHECK: return %[[RESULT]] : complex<f32>
 
 // -----
@@ -679,8 +590,8 @@ func.func @complex_tanh(%arg: complex<f32>) -> complex<f32> {
 }
 // CHECK: %[[REAL:.*]] = complex.re %[[ARG]] : complex<f32>
 // CHECK: %[[IMAG:.*]] = complex.im %[[ARG]] : complex<f32>
-// CHECK: %[[INF:.*]] = arith.constant 0x7F800000 : f32
 // CHECK: %[[NEG_ONE:.*]] = arith.constant -1.000000e+00 : f32
+// CHECK: %[[INF:.*]] = arith.constant 0x7F800000 : f32
 // CHECK: %[[FOUR:.*]] = arith.constant 4.000000e+00 : f32
 // CHECK: %[[TWO_REAL:.*]] = arith.addf %[[REAL]], %[[REAL]] : f32
 // CHECK: %[[NEG_TWO_REAL:.*]] = arith.mulf %[[NEG_ONE]], %[[TWO_REAL]] : f32
@@ -1997,135 +1908,45 @@ func.func @complex_tan_with_fmf(%arg: complex<f32>) -> complex<f32> {
   return %tan : complex<f32>
 }
 
-// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]]
-// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]]
-// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32
-// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath<nnan,contract> : f32
-// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath<nnan,contract>
-// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath<nnan,contract>
-// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath<nnan,contract> : f32
-// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath<nnan,contract> : f32
-// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_REXP]], %[[HALF_EXP]] fastmath<nnan,contract>
-// CHECK-DAG: %[[COS_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[COS]] fastmath<nnan,contract>
-// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_REXP]], %[[HALF_EXP]] fastmath<nnan,contract>
-// CHECK-DAG: %[[COS_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[SIN]] fastmath<nnan,contract>
-// CHECK-DAG: %[[COS_COMP:.*]] = complex.create %[[COS_REAL]], %[[COS_IMAG]] : complex<f32>
-
-// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]]
-// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]]
-// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32
-// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath<nnan,contract> : f32
-// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath<nnan,contract>
-// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath<nnan,contract>
-// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath<nnan,contract> : f32
-// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath<nnan,contract> : f32
-// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_EXP]], %[[HALF_REXP]] fastmath<nnan,contract>
-// CHECK-DAG: %[[SIN_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[SIN]] fastmath<nnan,contract>
-// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_EXP]], %[[HALF_REXP]] fastmath<nnan,contract>
-// CHECK-DAG: %[[SIN_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[COS]] fastmath<nnan,contract>
-// CHECK-DAG: %[[SIN_COMP:.*]] = complex.create %[[SIN_REAL]], %[[SIN_IMAG]] : complex<f32>
-
-// CHECK: %[[LHS_REAL:.*]] = complex.re %[[SIN_COMP]] : complex<f32>
-// CHECK: %[[LHS_IMAG:.*]] = complex.im %[[SIN_COMP]] : complex<f32>
-// CHECK: %[[RHS_REAL:.*]] = complex.re %[[COS_COMP]] : complex<f32>
-// CHECK: %[[RHS_IMAG:.*]] = complex.im %[[COS_COMP]] : complex<f32>
-
-// CHECK: %[[RHS_REAL_IMAG_RATIO:.*]] = arith.divf %[[RHS_REAL]], %[[RHS_IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = arith.mulf %[[RHS_REAL_IMAG_RATIO]], %[[RHS_REAL]] fastmath<nnan,contract> : f32
-// CHECK: %[[RHS_REAL_IMAG_DENOM:.*]] = arith.addf %[[RHS_IMAG]], %[[RHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]] fastmath<nnan,contract> : f32
-// CHECK: %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO:.*]] = arith.mulf %[[LHS_REAL]], %[[RHS_REAL_IMAG_RATIO]] fastmath<nnan,contract> : f32
-// CHECK: %[[REAL_NUMERATOR_1:.*]] = arith.addf %[[LHS_REAL_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[RESULT_REAL_1:.*]] = arith.divf %[[REAL_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]] fastmath<nnan,contract> : f32
-// CHECK: %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO:.*]] = arith.mulf %[[LHS_IMAG]], %[[RHS_REAL_IMAG_RATIO]] fastmath<nnan,contract> : f32
-// CHECK: %[[IMAG_NUMERATOR_1:.*]] = arith.subf %[[LHS_IMAG_TIMES_RHS_REAL_IMAG_RATIO]], %[[LHS_REAL]] fastmath<nnan,contract> : f32
-// CHECK: %[[RESULT_IMAG_1:.*]] = arith.divf %[[IMAG_NUMERATOR_1]], %[[RHS_REAL_IMAG_DENOM]] fastmath<nnan,contract> : f32
-
-// CHECK: %[[RHS_IMAG_REAL_RATIO:.*]] = arith.divf %[[RHS_IMAG]], %[[RHS_REAL]] fastmath<nnan,contract> : f32
-// CHECK: %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = arith.mulf %[[RHS_IMAG_REAL_RATIO]], %[[RHS_IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[RHS_IMAG_REAL_DENOM:.*]] = arith.addf %[[RHS_REAL]], %[[RHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]] fastmath<nnan,contract> : f32
-// CHECK: %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO:.*]] = arith.mulf %[[LHS_IMAG]], %[[RHS_IMAG_REAL_RATIO]] fastmath<nnan,contract> : f32
-// CHECK: %[[REAL_NUMERATOR_2:.*]] = arith.addf %[[LHS_REAL]], %[[LHS_IMAG_TIMES_RHS_IMAG_REAL_RATIO]] fastmath<nnan,contract> : f32
-// CHECK: %[[RESULT_REAL_2:.*]] = arith.divf %[[REAL_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]] fastmath<nnan,contract> : f32
-// CHECK: %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO:.*]] = arith.mulf %[[LHS_REAL]], %[[RHS_IMAG_REAL_RATIO]] fastmath<nnan,contract> : f32
-// CHECK: %[[IMAG_NUMERATOR_2:.*]] = arith.subf %[[LHS_IMAG]], %[[LHS_REAL_TIMES_RHS_IMAG_REAL_RATIO]] fastmath<nnan,contract> : f32
-// CHECK: %[[RESULT_IMAG_2:.*]] = arith.divf %[[IMAG_NUMERATOR_2]], %[[RHS_IMAG_REAL_DENOM]] fastmath<nnan,contract> : f32
-
-// Case 1. Zero denominator, numerator contains at most one NaN value.
-// CHECK: %[[ZERO:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: %[[RHS_REAL_ABS:.*]] = math.absf %[[RHS_REAL]] fastmath<nnan,contract> : f32
-// CHECK: %[[RHS_REAL_ABS_IS_ZERO:.*]] = arith.cmpf oeq, %[[RHS_REAL_ABS]], %[[ZERO]] : f32
-// CHECK: %[[RHS_IMAG_ABS:.*]] = math.absf %[[RHS_IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[RHS_IMAG_ABS_IS_ZERO:.*]] = arith.cmpf oeq, %[[RHS_IMAG_ABS]], %[[ZERO]] : f32
-// CHECK: %[[LHS_REAL_IS_NOT_NAN:.*]] = arith.cmpf ord, %[[LHS_REAL]], %[[ZERO]] : f32
-// CHECK: %[[LHS_IMAG_IS_NOT_NAN:.*]] = arith.cmpf ord, %[[LHS_IMAG]], %[[ZERO]] : f32
-// CHECK: %[[LHS_CONTAINS_NOT_NAN_VALUE:.*]] = arith.ori %[[LHS_REAL_IS_NOT_NAN]], %[[LHS_IMAG_IS_NOT_NAN]] : i1
-// CHECK: %[[RHS_IS_ZERO:.*]] = arith.andi %[[RHS_REAL_ABS_IS_ZERO]], %[[RHS_IMAG_ABS_IS_ZERO]] : i1
-// CHECK: %[[RESULT_IS_INFINITY:.*]] = arith.andi %[[LHS_CONTAINS_NOT_NAN_VALUE]], %[[RHS_IS_ZERO]] : i1
+// CHECK: %[[IMAG:.*]] = complex.re %[[ARG]] : complex<f32>
+// CHECK: %[[V0:.*]] = complex.im %[[ARG]] : complex<f32>
+// CHECK: %[[NEG_ONE:.*]] = arith.constant -1.000000e+00 : f32
+// CHECK: %[[REAL:.*]] = arith.mulf %[[V0]], %cst fastmath<nnan,contract> : f32
 // CHECK: %[[INF:.*]] = arith.constant 0x7F800000 : f32
-// CHECK: %[[INF_WITH_SIGN_OF_RHS_REAL:.*]] = math.copysign %[[INF]], %[[RHS_REAL]] : f32
-// CHECK: %[[INFINITY_RESULT_REAL:.*]] = arith.mulf %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_REAL]] fastmath<nnan,contract> : f32
-// CHECK: %[[INFINITY_RESULT_IMAG:.*]] = arith.mulf %[[INF_WITH_SIGN_OF_RHS_REAL]], %[[LHS_IMAG]] fastmath<nnan,contract> : f32
-
-// Case 2. Infinite numerator, finite denominator.
-// CHECK: %[[RHS_REAL_FINITE:.*]] = arith.cmpf one, %[[RHS_REAL_ABS]], %[[INF]] : f32
-// CHECK: %[[RHS_IMAG_FINITE:.*]] = arith.cmpf one, %[[RHS_IMAG_ABS]], %[[INF]] : f32
-// CHECK: %[[RHS_IS_FINITE:.*]] = arith.andi %[[RHS_REAL_FINITE]], %[[RHS_IMAG_FINITE]] : i1
-// CHECK: %[[LHS_REAL_ABS:.*]] = math.absf %[[LHS_REAL]] fastmath<nnan,contract> : f32
-// CHECK: %[[LHS_REAL_INFINITE:.*]] = arith.cmpf oeq, %[[LHS_REAL_ABS]], %[[INF]] : f32
-// CHECK: %[[LHS_IMAG_ABS:.*]] = math.absf %[[LHS_IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[LHS_IMAG_INFINITE:.*]] = arith.cmpf oeq, %[[LHS_IMAG_ABS]], %[[INF]] : f32
-// CHECK: %[[LHS_IS_INFINITE:.*]] = arith.ori %[[LHS_REAL_INFINITE]], %[[LHS_IMAG_INFINITE]] : i1
-// CHECK: %[[INF_NUM_FINITE_DENOM:.*]] = arith.andi %[[LHS_IS_INFINITE]], %[[RHS_IS_FINITE]] : i1
-// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
-// CHECK: %[[LHS_REAL_IS_INF:.*]] = arith.select %[[LHS_REAL_INFINITE]], %[[ONE]], %[[ZERO]] : f32
-// CHECK: %[[LHS_REAL_IS_INF_WITH_SIGN:.*]] = math.copysign %[[LHS_REAL_IS_INF]], %[[LHS_REAL]] : f32
-// CHECK: %[[LHS_IMAG_IS_INF:.*]] = arith.select %[[LHS_IMAG_INFINITE]], %[[ONE]], %[[ZERO]] : f32
-// CHECK: %[[LHS_IMAG_IS_INF_WITH_SIGN:.*]] = math.copysign %[[LHS_IMAG_IS_INF]], %[[LHS_IMAG]] : f32
-// CHECK: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = arith.mulf %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_REAL]] fastmath<nnan,contract> : f32
-// CHECK: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = arith.mulf %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[INF_MULTIPLICATOR_1:.*]] = arith.addf %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[RESULT_REAL_3:.*]] = arith.mulf %[[INF]], %[[INF_MULTIPLICATOR_1]] fastmath<nnan,contract> : f32
-// CHECK: %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG:.*]] = arith.mulf %[[LHS_REAL_IS_INF_WITH_SIGN]], %[[RHS_IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL:.*]] = arith.mulf %[[LHS_IMAG_IS_INF_WITH_SIGN]], %[[RHS_REAL]] fastmath<nnan,contract> : f32
-// CHECK: %[[INF_MULTIPLICATOR_2:.*]] = arith.subf %[[LHS_IMAG_IS_INF_WITH_SIGN_TIMES_RHS_REAL]], %[[LHS_REAL_IS_INF_WITH_SIGN_TIMES_RHS_IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[RESULT_IMAG_3:.*]] = arith.mulf %[[INF]], %[[INF_MULTIPLICATOR_2]] fastmath<nnan,contract> : f32
-
-// Case 3. Finite numerator, infinite denominator.
-// CHECK: %[[LHS_REAL_FINITE:.*]] = arith.cmpf one, %[[LHS_REAL_ABS]], %[[INF]] : f32
-// CHECK: %[[LHS_IMAG_FINITE:.*]] = arith.cmpf one, %[[LHS_IMAG_ABS]], %[[INF]] : f32
-// CHECK: %[[LHS_IS_FINITE:.*]] = arith.andi %[[LHS_REAL_FINITE]], %[[LHS_IMAG_FINITE]] : i1
-// CHECK: %[[RHS_REAL_INFINITE:.*]] = arith.cmpf oeq, %[[RHS_REAL_ABS]], %[[INF]] : f32
-// CHECK: %[[RHS_IMAG_INFINITE:.*]] = arith.cmpf oeq, %[[RHS_IMAG_ABS]], %[[INF]] : f32
-// CHECK: %[[RHS_IS_INFINITE:.*]] = arith.ori %[[RHS_REAL_INFINITE]], %[[RHS_IMAG_INFINITE]] : i1
-// CHECK: %[[FINITE_NUM_INFINITE_DENOM:.*]] = arith.andi %[[LHS_IS_FINITE]], %[[RHS_IS_INFINITE]] : i1
-// CHECK: %[[RHS_REAL_IS_INF:.*]] = arith.select %[[RHS_REAL_INFINITE]], %[[ONE]], %[[ZERO]] : f32
-// CHECK: %[[RHS_REAL_IS_INF_WITH_SIGN:.*]] = math.copysign %[[RHS_REAL_IS_INF]], %[[RHS_REAL]] : f32
-// CHECK: %[[RHS_IMAG_IS_INF:.*]] = arith.select %[[RHS_IMAG_INFINITE]], %[[ONE]], %[[ZERO]] : f32
-// CHECK: %[[RHS_IMAG_IS_INF_WITH_SIGN:.*]] = math.copysign %[[RHS_IMAG_IS_INF]], %[[RHS_IMAG]] : f32
-// CHECK: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = arith.mulf %[[LHS_REAL]], %[[RHS_REAL_IS_INF_WITH_SIGN]] fastmath<nnan,contract> : f32
-// CHECK: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = arith.mulf %[[LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN]] fastmath<nnan,contract> : f32
-// CHECK: %[[ZERO_MULTIPLICATOR_1:.*]] = arith.addf %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[RESULT_REAL_4:.*]] = arith.mulf %[[ZERO]], %[[ZERO_MULTIPLICATOR_1]] fastmath<nnan,contract> : f32
-// CHECK: %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG:.*]] = arith.mulf %[[LHS_IMAG]], %[[RHS_REAL_IS_INF_WITH_SIGN]] fastmath<nnan,contract> : f32
-// CHECK: %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL:.*]] = arith.mulf %[[LHS_REAL]], %[[RHS_IMAG_IS_INF_WITH_SIGN]] fastmath<nnan,contract> : f32
-// CHECK: %[[ZERO_MULTIPLICATOR_2:.*]] = arith.subf %[[RHS_REAL_IS_INF_WITH_SIGN_TIMES_LHS_IMAG]], %[[RHS_IMAG_IS_INF_WITH_SIGN_TIMES_LHS_REAL]] fastmath<nnan,contract> : f32
-// CHECK: %[[RESULT_IMAG_4:.*]] = arith.mulf %[[ZERO]], %[[ZERO_MULTIPLICATOR_2]] fastmath<nnan,contract> : f32
-
-// CHECK: %[[REAL_ABS_SMALLER_THAN_IMAG_ABS:.*]] = arith.cmpf olt, %[[RHS_REAL_ABS]], %[[RHS_IMAG_ABS]] : f32
-// CHECK: %[[RESULT_REAL:.*]] = arith.select %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], %[[RESULT_REAL_1]], %[[RESULT_REAL_2]] : f32
-// CHECK: %[[RESULT_IMAG:.*]] = arith.select %[[REAL_ABS_SMALLER_THAN_IMAG_ABS]], %[[RESULT_IMAG_1]], %[[RESULT_IMAG_2]] : f32
-// CHECK: %[[RESULT_REAL_SPECIAL_CASE_3:.*]] = arith.select %[[FINITE_NUM_INFINITE_DENOM]], %[[RESULT_REAL_4]], %[[RESULT_REAL]] : f32
-// CHECK: %[[RESULT_IMAG_SPECIAL_CASE_3:.*]] = arith.select %[[FINITE_NUM_INFINITE_DENOM]], %[[RESULT_IMAG_4]], %[[RESULT_IMAG]] : f32
-// CHECK: %[[RESULT_REAL_SPECIAL_CASE_2:.*]] = arith.select %[[INF_NUM_FINITE_DENOM]], %[[RESULT_REAL_3]], %[[RESULT_REAL_SPECIAL_CASE_3]] : f32
-// CHECK: %[[RESULT_IMAG_SPECIAL_CASE_2:.*]] = arith.select %[[INF_NUM_FINITE_DENOM]], %[[RESULT_IMAG_3]], %[[RESULT_IMAG_SPECIAL_CASE_3]] : f32
-// CHECK: %[[RESULT_REAL_SPECIAL_CASE_1:.*]] = arith.select %[[RESULT_IS_INFINITY]], %[[INFINITY_RESULT_REAL]], %[[RESULT_REAL_SPECIAL_CASE_2]] : f32
-// CHECK: %[[RESULT_IMAG_SPECIAL_CASE_1:.*]] = arith.select %[[RESULT_IS_INFINITY]], %[[INFINITY_RESULT_IMAG]], %[[RESULT_IMAG_SPECIAL_CASE_2]] : f32
-// CHECK: %[[RESULT_REAL_IS_NAN:.*]] = arith.cmpf uno, %[[RESULT_REAL]], %[[ZERO]] : f32
-// CHECK: %[[RESULT_IMAG_IS_NAN:.*]] = arith.cmpf uno, %[[RESULT_IMAG]], %[[ZERO]] : f32
-// CHECK: %[[RESULT_IS_NAN:.*]] = arith.andi %[[RESULT_REAL_IS_NAN]], %[[RESULT_IMAG_IS_NAN]] : i1
-// CHECK: %[[RESULT_REAL_WITH_SPECIAL_CASES:.*]] = arith.select %[[RESULT_IS_NAN]], %[[RESULT_REAL_SPECIAL_CASE_1]], %[[RESULT_REAL]] : f32
-// CHECK: %[[RESULT_IMAG_WITH_SPECIAL_CASES:.*]] = arith.select %[[RESULT_IS_NAN]], %[[RESULT_IMAG_SPECIAL_CASE_1]], %[[RESULT_IMAG]] : f32
-// CHECK: %[[RESULT:.*]] = complex.create %[[RESULT_REAL_WITH_SPECIAL_CASES]], %[[RESULT_IMAG_WITH_SPECIAL_CASES]] : complex<f32>
+// CHECK: %[[FOUR:.*]] = arith.constant 4.000000e+00 : f32
+// CHECK: %[[TWO_REAL:.*]] = arith.addf %[[REAL]], %[[REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[NEG_TWO_REAL:.*]] = arith.mulf %[[NEG_ONE]], %[[TWO_REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[EXPM1:.*]] = math.expm1 %[[TWO_REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[EXPM1_2:.*]] = math.expm1 %[[NEG_TWO_REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[REAL_NUM:.*]] = arith.subf %[[EXPM1]], %[[EXPM1_2]] fastmath<nnan,contract> : f32
+// CHECK: %[[COS:.*]] = math.cos %[[IMAG]] fastmath<nnan,contract> : f32
+// CHECK: %[[COS_SQ:.*]] = arith.mulf %[[COS]], %[[COS]] fastmath<nnan,contract> : f32
+// CHECK: %[[FOUR_COS_SQ:.*]] = arith.mulf %[[COS_SQ]], %[[FOUR]] fastmath<nnan,contract> : f32
+// CHECK: %[[SIN:.*]] = math.sin %[[IMAG]] fastmath<nnan,contract> : f32
+// CHECK: %[[MUL:.*]] = arith.mulf %[[COS]], %[[SIN]] fastmath<nnan,contract> : f32
+// CHECK: %[[IMAG_NUM:.*]] = arith.mulf %[[FOUR]], %[[MUL]] fastmath<nnan,contract> : f32
+// CHECK: %[[ADD:.*]] = arith.addf %[[EXPM1]], %[[EXPM1_2]] fastmath<nnan,contract> : f32
+// CHECK: %[[DENOM:.*]] = arith.addf %[[ADD]], %[[FOUR_COS_SQ]] fastmath<nnan,contract> : f32
+// CHECK: %[[IS_INF:.*]] = arith.cmpf oeq, %[[ADD]], %[[INF]] fastmath<nnan,contract> : f32
+// CHECK: %[[LIMIT:.*]] = math.copysign %[[NEG_ONE]], %[[REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_REAL:.*]] = arith.divf %[[REAL_NUM]], %[[DENOM]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_REAL2:.*]] = arith.select %[[IS_INF]], %[[LIMIT]], %[[RESULT_REAL]] : f32
+// CHECK: %[[RESULT_IMAG:.*]] = arith.divf %[[IMAG_NUM]], %[[DENOM]] fastmath<nnan,contract> : f32
+// CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[ZERO:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[NAN:.*]] = arith.constant 0x7FC00000 : f32
+// CHECK: %[[ABS_REAL_INF:.*]] = arith.cmpf oeq, %[[ABS_REAL]], %[[INF]] fastmath<nnan,contract> : f32
+// CHECK: %[[IMAG_ZERO:.*]] = arith.cmpf oeq, %[[IMAG]], %[[ZERO]] fastmath<nnan,contract> : f32
+// CHECK: %true = arith.constant true
+// CHECK: %[[ABS_REAL_NOT_INF:.*]] = arith.xori %[[ABS_REAL_INF]], %true : i1
+// CHECK: %[[IMAG_IS_NAN:.*]] = arith.cmpf uno, %[[IMAG_NUM]], %[[IMAG_NUM]] fastmath<nnan,contract> : f32
+// CHECK: %[[REAL_IS_NAN:.*]] = arith.andi %[[IMAG_IS_NAN]], %[[ABS_REAL_NOT_INF]] : i1
+// CHECK: %[[AND:.*]] = arith.andi %[[ABS_REAL_INF]], %[[IMAG_IS_NAN]] : i1
+// CHECK: %[[IMAG_IS_NAN2:.*]] = arith.ori %[[IMAG_ZERO]], %[[AND]] : i1
+// CHECK: %[[RESULT_REAL3:.*]] = arith.select %[[REAL_IS_NAN]], %[[NAN]], %[[RESULT_REAL2]] : f32
+// CHECK: %[[RESULT_REAL:.*]] = arith.select %[[IMAG_IS_NAN2]], %[[ZERO]], %[[RESULT_IMAG]] : f32
+// CHECK: %[[RESULT_IMAG:.*]] = arith.mulf %[[RESULT_REAL3]], %[[NEG_ONE]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex<f32>
 // CHECK: return %[[RESULT]] : complex<f32>
 
 // -----
@@ -2139,8 +1960,8 @@ func.func @complex_tanh_with_fmf(%arg: complex<f32>) -> complex<f32> {
 
 // CHECK: %[[REAL:.*]] = complex.re %[[ARG]] : complex<f32>
 // CHECK: %[[IMAG:.*]] = complex.im %[[ARG]] : complex<f32>
-// CHECK: %[[INF:.*]] = arith.constant 0x7F800000 : f32
 // CHECK: %[[NEG_ONE:.*]] = arith.constant -1.000000e+00 : f32
+// CHECK: %[[INF:.*]] = arith.constant 0x7F800000 : f32
 // CHECK: %[[FOUR:.*]] = arith.constant 4.000000e+00 : f32
 // CHECK: %[[TWO_REAL:.*]] = arith.addf %[[REAL]], %[[REAL]] fastmath<nnan,contract> : f32
 // CHECK: %[[NEG_TWO_REAL:.*]] = arith.mulf %[[NEG_ONE]], %[[TWO_REAL]] fastmath<nnan,contract> : f32
-- 
GitLab


From 5b7088c3619e95eaa8e154765470ddb4d3859fa4 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 17 May 2024 08:45:00 +0200
Subject: [PATCH 1092/1206] TargetLibraryInfo: Assume no libcalls in the
 default constructor (#92400)

The only tricky point here is PlaceSafepoints has an awful hack where
it's creating a legacy PassManager inside it's runImpl, which was not
propagating the incoming TLI. This means there's an implicit bug fix,
where PlaceSafepoints would have been treating too many calls as
builtins.

I'm trying to delete the default constructor altogether, but this seems
to be more difficult.
---
 llvm/lib/Analysis/TargetLibraryInfo.cpp | 51 +++++++++++++++----------
 1 file changed, 31 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 684b0759f315..7ce42447b630 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -160,11 +160,28 @@ bool TargetLibraryInfoImpl::isCallingConvCCompatible(Function *F) {
                                     F->getFunctionType());
 }
 
+static void initializeBase(TargetLibraryInfoImpl &TLI, const Triple &T) {
+  bool ShouldExtI32Param, ShouldExtI32Return;
+  bool ShouldSignExtI32Param, ShouldSignExtI32Return;
+  TargetLibraryInfo::initExtensionsForTriple(
+      ShouldExtI32Param, ShouldExtI32Return, ShouldSignExtI32Param,
+      ShouldSignExtI32Return, T);
+  TLI.setShouldExtI32Param(ShouldExtI32Param);
+  TLI.setShouldExtI32Return(ShouldExtI32Return);
+  TLI.setShouldSignExtI32Param(ShouldSignExtI32Param);
+  TLI.setShouldSignExtI32Return(ShouldSignExtI32Return);
+
+  // Let's assume by default that the size of int is 32 bits, unless the target
+  // is a 16-bit architecture because then it most likely is 16 bits. If that
+  // isn't true for a target those defaults should be overridden below.
+  TLI.setIntSize(T.isArch16Bit() ? 16 : 32);
+}
+
 /// Initialize the set of available library functions based on the specified
 /// target triple. This should be carefully written so that a missing target
 /// triple gets a sane set of defaults.
-static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
-                       ArrayRef<StringLiteral> StandardNames) {
+static void initializeLibCalls(TargetLibraryInfoImpl &TLI, const Triple &T,
+                               ArrayRef<StringLiteral> StandardNames) {
   // Set IO unlocked variants as unavailable
   // Set them as available per system below
   TLI.setUnavailable(LibFunc_getc_unlocked);
@@ -178,20 +195,6 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
   TLI.setUnavailable(LibFunc_fputs_unlocked);
   TLI.setUnavailable(LibFunc_fgets_unlocked);
 
-  bool ShouldExtI32Param, ShouldExtI32Return;
-  bool ShouldSignExtI32Param, ShouldSignExtI32Return;
-  TargetLibraryInfo::initExtensionsForTriple(ShouldExtI32Param,
-       ShouldExtI32Return, ShouldSignExtI32Param, ShouldSignExtI32Return, T);
-  TLI.setShouldExtI32Param(ShouldExtI32Param);
-  TLI.setShouldExtI32Return(ShouldExtI32Return);
-  TLI.setShouldSignExtI32Param(ShouldSignExtI32Param);
-  TLI.setShouldSignExtI32Return(ShouldSignExtI32Return);
-
-  // Let's assume by default that the size of int is 32 bits, unless the target
-  // is a 16-bit architecture because then it most likely is 16 bits. If that
-  // isn't true for a target those defaults should be overridden below.
-  TLI.setIntSize(T.isArch16Bit() ? 16 : 32);
-
   // There is really no runtime library on AMDGPU, apart from
   // __kmpc_alloc/free_shared.
   if (T.isAMDGPU()) {
@@ -882,11 +885,19 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
   TLI.addVectorizableFunctionsFromVecLib(ClVectorLibrary, T);
 }
 
-TargetLibraryInfoImpl::TargetLibraryInfoImpl() {
-  // Default to everything being available.
-  memset(AvailableArray, -1, sizeof(AvailableArray));
+/// Initialize the set of available library functions based on the specified
+/// target triple. This should be carefully written so that a missing target
+/// triple gets a sane set of defaults.
+static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
+                       ArrayRef<StringLiteral> StandardNames) {
+  initializeBase(TLI, T);
+  initializeLibCalls(TLI, T, StandardNames);
+}
 
-  initialize(*this, Triple(), StandardNames);
+TargetLibraryInfoImpl::TargetLibraryInfoImpl() {
+  // Default to nothing being available.
+  memset(AvailableArray, 0, sizeof(AvailableArray));
+  initializeBase(*this, Triple());
 }
 
 TargetLibraryInfoImpl::TargetLibraryInfoImpl(const Triple &T) {
-- 
GitLab


From 37d00635c4ff1b656a49635e826aeff8063dc1e8 Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy@intel.com>
Date: Fri, 17 May 2024 09:01:29 +0200
Subject: [PATCH 1093/1206] [SPIR-V] Ensure that internal intrinsic functions
 for PHI's operand are inserted at the correct positions (#92316)

This PR is to ensure that internal intrinsic functions for PHI's operand
are inserted at the correct positions and don't break rules of
instruction domination and PHI nodes grouping at top of basic block.
---
 llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp |  4 +-
 llvm/test/CodeGen/SPIRV/phi-insert-point.ll   | 59 +++++++++++++++++++
 2 files changed, 61 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/phi-insert-point.ll

diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index c00066f5dca6..d90249f7c2e6 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -174,7 +174,7 @@ static bool isAggrToReplace(const Value *V) {
 
 static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
   if (isa<PHINode>(I))
-    B.SetInsertPoint(I->getParent(), I->getParent()->getFirstInsertionPt());
+    B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
   else
     B.SetInsertPoint(I);
 }
@@ -491,7 +491,7 @@ void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I) {
     if (Instruction *User = dyn_cast<Instruction>(Op->use_begin()->get()))
       setInsertPointSkippingPhis(B, User->getNextNode());
     else
-      B.SetInsertPoint(I);
+      setInsertPointSkippingPhis(B, I);
     Value *OpTyVal = Constant::getNullValue(KnownElemTy);
     Type *OpTy = Op->getType();
     if (!Ty) {
diff --git a/llvm/test/CodeGen/SPIRV/phi-insert-point.ll b/llvm/test/CodeGen/SPIRV/phi-insert-point.ll
new file mode 100644
index 000000000000..41cc514f4dba
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/phi-insert-point.ll
@@ -0,0 +1,59 @@
+; The goal of the test is to check that internal intrinsic functions for PHI's
+; operand are inserted at the correct positions, and don't break rules of
+; instruction domination and PHI nodes grouping at top of basic block.
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: OpName %[[#Foo:]] "foo"
+; CHECK-DAG: OpName %[[#Bar:]] "bar"
+; CHECK: %[[#Foo]] = OpFunction
+; CHECK: OpPhi
+; CHECK-NEXT: OpPhi
+; CHECK-NEXT: OpPhi
+; CHECK-NEXT: OpPhi
+; CHECK: %[[#Bar]] = OpFunction
+; CHECK: OpPhi
+; CHECK-NEXT: OpPhi
+; CHECK-NEXT: OpPhi
+; CHECK-NEXT: OpPhi
+
+%struct = type { i64, i64 }
+
+define spir_kernel void @foo(i64 %arg_val, ptr addrspace(4) byval(%struct) %arg_ptr) {
+entry:
+  %fl = icmp eq i64 %arg_val, 0
+  br i1 %fl, label %ok, label %err
+
+err:
+  br label %ok
+
+ok:
+  %r1 = phi i64 [ undef, %err ], [ %arg_val, %entry ]
+  %r2 = phi i64 [ undef, %err ], [ %arg_val, %entry ]
+  %r3 = phi ptr addrspace(4) [ undef, %err ], [ %arg_ptr, %entry ]
+  %r4 = phi ptr addrspace(4) [ undef, %err ], [ %arg_ptr, %entry ]
+  br label %exit
+
+exit:
+  ret void
+}
+
+define spir_kernel void @bar(i64 %arg_val, i64 %arg_val_def, ptr addrspace(4) byval(%struct) %arg_ptr, ptr addrspace(4) %arg_ptr_def) {
+entry:
+  %fl = icmp eq i64 %arg_val, 0
+  br i1 %fl, label %ok, label %err
+
+err:
+  br label %ok
+
+ok:
+  %r1 = phi i64 [ %arg_val_def, %err ], [ %arg_val, %entry ]
+  %r2 = phi i64 [ %arg_val_def, %err ], [ %arg_val, %entry ]
+  %r3 = phi ptr addrspace(4) [ %arg_ptr_def, %err ], [ %arg_ptr, %entry ]
+  %r4 = phi ptr addrspace(4) [ %arg_ptr_def, %err ], [ %arg_ptr, %entry ]
+  br label %exit
+
+exit:
+  ret void
+}
-- 
GitLab


From e0a293d12f3fe1c2a284e2e116cfa77d894da49f Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 17 May 2024 00:10:00 -0700
Subject: [PATCH 1094/1206] [mlir] Fix a warning

This patch fixes:

  mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp:964:26:
  error: missing 'typename' prior to dependent type name Op::Adaptor;
  implicit 'typename' is a C++20 extension
  [-Werror,-Wc++20-extensions]
---
 mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index edca8eaecbff..d8150aeb828a 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -961,7 +961,7 @@ struct TanTanhOpConversion : public OpConversionPattern<Op> {
   using OpConversionPattern<Op>::OpConversionPattern;
 
   LogicalResult
-  matchAndRewrite(Op op, Op::Adaptor adaptor,
+  matchAndRewrite(Op op, typename Op::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     auto loc = op.getLoc();
-- 
GitLab


From b27eb0ae8280675fc8fb249d39f1ccafa3ee2187 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Fri, 17 May 2024 07:32:02 +0000
Subject: [PATCH 1095/1206] [lldb] Avoid modifying the source tree in
 TestCompletion.py

This was a side-effect of the "optimization" in #92281. Deoptimize the
code slightly.
---
 .../completion/TestCompletion.py               | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/lldb/test/API/functionalities/completion/TestCompletion.py b/lldb/test/API/functionalities/completion/TestCompletion.py
index 63842487fc33..15aeaf8d0e89 100644
--- a/lldb/test/API/functionalities/completion/TestCompletion.py
+++ b/lldb/test/API/functionalities/completion/TestCompletion.py
@@ -108,15 +108,17 @@ class CommandLineCompletionTestCase(TestBase):
         )
         err = lldb.SBError()
         local_spec = lldb.SBFileSpec(self.getBuildArtifact("libshared.so"))
-        remote_spec = (
-            lldb.SBFileSpec(
-                lldbutil.append_to_process_working_directory(self, "libshared.so"),
-                False,
+        if lldb.remote_platform:
+            self.process().LoadImage(
+                local_spec,
+                lldb.SBFileSpec(
+                    lldbutil.append_to_process_working_directory(self, "libshared.so"),
+                    False,
+                ),
+                err,
             )
-            if lldb.remote_platform
-            else lldb.SBFileSpec()
-        )
-        self.process().LoadImage(local_spec, remote_spec, err)
+        else:
+            self.process().LoadImage(local_spec, err)
         self.assertSuccess(err)
 
         self.complete_from_to("process unload ", "process unload 0")
-- 
GitLab


From c4bac7f7dcd931a5e561604e95656a24c3d1c9d9 Mon Sep 17 00:00:00 2001
From: CarolineConcatto <caroline.concatto@arm.com>
Date: Fri, 17 May 2024 09:25:21 +0100
Subject: [PATCH 1096/1206] =?UTF-8?q?[LLVM][AArch64]Use=20load/store=20wit?=
 =?UTF-8?q?h=20consecutive=20registers=20in=20SME2=20or=20S=E2=80=A6=20(#7?=
 =?UTF-8?q?7665)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…VE2.1 for spill/fill

When possible the spill/fill register in Frame Lowering uses the ld/st
consecutive pairs available in sme or sve2.1.
---
 .../Target/AArch64/AArch64FrameLowering.cpp   |  204 ++-
 .../AArch64/AArch64MachineFunctionInfo.h      |   11 +
 .../CodeGen/AArch64/sme2-intrinsics-ld1.ll    | 1488 ++++++-----------
 .../CodeGen/AArch64/sme2-intrinsics-ldnt1.ll  | 1488 ++++++-----------
 .../AArch64/sve-callee-save-restore-pairs.ll  |  556 ++++++
 5 files changed, 1815 insertions(+), 1932 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll

diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index c86c98eed24f..239862756fc4 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -1508,6 +1508,9 @@ static bool IsSVECalleeSave(MachineBasicBlock::iterator I) {
   switch (I->getOpcode()) {
   default:
     return false;
+  case AArch64::PTRUE_C_B:
+  case AArch64::LD1B_2Z_IMM:
+  case AArch64::ST1B_2Z_IMM:
   case AArch64::STR_ZXI:
   case AArch64::STR_PXI:
   case AArch64::LDR_ZXI:
@@ -2781,6 +2784,16 @@ struct RegPairInfo {
 
 } // end anonymous namespace
 
+unsigned findFreePredicateReg(BitVector &SavedRegs) {
+  for (unsigned PReg = AArch64::P8; PReg <= AArch64::P15; ++PReg) {
+    if (SavedRegs.test(PReg)) {
+      unsigned PNReg = PReg - AArch64::P0 + AArch64::PN0;
+      return PNReg;
+    }
+  }
+  return AArch64::NoRegister;
+}
+
 static void computeCalleeSaveRegisterPairs(
     MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI,
     const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
@@ -2859,7 +2872,11 @@ static void computeCalleeSaveRegisterPairs(
           RPI.Reg2 = NextReg;
         break;
       case RegPairInfo::PPR:
+        break;
       case RegPairInfo::ZPR:
+        if (AFI->getPredicateRegForFillSpill() != 0)
+          if (((RPI.Reg1 - AArch64::Z0) & 1) == 0 && (NextReg == RPI.Reg1 + 1))
+            RPI.Reg2 = NextReg;
         break;
       }
     }
@@ -2897,14 +2914,13 @@ static void computeCalleeSaveRegisterPairs(
     if (NeedsWinCFI &&
         RPI.isPaired()) // RPI.FrameIdx must be the lower index of the pair
       RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
-
     int Scale = RPI.getScale();
 
     int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
     assert(OffsetPre % Scale == 0);
 
     if (RPI.isScalable())
-      ScalableByteOffset += StackFillDir * Scale;
+      ScalableByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
     else
       ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
 
@@ -2915,9 +2931,6 @@ static void computeCalleeSaveRegisterPairs(
          (IsWindows && RPI.Reg2 == AArch64::LR)))
       ByteOffset += StackFillDir * 8;
 
-    assert(!(RPI.isScalable() && RPI.isPaired()) &&
-           "Paired spill/fill instructions don't exist for SVE vectors");
-
     // Round up size of non-pair to pair size if we need to pad the
     // callee-save area to ensure 16-byte alignment.
     if (NeedGapToAlignStack && !NeedsWinCFI &&
@@ -3004,6 +3017,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
     }
     return true;
   }
+  bool PTrueCreated = false;
   for (const RegPairInfo &RPI : llvm::reverse(RegPairs)) {
     unsigned Reg1 = RPI.Reg1;
     unsigned Reg2 = RPI.Reg2;
@@ -3038,10 +3052,10 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
        Alignment = Align(16);
        break;
     case RegPairInfo::ZPR:
-       StrOpc = AArch64::STR_ZXI;
-       Size = 16;
-       Alignment = Align(16);
-       break;
+      StrOpc = RPI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI;
+      Size = 16;
+      Alignment = Align(16);
+      break;
     case RegPairInfo::PPR:
        StrOpc = AArch64::STR_PXI;
        Size = 2;
@@ -3065,33 +3079,79 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
       std::swap(Reg1, Reg2);
       std::swap(FrameIdxReg1, FrameIdxReg2);
     }
-    MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
-    if (!MRI.isReserved(Reg1))
-      MBB.addLiveIn(Reg1);
-    if (RPI.isPaired()) {
+
+    if (RPI.isPaired() && RPI.isScalable()) {
+      const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
+      AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+      unsigned PnReg = AFI->getPredicateRegForFillSpill();
+      assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
+             "Expects SVE2.1 or SME2 target and a predicate register");
+#ifdef EXPENSIVE_CHECKS
+      auto IsPPR = [](const RegPairInfo &c) {
+        return c.Reg1 == RegPairInfo::PPR;
+      };
+      auto PPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsPPR);
+      auto IsZPR = [](const RegPairInfo &c) {
+        return c.Type == RegPairInfo::ZPR;
+      };
+      auto ZPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsZPR);
+      assert(!(PPRBegin < ZPRBegin) &&
+             "Expected callee save predicate to be handled first");
+#endif
+      if (!PTrueCreated) {
+        PTrueCreated = true;
+        BuildMI(MBB, MI, DL, TII.get(AArch64::PTRUE_C_B), PnReg)
+            .setMIFlags(MachineInstr::FrameSetup);
+      }
+      MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
+      if (!MRI.isReserved(Reg1))
+        MBB.addLiveIn(Reg1);
       if (!MRI.isReserved(Reg2))
         MBB.addLiveIn(Reg2);
-      MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
+      MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0));
       MIB.addMemOperand(MF.getMachineMemOperand(
           MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
           MachineMemOperand::MOStore, Size, Alignment));
+      MIB.addReg(PnReg);
+      MIB.addReg(AArch64::SP)
+          .addImm(RPI.Offset) // [sp, #offset*scale],
+                              // where factor*scale is implicit
+          .setMIFlag(MachineInstr::FrameSetup);
+      MIB.addMemOperand(MF.getMachineMemOperand(
+          MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
+          MachineMemOperand::MOStore, Size, Alignment));
+      if (NeedsWinCFI)
+        InsertSEH(MIB, TII, MachineInstr::FrameSetup);
+    } else { // The code when the pair of ZReg is not present
+      MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
+      if (!MRI.isReserved(Reg1))
+        MBB.addLiveIn(Reg1);
+      if (RPI.isPaired()) {
+        if (!MRI.isReserved(Reg2))
+          MBB.addLiveIn(Reg2);
+        MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
+        MIB.addMemOperand(MF.getMachineMemOperand(
+            MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
+            MachineMemOperand::MOStore, Size, Alignment));
+      }
+      MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
+          .addReg(AArch64::SP)
+          .addImm(RPI.Offset) // [sp, #offset*scale],
+                              // where factor*scale is implicit
+          .setMIFlag(MachineInstr::FrameSetup);
+      MIB.addMemOperand(MF.getMachineMemOperand(
+          MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
+          MachineMemOperand::MOStore, Size, Alignment));
+      if (NeedsWinCFI)
+        InsertSEH(MIB, TII, MachineInstr::FrameSetup);
     }
-    MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
-        .addReg(AArch64::SP)
-        .addImm(RPI.Offset) // [sp, #offset*scale],
-                            // where factor*scale is implicit
-        .setMIFlag(MachineInstr::FrameSetup);
-    MIB.addMemOperand(MF.getMachineMemOperand(
-        MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
-        MachineMemOperand::MOStore, Size, Alignment));
-    if (NeedsWinCFI)
-      InsertSEH(MIB, TII, MachineInstr::FrameSetup);
-
     // Update the StackIDs of the SVE stack slots.
     MachineFrameInfo &MFI = MF.getFrameInfo();
-    if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR)
-      MFI.setStackID(RPI.FrameIdx, TargetStackID::ScalableVector);
-
+    if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR) {
+      MFI.setStackID(FrameIdxReg1, TargetStackID::ScalableVector);
+      if (RPI.isPaired())
+        MFI.setStackID(FrameIdxReg2, TargetStackID::ScalableVector);
+    }
   }
   return true;
 }
@@ -3109,7 +3169,6 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
     DL = MBBI->getDebugLoc();
 
   computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF));
-
   if (homogeneousPrologEpilog(MF, &MBB)) {
     auto MIB = BuildMI(MBB, MBBI, DL, TII.get(AArch64::HOM_Epilog))
                    .setMIFlag(MachineInstr::FrameDestroy);
@@ -3130,6 +3189,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
   auto ZPREnd = std::find_if_not(ZPRBegin, RegPairs.end(), IsZPR);
   std::reverse(ZPRBegin, ZPREnd);
 
+  bool PTrueCreated = false;
   for (const RegPairInfo &RPI : RegPairs) {
     unsigned Reg1 = RPI.Reg1;
     unsigned Reg2 = RPI.Reg2;
@@ -3162,7 +3222,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
        Alignment = Align(16);
        break;
     case RegPairInfo::ZPR:
-       LdrOpc = AArch64::LDR_ZXI;
+       LdrOpc = RPI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI;
        Size = 16;
        Alignment = Align(16);
        break;
@@ -3187,25 +3247,58 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
       std::swap(Reg1, Reg2);
       std::swap(FrameIdxReg1, FrameIdxReg2);
     }
-    MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
-    if (RPI.isPaired()) {
-      MIB.addReg(Reg2, getDefRegState(true));
+
+    AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+    if (RPI.isPaired() && RPI.isScalable()) {
+      const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
+      unsigned PnReg = AFI->getPredicateRegForFillSpill();
+      assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
+             "Expects SVE2.1 or SME2 target and a predicate register");
+#ifdef EXPENSIVE_CHECKS
+      assert(!(PPRBegin < ZPRBegin) &&
+             "Expected callee save predicate to be handled first");
+#endif
+      if (!PTrueCreated) {
+        PTrueCreated = true;
+        BuildMI(MBB, MBBI, DL, TII.get(AArch64::PTRUE_C_B), PnReg)
+            .setMIFlags(MachineInstr::FrameDestroy);
+      }
+      MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
+      MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0),
+                 getDefRegState(true));
       MIB.addMemOperand(MF.getMachineMemOperand(
           MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
           MachineMemOperand::MOLoad, Size, Alignment));
+      MIB.addReg(PnReg);
+      MIB.addReg(AArch64::SP)
+          .addImm(RPI.Offset) // [sp, #offset*scale]
+                              // where factor*scale is implicit
+          .setMIFlag(MachineInstr::FrameDestroy);
+      MIB.addMemOperand(MF.getMachineMemOperand(
+          MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
+          MachineMemOperand::MOLoad, Size, Alignment));
+      if (NeedsWinCFI)
+        InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
+    } else {
+      MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
+      if (RPI.isPaired()) {
+        MIB.addReg(Reg2, getDefRegState(true));
+        MIB.addMemOperand(MF.getMachineMemOperand(
+            MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
+            MachineMemOperand::MOLoad, Size, Alignment));
+      }
+      MIB.addReg(Reg1, getDefRegState(true));
+      MIB.addReg(AArch64::SP)
+          .addImm(RPI.Offset) // [sp, #offset*scale]
+                              // where factor*scale is implicit
+          .setMIFlag(MachineInstr::FrameDestroy);
+      MIB.addMemOperand(MF.getMachineMemOperand(
+          MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
+          MachineMemOperand::MOLoad, Size, Alignment));
+      if (NeedsWinCFI)
+        InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
     }
-    MIB.addReg(Reg1, getDefRegState(true))
-        .addReg(AArch64::SP)
-        .addImm(RPI.Offset) // [sp, #offset*scale]
-                            // where factor*scale is implicit
-        .setMIFlag(MachineInstr::FrameDestroy);
-    MIB.addMemOperand(MF.getMachineMemOperand(
-        MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
-        MachineMemOperand::MOLoad, Size, Alignment));
-    if (NeedsWinCFI)
-      InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
   }
-
   return true;
 }
 
@@ -3234,6 +3327,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
 
   unsigned ExtraCSSpill = 0;
   bool HasUnpairedGPR64 = false;
+  bool HasPairZReg = false;
   // Figure out which callee-saved registers to save/restore.
   for (unsigned i = 0; CSRegs[i]; ++i) {
     const unsigned Reg = CSRegs[i];
@@ -3287,6 +3381,28 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
           !RegInfo->isReservedReg(MF, PairedReg))
         ExtraCSSpill = PairedReg;
     }
+    // Check if there is a pair of ZRegs, so it can select PReg for spill/fill
+    HasPairZReg |= (AArch64::ZPRRegClass.contains(Reg, CSRegs[i ^ 1]) &&
+                    SavedRegs.test(CSRegs[i ^ 1]));
+  }
+
+  if (HasPairZReg && (Subtarget.hasSVE2p1() || Subtarget.hasSME2())) {
+    AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+    // Find a suitable predicate register for the multi-vector spill/fill
+    // instructions.
+    unsigned PnReg = findFreePredicateReg(SavedRegs);
+    if (PnReg != AArch64::NoRegister)
+      AFI->setPredicateRegForFillSpill(PnReg);
+    // If no free callee-save has been found assign one.
+    if (!AFI->getPredicateRegForFillSpill() &&
+        MF.getFunction().getCallingConv() ==
+            CallingConv::AArch64_SVE_VectorCall) {
+      SavedRegs.set(AArch64::P8);
+      AFI->setPredicateRegForFillSpill(AArch64::PN8);
+    }
+
+    assert(!RegInfo->isReservedReg(MF, AFI->getPredicateRegForFillSpill()) &&
+           "Predicate cannot be a reserved register");
   }
 
   if (MF.getFunction().getCallingConv() == CallingConv::Win64 &&
diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index d5941e628411..df09fc5592ed 100644
--- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -212,6 +212,10 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
   // on function entry to record the initial pstate of a function.
   Register PStateSMReg = MCRegister::NoRegister;
 
+  // Has the PNReg used to build PTRUE instruction.
+  // The PTRUE is used for the LD/ST of ZReg pairs in save and restore.
+  unsigned PredicateRegForFillSpill = 0;
+
 public:
   AArch64FunctionInfo(const Function &F, const AArch64Subtarget *STI);
 
@@ -220,6 +224,13 @@ public:
         const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
       const override;
 
+  void setPredicateRegForFillSpill(unsigned Reg) {
+    PredicateRegForFillSpill = Reg;
+  }
+  unsigned getPredicateRegForFillSpill() const {
+    return PredicateRegForFillSpill;
+  }
+
   Register getPStateSMReg() const { return PStateSMReg; };
   void setPStateSMReg(Register Reg) { PStateSMReg = Reg; };
 
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-ld1.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-ld1.ll
index ea7808d73093..3a94b0333e26 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-ld1.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-ld1.ll
@@ -55,45 +55,31 @@ define <vscale x 32 x i8> @ld1_x2_i8_z0_z8(<vscale x 16 x i8> %unused, <vscale x
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1b { z0.b, z8.b }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -103,20 +89,14 @@ define <vscale x 32 x i8> @ld1_x2_i8_z0_z8(<vscale x 16 x i8> %unused, <vscale x
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -129,21 +109,15 @@ define <vscale x 32 x i8> @ld1_x2_i8_z0_z8(<vscale x 16 x i8> %unused, <vscale x
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -210,45 +184,31 @@ define <vscale x 32 x i8> @ld1_x2_i8_z0_z8_scalar(<vscale x 16 x i8> %unused, <v
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1b { z0.b, z8.b }, pn8/z, [x0, x1]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -258,20 +218,14 @@ define <vscale x 32 x i8> @ld1_x2_i8_z0_z8_scalar(<vscale x 16 x i8> %unused, <v
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -284,21 +238,15 @@ define <vscale x 32 x i8> @ld1_x2_i8_z0_z8_scalar(<vscale x 16 x i8> %unused, <v
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -366,45 +314,31 @@ define <vscale x 16 x i16> @ld1_x2_i16_z0_z8(<vscale x 8 x i16> %unused, <vscale
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1h { z0.h, z8.h }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -414,20 +348,14 @@ define <vscale x 16 x i16> @ld1_x2_i16_z0_z8(<vscale x 8 x i16> %unused, <vscale
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -440,21 +368,15 @@ define <vscale x 16 x i16> @ld1_x2_i16_z0_z8(<vscale x 8 x i16> %unused, <vscale
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -521,45 +443,31 @@ define <vscale x 16 x i16> @ld1_x2_i16_z0_z8_scalar(<vscale x 8 x i16> %unused,
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1h { z0.h, z8.h }, pn8/z, [x0, x1, lsl #1]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -569,20 +477,14 @@ define <vscale x 16 x i16> @ld1_x2_i16_z0_z8_scalar(<vscale x 8 x i16> %unused,
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -595,21 +497,15 @@ define <vscale x 16 x i16> @ld1_x2_i16_z0_z8_scalar(<vscale x 8 x i16> %unused,
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -677,45 +573,31 @@ define <vscale x 8 x i32> @ld1_x2_i32_z0_z8(<vscale x 4 x i32> %unused, <vscale
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1w { z0.s, z8.s }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -725,20 +607,14 @@ define <vscale x 8 x i32> @ld1_x2_i32_z0_z8(<vscale x 4 x i32> %unused, <vscale
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -751,21 +627,15 @@ define <vscale x 8 x i32> @ld1_x2_i32_z0_z8(<vscale x 4 x i32> %unused, <vscale
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -832,45 +702,31 @@ define <vscale x 8 x i32> @ld1_x2_i32_z0_z8_scalar(<vscale x 4 x i32> %unused, <
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1w { z0.s, z8.s }, pn8/z, [x0, x1, lsl #2]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -880,20 +736,14 @@ define <vscale x 8 x i32> @ld1_x2_i32_z0_z8_scalar(<vscale x 4 x i32> %unused, <
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -906,21 +756,15 @@ define <vscale x 8 x i32> @ld1_x2_i32_z0_z8_scalar(<vscale x 4 x i32> %unused, <
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -988,45 +832,31 @@ define <vscale x 4 x i64> @ld1_x2_i64_z0_z8(<vscale x 2 x i64> %unused, <vscale
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1d { z0.d, z8.d }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -1036,20 +866,14 @@ define <vscale x 4 x i64> @ld1_x2_i64_z0_z8(<vscale x 2 x i64> %unused, <vscale
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1062,21 +886,15 @@ define <vscale x 4 x i64> @ld1_x2_i64_z0_z8(<vscale x 2 x i64> %unused, <vscale
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1143,45 +961,31 @@ define <vscale x 4 x i64> @ld1_x2_i64_z0_z8_scalar(<vscale x 2 x i64> %unused, <
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1d { z0.d, z8.d }, pn8/z, [x0, x1, lsl #3]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -1191,20 +995,14 @@ define <vscale x 4 x i64> @ld1_x2_i64_z0_z8_scalar(<vscale x 2 x i64> %unused, <
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1217,21 +1015,15 @@ define <vscale x 4 x i64> @ld1_x2_i64_z0_z8_scalar(<vscale x 2 x i64> %unused, <
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1301,46 +1093,32 @@ define <vscale x 64 x i8> @ld1_x4_i8_z0_z4_z8_z12(<vscale x 16 x i8> %unused, <v
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1351,19 +1129,14 @@ define <vscale x 64 x i8> @ld1_x4_i8_z0_z4_z8_z12(<vscale x 16 x i8> %unused, <v
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1380,20 +1153,15 @@ define <vscale x 64 x i8> @ld1_x4_i8_z0_z4_z8_z12(<vscale x 16 x i8> %unused, <v
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1466,46 +1234,32 @@ define <vscale x 64 x i8> @ld1_x4_i8_z0_z4_z8_z12_scalar(<vscale x 16 x i8> %unu
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0, x1]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1516,19 +1270,14 @@ define <vscale x 64 x i8> @ld1_x4_i8_z0_z4_z8_z12_scalar(<vscale x 16 x i8> %unu
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1545,20 +1294,15 @@ define <vscale x 64 x i8> @ld1_x4_i8_z0_z4_z8_z12_scalar(<vscale x 16 x i8> %unu
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1632,46 +1376,32 @@ define <vscale x 32 x i16> @ld1_x4_i16_z0_z4_z8_z12(<vscale x 8 x i16> %unused,
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1h { z0.h, z4.h, z8.h, z12.h }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1682,19 +1412,14 @@ define <vscale x 32 x i16> @ld1_x4_i16_z0_z4_z8_z12(<vscale x 8 x i16> %unused,
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1711,20 +1436,15 @@ define <vscale x 32 x i16> @ld1_x4_i16_z0_z4_z8_z12(<vscale x 8 x i16> %unused,
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1798,46 +1518,32 @@ define <vscale x 32 x i16> @ld1_x4_i16_z0_z4_z8_z12_scalar(<vscale x 8 x i16> %u
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1h { z0.h, z4.h, z8.h, z12.h }, pn8/z, [x0, x1, lsl #1]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1848,19 +1554,14 @@ define <vscale x 32 x i16> @ld1_x4_i16_z0_z4_z8_z12_scalar(<vscale x 8 x i16> %u
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1877,20 +1578,15 @@ define <vscale x 32 x i16> @ld1_x4_i16_z0_z4_z8_z12_scalar(<vscale x 8 x i16> %u
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1964,46 +1660,32 @@ define <vscale x 16 x i32> @ld1_x4_i32_z0_z4_z8_z12(<vscale x 4 x i32> %unused,
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1w { z0.s, z4.s, z8.s, z12.s }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -2014,19 +1696,14 @@ define <vscale x 16 x i32> @ld1_x4_i32_z0_z4_z8_z12(<vscale x 4 x i32> %unused,
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -2043,20 +1720,15 @@ define <vscale x 16 x i32> @ld1_x4_i32_z0_z4_z8_z12(<vscale x 4 x i32> %unused,
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -2130,46 +1802,32 @@ define <vscale x 16 x i32> @ld1_x4_i32_z0_z4_z8_z12_scalar(<vscale x 4 x i32> %u
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1w { z0.s, z4.s, z8.s, z12.s }, pn8/z, [x0, x1, lsl #2]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -2180,19 +1838,14 @@ define <vscale x 16 x i32> @ld1_x4_i32_z0_z4_z8_z12_scalar(<vscale x 4 x i32> %u
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -2209,20 +1862,15 @@ define <vscale x 16 x i32> @ld1_x4_i32_z0_z4_z8_z12_scalar(<vscale x 4 x i32> %u
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -2296,46 +1944,32 @@ define <vscale x 8 x i64> @ld1_x4_i64_z0_z4_z8_z12(<vscale x 2 x i64> %unused, <
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1d { z0.d, z4.d, z8.d, z12.d }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -2346,19 +1980,14 @@ define <vscale x 8 x i64> @ld1_x4_i64_z0_z4_z8_z12(<vscale x 2 x i64> %unused, <
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -2375,20 +2004,15 @@ define <vscale x 8 x i64> @ld1_x4_i64_z0_z4_z8_z12(<vscale x 2 x i64> %unused, <
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -2462,46 +2086,32 @@ define <vscale x 8 x i64> @ld1_x4_i64_z0_z4_z8_z12_scalar(<vscale x 2 x i64> %un
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ld1d { z0.d, z4.d, z8.d, z12.d }, pn8/z, [x0, x1, lsl #3]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -2512,19 +2122,14 @@ define <vscale x 8 x i64> @ld1_x4_i64_z0_z4_z8_z12_scalar(<vscale x 2 x i64> %un
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -2541,20 +2146,15 @@ define <vscale x 8 x i64> @ld1_x4_i64_z0_z4_z8_z12_scalar(<vscale x 2 x i64> %un
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-ldnt1.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-ldnt1.ll
index 7e2d28fbf798..8ecb7c858c6a 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-ldnt1.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-ldnt1.ll
@@ -8,45 +8,31 @@ define <vscale x 32 x i8> @ldnt1_x2_i8_z0_z8(<vscale x 16 x i8> %unused, <vscale
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1b { z0.b, z8.b }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -56,20 +42,14 @@ define <vscale x 32 x i8> @ldnt1_x2_i8_z0_z8(<vscale x 16 x i8> %unused, <vscale
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -82,21 +62,15 @@ define <vscale x 32 x i8> @ldnt1_x2_i8_z0_z8(<vscale x 16 x i8> %unused, <vscale
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -116,45 +90,31 @@ define <vscale x 32 x i8> @ldnt1_x2_i8_z0_z8_scalar(<vscale x 16 x i8> %unused,
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1b { z0.b, z8.b }, pn8/z, [x0, x1]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -164,20 +124,14 @@ define <vscale x 32 x i8> @ldnt1_x2_i8_z0_z8_scalar(<vscale x 16 x i8> %unused,
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -190,21 +144,15 @@ define <vscale x 32 x i8> @ldnt1_x2_i8_z0_z8_scalar(<vscale x 16 x i8> %unused,
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -225,45 +173,31 @@ define <vscale x 16 x i16> @ldnt1_x2_i16_z0_z8(<vscale x 8 x i16> %unused, <vsca
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1h { z0.h, z8.h }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -273,20 +207,14 @@ define <vscale x 16 x i16> @ldnt1_x2_i16_z0_z8(<vscale x 8 x i16> %unused, <vsca
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -299,21 +227,15 @@ define <vscale x 16 x i16> @ldnt1_x2_i16_z0_z8(<vscale x 8 x i16> %unused, <vsca
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -333,45 +255,31 @@ define <vscale x 16 x i16> @ldnt1_x2_i16_z0_z8_scalar(<vscale x 8 x i16> %unused
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1h { z0.h, z8.h }, pn8/z, [x0, x1, lsl #1]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -381,20 +289,14 @@ define <vscale x 16 x i16> @ldnt1_x2_i16_z0_z8_scalar(<vscale x 8 x i16> %unused
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -407,21 +309,15 @@ define <vscale x 16 x i16> @ldnt1_x2_i16_z0_z8_scalar(<vscale x 8 x i16> %unused
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -442,45 +338,31 @@ define <vscale x 8 x i32> @ldnt1_x2_i32_z0_z8(<vscale x 4 x i32> %unused, <vscal
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1w { z0.s, z8.s }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -490,20 +372,14 @@ define <vscale x 8 x i32> @ldnt1_x2_i32_z0_z8(<vscale x 4 x i32> %unused, <vscal
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -516,21 +392,15 @@ define <vscale x 8 x i32> @ldnt1_x2_i32_z0_z8(<vscale x 4 x i32> %unused, <vscal
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -550,45 +420,31 @@ define <vscale x 8 x i32> @ldnt1_x2_i32_z0_z8_scalar(<vscale x 4 x i32> %unused,
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1w { z0.s, z8.s }, pn8/z, [x0, x1, lsl #2]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -598,20 +454,14 @@ define <vscale x 8 x i32> @ldnt1_x2_i32_z0_z8_scalar(<vscale x 4 x i32> %unused,
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -624,21 +474,15 @@ define <vscale x 8 x i32> @ldnt1_x2_i32_z0_z8_scalar(<vscale x 4 x i32> %unused,
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -659,45 +503,31 @@ define <vscale x 4 x i64> @ldnt1_x2_i64_z0_z8(<vscale x 2 x i64> %unused, <vscal
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1d { z0.d, z8.d }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -707,20 +537,14 @@ define <vscale x 4 x i64> @ldnt1_x2_i64_z0_z8(<vscale x 2 x i64> %unused, <vscal
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -733,21 +557,15 @@ define <vscale x 4 x i64> @ldnt1_x2_i64_z0_z8(<vscale x 2 x i64> %unused, <vscal
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -767,45 +585,31 @@ define <vscale x 4 x i64> @ldnt1_x2_i64_z0_z8_scalar(<vscale x 2 x i64> %unused,
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1d { z0.d, z8.d }, pn8/z, [x0, x1, lsl #3]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z8.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; STRIDED-NEXT:    ret
@@ -815,20 +619,14 @@ define <vscale x 4 x i64> @ldnt1_x2_i64_z0_z8_scalar(<vscale x 2 x i64> %unused,
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-16
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-2
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -841,21 +639,15 @@ define <vscale x 4 x i64> @ldnt1_x2_i64_z0_z8_scalar(<vscale x 2 x i64> %unused,
 ; CONTIGUOUS-NEXT:    ldr z0, [sp]
 ; CONTIGUOUS-NEXT:    ldr z1, [sp, #1, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #2
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #16
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -876,46 +668,32 @@ define <vscale x 64 x i8> @ldnt1_x4_i8_z0_z4_z8_z12(<vscale x 16 x i8> %unused,
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -926,19 +704,14 @@ define <vscale x 64 x i8> @ldnt1_x4_i8_z0_z4_z8_z12(<vscale x 16 x i8> %unused,
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -955,20 +728,15 @@ define <vscale x 64 x i8> @ldnt1_x4_i8_z0_z4_z8_z12(<vscale x 16 x i8> %unused,
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -992,46 +760,32 @@ define <vscale x 64 x i8> @ldnt1_x4_i8_z0_z4_z8_z12_scalar(<vscale x 16 x i8> %u
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1b { z0.b, z4.b, z8.b, z12.b }, pn8/z, [x0, x1]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1042,19 +796,14 @@ define <vscale x 64 x i8> @ldnt1_x4_i8_z0_z4_z8_z12_scalar(<vscale x 16 x i8> %u
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1071,20 +820,15 @@ define <vscale x 64 x i8> @ldnt1_x4_i8_z0_z4_z8_z12_scalar(<vscale x 16 x i8> %u
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1109,46 +853,32 @@ define <vscale x 32 x i16> @ldnt1_x4_i16_z0_z4_z8_z12(<vscale x 8 x i16> %unused
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1h { z0.h, z4.h, z8.h, z12.h }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1159,19 +889,14 @@ define <vscale x 32 x i16> @ldnt1_x4_i16_z0_z4_z8_z12(<vscale x 8 x i16> %unused
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1188,20 +913,15 @@ define <vscale x 32 x i16> @ldnt1_x4_i16_z0_z4_z8_z12(<vscale x 8 x i16> %unused
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1225,46 +945,32 @@ define <vscale x 32 x i16> @ldnt1_x4_i16_z0_z4_z8_z12_scalar(<vscale x 8 x i16>
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1h { z0.h, z4.h, z8.h, z12.h }, pn8/z, [x0, x1, lsl #1]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1275,19 +981,14 @@ define <vscale x 32 x i16> @ldnt1_x4_i16_z0_z4_z8_z12_scalar(<vscale x 8 x i16>
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1304,20 +1005,15 @@ define <vscale x 32 x i16> @ldnt1_x4_i16_z0_z4_z8_z12_scalar(<vscale x 8 x i16>
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1342,46 +1038,32 @@ define <vscale x 16 x i32> @ldnt1_x4_i32_z0_z4_z8_z12(<vscale x 4 x i32> %unused
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1w { z0.s, z4.s, z8.s, z12.s }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1392,19 +1074,14 @@ define <vscale x 16 x i32> @ldnt1_x4_i32_z0_z4_z8_z12(<vscale x 4 x i32> %unused
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1421,20 +1098,15 @@ define <vscale x 16 x i32> @ldnt1_x4_i32_z0_z4_z8_z12(<vscale x 4 x i32> %unused
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1458,46 +1130,32 @@ define <vscale x 16 x i32> @ldnt1_x4_i32_z0_z4_z8_z12_scalar(<vscale x 4 x i32>
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1w { z0.s, z4.s, z8.s, z12.s }, pn8/z, [x0, x1, lsl #2]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1508,19 +1166,14 @@ define <vscale x 16 x i32> @ldnt1_x4_i32_z0_z4_z8_z12_scalar(<vscale x 4 x i32>
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1537,20 +1190,15 @@ define <vscale x 16 x i32> @ldnt1_x4_i32_z0_z4_z8_z12_scalar(<vscale x 4 x i32>
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1575,46 +1223,32 @@ define <vscale x 8 x i64> @ldnt1_x4_i64_z0_z4_z8_z12(<vscale x 2 x i64> %unused,
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1d { z0.d, z4.d, z8.d, z12.d }, pn8/z, [x0]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1625,19 +1259,14 @@ define <vscale x 8 x i64> @ldnt1_x4_i64_z0_z4_z8_z12(<vscale x 2 x i64> %unused,
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1654,20 +1283,15 @@ define <vscale x 8 x i64> @ldnt1_x4_i64_z0_z4_z8_z12(<vscale x 2 x i64> %unused,
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1691,46 +1315,32 @@ define <vscale x 8 x i64> @ldnt1_x4_i64_z0_z4_z8_z12_scalar(<vscale x 2 x i64> %
 ; STRIDED-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; STRIDED-NEXT:    addvl sp, sp, #-17
 ; STRIDED-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #22, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #26, mul vl] // 32-byte Folded Spill
+; STRIDED-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #30, mul vl] // 32-byte Folded Spill
 ; STRIDED-NEXT:    mov p8.b, p0.b
-; STRIDED-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; STRIDED-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
 ; STRIDED-NEXT:    ldnt1d { z0.d, z4.d, z8.d, z12.d }, pn8/z, [x0, x1, lsl #3]
 ; STRIDED-NEXT:    //APP
 ; STRIDED-NEXT:    nop
 ; STRIDED-NEXT:    //NO_APP
-; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
-; STRIDED-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ptrue pn8.b
+; STRIDED-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #26, mul vl] // 32-byte Folded Reload
 ; STRIDED-NEXT:    mov z2.d, z8.d
 ; STRIDED-NEXT:    mov z3.d, z12.d
-; STRIDED-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; STRIDED-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #22, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #30, mul vl] // 32-byte Folded Reload
+; STRIDED-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; STRIDED-NEXT:    mov z1.d, z4.d
 ; STRIDED-NEXT:    addvl sp, sp, #17
 ; STRIDED-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1741,19 +1351,14 @@ define <vscale x 8 x i64> @ldnt1_x4_i64_z0_z4_z8_z12_scalar(<vscale x 2 x i64> %
 ; CONTIGUOUS-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-15
 ; CONTIGUOUS-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    ptrue pn8.b
+; CONTIGUOUS-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #18, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z11, [sp, #12, mul vl] // 16-byte Folded Spill
-; CONTIGUOUS-NEXT:    str z10, [sp, #13, mul vl] // 16-byte Folded Spill
+; CONTIGUOUS-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
 ; CONTIGUOUS-NEXT:    str z9, [sp, #14, mul vl] // 16-byte Folded Spill
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #-4
 ; CONTIGUOUS-NEXT:    mov p8.b, p0.b
@@ -1770,20 +1375,15 @@ define <vscale x 8 x i64> @ldnt1_x4_i64_z0_z4_z8_z12_scalar(<vscale x 2 x i64> %
 ; CONTIGUOUS-NEXT:    ldr z2, [sp, #2, mul vl]
 ; CONTIGUOUS-NEXT:    ldr z3, [sp, #3, mul vl]
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #4
-; CONTIGUOUS-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ptrue pn8.b
 ; CONTIGUOUS-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload
-; CONTIGUOUS-NEXT:    ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #18, mul vl] // 32-byte Folded Reload
+; CONTIGUOUS-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
 ; CONTIGUOUS-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CONTIGUOUS-NEXT:    addvl sp, sp, #15
 ; CONTIGUOUS-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll b/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll
new file mode 100644
index 000000000000..c62016d8ea01
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll
@@ -0,0 +1,556 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -verify-machineinstrs < %s | FileCheck %s --check-prefixes=NOPAIR
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=NOPAIR
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=PAIR
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=PAIR
+
+
+declare void @my_func()
+
+define void @fbyte(<vscale x 16 x i8> %v) {
+; NOPAIR-LABEL: fbyte:
+; NOPAIR:       // %bb.0:
+; NOPAIR-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; NOPAIR-NEXT:    addvl sp, sp, #-18
+; NOPAIR-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
+; NOPAIR-NEXT:    .cfi_offset w30, -8
+; NOPAIR-NEXT:    .cfi_offset w29, -16
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
+; NOPAIR-NEXT:    bl my_func
+; NOPAIR-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    addvl sp, sp, #18
+; NOPAIR-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; NOPAIR-NEXT:    ret
+;
+; PAIR-LABEL: fbyte:
+; PAIR:       // %bb.0:
+; PAIR-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; PAIR-NEXT:    addvl sp, sp, #-18
+; PAIR-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    ptrue pn8.b
+; PAIR-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #16, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #20, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #28, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #32, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
+; PAIR-NEXT:    .cfi_offset w30, -8
+; PAIR-NEXT:    .cfi_offset w29, -16
+; PAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
+; PAIR-NEXT:    bl my_func
+; PAIR-NEXT:    ptrue pn8.b
+; PAIR-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #16, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #20, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #28, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #32, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    addvl sp, sp, #18
+; PAIR-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; PAIR-NEXT:    ret
+  call void @my_func()
+  ret void
+}
+
+define void @fhalf(<vscale x 8 x half> %v) {
+; NOPAIR-LABEL: fhalf:
+; NOPAIR:       // %bb.0:
+; NOPAIR-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; NOPAIR-NEXT:    addvl sp, sp, #-18
+; NOPAIR-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
+; NOPAIR-NEXT:    .cfi_offset w30, -8
+; NOPAIR-NEXT:    .cfi_offset w29, -16
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
+; NOPAIR-NEXT:    bl my_func
+; NOPAIR-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    addvl sp, sp, #18
+; NOPAIR-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; NOPAIR-NEXT:    ret
+;
+; PAIR-LABEL: fhalf:
+; PAIR:       // %bb.0:
+; PAIR-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; PAIR-NEXT:    addvl sp, sp, #-18
+; PAIR-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    ptrue pn8.b
+; PAIR-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    st1b { z22.b, z23.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    st1b { z20.b, z21.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    st1b { z18.b, z19.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    st1b { z16.b, z17.b }, pn8, [sp, #16, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    st1b { z14.b, z15.b }, pn8, [sp, #20, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    st1b { z12.b, z13.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #28, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #32, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x01, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 144 * VG
+; PAIR-NEXT:    .cfi_offset w30, -8
+; PAIR-NEXT:    .cfi_offset w29, -16
+; PAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 16 - 32 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 16 - 40 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 16 - 48 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 16 - 56 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 16 - 64 * VG
+; PAIR-NEXT:    bl my_func
+; PAIR-NEXT:    ptrue pn8.b
+; PAIR-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ld1b { z22.b, z23.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z20.b, z21.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z18.b, z19.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z16.b, z17.b }, pn8/z, [sp, #16, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z14.b, z15.b }, pn8/z, [sp, #20, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z12.b, z13.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #28, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #32, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    addvl sp, sp, #18
+; PAIR-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; PAIR-NEXT:    ret
+  call void @my_func()
+  ret void
+}
+
+;; Do NOT group Z10
+;; DO group Z8 and Z9 and save P8
+define aarch64_sve_vector_pcs void @test_clobbers_z_p_regs() {
+; NOPAIR-LABEL: test_clobbers_z_p_regs:
+; NOPAIR:       // %bb.0:
+; NOPAIR-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; NOPAIR-NEXT:    addvl sp, sp, #-4
+; NOPAIR-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z8, [sp, #3, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG
+; NOPAIR-NEXT:    .cfi_offset w29, -16
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; NOPAIR-NEXT:    //APP
+; NOPAIR-NEXT:    //NO_APP
+; NOPAIR-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z8, [sp, #3, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    addvl sp, sp, #4
+; NOPAIR-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; NOPAIR-NEXT:    ret
+;
+; PAIR-LABEL: test_clobbers_z_p_regs:
+; PAIR:       // %bb.0:
+; PAIR-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; PAIR-NEXT:    addvl sp, sp, #-4
+; PAIR-NEXT:    str p8, [sp, #5, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    ptrue pn8.b
+; PAIR-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG
+; PAIR-NEXT:    .cfi_offset w29, -16
+; PAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; PAIR-NEXT:    //APP
+; PAIR-NEXT:    //NO_APP
+; PAIR-NEXT:    ptrue pn8.b
+; PAIR-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ldr p8, [sp, #5, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    addvl sp, sp, #4
+; PAIR-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; PAIR-NEXT:    ret
+  call void asm sideeffect "", "~{p4},~{p5},~{z8},~{z9},~{z10}"()
+  ret void
+}
+
+;; Do NOT group Z10
+;; DO group Z8 and Z9 and use P9
+define aarch64_sve_vector_pcs  void @test_clobbers_z_p_regs2() {
+; NOPAIR-LABEL: test_clobbers_z_p_regs2:
+; NOPAIR:       // %bb.0:
+; NOPAIR-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; NOPAIR-NEXT:    addvl sp, sp, #-4
+; NOPAIR-NEXT:    str p10, [sp, #6, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str p9, [sp, #7, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z9, [sp, #2, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z8, [sp, #3, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG
+; NOPAIR-NEXT:    .cfi_offset w29, -16
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; NOPAIR-NEXT:    //APP
+; NOPAIR-NEXT:    //NO_APP
+; NOPAIR-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z8, [sp, #3, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr p10, [sp, #6, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    ldr p9, [sp, #7, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    addvl sp, sp, #4
+; NOPAIR-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; NOPAIR-NEXT:    ret
+;
+; PAIR-LABEL: test_clobbers_z_p_regs2:
+; PAIR:       // %bb.0:
+; PAIR-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; PAIR-NEXT:    addvl sp, sp, #-4
+; PAIR-NEXT:    str p9, [sp, #7, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    ptrue pn9.b
+; PAIR-NEXT:    str p10, [sp, #6, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT:    st1b { z8.b, z9.b }, pn9, [sp, #4, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG
+; PAIR-NEXT:    .cfi_offset w29, -16
+; PAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 24 * VG
+; PAIR-NEXT:    //APP
+; PAIR-NEXT:    //NO_APP
+; PAIR-NEXT:    ptrue pn9.b
+; PAIR-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT:    ldr p10, [sp, #6, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    ld1b { z8.b, z9.b }, pn9/z, [sp, #4, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ldr p9, [sp, #7, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    addvl sp, sp, #4
+; PAIR-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; PAIR-NEXT:    ret
+  call void asm sideeffect "", "~{p9},~{p10},~{z8},~{z9},~{z10}"()
+  ret void
+}
+
+
+;; Test order of PRegs and ZRegs when there is no PReg being clobbered
+define aarch64_sve_vector_pcs  void @test_clobbers_z_regs() {
+; NOPAIR-LABEL: test_clobbers_z_regs:
+; NOPAIR:       // %bb.0:
+; NOPAIR-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; NOPAIR-NEXT:    addvl sp, sp, #-2
+; NOPAIR-NEXT:    str z9, [sp] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z8, [sp, #1, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; NOPAIR-NEXT:    .cfi_offset w29, -16
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; NOPAIR-NEXT:    //APP
+; NOPAIR-NEXT:    //NO_APP
+; NOPAIR-NEXT:    ldr z9, [sp] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    addvl sp, sp, #2
+; NOPAIR-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; NOPAIR-NEXT:    ret
+;
+; PAIR-LABEL: test_clobbers_z_regs:
+; PAIR:       // %bb.0:
+; PAIR-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; PAIR-NEXT:    addvl sp, sp, #-3
+; PAIR-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    ptrue pn8.b
+; PAIR-NEXT:    st1b { z8.b, z9.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
+; PAIR-NEXT:    .cfi_offset w29, -16
+; PAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 16 - 16 * VG
+; PAIR-NEXT:    //APP
+; PAIR-NEXT:    //NO_APP
+; PAIR-NEXT:    ptrue pn8.b
+; PAIR-NEXT:    ld1b { z8.b, z9.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    addvl sp, sp, #3
+; PAIR-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; PAIR-NEXT:    ret
+  call void asm sideeffect "", "~{z8},~{z9}"()
+  ret void
+}
+
+;; DO NOT group Z8 and Z9 and
+;; DO NOT save P8
+;; It does not belong to the allowed calling conventions
+;; NOPAIR and PAIR should have the same assembly
+define  void @test_clobbers_z_regs_negative() {
+; NOPAIR-LABEL: test_clobbers_z_regs_negative:
+; NOPAIR:       // %bb.0:
+; NOPAIR-NEXT:    stp d9, d8, [sp, #-16]! // 16-byte Folded Spill
+; NOPAIR-NEXT:    .cfi_def_cfa_offset 16
+; NOPAIR-NEXT:    .cfi_offset b8, -8
+; NOPAIR-NEXT:    .cfi_offset b9, -16
+; NOPAIR-NEXT:    //APP
+; NOPAIR-NEXT:    //NO_APP
+; NOPAIR-NEXT:    ldp d9, d8, [sp], #16 // 16-byte Folded Reload
+; NOPAIR-NEXT:    ret
+;
+; PAIR-LABEL: test_clobbers_z_regs_negative:
+; PAIR:       // %bb.0:
+; PAIR-NEXT:    stp d9, d8, [sp, #-16]! // 16-byte Folded Spill
+; PAIR-NEXT:    .cfi_def_cfa_offset 16
+; PAIR-NEXT:    .cfi_offset b8, -8
+; PAIR-NEXT:    .cfi_offset b9, -16
+; PAIR-NEXT:    //APP
+; PAIR-NEXT:    //NO_APP
+; PAIR-NEXT:    ldp d9, d8, [sp], #16 // 16-byte Folded Reload
+; PAIR-NEXT:    ret
+  call void asm sideeffect "", "~{z8},~{z9}"()
+  ret void
+}
+
+;; Do NOT save P8 and NOT group any Z8 and Z10 register
+define aarch64_sve_vector_pcs void @test_clobbers_2_z_regs_negative() {
+; NOPAIR-LABEL: test_clobbers_2_z_regs_negative:
+; NOPAIR:       // %bb.0:
+; NOPAIR-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; NOPAIR-NEXT:    addvl sp, sp, #-2
+; NOPAIR-NEXT:    str z10, [sp] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str z8, [sp, #1, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; NOPAIR-NEXT:    .cfi_offset w29, -16
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; NOPAIR-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 16 * VG
+; NOPAIR-NEXT:    //APP
+; NOPAIR-NEXT:    //NO_APP
+; NOPAIR-NEXT:    ldr z10, [sp] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT:    addvl sp, sp, #2
+; NOPAIR-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; NOPAIR-NEXT:    ret
+;
+; PAIR-LABEL: test_clobbers_2_z_regs_negative:
+; PAIR:       // %bb.0:
+; PAIR-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; PAIR-NEXT:    addvl sp, sp, #-2
+; PAIR-NEXT:    str z10, [sp] // 16-byte Folded Spill
+; PAIR-NEXT:    str z8, [sp, #1, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; PAIR-NEXT:    .cfi_offset w29, -16
+; PAIR-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG
+; PAIR-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 16 - 16 * VG
+; PAIR-NEXT:    //APP
+; PAIR-NEXT:    //NO_APP
+; PAIR-NEXT:    ldr z10, [sp] // 16-byte Folded Reload
+; PAIR-NEXT:    ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT:    addvl sp, sp, #2
+; PAIR-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; PAIR-NEXT:    ret
+  call void asm sideeffect "", "~{z8},~{z10}"()
+  ret void
+}
+
+
+;; NOTHING TO DO HERE
+;; There is no ZReg pairs to save
+define aarch64_sve_vector_pcs  void @test_clobbers_p_reg_negative() {
+; NOPAIR-LABEL: test_clobbers_p_reg_negative:
+; NOPAIR:       // %bb.0:
+; NOPAIR-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; NOPAIR-NEXT:    addvl sp, sp, #-1
+; NOPAIR-NEXT:    str p10, [sp, #7, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; NOPAIR-NEXT:    .cfi_offset w29, -16
+; NOPAIR-NEXT:    //APP
+; NOPAIR-NEXT:    //NO_APP
+; NOPAIR-NEXT:    ldr p10, [sp, #7, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT:    addvl sp, sp, #1
+; NOPAIR-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; NOPAIR-NEXT:    ret
+;
+; PAIR-LABEL: test_clobbers_p_reg_negative:
+; PAIR:       // %bb.0:
+; PAIR-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; PAIR-NEXT:    addvl sp, sp, #-1
+; PAIR-NEXT:    str p10, [sp, #7, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG
+; PAIR-NEXT:    .cfi_offset w29, -16
+; PAIR-NEXT:    //APP
+; PAIR-NEXT:    //NO_APP
+; PAIR-NEXT:    ldr p10, [sp, #7, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT:    addvl sp, sp, #1
+; PAIR-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; PAIR-NEXT:    ret
+ call void asm sideeffect "", "~{p10}"()
+  ret void
+}
-- 
GitLab


From 5a20a07fce88d54cf01cafde489bfc2fc447acc0 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Fri, 17 May 2024 16:37:43 +0800
Subject: [PATCH 1097/1206] [AArch64] Fix -Wunused-variable in
 AArch64FrameLowering.cpp (NFC)

llvm-project/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp:3084:31:
error: unused variable 'Subtarget' [-Werror,-Wunused-variable]
      const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
                              ^
llvm-project/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp:3253:31:
error: unused variable 'Subtarget' [-Werror,-Wunused-variable]
      const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
                              ^
2 errors generated.
---
 llvm/lib/Target/AArch64/AArch64FrameLowering.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 239862756fc4..dc7759367687 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -3081,7 +3081,8 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
     }
 
     if (RPI.isPaired() && RPI.isScalable()) {
-      const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
+      [[maybe_unused]] const AArch64Subtarget &Subtarget =
+                              MF.getSubtarget<AArch64Subtarget>();
       AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
       unsigned PnReg = AFI->getPredicateRegForFillSpill();
       assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
@@ -3250,7 +3251,8 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
 
     AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     if (RPI.isPaired() && RPI.isScalable()) {
-      const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
+      [[maybe_unused]] const AArch64Subtarget &Subtarget =
+                              MF.getSubtarget<AArch64Subtarget>();
       unsigned PnReg = AFI->getPredicateRegForFillSpill();
       assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
              "Expects SVE2.1 or SME2 target and a predicate register");
-- 
GitLab


From ee54c86ef70d7809e7d67cd44de2d3153b31c46e Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 17 May 2024 13:01:37 +0400
Subject: [PATCH 1098/1206] [clang][NFC] Improve const-correctness in
 `SourceManager` (#92436)

This patch adds several const-qualified variants of existing member
functions to `SourceManager`.
I started with removing const qualification from
`setNumCreatedFIDsForFileID`, and removing `const_cast` in the body of
this function, as I think it doesn't make sense to const-qualify
setters.
---
 clang/include/clang/Basic/SourceManager.h | 44 +++++++++++++++++++++--
 clang/lib/Basic/SourceManager.cpp         |  4 +++
 2 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 5258bab584f4..ce3342355103 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -497,6 +497,10 @@ public:
   bool isFile() const { return !isExpansion(); }
 
   const FileInfo &getFile() const {
+    return const_cast<SLocEntry *>(this)->getFile();
+  }
+
+  FileInfo &getFile() {
     assert(isFile() && "Not a file SLocEntry!");
     return File;
   }
@@ -1120,12 +1124,12 @@ public:
   /// Set the number of FileIDs (files and macros) that were created
   /// during preprocessing of \p FID, including it.
   void setNumCreatedFIDsForFileID(FileID FID, unsigned NumFIDs,
-                                  bool Force = false) const {
+                                  bool Force = false) {
     auto *Entry = getSLocEntryForFile(FID);
     if (!Entry)
       return;
     assert((Force || Entry->getFile().NumCreatedFIDs == 0) && "Already set!");
-    const_cast<SrcMgr::FileInfo &>(Entry->getFile()).NumCreatedFIDs = NumFIDs;
+    Entry->getFile().NumCreatedFIDs = NumFIDs;
   }
 
   //===--------------------------------------------------------------------===//
@@ -1730,6 +1734,11 @@ public:
 
   /// Get a local SLocEntry. This is exposed for indexing.
   const SrcMgr::SLocEntry &getLocalSLocEntry(unsigned Index) const {
+    return const_cast<SourceManager *>(this)->getLocalSLocEntry(Index);
+  }
+
+  /// Get a local SLocEntry. This is exposed for indexing.
+  SrcMgr::SLocEntry &getLocalSLocEntry(unsigned Index) {
     assert(Index < LocalSLocEntryTable.size() && "Invalid index");
     return LocalSLocEntryTable[Index];
   }
@@ -1740,6 +1749,13 @@ public:
   /// Get a loaded SLocEntry. This is exposed for indexing.
   const SrcMgr::SLocEntry &getLoadedSLocEntry(unsigned Index,
                                               bool *Invalid = nullptr) const {
+    return const_cast<SourceManager *>(this)->getLoadedSLocEntry(Index,
+                                                                 Invalid);
+  }
+
+  /// Get a loaded SLocEntry. This is exposed for indexing.
+  SrcMgr::SLocEntry &getLoadedSLocEntry(unsigned Index,
+                                        bool *Invalid = nullptr) {
     assert(Index < LoadedSLocEntryTable.size() && "Invalid index");
     if (SLocEntryLoaded[Index])
       return LoadedSLocEntryTable[Index];
@@ -1748,6 +1764,10 @@ public:
 
   const SrcMgr::SLocEntry &getSLocEntry(FileID FID,
                                         bool *Invalid = nullptr) const {
+    return const_cast<SourceManager *>(this)->getSLocEntry(FID, Invalid);
+  }
+
+  SrcMgr::SLocEntry &getSLocEntry(FileID FID, bool *Invalid = nullptr) {
     if (FID.ID == 0 || FID.ID == -1) {
       if (Invalid) *Invalid = true;
       return LocalSLocEntryTable[0];
@@ -1821,14 +1841,23 @@ private:
   SrcMgr::ContentCache &getFakeContentCacheForRecovery() const;
 
   const SrcMgr::SLocEntry &loadSLocEntry(unsigned Index, bool *Invalid) const;
+  SrcMgr::SLocEntry &loadSLocEntry(unsigned Index, bool *Invalid);
 
   const SrcMgr::SLocEntry *getSLocEntryOrNull(FileID FID) const {
+    return const_cast<SourceManager *>(this)->getSLocEntryOrNull(FID);
+  }
+
+  SrcMgr::SLocEntry *getSLocEntryOrNull(FileID FID) {
     bool Invalid = false;
-    const SrcMgr::SLocEntry &Entry = getSLocEntry(FID, &Invalid);
+    SrcMgr::SLocEntry &Entry = getSLocEntry(FID, &Invalid);
     return Invalid ? nullptr : &Entry;
   }
 
   const SrcMgr::SLocEntry *getSLocEntryForFile(FileID FID) const {
+    return const_cast<SourceManager *>(this)->getSLocEntryForFile(FID);
+  }
+
+  SrcMgr::SLocEntry *getSLocEntryForFile(FileID FID) {
     if (auto *Entry = getSLocEntryOrNull(FID))
       if (Entry->isFile())
         return Entry;
@@ -1839,6 +1868,10 @@ private:
   /// Invalid will not be modified for Local IDs.
   const SrcMgr::SLocEntry &getSLocEntryByID(int ID,
                                             bool *Invalid = nullptr) const {
+    return const_cast<SourceManager *>(this)->getSLocEntryByID(ID, Invalid);
+  }
+
+  SrcMgr::SLocEntry &getSLocEntryByID(int ID, bool *Invalid = nullptr) {
     assert(ID != -1 && "Using FileID sentinel value");
     if (ID < 0)
       return getLoadedSLocEntryByID(ID, Invalid);
@@ -1847,6 +1880,11 @@ private:
 
   const SrcMgr::SLocEntry &
   getLoadedSLocEntryByID(int ID, bool *Invalid = nullptr) const {
+    return const_cast<SourceManager *>(this)->getLoadedSLocEntryByID(ID,
+                                                                     Invalid);
+  }
+
+  SrcMgr::SLocEntry &getLoadedSLocEntryByID(int ID, bool *Invalid = nullptr) {
     return getLoadedSLocEntry(static_cast<unsigned>(-ID - 2), Invalid);
   }
 
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 37734d3b10e7..afdb5b0e92d6 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -431,6 +431,10 @@ ContentCache &SourceManager::createMemBufferContentCache(
 
 const SrcMgr::SLocEntry &SourceManager::loadSLocEntry(unsigned Index,
                                                       bool *Invalid) const {
+  return const_cast<SourceManager *>(this)->loadSLocEntry(Index, Invalid);
+}
+
+SrcMgr::SLocEntry &SourceManager::loadSLocEntry(unsigned Index, bool *Invalid) {
   assert(!SLocEntryLoaded[Index]);
   if (ExternalSLocEntries->ReadSLocEntry(-(static_cast<int>(Index) + 2))) {
     if (Invalid)
-- 
GitLab


From 33bf08ec36efc2fc2df8217eddd751cef9bc6be6 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Fri, 17 May 2024 11:24:20 +0200
Subject: [PATCH 1099/1206] [lldb-dap] Correctly detect alias commands with
 arguments in repl (#92137)

ResolveCommand will not succeed for an alias command with arguments, and
the code wasn't providing any. Replace that with explicit query(ies) for
the existence of a command with the given name.
---
 .../API/tools/lldb-dap/repl-mode/Makefile     |  3 +
 .../repl-mode/TestDAP_repl_mode_detection.py  | 55 +++++++++++++++++++
 .../API/tools/lldb-dap/repl-mode/main.cpp     | 15 +++++
 lldb/tools/lldb-dap/DAP.cpp                   |  8 ++-
 4 files changed, 78 insertions(+), 3 deletions(-)
 create mode 100644 lldb/test/API/tools/lldb-dap/repl-mode/Makefile
 create mode 100644 lldb/test/API/tools/lldb-dap/repl-mode/TestDAP_repl_mode_detection.py
 create mode 100644 lldb/test/API/tools/lldb-dap/repl-mode/main.cpp

diff --git a/lldb/test/API/tools/lldb-dap/repl-mode/Makefile b/lldb/test/API/tools/lldb-dap/repl-mode/Makefile
new file mode 100644
index 000000000000..99998b20bcb0
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/repl-mode/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/tools/lldb-dap/repl-mode/TestDAP_repl_mode_detection.py b/lldb/test/API/tools/lldb-dap/repl-mode/TestDAP_repl_mode_detection.py
new file mode 100644
index 000000000000..7c77fc8541b9
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/repl-mode/TestDAP_repl_mode_detection.py
@@ -0,0 +1,55 @@
+"""
+Test lldb-dap repl mode detection
+"""
+
+import lldbdap_testcase
+import dap_server
+from lldbsuite.test import lldbutil
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+
+
+class TestDAP_repl_mode_detection(lldbdap_testcase.DAPTestCaseBase):
+    def assertEvaluate(self, expression, regex):
+        self.assertRegex(
+            self.dap_server.request_evaluate(expression, context="repl")["body"][
+                "result"
+            ],
+            regex,
+        )
+
+    def test_completions(self):
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+
+        source = "main.cpp"
+        breakpoint1_line = line_number(source, "// breakpoint 1")
+        breakpoint2_line = line_number(source, "// breakpoint 2")
+
+        self.set_source_breakpoints(source, [breakpoint1_line, breakpoint2_line])
+
+        self.assertEvaluate(
+            "`command regex user_command s/^$/platform/", r"\(lldb\) command regex"
+        )
+        self.assertEvaluate(
+            "`command alias alias_command platform", r"\(lldb\) command alias"
+        )
+        self.assertEvaluate(
+            "`command alias alias_command_with_arg platform select --sysroot %1 remote-linux",
+            r"\(lldb\) command alias",
+        )
+
+        self.continue_to_next_stop()
+        self.assertEvaluate("user_command", "474747")
+        self.assertEvaluate("alias_command", "474747")
+        self.assertEvaluate("alias_command_with_arg", "474747")
+        self.assertEvaluate("platform", "474747")
+
+        self.continue_to_next_stop()
+        platform_help_needle = "Commands to manage and create platforms"
+        self.assertEvaluate("user_command", platform_help_needle)
+        self.assertEvaluate("alias_command", platform_help_needle)
+        self.assertEvaluate(
+            "alias_command_with_arg " + self.getBuildDir(), "Platform: remote-linux"
+        )
+        self.assertEvaluate("platform", platform_help_needle)
diff --git a/lldb/test/API/tools/lldb-dap/repl-mode/main.cpp b/lldb/test/API/tools/lldb-dap/repl-mode/main.cpp
new file mode 100644
index 000000000000..52561d3471ab
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/repl-mode/main.cpp
@@ -0,0 +1,15 @@
+void noop() {}
+
+void fun() {
+  int user_command = 474747;
+  int alias_command = 474747;
+  int alias_command_with_arg = 474747;
+  int platform = 474747; // built-in command
+  noop();                // breakpoint 1
+}
+
+int main() {
+  fun();
+  noop(); // breakpoint 2
+  return 0;
+}
diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp
index 55ff1493c101..c7eb3db4304a 100644
--- a/lldb/tools/lldb-dap/DAP.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -14,6 +14,7 @@
 
 #include "DAP.h"
 #include "LLDBUtils.h"
+#include "lldb/API/SBCommandInterpreter.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/FormatVariadic.h"
 
@@ -405,9 +406,10 @@ ExpressionContext DAP::DetectExpressionContext(lldb::SBFrame frame,
     std::pair<llvm::StringRef, llvm::StringRef> token =
         llvm::getToken(expression);
     std::string term = token.first.str();
-    lldb::SBCommandReturnObject result;
-    debugger.GetCommandInterpreter().ResolveCommand(term.c_str(), result);
-    bool term_is_command = result.Succeeded();
+    lldb::SBCommandInterpreter interpreter = debugger.GetCommandInterpreter();
+    bool term_is_command = interpreter.CommandExists(term.c_str()) ||
+                           interpreter.UserCommandExists(term.c_str()) ||
+                           interpreter.AliasExists(term.c_str());
     bool term_is_variable = frame.FindVariable(term.c_str()).IsValid();
 
     // If we have both a variable and command, warn the user about the conflict.
-- 
GitLab


From 7a6747939218efbe3b1d2cc0f896dfa97c0ff40f Mon Sep 17 00:00:00 2001
From: Daniel Grumberg <dgrumberg@apple.com>
Date: Fri, 17 May 2024 10:33:31 +0100
Subject: [PATCH 1100/1206] [clang][ExtractAPI] Correctly generate declaration
 fragments for non-type template parameters (#91958)

Previously we only generated declaration fragments for template type
parameters/arguments, this adds supports for most other possible
template parameters/arguments.

rdar://127732598
---
 clang/lib/ExtractAPI/DeclarationFragments.cpp | 226 ++++++++++---
 clang/test/ExtractAPI/class_template.cpp      |   2 +-
 .../class_template_param_inheritance.cpp      |   2 +-
 .../class_template_partial_spec.cpp           |   4 +-
 clang/test/ExtractAPI/class_template_spec.cpp |   4 +-
 clang/test/ExtractAPI/concept.cpp             |   2 +-
 clang/test/ExtractAPI/field_template.cpp      |   2 +-
 .../test/ExtractAPI/global_func_template.cpp  |   4 +-
 .../ExtractAPI/global_func_template_spec.cpp  |   4 +-
 clang/test/ExtractAPI/global_var_template.cpp |   2 +-
 .../global_var_template_partial_spec.cpp      |   4 +-
 .../ExtractAPI/global_var_template_spec.cpp   |   4 +-
 clang/test/ExtractAPI/method_template.cpp     |   2 +-
 .../test/ExtractAPI/method_template_spec.cpp  |   4 +-
 clang/test/ExtractAPI/non_type_template.cpp   | 313 ++++++++++++++++++
 15 files changed, 522 insertions(+), 57 deletions(-)
 create mode 100644 clang/test/ExtractAPI/non_type_template.cpp

diff --git a/clang/lib/ExtractAPI/DeclarationFragments.cpp b/clang/lib/ExtractAPI/DeclarationFragments.cpp
index 9bf7950888db..98b9343924a8 100644
--- a/clang/lib/ExtractAPI/DeclarationFragments.cpp
+++ b/clang/lib/ExtractAPI/DeclarationFragments.cpp
@@ -12,13 +12,19 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/ExtractAPI/DeclarationFragments.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/TemplateBase.h"
+#include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/ExtractAPI/TypedefUnderlyingTypeResolver.h"
 #include "clang/Index/USRGeneration.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#include <optional>
 
 using namespace clang::extractapi;
 using namespace llvm;
@@ -386,6 +392,25 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForType(
         getFragmentsForType(AT->getElementType(), Context, After));
   }
 
+  if (const TemplateSpecializationType *TemplSpecTy =
+          dyn_cast<TemplateSpecializationType>(T)) {
+    const auto TemplName = TemplSpecTy->getTemplateName();
+    std::string Str;
+    raw_string_ostream Stream(Str);
+    TemplName.print(Stream, Context.getPrintingPolicy(),
+                    TemplateName::Qualified::AsWritten);
+    SmallString<64> USR("");
+    if (const auto *TemplDecl = TemplName.getAsTemplateDecl())
+      index::generateUSRForDecl(TemplDecl, USR);
+
+    return Fragments
+        .append(Str, DeclarationFragments::FragmentKind::TypeIdentifier, USR)
+        .append("<", DeclarationFragments::FragmentKind::Text)
+        .append(getFragmentsForTemplateArguments(
+            TemplSpecTy->template_arguments(), Context, std::nullopt))
+        .append(">", DeclarationFragments::FragmentKind::Text);
+  }
+
   // Everything we care about has been handled now, reduce to the canonical
   // unqualified base type.
   QualType Base = T->getCanonicalTypeUnqualified();
@@ -650,7 +675,6 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForBlock(
 DeclarationFragments
 DeclarationFragmentsBuilder::getFragmentsForFunction(const FunctionDecl *Func) {
   DeclarationFragments Fragments;
-  // FIXME: Handle template specialization
   switch (Func->getStorageClass()) {
   case SC_None:
   case SC_PrivateExtern:
@@ -952,27 +976,84 @@ DeclarationFragmentsBuilder::getFragmentsForTemplateParameters(
       Fragments.append(",", DeclarationFragments::FragmentKind::Text)
           .appendSpace();
 
-    const auto *TemplateParam =
-        dyn_cast<TemplateTypeParmDecl>(ParameterArray[i]);
-    if (!TemplateParam)
-      continue;
-    if (TemplateParam->hasTypeConstraint())
-      Fragments.append(TemplateParam->getTypeConstraint()
-                           ->getNamedConcept()
-                           ->getName()
-                           .str(),
-                       DeclarationFragments::FragmentKind::TypeIdentifier);
-    else if (TemplateParam->wasDeclaredWithTypename())
-      Fragments.append("typename", DeclarationFragments::FragmentKind::Keyword);
-    else
-      Fragments.append("class", DeclarationFragments::FragmentKind::Keyword);
-
-    if (TemplateParam->isParameterPack())
-      Fragments.append("...", DeclarationFragments::FragmentKind::Text);
-
-    Fragments.appendSpace().append(
-        TemplateParam->getName(),
-        DeclarationFragments::FragmentKind::GenericParameter);
+    if (const auto *TemplateParam =
+            dyn_cast<TemplateTypeParmDecl>(ParameterArray[i])) {
+      if (TemplateParam->hasTypeConstraint())
+        Fragments.append(TemplateParam->getTypeConstraint()
+                             ->getNamedConcept()
+                             ->getName()
+                             .str(),
+                         DeclarationFragments::FragmentKind::TypeIdentifier);
+      else if (TemplateParam->wasDeclaredWithTypename())
+        Fragments.append("typename",
+                         DeclarationFragments::FragmentKind::Keyword);
+      else
+        Fragments.append("class", DeclarationFragments::FragmentKind::Keyword);
+
+      if (TemplateParam->isParameterPack())
+        Fragments.append("...", DeclarationFragments::FragmentKind::Text);
+
+      if (!TemplateParam->getName().empty())
+        Fragments.appendSpace().append(
+            TemplateParam->getName(),
+            DeclarationFragments::FragmentKind::GenericParameter);
+
+      if (TemplateParam->hasDefaultArgument()) {
+        DeclarationFragments After;
+        Fragments.append(" = ", DeclarationFragments::FragmentKind::Text)
+            .append(getFragmentsForType(TemplateParam->getDefaultArgument(),
+                                        TemplateParam->getASTContext(), After));
+        Fragments.append(std::move(After));
+      }
+    } else if (const auto *NTP =
+                   dyn_cast<NonTypeTemplateParmDecl>(ParameterArray[i])) {
+      DeclarationFragments After;
+      const auto TyFragments =
+          getFragmentsForType(NTP->getType(), NTP->getASTContext(), After);
+      Fragments.append(std::move(TyFragments)).append(std::move(After));
+
+      if (NTP->isParameterPack())
+        Fragments.append("...", DeclarationFragments::FragmentKind::Text);
+
+      if (!NTP->getName().empty())
+        Fragments.appendSpace().append(
+            NTP->getName(),
+            DeclarationFragments::FragmentKind::GenericParameter);
+
+      if (NTP->hasDefaultArgument()) {
+        SmallString<8> ExprStr;
+        raw_svector_ostream Output(ExprStr);
+        NTP->getDefaultArgument()->printPretty(
+            Output, nullptr, NTP->getASTContext().getPrintingPolicy());
+        Fragments.append(" = ", DeclarationFragments::FragmentKind::Text)
+            .append(ExprStr, DeclarationFragments::FragmentKind::Text);
+      }
+    } else if (const auto *TTP =
+                   dyn_cast<TemplateTemplateParmDecl>(ParameterArray[i])) {
+      Fragments.append("template", DeclarationFragments::FragmentKind::Keyword)
+          .appendSpace()
+          .append("<", DeclarationFragments::FragmentKind::Text)
+          .append(getFragmentsForTemplateParameters(
+              TTP->getTemplateParameters()->asArray()))
+          .append(">", DeclarationFragments::FragmentKind::Text)
+          .appendSpace()
+          .append(TTP->wasDeclaredWithTypename() ? "typename" : "class",
+                  DeclarationFragments::FragmentKind::Keyword);
+
+      if (TTP->isParameterPack())
+        Fragments.append("...", DeclarationFragments::FragmentKind::Text);
+
+      if (!TTP->getName().empty())
+        Fragments.appendSpace().append(
+            TTP->getName(),
+            DeclarationFragments::FragmentKind::GenericParameter);
+      if (TTP->hasDefaultArgument()) {
+        const auto Default = TTP->getDefaultArgument();
+        Fragments.append(" = ", DeclarationFragments::FragmentKind::Text)
+            .append(getFragmentsForTemplateArguments(
+                {Default.getArgument()}, TTP->getASTContext(), {Default}));
+      }
+    }
   }
   return Fragments;
 }
@@ -993,23 +1074,85 @@ DeclarationFragmentsBuilder::getFragmentsForTemplateArguments(
       Fragments.append(",", DeclarationFragments::FragmentKind::Text)
           .appendSpace();
 
-    std::string Type = TemplateArguments[i].getAsType().getAsString();
-    DeclarationFragments After;
-    DeclarationFragments ArgumentFragment =
-        getFragmentsForType(TemplateArguments[i].getAsType(), Context, After);
-
-    if (StringRef(ArgumentFragment.begin()->Spelling)
-            .starts_with("type-parameter")) {
-      std::string ProperArgName = TemplateArgumentLocs.value()[i]
-                                      .getTypeSourceInfo()
-                                      ->getType()
-                                      .getAsString();
-      ArgumentFragment.begin()->Spelling.swap(ProperArgName);
+    const auto &CTA = TemplateArguments[i];
+    switch (CTA.getKind()) {
+    case TemplateArgument::Type: {
+      DeclarationFragments After;
+      DeclarationFragments ArgumentFragment =
+          getFragmentsForType(CTA.getAsType(), Context, After);
+
+      if (StringRef(ArgumentFragment.begin()->Spelling)
+              .starts_with("type-parameter")) {
+        std::string ProperArgName = TemplateArgumentLocs.value()[i]
+                                        .getTypeSourceInfo()
+                                        ->getType()
+                                        .getAsString();
+        ArgumentFragment.begin()->Spelling.swap(ProperArgName);
+      }
+      Fragments.append(std::move(ArgumentFragment));
+      break;
     }
-    Fragments.append(std::move(ArgumentFragment));
+    case TemplateArgument::Declaration: {
+      const auto *VD = CTA.getAsDecl();
+      SmallString<128> USR;
+      index::generateUSRForDecl(VD, USR);
+      Fragments.append(VD->getNameAsString(),
+                       DeclarationFragments::FragmentKind::Identifier, USR);
+      break;
+    }
+    case TemplateArgument::NullPtr:
+      Fragments.append("nullptr", DeclarationFragments::FragmentKind::Keyword);
+      break;
 
-    if (TemplateArguments[i].isPackExpansion())
-      Fragments.append("...", DeclarationFragments::FragmentKind::Text);
+    case TemplateArgument::Integral: {
+      SmallString<4> Str;
+      CTA.getAsIntegral().toString(Str);
+      Fragments.append(Str, DeclarationFragments::FragmentKind::Text);
+      break;
+    }
+
+    case TemplateArgument::StructuralValue: {
+      const auto SVTy = CTA.getStructuralValueType();
+      Fragments.append(CTA.getAsStructuralValue().getAsString(Context, SVTy),
+                       DeclarationFragments::FragmentKind::Text);
+      break;
+    }
+
+    case TemplateArgument::TemplateExpansion:
+    case TemplateArgument::Template: {
+      std::string Str;
+      raw_string_ostream Stream(Str);
+      CTA.getAsTemplate().print(Stream, Context.getPrintingPolicy());
+      SmallString<64> USR("");
+      if (const auto *TemplDecl =
+              CTA.getAsTemplateOrTemplatePattern().getAsTemplateDecl())
+        index::generateUSRForDecl(TemplDecl, USR);
+      Fragments.append(Str, DeclarationFragments::FragmentKind::TypeIdentifier,
+                       USR);
+      if (CTA.getKind() == TemplateArgument::TemplateExpansion)
+        Fragments.append("...", DeclarationFragments::FragmentKind::Text);
+      break;
+    }
+
+    case TemplateArgument::Pack:
+      Fragments.append("<", DeclarationFragments::FragmentKind::Text)
+          .append(getFragmentsForTemplateArguments(CTA.pack_elements(), Context,
+                                                   {}))
+          .append(">", DeclarationFragments::FragmentKind::Text);
+      break;
+
+    case TemplateArgument::Expression: {
+      SmallString<8> ExprStr;
+      raw_svector_ostream Output(ExprStr);
+      CTA.getAsExpr()->printPretty(Output, nullptr,
+                                   Context.getPrintingPolicy());
+      Fragments.append(ExprStr, DeclarationFragments::FragmentKind::Text);
+      break;
+    }
+
+    case TemplateArgument::Null:
+      break;
+    }
   }
   return Fragments;
 }
@@ -1019,10 +1162,12 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForConcept(
   DeclarationFragments Fragments;
   return Fragments
       .append("template", DeclarationFragments::FragmentKind::Keyword)
+      .appendSpace()
       .append("<", DeclarationFragments::FragmentKind::Text)
       .append(getFragmentsForTemplateParameters(
           Concept->getTemplateParameters()->asArray()))
       .append("> ", DeclarationFragments::FragmentKind::Text)
+      .appendSpace()
       .append("concept", DeclarationFragments::FragmentKind::Keyword)
       .appendSpace()
       .append(Concept->getName().str(),
@@ -1035,6 +1180,7 @@ DeclarationFragmentsBuilder::getFragmentsForRedeclarableTemplate(
     const RedeclarableTemplateDecl *RedeclarableTemplate) {
   DeclarationFragments Fragments;
   Fragments.append("template", DeclarationFragments::FragmentKind::Keyword)
+      .appendSpace()
       .append("<", DeclarationFragments::FragmentKind::Text)
       .append(getFragmentsForTemplateParameters(
           RedeclarableTemplate->getTemplateParameters()->asArray()))
@@ -1057,6 +1203,7 @@ DeclarationFragmentsBuilder::getFragmentsForClassTemplateSpecialization(
   DeclarationFragments Fragments;
   return Fragments
       .append("template", DeclarationFragments::FragmentKind::Keyword)
+      .appendSpace()
       .append("<", DeclarationFragments::FragmentKind::Text)
       .append(">", DeclarationFragments::FragmentKind::Text)
       .appendSpace()
@@ -1077,6 +1224,7 @@ DeclarationFragmentsBuilder::getFragmentsForClassTemplatePartialSpecialization(
   DeclarationFragments Fragments;
   return Fragments
       .append("template", DeclarationFragments::FragmentKind::Keyword)
+      .appendSpace()
       .append("<", DeclarationFragments::FragmentKind::Text)
       .append(getFragmentsForTemplateParameters(
           Decl->getTemplateParameters()->asArray()))
@@ -1099,6 +1247,7 @@ DeclarationFragmentsBuilder::getFragmentsForVarTemplateSpecialization(
   DeclarationFragments Fragments;
   return Fragments
       .append("template", DeclarationFragments::FragmentKind::Keyword)
+      .appendSpace()
       .append("<", DeclarationFragments::FragmentKind::Text)
       .append(">", DeclarationFragments::FragmentKind::Text)
       .appendSpace()
@@ -1118,6 +1267,7 @@ DeclarationFragmentsBuilder::getFragmentsForVarTemplatePartialSpecialization(
   DeclarationFragments Fragments;
   return Fragments
       .append("template", DeclarationFragments::FragmentKind::Keyword)
+      .appendSpace()
       .append("<", DeclarationFragments::FragmentKind::Text)
       // Partial specs may have new params.
       .append(getFragmentsForTemplateParameters(
@@ -1140,6 +1290,7 @@ DeclarationFragmentsBuilder::getFragmentsForFunctionTemplate(
   DeclarationFragments Fragments;
   return Fragments
       .append("template", DeclarationFragments::FragmentKind::Keyword)
+      .appendSpace()
       .append("<", DeclarationFragments::FragmentKind::Text)
       // Partial specs may have new params.
       .append(getFragmentsForTemplateParameters(
@@ -1156,6 +1307,7 @@ DeclarationFragmentsBuilder::getFragmentsForFunctionTemplateSpecialization(
   DeclarationFragments Fragments;
   return Fragments
       .append("template", DeclarationFragments::FragmentKind::Keyword)
+      .appendSpace()
       .append("<>", DeclarationFragments::FragmentKind::Text)
       .appendSpace()
       .append(DeclarationFragmentsBuilder::getFragmentsForFunction(Decl));
diff --git a/clang/test/ExtractAPI/class_template.cpp b/clang/test/ExtractAPI/class_template.cpp
index 4f2670d7b699..203d47394dd6 100644
--- a/clang/test/ExtractAPI/class_template.cpp
+++ b/clang/test/ExtractAPI/class_template.cpp
@@ -51,7 +51,7 @@ template<typename T> class Foo {};
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
diff --git a/clang/test/ExtractAPI/class_template_param_inheritance.cpp b/clang/test/ExtractAPI/class_template_param_inheritance.cpp
index 3d7b09f93ed6..53b331e0b460 100644
--- a/clang/test/ExtractAPI/class_template_param_inheritance.cpp
+++ b/clang/test/ExtractAPI/class_template_param_inheritance.cpp
@@ -58,7 +58,7 @@ template<typename T> class Foo : public T {};
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
diff --git a/clang/test/ExtractAPI/class_template_partial_spec.cpp b/clang/test/ExtractAPI/class_template_partial_spec.cpp
index c8d9cc78d41c..7d244f49c735 100644
--- a/clang/test/ExtractAPI/class_template_partial_spec.cpp
+++ b/clang/test/ExtractAPI/class_template_partial_spec.cpp
@@ -53,7 +53,7 @@ template<typename Z> class Foo<Z, int> {};
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
@@ -161,7 +161,7 @@ template<typename Z> class Foo<Z, int> {};
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
diff --git a/clang/test/ExtractAPI/class_template_spec.cpp b/clang/test/ExtractAPI/class_template_spec.cpp
index 06a95314dc4a..9dbd887510c7 100644
--- a/clang/test/ExtractAPI/class_template_spec.cpp
+++ b/clang/test/ExtractAPI/class_template_spec.cpp
@@ -53,7 +53,7 @@ template<> class Foo<int> {};
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
@@ -140,7 +140,7 @@ template<> class Foo<int> {};
         },
         {
           "kind": "text",
-          "spelling": "<> "
+          "spelling": " <> "
         },
         {
           "kind": "keyword",
diff --git a/clang/test/ExtractAPI/concept.cpp b/clang/test/ExtractAPI/concept.cpp
index 443eac2971f0..4396a39a63ef 100644
--- a/clang/test/ExtractAPI/concept.cpp
+++ b/clang/test/ExtractAPI/concept.cpp
@@ -51,7 +51,7 @@ template<typename T> concept Foo = true;
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
diff --git a/clang/test/ExtractAPI/field_template.cpp b/clang/test/ExtractAPI/field_template.cpp
index 2058ed008cfe..973a9de5c998 100644
--- a/clang/test/ExtractAPI/field_template.cpp
+++ b/clang/test/ExtractAPI/field_template.cpp
@@ -114,7 +114,7 @@ class Foo {
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
diff --git a/clang/test/ExtractAPI/global_func_template.cpp b/clang/test/ExtractAPI/global_func_template.cpp
index f43a618ec0c3..044757c316d5 100644
--- a/clang/test/ExtractAPI/global_func_template.cpp
+++ b/clang/test/ExtractAPI/global_func_template.cpp
@@ -52,7 +52,7 @@ template<typename T> T Fizz(int Buzz);
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
@@ -186,7 +186,7 @@ template<typename T> T Fizz(int Buzz);
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
diff --git a/clang/test/ExtractAPI/global_func_template_spec.cpp b/clang/test/ExtractAPI/global_func_template_spec.cpp
index fe046e9c3b9d..b88063aacfed 100644
--- a/clang/test/ExtractAPI/global_func_template_spec.cpp
+++ b/clang/test/ExtractAPI/global_func_template_spec.cpp
@@ -52,7 +52,7 @@ template<> void Foo<int>(int Bar);
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
@@ -186,7 +186,7 @@ template<> void Foo<int>(int Bar);
         },
         {
           "kind": "text",
-          "spelling": "<> "
+          "spelling": " <> "
         },
         {
           "kind": "typeIdentifier",
diff --git a/clang/test/ExtractAPI/global_var_template.cpp b/clang/test/ExtractAPI/global_var_template.cpp
index 94f3713cd3d3..5c9194141332 100644
--- a/clang/test/ExtractAPI/global_var_template.cpp
+++ b/clang/test/ExtractAPI/global_var_template.cpp
@@ -50,7 +50,7 @@ template<typename T> T Foo = T(3.14);
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
diff --git a/clang/test/ExtractAPI/global_var_template_partial_spec.cpp b/clang/test/ExtractAPI/global_var_template_partial_spec.cpp
index 91084f258878..ffb1557fd758 100644
--- a/clang/test/ExtractAPI/global_var_template_partial_spec.cpp
+++ b/clang/test/ExtractAPI/global_var_template_partial_spec.cpp
@@ -52,7 +52,7 @@ template<typename Z> int Foo<int, Z> = 0;
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
@@ -161,7 +161,7 @@ template<typename Z> int Foo<int, Z> = 0;
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
diff --git a/clang/test/ExtractAPI/global_var_template_spec.cpp b/clang/test/ExtractAPI/global_var_template_spec.cpp
index ff4d8d17aecb..e465da97db1f 100644
--- a/clang/test/ExtractAPI/global_var_template_spec.cpp
+++ b/clang/test/ExtractAPI/global_var_template_spec.cpp
@@ -52,7 +52,7 @@ template<> int Foo<int>;
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
@@ -140,7 +140,7 @@ template<> int Foo<int>;
         },
         {
           "kind": "text",
-          "spelling": "<> "
+          "spelling": " <> "
         },
         {
           "kind": "typeIdentifier",
diff --git a/clang/test/ExtractAPI/method_template.cpp b/clang/test/ExtractAPI/method_template.cpp
index 714f9cac26c2..bf91a208c548 100644
--- a/clang/test/ExtractAPI/method_template.cpp
+++ b/clang/test/ExtractAPI/method_template.cpp
@@ -114,7 +114,7 @@ class Foo {
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
diff --git a/clang/test/ExtractAPI/method_template_spec.cpp b/clang/test/ExtractAPI/method_template_spec.cpp
index 8eaffdefd827..978dbe1b6c1d 100644
--- a/clang/test/ExtractAPI/method_template_spec.cpp
+++ b/clang/test/ExtractAPI/method_template_spec.cpp
@@ -122,7 +122,7 @@ class Foo {
         },
         {
           "kind": "text",
-          "spelling": "<"
+          "spelling": " <"
         },
         {
           "kind": "keyword",
@@ -257,7 +257,7 @@ class Foo {
         },
         {
           "kind": "text",
-          "spelling": "<> "
+          "spelling": " <> "
         },
         {
           "kind": "typeIdentifier",
diff --git a/clang/test/ExtractAPI/non_type_template.cpp b/clang/test/ExtractAPI/non_type_template.cpp
new file mode 100644
index 000000000000..4e65eb790ca1
--- /dev/null
+++ b/clang/test/ExtractAPI/non_type_template.cpp
@@ -0,0 +1,313 @@
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -extract-api --pretty-sgf --emit-sgf-symbol-labels-for-testing \
+// RUN:   -triple arm64-apple-macosx -std=c++17 -x c++-header %s -o %t/output.symbols.json -verify
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix FOO
+template <typename T, int N = 4> class Foo { };
+// FOO-LABEL: "!testLabel": "c:@ST>2#T#NI@Foo"
+// FOO:      "declarationFragments": [
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "keyword",
+// FOO-NEXT:     "spelling": "template"
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "text",
+// FOO-NEXT:     "spelling": " <"
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "keyword",
+// FOO-NEXT:     "spelling": "typename"
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "text",
+// FOO-NEXT:     "spelling": " "
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "genericParameter",
+// FOO-NEXT:     "spelling": "T"
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "text",
+// FOO-NEXT:     "spelling": ", "
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "typeIdentifier",
+// FOO-NEXT:     "preciseIdentifier": "c:I",
+// FOO-NEXT:     "spelling": "int"
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "text",
+// FOO-NEXT:     "spelling": " "
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "genericParameter",
+// FOO-NEXT:     "spelling": "N"
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "text",
+// FOO-NEXT:     "spelling": " = 4> "
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "keyword",
+// FOO-NEXT:     "spelling": "class"
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "text",
+// FOO-NEXT:     "spelling": " "
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "identifier",
+// FOO-NEXT:     "spelling": "Foo"
+// FOO-NEXT:   },
+// FOO-NEXT:   {
+// FOO-NEXT:     "kind": "text",
+// FOO-NEXT:     "spelling": ";"
+// FOO-NEXT:   }
+// FOO-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix FOO-SPEC
+template <typename T> class Foo <T, 4> { };
+// FOO-SPEC-LABEL: "!testLabel": "c:@SP>1#T@Foo>#t0.0#VI4"
+// FOO-SPEC:      "declarationFragments": [
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "keyword",
+// FOO-SPEC-NEXT:     "spelling": "template"
+// FOO-SPEC-NEXT:   },
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "text",
+// FOO-SPEC-NEXT:     "spelling": " <"
+// FOO-SPEC-NEXT:   },
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "keyword",
+// FOO-SPEC-NEXT:     "spelling": "typename"
+// FOO-SPEC-NEXT:   },
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "text",
+// FOO-SPEC-NEXT:     "spelling": " "
+// FOO-SPEC-NEXT:   },
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "genericParameter",
+// FOO-SPEC-NEXT:     "spelling": "T"
+// FOO-SPEC-NEXT:   },
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "text",
+// FOO-SPEC-NEXT:     "spelling": "> "
+// FOO-SPEC-NEXT:   },
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "keyword",
+// FOO-SPEC-NEXT:     "spelling": "class"
+// FOO-SPEC-NEXT:   },
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "text",
+// FOO-SPEC-NEXT:     "spelling": " "
+// FOO-SPEC-NEXT:   },
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "identifier",
+// FOO-SPEC-NEXT:     "spelling": "Foo"
+// FOO-SPEC-NEXT:   },
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "text",
+// FOO-SPEC-NEXT:     "spelling": "<"
+// FOO-SPEC-NEXT:   },
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "typeIdentifier",
+// FOO-SPEC-NEXT:     "preciseIdentifier": "c:t0.0",
+// FOO-SPEC-NEXT:     "spelling": "T"
+// FOO-SPEC-NEXT:   },
+// FOO-SPEC-NEXT:   {
+// FOO-SPEC-NEXT:     "kind": "text",
+// FOO-SPEC-NEXT:     "spelling": ", 4>;"
+// FOO-SPEC-NEXT:   }
+// FOO-SPEC-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix NEST
+template <template <template <typename> typename> class... Bs> class NestedTemplateTemplateParamPack{ };
+// NEST-LABEL: "!testLabel": "c:@ST>1#pt>1#t>1#T@NestedTemplateTemplateParamPack"
+// NEST:      "declarationFragments": [
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "keyword",
+// NEST-NEXT:     "spelling": "template"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "text",
+// NEST-NEXT:     "spelling": " <"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "keyword",
+// NEST-NEXT:     "spelling": "template"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "text",
+// NEST-NEXT:     "spelling": " <"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "keyword",
+// NEST-NEXT:     "spelling": "template"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "text",
+// NEST-NEXT:     "spelling": " <"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "keyword",
+// NEST-NEXT:     "spelling": "typename"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "text",
+// NEST-NEXT:     "spelling": "> "
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "keyword",
+// NEST-NEXT:     "spelling": "typename"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "text",
+// NEST-NEXT:     "spelling": "> "
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "keyword",
+// NEST-NEXT:     "spelling": "class"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "text",
+// NEST-NEXT:     "spelling": "... "
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "genericParameter",
+// NEST-NEXT:     "spelling": "Bs"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "text",
+// NEST-NEXT:     "spelling": "> "
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "keyword",
+// NEST-NEXT:     "spelling": "class"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "text",
+// NEST-NEXT:     "spelling": " "
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "identifier",
+// NEST-NEXT:     "spelling": "NestedTemplateTemplateParamPack"
+// NEST-NEXT:   },
+// NEST-NEXT:   {
+// NEST-NEXT:     "kind": "text",
+// NEST-NEXT:     "spelling": ";"
+// NEST-NEXT:   }
+// NEST-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix BAR
+template <template <typename> typename T = Foo> struct Bar { };
+// BAR-LABEL: "!testLabel": "c:@ST>1#t>1#T@Bar"
+// BAR:      "declarationFragments": [
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "keyword",
+// BAR-NEXT:     "spelling": "template"
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "text",
+// BAR-NEXT:     "spelling": " <"
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "keyword",
+// BAR-NEXT:     "spelling": "template"
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "text",
+// BAR-NEXT:     "spelling": " <"
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "keyword",
+// BAR-NEXT:     "spelling": "typename"
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "text",
+// BAR-NEXT:     "spelling": "> "
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "keyword",
+// BAR-NEXT:     "spelling": "typename"
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "text",
+// BAR-NEXT:     "spelling": " "
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "genericParameter",
+// BAR-NEXT:     "spelling": "T"
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "text",
+// BAR-NEXT:     "spelling": " = "
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "typeIdentifier",
+// BAR-NEXT:     "preciseIdentifier": "c:@ST>2#T#NI@Foo",
+// BAR-NEXT:     "spelling": "Foo"
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "text",
+// BAR-NEXT:     "spelling": "> "
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "keyword",
+// BAR-NEXT:     "spelling": "struct"
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "text",
+// BAR-NEXT:     "spelling": " "
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "identifier",
+// BAR-NEXT:     "spelling": "Bar"
+// BAR-NEXT:   },
+// BAR-NEXT:   {
+// BAR-NEXT:     "kind": "text",
+// BAR-NEXT:     "spelling": ";"
+// BAR-NEXT:   }
+// BAR-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix VAR
+NestedTemplateTemplateParamPack<Bar, Bar> var;
+// VAR-LABEL: "!testLabel": "c:@var"
+// VAR:      "declarationFragments": [
+// VAR-NEXT:   {
+// VAR-NEXT:     "kind": "typeIdentifier",
+// VAR-NEXT:     "preciseIdentifier": "c:@ST>1#pt>1#t>1#T@NestedTemplateTemplateParamPack",
+// VAR-NEXT:     "spelling": "NestedTemplateTemplateParamPack"
+// VAR-NEXT:   },
+// VAR-NEXT:   {
+// VAR-NEXT:     "kind": "text",
+// VAR-NEXT:     "spelling": "<"
+// VAR-NEXT:   },
+// VAR-NEXT:   {
+// VAR-NEXT:     "kind": "typeIdentifier",
+// VAR-NEXT:     "preciseIdentifier": "c:@ST>1#t>1#T@Bar",
+// VAR-NEXT:     "spelling": "Bar"
+// VAR-NEXT:   },
+// VAR-NEXT:   {
+// VAR-NEXT:     "kind": "text",
+// VAR-NEXT:     "spelling": ", "
+// VAR-NEXT:   },
+// VAR-NEXT:   {
+// VAR-NEXT:     "kind": "typeIdentifier",
+// VAR-NEXT:     "preciseIdentifier": "c:@ST>1#t>1#T@Bar",
+// VAR-NEXT:     "spelling": "Bar"
+// VAR-NEXT:   },
+// VAR-NEXT:   {
+// VAR-NEXT:     "kind": "text",
+// VAR-NEXT:     "spelling": "> "
+// VAR-NEXT:   },
+// VAR-NEXT:   {
+// VAR-NEXT:     "kind": "identifier",
+// VAR-NEXT:     "spelling": "var"
+// VAR-NEXT:   },
+// VAR-NEXT:   {
+// VAR-NEXT:     "kind": "text",
+// VAR-NEXT:     "spelling": ";"
+// VAR-NEXT:   }
+// VAR-NEXT: ]
+
+// expected-no-diagnostics
-- 
GitLab


From b6fa78d54cf761b232b00222a39fbfe974cb8a16 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <r@artagnon.com>
Date: Fri, 17 May 2024 10:41:41 +0100
Subject: [PATCH 1101/1206] [LAA] refactor sortPtrAccesses (NFC) (#92256)

Use the destructuring syntax in C++ and llvm::enumerate to make
sortPtrAccesses a little more readable.
---
 llvm/lib/Analysis/LoopAccessAnalysis.cpp | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 4ba2e1522210..13dec3b1e1b0 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1621,9 +1621,8 @@ bool llvm::sortPtrAccesses(ArrayRef<Value *> VL, Type *ElemTy,
   auto Compare = llvm::less_first();
   std::set<DistOrdPair, decltype(Compare)> Offsets(Compare);
   Offsets.emplace(0, 0);
-  int Cnt = 1;
   bool IsConsecutive = true;
-  for (auto *Ptr : VL.drop_front()) {
+  for (auto [Idx, Ptr] : drop_begin(enumerate(VL))) {
     std::optional<int> Diff = getPointersDiff(ElemTy, Ptr0, ElemTy, Ptr, DL, SE,
                                               /*StrictCheck=*/true);
     if (!Diff)
@@ -1631,22 +1630,18 @@ bool llvm::sortPtrAccesses(ArrayRef<Value *> VL, Type *ElemTy,
 
     // Check if the pointer with the same offset is found.
     int64_t Offset = *Diff;
-    auto Res = Offsets.emplace(Offset, Cnt);
-    if (!Res.second)
+    auto [It, IsInserted] = Offsets.emplace(Offset, Idx);
+    if (!IsInserted)
       return false;
     // Consecutive order if the inserted element is the last one.
-    IsConsecutive = IsConsecutive && std::next(Res.first) == Offsets.end();
-    ++Cnt;
+    IsConsecutive &= std::next(It) == Offsets.end();
   }
   SortedIndices.clear();
   if (!IsConsecutive) {
     // Fill SortedIndices array only if it is non-consecutive.
     SortedIndices.resize(VL.size());
-    Cnt = 0;
-    for (const std::pair<int64_t, int> &Pair : Offsets) {
-      SortedIndices[Cnt] = Pair.second;
-      ++Cnt;
-    }
+    for (auto [Idx, Off] : enumerate(Offsets))
+      SortedIndices[Idx] = Off.second;
   }
   return true;
 }
@@ -2639,7 +2634,7 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
                                SymbolicStrides, UncomputablePtr, false);
   if (!CanDoRTIfNeeded) {
     auto *I = dyn_cast_or_null<Instruction>(UncomputablePtr);
-    recordAnalysis("CantIdentifyArrayBounds", I) 
+    recordAnalysis("CantIdentifyArrayBounds", I)
         << "cannot identify array bounds";
     LLVM_DEBUG(dbgs() << "LAA: We can't vectorize because we can't find "
                       << "the array bounds.\n");
-- 
GitLab


From 2ed8ff3bf84e682b7c6ae9ad6370db629166653c Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy@intel.com>
Date: Fri, 17 May 2024 11:42:37 +0200
Subject: [PATCH 1102/1206] [SPIR-V] Fix types of internal intrinsic functions
 and add a test case for __builtin_alloca() (#92265)

This PR generation of argument types of internal intrinsic functions
`spv_const_composite` and `spv_track_constant`, so that composite
constants of ConstantVector type preserve their correct type in
transformation passes and can be successfully used further by LLVM
intrinsic functions.

The added test case serves two purposes: it is to check the above
mentioned fix and to demonstrate that a call to __builtin_alloca() maps
to instructions from SPV_INTEL_variable_length_array when this extension
is available.
---
 llvm/docs/SPIRVUsage.rst                      |  2 +-
 llvm/include/llvm/IR/IntrinsicsSPIRV.td       |  2 +-
 llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 79 ++++++++++---------
 .../builtin_alloca.ll                         | 48 +++++++++++
 4 files changed, 93 insertions(+), 38 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/builtin_alloca.ll

diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index 589ee7646ce1..d27177a4541a 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -298,7 +298,7 @@ SPIR-V backend, along with their descriptions and argument details.
      - `[Type, Type, Any Integer]`
      - Inserts an element into an aggregate type at a specified index. Allows for building and modifying arrays and vectors.
    * - `int_spv_const_composite`
-     - 32-bit Integer
+     - Type
      - `[Vararg]`
      - Constructs a composite type from given elements. Key for creating arrays, structs, and vectors from individual components.
    * - `int_spv_bitcast`
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 931786ab9647..cc84decc4340 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -27,7 +27,7 @@ let TargetPrefix = "spv" in {
   def int_spv_insertv : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_any_ty, llvm_vararg_ty]>;
   def int_spv_extractelt : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_anyint_ty]>;
   def int_spv_insertelt : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_any_ty, llvm_anyint_ty]>;
-  def int_spv_const_composite : Intrinsic<[llvm_i32_ty], [llvm_vararg_ty]>;
+  def int_spv_const_composite : Intrinsic<[llvm_any_ty], [llvm_vararg_ty]>;
   def int_spv_bitcast : Intrinsic<[llvm_any_ty], [llvm_any_ty]>;
   def int_spv_ptrcast : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_metadata_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>;
   def int_spv_switch : Intrinsic<[], [llvm_any_ty, llvm_vararg_ty]>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index d90249f7c2e6..32df2403dfe5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -167,8 +167,9 @@ static bool isMemInstrToReplace(Instruction *I) {
          isa<ExtractValueInst>(I) || isa<AtomicCmpXchgInst>(I);
 }
 
-static bool isAggrToReplace(const Value *V) {
-  return isa<ConstantAggregate>(V) || isa<ConstantDataArray>(V) ||
+static bool isAggrConstForceInt32(const Value *V) {
+  return isa<ConstantArray>(V) || isa<ConstantStruct>(V) ||
+         isa<ConstantDataArray>(V) ||
          (isa<ConstantAggregateZero>(V) && !V->getType()->isVectorTy());
 }
 
@@ -576,36 +577,42 @@ void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
     assert(I);
     bool KeepInst = false;
     for (const auto &Op : I->operands()) {
-      auto BuildCompositeIntrinsic =
-          [](Constant *AggrC, ArrayRef<Value *> Args, Value *Op, Instruction *I,
-             IRBuilder<> &B, std::queue<Instruction *> &Worklist,
-             bool &KeepInst, SPIRVEmitIntrinsics &SEI) {
-            B.SetInsertPoint(I);
-            auto *CCI =
-                B.CreateIntrinsic(Intrinsic::spv_const_composite, {}, {Args});
-            Worklist.push(CCI);
-            I->replaceUsesOfWith(Op, CCI);
-            KeepInst = true;
-            SEI.AggrConsts[CCI] = AggrC;
-            SEI.AggrConstTypes[CCI] = SEI.deduceNestedTypeHelper(AggrC);
-          };
-
-      if (auto *AggrC = dyn_cast<ConstantAggregate>(Op)) {
-        SmallVector<Value *> Args(AggrC->op_begin(), AggrC->op_end());
-        BuildCompositeIntrinsic(AggrC, Args, Op, I, B, Worklist, KeepInst,
-                                *this);
-      } else if (auto *AggrC = dyn_cast<ConstantDataArray>(Op)) {
+      Constant *AggrConst = nullptr;
+      Type *ResTy = nullptr;
+      if (auto *COp = dyn_cast<ConstantVector>(Op)) {
+        AggrConst = cast<Constant>(COp);
+        ResTy = COp->getType();
+      } else if (auto *COp = dyn_cast<ConstantArray>(Op)) {
+        AggrConst = cast<Constant>(COp);
+        ResTy = B.getInt32Ty();
+      } else if (auto *COp = dyn_cast<ConstantStruct>(Op)) {
+        AggrConst = cast<Constant>(COp);
+        ResTy = B.getInt32Ty();
+      } else if (auto *COp = dyn_cast<ConstantDataArray>(Op)) {
+        AggrConst = cast<Constant>(COp);
+        ResTy = B.getInt32Ty();
+      } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Op)) {
+        if (!Op->getType()->isVectorTy()) {
+          AggrConst = cast<Constant>(COp);
+          ResTy = B.getInt32Ty();
+        }
+      }
+      if (AggrConst) {
         SmallVector<Value *> Args;
-        for (unsigned i = 0; i < AggrC->getNumElements(); ++i)
-          Args.push_back(AggrC->getElementAsConstant(i));
-        BuildCompositeIntrinsic(AggrC, Args, Op, I, B, Worklist, KeepInst,
-                                *this);
-      } else if (isa<ConstantAggregateZero>(Op) &&
-                 !Op->getType()->isVectorTy()) {
-        auto *AggrC = cast<ConstantAggregateZero>(Op);
-        SmallVector<Value *> Args(AggrC->op_begin(), AggrC->op_end());
-        BuildCompositeIntrinsic(AggrC, Args, Op, I, B, Worklist, KeepInst,
-                                *this);
+        if (auto *COp = dyn_cast<ConstantDataSequential>(Op))
+          for (unsigned i = 0; i < COp->getNumElements(); ++i)
+            Args.push_back(COp->getElementAsConstant(i));
+        else
+          for (auto &COp : AggrConst->operands())
+            Args.push_back(COp);
+        B.SetInsertPoint(I);
+        auto *CI =
+            B.CreateIntrinsic(Intrinsic::spv_const_composite, {ResTy}, {Args});
+        Worklist.push(CI);
+        I->replaceUsesOfWith(Op, CI);
+        KeepInst = true;
+        AggrConsts[CI] = AggrConst;
+        AggrConstTypes[CI] = deduceNestedTypeHelper(AggrConst);
       }
     }
     if (!KeepInst)
@@ -1054,8 +1061,8 @@ void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
     // by llvm IR general logic.
     deduceElementTypeHelper(&GV);
     Constant *Init = GV.getInitializer();
-    Type *Ty = isAggrToReplace(Init) ? B.getInt32Ty() : Init->getType();
-    Constant *Const = isAggrToReplace(Init) ? B.getInt32(1) : Init;
+    Type *Ty = isAggrConstForceInt32(Init) ? B.getInt32Ty() : Init->getType();
+    Constant *Const = isAggrConstForceInt32(Init) ? B.getInt32(1) : Init;
     auto *InitInst = B.CreateIntrinsic(Intrinsic::spv_init_global,
                                        {GV.getType(), Ty}, {&GV, Const});
     InitInst->setArgOperand(1, Init);
@@ -1132,11 +1139,11 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
   if (II && II->getIntrinsicID() == Intrinsic::spv_const_composite &&
       TrackConstants) {
     B.SetInsertPoint(I->getNextNode());
-    Type *Ty = B.getInt32Ty();
     auto t = AggrConsts.find(I);
     assert(t != AggrConsts.end());
-    auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant, {Ty, Ty},
-                                  t->second, I, {}, B);
+    auto *NewOp =
+        buildIntrWithMD(Intrinsic::spv_track_constant,
+                        {II->getType(), II->getType()}, t->second, I, {}, B);
     I->replaceAllUsesWith(NewOp);
     NewOp->setArgOperand(0, I);
   }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/builtin_alloca.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/builtin_alloca.ll
new file mode 100644
index 000000000000..4d6173e5b723
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/builtin_alloca.ll
@@ -0,0 +1,48 @@
+; The goal of the test is to:
+; 1) check that composite constants of ConstantVector type preserve their
+;    type and can be successfully used further in LLVM intrinsic functions;
+; 2) demonstrate that a call to __builtin_alloca() maps to instructions
+;    from SPV_INTEL_variable_length_array when this extension is available.
+
+; Test LLVM IR is an artificial example, but it's similar to what can be
+; generated by DPC++ compiler from the code snippet:
+;   ...
+;   size_t Sz = ...;
+;   queue Q;
+;   Q.submit([&](sycl::handler &CGH) {
+;     ...
+;     CGH.single_task([=](sycl::kernel_handler KH) SYCL_ESIMD_KERNEL {
+;       int *PrivateArray = (int *)__builtin_alloca(sizeof(int) * Sz);
+;       ...
+;       simd<int, 8> InitVec(100, 10);
+;       InitVec.copy_to(PrivateArray);
+;       ...
+;     });
+;   }).wait();
+;   ...
+
+; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_variable_length_array %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_variable_length_array %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-ERROR: LLVM ERROR: array allocation: this instruction requires the following SPIR-V extension: SPV_INTEL_variable_length_array
+
+; CHECK-SPIRV: Capability VariableLengthArrayINTEL
+; CHECK-SPIRV: Extension "SPV_INTEL_variable_length_array"
+; CHECK-SPIRV: OpVariableLengthArrayINTEL %[[#]] %[[#]]
+
+define spir_kernel void @foo(i64 %_arg_sz) {
+entry:
+  %sz = shl i64 %_arg_sz, 2
+  %p1 = alloca i8, i64 %sz, align 8
+  %p4 = addrspacecast ptr %p1 to ptr addrspace(4)
+  %i = ptrtoint ptr addrspace(4) %p4 to i64
+  %splat_ins = insertelement <8 x i64> poison, i64 %i, i64 0
+  %splat_v = shufflevector <8 x i64> %splat_ins, <8 x i64> poison, <8 x i32> zeroinitializer
+  %sum_r = add <8 x i64> %splat_v, <i64 0, i64 4, i64 8, i64 12, i64 16, i64 20, i64 24, i64 28>
+  call void @llvm.genx.svm.scatter.v8i1.v8i64.v8i32(<8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i32 0, <8 x i64> %sum_r, <8 x i32> <i32 100, i32 110, i32 120, i32 130, i32 140, i32 150, i32 160, i32 170>)
+  ret void
+}
+
+declare void @llvm.genx.svm.scatter.v8i1.v8i64.v8i32(<8 x i1>, i32, <8 x i64>, <8 x i32>)
-- 
GitLab


From e3e06135eb16a1b9d84796a6cbb14cac1c1cf543 Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy@intel.com>
Date: Fri, 17 May 2024 11:43:02 +0200
Subject: [PATCH 1103/1206] [SPIR-V] Ensure that we don't have a dangling
 BlockAddress constants after internal intrinsic 'spv_switch' is processed
 (#92390)

After internal intrinsic 'spv_switch' is processed we need to delete
G_BLOCK_ADDR instructions that were generated to keep track of the
corresponding basic blocks. If we just delete G_BLOCK_ADDR instructions
with BlockAddress operands, this leaves their BasicBlock counterparts in
a "address taken" status. This would make AsmPrinter to generate a
series of unneeded labels of a `"Address of block that was removed by
CodeGen"` kind. This PR is to ensure that we don't have a dangling
BlockAddress constants by zapping the BlockAddress nodes, and only after
that proceed with erasing G_BLOCK_ADDR instructions.

See also https://github.com/llvm/llvm-project/pull/87823 for more
details.
---
 llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp   | 19 ++++++++++++++++++-
 .../SPIRV/branching/switch-range-check.ll     |  3 +++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 6ee5d90d9afe..9bff23dd9666 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -602,8 +602,25 @@ static void processSwitches(MachineFunction &MF, SPIRVGlobalRegistry *GR,
         ToEraseMI.insert(Next);
     }
   }
-  for (MachineInstr *BlockAddrI : ToEraseMI)
+
+  // If we just delete G_BLOCK_ADDR instructions with BlockAddress operands,
+  // this leaves their BasicBlock counterparts in a "address taken" status. This
+  // would make AsmPrinter to generate a series of unneeded labels of a "Address
+  // of block that was removed by CodeGen" kind. Let's first ensure that we
+  // don't have a dangling BlockAddress constants by zapping the BlockAddress
+  // nodes, and only after that proceed with erasing G_BLOCK_ADDR instructions.
+  Constant *Replacement =
+      ConstantInt::get(Type::getInt32Ty(MF.getFunction().getContext()), 1);
+  for (MachineInstr *BlockAddrI : ToEraseMI) {
+    if (BlockAddrI->getOpcode() == TargetOpcode::G_BLOCK_ADDR) {
+      BlockAddress *BA = const_cast<BlockAddress *>(
+          BlockAddrI->getOperand(1).getBlockAddress());
+      BA->replaceAllUsesWith(
+          ConstantExpr::getIntToPtr(Replacement, BA->getType()));
+      BA->destroyConstant();
+    }
     BlockAddrI->eraseFromParent();
+  }
 }
 
 static bool isImplicitFallthrough(MachineBasicBlock &MBB) {
diff --git a/llvm/test/CodeGen/SPIRV/branching/switch-range-check.ll b/llvm/test/CodeGen/SPIRV/branching/switch-range-check.ll
index 85a4d4db089c..f8ce15323aac 100644
--- a/llvm/test/CodeGen/SPIRV/branching/switch-range-check.ll
+++ b/llvm/test/CodeGen/SPIRV/branching/switch-range-check.ll
@@ -1,10 +1,13 @@
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
+; CHECK: OpFunction
 ; CHECK: %[[#Var:]] = OpPhi
 ; CHECK: OpSwitch %[[#Var]] %[[#]] [[#]] %[[#]] [[#]] %[[#]] [[#]] %[[#]] [[#]] %[[#]] [[#]] %[[#]] [[#]] %[[#]] [[#]] %[[#]] [[#]] %[[#]] [[#]] %[[#]] [[#]] %[[#]] [[#]] %[[#]] [[#]] %[[#]]
 ; CHECK-COUNT-11: OpBranch
 ; CHECK-NOT: OpBranch
+; CHECK: OpReturn
+; CHECK-NEXT: OpFunctionEnd
 
 define spir_func void @foo(i64 noundef %addr, i64 noundef %as) {
 entry:
-- 
GitLab


From 7c16e7d9cd37d7d1162866c5ac29fe4ab2e24a5a Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Fri, 17 May 2024 10:53:29 +0100
Subject: [PATCH 1104/1206] [flang][NFC] Use tablegen to reduce AddAliasTags
 boilerplate (#92373)

I will make this pass run properly on more top-level operations in a
follow up patch.
---
 flang/include/flang/Optimizer/Transforms/Passes.h  | 2 +-
 flang/include/flang/Optimizer/Transforms/Passes.td | 1 -
 flang/include/flang/Tools/CLOptions.inc            | 2 +-
 flang/lib/Optimizer/Transforms/AddAliasTags.cpp    | 4 ----
 4 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index 25fe61488f4f..e40e2faed533 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -49,6 +49,7 @@ namespace fir {
 #define GEN_PASS_DECL_ADDDEBUGINFO
 #define GEN_PASS_DECL_STACKARRAYS
 #define GEN_PASS_DECL_LOOPVERSIONING
+#define GEN_PASS_DECL_ADDALIASTAGS
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 
 std::unique_ptr<mlir::Pass> createAffineDemotionPass();
@@ -60,7 +61,6 @@ std::unique_ptr<mlir::Pass>
 createExternalNameConversionPass(bool appendUnderscore);
 std::unique_ptr<mlir::Pass> createMemDataFlowOptPass();
 std::unique_ptr<mlir::Pass> createPromoteToAffinePass();
-std::unique_ptr<mlir::Pass> createAliasTagsPass();
 std::unique_ptr<mlir::Pass>
 createAddDebugInfoPass(fir::AddDebugInfoOptions options = {});
 
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 622c9465754c..28420a8b3f70 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -278,7 +278,6 @@ def AddAliasTags : Pass<"fir-add-alias-tags", "mlir::ModuleOp"> {
     aren't races adding new tags to the mlir context.
   }];
   let dependentDialects = [ "fir::FIROpsDialect" ];
-  let constructor = "::fir::createAliasTagsPass()";
 }
 
 def SimplifyRegionLite : Pass<"simplify-region-lite", "mlir::ModuleOp"> {
diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc
index 9c4a30dcf301..61e591f2086d 100644
--- a/flang/include/flang/Tools/CLOptions.inc
+++ b/flang/include/flang/Tools/CLOptions.inc
@@ -294,7 +294,7 @@ inline void createDefaultFIROptimizerPassPipeline(
   pm.addPass(fir::createPolymorphicOpConversion());
 
   if (pc.AliasAnalysis && !disableFirAliasTags && !useOldAliasTags)
-    pm.addPass(fir::createAliasTagsPass());
+    pm.addPass(fir::createAddAliasTags());
 
   // convert control flow to CFG form
   fir::addCfgConversionPass(pm, pc);
diff --git a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
index caece9384004..40df38e41760 100644
--- a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
+++ b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
@@ -227,7 +227,3 @@ void AddAliasTagsPass::runOnOperation() {
 
   LLVM_DEBUG(llvm::dbgs() << "=== End " DEBUG_TYPE " ===\n");
 }
-
-std::unique_ptr<mlir::Pass> fir::createAliasTagsPass() {
-  return std::make_unique<AddAliasTagsPass>();
-}
-- 
GitLab


From d0e2808f806ece6d2aa8d359a700afab87ded16b Mon Sep 17 00:00:00 2001
From: Shan Huang <52285902006@stu.ecnu.edu.cn>
Date: Fri, 17 May 2024 17:56:05 +0800
Subject: [PATCH 1105/1206] [DebugInfo][LoopLoadElim] Fix missing debug
 location updates (#91839)

---
 .../Transforms/Scalar/LoopLoadElimination.cpp | 12 ++-
 .../update-debugloc-store-forwarded.ll        | 85 +++++++++++++++++++
 2 files changed, 96 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/LoopLoadElim/update-debugloc-store-forwarded.ll

diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
index f611ef6b2fa2..38aea1371e1e 100644
--- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -448,6 +448,10 @@ public:
         new LoadInst(Cand.Load->getType(), InitialPtr, "load_initial",
                      /* isVolatile */ false, Cand.Load->getAlign(),
                      PH->getTerminator()->getIterator());
+    // We don't give any debug location to Initial, because it is inserted
+    // into the loop's preheader. A debug location inside the loop will cause
+    // a misleading stepping when debugging. The test update-debugloc-store
+    // -forwarded.ll checks this.
 
     PHINode *PHI = PHINode::Create(Initial->getType(), 2, "store_forwarded");
     PHI->insertBefore(L->getHeader()->begin());
@@ -462,14 +466,20 @@ public:
            "The type sizes should match!");
 
     Value *StoreValue = Cand.Store->getValueOperand();
-    if (LoadType != StoreType)
+    if (LoadType != StoreType) {
       StoreValue = CastInst::CreateBitOrPointerCast(StoreValue, LoadType,
                                                     "store_forward_cast",
                                                     Cand.Store->getIterator());
+      // Because it casts the old `load` value and is used by the new `phi`
+      // which replaces the old `load`, we give the `load`'s debug location
+      // to it.
+      cast<Instruction>(StoreValue)->setDebugLoc(Cand.Load->getDebugLoc());
+    }
 
     PHI->addIncoming(StoreValue, L->getLoopLatch());
 
     Cand.Load->replaceAllUsesWith(PHI);
+    PHI->setDebugLoc(Cand.Load->getDebugLoc());
   }
 
   /// Top-level driver for each loop: find store->load forwarding
diff --git a/llvm/test/Transforms/LoopLoadElim/update-debugloc-store-forwarded.ll b/llvm/test/Transforms/LoopLoadElim/update-debugloc-store-forwarded.ll
new file mode 100644
index 000000000000..6833a3fd23cb
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/update-debugloc-store-forwarded.ll
@@ -0,0 +1,85 @@
+; RUN: opt -passes=loop-load-elim -S < %s | FileCheck %s
+
+; LoopLoadElimination's propagateStoredValueToLoadUsers() replaces the
+; `load` (`%a`) with a hoisted initial `load` and a `phi` that forwards
+; the stored value.
+; This test checks that the debug location is propagated to the new `phi`
+; from the original `load` it replaces in block `%for.body` and the debug
+; location drop of the hoisted `load` in block `%entry`.
+; Moreover, this test also checks the debug location update of the new
+; `bitcast` created when the `load` type is mismatched with the `store` type:
+;   store i32 ...
+;   %a = load float, ...
+; Because the `bitcast` casts the old `load` value, it has the same debug
+; location as the old `load` (ie., the same as the new `phi`).
+
+; If the store and the load use different types, but have the same
+; size then we should still be able to forward the value.
+;
+;   for (unsigned i = 0; i < 100; i++) {
+;     A[i+1] = B[i] + 2;
+;     C[i] = ((float*)A)[i] * 2;
+;   }
+
+define void @f(ptr noalias %A, ptr noalias %B, ptr noalias %C, i64 %N) !dbg !5 {
+; CHECK-LABEL: define void @f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD_INITIAL:%.*]] = load float, ptr {{.*}}, align 4{{$}}
+; CHECK:       for.body:
+; CHECK-NEXT:    [[STORE_FORWARDED:%.*]] = phi float [ [[LOAD_INITIAL]], %entry ], [ [[STORE_FORWARD_CAST:%.*]], %for.body ], !dbg [[DBG9:![0-9]+]]
+; CHECK:         [[STORE_FORWARD_CAST]] = bitcast i32 {{.*}} to float, !dbg [[DBG9]]
+; CHECK:       [[DBG9]] = !DILocation(line: 11,
+;
+entry:
+  br label %for.body, !dbg !8
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ], !dbg !9
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !10
+  %Aidx_next = getelementptr inbounds i32, ptr %A, i64 %indvars.iv.next, !dbg !11
+  %Bidx = getelementptr inbounds i32, ptr %B, i64 %indvars.iv, !dbg !12
+  %Cidx = getelementptr inbounds i32, ptr %C, i64 %indvars.iv, !dbg !13
+  %Aidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv, !dbg !14
+  %b = load i32, ptr %Bidx, align 4, !dbg !15
+  %a_p1 = add i32 %b, 2, !dbg !16
+  store i32 %a_p1, ptr %Aidx_next, align 4, !dbg !17
+  %a = load float, ptr %Aidx, align 4, !dbg !18
+  %c = fmul float %a, 2.000000e+00, !dbg !19
+  %c.int = fptosi float %c to i32, !dbg !20
+  store i32 %c.int, ptr %Cidx, align 4, !dbg !21
+  %exitcond = icmp eq i64 %indvars.iv.next, %N, !dbg !22
+  br i1 %exitcond, label %for.end, label %for.body, !dbg !23
+
+for.end:                                          ; preds = %for.body
+  ret void, !dbg !24
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!2, !3}
+!llvm.module.flags = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "type-mismatch.ll", directory: "/")
+!2 = !{i32 17}
+!3 = !{i32 0}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "f", linkageName: "f", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !DILocation(line: 1, column: 1, scope: !5)
+!9 = !DILocation(line: 2, column: 1, scope: !5)
+!10 = !DILocation(line: 3, column: 1, scope: !5)
+!11 = !DILocation(line: 4, column: 1, scope: !5)
+!12 = !DILocation(line: 5, column: 1, scope: !5)
+!13 = !DILocation(line: 6, column: 1, scope: !5)
+!14 = !DILocation(line: 7, column: 1, scope: !5)
+!15 = !DILocation(line: 8, column: 1, scope: !5)
+!16 = !DILocation(line: 9, column: 1, scope: !5)
+!17 = !DILocation(line: 10, column: 1, scope: !5)
+!18 = !DILocation(line: 11, column: 1, scope: !5)
+!19 = !DILocation(line: 12, column: 1, scope: !5)
+!20 = !DILocation(line: 13, column: 1, scope: !5)
+!21 = !DILocation(line: 14, column: 1, scope: !5)
+!22 = !DILocation(line: 15, column: 1, scope: !5)
+!23 = !DILocation(line: 16, column: 1, scope: !5)
+!24 = !DILocation(line: 17, column: 1, scope: !5)
-- 
GitLab


From 698cf0176b3146993134354b02c9a67352d9d27e Mon Sep 17 00:00:00 2001
From: Johannes Reifferscheid <jreiffers@google.com>
Date: Fri, 17 May 2024 12:06:35 +0200
Subject: [PATCH 1106/1206] Fix i1 array global crash in NVPTXAsmPrinter.
 (#92506)

See the test file. At head, this crashes with

```
assertion failed at llvm/lib/Support/APInt.cpp:492 in
uint64_t llvm::APInt::extractBitsAsZExtValue(unsigned int, unsigned int) const:
  bitPosition < BitWidth && (numBits + bitPosition) <= BitWidth &&
  "Illegal bit extraction"
```
---
 llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp  |  6 +++++-
 llvm/test/CodeGen/NVPTX/i1-array-global.ll | 19 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/i1-array-global.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index 9f31b72bbceb..f8b246a1a36c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -1847,9 +1847,13 @@ void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
   auto AddIntToBuffer = [AggBuffer, Bytes](const APInt &Val) {
     size_t NumBytes = (Val.getBitWidth() + 7) / 8;
     SmallVector<unsigned char, 16> Buf(NumBytes);
-    for (unsigned I = 0; I < NumBytes; ++I) {
+    for (unsigned I = 0; I < NumBytes - 1; ++I) {
       Buf[I] = Val.extractBitsAsZExtValue(8, I * 8);
     }
+    size_t LastBytePosition = (NumBytes - 1) * 8;
+    size_t LastByteBits = Val.getBitWidth() - LastBytePosition;
+    Buf[NumBytes - 1] =
+        Val.extractBitsAsZExtValue(LastByteBits, LastBytePosition);
     AggBuffer->addBytes(Buf.data(), NumBytes, Bytes);
   };
 
diff --git a/llvm/test/CodeGen/NVPTX/i1-array-global.ll b/llvm/test/CodeGen/NVPTX/i1-array-global.ll
new file mode 100644
index 000000000000..a177f75a18dd
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/i1-array-global.ll
@@ -0,0 +1,19 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
+
+target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
+target triple = "nvptx-nvidia-cuda"
+
+@global_cst = private constant [6 x i1] [i1 true, i1 false, i1 true, i1 false, i1 true, i1 false]
+
+; CHECK: .global .align 1 .b8 global_cst[6] = {1, 0, 1, 0, 1}
+define void @kernel(i32 %i, ptr %out) {
+  %5 = getelementptr inbounds i1, ptr @global_cst, i32 %i
+  %6 = load i1, ptr %5, align 1
+  store i1 %6, ptr %out, align 1
+  ret void
+}
+
+!nvvm.annotations = !{!0}
+!0 = !{ptr @kernel, !"kernel", i32 1}
+
-- 
GitLab


From c93b45aaee62700f61ddc366d971bde594459324 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Fri, 17 May 2024 11:20:11 +0100
Subject: [PATCH 1107/1206] [mlir][ArmSME] Reword in-memory tile warning (NFC)
 (#92415)

It did not make sense that this said "all tile operations will go
through memory". Only the operations where the warning is emitted will
go through memory. The message has been updated to reflect that.
---
 mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp    |  4 ++--
 .../ArmSMEToLLVM/tile-spills-and-fills.mlir          |  4 ++--
 .../Linalg/CPU/ArmSME/use-too-many-tiles.mlir        | 12 ++++++------
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index 7e7706793674..4d96091a637c 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -248,8 +248,8 @@ struct ConvertArmSMESpillsAndFillsToLLVM : public ConvertToLLVMPattern {
       return failure();
 
     tileOp->emitWarning(
-        "failed to allocate SME virtual tile to operation, all tile "
-        "operations will go through memory, expect degraded performance");
+        "failed to allocate SME virtual tile to operation, tile value will go "
+        "through memory, expect degraded performance");
 
     // Step 1. Create an alloca for the tile at the top of the function (if one
     // does not already exist).
diff --git a/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir b/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
index 91c1b92b0122..1dced0fcd18c 100644
--- a/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
+++ b/mlir/test/Conversion/ArmSMEToLLVM/tile-spills-and-fills.mlir
@@ -54,7 +54,7 @@
 func.func @use_too_many_tiles() {
   %0 = arm_sme.zero : vector<[4]x[4]xi32>
   %1 = arm_sme.zero : vector<[4]x[4]xi32>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, tile value will go through memory, expect degraded performance}}
   %2 = arm_sme.zero : vector<[8]x[8]xi16>
   "test.some_use"(%0) : (vector<[4]x[4]xi32>) -> ()
   "test.some_use"(%1) : (vector<[4]x[4]xi32>) -> ()
@@ -138,7 +138,7 @@ func.func @very_excessive_spills(%useAllTiles : vector<[16]x[16]xi8>, %memref: m
   %c0 = arith.constant 0 : index
   %tile = arm_sme.get_tile : vector<[4]x[4]xf32>
   %mask = vector.constant_mask [4] : vector<[4]xi1>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, tile value will go through memory, expect degraded performance}}
   %loadSlice = arm_sme.load_tile_slice %memref[%c0, %c0], %mask, %tile, %c0 : memref<?x?xf32>, vector<[4]xi1>, vector<[4]x[4]xf32>
   "test.some_use"(%useAllTiles) : (vector<[16]x[16]xi8>) -> ()
   "test.some_use"(%loadSlice) : (vector<[4]x[4]xf32>) -> ()
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
index 14d9712e971a..0b84b5a71b39 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
@@ -13,11 +13,11 @@
 /// performance (hence the warning).
 func.func @use_too_many_tiles(%a: memref<?x?xi16>, %b:  memref<?x?xi16>, %c: memref<?x?xi16>) {
   %c0 = arith.constant 0 : index
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, tile value will go through memory, expect degraded performance}}
   %tile_a = arith.constant dense<0> : vector<[8]x[8]xi16>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, tile value will go through memory, expect degraded performance}}
   %tile_b = arith.constant dense<1> : vector<[8]x[8]xi16>
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, tile value will go through memory, expect degraded performance}}
   %tile_c = arm_sme.tile_load %a[%c0, %c0] : memref<?x?xi16>, vector<[8]x[8]xi16>
   %tile_d = arm_sme.tile_load %b[%c0, %c0] : memref<?x?xi16>, vector<[8]x[8]xi16>
   %tile_e = arm_sme.tile_load %c[%c0, %c0] : memref<?x?xi16>, vector<[8]x[8]xi16>
@@ -25,17 +25,17 @@ func.func @use_too_many_tiles(%a: memref<?x?xi16>, %b:  memref<?x?xi16>, %c: mem
   // CHECK-LABEL: tile_a:
   // CHECK-COUNT-8: ( 0, 0, 0, 0, 0, 0, 0, 0
   vector.print str "tile_a:\n"
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, tile value will go through memory, expect degraded performance}}
   vector.print %tile_a : vector<[8]x[8]xi16>
   // CHECK-LABEL: tile_b:
   // CHECK-COUNT-8: ( 1, 1, 1, 1, 1, 1, 1, 1
   vector.print str "tile_b:\n"
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, tile value will go through memory, expect degraded performance}}
   vector.print %tile_b : vector<[8]x[8]xi16>
   // CHECK-LABEL: tile_c:
   // CHECK-COUNT-8: ( 2, 2, 2, 2, 2, 2, 2, 2
   vector.print str "tile_c:\n"
-  // expected-warning @below {{failed to allocate SME virtual tile to operation, all tile operations will go through memory, expect degraded performance}}
+  // expected-warning @below {{failed to allocate SME virtual tile to operation, tile value will go through memory, expect degraded performance}}
   vector.print %tile_c : vector<[8]x[8]xi16>
   // CHECK-LABEL: tile_d:
   // CHECK-COUNT-8: ( 3, 3, 3, 3, 3, 3, 3, 3
-- 
GitLab


From bbe40b9e0a24605e0526475e643312839772294f Mon Sep 17 00:00:00 2001
From: Ivan Kosarev <ivan.kosarev@amd.com>
Date: Fri, 17 May 2024 13:20:33 +0300
Subject: [PATCH 1108/1206] [MCAsmStreamer] Do not crash on switching to
 sections of unknown types (#92380)

MCObjectStreamer already accepts unknown types.
---
 llvm/lib/MC/MCSectionELF.cpp                    | 3 +--
 llvm/test/MC/ELF/section-numeric-invalid-type.s | 6 +++---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/MC/MCSectionELF.cpp b/llvm/lib/MC/MCSectionELF.cpp
index b1efb839ba75..6bbdef3dce32 100644
--- a/llvm/lib/MC/MCSectionELF.cpp
+++ b/llvm/lib/MC/MCSectionELF.cpp
@@ -175,8 +175,7 @@ void MCSectionELF::printSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
   else if (Type == ELF::SHT_LLVM_LTO)
     OS << "llvm_lto";
   else
-    report_fatal_error("unsupported type 0x" + Twine::utohexstr(Type) +
-                       " for section " + getName());
+    OS << "0x" << Twine::utohexstr(Type);
 
   if (EntrySize) {
     assert(Flags & ELF::SHF_MERGE);
diff --git a/llvm/test/MC/ELF/section-numeric-invalid-type.s b/llvm/test/MC/ELF/section-numeric-invalid-type.s
index fefc364d9917..b707b51c5ecc 100644
--- a/llvm/test/MC/ELF/section-numeric-invalid-type.s
+++ b/llvm/test/MC/ELF/section-numeric-invalid-type.s
@@ -1,7 +1,7 @@
-// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux-gnu %s -o - \
+// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux-gnu %s \
 // RUN:   | llvm-readobj -S --symbols - | FileCheck --check-prefix=OBJ %s
 
-// RUN: not --crash llvm-mc -filetype=asm -triple=x86_64-pc-linux-gnu %s -o - 2>&1 \
+// RUN: llvm-mc -filetype=asm -triple=x86_64-pc-linux-gnu %s \
 // RUN:   | FileCheck --check-prefix=ASM %s
 
   .section .sec,"a",@0x7fffffff
@@ -11,4 +11,4 @@
 // OBJ-NEXT:   Type: Unknown (0x7FFFFFFF)
 // OBJ:      }
 
-// ASM: unsupported type 0x7fffffff for section .sec
+// ASM: .section .sec,"a",@0x7fffffff
-- 
GitLab


From 4349ffb3fa268dba180b7c65fdce20a0be60272b Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Fri, 17 May 2024 11:34:01 +0100
Subject: [PATCH 1109/1206] [SelectOpt] Add tests for not select conditions.
 NFC

---
 llvm/test/CodeGen/AArch64/selectopt-not.ll | 668 +++++++++++++++++++++
 1 file changed, 668 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/selectopt-not.ll

diff --git a/llvm/test/CodeGen/AArch64/selectopt-not.ll b/llvm/test/CodeGen/AArch64/selectopt-not.ll
new file mode 100644
index 000000000000..7a949d11c80d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/selectopt-not.ll
@@ -0,0 +1,668 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -select-optimize -mtriple=aarch64-linux-gnu -mcpu=neoverse-v2 -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64"
+
+define i32 @minloc1(ptr nocapture readonly %0, ptr nocapture readonly %1, ptr nocapture readonly %2) {
+; CHECK-LABEL: @minloc1(
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP0:%.*]], i64 40
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP0]], i64 64
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP0]], i64 80
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP0]], i64 88
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP1:%.*]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = add nsw i64 [[TMP14]], -1
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = shl i64 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP2:%.*]], align 4
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp slt i64 [[TMP9]], 1
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[DOTPREHEADER:%.*]], label [[DOTPREHEADER35_LR_PH:%.*]]
+; CHECK:       .preheader35.lr.ph:
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i64 0, [[TMP7]]
+; CHECK-NEXT:    br label [[DOTPREHEADER35:%.*]]
+; CHECK:       .preheader35:
+; CHECK-NEXT:    [[TMP22:%.*]] = phi i32 [ 2147483647, [[DOTPREHEADER35_LR_PH]] ], [ [[TMP30:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = phi i64 [ 0, [[DOTPREHEADER35_LR_PH]] ], [ [[IV_N:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[DOT045:%.*]] = phi i1 [ false, [[DOTPREHEADER35_LR_PH]] ], [ [[DOT2:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[DOTLCSSA364144:%.*]] = phi i32 [ 0, [[DOTPREHEADER35_LR_PH]] ], [ [[TMP29:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = mul nsw i64 [[TMP23]], [[TMP11]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP19]], i64 [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ne i32 [[TMP26]], [[TMP20]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp sge i32 [[TMP26]], [[TMP22]]
+; CHECK-NEXT:    [[DOTNOT33:%.*]] = and i1 [[DOT045]], [[TMP28]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[TMP27]], i1 true, i1 [[DOTNOT33]]
+; CHECK-NEXT:    [[TMP29]] = select i1 [[OR_COND]], i32 [[DOTLCSSA364144]], i32 1
+; CHECK-NEXT:    [[NOT_OR_COND:%.*]] = xor i1 [[OR_COND]], true
+; CHECK-NEXT:    [[DOT2]] = select i1 [[NOT_OR_COND]], i1 true, i1 [[DOT045]]
+; CHECK-NEXT:    [[TMP30]] = select i1 [[OR_COND]], i32 [[TMP22]], i32 [[TMP20]]
+; CHECK-NEXT:    [[IV_N]] = add nuw nsw i64 [[TMP23]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_N]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[DOTPREHEADER]], label [[DOTPREHEADER35]]
+; CHECK:       .preheader:
+; CHECK-NEXT:    [[DOTLCSSA3641_LCSSA:%.*]] = phi i32 [ 0, [[TMP3:%.*]] ], [ [[TMP29]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    ret i32 [[DOTLCSSA3641_LCSSA]]
+;
+  %4 = getelementptr i8, ptr %0, i64 40
+  %5 = load i64, ptr %4, align 8
+  %6 = getelementptr i8, ptr %0, i64 64
+  %7 = load i64, ptr %6, align 8
+  %8 = getelementptr i8, ptr %0, i64 80
+  %9 = load i64, ptr %8, align 8
+  %10 = getelementptr i8, ptr %0, i64 88
+  %11 = load i64, ptr %10, align 8
+  %12 = load ptr, ptr %0, align 8
+  %13 = load i32, ptr %1, align 4
+  %14 = sext i32 %13 to i64
+  %15 = add nsw i64 %14, -1
+  %16 = mul i64 %15, %5
+  %17 = getelementptr i8, ptr %12, i64 %16
+  %18 = shl i64 %7, 3
+  %19 = getelementptr i8, ptr %17, i64 %18
+  %20 = load i32, ptr %2, align 4
+  %.not = icmp slt i64 %9, 1
+  br i1 %.not, label %.preheader, label %.preheader35.lr.ph
+
+.preheader35.lr.ph:                               ; preds = %3
+  %21 = sub i64 0, %7
+  br label %.preheader35
+
+.preheader35:                                     ; preds = %.preheader35.lr.ph, %.preheader35
+  %22 = phi i32 [ 2147483647, %.preheader35.lr.ph ], [ %30, %.preheader35 ]
+  %23 = phi i64 [ 0, %.preheader35.lr.ph ], [ %iv.n, %.preheader35 ]
+  %.045 = phi i1 [ false, %.preheader35.lr.ph ], [ %.2, %.preheader35 ]
+  %.lcssa364144 = phi i32 [ 0, %.preheader35.lr.ph ], [ %29, %.preheader35 ]
+  %24 = mul nsw i64 %23, %11
+  %25 = getelementptr i8, ptr %19, i64 %24
+  %26 = load i32, ptr %25, align 4
+  %27 = icmp ne i32 %26, %20
+  %28 = icmp sge i32 %26, %22
+  %.not33 = and i1 %.045, %28
+  %or.cond = select i1 %27, i1 true, i1 %.not33
+  %29 = select i1 %or.cond, i32 %.lcssa364144, i32 1
+  %not.or.cond = xor i1 %or.cond, true
+  %.2 = select i1 %not.or.cond, i1 true, i1 %.045
+  %30 = select i1 %or.cond, i32 %22, i32 %20
+  %iv.n = add nuw nsw i64 %23, 1
+  %exitcond.not = icmp eq i64 %iv.n, %9
+  br i1 %exitcond.not, label %.preheader, label %.preheader35
+
+.preheader:                                       ; preds = %.preheader35, %3
+  %.lcssa3641.lcssa = phi i32 [ 0, %3 ], [ %29, %.preheader35 ]
+  ret i32 %.lcssa3641.lcssa
+}
+
+define i32 @minloc1_otherunusednot(ptr nocapture readonly %0, ptr nocapture readonly %1, ptr nocapture readonly %2) {
+; CHECK-LABEL: @minloc1_otherunusednot(
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP0:%.*]], i64 40
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP0]], i64 64
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP0]], i64 80
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP0]], i64 88
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP1:%.*]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = add nsw i64 [[TMP14]], -1
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = shl i64 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP2:%.*]], align 4
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp slt i64 [[TMP9]], 1
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[DOTPREHEADER:%.*]], label [[DOTPREHEADER35_LR_PH:%.*]]
+; CHECK:       .preheader35.lr.ph:
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i64 0, [[TMP7]]
+; CHECK-NEXT:    br label [[DOTPREHEADER35:%.*]]
+; CHECK:       .preheader35:
+; CHECK-NEXT:    [[TMP22:%.*]] = phi i32 [ 2147483647, [[DOTPREHEADER35_LR_PH]] ], [ [[TMP30:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = phi i64 [ 0, [[DOTPREHEADER35_LR_PH]] ], [ [[IV_N:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[DOT045:%.*]] = phi i1 [ false, [[DOTPREHEADER35_LR_PH]] ], [ [[DOT2:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[DOTLCSSA364144:%.*]] = phi i32 [ 0, [[DOTPREHEADER35_LR_PH]] ], [ [[TMP29:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = mul nsw i64 [[TMP23]], [[TMP11]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP19]], i64 [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ne i32 [[TMP26]], [[TMP20]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp sge i32 [[TMP26]], [[TMP22]]
+; CHECK-NEXT:    [[DOTNOT33:%.*]] = and i1 [[DOT045]], [[TMP28]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[TMP27]], i1 true, i1 [[DOTNOT33]]
+; CHECK-NEXT:    [[TMP29]] = select i1 [[OR_COND]], i32 [[DOTLCSSA364144]], i32 1
+; CHECK-NEXT:    [[DOT2]] = select i1 [[OR_COND]], i1 [[DOT045]], i1 true
+; CHECK-NEXT:    [[NOT_OR_COND:%.*]] = xor i1 [[OR_COND]], true
+; CHECK-NEXT:    [[TMP30]] = select i1 [[OR_COND]], i32 [[TMP22]], i32 [[TMP20]]
+; CHECK-NEXT:    [[IV_N]] = add nuw nsw i64 [[TMP23]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_N]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[DOTPREHEADER]], label [[DOTPREHEADER35]]
+; CHECK:       .preheader:
+; CHECK-NEXT:    [[DOTLCSSA3641_LCSSA:%.*]] = phi i32 [ 0, [[TMP3:%.*]] ], [ [[TMP29]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[P:%.*]] = phi i1 [ false, [[TMP3]] ], [ [[NOT_OR_COND]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[Q:%.*]] = select i1 [[P]], i32 [[DOTLCSSA3641_LCSSA]], i32 1
+; CHECK-NEXT:    ret i32 [[Q]]
+;
+  %4 = getelementptr i8, ptr %0, i64 40
+  %5 = load i64, ptr %4, align 8
+  %6 = getelementptr i8, ptr %0, i64 64
+  %7 = load i64, ptr %6, align 8
+  %8 = getelementptr i8, ptr %0, i64 80
+  %9 = load i64, ptr %8, align 8
+  %10 = getelementptr i8, ptr %0, i64 88
+  %11 = load i64, ptr %10, align 8
+  %12 = load ptr, ptr %0, align 8
+  %13 = load i32, ptr %1, align 4
+  %14 = sext i32 %13 to i64
+  %15 = add nsw i64 %14, -1
+  %16 = mul i64 %15, %5
+  %17 = getelementptr i8, ptr %12, i64 %16
+  %18 = shl i64 %7, 3
+  %19 = getelementptr i8, ptr %17, i64 %18
+  %20 = load i32, ptr %2, align 4
+  %.not = icmp slt i64 %9, 1
+  br i1 %.not, label %.preheader, label %.preheader35.lr.ph
+
+.preheader35.lr.ph:                               ; preds = %3
+  %21 = sub i64 0, %7
+  br label %.preheader35
+
+.preheader35:                                     ; preds = %.preheader35.lr.ph, %.preheader35
+  %22 = phi i32 [ 2147483647, %.preheader35.lr.ph ], [ %30, %.preheader35 ]
+  %23 = phi i64 [ 0, %.preheader35.lr.ph ], [ %iv.n, %.preheader35 ]
+  %.045 = phi i1 [ false, %.preheader35.lr.ph ], [ %.2, %.preheader35 ]
+  %.lcssa364144 = phi i32 [ 0, %.preheader35.lr.ph ], [ %29, %.preheader35 ]
+  %24 = mul nsw i64 %23, %11
+  %25 = getelementptr i8, ptr %19, i64 %24
+  %26 = load i32, ptr %25, align 4
+  %27 = icmp ne i32 %26, %20
+  %28 = icmp sge i32 %26, %22
+  %.not33 = and i1 %.045, %28
+  %or.cond = select i1 %27, i1 true, i1 %.not33
+  %29 = select i1 %or.cond, i32 %.lcssa364144, i32 1
+  %.2 = select i1 %or.cond, i1 %.045, i1 true
+  %not.or.cond = xor i1 %or.cond, true
+  %30 = select i1 %or.cond, i32 %22, i32 %20
+  %iv.n = add nuw nsw i64 %23, 1
+  %exitcond.not = icmp eq i64 %iv.n, %9
+  br i1 %exitcond.not, label %.preheader, label %.preheader35
+
+.preheader:                                       ; preds = %.preheader35, %3
+  %.lcssa3641.lcssa = phi i32 [ 0, %3 ], [ %29, %.preheader35 ]
+  %p = phi i1 [ false, %3 ], [ %not.or.cond, %.preheader35 ]
+  %q = select i1 %p, i32 %.lcssa3641.lcssa, i32 1
+  ret i32 %q
+}
+
+define i32 @minloc1_twonot(ptr nocapture readonly %0, ptr nocapture readonly %1, ptr nocapture readonly %2) {
+; CHECK-LABEL: @minloc1_twonot(
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP0:%.*]], i64 40
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP0]], i64 64
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP0]], i64 80
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP0]], i64 88
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP1:%.*]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = add nsw i64 [[TMP14]], -1
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = shl i64 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP2:%.*]], align 4
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp slt i64 [[TMP9]], 1
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[DOTPREHEADER:%.*]], label [[DOTPREHEADER35_LR_PH:%.*]]
+; CHECK:       .preheader35.lr.ph:
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i64 0, [[TMP7]]
+; CHECK-NEXT:    br label [[DOTPREHEADER35:%.*]]
+; CHECK:       .preheader35:
+; CHECK-NEXT:    [[TMP22:%.*]] = phi i32 [ 2147483647, [[DOTPREHEADER35_LR_PH]] ], [ [[TMP30:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = phi i64 [ 0, [[DOTPREHEADER35_LR_PH]] ], [ [[IV_N:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[DOT045:%.*]] = phi i1 [ false, [[DOTPREHEADER35_LR_PH]] ], [ [[DOT3:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[DOTLCSSA364144:%.*]] = phi i32 [ 0, [[DOTPREHEADER35_LR_PH]] ], [ [[TMP29:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = mul nsw i64 [[TMP23]], [[TMP11]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP19]], i64 [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ne i32 [[TMP26]], [[TMP20]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp sge i32 [[TMP26]], [[TMP22]]
+; CHECK-NEXT:    [[DOTNOT33:%.*]] = and i1 [[DOT045]], [[TMP28]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[TMP27]], i1 true, i1 [[DOTNOT33]]
+; CHECK-NEXT:    [[TMP29]] = select i1 [[OR_COND]], i32 [[DOTLCSSA364144]], i32 1
+; CHECK-NEXT:    [[NOT_OR_COND:%.*]] = xor i1 [[OR_COND]], true
+; CHECK-NEXT:    [[DOT2:%.*]] = select i1 [[NOT_OR_COND]], i1 true, i1 [[DOT045]]
+; CHECK-NEXT:    [[DOT3]] = select i1 [[NOT_OR_COND]], i1 true, i1 [[DOT2]]
+; CHECK-NEXT:    [[TMP30]] = select i1 [[OR_COND]], i32 [[TMP22]], i32 [[TMP20]]
+; CHECK-NEXT:    [[IV_N]] = add nuw nsw i64 [[TMP23]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_N]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[DOTPREHEADER]], label [[DOTPREHEADER35]]
+; CHECK:       .preheader:
+; CHECK-NEXT:    [[DOTLCSSA3641_LCSSA:%.*]] = phi i32 [ 0, [[TMP3:%.*]] ], [ [[TMP29]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    ret i32 [[DOTLCSSA3641_LCSSA]]
+;
+  %4 = getelementptr i8, ptr %0, i64 40
+  %5 = load i64, ptr %4, align 8
+  %6 = getelementptr i8, ptr %0, i64 64
+  %7 = load i64, ptr %6, align 8
+  %8 = getelementptr i8, ptr %0, i64 80
+  %9 = load i64, ptr %8, align 8
+  %10 = getelementptr i8, ptr %0, i64 88
+  %11 = load i64, ptr %10, align 8
+  %12 = load ptr, ptr %0, align 8
+  %13 = load i32, ptr %1, align 4
+  %14 = sext i32 %13 to i64
+  %15 = add nsw i64 %14, -1
+  %16 = mul i64 %15, %5
+  %17 = getelementptr i8, ptr %12, i64 %16
+  %18 = shl i64 %7, 3
+  %19 = getelementptr i8, ptr %17, i64 %18
+  %20 = load i32, ptr %2, align 4
+  %.not = icmp slt i64 %9, 1
+  br i1 %.not, label %.preheader, label %.preheader35.lr.ph
+
+.preheader35.lr.ph:                               ; preds = %3
+  %21 = sub i64 0, %7
+  br label %.preheader35
+
+.preheader35:                                     ; preds = %.preheader35.lr.ph, %.preheader35
+  %22 = phi i32 [ 2147483647, %.preheader35.lr.ph ], [ %30, %.preheader35 ]
+  %23 = phi i64 [ 0, %.preheader35.lr.ph ], [ %iv.n, %.preheader35 ]
+  %.045 = phi i1 [ false, %.preheader35.lr.ph ], [ %.3, %.preheader35 ]
+  %.lcssa364144 = phi i32 [ 0, %.preheader35.lr.ph ], [ %29, %.preheader35 ]
+  %24 = mul nsw i64 %23, %11
+  %25 = getelementptr i8, ptr %19, i64 %24
+  %26 = load i32, ptr %25, align 4
+  %27 = icmp ne i32 %26, %20
+  %28 = icmp sge i32 %26, %22
+  %.not33 = and i1 %.045, %28
+  %or.cond = select i1 %27, i1 true, i1 %.not33
+  %29 = select i1 %or.cond, i32 %.lcssa364144, i32 1
+  %not.or.cond = xor i1 %or.cond, true
+  %.2 = select i1 %not.or.cond, i1 true, i1 %.045
+  %.3 = select i1 %not.or.cond, i1 true, i1 %.2
+  %30 = select i1 %or.cond, i32 %22, i32 %20
+  %iv.n = add nuw nsw i64 %23, 1
+  %exitcond.not = icmp eq i64 %iv.n, %9
+  br i1 %exitcond.not, label %.preheader, label %.preheader35
+
+.preheader:                                       ; preds = %.preheader35, %3
+  %.lcssa3641.lcssa = phi i32 [ 0, %3 ], [ %29, %.preheader35 ]
+  ret i32 %.lcssa3641.lcssa
+}
+
+define i32 @minloc1_onenotdependent(ptr nocapture readonly %0, ptr nocapture readonly %1, ptr nocapture readonly %2) {
+; CHECK-LABEL: @minloc1_onenotdependent(
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP0:%.*]], i64 40
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP0]], i64 64
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP0]], i64 80
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP0]], i64 88
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP1:%.*]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = add nsw i64 [[TMP14]], -1
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = shl i64 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP2:%.*]], align 4
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp slt i64 [[TMP9]], 1
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[DOTPREHEADER:%.*]], label [[DOTPREHEADER35_LR_PH:%.*]]
+; CHECK:       .preheader35.lr.ph:
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i64 0, [[TMP7]]
+; CHECK-NEXT:    br label [[DOTPREHEADER35:%.*]]
+; CHECK:       .preheader35:
+; CHECK-NEXT:    [[TMP22:%.*]] = phi i32 [ 2147483647, [[DOTPREHEADER35_LR_PH]] ], [ [[TMP30:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = phi i64 [ 0, [[DOTPREHEADER35_LR_PH]] ], [ [[IV_N:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[DOT045:%.*]] = phi i1 [ false, [[DOTPREHEADER35_LR_PH]] ], [ [[DOT3:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[DOTLCSSA364144:%.*]] = phi i32 [ 0, [[DOTPREHEADER35_LR_PH]] ], [ [[TMP29:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = mul nsw i64 [[TMP23]], [[TMP11]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP19]], i64 [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ne i32 [[TMP26]], [[TMP20]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp sge i32 [[TMP26]], [[TMP22]]
+; CHECK-NEXT:    [[DOTNOT33:%.*]] = and i1 [[DOT045]], [[TMP28]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[TMP27]], i1 true, i1 [[DOTNOT33]]
+; CHECK-NEXT:    [[TMP29]] = select i1 [[OR_COND]], i32 [[DOTLCSSA364144]], i32 1
+; CHECK-NEXT:    [[NOT_OR_COND:%.*]] = xor i1 [[OR_COND]], true
+; CHECK-NEXT:    [[DOT2:%.*]] = select i1 [[OR_COND]], i1 true, i1 [[DOT045]]
+; CHECK-NEXT:    [[DOT3]] = select i1 [[NOT_OR_COND]], i1 true, i1 [[DOT2]]
+; CHECK-NEXT:    [[TMP30]] = select i1 [[OR_COND]], i32 [[TMP22]], i32 [[TMP20]]
+; CHECK-NEXT:    [[IV_N]] = add nuw nsw i64 [[TMP23]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_N]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[DOTPREHEADER]], label [[DOTPREHEADER35]]
+; CHECK:       .preheader:
+; CHECK-NEXT:    [[DOTLCSSA3641_LCSSA:%.*]] = phi i32 [ 0, [[TMP3:%.*]] ], [ [[TMP29]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    ret i32 [[DOTLCSSA3641_LCSSA]]
+;
+  %4 = getelementptr i8, ptr %0, i64 40
+  %5 = load i64, ptr %4, align 8
+  %6 = getelementptr i8, ptr %0, i64 64
+  %7 = load i64, ptr %6, align 8
+  %8 = getelementptr i8, ptr %0, i64 80
+  %9 = load i64, ptr %8, align 8
+  %10 = getelementptr i8, ptr %0, i64 88
+  %11 = load i64, ptr %10, align 8
+  %12 = load ptr, ptr %0, align 8
+  %13 = load i32, ptr %1, align 4
+  %14 = sext i32 %13 to i64
+  %15 = add nsw i64 %14, -1
+  %16 = mul i64 %15, %5
+  %17 = getelementptr i8, ptr %12, i64 %16
+  %18 = shl i64 %7, 3
+  %19 = getelementptr i8, ptr %17, i64 %18
+  %20 = load i32, ptr %2, align 4
+  %.not = icmp slt i64 %9, 1
+  br i1 %.not, label %.preheader, label %.preheader35.lr.ph
+
+.preheader35.lr.ph:                               ; preds = %3
+  %21 = sub i64 0, %7
+  br label %.preheader35
+
+.preheader35:                                     ; preds = %.preheader35.lr.ph, %.preheader35
+  %22 = phi i32 [ 2147483647, %.preheader35.lr.ph ], [ %30, %.preheader35 ]
+  %23 = phi i64 [ 0, %.preheader35.lr.ph ], [ %iv.n, %.preheader35 ]
+  %.045 = phi i1 [ false, %.preheader35.lr.ph ], [ %.3, %.preheader35 ]
+  %.lcssa364144 = phi i32 [ 0, %.preheader35.lr.ph ], [ %29, %.preheader35 ]
+  %24 = mul nsw i64 %23, %11
+  %25 = getelementptr i8, ptr %19, i64 %24
+  %26 = load i32, ptr %25, align 4
+  %27 = icmp ne i32 %26, %20
+  %28 = icmp sge i32 %26, %22
+  %.not33 = and i1 %.045, %28
+  %or.cond = select i1 %27, i1 true, i1 %.not33
+  %29 = select i1 %or.cond, i32 %.lcssa364144, i32 1
+  %not.or.cond = xor i1 %or.cond, true
+  %.2 = select i1 %or.cond, i1 true, i1 %.045
+  %.3 = select i1 %not.or.cond, i1 true, i1 %.2
+  %30 = select i1 %or.cond, i32 %22, i32 %20
+  %iv.n = add nuw nsw i64 %23, 1
+  %exitcond.not = icmp eq i64 %iv.n, %9
+  br i1 %exitcond.not, label %.preheader, label %.preheader35
+
+.preheader:                                       ; preds = %.preheader35, %3
+  %.lcssa3641.lcssa = phi i32 [ 0, %3 ], [ %29, %.preheader35 ]
+  ret i32 %.lcssa3641.lcssa
+}
+
+
+define i32 @minloc9(ptr nocapture readonly %0, ptr nocapture readonly %1, ptr nocapture readonly %2) {
+; CHECK-LABEL: @minloc9(
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP0:%.*]], i64 40
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP0]], i64 64
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP0]], i64 80
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP0]], i64 88
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP1:%.*]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = add nsw i64 [[TMP14]], -1
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = shl i64 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP2:%.*]], align 4
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp slt i64 [[TMP9]], 1
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[DOTPREHEADER:%.*]], label [[DOTPREHEADER35_LR_PH:%.*]]
+; CHECK:       .preheader35.lr.ph:
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i64 0, [[TMP7]]
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul i64 [[TMP7]], -2
+; CHECK-NEXT:    [[DOTNEG50:%.*]] = mul i64 [[TMP7]], -3
+; CHECK-NEXT:    [[DOTNEG51:%.*]] = mul i64 [[TMP7]], -4
+; CHECK-NEXT:    [[DOTNEG52:%.*]] = mul i64 [[TMP7]], -5
+; CHECK-NEXT:    [[DOTNEG53:%.*]] = mul i64 [[TMP7]], -6
+; CHECK-NEXT:    [[DOTNEG54:%.*]] = mul i64 [[TMP7]], -7
+; CHECK-NEXT:    [[DOTNEG55:%.*]] = mul i64 [[TMP7]], -8
+; CHECK-NEXT:    br label [[DOTPREHEADER35:%.*]]
+; CHECK:       .preheader35:
+; CHECK-NEXT:    [[TMP22:%.*]] = phi i32 [ 2147483647, [[DOTPREHEADER35_LR_PH]] ], [ [[TMP78:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = phi i64 [ 0, [[DOTPREHEADER35_LR_PH]] ], [ [[TMP79:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[DOT045:%.*]] = phi i1 [ false, [[DOTPREHEADER35_LR_PH]] ], [ [[DOT2_8:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[DOTLCSSA364144:%.*]] = phi i32 [ 0, [[DOTPREHEADER35_LR_PH]] ], [ [[TMP77:%.*]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = mul nsw i64 [[TMP23]], [[TMP11]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP19]], i64 [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ne i32 [[TMP26]], [[TMP20]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp sge i32 [[TMP26]], [[TMP22]]
+; CHECK-NEXT:    [[DOTNOT33:%.*]] = and i1 [[DOT045]], [[TMP28]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[TMP27]], i1 true, i1 [[DOTNOT33]]
+; CHECK-NEXT:    [[TMP29:%.*]] = select i1 [[OR_COND]], i32 [[DOTLCSSA364144]], i32 1
+; CHECK-NEXT:    [[NOT_OR_COND:%.*]] = xor i1 [[OR_COND]], true
+; CHECK-NEXT:    [[DOT2:%.*]] = select i1 [[NOT_OR_COND]], i1 true, i1 [[DOT045]]
+; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[OR_COND]], i32 [[TMP22]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP25]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne i32 [[TMP32]], [[TMP20]]
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp sge i32 [[TMP32]], [[TMP30]]
+; CHECK-NEXT:    [[DOTNOT33_1:%.*]] = and i1 [[DOT2]], [[TMP34]]
+; CHECK-NEXT:    [[OR_COND_1:%.*]] = select i1 [[TMP33]], i1 true, i1 [[DOTNOT33_1]]
+; CHECK-NEXT:    [[TMP35:%.*]] = select i1 [[OR_COND_1]], i32 [[TMP29]], i32 2
+; CHECK-NEXT:    [[NOT_OR_COND_1:%.*]] = xor i1 [[OR_COND_1]], true
+; CHECK-NEXT:    [[DOT2_1:%.*]] = select i1 [[NOT_OR_COND_1]], i1 true, i1 [[DOT2]]
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[OR_COND_1]], i32 [[TMP30]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr i8, ptr [[TMP25]], i64 [[DOTNEG]]
+; CHECK-NEXT:    [[TMP38:%.*]] = load i32, ptr [[TMP37]], align 4
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp ne i32 [[TMP38]], [[TMP20]]
+; CHECK-NEXT:    [[TMP40:%.*]] = icmp sge i32 [[TMP38]], [[TMP36]]
+; CHECK-NEXT:    [[DOTNOT33_2:%.*]] = and i1 [[DOT2_1]], [[TMP40]]
+; CHECK-NEXT:    [[OR_COND_2:%.*]] = select i1 [[TMP39]], i1 true, i1 [[DOTNOT33_2]]
+; CHECK-NEXT:    [[TMP41:%.*]] = select i1 [[OR_COND_2]], i32 [[TMP35]], i32 3
+; CHECK-NEXT:    [[NOT_OR_COND_2:%.*]] = xor i1 [[OR_COND_2]], true
+; CHECK-NEXT:    [[DOT2_2:%.*]] = select i1 [[NOT_OR_COND_2]], i1 true, i1 [[DOT2_1]]
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[OR_COND_2]], i32 [[TMP36]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr i8, ptr [[TMP25]], i64 [[DOTNEG50]]
+; CHECK-NEXT:    [[TMP44:%.*]] = load i32, ptr [[TMP43]], align 4
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne i32 [[TMP44]], [[TMP20]]
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp sge i32 [[TMP44]], [[TMP42]]
+; CHECK-NEXT:    [[DOTNOT33_3:%.*]] = and i1 [[DOT2_2]], [[TMP46]]
+; CHECK-NEXT:    [[OR_COND_3:%.*]] = select i1 [[TMP45]], i1 true, i1 [[DOTNOT33_3]]
+; CHECK-NEXT:    [[TMP47:%.*]] = select i1 [[OR_COND_3]], i32 [[TMP41]], i32 4
+; CHECK-NEXT:    [[NOT_OR_COND_3:%.*]] = xor i1 [[OR_COND_3]], true
+; CHECK-NEXT:    [[DOT2_3:%.*]] = select i1 [[NOT_OR_COND_3]], i1 true, i1 [[DOT2_2]]
+; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[OR_COND_3]], i32 [[TMP42]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr i8, ptr [[TMP25]], i64 [[DOTNEG51]]
+; CHECK-NEXT:    [[TMP50:%.*]] = load i32, ptr [[TMP49]], align 4
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp ne i32 [[TMP50]], [[TMP20]]
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp sge i32 [[TMP50]], [[TMP48]]
+; CHECK-NEXT:    [[DOTNOT33_4:%.*]] = and i1 [[DOT2_3]], [[TMP52]]
+; CHECK-NEXT:    [[OR_COND_4:%.*]] = select i1 [[TMP51]], i1 true, i1 [[DOTNOT33_4]]
+; CHECK-NEXT:    [[TMP53:%.*]] = select i1 [[OR_COND_4]], i32 [[TMP47]], i32 5
+; CHECK-NEXT:    [[NOT_OR_COND_4:%.*]] = xor i1 [[OR_COND_4]], true
+; CHECK-NEXT:    [[DOT2_4:%.*]] = select i1 [[NOT_OR_COND_4]], i1 true, i1 [[DOT2_3]]
+; CHECK-NEXT:    [[TMP54:%.*]] = select i1 [[OR_COND_4]], i32 [[TMP48]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP55:%.*]] = getelementptr i8, ptr [[TMP25]], i64 [[DOTNEG52]]
+; CHECK-NEXT:    [[TMP56:%.*]] = load i32, ptr [[TMP55]], align 4
+; CHECK-NEXT:    [[TMP57:%.*]] = icmp ne i32 [[TMP56]], [[TMP20]]
+; CHECK-NEXT:    [[TMP58:%.*]] = icmp sge i32 [[TMP56]], [[TMP54]]
+; CHECK-NEXT:    [[DOTNOT33_5:%.*]] = and i1 [[DOT2_4]], [[TMP58]]
+; CHECK-NEXT:    [[OR_COND_5:%.*]] = select i1 [[TMP57]], i1 true, i1 [[DOTNOT33_5]]
+; CHECK-NEXT:    [[TMP59:%.*]] = select i1 [[OR_COND_5]], i32 [[TMP53]], i32 6
+; CHECK-NEXT:    [[NOT_OR_COND_5:%.*]] = xor i1 [[OR_COND_5]], true
+; CHECK-NEXT:    [[DOT2_5:%.*]] = select i1 [[NOT_OR_COND_5]], i1 true, i1 [[DOT2_4]]
+; CHECK-NEXT:    [[TMP60:%.*]] = select i1 [[OR_COND_5]], i32 [[TMP54]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP61:%.*]] = getelementptr i8, ptr [[TMP25]], i64 [[DOTNEG53]]
+; CHECK-NEXT:    [[TMP62:%.*]] = load i32, ptr [[TMP61]], align 4
+; CHECK-NEXT:    [[TMP63:%.*]] = icmp ne i32 [[TMP62]], [[TMP20]]
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp sge i32 [[TMP62]], [[TMP60]]
+; CHECK-NEXT:    [[DOTNOT33_6:%.*]] = and i1 [[DOT2_5]], [[TMP64]]
+; CHECK-NEXT:    [[OR_COND_6:%.*]] = select i1 [[TMP63]], i1 true, i1 [[DOTNOT33_6]]
+; CHECK-NEXT:    [[TMP65:%.*]] = select i1 [[OR_COND_6]], i32 [[TMP59]], i32 7
+; CHECK-NEXT:    [[NOT_OR_COND_6:%.*]] = xor i1 [[OR_COND_6]], true
+; CHECK-NEXT:    [[DOT2_6:%.*]] = select i1 [[NOT_OR_COND_6]], i1 true, i1 [[DOT2_5]]
+; CHECK-NEXT:    [[TMP66:%.*]] = select i1 [[OR_COND_6]], i32 [[TMP60]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr i8, ptr [[TMP25]], i64 [[DOTNEG54]]
+; CHECK-NEXT:    [[TMP68:%.*]] = load i32, ptr [[TMP67]], align 4
+; CHECK-NEXT:    [[TMP69:%.*]] = icmp ne i32 [[TMP68]], [[TMP20]]
+; CHECK-NEXT:    [[TMP70:%.*]] = icmp sge i32 [[TMP68]], [[TMP66]]
+; CHECK-NEXT:    [[DOTNOT33_7:%.*]] = and i1 [[DOT2_6]], [[TMP70]]
+; CHECK-NEXT:    [[OR_COND_7:%.*]] = select i1 [[TMP69]], i1 true, i1 [[DOTNOT33_7]]
+; CHECK-NEXT:    [[TMP71:%.*]] = select i1 [[OR_COND_7]], i32 [[TMP65]], i32 8
+; CHECK-NEXT:    [[NOT_OR_COND_7:%.*]] = xor i1 [[OR_COND_7]], true
+; CHECK-NEXT:    [[DOT2_7:%.*]] = select i1 [[NOT_OR_COND_7]], i1 true, i1 [[DOT2_6]]
+; CHECK-NEXT:    [[TMP72:%.*]] = select i1 [[OR_COND_7]], i32 [[TMP66]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP73:%.*]] = getelementptr i8, ptr [[TMP25]], i64 [[DOTNEG55]]
+; CHECK-NEXT:    [[TMP74:%.*]] = load i32, ptr [[TMP73]], align 4
+; CHECK-NEXT:    [[TMP75:%.*]] = icmp ne i32 [[TMP74]], [[TMP20]]
+; CHECK-NEXT:    [[TMP76:%.*]] = icmp sge i32 [[TMP74]], [[TMP72]]
+; CHECK-NEXT:    [[DOTNOT33_8:%.*]] = and i1 [[DOT2_7]], [[TMP76]]
+; CHECK-NEXT:    [[OR_COND_8:%.*]] = select i1 [[TMP75]], i1 true, i1 [[DOTNOT33_8]]
+; CHECK-NEXT:    [[TMP77]] = select i1 [[OR_COND_8]], i32 [[TMP71]], i32 9
+; CHECK-NEXT:    [[NOT_OR_COND_8:%.*]] = xor i1 [[OR_COND_8]], true
+; CHECK-NEXT:    [[DOT2_8]] = select i1 [[NOT_OR_COND_8]], i1 true, i1 [[DOT2_7]]
+; CHECK-NEXT:    [[TMP78]] = select i1 [[OR_COND_8]], i32 [[TMP72]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP79]] = add nuw nsw i64 [[TMP23]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[TMP79]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[DOTPREHEADER]], label [[DOTPREHEADER35]]
+; CHECK:       .preheader:
+; CHECK-NEXT:    [[DOTLCSSA3641_LCSSA:%.*]] = phi i32 [ 0, [[TMP3:%.*]] ], [ [[TMP77]], [[DOTPREHEADER35]] ]
+; CHECK-NEXT:    ret i32 [[DOTLCSSA3641_LCSSA]]
+;
+  %4 = getelementptr i8, ptr %0, i64 40
+  %5 = load i64, ptr %4, align 8
+  %6 = getelementptr i8, ptr %0, i64 64
+  %7 = load i64, ptr %6, align 8
+  %8 = getelementptr i8, ptr %0, i64 80
+  %9 = load i64, ptr %8, align 8
+  %10 = getelementptr i8, ptr %0, i64 88
+  %11 = load i64, ptr %10, align 8
+  %12 = load ptr, ptr %0, align 8
+  %13 = load i32, ptr %1, align 4
+  %14 = sext i32 %13 to i64
+  %15 = add nsw i64 %14, -1
+  %16 = mul i64 %15, %5
+  %17 = getelementptr i8, ptr %12, i64 %16
+  %18 = shl i64 %7, 3
+  %19 = getelementptr i8, ptr %17, i64 %18
+  %20 = load i32, ptr %2, align 4
+  %.not = icmp slt i64 %9, 1
+  br i1 %.not, label %.preheader, label %.preheader35.lr.ph
+
+.preheader35.lr.ph:                               ; preds = %3
+  %21 = sub i64 0, %7
+  %.neg = mul i64 %7, -2
+  %.neg50 = mul i64 %7, -3
+  %.neg51 = mul i64 %7, -4
+  %.neg52 = mul i64 %7, -5
+  %.neg53 = mul i64 %7, -6
+  %.neg54 = mul i64 %7, -7
+  %.neg55 = mul i64 %7, -8
+  br label %.preheader35
+
+.preheader35:                                     ; preds = %.preheader35.lr.ph, %.preheader35
+  %22 = phi i32 [ 2147483647, %.preheader35.lr.ph ], [ %78, %.preheader35 ]
+  %23 = phi i64 [ 0, %.preheader35.lr.ph ], [ %79, %.preheader35 ]
+  %.045 = phi i1 [ false, %.preheader35.lr.ph ], [ %.2.8, %.preheader35 ]
+  %.lcssa364144 = phi i32 [ 0, %.preheader35.lr.ph ], [ %77, %.preheader35 ]
+  %24 = mul nsw i64 %23, %11
+  %25 = getelementptr i8, ptr %19, i64 %24
+  %26 = load i32, ptr %25, align 4
+  %27 = icmp ne i32 %26, %20
+  %28 = icmp sge i32 %26, %22
+  %.not33 = and i1 %.045, %28
+  %or.cond = select i1 %27, i1 true, i1 %.not33
+  %29 = select i1 %or.cond, i32 %.lcssa364144, i32 1
+  %not.or.cond = xor i1 %or.cond, true
+  %.2 = select i1 %not.or.cond, i1 true, i1 %.045
+  %30 = select i1 %or.cond, i32 %22, i32 %20
+  %31 = getelementptr i8, ptr %25, i64 %21
+  %32 = load i32, ptr %31, align 4
+  %33 = icmp ne i32 %32, %20
+  %34 = icmp sge i32 %32, %30
+  %.not33.1 = and i1 %.2, %34
+  %or.cond.1 = select i1 %33, i1 true, i1 %.not33.1
+  %35 = select i1 %or.cond.1, i32 %29, i32 2
+  %not.or.cond.1 = xor i1 %or.cond.1, true
+  %.2.1 = select i1 %not.or.cond.1, i1 true, i1 %.2
+  %36 = select i1 %or.cond.1, i32 %30, i32 %20
+  %37 = getelementptr i8, ptr %25, i64 %.neg
+  %38 = load i32, ptr %37, align 4
+  %39 = icmp ne i32 %38, %20
+  %40 = icmp sge i32 %38, %36
+  %.not33.2 = and i1 %.2.1, %40
+  %or.cond.2 = select i1 %39, i1 true, i1 %.not33.2
+  %41 = select i1 %or.cond.2, i32 %35, i32 3
+  %not.or.cond.2 = xor i1 %or.cond.2, true
+  %.2.2 = select i1 %not.or.cond.2, i1 true, i1 %.2.1
+  %42 = select i1 %or.cond.2, i32 %36, i32 %20
+  %43 = getelementptr i8, ptr %25, i64 %.neg50
+  %44 = load i32, ptr %43, align 4
+  %45 = icmp ne i32 %44, %20
+  %46 = icmp sge i32 %44, %42
+  %.not33.3 = and i1 %.2.2, %46
+  %or.cond.3 = select i1 %45, i1 true, i1 %.not33.3
+  %47 = select i1 %or.cond.3, i32 %41, i32 4
+  %not.or.cond.3 = xor i1 %or.cond.3, true
+  %.2.3 = select i1 %not.or.cond.3, i1 true, i1 %.2.2
+  %48 = select i1 %or.cond.3, i32 %42, i32 %20
+  %49 = getelementptr i8, ptr %25, i64 %.neg51
+  %50 = load i32, ptr %49, align 4
+  %51 = icmp ne i32 %50, %20
+  %52 = icmp sge i32 %50, %48
+  %.not33.4 = and i1 %.2.3, %52
+  %or.cond.4 = select i1 %51, i1 true, i1 %.not33.4
+  %53 = select i1 %or.cond.4, i32 %47, i32 5
+  %not.or.cond.4 = xor i1 %or.cond.4, true
+  %.2.4 = select i1 %not.or.cond.4, i1 true, i1 %.2.3
+  %54 = select i1 %or.cond.4, i32 %48, i32 %20
+  %55 = getelementptr i8, ptr %25, i64 %.neg52
+  %56 = load i32, ptr %55, align 4
+  %57 = icmp ne i32 %56, %20
+  %58 = icmp sge i32 %56, %54
+  %.not33.5 = and i1 %.2.4, %58
+  %or.cond.5 = select i1 %57, i1 true, i1 %.not33.5
+  %59 = select i1 %or.cond.5, i32 %53, i32 6
+  %not.or.cond.5 = xor i1 %or.cond.5, true
+  %.2.5 = select i1 %not.or.cond.5, i1 true, i1 %.2.4
+  %60 = select i1 %or.cond.5, i32 %54, i32 %20
+  %61 = getelementptr i8, ptr %25, i64 %.neg53
+  %62 = load i32, ptr %61, align 4
+  %63 = icmp ne i32 %62, %20
+  %64 = icmp sge i32 %62, %60
+  %.not33.6 = and i1 %.2.5, %64
+  %or.cond.6 = select i1 %63, i1 true, i1 %.not33.6
+  %65 = select i1 %or.cond.6, i32 %59, i32 7
+  %not.or.cond.6 = xor i1 %or.cond.6, true
+  %.2.6 = select i1 %not.or.cond.6, i1 true, i1 %.2.5
+  %66 = select i1 %or.cond.6, i32 %60, i32 %20
+  %67 = getelementptr i8, ptr %25, i64 %.neg54
+  %68 = load i32, ptr %67, align 4
+  %69 = icmp ne i32 %68, %20
+  %70 = icmp sge i32 %68, %66
+  %.not33.7 = and i1 %.2.6, %70
+  %or.cond.7 = select i1 %69, i1 true, i1 %.not33.7
+  %71 = select i1 %or.cond.7, i32 %65, i32 8
+  %not.or.cond.7 = xor i1 %or.cond.7, true
+  %.2.7 = select i1 %not.or.cond.7, i1 true, i1 %.2.6
+  %72 = select i1 %or.cond.7, i32 %66, i32 %20
+  %73 = getelementptr i8, ptr %25, i64 %.neg55
+  %74 = load i32, ptr %73, align 4
+  %75 = icmp ne i32 %74, %20
+  %76 = icmp sge i32 %74, %72
+  %.not33.8 = and i1 %.2.7, %76
+  %or.cond.8 = select i1 %75, i1 true, i1 %.not33.8
+  %77 = select i1 %or.cond.8, i32 %71, i32 9
+  %not.or.cond.8 = xor i1 %or.cond.8, true
+  %.2.8 = select i1 %not.or.cond.8, i1 true, i1 %.2.7
+  %78 = select i1 %or.cond.8, i32 %72, i32 %20
+  %79 = add nuw nsw i64 %23, 1
+  %exitcond.not = icmp eq i64 %79, %9
+  br i1 %exitcond.not, label %.preheader, label %.preheader35
+
+.preheader:                                       ; preds = %.preheader35, %3
+  %.lcssa3641.lcssa = phi i32 [ 0, %3 ], [ %77, %.preheader35 ]
+  ret i32 %.lcssa3641.lcssa
+}
-- 
GitLab


From 37c6b9ff7245a76056ca3363bdef4e593e7e6c9d Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Tue, 14 May 2024 15:52:07 +0000
Subject: [PATCH 1110/1206] [NFC][LLVM] Mainly whitespace changes.

Also marks AliasSetTracker::size() as const.
---
 llvm/include/llvm/Analysis/AliasSetTracker.h | 2 +-
 llvm/include/llvm/Analysis/VectorUtils.h     | 1 +
 llvm/lib/IR/ConstantFold.cpp                 | 1 +
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 +-
 4 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Analysis/AliasSetTracker.h b/llvm/include/llvm/Analysis/AliasSetTracker.h
index 3030cb30864a..b8e8e7ef047f 100644
--- a/llvm/include/llvm/Analysis/AliasSetTracker.h
+++ b/llvm/include/llvm/Analysis/AliasSetTracker.h
@@ -120,7 +120,7 @@ public:
   iterator begin() const { return MemoryLocs.begin(); }
   iterator end() const { return MemoryLocs.end(); }
 
-  unsigned size() { return MemoryLocs.size(); }
+  unsigned size() const { return MemoryLocs.size(); }
 
   /// Retrieve the pointer values for the memory locations in this alias set.
   /// The order matches that of the memory locations, but duplicate pointer
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 521dac08792f..7a740d1206f7 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -96,6 +96,7 @@ public:
   VFDatabase(CallInst &CI)
       : M(CI.getModule()), CI(CI),
         ScalarToVectorMappings(VFDatabase::getMappings(CI)) {}
+
   /// \defgroup VFDatabase query interface.
   ///
   /// @{
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index a766b1fe6018..4622ad7e9a0e 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -542,6 +542,7 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
     } else if (!MaskEltCount.isScalable())
       return ConstantVector::getSplat(MaskEltCount, Elt);
   }
+
   // Do not iterate on scalable vector. The num of elements is unknown at
   // compile-time.
   if (isa<ScalableVectorType>(V1VTy))
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 55fecc4b4845..aa0b7c93f866 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -5717,7 +5717,7 @@ int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
     Offset = Remainder;
   else {
     NewOffset = NewOffset < 0 ? MinOff : MaxOff;
-    Offset = Offset - NewOffset * Scale;
+    Offset = Offset - (NewOffset * Scale);
   }
 
   if (EmittableOffset)
-- 
GitLab


From 58bad2862cf136f9483eb005bbfa6915d459b46d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Don=C3=A1t=20Nagy?= <donat.nagy@ericsson.com>
Date: Fri, 17 May 2024 13:08:45 +0200
Subject: [PATCH 1111/1206] [analyzer][NFC] Require explicit matching mode for
 CallDescriptions (#92454)

This commit deletes the "simple" constructor of `CallDescription` which
did not require a `CallDescription::Mode` argument and always used the
"wildcard" mode `CDM::Unspecified`.

A few months ago, this vague matching mode was used by many checkers,
which caused bugs like https://github.com/llvm/llvm-project/issues/81597
and https://github.com/llvm/llvm-project/issues/88181. Since then, my
commits improved the available matching modes and ensured that all
checkers explicitly specify the right matching mode.

After those commits, the only remaining references to the "simple"
constructor were some unit tests; this commit updates them to use an
explicitly specified matching mode (often `CDM::SimpleFunc`).

The mode `CDM::Unspecified` was not deleted in this commit because it's
still a reasonable choice in `GenericTaintChecker` and a few unit tests.
---
 .../Core/PathSensitive/CallDescription.h      | 14 ++---
 .../StaticAnalyzer/Core/CallDescription.cpp   |  7 ---
 .../BugReportInterestingnessTest.cpp          | 10 ++--
 .../StaticAnalyzer/CallDescriptionTest.cpp    | 57 ++++++++++---------
 .../ConflictingEvalCallsTest.cpp              |  2 +-
 .../FalsePositiveRefutationBRVisitorTest.cpp  |  8 ++-
 .../MemRegionDescriptiveNameTest.cpp          |  3 +-
 .../NoStateChangeFuncVisitorTest.cpp          |  6 +-
 8 files changed, 51 insertions(+), 56 deletions(-)

diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
index ccfe8d47c290..a99c11766f11 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
@@ -56,9 +56,10 @@ public:
     /// overloaded operator, a constructor or a destructor).
     CXXMethod,
 
-    /// Match any CallEvent that is not an ObjCMethodCall.
-    /// FIXME: Previously this was the default behavior of CallDescription, but
-    /// its use should be replaced by a more specific mode almost everywhere.
+    /// Match any CallEvent that is not an ObjCMethodCall. This should not be
+    /// used when the checker looks for a concrete function (and knows whether
+    /// it is a method); but GenericTaintChecker uses this mode to match
+    /// functions whose name was configured by the user.
     Unspecified,
 
     /// FIXME: Add support for ObjCMethodCall events (I'm not adding it because
@@ -100,13 +101,6 @@ public:
                   MaybeCount RequiredArgs = std::nullopt,
                   MaybeCount RequiredParams = std::nullopt);
 
-  /// Construct a CallDescription with default flags.
-  CallDescription(ArrayRef<StringRef> QualifiedName,
-                  MaybeCount RequiredArgs = std::nullopt,
-                  MaybeCount RequiredParams = std::nullopt);
-
-  CallDescription(std::nullptr_t) = delete;
-
   /// Get the name of the function that this object matches.
   StringRef getFunctionName() const { return QualifiedName.back(); }
 
diff --git a/clang/lib/StaticAnalyzer/Core/CallDescription.cpp b/clang/lib/StaticAnalyzer/Core/CallDescription.cpp
index 0bb0fe66e54f..cd23b381f879 100644
--- a/clang/lib/StaticAnalyzer/Core/CallDescription.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CallDescription.cpp
@@ -48,13 +48,6 @@ ento::CallDescription::CallDescription(Mode MatchAs,
                   [](StringRef From) { return From.str(); });
 }
 
-/// Construct a CallDescription with default flags.
-ento::CallDescription::CallDescription(ArrayRef<StringRef> QualifiedName,
-                                       MaybeCount RequiredArgs /*= None*/,
-                                       MaybeCount RequiredParams /*= None*/)
-    : CallDescription(Mode::Unspecified, QualifiedName, RequiredArgs,
-                      RequiredParams) {}
-
 bool ento::CallDescription::matches(const CallEvent &Call) const {
   // FIXME: Add ObjC Message support.
   if (Call.getKind() == CE_ObjCMessage)
diff --git a/clang/unittests/StaticAnalyzer/BugReportInterestingnessTest.cpp b/clang/unittests/StaticAnalyzer/BugReportInterestingnessTest.cpp
index 5f562b1c98b0..70a58026da95 100644
--- a/clang/unittests/StaticAnalyzer/BugReportInterestingnessTest.cpp
+++ b/clang/unittests/StaticAnalyzer/BugReportInterestingnessTest.cpp
@@ -33,11 +33,13 @@ class InterestingnessTestChecker : public Checker<check::PreCall> {
                                        const CallEvent &, CheckerContext &)>;
 
   CallDescriptionMap<HandlerFn> Handlers = {
-      {{{"setInteresting"}, 1}, &InterestingnessTestChecker::handleInteresting},
-      {{{"setNotInteresting"}, 1},
+      {{CDM::SimpleFunc, {"setInteresting"}, 1},
+       &InterestingnessTestChecker::handleInteresting},
+      {{CDM::SimpleFunc, {"setNotInteresting"}, 1},
        &InterestingnessTestChecker::handleNotInteresting},
-      {{{"check"}, 1}, &InterestingnessTestChecker::handleCheck},
-      {{{"bug"}, 1}, &InterestingnessTestChecker::handleBug},
+      {{CDM::SimpleFunc, {"check"}, 1},
+       &InterestingnessTestChecker::handleCheck},
+      {{CDM::SimpleFunc, {"bug"}, 1}, &InterestingnessTestChecker::handleBug},
   };
 
   void handleInteresting(const CallEvent &Call, CheckerContext &C) const;
diff --git a/clang/unittests/StaticAnalyzer/CallDescriptionTest.cpp b/clang/unittests/StaticAnalyzer/CallDescriptionTest.cpp
index 238f954d7133..aed49a1f5f78 100644
--- a/clang/unittests/StaticAnalyzer/CallDescriptionTest.cpp
+++ b/clang/unittests/StaticAnalyzer/CallDescriptionTest.cpp
@@ -138,8 +138,10 @@ public:
 TEST(CallDescription, SimpleNameMatching) {
   EXPECT_TRUE(tooling::runToolOnCode(
       std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-          {{{"bar"}}, false}, // false: there's no call to 'bar' in this code.
-          {{{"foo"}}, true},  // true: there's a call to 'foo' in this code.
+          {{CDM::SimpleFunc, {"bar"}},
+           false}, // false: there's no call to 'bar' in this code.
+          {{CDM::SimpleFunc, {"foo"}},
+           true}, // true: there's a call to 'foo' in this code.
       })),
       "void foo(); void bar() { foo(); }"));
 }
@@ -147,8 +149,8 @@ TEST(CallDescription, SimpleNameMatching) {
 TEST(CallDescription, RequiredArguments) {
   EXPECT_TRUE(tooling::runToolOnCode(
       std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-          {{{"foo"}, 1}, true},
-          {{{"foo"}, 2}, false},
+          {{CDM::SimpleFunc, {"foo"}, 1}, true},
+          {{CDM::SimpleFunc, {"foo"}, 2}, false},
       })),
       "void foo(int); void foo(int, int); void bar() { foo(1); }"));
 }
@@ -156,8 +158,8 @@ TEST(CallDescription, RequiredArguments) {
 TEST(CallDescription, LackOfRequiredArguments) {
   EXPECT_TRUE(tooling::runToolOnCode(
       std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-          {{{"foo"}, std::nullopt}, true},
-          {{{"foo"}, 2}, false},
+          {{CDM::SimpleFunc, {"foo"}, std::nullopt}, true},
+          {{CDM::SimpleFunc, {"foo"}, 2}, false},
       })),
       "void foo(int); void foo(int, int); void bar() { foo(1); }"));
 }
@@ -187,7 +189,7 @@ TEST(CallDescription, QualifiedNames) {
   const std::string Code = (Twine{MockStdStringHeader} + AdditionalCode).str();
   EXPECT_TRUE(tooling::runToolOnCode(
       std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-          {{{"std", "basic_string", "c_str"}}, true},
+          {{CDM::CXXMethod, {"std", "basic_string", "c_str"}}, true},
       })),
       Code));
 }
@@ -202,7 +204,8 @@ TEST(CallDescription, MatchConstructor) {
   EXPECT_TRUE(tooling::runToolOnCode(
       std::unique_ptr<FrontendAction>(
           new CallDescriptionAction<CXXConstructExpr>({
-              {{{"std", "basic_string", "basic_string"}, 2, 2}, true},
+              {{CDM::CXXMethod, {"std", "basic_string", "basic_string"}, 2, 2},
+               true},
           })),
       Code));
 }
@@ -228,7 +231,7 @@ TEST(CallDescription, MatchConversionOperator) {
     })code";
   EXPECT_TRUE(tooling::runToolOnCode(
       std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-          {{{"aaa", "bbb", "Bar", "operator int"}}, true},
+          {{CDM::CXXMethod, {"aaa", "bbb", "Bar", "operator int"}}, true},
       })),
       Code));
 }
@@ -252,7 +255,7 @@ TEST(CallDescription, RejectOverQualifiedNames) {
   // FIXME: We should **not** match.
   EXPECT_TRUE(tooling::runToolOnCode(
       std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-          {{{"std", "container", "data"}}, true},
+          {{CDM::CXXMethod, {"std", "container", "data"}}, true},
       })),
       Code));
 }
@@ -272,7 +275,7 @@ TEST(CallDescription, DontSkipNonInlineNamespaces) {
     SCOPED_TRACE("my v1 bar");
     EXPECT_TRUE(tooling::runToolOnCode(
         std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-            {{{"my", "v1", "bar"}}, true},
+            {{CDM::SimpleFunc, {"my", "v1", "bar"}}, true},
         })),
         Code));
   }
@@ -281,7 +284,7 @@ TEST(CallDescription, DontSkipNonInlineNamespaces) {
     SCOPED_TRACE("my bar");
     EXPECT_TRUE(tooling::runToolOnCode(
         std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-            {{{"my", "bar"}}, true},
+            {{CDM::SimpleFunc, {"my", "bar"}}, true},
         })),
         Code));
   }
@@ -303,7 +306,7 @@ TEST(CallDescription, SkipTopInlineNamespaces) {
     SCOPED_TRACE("my v1 bar");
     EXPECT_TRUE(tooling::runToolOnCode(
         std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-            {{{"my", "v1", "bar"}}, true},
+            {{CDM::SimpleFunc, {"my", "v1", "bar"}}, true},
         })),
         Code));
   }
@@ -311,7 +314,7 @@ TEST(CallDescription, SkipTopInlineNamespaces) {
     SCOPED_TRACE("v1 bar");
     EXPECT_TRUE(tooling::runToolOnCode(
         std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-            {{{"v1", "bar"}}, true},
+            {{CDM::SimpleFunc, {"v1", "bar"}}, true},
         })),
         Code));
   }
@@ -338,7 +341,7 @@ TEST(CallDescription, SkipAnonimousNamespaces) {
 
   EXPECT_TRUE(tooling::runToolOnCode(
       std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-          {{{"std", "container", "data"}}, true},
+          {{CDM::CXXMethod, {"std", "container", "data"}}, true},
       })),
       Code));
 }
@@ -376,7 +379,7 @@ TEST(CallDescription, AliasNames) {
       SCOPED_TRACE("std container data");
       EXPECT_TRUE(tooling::runToolOnCode(
           std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-              {{{"std", "container", "data"}}, true},
+              {{CDM::CXXMethod, {"std", "container", "data"}}, true},
           })),
           UseAliasInSpellingCode));
     }
@@ -385,7 +388,7 @@ TEST(CallDescription, AliasNames) {
       SCOPED_TRACE("std cont data");
       EXPECT_TRUE(tooling::runToolOnCode(
           std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-              {{{"std", "cont", "data"}}, false},
+              {{CDM::CXXMethod, {"std", "cont", "data"}}, false},
           })),
           UseAliasInSpellingCode));
     }
@@ -399,7 +402,7 @@ TEST(CallDescription, AliasNames) {
       SCOPED_TRACE("std container data");
       EXPECT_TRUE(tooling::runToolOnCode(
           std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-              {{{"std", "container", "data"}}, true},
+              {{CDM::CXXMethod, {"std", "container", "data"}}, true},
           })),
           UseAliasInSpellingCode));
     }
@@ -408,7 +411,7 @@ TEST(CallDescription, AliasNames) {
       SCOPED_TRACE("std cont data");
       EXPECT_TRUE(tooling::runToolOnCode(
           std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-              {{{"std", "cont", "data"}}, false},
+              {{CDM::CXXMethod, {"std", "cont", "data"}}, false},
           })),
           UseAliasInSpellingCode));
     }
@@ -431,7 +434,7 @@ TEST(CallDescription, AliasSingleNamespace) {
     SCOPED_TRACE("aaa bbb ccc bar");
     EXPECT_TRUE(tooling::runToolOnCode(
         std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-            {{{"aaa", "bbb", "ccc", "bar"}}, true},
+            {{CDM::SimpleFunc, {"aaa", "bbb", "ccc", "bar"}}, true},
         })),
         Code));
   }
@@ -440,7 +443,7 @@ TEST(CallDescription, AliasSingleNamespace) {
     SCOPED_TRACE("aaa bbb_alias ccc bar");
     EXPECT_TRUE(tooling::runToolOnCode(
         std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-            {{{"aaa", "bbb_alias", "ccc", "bar"}}, false},
+            {{CDM::SimpleFunc, {"aaa", "bbb_alias", "ccc", "bar"}}, false},
         })),
         Code));
   }
@@ -462,7 +465,7 @@ TEST(CallDescription, AliasMultipleNamespaces) {
     SCOPED_TRACE("aaa bbb ccc bar");
     EXPECT_TRUE(tooling::runToolOnCode(
         std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-            {{{"aaa", "bbb", "ccc", "bar"}}, true},
+            {{CDM::SimpleFunc, {"aaa", "bbb", "ccc", "bar"}}, true},
         })),
         Code));
   }
@@ -471,7 +474,7 @@ TEST(CallDescription, AliasMultipleNamespaces) {
     SCOPED_TRACE("aaa_bbb_ccc bar");
     EXPECT_TRUE(tooling::runToolOnCode(
         std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-            {{{"aaa_bbb_ccc", "bar"}}, false},
+            {{CDM::SimpleFunc, {"aaa_bbb_ccc", "bar"}}, false},
         })),
         Code));
   }
@@ -480,9 +483,9 @@ TEST(CallDescription, AliasMultipleNamespaces) {
 TEST(CallDescription, NegativeMatchQualifiedNames) {
   EXPECT_TRUE(tooling::runToolOnCode(
       std::unique_ptr<FrontendAction>(new CallDescriptionAction<>({
-          {{{"foo", "bar"}}, false},
-          {{{"bar", "foo"}}, false},
-          {{{"foo"}}, true},
+          {{CDM::Unspecified, {"foo", "bar"}}, false},
+          {{CDM::Unspecified, {"bar", "foo"}}, false},
+          {{CDM::Unspecified, {"foo"}}, true},
       })),
       "void foo(); struct bar { void foo(); }; void test() { foo(); }"));
 }
@@ -598,7 +601,7 @@ TEST(CallDescription, MatchBuiltins) {
 
 class CallDescChecker
     : public Checker<check::PreCall, check::PreStmt<CallExpr>> {
-  CallDescriptionSet Set = {{{"bar"}, 0}};
+  CallDescriptionSet Set = {{CDM::SimpleFunc, {"bar"}, 0}};
 
 public:
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const {
diff --git a/clang/unittests/StaticAnalyzer/ConflictingEvalCallsTest.cpp b/clang/unittests/StaticAnalyzer/ConflictingEvalCallsTest.cpp
index d3eb4b7a94d3..e410cca07663 100644
--- a/clang/unittests/StaticAnalyzer/ConflictingEvalCallsTest.cpp
+++ b/clang/unittests/StaticAnalyzer/ConflictingEvalCallsTest.cpp
@@ -18,7 +18,7 @@ using namespace ento;
 
 namespace {
 class EvalCallBase : public Checker<eval::Call> {
-  const CallDescription Foo = {{"foo"}, 0};
+  const CallDescription Foo = {CDM::SimpleFunc, {"foo"}, 0};
 
 public:
   bool evalCall(const CallEvent &Call, CheckerContext &C) const {
diff --git a/clang/unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp b/clang/unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp
index 3e12eec549c4..8f0a96d41e75 100644
--- a/clang/unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp
+++ b/clang/unittests/StaticAnalyzer/FalsePositiveRefutationBRVisitorTest.cpp
@@ -30,9 +30,11 @@ class FalsePositiveGenerator : public Checker<eval::Call> {
   using HandlerFn = bool (Self::*)(const CallEvent &Call,
                                    CheckerContext &) const;
   CallDescriptionMap<HandlerFn> Callbacks = {
-      {{{"reachedWithContradiction"}, 0}, &Self::reachedWithContradiction},
-      {{{"reachedWithNoContradiction"}, 0}, &Self::reachedWithNoContradiction},
-      {{{"reportIfCanBeTrue"}, 1}, &Self::reportIfCanBeTrue},
+      {{CDM::SimpleFunc, {"reachedWithContradiction"}, 0},
+       &Self::reachedWithContradiction},
+      {{CDM::SimpleFunc, {"reachedWithNoContradiction"}, 0},
+       &Self::reachedWithNoContradiction},
+      {{CDM::SimpleFunc, {"reportIfCanBeTrue"}, 1}, &Self::reportIfCanBeTrue},
   };
 
   bool report(CheckerContext &C, ProgramStateRef State,
diff --git a/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp b/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp
index ba0c4d25e13b..03aee56a200f 100644
--- a/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp
+++ b/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp
@@ -38,7 +38,8 @@ public:
 
 private:
   const BugType Bug{this, "DescriptiveNameBug"};
-  const CallDescription HandlerFn = {{"reportDescriptiveName"}, 1};
+  const CallDescription HandlerFn = {
+      CDM::SimpleFunc, {"reportDescriptiveName"}, 1};
 };
 
 void addDescriptiveNameChecker(AnalysisASTConsumer &AnalysisConsumer,
diff --git a/clang/unittests/StaticAnalyzer/NoStateChangeFuncVisitorTest.cpp b/clang/unittests/StaticAnalyzer/NoStateChangeFuncVisitorTest.cpp
index 51aca42a26b0..a9033425dfb5 100644
--- a/clang/unittests/StaticAnalyzer/NoStateChangeFuncVisitorTest.cpp
+++ b/clang/unittests/StaticAnalyzer/NoStateChangeFuncVisitorTest.cpp
@@ -86,17 +86,17 @@ class StatefulChecker : public Checker<check::PreCall> {
 
 public:
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const {
-    if (CallDescription{{"preventError"}, 0}.matches(Call)) {
+    if (CallDescription{CDM::SimpleFunc, {"preventError"}, 0}.matches(Call)) {
       C.addTransition(C.getState()->set<ErrorPrevented>(true));
       return;
     }
 
-    if (CallDescription{{"allowError"}, 0}.matches(Call)) {
+    if (CallDescription{CDM::SimpleFunc, {"allowError"}, 0}.matches(Call)) {
       C.addTransition(C.getState()->set<ErrorPrevented>(false));
       return;
     }
 
-    if (CallDescription{{"error"}, 0}.matches(Call)) {
+    if (CallDescription{CDM::SimpleFunc, {"error"}, 0}.matches(Call)) {
       if (C.getState()->get<ErrorPrevented>())
         return;
       const ExplodedNode *N = C.generateErrorNode();
-- 
GitLab


From ddb87e0f96c7d23b9b824fa9e2710c0fe11bb86b Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 17 May 2024 13:16:34 +0200
Subject: [PATCH 1112/1206] SystemZ: Use REG_SEQUENCE for PAIR128 (#90640)

PAIR128 should probably just be removed entirely

Depends #90638
---
 .../Target/SystemZ/SystemZISelLowering.cpp    | 20 ++++++-------------
 .../CodeGen/SystemZ/atomicrmw-ops-i128.ll     | 14 ++++++-------
 2 files changed, 13 insertions(+), 21 deletions(-)

diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 2920c1f02a31..5cb67ebf4e4b 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -8868,23 +8868,15 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
 MachineBasicBlock *
 SystemZTargetLowering::emitPair128(MachineInstr &MI,
                                    MachineBasicBlock *MBB) const {
-  MachineFunction &MF = *MBB->getParent();
   const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
-  MachineRegisterInfo &MRI = MF.getRegInfo();
-  DebugLoc DL = MI.getDebugLoc();
+  const DebugLoc &DL = MI.getDebugLoc();
 
   Register Dest = MI.getOperand(0).getReg();
-  Register Hi = MI.getOperand(1).getReg();
-  Register Lo = MI.getOperand(2).getReg();
-  Register Tmp1 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
-  Register Tmp2 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
-
-  BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), Tmp1);
-  BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Tmp2)
-    .addReg(Tmp1).addReg(Hi).addImm(SystemZ::subreg_h64);
-  BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest)
-    .addReg(Tmp2).addReg(Lo).addImm(SystemZ::subreg_l64);
-
+  BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest)
+      .add(MI.getOperand(1))
+      .addImm(SystemZ::subreg_h64)
+      .add(MI.getOperand(2))
+      .addImm(SystemZ::subreg_l64);
   MI.eraseFromParent();
   return MBB;
 }
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
index 604fd50baade..c088f6d862e7 100644
--- a/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
@@ -372,8 +372,8 @@ define i128 @atomicrmw_udec_wrap(ptr %src, i128 %b) {
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
 ; CHECK-NEXT:    vlgvg %r1, %v3, 1
 ; CHECK-NEXT:    vlgvg %r0, %v3, 0
-; CHECK-NEXT:    vlgvg %r5, %v4, 1
-; CHECK-NEXT:    vlgvg %r4, %v4, 0
+; CHECK-NEXT:    vlgvg %r5, %v5, 1
+; CHECK-NEXT:    vlgvg %r4, %v5, 0
 ; CHECK-NEXT:    cdsg %r0, %r4, 0(%r3)
 ; CHECK-NEXT:    vlvgp %v3, %r0, %r1
 ; CHECK-NEXT:    je .LBB12_8
@@ -386,19 +386,19 @@ define i128 @atomicrmw_udec_wrap(ptr %src, i128 %b) {
 ; CHECK-NEXT:    vchlgs %v4, %v3, %v0
 ; CHECK-NEXT:  .LBB12_4: # %atomicrmw.start
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
-; CHECK-NEXT:    vlr %v5, %v0
+; CHECK-NEXT:    vlr %v4, %v0
 ; CHECK-NEXT:    jl .LBB12_6
 ; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
-; CHECK-NEXT:    vaq %v5, %v3, %v1
+; CHECK-NEXT:    vaq %v4, %v3, %v1
 ; CHECK-NEXT:  .LBB12_6: # %atomicrmw.start
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
-; CHECK-NEXT:    vceqgs %v4, %v3, %v2
-; CHECK-NEXT:    vlr %v4, %v0
+; CHECK-NEXT:    vceqgs %v5, %v3, %v2
+; CHECK-NEXT:    vlr %v5, %v0
 ; CHECK-NEXT:    je .LBB12_1
 ; CHECK-NEXT:  # %bb.7: # %atomicrmw.start
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
-; CHECK-NEXT:    vlr %v4, %v5
+; CHECK-NEXT:    vlr %v5, %v4
 ; CHECK-NEXT:    j .LBB12_1
 ; CHECK-NEXT:  .LBB12_8: # %atomicrmw.end
 ; CHECK-NEXT:    vst %v3, 0(%r2), 3
-- 
GitLab


From 1e7d047c71ce0a8076f774b9106cd7a499a1c542 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 17 May 2024 12:18:01 +0100
Subject: [PATCH 1113/1206] [VPlan] Mark LoopInfo preserved in native-path as
 well (NFC).

LoopInfo is updated during VPlan execution now, so it will also be
updated correctly in the native path.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp        | 6 +++---
 llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f2a541f5167a..fd652cb78954 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -10390,15 +10390,15 @@ PreservedAnalyses LoopVectorizePass::run(Function &F,
         RemoveRedundantDbgInstrs(&BB);
     }
 
-    // We currently do not preserve loopinfo/dominator analyses with outer loop
+    // We currently do not preserve dominator analyses with outer loop
     // vectorization. Until this is addressed, mark these analyses as preserved
     // only for non-VPlan-native path.
-    // TODO: Preserve Loop and Dominator analyses for VPlan-native path.
+    // TODO: Preserve Dominator analysis for VPlan-native path.
     if (!EnableVPlanNativePath) {
-      PA.preserve<LoopAnalysis>();
       PA.preserve<DominatorTreeAnalysis>();
       PA.preserve<ScalarEvolutionAnalysis>();
     }
+    PA.preserve<LoopAnalysis>();
 
     if (Result.MadeCFGChange) {
       // Making CFG changes likely means a loop got vectorized. Indicate that
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
index 4294212ecb74..3b5811d2c9f0 100644
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
@@ -13,7 +13,7 @@
 ;   }
 ; }
 ;
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path < %s | FileCheck %s
+; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -verify-loop-info < %s | FileCheck %s
 ; CHECK-LABEL: vector.ph:
 ; CHECK: %[[SplatVal:.*]] = insertelement <4 x i32> poison, i32 %n, i64 0
 ; CHECK: %[[Splat:.*]] = shufflevector <4 x i32> %[[SplatVal]], <4 x i32> poison, <4 x i32> zeroinitializer
-- 
GitLab


From ac092925c3fa61144e606c7a9d0d375b96e1abda Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Fri, 17 May 2024 12:39:40 +0100
Subject: [PATCH 1114/1206] [SelectionDAG] Widen cttz to cttz_zero_undef
 (#92514)

Instead of widening e.g. i8 cttz(x) to i16 cttz(x | 0x100), use the more
optimizable form cttz_zero_undef(x | 0x100) since the widened operand is
definitely not zero.
---
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 12 ++++----
 .../SelectionDAG/LegalizeIntegerTypes.cpp     | 15 ++++++----
 llvm/test/CodeGen/AMDGPU/cttz.ll              |  2 --
 llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll   |  1 -
 llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll        | 30 +++++--------------
 5 files changed, 24 insertions(+), 36 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index fd97a1283b65..0543c211c497 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -5032,7 +5032,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
   case ISD::CTTZ_ZERO_UNDEF:
   case ISD::CTLZ:
   case ISD::CTLZ_ZERO_UNDEF:
-  case ISD::CTPOP:
+  case ISD::CTPOP: {
     // Zero extend the argument unless its cttz, then use any_extend.
     if (Node->getOpcode() == ISD::CTTZ ||
         Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
@@ -5040,7 +5040,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     else
       Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
 
-    if (Node->getOpcode() == ISD::CTTZ) {
+    unsigned NewOpc = Node->getOpcode();
+    if (NewOpc == ISD::CTTZ) {
       // The count is the same in the promoted type except if the original
       // value was zero.  This can be handled by setting the bit just off
       // the top of the original type.
@@ -5048,12 +5049,12 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
                                         OVT.getSizeInBits());
       Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
                          DAG.getConstant(TopBit, dl, NVT));
+      NewOpc = ISD::CTTZ_ZERO_UNDEF;
     }
     // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
     // already the correct result.
-    Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
-    if (Node->getOpcode() == ISD::CTLZ ||
-        Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
+    Tmp1 = DAG.getNode(NewOpc, dl, NVT, Tmp1);
+    if (NewOpc == ISD::CTLZ || NewOpc == ISD::CTLZ_ZERO_UNDEF) {
       // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
       Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
                           DAG.getConstant(NVT.getSizeInBits() -
@@ -5061,6 +5062,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     }
     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
     break;
+  }
   case ISD::BITREVERSE:
   case ISD::BSWAP: {
     unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 0aa36deda79d..98f64947bcab 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -709,23 +709,26 @@ SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
     }
   }
 
-  if (N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::VP_CTTZ) {
+  unsigned NewOpc = N->getOpcode();
+  if (NewOpc == ISD::CTTZ || NewOpc == ISD::VP_CTTZ) {
     // The count is the same in the promoted type except if the original
     // value was zero.  This can be handled by setting the bit just off
     // the top of the original type.
     auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
                                       OVT.getScalarSizeInBits());
-    if (N->getOpcode() == ISD::CTTZ)
+    if (NewOpc == ISD::CTTZ) {
       Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
-    else
+      NewOpc = ISD::CTTZ_ZERO_UNDEF;
+    } else {
       Op =
           DAG.getNode(ISD::VP_OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT),
                       N->getOperand(1), N->getOperand(2));
+      NewOpc = ISD::VP_CTTZ_ZERO_UNDEF;
+    }
   }
   if (!N->isVPOpcode())
-    return DAG.getNode(N->getOpcode(), dl, NVT, Op);
-  return DAG.getNode(N->getOpcode(), dl, NVT, Op, N->getOperand(1),
-                     N->getOperand(2));
+    return DAG.getNode(NewOpc, dl, NVT, Op);
+  return DAG.getNode(NewOpc, dl, NVT, Op, N->getOperand(1), N->getOperand(2));
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(SDNode *N) {
diff --git a/llvm/test/CodeGen/AMDGPU/cttz.ll b/llvm/test/CodeGen/AMDGPU/cttz.ll
index 118d6c123046..ee2894a66fbf 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz.ll
@@ -1408,7 +1408,6 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_e32 v2, 0x10000, v0
 ; VI-NEXT:    v_ffbl_b32_e32 v2, v2
-; VI-NEXT:    v_min_u32_e32 v2, 32, v2
 ; VI-NEXT:    v_cmp_ne_u16_e32 vcc, 0, v0
 ; VI-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
 ; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
@@ -1451,7 +1450,6 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
 ; GFX10-NEXT:    v_or_b32_e32 v2, 0x10000, v1
 ; GFX10-NEXT:    v_cmp_ne_u16_e32 vcc_lo, 0, v1
 ; GFX10-NEXT:    v_ffbl_b32_e32 v2, v2
-; GFX10-NEXT:    v_min_u32_e32 v2, 32, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0xffff, v2, vcc_lo
 ; GFX10-NEXT:    global_store_short v0, v1, s[0:1]
 ; GFX10-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
index 71f1cd54d705..392a44318b0a 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
@@ -1561,7 +1561,6 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
 ; VI-NEXT:    v_or_b32_e32 v0, v2, v0
 ; VI-NEXT:    v_or_b32_e32 v2, 0x10000, v0
 ; VI-NEXT:    v_ffbl_b32_e32 v2, v2
-; VI-NEXT:    v_min_u32_e32 v2, 32, v2
 ; VI-NEXT:    v_cmp_ne_u16_e32 vcc, 0, v0
 ; VI-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
diff --git a/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
index ef8a6c704a44..4a001662ce2c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
@@ -4145,29 +4145,15 @@ define <vscale x 1 x i9> @vp_cttz_nxv1i9(<vscale x 1 x i9> %va, <vscale x 1 x i1
 ; CHECK-NEXT:    li a1, 512
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vor.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    li a0, 1
-; CHECK-NEXT:    vsub.vx v9, v8, a0, v0.t
-; CHECK-NEXT:    vnot.v v8, v8, v0.t
+; CHECK-NEXT:    vrsub.vi v9, v8, 0, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
-; CHECK-NEXT:    vsrl.vi v9, v8, 1, v0.t
-; CHECK-NEXT:    lui a0, 5
-; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vx v9, v9, a0, v0.t
-; CHECK-NEXT:    vsub.vv v8, v8, v9, v0.t
-; CHECK-NEXT:    lui a0, 3
-; CHECK-NEXT:    addi a0, a0, 819
-; CHECK-NEXT:    vand.vx v9, v8, a0, v0.t
-; CHECK-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; CHECK-NEXT:    vand.vx v8, v8, a0, v0.t
-; CHECK-NEXT:    vadd.vv v8, v9, v8, v0.t
-; CHECK-NEXT:    vsrl.vi v9, v8, 4, v0.t
-; CHECK-NEXT:    vadd.vv v8, v8, v9, v0.t
-; CHECK-NEXT:    lui a0, 1
-; CHECK-NEXT:    addi a0, a0, -241
-; CHECK-NEXT:    vand.vx v8, v8, a0, v0.t
-; CHECK-NEXT:    li a0, 257
-; CHECK-NEXT:    vmul.vx v8, v8, a0, v0.t
-; CHECK-NEXT:    vsrl.vi v8, v8, 8, v0.t
+; CHECK-NEXT:    vfwcvt.f.xu.v v9, v8, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsrl.vi v8, v9, 23, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0, v0.t
+; CHECK-NEXT:    li a0, 127
+; CHECK-NEXT:    vsub.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_cttz_nxv1i9:
-- 
GitLab


From d1cff36e5e37cd552ec049335feb1dd8f94517ea Mon Sep 17 00:00:00 2001
From: Andi Drebes <47449897+andidr@users.noreply.github.com>
Date: Fri, 17 May 2024 13:41:30 +0200
Subject: [PATCH 1115/1206] [MLIR][analysis] Lattice: Fix automatic delegation
 of meet to lattice value classes (#82620)

The class `Lattice` should automatically delegate invocations of the
meet operator to the meet operation of the associated lattice value
class if that class provides a static function called `meet`. This
process fails for two reasons:

1. `Lattice::has_meet` checks for a member function `meet` without
arguments of the lattice value class, although it should check for a
static member function.

2. The function template `Lattice::meet<VT>()` implementing the default
meet operation directly in the lattice is always present and takes
precedence over the delegating function template `Lattice::meet<VT,
std::integral_constant<bool, true>>()`.

This change fixes the automatic delegation of the meet operation of a
lattice to the lattice value class in the presence of a static `meet`
function by conditionally enabling either the delegating function
template or the non-delegating function template and by changing
`Lattice::has_meet` so that it checks for a static `meet` member
function in the lattice value type.

The test from `TestSparseBackwardDataFlowAnalysis.cpp` is changed, such
that the `meet` function is not provided directly in the `WrittenTo`
lattice, but by the `Lattice` base class in order to trigger delegation
to a lattice value class.
---
 .../mlir/Analysis/DataFlow/SparseAnalysis.h   |  8 ++-
 .../TestSparseBackwardDataFlowAnalysis.cpp    | 60 +++++++++++++------
 2 files changed, 47 insertions(+), 21 deletions(-)

diff --git a/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h b/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
index b65ac8bb1dec..7aadd5409cc6 100644
--- a/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
+++ b/mlir/include/mlir/Analysis/DataFlow/SparseAnalysis.h
@@ -132,14 +132,15 @@ public:
   /// analysis, lattices will only have a `join`, no `meet`, but we want to use
   /// the same `Lattice` class for both directions.
   template <typename T, typename... Args>
-  using has_meet = decltype(std::declval<T>().meet());
+  using has_meet = decltype(&T::meet);
   template <typename T>
   using lattice_has_meet = llvm::is_detected<has_meet, T>;
 
   /// Meet (intersect) the information contained in the 'rhs' value with this
   /// lattice. Returns if the state of the current lattice changed.  If the
   /// lattice elements don't have a `meet` method, this is a no-op (see below.)
-  template <typename VT, std::enable_if_t<lattice_has_meet<VT>::value>>
+  template <typename VT,
+            std::enable_if_t<lattice_has_meet<VT>::value> * = nullptr>
   ChangeResult meet(const VT &rhs) {
     ValueT newValue = ValueT::meet(value, rhs);
     assert(ValueT::meet(newValue, value) == newValue &&
@@ -155,7 +156,8 @@ public:
     return ChangeResult::Change;
   }
 
-  template <typename VT>
+  template <typename VT,
+            std::enable_if_t<!lattice_has_meet<VT>::value> * = nullptr>
   ChangeResult meet(const VT &rhs) {
     return ChangeResult::NoChange;
   }
diff --git a/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp
index e1c60f06a6b5..302973804664 100644
--- a/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp
+++ b/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp
@@ -18,18 +18,27 @@ using namespace mlir::dataflow;
 
 namespace {
 
-/// This lattice represents, for a given value, the set of memory resources that
-/// this value, or anything derived from this value, is potentially written to.
-struct WrittenTo : public AbstractSparseLattice {
-  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(WrittenTo)
-  using AbstractSparseLattice::AbstractSparseLattice;
+/// Lattice value storing the a set of memory resources that something
+/// is written to.
+struct WrittenToLatticeValue {
+  bool operator==(const WrittenToLatticeValue &other) {
+    return this->writes == other.writes;
+  }
 
-  void print(raw_ostream &os) const override {
-    os << "[";
-    llvm::interleave(
-        writes, os, [&](const StringAttr &a) { os << a.str(); }, " ");
-    os << "]";
+  static WrittenToLatticeValue meet(const WrittenToLatticeValue &lhs,
+                                    const WrittenToLatticeValue &rhs) {
+    WrittenToLatticeValue res = lhs;
+    (void)res.addWrites(rhs.writes);
+
+    return res;
   }
+
+  static WrittenToLatticeValue join(const WrittenToLatticeValue &lhs,
+                                    const WrittenToLatticeValue &rhs) {
+    // Should not be triggered by this test, but required by `Lattice<T>`
+    llvm_unreachable("Join should not be triggered by this test");
+  }
+
   ChangeResult addWrites(const SetVector<StringAttr> &writes) {
     int sizeBefore = this->writes.size();
     this->writes.insert(writes.begin(), writes.end());
@@ -37,14 +46,26 @@ struct WrittenTo : public AbstractSparseLattice {
     return sizeBefore == sizeAfter ? ChangeResult::NoChange
                                    : ChangeResult::Change;
   }
-  ChangeResult meet(const AbstractSparseLattice &other) override {
-    const auto *rhs = reinterpret_cast<const WrittenTo *>(&other);
-    return addWrites(rhs->writes);
+
+  void print(raw_ostream &os) const {
+    os << "[";
+    llvm::interleave(
+        writes, os, [&](const StringAttr &a) { os << a.str(); }, " ");
+    os << "]";
   }
 
+  void clear() { writes.clear(); }
+
   SetVector<StringAttr> writes;
 };
 
+/// This lattice represents, for a given value, the set of memory resources that
+/// this value, or anything derived from this value, is potentially written to.
+struct WrittenTo : public Lattice<WrittenToLatticeValue> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(WrittenTo)
+  using Lattice::Lattice;
+};
+
 /// An analysis that, by going backwards along the dataflow graph, annotates
 /// each value with all the memory resources it (or anything derived from it)
 /// is eventually written to.
@@ -65,7 +86,9 @@ public:
   void visitExternalCall(CallOpInterface call, ArrayRef<WrittenTo *> operands,
                          ArrayRef<const WrittenTo *> results) override;
 
-  void setToExitState(WrittenTo *lattice) override { lattice->writes.clear(); }
+  void setToExitState(WrittenTo *lattice) override {
+    lattice->getValue().clear();
+  }
 
 private:
   bool assumeFuncWrites;
@@ -77,7 +100,8 @@ void WrittenToAnalysis::visitOperation(Operation *op,
   if (auto store = dyn_cast<memref::StoreOp>(op)) {
     SetVector<StringAttr> newWrites;
     newWrites.insert(op->getAttrOfType<StringAttr>("tag_name"));
-    propagateIfChanged(operands[0], operands[0]->addWrites(newWrites));
+    propagateIfChanged(operands[0],
+                       operands[0]->getValue().addWrites(newWrites));
     return;
   } // By default, every result of an op depends on every operand.
   for (const WrittenTo *r : results) {
@@ -95,7 +119,7 @@ void WrittenToAnalysis::visitBranchOperand(OpOperand &operand) {
   newWrites.insert(
       StringAttr::get(operand.getOwner()->getContext(),
                       "brancharg" + Twine(operand.getOperandNumber())));
-  propagateIfChanged(lattice, lattice->addWrites(newWrites));
+  propagateIfChanged(lattice, lattice->getValue().addWrites(newWrites));
 }
 
 void WrittenToAnalysis::visitCallOperand(OpOperand &operand) {
@@ -105,7 +129,7 @@ void WrittenToAnalysis::visitCallOperand(OpOperand &operand) {
   newWrites.insert(
       StringAttr::get(operand.getOwner()->getContext(),
                       "callarg" + Twine(operand.getOperandNumber())));
-  propagateIfChanged(lattice, lattice->addWrites(newWrites));
+  propagateIfChanged(lattice, lattice->getValue().addWrites(newWrites));
 }
 
 void WrittenToAnalysis::visitExternalCall(CallOpInterface call,
@@ -124,7 +148,7 @@ void WrittenToAnalysis::visitExternalCall(CallOpInterface call,
                              call.getOperation()->getName().getStringRef());
     }
     newWrites.insert(name);
-    propagateIfChanged(lattice, lattice->addWrites(newWrites));
+    propagateIfChanged(lattice, lattice->getValue().addWrites(newWrites));
   }
 }
 
-- 
GitLab


From a2590e0c145c56928a8870d9a6ea76ccbf4fcfeb Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Fri, 17 May 2024 13:44:00 +0200
Subject: [PATCH 1116/1206] [mlir][Transforms] Make 1:N function conversion
 pattern interface-based (#92395)

This commit turns the 1:N dialect conversion pattern for function
signatures into a pattern for `FunctionOpInterface`. This is similar to
the interface-based pattern that is provided with the 1:1 dialect
conversion (`populateFunctionOpInterfaceTypeConversionPattern`). No
change in functionality apart from supporting all `FunctionOpInterface`
ops and not just `func::FuncOp`.
---
 .../mlir/Transforms/OneToNTypeConversion.h    | 14 +++++
 .../Func/Transforms/OneToNFuncConversions.cpp | 47 +--------------
 .../Transforms/Utils/OneToNTypeConversion.cpp | 59 +++++++++++++++++++
 3 files changed, 75 insertions(+), 45 deletions(-)

diff --git a/mlir/include/mlir/Transforms/OneToNTypeConversion.h b/mlir/include/mlir/Transforms/OneToNTypeConversion.h
index 933961814cbe..4c689ba219e8 100644
--- a/mlir/include/mlir/Transforms/OneToNTypeConversion.h
+++ b/mlir/include/mlir/Transforms/OneToNTypeConversion.h
@@ -297,6 +297,20 @@ LogicalResult
 applyPartialOneToNConversion(Operation *op, OneToNTypeConverter &typeConverter,
                              const FrozenRewritePatternSet &patterns);
 
+/// Add a pattern to the given pattern list to convert the signature of a
+/// FunctionOpInterface op with the given type converter. This only supports
+/// ops which use FunctionType to represent their type. This is intended to be
+/// used with the 1:N dialect conversion.
+void populateOneToNFunctionOpInterfaceTypeConversionPattern(
+    StringRef functionLikeOpName, TypeConverter &converter,
+    RewritePatternSet &patterns);
+template <typename FuncOpT>
+void populateOneToNFunctionOpInterfaceTypeConversionPattern(
+    TypeConverter &converter, RewritePatternSet &patterns) {
+  populateOneToNFunctionOpInterfaceTypeConversionPattern(
+      FuncOpT::getOperationName(), converter, patterns);
+}
+
 } // namespace mlir
 
 #endif // MLIR_TRANSFORMS_ONETONTYPECONVERSION_H
diff --git a/mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp b/mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp
index a5b88338e638..8489396da7a2 100644
--- a/mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp
+++ b/mlir/lib/Dialect/Func/Transforms/OneToNFuncConversions.cpp
@@ -49,50 +49,6 @@ public:
   }
 };
 
-class ConvertTypesInFuncFuncOp : public OneToNOpConversionPattern<FuncOp> {
-public:
-  using OneToNOpConversionPattern<FuncOp>::OneToNOpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(FuncOp op, OpAdaptor adaptor,
-                  OneToNPatternRewriter &rewriter) const override {
-    auto *typeConverter = getTypeConverter<OneToNTypeConverter>();
-
-    // Construct mapping for function arguments.
-    OneToNTypeMapping argumentMapping(op.getArgumentTypes());
-    if (failed(typeConverter->computeTypeMapping(op.getArgumentTypes(),
-                                                 argumentMapping)))
-      return failure();
-
-    // Construct mapping for function results.
-    OneToNTypeMapping funcResultMapping(op.getResultTypes());
-    if (failed(typeConverter->computeTypeMapping(op.getResultTypes(),
-                                                 funcResultMapping)))
-      return failure();
-
-    // Nothing to do if the op doesn't have any non-identity conversions for its
-    // operands or results.
-    if (!argumentMapping.hasNonIdentityConversion() &&
-        !funcResultMapping.hasNonIdentityConversion())
-      return failure();
-
-    // Update the function signature in-place.
-    auto newType = FunctionType::get(rewriter.getContext(),
-                                     argumentMapping.getConvertedTypes(),
-                                     funcResultMapping.getConvertedTypes());
-    rewriter.modifyOpInPlace(op, [&] { op.setType(newType); });
-
-    // Update block signatures.
-    if (!op.isExternal()) {
-      Region *region = &op.getBody();
-      Block *block = &region->front();
-      rewriter.applySignatureConversion(block, argumentMapping);
-    }
-
-    return success();
-  }
-};
-
 class ConvertTypesInFuncReturnOp : public OneToNOpConversionPattern<ReturnOp> {
 public:
   using OneToNOpConversionPattern<ReturnOp>::OneToNOpConversionPattern;
@@ -121,10 +77,11 @@ void populateFuncTypeConversionPatterns(TypeConverter &typeConverter,
   patterns.add<
       // clang-format off
       ConvertTypesInFuncCallOp,
-      ConvertTypesInFuncFuncOp,
       ConvertTypesInFuncReturnOp
       // clang-format on
       >(typeConverter, patterns.getContext());
+  populateOneToNFunctionOpInterfaceTypeConversionPattern<func::FuncOp>(
+      typeConverter, patterns);
 }
 
 } // namespace mlir
diff --git a/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp b/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp
index fef9d8eb0fef..f6e8e9e7ad33 100644
--- a/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp
+++ b/mlir/lib/Transforms/Utils/OneToNTypeConversion.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Transforms/OneToNTypeConversion.h"
 
+#include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/SmallSet.h"
 
@@ -412,4 +413,62 @@ applyPartialOneToNConversion(Operation *op, OneToNTypeConverter &typeConverter,
   return success();
 }
 
+namespace {
+class FunctionOpInterfaceSignatureConversion : public OneToNConversionPattern {
+public:
+  FunctionOpInterfaceSignatureConversion(StringRef functionLikeOpName,
+                                         MLIRContext *ctx,
+                                         TypeConverter &converter)
+      : OneToNConversionPattern(converter, functionLikeOpName, /*benefit=*/1,
+                                ctx) {}
+
+  LogicalResult matchAndRewrite(Operation *op, OneToNPatternRewriter &rewriter,
+                                const OneToNTypeMapping &operandMapping,
+                                const OneToNTypeMapping &resultMapping,
+                                ValueRange convertedOperands) const override {
+    auto funcOp = cast<FunctionOpInterface>(op);
+    auto *typeConverter = getTypeConverter<OneToNTypeConverter>();
+
+    // Construct mapping for function arguments.
+    OneToNTypeMapping argumentMapping(funcOp.getArgumentTypes());
+    if (failed(typeConverter->computeTypeMapping(funcOp.getArgumentTypes(),
+                                                 argumentMapping)))
+      return failure();
+
+    // Construct mapping for function results.
+    OneToNTypeMapping funcResultMapping(funcOp.getResultTypes());
+    if (failed(typeConverter->computeTypeMapping(funcOp.getResultTypes(),
+                                                 funcResultMapping)))
+      return failure();
+
+    // Nothing to do if the op doesn't have any non-identity conversions for its
+    // operands or results.
+    if (!argumentMapping.hasNonIdentityConversion() &&
+        !funcResultMapping.hasNonIdentityConversion())
+      return failure();
+
+    // Update the function signature in-place.
+    auto newType = FunctionType::get(rewriter.getContext(),
+                                     argumentMapping.getConvertedTypes(),
+                                     funcResultMapping.getConvertedTypes());
+    rewriter.modifyOpInPlace(op, [&] { funcOp.setType(newType); });
+
+    // Update block signatures.
+    if (!funcOp.isExternal()) {
+      Region *region = &funcOp.getFunctionBody();
+      Block *block = &region->front();
+      rewriter.applySignatureConversion(block, argumentMapping);
+    }
+
+    return success();
+  }
+};
+} // namespace
+
+void populateOneToNFunctionOpInterfaceTypeConversionPattern(
+    StringRef functionLikeOpName, TypeConverter &converter,
+    RewritePatternSet &patterns) {
+  patterns.add<FunctionOpInterfaceSignatureConversion>(
+      functionLikeOpName, patterns.getContext(), converter);
+}
 } // namespace mlir
-- 
GitLab


From 98deedae929fce01f0c40d5a55d00e93fb121981 Mon Sep 17 00:00:00 2001
From: Shengchen Kan <shengchen.kan@intel.com>
Date: Fri, 17 May 2024 16:24:20 +0800
Subject: [PATCH 1117/1206] [X86][CodeGen] Use switch-case for transform in
 X86DAGToDAGISel::PostprocessISelDAG, NFCI

This is to simplify code for #91747
---
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 227 +++++++++++++-----------
 1 file changed, 120 insertions(+), 107 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index ea3b84d0ca9e..7f76324fa570 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -1549,136 +1549,147 @@ void X86DAGToDAGISel::PostprocessISelDAG() {
       continue;
     }
 
-    // Look for a TESTrr+ANDrr pattern where both operands of the test are
-    // the same. Rewrite to remove the AND.
     unsigned Opc = N->getMachineOpcode();
-    if ((Opc == X86::TEST8rr || Opc == X86::TEST16rr ||
-         Opc == X86::TEST32rr || Opc == X86::TEST64rr) &&
-        N->getOperand(0) == N->getOperand(1) &&
-        N->getOperand(0)->hasNUsesOfValue(2, N->getOperand(0).getResNo()) &&
-        N->getOperand(0).isMachineOpcode()) {
-      SDValue And = N->getOperand(0);
-      unsigned N0Opc = And.getMachineOpcode();
-      if ((N0Opc == X86::AND8rr || N0Opc == X86::AND16rr ||
-           N0Opc == X86::AND32rr || N0Opc == X86::AND64rr ||
-           N0Opc == X86::AND8rr_ND || N0Opc == X86::AND16rr_ND ||
-           N0Opc == X86::AND32rr_ND || N0Opc == X86::AND64rr_ND) &&
-          !And->hasAnyUseOfValue(1)) {
-        MachineSDNode *Test = CurDAG->getMachineNode(Opc, SDLoc(N),
-                                                     MVT::i32,
-                                                     And.getOperand(0),
-                                                     And.getOperand(1));
-        ReplaceUses(N, Test);
-        MadeChange = true;
+    switch (Opc) {
+    default:
+      continue;
+    // TESTrr+ANDrr/rm -> TESTrr/TESTmr
+    case X86::TEST8rr:
+    case X86::TEST16rr:
+    case X86::TEST32rr:
+    case X86::TEST64rr: {
+      auto &Op0 = N->getOperand(0);
+      if (Op0 != N->getOperand(1) || !Op0->hasNUsesOfValue(2, Op0.getResNo()) ||
+          !Op0.isMachineOpcode())
         continue;
-      }
-      if ((N0Opc == X86::AND8rm || N0Opc == X86::AND16rm ||
-           N0Opc == X86::AND32rm || N0Opc == X86::AND64rm ||
-           N0Opc == X86::AND8rm_ND || N0Opc == X86::AND16rm_ND ||
-           N0Opc == X86::AND32rm_ND || N0Opc == X86::AND64rm_ND) &&
-          !And->hasAnyUseOfValue(1)) {
-        unsigned NewOpc;
+      SDValue And = N->getOperand(0);
 #define CASE_ND(OP)                                                            \
   case X86::OP:                                                                \
   case X86::OP##_ND:
+      switch (And.getMachineOpcode()) {
+      default:
+        continue;
+        CASE_ND(AND8rr)
+        CASE_ND(AND16rr)
+        CASE_ND(AND32rr)
+        CASE_ND(AND64rr) {
+          if (And->hasAnyUseOfValue(1))
+            continue;
+          MachineSDNode *Test = CurDAG->getMachineNode(
+              Opc, SDLoc(N), MVT::i32, And.getOperand(0), And.getOperand(1));
+          ReplaceUses(N, Test);
+          MadeChange = true;
+          continue;
+        }
+        CASE_ND(AND8rm)
+        CASE_ND(AND16rm)
+        CASE_ND(AND32rm)
+        CASE_ND(AND64rm) {
+          if (And->hasAnyUseOfValue(1))
+            continue;
+          unsigned NewOpc;
 #define FROM_TO(A, B)                                                          \
   CASE_ND(A) NewOpc = X86::B;                                                  \
   break;
-        switch (N0Opc) {
-          FROM_TO(AND8rm, TEST8mr);
-          FROM_TO(AND16rm, TEST16mr);
-          FROM_TO(AND32rm, TEST32mr);
-          FROM_TO(AND64rm, TEST64mr);
-        }
+          switch (And.getMachineOpcode()) {
+            FROM_TO(AND8rm, TEST8mr);
+            FROM_TO(AND16rm, TEST16mr);
+            FROM_TO(AND32rm, TEST32mr);
+            FROM_TO(AND64rm, TEST64mr);
+          }
 #undef FROM_TO
 #undef CASE_ND
-
-        // Need to swap the memory and register operand.
-        SDValue Ops[] = { And.getOperand(1),
-                          And.getOperand(2),
-                          And.getOperand(3),
-                          And.getOperand(4),
-                          And.getOperand(5),
-                          And.getOperand(0),
-                          And.getOperand(6)  /* Chain */ };
-        MachineSDNode *Test = CurDAG->getMachineNode(NewOpc, SDLoc(N),
-                                                     MVT::i32, MVT::Other, Ops);
-        CurDAG->setNodeMemRefs(
-            Test, cast<MachineSDNode>(And.getNode())->memoperands());
-        ReplaceUses(And.getValue(2), SDValue(Test, 1));
-        ReplaceUses(SDValue(N, 0), SDValue(Test, 0));
-        MadeChange = true;
-        continue;
+          // Need to swap the memory and register operand.
+          SDValue Ops[] = {And.getOperand(1), And.getOperand(2),
+                           And.getOperand(3), And.getOperand(4),
+                           And.getOperand(5), And.getOperand(0),
+                           And.getOperand(6) /* Chain */};
+          MachineSDNode *Test = CurDAG->getMachineNode(
+              NewOpc, SDLoc(N), MVT::i32, MVT::Other, Ops);
+          CurDAG->setNodeMemRefs(
+              Test, cast<MachineSDNode>(And.getNode())->memoperands());
+          ReplaceUses(And.getValue(2), SDValue(Test, 1));
+          ReplaceUses(SDValue(N, 0), SDValue(Test, 0));
+          MadeChange = true;
+          continue;
+        }
       }
     }
-
     // Look for a KAND+KORTEST and turn it into KTEST if only the zero flag is
     // used. We're doing this late so we can prefer to fold the AND into masked
     // comparisons. Doing that can be better for the live range of the mask
     // register.
-    if ((Opc == X86::KORTESTBrr || Opc == X86::KORTESTWrr ||
-         Opc == X86::KORTESTDrr || Opc == X86::KORTESTQrr) &&
-        N->getOperand(0) == N->getOperand(1) &&
-        N->isOnlyUserOf(N->getOperand(0).getNode()) &&
-        N->getOperand(0).isMachineOpcode() &&
-        onlyUsesZeroFlag(SDValue(N, 0))) {
-      SDValue And = N->getOperand(0);
-      unsigned N0Opc = And.getMachineOpcode();
+    case X86::KORTESTBrr:
+    case X86::KORTESTWrr:
+    case X86::KORTESTDrr:
+    case X86::KORTESTQrr: {
+      SDValue Op0 = N->getOperand(0);
+      if (Op0 != N->getOperand(1) || !N->isOnlyUserOf(Op0.getNode()) ||
+          !Op0.isMachineOpcode() || !onlyUsesZeroFlag(SDValue(N, 0)))
+        continue;
+#define CASE(A)                                                                \
+  case X86::A:                                                                 \
+    break;
+      switch (Op0.getMachineOpcode()) {
+      default:
+        continue;
+        CASE(KANDBrr)
+        CASE(KANDWrr)
+        CASE(KANDDrr)
+        CASE(KANDQrr)
+      }
+      unsigned NewOpc;
+#define FROM_TO(A, B)                                                          \
+  case X86::A:                                                                 \
+    NewOpc = X86::B;                                                           \
+    break;
+      switch (Opc) {
+        FROM_TO(KORTESTBrr, KTESTBrr)
+        FROM_TO(KORTESTWrr, KTESTWrr)
+        FROM_TO(KORTESTDrr, KTESTDrr)
+        FROM_TO(KORTESTQrr, KTESTQrr)
+      }
       // KANDW is legal with AVX512F, but KTESTW requires AVX512DQ. The other
       // KAND instructions and KTEST use the same ISA feature.
-      if (N0Opc == X86::KANDBrr ||
-          (N0Opc == X86::KANDWrr && Subtarget->hasDQI()) ||
-          N0Opc == X86::KANDDrr || N0Opc == X86::KANDQrr) {
-        unsigned NewOpc;
-        switch (Opc) {
-        default: llvm_unreachable("Unexpected opcode!");
-        case X86::KORTESTBrr: NewOpc = X86::KTESTBrr; break;
-        case X86::KORTESTWrr: NewOpc = X86::KTESTWrr; break;
-        case X86::KORTESTDrr: NewOpc = X86::KTESTDrr; break;
-        case X86::KORTESTQrr: NewOpc = X86::KTESTQrr; break;
-        }
-        MachineSDNode *KTest = CurDAG->getMachineNode(NewOpc, SDLoc(N),
-                                                      MVT::i32,
-                                                      And.getOperand(0),
-                                                      And.getOperand(1));
-        ReplaceUses(N, KTest);
-        MadeChange = true;
+      if (NewOpc == X86::KTESTWrr && !Subtarget->hasDQI())
         continue;
-      }
+#undef FROM_TO
+      MachineSDNode *KTest = CurDAG->getMachineNode(
+          NewOpc, SDLoc(N), MVT::i32, Op0.getOperand(0), Op0.getOperand(1));
+      ReplaceUses(N, KTest);
+      MadeChange = true;
+      continue;
     }
-
     // Attempt to remove vectors moves that were inserted to zero upper bits.
-    if (Opc != TargetOpcode::SUBREG_TO_REG)
-      continue;
-
-    unsigned SubRegIdx = N->getConstantOperandVal(2);
-    if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
-      continue;
+    case TargetOpcode::SUBREG_TO_REG: {
+      unsigned SubRegIdx = N->getConstantOperandVal(2);
+      if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
+        continue;
 
-    SDValue Move = N->getOperand(1);
-    if (!Move.isMachineOpcode())
-      continue;
+      SDValue Move = N->getOperand(1);
+      if (!Move.isMachineOpcode())
+        continue;
 
-    // Make sure its one of the move opcodes we recognize.
-    switch (Move.getMachineOpcode()) {
-    default:
-      continue;
-    case X86::VMOVAPDrr:       case X86::VMOVUPDrr:
-    case X86::VMOVAPSrr:       case X86::VMOVUPSrr:
-    case X86::VMOVDQArr:       case X86::VMOVDQUrr:
-    case X86::VMOVAPDYrr:      case X86::VMOVUPDYrr:
-    case X86::VMOVAPSYrr:      case X86::VMOVUPSYrr:
-    case X86::VMOVDQAYrr:      case X86::VMOVDQUYrr:
-    case X86::VMOVAPDZ128rr:   case X86::VMOVUPDZ128rr:
-    case X86::VMOVAPSZ128rr:   case X86::VMOVUPSZ128rr:
-    case X86::VMOVDQA32Z128rr: case X86::VMOVDQU32Z128rr:
-    case X86::VMOVDQA64Z128rr: case X86::VMOVDQU64Z128rr:
-    case X86::VMOVAPDZ256rr:   case X86::VMOVUPDZ256rr:
-    case X86::VMOVAPSZ256rr:   case X86::VMOVUPSZ256rr:
-    case X86::VMOVDQA32Z256rr: case X86::VMOVDQU32Z256rr:
-    case X86::VMOVDQA64Z256rr: case X86::VMOVDQU64Z256rr:
-      break;
-    }
+      // Make sure its one of the move opcodes we recognize.
+      switch (Move.getMachineOpcode()) {
+      default:
+        continue;
+        CASE(VMOVAPDrr)       CASE(VMOVUPDrr)
+        CASE(VMOVAPSrr)       CASE(VMOVUPSrr)
+        CASE(VMOVDQArr)       CASE(VMOVDQUrr)
+        CASE(VMOVAPDYrr)      CASE(VMOVUPDYrr)
+        CASE(VMOVAPSYrr)      CASE(VMOVUPSYrr)
+        CASE(VMOVDQAYrr)      CASE(VMOVDQUYrr)
+        CASE(VMOVAPDZ128rr)   CASE(VMOVUPDZ128rr)
+        CASE(VMOVAPSZ128rr)   CASE(VMOVUPSZ128rr)
+        CASE(VMOVDQA32Z128rr) CASE(VMOVDQU32Z128rr)
+        CASE(VMOVDQA64Z128rr) CASE(VMOVDQU64Z128rr)
+        CASE(VMOVAPDZ256rr)   CASE(VMOVUPDZ256rr)
+        CASE(VMOVAPSZ256rr)   CASE(VMOVUPSZ256rr)
+        CASE(VMOVDQA32Z256rr) CASE(VMOVDQU32Z256rr)
+        CASE(VMOVDQA64Z256rr) CASE(VMOVDQU64Z256rr)
+      }
+#undef CASE
 
     SDValue In = Move.getOperand(0);
     if (!In.isMachineOpcode() ||
@@ -1697,6 +1708,8 @@ void X86DAGToDAGISel::PostprocessISelDAG() {
     // move.
     CurDAG->UpdateNodeOperands(N, N->getOperand(0), In, N->getOperand(2));
     MadeChange = true;
+    }
+    }
   }
 
   if (MadeChange)
-- 
GitLab


From 0ad275c1588065c9f4ef45a6a88f062182ad446b Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Fri, 17 May 2024 12:58:14 +0100
Subject: [PATCH 1118/1206] [InstCombine] Fold
 vector.reduce.op(vector.reverse(X)) -> vector.reduce.op(X) (#91743)

For all of the following reductions:

vector.reduce.or
vector.reduce.and
vector.reduce.xor
vector.reduce.add
vector.reduce.mul
vector.reduce.umin
vector.reduce.umax
vector.reduce.smin
vector.reduce.smax
vector.reduce.fmin
vector.reduce.fmax

if the input operand is the result of a vector.reverse then we can
perform a reduction on the vector.reverse input instead since the answer
is the same. If the reassociation is permitted we can also do the same
folds for these:

vector.reduce.fadd
vector.reduce.fmul
---
 .../InstCombine/InstCombineCalls.cpp          |  94 +++++++---
 .../InstCombine/vector-logical-reductions.ll  |  72 ++++++++
 .../InstCombine/vector-reductions.ll          | 162 ++++++++++++++++++
 3 files changed, 308 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 77534e0d3613..cdec8c8c7c80 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1435,6 +1435,33 @@ static Instruction *foldBitOrderCrossLogicOp(Value *V,
   return nullptr;
 }
 
+static Value *simplifyReductionOperand(Value *Arg, bool CanReorderLanes) {
+  if (!CanReorderLanes)
+    return nullptr;
+
+  Value *V;
+  if (match(Arg, m_VecReverse(m_Value(V))))
+    return V;
+
+  ArrayRef<int> Mask;
+  if (!isa<FixedVectorType>(Arg->getType()) ||
+      !match(Arg, m_Shuffle(m_Value(V), m_Undef(), m_Mask(Mask))) ||
+      !cast<ShuffleVectorInst>(Arg)->isSingleSource())
+    return nullptr;
+
+  int Sz = Mask.size();
+  SmallBitVector UsedIndices(Sz);
+  for (int Idx : Mask) {
+    if (Idx == PoisonMaskElem || UsedIndices.test(Idx))
+      return nullptr;
+    UsedIndices.set(Idx);
+  }
+
+  // Can remove shuffle iff just shuffled elements, no repeats, undefs, or
+  // other changes.
+  return UsedIndices.all() ? V : nullptr;
+}
+
 /// CallInst simplification. This mostly only handles folding of intrinsic
 /// instructions. For normal calls, it allows visitCallBase to do the heavy
 /// lifting.
@@ -3223,6 +3250,13 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     // %res = cmp eq iReduxWidth %val, 11111
     Value *Arg = II->getArgOperand(0);
     Value *Vect;
+
+    if (Value *NewOp =
+            simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
+      replaceUse(II->getOperandUse(0), NewOp);
+      return II;
+    }
+
     if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
       if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
         if (FTy->getElementType() == Builder.getInt1Ty()) {
@@ -3254,6 +3288,13 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       // Trunc(ctpop(bitcast <n x i1> to in)).
       Value *Arg = II->getArgOperand(0);
       Value *Vect;
+
+      if (Value *NewOp =
+              simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
+        replaceUse(II->getOperandUse(0), NewOp);
+        return II;
+      }
+
       if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
         if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
           if (FTy->getElementType() == Builder.getInt1Ty()) {
@@ -3282,6 +3323,13 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       //   ?ext(vector_reduce_add(<n x i1>))
       Value *Arg = II->getArgOperand(0);
       Value *Vect;
+
+      if (Value *NewOp =
+              simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
+        replaceUse(II->getOperandUse(0), NewOp);
+        return II;
+      }
+
       if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
         if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
           if (FTy->getElementType() == Builder.getInt1Ty()) {
@@ -3305,6 +3353,13 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       //   zext(vector_reduce_and(<n x i1>))
       Value *Arg = II->getArgOperand(0);
       Value *Vect;
+
+      if (Value *NewOp =
+              simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
+        replaceUse(II->getOperandUse(0), NewOp);
+        return II;
+      }
+
       if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
         if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
           if (FTy->getElementType() == Builder.getInt1Ty()) {
@@ -3329,6 +3384,13 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       //   ?ext(vector_reduce_{and,or}(<n x i1>))
       Value *Arg = II->getArgOperand(0);
       Value *Vect;
+
+      if (Value *NewOp =
+              simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
+        replaceUse(II->getOperandUse(0), NewOp);
+        return II;
+      }
+
       if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
         if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
           if (FTy->getElementType() == Builder.getInt1Ty()) {
@@ -3364,6 +3426,13 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       //   zext(vector_reduce_{and,or}(<n x i1>))
       Value *Arg = II->getArgOperand(0);
       Value *Vect;
+
+      if (Value *NewOp =
+              simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
+        replaceUse(II->getOperandUse(0), NewOp);
+        return II;
+      }
+
       if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
         if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
           if (FTy->getElementType() == Builder.getInt1Ty()) {
@@ -3386,31 +3455,16 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
   case Intrinsic::vector_reduce_fmin:
   case Intrinsic::vector_reduce_fadd:
   case Intrinsic::vector_reduce_fmul: {
-    bool CanBeReassociated = (IID != Intrinsic::vector_reduce_fadd &&
-                              IID != Intrinsic::vector_reduce_fmul) ||
-                             II->hasAllowReassoc();
+    bool CanReorderLanes = (IID != Intrinsic::vector_reduce_fadd &&
+                            IID != Intrinsic::vector_reduce_fmul) ||
+                           II->hasAllowReassoc();
     const unsigned ArgIdx = (IID == Intrinsic::vector_reduce_fadd ||
                              IID == Intrinsic::vector_reduce_fmul)
                                 ? 1
                                 : 0;
     Value *Arg = II->getArgOperand(ArgIdx);
-    Value *V;
-    ArrayRef<int> Mask;
-    if (!isa<FixedVectorType>(Arg->getType()) || !CanBeReassociated ||
-        !match(Arg, m_Shuffle(m_Value(V), m_Undef(), m_Mask(Mask))) ||
-        !cast<ShuffleVectorInst>(Arg)->isSingleSource())
-      break;
-    int Sz = Mask.size();
-    SmallBitVector UsedIndices(Sz);
-    for (int Idx : Mask) {
-      if (Idx == PoisonMaskElem || UsedIndices.test(Idx))
-        break;
-      UsedIndices.set(Idx);
-    }
-    // Can remove shuffle iff just shuffled elements, no repeats, undefs, or
-    // other changes.
-    if (UsedIndices.all()) {
-      replaceUse(II->getOperandUse(ArgIdx), V);
+    if (Value *NewOp = simplifyReductionOperand(Arg, CanReorderLanes)) {
+      replaceUse(II->getOperandUse(ArgIdx), NewOp);
       return nullptr;
     }
     break;
diff --git a/llvm/test/Transforms/InstCombine/vector-logical-reductions.ll b/llvm/test/Transforms/InstCombine/vector-logical-reductions.ll
index 9bb307ebf71e..da4a0ca75468 100644
--- a/llvm/test/Transforms/InstCombine/vector-logical-reductions.ll
+++ b/llvm/test/Transforms/InstCombine/vector-logical-reductions.ll
@@ -21,5 +21,77 @@ define i1 @reduction_logical_and(<4 x i1> %x) {
   ret i1 %r
 }
 
+define i1 @reduction_logical_or_reverse_nxv2i1(<vscale x 2 x i1> %p) {
+; CHECK-LABEL: @reduction_logical_or_reverse_nxv2i1(
+; CHECK-NEXT:    [[RED:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[P:%.*]])
+; CHECK-NEXT:    ret i1 [[RED]]
+;
+  %rev = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %p)
+  %red = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> %rev)
+  ret i1 %red
+}
+
+define i1 @reduction_logical_or_reverse_v2i1(<2 x i1> %p) {
+; CHECK-LABEL: @reduction_logical_or_reverse_v2i1(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i1> [[P:%.*]] to i2
+; CHECK-NEXT:    [[RED:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[RED]]
+;
+  %rev = call <2 x i1> @llvm.vector.reverse.v2i1(<2 x i1> %p)
+  %red = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> %rev)
+  ret i1 %red
+}
+
+define i1 @reduction_logical_and_reverse_nxv2i1(<vscale x 2 x i1> %p) {
+; CHECK-LABEL: @reduction_logical_and_reverse_nxv2i1(
+; CHECK-NEXT:    [[RED:%.*]] = call i1 @llvm.vector.reduce.and.nxv2i1(<vscale x 2 x i1> [[P:%.*]])
+; CHECK-NEXT:    ret i1 [[RED]]
+;
+  %rev = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %p)
+  %red = call i1 @llvm.vector.reduce.and.nxv2i1(<vscale x 2 x i1> %rev)
+  ret i1 %red
+}
+
+define i1 @reduction_logical_and_reverse_v2i1(<2 x i1> %p) {
+; CHECK-LABEL: @reduction_logical_and_reverse_v2i1(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i1> [[P:%.*]] to i2
+; CHECK-NEXT:    [[RED:%.*]] = icmp eq i2 [[TMP1]], -1
+; CHECK-NEXT:    ret i1 [[RED]]
+;
+  %rev = call <2 x i1> @llvm.vector.reverse.v2i1(<2 x i1> %p)
+  %red = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> %rev)
+  ret i1 %red
+}
+
+define i1 @reduction_logical_xor_reverse_nxv2i1(<vscale x 2 x i1> %p) {
+; CHECK-LABEL: @reduction_logical_xor_reverse_nxv2i1(
+; CHECK-NEXT:    [[RED:%.*]] = call i1 @llvm.vector.reduce.xor.nxv2i1(<vscale x 2 x i1> [[P:%.*]])
+; CHECK-NEXT:    ret i1 [[RED]]
+;
+  %rev = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> %p)
+  %red = call i1 @llvm.vector.reduce.xor.nxv2i1(<vscale x 2 x i1> %rev)
+  ret i1 %red
+}
+
+define i1 @reduction_logical_xor_reverse_v2i1(<2 x i1> %p) {
+; CHECK-LABEL: @reduction_logical_xor_reverse_v2i1(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i1> [[P:%.*]] to i2
+; CHECK-NEXT:    [[TMP2:%.*]] = call range(i2 0, -1) i2 @llvm.ctpop.i2(i2 [[TMP1]])
+; CHECK-NEXT:    [[RED:%.*]] = trunc i2 [[TMP2]] to i1
+; CHECK-NEXT:    ret i1 [[RED]]
+;
+  %rev = call <2 x i1> @llvm.vector.reverse.v2i1(<2 x i1> %p)
+  %red = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> %rev)
+  ret i1 %red
+}
+
 declare i1 @llvm.vector.reduce.or.v4i1(<4 x i1>)
+declare i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1>)
+declare i1 @llvm.vector.reduce.or.v2i1(<2 x i1>)
 declare i1 @llvm.vector.reduce.and.v4i1(<4 x i1>)
+declare i1 @llvm.vector.reduce.and.nxv2i1(<vscale x 2 x i1>)
+declare i1 @llvm.vector.reduce.and.v2i1(<2 x i1>)
+declare i1 @llvm.vector.reduce.xor.nxv2i1(<vscale x 2 x i1>)
+declare i1 @llvm.vector.reduce.xor.v2i1(<2 x i1>)
+declare <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1>)
+declare <2 x i1> @llvm.vector.reverse.v2i1(<2 x i1>)
diff --git a/llvm/test/Transforms/InstCombine/vector-reductions.ll b/llvm/test/Transforms/InstCombine/vector-reductions.ll
index 2614ffd38695..10f4aca72dbc 100644
--- a/llvm/test/Transforms/InstCombine/vector-reductions.ll
+++ b/llvm/test/Transforms/InstCombine/vector-reductions.ll
@@ -3,12 +3,29 @@
 
 declare float @llvm.vector.reduce.fadd.f32.v4f32(float, <4 x float>)
 declare float @llvm.vector.reduce.fadd.f32.v8f32(float, <8 x float>)
+declare float @llvm.vector.reduce.fmul.f32.nxv4f32(float, <vscale x 4 x float>)
+declare float @llvm.vector.reduce.fmin.f32.v4f32(float, <4 x float>)
+declare float @llvm.vector.reduce.fmax.f32.nxv4f32(float, <vscale x 4 x float>)
 declare void @use_f32(float)
 
 declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32>)
 declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
 declare void @use_i32(i32)
 
+declare i32 @llvm.vector.reduce.mul.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32>)
+
+declare i32 @llvm.vector.reduce.smin.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32>)
+declare i32 @llvm.vector.reduce.umin.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32>)
+
+declare <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float>)
+declare <4 x i32> @llvm.vector.reverse.v4i32(<4 x i32>)
+declare <4 x float> @llvm.vector.reverse.v4f32(<4 x float>)
+
 define float @diff_of_sums_v4f32(float %a0, <4 x float> %v0, float %a1, <4 x float> %v1) {
 ; CHECK-LABEL: @diff_of_sums_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc nsz <4 x float> [[V0:%.*]], [[V1:%.*]]
@@ -22,6 +39,71 @@ define float @diff_of_sums_v4f32(float %a0, <4 x float> %v0, float %a1, <4 x flo
   ret float %r
 }
 
+define float @reassoc_sum_of_reverse_v4f32(<4 x float> %v0) {
+; CHECK-LABEL: @reassoc_sum_of_reverse_v4f32(
+; CHECK-NEXT:    [[RED:%.*]] = call reassoc float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[V0:%.*]])
+; CHECK-NEXT:    ret float [[RED]]
+;
+  %rev = call <4 x float> @llvm.vector.reverse.v4f32(<4 x float> %v0)
+  %red = call reassoc float @llvm.vector.reduce.fadd.v4f32(float zeroinitializer, <4 x float> %rev)
+  ret float %red
+}
+
+define float @reassoc_mul_reduction_of_reverse_nxv4f32(<vscale x 4 x float> %v0) {
+; CHECK-LABEL: @reassoc_mul_reduction_of_reverse_nxv4f32(
+; CHECK-NEXT:    [[RED:%.*]] = call reassoc float @llvm.vector.reduce.fmul.nxv4f32(float 1.000000e+00, <vscale x 4 x float> [[V0:%.*]])
+; CHECK-NEXT:    ret float [[RED]]
+;
+  %rev = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %v0)
+  %red = call reassoc float @llvm.vector.reduce.fmul.nxv4f32(float 1.0, <vscale x 4 x float> %rev)
+  ret float %red
+}
+
+define float @fmax_of_reverse_v4f32(<4 x float> %v0) {
+; CHECK-LABEL: @fmax_of_reverse_v4f32(
+; CHECK-NEXT:    [[RED:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[V0:%.*]])
+; CHECK-NEXT:    ret float [[RED]]
+;
+  %rev = call <4 x float> @llvm.vector.reverse.v4f32(<4 x float> %v0)
+  %red = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> %rev)
+  ret float %red
+}
+
+define float @fmin_of_reverse_nxv4f32(<vscale x 4 x float> %v0) {
+; CHECK-LABEL: @fmin_of_reverse_nxv4f32(
+; CHECK-NEXT:    [[RED:%.*]] = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> [[V0:%.*]])
+; CHECK-NEXT:    ret float [[RED]]
+;
+  %rev = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %v0)
+  %red = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %rev)
+  ret float %red
+}
+
+; negative test - fadd cannot be folded with reverse due to lack of reassoc
+define float @sum_of_reverse_v4f32(<4 x float> %v0) {
+; CHECK-LABEL: @sum_of_reverse_v4f32(
+; CHECK-NEXT:    [[REV:%.*]] = call <4 x float> @llvm.vector.reverse.v4f32(<4 x float> [[V0:%.*]])
+; CHECK-NEXT:    [[RED:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[REV]])
+; CHECK-NEXT:    ret float [[RED]]
+;
+  %rev = call <4 x float> @llvm.vector.reverse.v4f32(<4 x float> %v0)
+  %red = call float @llvm.vector.reduce.fadd.v4f32(float zeroinitializer, <4 x float> %rev)
+  ret float %red
+}
+
+; negative test - fmul cannot be folded with reverse due to lack of reassoc
+define float @mul_reduction_of_reverse_nxv4f32(<vscale x 4 x float> %v0) {
+; CHECK-LABEL: @mul_reduction_of_reverse_nxv4f32(
+; CHECK-NEXT:    [[REV:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[V0:%.*]])
+; CHECK-NEXT:    [[RED:%.*]] = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[REV]])
+; CHECK-NEXT:    ret float [[RED]]
+;
+  %rev = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %v0)
+  %red = call float @llvm.vector.reduce.fmul.nxv4f32(float zeroinitializer, <vscale x 4 x float> %rev)
+  ret float %red
+}
+
+
 ; negative test - fsub must allow reassociation
 
 define float @diff_of_sums_v4f32_fmf(float %a0, <4 x float> %v0, float %a1, <4 x float> %v1) {
@@ -98,6 +180,86 @@ define i32 @diff_of_sums_v4i32(<4 x i32> %v0, <4 x i32> %v1) {
   ret i32 %r
 }
 
+define i32 @sum_of_reverse_v4i32(<4 x i32> %v0) {
+; CHECK-LABEL: @sum_of_reverse_v4i32(
+; CHECK-NEXT:    [[RED:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[V0:%.*]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %rev = call <4 x i32> @llvm.vector.reverse.v4i32(<4 x i32> %v0)
+  %red = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %rev)
+  ret i32 %red
+}
+
+define i32 @sum_of_reverse_nxv4i32(<vscale x 4 x i32> %v0) {
+; CHECK-LABEL: @sum_of_reverse_nxv4i32(
+; CHECK-NEXT:    [[RED:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[V0:%.*]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %rev = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %v0)
+  %red = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %rev)
+  ret i32 %red
+}
+
+define i32 @mul_reduce_of_reverse_v4i32(<4 x i32> %v0) {
+; CHECK-LABEL: @mul_reduce_of_reverse_v4i32(
+; CHECK-NEXT:    [[RED:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[V0:%.*]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %rev = call <4 x i32> @llvm.vector.reverse.v4i32(<4 x i32> %v0)
+  %red = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %rev)
+  ret i32 %red
+}
+
+define i32 @mul_reduce_of_reverse_nxv4i32(<vscale x 4 x i32> %v0) {
+; CHECK-LABEL: @mul_reduce_of_reverse_nxv4i32(
+; CHECK-NEXT:    [[RED:%.*]] = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> [[V0:%.*]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %rev = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %v0)
+  %red = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %rev)
+  ret i32 %red
+}
+
+define i32 @smin_reduce_of_reverse_v4i32(<4 x i32> %v0) {
+; CHECK-LABEL: @smin_reduce_of_reverse_v4i32(
+; CHECK-NEXT:    [[RED:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[V0:%.*]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %rev = call <4 x i32> @llvm.vector.reverse.v4i32(<4 x i32> %v0)
+  %red = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> %rev)
+  ret i32 %red
+}
+
+define i32 @smax_reduce_of_reverse_nxv4i32(<vscale x 4 x i32> %v0) {
+; CHECK-LABEL: @smax_reduce_of_reverse_nxv4i32(
+; CHECK-NEXT:    [[RED:%.*]] = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> [[V0:%.*]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %rev = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %v0)
+  %red = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %rev)
+  ret i32 %red
+}
+
+define i32 @umin_reduce_of_reverse_v4i32(<4 x i32> %v0) {
+; CHECK-LABEL: @umin_reduce_of_reverse_v4i32(
+; CHECK-NEXT:    [[RED:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[V0:%.*]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %rev = call <4 x i32> @llvm.vector.reverse.v4i32(<4 x i32> %v0)
+  %red = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> %rev)
+  ret i32 %red
+}
+
+define i32 @umax_reduce_of_reverse_nxv4i32(<vscale x 4 x i32> %v0) {
+; CHECK-LABEL: @umax_reduce_of_reverse_nxv4i32(
+; CHECK-NEXT:    [[RED:%.*]] = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> [[V0:%.*]])
+; CHECK-NEXT:    ret i32 [[RED]]
+;
+  %rev = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %v0)
+  %red = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %rev)
+  ret i32 %red
+}
+
 ; negative test - extra uses could create extra instructions
 
 define i32 @diff_of_sums_v4i32_extra_use1(<4 x i32> %v0, <4 x i32> %v1) {
-- 
GitLab


From a68d20e986053ec571223a9f3ead3e146a27dc82 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 17 May 2024 16:02:44 +0400
Subject: [PATCH 1119/1206] [clang] Implement CWG2428 "Deprecating a concept"
 (#92295)

This patch allows attributes to be attached to C++20 concepts,
implementing
[CWG2428](https://cplusplus.github.io/CWG/issues/2428.html).
---
 clang/docs/ReleaseNotes.rst           |  3 ++
 clang/include/clang/Sema/Sema.h       |  5 +++-
 clang/lib/Parse/ParseTemplate.cpp     | 17 +++++++----
 clang/lib/Sema/SemaChecking.cpp       |  6 ++++
 clang/lib/Sema/SemaDecl.cpp           |  5 ++++
 clang/lib/Sema/SemaTemplate.cpp       |  8 ++++-
 clang/lib/Sema/SemaType.cpp           |  5 ++++
 clang/test/CXX/drs/cwg24xx.cpp        | 42 +++++++++++++++++++++++++++
 clang/test/SemaCXX/cxx-deprecated.cpp | 12 +++++++-
 clang/www/cxx_dr_status.html          |  2 +-
 10 files changed, 95 insertions(+), 10 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 65d191b6161a..40e41c940b6a 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -226,6 +226,9 @@ Resolutions to C++ Defect Reports
 - Clang now diagnoses declarative nested-name-specifiers with pack-index-specifiers.
   (`CWG2858: Declarative nested-name-specifiers and pack-index-specifiers <https://cplusplus.github.io/CWG/issues/2858.html>`_).
 
+- Clang now allows attributes on concepts.
+  (`CWG2428: Deprecating a concept <https://cplusplus.github.io/CWG/issues/2428.html>`_).
+
 - P0522 implementation is enabled by default in all language versions, and
   provisional wording for CWG2398 is implemented.
 
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 873592a2d430..226987b97570 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -2022,6 +2022,8 @@ public:
   void CheckTCBEnforcement(const SourceLocation CallExprLoc,
                            const NamedDecl *Callee);
 
+  void CheckConstrainedAuto(const AutoType *AutoT, SourceLocation Loc);
+
 private:
   void CheckArrayAccess(const Expr *BaseExpr, const Expr *IndexExpr,
                         const ArraySubscriptExpr *ASE = nullptr,
@@ -9356,7 +9358,8 @@ public:
   Decl *ActOnConceptDefinition(Scope *S,
                                MultiTemplateParamsArg TemplateParameterLists,
                                const IdentifierInfo *Name,
-                               SourceLocation NameLoc, Expr *ConstraintExpr);
+                               SourceLocation NameLoc, Expr *ConstraintExpr,
+                               const ParsedAttributesView &Attrs);
 
   void CheckConceptRedefinition(ConceptDecl *NewDecl, LookupResult &Previous,
                                 bool &AddToScope);
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index 665253a6674d..8f841be422dd 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -167,9 +167,13 @@ Parser::DeclGroupPtrTy Parser::ParseTemplateDeclarationOrSpecialization(
                                   LastParamListWasEmpty);
 
   // Parse the actual template declaration.
-  if (Tok.is(tok::kw_concept))
-    return Actions.ConvertDeclToDeclGroup(
-        ParseConceptDefinition(TemplateInfo, DeclEnd));
+  if (Tok.is(tok::kw_concept)) {
+    Decl *ConceptDecl = ParseConceptDefinition(TemplateInfo, DeclEnd);
+    // We need to explicitly pass ConceptDecl to ParsingDeclRAIIObject, so that
+    // delayed diagnostics (e.g. warn_deprecated) have a Decl to work with.
+    ParsingTemplateParams.complete(ConceptDecl);
+    return Actions.ConvertDeclToDeclGroup(ConceptDecl);
+  }
 
   return ParseDeclarationAfterTemplate(
       Context, TemplateInfo, ParsingTemplateParams, DeclEnd, AccessAttrs, AS);
@@ -316,7 +320,8 @@ Parser::ParseConceptDefinition(const ParsedTemplateInfo &TemplateInfo,
   const IdentifierInfo *Id = Result.Identifier;
   SourceLocation IdLoc = Result.getBeginLoc();
 
-  DiagnoseAndSkipCXX11Attributes();
+  ParsedAttributes Attrs(AttrFactory);
+  MaybeParseAttributes(PAKM_GNU | PAKM_CXX11, Attrs);
 
   if (!TryConsumeToken(tok::equal)) {
     Diag(Tok.getLocation(), diag::err_expected) << tok::equal;
@@ -335,8 +340,8 @@ Parser::ParseConceptDefinition(const ParsedTemplateInfo &TemplateInfo,
   ExpectAndConsumeSemi(diag::err_expected_semi_declaration);
   Expr *ConstraintExpr = ConstraintExprResult.get();
   return Actions.ActOnConceptDefinition(getCurScope(),
-                                        *TemplateInfo.TemplateParams,
-                                        Id, IdLoc, ConstraintExpr);
+                                        *TemplateInfo.TemplateParams, Id, IdLoc,
+                                        ConstraintExpr, Attrs);
 }
 
 /// ParseTemplateParameters - Parses a template-parameter-list enclosed in
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index ecd182165114..f2dc8e9dd005 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -8092,6 +8092,12 @@ void Sema::checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto,
     diagnoseArgDependentDiagnoseIfAttrs(FD, ThisArg, Args, Loc);
 }
 
+void Sema::CheckConstrainedAuto(const AutoType *AutoT, SourceLocation Loc) {
+  if (ConceptDecl *Decl = AutoT->getTypeConstraintConcept()) {
+    DiagnoseUseOfDecl(Decl, Loc);
+  }
+}
+
 /// CheckConstructorCall - Check a constructor call for correctness and safety
 /// properties not enforced by the C type system.
 void Sema::CheckConstructorCall(FunctionDecl *FDecl, QualType ThisType,
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 0dbdf923df95..bb0bee0d234c 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -7573,6 +7573,11 @@ NamedDecl *Sema::ActOnVariableDeclarator(
     tryToFixVariablyModifiedVarType(TInfo, R, D.getIdentifierLoc(),
                                     /*DiagID=*/0);
 
+  if (const AutoType *AutoT = R->getAs<AutoType>())
+    CheckConstrainedAuto(
+        AutoT,
+        TInfo->getTypeLoc().getContainedAutoTypeLoc().getConceptNameLoc());
+
   bool IsMemberSpecialization = false;
   bool IsVariableTemplateSpecialization = false;
   bool IsPartialSpecialization = false;
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 1b3f1e8bf80f..0462d3520037 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -5653,6 +5653,8 @@ Sema::CheckConceptTemplateId(const CXXScopeSpec &SS,
           /*UpdateArgsWithConversions=*/false))
     return ExprError();
 
+  DiagnoseUseOfDecl(NamedConcept, ConceptNameInfo.getLoc());
+
   auto *CSD = ImplicitConceptSpecializationDecl::Create(
       Context, NamedConcept->getDeclContext(), NamedConcept->getLocation(),
       CanonicalConverted);
@@ -9859,7 +9861,8 @@ Decl *Sema::ActOnTemplateDeclarator(Scope *S,
 
 Decl *Sema::ActOnConceptDefinition(
     Scope *S, MultiTemplateParamsArg TemplateParameterLists,
-    const IdentifierInfo *Name, SourceLocation NameLoc, Expr *ConstraintExpr) {
+    const IdentifierInfo *Name, SourceLocation NameLoc, Expr *ConstraintExpr,
+    const ParsedAttributesView &Attrs) {
   DeclContext *DC = CurContext;
 
   if (!DC->getRedeclContext()->isFileContext()) {
@@ -9921,6 +9924,9 @@ Decl *Sema::ActOnConceptDefinition(
   ActOnDocumentableDecl(NewDecl);
   if (AddToScope)
     PushOnScopeChains(NewDecl, S);
+
+  ProcessDeclAttributeList(S, NewDecl, Attrs);
+
   return NewDecl;
 }
 
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 9c002d56b5bd..c19c8cc34dd3 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -6392,6 +6392,11 @@ TypeResult Sema::ActOnTypeName(Declarator &D) {
     CheckExtraCXXDefaultArguments(D);
   }
 
+  if (const AutoType *AutoT = T->getAs<AutoType>())
+    CheckConstrainedAuto(
+        AutoT,
+        TInfo->getTypeLoc().getContainedAutoTypeLoc().getConceptNameLoc());
+
   return CreateParsedType(T, TInfo);
 }
 
diff --git a/clang/test/CXX/drs/cwg24xx.cpp b/clang/test/CXX/drs/cwg24xx.cpp
index 75e1a614765c..4c7b1506e86e 100644
--- a/clang/test/CXX/drs/cwg24xx.cpp
+++ b/clang/test/CXX/drs/cwg24xx.cpp
@@ -45,6 +45,48 @@ void fallthrough(int n) {
 #endif
 }
 
+namespace cwg2428 { // cwg2428: 19
+#if __cplusplus >= 202002L
+template <typename>
+concept C [[deprecated]] = true; // #cwg2428-C
+
+template <typename>
+[[deprecated]] concept C2 = true;
+// expected-error@-1 {{expected unqualified-id}}
+
+template <typename T>
+concept C3 = C<T>;
+// expected-warning@-1 {{'C' is deprecated}}
+//   expected-note@#cwg2428-C {{'C' has been explicitly marked deprecated here}}
+
+template <typename T, C U>
+// expected-warning@-1 {{'C' is deprecated}}
+//   expected-note@#cwg2428-C {{'C' has been explicitly marked deprecated here}}
+requires C<T>
+// expected-warning@-1 {{'C' is deprecated}}
+//   expected-note@#cwg2428-C {{'C' has been explicitly marked deprecated here}}
+void f() {
+  bool b = C<int>;
+  // expected-warning@-1 {{'C' is deprecated}}
+  //   expected-note@#cwg2428-C {{'C' has been explicitly marked deprecated here}}
+};
+
+void g(C auto a) {};
+// expected-warning@-1 {{'C' is deprecated}}
+//   expected-note@#cwg2428-C {{'C' has been explicitly marked deprecated here}}
+
+template <typename T>
+auto h() -> C auto {
+// expected-warning@-1 {{'C' is deprecated}}
+//   expected-note@#cwg2428-C {{'C' has been explicitly marked deprecated here}}
+  C auto foo = T();
+  // expected-warning@-1 {{'C' is deprecated}}
+  //   expected-note@#cwg2428-C {{'C' has been explicitly marked deprecated here}}
+  return foo;
+}
+#endif
+} // namespace cwg2428
+
 namespace cwg2430 { // cwg2430: 2.7
 struct S {
   S f(S s) { return s; }
diff --git a/clang/test/SemaCXX/cxx-deprecated.cpp b/clang/test/SemaCXX/cxx-deprecated.cpp
index aa4501d53197..81eb07608300 100644
--- a/clang/test/SemaCXX/cxx-deprecated.cpp
+++ b/clang/test/SemaCXX/cxx-deprecated.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -std=c++1z %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++20 %s
 
 namespace [[deprecated]] {}  // expected-warning {{'deprecated' attribute on anonymous namespace ignored}}
 
@@ -27,3 +27,13 @@ namespace M = N; // expected-warning {{'N' is deprecated}}
 
 // Shouldn't diag:
 [[nodiscard, deprecated("")]] int PR37935();
+
+namespace cxx20_concept {
+template <typename>
+concept C __attribute__((deprecated)) = true; // #C
+
+template <C T>
+// expected-warning@-1 {{'C' is deprecated}}
+//   expected-note@#C {{'C' has been explicitly marked deprecated here}}
+void f();
+} // namespace cxx20_concept
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 2ed79fa21db2..5d517d358672 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -14376,7 +14376,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/2428.html">2428</a></td>
     <td>C++23</td>
     <td>Deprecating a concept</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="unreleased" align="center">Clang 19</td>
   </tr>
   <tr id="2429">
     <td><a href="https://cplusplus.github.io/CWG/issues/2429.html">2429</a></td>
-- 
GitLab


From 371eccd5dfed88c8e76449233d8388c12be3464b Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Fri, 17 May 2024 08:06:46 -0400
Subject: [PATCH 1120/1206] [clang-query] Remove support for srcloc output
 (#92442)

This functionality was added about three years ago, but has been in a
significantly broken state since it was added. It has begun to cause a
maintenance burden for work in Clang (largely due to the complexity of
having two levels of code generation involved), and the original author
is unable to help maintain it. Because it only worked under limited
circumstances and because of the maintenance burden, it is being
removed. If someone wishes to resurrect the functionality, they should
hopefully be able to do so from this one commit.

Fixes #82591
---
 clang-tools-extra/clang-query/CMakeLists.txt  |    1 -
 clang-tools-extra/clang-query/Query.cpp       |   98 +-
 clang-tools-extra/clang-query/Query.h         |    3 +-
 clang-tools-extra/clang-query/QueryParser.cpp |   26 +-
 clang-tools-extra/clang-query/QuerySession.h  |    5 +-
 clang-tools-extra/docs/ReleaseNotes.rst       |    2 +
 .../unittests/clang-query/QueryParserTest.cpp |   32 +-
 clang/docs/tools/clang-formatted-files.txt    |    5 -
 .../include/clang/Tooling/NodeIntrospection.h |  101 -
 clang/lib/Tooling/CMakeLists.txt              |   90 -
 clang/lib/Tooling/DumpTool/APIData.h          |   31 -
 .../Tooling/DumpTool/ASTSrcLocProcessor.cpp   |  271 ---
 .../lib/Tooling/DumpTool/ASTSrcLocProcessor.h |   53 -
 clang/lib/Tooling/DumpTool/CMakeLists.txt     |   16 -
 .../lib/Tooling/DumpTool/ClangSrcLocDump.cpp  |  155 --
 .../Tooling/DumpTool/generate_cxx_src_locs.py |  452 -----
 .../lib/Tooling/EmptyNodeIntrospection.inc.in |   48 -
 clang/lib/Tooling/NodeIntrospection.cpp       |   88 -
 clang/unittests/CMakeLists.txt                |    1 -
 clang/unittests/Introspection/CMakeLists.txt  |   22 -
 .../Introspection/IntrospectionTest.cpp       | 1679 -----------------
 .../gn/secondary/clang/lib/Tooling/BUILD.gn   |   16 -
 .../clang/lib/Tooling/DumpTool/BUILD.gn       |   20 -
 .../gn/secondary/clang/unittests/BUILD.gn     |    1 -
 .../clang/unittests/Introspection/BUILD.gn    |   19 -
 25 files changed, 22 insertions(+), 3213 deletions(-)
 delete mode 100644 clang/include/clang/Tooling/NodeIntrospection.h
 delete mode 100644 clang/lib/Tooling/DumpTool/APIData.h
 delete mode 100644 clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp
 delete mode 100644 clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h
 delete mode 100644 clang/lib/Tooling/DumpTool/CMakeLists.txt
 delete mode 100644 clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp
 delete mode 100755 clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py
 delete mode 100644 clang/lib/Tooling/EmptyNodeIntrospection.inc.in
 delete mode 100644 clang/lib/Tooling/NodeIntrospection.cpp
 delete mode 100644 clang/unittests/Introspection/CMakeLists.txt
 delete mode 100644 clang/unittests/Introspection/IntrospectionTest.cpp
 delete mode 100644 llvm/utils/gn/secondary/clang/lib/Tooling/DumpTool/BUILD.gn
 delete mode 100644 llvm/utils/gn/secondary/clang/unittests/Introspection/BUILD.gn

diff --git a/clang-tools-extra/clang-query/CMakeLists.txt b/clang-tools-extra/clang-query/CMakeLists.txt
index 8a58d4224e04..34f018c4a03f 100644
--- a/clang-tools-extra/clang-query/CMakeLists.txt
+++ b/clang-tools-extra/clang-query/CMakeLists.txt
@@ -20,7 +20,6 @@ clang_target_link_libraries(clangQuery
   clangBasic
   clangDynamicASTMatchers
   clangFrontend
-  clangTooling
   clangSerialization
   )
 
diff --git a/clang-tools-extra/clang-query/Query.cpp b/clang-tools-extra/clang-query/Query.cpp
index 9d5807a52fa8..93f4104d39db 100644
--- a/clang-tools-extra/clang-query/Query.cpp
+++ b/clang-tools-extra/clang-query/Query.cpp
@@ -13,7 +13,6 @@
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Frontend/TextDiagnostic.h"
-#include "clang/Tooling/NodeIntrospection.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 
@@ -69,8 +68,6 @@ bool HelpQuery::run(llvm::raw_ostream &OS, QuerySession &QS) const {
         "Diagnostic location for bound nodes.\n"
         "  detailed-ast                      "
         "Detailed AST output for bound nodes.\n"
-        "  srcloc                            "
-        "Source locations and ranges for bound nodes.\n"
         "  dump                              "
         "Detailed AST output for bound nodes (alias of detailed-ast).\n\n";
   return true;
@@ -91,90 +88,6 @@ struct CollectBoundNodes : MatchFinder::MatchCallback {
   }
 };
 
-void dumpLocations(llvm::raw_ostream &OS, DynTypedNode Node, ASTContext &Ctx,
-                   const DiagnosticsEngine &Diags, SourceManager const &SM) {
-  auto Locs = clang::tooling::NodeIntrospection::GetLocations(Node);
-
-  auto PrintLocations = [](llvm::raw_ostream &OS, auto Iter, auto End) {
-    auto CommonEntry = Iter->first;
-    auto Scout = Iter;
-    SmallVector<std::string> LocationStrings;
-    while (Scout->first == CommonEntry) {
-      LocationStrings.push_back(
-          tooling::LocationCallFormatterCpp::format(*Iter->second));
-      if (Scout == End)
-        break;
-      ++Scout;
-      if (Scout->first == CommonEntry)
-        ++Iter;
-    }
-    llvm::sort(LocationStrings);
-    for (auto &LS : LocationStrings) {
-      OS << " * \"" << LS << "\"\n";
-    }
-    return Iter;
-  };
-
-  TextDiagnostic TD(OS, Ctx.getLangOpts(), &Diags.getDiagnosticOptions());
-
-  for (auto Iter = Locs.LocationAccessors.begin();
-       Iter != Locs.LocationAccessors.end(); ++Iter) {
-    if (!Iter->first.isValid())
-      continue;
-
-    TD.emitDiagnostic(FullSourceLoc(Iter->first, SM), DiagnosticsEngine::Note,
-                      "source locations here", std::nullopt, std::nullopt);
-
-    Iter = PrintLocations(OS, Iter, Locs.LocationAccessors.end());
-    OS << '\n';
-  }
-
-  for (auto Iter = Locs.RangeAccessors.begin();
-       Iter != Locs.RangeAccessors.end(); ++Iter) {
-
-    if (!Iter->first.getBegin().isValid())
-      continue;
-
-    if (SM.getPresumedLineNumber(Iter->first.getBegin()) !=
-        SM.getPresumedLineNumber(Iter->first.getEnd()))
-      continue;
-
-    TD.emitDiagnostic(
-        FullSourceLoc(Iter->first.getBegin(), SM), DiagnosticsEngine::Note,
-        "source ranges here " + Iter->first.printToString(SM),
-        CharSourceRange::getTokenRange(Iter->first), std::nullopt);
-
-    Iter = PrintLocations(OS, Iter, Locs.RangeAccessors.end());
-  }
-  for (auto Iter = Locs.RangeAccessors.begin();
-       Iter != Locs.RangeAccessors.end(); ++Iter) {
-
-    if (!Iter->first.getBegin().isValid())
-      continue;
-
-    if (SM.getPresumedLineNumber(Iter->first.getBegin()) ==
-        SM.getPresumedLineNumber(Iter->first.getEnd()))
-      continue;
-
-    TD.emitDiagnostic(
-        FullSourceLoc(Iter->first.getBegin(), SM), DiagnosticsEngine::Note,
-        "source range " + Iter->first.printToString(SM) + " starting here...",
-        CharSourceRange::getTokenRange(Iter->first), std::nullopt);
-
-    auto ColNum = SM.getPresumedColumnNumber(Iter->first.getEnd());
-    auto LastLineLoc = Iter->first.getEnd().getLocWithOffset(-(ColNum - 1));
-
-    TD.emitDiagnostic(FullSourceLoc(Iter->first.getEnd(), SM),
-                      DiagnosticsEngine::Note, "... ending here",
-                      CharSourceRange::getTokenRange(
-                          SourceRange(LastLineLoc, Iter->first.getEnd())),
-                      std::nullopt);
-
-    Iter = PrintLocations(OS, Iter, Locs.RangeAccessors.end());
-  }
-  OS << "\n";
-}
-
 } // namespace
 
 bool MatchQuery::run(llvm::raw_ostream &OS, QuerySession &QS) const {
@@ -195,8 +108,7 @@ bool MatchQuery::run(llvm::raw_ostream &OS, QuerySession &QS) const {
       return false;
     }
 
-    auto &Ctx = AST->getASTContext();
-    const auto &SM = Ctx.getSourceManager();
+    ASTContext &Ctx = AST->getASTContext();
     Ctx.getParentMapContext().setTraversalKind(QS.TK);
     Finder.matchAST(Ctx);
 
@@ -244,19 +156,11 @@ bool MatchQuery::run(llvm::raw_ostream &OS, QuerySession &QS) const {
         }
         if (QS.DetailedASTOutput) {
           OS << "Binding for \"" << BI->first << "\":\n";
-          const ASTContext &Ctx = AST->getASTContext();
           ASTDumper Dumper(OS, Ctx, AST->getDiagnostics().getShowColors());
           Dumper.SetTraversalKind(QS.TK);
           Dumper.Visit(BI->second);
           OS << "\n";
         }
-        if (QS.SrcLocOutput) {
-          OS << "\n  \"" << BI->first << "\" Source locations\n";
-          OS << "  " << std::string(19 + BI->first.size(), '-') << '\n';
-
-          dumpLocations(OS, BI->second, Ctx, AST->getDiagnostics(), SM);
-          OS << "\n";
-        }
       }
 
       if (MI->getMap().empty())
diff --git a/clang-tools-extra/clang-query/Query.h b/clang-tools-extra/clang-query/Query.h
index 7242479633c2..af250fbe13ce 100644
--- a/clang-tools-extra/clang-query/Query.h
+++ b/clang-tools-extra/clang-query/Query.h
@@ -17,7 +17,7 @@
 namespace clang {
 namespace query {
 
-enum OutputKind { OK_Diag, OK_Print, OK_DetailedAST, OK_SrcLoc };
+enum OutputKind { OK_Diag, OK_Print, OK_DetailedAST };
 
 enum QueryKind {
   QK_Invalid,
@@ -149,7 +149,6 @@ struct SetExclusiveOutputQuery : Query {
     QS.DiagOutput = false;
     QS.DetailedASTOutput = false;
     QS.PrintOutput = false;
-    QS.SrcLocOutput = false;
     QS.*Var = true;
     return true;
   }
diff --git a/clang-tools-extra/clang-query/QueryParser.cpp b/clang-tools-extra/clang-query/QueryParser.cpp
index 85a442bdd7de..1d0b7d9bc6fc 100644
--- a/clang-tools-extra/clang-query/QueryParser.cpp
+++ b/clang-tools-extra/clang-query/QueryParser.cpp
@@ -11,7 +11,6 @@
 #include "QuerySession.h"
 #include "clang/ASTMatchers/Dynamic/Parser.h"
 #include "clang/Basic/CharInfo.h"
-#include "clang/Tooling/NodeIntrospection.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include <optional>
@@ -104,19 +103,16 @@ QueryRef QueryParser::parseSetBool(bool QuerySession::*Var) {
 
 template <typename QueryType> QueryRef QueryParser::parseSetOutputKind() {
   StringRef ValStr;
-  bool HasIntrospection = tooling::NodeIntrospection::hasIntrospectionSupport();
-  unsigned OutKind =
-      LexOrCompleteWord<unsigned>(this, ValStr)
-          .Case("diag", OK_Diag)
-          .Case("print", OK_Print)
-          .Case("detailed-ast", OK_DetailedAST)
-          .Case("srcloc", OK_SrcLoc, /*IsCompletion=*/HasIntrospection)
-          .Case("dump", OK_DetailedAST)
-          .Default(~0u);
+  unsigned OutKind = LexOrCompleteWord<unsigned>(this, ValStr)
+                         .Case("diag", OK_Diag)
+                         .Case("print", OK_Print)
+                         .Case("detailed-ast", OK_DetailedAST)
+                         .Case("dump", OK_DetailedAST)
+                         .Default(~0u);
   if (OutKind == ~0u) {
-    return new InvalidQuery("expected 'diag', 'print', 'detailed-ast'" +
-                            StringRef(HasIntrospection ? ", 'srcloc'" : "") +
-                            " or 'dump', got '" + ValStr + "'");
+    return new InvalidQuery("expected 'diag', 'print', 'detailed-ast' or "
+                            "'dump', got '" +
+                            ValStr + "'");
   }
 
   switch (OutKind) {
@@ -126,10 +122,6 @@ template <typename QueryType> QueryRef QueryParser::parseSetOutputKind() {
     return new QueryType(&QuerySession::DiagOutput);
   case OK_Print:
     return new QueryType(&QuerySession::PrintOutput);
-  case OK_SrcLoc:
-    if (HasIntrospection)
-      return new QueryType(&QuerySession::SrcLocOutput);
-    return new InvalidQuery("'srcloc' output support is not available.");
   }
 
   llvm_unreachable("Invalid output kind");
diff --git a/clang-tools-extra/clang-query/QuerySession.h b/clang-tools-extra/clang-query/QuerySession.h
index 9a08289a2534..31a4900e2619 100644
--- a/clang-tools-extra/clang-query/QuerySession.h
+++ b/clang-tools-extra/clang-query/QuerySession.h
@@ -25,15 +25,14 @@ class QuerySession {
 public:
   QuerySession(llvm::ArrayRef<std::unique_ptr<ASTUnit>> ASTs)
       : ASTs(ASTs), PrintOutput(false), DiagOutput(true),
-        DetailedASTOutput(false), SrcLocOutput(false), BindRoot(true),
-        PrintMatcher(false), Terminate(false), TK(TK_AsIs) {}
+        DetailedASTOutput(false), BindRoot(true), PrintMatcher(false),
+        Terminate(false), TK(TK_AsIs) {}
 
   llvm::ArrayRef<std::unique_ptr<ASTUnit>> ASTs;
 
   bool PrintOutput;
   bool DiagOutput;
   bool DetailedASTOutput;
-  bool SrcLocOutput;
 
   bool BindRoot;
   bool PrintMatcher;
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 71734617bf7a..6a9892bada91 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -94,6 +94,8 @@ Improvements to clang-query
   from an external file, allowing the cost of reading the compilation database
   and building the AST to be imposed just once for faster prototyping.
 
+- Removed support for ``enable output srcloc``. Fixes #GH82591
+
 Improvements to clang-rename
 ----------------------------
 
diff --git a/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp b/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp
index b561e2bb9833..e414587c568b 100644
--- a/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp
+++ b/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp
@@ -9,7 +9,6 @@
 #include "QueryParser.h"
 #include "Query.h"
 #include "QuerySession.h"
-#include "clang/Tooling/NodeIntrospection.h"
 #include "llvm/LineEditor/LineEditor.h"
 #include "gtest/gtest.h"
 
@@ -61,7 +60,6 @@ TEST_F(QueryParserTest, Quit) {
 
 TEST_F(QueryParserTest, Set) {
 
-  bool HasIntrospection = tooling::NodeIntrospection::hasIntrospectionSupport();
   QueryRef Q = parse("set");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
   EXPECT_EQ("expected variable name", cast<InvalidQuery>(Q)->ErrStr);
@@ -72,13 +70,8 @@ TEST_F(QueryParserTest, Set) {
 
   Q = parse("set output");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
-  if (HasIntrospection)
-    EXPECT_EQ(
-        "expected 'diag', 'print', 'detailed-ast', 'srcloc' or 'dump', got ''",
-        cast<InvalidQuery>(Q)->ErrStr);
-  else
-    EXPECT_EQ("expected 'diag', 'print', 'detailed-ast' or 'dump', got ''",
-              cast<InvalidQuery>(Q)->ErrStr);
+  EXPECT_EQ("expected 'diag', 'print', 'detailed-ast' or 'dump', got ''",
+            cast<InvalidQuery>(Q)->ErrStr);
 
   Q = parse("set bind-root true foo");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
@@ -86,13 +79,8 @@ TEST_F(QueryParserTest, Set) {
 
   Q = parse("set output foo");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
-  if (HasIntrospection)
-    EXPECT_EQ("expected 'diag', 'print', 'detailed-ast', 'srcloc' or 'dump', "
-              "got 'foo'",
-              cast<InvalidQuery>(Q)->ErrStr);
-  else
-    EXPECT_EQ("expected 'diag', 'print', 'detailed-ast' or 'dump', got 'foo'",
-              cast<InvalidQuery>(Q)->ErrStr);
+  EXPECT_EQ("expected 'diag', 'print', 'detailed-ast' or 'dump', got 'foo'",
+            cast<InvalidQuery>(Q)->ErrStr);
 
   Q = parse("set output dump");
   ASSERT_TRUE(isa<SetExclusiveOutputQuery >(Q));
@@ -232,10 +220,8 @@ TEST_F(QueryParserTest, Complete) {
   EXPECT_EQ("output ", Comps[0].TypedText);
   EXPECT_EQ("output", Comps[0].DisplayText);
 
-  bool HasIntrospection = tooling::NodeIntrospection::hasIntrospectionSupport();
-
   Comps = QueryParser::complete("enable output ", 14, QS);
-  ASSERT_EQ(HasIntrospection ? 5u : 4u, Comps.size());
+  ASSERT_EQ(4u, Comps.size());
 
   EXPECT_EQ("diag ", Comps[0].TypedText);
   EXPECT_EQ("diag", Comps[0].DisplayText);
@@ -243,12 +229,8 @@ TEST_F(QueryParserTest, Complete) {
   EXPECT_EQ("print", Comps[1].DisplayText);
   EXPECT_EQ("detailed-ast ", Comps[2].TypedText);
   EXPECT_EQ("detailed-ast", Comps[2].DisplayText);
-  if (HasIntrospection) {
-    EXPECT_EQ("srcloc ", Comps[3].TypedText);
-    EXPECT_EQ("srcloc", Comps[3].DisplayText);
-  }
-  EXPECT_EQ("dump ", Comps[HasIntrospection ? 4 : 3].TypedText);
-  EXPECT_EQ("dump", Comps[HasIntrospection ? 4 : 3].DisplayText);
+  EXPECT_EQ("dump ", Comps[3].TypedText);
+  EXPECT_EQ("dump", Comps[3].DisplayText);
 
   Comps = QueryParser::complete("set traversal ", 14, QS);
   ASSERT_EQ(2u, Comps.size());
diff --git a/clang/docs/tools/clang-formatted-files.txt b/clang/docs/tools/clang-formatted-files.txt
index eaeadf2656b0..b747adfb6992 100644
--- a/clang/docs/tools/clang-formatted-files.txt
+++ b/clang/docs/tools/clang-formatted-files.txt
@@ -252,7 +252,6 @@ clang/include/clang/Tooling/CompilationDatabasePluginRegistry.h
 clang/include/clang/Tooling/DiagnosticsYaml.h
 clang/include/clang/Tooling/Execution.h
 clang/include/clang/Tooling/JSONCompilationDatabase.h
-clang/include/clang/Tooling/NodeIntrospection.h
 clang/include/clang/Tooling/Refactoring.h
 clang/include/clang/Tooling/StandaloneExecution.h
 clang/include/clang/Tooling/ToolExecutorPluginRegistry.h
@@ -562,15 +561,11 @@ clang/lib/Tooling/Execution.cpp
 clang/lib/Tooling/ExpandResponseFilesCompilationDatabase.cpp
 clang/lib/Tooling/FixIt.cpp
 clang/lib/Tooling/GuessTargetAndModeCompilationDatabase.cpp
-clang/lib/Tooling/NodeIntrospection.cpp
 clang/lib/Tooling/StandaloneExecution.cpp
 clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
 clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
 clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
 clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
-clang/lib/Tooling/DumpTool/APIData.h
-clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h
-clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp
 clang/lib/Tooling/Inclusions/HeaderIncludes.cpp
 clang/lib/Tooling/Inclusions/IncludeStyle.cpp
 clang/lib/Tooling/Inclusions/StandardLibrary.cpp
diff --git a/clang/include/clang/Tooling/NodeIntrospection.h b/clang/include/clang/Tooling/NodeIntrospection.h
deleted file mode 100644
index 91552cad2eca..000000000000
--- a/clang/include/clang/Tooling/NodeIntrospection.h
+++ /dev/null
@@ -1,101 +0,0 @@
-//===- NodeIntrospection.h ------------------------------------*- C++ -*---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file contains the implementation of the NodeIntrospection.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_TOOLING_NODEINTROSPECTION_H
-#define LLVM_CLANG_TOOLING_NODEINTROSPECTION_H
-
-#include "clang/AST/ASTTypeTraits.h"
-#include "clang/AST/DeclarationName.h"
-#include "llvm/ADT/IntrusiveRefCntPtr.h"
-#include <set>
-
-namespace clang {
-
-class Stmt;
-class Decl;
-class CXXCtorInitializer;
-class NestedNameSpecifierLoc;
-class TemplateArgumentLoc;
-class CXXBaseSpecifier;
-struct DeclarationNameInfo;
-
-namespace tooling {
-
-class LocationCall;
-using SharedLocationCall = llvm::IntrusiveRefCntPtr<LocationCall>;
-
-class LocationCall : public llvm::ThreadSafeRefCountedBase<LocationCall> {
-public:
-  enum LocationCallFlags { NoFlags, ReturnsPointer, IsCast };
-  LocationCall(SharedLocationCall on, std::string name,
-               LocationCallFlags flags = NoFlags)
-      : m_flags(flags), m_on(std::move(on)), m_name(std::move(name)) {}
-
-  LocationCall *on() const { return m_on.get(); }
-  StringRef name() const { return m_name; }
-  bool returnsPointer() const { return m_flags & ReturnsPointer; }
-  bool isCast() const { return m_flags & IsCast; }
-
-private:
-  LocationCallFlags m_flags;
-  SharedLocationCall m_on;
-  std::string m_name;
-};
-
-class LocationCallFormatterCpp {
-public:
-  static void print(const LocationCall &Call, llvm::raw_ostream &OS);
-  static std::string format(const LocationCall &Call);
-};
-
-namespace internal {
-struct RangeLessThan {
-  bool operator()(std::pair<SourceRange, SharedLocationCall> const &LHS,
-                  std::pair<SourceRange, SharedLocationCall> const &RHS) const;
-  bool
-  operator()(std::pair<SourceLocation, SharedLocationCall> const &LHS,
-             std::pair<SourceLocation, SharedLocationCall> const &RHS) const;
-};
-
-} // namespace internal
-
-// Note that this container stores unique results in a deterministic, but
-// the location calls are in an unspecified order.  Clients which desire
-// a particular order for the location calls, such as alphabetical,
-// should sort results after retrieval, because the order is dependent
-// on how the LocationCalls are formatted.
-template <typename T, typename U>
-using UniqueMultiMap = std::set<std::pair<T, U>, internal::RangeLessThan>;
-
-using SourceLocationMap = UniqueMultiMap<SourceLocation, SharedLocationCall>;
-using SourceRangeMap = UniqueMultiMap<SourceRange, SharedLocationCall>;
-
-struct NodeLocationAccessors {
-  SourceLocationMap LocationAccessors;
-  SourceRangeMap RangeAccessors;
-};
-
-namespace NodeIntrospection {
-bool hasIntrospectionSupport();
-NodeLocationAccessors GetLocations(clang::Stmt const *Object);
-NodeLocationAccessors GetLocations(clang::Decl const *Object);
-NodeLocationAccessors GetLocations(clang::CXXCtorInitializer const *Object);
-NodeLocationAccessors GetLocations(clang::NestedNameSpecifierLoc const &);
-NodeLocationAccessors GetLocations(clang::TemplateArgumentLoc const &);
-NodeLocationAccessors GetLocations(clang::CXXBaseSpecifier const *);
-NodeLocationAccessors GetLocations(clang::TypeLoc const &);
-NodeLocationAccessors GetLocations(clang::DeclarationNameInfo const &);
-NodeLocationAccessors GetLocations(clang::DynTypedNode const &Node);
-} // namespace NodeIntrospection
-} // namespace tooling
-} // namespace clang
-#endif
diff --git a/clang/lib/Tooling/CMakeLists.txt b/clang/lib/Tooling/CMakeLists.txt
index 91e6cbdcbc44..93a9e707a134 100644
--- a/clang/lib/Tooling/CMakeLists.txt
+++ b/clang/lib/Tooling/CMakeLists.txt
@@ -9,98 +9,10 @@ add_subdirectory(Core)
 add_subdirectory(Inclusions)
 add_subdirectory(Refactoring)
 add_subdirectory(ASTDiff)
-add_subdirectory(DumpTool)
 add_subdirectory(Syntax)
 add_subdirectory(DependencyScanning)
 add_subdirectory(Transformer)
 
-# Replace the last lib component of the current binary directory with include
-string(FIND ${CMAKE_CURRENT_BINARY_DIR} "/lib/" PATH_LIB_START REVERSE)
-if(PATH_LIB_START EQUAL -1)
-  message(FATAL_ERROR "Couldn't find lib component in binary directory")
-endif()
-math(EXPR PATH_LIB_END "${PATH_LIB_START}+5")
-string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} 0 ${PATH_LIB_START} PATH_HEAD)
-string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} ${PATH_LIB_END} -1 PATH_TAIL)
-string(CONCAT BINARY_INCLUDE_DIR ${PATH_HEAD} "/include/clang/" ${PATH_TAIL})
-
-if (NOT Python3_EXECUTABLE
-    OR APPLE
-    OR CMAKE_CROSSCOMPILING
-    OR GENERATOR_IS_MULTI_CONFIG
-    OR NOT LLVM_NATIVE_ARCH IN_LIST LLVM_TARGETS_TO_BUILD
-    )
-    configure_file(
-      EmptyNodeIntrospection.inc.in
-      ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
-      COPYONLY
-    )
-    set(CLANG_TOOLING_BUILD_AST_INTROSPECTION "OFF" CACHE BOOL "")
-else()
-  # The generation of ASTNodeAPI.json takes a long time in a
-  # Debug build due to parsing AST.h. Disable the processing
-  # but setting CLANG_TOOLING_BUILD_AST_INTROSPECTION as an
-  # internal hidden setting to override.
-  # When the processing is disabled, a trivial/empty JSON
-  # file is generated by clang-ast-dump and generate_cxx_src_locs.py
-  # generates the same API, but with a trivial implementation.
-  option(CLANG_TOOLING_BUILD_AST_INTROSPECTION "Enable AST introspection" TRUE)
-
-  set(skip_expensive_processing $<OR:$<CONFIG:Debug>,$<NOT:$<BOOL:${CLANG_TOOLING_BUILD_AST_INTROSPECTION}>>>)
-
-  set(implicitDirs)
-  foreach(implicitDir ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES})
-    list(APPEND implicitDirs -I ${implicitDir})
-  endforeach()
-
-  include(GetClangResourceDir)
-  get_clang_resource_dir(resource_dir PREFIX ${LLVM_BINARY_DIR})
-  add_custom_command(
-      COMMENT Generate ASTNodeAPI.json
-      OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json
-      DEPENDS clang-ast-dump clang-resource-headers
-      COMMAND
-      $<TARGET_FILE:clang-ast-dump>
-        # Skip this in debug mode because parsing AST.h is too slow
-        --skip-processing=${skip_expensive_processing}
-        -I ${resource_dir}/include
-        -I ${CLANG_SOURCE_DIR}/include
-        -I ${LLVM_BINARY_DIR}/tools/clang/include
-        -I ${LLVM_BINARY_DIR}/include
-        -I ${LLVM_SOURCE_DIR}/include
-        ${implicitDirs}
-        --json-output-path ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json
-  )
-
-  add_custom_target(run-ast-api-dump-tool
-      DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json
-  )
-
-  add_custom_command(
-      COMMENT Generate NodeIntrospection.inc
-      OUTPUT ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
-      DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json
-        ${CMAKE_CURRENT_SOURCE_DIR}/DumpTool/generate_cxx_src_locs.py
-        ${CMAKE_CURRENT_SOURCE_DIR}/EmptyNodeIntrospection.inc.in
-      COMMAND
-      ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/DumpTool/generate_cxx_src_locs.py
-        --json-input-path ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json
-        --output-file NodeIntrospection.inc
-        --use-empty-implementation ${skip_expensive_processing}
-        --empty-implementation
-          "${CMAKE_CURRENT_SOURCE_DIR}/EmptyNodeIntrospection.inc.in"
-      COMMAND
-      ${CMAKE_COMMAND} -E copy_if_different
-        ${CMAKE_CURRENT_BINARY_DIR}/NodeIntrospection.inc
-        ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
-  )
-
-  add_custom_target(run-ast-api-generate-tool
-      DEPENDS
-      ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
-  )
-endif()
-
 add_clang_library(clangTooling
   AllTUsExecution.cpp
   ArgumentsAdjusters.cpp
@@ -116,8 +28,6 @@ add_clang_library(clangTooling
   Refactoring.cpp
   RefactoringCallbacks.cpp
   StandaloneExecution.cpp
-  NodeIntrospection.cpp
-  ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
   Tooling.cpp
 
   DEPENDS
diff --git a/clang/lib/Tooling/DumpTool/APIData.h b/clang/lib/Tooling/DumpTool/APIData.h
deleted file mode 100644
index 03e247a8bd95..000000000000
--- a/clang/lib/Tooling/DumpTool/APIData.h
+++ /dev/null
@@ -1,31 +0,0 @@
-//===- APIData.h ---------------------------------------------*- C++ -*----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_LIB_TOOLING_DUMPTOOL_APIDATA_H
-#define LLVM_CLANG_LIB_TOOLING_DUMPTOOL_APIDATA_H
-
-#include <string>
-#include <vector>
-
-namespace clang {
-namespace tooling {
-
-struct ClassData {
-  std::vector<std::string> ASTClassLocations;
-  std::vector<std::string> ASTClassRanges;
-  std::vector<std::string> TemplateParms;
-  std::vector<std::string> TypeSourceInfos;
-  std::vector<std::string> TypeLocs;
-  std::vector<std::string> NestedNameLocs;
-  std::vector<std::string> DeclNameInfos;
-};
-
-} // namespace tooling
-} // namespace clang
-
-#endif
diff --git a/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp b/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp
deleted file mode 100644
index 42691d556d98..000000000000
--- a/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.cpp
+++ /dev/null
@@ -1,271 +0,0 @@
-//===- ASTSrcLocProcessor.cpp --------------------------------*- C++ -*----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "ASTSrcLocProcessor.h"
-
-#include "clang/Frontend/CompilerInstance.h"
-#include "llvm/Support/JSON.h"
-#include "llvm/Support/MemoryBuffer.h"
-
-using namespace clang::tooling;
-using namespace llvm;
-using namespace clang::ast_matchers;
-
-ASTSrcLocProcessor::ASTSrcLocProcessor(StringRef JsonPath)
-    : JsonPath(JsonPath) {
-
-  MatchFinder::MatchFinderOptions FinderOptions;
-
-  Finder = std::make_unique<MatchFinder>(std::move(FinderOptions));
-  Finder->addMatcher(
-      cxxRecordDecl(
-          isDefinition(),
-          isSameOrDerivedFrom(
-              namedDecl(
-                  hasAnyName(
-                      "clang::Stmt", "clang::Decl", "clang::CXXCtorInitializer",
-                      "clang::NestedNameSpecifierLoc",
-                      "clang::TemplateArgumentLoc", "clang::CXXBaseSpecifier",
-                      "clang::DeclarationNameInfo", "clang::TypeLoc"))
-                  .bind("nodeClade")),
-          optionally(isDerivedFrom(cxxRecordDecl().bind("derivedFrom"))))
-          .bind("className"),
-      this);
-  Finder->addMatcher(
-          cxxRecordDecl(isDefinition(), hasAnyName("clang::PointerLikeTypeLoc",
-                                                   "clang::TypeofLikeTypeLoc"))
-              .bind("templateName"),
-      this);
-}
-
-std::unique_ptr<clang::ASTConsumer>
-ASTSrcLocProcessor::createASTConsumer(clang::CompilerInstance &Compiler,
-                                      StringRef File) {
-  return Finder->newASTConsumer();
-}
-
-llvm::json::Object toJSON(llvm::StringMap<std::vector<StringRef>> const &Obj) {
-  using llvm::json::toJSON;
-
-  llvm::json::Object JsonObj;
-  for (const auto &Item : Obj) {
-    JsonObj[Item.first()] = Item.second;
-  }
-  return JsonObj;
-}
-
-llvm::json::Object toJSON(llvm::StringMap<std::string> const &Obj) {
-  using llvm::json::toJSON;
-
-  llvm::json::Object JsonObj;
-  for (const auto &Item : Obj) {
-    JsonObj[Item.first()] = Item.second;
-  }
-  return JsonObj;
-}
-
-llvm::json::Object toJSON(ClassData const &Obj) {
-  llvm::json::Object JsonObj;
-
-  if (!Obj.ASTClassLocations.empty())
-    JsonObj["sourceLocations"] = Obj.ASTClassLocations;
-  if (!Obj.ASTClassRanges.empty())
-    JsonObj["sourceRanges"] = Obj.ASTClassRanges;
-  if (!Obj.TemplateParms.empty())
-    JsonObj["templateParms"] = Obj.TemplateParms;
-  if (!Obj.TypeSourceInfos.empty())
-    JsonObj["typeSourceInfos"] = Obj.TypeSourceInfos;
-  if (!Obj.TypeLocs.empty())
-    JsonObj["typeLocs"] = Obj.TypeLocs;
-  if (!Obj.NestedNameLocs.empty())
-    JsonObj["nestedNameLocs"] = Obj.NestedNameLocs;
-  if (!Obj.DeclNameInfos.empty())
-    JsonObj["declNameInfos"] = Obj.DeclNameInfos;
-  return JsonObj;
-}
-
-llvm::json::Object toJSON(llvm::StringMap<ClassData> const &Obj) {
-  using llvm::json::toJSON;
-
-  llvm::json::Object JsonObj;
-  for (const auto &Item : Obj)
-    JsonObj[Item.first()] = ::toJSON(Item.second);
-  return JsonObj;
-}
-
-void WriteJSON(StringRef JsonPath, llvm::json::Object &&ClassInheritance,
-               llvm::json::Object &&ClassesInClade,
-               llvm::json::Object &&ClassEntries) {
-  llvm::json::Object JsonObj;
-
-  using llvm::json::toJSON;
-
-  JsonObj["classInheritance"] = std::move(ClassInheritance);
-  JsonObj["classesInClade"] = std::move(ClassesInClade);
-  JsonObj["classEntries"] = std::move(ClassEntries);
-
-  llvm::json::Value JsonVal(std::move(JsonObj));
-
-  bool WriteChange = false;
-  std::string OutString;
-  if (auto ExistingOrErr = MemoryBuffer::getFile(JsonPath, /*IsText=*/true)) {
-    raw_string_ostream Out(OutString);
-    Out << formatv("{0:2}", JsonVal);
-    if (ExistingOrErr.get()->getBuffer() == Out.str())
-      return;
-    WriteChange = true;
-  }
-
-  std::error_code EC;
-  llvm::raw_fd_ostream JsonOut(JsonPath, EC, llvm::sys::fs::OF_Text);
-  if (EC)
-    return;
-
-  if (WriteChange)
-    JsonOut << OutString;
-  else
-    JsonOut << formatv("{0:2}", JsonVal);
-}
-
-void ASTSrcLocProcessor::generate() {
-  WriteJSON(JsonPath, ::toJSON(ClassInheritance), ::toJSON(ClassesInClade),
-            ::toJSON(ClassEntries));
-}
-
-void ASTSrcLocProcessor::generateEmpty() { WriteJSON(JsonPath, {}, {}, {}); }
-
-std::vector<std::string>
-CaptureMethods(std::string TypeString, const clang::CXXRecordDecl *ASTClass,
-               const MatchFinder::MatchResult &Result) {
-
-  auto publicAccessor = [](auto... InnerMatcher) {
-    return cxxMethodDecl(isPublic(), parameterCountIs(0), isConst(),
-                         InnerMatcher...);
-  };
-
-  auto BoundNodesVec = match(
-      findAll(
-          publicAccessor(
-              ofClass(cxxRecordDecl(
-                  equalsNode(ASTClass),
-                  optionally(isDerivedFrom(
-                      cxxRecordDecl(hasAnyName("clang::Stmt", "clang::Decl"))
-                          .bind("stmtOrDeclBase"))),
-                  optionally(isDerivedFrom(
-                      cxxRecordDecl(hasName("clang::Expr")).bind("exprBase"))),
-                  optionally(
-                      isDerivedFrom(cxxRecordDecl(hasName("clang::TypeLoc"))
-                                        .bind("typeLocBase"))))),
-              returns(hasCanonicalType(asString(TypeString))))
-              .bind("classMethod")),
-      *ASTClass, *Result.Context);
-
-  std::vector<std::string> Methods;
-  for (const auto &BN : BoundNodesVec) {
-    if (const auto *Node = BN.getNodeAs<clang::NamedDecl>("classMethod")) {
-      const auto *StmtOrDeclBase =
-          BN.getNodeAs<clang::CXXRecordDecl>("stmtOrDeclBase");
-      const auto *TypeLocBase =
-          BN.getNodeAs<clang::CXXRecordDecl>("typeLocBase");
-      const auto *ExprBase = BN.getNodeAs<clang::CXXRecordDecl>("exprBase");
-      // The clang AST has several methods on base classes which are overriden
-      // pseudo-virtually by derived classes.
-      // We record only the pseudo-virtual methods on the base classes to
-      // avoid duplication.
-      if (StmtOrDeclBase &&
-          (Node->getName() == "getBeginLoc" || Node->getName() == "getEndLoc" ||
-           Node->getName() == "getSourceRange"))
-        continue;
-      if (ExprBase && Node->getName() == "getExprLoc")
-        continue;
-      if (TypeLocBase && Node->getName() == "getLocalSourceRange")
-        continue;
-      if ((ASTClass->getName() == "PointerLikeTypeLoc" ||
-           ASTClass->getName() == "TypeofLikeTypeLoc") &&
-          Node->getName() == "getLocalSourceRange")
-        continue;
-      Methods.push_back(Node->getName().str());
-    }
-  }
-  return Methods;
-}
-
-void ASTSrcLocProcessor::run(const MatchFinder::MatchResult &Result) {
-
-  const auto *ASTClass =
-      Result.Nodes.getNodeAs<clang::CXXRecordDecl>("className");
-
-  StringRef CladeName;
-  if (ASTClass) {
-    if (const auto *NodeClade =
-            Result.Nodes.getNodeAs<clang::CXXRecordDecl>("nodeClade"))
-      CladeName = NodeClade->getName();
-  } else {
-    ASTClass = Result.Nodes.getNodeAs<clang::CXXRecordDecl>("templateName");
-    CladeName = "TypeLoc";
-  }
-
-  StringRef ClassName = ASTClass->getName();
-
-  ClassData CD;
-
-  CD.ASTClassLocations =
-      CaptureMethods("class clang::SourceLocation", ASTClass, Result);
-  CD.ASTClassRanges =
-      CaptureMethods("class clang::SourceRange", ASTClass, Result);
-  CD.TypeSourceInfos =
-      CaptureMethods("class clang::TypeSourceInfo *", ASTClass, Result);
-  CD.TypeLocs = CaptureMethods("class clang::TypeLoc", ASTClass, Result);
-  CD.NestedNameLocs =
-      CaptureMethods("class clang::NestedNameSpecifierLoc", ASTClass, Result);
-  CD.DeclNameInfos =
-      CaptureMethods("struct clang::DeclarationNameInfo", ASTClass, Result);
-  auto DI = CaptureMethods("const struct clang::DeclarationNameInfo &",
-                           ASTClass, Result);
-  CD.DeclNameInfos.insert(CD.DeclNameInfos.end(), DI.begin(), DI.end());
-
-  if (const auto *DerivedFrom =
-          Result.Nodes.getNodeAs<clang::CXXRecordDecl>("derivedFrom")) {
-
-    if (const auto *Templ =
-            llvm::dyn_cast<clang::ClassTemplateSpecializationDecl>(
-                DerivedFrom)) {
-
-      const auto &TArgs = Templ->getTemplateArgs();
-
-      SmallString<256> TArgsString;
-      llvm::raw_svector_ostream OS(TArgsString);
-      OS << DerivedFrom->getName() << '<';
-
-      clang::PrintingPolicy PPol(Result.Context->getLangOpts());
-      PPol.TerseOutput = true;
-
-      for (unsigned I = 0; I < TArgs.size(); ++I) {
-        if (I > 0)
-          OS << ", ";
-        TArgs.get(I).getAsType().print(OS, PPol);
-      }
-      OS << '>';
-
-      ClassInheritance[ClassName] = TArgsString.str().str();
-    } else {
-      ClassInheritance[ClassName] = DerivedFrom->getName().str();
-    }
-  }
-
-  if (const auto *Templ = ASTClass->getDescribedClassTemplate()) {
-    if (auto *TParams = Templ->getTemplateParameters()) {
-      for (const auto &TParam : *TParams) {
-        CD.TemplateParms.push_back(TParam->getName().str());
-      }
-    }
-  }
-
-  ClassEntries[ClassName] = CD;
-  ClassesInClade[CladeName].push_back(ClassName);
-}
diff --git a/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h b/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h
deleted file mode 100644
index 5f2b48173f28..000000000000
--- a/clang/lib/Tooling/DumpTool/ASTSrcLocProcessor.h
+++ /dev/null
@@ -1,53 +0,0 @@
-//===- ASTSrcLocProcessor.h ---------------------------------*- C++ -*-----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_TOOLING_DUMPTOOL_ASTSRCLOCPROCESSOR_H
-#define LLVM_CLANG_TOOLING_DUMPTOOL_ASTSRCLOCPROCESSOR_H
-
-#include "APIData.h"
-#include "clang/ASTMatchers/ASTMatchFinder.h"
-#include "llvm/ADT/StringRef.h"
-#include <memory>
-#include <string>
-#include <vector>
-
-namespace clang {
-
-class CompilerInstance;
-
-namespace tooling {
-
-class ASTSrcLocProcessor : public ast_matchers::MatchFinder::MatchCallback {
-public:
-  explicit ASTSrcLocProcessor(StringRef JsonPath);
-
-  std::unique_ptr<ASTConsumer> createASTConsumer(CompilerInstance &Compiler,
-                                                 StringRef File);
-
-  void generate();
-  void generateEmpty();
-
-private:
-  void run(const ast_matchers::MatchFinder::MatchResult &Result) override;
-
-  std::optional<TraversalKind> getCheckTraversalKind() const override {
-    return TK_IgnoreUnlessSpelledInSource;
-  }
-
-  llvm::StringMap<std::string> ClassInheritance;
-  llvm::StringMap<std::vector<StringRef>> ClassesInClade;
-  llvm::StringMap<ClassData> ClassEntries;
-
-  std::string JsonPath;
-  std::unique_ptr<clang::ast_matchers::MatchFinder> Finder;
-};
-
-} // namespace tooling
-} // namespace clang
-
-#endif
diff --git a/clang/lib/Tooling/DumpTool/CMakeLists.txt b/clang/lib/Tooling/DumpTool/CMakeLists.txt
deleted file mode 100644
index 712985bf7e2b..000000000000
--- a/clang/lib/Tooling/DumpTool/CMakeLists.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-
-add_clang_executable(clang-ast-dump
-  ASTSrcLocProcessor.cpp
-  ClangSrcLocDump.cpp
-)
-
-target_link_libraries(clang-ast-dump
-  PRIVATE
-  clangAST
-  clangASTMatchers
-  clangBasic
-  clangDriver
-  clangFrontend
-  clangSerialization
-  clangToolingCore
-)
diff --git a/clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp b/clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp
deleted file mode 100644
index 1529bfa75d6d..000000000000
--- a/clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp
+++ /dev/null
@@ -1,155 +0,0 @@
-//===- ClangSrcLocDump.cpp ------------------------------------*- C++ -*---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Driver/Compilation.h"
-#include "clang/Driver/Driver.h"
-#include "clang/Driver/Job.h"
-#include "clang/Driver/Options.h"
-#include "clang/Driver/Tool.h"
-#include "clang/Frontend/CompilerInstance.h"
-#include "clang/Frontend/TextDiagnosticPrinter.h"
-#include "clang/Lex/PreprocessorOptions.h"
-#include "clang/Tooling/Tooling.h"
-#include "llvm/Option/ArgList.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/JSON.h"
-#include "llvm/TargetParser/Host.h"
-
-#include "ASTSrcLocProcessor.h"
-
-using namespace clang::tooling;
-using namespace clang;
-using namespace llvm;
-
-static cl::list<std::string> IncludeDirectories(
-    "I", cl::desc("Include directories to use while compiling"),
-    cl::value_desc("directory"), cl::Required, cl::OneOrMore, cl::Prefix);
-
-static cl::opt<bool>
-    SkipProcessing("skip-processing",
-                   cl::desc("Avoid processing the AST header file"),
-                   cl::Required, cl::value_desc("bool"));
-
-static cl::opt<std::string> JsonOutputPath("json-output-path",
-                                           cl::desc("json output path"),
-                                           cl::Required,
-                                           cl::value_desc("path"));
-
-class ASTSrcLocGenerationAction : public clang::ASTFrontendAction {
-public:
-  ASTSrcLocGenerationAction() : Processor(JsonOutputPath) {}
-
-  void ExecuteAction() override {
-    clang::ASTFrontendAction::ExecuteAction();
-    if (getCompilerInstance().getDiagnostics().getNumErrors() > 0)
-      Processor.generateEmpty();
-    else
-      Processor.generate();
-  }
-
-  std::unique_ptr<clang::ASTConsumer>
-  CreateASTConsumer(clang::CompilerInstance &Compiler,
-                    llvm::StringRef File) override {
-    return Processor.createASTConsumer(Compiler, File);
-  }
-
-private:
-  ASTSrcLocProcessor Processor;
-};
-
-static const char Filename[] = "ASTTU.cpp";
-
-int main(int argc, const char **argv) {
-
-  cl::ParseCommandLineOptions(argc, argv);
-
-  if (SkipProcessing) {
-    std::error_code EC;
-    llvm::raw_fd_ostream JsonOut(JsonOutputPath, EC, llvm::sys::fs::OF_Text);
-    if (EC)
-      return 1;
-    JsonOut << formatv("{0:2}", llvm::json::Value(llvm::json::Object()));
-    return 0;
-  }
-
-  std::vector<std::string> Args;
-  Args.push_back("-cc1");
-
-  llvm::transform(IncludeDirectories, std::back_inserter(Args),
-                  [](const std::string &IncDir) { return "-I" + IncDir; });
-
-  Args.push_back("-fsyntax-only");
-  Args.push_back(Filename);
-
-  std::vector<const char *> Argv(Args.size(), nullptr);
-  llvm::transform(Args, Argv.begin(),
-                  [](const std::string &Arg) { return Arg.c_str(); });
-
-  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
-      CreateAndPopulateDiagOpts(Argv);
-
-  // Don't output diagnostics, because common scenarios such as
-  // cross-compiling fail with diagnostics.  This is not fatal, but
-  // just causes attempts to use the introspection API to return no data.
-  TextDiagnosticPrinter DiagnosticPrinter(llvm::nulls(), &*DiagOpts);
-  DiagnosticsEngine Diagnostics(
-      IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts,
-      &DiagnosticPrinter, false);
-
-  auto *OFS = new llvm::vfs::OverlayFileSystem(vfs::getRealFileSystem());
-
-  auto *MemFS = new llvm::vfs::InMemoryFileSystem();
-  OFS->pushOverlay(MemFS);
-  MemFS->addFile(Filename, 0,
-                 MemoryBuffer::getMemBuffer("#include \"clang/AST/AST.h\"\n"));
-
-  auto Files = llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOptions(), OFS);
-
-  auto Driver = std::make_unique<clang::driver::Driver>(
-      "clang", llvm::sys::getDefaultTargetTriple(), Diagnostics,
-      "ast-api-dump-tool", OFS);
-
-  std::unique_ptr<clang::driver::Compilation> Comp(
-      Driver->BuildCompilation(llvm::ArrayRef(Argv)));
-  if (!Comp)
-    return 1;
-
-  const auto &Jobs = Comp->getJobs();
-  if (Jobs.size() != 1 || !isa<clang::driver::Command>(*Jobs.begin())) {
-    SmallString<256> error_msg;
-    llvm::raw_svector_ostream error_stream(error_msg);
-    Jobs.Print(error_stream, "; ", true);
-    return 1;
-  }
-
-  const auto &Cmd = cast<clang::driver::Command>(*Jobs.begin());
-  const llvm::opt::ArgStringList &CC1Args = Cmd.getArguments();
-
-  auto Invocation = std::make_unique<CompilerInvocation>();
-  CompilerInvocation::CreateFromArgs(*Invocation, CC1Args, Diagnostics);
-
-  CompilerInstance Compiler(std::make_shared<clang::PCHContainerOperations>());
-  Compiler.setInvocation(std::move(Invocation));
-
-  Compiler.createDiagnostics(&DiagnosticPrinter, false);
-  if (!Compiler.hasDiagnostics())
-    return 1;
-
-  // Suppress "2 errors generated" or similar messages
-  Compiler.getDiagnosticOpts().ShowCarets = false;
-  Compiler.createSourceManager(*Files);
-  Compiler.setFileManager(Files.get());
-
-  ASTSrcLocGenerationAction ScopedToolAction;
-  Compiler.ExecuteAction(ScopedToolAction);
-
-  Files->clearStatCache();
-
-  return 0;
-}
diff --git a/clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py b/clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py
deleted file mode 100755
index 7671f9691c09..000000000000
--- a/clang/lib/Tooling/DumpTool/generate_cxx_src_locs.py
+++ /dev/null
@@ -1,452 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-import os
-import sys
-import json
-import filecmp
-import shutil
-import argparse
-
-
-class Generator(object):
-
-    implementationContent = ""
-
-    RefClades = {
-        "DeclarationNameInfo",
-        "NestedNameSpecifierLoc",
-        "TemplateArgumentLoc",
-        "TypeLoc",
-    }
-
-    def __init__(self, templateClasses):
-        self.templateClasses = templateClasses
-
-    def GeneratePrologue(self):
-
-        self.implementationContent += r"""
-/*===- Generated file -------------------------------------------*- C++ -*-===*\
-|*                                                                            *|
-|* Introspection of available AST node SourceLocations                        *|
-|*                                                                            *|
-|* Automatically generated file, do not edit!                                 *|
-|*                                                                            *|
-\*===----------------------------------------------------------------------===*/
-
-namespace clang {
-namespace tooling {
-
-using LocationAndString = SourceLocationMap::value_type;
-using RangeAndString = SourceRangeMap::value_type;
-
-bool NodeIntrospection::hasIntrospectionSupport() { return true; }
-
-struct RecursionPopper
-{
-    RecursionPopper(std::vector<clang::TypeLoc> &TypeLocRecursionGuard)
-    :  TLRG(TypeLocRecursionGuard)
-    {
-
-    }
-
-    ~RecursionPopper()
-    {
-    TLRG.pop_back();
-    }
-
-private:
-std::vector<clang::TypeLoc> &TLRG;
-};
-"""
-
-    def GenerateBaseGetLocationsDeclaration(self, CladeName):
-        InstanceDecoration = "*"
-        if CladeName in self.RefClades:
-            InstanceDecoration = "&"
-
-        self.implementationContent += """
-void GetLocationsImpl(SharedLocationCall const& Prefix,
-    clang::{0} const {1}Object, SourceLocationMap &Locs,
-    SourceRangeMap &Rngs,
-    std::vector<clang::TypeLoc> &TypeLocRecursionGuard);
-""".format(
-            CladeName, InstanceDecoration
-        )
-
-    def GenerateSrcLocMethod(self, ClassName, ClassData, CreateLocalRecursionGuard):
-
-        NormalClassName = ClassName
-        RecursionGuardParam = (
-            ""
-            if CreateLocalRecursionGuard
-            else ", std::vector<clang::TypeLoc>& TypeLocRecursionGuard"
-        )
-
-        if "templateParms" in ClassData:
-            TemplatePreamble = "template <typename "
-            ClassName += "<"
-            First = True
-            for TA in ClassData["templateParms"]:
-                if not First:
-                    ClassName += ", "
-                    TemplatePreamble += ", typename "
-
-                First = False
-                ClassName += TA
-                TemplatePreamble += TA
-
-            ClassName += ">"
-            TemplatePreamble += ">\n"
-            self.implementationContent += TemplatePreamble
-
-        self.implementationContent += """
-static void GetLocations{0}(SharedLocationCall const& Prefix,
-    clang::{1} const &Object,
-    SourceLocationMap &Locs, SourceRangeMap &Rngs {2})
-{{
-""".format(
-            NormalClassName, ClassName, RecursionGuardParam
-        )
-
-        if "sourceLocations" in ClassData:
-            for locName in ClassData["sourceLocations"]:
-                self.implementationContent += """
-  Locs.insert(LocationAndString(Object.{0}(),
-    llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}")));
-""".format(
-                    locName
-                )
-
-            self.implementationContent += "\n"
-
-        if "sourceRanges" in ClassData:
-            for rngName in ClassData["sourceRanges"]:
-                self.implementationContent += """
-  Rngs.insert(RangeAndString(Object.{0}(),
-    llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}")));
-""".format(
-                    rngName
-                )
-
-            self.implementationContent += "\n"
-
-        if (
-            "typeLocs" in ClassData
-            or "typeSourceInfos" in ClassData
-            or "nestedNameLocs" in ClassData
-            or "declNameInfos" in ClassData
-        ):
-            if CreateLocalRecursionGuard:
-                self.implementationContent += (
-                    "std::vector<clang::TypeLoc> TypeLocRecursionGuard;\n"
-                )
-
-            self.implementationContent += "\n"
-
-            if "typeLocs" in ClassData:
-                for typeLoc in ClassData["typeLocs"]:
-
-                    self.implementationContent += """
-              if (Object.{0}()) {{
-                GetLocationsImpl(
-                    llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}"),
-                    Object.{0}(), Locs, Rngs, TypeLocRecursionGuard);
-                }}
-              """.format(
-                        typeLoc
-                    )
-
-            self.implementationContent += "\n"
-            if "typeSourceInfos" in ClassData:
-                for tsi in ClassData["typeSourceInfos"]:
-                    self.implementationContent += """
-              if (Object.{0}()) {{
-                GetLocationsImpl(llvm::makeIntrusiveRefCnt<LocationCall>(
-                    llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}",
-                        LocationCall::ReturnsPointer), "getTypeLoc"),
-                    Object.{0}()->getTypeLoc(), Locs, Rngs, TypeLocRecursionGuard);
-                    }}
-              """.format(
-                        tsi
-                    )
-
-                self.implementationContent += "\n"
-
-            if "nestedNameLocs" in ClassData:
-                for NN in ClassData["nestedNameLocs"]:
-                    self.implementationContent += """
-              if (Object.{0}())
-                GetLocationsImpl(
-                    llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}"),
-                    Object.{0}(), Locs, Rngs, TypeLocRecursionGuard);
-              """.format(
-                        NN
-                    )
-
-            if "declNameInfos" in ClassData:
-                for declName in ClassData["declNameInfos"]:
-
-                    self.implementationContent += """
-                      GetLocationsImpl(
-                          llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "{0}"),
-                          Object.{0}(), Locs, Rngs, TypeLocRecursionGuard);
-                      """.format(
-                        declName
-                    )
-
-        self.implementationContent += "}\n"
-
-    def GenerateFiles(self, OutputFile):
-        with open(os.path.join(os.getcwd(), OutputFile), "w") as f:
-            f.write(self.implementationContent)
-
-    def GenerateBaseGetLocationsFunction(
-        self,
-        ASTClassNames,
-        ClassEntries,
-        CladeName,
-        InheritanceMap,
-        CreateLocalRecursionGuard,
-    ):
-
-        MethodReturnType = "NodeLocationAccessors"
-        InstanceDecoration = "*"
-        if CladeName in self.RefClades:
-            InstanceDecoration = "&"
-
-        Signature = "GetLocations(clang::{0} const {1}Object)".format(
-            CladeName, InstanceDecoration
-        )
-        ImplSignature = """
-    GetLocationsImpl(SharedLocationCall const& Prefix,
-        clang::{0} const {1}Object, SourceLocationMap &Locs,
-        SourceRangeMap &Rngs,
-        std::vector<clang::TypeLoc> &TypeLocRecursionGuard)
-    """.format(
-            CladeName, InstanceDecoration
-        )
-
-        self.implementationContent += "void {0} {{ ".format(ImplSignature)
-
-        if CladeName == "TypeLoc":
-            self.implementationContent += "if (Object.isNull()) return;"
-
-            self.implementationContent += """
-            if (llvm::find(TypeLocRecursionGuard, Object) != TypeLocRecursionGuard.end())
-              return;
-            TypeLocRecursionGuard.push_back(Object);
-            RecursionPopper RAII(TypeLocRecursionGuard);
-                """
-
-        RecursionGuardParam = ""
-        if not CreateLocalRecursionGuard:
-            RecursionGuardParam = ", TypeLocRecursionGuard"
-
-        ArgPrefix = "*"
-        if CladeName in self.RefClades:
-            ArgPrefix = ""
-        self.implementationContent += (
-            "GetLocations{0}(Prefix, {1}Object, Locs, Rngs {2});".format(
-                CladeName, ArgPrefix, RecursionGuardParam
-            )
-        )
-
-        if CladeName == "TypeLoc":
-            self.implementationContent += """
-        if (auto QTL = Object.getAs<clang::QualifiedTypeLoc>()) {
-            auto Dequalified = QTL.getNextTypeLoc();
-            return GetLocationsImpl(llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "getNextTypeLoc"),
-                                Dequalified,
-                                Locs,
-                                Rngs,
-                                TypeLocRecursionGuard);
-        }"""
-
-        for ASTClassName in ASTClassNames:
-            if ASTClassName in self.templateClasses:
-                continue
-            if ASTClassName == CladeName:
-                continue
-            if CladeName != "TypeLoc":
-                self.implementationContent += """
-if (auto Derived = llvm::dyn_cast<clang::{0}>(Object)) {{
-  GetLocations{0}(Prefix, *Derived, Locs, Rngs {1});
-}}
-""".format(
-                    ASTClassName, RecursionGuardParam
-                )
-                continue
-
-            self.GenerateBaseTypeLocVisit(
-                ASTClassName, ClassEntries, RecursionGuardParam, InheritanceMap
-            )
-
-        self.implementationContent += "}"
-
-        self.implementationContent += """
-{0} NodeIntrospection::{1} {{
-  NodeLocationAccessors Result;
-  SharedLocationCall Prefix;
-  std::vector<clang::TypeLoc> TypeLocRecursionGuard;
-
-  GetLocationsImpl(Prefix, Object, Result.LocationAccessors,
-                   Result.RangeAccessors, TypeLocRecursionGuard);
-""".format(
-            MethodReturnType, Signature
-        )
-
-        self.implementationContent += "return Result; }"
-
-    def GenerateBaseTypeLocVisit(
-        self, ASTClassName, ClassEntries, RecursionGuardParam, InheritanceMap
-    ):
-        CallPrefix = "Prefix"
-        if ASTClassName != "TypeLoc":
-            CallPrefix = """llvm::makeIntrusiveRefCnt<LocationCall>(Prefix,
-                    "getAs<clang::{0}>", LocationCall::IsCast)
-                """.format(
-                ASTClassName
-            )
-
-        if ASTClassName in ClassEntries:
-
-            self.implementationContent += """
-            if (auto ConcreteTL = Object.getAs<clang::{0}>())
-              GetLocations{1}({2}, ConcreteTL, Locs, Rngs {3});
-            """.format(
-                ASTClassName, ASTClassName, CallPrefix, RecursionGuardParam
-            )
-
-        if ASTClassName in InheritanceMap:
-            for baseTemplate in self.templateClasses:
-                if baseTemplate in InheritanceMap[ASTClassName]:
-                    self.implementationContent += """
-    if (auto ConcreteTL = Object.getAs<clang::{0}>())
-      GetLocations{1}({2}, ConcreteTL, Locs, Rngs {3});
-    """.format(
-                        InheritanceMap[ASTClassName],
-                        baseTemplate,
-                        CallPrefix,
-                        RecursionGuardParam,
-                    )
-
-    def GenerateDynNodeVisitor(self, CladeNames):
-        MethodReturnType = "NodeLocationAccessors"
-
-        Signature = "GetLocations(clang::DynTypedNode const &Node)"
-
-        self.implementationContent += (
-            MethodReturnType + " NodeIntrospection::" + Signature + "{"
-        )
-
-        for CladeName in CladeNames:
-            if CladeName == "DeclarationNameInfo":
-                continue
-            self.implementationContent += """
-    if (const auto *N = Node.get<{0}>())
-    """.format(
-                CladeName
-            )
-            ArgPrefix = ""
-            if CladeName in self.RefClades:
-                ArgPrefix = "*"
-            self.implementationContent += """
-      return GetLocations({0}const_cast<{1} *>(N));""".format(
-                ArgPrefix, CladeName
-            )
-
-        self.implementationContent += "\nreturn {}; }"
-
-    def GenerateEpilogue(self):
-
-        self.implementationContent += """
-  }
-}
-"""
-
-
-def main():
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--json-input-path", help="Read API description from FILE", metavar="FILE"
-    )
-    parser.add_argument(
-        "--output-file", help="Generate output in FILEPATH", metavar="FILEPATH"
-    )
-    parser.add_argument(
-        "--use-empty-implementation",
-        help="Generate empty implementation",
-        action="store",
-        type=int,
-    )
-    parser.add_argument(
-        "--empty-implementation",
-        help="Copy empty implementation from FILEPATH",
-        action="store",
-        metavar="FILEPATH",
-    )
-
-    options = parser.parse_args()
-
-    use_empty_implementation = options.use_empty_implementation
-
-    if not use_empty_implementation and not os.path.exists(options.json_input_path):
-        use_empty_implementation = True
-
-    if not use_empty_implementation:
-        with open(options.json_input_path) as f:
-            jsonData = json.load(f)
-
-        if not "classesInClade" in jsonData or not jsonData["classesInClade"]:
-            use_empty_implementation = True
-
-    if use_empty_implementation:
-        if not os.path.exists(options.output_file) or not filecmp.cmp(
-            options.empty_implementation, options.output_file
-        ):
-            shutil.copyfile(options.empty_implementation, options.output_file)
-        sys.exit(0)
-
-    templateClasses = []
-    for (ClassName, ClassAccessors) in jsonData["classEntries"].items():
-        if "templateParms" in ClassAccessors:
-            templateClasses.append(ClassName)
-
-    g = Generator(templateClasses)
-
-    g.GeneratePrologue()
-
-    for (CladeName, ClassNameData) in jsonData["classesInClade"].items():
-        g.GenerateBaseGetLocationsDeclaration(CladeName)
-
-    def getCladeName(ClassName):
-        for (CladeName, ClassNameData) in jsonData["classesInClade"].items():
-            if ClassName in ClassNameData:
-                return CladeName
-
-    for (ClassName, ClassAccessors) in jsonData["classEntries"].items():
-        cladeName = getCladeName(ClassName)
-        g.GenerateSrcLocMethod(
-            ClassName, ClassAccessors, cladeName not in Generator.RefClades
-        )
-
-    for (CladeName, ClassNameData) in jsonData["classesInClade"].items():
-        g.GenerateBaseGetLocationsFunction(
-            ClassNameData,
-            jsonData["classEntries"],
-            CladeName,
-            jsonData["classInheritance"],
-            CladeName not in Generator.RefClades,
-        )
-
-    g.GenerateDynNodeVisitor(jsonData["classesInClade"].keys())
-
-    g.GenerateEpilogue()
-
-    g.GenerateFiles(options.output_file)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/clang/lib/Tooling/EmptyNodeIntrospection.inc.in b/clang/lib/Tooling/EmptyNodeIntrospection.inc.in
deleted file mode 100644
index 2071c34cbd04..000000000000
--- a/clang/lib/Tooling/EmptyNodeIntrospection.inc.in
+++ /dev/null
@@ -1,48 +0,0 @@
-//===- EmptyNodeIntrospection.inc.in --------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-namespace clang {
-namespace tooling {
-bool NodeIntrospection::hasIntrospectionSupport() { return false; }
-
-NodeLocationAccessors NodeIntrospection::GetLocations(clang::Stmt const *) {
-  return {};
-}
-NodeLocationAccessors NodeIntrospection::GetLocations(clang::Decl const *) {
-  return {};
-}
-NodeLocationAccessors NodeIntrospection::GetLocations(
-    clang::CXXCtorInitializer const *) {
-  return {};
-}
-NodeLocationAccessors NodeIntrospection::GetLocations(
-    clang::NestedNameSpecifierLoc const&) {
-  return {};
-}
-NodeLocationAccessors NodeIntrospection::GetLocations(
-    clang::TemplateArgumentLoc const&) {
-  return {};
-}
-NodeLocationAccessors NodeIntrospection::GetLocations(
-    clang::CXXBaseSpecifier const*) {
-  return {};
-}
-NodeLocationAccessors NodeIntrospection::GetLocations(
-    clang::TypeLoc const&) {
-  return {};
-}
-NodeLocationAccessors NodeIntrospection::GetLocations(
-    clang::DeclarationNameInfo const&) {
-  return {};
-}
-NodeLocationAccessors
-NodeIntrospection::GetLocations(clang::DynTypedNode const &) {
-  return {};
-}
-} // namespace tooling
-} // namespace clang
diff --git a/clang/lib/Tooling/NodeIntrospection.cpp b/clang/lib/Tooling/NodeIntrospection.cpp
deleted file mode 100644
index f01bb1cb9c3c..000000000000
--- a/clang/lib/Tooling/NodeIntrospection.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-//===- NodeIntrospection.h -----------------------------------*- C++ -*----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file contains the implementation of the NodeIntrospection.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Tooling/NodeIntrospection.h"
-
-#include "clang/AST/AST.h"
-#include "llvm/Support/raw_ostream.h"
-
-namespace clang {
-
-namespace tooling {
-
-void LocationCallFormatterCpp::print(const LocationCall &Call,
-                                     llvm::raw_ostream &OS) {
-  if (const LocationCall *On = Call.on()) {
-    print(*On, OS);
-    if (On->returnsPointer())
-      OS << "->";
-    else
-      OS << '.';
-  }
-
-  OS << Call.name() << "()";
-}
-
-std::string LocationCallFormatterCpp::format(const LocationCall &Call) {
-  std::string Result;
-  llvm::raw_string_ostream OS(Result);
-  print(Call, OS);
-  OS.flush();
-  return Result;
-}
-
-namespace internal {
-
-static bool locationCallLessThan(const LocationCall *LHS,
-                                 const LocationCall *RHS) {
-  if (!LHS && !RHS)
-    return false;
-  if (LHS && !RHS)
-    return true;
-  if (!LHS && RHS)
-    return false;
-  auto compareResult = LHS->name().compare(RHS->name());
-  if (compareResult < 0)
-    return true;
-  if (compareResult > 0)
-    return false;
-  return locationCallLessThan(LHS->on(), RHS->on());
-}
-
-bool RangeLessThan::operator()(
-    std::pair<SourceRange, SharedLocationCall> const &LHS,
-    std::pair<SourceRange, SharedLocationCall> const &RHS) const {
-  if (LHS.first.getBegin() < RHS.first.getBegin())
-    return true;
-  else if (LHS.first.getBegin() != RHS.first.getBegin())
-    return false;
-
-  if (LHS.first.getEnd() < RHS.first.getEnd())
-    return true;
-  else if (LHS.first.getEnd() != RHS.first.getEnd())
-    return false;
-
-  return locationCallLessThan(LHS.second.get(), RHS.second.get());
-}
-bool RangeLessThan::operator()(
-    std::pair<SourceLocation, SharedLocationCall> const &LHS,
-    std::pair<SourceLocation, SharedLocationCall> const &RHS) const {
-  if (LHS.first == RHS.first)
-    return locationCallLessThan(LHS.second.get(), RHS.second.get());
-  return LHS.first < RHS.first;
-}
-} // namespace internal
-
-} // namespace tooling
-} // namespace clang
-
-#include "clang/Tooling/NodeIntrospection.inc"
diff --git a/clang/unittests/CMakeLists.txt b/clang/unittests/CMakeLists.txt
index 37ca3107b547..a8363ffa481e 100644
--- a/clang/unittests/CMakeLists.txt
+++ b/clang/unittests/CMakeLists.txt
@@ -34,7 +34,6 @@ add_subdirectory(ASTMatchers)
 add_subdirectory(AST)
 add_subdirectory(CrossTU)
 add_subdirectory(Tooling)
-add_subdirectory(Introspection)
 add_subdirectory(Format)
 add_subdirectory(Frontend)
 add_subdirectory(Rewrite)
diff --git a/clang/unittests/Introspection/CMakeLists.txt b/clang/unittests/Introspection/CMakeLists.txt
deleted file mode 100644
index edc359ef4fcf..000000000000
--- a/clang/unittests/Introspection/CMakeLists.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  FrontendOpenMP
-  Support
-  )
-
-add_clang_unittest(IntrospectionTests
-  IntrospectionTest.cpp
-  )
-
-clang_target_link_libraries(IntrospectionTests
-  PRIVATE
-  clangAST
-  clangASTMatchers
-  clangBasic
-  clangFrontend
-  clangSerialization
-  clangTooling
-  )
-target_link_libraries(IntrospectionTests
-  PRIVATE
-  LLVMTestingSupport
-)
diff --git a/clang/unittests/Introspection/IntrospectionTest.cpp b/clang/unittests/Introspection/IntrospectionTest.cpp
deleted file mode 100644
index 22321f241f3f..000000000000
--- a/clang/unittests/Introspection/IntrospectionTest.cpp
+++ /dev/null
@@ -1,1679 +0,0 @@
-//===- unittest/Introspection/IntrospectionTest.cpp ----------*- C++ -*---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Tests for AST location API introspection.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/AST/ASTContext.h"
-#include "clang/ASTMatchers/ASTMatchFinder.h"
-#include "clang/ASTMatchers/ASTMatchers.h"
-#include "clang/Tooling/NodeIntrospection.h"
-#include "clang/Tooling/Tooling.h"
-#include "gmock/gmock-matchers.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-using namespace clang;
-using namespace clang::ast_matchers;
-using namespace clang::tooling;
-
-using ::testing::Pair;
-using ::testing::UnorderedElementsAre;
-
-template <typename T, typename MapType>
-std::vector<std::pair<std::string, T>>
-FormatExpected(const MapType &Accessors) {
-  std::vector<std::pair<std::string, T>> Result;
-  llvm::transform(llvm::make_filter_range(Accessors,
-                                          [](const auto &Accessor) {
-                                            return Accessor.first.isValid();
-                                          }),
-                  std::back_inserter(Result), [](const auto &Accessor) {
-                    return std::make_pair(
-                        LocationCallFormatterCpp::format(*Accessor.second),
-                        Accessor.first);
-                  });
-  return Result;
-}
-
-#define STRING_LOCATION_PAIR(INSTANCE, LOC) Pair(#LOC, INSTANCE->LOC)
-
-#define STRING_LOCATION_STDPAIR(INSTANCE, LOC)                                 \
-  std::make_pair(std::string(#LOC), INSTANCE->LOC)
-
-/**
-  A test formatter for a hypothetical language which needs
-  neither casts nor '->'.
-*/
-class LocationCallFormatterSimple {
-public:
-  static void print(const LocationCall &Call, llvm::raw_ostream &OS) {
-    if (Call.isCast()) {
-      if (const LocationCall *On = Call.on())
-        print(*On, OS);
-      return;
-    }
-    if (const LocationCall *On = Call.on()) {
-      print(*On, OS);
-      OS << '.';
-    }
-    OS << Call.name() << "()";
-  }
-
-  static std::string format(const LocationCall &Call) {
-    std::string Result;
-    llvm::raw_string_ostream OS(Result);
-    print(Call, OS);
-    OS.flush();
-    return Result;
-  }
-};
-
-TEST(Introspection, SourceLocations_CallContainer) {
-  SourceLocationMap slm;
-  SharedLocationCall Prefix;
-  slm.insert(std::make_pair(
-      SourceLocation(),
-      llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "getSourceRange")));
-  EXPECT_EQ(slm.size(), 1u);
-
-  auto callTypeLoc =
-      llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "getTypeLoc");
-  slm.insert(std::make_pair(
-      SourceLocation(),
-      llvm::makeIntrusiveRefCnt<LocationCall>(callTypeLoc, "getSourceRange")));
-  EXPECT_EQ(slm.size(), 2u);
-}
-
-TEST(Introspection, SourceLocations_CallContainer2) {
-  SourceRangeMap slm;
-  SharedLocationCall Prefix;
-  slm.insert(
-      std::make_pair(SourceRange(), llvm::makeIntrusiveRefCnt<LocationCall>(
-                                        Prefix, "getCXXOperatorNameRange")));
-  EXPECT_EQ(slm.size(), 1u);
-
-  slm.insert(std::make_pair(
-      SourceRange(),
-      llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "getSourceRange")));
-  EXPECT_EQ(slm.size(), 2u);
-}
-
-TEST(Introspection, SourceLocations_CallChainFormatting) {
-  SharedLocationCall Prefix;
-  auto chainedCall = llvm::makeIntrusiveRefCnt<LocationCall>(
-      llvm::makeIntrusiveRefCnt<LocationCall>(Prefix, "getTypeLoc"),
-      "getSourceRange");
-  EXPECT_EQ(LocationCallFormatterCpp::format(*chainedCall),
-            "getTypeLoc().getSourceRange()");
-}
-
-TEST(Introspection, SourceLocations_Formatter) {
-  SharedLocationCall Prefix;
-  auto chainedCall = llvm::makeIntrusiveRefCnt<LocationCall>(
-      llvm::makeIntrusiveRefCnt<LocationCall>(
-          llvm::makeIntrusiveRefCnt<LocationCall>(
-              llvm::makeIntrusiveRefCnt<LocationCall>(
-                  Prefix, "getTypeSourceInfo", LocationCall::ReturnsPointer),
-              "getTypeLoc"),
-          "getAs<clang::TypeSpecTypeLoc>", LocationCall::IsCast),
-      "getNameLoc");
-
-  EXPECT_EQ("getTypeSourceInfo()->getTypeLoc().getAs<clang::TypeSpecTypeLoc>()."
-            "getNameLoc()",
-            LocationCallFormatterCpp::format(*chainedCall));
-  EXPECT_EQ("getTypeSourceInfo().getTypeLoc().getNameLoc()",
-            LocationCallFormatterSimple::format(*chainedCall));
-}
-
-TEST(Introspection, SourceLocations_Stmt) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST = buildASTFromCode("void foo() {} void bar() { foo(); }", "foo.cpp",
-                              std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(
-          callExpr(callee(functionDecl(hasName("foo")))).bind("fooCall"))),
-      TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  auto *FooCall = BoundNodes[0].getNodeAs<CallExpr>("fooCall");
-
-  auto Result = NodeIntrospection::GetLocations(FooCall);
-
-  auto ExpectedLocations =
-    FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  EXPECT_THAT(
-      ExpectedLocations,
-      UnorderedElementsAre(STRING_LOCATION_PAIR(FooCall, getBeginLoc()),
-                           STRING_LOCATION_PAIR(FooCall, getEndLoc()),
-                           STRING_LOCATION_PAIR(FooCall, getExprLoc()),
-                           STRING_LOCATION_PAIR(FooCall, getRParenLoc())));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(ExpectedRanges, UnorderedElementsAre(STRING_LOCATION_PAIR(
-                                  FooCall, getSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_Decl) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-namespace ns1 {
-namespace ns2 {
-template <typename T, typename U> struct Foo {};
-template <typename T, typename U> struct Bar {
-  struct Nested {
-    template <typename A, typename B>
-    Foo<A, B> method(int i, bool b) const noexcept(true);
-  };
-};
-} // namespace ns2
-} // namespace ns1
-
-template <typename T, typename U>
-template <typename A, typename B>
-ns1::ns2::Foo<A, B> ns1::ns2::Bar<T, U>::Nested::method(int i, bool b) const
-    noexcept(true) {}
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(
-          cxxMethodDecl(hasName("method"), isDefinition()).bind("method"))),
-      TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *MethodDecl = BoundNodes[0].getNodeAs<CXXMethodDecl>("method");
-
-  auto Result = NodeIntrospection::GetLocations(MethodDecl);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  llvm::sort(ExpectedLocations);
-
-  // clang-format off
-  std::vector<std::pair<std::string, SourceLocation>> ActualLocations{
-STRING_LOCATION_STDPAIR(MethodDecl, getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getBodyRBrace()),
-STRING_LOCATION_STDPAIR(MethodDecl, getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getInnerLocStart()),
-STRING_LOCATION_STDPAIR(MethodDecl, getLocation()),
-STRING_LOCATION_STDPAIR(MethodDecl, getNameInfo().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getNameInfo().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getNameInfo().getLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getOuterLocStart()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getLocalBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getLocalEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getLocalBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getLocalEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getLocalBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getLocalEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getPrefix().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getPrefix().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getPrefix().getLocalBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getPrefix().getLocalEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getLAngleLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getRAngleLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getTemplateNameLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getLParenLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getLocalRangeBegin()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getLocalRangeEnd()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getRParenLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getLAngleLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getRAngleLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getTemplateNameLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getLocalBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getLocalEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getLocalBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getLocalEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getLAngleLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getRAngleLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getTemplateNameLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getNextTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getLAngleLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getRAngleLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getTemplateNameLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getLocalBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getLocalEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getLocalBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getLocalEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getLAngleLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getRAngleLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getTemplateNameLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getNextTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSpecEndLoc()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSpecStartLoc())
-  };
-  // clang-format on
-
-  EXPECT_EQ(ExpectedLocations, ActualLocations);
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  llvm::sort(ExpectedRanges, [](const auto &LHS, const auto &RHS) {
-    return LHS.first < RHS.first;
-  });
-
-  // clang-format off
-  EXPECT_EQ(
-            llvm::ArrayRef(ExpectedRanges),
-      (ArrayRef<std::pair<std::string, SourceRange>>{
-STRING_LOCATION_STDPAIR(MethodDecl, getExceptionSpecSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getNameInfo().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getParametersSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getPrefix().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getPrefix().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getPrefix().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getPrefix().getTypeLoc().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getQualifierLoc().getTypeLoc().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getReturnTypeSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getExceptionSpecRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getParensRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getAs<clang::FunctionTypeLoc>().getReturnLoc().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getPrefix().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::ElaboratedTypeLoc>().getQualifierLoc().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_STDPAIR(MethodDecl, getTypeSourceInfo()->getTypeLoc().getSourceRange())
-  }));
-  // clang-format on
-}
-
-TEST(Introspection, SourceLocations_NNS) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-namespace ns
-{
-  struct A {
-  void foo();
-};
-}
-void ns::A::foo() {}
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(nestedNameSpecifierLoc().bind("nns"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *NNS = BoundNodes[0].getNodeAs<NestedNameSpecifierLoc>("nns");
-
-  auto Result = NodeIntrospection::GetLocations(*NNS);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  llvm::sort(ExpectedLocations);
-
-  EXPECT_EQ(
-      llvm::ArrayRef(ExpectedLocations),
-      (ArrayRef<std::pair<std::string, SourceLocation>>{
-          STRING_LOCATION_STDPAIR(NNS, getBeginLoc()),
-          STRING_LOCATION_STDPAIR(NNS, getEndLoc()),
-          STRING_LOCATION_STDPAIR(NNS, getLocalBeginLoc()),
-          STRING_LOCATION_STDPAIR(NNS, getLocalEndLoc()),
-          STRING_LOCATION_STDPAIR(NNS, getPrefix().getBeginLoc()),
-          STRING_LOCATION_STDPAIR(NNS, getPrefix().getEndLoc()),
-          STRING_LOCATION_STDPAIR(NNS, getPrefix().getLocalBeginLoc()),
-          STRING_LOCATION_STDPAIR(NNS, getPrefix().getLocalEndLoc()),
-          STRING_LOCATION_STDPAIR(
-              NNS, getTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-          STRING_LOCATION_STDPAIR(NNS, getTypeLoc().getBeginLoc()),
-          STRING_LOCATION_STDPAIR(NNS, getTypeLoc().getEndLoc())}));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(
-      ExpectedRanges,
-      UnorderedElementsAre(
-          STRING_LOCATION_PAIR(NNS, getPrefix().getLocalSourceRange()),
-          STRING_LOCATION_PAIR(NNS, getPrefix().getSourceRange()),
-          STRING_LOCATION_PAIR(NNS, getLocalSourceRange()),
-          STRING_LOCATION_PAIR(NNS, getSourceRange()),
-          STRING_LOCATION_PAIR(NNS, getTypeLoc().getSourceRange()),
-          STRING_LOCATION_PAIR(NNS, getTypeLoc().getLocalSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_TA_Type) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-template<typename T>
-  struct A {
-  void foo();
-};
-
-void foo()
-{
-  A<int> a;
-}
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(templateArgumentLoc().bind("ta"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *TA = BoundNodes[0].getNodeAs<TemplateArgumentLoc>("ta");
-
-  auto Result = NodeIntrospection::GetLocations(*TA);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(
-STRING_LOCATION_PAIR(TA, getLocation()),
-STRING_LOCATION_PAIR(TA,
-  getTypeSourceInfo()->getTypeLoc().getAs<clang::BuiltinTypeLoc>().getBuiltinLoc()),
-STRING_LOCATION_PAIR(TA,
-  getTypeSourceInfo()->getTypeLoc().getAs<clang::BuiltinTypeLoc>().getNameLoc()),
-STRING_LOCATION_PAIR(
-    TA, getTypeSourceInfo()->getTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(
-    TA, getTypeSourceInfo()->getTypeLoc().getEndLoc())
-  ));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(
-      ExpectedRanges,
-      UnorderedElementsAre(
-          STRING_LOCATION_PAIR(TA, getSourceRange()),
-          STRING_LOCATION_PAIR(
-              TA, getTypeSourceInfo()->getTypeLoc().getSourceRange()),
-          STRING_LOCATION_PAIR(
-              TA, getTypeSourceInfo()->getTypeLoc().getLocalSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_TA_Decl) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-template<void(*Ty)()>
-void test2() {}
-void doNothing() {}
-void test() {
-    test2<doNothing>();
-}
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(templateArgumentLoc().bind("ta"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *TA = BoundNodes[0].getNodeAs<TemplateArgumentLoc>("ta");
-
-  auto Result = NodeIntrospection::GetLocations(*TA);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getLocation())));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(ExpectedRanges,
-              UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_TA_Nullptr) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-template<void(*Ty)()>
-void test2() {}
-void doNothing() {}
-void test() {
-    test2<nullptr>();
-}
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(templateArgumentLoc().bind("ta"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *TA = BoundNodes[0].getNodeAs<TemplateArgumentLoc>("ta");
-
-  auto Result = NodeIntrospection::GetLocations(*TA);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getLocation())));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(ExpectedRanges,
-              UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_TA_Integral) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-template<int>
-void test2() {}
-void test() {
-    test2<42>();
-}
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(templateArgumentLoc().bind("ta"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *TA = BoundNodes[0].getNodeAs<TemplateArgumentLoc>("ta");
-
-  auto Result = NodeIntrospection::GetLocations(*TA);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getLocation())));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(ExpectedRanges,
-              UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_TA_Template) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-template<typename T> class A;
-template <template <typename> class T> void foo();
-void bar()
-{
-  foo<A>();
-}
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(templateArgumentLoc().bind("ta"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *TA = BoundNodes[0].getNodeAs<TemplateArgumentLoc>("ta");
-
-  auto Result = NodeIntrospection::GetLocations(*TA);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  EXPECT_THAT(
-      ExpectedLocations,
-      UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getLocation()),
-                           STRING_LOCATION_PAIR(TA, getTemplateNameLoc())));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(ExpectedRanges,
-              UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_TA_TemplateExpansion) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST = buildASTFromCodeWithArgs(
-      R"cpp(
-template<template<typename> class ...> class B { };
-  template<template<typename> class ...T> class C {
-  B<T...> testTemplateExpansion;
-};
-)cpp",
-      {"-fno-delayed-template-parsing"}, "foo.cpp", "clang-tool",
-      std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(templateArgumentLoc().bind("ta"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *TA = BoundNodes[0].getNodeAs<TemplateArgumentLoc>("ta");
-
-  auto Result = NodeIntrospection::GetLocations(*TA);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  EXPECT_THAT(
-      ExpectedLocations,
-      UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getLocation()),
-                           STRING_LOCATION_PAIR(TA, getTemplateNameLoc()),
-                           STRING_LOCATION_PAIR(TA, getTemplateEllipsisLoc())));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(ExpectedRanges,
-              UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_TA_Expression) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-template<int, int = 0> class testExpr;
-template<int I> class testExpr<I> { };
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(templateArgumentLoc().bind("ta"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *TA = BoundNodes[0].getNodeAs<TemplateArgumentLoc>("ta");
-
-  auto Result = NodeIntrospection::GetLocations(*TA);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getLocation())));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(ExpectedRanges,
-              UnorderedElementsAre(STRING_LOCATION_PAIR(TA, getSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_TA_Pack) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST = buildASTFromCodeWithArgs(
-      R"cpp(
-template<typename... T> class A {};
-void foo()
-{
-    A<int> ai;
-}
-)cpp",
-      {"-fno-delayed-template-parsing"}, "foo.cpp", "clang-tool",
-      std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(templateArgumentLoc().bind("ta"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *TA = BoundNodes[0].getNodeAs<TemplateArgumentLoc>("ta");
-
-  auto Result = NodeIntrospection::GetLocations(*TA);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(
-STRING_LOCATION_PAIR(TA, getLocation()),
-STRING_LOCATION_PAIR(TA,
-  getTypeSourceInfo()->getTypeLoc().getAs<clang::BuiltinTypeLoc>().getBuiltinLoc()),
-STRING_LOCATION_PAIR(TA,
-  getTypeSourceInfo()->getTypeLoc().getAs<clang::BuiltinTypeLoc>().getNameLoc()),
-STRING_LOCATION_PAIR(
-    TA, getTypeSourceInfo()->getTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(
-    TA, getTypeSourceInfo()->getTypeLoc().getEndLoc())
-  ));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(
-      ExpectedRanges,
-      UnorderedElementsAre(
-          STRING_LOCATION_PAIR(TA, getSourceRange()),
-          STRING_LOCATION_PAIR(
-              TA, getTypeSourceInfo()->getTypeLoc().getSourceRange()),
-          STRING_LOCATION_PAIR(
-              TA, getTypeSourceInfo()->getTypeLoc().getLocalSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_CXXCtorInitializer_base) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-struct A {
-};
-
-struct B : A {
-  B() : A() {}
-};
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(cxxConstructorDecl(
-          hasAnyConstructorInitializer(cxxCtorInitializer().bind("init"))))),
-      TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *CtorInit = BoundNodes[0].getNodeAs<CXXCtorInitializer>("init");
-
-  auto Result = NodeIntrospection::GetLocations(CtorInit);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  // clang-format off
-  EXPECT_THAT(
-      ExpectedLocations,
-      UnorderedElementsAre(
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getEndLoc()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-STRING_LOCATION_PAIR(CtorInit, getLParenLoc()),
-STRING_LOCATION_PAIR(CtorInit, getRParenLoc()),
-STRING_LOCATION_PAIR(CtorInit, getSourceLocation()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getEndLoc())
- ));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  // clang-format off
-  EXPECT_THAT(
-      ExpectedRanges,
-      UnorderedElementsAre(
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getSourceRange())));
-  // clang-format on
-}
-
-TEST(Introspection, SourceLocations_CXXCtorInitializer_member) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-struct A {
-  int m_i;
-  A() : m_i(42) {}
-};
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(cxxConstructorDecl(
-          hasAnyConstructorInitializer(cxxCtorInitializer().bind("init"))))),
-      TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *CtorInit = BoundNodes[0].getNodeAs<CXXCtorInitializer>("init");
-
-  auto Result = NodeIntrospection::GetLocations(CtorInit);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(
-                  STRING_LOCATION_PAIR(CtorInit, getLParenLoc()),
-                  STRING_LOCATION_PAIR(CtorInit, getMemberLocation()),
-                  STRING_LOCATION_PAIR(CtorInit, getRParenLoc()),
-                  STRING_LOCATION_PAIR(CtorInit, getSourceLocation())));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(ExpectedRanges, UnorderedElementsAre(STRING_LOCATION_PAIR(
-                                  CtorInit, getSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_CXXCtorInitializer_ctor) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-struct C {
-  C() : C(42) {}
-  C(int) {}
-};
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(cxxConstructorDecl(
-          hasAnyConstructorInitializer(cxxCtorInitializer().bind("init"))))),
-      TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *CtorInit = BoundNodes[0].getNodeAs<CXXCtorInitializer>("init");
-
-  auto Result = NodeIntrospection::GetLocations(CtorInit);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  // clang-format off
-  EXPECT_THAT(
-      ExpectedLocations,
-      UnorderedElementsAre(
-STRING_LOCATION_PAIR(CtorInit, getLParenLoc()),
-STRING_LOCATION_PAIR(CtorInit, getRParenLoc()),
-STRING_LOCATION_PAIR(CtorInit, getSourceLocation()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc())
-  ));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  // clang-format off
-  EXPECT_THAT(
-      ExpectedRanges,
-      UnorderedElementsAre(
-STRING_LOCATION_PAIR(CtorInit, getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getSourceRange())
-  ));
-  // clang-format on
-}
-
-TEST(Introspection, SourceLocations_CXXCtorInitializer_pack) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST = buildASTFromCodeWithArgs(
-      R"cpp(
-template<typename... T>
-struct Templ {
-};
-
-template<typename... T>
-struct D : Templ<T...> {
-  D(T... t) : Templ<T>(t)... {}
-};
-)cpp",
-      {"-fno-delayed-template-parsing"}, "foo.cpp", "clang-tool",
-      std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(cxxConstructorDecl(
-          hasAnyConstructorInitializer(cxxCtorInitializer().bind("init"))))),
-      TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *CtorInit = BoundNodes[0].getNodeAs<CXXCtorInitializer>("init");
-
-  auto Result = NodeIntrospection::GetLocations(CtorInit);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  llvm::sort(ExpectedLocations);
-
-  // clang-format off
-  EXPECT_EQ(
-     llvm::ArrayRef(ExpectedLocations),
-      (ArrayRef<std::pair<std::string, SourceLocation>>{
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getLAngleLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getRAngleLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getTemplateNameLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getLAngleLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getRAngleLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getTemplateNameLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getEllipsisLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getLParenLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getMemberLocation()),
-STRING_LOCATION_STDPAIR(CtorInit, getRParenLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getSourceLocation()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getLAngleLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getRAngleLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getTemplateNameLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getLAngleLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getRAngleLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getTemplateNameLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getEndLoc())
-  }));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  // clang-format off
-  EXPECT_THAT(
-      ExpectedRanges,
-      UnorderedElementsAre(
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getBaseClassLoc().getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getTypeSourceInfo()->getTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(CtorInit, getSourceRange())
-  ));
-  // clang-format on
-}
-
-TEST(Introspection, SourceLocations_CXXBaseSpecifier_plain) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-class A {};
-class B : A {};
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(cxxRecordDecl(hasDirectBase(
-          cxxBaseSpecifier(hasType(asString("A"))).bind("base"))))),
-      TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *Base = BoundNodes[0].getNodeAs<CXXBaseSpecifier>("base");
-
-  auto Result = NodeIntrospection::GetLocations(Base);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(
-STRING_LOCATION_PAIR(Base, getBaseTypeLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc())
-  ));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedRanges, UnorderedElementsAre(
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getSourceRange())
-    ));
-  // clang-format on
-}
-
-TEST(Introspection, SourceLocations_CXXBaseSpecifier_accessspec) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-class A {};
-class B : public A {};
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(cxxRecordDecl(hasDirectBase(
-          cxxBaseSpecifier(hasType(asString("A"))).bind("base"))))),
-      TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *Base = BoundNodes[0].getNodeAs<CXXBaseSpecifier>("base");
-
-  auto Result = NodeIntrospection::GetLocations(Base);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(
-STRING_LOCATION_PAIR(Base, getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getBaseTypeLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc())
-  ));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedRanges, UnorderedElementsAre(
-STRING_LOCATION_PAIR(Base, getSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getSourceRange())
-  ));
-  // clang-format on
-}
-
-TEST(Introspection, SourceLocations_CXXBaseSpecifier_virtual) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-class A {};
-class B {};
-class C : virtual B, A {};
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(cxxRecordDecl(hasDirectBase(
-          cxxBaseSpecifier(hasType(asString("A"))).bind("base"))))),
-      TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *Base = BoundNodes[0].getNodeAs<CXXBaseSpecifier>("base");
-
-  auto Result = NodeIntrospection::GetLocations(Base);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(
-STRING_LOCATION_PAIR(Base, getBaseTypeLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc())
-  ));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedRanges, UnorderedElementsAre(
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getSourceRange())
-  ));
-  // clang-format on
-}
-
-TEST(Introspection, SourceLocations_CXXBaseSpecifier_template_base) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-template<typename T, typename U>
-class A {};
-class B : A<int, bool> {};
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes =
-      ast_matchers::match(decl(hasDescendant(cxxRecordDecl(
-                              hasDirectBase(cxxBaseSpecifier().bind("base"))))),
-                          TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *Base = BoundNodes[0].getNodeAs<CXXBaseSpecifier>("base");
-
-  auto Result = NodeIntrospection::GetLocations(Base);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(
-STRING_LOCATION_PAIR(Base, getBaseTypeLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getTemplateNameLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getTemplateNameLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getLAngleLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getLAngleLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getRAngleLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getAs<clang::TemplateSpecializationTypeLoc>().getRAngleLoc())
-  ));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedRanges, UnorderedElementsAre(
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::ElaboratedTypeLoc>().getNamedTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getSourceRange())
-  ));
-  // clang-format on
-}
-
-TEST(Introspection, SourceLocations_CXXBaseSpecifier_pack) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST = buildASTFromCodeWithArgs(
-      R"cpp(
-template<typename... T>
-struct Templ : T... {
-};
-)cpp",
-      {"-fno-delayed-template-parsing"}, "foo.cpp", "clang-tool",
-      std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes =
-      ast_matchers::match(decl(hasDescendant(cxxRecordDecl(
-                              hasDirectBase(cxxBaseSpecifier().bind("base"))))),
-                          TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *Base = BoundNodes[0].getNodeAs<CXXBaseSpecifier>("base");
-
-  auto Result = NodeIntrospection::GetLocations(Base);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(
-STRING_LOCATION_PAIR(Base, getBaseTypeLoc()),
-STRING_LOCATION_PAIR(Base, getEllipsisLoc()),
-STRING_LOCATION_PAIR(Base, getBeginLoc()),
-STRING_LOCATION_PAIR(Base, getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getEndLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getBeginLoc())
-  ));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  // clang-format off
-  EXPECT_THAT(ExpectedRanges, UnorderedElementsAre(
-STRING_LOCATION_PAIR(Base, getSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(Base, getTypeSourceInfo()->getTypeLoc().getLocalSourceRange())
-  ));
-  // clang-format on
-}
-
-TEST(Introspection, SourceLocations_FunctionProtoTypeLoc) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-int foo();
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(loc(functionProtoType()).bind("tl"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *TL = BoundNodes[0].getNodeAs<TypeLoc>("tl");
-  auto Result = NodeIntrospection::GetLocations(*TL);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  llvm::sort(ExpectedLocations);
-
-  // clang-format off
-  EXPECT_EQ(
-      llvm::ArrayRef(ExpectedLocations),
-          (ArrayRef<std::pair<std::string, SourceLocation>>{
-STRING_LOCATION_STDPAIR(TL, getAs<clang::FunctionTypeLoc>().getLParenLoc()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::FunctionTypeLoc>().getLocalRangeBegin()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::FunctionTypeLoc>().getLocalRangeEnd()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::FunctionTypeLoc>().getRParenLoc()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::BuiltinTypeLoc>().getBuiltinLoc()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::FunctionTypeLoc>().getReturnLoc().getAs<clang::BuiltinTypeLoc>().getNameLoc()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::FunctionTypeLoc>().getReturnLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::FunctionTypeLoc>().getReturnLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(TL, getBeginLoc()),
-STRING_LOCATION_STDPAIR(TL, getEndLoc()),
-STRING_LOCATION_STDPAIR(TL, getNextTypeLoc().getAs<clang::BuiltinTypeLoc>().getBuiltinLoc()),
-STRING_LOCATION_STDPAIR(TL, getNextTypeLoc().getAs<clang::BuiltinTypeLoc>().getNameLoc()),
-STRING_LOCATION_STDPAIR(TL, getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(TL, getNextTypeLoc().getEndLoc())
-        }));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  // clang-format off
-  EXPECT_THAT(
-      ExpectedRanges,
-      UnorderedElementsAre(
-STRING_LOCATION_PAIR(TL, getAs<clang::FunctionTypeLoc>().getParensRange()),
-STRING_LOCATION_PAIR(TL, getAs<clang::FunctionTypeLoc>().getReturnLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(TL, getAs<clang::FunctionTypeLoc>().getReturnLoc().getSourceRange()),
-STRING_LOCATION_PAIR(TL, getLocalSourceRange()),
-STRING_LOCATION_PAIR(TL, getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(TL, getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(TL, getSourceRange())
-          ));
-  // clang-format on
-}
-
-TEST(Introspection, SourceLocations_PointerTypeLoc) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-int* i;
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(
-          varDecl(hasName("i"), hasDescendant(loc(pointerType()).bind("tl"))))),
-      TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *TL = BoundNodes[0].getNodeAs<TypeLoc>("tl");
-  auto Result = NodeIntrospection::GetLocations(*TL);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  llvm::sort(ExpectedLocations);
-
-  // clang-format off
-  EXPECT_EQ(
-      llvm::ArrayRef(ExpectedLocations),
-      (ArrayRef<std::pair<std::string, SourceLocation>>{
-STRING_LOCATION_STDPAIR(TL, getAs<clang::PointerTypeLoc>().getPointeeLoc().getAs<clang::BuiltinTypeLoc>().getBuiltinLoc()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::PointerTypeLoc>().getPointeeLoc().getAs<clang::BuiltinTypeLoc>().getNameLoc()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::PointerTypeLoc>().getPointeeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::PointerTypeLoc>().getPointeeLoc().getEndLoc()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::PointerTypeLoc>().getSigilLoc()),
-STRING_LOCATION_STDPAIR(TL, getAs<clang::PointerTypeLoc>().getStarLoc()),
-STRING_LOCATION_STDPAIR(TL, getBeginLoc()),
-STRING_LOCATION_STDPAIR(TL, getEndLoc()),
-STRING_LOCATION_STDPAIR(TL, getNextTypeLoc().getAs<clang::BuiltinTypeLoc>().getBuiltinLoc()),
-STRING_LOCATION_STDPAIR(TL, getNextTypeLoc().getAs<clang::BuiltinTypeLoc>().getNameLoc()),
-STRING_LOCATION_STDPAIR(TL, getNextTypeLoc().getBeginLoc()),
-STRING_LOCATION_STDPAIR(TL, getNextTypeLoc().getEndLoc())
-}));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  // clang-format off
-  EXPECT_THAT(
-      ExpectedRanges,
-      UnorderedElementsAre(
-STRING_LOCATION_PAIR(TL, getAs<clang::PointerTypeLoc>().getPointeeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(TL, getAs<clang::PointerTypeLoc>().getPointeeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(TL, getLocalSourceRange()),
-STRING_LOCATION_PAIR(TL, getNextTypeLoc().getLocalSourceRange()),
-STRING_LOCATION_PAIR(TL, getNextTypeLoc().getSourceRange()),
-STRING_LOCATION_PAIR(TL, getSourceRange())
-          ));
-  // clang-format on
-}
-
-#ifndef _WIN32
-// This test doesn't work on windows due to use of the typeof extension.
-TEST(Introspection, SourceLocations_TypeOfTypeLoc) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-typeof (static_cast<void *>(0)) i;
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(
-          varDecl(hasName("i"), hasDescendant(loc(type()).bind("tl"))))),
-      TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *TL = BoundNodes[0].getNodeAs<TypeLoc>("tl");
-  auto Result = NodeIntrospection::GetLocations(*TL);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  EXPECT_THAT(ExpectedLocations,
-              UnorderedElementsAre(
-                  STRING_LOCATION_PAIR(TL, getBeginLoc()),
-                  STRING_LOCATION_PAIR(TL, getEndLoc()),
-                  STRING_LOCATION_PAIR(
-                      TL, getAs<clang::TypeOfExprTypeLoc>().getTypeofLoc()),
-                  STRING_LOCATION_PAIR(
-                      TL, getAs<clang::TypeOfExprTypeLoc>().getLParenLoc()),
-                  STRING_LOCATION_PAIR(
-                      TL, getAs<clang::TypeOfExprTypeLoc>().getRParenLoc())));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(ExpectedRanges,
-              UnorderedElementsAre(
-                  STRING_LOCATION_PAIR(TL, getLocalSourceRange()),
-                  STRING_LOCATION_PAIR(TL, getSourceRange()),
-                  STRING_LOCATION_PAIR(
-                      TL, getAs<clang::TypeOfExprTypeLoc>().getParensRange())));
-}
-#endif
-
-TEST(Introspection, SourceLocations_DeclarationNameInfo_Dtor) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-class Foo
-{
-  ~Foo() {}
-};
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(cxxDestructorDecl(hasName("~Foo")).bind("dtor"))), TU,
-      Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *Dtor = BoundNodes[0].getNodeAs<CXXDestructorDecl>("dtor");
-  auto NI = Dtor->getNameInfo();
-  auto Result = NodeIntrospection::GetLocations(NI);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  llvm::sort(ExpectedLocations);
-
-  // clang-format off
-  EXPECT_EQ(
-      llvm::ArrayRef(ExpectedLocations),
-      (ArrayRef<std::pair<std::string, SourceLocation>>{
-          STRING_LOCATION_STDPAIR((&NI), getBeginLoc()),
-          STRING_LOCATION_STDPAIR((&NI), getEndLoc()),
-          STRING_LOCATION_STDPAIR((&NI), getLoc()),
-          STRING_LOCATION_STDPAIR((&NI),
-getNamedTypeInfo()->getTypeLoc().getAs<clang::TypeSpecTypeLoc>().getNameLoc()),
-          STRING_LOCATION_STDPAIR(
-              (&NI), getNamedTypeInfo()->getTypeLoc().getBeginLoc()),
-          STRING_LOCATION_STDPAIR(
-              (&NI), getNamedTypeInfo()->getTypeLoc().getEndLoc())}));
-  // clang-format on
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(
-      ExpectedRanges,
-      UnorderedElementsAre(
-          STRING_LOCATION_PAIR(
-              (&NI), getNamedTypeInfo()->getTypeLoc().getLocalSourceRange()),
-          STRING_LOCATION_PAIR(
-              (&NI), getNamedTypeInfo()->getTypeLoc().getSourceRange()),
-          STRING_LOCATION_PAIR((&NI), getSourceRange())));
-}
-
-TEST(Introspection, SourceLocations_DeclarationNameInfo_CRef) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-
-  auto AST = buildASTFromCodeWithArgs(
-      R"cpp(
-template<typename T>
-struct MyContainer
-{
-    template <typename U>
-    void pushBack();
-};
-
-template<typename T>
-void foo()
-{
-    MyContainer<T> mc;
-    mc.template pushBack<int>();
-}
-)cpp",
-      {"-fno-delayed-template-parsing"}, "foo.cpp", "clang-tool",
-      std::make_shared<PCHContainerOperations>());
-
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(cxxDependentScopeMemberExpr(hasMemberName("pushBack")).bind("member"))), TU,
-      Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *Member = BoundNodes[0].getNodeAs<CXXDependentScopeMemberExpr>("member");
-  auto Result = NodeIntrospection::GetLocations(Member);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  llvm::sort(ExpectedLocations);
-
-  EXPECT_EQ(
-      llvm::ArrayRef(ExpectedLocations),
-      (ArrayRef<std::pair<std::string, SourceLocation>>{
-          STRING_LOCATION_STDPAIR(Member, getBeginLoc()),
-          STRING_LOCATION_STDPAIR(Member, getEndLoc()),
-          STRING_LOCATION_STDPAIR(Member, getExprLoc()),
-          STRING_LOCATION_STDPAIR(Member, getLAngleLoc()),
-          STRING_LOCATION_STDPAIR(Member, getMemberLoc()),
-          STRING_LOCATION_STDPAIR(Member, getMemberNameInfo().getBeginLoc()),
-          STRING_LOCATION_STDPAIR(Member, getMemberNameInfo().getEndLoc()),
-          STRING_LOCATION_STDPAIR(Member, getMemberNameInfo().getLoc()),
-          STRING_LOCATION_STDPAIR(Member, getOperatorLoc()),
-          STRING_LOCATION_STDPAIR(Member, getRAngleLoc()),
-          STRING_LOCATION_STDPAIR(Member, getTemplateKeywordLoc())}));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(
-      ExpectedRanges,
-      UnorderedElementsAre(
-          STRING_LOCATION_PAIR(Member, getMemberNameInfo().getSourceRange()),
-          STRING_LOCATION_PAIR(Member, getSourceRange())
-          ));
-}
-
-TEST(Introspection, SourceLocations_DeclarationNameInfo_ConvOp) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-class Foo
-{
-  bool operator==(const Foo&) const { return false; }
-};
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(cxxMethodDecl().bind("opeq"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *Opeq = BoundNodes[0].getNodeAs<CXXMethodDecl>("opeq");
-  auto NI = Opeq->getNameInfo();
-  auto Result = NodeIntrospection::GetLocations(NI);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  llvm::sort(ExpectedLocations);
-
-  EXPECT_EQ(llvm::ArrayRef(ExpectedLocations),
-            (ArrayRef<std::pair<std::string, SourceLocation>>{
-                STRING_LOCATION_STDPAIR((&NI), getBeginLoc()),
-                STRING_LOCATION_STDPAIR((&NI), getEndLoc()),
-                STRING_LOCATION_STDPAIR((&NI), getLoc())}));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(ExpectedRanges,
-              UnorderedElementsAre(
-                  STRING_LOCATION_PAIR((&NI), getSourceRange()),
-                  STRING_LOCATION_PAIR((&NI), getCXXOperatorNameRange())));
-}
-
-TEST(Introspection, SourceLocations_DeclarationNameInfo_LitOp) {
-  if (!NodeIntrospection::hasIntrospectionSupport())
-    GTEST_SKIP();
-  auto AST =
-      buildASTFromCode(R"cpp(
-long double operator"" _identity ( long double val )
-{
-    return val;
-}
-)cpp",
-                       "foo.cpp", std::make_shared<PCHContainerOperations>());
-  auto &Ctx = AST->getASTContext();
-  auto &TU = *Ctx.getTranslationUnitDecl();
-
-  auto BoundNodes = ast_matchers::match(
-      decl(hasDescendant(functionDecl().bind("litop"))), TU, Ctx);
-
-  EXPECT_EQ(BoundNodes.size(), 1u);
-
-  const auto *LitOp = BoundNodes[0].getNodeAs<FunctionDecl>("litop");
-  auto NI = LitOp->getNameInfo();
-  auto Result = NodeIntrospection::GetLocations(NI);
-
-  auto ExpectedLocations =
-      FormatExpected<SourceLocation>(Result.LocationAccessors);
-
-  llvm::sort(ExpectedLocations);
-
-  EXPECT_EQ(llvm::ArrayRef(ExpectedLocations),
-            (ArrayRef<std::pair<std::string, SourceLocation>>{
-                STRING_LOCATION_STDPAIR((&NI), getBeginLoc()),
-                STRING_LOCATION_STDPAIR((&NI), getCXXLiteralOperatorNameLoc()),
-                STRING_LOCATION_STDPAIR((&NI), getEndLoc()),
-                STRING_LOCATION_STDPAIR((&NI), getLoc())}));
-
-  auto ExpectedRanges = FormatExpected<SourceRange>(Result.RangeAccessors);
-
-  EXPECT_THAT(ExpectedRanges, UnorderedElementsAre(STRING_LOCATION_PAIR(
-                                  (&NI), getSourceRange())));
-}
diff --git a/llvm/utils/gn/secondary/clang/lib/Tooling/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Tooling/BUILD.gn
index e8240f2099ea..f8d5c3729775 100644
--- a/llvm/utils/gn/secondary/clang/lib/Tooling/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Tooling/BUILD.gn
@@ -1,21 +1,7 @@
-# FIXME: The cmake build runs DumpTool:clang-ast-dump to generate a json
-# file and feeds it into this step in non-debug builds or if an option is set.
-action("node_introspection_inc") {
-  script = "DumpTool/generate_cxx_src_locs.py"
-  outputs = [ "$target_gen_dir/clang/Tooling/NodeIntrospection.inc" ]
-  sources = [ "EmptyNodeIntrospection.inc.in" ]
-  args = [
-    "--use-empty-implementation=1",
-    "--empty-implementation=" + rebase_path(sources[0], root_build_dir),
-    "--output-file=" + rebase_path(outputs[0], root_build_dir),
-  ]
-}
-
 static_library("Tooling") {
   output_name = "clangTooling"
   configs += [ "//llvm/utils/gn/build:clang_code" ]
   deps = [
-    ":node_introspection_inc",
     "//clang/include/clang/Driver:Options",
     "//clang/lib/AST",
     "//clang/lib/ASTMatchers",
@@ -28,7 +14,6 @@ static_library("Tooling") {
     "//clang/lib/Tooling/Core",
     "//llvm/lib/TargetParser",
   ]
-  include_dirs = [ target_gen_dir ]
   sources = [
     "AllTUsExecution.cpp",
     "ArgumentsAdjusters.cpp",
@@ -41,7 +26,6 @@ static_library("Tooling") {
     "GuessTargetAndModeCompilationDatabase.cpp",
     "InterpolatingCompilationDatabase.cpp",
     "JSONCompilationDatabase.cpp",
-    "NodeIntrospection.cpp",
     "Refactoring.cpp",
     "RefactoringCallbacks.cpp",
     "StandaloneExecution.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Tooling/DumpTool/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Tooling/DumpTool/BUILD.gn
deleted file mode 100644
index 7d9d2113d459..000000000000
--- a/llvm/utils/gn/secondary/clang/lib/Tooling/DumpTool/BUILD.gn
+++ /dev/null
@@ -1,20 +0,0 @@
-executable("clang-ast-dump") {
-  configs += [ "//llvm/utils/gn/build:clang_code" ]
-  deps = [
-    "//clang/lib/AST",
-    "//clang/lib/ASTMatchers",
-    "//clang/lib/Basic",
-    "//clang/lib/Driver",
-    "//clang/lib/Format",
-    "//clang/lib/Frontend",
-    "//clang/lib/Lex",
-    "//clang/lib/Rewrite",
-    "//clang/lib/Serialization",
-    "//clang/lib/Tooling/Core",
-  ]
-
-  sources = [
-    "ASTSrcLocProcessor.cpp",
-    "ClangSrcLocDump.cpp",
-  ]
-}
diff --git a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
index 354934f4b18a..8b98e3beec28 100644
--- a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
@@ -15,7 +15,6 @@ group("unittests") {
     "Index:IndexTests",
     "InstallAPI:InstallAPITests",
     "Interpreter:ClangReplInterpreterTests",
-    "Introspection:IntrospectionTests",
     "Lex:LexTests",
     "Rename:ClangRenameTests",
     "Rewrite:RewriteTests",
diff --git a/llvm/utils/gn/secondary/clang/unittests/Introspection/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Introspection/BUILD.gn
deleted file mode 100644
index 687ac2b81265..000000000000
--- a/llvm/utils/gn/secondary/clang/unittests/Introspection/BUILD.gn
+++ /dev/null
@@ -1,19 +0,0 @@
-import("//third-party/unittest/unittest.gni")
-
-unittest("IntrospectionTests") {
-  configs += [ "//llvm/utils/gn/build:clang_code" ]
-  deps = [
-    "//clang/lib/AST",
-    "//clang/lib/ASTMatchers",
-    "//clang/lib/Basic",
-    "//clang/lib/Frontend",
-    "//clang/lib/Serialization",
-    "//clang/lib/Tooling",
-    "//llvm/lib/Support",
-    "//llvm/lib/Testing/Support",
-  ]
-
-  defines = [ "SKIP_INTROSPECTION_GENERATION" ]
-
-  sources = [ "IntrospectionTest.cpp" ]
-}
-- 
GitLab


From f5c8242042c959cbf476778e1d8a874fc8747801 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 17 May 2024 14:28:03 +0200
Subject: [PATCH 1121/1206] SimplifyLibCalls: Prefer to emit intrinsic in
 pow(2, x) -> ldexp(1, x) (#92363)

---
 .../lib/Transforms/Utils/SimplifyLibCalls.cpp |  18 +-
 .../Transforms/InstCombine/pow-to-ldexp.ll    | 492 ++++++++++++++++++
 .../test/Transforms/InstCombine/pow_fp_int.ll |   6 +-
 .../Transforms/InstCombine/pow_fp_int16.ll    |   6 +-
 4 files changed, 511 insertions(+), 11 deletions(-)
 create mode 100644 llvm/test/Transforms/InstCombine/pow-to-ldexp.ll

diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index c9f6839e4b94..c9567b740026 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -2092,11 +2092,19 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
   if (!Ty->isVectorTy() && match(Base, m_SpecificFP(2.0)) &&
       (isa<SIToFPInst>(Expo) || isa<UIToFPInst>(Expo)) &&
       hasFloatFn(M, TLI, Ty, LibFunc_ldexp, LibFunc_ldexpf, LibFunc_ldexpl)) {
-    if (Value *ExpoI = getIntToFPVal(Expo, B, TLI->getIntSize()))
-      return copyFlags(*Pow,
-                       emitBinaryFloatFnCall(ConstantFP::get(Ty, 1.0), ExpoI,
-                                             TLI, LibFunc_ldexp, LibFunc_ldexpf,
-                                             LibFunc_ldexpl, B, NoAttrs));
+    if (Value *ExpoI = getIntToFPVal(Expo, B, TLI->getIntSize())) {
+      Constant *One = ConstantFP::get(Ty, 1.0);
+
+      if (Pow->doesNotAccessMemory()) {
+        return copyFlags(*Pow, B.CreateIntrinsic(Intrinsic::ldexp,
+                                                 {Ty, ExpoI->getType()},
+                                                 {One, ExpoI}, Pow, "exp2"));
+      }
+
+      return copyFlags(*Pow, emitBinaryFloatFnCall(
+                                 One, ExpoI, TLI, LibFunc_ldexp, LibFunc_ldexpf,
+                                 LibFunc_ldexpl, B, NoAttrs));
+    }
   }
 
   // pow(2.0 ** n, x) -> exp2(n * x)
diff --git a/llvm/test/Transforms/InstCombine/pow-to-ldexp.ll b/llvm/test/Transforms/InstCombine/pow-to-ldexp.ll
new file mode 100644
index 000000000000..27249dd5d72a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pow-to-ldexp.ll
@@ -0,0 +1,492 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S -passes=instcombine %s | FileCheck -check-prefixes=CHECK,LDEXP,LDEXP-EXP2 %s
+; RUN: opt -S -passes=instcombine -disable-builtin=exp2f -disable-builtin=exp2 -disable-builtin=exp2l %s | FileCheck -check-prefixes=CHECK,LDEXP,LDEXP-NOEXP2 %s
+; RUN: opt -S -passes=instcombine -disable-builtin=ldexpf -disable-builtin=ldexp -disable-builtin=ldexpl %s | FileCheck -check-prefixes=CHECK,NOLDEXP %s
+
+
+define float @pow_sitofp_f32_const_base_2(i32 %x) {
+; LDEXP-LABEL: define float @pow_sitofp_f32_const_base_2(
+; LDEXP-SAME: i32 [[X:%.*]]) {
+; LDEXP-NEXT:    [[LDEXPF:%.*]] = tail call float @llvm.ldexp.f32.i32(float 1.000000e+00, i32 [[X]])
+; LDEXP-NEXT:    ret float [[LDEXPF]]
+;
+; NOLDEXP-LABEL: define float @pow_sitofp_f32_const_base_2(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to float
+; NOLDEXP-NEXT:    [[POW:%.*]] = tail call float @llvm.exp2.f32(float [[ITOFP]])
+; NOLDEXP-NEXT:    ret float [[POW]]
+;
+  %itofp = sitofp i32 %x to float
+  %pow = tail call float @llvm.pow.f32(float 2.000000e+00, float %itofp)
+  ret float %pow
+}
+
+define float @pow_sitofp_f32_const_base_2__flags(i32 %x) {
+; LDEXP-LABEL: define float @pow_sitofp_f32_const_base_2__flags(
+; LDEXP-SAME: i32 [[X:%.*]]) {
+; LDEXP-NEXT:    [[LDEXPF:%.*]] = tail call nnan nsz float @llvm.ldexp.f32.i32(float 1.000000e+00, i32 [[X]])
+; LDEXP-NEXT:    ret float [[LDEXPF]]
+;
+; NOLDEXP-LABEL: define float @pow_sitofp_f32_const_base_2__flags(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to float
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = tail call nnan nsz float @llvm.exp2.f32(float [[ITOFP]])
+; NOLDEXP-NEXT:    ret float [[EXP2]]
+;
+  %itofp = sitofp i32 %x to float
+  %pow = tail call nsz nnan float @llvm.pow.f32(float 2.000000e+00, float %itofp)
+  ret float %pow
+}
+
+define float @pow_uitofp_f32_const_base_2(i32 %x) {
+; LDEXP-EXP2-LABEL: define float @pow_uitofp_f32_const_base_2(
+; LDEXP-EXP2-SAME: i32 [[X:%.*]]) {
+; LDEXP-EXP2-NEXT:    [[ITOFP:%.*]] = uitofp i32 [[X]] to float
+; LDEXP-EXP2-NEXT:    [[EXP2:%.*]] = tail call float @llvm.exp2.f32(float [[ITOFP]])
+; LDEXP-EXP2-NEXT:    ret float [[EXP2]]
+;
+; LDEXP-NOEXP2-LABEL: define float @pow_uitofp_f32_const_base_2(
+; LDEXP-NOEXP2-SAME: i32 [[X:%.*]]) {
+; LDEXP-NOEXP2-NEXT:    [[ITOFP:%.*]] = uitofp i32 [[X]] to float
+; LDEXP-NOEXP2-NEXT:    [[POW:%.*]] = tail call float @llvm.pow.f32(float 2.000000e+00, float [[ITOFP]])
+; LDEXP-NOEXP2-NEXT:    ret float [[POW]]
+;
+; NOLDEXP-LABEL: define float @pow_uitofp_f32_const_base_2(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = uitofp i32 [[X]] to float
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = tail call float @llvm.exp2.f32(float [[ITOFP]])
+; NOLDEXP-NEXT:    ret float [[EXP2]]
+;
+  %itofp = uitofp i32 %x to float
+  %pow = tail call float @llvm.pow.f32(float 2.000000e+00, float %itofp)
+  ret float %pow
+}
+
+define float @pow_sitofp_f32_const_base_4(i32 %x) {
+; LDEXP-EXP2-LABEL: define float @pow_sitofp_f32_const_base_4(
+; LDEXP-EXP2-SAME: i32 [[X:%.*]]) {
+; LDEXP-EXP2-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to float
+; LDEXP-EXP2-NEXT:    [[MUL:%.*]] = fmul float [[ITOFP]], 2.000000e+00
+; LDEXP-EXP2-NEXT:    [[EXP2:%.*]] = tail call float @llvm.exp2.f32(float [[MUL]])
+; LDEXP-EXP2-NEXT:    ret float [[EXP2]]
+;
+; LDEXP-NOEXP2-LABEL: define float @pow_sitofp_f32_const_base_4(
+; LDEXP-NOEXP2-SAME: i32 [[X:%.*]]) {
+; LDEXP-NOEXP2-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to float
+; LDEXP-NOEXP2-NEXT:    [[POW:%.*]] = tail call float @llvm.pow.f32(float 4.000000e+00, float [[ITOFP]])
+; LDEXP-NOEXP2-NEXT:    ret float [[POW]]
+;
+; NOLDEXP-LABEL: define float @pow_sitofp_f32_const_base_4(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to float
+; NOLDEXP-NEXT:    [[MUL:%.*]] = fmul float [[ITOFP]], 2.000000e+00
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = tail call float @llvm.exp2.f32(float [[MUL]])
+; NOLDEXP-NEXT:    ret float [[EXP2]]
+;
+  %itofp = sitofp i32 %x to float
+  %pow = tail call float @llvm.pow.f32(float 4.000000e+00, float %itofp)
+  ret float %pow
+}
+
+define float @pow_sitofp_f32_const_base_16(i32 %x) {
+; LDEXP-EXP2-LABEL: define float @pow_sitofp_f32_const_base_16(
+; LDEXP-EXP2-SAME: i32 [[X:%.*]]) {
+; LDEXP-EXP2-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to float
+; LDEXP-EXP2-NEXT:    [[MUL:%.*]] = fmul float [[ITOFP]], 4.000000e+00
+; LDEXP-EXP2-NEXT:    [[EXP2:%.*]] = tail call float @llvm.exp2.f32(float [[MUL]])
+; LDEXP-EXP2-NEXT:    ret float [[EXP2]]
+;
+; LDEXP-NOEXP2-LABEL: define float @pow_sitofp_f32_const_base_16(
+; LDEXP-NOEXP2-SAME: i32 [[X:%.*]]) {
+; LDEXP-NOEXP2-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to float
+; LDEXP-NOEXP2-NEXT:    [[POW:%.*]] = tail call float @llvm.pow.f32(float 1.600000e+01, float [[ITOFP]])
+; LDEXP-NOEXP2-NEXT:    ret float [[POW]]
+;
+; NOLDEXP-LABEL: define float @pow_sitofp_f32_const_base_16(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to float
+; NOLDEXP-NEXT:    [[MUL:%.*]] = fmul float [[ITOFP]], 4.000000e+00
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = tail call float @llvm.exp2.f32(float [[MUL]])
+; NOLDEXP-NEXT:    ret float [[EXP2]]
+;
+  %itofp = sitofp i32 %x to float
+  %pow = tail call float @llvm.pow.f32(float 16.000000e+00, float %itofp)
+  ret float %pow
+}
+
+define double @pow_sitofp_f64_const_base_2(i32 %x) {
+; LDEXP-LABEL: define double @pow_sitofp_f64_const_base_2(
+; LDEXP-SAME: i32 [[X:%.*]]) {
+; LDEXP-NEXT:    [[LDEXP:%.*]] = tail call double @llvm.ldexp.f64.i32(double 1.000000e+00, i32 [[X]])
+; LDEXP-NEXT:    ret double [[LDEXP]]
+;
+; NOLDEXP-LABEL: define double @pow_sitofp_f64_const_base_2(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to double
+; NOLDEXP-NEXT:    [[POW:%.*]] = tail call double @llvm.exp2.f64(double [[ITOFP]])
+; NOLDEXP-NEXT:    ret double [[POW]]
+;
+  %itofp = sitofp i32 %x to double
+  %pow = tail call double @llvm.pow.f64(double 2.000000e+00, double %itofp)
+  ret double %pow
+}
+
+define half @pow_sitofp_f16_const_base_2(i32 %x) {
+; CHECK-LABEL: define half @pow_sitofp_f16_const_base_2(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to half
+; CHECK-NEXT:    [[POW:%.*]] = tail call half @llvm.pow.f16(half 0xH4000, half [[ITOFP]])
+; CHECK-NEXT:    ret half [[POW]]
+;
+  %itofp = sitofp i32 %x to half
+  %pow = tail call half @llvm.pow.f16(half 2.000000e+00, half %itofp)
+  ret half %pow
+}
+
+define <2 x float> @pow_sitofp_v2f32_const_base_2(<2 x i32> %x) {
+; LDEXP-EXP2-LABEL: define <2 x float> @pow_sitofp_v2f32_const_base_2(
+; LDEXP-EXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-EXP2-NEXT:    [[EXP2:%.*]] = tail call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x i32> [[X]])
+; LDEXP-EXP2-NEXT:    ret <2 x float> [[EXP2]]
+;
+; LDEXP-NOEXP2-LABEL: define <2 x float> @pow_sitofp_v2f32_const_base_2(
+; LDEXP-NOEXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-NOEXP2-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x float>
+; LDEXP-NOEXP2-NEXT:    [[POW:%.*]] = tail call <2 x float> @llvm.pow.v2f32(<2 x float> <float 2.000000e+00, float 2.000000e+00>, <2 x float> [[ITOFP]])
+; LDEXP-NOEXP2-NEXT:    ret <2 x float> [[POW]]
+;
+; NOLDEXP-LABEL: define <2 x float> @pow_sitofp_v2f32_const_base_2(
+; NOLDEXP-SAME: <2 x i32> [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x float>
+; NOLDEXP-NEXT:    [[POW:%.*]] = tail call <2 x float> @llvm.exp2.v2f32(<2 x float> [[ITOFP]])
+; NOLDEXP-NEXT:    ret <2 x float> [[POW]]
+;
+  %itofp = sitofp <2 x i32> %x to <2 x float>
+  %pow = tail call <2 x float> @llvm.pow.v2f32(<2 x float> <float 2.000000e+00, float 2.000000e+00>, <2 x float> %itofp)
+  ret <2 x float> %pow
+}
+
+define <2 x float> @pow_sitofp_v2f32_const_base_8(<2 x i32> %x) {
+; LDEXP-EXP2-LABEL: define <2 x float> @pow_sitofp_v2f32_const_base_8(
+; LDEXP-EXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-EXP2-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x float>
+; LDEXP-EXP2-NEXT:    [[MUL:%.*]] = fmul <2 x float> [[ITOFP]], <float 3.000000e+00, float 3.000000e+00>
+; LDEXP-EXP2-NEXT:    [[EXP2:%.*]] = tail call <2 x float> @llvm.exp2.v2f32(<2 x float> [[MUL]])
+; LDEXP-EXP2-NEXT:    ret <2 x float> [[EXP2]]
+;
+; LDEXP-NOEXP2-LABEL: define <2 x float> @pow_sitofp_v2f32_const_base_8(
+; LDEXP-NOEXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-NOEXP2-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x float>
+; LDEXP-NOEXP2-NEXT:    [[POW:%.*]] = tail call <2 x float> @llvm.pow.v2f32(<2 x float> <float 8.000000e+00, float 8.000000e+00>, <2 x float> [[ITOFP]])
+; LDEXP-NOEXP2-NEXT:    ret <2 x float> [[POW]]
+;
+; NOLDEXP-LABEL: define <2 x float> @pow_sitofp_v2f32_const_base_8(
+; NOLDEXP-SAME: <2 x i32> [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x float>
+; NOLDEXP-NEXT:    [[MUL:%.*]] = fmul <2 x float> [[ITOFP]], <float 3.000000e+00, float 3.000000e+00>
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = tail call <2 x float> @llvm.exp2.v2f32(<2 x float> [[MUL]])
+; NOLDEXP-NEXT:    ret <2 x float> [[EXP2]]
+;
+  %itofp = sitofp <2 x i32> %x to <2 x float>
+  %pow = tail call <2 x float> @llvm.pow.v2f32(<2 x float> <float 8.000000e+00, float 8.000000e+00>, <2 x float> %itofp)
+  ret <2 x float> %pow
+}
+
+define <2 x float> @pow_sitofp_v2f32_const_base_mixed_2(<2 x i32> %x) {
+; CHECK-LABEL: define <2 x float> @pow_sitofp_v2f32_const_base_mixed_2(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x float>
+; CHECK-NEXT:    [[POW:%.*]] = tail call <2 x float> @llvm.pow.v2f32(<2 x float> <float 2.000000e+00, float 4.000000e+00>, <2 x float> [[ITOFP]])
+; CHECK-NEXT:    ret <2 x float> [[POW]]
+;
+  %itofp = sitofp <2 x i32> %x to <2 x float>
+  %pow = tail call <2 x float> @llvm.pow.v2f32(<2 x float> <float 2.000000e+00, float 4.000000e+00>, <2 x float> %itofp)
+  ret <2 x float> %pow
+}
+
+define <2 x float> @pow_sitofp_v2f32_const_base_2__flags(<2 x i32> %x) {
+; LDEXP-EXP2-LABEL: define <2 x float> @pow_sitofp_v2f32_const_base_2__flags(
+; LDEXP-EXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-EXP2-NEXT:    [[EXP2:%.*]] = tail call nsz afn <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x i32> [[X]])
+; LDEXP-EXP2-NEXT:    ret <2 x float> [[EXP2]]
+;
+; LDEXP-NOEXP2-LABEL: define <2 x float> @pow_sitofp_v2f32_const_base_2__flags(
+; LDEXP-NOEXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-NOEXP2-NEXT:    [[POW:%.*]] = tail call nsz afn <2 x float> @llvm.powi.v2f32.v2i32(<2 x float> <float 2.000000e+00, float 2.000000e+00>, <2 x i32> [[X]])
+; LDEXP-NOEXP2-NEXT:    ret <2 x float> [[POW]]
+;
+; NOLDEXP-LABEL: define <2 x float> @pow_sitofp_v2f32_const_base_2__flags(
+; NOLDEXP-SAME: <2 x i32> [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x float>
+; NOLDEXP-NEXT:    [[POW:%.*]] = tail call nsz afn <2 x float> @llvm.exp2.v2f32(<2 x float> [[ITOFP]])
+; NOLDEXP-NEXT:    ret <2 x float> [[POW]]
+;
+  %itofp = sitofp <2 x i32> %x to <2 x float>
+  %pow = tail call nsz afn <2 x float> @llvm.pow.v2f32(<2 x float> <float 2.000000e+00, float 2.000000e+00>, <2 x float> %itofp)
+  ret <2 x float> %pow
+}
+
+define <vscale x 4 x float> @pow_sitofp_nxv4f32_const_base_2(<vscale x 4 x i32> %x) {
+; LDEXP-EXP2-LABEL: define <vscale x 4 x float> @pow_sitofp_nxv4f32_const_base_2(
+; LDEXP-EXP2-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; LDEXP-EXP2-NEXT:    [[EXP2:%.*]] = tail call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i32(<vscale x 4 x float> shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 1.000000e+00, i64 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> [[X]])
+; LDEXP-EXP2-NEXT:    ret <vscale x 4 x float> [[EXP2]]
+;
+; LDEXP-NOEXP2-LABEL: define <vscale x 4 x float> @pow_sitofp_nxv4f32_const_base_2(
+; LDEXP-NOEXP2-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; LDEXP-NOEXP2-NEXT:    [[ITOFP:%.*]] = sitofp <vscale x 4 x i32> [[X]] to <vscale x 4 x float>
+; LDEXP-NOEXP2-NEXT:    [[POW:%.*]] = tail call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> shufflevector (<vscale x 4 x float> insertelement (<vscale x 4 x float> poison, float 2.000000e+00, i64 0), <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x float> [[ITOFP]])
+; LDEXP-NOEXP2-NEXT:    ret <vscale x 4 x float> [[POW]]
+;
+; NOLDEXP-LABEL: define <vscale x 4 x float> @pow_sitofp_nxv4f32_const_base_2(
+; NOLDEXP-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp <vscale x 4 x i32> [[X]] to <vscale x 4 x float>
+; NOLDEXP-NEXT:    [[POW:%.*]] = tail call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> [[ITOFP]])
+; NOLDEXP-NEXT:    ret <vscale x 4 x float> [[POW]]
+;
+  %itofp = sitofp <vscale x 4 x i32> %x to <vscale x 4 x float>
+  %pow = tail call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> splat (float 2.0), <vscale x 4 x float> %itofp)
+  ret <vscale x 4 x float> %pow
+}
+
+define <2 x half> @pow_sitofp_v2f16_const_base_2(<2 x i32> %x) {
+; LDEXP-EXP2-LABEL: define <2 x half> @pow_sitofp_v2f16_const_base_2(
+; LDEXP-EXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-EXP2-NEXT:    [[EXP2:%.*]] = tail call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> <half 0xH3C00, half 0xH3C00>, <2 x i32> [[X]])
+; LDEXP-EXP2-NEXT:    ret <2 x half> [[EXP2]]
+;
+; LDEXP-NOEXP2-LABEL: define <2 x half> @pow_sitofp_v2f16_const_base_2(
+; LDEXP-NOEXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-NOEXP2-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x half>
+; LDEXP-NOEXP2-NEXT:    [[POW:%.*]] = tail call <2 x half> @llvm.pow.v2f16(<2 x half> <half 0xH4000, half 0xH4000>, <2 x half> [[ITOFP]])
+; LDEXP-NOEXP2-NEXT:    ret <2 x half> [[POW]]
+;
+; NOLDEXP-LABEL: define <2 x half> @pow_sitofp_v2f16_const_base_2(
+; NOLDEXP-SAME: <2 x i32> [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x half>
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = tail call <2 x half> @llvm.exp2.v2f16(<2 x half> [[ITOFP]])
+; NOLDEXP-NEXT:    ret <2 x half> [[EXP2]]
+;
+  %itofp = sitofp <2 x i32> %x to <2 x half>
+  %pow = tail call <2 x half> @llvm.pow.v2f16(<2 x half> <half 2.000000e+00, half 2.000000e+00>, <2 x half> %itofp)
+  ret <2 x half> %pow
+}
+
+define <2 x double> @pow_sitofp_v2f64_const_base_2(<2 x i32> %x) {
+; LDEXP-EXP2-LABEL: define <2 x double> @pow_sitofp_v2f64_const_base_2(
+; LDEXP-EXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-EXP2-NEXT:    [[EXP2:%.*]] = tail call <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double> <double 1.000000e+00, double 1.000000e+00>, <2 x i32> [[X]])
+; LDEXP-EXP2-NEXT:    ret <2 x double> [[EXP2]]
+;
+; LDEXP-NOEXP2-LABEL: define <2 x double> @pow_sitofp_v2f64_const_base_2(
+; LDEXP-NOEXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-NOEXP2-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x double>
+; LDEXP-NOEXP2-NEXT:    [[POW:%.*]] = tail call <2 x double> @llvm.pow.v2f64(<2 x double> <double 2.000000e+00, double 2.000000e+00>, <2 x double> [[ITOFP]])
+; LDEXP-NOEXP2-NEXT:    ret <2 x double> [[POW]]
+;
+; NOLDEXP-LABEL: define <2 x double> @pow_sitofp_v2f64_const_base_2(
+; NOLDEXP-SAME: <2 x i32> [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x double>
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = tail call <2 x double> @llvm.exp2.v2f64(<2 x double> [[ITOFP]])
+; NOLDEXP-NEXT:    ret <2 x double> [[EXP2]]
+;
+  %itofp = sitofp <2 x i32> %x to <2 x double>
+  %pow = tail call <2 x double> @llvm.pow.v2f64(<2 x double> <double 2.000000e+00, double 2.000000e+00>, <2 x double> %itofp)
+  ret <2 x double> %pow
+}
+
+define <2 x half> @pow_sitofp_v2f16_const_base_8(<2 x i32> %x) {
+; EXP2-LABEL: define <2 x half> @pow_sitofp_v2f16_const_base_8(
+; EXP2-SAME: <2 x i32> [[X:%.*]]) {
+; EXP2-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x half>
+; EXP2-NEXT:    [[MUL:%.*]] = fmul <2 x half> [[ITOFP]], <half 0xH4200, half 0xH4200>
+; EXP2-NEXT:    [[EXP2:%.*]] = tail call <2 x half> @llvm.exp2.v2f16(<2 x half> [[MUL]])
+; EXP2-NEXT:    ret <2 x half> [[EXP2]]
+;
+; LDEXP-EXP2-LABEL: define <2 x half> @pow_sitofp_v2f16_const_base_8(
+; LDEXP-EXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-EXP2-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x half>
+; LDEXP-EXP2-NEXT:    [[MUL:%.*]] = fmul <2 x half> [[ITOFP]], <half 0xH4200, half 0xH4200>
+; LDEXP-EXP2-NEXT:    [[EXP2:%.*]] = tail call <2 x half> @llvm.exp2.v2f16(<2 x half> [[MUL]])
+; LDEXP-EXP2-NEXT:    ret <2 x half> [[EXP2]]
+;
+; LDEXP-NOEXP2-LABEL: define <2 x half> @pow_sitofp_v2f16_const_base_8(
+; LDEXP-NOEXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-NOEXP2-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x half>
+; LDEXP-NOEXP2-NEXT:    [[POW:%.*]] = tail call <2 x half> @llvm.pow.v2f16(<2 x half> <half 0xH4800, half 0xH4800>, <2 x half> [[ITOFP]])
+; LDEXP-NOEXP2-NEXT:    ret <2 x half> [[POW]]
+;
+; NOLDEXP-LABEL: define <2 x half> @pow_sitofp_v2f16_const_base_8(
+; NOLDEXP-SAME: <2 x i32> [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x half>
+; NOLDEXP-NEXT:    [[MUL:%.*]] = fmul <2 x half> [[ITOFP]], <half 0xH4200, half 0xH4200>
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = tail call <2 x half> @llvm.exp2.v2f16(<2 x half> [[MUL]])
+; NOLDEXP-NEXT:    ret <2 x half> [[EXP2]]
+;
+  %itofp = sitofp <2 x i32> %x to <2 x half>
+  %pow = tail call <2 x half> @llvm.pow.v2f16(<2 x half> <half 8.000000e+00, half 8.000000e+00>, <2 x half> %itofp)
+  ret <2 x half> %pow
+}
+
+define <2 x double> @pow_sitofp_v2f64_const_base_8(<2 x i32> %x) {
+; EXP2-LABEL: define <2 x double> @pow_sitofp_v2f64_const_base_8(
+; EXP2-SAME: <2 x i32> [[X:%.*]]) {
+; EXP2-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x double>
+; EXP2-NEXT:    [[MUL:%.*]] = fmul <2 x double> [[ITOFP]], <double 3.000000e+00, double 3.000000e+00>
+; EXP2-NEXT:    [[EXP2:%.*]] = tail call <2 x double> @llvm.exp2.v2f64(<2 x double> [[MUL]])
+; EXP2-NEXT:    ret <2 x double> [[EXP2]]
+;
+; LDEXP-EXP2-LABEL: define <2 x double> @pow_sitofp_v2f64_const_base_8(
+; LDEXP-EXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-EXP2-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x double>
+; LDEXP-EXP2-NEXT:    [[MUL:%.*]] = fmul <2 x double> [[ITOFP]], <double 3.000000e+00, double 3.000000e+00>
+; LDEXP-EXP2-NEXT:    [[EXP2:%.*]] = tail call <2 x double> @llvm.exp2.v2f64(<2 x double> [[MUL]])
+; LDEXP-EXP2-NEXT:    ret <2 x double> [[EXP2]]
+;
+; LDEXP-NOEXP2-LABEL: define <2 x double> @pow_sitofp_v2f64_const_base_8(
+; LDEXP-NOEXP2-SAME: <2 x i32> [[X:%.*]]) {
+; LDEXP-NOEXP2-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x double>
+; LDEXP-NOEXP2-NEXT:    [[POW:%.*]] = tail call <2 x double> @llvm.pow.v2f64(<2 x double> <double 8.000000e+00, double 8.000000e+00>, <2 x double> [[ITOFP]])
+; LDEXP-NOEXP2-NEXT:    ret <2 x double> [[POW]]
+;
+; NOLDEXP-LABEL: define <2 x double> @pow_sitofp_v2f64_const_base_8(
+; NOLDEXP-SAME: <2 x i32> [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp <2 x i32> [[X]] to <2 x double>
+; NOLDEXP-NEXT:    [[MUL:%.*]] = fmul <2 x double> [[ITOFP]], <double 3.000000e+00, double 3.000000e+00>
+; NOLDEXP-NEXT:    [[EXP2:%.*]] = tail call <2 x double> @llvm.exp2.v2f64(<2 x double> [[MUL]])
+; NOLDEXP-NEXT:    ret <2 x double> [[EXP2]]
+;
+  %itofp = sitofp <2 x i32> %x to <2 x double>
+  %pow = tail call <2 x double> @llvm.pow.v2f64(<2 x double> <double 8.000000e+00, double 8.000000e+00>, <2 x double> %itofp)
+  ret <2 x double> %pow
+}
+
+define fp128 @pow_sitofp_fp128_const_base_2(i32 %x) {
+; LDEXP-LABEL: define fp128 @pow_sitofp_fp128_const_base_2(
+; LDEXP-SAME: i32 [[X:%.*]]) {
+; LDEXP-NEXT:    [[LDEXPL:%.*]] = tail call fp128 @llvm.ldexp.f128.i32(fp128 0xL00000000000000003FFF000000000000, i32 [[X]])
+; LDEXP-NEXT:    ret fp128 [[LDEXPL]]
+;
+; NOLDEXP-LABEL: define fp128 @pow_sitofp_fp128_const_base_2(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to fp128
+; NOLDEXP-NEXT:    [[POW:%.*]] = tail call fp128 @llvm.exp2.f128(fp128 [[ITOFP]])
+; NOLDEXP-NEXT:    ret fp128 [[POW]]
+;
+  %itofp = sitofp i32 %x to fp128
+  %pow = tail call fp128 @llvm.pow.fp128(fp128 0xL00000000000000004000000000000000, fp128 %itofp)
+  ret fp128 %pow
+}
+
+; FIXME: This asserts
+; define bfloat @pow_sitofp_bf16_const_base_2(i32 %x) {
+;   %itofp = sitofp i32 %x to bfloat
+;   %pow = tail call bfloat @llvm.pow.bf16(bfloat 2.000000e+00, bfloat %itofp)
+;   ret bfloat %pow
+; }
+
+; FIXME: This asserts
+; define x86_fp80 @pow_sitofp_x86_fp80_const_base_2(i32 %x) {
+;   %itofp = sitofp i32 %x to x86_fp80
+;   %fp2 = fpext float 2.0 to x86_fp80
+;   %pow = tail call x86_fp80 @llvm.pow.f80(x86_fp80 %fp2, x86_fp80 %itofp)
+;   ret x86_fp80 %pow
+; }
+
+; FIXME: This asserts
+; define ppc_fp128 @pow_sitofp_ppc_fp128_const_base_2(i32 %x) {
+;   %itofp = sitofp i32 %x to ppc_fp128
+;   %fp2 = fpext float 2.0 to ppc_fp128
+;   %pow = tail call ppc_fp128 @llvm.pow.ppcf128(ppc_fp128 %fp2, ppc_fp128 %itofp)
+;   ret ppc_fp128 %pow
+; }
+
+
+declare float @powf(float, float)
+declare double @pow(double, double)
+declare fp128 @powl(fp128, fp128)
+
+define float @libcall_powf_sitofp_f32_const_base_2(i32 %x) {
+; LDEXP-LABEL: define float @libcall_powf_sitofp_f32_const_base_2(
+; LDEXP-SAME: i32 [[X:%.*]]) {
+; LDEXP-NEXT:    [[LDEXPF:%.*]] = tail call float @ldexpf(float 1.000000e+00, i32 [[X]])
+; LDEXP-NEXT:    ret float [[LDEXPF]]
+;
+; NOLDEXP-LABEL: define float @libcall_powf_sitofp_f32_const_base_2(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to float
+; NOLDEXP-NEXT:    [[EXP2F:%.*]] = tail call float @exp2f(float [[ITOFP]])
+; NOLDEXP-NEXT:    ret float [[EXP2F]]
+;
+  %itofp = sitofp i32 %x to float
+  %pow = tail call float @powf(float 2.000000e+00, float %itofp)
+  ret float %pow
+}
+
+define float @libcall_powf_sitofp_f32_const_base_2__flags(i32 %x) {
+; LDEXP-LABEL: define float @libcall_powf_sitofp_f32_const_base_2__flags(
+; LDEXP-SAME: i32 [[X:%.*]]) {
+; LDEXP-NEXT:    [[LDEXPF:%.*]] = tail call nnan nsz float @ldexpf(float 1.000000e+00, i32 [[X]])
+; LDEXP-NEXT:    ret float [[LDEXPF]]
+;
+; NOLDEXP-LABEL: define float @libcall_powf_sitofp_f32_const_base_2__flags(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to float
+; NOLDEXP-NEXT:    [[EXP2F:%.*]] = tail call nnan nsz float @exp2f(float [[ITOFP]])
+; NOLDEXP-NEXT:    ret float [[EXP2F]]
+;
+  %itofp = sitofp i32 %x to float
+  %pow = tail call nnan nsz float @powf(float 2.000000e+00, float %itofp)
+  ret float %pow
+}
+
+define float @readnone_libcall_powf_sitofp_f32_const_base_2(i32 %x) {
+; LDEXP-LABEL: define float @readnone_libcall_powf_sitofp_f32_const_base_2(
+; LDEXP-SAME: i32 [[X:%.*]]) {
+; LDEXP-NEXT:    [[LDEXPF:%.*]] = tail call float @llvm.ldexp.f32.i32(float 1.000000e+00, i32 [[X]])
+; LDEXP-NEXT:    ret float [[LDEXPF]]
+;
+; NOLDEXP-LABEL: define float @readnone_libcall_powf_sitofp_f32_const_base_2(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to float
+; NOLDEXP-NEXT:    [[POW:%.*]] = tail call float @llvm.exp2.f32(float [[ITOFP]])
+; NOLDEXP-NEXT:    ret float [[POW]]
+;
+  %itofp = sitofp i32 %x to float
+  %pow = tail call float @powf(float 2.000000e+00, float %itofp) memory(none)
+  ret float %pow
+}
+
+define double @readnone_libcall_pow_sitofp_f32_const_base_2(i32 %x) {
+; LDEXP-LABEL: define double @readnone_libcall_pow_sitofp_f32_const_base_2(
+; LDEXP-SAME: i32 [[X:%.*]]) {
+; LDEXP-NEXT:    [[LDEXP:%.*]] = tail call double @llvm.ldexp.f64.i32(double 1.000000e+00, i32 [[X]])
+; LDEXP-NEXT:    ret double [[LDEXP]]
+;
+; NOLDEXP-LABEL: define double @readnone_libcall_pow_sitofp_f32_const_base_2(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to double
+; NOLDEXP-NEXT:    [[POW:%.*]] = tail call double @llvm.exp2.f64(double [[ITOFP]])
+; NOLDEXP-NEXT:    ret double [[POW]]
+;
+  %itofp = sitofp i32 %x to double
+  %pow = tail call double @pow(double 2.000000e+00, double %itofp) memory(none)
+  ret double %pow
+}
+
+define fp128 @readnone_libcall_powl_sitofp_fp128_const_base_2(i32 %x) {
+; LDEXP-LABEL: define fp128 @readnone_libcall_powl_sitofp_fp128_const_base_2(
+; LDEXP-SAME: i32 [[X:%.*]]) {
+; LDEXP-NEXT:    [[LDEXPL:%.*]] = tail call fp128 @llvm.ldexp.f128.i32(fp128 0xL00000000000000003FFF000000000000, i32 [[X]])
+; LDEXP-NEXT:    ret fp128 [[LDEXPL]]
+;
+; NOLDEXP-LABEL: define fp128 @readnone_libcall_powl_sitofp_fp128_const_base_2(
+; NOLDEXP-SAME: i32 [[X:%.*]]) {
+; NOLDEXP-NEXT:    [[ITOFP:%.*]] = sitofp i32 [[X]] to fp128
+; NOLDEXP-NEXT:    [[POW:%.*]] = tail call fp128 @llvm.exp2.f128(fp128 [[ITOFP]])
+; NOLDEXP-NEXT:    ret fp128 [[POW]]
+;
+  %itofp = sitofp i32 %x to fp128
+  %pow = tail call fp128 @powl(fp128 0xL00000000000000004000000000000000, fp128 %itofp) memory(none)
+  ret fp128 %pow
+}
diff --git a/llvm/test/Transforms/InstCombine/pow_fp_int.ll b/llvm/test/Transforms/InstCombine/pow_fp_int.ll
index 7b194b3f8925..e5546075edc9 100644
--- a/llvm/test/Transforms/InstCombine/pow_fp_int.ll
+++ b/llvm/test/Transforms/InstCombine/pow_fp_int.ll
@@ -58,7 +58,7 @@ define double @pow_uitofp_double_const_base_fast(i31 %x) {
 define double @pow_sitofp_double_const_base_2_fast(i32 %x) {
 ; CHECK-LABEL: define double @pow_sitofp_double_const_base_2_fast(
 ; CHECK-SAME: i32 [[X:%.*]]) {
-; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call afn float @ldexpf(float 1.000000e+00, i32 [[X]])
+; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call afn float @llvm.ldexp.f32.i32(float 1.000000e+00, i32 [[X]])
 ; CHECK-NEXT:    [[RES:%.*]] = fpext float [[LDEXPF]] to double
 ; CHECK-NEXT:    ret double [[RES]]
 ;
@@ -87,7 +87,7 @@ define double @pow_uitofp_const_base_2_fast(i31 %x) {
 ; CHECK-LABEL: define double @pow_uitofp_const_base_2_fast(
 ; CHECK-SAME: i31 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i31 [[X]] to i32
-; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call afn float @ldexpf(float 1.000000e+00, i32 [[TMP1]])
+; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call afn float @llvm.ldexp.f32.i32(float 1.000000e+00, i32 [[TMP1]])
 ; CHECK-NEXT:    [[RES:%.*]] = fpext float [[LDEXPF]] to double
 ; CHECK-NEXT:    ret double [[RES]]
 ;
@@ -373,7 +373,7 @@ define double @pow_uitofp_const_base_no_fast(i32 %x) {
 define double @pow_sitofp_const_base_2_no_fast(i32 %x) {
 ; CHECK-LABEL: define double @pow_sitofp_const_base_2_no_fast(
 ; CHECK-SAME: i32 [[X:%.*]]) {
-; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call float @ldexpf(float 1.000000e+00, i32 [[X]])
+; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call float @llvm.ldexp.f32.i32(float 1.000000e+00, i32 [[X]])
 ; CHECK-NEXT:    [[RES:%.*]] = fpext float [[LDEXPF]] to double
 ; CHECK-NEXT:    ret double [[RES]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/pow_fp_int16.ll b/llvm/test/Transforms/InstCombine/pow_fp_int16.ll
index 90e81c99d91c..8ba6b8a699ba 100644
--- a/llvm/test/Transforms/InstCombine/pow_fp_int16.ll
+++ b/llvm/test/Transforms/InstCombine/pow_fp_int16.ll
@@ -51,7 +51,7 @@ define double @pow_uitofp_double_const_base_fast(i15 %x) {
 
 define double @pow_sitofp_double_const_base_2_fast(i16 %x) {
 ; CHECK-LABEL: @pow_sitofp_double_const_base_2_fast(
-; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call afn float @ldexpf(float 1.000000e+00, i16 [[X:%.*]])
+; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call afn float @llvm.ldexp.f32.i16(float 1.000000e+00, i16 [[X:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = fpext float [[LDEXPF]] to double
 ; CHECK-NEXT:    ret double [[RES]]
 ;
@@ -78,7 +78,7 @@ define double @pow_sitofp_double_const_base_power_of_2_fast(i16 %x) {
 define double @pow_uitofp_const_base_2_fast(i15 %x) {
 ; CHECK-LABEL: @pow_uitofp_const_base_2_fast(
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i15 [[X:%.*]] to i16
-; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call afn float @ldexpf(float 1.000000e+00, i16 [[TMP1]])
+; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call afn float @llvm.ldexp.f32.i16(float 1.000000e+00, i16 [[TMP1]])
 ; CHECK-NEXT:    [[RES:%.*]] = fpext float [[LDEXPF]] to double
 ; CHECK-NEXT:    ret double [[RES]]
 ;
@@ -313,7 +313,7 @@ define double @pow_uitofp_const_base_no_fast(i16 %x) {
 
 define double @pow_sitofp_const_base_2_no_fast(i16 %x) {
 ; CHECK-LABEL: @pow_sitofp_const_base_2_no_fast(
-; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call float @ldexpf(float 1.000000e+00, i16 [[X:%.*]])
+; CHECK-NEXT:    [[LDEXPF:%.*]] = tail call float @llvm.ldexp.f32.i16(float 1.000000e+00, i16 [[X:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = fpext float [[LDEXPF]] to double
 ; CHECK-NEXT:    ret double [[RES]]
 ;
-- 
GitLab


From 3a32590f25482e283a79dea1f313a226a6dd392f Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Fri, 17 May 2024 14:11:28 +0100
Subject: [PATCH 1122/1206] [AArch64] Avoid using NEON FCVTXN in Streaming-SVE
 mode. (#91981)

---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  3 +-
 ...e-streaming-mode-fixed-length-fcopysign.ll | 60 ++++++++++++-------
 2 files changed, 39 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 6223c211b33b..e31a27e9428e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -19501,7 +19501,8 @@ static SDValue performBuildVectorCombine(SDNode *N,
   SDLoc DL(N);
   EVT VT = N->getValueType(0);
 
-  if (VT == MVT::v4f16 || VT == MVT::v4bf16) {
+  if (DAG.getSubtarget<AArch64Subtarget>().isNeonAvailable() &&
+      (VT == MVT::v4f16 || VT == MVT::v4bf16)) {
     SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1),
             Elt2 = N->getOperand(2), Elt3 = N->getOperand(3);
     if (Elt0->getOpcode() == ISD::FP_ROUND &&
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
index 0d6675def8b5..f017eead92cf 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
@@ -427,35 +427,49 @@ define void @test_copysign_v4f16_v4f32(ptr %ap, ptr %bp) {
 define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) {
 ; SVE-LABEL: test_copysign_v4f16_v4f64:
 ; SVE:       // %bb.0:
-; SVE-NEXT:    ldp q0, q1, [x1]
-; SVE-NEXT:    ptrue p0.s, vl2
-; SVE-NEXT:    fcvtxn v1.2s, v1.2d
-; SVE-NEXT:    fcvtxn v0.2s, v0.2d
-; SVE-NEXT:    splice z0.s, p0, z0.s, z1.s
-; SVE-NEXT:    ptrue p0.s
-; SVE-NEXT:    ldr d1, [x0]
-; SVE-NEXT:    and z1.h, z1.h, #0x7fff
-; SVE-NEXT:    fcvt z0.h, p0/m, z0.s
-; SVE-NEXT:    uzp1 z0.h, z0.h, z0.h
+; SVE-NEXT:    sub sp, sp, #16
+; SVE-NEXT:    .cfi_def_cfa_offset 16
+; SVE-NEXT:    ldp q1, q0, [x1]
+; SVE-NEXT:    ldr d4, [x0]
+; SVE-NEXT:    and z4.h, z4.h, #0x7fff
+; SVE-NEXT:    mov z2.d, z0.d[1]
+; SVE-NEXT:    mov z3.d, z1.d[1]
+; SVE-NEXT:    fcvt h0, d0
+; SVE-NEXT:    fcvt h1, d1
+; SVE-NEXT:    fcvt h2, d2
+; SVE-NEXT:    fcvt h3, d3
+; SVE-NEXT:    str h0, [sp, #12]
+; SVE-NEXT:    str h1, [sp, #8]
+; SVE-NEXT:    str h2, [sp, #14]
+; SVE-NEXT:    str h3, [sp, #10]
+; SVE-NEXT:    ldr d0, [sp, #8]
 ; SVE-NEXT:    and z0.h, z0.h, #0x8000
-; SVE-NEXT:    orr z0.d, z1.d, z0.d
+; SVE-NEXT:    orr z0.d, z4.d, z0.d
 ; SVE-NEXT:    str d0, [x0]
+; SVE-NEXT:    add sp, sp, #16
 ; SVE-NEXT:    ret
 ;
 ; SVE2-LABEL: test_copysign_v4f16_v4f64:
 ; SVE2:       // %bb.0:
-; SVE2-NEXT:    ldp q0, q1, [x1]
-; SVE2-NEXT:    ptrue p0.s, vl2
-; SVE2-NEXT:    ldr d2, [x0]
-; SVE2-NEXT:    fcvtxn v1.2s, v1.2d
-; SVE2-NEXT:    fcvtxn v0.2s, v0.2d
-; SVE2-NEXT:    splice z0.s, p0, z0.s, z1.s
-; SVE2-NEXT:    ptrue p0.s
-; SVE2-NEXT:    mov z1.h, #32767 // =0x7fff
-; SVE2-NEXT:    fcvt z0.h, p0/m, z0.s
-; SVE2-NEXT:    uzp1 z0.h, z0.h, z0.h
-; SVE2-NEXT:    bsl z2.d, z2.d, z0.d, z1.d
-; SVE2-NEXT:    str d2, [x0]
+; SVE2-NEXT:    sub sp, sp, #16
+; SVE2-NEXT:    .cfi_def_cfa_offset 16
+; SVE2-NEXT:    ldp q2, q1, [x1]
+; SVE2-NEXT:    mov z0.h, #32767 // =0x7fff
+; SVE2-NEXT:    ldr d5, [x0]
+; SVE2-NEXT:    mov z3.d, z1.d[1]
+; SVE2-NEXT:    mov z4.d, z2.d[1]
+; SVE2-NEXT:    fcvt h1, d1
+; SVE2-NEXT:    fcvt h2, d2
+; SVE2-NEXT:    fcvt h3, d3
+; SVE2-NEXT:    fcvt h4, d4
+; SVE2-NEXT:    str h1, [sp, #12]
+; SVE2-NEXT:    str h2, [sp, #8]
+; SVE2-NEXT:    str h3, [sp, #14]
+; SVE2-NEXT:    str h4, [sp, #10]
+; SVE2-NEXT:    ldr d1, [sp, #8]
+; SVE2-NEXT:    bsl z5.d, z5.d, z1.d, z0.d
+; SVE2-NEXT:    str d5, [x0]
+; SVE2-NEXT:    add sp, sp, #16
 ; SVE2-NEXT:    ret
   %a = load <4 x half>, ptr %ap
   %b = load <4 x double>, ptr %bp
-- 
GitLab


From 932ca85680db5e4579306f37e55746097fb8ec7f Mon Sep 17 00:00:00 2001
From: Romaric Jodin <89833130+rjodinchr@users.noreply.github.com>
Date: Fri, 17 May 2024 15:13:32 +0200
Subject: [PATCH 1123/1206] libclc: remove __attribute__((assume)) for clspv
 targets (#92126)

Instead add a proper attribute in clang, and add convert it to function
metadata to keep the information in the IR. The goal is to remove the
dependency on __attribute__((assume)) that should have not be there in
the first place.

Ref https://github.com/llvm/llvm-project/pull/84934
---
 clang/docs/ReleaseNotes.rst                        |  4 ++++
 clang/include/clang/Basic/Attr.td                  |  7 +++++++
 clang/include/clang/Basic/AttrDocs.td              | 14 ++++++++++++++
 clang/lib/CodeGen/CodeGenFunction.cpp              |  5 +++++
 clang/test/CodeGen/clspv_libclc_builtin.c          |  6 ++++++
 ...pragma-attribute-supported-attributes-list.test |  1 +
 libclc/generic/include/clc/clcfunc.h               |  3 +--
 7 files changed, 38 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/CodeGen/clspv_libclc_builtin.c

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 40e41c940b6a..9799f878d7d7 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -403,6 +403,10 @@ Attribute Changes in Clang
 - Clang now warns that the ``exclude_from_explicit_instantiation`` attribute
   is ignored when applied to a local class or a member thereof.
 
+- The ``clspv_libclc_builtin`` attribute has been added to allow clspv
+  (`OpenCL-C to Vulkan SPIR-V compiler <https://github.com/google/clspv>`_) to identify functions coming from libclc
+  (`OpenCL-C builtin library <https://libclc.llvm.org>`_).
+
 Improvements to Clang's diagnostics
 -----------------------------------
 - Clang now applies syntax highlighting to the code snippets it
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 52552ba48856..38ee8356583b 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4561,3 +4561,10 @@ def CodeAlign: StmtAttr {
     static constexpr int MaximumAlignment = 4096;
   }];
 }
+
+def ClspvLibclcBuiltin: InheritableAttr {
+  let Spellings = [Clang<"clspv_libclc_builtin">];
+  let Subjects = SubjectList<[Function]>;
+  let Documentation = [ClspvLibclcBuiltinDoc];
+  let SimpleHandler = 1;
+}
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index f351822ac74b..b48aaf65558a 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -8087,3 +8087,17 @@ requirement:
   }
   }];
 }
+
+def ClspvLibclcBuiltinDoc : Documentation {
+  let Category = DocCatFunction;
+  let Content = [{
+Attribute used by `clspv`_ (OpenCL-C to Vulkan SPIR-V compiler) to identify functions coming from `libclc`_ (OpenCL-C builtin library).
+
+.. code-block:: c
+
+  void __attribute__((clspv_libclc_builtin)) libclc_builtin() {}
+
+.. _`clspv`: https://github.com/google/clspv
+.. _`libclc`: https://libclc.llvm.org
+}];
+}
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 34dc0bd5c15b..04abdadd9537 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -985,6 +985,11 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
     EmitKernelMetadata(FD, Fn);
   }
 
+  if (FD && FD->hasAttr<ClspvLibclcBuiltinAttr>()) {
+    Fn->setMetadata("clspv_libclc_builtin",
+                    llvm::MDNode::get(getLLVMContext(), {}));
+  }
+
   // If we are checking function types, emit a function type signature as
   // prologue data.
   if (FD && SanOpts.has(SanitizerKind::Function)) {
diff --git a/clang/test/CodeGen/clspv_libclc_builtin.c b/clang/test/CodeGen/clspv_libclc_builtin.c
new file mode 100644
index 000000000000..ada2555c79e5
--- /dev/null
+++ b/clang/test/CodeGen/clspv_libclc_builtin.c
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -triple spir -emit-llvm %s -o - | FileCheck %s
+
+// CHECK: @foo()
+// CHECK-SAME: !clspv_libclc_builtin
+
+void __attribute__((clspv_libclc_builtin)) foo() {}
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index 318bfb2df2a7..fd0e6d71baa8 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -48,6 +48,7 @@
 // CHECK-NEXT: Capability (SubjectMatchRule_record, SubjectMatchRule_type_alias)
 // CHECK-NEXT: CarriesDependency (SubjectMatchRule_variable_is_parameter, SubjectMatchRule_objc_method, SubjectMatchRule_function)
 // CHECK-NEXT: Cleanup (SubjectMatchRule_variable_is_local)
+// CHECK-NEXT: ClspvLibclcBuiltin (SubjectMatchRule_function)
 // CHECK-NEXT: CmseNSEntry (SubjectMatchRule_function)
 // CHECK-NEXT: Cold (SubjectMatchRule_function)
 // CHECK-NEXT: Common (SubjectMatchRule_variable)
diff --git a/libclc/generic/include/clc/clcfunc.h b/libclc/generic/include/clc/clcfunc.h
index ad9eb23f2933..086d780b9708 100644
--- a/libclc/generic/include/clc/clcfunc.h
+++ b/libclc/generic/include/clc/clcfunc.h
@@ -7,8 +7,7 @@
 #if defined(CLC_SPIRV) || defined(CLC_SPIRV64)
 #define _CLC_DEF
 #elif defined(CLC_CLSPV) || defined(CLC_CLSPV64)
-#define _CLC_DEF                                                               \
-  __attribute__((noinline)) __attribute__((assume("clspv_libclc_builtin")))
+#define _CLC_DEF __attribute__((noinline)) __attribute__((clspv_libclc_builtin))
 #else
 #define _CLC_DEF __attribute__((always_inline))
 #endif
-- 
GitLab


From 9917f3ce641490ff95a11d22e37754a972c05420 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Fri, 17 May 2024 14:21:56 +0100
Subject: [PATCH 1124/1206] [Clang][AArch64] Require SVE or SSVE for scalable
 types. (#91356)

Scalable types are only available when:
* The function is compiled with +sve
* The function is compiled with +sme and the function is executed in
Streaming-SVE mode.
---
 .../clang/Basic/DiagnosticSemaKinds.td        |  2 +
 clang/lib/Sema/Sema.cpp                       | 12 ++-
 clang/lib/Sema/SemaDecl.cpp                   | 19 +++--
 .../acle_sme2_reinterpret_svcount_svbool.c    | 10 ++-
 .../aarch64-sve2-intrinsics/acle_sve2_revd.c  | 78 ++++++++++---------
 .../acle_sve2p1_bfadd.c                       | 18 +++--
 .../acle_sve2p1_bfmax.c                       | 18 +++--
 .../acle_sve2p1_bfmaxnm.c                     | 18 +++--
 .../acle_sve2p1_bfmin.c                       | 18 +++--
 .../acle_sve2p1_bfminnm.c                     | 18 +++--
 .../acle_sve2p1_bfmla.c                       | 18 +++--
 .../acle_sve2p1_bfmls.c                       | 18 +++--
 .../acle_sve2p1_bfmul.c                       | 18 +++--
 .../acle_sve2p1_bfsub.c                       | 18 +++--
 .../acle_sve2p1_create2_bool.c                | 20 ++---
 .../acle_sve2p1_get4_bool.c                   | 24 +++---
 .../acle_sve2p1_undef_bool.c                  | 10 ++-
 .../Sema/aarch64-sme2-sve2p1-diagnostics.c    |  2 +
 clang/test/Sema/aarch64-sme2p1-diagnostics.c  |  2 +-
 19 files changed, 216 insertions(+), 125 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 9372c862a36c..59f44bc26eaf 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -3209,6 +3209,8 @@ def warn_attribute_arm_zt0_builtin_no_zt0_state : Warning<
   InGroup<DiagGroup<"undefined-arm-zt0">>;
 def err_sve_vector_in_non_sve_target : Error<
   "SVE vector type %0 cannot be used in a target without sve">;
+def err_sve_vector_in_non_streaming_function : Error<
+  "SVE vector type %0 cannot be used in a non-streaming function">;
 def err_attribute_riscv_rvv_bits_unsupported : Error<
   "%0 is only supported when '-mrvv-vector-bits=<bits>' is specified with a "
   "value of \"zvl\" or a power 2 in the range [64,65536]">;
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 5d6ce233c775..c9bc46414c7d 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -2054,9 +2054,15 @@ void Sema::checkTypeSupport(QualType Ty, SourceLocation Loc, ValueDecl *D) {
     if (Ty->isSVESizelessBuiltinType() && FD && FD->hasBody()) {
       llvm::StringMap<bool> CallerFeatureMap;
       Context.getFunctionFeatureMap(CallerFeatureMap, FD);
-      if (!Builtin::evaluateRequiredTargetFeatures("sve", CallerFeatureMap) &&
-          !Builtin::evaluateRequiredTargetFeatures("sme", CallerFeatureMap))
-        Diag(D->getLocation(), diag::err_sve_vector_in_non_sve_target) << Ty;
+      if (!Builtin::evaluateRequiredTargetFeatures("sve", CallerFeatureMap)) {
+        if (!Builtin::evaluateRequiredTargetFeatures("sme", CallerFeatureMap))
+          Diag(D->getLocation(), diag::err_sve_vector_in_non_sve_target) << Ty;
+        else if (!IsArmStreamingFunction(FD,
+                                         /*IncludeLocallyStreaming=*/true)) {
+          Diag(D->getLocation(), diag::err_sve_vector_in_non_streaming_function)
+              << Ty;
+        }
+      }
     }
   };
 
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index bb0bee0d234c..6c545d695fc9 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -8905,11 +8905,20 @@ void Sema::CheckVariableDeclarationType(VarDecl *NewVD) {
     const FunctionDecl *FD = cast<FunctionDecl>(CurContext);
     llvm::StringMap<bool> CallerFeatureMap;
     Context.getFunctionFeatureMap(CallerFeatureMap, FD);
-    if (!Builtin::evaluateRequiredTargetFeatures(
-        "sve", CallerFeatureMap)) {
-      Diag(NewVD->getLocation(), diag::err_sve_vector_in_non_sve_target) << T;
-      NewVD->setInvalidDecl();
-      return;
+
+    if (!Builtin::evaluateRequiredTargetFeatures("sve", CallerFeatureMap)) {
+      if (!Builtin::evaluateRequiredTargetFeatures("sme", CallerFeatureMap)) {
+        Diag(NewVD->getLocation(), diag::err_sve_vector_in_non_sve_target) << T;
+        NewVD->setInvalidDecl();
+        return;
+      } else if (!IsArmStreamingFunction(FD,
+                                         /*IncludeLocallyStreaming=*/true)) {
+        Diag(NewVD->getLocation(),
+             diag::err_sve_vector_in_non_streaming_function)
+            << T;
+        NewVD->setInvalidDecl();
+        return;
+      }
     }
   }
 
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c
index b3d5f4a4c4a5..c442d2c0c475 100644
--- a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c
+++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c
@@ -9,6 +9,12 @@
 
 #include <arm_sme.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.§
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
@@ -26,7 +32,7 @@
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[CNT:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP0]]
 //
-svbool_t test_svreinterpret_svbool_svcnt(svcount_t cnt) __arm_streaming_compatible
+svbool_t test_svreinterpret_svbool_svcnt(svcount_t cnt) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svreinterpret,_b,,)(cnt);
 }
@@ -41,7 +47,7 @@ svbool_t test_svreinterpret_svbool_svcnt(svcount_t cnt) __arm_streaming_compatib
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
 //
-svcount_t test_svreinterpret_svcnt_svbool(svbool_t pg) __arm_streaming_compatible
+svcount_t test_svreinterpret_svcnt_svbool(svbool_t pg) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svreinterpret,_c,,)(pg);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c
index 128a7eb102da..b4432c4f4477 100644
--- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c
@@ -12,6 +12,12 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
@@ -29,7 +35,7 @@
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-svint8_t test_svrevd_s8_z(svbool_t pg, svint8_t op) {
+svint8_t test_svrevd_s8_z(svbool_t pg, svint8_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s8, _z, )(pg, op);
 }
 
@@ -45,7 +51,7 @@ svint8_t test_svrevd_s8_z(svbool_t pg, svint8_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-svint16_t test_svrevd_s16_z(svbool_t pg, svint16_t op) {
+svint16_t test_svrevd_s16_z(svbool_t pg, svint16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s16, _z, )(pg, op);
 }
 
@@ -61,7 +67,7 @@ svint16_t test_svrevd_s16_z(svbool_t pg, svint16_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-svint32_t test_svrevd_s32_z(svbool_t pg, svint32_t op) {
+svint32_t test_svrevd_s32_z(svbool_t pg, svint32_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s32, _z, )(pg, op);
 }
 
@@ -77,7 +83,7 @@ svint32_t test_svrevd_s32_z(svbool_t pg, svint32_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-svint64_t test_svrevd_s64_z(svbool_t pg, svint64_t op) {
+svint64_t test_svrevd_s64_z(svbool_t pg, svint64_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s64, _z, )(pg, op);
 }
 
@@ -91,7 +97,7 @@ svint64_t test_svrevd_s64_z(svbool_t pg, svint64_t op) {
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-svuint8_t test_svrevd_u8_z(svbool_t pg, svuint8_t op) {
+svuint8_t test_svrevd_u8_z(svbool_t pg, svuint8_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u8, _z, )(pg, op);
 }
 // CHECK-LABEL: @test_svrevd_u16_z(
@@ -106,7 +112,7 @@ svuint8_t test_svrevd_u8_z(svbool_t pg, svuint8_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-svuint16_t test_svrevd_u16_z(svbool_t pg, svuint16_t op) {
+svuint16_t test_svrevd_u16_z(svbool_t pg, svuint16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u16, _z, )(pg, op);
 }
 
@@ -122,7 +128,7 @@ svuint16_t test_svrevd_u16_z(svbool_t pg, svuint16_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-svuint32_t test_svrevd_u32_z(svbool_t pg, svuint32_t op) {
+svuint32_t test_svrevd_u32_z(svbool_t pg, svuint32_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u32, _z, )(pg, op);
 }
 
@@ -138,7 +144,7 @@ svuint32_t test_svrevd_u32_z(svbool_t pg, svuint32_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-svuint64_t test_svrevd_u64_z(svbool_t pg, svuint64_t op) {
+svuint64_t test_svrevd_u64_z(svbool_t pg, svuint64_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u64, _z, )(pg, op);
 }
 
@@ -152,7 +158,7 @@ svuint64_t test_svrevd_u64_z(svbool_t pg, svuint64_t op) {
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> [[INACTIVE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-svint8_t test_svrevd_s8_m(svint8_t inactive, svbool_t pg, svint8_t op) {
+svint8_t test_svrevd_s8_m(svint8_t inactive, svbool_t pg, svint8_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s8, _m, )(inactive, pg, op);
 }
 
@@ -168,7 +174,7 @@ svint8_t test_svrevd_s8_m(svint8_t inactive, svbool_t pg, svint8_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> [[INACTIVE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-svint16_t test_svrevd_s16_m(svint16_t inactive, svbool_t pg, svint16_t op) {
+svint16_t test_svrevd_s16_m(svint16_t inactive, svbool_t pg, svint16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s16, _m, )(inactive, pg, op);
 }
 
@@ -184,7 +190,7 @@ svint16_t test_svrevd_s16_m(svint16_t inactive, svbool_t pg, svint16_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> [[INACTIVE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-svint32_t test_svrevd_s32_m(svint32_t inactive, svbool_t pg, svint32_t op) {
+svint32_t test_svrevd_s32_m(svint32_t inactive, svbool_t pg, svint32_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s32, _m, )(inactive, pg, op);
 }
 
@@ -200,7 +206,7 @@ svint32_t test_svrevd_s32_m(svint32_t inactive, svbool_t pg, svint32_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> [[INACTIVE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-svint64_t test_svrevd_s64_m(svint64_t inactive, svbool_t pg, svint64_t op) {
+svint64_t test_svrevd_s64_m(svint64_t inactive, svbool_t pg, svint64_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s64, _m, )(inactive, pg, op);
 }
 
@@ -214,7 +220,7 @@ svint64_t test_svrevd_s64_m(svint64_t inactive, svbool_t pg, svint64_t op) {
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> [[INACTIVE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-svuint8_t test_svrevd_u8_m(svuint8_t inactive, svbool_t pg, svuint8_t op) {
+svuint8_t test_svrevd_u8_m(svuint8_t inactive, svbool_t pg, svuint8_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u8, _m, )(inactive, pg, op);
 }
 
@@ -230,7 +236,7 @@ svuint8_t test_svrevd_u8_m(svuint8_t inactive, svbool_t pg, svuint8_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> [[INACTIVE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-svuint16_t test_svrevd_u16_m(svuint16_t inactive, svbool_t pg, svuint16_t op) {
+svuint16_t test_svrevd_u16_m(svuint16_t inactive, svbool_t pg, svuint16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u16, _m, )(inactive, pg, op);
 }
 
@@ -246,7 +252,7 @@ svuint16_t test_svrevd_u16_m(svuint16_t inactive, svbool_t pg, svuint16_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> [[INACTIVE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-svuint32_t test_svrevd_u32_m(svuint32_t inactive, svbool_t pg, svuint32_t op) {
+svuint32_t test_svrevd_u32_m(svuint32_t inactive, svbool_t pg, svuint32_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u32, _m, )(inactive, pg, op);
 }
 
@@ -262,7 +268,7 @@ svuint32_t test_svrevd_u32_m(svuint32_t inactive, svbool_t pg, svuint32_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> [[INACTIVE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-svuint64_t test_svrevd_u64_m(svuint64_t inactive, svbool_t pg, svuint64_t op) {
+svuint64_t test_svrevd_u64_m(svuint64_t inactive, svbool_t pg, svuint64_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u64, _m, )(inactive, pg, op);
 }
 
@@ -276,7 +282,7 @@ svuint64_t test_svrevd_u64_m(svuint64_t inactive, svbool_t pg, svuint64_t op) {
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-svint8_t test_svrevd_s8_x(svbool_t pg, svint8_t op) {
+svint8_t test_svrevd_s8_x(svbool_t pg, svint8_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s8, _x, )(pg, op);
 }
 
@@ -292,7 +298,7 @@ svint8_t test_svrevd_s8_x(svbool_t pg, svint8_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-svint16_t test_svrevd_s16_x(svbool_t pg, svint16_t op) {
+svint16_t test_svrevd_s16_x(svbool_t pg, svint16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s16, _x, )(pg, op);
 }
 
@@ -308,7 +314,7 @@ svint16_t test_svrevd_s16_x(svbool_t pg, svint16_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-svint32_t test_svrevd_s32_x(svbool_t pg, svint32_t op) {
+svint32_t test_svrevd_s32_x(svbool_t pg, svint32_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s32, _x, )(pg, op);
 }
 
@@ -324,7 +330,7 @@ svint32_t test_svrevd_s32_x(svbool_t pg, svint32_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-svint64_t test_svrevd_s64_x(svbool_t pg, svint64_t op) {
+svint64_t test_svrevd_s64_x(svbool_t pg, svint64_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _s64, _x, )(pg, op);
 }
 
@@ -338,7 +344,7 @@ svint64_t test_svrevd_s64_x(svbool_t pg, svint64_t op) {
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-svuint8_t test_svrevd_u8_x(svbool_t pg, svuint8_t op) {
+svuint8_t test_svrevd_u8_x(svbool_t pg, svuint8_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u8, _x, )(pg, op);
 }
 
@@ -354,7 +360,7 @@ svuint8_t test_svrevd_u8_x(svbool_t pg, svuint8_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-svuint16_t test_svrevd_u16_x(svbool_t pg, svuint16_t op) {
+svuint16_t test_svrevd_u16_x(svbool_t pg, svuint16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u16, _x, )(pg, op);
 }
 
@@ -370,7 +376,7 @@ svuint16_t test_svrevd_u16_x(svbool_t pg, svuint16_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-svuint32_t test_svrevd_u32_x(svbool_t pg, svuint32_t op) {
+svuint32_t test_svrevd_u32_x(svbool_t pg, svuint32_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u32, _x, )(pg, op);
 }
 
@@ -386,7 +392,7 @@ svuint32_t test_svrevd_u32_x(svbool_t pg, svuint32_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-svuint64_t test_svrevd_u64_x(svbool_t pg, svuint64_t op) {
+svuint64_t test_svrevd_u64_x(svbool_t pg, svuint64_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _u64, _x, )(pg, op);
 }
 
@@ -403,7 +409,7 @@ svuint64_t test_svrevd_u64_x(svbool_t pg, svuint64_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.revd.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svrevd_bf16_z(svbool_t pg, svbfloat16_t op) {
+svbfloat16_t test_svrevd_bf16_z(svbool_t pg, svbfloat16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _bf16, _z, )(pg, op);
 }
 
@@ -419,7 +425,7 @@ svbfloat16_t test_svrevd_bf16_z(svbool_t pg, svbfloat16_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.revd.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
-svfloat16_t test_svrevd_f16_z(svbool_t pg, svfloat16_t op) {
+svfloat16_t test_svrevd_f16_z(svbool_t pg, svfloat16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _f16, _z, )(pg, op);
 }
 
@@ -435,7 +441,7 @@ svfloat16_t test_svrevd_f16_z(svbool_t pg, svfloat16_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.revd.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-svfloat32_t test_svrevd_f32_z(svbool_t pg, svfloat32_t op) {
+svfloat32_t test_svrevd_f32_z(svbool_t pg, svfloat32_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _f32, _z, )(pg, op);
 }
 
@@ -451,7 +457,7 @@ svfloat32_t test_svrevd_f32_z(svbool_t pg, svfloat32_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.revd.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-svfloat64_t test_svrevd_f64_z(svbool_t pg, svfloat64_t op) {
+svfloat64_t test_svrevd_f64_z(svbool_t pg, svfloat64_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _f64, _z, )(pg, op);
 }
 
@@ -467,7 +473,7 @@ svfloat64_t test_svrevd_f64_z(svbool_t pg, svfloat64_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.revd.nxv8bf16(<vscale x 8 x bfloat> [[INACTIVE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svrevd_bf16_m(svbfloat16_t inactive, svbool_t pg, svbfloat16_t op) {
+svbfloat16_t test_svrevd_bf16_m(svbfloat16_t inactive, svbool_t pg, svbfloat16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _bf16, _m, )(inactive, pg, op);
 }
 
@@ -483,7 +489,7 @@ svbfloat16_t test_svrevd_bf16_m(svbfloat16_t inactive, svbool_t pg, svbfloat16_t
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.revd.nxv8f16(<vscale x 8 x half> [[INACTIVE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
-svfloat16_t test_svrevd_f16_m(svfloat16_t inactive, svbool_t pg, svfloat16_t op) {
+svfloat16_t test_svrevd_f16_m(svfloat16_t inactive, svbool_t pg, svfloat16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _f16, _m, )(inactive, pg, op);
 }
 
@@ -499,7 +505,7 @@ svfloat16_t test_svrevd_f16_m(svfloat16_t inactive, svbool_t pg, svfloat16_t op)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.revd.nxv4f32(<vscale x 4 x float> [[INACTIVE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-svfloat32_t test_svrevd_f32_m(svfloat32_t inactive, svbool_t pg, svfloat32_t op) {
+svfloat32_t test_svrevd_f32_m(svfloat32_t inactive, svbool_t pg, svfloat32_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _f32, _m, )(inactive, pg, op);
 }
 
@@ -515,7 +521,7 @@ svfloat32_t test_svrevd_f32_m(svfloat32_t inactive, svbool_t pg, svfloat32_t op)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.revd.nxv2f64(<vscale x 2 x double> [[INACTIVE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-svfloat64_t test_svrevd_f64_m(svfloat64_t inactive, svbool_t pg, svfloat64_t op) {
+svfloat64_t test_svrevd_f64_m(svfloat64_t inactive, svbool_t pg, svfloat64_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _f64, _m, )(inactive, pg, op);
 }
 
@@ -531,7 +537,7 @@ svfloat64_t test_svrevd_f64_m(svfloat64_t inactive, svbool_t pg, svfloat64_t op)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.revd.nxv8bf16(<vscale x 8 x bfloat> undef, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svrevd_bf16_x(svbool_t pg, svbfloat16_t op) {
+svbfloat16_t test_svrevd_bf16_x(svbool_t pg, svbfloat16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _bf16, _x, )(pg, op);
 }
 
@@ -547,7 +553,7 @@ svbfloat16_t test_svrevd_bf16_x(svbool_t pg, svbfloat16_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.revd.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
-svfloat16_t test_svrevd_f16_x(svbool_t pg, svfloat16_t op) {
+svfloat16_t test_svrevd_f16_x(svbool_t pg, svfloat16_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _f16, _x, )(pg, op);
 }
 
@@ -563,7 +569,7 @@ svfloat16_t test_svrevd_f16_x(svbool_t pg, svfloat16_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.revd.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-svfloat32_t test_svrevd_f32_x(svbool_t pg, svfloat32_t op) {
+svfloat32_t test_svrevd_f32_x(svbool_t pg, svfloat32_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _f32, _x, )(pg, op);
 }
 
@@ -579,6 +585,6 @@ svfloat32_t test_svrevd_f32_x(svbool_t pg, svfloat32_t op) {
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.revd.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-svfloat64_t test_svrevd_f64_x(svbool_t pg, svfloat64_t op) {
+svfloat64_t test_svrevd_f64_x(svbool_t pg, svfloat64_t op) MODE_ATTR {
   return SVE_ACLE_FUNC(svrevd, _f64, _x, )(pg, op);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfadd.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfadd.c
index 9d01ba773694..d5c24d7c97bd 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfadd.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfadd.c
@@ -8,6 +8,12 @@
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
@@ -27,7 +33,7 @@
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fadd.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svadd_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svadd_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svadd, _bf16, _m)(pg, op1, op2);
 }
@@ -46,7 +52,7 @@ svbfloat16_t test_svadd_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fadd.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svadd_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svadd_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svadd, _bf16, _z)(pg, op1, op2);
 }
@@ -63,7 +69,7 @@ svbfloat16_t test_svadd_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fadd.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svadd_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svadd_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svadd, _bf16, _x)(pg, op1, op2);
 }
@@ -84,7 +90,7 @@ svbfloat16_t test_svadd_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fadd.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svadd_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svadd_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svadd, _n_bf16, _m)(pg, op1, op2);
 }
@@ -107,7 +113,7 @@ svbfloat16_t test_svadd_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fadd.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svadd_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svadd_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svadd, _n_bf16, _z)(pg, op1, op2);
 }
@@ -128,7 +134,7 @@ svbfloat16_t test_svadd_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fadd.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svadd_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svadd_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svadd, _n_bf16, _x)(pg, op1, op2);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmax.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmax.c
index ce28a0b0653d..a160e52d662e 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmax.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmax.c
@@ -8,6 +8,12 @@
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
@@ -27,7 +33,7 @@
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmax.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmax_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmax_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmax, _bf16, _m)(pg, op1, op2);
 }
@@ -46,7 +52,7 @@ svbfloat16_t test_svmax_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmax.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmax_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmax_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmax, _bf16, _z)(pg, op1, op2);
 }
@@ -63,7 +69,7 @@ svbfloat16_t test_svmax_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmax.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmax_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmax_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmax, _bf16, _x)(pg, op1, op2);
 }
@@ -85,7 +91,7 @@ svbfloat16_t test_svmax_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmax.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmax_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmax_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmax, _n_bf16, _m)(pg, op1, op2);
 }
@@ -108,7 +114,7 @@ svbfloat16_t test_svmax_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmax.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmax_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmax_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svmax, _n_bf16, _z)(pg, op1, op2);
 }
@@ -129,7 +135,7 @@ svbfloat16_t test_svmax_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmax.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmax_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmax_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svmax, _n_bf16, _x)(pg, op1, op2);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmaxnm.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmaxnm.c
index d8fec8c5145e..392677c2daa1 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmaxnm.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmaxnm.c
@@ -8,6 +8,12 @@
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
@@ -27,7 +33,7 @@
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmaxnm.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmaxnm_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmaxnm_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmaxnm, _bf16, _m)(pg, op1, op2);
 }
@@ -46,7 +52,7 @@ svbfloat16_t test_svmaxnm_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmaxnm.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmaxnm_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmaxnm_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmaxnm, _bf16, _z)(pg, op1, op2);
 }
@@ -63,7 +69,7 @@ svbfloat16_t test_svmaxnm_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmaxnm.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmaxnm_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmaxnm_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmaxnm, _bf16, _x)(pg, op1, op2);
 }
@@ -85,7 +91,7 @@ svbfloat16_t test_svmaxnm_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmaxnm.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmaxnm_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmaxnm_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmaxnm, _n_bf16, _m)(pg, op1, op2);
 }
@@ -108,7 +114,7 @@ svbfloat16_t test_svmaxnm_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmaxnm.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmaxnm_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmaxnm_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svmaxnm, _n_bf16, _z)(pg, op1, op2);
 }
@@ -129,7 +135,7 @@ svbfloat16_t test_svmaxnm_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmaxnm.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmaxnm_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmaxnm_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svmaxnm, _n_bf16, _x)(pg, op1, op2);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmin.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmin.c
index 5efaa37a1464..5b0e1505c8e9 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmin.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmin.c
@@ -8,6 +8,12 @@
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
@@ -27,7 +33,7 @@
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmin.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmin_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmin_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmin, _bf16, _m)(pg, op1, op2);
 }
@@ -46,7 +52,7 @@ svbfloat16_t test_svmin_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmin.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmin_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmin_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmin, _bf16, _z)(pg, op1, op2);
 }
@@ -63,7 +69,7 @@ svbfloat16_t test_svmin_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmin.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmin_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmin_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmin, _bf16, _x)(pg, op1, op2);
 }
@@ -85,7 +91,7 @@ svbfloat16_t test_svmin_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmin.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmin_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmin_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmin, _n_bf16, _m)(pg, op1, op2);
 }
@@ -108,7 +114,7 @@ svbfloat16_t test_svmin_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmin.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmin_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmin_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svmin, _n_bf16, _z)(pg, op1, op2);
 }
@@ -129,7 +135,7 @@ svbfloat16_t test_svmin_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmin.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmin_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmin_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svmin, _n_bf16, _x)(pg, op1, op2);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfminnm.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfminnm.c
index 98fd12b3a839..7a7de42e1419 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfminnm.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfminnm.c
@@ -8,6 +8,12 @@
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
@@ -27,7 +33,7 @@
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fminnm.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svminnm_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svminnm_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svminnm, _bf16, _m)(pg, op1, op2);
 }
@@ -46,7 +52,7 @@ svbfloat16_t test_svminnm_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fminnm.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svminnm_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svminnm_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svminnm, _bf16, _z)(pg, op1, op2);
 }
@@ -63,7 +69,7 @@ svbfloat16_t test_svminnm_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fminnm.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svminnm_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svminnm_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svminnm, _bf16, _x)(pg, op1, op2);
 }
@@ -85,7 +91,7 @@ svbfloat16_t test_svminnm_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fminnm.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svminnm_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svminnm_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svminnm, _n_bf16, _m)(pg, op1, op2);
 }
@@ -108,7 +114,7 @@ svbfloat16_t test_svminnm_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fminnm.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svminnm_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svminnm_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svminnm, _n_bf16, _z)(pg, op1, op2);
 }
@@ -129,7 +135,7 @@ svbfloat16_t test_svminnm_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fminnm.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svminnm_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svminnm_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svminnm, _n_bf16, _x)(pg, op1, op2);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmla.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmla.c
index 0a70466a540d..3663d95b6279 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmla.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmla.c
@@ -8,6 +8,12 @@
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
@@ -27,7 +33,7 @@
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmla.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[OP3:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmla_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmla_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmla, _bf16, _m)(pg, op1, op2, op3);
 }
@@ -46,7 +52,7 @@ svbfloat16_t test_svmla_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2,
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmla.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[OP3:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmla_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmla_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmla, _bf16, _z)(pg, op1, op2, op3);
 }
@@ -63,7 +69,7 @@ svbfloat16_t test_svmla_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2,
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmla.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[OP3:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmla_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmla_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmla, _bf16, _x)(pg, op1, op2, op3);
 }
@@ -84,7 +90,7 @@ svbfloat16_t test_svmla_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2,
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmla.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmla_n_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmla_n_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmla, _n_bf16, _m)(pg, op1, op2, op3);
 }
@@ -107,7 +113,7 @@ svbfloat16_t test_svmla_n_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmla.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmla_n_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmla_n_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmla, _n_bf16, _z)(pg, op1, op2, op3);
 }
@@ -128,7 +134,7 @@ svbfloat16_t test_svmla_n_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmla.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmla_n_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmla_n_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmla, _n_bf16, _x)(pg, op1, op2, op3);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmls.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmls.c
index ed71d4e490bb..73619e10dfed 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmls.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmls.c
@@ -8,6 +8,12 @@
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
@@ -27,7 +33,7 @@
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmls.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[OP3:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmls_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmls_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmls, _bf16, _m)(pg, op1, op2, op3);
 }
@@ -46,7 +52,7 @@ svbfloat16_t test_svmls_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2,
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmls.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[OP3:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmls_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmls_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmls, _bf16, _z)(pg, op1, op2, op3);
 }
@@ -63,7 +69,7 @@ svbfloat16_t test_svmls_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2,
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmls.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[OP3:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmls_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmls_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, svbfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmls, _bf16, _x)(pg, op1, op2, op3);
 }
@@ -84,7 +90,7 @@ svbfloat16_t test_svmls_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2,
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmls.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmls_n_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmls_n_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmls, _n_bf16, _m)(pg, op1, op2, op3);
 }
@@ -107,7 +113,7 @@ svbfloat16_t test_svmls_n_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmls.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmls_n_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmls_n_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmls, _n_bf16, _z)(pg, op1, op2, op3);
 }
@@ -128,7 +134,7 @@ svbfloat16_t test_svmls_n_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmls.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmls_n_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) __arm_streaming_compatible
+svbfloat16_t test_svmls_n_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2, bfloat16_t op3) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmls, _n_bf16, _x)(pg, op1, op2, op3);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmul.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmul.c
index f0a3664426de..f5e96996c360 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmul.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmul.c
@@ -8,6 +8,12 @@
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
@@ -27,7 +33,7 @@
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmul.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmul_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmul_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmul, _bf16, _m)(pg, op1, op2);
 }
@@ -46,7 +52,7 @@ svbfloat16_t test_svmul_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmul.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmul_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmul_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmul, _bf16, _z)(pg, op1, op2);
 }
@@ -63,7 +69,7 @@ svbfloat16_t test_svmul_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmul.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmul_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmul_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmul, _bf16, _x)(pg, op1, op2);
 }
@@ -85,7 +91,7 @@ svbfloat16_t test_svmul_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmul.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmul_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmul_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svmul, _n_bf16, _m)(pg, op1, op2);
 }
@@ -108,7 +114,7 @@ svbfloat16_t test_svmul_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmul.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svmul_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmul_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svmul, _n_bf16, _z)(pg, op1, op2);
 }
@@ -129,7 +135,7 @@ svbfloat16_t test_svmul_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fmul.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svmul_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svmul_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svmul, _n_bf16, _x)(pg, op1, op2);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfsub.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfsub.c
index 6f08ea84ab82..464eeac67e51 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfsub.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfsub.c
@@ -8,6 +8,12 @@
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -target-feature +b16b16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
@@ -27,7 +33,7 @@
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fsub.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svsub_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svsub_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svsub, _bf16, _m)(pg, op1, op2);
 }
@@ -46,7 +52,7 @@ svbfloat16_t test_svsub_bf16_m(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fsub.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svsub_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svsub_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svsub, _bf16, _z)(pg, op1, op2);
 }
@@ -63,7 +69,7 @@ svbfloat16_t test_svsub_bf16_z(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fsub.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[OP2:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svsub_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svsub_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svsub, _bf16, _x)(pg, op1, op2);
 }
@@ -85,7 +91,7 @@ svbfloat16_t test_svsub_bf16_x(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fsub.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svsub_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svsub_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svsub, _n_bf16, _m)(pg, op1, op2);
 }
@@ -108,7 +114,7 @@ svbfloat16_t test_svsub_bf16_n_m(svbool_t pg, svbfloat16_t op1, bfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fsub.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-svbfloat16_t test_svsub_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svsub_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svsub, _n_bf16, _z)(pg, op1, op2);
 }
@@ -129,7 +135,7 @@ svbfloat16_t test_svsub_bf16_n_z(svbool_t pg, svbfloat16_t op1, bfloat16_t op2)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.fsub.u.nxv8bf16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[OP1:%.*]], <vscale x 8 x bfloat> [[DOTSPLAT]])
 // CPP-CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-svbfloat16_t test_svsub_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) __arm_streaming_compatible
+svbfloat16_t test_svsub_bf16_n_x(svbool_t pg, svbfloat16_t op1, bfloat16_t op2) MODE_ATTR
 {
    return SVE_ACLE_FUNC(svsub, _n_bf16, _x)(pg, op1, op2);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create2_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create2_bool.c
index d441c934bad0..3728053b26e1 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create2_bool.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create2_bool.c
@@ -5,19 +5,25 @@
 // RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
 // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve2p1 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \
 // RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -DTEST_SME2 -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \
 // RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
 // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve2p1 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\
 // RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -DTEST_SME -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \
 // RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
-// RUN: %clang_cc1 -DTEST_SME2 -triple aarch64 -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 
 // REQUIRES: aarch64-registered-target
 
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
@@ -25,12 +31,6 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-#ifndef TEST_SME2
-#define ATTR
-#else
-#define ATTR __arm_streaming
-#endif
-
 // CHECK-LABEL: @test_svcreate2_b(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 32 x i1> @llvm.vector.insert.nxv32i1.nxv16i1(<vscale x 32 x i1> poison, <vscale x 16 x i1> [[X0:%.*]], i64 0)
@@ -43,7 +43,7 @@
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 32 x i1> @llvm.vector.insert.nxv32i1.nxv16i1(<vscale x 32 x i1> [[TMP0]], <vscale x 16 x i1> [[X1:%.*]], i64 16)
 // CPP-CHECK-NEXT:    ret <vscale x 32 x i1> [[TMP1]]
 //
-svboolx2_t test_svcreate2_b(svbool_t x0, svbool_t x1) ATTR
+svboolx2_t test_svcreate2_b(svbool_t x0, svbool_t x1) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svcreate2,_b,,)(x0, x1);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get4_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get4_bool.c
index e5016d6cb3dc..0b2a382d71bd 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get4_bool.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get4_bool.c
@@ -5,19 +5,25 @@
 // RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
 // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve2p1 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \
 // RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -DTEST_SME2 -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \
 // RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
 // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve2p1 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\
 // RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
-// RUN: %clang_cc1 -DTEST_SME2 -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \
 // RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
-// RUN: %clang_cc1 -DTEST_SME2 -triple aarch64 -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 
 // REQUIRES: aarch64-registered-target
 
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 #ifdef SVE_OVERLOADED_FORMS
 // A simple used,unused... macro, long enough to represent any SVE builtin.
 #define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
@@ -25,12 +31,6 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-#ifndef TEST_SME
-#define ATTR
-#else
-#define ATTR __arm_streaming
-#endif
-
 // NOTE: For these tests clang converts the struct parameter into
 // several parameters, one for each member of the original struct.
 // CHECK-LABEL: @test_svget4_b_0(
@@ -43,7 +43,7 @@
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[TUPLE:%.*]], i64 0)
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP0]]
 //
-svbool_t test_svget4_b_0(svboolx4_t tuple) ATTR
+svbool_t test_svget4_b_0(svboolx4_t tuple) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svget4,_b,,)(tuple, 0);
 }
@@ -60,7 +60,7 @@ svbool_t test_svget4_b_0(svboolx4_t tuple) ATTR
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[TUPLE:%.*]], i64 16)
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP0]]
 //
-svbool_t test_svget4_b_1(svboolx4_t tuple) ATTR
+svbool_t test_svget4_b_1(svboolx4_t tuple) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svget4,_b,,)(tuple, 1);
 }
@@ -77,7 +77,7 @@ svbool_t test_svget4_b_1(svboolx4_t tuple) ATTR
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.vector.extract.nxv16i1.nxv64i1(<vscale x 64 x i1> [[TUPLE:%.*]], i64 48)
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP0]]
 //
-svbool_t test_svget4_b_3(svboolx4_t tuple) ATTR
+svbool_t test_svget4_b_3(svboolx4_t tuple) MODE_ATTR
 {
   return SVE_ACLE_FUNC(svget4,_b,,)(tuple, 3);
 }
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_undef_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_undef_bool.c
index 5197e41c1ffd..16ba689964a3 100644
--- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_undef_bool.c
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_undef_bool.c
@@ -9,6 +9,12 @@
 
 #include <arm_sve.h>
 
+#if defined __ARM_FEATURE_SME
+#define MODE_ATTR __arm_streaming
+#else
+#define MODE_ATTR
+#endif
+
 // CHECK-LABEL: define dso_local <vscale x 32 x i1> @test_svundef2_b(
 // CPP-CHECK-LABEL: define dso_local <vscale x 32 x i1> @_Z15test_svundef2_bv(
 //
@@ -18,7 +24,7 @@
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    ret <vscale x 32 x i1> undef
 //
-svboolx2_t test_svundef2_b() {
+svboolx2_t test_svundef2_b(void) MODE_ATTR {
     return svundef2_b();
 }
 
@@ -31,6 +37,6 @@ svboolx2_t test_svundef2_b() {
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    ret <vscale x 64 x i1> undef
 //
-svboolx4_t test_svundef4_b() {
+svboolx4_t test_svundef4_b(void) MODE_ATTR {
     return svundef4_b();
 }
diff --git a/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c b/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c
index 4debc14190aa..2012221b4804 100644
--- a/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c
+++ b/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c
@@ -6,6 +6,8 @@
 
 //svldnt1:
 
+// expected-error@+3 {{SVE vector type 'svcount_t' (aka '__SVCount_t') cannot be used in a non-streaming function}}
+// expected-error@+2 {{SVE vector type 'svuint8x2_t' (aka '__clang_svuint8x2_t') cannot be used in a non-streaming function}}
 __attribute__((target("+sme2")))
 svuint8x2_t sme2_or_sve2p1_intrinsic_test_sme2_invalid(svcount_t png, const uint8_t *rn) {
   // expected-warning@+1 {{builtin call has undefined behaviour when called from a non-streaming function}}
diff --git a/clang/test/Sema/aarch64-sme2p1-diagnostics.c b/clang/test/Sema/aarch64-sme2p1-diagnostics.c
index a0adb0403858..c3c8b4ede305 100644
--- a/clang/test/Sema/aarch64-sme2p1-diagnostics.c
+++ b/clang/test/Sema/aarch64-sme2p1-diagnostics.c
@@ -3,7 +3,7 @@
 // REQUIRES: aarch64-registered-target
 #include "arm_sme.h"
 
-svuint8x2_t  test_sme2p1(svuint8x2_t  x) {
+svuint8x2_t  test_sme2p1(svuint8x2_t  x) __arm_streaming {
   // expected-no-diagnostics
   return x;
 }
-- 
GitLab


From f39af73f3edcc949bf9dc3535effc59afbcdda22 Mon Sep 17 00:00:00 2001
From: Youngsuk Kim <youngsuk.kim@hpe.com>
Date: Fri, 17 May 2024 09:29:20 -0400
Subject: [PATCH 1125/1206] [clang][Sema] Warn consecutive builtin comparisons
 in an expression (#92200)

Add warning under `-Wparentheses` for consistency with `gcc 14.1`.

Closes #20456

---------

Co-authored-by: Aaron Ballman <aaron@aaronballman.com>
---
 clang/docs/ReleaseNotes.rst                       |  3 +++
 clang/include/clang/Basic/DiagnosticSemaKinds.td  |  4 ++++
 clang/lib/Sema/SemaExpr.cpp                       |  5 +++++
 clang/test/Sema/bool-compare.c                    |  4 ++--
 clang/test/Sema/parentheses.cpp                   | 11 +++++++++++
 clang/test/SemaCXX/bool-compare.cpp               |  4 ++--
 clang/test/SemaCXX/cxx2a-adl-only-template-id.cpp |  2 +-
 clang/test/SemaTemplate/typo-dependent-name.cpp   |  4 ++--
 clang/test/SemaTemplate/typo-template-name.cpp    |  2 +-
 9 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 9799f878d7d7..85327576548c 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -498,6 +498,9 @@ Improvements to Clang's diagnostics
        }
      };
 
+- Clang emits a ``-Wparentheses`` warning for expressions with consecutive comparisons like ``x < y < z``.
+  Fixes #GH20456.
+
 Improvements to Clang's time-trace
 ----------------------------------
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 59f44bc26eaf..09b1874f9fdd 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -6919,6 +6919,10 @@ def warn_precedence_bitwise_conditional : Warning<
 def note_precedence_conditional_first : Note<
   "place parentheses around the '?:' expression to evaluate it first">;
 
+def warn_consecutive_comparison : Warning<
+  "comparisons like 'X<=Y<=Z' don't have their mathematical meaning">,
+  InGroup<Parentheses>;
+
 def warn_enum_constant_in_bool_context : Warning<
   "converting the enum constant to a boolean">,
   InGroup<IntInBoolContext>, DefaultIgnore;
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index e0aae6333e1a..5ecfdee21f09 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -14867,6 +14867,11 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
   case BO_GT:
     ConvertHalfVec = true;
     ResultTy = CheckCompareOperands(LHS, RHS, OpLoc, Opc);
+
+    if (const auto *BI = dyn_cast<BinaryOperator>(LHSExpr);
+        BI && BI->isComparisonOp())
+      Diag(OpLoc, diag::warn_consecutive_comparison);
+
     break;
   case BO_EQ:
   case BO_NE:
diff --git a/clang/test/Sema/bool-compare.c b/clang/test/Sema/bool-compare.c
index 923ff4208e8b..2308dc4bf2bb 100644
--- a/clang/test/Sema/bool-compare.c
+++ b/clang/test/Sema/bool-compare.c
@@ -85,7 +85,7 @@ void f(int x, int y, int z) {
   if ((a<y) != -1) {}// expected-warning {{comparison of constant -1 with boolean expression is always true}}
 
   if ((a<y) == z) {} // no warning
-  if (a>y<z)      {} // no warning
+  if (a>y<z)      {} // expected-warning {{comparisons like 'X<=Y<=Z' don't have their mathematical meaning}}
   if ((a<y) > z)  {} // no warning
   if((a<y)>(z<y)) {} // no warning
   if((a<y)==(z<y)){} // no warning
@@ -145,7 +145,7 @@ void f(int x, int y, int z) {
   if (-1 !=(a<y)) {} // expected-warning {{comparison of constant -1 with boolean expression is always true}}
 
   if (z ==(a<y))  {}    // no warning
-  if (z<a>y)      {}        // no warning
+  if (z<a>y)      {} // expected-warning {{comparisons like 'X<=Y<=Z' don't have their mathematical meaning}}
   if (z > (a<y))  {}    // no warning
   if((z<y)>(a<y)) {}   // no warning
   if((z<y)==(a<y)){}  // no warning
diff --git a/clang/test/Sema/parentheses.cpp b/clang/test/Sema/parentheses.cpp
index 324d9b5f1e41..e991cee16b0e 100644
--- a/clang/test/Sema/parentheses.cpp
+++ b/clang/test/Sema/parentheses.cpp
@@ -215,3 +215,14 @@ namespace PR20735 {
     // fix-it:"{{.*}}":{[[@LINE-9]]:20-[[@LINE-9]]:20}:")"
   }
 }
+
+void consecutive_builtin_compare(int x, int y, int z) {
+  (void)(x < y < z);  // expected-warning {{comparisons like 'X<=Y<=Z' don't have their mathematical meaning}}
+  (void)(x < y > z);  // expected-warning {{comparisons like 'X<=Y<=Z' don't have their mathematical meaning}}
+  (void)(x < y <= z); // expected-warning {{comparisons like 'X<=Y<=Z' don't have their mathematical meaning}}
+  (void)(x <= y > z); // expected-warning {{comparisons like 'X<=Y<=Z' don't have their mathematical meaning}}
+  (void)((x < y) < z);  // no-warning
+  (void)((x < y) >= z); // no-warning
+
+  // TODO: Also issue warning for consecutive comparisons that use overloaded comparison operators?
+}
diff --git a/clang/test/SemaCXX/bool-compare.cpp b/clang/test/SemaCXX/bool-compare.cpp
index fe47633bb23b..4e33e076a7de 100644
--- a/clang/test/SemaCXX/bool-compare.cpp
+++ b/clang/test/SemaCXX/bool-compare.cpp
@@ -98,7 +98,7 @@ void f(int x, int y, int z) {
   if ((a<y) != -1) {}// expected-warning {{comparison of constant -1 with expression of type 'bool' is always true}}
 
   if ((a<y) == z) {} // no warning
-  if (a>y<z)      {} // no warning
+  if (a>y<z)      {} // expected-warning {{comparisons like 'X<=Y<=Z' don't have their mathematical meaning}}
   if ((a<y) > z)  {} // no warning
   if((a<y)>(z<y)) {} // no warning
   if((a<y)==(z<y)){} // no warning
@@ -159,7 +159,7 @@ void f(int x, int y, int z) {
   if (-1 !=(a<y)) {} // expected-warning {{comparison of constant -1 with expression of type 'bool' is always true}}
 
   if (z ==(a<y))  {} // no warning
-  if (z<a>y)      {} // no warning
+  if (z<a>y)      {} // expected-warning {{comparisons like 'X<=Y<=Z' don't have their mathematical meaning}}
   if (z > (a<y))  {} // no warning
   if((z<y)>(a<y)) {} // no warning
   if((z<y)==(a<y)){} // no warning
diff --git a/clang/test/SemaCXX/cxx2a-adl-only-template-id.cpp b/clang/test/SemaCXX/cxx2a-adl-only-template-id.cpp
index 28ecbade1b1a..b8397f41febc 100644
--- a/clang/test/SemaCXX/cxx2a-adl-only-template-id.cpp
+++ b/clang/test/SemaCXX/cxx2a-adl-only-template-id.cpp
@@ -27,7 +27,7 @@ int e = h<0>(q); // ok, found by unqualified lookup
 void fn() {
   f<0>(q);
   int f;
-  f<0>(q); // expected-error {{invalid operands to binary expression}}
+  f<0>(q); // expected-error {{invalid operands to binary expression}} // expected-warning {{comparisons like 'X<=Y<=Z' don't have their mathematical meaning}}
 }
 
 void disambig() {
diff --git a/clang/test/SemaTemplate/typo-dependent-name.cpp b/clang/test/SemaTemplate/typo-dependent-name.cpp
index 88b2fc373b1f..fb61b03e5010 100644
--- a/clang/test/SemaTemplate/typo-dependent-name.cpp
+++ b/clang/test/SemaTemplate/typo-dependent-name.cpp
@@ -20,11 +20,11 @@ struct X : Base<T> {
   bool f(T other) {
     // A pair of comparisons; 'inner' is a dependent name so can't be assumed
     // to be a template.
-    return this->inner < other > ::z;
+    return this->inner < other > ::z; // expected-warning {{comparisons like 'X<=Y<=Z' don't have their mathematical meaning}}
   }
 };
 
-void use_x(X<int> x) { x.f(0); }
+void use_x(X<int> x) { x.f(0); } // expected-note {{requested here}}
 
 template<typename T>
 struct Y {
diff --git a/clang/test/SemaTemplate/typo-template-name.cpp b/clang/test/SemaTemplate/typo-template-name.cpp
index fe5201a8e26c..741aa708e81f 100644
--- a/clang/test/SemaTemplate/typo-template-name.cpp
+++ b/clang/test/SemaTemplate/typo-template-name.cpp
@@ -36,7 +36,7 @@ namespace InExpr {
 
     // These are valid expressions.
     foo<foo; // expected-warning {{self-comparison}}
-    foo<int()>(0);
+    foo<int()>(0); // expected-warning {{comparisons like 'X<=Y<=Z' don't have their mathematical meaning}}
     foo<int(), true>(false);
     foo<Base{}.n;
   }
-- 
GitLab


From b1161b2a40d89d4146f6b81fda073a37479edf0f Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Fri, 17 May 2024 14:31:16 +0100
Subject: [PATCH 1126/1206] [Flang][OpenMP] Fix semantics check for nested
 DISTRIBUTE (#91592)

Composite OpenMP constructs where DISTRIBUTE is the first leaf
construct, as well as standalone DISTRIBUTE constructs, are allowed
inside of TEAMS regions.

Before this patch, nesting a DISTRIBUTE construct inside of a combined
TARGET TEAMS construct was disallowed, which it shouldn't be. Now both
TEAMS and TARGET TEAMS constructs can be immediate parents of DISTRIBUTE
constructs.
---
 flang/lib/Semantics/check-omp-structure.cpp       | 3 ++-
 flang/test/Semantics/OpenMP/nested-distribute.f90 | 7 +++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 2493eb3ed367..e9637b7bb591 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -280,7 +280,8 @@ void OmpStructureChecker::HasInvalidDistributeNesting(
       violation = true;
     } else {
       // `distribute` region has to be strictly nested inside `teams`
-      if (!llvm::omp::topTeamsSet.test(GetContextParent().directive)) {
+      if (!OmpDirectiveSet{llvm::omp::OMPD_teams, llvm::omp::OMPD_target_teams}
+               .test(GetContextParent().directive)) {
         violation = true;
       }
     }
diff --git a/flang/test/Semantics/OpenMP/nested-distribute.f90 b/flang/test/Semantics/OpenMP/nested-distribute.f90
index 510379039289..ba8c3bf04b33 100644
--- a/flang/test/Semantics/OpenMP/nested-distribute.f90
+++ b/flang/test/Semantics/OpenMP/nested-distribute.f90
@@ -74,6 +74,13 @@ program main
    !$omp end distribute
   !$omp end teams
 
+  !$omp target teams
+    !$omp distribute
+    do i = 1, 10
+    end do
+    !$omp end distribute
+  !$omp end target teams
+
   !$omp teams 
       !ERROR: Only `DISTRIBUTE` or `PARALLEL` regions are allowed to be strictly nested inside `TEAMS` region.
       !$omp task
-- 
GitLab


From a43d2f686a46fd9d971aa65fde4563375e16f3de Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Fri, 17 May 2024 14:32:17 +0100
Subject: [PATCH 1127/1206] [Flang][OpenMP] NFC: Fix typo in include guard
 (#91593)

---
 flang/lib/Lower/OpenMP/ClauseProcessor.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 7e7a291b94ae..4d3d4448e8f0 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -9,8 +9,8 @@
 // Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
 //
 //===----------------------------------------------------------------------===//
-#ifndef FORTRAN_LOWER_CLAUASEPROCESSOR_H
-#define FORTRAN_LOWER_CLAUASEPROCESSOR_H
+#ifndef FORTRAN_LOWER_CLAUSEPROCESSOR_H
+#define FORTRAN_LOWER_CLAUSEPROCESSOR_H
 
 #include "Clauses.h"
 #include "DirectivesCommon.h"
@@ -312,4 +312,4 @@ bool ClauseProcessor::markClauseOccurrence(mlir::UnitAttr &result) const {
 } // namespace lower
 } // namespace Fortran
 
-#endif // FORTRAN_LOWER_CLAUASEPROCESSOR_H
+#endif // FORTRAN_LOWER_CLAUSEPROCESSOR_H
-- 
GitLab


From 502ccd81c6bcf913d2eddf9ed24496a9a634e808 Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Fri, 17 May 2024 14:33:10 +0100
Subject: [PATCH 1128/1206] [MLIR][OpenMP] NFC: Remove redundant check (#91594)

The check removed by this patch in the OpenMP to LLVM IR translation
pass already exists as part of the op verifier for `omp.loop_nest`.
---
 .../LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp       | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 8576be67f04a..50ee642b5098 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -939,10 +939,6 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   llvm::ArrayRef<bool> isByRef = getIsByRef(wsloopOp.getReductionVarsByref());
   assert(isByRef.size() == wsloopOp.getNumReductionVars());
 
-  // TODO: this should be in the op verifier instead.
-  if (loopOp.getLowerBound().empty())
-    return failure();
-
   // Static is the default.
   auto schedule =
       wsloopOp.getScheduleVal().value_or(omp::ClauseScheduleKind::Static);
-- 
GitLab


From d3d5a30021b8df26ac4b2b01e659ab8487fb4080 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 17 May 2024 16:40:56 +0300
Subject: [PATCH 1129/1206] [clang][NFC] Remove `const_cast` from
 `ParseClassSpecifier`

---
 clang/include/clang/Parse/Parser.h | 29 ++++++++++++-------------
 clang/lib/Parse/ParseDecl.cpp      | 12 ++++++-----
 clang/lib/Parse/ParseDeclCXX.cpp   | 34 ++++++++++++++----------------
 clang/lib/Parse/ParseObjc.cpp      |  6 ++++--
 clang/lib/Parse/ParseTemplate.cpp  |  3 ++-
 clang/lib/Parse/Parser.cpp         |  3 ++-
 6 files changed, 45 insertions(+), 42 deletions(-)

diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 7a8c2bcde804..c7d75985ce8b 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -2480,24 +2480,23 @@ private:
   bool trySkippingFunctionBody();
 
   bool ParseImplicitInt(DeclSpec &DS, CXXScopeSpec *SS,
-                        const ParsedTemplateInfo &TemplateInfo,
-                        AccessSpecifier AS, DeclSpecContext DSC,
-                        ParsedAttributes &Attrs);
+                        ParsedTemplateInfo &TemplateInfo, AccessSpecifier AS,
+                        DeclSpecContext DSC, ParsedAttributes &Attrs);
   DeclSpecContext
   getDeclSpecContextFromDeclaratorContext(DeclaratorContext Context);
-  void ParseDeclarationSpecifiers(
-      DeclSpec &DS,
-      const ParsedTemplateInfo &TemplateInfo = ParsedTemplateInfo(),
-      AccessSpecifier AS = AS_none,
-      DeclSpecContext DSC = DeclSpecContext::DSC_normal,
-      LateParsedAttrList *LateAttrs = nullptr) {
+  void
+  ParseDeclarationSpecifiers(DeclSpec &DS, ParsedTemplateInfo &TemplateInfo,
+                             AccessSpecifier AS = AS_none,
+                             DeclSpecContext DSC = DeclSpecContext::DSC_normal,
+                             LateParsedAttrList *LateAttrs = nullptr) {
     return ParseDeclarationSpecifiers(DS, TemplateInfo, AS, DSC, LateAttrs,
                                       getImplicitTypenameContext(DSC));
   }
-  void ParseDeclarationSpecifiers(
-      DeclSpec &DS, const ParsedTemplateInfo &TemplateInfo, AccessSpecifier AS,
-      DeclSpecContext DSC, LateParsedAttrList *LateAttrs,
-      ImplicitTypenameContext AllowImplicitTypename);
+  void
+  ParseDeclarationSpecifiers(DeclSpec &DS, ParsedTemplateInfo &TemplateInfo,
+                             AccessSpecifier AS, DeclSpecContext DSC,
+                             LateParsedAttrList *LateAttrs,
+                             ImplicitTypenameContext AllowImplicitTypename);
 
   SourceLocation ParsePackIndexingType(DeclSpec &DS);
   void AnnotateExistingIndexedTypeNamePack(ParsedType T,
@@ -3308,7 +3307,7 @@ private:
   // C++ 9: classes [class] and C structs/unions.
   bool isValidAfterTypeSpecifier(bool CouldBeBitfield);
   void ParseClassSpecifier(tok::TokenKind TagTokKind, SourceLocation TagLoc,
-                           DeclSpec &DS, const ParsedTemplateInfo &TemplateInfo,
+                           DeclSpec &DS, ParsedTemplateInfo &TemplateInfo,
                            AccessSpecifier AS, bool EnteringContext,
                            DeclSpecContext DSC, ParsedAttributes &Attributes);
   void SkipCXXMemberSpecification(SourceLocation StartLoc,
@@ -3330,7 +3329,7 @@ private:
                                                                VirtSpecifiers &VS);
   DeclGroupPtrTy ParseCXXClassMemberDeclaration(
       AccessSpecifier AS, ParsedAttributes &Attr,
-      const ParsedTemplateInfo &TemplateInfo = ParsedTemplateInfo(),
+      ParsedTemplateInfo &TemplateInfo,
       ParsingDeclRAIIObject *DiagsFromTParams = nullptr);
   DeclGroupPtrTy
   ParseCXXClassMemberDeclarationWithPragmas(AccessSpecifier &AS,
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 6d026878d327..179564ae41a3 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -2845,10 +2845,11 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
 void Parser::ParseSpecifierQualifierList(
     DeclSpec &DS, ImplicitTypenameContext AllowImplicitTypename,
     AccessSpecifier AS, DeclSpecContext DSC) {
+  ParsedTemplateInfo TemplateInfo;
   /// specifier-qualifier-list is a subset of declaration-specifiers.  Just
   /// parse declaration-specifiers and complain about extra stuff.
   /// TODO: diagnose attribute-specifiers and alignment-specifiers.
-  ParseDeclarationSpecifiers(DS, ParsedTemplateInfo(), AS, DSC, nullptr,
+  ParseDeclarationSpecifiers(DS, TemplateInfo, AS, DSC, nullptr,
                              AllowImplicitTypename);
 
   // Validate declspec for type-name.
@@ -2927,7 +2928,7 @@ static bool isValidAfterIdentifierInDeclarator(const Token &T) {
 /// other pieces of declspec after it, it returns true.
 ///
 bool Parser::ParseImplicitInt(DeclSpec &DS, CXXScopeSpec *SS,
-                              const ParsedTemplateInfo &TemplateInfo,
+                              ParsedTemplateInfo &TemplateInfo,
                               AccessSpecifier AS, DeclSpecContext DSC,
                               ParsedAttributes &Attrs) {
   assert(Tok.is(tok::identifier) && "should have identifier");
@@ -3489,7 +3490,7 @@ Parser::DiagnoseMissingSemiAfterTagDefinition(DeclSpec &DS, AccessSpecifier AS,
 ///       'friend': [C++ dcl.friend]
 ///       'constexpr': [C++0x dcl.constexpr]
 void Parser::ParseDeclarationSpecifiers(
-    DeclSpec &DS, const ParsedTemplateInfo &TemplateInfo, AccessSpecifier AS,
+    DeclSpec &DS, ParsedTemplateInfo &TemplateInfo, AccessSpecifier AS,
     DeclSpecContext DSContext, LateParsedAttrList *LateAttrs,
     ImplicitTypenameContext AllowImplicitTypename) {
   if (DS.getSourceRange().isInvalid()) {
@@ -7736,8 +7737,9 @@ void Parser::ParseParameterDeclarationClause(
         Diag(ThisLoc, diag::err_requires_expr_explicit_object_parameter);
     }
 
-    ParseDeclarationSpecifiers(DS, /*TemplateInfo=*/ParsedTemplateInfo(),
-                               AS_none, DeclSpecContext::DSC_normal,
+    ParsedTemplateInfo TemplateInfo;
+    ParseDeclarationSpecifiers(DS, TemplateInfo, AS_none,
+                               DeclSpecContext::DSC_normal,
                                /*LateAttrs=*/nullptr, AllowImplicitTypename);
 
     DS.takeAttributesFrom(ArgDeclSpecAttrs);
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 32c4e923243a..3caad5c75d7d 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -1661,7 +1661,7 @@ bool Parser::isValidAfterTypeSpecifier(bool CouldBeBitfield) {
 ///         'union'
 void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
                                  SourceLocation StartLoc, DeclSpec &DS,
-                                 const ParsedTemplateInfo &TemplateInfo,
+                                 ParsedTemplateInfo &TemplateInfo,
                                  AccessSpecifier AS, bool EnteringContext,
                                  DeclSpecContext DSC,
                                  ParsedAttributes &Attributes) {
@@ -1861,18 +1861,14 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
         TemplateParams->pop_back();
       } else {
         TemplateParams = nullptr;
-        const_cast<ParsedTemplateInfo &>(TemplateInfo).Kind =
-            ParsedTemplateInfo::NonTemplate;
+        TemplateInfo.Kind = ParsedTemplateInfo::NonTemplate;
       }
     } else if (TemplateInfo.Kind == ParsedTemplateInfo::ExplicitInstantiation) {
       // Pretend this is just a forward declaration.
       TemplateParams = nullptr;
-      const_cast<ParsedTemplateInfo &>(TemplateInfo).Kind =
-          ParsedTemplateInfo::NonTemplate;
-      const_cast<ParsedTemplateInfo &>(TemplateInfo).TemplateLoc =
-          SourceLocation();
-      const_cast<ParsedTemplateInfo &>(TemplateInfo).ExternLoc =
-          SourceLocation();
+      TemplateInfo.Kind = ParsedTemplateInfo::NonTemplate;
+      TemplateInfo.TemplateLoc = SourceLocation();
+      TemplateInfo.ExternLoc = SourceLocation();
     }
   };
 
@@ -2804,11 +2800,9 @@ void Parser::MaybeParseAndDiagnoseDeclSpecAfterCXX11VirtSpecifierSeq(
 ///       constant-initializer:
 ///         '=' constant-expression
 ///
-Parser::DeclGroupPtrTy
-Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
-                                       ParsedAttributes &AccessAttrs,
-                                       const ParsedTemplateInfo &TemplateInfo,
-                                       ParsingDeclRAIIObject *TemplateDiags) {
+Parser::DeclGroupPtrTy Parser::ParseCXXClassMemberDeclaration(
+    AccessSpecifier AS, ParsedAttributes &AccessAttrs,
+    ParsedTemplateInfo &TemplateInfo, ParsingDeclRAIIObject *TemplateDiags) {
   assert(getLangOpts().CPlusPlus &&
          "ParseCXXClassMemberDeclaration should only be called in C++ mode");
   if (Tok.is(tok::at)) {
@@ -3525,8 +3519,10 @@ Parser::DeclGroupPtrTy Parser::ParseCXXClassMemberDeclarationWithPragmas(
   case tok::kw_private:
     // FIXME: We don't accept GNU attributes on access specifiers in OpenCL mode
     // yet.
-    if (getLangOpts().OpenCL && !NextToken().is(tok::colon))
-      return ParseCXXClassMemberDeclaration(AS, AccessAttrs);
+    if (getLangOpts().OpenCL && !NextToken().is(tok::colon)) {
+      ParsedTemplateInfo TemplateInfo;
+      return ParseCXXClassMemberDeclaration(AS, AccessAttrs, TemplateInfo);
+    }
     [[fallthrough]];
   case tok::kw_public:
   case tok::kw_protected: {
@@ -3582,7 +3578,8 @@ Parser::DeclGroupPtrTy Parser::ParseCXXClassMemberDeclarationWithPragmas(
       ConsumeAnnotationToken();
       return nullptr;
     }
-    return ParseCXXClassMemberDeclaration(AS, AccessAttrs);
+    ParsedTemplateInfo TemplateInfo;
+    return ParseCXXClassMemberDeclaration(AS, AccessAttrs, TemplateInfo);
   }
 }
 
@@ -5167,8 +5164,9 @@ void Parser::ParseMicrosoftIfExistsClassDeclaration(
       continue;
     }
 
+    ParsedTemplateInfo TemplateInfo;
     // Parse all the comma separated declarators.
-    ParseCXXClassMemberDeclaration(CurAS, AccessAttrs);
+    ParseCXXClassMemberDeclaration(CurAS, AccessAttrs, TemplateInfo);
   }
 
   Braces.consumeClose();
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index 4cb04b353473..263aea2e5524 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -1533,7 +1533,8 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
       cStyleParamWarned = true;
     }
     DeclSpec DS(AttrFactory);
-    ParseDeclarationSpecifiers(DS);
+    ParsedTemplateInfo TemplateInfo;
+    ParseDeclarationSpecifiers(DS, TemplateInfo);
     // Parse the declarator.
     Declarator ParmDecl(DS, ParsedAttributesView::none(),
                         DeclaratorContext::Prototype);
@@ -2588,7 +2589,8 @@ StmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
                                         Scope::AtCatchScope);
         if (Tok.isNot(tok::ellipsis)) {
           DeclSpec DS(AttrFactory);
-          ParseDeclarationSpecifiers(DS);
+          ParsedTemplateInfo TemplateInfo;
+          ParseDeclarationSpecifiers(DS, TemplateInfo);
           Declarator ParmDecl(DS, ParsedAttributesView::none(),
                               DeclaratorContext::ObjCCatch);
           ParseDeclarator(ParmDecl);
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index 8f841be422dd..22ef44aaa087 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -907,7 +907,8 @@ Parser::ParseNonTypeTemplateParameter(unsigned Depth, unsigned Position) {
   // FIXME: The type should probably be restricted in some way... Not all
   // declarators (parts of declarators?) are accepted for parameters.
   DeclSpec DS(AttrFactory);
-  ParseDeclarationSpecifiers(DS, ParsedTemplateInfo(), AS_none,
+  ParsedTemplateInfo TemplateInfo;
+  ParseDeclarationSpecifiers(DS, TemplateInfo, AS_none,
                              DeclSpecContext::DSC_template_param);
 
   // Parse this as a typename.
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index 869b9c6669c2..e3e72a441ca6 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -1562,7 +1562,8 @@ void Parser::ParseKNRParamDeclarations(Declarator &D) {
 
     // Parse the common declaration-specifiers piece.
     DeclSpec DS(AttrFactory);
-    ParseDeclarationSpecifiers(DS);
+    ParsedTemplateInfo TemplateInfo;
+    ParseDeclarationSpecifiers(DS, TemplateInfo);
 
     // C99 6.9.1p6: 'each declaration in the declaration list shall have at
     // least one declarator'.
-- 
GitLab


From 89b83d2d3f2ae7b806987143167e8ccb79c6d674 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar@redhat.com>
Date: Fri, 17 May 2024 06:49:27 -0700
Subject: [PATCH 1130/1206] [git] Add hashes to requirements.txt for extra
 security (#92305)

https://pip.pypa.io/en/stable/topics/secure-installs/
---
 .github/workflows/issue-release-workflow.yml  |   2 +-
 .github/workflows/issue-subscriber.yml        |   2 +-
 .github/workflows/merged-prs.yml              |   2 +-
 .github/workflows/new-prs.yml                 |   2 +-
 .github/workflows/pr-request-release-note.yml |   2 +-
 .github/workflows/pr-subscriber.yml           |   2 +-
 .github/workflows/release-binaries.yml        |   2 +-
 .github/workflows/version-check.yml           |   2 +-
 llvm/utils/git/requirements.txt               | 310 +++++++++++++++++-
 9 files changed, 301 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/issue-release-workflow.yml b/.github/workflows/issue-release-workflow.yml
index eb88ec6e43c5..5027d4f3ea6f 100644
--- a/.github/workflows/issue-release-workflow.yml
+++ b/.github/workflows/issue-release-workflow.yml
@@ -53,7 +53,7 @@ jobs:
 
       - name: Setup Environment
         run: |
-          pip install -r ./llvm/utils/git/requirements.txt
+          pip install --require-hashes -r ./llvm/utils/git/requirements.txt
           ./llvm/utils/git/github-automation.py --token ${{ github.token }} setup-llvmbot-git
 
       - name: Backport Commits
diff --git a/.github/workflows/issue-subscriber.yml b/.github/workflows/issue-subscriber.yml
index ef6cd0674e80..ef4fdf441819 100644
--- a/.github/workflows/issue-subscriber.yml
+++ b/.github/workflows/issue-subscriber.yml
@@ -22,7 +22,7 @@ jobs:
       - name: Setup Automation Script
         working-directory: ./llvm/utils/git/
         run: |
-          pip install -r requirements.txt
+          pip install --require-hashes -r requirements.txt
 
       - name: Update watchers
         working-directory: ./llvm/utils/git/
diff --git a/.github/workflows/merged-prs.yml b/.github/workflows/merged-prs.yml
index 37fc6c67f000..e29afd4097f9 100644
--- a/.github/workflows/merged-prs.yml
+++ b/.github/workflows/merged-prs.yml
@@ -29,7 +29,7 @@ jobs:
       - name: Setup Automation Script
         working-directory: ./llvm/utils/git/
         run: |
-          pip install -r requirements.txt
+          pip install --require-hashes -r requirements.txt
 
       - name: Add Buildbot information comment
         working-directory: ./llvm/utils/git/
diff --git a/.github/workflows/new-prs.yml b/.github/workflows/new-prs.yml
index a60f82ce35d1..88175d6f8d64 100644
--- a/.github/workflows/new-prs.yml
+++ b/.github/workflows/new-prs.yml
@@ -43,7 +43,7 @@ jobs:
       - name: Setup Automation Script
         working-directory: ./llvm/utils/git/
         run: |
-          pip install -r requirements.txt
+          pip install --require-hashes -r requirements.txt
 
       - name: Greet Author
         working-directory: ./llvm/utils/git/
diff --git a/.github/workflows/pr-request-release-note.yml b/.github/workflows/pr-request-release-note.yml
index 0fcb95f1fe29..5e48ce7aee2e 100644
--- a/.github/workflows/pr-request-release-note.yml
+++ b/.github/workflows/pr-request-release-note.yml
@@ -29,7 +29,7 @@ jobs:
 
       - name: Install Dependencies
         run: |
-          pip install -r llvm/utils/git/requirements.txt
+          pip install --require-hashes -r llvm/utils/git/requirements.txt
 
       - name: Request Release Note
         env:
diff --git a/.github/workflows/pr-subscriber.yml b/.github/workflows/pr-subscriber.yml
index 3952493bb698..272d3e2f9ef8 100644
--- a/.github/workflows/pr-subscriber.yml
+++ b/.github/workflows/pr-subscriber.yml
@@ -22,7 +22,7 @@ jobs:
       - name: Setup Automation Script
         working-directory: ./llvm/utils/git/
         run: |
-          pip install -r requirements.txt
+          pip install --require-hashes -r requirements.txt
 
       - name: Update watchers
         working-directory: ./llvm/utils/git/
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 02082a84d8c1..fc497a7de94f 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -47,7 +47,7 @@ jobs:
 
     - name: Install Dependencies
       run: |
-        pip install -r ./llvm/utils/git/requirements.txt
+        pip install --require-hashes -r ./llvm/utils/git/requirements.txt
 
     - name: Check Permissions
       env:
diff --git a/.github/workflows/version-check.yml b/.github/workflows/version-check.yml
index c6d779080bbe..4ce6119a407f 100644
--- a/.github/workflows/version-check.yml
+++ b/.github/workflows/version-check.yml
@@ -23,7 +23,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -r ./llvm/utils/git/requirements.txt
+          pip install --require-hashes -r ./llvm/utils/git/requirements.txt
 
       - name: Version Check
         run: |
diff --git a/llvm/utils/git/requirements.txt b/llvm/utils/git/requirements.txt
index 0ff62a8ea5be..e354c91a4d5b 100644
--- a/llvm/utils/git/requirements.txt
+++ b/llvm/utils/git/requirements.txt
@@ -2,41 +2,317 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --output-file=requirements.txt requirements.txt.in
+#    pip-compile --generate-hashes --output-file=requirements.txt requirements.txt.in
 #
-certifi==2024.2.2
+certifi==2024.2.2 \
+    --hash=sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f \
+    --hash=sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1
     # via
     #   -r requirements.txt.in
     #   requests
-cffi==1.16.0
+cffi==1.16.0 \
+    --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \
+    --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \
+    --hash=sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417 \
+    --hash=sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab \
+    --hash=sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520 \
+    --hash=sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36 \
+    --hash=sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743 \
+    --hash=sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8 \
+    --hash=sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed \
+    --hash=sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684 \
+    --hash=sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56 \
+    --hash=sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324 \
+    --hash=sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d \
+    --hash=sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235 \
+    --hash=sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e \
+    --hash=sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088 \
+    --hash=sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000 \
+    --hash=sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7 \
+    --hash=sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e \
+    --hash=sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673 \
+    --hash=sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c \
+    --hash=sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe \
+    --hash=sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2 \
+    --hash=sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098 \
+    --hash=sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8 \
+    --hash=sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a \
+    --hash=sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0 \
+    --hash=sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b \
+    --hash=sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896 \
+    --hash=sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e \
+    --hash=sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9 \
+    --hash=sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2 \
+    --hash=sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b \
+    --hash=sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6 \
+    --hash=sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404 \
+    --hash=sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f \
+    --hash=sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0 \
+    --hash=sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4 \
+    --hash=sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc \
+    --hash=sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936 \
+    --hash=sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba \
+    --hash=sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872 \
+    --hash=sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb \
+    --hash=sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614 \
+    --hash=sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1 \
+    --hash=sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d \
+    --hash=sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969 \
+    --hash=sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b \
+    --hash=sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4 \
+    --hash=sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627 \
+    --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \
+    --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357
     # via
     #   cryptography
     #   pynacl
-charset-normalizer==3.3.2
+charset-normalizer==3.3.2 \
+    --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \
+    --hash=sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087 \
+    --hash=sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786 \
+    --hash=sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8 \
+    --hash=sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09 \
+    --hash=sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185 \
+    --hash=sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574 \
+    --hash=sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e \
+    --hash=sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519 \
+    --hash=sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898 \
+    --hash=sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269 \
+    --hash=sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3 \
+    --hash=sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f \
+    --hash=sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6 \
+    --hash=sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8 \
+    --hash=sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a \
+    --hash=sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73 \
+    --hash=sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc \
+    --hash=sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714 \
+    --hash=sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2 \
+    --hash=sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc \
+    --hash=sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce \
+    --hash=sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d \
+    --hash=sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e \
+    --hash=sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6 \
+    --hash=sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269 \
+    --hash=sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96 \
+    --hash=sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d \
+    --hash=sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a \
+    --hash=sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4 \
+    --hash=sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77 \
+    --hash=sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d \
+    --hash=sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0 \
+    --hash=sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed \
+    --hash=sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068 \
+    --hash=sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac \
+    --hash=sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25 \
+    --hash=sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8 \
+    --hash=sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab \
+    --hash=sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26 \
+    --hash=sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2 \
+    --hash=sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db \
+    --hash=sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f \
+    --hash=sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5 \
+    --hash=sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99 \
+    --hash=sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c \
+    --hash=sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d \
+    --hash=sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811 \
+    --hash=sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa \
+    --hash=sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a \
+    --hash=sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03 \
+    --hash=sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b \
+    --hash=sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04 \
+    --hash=sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c \
+    --hash=sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001 \
+    --hash=sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458 \
+    --hash=sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389 \
+    --hash=sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99 \
+    --hash=sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985 \
+    --hash=sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537 \
+    --hash=sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238 \
+    --hash=sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f \
+    --hash=sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d \
+    --hash=sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796 \
+    --hash=sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a \
+    --hash=sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143 \
+    --hash=sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8 \
+    --hash=sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c \
+    --hash=sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5 \
+    --hash=sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5 \
+    --hash=sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711 \
+    --hash=sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4 \
+    --hash=sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6 \
+    --hash=sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c \
+    --hash=sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7 \
+    --hash=sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4 \
+    --hash=sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b \
+    --hash=sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae \
+    --hash=sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12 \
+    --hash=sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c \
+    --hash=sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae \
+    --hash=sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8 \
+    --hash=sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887 \
+    --hash=sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b \
+    --hash=sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4 \
+    --hash=sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f \
+    --hash=sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5 \
+    --hash=sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33 \
+    --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \
+    --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561
     # via requests
-cryptography==42.0.5
+cryptography==42.0.5 \
+    --hash=sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee \
+    --hash=sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576 \
+    --hash=sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d \
+    --hash=sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30 \
+    --hash=sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413 \
+    --hash=sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb \
+    --hash=sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da \
+    --hash=sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4 \
+    --hash=sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd \
+    --hash=sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc \
+    --hash=sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8 \
+    --hash=sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1 \
+    --hash=sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc \
+    --hash=sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e \
+    --hash=sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8 \
+    --hash=sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940 \
+    --hash=sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400 \
+    --hash=sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7 \
+    --hash=sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16 \
+    --hash=sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278 \
+    --hash=sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74 \
+    --hash=sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec \
+    --hash=sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1 \
+    --hash=sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2 \
+    --hash=sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c \
+    --hash=sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922 \
+    --hash=sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a \
+    --hash=sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6 \
+    --hash=sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1 \
+    --hash=sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e \
+    --hash=sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac \
+    --hash=sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7
     # via pyjwt
-deprecated==1.2.14
+deprecated==1.2.14 \
+    --hash=sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c \
+    --hash=sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3
     # via pygithub
-gitdb==4.0.11
+gitdb==4.0.11 \
+    --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \
+    --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b
     # via gitpython
-gitpython==3.1.43
+gitpython==3.1.43 \
+    --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \
+    --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff
     # via -r requirements.txt.in
-idna==3.7
+idna==3.7 \
+    --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \
+    --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0
     # via requests
-pycparser==2.22
+pycparser==2.22 \
+    --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \
+    --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc
     # via cffi
-pygithub==1.59.1
+pygithub==1.59.1 \
+    --hash=sha256:3d87a822e6c868142f0c2c4bf16cce4696b5a7a4d142a7bd160e1bdf75bc54a9 \
+    --hash=sha256:c44e3a121c15bf9d3a5cc98d94c9a047a5132a9b01d22264627f58ade9ddc217
     # via -r requirements.txt.in
-pyjwt[crypto]==2.8.0
+pyjwt[crypto]==2.8.0 \
+    --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \
+    --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320
     # via pygithub
-pynacl==1.5.0
+pynacl==1.5.0 \
+    --hash=sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858 \
+    --hash=sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d \
+    --hash=sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93 \
+    --hash=sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1 \
+    --hash=sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92 \
+    --hash=sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff \
+    --hash=sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba \
+    --hash=sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394 \
+    --hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \
+    --hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543
     # via pygithub
-requests==2.31.0
+requests==2.31.0 \
+    --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
+    --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
     # via pygithub
-smmap==5.0.1
+smmap==5.0.1 \
+    --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \
+    --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da
     # via gitdb
-urllib3==2.2.1
+urllib3==2.2.1 \
+    --hash=sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d \
+    --hash=sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19
     # via requests
-wrapt==1.16.0
+wrapt==1.16.0 \
+    --hash=sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc \
+    --hash=sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81 \
+    --hash=sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09 \
+    --hash=sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e \
+    --hash=sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca \
+    --hash=sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0 \
+    --hash=sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb \
+    --hash=sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487 \
+    --hash=sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40 \
+    --hash=sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c \
+    --hash=sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060 \
+    --hash=sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202 \
+    --hash=sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41 \
+    --hash=sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9 \
+    --hash=sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b \
+    --hash=sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664 \
+    --hash=sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d \
+    --hash=sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362 \
+    --hash=sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00 \
+    --hash=sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc \
+    --hash=sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1 \
+    --hash=sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267 \
+    --hash=sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956 \
+    --hash=sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966 \
+    --hash=sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1 \
+    --hash=sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228 \
+    --hash=sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72 \
+    --hash=sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d \
+    --hash=sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292 \
+    --hash=sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0 \
+    --hash=sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0 \
+    --hash=sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36 \
+    --hash=sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c \
+    --hash=sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5 \
+    --hash=sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f \
+    --hash=sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73 \
+    --hash=sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b \
+    --hash=sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2 \
+    --hash=sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593 \
+    --hash=sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39 \
+    --hash=sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389 \
+    --hash=sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf \
+    --hash=sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf \
+    --hash=sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89 \
+    --hash=sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c \
+    --hash=sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c \
+    --hash=sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f \
+    --hash=sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440 \
+    --hash=sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465 \
+    --hash=sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136 \
+    --hash=sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b \
+    --hash=sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8 \
+    --hash=sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3 \
+    --hash=sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8 \
+    --hash=sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6 \
+    --hash=sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e \
+    --hash=sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f \
+    --hash=sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c \
+    --hash=sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e \
+    --hash=sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8 \
+    --hash=sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2 \
+    --hash=sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020 \
+    --hash=sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35 \
+    --hash=sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d \
+    --hash=sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3 \
+    --hash=sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537 \
+    --hash=sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809 \
+    --hash=sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d \
+    --hash=sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a \
+    --hash=sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4
     # via deprecated
-- 
GitLab


From 6677aef28583526c65073bd2406ca1246a41c2a8 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 17 May 2024 17:49:50 +0400
Subject: [PATCH 1131/1206] [clang][NFC] Remove const-qualification from
 `FunctionTemplateSpecializationInfo::TemplateArguments` (#92500)

This patch remove const-qualification from pointee type of
`FunctionTemplateSpecializationInfo::TemplateArguments`, because it's
(eventually) used to initialize `MultiLevelTemplateArgumentList`, which
can actually mutate the arguments via

https://github.com/llvm/llvm-project/blob/f42f57b52dd279e6ae19270d063aeb8d59e3f11c/clang/include/clang/Sema/Template.h#L197-L204
Mutation seems to be required to correctly handle packs:

https://github.com/llvm/llvm-project/blob/9144553207052a868efc5a8ce61a0afbb0eaf236/clang/lib/Sema/SemaTemplateInstantiate.cpp#L1440-L1469
---
 clang/include/clang/AST/Decl.h         | 25 ++++++++++++-------------
 clang/include/clang/AST/DeclTemplate.h |  7 +++----
 clang/lib/AST/Decl.cpp                 | 14 ++++++--------
 clang/lib/AST/DeclTemplate.cpp         |  2 +-
 clang/lib/Sema/SemaTemplate.cpp        |  2 +-
 5 files changed, 23 insertions(+), 27 deletions(-)

diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index de8b923645f8..5e485ccb85a1 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -2096,13 +2096,12 @@ private:
   ///
   /// \param PointOfInstantiation point at which the function template
   /// specialization was first instantiated.
-  void setFunctionTemplateSpecialization(ASTContext &C,
-                                         FunctionTemplateDecl *Template,
-                                       const TemplateArgumentList *TemplateArgs,
-                                         void *InsertPos,
-                                         TemplateSpecializationKind TSK,
-                          const TemplateArgumentListInfo *TemplateArgsAsWritten,
-                                         SourceLocation PointOfInstantiation);
+  void setFunctionTemplateSpecialization(
+      ASTContext &C, FunctionTemplateDecl *Template,
+      TemplateArgumentList *TemplateArgs, void *InsertPos,
+      TemplateSpecializationKind TSK,
+      const TemplateArgumentListInfo *TemplateArgsAsWritten,
+      SourceLocation PointOfInstantiation);
 
   /// Specify that this record is an instantiation of the
   /// member function FD.
@@ -2981,12 +2980,12 @@ public:
   ///
   /// \param PointOfInstantiation point at which the function template
   /// specialization was first instantiated.
-  void setFunctionTemplateSpecialization(FunctionTemplateDecl *Template,
-                const TemplateArgumentList *TemplateArgs,
-                void *InsertPos,
-                TemplateSpecializationKind TSK = TSK_ImplicitInstantiation,
-                const TemplateArgumentListInfo *TemplateArgsAsWritten = nullptr,
-                SourceLocation PointOfInstantiation = SourceLocation()) {
+  void setFunctionTemplateSpecialization(
+      FunctionTemplateDecl *Template, TemplateArgumentList *TemplateArgs,
+      void *InsertPos,
+      TemplateSpecializationKind TSK = TSK_ImplicitInstantiation,
+      TemplateArgumentListInfo *TemplateArgsAsWritten = nullptr,
+      SourceLocation PointOfInstantiation = SourceLocation()) {
     setFunctionTemplateSpecialization(getASTContext(), Template, TemplateArgs,
                                       InsertPos, TSK, TemplateArgsAsWritten,
                                       PointOfInstantiation);
diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index 268aeacf2f20..f3d6a321ecf1 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -478,7 +478,7 @@ class FunctionTemplateSpecializationInfo final
 public:
   /// The template arguments used to produce the function template
   /// specialization from the function template.
-  const TemplateArgumentList *TemplateArguments;
+  TemplateArgumentList *TemplateArguments;
 
   /// The template arguments as written in the sources, if provided.
   /// FIXME: Normally null; tail-allocate this.
@@ -491,7 +491,7 @@ public:
 private:
   FunctionTemplateSpecializationInfo(
       FunctionDecl *FD, FunctionTemplateDecl *Template,
-      TemplateSpecializationKind TSK, const TemplateArgumentList *TemplateArgs,
+      TemplateSpecializationKind TSK, TemplateArgumentList *TemplateArgs,
       const ASTTemplateArgumentListInfo *TemplateArgsAsWritten,
       SourceLocation POI, MemberSpecializationInfo *MSInfo)
       : Function(FD, MSInfo ? true : false), Template(Template, TSK - 1),
@@ -511,8 +511,7 @@ public:
 
   static FunctionTemplateSpecializationInfo *
   Create(ASTContext &C, FunctionDecl *FD, FunctionTemplateDecl *Template,
-         TemplateSpecializationKind TSK,
-         const TemplateArgumentList *TemplateArgs,
+         TemplateSpecializationKind TSK, TemplateArgumentList *TemplateArgs,
          const TemplateArgumentListInfo *TemplateArgsAsWritten,
          SourceLocation POI, MemberSpecializationInfo *MSInfo);
 
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index ec851c9371e1..41fbfe281ef6 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -4199,14 +4199,12 @@ FunctionDecl::getTemplateSpecializationArgsAsWritten() const {
   return nullptr;
 }
 
-void
-FunctionDecl::setFunctionTemplateSpecialization(ASTContext &C,
-                                                FunctionTemplateDecl *Template,
-                                     const TemplateArgumentList *TemplateArgs,
-                                                void *InsertPos,
-                                                TemplateSpecializationKind TSK,
-                        const TemplateArgumentListInfo *TemplateArgsAsWritten,
-                                          SourceLocation PointOfInstantiation) {
+void FunctionDecl::setFunctionTemplateSpecialization(
+    ASTContext &C, FunctionTemplateDecl *Template,
+    TemplateArgumentList *TemplateArgs, void *InsertPos,
+    TemplateSpecializationKind TSK,
+    const TemplateArgumentListInfo *TemplateArgsAsWritten,
+    SourceLocation PointOfInstantiation) {
   assert((TemplateOrSpecialization.isNull() ||
           TemplateOrSpecialization.is<MemberSpecializationInfo *>()) &&
          "Member function is already a specialization");
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index d22ecc3c032e..26765a5da1dc 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -884,7 +884,7 @@ TemplateArgumentList::CreateCopy(ASTContext &Context,
 
 FunctionTemplateSpecializationInfo *FunctionTemplateSpecializationInfo::Create(
     ASTContext &C, FunctionDecl *FD, FunctionTemplateDecl *Template,
-    TemplateSpecializationKind TSK, const TemplateArgumentList *TemplateArgs,
+    TemplateSpecializationKind TSK, TemplateArgumentList *TemplateArgs,
     const TemplateArgumentListInfo *TemplateArgsAsWritten, SourceLocation POI,
     MemberSpecializationInfo *MSInfo) {
   const ASTTemplateArgumentListInfo *ArgsAsWritten = nullptr;
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 0462d3520037..4937cce4621f 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -10500,7 +10500,7 @@ bool Sema::CheckFunctionTemplateSpecialization(
   // specialization, with the template arguments from the previous
   // specialization.
   // Take copies of (semantic and syntactic) template argument lists.
-  const TemplateArgumentList *TemplArgs = TemplateArgumentList::CreateCopy(
+  TemplateArgumentList *TemplArgs = TemplateArgumentList::CreateCopy(
       Context, Specialization->getTemplateSpecializationArgs()->asArray());
   FD->setFunctionTemplateSpecialization(
       Specialization->getPrimaryTemplate(), TemplArgs, /*InsertPos=*/nullptr,
-- 
GitLab


From a0cc1ab97888ea15647d9dfb82d51aed7f9e446e Mon Sep 17 00:00:00 2001
From: Momchil Velikov <momchil.velikov@arm.com>
Date: Fri, 17 May 2024 15:02:53 +0100
Subject: [PATCH 1132/1206] [AArch64] Add intrinsics for multi-vector to ZA
 array vector accumulators (#91606)

[Recommit of e88ba6d975d887ca001cae30bfa0c53d91165148]

According to the specification in
https://github.com/ARM-software/acle/pull/309 this adds the intrinsics

void_svadd_za16_vg1x2_f16(uint32_t slice, svfloat16x2_t zn)
__arm_streaming __arm_inout("za");
void_svadd_za16_vg1x4_f16(uint32_t slice, svfloat16x4_t zn)
__arm_streaming __arm_inout("za");
void_svsub_za16_vg1x2_f16(uint32_t slice, svfloat16x2_t zn)
__arm_streaming __arm_inout("za");
void_svsub_za16_vg1x4_f16(uint32_t slice, svfloat16x4_t zn)
__arm_streaming __arm_inout("za");

as well as the corresponding `bf16` variants.
---
 clang/include/clang/Basic/arm_sme.td          |  10 +
 .../acle_sme2_add_sub_za16.c                  | 193 ++++++++++++++++++
 .../acle_sme2_add_sub_za16.c                  |  29 +++
 llvm/include/llvm/IR/IntrinsicsAArch64.td     |   2 +-
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |  16 +-
 .../AArch64/sme2-intrinsics-add-sub-za16.ll   | 148 ++++++++++++++
 6 files changed, 389 insertions(+), 9 deletions(-)
 create mode 100644 clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
 create mode 100644 clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
 create mode 100644 llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll

diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index 7808ee559932..80e635e4a57e 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -298,6 +298,16 @@ multiclass ZAAddSub<string n_suffix> {
     def NAME # _ZA64_VG1X2_F64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x2", "vm2", "d", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x2", [IsStreaming, IsInOutZA], []>;
     def NAME # _ZA64_VG1X4_F64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x4", "vm4", "d", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x4", [IsStreaming, IsInOutZA], []>;
   }
+
+  let TargetGuard = "sme-f16f16|sme-f8f16" in {
+    def NAME # _ZA16_VG1X2_F16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x2", "vm2", "h", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x2", [IsStreaming, IsInOutZA], []>;
+    def NAME # _ZA16_VG1X4_F16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x4", "vm4", "h", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x4", [IsStreaming, IsInOutZA], []>;
+  }
+
+  let TargetGuard = "sme2,b16b16" in {
+    def NAME # _ZA16_VG1X2_BF16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x2", "vm2", "b", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x2", [IsStreaming, IsInOutZA], []>;
+    def NAME # _ZA16_VG1X4_BF16 : Inst<"sv" # n_suffix # "_za16[_{d}]_vg1x4", "vm4", "b", MergeNone, "aarch64_sme_" # n_suffix # "_za16_vg1x4", [IsStreaming, IsInOutZA], []>;
+  }
 }
 
 defm SVADD : ZAAddSub<"add">;
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
new file mode 100644
index 000000000000..9a8aa448d378
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
@@ -0,0 +1,193 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
+// RUN: %clang_cc1                               -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1                        -x c++ -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature  +sme-f8f16 -target-feature +b16b16 -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s -check-prefix CHECK-CXX
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS        -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature  +sme-f8f16 -target-feature +b16b16 -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -x c++ -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -Werror -Wall -emit-llvm -o - %s | FileCheck %s -check-prefix CHECK-CXX
+
+// RUN: %clang_cc1                               -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sme-f16f16 -target-feature +b16b16 -O2 -S -Werror -Wall -o /dev/null
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3) A1##A2##A3
+#endif
+
+// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x2_f16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svadd_za16_vg1x2_f16j13svfloat16x2_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svadd_za16_vg1x2_f16(uint32_t slice, svfloat16x2_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svadd_za16,_f16,_vg1x2)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x4_f16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svadd_za16_vg1x4_f16j13svfloat16x4_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svadd_za16_vg1x4_f16(uint32_t slice, svfloat16x4_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svadd_za16,_f16,_vg1x4)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x2_f16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svsub_za16_vg1x2_f16j13svfloat16x2_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svsub_za16_vg1x2_f16(uint32_t slice, svfloat16x2_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svsub_za16,_f16,_vg1x2)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x4_f16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svsub_za16_vg1x4_f16j13svfloat16x4_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 [[SLICE]], <vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svsub_za16_vg1x4_f16(uint32_t slice, svfloat16x4_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svsub_za16,_f16,_vg1x4)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x2_bf16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svadd_za16_vg1x2_bf16j14svbfloat16x2_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svadd_za16_vg1x2_bf16(uint32_t slice, svbfloat16x2_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svadd_za16,_bf16,_vg1x2)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svadd_za16_vg1x4_bf16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svadd_za16_vg1x4_bf16j14svbfloat16x4_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svadd_za16_vg1x4_bf16(uint32_t slice, svbfloat16x4_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svadd_za16,_bf16,_vg1x4)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x2_bf16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svsub_za16_vg1x2_bf16j14svbfloat16x2_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svsub_za16_vg1x2_bf16(uint32_t slice, svbfloat16x2_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svsub_za16,_bf16,_vg1x2)(slice, zn);
+}
+
+// CHECK-LABEL: define dso_local void @test_svsub_za16_vg1x4_bf16(
+// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svsub_za16_vg1x4_bf16j14svbfloat16x4_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE:%.*]], <vscale x 32 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 0)
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 8)
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 16)
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[ZN]], i64 24)
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 [[SLICE]], <vscale x 8 x bfloat> [[TMP0]], <vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svsub_za16_vg1x4_bf16(uint32_t slice, svbfloat16x4_t zn) __arm_streaming __arm_inout("za") {
+  SVE_ACLE_FUNC(svsub_za16,_bf16,_vg1x4)(slice, zn);
+}
diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
new file mode 100644
index 000000000000..fb43bc633708
--- /dev/null
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -verify -emit-llvm %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+void test_features(uint32_t slice, svfloat16x2_t zn2, svfloat16x4_t zn4,
+                   svbfloat16x2_t bzn2, svbfloat16x4_t bzn4) __arm_streaming __arm_inout("za") {
+  // expected-error@+1 {{'svadd_za16_f16_vg1x2' needs target feature sme-f16f16|sme-f8f16}}
+  svadd_za16_f16_vg1x2(slice, zn2);
+  // expected-error@+1 {{'svadd_za16_f16_vg1x4' needs target feature sme-f16f16|sme-f8f16}}
+  svadd_za16_f16_vg1x4(slice, zn4);
+  // expected-error@+1 {{'svsub_za16_f16_vg1x2' needs target feature sme-f16f16|sme-f8f16}}
+  svsub_za16_f16_vg1x2(slice, zn2);
+  // expected-error@+1 {{'svsub_za16_f16_vg1x4' needs target feature sme-f16f16|sme-f8f16}}
+  svsub_za16_f16_vg1x4(slice, zn4);
+
+  // expected-error@+1 {{'svadd_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
+  svadd_za16_bf16_vg1x2(slice, bzn2);
+  // expected-error@+1 {{'svadd_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
+  svadd_za16_bf16_vg1x4(slice, bzn4);
+  // expected-error@+1 {{'svsub_za16_bf16_vg1x2' needs target feature sme2,b16b16}}
+  svsub_za16_bf16_vg1x2(slice, bzn2);
+  // expected-error@+1 {{'svsub_za16_bf16_vg1x4' needs target feature sme2,b16b16}}
+  svsub_za16_bf16_vg1x4(slice, bzn4);
+}
+
+
+
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index e0630a6649dd..4544cf35fb7b 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -3481,7 +3481,7 @@ let TargetPrefix = "aarch64" in {
   // Multi-vector add/sub and accumulate into ZA
   //
   foreach intr = ["add", "sub"] in {
-    foreach za = ["za32", "za64"] in {
+    foreach za = ["za16","za32", "za64"] in {
       def int_aarch64_sme_ # intr # _ # za # _vg1x2 : SME2_ZA_Write_VG2_Intrinsic;
       def int_aarch64_sme_ # intr # _ # za # _vg1x4 : SME2_ZA_Write_VG4_Intrinsic;
     }
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index c5cbdce476ca..2b70c4715bf9 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -793,10 +793,10 @@ defm LUTI4_S_4ZTZI : sme2p1_luti4_vector_vg4_index<"luti4">;
 }
 
 let Predicates = [HasSMEF16F16orSMEF8F16] in {
-defm FADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fadd", 0b0100, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
-defm FADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fadd", 0b0100, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
-defm FSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fsub", 0b0101, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
-defm FSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fsub", 0b0101, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
+defm FADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fadd", 0b0100, MatrixOp16, ZZ_h_mul_r, nxv8f16, int_aarch64_sme_add_za16_vg1x2>;
+defm FADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fadd", 0b0100, MatrixOp16, ZZZZ_h_mul_r, nxv8f16,  int_aarch64_sme_add_za16_vg1x4>;
+defm FSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fsub", 0b0101, MatrixOp16, ZZ_h_mul_r, nxv8f16,  int_aarch64_sme_sub_za16_vg1x2>;
+defm FSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fsub", 0b0101, MatrixOp16, ZZZZ_h_mul_r, nxv8f16,  int_aarch64_sme_sub_za16_vg1x4>;
 
 defm FMLA_VG2_M2ZZI_H : sme2p1_multi_vec_array_vg2_index_16b<"fmla", 0b00, 0b100, ZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fmla_lane_vg1x2>;
 defm FMLA_VG4_M4ZZI_H : sme2p1_multi_vec_array_vg4_index_16b<"fmla", 0b000, ZZZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fmla_lane_vg1x4>;
@@ -820,10 +820,10 @@ defm FMOPS_MPPZZ_H : sme2p1_fmop_tile_fp16<"fmops", 0b0, 0b1, nxv8f16, int_aarch
 }
 
 let Predicates = [HasSME2, HasB16B16] in {
-defm BFADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfadd", 0b1100, MatrixOp16, ZZ_h_mul_r, nxv8bf16, null_frag>;
-defm BFADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfadd", 0b1100, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16, null_frag>;
-defm BFSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfsub", 0b1101, MatrixOp16, ZZ_h_mul_r,  nxv8bf16, null_frag>;
-defm BFSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfsub", 0b1101, MatrixOp16, ZZZZ_h_mul_r,  nxv8bf16, null_frag>;
+defm BFADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfadd", 0b1100, MatrixOp16, ZZ_h_mul_r, nxv8bf16,    int_aarch64_sme_add_za16_vg1x2>;
+defm BFADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfadd", 0b1100, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16,  int_aarch64_sme_add_za16_vg1x4>;
+defm BFSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfsub", 0b1101, MatrixOp16, ZZ_h_mul_r,  nxv8bf16,   int_aarch64_sme_sub_za16_vg1x2>;
+defm BFSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfsub", 0b1101, MatrixOp16, ZZZZ_h_mul_r,  nxv8bf16, int_aarch64_sme_sub_za16_vg1x4>;
 
 defm BFMLA_VG2_M2ZZI : sme2p1_multi_vec_array_vg2_index_16b<"bfmla", 0b00, 0b110, ZZ_h_mul_r, ZPR4b16, nxv8bf16, int_aarch64_sme_fmla_lane_vg1x2>;
 defm BFMLA_VG4_M4ZZI : sme2p1_multi_vec_array_vg4_index_16b<"bfmla", 0b010, ZZZZ_h_mul_r, ZPR4b16, nxv8bf16, int_aarch64_sme_fmla_lane_vg1x4>;
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll
new file mode 100644
index 000000000000..e7a6c0d6c549
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-add-sub-za16.ll
@@ -0,0 +1,148 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+
+target triple = "aarch64-linux"
+
+define void @add_f16_vg1x2(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1) #0 {
+; CHECK-LABEL: add_f16_vg1x2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    fadd za.h[w8, 0, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    fadd za.h[w8, 7, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
+  ret void
+}
+
+define void @add_f16_vg1x4(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+; CHECK-LABEL: add_f16_vg1x4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    fadd za.h[w8, 0, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    fadd za.h[w8, 7, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    ret
+                            <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3) #1 {
+  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
+  ret void
+}
+
+define void @sub_f16_vg1x2(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1) #1 {
+; CHECK-LABEL: sub_f16_vg1x2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    fsub za.h[w8, 0, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    fsub za.h[w8, 7, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1)
+  ret void
+}
+
+define void @sub_f16_vg1x4(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+; CHECK-LABEL: sub_f16_vg1x4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    fsub za.h[w8, 0, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    fsub za.h[w8, 7, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    ret
+                            <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3) #0 {
+  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8f16(i32 %slice.7, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,
+                                                     <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3);
+  ret void
+}
+
+define void @add_bf16_vg1x2(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1) #2 {
+; CHECK-LABEL: add_bf16_vg1x2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    bfadd za.h[w8, 0, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    bfadd za.h[w8, 7, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.add.za16.vg1x2.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
+  ret void
+}
+
+define void @add_bf16_vg1x4(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+; CHECK-LABEL: add_bf16_vg1x4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    bfadd za.h[w8, 0, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    bfadd za.h[w8, 7, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    ret
+                            <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3) #2 {
+  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.add.za16.vg1x4.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
+  ret void
+}
+
+define void @sub_bf16_vg1x2(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1) #2 {
+; CHECK-LABEL: sub_bf16_vg1x2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    bfsub za.h[w8, 0, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    bfsub za.h[w8, 7, vgx2], { z0.h, z1.h }
+; CHECK-NEXT:    ret
+  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.sub.za16.vg1x2.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1)
+  ret void
+}
+
+define void @sub_bf16_vg1x4(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+; CHECK-LABEL: sub_bf16_vg1x4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    bfsub za.h[w8, 0, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    bfsub za.h[w8, 7, vgx4], { z0.h - z3.h }
+; CHECK-NEXT:    ret
+                            <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3) #2 {
+  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
+  %slice.7 = add i32 %slice, 7
+  call void @llvm.aarch64.sme.sub.za16.vg1x4.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,
+                                                     <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3);
+  ret void
+}
+
+attributes #0 = { nounwind "target-features"="+sme-f16f16" }
+attributes #1 = { nounwind "target-features"="+sme-f8f16" }
+attributes #2 = { nounwind "target-features"="+sme2,+bf16,+b16b16" }
-- 
GitLab


From fa9881d6a9d0bf3c72754a310c800d424c2db932 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 17 May 2024 14:29:14 +0100
Subject: [PATCH 1133/1206] [X86] vector-bitreverse.ll - add AVX512BW+AVX512VL
 test coverage

---
 llvm/test/CodeGen/X86/vector-bitreverse.ll | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/test/CodeGen/X86/vector-bitreverse.ll b/llvm/test/CodeGen/X86/vector-bitreverse.ll
index 90cc3d5fdde8..c54a7f464225 100644
--- a/llvm/test/CodeGen/X86/vector-bitreverse.ll
+++ b/llvm/test/CodeGen/X86/vector-bitreverse.ll
@@ -5,6 +5,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=ALL,AVX,AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=ALL,AVX,AVX512,AVX512F
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=ALL,AVX,AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=ALL,AVX,AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=ALL,XOP,XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=ALL,XOP,XOPAVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3,+gfni | FileCheck %s --check-prefixes=ALL,GFNISSE
-- 
GitLab


From c4017cda006c437c8400e280f8473aa65c302b40 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 17 May 2024 09:05:03 -0500
Subject: [PATCH 1134/1206] [Offload][NFC] Remove header license in CMake files
 (#92544)

Summary:
No other project has these in the CMake itself, and they're wildly
inconsistent even within the project. These don't really add anything so
I think they should be removed.
---
 offload/CMakeLists.txt                           | 13 ++-----------
 offload/DeviceRTL/CMakeLists.txt                 | 12 ------------
 .../Modules/LibomptargetGetDependencies.cmake    | 10 ----------
 offload/cmake/Modules/LibomptargetUtils.cmake    | 10 ----------
 offload/plugins-nextgen/CMakeLists.txt           | 12 ------------
 offload/plugins-nextgen/amdgpu/CMakeLists.txt    | 16 +---------------
 offload/plugins-nextgen/common/CMakeLists.txt    | 12 ------------
 offload/plugins-nextgen/cuda/CMakeLists.txt      | 11 -----------
 offload/src/CMakeLists.txt                       | 12 ------------
 offload/tools/CMakeLists.txt                     | 12 ------------
 offload/tools/deviceinfo/CMakeLists.txt          | 12 ------------
 offload/tools/kernelreplay/CMakeLists.txt        | 12 ------------
 12 files changed, 3 insertions(+), 141 deletions(-)

diff --git a/offload/CMakeLists.txt b/offload/CMakeLists.txt
index e2a4c2597636..31adbe45388a 100644
--- a/offload/CMakeLists.txt
+++ b/offload/CMakeLists.txt
@@ -1,14 +1,5 @@
-##===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-##===----------------------------------------------------------------------===##
-#
-# Build offloading library and related plugins.
-#
-##===----------------------------------------------------------------------===##
+# See for https://openmp.llvm.org/SupportAndFAQ.html for instructions on how
+# to build offload with CMake.
 
 cmake_minimum_required(VERSION 3.20.0)
 
diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt
index cbc859059100..ae7d54d127ed 100644
--- a/offload/DeviceRTL/CMakeLists.txt
+++ b/offload/DeviceRTL/CMakeLists.txt
@@ -1,15 +1,3 @@
-##===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-##===----------------------------------------------------------------------===##
-#
-# Build the DeviceRTL for all toolchains that are available
-#
-##===----------------------------------------------------------------------===##
-
 set(LIBOMPTARGET_BUILD_DEVICERTL_BCLIB TRUE CACHE BOOL
   "Can be set to false to disable building this library.")
 
diff --git a/offload/cmake/Modules/LibomptargetGetDependencies.cmake b/offload/cmake/Modules/LibomptargetGetDependencies.cmake
index bbf2b9836c70..e37b86b2a81f 100644
--- a/offload/cmake/Modules/LibomptargetGetDependencies.cmake
+++ b/offload/cmake/Modules/LibomptargetGetDependencies.cmake
@@ -1,13 +1,3 @@
-#
-#//===----------------------------------------------------------------------===//
-#//
-#// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#// See https://llvm.org/LICENSE.txt for license information.
-#// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#//
-#//===----------------------------------------------------------------------===//
-#
-
 # Try to detect in the system several dependencies required by the different
 # components of libomptarget. These are the dependencies we have:
 #
diff --git a/offload/cmake/Modules/LibomptargetUtils.cmake b/offload/cmake/Modules/LibomptargetUtils.cmake
index 7339cc0b56ed..57682dc8e367 100644
--- a/offload/cmake/Modules/LibomptargetUtils.cmake
+++ b/offload/cmake/Modules/LibomptargetUtils.cmake
@@ -1,13 +1,3 @@
-#
-#//===----------------------------------------------------------------------===//
-#//
-#// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#// See https://llvm.org/LICENSE.txt for license information.
-#// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#//
-#//===----------------------------------------------------------------------===//
-#
-
 # void libomptarget_say(string message_to_user);
 # - prints out message_to_user
 macro(libomptarget_say message_to_user)
diff --git a/offload/plugins-nextgen/CMakeLists.txt b/offload/plugins-nextgen/CMakeLists.txt
index 79f1c6dc66e8..5d684420c290 100644
--- a/offload/plugins-nextgen/CMakeLists.txt
+++ b/offload/plugins-nextgen/CMakeLists.txt
@@ -1,15 +1,3 @@
-##===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-##===----------------------------------------------------------------------===##
-#
-# Build plugins for the user system if available.
-#
-##===----------------------------------------------------------------------===##
-
 # Common interface to handle creating a plugin library.
 set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common)
 add_subdirectory(common)
diff --git a/offload/plugins-nextgen/amdgpu/CMakeLists.txt b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
index 280cb57e22e9..4a79fa83420c 100644
--- a/offload/plugins-nextgen/amdgpu/CMakeLists.txt
+++ b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
@@ -1,18 +1,4 @@
-##===----------------------------------------------------------------------===##
-#
-#                     The LLVM Compiler Infrastructure
-#
-# This file is dual licensed under the MIT and the University of Illinois Open
-# Source Licenses. See LICENSE.txt for details.
-#
-##===----------------------------------------------------------------------===##
-#
-# Build a plugin for an AMDGPU machine if available.
-#
-##===----------------------------------------------------------------------===##
-
-################################################################################
-# as of rocm-3.7, hsa is installed with cmake packages and kmt is found via hsa
+# As of rocm-3.7, hsa is installed with cmake packages and kmt is found via hsa
 find_package(hsa-runtime64 QUIET 1.2.0 HINTS ${CMAKE_INSTALL_PREFIX} PATHS /opt/rocm)
 
 if(NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)|(aarch64)$" AND CMAKE_SYSTEM_NAME MATCHES "Linux"))
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 6064468ea5b5..43e08d575a4a 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -1,15 +1,3 @@
-##===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-##===----------------------------------------------------------------------===##
-#
-# Common parts which can be used by all plugins
-#
-##===----------------------------------------------------------------------===##
-
 # NOTE: Don't try to build `PluginInterface` using `add_llvm_library` because we
 # don't want to export `PluginInterface` while `add_llvm_library` requires that.
 add_library(PluginCommon OBJECT
diff --git a/offload/plugins-nextgen/cuda/CMakeLists.txt b/offload/plugins-nextgen/cuda/CMakeLists.txt
index 3f3388e56c5f..4be33c6b69dc 100644
--- a/offload/plugins-nextgen/cuda/CMakeLists.txt
+++ b/offload/plugins-nextgen/cuda/CMakeLists.txt
@@ -1,14 +1,3 @@
-##===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-##===----------------------------------------------------------------------===##
-#
-# Build a plugin for a CUDA machine if available.
-#
-##===----------------------------------------------------------------------===##
 if (NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)|(aarch64)$" AND CMAKE_SYSTEM_NAME MATCHES "Linux"))
   libomptarget_say("Not building CUDA NextGen offloading plugin: only support CUDA in Linux x86_64, ppc64le, or aarch64 hosts.")
   return()
diff --git a/offload/src/CMakeLists.txt b/offload/src/CMakeLists.txt
index c5338130297f..590fb650c0ae 100644
--- a/offload/src/CMakeLists.txt
+++ b/offload/src/CMakeLists.txt
@@ -1,15 +1,3 @@
-##===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-##===----------------------------------------------------------------------===##
-#
-# Build offloading library libomptarget.so.
-#
-##===----------------------------------------------------------------------===##
-
 libomptarget_say("Building offloading runtime library libomptarget.")
 
 if(LIBOMP_STANDALONE)
diff --git a/offload/tools/CMakeLists.txt b/offload/tools/CMakeLists.txt
index a850647fbd58..13772a4a675e 100644
--- a/offload/tools/CMakeLists.txt
+++ b/offload/tools/CMakeLists.txt
@@ -1,15 +1,3 @@
-##===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-##===----------------------------------------------------------------------===##
-#
-# Adding omptarget tools
-#
-##===----------------------------------------------------------------------===##
-
 set(OPENMP_TOOLS_INSTALL_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH
     "Path for binary subdirectory (defaults to '${CMAKE_INSTALL_BINDIR}')")
 mark_as_advanced(OPENMP_TOOLS_INSTALL_DIR)
diff --git a/offload/tools/deviceinfo/CMakeLists.txt b/offload/tools/deviceinfo/CMakeLists.txt
index 62d36d7cc92f..b83f02fd6eb9 100644
--- a/offload/tools/deviceinfo/CMakeLists.txt
+++ b/offload/tools/deviceinfo/CMakeLists.txt
@@ -1,15 +1,3 @@
-##===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-##===----------------------------------------------------------------------===##
-#
-# Build llvm-omp-device-info tool
-#
-##===----------------------------------------------------------------------===##
-
 libomptarget_say("Building the llvm-omp-device-info tool")
 
 add_openmp_tool(llvm-omp-device-info llvm-omp-device-info.cpp)
diff --git a/offload/tools/kernelreplay/CMakeLists.txt b/offload/tools/kernelreplay/CMakeLists.txt
index 6f3dc33d6736..ae0c5caac4c9 100644
--- a/offload/tools/kernelreplay/CMakeLists.txt
+++ b/offload/tools/kernelreplay/CMakeLists.txt
@@ -1,15 +1,3 @@
-##===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-##===----------------------------------------------------------------------===##
-#
-# Build llvm-omp-kernel-replay tool
-#
-##===----------------------------------------------------------------------===##
-
 libomptarget_say("Building the llvm-omp-kernel-replay tool")
 
 add_openmp_tool(llvm-omp-kernel-replay llvm-omp-kernel-replay.cpp)
-- 
GitLab


From 8c5e487ee3160cf4d40c0680ed40e4b0a494e956 Mon Sep 17 00:00:00 2001
From: Chris B <chris.bieneman@me.com>
Date: Fri, 17 May 2024 09:10:27 -0500
Subject: [PATCH 1135/1206] [HLSL][CMake] Add clangd and distribution settings
 (#92011)

This just adds some simple distribution settings and includes clangd in
the build for distribution.
---
 clang/cmake/caches/HLSL.cmake | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/clang/cmake/caches/HLSL.cmake b/clang/cmake/caches/HLSL.cmake
index 84850c86f12c..27f848fdccf0 100644
--- a/clang/cmake/caches/HLSL.cmake
+++ b/clang/cmake/caches/HLSL.cmake
@@ -8,6 +8,12 @@ set(LLVM_EXPERIMENTAL_TARGETS_TO_BUILD "DirectX;SPIRV" CACHE STRING "")
 
 # HLSL support is currently limted to clang, eventually it will expand to
 # clang-tools-extra too.
-set(LLVM_ENABLE_PROJECTS "clang" CACHE STRING "")
+set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra" CACHE STRING "")
 
 set(CLANG_ENABLE_HLSL On CACHE BOOL "")
+
+if (NOT CMAKE_CONFIGURATION_TYPES)
+  set(LLVM_DISTRIBUTION_COMPONENTS
+      "clang;hlsl-resource-headers;clangd"
+      CACHE STRING "")
+endif()
-- 
GitLab


From 30d0850e0f780b17a37522e6503c98ebe197c5fa Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 17 May 2024 17:15:06 +0300
Subject: [PATCH 1136/1206] [clang][NFC] Improve const-correctness in
 `ParseLexedMethodDeclaration`

---
 clang/lib/Parse/ParseCXXInlineMethods.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index a26568dfd6aa..943ce0fdde3a 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -466,14 +466,14 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
         ConsumeAnyToken();
     } else if (HasUnparsed) {
       assert(Param->hasInheritedDefaultArg());
-      const FunctionDecl *Old;
+      FunctionDecl *Old;
       if (const auto *FunTmpl = dyn_cast<FunctionTemplateDecl>(LM.Method))
         Old =
             cast<FunctionDecl>(FunTmpl->getTemplatedDecl())->getPreviousDecl();
       else
         Old = cast<FunctionDecl>(LM.Method)->getPreviousDecl();
       if (Old) {
-        ParmVarDecl *OldParam = const_cast<ParmVarDecl*>(Old->getParamDecl(I));
+        ParmVarDecl *OldParam = Old->getParamDecl(I);
         assert(!OldParam->hasUnparsedDefaultArg());
         if (OldParam->hasUninstantiatedDefaultArg())
           Param->setUninstantiatedDefaultArg(
-- 
GitLab


From 9ba0a7725680fec5e071634f814a9d32ba33e852 Mon Sep 17 00:00:00 2001
From: maxbartel <bartel@roofline.ai>
Date: Fri, 17 May 2024 16:29:48 +0200
Subject: [PATCH 1137/1206] [mlir][spirv] Add support for dense_resource in
 arith to spirv (#91318)

This adds support for `dense_resource` in arith to spirv.

Note that this inlines the blob into the IR. Another possibility would
be to add proper dense_resource support to spirv, but there is a lot of
special handling going on to convert a `DenseElementsAttr` to the
correct SPIRV type. Some of that even iterates over all the values in
the Attribute. For proper support of a `DenseResourceElementsAttr` this
probably needs a redesign. I would like to hear some opinions on that!

The test is disabled on non little Endian machines. See
https://github.com/llvm/llvm-project/issues/63469 for more information.
---
 .../Conversion/ArithToSPIRV/ArithToSPIRV.cpp  | 36 +++++++++++++++---
 .../arith-to-spirv-le-specific.mlir           | 38 +++++++++++++++++++
 .../arith-to-spirv-unsupported.mlir           | 28 ++++++++++++++
 3 files changed, 97 insertions(+), 5 deletions(-)
 create mode 100644 mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-le-specific.mlir

diff --git a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
index 806981728561..4c3237b24b78 100644
--- a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
+++ b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
@@ -17,6 +17,7 @@
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/DialectResourceBlobManager.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
@@ -229,16 +230,41 @@ struct ConstantCompositeOpPattern final
     if (!srcType || srcType.getNumElements() == 1)
       return failure();
 
-    // arith.constant should only have vector or tenor types.
-    assert((isa<VectorType, RankedTensorType>(srcType)));
+    // arith.constant should only have vector or tensor types. This is a MLIR
+    // wide problem at the moment.
+    if (!isa<VectorType, RankedTensorType>(srcType))
+      return rewriter.notifyMatchFailure(constOp, "unsupported ShapedType");
 
     Type dstType = getTypeConverter()->convertType(srcType);
     if (!dstType)
       return failure();
 
-    auto dstElementsAttr = dyn_cast<DenseElementsAttr>(constOp.getValue());
-    if (!dstElementsAttr)
-      return failure();
+    // Import the resource into the IR to make use of the special handling of
+    // element types later on.
+    mlir::DenseElementsAttr dstElementsAttr;
+    if (auto denseElementsAttr =
+            dyn_cast<DenseElementsAttr>(constOp.getValue())) {
+      dstElementsAttr = denseElementsAttr;
+    } else if (auto resourceAttr =
+                   dyn_cast<DenseResourceElementsAttr>(constOp.getValue())) {
+
+      AsmResourceBlob *blob = resourceAttr.getRawHandle().getBlob();
+      if (!blob)
+        return constOp->emitError("could not find resource blob");
+
+      ArrayRef<char> ptr = blob->getData();
+
+      // Check that the buffer meets the requirements to get converted to a
+      // DenseElementsAttr
+      bool detectedSplat = false;
+      if (!DenseElementsAttr::isValidRawBuffer(srcType, ptr, detectedSplat))
+        return constOp->emitError("resource is not a valid buffer");
+
+      dstElementsAttr =
+          DenseElementsAttr::getFromRawBuffer(resourceAttr.getType(), ptr);
+    } else {
+      return constOp->emitError("unsupported elements attribute");
+    }
 
     ShapedType dstAttrType = dstElementsAttr.getType();
 
diff --git a/mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-le-specific.mlir b/mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-le-specific.mlir
new file mode 100644
index 000000000000..7233a8bfffa9
--- /dev/null
+++ b/mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-le-specific.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-opt -split-input-file -convert-arith-to-spirv -verify-diagnostics %s | FileCheck %s
+
+
+//===----------------------------------------------------------------------===//
+// arith.constant dense_resource
+//
+// The decoding of dense_resource differs between little and big endian
+// machines. At the moment only litte endian is supported.
+// See https://github.com/llvm/llvm-project/issues/63469 for more infos.
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: target=s390x-{{.*}}
+
+module attributes {
+  spirv.target_env = #spirv.target_env<
+    #spirv.vce<v1.0, [Int8, Int16, Int64, Float16, Float64], []>, #spirv.resource_limits<>>
+} {
+func.func @constant_dense_resource() {
+  // CHECK:    %{{.*}} = spirv.Constant dense<[0.203224242, -0.254296064, -0.365104556, -0.469196141, 0.466041982]> : tensor<5xf32> : !spirv.array<5 x f32>
+  %0 = arith.constant dense_resource<dense_resource_test_5xf32> : tensor<5xf32>  
+  // CHECK:    %{{.*}} = spirv.Constant dense<[1, 2]> : vector<2xi32>
+  %1 = arith.constant dense_resource<dense_resource_test_2xi32> : vector<2xi32>  
+  // CHECK:    %{{.*}} = spirv.Constant dense<[0.35476172, 0.351080596, -0.0795008316, 0.366843373]> : tensor<4xf32> : !spirv.array<4 x f32>
+  %2 = arith.constant dense_resource<dense_resource_test_2x2xf32> : tensor<1x2x2xf32>  
+  return
+  }
+}
+// Resources are kept at end of file. New tests should be added above this.
+{-#
+  dialect_resources: {
+    builtin: {
+      dense_resource_test_2xi32: "0x400000000100000002000000",
+      dense_resource_test_5xf32: "0x08000000041A503E183382BEFCEEBABE7A3AF0BE0E9DEE3E",
+      dense_resource_test_2x2xf32: "0x0800000054A3B53ED6C0B33E55D1A2BDE5D2BB3E"
+    }
+  }
+#-}
diff --git a/mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-unsupported.mlir b/mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-unsupported.mlir
index 0d92a8e676d8..2512254b443d 100644
--- a/mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-unsupported.mlir
+++ b/mlir/test/Conversion/ArithToSPIRV/arith-to-spirv-unsupported.mlir
@@ -95,6 +95,34 @@ func.func @unsupported_constant_tensor_2xf64_0() {
   return
 }
 
+// -----
+
+func.func @constant_dense_resource_non_existant() {
+  // expected-error @+2 {{failed to legalize operation 'arith.constant'}}
+  // expected-error @+1 {{could not find resource blob}}
+  %0 = arith.constant dense_resource<non_existant> : tensor<5xf32>  
+  return
+}
+
+// -----
+
+module {
+func.func @constant_dense_resource_invalid_buffer() {
+  // expected-error @+2 {{failed to legalize operation 'arith.constant'}}
+  // expected-error @+1 {{resource is not a valid buffer}}
+  %0 = arith.constant dense_resource<dense_resource_test_2xi32> : vector<2xi32>  
+  return
+  }
+}
+// This is a buffer of wrong type and shape
+{-#
+  dialect_resources: {
+    builtin: {
+      dense_resource_test_2xi32: "0x0800000054A3B53ED6C0B33E55D1A2BDE5D2BB3E"
+    }
+  }
+#-}
+
 ///===----------------------------------------------------------------------===//
 // Type emulation
 //===----------------------------------------------------------------------===//
-- 
GitLab


From 9b6f0735fbff4e9a51f0c76f5630be649490f6ef Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 17 May 2024 07:30:58 -0700
Subject: [PATCH 1138/1206] [Support] Drop nop conversions of StringRef to
 StringRef (NFC)

Both sides here are known to be of StringRef.
---
 llvm/lib/Support/CommandLine.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp
index 056340bbab5a..8a00d4798f33 100644
--- a/llvm/lib/Support/CommandLine.cpp
+++ b/llvm/lib/Support/CommandLine.cpp
@@ -534,7 +534,7 @@ SubCommand *CommandLineParser::LookupSubCommand(StringRef Name,
     if (S->getName().empty())
       continue;
 
-    if (StringRef(S->getName()) == StringRef(Name))
+    if (S->getName() == Name)
       return S;
 
     if (!NearestMatch && S->getName().edit_distance(Name) < 2)
-- 
GitLab


From c0cdc1d9569fdb8e3b672befda80cf90db24e814 Mon Sep 17 00:00:00 2001
From: Kelvin Li <kkwli@users.noreply.github.com>
Date: Fri, 17 May 2024 10:49:20 -0400
Subject: [PATCH 1139/1206] [flang] Allow flexible producer name in
 #di_compile_unit (NFC) (#92462)

---
 flang/test/Transforms/debug-line-table-inc-file.fir | 2 +-
 flang/test/Transforms/debug-line-table.fir          | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang/test/Transforms/debug-line-table-inc-file.fir b/flang/test/Transforms/debug-line-table-inc-file.fir
index d7f60a1a86db..9370c138fd42 100644
--- a/flang/test/Transforms/debug-line-table-inc-file.fir
+++ b/flang/test/Transforms/debug-line-table-inc-file.fir
@@ -30,7 +30,7 @@ module attributes {} {
 // CHECK: #[[MODULE_LOC]] = loc("{{.*}}simple.f90":0:0)
 // CHECK: #[[LOC_INC_FILE:.*]] = loc("{{.*}}inc.f90":1:1)
 // CHECK: #[[LOC_FILE:.*]] = loc("{{.*}}simple.f90":3:1)
-// CHECK: #[[DI_CU:.*]] = #llvm.di_compile_unit<id = distinct[{{.*}}]<>, sourceLanguage = DW_LANG_Fortran95, file = #[[DI_FILE]], producer = "flang{{.*}}", isOptimized = false, emissionKind = LineTablesOnly>
+// CHECK: #[[DI_CU:.*]] = #llvm.di_compile_unit<id = distinct[{{.*}}]<>, sourceLanguage = DW_LANG_Fortran95, file = #[[DI_FILE]], producer = "{{.*}}flang{{.*}}", isOptimized = false, emissionKind = LineTablesOnly>
 // CHECK: #[[DI_SP_INC:.*]] = #llvm.di_subprogram<id = distinct[{{.*}}]<>, compileUnit = #[[DI_CU]], scope = #[[DI_FILE]], name = "sinc", linkageName = "_QPsinc", file = #[[DI_INC_FILE]], {{.*}}>
 // CHECK: #[[DI_SP:.*]] = #llvm.di_subprogram<id = distinct[{{.*}}]<>, compileUnit = #[[DI_CU]], scope = #[[DI_FILE]], name = "_QQmain", linkageName = "_QQmain", file = #[[DI_FILE]], {{.*}}>
 // CHECK: #[[FUSED_LOC_INC_FILE]] = loc(fused<#[[DI_SP_INC]]>[#[[LOC_INC_FILE]]])
diff --git a/flang/test/Transforms/debug-line-table.fir b/flang/test/Transforms/debug-line-table.fir
index 3b3a39174df0..8a72ca2a856a 100644
--- a/flang/test/Transforms/debug-line-table.fir
+++ b/flang/test/Transforms/debug-line-table.fir
@@ -24,9 +24,9 @@ module attributes { fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.dat
 // CHECK: #[[MODULE_LOC]] = loc("[[DIR_NAME]]/[[FILE_NAME]]":1:1)
 // CHECK: #[[SB_LOC]] = loc("./simple.f90":2:1)
 // CHECK: #[[DECL_LOC:.*]] = loc("./simple.f90":10:1)
-// FULL: #di_compile_unit = #llvm.di_compile_unit<id = distinct[{{.*}}]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang{{.*}}", isOptimized = false, emissionKind = Full>
-// OPT: #di_compile_unit = #llvm.di_compile_unit<id = distinct[{{.*}}]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang{{.*}}", isOptimized = true, emissionKind = Full>
-// LINETABLE: #di_compile_unit = #llvm.di_compile_unit<id = distinct[{{.*}}]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang{{.*}}", isOptimized = false, emissionKind = LineTablesOnly>
+// FULL: #di_compile_unit = #llvm.di_compile_unit<id = distinct[{{.*}}]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "{{.*}}flang{{.*}}", isOptimized = false, emissionKind = Full>
+// OPT: #di_compile_unit = #llvm.di_compile_unit<id = distinct[{{.*}}]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "{{.*}}flang{{.*}}", isOptimized = true, emissionKind = Full>
+// LINETABLE: #di_compile_unit = #llvm.di_compile_unit<id = distinct[{{.*}}]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "{{.*}}flang{{.*}}", isOptimized = false, emissionKind = LineTablesOnly>
 // CHECK: #di_subroutine_type = #llvm.di_subroutine_type<callingConvention = DW_CC_normal, types = #di_basic_type, #di_basic_type>
 // CHECK: #[[SB_SUBPROGRAM:.*]] = #llvm.di_subprogram<id = distinct[{{.*}}]<>, compileUnit = #di_compile_unit, scope = #di_file, name = "[[SB_NAME]]", linkageName = "[[SB_NAME]]", file = #di_file, line = 1, scopeLine = 1, subprogramFlags = "Definition|Optimized", type = #di_subroutine_type>
 // CHECK: #[[DECL_SUBPROGRAM:.*]] = #llvm.di_subprogram<scope = #di_file, name = "[[DECL_NAME]]", linkageName = "[[DECL_NAME]]", file = #di_file, line = 1, scopeLine = 1, subprogramFlags = Optimized, type = #di_subroutine_type>
-- 
GitLab


From 84aee95124549c5d13e22053af254e3fcc02bc84 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 17 May 2024 18:11:24 +0300
Subject: [PATCH 1140/1206] [clang][NFC] Remove `const_cast` from
 `Preprocessor::addModuleMacro()`

---
 clang/include/clang/Lex/Preprocessor.h | 4 ++--
 clang/lib/Lex/PPLexerChange.cpp        | 2 +-
 clang/lib/Lex/PPMacroExpansion.cpp     | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index e89b4a2c5230..c0850a8fa9f7 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -1010,7 +1010,7 @@ private:
   llvm::FoldingSet<ModuleMacro> ModuleMacros;
 
   /// The names of potential module macros that we've not yet processed.
-  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
+  llvm::SmallVector<IdentifierInfo *, 32> PendingModuleMacroNames;
 
   /// The list of module macros, for each identifier, that are not overridden by
   /// any other module macro.
@@ -1432,7 +1432,7 @@ public:
                                MacroDirective *MD);
 
   /// Register an exported macro for a module and identifier.
-  ModuleMacro *addModuleMacro(Module *Mod, const IdentifierInfo *II,
+  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II,
                               MacroInfo *Macro,
                               ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
   ModuleMacro *getModuleMacro(Module *Mod, const IdentifierInfo *II);
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index 2ca2122ac710..8221db46e06a 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -804,7 +804,7 @@ Module *Preprocessor::LeaveSubmodule(bool ForPragma) {
   llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
   for (unsigned I = Info.OuterPendingModuleMacroNames;
        I != PendingModuleMacroNames.size(); ++I) {
-    const auto *II = PendingModuleMacroNames[I];
+    auto *II = PendingModuleMacroNames[I];
     if (!VisitedMacros.insert(II).second)
       continue;
 
diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
index a478e0badb0c..8af4a97d00cb 100644
--- a/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -129,7 +129,7 @@ void Preprocessor::setLoadedMacroDirective(IdentifierInfo *II,
     II->setHasMacroDefinition(false);
 }
 
-ModuleMacro *Preprocessor::addModuleMacro(Module *Mod, const IdentifierInfo *II,
+ModuleMacro *Preprocessor::addModuleMacro(Module *Mod, IdentifierInfo *II,
                                           MacroInfo *Macro,
                                           ArrayRef<ModuleMacro *> Overrides,
                                           bool &New) {
@@ -162,7 +162,7 @@ ModuleMacro *Preprocessor::addModuleMacro(Module *Mod, const IdentifierInfo *II,
   // The new macro is always a leaf macro.
   LeafMacros.push_back(MM);
   // The identifier now has defined macros (that may or may not be visible).
-  const_cast<IdentifierInfo *>(II)->setHasMacroDefinition(true);
+  II->setHasMacroDefinition(true);
 
   New = true;
   return MM;
-- 
GitLab


From e919df57770d43217cbd4d05f92de8dc7aac5035 Mon Sep 17 00:00:00 2001
From: Christian Ulmann <christianulmann@gmail.com>
Date: Fri, 17 May 2024 17:36:09 +0200
Subject: [PATCH 1141/1206] [MLIR][Transforms] Correct block sorting utils name
 (NFC) (#92558)

This commit renames the name of the block sorting utility function to
`getBlocksSortedByDominance`. A topological order is not defined on a
general directed graph, so the previous name did not make sense.
---
 mlir/include/mlir/Transforms/RegionUtils.h    |  5 +-
 .../OpenACC/OpenACCToLLVMIRTranslation.cpp    |  2 +-
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 54 +++++++++----------
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp  |  2 +-
 mlir/lib/Transforms/Mem2Reg.cpp               |  2 +-
 mlir/lib/Transforms/Utils/RegionUtils.cpp     |  2 +-
 6 files changed, 33 insertions(+), 34 deletions(-)

diff --git a/mlir/include/mlir/Transforms/RegionUtils.h b/mlir/include/mlir/Transforms/RegionUtils.h
index 192ff7138405..f65d0d44eef4 100644
--- a/mlir/include/mlir/Transforms/RegionUtils.h
+++ b/mlir/include/mlir/Transforms/RegionUtils.h
@@ -87,8 +87,9 @@ LogicalResult eraseUnreachableBlocks(RewriterBase &rewriter,
 LogicalResult runRegionDCE(RewriterBase &rewriter,
                            MutableArrayRef<Region> regions);
 
-/// Get a topologically sorted list of blocks of the given region.
-SetVector<Block *> getTopologicallySortedBlocks(Region &region);
+/// Get a list of blocks that is sorted according to dominance. This sort is
+/// stable.
+SetVector<Block *> getBlocksSortedByDominance(Region &region);
 
 } // namespace mlir
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
index b964d1c082b2..eeda245ce969 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
@@ -392,7 +392,7 @@ static LogicalResult convertDataOp(acc::DataOp &op,
   llvm::BasicBlock *endDataBlock = llvm::BasicBlock::Create(
       ctx, "acc.end_data", builder.GetInsertBlock()->getParent());
 
-  SetVector<Block *> blocks = getTopologicallySortedBlocks(op.getRegion());
+  SetVector<Block *> blocks = getBlocksSortedByDominance(op.getRegion());
   for (Block *bb : blocks) {
     llvm::BasicBlock *llvmBB = moduleTranslation.lookupBlock(bb);
     if (bb->isEntryBlock()) {
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 50ee642b5098..34b6903f8da0 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -199,7 +199,7 @@ static llvm::BasicBlock *convertOmpOpRegions(
 
   // Convert blocks one by one in topological order to ensure
   // defs are converted before uses.
-  SetVector<Block *> blocks = getTopologicallySortedBlocks(region);
+  SetVector<Block *> blocks = getBlocksSortedByDominance(region);
   for (Block *bb : blocks) {
     llvm::BasicBlock *llvmBB = moduleTranslation.lookupBlock(bb);
     // Retarget the branch of the entry block to the entry block of the
@@ -2153,40 +2153,38 @@ getFirstOrLastMappedMemberPtr(mlir::omp::MapInfoOp mapInfo, bool first) {
   llvm::SmallVector<size_t> indices(shape[0]);
   std::iota(indices.begin(), indices.end(), 0);
 
-  llvm::sort(
-      indices.begin(), indices.end(), [&](const size_t a, const size_t b) {
-        auto indexValues = indexAttr.getValues<int32_t>();
-        for (int i = 0;
-             i < shape[1];
-             ++i) {
-          int aIndex = indexValues[a * shape[1] + i];
-          int bIndex = indexValues[b * shape[1] + i];
+  llvm::sort(indices.begin(), indices.end(),
+             [&](const size_t a, const size_t b) {
+               auto indexValues = indexAttr.getValues<int32_t>();
+               for (int i = 0; i < shape[1]; ++i) {
+                 int aIndex = indexValues[a * shape[1] + i];
+                 int bIndex = indexValues[b * shape[1] + i];
 
-          if (aIndex == bIndex)
-            continue;
+                 if (aIndex == bIndex)
+                   continue;
 
-          if (aIndex != -1 && bIndex == -1)
-            return false;
+                 if (aIndex != -1 && bIndex == -1)
+                   return false;
 
-          if (aIndex == -1 && bIndex != -1)
-            return true;
+                 if (aIndex == -1 && bIndex != -1)
+                   return true;
 
-          // A is earlier in the record type layout than B
-          if (aIndex < bIndex)
-            return first;
+                 // A is earlier in the record type layout than B
+                 if (aIndex < bIndex)
+                   return first;
 
-          if (bIndex < aIndex)
-            return !first;
-        }
+                 if (bIndex < aIndex)
+                   return !first;
+               }
 
-        // Iterated the entire list and couldn't make a decision, all elements
-        // were likely the same. Return false, since the sort comparator should
-        // return false for equal elements.
-        return false;
-      });
+               // Iterated the entire list and couldn't make a decision, all
+               // elements were likely the same. Return false, since the sort
+               // comparator should return false for equal elements.
+               return false;
+             });
 
-    return llvm::cast<mlir::omp::MapInfoOp>(
-          mapInfo.getMembers()[indices.front()].getDefiningOp());
+  return llvm::cast<mlir::omp::MapInfoOp>(
+      mapInfo.getMembers()[indices.front()].getDefiningOp());
 }
 
 /// This function calculates the array/pointer offset for map data provided
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 2e4b0feb1973..cf3257c8b9b8 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -1331,7 +1331,7 @@ LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
 
   // Then, convert blocks one by one in topological order to ensure defs are
   // converted before uses.
-  auto blocks = getTopologicallySortedBlocks(func.getBody());
+  auto blocks = getBlocksSortedByDominance(func.getBody());
   for (Block *bb : blocks) {
     CapturingIRBuilder builder(llvmContext);
     if (failed(convertBlockImpl(*bb, bb->isEntryBlock(), builder,
diff --git a/mlir/lib/Transforms/Mem2Reg.cpp b/mlir/lib/Transforms/Mem2Reg.cpp
index e096747741c0..e2e240ad865c 100644
--- a/mlir/lib/Transforms/Mem2Reg.cpp
+++ b/mlir/lib/Transforms/Mem2Reg.cpp
@@ -517,7 +517,7 @@ getOrCreateBlockIndices(BlockIndexCache &blockIndexCache, Region *region) {
     return it->second;
 
   DenseMap<Block *, size_t> &blockIndices = it->second;
-  SetVector<Block *> topologicalOrder = getTopologicallySortedBlocks(*region);
+  SetVector<Block *> topologicalOrder = getBlocksSortedByDominance(*region);
   for (auto [index, block] : llvm::enumerate(topologicalOrder))
     blockIndices[block] = index;
   return blockIndices;
diff --git a/mlir/lib/Transforms/Utils/RegionUtils.cpp b/mlir/lib/Transforms/Utils/RegionUtils.cpp
index e25867b527b7..192f59b35329 100644
--- a/mlir/lib/Transforms/Utils/RegionUtils.cpp
+++ b/mlir/lib/Transforms/Utils/RegionUtils.cpp
@@ -837,7 +837,7 @@ LogicalResult mlir::simplifyRegions(RewriterBase &rewriter,
                  mergedIdenticalBlocks);
 }
 
-SetVector<Block *> mlir::getTopologicallySortedBlocks(Region &region) {
+SetVector<Block *> mlir::getBlocksSortedByDominance(Region &region) {
   // For each block that has not been visited yet (i.e. that has no
   // predecessors), add it to the list as well as its successors.
   SetVector<Block *> blocks;
-- 
GitLab


From 9e39a0c723ced1aefdb30cece554356644aac148 Mon Sep 17 00:00:00 2001
From: Christian Ulmann <christian.ulmann@nextsilicon.com>
Date: Fri, 17 May 2024 15:59:04 +0000
Subject: [PATCH 1142/1206] [MLIR][ArmSME] Fix for block sorting refactor

This commit fixes a breakage introduced by changing the name of the
block sorting function.

Related PR: https://github.com/llvm/llvm-project/pull/92558
---
 mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
index 1e1e0e569124..acbbbe9932e1 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/TileAllocation.cpp
@@ -333,7 +333,7 @@ DenseMap<Operation *, unsigned>
 generateOperationNumbering(FunctionOpInterface function) {
   unsigned index = 0;
   SetVector<Block *> blocks =
-      getTopologicallySortedBlocks(function.getFunctionBody());
+      getBlocksSortedByDominance(function.getFunctionBody());
   DenseMap<Operation *, unsigned> operationToIndexMap;
   for (Block *block : blocks) {
     index++; // We want block args to have their own number.
-- 
GitLab


From 621d0f3e86ecc46c2df302041777e42dfc10171e Mon Sep 17 00:00:00 2001
From: Mital Ashok <mital@mitalashok.co.uk>
Date: Fri, 17 May 2024 17:00:43 +0100
Subject: [PATCH 1143/1206] [Clang][Sema] Fix last argument not being used when
 comparing function template specializations when one has an explicit object
 argument (#92263)

Fixes #92188
---
 clang/docs/ReleaseNotes.rst                |  2 +
 clang/lib/Sema/SemaTemplateDeduction.cpp   |  9 +++-
 clang/test/SemaCXX/cxx2b-deducing-this.cpp | 57 ++++++++++++++++++++++
 3 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 85327576548c..7af5869d2176 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -737,6 +737,8 @@ Bug Fixes to C++ Support
   templates during partial ordering when deducing template arguments from a function declaration or when
   taking the address of a function template.
 - Fix a bug with checking constrained non-type template parameters for equivalence. Fixes (#GH77377).
+- Fix a bug where the last argument was not considered when considering the most viable function for
+  explicit object argument member functions. Fixes (#GH92188).
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index d0b780c28eae..41fd210f29d0 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -5638,7 +5638,8 @@ static bool isAtLeastAsSpecializedAs(Sema &S, SourceLocation Loc,
 /// function templates.
 ///
 /// \param NumCallArguments1 The number of arguments in the call to FT1, used
-/// only when \c TPOC is \c TPOC_Call.
+/// only when \c TPOC is \c TPOC_Call. Does not include the object argument when
+/// calling a member function.
 ///
 /// \param RawObj1Ty The type of the object parameter of FT1 if a member
 /// function only used if \c TPOC is \c TPOC_Call and FT1 is a Function
@@ -5707,7 +5708,11 @@ FunctionTemplateDecl *Sema::getMoreSpecializedTemplate(
                                               IsRValRef1);
       Args2.push_back(Obj2Ty);
     }
-    size_t NumComparedArguments = NumCallArguments1 + ShouldConvert1;
+    size_t NumComparedArguments = NumCallArguments1;
+    // Either added an argument above or the prototype includes an explicit
+    // object argument we need to count
+    if (Method1)
+      ++NumComparedArguments;
 
     Args1.insert(Args1.end(), Proto1->param_type_begin(),
                  Proto1->param_type_end());
diff --git a/clang/test/SemaCXX/cxx2b-deducing-this.cpp b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
index aa64530bd5be..cdb9d1324b97 100644
--- a/clang/test/SemaCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
@@ -836,3 +836,60 @@ int h() {
   }();
 }
 }
+
+namespace GH92188 {
+struct A {
+  template<auto N>
+  void operator+=(this auto &&, const char (&)[N]);
+  void operator+=(this auto &&, auto &&) = delete;
+
+  void f1(this A &, auto &);
+  void f1(this A &, auto &&) = delete;
+
+  void f2(this auto&);
+  void f2(this auto&&) = delete;
+
+  void f3(auto&) &;
+  void f3(this A&, auto&&) = delete;
+
+  void f4(auto&&) & = delete;
+  void f4(this A&, auto&);
+
+  static void f5(auto&);
+  void f5(this A&, auto&&) = delete;
+
+  static void f6(auto&&) = delete;
+  void f6(this A&, auto&);
+
+  void implicit_this() {
+    int lval;
+    operator+=("123");
+    f1(lval);
+    f2();
+    f3(lval);
+    f4(lval);
+    f5(lval);
+    f6(lval);
+  }
+
+  void operator-(this A&, auto&&) = delete;
+  friend void operator-(A&, auto&);
+
+  void operator*(this A&, auto&);
+  friend void operator*(A&, auto&&) = delete;
+};
+
+void g() {
+  A a;
+  int lval;
+  a += "123";
+  a.f1(lval);
+  a.f2();
+  a.f3(lval);
+  a.f4(lval);
+  a.f5(lval);
+  a.f6(lval);
+  a - lval;
+  a * lval;
+}
+}
-- 
GitLab


From 8aa6511f4209bba33a74c4ef6e208fda5c0f3d27 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 17 May 2024 19:10:04 +0300
Subject: [PATCH 1144/1206] [clang][NFC] Remove an unnecessary `const_cast`

---
 clang/lib/Sema/Sema.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index c9bc46414c7d..6c548f374cbe 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -2462,7 +2462,7 @@ bool Sema::tryExprAsCall(Expr &E, QualType &ZeroArgCallReturnTy,
   const OverloadExpr *Overloads = nullptr;
   bool IsMemExpr = false;
   if (E.getType() == Context.OverloadTy) {
-    OverloadExpr::FindResult FR = OverloadExpr::find(const_cast<Expr*>(&E));
+    OverloadExpr::FindResult FR = OverloadExpr::find(&E);
 
     // Ignore overloads that are pointer-to-member constants.
     if (FR.HasFormOfMemberPointer)
-- 
GitLab


From e01ff8238cf62c7149de7b8046bccec9adefbe67 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Fri, 17 May 2024 17:13:59 +0100
Subject: [PATCH 1145/1206] [mlir][vector] Fix scalability issues in drop
 innermost unit dims transfer patterns (#92402)

Previously, these rewrites would drop scalable dimensions and treated
`[1]` (scalable one dim) as a unit dimension. This patch propagates
scalable dimensions and ensures `[1]` is not treated as a unit
dimension.
---
 .../Vector/Transforms/VectorTransforms.cpp    | 13 +++--
 ...tor-transfer-collapse-inner-most-dims.mlir | 56 +++++++++++++++++++
 2 files changed, 65 insertions(+), 4 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index 69c497264fd1..f29eba90c3ce 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -1237,6 +1237,10 @@ getTransferFoldableInnerUnitDims(MemRefType srcType, VectorType vectorType) {
   if (failed(getStridesAndOffset(srcType, srcStrides, srcOffset)))
     return failure();
 
+  auto isUnitDim = [](VectorType type, int dim) {
+    return type.getDimSize(dim) == 1 && !type.getScalableDims()[dim];
+  };
+
   // According to vector.transfer_read/write semantics, the vector can be a
   // slice. Thus, we have to offset the check index with `rankDiff` in
   // `srcStrides` and source dim sizes.
@@ -1247,8 +1251,7 @@ getTransferFoldableInnerUnitDims(MemRefType srcType, VectorType vectorType) {
     // It can be folded only if they are 1 and the stride is 1.
     int dim = vectorType.getRank() - i - 1;
     if (srcStrides[dim + rankDiff] != 1 ||
-        srcType.getDimSize(dim + rankDiff) != 1 ||
-        vectorType.getDimSize(dim) != 1)
+        srcType.getDimSize(dim + rankDiff) != 1 || !isUnitDim(vectorType, dim))
       break;
     result++;
   }
@@ -1292,7 +1295,8 @@ class DropInnerMostUnitDimsTransferRead
 
     auto resultTargetVecType =
         VectorType::get(targetType.getShape().drop_back(dimsToDrop),
-                        targetType.getElementType());
+                        targetType.getElementType(),
+                        targetType.getScalableDims().drop_back(dimsToDrop));
 
     auto loc = readOp.getLoc();
     SmallVector<OpFoldResult> sizes =
@@ -1378,7 +1382,8 @@ class DropInnerMostUnitDimsTransferWrite
 
     auto resultTargetVecType =
         VectorType::get(targetType.getShape().drop_back(dimsToDrop),
-                        targetType.getElementType());
+                        targetType.getElementType(),
+                        targetType.getScalableDims().drop_back(dimsToDrop));
 
     Location loc = writeOp.getLoc();
     SmallVector<OpFoldResult> sizes =
diff --git a/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir b/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir
index 477755b66c02..b4cb640108ba 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-collapse-inner-most-dims.mlir
@@ -174,3 +174,59 @@ func.func @non_unit_strides(%arg0: memref<512x16x1xf32, strided<[8192, 16, 4], o
 // The inner most unit dims can not be dropped if the strides are not ones.
 // CHECK:     func.func @non_unit_strides
 // CHECK-NOT:   memref.subview
+
+// -----
+
+func.func @leading_scalable_dimension_transfer_read(%dest : memref<24x1xf32>) -> vector<[4]x1xf32> {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f32
+  %0 = vector.transfer_read %dest[%c0, %c0], %pad {in_bounds = [true, true]} : memref<24x1xf32>, vector<[4]x1xf32>
+  return %0 : vector<[4]x1xf32>
+}
+// CHECK:      func.func @leading_scalable_dimension_transfer_read
+// CHECK-SAME:   %[[DEST:[a-zA-Z0-9]+]]
+// CHECK:        %[[SUBVIEW:.+]] = memref.subview %[[DEST]][0, 0] [24, 1] [1, 1] : memref<24x1xf32> to memref<24xf32, strided<[1]>>
+// CHECK:        %[[READ:.+]] = vector.transfer_read %[[SUBVIEW]]{{.*}} {in_bounds = [true]} : memref<24xf32, strided<[1]>>, vector<[4]xf32>
+// CHECK:        %[[CAST:.+]] = vector.shape_cast %[[READ]] : vector<[4]xf32> to vector<[4]x1xf32>
+// CHECK:        return %[[CAST]]
+
+// -----
+
+// Negative test: [1] (scalable 1) is _not_ a unit dimension.
+func.func @trailing_scalable_one_dim_transfer_read(%dest : memref<24x1xf32>) -> vector<4x[1]xf32> {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f32
+  %0 = vector.transfer_read %dest[%c0, %c0], %pad {in_bounds = [true, true]} : memref<24x1xf32>, vector<4x[1]xf32>
+  return %0 : vector<4x[1]xf32>
+}
+// CHECK:      func.func @trailing_scalable_one_dim_transfer_read
+// CHECK-NOT:    vector.shape_cast
+// CHECK:        vector.transfer_read {{.*}} : memref<24x1xf32>, vector<4x[1]xf32>
+// CHECK-NOT:    vector.shape_cast
+
+// -----
+
+func.func @leading_scalable_dimension_transfer_write(%dest : memref<24x1xf32>, %vec: vector<[4]x1xf32>) {
+  %c0 = arith.constant 0 : index
+  vector.transfer_write %vec, %dest[%c0, %c0] {in_bounds = [true, true]} : vector<[4]x1xf32>,  memref<24x1xf32>
+  return
+}
+// CHECK:      func.func @leading_scalable_dimension_transfer_write
+// CHECK-SAME:   %[[DEST:[a-zA-Z0-9]+]]
+// CHECK-SAME:   %[[VEC:[a-zA-Z0-9]+]]
+// CHECK:        %[[SUBVIEW:.+]] = memref.subview %[[DEST]][0, 0] [24, 1] [1, 1] : memref<24x1xf32> to memref<24xf32, strided<[1]>>
+// CHECK:        %[[CAST:.+]] = vector.shape_cast %[[VEC]] : vector<[4]x1xf32> to vector<[4]xf32>
+// CHECK:        vector.transfer_write %[[CAST]], %[[SUBVIEW]]{{.*}} {in_bounds = [true]} : vector<[4]xf32>, memref<24xf32, strided<[1]>>
+
+// -----
+
+// Negative test: [1] (scalable 1) is _not_ a unit dimension.
+func.func @trailing_scalable_one_dim_transfer_write(%dest : memref<24x1xf32>, %vec: vector<4x[1]xf32>, %index: index) {
+  %c0 = arith.constant 0 : index
+  vector.transfer_write %vec, %dest[%index, %c0] {in_bounds = [true, true]} : vector<4x[1]xf32>,  memref<24x1xf32>
+  return
+}
+// CHECK:      func.func @trailing_scalable_one_dim_transfer_write
+// CHECK-NOT:    vector.shape_cast
+// CHECK:        vector.transfer_write {{.*}} : vector<4x[1]xf32>,  memref<24x1xf32>
+// CHECK-NOT:    vector.shape_cast
-- 
GitLab


From d38ea8c4c84be9496249098053599c24b87f1376 Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Fri, 17 May 2024 20:14:09 +0400
Subject: [PATCH 1146/1206] [lldb] Fixed the test TestGdbRemoteAttachWait
 running on a remote target (#92413)

Install `_exe_to_attach` to a remote target if necessary.
---
 .../tools/lldb-server/attach-wait/TestGdbRemoteAttachWait.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lldb/test/API/tools/lldb-server/attach-wait/TestGdbRemoteAttachWait.py b/lldb/test/API/tools/lldb-server/attach-wait/TestGdbRemoteAttachWait.py
index f4c31fe2f5c0..84aab9c969aa 100644
--- a/lldb/test/API/tools/lldb-server/attach-wait/TestGdbRemoteAttachWait.py
+++ b/lldb/test/API/tools/lldb-server/attach-wait/TestGdbRemoteAttachWait.py
@@ -17,7 +17,10 @@ class TestGdbRemoteAttachWait(gdbremote_testcase.GdbRemoteTestCaseBase):
             # Use a shim to ensure that the process is ready to be attached from
             # the get-go.
             self._exe_to_run = "shim"
-            self._run_args = [self.getBuildArtifact(self._exe_to_attach)]
+            self._exe_to_attach = lldbutil.install_to_target(
+                self, self.getBuildArtifact(self._exe_to_attach)
+            )
+            self._run_args = [self._exe_to_attach]
             self.build(dictionary={"EXE": self._exe_to_run, "CXX_SOURCES": "shim.cpp"})
         else:
             self._exe_to_run = self._exe_to_attach
-- 
GitLab


From d90159add4140d6f4994f5767fa6ef31cc28897b Mon Sep 17 00:00:00 2001
From: Walter Erquinigo <a20012251@gmail.com>
Date: Fri, 17 May 2024 18:32:59 +0200
Subject: [PATCH 1147/1206] [LSP] Use std::string for storing the Reply method
 (#92464)

This was using a StringRef, which is very unsafe because the method name
might just get disposed due to the async nature of the response.
This was causing weird characters being printed in the output logs.
---
 mlir/lib/Tools/lsp-server-support/Transport.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Tools/lsp-server-support/Transport.cpp b/mlir/lib/Tools/lsp-server-support/Transport.cpp
index 1e90ab32281f..ca7ffdf78d3a 100644
--- a/mlir/lib/Tools/lsp-server-support/Transport.cpp
+++ b/mlir/lib/Tools/lsp-server-support/Transport.cpp
@@ -41,7 +41,7 @@ public:
   void operator()(llvm::Expected<llvm::json::Value> reply);
 
 private:
-  StringRef method;
+  std::string method;
   std::atomic<bool> replied = {false};
   llvm::json::Value id;
   JSONTransport *transport;
-- 
GitLab


From 45daa4fdc68f5faa5bd5c33da052d2415cd88540 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 17 May 2024 09:37:53 -0700
Subject: [PATCH 1148/1206] [flang][cuda] Move CUDA Fortran operations to a CUF
 dialect (#92317)

The number of operations dedicated to CUF grew and where all still in
FIR. In order to have a better organization, the CUF operations,
attributes and code is moved into their specific dialect and files. CUF
dialect is tightly coupled with HLFIR/FIR and their types.

The CUF attributes are bundled into their own library since some
HLFIR/FIR operations depend on them and the CUF dialect depends on the
FIR types. Without having the attributes into a separate library there
would be a dependency cycle.
---
 flang/include/flang/Lower/ConvertVariable.h   |  10 +-
 .../flang/Optimizer/Builder/FIRBuilder.h      |   4 +-
 .../flang/Optimizer/Builder/HLFIRTools.h      |   2 +-
 .../flang/Optimizer/Dialect/CMakeLists.txt    |   2 +
 .../Dialect/CUF/Attributes/CMakeLists.txt     |   7 +
 .../Dialect/CUF/Attributes/CUFAttr.h          | 106 +++++++
 .../Dialect/CUF/Attributes/CUFAttr.td         | 100 +++++++
 .../Optimizer/Dialect/CUF/CMakeLists.txt      |  11 +
 .../flang/Optimizer/Dialect/CUF/CUFDialect.h  |  26 ++
 .../flang/Optimizer/Dialect/CUF/CUFDialect.td |  43 +++
 .../flang/Optimizer/Dialect/CUF/CUFOps.h      |  20 ++
 .../flang/Optimizer/Dialect/CUF/CUFOps.td     | 263 ++++++++++++++++++
 .../flang/Optimizer/Dialect/FIRAttr.td        |  83 ------
 .../include/flang/Optimizer/Dialect/FIROps.h  |   1 +
 .../include/flang/Optimizer/Dialect/FIROps.td | 242 +---------------
 .../flang/Optimizer/Dialect/FIROpsSupport.h   |  13 -
 .../include/flang/Optimizer/HLFIR/HLFIROps.td |   5 +-
 .../include/flang/Optimizer/Support/InitFIR.h |   3 +-
 flang/include/flang/Optimizer/Support/Utils.h |  61 +---
 flang/lib/Frontend/CMakeLists.txt             |   2 +
 flang/lib/Lower/Allocatable.cpp               |  17 +-
 flang/lib/Lower/Bridge.cpp                    |  52 ++--
 flang/lib/Lower/CMakeLists.txt                |   4 +
 flang/lib/Lower/CallInterface.cpp             |  21 +-
 flang/lib/Lower/ConvertCall.cpp               |   3 +-
 flang/lib/Lower/ConvertVariable.cpp           |  57 ++--
 flang/lib/Optimizer/Builder/FIRBuilder.cpp    |   8 +-
 flang/lib/Optimizer/Builder/HLFIRTools.cpp    |   4 +-
 flang/lib/Optimizer/Dialect/CMakeLists.txt    |   3 +
 .../Dialect/CUF/Attributes/CMakeLists.txt     |  16 ++
 .../Dialect/CUF/Attributes/CUFAttr.cpp        |  32 +++
 .../lib/Optimizer/Dialect/CUF/CMakeLists.txt  |  22 ++
 .../lib/Optimizer/Dialect/CUF/CUFDialect.cpp  |  25 ++
 flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp    | 219 +++++++++++++++
 flang/lib/Optimizer/Dialect/FIRAttr.cpp       |   4 +-
 flang/lib/Optimizer/Dialect/FIRDialect.cpp    |   1 +
 flang/lib/Optimizer/Dialect/FIROps.cpp        | 163 -----------
 flang/lib/Optimizer/HLFIR/IR/CMakeLists.txt   |   2 +
 flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp     |   4 +-
 .../Optimizer/HLFIR/Transforms/CMakeLists.txt |   2 +
 .../HLFIR/Transforms/ConvertToFIR.cpp         |   8 +-
 flang/test/Fir/cuf-invalid.fir                |  50 ++--
 flang/test/Fir/cuf.mlir                       |  46 +--
 flang/test/Lower/CUDA/cuda-allocatable.cuf    |  48 ++--
 flang/test/Lower/CUDA/cuda-data-attribute.cuf |  60 ++--
 flang/test/Lower/CUDA/cuda-data-transfer.cuf  |  50 ++--
 flang/test/Lower/CUDA/cuda-kernel-calls.cuf   |  12 +-
 .../Lower/CUDA/cuda-kernel-loop-directive.cuf |  10 +-
 flang/test/Lower/CUDA/cuda-mod.cuf            |   4 +-
 flang/test/Lower/CUDA/cuda-module-use.cuf     |   8 +-
 flang/test/Lower/CUDA/cuda-proc-attribute.cuf |  26 +-
 flang/tools/bbc/CMakeLists.txt                |   2 +
 flang/tools/fir-opt/CMakeLists.txt            |   2 +
 flang/tools/tco/CMakeLists.txt                |   2 +
 flang/unittests/Optimizer/CMakeLists.txt      |   2 +
 .../Optimizer/FortranVariableTest.cpp         |   8 +-
 56 files changed, 1184 insertions(+), 817 deletions(-)
 create mode 100644 flang/include/flang/Optimizer/Dialect/CUF/Attributes/CMakeLists.txt
 create mode 100644 flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h
 create mode 100644 flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td
 create mode 100644 flang/include/flang/Optimizer/Dialect/CUF/CMakeLists.txt
 create mode 100644 flang/include/flang/Optimizer/Dialect/CUF/CUFDialect.h
 create mode 100644 flang/include/flang/Optimizer/Dialect/CUF/CUFDialect.td
 create mode 100644 flang/include/flang/Optimizer/Dialect/CUF/CUFOps.h
 create mode 100644 flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
 create mode 100644 flang/lib/Optimizer/Dialect/CUF/Attributes/CMakeLists.txt
 create mode 100644 flang/lib/Optimizer/Dialect/CUF/Attributes/CUFAttr.cpp
 create mode 100644 flang/lib/Optimizer/Dialect/CUF/CMakeLists.txt
 create mode 100644 flang/lib/Optimizer/Dialect/CUF/CUFDialect.cpp
 create mode 100644 flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp

diff --git a/flang/include/flang/Lower/ConvertVariable.h b/flang/include/flang/Lower/ConvertVariable.h
index d70d3268acac..515f4695951b 100644
--- a/flang/include/flang/Lower/ConvertVariable.h
+++ b/flang/include/flang/Lower/ConvertVariable.h
@@ -23,6 +23,10 @@
 #include "mlir/IR/Value.h"
 #include "llvm/ADT/DenseMap.h"
 
+namespace cuf {
+class DataAttributeAttr;
+}
+
 namespace fir {
 class ExtendedValue;
 class FirOpBuilder;
@@ -146,9 +150,9 @@ translateSymbolAttributes(mlir::MLIRContext *mlirContext,
 
 /// Translate the CUDA Fortran attributes of \p sym into the FIR CUDA attribute
 /// representation.
-fir::CUDADataAttributeAttr
-translateSymbolCUDADataAttribute(mlir::MLIRContext *mlirContext,
-                                 const Fortran::semantics::Symbol &sym);
+cuf::DataAttributeAttr
+translateSymbolCUFDataAttribute(mlir::MLIRContext *mlirContext,
+                                const Fortran::semantics::Symbol &sym);
 
 /// Map a symbol to a given fir::ExtendedValue. This will generate an
 /// hlfir.declare when lowering to HLFIR and map the hlfir.declare result to the
diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index 0d650f830b64..287730ef2ac8 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -254,13 +254,13 @@ public:
                              mlir::StringAttr linkage = {},
                              mlir::Attribute value = {}, bool isConst = false,
                              bool isTarget = false,
-                             fir::CUDADataAttributeAttr cudaAttr = {});
+                             cuf::DataAttributeAttr dataAttr = {});
 
   fir::GlobalOp createGlobal(mlir::Location loc, mlir::Type type,
                              llvm::StringRef name, bool isConst, bool isTarget,
                              std::function<void(FirOpBuilder &)> bodyBuilder,
                              mlir::StringAttr linkage = {},
-                             fir::CUDADataAttributeAttr cudaAttr = {});
+                             cuf::DataAttributeAttr dataAttr = {});
 
   /// Create a global constant (read-only) value.
   fir::GlobalOp createGlobalConstant(mlir::Location loc, mlir::Type type,
diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index 6cc8e71b3b18..43aa1661550e 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -239,7 +239,7 @@ genDeclare(mlir::Location loc, fir::FirOpBuilder &builder,
            const fir::ExtendedValue &exv, llvm::StringRef name,
            fir::FortranVariableFlagsAttr flags,
            mlir::Value dummyScope = nullptr,
-           fir::CUDADataAttributeAttr cudaAttr = {});
+           cuf::DataAttributeAttr dataAttr = {});
 
 /// Generate an hlfir.associate to build a variable from an expression value.
 /// The type of the variable must be provided so that scalar logicals are
diff --git a/flang/include/flang/Optimizer/Dialect/CMakeLists.txt b/flang/include/flang/Optimizer/Dialect/CMakeLists.txt
index f00993d4d377..301a93c1fe5b 100644
--- a/flang/include/flang/Optimizer/Dialect/CMakeLists.txt
+++ b/flang/include/flang/Optimizer/Dialect/CMakeLists.txt
@@ -1,3 +1,5 @@
+add_subdirectory(CUF)
+
 # This replicates part of the add_mlir_dialect cmake function from MLIR that
 # cannot be used her because it expects to be run inside MLIR directory which
 # is not the case for FIR.
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CMakeLists.txt b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CMakeLists.txt
new file mode 100644
index 000000000000..bae7fe3484f4
--- /dev/null
+++ b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CMakeLists.txt
@@ -0,0 +1,7 @@
+set(LLVM_TARGET_DEFINITIONS CUFAttr.td)
+mlir_tablegen(CUFEnumAttr.h.inc -gen-enum-decls)
+mlir_tablegen(CUFEnumAttr.cpp.inc -gen-enum-defs)
+mlir_tablegen(CUFAttr.h.inc --gen-attrdef-decls)
+mlir_tablegen(CUFAttr.cpp.inc -gen-attrdef-defs)
+
+add_public_tablegen_target(CUFAttrsIncGen)
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h
new file mode 100644
index 000000000000..f32e39b543e3
--- /dev/null
+++ b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h
@@ -0,0 +1,106 @@
+//===-- Optimizer/Dialect/CUF/Attributes/CUFAttr.h -- CUF attributes ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_OPTIMIZER_DIALECT_CUF_CUFATTR_H
+#define FORTRAN_OPTIMIZER_DIALECT_CUF_CUFATTR_H
+
+#include "flang/Common/Fortran.h"
+#include "mlir/IR/BuiltinAttributes.h"
+
+namespace llvm {
+class StringRef;
+}
+
+#include "flang/Optimizer/Dialect/CUF/Attributes/CUFEnumAttr.h.inc"
+
+#define GET_ATTRDEF_CLASSES
+#include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h.inc"
+
+namespace cuf {
+
+/// Attribute to mark Fortran entities with the CUDA attribute.
+static constexpr llvm::StringRef getDataAttrName() { return "cuf.data_attr"; }
+static constexpr llvm::StringRef getProcAttrName() { return "cuf.proc_attr"; }
+
+/// Attribute to carry CUDA launch_bounds values.
+static constexpr llvm::StringRef getLaunchBoundsAttrName() {
+  return "cuf.launch_bounds";
+}
+
+/// Attribute to carry CUDA cluster_dims values.
+static constexpr llvm::StringRef getClusterDimsAttrName() {
+  return "cuf.cluster_dims";
+}
+
+inline cuf::DataAttributeAttr
+getDataAttribute(mlir::MLIRContext *mlirContext,
+                 std::optional<Fortran::common::CUDADataAttr> cudaAttr) {
+  if (cudaAttr) {
+    cuf::DataAttribute attr;
+    switch (*cudaAttr) {
+    case Fortran::common::CUDADataAttr::Constant:
+      attr = cuf::DataAttribute::Constant;
+      break;
+    case Fortran::common::CUDADataAttr::Device:
+      attr = cuf::DataAttribute::Device;
+      break;
+    case Fortran::common::CUDADataAttr::Managed:
+      attr = cuf::DataAttribute::Managed;
+      break;
+    case Fortran::common::CUDADataAttr::Pinned:
+      attr = cuf::DataAttribute::Pinned;
+      break;
+    case Fortran::common::CUDADataAttr::Shared:
+      attr = cuf::DataAttribute::Shared;
+      break;
+    case Fortran::common::CUDADataAttr::Texture:
+      // Obsolete attribute
+      return {};
+    case Fortran::common::CUDADataAttr::Unified:
+      attr = cuf::DataAttribute::Unified;
+      break;
+    }
+    return cuf::DataAttributeAttr::get(mlirContext, attr);
+  }
+  return {};
+}
+
+inline cuf::ProcAttributeAttr
+getProcAttribute(mlir::MLIRContext *mlirContext,
+                 std::optional<Fortran::common::CUDASubprogramAttrs> cudaAttr) {
+  if (cudaAttr) {
+    cuf::ProcAttribute attr;
+    switch (*cudaAttr) {
+    case Fortran::common::CUDASubprogramAttrs::Host:
+      attr = cuf::ProcAttribute::Host;
+      break;
+    case Fortran::common::CUDASubprogramAttrs::Device:
+      attr = cuf::ProcAttribute::Device;
+      break;
+    case Fortran::common::CUDASubprogramAttrs::HostDevice:
+      attr = cuf::ProcAttribute::HostDevice;
+      break;
+    case Fortran::common::CUDASubprogramAttrs::Global:
+      attr = cuf::ProcAttribute::Global;
+      break;
+    case Fortran::common::CUDASubprogramAttrs::Grid_Global:
+      attr = cuf::ProcAttribute::GridGlobal;
+      break;
+    }
+    return cuf::ProcAttributeAttr::get(mlirContext, attr);
+  }
+  return {};
+}
+
+} // namespace cuf
+
+#endif // FORTRAN_OPTIMIZER_DIALECT_CUF_CUFATTR_H
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td
new file mode 100644
index 000000000000..8e2b54672527
--- /dev/null
+++ b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td
@@ -0,0 +1,100 @@
+//===- CUFAttr.td - CUF Attributes -------------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the CUF dialect attributes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_DIALECT_CUF_CUFATTRS
+#define FORTRAN_DIALECT_CUF_CUFATTRS
+
+include "flang/Optimizer/Dialect/CUF/CUFDialect.td"
+include "mlir/IR/EnumAttr.td"
+
+class cuf_Attr<string name> : AttrDef<CUFDialect, name>;
+
+def cuf_DataAttribute : I32EnumAttr<
+    "DataAttribute",
+    "CUDA Fortran variable attributes",
+    [
+      I32EnumAttrCase<"Constant", 0, "constant">,
+      I32EnumAttrCase<"Device", 1, "device">,
+      I32EnumAttrCase<"Managed", 2, "managed">,
+      I32EnumAttrCase<"Pinned", 3, "pinned">,
+      I32EnumAttrCase<"Shared", 4, "shared">,
+      I32EnumAttrCase<"Unified", 5, "unified">,
+      // Texture is omitted since it is obsolete and rejected by semantic.
+    ]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::cuf";
+}
+
+def cuf_DataAttributeAttr :
+    EnumAttr<CUFDialect, cuf_DataAttribute, "cuda"> {
+  let assemblyFormat = [{ ```<` $value `>` }];
+}
+
+def cuf_ProcAttribute : I32EnumAttr<
+    "ProcAttribute", "CUDA Fortran procedure attributes",
+    [
+      I32EnumAttrCase<"Host", 0, "host">,
+      I32EnumAttrCase<"Device", 1, "device">,
+      I32EnumAttrCase<"HostDevice", 2, "host_device">,
+      I32EnumAttrCase<"Global", 3, "global">,
+      I32EnumAttrCase<"GridGlobal", 4, "grid_global">,
+    ]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::cuf";
+}
+
+def cuf_ProcAttributeAttr :
+    EnumAttr<CUFDialect, cuf_ProcAttribute, "cuda_proc"> {
+  let assemblyFormat = [{ ```<` $value `>` }];
+}
+
+def cuf_LaunchBoundsAttr : cuf_Attr<"LaunchBounds"> {
+  let mnemonic = "launch_bounds";
+
+  let parameters = (ins
+    "mlir::IntegerAttr":$maxTPB,
+    "mlir::IntegerAttr":$minBPM,
+    OptionalParameter<"mlir::IntegerAttr">:$upperBoundClusterSize
+  );
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def cuf_ClusterDimsAttr : cuf_Attr<"ClusterDims"> {
+  let mnemonic = "cluster_dims";
+
+  let parameters = (ins
+    "mlir::IntegerAttr":$x,
+    "mlir::IntegerAttr":$y,
+    "mlir::IntegerAttr":$z
+  );
+
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def cuf_DataTransferKind : I32EnumAttr<
+    "DataTransferKind", "CUDA Fortran data transfer kind",
+    [
+      I32EnumAttrCase<"DeviceHost", 0, "device_host">,
+      I32EnumAttrCase<"HostDevice", 1, "host_device">,
+      I32EnumAttrCase<"DeviceDevice", 2, "device_device">,
+    ]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::cuf";
+}
+
+def cuf_DataTransferKindAttr :
+    EnumAttr<CUFDialect, cuf_DataTransferKind, "cuda_transfer"> {
+  let assemblyFormat = [{ ```<` $value `>` }];
+}
+
+#endif // FORTRAN_DIALECT_CUF_CUFATTRS
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/CMakeLists.txt b/flang/include/flang/Optimizer/Dialect/CUF/CMakeLists.txt
new file mode 100644
index 000000000000..07490c7b9ca2
--- /dev/null
+++ b/flang/include/flang/Optimizer/Dialect/CUF/CMakeLists.txt
@@ -0,0 +1,11 @@
+add_subdirectory(Attributes)
+
+set(LLVM_TARGET_DEFINITIONS CUFDialect.td)
+mlir_tablegen(CUFDialect.h.inc -gen-dialect-decls -dialect=cuf)
+mlir_tablegen(CUFDialect.cpp.inc -gen-dialect-defs -dialect=cuf)
+
+set(LLVM_TARGET_DEFINITIONS CUFOps.td)
+mlir_tablegen(CUFOps.h.inc -gen-op-decls)
+mlir_tablegen(CUFOps.cpp.inc -gen-op-defs)
+
+add_public_tablegen_target(CUFOpsIncGen)
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/CUFDialect.h b/flang/include/flang/Optimizer/Dialect/CUF/CUFDialect.h
new file mode 100644
index 000000000000..cf562b226835
--- /dev/null
+++ b/flang/include/flang/Optimizer/Dialect/CUF/CUFDialect.h
@@ -0,0 +1,26 @@
+//===-- Optimizer/Dialect/CUFDialect.h -- CUF dialect -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_OPTIMIZER_DIALECT_CUF_CUFDIALECT_H
+#define FORTRAN_OPTIMIZER_DIALECT_CUF_CUFDIALECT_H
+
+#include "mlir/Bytecode/BytecodeOpInterface.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/SymbolTable.h"
+#include "mlir/Interfaces/CallInterfaces.h"
+#include "mlir/Interfaces/FunctionInterfaces.h"
+#include "mlir/Interfaces/LoopLikeInterface.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
+
+#include "flang/Optimizer/Dialect/CUF/CUFDialect.h.inc"
+
+#endif // FORTRAN_OPTIMIZER_DIALECT_CUF_CUFDIALECT_H
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/CUFDialect.td b/flang/include/flang/Optimizer/Dialect/CUF/CUFDialect.td
new file mode 100644
index 000000000000..df866e566406
--- /dev/null
+++ b/flang/include/flang/Optimizer/Dialect/CUF/CUFDialect.td
@@ -0,0 +1,43 @@
+//===-- CUFDialect.td - CUF dialect base definitions -------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Definition of the CUDA Fortran dialect
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_DIALECT_CUF_CUFDIALECT
+#define FORTRAN_DIALECT_CUF_CUFDIALECT
+
+include "mlir/IR/AttrTypeBase.td"
+include "mlir/IR/EnumAttr.td"
+include "mlir/IR/OpBase.td"
+
+def CUFDialect : Dialect {
+  let name = "cuf";
+
+  let summary = "CUDA Fortran dialect";
+
+  let description = [{
+    This dialect models CUDA Fortran operations. The CUF dialect operations use
+    the FIR types and are tightly coupled with FIR and HLFIR.
+  }];
+
+  let useDefaultAttributePrinterParser = 1;
+  let usePropertiesForAttributes = 1;
+  let cppNamespace = "::cuf";
+  let dependentDialects = ["fir::FIROpsDialect"];
+
+  let extraClassDeclaration = [{
+  private:
+    // Register the CUF Attributes.
+    void registerAttributes();
+  }];
+}
+
+#endif // FORTRAN_DIALECT_CUF_CUFDIALECT
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.h b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.h
new file mode 100644
index 000000000000..4132db672e39
--- /dev/null
+++ b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.h
@@ -0,0 +1,20 @@
+//===-- Optimizer/Dialect/CUF/CUFOps.h - CUF operations ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_OPTIMIZER_DIALECT_CUF_CUFOPS_H
+#define FORTRAN_OPTIMIZER_DIALECT_CUF_CUFOPS_H
+
+#include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h"
+#include "flang/Optimizer/Dialect/CUF/CUFDialect.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "mlir/IR/OpDefinition.h"
+
+#define GET_OP_CLASSES
+#include "flang/Optimizer/Dialect/CUF/CUFOps.h.inc"
+
+#endif // FORTRAN_OPTIMIZER_DIALECT_CUF_CUFOPS_H
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
new file mode 100644
index 000000000000..72157bce4f76
--- /dev/null
+++ b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
@@ -0,0 +1,263 @@
+//===-- CUFOps.td - CUF operation definitions --------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Definition of the CUF dialect operations
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_DIALECT_CUF_CUF_OPS
+#define FORTRAN_DIALECT_CUF_CUF_OPS
+
+include "flang/Optimizer/Dialect/CUF/CUFDialect.td"
+include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td"
+include "flang/Optimizer/Dialect/FIRTypes.td"
+include "mlir/Interfaces/LoopLikeInterface.td"
+include "mlir/IR/BuiltinAttributes.td"
+
+class cuf_Op<string mnemonic, list<Trait> traits>
+    : Op<CUFDialect, mnemonic, traits>;
+
+def cuf_AllocOp : cuf_Op<"alloc", [AttrSizedOperandSegments,
+    MemoryEffects<[MemAlloc]>]> {
+  let summary = "Allocate an object on device";
+
+  let description = [{
+    This is a drop in replacement for fir.alloca and fir.allocmem for device
+    object. Any device, managed or unified object declared in an host
+    subprogram needs to be allocated in the device memory through runtime calls.
+    The cuf.alloc is an abstraction to the runtime calls and works together
+    with cuf.free.
+  }];
+
+  let arguments = (ins
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<AnyIntegerType>:$typeparams,
+    Variadic<AnyIntegerType>:$shape,
+    cuf_DataAttributeAttr:$data_attr
+  );
+
+  let results = (outs fir_ReferenceType:$ptr);
+
+  let assemblyFormat = [{
+    $in_type (`(` $typeparams^ `:` type($typeparams) `)`)?
+        (`,` $shape^ `:` type($shape) )?  attr-dict `->` qualified(type($ptr))
+  }];
+
+  let builders = [
+    OpBuilder<(ins "mlir::Type":$inType, "llvm::StringRef":$uniqName,
+      "llvm::StringRef":$bindcName,
+      "cuf::DataAttributeAttr":$cudaAttr,
+      CArg<"mlir::ValueRange", "{}">:$typeparams,
+      CArg<"mlir::ValueRange", "{}">:$shape,
+      CArg<"llvm::ArrayRef<mlir::NamedAttribute>", "{}">:$attributes)>];
+
+  let hasVerifier = 1;
+}
+
+def cuf_FreeOp : cuf_Op<"free", [MemoryEffects<[MemFree]>]> {
+  let summary = "Free a device allocated object";
+
+  let description = [{
+    The cuf.free operation frees the memory allocated by cuf.alloc.
+    This is used for non-allocatable device, managed and unified device
+    variables declare in host subprogram.
+  }];
+
+  let arguments = (ins
+    Arg<AnyReferenceLike, "", [MemFree]>:$devptr,
+    cuf_DataAttributeAttr:$data_attr
+  );
+
+  let assemblyFormat = "$devptr `:` qualified(type($devptr)) attr-dict";
+
+  let hasVerifier = 1;
+}
+
+def cuf_AllocateOp : cuf_Op<"allocate", [AttrSizedOperandSegments,
+    MemoryEffects<[MemAlloc<DefaultResource>]>]> {
+  let summary = "Perform the device allocation of data of an allocatable";
+
+  let description = [{
+    The cuf.allocate operation performs the allocation on the device
+    of the data of an allocatable. The descriptor passed to the operation
+    is initialized before with the standard flang runtime calls.
+  }];
+
+  let arguments = (ins Arg<fir_ReferenceType, "", [MemRead, MemWrite]>:$box,
+                       Arg<Optional<AnyRefOrBoxType>, "", [MemWrite]>:$errmsg,
+                       Optional<AnyIntegerType>:$stream,
+                       Arg<Optional<AnyRefOrBoxType>, "", [MemWrite]>:$pinned,
+                       Arg<Optional<AnyRefOrBoxType>, "", [MemRead]>:$source,
+                       cuf_DataAttributeAttr:$data_attr,
+                       UnitAttr:$hasStat);
+
+  let results = (outs AnyIntegerType:$stat);
+
+  let assemblyFormat = [{
+    $box `:` qualified(type($box))
+    ( `source` `(` $source^ `:` qualified(type($source) )`)` )?
+    ( `errmsg` `(` $errmsg^ `:` type($errmsg) `)` )?
+    ( `stream` `(` $stream^ `:` type($stream) `)` )?
+    ( `pinned` `(` $pinned^ `:` type($pinned) `)` )?
+    attr-dict `->` type($stat)
+  }];
+
+  let hasVerifier = 1;
+}
+
+def cuf_DeallocateOp : cuf_Op<"deallocate",
+    [MemoryEffects<[MemFree<DefaultResource>]>]> {
+  let summary = "Perform the device deallocation of data of an allocatable";
+
+  let description = [{
+    The cuf.deallocate operation performs the deallocation on the device
+    of the data of an allocatable.
+  }];
+
+  let arguments = (ins Arg<fir_ReferenceType, "", [MemRead, MemWrite]>:$box,
+                       Arg<Optional<AnyRefOrBoxType>, "", [MemWrite]>:$errmsg,
+                       cuf_DataAttributeAttr:$data_attr,
+                       UnitAttr:$hasStat);
+
+  let results = (outs AnyIntegerType:$stat);
+
+  let assemblyFormat = [{
+    $box `:` qualified(type($box))
+    ( `errmsg` `(` $errmsg^ `:` type($errmsg) `)` )?
+    attr-dict `->` type($stat)
+  }];
+
+  let hasVerifier = 1;
+}
+
+def cuf_DataTransferOp : cuf_Op<"data_transfer", []> {
+  let summary = "Represent a data transfer between host and device memory";
+
+  let description = [{
+    CUDA Fortran allows data transfer to be done via intrinsic assignment
+    between a host and a device variable. This operation is used to materialized
+    the data transfer between the lhs and rhs memory references.
+    The kind of transfer is specified in the attribute. 
+
+    ```
+      adev = a ! transfer host to device
+      a = adev ! transfer device to host
+      bdev = adev ! transfer device to device
+    ```
+  }];
+
+  let arguments = (ins Arg<AnyReferenceLike, "", [MemWrite]>:$src,
+                       Arg<AnyReferenceLike, "", [MemRead]>:$dst,
+                       cuf_DataTransferKindAttr:$transfer_kind);
+
+  let assemblyFormat = [{
+    $src `to` $dst attr-dict `:` type(operands)
+  }];
+}
+
+def cuf_KernelLaunchOp : cuf_Op<"kernel_launch", [CallOpInterface,
+    AttrSizedOperandSegments]> {
+  let summary = "call CUDA kernel";
+
+  let description = [{
+    Launch a CUDA kernel from the host.
+
+    ```
+      // launch simple kernel with no arguments. bytes and stream value are
+      // optional in the chevron notation.
+      cuf.kernel_launch @kernel<<<%gx, %gy, %bx, %by, %bz>>>()
+    ```
+  }];
+
+  let arguments = (ins
+    SymbolRefAttr:$callee,
+    I32:$grid_x,
+    I32:$grid_y,
+    I32:$grid_z,
+    I32:$block_x,
+    I32:$block_y,
+    I32:$block_z,
+    Optional<I32>:$bytes,
+    Optional<I32>:$stream,
+    Variadic<AnyType>:$args
+  );
+
+  let assemblyFormat = [{
+    $callee `<` `<` `<` $grid_x `,` $grid_y `,` $grid_z `,`$block_x `,`
+        $block_y `,` $block_z ( `,` $bytes^ ( `,` $stream^ )? )? `>` `>` `>`
+        `` `(` $args `)` ( `:` `(` type($args)^ `)` )? attr-dict
+  }];
+
+  let extraClassDeclaration = [{
+    mlir::CallInterfaceCallable getCallableForCallee() {
+      return getCalleeAttr();
+    }
+
+    void setCalleeFromCallable(mlir::CallInterfaceCallable callee) {
+      (*this)->setAttr(getCalleeAttrName(), callee.get<mlir::SymbolRefAttr>());
+    }
+    mlir::FunctionType getFunctionType();
+
+    unsigned getNbNoArgOperand() {
+      unsigned nbNoArgOperand = 5; // grids and blocks values are always present.
+      if (getBytes()) ++nbNoArgOperand;
+      if (getStream()) ++nbNoArgOperand;
+      return nbNoArgOperand;
+    }
+
+    operand_range getArgOperands() {
+      return {operand_begin() + getNbNoArgOperand(), operand_end()};
+    }
+    mlir::MutableOperandRange getArgOperandsMutable() {
+      return mlir::MutableOperandRange(
+          *this, getNbNoArgOperand(), getArgs().size() - 1);
+    }
+  }];
+}
+
+def cuf_KernelOp : cuf_Op<"kernel", [AttrSizedOperandSegments,
+    DeclareOpInterfaceMethods<LoopLikeOpInterface>]> {
+
+  let description = [{
+    Represent the CUDA Fortran kernel directive. The operation is a loop like
+    operation that represents the iteration range of the embedded loop nest.
+
+    When grid or block variadic operands are empty, a `*` only syntax was used
+    in the Fortran code.
+    If the `*` is mixed with values for either grid or block, these are
+    represented by a 0 constant value.
+  }];
+
+  let arguments = (ins
+    Variadic<I32>:$grid, // empty means `*`
+    Variadic<I32>:$block, // empty means `*`
+    Optional<I32>:$stream,
+    Variadic<Index>:$lowerbound,
+    Variadic<Index>:$upperbound,
+    Variadic<Index>:$step,
+    OptionalAttr<I64Attr>:$n
+  );
+
+  let regions = (region AnyRegion:$region);
+
+  let assemblyFormat = [{
+    `<` `<` `<` custom<CUFKernelValues>($grid, type($grid)) `,` 
+                custom<CUFKernelValues>($block, type($block))
+        ( `,` `stream` `=` $stream^ )? `>` `>` `>`
+        custom<CUFKernelLoopControl>($region, $lowerbound, type($lowerbound),
+            $upperbound, type($upperbound), $step, type($step))
+        attr-dict
+  }];
+
+  let hasVerifier = 1;
+}
+
+#endif // FORTRAN_DIALECT_CUF_CUF_OPS
diff --git a/flang/include/flang/Optimizer/Dialect/FIRAttr.td b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
index f8b3fb861cc6..989319ff3dda 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRAttr.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
@@ -70,87 +70,4 @@ def fir_BoxFieldAttr : I32EnumAttr<
 // mlir::SideEffects::Resource for modelling operations which add debugging information
 def DebuggingResource : Resource<"::fir::DebuggingResource">;
 
-//===----------------------------------------------------------------------===//
-// CUDA Fortran specific attributes
-//===----------------------------------------------------------------------===//
-
-def fir_CUDADataAttribute : I32EnumAttr<
-    "CUDADataAttribute",
-    "CUDA Fortran variable attributes",
-    [
-      I32EnumAttrCase<"Constant", 0, "constant">,
-      I32EnumAttrCase<"Device", 1, "device">,
-      I32EnumAttrCase<"Managed", 2, "managed">,
-      I32EnumAttrCase<"Pinned", 3, "pinned">,
-      I32EnumAttrCase<"Shared", 4, "shared">,
-      I32EnumAttrCase<"Unified", 5, "unified">,
-      // Texture is omitted since it is obsolete and rejected by semantic.
-    ]> {
-  let genSpecializedAttr = 0;
-  let cppNamespace = "::fir";
-}
-
-def fir_CUDADataAttributeAttr :
-    EnumAttr<FIROpsDialect, fir_CUDADataAttribute, "cuda"> {
-  let assemblyFormat = [{ ```<` $value `>` }];
-}
-
-def fir_CUDAProcAttribute : I32EnumAttr<
-    "CUDAProcAttribute", "CUDA Fortran procedure attributes",
-    [
-      I32EnumAttrCase<"Host", 0, "host">,
-      I32EnumAttrCase<"Device", 1, "device">,
-      I32EnumAttrCase<"HostDevice", 2, "host_device">,
-      I32EnumAttrCase<"Global", 3, "global">,
-      I32EnumAttrCase<"GridGlobal", 4, "grid_global">,
-    ]> {
-  let genSpecializedAttr = 0;
-  let cppNamespace = "::fir";
-}
-
-def fir_CUDAProcAttributeAttr :
-    EnumAttr<FIROpsDialect, fir_CUDAProcAttribute, "cuda_proc"> {
-  let assemblyFormat = [{ ```<` $value `>` }];
-}
-
-def fir_CUDALaunchBoundsAttr : fir_Attr<"CUDALaunchBounds"> {
-  let mnemonic = "launch_bounds";
-
-  let parameters = (ins
-    "mlir::IntegerAttr":$maxTPB,
-    "mlir::IntegerAttr":$minBPM,
-    OptionalParameter<"mlir::IntegerAttr">:$upperBoundClusterSize
-  );
-
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
-def fir_CUDAClusterDimsAttr : fir_Attr<"CUDAClusterDims"> {
-  let mnemonic = "cluster_dims";
-
-  let parameters = (ins
-    "mlir::IntegerAttr":$x,
-    "mlir::IntegerAttr":$y,
-    "mlir::IntegerAttr":$z
-  );
-
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
-def fir_CUDADataTransferKind : I32EnumAttr<
-    "CUDADataTransferKind", "CUDA Fortran data transfer kind",
-    [
-      I32EnumAttrCase<"DeviceHost", 0, "device_host">,
-      I32EnumAttrCase<"HostDevice", 1, "host_device">,
-      I32EnumAttrCase<"DeviceDevice", 2, "device_device">,
-    ]> {
-  let genSpecializedAttr = 0;
-  let cppNamespace = "::fir";
-}
-
-def fir_CUDADataTransferKindAttr :
-    EnumAttr<FIROpsDialect, fir_CUDADataTransferKind, "cuda_transfer"> {
-  let assemblyFormat = [{ ```<` $value `>` }];
-}
-
 #endif // FIR_DIALECT_FIR_ATTRS
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.h b/flang/include/flang/Optimizer/Dialect/FIROps.h
index 016ad0433ed8..9f07364ddb62 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.h
@@ -9,6 +9,7 @@
 #ifndef FORTRAN_OPTIMIZER_DIALECT_FIROPS_H
 #define FORTRAN_OPTIMIZER_DIALECT_FIROPS_H
 
+#include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h"
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Dialect/FirAliasTagOpInterface.h"
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 64c5e360b28f..d9c114904006 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -17,6 +17,7 @@
 include "mlir/Dialect/Arith/IR/ArithBase.td"
 include "mlir/Dialect/Arith/IR/ArithOpsInterfaces.td"
 include "mlir/Dialect/LLVMIR/LLVMAttrDefs.td"
+include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td"
 include "flang/Optimizer/Dialect/FIRDialect.td"
 include "flang/Optimizer/Dialect/FIRTypes.td"
 include "flang/Optimizer/Dialect/FIRAttr.td"
@@ -2436,66 +2437,6 @@ def fir_DispatchOp : fir_Op<"dispatch", []> {
   }];
 }
 
-def fir_CUDAKernelLaunch : fir_Op<"cuda_kernel_launch", [CallOpInterface,
-    AttrSizedOperandSegments]> {
-  let summary = "call CUDA kernel";
-
-  let description = [{
-    Launch a CUDA kernel from the host.
-
-    ```
-      // launch simple kernel with no arguments. bytes and stream value are
-      // optional in the chevron notation.
-      fir.cuda_kernel_launch @kernel<<<%gx, %gy, %bx, %by, %bz>>>()
-    ```
-  }];
-
-  let arguments = (ins
-    SymbolRefAttr:$callee,
-    I32:$grid_x,
-    I32:$grid_y,
-    I32:$grid_z,
-    I32:$block_x,
-    I32:$block_y,
-    I32:$block_z,
-    Optional<I32>:$bytes,
-    Optional<I32>:$stream,
-    Variadic<AnyType>:$args
-  );
-
-  let assemblyFormat = [{
-    $callee `<` `<` `<` $grid_x `,` $grid_y `,` $grid_z `,`$block_x `,`
-        $block_y `,` $block_z ( `,` $bytes^ ( `,` $stream^ )? )? `>` `>` `>`
-        `` `(` $args `)` ( `:` `(` type($args)^ `)` )? attr-dict
-  }];
-
-  let extraClassDeclaration = [{
-    mlir::CallInterfaceCallable getCallableForCallee() {
-      return getCalleeAttr();
-    }
-
-    void setCalleeFromCallable(mlir::CallInterfaceCallable callee) {
-      (*this)->setAttr(getCalleeAttrName(), callee.get<mlir::SymbolRefAttr>());
-    }
-    mlir::FunctionType getFunctionType();
-
-    unsigned getNbNoArgOperand() {
-      unsigned nbNoArgOperand = 5; // grids and blocks values are always present.
-      if (getBytes()) ++nbNoArgOperand;
-      if (getStream()) ++nbNoArgOperand;
-      return nbNoArgOperand;
-    }
-
-    operand_range getArgOperands() {
-      return {operand_begin() + getNbNoArgOperand(), operand_end()};
-    }
-    mlir::MutableOperandRange getArgOperandsMutable() {
-      return mlir::MutableOperandRange(
-          *this, getNbNoArgOperand(), getArgs().size() - 1);
-    }
-  }];
-}
-
 // Constant operations that support Fortran
 
 def fir_StringLitOp : fir_Op<"string_lit", [NoMemoryEffect]> {
@@ -2797,7 +2738,7 @@ def fir_GlobalOp : fir_Op<"global", [IsolatedFromAbove, Symbol]> {
     OptionalAttr<UnitAttr>:$constant,
     OptionalAttr<UnitAttr>:$target,
     OptionalAttr<StrAttr>:$linkName,
-    OptionalAttr<fir_CUDADataAttributeAttr>:$cuda_attr
+    OptionalAttr<cuf_DataAttributeAttr>:$data_attr
   );
 
   let regions = (region AtMostRegion<1>:$region);
@@ -3077,7 +3018,7 @@ def fir_DeclareOp : fir_Op<"declare", [AttrSizedOperandSegments,
     Optional<fir_DummyScopeType>:$dummy_scope,
     Builtin_StringAttr:$uniq_name,
     OptionalAttr<fir_FortranVariableFlagsAttr>:$fortran_attrs,
-    OptionalAttr<fir_CUDADataAttributeAttr>:$cuda_attr
+    OptionalAttr<cuf_DataAttributeAttr>:$data_attr
   );
 
   let results = (outs AnyRefOrBox);
@@ -3130,125 +3071,6 @@ def fir_BoxOffsetOp : fir_Op<"box_offset", [NoMemoryEffect]> {
   ];
 }
 
-def fir_CUDAKernelOp : fir_Op<"cuda_kernel", [AttrSizedOperandSegments,
-    DeclareOpInterfaceMethods<LoopLikeOpInterface>]> {
-
-  let description = [{
-    Represent the CUDA Fortran kernel directive. The operation is a loop like
-    operation that represents the iteration range of the embedded loop nest.
-
-    When grid or block variadic operands are empty, a `*` only syntax was used
-    in the Fortran code.
-    If the `*` is mixed with values for either grid or block, these are
-    represented by a 0 constant value.
-  }];
-
-  let arguments = (ins
-    Variadic<I32>:$grid, // empty means `*`
-    Variadic<I32>:$block, // empty means `*`
-    Optional<I32>:$stream,
-    Variadic<Index>:$lowerbound,
-    Variadic<Index>:$upperbound,
-    Variadic<Index>:$step,
-    OptionalAttr<I64Attr>:$n
-  );
-
-  let regions = (region AnyRegion:$region);
-
-  let assemblyFormat = [{
-    `<` `<` `<` custom<CUFKernelValues>($grid, type($grid)) `,` 
-                custom<CUFKernelValues>($block, type($block))
-        ( `,` `stream` `=` $stream^ )? `>` `>` `>`
-        custom<CUFKernelLoopControl>($region, $lowerbound, type($lowerbound),
-            $upperbound, type($upperbound), $step, type($step))
-        attr-dict
-  }];
-
-  let hasVerifier = 1;
-}
-
-def fir_CUDADataTransferOp : fir_Op<"cuda_data_transfer", []> {
-  let summary = "Represent a data transfer between host and device memory";
-
-  let description = [{
-    CUDA Fortran allows data transfer to be done via intrinsic assignment
-    between a host and a device variable. This operation is used to materialized
-    the data transfer between the lhs and rhs memory references.
-    The kind of transfer is specified in the attribute. 
-
-    ```
-      adev = a ! transfer host to device
-      a = adev ! transfer device to host
-      bdev = adev ! transfer device to device
-    ```
-  }];
-
-  let arguments = (ins Arg<AnyReferenceLike, "", [MemWrite]>:$src,
-                       Arg<AnyReferenceLike, "", [MemRead]>:$dst,
-                       fir_CUDADataTransferKindAttr:$transfer_kind);
-
-  let assemblyFormat = [{
-    $src `to` $dst attr-dict `:` type(operands)
-  }];
-}
-
-def fir_CUDAAllocateOp : fir_Op<"cuda_allocate", [AttrSizedOperandSegments,
-    MemoryEffects<[MemAlloc<DefaultResource>]>]> {
-  let summary = "Perform the device allocation of data of an allocatable";
-
-  let description = [{
-    The fir.cuda_allocate operation performs the allocation on the device
-    of the data of an allocatable. The descriptor passed to the operation
-    is initialized before with the standard flang runtime calls.
-  }];
-
-  let arguments = (ins Arg<fir_ReferenceType, "", [MemRead, MemWrite]>:$box,
-                       Arg<Optional<AnyRefOrBoxType>, "", [MemWrite]>:$errmsg,
-                       Optional<AnyIntegerType>:$stream,
-                       Arg<Optional<AnyRefOrBoxType>, "", [MemWrite]>:$pinned,
-                       Arg<Optional<AnyRefOrBoxType>, "", [MemRead]>:$source,
-                       fir_CUDADataAttributeAttr:$cuda_attr,
-                       UnitAttr:$hasStat);
-
-  let results = (outs AnyIntegerType:$stat);
-
-  let assemblyFormat = [{
-    $box `:` qualified(type($box))
-    ( `source` `(` $source^ `:` qualified(type($source) )`)` )?
-    ( `errmsg` `(` $errmsg^ `:` type($errmsg) `)` )?
-    ( `stream` `(` $stream^ `:` type($stream) `)` )?
-    ( `pinned` `(` $pinned^ `:` type($pinned) `)` )?
-    attr-dict `->` type($stat)
-  }];
-
-  let hasVerifier = 1;
-}
-
-def fir_CUDADeallocateOp : fir_Op<"cuda_deallocate",
-    [MemoryEffects<[MemFree<DefaultResource>]>]> {
-  let summary = "Perform the device deallocation of data of an allocatable";
-
-  let description = [{
-    The fir.cuda_deallocate operation performs the deallocation on the device
-    of the data of an allocatable.
-  }];
-
-  let arguments = (ins Arg<fir_ReferenceType, "", [MemRead, MemWrite]>:$box,
-                       Arg<Optional<AnyRefOrBoxType>, "", [MemWrite]>:$errmsg,
-                       fir_CUDADataAttributeAttr:$cuda_attr,
-                       UnitAttr:$hasStat);
-
-  let results = (outs AnyIntegerType:$stat);
-
-  let assemblyFormat = [{
-    $box `:` qualified(type($box))
-    ( `errmsg` `(` $errmsg^ `:` type($errmsg) `)` )?
-    attr-dict `->` type($stat)
-  }];
-
-  let hasVerifier = 1;
-}
-
 def fir_DummyScopeOp : fir_Op<"dummy_scope",
     [MemoryEffects<[MemWrite<DebuggingResource>]>]> {
   let summary = "Define a scope for dummy arguments";
@@ -3329,62 +3151,4 @@ def fir_DummyScopeOp : fir_Op<"dummy_scope",
   let assemblyFormat = "attr-dict `:` type(results)";
 }
 
-def fir_CUDAAllocOp : fir_Op<"cuda_alloc", [AttrSizedOperandSegments,
-    MemoryEffects<[MemAlloc]>]> {
-  let summary = "Allocate an object on device";
-
-  let description = [{
-    This is a drop in replacement for fir.alloca and fir.allocmem for device
-    object. Any device, managed or unified object declared in an host
-    subprogram needs to be allocated in the device memory through runtime calls.
-    The fir.cuda_alloc is an abstraction to the runtime calls and works together
-    with fir.cuda_free.
-  }];
-
-  let arguments = (ins
-    TypeAttr:$in_type,
-    OptionalAttr<StrAttr>:$uniq_name,
-    OptionalAttr<StrAttr>:$bindc_name,
-    Variadic<AnyIntegerType>:$typeparams,
-    Variadic<AnyIntegerType>:$shape,
-    fir_CUDADataAttributeAttr:$cuda_attr
-  );
-
-  let results = (outs fir_ReferenceType:$ptr);
-
-  let assemblyFormat = [{
-    $in_type (`(` $typeparams^ `:` type($typeparams) `)`)?
-        (`,` $shape^ `:` type($shape) )?  attr-dict `->` qualified(type($ptr))
-  }];
-
-  let builders = [
-    OpBuilder<(ins "mlir::Type":$inType, "llvm::StringRef":$uniqName,
-      "llvm::StringRef":$bindcName,
-      "fir::CUDADataAttributeAttr":$cudaAttr,
-      CArg<"mlir::ValueRange", "{}">:$typeparams,
-      CArg<"mlir::ValueRange", "{}">:$shape,
-      CArg<"llvm::ArrayRef<mlir::NamedAttribute>", "{}">:$attributes)>];
-
-  let hasVerifier = 1;
-}
-
-def fir_CUDAFreeOp : fir_Op<"cuda_free", [MemoryEffects<[MemFree]>]> {
-  let summary = "Free a device allocated object";
-
-  let description = [{
-    The fir.cuda_free operation frees the memory allocated by fir.cuda_alloc.
-    This is used for non-allocatable device, managed and unified device
-    variables declare in host subprogram.
-  }];
-
-  let arguments = (ins
-    Arg<AnyReferenceLike, "", [MemFree]>:$devptr,
-    fir_CUDADataAttributeAttr:$cuda_attr
-  );
-
-  let assemblyFormat = "$devptr `:` qualified(type($devptr)) attr-dict";
-
-  let hasVerifier = 1;
-}
-
 #endif
diff --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
index 46b62d8de8d3..b68a39bf374b 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
@@ -75,19 +75,6 @@ constexpr llvm::StringRef getOptionalAttrName() { return "fir.optional"; }
 /// Attribute to mark Fortran entities with the TARGET attribute.
 static constexpr llvm::StringRef getTargetAttrName() { return "fir.target"; }
 
-/// Attribute to mark Fortran entities with the CUDA attribute.
-static constexpr llvm::StringRef getCUDAAttrName() { return "fir.cuda_attr"; }
-
-/// Attribute to carry CUDA launch_bounds values.
-static constexpr llvm::StringRef getCUDALaunchBoundsAttrName() {
-  return "fir.cuda_launch_bounds";
-}
-
-/// Attribute to carry CUDA cluster_dims values.
-static constexpr llvm::StringRef getCUDAClusterDimsAttrName() {
-  return "fir.cuda_cluster_dims";
-}
-
 /// Attribute to mark that a function argument is a character dummy procedure.
 /// Character dummy procedure have special ABI constraints.
 static constexpr llvm::StringRef getCharacterProcedureDummyAttrName() {
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
index 376417e3c353..b537d9e11ef8 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
@@ -15,6 +15,7 @@
 #define FORTRAN_DIALECT_HLFIR_OPS
 
 include "flang/Optimizer/HLFIR/HLFIROpBase.td"
+include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.td"
 include "flang/Optimizer/Dialect/FIRTypes.td"
 include "flang/Optimizer/Dialect/FIRAttr.td"
 include "flang/Optimizer/Dialect/FortranVariableInterface.td"
@@ -90,7 +91,7 @@ def hlfir_DeclareOp : hlfir_Op<"declare", [AttrSizedOperandSegments,
     Optional<fir_DummyScopeType>:$dummy_scope,
     Builtin_StringAttr:$uniq_name,
     OptionalAttr<fir_FortranVariableFlagsAttr>:$fortran_attrs,
-    OptionalAttr<fir_CUDADataAttributeAttr>:$cuda_attr
+    OptionalAttr<cuf_DataAttributeAttr>:$data_attr
   );
 
   let results = (outs AnyFortranVariable, AnyRefOrBoxLike);
@@ -106,7 +107,7 @@ def hlfir_DeclareOp : hlfir_Op<"declare", [AttrSizedOperandSegments,
       CArg<"mlir::Value", "{}">:$shape, CArg<"mlir::ValueRange", "{}">:$typeparams,
       CArg<"mlir::Value", "{}">:$dummy_scope,
       CArg<"fir::FortranVariableFlagsAttr", "{}">:$fortran_attrs,
-      CArg<"fir::CUDADataAttributeAttr", "{}">:$cuda_attr)>];
+      CArg<"cuf::DataAttributeAttr", "{}">:$data_attr)>];
 
   let extraClassDeclaration = [{
     /// Get the variable original base (same as input). It lacks
diff --git a/flang/include/flang/Optimizer/Support/InitFIR.h b/flang/include/flang/Optimizer/Support/InitFIR.h
index 9f4c4ed28a4a..48cc1cbc6456 100644
--- a/flang/include/flang/Optimizer/Support/InitFIR.h
+++ b/flang/include/flang/Optimizer/Support/InitFIR.h
@@ -13,6 +13,7 @@
 #ifndef FORTRAN_OPTIMIZER_SUPPORT_INITFIR_H
 #define FORTRAN_OPTIMIZER_SUPPORT_INITFIR_H
 
+#include "flang/Optimizer/Dialect/CUF/CUFDialect.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/HLFIR/HLFIRDialect.h"
 #include "mlir/Conversion/Passes.h"
@@ -34,7 +35,7 @@ namespace fir::support {
       mlir::scf::SCFDialect, mlir::arith::ArithDialect,                        \
       mlir::cf::ControlFlowDialect, mlir::func::FuncDialect,                   \
       mlir::vector::VectorDialect, mlir::math::MathDialect,                    \
-      mlir::complex::ComplexDialect, mlir::DLTIDialect
+      mlir::complex::ComplexDialect, mlir::DLTIDialect, cuf::CUFDialect
 
 #define FLANG_CODEGEN_DIALECT_LIST FIRCodeGenDialect, mlir::LLVM::LLVMDialect
 
diff --git a/flang/include/flang/Optimizer/Support/Utils.h b/flang/include/flang/Optimizer/Support/Utils.h
index 2da6f24da40e..d8bcb5fae034 100644
--- a/flang/include/flang/Optimizer/Support/Utils.h
+++ b/flang/include/flang/Optimizer/Support/Utils.h
@@ -16,6 +16,7 @@
 #include "flang/Common/default-kinds.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Support/FatalError.h"
@@ -136,66 +137,6 @@ inline void intrinsicTypeTODO(fir::FirOpBuilder &builder, mlir::Type type,
            " in " + intrinsicName);
 }
 
-inline fir::CUDADataAttributeAttr
-getCUDADataAttribute(mlir::MLIRContext *mlirContext,
-                     std::optional<Fortran::common::CUDADataAttr> cudaAttr) {
-  if (cudaAttr) {
-    fir::CUDADataAttribute attr;
-    switch (*cudaAttr) {
-    case Fortran::common::CUDADataAttr::Constant:
-      attr = fir::CUDADataAttribute::Constant;
-      break;
-    case Fortran::common::CUDADataAttr::Device:
-      attr = fir::CUDADataAttribute::Device;
-      break;
-    case Fortran::common::CUDADataAttr::Managed:
-      attr = fir::CUDADataAttribute::Managed;
-      break;
-    case Fortran::common::CUDADataAttr::Pinned:
-      attr = fir::CUDADataAttribute::Pinned;
-      break;
-    case Fortran::common::CUDADataAttr::Shared:
-      attr = fir::CUDADataAttribute::Shared;
-      break;
-    case Fortran::common::CUDADataAttr::Texture:
-      // Obsolete attribute
-      return {};
-    case Fortran::common::CUDADataAttr::Unified:
-      attr = fir::CUDADataAttribute::Unified;
-      break;
-    }
-    return fir::CUDADataAttributeAttr::get(mlirContext, attr);
-  }
-  return {};
-}
-
-inline fir::CUDAProcAttributeAttr getCUDAProcAttribute(
-    mlir::MLIRContext *mlirContext,
-    std::optional<Fortran::common::CUDASubprogramAttrs> cudaAttr) {
-  if (cudaAttr) {
-    fir::CUDAProcAttribute attr;
-    switch (*cudaAttr) {
-    case Fortran::common::CUDASubprogramAttrs::Host:
-      attr = fir::CUDAProcAttribute::Host;
-      break;
-    case Fortran::common::CUDASubprogramAttrs::Device:
-      attr = fir::CUDAProcAttribute::Device;
-      break;
-    case Fortran::common::CUDASubprogramAttrs::HostDevice:
-      attr = fir::CUDAProcAttribute::HostDevice;
-      break;
-    case Fortran::common::CUDASubprogramAttrs::Global:
-      attr = fir::CUDAProcAttribute::Global;
-      break;
-    case Fortran::common::CUDASubprogramAttrs::Grid_Global:
-      attr = fir::CUDAProcAttribute::GridGlobal;
-      break;
-    }
-    return fir::CUDAProcAttributeAttr::get(mlirContext, attr);
-  }
-  return {};
-}
-
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_SUPPORT_UTILS_H
diff --git a/flang/lib/Frontend/CMakeLists.txt b/flang/lib/Frontend/CMakeLists.txt
index a701c264bc4c..f85665d11429 100644
--- a/flang/lib/Frontend/CMakeLists.txt
+++ b/flang/lib/Frontend/CMakeLists.txt
@@ -14,6 +14,7 @@ add_flang_library(flangFrontend
   TextDiagnostic.cpp
 
   DEPENDS
+  CUFDialect
   FIRDialect
   FIROptCodeGenPassIncGen
   FIROptTransformsPassIncGen
@@ -23,6 +24,7 @@ add_flang_library(flangFrontend
   ${extension_libs}
 
   LINK_LIBS
+  CUFDialect
   FortranParser
   FortranSemantics
   FortranEvaluate
diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp
index a1957c0eb1bb..61f4bbd856a8 100644
--- a/flang/lib/Lower/Allocatable.cpp
+++ b/flang/lib/Lower/Allocatable.cpp
@@ -24,6 +24,7 @@
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/Runtime/RTBuilder.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/Dialect/CUF/CUFOps.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/Support/FatalError.h"
@@ -729,9 +730,9 @@ private:
                               ErrorManager &errorManager,
                               const Fortran::semantics::Symbol &sym) {
     Fortran::lower::StatementContext stmtCtx;
-    fir::CUDADataAttributeAttr cudaAttr =
-        Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(),
-                                                         sym);
+    cuf::DataAttributeAttr cudaAttr =
+        Fortran::lower::translateSymbolCUFDataAttribute(builder.getContext(),
+                                                        sym);
     mlir::Value errmsg = errMsgExpr ? errorManager.errMsgAddr : nullptr;
     mlir::Value stream =
         streamExpr
@@ -746,7 +747,7 @@ private:
     // Keep return type the same as a standard AllocatableAllocate call.
     mlir::Type retTy = fir::runtime::getModel<int>()(builder.getContext());
     return builder
-        .create<fir::CUDAAllocateOp>(
+        .create<cuf::AllocateOp>(
             loc, retTy, box.getAddr(), errmsg, stream, pinned, source, cudaAttr,
             errorManager.hasStatSpec() ? builder.getUnitAttr() : nullptr)
         .getResult();
@@ -804,9 +805,9 @@ static mlir::Value genCudaDeallocate(fir::FirOpBuilder &builder,
                                      const fir::MutableBoxValue &box,
                                      ErrorManager &errorManager,
                                      const Fortran::semantics::Symbol &sym) {
-  fir::CUDADataAttributeAttr cudaAttr =
-      Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(),
-                                                       sym);
+  cuf::DataAttributeAttr cudaAttr =
+      Fortran::lower::translateSymbolCUFDataAttribute(builder.getContext(),
+                                                      sym);
   mlir::Value errmsg =
       mlir::isa<fir::AbsentOp>(errorManager.errMsgAddr.getDefiningOp())
           ? nullptr
@@ -815,7 +816,7 @@ static mlir::Value genCudaDeallocate(fir::FirOpBuilder &builder,
   // Keep return type the same as a standard AllocatableAllocate call.
   mlir::Type retTy = fir::runtime::getModel<int>()(builder.getContext());
   return builder
-      .create<fir::CUDADeallocateOp>(
+      .create<cuf::DeallocateOp>(
           loc, retTy, box.getAddr(), errmsg, cudaAttr,
           errorManager.hasStatSpec() ? builder.getUnitAttr() : nullptr)
       .getResult();
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index afbc1122de86..4a29c6b8fae7 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -40,6 +40,8 @@
 #include "flang/Optimizer/Builder/Runtime/Ragged.h"
 #include "flang/Optimizer/Builder/Runtime/Stop.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h"
+#include "flang/Optimizer/Dialect/CUF/CUFOps.h"
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
@@ -2648,8 +2650,8 @@ private:
         loopEval = &*std::next(loopEval->getNestedEvaluations().begin());
     }
 
-    auto op = builder->create<fir::CUDAKernelOp>(
-        loc, gridValues, blockValues, streamValue, lbs, ubs, steps, n);
+    auto op = builder->create<cuf::KernelOp>(loc, gridValues, blockValues,
+                                             streamValue, lbs, ubs, steps, n);
     builder->createBlock(&op.getRegion(), op.getRegion().end(), ivTypes,
                          ivLocs);
     mlir::Block &b = op.getRegion().back();
@@ -3719,46 +3721,44 @@ private:
 
     // device = host
     if (lhsIsDevice && !rhsIsDevice) {
-      auto transferKindAttr = fir::CUDADataTransferKindAttr::get(
-          builder.getContext(), fir::CUDADataTransferKind::HostDevice);
+      auto transferKindAttr = cuf::DataTransferKindAttr::get(
+          builder.getContext(), cuf::DataTransferKind::HostDevice);
       if (!rhs.isVariable()) {
         auto associate = hlfir::genAssociateExpr(
             loc, builder, rhs, rhs.getType(), ".cuf_host_tmp");
-        builder.create<fir::CUDADataTransferOp>(loc, associate.getBase(), lhs,
-                                                transferKindAttr);
+        builder.create<cuf::DataTransferOp>(loc, associate.getBase(), lhs,
+                                            transferKindAttr);
         builder.create<hlfir::EndAssociateOp>(loc, associate);
       } else {
-        builder.create<fir::CUDADataTransferOp>(loc, rhs, lhs,
-                                                transferKindAttr);
+        builder.create<cuf::DataTransferOp>(loc, rhs, lhs, transferKindAttr);
       }
       return;
     }
 
     // host = device
     if (!lhsIsDevice && rhsIsDevice) {
-      auto transferKindAttr = fir::CUDADataTransferKindAttr::get(
-          builder.getContext(), fir::CUDADataTransferKind::DeviceHost);
+      auto transferKindAttr = cuf::DataTransferKindAttr::get(
+          builder.getContext(), cuf::DataTransferKind::DeviceHost);
       if (!rhs.isVariable()) {
         // evaluateRhs loads scalar. Look for the memory reference to be used in
         // the transfer.
         if (mlir::isa_and_nonnull<fir::LoadOp>(rhs.getDefiningOp())) {
           auto loadOp = mlir::dyn_cast<fir::LoadOp>(rhs.getDefiningOp());
-          builder.create<fir::CUDADataTransferOp>(loc, loadOp.getMemref(), lhs,
-                                                  transferKindAttr);
+          builder.create<cuf::DataTransferOp>(loc, loadOp.getMemref(), lhs,
+                                              transferKindAttr);
           return;
         }
       } else {
-        builder.create<fir::CUDADataTransferOp>(loc, rhs, lhs,
-                                                transferKindAttr);
+        builder.create<cuf::DataTransferOp>(loc, rhs, lhs, transferKindAttr);
       }
       return;
     }
 
     if (lhsIsDevice && rhsIsDevice) {
       assert(rhs.isVariable() && "CUDA Fortran assignment rhs is not legal");
-      auto transferKindAttr = fir::CUDADataTransferKindAttr::get(
-          builder.getContext(), fir::CUDADataTransferKind::DeviceDevice);
-      builder.create<fir::CUDADataTransferOp>(loc, rhs, lhs, transferKindAttr);
+      auto transferKindAttr = cuf::DataTransferKindAttr::get(
+          builder.getContext(), cuf::DataTransferKind::DeviceDevice);
+      builder.create<cuf::DataTransferOp>(loc, rhs, lhs, transferKindAttr);
       return;
     }
     llvm_unreachable("Unhandled CUDA data transfer");
@@ -3769,8 +3769,8 @@ private:
                               const Fortran::evaluate::Assignment &assign) {
     llvm::SmallVector<mlir::Value> temps;
     localSymbols.pushScope();
-    auto transferKindAttr = fir::CUDADataTransferKindAttr::get(
-        builder.getContext(), fir::CUDADataTransferKind::DeviceHost);
+    auto transferKindAttr = cuf::DataTransferKindAttr::get(
+        builder.getContext(), cuf::DataTransferKind::DeviceHost);
     [[maybe_unused]] unsigned nbDeviceResidentObject = 0;
     for (const Fortran::semantics::Symbol &sym :
          Fortran::evaluate::CollectSymbols(assign.rhs)) {
@@ -3795,8 +3795,8 @@ private:
           addSymbol(sym,
                     hlfir::translateToExtendedValue(loc, builder, temp).first,
                     /*forced=*/true);
-          builder.create<fir::CUDADataTransferOp>(loc, addr, temp,
-                                                  transferKindAttr);
+          builder.create<cuf::DataTransferOp>(loc, addr, temp,
+                                              transferKindAttr);
           ++nbDeviceResidentObject;
         }
       }
@@ -3808,15 +3808,15 @@ private:
   // subprogram are not considered fully device context so it will return false
   // for it.
   static bool isDeviceContext(fir::FirOpBuilder &builder) {
-    if (builder.getRegion().getParentOfType<fir::CUDAKernelOp>())
+    if (builder.getRegion().getParentOfType<cuf::KernelOp>())
       return true;
     if (auto funcOp =
             builder.getRegion().getParentOfType<mlir::func::FuncOp>()) {
       if (auto cudaProcAttr =
-              funcOp.getOperation()->getAttrOfType<fir::CUDAProcAttributeAttr>(
-                  fir::getCUDAAttrName())) {
-        return cudaProcAttr.getValue() != fir::CUDAProcAttribute::Host &&
-               cudaProcAttr.getValue() != fir::CUDAProcAttribute::HostDevice;
+              funcOp.getOperation()->getAttrOfType<cuf::ProcAttributeAttr>(
+                  cuf::getProcAttrName())) {
+        return cudaProcAttr.getValue() != cuf::ProcAttribute::Host &&
+               cudaProcAttr.getValue() != cuf::ProcAttribute::HostDevice;
       }
     }
     return false;
diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index 1546409752e7..ba6622d8504a 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -37,6 +37,8 @@ add_flang_library(FortranLower
   VectorSubscripts.cpp
   
   DEPENDS
+  CUFAttrs
+  CUFDialect
   FIRDialect
   FIRTransforms
   HLFIRDialect
@@ -44,6 +46,8 @@ add_flang_library(FortranLower
   ${extension_libs}
 
   LINK_LIBS
+  CUFAttrs
+  CUFDialect
   FIRDialect
   FIRDialectSupport
   FIRBuilder
diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index c1f54ad39287..cfbb7c7f6b4f 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -627,9 +627,9 @@ setCUDAAttributes(mlir::func::FuncOp func,
                       characteristic) {
   if (characteristic && characteristic->cudaSubprogramAttrs) {
     func.getOperation()->setAttr(
-        fir::getCUDAAttrName(),
-        fir::getCUDAProcAttribute(func.getContext(),
-                                  *characteristic->cudaSubprogramAttrs));
+        cuf::getProcAttrName(),
+        cuf::getProcAttribute(func.getContext(),
+                              *characteristic->cudaSubprogramAttrs));
   }
 
   if (sym) {
@@ -649,9 +649,9 @@ setCUDAAttributes(mlir::func::FuncOp func,
           ubAttr =
               mlir::IntegerAttr::get(i64Ty, details->cudaLaunchBounds()[2]);
         func.getOperation()->setAttr(
-            fir::getCUDALaunchBoundsAttrName(),
-            fir::CUDALaunchBoundsAttr::get(func.getContext(), maxTPBAttr,
-                                           minBPMAttr, ubAttr));
+            cuf::getLaunchBoundsAttrName(),
+            cuf::LaunchBoundsAttr::get(func.getContext(), maxTPBAttr,
+                                       minBPMAttr, ubAttr));
       }
 
       if (!details->cudaClusterDims().empty()) {
@@ -663,9 +663,8 @@ setCUDAAttributes(mlir::func::FuncOp func,
         auto zAttr =
             mlir::IntegerAttr::get(i64Ty, details->cudaClusterDims()[2]);
         func.getOperation()->setAttr(
-            fir::getCUDAClusterDimsAttrName(),
-            fir::CUDAClusterDimsAttr::get(func.getContext(), xAttr, yAttr,
-                                          zAttr));
+            cuf::getClusterDimsAttrName(),
+            cuf::ClusterDimsAttr::get(func.getContext(), xAttr, yAttr, zAttr));
       }
     }
   }
@@ -1116,8 +1115,8 @@ private:
       addMLIRAttr(fir::getTargetAttrName());
     if (obj.cudaDataAttr)
       attrs.emplace_back(
-          mlir::StringAttr::get(&mlirContext, fir::getCUDAAttrName()),
-          fir::getCUDADataAttribute(&mlirContext, obj.cudaDataAttr));
+          mlir::StringAttr::get(&mlirContext, cuf::getDataAttrName()),
+          cuf::getDataAttribute(&mlirContext, obj.cudaDataAttr));
 
     // TODO: intents that require special care (e.g finalization)
 
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index c6bfe3592169..7ec719a2cb9e 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -28,6 +28,7 @@
 #include "flang/Optimizer/Builder/MutableBox.h"
 #include "flang/Optimizer/Builder/Runtime/Derived.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/Dialect/CUF/CUFOps.h"
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "mlir/IR/IRMapping.h"
@@ -589,7 +590,7 @@ std::pair<fir::ExtendedValue, bool> Fortran::lower::genCallOpAndResult(
           fir::getBase(converter.genExprValue(
               caller.getCallDescription().chevrons()[3], stmtCtx)));
 
-    builder.create<fir::CUDAKernelLaunch>(
+    builder.create<cuf::KernelLaunchOp>(
         loc, funcType.getResults(), funcSymbolAttr, grid_x, grid_y, grid_z,
         block_x, block_y, block_z, bytes, stream, operands);
     callNumResults = 0;
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 5ddd8a6a9d41..b8868161fa05 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -30,6 +30,7 @@
 #include "flang/Optimizer/Builder/IntrinsicCall.h"
 #include "flang/Optimizer/Builder/Runtime/Derived.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/Dialect/CUF/CUFOps.h"
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
@@ -139,7 +140,7 @@ static fir::GlobalOp defineGlobal(Fortran::lower::AbstractConverter &converter,
                                   const Fortran::lower::pft::Variable &var,
                                   llvm::StringRef globalName,
                                   mlir::StringAttr linkage,
-                                  fir::CUDADataAttributeAttr cudaAttr = {});
+                                  cuf::DataAttributeAttr dataAttr = {});
 
 static mlir::Location genLocation(Fortran::lower::AbstractConverter &converter,
                                   const Fortran::semantics::Symbol &sym) {
@@ -172,12 +173,12 @@ static fir::GlobalOp declareGlobal(Fortran::lower::AbstractConverter &converter,
       !Fortran::semantics::IsProcedurePointer(ultimate))
     mlir::emitError(loc, "processing global declaration: symbol '")
         << toStringRef(sym.name()) << "' has unexpected details\n";
-  fir::CUDADataAttributeAttr cudaAttr =
-      Fortran::lower::translateSymbolCUDADataAttribute(
+  cuf::DataAttributeAttr dataAttr =
+      Fortran::lower::translateSymbolCUFDataAttribute(
           converter.getFirOpBuilder().getContext(), sym);
   return builder.createGlobal(loc, converter.genType(var), globalName, linkage,
                               mlir::Attribute{}, isConstant(ultimate),
-                              var.isTarget(), cudaAttr);
+                              var.isTarget(), dataAttr);
 }
 
 /// Temporary helper to catch todos in initial data target lowering.
@@ -474,7 +475,7 @@ static fir::GlobalOp defineGlobal(Fortran::lower::AbstractConverter &converter,
                                   const Fortran::lower::pft::Variable &var,
                                   llvm::StringRef globalName,
                                   mlir::StringAttr linkage,
-                                  fir::CUDADataAttributeAttr cudaAttr) {
+                                  cuf::DataAttributeAttr dataAttr) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   const Fortran::semantics::Symbol &sym = var.getSymbol();
   mlir::Location loc = genLocation(converter, sym);
@@ -514,7 +515,7 @@ static fir::GlobalOp defineGlobal(Fortran::lower::AbstractConverter &converter,
   if (!global)
     global =
         builder.createGlobal(loc, symTy, globalName, linkage, mlir::Attribute{},
-                             isConst, var.isTarget(), cudaAttr);
+                             isConst, var.isTarget(), dataAttr);
   if (Fortran::semantics::IsAllocatableOrPointer(sym) &&
       !Fortran::semantics::IsProcedure(sym)) {
     const auto *details =
@@ -694,9 +695,9 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
     return builder.create<fir::ZeroOp>(loc, fir::ReferenceType::get(ty));
 
   if (Fortran::semantics::NeedCUDAAlloc(ultimateSymbol)) {
-    fir::CUDADataAttributeAttr cudaAttr =
-        Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(),
-                                                         ultimateSymbol);
+    cuf::DataAttributeAttr dataAttr =
+        Fortran::lower::translateSymbolCUFDataAttribute(builder.getContext(),
+                                                        ultimateSymbol);
     llvm::SmallVector<mlir::Value> indices;
     llvm::SmallVector<mlir::Value> elidedShape =
         fir::factory::elideExtentsAlreadyInType(ty, shape);
@@ -705,8 +706,8 @@ static mlir::Value createNewLocal(Fortran::lower::AbstractConverter &converter,
     auto idxTy = builder.getIndexType();
     for (mlir::Value sh : elidedShape)
       indices.push_back(builder.createConvert(loc, idxTy, sh));
-    return builder.create<fir::CUDAAllocOp>(loc, ty, nm, symNm, cudaAttr,
-                                            lenParams, indices);
+    return builder.create<cuf::AllocOp>(loc, ty, nm, symNm, dataAttr, lenParams,
+                                        indices);
   }
 
   // Let the builder do all the heavy lifting.
@@ -950,10 +951,10 @@ static void instantiateLocal(Fortran::lower::AbstractConverter &converter,
         converter.getSymbolExtendedValue(var.getSymbol(), &symMap);
     auto *sym = &var.getSymbol();
     converter.getFctCtx().attachCleanup([builder, loc, exv, sym]() {
-      fir::CUDADataAttributeAttr cudaAttr =
-          Fortran::lower::translateSymbolCUDADataAttribute(
-              builder->getContext(), *sym);
-      builder->create<fir::CUDAFreeOp>(loc, fir::getBase(exv), cudaAttr);
+      cuf::DataAttributeAttr dataAttr =
+          Fortran::lower::translateSymbolCUFDataAttribute(builder->getContext(),
+                                                          *sym);
+      builder->create<cuf::FreeOp>(loc, fir::getBase(exv), dataAttr);
     });
   }
 }
@@ -1628,11 +1629,11 @@ fir::FortranVariableFlagsAttr Fortran::lower::translateSymbolAttributes(
   return fir::FortranVariableFlagsAttr::get(mlirContext, flags);
 }
 
-fir::CUDADataAttributeAttr Fortran::lower::translateSymbolCUDADataAttribute(
+cuf::DataAttributeAttr Fortran::lower::translateSymbolCUFDataAttribute(
     mlir::MLIRContext *mlirContext, const Fortran::semantics::Symbol &sym) {
   std::optional<Fortran::common::CUDADataAttr> cudaAttr =
       Fortran::semantics::GetCUDADataAttr(&sym.GetUltimate());
-  return fir::getCUDADataAttribute(mlirContext, cudaAttr);
+  return cuf::getDataAttribute(mlirContext, cudaAttr);
 }
 
 /// Map a symbol to its FIR address and evaluated specification expressions.
@@ -1672,9 +1673,9 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
     auto name = converter.mangleName(sym);
     fir::FortranVariableFlagsAttr attributes =
         Fortran::lower::translateSymbolAttributes(builder.getContext(), sym);
-    fir::CUDADataAttributeAttr cudaAttr =
-        Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(),
-                                                         sym);
+    cuf::DataAttributeAttr dataAttr =
+        Fortran::lower::translateSymbolCUFDataAttribute(builder.getContext(),
+                                                        sym);
 
     if (sym.test(Fortran::semantics::Symbol::Flag::CrayPointee)) {
       mlir::Type ptrBoxType =
@@ -1716,7 +1717,7 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
       dummyScope = converter.dummyArgsScopeValue();
     auto newBase = builder.create<hlfir::DeclareOp>(
         loc, base, name, shapeOrShift, lenParams, dummyScope, attributes,
-        cudaAttr);
+        dataAttr);
     symMap.addVariableDefinition(sym, newBase, force);
     return;
   }
@@ -1762,15 +1763,15 @@ void Fortran::lower::genDeclareSymbol(
     fir::FortranVariableFlagsAttr attributes =
         Fortran::lower::translateSymbolAttributes(
             builder.getContext(), sym.GetUltimate(), extraFlags);
-    fir::CUDADataAttributeAttr cudaAttr =
-        Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(),
-                                                         sym.GetUltimate());
+    cuf::DataAttributeAttr dataAttr =
+        Fortran::lower::translateSymbolCUFDataAttribute(builder.getContext(),
+                                                        sym.GetUltimate());
     auto name = converter.mangleName(sym);
     mlir::Value dummyScope;
     if (converter.isRegisteredDummySymbol(sym))
       dummyScope = converter.dummyArgsScopeValue();
     hlfir::EntityWithAttributes declare = hlfir::genDeclare(
-        loc, builder, exv, name, attributes, dummyScope, cudaAttr);
+        loc, builder, exv, name, attributes, dummyScope, dataAttr);
     symMap.addVariableDefinition(sym, declare.getIfVariableInterface(), force);
     return;
   }
@@ -2272,10 +2273,10 @@ void Fortran::lower::defineModuleVariable(
     // Do nothing. Mapping will be done on user side.
   } else {
     std::string globalName = converter.mangleName(sym);
-    fir::CUDADataAttributeAttr cudaAttr =
-        Fortran::lower::translateSymbolCUDADataAttribute(
+    cuf::DataAttributeAttr dataAttr =
+        Fortran::lower::translateSymbolCUFDataAttribute(
             converter.getFirOpBuilder().getContext(), sym);
-    defineGlobal(converter, var, globalName, linkage, cudaAttr);
+    defineGlobal(converter, var, globalName, linkage, dataAttr);
   }
 }
 
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index bd018d7f015b..3c3fd02d7c88 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -322,18 +322,18 @@ mlir::Value fir::FirOpBuilder::createHeapTemporary(
 fir::GlobalOp fir::FirOpBuilder::createGlobal(
     mlir::Location loc, mlir::Type type, llvm::StringRef name,
     mlir::StringAttr linkage, mlir::Attribute value, bool isConst,
-    bool isTarget, fir::CUDADataAttributeAttr cudaAttr) {
+    bool isTarget, cuf::DataAttributeAttr dataAttr) {
   if (auto global = getNamedGlobal(name))
     return global;
   auto module = getModule();
   auto insertPt = saveInsertionPoint();
   setInsertionPoint(module.getBody(), module.getBody()->end());
   llvm::SmallVector<mlir::NamedAttribute> attrs;
-  if (cudaAttr) {
+  if (dataAttr) {
     auto globalOpName = mlir::OperationName(fir::GlobalOp::getOperationName(),
                                             module.getContext());
     attrs.push_back(mlir::NamedAttribute(
-        fir::GlobalOp::getCudaAttrAttrName(globalOpName), cudaAttr));
+        fir::GlobalOp::getDataAttrAttrName(globalOpName), dataAttr));
   }
   auto glob = create<fir::GlobalOp>(loc, name, isConst, isTarget, type, value,
                                     linkage, attrs);
@@ -346,7 +346,7 @@ fir::GlobalOp fir::FirOpBuilder::createGlobal(
 fir::GlobalOp fir::FirOpBuilder::createGlobal(
     mlir::Location loc, mlir::Type type, llvm::StringRef name, bool isConst,
     bool isTarget, std::function<void(FirOpBuilder &)> bodyBuilder,
-    mlir::StringAttr linkage, fir::CUDADataAttributeAttr cudaAttr) {
+    mlir::StringAttr linkage, cuf::DataAttributeAttr dataAttr) {
   if (auto global = getNamedGlobal(name))
     return global;
   auto module = getModule();
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index 8fdab2a57181..511585dc7689 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -199,7 +199,7 @@ fir::FortranVariableOpInterface
 hlfir::genDeclare(mlir::Location loc, fir::FirOpBuilder &builder,
                   const fir::ExtendedValue &exv, llvm::StringRef name,
                   fir::FortranVariableFlagsAttr flags, mlir::Value dummyScope,
-                  fir::CUDADataAttributeAttr cudaAttr) {
+                  cuf::DataAttributeAttr dataAttr) {
 
   mlir::Value base = fir::getBase(exv);
   assert(fir::conformsWithPassByRef(base.getType()) &&
@@ -229,7 +229,7 @@ hlfir::genDeclare(mlir::Location loc, fir::FirOpBuilder &builder,
       },
       [](const auto &) {});
   auto declareOp = builder.create<hlfir::DeclareOp>(
-      loc, base, name, shapeOrShift, lenParams, dummyScope, flags, cudaAttr);
+      loc, base, name, shapeOrShift, lenParams, dummyScope, flags, dataAttr);
   return mlir::cast<fir::FortranVariableOpInterface>(declareOp.getOperation());
 }
 
diff --git a/flang/lib/Optimizer/Dialect/CMakeLists.txt b/flang/lib/Optimizer/Dialect/CMakeLists.txt
index 745439b7e1e5..a8235f841b87 100644
--- a/flang/lib/Optimizer/Dialect/CMakeLists.txt
+++ b/flang/lib/Optimizer/Dialect/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_subdirectory(Support)
+add_subdirectory(CUF)
 
 add_flang_library(FIRDialect
   FIRAttr.cpp
@@ -13,9 +14,11 @@ add_flang_library(FIRDialect
   CanonicalizationPatternsIncGen
   MLIRIR
   FIROpsIncGen
+  CUFAttrsIncGen
   intrinsics_gen
 
   LINK_LIBS
+  CUFAttrs
   FIRDialectSupport
   MLIRArithDialect
   MLIRBuiltinToLLVMIRTranslation
diff --git a/flang/lib/Optimizer/Dialect/CUF/Attributes/CMakeLists.txt b/flang/lib/Optimizer/Dialect/CUF/Attributes/CMakeLists.txt
new file mode 100644
index 000000000000..81db40f3ba46
--- /dev/null
+++ b/flang/lib/Optimizer/Dialect/CUF/Attributes/CMakeLists.txt
@@ -0,0 +1,16 @@
+# Keep CUF attributes as a separate library as FIR and HLFIR depend on it. 
+add_flang_library(CUFAttrs
+  CUFAttr.cpp
+
+  DEPENDS
+  MLIRIR
+  CUFAttrsIncGen
+
+  LINK_LIBS
+  MLIRTargetLLVMIRExport
+
+  LINK_COMPONENTS
+  AsmParser
+  AsmPrinter
+  Remarks
+)
diff --git a/flang/lib/Optimizer/Dialect/CUF/Attributes/CUFAttr.cpp b/flang/lib/Optimizer/Dialect/CUF/Attributes/CUFAttr.cpp
new file mode 100644
index 000000000000..52c733dcad6f
--- /dev/null
+++ b/flang/lib/Optimizer/Dialect/CUF/Attributes/CUFAttr.cpp
@@ -0,0 +1,32 @@
+//===-- CUFAttr.cpp -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h"
+#include "flang/Optimizer/Dialect/CUF/CUFDialect.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/DialectImplementation.h"
+#include "mlir/IR/OpDefinition.h"
+#include "llvm/ADT/TypeSwitch.h"
+
+#include "flang/Optimizer/Dialect/CUF/Attributes/CUFEnumAttr.cpp.inc"
+#define GET_ATTRDEF_CLASSES
+#include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.cpp.inc"
+
+namespace cuf {
+
+void CUFDialect::registerAttributes() {
+  addAttributes<ClusterDimsAttr, DataAttributeAttr, DataTransferKindAttr,
+                LaunchBoundsAttr, ProcAttributeAttr>();
+}
+
+} // namespace cuf
diff --git a/flang/lib/Optimizer/Dialect/CUF/CMakeLists.txt b/flang/lib/Optimizer/Dialect/CUF/CMakeLists.txt
new file mode 100644
index 000000000000..d5ce5e0a7614
--- /dev/null
+++ b/flang/lib/Optimizer/Dialect/CUF/CMakeLists.txt
@@ -0,0 +1,22 @@
+add_subdirectory(Attributes)
+
+add_flang_library(CUFDialect
+  CUFDialect.cpp
+  CUFOps.cpp
+
+  DEPENDS
+  MLIRIR
+  CUFOpsIncGen
+
+  LINK_LIBS
+  CUFAttrs
+  FIRDialect
+  FIRDialectSupport
+  MLIRIR
+  MLIRTargetLLVMIRExport
+
+  LINK_COMPONENTS
+  AsmParser
+  AsmPrinter
+  Remarks
+)
diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFDialect.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFDialect.cpp
new file mode 100644
index 000000000000..47d3636df4f9
--- /dev/null
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFDialect.cpp
@@ -0,0 +1,25 @@
+//===-- CUFDialect.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Dialect/CUF/CUFDialect.h"
+#include "flang/Optimizer/Dialect/CUF/CUFOps.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+
+#include "flang/Optimizer/Dialect/CUF/CUFDialect.cpp.inc"
+
+void cuf::CUFDialect::initialize() {
+  registerAttributes();
+  addOperations<
+#define GET_OP_LIST
+#include "flang/Optimizer/Dialect/CUF/CUFOps.cpp.inc"
+      >();
+}
diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
new file mode 100644
index 000000000000..870652c72fab
--- /dev/null
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
@@ -0,0 +1,219 @@
+//===-- CUFOps.cpp --------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Dialect/CUF/CUFOps.h"
+#include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h"
+#include "flang/Optimizer/Dialect/CUF/CUFDialect.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/Matchers.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/IR/PatternMatch.h"
+#include "llvm/ADT/SmallVector.h"
+
+//===----------------------------------------------------------------------===//
+// AllocOp
+//===----------------------------------------------------------------------===//
+
+static mlir::Type wrapAllocaResultType(mlir::Type intype) {
+  if (mlir::isa<fir::ReferenceType>(intype))
+    return {};
+  return fir::ReferenceType::get(intype);
+}
+
+void cuf::AllocOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
+                         mlir::Type inType, llvm::StringRef uniqName,
+                         llvm::StringRef bindcName,
+                         cuf::DataAttributeAttr cudaAttr,
+                         mlir::ValueRange typeparams, mlir::ValueRange shape,
+                         llvm::ArrayRef<mlir::NamedAttribute> attributes) {
+  mlir::StringAttr nameAttr =
+      uniqName.empty() ? mlir::StringAttr{} : builder.getStringAttr(uniqName);
+  mlir::StringAttr bindcAttr =
+      bindcName.empty() ? mlir::StringAttr{} : builder.getStringAttr(bindcName);
+  build(builder, result, wrapAllocaResultType(inType),
+        mlir::TypeAttr::get(inType), nameAttr, bindcAttr, typeparams, shape,
+        cudaAttr);
+  result.addAttributes(attributes);
+}
+
+template <typename Op>
+static mlir::LogicalResult checkCudaAttr(Op op) {
+  if (op.getDataAttr() == cuf::DataAttribute::Device ||
+      op.getDataAttr() == cuf::DataAttribute::Managed ||
+      op.getDataAttr() == cuf::DataAttribute::Unified)
+    return mlir::success();
+  return op.emitOpError("expect device, managed or unified cuda attribute");
+}
+
+mlir::LogicalResult cuf::AllocOp::verify() { return checkCudaAttr(*this); }
+
+//===----------------------------------------------------------------------===//
+// FreeOp
+//===----------------------------------------------------------------------===//
+
+mlir::LogicalResult cuf::FreeOp::verify() { return checkCudaAttr(*this); }
+
+//===----------------------------------------------------------------------===//
+// AllocateOp
+//===----------------------------------------------------------------------===//
+
+mlir::LogicalResult cuf::AllocateOp::verify() {
+  if (getPinned() && getStream())
+    return emitOpError("pinned and stream cannot appears at the same time");
+  if (!mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(getBox().getType())))
+    return emitOpError(
+        "expect box to be a reference to a class or box type value");
+  if (getSource() &&
+      !mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(getSource().getType())))
+    return emitOpError(
+        "expect source to be a reference to/or a class or box type value");
+  if (getErrmsg() &&
+      !mlir::isa<fir::BoxType>(fir::unwrapRefType(getErrmsg().getType())))
+    return emitOpError(
+        "expect errmsg to be a reference to/or a box type value");
+  if (getErrmsg() && !getHasStat())
+    return emitOpError("expect stat attribute when errmsg is provided");
+  return mlir::success();
+}
+
+//===----------------------------------------------------------------------===//
+// DeallocateOp
+//===----------------------------------------------------------------------===//
+
+mlir::LogicalResult cuf::DeallocateOp::verify() {
+  if (!mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(getBox().getType())))
+    return emitOpError(
+        "expect box to be a reference to class or box type value");
+  if (getErrmsg() &&
+      !mlir::isa<fir::BoxType>(fir::unwrapRefType(getErrmsg().getType())))
+    return emitOpError(
+        "expect errmsg to be a reference to/or a box type value");
+  if (getErrmsg() && !getHasStat())
+    return emitOpError("expect stat attribute when errmsg is provided");
+  return mlir::success();
+}
+
+//===----------------------------------------------------------------------===//
+// KernelOp
+//===----------------------------------------------------------------------===//
+
+llvm::SmallVector<mlir::Region *> cuf::KernelOp::getLoopRegions() {
+  return {&getRegion()};
+}
+
+mlir::ParseResult parseCUFKernelValues(
+    mlir::OpAsmParser &parser,
+    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &values,
+    llvm::SmallVectorImpl<mlir::Type> &types) {
+  if (mlir::succeeded(parser.parseOptionalStar()))
+    return mlir::success();
+
+  if (mlir::succeeded(parser.parseOptionalLParen())) {
+    if (mlir::failed(parser.parseCommaSeparatedList(
+            mlir::AsmParser::Delimiter::None, [&]() {
+              if (parser.parseOperand(values.emplace_back()))
+                return mlir::failure();
+              return mlir::success();
+            })))
+      return mlir::failure();
+    auto builder = parser.getBuilder();
+    for (size_t i = 0; i < values.size(); i++) {
+      types.emplace_back(builder.getI32Type());
+    }
+    if (parser.parseRParen())
+      return mlir::failure();
+  } else {
+    if (parser.parseOperand(values.emplace_back()))
+      return mlir::failure();
+    auto builder = parser.getBuilder();
+    types.emplace_back(builder.getI32Type());
+    return mlir::success();
+  }
+  return mlir::success();
+}
+
+void printCUFKernelValues(mlir::OpAsmPrinter &p, mlir::Operation *op,
+                          mlir::ValueRange values, mlir::TypeRange types) {
+  if (values.empty())
+    p << "*";
+
+  if (values.size() > 1)
+    p << "(";
+  llvm::interleaveComma(values, p, [&p](mlir::Value v) { p << v; });
+  if (values.size() > 1)
+    p << ")";
+}
+
+mlir::ParseResult parseCUFKernelLoopControl(
+    mlir::OpAsmParser &parser, mlir::Region &region,
+    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &lowerbound,
+    llvm::SmallVectorImpl<mlir::Type> &lowerboundType,
+    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &upperbound,
+    llvm::SmallVectorImpl<mlir::Type> &upperboundType,
+    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &step,
+    llvm::SmallVectorImpl<mlir::Type> &stepType) {
+
+  llvm::SmallVector<mlir::OpAsmParser::Argument> inductionVars;
+  if (parser.parseLParen() ||
+      parser.parseArgumentList(inductionVars,
+                               mlir::OpAsmParser::Delimiter::None,
+                               /*allowType=*/true) ||
+      parser.parseRParen() || parser.parseEqual() || parser.parseLParen() ||
+      parser.parseOperandList(lowerbound, inductionVars.size(),
+                              mlir::OpAsmParser::Delimiter::None) ||
+      parser.parseColonTypeList(lowerboundType) || parser.parseRParen() ||
+      parser.parseKeyword("to") || parser.parseLParen() ||
+      parser.parseOperandList(upperbound, inductionVars.size(),
+                              mlir::OpAsmParser::Delimiter::None) ||
+      parser.parseColonTypeList(upperboundType) || parser.parseRParen() ||
+      parser.parseKeyword("step") || parser.parseLParen() ||
+      parser.parseOperandList(step, inductionVars.size(),
+                              mlir::OpAsmParser::Delimiter::None) ||
+      parser.parseColonTypeList(stepType) || parser.parseRParen())
+    return mlir::failure();
+  return parser.parseRegion(region, inductionVars);
+}
+
+void printCUFKernelLoopControl(
+    mlir::OpAsmPrinter &p, mlir::Operation *op, mlir::Region &region,
+    mlir::ValueRange lowerbound, mlir::TypeRange lowerboundType,
+    mlir::ValueRange upperbound, mlir::TypeRange upperboundType,
+    mlir::ValueRange steps, mlir::TypeRange stepType) {
+  mlir::ValueRange regionArgs = region.front().getArguments();
+  if (!regionArgs.empty()) {
+    p << "(";
+    llvm::interleaveComma(
+        regionArgs, p, [&p](mlir::Value v) { p << v << " : " << v.getType(); });
+    p << ") = (" << lowerbound << " : " << lowerboundType << ") to ("
+      << upperbound << " : " << upperboundType << ") "
+      << " step (" << steps << " : " << stepType << ") ";
+  }
+  p.printRegion(region, /*printEntryBlockArgs=*/false);
+}
+
+mlir::LogicalResult cuf::KernelOp::verify() {
+  if (getLowerbound().size() != getUpperbound().size() ||
+      getLowerbound().size() != getStep().size())
+    return emitOpError(
+        "expect same number of values in lowerbound, upperbound and step");
+
+  return mlir::success();
+}
+
+// Tablegen operators
+
+#define GET_OP_CLASSES
+#include "flang/Optimizer/Dialect/CUF/CUFOps.cpp.inc"
diff --git a/flang/lib/Optimizer/Dialect/FIRAttr.cpp b/flang/lib/Optimizer/Dialect/FIRAttr.cpp
index 9ea3a0568f69..2faba63dfba0 100644
--- a/flang/lib/Optimizer/Dialect/FIRAttr.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRAttr.cpp
@@ -298,7 +298,5 @@ void fir::printFirAttribute(FIROpsDialect *dialect, mlir::Attribute attr,
 void FIROpsDialect::registerAttributes() {
   addAttributes<ClosedIntervalAttr, ExactTypeAttr, FortranVariableFlagsAttr,
                 LowerBoundAttr, PointIntervalAttr, RealAttr, SubclassAttr,
-                UpperBoundAttr, CUDADataAttributeAttr, CUDAProcAttributeAttr,
-                CUDALaunchBoundsAttr, CUDAClusterDimsAttr,
-                CUDADataTransferKindAttr>();
+                UpperBoundAttr>();
 }
diff --git a/flang/lib/Optimizer/Dialect/FIRDialect.cpp b/flang/lib/Optimizer/Dialect/FIRDialect.cpp
index 4d1e8cd1405a..4b1dadaac672 100644
--- a/flang/lib/Optimizer/Dialect/FIRDialect.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRDialect.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h"
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index edf7f7f4b1a9..94113da9a46c 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -3898,169 +3898,6 @@ mlir::LogicalResult fir::DeclareOp::verify() {
   return fortranVar.verifyDeclareLikeOpImpl(getMemref());
 }
 
-llvm::SmallVector<mlir::Region *> fir::CUDAKernelOp::getLoopRegions() {
-  return {&getRegion()};
-}
-
-mlir::ParseResult parseCUFKernelValues(
-    mlir::OpAsmParser &parser,
-    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &values,
-    llvm::SmallVectorImpl<mlir::Type> &types) {
-  if (mlir::succeeded(parser.parseOptionalStar()))
-    return mlir::success();
-
-  if (mlir::succeeded(parser.parseOptionalLParen())) {
-    if (mlir::failed(parser.parseCommaSeparatedList(
-            mlir::AsmParser::Delimiter::None, [&]() {
-              if (parser.parseOperand(values.emplace_back()))
-                return mlir::failure();
-              return mlir::success();
-            })))
-      return mlir::failure();
-    auto builder = parser.getBuilder();
-    for (size_t i = 0; i < values.size(); i++) {
-      types.emplace_back(builder.getI32Type());
-    }
-    if (parser.parseRParen())
-      return mlir::failure();
-  } else {
-    if (parser.parseOperand(values.emplace_back()))
-      return mlir::failure();
-    auto builder = parser.getBuilder();
-    types.emplace_back(builder.getI32Type());
-    return mlir::success();
-  }
-  return mlir::success();
-}
-
-void printCUFKernelValues(mlir::OpAsmPrinter &p, mlir::Operation *op,
-                          mlir::ValueRange values, mlir::TypeRange types) {
-  if (values.empty())
-    p << "*";
-
-  if (values.size() > 1)
-    p << "(";
-  llvm::interleaveComma(values, p, [&p](mlir::Value v) { p << v; });
-  if (values.size() > 1)
-    p << ")";
-}
-
-mlir::ParseResult parseCUFKernelLoopControl(
-    mlir::OpAsmParser &parser, mlir::Region &region,
-    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &lowerbound,
-    llvm::SmallVectorImpl<mlir::Type> &lowerboundType,
-    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &upperbound,
-    llvm::SmallVectorImpl<mlir::Type> &upperboundType,
-    llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &step,
-    llvm::SmallVectorImpl<mlir::Type> &stepType) {
-
-  llvm::SmallVector<mlir::OpAsmParser::Argument> inductionVars;
-  if (parser.parseLParen() ||
-      parser.parseArgumentList(inductionVars,
-                               mlir::OpAsmParser::Delimiter::None,
-                               /*allowType=*/true) ||
-      parser.parseRParen() || parser.parseEqual() || parser.parseLParen() ||
-      parser.parseOperandList(lowerbound, inductionVars.size(),
-                              mlir::OpAsmParser::Delimiter::None) ||
-      parser.parseColonTypeList(lowerboundType) || parser.parseRParen() ||
-      parser.parseKeyword("to") || parser.parseLParen() ||
-      parser.parseOperandList(upperbound, inductionVars.size(),
-                              mlir::OpAsmParser::Delimiter::None) ||
-      parser.parseColonTypeList(upperboundType) || parser.parseRParen() ||
-      parser.parseKeyword("step") || parser.parseLParen() ||
-      parser.parseOperandList(step, inductionVars.size(),
-                              mlir::OpAsmParser::Delimiter::None) ||
-      parser.parseColonTypeList(stepType) || parser.parseRParen())
-    return mlir::failure();
-  return parser.parseRegion(region, inductionVars);
-}
-
-void printCUFKernelLoopControl(
-    mlir::OpAsmPrinter &p, mlir::Operation *op, mlir::Region &region,
-    mlir::ValueRange lowerbound, mlir::TypeRange lowerboundType,
-    mlir::ValueRange upperbound, mlir::TypeRange upperboundType,
-    mlir::ValueRange steps, mlir::TypeRange stepType) {
-  mlir::ValueRange regionArgs = region.front().getArguments();
-  if (!regionArgs.empty()) {
-    p << "(";
-    llvm::interleaveComma(
-        regionArgs, p, [&p](mlir::Value v) { p << v << " : " << v.getType(); });
-    p << ") = (" << lowerbound << " : " << lowerboundType << ") to ("
-      << upperbound << " : " << upperboundType << ") "
-      << " step (" << steps << " : " << stepType << ") ";
-  }
-  p.printRegion(region, /*printEntryBlockArgs=*/false);
-}
-
-mlir::LogicalResult fir::CUDAKernelOp::verify() {
-  if (getLowerbound().size() != getUpperbound().size() ||
-      getLowerbound().size() != getStep().size())
-    return emitOpError(
-        "expect same number of values in lowerbound, upperbound and step");
-
-  return mlir::success();
-}
-
-mlir::LogicalResult fir::CUDAAllocateOp::verify() {
-  if (getPinned() && getStream())
-    return emitOpError("pinned and stream cannot appears at the same time");
-  if (!mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(getBox().getType())))
-    return emitOpError(
-        "expect box to be a reference to a class or box type value");
-  if (getSource() &&
-      !mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(getSource().getType())))
-    return emitOpError(
-        "expect source to be a reference to/or a class or box type value");
-  if (getErrmsg() &&
-      !mlir::isa<fir::BoxType>(fir::unwrapRefType(getErrmsg().getType())))
-    return emitOpError(
-        "expect errmsg to be a reference to/or a box type value");
-  if (getErrmsg() && !getHasStat())
-    return emitOpError("expect stat attribute when errmsg is provided");
-  return mlir::success();
-}
-
-mlir::LogicalResult fir::CUDADeallocateOp::verify() {
-  if (!mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(getBox().getType())))
-    return emitOpError(
-        "expect box to be a reference to class or box type value");
-  if (getErrmsg() &&
-      !mlir::isa<fir::BoxType>(fir::unwrapRefType(getErrmsg().getType())))
-    return emitOpError(
-        "expect errmsg to be a reference to/or a box type value");
-  if (getErrmsg() && !getHasStat())
-    return emitOpError("expect stat attribute when errmsg is provided");
-  return mlir::success();
-}
-
-void fir::CUDAAllocOp::build(
-    mlir::OpBuilder &builder, mlir::OperationState &result, mlir::Type inType,
-    llvm::StringRef uniqName, llvm::StringRef bindcName,
-    fir::CUDADataAttributeAttr cudaAttr, mlir::ValueRange typeparams,
-    mlir::ValueRange shape, llvm::ArrayRef<mlir::NamedAttribute> attributes) {
-  mlir::StringAttr nameAttr =
-      uniqName.empty() ? mlir::StringAttr{} : builder.getStringAttr(uniqName);
-  mlir::StringAttr bindcAttr =
-      bindcName.empty() ? mlir::StringAttr{} : builder.getStringAttr(bindcName);
-  build(builder, result, wrapAllocaResultType(inType),
-        mlir::TypeAttr::get(inType), nameAttr, bindcAttr, typeparams, shape,
-        cudaAttr);
-  result.addAttributes(attributes);
-}
-
-template <typename Op>
-static mlir::LogicalResult checkCudaAttr(Op op) {
-  if (op.getCudaAttr() == fir::CUDADataAttribute::Device ||
-      op.getCudaAttr() == fir::CUDADataAttribute::Managed ||
-      op.getCudaAttr() == fir::CUDADataAttribute::Unified)
-    return mlir::success();
-  return op.emitOpError("expect device, managed or unified cuda attribute");
-}
-
-mlir::LogicalResult fir::CUDAAllocOp::verify() { return checkCudaAttr(*this); }
-
-mlir::LogicalResult fir::CUDAFreeOp::verify() { return checkCudaAttr(*this); }
-
 //===----------------------------------------------------------------------===//
 // FIROpsDialect
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/HLFIR/IR/CMakeLists.txt b/flang/lib/Optimizer/HLFIR/IR/CMakeLists.txt
index dc9e080b0f8b..267d6469ee7a 100644
--- a/flang/lib/Optimizer/HLFIR/IR/CMakeLists.txt
+++ b/flang/lib/Optimizer/HLFIR/IR/CMakeLists.txt
@@ -5,11 +5,13 @@ add_flang_library(HLFIRDialect
   HLFIROps.cpp
 
   DEPENDS
+  CUFAttrsIncGen
   FIRDialect
   HLFIROpsIncGen
   ${dialect_libs}
 
   LINK_LIBS
+  CUFAttrs
   FIRDialect
   MLIRIR
   ${dialect_libs}
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index c232ae165d4c..11196353b07c 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -127,14 +127,14 @@ void hlfir::DeclareOp::build(mlir::OpBuilder &builder,
                              mlir::ValueRange typeparams,
                              mlir::Value dummy_scope,
                              fir::FortranVariableFlagsAttr fortran_attrs,
-                             fir::CUDADataAttributeAttr cuda_attr) {
+                             cuf::DataAttributeAttr data_attr) {
   auto nameAttr = builder.getStringAttr(uniq_name);
   mlir::Type inputType = memref.getType();
   bool hasExplicitLbs = hasExplicitLowerBounds(shape);
   mlir::Type hlfirVariableType =
       getHLFIRVariableType(inputType, hasExplicitLbs);
   build(builder, result, {hlfirVariableType, inputType}, memref, shape,
-        typeparams, dummy_scope, nameAttr, fortran_attrs, cuda_attr);
+        typeparams, dummy_scope, nameAttr, fortran_attrs, data_attr);
 }
 
 mlir::LogicalResult hlfir::DeclareOp::verify() {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt b/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
index ad569ce3b41f..fa3a59303137 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
@@ -11,11 +11,13 @@ add_flang_library(HLFIRTransforms
   OptimizedBufferization.cpp
 
   DEPENDS
+  CUFAttrsIncGen
   FIRDialect
   HLFIROpsIncGen
   ${dialect_libs}
 
   LINK_LIBS
+  CUFAttrs
   FIRAnalysis
   FIRDialect
   FIRBuilder
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
index 3570e0011ca7..e56595d1c8e2 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
@@ -320,16 +320,16 @@ public:
     mlir::Location loc = declareOp->getLoc();
     mlir::Value memref = declareOp.getMemref();
     fir::FortranVariableFlagsAttr fortranAttrs;
-    fir::CUDADataAttributeAttr cudaAttr;
+    cuf::DataAttributeAttr dataAttr;
     if (auto attrs = declareOp.getFortranAttrs())
       fortranAttrs =
           fir::FortranVariableFlagsAttr::get(rewriter.getContext(), *attrs);
-    if (auto attr = declareOp.getCudaAttr())
-      cudaAttr = fir::CUDADataAttributeAttr::get(rewriter.getContext(), *attr);
+    if (auto attr = declareOp.getDataAttr())
+      dataAttr = cuf::DataAttributeAttr::get(rewriter.getContext(), *attr);
     auto firDeclareOp = rewriter.create<fir::DeclareOp>(
         loc, memref.getType(), memref, declareOp.getShape(),
         declareOp.getTypeparams(), declareOp.getDummyScope(),
-        declareOp.getUniqName(), fortranAttrs, cudaAttr);
+        declareOp.getUniqName(), fortranAttrs, dataAttr);
 
     // Propagate other attributes from hlfir.declare to fir.declare.
     // OpenACC's acc.declare is one example. Right now, the propagation
diff --git a/flang/test/Fir/cuf-invalid.fir b/flang/test/Fir/cuf-invalid.fir
index 5a12e3c1a4bf..6e18e48ac82f 100644
--- a/flang/test/Fir/cuf-invalid.fir
+++ b/flang/test/Fir/cuf-invalid.fir
@@ -4,11 +4,11 @@ func.func @_QPsub1() {
   %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
   %1 = fir.alloca i32
   %pinned = fir.alloca i1
-  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
   %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
   %s = fir.load %1 : !fir.ref<i32>
-  // expected-error@+1{{'fir.cuda_allocate' op pinned and stream cannot appears at the same time}}
-  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> stream(%s : i32) pinned(%pinned : !fir.ref<i1>) {cuda_attr = #fir.cuda<device>} -> i32
+  // expected-error@+1{{'cuf.allocate' op pinned and stream cannot appears at the same time}}
+  %13 = cuf.allocate %11 : !fir.ref<!fir.box<none>> stream(%s : i32) pinned(%pinned : !fir.ref<i1>) {data_attr = #cuf.cuda<device>} -> i32
   return
 }
 
@@ -16,8 +16,8 @@ func.func @_QPsub1() {
 
 func.func @_QPsub1() {
   %1 = fir.alloca i32
-  // expected-error@+1{{'fir.cuda_allocate' op expect box to be a reference to a class or box type value}}
-  %2 = fir.cuda_allocate %1 : !fir.ref<i32> {cuda_attr = #fir.cuda<device>} -> i32
+  // expected-error@+1{{'cuf.allocate' op expect box to be a reference to a class or box type value}}
+  %2 = cuf.allocate %1 : !fir.ref<i32> {data_attr = #cuf.cuda<device>} -> i32
   return
 }
 
@@ -25,15 +25,15 @@ func.func @_QPsub1() {
 
 func.func @_QPsub1() {
   %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
-  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
   %c100 = arith.constant 100 : index
   %7 = fir.alloca !fir.char<1,100> {bindc_name = "msg", uniq_name = "_QFsub1Emsg"}
   %8:2 = hlfir.declare %7 typeparams %c100 {uniq_name = "_QFsub1Emsg"} : (!fir.ref<!fir.char<1,100>>, index) -> (!fir.ref<!fir.char<1,100>>, !fir.ref<!fir.char<1,100>>)
   %9 = fir.embox %8#1 : (!fir.ref<!fir.char<1,100>>) -> !fir.box<!fir.char<1,100>>
   %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
   %16 = fir.convert %9 : (!fir.box<!fir.char<1,100>>) -> !fir.box<none>
-  // expected-error@+1{{'fir.cuda_allocate' op expect stat attribute when errmsg is provided}}
-  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> errmsg(%16 : !fir.box<none>) {cuda_attr = #fir.cuda<device>} -> i32
+  // expected-error@+1{{'cuf.allocate' op expect stat attribute when errmsg is provided}}
+  %13 = cuf.allocate %11 : !fir.ref<!fir.box<none>> errmsg(%16 : !fir.box<none>) {data_attr = #cuf.cuda<device>} -> i32
   return
 }
 
@@ -41,11 +41,11 @@ func.func @_QPsub1() {
 
 func.func @_QPsub1() {
   %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
-  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
   %1 = fir.alloca i32
   %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
-  // expected-error@+1{{'fir.cuda_allocate' op expect errmsg to be a reference to/or a box type value}}
-  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> errmsg(%1 : !fir.ref<i32>) {cuda_attr = #fir.cuda<device>, hasStat} -> i32
+  // expected-error@+1{{'cuf.allocate' op expect errmsg to be a reference to/or a box type value}}
+  %13 = cuf.allocate %11 : !fir.ref<!fir.box<none>> errmsg(%1 : !fir.ref<i32>) {data_attr = #cuf.cuda<device>, hasStat} -> i32
   return
 }
 
@@ -53,8 +53,8 @@ func.func @_QPsub1() {
 
 func.func @_QPsub1() {
   %1 = fir.alloca i32
-  // expected-error@+1{{'fir.cuda_deallocate' op expect box to be a reference to class or box type value}}
-  %2 = fir.cuda_deallocate %1 : !fir.ref<i32> {cuda_attr = #fir.cuda<device>} -> i32
+  // expected-error@+1{{'cuf.deallocate' op expect box to be a reference to class or box type value}}
+  %2 = cuf.deallocate %1 : !fir.ref<i32> {data_attr = #cuf.cuda<device>} -> i32
   return
 }
 
@@ -62,11 +62,11 @@ func.func @_QPsub1() {
 
 func.func @_QPsub1() {
   %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
-  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
   %1 = fir.alloca i32
   %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
-  // expected-error@+1{{'fir.cuda_deallocate' op expect errmsg to be a reference to/or a box type value}}
-  %13 = fir.cuda_deallocate %11 : !fir.ref<!fir.box<none>> errmsg(%1 : !fir.ref<i32>) {cuda_attr = #fir.cuda<device>, hasStat} -> i32
+  // expected-error@+1{{'cuf.deallocate' op expect errmsg to be a reference to/or a box type value}}
+  %13 = cuf.deallocate %11 : !fir.ref<!fir.box<none>> errmsg(%1 : !fir.ref<i32>) {data_attr = #cuf.cuda<device>, hasStat} -> i32
   return
 }
 
@@ -74,32 +74,32 @@ func.func @_QPsub1() {
 
 func.func @_QPsub1() {
   %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
-  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
   %c100 = arith.constant 100 : index
   %7 = fir.alloca !fir.char<1,100> {bindc_name = "msg", uniq_name = "_QFsub1Emsg"}
   %8:2 = hlfir.declare %7 typeparams %c100 {uniq_name = "_QFsub1Emsg"} : (!fir.ref<!fir.char<1,100>>, index) -> (!fir.ref<!fir.char<1,100>>, !fir.ref<!fir.char<1,100>>)
   %9 = fir.embox %8#1 : (!fir.ref<!fir.char<1,100>>) -> !fir.box<!fir.char<1,100>>
   %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
   %16 = fir.convert %9 : (!fir.box<!fir.char<1,100>>) -> !fir.box<none>
-  // expected-error@+1{{'fir.cuda_deallocate' op expect stat attribute when errmsg is provided}}
-  %13 = fir.cuda_deallocate %11 : !fir.ref<!fir.box<none>> errmsg(%16 : !fir.box<none>) {cuda_attr = #fir.cuda<device>} -> i32
+  // expected-error@+1{{'cuf.deallocate' op expect stat attribute when errmsg is provided}}
+  %13 = cuf.deallocate %11 : !fir.ref<!fir.box<none>> errmsg(%16 : !fir.box<none>) {data_attr = #cuf.cuda<device>} -> i32
   return
 }
 
 // -----
 
 func.func @_QPsub1() {
-  // expected-error@+1{{'fir.cuda_alloc' op expect device, managed or unified cuda attribute}}
-  %0 = fir.cuda_alloc f32 {bindc_name = "r", cuda_attr = #fir.cuda<pinned>, uniq_name = "_QFsub1Er"} -> !fir.ref<f32>
-  fir.cuda_free %0 : !fir.ref<f32> {cuda_attr = #fir.cuda<constant>}
+  // expected-error@+1{{'cuf.alloc' op expect device, managed or unified cuda attribute}}
+  %0 = cuf.alloc f32 {bindc_name = "r", data_attr = #cuf.cuda<pinned>, uniq_name = "_QFsub1Er"} -> !fir.ref<f32>
+  cuf.free %0 : !fir.ref<f32> {data_attr = #cuf.cuda<constant>}
   return
 }
 
 // -----
 
 func.func @_QPsub1() {
-  %0 = fir.cuda_alloc f32 {bindc_name = "r", cuda_attr = #fir.cuda<device>, uniq_name = "_QFsub1Er"} -> !fir.ref<f32>
-  // expected-error@+1{{'fir.cuda_free' op expect device, managed or unified cuda attribute}}
-  fir.cuda_free %0 : !fir.ref<f32> {cuda_attr = #fir.cuda<constant>}
+  %0 = cuf.alloc f32 {bindc_name = "r", data_attr = #cuf.cuda<device>, uniq_name = "_QFsub1Er"} -> !fir.ref<f32>
+  // expected-error@+1{{'cuf.free' op expect device, managed or unified cuda attribute}}
+  cuf.free %0 : !fir.ref<f32> {data_attr = #cuf.cuda<constant>}
   return
 }
diff --git a/flang/test/Fir/cuf.mlir b/flang/test/Fir/cuf.mlir
index 8e2346def43e..188044d04b84 100644
--- a/flang/test/Fir/cuf.mlir
+++ b/flang/test/Fir/cuf.mlir
@@ -4,85 +4,85 @@
 
 func.func @_QPsub1() {
   %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
-  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
   %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
-  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> {cuda_attr = #fir.cuda<device>} -> i32
-  %14 = fir.cuda_deallocate %11 : !fir.ref<!fir.box<none>> {cuda_attr = #fir.cuda<device>} -> i32
+  %13 = cuf.allocate %11 : !fir.ref<!fir.box<none>> {data_attr = #cuf.cuda<device>} -> i32
+  %14 = cuf.deallocate %11 : !fir.ref<!fir.box<none>> {data_attr = #cuf.cuda<device>} -> i32
   return
 }
 
-// CHECK: fir.cuda_allocate %{{.*}} : !fir.ref<!fir.box<none>> {cuda_attr = #fir.cuda<device>} -> i32
-// CHECK: fir.cuda_deallocate %{{.*}} : !fir.ref<!fir.box<none>> {cuda_attr = #fir.cuda<device>} -> i32
+// CHECK: cuf.allocate %{{.*}} : !fir.ref<!fir.box<none>> {data_attr = #cuf.cuda<device>} -> i32
+// CHECK: cuf.deallocate %{{.*}} : !fir.ref<!fir.box<none>> {data_attr = #cuf.cuda<device>} -> i32
 
 // -----
 
 func.func @_QPsub1() {
   %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
   %1 = fir.alloca i32
-  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
   %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
   %s = fir.load %1 : !fir.ref<i32>
-  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> stream(%s : i32) {cuda_attr = #fir.cuda<device>} -> i32
+  %13 = cuf.allocate %11 : !fir.ref<!fir.box<none>> stream(%s : i32) {data_attr = #cuf.cuda<device>} -> i32
   return
 }
 
-// CHECK: fir.cuda_allocate %{{.*}} : !fir.ref<!fir.box<none>> stream(%{{.*}} : i32) {cuda_attr = #fir.cuda<device>} -> i32
+// CHECK: cuf.allocate %{{.*}} : !fir.ref<!fir.box<none>> stream(%{{.*}} : i32) {data_attr = #cuf.cuda<device>} -> i32
 
 // -----
 
 func.func @_QPsub1() {
   %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
   %1 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "b", uniq_name = "_QFsub1Eb"}
-  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
   %5:2 = hlfir.declare %1 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
   %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
   %12 = fir.convert %5#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
-  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> source(%12 : !fir.ref<!fir.box<none>>) {cuda_attr = #fir.cuda<device>} -> i32
+  %13 = cuf.allocate %11 : !fir.ref<!fir.box<none>> source(%12 : !fir.ref<!fir.box<none>>) {data_attr = #cuf.cuda<device>} -> i32
   return
 }
 
-// CHECK: fir.cuda_allocate %{{.*}} : !fir.ref<!fir.box<none>> source(%{{.*}} : !fir.ref<!fir.box<none>>) {cuda_attr = #fir.cuda<device>} -> i32
+// CHECK: cuf.allocate %{{.*}} : !fir.ref<!fir.box<none>> source(%{{.*}} : !fir.ref<!fir.box<none>>) {data_attr = #cuf.cuda<device>} -> i32
 
 // -----
 
 func.func @_QPsub1() {
   %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
   %pinned = fir.alloca i1
-  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
   %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
-  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> pinned(%pinned : !fir.ref<i1>) {cuda_attr = #fir.cuda<device>} -> i32
+  %13 = cuf.allocate %11 : !fir.ref<!fir.box<none>> pinned(%pinned : !fir.ref<i1>) {data_attr = #cuf.cuda<device>} -> i32
   return
 }
 
-// CHECK: fir.cuda_allocate %{{.*}} : !fir.ref<!fir.box<none>> pinned(%{{.*}} : !fir.ref<i1>) {cuda_attr = #fir.cuda<device>} -> i32
+// CHECK: cuf.allocate %{{.*}} : !fir.ref<!fir.box<none>> pinned(%{{.*}} : !fir.ref<i1>) {data_attr = #cuf.cuda<device>} -> i32
 
 // -----
 
 func.func @_QPsub1() {
   %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
-  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %4:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
   %c100 = arith.constant 100 : index
   %7 = fir.alloca !fir.char<1,100> {bindc_name = "msg", uniq_name = "_QFsub1Emsg"}
   %8:2 = hlfir.declare %7 typeparams %c100 {uniq_name = "_QFsub1Emsg"} : (!fir.ref<!fir.char<1,100>>, index) -> (!fir.ref<!fir.char<1,100>>, !fir.ref<!fir.char<1,100>>)
   %9 = fir.embox %8#1 : (!fir.ref<!fir.char<1,100>>) -> !fir.box<!fir.char<1,100>>
   %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
   %16 = fir.convert %9 : (!fir.box<!fir.char<1,100>>) -> !fir.box<none>
-  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> errmsg(%16 : !fir.box<none>) {cuda_attr = #fir.cuda<device>, hasStat} -> i32
-  %14 = fir.cuda_deallocate %11 : !fir.ref<!fir.box<none>> errmsg(%16 : !fir.box<none>) {cuda_attr = #fir.cuda<device>, hasStat} -> i32
+  %13 = cuf.allocate %11 : !fir.ref<!fir.box<none>> errmsg(%16 : !fir.box<none>) {data_attr = #cuf.cuda<device>, hasStat} -> i32
+  %14 = cuf.deallocate %11 : !fir.ref<!fir.box<none>> errmsg(%16 : !fir.box<none>) {data_attr = #cuf.cuda<device>, hasStat} -> i32
   return
 }
 
-// CHECK: fir.cuda_allocate %{{.*}} : !fir.ref<!fir.box<none>> errmsg(%{{.*}} : !fir.box<none>) {cuda_attr = #fir.cuda<device>, hasStat} -> i32
-// CHECK: fir.cuda_deallocate %{{.*}} : !fir.ref<!fir.box<none>> errmsg(%{{.*}} : !fir.box<none>) {cuda_attr = #fir.cuda<device>, hasStat} -> i32
+// CHECK: cuf.allocate %{{.*}} : !fir.ref<!fir.box<none>> errmsg(%{{.*}} : !fir.box<none>) {data_attr = #cuf.cuda<device>, hasStat} -> i32
+// CHECK: cuf.deallocate %{{.*}} : !fir.ref<!fir.box<none>> errmsg(%{{.*}} : !fir.box<none>) {data_attr = #cuf.cuda<device>, hasStat} -> i32
 
 // -----
 
 func.func @_QPsub1() {
-  %0 = fir.cuda_alloc f32 {bindc_name = "r", cuda_attr = #fir.cuda<device>, uniq_name = "_QFsub1Er"} -> !fir.ref<f32>
-  fir.cuda_free %0 : !fir.ref<f32> {cuda_attr = #fir.cuda<device>}
+  %0 = cuf.alloc f32 {bindc_name = "r", data_attr = #cuf.cuda<device>, uniq_name = "_QFsub1Er"} -> !fir.ref<f32>
+  cuf.free %0 : !fir.ref<f32> {data_attr = #cuf.cuda<device>}
   return
 }
 
-// CHECK: fir.cuda_alloc
-// CHECK: fir.cuda_free
+// CHECK: cuf.alloc
+// CHECK: cuf.free
 
diff --git a/flang/test/Lower/CUDA/cuda-allocatable.cuf b/flang/test/Lower/CUDA/cuda-allocatable.cuf
index eff5f13669e9..74a3ec100a8f 100644
--- a/flang/test/Lower/CUDA/cuda-allocatable.cuf
+++ b/flang/test/Lower/CUDA/cuda-allocatable.cuf
@@ -11,11 +11,11 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub1()
 ! CHECK: %[[BOX:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
-! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 ! CHECK: fir.call @_FortranAAllocatableSetBounds
-! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuda_attr = #fir.cuda<device>} -> i32
+! CHECK: %{{.*}} = cuf.allocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>} -> i32
 
-! CHECK: %{{.*}} = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuda_attr = #fir.cuda<device>} -> i32
+! CHECK: %{{.*}} = cuf.deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>} -> i32
 
 ! CHECK: %[[BOX_LOAD:.*]] = fir.load %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: %[[ADDR:.*]] = fir.box_addr %[[BOX_LOAD]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
@@ -23,7 +23,7 @@ end subroutine
 ! CHECK: %[[C0:.*]] = arith.constant 0 : i64
 ! CHECK: %[[NE_C0:.*]] = arith.cmpi ne, %[[ADDR_I64]], %[[C0]] : i64
 ! CHECK: fir.if %[[NE_C0]] {
-! CHECK:   %{{.*}} = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuda_attr = #fir.cuda<device>} -> i32
+! CHECK:   %{{.*}} = cuf.deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>} -> i32
 ! CHECK: }
 
 subroutine sub2()
@@ -36,18 +36,18 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub2()
 ! CHECK: %[[BOX:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub2Ea"}
-! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub2Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub2Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 ! CHECK: %[[ISTAT:.*]] = fir.alloca i32 {bindc_name = "istat", uniq_name = "_QFsub2Eistat"}
 ! CHECK: %[[ISTAT_DECL:.*]]:2 = hlfir.declare %[[ISTAT]] {uniq_name = "_QFsub2Eistat"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: fir.call @_FortranAAllocatableSetBounds
-! CHECK: %[[STAT:.*]] = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuda_attr = #fir.cuda<managed>, hasStat} -> i32
+! CHECK: %[[STAT:.*]] = cuf.allocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<managed>, hasStat} -> i32
 ! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref<i32>
 
-! CHECK: %[[STAT:.*]] = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuda_attr = #fir.cuda<managed>, hasStat} -> i32
+! CHECK: %[[STAT:.*]] = cuf.deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<managed>, hasStat} -> i32
 ! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref<i32>
 
 ! CHECK: fir.if %{{.*}} {
-! CHECK:   %{{.*}} = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuda_attr = #fir.cuda<managed>} -> i32
+! CHECK:   %{{.*}} = cuf.deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<managed>} -> i32
 ! CHECK: }
 
 subroutine sub3()
@@ -58,13 +58,13 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub3()
 ! CHECK: %[[BOX:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xi32>>> {bindc_name = "a", uniq_name = "_QFsub3Ea"}
-! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub3Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {data_attr = #cuf.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub3Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
 ! CHECK: %[[PLOG:.*]] = fir.alloca !fir.logical<4> {bindc_name = "plog", uniq_name = "_QFsub3Eplog"}
 ! CHECK: %[[PLOG_DECL:.*]]:2 = hlfir.declare %5 {uniq_name = "_QFsub3Eplog"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK-2: fir.call @_FortranAAllocatableSetBounds
-! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> pinned(%[[PLOG_DECL]]#1 : !fir.ref<!fir.logical<4>>) {cuda_attr = #fir.cuda<pinned>} -> i32
+! CHECK: %{{.*}} = cuf.allocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> pinned(%[[PLOG_DECL]]#1 : !fir.ref<!fir.logical<4>>) {data_attr = #cuf.cuda<pinned>} -> i32
 ! CHECK: fir.if %{{.*}} {
-! CHECK:   %{{.*}} = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {cuda_attr = #fir.cuda<pinned>} -> i32
+! CHECK:   %{{.*}} = cuf.deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {data_attr = #cuf.cuda<pinned>} -> i32
 ! CHECK: }
 
 subroutine sub4()
@@ -75,14 +75,14 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub4()
 ! CHECK: %[[BOX:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub4Ea"}
-! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub4Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %0 {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub4Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 ! CHECK: %[[ISTREAM:.*]] = fir.alloca i32 {bindc_name = "istream", uniq_name = "_QFsub4Eistream"}
 ! CHECK: %[[ISTREAM_DECL:.*]]:2 = hlfir.declare %[[ISTREAM]] {uniq_name = "_QFsub4Eistream"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: fir.call @_FortranAAllocatableSetBounds
 ! CHECK: %[[STREAM:.*]] = fir.load %[[ISTREAM_DECL]]#0 : !fir.ref<i32>
-! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> stream(%[[STREAM]] : i32) {cuda_attr = #fir.cuda<device>} -> i32
+! CHECK: %{{.*}} = cuf.allocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> stream(%[[STREAM]] : i32) {data_attr = #cuf.cuda<device>} -> i32
 ! CHECK: fir.if %{{.*}} {
-! CHECK:   %{{.*}} = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuda_attr = #fir.cuda<device>} -> i32
+! CHECK:   %{{.*}} = cuf.deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>} -> i32
 ! CHECK: }
 
 subroutine sub5()
@@ -93,16 +93,16 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub5()
 ! CHECK: %[[BOX_A:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub5Ea"}
-! CHECK: %[[BOX_A_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub5Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+! CHECK: %[[BOX_A_DECL:.*]]:2 = hlfir.declare %[[BOX]] {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub5Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 ! CHECK: %[[BOX_B:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "b", uniq_name = "_QFsub5Eb"}
 ! CHECK: %[[BOX_B_DECL:.*]]:2 = hlfir.declare %[[BOX_B]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub5Eb"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 ! CHECK: %[[LOAD_B:.*]] = fir.load %[[BOX_B_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: fir.call @_FortranAAllocatableSetBounds
-! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_A_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> source(%[[LOAD_B]] : !fir.box<!fir.heap<!fir.array<?xf32>>>) {cuda_attr = #fir.cuda<device>} -> i32
+! CHECK: %{{.*}} = cuf.allocate %[[BOX_A_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> source(%[[LOAD_B]] : !fir.box<!fir.heap<!fir.array<?xf32>>>) {data_attr = #cuf.cuda<device>} -> i32
 ! CHECK: fir.if
 ! CHECK: fir.freemem
 ! CHECK: fir.if %{{.*}} {
-! CHECK:   %{{.*}} = fir.cuda_deallocate %[[BOX_A_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuda_attr = #fir.cuda<device>} -> i32
+! CHECK:   %{{.*}} = cuf.deallocate %[[BOX_A_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>} -> i32
 ! CHECK: }
 
 subroutine sub6()
@@ -113,14 +113,14 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub6()
 ! CHECK: %[[BOX_A:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub6Ea"}
-! CHECK: %[[BOX_A_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub6Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+! CHECK: %[[BOX_A_DECL:.*]]:2 = hlfir.declare %[[BOX]] {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub6Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 ! CHECK: %[[BOX_B:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "b", uniq_name = "_QFsub6Eb"}
 ! CHECK: %[[BOX_B_DECL:.*]]:2 = hlfir.declare %[[BOX_B]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub6Eb"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 ! CHECK: %[[LOAD_B:.*]] = fir.load %[[BOX_B_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
 ! CHECK: fir.call @_FortranAAllocatableApplyMold
-! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_A_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuda_attr = #fir.cuda<device>} -> i32
+! CHECK: %{{.*}} = cuf.allocate %[[BOX_A_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>} -> i32
 ! CHECK: fir.if %{{.*}} {
-! CHECK:   %{{.*}} = fir.cuda_deallocate %[[BOX_A_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuda_attr = #fir.cuda<device>} -> i32
+! CHECK:   %{{.*}} = cuf.deallocate %[[BOX_A_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>} -> i32
 ! CHECK: }
 
 subroutine sub7()
@@ -134,19 +134,19 @@ end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub7()
 ! CHECK: %[[BOX:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub7Ea"}
-! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub7Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {data_attr = #cuf.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub7Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
 ! CHECK: %[[ERR:.*]] = fir.alloca !fir.char<1,50> {bindc_name = "err", uniq_name = "_QFsub7Eerr"}
 ! CHECK: %[[ERR_DECL:.*]]:2 = hlfir.declare %[[ERR]] typeparams %{{.*}} {uniq_name = "_QFsub7Eerr"} : (!fir.ref<!fir.char<1,50>>, index) -> (!fir.ref<!fir.char<1,50>>, !fir.ref<!fir.char<1,50>>)
 ! CHECK: %[[ISTAT:.*]] = fir.alloca i32 {bindc_name = "istat", uniq_name = "_QFsub7Eistat"}
 ! CHECK: %[[ISTAT_DECL:.*]]:2 = hlfir.declare %[[ISTAT]] {uniq_name = "_QFsub7Eistat"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: %[[ERR_BOX:.*]] = fir.embox %[[ERR_DECL]]#1 : (!fir.ref<!fir.char<1,50>>) -> !fir.box<!fir.char<1,50>>
 ! CHECK: fir.call @_FortranAAllocatableSetBounds
-! CHECK: %[[STAT:.*]] = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> errmsg(%[[ERR_BOX]] : !fir.box<!fir.char<1,50>>) {cuda_attr = #fir.cuda<device>, hasStat} -> i32
+! CHECK: %[[STAT:.*]] = cuf.allocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> errmsg(%[[ERR_BOX]] : !fir.box<!fir.char<1,50>>) {data_attr = #cuf.cuda<device>, hasStat} -> i32
 ! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref<i32>
 
 ! CHECK: %[[ERR_BOX:.*]] = fir.embox %[[ERR_DECL]]#1 : (!fir.ref<!fir.char<1,50>>) -> !fir.box<!fir.char<1,50>>
-! CHECK: %[[STAT:.*]] = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> errmsg(%15 : !fir.box<!fir.char<1,50>>) {cuda_attr = #fir.cuda<device>, hasStat} -> i32
+! CHECK: %[[STAT:.*]] = cuf.deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> errmsg(%15 : !fir.box<!fir.char<1,50>>) {data_attr = #cuf.cuda<device>, hasStat} -> i32
 ! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref<i32>
 ! CHECK: fir.if %{{.*}} {
-! CHECK:   %{{.*}} = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {cuda_attr = #fir.cuda<device>} -> i32
+! CHECK:   %{{.*}} = cuf.deallocate %[[BOX_DECL]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>> {data_attr = #cuf.cuda<device>} -> i32
 ! CHECK: }
diff --git a/flang/test/Lower/CUDA/cuda-data-attribute.cuf b/flang/test/Lower/CUDA/cuda-data-attribute.cuf
index 3eb42a6a5d40..f7f58a43a143 100644
--- a/flang/test/Lower/CUDA/cuda-data-attribute.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-attribute.cuf
@@ -5,13 +5,13 @@
 
 module cuda_var
   real, constant :: mod_a_rc
-! CHECK: fir.global @_QMcuda_varEmod_a_rc {cuda_attr = #fir.cuda<constant>} : f32 
+! CHECK: fir.global @_QMcuda_varEmod_a_rc {data_attr = #cuf.cuda<constant>} : f32 
   real, device :: mod_b_ra
-! CHECK: fir.global @_QMcuda_varEmod_b_ra {cuda_attr = #fir.cuda<device>} : f32
+! CHECK: fir.global @_QMcuda_varEmod_b_ra {data_attr = #cuf.cuda<device>} : f32
   real, allocatable, managed :: mod_c_rm
-! CHECK: fir.global @_QMcuda_varEmod_c_rm {cuda_attr = #fir.cuda<managed>} : !fir.box<!fir.heap<f32>>
+! CHECK: fir.global @_QMcuda_varEmod_c_rm {data_attr = #cuf.cuda<managed>} : !fir.box<!fir.heap<f32>>
   real, allocatable, pinned :: mod_d_rp
-! CHECK: fir.global @_QMcuda_varEmod_d_rp {cuda_attr = #fir.cuda<pinned>} : !fir.box<!fir.heap<f32>>
+! CHECK: fir.global @_QMcuda_varEmod_d_rp {data_attr = #cuf.cuda<pinned>} : !fir.box<!fir.heap<f32>>
 
 contains
 
@@ -23,44 +23,44 @@ subroutine local_var_attrs
 end subroutine
 
 ! CHECK-LABEL: func.func @_QMcuda_varPlocal_var_attrs()
-! CHECK: %{{.*}}:2 = hlfir.declare %{{.*}} {cuda_attr = #fir.cuda<device>, uniq_name = "_QMcuda_varFlocal_var_attrsErd"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-! CHECK: %{{.*}}:2 = hlfir.declare %{{.*}} {cuda_attr = #fir.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFlocal_var_attrsErm"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
-! CHECK: %{{.*}}:2 = hlfir.declare %{{.*}} {cuda_attr = #fir.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFlocal_var_attrsErp"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
-! CHECK: %{{.*}}:2 = hlfir.declare %{{.*}} {cuda_attr = #fir.cuda<unified>, uniq_name = "_QMcuda_varFlocal_var_attrsEru"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %{{.*}}:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QMcuda_varFlocal_var_attrsErd"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %{{.*}}:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFlocal_var_attrsErm"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
+! CHECK: %{{.*}}:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFlocal_var_attrsErp"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
+! CHECK: %{{.*}}:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<unified>, uniq_name = "_QMcuda_varFlocal_var_attrsEru"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 
 
-! FIR: %{{.*}} = fir.declare %{{.*}} {cuda_attr = #fir.cuda<device>, uniq_name = "_QMcuda_varFlocal_var_attrsErd"} : (!fir.ref<f32>) -> !fir.ref<f32>
-! FIR: %{{.*}} = fir.declare %{{.*}} {cuda_attr = #fir.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFlocal_var_attrsErm"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> !fir.ref<!fir.box<!fir.heap<f32>>>
-! FIR: %{{.*}} = fir.declare %{{.*}} {cuda_attr = #fir.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFlocal_var_attrsErp"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> !fir.ref<!fir.box<!fir.heap<f32>>>
-! FIR: %{{.*}} = fir.declare %{{.*}} {cuda_attr = #fir.cuda<unified>, uniq_name = "_QMcuda_varFlocal_var_attrsEru"} : (!fir.ref<f32>) -> !fir.ref<f32>
+! FIR: %{{.*}} = fir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QMcuda_varFlocal_var_attrsErd"} : (!fir.ref<f32>) -> !fir.ref<f32>
+! FIR: %{{.*}} = fir.declare %{{.*}} {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFlocal_var_attrsErm"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> !fir.ref<!fir.box<!fir.heap<f32>>>
+! FIR: %{{.*}} = fir.declare %{{.*}} {data_attr = #cuf.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFlocal_var_attrsErp"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> !fir.ref<!fir.box<!fir.heap<f32>>>
+! FIR: %{{.*}} = fir.declare %{{.*}} {data_attr = #cuf.cuda<unified>, uniq_name = "_QMcuda_varFlocal_var_attrsEru"} : (!fir.ref<f32>) -> !fir.ref<f32>
 
 subroutine dummy_arg_device(dd)
   real, device :: dd
 end subroutine
 ! CHECK-LABEL: func.func @_QMcuda_varPdummy_arg_device(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<f32> {fir.bindc_name = "dd", fir.cuda_attr = #fir.cuda<device>}) {
-! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {cuda_attr = #fir.cuda<device>, uniq_name = "_QMcuda_varFdummy_arg_deviceEdd"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<f32> {cuf.data_attr = #cuf.cuda<device>, fir.bindc_name = "dd"}) {
+! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {data_attr = #cuf.cuda<device>, uniq_name = "_QMcuda_varFdummy_arg_deviceEdd"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 
 subroutine dummy_arg_managed(dm)
   real, allocatable, managed :: dm
 end subroutine
 ! CHECK-LABEL: func.func @_QMcuda_varPdummy_arg_managed(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<f32>>> {fir.bindc_name = "dm", fir.cuda_attr = #fir.cuda<managed>}) {
-! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {cuda_attr = #fir.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFdummy_arg_managedEdm"} : (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<f32>>> {cuf.data_attr = #cuf.cuda<managed>, fir.bindc_name = "dm"}) {
+! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFdummy_arg_managedEdm"} : (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
 
 subroutine dummy_arg_pinned(dp)
   real, allocatable, pinned :: dp
 end subroutine
 ! CHECK-LABEL: func.func @_QMcuda_varPdummy_arg_pinned(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<f32>>> {fir.bindc_name = "dp", fir.cuda_attr = #fir.cuda<pinned>}) {
-! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {cuda_attr = #fir.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFdummy_arg_pinnedEdp"} : (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<f32>>> {cuf.data_attr = #cuf.cuda<pinned>, fir.bindc_name = "dp"}) {
+! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {data_attr = #cuf.cuda<pinned>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMcuda_varFdummy_arg_pinnedEdp"} : (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
 
 subroutine dummy_arg_unified(du)
   real, unified :: du
 end subroutine
 ! CHECK-LABEL: func.func @_QMcuda_varPdummy_arg_unified(
-! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<f32> {fir.bindc_name = "du", fir.cuda_attr = #fir.cuda<unified>})
-! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {cuda_attr = #fir.cuda<unified>, uniq_name = "_QMcuda_varFdummy_arg_unifiedEdu"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<f32> {cuf.data_attr = #cuf.cuda<unified>, fir.bindc_name = "du"})
+! CHECK: %{{.*}}:2 = hlfir.declare %[[ARG0]] dummy_scope %{{[0-9]+}} {data_attr = #cuf.cuda<unified>, uniq_name = "_QMcuda_varFdummy_arg_unifiedEdu"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
 
 subroutine cuda_alloc_free(n)
   integer :: n
@@ -70,27 +70,27 @@ subroutine cuda_alloc_free(n)
 end
 
 ! CHECK-LABEL: func.func @_QMcuda_varPcuda_alloc_free
-! CHECK: %[[ALLOC_A:.*]] = fir.cuda_alloc !fir.array<10xf32> {bindc_name = "a", cuda_attr = #fir.cuda<device>, uniq_name = "_QMcuda_varFcuda_alloc_freeEa"} -> !fir.ref<!fir.array<10xf32>>
+! CHECK: %[[ALLOC_A:.*]] = cuf.alloc !fir.array<10xf32> {bindc_name = "a", data_attr = #cuf.cuda<device>, uniq_name = "_QMcuda_varFcuda_alloc_freeEa"} -> !fir.ref<!fir.array<10xf32>>
 ! CHECK: %[[SHAPE:.*]] = fir.shape %c10 : (index) -> !fir.shape<1>
-! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ALLOC_A]](%[[SHAPE]]) {cuda_attr = #fir.cuda<device>, uniq_name = "_QMcuda_varFcuda_alloc_freeEa"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
+! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ALLOC_A]](%[[SHAPE]]) {data_attr = #cuf.cuda<device>, uniq_name = "_QMcuda_varFcuda_alloc_freeEa"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 
-! CHECK: %[[ALLOC_U:.*]] = fir.cuda_alloc i32 {bindc_name = "u", cuda_attr = #fir.cuda<unified>, uniq_name = "_QMcuda_varFcuda_alloc_freeEu"} -> !fir.ref<i32>
-! CHECK: %[[DECL_U:.*]]:2 = hlfir.declare %[[ALLOC_U]] {cuda_attr = #fir.cuda<unified>, uniq_name = "_QMcuda_varFcuda_alloc_freeEu"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[ALLOC_U:.*]] = cuf.alloc i32 {bindc_name = "u", data_attr = #cuf.cuda<unified>, uniq_name = "_QMcuda_varFcuda_alloc_freeEu"} -> !fir.ref<i32>
+! CHECK: %[[DECL_U:.*]]:2 = hlfir.declare %[[ALLOC_U]] {data_attr = #cuf.cuda<unified>, uniq_name = "_QMcuda_varFcuda_alloc_freeEu"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
-! CHECK: %[[ALLOC_B:.*]] = fir.cuda_alloc !fir.array<?xf32>, %{{.*}} : index {bindc_name = "b", cuda_attr = #fir.cuda<managed>, uniq_name = "_QMcuda_varFcuda_alloc_freeEb"} -> !fir.ref<!fir.array<?xf32>>
+! CHECK: %[[ALLOC_B:.*]] = cuf.alloc !fir.array<?xf32>, %{{.*}} : index {bindc_name = "b", data_attr = #cuf.cuda<managed>, uniq_name = "_QMcuda_varFcuda_alloc_freeEb"} -> !fir.ref<!fir.array<?xf32>>
 ! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
-! CHECK: %[[DECL_B:.*]]:2 = hlfir.declare %[[ALLOC_B]](%[[SHAPE]]) {cuda_attr = #fir.cuda<managed>, uniq_name = "_QMcuda_varFcuda_alloc_freeEb"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+! CHECK: %[[DECL_B:.*]]:2 = hlfir.declare %[[ALLOC_B]](%[[SHAPE]]) {data_attr = #cuf.cuda<managed>, uniq_name = "_QMcuda_varFcuda_alloc_freeEb"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
 
-! CHECK: fir.cuda_free %[[DECL_B]]#1 : !fir.ref<!fir.array<?xf32>> {cuda_attr = #fir.cuda<managed>}
-! CHECK: fir.cuda_free %[[DECL_U]]#1 : !fir.ref<i32> {cuda_attr = #fir.cuda<unified>}
-! CHECK: fir.cuda_free %[[DECL_A]]#1 : !fir.ref<!fir.array<10xf32>> {cuda_attr = #fir.cuda<device>}
+! CHECK: cuf.free %[[DECL_B]]#1 : !fir.ref<!fir.array<?xf32>> {data_attr = #cuf.cuda<managed>}
+! CHECK: cuf.free %[[DECL_U]]#1 : !fir.ref<i32> {data_attr = #cuf.cuda<unified>}
+! CHECK: cuf.free %[[DECL_A]]#1 : !fir.ref<!fir.array<10xf32>> {data_attr = #cuf.cuda<device>}
 
 subroutine dummy(x)
   real, target, device :: x
 end subroutine
 
 ! CHECK: func.func @_QMcuda_varPdummy
-! CHECK-NOT: fir.cuda_free
+! CHECK-NOT: cuf.free
 
 end module
 
diff --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index 0a2608639bce..084314ed63ec 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -29,37 +29,37 @@ end
 
 ! CHECK-LABEL: func.func @_QPsub1()
 
-! CHECK: %[[ADEV:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {cuda_attr = #fir.cuda<device>, uniq_name = "_QFsub1Eadev"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK: %[[ADEV:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {data_attr = #cuf.cuda<device>, uniq_name = "_QFsub1Eadev"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK: %[[AHOST:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFsub1Eahost"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK: %[[I:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsub1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK: %[[M:.*]]:2 = hlfir.declare %{{.*}} {cuda_attr = #fir.cuda<device>, uniq_name = "_QFsub1Em"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[M:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QFsub1Em"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
 ! CHECK: %[[C1:.*]] = arith.constant 1 : i32
 ! CHECK: %[[LOADED_I:.*]] = fir.load %[[I]]#0 : !fir.ref<i32>
 ! CHECK: %[[ADD:.*]] = arith.addi %[[C1]], %[[LOADED_I]] : i32
 ! CHECK: %[[ASSOC:.*]]:3 = hlfir.associate %[[ADD]] {uniq_name = ".cuf_host_tmp"} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
-! CHECK: fir.cuda_data_transfer %[[ASSOC]]#0 to %[[M]]#0 {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<i32>, !fir.ref<i32>
+! CHECK: cuf.data_transfer %[[ASSOC]]#0 to %[[M]]#0 {transfer_kind = #cuf.cuda_transfer<host_device>} : !fir.ref<i32>, !fir.ref<i32>
 ! CHECK: hlfir.end_associate %[[ASSOC]]#1, %[[ASSOC]]#2 : !fir.ref<i32>, i1
 
 ! CHECK: %[[C1:.*]] = arith.constant 1 : i32
 ! CHECK: %[[ASSOC:.*]]:3 = hlfir.associate %[[C1]] {uniq_name = ".cuf_host_tmp"} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
-! CHECK: fir.cuda_data_transfer %[[ASSOC]]#0 to %[[M]]#0 {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<i32>, !fir.ref<i32>
+! CHECK: cuf.data_transfer %[[ASSOC]]#0 to %[[M]]#0 {transfer_kind = #cuf.cuda_transfer<host_device>} : !fir.ref<i32>, !fir.ref<i32>
 ! CHECK: hlfir.end_associate %[[ASSOC]]#1, %[[ASSOC]]#2 : !fir.ref<i32>, i1
 
-! CHECK: fir.cuda_data_transfer %[[AHOST]]#0 to %[[ADEV]]#0 {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
+! CHECK: cuf.data_transfer %[[AHOST]]#0 to %[[ADEV]]#0 {transfer_kind = #cuf.cuda_transfer<host_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
 
 ! CHECK: %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<10xi32> {
 ! CHECK: %[[ASSOC:.*]]:3 = hlfir.associate %[[ELEMENTAL]](%{{.*}}) {uniq_name = ".cuf_host_tmp"} : (!hlfir.expr<10xi32>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>, i1)
-! CHECK: fir.cuda_data_transfer %[[ASSOC]]#0 to %[[ADEV]]#0 {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
+! CHECK: cuf.data_transfer %[[ASSOC]]#0 to %[[ADEV]]#0 {transfer_kind = #cuf.cuda_transfer<host_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
 ! CHECK: hlfir.end_associate %[[ASSOC]]#1, %[[ASSOC]]#2 : !fir.ref<!fir.array<10xi32>>, i1
 
 ! CHECK: %[[DES_AHOST:.*]] = hlfir.designate %[[AHOST]]#0 (%c1{{.*}}:%c5{{.*}}:%c1{{.*}})  shape %{{.*}} : (!fir.ref<!fir.array<10xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<5xi32>>
 ! CHECK: %[[DES_ADEV:.*]] = hlfir.designate %[[ADEV]]#0 (%c1{{.*}}:%c5{{.*}}:%c1{{.*}})  shape %{{.*}} : (!fir.ref<!fir.array<10xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<5xi32>>
-! CHECK: fir.cuda_data_transfer %[[DES_AHOST]] to %[[DES_ADEV]] {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<!fir.array<5xi32>>, !fir.ref<!fir.array<5xi32>>
+! CHECK: cuf.data_transfer %[[DES_AHOST]] to %[[DES_ADEV]] {transfer_kind = #cuf.cuda_transfer<host_device>} : !fir.ref<!fir.array<5xi32>>, !fir.ref<!fir.array<5xi32>>
 
 ! CHECK: %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<10xi32>
 ! CHECK: %[[ASSOC:.*]]:3 = hlfir.associate %[[ELEMENTAL]](%{{.*}}) {uniq_name = ".cuf_host_tmp"} : (!hlfir.expr<10xi32>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>, i1)
-! CHECK: fir.cuda_data_transfer %[[ASSOC]]#0 to %[[ADEV]]#0 {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
+! CHECK: cuf.data_transfer %[[ASSOC]]#0 to %[[ADEV]]#0 {transfer_kind = #cuf.cuda_transfer<host_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
 ! CHECK: hlfir.end_associate %[[ASSOC]]#1, %[[ASSOC]]#2 : !fir.ref<!fir.array<10xi32>>, i1
 
 subroutine sub2()
@@ -81,25 +81,25 @@ subroutine sub2()
 end
 
 ! CHECK-LABEL: func.func @_QPsub2()
-! CHECK: %[[ADEV:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {cuda_attr = #fir.cuda<device>, uniq_name = "_QFsub2Eadev"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK: %[[ADEV:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {data_attr = #cuf.cuda<device>, uniq_name = "_QFsub2Eadev"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK: %[[AHOST:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFsub2Eahost"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
-! CHECK: %[[BDEV:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {cuda_attr = #fir.cuda<device>, uniq_name = "_QFsub2Ebdev"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK: %[[BDEV:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {data_attr = #cuf.cuda<device>, uniq_name = "_QFsub2Ebdev"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK: %[[BHOST:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFsub2Ebhost"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK: %[[I:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsub2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK: %[[M:.*]]:2 = hlfir.declare %{{.*}} {cuda_attr = #fir.cuda<device>, uniq_name = "_QFsub2Em"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK: fir.cuda_data_transfer %[[ADEV]]#0 to %[[AHOST]]#0 {transfer_kind = #fir.cuda_transfer<device_host>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
-! CHECK: fir.cuda_data_transfer %[[M]]#0 to %[[I]]#0 {transfer_kind = #fir.cuda_transfer<device_host>} : !fir.ref<i32>, !fir.ref<i32>
+! CHECK: %[[M:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QFsub2Em"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: cuf.data_transfer %[[ADEV]]#0 to %[[AHOST]]#0 {transfer_kind = #cuf.cuda_transfer<device_host>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
+! CHECK: cuf.data_transfer %[[M]]#0 to %[[I]]#0 {transfer_kind = #cuf.cuda_transfer<device_host>} : !fir.ref<i32>, !fir.ref<i32>
 
 ! CHECK: %[[DES_ADEV:.*]] = hlfir.designate %[[ADEV]]#0 (%{{.*}}:%{{.*}}:%{{.*}})  shape %{{.*}} : (!fir.ref<!fir.array<10xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<5xi32>>
 ! CHECK: %[[DES_AHOST:.*]] = hlfir.designate %[[AHOST]]#0 (%{{.*}}:%{{.*}}:%{{.*}})  shape %{{.*}} : (!fir.ref<!fir.array<10xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<5xi32>>
-! CHECK: fir.cuda_data_transfer %[[DES_ADEV]] to %[[DES_AHOST]] {transfer_kind = #fir.cuda_transfer<device_host>} : !fir.ref<!fir.array<5xi32>>, !fir.ref<!fir.array<5xi32>>
+! CHECK: cuf.data_transfer %[[DES_ADEV]] to %[[DES_AHOST]] {transfer_kind = #cuf.cuda_transfer<device_host>} : !fir.ref<!fir.array<5xi32>>, !fir.ref<!fir.array<5xi32>>
 
-! CHECK: fir.cuda_data_transfer %[[ADEV]]#0 to %[[BDEV]]#0 {transfer_kind = #fir.cuda_transfer<device_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
+! CHECK: cuf.data_transfer %[[ADEV]]#0 to %[[BDEV]]#0 {transfer_kind = #cuf.cuda_transfer<device_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
 
 ! CHECK: %[[TEMP:.*]] = fir.allocmem !fir.array<10xi32> {bindc_name = ".tmp", uniq_name = ""}
 ! CHECK: %[[DECL_TEMP:.*]]:2 = hlfir.declare %[[TEMP]](%{{.*}}) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.heap<!fir.array<10xi32>>, !fir.heap<!fir.array<10xi32>>)
-! CHECK: %[[ADEV_TEMP:.*]]:2 = hlfir.declare %[[DECL_TEMP]]#1(%{{.*}}) {cuda_attr = #fir.cuda<device>, uniq_name = "_QFsub2Eadev"} : (!fir.heap<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.heap<!fir.array<10xi32>>, !fir.heap<!fir.array<10xi32>>)
-! CHECK: fir.cuda_data_transfer %[[ADEV]]#1 to %[[DECL_TEMP]]#0 {transfer_kind = #fir.cuda_transfer<device_host>} : !fir.ref<!fir.array<10xi32>>, !fir.heap<!fir.array<10xi32>>
+! CHECK: %[[ADEV_TEMP:.*]]:2 = hlfir.declare %[[DECL_TEMP]]#1(%{{.*}}) {data_attr = #cuf.cuda<device>, uniq_name = "_QFsub2Eadev"} : (!fir.heap<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.heap<!fir.array<10xi32>>, !fir.heap<!fir.array<10xi32>>)
+! CHECK: cuf.data_transfer %[[ADEV]]#1 to %[[DECL_TEMP]]#0 {transfer_kind = #cuf.cuda_transfer<device_host>} : !fir.ref<!fir.array<10xi32>>, !fir.heap<!fir.array<10xi32>>
 ! CHECK: %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<10xi32>
 ! CHECK: hlfir.assign %[[ELEMENTAL]] to %[[BHOST]]#0 : !hlfir.expr<10xi32>, !fir.ref<!fir.array<10xi32>>
 ! CHECK: fir.freemem %[[DECL_TEMP]]#0 : !fir.heap<!fir.array<10xi32>>
@@ -116,12 +116,12 @@ end
 ! CHECK: %[[TMP:.*]] = fir.alloca !fir.type<_QMmod1Tt1{i:i32}> {bindc_name = ".tmp"}
 ! CHECK: %[[AHOST:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFsub3Eahost"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
 ! CHECK: %[[BHOST:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFsub3Ebhost"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
-! CHECK: %[[T:.*]]:2 = hlfir.declare %7 {cuda_attr = #fir.cuda<device>, uniq_name = "_QFsub3Et"} : (!fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>) -> (!fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>, !fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>)
+! CHECK: %[[T:.*]]:2 = hlfir.declare %7 {data_attr = #cuf.cuda<device>, uniq_name = "_QFsub3Et"} : (!fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>) -> (!fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>, !fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>)
 ! CHECK: %[[TMP_DECL:.*]]:2 = hlfir.declare %0 {uniq_name = ".tmp"} : (!fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>) -> (!fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>, !fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>)
-! CHECK: fir.cuda_data_transfer %[[T]]#1 to %[[TMP_DECL]]#0 {transfer_kind = #fir.cuda_transfer<device_host>} : !fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>, !fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>
+! CHECK: cuf.data_transfer %[[T]]#1 to %[[TMP_DECL]]#0 {transfer_kind = #cuf.cuda_transfer<device_host>} : !fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>, !fir.ref<!fir.type<_QMmod1Tt1{i:i32}>>
 
 
-! Check that fir.cuda_data_transfer are not generated within cuf kernel
+! Check that cuf.data_transfer are not generated within cuf kernel
 subroutine sub4()
   integer, parameter :: n = 10
   real, device :: adev(n)
@@ -137,9 +137,9 @@ subroutine sub4()
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub4()
-! CHECK: fir.cuda_data_transfer
-! CHECK: fir.cuda_kernel<<<*, *>>>
-! CHECK-NOT: fir.cuda_data_transfer
+! CHECK: cuf.data_transfer
+! CHECK: cuf.kernel<<<*, *>>>
+! CHECK-NOT: cuf.data_transfer
 ! CHECK: hlfir.assign
 
 attributes(global) subroutine sub5(a)
@@ -149,7 +149,7 @@ attributes(global) subroutine sub5(a)
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub5
-! CHECK-NOT: fir.cuda_data_transfer
+! CHECK-NOT: cuf.data_transfer
 
 attributes(host,device) subroutine sub6(a)
   integer, device :: a
@@ -158,4 +158,4 @@ attributes(host,device) subroutine sub6(a)
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub6
-! CHECK: fir.cuda_data_transfer
+! CHECK: cuf.data_transfer
diff --git a/flang/test/Lower/CUDA/cuda-kernel-calls.cuf b/flang/test/Lower/CUDA/cuda-kernel-calls.cuf
index 7e28fbb2231a..82d1a61f8e15 100644
--- a/flang/test/Lower/CUDA/cuda-kernel-calls.cuf
+++ b/flang/test/Lower/CUDA/cuda-kernel-calls.cuf
@@ -16,10 +16,10 @@ contains
   subroutine host()
     real, device :: a
 ! CHECK-LABEL: func.func @_QMtest_callPhost()
-! CHECK: %[[A:.*]]:2 = hlfir.declare %{{.*}} {cuda_attr = #fir.cuda<device>, uniq_name = "_QMtest_callFhostEa"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %[[A:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QMtest_callFhostEa"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 
     call dev_kernel0<<<10, 20>>>()
-! CHECK: fir.cuda_kernel_launch @_QMtest_callPdev_kernel0<<<%c10{{.*}}, %c1{{.*}}, %c1{{.*}}, %c20{{.*}}, %c1{{.*}}, %c1{{.*}}>>>()
+! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel0<<<%c10{{.*}}, %c1{{.*}}, %c1{{.*}}, %c20{{.*}}, %c1{{.*}}, %c1{{.*}}>>>()
 
     call dev_kernel0<<< __builtin_dim3(1,1,4), __builtin_dim3(32,1,1) >>>
 ! CHECK: %[[ADDR_DIM3_GRID:.*]] = fir.address_of(@_QQro._QM__fortran_builtinsT__builtin_dim3.{{.*}}) : !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>
@@ -38,16 +38,16 @@ contains
 ! CHECK: %[[BLOCK_Y_LOAD:.*]] = fir.load %[[BLOCK_Y]] : !fir.ref<i32>
 ! CHECK: %[[BLOCK_Z:.*]] = hlfir.designate %[[DIM3_BLOCK]]#1{"z"}   : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>) -> !fir.ref<i32>
 ! CHECK: %[[BLOCK_Z_LOAD:.*]] = fir.load %[[BLOCK_Z]] : !fir.ref<i32>
-! CHECK: fir.cuda_kernel_launch @_QMtest_callPdev_kernel0<<<%[[GRID_X_LOAD]], %[[GRID_Y_LOAD]], %[[GRID_Z_LOAD]], %[[BLOCK_X_LOAD]], %[[BLOCK_Y_LOAD]], %[[BLOCK_Z_LOAD]]>>>()
+! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel0<<<%[[GRID_X_LOAD]], %[[GRID_Y_LOAD]], %[[GRID_Z_LOAD]], %[[BLOCK_X_LOAD]], %[[BLOCK_Y_LOAD]], %[[BLOCK_Z_LOAD]]>>>()
 
     call dev_kernel0<<<10, 20, 2>>>()
-! CHECK: fir.cuda_kernel_launch @_QMtest_callPdev_kernel0<<<%c10{{.*}}, %c1{{.*}}, %c1{{.*}}, %c20{{.*}}, %c1{{.*}}, %c1{{.*}}, %c2{{.*}}>>>() 
+! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel0<<<%c10{{.*}}, %c1{{.*}}, %c1{{.*}}, %c20{{.*}}, %c1{{.*}}, %c1{{.*}}, %c2{{.*}}>>>() 
 
     call dev_kernel0<<<10, 20, 2, 0>>>()
-! CHECK: fir.cuda_kernel_launch @_QMtest_callPdev_kernel0<<<%c10{{.*}}, %c1{{.*}}, %c1{{.*}}, %c20{{.*}}, %c1{{.*}}, %c1{{.*}}, %c2{{.*}}, %c0{{.*}}>>>()
+! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel0<<<%c10{{.*}}, %c1{{.*}}, %c1{{.*}}, %c20{{.*}}, %c1{{.*}}, %c1{{.*}}, %c2{{.*}}, %c0{{.*}}>>>()
 
     call dev_kernel1<<<1, 32>>>(a)
-! CHECK: fir.cuda_kernel_launch @_QMtest_callPdev_kernel1<<<%c1{{.*}}, %c1{{.*}}, %c1{{.*}}, %c32{{.*}}, %c1{{.*}}, %c1{{.*}}>>>(%1#1) : (!fir.ref<f32>)
+! CHECK: cuf.kernel_launch @_QMtest_callPdev_kernel1<<<%c1{{.*}}, %c1{{.*}}, %c1{{.*}}, %c32{{.*}}, %c1{{.*}}, %c1{{.*}}>>>(%1#1) : (!fir.ref<f32>)
   end
 
 end
diff --git a/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf b/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
index e1cc35772618..89de367b723f 100644
--- a/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
+++ b/flang/test/Lower/CUDA/cuda-kernel-loop-directive.cuf
@@ -20,7 +20,7 @@ subroutine sub1()
 ! CHECK: %[[LB:.*]] = fir.convert %c1{{.*}} : (i32) -> index
 ! CHECK: %[[UB:.*]] = fir.convert %c100{{.*}} : (i32) -> index
 ! CHECK: %[[STEP:.*]] = arith.constant 1 : index
-! CHECK: fir.cuda_kernel<<<%c1_i32, %c2_i32>>> (%[[ARG0:.*]] : index) = (%[[LB]] : index) to (%[[UB]] : index)  step (%[[STEP]] : index)
+! CHECK: cuf.kernel<<<%c1_i32, %c2_i32>>> (%[[ARG0:.*]] : index) = (%[[LB]] : index) to (%[[UB]] : index)  step (%[[STEP]] : index)
 ! CHECK-NOT: fir.do_loop
 ! CHECK: %[[ARG0_I32:.*]] = fir.convert %[[ARG0]] : (index) -> i32
 ! CHECK: fir.store %[[ARG0_I32]] to %[[IV]]#1 : !fir.ref<i32>
@@ -32,7 +32,7 @@ subroutine sub1()
     a(i) = a(i) * b(i)
   end do
 
-! CHECK: fir.cuda_kernel<<<*, *>>> (%{{.*}} : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index)
+! CHECK: cuf.kernel<<<*, *>>> (%{{.*}} : index) = (%{{.*}} : index) to (%{{.*}} : index)  step (%{{.*}} : index)
 
   !$cuf kernel do(2) <<< 1, (256,1) >>>
   do i = 1, n
@@ -41,7 +41,7 @@ subroutine sub1()
     end do
   end do
 
-! CHECK: fir.cuda_kernel<<<%c1{{.*}}, (%c256{{.*}}, %c1{{.*}})>>> (%[[ARG0:.*]] : index, %[[ARG1:.*]] : index) = (%{{.*}}, %{{.*}} : index, index) to (%{{.*}}, %{{.*}} : index, index) step (%{{.*}}, %{{.*}} : index, index)
+! CHECK: cuf.kernel<<<%c1{{.*}}, (%c256{{.*}}, %c1{{.*}})>>> (%[[ARG0:.*]] : index, %[[ARG1:.*]] : index) = (%{{.*}}, %{{.*}} : index, index) to (%{{.*}}, %{{.*}} : index, index) step (%{{.*}}, %{{.*}} : index, index)
 ! CHECK: %[[ARG0_I32:.*]] = fir.convert %[[ARG0]] : (index) -> i32
 ! CHECK: fir.store %[[ARG0_I32]] to %[[IV]]#1 : !fir.ref<i32>
 ! CHECK: %[[ARG1_I32:.*]] = fir.convert %[[ARG1]] : (index) -> i32
@@ -54,7 +54,7 @@ subroutine sub1()
       c(i,j) = c(i,j) * d(i,j)
     end do
   end do
-! CHECK: fir.cuda_kernel<<<(%c1{{.*}}, %c0{{.*}}), (%c256{{.*}}, %c1{{.*}})>>> (%{{.*}} : index, %{{.*}} : index) = (%{{.*}}, %{{.*}} : index, index) to (%{{.*}}, %{{.*}} : index, index)  step (%{{.*}}, %{{.*}} : index, index)
+! CHECK: cuf.kernel<<<(%c1{{.*}}, %c0{{.*}}), (%c256{{.*}}, %c1{{.*}})>>> (%{{.*}} : index, %{{.*}} : index) = (%{{.*}}, %{{.*}} : index, index) to (%{{.*}}, %{{.*}} : index, index)  step (%{{.*}}, %{{.*}} : index, index)
 
 !$cuf kernel do(2) <<< (*,*), (32,4) >>>
   do i = 1, n
@@ -63,5 +63,5 @@ subroutine sub1()
     end do
   end do
 
-! CHECK: fir.cuda_kernel<<<*, (%c32{{.*}}, %c4{{.*}})>>> (%{{.*}} : index, %{{.*}} : index) = (%{{.*}}, %{{.*}} : index, index) to (%{{.*}}, %{{.*}} : index, index)  step (%{{.*}}, %{{.*}} : index, index)
+! CHECK: cuf.kernel<<<*, (%c32{{.*}}, %c4{{.*}})>>> (%{{.*}} : index, %{{.*}} : index) = (%{{.*}}, %{{.*}} : index, index) to (%{{.*}}, %{{.*}} : index, index)  step (%{{.*}}, %{{.*}} : index, index)
 end
diff --git a/flang/test/Lower/CUDA/cuda-mod.cuf b/flang/test/Lower/CUDA/cuda-mod.cuf
index ae5bf63d2da4..f03e72e94780 100644
--- a/flang/test/Lower/CUDA/cuda-mod.cuf
+++ b/flang/test/Lower/CUDA/cuda-mod.cuf
@@ -10,6 +10,6 @@ contains
   end
 end module
 
-! CHECK: fir.global @_QMcuf_modEmd {cuda_attr = #fir.cuda<device>} : f32
+! CHECK: fir.global @_QMcuf_modEmd {data_attr = #cuf.cuda<device>} : f32
 
-! CHECK: func.func @_QMcuf_modPdevsub() attributes {fir.cuda_attr = #fir.cuda_proc<device>}
+! CHECK: func.func @_QMcuf_modPdevsub() attributes {cuf.proc_attr = #cuf.cuda_proc<device>}
diff --git a/flang/test/Lower/CUDA/cuda-module-use.cuf b/flang/test/Lower/CUDA/cuda-module-use.cuf
index 47d3805065a5..130fefab24d9 100644
--- a/flang/test/Lower/CUDA/cuda-module-use.cuf
+++ b/flang/test/Lower/CUDA/cuda-module-use.cuf
@@ -10,16 +10,16 @@ end
 
 ! CHECK-LABEL: func.func @_QPsub1()
 ! CHECK: %[[ADDR:.*]] = fir.address_of(@_QMcuf_modEmd) : !fir.ref<f32>
-! CHECK: %{{.*}}:2 = hlfir.declare %[[ADDR]] {cuda_attr = #fir.cuda<device>, uniq_name = "_QMcuf_modEmd"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %{{.*}}:2 = hlfir.declare %[[ADDR]] {data_attr = #cuf.cuda<device>, uniq_name = "_QMcuf_modEmd"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 
 attributes(device) subroutine sub2()
   use cuf_mod
   call devsub()
 end
 
-! CHECK-LABEL: func.func @_QPsub2() attributes {fir.cuda_attr = #fir.cuda_proc<device>}
+! CHECK-LABEL: func.func @_QPsub2() attributes {cuf.proc_attr = #cuf.cuda_proc<device>}
 ! CHECK: fir.call @_QMcuf_modPdevsub()
 
-! CHECK-LABEL: fir.global @_QMcuf_modEmd {cuda_attr = #fir.cuda<device>} : f32
+! CHECK-LABEL: fir.global @_QMcuf_modEmd {data_attr = #cuf.cuda<device>} : f32
 
-! CHECK-LABEL: func.func private @_QMcuf_modPdevsub() attributes {fir.cuda_attr = #fir.cuda_proc<device>}
+! CHECK-LABEL: func.func private @_QMcuf_modPdevsub() attributes {cuf.proc_attr = #cuf.cuda_proc<device>}
diff --git a/flang/test/Lower/CUDA/cuda-proc-attribute.cuf b/flang/test/Lower/CUDA/cuda-proc-attribute.cuf
index d9765f6cd2fe..f8b8dd8e296b 100644
--- a/flang/test/Lower/CUDA/cuda-proc-attribute.cuf
+++ b/flang/test/Lower/CUDA/cuda-proc-attribute.cuf
@@ -4,40 +4,40 @@
 ! Test lowering of CUDA attribute on procedures.
 
 attributes(host) subroutine sub_host(); end
-! CHECK: func.func @_QPsub_host() attributes {fir.cuda_attr = #fir.cuda_proc<host>}
+! CHECK: func.func @_QPsub_host() attributes {cuf.proc_attr = #cuf.cuda_proc<host>}
 
 attributes(device) subroutine sub_device(); end
-! CHECK: func.func @_QPsub_device() attributes {fir.cuda_attr = #fir.cuda_proc<device>}
+! CHECK: func.func @_QPsub_device() attributes {cuf.proc_attr = #cuf.cuda_proc<device>}
 
 attributes(host) attributes(device) subroutine sub_host_device; end
-! CHECK: func.func @_QPsub_host_device() attributes {fir.cuda_attr = #fir.cuda_proc<host_device>}
+! CHECK: func.func @_QPsub_host_device() attributes {cuf.proc_attr = #cuf.cuda_proc<host_device>}
 
 attributes(device) attributes(host) subroutine sub_device_host; end
-! CHECK: func.func @_QPsub_device_host() attributes {fir.cuda_attr = #fir.cuda_proc<host_device>}
+! CHECK: func.func @_QPsub_device_host() attributes {cuf.proc_attr = #cuf.cuda_proc<host_device>}
 
 attributes(global) subroutine sub_global(); end
-! CHECK: func.func @_QPsub_global() attributes {fir.cuda_attr = #fir.cuda_proc<global>}
+! CHECK: func.func @_QPsub_global() attributes {cuf.proc_attr = #cuf.cuda_proc<global>}
 
 attributes(grid_global) subroutine sub_grid_global(); end
-! CHECK: func.func @_QPsub_grid_global() attributes {fir.cuda_attr = #fir.cuda_proc<grid_global>}
+! CHECK: func.func @_QPsub_grid_global() attributes {cuf.proc_attr = #cuf.cuda_proc<grid_global>}
 
 attributes(host) integer function fct_host(); end
-! CHECK: func.func @_QPfct_host() -> i32 attributes {fir.cuda_attr = #fir.cuda_proc<host>}
+! CHECK: func.func @_QPfct_host() -> i32 attributes {cuf.proc_attr = #cuf.cuda_proc<host>}
 
 attributes(device) integer function fct_device(); end
-! CHECK: func.func @_QPfct_device() -> i32 attributes {fir.cuda_attr = #fir.cuda_proc<device>}
+! CHECK: func.func @_QPfct_device() -> i32 attributes {cuf.proc_attr = #cuf.cuda_proc<device>}
 
 attributes(host) attributes(device) integer function fct_host_device; end
-! CHECK: func.func @_QPfct_host_device() -> i32 attributes {fir.cuda_attr = #fir.cuda_proc<host_device>}
+! CHECK: func.func @_QPfct_host_device() -> i32 attributes {cuf.proc_attr = #cuf.cuda_proc<host_device>}
 
 attributes(device) attributes(host) integer function fct_device_host; end
-! CHECK: func.func @_QPfct_device_host() -> i32 attributes {fir.cuda_attr = #fir.cuda_proc<host_device>}
+! CHECK: func.func @_QPfct_device_host() -> i32 attributes {cuf.proc_attr = #cuf.cuda_proc<host_device>}
 
 attributes(global) launch_bounds(1, 2) subroutine sub_lbounds1(); end
-! CHECK: func.func @_QPsub_lbounds1() attributes {fir.cuda_attr = #fir.cuda_proc<global>, fir.cuda_launch_bounds = #fir.launch_bounds<maxTPB = 1 : i64, minBPM = 2 : i64>}
+! CHECK: func.func @_QPsub_lbounds1() attributes {cuf.launch_bounds = #cuf.launch_bounds<maxTPB = 1 : i64, minBPM = 2 : i64>, cuf.proc_attr = #cuf.cuda_proc<global>}
 
 attributes(global) launch_bounds(1, 2, 3) subroutine sub_lbounds2(); end
-! CHECK: func.func @_QPsub_lbounds2() attributes {fir.cuda_attr = #fir.cuda_proc<global>, fir.cuda_launch_bounds = #fir.launch_bounds<maxTPB = 1 : i64, minBPM = 2 : i64, upperBoundClusterSize = 3 : i64>}
+! CHECK: func.func @_QPsub_lbounds2() attributes {cuf.launch_bounds = #cuf.launch_bounds<maxTPB = 1 : i64, minBPM = 2 : i64, upperBoundClusterSize = 3 : i64>, cuf.proc_attr = #cuf.cuda_proc<global>}
 
 attributes(global) cluster_dims(1, 2, 3) subroutine sub_clusterdims1(); end
-! CHECK: func.func @_QPsub_clusterdims1() attributes {fir.cuda_attr = #fir.cuda_proc<global>, fir.cuda_cluster_dims = #fir.cluster_dims<x = 1 : i64, y = 2 : i64, z = 3 : i64>}
+! CHECK: func.func @_QPsub_clusterdims1() attributes {cuf.cluster_dims = #cuf.cluster_dims<x = 1 : i64, y = 2 : i64, z = 3 : i64>, cuf.proc_attr = #cuf.cuda_proc<global>}
diff --git a/flang/tools/bbc/CMakeLists.txt b/flang/tools/bbc/CMakeLists.txt
index f21fa3b7bae3..9410fd005660 100644
--- a/flang/tools/bbc/CMakeLists.txt
+++ b/flang/tools/bbc/CMakeLists.txt
@@ -16,6 +16,8 @@ get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 get_property(extension_libs GLOBAL PROPERTY MLIR_EXTENSION_LIBS)
 
 target_link_libraries(bbc PRIVATE
+CUFAttrs
+CUFDialect
 FIRDialect
 FIRDialectSupport
 FIRSupport
diff --git a/flang/tools/fir-opt/CMakeLists.txt b/flang/tools/fir-opt/CMakeLists.txt
index 43b0c74696f5..43679a9d5357 100644
--- a/flang/tools/fir-opt/CMakeLists.txt
+++ b/flang/tools/fir-opt/CMakeLists.txt
@@ -11,6 +11,8 @@ if(FLANG_INCLUDE_TESTS)
 endif()
 
 target_link_libraries(fir-opt PRIVATE
+  CUFAttrs
+  CUFDialect
   FIRDialect
   FIRSupport
   FIRTransforms
diff --git a/flang/tools/tco/CMakeLists.txt b/flang/tools/tco/CMakeLists.txt
index 6d83353b4e0d..808219ac361f 100644
--- a/flang/tools/tco/CMakeLists.txt
+++ b/flang/tools/tco/CMakeLists.txt
@@ -7,6 +7,8 @@ llvm_update_compile_flags(tco)
 get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 get_property(extension_libs GLOBAL PROPERTY MLIR_EXTENSION_LIBS)
 target_link_libraries(tco PRIVATE
+  CUFAttrs
+  CUFDialect
   FIRCodeGen
   FIRDialect
   FIRDialectSupport
diff --git a/flang/unittests/Optimizer/CMakeLists.txt b/flang/unittests/Optimizer/CMakeLists.txt
index 9c165d998e2e..7299e3ee0529 100644
--- a/flang/unittests/Optimizer/CMakeLists.txt
+++ b/flang/unittests/Optimizer/CMakeLists.txt
@@ -2,6 +2,7 @@ get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 get_property(extension_libs GLOBAL PROPERTY MLIR_EXTENSION_LIBS)
 
 set(LIBS
+  CUFDialect
   FIRBuilder
   FIRCodeGen
   FIRDialect
@@ -36,6 +37,7 @@ add_flang_unittest(FlangOptimizerTests
   KindMappingTest.cpp
   RTBuilder.cpp
 DEPENDS
+  CUFDialect
   FIRDialect
   FIRSupport
   HLFIRDialect
diff --git a/flang/unittests/Optimizer/FortranVariableTest.cpp b/flang/unittests/Optimizer/FortranVariableTest.cpp
index f5f559ef887c..87efb624735c 100644
--- a/flang/unittests/Optimizer/FortranVariableTest.cpp
+++ b/flang/unittests/Optimizer/FortranVariableTest.cpp
@@ -51,7 +51,7 @@ TEST_F(FortranVariableTest, SimpleScalar) {
       /*shape=*/mlir::Value{}, /*typeParams=*/std::nullopt,
       /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
-      /*cuda_attr=*/fir::CUDADataAttributeAttr{});
+      /*data_attr=*/cuf::DataAttributeAttr{});
 
   fir::FortranVariableOpInterface fortranVariable = declare;
   EXPECT_FALSE(fortranVariable.isArray());
@@ -77,7 +77,7 @@ TEST_F(FortranVariableTest, CharacterScalar) {
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
       /*shape=*/mlir::Value{}, typeParams, /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
-      /*cuda_attr=*/fir::CUDADataAttributeAttr{});
+      /*data_attr=*/cuf::DataAttributeAttr{});
 
   fir::FortranVariableOpInterface fortranVariable = declare;
   EXPECT_FALSE(fortranVariable.isArray());
@@ -108,7 +108,7 @@ TEST_F(FortranVariableTest, SimpleArray) {
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
       shape, /*typeParams*/ std::nullopt, /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
-      /*cuda_attr=*/fir::CUDADataAttributeAttr{});
+      /*data_attr=*/cuf::DataAttributeAttr{});
 
   fir::FortranVariableOpInterface fortranVariable = declare;
   EXPECT_TRUE(fortranVariable.isArray());
@@ -139,7 +139,7 @@ TEST_F(FortranVariableTest, CharacterArray) {
   auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
       shape, typeParams, /*dummy_scope=*/nullptr, name,
       /*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
-      /*cuda_attr=*/fir::CUDADataAttributeAttr{});
+      /*data_attr=*/cuf::DataAttributeAttr{});
 
   fir::FortranVariableOpInterface fortranVariable = declare;
   EXPECT_TRUE(fortranVariable.isArray());
-- 
GitLab


From e3686755eab66604335e246ed7b30033ab71dbbc Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Fri, 17 May 2024 09:52:13 -0700
Subject: [PATCH 1149/1206] [mlir][polynomial] implement add for polynomial
 data structure (#92169)

A change extracted from https://github.com/llvm/llvm-project/pull/91655,
where I'm still trying to get the attributes working for elementwise
constant folding of polynomial ops. This piece is self-contained.

- use CRTP for base classes
- Add unit test

---------

Co-authored-by: Jeremy Kun <j2kun@users.noreply.github.com>
---
 .../mlir/Dialect/Polynomial/IR/Polynomial.h   | 83 +++++++++++++++----
 mlir/unittests/Dialect/CMakeLists.txt         |  1 +
 .../Dialect/Polynomial/CMakeLists.txt         |  8 ++
 .../Dialect/Polynomial/PolynomialMathTest.cpp | 44 ++++++++++
 4 files changed, 118 insertions(+), 18 deletions(-)
 create mode 100644 mlir/unittests/Dialect/Polynomial/CMakeLists.txt
 create mode 100644 mlir/unittests/Dialect/Polynomial/PolynomialMathTest.cpp

diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
index 7f44c29a9870..e14cef51185e 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
@@ -30,7 +30,7 @@ namespace polynomial {
 /// would want to specify 128-bit polynomials statically in the source code.
 constexpr unsigned apintBitWidth = 64;
 
-template <typename CoefficientType>
+template <class Derived, typename CoefficientType>
 class MonomialBase {
 public:
   MonomialBase(const CoefficientType &coeff, const APInt &expo)
@@ -55,12 +55,21 @@ public:
     return (exponent.ult(other.exponent));
   }
 
+  Derived add(const Derived &other) {
+    assert(exponent == other.exponent);
+    CoefficientType newCoeff = coefficient + other.coefficient;
+    Derived result;
+    result.setCoefficient(newCoeff);
+    result.setExponent(exponent);
+    return result;
+  }
+
   virtual bool isMonic() const = 0;
   virtual void
   coefficientToString(llvm::SmallString<16> &coeffString) const = 0;
 
-  template <typename T>
-  friend ::llvm::hash_code hash_value(const MonomialBase<T> &arg);
+  template <class D, typename T>
+  friend ::llvm::hash_code hash_value(const MonomialBase<D, T> &arg);
 
 protected:
   CoefficientType coefficient;
@@ -69,7 +78,7 @@ protected:
 
 /// A class representing a monomial of a single-variable polynomial with integer
 /// coefficients.
-class IntMonomial : public MonomialBase<APInt> {
+class IntMonomial : public MonomialBase<IntMonomial, APInt> {
 public:
   IntMonomial(int64_t coeff, uint64_t expo)
       : MonomialBase(APInt(apintBitWidth, coeff), APInt(apintBitWidth, expo)) {}
@@ -77,7 +86,7 @@ public:
   IntMonomial()
       : MonomialBase(APInt(apintBitWidth, 0), APInt(apintBitWidth, 0)) {}
 
-  ~IntMonomial() = default;
+  ~IntMonomial() override = default;
 
   bool isMonic() const override { return coefficient == 1; }
 
@@ -88,14 +97,14 @@ public:
 
 /// A class representing a monomial of a single-variable polynomial with integer
 /// coefficients.
-class FloatMonomial : public MonomialBase<APFloat> {
+class FloatMonomial : public MonomialBase<FloatMonomial, APFloat> {
 public:
   FloatMonomial(double coeff, uint64_t expo)
       : MonomialBase(APFloat(coeff), APInt(apintBitWidth, expo)) {}
 
   FloatMonomial() : MonomialBase(APFloat((double)0), APInt(apintBitWidth, 0)) {}
 
-  ~FloatMonomial() = default;
+  ~FloatMonomial() override = default;
 
   bool isMonic() const override { return coefficient == APFloat(1.0); }
 
@@ -104,7 +113,7 @@ public:
   }
 };
 
-template <typename Monomial>
+template <class Derived, typename Monomial>
 class PolynomialBase {
 public:
   PolynomialBase() = delete;
@@ -149,6 +158,44 @@ public:
     }
   }
 
+  Derived add(const Derived &other) {
+    SmallVector<Monomial> newTerms;
+    auto it1 = terms.begin();
+    auto it2 = other.terms.begin();
+    while (it1 != terms.end() || it2 != other.terms.end()) {
+      if (it1 == terms.end()) {
+        newTerms.emplace_back(*it2);
+        it2++;
+        continue;
+      }
+
+      if (it2 == other.terms.end()) {
+        newTerms.emplace_back(*it1);
+        it1++;
+        continue;
+      }
+
+      while (it1->getExponent().ult(it2->getExponent())) {
+        newTerms.emplace_back(*it1);
+        it1++;
+        if (it1 == terms.end())
+          break;
+      }
+
+      while (it2->getExponent().ult(it1->getExponent())) {
+        newTerms.emplace_back(*it2);
+        it2++;
+        if (it2 == terms.end())
+          break;
+      }
+
+      newTerms.emplace_back(it1->add(*it2));
+      it1++;
+      it2++;
+    }
+    return Derived(newTerms);
+  }
+
   // Prints polynomial to 'os'.
   void print(raw_ostream &os) const { print(os, " + ", "**"); }
 
@@ -168,8 +215,8 @@ public:
 
   ArrayRef<Monomial> getTerms() const { return terms; }
 
-  template <typename T>
-  friend ::llvm::hash_code hash_value(const PolynomialBase<T> &arg);
+  template <class D, typename T>
+  friend ::llvm::hash_code hash_value(const PolynomialBase<D, T> &arg);
 
 private:
   // The monomial terms for this polynomial.
@@ -179,7 +226,7 @@ private:
 /// A single-variable polynomial with integer coefficients.
 ///
 /// Eg: x^1024 + x + 1
-class IntPolynomial : public PolynomialBase<IntMonomial> {
+class IntPolynomial : public PolynomialBase<IntPolynomial, IntMonomial> {
 public:
   explicit IntPolynomial(ArrayRef<IntMonomial> terms) : PolynomialBase(terms) {}
 
@@ -196,7 +243,7 @@ public:
 /// A single-variable polynomial with double coefficients.
 ///
 /// Eg: 1.0 x^1024 + 3.5 x + 1e-05
-class FloatPolynomial : public PolynomialBase<FloatMonomial> {
+class FloatPolynomial : public PolynomialBase<FloatPolynomial, FloatMonomial> {
 public:
   explicit FloatPolynomial(ArrayRef<FloatMonomial> terms)
       : PolynomialBase(terms) {}
@@ -212,20 +259,20 @@ public:
 };
 
 // Make Polynomials hashable.
-template <typename T>
-inline ::llvm::hash_code hash_value(const PolynomialBase<T> &arg) {
+template <class D, typename T>
+inline ::llvm::hash_code hash_value(const PolynomialBase<D, T> &arg) {
   return ::llvm::hash_combine_range(arg.terms.begin(), arg.terms.end());
 }
 
-template <typename T>
-inline ::llvm::hash_code hash_value(const MonomialBase<T> &arg) {
+template <class D, typename T>
+inline ::llvm::hash_code hash_value(const MonomialBase<D, T> &arg) {
   return llvm::hash_combine(::llvm::hash_value(arg.coefficient),
                             ::llvm::hash_value(arg.exponent));
 }
 
-template <typename T>
+template <class D, typename T>
 inline raw_ostream &operator<<(raw_ostream &os,
-                               const PolynomialBase<T> &polynomial) {
+                               const PolynomialBase<D, T> &polynomial) {
   polynomial.print(os);
   return os;
 }
diff --git a/mlir/unittests/Dialect/CMakeLists.txt b/mlir/unittests/Dialect/CMakeLists.txt
index 13393569f36f..90a75d5a46ad 100644
--- a/mlir/unittests/Dialect/CMakeLists.txt
+++ b/mlir/unittests/Dialect/CMakeLists.txt
@@ -11,6 +11,7 @@ add_subdirectory(Index)
 add_subdirectory(LLVMIR)
 add_subdirectory(MemRef)
 add_subdirectory(OpenACC)
+add_subdirectory(Polynomial)
 add_subdirectory(SCF)
 add_subdirectory(SparseTensor)
 add_subdirectory(SPIRV)
diff --git a/mlir/unittests/Dialect/Polynomial/CMakeLists.txt b/mlir/unittests/Dialect/Polynomial/CMakeLists.txt
new file mode 100644
index 000000000000..807deeca41c0
--- /dev/null
+++ b/mlir/unittests/Dialect/Polynomial/CMakeLists.txt
@@ -0,0 +1,8 @@
+add_mlir_unittest(MLIRPolynomialTests
+  PolynomialMathTest.cpp
+)
+target_link_libraries(MLIRPolynomialTests
+  PRIVATE
+  MLIRIR
+  MLIRPolynomialDialect
+)
diff --git a/mlir/unittests/Dialect/Polynomial/PolynomialMathTest.cpp b/mlir/unittests/Dialect/Polynomial/PolynomialMathTest.cpp
new file mode 100644
index 000000000000..95906ad42588
--- /dev/null
+++ b/mlir/unittests/Dialect/Polynomial/PolynomialMathTest.cpp
@@ -0,0 +1,44 @@
+//===- PolynomialMathTest.cpp - Polynomial math Tests ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Polynomial/IR/Polynomial.h"
+#include "gtest/gtest.h"
+
+using namespace mlir;
+using namespace mlir::polynomial;
+
+TEST(AddTest, checkSameDegreeAdditionOfIntPolynomial) {
+  IntPolynomial x = IntPolynomial::fromCoefficients({1, 2, 3});
+  IntPolynomial y = IntPolynomial::fromCoefficients({2, 3, 4});
+  IntPolynomial expected = IntPolynomial::fromCoefficients({3, 5, 7});
+  EXPECT_EQ(expected, x.add(y));
+}
+
+TEST(AddTest, checkDifferentDegreeAdditionOfIntPolynomial) {
+  IntMonomial term2t = IntMonomial(2, 1);
+  IntPolynomial x = IntPolynomial::fromMonomials({term2t}).value();
+  IntPolynomial y = IntPolynomial::fromCoefficients({2, 3, 4});
+  IntPolynomial expected = IntPolynomial::fromCoefficients({2, 5, 4});
+  EXPECT_EQ(expected, x.add(y));
+  EXPECT_EQ(expected, y.add(x));
+}
+
+TEST(AddTest, checkSameDegreeAdditionOfFloatPolynomial) {
+  FloatPolynomial x = FloatPolynomial::fromCoefficients({1.5, 2.5, 3.5});
+  FloatPolynomial y = FloatPolynomial::fromCoefficients({2.5, 3.5, 4.5});
+  FloatPolynomial expected = FloatPolynomial::fromCoefficients({4, 6, 8});
+  EXPECT_EQ(expected, x.add(y));
+}
+
+TEST(AddTest, checkDifferentDegreeAdditionOfFloatPolynomial) {
+  FloatPolynomial x = FloatPolynomial::fromCoefficients({1.5, 2.5});
+  FloatPolynomial y = FloatPolynomial::fromCoefficients({2.5, 3.5, 4.5});
+  FloatPolynomial expected = FloatPolynomial::fromCoefficients({4, 6, 4.5});
+  EXPECT_EQ(expected, x.add(y));
+  EXPECT_EQ(expected, y.add(x));
+}
-- 
GitLab


From 874f511ae779a38bd9ad9d80db63c0a10252e526 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 17 May 2024 20:55:37 +0400
Subject: [PATCH 1150/1206] [clang] Introduce `SemaCodeCompletion` (#92311)

This patch continues previous efforts to split `Sema` up, this time
covering code completion.
Context can be found in #84184.
Dropping `Code` prefix from function names in `SemaCodeCompletion` would
make sense, but I think this PR has enough changes already.
As usual, formatting changes are done as a separate commit. Hopefully
this helps with the review.
---
 clang/include/clang/Parse/Parser.h            |    4 +-
 clang/include/clang/Sema/Sema.h               |  225 +--
 clang/include/clang/Sema/SemaCodeCompletion.h |  237 +++
 clang/lib/Frontend/ASTUnit.cpp                |    5 +-
 clang/lib/Parse/ParseDecl.cpp                 |   62 +-
 clang/lib/Parse/ParseDeclCXX.cpp              |   57 +-
 clang/lib/Parse/ParseExpr.cpp                 |   31 +-
 clang/lib/Parse/ParseExprCXX.cpp              |   46 +-
 clang/lib/Parse/ParseInit.cpp                 |    8 +-
 clang/lib/Parse/ParseObjc.cpp                 |  148 +-
 clang/lib/Parse/ParseOpenMP.cpp               |   11 +-
 clang/lib/Parse/ParsePragma.cpp               |    4 +-
 clang/lib/Parse/ParseStmt.cpp                 |   24 +-
 clang/lib/Parse/ParseTemplate.cpp             |    4 +-
 clang/lib/Parse/Parser.cpp                    |   47 +-
 clang/lib/Sema/Sema.cpp                       |    6 +-
 clang/lib/Sema/SemaCodeComplete.cpp           | 1306 +++++++++--------
 17 files changed, 1203 insertions(+), 1022 deletions(-)
 create mode 100644 clang/include/clang/Sema/SemaCodeCompletion.h

diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index c7d75985ce8b..1e796e828b10 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -18,6 +18,7 @@
 #include "clang/Lex/CodeCompletionHandler.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "llvm/ADT/SmallVector.h"
@@ -2997,7 +2998,8 @@ private:
 
   IdentifierInfo *TryParseCXX11AttributeIdentifier(
       SourceLocation &Loc,
-      Sema::AttributeCompletion Completion = Sema::AttributeCompletion::None,
+      SemaCodeCompletion::AttributeCompletion Completion =
+          SemaCodeCompletion::AttributeCompletion::None,
       const IdentifierInfo *EnclosingScope = nullptr);
 
   void MaybeParseHLSLAnnotations(Declarator &D,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 226987b97570..df562e93fbf0 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -167,6 +167,7 @@ class Preprocessor;
 class PseudoDestructorTypeStorage;
 class PseudoObjectExpr;
 class QualType;
+class SemaCodeCompletion;
 class SemaCUDA;
 class SemaHLSL;
 class SemaObjC;
@@ -481,9 +482,8 @@ class Sema final : public SemaBase {
   // 29. C++ Variadic Templates (SemaTemplateVariadic.cpp)
   // 30. Constraints and Concepts (SemaConcept.cpp)
   // 31. Types (SemaType.cpp)
-  // 32. Code Completion (SemaCodeComplete.cpp)
-  // 33. FixIt Helpers (SemaFixItUtils.cpp)
-  // 34. Name Lookup for RISC-V Vector Intrinsic (SemaRISCVVectorLookup.cpp)
+  // 32. FixIt Helpers (SemaFixItUtils.cpp)
+  // 33. Name Lookup for RISC-V Vector Intrinsic (SemaRISCVVectorLookup.cpp)
 
   /// \name Semantic Analysis
   /// Implementations are in Sema.cpp
@@ -984,6 +984,11 @@ public:
   /// CurContext - This is the current declaration context of parsing.
   DeclContext *CurContext;
 
+  SemaCodeCompletion &CodeCompletion() {
+    assert(CodeCompletionPtr);
+    return *CodeCompletionPtr;
+  }
+
   SemaCUDA &CUDA() {
     assert(CUDAPtr);
     return *CUDAPtr;
@@ -1044,6 +1049,7 @@ private:
 
   mutable IdentifierInfo *Ident_super;
 
+  std::unique_ptr<SemaCodeCompletion> CodeCompletionPtr;
   std::unique_ptr<SemaCUDA> CUDAPtr;
   std::unique_ptr<SemaHLSL> HLSLPtr;
   std::unique_ptr<SemaObjC> ObjCPtr;
@@ -4801,6 +4807,11 @@ public:
     DelayedDiagnostics.popUndelayed(state);
   }
 
+  ValueDecl *tryLookupCtorInitMemberDecl(CXXRecordDecl *ClassDecl,
+                                         CXXScopeSpec &SS,
+                                         ParsedType TemplateTypeTy,
+                                         IdentifierInfo *MemberOrBase);
+
 private:
   void setupImplicitSpecialMemberType(CXXMethodDecl *SpecialMem,
                                       QualType ResultTy,
@@ -4811,11 +4822,6 @@ private:
   // of a ComparisonCategoryType enumerator.
   llvm::SmallBitVector FullyCheckedComparisonCategories;
 
-  ValueDecl *tryLookupCtorInitMemberDecl(CXXRecordDecl *ClassDecl,
-                                         CXXScopeSpec &SS,
-                                         ParsedType TemplateTypeTy,
-                                         IdentifierInfo *MemberOrBase);
-
   /// Check if there is a field shadowing.
   void CheckShadowInheritedFields(const SourceLocation &Loc,
                                   DeclarationName FieldName,
@@ -11664,209 +11670,6 @@ private:
   //
   //
 
-  /// \name Code Completion
-  /// Implementations are in SemaCodeComplete.cpp
-  ///@{
-
-public:
-  /// Code-completion consumer.
-  CodeCompleteConsumer *CodeCompleter;
-
-  /// Describes the context in which code completion occurs.
-  enum ParserCompletionContext {
-    /// Code completion occurs at top-level or namespace context.
-    PCC_Namespace,
-    /// Code completion occurs within a class, struct, or union.
-    PCC_Class,
-    /// Code completion occurs within an Objective-C interface, protocol,
-    /// or category.
-    PCC_ObjCInterface,
-    /// Code completion occurs within an Objective-C implementation or
-    /// category implementation
-    PCC_ObjCImplementation,
-    /// Code completion occurs within the list of instance variables
-    /// in an Objective-C interface, protocol, category, or implementation.
-    PCC_ObjCInstanceVariableList,
-    /// Code completion occurs following one or more template
-    /// headers.
-    PCC_Template,
-    /// Code completion occurs following one or more template
-    /// headers within a class.
-    PCC_MemberTemplate,
-    /// Code completion occurs within an expression.
-    PCC_Expression,
-    /// Code completion occurs within a statement, which may
-    /// also be an expression or a declaration.
-    PCC_Statement,
-    /// Code completion occurs at the beginning of the
-    /// initialization statement (or expression) in a for loop.
-    PCC_ForInit,
-    /// Code completion occurs within the condition of an if,
-    /// while, switch, or for statement.
-    PCC_Condition,
-    /// Code completion occurs within the body of a function on a
-    /// recovery path, where we do not have a specific handle on our position
-    /// in the grammar.
-    PCC_RecoveryInFunction,
-    /// Code completion occurs where only a type is permitted.
-    PCC_Type,
-    /// Code completion occurs in a parenthesized expression, which
-    /// might also be a type cast.
-    PCC_ParenthesizedExpression,
-    /// Code completion occurs within a sequence of declaration
-    /// specifiers within a function, method, or block.
-    PCC_LocalDeclarationSpecifiers,
-    /// Code completion occurs at top-level in a REPL session
-    PCC_TopLevelOrExpression,
-  };
-
-  void CodeCompleteModuleImport(SourceLocation ImportLoc, ModuleIdPath Path);
-  void CodeCompleteOrdinaryName(Scope *S,
-                                ParserCompletionContext CompletionContext);
-  void CodeCompleteDeclSpec(Scope *S, DeclSpec &DS, bool AllowNonIdentifiers,
-                            bool AllowNestedNameSpecifiers);
-
-  struct CodeCompleteExpressionData;
-  void CodeCompleteExpression(Scope *S, const CodeCompleteExpressionData &Data);
-  void CodeCompleteExpression(Scope *S, QualType PreferredType,
-                              bool IsParenthesized = false);
-  void CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base, Expr *OtherOpBase,
-                                       SourceLocation OpLoc, bool IsArrow,
-                                       bool IsBaseExprStatement,
-                                       QualType PreferredType);
-  void CodeCompletePostfixExpression(Scope *S, ExprResult LHS,
-                                     QualType PreferredType);
-  void CodeCompleteTag(Scope *S, unsigned TagSpec);
-  void CodeCompleteTypeQualifiers(DeclSpec &DS);
-  void CodeCompleteFunctionQualifiers(DeclSpec &DS, Declarator &D,
-                                      const VirtSpecifiers *VS = nullptr);
-  void CodeCompleteBracketDeclarator(Scope *S);
-  void CodeCompleteCase(Scope *S);
-  enum class AttributeCompletion {
-    Attribute,
-    Scope,
-    None,
-  };
-  void CodeCompleteAttribute(
-      AttributeCommonInfo::Syntax Syntax,
-      AttributeCompletion Completion = AttributeCompletion::Attribute,
-      const IdentifierInfo *Scope = nullptr);
-  /// Determines the preferred type of the current function argument, by
-  /// examining the signatures of all possible overloads.
-  /// Returns null if unknown or ambiguous, or if code completion is off.
-  ///
-  /// If the code completion point has been reached, also reports the function
-  /// signatures that were considered.
-  ///
-  /// FIXME: rename to GuessCallArgumentType to reduce confusion.
-  QualType ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
-                                    SourceLocation OpenParLoc);
-  QualType ProduceConstructorSignatureHelp(QualType Type, SourceLocation Loc,
-                                           ArrayRef<Expr *> Args,
-                                           SourceLocation OpenParLoc,
-                                           bool Braced);
-  QualType ProduceCtorInitMemberSignatureHelp(
-      Decl *ConstructorDecl, CXXScopeSpec SS, ParsedType TemplateTypeTy,
-      ArrayRef<Expr *> ArgExprs, IdentifierInfo *II, SourceLocation OpenParLoc,
-      bool Braced);
-  QualType ProduceTemplateArgumentSignatureHelp(
-      TemplateTy, ArrayRef<ParsedTemplateArgument>, SourceLocation LAngleLoc);
-  void CodeCompleteInitializer(Scope *S, Decl *D);
-  /// Trigger code completion for a record of \p BaseType. \p InitExprs are
-  /// expressions in the initializer list seen so far and \p D is the current
-  /// Designation being parsed.
-  void CodeCompleteDesignator(const QualType BaseType,
-                              llvm::ArrayRef<Expr *> InitExprs,
-                              const Designation &D);
-  void CodeCompleteAfterIf(Scope *S, bool IsBracedThen);
-
-  void CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS, bool EnteringContext,
-                               bool IsUsingDeclaration, QualType BaseType,
-                               QualType PreferredType);
-  void CodeCompleteUsing(Scope *S);
-  void CodeCompleteUsingDirective(Scope *S);
-  void CodeCompleteNamespaceDecl(Scope *S);
-  void CodeCompleteNamespaceAliasDecl(Scope *S);
-  void CodeCompleteOperatorName(Scope *S);
-  void CodeCompleteConstructorInitializer(
-      Decl *Constructor, ArrayRef<CXXCtorInitializer *> Initializers);
-
-  void CodeCompleteLambdaIntroducer(Scope *S, LambdaIntroducer &Intro,
-                                    bool AfterAmpersand);
-  void CodeCompleteAfterFunctionEquals(Declarator &D);
-
-  void CodeCompleteObjCAtDirective(Scope *S);
-  void CodeCompleteObjCAtVisibility(Scope *S);
-  void CodeCompleteObjCAtStatement(Scope *S);
-  void CodeCompleteObjCAtExpression(Scope *S);
-  void CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS);
-  void CodeCompleteObjCPropertyGetter(Scope *S);
-  void CodeCompleteObjCPropertySetter(Scope *S);
-  void CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS,
-                                   bool IsParameter);
-  void CodeCompleteObjCMessageReceiver(Scope *S);
-  void CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc,
-                                    ArrayRef<const IdentifierInfo *> SelIdents,
-                                    bool AtArgumentExpression);
-  void CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
-                                    ArrayRef<const IdentifierInfo *> SelIdents,
-                                    bool AtArgumentExpression,
-                                    bool IsSuper = false);
-  void CodeCompleteObjCInstanceMessage(
-      Scope *S, Expr *Receiver, ArrayRef<const IdentifierInfo *> SelIdents,
-      bool AtArgumentExpression, ObjCInterfaceDecl *Super = nullptr);
-  void CodeCompleteObjCForCollection(Scope *S, DeclGroupPtrTy IterationVar);
-  void CodeCompleteObjCSelector(Scope *S,
-                                ArrayRef<const IdentifierInfo *> SelIdents);
-  void
-  CodeCompleteObjCProtocolReferences(ArrayRef<IdentifierLocPair> Protocols);
-  void CodeCompleteObjCProtocolDecl(Scope *S);
-  void CodeCompleteObjCInterfaceDecl(Scope *S);
-  void CodeCompleteObjCClassForwardDecl(Scope *S);
-  void CodeCompleteObjCSuperclass(Scope *S, IdentifierInfo *ClassName,
-                                  SourceLocation ClassNameLoc);
-  void CodeCompleteObjCImplementationDecl(Scope *S);
-  void CodeCompleteObjCInterfaceCategory(Scope *S, IdentifierInfo *ClassName,
-                                         SourceLocation ClassNameLoc);
-  void CodeCompleteObjCImplementationCategory(Scope *S,
-                                              IdentifierInfo *ClassName,
-                                              SourceLocation ClassNameLoc);
-  void CodeCompleteObjCPropertyDefinition(Scope *S);
-  void CodeCompleteObjCPropertySynthesizeIvar(Scope *S,
-                                              IdentifierInfo *PropertyName);
-  void CodeCompleteObjCMethodDecl(Scope *S,
-                                  std::optional<bool> IsInstanceMethod,
-                                  ParsedType ReturnType);
-  void CodeCompleteObjCMethodDeclSelector(
-      Scope *S, bool IsInstanceMethod, bool AtParameterName,
-      ParsedType ReturnType, ArrayRef<const IdentifierInfo *> SelIdents);
-  void CodeCompleteObjCClassPropertyRefExpr(Scope *S,
-                                            const IdentifierInfo &ClassName,
-                                            SourceLocation ClassNameLoc,
-                                            bool IsBaseExprStatement);
-  void CodeCompletePreprocessorDirective(bool InConditional);
-  void CodeCompleteInPreprocessorConditionalExclusion(Scope *S);
-  void CodeCompletePreprocessorMacroName(bool IsDefinition);
-  void CodeCompletePreprocessorExpression();
-  void CodeCompletePreprocessorMacroArgument(Scope *S, IdentifierInfo *Macro,
-                                             MacroInfo *MacroInfo,
-                                             unsigned Argument);
-  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
-  void CodeCompleteNaturalLanguage();
-  void CodeCompleteAvailabilityPlatformName();
-  void
-  GatherGlobalCodeCompletions(CodeCompletionAllocator &Allocator,
-                              CodeCompletionTUInfo &CCTUInfo,
-                              SmallVectorImpl<CodeCompletionResult> &Results);
-
-  ///@}
-
-  //
-  //
-  // -------------------------------------------------------------------------
-  //
-  //
-
   /// \name FixIt Helpers
   /// Implementations are in SemaFixItUtils.cpp
   ///@{
diff --git a/clang/include/clang/Sema/SemaCodeCompletion.h b/clang/include/clang/Sema/SemaCodeCompletion.h
new file mode 100644
index 000000000000..6bf0f2473cbf
--- /dev/null
+++ b/clang/include/clang/Sema/SemaCodeCompletion.h
@@ -0,0 +1,237 @@
+//===----- SemaCodeCompletion.h ------ Code completion support ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file declares facilities that support code completion.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SEMA_SEMACODECOMPLETION_H
+#define LLVM_CLANG_SEMA_SEMACODECOMPLETION_H
+
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/Type.h"
+#include "clang/Basic/AttributeCommonInfo.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/ModuleLoader.h"
+#include "clang/Sema/CodeCompleteConsumer.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Designator.h"
+#include "clang/Sema/Ownership.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaBase.h"
+#include "llvm/ADT/StringRef.h"
+#include <optional>
+
+namespace clang {
+
+class SemaCodeCompletion : public SemaBase {
+public:
+  SemaCodeCompletion(Sema &S, CodeCompleteConsumer *CompletionConsumer);
+
+  using TemplateTy = OpaquePtr<TemplateName>;
+  using DeclGroupPtrTy = OpaquePtr<DeclGroupRef>;
+
+  /// Code-completion consumer.
+  CodeCompleteConsumer *CodeCompleter;
+
+  /// Describes the context in which code completion occurs.
+  enum ParserCompletionContext {
+    /// Code completion occurs at top-level or namespace context.
+    PCC_Namespace,
+    /// Code completion occurs within a class, struct, or union.
+    PCC_Class,
+    /// Code completion occurs within an Objective-C interface, protocol,
+    /// or category.
+    PCC_ObjCInterface,
+    /// Code completion occurs within an Objective-C implementation or
+    /// category implementation
+    PCC_ObjCImplementation,
+    /// Code completion occurs within the list of instance variables
+    /// in an Objective-C interface, protocol, category, or implementation.
+    PCC_ObjCInstanceVariableList,
+    /// Code completion occurs following one or more template
+    /// headers.
+    PCC_Template,
+    /// Code completion occurs following one or more template
+    /// headers within a class.
+    PCC_MemberTemplate,
+    /// Code completion occurs within an expression.
+    PCC_Expression,
+    /// Code completion occurs within a statement, which may
+    /// also be an expression or a declaration.
+    PCC_Statement,
+    /// Code completion occurs at the beginning of the
+    /// initialization statement (or expression) in a for loop.
+    PCC_ForInit,
+    /// Code completion occurs within the condition of an if,
+    /// while, switch, or for statement.
+    PCC_Condition,
+    /// Code completion occurs within the body of a function on a
+    /// recovery path, where we do not have a specific handle on our position
+    /// in the grammar.
+    PCC_RecoveryInFunction,
+    /// Code completion occurs where only a type is permitted.
+    PCC_Type,
+    /// Code completion occurs in a parenthesized expression, which
+    /// might also be a type cast.
+    PCC_ParenthesizedExpression,
+    /// Code completion occurs within a sequence of declaration
+    /// specifiers within a function, method, or block.
+    PCC_LocalDeclarationSpecifiers,
+    /// Code completion occurs at top-level in a REPL session
+    PCC_TopLevelOrExpression,
+  };
+
+  void CodeCompleteModuleImport(SourceLocation ImportLoc, ModuleIdPath Path);
+  void CodeCompleteOrdinaryName(Scope *S,
+                                ParserCompletionContext CompletionContext);
+  void CodeCompleteDeclSpec(Scope *S, DeclSpec &DS, bool AllowNonIdentifiers,
+                            bool AllowNestedNameSpecifiers);
+
+  struct CodeCompleteExpressionData;
+  void CodeCompleteExpression(Scope *S, const CodeCompleteExpressionData &Data);
+  void CodeCompleteExpression(Scope *S, QualType PreferredType,
+                              bool IsParenthesized = false);
+  void CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base, Expr *OtherOpBase,
+                                       SourceLocation OpLoc, bool IsArrow,
+                                       bool IsBaseExprStatement,
+                                       QualType PreferredType);
+  void CodeCompletePostfixExpression(Scope *S, ExprResult LHS,
+                                     QualType PreferredType);
+  void CodeCompleteTag(Scope *S, unsigned TagSpec);
+  void CodeCompleteTypeQualifiers(DeclSpec &DS);
+  void CodeCompleteFunctionQualifiers(DeclSpec &DS, Declarator &D,
+                                      const VirtSpecifiers *VS = nullptr);
+  void CodeCompleteBracketDeclarator(Scope *S);
+  void CodeCompleteCase(Scope *S);
+  enum class AttributeCompletion {
+    Attribute,
+    Scope,
+    None,
+  };
+  void CodeCompleteAttribute(
+      AttributeCommonInfo::Syntax Syntax,
+      AttributeCompletion Completion = AttributeCompletion::Attribute,
+      const IdentifierInfo *Scope = nullptr);
+  /// Determines the preferred type of the current function argument, by
+  /// examining the signatures of all possible overloads.
+  /// Returns null if unknown or ambiguous, or if code completion is off.
+  ///
+  /// If the code completion point has been reached, also reports the function
+  /// signatures that were considered.
+  ///
+  /// FIXME: rename to GuessCallArgumentType to reduce confusion.
+  QualType ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
+                                    SourceLocation OpenParLoc);
+  QualType ProduceConstructorSignatureHelp(QualType Type, SourceLocation Loc,
+                                           ArrayRef<Expr *> Args,
+                                           SourceLocation OpenParLoc,
+                                           bool Braced);
+  QualType ProduceCtorInitMemberSignatureHelp(
+      Decl *ConstructorDecl, CXXScopeSpec SS, ParsedType TemplateTypeTy,
+      ArrayRef<Expr *> ArgExprs, IdentifierInfo *II, SourceLocation OpenParLoc,
+      bool Braced);
+  QualType ProduceTemplateArgumentSignatureHelp(
+      TemplateTy, ArrayRef<ParsedTemplateArgument>, SourceLocation LAngleLoc);
+  void CodeCompleteInitializer(Scope *S, Decl *D);
+  /// Trigger code completion for a record of \p BaseType. \p InitExprs are
+  /// expressions in the initializer list seen so far and \p D is the current
+  /// Designation being parsed.
+  void CodeCompleteDesignator(const QualType BaseType,
+                              llvm::ArrayRef<Expr *> InitExprs,
+                              const Designation &D);
+  void CodeCompleteAfterIf(Scope *S, bool IsBracedThen);
+
+  void CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS, bool EnteringContext,
+                               bool IsUsingDeclaration, QualType BaseType,
+                               QualType PreferredType);
+  void CodeCompleteUsing(Scope *S);
+  void CodeCompleteUsingDirective(Scope *S);
+  void CodeCompleteNamespaceDecl(Scope *S);
+  void CodeCompleteNamespaceAliasDecl(Scope *S);
+  void CodeCompleteOperatorName(Scope *S);
+  void CodeCompleteConstructorInitializer(
+      Decl *Constructor, ArrayRef<CXXCtorInitializer *> Initializers);
+
+  void CodeCompleteLambdaIntroducer(Scope *S, LambdaIntroducer &Intro,
+                                    bool AfterAmpersand);
+  void CodeCompleteAfterFunctionEquals(Declarator &D);
+
+  void CodeCompleteObjCAtDirective(Scope *S);
+  void CodeCompleteObjCAtVisibility(Scope *S);
+  void CodeCompleteObjCAtStatement(Scope *S);
+  void CodeCompleteObjCAtExpression(Scope *S);
+  void CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS);
+  void CodeCompleteObjCPropertyGetter(Scope *S);
+  void CodeCompleteObjCPropertySetter(Scope *S);
+  void CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS,
+                                   bool IsParameter);
+  void CodeCompleteObjCMessageReceiver(Scope *S);
+  void CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc,
+                                    ArrayRef<const IdentifierInfo *> SelIdents,
+                                    bool AtArgumentExpression);
+  void CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
+                                    ArrayRef<const IdentifierInfo *> SelIdents,
+                                    bool AtArgumentExpression,
+                                    bool IsSuper = false);
+  void CodeCompleteObjCInstanceMessage(
+      Scope *S, Expr *Receiver, ArrayRef<const IdentifierInfo *> SelIdents,
+      bool AtArgumentExpression, ObjCInterfaceDecl *Super = nullptr);
+  void CodeCompleteObjCForCollection(Scope *S, DeclGroupPtrTy IterationVar);
+  void CodeCompleteObjCSelector(Scope *S,
+                                ArrayRef<const IdentifierInfo *> SelIdents);
+  void
+  CodeCompleteObjCProtocolReferences(ArrayRef<IdentifierLocPair> Protocols);
+  void CodeCompleteObjCProtocolDecl(Scope *S);
+  void CodeCompleteObjCInterfaceDecl(Scope *S);
+  void CodeCompleteObjCClassForwardDecl(Scope *S);
+  void CodeCompleteObjCSuperclass(Scope *S, IdentifierInfo *ClassName,
+                                  SourceLocation ClassNameLoc);
+  void CodeCompleteObjCImplementationDecl(Scope *S);
+  void CodeCompleteObjCInterfaceCategory(Scope *S, IdentifierInfo *ClassName,
+                                         SourceLocation ClassNameLoc);
+  void CodeCompleteObjCImplementationCategory(Scope *S,
+                                              IdentifierInfo *ClassName,
+                                              SourceLocation ClassNameLoc);
+  void CodeCompleteObjCPropertyDefinition(Scope *S);
+  void CodeCompleteObjCPropertySynthesizeIvar(Scope *S,
+                                              IdentifierInfo *PropertyName);
+  void CodeCompleteObjCMethodDecl(Scope *S,
+                                  std::optional<bool> IsInstanceMethod,
+                                  ParsedType ReturnType);
+  void CodeCompleteObjCMethodDeclSelector(
+      Scope *S, bool IsInstanceMethod, bool AtParameterName,
+      ParsedType ReturnType, ArrayRef<const IdentifierInfo *> SelIdents);
+  void CodeCompleteObjCClassPropertyRefExpr(Scope *S,
+                                            const IdentifierInfo &ClassName,
+                                            SourceLocation ClassNameLoc,
+                                            bool IsBaseExprStatement);
+  void CodeCompletePreprocessorDirective(bool InConditional);
+  void CodeCompleteInPreprocessorConditionalExclusion(Scope *S);
+  void CodeCompletePreprocessorMacroName(bool IsDefinition);
+  void CodeCompletePreprocessorExpression();
+  void CodeCompletePreprocessorMacroArgument(Scope *S, IdentifierInfo *Macro,
+                                             MacroInfo *MacroInfo,
+                                             unsigned Argument);
+  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
+  void CodeCompleteNaturalLanguage();
+  void CodeCompleteAvailabilityPlatformName();
+  void
+  GatherGlobalCodeCompletions(CodeCompletionAllocator &Allocator,
+                              CodeCompletionTUInfo &CCTUInfo,
+                              SmallVectorImpl<CodeCompletionResult> &Results);
+};
+
+} // namespace clang
+
+#endif // LLVM_CLANG_SEMA_SEMACODECOMPLETION_H
diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index 755aaddc0ad7..8d24d5d42252 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -56,6 +56,7 @@
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "clang/Sema/CodeCompleteOptions.h"
 #include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/ASTWriter.h"
@@ -375,8 +376,8 @@ void ASTUnit::CacheCodeCompletionResults() {
   SmallVector<Result, 8> Results;
   CachedCompletionAllocator = std::make_shared<GlobalCodeCompletionAllocator>();
   CodeCompletionTUInfo CCTUInfo(CachedCompletionAllocator);
-  TheSema->GatherGlobalCodeCompletions(*CachedCompletionAllocator,
-                                       CCTUInfo, Results);
+  TheSema->CodeCompletion().GatherGlobalCodeCompletions(
+      *CachedCompletionAllocator, CCTUInfo, Results);
 
   // Translate global code completions into cached completions.
   llvm::DenseMap<CanQualType, unsigned> CompletionTypes;
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 179564ae41a3..2ce8fa98089f 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -27,6 +27,7 @@
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaCUDA.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "clang/Sema/SemaDiagnostic.h"
 #include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
@@ -218,7 +219,8 @@ void Parser::ParseGNUAttributes(ParsedAttributes &Attrs,
         break;
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
-        Actions.CodeCompleteAttribute(AttributeCommonInfo::Syntax::AS_GNU);
+        Actions.CodeCompletion().CodeCompleteAttribute(
+            AttributeCommonInfo::Syntax::AS_GNU);
         break;
       }
       IdentifierInfo *AttrName = Tok.getIdentifierInfo();
@@ -895,7 +897,8 @@ void Parser::ParseMicrosoftDeclSpecs(ParsedAttributes &Attrs) {
 
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
-        Actions.CodeCompleteAttribute(AttributeCommonInfo::AS_Declspec);
+        Actions.CodeCompletion().CodeCompleteAttribute(
+            AttributeCommonInfo::AS_Declspec);
         return;
       }
 
@@ -2286,7 +2289,7 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
     // Check to see if we have a function *definition* which must have a body.
     if (Tok.is(tok::equal) && NextToken().is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteAfterFunctionEquals(D);
+      Actions.CodeCompletion().CodeCompleteAfterFunctionEquals(D);
       return nullptr;
     }
     // We're at the point where the parsing of function declarator is finished.
@@ -2722,7 +2725,8 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
 
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
-        Actions.CodeCompleteInitializer(getCurScope(), ThisDecl);
+        Actions.CodeCompletion().CodeCompleteInitializer(getCurScope(),
+                                                         ThisDecl);
         Actions.FinalizeDeclaration(ThisDecl);
         return nullptr;
       }
@@ -2767,10 +2771,11 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
 
     auto ThisVarDecl = dyn_cast_or_null<VarDecl>(ThisDecl);
     auto RunSignatureHelp = [&]() {
-      QualType PreferredType = Actions.ProduceConstructorSignatureHelp(
-          ThisVarDecl->getType()->getCanonicalTypeInternal(),
-          ThisDecl->getLocation(), Exprs, T.getOpenLocation(),
-          /*Braced=*/false);
+      QualType PreferredType =
+          Actions.CodeCompletion().ProduceConstructorSignatureHelp(
+              ThisVarDecl->getType()->getCanonicalTypeInternal(),
+              ThisDecl->getLocation(), Exprs, T.getOpenLocation(),
+              /*Braced=*/false);
       CalledSignatureHelp = true;
       return PreferredType;
     };
@@ -2791,7 +2796,7 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
 
     if (SawError) {
       if (ThisVarDecl && PP.isCodeCompletionReached() && !CalledSignatureHelp) {
-        Actions.ProduceConstructorSignatureHelp(
+        Actions.CodeCompletion().ProduceConstructorSignatureHelp(
             ThisVarDecl->getType()->getCanonicalTypeInternal(),
             ThisDecl->getLocation(), Exprs, T.getOpenLocation(),
             /*Braced=*/false);
@@ -3635,7 +3640,8 @@ void Parser::ParseDeclarationSpecifiers(
       continue;
 
     case tok::code_completion: {
-      Sema::ParserCompletionContext CCC = Sema::PCC_Namespace;
+      SemaCodeCompletion::ParserCompletionContext CCC =
+          SemaCodeCompletion::PCC_Namespace;
       if (DS.hasTypeSpecifier()) {
         bool AllowNonIdentifiers
           = (getCurScope()->getFlags() & (Scope::ControlScope |
@@ -3648,25 +3654,25 @@ void Parser::ParseDeclarationSpecifiers(
             (DSContext == DeclSpecContext::DSC_class && DS.isFriendSpecified());
 
         cutOffParsing();
-        Actions.CodeCompleteDeclSpec(getCurScope(), DS,
-                                     AllowNonIdentifiers,
-                                     AllowNestedNameSpecifiers);
+        Actions.CodeCompletion().CodeCompleteDeclSpec(
+            getCurScope(), DS, AllowNonIdentifiers, AllowNestedNameSpecifiers);
         return;
       }
 
       // Class context can appear inside a function/block, so prioritise that.
       if (TemplateInfo.Kind != ParsedTemplateInfo::NonTemplate)
-        CCC = DSContext == DeclSpecContext::DSC_class ? Sema::PCC_MemberTemplate
-                                                      : Sema::PCC_Template;
+        CCC = DSContext == DeclSpecContext::DSC_class
+                  ? SemaCodeCompletion::PCC_MemberTemplate
+                  : SemaCodeCompletion::PCC_Template;
       else if (DSContext == DeclSpecContext::DSC_class)
-        CCC = Sema::PCC_Class;
+        CCC = SemaCodeCompletion::PCC_Class;
       else if (getCurScope()->getFnParent() || getCurScope()->getBlockParent())
-        CCC = Sema::PCC_LocalDeclarationSpecifiers;
+        CCC = SemaCodeCompletion::PCC_LocalDeclarationSpecifiers;
       else if (CurParsedObjCImpl)
-        CCC = Sema::PCC_ObjCImplementation;
+        CCC = SemaCodeCompletion::PCC_ObjCImplementation;
 
       cutOffParsing();
-      Actions.CodeCompleteOrdinaryName(getCurScope(), CCC);
+      Actions.CodeCompletion().CodeCompleteOrdinaryName(getCurScope(), CCC);
       return;
     }
 
@@ -5076,7 +5082,7 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
   if (Tok.is(tok::code_completion)) {
     // Code completion for an enum name.
     cutOffParsing();
-    Actions.CodeCompleteTag(getCurScope(), DeclSpec::TST_enum);
+    Actions.CodeCompletion().CodeCompleteTag(getCurScope(), DeclSpec::TST_enum);
     DS.SetTypeSpecError(); // Needed by ActOnUsingDeclaration.
     return;
   }
@@ -6266,7 +6272,7 @@ void Parser::ParseTypeQualifierListOpt(
       if (CodeCompletionHandler)
         (*CodeCompletionHandler)();
       else
-        Actions.CodeCompleteTypeQualifiers(DS);
+        Actions.CodeCompletion().CodeCompleteTypeQualifiers(DS);
       return;
 
     case tok::kw_const:
@@ -7403,12 +7409,12 @@ void Parser::ParseFunctionDeclarator(Declarator &D,
       // with the pure-specifier in the same way.
 
       // Parse cv-qualifier-seq[opt].
-      ParseTypeQualifierListOpt(DS, AR_NoAttributesParsed,
-                                /*AtomicAllowed*/ false,
-                                /*IdentifierRequired=*/false,
-                                llvm::function_ref<void()>([&]() {
-                                  Actions.CodeCompleteFunctionQualifiers(DS, D);
-                                }));
+      ParseTypeQualifierListOpt(
+          DS, AR_NoAttributesParsed,
+          /*AtomicAllowed*/ false,
+          /*IdentifierRequired=*/false, llvm::function_ref<void()>([&]() {
+            Actions.CodeCompletion().CodeCompleteFunctionQualifiers(DS, D);
+          }));
       if (!DS.getSourceRange().getEnd().isInvalid()) {
         EndLoc = DS.getSourceRange().getEnd();
       }
@@ -7971,7 +7977,7 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
     return;
   } else if (Tok.getKind() == tok::code_completion) {
     cutOffParsing();
-    Actions.CodeCompleteBracketDeclarator(getCurScope());
+    Actions.CodeCompletion().CodeCompleteBracketDeclarator(getCurScope());
     return;
   }
 
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 3caad5c75d7d..5eaec2b621e6 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -27,6 +27,7 @@
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/TimeProfiler.h"
 #include <optional>
@@ -69,7 +70,7 @@ Parser::DeclGroupPtrTy Parser::ParseNamespace(DeclaratorContext Context,
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteNamespaceDecl(getCurScope());
+    Actions.CodeCompletion().CodeCompleteNamespaceDecl(getCurScope());
     return nullptr;
   }
 
@@ -309,7 +310,7 @@ Decl *Parser::ParseNamespaceAlias(SourceLocation NamespaceLoc,
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteNamespaceAliasDecl(getCurScope());
+    Actions.CodeCompletion().CodeCompleteNamespaceAliasDecl(getCurScope());
     return nullptr;
   }
 
@@ -492,7 +493,7 @@ Parser::DeclGroupPtrTy Parser::ParseUsingDirectiveOrDeclaration(
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteUsing(getCurScope());
+    Actions.CodeCompletion().CodeCompleteUsing(getCurScope());
     return nullptr;
   }
 
@@ -542,7 +543,7 @@ Decl *Parser::ParseUsingDirective(DeclaratorContext Context,
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteUsingDirective(getCurScope());
+    Actions.CodeCompletion().CodeCompleteUsingDirective(getCurScope());
     return nullptr;
   }
 
@@ -733,7 +734,7 @@ Parser::DeclGroupPtrTy Parser::ParseUsingDeclaration(
 
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteUsing(getCurScope());
+      Actions.CodeCompletion().CodeCompleteUsing(getCurScope());
       return nullptr;
     }
 
@@ -1680,7 +1681,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   if (Tok.is(tok::code_completion)) {
     // Code completion for a struct, class, or union name.
     cutOffParsing();
-    Actions.CodeCompleteTag(getCurScope(), TagType);
+    Actions.CodeCompletion().CodeCompleteTag(getCurScope(), TagType);
     return;
   }
 
@@ -2719,7 +2720,7 @@ void Parser::MaybeParseAndDiagnoseDeclSpecAfterCXX11VirtSpecifierSeq(
   ParseTypeQualifierListOpt(
       DS, AR_NoAttributesParsed, false,
       /*IdentifierRequired=*/false, llvm::function_ref<void()>([&]() {
-        Actions.CodeCompleteFunctionQualifiers(DS, D, &VS);
+        Actions.CodeCompletion().CodeCompleteFunctionQualifiers(DS, D, &VS);
       }));
   D.ExtendWithDeclSpec(DS);
 
@@ -3080,7 +3081,8 @@ Parser::DeclGroupPtrTy Parser::ParseCXXClassMemberDeclaration(
           DefinitionKind = FunctionDefinitionKind::Deleted;
         else if (KW.is(tok::code_completion)) {
           cutOffParsing();
-          Actions.CodeCompleteAfterFunctionEquals(DeclaratorInfo);
+          Actions.CodeCompletion().CodeCompleteAfterFunctionEquals(
+              DeclaratorInfo);
           return nullptr;
         }
       }
@@ -3892,8 +3894,8 @@ void Parser::ParseConstructorInitializer(Decl *ConstructorDecl) {
   do {
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteConstructorInitializer(ConstructorDecl,
-                                                 MemInitializers);
+      Actions.CodeCompletion().CodeCompleteConstructorInitializer(
+          ConstructorDecl, MemInitializers);
       return;
     }
 
@@ -4013,9 +4015,10 @@ MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
     auto RunSignatureHelp = [&] {
       if (TemplateTypeTy.isInvalid())
         return QualType();
-      QualType PreferredType = Actions.ProduceCtorInitMemberSignatureHelp(
-          ConstructorDecl, SS, TemplateTypeTy.get(), ArgExprs, II,
-          T.getOpenLocation(), /*Braced=*/false);
+      QualType PreferredType =
+          Actions.CodeCompletion().ProduceCtorInitMemberSignatureHelp(
+              ConstructorDecl, SS, TemplateTypeTy.get(), ArgExprs, II,
+              T.getOpenLocation(), /*Braced=*/false);
       CalledSignatureHelp = true;
       return PreferredType;
     };
@@ -4400,10 +4403,9 @@ void Parser::PopParsingClass(Sema::ParsingClassState state) {
 ///   If a keyword or an alternative token that satisfies the syntactic
 ///   requirements of an identifier is contained in an attribute-token,
 ///   it is considered an identifier.
-IdentifierInfo *
-Parser::TryParseCXX11AttributeIdentifier(SourceLocation &Loc,
-                                         Sema::AttributeCompletion Completion,
-                                         const IdentifierInfo *Scope) {
+IdentifierInfo *Parser::TryParseCXX11AttributeIdentifier(
+    SourceLocation &Loc, SemaCodeCompletion::AttributeCompletion Completion,
+    const IdentifierInfo *Scope) {
   switch (Tok.getKind()) {
   default:
     // Identifiers and keywords have identifier info attached.
@@ -4417,9 +4419,9 @@ Parser::TryParseCXX11AttributeIdentifier(SourceLocation &Loc,
 
   case tok::code_completion:
     cutOffParsing();
-    Actions.CodeCompleteAttribute(getLangOpts().CPlusPlus ? ParsedAttr::AS_CXX11
-                                                          : ParsedAttr::AS_C23,
-                                  Completion, Scope);
+    Actions.CodeCompletion().CodeCompleteAttribute(
+        getLangOpts().CPlusPlus ? ParsedAttr::AS_CXX11 : ParsedAttr::AS_C23,
+        Completion, Scope);
     return nullptr;
 
   case tok::numeric_constant: {
@@ -4802,7 +4804,7 @@ void Parser::ParseCXX11AttributeSpecifierInternal(ParsedAttributes &Attrs,
     ConsumeToken();
 
     CommonScopeName = TryParseCXX11AttributeIdentifier(
-        CommonScopeLoc, Sema::AttributeCompletion::Scope);
+        CommonScopeLoc, SemaCodeCompletion::AttributeCompletion::Scope);
     if (!CommonScopeName) {
       Diag(Tok.getLocation(), diag::err_expected) << tok::identifier;
       SkipUntil(tok::r_square, tok::colon, StopBeforeMatch);
@@ -4831,7 +4833,8 @@ void Parser::ParseCXX11AttributeSpecifierInternal(ParsedAttributes &Attrs,
     IdentifierInfo *ScopeName = nullptr, *AttrName = nullptr;
 
     AttrName = TryParseCXX11AttributeIdentifier(
-        AttrLoc, Sema::AttributeCompletion::Attribute, CommonScopeName);
+        AttrLoc, SemaCodeCompletion::AttributeCompletion::Attribute,
+        CommonScopeName);
     if (!AttrName)
       // Break out to the "expected ']'" diagnostic.
       break;
@@ -4842,7 +4845,8 @@ void Parser::ParseCXX11AttributeSpecifierInternal(ParsedAttributes &Attrs,
       ScopeLoc = AttrLoc;
 
       AttrName = TryParseCXX11AttributeIdentifier(
-          AttrLoc, Sema::AttributeCompletion::Attribute, ScopeName);
+          AttrLoc, SemaCodeCompletion::AttributeCompletion::Attribute,
+          ScopeName);
       if (!AttrName) {
         Diag(Tok.getLocation(), diag::err_expected) << tok::identifier;
         SkipUntil(tok::r_square, tok::comma, StopAtSemi | StopBeforeMatch);
@@ -5066,9 +5070,10 @@ void Parser::ParseMicrosoftAttributes(ParsedAttributes &Attrs) {
                 StopAtSemi | StopBeforeMatch | StopAtCodeCompletion);
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
-        Actions.CodeCompleteAttribute(AttributeCommonInfo::AS_Microsoft,
-                                      Sema::AttributeCompletion::Attribute,
-                                      /*Scope=*/nullptr);
+        Actions.CodeCompletion().CodeCompleteAttribute(
+            AttributeCommonInfo::AS_Microsoft,
+            SemaCodeCompletion::AttributeCompletion::Attribute,
+            /*Scope=*/nullptr);
         break;
       }
       if (Tok.isNot(tok::identifier)) // ']', but also eof
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 9cd7d20fc4d5..eb7447fa038e 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -31,6 +31,7 @@
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaCUDA.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenACC.h"
 #include "clang/Sema/SemaOpenMP.h"
@@ -168,8 +169,8 @@ Parser::ParseExpressionWithLeadingExtension(SourceLocation ExtLoc) {
 ExprResult Parser::ParseAssignmentExpression(TypeCastState isTypeCast) {
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteExpression(getCurScope(),
-                                   PreferredType.get(Tok.getLocation()));
+    Actions.CodeCompletion().CodeCompleteExpression(
+        getCurScope(), PreferredType.get(Tok.getLocation()));
     return ExprError();
   }
 
@@ -187,8 +188,8 @@ ExprResult Parser::ParseAssignmentExpression(TypeCastState isTypeCast) {
 ExprResult Parser::ParseConditionalExpression() {
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteExpression(getCurScope(),
-                                   PreferredType.get(Tok.getLocation()));
+    Actions.CodeCompletion().CodeCompleteExpression(
+        getCurScope(), PreferredType.get(Tok.getLocation()));
     return ExprError();
   }
 
@@ -1215,7 +1216,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
 
       if (Tok.is(tok::code_completion) && &II != Ident_super) {
         cutOffParsing();
-        Actions.CodeCompleteObjCClassPropertyRefExpr(
+        Actions.CodeCompletion().CodeCompleteObjCClassPropertyRefExpr(
             getCurScope(), II, ILoc, ExprStatementTokLoc == ILoc);
         return ExprError();
       }
@@ -1811,8 +1812,8 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     break;
   case tok::code_completion: {
     cutOffParsing();
-    Actions.CodeCompleteExpression(getCurScope(),
-                                   PreferredType.get(Tok.getLocation()));
+    Actions.CodeCompletion().CodeCompleteExpression(
+        getCurScope(), PreferredType.get(Tok.getLocation()));
     return ExprError();
   }
 #define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) case tok::kw___##Trait:
@@ -1956,7 +1957,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         return LHS;
 
       cutOffParsing();
-      Actions.CodeCompletePostfixExpression(
+      Actions.CodeCompletion().CodeCompletePostfixExpression(
           getCurScope(), LHS, PreferredType.get(Tok.getLocation()));
       return ExprError();
 
@@ -2154,8 +2155,9 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
 
       ExprVector ArgExprs;
       auto RunSignatureHelp = [&]() -> QualType {
-        QualType PreferredType = Actions.ProduceCallSignatureHelp(
-            LHS.get(), ArgExprs, PT.getOpenLocation());
+        QualType PreferredType =
+            Actions.CodeCompletion().ProduceCallSignatureHelp(
+                LHS.get(), ArgExprs, PT.getOpenLocation());
         CalledSignatureHelp = true;
         return PreferredType;
       };
@@ -2279,7 +2281,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
 
         // Code completion for a member access expression.
         cutOffParsing();
-        Actions.CodeCompleteMemberReferenceExpr(
+        Actions.CodeCompletion().CodeCompleteMemberReferenceExpr(
             getCurScope(), Base, CorrectedBase, OpLoc, OpKind == tok::arrow,
             Base && ExprStatementTokLoc == Base->getBeginLoc(),
             PreferredType.get(Tok.getLocation()));
@@ -3001,7 +3003,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteExpression(
+    Actions.CodeCompletion().CodeCompleteExpression(
         getCurScope(), PreferredType.get(Tok.getLocation()),
         /*IsParenthesized=*/ExprType >= CompoundLiteral);
     return ExprError();
@@ -3678,7 +3680,8 @@ bool Parser::ParseSimpleExpressionList(SmallVectorImpl<Expr *> &Exprs) {
 void Parser::ParseBlockId(SourceLocation CaretLoc) {
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Type);
+    Actions.CodeCompletion().CodeCompleteOrdinaryName(
+        getCurScope(), SemaCodeCompletion::PCC_Type);
     return;
   }
 
@@ -3867,7 +3870,7 @@ std::optional<AvailabilitySpec> Parser::ParseAvailabilitySpec() {
     // Parse the platform name.
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteAvailabilityPlatformName();
+      Actions.CodeCompletion().CodeCompleteAvailabilityPlatformName();
       return std::nullopt;
     }
     if (Tok.isNot(tok::identifier)) {
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 825031da358a..e149b1a0fb5e 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -23,6 +23,7 @@
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <numeric>
@@ -270,9 +271,9 @@ bool Parser::ParseOptionalCXXScopeSpecifier(
         cutOffParsing();
         // Code completion for a nested-name-specifier, where the code
         // completion token follows the '::'.
-        Actions.CodeCompleteQualifiedId(getCurScope(), SS, EnteringContext,
-                                        InUsingDeclaration, ObjectType.get(),
-                                        SavedType.get(SS.getBeginLoc()));
+        Actions.CodeCompletion().CodeCompleteQualifiedId(
+            getCurScope(), SS, EnteringContext, InUsingDeclaration,
+            ObjectType.get(), SavedType.get(SS.getBeginLoc()));
         // Include code completion token into the range of the scope otherwise
         // when we try to annotate the scope tokens the dangling code completion
         // token will cause assertion in
@@ -954,8 +955,9 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro,
         if (Tok.is(tok::code_completion) &&
             !(getLangOpts().ObjC && Tentative)) {
           cutOffParsing();
-          Actions.CodeCompleteLambdaIntroducer(getCurScope(), Intro,
-                                               /*AfterAmpersand=*/false);
+          Actions.CodeCompletion().CodeCompleteLambdaIntroducer(
+              getCurScope(), Intro,
+              /*AfterAmpersand=*/false);
           break;
         }
 
@@ -971,10 +973,11 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro,
       // If we're in Objective-C++ and we have a bare '[', then this is more
       // likely to be a message receiver.
       if (getLangOpts().ObjC && Tentative && First)
-        Actions.CodeCompleteObjCMessageReceiver(getCurScope());
+        Actions.CodeCompletion().CodeCompleteObjCMessageReceiver(getCurScope());
       else
-        Actions.CodeCompleteLambdaIntroducer(getCurScope(), Intro,
-                                             /*AfterAmpersand=*/false);
+        Actions.CodeCompletion().CodeCompleteLambdaIntroducer(
+            getCurScope(), Intro,
+            /*AfterAmpersand=*/false);
       break;
     }
 
@@ -1020,8 +1023,9 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro,
 
         if (Tok.is(tok::code_completion)) {
           cutOffParsing();
-          Actions.CodeCompleteLambdaIntroducer(getCurScope(), Intro,
-                                               /*AfterAmpersand=*/true);
+          Actions.CodeCompletion().CodeCompleteLambdaIntroducer(
+              getCurScope(), Intro,
+              /*AfterAmpersand=*/true);
           break;
         }
       }
@@ -2031,9 +2035,10 @@ Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
     auto RunSignatureHelp = [&]() {
       QualType PreferredType;
       if (TypeRep)
-        PreferredType = Actions.ProduceConstructorSignatureHelp(
-            TypeRep.get()->getCanonicalTypeInternal(), DS.getEndLoc(), Exprs,
-            T.getOpenLocation(), /*Braced=*/false);
+        PreferredType =
+            Actions.CodeCompletion().ProduceConstructorSignatureHelp(
+                TypeRep.get()->getCanonicalTypeInternal(), DS.getEndLoc(),
+                Exprs, T.getOpenLocation(), /*Braced=*/false);
       CalledSignatureHelp = true;
       return PreferredType;
     };
@@ -2140,7 +2145,8 @@ Parser::ParseCXXCondition(StmtResult *InitStmt, SourceLocation Loc,
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Condition);
+    Actions.CodeCompletion().CodeCompleteOrdinaryName(
+        getCurScope(), SemaCodeCompletion::PCC_Condition);
     return Sema::ConditionError();
   }
 
@@ -2792,7 +2798,7 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
       // Don't try to parse any further.
       cutOffParsing();
       // Code completion for the operator name.
-      Actions.CodeCompleteOperatorName(getCurScope());
+      Actions.CodeCompletion().CodeCompleteOperatorName(getCurScope());
       return true;
     }
 
@@ -3350,10 +3356,12 @@ Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
         // the passing DeclaratorInfo is valid, e.g. running SignatureHelp on
         // `new decltype(invalid) (^)`.
         if (TypeRep)
-          PreferredType = Actions.ProduceConstructorSignatureHelp(
-              TypeRep.get()->getCanonicalTypeInternal(),
-              DeclaratorInfo.getEndLoc(), ConstructorArgs, ConstructorLParen,
-              /*Braced=*/false);
+          PreferredType =
+              Actions.CodeCompletion().ProduceConstructorSignatureHelp(
+                  TypeRep.get()->getCanonicalTypeInternal(),
+                  DeclaratorInfo.getEndLoc(), ConstructorArgs,
+                  ConstructorLParen,
+                  /*Braced=*/false);
         CalledSignatureHelp = true;
         return PreferredType;
       };
diff --git a/clang/lib/Parse/ParseInit.cpp b/clang/lib/Parse/ParseInit.cpp
index 04e4419f4d45..432ddc74b108 100644
--- a/clang/lib/Parse/ParseInit.cpp
+++ b/clang/lib/Parse/ParseInit.cpp
@@ -18,6 +18,7 @@
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
@@ -204,8 +205,9 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator(
 
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
-        Actions.CodeCompleteDesignator(DesignatorCompletion.PreferredBaseType,
-                                       DesignatorCompletion.InitExprs, Desig);
+        Actions.CodeCompletion().CodeCompleteDesignator(
+            DesignatorCompletion.PreferredBaseType,
+            DesignatorCompletion.InitExprs, Desig);
         return ExprError();
       }
       if (Tok.isNot(tok::identifier)) {
@@ -471,7 +473,7 @@ ExprResult Parser::ParseBraceInitializer() {
   auto RunSignatureHelp = [&] {
     QualType PreferredType;
     if (!LikelyType.isNull())
-      PreferredType = Actions.ProduceConstructorSignatureHelp(
+      PreferredType = Actions.CodeCompletion().ProduceConstructorSignatureHelp(
           LikelyType->getCanonicalTypeInternal(), T.getOpenLocation(),
           InitExprs, T.getOpenLocation(), /*Braced=*/true);
     CalledSignatureHelp = true;
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index 263aea2e5524..89f4acbd25e4 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -20,6 +20,7 @@
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
@@ -56,7 +57,7 @@ Parser::ParseObjCAtDirectives(ParsedAttributes &DeclAttrs,
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteObjCAtDirective(getCurScope());
+    Actions.CodeCompletion().CodeCompleteObjCAtDirective(getCurScope());
     return nullptr;
   }
 
@@ -156,7 +157,7 @@ Parser::ParseObjCAtClassDeclaration(SourceLocation atLoc) {
     MaybeSkipAttributes(tok::objc_class);
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteObjCClassForwardDecl(getCurScope());
+      Actions.CodeCompletion().CodeCompleteObjCClassForwardDecl(getCurScope());
       return Actions.ConvertDeclToDeclGroup(nullptr);
     }
     if (expectIdentifier()) {
@@ -242,7 +243,7 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
   // Code completion after '@interface'.
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteObjCInterfaceDecl(getCurScope());
+    Actions.CodeCompletion().CodeCompleteObjCInterfaceDecl(getCurScope());
     return nullptr;
   }
 
@@ -276,7 +277,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
     IdentifierInfo *categoryId = nullptr;
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteObjCInterfaceCategory(getCurScope(), nameId, nameLoc);
+      Actions.CodeCompletion().CodeCompleteObjCInterfaceCategory(
+          getCurScope(), nameId, nameLoc);
       return nullptr;
     }
 
@@ -331,7 +333,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc,
     // Code completion of superclass names.
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteObjCSuperclass(getCurScope(), nameId, nameLoc);
+      Actions.CodeCompletion().CodeCompleteObjCSuperclass(getCurScope(), nameId,
+                                                          nameLoc);
       return nullptr;
     }
 
@@ -508,7 +511,8 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
         // FIXME: If these aren't protocol references, we'll need different
         // completions.
         cutOffParsing();
-        Actions.CodeCompleteObjCProtocolReferences(protocolIdents);
+        Actions.CodeCompletion().CodeCompleteObjCProtocolReferences(
+            protocolIdents);
 
         // FIXME: Better recovery here?.
         return nullptr;
@@ -681,9 +685,10 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
     // Code completion within an Objective-C interface.
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteOrdinaryName(getCurScope(),
-                            CurParsedObjCImpl? Sema::PCC_ObjCImplementation
-                                             : Sema::PCC_ObjCInterface);
+      Actions.CodeCompletion().CodeCompleteOrdinaryName(
+          getCurScope(), CurParsedObjCImpl
+                             ? SemaCodeCompletion::PCC_ObjCImplementation
+                             : SemaCodeCompletion::PCC_ObjCInterface);
       return;
     }
 
@@ -720,7 +725,7 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
     const auto &NextTok = NextToken();
     if (NextTok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteObjCAtDirective(getCurScope());
+      Actions.CodeCompletion().CodeCompleteObjCAtDirective(getCurScope());
       return;
     }
 
@@ -895,7 +900,7 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS) {
   while (true) {
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteObjCPropertyFlags(getCurScope(), DS);
+      Actions.CodeCompletion().CodeCompleteObjCPropertyFlags(getCurScope(), DS);
       return;
     }
     const IdentifierInfo *II = Tok.getIdentifierInfo();
@@ -943,9 +948,11 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS) {
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
         if (IsSetter)
-          Actions.CodeCompleteObjCPropertySetter(getCurScope());
+          Actions.CodeCompletion().CodeCompleteObjCPropertySetter(
+              getCurScope());
         else
-          Actions.CodeCompleteObjCPropertyGetter(getCurScope());
+          Actions.CodeCompletion().CodeCompleteObjCPropertyGetter(
+              getCurScope());
         return;
       }
 
@@ -1196,7 +1203,7 @@ void Parser::ParseObjCTypeQualifierList(ObjCDeclSpec &DS,
   while (true) {
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteObjCPassingType(
+      Actions.CodeCompletion().CodeCompleteObjCPassingType(
           getCurScope(), DS, Context == DeclaratorContext::ObjCParameter);
       return;
     }
@@ -1390,8 +1397,9 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus,
-                                       /*ReturnType=*/nullptr);
+    Actions.CodeCompletion().CodeCompleteObjCMethodDecl(getCurScope(),
+                                                        mType == tok::minus,
+                                                        /*ReturnType=*/nullptr);
     return nullptr;
   }
 
@@ -1409,8 +1417,8 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus,
-                                       ReturnType);
+    Actions.CodeCompletion().CodeCompleteObjCMethodDecl(
+        getCurScope(), mType == tok::minus, ReturnType);
     return nullptr;
   }
 
@@ -1472,10 +1480,9 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
       KeyIdents.push_back(SelIdent);
-      Actions.CodeCompleteObjCMethodDeclSelector(getCurScope(),
-                                                 mType == tok::minus,
-                                                 /*AtParameterName=*/true,
-                                                 ReturnType, KeyIdents);
+      Actions.CodeCompletion().CodeCompleteObjCMethodDeclSelector(
+          getCurScope(), mType == tok::minus,
+          /*AtParameterName=*/true, ReturnType, KeyIdents);
       return nullptr;
     }
 
@@ -1496,10 +1503,9 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
     // Code completion for the next piece of the selector.
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteObjCMethodDeclSelector(getCurScope(),
-                                                 mType == tok::minus,
-                                                 /*AtParameterName=*/false,
-                                                 ReturnType, KeyIdents);
+      Actions.CodeCompletion().CodeCompleteObjCMethodDeclSelector(
+          getCurScope(), mType == tok::minus,
+          /*AtParameterName=*/false, ReturnType, KeyIdents);
       return nullptr;
     }
 
@@ -1584,7 +1590,8 @@ ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
   while (true) {
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteObjCProtocolReferences(ProtocolIdents);
+      Actions.CodeCompletion().CodeCompleteObjCProtocolReferences(
+          ProtocolIdents);
       return true;
     }
 
@@ -1682,9 +1689,11 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers(
       QualType BaseT = Actions.GetTypeFromParser(baseType);
       cutOffParsing();
       if (!BaseT.isNull() && BaseT->acceptsObjCTypeParams()) {
-        Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Type);
+        Actions.CodeCompletion().CodeCompleteOrdinaryName(
+            getCurScope(), SemaCodeCompletion::PCC_Type);
       } else {
-        Actions.CodeCompleteObjCProtocolReferences(identifierLocPairs);
+        Actions.CodeCompletion().CodeCompleteObjCProtocolReferences(
+            identifierLocPairs);
       }
       return;
     }
@@ -1959,7 +1968,7 @@ void Parser::ParseObjCClassInstanceVariables(ObjCContainerDecl *interfaceDecl,
     if (TryConsumeToken(tok::at)) { // parse objc-visibility-spec
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
-        Actions.CodeCompleteObjCAtVisibility(getCurScope());
+        Actions.CodeCompletion().CodeCompleteObjCAtVisibility(getCurScope());
         return;
       }
 
@@ -1990,8 +1999,8 @@ void Parser::ParseObjCClassInstanceVariables(ObjCContainerDecl *interfaceDecl,
 
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteOrdinaryName(getCurScope(),
-                                       Sema::PCC_ObjCInstanceVariableList);
+      Actions.CodeCompletion().CodeCompleteOrdinaryName(
+          getCurScope(), SemaCodeCompletion::PCC_ObjCInstanceVariableList);
       return;
     }
 
@@ -2058,7 +2067,7 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteObjCProtocolDecl(getCurScope());
+    Actions.CodeCompletion().CodeCompleteObjCProtocolDecl(getCurScope());
     return nullptr;
   }
 
@@ -2156,7 +2165,7 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc,
   // Code completion after '@implementation'.
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteObjCImplementationDecl(getCurScope());
+    Actions.CodeCompletion().CodeCompleteObjCImplementationDecl(getCurScope());
     return nullptr;
   }
 
@@ -2194,7 +2203,8 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc,
 
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteObjCImplementationCategory(getCurScope(), nameId, nameLoc);
+      Actions.CodeCompletion().CodeCompleteObjCImplementationCategory(
+          getCurScope(), nameId, nameLoc);
       return nullptr;
     }
 
@@ -2368,7 +2378,8 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
   while (true) {
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteObjCPropertyDefinition(getCurScope());
+      Actions.CodeCompletion().CodeCompleteObjCPropertyDefinition(
+          getCurScope());
       return nullptr;
     }
 
@@ -2386,7 +2397,8 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
       // property '=' ivar-name
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
-        Actions.CodeCompleteObjCPropertySynthesizeIvar(getCurScope(), propertyId);
+        Actions.CodeCompletion().CodeCompleteObjCPropertySynthesizeIvar(
+            getCurScope(), propertyId);
         return nullptr;
       }
 
@@ -2445,7 +2457,8 @@ Decl *Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
   while (true) {
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteObjCPropertyDefinition(getCurScope());
+      Actions.CodeCompletion().CodeCompleteObjCPropertyDefinition(
+          getCurScope());
       return nullptr;
     }
 
@@ -2784,7 +2797,7 @@ StmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc,
                                         ParsedStmtContext StmtCtx) {
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteObjCAtStatement(getCurScope());
+    Actions.CodeCompletion().CodeCompleteObjCAtStatement(getCurScope());
     return StmtError();
   }
 
@@ -2825,7 +2838,7 @@ ExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) {
   switch (Tok.getKind()) {
   case tok::code_completion:
     cutOffParsing();
-    Actions.CodeCompleteObjCAtExpression(getCurScope());
+    Actions.CodeCompletion().CodeCompleteObjCAtExpression(getCurScope());
     return ExprError();
 
   case tok::minus:
@@ -3073,7 +3086,7 @@ ExprResult Parser::ParseObjCMessageExpression() {
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteObjCMessageReceiver(getCurScope());
+    Actions.CodeCompletion().CodeCompleteObjCMessageReceiver(getCurScope());
     return ExprError();
   }
 
@@ -3210,14 +3223,14 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
     if (SuperLoc.isValid())
-      Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc,
-                                           std::nullopt, false);
+      Actions.CodeCompletion().CodeCompleteObjCSuperMessage(
+          getCurScope(), SuperLoc, std::nullopt, false);
     else if (ReceiverType)
-      Actions.CodeCompleteObjCClassMessage(getCurScope(), ReceiverType,
-                                           std::nullopt, false);
+      Actions.CodeCompletion().CodeCompleteObjCClassMessage(
+          getCurScope(), ReceiverType, std::nullopt, false);
     else
-      Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
-                                              std::nullopt, false);
+      Actions.CodeCompletion().CodeCompleteObjCInstanceMessage(
+          getCurScope(), ReceiverExpr, std::nullopt, false);
     return ExprError();
   }
 
@@ -3248,17 +3261,17 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
         if (SuperLoc.isValid())
-          Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc,
-                                               KeyIdents,
-                                               /*AtArgumentExpression=*/true);
+          Actions.CodeCompletion().CodeCompleteObjCSuperMessage(
+              getCurScope(), SuperLoc, KeyIdents,
+              /*AtArgumentExpression=*/true);
         else if (ReceiverType)
-          Actions.CodeCompleteObjCClassMessage(getCurScope(), ReceiverType,
-                                               KeyIdents,
-                                               /*AtArgumentExpression=*/true);
+          Actions.CodeCompletion().CodeCompleteObjCClassMessage(
+              getCurScope(), ReceiverType, KeyIdents,
+              /*AtArgumentExpression=*/true);
         else
-          Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
-                                                  KeyIdents,
-                                                  /*AtArgumentExpression=*/true);
+          Actions.CodeCompletion().CodeCompleteObjCInstanceMessage(
+              getCurScope(), ReceiverExpr, KeyIdents,
+              /*AtArgumentExpression=*/true);
 
         return ExprError();
       }
@@ -3286,17 +3299,17 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
         if (SuperLoc.isValid())
-          Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc,
-                                               KeyIdents,
-                                               /*AtArgumentExpression=*/false);
+          Actions.CodeCompletion().CodeCompleteObjCSuperMessage(
+              getCurScope(), SuperLoc, KeyIdents,
+              /*AtArgumentExpression=*/false);
         else if (ReceiverType)
-          Actions.CodeCompleteObjCClassMessage(getCurScope(), ReceiverType,
-                                               KeyIdents,
-                                               /*AtArgumentExpression=*/false);
+          Actions.CodeCompletion().CodeCompleteObjCClassMessage(
+              getCurScope(), ReceiverType, KeyIdents,
+              /*AtArgumentExpression=*/false);
         else
-          Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
-                                                  KeyIdents,
-                                                /*AtArgumentExpression=*/false);
+          Actions.CodeCompletion().CodeCompleteObjCInstanceMessage(
+              getCurScope(), ReceiverExpr, KeyIdents,
+              /*AtArgumentExpression=*/false);
         return ExprError();
       }
 
@@ -3637,7 +3650,7 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents);
+    Actions.CodeCompletion().CodeCompleteObjCSelector(getCurScope(), KeyIdents);
     return ExprError();
   }
 
@@ -3663,7 +3676,8 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
 
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
-        Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents);
+        Actions.CodeCompletion().CodeCompleteObjCSelector(getCurScope(),
+                                                          KeyIdents);
         return ExprError();
       }
 
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index ca2c6d69eb98..e959dd6378f4 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -21,6 +21,7 @@
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -460,7 +461,8 @@ void Parser::ParseOpenMPReductionInitializerForDecl(VarDecl *OmpPrivParm) {
 
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteInitializer(getCurScope(), OmpPrivParm);
+      Actions.CodeCompletion().CodeCompleteInitializer(getCurScope(),
+                                                       OmpPrivParm);
       Actions.FinalizeDeclaration(OmpPrivParm);
       return;
     }
@@ -484,9 +486,10 @@ void Parser::ParseOpenMPReductionInitializerForDecl(VarDecl *OmpPrivParm) {
 
     SourceLocation LParLoc = T.getOpenLocation();
     auto RunSignatureHelp = [this, OmpPrivParm, LParLoc, &Exprs]() {
-      QualType PreferredType = Actions.ProduceConstructorSignatureHelp(
-          OmpPrivParm->getType()->getCanonicalTypeInternal(),
-          OmpPrivParm->getLocation(), Exprs, LParLoc, /*Braced=*/false);
+      QualType PreferredType =
+          Actions.CodeCompletion().ProduceConstructorSignatureHelp(
+              OmpPrivParm->getType()->getCanonicalTypeInternal(),
+              OmpPrivParm->getLocation(), Exprs, LParLoc, /*Braced=*/false);
       CalledSignatureHelp = true;
       return PreferredType;
     };
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index cd0fab5fe31d..643fdac287d1 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -22,6 +22,7 @@
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaCUDA.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include <optional>
@@ -1924,7 +1925,8 @@ void Parser::HandlePragmaAttribute() {
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
       // FIXME: suppress completion of unsupported attributes?
-      Actions.CodeCompleteAttribute(AttributeCommonInfo::Syntax::AS_GNU);
+      Actions.CodeCompletion().CodeCompleteAttribute(
+          AttributeCommonInfo::Syntax::AS_GNU);
       return SkipToEnd();
     }
 
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index e0116d300310..b0af04451166 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -22,6 +22,7 @@
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/TypoCorrection.h"
@@ -191,7 +192,8 @@ Retry:
 
   case tok::code_completion:
     cutOffParsing();
-    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Statement);
+    Actions.CodeCompletion().CodeCompleteOrdinaryName(
+        getCurScope(), SemaCodeCompletion::PCC_Statement);
     return StmtError();
 
   case tok::identifier:
@@ -843,7 +845,7 @@ StmtResult Parser::ParseCaseStatement(ParsedStmtContext StmtCtx,
 
     if (Tok.is(tok::code_completion)) {
       cutOffParsing();
-      Actions.CodeCompleteCase(getCurScope());
+      Actions.CodeCompletion().CodeCompleteCase(getCurScope());
       return StmtError();
     }
 
@@ -1649,7 +1651,7 @@ StmtResult Parser::ParseIfStatement(SourceLocation *TrailingElseLoc) {
     InnerScope.Exit();
   } else if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteAfterIf(getCurScope(), IsBracedThen);
+    Actions.CodeCompletion().CodeCompleteAfterIf(getCurScope(), IsBracedThen);
     return StmtError();
   } else if (InnerStatementTrailingElseLoc.isValid()) {
     Diag(InnerStatementTrailingElseLoc, diag::warn_dangling_else);
@@ -2040,9 +2042,9 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
 
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
-    Actions.CodeCompleteOrdinaryName(getCurScope(),
-                                     C99orCXXorObjC? Sema::PCC_ForInit
-                                                   : Sema::PCC_Expression);
+    Actions.CodeCompletion().CodeCompleteOrdinaryName(
+        getCurScope(), C99orCXXorObjC ? SemaCodeCompletion::PCC_ForInit
+                                      : SemaCodeCompletion::PCC_Expression);
     return StmtError();
   }
 
@@ -2117,7 +2119,8 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
 
         if (Tok.is(tok::code_completion)) {
           cutOffParsing();
-          Actions.CodeCompleteObjCForCollection(getCurScope(), DG);
+          Actions.CodeCompletion().CodeCompleteObjCForCollection(getCurScope(),
+                                                                 DG);
           return StmtError();
         }
         Collection = ParseExpression();
@@ -2154,7 +2157,8 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
 
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
-        Actions.CodeCompleteObjCForCollection(getCurScope(), nullptr);
+        Actions.CodeCompletion().CodeCompleteObjCForCollection(getCurScope(),
+                                                               nullptr);
         return StmtError();
       }
       Collection = ParseExpression();
@@ -2431,8 +2435,8 @@ StmtResult Parser::ParseReturnStatement() {
     // FIXME: Code completion for co_return.
     if (Tok.is(tok::code_completion) && !IsCoreturn) {
       cutOffParsing();
-      Actions.CodeCompleteExpression(getCurScope(),
-                                     PreferredType.get(Tok.getLocation()));
+      Actions.CodeCompletion().CodeCompleteExpression(
+          getCurScope(), PreferredType.get(Tok.getLocation()));
       return StmtError();
     }
 
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index 22ef44aaa087..a5130f56600e 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -1541,8 +1541,8 @@ bool Parser::ParseTemplateArgumentList(TemplateArgList &TemplateArgs,
     if (!Template)
       return QualType();
     CalledSignatureHelp = true;
-    return Actions.ProduceTemplateArgumentSignatureHelp(Template, TemplateArgs,
-                                                        OpenLoc);
+    return Actions.CodeCompletion().ProduceTemplateArgumentSignatureHelp(
+        Template, TemplateArgs, OpenLoc);
   };
 
   do {
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index e3e72a441ca6..6d0cf7b174e5 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -21,6 +21,7 @@
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TimeProfiler.h"
 using namespace clang;
@@ -944,20 +945,21 @@ Parser::ParseExternalDeclaration(ParsedAttributes &Attrs,
     cutOffParsing();
     if (CurParsedObjCImpl) {
       // Code-complete Objective-C methods even without leading '-'/'+' prefix.
-      Actions.CodeCompleteObjCMethodDecl(getCurScope(),
-                                         /*IsInstanceMethod=*/std::nullopt,
-                                         /*ReturnType=*/nullptr);
+      Actions.CodeCompletion().CodeCompleteObjCMethodDecl(
+          getCurScope(),
+          /*IsInstanceMethod=*/std::nullopt,
+          /*ReturnType=*/nullptr);
     }
 
-    Sema::ParserCompletionContext PCC;
+    SemaCodeCompletion::ParserCompletionContext PCC;
     if (CurParsedObjCImpl) {
-      PCC = Sema::PCC_ObjCImplementation;
+      PCC = SemaCodeCompletion::PCC_ObjCImplementation;
     } else if (PP.isIncrementalProcessingEnabled()) {
-      PCC = Sema::PCC_TopLevelOrExpression;
+      PCC = SemaCodeCompletion::PCC_TopLevelOrExpression;
     } else {
-      PCC = Sema::PCC_Namespace;
+      PCC = SemaCodeCompletion::PCC_Namespace;
     };
-    Actions.CodeCompleteOrdinaryName(getCurScope(), PCC);
+    Actions.CodeCompletion().CodeCompleteOrdinaryName(getCurScope(), PCC);
     return nullptr;
   case tok::kw_import: {
     Sema::ModuleImportState IS = Sema::ModuleImportState::NotACXX20Module;
@@ -2281,54 +2283,57 @@ SourceLocation Parser::handleUnexpectedCodeCompletionToken() {
   for (Scope *S = getCurScope(); S; S = S->getParent()) {
     if (S->isFunctionScope()) {
       cutOffParsing();
-      Actions.CodeCompleteOrdinaryName(getCurScope(),
-                                       Sema::PCC_RecoveryInFunction);
+      Actions.CodeCompletion().CodeCompleteOrdinaryName(
+          getCurScope(), SemaCodeCompletion::PCC_RecoveryInFunction);
       return PrevTokLocation;
     }
 
     if (S->isClassScope()) {
       cutOffParsing();
-      Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Class);
+      Actions.CodeCompletion().CodeCompleteOrdinaryName(
+          getCurScope(), SemaCodeCompletion::PCC_Class);
       return PrevTokLocation;
     }
   }
 
   cutOffParsing();
-  Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Namespace);
+  Actions.CodeCompletion().CodeCompleteOrdinaryName(
+      getCurScope(), SemaCodeCompletion::PCC_Namespace);
   return PrevTokLocation;
 }
 
 // Code-completion pass-through functions
 
 void Parser::CodeCompleteDirective(bool InConditional) {
-  Actions.CodeCompletePreprocessorDirective(InConditional);
+  Actions.CodeCompletion().CodeCompletePreprocessorDirective(InConditional);
 }
 
 void Parser::CodeCompleteInConditionalExclusion() {
-  Actions.CodeCompleteInPreprocessorConditionalExclusion(getCurScope());
+  Actions.CodeCompletion().CodeCompleteInPreprocessorConditionalExclusion(
+      getCurScope());
 }
 
 void Parser::CodeCompleteMacroName(bool IsDefinition) {
-  Actions.CodeCompletePreprocessorMacroName(IsDefinition);
+  Actions.CodeCompletion().CodeCompletePreprocessorMacroName(IsDefinition);
 }
 
 void Parser::CodeCompletePreprocessorExpression() {
-  Actions.CodeCompletePreprocessorExpression();
+  Actions.CodeCompletion().CodeCompletePreprocessorExpression();
 }
 
 void Parser::CodeCompleteMacroArgument(IdentifierInfo *Macro,
                                        MacroInfo *MacroInfo,
                                        unsigned ArgumentIndex) {
-  Actions.CodeCompletePreprocessorMacroArgument(getCurScope(), Macro, MacroInfo,
-                                                ArgumentIndex);
+  Actions.CodeCompletion().CodeCompletePreprocessorMacroArgument(
+      getCurScope(), Macro, MacroInfo, ArgumentIndex);
 }
 
 void Parser::CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled) {
-  Actions.CodeCompleteIncludedFile(Dir, IsAngled);
+  Actions.CodeCompletion().CodeCompleteIncludedFile(Dir, IsAngled);
 }
 
 void Parser::CodeCompleteNaturalLanguage() {
-  Actions.CodeCompleteNaturalLanguage();
+  Actions.CodeCompletion().CodeCompleteNaturalLanguage();
 }
 
 bool Parser::ParseMicrosoftIfExistsCondition(IfExistsCondition& Result) {
@@ -2682,7 +2687,7 @@ bool Parser::ParseModuleName(
     if (!Tok.is(tok::identifier)) {
       if (Tok.is(tok::code_completion)) {
         cutOffParsing();
-        Actions.CodeCompleteModuleImport(UseLoc, Path);
+        Actions.CodeCompletion().CodeCompleteModuleImport(UseLoc, Path);
         return true;
       }
 
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 6c548f374cbe..f847c49920cf 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -42,6 +42,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "clang/Sema/SemaConsumer.h"
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
@@ -202,6 +203,8 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
       LateTemplateParser(nullptr), LateTemplateParserCleanup(nullptr),
       OpaqueParser(nullptr), CurContext(nullptr), ExternalSource(nullptr),
       CurScope(nullptr), Ident_super(nullptr),
+      CodeCompletionPtr(
+          std::make_unique<SemaCodeCompletion>(*this, CodeCompleter)),
       CUDAPtr(std::make_unique<SemaCUDA>(*this)),
       HLSLPtr(std::make_unique<SemaHLSL>(*this)),
       ObjCPtr(std::make_unique<SemaObjC>(*this)),
@@ -225,8 +228,7 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
       TyposCorrected(0), IsBuildingRecoveryCallExpr(false), NumSFINAEErrors(0),
       AccessCheckingSFINAE(false), CurrentInstantiationScope(nullptr),
       InNonInstantiationSFINAEContext(false), NonInstantiationEntries(0),
-      ArgumentPackSubstitutionIndex(-1), SatisfactionCache(Context),
-      CodeCompleter(CodeCompleter) {
+      ArgumentPackSubstitutionIndex(-1), SatisfactionCache(Context) {
   assert(pp.TUKind == TUKind);
   TUScope = nullptr;
 
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 328641ed9488..ad3ca4cc94ca 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -41,6 +41,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaCodeCompletion.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/SemaObjC.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -246,8 +247,8 @@ public:
   /// Whether we should include code patterns in the completion
   /// results.
   bool includeCodePatterns() const {
-    return SemaRef.CodeCompleter &&
-           SemaRef.CodeCompleter->includeCodePatterns();
+    return SemaRef.CodeCompletion().CodeCompleter &&
+           SemaRef.CodeCompletion().CodeCompleter->includeCodePatterns();
   }
 
   /// Set the filter used for code-completion results.
@@ -1863,9 +1864,9 @@ static void AddTypeSpecifierResults(const LangOptions &LangOpts,
   Results.AddResult(Result("_Nullable", CCP_Type));
 }
 
-static void AddStorageSpecifiers(Sema::ParserCompletionContext CCC,
-                                 const LangOptions &LangOpts,
-                                 ResultBuilder &Results) {
+static void
+AddStorageSpecifiers(SemaCodeCompletion::ParserCompletionContext CCC,
+                     const LangOptions &LangOpts, ResultBuilder &Results) {
   typedef CodeCompletionResult Result;
   // Note: we don't suggest either "auto" or "register", because both
   // are pointless as storage specifiers. Elsewhere, we suggest "auto"
@@ -1889,13 +1890,13 @@ static void AddStorageSpecifiers(Sema::ParserCompletionContext CCC,
   }
 }
 
-static void AddFunctionSpecifiers(Sema::ParserCompletionContext CCC,
-                                  const LangOptions &LangOpts,
-                                  ResultBuilder &Results) {
+static void
+AddFunctionSpecifiers(SemaCodeCompletion::ParserCompletionContext CCC,
+                      const LangOptions &LangOpts, ResultBuilder &Results) {
   typedef CodeCompletionResult Result;
   switch (CCC) {
-  case Sema::PCC_Class:
-  case Sema::PCC_MemberTemplate:
+  case SemaCodeCompletion::PCC_Class:
+  case SemaCodeCompletion::PCC_MemberTemplate:
     if (LangOpts.CPlusPlus) {
       Results.AddResult(Result("explicit"));
       Results.AddResult(Result("friend"));
@@ -1904,24 +1905,24 @@ static void AddFunctionSpecifiers(Sema::ParserCompletionContext CCC,
     }
     [[fallthrough]];
 
-  case Sema::PCC_ObjCInterface:
-  case Sema::PCC_ObjCImplementation:
-  case Sema::PCC_Namespace:
-  case Sema::PCC_Template:
+  case SemaCodeCompletion::PCC_ObjCInterface:
+  case SemaCodeCompletion::PCC_ObjCImplementation:
+  case SemaCodeCompletion::PCC_Namespace:
+  case SemaCodeCompletion::PCC_Template:
     if (LangOpts.CPlusPlus || LangOpts.C99)
       Results.AddResult(Result("inline"));
     break;
 
-  case Sema::PCC_ObjCInstanceVariableList:
-  case Sema::PCC_Expression:
-  case Sema::PCC_Statement:
-  case Sema::PCC_TopLevelOrExpression:
-  case Sema::PCC_ForInit:
-  case Sema::PCC_Condition:
-  case Sema::PCC_RecoveryInFunction:
-  case Sema::PCC_Type:
-  case Sema::PCC_ParenthesizedExpression:
-  case Sema::PCC_LocalDeclarationSpecifiers:
+  case SemaCodeCompletion::PCC_ObjCInstanceVariableList:
+  case SemaCodeCompletion::PCC_Expression:
+  case SemaCodeCompletion::PCC_Statement:
+  case SemaCodeCompletion::PCC_TopLevelOrExpression:
+  case SemaCodeCompletion::PCC_ForInit:
+  case SemaCodeCompletion::PCC_Condition:
+  case SemaCodeCompletion::PCC_RecoveryInFunction:
+  case SemaCodeCompletion::PCC_Type:
+  case SemaCodeCompletion::PCC_ParenthesizedExpression:
+  case SemaCodeCompletion::PCC_LocalDeclarationSpecifiers:
     break;
   }
 }
@@ -1960,31 +1961,31 @@ static void AddUsingAliasResult(CodeCompletionBuilder &Builder,
   Results.AddResult(CodeCompletionResult(Builder.TakeString()));
 }
 
-static bool WantTypesInContext(Sema::ParserCompletionContext CCC,
+static bool WantTypesInContext(SemaCodeCompletion::ParserCompletionContext CCC,
                                const LangOptions &LangOpts) {
   switch (CCC) {
-  case Sema::PCC_Namespace:
-  case Sema::PCC_Class:
-  case Sema::PCC_ObjCInstanceVariableList:
-  case Sema::PCC_Template:
-  case Sema::PCC_MemberTemplate:
-  case Sema::PCC_Statement:
-  case Sema::PCC_RecoveryInFunction:
-  case Sema::PCC_Type:
-  case Sema::PCC_ParenthesizedExpression:
-  case Sema::PCC_LocalDeclarationSpecifiers:
-  case Sema::PCC_TopLevelOrExpression:
+  case SemaCodeCompletion::PCC_Namespace:
+  case SemaCodeCompletion::PCC_Class:
+  case SemaCodeCompletion::PCC_ObjCInstanceVariableList:
+  case SemaCodeCompletion::PCC_Template:
+  case SemaCodeCompletion::PCC_MemberTemplate:
+  case SemaCodeCompletion::PCC_Statement:
+  case SemaCodeCompletion::PCC_RecoveryInFunction:
+  case SemaCodeCompletion::PCC_Type:
+  case SemaCodeCompletion::PCC_ParenthesizedExpression:
+  case SemaCodeCompletion::PCC_LocalDeclarationSpecifiers:
+  case SemaCodeCompletion::PCC_TopLevelOrExpression:
     return true;
 
-  case Sema::PCC_Expression:
-  case Sema::PCC_Condition:
+  case SemaCodeCompletion::PCC_Expression:
+  case SemaCodeCompletion::PCC_Condition:
     return LangOpts.CPlusPlus;
 
-  case Sema::PCC_ObjCInterface:
-  case Sema::PCC_ObjCImplementation:
+  case SemaCodeCompletion::PCC_ObjCInterface:
+  case SemaCodeCompletion::PCC_ObjCImplementation:
     return false;
 
-  case Sema::PCC_ForInit:
+  case SemaCodeCompletion::PCC_ForInit:
     return LangOpts.CPlusPlus || LangOpts.ObjC || LangOpts.C99;
   }
 
@@ -2131,14 +2132,15 @@ static void AddOverrideResults(ResultBuilder &Results,
 }
 
 /// Add language constructs that show up for "ordinary" names.
-static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC, Scope *S,
-                                   Sema &SemaRef, ResultBuilder &Results) {
+static void
+AddOrdinaryNameResults(SemaCodeCompletion::ParserCompletionContext CCC,
+                       Scope *S, Sema &SemaRef, ResultBuilder &Results) {
   CodeCompletionAllocator &Allocator = Results.getAllocator();
   CodeCompletionBuilder Builder(Allocator, Results.getCodeCompletionTUInfo());
 
   typedef CodeCompletionResult Result;
   switch (CCC) {
-  case Sema::PCC_Namespace:
+  case SemaCodeCompletion::PCC_Namespace:
     if (SemaRef.getLangOpts().CPlusPlus) {
       if (Results.includeCodePatterns()) {
         // namespace <identifier> { declarations }
@@ -2194,7 +2196,7 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC, Scope *S,
     AddTypedefResult(Results);
     [[fallthrough]];
 
-  case Sema::PCC_Class:
+  case SemaCodeCompletion::PCC_Class:
     if (SemaRef.getLangOpts().CPlusPlus) {
       // Using declaration
       Builder.AddTypedTextChunk("using");
@@ -2221,7 +2223,7 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC, Scope *S,
 
       AddStaticAssertResult(Builder, Results, SemaRef.getLangOpts());
 
-      if (CCC == Sema::PCC_Class) {
+      if (CCC == SemaCodeCompletion::PCC_Class) {
         AddTypedefResult(Results);
 
         bool IsNotInheritanceScope = !S->isClassInheritanceScope();
@@ -2252,8 +2254,8 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC, Scope *S,
     }
     [[fallthrough]];
 
-  case Sema::PCC_Template:
-  case Sema::PCC_MemberTemplate:
+  case SemaCodeCompletion::PCC_Template:
+  case SemaCodeCompletion::PCC_MemberTemplate:
     if (SemaRef.getLangOpts().CPlusPlus && Results.includeCodePatterns()) {
       // template < parameters >
       Builder.AddTypedTextChunk("template");
@@ -2269,25 +2271,25 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC, Scope *S,
     AddFunctionSpecifiers(CCC, SemaRef.getLangOpts(), Results);
     break;
 
-  case Sema::PCC_ObjCInterface:
+  case SemaCodeCompletion::PCC_ObjCInterface:
     AddObjCInterfaceResults(SemaRef.getLangOpts(), Results, true);
     AddStorageSpecifiers(CCC, SemaRef.getLangOpts(), Results);
     AddFunctionSpecifiers(CCC, SemaRef.getLangOpts(), Results);
     break;
 
-  case Sema::PCC_ObjCImplementation:
+  case SemaCodeCompletion::PCC_ObjCImplementation:
     AddObjCImplementationResults(SemaRef.getLangOpts(), Results, true);
     AddStorageSpecifiers(CCC, SemaRef.getLangOpts(), Results);
     AddFunctionSpecifiers(CCC, SemaRef.getLangOpts(), Results);
     break;
 
-  case Sema::PCC_ObjCInstanceVariableList:
+  case SemaCodeCompletion::PCC_ObjCInstanceVariableList:
     AddObjCVisibilityResults(SemaRef.getLangOpts(), Results, true);
     break;
 
-  case Sema::PCC_RecoveryInFunction:
-  case Sema::PCC_TopLevelOrExpression:
-  case Sema::PCC_Statement: {
+  case SemaCodeCompletion::PCC_RecoveryInFunction:
+  case SemaCodeCompletion::PCC_TopLevelOrExpression:
+  case SemaCodeCompletion::PCC_Statement: {
     if (SemaRef.getLangOpts().CPlusPlus11)
       AddUsingAliasResult(Builder, Results);
 
@@ -2524,15 +2526,15 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC, Scope *S,
     [[fallthrough]];
 
   // Fall through (for statement expressions).
-  case Sema::PCC_ForInit:
-  case Sema::PCC_Condition:
+  case SemaCodeCompletion::PCC_ForInit:
+  case SemaCodeCompletion::PCC_Condition:
     AddStorageSpecifiers(CCC, SemaRef.getLangOpts(), Results);
     // Fall through: conditions and statements can have expressions.
     [[fallthrough]];
 
-  case Sema::PCC_ParenthesizedExpression:
+  case SemaCodeCompletion::PCC_ParenthesizedExpression:
     if (SemaRef.getLangOpts().ObjCAutoRefCount &&
-        CCC == Sema::PCC_ParenthesizedExpression) {
+        CCC == SemaCodeCompletion::PCC_ParenthesizedExpression) {
       // (__bridge <type>)<expression>
       Builder.AddTypedTextChunk("__bridge");
       Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
@@ -2560,7 +2562,7 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC, Scope *S,
     // Fall through
     [[fallthrough]];
 
-  case Sema::PCC_Expression: {
+  case SemaCodeCompletion::PCC_Expression: {
     if (SemaRef.getLangOpts().CPlusPlus) {
       // 'this', if we're in a non-static member function.
       addThisCompletion(SemaRef, Results);
@@ -2758,15 +2760,15 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC, Scope *S,
     break;
   }
 
-  case Sema::PCC_Type:
-  case Sema::PCC_LocalDeclarationSpecifiers:
+  case SemaCodeCompletion::PCC_Type:
+  case SemaCodeCompletion::PCC_LocalDeclarationSpecifiers:
     break;
   }
 
   if (WantTypesInContext(CCC, SemaRef.getLangOpts()))
     AddTypeSpecifierResults(SemaRef.getLangOpts(), Results);
 
-  if (SemaRef.getLangOpts().CPlusPlus && CCC != Sema::PCC_Type)
+  if (SemaRef.getLangOpts().CPlusPlus && CCC != SemaCodeCompletion::PCC_Type)
     Results.AddResult(Result("operator"));
 }
 
@@ -4241,59 +4243,60 @@ static void HandleCodeCompleteResults(Sema *S,
 }
 
 static CodeCompletionContext
-mapCodeCompletionContext(Sema &S, Sema::ParserCompletionContext PCC) {
+mapCodeCompletionContext(Sema &S,
+                         SemaCodeCompletion::ParserCompletionContext PCC) {
   switch (PCC) {
-  case Sema::PCC_Namespace:
+  case SemaCodeCompletion::PCC_Namespace:
     return CodeCompletionContext::CCC_TopLevel;
 
-  case Sema::PCC_Class:
+  case SemaCodeCompletion::PCC_Class:
     return CodeCompletionContext::CCC_ClassStructUnion;
 
-  case Sema::PCC_ObjCInterface:
+  case SemaCodeCompletion::PCC_ObjCInterface:
     return CodeCompletionContext::CCC_ObjCInterface;
 
-  case Sema::PCC_ObjCImplementation:
+  case SemaCodeCompletion::PCC_ObjCImplementation:
     return CodeCompletionContext::CCC_ObjCImplementation;
 
-  case Sema::PCC_ObjCInstanceVariableList:
+  case SemaCodeCompletion::PCC_ObjCInstanceVariableList:
     return CodeCompletionContext::CCC_ObjCIvarList;
 
-  case Sema::PCC_Template:
-  case Sema::PCC_MemberTemplate:
+  case SemaCodeCompletion::PCC_Template:
+  case SemaCodeCompletion::PCC_MemberTemplate:
     if (S.CurContext->isFileContext())
       return CodeCompletionContext::CCC_TopLevel;
     if (S.CurContext->isRecord())
       return CodeCompletionContext::CCC_ClassStructUnion;
     return CodeCompletionContext::CCC_Other;
 
-  case Sema::PCC_RecoveryInFunction:
+  case SemaCodeCompletion::PCC_RecoveryInFunction:
     return CodeCompletionContext::CCC_Recovery;
 
-  case Sema::PCC_ForInit:
+  case SemaCodeCompletion::PCC_ForInit:
     if (S.getLangOpts().CPlusPlus || S.getLangOpts().C99 ||
         S.getLangOpts().ObjC)
       return CodeCompletionContext::CCC_ParenthesizedExpression;
     else
       return CodeCompletionContext::CCC_Expression;
 
-  case Sema::PCC_Expression:
+  case SemaCodeCompletion::PCC_Expression:
     return CodeCompletionContext::CCC_Expression;
-  case Sema::PCC_Condition:
+  case SemaCodeCompletion::PCC_Condition:
     return CodeCompletionContext(CodeCompletionContext::CCC_Expression,
                                  S.getASTContext().BoolTy);
 
-  case Sema::PCC_Statement:
+  case SemaCodeCompletion::PCC_Statement:
     return CodeCompletionContext::CCC_Statement;
 
-  case Sema::PCC_Type:
+  case SemaCodeCompletion::PCC_Type:
     return CodeCompletionContext::CCC_Type;
 
-  case Sema::PCC_ParenthesizedExpression:
+  case SemaCodeCompletion::PCC_ParenthesizedExpression:
     return CodeCompletionContext::CCC_ParenthesizedExpression;
 
-  case Sema::PCC_LocalDeclarationSpecifiers:
+  case SemaCodeCompletion::PCC_LocalDeclarationSpecifiers:
     return CodeCompletionContext::CCC_Type;
-  case Sema::PCC_TopLevelOrExpression:
+  case SemaCodeCompletion::PCC_TopLevelOrExpression:
     return CodeCompletionContext::CCC_TopLevelOrExpression;
   }
 
@@ -4366,10 +4369,10 @@ static void MaybeAddOverrideCalls(Sema &S, DeclContext *InContext,
   }
 }
 
-void Sema::CodeCompleteModuleImport(SourceLocation ImportLoc,
-                                    ModuleIdPath Path) {
+void SemaCodeCompletion::CodeCompleteModuleImport(SourceLocation ImportLoc,
+                                                  ModuleIdPath Path) {
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
@@ -4380,7 +4383,7 @@ void Sema::CodeCompleteModuleImport(SourceLocation ImportLoc,
   if (Path.empty()) {
     // Enumerate all top-level modules.
     SmallVector<Module *, 8> Modules;
-    PP.getHeaderSearchInfo().collectAllModules(Modules);
+    SemaRef.PP.getHeaderSearchInfo().collectAllModules(Modules);
     for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
       Builder.AddTypedTextChunk(
           Builder.getAllocator().CopyString(Modules[I]->Name));
@@ -4391,9 +4394,9 @@ void Sema::CodeCompleteModuleImport(SourceLocation ImportLoc,
     }
   } else if (getLangOpts().Modules) {
     // Load the named module.
-    Module *Mod =
-        PP.getModuleLoader().loadModule(ImportLoc, Path, Module::AllVisible,
-                                        /*IsInclusionDirective=*/false);
+    Module *Mod = SemaRef.PP.getModuleLoader().loadModule(
+        ImportLoc, Path, Module::AllVisible,
+        /*IsInclusionDirective=*/false);
     // Enumerate submodules.
     if (Mod) {
       for (auto *Submodule : Mod->submodules()) {
@@ -4407,15 +4410,16 @@ void Sema::CodeCompleteModuleImport(SourceLocation ImportLoc,
     }
   }
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteOrdinaryName(Scope *S,
-                                    ParserCompletionContext CompletionContext) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteOrdinaryName(
+    Scope *S, SemaCodeCompletion::ParserCompletionContext CompletionContext) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
-                        mapCodeCompletionContext(*this, CompletionContext));
+                        mapCodeCompletionContext(SemaRef, CompletionContext));
   Results.EnterNewScope();
 
   // Determine how to filter results, e.g., so that the names of
@@ -4446,7 +4450,7 @@ void Sema::CodeCompleteOrdinaryName(Scope *S,
       Results.setFilter(&ResultBuilder::IsOrdinaryNonTypeName);
 
     if (getLangOpts().CPlusPlus)
-      MaybeAddOverrideCalls(*this, /*InContext=*/nullptr, Results);
+      MaybeAddOverrideCalls(SemaRef, /*InContext=*/nullptr, Results);
     break;
 
   case PCC_RecoveryInFunction:
@@ -4456,17 +4460,17 @@ void Sema::CodeCompleteOrdinaryName(Scope *S,
 
   // If we are in a C++ non-static member function, check the qualifiers on
   // the member function to filter/prioritize the results list.
-  auto ThisType = getCurrentThisType();
+  auto ThisType = SemaRef.getCurrentThisType();
   if (!ThisType.isNull())
     Results.setObjectTypeQualifiers(ThisType->getPointeeType().getQualifiers(),
                                     VK_LValue);
 
-  CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
-                     CodeCompleter->includeGlobals(),
-                     CodeCompleter->loadExternal());
+  CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
+  SemaRef.LookupVisibleDecls(S, SemaRef.LookupOrdinaryName, Consumer,
+                             CodeCompleter->includeGlobals(),
+                             CodeCompleter->loadExternal());
 
-  AddOrdinaryNameResults(CompletionContext, S, *this, Results);
+  AddOrdinaryNameResults(CompletionContext, S, SemaRef, Results);
   Results.ExitScope();
 
   switch (CompletionContext) {
@@ -4494,10 +4498,11 @@ void Sema::CodeCompleteOrdinaryName(Scope *S,
   }
 
   if (CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results, CodeCompleter->loadExternal(), false);
+    AddMacroResults(SemaRef.PP, Results, CodeCompleter->loadExternal(), false);
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 static void
@@ -4506,12 +4511,12 @@ AddClassMessageCompletions(Sema &SemaRef, Scope *S, ParsedType Receiver,
                            bool AtArgumentExpression, bool IsSuper,
                            ResultBuilder &Results);
 
-void Sema::CodeCompleteDeclSpec(Scope *S, DeclSpec &DS,
-                                bool AllowNonIdentifiers,
-                                bool AllowNestedNameSpecifiers) {
+void SemaCodeCompletion::CodeCompleteDeclSpec(Scope *S, DeclSpec &DS,
+                                              bool AllowNonIdentifiers,
+                                              bool AllowNestedNameSpecifiers) {
   typedef CodeCompletionResult Result;
   ResultBuilder Results(
-      *this, CodeCompleter->getAllocator(),
+      SemaRef, CodeCompleter->getAllocator(),
       CodeCompleter->getCodeCompletionTUInfo(),
       AllowNestedNameSpecifiers
           // FIXME: Try to separate codepath leading here to deduce whether we
@@ -4540,10 +4545,10 @@ void Sema::CodeCompleteDeclSpec(Scope *S, DeclSpec &DS,
     if (AllowNestedNameSpecifiers) {
       Results.allowNestedNameSpecifiers();
       Results.setFilter(&ResultBuilder::IsImpossibleToSatisfy);
-      CodeCompletionDeclConsumer Consumer(Results, CurContext);
-      LookupVisibleDecls(S, LookupNestedNameSpecifierName, Consumer,
-                         CodeCompleter->includeGlobals(),
-                         CodeCompleter->loadExternal());
+      CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
+      SemaRef.LookupVisibleDecls(S, Sema::LookupNestedNameSpecifierName,
+                                 Consumer, CodeCompleter->includeGlobals(),
+                                 CodeCompleter->loadExternal());
       Results.setFilter(nullptr);
     }
   }
@@ -4565,15 +4570,16 @@ void Sema::CodeCompleteDeclSpec(Scope *S, DeclSpec &DS,
           0) {
     ParsedType T = DS.getRepAsType();
     if (!T.get().isNull() && T.get()->isObjCObjectOrInterfaceType())
-      AddClassMessageCompletions(*this, S, T, std::nullopt, false, false,
+      AddClassMessageCompletions(SemaRef, S, T, std::nullopt, false, false,
                                  Results);
   }
 
   // Note that we intentionally suppress macro results here, since we do not
   // encourage using macros to produce the names of entities.
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 static const char *underscoreAttrScope(llvm::StringRef Scope) {
@@ -4592,12 +4598,12 @@ static const char *noUnderscoreAttrScope(llvm::StringRef Scope) {
   return nullptr;
 }
 
-void Sema::CodeCompleteAttribute(AttributeCommonInfo::Syntax Syntax,
-                                 AttributeCompletion Completion,
-                                 const IdentifierInfo *InScope) {
+void SemaCodeCompletion::CodeCompleteAttribute(
+    AttributeCommonInfo::Syntax Syntax, AttributeCompletion Completion,
+    const IdentifierInfo *InScope) {
   if (Completion == AttributeCompletion::None)
     return;
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Attribute);
 
@@ -4626,7 +4632,8 @@ void Sema::CodeCompleteAttribute(AttributeCommonInfo::Syntax Syntax,
 
   llvm::DenseSet<llvm::StringRef> FoundScopes;
   auto AddCompletions = [&](const ParsedAttrInfo &A) {
-    if (A.IsTargetSpecific && !A.existsInTarget(Context.getTargetInfo()))
+    if (A.IsTargetSpecific &&
+        !A.existsInTarget(getASTContext().getTargetInfo()))
       return;
     if (!A.acceptsLangOpts(getLangOpts()))
       return;
@@ -4724,11 +4731,12 @@ void Sema::CodeCompleteAttribute(AttributeCommonInfo::Syntax Syntax,
   for (const auto &Entry : ParsedAttrInfoRegistry::entries())
     AddCompletions(*Entry.instantiate());
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-struct Sema::CodeCompleteExpressionData {
+struct SemaCodeCompletion::CodeCompleteExpressionData {
   CodeCompleteExpressionData(QualType PreferredType = QualType(),
                              bool IsParenthesized = false)
       : PreferredType(PreferredType), IntegralConstantExpression(false),
@@ -4841,10 +4849,10 @@ static void AddLambdaCompletion(ResultBuilder &Results,
 
 /// Perform code-completion in an expression context when we know what
 /// type we're looking for.
-void Sema::CodeCompleteExpression(Scope *S,
-                                  const CodeCompleteExpressionData &Data) {
+void SemaCodeCompletion::CodeCompleteExpression(
+    Scope *S, const CodeCompleteExpressionData &Data) {
   ResultBuilder Results(
-      *this, CodeCompleter->getAllocator(),
+      SemaRef, CodeCompleter->getAllocator(),
       CodeCompleter->getCodeCompletionTUInfo(),
       CodeCompletionContext(
           Data.IsParenthesized
@@ -4869,13 +4877,13 @@ void Sema::CodeCompleteExpression(Scope *S,
   for (unsigned I = 0, N = Data.IgnoreDecls.size(); I != N; ++I)
     Results.Ignore(Data.IgnoreDecls[I]);
 
-  CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
-                     CodeCompleter->includeGlobals(),
-                     CodeCompleter->loadExternal());
+  CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
+  SemaRef.LookupVisibleDecls(S, Sema::LookupOrdinaryName, Consumer,
+                             CodeCompleter->includeGlobals(),
+                             CodeCompleter->loadExternal());
 
   Results.EnterNewScope();
-  AddOrdinaryNameResults(PCC, S, *this, Results);
+  AddOrdinaryNameResults(PCC, S, SemaRef, Results);
   Results.ExitScope();
 
   bool PreferredTypeIsPointer = false;
@@ -4889,7 +4897,8 @@ void Sema::CodeCompleteExpression(Scope *S,
         Enum = Def;
       // FIXME: collect covered enumerators in cases like:
       //        if (x == my_enum::one) { ... } else if (x == ^) {}
-      AddEnumerators(Results, Context, Enum, CurContext, CoveredEnumerators());
+      AddEnumerators(Results, getASTContext(), Enum, SemaRef.CurContext,
+                     CoveredEnumerators());
     }
   }
 
@@ -4898,7 +4907,7 @@ void Sema::CodeCompleteExpression(Scope *S,
     AddPrettyFunctionResults(getLangOpts(), Results);
 
   if (CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results, CodeCompleter->loadExternal(), false,
+    AddMacroResults(SemaRef.PP, Results, CodeCompleter->loadExternal(), false,
                     PreferredTypeIsPointer);
 
   // Complete a lambda expression when preferred type is a function.
@@ -4908,18 +4917,20 @@ void Sema::CodeCompleteExpression(Scope *S,
       AddLambdaCompletion(Results, F->getParamTypes(), getLangOpts());
   }
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteExpression(Scope *S, QualType PreferredType,
-                                  bool IsParenthesized) {
+void SemaCodeCompletion::CodeCompleteExpression(Scope *S,
+                                                QualType PreferredType,
+                                                bool IsParenthesized) {
   return CodeCompleteExpression(
       S, CodeCompleteExpressionData(PreferredType, IsParenthesized));
 }
 
-void Sema::CodeCompletePostfixExpression(Scope *S, ExprResult E,
-                                         QualType PreferredType) {
+void SemaCodeCompletion::CodeCompletePostfixExpression(Scope *S, ExprResult E,
+                                                       QualType PreferredType) {
   if (E.isInvalid())
     CodeCompleteExpression(S, PreferredType);
   else if (getLangOpts().ObjC)
@@ -5182,10 +5193,11 @@ AddRecordMembersCompletionResults(Sema &SemaRef, ResultBuilder &Results,
   if (AccessOpFixIt)
     FixIts.emplace_back(*AccessOpFixIt);
   CodeCompletionDeclConsumer Consumer(Results, RD, BaseType, std::move(FixIts));
-  SemaRef.LookupVisibleDecls(RD, Sema::LookupMemberName, Consumer,
-                             SemaRef.CodeCompleter->includeGlobals(),
-                             /*IncludeDependentBases=*/true,
-                             SemaRef.CodeCompleter->loadExternal());
+  SemaRef.LookupVisibleDecls(
+      RD, Sema::LookupMemberName, Consumer,
+      SemaRef.CodeCompletion().CodeCompleter->includeGlobals(),
+      /*IncludeDependentBases=*/true,
+      SemaRef.CodeCompletion().CodeCompleter->loadExternal());
 
   if (SemaRef.getLangOpts().CPlusPlus) {
     if (!Results.empty()) {
@@ -5701,17 +5713,16 @@ Expr *unwrapParenList(Expr *Base) {
 
 } // namespace
 
-void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base,
-                                           Expr *OtherOpBase,
-                                           SourceLocation OpLoc, bool IsArrow,
-                                           bool IsBaseExprStatement,
-                                           QualType PreferredType) {
+void SemaCodeCompletion::CodeCompleteMemberReferenceExpr(
+    Scope *S, Expr *Base, Expr *OtherOpBase, SourceLocation OpLoc, bool IsArrow,
+    bool IsBaseExprStatement, QualType PreferredType) {
   Base = unwrapParenList(Base);
   OtherOpBase = unwrapParenList(OtherOpBase);
   if (!Base || !CodeCompleter)
     return;
 
-  ExprResult ConvertedBase = PerformMemberExprBaseConversion(Base, IsArrow);
+  ExprResult ConvertedBase =
+      SemaRef.PerformMemberExprBaseConversion(Base, IsArrow);
   if (ConvertedBase.isInvalid())
     return;
   QualType ConvertedBaseType = getApproximateType(ConvertedBase.get());
@@ -5736,7 +5747,7 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base,
 
   CodeCompletionContext CCContext(contextKind, ConvertedBaseType);
   CCContext.setPreferredType(PreferredType);
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(), CCContext,
                         &ResultBuilder::IsMember);
 
@@ -5745,7 +5756,8 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base,
     if (!Base)
       return false;
 
-    ExprResult ConvertedBase = PerformMemberExprBaseConversion(Base, IsArrow);
+    ExprResult ConvertedBase =
+        SemaRef.PerformMemberExprBaseConversion(Base, IsArrow);
     if (ConvertedBase.isInvalid())
       return false;
     Base = ConvertedBase.get();
@@ -5768,7 +5780,7 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base,
     }
 
     if (RecordDecl *RD = getAsRecordDecl(BaseType)) {
-      AddRecordMembersCompletionResults(*this, Results, S, BaseType, BaseKind,
+      AddRecordMembersCompletionResults(SemaRef, Results, S, BaseType, BaseKind,
                                         RD, std::move(AccessOpFixIt));
     } else if (const auto *TTPT =
                    dyn_cast<TemplateTypeParmType>(BaseType.getTypePtr())) {
@@ -5778,7 +5790,7 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base,
         if (R.Operator != Operator)
           continue;
         CodeCompletionResult Result(
-            R.render(*this, CodeCompleter->getAllocator(),
+            R.render(SemaRef, CodeCompleter->getAllocator(),
                      CodeCompleter->getCodeCompletionTUInfo()));
         if (AccessOpFixIt)
           Result.FixIts.push_back(*AccessOpFixIt);
@@ -5798,14 +5810,14 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base,
         // Add property results based on our interface.
         assert(ObjCPtr && "Non-NULL pointer guaranteed above!");
         AddObjCProperties(CCContext, ObjCPtr->getInterfaceDecl(), true,
-                          /*AllowNullaryMethods=*/true, CurContext,
+                          /*AllowNullaryMethods=*/true, SemaRef.CurContext,
                           AddedProperties, Results, IsBaseExprStatement);
       }
 
       // Add properties from the protocols in a qualified interface.
       for (auto *I : BaseType->castAs<ObjCObjectPointerType>()->quals())
         AddObjCProperties(CCContext, I, true, /*AllowNullaryMethods=*/true,
-                          CurContext, AddedProperties, Results,
+                          SemaRef.CurContext, AddedProperties, Results,
                           IsBaseExprStatement, /*IsClassProperty*/ false,
                           /*InOriginalClass*/ false);
     } else if ((IsArrow && BaseType->isObjCObjectPointerType()) ||
@@ -5827,9 +5839,10 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base,
       if (Class) {
         CodeCompletionDeclConsumer Consumer(Results, Class, BaseType);
         Results.setFilter(&ResultBuilder::IsObjCIvar);
-        LookupVisibleDecls(
-            Class, LookupMemberName, Consumer, CodeCompleter->includeGlobals(),
-            /*IncludeDependentBases=*/false, CodeCompleter->loadExternal());
+        SemaRef.LookupVisibleDecls(Class, Sema::LookupMemberName, Consumer,
+                                   CodeCompleter->includeGlobals(),
+                                   /*IncludeDependentBases=*/false,
+                                   CodeCompleter->loadExternal());
       }
     }
 
@@ -5854,36 +5867,37 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base,
     return;
 
   // Hand off the results found for code completion.
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCClassPropertyRefExpr(Scope *S,
-                                                const IdentifierInfo &ClassName,
-                                                SourceLocation ClassNameLoc,
-                                                bool IsBaseExprStatement) {
+void SemaCodeCompletion::CodeCompleteObjCClassPropertyRefExpr(
+    Scope *S, const IdentifierInfo &ClassName, SourceLocation ClassNameLoc,
+    bool IsBaseExprStatement) {
   const IdentifierInfo *ClassNamePtr = &ClassName;
   ObjCInterfaceDecl *IFace =
-      ObjC().getObjCInterfaceDecl(ClassNamePtr, ClassNameLoc);
+      SemaRef.ObjC().getObjCInterfaceDecl(ClassNamePtr, ClassNameLoc);
   if (!IFace)
     return;
   CodeCompletionContext CCContext(
       CodeCompletionContext::CCC_ObjCPropertyAccess);
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(), CCContext,
                         &ResultBuilder::IsMember);
   Results.EnterNewScope();
   AddedPropertiesSet AddedProperties;
   AddObjCProperties(CCContext, IFace, true,
-                    /*AllowNullaryMethods=*/true, CurContext, AddedProperties,
-                    Results, IsBaseExprStatement,
+                    /*AllowNullaryMethods=*/true, SemaRef.CurContext,
+                    AddedProperties, Results, IsBaseExprStatement,
                     /*IsClassProperty=*/true);
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteTag(Scope *S, unsigned TagSpec) {
+void SemaCodeCompletion::CodeCompleteTag(Scope *S, unsigned TagSpec) {
   if (!CodeCompleter)
     return;
 
@@ -5912,26 +5926,27 @@ void Sema::CodeCompleteTag(Scope *S, unsigned TagSpec) {
     llvm_unreachable("Unknown type specifier kind in CodeCompleteTag");
   }
 
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(), ContextKind);
-  CodeCompletionDeclConsumer Consumer(Results, CurContext);
+  CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
 
   // First pass: look for tags.
   Results.setFilter(Filter);
-  LookupVisibleDecls(S, LookupTagName, Consumer,
-                     CodeCompleter->includeGlobals(),
-                     CodeCompleter->loadExternal());
+  SemaRef.LookupVisibleDecls(S, Sema::LookupTagName, Consumer,
+                             CodeCompleter->includeGlobals(),
+                             CodeCompleter->loadExternal());
 
   if (CodeCompleter->includeGlobals()) {
     // Second pass: look for nested name specifiers.
     Results.setFilter(&ResultBuilder::IsNestedNameSpecifier);
-    LookupVisibleDecls(S, LookupNestedNameSpecifierName, Consumer,
-                       CodeCompleter->includeGlobals(),
-                       CodeCompleter->loadExternal());
+    SemaRef.LookupVisibleDecls(S, Sema::LookupNestedNameSpecifierName, Consumer,
+                               CodeCompleter->includeGlobals(),
+                               CodeCompleter->loadExternal());
   }
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 static void AddTypeQualifierResults(DeclSpec &DS, ResultBuilder &Results,
@@ -5948,25 +5963,26 @@ static void AddTypeQualifierResults(DeclSpec &DS, ResultBuilder &Results,
     Results.AddResult("__unaligned");
 }
 
-void Sema::CodeCompleteTypeQualifiers(DeclSpec &DS) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteTypeQualifiers(DeclSpec &DS) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_TypeQualifiers);
   Results.EnterNewScope();
-  AddTypeQualifierResults(DS, Results, LangOpts);
+  AddTypeQualifierResults(DS, Results, getLangOpts());
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteFunctionQualifiers(DeclSpec &DS, Declarator &D,
-                                          const VirtSpecifiers *VS) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteFunctionQualifiers(
+    DeclSpec &DS, Declarator &D, const VirtSpecifiers *VS) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_TypeQualifiers);
   Results.EnterNewScope();
-  AddTypeQualifierResults(DS, Results, LangOpts);
-  if (LangOpts.CPlusPlus11) {
+  AddTypeQualifierResults(DS, Results, getLangOpts());
+  if (getLangOpts().CPlusPlus11) {
     Results.AddResult("noexcept");
     if (D.getContext() == DeclaratorContext::Member && !D.isCtorOrDtor() &&
         !D.isStaticMember()) {
@@ -5977,19 +5993,21 @@ void Sema::CodeCompleteFunctionQualifiers(DeclSpec &DS, Declarator &D,
     }
   }
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteBracketDeclarator(Scope *S) {
+void SemaCodeCompletion::CodeCompleteBracketDeclarator(Scope *S) {
   CodeCompleteExpression(S, QualType(getASTContext().getSizeType()));
 }
 
-void Sema::CodeCompleteCase(Scope *S) {
-  if (getCurFunction()->SwitchStack.empty() || !CodeCompleter)
+void SemaCodeCompletion::CodeCompleteCase(Scope *S) {
+  if (SemaRef.getCurFunction()->SwitchStack.empty() || !CodeCompleter)
     return;
 
-  SwitchStmt *Switch = getCurFunction()->SwitchStack.back().getPointer();
+  SwitchStmt *Switch =
+      SemaRef.getCurFunction()->SwitchStack.back().getPointer();
   // Condition expression might be invalid, do not continue in this case.
   if (!Switch->getCond())
     return;
@@ -6046,16 +6064,18 @@ void Sema::CodeCompleteCase(Scope *S) {
   }
 
   // Add any enumerators that have not yet been mentioned.
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Expression);
-  AddEnumerators(Results, Context, Enum, CurContext, Enumerators);
+  AddEnumerators(Results, getASTContext(), Enum, SemaRef.CurContext,
+                 Enumerators);
 
   if (CodeCompleter->includeMacros()) {
-    AddMacroResults(PP, Results, CodeCompleter->loadExternal(), false);
+    AddMacroResults(SemaRef.PP, Results, CodeCompleter->loadExternal(), false);
   }
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 static bool anyNullArguments(ArrayRef<Expr *> Args) {
@@ -6136,7 +6156,7 @@ ProduceSignatureHelp(Sema &SemaRef, MutableArrayRef<ResultCandidate> Candidates,
   if (Candidates.empty())
     return QualType();
   if (SemaRef.getPreprocessor().isCodeCompletionReached())
-    SemaRef.CodeCompleter->ProcessOverloadCandidates(
+    SemaRef.CodeCompletion().CodeCompleter->ProcessOverloadCandidates(
         SemaRef, CurrentArg, Candidates.data(), Candidates.size(), OpenParLoc,
         Braced);
   return getParamType(SemaRef, Candidates, CurrentArg);
@@ -6190,8 +6210,9 @@ static FunctionProtoTypeLoc GetPrototypeLoc(Expr *Fn) {
   return {};
 }
 
-QualType Sema::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
-                                        SourceLocation OpenParLoc) {
+QualType
+SemaCodeCompletion::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
+                                             SourceLocation OpenParLoc) {
   Fn = unwrapParenList(Fn);
   if (!CodeCompleter || !Fn)
     return QualType();
@@ -6214,8 +6235,9 @@ QualType Sema::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
   OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
 
   if (auto ULE = dyn_cast<UnresolvedLookupExpr>(NakedFn)) {
-    AddOverloadedCallCandidates(ULE, ArgsWithoutDependentTypes, CandidateSet,
-                                /*PartialOverloading=*/true);
+    SemaRef.AddOverloadedCallCandidates(ULE, ArgsWithoutDependentTypes,
+                                        CandidateSet,
+                                        /*PartialOverloading=*/true);
   } else if (auto UME = dyn_cast<UnresolvedMemberExpr>(NakedFn)) {
     TemplateArgumentListInfo TemplateArgsBuffer, *TemplateArgs = nullptr;
     if (UME->hasExplicitTemplateArgs()) {
@@ -6231,9 +6253,10 @@ QualType Sema::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
     UnresolvedSet<8> Decls;
     Decls.append(UME->decls_begin(), UME->decls_end());
     const bool FirstArgumentIsBase = !UME->isImplicitAccess() && UME->getBase();
-    AddFunctionCandidates(Decls, ArgExprs, CandidateSet, TemplateArgs,
-                          /*SuppressUserConversions=*/false,
-                          /*PartialOverloading=*/true, FirstArgumentIsBase);
+    SemaRef.AddFunctionCandidates(Decls, ArgExprs, CandidateSet, TemplateArgs,
+                                  /*SuppressUserConversions=*/false,
+                                  /*PartialOverloading=*/true,
+                                  FirstArgumentIsBase);
   } else {
     FunctionDecl *FD = nullptr;
     if (auto *MCE = dyn_cast<MemberExpr>(NakedFn))
@@ -6245,28 +6268,30 @@ QualType Sema::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
           !FD->getType()->getAs<FunctionProtoType>())
         Results.push_back(ResultCandidate(FD));
       else
-        AddOverloadCandidate(FD, DeclAccessPair::make(FD, FD->getAccess()),
-                             ArgsWithoutDependentTypes, CandidateSet,
-                             /*SuppressUserConversions=*/false,
-                             /*PartialOverloading=*/true);
+        SemaRef.AddOverloadCandidate(FD,
+                                     DeclAccessPair::make(FD, FD->getAccess()),
+                                     ArgsWithoutDependentTypes, CandidateSet,
+                                     /*SuppressUserConversions=*/false,
+                                     /*PartialOverloading=*/true);
 
     } else if (auto DC = NakedFn->getType()->getAsCXXRecordDecl()) {
       // If expression's type is CXXRecordDecl, it may overload the function
       // call operator, so we check if it does and add them as candidates.
       // A complete type is needed to lookup for member function call operators.
-      if (isCompleteType(Loc, NakedFn->getType())) {
+      if (SemaRef.isCompleteType(Loc, NakedFn->getType())) {
         DeclarationName OpName =
-            Context.DeclarationNames.getCXXOperatorName(OO_Call);
-        LookupResult R(*this, OpName, Loc, LookupOrdinaryName);
-        LookupQualifiedName(R, DC);
+            getASTContext().DeclarationNames.getCXXOperatorName(OO_Call);
+        LookupResult R(SemaRef, OpName, Loc, Sema::LookupOrdinaryName);
+        SemaRef.LookupQualifiedName(R, DC);
         R.suppressDiagnostics();
         SmallVector<Expr *, 12> ArgExprs(1, NakedFn);
         ArgExprs.append(ArgsWithoutDependentTypes.begin(),
                         ArgsWithoutDependentTypes.end());
-        AddFunctionCandidates(R.asUnresolvedSet(), ArgExprs, CandidateSet,
-                              /*ExplicitArgs=*/nullptr,
-                              /*SuppressUserConversions=*/false,
-                              /*PartialOverloading=*/true);
+        SemaRef.AddFunctionCandidates(R.asUnresolvedSet(), ArgExprs,
+                                      CandidateSet,
+                                      /*ExplicitArgs=*/nullptr,
+                                      /*SuppressUserConversions=*/false,
+                                      /*PartialOverloading=*/true);
       }
     } else {
       // Lastly we check whether expression's type is function pointer or
@@ -6278,9 +6303,9 @@ QualType Sema::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
         T = T->getPointeeType();
 
       if (auto FP = T->getAs<FunctionProtoType>()) {
-        if (!TooManyArguments(FP->getNumParams(),
-                              ArgsWithoutDependentTypes.size(),
-                              /*PartialOverloading=*/true) ||
+        if (!SemaRef.TooManyArguments(FP->getNumParams(),
+                                      ArgsWithoutDependentTypes.size(),
+                                      /*PartialOverloading=*/true) ||
             FP->isVariadic()) {
           if (P) {
             Results.push_back(ResultCandidate(P));
@@ -6293,8 +6318,8 @@ QualType Sema::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
         Results.push_back(ResultCandidate(FT));
     }
   }
-  mergeCandidatesWithResults(*this, Results, CandidateSet, Loc, Args.size());
-  QualType ParamType = ProduceSignatureHelp(*this, Results, Args.size(),
+  mergeCandidatesWithResults(SemaRef, Results, CandidateSet, Loc, Args.size());
+  QualType ParamType = ProduceSignatureHelp(SemaRef, Results, Args.size(),
                                             OpenParLoc, /*Braced=*/false);
   return !CandidateSet.empty() ? ParamType : QualType();
 }
@@ -6366,18 +6391,16 @@ getNextAggregateIndexAfterDesignatedInit(const ResultCandidate &Aggregate,
   return DesignatedIndex + ArgsAfterDesignator + 1;
 }
 
-QualType Sema::ProduceConstructorSignatureHelp(QualType Type,
-                                               SourceLocation Loc,
-                                               ArrayRef<Expr *> Args,
-                                               SourceLocation OpenParLoc,
-                                               bool Braced) {
+QualType SemaCodeCompletion::ProduceConstructorSignatureHelp(
+    QualType Type, SourceLocation Loc, ArrayRef<Expr *> Args,
+    SourceLocation OpenParLoc, bool Braced) {
   if (!CodeCompleter)
     return QualType();
   SmallVector<ResultCandidate, 8> Results;
 
   // A complete type is needed to lookup for constructors.
   RecordDecl *RD =
-      isCompleteType(Loc, Type) ? Type->getAsRecordDecl() : nullptr;
+      SemaRef.isCompleteType(Loc, Type) ? Type->getAsRecordDecl() : nullptr;
   if (!RD)
     return Type;
   CXXRecordDecl *CRD = dyn_cast<CXXRecordDecl>(RD);
@@ -6389,7 +6412,7 @@ QualType Sema::ProduceConstructorSignatureHelp(QualType Type,
   // FIXME: it would be nice to support "unwrapping" aggregates that contain
   // a single subaggregate, like std::array<T, N> -> T __elements[N].
   if (Braced && !RD->isUnion() &&
-      (!LangOpts.CPlusPlus || (CRD && CRD->isAggregate()))) {
+      (!getLangOpts().CPlusPlus || (CRD && CRD->isAggregate()))) {
     ResultCandidate AggregateSig(RD);
     unsigned AggregateSize = AggregateSig.getNumParams();
 
@@ -6399,7 +6422,7 @@ QualType Sema::ProduceConstructorSignatureHelp(QualType Type,
       if (*NextIndex >= AggregateSize)
         return Type;
       Results.push_back(AggregateSig);
-      return ProduceSignatureHelp(*this, Results, *NextIndex, OpenParLoc,
+      return ProduceSignatureHelp(SemaRef, Results, *NextIndex, OpenParLoc,
                                   Braced);
     }
 
@@ -6413,36 +6436,39 @@ QualType Sema::ProduceConstructorSignatureHelp(QualType Type,
 
   if (CRD) {
     OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
-    for (NamedDecl *C : LookupConstructors(CRD)) {
+    for (NamedDecl *C : SemaRef.LookupConstructors(CRD)) {
       if (auto *FD = dyn_cast<FunctionDecl>(C)) {
         // FIXME: we can't yet provide correct signature help for initializer
         //        list constructors, so skip them entirely.
-        if (Braced && LangOpts.CPlusPlus && isInitListConstructor(FD))
+        if (Braced && getLangOpts().CPlusPlus &&
+            SemaRef.isInitListConstructor(FD))
           continue;
-        AddOverloadCandidate(FD, DeclAccessPair::make(FD, C->getAccess()), Args,
-                             CandidateSet,
-                             /*SuppressUserConversions=*/false,
-                             /*PartialOverloading=*/true,
-                             /*AllowExplicit*/ true);
+        SemaRef.AddOverloadCandidate(
+            FD, DeclAccessPair::make(FD, C->getAccess()), Args, CandidateSet,
+            /*SuppressUserConversions=*/false,
+            /*PartialOverloading=*/true,
+            /*AllowExplicit*/ true);
       } else if (auto *FTD = dyn_cast<FunctionTemplateDecl>(C)) {
-        if (Braced && LangOpts.CPlusPlus &&
-            isInitListConstructor(FTD->getTemplatedDecl()))
+        if (Braced && getLangOpts().CPlusPlus &&
+            SemaRef.isInitListConstructor(FTD->getTemplatedDecl()))
           continue;
 
-        AddTemplateOverloadCandidate(
+        SemaRef.AddTemplateOverloadCandidate(
             FTD, DeclAccessPair::make(FTD, C->getAccess()),
             /*ExplicitTemplateArgs=*/nullptr, Args, CandidateSet,
             /*SuppressUserConversions=*/false,
             /*PartialOverloading=*/true);
       }
     }
-    mergeCandidatesWithResults(*this, Results, CandidateSet, Loc, Args.size());
+    mergeCandidatesWithResults(SemaRef, Results, CandidateSet, Loc,
+                               Args.size());
   }
 
-  return ProduceSignatureHelp(*this, Results, Args.size(), OpenParLoc, Braced);
+  return ProduceSignatureHelp(SemaRef, Results, Args.size(), OpenParLoc,
+                              Braced);
 }
 
-QualType Sema::ProduceCtorInitMemberSignatureHelp(
+QualType SemaCodeCompletion::ProduceCtorInitMemberSignatureHelp(
     Decl *ConstructorDecl, CXXScopeSpec SS, ParsedType TemplateTypeTy,
     ArrayRef<Expr *> ArgExprs, IdentifierInfo *II, SourceLocation OpenParLoc,
     bool Braced) {
@@ -6454,7 +6480,7 @@ QualType Sema::ProduceCtorInitMemberSignatureHelp(
   if (!Constructor)
     return QualType();
   // FIXME: Add support for Base class constructors as well.
-  if (ValueDecl *MemberDecl = tryLookupCtorInitMemberDecl(
+  if (ValueDecl *MemberDecl = SemaRef.tryLookupCtorInitMemberDecl(
           Constructor->getParent(), SS, TemplateTypeTy, II))
     return ProduceConstructorSignatureHelp(MemberDecl->getType(),
                                            MemberDecl->getLocation(), ArgExprs,
@@ -6484,7 +6510,7 @@ static bool argMatchesTemplateParams(const ParsedTemplateArgument &Arg,
   llvm_unreachable("Unhandled switch case");
 }
 
-QualType Sema::ProduceTemplateArgumentSignatureHelp(
+QualType SemaCodeCompletion::ProduceTemplateArgumentSignatureHelp(
     TemplateTy ParsedTemplate, ArrayRef<ParsedTemplateArgument> Args,
     SourceLocation LAngleLoc) {
   if (!CodeCompleter || !ParsedTemplate)
@@ -6512,7 +6538,7 @@ QualType Sema::ProduceTemplateArgumentSignatureHelp(
       if (const auto *TD = llvm::dyn_cast<TemplateDecl>(ND))
         Consider(TD);
   }
-  return ProduceSignatureHelp(*this, Results, Args.size(), LAngleLoc,
+  return ProduceSignatureHelp(SemaRef, Results, Args.size(), LAngleLoc,
                               /*Braced=*/false);
 }
 
@@ -6541,9 +6567,8 @@ static QualType getDesignatedType(QualType BaseType, const Designation &Desig) {
   return BaseType;
 }
 
-void Sema::CodeCompleteDesignator(QualType BaseType,
-                                  llvm::ArrayRef<Expr *> InitExprs,
-                                  const Designation &D) {
+void SemaCodeCompletion::CodeCompleteDesignator(
+    QualType BaseType, llvm::ArrayRef<Expr *> InitExprs, const Designation &D) {
   BaseType = getDesignatedType(BaseType, D);
   if (BaseType.isNull())
     return;
@@ -6553,7 +6578,7 @@ void Sema::CodeCompleteDesignator(QualType BaseType,
 
   CodeCompletionContext CCC(CodeCompletionContext::CCC_DotMemberAccess,
                             BaseType);
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(), CCC);
 
   Results.EnterNewScope();
@@ -6569,14 +6594,15 @@ void Sema::CodeCompleteDesignator(QualType BaseType,
     // FIXME: Make use of previous designators to mark any fields before those
     // inaccessible, and also compute the next initializer priority.
     ResultBuilder::Result Result(FD, Results.getBasePriority(FD));
-    Results.AddResult(Result, CurContext, /*Hiding=*/nullptr);
+    Results.AddResult(Result, SemaRef.CurContext, /*Hiding=*/nullptr);
   }
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteInitializer(Scope *S, Decl *D) {
+void SemaCodeCompletion::CodeCompleteInitializer(Scope *S, Decl *D) {
   ValueDecl *VD = dyn_cast_or_null<ValueDecl>(D);
   if (!VD) {
     CodeCompleteOrdinaryName(S, PCC_Expression);
@@ -6591,19 +6617,19 @@ void Sema::CodeCompleteInitializer(Scope *S, Decl *D) {
   CodeCompleteExpression(S, Data);
 }
 
-void Sema::CodeCompleteAfterIf(Scope *S, bool IsBracedThen) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteAfterIf(Scope *S, bool IsBracedThen) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
-                        mapCodeCompletionContext(*this, PCC_Statement));
+                        mapCodeCompletionContext(SemaRef, PCC_Statement));
   Results.setFilter(&ResultBuilder::IsOrdinaryName);
   Results.EnterNewScope();
 
-  CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
-                     CodeCompleter->includeGlobals(),
-                     CodeCompleter->loadExternal());
+  CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
+  SemaRef.LookupVisibleDecls(S, Sema::LookupOrdinaryName, Consumer,
+                             CodeCompleter->includeGlobals(),
+                             CodeCompleter->loadExternal());
 
-  AddOrdinaryNameResults(PCC_Statement, S, *this, Results);
+  AddOrdinaryNameResults(PCC_Statement, S, SemaRef, Results);
 
   // "else" block
   CodeCompletionBuilder Builder(Results.getAllocator(),
@@ -6649,16 +6675,18 @@ void Sema::CodeCompleteAfterIf(Scope *S, bool IsBracedThen) {
     AddPrettyFunctionResults(getLangOpts(), Results);
 
   if (CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results, CodeCompleter->loadExternal(), false);
+    AddMacroResults(SemaRef.PP, Results, CodeCompleter->loadExternal(), false);
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
-                                   bool EnteringContext,
-                                   bool IsUsingDeclaration, QualType BaseType,
-                                   QualType PreferredType) {
+void SemaCodeCompletion::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
+                                                 bool EnteringContext,
+                                                 bool IsUsingDeclaration,
+                                                 QualType BaseType,
+                                                 QualType PreferredType) {
   if (SS.isEmpty() || !CodeCompleter)
     return;
 
@@ -6674,34 +6702,34 @@ void Sema::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
     // As SS is invalid, we try to collect accessible contexts from the current
     // scope with a dummy lookup so that the completion consumer can try to
     // guess what the specified scope is.
-    ResultBuilder DummyResults(*this, CodeCompleter->getAllocator(),
+    ResultBuilder DummyResults(SemaRef, CodeCompleter->getAllocator(),
                                CodeCompleter->getCodeCompletionTUInfo(), CC);
     if (!PreferredType.isNull())
       DummyResults.setPreferredType(PreferredType);
     if (S->getEntity()) {
       CodeCompletionDeclConsumer Consumer(DummyResults, S->getEntity(),
                                           BaseType);
-      LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
-                         /*IncludeGlobalScope=*/false,
-                         /*LoadExternal=*/false);
+      SemaRef.LookupVisibleDecls(S, Sema::LookupOrdinaryName, Consumer,
+                                 /*IncludeGlobalScope=*/false,
+                                 /*LoadExternal=*/false);
     }
-    HandleCodeCompleteResults(this, CodeCompleter,
+    HandleCodeCompleteResults(&SemaRef, CodeCompleter,
                               DummyResults.getCompletionContext(), nullptr, 0);
     return;
   }
   // Always pretend to enter a context to ensure that a dependent type
   // resolves to a dependent record.
-  DeclContext *Ctx = computeDeclContext(SS, /*EnteringContext=*/true);
+  DeclContext *Ctx = SemaRef.computeDeclContext(SS, /*EnteringContext=*/true);
 
   // Try to instantiate any non-dependent declaration contexts before
   // we look in them. Bail out if we fail.
   NestedNameSpecifier *NNS = SS.getScopeRep();
   if (NNS != nullptr && SS.isValid() && !NNS->isDependent()) {
-    if (Ctx == nullptr || RequireCompleteDeclContext(SS, Ctx))
+    if (Ctx == nullptr || SemaRef.RequireCompleteDeclContext(SS, Ctx))
       return;
   }
 
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(), CC);
   if (!PreferredType.isNull())
     Results.setPreferredType(PreferredType);
@@ -6722,7 +6750,7 @@ void Sema::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
         if (R.Operator != ConceptInfo::Member::Colons)
           continue;
         Results.AddResult(CodeCompletionResult(
-            R.render(*this, CodeCompleter->getAllocator(),
+            R.render(SemaRef, CodeCompleter->getAllocator(),
                      CodeCompleter->getCodeCompletionTUInfo())));
       }
     }
@@ -6734,23 +6762,24 @@ void Sema::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
   // in a context that permits expressions. This is a general issue with
   // qualified-id completions.
   if (Ctx && !EnteringContext)
-    MaybeAddOverrideCalls(*this, Ctx, Results);
+    MaybeAddOverrideCalls(SemaRef, Ctx, Results);
   Results.ExitScope();
 
   if (Ctx &&
       (CodeCompleter->includeNamespaceLevelDecls() || !Ctx->isFileContext())) {
     CodeCompletionDeclConsumer Consumer(Results, Ctx, BaseType);
-    LookupVisibleDecls(Ctx, LookupOrdinaryName, Consumer,
-                       /*IncludeGlobalScope=*/true,
-                       /*IncludeDependentBases=*/true,
-                       CodeCompleter->loadExternal());
+    SemaRef.LookupVisibleDecls(Ctx, Sema::LookupOrdinaryName, Consumer,
+                               /*IncludeGlobalScope=*/true,
+                               /*IncludeDependentBases=*/true,
+                               CodeCompleter->loadExternal());
   }
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteUsing(Scope *S) {
+void SemaCodeCompletion::CodeCompleteUsing(Scope *S) {
   if (!CodeCompleter)
     return;
 
@@ -6759,7 +6788,7 @@ void Sema::CodeCompleteUsing(Scope *S) {
   CodeCompletionContext Context(CodeCompletionContext::CCC_SymbolOrNewName);
   Context.setIsUsingDeclaration(true);
 
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(), Context,
                         &ResultBuilder::IsNestedNameSpecifier);
   Results.EnterNewScope();
@@ -6770,48 +6799,50 @@ void Sema::CodeCompleteUsing(Scope *S) {
 
   // After "using", we can see anything that would start a
   // nested-name-specifier.
-  CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
-                     CodeCompleter->includeGlobals(),
-                     CodeCompleter->loadExternal());
+  CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
+  SemaRef.LookupVisibleDecls(S, Sema::LookupOrdinaryName, Consumer,
+                             CodeCompleter->includeGlobals(),
+                             CodeCompleter->loadExternal());
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteUsingDirective(Scope *S) {
+void SemaCodeCompletion::CodeCompleteUsingDirective(Scope *S) {
   if (!CodeCompleter)
     return;
 
   // After "using namespace", we expect to see a namespace name or namespace
   // alias.
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Namespace,
                         &ResultBuilder::IsNamespaceOrAlias);
   Results.EnterNewScope();
-  CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
-                     CodeCompleter->includeGlobals(),
-                     CodeCompleter->loadExternal());
+  CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
+  SemaRef.LookupVisibleDecls(S, Sema::LookupOrdinaryName, Consumer,
+                             CodeCompleter->includeGlobals(),
+                             CodeCompleter->loadExternal());
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteNamespaceDecl(Scope *S) {
+void SemaCodeCompletion::CodeCompleteNamespaceDecl(Scope *S) {
   if (!CodeCompleter)
     return;
 
   DeclContext *Ctx = S->getEntity();
   if (!S->getParent())
-    Ctx = Context.getTranslationUnitDecl();
+    Ctx = getASTContext().getTranslationUnitDecl();
 
   bool SuppressedGlobalResults =
       Ctx && !CodeCompleter->includeGlobals() && isa<TranslationUnitDecl>(Ctx);
 
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         SuppressedGlobalResults
                             ? CodeCompletionContext::CCC_Namespace
@@ -6840,37 +6871,39 @@ void Sema::CodeCompleteNamespaceDecl(Scope *S) {
       Results.AddResult(
           CodeCompletionResult(NS->second, Results.getBasePriority(NS->second),
                                nullptr),
-          CurContext, nullptr, false);
+          SemaRef.CurContext, nullptr, false);
     Results.ExitScope();
   }
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteNamespaceAliasDecl(Scope *S) {
+void SemaCodeCompletion::CodeCompleteNamespaceAliasDecl(Scope *S) {
   if (!CodeCompleter)
     return;
 
   // After "namespace", we expect to see a namespace or alias.
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Namespace,
                         &ResultBuilder::IsNamespaceOrAlias);
-  CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
-                     CodeCompleter->includeGlobals(),
-                     CodeCompleter->loadExternal());
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
+  SemaRef.LookupVisibleDecls(S, Sema::LookupOrdinaryName, Consumer,
+                             CodeCompleter->includeGlobals(),
+                             CodeCompleter->loadExternal());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteOperatorName(Scope *S) {
+void SemaCodeCompletion::CodeCompleteOperatorName(Scope *S) {
   if (!CodeCompleter)
     return;
 
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Type,
                         &ResultBuilder::IsType);
@@ -6885,31 +6918,32 @@ void Sema::CodeCompleteOperatorName(Scope *S) {
 
   // Add any type names visible from the current scope
   Results.allowNestedNameSpecifiers();
-  CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
-                     CodeCompleter->includeGlobals(),
-                     CodeCompleter->loadExternal());
+  CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
+  SemaRef.LookupVisibleDecls(S, Sema::LookupOrdinaryName, Consumer,
+                             CodeCompleter->includeGlobals(),
+                             CodeCompleter->loadExternal());
 
   // Add any type specifiers
   AddTypeSpecifierResults(getLangOpts(), Results);
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteConstructorInitializer(
+void SemaCodeCompletion::CodeCompleteConstructorInitializer(
     Decl *ConstructorD, ArrayRef<CXXCtorInitializer *> Initializers) {
   if (!ConstructorD)
     return;
 
-  AdjustDeclIfTemplate(ConstructorD);
+  SemaRef.AdjustDeclIfTemplate(ConstructorD);
 
   auto *Constructor = dyn_cast<CXXConstructorDecl>(ConstructorD);
   if (!Constructor)
     return;
 
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Symbol);
   Results.EnterNewScope();
@@ -6919,7 +6953,7 @@ void Sema::CodeCompleteConstructorInitializer(
   llvm::SmallPtrSet<CanQualType, 4> InitializedBases;
   for (unsigned I = 0, E = Initializers.size(); I != E; ++I) {
     if (Initializers[I]->isBaseInitializer())
-      InitializedBases.insert(Context.getCanonicalType(
+      InitializedBases.insert(getASTContext().getCanonicalType(
           QualType(Initializers[I]->getBaseClass(), 0)));
     else
       InitializedFields.insert(
@@ -6927,7 +6961,7 @@ void Sema::CodeCompleteConstructorInitializer(
   }
 
   // Add completions for base classes.
-  PrintingPolicy Policy = getCompletionPrintingPolicy(*this);
+  PrintingPolicy Policy = getCompletionPrintingPolicy(SemaRef);
   bool SawLastInitializer = Initializers.empty();
   CXXRecordDecl *ClassDecl = Constructor->getParent();
 
@@ -6937,10 +6971,10 @@ void Sema::CodeCompleteConstructorInitializer(
     Builder.AddTypedTextChunk(Name);
     Builder.AddChunk(CodeCompletionString::CK_LeftParen);
     if (const auto *Function = dyn_cast<FunctionDecl>(ND))
-      AddFunctionParameterChunks(PP, Policy, Function, Builder);
+      AddFunctionParameterChunks(SemaRef.PP, Policy, Function, Builder);
     else if (const auto *FunTemplDecl = dyn_cast<FunctionTemplateDecl>(ND))
-      AddFunctionParameterChunks(PP, Policy, FunTemplDecl->getTemplatedDecl(),
-                                 Builder);
+      AddFunctionParameterChunks(SemaRef.PP, Policy,
+                                 FunTemplDecl->getTemplatedDecl(), Builder);
     Builder.AddChunk(CodeCompletionString::CK_RightParen);
     return Builder.TakeString();
   };
@@ -6972,7 +7006,7 @@ void Sema::CodeCompleteConstructorInitializer(
                                          FD->getType().getAsString(Policy))
                                    : Name,
                                 FD);
-    auto Ctors = getConstructors(Context, RD);
+    auto Ctors = getConstructors(getASTContext(), RD);
     if (Ctors.begin() == Ctors.end())
       return AddDefaultCtorInit(Name, Name, RD);
     for (const NamedDecl *Ctor : Ctors) {
@@ -6999,11 +7033,12 @@ void Sema::CodeCompleteConstructorInitializer(
   };
 
   for (const auto &Base : ClassDecl->bases()) {
-    if (!InitializedBases.insert(Context.getCanonicalType(Base.getType()))
+    if (!InitializedBases
+             .insert(getASTContext().getCanonicalType(Base.getType()))
              .second) {
       SawLastInitializer =
           !Initializers.empty() && Initializers.back()->isBaseInitializer() &&
-          Context.hasSameUnqualifiedType(
+          getASTContext().hasSameUnqualifiedType(
               Base.getType(), QualType(Initializers.back()->getBaseClass(), 0));
       continue;
     }
@@ -7014,11 +7049,12 @@ void Sema::CodeCompleteConstructorInitializer(
 
   // Add completions for virtual base classes.
   for (const auto &Base : ClassDecl->vbases()) {
-    if (!InitializedBases.insert(Context.getCanonicalType(Base.getType()))
+    if (!InitializedBases
+             .insert(getASTContext().getCanonicalType(Base.getType()))
              .second) {
       SawLastInitializer =
           !Initializers.empty() && Initializers.back()->isBaseInitializer() &&
-          Context.hasSameUnqualifiedType(
+          getASTContext().hasSameUnqualifiedType(
               Base.getType(), QualType(Initializers.back()->getBaseClass(), 0));
       continue;
     }
@@ -7045,8 +7081,9 @@ void Sema::CodeCompleteConstructorInitializer(
   }
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 /// Determine whether this scope denotes a namespace.
@@ -7058,9 +7095,10 @@ static bool isNamespaceScope(Scope *S) {
   return DC->isFileContext();
 }
 
-void Sema::CodeCompleteLambdaIntroducer(Scope *S, LambdaIntroducer &Intro,
-                                        bool AfterAmpersand) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteLambdaIntroducer(Scope *S,
+                                                      LambdaIntroducer &Intro,
+                                                      bool AfterAmpersand) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
@@ -7086,24 +7124,25 @@ void Sema::CodeCompleteLambdaIntroducer(Scope *S, LambdaIntroducer &Intro,
 
       if (Known.insert(Var->getIdentifier()).second)
         Results.AddResult(CodeCompletionResult(Var, CCP_LocalDeclaration),
-                          CurContext, nullptr, false);
+                          SemaRef.CurContext, nullptr, false);
     }
   }
 
   // Add 'this', if it would be valid.
   if (!IncludedThis && !AfterAmpersand && Intro.Default != LCD_ByCopy)
-    addThisCompletion(*this, Results);
+    addThisCompletion(SemaRef, Results);
 
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteAfterFunctionEquals(Declarator &D) {
-  if (!LangOpts.CPlusPlus11)
+void SemaCodeCompletion::CodeCompleteAfterFunctionEquals(Declarator &D) {
+  if (!getLangOpts().CPlusPlus11)
     return;
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   auto ShouldAddDefault = [&D, this]() {
@@ -7123,7 +7162,7 @@ void Sema::CodeCompleteAfterFunctionEquals(Declarator &D) {
       // verify that it is the copy or move assignment?
       if (Op == OverloadedOperatorKind::OO_Equal)
         return true;
-      if (LangOpts.CPlusPlus20 &&
+      if (getLangOpts().CPlusPlus20 &&
           (Op == OverloadedOperatorKind::OO_EqualEqual ||
            Op == OverloadedOperatorKind::OO_ExclaimEqual ||
            Op == OverloadedOperatorKind::OO_Less ||
@@ -7143,8 +7182,9 @@ void Sema::CodeCompleteAfterFunctionEquals(Declarator &D) {
   // first function declaration.
   Results.AddResult("delete");
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 /// Macro that optionally prepends an "@" to the string literal passed in via
@@ -7244,20 +7284,21 @@ static void AddObjCTopLevelResults(ResultBuilder &Results, bool NeedAt) {
   }
 }
 
-void Sema::CodeCompleteObjCAtDirective(Scope *S) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteObjCAtDirective(Scope *S) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
-  if (isa<ObjCImplDecl>(CurContext))
+  if (isa<ObjCImplDecl>(SemaRef.CurContext))
     AddObjCImplementationResults(getLangOpts(), Results, false);
-  else if (CurContext->isObjCContainer())
+  else if (SemaRef.CurContext->isObjCContainer())
     AddObjCInterfaceResults(getLangOpts(), Results, false);
   else
     AddObjCTopLevelResults(Results, false);
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 static void AddObjCExpressionResults(ResultBuilder &Results, bool NeedAt) {
@@ -7381,38 +7422,41 @@ static void AddObjCVisibilityResults(const LangOptions &LangOpts,
     Results.AddResult(Result(OBJC_AT_KEYWORD_NAME(NeedAt, "package")));
 }
 
-void Sema::CodeCompleteObjCAtVisibility(Scope *S) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteObjCAtVisibility(Scope *S) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   AddObjCVisibilityResults(getLangOpts(), Results, false);
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCAtStatement(Scope *S) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteObjCAtStatement(Scope *S) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   AddObjCStatementResults(Results, false);
   AddObjCExpressionResults(Results, false);
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCAtExpression(Scope *S) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteObjCAtExpression(Scope *S) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   AddObjCExpressionResults(Results, false);
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 /// Determine whether the addition of the given flag to an Objective-C
@@ -7448,13 +7492,14 @@ static bool ObjCPropertyFlagConflicts(unsigned Attributes, unsigned NewFlag) {
   return false;
 }
 
-void Sema::CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS) {
+void SemaCodeCompletion::CodeCompleteObjCPropertyFlags(Scope *S,
+                                                       ObjCDeclSpec &ODS) {
   if (!CodeCompleter)
     return;
 
   unsigned Attributes = ODS.getPropertyAttributes();
 
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
@@ -7517,8 +7562,9 @@ void Sema::CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS) {
     Results.AddResult(CodeCompletionResult("null_resettable"));
   }
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 /// Describes the kind of Objective-C method that we want to find
@@ -7677,12 +7723,13 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
                    IsRootClass);
 }
 
-void Sema::CodeCompleteObjCPropertyGetter(Scope *S) {
+void SemaCodeCompletion::CodeCompleteObjCPropertyGetter(Scope *S) {
   // Try to find the interface where getters might live.
-  ObjCInterfaceDecl *Class = dyn_cast_or_null<ObjCInterfaceDecl>(CurContext);
+  ObjCInterfaceDecl *Class =
+      dyn_cast_or_null<ObjCInterfaceDecl>(SemaRef.CurContext);
   if (!Class) {
     if (ObjCCategoryDecl *Category =
-            dyn_cast_or_null<ObjCCategoryDecl>(CurContext))
+            dyn_cast_or_null<ObjCCategoryDecl>(SemaRef.CurContext))
       Class = Category->getClassInterface();
 
     if (!Class)
@@ -7690,26 +7737,28 @@ void Sema::CodeCompleteObjCPropertyGetter(Scope *S) {
   }
 
   // Find all of the potential getters.
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
 
   VisitedSelectorSet Selectors;
-  AddObjCMethods(Class, true, MK_ZeroArgSelector, std::nullopt, CurContext,
-                 Selectors,
+  AddObjCMethods(Class, true, MK_ZeroArgSelector, std::nullopt,
+                 SemaRef.CurContext, Selectors,
                  /*AllowSameLength=*/true, Results);
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCPropertySetter(Scope *S) {
+void SemaCodeCompletion::CodeCompleteObjCPropertySetter(Scope *S) {
   // Try to find the interface where setters might live.
-  ObjCInterfaceDecl *Class = dyn_cast_or_null<ObjCInterfaceDecl>(CurContext);
+  ObjCInterfaceDecl *Class =
+      dyn_cast_or_null<ObjCInterfaceDecl>(SemaRef.CurContext);
   if (!Class) {
     if (ObjCCategoryDecl *Category =
-            dyn_cast_or_null<ObjCCategoryDecl>(CurContext))
+            dyn_cast_or_null<ObjCCategoryDecl>(SemaRef.CurContext))
       Class = Category->getClassInterface();
 
     if (!Class)
@@ -7717,24 +7766,25 @@ void Sema::CodeCompleteObjCPropertySetter(Scope *S) {
   }
 
   // Find all of the potential getters.
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
 
   VisitedSelectorSet Selectors;
-  AddObjCMethods(Class, true, MK_OneArgSelector, std::nullopt, CurContext,
-                 Selectors,
+  AddObjCMethods(Class, true, MK_OneArgSelector, std::nullopt,
+                 SemaRef.CurContext, Selectors,
                  /*AllowSameLength=*/true, Results);
 
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS,
-                                       bool IsParameter) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS,
+                                                     bool IsParameter) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Type);
   Results.EnterNewScope();
@@ -7771,7 +7821,7 @@ void Sema::CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS,
   // an action, e.g.,
   //   IBAction)<#selector#>:(id)sender
   if (DS.getObjCDeclQualifier() == 0 && !IsParameter &&
-      PP.isMacroDefined("IBAction")) {
+      SemaRef.PP.isMacroDefined("IBAction")) {
     CodeCompletionBuilder Builder(Results.getAllocator(),
                                   Results.getCodeCompletionTUInfo(),
                                   CCP_CodePattern, CXAvailability_Available);
@@ -7792,21 +7842,22 @@ void Sema::CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS,
   }
 
   // Add various builtin type names and specifiers.
-  AddOrdinaryNameResults(PCC_Type, S, *this, Results);
+  AddOrdinaryNameResults(PCC_Type, S, SemaRef, Results);
   Results.ExitScope();
 
   // Add the various type names
   Results.setFilter(&ResultBuilder::IsOrdinaryNonValueName);
-  CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
-                     CodeCompleter->includeGlobals(),
-                     CodeCompleter->loadExternal());
+  CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
+  SemaRef.LookupVisibleDecls(S, Sema::LookupOrdinaryName, Consumer,
+                             CodeCompleter->includeGlobals(),
+                             CodeCompleter->loadExternal());
 
   if (CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results, CodeCompleter->loadExternal(), false);
+    AddMacroResults(SemaRef.PP, Results, CodeCompleter->loadExternal(), false);
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 /// When we have an expression with type "id", we may assume
@@ -7997,49 +8048,50 @@ AddSuperSendCompletion(Sema &S, bool NeedSuperKeyword,
   return SuperMethod;
 }
 
-void Sema::CodeCompleteObjCMessageReceiver(Scope *S) {
+void SemaCodeCompletion::CodeCompleteObjCMessageReceiver(Scope *S) {
   typedef CodeCompletionResult Result;
   ResultBuilder Results(
-      *this, CodeCompleter->getAllocator(),
+      SemaRef, CodeCompleter->getAllocator(),
       CodeCompleter->getCodeCompletionTUInfo(),
       CodeCompletionContext::CCC_ObjCMessageReceiver,
       getLangOpts().CPlusPlus11
           ? &ResultBuilder::IsObjCMessageReceiverOrLambdaCapture
           : &ResultBuilder::IsObjCMessageReceiver);
 
-  CodeCompletionDeclConsumer Consumer(Results, CurContext);
+  CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
   Results.EnterNewScope();
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
-                     CodeCompleter->includeGlobals(),
-                     CodeCompleter->loadExternal());
+  SemaRef.LookupVisibleDecls(S, Sema::LookupOrdinaryName, Consumer,
+                             CodeCompleter->includeGlobals(),
+                             CodeCompleter->loadExternal());
 
   // If we are in an Objective-C method inside a class that has a superclass,
   // add "super" as an option.
-  if (ObjCMethodDecl *Method = getCurMethodDecl())
+  if (ObjCMethodDecl *Method = SemaRef.getCurMethodDecl())
     if (ObjCInterfaceDecl *Iface = Method->getClassInterface())
       if (Iface->getSuperClass()) {
         Results.AddResult(Result("super"));
 
-        AddSuperSendCompletion(*this, /*NeedSuperKeyword=*/true, std::nullopt,
+        AddSuperSendCompletion(SemaRef, /*NeedSuperKeyword=*/true, std::nullopt,
                                Results);
       }
 
   if (getLangOpts().CPlusPlus11)
-    addThisCompletion(*this, Results);
+    addThisCompletion(SemaRef, Results);
 
   Results.ExitScope();
 
   if (CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results, CodeCompleter->loadExternal(), false);
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+    AddMacroResults(SemaRef.PP, Results, CodeCompleter->loadExternal(), false);
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCSuperMessage(
+void SemaCodeCompletion::CodeCompleteObjCSuperMessage(
     Scope *S, SourceLocation SuperLoc,
     ArrayRef<const IdentifierInfo *> SelIdents, bool AtArgumentExpression) {
   ObjCInterfaceDecl *CDecl = nullptr;
-  if (ObjCMethodDecl *CurMethod = getCurMethodDecl()) {
+  if (ObjCMethodDecl *CurMethod = SemaRef.getCurMethodDecl()) {
     // Figure out which interface we're in.
     CDecl = CurMethod->getClassInterface();
     if (!CDecl)
@@ -8062,13 +8114,14 @@ void Sema::CodeCompleteObjCSuperMessage(
   } else {
     // "super" may be the name of a type or variable. Figure out which
     // it is.
-    const IdentifierInfo *Super = getSuperIdentifier();
-    NamedDecl *ND = LookupSingleName(S, Super, SuperLoc, LookupOrdinaryName);
+    const IdentifierInfo *Super = SemaRef.getSuperIdentifier();
+    NamedDecl *ND =
+        SemaRef.LookupSingleName(S, Super, SuperLoc, Sema::LookupOrdinaryName);
     if ((CDecl = dyn_cast_or_null<ObjCInterfaceDecl>(ND))) {
       // "super" names an interface. Use it.
     } else if (TypeDecl *TD = dyn_cast_or_null<TypeDecl>(ND)) {
       if (const ObjCObjectType *Iface =
-              Context.getTypeDeclType(TD)->getAs<ObjCObjectType>())
+              getASTContext().getTypeDeclType(TD)->getAs<ObjCObjectType>())
         CDecl = Iface->getInterface();
     } else if (ND && isa<UnresolvedUsingTypenameDecl>(ND)) {
       // "super" names an unresolved type; we can't be more specific.
@@ -8078,9 +8131,10 @@ void Sema::CodeCompleteObjCSuperMessage(
       SourceLocation TemplateKWLoc;
       UnqualifiedId id;
       id.setIdentifier(Super, SuperLoc);
-      ExprResult SuperExpr = ActOnIdExpression(S, SS, TemplateKWLoc, id,
-                                               /*HasTrailingLParen=*/false,
-                                               /*IsAddressOfOperand=*/false);
+      ExprResult SuperExpr =
+          SemaRef.ActOnIdExpression(S, SS, TemplateKWLoc, id,
+                                    /*HasTrailingLParen=*/false,
+                                    /*IsAddressOfOperand=*/false);
       return CodeCompleteObjCInstanceMessage(S, (Expr *)SuperExpr.get(),
                                              SelIdents, AtArgumentExpression);
     }
@@ -8090,7 +8144,7 @@ void Sema::CodeCompleteObjCSuperMessage(
 
   ParsedType Receiver;
   if (CDecl)
-    Receiver = ParsedType::make(Context.getObjCInterfaceType(CDecl));
+    Receiver = ParsedType::make(getASTContext().getObjCInterfaceType(CDecl));
   return CodeCompleteObjCClassMessage(S, Receiver, SelIdents,
                                       AtArgumentExpression,
                                       /*IsSuper=*/true);
@@ -8206,19 +8260,19 @@ AddClassMessageCompletions(Sema &SemaRef, Scope *S, ParsedType Receiver,
   Results.ExitScope();
 }
 
-void Sema::CodeCompleteObjCClassMessage(
+void SemaCodeCompletion::CodeCompleteObjCClassMessage(
     Scope *S, ParsedType Receiver, ArrayRef<const IdentifierInfo *> SelIdents,
     bool AtArgumentExpression, bool IsSuper) {
 
-  QualType T = this->GetTypeFromParser(Receiver);
+  QualType T = SemaRef.GetTypeFromParser(Receiver);
 
   ResultBuilder Results(
-      *this, CodeCompleter->getAllocator(),
+      SemaRef, CodeCompleter->getAllocator(),
       CodeCompleter->getCodeCompletionTUInfo(),
       CodeCompletionContext(CodeCompletionContext::CCC_ObjCClassMessage, T,
                             SelIdents));
 
-  AddClassMessageCompletions(*this, S, Receiver, SelIdents,
+  AddClassMessageCompletions(SemaRef, S, Receiver, SelIdents,
                              AtArgumentExpression, IsSuper, Results);
 
   // If we're actually at the argument expression (rather than prior to the
@@ -8236,21 +8290,23 @@ void Sema::CodeCompleteObjCClassMessage(
     return;
   }
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCInstanceMessage(
+void SemaCodeCompletion::CodeCompleteObjCInstanceMessage(
     Scope *S, Expr *Receiver, ArrayRef<const IdentifierInfo *> SelIdents,
     bool AtArgumentExpression, ObjCInterfaceDecl *Super) {
   typedef CodeCompletionResult Result;
+  ASTContext &Context = getASTContext();
 
   Expr *RecExpr = static_cast<Expr *>(Receiver);
 
   // If necessary, apply function/array conversion to the receiver.
   // C99 6.7.5.3p[7,8].
   if (RecExpr) {
-    ExprResult Conv = DefaultFunctionArrayLvalueConversion(RecExpr);
+    ExprResult Conv = SemaRef.DefaultFunctionArrayLvalueConversion(RecExpr);
     if (Conv.isInvalid()) // conversion failed. bail.
       return;
     RecExpr = Conv.get();
@@ -8275,7 +8331,7 @@ void Sema::CodeCompleteObjCInstanceMessage(
           Context.getObjCObjectPointerType(Context.getObjCInterfaceType(IFace));
     }
   } else if (RecExpr && getLangOpts().CPlusPlus) {
-    ExprResult Conv = PerformContextuallyConvertToObjCPointer(RecExpr);
+    ExprResult Conv = SemaRef.PerformContextuallyConvertToObjCPointer(RecExpr);
     if (Conv.isUsable()) {
       RecExpr = Conv.get();
       ReceiverType = RecExpr->getType();
@@ -8284,7 +8340,7 @@ void Sema::CodeCompleteObjCInstanceMessage(
 
   // Build the set of methods we can see.
   ResultBuilder Results(
-      *this, CodeCompleter->getAllocator(),
+      SemaRef, CodeCompleter->getAllocator(),
       CodeCompleter->getCodeCompletionTUInfo(),
       CodeCompletionContext(CodeCompletionContext::CCC_ObjCInstanceMessage,
                             ReceiverType, SelIdents));
@@ -8295,13 +8351,13 @@ void Sema::CodeCompleteObjCInstanceMessage(
   // completion.
   if (Super) {
     if (ObjCMethodDecl *SuperMethod =
-            AddSuperSendCompletion(*this, false, SelIdents, Results))
+            AddSuperSendCompletion(SemaRef, false, SelIdents, Results))
       Results.Ignore(SuperMethod);
   }
 
   // If we're inside an Objective-C method definition, prefer its selector to
   // others.
-  if (ObjCMethodDecl *CurMethod = getCurMethodDecl())
+  if (ObjCMethodDecl *CurMethod = SemaRef.getCurMethodDecl())
     Results.setPreferredSelector(CurMethod->getSelector());
 
   // Keep track of the selectors we've already added.
@@ -8312,9 +8368,9 @@ void Sema::CodeCompleteObjCInstanceMessage(
   // class method.
   if (ReceiverType->isObjCClassType() ||
       ReceiverType->isObjCQualifiedClassType()) {
-    if (ObjCMethodDecl *CurMethod = getCurMethodDecl()) {
+    if (ObjCMethodDecl *CurMethod = SemaRef.getCurMethodDecl()) {
       if (ObjCInterfaceDecl *ClassDecl = CurMethod->getClassInterface())
-        AddObjCMethods(ClassDecl, false, MK_Any, SelIdents, CurContext,
+        AddObjCMethods(ClassDecl, false, MK_Any, SelIdents, SemaRef.CurContext,
                        Selectors, AtArgumentExpression, Results);
     }
   }
@@ -8323,7 +8379,7 @@ void Sema::CodeCompleteObjCInstanceMessage(
                ReceiverType->getAsObjCQualifiedIdType()) {
     // Search protocols for instance methods.
     for (auto *I : QualID->quals())
-      AddObjCMethods(I, true, MK_Any, SelIdents, CurContext, Selectors,
+      AddObjCMethods(I, true, MK_Any, SelIdents, SemaRef.CurContext, Selectors,
                      AtArgumentExpression, Results);
   }
   // Handle messages to a pointer to interface type.
@@ -8331,11 +8387,12 @@ void Sema::CodeCompleteObjCInstanceMessage(
                ReceiverType->getAsObjCInterfacePointerType()) {
     // Search the class, its superclasses, etc., for instance methods.
     AddObjCMethods(IFacePtr->getInterfaceDecl(), true, MK_Any, SelIdents,
-                   CurContext, Selectors, AtArgumentExpression, Results);
+                   SemaRef.CurContext, Selectors, AtArgumentExpression,
+                   Results);
 
     // Search protocols for instance methods.
     for (auto *I : IFacePtr->quals())
-      AddObjCMethods(I, true, MK_Any, SelIdents, CurContext, Selectors,
+      AddObjCMethods(I, true, MK_Any, SelIdents, SemaRef.CurContext, Selectors,
                      AtArgumentExpression, Results);
   }
   // Handle messages to "id".
@@ -8345,19 +8402,21 @@ void Sema::CodeCompleteObjCInstanceMessage(
 
     // If we have an external source, load the entire class method
     // pool from the AST file.
-    if (ExternalSource) {
-      for (uint32_t I = 0, N = ExternalSource->GetNumExternalSelectors();
+    if (SemaRef.ExternalSource) {
+      for (uint32_t I = 0,
+                    N = SemaRef.ExternalSource->GetNumExternalSelectors();
            I != N; ++I) {
-        Selector Sel = ExternalSource->GetExternalSelector(I);
-        if (Sel.isNull() || ObjC().MethodPool.count(Sel))
+        Selector Sel = SemaRef.ExternalSource->GetExternalSelector(I);
+        if (Sel.isNull() || SemaRef.ObjC().MethodPool.count(Sel))
           continue;
 
-        ObjC().ReadMethodPool(Sel);
+        SemaRef.ObjC().ReadMethodPool(Sel);
       }
     }
 
-    for (SemaObjC::GlobalMethodPool::iterator M = ObjC().MethodPool.begin(),
-                                              MEnd = ObjC().MethodPool.end();
+    for (SemaObjC::GlobalMethodPool::iterator
+             M = SemaRef.ObjC().MethodPool.begin(),
+             MEnd = SemaRef.ObjC().MethodPool.end();
          M != MEnd; ++M) {
       for (ObjCMethodList *MethList = &M->second.first;
            MethList && MethList->getMethod(); MethList = MethList->getNext()) {
@@ -8371,7 +8430,7 @@ void Sema::CodeCompleteObjCInstanceMessage(
                  Results.getBasePriority(MethList->getMethod()), nullptr);
         R.StartParameter = SelIdents.size();
         R.AllParametersAreInformative = false;
-        Results.MaybeAddResult(R, CurContext);
+        Results.MaybeAddResult(R, SemaRef.CurContext);
       }
     }
   }
@@ -8392,12 +8451,13 @@ void Sema::CodeCompleteObjCInstanceMessage(
     return;
   }
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCForCollection(Scope *S,
-                                         DeclGroupPtrTy IterationVar) {
+void SemaCodeCompletion::CodeCompleteObjCForCollection(
+    Scope *S, DeclGroupPtrTy IterationVar) {
   CodeCompleteExpressionData Data;
   Data.ObjCCollection = true;
 
@@ -8412,27 +8472,28 @@ void Sema::CodeCompleteObjCForCollection(Scope *S,
   CodeCompleteExpression(S, Data);
 }
 
-void Sema::CodeCompleteObjCSelector(
+void SemaCodeCompletion::CodeCompleteObjCSelector(
     Scope *S, ArrayRef<const IdentifierInfo *> SelIdents) {
   // If we have an external source, load the entire class method
   // pool from the AST file.
-  if (ExternalSource) {
-    for (uint32_t I = 0, N = ExternalSource->GetNumExternalSelectors(); I != N;
-         ++I) {
-      Selector Sel = ExternalSource->GetExternalSelector(I);
-      if (Sel.isNull() || ObjC().MethodPool.count(Sel))
+  if (SemaRef.ExternalSource) {
+    for (uint32_t I = 0, N = SemaRef.ExternalSource->GetNumExternalSelectors();
+         I != N; ++I) {
+      Selector Sel = SemaRef.ExternalSource->GetExternalSelector(I);
+      if (Sel.isNull() || SemaRef.ObjC().MethodPool.count(Sel))
         continue;
 
-      ObjC().ReadMethodPool(Sel);
+      SemaRef.ObjC().ReadMethodPool(Sel);
     }
   }
 
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_SelectorName);
   Results.EnterNewScope();
-  for (SemaObjC::GlobalMethodPool::iterator M = ObjC().MethodPool.begin(),
-                                            MEnd = ObjC().MethodPool.end();
+  for (SemaObjC::GlobalMethodPool::iterator
+           M = SemaRef.ObjC().MethodPool.begin(),
+           MEnd = SemaRef.ObjC().MethodPool.end();
        M != MEnd; ++M) {
 
     Selector Sel = M->first;
@@ -8466,8 +8527,9 @@ void Sema::CodeCompleteObjCSelector(
   }
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 /// Add all of the protocol declarations that we find in the given
@@ -8487,9 +8549,9 @@ static void AddProtocolResults(DeclContext *Ctx, DeclContext *CurContext,
   }
 }
 
-void Sema::CodeCompleteObjCProtocolReferences(
+void SemaCodeCompletion::CodeCompleteObjCProtocolReferences(
     ArrayRef<IdentifierLocPair> Protocols) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_ObjCProtocolName);
 
@@ -8501,22 +8563,23 @@ void Sema::CodeCompleteObjCProtocolReferences(
     // FIXME: This doesn't work when caching code-completion results.
     for (const IdentifierLocPair &Pair : Protocols)
       if (ObjCProtocolDecl *Protocol =
-              ObjC().LookupProtocol(Pair.first, Pair.second))
+              SemaRef.ObjC().LookupProtocol(Pair.first, Pair.second))
         Results.Ignore(Protocol);
 
     // Add all protocols.
-    AddProtocolResults(Context.getTranslationUnitDecl(), CurContext, false,
-                       Results);
+    AddProtocolResults(getASTContext().getTranslationUnitDecl(),
+                       SemaRef.CurContext, false, Results);
 
     Results.ExitScope();
   }
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCProtocolDecl(Scope *) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteObjCProtocolDecl(Scope *) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_ObjCProtocolName);
 
@@ -8524,14 +8587,15 @@ void Sema::CodeCompleteObjCProtocolDecl(Scope *) {
     Results.EnterNewScope();
 
     // Add all protocols.
-    AddProtocolResults(Context.getTranslationUnitDecl(), CurContext, true,
-                       Results);
+    AddProtocolResults(getASTContext().getTranslationUnitDecl(),
+                       SemaRef.CurContext, true, Results);
 
     Results.ExitScope();
   }
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 /// Add all of the Objective-C interface declarations that we find in
@@ -8553,99 +8617,102 @@ static void AddInterfaceResults(DeclContext *Ctx, DeclContext *CurContext,
   }
 }
 
-void Sema::CodeCompleteObjCInterfaceDecl(Scope *S) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteObjCInterfaceDecl(Scope *S) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_ObjCInterfaceName);
   Results.EnterNewScope();
 
   if (CodeCompleter->includeGlobals()) {
     // Add all classes.
-    AddInterfaceResults(Context.getTranslationUnitDecl(), CurContext, false,
-                        false, Results);
+    AddInterfaceResults(getASTContext().getTranslationUnitDecl(),
+                        SemaRef.CurContext, false, false, Results);
   }
 
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCClassForwardDecl(Scope *S) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteObjCClassForwardDecl(Scope *S) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_ObjCClassForwardDecl);
   Results.EnterNewScope();
 
   if (CodeCompleter->includeGlobals()) {
     // Add all classes.
-    AddInterfaceResults(Context.getTranslationUnitDecl(), CurContext, false,
-                        false, Results);
+    AddInterfaceResults(getASTContext().getTranslationUnitDecl(),
+                        SemaRef.CurContext, false, false, Results);
   }
 
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCSuperclass(Scope *S, IdentifierInfo *ClassName,
-                                      SourceLocation ClassNameLoc) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteObjCSuperclass(
+    Scope *S, IdentifierInfo *ClassName, SourceLocation ClassNameLoc) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_ObjCInterfaceName);
   Results.EnterNewScope();
 
   // Make sure that we ignore the class we're currently defining.
-  NamedDecl *CurClass =
-      LookupSingleName(TUScope, ClassName, ClassNameLoc, LookupOrdinaryName);
+  NamedDecl *CurClass = SemaRef.LookupSingleName(
+      SemaRef.TUScope, ClassName, ClassNameLoc, Sema::LookupOrdinaryName);
   if (CurClass && isa<ObjCInterfaceDecl>(CurClass))
     Results.Ignore(CurClass);
 
   if (CodeCompleter->includeGlobals()) {
     // Add all classes.
-    AddInterfaceResults(Context.getTranslationUnitDecl(), CurContext, false,
-                        false, Results);
+    AddInterfaceResults(getASTContext().getTranslationUnitDecl(),
+                        SemaRef.CurContext, false, false, Results);
   }
 
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCImplementationDecl(Scope *S) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteObjCImplementationDecl(Scope *S) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_ObjCImplementation);
   Results.EnterNewScope();
 
   if (CodeCompleter->includeGlobals()) {
     // Add all unimplemented classes.
-    AddInterfaceResults(Context.getTranslationUnitDecl(), CurContext, false,
-                        true, Results);
+    AddInterfaceResults(getASTContext().getTranslationUnitDecl(),
+                        SemaRef.CurContext, false, true, Results);
   }
 
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCInterfaceCategory(Scope *S,
-                                             IdentifierInfo *ClassName,
-                                             SourceLocation ClassNameLoc) {
+void SemaCodeCompletion::CodeCompleteObjCInterfaceCategory(
+    Scope *S, IdentifierInfo *ClassName, SourceLocation ClassNameLoc) {
   typedef CodeCompletionResult Result;
 
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_ObjCCategoryName);
 
   // Ignore any categories we find that have already been implemented by this
   // interface.
   llvm::SmallPtrSet<IdentifierInfo *, 16> CategoryNames;
-  NamedDecl *CurClass =
-      LookupSingleName(TUScope, ClassName, ClassNameLoc, LookupOrdinaryName);
+  NamedDecl *CurClass = SemaRef.LookupSingleName(
+      SemaRef.TUScope, ClassName, ClassNameLoc, Sema::LookupOrdinaryName);
   if (ObjCInterfaceDecl *Class =
           dyn_cast_or_null<ObjCInterfaceDecl>(CurClass)) {
     for (const auto *Cat : Class->visible_categories())
@@ -8654,34 +8721,34 @@ void Sema::CodeCompleteObjCInterfaceCategory(Scope *S,
 
   // Add all of the categories we know about.
   Results.EnterNewScope();
-  TranslationUnitDecl *TU = Context.getTranslationUnitDecl();
+  TranslationUnitDecl *TU = getASTContext().getTranslationUnitDecl();
   for (const auto *D : TU->decls())
     if (const auto *Category = dyn_cast<ObjCCategoryDecl>(D))
       if (CategoryNames.insert(Category->getIdentifier()).second)
         Results.AddResult(
             Result(Category, Results.getBasePriority(Category), nullptr),
-            CurContext, nullptr, false);
+            SemaRef.CurContext, nullptr, false);
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCImplementationCategory(Scope *S,
-                                                  IdentifierInfo *ClassName,
-                                                  SourceLocation ClassNameLoc) {
+void SemaCodeCompletion::CodeCompleteObjCImplementationCategory(
+    Scope *S, IdentifierInfo *ClassName, SourceLocation ClassNameLoc) {
   typedef CodeCompletionResult Result;
 
   // Find the corresponding interface. If we couldn't find the interface, the
   // program itself is ill-formed. However, we'll try to be helpful still by
   // providing the list of all of the categories we know about.
-  NamedDecl *CurClass =
-      LookupSingleName(TUScope, ClassName, ClassNameLoc, LookupOrdinaryName);
+  NamedDecl *CurClass = SemaRef.LookupSingleName(
+      SemaRef.TUScope, ClassName, ClassNameLoc, Sema::LookupOrdinaryName);
   ObjCInterfaceDecl *Class = dyn_cast_or_null<ObjCInterfaceDecl>(CurClass);
   if (!Class)
     return CodeCompleteObjCInterfaceCategory(S, ClassName, ClassNameLoc);
 
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_ObjCCategoryName);
 
@@ -8696,7 +8763,7 @@ void Sema::CodeCompleteObjCImplementationCategory(Scope *S,
       if ((!IgnoreImplemented || !Cat->getImplementation()) &&
           CategoryNames.insert(Cat->getIdentifier()).second)
         Results.AddResult(Result(Cat, Results.getBasePriority(Cat), nullptr),
-                          CurContext, nullptr, false);
+                          SemaRef.CurContext, nullptr, false);
     }
 
     Class = Class->getSuperClass();
@@ -8704,18 +8771,19 @@ void Sema::CodeCompleteObjCImplementationCategory(Scope *S,
   }
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCPropertyDefinition(Scope *S) {
+void SemaCodeCompletion::CodeCompleteObjCPropertyDefinition(Scope *S) {
   CodeCompletionContext CCContext(CodeCompletionContext::CCC_Other);
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(), CCContext);
 
   // Figure out where this @synthesize lives.
   ObjCContainerDecl *Container =
-      dyn_cast_or_null<ObjCContainerDecl>(CurContext);
+      dyn_cast_or_null<ObjCContainerDecl>(SemaRef.CurContext);
   if (!Container || (!isa<ObjCImplementationDecl>(Container) &&
                      !isa<ObjCCategoryImplDecl>(Container)))
     return;
@@ -8732,29 +8800,30 @@ void Sema::CodeCompleteObjCPropertyDefinition(Scope *S) {
   if (ObjCImplementationDecl *ClassImpl =
           dyn_cast<ObjCImplementationDecl>(Container))
     AddObjCProperties(CCContext, ClassImpl->getClassInterface(), false,
-                      /*AllowNullaryMethods=*/false, CurContext,
+                      /*AllowNullaryMethods=*/false, SemaRef.CurContext,
                       AddedProperties, Results);
   else
     AddObjCProperties(CCContext,
                       cast<ObjCCategoryImplDecl>(Container)->getCategoryDecl(),
-                      false, /*AllowNullaryMethods=*/false, CurContext,
+                      false, /*AllowNullaryMethods=*/false, SemaRef.CurContext,
                       AddedProperties, Results);
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCPropertySynthesizeIvar(
+void SemaCodeCompletion::CodeCompleteObjCPropertySynthesizeIvar(
     Scope *S, IdentifierInfo *PropertyName) {
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
 
   // Figure out where this @synthesize lives.
   ObjCContainerDecl *Container =
-      dyn_cast_or_null<ObjCContainerDecl>(CurContext);
+      dyn_cast_or_null<ObjCContainerDecl>(SemaRef.CurContext);
   if (!Container || (!isa<ObjCImplementationDecl>(Container) &&
                      !isa<ObjCCategoryImplDecl>(Container)))
     return;
@@ -8770,7 +8839,7 @@ void Sema::CodeCompleteObjCPropertySynthesizeIvar(
                 ->getClassInterface();
 
   // Determine the type of the property we're synthesizing.
-  QualType PropertyType = Context.getObjCIdType();
+  QualType PropertyType = getASTContext().getObjCIdType();
   if (Class) {
     if (ObjCPropertyDecl *Property = Class->FindPropertyDeclaration(
             PropertyName, ObjCPropertyQueryKind::OBJC_PR_query_instance)) {
@@ -8794,7 +8863,7 @@ void Sema::CodeCompleteObjCPropertySynthesizeIvar(
     for (ObjCIvarDecl *Ivar = Class->all_declared_ivar_begin(); Ivar;
          Ivar = Ivar->getNextIvar()) {
       Results.AddResult(Result(Ivar, Results.getBasePriority(Ivar), nullptr),
-                        CurContext, nullptr, false);
+                        SemaRef.CurContext, nullptr, false);
 
       // Determine whether we've seen an ivar with a name similar to the
       // property.
@@ -8823,9 +8892,9 @@ void Sema::CodeCompleteObjCPropertySynthesizeIvar(
     CodeCompletionBuilder Builder(Allocator, Results.getCodeCompletionTUInfo(),
                                   Priority, CXAvailability_Available);
 
-    PrintingPolicy Policy = getCompletionPrintingPolicy(*this);
-    Builder.AddResultTypeChunk(
-        GetCompletionTypeString(PropertyType, Context, Policy, Allocator));
+    PrintingPolicy Policy = getCompletionPrintingPolicy(SemaRef);
+    Builder.AddResultTypeChunk(GetCompletionTypeString(
+        PropertyType, getASTContext(), Policy, Allocator));
     Builder.AddTypedTextChunk(Allocator.CopyString(NameWithPrefix));
     Results.AddResult(
         Result(Builder.TakeString(), Priority, CXCursor_ObjCIvarDecl));
@@ -8833,8 +8902,9 @@ void Sema::CodeCompleteObjCPropertySynthesizeIvar(
 
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
 // Mapping from selectors to the methods that implement that selector, along
@@ -9573,15 +9643,15 @@ static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
   }
 }
 
-void Sema::CodeCompleteObjCMethodDecl(Scope *S,
-                                      std::optional<bool> IsInstanceMethod,
-                                      ParsedType ReturnTy) {
+void SemaCodeCompletion::CodeCompleteObjCMethodDecl(
+    Scope *S, std::optional<bool> IsInstanceMethod, ParsedType ReturnTy) {
+  ASTContext &Context = getASTContext();
   // Determine the return type of the method we're declaring, if
   // provided.
-  QualType ReturnType = GetTypeFromParser(ReturnTy);
+  QualType ReturnType = SemaRef.GetTypeFromParser(ReturnTy);
   Decl *IDecl = nullptr;
-  if (CurContext->isObjCContainer()) {
-    ObjCContainerDecl *OCD = dyn_cast<ObjCContainerDecl>(CurContext);
+  if (SemaRef.CurContext->isObjCContainer()) {
+    ObjCContainerDecl *OCD = dyn_cast<ObjCContainerDecl>(SemaRef.CurContext);
     IDecl = OCD;
   }
   // Determine where we should start searching for methods.
@@ -9605,7 +9675,7 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
   }
 
   if (!SearchDecl) {
-    HandleCodeCompleteResults(this, CodeCompleter,
+    HandleCodeCompleteResults(&SemaRef, CodeCompleter,
                               CodeCompletionContext::CCC_Other, nullptr, 0);
     return;
   }
@@ -9617,11 +9687,11 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
 
   // Add declarations or definitions for each of the known methods.
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
-  PrintingPolicy Policy = getCompletionPrintingPolicy(*this);
+  PrintingPolicy Policy = getCompletionPrintingPolicy(SemaRef);
   for (KnownMethodsMap::iterator M = KnownMethods.begin(),
                                  MEnd = KnownMethods.end();
        M != MEnd; ++M) {
@@ -9746,38 +9816,41 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
 
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteObjCMethodDeclSelector(
+void SemaCodeCompletion::CodeCompleteObjCMethodDeclSelector(
     Scope *S, bool IsInstanceMethod, bool AtParameterName, ParsedType ReturnTy,
     ArrayRef<const IdentifierInfo *> SelIdents) {
   // If we have an external source, load the entire class method
   // pool from the AST file.
-  if (ExternalSource) {
-    for (uint32_t I = 0, N = ExternalSource->GetNumExternalSelectors(); I != N;
-         ++I) {
-      Selector Sel = ExternalSource->GetExternalSelector(I);
-      if (Sel.isNull() || ObjC().MethodPool.count(Sel))
+  if (SemaRef.ExternalSource) {
+    for (uint32_t I = 0, N = SemaRef.ExternalSource->GetNumExternalSelectors();
+         I != N; ++I) {
+      Selector Sel = SemaRef.ExternalSource->GetExternalSelector(I);
+      if (Sel.isNull() || SemaRef.ObjC().MethodPool.count(Sel))
         continue;
 
-      ObjC().ReadMethodPool(Sel);
+      SemaRef.ObjC().ReadMethodPool(Sel);
     }
   }
 
   // Build the set of methods we can see.
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
 
   if (ReturnTy)
-    Results.setPreferredType(GetTypeFromParser(ReturnTy).getNonReferenceType());
+    Results.setPreferredType(
+        SemaRef.GetTypeFromParser(ReturnTy).getNonReferenceType());
 
   Results.EnterNewScope();
-  for (SemaObjC::GlobalMethodPool::iterator M = ObjC().MethodPool.begin(),
-                                            MEnd = ObjC().MethodPool.end();
+  for (SemaObjC::GlobalMethodPool::iterator
+           M = SemaRef.ObjC().MethodPool.begin(),
+           MEnd = SemaRef.ObjC().MethodPool.end();
        M != MEnd; ++M) {
     for (ObjCMethodList *MethList = IsInstanceMethod ? &M->second.first
                                                      : &M->second.second;
@@ -9809,7 +9882,7 @@ void Sema::CodeCompleteObjCMethodDeclSelector(
       R.StartParameter = SelIdents.size();
       R.AllParametersAreInformative = false;
       R.DeclaringEntity = true;
-      Results.MaybeAddResult(R, CurContext);
+      Results.MaybeAddResult(R, SemaRef.CurContext);
     }
   }
 
@@ -9817,7 +9890,7 @@ void Sema::CodeCompleteObjCMethodDeclSelector(
 
   if (!AtParameterName && !SelIdents.empty() &&
       SelIdents.front()->getName().starts_with("init")) {
-    for (const auto &M : PP.macros()) {
+    for (const auto &M : SemaRef.PP.macros()) {
       if (M.first->getName() != "NS_DESIGNATED_INITIALIZER")
         continue;
       Results.EnterNewScope();
@@ -9831,12 +9904,13 @@ void Sema::CodeCompleteObjCMethodDeclSelector(
     }
   }
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompletePreprocessorDirective(bool InConditional) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompletePreprocessorDirective(bool InConditional) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_PreprocessorDirective);
   Results.EnterNewScope();
@@ -10001,17 +10075,20 @@ void Sema::CodeCompletePreprocessorDirective(bool InConditional) {
   // FIXME: we don't support #assert or #unassert, so don't suggest them.
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteInPreprocessorConditionalExclusion(Scope *S) {
-  CodeCompleteOrdinaryName(S, S->getFnParent() ? Sema::PCC_RecoveryInFunction
-                                               : Sema::PCC_Namespace);
+void SemaCodeCompletion::CodeCompleteInPreprocessorConditionalExclusion(
+    Scope *S) {
+  CodeCompleteOrdinaryName(S, S->getFnParent()
+                                  ? SemaCodeCompletion::PCC_RecoveryInFunction
+                                  : SemaCodeCompletion::PCC_Namespace);
 }
 
-void Sema::CodeCompletePreprocessorMacroName(bool IsDefinition) {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompletePreprocessorMacroName(bool IsDefinition) {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         IsDefinition ? CodeCompletionContext::CCC_MacroName
                                      : CodeCompletionContext::CCC_MacroNameUse);
@@ -10020,8 +10097,8 @@ void Sema::CodeCompletePreprocessorMacroName(bool IsDefinition) {
     CodeCompletionBuilder Builder(Results.getAllocator(),
                                   Results.getCodeCompletionTUInfo());
     Results.EnterNewScope();
-    for (Preprocessor::macro_iterator M = PP.macro_begin(),
-                                      MEnd = PP.macro_end();
+    for (Preprocessor::macro_iterator M = SemaRef.PP.macro_begin(),
+                                      MEnd = SemaRef.PP.macro_end();
          M != MEnd; ++M) {
       Builder.AddTypedTextChunk(
           Builder.getAllocator().CopyString(M->first->getName()));
@@ -10033,17 +10110,18 @@ void Sema::CodeCompletePreprocessorMacroName(bool IsDefinition) {
     // FIXME: Can we detect when the user just wrote an include guard above?
   }
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompletePreprocessorExpression() {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompletePreprocessorExpression() {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_PreprocessorExpression);
 
   if (CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results, CodeCompleter->loadExternal(), true);
+    AddMacroResults(SemaRef.PP, Results, CodeCompleter->loadExternal(), true);
 
   // defined (<macro>)
   Results.EnterNewScope();
@@ -10057,14 +10135,13 @@ void Sema::CodeCompletePreprocessorExpression() {
   Results.AddResult(Builder.TakeString());
   Results.ExitScope();
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompletePreprocessorMacroArgument(Scope *S,
-                                                 IdentifierInfo *Macro,
-                                                 MacroInfo *MacroInfo,
-                                                 unsigned Argument) {
+void SemaCodeCompletion::CodeCompletePreprocessorMacroArgument(
+    Scope *S, IdentifierInfo *Macro, MacroInfo *MacroInfo, unsigned Argument) {
   // FIXME: In the future, we could provide "overload" results, much like we
   // do for function calls.
 
@@ -10075,7 +10152,8 @@ void Sema::CodeCompletePreprocessorMacroArgument(Scope *S,
 // This handles completion inside an #include filename, e.g. #include <foo/ba
 // We look for the directory "foo" under each directory on the include path,
 // list its files, and reassemble the appropriate #include.
-void Sema::CodeCompleteIncludedFile(llvm::StringRef Dir, bool Angled) {
+void SemaCodeCompletion::CodeCompleteIncludedFile(llvm::StringRef Dir,
+                                                  bool Angled) {
   // RelDir should use /, but unescaped \ is possible on windows!
   // Our completions will normalize to / for simplicity, this case is rare.
   std::string RelDir = llvm::sys::path::convert_to_slash(Dir);
@@ -10083,9 +10161,9 @@ void Sema::CodeCompleteIncludedFile(llvm::StringRef Dir, bool Angled) {
   SmallString<128> NativeRelDir = StringRef(RelDir);
   llvm::sys::path::native(NativeRelDir);
   llvm::vfs::FileSystem &FS =
-      getSourceManager().getFileManager().getVirtualFileSystem();
+      SemaRef.getSourceManager().getFileManager().getVirtualFileSystem();
 
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_IncludedFile);
   llvm::DenseSet<StringRef> SeenResults; // To deduplicate results.
@@ -10197,11 +10275,11 @@ void Sema::CodeCompleteIncludedFile(llvm::StringRef Dir, bool Angled) {
   // Finally with all our helpers, we can scan the include path.
   // Do this in standard order so deduplication keeps the right file.
   // (In case we decide to add more details to the results later).
-  const auto &S = PP.getHeaderSearchInfo();
+  const auto &S = SemaRef.PP.getHeaderSearchInfo();
   using llvm::make_range;
   if (!Angled) {
     // The current directory is on the include path for "quoted" includes.
-    if (auto CurFile = PP.getCurrentFileLexer()->getFileEntry())
+    if (auto CurFile = SemaRef.PP.getCurrentFileLexer()->getFileEntry())
       AddFilesFromIncludeDir(CurFile->getDir().getName(), false,
                              DirectoryLookup::LT_NormalDir);
     for (const auto &D : make_range(S.quoted_dir_begin(), S.quoted_dir_end()))
@@ -10212,18 +10290,19 @@ void Sema::CodeCompleteIncludedFile(llvm::StringRef Dir, bool Angled) {
   for (const auto &D : make_range(S.system_dir_begin(), S.system_dir_end()))
     AddFilesFromDirLookup(D, true);
 
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::CodeCompleteNaturalLanguage() {
-  HandleCodeCompleteResults(this, CodeCompleter,
+void SemaCodeCompletion::CodeCompleteNaturalLanguage() {
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
                             CodeCompletionContext::CCC_NaturalLanguage, nullptr,
                             0);
 }
 
-void Sema::CodeCompleteAvailabilityPlatformName() {
-  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+void SemaCodeCompletion::CodeCompleteAvailabilityPlatformName() {
+  ResultBuilder Results(SemaRef, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
@@ -10234,28 +10313,33 @@ void Sema::CodeCompleteAvailabilityPlatformName() {
         Twine(Platform) + "ApplicationExtension")));
   }
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
-                            Results.data(), Results.size());
+  HandleCodeCompleteResults(&SemaRef, CodeCompleter,
+                            Results.getCompletionContext(), Results.data(),
+                            Results.size());
 }
 
-void Sema::GatherGlobalCodeCompletions(
+void SemaCodeCompletion::GatherGlobalCodeCompletions(
     CodeCompletionAllocator &Allocator, CodeCompletionTUInfo &CCTUInfo,
     SmallVectorImpl<CodeCompletionResult> &Results) {
-  ResultBuilder Builder(*this, Allocator, CCTUInfo,
+  ResultBuilder Builder(SemaRef, Allocator, CCTUInfo,
                         CodeCompletionContext::CCC_Recovery);
   if (!CodeCompleter || CodeCompleter->includeGlobals()) {
-    CodeCompletionDeclConsumer Consumer(Builder,
-                                        Context.getTranslationUnitDecl());
-    LookupVisibleDecls(Context.getTranslationUnitDecl(), LookupAnyName,
-                       Consumer,
-                       !CodeCompleter || CodeCompleter->loadExternal());
+    CodeCompletionDeclConsumer Consumer(
+        Builder, getASTContext().getTranslationUnitDecl());
+    SemaRef.LookupVisibleDecls(getASTContext().getTranslationUnitDecl(),
+                               Sema::LookupAnyName, Consumer,
+                               !CodeCompleter || CodeCompleter->loadExternal());
   }
 
   if (!CodeCompleter || CodeCompleter->includeMacros())
-    AddMacroResults(PP, Builder,
+    AddMacroResults(SemaRef.PP, Builder,
                     !CodeCompleter || CodeCompleter->loadExternal(), true);
 
   Results.clear();
   Results.insert(Results.end(), Builder.data(),
                  Builder.data() + Builder.size());
 }
+
+SemaCodeCompletion::SemaCodeCompletion(Sema &S,
+                                       CodeCompleteConsumer *CompletionConsumer)
+    : SemaBase(S), CodeCompleter(CompletionConsumer) {}
-- 
GitLab


From d74bc823beabbb7067a4b4ae2d69a36d874f5132 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 16 May 2024 19:31:43 -0700
Subject: [PATCH 1151/1206] [lldb] Include SBLanguages in the SWIG bindings
 (#92470)

---
 lldb/bindings/CMakeLists.txt        | 2 ++
 lldb/bindings/headers.swig          | 1 +
 lldb/bindings/interfaces.swig       | 1 +
 lldb/bindings/lua/CMakeLists.txt    | 1 +
 lldb/bindings/python/CMakeLists.txt | 1 +
 5 files changed, 6 insertions(+)

diff --git a/lldb/bindings/CMakeLists.txt b/lldb/bindings/CMakeLists.txt
index 296eae1ae77f..bec694e43bd7 100644
--- a/lldb/bindings/CMakeLists.txt
+++ b/lldb/bindings/CMakeLists.txt
@@ -3,6 +3,7 @@ file(GLOB_RECURSE SWIG_SOURCES *.swig)
 file(GLOB SWIG_HEADERS
   ${LLDB_SOURCE_DIR}/include/lldb/API/*.h
   ${LLDB_SOURCE_DIR}/include/lldb/*.h
+  ${LLDB_BINARY_DIR}/include/lldb/API/SBLanguages.h
 )
 file(GLOB SWIG_PRIVATE_HEADERS
   ${LLDB_SOURCE_DIR}/include/lldb/lldb-private*.h
@@ -30,6 +31,7 @@ set(SWIG_COMMON_FLAGS
   -w361,362,509
   -features autodoc
   -I${LLDB_SOURCE_DIR}/include
+  -I${LLDB_BINARY_DIR}/include
   -I${CMAKE_CURRENT_SOURCE_DIR}
   ${DARWIN_EXTRAS}
 )
diff --git a/lldb/bindings/headers.swig b/lldb/bindings/headers.swig
index e8d0cda28814..ffdc3c31ec88 100644
--- a/lldb/bindings/headers.swig
+++ b/lldb/bindings/headers.swig
@@ -36,6 +36,7 @@
 #include "lldb/API/SBHostOS.h"
 #include "lldb/API/SBInstruction.h"
 #include "lldb/API/SBInstructionList.h"
+#include "lldb/API/SBLanguages.h"
 #include "lldb/API/SBLanguageRuntime.h"
 #include "lldb/API/SBLaunchInfo.h"
 #include "lldb/API/SBLineEntry.h"
diff --git a/lldb/bindings/interfaces.swig b/lldb/bindings/interfaces.swig
index a31a0b4af1eb..2a29a8dd7ef0 100644
--- a/lldb/bindings/interfaces.swig
+++ b/lldb/bindings/interfaces.swig
@@ -114,6 +114,7 @@
 %include "lldb/API/SBHostOS.h"
 %include "lldb/API/SBInstruction.h"
 %include "lldb/API/SBInstructionList.h"
+%include "lldb/API/SBLanguages.h"
 %include "lldb/API/SBLanguageRuntime.h"
 %include "lldb/API/SBLaunchInfo.h"
 %include "lldb/API/SBLineEntry.h"
diff --git a/lldb/bindings/lua/CMakeLists.txt b/lldb/bindings/lua/CMakeLists.txt
index 2d128cc1864c..4a110f7829b0 100644
--- a/lldb/bindings/lua/CMakeLists.txt
+++ b/lldb/bindings/lua/CMakeLists.txt
@@ -3,6 +3,7 @@ add_custom_command(
   DEPENDS ${SWIG_SOURCES}
   DEPENDS ${SWIG_INTERFACES}
   DEPENDS ${SWIG_HEADERS}
+  DEPENDS lldb-sbapi-dwarf-enums
   COMMAND ${SWIG_EXECUTABLE}
       ${SWIG_COMMON_FLAGS}
       -I${CMAKE_CURRENT_SOURCE_DIR}
diff --git a/lldb/bindings/python/CMakeLists.txt b/lldb/bindings/python/CMakeLists.txt
index 73b1239495e2..def6941e802b 100644
--- a/lldb/bindings/python/CMakeLists.txt
+++ b/lldb/bindings/python/CMakeLists.txt
@@ -11,6 +11,7 @@ add_custom_command(
   DEPENDS ${SWIG_SOURCES}
   DEPENDS ${SWIG_INTERFACES}
   DEPENDS ${SWIG_HEADERS}
+  DEPENDS lldb-sbapi-dwarf-enums
   COMMAND ${SWIG_EXECUTABLE}
       ${SWIG_COMMON_FLAGS}
       -I${CMAKE_CURRENT_SOURCE_DIR}
-- 
GitLab


From dc8d70acf7afd6d8c9eef7d240fb8c917eddab50 Mon Sep 17 00:00:00 2001
From: jiajie zhang <56027356+JumpMasterJJ@users.noreply.github.com>
Date: Sat, 18 May 2024 01:00:25 +0800
Subject: [PATCH 1152/1206] [reland][flang] Add ETIME runtime and lowering
 intrinsics implementation (#92571)

This is same as https://github.com/llvm/llvm-project/pull/90578 with an
added fix. This PR updated tests of etime intrinsic due to Lowering
changes for assigning dummy_scope to hlfir.declare. Referring to
https://github.com/llvm/llvm-project/pull/92472 and
https://github.com/llvm/llvm-project/pull/90989
---
 etime-function.mlir                           | 25 +++++
 flang/docs/Intrinsics.md                      | 52 ++++++++++
 .../flang/Optimizer/Builder/IntrinsicCall.h   | 12 ++-
 .../Optimizer/Builder/Runtime/Intrinsics.h    |  2 +
 flang/include/flang/Runtime/time-intrinsic.h  |  3 +
 flang/lib/Evaluate/intrinsics.cpp             | 26 ++++-
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 99 +++++++++++++++++++
 .../Optimizer/Builder/Runtime/Intrinsics.cpp  | 14 +++
 flang/runtime/time-intrinsic.cpp              | 70 ++++++++++++-
 flang/runtime/tools.h                         |  9 ++
 .../test/Lower/Intrinsics/etime-function.f90  | 25 +++++
 flang/test/Lower/Intrinsics/etime.f90         | 22 +++++
 flang/test/Semantics/etime.f90                | 30 ++++++
 13 files changed, 384 insertions(+), 5 deletions(-)
 create mode 100644 etime-function.mlir
 create mode 100644 flang/test/Lower/Intrinsics/etime-function.f90
 create mode 100644 flang/test/Lower/Intrinsics/etime.f90
 create mode 100644 flang/test/Semantics/etime.f90

diff --git a/etime-function.mlir b/etime-function.mlir
new file mode 100644
index 000000000000..740dfd4866aa
--- /dev/null
+++ b/etime-function.mlir
@@ -0,0 +1,25 @@
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+  func.func @_QPetime_test(%arg0: !fir.ref<!fir.array<2xf32>> {fir.bindc_name = "values"}, %arg1: !fir.ref<f32> {fir.bindc_name = "time"}) {
+    %c9_i32 = arith.constant 9 : i32
+    %c2 = arith.constant 2 : index
+    %0 = fir.alloca f32
+    %1 = fir.declare %arg1 {uniq_name = "_QFetime_testEtime"} : (!fir.ref<f32>) -> !fir.ref<f32>
+    %2 = fir.shape %c2 : (index) -> !fir.shape<1>
+    %3 = fir.declare %arg0(%2) {uniq_name = "_QFetime_testEvalues"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<2xf32>>
+    %4 = fir.embox %3(%2) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
+    %5 = fir.embox %0 : (!fir.ref<f32>) -> !fir.box<f32>
+    %6 = fir.address_of(@_QQclX116781708dcf8f012d7ec1e40d743d97) : !fir.ref<!fir.char<1,71>>
+    %7 = fir.convert %4 : (!fir.box<!fir.array<2xf32>>) -> !fir.box<none>
+    %8 = fir.convert %5 : (!fir.box<f32>) -> !fir.box<none>
+    %9 = fir.convert %6 : (!fir.ref<!fir.char<1,71>>) -> !fir.ref<i8>
+    %10 = fir.call @_FortranAEtime(%7, %8, %9, %c9_i32) fastmath<contract> : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+    %11 = fir.load %0 : !fir.ref<f32>
+    fir.store %11 to %1 : !fir.ref<f32>
+    return
+  }
+  func.func private @_FortranAEtime(!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none attributes {fir.runtime}
+  fir.global linkonce @_QQclX116781708dcf8f012d7ec1e40d743d97 constant : !fir.char<1,71> {
+    %0 = fir.string_lit "/home/jump/llvm-project/flang/test/Lower/Intrinsics/etime-function.f90\00"(71) : !fir.char<1,71>
+    fir.has_value %0 : !fir.char<1,71>
+  }
+}
diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index 848619cb65d9..41129b10083b 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -916,3 +916,55 @@ used in constant expressions have currently no folding support at all.
   - If a condition occurs that would assign a nonzero value to `CMDSTAT` but the `CMDSTAT` variable is not present, error termination is initiated.
     - On POSIX-compatible systems, the child process (async process) will be terminated with no effect on the parent process (continues).
     - On Windows, error termination is not initiated.
+
+### Non-Standard Intrinsics: ETIME
+
+#### Description
+`ETIME(VALUES, TIME)` returns the number of seconds of runtime since the start of the process’s execution in *TIME*. *VALUES* returns the user and system components of this time in `VALUES(1)` and `VALUES(2)` respectively. *TIME* is equal to `VALUES(1) + VALUES(2)`.
+
+On some systems, the underlying timings are represented using types with sufficiently small limits that overflows (wrap around) are possible, such as 32-bit types. Therefore, the values returned by this intrinsic might be, or become, negative, or numerically less than previous values, during a single run of the compiled program.
+
+This intrinsic is provided in both subroutine and function forms; however, only one form can be used in any given program unit.
+
+*VALUES* and *TIME* are `INTENT(OUT)` and provide the following:
+
+
+|               |                                   |
+|---------------|-----------------------------------|
+| `VALUES(1)`   | User time in seconds.             |
+| `VALUES(2)`   | System time in seconds.           |
+| `TIME`        | Run time since start in seconds.  |
+
+#### Usage and Info
+
+- **Standard:** GNU extension
+- **Class:** Subroutine, function
+- **Syntax:** `CALL ETIME(VALUES, TIME)`
+- **Arguments:**
+- **Return value** Elapsed time in seconds since the start of program execution.
+
+| Argument   | Description                                                           |
+|------------|-----------------------------------------------------------------------|
+| `VALUES`   | The type shall be REAL(4), DIMENSION(2).                              |
+| `TIME`     | The type shall be REAL(4).                                            |
+
+#### Example
+Here is an example usage from [Gfortran ETIME](https://gcc.gnu.org/onlinedocs/gfortran/ETIME.html)
+```Fortran
+program test_etime
+    integer(8) :: i, j
+    real, dimension(2) :: tarray
+    real :: result
+    call ETIME(tarray, result)
+    print *, result
+    print *, tarray(1)
+    print *, tarray(2)   
+    do i=1,100000000    ! Just a delay
+        j = i * i - i
+    end do
+    call ETIME(tarray, result)
+    print *, result
+    print *, tarray(1)
+    print *, tarray(2)
+end program test_etime
+```
\ No newline at end of file
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index b7d060926761..977a69af5281 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -222,6 +222,8 @@ struct IntrinsicLibrary {
   fir::ExtendedValue genEoshift(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   void genExit(llvm::ArrayRef<fir::ExtendedValue>);
   void genExecuteCommandLine(mlir::ArrayRef<fir::ExtendedValue> args);
+  fir::ExtendedValue genEtime(std::optional<mlir::Type>,
+                              mlir::ArrayRef<fir::ExtendedValue> args);
   mlir::Value genExponent(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genExtendsTypeOf(mlir::Type,
                                       llvm::ArrayRef<fir::ExtendedValue>);
@@ -400,8 +402,10 @@ struct IntrinsicLibrary {
   using ElementalGenerator = decltype(&IntrinsicLibrary::genAbs);
   using ExtendedGenerator = decltype(&IntrinsicLibrary::genLenTrim);
   using SubroutineGenerator = decltype(&IntrinsicLibrary::genDateAndTime);
-  using Generator =
-      std::variant<ElementalGenerator, ExtendedGenerator, SubroutineGenerator>;
+  /// The generator for intrinsic that has both function and subroutine form.
+  using DualGenerator = decltype(&IntrinsicLibrary::genEtime);
+  using Generator = std::variant<ElementalGenerator, ExtendedGenerator,
+                                 SubroutineGenerator, DualGenerator>;
 
   /// All generators can be outlined. This will build a function named
   /// "fir."+ <generic name> + "." + <result type code> and generate the
@@ -442,6 +446,10 @@ struct IntrinsicLibrary {
                               llvm::ArrayRef<mlir::Value> args);
   mlir::Value invokeGenerator(SubroutineGenerator generator,
                               llvm::ArrayRef<mlir::Value> args);
+  mlir::Value invokeGenerator(DualGenerator generator,
+                              llvm::ArrayRef<mlir::Value> args);
+  mlir::Value invokeGenerator(DualGenerator generator, mlir::Type resultType,
+                              llvm::ArrayRef<mlir::Value> args);
 
   /// Get pointer to unrestricted intrinsic. Generate the related unrestricted
   /// intrinsic if it is not defined yet.
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
index 737c631e45c1..7497a4bc3564 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
@@ -44,6 +44,8 @@ void genDateAndTime(fir::FirOpBuilder &, mlir::Location,
                     std::optional<fir::CharBoxValue> date,
                     std::optional<fir::CharBoxValue> time,
                     std::optional<fir::CharBoxValue> zone, mlir::Value values);
+void genEtime(fir::FirOpBuilder &builder, mlir::Location loc,
+              mlir::Value values, mlir::Value time);
 
 void genRandomInit(fir::FirOpBuilder &, mlir::Location, mlir::Value repeatable,
                    mlir::Value imageDistinct);
diff --git a/flang/include/flang/Runtime/time-intrinsic.h b/flang/include/flang/Runtime/time-intrinsic.h
index 650c02436ee4..80490a17e455 100644
--- a/flang/include/flang/Runtime/time-intrinsic.h
+++ b/flang/include/flang/Runtime/time-intrinsic.h
@@ -43,6 +43,9 @@ void RTNAME(DateAndTime)(char *date, std::size_t dateChars, char *time,
     const char *source = nullptr, int line = 0,
     const Descriptor *values = nullptr);
 
+void RTNAME(Etime)(const Descriptor *values, const Descriptor *time,
+    const char *sourceFile, int line);
+
 } // extern "C"
 } // namespace Fortran::runtime
 #endif // FORTRAN_RUNTIME_TIME_INTRINSIC_H_
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 441a762c930d..ded277877f49 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -454,6 +454,10 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"erf", {{"x", SameReal}}, SameReal},
     {"erfc", {{"x", SameReal}}, SameReal},
     {"erfc_scaled", {{"x", SameReal}}, SameReal},
+    {"etime",
+        {{"values", TypePattern{RealType, KindCode::exactKind, 4}, Rank::vector,
+            Optionality::required, common::Intent::Out}},
+        TypePattern{RealType, KindCode::exactKind, 4}},
     {"exp", {{"x", SameFloating}}, SameFloating},
     {"exp", {{"x", SameFloating}}, SameFloating},
     {"exponent", {{"x", AnyReal}}, DefaultInt},
@@ -1342,6 +1346,12 @@ static const IntrinsicInterface intrinsicSubroutine[]{
             {"values", AnyInt, Rank::vector, Optionality::optional,
                 common::Intent::Out}},
         {}, Rank::elemental, IntrinsicClass::impureSubroutine},
+    {"etime",
+        {{"values", TypePattern{RealType, KindCode::exactKind, 4}, Rank::vector,
+             Optionality::required, common::Intent::Out},
+            {"time", TypePattern{RealType, KindCode::exactKind, 4},
+                Rank::scalar, Optionality::required, common::Intent::Out}},
+        {}, Rank::elemental, IntrinsicClass::impureSubroutine},
     {"execute_command_line",
         {{"command", DefaultChar, Rank::scalar},
             {"wait", AnyLogical, Rank::scalar, Optionality::optional},
@@ -2484,6 +2494,7 @@ public:
   bool IsIntrinsic(const std::string &) const;
   bool IsIntrinsicFunction(const std::string &) const;
   bool IsIntrinsicSubroutine(const std::string &) const;
+  bool IsDualIntrinsic(const std::string &) const;
 
   IntrinsicClass GetIntrinsicClass(const std::string &) const;
   std::string GetGenericIntrinsicName(const std::string &) const;
@@ -2545,6 +2556,17 @@ bool IntrinsicProcTable::Implementation::IsIntrinsic(
     const std::string &name) const {
   return IsIntrinsicFunction(name) || IsIntrinsicSubroutine(name);
 }
+bool IntrinsicProcTable::Implementation::IsDualIntrinsic(
+    const std::string &name) const {
+  // Collection for some intrinsics with function and subroutine form,
+  // in order to pass the semantic check.
+  static const std::string dualIntrinsic[]{{"etime"}};
+
+  return std::find_if(std::begin(dualIntrinsic), std::end(dualIntrinsic),
+             [&name](const std::string &dualName) {
+               return dualName == name;
+             }) != std::end(dualIntrinsic);
+}
 
 IntrinsicClass IntrinsicProcTable::Implementation::GetIntrinsicClass(
     const std::string &name) const {
@@ -3083,7 +3105,7 @@ std::optional<SpecificCall> IntrinsicProcTable::Implementation::Probe(
         return specificCall;
       }
     }
-    if (IsIntrinsicFunction(call.name)) {
+    if (IsIntrinsicFunction(call.name) && !IsDualIntrinsic(call.name)) {
       context.messages().Say(
           "Cannot use intrinsic function '%s' as a subroutine"_err_en_US,
           call.name);
@@ -3218,7 +3240,7 @@ std::optional<SpecificCall> IntrinsicProcTable::Implementation::Probe(
   }
 
   if (specificBuffer.empty() && genericBuffer.empty() &&
-      IsIntrinsicSubroutine(call.name)) {
+      IsIntrinsicSubroutine(call.name) && !IsDualIntrinsic(call.name)) {
     context.messages().Say(
         "Cannot use intrinsic subroutine '%s' as a function"_err_en_US,
         call.name);
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 58064d23eb08..ae7e65098744 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -35,6 +35,7 @@
 #include "flang/Optimizer/Builder/Runtime/Stop.h"
 #include "flang/Optimizer/Builder/Runtime/Transformational.h"
 #include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Support/FatalError.h"
@@ -49,6 +50,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include <mlir/IR/Value.h>
 #include <optional>
 
 #define DEBUG_TYPE "flang-lower-intrinsic"
@@ -222,6 +224,10 @@ static constexpr IntrinsicHandler handlers[]{
        {"boundary", asBox, handleDynamicOptional},
        {"dim", asValue}}},
      /*isElemental=*/false},
+    {"etime",
+     &I::genEtime,
+     {{{"values", asBox}, {"time", asBox}}},
+     /*isElemental=*/false},
     {"execute_command_line",
      &I::genExecuteCommandLine,
      {{{"command", asBox},
@@ -1682,6 +1688,24 @@ IntrinsicLibrary::genElementalCall<IntrinsicLibrary::SubroutineGenerator>(
   return mlir::Value();
 }
 
+template <>
+fir::ExtendedValue
+IntrinsicLibrary::genElementalCall<IntrinsicLibrary::DualGenerator>(
+    DualGenerator generator, llvm::StringRef name, mlir::Type resultType,
+    llvm::ArrayRef<fir::ExtendedValue> args, bool outline) {
+  assert(resultType.getImpl() && "expect elemental intrinsic to be functions");
+
+  for (const fir::ExtendedValue &arg : args)
+    if (!arg.getUnboxed() && !arg.getCharBox())
+      // fir::emitFatalError(loc, "nonscalar intrinsic argument");
+      crashOnMissingIntrinsic(loc, name);
+  if (outline)
+    return outlineInExtendedWrapper(generator, name, resultType, args);
+
+  return std::invoke(generator, *this, std::optional<mlir::Type>{resultType},
+                     args);
+}
+
 static fir::ExtendedValue
 invokeHandler(IntrinsicLibrary::ElementalGenerator generator,
               const IntrinsicHandler &handler,
@@ -1725,6 +1749,22 @@ invokeHandler(IntrinsicLibrary::SubroutineGenerator generator,
   return mlir::Value{};
 }
 
+static fir::ExtendedValue
+invokeHandler(IntrinsicLibrary::DualGenerator generator,
+              const IntrinsicHandler &handler,
+              std::optional<mlir::Type> resultType,
+              llvm::ArrayRef<fir::ExtendedValue> args, bool outline,
+              IntrinsicLibrary &lib) {
+  if (handler.isElemental)
+    return lib.genElementalCall(generator, handler.name, mlir::Type{}, args,
+                                outline);
+  if (outline)
+    return lib.outlineInExtendedWrapper(generator, handler.name, resultType,
+                                        args);
+
+  return std::invoke(generator, lib, resultType, args);
+}
+
 std::pair<fir::ExtendedValue, bool>
 IntrinsicLibrary::genIntrinsicCall(llvm::StringRef specificName,
                                    std::optional<mlir::Type> resultType,
@@ -1820,6 +1860,34 @@ IntrinsicLibrary::invokeGenerator(SubroutineGenerator generator,
   return {};
 }
 
+mlir::Value
+IntrinsicLibrary::invokeGenerator(DualGenerator generator,
+                                  llvm::ArrayRef<mlir::Value> args) {
+  llvm::SmallVector<fir::ExtendedValue> extendedArgs;
+  for (mlir::Value arg : args)
+    extendedArgs.emplace_back(toExtendedValue(arg, builder, loc));
+  std::invoke(generator, *this, std::optional<mlir::Type>{}, extendedArgs);
+  return {};
+}
+
+mlir::Value
+IntrinsicLibrary::invokeGenerator(DualGenerator generator,
+                                  mlir::Type resultType,
+                                  llvm::ArrayRef<mlir::Value> args) {
+  llvm::SmallVector<fir::ExtendedValue> extendedArgs;
+  for (mlir::Value arg : args)
+    extendedArgs.emplace_back(toExtendedValue(arg, builder, loc));
+
+  if (resultType.getImpl() == nullptr) {
+    // TODO:
+    assert(false && "result type is null");
+  }
+
+  auto extendedResult = std::invoke(
+      generator, *this, std::optional<mlir::Type>{resultType}, extendedArgs);
+  return toValue(extendedResult, builder, loc);
+}
+
 //===----------------------------------------------------------------------===//
 // Intrinsic Procedure Mangling
 //===----------------------------------------------------------------------===//
@@ -3235,6 +3303,37 @@ void IntrinsicLibrary::genExecuteCommandLine(
                                       exitstatBox, cmdstatBox, cmdmsgBox);
 }
 
+// ETIME
+fir::ExtendedValue
+IntrinsicLibrary::genEtime(std::optional<mlir::Type> resultType,
+                           llvm::ArrayRef<fir::ExtendedValue> args) {
+  assert((args.size() == 2 && !resultType.has_value()) ||
+         (args.size() == 1 && resultType.has_value()));
+
+  mlir::Value values = fir::getBase(args[0]);
+  if (resultType.has_value()) {
+    // function form
+    if (!values)
+      fir::emitFatalError(loc, "expected VALUES parameter");
+
+    auto timeAddr = builder.createTemporary(loc, *resultType);
+    auto timeBox = builder.createBox(loc, timeAddr);
+    fir::runtime::genEtime(builder, loc, values, timeBox);
+    return builder.create<fir::LoadOp>(loc, timeAddr);
+  } else {
+    // subroutine form
+    mlir::Value time = fir::getBase(args[1]);
+    if (!values)
+      fir::emitFatalError(loc, "expected VALUES parameter");
+    if (!time)
+      fir::emitFatalError(loc, "expected TIME parameter");
+
+    fir::runtime::genEtime(builder, loc, values, time);
+    return {};
+  }
+  return {};
+}
+
 // EXIT
 void IntrinsicLibrary::genExit(llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 1);
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 8b78a1688c73..3f36d639861b 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -106,6 +106,20 @@ void fir::runtime::genDateAndTime(fir::FirOpBuilder &builder,
   builder.create<fir::CallOp>(loc, callee, args);
 }
 
+void fir::runtime::genEtime(fir::FirOpBuilder &builder, mlir::Location loc,
+                            mlir::Value values, mlir::Value time) {
+  auto runtimeFunc = fir::runtime::getRuntimeFunc<mkRTKey(Etime)>(loc, builder);
+  mlir::FunctionType runtimeFuncTy = runtimeFunc.getFunctionType();
+
+  mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
+  mlir::Value sourceLine =
+      fir::factory::locationToLineNo(builder, loc, runtimeFuncTy.getInput(3));
+
+  llvm::SmallVector<mlir::Value> args = fir::runtime::createArguments(
+      builder, loc, runtimeFuncTy, values, time, sourceFile, sourceLine);
+  builder.create<fir::CallOp>(loc, runtimeFunc, args);
+}
+
 void fir::runtime::genRandomInit(fir::FirOpBuilder &builder, mlir::Location loc,
                                  mlir::Value repeatable,
                                  mlir::Value imageDistinct) {
diff --git a/flang/runtime/time-intrinsic.cpp b/flang/runtime/time-intrinsic.cpp
index 68d63253139f..989d4f804c5f 100644
--- a/flang/runtime/time-intrinsic.cpp
+++ b/flang/runtime/time-intrinsic.cpp
@@ -19,8 +19,12 @@
 #include <cstdlib>
 #include <cstring>
 #include <ctime>
-#ifndef _WIN32
+#ifdef _WIN32
+#include "flang/Common/windows-include.h"
+#else
 #include <sys/time.h> // gettimeofday
+#include <sys/times.h>
+#include <unistd.h>
 #endif
 
 // CPU_TIME (Fortran 2018 16.9.57)
@@ -370,5 +374,69 @@ void RTNAME(DateAndTime)(char *date, std::size_t dateChars, char *time,
       terminator, date, dateChars, time, timeChars, zone, zoneChars, values);
 }
 
+void RTNAME(Etime)(const Descriptor *values, const Descriptor *time,
+    const char *sourceFile, int line) {
+  Fortran::runtime::Terminator terminator{sourceFile, line};
+
+  double usrTime = -1.0, sysTime = -1.0, realTime = -1.0;
+
+#ifdef _WIN32
+  FILETIME creationTime;
+  FILETIME exitTime;
+  FILETIME kernelTime;
+  FILETIME userTime;
+
+  if (GetProcessTimes(GetCurrentProcess(), &creationTime, &exitTime,
+          &kernelTime, &userTime) == 0) {
+    ULARGE_INTEGER userSystemTime;
+    ULARGE_INTEGER kernelSystemTime;
+
+    memcpy(&userSystemTime, &userTime, sizeof(FILETIME));
+    memcpy(&kernelSystemTime, &kernelTime, sizeof(FILETIME));
+
+    usrTime = ((double)(userSystemTime.QuadPart)) / 10000000.0;
+    sysTime = ((double)(kernelSystemTime.QuadPart)) / 10000000.0;
+    realTime = usrTime + sysTime;
+  }
+#else
+  struct tms tms;
+  if (times(&tms) != -1) {
+    usrTime = ((double)(tms.tms_utime)) / sysconf(_SC_CLK_TCK);
+    sysTime = ((double)(tms.tms_stime)) / sysconf(_SC_CLK_TCK);
+    realTime = usrTime + sysTime;
+  }
+#endif
+
+  if (values) {
+    auto typeCode{values->type().GetCategoryAndKind()};
+    // ETIME values argument must have decimal range == 2.
+    RUNTIME_CHECK(terminator,
+        values->rank() == 1 && values->GetDimension(0).Extent() == 2 &&
+            typeCode && typeCode->first == Fortran::common::TypeCategory::Real);
+    // Only accept KIND=4 here.
+    int kind{typeCode->second};
+    RUNTIME_CHECK(terminator, kind == 4);
+
+    ApplyFloatingPointKind<StoreFloatingPointAt, void>(
+        kind, terminator, *values, /* atIndex = */ 0, usrTime);
+    ApplyFloatingPointKind<StoreFloatingPointAt, void>(
+        kind, terminator, *values, /* atIndex = */ 1, sysTime);
+  }
+
+  if (time) {
+    auto typeCode{time->type().GetCategoryAndKind()};
+    // ETIME time argument must have decimal range == 0.
+    RUNTIME_CHECK(terminator,
+        time->rank() == 0 && typeCode &&
+            typeCode->first == Fortran::common::TypeCategory::Real);
+    // Only accept KIND=4 here.
+    int kind{typeCode->second};
+    RUNTIME_CHECK(terminator, kind == 4);
+
+    ApplyFloatingPointKind<StoreFloatingPointAt, void>(
+        kind, terminator, *time, /* atIndex = */ 0, realTime);
+  }
+}
+
 } // extern "C"
 } // namespace Fortran::runtime
diff --git a/flang/runtime/tools.h b/flang/runtime/tools.h
index 52049c511f13..dc12e5c4533e 100644
--- a/flang/runtime/tools.h
+++ b/flang/runtime/tools.h
@@ -99,6 +99,15 @@ template <int KIND> struct StoreIntegerAt {
   }
 };
 
+// Helper to store floating value in result[at].
+template <int KIND> struct StoreFloatingPointAt {
+  RT_API_ATTRS void operator()(const Fortran::runtime::Descriptor &result,
+      std::size_t at, std::double_t value) const {
+    *result.ZeroBasedIndexedElement<Fortran::runtime::CppTypeFor<
+        Fortran::common::TypeCategory::Real, KIND>>(at) = value;
+  }
+};
+
 // Validate a KIND= argument
 RT_API_ATTRS void CheckIntegerKind(
     Terminator &, int kind, const char *intrinsic);
diff --git a/flang/test/Lower/Intrinsics/etime-function.f90 b/flang/test/Lower/Intrinsics/etime-function.f90
new file mode 100644
index 000000000000..e3279189c752
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/etime-function.f90
@@ -0,0 +1,25 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+
+! CHECK-LABEL: func.func @_QPetime_test(
+! CHECK-SAME: %[[valuesArg:.*]]: !fir.ref<!fir.array<2xf32>> {fir.bindc_name = "values"},
+! CHECK-SAME: %[[timeArg:.*]]: !fir.ref<f32> {fir.bindc_name = "time"}) {
+subroutine etime_test(values, time)
+  REAL(4), DIMENSION(2) :: values
+  REAL(4) :: time
+  time = etime(values)
+  ! CHECK-NEXT:        %[[c9:.*]] = arith.constant 9 : i32
+  ! CHECK-NEXT:        %[[c2:.*]] = arith.constant 2 : index
+  ! CHECK-NEXT:        %[[timeTmpAddr:.*]] = fir.alloca f32
+  ! CHECK-NEXT:        %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+  ! CHECK-NEXT:        %[[timeDeclare:.*]] = fir.declare %[[timeArg]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFetime_testEtime"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
+  ! CHECK-NEXT:        %[[shape:.*]] = fir.shape %[[c2]] : (index) -> !fir.shape<1>
+  ! CHECK-NEXT:        %[[valuesDeclare:.*]] = fir.declare %[[valuesArg]](%[[shape]]) dummy_scope %[[DSCOPE]] {uniq_name = "_QFetime_testEvalues"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<2xf32>>
+  ! CHECK-NEXT:        %[[valuesBox:.*]] = fir.embox %[[valuesDeclare]](%[[shape]]) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
+  ! CHECK-NEXT:        %[[timeTmpBox:.*]] = fir.embox %[[timeTmpAddr]] : (!fir.ref<f32>) -> !fir.box<f32>
+  ! CHECK:             %[[values:.*]] = fir.convert %[[valuesBox]] : (!fir.box<!fir.array<2xf32>>) -> !fir.box<none>
+  ! CHECK:             %[[timeTmp:.*]] = fir.convert %[[timeTmpBox]] : (!fir.box<f32>) -> !fir.box<none>
+  ! CHECK:             %[[VAL_9:.*]] = fir.call @_FortranAEtime(%[[values]], %[[timeTmp]], %[[VAL_7:.*]], %[[c9]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+  ! CHECK-NEXT:        %[[timeValue:.*]] = fir.load %[[timeTmpAddr]] : !fir.ref<f32>
+  ! CHECK-NEXT:        fir.store %[[timeValue]] to %[[timeDeclare]] : !fir.ref<f32>
+  ! CHECK-NEXT:        return
+end subroutine etime_test
\ No newline at end of file
diff --git a/flang/test/Lower/Intrinsics/etime.f90 b/flang/test/Lower/Intrinsics/etime.f90
new file mode 100644
index 000000000000..3e7ae0e9a406
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/etime.f90
@@ -0,0 +1,22 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+
+! CHECK-LABEL: func.func @_QPetime_test(
+! CHECK-SAME: %[[valuesArg:.*]]: !fir.ref<!fir.array<2xf32>> {fir.bindc_name = "values"},
+! CHECK-SAME: %[[timeArg:.*]]: !fir.ref<f32> {fir.bindc_name = "time"}) {
+subroutine etime_test(values, time)
+  REAL(4), DIMENSION(2) :: values
+  REAL(4) :: time
+  call etime(values, time)
+  ! CHECK-NEXT:        %[[c9:.*]] = arith.constant 9 : i32
+  ! CHECK-NEXT:        %[[c2:.*]] = arith.constant 2 : index
+  ! CHECK-NEXT:        %[[DSCOPE:.*]] = fir.dummy_scope : !fir.dscope
+  ! CHECK-NEXT:        %[[timeDeclare:.*]] = fir.declare %[[timeArg]] dummy_scope %[[DSCOPE]] {uniq_name = "_QFetime_testEtime"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
+  ! CHECK-NEXT:        %[[shape:.*]] = fir.shape %[[c2]] : (index) -> !fir.shape<1>
+  ! CHECK-NEXT:        %[[valuesDeclare:.*]] = fir.declare %[[valuesArg]](%[[shape]]) dummy_scope %[[DSCOPE]] {uniq_name = "_QFetime_testEvalues"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>, !fir.dscope) -> !fir.ref<!fir.array<2xf32>>
+  ! CHECK-NEXT:        %[[valuesBox:.*]] = fir.embox %[[valuesDeclare]](%[[shape]]) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
+  ! CHECK-NEXT:        %[[timeBox:.*]] = fir.embox %[[timeDeclare]] : (!fir.ref<f32>) -> !fir.box<f32>
+  ! CHECK:             %[[values:.*]] = fir.convert %[[valuesBox]] : (!fir.box<!fir.array<2xf32>>) -> !fir.box<none>
+  ! CHECK:             %[[time:.*]] = fir.convert %[[timeBox]] : (!fir.box<f32>) -> !fir.box<none>
+  ! CHECK:             %[[VAL_9:.*]] = fir.call @_FortranAEtime(%[[values]], %[[time]], %[[VAL_7:.*]], %[[c9]]) fastmath<contract> : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+  ! CHECK-NEXT:        return
+end subroutine etime_test
\ No newline at end of file
diff --git a/flang/test/Semantics/etime.f90 b/flang/test/Semantics/etime.f90
new file mode 100644
index 000000000000..28735c2a7aac
--- /dev/null
+++ b/flang/test/Semantics/etime.f90
@@ -0,0 +1,30 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
+! Tests for the ETIME intrinsics
+
+subroutine bad_kind_error(values, time)
+  REAL(KIND=8), DIMENSION(2) :: values
+  REAL(KIND=8) :: time
+  !ERROR: Actual argument for 'values=' has bad type or kind 'REAL(8)'
+  call etime(values, time)
+end subroutine bad_kind_error
+  
+subroutine bad_args_error(values)
+  REAL(KIND=4), DIMENSION(2) :: values
+  !ERROR: missing mandatory 'time=' argument
+  call etime(values)
+end subroutine bad_args_error
+
+subroutine bad_apply_form(values)
+  REAL(KIND=4), DIMENSION(2) :: values
+  REAL(KIND=4) :: time
+  !Declaration of 'etime'
+  call etime(values, time)
+  !ERROR: Cannot call subroutine 'etime' like a function
+  time = etime(values)
+end subroutine bad_apply_form
+
+subroutine good_kind_equal(values, time)
+  REAL(KIND=4), DIMENSION(2) :: values
+  REAL(KIND=4) :: time
+  call etime(values, time)
+end subroutine good_kind_equal
\ No newline at end of file
-- 
GitLab


From a4ad05284e97dd188c44252846486cbfb74a884c Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Fri, 17 May 2024 10:11:29 -0700
Subject: [PATCH 1153/1206] [lldb-dap] Replace `assertEquals` with
 `assertEqual` (NFC)

Fixes new test that were added or modified after #82073. Also fixes a
formatting issue.
---
 .../TestDAP_setDataBreakpoints.py             | 62 +++++++++----------
 .../lldb-dap/variables/TestDAP_variables.py   | 44 +++++++------
 2 files changed, 57 insertions(+), 49 deletions(-)

diff --git a/lldb/test/API/tools/lldb-dap/databreakpoint/TestDAP_setDataBreakpoints.py b/lldb/test/API/tools/lldb-dap/databreakpoint/TestDAP_setDataBreakpoints.py
index 52c0bbfb33da..1e0e40d4a013 100644
--- a/lldb/test/API/tools/lldb-dap/databreakpoint/TestDAP_setDataBreakpoints.py
+++ b/lldb/test/API/tools/lldb-dap/databreakpoint/TestDAP_setDataBreakpoints.py
@@ -27,10 +27,10 @@ class TestDAP_setDataBreakpoints(lldbdap_testcase.DAPTestCaseBase):
         response_x = self.dap_server.request_dataBreakpointInfo(0, "&x")
         response_arr_2 = self.dap_server.request_dataBreakpointInfo(0, "arr+2")
         # Test response from dataBreakpointInfo request.
-        self.assertEquals(response_x["body"]["dataId"].split("/")[1], "4")
-        self.assertEquals(response_x["body"]["accessTypes"], self.accessTypes)
-        self.assertEquals(response_arr_2["body"]["dataId"].split("/")[1], "4")
-        self.assertEquals(response_arr_2["body"]["accessTypes"], self.accessTypes)
+        self.assertEqual(response_x["body"]["dataId"].split("/")[1], "4")
+        self.assertEqual(response_x["body"]["accessTypes"], self.accessTypes)
+        self.assertEqual(response_arr_2["body"]["dataId"].split("/")[1], "4")
+        self.assertEqual(response_arr_2["body"]["accessTypes"], self.accessTypes)
         # The first one should be overwritten by the third one as they start at
         # the same address. This is indicated by returning {verified: False} for
         # the first one.
@@ -40,7 +40,7 @@ class TestDAP_setDataBreakpoints(lldbdap_testcase.DAPTestCaseBase):
             {"dataId": response_x["body"]["dataId"], "accessType": "write"},
         ]
         set_response = self.dap_server.request_setDataBreakpoint(dataBreakpoints)
-        self.assertEquals(
+        self.assertEqual(
             set_response["body"]["breakpoints"],
             [{"verified": False}, {"verified": True}, {"verified": True}],
         )
@@ -48,14 +48,14 @@ class TestDAP_setDataBreakpoints(lldbdap_testcase.DAPTestCaseBase):
         self.continue_to_next_stop()
         x_val = self.dap_server.get_local_variable_value("x")
         i_val = self.dap_server.get_local_variable_value("i")
-        self.assertEquals(x_val, "2")
-        self.assertEquals(i_val, "1")
+        self.assertEqual(x_val, "2")
+        self.assertEqual(i_val, "1")
 
         self.continue_to_next_stop()
         arr_2 = self.dap_server.get_local_variable_child("arr", "[2]")
         i_val = self.dap_server.get_local_variable_value("i")
-        self.assertEquals(arr_2["value"], "42")
-        self.assertEquals(i_val, "2")
+        self.assertEqual(arr_2["value"], "42")
+        self.assertEqual(i_val, "2")
 
     @skipIfWindows
     @skipIfRemote
@@ -72,16 +72,16 @@ class TestDAP_setDataBreakpoints(lldbdap_testcase.DAPTestCaseBase):
         response_x = self.dap_server.request_dataBreakpointInfo(0, "&x")
         response_arr_2 = self.dap_server.request_dataBreakpointInfo(0, "arr+2")
         # Test response from dataBreakpointInfo request.
-        self.assertEquals(response_x["body"]["dataId"].split("/")[1], "4")
-        self.assertEquals(response_x["body"]["accessTypes"], self.accessTypes)
-        self.assertEquals(response_arr_2["body"]["dataId"].split("/")[1], "4")
-        self.assertEquals(response_arr_2["body"]["accessTypes"], self.accessTypes)
+        self.assertEqual(response_x["body"]["dataId"].split("/")[1], "4")
+        self.assertEqual(response_x["body"]["accessTypes"], self.accessTypes)
+        self.assertEqual(response_arr_2["body"]["dataId"].split("/")[1], "4")
+        self.assertEqual(response_arr_2["body"]["accessTypes"], self.accessTypes)
         dataBreakpoints = [
             {"dataId": response_x["body"]["dataId"], "accessType": "write"},
             {"dataId": response_arr_2["body"]["dataId"], "accessType": "write"},
         ]
         set_response = self.dap_server.request_setDataBreakpoint(dataBreakpoints)
-        self.assertEquals(
+        self.assertEqual(
             set_response["body"]["breakpoints"],
             [{"verified": True}, {"verified": True}],
         )
@@ -89,14 +89,14 @@ class TestDAP_setDataBreakpoints(lldbdap_testcase.DAPTestCaseBase):
         self.continue_to_next_stop()
         x_val = self.dap_server.get_local_variable_value("x")
         i_val = self.dap_server.get_local_variable_value("i")
-        self.assertEquals(x_val, "2")
-        self.assertEquals(i_val, "1")
+        self.assertEqual(x_val, "2")
+        self.assertEqual(i_val, "1")
 
         self.continue_to_next_stop()
         arr_2 = self.dap_server.get_local_variable_child("arr", "[2]")
         i_val = self.dap_server.get_local_variable_value("i")
-        self.assertEquals(arr_2["value"], "42")
-        self.assertEquals(i_val, "2")
+        self.assertEqual(arr_2["value"], "42")
+        self.assertEqual(i_val, "2")
 
     @skipIfWindows
     @skipIfRemote
@@ -117,16 +117,16 @@ class TestDAP_setDataBreakpoints(lldbdap_testcase.DAPTestCaseBase):
         )
 
         # Test response from dataBreakpointInfo request.
-        self.assertEquals(response_x["body"]["dataId"].split("/")[1], "4")
-        self.assertEquals(response_x["body"]["accessTypes"], self.accessTypes)
-        self.assertEquals(response_arr_2["body"]["dataId"].split("/")[1], "4")
-        self.assertEquals(response_arr_2["body"]["accessTypes"], self.accessTypes)
+        self.assertEqual(response_x["body"]["dataId"].split("/")[1], "4")
+        self.assertEqual(response_x["body"]["accessTypes"], self.accessTypes)
+        self.assertEqual(response_arr_2["body"]["dataId"].split("/")[1], "4")
+        self.assertEqual(response_arr_2["body"]["accessTypes"], self.accessTypes)
         dataBreakpoints = [
             {"dataId": response_x["body"]["dataId"], "accessType": "write"},
             {"dataId": response_arr_2["body"]["dataId"], "accessType": "write"},
         ]
         set_response = self.dap_server.request_setDataBreakpoint(dataBreakpoints)
-        self.assertEquals(
+        self.assertEqual(
             set_response["body"]["breakpoints"],
             [{"verified": True}, {"verified": True}],
         )
@@ -134,14 +134,14 @@ class TestDAP_setDataBreakpoints(lldbdap_testcase.DAPTestCaseBase):
         self.continue_to_next_stop()
         x_val = self.dap_server.get_local_variable_value("x")
         i_val = self.dap_server.get_local_variable_value("i")
-        self.assertEquals(x_val, "2")
-        self.assertEquals(i_val, "1")
+        self.assertEqual(x_val, "2")
+        self.assertEqual(i_val, "1")
 
         self.continue_to_next_stop()
         arr_2 = self.dap_server.get_local_variable_child("arr", "[2]")
         i_val = self.dap_server.get_local_variable_value("i")
-        self.assertEquals(arr_2["value"], "42")
-        self.assertEquals(i_val, "2")
+        self.assertEqual(arr_2["value"], "42")
+        self.assertEqual(i_val, "2")
         self.dap_server.request_setDataBreakpoint([])
 
         # Test hit condition
@@ -156,10 +156,10 @@ class TestDAP_setDataBreakpoints(lldbdap_testcase.DAPTestCaseBase):
             }
         ]
         set_response = self.dap_server.request_setDataBreakpoint(dataBreakpoints)
-        self.assertEquals(set_response["body"]["breakpoints"], [{"verified": True}])
+        self.assertEqual(set_response["body"]["breakpoints"], [{"verified": True}])
         self.continue_to_next_stop()
         x_val = self.dap_server.get_local_variable_value("x")
-        self.assertEquals(x_val, "3")
+        self.assertEqual(x_val, "3")
 
         # Test condition
         dataBreakpoints = [
@@ -170,7 +170,7 @@ class TestDAP_setDataBreakpoints(lldbdap_testcase.DAPTestCaseBase):
             }
         ]
         set_response = self.dap_server.request_setDataBreakpoint(dataBreakpoints)
-        self.assertEquals(set_response["body"]["breakpoints"], [{"verified": True}])
+        self.assertEqual(set_response["body"]["breakpoints"], [{"verified": True}])
         self.continue_to_next_stop()
         x_val = self.dap_server.get_local_variable_value("x")
-        self.assertEquals(x_val, "10")
+        self.assertEqual(x_val, "10")
diff --git a/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py b/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py
index ab7dfb5216ae..07ab6d5a63eb 100644
--- a/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py
+++ b/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py
@@ -254,18 +254,22 @@ class TestDAP_variables(lldbdap_testcase.DAPTestCaseBase):
             "pt": {
                 "equals": {"type": "PointType"},
                 "startswith": {
-                    "result": "{x:11, y:22, buffer:{...}}"
-                    if enableAutoVariableSummaries
-                    else "PointType @ 0x"
+                    "result": (
+                        "{x:11, y:22, buffer:{...}}"
+                        if enableAutoVariableSummaries
+                        else "PointType @ 0x"
+                    )
                 },
                 "hasVariablesReference": True,
             },
             "pt.buffer": {
                 "equals": {"type": "int[16]"},
                 "startswith": {
-                    "result": "{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ...}"
-                    if enableAutoVariableSummaries
-                    else "int[16] @ 0x"
+                    "result": (
+                        "{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ...}"
+                        if enableAutoVariableSummaries
+                        else "int[16] @ 0x"
+                    )
                 },
                 "hasVariablesReference": True,
             },
@@ -528,9 +532,11 @@ class TestDAP_variables(lldbdap_testcase.DAPTestCaseBase):
                 "watch": {
                     "equals": {"type": "PointType"},
                     "startswith": {
-                        "result": "{x:11, y:22, buffer:{...}}"
-                        if enableAutoVariableSummaries
-                        else "PointType @ 0x"
+                        "result": (
+                            "{x:11, y:22, buffer:{...}}"
+                            if enableAutoVariableSummaries
+                            else "PointType @ 0x"
+                        )
                     },
                     "missing": ["indexedVariables"],
                     "hasVariablesReference": True,
@@ -538,9 +544,11 @@ class TestDAP_variables(lldbdap_testcase.DAPTestCaseBase):
                 "variables": {
                     "equals": {"type": "PointType"},
                     "startswith": {
-                        "result": "{x:11, y:22, buffer:{...}}"
-                        if enableAutoVariableSummaries
-                        else "PointType @ 0x"
+                        "result": (
+                            "{x:11, y:22, buffer:{...}}"
+                            if enableAutoVariableSummaries
+                            else "PointType @ 0x"
+                        )
                     },
                     "missing": ["indexedVariables"],
                     "hasVariablesReference": True,
@@ -777,20 +785,20 @@ class TestDAP_variables(lldbdap_testcase.DAPTestCaseBase):
         # Verify locals value format decimal
         is_hex = False
         var_pt_x = self.dap_server.get_local_variable_child("pt", "x", is_hex=is_hex)
-        self.assertEquals(var_pt_x["value"], "11")
+        self.assertEqual(var_pt_x["value"], "11")
         var_pt_y = self.dap_server.get_local_variable_child("pt", "y", is_hex=is_hex)
-        self.assertEquals(var_pt_y["value"], "22")
+        self.assertEqual(var_pt_y["value"], "22")
 
         # Verify locals value format hexical
         is_hex = True
         var_pt_x = self.dap_server.get_local_variable_child("pt", "x", is_hex=is_hex)
-        self.assertEquals(var_pt_x["value"], "0x0000000b")
+        self.assertEqual(var_pt_x["value"], "0x0000000b")
         var_pt_y = self.dap_server.get_local_variable_child("pt", "y", is_hex=is_hex)
-        self.assertEquals(var_pt_y["value"], "0x00000016")
+        self.assertEqual(var_pt_y["value"], "0x00000016")
 
         # Toggle and verify locals value format decimal again
         is_hex = False
         var_pt_x = self.dap_server.get_local_variable_child("pt", "x", is_hex=is_hex)
-        self.assertEquals(var_pt_x["value"], "11")
+        self.assertEqual(var_pt_x["value"], "11")
         var_pt_y = self.dap_server.get_local_variable_child("pt", "y", is_hex=is_hex)
-        self.assertEquals(var_pt_y["value"], "22")
+        self.assertEqual(var_pt_y["value"], "22")
-- 
GitLab


From 932bef23276da05c5eed2ffe7c62b7fc5f39caf1 Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Fri, 17 May 2024 10:18:26 -0700
Subject: [PATCH 1154/1206] Poly canonicalization (#91410)

Adds simple canonicalization rules to the polynomial dialect. Mainly to
get the boilerplate incorporated before more substantial
canonicalization patterns are added.

---------

Co-authored-by: Jeremy Kun <j2kun@users.noreply.github.com>
---
 .../mlir/Dialect/Polynomial/IR/Polynomial.td  |  3 ++
 mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt |  5 +++
 .../IR/PolynomialCanonicalization.td          | 42 +++++++++++++++++
 .../Dialect/Polynomial/IR/PolynomialOps.cpp   | 25 +++++++++++
 .../Dialect/Polynomial/canonicalization.mlir  | 45 +++++++++++++++++++
 .../llvm-project-overlay/mlir/BUILD.bazel     | 18 ++++++++
 6 files changed, 138 insertions(+)
 create mode 100644 mlir/lib/Dialect/Polynomial/IR/PolynomialCanonicalization.td
 create mode 100644 mlir/test/Dialect/Polynomial/canonicalization.mlir

diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
index ae8484501a50..537be4832e8f 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
@@ -245,6 +245,7 @@ def Polynomial_SubOp : Polynomial_BinaryOp<"sub"> {
     %2 = polynomial.sub %0, %1 : !polynomial.polynomial<#ring>
     ```
   }];
+  let hasCanonicalizer = 1;
 }
 
 def Polynomial_MulOp : Polynomial_BinaryOp<"mul", [Commutative]> {
@@ -480,6 +481,7 @@ def Polynomial_NTTOp : Polynomial_Op<"ntt", [Pure]> {
   let arguments = (ins Polynomial_PolynomialType:$input);
   let results = (outs RankedTensorOf<[AnyInteger]>:$output);
   let assemblyFormat = "$input attr-dict `:` qualified(type($input)) `->` type($output)";
+  let hasCanonicalizer = 1;
   let hasVerifier = 1;
 }
 
@@ -498,6 +500,7 @@ def Polynomial_INTTOp : Polynomial_Op<"intt", [Pure]> {
   let arguments = (ins RankedTensorOf<[AnyInteger]>:$input);
   let results = (outs Polynomial_PolynomialType:$output);
   let assemblyFormat = "$input attr-dict `:` qualified(type($input)) `->` type($output)";
+  let hasCanonicalizer = 1;
   let hasVerifier = 1;
 }
 
diff --git a/mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt b/mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt
index d6e703b8b359..8fa49555dd59 100644
--- a/mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt
@@ -1,3 +1,7 @@
+set(LLVM_TARGET_DEFINITIONS PolynomialCanonicalization.td)
+mlir_tablegen(PolynomialCanonicalization.inc -gen-rewriters)
+add_public_tablegen_target(MLIRPolynomialCanonicalizationIncGen)
+
 add_mlir_dialect_library(MLIRPolynomialDialect
   Polynomial.cpp
   PolynomialAttributes.cpp
@@ -10,6 +14,7 @@ add_mlir_dialect_library(MLIRPolynomialDialect
   DEPENDS
   MLIRPolynomialIncGen
   MLIRPolynomialAttributesIncGen
+  MLIRPolynomialCanonicalizationIncGen
   MLIRBuiltinAttributesIncGen
 
   LINK_LIBS PUBLIC
diff --git a/mlir/lib/Dialect/Polynomial/IR/PolynomialCanonicalization.td b/mlir/lib/Dialect/Polynomial/IR/PolynomialCanonicalization.td
new file mode 100644
index 000000000000..9d09799c1763
--- /dev/null
+++ b/mlir/lib/Dialect/Polynomial/IR/PolynomialCanonicalization.td
@@ -0,0 +1,42 @@
+//===- PolynomialCanonicalization.td - Polynomial patterns -*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef POLYNOMIAL_CANONICALIZATION
+#define POLYNOMIAL_CANONICALIZATION
+
+include "mlir/Dialect/Polynomial/IR/Polynomial.td"
+include "mlir/Dialect/Arith/IR/ArithOps.td"
+include "mlir/IR/OpBase.td"
+include "mlir/IR/PatternBase.td"
+
+// Get a -1 integer attribute of the same type as the polynomial SSA value's
+// ring coefficient type.
+def getMinusOne
+  : NativeCodeCall<
+      "$_builder.getIntegerAttr("
+        "cast<PolynomialType>($0.getType()).getRing().getCoefficientType(), -1)">;
+
+def SubAsAdd : Pat<
+  (Polynomial_SubOp $f, $g),
+  (Polynomial_AddOp $f,
+    (Polynomial_MulScalarOp $g,
+      (Arith_ConstantOp (getMinusOne $g))))>;
+
+def INTTAfterNTT : Pat<
+  (Polynomial_INTTOp (Polynomial_NTTOp $poly)),
+  (replaceWithValue $poly),
+  []
+>;
+
+def NTTAfterINTT : Pat<
+  (Polynomial_NTTOp (Polynomial_INTTOp $tensor)),
+  (replaceWithValue $tensor),
+  []
+>;
+
+#endif  // POLYNOMIAL_CANONICALIZATION
diff --git a/mlir/lib/Dialect/Polynomial/IR/PolynomialOps.cpp b/mlir/lib/Dialect/Polynomial/IR/PolynomialOps.cpp
index 12010de34823..1a2439fe810b 100644
--- a/mlir/lib/Dialect/Polynomial/IR/PolynomialOps.cpp
+++ b/mlir/lib/Dialect/Polynomial/IR/PolynomialOps.cpp
@@ -7,12 +7,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Polynomial/IR/PolynomialOps.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Polynomial/IR/Polynomial.h"
 #include "mlir/Dialect/Polynomial/IR/PolynomialAttributes.h"
 #include "mlir/Dialect/Polynomial/IR/PolynomialTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
+#include "mlir/IR/PatternMatch.h"
 #include "mlir/Support/LogicalResult.h"
 #include "llvm/ADT/APInt.h"
 
@@ -183,3 +185,26 @@ LogicalResult INTTOp::verify() {
   auto ring = getOutput().getType().getRing();
   return verifyNTTOp(this->getOperation(), ring, tensorType);
 }
+
+//===----------------------------------------------------------------------===//
+// TableGen'd canonicalization patterns
+//===----------------------------------------------------------------------===//
+
+namespace {
+#include "PolynomialCanonicalization.inc"
+} // namespace
+
+void SubOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                        MLIRContext *context) {
+  results.add<SubAsAdd>(context);
+}
+
+void NTTOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                        MLIRContext *context) {
+  results.add<NTTAfterINTT>(context);
+}
+
+void INTTOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                         MLIRContext *context) {
+  results.add<INTTAfterNTT>(context);
+}
diff --git a/mlir/test/Dialect/Polynomial/canonicalization.mlir b/mlir/test/Dialect/Polynomial/canonicalization.mlir
new file mode 100644
index 000000000000..dbfbf2d93f11
--- /dev/null
+++ b/mlir/test/Dialect/Polynomial/canonicalization.mlir
@@ -0,0 +1,45 @@
+// RUN: mlir-opt -canonicalize %s | FileCheck %s
+#ntt_poly = #polynomial.int_polynomial<-1 + x**8>
+#ntt_ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256, polynomialModulus=#ntt_poly, primitiveRoot=31>
+!ntt_poly_ty = !polynomial.polynomial<ring=#ntt_ring>
+!tensor_ty = tensor<8xi32, #ntt_ring>
+
+// CHECK-LABEL: @test_canonicalize_intt_after_ntt
+// CHECK: (%[[P:.*]]: [[T:.*]]) -> [[T]]
+func.func @test_canonicalize_intt_after_ntt(%p0 : !ntt_poly_ty) -> !ntt_poly_ty {
+  // CHECK-NOT: polynomial.ntt
+  // CHECK-NOT: polynomial.intt
+  // CHECK: %[[RESULT:.+]] = polynomial.add %[[P]], %[[P]]  : [[T]]
+  %t0 = polynomial.ntt %p0 : !ntt_poly_ty -> !tensor_ty
+  %p1 = polynomial.intt %t0: !tensor_ty -> !ntt_poly_ty
+  %p2 = polynomial.add %p1, %p1 : !ntt_poly_ty
+  // CHECK: return %[[RESULT]] : [[T]]
+  return %p2 : !ntt_poly_ty
+}
+
+// CHECK-LABEL: @test_canonicalize_ntt_after_intt
+// CHECK: (%[[X:.*]]: [[T:.*]]) -> [[T]]
+func.func @test_canonicalize_ntt_after_intt(%t0 : !tensor_ty) -> !tensor_ty {
+  // CHECK-NOT: polynomial.intt
+  // CHECK-NOT: polynomial.ntt
+  // CHECK: %[[RESULT:.+]] = arith.addi %[[X]], %[[X]] : [[T]]
+  %p0 = polynomial.intt %t0 : !tensor_ty -> !ntt_poly_ty
+  %t1 = polynomial.ntt %p0 : !ntt_poly_ty -> !tensor_ty
+  %t2 = arith.addi %t1, %t1 : !tensor_ty
+  // CHECK: return %[[RESULT]] : [[T]]
+  return %t2 : !tensor_ty
+}
+
+#cycl_2048 = #polynomial.int_polynomial<1 + x**1024>
+#ring = #polynomial.ring<coefficientType=i32, coefficientModulus=256:i32, polynomialModulus=#cycl_2048>
+!sub_ty = !polynomial.polynomial<ring=#ring>
+
+// CHECK-LABEL: test_canonicalize_sub
+// CHECK-SAME: (%[[p0:.*]]: [[T:.*]], %[[p1:.*]]: [[T]]) -> [[T]] {
+func.func @test_canonicalize_sub(%poly0 : !sub_ty, %poly1 : !sub_ty) -> !sub_ty {
+  %0 = polynomial.sub %poly0, %poly1  : !sub_ty
+  // CHECK: %[[minus_one:.+]] = arith.constant -1 : i32
+  // CHECK: %[[p1neg:.+]] = polynomial.mul_scalar %[[p1]], %[[minus_one]]
+  // CHECK: [[ADD:%.+]] = polynomial.add %[[p0]], %[[p1neg]]
+  return %0 : !sub_ty
+}
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index f7495a202669..dac759cfd100 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -6728,6 +6728,7 @@ cc_library(
         ":IR",
         ":InferTypeOpInterface",
         ":PolynomialAttributesIncGen",
+        ":PolynomialCanonicalizationIncGen",
         ":PolynomialIncGen",
         ":Support",
         "//llvm:Support",
@@ -6818,6 +6819,23 @@ gentbl_cc_library(
     deps = [":PolynomialTdFiles"],
 )
 
+gentbl_cc_library(
+    name = "PolynomialCanonicalizationIncGen",
+    strip_include_prefix = "include/mlir/Dialect/Polynomial/IR",
+    tbl_outs = [
+        (
+            ["-gen-rewriters"],
+            "include/mlir/Dialect/Polynomial/IR/PolynomialCanonicalization.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "lib/Dialect/Polynomial/IR/PolynomialCanonicalization.td",
+    deps = [
+        ":ArithOpsTdFiles",
+        ":PolynomialTdFiles",
+    ],
+)
+
 td_library(
     name = "SPIRVOpsTdFiles",
     srcs = glob(["include/mlir/Dialect/SPIRV/IR/*.td"]),
-- 
GitLab


From e18c4838811f2a6f9592ddeaed585a3830ebc8aa Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Fri, 17 May 2024 10:50:39 -0700
Subject: [PATCH 1155/1206] [mlir] add arith dialect dep to fix buildbot
 failure (#92585)

https://lab.llvm.org/buildbot/#/builders/268/builds/14288

```undefined reference to `mlir::detail::TypeIDResolver<mlir::arith::ConstantOp, void>::id'```
---
 mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt b/mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt
index 8fa49555dd59..975315ff4915 100644
--- a/mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt
@@ -18,6 +18,7 @@ add_mlir_dialect_library(MLIRPolynomialDialect
   MLIRBuiltinAttributesIncGen
 
   LINK_LIBS PUBLIC
+  MLIRArithDialect
   MLIRSupport
   MLIRDialect
   MLIRIR
-- 
GitLab


From f7516c7f3f00fca7271056cc9abeda2da4687c17 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 17 May 2024 10:59:39 -0700
Subject: [PATCH 1156/1206] [CodeGen] Support arrays with initializers of
 64-bit size

Based on @OfekShochat's https://reviews.llvm.org/D133648

init.c is the primary test for array initialization, but it uses a
32-bit triple, which would lead to an "array is too large" error. Add
the new test to array-init.c instead.

Fix #57353

Pull Request: https://github.com/llvm/llvm-project/pull/92473
---
 clang/lib/CodeGen/CGExprConstant.cpp | 14 +++++++-------
 clang/lib/Sema/SemaInit.cpp          |  4 ++--
 clang/test/CodeGen/array-init.c      | 10 ++++++++++
 3 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index 94962091116a..4eb65b34a89f 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -393,7 +393,7 @@ bool ConstantAggregateBuilder::split(size_t Index, CharUnits Hint) {
 
 static llvm::Constant *
 EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType,
-                  llvm::Type *CommonElementType, unsigned ArrayBound,
+                  llvm::Type *CommonElementType, uint64_t ArrayBound,
                   SmallVectorImpl<llvm::Constant *> &Elements,
                   llvm::Constant *Filler);
 
@@ -949,11 +949,11 @@ tryEmitGlobalCompoundLiteral(ConstantEmitter &emitter,
 
 static llvm::Constant *
 EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType,
-                  llvm::Type *CommonElementType, unsigned ArrayBound,
+                  llvm::Type *CommonElementType, uint64_t ArrayBound,
                   SmallVectorImpl<llvm::Constant *> &Elements,
                   llvm::Constant *Filler) {
   // Figure out how long the initial prefix of non-zero elements is.
-  unsigned NonzeroLength = ArrayBound;
+  uint64_t NonzeroLength = ArrayBound;
   if (Elements.size() < NonzeroLength && Filler->isNullValue())
     NonzeroLength = Elements.size();
   if (NonzeroLength == Elements.size()) {
@@ -965,7 +965,7 @@ EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType,
     return llvm::ConstantAggregateZero::get(DesiredType);
 
   // Add a zeroinitializer array filler if we have lots of trailing zeroes.
-  unsigned TrailingZeroes = ArrayBound - NonzeroLength;
+  uint64_t TrailingZeroes = ArrayBound - NonzeroLength;
   if (TrailingZeroes >= 8) {
     assert(Elements.size() >= NonzeroLength &&
            "missing initializer for non-zero element");
@@ -1252,12 +1252,12 @@ public:
   llvm::Constant *EmitArrayInitialization(const InitListExpr *ILE, QualType T) {
     auto *CAT = CGM.getContext().getAsConstantArrayType(ILE->getType());
     assert(CAT && "can't emit array init for non-constant-bound array");
-    unsigned NumInitElements = ILE->getNumInits();
-    unsigned NumElements = CAT->getZExtSize();
+    const uint64_t NumElements = CAT->getZExtSize();
 
     // Initialising an array requires us to automatically
     // initialise any elements that have not been initialised explicitly
-    unsigned NumInitableElts = std::min(NumInitElements, NumElements);
+    uint64_t NumInitableElts =
+        std::min<uint64_t>(ILE->getNumInits(), NumElements);
 
     QualType EltType = CAT->getElementType();
 
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 2177972f3af2..353e911c5cc3 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -876,7 +876,7 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity,
 
   InitializedEntity ElementEntity = Entity;
   unsigned NumInits = ILE->getNumInits();
-  unsigned NumElements = NumInits;
+  uint64_t NumElements = NumInits;
   if (const ArrayType *AType = SemaRef.Context.getAsArrayType(ILE->getType())) {
     ElementType = AType->getElementType();
     if (const auto *CAType = dyn_cast<ConstantArrayType>(AType))
@@ -896,7 +896,7 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity,
     ElementType = ILE->getType();
 
   bool SkipEmptyInitChecks = false;
-  for (unsigned Init = 0; Init != NumElements; ++Init) {
+  for (uint64_t Init = 0; Init != NumElements; ++Init) {
     if (hadError)
       return;
 
diff --git a/clang/test/CodeGen/array-init.c b/clang/test/CodeGen/array-init.c
index 62e87edc2974..b3accf482994 100644
--- a/clang/test/CodeGen/array-init.c
+++ b/clang/test/CodeGen/array-init.c
@@ -13,3 +13,13 @@ void testConstArrayInits(void)
   const int a2[5] = {0,0,0};
   const int a3[5] = {0};
 }
+
+/// https://github.com/llvm/llvm-project/issues/57353
+// CHECK: @big_char ={{.*}} global <{ i8, [4294967295 x i8] }> <{ i8 1, [4294967295 x i8] zeroinitializer }>
+char big_char[4294967296] = {1};
+
+// CHECK: @big_char2 ={{.*}} global <{ i8, i8, [4294967296 x i8] }> <{ i8 1, i8 2, [4294967296 x i8] zeroinitializer }>
+char big_char2[4294967298] = {1, 2};
+
+// CHECK: @big_int ={{.*}} global <{ i32, i32, i32, [2147483647 x i32] }> <{ i32 1, i32 2, i32 3, [2147483645 x i32] zeroinitializer }>
+int big_int[0x200000000 >> 2] = {1, 2, 3};
-- 
GitLab


From 487d5af6482ea5f074c12d29d7e376d3fc697706 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Fri, 17 May 2024 11:08:14 -0700
Subject: [PATCH 1157/1206] [ctx_profile] Integration test (#92456)

Compile with clang a program that's instrumented for contextual profiling and verify a profile can be collected.
---
 compiler-rt/lib/ctx_profile/CMakeLists.txt    |   9 ++
 .../TestCases/generate-context.cpp            | 103 ++++++++++++++++++
 compiler-rt/test/ctx_profile/lit.cfg.py       |   4 +
 3 files changed, 116 insertions(+)
 create mode 100644 compiler-rt/test/ctx_profile/TestCases/generate-context.cpp

diff --git a/compiler-rt/lib/ctx_profile/CMakeLists.txt b/compiler-rt/lib/ctx_profile/CMakeLists.txt
index 1fa70594b28a..ab7bf3241fd6 100644
--- a/compiler-rt/lib/ctx_profile/CMakeLists.txt
+++ b/compiler-rt/lib/ctx_profile/CMakeLists.txt
@@ -18,3 +18,12 @@ append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ EXTRA_FLAGS)
 if(COMPILER_RT_INCLUDE_TESTS)
   add_subdirectory(tests)
 endif()
+
+add_compiler_rt_runtime(clang_rt.ctx_profile
+  STATIC
+  ARCHS ${CTX_PROFILE_SUPPORTED_ARCH}
+  OBJECT_LIBS RTSanitizerCommon RTSanitizerCommonLibc
+  CFLAGS ${EXTRA_FLAGS}
+  SOURCES ${CTX_PROFILE_SOURCES}
+  ADDITIONAL_HEADERS ${CTX_PROFILE_HEADERS}
+  PARENT_TARGET ctx_profile)
\ No newline at end of file
diff --git a/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp b/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp
new file mode 100644
index 000000000000..981d6170091c
--- /dev/null
+++ b/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp
@@ -0,0 +1,103 @@
+// Simple integration test for contextual instrumentation
+//
+// Copy the header defining ContextNode.
+// RUN: mkdir -p %t_include
+// RUN: cp %llvm_src/include/llvm/ProfileData/CtxInstrContextNode.h %t_include/
+//
+// Compile with ctx instrumentation "on". We treat "theRoot" as callgraph root.
+// RUN: %clangxx %s -lclang_rt.ctx_profile -I%t_include -O2 -o %t.bin -mllvm -profile-context-root=theRoot
+//
+// Run the binary, and observe the profile fetch handler's output.
+// RUN: %t.bin | FileCheck %s
+
+#include "CtxInstrContextNode.h"
+#include <cstdio>
+#include <iostream>
+
+using namespace llvm::ctx_profile;
+extern "C" bool __llvm_ctx_profile_fetch(void *Data,
+                                         bool (*Writer)(void *,
+                                                        const ContextNode &));
+
+// avoid name mangling
+extern "C" {
+__attribute__((noinline)) void someFunction(int I) {
+  if (I % 2)
+    printf("check odd\n");
+  else
+    printf("check even\n");
+}
+
+// block inlining because the pre-inliner otherwise will inline this - it's
+// too small.
+__attribute__((noinline)) void theRoot() {
+  printf("check 1\n");
+  someFunction(1);
+#pragma nounroll
+  for (auto I = 0; I < 2; ++I) {
+    someFunction(I);
+  }
+}
+}
+
+// Make sure the program actually ran correctly.
+// CHECK: check 1
+// CHECK-NEXT: check odd
+// CHECK-NEXT: check even
+// CHECK-NEXT: check odd
+
+void printProfile(const ContextNode &Node, const std::string &Indent,
+                  const std::string &Increment) {
+  std::cout << Indent << "Guid: " << Node.guid() << std::endl;
+  std::cout << Indent << "Entries: " << Node.entrycount() << std::endl;
+  std::cout << Indent << Node.counters_size() << " counters and "
+            << Node.callsites_size() << " callsites" << std::endl;
+  std::cout << Indent << "Counter values: ";
+  for (uint32_t I = 0U; I < Node.counters_size(); ++I)
+    std::cout << Node.counters()[I] << " ";
+  std::cout << std::endl;
+  for (uint32_t I = 0U; I < Node.callsites_size(); ++I)
+    for (const auto *N = Node.subContexts()[I]; N; N = N->next()) {
+      std::cout << Indent << "At Index " << I << ":" << std::endl;
+      printProfile(*N, Indent + Increment, Increment);
+    }
+}
+
+// 8657661246551306189 is theRoot. We expect 2 callsites and 2 counters - one
+// for the entry basic block and one for the loop.
+// 6759619411192316602 is someFunction. We expect all context instances to show
+// the same nr of counters and callsites, but the counters will be different.
+// The first context is for the first callsite with theRoot as parent, and the
+// second counter in someFunction will be 0 (we pass an odd nr, and the other
+// path gets instrumented).
+// The second context is in the loop. We expect 2 entries and each of the
+// branches would be taken once, so the second counter is 1.
+// CHECK-NEXT: Guid: 8657661246551306189
+// CHECK-NEXT: Entries: 1
+// CHECK-NEXT: 2 counters and 3 callsites
+// CHECK-NEXT: Counter values: 1 2
+// CHECK-NEXT: At Index 1:
+// CHECK-NEXT:   Guid: 6759619411192316602
+// CHECK-NEXT:   Entries: 1
+// CHECK-NEXT:   2 counters and 2 callsites
+// CHECK-NEXT:   Counter values: 1 0
+// CHECK-NEXT: At Index 2:
+// CHECK-NEXT:   Guid: 6759619411192316602
+// CHECK-NEXT:   Entries: 2
+// CHECK-NEXT:   2 counters and 2 callsites
+// CHECK-NEXT:   Counter values: 2 1
+
+bool profileWriter() {
+  return __llvm_ctx_profile_fetch(
+      nullptr, +[](void *, const ContextNode &Node) {
+        printProfile(Node, "", "  ");
+        return true;
+      });
+}
+
+int main(int argc, char **argv) {
+  theRoot();
+  // This would be implemented in a specific RPC handler, but here we just call
+  // it directly.
+  return !profileWriter();
+}
diff --git a/compiler-rt/test/ctx_profile/lit.cfg.py b/compiler-rt/test/ctx_profile/lit.cfg.py
index a56dabb8ebeb..520552974f76 100644
--- a/compiler-rt/test/ctx_profile/lit.cfg.py
+++ b/compiler-rt/test/ctx_profile/lit.cfg.py
@@ -29,3 +29,7 @@ config.name = "CtxProfile" + config.name_suffix
 config.test_source_root = os.path.dirname(__file__)
 # Default test suffixes.
 config.suffixes = [".c", ".cpp", ".test"]
+
+config.substitutions.append(
+    ("%clangxx ", " ".join([config.clang] + config.cxx_mode_flags) + " ")
+)
-- 
GitLab


From 16bb7e89a995dfe8b14a8ddbce3da79c217e884a Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 17 May 2024 13:15:04 -0500
Subject: [PATCH 1158/1206] [Offload][NFC] Remove all trailing whitespace from
 offload/ (#92578)

Summary:
This patch cleans up the training whitespace in a bunch of tests and
CMake files. Most just in preparation for other cleanups.
---
 offload/CMakeLists.txt                        |  2 +-
 offload/DeviceRTL/CMakeLists.txt              |  2 +-
 offload/plugins-nextgen/CMakeLists.txt        |  2 +-
 offload/plugins-nextgen/amdgpu/CMakeLists.txt |  2 +-
 offload/plugins-nextgen/common/CMakeLists.txt |  2 +-
 offload/plugins-nextgen/cuda/CMakeLists.txt   |  2 +-
 offload/plugins-nextgen/host/CMakeLists.txt   | 10 ++--
 offload/src/CMakeLists.txt                    |  2 +-
 .../mapping/array_section_implicit_capture.c  |  2 +-
 .../test/offloading/default_thread_limit.c    |  2 +-
 .../basic-target-region-1D-array-section.f90  |  4 +-
 .../basic-target-region-3D-array-section.f90  |  4 +-
 ...double-target-call-with-declare-target.f90 |  6 +--
 .../target-map-allocatable-map-scopes.f90     | 46 +++++++++----------
 .../target-map-derived-type-full-1.f90        | 14 +++---
 .../target-map-derived-type-full-2.f90        | 20 ++++----
 ...arget-map-derived-type-full-implicit-1.f90 | 13 +++---
 ...arget-map-derived-type-full-implicit-2.f90 | 20 ++++----
 ...double-large-nested-dtype-multi-member.f90 | 24 +++++-----
 ...t-map-double-nested-dtype-array-bounds.f90 | 16 +++----
 ...ouble-nested-dtype-double-array-bounds.f90 | 16 +++----
 ...-map-double-nested-dtype-single-member.f90 | 10 ++--
 ...pe-arr-bounds-member-enter-exit-update.f90 | 12 ++---
 ...map-dtype-arr-bounds-member-enter-exit.f90 | 16 +++----
 ...dtype-explicit-individual-array-member.f90 |  6 +--
 ...-multi-explicit-array-3D-member-bounds.f90 |  8 ++--
 ...ype-multi-explicit-array-member-bounds.f90 |  8 ++--
 ...-map-dtype-multi-explicit-array-member.f90 |  4 +-
 ...target-map-dtype-multi-explicit-member.f90 | 16 +++----
 .../target-map-enter-exit-allocatables.f90    | 10 ++--
 .../fortran/target-map-enter-exit-array-2.f90 |  4 +-
 .../target-map-enter-exit-array-bounds.f90    |  6 +--
 .../fortran/target-map-enter-exit-array.f90   |  6 +--
 .../fortran/target-map-enter-exit-scalar.f90  |  6 +--
 ...target-map-individual-dtype-member-map.f90 |  4 +-
 ...et-map-large-nested-dtype-multi-member.f90 | 20 ++++----
 ...target-map-nested-dtype-complex-member.f90 | 10 ++--
 ...target-map-nested-dtype-derived-member.f90 | 12 ++---
 .../target-map-nested-dtype-multi-member.f90  |  8 ++--
 .../target-map-nested-dtype-single-member.f90 |  8 ++--
 .../target-map-pointer-scopes-enter-exit.f90  | 46 +++++++++----------
 ...pointer-target-array-section-3d-bounds.f90 |  6 +--
 .../target-map-pointer-target-scopes.f90      | 44 +++++++++---------
 .../target-map-two-dtype-explicit-member.f90  |  4 +-
 ...type-individual-member-array-1D-bounds.f90 |  8 ++--
 ...type-mixed-implicit-explicit-capture-1.f90 |  4 +-
 ...type-mixed-implicit-explicit-capture-2.f90 |  6 +--
 ...two-dtype-multi-member-array-1D-bounds.f90 |  8 ++--
 ...-map-two-nested-dtype-member-array-map.f90 |  6 +--
 offload/test/offloading/spmdization.c         | 10 +++-
 offload/test/offloading/thread_limit.c        |  4 +-
 51 files changed, 268 insertions(+), 263 deletions(-)

diff --git a/offload/CMakeLists.txt b/offload/CMakeLists.txt
index 31adbe45388a..4e552e058959 100644
--- a/offload/CMakeLists.txt
+++ b/offload/CMakeLists.txt
@@ -134,7 +134,7 @@ if (NOT LIBOMPTARGET_LLVM_INCLUDE_DIRS)
   message(FATAL_ERROR "Missing definition for LIBOMPTARGET_LLVM_INCLUDE_DIRS")
 endif()
 
-if(DEFINED LIBOMPTARGET_BUILD_CUDA_PLUGIN OR 
+if(DEFINED LIBOMPTARGET_BUILD_CUDA_PLUGIN OR
    DEFINED LIBOMPTARGET_BUILD_AMDGPU_PLUGIN)
   message(WARNING "Option removed, use 'LIBOMPTARGET_PLUGINS_TO_BUILD' instead")
 endif()
diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt
index ae7d54d127ed..0c9b77ded5ab 100644
--- a/offload/DeviceRTL/CMakeLists.txt
+++ b/offload/DeviceRTL/CMakeLists.txt
@@ -104,7 +104,7 @@ set(src_files
 # vectorized accesses to the shared state. Generally, those are "good" but
 # the optimizer pipeline (esp. Attributor) does not fully support vectorized
 # instructions yet and we end up missing out on way more important constant
-# propagation. That said, we will run the vectorizer again after the runtime 
+# propagation. That said, we will run the vectorizer again after the runtime
 # has been linked into the user program.
 set(clang_opt_flags -O3 -mllvm -openmp-opt-disable -DSHARED_SCRATCHPAD_SIZE=512 -mllvm -vectorize-slp=false )
 set(link_opt_flags  -O3        -openmp-opt-disable -attributor-enable=module -vectorize-slp=false )
diff --git a/offload/plugins-nextgen/CMakeLists.txt b/offload/plugins-nextgen/CMakeLists.txt
index 5d684420c290..d31bf557669e 100644
--- a/offload/plugins-nextgen/CMakeLists.txt
+++ b/offload/plugins-nextgen/CMakeLists.txt
@@ -38,7 +38,7 @@ function(add_target_library target_name lib_name)
                         PluginCommon ${OPENMP_PTHREAD_LIB})
 
   target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name})
-  target_compile_definitions(${target_name} PRIVATE 
+  target_compile_definitions(${target_name} PRIVATE
                              DEBUG_PREFIX="TARGET ${lib_name} RTL")
   set_target_properties(${target_name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
diff --git a/offload/plugins-nextgen/amdgpu/CMakeLists.txt b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
index 4a79fa83420c..5f72568e9a53 100644
--- a/offload/plugins-nextgen/amdgpu/CMakeLists.txt
+++ b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
@@ -28,7 +28,7 @@ endif()
 option(LIBOMPTARGET_FORCE_AMDGPU_TESTS "Build AMDGPU libomptarget tests" OFF)
 if (LIBOMPTARGET_FOUND_AMDGPU_GPU OR LIBOMPTARGET_FORCE_AMDGPU_TESTS)
   # Report to the parent scope that we are building a plugin for amdgpu
-  set(LIBOMPTARGET_SYSTEM_TARGETS 
+  set(LIBOMPTARGET_SYSTEM_TARGETS
       "${LIBOMPTARGET_SYSTEM_TARGETS} amdgcn-amd-amdhsa" PARENT_SCOPE)
   list(APPEND LIBOMPTARGET_TESTED_PLUGINS "omptarget.rtl.amdgpu")
   set(LIBOMPTARGET_TESTED_PLUGINS "${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE)
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 43e08d575a4a..284f98875170 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -51,7 +51,7 @@ target_compile_definitions(PluginCommon PRIVATE
 target_compile_options(PluginCommon PUBLIC ${offload_compile_flags})
 target_link_options(PluginCommon PUBLIC ${offload_link_flags})
 
-target_include_directories(PluginCommon PUBLIC 
+target_include_directories(PluginCommon PUBLIC
   ${CMAKE_CURRENT_SOURCE_DIR}/include
   ${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
   ${LIBOMPTARGET_BINARY_INCLUDE_DIR}
diff --git a/offload/plugins-nextgen/cuda/CMakeLists.txt b/offload/plugins-nextgen/cuda/CMakeLists.txt
index 4be33c6b69dc..51507e7537a4 100644
--- a/offload/plugins-nextgen/cuda/CMakeLists.txt
+++ b/offload/plugins-nextgen/cuda/CMakeLists.txt
@@ -25,7 +25,7 @@ endif()
 option(LIBOMPTARGET_FORCE_NVIDIA_TESTS "Build NVIDIA libomptarget tests" OFF)
 if (LIBOMPTARGET_FOUND_NVIDIA_GPU OR LIBOMPTARGET_FORCE_NVIDIA_TESTS)
   libomptarget_say("Enable tests using CUDA plugin")
-  set(LIBOMPTARGET_SYSTEM_TARGETS 
+  set(LIBOMPTARGET_SYSTEM_TARGETS
       "${LIBOMPTARGET_SYSTEM_TARGETS} nvptx64-nvidia-cuda nvptx64-nvidia-cuda-LTO" PARENT_SCOPE)
   list(APPEND LIBOMPTARGET_TESTED_PLUGINS "omptarget.rtl.cuda")
   set(LIBOMPTARGET_TESTED_PLUGINS "${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE)
diff --git a/offload/plugins-nextgen/host/CMakeLists.txt b/offload/plugins-nextgen/host/CMakeLists.txt
index 72b5681283fe..d35aadc56aa8 100644
--- a/offload/plugins-nextgen/host/CMakeLists.txt
+++ b/offload/plugins-nextgen/host/CMakeLists.txt
@@ -44,23 +44,23 @@ endif()
 
 # Define the target specific triples and ELF machine values.
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$")
-  list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
+  list(APPEND LIBOMPTARGET_SYSTEM_TARGETS
        "powerpc64le-ibm-linux-gnu" "powerpc64le-ibm-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64$")
-  list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
+  list(APPEND LIBOMPTARGET_SYSTEM_TARGETS
        "powerpc64-ibm-linux-gnu" "powerpc64-ibm-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64$")
-  list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
+  list(APPEND LIBOMPTARGET_SYSTEM_TARGETS
        "x86_64-pc-linux-gnu" "x86_64-pc-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64$")
-  list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
+  list(APPEND LIBOMPTARGET_SYSTEM_TARGETS
        "aarch64-unknown-linux-gnu" "aarch64-unknown-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x$")
-  list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
+  list(APPEND LIBOMPTARGET_SYSTEM_TARGETS
        "s390x-ibm-linux-gnu" "s390x-ibm-linux-gnu-LTO")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
 endif()
diff --git a/offload/src/CMakeLists.txt b/offload/src/CMakeLists.txt
index 590fb650c0ae..c54bcbe7ed26 100644
--- a/offload/src/CMakeLists.txt
+++ b/offload/src/CMakeLists.txt
@@ -38,7 +38,7 @@ add_llvm_library(omptarget
   NO_INSTALL_RPATH
   BUILDTREE_ONLY
 )
-target_include_directories(omptarget PRIVATE 
+target_include_directories(omptarget PRIVATE
   ${LIBOMPTARGET_INCLUDE_DIR} ${LIBOMPTARGET_BINARY_INCLUDE_DIR}
 )
 
diff --git a/offload/test/mapping/array_section_implicit_capture.c b/offload/test/mapping/array_section_implicit_capture.c
index 210b7e51cbdf..1042bd23bd5e 100644
--- a/offload/test/mapping/array_section_implicit_capture.c
+++ b/offload/test/mapping/array_section_implicit_capture.c
@@ -1,4 +1,4 @@
-// RUN: %libomptarget-compile-generic 
+// RUN: %libomptarget-compile-generic
 // RUN: %libomptarget-run-generic 2>&1 \
 // RUN: | %fcheck-generic
 
diff --git a/offload/test/offloading/default_thread_limit.c b/offload/test/offloading/default_thread_limit.c
index 4da02bbb152e..d9be843dfbda 100644
--- a/offload/test/offloading/default_thread_limit.c
+++ b/offload/test/offloading/default_thread_limit.c
@@ -33,7 +33,7 @@ int main() {
     optnone();
   }
 // DEFAULT: [[NT:(128|256)]] (MaxFlatWorkGroupSize: [[NT]]
-#pragma omp target 
+#pragma omp target
 #pragma omp teams distribute parallel for
   for (int i = 0; i < N; ++i) {
     optnone();
diff --git a/offload/test/offloading/fortran/basic-target-region-1D-array-section.f90 b/offload/test/offloading/fortran/basic-target-region-1D-array-section.f90
index 476b77e4a549..e0b0f2ea734e 100644
--- a/offload/test/offloading/fortran/basic-target-region-1D-array-section.f90
+++ b/offload/test/offloading/fortran/basic-target-region-1D-array-section.f90
@@ -1,4 +1,4 @@
-! Basic offloading test of arrays with provided lower 
+! Basic offloading test of arrays with provided lower
 ! and upper bounds as specified by OpenMP's sectioning
 ! REQUIRES: flang
 ! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
@@ -20,7 +20,7 @@ program main
             i = i + 1
         end do
     !$omp end target
-    
+
     print *, write_arr(:)
 end program
 
diff --git a/offload/test/offloading/fortran/basic-target-region-3D-array-section.f90 b/offload/test/offloading/fortran/basic-target-region-3D-array-section.f90
index 229798b57477..1b3f86c0dbfa 100644
--- a/offload/test/offloading/fortran/basic-target-region-3D-array-section.f90
+++ b/offload/test/offloading/fortran/basic-target-region-3D-array-section.f90
@@ -12,7 +12,7 @@ program main
     implicit none
     integer :: inArray(3,3,3)
     integer :: outArray(3,3,3)
-    integer :: i, j, k 
+    integer :: i, j, k
     integer :: j2 = 3, k2 = 3
 
     do i = 1, 3
@@ -25,7 +25,7 @@ program main
     end do
 
 j = 1
-k = 1 
+k = 1
 !$omp target map(tofrom:inArray(1:3, 1:3, 2:2), outArray(1:3, 1:3, 1:3), j, k, j2, k2)
     do while (j <= j2)
       k = 1
diff --git a/offload/test/offloading/fortran/double-target-call-with-declare-target.f90 b/offload/test/offloading/fortran/double-target-call-with-declare-target.f90
index 56c96727d475..38cb055bfa2d 100644
--- a/offload/test/offloading/fortran/double-target-call-with-declare-target.f90
+++ b/offload/test/offloading/fortran/double-target-call-with-declare-target.f90
@@ -1,6 +1,6 @@
 ! Offloading test with two target regions mapping the same
-! declare target Fortran array and writing some values to 
-! it before checking the host correctly receives the 
+! declare target Fortran array and writing some values to
+! it before checking the host correctly receives the
 ! correct updates made on the device.
 ! REQUIRES: flang
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -35,7 +35,7 @@ program main
         i = i + 1
     end do
 !$omp end target
-    
+
 print *, sp(:)
 
 end program
diff --git a/offload/test/offloading/fortran/target-map-allocatable-map-scopes.f90 b/offload/test/offloading/fortran/target-map-allocatable-map-scopes.f90
index bb47d3de96d2..506976b339df 100644
--- a/offload/test/offloading/fortran/target-map-allocatable-map-scopes.f90
+++ b/offload/test/offloading/fortran/target-map-allocatable-map-scopes.f90
@@ -11,55 +11,55 @@
 ! RUN: %libomptarget-compile-fortran-run-and-check-generic
 module test
     contains
-  subroutine func_arg(arg_alloc) 
+  subroutine func_arg(arg_alloc)
     integer,  allocatable, intent (inout) :: arg_alloc(:)
-  
-  !$omp target map(tofrom: arg_alloc) 
+
+  !$omp target map(tofrom: arg_alloc)
     do index = 1, 10
       arg_alloc(index) = arg_alloc(index) + index
     end do
   !$omp end target
-    
+
     print *, arg_alloc
   end subroutine func_arg
-end module 
-  
-subroutine func 
-    integer,  allocatable :: local_alloc(:) 
+end module
+
+subroutine func
+    integer,  allocatable :: local_alloc(:)
     allocate(local_alloc(10))
-  
-  !$omp target map(tofrom: local_alloc) 
+
+  !$omp target map(tofrom: local_alloc)
     do index = 1, 10
       local_alloc(index) = index
     end do
   !$omp end target
-  
+
     print *, local_alloc
-  
+
     deallocate(local_alloc)
-end subroutine func 
-  
-  
-program main 
-  use test 
-  integer,  allocatable :: map_ptr(:) 
-  
+end subroutine func
+
+
+program main
+  use test
+  integer,  allocatable :: map_ptr(:)
+
   allocate(map_ptr(10))
-  
-  !$omp target map(tofrom: map_ptr) 
+
+  !$omp target map(tofrom: map_ptr)
     do index = 1, 10
       map_ptr(index) = index
     end do
   !$omp end target
 
-   call func 
+   call func
 
    print *, map_ptr
 
    call func_arg(map_ptr)
 
    deallocate(map_ptr)
-end program 
+end program
 
 ! CHECK: 1 2 3 4 5 6 7 8 9 10
 ! CHECK: 1 2 3 4 5 6 7 8 9 10
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-1.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
index cb03708554fe..f4e11f8b9e32 100644
--- a/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-1.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of an
-! explicit derived type mapping when mapped 
+! explicit derived type mapping when mapped
 ! to target and assinging one derived type
 ! to another
 ! REQUIRES: flang, amdgcn-amd-amdhsa
@@ -16,22 +16,22 @@ program main
     integer(4) :: ix = 0
     real(4) :: rx = 0.0
     complex(4) :: zx = (0,0)
-    end type scalar  
-  
+    end type scalar
+
     type(scalar) :: in
     type(scalar) :: out
     in%ix = 10
     in%rx = 2.0
     in%zx = (2, 10)
-  
+
   !$omp target map(from:out) map(to:in)
-      out = in 
+      out = in
   !$omp end target
-  
+
     print*, in%ix
     print*, in%rx
     write (*,*) in%zx
-  
+
     print*, out%ix
     print*, out%rx
     write (*,*)  out%zx
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-2.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-2.f90
index 0095b0fdf86a..f97880a2023d 100644
--- a/offload/test/offloading/fortran/target-map-derived-type-full-2.f90
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-2.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of an
-! explicit derived type mapping when mapped to 
+! explicit derived type mapping when mapped to
 ! target and assigning to individual members
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -16,29 +16,29 @@ program main
     real(4) :: rx = 0.0
     complex(4) :: zx = (0,0)
     integer(4) :: array(5)
-    end type scalar 
-  
+    end type scalar
+
     type(scalar) :: out
     type(scalar) :: in
-  
+
     in%ix = 10
     in%rx = 2.0
     in%zx = (2, 10)
-  
+
     do i = 1, 5
       in%array(i) = i
-    end do 
-  
+    end do
+
   !$omp target map(from:out) map(to:in)
     out%ix = in%ix
     out%rx = in%rx
     out%zx = in%zx
-  
+
     do i = 1, 5
       out%array(i) = in%array(i)
-    end do 
+    end do
   !$omp end target
-  
+
     print*, in%ix
     print*, in%rx
     print*, in%array
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
index f57e2c70d155..0f1d58de961c 100644
--- a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-1.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of an
-! implicit derived type mapping when mapped 
+! implicit derived type mapping when mapped
 ! to target and assinging one derived type
 ! to another
 ! REQUIRES: flang, amdgcn-amd-amdhsa
@@ -16,18 +16,18 @@ program main
     integer(4) :: ix = 0
     real(4) :: rx = 0.0
     complex(4) :: zx = (0,0)
-    end type scalar  
-  
+    end type scalar
+
     type(scalar) :: in
     type(scalar) :: out
     in%ix = 10
     in%rx = 2.0
     in%zx = (2, 10)
-  
+
   !$omp target map(from:out)
-      out = in 
+      out = in
   !$omp end target
-  
+
     print*, in%ix
     print*, in%rx
     write (*,*) in%zx
@@ -43,4 +43,3 @@ program main
 !CHECK: 10
 !CHECK: 2.
 !CHECK: (2.,10.)
-  
\ No newline at end of file
diff --git a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90 b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
index 92d3454d462a..a20b644652f8 100644
--- a/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
+++ b/offload/test/offloading/fortran/target-map-derived-type-full-implicit-2.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of an
-! explicit derived type mapping when mapped 
+! explicit derived type mapping when mapped
 ! to target and assinging one derived type
 ! to another
 ! REQUIRES: flang, amdgcn-amd-amdhsa
@@ -17,29 +17,29 @@ program main
     real(4) :: rx = 0.0
     complex(4) :: zx = (0,0)
     integer(4) :: array(5)
-    end type scalar 
-  
+    end type scalar
+
     type(scalar) :: out
     type(scalar) :: in
-  
+
     in%ix = 10
     in%rx = 2.0
     in%zx = (2, 10)
-  
+
     do i = 1, 5
       in%array(i) = i
-    end do 
-  
+    end do
+
   !$omp target
     out%ix = in%ix
     out%rx = in%rx
     out%zx = in%zx
-  
+
     do i = 1, 5
       out%array(i) = in%array(i)
-    end do 
+    end do
   !$omp end target
-  
+
     print*, in%ix
     print*, in%rx
     print*, in%array
diff --git a/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
index 31774be19146..17e1cd9549f0 100644
--- a/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
+++ b/offload/test/offloading/fortran/target-map-double-large-nested-dtype-multi-member.f90
@@ -21,14 +21,14 @@ program main
       integer(4) :: i3
       integer(4) :: j3
       integer(4) :: k3
-    end type bottom_layer2 
+    end type bottom_layer2
 
     type :: middle_layer
      real(4) :: array_i2(10)
      real(4) :: i2
      real(4) :: array_j2(10)
-     type(bottom_layer1) :: nest 
-     type(bottom_layer2) :: nest2 
+     type(bottom_layer1) :: nest
+     type(bottom_layer2) :: nest2
     end type middle_layer
 
     type :: top_layer
@@ -39,18 +39,18 @@ program main
     integer(4) :: k
     type(middle_layer) :: nested
     end type top_layer
-    
+
     type(top_layer) :: top_dtype
     type(top_layer) :: top_dtype2
 
     top_dtype2%nested%nest%i4 = 10
     top_dtype2%nested%nest%j4 = 12
     top_dtype2%nested%nest%k4 = 54
-    
+
     top_dtype2%nested%nest2%i3 = 20
     top_dtype2%nested%nest2%j3 = 40
     top_dtype2%nested%nest2%k3 = 60
-    
+
     top_dtype2%nested%i2 = 200
 
       do i = 1, 10
@@ -64,20 +64,20 @@ program main
 !$omp map(to: top_dtype2%array_i, top_dtype2%nested%nest2%i3, top_dtype2%nested%i2) &
 !$omp map(to: top_dtype2%nested%nest2%k3, top_dtype2%nested%nest2%j3)
     top_dtype%nested%nest%i4 = top_dtype2%nested%nest%i4
-    top_dtype%nested%nest%j4 = top_dtype2%nested%nest%j4 
+    top_dtype%nested%nest%j4 = top_dtype2%nested%nest%j4
     top_dtype%nested%nest%k4 = top_dtype2%nested%nest%k4
-    
+
     top_dtype%nested%nest2%i3 = top_dtype2%nested%nest2%i3
     top_dtype%nested%nest2%j3 = top_dtype2%nested%nest2%j3
     top_dtype%nested%nest2%k3 = top_dtype2%nested%nest2%k3
-    
+
     top_dtype%nested%i2 = top_dtype2%nested%i2
 
     do i = 1, 10
       top_dtype%array_i(i) = top_dtype2%array_i(i)
     end do
 !$omp end target
-  
+
   print *, top_dtype%nested%nest%i4
   print *, top_dtype%nested%nest%j4
   print *, top_dtype%nested%nest%k4
@@ -85,10 +85,10 @@ program main
   print *, top_dtype%nested%nest2%i3
   print *, top_dtype%nested%nest2%j3
   print *, top_dtype%nested%nest2%k3
-  
+
   print *, top_dtype%nested%i2
 
-  print *, top_dtype%array_i 
+  print *, top_dtype%array_i
 end program main
 
 !CHECK: 10.
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
index cecfb9e84a59..d327ebf8c1a9 100644
--- a/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
+++ b/offload/test/offloading/fortran/target-map-double-nested-dtype-array-bounds.f90
@@ -1,6 +1,6 @@
 ! Offloading test checking interaction of two
-! explicit arrau member maps with bounds from 
-! two nested derived types 
+! explicit arrau member maps with bounds from
+! two nested derived types
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
 ! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
@@ -25,20 +25,20 @@ program main
       integer, allocatable :: array_j(:)
       integer(4) :: k
     end type top_layer
-    
+
     type(top_layer) :: top_dtype
     type(top_layer) :: top_dtype2
 
 !$omp target map(tofrom: top_dtype%nested%array_i2(4:8), top_dtype2%nested%array_j2(4:8))
-    do i = 4, 8 
+    do i = 4, 8
       top_dtype%nested%array_i2(i) = i * 2
-    end do 
+    end do
 
-    do i = 4, 8 
+    do i = 4, 8
       top_dtype2%nested%array_j2(i) = i * 2
-    end do 
+    end do
 !$omp end target
-  
+
   print *, top_dtype%nested%array_i2
   print *, top_dtype2%nested%array_j2
 end program main
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
index a8762a0829cc..d80ec8a4e069 100644
--- a/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
+++ b/offload/test/offloading/fortran/target-map-double-nested-dtype-double-array-bounds.f90
@@ -1,6 +1,6 @@
 ! Offloading test checking interaction of two
-! explicit array member maps with array bounds 
-! from two nested derived types 
+! explicit array member maps with array bounds
+! from two nested derived types
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
 ! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
@@ -25,20 +25,20 @@ program main
       integer, allocatable :: array_j(:)
       integer(4) :: k
     end type top_layer
-    
+
     type(top_layer) :: top_dtype
     type(top_layer) :: top_dtype2
 
 !$omp target map(tofrom: top_dtype%nested%array_i2(4:8), top_dtype2%nested%array_j2(4:8))
-    do i = 4, 8 
+    do i = 4, 8
       top_dtype%nested%array_i2(i) = i * 2
-    end do 
+    end do
 
-    do i = 4, 8 
+    do i = 4, 8
       top_dtype2%nested%array_j2(i) = i * 2
-    end do 
+    end do
 !$omp end target
-  
+
   print *, top_dtype%nested%array_i2
   print *, top_dtype2%nested%array_j2
 end program main
diff --git a/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90 b/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
index 9ecb394dbe46..59bcd4cc3834 100644
--- a/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
+++ b/offload/test/offloading/fortran/target-map-double-nested-dtype-single-member.f90
@@ -25,18 +25,18 @@ program main
       integer, allocatable :: array_j(:)
       integer(4) :: k
     end type top_layer
-    
+
     type(top_layer) :: top_dtype
     type(top_layer) :: top_dtype2
 
 !$omp target map(tofrom: top_dtype%nested%array_i2, top_dtype2%nested%array_j2)
-    do i = 1, 10 
+    do i = 1, 10
       top_dtype%nested%array_i2(i) = i * 2
-    end do 
+    end do
 
-    do i = 1, 10 
+    do i = 1, 10
       top_dtype2%nested%array_j2(i) = i * 2
-    end do 
+    end do
 !$omp end target
 
   print *, top_dtype%nested%array_i2
diff --git a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90 b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
index 3b3ec96b9bab..47fb6d210e97 100644
--- a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
+++ b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit-update.f90
@@ -1,6 +1,6 @@
 ! Offloading test checking interaction of an
-! explicit derived type member mapping of 
-! an array with bounds when mapped to 
+! explicit derived type member mapping of
+! an array with bounds when mapped to
 ! target using a combination of update,
 ! enter and exit directives.
 ! REQUIRES: flang, amdgcn-amd-amdhsa
@@ -25,24 +25,24 @@ program main
 
   !$omp target enter data map(to: scalar_arr%array(3:6))
 
-    ! overwrite our target data with an update.  
+    ! overwrite our target data with an update.
     do I = 1, 10
         scalar_arr%array(I) = 10
     end do
 
   !$omp target update to(scalar_arr%array(3:6))
 
-  ! The compiler/runtime is less friendly about read/write out of 
+  ! The compiler/runtime is less friendly about read/write out of
   ! bounds when using enter and exit, we have to specifically loop
   ! over the correct range
    !$omp target
     do i=3,6
         scalar_arr%array(i) = scalar_arr%array(i) + i
     end do
-  !$omp end target 
+  !$omp end target
 
   !$omp target exit data map(from: scalar_arr%array(3:6))
-  
+
   print*, scalar_arr%array
 end program
 
diff --git a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90 b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
index 5f7e9f946826..e92e85ed0bc0 100644
--- a/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
+++ b/offload/test/offloading/fortran/target-map-dtype-arr-bounds-member-enter-exit.f90
@@ -1,7 +1,7 @@
 ! Offloading test checking interaction of an
-! explicit derived type member mapping of 
-! an array with bounds when mapped to 
-! target using a combination of enter and 
+! explicit derived type member mapping of
+! an array with bounds when mapped to
+! target using a combination of enter and
 ! exit directives.
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -24,25 +24,25 @@ program main
     end do
 
     !$omp target enter data map(to: scalar_arr%array(3:6))
-    
+
     ! Shouldn't overwrite data already locked in
-    ! on target via enter, which will then be 
+    ! on target via enter, which will then be
     ! overwritten by our exit
     do I = 1, 10
         scalar_arr%array(I) = 10
     end do
 
-  ! The compiler/runtime is less friendly about read/write out of 
+  ! The compiler/runtime is less friendly about read/write out of
   ! bounds when using enter and exit, we have to specifically loop
   ! over the correct range
    !$omp target
     do i=3,6
         scalar_arr%array(i) = scalar_arr%array(i) + i
     end do
-  !$omp end target 
+  !$omp end target
 
   !$omp target exit data map(from: scalar_arr%array(3:6))
-  
+
   print*, scalar_arr%array
 end program
 
diff --git a/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90 b/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
index 907b16ffedf5..97b804e6d1d9 100644
--- a/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
+++ b/offload/test/offloading/fortran/target-map-dtype-explicit-individual-array-member.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of an
-! explicit derived type member mapping of 
+! explicit derived type member mapping of
 ! an array when mapped to target
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -18,9 +18,9 @@ type :: scalar_array
     real(4) :: array_y(10)
     real(4) :: break_3
 end type scalar_array
-  
+
    type(scalar_array) :: scalar_arr
-    
+
   !$omp target map(tofrom:scalar_arr%array_y)
     do i = 1, 10
       scalar_arr%array_y(i) = i
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
index 110fb648980c..44727bec73c0 100644
--- a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-3D-member-bounds.f90
@@ -1,6 +1,6 @@
 ! Offloading test checking interaction of an
-! explicit derived type member mapping of 
-! two arrays with explicit bounds when 
+! explicit derived type member mapping of
+! two arrays with explicit bounds when
 ! mapped to target
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -19,9 +19,9 @@ program main
     integer(4) :: array_y(3,3,3)
     real(4) :: break_3
     end type scalar_array
-  
+
     type(scalar_array) :: scalar_arr
-    
+
     do i = 1, 3
       do j = 1, 3
         do k = 1, 3
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
index b7f6e2ddfb3b..fddc956e3fd5 100644
--- a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member-bounds.f90
@@ -1,6 +1,6 @@
 ! Offloading test checking interaction of an
-! explicit derived type member mapping of 
-! two arrays with explicit bounds when 
+! explicit derived type member mapping of
+! two arrays with explicit bounds when
 ! mapped to target
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -19,9 +19,9 @@ program main
     real(4) :: array_y(10)
     real(4) :: break_3
     end type scalar_array
-  
+
     type(scalar_array) :: scalar_arr
-    
+
   do i = 1, 10
     scalar_arr%array_x(i) = i
   end do
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
index c44a58dbebc8..2f74b28a67f6 100644
--- a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-array-member.f90
@@ -18,9 +18,9 @@ program main
     real(4) :: array_y(10)
     real(4) :: break_3
     end type scalar_array
-  
+
     type(scalar_array) :: scalar_arr
-    
+
   do i = 1, 10
     scalar_arr%array_x(i) = i
   end do
diff --git a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90 b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
index a4205bc0179e..9f2c185aa1ad 100644
--- a/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
+++ b/offload/test/offloading/fortran/target-map-dtype-multi-explicit-member.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of an
-! derived type mapping of two explicit 
+! derived type mapping of two explicit
 ! members to target
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -11,23 +11,23 @@
 
 ! RUN: %libomptarget-compile-fortran-run-and-check-generic
 program main
-    type :: scalar   
-      integer(4) :: ix = 0  
+    type :: scalar
+      integer(4) :: ix = 0
       real(4) :: rx = 0.0
       complex(4) :: zx = (0,0)
       real(4) :: ry = 1.0
-    end type scalar  
-    
+    end type scalar
+
       type(scalar) :: scalar_struct
-      
+
     !$omp target map(from:scalar_struct%rx, scalar_struct%ry)
       scalar_struct%rx = 21.0
       scalar_struct%ry = 27.0
     !$omp end target
-  
+
     print*, scalar_struct%rx
     print*, scalar_struct%ry
 end program main
-  
+
 !CHECK: 21.
 !CHECK: 27.
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-allocatables.f90 b/offload/test/offloading/fortran/target-map-enter-exit-allocatables.f90
index 865be95ba968..c14b23f6170d 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-allocatables.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-allocatables.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of allocatables
-! with enter, exit and target 
+! with enter, exit and target
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
 ! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
@@ -10,9 +10,9 @@
 
 ! RUN: %libomptarget-compile-fortran-run-and-check-generic
 program main
-    integer, allocatable :: A(:)    
+    integer, allocatable :: A(:)
     allocate(A(10))
-    
+
    !$omp target enter data map(alloc: A)
 
     !$omp target
@@ -24,11 +24,11 @@ program main
     !$omp target exit data map(from: A)
 
     !$omp target exit data map(delete: A)
-    
+
     do i = 1, 10
         print *, A(i)
     end do
-    
+
     deallocate(A)
 end program
 
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90 b/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
index 8d35a281caf9..fe65d6a00cbb 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-array-2.f90
@@ -18,7 +18,7 @@ program main
 
     !$omp target enter data map(to: array)
     ! Shouldn't overwrite data already locked in
-    ! on target via enter, this will then be 
+    ! on target via enter, this will then be
     ! overwritten by our exit
     do I = 1, 10
       array(I) = 10
@@ -28,7 +28,7 @@ program main
     do i=1,10
       array(i) = array(i) + i
     end do
-  !$omp end target 
+  !$omp end target
 
   !$omp target exit data map(from: array)
   print*, array
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 b/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
index d842cd15c65a..dfb5d59273c6 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-array-bounds.f90
@@ -20,20 +20,20 @@ program main
 
     !$omp target enter data map(to: array(3:6))
     ! Shouldn't overwrite data already locked in
-    ! on target via enter, which will then be 
+    ! on target via enter, which will then be
     ! overwritten by our exit
     do I = 1, 10
       array(I) = 10
     end do
 
-  ! The compiler/runtime is less lenient about read/write out of 
+  ! The compiler/runtime is less lenient about read/write out of
   ! bounds when using enter and exit, we have to specifically loop
   ! over the correctly mapped range
    !$omp target
     do i=3,6
       array(i) = array(i) + i
     end do
-  !$omp end target 
+  !$omp end target
 
   !$omp target exit data map(from: array(3:6))
   print *, array
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-array.f90 b/offload/test/offloading/fortran/target-map-enter-exit-array.f90
index 4a9fb6ee177f..341c36647c4e 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-array.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-array.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of fixed size
-! arrays with enter, exit and target 
+! arrays with enter, exit and target
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
 ! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
@@ -11,7 +11,7 @@
 ! RUN: %libomptarget-compile-fortran-run-and-check-generic
 program main
     integer :: A(10)
-    
+
    !$omp target enter data map(alloc: A)
 
     !$omp target
@@ -23,7 +23,7 @@ program main
     !$omp target exit data map(from: A)
 
     !$omp target exit data map(delete: A)
-    
+
     do i = 1, 10
         print *, A(i)
     end do
diff --git a/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90 b/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
index 70ed3d747829..f2d7099c041c 100644
--- a/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
+++ b/offload/test/offloading/fortran/target-map-enter-exit-scalar.f90
@@ -19,12 +19,12 @@ program main
 
    !$omp target
       scalar = scalar + 50
-   !$omp end target 
+   !$omp end target
 
   !$omp target exit data map(from: scalar)
 
-  ! not the answer one may expect, but it is the same 
-  ! answer Clang gives so we are correctly on par with 
+  ! not the answer one may expect, but it is the same
+  ! answer Clang gives so we are correctly on par with
   ! Clang for the moment.
   print *, scalar
 end program
diff --git a/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90 b/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
index 4cdf41db70d4..9a65053fd145 100644
--- a/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
+++ b/offload/test/offloading/fortran/target-map-individual-dtype-member-map.f90
@@ -17,8 +17,8 @@ program main
         real(4) :: rx = 0.0
         complex(4) :: zx = (0,0)
         real(4) :: ry = 1.0
-    end type scalar  
-  
+    end type scalar
+
     type(scalar) :: scalar_struct
     scalar_struct%rx = 2.0
     test = 21.0
diff --git a/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
index 2412381e62e7..578b88504604 100644
--- a/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
+++ b/offload/test/offloading/fortran/target-map-large-nested-dtype-multi-member.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of an
-! explicit member map a large nested derived 
+! explicit member map a large nested derived
 ! type
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -21,14 +21,14 @@ program main
       integer(4) :: i3
       integer(4) :: j3
       integer(4) :: k3
-    end type bottom_layer2 
+    end type bottom_layer2
 
     type :: middle_layer
      real(4) :: array_i2(10)
      real(4) :: i2
      real(4) :: array_j2(10)
-     type(bottom_layer1) :: nest 
-     type(bottom_layer2) :: nest2 
+     type(bottom_layer1) :: nest
+     type(bottom_layer2) :: nest2
     end type middle_layer
 
     type :: top_layer
@@ -39,7 +39,7 @@ program main
     integer(4) :: k
     type(middle_layer) :: nested
     end type top_layer
-    
+
     type(top_layer) :: top_dtype
 
     top_dtype%nested%nest%j4 = 12
@@ -48,18 +48,18 @@ program main
     top_dtype%nested%nest%i4 = 10
     top_dtype%nested%nest%j4 = 12 + top_dtype%nested%nest%j4
     top_dtype%nested%nest%k4 = 54
-    
+
     top_dtype%nested%nest2%i3 = 20
     top_dtype%nested%nest2%j3 = 40
     top_dtype%nested%nest2%k3 = 60
-    
+
     top_dtype%nested%i2 = 200
 
       do i = 1, 10
         top_dtype%array_i(i) = i
       end do
 !$omp end target
-  
+
   print *, top_dtype%nested%nest%i4
   print *, top_dtype%nested%nest%j4
   print *, top_dtype%nested%nest%k4
@@ -67,10 +67,10 @@ program main
   print *, top_dtype%nested%nest2%i3
   print *, top_dtype%nested%nest2%j3
   print *, top_dtype%nested%nest2%k3
-  
+
   print *, top_dtype%nested%i2
 
-  print *, top_dtype%array_i 
+  print *, top_dtype%array_i
 end program main
 
 !CHECK: 10.
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
index 9d29639ca101..e2e34e6b9b5c 100644
--- a/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-complex-member.f90
@@ -27,21 +27,21 @@ program main
       integer(4) :: k
       complex :: l
     end type top_layer
-    
+
     type(top_layer) :: top_dtype
 
 !$omp target map(tofrom: top_dtype%nested%i2, top_dtype%k, top_dtype%nested%j2, top_dtype%nested%array_i2, top_dtype%l)
-    do i = 1, 10 
+    do i = 1, 10
       top_dtype%nested%array_i2(i) = i * 2
-    end do 
+    end do
 
     top_dtype%l = (10,20)
     top_dtype%nested%j2 = (510,210)
-    
+
     top_dtype%nested%i2 = 30.30
     top_dtype%k = 74
 !$omp end target
-  
+
   print *, top_dtype%nested%i2
   print *, top_dtype%k
   print *, top_dtype%nested%array_i2
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
index 1be8cb32c8e6..b94915594a15 100644
--- a/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-derived-member.f90
@@ -1,6 +1,6 @@
 ! Offloading test checking interaction of an
-! nested derived type member map with the 
-! inclusion of an entire nested derived 
+! nested derived type member map with the
+! inclusion of an entire nested derived
 ! type being mapped
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -27,19 +27,19 @@ program main
       integer(4) :: k
       type(bottom_layer) :: nested2
     end type top_layer
-    
+
     type(top_layer) :: top_dtype
 
 !$omp target map(tofrom: top_dtype%k, top_dtype%nested2%array_i2, top_dtype%nested)
-    do i = 1, 10 
+    do i = 1, 10
       top_dtype%nested2%array_i2(i) = i * 2
       top_dtype%nested%array_i2(i) = i * 2
-    end do 
+    end do
 
     top_dtype%nested%i2 = 30.30
     top_dtype%k = 74
 !$omp end target
-  
+
   print *, top_dtype%nested%i2
   print *, top_dtype%k
   print *, top_dtype%nested%array_i2
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
index 6f4d5ad5c15b..c3d230853872 100644
--- a/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-multi-member.f90
@@ -25,18 +25,18 @@ program main
       integer, allocatable :: array_j(:)
       integer(4) :: k
     end type top_layer
-    
+
     type(top_layer) :: top_dtype
 
 !$omp target map(tofrom: top_dtype%nested%i2, top_dtype%k, top_dtype%nested%array_i2)
-    do i = 1, 10 
+    do i = 1, 10
       top_dtype%nested%array_i2(i) = i * 2
-    end do 
+    end do
 
     top_dtype%nested%i2 = 30.30
     top_dtype%k = 74
 !$omp end target
-  
+
   print *, top_dtype%nested%i2
   print *, top_dtype%k
   print *, top_dtype%nested%array_i2
diff --git a/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90 b/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
index 046fc13eb93c..de436edb2ef2 100644
--- a/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
+++ b/offload/test/offloading/fortran/target-map-nested-dtype-single-member.f90
@@ -25,15 +25,15 @@ program main
       integer, allocatable :: array_j(:)
       integer(4) :: k
     end type top_layer
-    
+
     type(top_layer) :: top_dtype
 
 !$omp target map(tofrom: top_dtype%nested%array_i2)
-    do i = 1, 10 
+    do i = 1, 10
       top_dtype%nested%array_i2(i) = i * 2
-    end do 
+    end do
 !$omp end target
-  
+
   print *, top_dtype%nested%array_i2
 end program main
 
diff --git a/offload/test/offloading/fortran/target-map-pointer-scopes-enter-exit.f90 b/offload/test/offloading/fortran/target-map-pointer-scopes-enter-exit.f90
index dee75af06927..e148fb123ecb 100644
--- a/offload/test/offloading/fortran/target-map-pointer-scopes-enter-exit.f90
+++ b/offload/test/offloading/fortran/target-map-pointer-scopes-enter-exit.f90
@@ -11,52 +11,52 @@
 ! RUN: %libomptarget-compile-fortran-run-and-check-generic
 module test
     contains
-  subroutine func_arg(arg_alloc) 
+  subroutine func_arg(arg_alloc)
     integer,  pointer, intent (inout) :: arg_alloc(:)
-    
+
   !$omp target enter data map(alloc: arg_alloc)
-    
-  !$omp target 
+
+  !$omp target
     do index = 1, 10
       arg_alloc(index) = arg_alloc(index) + index
     end do
   !$omp end target
-    
+
   !$omp target exit data map(from: arg_alloc)
-  
+
   !$omp target exit data map(delete: arg_alloc)
-    
+
     print *, arg_alloc
   end subroutine func_arg
-end module 
-  
-subroutine func 
-  integer,  pointer :: local_alloc(:) 
+end module
+
+subroutine func
+  integer,  pointer :: local_alloc(:)
   allocate(local_alloc(10))
 
   !$omp target enter data map(alloc: local_alloc)
 
-  !$omp target 
+  !$omp target
     do index = 1, 10
       local_alloc(index) = index
     end do
   !$omp end target
-  
+
   !$omp target exit data map(from: local_alloc)
 
   !$omp target exit data map(delete: local_alloc)
 
   print *, local_alloc
-    
+
   deallocate(local_alloc)
-end subroutine func 
-  
-  
-program main 
-  use test 
-  integer,  pointer :: map_ptr(:) 
+end subroutine func
+
+
+program main
+  use test
+  integer,  pointer :: map_ptr(:)
   allocate(map_ptr(10))
-  
+
   !$omp target enter data map(alloc: map_ptr)
 
   !$omp target
@@ -64,12 +64,12 @@ program main
       map_ptr(index) = index
     end do
   !$omp end target
-  
+
   !$omp target exit data map(from: map_ptr)
 
   !$omp target exit data map(delete: map_ptr)
 
-  call func 
+  call func
 
   print *, map_ptr
 
diff --git a/offload/test/offloading/fortran/target-map-pointer-target-array-section-3d-bounds.f90 b/offload/test/offloading/fortran/target-map-pointer-target-array-section-3d-bounds.f90
index ff2298cf5dbc..c20fbbc0ff35 100644
--- a/offload/test/offloading/fortran/target-map-pointer-target-array-section-3d-bounds.f90
+++ b/offload/test/offloading/fortran/target-map-pointer-target-array-section-3d-bounds.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of pointer
-! and target with target where 3-D bounds have 
+! and target with target where 3-D bounds have
 ! been specified
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -15,10 +15,10 @@ program main
     integer, pointer :: outArray(:,:,:)
     integer, target :: in(3,3,3)
     integer, target :: out(3,3,3)
-    
+
     inArray => in
     outArray => out
-    
+
     do i = 1, 3
       do j = 1, 3
         do k = 1, 3
diff --git a/offload/test/offloading/fortran/target-map-pointer-target-scopes.f90 b/offload/test/offloading/fortran/target-map-pointer-target-scopes.f90
index d9a7000719f0..4d5b995a7dfb 100644
--- a/offload/test/offloading/fortran/target-map-pointer-target-scopes.f90
+++ b/offload/test/offloading/fortran/target-map-pointer-target-scopes.f90
@@ -1,5 +1,5 @@
 ! Offloading test checking interaction of pointer
-! and target with target across multiple scopes 
+! and target with target across multiple scopes
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
 ! UNSUPPORTED: nvptx64-nvidia-cuda-LTO
@@ -11,48 +11,48 @@
 ! RUN: %libomptarget-compile-fortran-run-and-check-generic
 module test
     contains
-  subroutine func_arg(arg_alloc) 
+  subroutine func_arg(arg_alloc)
     integer,  pointer, intent (inout) :: arg_alloc(:)
-  
-  !$omp target map(tofrom: arg_alloc) 
+
+  !$omp target map(tofrom: arg_alloc)
     do index = 1, 10
       arg_alloc(index) = arg_alloc(index) + index
     end do
   !$omp end target
-    
+
     print *, arg_alloc
   end subroutine func_arg
 end module
-  
-subroutine func 
-   integer,  pointer :: local_alloc(:) 
+
+subroutine func
+   integer,  pointer :: local_alloc(:)
    integer, target :: b(10)
    local_alloc => b
-  
-  !$omp target map(tofrom: local_alloc) 
+
+  !$omp target map(tofrom: local_alloc)
     do index = 1, 10
       local_alloc(index) = index
     end do
   !$omp end target
-  
+
     print *, local_alloc
-  end subroutine func 
-  
-  
-  program main 
-  use test 
-  integer,  pointer :: map_ptr(:) 
+  end subroutine func
+
+
+  program main
+  use test
+  integer,  pointer :: map_ptr(:)
   integer, target :: b(10)
-  
+
   map_ptr => b
-  
-  !$omp target map(tofrom: map_ptr) 
+
+  !$omp target map(tofrom: map_ptr)
     do index = 1, 10
       map_ptr(index) = index
     end do
   !$omp end target
-  
-   call func 
+
+   call func
 
    print *, map_ptr
 
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90 b/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
index b080b437e381..2ca621faf6ae 100644
--- a/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
+++ b/offload/test/offloading/fortran/target-map-two-dtype-explicit-member.f90
@@ -18,10 +18,10 @@ program main
         real(4) :: array_y(10)
         real(4) :: break_3
     end type scalar_array
-  
+
     type(scalar_array) :: scalar_arr1
     type(scalar_array) :: scalar_arr2
-    
+
   !$omp target map(tofrom:scalar_arr1%break_1, scalar_arr2%break_3)
     scalar_arr2%break_3 = 10
     scalar_arr1%break_1 = 15
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90 b/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
index 7cb7846ed0b2..77a5e1bc9e5f 100644
--- a/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
+++ b/offload/test/offloading/fortran/target-map-two-dtype-individual-member-array-1D-bounds.f90
@@ -1,6 +1,6 @@
 ! Offloading test checking interaction of two
 ! derived type's with a single explicit array
-! member each being mapped with bounds to 
+! member each being mapped with bounds to
 ! target
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -19,10 +19,10 @@ program main
     real(4) :: array_y(10)
     real(4) :: break_3
     end type scalar_array
-  
+
     type(scalar_array) :: scalar_arr1
     type(scalar_array) :: scalar_arr2
-    
+
 
   !$omp target map(tofrom:scalar_arr1%array_x(3:6), scalar_arr2%array_x(3:6))
     do i = 3, 6
@@ -31,7 +31,7 @@ program main
     end do
   !$omp end target
 
-  print*, scalar_arr1%array_x  
+  print*, scalar_arr1%array_x
   print*, scalar_arr2%array_x
 end program main
 
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90 b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
index fbe6b305464a..8675f470e1cb 100644
--- a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
+++ b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-1.f90
@@ -18,10 +18,10 @@ program main
         real(4) :: array_y(10)
         real(4) :: break_3
     end type scalar_array
-  
+
     type(scalar_array) :: scalar_arr1
     type(scalar_array) :: scalar_arr2
-    
+
   !$omp target map(tofrom:scalar_arr1%break_1)
     scalar_arr2%break_3 = 10
     scalar_arr1%break_1 = 15
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90 b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
index 503329d90628..d13d9e8068ec 100644
--- a/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
+++ b/offload/test/offloading/fortran/target-map-two-dtype-mixed-implicit-explicit-capture-2.f90
@@ -19,13 +19,13 @@ program main
         real(4) :: array_y(10)
         real(4) :: break_3
     end type scalar_array
-  
+
     type(scalar_array) :: scalar_arr1
     type(scalar_array) :: scalar_arr2
-    
+
   do i = 1, 10
     scalar_arr1%array_x(i) = i
-  end do 
+  end do
 
   !$omp target map(tofrom:scalar_arr2%array_x(3:6))
     do i = 3, 6
diff --git a/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90 b/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
index ed350c54dcb2..b788c6c5bcf2 100644
--- a/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
+++ b/offload/test/offloading/fortran/target-map-two-dtype-multi-member-array-1D-bounds.f90
@@ -1,6 +1,6 @@
 ! Offloading test checking interaction of two
 ! derived type's with two explicit array
-! members each being mapped with bounds to 
+! members each being mapped with bounds to
 ! target
 ! REQUIRES: flang, amdgcn-amd-amdhsa
 ! UNSUPPORTED: nvptx64-nvidia-cuda
@@ -19,10 +19,10 @@ program main
     real(4) :: array_y(10)
     real(4) :: break_3
     end type scalar_array
-  
+
     type(scalar_array) :: scalar_arr1
     type(scalar_array) :: scalar_arr2
-    
+
   do i = 1, 10
     scalar_arr1%array_x(i) = i
     scalar_arr2%array_x(i) = i
@@ -32,7 +32,7 @@ program main
     do i = 1, 10
       scalar_arr2%array_y(i) = scalar_arr1%array_x(i)
     end do
-    
+
     do i = 1, 10
       scalar_arr1%array_y(i) = scalar_arr2%array_x(i)
     end do
diff --git a/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90 b/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90
index 42d9197f6e1e..2114a33e7ba1 100644
--- a/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90
+++ b/offload/test/offloading/fortran/target-map-two-nested-dtype-member-array-map.f90
@@ -24,10 +24,10 @@ program main
     real(4) :: break_3
     type(array) :: nested
     end type scalar_array
-  
+
     type(scalar_array) :: scalar_arr1
     type(scalar_array) :: scalar_arr2
-    
+
   do i = 1, 10
     scalar_arr1%nested%array_z(i) = i
     scalar_arr2%nested%array_z(i) = i
@@ -37,7 +37,7 @@ program main
     do i = 3, 6
       scalar_arr2%nested%array_ix(i) = scalar_arr1%nested%array_z(i)
     end do
-    
+
     do i = 3, 6
       scalar_arr1%nested%array_ix(i) = scalar_arr2%nested%array_z(i)
     end do
diff --git a/offload/test/offloading/spmdization.c b/offload/test/offloading/spmdization.c
index 77913bec8342..ab26456823e4 100644
--- a/offload/test/offloading/spmdization.c
+++ b/offload/test/offloading/spmdization.c
@@ -32,13 +32,19 @@ int main(void) {
     tid = omp_get_thread_num();
     maxt = omp_get_max_threads();
     #pragma omp parallel
-    noop();  
+    noop();
   }
   printf("NumThreads: %i, InParallel: %i, Level: %i, ActiveLevel: %i, Nested: %i, "
          "ThreadNum: %i, MaxThreads: %i\n",
          nthreads, ip, lvl, alvl, nested, tid, maxt);
   // GENERIC: Generic mode
   // SPMD: Generic-SPMD mode
-  // CHECK: NumThreads: 1, InParallel: 0, Level: 0, ActiveLevel: 0, Nested: 0, ThreadNum: 0, MaxThreads: 
+  // CHECK: NumThreads: 1
+  // CHECK: InParallel: 0
+  // CHECK: Level: 0
+  // CHECK: ActiveLevel: 0
+  // CHECK: Nested: 0
+  // CHECK: ThreadNum: 0
+  // CHECK: MaxThreads:
   return 0;
 }
diff --git a/offload/test/offloading/thread_limit.c b/offload/test/offloading/thread_limit.c
index a8cc51b651dc..f18a306614c7 100644
--- a/offload/test/offloading/thread_limit.c
+++ b/offload/test/offloading/thread_limit.c
@@ -16,13 +16,13 @@ int main() {
   int n = 1 << 20;
   int th = 12;
   int te = n / th;
-// DEFAULT: 12 (MaxFlatWorkGroupSize: 
+// DEFAULT: 12 (MaxFlatWorkGroupSize:
 #pragma omp target
 #pragma omp teams loop num_teams(te), thread_limit(th)
   for (int i = 0; i < n; i++) {
   }
 
-// DEFAULT: 13 (MaxFlatWorkGroupSize: 
+// DEFAULT: 13 (MaxFlatWorkGroupSize:
   #pragma omp target
   #pragma omp teams distribute parallel for simd num_teams(te), thread_limit(th+1) simdlen(64)
   for(int i = 0; i < n; i++) {
-- 
GitLab


From 770d92830311b21c82ff12ba85bd131950db23d0 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 17 May 2024 13:24:32 -0500
Subject: [PATCH 1159/1206] [Offload][NFC] Remove 'libomptarget' message
 helpers (#92581)

Summary:
This isn't `libomptarget` anymore, and these messages were always
unnecessary because no other project uses these prefixed messages. The
effect of this is that no longer will the logs have `LIBOMPTARGET --` in
front of everything. We have a message stating when we start building
the offload project so it'll still be trivial to find.
---
 offload/CMakeLists.txt                        |  6 ++----
 offload/DeviceRTL/CMakeLists.txt              | 12 ++++++------
 offload/cmake/Modules/LibomptargetUtils.cmake | 17 -----------------
 offload/plugins-nextgen/amdgpu/CMakeLists.txt |  8 ++++----
 offload/plugins-nextgen/cuda/CMakeLists.txt   | 12 ++++++------
 offload/plugins-nextgen/host/CMakeLists.txt   |  8 ++++----
 offload/src/CMakeLists.txt                    |  2 +-
 offload/test/CMakeLists.txt                   |  2 +-
 offload/tools/deviceinfo/CMakeLists.txt       |  2 +-
 offload/tools/kernelreplay/CMakeLists.txt     |  2 +-
 offload/unittests/Plugins/CMakeLists.txt      |  2 +-
 11 files changed, 27 insertions(+), 46 deletions(-)
 delete mode 100644 offload/cmake/Modules/LibomptargetUtils.cmake

diff --git a/offload/CMakeLists.txt b/offload/CMakeLists.txt
index 4e552e058959..c3dcebfb7301 100644
--- a/offload/CMakeLists.txt
+++ b/offload/CMakeLists.txt
@@ -118,9 +118,6 @@ else()
   set(LIBOMPTARGET_INTDIR ${LLVM_LIBRARY_OUTPUT_INTDIR})
 endif()
 
-# Message utilities.
-include(LibomptargetUtils)
-
 # Get dependencies for the different components of the project.
 include(LibomptargetGetDependencies)
 
@@ -146,7 +143,8 @@ set(LIBOMPTARGET_PLUGINS_TO_BUILD "all" CACHE STRING
 if(LIBOMPTARGET_PLUGINS_TO_BUILD STREQUAL "all")
   set(LIBOMPTARGET_PLUGINS_TO_BUILD ${LIBOMPTARGET_ALL_PLUGIN_TARGETS})
 endif()
-message("Building with support for ${LIBOMPTARGET_PLUGINS_TO_BUILD} plugins")
+message(STATUS "Building the offload library with support for "
+               "the \"${LIBOMPTARGET_PLUGINS_TO_BUILD}\" plugins")
 
 set(LIBOMPTARGET_ENUM_PLUGIN_TARGETS "")
 foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD)
diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt
index 0c9b77ded5ab..b35600367276 100644
--- a/offload/DeviceRTL/CMakeLists.txt
+++ b/offload/DeviceRTL/CMakeLists.txt
@@ -2,13 +2,13 @@ set(LIBOMPTARGET_BUILD_DEVICERTL_BCLIB TRUE CACHE BOOL
   "Can be set to false to disable building this library.")
 
 if (NOT LIBOMPTARGET_BUILD_DEVICERTL_BCLIB)
-  libomptarget_say("Not building DeviceRTL: Disabled by LIBOMPTARGET_BUILD_DEVICERTL_BCLIB")
+  message(STATUS "Not building DeviceRTL: Disabled by LIBOMPTARGET_BUILD_DEVICERTL_BCLIB")
   return()
 endif()
 
 # Check to ensure the host system is a supported host architecture.
 if(NOT ${CMAKE_SIZEOF_VOID_P} EQUAL "8")
-  libomptarget_say("Not building DeviceRTL: Runtime does not support 32-bit hosts")
+  message(STATUS "Not building DeviceRTL: Runtime does not support 32-bit hosts")
   return()
 endif()
 
@@ -20,10 +20,10 @@ if (LLVM_DIR)
   find_program(LINK_TOOL llvm-link PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
   find_program(OPT_TOOL opt PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
   if ((NOT CLANG_TOOL) OR (NOT LINK_TOOL) OR (NOT OPT_TOOL) OR (NOT PACKAGER_TOOL))
-    libomptarget_say("Not building DeviceRTL. Missing clang: ${CLANG_TOOL}, llvm-link: ${LINK_TOOL}, opt: ${OPT_TOOL}, or clang-offload-packager: ${PACKAGER_TOOL}")
+    message(STATUS "Not building DeviceRTL. Missing clang: ${CLANG_TOOL}, llvm-link: ${LINK_TOOL}, opt: ${OPT_TOOL}, or clang-offload-packager: ${PACKAGER_TOOL}")
     return()
   else()
-    libomptarget_say("Building DeviceRTL. Using clang: ${CLANG_TOOL}, llvm-link: ${LINK_TOOL} and opt: ${OPT_TOOL}")
+    message(STATUS "Building DeviceRTL. Using clang: ${CLANG_TOOL}, llvm-link: ${LINK_TOOL} and opt: ${OPT_TOOL}")
   endif()
 elseif (LLVM_TOOL_CLANG_BUILD AND NOT CMAKE_CROSSCOMPILING AND NOT OPENMP_STANDALONE_BUILD)
   # LLVM in-tree builds may use CMake target names to discover the tools.
@@ -32,9 +32,9 @@ elseif (LLVM_TOOL_CLANG_BUILD AND NOT CMAKE_CROSSCOMPILING AND NOT OPENMP_STANDA
   set(PACKAGER_TOOL $<TARGET_FILE:clang-offload-packager>)
   set(LINK_TOOL $<TARGET_FILE:llvm-link>)
   set(OPT_TOOL $<TARGET_FILE:opt>)
-  libomptarget_say("Building DeviceRTL. Using clang from in-tree build")
+  message(STATUS "Building DeviceRTL. Using clang from in-tree build")
 else()
-  libomptarget_say("Not building DeviceRTL. No appropriate clang found")
+  message(STATUS "Not building DeviceRTL. No appropriate clang found")
   return()
 endif()
 
diff --git a/offload/cmake/Modules/LibomptargetUtils.cmake b/offload/cmake/Modules/LibomptargetUtils.cmake
deleted file mode 100644
index 57682dc8e367..000000000000
--- a/offload/cmake/Modules/LibomptargetUtils.cmake
+++ /dev/null
@@ -1,17 +0,0 @@
-# void libomptarget_say(string message_to_user);
-# - prints out message_to_user
-macro(libomptarget_say message_to_user)
-  message(STATUS "LIBOMPTARGET: ${message_to_user}")
-endmacro()
-
-# void libomptarget_warning_say(string message_to_user);
-# - prints out message_to_user with a warning
-macro(libomptarget_warning_say message_to_user)
-  message(WARNING "LIBOMPTARGET: ${message_to_user}")
-endmacro()
-
-# void libomptarget_error_say(string message_to_user);
-# - prints out message_to_user with an error and exits cmake
-macro(libomptarget_error_say message_to_user)
-  message(FATAL_ERROR "LIBOMPTARGET: ${message_to_user}")
-endmacro()
diff --git a/offload/plugins-nextgen/amdgpu/CMakeLists.txt b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
index 5f72568e9a53..2f4057c0ae7e 100644
--- a/offload/plugins-nextgen/amdgpu/CMakeLists.txt
+++ b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
@@ -2,7 +2,7 @@
 find_package(hsa-runtime64 QUIET 1.2.0 HINTS ${CMAKE_INSTALL_PREFIX} PATHS /opt/rocm)
 
 if(NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)|(aarch64)$" AND CMAKE_SYSTEM_NAME MATCHES "Linux"))
-  libomptarget_say("Not building AMDGPU NextGen plugin: only support AMDGPU in Linux x86_64, ppc64le, or aarch64 hosts")
+  message(STATUS "Not building AMDGPU NextGen plugin: only support AMDGPU in Linux x86_64, ppc64le, or aarch64 hosts")
   return()
 endif()
 
@@ -15,10 +15,10 @@ target_include_directories(omptarget.rtl.amdgpu PRIVATE
 
 option(LIBOMPTARGET_FORCE_DLOPEN_LIBHSA "Build with dlopened libhsa" OFF)
 if(hsa-runtime64_FOUND AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBHSA)
-  libomptarget_say("Building AMDGPU plugin linked against libhsa")
+  message(STATUS "Building AMDGPU plugin linked against libhsa")
   target_link_libraries(omptarget.rtl.amdgpu PRIVATE hsa-runtime64::hsa-runtime64)
 else()
-  libomptarget_say("Building AMDGPU plugin for dlopened libhsa")
+  message(STATUS "Building AMDGPU plugin for dlopened libhsa")
   target_include_directories(omptarget.rtl.amdgpu PRIVATE dynamic_hsa)
   target_sources(omptarget.rtl.amdgpu PRIVATE dynamic_hsa/hsa.cpp)
 endif()
@@ -33,6 +33,6 @@ if (LIBOMPTARGET_FOUND_AMDGPU_GPU OR LIBOMPTARGET_FORCE_AMDGPU_TESTS)
   list(APPEND LIBOMPTARGET_TESTED_PLUGINS "omptarget.rtl.amdgpu")
   set(LIBOMPTARGET_TESTED_PLUGINS "${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE)
 else()
-  libomptarget_say("Not generating AMDGPU tests, no supported devices detected."
+  message(STATUS "Not generating AMDGPU tests, no supported devices detected."
                    " Use 'LIBOMPTARGET_FORCE_AMDGPU_TESTS' to override.")
 endif()
diff --git a/offload/plugins-nextgen/cuda/CMakeLists.txt b/offload/plugins-nextgen/cuda/CMakeLists.txt
index 51507e7537a4..10ff612848ad 100644
--- a/offload/plugins-nextgen/cuda/CMakeLists.txt
+++ b/offload/plugins-nextgen/cuda/CMakeLists.txt
@@ -1,9 +1,9 @@
 if (NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)|(aarch64)$" AND CMAKE_SYSTEM_NAME MATCHES "Linux"))
-  libomptarget_say("Not building CUDA NextGen offloading plugin: only support CUDA in Linux x86_64, ppc64le, or aarch64 hosts.")
+  message(STATUS "Not building CUDA NextGen offloading plugin: only support CUDA in Linux x86_64, ppc64le, or aarch64 hosts.")
   return()
 endif()
 
-libomptarget_say("Building CUDA NextGen offloading plugin.")
+message(STATUS "Building CUDA NextGen offloading plugin.")
 
 # Create the library and add the default arguments.
 add_target_library(omptarget.rtl.cuda CUDA)
@@ -12,10 +12,10 @@ target_sources(omptarget.rtl.cuda PRIVATE src/rtl.cpp)
 
 option(LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA "Build with dlopened libcuda" OFF)
 if(LIBOMPTARGET_DEP_CUDA_FOUND AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA)
-  libomptarget_say("Building CUDA plugin linked against libcuda")
+  message(STATUS "Building CUDA plugin linked against libcuda")
   target_link_libraries(omptarget.rtl.cuda PRIVATE CUDA::cuda_driver)
 else()
-  libomptarget_say("Building CUDA plugin for dlopened libcuda")
+  message(STATUS "Building CUDA plugin for dlopened libcuda")
   target_include_directories(omptarget.rtl.cuda PRIVATE dynamic_cuda)
   target_sources(omptarget.rtl.cuda PRIVATE dynamic_cuda/cuda.cpp)
 endif()
@@ -24,12 +24,12 @@ endif()
 # functional NVIDIA GPU on the system, or if manually specifies by the user.
 option(LIBOMPTARGET_FORCE_NVIDIA_TESTS "Build NVIDIA libomptarget tests" OFF)
 if (LIBOMPTARGET_FOUND_NVIDIA_GPU OR LIBOMPTARGET_FORCE_NVIDIA_TESTS)
-  libomptarget_say("Enable tests using CUDA plugin")
+  message(STATUS "Enable tests using CUDA plugin")
   set(LIBOMPTARGET_SYSTEM_TARGETS
       "${LIBOMPTARGET_SYSTEM_TARGETS} nvptx64-nvidia-cuda nvptx64-nvidia-cuda-LTO" PARENT_SCOPE)
   list(APPEND LIBOMPTARGET_TESTED_PLUGINS "omptarget.rtl.cuda")
   set(LIBOMPTARGET_TESTED_PLUGINS "${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE)
 else()
-  libomptarget_say("Not generating NVIDIA tests, no supported devices detected."
+  message(STATUS "Not generating NVIDIA tests, no supported devices detected."
                    " Use 'LIBOMPTARGET_FORCE_NVIDIA_TESTS' to override.")
 endif()
diff --git a/offload/plugins-nextgen/host/CMakeLists.txt b/offload/plugins-nextgen/host/CMakeLists.txt
index d35aadc56aa8..9c6aa274921b 100644
--- a/offload/plugins-nextgen/host/CMakeLists.txt
+++ b/offload/plugins-nextgen/host/CMakeLists.txt
@@ -4,7 +4,7 @@ endif()
 
 set(supported_targets x86_64 aarch64 ppc64 ppc64le s390x)
 if(NOT ${CMAKE_SYSTEM_PROCESSOR} IN_LIST supported_targets)
-  libomptarget_say("Not building ${machine} NextGen offloading plugin")
+  message(STATUS "Not building ${machine} NextGen offloading plugin")
   return()
 endif()
 
@@ -19,14 +19,14 @@ add_target_library(omptarget.rtl.host ${machine})
 target_sources(omptarget.rtl.host PRIVATE src/rtl.cpp)
 
 if(LIBOMPTARGET_DEP_LIBFFI_FOUND)
-  libomptarget_say("Building ${machine} plugin linked with libffi")
+  message(STATUS "Building ${machine} plugin linked with libffi")
   if(FFI_STATIC_LIBRARIES)
     target_link_libraries(omptarget.rtl.host PRIVATE FFI::ffi_static)
   else()
     target_link_libraries(omptarget.rtl.host PRIVATE FFI::ffi)
   endif()
 else()
-  libomptarget_say("Building ${machine} plugin for dlopened libffi")
+  message(STATUS "Building ${machine} plugin for dlopened libffi")
   target_sources(omptarget.rtl.host PRIVATE dynamic_ffi/ffi.cpp)
   target_include_directories(omptarget.rtl.host PRIVATE dynamic_ffi)
 endif()
@@ -39,7 +39,7 @@ if(LIBOMPTARGET_DEP_LIBFFI_FOUND)
   set(LIBOMPTARGET_TESTED_PLUGINS
       "${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE)
 else()
-  libomptarget_say("Not generating ${tmachine_name} tests. LibFFI not found.")
+  message(STATUS "Not generating ${tmachine_name} tests. LibFFI not found.")
 endif()
 
 # Define the target specific triples and ELF machine values.
diff --git a/offload/src/CMakeLists.txt b/offload/src/CMakeLists.txt
index c54bcbe7ed26..efa5cdab33ec 100644
--- a/offload/src/CMakeLists.txt
+++ b/offload/src/CMakeLists.txt
@@ -1,4 +1,4 @@
-libomptarget_say("Building offloading runtime library libomptarget.")
+message(STATUS "Building offloading runtime library libomptarget.")
 
 if(LIBOMP_STANDALONE)
   set(LIBOMP ${LIBOMP_STANDALONE})
diff --git a/offload/test/CMakeLists.txt b/offload/test/CMakeLists.txt
index 59c9dd98f712..58cd56911f24 100644
--- a/offload/test/CMakeLists.txt
+++ b/offload/test/CMakeLists.txt
@@ -1,7 +1,7 @@
 # CMakeLists.txt file for unit testing OpenMP offloading runtime library.
 if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR
    CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0.0)
-  libomptarget_say("Can only test with Clang compiler in version 6.0.0 or later.")
+  message(STATUS "Can only test with Clang compiler in version 6.0.0 or later.")
   libomptarget_warning_say("The check-libomptarget target will not be available!")
   return()
 endif()
diff --git a/offload/tools/deviceinfo/CMakeLists.txt b/offload/tools/deviceinfo/CMakeLists.txt
index b83f02fd6eb9..a39de8833bf1 100644
--- a/offload/tools/deviceinfo/CMakeLists.txt
+++ b/offload/tools/deviceinfo/CMakeLists.txt
@@ -1,4 +1,4 @@
-libomptarget_say("Building the llvm-omp-device-info tool")
+message(STATUS "Building the llvm-omp-device-info tool")
 
 add_openmp_tool(llvm-omp-device-info llvm-omp-device-info.cpp)
 
diff --git a/offload/tools/kernelreplay/CMakeLists.txt b/offload/tools/kernelreplay/CMakeLists.txt
index ae0c5caac4c9..690edd6d5a68 100644
--- a/offload/tools/kernelreplay/CMakeLists.txt
+++ b/offload/tools/kernelreplay/CMakeLists.txt
@@ -1,4 +1,4 @@
-libomptarget_say("Building the llvm-omp-kernel-replay tool")
+message(STATUS "Building the llvm-omp-kernel-replay tool")
 
 add_openmp_tool(llvm-omp-kernel-replay llvm-omp-kernel-replay.cpp)
 
diff --git a/offload/unittests/Plugins/CMakeLists.txt b/offload/unittests/Plugins/CMakeLists.txt
index 67d4d9b463b0..06e5288ad6f6 100644
--- a/offload/unittests/Plugins/CMakeLists.txt
+++ b/offload/unittests/Plugins/CMakeLists.txt
@@ -3,7 +3,7 @@ set(PLUGINS_TEST_SOURCES NextgenPluginsTest.cpp)
 set(PLUGINS_TEST_INCLUDE ${LIBOMPTARGET_INCLUDE_DIR})
 
 foreach(PLUGIN IN LISTS LIBOMPTARGET_TESTED_PLUGINS)
-  libomptarget_say("Building plugin unit tests for ${PLUGIN}")
+  message(STATUS "Building plugin unit tests for ${PLUGIN}")
   add_libompt_unittest("${PLUGIN}.unittests" ${PLUGINS_TEST_SOURCES})
   add_dependencies("${PLUGIN}.unittests" ${PLUGINS_TEST_COMMON} ${PLUGIN})
   target_link_libraries("${PLUGIN}.unittests" PRIVATE ${PLUGINS_TEST_COMMON} ${PLUGIN})
-- 
GitLab


From 48b23c09c0dd70c4d5645d781caa1e0a7d51e0fc Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 17 May 2024 20:51:27 +0200
Subject: [PATCH 1160/1206] AMDGPU: Handle undef correctly in isKnownIntegral
 (#92566)

---
 llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp     | 20 +++++++---
 .../AMDGPU/amdgpu-simplify-libcall-pow.ll     | 40 +++++++++++++++++++
 2 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
index 978a2d49b08b..faf04c3c7e70 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
@@ -577,18 +577,26 @@ bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
 
 static bool isKnownIntegral(const Value *V, const DataLayout &DL,
                             FastMathFlags FMF) {
-  if (isa<UndefValue>(V))
+  if (isa<PoisonValue>(V))
     return true;
+  if (isa<UndefValue>(V))
+    return false;
 
   if (const ConstantFP *CF = dyn_cast<ConstantFP>(V))
     return CF->getValueAPF().isInteger();
 
-  if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(V)) {
-    for (unsigned i = 0, e = CDV->getNumElements(); i != e; ++i) {
-      Constant *ConstElt = CDV->getElementAsConstant(i);
-      if (isa<UndefValue>(ConstElt))
+  auto *VFVTy = dyn_cast<FixedVectorType>(V->getType());
+  const Constant *CV = dyn_cast<Constant>(V);
+  if (VFVTy && CV) {
+    unsigned NumElts = VFVTy->getNumElements();
+    for (unsigned i = 0; i != NumElts; ++i) {
+      Constant *Elt = CV->getAggregateElement(i);
+      if (!Elt)
+        return false;
+      if (isa<PoisonValue>(Elt))
         continue;
-      const ConstantFP *CFP = dyn_cast<ConstantFP>(ConstElt);
+
+      const ConstantFP *CFP = dyn_cast<ConstantFP>(Elt);
       if (!CFP || !CFP->getValue().isInteger())
         return false;
     }
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
index 5db25a59d33f..6b4b0f881f3b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow.ll
@@ -2673,6 +2673,46 @@ define float @test_pow_f32__y_known_integral_roundeven(float %x, float nofpclass
   ret float %pow
 }
 
+define float @test_pow_f32_known_integral_undef(float %x) {
+; CHECK-LABEL: define float @test_pow_f32_known_integral_undef
+; CHECK-SAME: (float [[X:%.*]]) {
+; CHECK-NEXT:    [[POW:%.*]] = tail call float @_Z3powff(float [[X]], float undef)
+; CHECK-NEXT:    ret float [[POW]]
+;
+  %pow = tail call float @_Z3powff(float %x, float undef)
+  ret float %pow
+}
+
+define float @test_pow_f32_known_integral_poison(float %x) {
+; CHECK-LABEL: define float @test_pow_f32_known_integral_poison
+; CHECK-SAME: (float [[X:%.*]]) {
+; CHECK-NEXT:    [[POW:%.*]] = tail call float @_Z4pownfi(float [[X]], i32 poison)
+; CHECK-NEXT:    ret float [[POW]]
+;
+  %pow = tail call float @_Z3powff(float %x, float poison)
+  ret float %pow
+}
+
+define <2 x float> @test_pow_v2f32_known_integral_constant_vector_undef_elt(<2 x float> %x) {
+; CHECK-LABEL: define <2 x float> @test_pow_v2f32_known_integral_constant_vector_undef_elt
+; CHECK-SAME: (<2 x float> [[X:%.*]]) {
+; CHECK-NEXT:    [[POW:%.*]] = tail call <2 x float> @_Z3powDv2_fS_(<2 x float> [[X]], <2 x float> <float 4.000000e+00, float undef>)
+; CHECK-NEXT:    ret <2 x float> [[POW]]
+;
+  %pow = tail call <2 x float> @_Z3powDv2_fS_(<2 x float> %x, <2 x float> <float 4.0, float undef>)
+  ret <2 x float> %pow
+}
+
+define <2 x float> @test_pow_v2f32_known_integral_constant_vector_poison_elt(<2 x float> %x) {
+; CHECK-LABEL: define <2 x float> @test_pow_v2f32_known_integral_constant_vector_poison_elt
+; CHECK-SAME: (<2 x float> [[X:%.*]]) {
+; CHECK-NEXT:    [[POW:%.*]] = tail call <2 x float> @_Z4pownDv2_fDv2_i(<2 x float> [[X]], <2 x i32> <i32 4, i32 poison>)
+; CHECK-NEXT:    ret <2 x float> [[POW]]
+;
+  %pow = tail call <2 x float> @_Z3powDv2_fS_(<2 x float> %x, <2 x float> <float 4.0, float poison>)
+  ret <2 x float> %pow
+}
+
 attributes #0 = { minsize }
 attributes #1 = { noinline }
 attributes #2 = { strictfp }
-- 
GitLab


From 1e5f29af81a5f6fda308074f6345b9fba4faa71c Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 17 May 2024 11:55:21 -0700
Subject: [PATCH 1161/1206] Revert "[MCAsmParser] .rept/.irp/.irpc: remove
 excess tail EOL in expansion"

This reverts commit c6e787f771d1f9d6a846b2d9b8db6adcd87e8dba.

parseEOL() would remove \n # after .endr, not recognizing the line marker.

```
// reduced from Linux kernel arch/x86/crypto/sha1_avx2_x86_64_asm.S
.rept 1
nop
.endr
# 512 "a.s"
```
---
 llvm/lib/MC/MCParser/AsmParser.cpp  | 31 ++++++++++++++++++-----------
 llvm/test/MC/AsmParser/macro-rept.s | 14 ++++++-------
 2 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 56da5e2f9a40..ce155be9c1c7 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -5624,20 +5624,27 @@ MCAsmMacro *AsmParser::parseMacroLikeBody(SMLoc DirectiveLoc) {
       return nullptr;
     }
 
-    if (Lexer.is(AsmToken::Identifier)) {
-      StringRef Ident = getTok().getIdentifier();
-      if (Ident == ".rep" || Ident == ".rept" || Ident == ".irp" ||
-          Ident == ".irpc") {
-        ++NestLevel;
-      } else if (Ident == ".endr") {
-        if (NestLevel == 0) {
-          EndToken = getTok();
-          Lex();
-          if (!parseEOL())
-            break;
+    if (Lexer.is(AsmToken::Identifier) &&
+        (getTok().getIdentifier() == ".rep" ||
+         getTok().getIdentifier() == ".rept" ||
+         getTok().getIdentifier() == ".irp" ||
+         getTok().getIdentifier() == ".irpc")) {
+      ++NestLevel;
+    }
+
+    // Otherwise, check whether we have reached the .endr.
+    if (Lexer.is(AsmToken::Identifier) && getTok().getIdentifier() == ".endr") {
+      if (NestLevel == 0) {
+        EndToken = getTok();
+        Lex();
+        if (Lexer.isNot(AsmToken::EndOfStatement)) {
+          printError(getTok().getLoc(),
+                     "unexpected token in '.endr' directive");
+          return nullptr;
         }
-        --NestLevel;
+        break;
       }
+      --NestLevel;
     }
 
     // Otherwise, scan till the end of the statement.
diff --git a/llvm/test/MC/AsmParser/macro-rept.s b/llvm/test/MC/AsmParser/macro-rept.s
index 2a6a4070bff5..1dc8060e1d87 100644
--- a/llvm/test/MC/AsmParser/macro-rept.s
+++ b/llvm/test/MC/AsmParser/macro-rept.s
@@ -13,10 +13,10 @@
 // CHECK: .long	1
 // CHECK: .long	1
 
-// CHECK:      .long	0
-// CHECK-NEXT: .long	0
-// CHECK-NEXT: .long	0
-// CHECK-NEXT: .long	0
-// CHECK-NEXT: .long	0
-// CHECK-NEXT: .long	0
-// CHECK-EMPTY:
+// CHECK: .long	0
+// CHECK: .long	0
+// CHECK: .long	0
+
+// CHECK: .long	0
+// CHECK: .long	0
+// CHECK: .long	0
-- 
GitLab


From 0ec3b972e58bcbcdc1bebe1696ea37f2931287c3 Mon Sep 17 00:00:00 2001
From: Dan Liew <delcypher@gmail.com>
Date: Fri, 17 May 2024 12:07:40 -0700
Subject: [PATCH 1162/1206] [BoundsSafety] Allow 'counted_by' attribute on
 pointers in structs in C (#90786)

Previously the attribute was only allowed on flexible array members.
This patch patch changes this to also allow the attribute on pointer
fields in structs and also allows late parsing of the attribute in some
contexts.

For example this previously wasn't allowed:

```
struct BufferTypeDeclAttributePosition {
  size_t count;
  char* buffer __counted_by(count); // Now allowed
}
```

Note the attribute is prevented on pointee types where the size isn't
known at compile time. In particular pointee types that are:

* Incomplete (e.g. `void`) and sizeless types
* Function types (e.g. the pointee of a function pointer)
* Struct types with a flexible array member

This patch also introduces late parsing of the attribute when used in
the declaration attribute position. For example

```
struct BufferTypeDeclAttributePosition {
  char* buffer __counted_by(count); // Now allowed
  size_t count;
}
```

is now allowed but **only** when passing
`-fexperimental-late-parse-attributes`. The motivation for using late
parsing here is to avoid breaking the data layout of structs in existing
code that want to use the `counted_by` attribute. This patch is the
first use of `LateAttrParseExperimentalExt` in `Attr.td` that was
introduced in a previous patch.

Note by allowing the attribute on struct member pointers this now allows
the possiblity of writing the attribute in the type attribute position.
For example:

```
struct BufferTypeAttributePosition {
  size_t count;
  char *__counted_by(count) buffer; // Now allowed
}
```

However, the attribute in this position is still currently parsed
immediately rather than late parsed. So this will not parse currently:

```
struct BufferTypeAttributePosition {
  char *__counted_by(count) buffer; // Fails to parse
  size_t count;
}
```

The intention is to lift this restriction in future patches. It has not
been done in this patch to keep this size of this commit small.

There are also several other follow up changes that will need to be
addressed in future patches:

* Make late parsing working with anonymous structs (see
`on_pointer_anon_buf` in `attr-counted-by-late-parsed-struct-ptrs.c`).
* Allow `counted_by` on more subjects (e.g. parameters, returns types)
when `-fbounds-safety` is enabled.
* Make use of the attribute on pointer types in code gen (e.g. for
`_builtin_dynamic_object_size` and UBSan's array-bounds checks).

This work is heavily based on a patch originally written by Yeoul Na.

rdar://125400257

Co-authored-by: Dan Liew <dan@su-root.co.uk>
---
 clang/docs/ReleaseNotes.rst                   |  21 +-
 clang/include/clang/AST/Type.h                |   1 +
 clang/include/clang/Basic/Attr.td             |   3 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |  17 +-
 clang/include/clang/Parse/Parser.h            |   7 +-
 clang/include/clang/Sema/Sema.h               |   3 +-
 clang/lib/AST/Type.cpp                        |  10 +
 clang/lib/Parse/ParseDecl.cpp                 | 104 ++++++-
 clang/lib/Parse/ParseObjc.cpp                 |  10 +-
 clang/lib/Sema/SemaDeclAttr.cpp               |  82 ++++--
 clang/lib/Sema/SemaType.cpp                   |   6 +-
 clang/lib/Sema/TreeTransform.h                |   2 +-
 .../attr-counted-by-late-parsed-struct-ptrs.c |  45 ++++
 clang/test/AST/attr-counted-by-struct-ptrs.c  | 117 ++++++++
 .../Sema/attr-counted-by-late-parsed-off.c    |  26 ++
 .../attr-counted-by-late-parsed-struct-ptrs.c | 254 ++++++++++++++++++
 ...tr-counted-by-struct-ptrs-sizeless-types.c |  17 ++
 clang/test/Sema/attr-counted-by-struct-ptrs.c | 224 +++++++++++++++
 .../Sema/attr-counted-by-vla-sizeless-types.c |  11 +
 clang/test/Sema/attr-counted-by-vla.c         | 193 +++++++++++++
 clang/test/Sema/attr-counted-by.c             | 112 --------
 21 files changed, 1117 insertions(+), 148 deletions(-)
 create mode 100644 clang/test/AST/attr-counted-by-late-parsed-struct-ptrs.c
 create mode 100644 clang/test/AST/attr-counted-by-struct-ptrs.c
 create mode 100644 clang/test/Sema/attr-counted-by-late-parsed-off.c
 create mode 100644 clang/test/Sema/attr-counted-by-late-parsed-struct-ptrs.c
 create mode 100644 clang/test/Sema/attr-counted-by-struct-ptrs-sizeless-types.c
 create mode 100644 clang/test/Sema/attr-counted-by-struct-ptrs.c
 create mode 100644 clang/test/Sema/attr-counted-by-vla-sizeless-types.c
 create mode 100644 clang/test/Sema/attr-counted-by-vla.c
 delete mode 100644 clang/test/Sema/attr-counted-by.c

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 7af5869d2176..2f83f5c6d54e 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -317,7 +317,8 @@ New Compiler Flags
 
 - ``-fexperimental-late-parse-attributes`` enables an experimental feature to
   allow late parsing certain attributes in specific contexts where they would
-  not normally be late parsed.
+  not normally be late parsed. Currently this allows late parsing the
+  `counted_by` attribute in C. See `Attribute Changes in Clang`_.
 
 - ``-fseparate-named-sections`` uses separate unique sections for global
   symbols in named special sections (i.e. symbols annotated with
@@ -406,6 +407,24 @@ Attribute Changes in Clang
 - The ``clspv_libclc_builtin`` attribute has been added to allow clspv
   (`OpenCL-C to Vulkan SPIR-V compiler <https://github.com/google/clspv>`_) to identify functions coming from libclc
   (`OpenCL-C builtin library <https://libclc.llvm.org>`_).
+- The ``counted_by`` attribute is now allowed on pointers that are members of a
+  struct in C.
+
+- The ``counted_by`` attribute can now be late parsed in C when
+  ``-fexperimental-late-parse-attributes`` is passed but only when attribute is
+  used in the declaration attribute position. This allows using the
+  attribute on existing code where it previously impossible to do so without
+  re-ordering struct field declarations would break ABI as shown below.
+
+  .. code-block:: c
+
+     struct BufferTy {
+       /* Refering to `count` requires late parsing */
+       char* buffer __counted_by(count);
+       /* Swapping `buffer` and `count` to avoid late parsing would break ABI */
+       size_t count;
+     };
+
 
 Improvements to Clang's diagnostics
 -----------------------------------
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index da3834f19ca0..c7a8e785913b 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -2515,6 +2515,7 @@ public:
   bool isRecordType() const;
   bool isClassType() const;
   bool isStructureType() const;
+  bool isStructureTypeWithFlexibleArrayMember() const;
   bool isObjCBoxableRecordType() const;
   bool isInterfaceType() const;
   bool isStructureOrClassType() const;
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 38ee8356583b..7a7721239a28 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -2229,7 +2229,8 @@ def TypeNullUnspecified : TypeAttr {
 def CountedBy : DeclOrTypeAttr {
   let Spellings = [Clang<"counted_by">];
   let Subjects = SubjectList<[Field], ErrorDiag>;
-  let Args = [ExprArgument<"Count">, IntArgument<"NestedLevel">];
+  let Args = [ExprArgument<"Count">, IntArgument<"NestedLevel", 1>];
+  let LateParsed = LateAttrParseExperimentalExt;
   let ParseArgumentsAsUnevaluated = 1;
   let Documentation = [CountedByDocs];
   let LangOpts = [COnly];
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 09b1874f9fdd..8e6596410c5d 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -6533,8 +6533,10 @@ def warn_superclass_variable_sized_type_not_at_end : Warning<
 
 def err_flexible_array_count_not_in_same_struct : Error<
   "'counted_by' field %0 isn't within the same struct as the flexible array">;
-def err_counted_by_attr_not_on_flexible_array_member : Error<
-  "'counted_by' only applies to C99 flexible array members">;
+def err_counted_by_attr_not_on_ptr_or_flexible_array_member : Error<
+  "'counted_by' only applies to pointers or C99 flexible array members">;
+def err_counted_by_attr_on_array_not_flexible_array_member : Error<
+  "'counted_by' on arrays only applies to C99 flexible array members">;
 def err_counted_by_attr_refer_to_itself : Error<
   "'counted_by' cannot refer to the flexible array member %0">;
 def err_counted_by_must_be_in_structure : Error<
@@ -6549,6 +6551,17 @@ def err_counted_by_attr_refer_to_union : Error<
   "'counted_by' argument cannot refer to a union member">;
 def note_flexible_array_counted_by_attr_field : Note<
   "field %0 declared here">;
+def err_counted_by_attr_pointee_unknown_size : Error<
+  "'counted_by' cannot be applied to %select{"
+    "a pointer with pointee|" // pointer
+    "an array with element}0" // array
+  " of unknown size because %1 is %select{"
+    "an incomplete type|"  // CountedByInvalidPointeeTypeKind::INCOMPLETE
+    "a sizeless type|"     // CountedByInvalidPointeeTypeKind::SIZELESS
+    "a function type|"     // CountedByInvalidPointeeTypeKind::FUNCTION
+    // CountedByInvalidPointeeTypeKind::FLEXIBLE_ARRAY_MEMBER
+    "a struct type with a flexible array member"
+  "}2">;
 
 let CategoryName = "ARC Semantic Issue" in {
 
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 1e796e828b10..af50164a8f93 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -1645,6 +1645,8 @@ private:
                                bool EnterScope, bool OnDefinition);
   void ParseLexedAttribute(LateParsedAttribute &LA,
                            bool EnterScope, bool OnDefinition);
+  void ParseLexedCAttribute(LateParsedAttribute &LA,
+                            ParsedAttributes *OutAttrs = nullptr);
   void ParseLexedMethodDeclarations(ParsingClass &Class);
   void ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM);
   void ParseLexedMethodDefs(ParsingClass &Class);
@@ -2531,7 +2533,8 @@ private:
 
   void ParseStructDeclaration(
       ParsingDeclSpec &DS,
-      llvm::function_ref<void(ParsingFieldDeclarator &)> FieldsCallback);
+      llvm::function_ref<Decl *(ParsingFieldDeclarator &)> FieldsCallback,
+      LateParsedAttrList *LateFieldAttrs = nullptr);
 
   DeclGroupPtrTy ParseTopLevelStmtDecl();
 
@@ -3109,6 +3112,8 @@ private:
                                  SourceLocation ScopeLoc,
                                  ParsedAttr::Form Form);
 
+  void DistributeCLateParsedAttrs(Decl *Dcl, LateParsedAttrList *LateAttrs);
+
   void ParseBoundsAttribute(IdentifierInfo &AttrName,
                             SourceLocation AttrNameLoc, ParsedAttributes &Attrs,
                             IdentifierInfo *ScopeName, SourceLocation ScopeLoc,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index df562e93fbf0..cd06bd9c230d 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -11396,7 +11396,8 @@ public:
   QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns,
                            SourceLocation AttrLoc);
 
-  QualType BuildCountAttributedArrayType(QualType WrappedTy, Expr *CountExpr);
+  QualType BuildCountAttributedArrayOrPointerType(QualType WrappedTy,
+                                                  Expr *CountExpr);
 
   QualType BuildAddressSpaceAttr(QualType &T, LangAS ASIdx, Expr *AddrSpace,
                                  SourceLocation AttrLoc);
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index e31741cd4424..f69a8f80a639 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -632,6 +632,16 @@ bool Type::isStructureType() const {
   return false;
 }
 
+bool Type::isStructureTypeWithFlexibleArrayMember() const {
+  const auto *RT = getAs<RecordType>();
+  if (!RT)
+    return false;
+  const auto *Decl = RT->getDecl();
+  if (!Decl->isStruct())
+    return false;
+  return Decl->hasFlexibleArrayMember();
+}
+
 bool Type::isObjCBoxableRecordType() const {
   if (const auto *RT = getAs<RecordType>())
     return RT->getDecl()->hasAttr<ObjCBoxableAttr>();
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 2ce8fa98089f..8405b44685ae 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -3288,6 +3288,19 @@ void Parser::ParseAlignmentSpecifier(ParsedAttributes &Attrs,
   }
 }
 
+void Parser::DistributeCLateParsedAttrs(Decl *Dcl,
+                                        LateParsedAttrList *LateAttrs) {
+  assert(Dcl && "Dcl cannot be null");
+
+  if (!LateAttrs)
+    return;
+
+  for (auto *LateAttr : *LateAttrs) {
+    if (LateAttr->Decls.empty())
+      LateAttr->addDecl(Dcl);
+  }
+}
+
 /// Bounds attributes (e.g., counted_by):
 ///   AttrName '(' expression ')'
 void Parser::ParseBoundsAttribute(IdentifierInfo &AttrName,
@@ -4825,13 +4838,14 @@ static void DiagnoseCountAttributedTypeInUnnamedAnon(ParsingDeclSpec &DS,
 ///
 void Parser::ParseStructDeclaration(
     ParsingDeclSpec &DS,
-    llvm::function_ref<void(ParsingFieldDeclarator &)> FieldsCallback) {
+    llvm::function_ref<Decl *(ParsingFieldDeclarator &)> FieldsCallback,
+    LateParsedAttrList *LateFieldAttrs) {
 
   if (Tok.is(tok::kw___extension__)) {
     // __extension__ silences extension warnings in the subexpression.
     ExtensionRAIIObject O(Diags);  // Use RAII to do this.
     ConsumeToken();
-    return ParseStructDeclaration(DS, FieldsCallback);
+    return ParseStructDeclaration(DS, FieldsCallback, LateFieldAttrs);
   }
 
   // Parse leading attributes.
@@ -4896,10 +4910,12 @@ void Parser::ParseStructDeclaration(
     }
 
     // If attributes exist after the declarator, parse them.
-    MaybeParseGNUAttributes(DeclaratorInfo.D);
+    MaybeParseGNUAttributes(DeclaratorInfo.D, LateFieldAttrs);
 
     // We're done with this declarator;  invoke the callback.
-    FieldsCallback(DeclaratorInfo);
+    Decl *Field = FieldsCallback(DeclaratorInfo);
+    if (Field)
+      DistributeCLateParsedAttrs(Field, LateFieldAttrs);
 
     // If we don't have a comma, it is either the end of the list (a ';')
     // or an error, bail out.
@@ -4910,6 +4926,69 @@ void Parser::ParseStructDeclaration(
   }
 }
 
+/// Finish parsing an attribute for which parsing was delayed.
+/// This will be called at the end of parsing a class declaration
+/// for each LateParsedAttribute. We consume the saved tokens and
+/// create an attribute with the arguments filled in. We add this
+/// to the Attribute list for the decl.
+void Parser::ParseLexedCAttribute(LateParsedAttribute &LA,
+                                  ParsedAttributes *OutAttrs) {
+  // Create a fake EOF so that attribute parsing won't go off the end of the
+  // attribute.
+  Token AttrEnd;
+  AttrEnd.startToken();
+  AttrEnd.setKind(tok::eof);
+  AttrEnd.setLocation(Tok.getLocation());
+  AttrEnd.setEofData(LA.Toks.data());
+  LA.Toks.push_back(AttrEnd);
+
+  // Append the current token at the end of the new token stream so that it
+  // doesn't get lost.
+  LA.Toks.push_back(Tok);
+  PP.EnterTokenStream(LA.Toks, /*DisableMacroExpansion=*/true,
+                      /*IsReinject=*/true);
+  // Drop the current token and bring the first cached one. It's the same token
+  // as when we entered this function.
+  ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
+
+  ParsedAttributes Attrs(AttrFactory);
+
+  assert(LA.Decls.size() <= 1 &&
+         "late field attribute expects to have at most one declaration.");
+
+  // Dispatch based on the attribute and parse it
+  const AttributeCommonInfo::Form ParsedForm = ParsedAttr::Form::GNU();
+  IdentifierInfo *ScopeName = nullptr;
+  const ParsedAttr::Kind AttrKind =
+      ParsedAttr::getParsedKind(&LA.AttrName, /*ScopeName=*/ScopeName,
+                                /*SyntaxUsed=*/ParsedForm.getSyntax());
+  switch (AttrKind) {
+  case ParsedAttr::Kind::AT_CountedBy:
+    ParseBoundsAttribute(LA.AttrName, LA.AttrNameLoc, Attrs,
+                         /*ScopeName=*/ScopeName, SourceLocation(),
+                         /*Form=*/ParsedForm);
+    break;
+  default:
+    llvm_unreachable("Unhandled late parsed attribute");
+  }
+
+  for (auto *D : LA.Decls)
+    Actions.ActOnFinishDelayedAttribute(getCurScope(), D, Attrs);
+
+  // Due to a parsing error, we either went over the cached tokens or
+  // there are still cached tokens left, so we skip the leftover tokens.
+  while (Tok.isNot(tok::eof))
+    ConsumeAnyToken();
+
+  // Consume the fake EOF token if it's there
+  if (Tok.is(tok::eof) && Tok.getEofData() == AttrEnd.getEofData())
+    ConsumeAnyToken();
+
+  if (OutAttrs) {
+    OutAttrs->takeAllFrom(Attrs);
+  }
+}
+
 /// ParseStructUnionBody
 ///       struct-contents:
 ///         struct-declaration-list
@@ -4933,6 +5012,11 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
   ParseScope StructScope(this, Scope::ClassScope|Scope::DeclScope);
   Actions.ActOnTagStartDefinition(getCurScope(), TagDecl);
 
+  // `LateAttrParseExperimentalExtOnly=true` requests that only attributes
+  // marked with `LateAttrParseExperimentalExt` are late parsed.
+  LateParsedAttrList LateFieldAttrs(/*PSoon=*/false,
+                                    /*LateAttrParseExperimentalExtOnly=*/true);
+
   // While we still have something to read, read the declarations in the struct.
   while (!tryParseMisplacedModuleImport() && Tok.isNot(tok::r_brace) &&
          Tok.isNot(tok::eof)) {
@@ -4983,18 +5067,19 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
     }
 
     if (!Tok.is(tok::at)) {
-      auto CFieldCallback = [&](ParsingFieldDeclarator &FD) {
+      auto CFieldCallback = [&](ParsingFieldDeclarator &FD) -> Decl * {
         // Install the declarator into the current TagDecl.
         Decl *Field =
             Actions.ActOnField(getCurScope(), TagDecl,
                                FD.D.getDeclSpec().getSourceRange().getBegin(),
                                FD.D, FD.BitfieldSize);
         FD.complete(Field);
+        return Field;
       };
 
       // Parse all the comma separated declarators.
       ParsingDeclSpec DS(*this);
-      ParseStructDeclaration(DS, CFieldCallback);
+      ParseStructDeclaration(DS, CFieldCallback, &LateFieldAttrs);
     } else { // Handle @defs
       ConsumeToken();
       if (!Tok.isObjCAtKeyword(tok::objc_defs)) {
@@ -5035,7 +5120,12 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
 
   ParsedAttributes attrs(AttrFactory);
   // If attributes exist after struct contents, parse them.
-  MaybeParseGNUAttributes(attrs);
+  MaybeParseGNUAttributes(attrs, &LateFieldAttrs);
+
+  // Late parse field attributes if necessary.
+  assert(!getLangOpts().CPlusPlus);
+  for (auto *LateAttr : LateFieldAttrs)
+    ParseLexedCAttribute(*LateAttr);
 
   SmallVector<Decl *, 32> FieldDecls(TagDecl->fields());
 
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index 89f4acbd25e4..6a2088a73c55 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -780,16 +780,16 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
       }
 
       bool addedToDeclSpec = false;
-      auto ObjCPropertyCallback = [&](ParsingFieldDeclarator &FD) {
+      auto ObjCPropertyCallback = [&](ParsingFieldDeclarator &FD) -> Decl * {
         if (FD.D.getIdentifier() == nullptr) {
           Diag(AtLoc, diag::err_objc_property_requires_field_name)
               << FD.D.getSourceRange();
-          return;
+          return nullptr;
         }
         if (FD.BitfieldSize) {
           Diag(AtLoc, diag::err_objc_property_bitfield)
               << FD.D.getSourceRange();
-          return;
+          return nullptr;
         }
 
         // Map a nullability property attribute to a context-sensitive keyword
@@ -818,6 +818,7 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
             MethodImplKind);
 
         FD.complete(Property);
+        return Property;
       };
 
       // Parse all the comma separated declarators.
@@ -2013,7 +2014,7 @@ void Parser::ParseObjCClassInstanceVariables(ObjCContainerDecl *interfaceDecl,
       continue;
     }
 
-    auto ObjCIvarCallback = [&](ParsingFieldDeclarator &FD) {
+    auto ObjCIvarCallback = [&](ParsingFieldDeclarator &FD) -> Decl * {
       assert(getObjCDeclContext() == interfaceDecl &&
              "Ivar should have interfaceDecl as its decl context");
       // Install the declarator into the interface decl.
@@ -2024,6 +2025,7 @@ void Parser::ParseObjCClassInstanceVariables(ObjCContainerDecl *interfaceDecl,
       if (Field)
         AllIvarDecls.push_back(Field);
       FD.complete(Field);
+      return Field;
     };
 
     // Parse all the comma separated declarators.
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 30776ff537fb..c8b71631076b 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -8633,31 +8633,82 @@ static const RecordDecl *GetEnclosingNamedOrTopAnonRecord(const FieldDecl *FD) {
   return RD;
 }
 
-static bool
-CheckCountExpr(Sema &S, FieldDecl *FD, Expr *E,
-               llvm::SmallVectorImpl<TypeCoupledDeclRefInfo> &Decls) {
+enum class CountedByInvalidPointeeTypeKind {
+  INCOMPLETE,
+  SIZELESS,
+  FUNCTION,
+  FLEXIBLE_ARRAY_MEMBER,
+  VALID,
+};
+
+static bool CheckCountedByAttrOnField(
+    Sema &S, FieldDecl *FD, Expr *E,
+    llvm::SmallVectorImpl<TypeCoupledDeclRefInfo> &Decls) {
+  // Check the context the attribute is used in
+
   if (FD->getParent()->isUnion()) {
     S.Diag(FD->getBeginLoc(), diag::err_counted_by_attr_in_union)
         << FD->getSourceRange();
     return true;
   }
 
-  if (!E->getType()->isIntegerType() || E->getType()->isBooleanType()) {
-    S.Diag(E->getBeginLoc(), diag::err_counted_by_attr_argument_not_integer)
-        << E->getSourceRange();
+  const auto FieldTy = FD->getType();
+  if (!FieldTy->isArrayType() && !FieldTy->isPointerType()) {
+    S.Diag(FD->getBeginLoc(),
+           diag::err_counted_by_attr_not_on_ptr_or_flexible_array_member)
+        << FD->getLocation();
     return true;
   }
 
   LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
       LangOptions::StrictFlexArraysLevelKind::IncompleteOnly;
-
-  if (!Decl::isFlexibleArrayMemberLike(S.getASTContext(), FD, FD->getType(),
+  if (FieldTy->isArrayType() &&
+      !Decl::isFlexibleArrayMemberLike(S.getASTContext(), FD, FieldTy,
                                        StrictFlexArraysLevel, true)) {
-    // The "counted_by" attribute must be on a flexible array member.
-    SourceRange SR = FD->getLocation();
-    S.Diag(SR.getBegin(),
-           diag::err_counted_by_attr_not_on_flexible_array_member)
-        << SR;
+    S.Diag(FD->getBeginLoc(),
+           diag::err_counted_by_attr_on_array_not_flexible_array_member)
+        << FD->getLocation();
+    return true;
+  }
+
+  CountedByInvalidPointeeTypeKind InvalidTypeKind =
+      CountedByInvalidPointeeTypeKind::VALID;
+  QualType PointeeTy;
+  int SelectPtrOrArr = 0;
+  if (FieldTy->isPointerType()) {
+    PointeeTy = FieldTy->getPointeeType();
+    SelectPtrOrArr = 0;
+  } else {
+    assert(FieldTy->isArrayType());
+    const ArrayType *AT = S.getASTContext().getAsArrayType(FieldTy);
+    PointeeTy = AT->getElementType();
+    SelectPtrOrArr = 1;
+  }
+  // Note: The `Decl::isFlexibleArrayMemberLike` check earlier on means
+  // only `PointeeTy->isStructureTypeWithFlexibleArrayMember()` is reachable
+  // when `FieldTy->isArrayType()`.
+  if (PointeeTy->isIncompleteType()) {
+    InvalidTypeKind = CountedByInvalidPointeeTypeKind::INCOMPLETE;
+  } else if (PointeeTy->isSizelessType()) {
+    InvalidTypeKind = CountedByInvalidPointeeTypeKind::SIZELESS;
+  } else if (PointeeTy->isFunctionType()) {
+    InvalidTypeKind = CountedByInvalidPointeeTypeKind::FUNCTION;
+  } else if (PointeeTy->isStructureTypeWithFlexibleArrayMember()) {
+    InvalidTypeKind = CountedByInvalidPointeeTypeKind::FLEXIBLE_ARRAY_MEMBER;
+  }
+
+  if (InvalidTypeKind != CountedByInvalidPointeeTypeKind::VALID) {
+    S.Diag(FD->getBeginLoc(), diag::err_counted_by_attr_pointee_unknown_size)
+        << SelectPtrOrArr << PointeeTy << (int)InvalidTypeKind
+        << FD->getSourceRange();
+    return true;
+  }
+
+  // Check the expression
+
+  if (!E->getType()->isIntegerType() || E->getType()->isBooleanType()) {
+    S.Diag(E->getBeginLoc(), diag::err_counted_by_attr_argument_not_integer)
+        << E->getSourceRange();
     return true;
   }
 
@@ -8720,10 +8771,11 @@ static void handleCountedByAttrField(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
 
   llvm::SmallVector<TypeCoupledDeclRefInfo, 1> Decls;
-  if (CheckCountExpr(S, FD, CountExpr, Decls))
+  if (CheckCountedByAttrOnField(S, FD, CountExpr, Decls))
     return;
 
-  QualType CAT = S.BuildCountAttributedArrayType(FD->getType(), CountExpr);
+  QualType CAT =
+      S.BuildCountAttributedArrayOrPointerType(FD->getType(), CountExpr);
   FD->setType(CAT);
 }
 
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index c19c8cc34dd3..ef0b6b701a52 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -9345,9 +9345,9 @@ BuildTypeCoupledDecls(Expr *E,
   Decls.push_back(TypeCoupledDeclRefInfo(CountDecl, /*IsDref*/ false));
 }
 
-QualType Sema::BuildCountAttributedArrayType(QualType WrappedTy,
-                                             Expr *CountExpr) {
-  assert(WrappedTy->isIncompleteArrayType());
+QualType Sema::BuildCountAttributedArrayOrPointerType(QualType WrappedTy,
+                                                      Expr *CountExpr) {
+  assert(WrappedTy->isIncompleteArrayType() || WrappedTy->isPointerType());
 
   llvm::SmallVector<TypeCoupledDeclRefInfo, 1> Decls;
   BuildTypeCoupledDecls(CountExpr, Decls);
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index b10e5ba65eb1..29444f0edc2a 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -7344,7 +7344,7 @@ QualType TreeTransform<Derived>::TransformCountAttributedType(
   if (getDerived().AlwaysRebuild() || InnerTy != OldTy->desugar() ||
       OldCount != NewCount) {
     // Currently, CountAttributedType can only wrap incomplete array types.
-    Result = SemaRef.BuildCountAttributedArrayType(InnerTy, NewCount);
+    Result = SemaRef.BuildCountAttributedArrayOrPointerType(InnerTy, NewCount);
   }
 
   TLB.push<CountAttributedTypeLoc>(Result);
diff --git a/clang/test/AST/attr-counted-by-late-parsed-struct-ptrs.c b/clang/test/AST/attr-counted-by-late-parsed-struct-ptrs.c
new file mode 100644
index 000000000000..a585a45eeff0
--- /dev/null
+++ b/clang/test/AST/attr-counted-by-late-parsed-struct-ptrs.c
@@ -0,0 +1,45 @@
+// RUN: %clang_cc1 -fexperimental-late-parse-attributes %s -ast-dump | FileCheck %s
+
+#define __counted_by(f)  __attribute__((counted_by(f)))
+
+struct size_known {
+  int field;
+};
+
+//==============================================================================
+// __counted_by on struct member pointer in decl attribute position
+//==============================================================================
+
+struct on_member_pointer_complete_ty {
+  struct size_known *buf __counted_by(count);
+  int count;
+};
+// CHECK-LABEL: struct on_member_pointer_complete_ty definition
+// CHECK-NEXT: |-FieldDecl {{.*}} buf 'struct size_known * __counted_by(count)':'struct size_known *'
+// CHECK-NEXT: `-FieldDecl {{.*}} referenced count 'int'
+
+struct on_pointer_anon_count {
+  struct size_known *buf __counted_by(count);
+  struct {
+    int count;
+  };
+};
+
+// CHECK-LABEL: struct on_pointer_anon_count definition
+// CHECK-NEXT:  |-FieldDecl {{.*}} buf 'struct size_known * __counted_by(count)':'struct size_known *'
+// CHECK-NEXT:  |-RecordDecl {{.*}} struct definition
+// CHECK-NEXT:  | `-FieldDecl {{.*}} count 'int'
+// CHECK-NEXT:  |-FieldDecl {{.*}} implicit 'struct on_pointer_anon_count::(anonymous at {{.*}})'
+// CHECK-NEXT:  `-IndirectFieldDecl {{.*}} implicit referenced count 'int'
+// CHECK-NEXT:    |-Field {{.*}} '' 'struct on_pointer_anon_count::(anonymous at {{.*}})'
+// CHECK-NEXT:    `-Field {{.*}} 'count' 'int'
+
+//==============================================================================
+// __counted_by on struct member pointer in type attribute position
+//==============================================================================
+// TODO: Correctly parse counted_by as a type attribute. Currently it is parsed
+// as a declaration attribute and is **not** late parsed resulting in the `count`
+// field being unavailable.
+//
+// See `clang/test/Sema/attr-counted-by-late-parsed-struct-ptrs.c` for test
+// cases.
diff --git a/clang/test/AST/attr-counted-by-struct-ptrs.c b/clang/test/AST/attr-counted-by-struct-ptrs.c
new file mode 100644
index 000000000000..79a453d239cd
--- /dev/null
+++ b/clang/test/AST/attr-counted-by-struct-ptrs.c
@@ -0,0 +1,117 @@
+// RUN: %clang_cc1 %s -ast-dump | FileCheck %s
+
+#define __counted_by(f)  __attribute__((counted_by(f)))
+
+struct size_unknown;
+struct size_known {
+  int field;
+};
+
+//==============================================================================
+// __counted_by on struct member pointer in decl attribute position
+//==============================================================================
+
+// CHECK-LABEL: RecordDecl {{.+}} struct on_member_pointer_complete_ty definition
+// CHECK-NEXT: |-FieldDecl {{.+}} referenced count 'int'
+// CHECK-NEXT: `-FieldDecl {{.+}} buf 'struct size_known * __counted_by(count)':'struct size_known *'
+struct on_member_pointer_complete_ty {
+  int count;
+  struct size_known * buf __counted_by(count);
+};
+
+// CHECK-LABEL: RecordDecl {{.+}} struct on_pointer_anon_buf definition
+// CHECK-NEXT:  |-FieldDecl {{.+}} referenced count 'int'
+// CHECK-NEXT:  |-RecordDecl {{.+}} struct definition
+// CHECK-NEXT:  | `-FieldDecl {{.+}} buf 'struct size_known * __counted_by(count)':'struct size_known *'
+// CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH:.+]])'
+// CHECK-NEXT:  `-IndirectFieldDecl {{.+}} implicit buf 'struct size_known * __counted_by(count)':'struct size_known *'
+// CHECK-NEXT:    |-Field {{.+}} '' 'struct on_pointer_anon_buf::(anonymous at [[ANON_STRUCT_PATH]])'
+// CHECK-NEXT:    `-Field {{.+}} 'buf' 'struct size_known * __counted_by(count)':'struct size_known *'
+struct on_pointer_anon_buf {
+  int count;
+  struct {
+    struct size_known *buf __counted_by(count);
+  };
+};
+
+struct on_pointer_anon_count {
+  struct {
+    int count;
+  };
+  struct size_known *buf __counted_by(count);
+};
+
+//==============================================================================
+// __counted_by on struct member pointer in type attribute position
+//==============================================================================
+// TODO: Correctly parse counted_by as a type attribute. Currently it is parsed
+// as a declaration attribute
+
+// CHECK-LABEL: RecordDecl {{.+}} struct on_member_pointer_complete_ty_ty_pos definition
+// CHECK-NEXT:  |-FieldDecl {{.+}} referenced count 'int'
+// CHECK-NEXT:  `-FieldDecl {{.+}} buf 'struct size_known * __counted_by(count)':'struct size_known *'
+struct on_member_pointer_complete_ty_ty_pos {
+  int count;
+  struct size_known *__counted_by(count) buf;
+};
+
+// TODO: This should be forbidden but isn't due to counted_by being treated as a
+// declaration attribute. The attribute ends up on the outer most pointer
+// (allowed by sema) even though syntactically its supposed to be on the inner
+// pointer (would not allowed by sema due to pointee being a function type).
+// CHECK-LABEL: RecordDecl {{.+}} struct on_member_pointer_fn_ptr_ty_ty_pos_inner definition
+// CHECK-NEXT:  |-FieldDecl {{.+}} referenced count 'int'
+// CHECK-NEXT:  `-FieldDecl {{.+}} fn_ptr 'void (** __counted_by(count))(void)':'void (**)(void)'
+struct on_member_pointer_fn_ptr_ty_ty_pos_inner {
+  int count;
+  void (* __counted_by(count) * fn_ptr)(void);
+};
+
+// FIXME: The generated AST here is wrong. The attribute should be on the inner
+// pointer.
+// CHECK-LABEL: RecordDecl {{.+}} struct on_nested_pointer_inner definition
+// CHECK-NEXT:  |-FieldDecl {{.+}} referenced count 'int'
+// CHECK-NEXT:  `-FieldDecl {{.+}} buf 'struct size_known ** __counted_by(count)':'struct size_known **'
+struct on_nested_pointer_inner {
+  int count;
+  // TODO: This should be disallowed because in the `-fbounds-safety` model
+  // `__counted_by` can only be nested when used in function parameters.
+  struct size_known *__counted_by(count) *buf;
+};
+
+// CHECK-LABEL: RecordDecl {{.+}} struct on_nested_pointer_outer definition
+// CHECK-NEXT:  |-FieldDecl {{.+}} referenced count 'int'
+// CHECK-NEXT:  `-FieldDecl {{.+}} buf 'struct size_known ** __counted_by(count)':'struct size_known **'
+struct on_nested_pointer_outer {
+  int count;
+  struct size_known **__counted_by(count) buf;
+};
+
+// CHECK-LABEL: RecordDecl {{.+}} struct on_pointer_anon_buf_ty_pos definition
+// CHECK-NEXT:  |-FieldDecl {{.+}} referenced count 'int'
+// CHECK-NEXT:  |-RecordDecl {{.+}} struct definition
+// CHECK-NEXT:  | `-FieldDecl {{.+}} buf 'struct size_known * __counted_by(count)':'struct size_known *'
+// CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2:.+]])'
+// CHECK-NEXT:  `-IndirectFieldDecl {{.+}} implicit buf 'struct size_known * __counted_by(count)':'struct size_known *'
+// CHECK-NEXT:    |-Field {{.+}} '' 'struct on_pointer_anon_buf_ty_pos::(anonymous at [[ANON_STRUCT_PATH2]])'
+// CHECK-NEXT:    `-Field {{.+}} 'buf' 'struct size_known * __counted_by(count)':'struct size_known *'
+struct on_pointer_anon_buf_ty_pos {
+  int count;
+  struct {
+    struct size_known * __counted_by(count) buf;
+  };
+};
+
+// CHECK-LABEL: RecordDecl {{.+}} struct on_pointer_anon_count_ty_pos definition
+// CHECK-NEXT:  |-RecordDecl {{.+}} struct definition
+// CHECK-NEXT:  | `-FieldDecl {{.+}} count 'int'
+// CHECK-NEXT:  |-FieldDecl {{.+}} implicit 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3:.+]])'
+// CHECK-NEXT:  |-IndirectFieldDecl {{.+}} implicit referenced count 'int'
+// CHECK-NEXT:  | |-Field {{.+}} '' 'struct on_pointer_anon_count_ty_pos::(anonymous at [[ANON_STRUCT_PATH3]])'
+// CHECK-NEXT:  | `-Field {{.+}} 'count' 'int'
+struct on_pointer_anon_count_ty_pos {
+  struct {
+    int count;
+  };
+  struct size_known *__counted_by(count) buf;
+};
diff --git a/clang/test/Sema/attr-counted-by-late-parsed-off.c b/clang/test/Sema/attr-counted-by-late-parsed-off.c
new file mode 100644
index 000000000000..34f51d10c083
--- /dev/null
+++ b/clang/test/Sema/attr-counted-by-late-parsed-off.c
@@ -0,0 +1,26 @@
+// RUN: %clang_cc1 -DNEEDS_LATE_PARSING -fno-experimental-late-parse-attributes -fsyntax-only -verify %s
+// RUN: %clang_cc1 -DNEEDS_LATE_PARSING -fsyntax-only -verify %s
+
+// RUN: %clang_cc1 -UNEEDS_LATE_PARSING -fno-experimental-late-parse-attributes -fsyntax-only -verify=ok %s
+// RUN: %clang_cc1 -UNEEDS_LATE_PARSING -fsyntax-only -verify=ok %s
+
+#define __counted_by(f)  __attribute__((counted_by(f)))
+
+struct size_known { int dummy; };
+
+#ifdef NEEDS_LATE_PARSING
+struct on_decl {
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  struct size_known *buf __counted_by(count);
+  int count;
+};
+
+#else
+
+// ok-no-diagnostics
+struct on_decl {
+  int count;
+  struct size_known *buf __counted_by(count);
+};
+
+#endif
diff --git a/clang/test/Sema/attr-counted-by-late-parsed-struct-ptrs.c b/clang/test/Sema/attr-counted-by-late-parsed-struct-ptrs.c
new file mode 100644
index 000000000000..9ff3b080f657
--- /dev/null
+++ b/clang/test/Sema/attr-counted-by-late-parsed-struct-ptrs.c
@@ -0,0 +1,254 @@
+// RUN: %clang_cc1 -fexperimental-late-parse-attributes -fsyntax-only -verify %s
+
+#define __counted_by(f)  __attribute__((counted_by(f)))
+
+struct size_unknown;
+struct size_known {
+  int field;
+};
+
+typedef void(*fn_ptr_ty)(void);
+
+//==============================================================================
+// __counted_by on struct member pointer in decl attribute position
+//==============================================================================
+
+struct on_member_pointer_complete_ty {
+  struct size_known * buf __counted_by(count);
+  int count;
+};
+
+struct on_member_pointer_incomplete_ty {
+  struct size_unknown * buf __counted_by(count); // expected-error{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct size_unknown' is an incomplete type}}
+  int count;
+};
+
+struct on_member_pointer_const_incomplete_ty {
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'const struct size_unknown' is an incomplete type}}
+  const struct size_unknown * buf __counted_by(count);
+  int count;
+};
+
+struct on_member_pointer_void_ty {
+  void* buf __counted_by(count); // expected-error{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'void' is an incomplete type}}
+  int count;
+};
+
+struct on_member_pointer_fn_ptr_ty {
+  // buffer of `count` function pointers is allowed
+  void (**fn_ptr)(void) __counted_by(count);
+  int count;
+};
+
+
+struct on_member_pointer_fn_ptr_ty_ptr_ty {
+  // buffer of `count` function pointers is allowed
+  fn_ptr_ty* fn_ptr __counted_by(count);
+  int count;
+};
+
+struct on_member_pointer_fn_ty {
+  // buffer of `count` functions is not allowed
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'void (void)' is a function type}}
+  void (*fn_ptr)(void) __counted_by(count);
+  int count;
+};
+
+struct on_member_pointer_fn_ptr_ty_ty {
+  // buffer of `count` functions is not allowed
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'void (void)' is a function type}}
+  fn_ptr_ty fn_ptr __counted_by(count);
+  int count;
+};
+
+struct has_unannotated_vla {
+  int count;
+  int buffer[];
+};
+
+struct on_member_pointer_struct_with_vla {
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct has_unannotated_vla' is a struct type with a flexible array member}}
+  struct has_unannotated_vla* objects __counted_by(count);
+  int count;
+};
+
+struct has_annotated_vla {
+  int count;
+  int buffer[] __counted_by(count);
+};
+
+// Currently prevented because computing the size of `objects` at runtime would
+// require an O(N) walk of `objects` to take into account the length of the VLA
+// in each struct instance.
+struct on_member_pointer_struct_with_annotated_vla {
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct has_annotated_vla' is a struct type with a flexible array member}}
+  struct has_annotated_vla* objects __counted_by(count);
+  int count;
+};
+
+struct on_pointer_anon_buf {
+  // TODO: Support referring to parent scope
+  struct {
+    // expected-error@+1{{use of undeclared identifier 'count'}}
+    struct size_known *buf __counted_by(count);
+  };
+  int count;
+};
+
+struct on_pointer_anon_count {
+  struct size_known *buf __counted_by(count);
+  struct {
+    int count;
+  };
+};
+
+//==============================================================================
+// __counted_by on struct member pointer in type attribute position
+//==============================================================================
+// TODO: Correctly parse counted_by as a type attribute. Currently it is parsed
+// as a declaration attribute and is **not** late parsed resulting in the `count`
+// field being unavailable.
+
+struct on_member_pointer_complete_ty_ty_pos {
+  // TODO: Allow this
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  struct size_known *__counted_by(count) buf;
+  int count;
+};
+
+struct on_member_pointer_incomplete_ty_ty_pos {
+  // TODO: Allow this
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  struct size_unknown * __counted_by(count) buf;
+  int count;
+};
+
+struct on_member_pointer_const_incomplete_ty_ty_pos {
+  // TODO: Allow this
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  const struct size_unknown * __counted_by(count) buf;
+  int count;
+};
+
+struct on_member_pointer_void_ty_ty_pos {
+  // TODO: This should fail because the attribute is
+  // on a pointer with the pointee being an incomplete type.
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  void *__counted_by(count) buf;
+  int count;
+};
+
+// -
+
+struct on_member_pointer_fn_ptr_ty_pos {
+  // TODO: buffer of `count` function pointers should be allowed
+  // but fails because this isn't late parsed.
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  void (** __counted_by(count) fn_ptr)(void);
+  int count;
+};
+
+struct on_member_pointer_fn_ptr_ty_ptr_ty_pos {
+  // TODO: buffer of `count` function pointers should be allowed
+  // but fails because this isn't late parsed.
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  fn_ptr_ty* __counted_by(count) fn_ptr;
+  int count;
+};
+
+struct on_member_pointer_fn_ty_ty_pos {
+  // TODO: This should fail because the attribute is
+  // on a pointer with the pointee being a function type.
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  void (* __counted_by(count) fn_ptr)(void);
+  int count;
+};
+
+struct on_member_pointer_fn_ptr_ty_ty_pos {
+  // TODO: buffer of `count` function pointers should be allowed
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  void (** __counted_by(count) fn_ptr)(void);
+  int count;
+};
+
+struct on_member_pointer_fn_ptr_ty_typedef_ty_pos {
+  // TODO: This should fail because the attribute is
+  // on a pointer with the pointee being a function type.
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  fn_ptr_ty __counted_by(count) fn_ptr;
+  int count;
+};
+
+struct on_member_pointer_fn_ptr_ty_ty_pos_inner {
+  // TODO: This should fail because the attribute is
+  // on a pointer with the pointee being a function type.
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  void (* __counted_by(count) * fn_ptr)(void);
+  int count;
+};
+
+struct on_member_pointer_struct_with_vla_ty_pos {
+  // TODO: This should fail because the attribute is
+  // on a pointer with the pointee being a struct type with a VLA.
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  struct has_unannotated_vla *__counted_by(count) objects;
+  int count;
+};
+
+struct on_member_pointer_struct_with_annotated_vla_ty_pos {
+  // TODO: This should fail because the attribute is
+  // on a pointer with the pointee being a struct type with a VLA.
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  struct has_annotated_vla* __counted_by(count) objects;
+  int count;
+};
+
+struct on_nested_pointer_inner {
+  // TODO: This should be disallowed because in the `-fbounds-safety` model
+  // `__counted_by` can only be nested when used in function parameters.
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  struct size_known *__counted_by(count) *buf;
+  int count;
+};
+
+struct on_nested_pointer_outer {
+  // TODO: Allow this
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  struct size_known **__counted_by(count) buf;
+  int count;
+};
+
+struct on_pointer_anon_buf_ty_pos {
+  struct {
+    // TODO: Support referring to parent scope
+    // expected-error@+1{{use of undeclared identifier 'count'}}
+    struct size_known * __counted_by(count) buf;
+  };
+  int count;
+};
+
+struct on_pointer_anon_count_ty_pos {
+  // TODO: Allow this
+  // expected-error@+1{{use of undeclared identifier 'count'}}
+  struct size_known *__counted_by(count) buf;
+  struct {
+    int count;
+  };
+};
+
+//==============================================================================
+// __counted_by on struct non-pointer members
+//==============================================================================
+
+struct on_pod_ty {
+  // expected-error@+1{{'counted_by' only applies to pointers or C99 flexible array members}}
+  int wrong_ty __counted_by(count);
+  int count;
+};
+
+struct on_void_ty {
+  // expected-error@+2{{'counted_by' only applies to pointers or C99 flexible array members}}
+  // expected-error@+1{{field has incomplete type 'void'}}
+  void wrong_ty __counted_by(count);
+  int count;
+};
diff --git a/clang/test/Sema/attr-counted-by-struct-ptrs-sizeless-types.c b/clang/test/Sema/attr-counted-by-struct-ptrs-sizeless-types.c
new file mode 100644
index 000000000000..9b0f2eafb13c
--- /dev/null
+++ b/clang/test/Sema/attr-counted-by-struct-ptrs-sizeless-types.c
@@ -0,0 +1,17 @@
+// __SVInt8_t is specific to ARM64 so specify that in the target triple
+// RUN: %clang_cc1 -triple arm64-apple-darwin -fsyntax-only -verify %s
+
+#define __counted_by(f)  __attribute__((counted_by(f)))
+
+struct on_sizeless_pointee_ty {
+    int count;
+    // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because '__SVInt8_t' is a sizeless type}}
+    __SVInt8_t* member __counted_by(count);
+};
+
+struct on_sizeless_ty {
+    int count;
+    // expected-error@+2{{'counted_by' only applies to pointers or C99 flexible array members}}
+    // expected-error@+1{{field has sizeless type '__SVInt8_t'}}
+    __SVInt8_t member __counted_by(count);
+};
diff --git a/clang/test/Sema/attr-counted-by-struct-ptrs.c b/clang/test/Sema/attr-counted-by-struct-ptrs.c
new file mode 100644
index 000000000000..cd2bfe36938b
--- /dev/null
+++ b/clang/test/Sema/attr-counted-by-struct-ptrs.c
@@ -0,0 +1,224 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+#define __counted_by(f)  __attribute__((counted_by(f)))
+
+struct size_unknown;
+struct size_known {
+  int field;
+};
+
+typedef void(*fn_ptr_ty)(void);
+
+//==============================================================================
+// __counted_by on struct member pointer in decl attribute position
+//==============================================================================
+
+struct on_member_pointer_complete_ty {
+  int count;
+  struct size_known * buf __counted_by(count);
+};
+
+struct on_member_pointer_incomplete_ty {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct size_unknown' is an incomplete type}}
+  struct size_unknown * buf __counted_by(count);
+};
+
+struct on_member_pointer_const_incomplete_ty {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'const struct size_unknown' is an incomplete type}}
+  const struct size_unknown * buf __counted_by(count);
+};
+
+struct on_member_pointer_void_ty {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'void' is an incomplete type}}
+  void* buf __counted_by(count);
+};
+
+struct on_member_pointer_fn_ptr_ty {
+  int count;
+  // buffer of `count` function pointers is allowed
+  void (**fn_ptr)(void) __counted_by(count);
+};
+
+struct on_member_pointer_fn_ptr_ty_ptr_ty {
+  int count;
+  // buffer of `count` function pointers is allowed
+  fn_ptr_ty* fn_ptr __counted_by(count);
+};
+
+struct on_member_pointer_fn_ty {
+  int count;
+  // buffer of `count` functions is not allowed
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'void (void)' is a function type}}
+  void (*fn_ptr)(void) __counted_by(count);
+};
+
+struct on_member_pointer_fn_ptr_ty_ty {
+  int count;
+  // buffer of `count` functions is not allowed
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'void (void)' is a function type}}
+  fn_ptr_ty fn_ptr __counted_by(count);
+};
+
+struct has_unannotated_vla {
+  int count;
+  int buffer[];
+};
+
+struct on_member_pointer_struct_with_vla {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct has_unannotated_vla' is a struct type with a flexible array member}}
+  struct has_unannotated_vla* objects __counted_by(count);
+};
+
+struct has_annotated_vla {
+  int count;
+  int buffer[] __counted_by(count);
+};
+
+// Currently prevented because computing the size of `objects` at runtime would
+// require an O(N) walk of `objects` to take into account the length of the VLA
+// in each struct instance.
+struct on_member_pointer_struct_with_annotated_vla {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct has_annotated_vla' is a struct type with a flexible array member}}
+  struct has_annotated_vla* objects __counted_by(count);
+};
+
+struct on_pointer_anon_buf {
+  int count;
+  struct {
+    struct size_known *buf __counted_by(count);
+  };
+};
+
+struct on_pointer_anon_count {
+  struct {
+    int count;
+  };
+  struct size_known *buf __counted_by(count);
+};
+
+//==============================================================================
+// __counted_by on struct member pointer in type attribute position
+//==============================================================================
+// TODO: Correctly parse counted_by as a type attribute. Currently it is parsed
+// as a declaration attribute
+
+struct on_member_pointer_complete_ty_ty_pos {
+  int count;
+  struct size_known *__counted_by(count) buf;
+};
+
+struct on_member_pointer_incomplete_ty_ty_pos {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct size_unknown' is an incomplete type}}
+  struct size_unknown * __counted_by(count) buf;
+};
+
+struct on_member_pointer_const_incomplete_ty_ty_pos {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'const struct size_unknown' is an incomplete type}}
+  const struct size_unknown * __counted_by(count) buf;
+};
+
+struct on_member_pointer_void_ty_ty_pos {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'void' is an incomplete type}}
+  void *__counted_by(count) buf;
+};
+
+// -
+
+struct on_member_pointer_fn_ptr_ty_pos {
+  int count;
+  // buffer of `count` function pointers is allowed
+  void (** __counted_by(count) fn_ptr)(void);
+};
+
+struct on_member_pointer_fn_ptr_ty_ptr_ty_pos {
+  int count;
+  // buffer of `count` function pointers is allowed
+  fn_ptr_ty* __counted_by(count) fn_ptr;
+};
+
+struct on_member_pointer_fn_ty_ty_pos {
+  int count;
+  // buffer of `count` functions is not allowed
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'void (void)' is a function type}}
+  void (* __counted_by(count) fn_ptr)(void);
+};
+
+struct on_member_pointer_fn_ptr_ty_ty_pos {
+  int count;
+  // buffer of `count` functions is not allowed
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'void (void)' is a function type}}
+  fn_ptr_ty __counted_by(count) fn_ptr;
+};
+
+// TODO: This should be forbidden but isn't due to counted_by being treated
+// as a declaration attribute.
+struct on_member_pointer_fn_ptr_ty_ty_pos_inner {
+  int count;
+  void (* __counted_by(count) * fn_ptr)(void);
+};
+
+struct on_member_pointer_struct_with_vla_ty_pos {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct has_unannotated_vla' is a struct type with a flexible array member}}
+  struct has_unannotated_vla *__counted_by(count) objects;
+};
+
+// Currently prevented because computing the size of `objects` at runtime would
+// require an O(N) walk of `objects` to take into account the length of the VLA
+// in each struct instance.
+struct on_member_pointer_struct_with_annotated_vla_ty_pos {
+  int count;
+  // expected-error@+1{{counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct has_annotated_vla' is a struct type with a flexible array member}}
+  struct has_annotated_vla* __counted_by(count) objects;
+};
+
+struct on_nested_pointer_inner {
+  // TODO: This should be disallowed because in the `-fbounds-safety` model
+  // `__counted_by` can only be nested when used in function parameters.
+  int count;
+  struct size_known *__counted_by(count) *buf;
+};
+
+struct on_nested_pointer_outer {
+  int count;
+  struct size_known **__counted_by(count) buf;
+};
+
+struct on_pointer_anon_buf_ty_pos {
+  int count;
+  struct {
+    struct size_known * __counted_by(count) buf;
+  };
+};
+
+struct on_pointer_anon_count_ty_pos {
+  struct {
+    int count;
+  };
+  struct size_known *__counted_by(count) buf;
+};
+
+//==============================================================================
+// __counted_by on struct non-pointer members
+//==============================================================================
+
+struct on_pod_ty {
+  int count;
+  // expected-error@+1{{'counted_by' only applies to pointers or C99 flexible array members}}
+  int wrong_ty __counted_by(count);
+};
+
+struct on_void_ty {
+  int count;
+  // expected-error@+2{{'counted_by' only applies to pointers or C99 flexible array members}}
+  // expected-error@+1{{field has incomplete type 'void'}}
+  void wrong_ty __counted_by(count);
+};
diff --git a/clang/test/Sema/attr-counted-by-vla-sizeless-types.c b/clang/test/Sema/attr-counted-by-vla-sizeless-types.c
new file mode 100644
index 000000000000..31c0007501c4
--- /dev/null
+++ b/clang/test/Sema/attr-counted-by-vla-sizeless-types.c
@@ -0,0 +1,11 @@
+// __SVInt8_t is specific to ARM64 so specify that in the target triple
+// RUN: %clang_cc1 -triple arm64-apple-darwin -fsyntax-only -verify %s
+
+#define __counted_by(f)  __attribute__((counted_by(f)))
+
+struct on_sizeless_elt_ty {
+    int count;
+    // expected-error@+2{{'counted_by' only applies to pointers or C99 flexible array members}}
+    // expected-error@+1{{array has sizeless element type '__SVInt8_t'}}
+    __SVInt8_t arr[] __counted_by(count);
+};
diff --git a/clang/test/Sema/attr-counted-by-vla.c b/clang/test/Sema/attr-counted-by-vla.c
new file mode 100644
index 000000000000..3de6bd55e2d8
--- /dev/null
+++ b/clang/test/Sema/attr-counted-by-vla.c
@@ -0,0 +1,193 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+#define __counted_by(f)  __attribute__((counted_by(f)))
+
+struct bar;
+
+struct not_found {
+  int count;
+  struct bar *fam[] __counted_by(bork); // expected-error {{use of undeclared identifier 'bork'}}
+};
+
+struct no_found_count_not_in_substruct {
+  unsigned long flags;
+  unsigned char count; // expected-note {{'count' declared here}}
+  struct A {
+    int dummy;
+    int array[] __counted_by(count); // expected-error {{'counted_by' field 'count' isn't within the same struct as the flexible array}}
+  } a;
+};
+
+struct not_found_count_not_in_unnamed_substruct {
+  unsigned char count; // expected-note {{'count' declared here}}
+  struct {
+    int dummy;
+    int array[] __counted_by(count); // expected-error {{'counted_by' field 'count' isn't within the same struct as the flexible array}}
+  } a;
+};
+
+struct not_found_count_not_in_unnamed_substruct_2 {
+  struct {
+    unsigned char count; // expected-note {{'count' declared here}}
+  };
+  struct {
+    int dummy;
+    int array[] __counted_by(count); // expected-error {{'counted_by' field 'count' isn't within the same struct as the flexible array}}
+  } a;
+};
+
+struct not_found_count_in_other_unnamed_substruct {
+  struct {
+    unsigned char count;
+  } a1;
+
+  struct {
+    int dummy;
+    int array[] __counted_by(count); // expected-error {{use of undeclared identifier 'count'}}
+  };
+};
+
+struct not_found_count_in_other_substruct {
+  struct _a1 {
+    unsigned char count;
+  } a1;
+
+  struct {
+    int dummy;
+    int array[] __counted_by(count); // expected-error {{use of undeclared identifier 'count'}}
+  };
+};
+
+struct not_found_count_in_other_substruct_2 {
+  struct _a2 {
+    unsigned char count;
+  } a2;
+
+  int array[] __counted_by(count); // expected-error {{use of undeclared identifier 'count'}}
+};
+
+struct not_found_suggest {
+  int bork;
+  struct bar *fam[] __counted_by(blork); // expected-error {{use of undeclared identifier 'blork'}}
+};
+
+int global; // expected-note {{'global' declared here}}
+
+struct found_outside_of_struct {
+  int bork;
+  struct bar *fam[] __counted_by(global); // expected-error {{field 'global' in 'counted_by' not inside structure}}
+};
+
+struct self_referrential {
+  int bork;
+  struct bar *self[] __counted_by(self); // expected-error {{use of undeclared identifier 'self'}}
+};
+
+struct non_int_count {
+  double dbl_count;
+  struct bar *fam[] __counted_by(dbl_count); // expected-error {{'counted_by' requires a non-boolean integer type argument}}
+};
+
+struct array_of_ints_count {
+  int integers[2];
+  struct bar *fam[] __counted_by(integers); // expected-error {{'counted_by' requires a non-boolean integer type argument}}
+};
+
+struct not_a_fam {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to a pointer with pointee of unknown size because 'struct bar' is an incomplete type}}
+  struct bar *non_fam __counted_by(count);
+};
+
+struct not_a_c99_fam {
+  int count;
+  struct bar *non_c99_fam[0] __counted_by(count); // expected-error {{'counted_by' on arrays only applies to C99 flexible array members}}
+};
+
+struct annotated_with_anon_struct {
+  unsigned long flags;
+  struct {
+    unsigned char count;
+    int array[] __counted_by(crount); // expected-error {{use of undeclared identifier 'crount'}}
+  };
+};
+
+//==============================================================================
+// __counted_by on a struct VLA with element type that has unknown size
+//==============================================================================
+
+struct size_unknown; // expected-note 2{{forward declaration of 'struct size_unknown'}}
+struct on_member_arr_incomplete_ty_ty_pos {
+  int count;
+  // expected-error@+2{{'counted_by' only applies to pointers or C99 flexible array members}}
+  // expected-error@+1{{array has incomplete element type 'struct size_unknown'}}
+  struct size_unknown buf[] __counted_by(count);
+};
+
+struct on_member_arr_incomplete_const_ty_ty_pos {
+  int count;
+  // expected-error@+2{{'counted_by' only applies to pointers or C99 flexible array members}}
+  // expected-error@+1{{array has incomplete element type 'const struct size_unknown'}}
+  const struct size_unknown buf[] __counted_by(count);
+};
+
+struct on_member_arr_void_ty_ty_pos {
+  int count;
+  // expected-error@+2{{'counted_by' only applies to pointers or C99 flexible array members}}
+  // expected-error@+1{{array has incomplete element type 'void'}}
+  void buf[] __counted_by(count);
+};
+
+typedef void(fn_ty)(int);
+
+struct on_member_arr_fn_ptr_ty {
+  int count;
+  // An Array of function pointers is allowed
+  fn_ty* buf[] __counted_by(count);
+};
+
+struct on_member_arr_fn_ty {
+  int count;
+  // An array of functions is not allowed.
+  // expected-error@+2{{'counted_by' only applies to pointers or C99 flexible array members}}
+  // expected-error@+1{{'buf' declared as array of functions of type 'fn_ty' (aka 'void (int)')}}
+  fn_ty buf[] __counted_by(count);
+};
+
+
+// `buffer_of_structs_with_unnannotated_vla`,
+// `buffer_of_structs_with_annotated_vla`, and
+// `buffer_of_const_structs_with_annotated_vla` are currently prevented because
+// computing the size of `Arr` at runtime would require an O(N) walk of `Arr`
+// elements to take into account the length of the VLA in each struct instance.
+
+struct has_unannotated_VLA {
+  int count;
+  char buffer[];
+};
+
+struct has_annotated_VLA {
+  int count;
+  char buffer[] __counted_by(count);
+};
+
+struct buffer_of_structs_with_unnannotated_vla {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to an array with element of unknown size because 'struct has_unannotated_VLA' is a struct type with a flexible array member}}
+  struct has_unannotated_VLA Arr[] __counted_by(count);
+};
+
+
+struct buffer_of_structs_with_annotated_vla {
+  int count;
+  // expected-error@+1{{'counted_by' cannot be applied to an array with element of unknown size because 'struct has_annotated_VLA' is a struct type with a flexible array member}}
+  struct has_annotated_VLA Arr[] __counted_by(count);
+};
+
+struct buffer_of_const_structs_with_annotated_vla {
+  int count;
+  // Make sure the `const` qualifier is printed when printing the element type.
+  // expected-error@+1{{'counted_by' cannot be applied to an array with element of unknown size because 'const struct has_annotated_VLA' is a struct type with a flexible array member}}
+  const struct has_annotated_VLA Arr[] __counted_by(count);
+};
+
diff --git a/clang/test/Sema/attr-counted-by.c b/clang/test/Sema/attr-counted-by.c
deleted file mode 100644
index d5d4ebf55739..000000000000
--- a/clang/test/Sema/attr-counted-by.c
+++ /dev/null
@@ -1,112 +0,0 @@
-// RUN: %clang_cc1 -fsyntax-only -verify %s
-
-#define __counted_by(f)  __attribute__((counted_by(f)))
-
-struct bar;
-
-struct not_found {
-  int count;
-  struct bar *fam[] __counted_by(bork); // expected-error {{use of undeclared identifier 'bork'}}
-};
-
-struct no_found_count_not_in_substruct {
-  unsigned long flags;
-  unsigned char count; // expected-note {{'count' declared here}}
-  struct A {
-    int dummy;
-    int array[] __counted_by(count); // expected-error {{'counted_by' field 'count' isn't within the same struct as the flexible array}}
-  } a;
-};
-
-struct not_found_count_not_in_unnamed_substruct {
-  unsigned char count; // expected-note {{'count' declared here}}
-  struct {
-    int dummy;
-    int array[] __counted_by(count); // expected-error {{'counted_by' field 'count' isn't within the same struct as the flexible array}}
-  } a;
-};
-
-struct not_found_count_not_in_unnamed_substruct_2 {
-  struct {
-    unsigned char count; // expected-note {{'count' declared here}}
-  };
-  struct {
-    int dummy;
-    int array[] __counted_by(count); // expected-error {{'counted_by' field 'count' isn't within the same struct as the flexible array}}
-  } a;
-};
-
-struct not_found_count_in_other_unnamed_substruct {
-  struct {
-    unsigned char count;
-  } a1;
-
-  struct {
-    int dummy;
-    int array[] __counted_by(count); // expected-error {{use of undeclared identifier 'count'}}
-  };
-};
-
-struct not_found_count_in_other_substruct {
-  struct _a1 {
-    unsigned char count;
-  } a1;
-
-  struct {
-    int dummy;
-    int array[] __counted_by(count); // expected-error {{use of undeclared identifier 'count'}}
-  };
-};
-
-struct not_found_count_in_other_substruct_2 {
-  struct _a2 {
-    unsigned char count;
-  } a2;
-
-  int array[] __counted_by(count); // expected-error {{use of undeclared identifier 'count'}}
-};
-
-struct not_found_suggest {
-  int bork;
-  struct bar *fam[] __counted_by(blork); // expected-error {{use of undeclared identifier 'blork'}}
-};
-
-int global; // expected-note {{'global' declared here}}
-
-struct found_outside_of_struct {
-  int bork;
-  struct bar *fam[] __counted_by(global); // expected-error {{field 'global' in 'counted_by' not inside structure}}
-};
-
-struct self_referrential {
-  int bork;
-  struct bar *self[] __counted_by(self); // expected-error {{use of undeclared identifier 'self'}}
-};
-
-struct non_int_count {
-  double dbl_count;
-  struct bar *fam[] __counted_by(dbl_count); // expected-error {{'counted_by' requires a non-boolean integer type argument}}
-};
-
-struct array_of_ints_count {
-  int integers[2];
-  struct bar *fam[] __counted_by(integers); // expected-error {{'counted_by' requires a non-boolean integer type argument}}
-};
-
-struct not_a_fam {
-  int count;
-  struct bar *non_fam __counted_by(count); // expected-error {{'counted_by' only applies to C99 flexible array members}}
-};
-
-struct not_a_c99_fam {
-  int count;
-  struct bar *non_c99_fam[0] __counted_by(count); // expected-error {{'counted_by' only applies to C99 flexible array members}}
-};
-
-struct annotated_with_anon_struct {
-  unsigned long flags;
-  struct {
-    unsigned char count;
-    int array[] __counted_by(crount); // expected-error {{use of undeclared identifier 'crount'}}
-  };
-};
-- 
GitLab


From 1b58940c88edaad9320bd21eaa7da65f7e791552 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 17 May 2024 12:08:47 -0700
Subject: [PATCH 1163/1206] [MCAsmParser,test] Test line marker after .endr \n

Regression test for 1e5f29af81a5f6fda308074f6345b9fba4faa71c

Reduced from Linux kernel arch/x86/crypto/sha1_avx2_x86_64_asm.S
---
 llvm/test/MC/AsmParser/macro-rept.s | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/test/MC/AsmParser/macro-rept.s b/llvm/test/MC/AsmParser/macro-rept.s
index 1dc8060e1d87..13599dd81841 100644
--- a/llvm/test/MC/AsmParser/macro-rept.s
+++ b/llvm/test/MC/AsmParser/macro-rept.s
@@ -3,6 +3,8 @@
 .rept 2
     .long 1
 .endr
+# 3 "a.s"
+/// Test line marker after .endr \n.
 
 .rept 3
 .rept 2
-- 
GitLab


From 76fdc2e52719c7164e3f2cb05158a74df2b3d22e Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Fri, 17 May 2024 15:40:40 -0400
Subject: [PATCH 1164/1206] [BOLT][NFC] Rename isUnsupportedBranch to
 isReversibleBranch (#92447)

`isUnsupportedBranch` is not a very informative name, and doesn't match
its corresponding `reverseBranchCondition`, as I noted in PR #92018.
Here's a renaming to a more mnemonic name.
---
 bolt/include/bolt/Core/MCPlusBuilder.h   |  4 ++--
 bolt/lib/Core/BinaryFunction.cpp         |  4 ++--
 bolt/lib/Passes/Instrumentation.cpp      |  2 +-
 bolt/lib/Target/X86/X86MCPlusBuilder.cpp | 10 +++++-----
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index f7614cf9ac97..f7cf538bd0e8 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -438,8 +438,8 @@ public:
     return false;
   }
 
-  /// Check whether we support inverting this branch
-  virtual bool isUnsupportedBranch(const MCInst &Inst) const { return false; }
+  /// Check whether this conditional branch can be reversed
+  virtual bool isReversibleBranch(const MCInst &Inst) const { return true; }
 
   /// Return true of the instruction is of pseudo kind.
   virtual bool isPseudo(const MCInst &Inst) const {
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 4f44ba0d970c..10b93e702984 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -1284,7 +1284,7 @@ Error BinaryFunction::disassemble() {
         const bool IsCondBranch = MIB->isConditionalBranch(Instruction);
         MCSymbol *TargetSymbol = nullptr;
 
-        if (BC.MIB->isUnsupportedBranch(Instruction)) {
+        if (!BC.MIB->isReversibleBranch(Instruction)) {
           setIgnored();
           if (BinaryFunction *TargetFunc =
                   BC.getBinaryFunctionContainingAddress(TargetAddress))
@@ -3381,7 +3381,7 @@ void BinaryFunction::fixBranches() {
 
       // Reverse branch condition and swap successors.
       auto swapSuccessors = [&]() {
-        if (MIB->isUnsupportedBranch(*CondBranch)) {
+        if (!MIB->isReversibleBranch(*CondBranch)) {
           if (opts::Verbosity) {
             BC.outs() << "BOLT-INFO: unable to swap successors in " << *this
                       << '\n';
diff --git a/bolt/lib/Passes/Instrumentation.cpp b/bolt/lib/Passes/Instrumentation.cpp
index 68acff7e6a86..14f506f9ca96 100644
--- a/bolt/lib/Passes/Instrumentation.cpp
+++ b/bolt/lib/Passes/Instrumentation.cpp
@@ -480,7 +480,7 @@ void Instrumentation::instrumentFunction(BinaryFunction &Function,
       else if (BC.MIB->isUnconditionalBranch(Inst))
         HasUnconditionalBranch = true;
       else if ((!BC.MIB->isCall(Inst) && !BC.MIB->isConditionalBranch(Inst)) ||
-               BC.MIB->isUnsupportedBranch(Inst))
+               !BC.MIB->isReversibleBranch(Inst))
         continue;
 
       const uint32_t FromOffset = *BC.MIB->getOffset(Inst);
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index 8b1894953f37..8fdacffcb147 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -328,19 +328,19 @@ public:
     return false;
   }
 
-  bool isUnsupportedBranch(const MCInst &Inst) const override {
+  bool isReversibleBranch(const MCInst &Inst) const override {
     if (isDynamicBranch(Inst))
-      return true;
+      return false;
 
     switch (Inst.getOpcode()) {
     default:
-      return false;
+      return true;
     case X86::LOOP:
     case X86::LOOPE:
     case X86::LOOPNE:
     case X86::JECXZ:
     case X86::JRCXZ:
-      return true;
+      return false;
     }
   }
 
@@ -1874,7 +1874,7 @@ public:
       }
 
       // Handle conditional branches and ignore indirect branches
-      if (!isUnsupportedBranch(*I) && getCondCode(*I) == X86::COND_INVALID) {
+      if (isReversibleBranch(*I) && getCondCode(*I) == X86::COND_INVALID) {
         // Indirect branch
         return false;
       }
-- 
GitLab


From 112eadd55f06bee15caadff688ea0b45acbfa804 Mon Sep 17 00:00:00 2001
From: Dan Liew <dan@su-root.co.uk>
Date: Fri, 17 May 2024 13:05:05 -0700
Subject: [PATCH 1165/1206] [Bounds-Safety] Fix
 `pragma-attribute-supported-attributes-list.test`

0ec3b972e58bcbcdc1bebe1696ea37f2931287c3 changed the `counted_by`
attribute to be `LateAttrParseExperimentalExt`. This means the attribute
is no longer supported by `#pragma clang attribute`. However, the
`pragma-attribute-supported-attributes-list.test` wasn't updated
to account for that.

rdar://125400257
---
 clang/test/Misc/pragma-attribute-supported-attributes-list.test | 1 -
 1 file changed, 1 deletion(-)

diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index fd0e6d71baa8..99732694f72a 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -63,7 +63,6 @@
 // CHECK-NEXT: CoroOnlyDestroyWhenComplete (SubjectMatchRule_record)
 // CHECK-NEXT: CoroReturnType (SubjectMatchRule_record)
 // CHECK-NEXT: CoroWrapper (SubjectMatchRule_function)
-// CHECK-NEXT: CountedBy (SubjectMatchRule_field)
 // CHECK-NEXT: DLLExport (SubjectMatchRule_function, SubjectMatchRule_variable, SubjectMatchRule_record, SubjectMatchRule_objc_interface)
 // CHECK-NEXT: DLLImport (SubjectMatchRule_function, SubjectMatchRule_variable, SubjectMatchRule_record, SubjectMatchRule_objc_interface)
 // CHECK-NEXT: Destructor (SubjectMatchRule_function)
-- 
GitLab


From df575be9d864886684e536cd76c5a96bb0d443a6 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 17 May 2024 23:09:35 +0300
Subject: [PATCH 1166/1206] [clang][NFC] Add `const` qualifier in
 `Sema::isIncompatibleTypedef`

---
 clang/include/clang/Sema/Sema.h |  2 +-
 clang/lib/Sema/SemaDecl.cpp     | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index cd06bd9c230d..d4d4a82525a0 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3309,7 +3309,7 @@ public:
   /// Subroutines of ActOnDeclarator().
   TypedefDecl *ParseTypedefDecl(Scope *S, Declarator &D, QualType T,
                                 TypeSourceInfo *TInfo);
-  bool isIncompatibleTypedef(TypeDecl *Old, TypedefNameDecl *New);
+  bool isIncompatibleTypedef(const TypeDecl *Old, TypedefNameDecl *New);
 
   /// Describes the kind of merge to perform for availability
   /// attributes (including "deprecated", "unavailable", and "availability").
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 6c545d695fc9..f2b9202255cd 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -2465,9 +2465,9 @@ NamedDecl *Sema::LazilyCreateBuiltin(IdentifierInfo *II, unsigned ID,
 /// entity if their types are the same.
 /// FIXME: This is notionally doing the same thing as ASTReaderDecl's
 /// isSameEntity.
-static void filterNonConflictingPreviousTypedefDecls(Sema &S,
-                                                     TypedefNameDecl *Decl,
-                                                     LookupResult &Previous) {
+static void
+filterNonConflictingPreviousTypedefDecls(Sema &S, const TypedefNameDecl *Decl,
+                                         LookupResult &Previous) {
   // This is only interesting when modules are enabled.
   if (!S.getLangOpts().Modules && !S.getLangOpts().ModulesLocalVisibility)
     return;
@@ -2504,9 +2504,9 @@ static void filterNonConflictingPreviousTypedefDecls(Sema &S,
   Filter.done();
 }
 
-bool Sema::isIncompatibleTypedef(TypeDecl *Old, TypedefNameDecl *New) {
+bool Sema::isIncompatibleTypedef(const TypeDecl *Old, TypedefNameDecl *New) {
   QualType OldType;
-  if (TypedefNameDecl *OldTypedef = dyn_cast<TypedefNameDecl>(Old))
+  if (const TypedefNameDecl *OldTypedef = dyn_cast<TypedefNameDecl>(Old))
     OldType = OldTypedef->getUnderlyingType();
   else
     OldType = Context.getTypeDeclType(Old);
-- 
GitLab


From af7467ce9f447d6fe977b73db1f03a18d6bbd511 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 17 May 2024 16:49:04 -0400
Subject: [PATCH 1167/1206] [libcxx][libcxxabi] Fix build for OpenBSD (#92186)

- No indirect syscalls on OpenBSD. Instead there is a `futex` function
which issues a direct syscall.

- Monotonic clock is available despite the full POSIX suite of timers
not being available in its entirety.

  See https://lists.boost.org/boost-bugs/2015/07/41690.php and
  https://github.com/boostorg/log/commit/c98b1f459add14d5ce3e9e63e2469064601d7f71
  for a description of an analogous problem and fix for Boost.
---
 libcxx/src/atomic.cpp          | 16 ++++++++++++++--
 libcxx/src/chrono.cpp          |  4 +++-
 libcxxabi/src/cxa_guard_impl.h | 16 +++++++++++++++-
 3 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/libcxx/src/atomic.cpp b/libcxx/src/atomic.cpp
index 2b67685c8a0a..44100d4414e0 100644
--- a/libcxx/src/atomic.cpp
+++ b/libcxx/src/atomic.cpp
@@ -25,16 +25,28 @@
 #  if !defined(SYS_futex) && defined(SYS_futex_time64)
 #    define SYS_futex SYS_futex_time64
 #  endif
+#  define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__)
 
 #elif defined(__FreeBSD__)
 
 #  include <sys/types.h>
 #  include <sys/umtx.h>
 
+#  define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__)
+
+#elif defined(__OpenBSD__)
+
+#  include <sys/futex.h>
+
+// OpenBSD has no indirect syscalls
+#  define _LIBCPP_FUTEX(...) futex(__VA_ARGS__)
+
 #else // <- Add other operating systems here
 
 // Baseline needs no new headers
 
+#  define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__)
+
 #endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
@@ -44,11 +56,11 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 static void
 __libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, __cxx_contention_t __val) {
   static constexpr timespec __timeout = {2, 0};
-  syscall(SYS_futex, __ptr, FUTEX_WAIT_PRIVATE, __val, &__timeout, 0, 0);
+  _LIBCPP_FUTEX(__ptr, FUTEX_WAIT_PRIVATE, __val, &__timeout, 0, 0);
 }
 
 static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile* __ptr, bool __notify_one) {
-  syscall(SYS_futex, __ptr, FUTEX_WAKE_PRIVATE, __notify_one ? 1 : INT_MAX, 0, 0, 0);
+  _LIBCPP_FUTEX(__ptr, FUTEX_WAKE_PRIVATE, __notify_one ? 1 : INT_MAX, 0, 0, 0);
 }
 
 #elif defined(__APPLE__) && defined(_LIBCPP_USE_ULOCK)
diff --git a/libcxx/src/chrono.cpp b/libcxx/src/chrono.cpp
index c5e827c0cb59..e7d6dfbc2292 100644
--- a/libcxx/src/chrono.cpp
+++ b/libcxx/src/chrono.cpp
@@ -31,7 +31,9 @@
 #  include <sys/time.h> // for gettimeofday and timeval
 #endif
 
-#if defined(__APPLE__) || defined(__gnu_hurd__) || (defined(_POSIX_TIMERS) && _POSIX_TIMERS > 0)
+// OpenBSD does not have a fully conformant suite of POSIX timers, but
+// it does have clock_gettime and CLOCK_MONOTONIC which is all we need.
+#if defined(__APPLE__) || defined(__gnu_hurd__) || defined(__OpenBSD__) || (defined(_POSIX_TIMERS) && _POSIX_TIMERS > 0)
 #  define _LIBCPP_HAS_CLOCK_GETTIME
 #endif
 
diff --git a/libcxxabi/src/cxa_guard_impl.h b/libcxxabi/src/cxa_guard_impl.h
index 7b140d3c3604..320501cb8593 100644
--- a/libcxxabi/src/cxa_guard_impl.h
+++ b/libcxxabi/src/cxa_guard_impl.h
@@ -47,6 +47,9 @@
 #include "__cxxabi_config.h"
 #include "include/atomic_support.h" // from libc++
 #if defined(__has_include)
+#  if __has_include(<sys/futex.h>)
+#    include <sys/futex.h>
+#  endif
 #  if __has_include(<sys/syscall.h>)
 #    include <sys/syscall.h>
 #  endif
@@ -411,7 +414,18 @@ private:
 //                         Futex Implementation
 //===----------------------------------------------------------------------===//
 
-#if defined(SYS_futex)
+#if defined(__OpenBSD__)
+void PlatformFutexWait(int* addr, int expect) {
+  constexpr int WAIT = 0;
+  futex(reinterpret_cast<volatile uint32_t*>(addr), WAIT, expect, NULL, NULL);
+  __tsan_acquire(addr);
+}
+void PlatformFutexWake(int* addr) {
+  constexpr int WAKE = 1;
+  __tsan_release(addr);
+  futex(reinterpret_cast<volatile uint32_t*>(addr), WAKE, INT_MAX, NULL, NULL);
+}
+#elif defined(SYS_futex)
 void PlatformFutexWait(int* addr, int expect) {
   constexpr int WAIT = 0;
   syscall(SYS_futex, addr, WAIT, expect, 0);
-- 
GitLab


From 1a5bc7c895ddeafa0e1d269da67bbeafcd8bc1a4 Mon Sep 17 00:00:00 2001
From: Julian Lettner <yln@users.noreply.github.com>
Date: Fri, 17 May 2024 13:58:34 -0700
Subject: [PATCH 1168/1206] Fix flaky test: signal_in_mutex_lock.cpp (#92587)

Fix flaky test: the spawned thread keeps spinning
on `sampler_mutex` which may be released before
the thread is terminated based on termination
ordering.

My understanding of C++ semantics are that the
program here is invalid: the destructors of global
variables are invoked at the time of program
termination, and it is the responsibility of the
program to ensure that invoking those destructors
is safe.

rdar://126768628
---
 .../test/tsan/signal_in_mutex_lock.cpp        | 32 ++++++++++++-------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/compiler-rt/test/tsan/signal_in_mutex_lock.cpp b/compiler-rt/test/tsan/signal_in_mutex_lock.cpp
index ec99e2319840..49b853653938 100644
--- a/compiler-rt/test/tsan/signal_in_mutex_lock.cpp
+++ b/compiler-rt/test/tsan/signal_in_mutex_lock.cpp
@@ -5,6 +5,7 @@
 #include <signal.h>
 #include <stdio.h>
 
+#include <atomic>
 #include <cassert>
 #include <condition_variable>
 #include <mutex>
@@ -13,9 +14,10 @@ std::mutex sampler_mutex; //dummy mutex to lock in the thread we spawn.
 std::mutex done_mutex;    // guards the cv and done variables.
 std::condition_variable cv;
 bool done = false;
+std::atomic<bool> spin = true;
 
 void *ThreadFunc(void *x) {
-  while (true) {
+  while (spin) {
     // Lock the mutex
     std::lock_guard<std::mutex> guard(sampler_mutex);
     // Mutex is released at the end
@@ -51,20 +53,26 @@ int main() {
   pthread_t thread;
   pthread_create(&thread, NULL, ThreadFunc, NULL);
 
-  // Lock the mutex before sending the signal
-  std::lock_guard<std::mutex> guard(sampler_mutex);
-  // From now on thread 1 will be waiting for the lock
+  {
+    // Lock the mutex before sending the signal
+    std::lock_guard<std::mutex> guard(sampler_mutex);
+    // From now on thread 1 will be waiting for the lock
 
-  // Send the SIGPROF signal to thread.
-  int r = pthread_kill(thread, SIGPROF);
-  assert(r == 0);
+    // Send the SIGPROF signal to thread.
+    int r = pthread_kill(thread, SIGPROF);
+    assert(r == 0);
 
-  // Wait until signal handler sends the data.
-  std::unique_lock lk(done_mutex);
-  cv.wait(lk, [] { return done; });
+    // Wait until signal handler sends the data.
+    std::unique_lock lk(done_mutex);
+    cv.wait(lk, [] { return done; });
+
+    // We got the done variable from the signal handler. Exiting successfully.
+    fprintf(stderr, "PASS\n");
+  }
 
-  // We got the done variable from the signal handler. Exiting successfully.
-  fprintf(stderr, "PASS\n");
+  // Wait for thread to prevent it from spinning on a released mutex.
+  spin = false;
+  pthread_join(thread, nullptr);
 }
 
 // CHECK-NOT: WARNING: ThreadSanitizer:
-- 
GitLab


From 6d50a79b721760fa5e10b81ebe6d1e37f141f61b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 17 May 2024 13:58:49 -0700
Subject: [PATCH 1169/1206] [flang][cuda] Implicitly load cudadevice module in
 device/global subprogram (#92038)

This is a re-worked version of #91668. It adds the `cudadevice` module
and set the `device` attributes on its functions/subroutines so there is
no need for special case in semantic check.
`cudadevice` module is implicitly USE'd in `global`/`device` subprogram.
---
 flang/include/flang/Semantics/semantics.h     |  5 +-
 flang/lib/Semantics/resolve-names.cpp         | 20 +++++
 flang/lib/Semantics/semantics.cpp             |  8 ++
 flang/module/cudadevice.f90                   | 74 +++++++++++++++++++
 .../Semantics/cuf-device-procedures01.cuf     | 35 +++++++++
 .../Semantics/cuf-device-procedures02.cuf     | 17 +++++
 flang/tools/f18/CMakeLists.txt                | 12 ++-
 7 files changed, 166 insertions(+), 5 deletions(-)
 create mode 100644 flang/module/cudadevice.f90
 create mode 100644 flang/test/Semantics/cuf-device-procedures01.cuf
 create mode 100644 flang/test/Semantics/cuf-device-procedures02.cuf

diff --git a/flang/include/flang/Semantics/semantics.h b/flang/include/flang/Semantics/semantics.h
index e6ba71d53e92..167e61381639 100644
--- a/flang/include/flang/Semantics/semantics.h
+++ b/flang/include/flang/Semantics/semantics.h
@@ -215,8 +215,10 @@ public:
   void UseFortranBuiltinsModule();
   const Scope *GetBuiltinsScope() const { return builtinsScope_; }
 
-  void UsePPCBuiltinTypesModule();
   const Scope &GetCUDABuiltinsScope();
+  const Scope &GetCUDADeviceScope();
+
+  void UsePPCBuiltinTypesModule();
   void UsePPCBuiltinsModule();
   Scope *GetPPCBuiltinTypesScope() { return ppcBuiltinTypesScope_; }
   const Scope *GetPPCBuiltinsScope() const { return ppcBuiltinsScope_; }
@@ -292,6 +294,7 @@ private:
   const Scope *builtinsScope_{nullptr}; // module __Fortran_builtins
   Scope *ppcBuiltinTypesScope_{nullptr}; // module __Fortran_PPC_types
   std::optional<const Scope *> cudaBuiltinsScope_; // module __CUDA_builtins
+  std::optional<const Scope *> cudaDeviceScope_; // module cudadevice
   const Scope *ppcBuiltinsScope_{nullptr}; // module __ppc_intrinsics
   std::list<parser::Program> modFileParseTrees_;
   std::unique_ptr<CommonBlockMap> commonBlockMap_;
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 40eee89de131..a46c0f378d5d 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -3797,6 +3797,26 @@ bool SubprogramVisitor::Pre(const parser::PrefixSpec::Attributes &attrs) {
         subp->set_cudaSubprogramAttrs(attr);
       }
     }
+    if (auto attrs{subp->cudaSubprogramAttrs()}) {
+      if (*attrs == common::CUDASubprogramAttrs::Global ||
+          *attrs == common::CUDASubprogramAttrs::Device) {
+        const Scope &scope{currScope()};
+        const Scope *mod{FindModuleContaining(scope)};
+        if (mod && mod->GetName().value() == "cudadevice") {
+          return false;
+        }
+        // Implicitly USE the cudadevice module by copying its symbols in the
+        // current scope.
+        const Scope &cudaDeviceScope{context().GetCUDADeviceScope()};
+        for (auto sym : cudaDeviceScope.GetSymbols()) {
+          if (!currScope().FindSymbol(sym->name())) {
+            auto &localSymbol{MakeSymbol(
+                sym->name(), Attrs{}, UseDetails{sym->name(), *sym})};
+            localSymbol.flags() = sym->flags();
+          }
+        }
+      }
+    }
   }
   return false;
 }
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index 6ccd915c4dcb..d51cc62d804e 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -543,6 +543,14 @@ const Scope &SemanticsContext::GetCUDABuiltinsScope() {
   return **cudaBuiltinsScope_;
 }
 
+const Scope &SemanticsContext::GetCUDADeviceScope() {
+  if (!cudaDeviceScope_) {
+    cudaDeviceScope_ = GetBuiltinModule("cudadevice");
+    CHECK(cudaDeviceScope_.value() != nullptr);
+  }
+  return **cudaDeviceScope_;
+}
+
 void SemanticsContext::UsePPCBuiltinsModule() {
   if (ppcBuiltinsScope_ == nullptr) {
     ppcBuiltinsScope_ = GetBuiltinModule("__ppc_intrinsics");
diff --git a/flang/module/cudadevice.f90 b/flang/module/cudadevice.f90
new file mode 100644
index 000000000000..f34820dd1079
--- /dev/null
+++ b/flang/module/cudadevice.f90
@@ -0,0 +1,74 @@
+!===-- module/cudedevice.f90 -----------------------------------------------===!
+!
+! Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+! See https://llvm.org/LICENSE.txt for license information.
+! SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+!
+!===------------------------------------------------------------------------===!
+
+! CUDA Fortran procedures available in device subprogram
+
+module cudadevice
+implicit none
+
+  ! Set PRIVATE by default to explicitly only export what is meant
+  ! to be exported by this MODULE.
+  private
+
+  ! Synchronization Functions
+
+  interface
+    attributes(device) subroutine syncthreads()
+    end subroutine
+  end interface
+  public :: syncthreads
+
+  interface
+    attributes(device) integer function syncthreads_and(value)
+      integer :: value
+    end function
+  end interface
+  public :: syncthreads_and
+
+  interface
+    attributes(device) integer function syncthreads_count(value)
+      integer :: value
+    end function
+  end interface
+  public :: syncthreads_count
+
+  interface
+    attributes(device) integer function syncthreads_or(value)
+      integer :: value
+    end function
+  end interface
+  public :: syncthreads_or
+
+  interface
+    attributes(device) subroutine syncwarp(mask)
+      integer :: mask
+    end subroutine
+  end interface
+  public :: syncwarp
+
+  ! Memory Fences
+
+  interface
+    attributes(device) subroutine threadfence()
+    end subroutine
+  end interface
+  public :: threadfence
+
+  interface
+    attributes(device) subroutine threadfence_block()
+    end subroutine
+  end interface
+  public :: threadfence_block
+
+  interface
+    attributes(device) subroutine threadfence_system()
+    end subroutine
+  end interface
+  public :: threadfence_system
+
+end module
diff --git a/flang/test/Semantics/cuf-device-procedures01.cuf b/flang/test/Semantics/cuf-device-procedures01.cuf
new file mode 100644
index 000000000000..b9918d8a4ae4
--- /dev/null
+++ b/flang/test/Semantics/cuf-device-procedures01.cuf
@@ -0,0 +1,35 @@
+! RUN: %flang_fc1 -fdebug-dump-symbols %s | FileCheck %s
+
+! Test CUDA Fortran intrinsic can pass semantic
+
+attributes(global) subroutine devsub()
+  implicit none
+  integer :: ret
+
+  ! 3.6.4. Synchronization Functions
+  call syncthreads()
+  call syncwarp(1)
+  call threadfence()
+  call threadfence_block()
+  call threadfence_system()
+  ret = syncthreads_and(1)
+  ret = syncthreads_count(1)
+  ret = syncthreads_or(1)
+end
+
+! CHECK-LABEL: Subprogram scope: devsub
+! CHECK: syncthreads (Subroutine): Use from syncthreads in cudadevice
+! CHECK: syncthreads_and (Function): Use from syncthreads_and in cudadevice
+! CHECK: syncthreads_count (Function): Use from syncthreads_count in cudadevice
+! CHECK: syncthreads_or (Function): Use from syncthreads_or in cudadevice
+! CHECK: syncwarp (Subroutine): Use from syncwarp in cudadevice
+! CHECK: threadfence (Subroutine): Use from threadfence in cudadevice
+! CHECK: threadfence_block (Subroutine): Use from threadfence_block in cudadevice
+! CHECK: threadfence_system (Subroutine): Use from threadfence_system in cudadevice
+
+subroutine host()
+  call syncthreads()
+end subroutine
+
+! CHECK-LABEL: Subprogram scope: host
+! CHECK: syncthreads, EXTERNAL: HostAssoc{{$}}
diff --git a/flang/test/Semantics/cuf-device-procedures02.cuf b/flang/test/Semantics/cuf-device-procedures02.cuf
new file mode 100644
index 000000000000..c93fc4033b8f
--- /dev/null
+++ b/flang/test/Semantics/cuf-device-procedures02.cuf
@@ -0,0 +1,17 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+
+module dev
+  integer, device :: syncthreads
+
+contains
+
+  attributes(device) subroutine sub1()
+    syncthreads = 1 ! syncthreads not overwritten by cudadevice
+  end subroutine
+
+  attributes(global) subroutine sub2()
+!ERROR: 'threadfence' is use-associated from module 'cudadevice' and cannot be re-declared
+    integer :: threadfence
+  end subroutine
+end module
+
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
index 64815a1f5da6..35e1cdafd3ae 100644
--- a/flang/tools/f18/CMakeLists.txt
+++ b/flang/tools/f18/CMakeLists.txt
@@ -12,6 +12,7 @@ set(MODULES
   "__ppc_intrinsics"
   "mma"
   "__cuda_builtins"
+  "cudadevice"
   "ieee_arithmetic"
   "ieee_exceptions"
   "ieee_features"
@@ -26,11 +27,15 @@ set(MODULES
 if (NOT CMAKE_CROSSCOMPILING)
   foreach(filename ${MODULES})
     set(depends "")
+    set(opts "")
     if(${filename} STREQUAL "__fortran_builtins" OR
        ${filename} STREQUAL "__ppc_types")
     elseif(${filename} STREQUAL "__ppc_intrinsics" OR
            ${filename} STREQUAL "mma")
       set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
+    elseif(${filename} STREQUAL "cudadevice")
+      set(opts -fc1 -xcuda)
+      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
     else()
       set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
       if(NOT ${filename} STREQUAL "__fortran_type_info")
@@ -43,9 +48,8 @@ if (NOT CMAKE_CROSSCOMPILING)
     endif()
 
     # The module contains PPC vector types that needs the PPC target.
-    set(opts "")
-      if(${filename} STREQUAL "__ppc_intrinsics" OR
-         ${filename} STREQUAL "mma")
+    if(${filename} STREQUAL "__ppc_intrinsics" OR
+       ${filename} STREQUAL "mma")
       if (PowerPC IN_LIST LLVM_TARGETS_TO_BUILD)
         set(opts "--target=ppc64le")
       else()
@@ -58,7 +62,7 @@ if (NOT CMAKE_CROSSCOMPILING)
     # TODO: We may need to flag this with conditional, in case Flang is built w/o OpenMP support
     add_custom_command(OUTPUT ${base}.mod
       COMMAND ${CMAKE_COMMAND} -E make_directory ${FLANG_INTRINSIC_MODULES_DIR}
-      COMMAND flang-new -cpp -fsyntax-only ${opts} -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
+      COMMAND flang-new ${opts} -cpp -fsyntax-only -module-dir ${FLANG_INTRINSIC_MODULES_DIR}
         ${FLANG_SOURCE_DIR}/module/${filename}.f90
       DEPENDS flang-new ${FLANG_SOURCE_DIR}/module/${filename}.f90 ${FLANG_SOURCE_DIR}/module/__fortran_builtins.f90 ${depends}
     )
-- 
GitLab


From bdfb04a63d73c31ee75395064762d0d6ccb45819 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Fri, 17 May 2024 13:59:31 -0700
Subject: [PATCH 1170/1206] [lldb-dap] Bump the version to 0.2.1

Bump the version to 0.2.1 to test the publishing workflow and update the
extension README and URL.
---
 lldb/tools/lldb-dap/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/tools/lldb-dap/package.json b/lldb/tools/lldb-dap/package.json
index 45fcb83cf81d..adffc3f0a2c5 100644
--- a/lldb/tools/lldb-dap/package.json
+++ b/lldb/tools/lldb-dap/package.json
@@ -1,7 +1,7 @@
 {
   "name": "lldb-dap",
   "displayName": "LLDB DAP",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "publisher": "llvm-vs-code-extensions",
   "homepage": "https://lldb.llvm.org",
   "description": "LLDB debugging from VSCode",
-- 
GitLab


From e2db08f8f1a7dd37cd18705164f0c1188707e5b6 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 17 May 2024 14:42:48 -0700
Subject: [PATCH 1171/1206] [test] Use conventional -emit-llvm-only

Similar to a6d7828f4c50c1ec7b0b5f61fe59d7a768175dcc
---
 .../test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
index fb43bc633708..4a3a5a86a668 100644
--- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_add_sub_za16.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -verify -emit-llvm %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -verify -emit-llvm-only %s
 
 // REQUIRES: aarch64-registered-target
 
-- 
GitLab


From 8da3a8f550d0b266411b2dc177dff8591ee86cb2 Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Fri, 17 May 2024 15:13:36 -0700
Subject: [PATCH 1172/1206] [NVPTX] fixup support for over-aligned parameters
 (#92457)

This extends the NVPTX support for over-aligned parameters and return
values in a few related ways:

- Support for `alignstack` attribute, as an alternative to legacy nvvm
`!"align"` metadata entries. While we still maintain the legacy support,
long term it might be nice to auto-upgrade to `alignstack`.
- Check the alignment info when emitting the parameter list to prevent a
mismatch between alignment of caller and callee, which would previously
cause a fatal error for `ptxas`.
- Check the alignment info when emitting loads for parameters,
potentially enabling better vectorization.
---
 llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp   |  14 ++-
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp |  33 +++---
 llvm/lib/Target/NVPTX/NVPTXISelLowering.h   |   3 +
 llvm/lib/Target/NVPTX/NVPTXUtilities.cpp    |  47 +++++----
 llvm/lib/Target/NVPTX/NVPTXUtilities.h      |   5 +-
 llvm/test/CodeGen/NVPTX/param-overalign.ll  | 109 ++++++++++++++++++++
 6 files changed, 168 insertions(+), 43 deletions(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/param-overalign.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index f8b246a1a36c..f63697916d90 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -72,6 +72,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Alignment.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Endian.h"
@@ -370,11 +371,10 @@ void NVPTXAsmPrinter::printReturnValStr(const Function *F, raw_ostream &O) {
         << " func_retval0";
     } else if (ShouldPassAsArray(Ty)) {
       unsigned totalsz = DL.getTypeAllocSize(Ty);
-      unsigned retAlignment = 0;
-      if (!getAlign(*F, 0, retAlignment))
-        retAlignment = TLI->getFunctionParamOptimizedAlign(F, Ty, DL).value();
-      O << ".param .align " << retAlignment << " .b8 func_retval0[" << totalsz
-        << "]";
+      Align RetAlignment = TLI->getFunctionArgumentAlignment(
+          F, Ty, AttributeList::ReturnIndex, DL);
+      O << ".param .align " << RetAlignment.value() << " .b8 func_retval0["
+        << totalsz << "]";
     } else
       llvm_unreachable("Unknown return type");
   } else {
@@ -1558,6 +1558,10 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
 
     auto getOptimalAlignForParam = [TLI, &DL, &PAL, F,
                                     paramIndex](Type *Ty) -> Align {
+      if (MaybeAlign StackAlign =
+              getAlign(*F, paramIndex + AttributeList::FirstArgIndex))
+        return StackAlign.value();
+
       Align TypeAlign = TLI->getFunctionParamOptimizedAlign(F, Ty, DL);
       MaybeAlign ParamAlign = PAL.getParamAlignment(paramIndex);
       return std::max(TypeAlign, ParamAlign.valueOrOne());
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index b03803f52b78..1e7477cf9d60 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -1434,12 +1434,11 @@ std::string NVPTXTargetLowering::getPrototype(
 
     if (!Outs[OIdx].Flags.isByVal()) {
       if (IsTypePassedAsArray(Ty)) {
-        unsigned ParamAlign = 0;
         const CallInst *CallI = cast<CallInst>(&CB);
-        // +1 because index 0 is reserved for return type alignment
-        if (!getAlign(*CallI, i + 1, ParamAlign))
-          ParamAlign = getFunctionParamOptimizedAlign(F, Ty, DL).value();
-        O << ".param .align " << ParamAlign << " .b8 ";
+        Align ParamAlign =
+            getAlign(*CallI, i + AttributeList::FirstArgIndex)
+                .value_or(getFunctionParamOptimizedAlign(F, Ty, DL));
+        O << ".param .align " << ParamAlign.value() << " .b8 ";
         O << "_";
         O << "[" << DL.getTypeAllocSize(Ty) << "]";
         // update the index for Outs
@@ -1489,6 +1488,11 @@ std::string NVPTXTargetLowering::getPrototype(
   return Prototype;
 }
 
+Align NVPTXTargetLowering::getFunctionArgumentAlignment(
+    const Function *F, Type *Ty, unsigned Idx, const DataLayout &DL) const {
+  return getAlign(*F, Idx).value_or(getFunctionParamOptimizedAlign(F, Ty, DL));
+}
+
 Align NVPTXTargetLowering::getArgumentAlignment(const CallBase *CB, Type *Ty,
                                                 unsigned Idx,
                                                 const DataLayout &DL) const {
@@ -1497,7 +1501,6 @@ Align NVPTXTargetLowering::getArgumentAlignment(const CallBase *CB, Type *Ty,
     return DL.getABITypeAlign(Ty);
   }
 
-  unsigned Alignment = 0;
   const Function *DirectCallee = CB->getCalledFunction();
 
   if (!DirectCallee) {
@@ -1507,21 +1510,16 @@ Align NVPTXTargetLowering::getArgumentAlignment(const CallBase *CB, Type *Ty,
     // With bitcast'd call targets, the instruction will be the call
     if (const auto *CI = dyn_cast<CallInst>(CB)) {
       // Check if we have call alignment metadata
-      if (getAlign(*CI, Idx, Alignment))
-        return Align(Alignment);
+      if (MaybeAlign StackAlign = getAlign(*CI, Idx))
+        return StackAlign.value();
     }
     DirectCallee = getMaybeBitcastedCallee(CB);
   }
 
   // Check for function alignment information if we found that the
   // ultimate target is a Function
-  if (DirectCallee) {
-    if (getAlign(*DirectCallee, Idx, Alignment))
-      return Align(Alignment);
-    // If alignment information is not available, fall back to the
-    // default function param optimized type alignment
-    return getFunctionParamOptimizedAlign(DirectCallee, Ty, DL);
-  }
+  if (DirectCallee)
+    return getFunctionArgumentAlignment(DirectCallee, Ty, Idx, DL);
 
   // Call is indirect, fall back to the ABI type alignment
   return DL.getABITypeAlign(Ty);
@@ -3195,8 +3193,9 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
       if (VTs.empty())
         report_fatal_error("Empty parameter types are not supported");
 
-      auto VectorInfo =
-          VectorizePTXValueVTs(VTs, Offsets, DL.getABITypeAlign(Ty));
+      Align ArgAlign = getFunctionArgumentAlignment(
+          F, Ty, i + AttributeList::FirstArgIndex, DL);
+      auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, ArgAlign);
 
       SDValue Arg = getParamSymbol(DAG, i, PtrVT);
       int VecIdx = -1; // Index of the first element of the current vector.
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index c9db10e555ce..e211286fcc55 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -462,6 +462,9 @@ public:
                           MachineFunction &MF,
                           unsigned Intrinsic) const override;
 
+  Align getFunctionArgumentAlignment(const Function *F, Type *Ty, unsigned Idx,
+                                     const DataLayout &DL) const;
+
   /// getFunctionParamOptimizedAlign - since function arguments are passed via
   /// .param space, we may want to increase their alignment in a way that
   /// ensures that we can effectively vectorize their loads & stores. We can
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
index 35302889095f..013afe916e86 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
@@ -19,11 +19,13 @@
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
+#include "llvm/Support/Alignment.h"
 #include "llvm/Support/Mutex.h"
 #include <algorithm>
 #include <cstring>
 #include <map>
 #include <mutex>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -296,37 +298,44 @@ bool isKernelFunction(const Function &F) {
   return (x == 1);
 }
 
-bool getAlign(const Function &F, unsigned index, unsigned &align) {
+MaybeAlign getAlign(const Function &F, unsigned Index) {
+  // First check the alignstack metadata
+  if (MaybeAlign StackAlign =
+          F.getAttributes().getAttributes(Index).getStackAlignment())
+    return StackAlign;
+
+  // If that is missing, check the legacy nvvm metadata
   std::vector<unsigned> Vs;
   bool retval = findAllNVVMAnnotation(&F, "align", Vs);
   if (!retval)
-    return false;
-  for (unsigned v : Vs) {
-    if ((v >> 16) == index) {
-      align = v & 0xFFFF;
-      return true;
-    }
-  }
-  return false;
+    return std::nullopt;
+  for (unsigned V : Vs)
+    if ((V >> 16) == Index)
+      return Align(V & 0xFFFF);
+
+  return std::nullopt;
 }
 
-bool getAlign(const CallInst &I, unsigned index, unsigned &align) {
+MaybeAlign getAlign(const CallInst &I, unsigned Index) {
+  // First check the alignstack metadata
+  if (MaybeAlign StackAlign =
+          I.getAttributes().getAttributes(Index).getStackAlignment())
+    return StackAlign;
+
+  // If that is missing, check the legacy nvvm metadata
   if (MDNode *alignNode = I.getMetadata("callalign")) {
     for (int i = 0, n = alignNode->getNumOperands(); i < n; i++) {
       if (const ConstantInt *CI =
               mdconst::dyn_extract<ConstantInt>(alignNode->getOperand(i))) {
-        unsigned v = CI->getZExtValue();
-        if ((v >> 16) == index) {
-          align = v & 0xFFFF;
-          return true;
-        }
-        if ((v >> 16) > index) {
-          return false;
-        }
+        unsigned V = CI->getZExtValue();
+        if ((V >> 16) == Index)
+          return Align(V & 0xFFFF);
+        if ((V >> 16) > Index)
+          return std::nullopt;
       }
     }
   }
-  return false;
+  return std::nullopt;
 }
 
 Function *getMaybeBitcastedCallee(const CallBase *CB) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.h b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
index 449973bb53de..2872db9fa213 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.h
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
@@ -18,6 +18,7 @@
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/Alignment.h"
 #include <cstdarg>
 #include <set>
 #include <string>
@@ -60,8 +61,8 @@ bool getMinCTASm(const Function &, unsigned &);
 bool getMaxNReg(const Function &, unsigned &);
 bool isKernelFunction(const Function &);
 
-bool getAlign(const Function &, unsigned index, unsigned &);
-bool getAlign(const CallInst &, unsigned index, unsigned &);
+MaybeAlign getAlign(const Function &, unsigned);
+MaybeAlign getAlign(const CallInst &, unsigned);
 Function *getMaybeBitcastedCallee(const CallBase *CB);
 
 // PTX ABI requires all scalar argument/return values to have
diff --git a/llvm/test/CodeGen/NVPTX/param-overalign.ll b/llvm/test/CodeGen/NVPTX/param-overalign.ll
new file mode 100644
index 000000000000..63e706982f39
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/param-overalign.ll
@@ -0,0 +1,109 @@
+; RUN: llc < %s -march=nvptx | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -march=nvptx -verify-machineinstrs | %ptxas-verify %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+%struct.float2 = type { float, float }
+
+; CHECK-LABEL: .visible .func  (.param .b32 func_retval0) callee_md
+; CHECK-NEXT: (
+; CHECK-NEXT:         .param .align 8 .b8 callee_md_param_0[8]
+; CHECK-NEXT: )
+; CHECK-NEXT: ;
+
+; CHECK-LABEL: .visible .func  (.param .b32 func_retval0) callee
+; CHECK-NEXT: (
+; CHECK-NEXT:         .param .align 8 .b8 callee_param_0[8]
+; CHECK-NEXT: )
+; CHECK-NEXT: ;
+
+define float @caller_md(float %a, float %b) {
+; CHECK-LABEL: .visible .func  (.param .b32 func_retval0) caller_md(
+; CHECK-NEXT:         .param .b32 caller_md_param_0,
+; CHECK-NEXT:         .param .b32 caller_md_param_1
+; CHECK-NEXT: )
+; CHECK-NEXT: {
+
+; CHECK:         ld.param.f32 %f1, [caller_md_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [caller_md_param_1];
+; CHECK-NEXT:    {
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v2.f32 [param0+0], {%f1, %f2};
+; CHECK-NEXT:    .param .b32 retval0;
+; CHECK-NEXT:    call.uni (retval0),
+; CHECK-NEXT:    callee_md,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    ld.param.f32 %f3, [retval0+0];
+; CHECK-NEXT:    }
+; CHECK-NEXT:    st.param.f32 [func_retval0+0], %f3;
+; CHECK-NEXT:    ret;
+  %s1 = insertvalue %struct.float2 poison, float %a, 0
+  %s2 = insertvalue %struct.float2 %s1, float %b, 1
+  %r = call float @callee_md(%struct.float2 %s2)
+  ret float %r
+}
+
+define float @callee_md(%struct.float2 %a) {
+; CHECK-LABEL: .visible .func  (.param .b32 func_retval0) callee_md(
+; CHECK-NEXT:         .param .align 8 .b8 callee_md_param_0[8]
+; CHECK-NEXT: )
+; CHECK-NEXT: {
+
+; CHECK:         ld.param.v2.f32 {%f1, %f2}, [callee_md_param_0];
+; CHECK-NEXT:    add.rn.f32 %f3, %f1, %f2;
+; CHECK-NEXT:    st.param.f32 [func_retval0+0], %f3;
+; CHECK-NEXT:    ret;
+  %v0 = extractvalue %struct.float2 %a, 0
+  %v1 = extractvalue %struct.float2 %a, 1
+  %2 = fadd float %v0, %v1
+  ret float %2
+}
+
+define float @caller(float %a, float %b) {
+; CHECK-LABEL: .visible .func  (.param .b32 func_retval0) caller(
+; CHECK-NEXT:         .param .b32 caller_param_0,
+; CHECK-NEXT:         .param .b32 caller_param_1
+; CHECK-NEXT: )
+; CHECK-NEXT: {
+
+; CHECK:         ld.param.f32 %f1, [caller_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [caller_param_1];
+; CHECK-NEXT:    {
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v2.f32 [param0+0], {%f1, %f2};
+; CHECK-NEXT:    .param .b32 retval0;
+; CHECK-NEXT:    call.uni (retval0),
+; CHECK-NEXT:    callee,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    ld.param.f32 %f3, [retval0+0];
+; CHECK-NEXT:    }
+; CHECK-NEXT:    st.param.f32 [func_retval0+0], %f3;
+; CHECK-NEXT:    ret;
+  %s1 = insertvalue %struct.float2 poison, float %a, 0
+  %s2 = insertvalue %struct.float2 %s1, float %b, 1
+  %r = call float @callee(%struct.float2 %s2)
+  ret float %r
+}
+
+define float @callee(%struct.float2 alignstack(8) %a ) {
+; CHECK-LABEL: .visible .func  (.param .b32 func_retval0) callee(
+; CHECK-NEXT:         .param .align 8 .b8 callee_param_0[8]
+; CHECK-NEXT: )
+; CHECK-NEXT: {
+
+; CHECK:         ld.param.v2.f32 {%f1, %f2}, [callee_param_0];
+; CHECK-NEXT:    add.rn.f32 %f3, %f1, %f2;
+; CHECK-NEXT:    st.param.f32 [func_retval0+0], %f3;
+; CHECK-NEXT:    ret;
+  %v0 = extractvalue %struct.float2 %a, 0
+  %v1 = extractvalue %struct.float2 %a, 1
+  %2 = fadd float %v0, %v1
+  ret float %2
+}
+
+!nvvm.annotations = !{!0}
+!0 = !{ptr @callee_md, !"align", i32 u0x00010008}
-- 
GitLab


From fe2ff54590c313551e7968179b48988ff0916290 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Fri, 17 May 2024 15:18:10 -0700
Subject: [PATCH 1173/1206] [flang][runtime] Decouple scalar output APIs from
 descriptors (#92444)

For testing purposes the implementations of the output APIs like
OutputInteger32 have been simply constructing descriptors and executing
the operation through the general DescriptorIO template. This patch
decouples those APIs from that mechanism so that programs using simple
"PRINT *" statements for output can link to a smaller portion of the I/O
runtime support library. (This is the only form of I/O accepted in GPU
device code by previous CUDA Fortran and Fortran OpenACC compilers.)
---
 flang/runtime/CMakeLists.txt     |   1 +
 flang/runtime/descriptor-io.h    |   3 +-
 flang/runtime/io-api-common.h    |  93 +++++++++++++
 flang/runtime/io-api-minimal.cpp | 163 +++++++++++++++++++++++
 flang/runtime/io-api.cpp         | 220 +------------------------------
 5 files changed, 264 insertions(+), 216 deletions(-)
 create mode 100644 flang/runtime/io-api-common.h
 create mode 100644 flang/runtime/io-api-minimal.cpp

diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index bc81e1b1887b..4f7627eac81f 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -136,6 +136,7 @@ set(sources
   inquiry.cpp
   internal-unit.cpp
   io-api.cpp
+  io-api-minimal.cpp
   io-error.cpp
   io-stmt.cpp
   iostat.cpp
diff --git a/flang/runtime/descriptor-io.h b/flang/runtime/descriptor-io.h
index 0b188a12a019..ff5f683c6da5 100644
--- a/flang/runtime/descriptor-io.h
+++ b/flang/runtime/descriptor-io.h
@@ -499,8 +499,7 @@ static RT_API_ATTRS bool DescriptorIO(IoStatementState &io,
     return false;
   }
   if (!io.get_if<IoDirectionState<DIR>>()) {
-    io.GetIoErrorHandler().Crash(
-        "DescriptorIO() called for wrong I/O direction");
+    handler.Crash("DescriptorIO() called for wrong I/O direction");
     return false;
   }
   if constexpr (DIR == Direction::Input) {
diff --git a/flang/runtime/io-api-common.h b/flang/runtime/io-api-common.h
new file mode 100644
index 000000000000..e9e050f6e7e6
--- /dev/null
+++ b/flang/runtime/io-api-common.h
@@ -0,0 +1,93 @@
+//===-- runtime/io-api-common.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FLANG_RUNTIME_IO_API_COMMON_H_
+#define FLANG_RUNTIME_IO_API_COMMON_H_
+
+#include "io-stmt.h"
+#include "terminator.h"
+#include "unit.h"
+#include "flang/Common/api-attrs.h"
+#include "flang/Common/optional.h"
+#include "flang/Runtime/io-api.h"
+
+namespace Fortran::runtime::io {
+
+static inline RT_API_ATTRS Cookie NoopUnit(const Terminator &terminator,
+    int unitNumber, enum Iostat iostat = IostatOk) {
+  Cookie cookie{&New<NoopStatementState>{terminator}(
+      terminator.sourceFileName(), terminator.sourceLine(), unitNumber)
+                     .release()
+                     ->ioStatementState()};
+  if (iostat != IostatOk) {
+    cookie->GetIoErrorHandler().SetPendingError(iostat);
+  }
+  return cookie;
+}
+
+static inline RT_API_ATTRS ExternalFileUnit *GetOrCreateUnit(int unitNumber,
+    Direction direction, Fortran::common::optional<bool> isUnformatted,
+    const Terminator &terminator, Cookie &errorCookie) {
+  if (ExternalFileUnit *
+      unit{ExternalFileUnit::LookUpOrCreateAnonymous(
+          unitNumber, direction, isUnformatted, terminator)}) {
+    errorCookie = nullptr;
+    return unit;
+  } else {
+    errorCookie = NoopUnit(terminator, unitNumber, IostatBadUnitNumber);
+    return nullptr;
+  }
+}
+
+template <Direction DIR, template <Direction> class STATE, typename... A>
+RT_API_ATTRS Cookie BeginExternalListIO(
+    int unitNumber, const char *sourceFile, int sourceLine, A &&...xs) {
+  Terminator terminator{sourceFile, sourceLine};
+  Cookie errorCookie{nullptr};
+  ExternalFileUnit *unit{GetOrCreateUnit(
+      unitNumber, DIR, false /*!unformatted*/, terminator, errorCookie)};
+  if (!unit) {
+    return errorCookie;
+  }
+  if (!unit->isUnformatted.has_value()) {
+    unit->isUnformatted = false;
+  }
+  Iostat iostat{IostatOk};
+  if (*unit->isUnformatted) {
+    iostat = IostatFormattedIoOnUnformattedUnit;
+  }
+  if (ChildIo * child{unit->GetChildIo()}) {
+    if (iostat == IostatOk) {
+      iostat = child->CheckFormattingAndDirection(false, DIR);
+    }
+    if (iostat == IostatOk) {
+      return &child->BeginIoStatement<ChildListIoStatementState<DIR>>(
+          *child, sourceFile, sourceLine);
+    } else {
+      return &child->BeginIoStatement<ErroneousIoStatementState>(
+          iostat, nullptr /* no unit */, sourceFile, sourceLine);
+    }
+  } else {
+    if (iostat == IostatOk && unit->access == Access::Direct) {
+      iostat = IostatListIoOnDirectAccessUnit;
+    }
+    if (iostat == IostatOk) {
+      iostat = unit->SetDirection(DIR);
+    }
+    if (iostat == IostatOk) {
+      return &unit->BeginIoStatement<STATE<DIR>>(
+          terminator, std::forward<A>(xs)..., *unit, sourceFile, sourceLine);
+    } else {
+      return &unit->BeginIoStatement<ErroneousIoStatementState>(
+          terminator, iostat, unit, sourceFile, sourceLine);
+    }
+  }
+}
+
+} // namespace Fortran::runtime::io
+#endif // FLANG_RUNTIME_IO_API_COMMON_H_
diff --git a/flang/runtime/io-api-minimal.cpp b/flang/runtime/io-api-minimal.cpp
new file mode 100644
index 000000000000..ad76fe3de032
--- /dev/null
+++ b/flang/runtime/io-api-minimal.cpp
@@ -0,0 +1,163 @@
+//===-- runtime/io-api-minimal.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Implements the subset of the I/O statement API needed for basic
+// list-directed output (PRINT *) of intrinsic types.
+
+#include "edit-output.h"
+#include "format.h"
+#include "io-api-common.h"
+#include "io-stmt.h"
+#include "terminator.h"
+#include "tools.h"
+#include "unit.h"
+#include "flang/Runtime/io-api.h"
+
+namespace Fortran::runtime::io {
+RT_EXT_API_GROUP_BEGIN
+
+Cookie IODEF(BeginExternalListOutput)(
+    ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
+  return BeginExternalListIO<Direction::Output, ExternalListIoStatementState>(
+      unitNumber, sourceFile, sourceLine);
+}
+
+enum Iostat IODEF(EndIoStatement)(Cookie cookie) {
+  IoStatementState &io{*cookie};
+  return static_cast<enum Iostat>(io.EndIoStatement());
+}
+
+template <int KIND, typename INT = CppTypeFor<TypeCategory::Integer, KIND>>
+inline RT_API_ATTRS bool FormattedScalarIntegerOutput(
+    IoStatementState &io, INT x, const char *whence) {
+  if (io.CheckFormattedStmtType<Direction::Output>(whence)) {
+    auto edit{io.GetNextDataEdit()};
+    return edit && EditIntegerOutput<KIND>(io, *edit, x);
+  } else {
+    return false;
+  }
+}
+
+bool IODEF(OutputInteger8)(Cookie cookie, std::int8_t n) {
+  return FormattedScalarIntegerOutput<1>(*cookie, n, "OutputInteger8");
+}
+
+bool IODEF(OutputInteger16)(Cookie cookie, std::int16_t n) {
+  return FormattedScalarIntegerOutput<2>(*cookie, n, "OutputInteger16");
+}
+
+bool IODEF(OutputInteger32)(Cookie cookie, std::int32_t n) {
+  return FormattedScalarIntegerOutput<4>(*cookie, n, "OutputInteger32");
+}
+
+bool IODEF(OutputInteger64)(Cookie cookie, std::int64_t n) {
+  return FormattedScalarIntegerOutput<8>(*cookie, n, "OutputInteger64");
+}
+
+#ifdef __SIZEOF_INT128__
+bool IODEF(OutputInteger128)(Cookie cookie, common::int128_t n) {
+  return FormattedScalarIntegerOutput<16>(*cookie, n, "OutputInteger128");
+}
+#endif
+
+template <int KIND,
+    typename REAL = typename RealOutputEditing<KIND>::BinaryFloatingPoint>
+inline RT_API_ATTRS bool FormattedScalarRealOutput(
+    IoStatementState &io, REAL x, const char *whence) {
+  if (io.CheckFormattedStmtType<Direction::Output>(whence)) {
+    auto edit{io.GetNextDataEdit()};
+    return edit && RealOutputEditing<KIND>{io, x}.Edit(*edit);
+  } else {
+    return false;
+  }
+}
+
+bool IODEF(OutputReal32)(Cookie cookie, float x) {
+  return FormattedScalarRealOutput<4>(*cookie, x, "OutputReal32");
+}
+
+bool IODEF(OutputReal64)(Cookie cookie, double x) {
+  return FormattedScalarRealOutput<8>(*cookie, x, "OutputReal64");
+}
+
+template <int KIND,
+    typename REAL = typename RealOutputEditing<KIND>::BinaryFloatingPoint>
+inline RT_API_ATTRS bool FormattedScalarComplexOutput(
+    IoStatementState &io, REAL re, REAL im, const char *whence) {
+  if (io.CheckFormattedStmtType<Direction::Output>(whence)) {
+    if (io.get_if<ListDirectedStatementState<Direction::Output>>() != nullptr) {
+      DataEdit rEdit, iEdit;
+      rEdit.descriptor = DataEdit::ListDirectedRealPart;
+      iEdit.descriptor = DataEdit::ListDirectedImaginaryPart;
+      rEdit.modes = iEdit.modes = io.mutableModes();
+      return RealOutputEditing<KIND>{io, re}.Edit(rEdit) &&
+          RealOutputEditing<KIND>{io, im}.Edit(iEdit);
+    } else {
+      auto reEdit{io.GetNextDataEdit()};
+      if (reEdit && RealOutputEditing<KIND>{io, re}.Edit(*reEdit)) {
+        auto imEdit{io.GetNextDataEdit()};
+        return imEdit && RealOutputEditing<KIND>{io, im}.Edit(*imEdit);
+      }
+    }
+  }
+  return false;
+}
+
+bool IODEF(OutputComplex32)(Cookie cookie, float re, float im) {
+  return FormattedScalarComplexOutput<4>(*cookie, re, im, "OutputComplex32");
+}
+
+bool IODEF(OutputComplex64)(Cookie cookie, double re, double im) {
+  return FormattedScalarComplexOutput<8>(*cookie, re, im, "OutputComplex64");
+}
+
+bool IODEF(OutputAscii)(Cookie cookie, const char *x, std::size_t length) {
+  IoStatementState &io{*cookie};
+  if (!x) {
+    io.GetIoErrorHandler().Crash("Null address for character output item");
+  } else if (auto *listOutput{
+                 io.get_if<ListDirectedStatementState<Direction::Output>>()}) {
+    return ListDirectedCharacterOutput(io, *listOutput, x, length);
+  } else if (io.CheckFormattedStmtType<Direction::Output>("OutputAscii")) {
+    auto edit{io.GetNextDataEdit()};
+    return edit && EditCharacterOutput(io, *edit, x, length);
+  } else {
+    return false;
+  }
+}
+
+bool IODEF(OutputLogical)(Cookie cookie, bool truth) {
+  IoStatementState &io{*cookie};
+  if (auto *listOutput{
+          io.get_if<ListDirectedStatementState<Direction::Output>>()}) {
+    return ListDirectedLogicalOutput(io, *listOutput, truth);
+  } else if (io.CheckFormattedStmtType<Direction::Output>("OutputAscii")) {
+    auto edit{io.GetNextDataEdit()};
+    return edit && EditLogicalOutput(io, *edit, truth);
+  } else {
+    return false;
+  }
+}
+
+} // namespace Fortran::runtime::io
+
+#if defined(_LIBCPP_VERBOSE_ABORT)
+// Provide own definition for `std::__libcpp_verbose_abort` to avoid dependency
+// on the version provided by libc++.
+
+void std::__libcpp_verbose_abort(char const *format, ...) {
+  va_list list;
+  va_start(list, format);
+  std::vfprintf(stderr, format, list);
+  va_end(list);
+
+  std::abort();
+}
+#endif
+
+RT_EXT_API_GROUP_END
diff --git a/flang/runtime/io-api.cpp b/flang/runtime/io-api.cpp
index ccb5b576451d..bfc678c2ed02 100644
--- a/flang/runtime/io-api.cpp
+++ b/flang/runtime/io-api.cpp
@@ -8,12 +8,18 @@
 
 // Implements the I/O statement API
 
+// template function BeginExternalListIo<> is in runtime/io-api-common.h.
+// APIs BeginExternalListOutput, OutputInteger{8,16,32,64,128},
+// OutputReal{32,64}, OutputComplex{32,64}, OutputAscii, & EndIoStatement()
+// are in runtime/io-api-minimal.cpp.
+
 #include "flang/Runtime/io-api.h"
 #include "descriptor-io.h"
 #include "edit-input.h"
 #include "edit-output.h"
 #include "environment.h"
 #include "format.h"
+#include "io-api-common.h"
 #include "io-stmt.h"
 #include "terminator.h"
 #include "tools.h"
@@ -157,83 +163,6 @@ Cookie IODEF(BeginInternalFormattedInput)(const char *internal,
       sourceFile, sourceLine);
 }
 
-static RT_API_ATTRS Cookie NoopUnit(const Terminator &terminator,
-    int unitNumber, enum Iostat iostat = IostatOk) {
-  Cookie cookie{&New<NoopStatementState>{terminator}(
-      terminator.sourceFileName(), terminator.sourceLine(), unitNumber)
-                     .release()
-                     ->ioStatementState()};
-  if (iostat != IostatOk) {
-    cookie->GetIoErrorHandler().SetPendingError(iostat);
-  }
-  return cookie;
-}
-
-static RT_API_ATTRS ExternalFileUnit *GetOrCreateUnit(int unitNumber,
-    Direction direction, Fortran::common::optional<bool> isUnformatted,
-    const Terminator &terminator, Cookie &errorCookie) {
-  if (ExternalFileUnit *
-      unit{ExternalFileUnit::LookUpOrCreateAnonymous(
-          unitNumber, direction, isUnformatted, terminator)}) {
-    errorCookie = nullptr;
-    return unit;
-  } else {
-    errorCookie = NoopUnit(terminator, unitNumber, IostatBadUnitNumber);
-    return nullptr;
-  }
-}
-
-template <Direction DIR, template <Direction> class STATE, typename... A>
-RT_API_ATTRS Cookie BeginExternalListIO(
-    int unitNumber, const char *sourceFile, int sourceLine, A &&...xs) {
-  Terminator terminator{sourceFile, sourceLine};
-  Cookie errorCookie{nullptr};
-  ExternalFileUnit *unit{GetOrCreateUnit(
-      unitNumber, DIR, false /*!unformatted*/, terminator, errorCookie)};
-  if (!unit) {
-    return errorCookie;
-  }
-  if (!unit->isUnformatted.has_value()) {
-    unit->isUnformatted = false;
-  }
-  Iostat iostat{IostatOk};
-  if (*unit->isUnformatted) {
-    iostat = IostatFormattedIoOnUnformattedUnit;
-  }
-  if (ChildIo * child{unit->GetChildIo()}) {
-    if (iostat == IostatOk) {
-      iostat = child->CheckFormattingAndDirection(false, DIR);
-    }
-    if (iostat == IostatOk) {
-      return &child->BeginIoStatement<ChildListIoStatementState<DIR>>(
-          *child, sourceFile, sourceLine);
-    } else {
-      return &child->BeginIoStatement<ErroneousIoStatementState>(
-          iostat, nullptr /* no unit */, sourceFile, sourceLine);
-    }
-  } else {
-    if (iostat == IostatOk && unit->access == Access::Direct) {
-      iostat = IostatListIoOnDirectAccessUnit;
-    }
-    if (iostat == IostatOk) {
-      iostat = unit->SetDirection(DIR);
-    }
-    if (iostat == IostatOk) {
-      return &unit->BeginIoStatement<STATE<DIR>>(
-          terminator, std::forward<A>(xs)..., *unit, sourceFile, sourceLine);
-    } else {
-      return &unit->BeginIoStatement<ErroneousIoStatementState>(
-          terminator, iostat, unit, sourceFile, sourceLine);
-    }
-  }
-}
-
-Cookie IODEF(BeginExternalListOutput)(
-    ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
-  return BeginExternalListIO<Direction::Output, ExternalListIoStatementState>(
-      unitNumber, sourceFile, sourceLine);
-}
-
 Cookie IODEF(BeginExternalListInput)(
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
   return BeginExternalListIO<Direction::Input, ExternalListIoStatementState>(
@@ -1139,63 +1068,6 @@ bool IODEF(InputDescriptor)(Cookie cookie, const Descriptor &descriptor) {
   return descr::DescriptorIO<Direction::Input>(*cookie, descriptor);
 }
 
-bool IODEF(OutputInteger8)(Cookie cookie, std::int8_t n) {
-  if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger8")) {
-    return false;
-  }
-  StaticDescriptor<0> staticDescriptor;
-  Descriptor &descriptor{staticDescriptor.descriptor()};
-  descriptor.Establish(
-      TypeCategory::Integer, 1, reinterpret_cast<void *>(&n), 0);
-  return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
-}
-
-bool IODEF(OutputInteger16)(Cookie cookie, std::int16_t n) {
-  if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger16")) {
-    return false;
-  }
-  StaticDescriptor<0> staticDescriptor;
-  Descriptor &descriptor{staticDescriptor.descriptor()};
-  descriptor.Establish(
-      TypeCategory::Integer, 2, reinterpret_cast<void *>(&n), 0);
-  return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
-}
-
-bool IODEF(OutputInteger32)(Cookie cookie, std::int32_t n) {
-  if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger32")) {
-    return false;
-  }
-  StaticDescriptor<0> staticDescriptor;
-  Descriptor &descriptor{staticDescriptor.descriptor()};
-  descriptor.Establish(
-      TypeCategory::Integer, 4, reinterpret_cast<void *>(&n), 0);
-  return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
-}
-
-bool IODEF(OutputInteger64)(Cookie cookie, std::int64_t n) {
-  if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger64")) {
-    return false;
-  }
-  StaticDescriptor<0> staticDescriptor;
-  Descriptor &descriptor{staticDescriptor.descriptor()};
-  descriptor.Establish(
-      TypeCategory::Integer, 8, reinterpret_cast<void *>(&n), 0);
-  return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
-}
-
-#ifdef __SIZEOF_INT128__
-bool IODEF(OutputInteger128)(Cookie cookie, common::int128_t n) {
-  if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger128")) {
-    return false;
-  }
-  StaticDescriptor<0> staticDescriptor;
-  Descriptor &descriptor{staticDescriptor.descriptor()};
-  descriptor.Establish(
-      TypeCategory::Integer, 16, reinterpret_cast<void *>(&n), 0);
-  return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
-}
-#endif
-
 bool IODEF(InputInteger)(Cookie cookie, std::int64_t &n, int kind) {
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputInteger")) {
     return false;
@@ -1207,26 +1079,6 @@ bool IODEF(InputInteger)(Cookie cookie, std::int64_t &n, int kind) {
   return descr::DescriptorIO<Direction::Input>(*cookie, descriptor);
 }
 
-bool IODEF(OutputReal32)(Cookie cookie, float x) {
-  if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputReal32")) {
-    return false;
-  }
-  StaticDescriptor<0> staticDescriptor;
-  Descriptor &descriptor{staticDescriptor.descriptor()};
-  descriptor.Establish(TypeCategory::Real, 4, reinterpret_cast<void *>(&x), 0);
-  return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
-}
-
-bool IODEF(OutputReal64)(Cookie cookie, double x) {
-  if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputReal64")) {
-    return false;
-  }
-  StaticDescriptor<0> staticDescriptor;
-  Descriptor &descriptor{staticDescriptor.descriptor()};
-  descriptor.Establish(TypeCategory::Real, 8, reinterpret_cast<void *>(&x), 0);
-  return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
-}
-
 bool IODEF(InputReal32)(Cookie cookie, float &x) {
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputReal32")) {
     return false;
@@ -1247,30 +1099,6 @@ bool IODEF(InputReal64)(Cookie cookie, double &x) {
   return descr::DescriptorIO<Direction::Input>(*cookie, descriptor);
 }
 
-bool IODEF(OutputComplex32)(Cookie cookie, float r, float i) {
-  if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputComplex32")) {
-    return false;
-  }
-  float z[2]{r, i};
-  StaticDescriptor<0> staticDescriptor;
-  Descriptor &descriptor{staticDescriptor.descriptor()};
-  descriptor.Establish(
-      TypeCategory::Complex, 4, reinterpret_cast<void *>(&z), 0);
-  return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
-}
-
-bool IODEF(OutputComplex64)(Cookie cookie, double r, double i) {
-  if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputComplex64")) {
-    return false;
-  }
-  double z[2]{r, i};
-  StaticDescriptor<0> staticDescriptor;
-  Descriptor &descriptor{staticDescriptor.descriptor()};
-  descriptor.Establish(
-      TypeCategory::Complex, 8, reinterpret_cast<void *>(&z), 0);
-  return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
-}
-
 bool IODEF(InputComplex32)(Cookie cookie, float z[2]) {
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputComplex32")) {
     return false;
@@ -1305,10 +1133,6 @@ bool IODEF(OutputCharacter)(
   return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
 }
 
-bool IODEF(OutputAscii)(Cookie cookie, const char *x, std::size_t length) {
-  return IONAME(OutputCharacter(cookie, x, length, 1));
-}
-
 bool IODEF(InputCharacter)(
     Cookie cookie, char *x, std::size_t length, int kind) {
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputCharacter")) {
@@ -1324,17 +1148,6 @@ bool IODEF(InputAscii)(Cookie cookie, char *x, std::size_t length) {
   return IONAME(InputCharacter)(cookie, x, length, 1);
 }
 
-bool IODEF(OutputLogical)(Cookie cookie, bool truth) {
-  if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputLogical")) {
-    return false;
-  }
-  StaticDescriptor<0> staticDescriptor;
-  Descriptor &descriptor{staticDescriptor.descriptor()};
-  descriptor.Establish(
-      TypeCategory::Logical, sizeof truth, reinterpret_cast<void *>(&truth), 0);
-  return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
-}
-
 bool IODEF(InputLogical)(Cookie cookie, bool &truth) {
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputLogical")) {
     return false;
@@ -1446,11 +1259,6 @@ bool IODEF(InquireInteger64)(
   return false;
 }
 
-enum Iostat IODEF(EndIoStatement)(Cookie cookie) {
-  IoStatementState &io{*cookie};
-  return static_cast<enum Iostat>(io.EndIoStatement());
-}
-
 template <typename INT>
 static RT_API_ATTRS enum Iostat CheckUnitNumberInRangeImpl(INT unit,
     bool handleError, char *ioMsg, std::size_t ioMsgLength,
@@ -1503,19 +1311,3 @@ enum Iostat IODEF(CheckUnitNumberInRange128)(common::int128_t unit,
 #endif
 
 } // namespace Fortran::runtime::io
-
-#if defined(_LIBCPP_VERBOSE_ABORT)
-// Provide own definition for `std::__libcpp_verbose_abort` to avoid dependency
-// on the version provided by libc++.
-
-void std::__libcpp_verbose_abort(char const *format, ...) {
-  va_list list;
-  va_start(list, format);
-  std::vfprintf(stderr, format, list);
-  va_end(list);
-
-  std::abort();
-}
-#endif
-
-RT_EXT_API_GROUP_END
-- 
GitLab


From ab7930bd9f088c4fcdfd438b5207a4052c9669b8 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Fri, 17 May 2024 15:50:46 -0700
Subject: [PATCH 1174/1206] [flang] Relax checking of dummy procedures under
 BIND(C) (#92474)

As was done recently to allow derived types that are not explicitly
BIND(C), but meet the requirements of BIND(C), to be acceptable for use
in contexts nominally requiring BIND(C), this patch allows procedures
that are not explicitly BIND(C) to be used in contexts that nominally
require BIND(C) so long as (1) they meet the requirements of BIND(C),
and (2) don't use dummy arguments whose implementations may vary under
BIND(C), such as VALUE.
---
 flang/include/flang/Semantics/tools.h      |  27 +-
 flang/lib/Semantics/check-declarations.cpp | 440 +++++++++++++--------
 flang/test/Semantics/bind-c03.f90          |   6 +-
 flang/test/Semantics/bind-c09.f90          |  18 +-
 flang/test/Semantics/bind-c12.f90          |  71 +++-
 flang/test/Semantics/resolve81.f90         |   1 +
 flang/test/Semantics/resolve82.f90         |   2 +-
 7 files changed, 361 insertions(+), 204 deletions(-)

diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index 46978441a640..0b5308d9242d 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -213,8 +213,7 @@ inline bool IsCUDADeviceContext(const Scope *scope) {
 }
 
 inline bool HasCUDAAttr(const Symbol &sym) {
-  if (const auto *details{
-          sym.GetUltimate().detailsIf<semantics::ObjectEntityDetails>()}) {
+  if (const auto *details{sym.GetUltimate().detailsIf<ObjectEntityDetails>()}) {
     if (details->cudaDataAttr()) {
       return true;
     }
@@ -224,17 +223,18 @@ inline bool HasCUDAAttr(const Symbol &sym) {
 
 inline bool NeedCUDAAlloc(const Symbol &sym) {
   bool inDeviceSubprogram{IsCUDADeviceContext(&sym.owner())};
-  if (Fortran::semantics::IsDummy(sym))
+  if (IsDummy(sym)) {
     return false;
-  if (const auto *details{
-          sym.GetUltimate().detailsIf<semantics::ObjectEntityDetails>()}) {
+  }
+  if (const auto *details{sym.GetUltimate().detailsIf<ObjectEntityDetails>()}) {
     if (details->cudaDataAttr() &&
         (*details->cudaDataAttr() == common::CUDADataAttr::Device ||
             *details->cudaDataAttr() == common::CUDADataAttr::Managed ||
             *details->cudaDataAttr() == common::CUDADataAttr::Unified)) {
       // Descriptor is allocated on host when in host context.
-      if (Fortran::semantics::IsAllocatable(sym))
+      if (IsAllocatable(sym)) {
         return inDeviceSubprogram;
+      }
       return true;
     }
   }
@@ -246,7 +246,7 @@ std::optional<common::CUDADataAttr> GetCUDADataAttr(const Symbol *);
 
 // Return an error if a symbol is not accessible from a scope
 std::optional<parser::MessageFormattedText> CheckAccessibleSymbol(
-    const semantics::Scope &, const Symbol &);
+    const Scope &, const Symbol &);
 
 // Analysis of image control statements
 bool IsImageControlStmt(const parser::ExecutableConstruct &);
@@ -706,14 +706,13 @@ inline const parser::Name *getDesignatorNameIfDataRef(
 bool CouldBeDataPointerValuedFunction(const Symbol *);
 
 template <typename R, typename T>
-std::optional<R> GetConstExpr(
-    Fortran::semantics::SemanticsContext &semanticsContext, const T &x) {
-  using DefaultCharConstantType = Fortran::evaluate::Ascii;
-  if (const auto *expr{Fortran::semantics::GetExpr(semanticsContext, x)}) {
-    const auto foldExpr{Fortran::evaluate::Fold(
-        semanticsContext.foldingContext(), Fortran::common::Clone(*expr))};
+std::optional<R> GetConstExpr(SemanticsContext &semanticsContext, const T &x) {
+  using DefaultCharConstantType = evaluate::Ascii;
+  if (const auto *expr{GetExpr(semanticsContext, x)}) {
+    const auto foldExpr{evaluate::Fold(
+        semanticsContext.foldingContext(), common::Clone(*expr))};
     if constexpr (std::is_same_v<R, std::string>) {
-      return Fortran::evaluate::GetScalarConstantValue<DefaultCharConstantType>(
+      return evaluate::GetScalarConstantValue<DefaultCharConstantType>(
           foldExpr);
     }
   }
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index 527a1a9539aa..f564a0b69671 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -139,8 +139,10 @@ private:
   void CheckProcedureAssemblyName(const Symbol &symbol);
   void CheckExplicitSave(const Symbol &);
   parser::Messages WhyNotInteroperableDerivedType(const Symbol &, bool isError);
+  parser::Messages WhyNotInteroperableObject(const Symbol &, bool isError);
+  parser::Messages WhyNotInteroperableFunctionResult(const Symbol &);
+  parser::Messages WhyNotInteroperableProcedure(const Symbol &, bool isError);
   void CheckBindC(const Symbol &);
-  void CheckBindCFunctionResult(const Symbol &);
   // Check functions for defined I/O procedures
   void CheckDefinedIoProc(
       const Symbol &, const GenericDetails &, common::DefinedIo);
@@ -189,8 +191,8 @@ private:
   // Collection of target dependent assembly names of external and BIND(C)
   // procedures.
   std::map<std::string, SymbolRef> procedureAssemblyNames_;
-  // Derived types that have been examined by WhyNotInteroperableDerivedType
-  UnorderedSymbolSet examinedByWhyNotInteroperableDerivedType_;
+  // Derived types that have been examined by WhyNotInteroperable_XXX
+  UnorderedSymbolSet examinedByWhyNotInteroperable_;
 };
 
 class DistinguishabilityHelper {
@@ -438,7 +440,6 @@ void CheckHelper::Check(const Symbol &symbol) {
       messages_.Say(
           "A function result may not also be a named constant"_err_en_US);
     }
-    CheckBindCFunctionResult(symbol);
   }
   if (IsAutomatic(symbol)) {
     if (const Symbol * common{FindCommonBlockContaining(symbol)}) {
@@ -510,35 +511,6 @@ void CheckHelper::CheckExplicitSave(const Symbol &symbol) {
   }
 }
 
-void CheckHelper::CheckBindCFunctionResult(const Symbol &symbol) { // C1553
-  if (!innermostSymbol_ || !IsBindCProcedure(*innermostSymbol_)) {
-    return;
-  }
-  if (IsPointer(symbol) || IsAllocatable(symbol)) {
-    messages_.Say(
-        "BIND(C) function result cannot have ALLOCATABLE or POINTER attribute"_err_en_US);
-  }
-  if (const DeclTypeSpec * type{symbol.GetType()};
-      type && type->category() == DeclTypeSpec::Character) {
-    bool isConstOne{false}; // 18.3.1(1)
-    if (const auto &len{type->characterTypeSpec().length().GetExplicit()}) {
-      if (auto constLen{evaluate::ToInt64(*len)}) {
-        isConstOne = constLen == 1;
-      }
-    }
-    if (!isConstOne) {
-      messages_.Say(
-          "BIND(C) character function result must have length one"_err_en_US);
-    }
-  }
-  if (symbol.Rank() > 0) {
-    messages_.Say("BIND(C) function result must be scalar"_err_en_US);
-  }
-  if (symbol.Corank()) {
-    messages_.Say("BIND(C) function result cannot be a coarray"_err_en_US);
-  }
-}
-
 void CheckHelper::CheckValue(
     const Symbol &symbol, const DerivedTypeSpec *derived) { // C863 - C865
   if (IsProcedure(symbol)) {
@@ -2870,12 +2842,12 @@ void CheckHelper::CheckProcedureAssemblyName(const Symbol &symbol) {
 parser::Messages CheckHelper::WhyNotInteroperableDerivedType(
     const Symbol &symbol, bool isError) {
   parser::Messages msgs;
-  if (examinedByWhyNotInteroperableDerivedType_.find(symbol) !=
-      examinedByWhyNotInteroperableDerivedType_.end()) {
+  if (examinedByWhyNotInteroperable_.find(symbol) !=
+      examinedByWhyNotInteroperable_.end()) {
     return msgs;
   }
   isError |= symbol.attrs().test(Attr::BIND_C);
-  examinedByWhyNotInteroperableDerivedType_.insert(symbol);
+  examinedByWhyNotInteroperable_.insert(symbol);
   if (const auto *derived{symbol.detailsIf<DerivedTypeDetails>()}) {
     if (derived->sequence()) { // C1801
       msgs.Say(symbol.name(),
@@ -2971,7 +2943,7 @@ parser::Messages CheckHelper::WhyNotInteroperableDerivedType(
     if (derived->componentNames().empty()) { // F'2023 C1805
       if (context_.ShouldWarn(common::LanguageFeature::EmptyBindCDerivedType)) {
         msgs.Say(symbol.name(),
-            "A derived type with the BIND attribute should not be empty"_port_en_US);
+            "A derived type with the BIND attribute should not be empty"_warn_en_US);
       }
     }
   }
@@ -2983,7 +2955,245 @@ parser::Messages CheckHelper::WhyNotInteroperableDerivedType(
     }
   }
   if (msgs.AnyFatalError()) {
-    examinedByWhyNotInteroperableDerivedType_.erase(symbol);
+    examinedByWhyNotInteroperable_.erase(symbol);
+  }
+  return msgs;
+}
+
+static UnorderedSymbolSet CollectEntryPointsWithDummy(const Symbol &dummy) {
+  UnorderedSymbolSet entries;
+  const Scope &subpScope{dummy.owner()};
+  for (const auto &[_, ref] : subpScope.parent()) {
+    const Symbol &x{*ref};
+    if (const auto *subp{x.detailsIf<SubprogramDetails>()}) {
+      if (x.scope() == &subpScope || subp->entryScope() == &dummy.owner()) {
+        if (std::find(subp->dummyArgs().begin(), subp->dummyArgs().end(),
+                &dummy) != subp->dummyArgs().end()) {
+          entries.insert(x);
+        }
+      }
+    }
+  }
+  return entries;
+}
+
+static bool AnyNonBindCEntry(const Symbol &dummy) {
+  for (const Symbol &subp : CollectEntryPointsWithDummy(dummy)) {
+    if (!subp.attrs().test(Attr::BIND_C)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+parser::Messages CheckHelper::WhyNotInteroperableObject(
+    const Symbol &symbol, bool isError) {
+  parser::Messages msgs;
+  if (examinedByWhyNotInteroperable_.find(symbol) !=
+      examinedByWhyNotInteroperable_.end()) {
+    return msgs;
+  }
+  bool isExplicitBindC{symbol.attrs().test(Attr::BIND_C)};
+  isError |= isExplicitBindC;
+  examinedByWhyNotInteroperable_.insert(symbol);
+  CHECK(symbol.has<ObjectEntityDetails>());
+  if (isExplicitBindC && !symbol.owner().IsModule()) {
+    messages_.Say(symbol.name(),
+        "A variable with BIND(C) attribute may only appear in the specification part of a module"_err_en_US);
+  }
+  auto shape{evaluate::GetShape(foldingContext_, symbol)};
+  if (shape) {
+    if (evaluate::GetRank(*shape) == 0) { // 18.3.4
+      if (IsAllocatableOrPointer(symbol) && !IsDummy(symbol)) {
+        messages_.Say(symbol.name(),
+            "A scalar interoperable variable may not be ALLOCATABLE or POINTER"_err_en_US);
+      }
+    } else if (auto extents{
+                   evaluate::AsConstantExtents(foldingContext_, *shape)}) {
+      if (evaluate::GetSize(*extents) == 0) {
+        msgs.Say(symbol.name(),
+            "Interoperable array must have at least one element"_err_en_US);
+      }
+    } else if (!evaluate::IsExplicitShape(symbol) &&
+        !IsAssumedSizeArray(symbol) &&
+        !(IsDummy(symbol) && !symbol.attrs().test(Attr::VALUE))) {
+      msgs.Say(symbol.name(),
+          "BIND(C) array must have explicit shape or be assumed-size unless a dummy argument without the VALUE attribute"_err_en_US);
+    }
+  }
+  if (const auto *type{symbol.GetType()}) {
+    const auto *derived{type->AsDerived()};
+    if (derived) {
+      if (derived->typeSymbol().attrs().test(Attr::BIND_C)) {
+      } else if (isError) {
+        if (auto *msg{messages_.Say(symbol.name(),
+                "The derived type of a BIND(C) object must also be BIND(C)"_err_en_US)}) {
+          msg->Attach(derived->typeSymbol().name(), "Non-BIND(C) type"_en_US);
+        }
+        context_.SetError(symbol);
+      } else if (auto bad{WhyNotInteroperableDerivedType(
+                     derived->typeSymbol(), /*isError=*/false)};
+                 bad.AnyFatalError()) {
+        if (auto *msg{messages_.Say(symbol.name(),
+                "The derived type of an interoperable object must be interoperable, but is not"_err_en_US)}) {
+          msg->Attach(
+              derived->typeSymbol().name(), "Non-interoperable type"_en_US);
+          bad.AttachTo(*msg, parser::Severity::None);
+        }
+      } else {
+        if (auto *msg{messages_.Say(symbol.name(),
+                "The derived type of an interoperable object should be BIND(C)"_warn_en_US)}) {
+          msg->Attach(derived->typeSymbol().name(), "Non-BIND(C) type"_en_US);
+        }
+      }
+    }
+    if (type->IsAssumedType()) { // ok
+    } else if (IsAssumedLengthCharacter(symbol)) {
+      if (AnyNonBindCEntry(symbol)) {
+        msgs.Say(symbol.name(),
+            "An assumed-length dummy argument must not appear in a non-BIND(C) entry in a subprogram with an entry that must be interoperable"_err_en_US);
+      }
+    } else if (IsAllocatableOrPointer(symbol) &&
+        type->category() == DeclTypeSpec::Character &&
+        type->characterTypeSpec().length().isDeferred()) {
+      // ok; F'2023 18.3.7 p2(6)
+    } else if (derived ||
+        IsInteroperableIntrinsicType(*type, context_.languageFeatures())) {
+      // F'2023 18.3.7 p2(4,5)
+    } else if (type->category() == DeclTypeSpec::Logical) {
+      if (context_.ShouldWarn(common::UsageWarning::LogicalVsCBool) &&
+          !InModuleFile()) {
+        if (IsDummy(symbol)) {
+          msgs.Say(symbol.name(),
+              "A BIND(C) LOGICAL dummy argument should have the interoperable KIND=C_BOOL"_port_en_US);
+        } else {
+          msgs.Say(symbol.name(),
+              "A BIND(C) LOGICAL object should have the interoperable KIND=C_BOOL"_port_en_US);
+        }
+      }
+    } else if (symbol.attrs().test(Attr::VALUE)) {
+      msgs.Say(symbol.name(),
+          "A BIND(C) VALUE dummy argument must have an interoperable type"_err_en_US);
+    } else {
+      msgs.Say(symbol.name(),
+          "A BIND(C) object must have an interoperable type"_err_en_US);
+    }
+  }
+  if (IsOptional(symbol) && !symbol.attrs().test(Attr::VALUE)) {
+    msgs.Say(symbol.name(),
+        "An interoperable procedure with an OPTIONAL dummy argument might not be portable"_port_en_US);
+  }
+  if (symbol.attrs().test(Attr::VALUE)) {
+    if (AnyNonBindCEntry(symbol)) {
+      msgs.Say(symbol.name(),
+          "A VALUE dummy argument must not appear in a non-BIND(C) entry of a subprogram with an entry that must be interoperable"_err_en_US);
+    }
+  }
+  if (IsDescriptor(symbol) && IsPointer(symbol) &&
+      symbol.attrs().test(Attr::CONTIGUOUS)) {
+    msgs.Say(symbol.name(),
+        "An interoperable pointer must not be CONTIGUOUS"_err_en_US);
+  }
+  if (msgs.AnyFatalError()) {
+    examinedByWhyNotInteroperable_.erase(symbol);
+  }
+  return msgs;
+}
+
+parser::Messages CheckHelper::WhyNotInteroperableFunctionResult(
+    const Symbol &symbol) {
+  parser::Messages msgs;
+  if (IsPointer(symbol) || IsAllocatable(symbol)) {
+    msgs.Say(symbol.name(),
+        "Interoperable function result may not have ALLOCATABLE or POINTER attribute"_err_en_US);
+  }
+  if (const DeclTypeSpec * type{symbol.GetType()};
+      type && type->category() == DeclTypeSpec::Character) {
+    bool isConstOne{false}; // 18.3.1(1)
+    if (const auto &len{type->characterTypeSpec().length().GetExplicit()}) {
+      if (auto constLen{evaluate::ToInt64(*len)}) {
+        isConstOne = constLen == 1;
+      }
+    }
+    if (!isConstOne) {
+      msgs.Say(symbol.name(),
+          "Interoperable character function result must have length one"_err_en_US);
+    }
+  }
+  if (symbol.Rank() > 0) {
+    msgs.Say(symbol.name(),
+        "Interoperable function result must be scalar"_err_en_US);
+  }
+  if (symbol.Corank()) {
+    msgs.Say(symbol.name(),
+        "Interoperable function result may not be a coarray"_err_en_US);
+  }
+  return msgs;
+}
+
+parser::Messages CheckHelper::WhyNotInteroperableProcedure(
+    const Symbol &symbol, bool isError) {
+  parser::Messages msgs;
+  if (examinedByWhyNotInteroperable_.find(symbol) !=
+      examinedByWhyNotInteroperable_.end()) {
+    return msgs;
+  }
+  isError |= symbol.attrs().test(Attr::BIND_C);
+  examinedByWhyNotInteroperable_.insert(symbol);
+  if (const auto *proc{symbol.detailsIf<ProcEntityDetails>()}) {
+    if (isError) {
+      if (!proc->procInterface() ||
+          !proc->procInterface()->attrs().test(Attr::BIND_C)) {
+        msgs.Say(symbol.name(),
+            "An interface name with the BIND attribute must appear if the BIND attribute appears in a procedure declaration"_err_en_US);
+      }
+    } else if (!proc->procInterface()) {
+      msgs.Say(symbol.name(),
+          "An interoperable procedure should have an interface"_port_en_US);
+    } else if (!proc->procInterface()->attrs().test(Attr::BIND_C)) {
+      auto bad{WhyNotInteroperableProcedure(
+          *proc->procInterface(), /*isError=*/false)};
+      if (bad.AnyFatalError()) {
+        bad.AttachTo(msgs.Say(symbol.name(),
+            "An interoperable procedure must have an interoperable interface"_err_en_US));
+      } else {
+        msgs.Say(symbol.name(),
+            "An interoperable procedure should have an interface with the BIND attribute"_warn_en_US);
+      }
+    }
+  } else if (const auto *subp{symbol.detailsIf<SubprogramDetails>()}) {
+    for (const Symbol *dummy : subp->dummyArgs()) {
+      if (dummy) {
+        parser::Messages dummyMsgs;
+        if (dummy->has<ProcEntityDetails>() ||
+            dummy->has<SubprogramDetails>()) {
+          dummyMsgs = WhyNotInteroperableProcedure(*dummy, /*isError=*/false);
+          if (dummyMsgs.empty() && !dummy->attrs().test(Attr::BIND_C)) {
+            dummyMsgs.Say(dummy->name(),
+                "A dummy procedure of an interoperable procedure should be BIND(C)"_warn_en_US);
+          }
+        } else if (dummy->has<ObjectEntityDetails>()) {
+          dummyMsgs = WhyNotInteroperableObject(*dummy, /*isError=*/false);
+        } else {
+          CheckBindC(*dummy);
+        }
+        msgs.Annex(std::move(dummyMsgs));
+      } else {
+        msgs.Say(symbol.name(),
+            "A subprogram interface with the BIND attribute may not have an alternate return argument"_err_en_US);
+      }
+    }
+    if (subp->isFunction()) {
+      if (subp->result().has<ObjectEntityDetails>()) {
+        msgs.Annex(WhyNotInteroperableFunctionResult(subp->result()));
+      } else {
+        msgs.Say(subp->result().name(),
+            "The result of an interoperable function must be a data object"_err_en_US);
+      }
+    }
+  }
+  if (msgs.AnyFatalError()) {
+    examinedByWhyNotInteroperable_.erase(symbol);
   }
   return msgs;
 }
@@ -2998,6 +3208,7 @@ void CheckHelper::CheckBindC(const Symbol &symbol) {
     // symbol must be interoperable (e.g., dummy argument of interoperable
     // procedure interface) but is not itself BIND(C).
   }
+  parser::Messages whyNot;
   if (const std::string * bindName{symbol.GetBindName()};
       bindName) { // has a binding name
     if (!bindName->empty()) {
@@ -3032,143 +3243,24 @@ void CheckHelper::CheckBindC(const Symbol &symbol) {
     }
   }
   if (symbol.has<ObjectEntityDetails>()) {
-    if (isExplicitBindC && !symbol.owner().IsModule()) {
-      messages_.Say(symbol.name(),
-          "A variable with BIND(C) attribute may only appear in the specification part of a module"_err_en_US);
-      context_.SetError(symbol);
-    }
-    auto shape{evaluate::GetShape(foldingContext_, symbol)};
-    if (shape) {
-      if (evaluate::GetRank(*shape) == 0) { // 18.3.4
-        if (isExplicitBindC && IsAllocatableOrPointer(symbol)) {
-          messages_.Say(symbol.name(),
-              "A scalar interoperable variable may not be ALLOCATABLE or POINTER"_err_en_US);
-          context_.SetError(symbol);
-        }
-      } else { // 18.3.5
-        if (auto extents{
-                evaluate::AsConstantExtents(foldingContext_, *shape)}) {
-          if (evaluate::GetSize(*extents) == 0) {
-            SayWithDeclaration(symbol, symbol.name(),
-                "Interoperable array must have at least one element"_err_en_US);
-            context_.SetError(symbol);
-          }
-        } else if ((isExplicitBindC || symbol.attrs().test(Attr::VALUE)) &&
-            !evaluate::IsExplicitShape(symbol) && !IsAssumedSizeArray(symbol)) {
-          SayWithDeclaration(symbol, symbol.name(),
-              "BIND(C) array must have explicit shape or be assumed-size unless a dummy argument without the VALUE attribute"_err_en_US);
-          context_.SetError(symbol);
-        }
-      }
-    }
-    if (const auto *type{symbol.GetType()}) {
-      const auto *derived{type->AsDerived()};
-      if (derived) {
-        if (derived->typeSymbol().attrs().test(Attr::BIND_C)) {
-        } else if (isExplicitBindC) {
-          if (auto *msg{messages_.Say(symbol.name(),
-                  "The derived type of a BIND(C) object must also be BIND(C)"_err_en_US)}) {
-            msg->Attach(derived->typeSymbol().name(), "Non-BIND(C) type"_en_US);
-          }
-          context_.SetError(symbol);
-        } else if (auto bad{WhyNotInteroperableDerivedType(
-                       derived->typeSymbol(), /*isError=*/false)};
-                   bad.AnyFatalError()) {
-          if (auto *msg{messages_.Say(symbol.name(),
-                  "The derived type of an interoperable object must be interoperable, but is not"_err_en_US)}) {
-            msg->Attach(
-                derived->typeSymbol().name(), "Non-interoperable type"_en_US);
-            bad.AttachTo(*msg, parser::Severity::None);
-          }
-          context_.SetError(symbol);
-        } else if (context_.ShouldWarn(
-                       common::LanguageFeature::NonBindCInteroperability) &&
-            !InModuleFile()) {
-          if (auto *msg{messages_.Say(symbol.name(),
-                  "The derived type of an interoperable object should be BIND(C)"_warn_en_US)}) {
-            msg->Attach(derived->typeSymbol().name(), "Non-BIND(C) type"_en_US);
-          }
-        }
-      }
-      if (type->IsAssumedType() || IsAssumedLengthCharacter(symbol)) {
-        // ok
-      } else if (IsAllocatableOrPointer(symbol) &&
-          type->category() == DeclTypeSpec::Character &&
-          type->characterTypeSpec().length().isDeferred()) {
-        // ok; F'2023 18.3.7 p2(6)
-      } else if (derived ||
-          IsInteroperableIntrinsicType(*type, context_.languageFeatures())) {
-        // F'2023 18.3.7 p2(4,5)
-      } else if (type->category() == DeclTypeSpec::Logical) {
-        if (context_.ShouldWarn(common::UsageWarning::LogicalVsCBool)) {
-          if (IsDummy(symbol)) {
-            WarnIfNotInModuleFile(symbol.name(),
-                "A BIND(C) LOGICAL dummy argument should have the interoperable KIND=C_BOOL"_port_en_US);
-          } else {
-            WarnIfNotInModuleFile(symbol.name(),
-                "A BIND(C) LOGICAL object should have the interoperable KIND=C_BOOL"_port_en_US);
-          }
-        }
-      } else if (symbol.attrs().test(Attr::VALUE)) {
-        messages_.Say(symbol.name(),
-            "A BIND(C) VALUE dummy argument must have an interoperable type"_err_en_US);
-        context_.SetError(symbol);
-      } else {
-        messages_.Say(symbol.name(),
-            "A BIND(C) object must have an interoperable type"_err_en_US);
-        context_.SetError(symbol);
-      }
-    }
-    if (IsOptional(symbol) && !symbol.attrs().test(Attr::VALUE)) {
-      if (context_.ShouldWarn(common::UsageWarning::Portability)) {
-        WarnIfNotInModuleFile(symbol.name(),
-            "An interoperable procedure with an OPTIONAL dummy argument might not be portable"_port_en_US);
-      }
-    }
-    if (IsDescriptor(symbol) && IsPointer(symbol) &&
-        symbol.attrs().test(Attr::CONTIGUOUS)) {
-      messages_.Say(symbol.name(),
-          "An interoperable pointer must not be CONTIGUOUS"_err_en_US);
-    }
-  } else if (const auto *proc{symbol.detailsIf<ProcEntityDetails>()}) {
-    if (!IsBindCProcedure(symbol) && proc->isDummy()) {
-      messages_.Say(symbol.name(),
-          "A dummy procedure to an interoperable procedure must also be interoperable"_err_en_US);
-      context_.SetError(symbol);
-    } else if (!proc->procInterface()) {
-      if (context_.ShouldWarn(
-              common::LanguageFeature::NonBindCInteroperability)) {
-        WarnIfNotInModuleFile(symbol.name(),
-            "An interface name with BIND attribute should be specified if the BIND attribute is specified in a procedure declaration statement"_warn_en_US);
-      }
-    } else if (!proc->procInterface()->attrs().test(Attr::BIND_C)) {
-      messages_.Say(symbol.name(),
-          "An interface name with BIND attribute must be specified if the BIND attribute is specified in a procedure declaration statement"_err_en_US);
-      context_.SetError(symbol);
-    }
-  } else if (const auto *subp{symbol.detailsIf<SubprogramDetails>()}) {
-    for (const Symbol *dummy : subp->dummyArgs()) {
-      if (dummy) {
-        CheckBindC(*dummy);
-      } else {
-        messages_.Say(symbol.name(),
-            "A subprogram interface with the BIND attribute may not have an alternate return argument"_err_en_US);
-        context_.SetError(symbol);
-      }
-    }
+    whyNot = WhyNotInteroperableObject(symbol, /*isError=*/isExplicitBindC);
+  } else if (symbol.has<ProcEntityDetails>() ||
+      symbol.has<SubprogramDetails>()) {
+    whyNot = WhyNotInteroperableProcedure(symbol, /*isError=*/isExplicitBindC);
   } else if (symbol.has<DerivedTypeDetails>()) {
-    if (auto msgs{WhyNotInteroperableDerivedType(symbol, /*isError=*/false)};
-        !msgs.empty()) {
-      bool anyFatal{msgs.AnyFatalError()};
-      if (msgs.AnyFatalError() ||
-          (!InModuleFile() &&
-              context_.ShouldWarn(
-                  common::LanguageFeature::NonBindCInteroperability))) {
-        context_.messages().Annex(std::move(msgs));
-      }
-      if (anyFatal) {
-        context_.SetError(symbol);
-      }
+    whyNot =
+        WhyNotInteroperableDerivedType(symbol, /*isError=*/isExplicitBindC);
+  }
+  if (!whyNot.empty()) {
+    bool anyFatal{whyNot.AnyFatalError()};
+    if (anyFatal ||
+        (!InModuleFile() &&
+            context_.ShouldWarn(
+                common::LanguageFeature::NonBindCInteroperability))) {
+      context_.messages().Annex(std::move(whyNot));
+    }
+    if (anyFatal) {
+      context_.SetError(symbol);
     }
   }
 }
diff --git a/flang/test/Semantics/bind-c03.f90 b/flang/test/Semantics/bind-c03.f90
index c37cb2bccb1f..c0d2fa1555d8 100644
--- a/flang/test/Semantics/bind-c03.f90
+++ b/flang/test/Semantics/bind-c03.f90
@@ -21,13 +21,13 @@ module m
   procedure(proc1), bind(c) :: pc1 ! no error
   procedure(proc3), bind(c) :: pc4 ! no error
 
-  !ERROR: An interface name with BIND attribute must be specified if the BIND attribute is specified in a procedure declaration statement
+  !ERROR: An interface name with the BIND attribute must appear if the BIND attribute appears in a procedure declaration
   procedure(proc2), bind(c) :: pc2
 
-  !WARNING: An interface name with BIND attribute should be specified if the BIND attribute is specified in a procedure declaration statement
+  !ERROR: An interface name with the BIND attribute must appear if the BIND attribute appears in a procedure declaration
   procedure(integer), bind(c) :: pc3
 
-  !WARNING: An interface name with BIND attribute should be specified if the BIND attribute is specified in a procedure declaration statement
+  !ERROR: An interface name with the BIND attribute must appear if the BIND attribute appears in a procedure declaration
   procedure(), bind(c) :: pc5
 
 end
diff --git a/flang/test/Semantics/bind-c09.f90 b/flang/test/Semantics/bind-c09.f90
index fe1972057e67..953f2d751234 100644
--- a/flang/test/Semantics/bind-c09.f90
+++ b/flang/test/Semantics/bind-c09.f90
@@ -2,33 +2,33 @@
 ! Check for C1553 and 18.3.4(1)
 
 function func1() result(res) bind(c)
-  ! ERROR: BIND(C) function result cannot have ALLOCATABLE or POINTER attribute
+  ! ERROR: Interoperable function result may not have ALLOCATABLE or POINTER attribute
   integer, pointer :: res
 end
 
 function func2() result(res) bind(c)
-  ! ERROR: BIND(C) function result cannot have ALLOCATABLE or POINTER attribute
+  ! ERROR: Interoperable function result may not have ALLOCATABLE or POINTER attribute
   integer, allocatable :: res
 end
 
 function func3() result(res) bind(c)
-  ! ERROR: BIND(C) function result must be scalar
+  !ERROR: Interoperable function result must be scalar
   integer :: res(2)
 end
 
 function func4() result(res) bind(c)
-  ! ERROR: BIND(C) character function result must have length one
+  ! ERROR: Interoperable character function result must have length one
   character(*) :: res
 end
 
 function func5(n) result(res) bind(c)
   integer :: n
-  ! ERROR: BIND(C) character function result must have length one
+  ! ERROR: Interoperable character function result must have length one
   character(n) :: res
 end
 
 function func6() result(res) bind(c)
-  ! ERROR: BIND(C) character function result must have length one
+  ! ERROR: Interoperable character function result must have length one
   character(2) :: res
 end
 
@@ -38,12 +38,12 @@ function func7() result(res) bind(c)
 end
 
 function func8() result(res) bind(c)
-  ! ERROR: BIND(C) function result cannot have ALLOCATABLE or POINTER attribute
-  ! ERROR: BIND(C) character function result must have length one
+  ! ERROR: Interoperable function result may not have ALLOCATABLE or POINTER attribute
+  ! ERROR: Interoperable character function result must have length one
   character(:), pointer :: res
 end
 
 function func9() result(res) bind(c)
-  ! ERROR: BIND(C) function result cannot be a coarray
+  ! ERROR: Interoperable function result may not be a coarray
   integer :: res[10, *]
 end
diff --git a/flang/test/Semantics/bind-c12.f90 b/flang/test/Semantics/bind-c12.f90
index 1b60967d8b31..55af8a93b5b5 100644
--- a/flang/test/Semantics/bind-c12.f90
+++ b/flang/test/Semantics/bind-c12.f90
@@ -1,5 +1,70 @@
-! RUN: %python %S/test_errors.py %s %flang_fc1
-!ERROR: A dummy procedure to an interoperable procedure must also be interoperable
-subroutine subr(e) bind(c)
+!RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic -Werror
+
+!PORTABILITY: An interoperable procedure should have an interface
+subroutine subr1(e) bind(c)
   external e
 end
+
+subroutine subr2(p) bind(c)
+  !PORTABILITY: An interoperable procedure should have an interface
+  procedure() :: p
+end
+
+subroutine subr3(p) bind(c)
+  !PORTABILITY: An interoperable procedure should have an interface
+  procedure(real) :: p
+end
+
+subroutine subr4(p) bind(c)
+  interface
+    !PORTABILITY: A dummy procedure of an interoperable procedure should be BIND(C)
+    subroutine p(n)
+      integer, intent(in) :: n
+    end
+  end interface
+end
+
+subroutine subr5(p) bind(c)
+  interface
+    subroutine p(c)
+      !ERROR: An assumed-length dummy argument must not appear in a non-BIND(C) entry in a subprogram with an entry that must be interoperable
+      character(*), intent(in) :: c
+    end
+  end interface
+end
+
+subroutine subr6(p) bind(c)
+  interface
+    function p()
+      !ERROR: Interoperable function result must be scalar
+      real p(1)
+    end
+  end interface
+end
+
+subroutine subr7(p) bind(c)
+  interface
+    !ERROR: Interoperable character function result must have length one
+    character(*) function p()
+    end
+  end interface
+end
+
+subroutine subr8(p) bind(c)
+  interface
+    subroutine p(n)
+      !ERROR: A VALUE dummy argument must not appear in a non-BIND(C) entry of a subprogram with an entry that must be interoperable
+      integer, intent(in), value :: n
+    end
+  end interface
+end
+
+subroutine subr9(p) bind(c)
+  !ERROR: An interface name with the BIND attribute must appear if the BIND attribute appears in a procedure declaration
+  procedure(q), bind(c), pointer :: p
+  interface
+    function q()
+      real q(1)
+    end
+  end interface
+end
diff --git a/flang/test/Semantics/resolve81.f90 b/flang/test/Semantics/resolve81.f90
index 5f0b66669423..db5b19f1155e 100644
--- a/flang/test/Semantics/resolve81.f90
+++ b/flang/test/Semantics/resolve81.f90
@@ -29,6 +29,7 @@ module m
   real, external, external :: externFunc
   !WARNING: Attribute 'INTRINSIC' cannot be used more than once
   !ERROR: 'cos' may not have both the BIND(C) and INTRINSIC attributes
+  !ERROR: An interface name with the BIND attribute must appear if the BIND attribute appears in a procedure declaration
   real, intrinsic, bind(c), intrinsic :: cos
   !WARNING: Attribute 'BIND(C)' cannot be used more than once
   integer, bind(c), volatile, bind(c) :: bindVar
diff --git a/flang/test/Semantics/resolve82.f90 b/flang/test/Semantics/resolve82.f90
index 99c0f4120218..88339742efdb 100644
--- a/flang/test/Semantics/resolve82.f90
+++ b/flang/test/Semantics/resolve82.f90
@@ -19,7 +19,7 @@ module m
   !WARNING: Attribute 'PRIVATE' cannot be used more than once
   procedure(procFunc), private, pointer, private :: proc2
   !WARNING: Attribute 'BIND(C)' cannot be used more than once
-  !ERROR: An interface name with BIND attribute must be specified if the BIND attribute is specified in a procedure declaration statement
+  !ERROR: An interface name with the BIND attribute must appear if the BIND attribute appears in a procedure declaration
   procedure(procFunc), bind(c), pointer, bind(c) :: proc3
   !WARNING: Attribute 'PROTECTED' cannot be used more than once
   procedure(procFunc), protected, pointer, protected :: proc4
-- 
GitLab


From b80e0fbcfb71b24d762d8b48069153b9e16679be Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Fri, 17 May 2024 16:03:44 -0700
Subject: [PATCH 1175/1206] [mlir][polynomial] split attributes into its own
 tablegen (#92613)

Out of tree we have other dialects that use the ring attribute, but we
get compilation errors when generating ops while pulling in all the
Polynomial tablegen ops (there's no `-dialect` flag in `mlir-tblgen` for
op generation like there is for attributes and types).

This PR simply moves the attributes into its own file, so it can be
included separately, and this also requires moving the dialect
declaration into its own file.
---
 .../mlir/Dialect/Polynomial/IR/CMakeLists.txt |   2 +-
 .../mlir/Dialect/Polynomial/IR/Polynomial.td  | 153 +-----------------
 .../Polynomial/IR/PolynomialAttributes.td     | 125 ++++++++++++++
 .../Polynomial/IR/PolynomialDialect.td        |  55 +++++++
 .../llvm-project-overlay/mlir/BUILD.bazel     |   2 +-
 5 files changed, 184 insertions(+), 153 deletions(-)
 create mode 100644 mlir/include/mlir/Dialect/Polynomial/IR/PolynomialAttributes.td
 create mode 100644 mlir/include/mlir/Dialect/Polynomial/IR/PolynomialDialect.td

diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
index e8f48f13be2b..ecdea158ddef 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
@@ -1,7 +1,7 @@
 add_mlir_dialect(Polynomial polynomial)
 add_mlir_doc(Polynomial PolynomialDialect Dialects/ -gen-dialect-doc -dialect=polynomial)
 
-set(LLVM_TARGET_DEFINITIONS Polynomial.td)
+set(LLVM_TARGET_DEFINITIONS PolynomialAttributes.td)
 mlir_tablegen(PolynomialAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=polynomial)
 mlir_tablegen(PolynomialAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=polynomial)
 add_public_tablegen_target(MLIRPolynomialAttributesIncGen)
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
index 537be4832e8f..294f58ae084b 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
@@ -13,157 +13,8 @@ include "mlir/IR/BuiltinAttributes.td"
 include "mlir/IR/OpBase.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
-
-def Polynomial_Dialect : Dialect {
-  let name = "polynomial";
-  let cppNamespace = "::mlir::polynomial";
-  let description = [{
-    The Polynomial dialect defines single-variable polynomial types and
-    operations.
-
-    The simplest use of `polynomial` is to represent mathematical operations in
-    a polynomial ring `R[x]`, where `R` is another MLIR type like `i32`.
-
-    More generally, this dialect supports representing polynomial operations in a
-    quotient ring `R[X]/(f(x))` for some statically fixed polynomial `f(x)`.
-    Two polyomials `p(x), q(x)` are considered equal in this ring if they have the
-    same remainder when dividing by `f(x)`. When a modulus is given, ring operations
-    are performed with reductions modulo `f(x)` and relative to the coefficient ring
-    `R`.
-
-    Examples:
-
-    ```mlir
-    // A constant polynomial in a ring with i32 coefficients and no polynomial modulus
-    #ring = #polynomial.ring<coefficientType=i32>
-    %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
-
-    // A constant polynomial in a ring with i32 coefficients, modulo (x^1024 + 1)
-    #modulus = #polynomial.int_polynomial<1 + x**1024>
-    #ring = #polynomial.ring<coefficientType=i32, polynomialModulus=#modulus>
-    %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
-
-    // A constant polynomial in a ring with i32 coefficients, with a polynomial
-    // modulus of (x^1024 + 1) and a coefficient modulus of 17.
-    #modulus = #polynomial.int_polynomial<1 + x**1024>
-    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=17:i32, polynomialModulus=#modulus>
-    %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
-    ```
-  }];
-
-  let useDefaultTypePrinterParser = 1;
-  let useDefaultAttributePrinterParser = 1;
-}
-
-class Polynomial_Attr<string name, string attrMnemonic, list<Trait> traits = []>
-    : AttrDef<Polynomial_Dialect, name, traits> {
-  let mnemonic = attrMnemonic;
-}
-
-def Polynomial_IntPolynomialAttr : Polynomial_Attr<"IntPolynomial", "int_polynomial"> {
-  let summary = "An attribute containing a single-variable polynomial with integer coefficients.";
-  let description = [{
-    A polynomial attribute represents a single-variable polynomial with integer
-    coefficients, which is used to define the modulus of a `RingAttr`, as well
-    as to define constants and perform constant folding for `polynomial` ops.
-
-    The polynomial must be expressed as a list of monomial terms, with addition
-    or subtraction between them. The choice of variable name is arbitrary, but
-    must be consistent across all the monomials used to define a single
-    attribute. The order of monomial terms is arbitrary, each monomial degree
-    must occur at most once.
-
-    Example:
-
-    ```mlir
-    #poly = #polynomial.int_polynomial<x**1024 + 1>
-    ```
-  }];
-  let parameters = (ins "::mlir::polynomial::IntPolynomial":$polynomial);
-  let hasCustomAssemblyFormat = 1;
-}
-
-def Polynomial_FloatPolynomialAttr : Polynomial_Attr<"FloatPolynomial", "float_polynomial"> {
-  let summary = "An attribute containing a single-variable polynomial with double precision floating point coefficients.";
-  let description = [{
-    A polynomial attribute represents a single-variable polynomial with double
-    precision floating point coefficients.
-
-    The polynomial must be expressed as a list of monomial terms, with addition
-    or subtraction between them. The choice of variable name is arbitrary, but
-    must be consistent across all the monomials used to define a single
-    attribute. The order of monomial terms is arbitrary, each monomial degree
-    must occur at most once.
-
-    Example:
-
-    ```mlir
-    #poly = #polynomial.float_polynomial<0.5 x**7 + 1.5>
-    ```
-  }];
-  let parameters = (ins "FloatPolynomial":$polynomial);
-  let hasCustomAssemblyFormat = 1;
-}
-
-def Polynomial_RingAttr : Polynomial_Attr<"Ring", "ring"> {
-  let summary = "An attribute specifying a polynomial ring.";
-  let description = [{
-    A ring describes the domain in which polynomial arithmetic occurs. The ring
-    attribute in `polynomial` represents the more specific case of polynomials
-    with a single indeterminate; whose coefficients can be represented by
-    another MLIR type (`coefficientType`); and, if the coefficient type is
-    integral, whose coefficients are taken modulo some statically known modulus
-    (`coefficientModulus`).
-
-    Additionally, a polynomial ring can specify a _polynomialModulus_, which converts
-    polynomial arithmetic to the analogue of modular integer arithmetic, where
-    each polynomial is represented as its remainder when dividing by the
-    modulus. For single-variable polynomials, an "polynomialModulus" is always specificed
-    via a single polynomial, which we call `polynomialModulus`.
-
-    An expressive example is polynomials with i32 coefficients, whose
-    coefficients are taken modulo `2**32 - 5`, with a polynomial modulus of
-    `x**1024 - 1`.
-
-    ```mlir
-    #poly_mod = #polynomial.int_polynomial<-1 + x**1024>
-    #ring = #polynomial.ring<coefficientType=i32,
-                             coefficientModulus=4294967291:i32,
-                             polynomialModulus=#poly_mod>
-
-    %0 = ... : polynomial.polynomial<#ring>
-    ```
-
-    In this case, the value of a polynomial is always "converted" to a
-    canonical form by applying repeated reductions by setting `x**1024 = 1`
-    and simplifying.
-
-    The coefficient and polynomial modulus parameters are optional, and the
-    coefficient modulus is only allowed if the coefficient type is integral.
-  }];
-
-  let parameters = (ins
-    "Type": $coefficientType,
-    OptionalParameter<"::mlir::IntegerAttr">: $coefficientModulus,
-    OptionalParameter<"::mlir::polynomial::IntPolynomialAttr">: $polynomialModulus,
-    OptionalParameter<"::mlir::IntegerAttr">: $primitiveRoot
-  );
-  let assemblyFormat = "`<` struct(params) `>`";
-  let builders = [
-    AttrBuilderWithInferredContext<
-        (ins "::mlir::Type":$coefficientTy,
-              CArg<"::mlir::IntegerAttr", "nullptr"> :$coefficientModulusAttr,
-              CArg<"::mlir::polynomial::IntPolynomialAttr", "nullptr"> :$polynomialModulusAttr,
-              CArg<"::mlir::IntegerAttr", "nullptr"> :$primitiveRootAttr), [{
-      return $_get(
-        coefficientTy.getContext(),
-        coefficientTy,
-        coefficientModulusAttr,
-        polynomialModulusAttr,
-        primitiveRootAttr);
-    }]>,
-  ];
-}
+include "mlir/Dialect/Polynomial/IR/PolynomialDialect.td"
+include "mlir/Dialect/Polynomial/IR/PolynomialAttributes.td"
 
 class Polynomial_Type<string name, string typeMnemonic>
     : TypeDef<Polynomial_Dialect, name> {
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialAttributes.td b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialAttributes.td
new file mode 100644
index 000000000000..e5dbfa7fa21e
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialAttributes.td
@@ -0,0 +1,125 @@
+//===- PolynomialOps.td - Polynomial dialect ---------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef POLYNOMIAL_ATTRIBUTES
+#define POLYNOMIAL_ATTRIBUTES
+
+include "mlir/IR/BuiltinAttributes.td"
+include "mlir/Dialect/Polynomial/IR/PolynomialDialect.td"
+
+class Polynomial_Attr<string name, string attrMnemonic, list<Trait> traits = []>
+    : AttrDef<Polynomial_Dialect, name, traits> {
+  let mnemonic = attrMnemonic;
+}
+
+def Polynomial_IntPolynomialAttr : Polynomial_Attr<"IntPolynomial", "int_polynomial"> {
+  let summary = "An attribute containing a single-variable polynomial with integer coefficients.";
+  let description = [{
+    A polynomial attribute represents a single-variable polynomial with integer
+    coefficients, which is used to define the modulus of a `RingAttr`, as well
+    as to define constants and perform constant folding for `polynomial` ops.
+
+    The polynomial must be expressed as a list of monomial terms, with addition
+    or subtraction between them. The choice of variable name is arbitrary, but
+    must be consistent across all the monomials used to define a single
+    attribute. The order of monomial terms is arbitrary, each monomial degree
+    must occur at most once.
+
+    Example:
+
+    ```mlir
+    #poly = #polynomial.int_polynomial<x**1024 + 1>
+    ```
+  }];
+  let parameters = (ins "::mlir::polynomial::IntPolynomial":$polynomial);
+  let hasCustomAssemblyFormat = 1;
+}
+
+def Polynomial_FloatPolynomialAttr : Polynomial_Attr<"FloatPolynomial", "float_polynomial"> {
+  let summary = "An attribute containing a single-variable polynomial with double precision floating point coefficients.";
+  let description = [{
+    A polynomial attribute represents a single-variable polynomial with double
+    precision floating point coefficients.
+
+    The polynomial must be expressed as a list of monomial terms, with addition
+    or subtraction between them. The choice of variable name is arbitrary, but
+    must be consistent across all the monomials used to define a single
+    attribute. The order of monomial terms is arbitrary, each monomial degree
+    must occur at most once.
+
+    Example:
+
+    ```mlir
+    #poly = #polynomial.float_polynomial<0.5 x**7 + 1.5>
+    ```
+  }];
+  let parameters = (ins "FloatPolynomial":$polynomial);
+  let hasCustomAssemblyFormat = 1;
+}
+
+def Polynomial_RingAttr : Polynomial_Attr<"Ring", "ring"> {
+  let summary = "An attribute specifying a polynomial ring.";
+  let description = [{
+    A ring describes the domain in which polynomial arithmetic occurs. The ring
+    attribute in `polynomial` represents the more specific case of polynomials
+    with a single indeterminate; whose coefficients can be represented by
+    another MLIR type (`coefficientType`); and, if the coefficient type is
+    integral, whose coefficients are taken modulo some statically known modulus
+    (`coefficientModulus`).
+
+    Additionally, a polynomial ring can specify a _polynomialModulus_, which converts
+    polynomial arithmetic to the analogue of modular integer arithmetic, where
+    each polynomial is represented as its remainder when dividing by the
+    modulus. For single-variable polynomials, an "polynomialModulus" is always specificed
+    via a single polynomial, which we call `polynomialModulus`.
+
+    An expressive example is polynomials with i32 coefficients, whose
+    coefficients are taken modulo `2**32 - 5`, with a polynomial modulus of
+    `x**1024 - 1`.
+
+    ```mlir
+    #poly_mod = #polynomial.int_polynomial<-1 + x**1024>
+    #ring = #polynomial.ring<coefficientType=i32,
+                             coefficientModulus=4294967291:i32,
+                             polynomialModulus=#poly_mod>
+
+    %0 = ... : polynomial.polynomial<#ring>
+    ```
+
+    In this case, the value of a polynomial is always "converted" to a
+    canonical form by applying repeated reductions by setting `x**1024 = 1`
+    and simplifying.
+
+    The coefficient and polynomial modulus parameters are optional, and the
+    coefficient modulus is only allowed if the coefficient type is integral.
+  }];
+
+  let parameters = (ins
+    "Type": $coefficientType,
+    OptionalParameter<"::mlir::IntegerAttr">: $coefficientModulus,
+    OptionalParameter<"::mlir::polynomial::IntPolynomialAttr">: $polynomialModulus,
+    OptionalParameter<"::mlir::IntegerAttr">: $primitiveRoot
+  );
+  let assemblyFormat = "`<` struct(params) `>`";
+  let builders = [
+    AttrBuilderWithInferredContext<
+        (ins "::mlir::Type":$coefficientTy,
+              CArg<"::mlir::IntegerAttr", "nullptr"> :$coefficientModulusAttr,
+              CArg<"::mlir::polynomial::IntPolynomialAttr", "nullptr"> :$polynomialModulusAttr,
+              CArg<"::mlir::IntegerAttr", "nullptr"> :$primitiveRootAttr), [{
+      return $_get(
+        coefficientTy.getContext(),
+        coefficientTy,
+        coefficientModulusAttr,
+        polynomialModulusAttr,
+        primitiveRootAttr);
+    }]>,
+  ];
+}
+
+#endif // POLYNOMIAL_ATTRIBUTES
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialDialect.td b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialDialect.td
new file mode 100644
index 000000000000..b0573b3715f7
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialDialect.td
@@ -0,0 +1,55 @@
+//===- PolynomialDialect.td - Polynomial dialect base ------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef POLYNOMIAL_DIALECT
+#define POLYNOMIAL_DIALECT
+
+include "mlir/IR/OpBase.td"
+
+def Polynomial_Dialect : Dialect {
+  let name = "polynomial";
+  let cppNamespace = "::mlir::polynomial";
+  let description = [{
+    The Polynomial dialect defines single-variable polynomial types and
+    operations.
+
+    The simplest use of `polynomial` is to represent mathematical operations in
+    a polynomial ring `R[x]`, where `R` is another MLIR type like `i32`.
+
+    More generally, this dialect supports representing polynomial operations in a
+    quotient ring `R[X]/(f(x))` for some statically fixed polynomial `f(x)`.
+    Two polyomials `p(x), q(x)` are considered equal in this ring if they have the
+    same remainder when dividing by `f(x)`. When a modulus is given, ring operations
+    are performed with reductions modulo `f(x)` and relative to the coefficient ring
+    `R`.
+
+    Examples:
+
+    ```mlir
+    // A constant polynomial in a ring with i32 coefficients and no polynomial modulus
+    #ring = #polynomial.ring<coefficientType=i32>
+    %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
+
+    // A constant polynomial in a ring with i32 coefficients, modulo (x^1024 + 1)
+    #modulus = #polynomial.int_polynomial<1 + x**1024>
+    #ring = #polynomial.ring<coefficientType=i32, polynomialModulus=#modulus>
+    %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
+
+    // A constant polynomial in a ring with i32 coefficients, with a polynomial
+    // modulus of (x^1024 + 1) and a coefficient modulus of 17.
+    #modulus = #polynomial.int_polynomial<1 + x**1024>
+    #ring = #polynomial.ring<coefficientType=i32, coefficientModulus=17:i32, polynomialModulus=#modulus>
+    %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
+    ```
+  }];
+
+  let useDefaultTypePrinterParser = 1;
+  let useDefaultAttributePrinterParser = 1;
+}
+
+#endif // POLYNOMIAL_OPS
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index dac759cfd100..71fca298e9b9 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -6737,7 +6737,7 @@ cc_library(
 
 td_library(
     name = "PolynomialTdFiles",
-    srcs = ["include/mlir/Dialect/Polynomial/IR/Polynomial.td"],
+    srcs = glob(["include/mlir/Dialect/Polynomial/IR/*.td"]),
     includes = ["include"],
     deps = [
         ":BuiltinDialectTdFiles",
-- 
GitLab


From cef6387e52578366c2332275dad88b9953b55336 Mon Sep 17 00:00:00 2001
From: Dan Liew <dan@su-root.co.uk>
Date: Fri, 17 May 2024 16:12:31 -0700
Subject: [PATCH 1176/1206] [Bounds-Safety] Temporarily relax a `counted_by`
 attribute restriction on flexible array members

In 0ec3b972e58bcbcdc1bebe1696ea37f2931287c3 an additional restriction
was added when applying the `counted_by` attribute to flexible array
members in structs. The restriction prevented the element type being
a struct that itself had a flexible array member. E.g.:

```
struct has_unannotated_VLA {
  int count;
  char buffer[];
};

struct buffer_of_structs_with_unnannotated_vla {
  int count;
  struct has_unannotated_VLA Arr[] __counted_by(count);
};
```

In this example assuming the size of `Arr` is `sizeof(struct
has_unannotated_VLA)*count` (which is what the attribute says) is wrong
because it doesn't account for the size of
`has_unannotated_VLA::buffer`. This is why this kind of code construct
was treated as an error.

However, it turns out existing Linux kernel code used the attribute
on a flexible array member in this way
(https://github.com/llvm/llvm-project/pull/90786#issuecomment-2118416515).

To unbreak the build this restriction is downgraded to a warning with
the plan to make it an error again once the errornous use of the
attribute in the Linux kernel is resolved.
---
 clang/include/clang/Basic/DiagnosticGroups.td   |  4 ++++
 .../include/clang/Basic/DiagnosticSemaKinds.td  |  8 +++++++-
 clang/lib/Sema/SemaDeclAttr.cpp                 | 17 +++++++++++++++--
 clang/test/Sema/attr-counted-by-vla.c           |  9 ++++++---
 4 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 4cb4f3d999f7..4fad4d1a0eca 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -1447,6 +1447,10 @@ def FunctionMultiVersioning
 
 def NoDeref : DiagGroup<"noderef">;
 
+// -fbounds-safety and bounds annotation related warnings
+def BoundsSafetyCountedByEltTyUnknownSize :
+  DiagGroup<"bounds-safety-counted-by-elt-type-unknown-size">;
+
 // A group for cross translation unit static analysis related warnings.
 def CrossTU : DiagGroup<"ctu">;
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 8e6596410c5d..1efa3af121c1 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -6552,7 +6552,7 @@ def err_counted_by_attr_refer_to_union : Error<
 def note_flexible_array_counted_by_attr_field : Note<
   "field %0 declared here">;
 def err_counted_by_attr_pointee_unknown_size : Error<
-  "'counted_by' cannot be applied to %select{"
+  "'counted_by' %select{cannot|should not}3 be applied to %select{"
     "a pointer with pointee|" // pointer
     "an array with element}0" // array
   " of unknown size because %1 is %select{"
@@ -6561,8 +6561,14 @@ def err_counted_by_attr_pointee_unknown_size : Error<
     "a function type|"     // CountedByInvalidPointeeTypeKind::FUNCTION
     // CountedByInvalidPointeeTypeKind::FLEXIBLE_ARRAY_MEMBER
     "a struct type with a flexible array member"
+    "%select{|. This will be an error in a future compiler version}3"
+    ""
   "}2">;
 
+def warn_counted_by_attr_elt_type_unknown_size :
+  Warning<err_counted_by_attr_pointee_unknown_size.Summary>,
+  InGroup<BoundsSafetyCountedByEltTyUnknownSize>;
+
 let CategoryName = "ARC Semantic Issue" in {
 
 // ARC-mode diagnostics.
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index c8b71631076b..e816ea3647a7 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -8687,6 +8687,7 @@ static bool CheckCountedByAttrOnField(
   // Note: The `Decl::isFlexibleArrayMemberLike` check earlier on means
   // only `PointeeTy->isStructureTypeWithFlexibleArrayMember()` is reachable
   // when `FieldTy->isArrayType()`.
+  bool ShouldWarn = false;
   if (PointeeTy->isIncompleteType()) {
     InvalidTypeKind = CountedByInvalidPointeeTypeKind::INCOMPLETE;
   } else if (PointeeTy->isSizelessType()) {
@@ -8694,13 +8695,25 @@ static bool CheckCountedByAttrOnField(
   } else if (PointeeTy->isFunctionType()) {
     InvalidTypeKind = CountedByInvalidPointeeTypeKind::FUNCTION;
   } else if (PointeeTy->isStructureTypeWithFlexibleArrayMember()) {
+    if (FieldTy->isArrayType()) {
+      // This is a workaround for the Linux kernel that has already adopted
+      // `counted_by` on a FAM where the pointee is a struct with a FAM. This
+      // should be an error because computing the bounds of the array cannot be
+      // done correctly without manually traversing every struct object in the
+      // array at runtime. To allow the code to be built this error is
+      // downgraded to a warning.
+      ShouldWarn = true;
+    }
     InvalidTypeKind = CountedByInvalidPointeeTypeKind::FLEXIBLE_ARRAY_MEMBER;
   }
 
   if (InvalidTypeKind != CountedByInvalidPointeeTypeKind::VALID) {
-    S.Diag(FD->getBeginLoc(), diag::err_counted_by_attr_pointee_unknown_size)
+    unsigned DiagID = ShouldWarn
+                          ? diag::warn_counted_by_attr_elt_type_unknown_size
+                          : diag::err_counted_by_attr_pointee_unknown_size;
+    S.Diag(FD->getBeginLoc(), DiagID)
         << SelectPtrOrArr << PointeeTy << (int)InvalidTypeKind
-        << FD->getSourceRange();
+        << (ShouldWarn ? 1 : 0) << FD->getSourceRange();
     return true;
   }
 
diff --git a/clang/test/Sema/attr-counted-by-vla.c b/clang/test/Sema/attr-counted-by-vla.c
index 3de6bd55e2d8..b25f719f3b95 100644
--- a/clang/test/Sema/attr-counted-by-vla.c
+++ b/clang/test/Sema/attr-counted-by-vla.c
@@ -173,21 +173,24 @@ struct has_annotated_VLA {
 
 struct buffer_of_structs_with_unnannotated_vla {
   int count;
-  // expected-error@+1{{'counted_by' cannot be applied to an array with element of unknown size because 'struct has_unannotated_VLA' is a struct type with a flexible array member}}
+  // Treating this as a warning is a temporary fix for existing attribute adopters. It **SHOULD BE AN ERROR**.
+  // expected-warning@+1{{'counted_by' should not be applied to an array with element of unknown size because 'struct has_unannotated_VLA' is a struct type with a flexible array member. This will be an error in a future compiler version}}
   struct has_unannotated_VLA Arr[] __counted_by(count);
 };
 
 
 struct buffer_of_structs_with_annotated_vla {
   int count;
-  // expected-error@+1{{'counted_by' cannot be applied to an array with element of unknown size because 'struct has_annotated_VLA' is a struct type with a flexible array member}}
+  // Treating this as a warning is a temporary fix for existing attribute adopters. It **SHOULD BE AN ERROR**.
+  // expected-warning@+1{{'counted_by' should not be applied to an array with element of unknown size because 'struct has_annotated_VLA' is a struct type with a flexible array member. This will be an error in a future compiler version}}
   struct has_annotated_VLA Arr[] __counted_by(count);
 };
 
 struct buffer_of_const_structs_with_annotated_vla {
   int count;
+  // Treating this as a warning is a temporary fix for existing attribute adopters. It **SHOULD BE AN ERROR**.
   // Make sure the `const` qualifier is printed when printing the element type.
-  // expected-error@+1{{'counted_by' cannot be applied to an array with element of unknown size because 'const struct has_annotated_VLA' is a struct type with a flexible array member}}
+  // expected-warning@+1{{'counted_by' should not be applied to an array with element of unknown size because 'const struct has_annotated_VLA' is a struct type with a flexible array member. This will be an error in a future compiler version}}
   const struct has_annotated_VLA Arr[] __counted_by(count);
 };
 
-- 
GitLab


From 86e511bc145ffe6e08f2f440b329c5ed9cb38532 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Fri, 17 May 2024 16:26:57 -0700
Subject: [PATCH 1177/1206] [flang] Fixed
 https://lab.llvm.org/buildbot/#/builders/270/builds/14406

---
 flang/runtime/io-api.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang/runtime/io-api.cpp b/flang/runtime/io-api.cpp
index bfc678c2ed02..e3c6b9e5ca89 100644
--- a/flang/runtime/io-api.cpp
+++ b/flang/runtime/io-api.cpp
@@ -1310,4 +1310,5 @@ enum Iostat IODEF(CheckUnitNumberInRange128)(common::int128_t unit,
 }
 #endif
 
+RT_EXT_API_GROUP_END
 } // namespace Fortran::runtime::io
-- 
GitLab


From 881f20e958e80bd30463fc57d2d3e891bcb8a571 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman@google.com>
Date: Fri, 17 May 2024 23:59:28 +0000
Subject: [PATCH 1178/1206] Revert "[ctx_profile] Integration test (#92456)"

This reverts commit 487d5af6482ea5f074c12d29d7e376d3fc697706.

This was causing failures on some buildbots.
https://lab.llvm.org/buildbot/#/builders/247/builds/18559
---
 compiler-rt/lib/ctx_profile/CMakeLists.txt    |   9 --
 .../TestCases/generate-context.cpp            | 103 ------------------
 compiler-rt/test/ctx_profile/lit.cfg.py       |   4 -
 3 files changed, 116 deletions(-)
 delete mode 100644 compiler-rt/test/ctx_profile/TestCases/generate-context.cpp

diff --git a/compiler-rt/lib/ctx_profile/CMakeLists.txt b/compiler-rt/lib/ctx_profile/CMakeLists.txt
index ab7bf3241fd6..1fa70594b28a 100644
--- a/compiler-rt/lib/ctx_profile/CMakeLists.txt
+++ b/compiler-rt/lib/ctx_profile/CMakeLists.txt
@@ -18,12 +18,3 @@ append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ EXTRA_FLAGS)
 if(COMPILER_RT_INCLUDE_TESTS)
   add_subdirectory(tests)
 endif()
-
-add_compiler_rt_runtime(clang_rt.ctx_profile
-  STATIC
-  ARCHS ${CTX_PROFILE_SUPPORTED_ARCH}
-  OBJECT_LIBS RTSanitizerCommon RTSanitizerCommonLibc
-  CFLAGS ${EXTRA_FLAGS}
-  SOURCES ${CTX_PROFILE_SOURCES}
-  ADDITIONAL_HEADERS ${CTX_PROFILE_HEADERS}
-  PARENT_TARGET ctx_profile)
\ No newline at end of file
diff --git a/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp b/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp
deleted file mode 100644
index 981d6170091c..000000000000
--- a/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp
+++ /dev/null
@@ -1,103 +0,0 @@
-// Simple integration test for contextual instrumentation
-//
-// Copy the header defining ContextNode.
-// RUN: mkdir -p %t_include
-// RUN: cp %llvm_src/include/llvm/ProfileData/CtxInstrContextNode.h %t_include/
-//
-// Compile with ctx instrumentation "on". We treat "theRoot" as callgraph root.
-// RUN: %clangxx %s -lclang_rt.ctx_profile -I%t_include -O2 -o %t.bin -mllvm -profile-context-root=theRoot
-//
-// Run the binary, and observe the profile fetch handler's output.
-// RUN: %t.bin | FileCheck %s
-
-#include "CtxInstrContextNode.h"
-#include <cstdio>
-#include <iostream>
-
-using namespace llvm::ctx_profile;
-extern "C" bool __llvm_ctx_profile_fetch(void *Data,
-                                         bool (*Writer)(void *,
-                                                        const ContextNode &));
-
-// avoid name mangling
-extern "C" {
-__attribute__((noinline)) void someFunction(int I) {
-  if (I % 2)
-    printf("check odd\n");
-  else
-    printf("check even\n");
-}
-
-// block inlining because the pre-inliner otherwise will inline this - it's
-// too small.
-__attribute__((noinline)) void theRoot() {
-  printf("check 1\n");
-  someFunction(1);
-#pragma nounroll
-  for (auto I = 0; I < 2; ++I) {
-    someFunction(I);
-  }
-}
-}
-
-// Make sure the program actually ran correctly.
-// CHECK: check 1
-// CHECK-NEXT: check odd
-// CHECK-NEXT: check even
-// CHECK-NEXT: check odd
-
-void printProfile(const ContextNode &Node, const std::string &Indent,
-                  const std::string &Increment) {
-  std::cout << Indent << "Guid: " << Node.guid() << std::endl;
-  std::cout << Indent << "Entries: " << Node.entrycount() << std::endl;
-  std::cout << Indent << Node.counters_size() << " counters and "
-            << Node.callsites_size() << " callsites" << std::endl;
-  std::cout << Indent << "Counter values: ";
-  for (uint32_t I = 0U; I < Node.counters_size(); ++I)
-    std::cout << Node.counters()[I] << " ";
-  std::cout << std::endl;
-  for (uint32_t I = 0U; I < Node.callsites_size(); ++I)
-    for (const auto *N = Node.subContexts()[I]; N; N = N->next()) {
-      std::cout << Indent << "At Index " << I << ":" << std::endl;
-      printProfile(*N, Indent + Increment, Increment);
-    }
-}
-
-// 8657661246551306189 is theRoot. We expect 2 callsites and 2 counters - one
-// for the entry basic block and one for the loop.
-// 6759619411192316602 is someFunction. We expect all context instances to show
-// the same nr of counters and callsites, but the counters will be different.
-// The first context is for the first callsite with theRoot as parent, and the
-// second counter in someFunction will be 0 (we pass an odd nr, and the other
-// path gets instrumented).
-// The second context is in the loop. We expect 2 entries and each of the
-// branches would be taken once, so the second counter is 1.
-// CHECK-NEXT: Guid: 8657661246551306189
-// CHECK-NEXT: Entries: 1
-// CHECK-NEXT: 2 counters and 3 callsites
-// CHECK-NEXT: Counter values: 1 2
-// CHECK-NEXT: At Index 1:
-// CHECK-NEXT:   Guid: 6759619411192316602
-// CHECK-NEXT:   Entries: 1
-// CHECK-NEXT:   2 counters and 2 callsites
-// CHECK-NEXT:   Counter values: 1 0
-// CHECK-NEXT: At Index 2:
-// CHECK-NEXT:   Guid: 6759619411192316602
-// CHECK-NEXT:   Entries: 2
-// CHECK-NEXT:   2 counters and 2 callsites
-// CHECK-NEXT:   Counter values: 2 1
-
-bool profileWriter() {
-  return __llvm_ctx_profile_fetch(
-      nullptr, +[](void *, const ContextNode &Node) {
-        printProfile(Node, "", "  ");
-        return true;
-      });
-}
-
-int main(int argc, char **argv) {
-  theRoot();
-  // This would be implemented in a specific RPC handler, but here we just call
-  // it directly.
-  return !profileWriter();
-}
diff --git a/compiler-rt/test/ctx_profile/lit.cfg.py b/compiler-rt/test/ctx_profile/lit.cfg.py
index 520552974f76..a56dabb8ebeb 100644
--- a/compiler-rt/test/ctx_profile/lit.cfg.py
+++ b/compiler-rt/test/ctx_profile/lit.cfg.py
@@ -29,7 +29,3 @@ config.name = "CtxProfile" + config.name_suffix
 config.test_source_root = os.path.dirname(__file__)
 # Default test suffixes.
 config.suffixes = [".c", ".cpp", ".test"]
-
-config.substitutions.append(
-    ("%clangxx ", " ".join([config.clang] + config.cxx_mode_flags) + " ")
-)
-- 
GitLab


From b672d9e3d3aa6f9796c09d691ca5c40dc816ab67 Mon Sep 17 00:00:00 2001
From: David Blaikie <dblaikie@gmail.com>
Date: Sat, 18 May 2024 01:07:03 +0000
Subject: [PATCH 1179/1206] DebugInfo: Don't emit .debug_names entries for
 split-dwarf-inlining info

---
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |  3 +-
 .../X86/debug-names-split-dwarf-inlining.ll   | 84 +++++++++++++++++++
 2 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/DebugInfo/X86/debug-names-split-dwarf-inlining.ll

diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index c7fa10775ada..b9c02aed848c 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -3564,7 +3564,8 @@ void DwarfDebug::addAccelNameImpl(
     const DwarfUnit &Unit,
     const DICompileUnit::DebugNameTableKind NameTableKind,
     AccelTable<DataT> &AppleAccel, StringRef Name, const DIE &Die) {
-  if (getAccelTableKind() == AccelTableKind::None || Name.empty())
+  if (getAccelTableKind() == AccelTableKind::None ||
+      Unit.getUnitDie().getTag() == dwarf::DW_TAG_skeleton_unit || Name.empty())
     return;
 
   if (getAccelTableKind() != AccelTableKind::Apple &&
diff --git a/llvm/test/DebugInfo/X86/debug-names-split-dwarf-inlining.ll b/llvm/test/DebugInfo/X86/debug-names-split-dwarf-inlining.ll
new file mode 100644
index 000000000000..83ef4a4267a1
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/debug-names-split-dwarf-inlining.ll
@@ -0,0 +1,84 @@
+; RUN: rm -rf %t && split-file %s %t && cd %t
+; RUN: llc -filetype=obj -o - a.ll -split-dwarf-file=foo.dwo \
+; RUN:     | llvm-dwarfdump - -debug-names -debug-info \
+; RUN:     | FileCheck --implicit-check-not=contents: %s
+
+; CHECK: .debug_info contents:
+; CHECK: .debug_info.dwo contents:
+; CHECK: DW_TAG_subprogram
+; CHECK: [[F3_DEF:0x[0-9a-f]*]]: DW_TAG_subprogram
+; CHECK-NOT: DW_TAG
+; CHECK: DW_AT_name ("f3")
+; CHECK-NOT: DW_TAG
+; CHECK: [[F2_INL:0x[0-9a-f]*]]:   DW_TAG_inlined_subroutine
+; CHECK-NOT: DW_TAG
+; CHECK:     DW_AT_abstract_origin ({{.*}} "_Z2f2v")
+; CHECK: .debug_names contents:
+; CHECK: String: {{.*}} "f2"
+; CHECK: Entry {{.*}}{
+; CHECK-NOT: {{^ *}}Entry
+; CHECK:   Tag: DW_TAG_inlined_subroutine
+; CHECK-NOT: {{^ *}}Entry
+; CHECK:   DW_IDX_die_offset: [[F2_INL]]
+; CHECK-NOT: {{^ *}}Entry
+; CHECK: }
+; CHECK-NEXT: }
+; CHECK: String: {{.*}} "f3"
+; CHECK: Entry {{.*}}{
+; CHECK-NOT: {{^ *}}Entry
+; CHECK:   Tag: DW_TAG_subprogram
+; CHECK-NOT: {{^ *}}Entry
+; CHECK:   DW_IDX_die_offset: [[F3_DEF]]
+; CHECK-NOT: {{^ *}}Entry
+; CHECK: }
+; CHECK-NEXT: }
+
+;--- a.cc
+void f1();
+inline void f2() {
+  f1();
+}
+void f3() {
+  f2();
+}
+;--- gen
+clang++ --target=x86_64-linux -S -emit-llvm -O3 -g -fsplit-dwarf-inlining -c \
+		 -gpubnames a.cc -o -
+
+;--- a.ll
+; ModuleID = 'a.cc'
+source_filename = "a.cc"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux"
+
+; Function Attrs: mustprogress uwtable
+define dso_local void @_Z2f3v() local_unnamed_addr #0 !dbg !8 {
+entry:
+  tail call void @_Z2f1v(), !dbg !11
+  ret void, !dbg !14
+}
+
+declare !dbg !15 void @_Z2f1v() local_unnamed_addr #1
+
+attributes #0 = { mustprogress uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
+attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "a.cc", directory: "/proc/self/cwd", checksumkind: CSK_MD5, checksum: "661bc2c3d7df6cc69861afc8fb17bf16")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 8, !"PIC Level", i32 2}
+!6 = !{i32 7, !"PIE Level", i32 2}
+!7 = !{i32 7, !"uwtable", i32 2}
+!8 = distinct !DISubprogram(name: "f3", linkageName: "_Z2f3v", scope: !1, file: !1, line: 5, type: !9, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !DILocation(line: 3, column: 3, scope: !12, inlinedAt: !13)
+!12 = distinct !DISubprogram(name: "f2", linkageName: "_Z2f2v", scope: !1, file: !1, line: 2, type: !9, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!13 = distinct !DILocation(line: 6, column: 3, scope: !8)
+!14 = !DILocation(line: 7, column: 1, scope: !8)
+!15 = !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 1, type: !9, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized)
-- 
GitLab


From 1e9324a8c734aaa933d2672522cc22d5022c6200 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Fri, 17 May 2024 18:43:09 -0700
Subject: [PATCH 1180/1206] [lldb] Namespace SBSourceLanguageName (NFC)

---
 lldb/include/lldb/API/SBExpressionOptions.h | 2 +-
 lldb/source/API/SBExpressionOptions.cpp     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/include/lldb/API/SBExpressionOptions.h b/lldb/include/lldb/API/SBExpressionOptions.h
index 19c416d0f3bc..a9e929a4c0bd 100644
--- a/lldb/include/lldb/API/SBExpressionOptions.h
+++ b/lldb/include/lldb/API/SBExpressionOptions.h
@@ -71,7 +71,7 @@ public:
   /// Set the language using a pair of language code and version as
   /// defined by the DWARF 6 specification.
   /// WARNING: These codes may change until DWARF 6 is finalized.
-  void SetLanguage(SBSourceLanguageName name, uint32_t version);
+  void SetLanguage(lldb::SBSourceLanguageName name, uint32_t version);
 
 #ifndef SWIG
   void SetCancelCallback(lldb::ExpressionCancelCallback callback, void *baton);
diff --git a/lldb/source/API/SBExpressionOptions.cpp b/lldb/source/API/SBExpressionOptions.cpp
index ce686112ff71..15ed403eaaea 100644
--- a/lldb/source/API/SBExpressionOptions.cpp
+++ b/lldb/source/API/SBExpressionOptions.cpp
@@ -156,7 +156,7 @@ void SBExpressionOptions::SetLanguage(lldb::LanguageType language) {
   m_opaque_up->SetLanguage(language);
 }
 
-void SBExpressionOptions::SetLanguage(SBSourceLanguageName name,
+void SBExpressionOptions::SetLanguage(lldb::SBSourceLanguageName name,
                                       uint32_t version) {
   LLDB_INSTRUMENT_VA(this, name, version);
 
-- 
GitLab


From faf39f45e3107d6bcb5f9cef8b5681c116b261d7 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 17 May 2024 19:58:10 -0700
Subject: [PATCH 1181/1206] Revert "[LoongArch] Use R_LARCH_ALIGN with section
 symbol (#84741)"

This reverts commit 01f79899ba349a0200586c8d05f5e22cca2ced31.

This unusual special case has been discussed on the binutils mailing
list. The approach will be revisited:
https://sourceware.org/pipermail/binutils/2024-May/134092.html

Pull Request: https://github.com/llvm/llvm-project/pull/92584
---
 lld/ELF/InputSection.cpp                      |  6 +---
 lld/test/ELF/loongarch-relax-align-ldr.s      | 28 -------------------
 lld/test/ELF/loongarch-relax-emit-relocs.s    |  5 ++--
 .../MCTargetDesc/LoongArchAsmBackend.cpp      |  7 +++--
 .../MC/LoongArch/Relocations/relax-addsub.s   |  2 +-
 .../MC/LoongArch/Relocations/relax-align.s    | 14 ++++------
 6 files changed, 15 insertions(+), 47 deletions(-)
 delete mode 100644 lld/test/ELF/loongarch-relax-align-ldr.s

diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 2a1ccd997f8b..e6c5996c0b39 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -471,11 +471,7 @@ void InputSection::copyRelocations(uint8_t *buf,
         addend += sec->getFile<ELFT>()->mipsGp0;
       }
 
-      if (config->emachine == EM_LOONGARCH && type == R_LARCH_ALIGN)
-        // LoongArch psABI v2.30, the R_LARCH_ALIGN requires symbol index.
-        // If it use the section symbol, the addend should not be changed.
-        p->r_addend = addend;
-      else if (RelTy::IsRela)
+      if (RelTy::IsRela)
         p->r_addend = sym.getVA(addend) - section->getOutputSection()->addr;
       // For SHF_ALLOC sections relocated by REL, append a relocation to
       // sec->relocations so that relocateAlloc transitively called by
diff --git a/lld/test/ELF/loongarch-relax-align-ldr.s b/lld/test/ELF/loongarch-relax-align-ldr.s
deleted file mode 100644
index 6534dc906cfd..000000000000
--- a/lld/test/ELF/loongarch-relax-align-ldr.s
+++ /dev/null
@@ -1,28 +0,0 @@
-# REQUIRES: loongarch
-## Test `ld -r` not changes the addend of R_LARCH_ALIGN.
-
-# RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax %s -o %t.64.o
-# RUN: ld.lld -r %t.64.o %t.64.o -o %t.64.r
-# RUN: llvm-objdump -dr --no-show-raw-insn %t.64.r | FileCheck %s
-
-# CHECK:      <.text>:
-# CHECK-NEXT:   break 1
-# CHECK-NEXT:   nop
-# CHECK-NEXT:   {{0*}}04:  R_LARCH_ALIGN        .text+0x804
-# CHECK-NEXT:   nop
-# CHECK-NEXT:   nop
-# CHECK-NEXT:   break 2
-# CHECK-NEXT:   break 0
-# CHECK-NEXT:   break 0
-# CHECK-NEXT:   break 0
-# CHECK-NEXT:   break 1
-# CHECK-NEXT:   nop
-# CHECK-NEXT:   {{0*}}24:  R_LARCH_ALIGN        .text+0x804
-# CHECK-NEXT:   nop
-# CHECK-NEXT:   nop
-# CHECK-NEXT:   break 2
-
-.text
-break 1
-.p2align 4, , 8
-break 2
diff --git a/lld/test/ELF/loongarch-relax-emit-relocs.s b/lld/test/ELF/loongarch-relax-emit-relocs.s
index 9007f8fcc114..581fce8c95ca 100644
--- a/lld/test/ELF/loongarch-relax-emit-relocs.s
+++ b/lld/test/ELF/loongarch-relax-emit-relocs.s
@@ -25,7 +25,7 @@
 # CHECK-NEXT:     R_LARCH_PCALA_LO12 _start
 # CHECK-NEXT:     R_LARCH_RELAX *ABS*
 # CHECK-NEXT:   nop
-# CHECK-NEXT:     R_LARCH_ALIGN .text+0x4
+# CHECK-NEXT:     R_LARCH_ALIGN .Lla-relax-align0+0x4
 # CHECK-NEXT:   nop
 # CHECK-NEXT:   ret
 
@@ -37,12 +37,11 @@
 # CHECKR-NEXT:     R_LARCH_PCALA_LO12 _start
 # CHECKR-NEXT:     R_LARCH_RELAX *ABS*
 # CHECKR-NEXT:   nop
-# CHECKR-NEXT:     R_LARCH_ALIGN .text+0x4
+# CHECKR-NEXT:     R_LARCH_ALIGN .Lla-relax-align0+0x4
 # CHECKR-NEXT:   nop
 # CHECKR-NEXT:   nop
 # CHECKR-NEXT:   ret
 
-.text
 .global _start
 _start:
   la.pcrel $a0, _start
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index 98f5014a34b1..de492f2b1f0a 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -226,8 +226,11 @@ bool LoongArchAsmBackend::shouldInsertFixupForCodeAlign(
       MCFixup::create(0, Dummy, MCFixupKind(LoongArch::fixup_loongarch_align));
   const MCSymbolRefExpr *MCSym = getSecToAlignSym()[Sec];
   if (MCSym == nullptr) {
-    // Use section symbol directly.
-    MCSym = MCSymbolRefExpr::create(Sec->getBeginSymbol(), Ctx);
+    // Create a symbol and make the value of symbol is zero.
+    MCSymbol *Sym = Ctx.createNamedTempSymbol("la-relax-align");
+    Sym->setFragment(&*Sec->getBeginSymbol()->getFragment());
+    Asm.registerSymbol(*Sym);
+    MCSym = MCSymbolRefExpr::create(Sym, Ctx);
     getSecToAlignSym()[Sec] = MCSym;
   }
 
diff --git a/llvm/test/MC/LoongArch/Relocations/relax-addsub.s b/llvm/test/MC/LoongArch/Relocations/relax-addsub.s
index 0e27d6301bb3..18e0ede5e293 100644
--- a/llvm/test/MC/LoongArch/Relocations/relax-addsub.s
+++ b/llvm/test/MC/LoongArch/Relocations/relax-addsub.s
@@ -28,7 +28,7 @@
 
 # RELAX:       Relocations [
 # RELAX-NEXT:    Section ({{.*}}) .rela.text {
-# RELAX-NEXT:      0x4 R_LARCH_ALIGN .text 0x4
+# RELAX-NEXT:      0x4 R_LARCH_ALIGN {{.*}} 0x4
 # RELAX-NEXT:      0x10 R_LARCH_PCALA_HI20 .L1 0x0
 # RELAX-NEXT:      0x10 R_LARCH_RELAX - 0x0
 # RELAX-NEXT:      0x14 R_LARCH_PCALA_LO12 .L1 0x0
diff --git a/llvm/test/MC/LoongArch/Relocations/relax-align.s b/llvm/test/MC/LoongArch/Relocations/relax-align.s
index 0246d5b46431..294fd9fb916c 100644
--- a/llvm/test/MC/LoongArch/Relocations/relax-align.s
+++ b/llvm/test/MC/LoongArch/Relocations/relax-align.s
@@ -63,19 +63,17 @@ ret
 ## Test the symbol index is different from .text.
 .section .text2, "ax"
 .p2align 4
-.p2align 4, , 4
 break 7
 
 # RELOC:            Relocations [
 # RELAX-RELOC-NEXT:   Section ({{.*}}) .rela.text {
-# RELAX-RELOC-NEXT:     0x24 R_LARCH_ALIGN .text 0x4
-# RELAX-RELOC-NEXT:     0x34 R_LARCH_ALIGN .text 0x5
-# RELAX-RELOC-NEXT:     0x50 R_LARCH_ALIGN .text 0x4
-# RELAX-RELOC-NEXT:     0x60 R_LARCH_ALIGN .text 0xB04
-# RELAX-RELOC-NEXT:     0x70 R_LARCH_ALIGN .text 0x4
+# RELAX-RELOC-NEXT:     0x24 R_LARCH_ALIGN .Lla-relax-align0 0x4
+# RELAX-RELOC-NEXT:     0x34 R_LARCH_ALIGN .Lla-relax-align0 0x5
+# RELAX-RELOC-NEXT:     0x50 R_LARCH_ALIGN .Lla-relax-align0 0x4
+# RELAX-RELOC-NEXT:     0x60 R_LARCH_ALIGN .Lla-relax-align0 0xB04
+# RELAX-RELOC-NEXT:     0x70 R_LARCH_ALIGN .Lla-relax-align0 0x4
 # RELAX-RELOC-NEXT:   }
 # RELAX-RELOC-NEXT:   Section ({{.*}}) .rela.text2 {
-# RELAX-RELOC-NEXT:     0x0 R_LARCH_ALIGN .text2 0x4
-# RELAX-RELOC-NEXT:     0xC R_LARCH_ALIGN .text2 0x404
+# RELAX-RELOC-NEXT:     0x0 R_LARCH_ALIGN .Lla-relax-align1 0x4
 # RELAX-RELOC-NEXT:   }
 # RELOC-NEXT:       ]
-- 
GitLab


From 702198fc9ac5dba392f9d9ba7c56467996343c0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 17 May 2024 20:56:10 -0700
Subject: [PATCH 1182/1206] [flang][cuda] Add data attribute to program globals
 (#92610)

---
 flang/lib/Lower/ConvertVariable.cpp           |  5 ++++-
 flang/test/Lower/CUDA/cuda-program-global.cuf | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Lower/CUDA/cuda-program-global.cuf

diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index b8868161fa05..075d0634fd1e 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -656,7 +656,10 @@ static void instantiateGlobal(Fortran::lower::AbstractConverter &converter,
     // Emit only a declaration if the global does not exist.
     global = declareGlobal(converter, var, globalName, linkage);
   } else {
-    global = defineGlobal(converter, var, globalName, linkage);
+    cuf::DataAttributeAttr dataAttr =
+        Fortran::lower::translateSymbolCUFDataAttribute(builder.getContext(),
+                                                        sym);
+    global = defineGlobal(converter, var, globalName, linkage, dataAttr);
   }
   auto addrOf = builder.create<fir::AddrOfOp>(loc, global.resultType(),
                                               global.getSymbol());
diff --git a/flang/test/Lower/CUDA/cuda-program-global.cuf b/flang/test/Lower/CUDA/cuda-program-global.cuf
new file mode 100644
index 000000000000..97b9927b3082
--- /dev/null
+++ b/flang/test/Lower/CUDA/cuda-program-global.cuf
@@ -0,0 +1,19 @@
+! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s
+
+! Test lowering of program local variable that are global
+
+program test
+  integer, device :: a(10)
+  integer :: b(10)
+  integer :: i
+  print*,i
+end
+
+! CHECK-LABEL: func.func @_QQmain()
+! CHECK: fir.address_of(@_QFEa) : !fir.ref<!fir.array<10xi32>>
+! CHECK: fir.address_of(@_QFEb) : !fir.ref<!fir.array<10xi32>>
+! CHECK: %[[ALLOCA:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
+! CHECK: hlfir.declare %[[ALLOCA]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+! CHECK: fir.global internal @_QFEa {data_attr = #cuf.cuda<device>} : !fir.array<10xi32> {{{$}}
+! CHECK: fir.global internal @_QFEb : !fir.array<10xi32> {{{$}}
-- 
GitLab


From cfe9deb1353021a1c1fe4731ec3e90f702dbd43d Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Fri, 17 May 2024 19:09:46 -0700
Subject: [PATCH 1183/1206] Reapply "[ctx_profile] Integration test (#92456)"

This reverts commit 881f20e958e80bd30463fc57d2d3e891bcb8a571.

Passing -ldl -lpthread explicitly
---
 .github/dependabot.yml                        |  26 ++---
 compiler-rt/lib/ctx_profile/CMakeLists.txt    |   9 ++
 .../TestCases/generate-context.cpp            | 103 ++++++++++++++++++
 compiler-rt/test/ctx_profile/lit.cfg.py       |   4 +
 4 files changed, 126 insertions(+), 16 deletions(-)
 create mode 100644 compiler-rt/test/ctx_profile/TestCases/generate-context.cpp

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 8c1dfd39b82c..f33a02cd16e4 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -1,18 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for more information:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://containers.dev/guide/dependabot
+
 version: 2
 updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "monthly"
-    groups:
-      github-actions:
-        patterns:
-          - "*"
-  - package-ecosystem: "pip"
-    directory: "/llvm/docs"
-    schedule:
-      interval: "monthly"
-    groups:
-      llvm-docs-requirements:
-        patterns:
-          - "*"
+ - package-ecosystem: "devcontainers"
+   directory: "/"
+   schedule:
+     interval: weekly
diff --git a/compiler-rt/lib/ctx_profile/CMakeLists.txt b/compiler-rt/lib/ctx_profile/CMakeLists.txt
index 1fa70594b28a..ab7bf3241fd6 100644
--- a/compiler-rt/lib/ctx_profile/CMakeLists.txt
+++ b/compiler-rt/lib/ctx_profile/CMakeLists.txt
@@ -18,3 +18,12 @@ append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ EXTRA_FLAGS)
 if(COMPILER_RT_INCLUDE_TESTS)
   add_subdirectory(tests)
 endif()
+
+add_compiler_rt_runtime(clang_rt.ctx_profile
+  STATIC
+  ARCHS ${CTX_PROFILE_SUPPORTED_ARCH}
+  OBJECT_LIBS RTSanitizerCommon RTSanitizerCommonLibc
+  CFLAGS ${EXTRA_FLAGS}
+  SOURCES ${CTX_PROFILE_SOURCES}
+  ADDITIONAL_HEADERS ${CTX_PROFILE_HEADERS}
+  PARENT_TARGET ctx_profile)
\ No newline at end of file
diff --git a/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp b/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp
new file mode 100644
index 000000000000..981d6170091c
--- /dev/null
+++ b/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp
@@ -0,0 +1,103 @@
+// Simple integration test for contextual instrumentation
+//
+// Copy the header defining ContextNode.
+// RUN: mkdir -p %t_include
+// RUN: cp %llvm_src/include/llvm/ProfileData/CtxInstrContextNode.h %t_include/
+//
+// Compile with ctx instrumentation "on". We treat "theRoot" as callgraph root.
+// RUN: %clangxx %s -lclang_rt.ctx_profile -I%t_include -O2 -o %t.bin -mllvm -profile-context-root=theRoot
+//
+// Run the binary, and observe the profile fetch handler's output.
+// RUN: %t.bin | FileCheck %s
+
+#include "CtxInstrContextNode.h"
+#include <cstdio>
+#include <iostream>
+
+using namespace llvm::ctx_profile;
+extern "C" bool __llvm_ctx_profile_fetch(void *Data,
+                                         bool (*Writer)(void *,
+                                                        const ContextNode &));
+
+// avoid name mangling
+extern "C" {
+__attribute__((noinline)) void someFunction(int I) {
+  if (I % 2)
+    printf("check odd\n");
+  else
+    printf("check even\n");
+}
+
+// block inlining because the pre-inliner otherwise will inline this - it's
+// too small.
+__attribute__((noinline)) void theRoot() {
+  printf("check 1\n");
+  someFunction(1);
+#pragma nounroll
+  for (auto I = 0; I < 2; ++I) {
+    someFunction(I);
+  }
+}
+}
+
+// Make sure the program actually ran correctly.
+// CHECK: check 1
+// CHECK-NEXT: check odd
+// CHECK-NEXT: check even
+// CHECK-NEXT: check odd
+
+void printProfile(const ContextNode &Node, const std::string &Indent,
+                  const std::string &Increment) {
+  std::cout << Indent << "Guid: " << Node.guid() << std::endl;
+  std::cout << Indent << "Entries: " << Node.entrycount() << std::endl;
+  std::cout << Indent << Node.counters_size() << " counters and "
+            << Node.callsites_size() << " callsites" << std::endl;
+  std::cout << Indent << "Counter values: ";
+  for (uint32_t I = 0U; I < Node.counters_size(); ++I)
+    std::cout << Node.counters()[I] << " ";
+  std::cout << std::endl;
+  for (uint32_t I = 0U; I < Node.callsites_size(); ++I)
+    for (const auto *N = Node.subContexts()[I]; N; N = N->next()) {
+      std::cout << Indent << "At Index " << I << ":" << std::endl;
+      printProfile(*N, Indent + Increment, Increment);
+    }
+}
+
+// 8657661246551306189 is theRoot. We expect 2 callsites and 2 counters - one
+// for the entry basic block and one for the loop.
+// 6759619411192316602 is someFunction. We expect all context instances to show
+// the same nr of counters and callsites, but the counters will be different.
+// The first context is for the first callsite with theRoot as parent, and the
+// second counter in someFunction will be 0 (we pass an odd nr, and the other
+// path gets instrumented).
+// The second context is in the loop. We expect 2 entries and each of the
+// branches would be taken once, so the second counter is 1.
+// CHECK-NEXT: Guid: 8657661246551306189
+// CHECK-NEXT: Entries: 1
+// CHECK-NEXT: 2 counters and 3 callsites
+// CHECK-NEXT: Counter values: 1 2
+// CHECK-NEXT: At Index 1:
+// CHECK-NEXT:   Guid: 6759619411192316602
+// CHECK-NEXT:   Entries: 1
+// CHECK-NEXT:   2 counters and 2 callsites
+// CHECK-NEXT:   Counter values: 1 0
+// CHECK-NEXT: At Index 2:
+// CHECK-NEXT:   Guid: 6759619411192316602
+// CHECK-NEXT:   Entries: 2
+// CHECK-NEXT:   2 counters and 2 callsites
+// CHECK-NEXT:   Counter values: 2 1
+
+bool profileWriter() {
+  return __llvm_ctx_profile_fetch(
+      nullptr, +[](void *, const ContextNode &Node) {
+        printProfile(Node, "", "  ");
+        return true;
+      });
+}
+
+int main(int argc, char **argv) {
+  theRoot();
+  // This would be implemented in a specific RPC handler, but here we just call
+  // it directly.
+  return !profileWriter();
+}
diff --git a/compiler-rt/test/ctx_profile/lit.cfg.py b/compiler-rt/test/ctx_profile/lit.cfg.py
index a56dabb8ebeb..bf62093601f1 100644
--- a/compiler-rt/test/ctx_profile/lit.cfg.py
+++ b/compiler-rt/test/ctx_profile/lit.cfg.py
@@ -29,3 +29,7 @@ config.name = "CtxProfile" + config.name_suffix
 config.test_source_root = os.path.dirname(__file__)
 # Default test suffixes.
 config.suffixes = [".c", ".cpp", ".test"]
+
+config.substitutions.append(
+    ("%clangxx ", " ".join([config.clang] + config.cxx_mode_flags) + " -ldl -lpthread ")
+)
-- 
GitLab


From 9e98815ef04f055e84d0b1a5cfb877bb1c9cb22a Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman@google.com>
Date: Sat, 18 May 2024 05:04:59 +0000
Subject: [PATCH 1184/1206] [Github] Revert accidental changes to dependabot
 config

f3524e9aebbfabed0c60d0087b39ce14d8f778da accidentally touched the
dependabot config. This patch reverts that change.
---
 .github/dependabot.yml | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index f33a02cd16e4..8c1dfd39b82c 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -1,12 +1,18 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for more information:
-# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
-# https://containers.dev/guide/dependabot
-
 version: 2
 updates:
- - package-ecosystem: "devcontainers"
-   directory: "/"
-   schedule:
-     interval: weekly
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    groups:
+      github-actions:
+        patterns:
+          - "*"
+  - package-ecosystem: "pip"
+    directory: "/llvm/docs"
+    schedule:
+      interval: "monthly"
+    groups:
+      llvm-docs-requirements:
+        patterns:
+          - "*"
-- 
GitLab


From 219476d20fcdb21644944b0c204b4ac6d6ef3760 Mon Sep 17 00:00:00 2001
From: jiajie zhang <56027356+JumpMasterJJ@users.noreply.github.com>
Date: Sat, 18 May 2024 13:33:00 +0800
Subject: [PATCH 1185/1206] Fix: remove wrongly pushed etime-function.mlir at
 toplevel (#92634)

The purpose of this PR is to remove the 'etime-function.mlir' file that
I mistakenly committed in
https://github.com/llvm/llvm-project/pull/92571. This file is not
necessary in source code control, and its presence may cause confusion
or misunderstanding.
---
 etime-function.mlir | 25 -------------------------
 1 file changed, 25 deletions(-)
 delete mode 100644 etime-function.mlir

diff --git a/etime-function.mlir b/etime-function.mlir
deleted file mode 100644
index 740dfd4866aa..000000000000
--- a/etime-function.mlir
+++ /dev/null
@@ -1,25 +0,0 @@
-module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
-  func.func @_QPetime_test(%arg0: !fir.ref<!fir.array<2xf32>> {fir.bindc_name = "values"}, %arg1: !fir.ref<f32> {fir.bindc_name = "time"}) {
-    %c9_i32 = arith.constant 9 : i32
-    %c2 = arith.constant 2 : index
-    %0 = fir.alloca f32
-    %1 = fir.declare %arg1 {uniq_name = "_QFetime_testEtime"} : (!fir.ref<f32>) -> !fir.ref<f32>
-    %2 = fir.shape %c2 : (index) -> !fir.shape<1>
-    %3 = fir.declare %arg0(%2) {uniq_name = "_QFetime_testEvalues"} : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<2xf32>>
-    %4 = fir.embox %3(%2) : (!fir.ref<!fir.array<2xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<2xf32>>
-    %5 = fir.embox %0 : (!fir.ref<f32>) -> !fir.box<f32>
-    %6 = fir.address_of(@_QQclX116781708dcf8f012d7ec1e40d743d97) : !fir.ref<!fir.char<1,71>>
-    %7 = fir.convert %4 : (!fir.box<!fir.array<2xf32>>) -> !fir.box<none>
-    %8 = fir.convert %5 : (!fir.box<f32>) -> !fir.box<none>
-    %9 = fir.convert %6 : (!fir.ref<!fir.char<1,71>>) -> !fir.ref<i8>
-    %10 = fir.call @_FortranAEtime(%7, %8, %9, %c9_i32) fastmath<contract> : (!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none
-    %11 = fir.load %0 : !fir.ref<f32>
-    fir.store %11 to %1 : !fir.ref<f32>
-    return
-  }
-  func.func private @_FortranAEtime(!fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> none attributes {fir.runtime}
-  fir.global linkonce @_QQclX116781708dcf8f012d7ec1e40d743d97 constant : !fir.char<1,71> {
-    %0 = fir.string_lit "/home/jump/llvm-project/flang/test/Lower/Intrinsics/etime-function.f90\00"(71) : !fir.char<1,71>
-    fir.has_value %0 : !fir.char<1,71>
-  }
-}
-- 
GitLab


From 195ba4572129befa25ca56439515d7cb91587c56 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 17 May 2024 23:02:54 -0700
Subject: [PATCH 1186/1206] [MCAsmParser] .macro/.rept/.irp/.irpc: remove
 excess \n after expansion

```
.irp foo,1
nop
.endr
nop
```

expands to an excess EOL between two nop lines. Other loop directives
and .macro have the same issue.

`Lex()` at "Jump to the macro instantiation and prime the lexer"
requires that there is one single \n token in CurTok. Therefore, we
cannot consume the trailing \n when parsing the macro(-like) body.
(commit c6e787f771d1f9d6a846b2d9b8db6adcd87e8dba (reverted by
1e5f29af81a5f6fda308074f6345b9fba4faa71c))

Instead, skip the potential \n after jumpToLoc at handleMacroExit.
---
 llvm/lib/MC/MCParser/AsmParser.cpp  | 35 ++++++++++++++---------------
 llvm/test/MC/AsmParser/macro-arg.s  |  6 ++---
 llvm/test/MC/AsmParser/macro-irp.s  |  2 ++
 llvm/test/MC/AsmParser/macro-rept.s | 25 +++++++++++----------
 4 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index ce155be9c1c7..2cddaf330b3b 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -2920,6 +2920,10 @@ void AsmParser::handleMacroExit() {
   // Jump to the EndOfStatement we should return to, and consume it.
   jumpToLoc(ActiveMacros.back()->ExitLoc, ActiveMacros.back()->ExitBuffer);
   Lex();
+  // If .endm/.endr is followed by \n instead of a comment, consume it so that
+  // we don't print an excess \n.
+  if (getTok().is(AsmToken::EndOfStatement))
+    Lex();
 
   // Pop the instantiation entry.
   delete ActiveMacros.back();
@@ -5624,27 +5628,22 @@ MCAsmMacro *AsmParser::parseMacroLikeBody(SMLoc DirectiveLoc) {
       return nullptr;
     }
 
-    if (Lexer.is(AsmToken::Identifier) &&
-        (getTok().getIdentifier() == ".rep" ||
-         getTok().getIdentifier() == ".rept" ||
-         getTok().getIdentifier() == ".irp" ||
-         getTok().getIdentifier() == ".irpc")) {
-      ++NestLevel;
-    }
-
-    // Otherwise, check whether we have reached the .endr.
-    if (Lexer.is(AsmToken::Identifier) && getTok().getIdentifier() == ".endr") {
-      if (NestLevel == 0) {
-        EndToken = getTok();
-        Lex();
-        if (Lexer.isNot(AsmToken::EndOfStatement)) {
-          printError(getTok().getLoc(),
-                     "unexpected token in '.endr' directive");
+    if (Lexer.is(AsmToken::Identifier)) {
+      StringRef Ident = getTok().getIdentifier();
+      if (Ident == ".rep" || Ident == ".rept" || Ident == ".irp" ||
+          Ident == ".irpc") {
+        ++NestLevel;
+      } else if (Ident == ".endr") {
+        if (NestLevel == 0) {
+          EndToken = getTok();
+          Lex();
+          if (Lexer.is(AsmToken::EndOfStatement))
+            break;
+          printError(getTok().getLoc(), "expected newline");
           return nullptr;
         }
-        break;
+        --NestLevel;
       }
-      --NestLevel;
     }
 
     // Otherwise, scan till the end of the statement.
diff --git a/llvm/test/MC/AsmParser/macro-arg.s b/llvm/test/MC/AsmParser/macro-arg.s
index a269873e9fa1..19065402309e 100644
--- a/llvm/test/MC/AsmParser/macro-arg.s
+++ b/llvm/test/MC/AsmParser/macro-arg.s
@@ -17,13 +17,11 @@ escape 3
 
 # CHECK:      .long -1
 # CHECK-NEXT: .long -1
-# CHECK-EMPTY:
 double
-# CHECK:      .long -1
 # CHECK-NEXT: .long -1
-# CHECK-EMPTY:
+# CHECK-NEXT: .long -1
 double ,
-# CHECK:      .long 1
+# CHECK-NEXT: .long 1
 # CHECK-NEXT: .long -1
 double 1
 # CHECK:      .long 2
diff --git a/llvm/test/MC/AsmParser/macro-irp.s b/llvm/test/MC/AsmParser/macro-irp.s
index 42902bc94b5f..7d4c7cc26942 100644
--- a/llvm/test/MC/AsmParser/macro-irp.s
+++ b/llvm/test/MC/AsmParser/macro-irp.s
@@ -3,9 +3,11 @@
 .irp reg,%eax,%ebx
         pushl \reg
 .endr
+pushl %ecx
 
 // CHECK: pushl %eax
 // CHECK: pushl %ebx
+// CHECK-NEXT: pushl %ecx
 
 .irp reg,%eax,%ebx
 .irp imm,4,3,5
diff --git a/llvm/test/MC/AsmParser/macro-rept.s b/llvm/test/MC/AsmParser/macro-rept.s
index 13599dd81841..c1a90b22c9f2 100644
--- a/llvm/test/MC/AsmParser/macro-rept.s
+++ b/llvm/test/MC/AsmParser/macro-rept.s
@@ -1,24 +1,25 @@
-// RUN: llvm-mc -triple x86_64-unknown-unknown %s | FileCheck %s
+# RUN: llvm-mc -triple x86_64 %s | FileCheck %s
 
 .rept 2
     .long 1
 .endr
 # 3 "a.s"
-/// Test line marker after .endr \n.
+## Test line marker after .endr \n.
 
 .rept 3
 .rept 2
     .long 0
 .endr
-.endr
-
-// CHECK: .long	1
-// CHECK: .long	1
+.endr # comment after .endr
+.long 42
 
-// CHECK: .long	0
-// CHECK: .long	0
-// CHECK: .long	0
+# CHECK:      .long 1
+# CHECK-NEXT: .long 1
 
-// CHECK: .long	0
-// CHECK: .long	0
-// CHECK: .long	0
+# CHECK:      .long 0
+# CHECK-NEXT: .long 0
+# CHECK-NEXT: .long 0
+# CHECK-NEXT: .long 0
+# CHECK-NEXT: .long 0
+# CHECK-NEXT: .long 0
+# CHECK-NEXT: .long 42
-- 
GitLab


From bfeebda3b1cc1a05e435e94f54bf2d2a2570b4e2 Mon Sep 17 00:00:00 2001
From: Michael Klemm <michael.klemm@amd.com>
Date: Sat, 18 May 2024 08:25:43 +0200
Subject: [PATCH 1187/1206] [flang][OpenMP] Re-enable tests when building
 OpenMP as a runtime (#89046)

---
 flang/test/Driver/include-omp-header.f90                 | 4 ++--
 flang/test/Lower/OpenMP/atomic-capture.f90               | 5 ++---
 flang/test/Lower/OpenMP/atomic-read.f90                  | 2 +-
 flang/test/Lower/OpenMP/atomic-update.f90                | 4 ++--
 flang/test/Lower/OpenMP/atomic-write.f90                 | 2 +-
 flang/test/Lower/OpenMP/critical.f90                     | 2 +-
 flang/test/Lower/OpenMP/omp-lib-num-threads.f90          | 8 ++++----
 flang/test/Lower/OpenMP/parallel-sections.f90            | 2 +-
 flang/test/Lower/OpenMP/parallel.f90                     | 2 +-
 flang/test/Lower/OpenMP/sections.f90                     | 4 ++--
 flang/test/Lower/OpenMP/single.f90                       | 4 ++--
 flang/test/Lower/OpenMP/task.f90                         | 2 +-
 flang/test/Lower/OpenMP/taskgroup.f90                    | 4 +++-
 flang/test/Lower/OpenMP/teams.f90                        | 2 +-
 flang/test/Lower/OpenMP/threadprivate-default-clause.f90 | 2 +-
 flang/test/Parser/OpenMP/allocate-tree-spec-part.f90     | 2 +-
 flang/test/Parser/OpenMP/allocate-tree.f90               | 4 ++--
 flang/test/Parser/OpenMP/target_device_parse.f90         | 4 ++--
 flang/test/Semantics/OpenMP/allocate-clause01.f90        | 2 +-
 flang/test/Semantics/OpenMP/allocate-directive.f90       | 2 +-
 flang/test/Semantics/OpenMP/allocate01.f90               | 2 +-
 flang/test/Semantics/OpenMP/allocate02.f90               | 2 +-
 flang/test/Semantics/OpenMP/allocate03.f90               | 2 +-
 flang/test/Semantics/OpenMP/allocate04.f90               | 2 +-
 flang/test/Semantics/OpenMP/allocate05.f90               | 2 +-
 flang/test/Semantics/OpenMP/allocate06.f90               | 2 +-
 flang/test/Semantics/OpenMP/allocate07.f90               | 2 +-
 flang/test/Semantics/OpenMP/allocate08.f90               | 2 +-
 flang/test/Semantics/OpenMP/allocate09.f90               | 2 +-
 flang/test/Semantics/OpenMP/allocators01.f90             | 2 +-
 flang/test/Semantics/OpenMP/allocators02.f90             | 2 +-
 flang/test/Semantics/OpenMP/allocators03.f90             | 2 +-
 flang/test/Semantics/OpenMP/allocators04.f90             | 2 +-
 flang/test/Semantics/OpenMP/allocators05.f90             | 2 +-
 flang/test/Semantics/OpenMP/allocators06.f90             | 2 +-
 flang/test/Semantics/OpenMP/atomic-hint-clause.f90       | 2 +-
 flang/test/Semantics/OpenMP/atomic01.f90                 | 2 +-
 flang/test/Semantics/OpenMP/atomic02.f90                 | 2 +-
 flang/test/Semantics/OpenMP/atomic03.f90                 | 2 +-
 flang/test/Semantics/OpenMP/atomic04.f90                 | 2 +-
 flang/test/Semantics/OpenMP/atomic05.f90                 | 2 +-
 flang/test/Semantics/OpenMP/clause-validity01.f90        | 2 +-
 flang/test/Semantics/OpenMP/critical-hint-clause.f90     | 2 +-
 flang/test/Semantics/OpenMP/declare-target01.f90         | 2 +-
 flang/test/Semantics/OpenMP/flush02.f90                  | 2 +-
 .../test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90 | 2 +-
 flang/test/Semantics/OpenMP/parallel-sections01.f90      | 2 +-
 flang/test/Semantics/OpenMP/resolve06.f90                | 2 +-
 flang/test/Semantics/OpenMP/sections02.f90               | 2 +-
 flang/test/Semantics/OpenMP/sync-critical02.f90          | 2 +-
 flang/test/Semantics/OpenMP/taskgroup01.f90              | 2 +-
 flang/test/Semantics/OpenMP/threadprivate01.f90          | 2 +-
 flang/test/lit.cfg.py                                    | 9 +++++++++
 flang/test/lit.site.cfg.py.in                            | 6 +++++-
 54 files changed, 78 insertions(+), 64 deletions(-)

diff --git a/flang/test/Driver/include-omp-header.f90 b/flang/test/Driver/include-omp-header.f90
index 6ddc6f3da2a4..7e54910a4b58 100644
--- a/flang/test/Driver/include-omp-header.f90
+++ b/flang/test/Driver/include-omp-header.f90
@@ -9,11 +9,11 @@
 
 ! This should just work
 ! RUN: not rm omp_lib.h
-! RUN: %flang -cpp -fsyntax-only -fopenmp %s  2>&1
+! RUN: %flang -cpp -fsyntax-only %openmp_flags %s  2>&1
 
 ! Create an empty omp_lib.h header that _does not_ define omp_default_mem_alloc - this should lead to semantic errors
 ! RUN: touch omp_lib.h
-! RUN: not %flang -cpp -fsyntax-only -fopenmp %s  2>&1 | FileCheck %s
+! RUN: not %flang -cpp -fsyntax-only %openmp_flags %s  2>&1 | FileCheck %s
 ! RUN: rm omp_lib.h
 
 ! CHECK: error: Must have INTEGER type, but is REAL(4)
diff --git a/flang/test/Lower/OpenMP/atomic-capture.f90 b/flang/test/Lower/OpenMP/atomic-capture.f90
index 2587c24cedf3..32d8cd7bbf32 100644
--- a/flang/test/Lower/OpenMP/atomic-capture.f90
+++ b/flang/test/Lower/OpenMP/atomic-capture.f90
@@ -2,9 +2,8 @@
 
 ! This test checks the lowering of atomic capture
 
-! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
-! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s 
-
+! RUN: bbc %openmp_flags -emit-hlfir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
 
 program OmpAtomicCapture
diff --git a/flang/test/Lower/OpenMP/atomic-read.f90 b/flang/test/Lower/OpenMP/atomic-read.f90
index 366da21f53f8..8c3f37c94975 100644
--- a/flang/test/Lower/OpenMP/atomic-read.f90
+++ b/flang/test/Lower/OpenMP/atomic-read.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
+! RUN: bbc %openmp_flags -emit-hlfir %s -o - | FileCheck %s
 
 ! This test checks the lowering of atomic read
 
diff --git a/flang/test/Lower/OpenMP/atomic-update.f90 b/flang/test/Lower/OpenMP/atomic-update.f90
index 051779e40816..74289ce046c1 100644
--- a/flang/test/Lower/OpenMP/atomic-update.f90
+++ b/flang/test/Lower/OpenMP/atomic-update.f90
@@ -1,8 +1,8 @@
 ! REQUIRES: openmp_runtime
 
 ! This test checks lowering of atomic and atomic update constructs
-! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
-! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+! RUN: bbc %openmp_flags -emit-hlfir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
 program OmpAtomicUpdate
     use omp_lib
diff --git a/flang/test/Lower/OpenMP/atomic-write.f90 b/flang/test/Lower/OpenMP/atomic-write.f90
index e0fa802976d9..8867dc592119 100644
--- a/flang/test/Lower/OpenMP/atomic-write.f90
+++ b/flang/test/Lower/OpenMP/atomic-write.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
+! RUN: bbc %openmp_flags -emit-hlfir %s -o - | FileCheck %s
 
 ! This test checks the lowering of atomic write
 
diff --git a/flang/test/Lower/OpenMP/critical.f90 b/flang/test/Lower/OpenMP/critical.f90
index 38e6d0eac1cb..98a90c959484 100644
--- a/flang/test/Lower/OpenMP/critical.f90
+++ b/flang/test/Lower/OpenMP/critical.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
 !CHECK: omp.critical.declare @help2
 !CHECK: omp.critical.declare @help1 hint(contended)
diff --git a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90 b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
index d2b558a886eb..6e76dce47305 100644
--- a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
+++ b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
@@ -1,9 +1,9 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - 2>&1 | FileCheck %s
-! RUN: bbc -fopenmp -emit-hlfir -o - %s 2>&1 | FileCheck %s
-! RUN: %flang_fc1 -emit-fir -fopenmp %s -o - 2>&1 | FileCheck %s
-! RUN: bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc %openmp_flags -emit-hlfir -o - %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc -emit-fir %openmp_flags -o - %s 2>&1 | FileCheck %s
 !
 ! Test that the calls to omp_lib's omp_get_num_threads and omp_set_num_threads
 ! get lowered even though their implementation is not in the omp_lib module
diff --git a/flang/test/Lower/OpenMP/parallel-sections.f90 b/flang/test/Lower/OpenMP/parallel-sections.f90
index 285102e06cad..594725779c13 100644
--- a/flang/test/Lower/OpenMP/parallel-sections.f90
+++ b/flang/test/Lower/OpenMP/parallel-sections.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
 !===============================================================================
 ! Parallel sections construct
diff --git a/flang/test/Lower/OpenMP/parallel.f90 b/flang/test/Lower/OpenMP/parallel.f90
index 6c062b706ed4..6b4307b4d435 100644
--- a/flang/test/Lower/OpenMP/parallel.f90
+++ b/flang/test/Lower/OpenMP/parallel.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
 !CHECK-LABEL: func @_QPparallel_simple
 subroutine parallel_simple()
diff --git a/flang/test/Lower/OpenMP/sections.f90 b/flang/test/Lower/OpenMP/sections.f90
index bd76bd53e5a0..c79d6b020f3f 100644
--- a/flang/test/Lower/OpenMP/sections.f90
+++ b/flang/test/Lower/OpenMP/sections.f90
@@ -2,8 +2,8 @@
 
 ! This test checks the lowering of OpenMP sections construct with several clauses present
 
-! RUN: %flang_fc1 -flang-experimental-hlfir -emit-hlfir -fopenmp %s -o - | FileCheck %s
-! RUN: bbc -hlfir -emit-hlfir -fopenmp %s -o - | FileCheck %s
+! RUN: %flang_fc1 -flang-experimental-hlfir -emit-hlfir %openmp_flags %s -o - | FileCheck %s
+! RUN: bbc -hlfir -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
 !CHECK: func @_QQmain() attributes {fir.bindc_name = "sample"} {
 !CHECK:   %[[COUNT:.*]] = fir.address_of(@_QFEcount) : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/single.f90 b/flang/test/Lower/OpenMP/single.f90
index 91f8a592909a..2e5d05dd7b47 100644
--- a/flang/test/Lower/OpenMP/single.f90
+++ b/flang/test/Lower/OpenMP/single.f90
@@ -1,7 +1,7 @@
 ! REQUIRES: openmp_runtime
 
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
-!RUN: bbc -emit-hlfir -fopenmp %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
+!RUN: bbc -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
 !===============================================================================
 ! Single construct
diff --git a/flang/test/Lower/OpenMP/task.f90 b/flang/test/Lower/OpenMP/task.f90
index f3718b8829cc..71ff57cd8fa5 100644
--- a/flang/test/Lower/OpenMP/task.f90
+++ b/flang/test/Lower/OpenMP/task.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
 !CHECK-LABEL: func @_QPomp_task_simple() {
 subroutine omp_task_simple
diff --git a/flang/test/Lower/OpenMP/taskgroup.f90 b/flang/test/Lower/OpenMP/taskgroup.f90
index d9d262bdd2c0..3828627be4f5 100644
--- a/flang/test/Lower/OpenMP/taskgroup.f90
+++ b/flang/test/Lower/OpenMP/taskgroup.f90
@@ -1,4 +1,6 @@
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+! REQUIRES: openmp_runtime
+
+!RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
 ! The "allocate" clause has been removed, because it needs to be used
 ! together with a privatizing clause. The only such clause for "taskgroup"
diff --git a/flang/test/Lower/OpenMP/teams.f90 b/flang/test/Lower/OpenMP/teams.f90
index b1b2e7080676..47d379d6c284 100644
--- a/flang/test/Lower/OpenMP/teams.f90
+++ b/flang/test/Lower/OpenMP/teams.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags -fopenmp-version=52 %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPteams_simple
 subroutine teams_simple()
diff --git a/flang/test/Lower/OpenMP/threadprivate-default-clause.f90 b/flang/test/Lower/OpenMP/threadprivate-default-clause.f90
index 10beaf6fefde..e4f922ad0506 100644
--- a/flang/test/Lower/OpenMP/threadprivate-default-clause.f90
+++ b/flang/test/Lower/OpenMP/threadprivate-default-clause.f90
@@ -2,7 +2,7 @@
 
 ! Simple test for lowering of OpenMP Threadprivate Directive with HLFIR.
 
-!RUN: %flang_fc1  -emit-hlfir -fopenmp %s -o - | FileCheck %s
+!RUN: %flang_fc1  -emit-hlfir %openmp_flags %s -o - | FileCheck %s
 
 !CHECK-LABEL: func.func @_QPsub1() {
 !CHECK:     %[[A:.*]] = fir.address_of(@_QFsub1Ea) : !fir.ref<i32>
diff --git a/flang/test/Parser/OpenMP/allocate-tree-spec-part.f90 b/flang/test/Parser/OpenMP/allocate-tree-spec-part.f90
index ab56b98b5783..afcaf44b09f0 100644
--- a/flang/test/Parser/OpenMP/allocate-tree-spec-part.f90
+++ b/flang/test/Parser/OpenMP/allocate-tree-spec-part.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %flang_fc1 -fopenmp -fdebug-dump-parse-tree %s | FileCheck %s
+! RUN: %flang_fc1 %openmp_flags -fdebug-dump-parse-tree %s | FileCheck %s
 ! Ensures associated declarative OMP allocations in the specification
 ! part are kept there
 
diff --git a/flang/test/Parser/OpenMP/allocate-tree.f90 b/flang/test/Parser/OpenMP/allocate-tree.f90
index 02393a536cf4..9de257b00dc3 100644
--- a/flang/test/Parser/OpenMP/allocate-tree.f90
+++ b/flang/test/Parser/OpenMP/allocate-tree.f90
@@ -1,7 +1,7 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %flang_fc1 -fopenmp -fdebug-dump-parse-tree %s | FileCheck %s
-! RUN: %flang_fc1 -fopenmp -fdebug-unparse %s | FileCheck %s --check-prefix="UNPARSE"
+! RUN: %flang_fc1 %openmp_flags -fdebug-dump-parse-tree %s | FileCheck %s
+! RUN: %flang_fc1 %openmp_flags -fdebug-unparse %s | FileCheck %s --check-prefix="UNPARSE"
 ! Ensures associated declarative OMP allocations are nested in their
 ! corresponding executable allocate directive
 
diff --git a/flang/test/Parser/OpenMP/target_device_parse.f90 b/flang/test/Parser/OpenMP/target_device_parse.f90
index dace34b6ec32..30464c5b08ab 100644
--- a/flang/test/Parser/OpenMP/target_device_parse.f90
+++ b/flang/test/Parser/OpenMP/target_device_parse.f90
@@ -1,7 +1,7 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %flang_fc1 -fdebug-unparse-no-sema -fopenmp %s | FileCheck --ignore-case %s
-! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+! RUN: %flang_fc1 -fdebug-unparse-no-sema %openmp_flags %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree %openmp_flags %s | FileCheck --check-prefix="PARSE-TREE" %s
 ! Checks the parsing of Openmp 5.0 Target Device constructs
 !
 PROGRAM main
diff --git a/flang/test/Semantics/OpenMP/allocate-clause01.f90 b/flang/test/Semantics/OpenMP/allocate-clause01.f90
index e059a4a61a90..2b9a72e928eb 100644
--- a/flang/test/Semantics/OpenMP/allocate-clause01.f90
+++ b/flang/test/Semantics/OpenMP/allocate-clause01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.2
 ! The allocate clause's allocator modifier must be of type allocator_handle
 ! and the align modifier must be constant, positive integer expression
diff --git a/flang/test/Semantics/OpenMP/allocate-directive.f90 b/flang/test/Semantics/OpenMP/allocate-directive.f90
index 1db13df41938..18a14b825f00 100644
--- a/flang/test/Semantics/OpenMP/allocate-directive.f90
+++ b/flang/test/Semantics/OpenMP/allocate-directive.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! Check OpenMP Allocate directive
 use omp_lib
 
diff --git a/flang/test/Semantics/OpenMP/allocate01.f90 b/flang/test/Semantics/OpenMP/allocate01.f90
index a91076c80b1f..6ccb8bb09e83 100644
--- a/flang/test/Semantics/OpenMP/allocate01.f90
+++ b/flang/test/Semantics/OpenMP/allocate01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.0
 ! 2.11.3 allocate Directive
 ! The allocate directive must appear in the same scope as the declarations of
diff --git a/flang/test/Semantics/OpenMP/allocate02.f90 b/flang/test/Semantics/OpenMP/allocate02.f90
index dc4194aca7f1..8f0579e810bb 100644
--- a/flang/test/Semantics/OpenMP/allocate02.f90
+++ b/flang/test/Semantics/OpenMP/allocate02.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.0
 ! 2.11.3 allocate Directive
 ! At most one allocator clause can appear on the allocate directive.
diff --git a/flang/test/Semantics/OpenMP/allocate03.f90 b/flang/test/Semantics/OpenMP/allocate03.f90
index 30a015f26c31..e35115f3897c 100644
--- a/flang/test/Semantics/OpenMP/allocate03.f90
+++ b/flang/test/Semantics/OpenMP/allocate03.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.0
 ! 2.11.3 allocate Directive
 ! A variable that is part of another variable (as an array or
diff --git a/flang/test/Semantics/OpenMP/allocate04.f90 b/flang/test/Semantics/OpenMP/allocate04.f90
index b8a870941161..ea89d9446cc1 100644
--- a/flang/test/Semantics/OpenMP/allocate04.f90
+++ b/flang/test/Semantics/OpenMP/allocate04.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.0
 ! 2.11.3 allocate Directive
 ! Only the allocator clause is allowed on the allocate directive
diff --git a/flang/test/Semantics/OpenMP/allocate05.f90 b/flang/test/Semantics/OpenMP/allocate05.f90
index 92eeaf4222ac..a787e8bb32a4 100644
--- a/flang/test/Semantics/OpenMP/allocate05.f90
+++ b/flang/test/Semantics/OpenMP/allocate05.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.0
 ! 2.11.3 allocate Directive
 ! allocate directives that appear in a target region must specify an allocator
diff --git a/flang/test/Semantics/OpenMP/allocate06.f90 b/flang/test/Semantics/OpenMP/allocate06.f90
index c4474560bea3..e14134cd0730 100644
--- a/flang/test/Semantics/OpenMP/allocate06.f90
+++ b/flang/test/Semantics/OpenMP/allocate06.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.0
 ! 2.11.3 allocate Directive 
 ! List items specified in the allocate directive must not have the ALLOCATABLE attribute unless the directive is associated with an
diff --git a/flang/test/Semantics/OpenMP/allocate07.f90 b/flang/test/Semantics/OpenMP/allocate07.f90
index 3211c0bf28b9..396df598b252 100644
--- a/flang/test/Semantics/OpenMP/allocate07.f90
+++ b/flang/test/Semantics/OpenMP/allocate07.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.0
 ! 2.11.3 allocate Directive 
 ! A type parameter inquiry cannot appear in an allocate directive.
diff --git a/flang/test/Semantics/OpenMP/allocate08.f90 b/flang/test/Semantics/OpenMP/allocate08.f90
index 152ad07544ea..fc950ea4fca3 100644
--- a/flang/test/Semantics/OpenMP/allocate08.f90
+++ b/flang/test/Semantics/OpenMP/allocate08.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.0
 ! 2.11.3 allocate Directive
 ! If list items within the ALLOCATE directive have the SAVE attribute, are a common block name, or are declared in the scope of a
diff --git a/flang/test/Semantics/OpenMP/allocate09.f90 b/flang/test/Semantics/OpenMP/allocate09.f90
index 639d23cd6d30..0f93a340fe1e 100644
--- a/flang/test/Semantics/OpenMP/allocate09.f90
+++ b/flang/test/Semantics/OpenMP/allocate09.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.0
 ! 2.11.3 allocate Directive
 ! List items specified in an allocate directive that is associated
diff --git a/flang/test/Semantics/OpenMP/allocators01.f90 b/flang/test/Semantics/OpenMP/allocators01.f90
index 02d686b49716..c75c522ecae1 100644
--- a/flang/test/Semantics/OpenMP/allocators01.f90
+++ b/flang/test/Semantics/OpenMP/allocators01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.2
 ! 6.7 allocators construct
 ! A list item that appears in an allocate clause must appear as
diff --git a/flang/test/Semantics/OpenMP/allocators02.f90 b/flang/test/Semantics/OpenMP/allocators02.f90
index 9d8644bc30c6..8055d21c6809 100644
--- a/flang/test/Semantics/OpenMP/allocators02.f90
+++ b/flang/test/Semantics/OpenMP/allocators02.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.2
 ! 6.7 allocators construct
 ! A variable that is part of another variable (as an array or
diff --git a/flang/test/Semantics/OpenMP/allocators03.f90 b/flang/test/Semantics/OpenMP/allocators03.f90
index 85ab91fde5e0..03cff1b1e991 100644
--- a/flang/test/Semantics/OpenMP/allocators03.f90
+++ b/flang/test/Semantics/OpenMP/allocators03.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.2
 ! 6.7 allocators construct
 ! Only the allocate clause is allowed on the allocators construct
diff --git a/flang/test/Semantics/OpenMP/allocators04.f90 b/flang/test/Semantics/OpenMP/allocators04.f90
index 0c6e994e137a..1d2e96443a9d 100644
--- a/flang/test/Semantics/OpenMP/allocators04.f90
+++ b/flang/test/Semantics/OpenMP/allocators04.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.2
 ! Inherited from 2.11.3 allocate Directive
 ! If list items within the ALLOCATE directive have the SAVE attribute, are a common block name, or are declared in the scope of a
diff --git a/flang/test/Semantics/OpenMP/allocators05.f90 b/flang/test/Semantics/OpenMP/allocators05.f90
index 07f1357b75b5..d0e11ca5874d 100644
--- a/flang/test/Semantics/OpenMP/allocators05.f90
+++ b/flang/test/Semantics/OpenMP/allocators05.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.2
 ! Inherited from 2.11.3 allocate directive
 ! allocate directives that appear in a target region must specify an
diff --git a/flang/test/Semantics/OpenMP/allocators06.f90 b/flang/test/Semantics/OpenMP/allocators06.f90
index d3773c05a485..a975204c1133 100644
--- a/flang/test/Semantics/OpenMP/allocators06.f90
+++ b/flang/test/Semantics/OpenMP/allocators06.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.2
 ! Inherited from 2.11.3 allocate directive
 ! The allocate directive must appear in the same scope as the declarations of
diff --git a/flang/test/Semantics/OpenMP/atomic-hint-clause.f90 b/flang/test/Semantics/OpenMP/atomic-hint-clause.f90
index 3cbde3ab586c..e157b7e1e73a 100644
--- a/flang/test/Semantics/OpenMP/atomic-hint-clause.f90
+++ b/flang/test/Semantics/OpenMP/atomic-hint-clause.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp 
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags 
 ! Semantic checks on hint clauses, as they appear on atomic constructs
 
 program sample
diff --git a/flang/test/Semantics/OpenMP/atomic01.f90 b/flang/test/Semantics/OpenMP/atomic01.f90
index 868c7623928a..f0e1b47d2fa1 100644
--- a/flang/test/Semantics/OpenMP/atomic01.f90
+++ b/flang/test/Semantics/OpenMP/atomic01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! Semantic checks for OpenMP 5.0 standard 2.17.7 atomic Construct.
 
 use omp_lib
diff --git a/flang/test/Semantics/OpenMP/atomic02.f90 b/flang/test/Semantics/OpenMP/atomic02.f90
index db67fa63dcdf..b823bc4c33b2 100644
--- a/flang/test/Semantics/OpenMP/atomic02.f90
+++ b/flang/test/Semantics/OpenMP/atomic02.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 
 ! OpenMP Atomic construct
 ! section 2.17.7
diff --git a/flang/test/Semantics/OpenMP/atomic03.f90 b/flang/test/Semantics/OpenMP/atomic03.f90
index 9bcbace10fb6..76367495b986 100644
--- a/flang/test/Semantics/OpenMP/atomic03.f90
+++ b/flang/test/Semantics/OpenMP/atomic03.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 
 ! OpenMP Atomic construct
 ! section 2.17.7
diff --git a/flang/test/Semantics/OpenMP/atomic04.f90 b/flang/test/Semantics/OpenMP/atomic04.f90
index c0419a69d2dd..a9644ad95aa3 100644
--- a/flang/test/Semantics/OpenMP/atomic04.f90
+++ b/flang/test/Semantics/OpenMP/atomic04.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 
 ! OpenMP Atomic construct
 ! section 2.17.7
diff --git a/flang/test/Semantics/OpenMP/atomic05.f90 b/flang/test/Semantics/OpenMP/atomic05.f90
index 7b970dba8e49..2d9566463309 100644
--- a/flang/test/Semantics/OpenMP/atomic05.f90
+++ b/flang/test/Semantics/OpenMP/atomic05.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang %openmp_flags
 
 ! This tests the various semantics related to the clauses of various OpenMP atomic constructs
 
diff --git a/flang/test/Semantics/OpenMP/clause-validity01.f90 b/flang/test/Semantics/OpenMP/clause-validity01.f90
index 21b99cb82549..22ac57065ffe 100644
--- a/flang/test/Semantics/OpenMP/clause-validity01.f90
+++ b/flang/test/Semantics/OpenMP/clause-validity01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags %openmp_module_flag
 use omp_lib
 ! Check OpenMP clause validity for the following directives:
 !
diff --git a/flang/test/Semantics/OpenMP/critical-hint-clause.f90 b/flang/test/Semantics/OpenMP/critical-hint-clause.f90
index f35eea149135..419187fa3bbf 100644
--- a/flang/test/Semantics/OpenMP/critical-hint-clause.f90
+++ b/flang/test/Semantics/OpenMP/critical-hint-clause.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp 
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags 
 ! Semantic checks on hint clauses, as they appear on critical construct
 
 program sample
diff --git a/flang/test/Semantics/OpenMP/declare-target01.f90 b/flang/test/Semantics/OpenMP/declare-target01.f90
index 549f708b08f9..2c50a9248280 100644
--- a/flang/test/Semantics/OpenMP/declare-target01.f90
+++ b/flang/test/Semantics/OpenMP/declare-target01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.1
 ! Check OpenMP construct validity for the following directives:
 ! 2.14.7 Declare Target Directive
diff --git a/flang/test/Semantics/OpenMP/flush02.f90 b/flang/test/Semantics/OpenMP/flush02.f90
index dbb57ac81450..18a0d0356bbd 100644
--- a/flang/test/Semantics/OpenMP/flush02.f90
+++ b/flang/test/Semantics/OpenMP/flush02.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 
 ! Check OpenMP 5.0 - 2.17.8 flush Construct
 ! Restriction -
diff --git a/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90 b/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90
index d9ced2cd7950..a346056dee38 100644
--- a/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90
+++ b/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! Semantic checks for various assignments related to atomic constructs
 
 program sample
diff --git a/flang/test/Semantics/OpenMP/parallel-sections01.f90 b/flang/test/Semantics/OpenMP/parallel-sections01.f90
index b073cc8223b6..6c5a053bf49c 100644
--- a/flang/test/Semantics/OpenMP/parallel-sections01.f90
+++ b/flang/test/Semantics/OpenMP/parallel-sections01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang %openmp_flags
 ! OpenMP version 5.0.0
 ! 2.13.3 parallel sections Construct
 ! The restrictions for the parallel construct and the sections construct apply
diff --git a/flang/test/Semantics/OpenMP/resolve06.f90 b/flang/test/Semantics/OpenMP/resolve06.f90
index f97290500559..358b1b1cc282 100644
--- a/flang/test/Semantics/OpenMP/resolve06.f90
+++ b/flang/test/Semantics/OpenMP/resolve06.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 use omp_lib
 !2.11.4 Allocate Clause
 !For any list item that is specified in the allocate
diff --git a/flang/test/Semantics/OpenMP/sections02.f90 b/flang/test/Semantics/OpenMP/sections02.f90
index b96455a960ef..ee29922a72c0 100644
--- a/flang/test/Semantics/OpenMP/sections02.f90
+++ b/flang/test/Semantics/OpenMP/sections02.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang %openmp_flags
 ! OpenMP version 5.0.0
 ! 2.8.1 sections construct
 ! The code enclosed in a sections construct must be a structured block.
diff --git a/flang/test/Semantics/OpenMP/sync-critical02.f90 b/flang/test/Semantics/OpenMP/sync-critical02.f90
index fdd95fa76ad7..1fa9d6ad84f2 100644
--- a/flang/test/Semantics/OpenMP/sync-critical02.f90
+++ b/flang/test/Semantics/OpenMP/sync-critical02.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang %openmp_flags
 
 ! OpenMP Version 5.0
 ! 2.17.1 critical construct
diff --git a/flang/test/Semantics/OpenMP/taskgroup01.f90 b/flang/test/Semantics/OpenMP/taskgroup01.f90
index e3185552a44b..9de1df91bf3b 100644
--- a/flang/test/Semantics/OpenMP/taskgroup01.f90
+++ b/flang/test/Semantics/OpenMP/taskgroup01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang %openmp_flags
 
 use omp_lib
   implicit none
diff --git a/flang/test/Semantics/OpenMP/threadprivate01.f90 b/flang/test/Semantics/OpenMP/threadprivate01.f90
index 5ba5f400eb20..c2cf9ba99ab0 100644
--- a/flang/test/Semantics/OpenMP/threadprivate01.f90
+++ b/flang/test/Semantics/OpenMP/threadprivate01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
 ! OpenMP Version 5.1
 ! Check OpenMP construct validity for the following directives:
 ! 2.21.2 Threadprivate Directive
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 5eeada0e9ee3..37869e7e2ecd 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -203,6 +203,15 @@ if result:
 # Determine if OpenMP runtime was built (enable OpenMP tests via REQUIRES in test file)
 if config.have_openmp_rtl:
     config.available_features.add("openmp_runtime")
+    # For the enabled OpenMP tests, add a substitution that is needed in the tests to find
+    # the omp_lib.{h,mod} files, depending on whether the OpenMP runtime was built as a
+    # project or runtime.
+    if config.openmp_module_dir:
+        config.substitutions.append(
+            ("%openmp_flags", f"-fopenmp -J {config.openmp_module_dir}")
+        )
+    else:
+        config.substitutions.append(("%openmp_flags", "-fopenmp"))
 
 # Add features and substitutions to test F128 math support.
 # %f128-lib substitution may be used to generate check prefixes
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index b9b7a7c5fd63..fe6186d71407 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -25,7 +25,11 @@ config.cc = "@CMAKE_C_COMPILER@"
 config.osx_sysroot = path(r"@CMAKE_OSX_SYSROOT@")
 config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.default_sysroot = "@DEFAULT_SYSROOT@"
-config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE")
+config.have_openmp_rtl = ("@LLVM_TOOL_OPENMP_BUILD@" == "TRUE") or ("openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"))
+if "openmp" in "@LLVM_ENABLE_RUNTIMES@".lower().split(";"):
+    config.openmp_module_dir = "@CMAKE_BINARY_DIR@/runtimes/runtimes-bins/openmp/runtime/src"
+else:
+    config.openmp_module_dir = None
 config.flang_runtime_f128_math_lib = "@FLANG_RUNTIME_F128_MATH_LIB@"
 
 import lit.llvm
-- 
GitLab


From 2a97b507dc643b7ee3bc651b3f21b754cfba433c Mon Sep 17 00:00:00 2001
From: Kareem Ergawy <kareem.ergawy@amd.com>
Date: Sat, 18 May 2024 08:39:58 +0200
Subject: [PATCH 1188/1206] [flang][OpenMP] Try to unify induction var
 privatization for OMP regions. (#91116)

---
 flang/include/flang/Lower/AbstractConverter.h |   3 +-
 flang/lib/Lower/Bridge.cpp                    |   3 +-
 .../lib/Lower/OpenMP/DataSharingProcessor.cpp | 108 ++++++++++++------
 flang/lib/Lower/OpenMP/DataSharingProcessor.h |   9 +-
 flang/lib/Lower/OpenMP/Decomposer.cpp         |   5 +
 flang/lib/Lower/OpenMP/Decomposer.h           |   3 +
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  27 +++--
 .../Todo/omp-default-clause-inner-loop.f90    |   2 +-
 flang/test/Lower/OpenMP/copyin.f90            |  16 ++-
 flang/test/Lower/OpenMP/critical.f90          |  23 ++++
 flang/test/Lower/OpenMP/default-clause.f90    |  27 +++--
 .../Lower/OpenMP/hlfir-seqloop-parallel.f90   |  13 ++-
 flang/test/Lower/OpenMP/hlfir-wsloop.f90      |   2 +-
 flang/test/Lower/OpenMP/lastprivate-iv.f90    |  14 ++-
 .../parallel-lastprivate-clause-scalar.f90    |  23 ++--
 .../OpenMP/parallel-private-clause-fixes.f90  |   4 +-
 .../Lower/OpenMP/parallel-private-clause.f90  |  17 +--
 .../parallel-reduction-allocatable-array.f90  |   2 +-
 .../test/Lower/OpenMP/parallel-reduction3.f90 |   2 +-
 .../OpenMP/parallel-wsloop-firstpriv.f90      |  15 ++-
 flang/test/Lower/OpenMP/parallel-wsloop.f90   |  32 ++++--
 .../test/Lower/OpenMP/stop-stmt-in-region.f90 |   8 +-
 flang/test/Lower/OpenMP/target.f90            |   2 +-
 flang/test/Lower/OpenMP/unstructured.f90      |  14 ++-
 flang/test/Lower/OpenMP/wsloop-collapse.f90   |  40 +++++--
 flang/test/Lower/OpenMP/wsloop-monotonic.f90  |   2 +-
 .../test/Lower/OpenMP/wsloop-nonmonotonic.f90 |   2 +-
 .../OpenMP/wsloop-reduction-add-byref.f90     |  14 +--
 .../wsloop-reduction-add-hlfir-byref.f90      |   2 +-
 .../OpenMP/wsloop-reduction-add-hlfir.f90     |   2 +-
 .../Lower/OpenMP/wsloop-reduction-add.f90     |  14 +--
 .../OpenMP/wsloop-reduction-allocatable.f90   |   2 +-
 .../wsloop-reduction-array-assumed-shape.f90  |   2 +-
 .../Lower/OpenMP/wsloop-reduction-array.f90   |   2 +-
 .../Lower/OpenMP/wsloop-reduction-array2.f90  |   2 +-
 .../OpenMP/wsloop-reduction-iand-byref.f90    |   2 +-
 .../Lower/OpenMP/wsloop-reduction-iand.f90    |   2 +-
 .../OpenMP/wsloop-reduction-ieor-byref.f90    |   2 +-
 .../Lower/OpenMP/wsloop-reduction-ieor.f90    |   2 +-
 .../OpenMP/wsloop-reduction-ior-byref.f90     |   2 +-
 .../Lower/OpenMP/wsloop-reduction-ior.f90     |   2 +-
 .../wsloop-reduction-logical-and-byref.f90    |   6 +-
 .../OpenMP/wsloop-reduction-logical-and.f90   |   6 +-
 .../wsloop-reduction-logical-eqv-byref.f90    |   6 +-
 .../OpenMP/wsloop-reduction-logical-eqv.f90   |   6 +-
 .../wsloop-reduction-logical-neqv-byref.f90   |   6 +-
 .../OpenMP/wsloop-reduction-logical-neqv.f90  |   6 +-
 .../wsloop-reduction-logical-or-byref.f90     |   6 +-
 .../OpenMP/wsloop-reduction-logical-or.f90    |   6 +-
 .../OpenMP/wsloop-reduction-max-byref.f90     |   6 +-
 .../wsloop-reduction-max-hlfir-byref.f90      |   2 +-
 .../OpenMP/wsloop-reduction-max-hlfir.f90     |   2 +-
 .../Lower/OpenMP/wsloop-reduction-max.f90     |   6 +-
 .../OpenMP/wsloop-reduction-min-byref.f90     |   6 +-
 .../Lower/OpenMP/wsloop-reduction-min.f90     |   6 +-
 .../Lower/OpenMP/wsloop-reduction-min2.f90    |   2 +-
 .../OpenMP/wsloop-reduction-mul-byref.f90     |  14 +--
 .../Lower/OpenMP/wsloop-reduction-mul.f90     |  14 +--
 .../wsloop-reduction-multiple-clauses.f90     |   2 +-
 flang/test/Lower/OpenMP/wsloop-variable.f90   |  20 +++-
 flang/test/Lower/OpenMP/wsloop.f90            |   6 +-
 61 files changed, 388 insertions(+), 216 deletions(-)

diff --git a/flang/include/flang/Lower/AbstractConverter.h b/flang/include/flang/Lower/AbstractConverter.h
index 0bc68de6938d..f43dfd8343ec 100644
--- a/flang/include/flang/Lower/AbstractConverter.h
+++ b/flang/include/flang/Lower/AbstractConverter.h
@@ -131,7 +131,8 @@ public:
 
   /// For a given symbol, check if it is present in the inner-most
   /// level of the symbol map.
-  virtual bool isPresentShallowLookup(Fortran::semantics::Symbol &sym) = 0;
+  virtual bool
+  isPresentShallowLookup(const Fortran::semantics::Symbol &sym) = 0;
 
   /// Collect the set of symbols with \p flag in \p eval
   /// region if \p collectSymbols is true. Otherwise, collect the
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4a29c6b8fae7..7ded9adcd5c2 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -601,7 +601,8 @@ public:
     return typeConstructionStack;
   }
 
-  bool isPresentShallowLookup(Fortran::semantics::Symbol &sym) override final {
+  bool
+  isPresentShallowLookup(const Fortran::semantics::Symbol &sym) override final {
     return bool(shallowLookupSymbol(sym));
   }
 
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 84e745f28f97..6a8c3e3ac9ba 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -29,9 +29,10 @@ void DataSharingProcessor::processStep1(
   collectSymbolsForPrivatization();
   collectDefaultSymbols();
   collectImplicitSymbols();
+  collectPreDeterminedSymbols();
+
   privatize(clauseOps, privateSyms);
-  defaultPrivatize(clauseOps, privateSyms);
-  implicitPrivatize(clauseOps, privateSyms);
+
   insertBarrier();
 }
 
@@ -57,7 +58,7 @@ void DataSharingProcessor::processStep2(mlir::Operation *op, bool isLoop) {
 }
 
 void DataSharingProcessor::insertDeallocs() {
-  for (const semantics::Symbol *sym : privatizedSymbols)
+  for (const semantics::Symbol *sym : allPrivatizedSymbols)
     if (semantics::IsAllocatable(sym->GetUltimate())) {
       if (!useDelayedPrivatization) {
         converter.createHostAssociateVarCloneDealloc(*sym);
@@ -92,10 +93,6 @@ void DataSharingProcessor::insertDeallocs() {
 }
 
 void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) {
-  // Privatization for symbols which are pre-determined (like loop index
-  // variables) happen separately, for everything else privatize here.
-  if (sym->test(semantics::Symbol::Flag::OmpPreDetermined))
-    return;
   bool success = converter.createHostAssociateVarClone(*sym);
   (void)success;
   assert(success && "Privatization failed due to existing binding");
@@ -126,20 +123,24 @@ void DataSharingProcessor::collectSymbolsForPrivatization() {
   for (const omp::Clause &clause : clauses) {
     if (const auto &privateClause =
             std::get_if<omp::clause::Private>(&clause.u)) {
-      collectOmpObjectListSymbol(privateClause->v, privatizedSymbols);
+      collectOmpObjectListSymbol(privateClause->v, explicitlyPrivatizedSymbols);
     } else if (const auto &firstPrivateClause =
                    std::get_if<omp::clause::Firstprivate>(&clause.u)) {
-      collectOmpObjectListSymbol(firstPrivateClause->v, privatizedSymbols);
+      collectOmpObjectListSymbol(firstPrivateClause->v,
+                                 explicitlyPrivatizedSymbols);
     } else if (const auto &lastPrivateClause =
                    std::get_if<omp::clause::Lastprivate>(&clause.u)) {
       const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t);
-      collectOmpObjectListSymbol(objects, privatizedSymbols);
+      collectOmpObjectListSymbol(objects, explicitlyPrivatizedSymbols);
       hasLastPrivateOp = true;
     } else if (std::get_if<omp::clause::Collapse>(&clause.u)) {
       hasCollapse = true;
     }
   }
 
+  for (auto *sym : explicitlyPrivatizedSymbols)
+    allPrivatizedSymbols.insert(sym);
+
   if (hasCollapse && hasLastPrivateOp)
     TODO(converter.getCurrentLocation(), "Collapse clause with lastprivate");
 }
@@ -149,7 +150,7 @@ bool DataSharingProcessor::needBarrier() {
   // initialization of firstprivate variables and post-update of lastprivate
   // variables.
   // Emit implicit barrier for linear clause. Maybe on somewhere else.
-  for (const semantics::Symbol *sym : privatizedSymbols) {
+  for (const semantics::Symbol *sym : allPrivatizedSymbols) {
     if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) &&
         sym->test(semantics::Symbol::Flag::OmpLastPrivate))
       return true;
@@ -283,10 +284,40 @@ void DataSharingProcessor::collectSymbolsInNestedRegions(
       if (nestedEval.isConstruct())
         // Recursively look for OpenMP constructs within `nestedEval`'s region
         collectSymbolsInNestedRegions(nestedEval, flag, symbolsInNestedRegions);
-      else
-        converter.collectSymbolSet(nestedEval, symbolsInNestedRegions, flag,
-                                   /*collectSymbols=*/true,
-                                   /*collectHostAssociatedSymbols=*/false);
+      else {
+        bool isOrderedConstruct = [&]() {
+          if (auto *ompConstruct =
+                  nestedEval.getIf<parser::OpenMPConstruct>()) {
+            if (auto *ompBlockConstruct =
+                    std::get_if<parser::OpenMPBlockConstruct>(
+                        &ompConstruct->u)) {
+              const auto &beginBlockDirective =
+                  std::get<parser::OmpBeginBlockDirective>(
+                      ompBlockConstruct->t);
+              const auto origDirective =
+                  std::get<parser::OmpBlockDirective>(beginBlockDirective.t).v;
+
+              return origDirective == llvm::omp::Directive::OMPD_ordered;
+            }
+          }
+
+          return false;
+        }();
+
+        bool isCriticalConstruct = [&]() {
+          if (auto *ompConstruct =
+                  nestedEval.getIf<parser::OpenMPConstruct>()) {
+            return std::get_if<parser::OpenMPCriticalConstruct>(
+                       &ompConstruct->u) != nullptr;
+          }
+          return false;
+        }();
+
+        if (!isOrderedConstruct && !isCriticalConstruct)
+          converter.collectSymbolSet(nestedEval, symbolsInNestedRegions, flag,
+                                     /*collectSymbols=*/true,
+                                     /*collectHostAssociatedSymbols=*/false);
+      }
     }
   }
 }
@@ -322,24 +353,39 @@ void DataSharingProcessor::collectSymbols(
   converter.collectSymbolSet(eval, allSymbols, flag,
                              /*collectSymbols=*/true,
                              /*collectHostAssociatedSymbols=*/true);
+
   llvm::SetVector<const semantics::Symbol *> symbolsInNestedRegions;
   collectSymbolsInNestedRegions(eval, flag, symbolsInNestedRegions);
   // Filter-out symbols that must not be privatized.
   bool collectImplicit = flag == semantics::Symbol::Flag::OmpImplicit;
+  bool collectPreDetermined = flag == semantics::Symbol::Flag::OmpPreDetermined;
+
   auto isPrivatizable = [](const semantics::Symbol &sym) -> bool {
     return !semantics::IsProcedure(sym) &&
            !sym.GetUltimate().has<semantics::DerivedTypeDetails>() &&
            !sym.GetUltimate().has<semantics::NamelistDetails>() &&
            !semantics::IsImpliedDoIndex(sym.GetUltimate());
   };
+
+  auto shouldCollectSymbol = [&](const semantics::Symbol *sym) {
+    if (collectImplicit)
+      return sym->test(semantics::Symbol::Flag::OmpImplicit);
+
+    if (collectPreDetermined)
+      return sym->test(semantics::Symbol::Flag::OmpPreDetermined);
+
+    return !sym->test(semantics::Symbol::Flag::OmpImplicit) &&
+           !sym->test(semantics::Symbol::Flag::OmpPreDetermined);
+  };
+
   for (const auto *sym : allSymbols) {
     assert(curScope && "couldn't find current scope");
     if (isPrivatizable(*sym) && !symbolsInNestedRegions.contains(sym) &&
-        !privatizedSymbols.contains(sym) &&
-        !sym->test(semantics::Symbol::Flag::OmpPreDetermined) &&
-        (collectImplicit || !sym->test(semantics::Symbol::Flag::OmpImplicit)) &&
-        clauseScopes.contains(&sym->owner()))
+        !explicitlyPrivatizedSymbols.contains(sym) &&
+        shouldCollectSymbol(sym) && clauseScopes.contains(&sym->owner())) {
+      allPrivatizedSymbols.insert(sym);
       symbols.insert(sym);
+    }
   }
 }
 
@@ -363,10 +409,16 @@ void DataSharingProcessor::collectImplicitSymbols() {
     collectSymbols(semantics::Symbol::Flag::OmpImplicit, implicitSymbols);
 }
 
+void DataSharingProcessor::collectPreDeterminedSymbols() {
+  if (shouldCollectPreDeterminedSymbols)
+    collectSymbols(semantics::Symbol::Flag::OmpPreDetermined,
+                   preDeterminedSymbols);
+}
+
 void DataSharingProcessor::privatize(
     mlir::omp::PrivateClauseOps *clauseOps,
     llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
-  for (const semantics::Symbol *sym : privatizedSymbols) {
+  for (const semantics::Symbol *sym : allPrivatizedSymbols) {
     if (const auto *commonDet =
             sym->detailsIf<semantics::CommonBlockDetails>()) {
       for (const auto &mem : commonDet->objects())
@@ -378,7 +430,7 @@ void DataSharingProcessor::privatize(
 
 void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
   insertLastPrivateCompare(op);
-  for (const semantics::Symbol *sym : privatizedSymbols)
+  for (const semantics::Symbol *sym : allPrivatizedSymbols)
     if (const auto *commonDet =
             sym->detailsIf<semantics::CommonBlockDetails>()) {
       for (const auto &mem : commonDet->objects()) {
@@ -389,20 +441,6 @@ void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
     }
 }
 
-void DataSharingProcessor::defaultPrivatize(
-    mlir::omp::PrivateClauseOps *clauseOps,
-    llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
-  for (const semantics::Symbol *sym : defaultSymbols)
-    doPrivatize(sym, clauseOps, privateSyms);
-}
-
-void DataSharingProcessor::implicitPrivatize(
-    mlir::omp::PrivateClauseOps *clauseOps,
-    llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
-  for (const semantics::Symbol *sym : implicitSymbols)
-    doPrivatize(sym, clauseOps, privateSyms);
-}
-
 void DataSharingProcessor::doPrivatize(
     const semantics::Symbol *sym, mlir::omp::PrivateClauseOps *clauseOps,
     llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms) {
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index 111266eeb784..9ec5304eb69d 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -37,9 +37,12 @@ private:
   mlir::OpBuilder::InsertPoint insPt;
   mlir::Value loopIV;
   // Symbols in private, firstprivate, and/or lastprivate clauses.
-  llvm::SetVector<const semantics::Symbol *> privatizedSymbols;
+  llvm::SetVector<const semantics::Symbol *> explicitlyPrivatizedSymbols;
   llvm::SetVector<const semantics::Symbol *> defaultSymbols;
   llvm::SetVector<const semantics::Symbol *> implicitSymbols;
+  llvm::SetVector<const semantics::Symbol *> preDeterminedSymbols;
+  llvm::SetVector<const semantics::Symbol *> allPrivatizedSymbols;
+
   llvm::DenseMap<const semantics::Symbol *, mlir::omp::PrivateClauseOp>
       symToPrivatizer;
   lower::AbstractConverter &converter;
@@ -47,6 +50,7 @@ private:
   fir::FirOpBuilder &firOpBuilder;
   omp::List<omp::Clause> clauses;
   lower::pft::Evaluation &eval;
+  bool shouldCollectPreDeterminedSymbols;
   bool useDelayedPrivatization;
   lower::SymMap *symTable;
 
@@ -63,6 +67,7 @@ private:
   void insertBarrier();
   void collectDefaultSymbols();
   void collectImplicitSymbols();
+  void collectPreDeterminedSymbols();
   void privatize(mlir::omp::PrivateClauseOps *clauseOps,
                  llvm::SmallVectorImpl<const semantics::Symbol *> *privateSyms);
   void defaultPrivatize(
@@ -90,10 +95,12 @@ public:
                        semantics::SemanticsContext &semaCtx,
                        const List<Clause> &clauses,
                        lower::pft::Evaluation &eval,
+                       bool shouldCollectPreDeterminedSymbols,
                        bool useDelayedPrivatization = false,
                        lower::SymMap *symTable = nullptr)
       : hasLastPrivateOp(false), converter(converter), semaCtx(semaCtx),
         firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval),
+        shouldCollectPreDeterminedSymbols(shouldCollectPreDeterminedSymbols),
         useDelayedPrivatization(useDelayedPrivatization), symTable(symTable) {}
 
   // Privatisation is split into two steps.
diff --git a/flang/lib/Lower/OpenMP/Decomposer.cpp b/flang/lib/Lower/OpenMP/Decomposer.cpp
index e6897cb81e94..66e4028c7a28 100644
--- a/flang/lib/Lower/OpenMP/Decomposer.cpp
+++ b/flang/lib/Lower/OpenMP/Decomposer.cpp
@@ -123,4 +123,9 @@ ConstructQueue buildConstructQueue(
 
   return constructs;
 }
+
+bool isLastItemInQueue(ConstructQueue::iterator item,
+                       const ConstructQueue &queue) {
+  return std::next(item) == queue.end();
+}
 } // namespace Fortran::lower::omp
diff --git a/flang/lib/Lower/OpenMP/Decomposer.h b/flang/lib/Lower/OpenMP/Decomposer.h
index f42d8f5c1740..a7851d8534e5 100644
--- a/flang/lib/Lower/OpenMP/Decomposer.h
+++ b/flang/lib/Lower/OpenMP/Decomposer.h
@@ -46,6 +46,9 @@ ConstructQueue buildConstructQueue(mlir::ModuleOp modOp,
                                    const parser::CharBlock &source,
                                    llvm::omp::Directive compound,
                                    const List<Clause> &clauses);
+
+bool isLastItemInQueue(ConstructQueue::iterator item,
+                       const ConstructQueue &queue);
 } // namespace Fortran::lower::omp
 
 #endif // FORTRAN_LOWER_OPENMP_DECOMPOSER_H
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 4ae1fe6d9902..ece098a5bfbb 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -215,12 +215,10 @@ createAndSetPrivatizedLoopVar(lower::AbstractConverter &converter,
   firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
 
   mlir::Type tempTy = converter.genType(*sym);
-  mlir::Value temp = firOpBuilder.create<fir::AllocaOp>(
-      loc, tempTy, /*pinned=*/true, /*lengthParams=*/mlir::ValueRange{},
-      /*shapeParams*/ mlir::ValueRange{},
-      llvm::ArrayRef<mlir::NamedAttribute>{
-          fir::getAdaptToByRefAttr(firOpBuilder)});
-  converter.bindSymbol(*sym, temp);
+
+  assert(converter.isPresentShallowLookup(*sym) &&
+         "Expected symbol to be in symbol table.");
+
   firOpBuilder.restoreInsertionPoint(insPt);
   mlir::Value cvtVal = firOpBuilder.createConvert(loc, tempTy, indexVal);
   mlir::Operation *storeOp = firOpBuilder.create<fir::StoreOp>(
@@ -580,7 +578,8 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
   std::optional<DataSharingProcessor> tempDsp;
   if (privatize) {
     if (!info.dsp) {
-      tempDsp.emplace(info.converter, info.semaCtx, *info.clauses, info.eval);
+      tempDsp.emplace(info.converter, info.semaCtx, *info.clauses, info.eval,
+                      Fortran::lower::omp::isLastItemInQueue(item, queue));
       tempDsp->processStep1();
     }
   }
@@ -1316,6 +1315,7 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
   bool privatize = !outerCombined;
   DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
+                           lower::omp::isLastItemInQueue(item, queue),
                            /*useDelayedPrivatization=*/true, &symTable);
 
   if (privatize)
@@ -1388,7 +1388,8 @@ genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
   // Insert privatizations before SECTIONS
   symTable.pushScope();
-  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval);
+  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
+                           lower::omp::isLastItemInQueue(item, queue));
   dsp.processStep1();
 
   List<Clause> nonDsaClauses;
@@ -1458,7 +1459,9 @@ genSimdOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
           mlir::Location loc, const ConstructQueue &queue,
           ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval);
+  symTable.pushScope();
+  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
+                           lower::omp::isLastItemInQueue(item, queue));
   dsp.processStep1();
 
   lower::StatementContext stmtCtx;
@@ -1496,6 +1499,7 @@ genSimdOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
                      .setGenRegionEntryCb(ivCallback),
                  queue, item);
 
+  symTable.popScope();
   return simdOp;
 }
 
@@ -1761,7 +1765,9 @@ genWsloopOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
             mlir::Location loc, const ConstructQueue &queue,
             ConstructQueue::iterator item) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval);
+  symTable.pushScope();
+  DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
+                           lower::omp::isLastItemInQueue(item, queue));
   dsp.processStep1();
 
   lower::StatementContext stmtCtx;
@@ -1804,6 +1810,7 @@ genWsloopOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
                      .setReductions(&reductionSyms, &reductionTypes)
                      .setGenRegionEntryCb(ivCallback),
                  queue, item);
+  symTable.popScope();
   return wsloopOp;
 }
 
diff --git a/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90 b/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90
index c245137f16c7..773452206993 100644
--- a/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90
+++ b/flang/test/Lower/OpenMP/Todo/omp-default-clause-inner-loop.f90
@@ -8,7 +8,7 @@
 ! CHECK: omp.parallel   {
 ! EXPECTED: %[[PRIVATE_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFEy"}
 ! EXPECTED: %[[PRIVATE_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFEz"}
-! CHECK: %[[TEMP:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK: %[[TEMP:.*]] = fir.alloca i32 {bindc_name = "x", pinned, {{.*}}}
 ! CHECK: %[[const_1:.*]] = arith.constant 1 : i32
 ! CHECK: %[[const_2:.*]] = arith.constant 10 : i32
 ! CHECK: %[[const_3:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/OpenMP/copyin.f90 b/flang/test/Lower/OpenMP/copyin.f90
index dda563303148..34c83fca4641 100644
--- a/flang/test/Lower/OpenMP/copyin.f90
+++ b/flang/test/Lower/OpenMP/copyin.f90
@@ -146,13 +146,17 @@ end
 ! CHECK:           %[[VAL_4:.*]] = omp.threadprivate %[[VAL_3]]#1 : !fir.ref<i32> -> !fir.ref<i32>
 ! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFcombined_parallel_worksharing_loopEx6"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_6:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFcombined_parallel_worksharing_loopEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[VAL_8:.*]] = omp.threadprivate %[[VAL_3]]#1 : !fir.ref<i32> -> !fir.ref<i32>
 ! CHECK:             %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFcombined_parallel_worksharing_loopEx6"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_10:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[VAL_10]] to %[[VAL_9]]#0 temporary_lhs : i32, !fir.ref<i32>
+
 ! CHECK:             omp.barrier
+
+! CHECK:             %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFcombined_parallel_worksharing_loopEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_12:.*]] = fir.load %[[VAL_9]]#0 : !fir.ref<i32>
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
@@ -280,7 +284,6 @@ subroutine common_1()
   !$omp end parallel sections
 end subroutine
 
-
 ! CHECK-LABEL:   func.func @_QPcommon_2() {
 ! CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFcommon_2Ei"}
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFcommon_2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
@@ -302,8 +305,7 @@ end subroutine
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (!fir.ref<i8>) -> !fir.ref<i32>
 ! CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFcommon_2Ey"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_19:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-! CHECK:             %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFcommon_2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[VAL_21:.*]] = omp.threadprivate %[[VAL_2]] : !fir.ref<!fir.array<8xi8>> -> !fir.ref<!fir.array<8xi8>>
 ! CHECK:             %[[VAL_22:.*]] = fir.convert %[[VAL_21]] : (!fir.ref<!fir.array<8xi8>>) -> !fir.ref<!fir.array<?xi8>>
 ! CHECK:             %[[VAL_23:.*]] = arith.constant 0 : index
@@ -320,6 +322,10 @@ end subroutine
 ! CHECK:             %[[VAL_33:.*]] = fir.load %[[VAL_18]]#0 : !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[VAL_33]] to %[[VAL_31]]#0 temporary_lhs : i32, !fir.ref<i32>
 ! CHECK:             omp.barrier
+
+! CHECK:             %[[VAL_19:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+! CHECK:             %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFcommon_2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[VAL_34:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_35:.*]] = fir.load %[[VAL_26]]#0 : !fir.ref<i32>
 ! CHECK:             %[[VAL_36:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/OpenMP/critical.f90 b/flang/test/Lower/OpenMP/critical.f90
index 98a90c959484..d62c58b3081a 100644
--- a/flang/test/Lower/OpenMP/critical.f90
+++ b/flang/test/Lower/OpenMP/critical.f90
@@ -28,3 +28,26 @@ subroutine omp_critical()
 !CHECK: omp.terminator
 !$OMP END CRITICAL
 end subroutine omp_critical
+
+
+! Tests that privatization for pre-determined variables (here `i`) is properly
+! handled.
+subroutine predetermined_privatization()
+  integer :: a(10), i
+
+  !CHECK: omp.parallel
+  !$omp parallel do
+
+  !CHECK: %[[PRIV_I_ALLOC:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+  !CHECK: %[[PRIV_I_DECL:.*]]:2 = hlfir.declare %[[PRIV_I_ALLOC]]
+  do i = 2, 10
+    !CHECK: omp.wsloop
+    !CHECK: omp.loop_nest (%[[IV:[^[:space:]]+]])
+    !CHECK: fir.store %[[IV]] to %[[PRIV_I_DECL]]#1
+    !CHECK: omp.critical
+    !$omp critical
+    a(i) = a(i-1) + 1
+    !$omp end critical
+  end do
+  !$omp end parallel do
+end
diff --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90
index 843ee6bb7910..60a9c5efbb2a 100644
--- a/flang/test/Lower/OpenMP/default-clause.f90
+++ b/flang/test/Lower/OpenMP/default-clause.f90
@@ -354,10 +354,13 @@ end subroutine
 
 !CHECK-LABEL: func @_QPnested_default_clause_test5
 !CHECK: omp.parallel {
-!CHECK: %[[LOOP_VAR_ALLOCA:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-!CHECK: %[[LOOP_VAR_DECLARE:.*]]:2 = hlfir.declare %[[LOOP_VAR_ALLOCA]] {{.*}}
+
 !CHECK: %[[X_ALLOCA:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_test5Ex"}
 !CHECK: %[[X_DECLARE:.*]]:2 = hlfir.declare %[[X_ALLOCA]] {{.*}}
+
+!CHECK: %[[LOOP_VAR_ALLOCA:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+!CHECK: %[[LOOP_VAR_DECLARE:.*]]:2 = hlfir.declare %[[LOOP_VAR_ALLOCA]] {{.*}}
+
 !CHECK: %[[CONST_LB:.*]] = arith.constant 1 : i32
 !CHECK: %[[CONST_UB:.*]] = arith.constant 50 : i32
 !CHECK: %[[CONST_STEP:.*]] = arith.constant 1 : i32
@@ -383,14 +386,18 @@ end subroutine
 
 !CHECK-LABEL: func @_QPnested_default_clause_test6
 !CHECK: omp.parallel {
-!CHECK: %[[LOOP_VAR:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-!CHECK: %[[LOOP_VAR_DECLARE:.*]]:2 = hlfir.declare %[[LOOP_VAR]] {{.*}}
 !CHECK: %[[X_VAR:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFnested_default_clause_test6Ex"}
 !CHECK: %[[X_VAR_DECLARE:.*]]:2 = hlfir.declare %[[X_VAR]] {{.*}}
+
 !CHECK: %[[Y_VAR:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_default_clause_test6Ey"}
 !CHECK: %[[Y_VAR_DECLARE:.*]]:2 = hlfir.declare %[[Y_VAR]] {{.*}}
+
 !CHECK: %[[Z_VAR:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_default_clause_test6Ez"}
 !CHECK: %[[Z_VAR_DECLARE:.*]]:2 = hlfir.declare %[[Z_VAR]] {{.*}}
+
+!CHECK: %[[LOOP_VAR:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+!CHECK: %[[LOOP_VAR_DECLARE:.*]]:2 = hlfir.declare %[[LOOP_VAR]] {{.*}}
+
 !CHECK: %[[CONST_LB:.*]] = arith.constant 1 : i32
 !CHECK: %[[CONST_UB:.*]] = arith.constant 10 : i32
 !CHECK: %[[CONST_STEP:.*]] = arith.constant 1 : i32
@@ -535,16 +542,20 @@ subroutine nested_constructs
 
     integer :: y, z
 !CHECK: omp.parallel {
-!CHECK: %[[INNER_J:.*]] = fir.alloca i32 {bindc_name = "j", pinned}
-!CHECK: %[[INNER_J_DECL:.*]]:2 = hlfir.declare %[[INNER_J]] {{.*}}
-!CHECK: %[[INNER_I:.*]] = fir.alloca i32 {bindc_name = "i", pinned}
-!CHECK: %[[INNER_I_DECL:.*]]:2 = hlfir.declare %[[INNER_I]] {{.*}}
+
 !CHECK: %[[INNER_Y:.*]] = fir.alloca i32 {bindc_name = "y", pinned, uniq_name = "_QFnested_constructsEy"}
 !CHECK: %[[INNER_Y_DECL:.*]]:2 = hlfir.declare %[[INNER_Y]] {{.*}}
 !CHECK: %[[TEMP:.*]] = fir.load %[[Y_DECL]]#0 : !fir.ref<i32>
 !CHECK: hlfir.assign %[[TEMP]] to %[[INNER_Y_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+
 !CHECK: %[[INNER_Z:.*]] = fir.alloca i32 {bindc_name = "z", pinned, uniq_name = "_QFnested_constructsEz"}
 !CHECK: %[[INNER_Z_DECL:.*]]:2 = hlfir.declare %[[INNER_Z]] {{.*}}
+
+!CHECK: %[[INNER_I:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+!CHECK: %[[INNER_I_DECL:.*]]:2 = hlfir.declare %[[INNER_I]] {{.*}}
+
+!CHECK: %[[INNER_J:.*]] = fir.alloca i32 {bindc_name = "j", pinned, {{.*}}}
+!CHECK: %[[INNER_J_DECL:.*]]:2 = hlfir.declare %[[INNER_J]] {{.*}}
     !$omp parallel default(private) firstprivate(y)
 !CHECK: {{.*}} = fir.do_loop {{.*}} {
       do i = 1, 10
diff --git a/flang/test/Lower/OpenMP/hlfir-seqloop-parallel.f90 b/flang/test/Lower/OpenMP/hlfir-seqloop-parallel.f90
index 68d20213a14b..271b97819e60 100644
--- a/flang/test/Lower/OpenMP/hlfir-seqloop-parallel.f90
+++ b/flang/test/Lower/OpenMP/hlfir-seqloop-parallel.f90
@@ -18,7 +18,7 @@ end subroutine
 !CHECK:    %[[I_ADDR:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsb1Ei"}
 !CHECK:    %[[I_DECL:.*]]:2 = hlfir.declare %[[I_ADDR]] {uniq_name = "_QFsb1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    omp.parallel   {
-!CHECK:      %[[I_PVT_ADDR:.*]] = fir.alloca i32 {bindc_name = "i", pinned}
+!CHECK:      %[[I_PVT_ADDR:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 !CHECK:      %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_ADDR]] {uniq_name = "_QFsb1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:      %[[I_FINAL_VAL:.*]]:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%[[I_VAL:.*]] = %{{.*}}) -> (index, i32) {
 !CHECK:        fir.store %[[I_VAL]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
@@ -51,12 +51,15 @@ end subroutine
 !CHECK:    %[[K_ADDR:.*]] = fir.alloca i32 {bindc_name = "k", uniq_name = "_QFsb2Ek"}
 !CHECK:    %[[K_DECL:.*]]:2 = hlfir.declare %[[K_ADDR]] {uniq_name = "_QFsb2Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    omp.parallel   {
-!CHECK:      %[[I_PVT_ADDR:.*]] = fir.alloca i32 {bindc_name = "i", pinned}
-!CHECK:      %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_ADDR]] {uniq_name = "_QFsb2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:      %[[J_PVT_ADDR:.*]] = fir.alloca i32 {bindc_name = "j", pinned}
+
+!CHECK:      %[[J_PVT_ADDR:.*]] = fir.alloca i32 {bindc_name = "j", pinned, {{.*}}}
 !CHECK:      %[[J_PVT_DECL:.*]]:2 = hlfir.declare %[[J_PVT_ADDR]] {uniq_name = "_QFsb2Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+!CHECK:      %[[I_PVT_ADDR:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+!CHECK:      %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_ADDR]] {uniq_name = "_QFsb2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:      %[[FINAL_J_VAL:.*]]:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%[[J_VAL:.*]] = %{{.*}}) -> (index, i32) {
-!CHECK:        fir.store %arg1 to %9#1 : !fir.ref<i32>
+!CHECK:        fir.store %arg1 to %[[J_PVT_DECL]]#1 : !fir.ref<i32>
 !CHECK:        fir.if %{{.*}} {
 !CHECK:          %[[FINAL_I_VAL:.*]]:2 = fir.do_loop %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%[[I_VAL:.*]] = %{{.*}}) -> (index, i32) {
 !CHECK:            fir.store %[[I_VAL]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/hlfir-wsloop.f90 b/flang/test/Lower/OpenMP/hlfir-wsloop.f90
index fea05ae3d6bc..fc5b5999f270 100644
--- a/flang/test/Lower/OpenMP/hlfir-wsloop.f90
+++ b/flang/test/Lower/OpenMP/hlfir-wsloop.f90
@@ -10,7 +10,7 @@ subroutine simple_loop
   ! CHECK-DAG:     %[[WS_END:.*]] = arith.constant 9 : i32
   ! CHECK:  omp.parallel
   !$OMP PARALLEL
-  ! CHECK-DAG:     %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned}
+  ! CHECK-DAG:     %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
   ! CHECK:         %[[IV:.*]]    = fir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loopEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
   ! CHECK:         omp.wsloop {
   ! CHECK-NEXT:      omp.loop_nest (%[[I:.*]]) : i32 = (%[[WS_ST]]) to (%[[WS_END]]) inclusive step (%[[WS_ST]]) {
diff --git a/flang/test/Lower/OpenMP/lastprivate-iv.f90 b/flang/test/Lower/OpenMP/lastprivate-iv.f90
index 24c20281b9c3..61c588732882 100644
--- a/flang/test/Lower/OpenMP/lastprivate-iv.f90
+++ b/flang/test/Lower/OpenMP/lastprivate-iv.f90
@@ -2,10 +2,13 @@
 ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
 
 !CHECK-LABEL: func @_QPlastprivate_iv_inc
-!CHECK:      %[[I_MEM:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-!CHECK:      %[[I:.*]]:2 = hlfir.declare %[[I_MEM]] {uniq_name = "_QFlastprivate_iv_incEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:      %[[I2_MEM:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFlastprivate_iv_incEi"}
 !CHECK:      %[[I2:.*]]:2 = hlfir.declare %[[I2_MEM]] {uniq_name = "_QFlastprivate_iv_incEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+!CHECK:      %[[I_MEM:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+!CHECK:      %[[I:.*]]:2 = hlfir.declare %[[I_MEM]] {uniq_name = "_QFlastprivate_iv_incEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:      %[[LB:.*]] = arith.constant 4 : i32
 !CHECK:      %[[UB:.*]] = arith.constant 10 : i32
 !CHECK:      %[[STEP:.*]]  = arith.constant 3 : i32
@@ -37,10 +40,13 @@ subroutine lastprivate_iv_inc()
 end subroutine
 
 !CHECK-LABEL: func @_QPlastprivate_iv_dec
-!CHECK:      %[[I_MEM:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-!CHECK:      %[[I:.*]]:2 = hlfir.declare %[[I_MEM]] {uniq_name = "_QFlastprivate_iv_decEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:      %[[I2_MEM:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFlastprivate_iv_decEi"}
 !CHECK:      %[[I2:.*]]:2 = hlfir.declare %[[I2_MEM]] {uniq_name = "_QFlastprivate_iv_decEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+!CHECK:      %[[I_MEM:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+!CHECK:      %[[I:.*]]:2 = hlfir.declare %[[I_MEM]] {uniq_name = "_QFlastprivate_iv_decEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:      %[[LB:.*]] = arith.constant 10 : i32
 !CHECK:      %[[UB:.*]] = arith.constant 1 : i32
 !CHECK:      %[[STEP:.*]]  = arith.constant -3 : i32
diff --git a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
index e6ee75c8a5be..be0cc4195c28 100644
--- a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
@@ -10,7 +10,7 @@
 !CHECK-DAG: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1_REF]] typeparams %[[FIVE]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFlastprivate_characterEarg1"} : (!fir.ref<!fir.char<1,5>>, index, !fir.dscope) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
 
 !CHECK: omp.parallel {
-!CHECK-DAG: %[[ARG1_PVT:.*]] = fir.alloca !fir.char<1,5> {bindc_name = "arg1",
+!CHECK-DAG: %[[ARG1_PVT:.*]] = fir.alloca !fir.char<1,5> {bindc_name = "arg1", pinned, {{.*}}}
 !CHECK-DAG: %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] typeparams %[[FIVE]] {uniq_name = "_QFlastprivate_characterEarg1"} : (!fir.ref<!fir.char<1,5>>, index) -> (!fir.ref<!fir.char<1,5>>, !fir.ref<!fir.char<1,5>>)
 
 ! Check that we are accessing the clone inside the loop
@@ -59,7 +59,7 @@ end subroutine
 !CHECK: func @_QPlastprivate_int(%[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "arg1"}) {
 !CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFlastprivate_intEarg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK-DAG: omp.parallel  {
-!CHECK-DAG: %[[CLONE:.*]] = fir.alloca i32 {bindc_name = "arg1"
+!CHECK-DAG: %[[CLONE:.*]] = fir.alloca i32 {bindc_name = "arg1", pinned, {{.*}}}
 !CHECK-DAG: %[[CLONE_DECL:.*]]:2 = hlfir.declare %[[CLONE]] {uniq_name = "_QFlastprivate_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.wsloop {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
@@ -100,9 +100,9 @@ end subroutine
 !CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_intEarg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_intEarg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
-!CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
+!CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1", pinned, {{.*}}}
 !CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK-DAG: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2"
+!CHECK-DAG: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2", pinned, {{.*}}}
 !CHECK-DAG: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFmult_lastprivate_intEarg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.wsloop {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
@@ -145,9 +145,9 @@ end subroutine
 !CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
-!CHECK-DAG: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2"
+!CHECK-DAG: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2", pinned, {{.*}}}
 !CHECK-DAG: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFmult_lastprivate_int2Earg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
+!CHECK-DAG: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1", pinned, {{.*}}}
 !CHECK-DAG: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFmult_lastprivate_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.wsloop {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
@@ -191,12 +191,12 @@ end subroutine
 !CHECK:    %[[ARG2_DECL:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstpriv_lastpriv_intEarg2"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
 ! Firstprivate update
-!CHECK: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
+!CHECK: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1", pinned, {{.*}}}
 !CHECK: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFfirstpriv_lastpriv_intEarg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: %[[FPV_LD:.*]] = fir.load %[[ARG1_DECL]]#0 : !fir.ref<i32>
 !CHECK: hlfir.assign %[[FPV_LD]] to %[[CLONE1_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
 ! Lastprivate Allocation
-!CHECK: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2"
+!CHECK: %[[CLONE2:.*]] = fir.alloca i32 {bindc_name = "arg2", pinned, {{.*}}}
 !CHECK: %[[CLONE2_DECL:.*]]:2 = hlfir.declare %[[CLONE2]] {uniq_name = "_QFfirstpriv_lastpriv_intEarg2"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK-NOT: omp.barrier
 !CHECK: omp.wsloop {
@@ -237,11 +237,16 @@ end subroutine
 !CHECK: func.func @_QPfirstpriv_lastpriv_int2(%[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "arg1"}) {
 !CHECK: %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstpriv_lastpriv_int2Earg1"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.parallel  {
+
 ! Firstprivate update
-!CHECK: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1"
+!CHECK: %[[CLONE1:.*]] = fir.alloca i32 {bindc_name = "arg1", pinned, {{.*}}}
 !CHECK: %[[CLONE1_DECL:.*]]:2 = hlfir.declare %[[CLONE1]] {uniq_name = "_QFfirstpriv_lastpriv_int2Earg1"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK-NEXT: %[[FPV_LD:.*]] = fir.load %[[ARG1_DECL]]#0 : !fir.ref<i32>
 !CHECK-NEXT: hlfir.assign %[[FPV_LD]] to %[[CLONE1_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+
+!CHECK-NEXT: %[[IV:.*]] = fir.alloca i32 {bindc_name = "n", pinned, {{.*}}}
+!CHECK-NEXT: hlfir.declare %[[IV]]
+
 !CHECK-NEXT: omp.barrier
 !CHECK: omp.wsloop {
 !CHECK-NEXT: omp.loop_nest (%[[INDX_WS:.*]]) : {{.*}} {
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
index d3843c8e241a..19c7b78298ea 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
@@ -12,11 +12,11 @@
 ! CHECK-DAG:         %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_private_fixEx"}
 ! CHECK-DAG:         %[[X_DECL:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFmultiple_private_fixEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:         omp.parallel {
-! CHECK-DAG:           %[[PRIV_I:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK-DAG:           %[[PRIV_I:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK-DAG:           %[[PRIV_I_DECL:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFmultiple_private_fixEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK-DAG:           %[[PRIV_J:.*]] = fir.alloca i32 {bindc_name = "j", pinned, uniq_name = "_QFmultiple_private_fixEj"}
 ! CHECK-DAG:           %[[PRIV_J_DECL:.*]]:2 = hlfir.declare %[[PRIV_J]] {uniq_name = "_QFmultiple_private_fixEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK-DAG:           %[[PRIV_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned
+! CHECK-DAG:           %[[PRIV_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, {{.*}}}
 ! CHECK-DAG:           %[[PRIV_X_DECL:.*]]:2 = hlfir.declare %[[PRIV_X]] {uniq_name = "_QFmultiple_private_fixEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[ONE:.*]] = arith.constant 1 : i32
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[GAMA_DECL]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause.f90 b/flang/test/Lower/OpenMP/parallel-private-clause.f90
index b9b58a135aaa..7f5bc2565e67 100644
--- a/flang/test/Lower/OpenMP/parallel-private-clause.f90
+++ b/flang/test/Lower/OpenMP/parallel-private-clause.f90
@@ -292,14 +292,14 @@ subroutine simple_loop_1
   real, allocatable :: r;
   ! FIRDialect:  omp.parallel
   !$OMP PARALLEL PRIVATE(r)
-  ! FIRDialect:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned}
-
-  ! FIRDialect:      %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loop_1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! FIRDialect:      [[R:%.*]] = fir.alloca !fir.box<!fir.heap<f32>> {bindc_name = "r", pinned, uniq_name = "{{.*}}Er"}
   ! FIRDialect:      fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
   ! FIRDialect:      fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
   ! FIRDialect:      %[[R_DECL:.*]]:2 = hlfir.declare [[R]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsimple_loop_1Er"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
 
+  ! FIRDialect:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
+  ! FIRDialect:      %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loop_1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
   ! FIRDialect:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! FIRDialect:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! FIRDialect:      %[[WS_STEP:.*]] = arith.constant 1 : i32
@@ -332,14 +332,14 @@ subroutine simple_loop_2
   real, allocatable :: r;
   ! FIRDialect:  omp.parallel
   !$OMP PARALLEL
-  ! FIRDialect:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned}
-
-  ! FIRDialect:      %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "{{.*}}Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! FIRDialect:      [[R:%.*]] = fir.alloca !fir.box<!fir.heap<f32>> {bindc_name = "r", pinned, uniq_name = "{{.*}}Er"}
   ! FIRDialect:      fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
   ! FIRDialect:      fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
   ! FIRDialect:      %[[R_DECL:.*]]:2 = hlfir.declare [[R]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "{{.*}}Er"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
 
+  ! FIRDialect:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
+  ! FIRDialect:      %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "{{.*}}Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
   ! FIRDialect:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! FIRDialect:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! FIRDialect:      %[[WS_STEP:.*]] = arith.constant 1 : i32
@@ -371,14 +371,15 @@ subroutine simple_loop_3
   integer :: i
   real, allocatable :: r;
   ! FIRDialect:  omp.parallel
-  ! FIRDialect:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned}
-  ! FIRDialect:      %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "{{.*}}Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
   ! FIRDialect:      [[R:%.*]] = fir.alloca !fir.box<!fir.heap<f32>> {bindc_name = "r", pinned, uniq_name = "{{.*}}Er"}
   ! FIRDialect:      fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
   ! FIRDialect:      fir.store {{%.*}} to [[R]] : !fir.ref<!fir.box<!fir.heap<f32>>>
   ! FIRDialect:      [[R_DECL:%.*]]:2 = hlfir.declare [[R]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "{{.*}}Er"} : (!fir.ref<!fir.box<!fir.heap<f32>>>) -> (!fir.ref<!fir.box<!fir.heap<f32>>>, !fir.ref<!fir.box<!fir.heap<f32>>>)
 
+  ! FIRDialect:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
+  ! FIRDialect:      %[[ALLOCA_IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "{{.*}}Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
   ! FIRDialect:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! FIRDialect:      %[[WS_UB:.*]] = arith.constant 9 : i32
   ! FIRDialect:      %[[WS_STEP:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
index f6d3b0b73f73..fdb7e974f1c5 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-allocatable-array.f90
@@ -90,7 +90,7 @@ end program
 ! CHECK:           %[[VAL_11:.*]] = fir.embox %[[VAL_9]](%[[VAL_10]]) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
 ! CHECK:           fir.store %[[VAL_11]] to %[[VAL_3]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_12:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_12:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_15:.*]] = arith.constant 10 : i32
diff --git a/flang/test/Lower/OpenMP/parallel-reduction3.f90 b/flang/test/Lower/OpenMP/parallel-reduction3.f90
index 47b743a558b4..879f59dfad6b 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction3.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction3.f90
@@ -67,7 +67,7 @@
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_13]] to %[[VAL_12]]#0 : i32, !fir.box<!fir.array<?xi32>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
index c32eb2400a34..33dab125b3b2 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
@@ -11,12 +11,15 @@ subroutine omp_do_firstprivate(a)
   n = a+1
   !$omp parallel do firstprivate(a)
   ! CHECK:  omp.parallel {
-  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_firstprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
   ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_do_firstprivateEa"}
   ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK-NEXT: %[[LD:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
   ! CHECK-NEXT: hlfir.assign %[[LD]] to %[[A_PVT_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+
+  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_firstprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
   ! CHECK: %[[LB:.*]] = arith.constant 1 : i32
   ! CHECK-NEXT: %[[UB:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
   ! CHECK-NEXT: %[[STEP:.*]] = arith.constant 1 : i32
@@ -45,17 +48,19 @@ subroutine omp_do_firstprivate2(a, n)
   n = a+1
   !$omp parallel do firstprivate(a, n)
   ! CHECK:  omp.parallel {
-  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_firstprivate2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned
+
+  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, {{.*}}}
   ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_firstprivate2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK: %[[LD:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
   ! CHECK: hlfir.assign %[[LD]] to %[[A_PVT_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+
   ! CHECK: %[[N_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "n", pinned, uniq_name = "_QFomp_do_firstprivate2En"}
   ! CHECK: %[[N_PVT_DECL:.*]]:2 = hlfir.declare %[[N_PVT_REF]] {uniq_name = "_QFomp_do_firstprivate2En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK: %[[LD1:.*]] = fir.load %[[ARG1_DECL]]#0 : !fir.ref<i32>
   ! CHECK: hlfir.assign %[[LD1]] to %[[N_PVT_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
 
+  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_firstprivate2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 
   ! CHECK: %[[LB:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
   ! CHECK: %[[UB:.*]] = fir.load %[[N_PVT_DECL]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop.f90 b/flang/test/Lower/OpenMP/parallel-wsloop.f90
index 5fa42da2269f..e5c303d7bb2e 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop.f90
@@ -147,14 +147,18 @@ end subroutine parallel_private_do
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "nt"}) {
 ! CHECK:           %[[NT_DECL:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFparallel_private_doEnt"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           omp.parallel   {
-! CHECK:             %[[I_PRIV:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV]] {uniq_name = "_QFparallel_private_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[COND_ADDR:.*]] = fir.alloca !fir.logical<4> {bindc_name = "cond", pinned, uniq_name = "_QFparallel_private_doEcond"}
 ! CHECK:             %[[COND_DECL:.*]]:2 = hlfir.declare %[[COND_ADDR]] {uniq_name = "_QFparallel_private_doEcond"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+
 ! CHECK:             %[[NT_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "nt", pinned, uniq_name = "_QFparallel_private_doEnt"}
 ! CHECK:             %[[NT_PRIV_DECL:.*]]:2 = hlfir.declare %[[NT_PRIV_ADDR]] {uniq_name = "_QFparallel_private_doEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[NT:.*]] = fir.load %[[NT_DECL]]#0 : !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[NT]] to %[[NT_PRIV_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+
+! CHECK:             %[[I_PRIV:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV]] {uniq_name = "_QFparallel_private_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 9 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
@@ -194,16 +198,20 @@ end subroutine omp_parallel_multiple_firstprivate_do
 ! CHECK:            %[[A_DECL:.*]]:2 = hlfir.declare %[[A_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:            %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           omp.parallel   {
-! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[A_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_parallel_multiple_firstprivate_doEa"}
 ! CHECK:             %[[A_PRIV_DECL:.*]]:2 = hlfir.declare %[[A_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[A:.*]] = fir.load %[[A_DECL]]#0 : !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[A]] to %[[A_PRIV_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+
 ! CHECK:             %[[B_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "b", pinned, uniq_name = "_QFomp_parallel_multiple_firstprivate_doEb"}
 ! CHECK:             %[[B_PRIV_DECL:.*]]:2 = hlfir.declare %[[B_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[B:.*]] = fir.load %[[B_DECL]]#0 : !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[B]] to %[[B_PRIV_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+
+! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_multiple_firstprivate_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 1 : i32
@@ -244,14 +252,18 @@ end subroutine parallel_do_private
 ! CHECK-SAME:                                      %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "nt"}) {
 ! CHECK:           %[[NT_DECL:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFparallel_do_privateEnt"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           omp.parallel   {
-! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFparallel_do_privateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[COND_PRIV_ADDR:.*]] = fir.alloca !fir.logical<4> {bindc_name = "cond", pinned, uniq_name = "_QFparallel_do_privateEcond"}
 ! CHECK:             %[[COND_PRIV_DECL:.*]]:2 = hlfir.declare %[[COND_PRIV_ADDR]] {uniq_name = "_QFparallel_do_privateEcond"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+
 ! CHECK:             %[[NT_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "nt", pinned, uniq_name = "_QFparallel_do_privateEnt"}
 ! CHECK:             %[[NT_PRIV_DECL:.*]]:2 = hlfir.declare %[[NT_PRIV_ADDR]] {uniq_name = "_QFparallel_do_privateEnt"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[NT_VAL:.*]] = fir.load %[[NT_DECL]]#0 : !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[NT_VAL]] to %[[NT_PRIV_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+
+! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFparallel_do_privateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 9 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
@@ -291,16 +303,20 @@ end subroutine omp_parallel_do_multiple_firstprivate
 ! CHECK:           %[[A_DECL:.*]]:2 = hlfir.declare %[[A_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEa"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[B_DECL:.*]]:2 = hlfir.declare %[[B_ADDR]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEb"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[A_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_parallel_do_multiple_firstprivateEa"}
 ! CHECK:             %[[A_PRIV_DECL:.*]]:2 = hlfir.declare %[[A_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[A:.*]] = fir.load %[[A_DECL]]#0 : !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[A]] to %[[A_PRIV_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+
 ! CHECK:             %[[B_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "b", pinned, uniq_name = "_QFomp_parallel_do_multiple_firstprivateEb"}
 ! CHECK:             %[[B_PRIV_DECL:.*]]:2 = hlfir.declare %[[B_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[B:.*]] = fir.load %[[B_DECL]]#0 : !fir.ref<i32>
 ! CHECK:             hlfir.assign %[[B]] to %[[B_PRIV_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+
+! CHECK:             %[[I_PRIV_ADDR:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+! CHECK:             %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_PRIV_ADDR]] {uniq_name = "_QFomp_parallel_do_multiple_firstprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 10 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/OpenMP/stop-stmt-in-region.f90 b/flang/test/Lower/OpenMP/stop-stmt-in-region.f90
index 4f3819c5e4eb..f8f2604caacc 100644
--- a/flang/test/Lower/OpenMP/stop-stmt-in-region.f90
+++ b/flang/test/Lower/OpenMP/stop-stmt-in-region.f90
@@ -73,12 +73,16 @@ subroutine test_stop_in_region3()
 end
 
 ! CHECK-LABEL: func.func @_QPtest_stop_in_region4() {
-! CHECK:         %[[VAL_0:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-! CHECK:         %[[VAL_0_DECL:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_stop_in_region4Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:         %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtest_stop_in_region4Ei"}
 ! CHECK:         %[[VAL_1_DECL:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest_stop_in_region4Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:         %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtest_stop_in_region4Ex"}
 ! CHECK:         %[[VAL_2_DECL:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtest_stop_in_region4Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+! CHECK:         %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+! CHECK:         %[[VAL_0_DECL:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_stop_in_region4Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:         %[[VAL_3:.*]] = arith.constant 1 : i32
 ! CHECK:         %[[VAL_4:.*]] = arith.constant 10 : i32
 ! CHECK:         %[[VAL_5:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/OpenMP/target.f90 b/flang/test/Lower/OpenMP/target.f90
index 9bb855e44694..0fcbad7e58fe 100644
--- a/flang/test/Lower/OpenMP/target.f90
+++ b/flang/test/Lower/OpenMP/target.f90
@@ -593,7 +593,7 @@ subroutine omp_target_parallel_do
       !CHECK: %[[VAL_0_DECL:.*]]:2 = hlfir.declare %[[ARG_0]](%{{.*}}) {uniq_name = "_QFomp_target_parallel_doEa"} : (!fir.ref<!fir.array<1024xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1024xi32>>, !fir.ref<!fir.array<1024xi32>>)
       !CHECK: omp.parallel
       !$omp target parallel do map(tofrom: a)
-         !CHECK: %[[I_PVT_ALLOCA:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+         !CHECK: %[[I_PVT_ALLOCA:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
          !CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_ALLOCA]] {uniq_name = "_QFomp_target_parallel_doEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
          !CHECK: omp.wsloop {
          !CHECK-NEXT: omp.loop_nest (%[[I_VAL:.*]]) : i32
diff --git a/flang/test/Lower/OpenMP/unstructured.f90 b/flang/test/Lower/OpenMP/unstructured.f90
index 6a1331799d54..b36e4f37a745 100644
--- a/flang/test/Lower/OpenMP/unstructured.f90
+++ b/flang/test/Lower/OpenMP/unstructured.f90
@@ -62,14 +62,14 @@ end
 ! CHECK:   omp.parallel {
 ! CHECK:     %[[ALLOCA_K:.*]] = fir.alloca i32 {bindc_name = "k", pinned}
 ! CHECK:     %[[K_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_K]] {uniq_name = "_QFss3Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:     %[[ALLOCA_1:.*]] = fir.alloca i32 {{{.*}}, pinned}
-! CHECK:     %[[OMP_LOOP_J_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_1]] {uniq_name = "_QFss3Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK:     %[[ALLOCA_2:.*]] = fir.alloca i32 {{{.*}}, pinned}
-! CHECK:     %[[OMP_LOOP_K_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_2]] {uniq_name = "_QFss3Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:     br ^bb1
 ! CHECK:   ^bb1:  // 2 preds: ^bb0, ^bb3
 ! CHECK:     cond_br %{{[0-9]*}}, ^bb2, ^bb4
 ! CHECK:   ^bb2:  // pred: ^bb1
+
+! CHECK:     %[[ALLOCA_2:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
+! CHECK:     %[[OMP_LOOP_K_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_2]] {uniq_name = "_QFss3Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:     omp.wsloop {
 ! CHECK:       omp.loop_nest (%[[ARG1:.*]]) : {{.*}} {
 ! CHECK:         fir.store %[[ARG1]] to %[[OMP_LOOP_K_DECL]]#1 : !fir.ref<i32>
@@ -80,6 +80,10 @@ end
 ! CHECK:       }
 ! CHECK:       omp.terminator
 ! CHECK:     }
+
+! CHECK:     %[[ALLOCA_1:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
+! CHECK:     %[[OMP_LOOP_J_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_1]] {uniq_name = "_QFss3Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 ! CHECK:     omp.wsloop {
 ! CHECK:       omp.loop_nest (%[[ARG2:.*]]) : {{.*}} {
 ! CHECK:         fir.store %[[ARG2]] to %[[OMP_LOOP_J_DECL]]#1 : !fir.ref<i32>
@@ -125,7 +129,7 @@ end
 
 ! CHECK-LABEL: func @_QPss4{{.*}} {
 ! CHECK:       omp.parallel {
-! CHECK:         %[[ALLOCA:.*]] = fir.alloca i32 {{{.*}}, pinned}
+! CHECK:         %[[ALLOCA:.*]] = fir.alloca i32 {{{.*}}, pinned, uniq_name = "_QFss4Ej"}
 ! CHECK:         %[[OMP_LOOP_J_DECL:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFss4Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:         omp.wsloop {
 ! CHECK-NEXT:      omp.loop_nest (%[[ARG:.*]]) : {{.*}} {
diff --git a/flang/test/Lower/OpenMP/wsloop-collapse.f90 b/flang/test/Lower/OpenMP/wsloop-collapse.f90
index d9541e176f6a..67351ca275ef 100644
--- a/flang/test/Lower/OpenMP/wsloop-collapse.f90
+++ b/flang/test/Lower/OpenMP/wsloop-collapse.f90
@@ -4,40 +4,56 @@
 
 !CHECK-LABEL:   func.func @_QQmain() attributes {fir.bindc_name = "wsloop_collapse"} {
 program wsloop_collapse
-!CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-!CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEk"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-!CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:           %[[VAL_4:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
-!CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:           %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFEa"}
 !CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:           %[[VAL_8:.*]] = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFEb"}
 !CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:           %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "c", uniq_name = "_QFEc"}
 !CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFEc"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+
 !CHECK:           %[[VAL_12:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFEi"}
 !CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:           %[[VAL_14:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFEj"}
 !CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:           %[[VAL_16:.*]] = fir.alloca i32 {bindc_name = "k", uniq_name = "_QFEk"}
 !CHECK:           %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFEk"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:           %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
 !CHECK:           %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-  integer :: i, j, k
-  integer :: a, b, c
-  integer :: x
+
 !CHECK:           %[[VAL_20:.*]] = arith.constant 3 : i32
 !CHECK:           hlfir.assign %[[VAL_20]] to %[[VAL_7]]#0 : i32, !fir.ref<i32>
-  a=3
+
 !CHECK:           %[[VAL_21:.*]] = arith.constant 2 : i32
 !CHECK:           hlfir.assign %[[VAL_21]] to %[[VAL_9]]#0 : i32, !fir.ref<i32>
-  b=2
+
 !CHECK:           %[[VAL_22:.*]] = arith.constant 5 : i32
 !CHECK:           hlfir.assign %[[VAL_22]] to %[[VAL_11]]#0 : i32, !fir.ref<i32>
-  c=5
+
 !CHECK:           %[[VAL_23:.*]] = arith.constant 0 : i32
 !CHECK:           hlfir.assign %[[VAL_23]] to %[[VAL_19]]#0 : i32, !fir.ref<i32>
+
+!CHECK:           %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
+!CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+!CHECK:           %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "j", pinned, {{.*}}}
+!CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+!CHECK:           %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "k", pinned, {{.*}}}
+!CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEk"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+  integer :: i, j, k
+  integer :: a, b, c
+  integer :: x
+
+  a=3
+  b=2
+  c=5
   x=0
 
 !CHECK:           %[[VAL_24:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-monotonic.f90 b/flang/test/Lower/OpenMP/wsloop-monotonic.f90
index 531d995052f6..2a5cc2321c64 100644
--- a/flang/test/Lower/OpenMP/wsloop-monotonic.f90
+++ b/flang/test/Lower/OpenMP/wsloop-monotonic.f90
@@ -11,7 +11,7 @@ program wsloop_dynamic
 !CHECK:  omp.parallel {
 
 !$OMP DO SCHEDULE(monotonic:dynamic)
-!CHECK:     %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned}
+!CHECK:     %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
 !CHECK:     %[[WS_LB:.*]] = arith.constant 1 : i32
 !CHECK:     %[[WS_UB:.*]] = arith.constant 9 : i32
 !CHECK:     %[[WS_STEP:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90 b/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90
index 420bc0bffaec..1e71807ae5b1 100644
--- a/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90
+++ b/flang/test/Lower/OpenMP/wsloop-nonmonotonic.f90
@@ -12,7 +12,7 @@ program wsloop_dynamic
 !CHECK:  omp.parallel {
 
 !$OMP DO SCHEDULE(nonmonotonic:dynamic)
-!CHECK:     %[[I_REF:.*]] = fir.alloca i32 {{{.*}}, pinned}
+!CHECK:     %[[I_REF:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
 !CHECK:     %[[ALLOCA_IV:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:     %[[WS_LB:.*]] = arith.constant 1 : i32
 !CHECK:     %[[WS_UB:.*]] = arith.constant 9 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
index cc4d055b3ea1..8dc2b43ad56a 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90
@@ -77,7 +77,7 @@
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
@@ -120,7 +120,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
@@ -164,7 +164,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
@@ -206,7 +206,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
@@ -257,7 +257,7 @@ end subroutine
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 100 : i32
@@ -321,7 +321,7 @@ end subroutine
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 100 : i32
@@ -392,7 +392,7 @@ end subroutine
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 0.000000e+00 : f64
 ! CHECK:           hlfir.assign %[[VAL_13]] to %[[VAL_3]]#0 : f64, !fir.ref<f64>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90
index ec9abba41db0..cef86d1c1bd4 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90
@@ -26,7 +26,7 @@
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir.f90
index c5cc5a95cef1..d0ba2cdff817 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir.f90
@@ -22,7 +22,7 @@
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add.f90
index 5b957959f40d..cd7c362e3c0d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-add.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-add.f90
@@ -53,7 +53,7 @@
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
@@ -96,7 +96,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
@@ -140,7 +140,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
@@ -182,7 +182,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 100 : i32
@@ -233,7 +233,7 @@ end subroutine
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 100 : i32
@@ -297,7 +297,7 @@ end subroutine
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 100 : i32
@@ -368,7 +368,7 @@ end subroutine
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 0.000000e+00 : f64
 ! CHECK:           hlfir.assign %[[VAL_13]] to %[[VAL_3]]#0 : f64, !fir.ref<f64>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
index ccf186cf9908..13858ff2f34e 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable.f90
@@ -75,7 +75,7 @@ end program
 ! CHECK:           %[[VAL_8:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_8]] to %[[VAL_5]]#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_9:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_9:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_9]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_11:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 10 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
index be1e3ec0cb9d..713cc2c0f02d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array-assumed-shape.f90
@@ -77,7 +77,7 @@ end program
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFFreduceEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = {{.*}}, uniq_name = "_QFFreduceEr"} : (!fir.box<!fir.array<?xf64>>, !fir.dscope) -> (!fir.box<!fir.array<?xf64>>, !fir.box<!fir.array<?xf64>>)
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_4:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFFreduceEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 10 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
index add422f5d956..18a76ed1d5a8 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90
@@ -71,7 +71,7 @@ end program
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
 ! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]](%[[VAL_4]]) {uniq_name = "_QFEr"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_6:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 10 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
index e988567fc337..9c2cb862ba6c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90
@@ -71,7 +71,7 @@ end program
 ! CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
 ! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]](%[[VAL_4]]) {uniq_name = "_QFEr"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_6:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_6:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 10 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
index bcc71415f1dd..8eb4f4c6eb4c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90
@@ -30,7 +30,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iandEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
index ae771c692b98..713e50f72e9d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
@@ -24,7 +24,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iandEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
index 6825800b2cd8..6a5d942cb74e 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90
@@ -26,7 +26,7 @@
 
 
 !CHECK: omp.parallel
-!CHECK: %[[I_REF:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+!CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 !CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFreduction_ieorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.wsloop reduction(byref @ieor_byref_i32 %[[X_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<i32>)
 !CHECK-NEXT: omp.loop_nest
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
index d50f6b854f48..958cc6f87e4f 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
@@ -17,7 +17,7 @@
 
 
 !CHECK: omp.parallel
-!CHECK: %[[I_REF:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+!CHECK: %[[I_REF:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 !CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFreduction_ieorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK: omp.wsloop reduction(@[[IEOR_DECLARE_I]] %[[X_DECL]]#0 -> %[[PRV:.+]] : !fir.ref<i32>)
 !CHECK-NEXT: omp.loop_nest
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
index 00f4552917aa..2956cd9ef53c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90
@@ -28,7 +28,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
index 182f1eaeeeb7..a80838047bac 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
@@ -24,7 +24,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_iorEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
index 331232a08e62..b505585e5cb0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
@@ -37,7 +37,7 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -87,7 +87,7 @@ end subroutine simple_reduction
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -146,7 +146,7 @@ end subroutine
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
index 078a463919e9..65781d6c3652 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
@@ -31,7 +31,7 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -81,7 +81,7 @@ end subroutine simple_reduction
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -140,7 +140,7 @@ end subroutine
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
index d1cb7b05f0af..a103bf58e16b 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
@@ -37,7 +37,7 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -86,7 +86,7 @@ end subroutine
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -145,7 +145,7 @@ end subroutine
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
index 8204e88815f3..c77a2bef0dd2 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
@@ -31,7 +31,7 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -80,7 +80,7 @@ end subroutine
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -139,7 +139,7 @@ end subroutine
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
index 64c056c9c6bd..8abc9b61c42e 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
@@ -37,7 +37,7 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -87,7 +87,7 @@ end subroutine
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -148,7 +148,7 @@ end subroutine
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
index 957de9b6741a..28e821bb41ec 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
@@ -31,7 +31,7 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -81,7 +81,7 @@ end subroutine
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -142,7 +142,7 @@ end subroutine
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
index 1e5f91de801c..e6def280cf70 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
@@ -36,7 +36,7 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -85,7 +85,7 @@ end subroutine
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -144,7 +144,7 @@ end subroutine
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
index d77566b109e5..2453efe59e25 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
@@ -31,7 +31,7 @@
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -80,7 +80,7 @@ end subroutine
 ! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_10:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
@@ -139,7 +139,7 @@ end subroutine
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i1) -> !fir.logical<4>
 ! CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_11]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_18:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
index 399afa106556..018fb28c6f68 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
@@ -41,7 +41,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
@@ -72,7 +72,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
@@ -93,7 +93,7 @@
 ! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_32:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_33:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90
index 2f3aee9f1300..130a580cd685 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90
@@ -28,7 +28,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90
index 71631fb14592..23e2ae98a027 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90
@@ -24,7 +24,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
index d4e827f3b7e2..9a93c75f5bd1 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
@@ -35,7 +35,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
@@ -66,7 +66,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
@@ -87,7 +87,7 @@
 ! CHECK:               omp.terminator
 ! CHECK:             omp.terminator
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_max_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_32:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_33:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
index ccab017a1b0c..754b3fd400d3 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
@@ -41,7 +41,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
@@ -72,7 +72,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
@@ -95,7 +95,7 @@
 ! CHECK:             omp.terminator
 ! CHECK:           }
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_32:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_33:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
index 80c056b5e8c5..50b2db9463d2 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
@@ -35,7 +35,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
@@ -66,7 +66,7 @@
 ! CHECK:           %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_4]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_7:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_9:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_10:.*]] = arith.constant 100 : i32
@@ -89,7 +89,7 @@
 ! CHECK:             omp.terminator
 ! CHECK:           }
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_30:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_31:.*]]:2 = hlfir.declare %[[VAL_30]] {uniq_name = "_QFreduction_min_realEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_32:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_33:.*]] = arith.constant 100 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
index a4c99f190dd2..834fb03c1d14 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90
@@ -34,7 +34,7 @@ end program
 ! CHECK:           %[[VAL_2:.*]] = fir.address_of(@_QFEr) : !fir.ref<i32>
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFEr"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_4:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_4:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_6:.*]] = arith.constant 0 : i32
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 10 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
index 762d9dd4f254..eddb9112d3b0 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90
@@ -80,7 +80,7 @@
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
@@ -118,7 +118,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
@@ -157,7 +157,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
@@ -195,7 +195,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
@@ -242,7 +242,7 @@ end subroutine
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
@@ -302,7 +302,7 @@ end subroutine
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
@@ -369,7 +369,7 @@ end subroutine
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 1.000000e+00 : f64
 ! CHECK:           hlfir.assign %[[VAL_13]] to %[[VAL_3]]#0 : f64, !fir.ref<f64>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 10 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90
index 08be4d84c1a6..573f1f431c18 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90
@@ -55,7 +55,7 @@
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
@@ -93,7 +93,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
@@ -132,7 +132,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_int_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
@@ -170,7 +170,7 @@ end subroutine
 ! CHECK:           %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_4]] to %[[VAL_3]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_5:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "_QFsimple_real_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_8:.*]] = arith.constant 10 : i32
@@ -217,7 +217,7 @@ end subroutine
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 1 : i32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : i32, !fir.ref<i32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_int_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
@@ -277,7 +277,7 @@ end subroutine
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 1.000000e+00 : f32
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_7]]#0 : f32, !fir.ref<f32>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFmultiple_real_reductions_same_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
@@ -344,7 +344,7 @@ end subroutine
 ! CHECK:           %[[VAL_13:.*]] = arith.constant 1.000000e+00 : f64
 ! CHECK:           hlfir.assign %[[VAL_13]] to %[[VAL_3]]#0 : f64, !fir.ref<f64>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_14:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFmultiple_reductions_different_typeEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_17:.*]] = arith.constant 10 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
index ed462b58bbf8..3551b90474aa 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-multiple-clauses.f90
@@ -107,7 +107,7 @@ endprogram
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f64
 ! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_4]]#0 : f64, !fir.ref<!fir.array<3x3xf64>>
 ! CHECK:           omp.parallel {
-! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:             %[[VAL_11:.*]] = fir.alloca i32 {bindc_name = "i", pinned, {{.*}}}
 ! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 10 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-variable.f90 b/flang/test/Lower/OpenMP/wsloop-variable.f90
index 4d83b3328803..dc2acf881f48 100644
--- a/flang/test/Lower/OpenMP/wsloop-variable.f90
+++ b/flang/test/Lower/OpenMP/wsloop-variable.f90
@@ -91,26 +91,30 @@ program wsloop_variable
 end program wsloop_variable
 
 !CHECK-LABEL: func.func @_QPwsloop_variable_sub() {
-!CHECK:           %[[VAL_0:.*]] = fir.alloca i8 {adapt.valuebyref, pinned}
-!CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFwsloop_variable_subEi1"} : (!fir.ref<i8>) -> (!fir.ref<i8>, !fir.ref<i8>)
-!CHECK:           %[[VAL_2:.*]] = fir.alloca i16 {adapt.valuebyref, pinned}
-!CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFwsloop_variable_subEi2"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
 !CHECK:           %[[VAL_4:.*]] = fir.alloca i8 {bindc_name = "i1", uniq_name = "_QFwsloop_variable_subEi1"}
 !CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {uniq_name = "_QFwsloop_variable_subEi1"} : (!fir.ref<i8>) -> (!fir.ref<i8>, !fir.ref<i8>)
+
 !CHECK:           %[[VAL_6:.*]] = fir.alloca i128 {bindc_name = "i16_lb", uniq_name = "_QFwsloop_variable_subEi16_lb"}
 !CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFwsloop_variable_subEi16_lb"} : (!fir.ref<i128>) -> (!fir.ref<i128>, !fir.ref<i128>)
+
 !CHECK:           %[[VAL_8:.*]] = fir.alloca i8 {bindc_name = "i1_ub", uniq_name = "_QFwsloop_variable_subEi1_ub"}
 !CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFwsloop_variable_subEi1_ub"} : (!fir.ref<i8>) -> (!fir.ref<i8>, !fir.ref<i8>)
+
 !CHECK:           %[[VAL_10:.*]] = fir.alloca i16 {bindc_name = "i2", uniq_name = "_QFwsloop_variable_subEi2"}
 !CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFwsloop_variable_subEi2"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
+
 !CHECK:           %[[VAL_12:.*]] = fir.alloca i16 {bindc_name = "i2_s", uniq_name = "_QFwsloop_variable_subEi2_s"}
 !CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFwsloop_variable_subEi2_s"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
+
 !CHECK:           %[[VAL_14:.*]] = fir.alloca i32 {bindc_name = "i4_s", uniq_name = "_QFwsloop_variable_subEi4_s"}
 !CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] {uniq_name = "_QFwsloop_variable_subEi4_s"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
 !CHECK:           %[[VAL_16:.*]] = fir.alloca i64 {bindc_name = "i8", uniq_name = "_QFwsloop_variable_subEi8"}
 !CHECK:           %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFwsloop_variable_subEi8"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
+
 !CHECK:           %[[VAL_18:.*]] = fir.alloca i8 {bindc_name = "j1", uniq_name = "_QFwsloop_variable_subEj1"}
 !CHECK:           %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFwsloop_variable_subEj1"} : (!fir.ref<i8>) -> (!fir.ref<i8>, !fir.ref<i8>)
+
 !CHECK:           %[[VAL_20:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFwsloop_variable_subEx"}
 !CHECK:           %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFwsloop_variable_subEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 
@@ -122,6 +126,9 @@ subroutine wsloop_variable_sub
   integer(kind=16) :: i16_lb
   real :: x
 
+!CHECK:           %[[VAL_2:.*]] = fir.alloca i16 {bindc_name = "i2", pinned, {{.*}}}
+!CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFwsloop_variable_subEi2"} : (!fir.ref<i16>) -> (!fir.ref<i16>, !fir.ref<i16>)
+
 !CHECK:           %[[VAL_22:.*]] = arith.constant 1 : i32
 !CHECK:           %[[VAL_23:.*]] = fir.load %[[VAL_9]]#0 : !fir.ref<i8>
 !CHECK:           %[[VAL_24:.*]] = fir.load %[[VAL_13]]#0 : !fir.ref<i16>
@@ -166,9 +173,12 @@ subroutine wsloop_variable_sub
   end do
   !$omp end do
 
-
 !CHECK:           %[[VAL_49:.*]] = arith.constant 5 : i8
 !CHECK:           hlfir.assign %[[VAL_49]] to %[[VAL_19]]#0 : i8, !fir.ref<i8>
+
+!CHECK:           %[[VAL_0:.*]] = fir.alloca i8 {bindc_name = "i1", pinned, {{.*}}}
+!CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFwsloop_variable_subEi1"} : (!fir.ref<i8>) -> (!fir.ref<i8>, !fir.ref<i8>)
+
 !CHECK:           %[[VAL_50:.*]] = arith.constant 1 : i32
 !CHECK:           %[[VAL_51:.*]] = arith.constant 10 : i32
 !CHECK:           %[[VAL_52:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/OpenMP/wsloop.f90 b/flang/test/Lower/OpenMP/wsloop.f90
index da90cb724159..6536f5b83d2c 100644
--- a/flang/test/Lower/OpenMP/wsloop.f90
+++ b/flang/test/Lower/OpenMP/wsloop.f90
@@ -7,7 +7,7 @@ subroutine simple_loop
   integer :: i
   ! CHECK:  omp.parallel
   !$OMP PARALLEL
-  ! CHECK:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned}
+  ! CHECK:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
   ! CHECK:      %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loopEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! CHECK:      %[[WS_UB:.*]] = arith.constant 9 : i32
@@ -33,7 +33,7 @@ subroutine simple_loop_with_step
   integer :: i
   ! CHECK:  omp.parallel
   !$OMP PARALLEL
-  ! CHECK:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned}
+  ! CHECK:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
   ! CHECK:      %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loop_with_stepEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! CHECK:      %[[WS_UB:.*]] = arith.constant 9 : i32
@@ -59,7 +59,7 @@ subroutine loop_with_schedule_nowait
   integer :: i
   ! CHECK:  omp.parallel
   !$OMP PARALLEL
-  ! CHECK:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned}
+  ! CHECK:      %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned, {{.*}}}
   ! CHECK:      %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFloop_with_schedule_nowaitEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
   ! CHECK:      %[[WS_LB:.*]] = arith.constant 1 : i32
   ! CHECK:      %[[WS_UB:.*]] = arith.constant 9 : i32
-- 
GitLab


From 7b4dfec8936f54145297fd2f14bf95a2b8c95f5f Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 17 May 2024 23:49:01 -0700
Subject: [PATCH 1189/1206] [MCAsmParser] Improve .rept/.irp tests

---
 llvm/test/MC/AsmParser/macro-irp.s       | 47 ++++++++++++++++--------
 llvm/test/MC/AsmParser/macro-rept-err1.s |  6 ---
 llvm/test/MC/AsmParser/macro-rept-err2.s |  7 ----
 llvm/test/MC/AsmParser/macro-rept.s      | 22 ++++++++++-
 4 files changed, 53 insertions(+), 29 deletions(-)
 delete mode 100644 llvm/test/MC/AsmParser/macro-rept-err1.s
 delete mode 100644 llvm/test/MC/AsmParser/macro-rept-err2.s

diff --git a/llvm/test/MC/AsmParser/macro-irp.s b/llvm/test/MC/AsmParser/macro-irp.s
index 7d4c7cc26942..d2ddd2a24a0b 100644
--- a/llvm/test/MC/AsmParser/macro-irp.s
+++ b/llvm/test/MC/AsmParser/macro-irp.s
@@ -1,23 +1,40 @@
-// RUN: llvm-mc -triple i386-unknown-unknown %s | FileCheck %s
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -triple=x86_64 a.s | FileCheck %s
 
-.irp reg,%eax,%ebx
-        pushl \reg
+#--- a.s
+# CHECK:      pushq %rax
+# CHECK-NEXT: pushq %rbx
+# CHECK-NEXT: pushq %rcx
+.irp reg,%rax,%rbx
+        pushq \reg
 .endr
-pushl %ecx
-
-// CHECK: pushl %eax
-// CHECK: pushl %ebx
-// CHECK-NEXT: pushl %ecx
+pushq %rcx
 
+# CHECK:      addl %eax, 4
+# CHECK-NEXT: addl %eax, 3
+# CHECK-NEXT: addl %eax, 5
+# CHECK-NEXT: addl %ebx, 4
+# CHECK-NEXT: addl %ebx, 3
+# CHECK-NEXT: addl %ebx, 5
+# CHECK-EMPTY:
+# CHECK-NEXT: nop
 .irp reg,%eax,%ebx
 .irp imm,4,3,5
         addl \reg, \imm
+.endr # comment after .endr
+.endr ;
+nop
+
+# CHECK:      xorl %eax, %eax
+# CHECK-EMPTY:
+# CHECK-NEXT: nop
+.irp reg,%eax
+xor \reg,\reg
 .endr
-.endr
+# 99 "a.s"
+nop
 
-// CHECK: addl %eax, 4
-// CHECK: addl %eax, 3
-// CHECK: addl %eax, 5
-// CHECK: addl %ebx, 4
-// CHECK: addl %ebx, 3
-// CHECK: addl %ebx, 5
+# RUN: not llvm-mc -triple=x86_64 err1.s 2>&1 | FileCheck %s --check-prefix=ERR1
+# ERR1: .s:1:1: error: no matching '.endr' in definition
+#--- err1.s
+.irp reg,%eax
diff --git a/llvm/test/MC/AsmParser/macro-rept-err1.s b/llvm/test/MC/AsmParser/macro-rept-err1.s
deleted file mode 100644
index cfa66878d979..000000000000
--- a/llvm/test/MC/AsmParser/macro-rept-err1.s
+++ /dev/null
@@ -1,6 +0,0 @@
-// RUN: not llvm-mc -triple x86_64-unknown-unknown %s 2> %t
-// RUN: FileCheck < %t %s
-
-.endr
-
-// CHECK: unmatched '.endr' directive
diff --git a/llvm/test/MC/AsmParser/macro-rept-err2.s b/llvm/test/MC/AsmParser/macro-rept-err2.s
deleted file mode 100644
index 678b4c7eeee0..000000000000
--- a/llvm/test/MC/AsmParser/macro-rept-err2.s
+++ /dev/null
@@ -1,7 +0,0 @@
-// RUN: not llvm-mc -triple x86_64-unknown-unknown %s 2> %t
-// RUN: FileCheck < %t %s
-
-.rept 3
-.long
-
-// CHECK: no matching '.endr' in definition
diff --git a/llvm/test/MC/AsmParser/macro-rept.s b/llvm/test/MC/AsmParser/macro-rept.s
index c1a90b22c9f2..5f4b607a2d55 100644
--- a/llvm/test/MC/AsmParser/macro-rept.s
+++ b/llvm/test/MC/AsmParser/macro-rept.s
@@ -1,5 +1,7 @@
-# RUN: llvm-mc -triple x86_64 %s | FileCheck %s
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -triple=x86_64 a.s | FileCheck %s
 
+#--- a.s
 .rept 2
     .long 1
 .endr
@@ -23,3 +25,21 @@
 # CHECK-NEXT: .long 0
 # CHECK-NEXT: .long 0
 # CHECK-NEXT: .long 42
+
+# RUN: not llvm-mc -triple=x86_64 err1.s 2>&1 | FileCheck %s --check-prefix=ERR1
+# ERR1: .s:1:6: error: unmatched '.endr' directive
+#--- err1.s
+.endr
+
+# RUN: not llvm-mc -triple=x86_64 err2.s 2>&1 | FileCheck %s --check-prefix=ERR2
+# ERR2: .s:1:1: error: no matching '.endr' in definition
+#--- err2.s
+.rept 3
+.long
+
+# RUN: not llvm-mc -triple=x86_64 err3.s 2>&1 | FileCheck %s --check-prefix=ERR3
+# ERR3: .s:3:7: error: expected newline
+#--- err3.s
+.rept 1
+.long 0
+.endr ab
-- 
GitLab


From 2c2e0507e92bdb77a01828f899ff59e44492b537 Mon Sep 17 00:00:00 2001
From: Antonio Frighetto <me@antoniofrighetto.com>
Date: Wed, 15 May 2024 17:03:02 +0200
Subject: [PATCH 1190/1206] [clang][ThreadSafety] Skip past implicit cast in
 `translateAttrExpr`

Ignore `ImplicitCastExpr` when building `AttrExp` for capability
attribute diagnostics.

Fixes: https://github.com/llvm/llvm-project/issues/92118.
---
 clang/lib/Analysis/ThreadSafetyCommon.cpp          | 2 +-
 clang/test/SemaCXX/warn-thread-safety-analysis.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Analysis/ThreadSafetyCommon.cpp b/clang/lib/Analysis/ThreadSafetyCommon.cpp
index a3b378c42df3..3e8c959ccee4 100644
--- a/clang/lib/Analysis/ThreadSafetyCommon.cpp
+++ b/clang/lib/Analysis/ThreadSafetyCommon.cpp
@@ -197,7 +197,7 @@ CapabilityExpr SExprBuilder::translateAttrExpr(const Expr *AttrExp,
   else if (const auto *UO = dyn_cast<UnaryOperator>(AttrExp)) {
     if (UO->getOpcode() == UO_LNot) {
       Neg = true;
-      AttrExp = UO->getSubExpr();
+      AttrExp = UO->getSubExpr()->IgnoreImplicit();
     }
   }
 
diff --git a/clang/test/SemaCXX/warn-thread-safety-analysis.cpp b/clang/test/SemaCXX/warn-thread-safety-analysis.cpp
index dfb966d3b590..749d9e135d94 100644
--- a/clang/test/SemaCXX/warn-thread-safety-analysis.cpp
+++ b/clang/test/SemaCXX/warn-thread-safety-analysis.cpp
@@ -5341,7 +5341,7 @@ void dispatch_log(const char *msg) __attribute__((requires_capability(!FlightCon
 void dispatch_log2(const char *msg) __attribute__((requires_capability(Logger))) {}
 
 void flight_control_entry(void) __attribute__((requires_capability(FlightControl))) {
-  dispatch_log("wrong"); /* expected-warning {{cannot call function 'dispatch_log' while mutex 'FlightControl' is held}} */
+  dispatch_log("wrong"); /* expected-warning {{cannot call function 'dispatch_log' while role 'FlightControl' is held}} */
   dispatch_log2("also wrong"); /* expected-warning {{calling function 'dispatch_log2' requires holding role 'Logger' exclusively}} */
 }
 
-- 
GitLab


From f7b0b99c52ee36ed6ca8abcce74a752e483768d6 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Sat, 18 May 2024 12:10:06 +0300
Subject: [PATCH 1191/1206] [clang][NFC] Further improvements to
 const-correctness

---
 clang/include/clang/Serialization/ASTWriter.h |  3 ++-
 clang/lib/Basic/SourceManager.cpp             |  6 ++---
 clang/lib/Frontend/CompilerInstance.cpp       |  3 +--
 clang/lib/Serialization/ASTReader.cpp         |  6 ++---
 clang/lib/Serialization/ASTWriter.cpp         | 23 +++++++++----------
 clang/lib/Serialization/ASTWriterStmt.cpp     |  2 +-
 6 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index 7bb0e81545bd..6aa2796a41e0 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -565,7 +565,8 @@ private:
 
   void GenerateNameLookupTable(const DeclContext *DC,
                                llvm::SmallVectorImpl<char> &LookupTable);
-  uint64_t WriteDeclContextLexicalBlock(ASTContext &Context, DeclContext *DC);
+  uint64_t WriteDeclContextLexicalBlock(ASTContext &Context,
+                                        const DeclContext *DC);
   uint64_t WriteDeclContextVisibleBlock(ASTContext &Context, DeclContext *DC);
   void WriteTypeDeclOffsets();
   void WriteFileDeclIDsMap();
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index afdb5b0e92d6..753601e01b5c 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -276,14 +276,14 @@ void SourceManager::AddLineNote(SourceLocation Loc, unsigned LineNo,
   std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
 
   bool Invalid = false;
-  const SLocEntry &Entry = getSLocEntry(LocInfo.first, &Invalid);
+  SLocEntry &Entry = getSLocEntry(LocInfo.first, &Invalid);
   if (!Entry.isFile() || Invalid)
     return;
 
-  const SrcMgr::FileInfo &FileInfo = Entry.getFile();
+  SrcMgr::FileInfo &FileInfo = Entry.getFile();
 
   // Remember that this file has #line directives now if it doesn't already.
-  const_cast<SrcMgr::FileInfo&>(FileInfo).setHasLineDirectives();
+  FileInfo.setHasLineDirectives();
 
   (void) getLineTable();
 
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 66a45b888f15..6242b5a7d9fe 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -411,8 +411,7 @@ static void InitializeFileRemapping(DiagnosticsEngine &Diags,
       SourceMgr.overrideFileContents(FromFile, RB.second->getMemBufferRef());
     else
       SourceMgr.overrideFileContents(
-          FromFile, std::unique_ptr<llvm::MemoryBuffer>(
-                        const_cast<llvm::MemoryBuffer *>(RB.second)));
+          FromFile, std::unique_ptr<llvm::MemoryBuffer>(RB.second));
   }
 
   // Remap files in the source manager (with other files).
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 510f61d9cccd..f50f9569c0a5 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1649,8 +1649,7 @@ bool ASTReader::ReadSLocEntry(int ID) {
       FileCharacter = (SrcMgr::CharacteristicKind)Record[2];
     FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, ID,
                                         BaseOffset + Record[0]);
-    SrcMgr::FileInfo &FileInfo =
-          const_cast<SrcMgr::FileInfo&>(SourceMgr.getSLocEntry(FID).getFile());
+    SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
     FileInfo.NumCreatedFIDs = Record[5];
     if (Record[3])
       FileInfo.setHasLineDirectives();
@@ -1693,8 +1692,7 @@ bool ASTReader::ReadSLocEntry(int ID) {
     FileID FID = SourceMgr.createFileID(std::move(Buffer), FileCharacter, ID,
                                         BaseOffset + Offset, IncludeLoc);
     if (Record[3]) {
-      auto &FileInfo =
-          const_cast<SrcMgr::FileInfo &>(SourceMgr.getSLocEntry(FID).getFile());
+      auto &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
       FileInfo.setHasLineDirectives();
     }
     break;
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index dd5d65b2eb20..2a107e4c56a3 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -3304,7 +3304,7 @@ static bool IsInternalDeclFromFileContext(const Decl *D) {
 /// \returns the offset of the DECL_CONTEXT_LEXICAL block within the
 /// bitstream, or 0 if no block was written.
 uint64_t ASTWriter::WriteDeclContextLexicalBlock(ASTContext &Context,
-                                                 DeclContext *DC) {
+                                                 const DeclContext *DC) {
   if (DC->decls_empty())
     return 0;
 
@@ -5698,8 +5698,8 @@ void ASTWriter::WriteDeclUpdatesBlocks(RecordDataImpl &OffsetsRecord) {
         break;
 
       case UPD_CXX_INSTANTIATED_DEFAULT_ARGUMENT:
-        Record.AddStmt(const_cast<Expr *>(
-            cast<ParmVarDecl>(Update.getDecl())->getDefaultArg()));
+        Record.writeStmtRef(
+            cast<ParmVarDecl>(Update.getDecl())->getDefaultArg());
         break;
 
       case UPD_CXX_INSTANTIATED_DEFAULT_MEMBER_INITIALIZER:
@@ -5713,8 +5713,7 @@ void ASTWriter::WriteDeclUpdatesBlocks(RecordDataImpl &OffsetsRecord) {
         Record.push_back(RD->isParamDestroyedInCallee());
         Record.push_back(llvm::to_underlying(RD->getArgPassingRestrictions()));
         Record.AddCXXDefinitionData(RD);
-        Record.AddOffset(WriteDeclContextLexicalBlock(
-            *Context, const_cast<CXXRecordDecl *>(RD)));
+        Record.AddOffset(WriteDeclContextLexicalBlock(*Context, RD));
 
         // This state is sometimes updated by template instantiation, when we
         // switch from the specialization referring to the template declaration
@@ -6357,7 +6356,7 @@ void ASTRecordWriter::AddTemplateParameterList(
     AddDeclRef(P);
   if (const Expr *RequiresClause = TemplateParams->getRequiresClause()) {
     Record->push_back(true);
-    AddStmt(const_cast<Expr*>(RequiresClause));
+    writeStmtRef(RequiresClause);
   } else {
     Record->push_back(false);
   }
@@ -7814,7 +7813,7 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   case OpenACCClauseKind::If: {
     const auto *IC = cast<OpenACCIfClause>(C);
     writeSourceLocation(IC->getLParenLoc());
-    AddStmt(const_cast<Expr *>(IC->getConditionExpr()));
+    writeStmtRef(IC->getConditionExpr());
     return;
   }
   case OpenACCClauseKind::Self: {
@@ -7822,7 +7821,7 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeSourceLocation(SC->getLParenLoc());
     writeBool(SC->hasConditionExpr());
     if (SC->hasConditionExpr())
-      AddStmt(const_cast<Expr *>(SC->getConditionExpr()));
+      writeStmtRef(SC->getConditionExpr());
     return;
   }
   case OpenACCClauseKind::NumGangs: {
@@ -7836,13 +7835,13 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   case OpenACCClauseKind::NumWorkers: {
     const auto *NWC = cast<OpenACCNumWorkersClause>(C);
     writeSourceLocation(NWC->getLParenLoc());
-    AddStmt(const_cast<Expr *>(NWC->getIntExpr()));
+    writeStmtRef(NWC->getIntExpr());
     return;
   }
   case OpenACCClauseKind::VectorLength: {
     const auto *NWC = cast<OpenACCVectorLengthClause>(C);
     writeSourceLocation(NWC->getLParenLoc());
-    AddStmt(const_cast<Expr *>(NWC->getIntExpr()));
+    writeStmtRef(NWC->getIntExpr());
     return;
   }
   case OpenACCClauseKind::Private: {
@@ -7921,7 +7920,7 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeSourceLocation(AC->getLParenLoc());
     writeBool(AC->hasIntExpr());
     if (AC->hasIntExpr())
-      AddStmt(const_cast<Expr *>(AC->getIntExpr()));
+      writeStmtRef(AC->getIntExpr());
     return;
   }
   case OpenACCClauseKind::Wait: {
@@ -7929,7 +7928,7 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeSourceLocation(WC->getLParenLoc());
     writeBool(WC->getDevNumExpr());
     if (const Expr *DNE = WC->getDevNumExpr())
-      AddStmt(const_cast<Expr *>(DNE));
+      writeStmtRef(DNE);
     writeSourceLocation(WC->getQueuesLoc());
 
     writeOpenACCIntExprList(WC->getQueueIdExprs());
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 39aec31b6d87..6f7c368ce9ca 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -473,7 +473,7 @@ addConstraintSatisfaction(ASTRecordWriter &Record,
   if (!Satisfaction.IsSatisfied) {
     Record.push_back(Satisfaction.NumRecords);
     for (const auto &DetailRecord : Satisfaction) {
-      Record.AddStmt(const_cast<Expr *>(DetailRecord.first));
+      Record.writeStmtRef(DetailRecord.first);
       auto *E = DetailRecord.second.dyn_cast<Expr *>();
       Record.push_back(E == nullptr);
       if (E)
-- 
GitLab


From 778826f0b8fb9f1a595ce8ab23faad6daa9466f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Sat, 18 May 2024 13:43:10 +0200
Subject: [PATCH 1192/1206] [GlobalIsel] Combine select to integer min max more
 (#92570)

---
 llvm/include/llvm/Target/GlobalISel/Combine.td      |  2 +-
 llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp      | 13 ++++++++++---
 .../CodeGen/AArch64/GlobalISel/combine-select.mir   |  3 +--
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index a1d39d910bc8..5d4b5a2479f6 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1309,7 +1309,7 @@ def select_to_minmax: GICombineRule<
 
 def select_to_iminmax: GICombineRule<
   (defs root:$root, build_fn_matchinfo:$info),
-  (match (G_ICMP $tst, $tst1, $x, $y),
+  (match (G_ICMP $tst, $tst1, $a, $b),
          (G_SELECT $root, $tst, $x, $y),
          [{ return Helper.matchSelectIMinMax(${root}, ${info}); }]),
   (apply [{ Helper.applyBuildFnMO(${root}, ${info}); }])>;
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index a87a26d72b45..22eb4a3e0d7c 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -6772,13 +6772,20 @@ bool CombinerHelper::matchSelectIMinMax(const MachineOperand &MO,
   if (CmpInst::isEquality(Pred))
     return false;
 
-  [[maybe_unused]] Register CmpLHS = Cmp->getLHSReg();
-  [[maybe_unused]] Register CmpRHS = Cmp->getRHSReg();
+  Register CmpLHS = Cmp->getLHSReg();
+  Register CmpRHS = Cmp->getRHSReg();
+
+  // We can swap CmpLHS and CmpRHS for higher hitrate.
+  if (True == CmpRHS && False == CmpLHS) {
+    std::swap(CmpLHS, CmpRHS);
+    Pred = CmpInst::getSwappedPredicate(Pred);
+  }
 
   // (icmp X, Y) ? X : Y -> integer minmax.
   // see matchSelectPattern in ValueTracking.
   // Legality between G_SELECT and integer minmax can differ.
-  assert(True == CmpLHS && False == CmpRHS && "unexpected MIR pattern");
+  if (True != CmpLHS || False != CmpRHS)
+    return false;
 
   switch (Pred) {
   case ICmpInst::ICMP_UGT:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
index 8d9ad8d7caca..353c1550d697 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
@@ -911,8 +911,7 @@ body:             |
     ; CHECK-NEXT: %f1:_(s32) = G_TRUNC [[COPY1]](s64)
     ; CHECK-NEXT: %t:_(<4 x s32>) = G_BUILD_VECTOR %t1(s32), %t1(s32), %t1(s32), %t1(s32)
     ; CHECK-NEXT: %f:_(<4 x s32>) = G_BUILD_VECTOR %f1(s32), %f1(s32), %f1(s32), %f1(s32)
-    ; CHECK-NEXT: %c:_(<4 x s32>) = G_ICMP intpred(sle), %f(<4 x s32>), %t
-    ; CHECK-NEXT: %sel:_(<4 x s32>) = exact G_SELECT %c(<4 x s32>), %t, %f
+    ; CHECK-NEXT: %sel:_(<4 x s32>) = G_SMAX %t, %f
     ; CHECK-NEXT: $q0 = COPY %sel(<4 x s32>)
     %0:_(s64) = COPY $x0
     %1:_(s64) = COPY $x1
-- 
GitLab


From 4b62afca64a3f18cd57a3b3247330effd4b38098 Mon Sep 17 00:00:00 2001
From: Shengchen Kan <shengchen.kan@intel.com>
Date: Sat, 18 May 2024 19:50:16 +0800
Subject: [PATCH 1193/1206] [X86][CodeGen] Support flags copy lowering for
 CCMP/CTEST (#91849)

```
%1:gr64 = COPY $eflags
OP1 may update eflags
$eflags = COPY %1
OP2 may use eflags
```

To use eflags as input at 4th instruction, we need to use SETcc to
preserve the eflags before 2, and update the source condition of OP2
according to value in GPR %1.

In this patch, we support CCMP/CTEST as OP2.
---
 llvm/lib/Target/X86/X86FlagsCopyLowering.cpp  | 33 ++++++++++++
 llvm/lib/Target/X86/X86InstrInfo.cpp          |  9 +++-
 llvm/lib/Target/X86/X86InstrInfo.h            |  3 ++
 .../X86/apx/ccmp-flags-copy-lowering.mir      | 51 +++++++++++++++++++
 4 files changed, 95 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/X86/apx/ccmp-flags-copy-lowering.mir

diff --git a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp
index 78355d355083..ea5ef5b5a602 100644
--- a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp
+++ b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp
@@ -124,6 +124,10 @@ private:
                     MachineBasicBlock::iterator TestPos,
                     const DebugLoc &TestLoc, MachineInstr &SetCCI,
                     MachineOperand &FlagUse, CondRegArray &CondRegs);
+  void rewriteCCMP(MachineBasicBlock &TestMBB,
+                   MachineBasicBlock::iterator TestPos, const DebugLoc &TestLoc,
+                   MachineInstr &CMovI, MachineOperand &FlagUse,
+                   CondRegArray &CondRegs);
 };
 
 } // end anonymous namespace
@@ -613,6 +617,9 @@ bool X86FlagsCopyLoweringPass::runOnMachineFunction(MachineFunction &MF) {
           rewriteFCMov(*TestMBB, TestPos, TestLoc, MI, *FlagUse, CondRegs);
         } else if (X86::getCondFromSETCC(MI) != X86::COND_INVALID) {
           rewriteSetCC(*TestMBB, TestPos, TestLoc, MI, *FlagUse, CondRegs);
+        } else if (X86::getCondFromCCMP(MI) != X86::COND_INVALID) {
+          rewriteCCMP(*TestMBB, TestPos, TestLoc, MI, *FlagUse, CondRegs);
+          FlagsKilled = true;
         } else if (MI.getOpcode() == TargetOpcode::COPY) {
           rewriteCopy(MI, *FlagUse, CopyDefI);
         } else {
@@ -970,3 +977,29 @@ void X86FlagsCopyLoweringPass::rewriteSetCC(MachineBasicBlock &TestMBB,
 
   SetCCI.eraseFromParent();
 }
+
+void X86FlagsCopyLoweringPass::rewriteCCMP(MachineBasicBlock &TestMBB,
+                                           MachineBasicBlock::iterator TestPos,
+                                           const DebugLoc &TestLoc,
+                                           MachineInstr &CCMPI,
+                                           MachineOperand &FlagUse,
+                                           CondRegArray &CondRegs) {
+  // First get the register containing this specific condition.
+  X86::CondCode Cond = X86::getCondFromCCMP(CCMPI);
+  unsigned CondReg;
+  bool Inverted;
+  std::tie(CondReg, Inverted) =
+      getCondOrInverseInReg(TestMBB, TestPos, TestLoc, Cond, CondRegs);
+
+  MachineBasicBlock &MBB = *CCMPI.getParent();
+
+  // Insert a direct test of the saved register.
+  insertTest(MBB, CCMPI.getIterator(), CCMPI.getDebugLoc(), CondReg);
+
+  // Rewrite the CCMP/CTEST to use the !ZF flag from the test, and then kill its
+  // use of the flags afterward.
+  CCMPI.getOperand(CCMPI.getDesc().getNumOperands() - 1)
+      .setImm(Inverted ? X86::COND_E : X86::COND_NE);
+  FlagUse.setIsKill(true);
+  LLVM_DEBUG(dbgs() << "    fixed ccmp/ctest: "; CCMPI.dump());
+}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 0e5e52d4d88e..26c68ce3c1a2 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -3121,7 +3121,8 @@ bool X86InstrInfo::hasCommutePreference(MachineInstr &MI, bool &Commute) const {
 int X86::getCondSrcNoFromDesc(const MCInstrDesc &MCID) {
   unsigned Opcode = MCID.getOpcode();
   if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isCMOVCC(Opcode) ||
-        X86::isCFCMOVCC(Opcode)))
+        X86::isCFCMOVCC(Opcode) || X86::isCCMPCC(Opcode) ||
+        X86::isCTESTCC(Opcode)))
     return -1;
   // Assume that condition code is always the last use operand.
   unsigned NumUses = MCID.getNumOperands() - MCID.getNumDefs();
@@ -3157,6 +3158,12 @@ X86::CondCode X86::getCondFromCFCMov(const MachineInstr &MI) {
                                          : X86::COND_INVALID;
 }
 
+X86::CondCode X86::getCondFromCCMP(const MachineInstr &MI) {
+  return X86::isCCMPCC(MI.getOpcode()) || X86::isCTESTCC(MI.getOpcode())
+             ? X86::getCondFromMI(MI)
+             : X86::COND_INVALID;
+}
+
 /// Return the inverse of the specified condition,
 /// e.g. turning COND_E to COND_NE.
 X86::CondCode X86::GetOppositeBranchCondition(X86::CondCode CC) {
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index 5407ede69a91..55deca73b1f3 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -71,6 +71,9 @@ CondCode getCondFromCMov(const MachineInstr &MI);
 // Turn CFCMOV instruction into condition code.
 CondCode getCondFromCFCMov(const MachineInstr &MI);
 
+// Turn CCMP instruction into condition code.
+CondCode getCondFromCCMP(const MachineInstr &MI);
+
 /// GetOppositeBranchCondition - Return the inverse of the specified cond,
 /// e.g. turning COND_E to COND_NE.
 CondCode GetOppositeBranchCondition(CondCode CC);
diff --git a/llvm/test/CodeGen/X86/apx/ccmp-flags-copy-lowering.mir b/llvm/test/CodeGen/X86/apx/ccmp-flags-copy-lowering.mir
new file mode 100644
index 000000000000..52d4c4cfb2aa
--- /dev/null
+++ b/llvm/test/CodeGen/X86/apx/ccmp-flags-copy-lowering.mir
@@ -0,0 +1,51 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -mtriple=x86_64 -run-pass x86-flags-copy-lowering -verify-machineinstrs -o - %s | FileCheck %s
+# Lower various interesting copy patterns of EFLAGS without using LAHF/SAHF.
+
+...
+---
+name:            test_ccmp
+body:             |
+  bb.0:
+    liveins: $edi
+
+    ; CHECK-LABEL: name: test_ccmp
+    ; CHECK: liveins: $edi
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: MUL32r $edi, implicit-def $eax, implicit-def dead $edx, implicit-def $eflags, implicit $eax
+    ; CHECK-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 1, implicit $eflags
+    ; CHECK-NEXT: [[ADD32rr:%[0-9]+]]:gr32 = ADD32rr $edi, $edi, implicit-def $eflags
+    ; CHECK-NEXT: TEST8rr [[SETCCr]], [[SETCCr]], implicit-def $eflags
+    ; CHECK-NEXT: CCMP32rr [[ADD32rr]], [[ADD32rr]], 0, 5, implicit-def $eflags, implicit killed $eflags
+    ; CHECK-NEXT: RET 0, $al
+    MUL32r $edi, implicit-def $eax, implicit-def dead $edx, implicit-def $eflags, implicit $eax
+    %1:gr64 = COPY $eflags
+    %2:gr32 = ADD32rr $edi, $edi, implicit-def $eflags
+    $eflags = COPY %1
+    CCMP32rr %2, %2, 0, 1, implicit-def $eflags, implicit $eflags
+    RET 0, $al
+
+...
+---
+name:            test_ctest
+body:             |
+  bb.0:
+    liveins: $edi
+
+    ; CHECK-LABEL: name: test_ctest
+    ; CHECK: liveins: $edi
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: MUL32r $edi, implicit-def $eax, implicit-def dead $edx, implicit-def $eflags, implicit $eax
+    ; CHECK-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 1, implicit $eflags
+    ; CHECK-NEXT: [[ADD32rr:%[0-9]+]]:gr32 = ADD32rr $edi, $edi, implicit-def $eflags
+    ; CHECK-NEXT: TEST8rr [[SETCCr]], [[SETCCr]], implicit-def $eflags
+    ; CHECK-NEXT: CTEST32rr [[ADD32rr]], [[ADD32rr]], 0, 5, implicit-def $eflags, implicit killed $eflags
+    ; CHECK-NEXT: RET 0, $al
+    MUL32r $edi, implicit-def $eax, implicit-def dead $edx, implicit-def $eflags, implicit $eax
+    %1:gr64 = COPY $eflags
+    %2:gr32 = ADD32rr $edi, $edi, implicit-def $eflags
+    $eflags = COPY %1
+    CTEST32rr %2, %2, 0, 1, implicit-def $eflags, implicit $eflags
+    RET 0, $al
+
+...
-- 
GitLab


From 33550b43f49a4e5b20e748936278e55f67e4b929 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Sat, 18 May 2024 08:03:19 -0500
Subject: [PATCH 1194/1206] [mlir] Add operator<< for printing `Block` (#92550)

Turns out it was already in Analysis/CFGLoopInfo, so just move it
to IR/AsmPrinter.
---
 mlir/include/mlir/Analysis/CFGLoopInfo.h | 2 --
 mlir/include/mlir/IR/Block.h             | 3 +++
 mlir/lib/Analysis/CFGLoopInfo.cpp        | 5 -----
 mlir/lib/IR/AsmPrinter.cpp               | 5 +++++
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/mlir/include/mlir/Analysis/CFGLoopInfo.h b/mlir/include/mlir/Analysis/CFGLoopInfo.h
index 41e75021942c..15ac0d71cb94 100644
--- a/mlir/include/mlir/Analysis/CFGLoopInfo.h
+++ b/mlir/include/mlir/Analysis/CFGLoopInfo.h
@@ -49,8 +49,6 @@ public:
   CFGLoopInfo(const llvm::DominatorTreeBase<mlir::Block, false> &domTree);
 };
 
-raw_ostream &operator<<(raw_ostream &os, mlir::Block &block);
-
 } // namespace mlir
 
 #endif // MLIR_ANALYSIS_LOOPINFO_H
diff --git a/mlir/include/mlir/IR/Block.h b/mlir/include/mlir/IR/Block.h
index c14e9aad8f6d..e4fddfcb7608 100644
--- a/mlir/include/mlir/IR/Block.h
+++ b/mlir/include/mlir/IR/Block.h
@@ -18,6 +18,7 @@
 
 namespace llvm {
 class BitVector;
+class raw_ostream;
 } // namespace llvm
 
 namespace mlir {
@@ -401,6 +402,8 @@ private:
 
   friend struct llvm::ilist_traits<Block>;
 };
+
+raw_ostream &operator<<(raw_ostream &, Block &);
 } // namespace mlir
 
 #endif // MLIR_IR_BLOCK_H
diff --git a/mlir/lib/Analysis/CFGLoopInfo.cpp b/mlir/lib/Analysis/CFGLoopInfo.cpp
index 785f6c05d76e..9fe220344dce 100644
--- a/mlir/lib/Analysis/CFGLoopInfo.cpp
+++ b/mlir/lib/Analysis/CFGLoopInfo.cpp
@@ -23,8 +23,3 @@ CFGLoopInfo::CFGLoopInfo(
     const llvm::DominatorTreeBase<mlir::Block, false> &domTree) {
   analyze(domTree);
 }
-
-raw_ostream &mlir::operator<<(raw_ostream &os, mlir::Block &block) {
-  block.print(os);
-  return os;
-}
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index 9a5c51ba738f..29e36210f127 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -3984,6 +3984,11 @@ void Block::printAsOperand(raw_ostream &os, AsmState &state) {
   printer.printBlockName(this);
 }
 
+raw_ostream &mlir::operator<<(raw_ostream &os, Block &block) {
+  block.print(os);
+  return os;
+}
+
 //===--------------------------------------------------------------------===//
 // Custom printers
 //===--------------------------------------------------------------------===//
-- 
GitLab


From e90126e0dd6f058e1602bb6c769dfa4c52d7ad47 Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval@gmail.com>
Date: Sat, 18 May 2024 06:35:51 -0700
Subject: [PATCH 1195/1206] [flang][cuf] Add attr gen dependency to fix #92635

---
 flang/lib/Optimizer/Dialect/CUF/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flang/lib/Optimizer/Dialect/CUF/CMakeLists.txt b/flang/lib/Optimizer/Dialect/CUF/CMakeLists.txt
index d5ce5e0a7614..83d468bafdfe 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CMakeLists.txt
+++ b/flang/lib/Optimizer/Dialect/CUF/CMakeLists.txt
@@ -6,6 +6,7 @@ add_flang_library(CUFDialect
 
   DEPENDS
   MLIRIR
+  CUFAttrsIncGen
   CUFOpsIncGen
 
   LINK_LIBS
-- 
GitLab


From 58c778565cd64a69ea86e7e67e6a87fff3b0b224 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Sat, 18 May 2024 08:46:26 -0700
Subject: [PATCH 1196/1206] [nfc][ctx_profile] Fix printf - related
 `-Wformat-pedantic`

---
 compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
index cff39eeafba6..a0a535015bf2 100644
--- a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
+++ b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
@@ -228,8 +228,9 @@ ContextNode *__llvm_ctx_profile_get_context(void *Callee, GUID Guid,
       Ret->counters_size() != NrCounters)
     __sanitizer::Printf("[ctxprof] Returned ctx differs from what's asked: "
                         "Context: %p, Asked: %lu %u %u, Got: %lu %u %u \n",
-                        Ret, Guid, NrCallsites, NrCounters, Ret->guid(),
-                        Ret->callsites_size(), Ret->counters_size());
+                        reinterpret_cast<void *>(Ret), Guid, NrCallsites,
+                        NrCounters, Ret->guid(), Ret->callsites_size(),
+                        Ret->counters_size());
   onContextEnter(*Ret);
   return Ret;
 }
-- 
GitLab


From c5b11a710e01304908e3f320e40bc9da9f6a8de4 Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Sat, 18 May 2024 10:33:05 -0700
Subject: [PATCH 1197/1206] [NVPTX] support immediate values in st.param
 instructions (#91523)

Add support for generating `st.param` instructions with direct use of
immediates. This eliminates the need for a `mov` instruction prior to
the `st.param` resulting in more concise emitted PTX.
---
 llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp |  161 +-
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td     |  100 +-
 llvm/test/CodeGen/NVPTX/st-param-imm.ll     | 2002 +++++++++++++++++++
 3 files changed, 2199 insertions(+), 64 deletions(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/st-param-imm.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 595395bb1b4b..2713b6859ff3 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -2182,6 +2182,100 @@ bool NVPTXDAGToDAGISel::tryStoreRetval(SDNode *N) {
   return true;
 }
 
+// Helpers for constructing opcode (ex: NVPTX::StoreParamV4F32_iiri)
+#define getOpcV2H(ty, opKind0, opKind1)                                        \
+  NVPTX::StoreParamV2##ty##_##opKind0##opKind1
+
+#define getOpcV2H1(ty, opKind0, isImm1)                                        \
+  (isImm1) ? getOpcV2H(ty, opKind0, i) : getOpcV2H(ty, opKind0, r)
+
+#define getOpcodeForVectorStParamV2(ty, isimm)                                 \
+  (isimm[0]) ? getOpcV2H1(ty, i, isimm[1]) : getOpcV2H1(ty, r, isimm[1])
+
+#define getOpcV4H(ty, opKind0, opKind1, opKind2, opKind3)                      \
+  NVPTX::StoreParamV4##ty##_##opKind0##opKind1##opKind2##opKind3
+
+#define getOpcV4H3(ty, opKind0, opKind1, opKind2, isImm3)                      \
+  (isImm3) ? getOpcV4H(ty, opKind0, opKind1, opKind2, i)                       \
+           : getOpcV4H(ty, opKind0, opKind1, opKind2, r)
+
+#define getOpcV4H2(ty, opKind0, opKind1, isImm2, isImm3)                       \
+  (isImm2) ? getOpcV4H3(ty, opKind0, opKind1, i, isImm3)                       \
+           : getOpcV4H3(ty, opKind0, opKind1, r, isImm3)
+
+#define getOpcV4H1(ty, opKind0, isImm1, isImm2, isImm3)                        \
+  (isImm1) ? getOpcV4H2(ty, opKind0, i, isImm2, isImm3)                        \
+           : getOpcV4H2(ty, opKind0, r, isImm2, isImm3)
+
+#define getOpcodeForVectorStParamV4(ty, isimm)                                 \
+  (isimm[0]) ? getOpcV4H1(ty, i, isimm[1], isimm[2], isimm[3])                 \
+             : getOpcV4H1(ty, r, isimm[1], isimm[2], isimm[3])
+
+#define getOpcodeForVectorStParam(n, ty, isimm)                                \
+  (n == 2) ? getOpcodeForVectorStParamV2(ty, isimm)                            \
+           : getOpcodeForVectorStParamV4(ty, isimm)
+
+static unsigned pickOpcodeForVectorStParam(SmallVector<SDValue, 8> &Ops,
+                                           unsigned NumElts,
+                                           MVT::SimpleValueType MemTy,
+                                           SelectionDAG *CurDAG, SDLoc DL) {
+  // Determine which inputs are registers and immediates make new operators
+  // with constant values
+  SmallVector<bool, 4> IsImm(NumElts, false);
+  for (unsigned i = 0; i < NumElts; i++) {
+    IsImm[i] = (isa<ConstantSDNode>(Ops[i]) || isa<ConstantFPSDNode>(Ops[i]));
+    if (IsImm[i]) {
+      SDValue Imm = Ops[i];
+      if (MemTy == MVT::f32 || MemTy == MVT::f64) {
+        const ConstantFPSDNode *ConstImm = cast<ConstantFPSDNode>(Imm);
+        const ConstantFP *CF = ConstImm->getConstantFPValue();
+        Imm = CurDAG->getTargetConstantFP(*CF, DL, Imm->getValueType(0));
+      } else {
+        const ConstantSDNode *ConstImm = cast<ConstantSDNode>(Imm);
+        const ConstantInt *CI = ConstImm->getConstantIntValue();
+        Imm = CurDAG->getTargetConstant(*CI, DL, Imm->getValueType(0));
+      }
+      Ops[i] = Imm;
+    }
+  }
+
+  // Get opcode for MemTy, size, and register/immediate operand ordering
+  switch (MemTy) {
+  case MVT::i8:
+    return getOpcodeForVectorStParam(NumElts, I8, IsImm);
+  case MVT::i16:
+    return getOpcodeForVectorStParam(NumElts, I16, IsImm);
+  case MVT::i32:
+    return getOpcodeForVectorStParam(NumElts, I32, IsImm);
+  case MVT::i64:
+    assert(NumElts == 2 && "MVT too large for NumElts > 2");
+    return getOpcodeForVectorStParamV2(I64, IsImm);
+  case MVT::f32:
+    return getOpcodeForVectorStParam(NumElts, F32, IsImm);
+  case MVT::f64:
+    assert(NumElts == 2 && "MVT too large for NumElts > 2");
+    return getOpcodeForVectorStParamV2(F64, IsImm);
+
+  // These cases don't support immediates, just use the all register version
+  // and generate moves.
+  case MVT::i1:
+    return (NumElts == 2) ? NVPTX::StoreParamV2I8_rr
+                          : NVPTX::StoreParamV4I8_rrrr;
+  case MVT::f16:
+  case MVT::bf16:
+    return (NumElts == 2) ? NVPTX::StoreParamV2I16_rr
+                          : NVPTX::StoreParamV4I16_rrrr;
+  case MVT::v2f16:
+  case MVT::v2bf16:
+  case MVT::v2i16:
+  case MVT::v4i8:
+    return (NumElts == 2) ? NVPTX::StoreParamV2I32_rr
+                          : NVPTX::StoreParamV4I32_rrrr;
+  default:
+    llvm_unreachable("Cannot select st.param for unknown MemTy");
+  }
+}
+
 bool NVPTXDAGToDAGISel::tryStoreParam(SDNode *N) {
   SDLoc DL(N);
   SDValue Chain = N->getOperand(0);
@@ -2193,10 +2287,10 @@ bool NVPTXDAGToDAGISel::tryStoreParam(SDNode *N) {
   SDValue Glue = N->getOperand(N->getNumOperands() - 1);
 
   // How many elements do we have?
-  unsigned NumElts = 1;
+  unsigned NumElts;
   switch (N->getOpcode()) {
   default:
-    return false;
+    llvm_unreachable("Unexpected opcode");
   case NVPTXISD::StoreParamU32:
   case NVPTXISD::StoreParamS32:
   case NVPTXISD::StoreParam:
@@ -2222,18 +2316,40 @@ bool NVPTXDAGToDAGISel::tryStoreParam(SDNode *N) {
   // Determine target opcode
   // If we have an i1, use an 8-bit store. The lowering code in
   // NVPTXISelLowering will have already emitted an upcast.
-  std::optional<unsigned> Opcode = 0;
+  std::optional<unsigned> Opcode;
   switch (N->getOpcode()) {
   default:
     switch (NumElts) {
     default:
-      return false;
-    case 1:
-      Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
-                               NVPTX::StoreParamI8, NVPTX::StoreParamI16,
-                               NVPTX::StoreParamI32, NVPTX::StoreParamI64,
-                               NVPTX::StoreParamF32, NVPTX::StoreParamF64);
-      if (Opcode == NVPTX::StoreParamI8) {
+      llvm_unreachable("Unexpected NumElts");
+    case 1: {
+      MVT::SimpleValueType MemTy = Mem->getMemoryVT().getSimpleVT().SimpleTy;
+      SDValue Imm = Ops[0];
+      if (MemTy != MVT::f16 && MemTy != MVT::v2f16 &&
+          (isa<ConstantSDNode>(Imm) || isa<ConstantFPSDNode>(Imm))) {
+        // Convert immediate to target constant
+        if (MemTy == MVT::f32 || MemTy == MVT::f64) {
+          const ConstantFPSDNode *ConstImm = cast<ConstantFPSDNode>(Imm);
+          const ConstantFP *CF = ConstImm->getConstantFPValue();
+          Imm = CurDAG->getTargetConstantFP(*CF, DL, Imm->getValueType(0));
+        } else {
+          const ConstantSDNode *ConstImm = cast<ConstantSDNode>(Imm);
+          const ConstantInt *CI = ConstImm->getConstantIntValue();
+          Imm = CurDAG->getTargetConstant(*CI, DL, Imm->getValueType(0));
+        }
+        Ops[0] = Imm;
+        // Use immediate version of store param
+        Opcode = pickOpcodeForVT(MemTy, NVPTX::StoreParamI8_i,
+                                 NVPTX::StoreParamI16_i, NVPTX::StoreParamI32_i,
+                                 NVPTX::StoreParamI64_i, NVPTX::StoreParamF32_i,
+                                 NVPTX::StoreParamF64_i);
+      } else
+        Opcode =
+            pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
+                            NVPTX::StoreParamI8_r, NVPTX::StoreParamI16_r,
+                            NVPTX::StoreParamI32_r, NVPTX::StoreParamI64_r,
+                            NVPTX::StoreParamF32_r, NVPTX::StoreParamF64_r);
+      if (Opcode == NVPTX::StoreParamI8_r) {
         // Fine tune the opcode depending on the size of the operand.
         // This helps to avoid creating redundant COPY instructions in
         // InstrEmitter::AddRegisterOperand().
@@ -2241,35 +2357,28 @@ bool NVPTXDAGToDAGISel::tryStoreParam(SDNode *N) {
         default:
           break;
         case MVT::i32:
-          Opcode = NVPTX::StoreParamI8TruncI32;
+          Opcode = NVPTX::StoreParamI8TruncI32_r;
           break;
         case MVT::i64:
-          Opcode = NVPTX::StoreParamI8TruncI64;
+          Opcode = NVPTX::StoreParamI8TruncI64_r;
           break;
         }
       }
       break;
+    }
     case 2:
-      Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
-                               NVPTX::StoreParamV2I8, NVPTX::StoreParamV2I16,
-                               NVPTX::StoreParamV2I32, NVPTX::StoreParamV2I64,
-                               NVPTX::StoreParamV2F32, NVPTX::StoreParamV2F64);
-      break;
-    case 4:
-      Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
-                               NVPTX::StoreParamV4I8, NVPTX::StoreParamV4I16,
-                               NVPTX::StoreParamV4I32, std::nullopt,
-                               NVPTX::StoreParamV4F32, std::nullopt);
+    case 4: {
+      MVT::SimpleValueType MemTy = Mem->getMemoryVT().getSimpleVT().SimpleTy;
+      Opcode = pickOpcodeForVectorStParam(Ops, NumElts, MemTy, CurDAG, DL);
       break;
     }
-    if (!Opcode)
-      return false;
+    }
     break;
   // Special case: if we have a sign-extend/zero-extend node, insert the
   // conversion instruction first, and use that as the value operand to
   // the selected StoreParam node.
   case NVPTXISD::StoreParamU32: {
-    Opcode = NVPTX::StoreParamI32;
+    Opcode = NVPTX::StoreParamI32_r;
     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
                                                 MVT::i32);
     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
@@ -2278,7 +2387,7 @@ bool NVPTXDAGToDAGISel::tryStoreParam(SDNode *N) {
     break;
   }
   case NVPTXISD::StoreParamS32: {
-    Opcode = NVPTX::StoreParamI32;
+    Opcode = NVPTX::StoreParamI32_r;
     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
                                                 MVT::i32);
     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 393fa29ff051..c4c35a1f74ba 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -2637,25 +2637,46 @@ class LoadParamRegInst<NVPTXRegClass regclass, string opstr> :
                 [(set regclass:$dst, (LoadParam (i32 0), (i32 imm:$b)))]>;
 
 let mayStore = true in {
-  class StoreParamInst<NVPTXRegClass regclass, string opstr> :
-        NVPTXInst<(outs), (ins regclass:$val, i32imm:$a, i32imm:$b),
-                  !strconcat("st.param", opstr, " \t[param$a+$b], $val;"),
-                  []>;
 
-  class StoreParamV2Inst<NVPTXRegClass regclass, string opstr> :
-        NVPTXInst<(outs), (ins regclass:$val, regclass:$val2,
-                               i32imm:$a, i32imm:$b),
-                  !strconcat("st.param.v2", opstr,
-                             " \t[param$a+$b], {{$val, $val2}};"),
-                  []>;
+  multiclass StoreParamInst<NVPTXRegClass regclass, Operand IMMType, string opstr, bit support_imm = true> {
+    foreach op = [IMMType, regclass] in
+      if !or(support_imm, !isa<NVPTXRegClass>(op)) then
+        def _ # !if(!isa<NVPTXRegClass>(op), "r", "i")
+          : NVPTXInst<(outs),
+                      (ins op:$val, i32imm:$a, i32imm:$b),
+                      "st.param" # opstr # " \t[param$a+$b], $val;",
+                      []>;
+  }
 
-  class StoreParamV4Inst<NVPTXRegClass regclass, string opstr> :
-        NVPTXInst<(outs), (ins regclass:$val, regclass:$val2, regclass:$val3,
-                               regclass:$val4, i32imm:$a,
-                               i32imm:$b),
-                  !strconcat("st.param.v4", opstr,
-                             " \t[param$a+$b], {{$val, $val2, $val3, $val4}};"),
-                  []>;
+  multiclass StoreParamV2Inst<NVPTXRegClass regclass, Operand IMMType, string opstr> {
+    foreach op1 = [IMMType, regclass] in
+      foreach op2 = [IMMType, regclass] in
+        def _ # !if(!isa<NVPTXRegClass>(op1), "r", "i")
+              # !if(!isa<NVPTXRegClass>(op2), "r", "i")
+          : NVPTXInst<(outs),
+                      (ins op1:$val1, op2:$val2,
+                           i32imm:$a, i32imm:$b),
+                      "st.param.v2" # opstr # " \t[param$a+$b], {{$val1, $val2}};",
+                      []>;
+  }
+
+  multiclass StoreParamV4Inst<NVPTXRegClass regclass, Operand IMMType, string opstr> {
+    foreach op1 = [IMMType, regclass] in
+      foreach op2 = [IMMType, regclass] in
+        foreach op3 = [IMMType, regclass] in
+          foreach op4 = [IMMType, regclass] in
+            def _ # !if(!isa<NVPTXRegClass>(op1), "r", "i")
+                  # !if(!isa<NVPTXRegClass>(op2), "r", "i")
+                  # !if(!isa<NVPTXRegClass>(op3), "r", "i")
+                  # !if(!isa<NVPTXRegClass>(op4), "r", "i")
+
+              : NVPTXInst<(outs),
+                          (ins op1:$val1, op2:$val2, op3:$val3, op4:$val4,
+                               i32imm:$a, i32imm:$b),
+                          "st.param.v4" # opstr #
+                          " \t[param$a+$b], {{$val1, $val2, $val3, $val4}};",
+                          []>;
+  }
 
   class StoreRetvalInst<NVPTXRegClass regclass, string opstr> :
         NVPTXInst<(outs), (ins regclass:$val, i32imm:$a),
@@ -2735,27 +2756,30 @@ def LoadParamMemV2F32  : LoadParamV2MemInst<Float32Regs, ".f32">;
 def LoadParamMemV2F64  : LoadParamV2MemInst<Float64Regs, ".f64">;
 def LoadParamMemV4F32  : LoadParamV4MemInst<Float32Regs, ".f32">;
 
-def StoreParamI64    : StoreParamInst<Int64Regs, ".b64">;
-def StoreParamI32    : StoreParamInst<Int32Regs, ".b32">;
-
-def StoreParamI16    : StoreParamInst<Int16Regs, ".b16">;
-def StoreParamI8     : StoreParamInst<Int16Regs, ".b8">;
-def StoreParamI8TruncI32 : StoreParamInst<Int32Regs, ".b8">;
-def StoreParamI8TruncI64 : StoreParamInst<Int64Regs, ".b8">;
-def StoreParamV2I64  : StoreParamV2Inst<Int64Regs, ".b64">;
-def StoreParamV2I32  : StoreParamV2Inst<Int32Regs, ".b32">;
-def StoreParamV2I16  : StoreParamV2Inst<Int16Regs, ".b16">;
-def StoreParamV2I8   : StoreParamV2Inst<Int16Regs, ".b8">;
-
-def StoreParamV4I32  : StoreParamV4Inst<Int32Regs, ".b32">;
-def StoreParamV4I16  : StoreParamV4Inst<Int16Regs, ".b16">;
-def StoreParamV4I8   : StoreParamV4Inst<Int16Regs, ".b8">;
-
-def StoreParamF32      : StoreParamInst<Float32Regs, ".f32">;
-def StoreParamF64      : StoreParamInst<Float64Regs, ".f64">;
-def StoreParamV2F32    : StoreParamV2Inst<Float32Regs, ".f32">;
-def StoreParamV2F64    : StoreParamV2Inst<Float64Regs, ".f64">;
-def StoreParamV4F32    : StoreParamV4Inst<Float32Regs, ".f32">;
+defm StoreParamI64    : StoreParamInst<Int64Regs, i64imm, ".b64">;
+defm StoreParamI32    : StoreParamInst<Int32Regs, i32imm, ".b32">;
+defm StoreParamI16    : StoreParamInst<Int16Regs, i16imm, ".b16">;
+defm StoreParamI8     : StoreParamInst<Int16Regs, i8imm,  ".b8">;
+
+defm StoreParamI8TruncI32 : StoreParamInst<Int32Regs, i8imm, ".b8", /* support_imm */ false>;
+defm StoreParamI8TruncI64 : StoreParamInst<Int64Regs, i8imm, ".b8", /* support_imm */ false>;
+
+defm StoreParamV2I64  : StoreParamV2Inst<Int64Regs, i64imm, ".b64">;
+defm StoreParamV2I32  : StoreParamV2Inst<Int32Regs, i32imm, ".b32">;
+defm StoreParamV2I16  : StoreParamV2Inst<Int16Regs, i16imm, ".b16">;
+defm StoreParamV2I8   : StoreParamV2Inst<Int16Regs, i8imm,  ".b8">;
+
+defm StoreParamV4I32  : StoreParamV4Inst<Int32Regs, i32imm, ".b32">;
+defm StoreParamV4I16  : StoreParamV4Inst<Int16Regs, i16imm, ".b16">;
+defm StoreParamV4I8   : StoreParamV4Inst<Int16Regs, i8imm,  ".b8">;
+
+defm StoreParamF32    : StoreParamInst<Float32Regs, f32imm, ".f32">;
+defm StoreParamF64    : StoreParamInst<Float64Regs, f64imm, ".f64">;
+
+defm StoreParamV2F32  : StoreParamV2Inst<Float32Regs, f32imm, ".f32">;
+defm StoreParamV2F64  : StoreParamV2Inst<Float64Regs, f64imm, ".f64">;
+
+defm StoreParamV4F32  : StoreParamV4Inst<Float32Regs, f32imm, ".f32">;
 
 def StoreRetvalI64    : StoreRetvalInst<Int64Regs, ".b64">;
 def StoreRetvalI32    : StoreRetvalInst<Int32Regs, ".b32">;
diff --git a/llvm/test/CodeGen/NVPTX/st-param-imm.ll b/llvm/test/CodeGen/NVPTX/st-param-imm.ll
new file mode 100644
index 000000000000..d9e005719238
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/st-param-imm.ll
@@ -0,0 +1,2002 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -march=nvptx64 | FileCheck %s
+; RUN: llc < %s -march=nvptx | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -march=nvptx -verify-machineinstrs | %ptxas-verify %}
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 -verify-machineinstrs | %ptxas-verify %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+%struct.A = type { i8, i16 }
+%struct.char2 = type { i8, i8 }
+%struct.char4 = type { i8, i8, i8, i8 }
+%struct.short2 = type { i16, i16 }
+%struct.short4 = type { i16, i16, i16, i16 }
+%struct.int2 = type { i32, i32 }
+%struct.int4 = type { i32, i32, i32, i32 }
+%struct.longlong2 = type { i64, i64 }
+%struct.float2 = type { float, float }
+%struct.float4 = type { float, float, float, float }
+%struct.double2 = type { double, double }
+
+define void @st_param_i8_i16() {
+; CHECK-LABEL: st_param_i8_i16(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 0, 0
+; CHECK-NEXT:    .param .align 2 .b8 param0[4];
+; CHECK-NEXT:    st.param.b8 [param0+0], 1;
+; CHECK-NEXT:    st.param.b16 [param0+2], 2;
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_i8_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 0
+; CHECK-NEXT:    ret;
+  call void @call_i8_i16(%struct.A { i8 1, i16 2 })
+  ret void
+}
+
+define void @st_param_i32() {
+; CHECK-LABEL: st_param_i32(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 1, 0
+; CHECK-NEXT:    .param .b32 param0;
+; CHECK-NEXT:    st.param.b32 [param0+0], 3;
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 1
+; CHECK-NEXT:    ret;
+  call void @call_i32(i32 3)
+  ret void
+}
+
+define void @st_param_i64() {
+; CHECK-LABEL: st_param_i64(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 2, 0
+; CHECK-NEXT:    .param .b64 param0;
+; CHECK-NEXT:    st.param.b64 [param0+0], 4;
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_i64,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 2
+; CHECK-NEXT:    ret;
+  call void @call_i64(i64 4)
+  ret void
+}
+
+define void @st_param_f32() {
+; CHECK-LABEL: st_param_f32(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 3, 0
+; CHECK-NEXT:    .param .b32 param0;
+; CHECK-NEXT:    st.param.f32 [param0+0], 0f40A00000;
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 3
+; CHECK-NEXT:    ret;
+  call void @call_f32(float 5.0)
+  ret void
+}
+
+define void @st_param_f64() {
+; CHECK-LABEL: st_param_f64(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 4, 0
+; CHECK-NEXT:    .param .b64 param0;
+; CHECK-NEXT:    st.param.f64 [param0+0], 0d4018000000000000;
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_f64,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 4
+; CHECK-NEXT:    ret;
+  call void @call_f64(double 6.0)
+  ret void
+}
+
+declare void @call_i8_i16(%struct.A)
+declare void @call_i32(i32)
+declare void @call_i64(i64)
+declare void @call_f32(float)
+declare void @call_f64(double)
+
+define void @st_param_v2_i8_ii() {
+; CHECK-LABEL: st_param_v2_i8_ii(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 5, 0
+; CHECK-NEXT:    .param .align 2 .b8 param0[2];
+; CHECK-NEXT:    st.param.v2.b8 [param0+0], {1, 2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 5
+; CHECK-NEXT:    ret;
+  call void @call_v2_i8(%struct.char2 { i8 1, i8 2 })
+  ret void
+}
+define void @st_param_v2_i8_ir(i8 %val) {
+; CHECK-LABEL: st_param_v2_i8_ir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v2_i8_ir_param_0];
+; CHECK-NEXT:    { // callseq 6, 0
+; CHECK-NEXT:    .param .align 2 .b8 param0[2];
+; CHECK-NEXT:    st.param.v2.b8 [param0+0], {1, %rs1};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 6
+; CHECK-NEXT:    ret;
+  %struct.ir0 = insertvalue %struct.char2 poison, i8 1, 0
+  %struct.ir1 = insertvalue %struct.char2 %struct.ir0, i8 %val, 1
+  call void @call_v2_i8(%struct.char2 %struct.ir1)
+  ret void
+}
+define void @st_param_v2_i8_ri(i8 %val) {
+; CHECK-LABEL: st_param_v2_i8_ri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v2_i8_ri_param_0];
+; CHECK-NEXT:    { // callseq 7, 0
+; CHECK-NEXT:    .param .align 2 .b8 param0[2];
+; CHECK-NEXT:    st.param.v2.b8 [param0+0], {%rs1, 2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 7
+; CHECK-NEXT:    ret;
+  %struct.ri0 = insertvalue %struct.char2 poison, i8 %val, 0
+  %struct.ri1 = insertvalue %struct.char2 %struct.ri0, i8 2, 1
+  call void @call_v2_i8(%struct.char2 %struct.ri1)
+  ret void
+}
+
+define void @st_param_v2_i16_ii() {
+; CHECK-LABEL: st_param_v2_i16_ii(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 8, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v2.b16 [param0+0], {1, 2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 8
+; CHECK-NEXT:    ret;
+  call void @call_v2_i16(%struct.short2 { i16 1, i16 2 })
+  ret void
+}
+define void @st_param_v2_i16_ir(i16 %val) {
+; CHECK-LABEL: st_param_v2_i16_ir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v2_i16_ir_param_0];
+; CHECK-NEXT:    { // callseq 9, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v2.b16 [param0+0], {1, %rs1};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 9
+; CHECK-NEXT:    ret;
+  %struct.ir0 = insertvalue %struct.short2 poison, i16 1, 0
+  %struct.ir1 = insertvalue %struct.short2 %struct.ir0, i16 %val, 1
+  call void @call_v2_i16(%struct.short2 %struct.ir1)
+  ret void
+}
+define void @st_param_v2_i16_ri(i16 %val) {
+; CHECK-LABEL: st_param_v2_i16_ri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v2_i16_ri_param_0];
+; CHECK-NEXT:    { // callseq 10, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v2.b16 [param0+0], {%rs1, 2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 10
+; CHECK-NEXT:    ret;
+  %struct.ri0 = insertvalue %struct.short2 poison, i16 %val, 0
+  %struct.ri1 = insertvalue %struct.short2 %struct.ri0, i16 2, 1
+  call void @call_v2_i16(%struct.short2 %struct.ri1)
+  ret void
+}
+
+define void @st_param_v2_i32_ii() {
+; CHECK-LABEL: st_param_v2_i32_ii(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 11, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v2.b32 [param0+0], {1, 2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 11
+; CHECK-NEXT:    ret;
+  call void @call_v2_i32(%struct.int2 { i32 1, i32 2 })
+  ret void
+}
+define void @st_param_v2_i32_ir(i32 %val) {
+; CHECK-LABEL: st_param_v2_i32_ir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v2_i32_ir_param_0];
+; CHECK-NEXT:    { // callseq 12, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v2.b32 [param0+0], {1, %r1};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 12
+; CHECK-NEXT:    ret;
+  %struct.ir0 = insertvalue %struct.int2 poison, i32 1, 0
+  %struct.ir1 = insertvalue %struct.int2 %struct.ir0, i32 %val, 1
+  call void @call_v2_i32(%struct.int2 %struct.ir1)
+  ret void
+}
+define void @st_param_v2_i32_ri(i32 %val) {
+; CHECK-LABEL: st_param_v2_i32_ri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v2_i32_ri_param_0];
+; CHECK-NEXT:    { // callseq 13, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v2.b32 [param0+0], {%r1, 2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 13
+; CHECK-NEXT:    ret;
+  %struct.ri0 = insertvalue %struct.int2 poison, i32 %val, 0
+  %struct.ri1 = insertvalue %struct.int2 %struct.ri0, i32 2, 1
+  call void @call_v2_i32(%struct.int2 %struct.ri1)
+  ret void
+}
+
+define void @st_param_v2_i64_ii() {
+; CHECK-LABEL: st_param_v2_i64_ii(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 14, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v2.b64 [param0+0], {1, 2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i64,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 14
+; CHECK-NEXT:    ret;
+  call void @call_v2_i64(%struct.longlong2 { i64 1, i64 2 })
+  ret void
+}
+define void @st_param_v2_i64_ir(i64 %val) {
+; CHECK-LABEL: st_param_v2_i64_ir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [st_param_v2_i64_ir_param_0];
+; CHECK-NEXT:    { // callseq 15, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v2.b64 [param0+0], {1, %rd1};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i64,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 15
+; CHECK-NEXT:    ret;
+  %struct.ir0 = insertvalue %struct.longlong2 poison, i64 1, 0
+  %struct.ir1 = insertvalue %struct.longlong2 %struct.ir0, i64 %val, 1
+  call void @call_v2_i64(%struct.longlong2 %struct.ir1)
+  ret void
+}
+define void @st_param_v2_i64_ri(i64 %val) {
+; CHECK-LABEL: st_param_v2_i64_ri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [st_param_v2_i64_ri_param_0];
+; CHECK-NEXT:    { // callseq 16, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v2.b64 [param0+0], {%rd1, 2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_i64,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 16
+; CHECK-NEXT:    ret;
+  %struct.ri0 = insertvalue %struct.longlong2 poison, i64 %val, 0
+  %struct.ri1 = insertvalue %struct.longlong2 %struct.ri0, i64 2, 1
+  call void @call_v2_i64(%struct.longlong2 %struct.ri1)
+  ret void
+}
+
+define void @st_param_v2_f32_ii(float %val) {
+; CHECK-LABEL: st_param_v2_f32_ii(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 17, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v2.f32 [param0+0], {0f3F800000, 0f40000000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 17
+; CHECK-NEXT:    ret;
+  call void @call_v2_f32(%struct.float2 { float 1.0, float 2.0 })
+  ret void
+}
+define void @st_param_v2_f32_ir(float %val) {
+; CHECK-LABEL: st_param_v2_f32_ir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v2_f32_ir_param_0];
+; CHECK-NEXT:    { // callseq 18, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v2.f32 [param0+0], {0f3F800000, %f1};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 18
+; CHECK-NEXT:    ret;
+  %struct.ir0 = insertvalue %struct.float2 poison, float 1.0, 0
+  %struct.ir1 = insertvalue %struct.float2 %struct.ir0, float %val, 1
+  call void @call_v2_f32(%struct.float2 %struct.ir1)
+  ret void
+}
+define void @st_param_v2_f32_ri(float %val) {
+; CHECK-LABEL: st_param_v2_f32_ri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v2_f32_ri_param_0];
+; CHECK-NEXT:    { // callseq 19, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v2.f32 [param0+0], {%f1, 0f40000000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 19
+; CHECK-NEXT:    ret;
+  %struct.ri0 = insertvalue %struct.float2 poison, float %val, 0
+  %struct.ri1 = insertvalue %struct.float2 %struct.ri0, float 2.0, 1
+  call void @call_v2_f32(%struct.float2 %struct.ri1)
+  ret void
+}
+
+define void @st_param_v2_f64_ii(double %val) {
+; CHECK-LABEL: st_param_v2_f64_ii(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 20, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v2.f64 [param0+0], {0d3FF0000000000000, 0d4000000000000000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_f64,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 20
+; CHECK-NEXT:    ret;
+  call void @call_v2_f64(%struct.double2 { double 1.0, double 2.0 })
+  ret void
+}
+define void @st_param_v2_f64_ir(double %val) {
+; CHECK-LABEL: st_param_v2_f64_ir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f64 %fd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f64 %fd1, [st_param_v2_f64_ir_param_0];
+; CHECK-NEXT:    { // callseq 21, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v2.f64 [param0+0], {0d3FF0000000000000, %fd1};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_f64,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 21
+; CHECK-NEXT:    ret;
+  %struct.ir0 = insertvalue %struct.double2 poison, double 1.0, 0
+  %struct.ir1 = insertvalue %struct.double2 %struct.ir0, double %val, 1
+  call void @call_v2_f64(%struct.double2 %struct.ir1)
+  ret void
+}
+define void @st_param_v2_f64_ri(double %val) {
+; CHECK-LABEL: st_param_v2_f64_ri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f64 %fd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f64 %fd1, [st_param_v2_f64_ri_param_0];
+; CHECK-NEXT:    { // callseq 22, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v2.f64 [param0+0], {%fd1, 0d4000000000000000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v2_f64,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 22
+; CHECK-NEXT:    ret;
+  %struct.ri0 = insertvalue %struct.double2 poison, double %val, 0
+  %struct.ri1 = insertvalue %struct.double2 %struct.ri0, double 2.0, 1
+  call void @call_v2_f64(%struct.double2 %struct.ri1)
+  ret void
+}
+
+declare void @call_v2_i8(%struct.char2 alignstack(2))
+declare void @call_v2_i16(%struct.short2 alignstack(4))
+declare void @call_v2_i32(%struct.int2 alignstack(8))
+declare void @call_v2_i64(%struct.longlong2 alignstack(16))
+declare void @call_v2_f32(%struct.float2 alignstack(8))
+declare void @call_v2_f64(%struct.double2 alignstack(16))
+
+define void @st_param_v4_i8_iiii() {
+; CHECK-LABEL: st_param_v4_i8_iiii(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 23, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {1, 2, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 23
+; CHECK-NEXT:    ret;
+  call void @call_v4_i8(%struct.char4 { i8 1, i8 2, i8 3, i8 4 })
+  ret void
+}
+define void @st_param_v4_i8_irrr(i8 %b, i8 %c, i8 %d) {
+; CHECK-LABEL: st_param_v4_i8_irrr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_irrr_param_0];
+; CHECK-NEXT:    ld.param.u8 %rs2, [st_param_v4_i8_irrr_param_1];
+; CHECK-NEXT:    ld.param.u8 %rs3, [st_param_v4_i8_irrr_param_2];
+; CHECK-NEXT:    { // callseq 24, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {1, %rs1, %rs2, %rs3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 24
+; CHECK-NEXT:    ret;
+  %struct.irrr0 = insertvalue %struct.char4 poison, i8 1, 0
+  %struct.irrr1 = insertvalue %struct.char4 %struct.irrr0, i8 %b, 1
+  %struct.irrr2 = insertvalue %struct.char4 %struct.irrr1, i8 %c, 2
+  %struct.irrr3 = insertvalue %struct.char4 %struct.irrr2, i8 %d, 3
+  call void @call_v4_i8(%struct.char4 %struct.irrr3)
+  ret void
+}
+define void @st_param_v4_i8_rirr(i8 %a, i8 %c, i8 %d) {
+; CHECK-LABEL: st_param_v4_i8_rirr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_rirr_param_0];
+; CHECK-NEXT:    ld.param.u8 %rs2, [st_param_v4_i8_rirr_param_1];
+; CHECK-NEXT:    ld.param.u8 %rs3, [st_param_v4_i8_rirr_param_2];
+; CHECK-NEXT:    { // callseq 25, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {%rs1, 2, %rs2, %rs3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 25
+; CHECK-NEXT:    ret;
+  %struct.rirr0 = insertvalue %struct.char4 poison, i8 %a, 0
+  %struct.rirr1 = insertvalue %struct.char4 %struct.rirr0, i8 2, 1
+  %struct.rirr2 = insertvalue %struct.char4 %struct.rirr1, i8 %c, 2
+  %struct.rirr3 = insertvalue %struct.char4 %struct.rirr2, i8 %d, 3
+  call void @call_v4_i8(%struct.char4 %struct.rirr3)
+  ret void
+}
+define void @st_param_v4_i8_rrir(i8 %a, i8 %b, i8 %d) {
+; CHECK-LABEL: st_param_v4_i8_rrir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_rrir_param_0];
+; CHECK-NEXT:    ld.param.u8 %rs2, [st_param_v4_i8_rrir_param_1];
+; CHECK-NEXT:    ld.param.u8 %rs3, [st_param_v4_i8_rrir_param_2];
+; CHECK-NEXT:    { // callseq 26, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {%rs1, %rs2, 3, %rs3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 26
+; CHECK-NEXT:    ret;
+  %struct.rrir0 = insertvalue %struct.char4 poison, i8 %a, 0
+  %struct.rrir1 = insertvalue %struct.char4 %struct.rrir0, i8 %b, 1
+  %struct.rrir2 = insertvalue %struct.char4 %struct.rrir1, i8 3, 2
+  %struct.rrir3 = insertvalue %struct.char4 %struct.rrir2, i8 %d, 3
+  call void @call_v4_i8(%struct.char4 %struct.rrir3)
+  ret void
+}
+define void @st_param_v4_i8_rrri(i8 %a, i8 %b, i8 %c) {
+; CHECK-LABEL: st_param_v4_i8_rrri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_rrri_param_0];
+; CHECK-NEXT:    ld.param.u8 %rs2, [st_param_v4_i8_rrri_param_1];
+; CHECK-NEXT:    ld.param.u8 %rs3, [st_param_v4_i8_rrri_param_2];
+; CHECK-NEXT:    { // callseq 27, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {%rs1, %rs2, %rs3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 27
+; CHECK-NEXT:    ret;
+  %struct.rrri0 = insertvalue %struct.char4 poison, i8 %a, 0
+  %struct.rrri1 = insertvalue %struct.char4 %struct.rrri0, i8 %b, 1
+  %struct.rrri2 = insertvalue %struct.char4 %struct.rrri1, i8 %c, 2
+  %struct.rrri3 = insertvalue %struct.char4 %struct.rrri2, i8 4, 3
+  call void @call_v4_i8(%struct.char4 %struct.rrri3)
+  ret void
+}
+define void @st_param_v4_i8_iirr(i8 %c, i8 %d) {
+; CHECK-LABEL: st_param_v4_i8_iirr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_iirr_param_0];
+; CHECK-NEXT:    ld.param.u8 %rs2, [st_param_v4_i8_iirr_param_1];
+; CHECK-NEXT:    { // callseq 28, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {1, 2, %rs1, %rs2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 28
+; CHECK-NEXT:    ret;
+  %struct.iirr0 = insertvalue %struct.char4 poison, i8 1, 0
+  %struct.iirr1 = insertvalue %struct.char4 %struct.iirr0, i8 2, 1
+  %struct.iirr2 = insertvalue %struct.char4 %struct.iirr1, i8 %c, 2
+  %struct.iirr3 = insertvalue %struct.char4 %struct.iirr2, i8 %d, 3
+  call void @call_v4_i8(%struct.char4 %struct.iirr3)
+  ret void
+}
+define void @st_param_v4_i8_irir(i8 %b, i8 %d) {
+; CHECK-LABEL: st_param_v4_i8_irir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_irir_param_0];
+; CHECK-NEXT:    ld.param.u8 %rs2, [st_param_v4_i8_irir_param_1];
+; CHECK-NEXT:    { // callseq 29, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {1, %rs1, 3, %rs2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 29
+; CHECK-NEXT:    ret;
+  %struct.irir0 = insertvalue %struct.char4 poison, i8 1, 0
+  %struct.irir1 = insertvalue %struct.char4 %struct.irir0, i8 %b, 1
+  %struct.irir2 = insertvalue %struct.char4 %struct.irir1, i8 3, 2
+  %struct.irir3 = insertvalue %struct.char4 %struct.irir2, i8 %d, 3
+  call void @call_v4_i8(%struct.char4 %struct.irir3)
+  ret void
+}
+define void @st_param_v4_i8_irri(i8 %b, i8 %c) {
+; CHECK-LABEL: st_param_v4_i8_irri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_irri_param_0];
+; CHECK-NEXT:    ld.param.u8 %rs2, [st_param_v4_i8_irri_param_1];
+; CHECK-NEXT:    { // callseq 30, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {1, %rs1, %rs2, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 30
+; CHECK-NEXT:    ret;
+  %struct.irri0 = insertvalue %struct.char4 poison, i8 1, 0
+  %struct.irri1 = insertvalue %struct.char4 %struct.irri0, i8 %b, 1
+  %struct.irri2 = insertvalue %struct.char4 %struct.irri1, i8 %c, 2
+  %struct.irri3 = insertvalue %struct.char4 %struct.irri2, i8 4, 3
+  call void @call_v4_i8(%struct.char4 %struct.irri3)
+  ret void
+}
+define void @st_param_v4_i8_riir(i8 %a, i8 %d) {
+; CHECK-LABEL: st_param_v4_i8_riir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_riir_param_0];
+; CHECK-NEXT:    ld.param.u8 %rs2, [st_param_v4_i8_riir_param_1];
+; CHECK-NEXT:    { // callseq 31, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {%rs1, 2, 3, %rs2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 31
+; CHECK-NEXT:    ret;
+  %struct.riir0 = insertvalue %struct.char4 poison, i8 %a, 0
+  %struct.riir1 = insertvalue %struct.char4 %struct.riir0, i8 2, 1
+  %struct.riir2 = insertvalue %struct.char4 %struct.riir1, i8 3, 2
+  %struct.riir3 = insertvalue %struct.char4 %struct.riir2, i8 %d, 3
+  call void @call_v4_i8(%struct.char4 %struct.riir3)
+  ret void
+}
+define void @st_param_v4_i8_riri(i8 %a, i8 %c) {
+; CHECK-LABEL: st_param_v4_i8_riri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_riri_param_0];
+; CHECK-NEXT:    ld.param.u8 %rs2, [st_param_v4_i8_riri_param_1];
+; CHECK-NEXT:    { // callseq 32, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {%rs1, 2, %rs2, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 32
+; CHECK-NEXT:    ret;
+  %struct.riri0 = insertvalue %struct.char4 poison, i8 %a, 0
+  %struct.riri1 = insertvalue %struct.char4 %struct.riri0, i8 2, 1
+  %struct.riri2 = insertvalue %struct.char4 %struct.riri1, i8 %c, 2
+  %struct.riri3 = insertvalue %struct.char4 %struct.riri2, i8 4, 3
+  call void @call_v4_i8(%struct.char4 %struct.riri3)
+  ret void
+}
+define void @st_param_v4_i8_rrii(i8 %a, i8 %b) {
+; CHECK-LABEL: st_param_v4_i8_rrii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_rrii_param_0];
+; CHECK-NEXT:    ld.param.u8 %rs2, [st_param_v4_i8_rrii_param_1];
+; CHECK-NEXT:    { // callseq 33, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {%rs1, %rs2, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 33
+; CHECK-NEXT:    ret;
+  %struct.rrii0 = insertvalue %struct.char4 poison, i8 %a, 0
+  %struct.rrii1 = insertvalue %struct.char4 %struct.rrii0, i8 %b, 1
+  %struct.rrii2 = insertvalue %struct.char4 %struct.rrii1, i8 3, 2
+  %struct.rrii3 = insertvalue %struct.char4 %struct.rrii2, i8 4, 3
+  call void @call_v4_i8(%struct.char4 %struct.rrii3)
+  ret void
+}
+define void @st_param_v4_i8_iiir(i8 %d) {
+; CHECK-LABEL: st_param_v4_i8_iiir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_iiir_param_0];
+; CHECK-NEXT:    { // callseq 34, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {1, 2, 3, %rs1};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 34
+; CHECK-NEXT:    ret;
+  %struct.iiir0 = insertvalue %struct.char4 poison, i8 1, 0
+  %struct.iiir1 = insertvalue %struct.char4 %struct.iiir0, i8 2, 1
+  %struct.iiir2 = insertvalue %struct.char4 %struct.iiir1, i8 3, 2
+  %struct.iiir3 = insertvalue %struct.char4 %struct.iiir2, i8 %d, 3
+  call void @call_v4_i8(%struct.char4 %struct.iiir3)
+  ret void
+}
+define void @st_param_v4_i8_iiri(i8 %c) {
+; CHECK-LABEL: st_param_v4_i8_iiri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_iiri_param_0];
+; CHECK-NEXT:    { // callseq 35, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {1, 2, %rs1, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 35
+; CHECK-NEXT:    ret;
+  %struct.iiri0 = insertvalue %struct.char4 poison, i8 1, 0
+  %struct.iiri1 = insertvalue %struct.char4 %struct.iiri0, i8 2, 1
+  %struct.iiri2 = insertvalue %struct.char4 %struct.iiri1, i8 %c, 2
+  %struct.iiri3 = insertvalue %struct.char4 %struct.iiri2, i8 4, 3
+  call void @call_v4_i8(%struct.char4 %struct.iiri3)
+  ret void
+}
+define void @st_param_v4_i8_irii(i8 %b) {
+; CHECK-LABEL: st_param_v4_i8_irii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_irii_param_0];
+; CHECK-NEXT:    { // callseq 36, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {1, %rs1, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 36
+; CHECK-NEXT:    ret;
+  %struct.irii0 = insertvalue %struct.char4 poison, i8 1, 0
+  %struct.irii1 = insertvalue %struct.char4 %struct.irii0, i8 %b, 1
+  %struct.irii2 = insertvalue %struct.char4 %struct.irii1, i8 3, 2
+  %struct.irii3 = insertvalue %struct.char4 %struct.irii2, i8 4, 3
+  call void @call_v4_i8(%struct.char4 %struct.irii3)
+  ret void
+}
+define void @st_param_v4_i8_riii(i8 %a) {
+; CHECK-LABEL: st_param_v4_i8_riii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u8 %rs1, [st_param_v4_i8_riii_param_0];
+; CHECK-NEXT:    { // callseq 37, 0
+; CHECK-NEXT:    .param .align 4 .b8 param0[4];
+; CHECK-NEXT:    st.param.v4.b8 [param0+0], {%rs1, 2, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i8,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 37
+; CHECK-NEXT:    ret;
+  %struct.riii0 = insertvalue %struct.char4 poison, i8 %a, 0
+  %struct.riii1 = insertvalue %struct.char4 %struct.riii0, i8 2, 1
+  %struct.riii2 = insertvalue %struct.char4 %struct.riii1, i8 3, 2
+  %struct.riii3 = insertvalue %struct.char4 %struct.riii2, i8 4, 3
+  call void @call_v4_i8(%struct.char4 %struct.riii3)
+  ret void
+}
+
+define void @st_param_v4_i16_iiii() {
+; CHECK-LABEL: st_param_v4_i16_iiii(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 38, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {1, 2, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 38
+; CHECK-NEXT:    ret;
+  call void @call_v4_i16(%struct.short4 { i16 1, i16 2, i16 3, i16 4 })
+  ret void
+}
+define void @st_param_v4_i16_irrr(i16 %b, i16 %c, i16 %d) {
+; CHECK-LABEL: st_param_v4_i16_irrr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_irrr_param_0];
+; CHECK-NEXT:    ld.param.u16 %rs2, [st_param_v4_i16_irrr_param_1];
+; CHECK-NEXT:    ld.param.u16 %rs3, [st_param_v4_i16_irrr_param_2];
+; CHECK-NEXT:    { // callseq 39, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {1, %rs1, %rs2, %rs3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 39
+; CHECK-NEXT:    ret;
+  %struct.irrr0 = insertvalue %struct.short4 poison, i16 1, 0
+  %struct.irrr1 = insertvalue %struct.short4 %struct.irrr0, i16 %b, 1
+  %struct.irrr2 = insertvalue %struct.short4 %struct.irrr1, i16 %c, 2
+  %struct.irrr3 = insertvalue %struct.short4 %struct.irrr2, i16 %d, 3
+  call void @call_v4_i16(%struct.short4 %struct.irrr3)
+  ret void
+}
+define void @st_param_v4_i16_rirr(i16 %a, i16 %c, i16 %d) {
+; CHECK-LABEL: st_param_v4_i16_rirr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_rirr_param_0];
+; CHECK-NEXT:    ld.param.u16 %rs2, [st_param_v4_i16_rirr_param_1];
+; CHECK-NEXT:    ld.param.u16 %rs3, [st_param_v4_i16_rirr_param_2];
+; CHECK-NEXT:    { // callseq 40, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {%rs1, 2, %rs2, %rs3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 40
+; CHECK-NEXT:    ret;
+  %struct.rirr0 = insertvalue %struct.short4 poison, i16 %a, 0
+  %struct.rirr1 = insertvalue %struct.short4 %struct.rirr0, i16 2, 1
+  %struct.rirr2 = insertvalue %struct.short4 %struct.rirr1, i16 %c, 2
+  %struct.rirr3 = insertvalue %struct.short4 %struct.rirr2, i16 %d, 3
+  call void @call_v4_i16(%struct.short4 %struct.rirr3)
+  ret void
+}
+define void @st_param_v4_i16_rrir(i16 %a, i16 %b, i16 %d) {
+; CHECK-LABEL: st_param_v4_i16_rrir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_rrir_param_0];
+; CHECK-NEXT:    ld.param.u16 %rs2, [st_param_v4_i16_rrir_param_1];
+; CHECK-NEXT:    ld.param.u16 %rs3, [st_param_v4_i16_rrir_param_2];
+; CHECK-NEXT:    { // callseq 41, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {%rs1, %rs2, 3, %rs3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 41
+; CHECK-NEXT:    ret;
+  %struct.rrir0 = insertvalue %struct.short4 poison, i16 %a, 0
+  %struct.rrir1 = insertvalue %struct.short4 %struct.rrir0, i16 %b, 1
+  %struct.rrir2 = insertvalue %struct.short4 %struct.rrir1, i16 3, 2
+  %struct.rrir3 = insertvalue %struct.short4 %struct.rrir2, i16 %d, 3
+  call void @call_v4_i16(%struct.short4 %struct.rrir3)
+  ret void
+}
+define void @st_param_v4_i16_rrri(i16 %a, i16 %b, i16 %c) {
+; CHECK-LABEL: st_param_v4_i16_rrri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_rrri_param_0];
+; CHECK-NEXT:    ld.param.u16 %rs2, [st_param_v4_i16_rrri_param_1];
+; CHECK-NEXT:    ld.param.u16 %rs3, [st_param_v4_i16_rrri_param_2];
+; CHECK-NEXT:    { // callseq 42, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {%rs1, %rs2, %rs3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 42
+; CHECK-NEXT:    ret;
+  %struct.rrri0 = insertvalue %struct.short4 poison, i16 %a, 0
+  %struct.rrri1 = insertvalue %struct.short4 %struct.rrri0, i16 %b, 1
+  %struct.rrri2 = insertvalue %struct.short4 %struct.rrri1, i16 %c, 2
+  %struct.rrri3 = insertvalue %struct.short4 %struct.rrri2, i16 4, 3
+  call void @call_v4_i16(%struct.short4 %struct.rrri3)
+  ret void
+}
+define void @st_param_v4_i16_iirr(i16 %c, i16 %d) {
+; CHECK-LABEL: st_param_v4_i16_iirr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_iirr_param_0];
+; CHECK-NEXT:    ld.param.u16 %rs2, [st_param_v4_i16_iirr_param_1];
+; CHECK-NEXT:    { // callseq 43, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {1, 2, %rs1, %rs2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 43
+; CHECK-NEXT:    ret;
+  %struct.iirr0 = insertvalue %struct.short4 poison, i16 1, 0
+  %struct.iirr1 = insertvalue %struct.short4 %struct.iirr0, i16 2, 1
+  %struct.iirr2 = insertvalue %struct.short4 %struct.iirr1, i16 %c, 2
+  %struct.iirr3 = insertvalue %struct.short4 %struct.iirr2, i16 %d, 3
+  call void @call_v4_i16(%struct.short4 %struct.iirr3)
+  ret void
+}
+define void @st_param_v4_i16_irir(i16 %b, i16 %d) {
+; CHECK-LABEL: st_param_v4_i16_irir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_irir_param_0];
+; CHECK-NEXT:    ld.param.u16 %rs2, [st_param_v4_i16_irir_param_1];
+; CHECK-NEXT:    { // callseq 44, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {1, %rs1, 3, %rs2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 44
+; CHECK-NEXT:    ret;
+  %struct.irir0 = insertvalue %struct.short4 poison, i16 1, 0
+  %struct.irir1 = insertvalue %struct.short4 %struct.irir0, i16 %b, 1
+  %struct.irir2 = insertvalue %struct.short4 %struct.irir1, i16 3, 2
+  %struct.irir3 = insertvalue %struct.short4 %struct.irir2, i16 %d, 3
+  call void @call_v4_i16(%struct.short4 %struct.irir3)
+  ret void
+}
+define void @st_param_v4_i16_irri(i16 %b, i16 %c) {
+; CHECK-LABEL: st_param_v4_i16_irri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_irri_param_0];
+; CHECK-NEXT:    ld.param.u16 %rs2, [st_param_v4_i16_irri_param_1];
+; CHECK-NEXT:    { // callseq 45, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {1, %rs1, %rs2, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 45
+; CHECK-NEXT:    ret;
+  %struct.irri0 = insertvalue %struct.short4 poison, i16 1, 0
+  %struct.irri1 = insertvalue %struct.short4 %struct.irri0, i16 %b, 1
+  %struct.irri2 = insertvalue %struct.short4 %struct.irri1, i16 %c, 2
+  %struct.irri3 = insertvalue %struct.short4 %struct.irri2, i16 4, 3
+  call void @call_v4_i16(%struct.short4 %struct.irri3)
+  ret void
+}
+define void @st_param_v4_i16_riir(i16 %a, i16 %d) {
+; CHECK-LABEL: st_param_v4_i16_riir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_riir_param_0];
+; CHECK-NEXT:    ld.param.u16 %rs2, [st_param_v4_i16_riir_param_1];
+; CHECK-NEXT:    { // callseq 46, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {%rs1, 2, 3, %rs2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 46
+; CHECK-NEXT:    ret;
+  %struct.riir0 = insertvalue %struct.short4 poison, i16 %a, 0
+  %struct.riir1 = insertvalue %struct.short4 %struct.riir0, i16 2, 1
+  %struct.riir2 = insertvalue %struct.short4 %struct.riir1, i16 3, 2
+  %struct.riir3 = insertvalue %struct.short4 %struct.riir2, i16 %d, 3
+  call void @call_v4_i16(%struct.short4 %struct.riir3)
+  ret void
+}
+define void @st_param_v4_i16_riri(i16 %a, i16 %c) {
+; CHECK-LABEL: st_param_v4_i16_riri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_riri_param_0];
+; CHECK-NEXT:    ld.param.u16 %rs2, [st_param_v4_i16_riri_param_1];
+; CHECK-NEXT:    { // callseq 47, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {%rs1, 2, %rs2, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 47
+; CHECK-NEXT:    ret;
+  %struct.riri0 = insertvalue %struct.short4 poison, i16 %a, 0
+  %struct.riri1 = insertvalue %struct.short4 %struct.riri0, i16 2, 1
+  %struct.riri2 = insertvalue %struct.short4 %struct.riri1, i16 %c, 2
+  %struct.riri3 = insertvalue %struct.short4 %struct.riri2, i16 4, 3
+  call void @call_v4_i16(%struct.short4 %struct.riri3)
+  ret void
+}
+define void @st_param_v4_i16_rrii(i16 %a, i16 %b) {
+; CHECK-LABEL: st_param_v4_i16_rrii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_rrii_param_0];
+; CHECK-NEXT:    ld.param.u16 %rs2, [st_param_v4_i16_rrii_param_1];
+; CHECK-NEXT:    { // callseq 48, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {%rs1, %rs2, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 48
+; CHECK-NEXT:    ret;
+  %struct.rrii0 = insertvalue %struct.short4 poison, i16 %a, 0
+  %struct.rrii1 = insertvalue %struct.short4 %struct.rrii0, i16 %b, 1
+  %struct.rrii2 = insertvalue %struct.short4 %struct.rrii1, i16 3, 2
+  %struct.rrii3 = insertvalue %struct.short4 %struct.rrii2, i16 4, 3
+  call void @call_v4_i16(%struct.short4 %struct.rrii3)
+  ret void
+}
+define void @st_param_v4_i16_iiir(i16 %d) {
+; CHECK-LABEL: st_param_v4_i16_iiir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_iiir_param_0];
+; CHECK-NEXT:    { // callseq 49, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {1, 2, 3, %rs1};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 49
+; CHECK-NEXT:    ret;
+  %struct.iiir0 = insertvalue %struct.short4 poison, i16 1, 0
+  %struct.iiir1 = insertvalue %struct.short4 %struct.iiir0, i16 2, 1
+  %struct.iiir2 = insertvalue %struct.short4 %struct.iiir1, i16 3, 2
+  %struct.iiir3 = insertvalue %struct.short4 %struct.iiir2, i16 %d, 3
+  call void @call_v4_i16(%struct.short4 %struct.iiir3)
+  ret void
+}
+define void @st_param_v4_i16_iiri(i16 %c) {
+; CHECK-LABEL: st_param_v4_i16_iiri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_iiri_param_0];
+; CHECK-NEXT:    { // callseq 50, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {1, 2, %rs1, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 50
+; CHECK-NEXT:    ret;
+  %struct.iiri0 = insertvalue %struct.short4 poison, i16 1, 0
+  %struct.iiri1 = insertvalue %struct.short4 %struct.iiri0, i16 2, 1
+  %struct.iiri2 = insertvalue %struct.short4 %struct.iiri1, i16 %c, 2
+  %struct.iiri3 = insertvalue %struct.short4 %struct.iiri2, i16 4, 3
+  call void @call_v4_i16(%struct.short4 %struct.iiri3)
+  ret void
+}
+define void @st_param_v4_i16_irii(i16 %b) {
+; CHECK-LABEL: st_param_v4_i16_irii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_irii_param_0];
+; CHECK-NEXT:    { // callseq 51, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {1, %rs1, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 51
+; CHECK-NEXT:    ret;
+  %struct.irii0 = insertvalue %struct.short4 poison, i16 1, 0
+  %struct.irii1 = insertvalue %struct.short4 %struct.irii0, i16 %b, 1
+  %struct.irii2 = insertvalue %struct.short4 %struct.irii1, i16 3, 2
+  %struct.irii3 = insertvalue %struct.short4 %struct.irii2, i16 4, 3
+  call void @call_v4_i16(%struct.short4 %struct.irii3)
+  ret void
+}
+define void @st_param_v4_i16_riii(i16 %a) {
+; CHECK-LABEL: st_param_v4_i16_riii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [st_param_v4_i16_riii_param_0];
+; CHECK-NEXT:    { // callseq 52, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NEXT:    st.param.v4.b16 [param0+0], {%rs1, 2, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i16,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 52
+; CHECK-NEXT:    ret;
+  %struct.riii0 = insertvalue %struct.short4 poison, i16 %a, 0
+  %struct.riii1 = insertvalue %struct.short4 %struct.riii0, i16 2, 1
+  %struct.riii2 = insertvalue %struct.short4 %struct.riii1, i16 3, 2
+  %struct.riii3 = insertvalue %struct.short4 %struct.riii2, i16 4, 3
+  call void @call_v4_i16(%struct.short4 %struct.riii3)
+  ret void
+}
+
+define void @st_param_v4_i32_iiii() {
+; CHECK-LABEL: st_param_v4_i32_iiii(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 53, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {1, 2, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 53
+; CHECK-NEXT:    ret;
+  call void @call_v4_i32(%struct.int4 { i32 1, i32 2, i32 3, i32 4 })
+  ret void
+}
+define void @st_param_v4_i32_irrr(i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: st_param_v4_i32_irrr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_irrr_param_0];
+; CHECK-NEXT:    ld.param.u32 %r2, [st_param_v4_i32_irrr_param_1];
+; CHECK-NEXT:    ld.param.u32 %r3, [st_param_v4_i32_irrr_param_2];
+; CHECK-NEXT:    { // callseq 54, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {1, %r1, %r2, %r3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 54
+; CHECK-NEXT:    ret;
+  %struct.irrr0 = insertvalue %struct.int4 poison, i32 1, 0
+  %struct.irrr1 = insertvalue %struct.int4 %struct.irrr0, i32 %b, 1
+  %struct.irrr2 = insertvalue %struct.int4 %struct.irrr1, i32 %c, 2
+  %struct.irrr3 = insertvalue %struct.int4 %struct.irrr2, i32 %d, 3
+  call void @call_v4_i32(%struct.int4 %struct.irrr3)
+  ret void
+}
+define void @st_param_v4_i32_rirr(i32 %a, i32 %c, i32 %d) {
+; CHECK-LABEL: st_param_v4_i32_rirr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_rirr_param_0];
+; CHECK-NEXT:    ld.param.u32 %r2, [st_param_v4_i32_rirr_param_1];
+; CHECK-NEXT:    ld.param.u32 %r3, [st_param_v4_i32_rirr_param_2];
+; CHECK-NEXT:    { // callseq 55, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {%r1, 2, %r2, %r3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 55
+; CHECK-NEXT:    ret;
+  %struct.rirr0 = insertvalue %struct.int4 poison, i32 %a, 0
+  %struct.rirr1 = insertvalue %struct.int4 %struct.rirr0, i32 2, 1
+  %struct.rirr2 = insertvalue %struct.int4 %struct.rirr1, i32 %c, 2
+  %struct.rirr3 = insertvalue %struct.int4 %struct.rirr2, i32 %d, 3
+  call void @call_v4_i32(%struct.int4 %struct.rirr3)
+  ret void
+}
+define void @st_param_v4_i32_rrir(i32 %a, i32 %b, i32 %d) {
+; CHECK-LABEL: st_param_v4_i32_rrir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_rrir_param_0];
+; CHECK-NEXT:    ld.param.u32 %r2, [st_param_v4_i32_rrir_param_1];
+; CHECK-NEXT:    ld.param.u32 %r3, [st_param_v4_i32_rrir_param_2];
+; CHECK-NEXT:    { // callseq 56, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {%r1, %r2, 3, %r3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 56
+; CHECK-NEXT:    ret;
+  %struct.rrir0 = insertvalue %struct.int4 poison, i32 %a, 0
+  %struct.rrir1 = insertvalue %struct.int4 %struct.rrir0, i32 %b, 1
+  %struct.rrir2 = insertvalue %struct.int4 %struct.rrir1, i32 3, 2
+  %struct.rrir3 = insertvalue %struct.int4 %struct.rrir2, i32 %d, 3
+  call void @call_v4_i32(%struct.int4 %struct.rrir3)
+  ret void
+}
+define void @st_param_v4_i32_rrri(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: st_param_v4_i32_rrri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_rrri_param_0];
+; CHECK-NEXT:    ld.param.u32 %r2, [st_param_v4_i32_rrri_param_1];
+; CHECK-NEXT:    ld.param.u32 %r3, [st_param_v4_i32_rrri_param_2];
+; CHECK-NEXT:    { // callseq 57, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {%r1, %r2, %r3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 57
+; CHECK-NEXT:    ret;
+  %struct.rrri0 = insertvalue %struct.int4 poison, i32 %a, 0
+  %struct.rrri1 = insertvalue %struct.int4 %struct.rrri0, i32 %b, 1
+  %struct.rrri2 = insertvalue %struct.int4 %struct.rrri1, i32 %c, 2
+  %struct.rrri3 = insertvalue %struct.int4 %struct.rrri2, i32 4, 3
+  call void @call_v4_i32(%struct.int4 %struct.rrri3)
+  ret void
+}
+define void @st_param_v4_i32_iirr(i32 %c, i32 %d) {
+; CHECK-LABEL: st_param_v4_i32_iirr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_iirr_param_0];
+; CHECK-NEXT:    ld.param.u32 %r2, [st_param_v4_i32_iirr_param_1];
+; CHECK-NEXT:    { // callseq 58, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {1, 2, %r1, %r2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 58
+; CHECK-NEXT:    ret;
+  %struct.iirr0 = insertvalue %struct.int4 poison, i32 1, 0
+  %struct.iirr1 = insertvalue %struct.int4 %struct.iirr0, i32 2, 1
+  %struct.iirr2 = insertvalue %struct.int4 %struct.iirr1, i32 %c, 2
+  %struct.iirr3 = insertvalue %struct.int4 %struct.iirr2, i32 %d, 3
+  call void @call_v4_i32(%struct.int4 %struct.iirr3)
+  ret void
+}
+define void @st_param_v4_i32_irir(i32 %b, i32 %d) {
+; CHECK-LABEL: st_param_v4_i32_irir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_irir_param_0];
+; CHECK-NEXT:    ld.param.u32 %r2, [st_param_v4_i32_irir_param_1];
+; CHECK-NEXT:    { // callseq 59, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {1, %r1, 3, %r2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 59
+; CHECK-NEXT:    ret;
+  %struct.irir0 = insertvalue %struct.int4 poison, i32 1, 0
+  %struct.irir1 = insertvalue %struct.int4 %struct.irir0, i32 %b, 1
+  %struct.irir2 = insertvalue %struct.int4 %struct.irir1, i32 3, 2
+  %struct.irir3 = insertvalue %struct.int4 %struct.irir2, i32 %d, 3
+  call void @call_v4_i32(%struct.int4 %struct.irir3)
+  ret void
+}
+define void @st_param_v4_i32_irri(i32 %b, i32 %c) {
+; CHECK-LABEL: st_param_v4_i32_irri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_irri_param_0];
+; CHECK-NEXT:    ld.param.u32 %r2, [st_param_v4_i32_irri_param_1];
+; CHECK-NEXT:    { // callseq 60, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {1, %r1, %r2, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 60
+; CHECK-NEXT:    ret;
+  %struct.irri0 = insertvalue %struct.int4 poison, i32 1, 0
+  %struct.irri1 = insertvalue %struct.int4 %struct.irri0, i32 %b, 1
+  %struct.irri2 = insertvalue %struct.int4 %struct.irri1, i32 %c, 2
+  %struct.irri3 = insertvalue %struct.int4 %struct.irri2, i32 4, 3
+  call void @call_v4_i32(%struct.int4 %struct.irri3)
+  ret void
+}
+define void @st_param_v4_i32_riir(i32 %a, i32 %d) {
+; CHECK-LABEL: st_param_v4_i32_riir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_riir_param_0];
+; CHECK-NEXT:    ld.param.u32 %r2, [st_param_v4_i32_riir_param_1];
+; CHECK-NEXT:    { // callseq 61, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {%r1, 2, 3, %r2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 61
+; CHECK-NEXT:    ret;
+  %struct.riir0 = insertvalue %struct.int4 poison, i32 %a, 0
+  %struct.riir1 = insertvalue %struct.int4 %struct.riir0, i32 2, 1
+  %struct.riir2 = insertvalue %struct.int4 %struct.riir1, i32 3, 2
+  %struct.riir3 = insertvalue %struct.int4 %struct.riir2, i32 %d, 3
+  call void @call_v4_i32(%struct.int4 %struct.riir3)
+  ret void
+}
+define void @st_param_v4_i32_riri(i32 %a, i32 %c) {
+; CHECK-LABEL: st_param_v4_i32_riri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_riri_param_0];
+; CHECK-NEXT:    ld.param.u32 %r2, [st_param_v4_i32_riri_param_1];
+; CHECK-NEXT:    { // callseq 62, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {%r1, 2, %r2, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 62
+; CHECK-NEXT:    ret;
+  %struct.riri0 = insertvalue %struct.int4 poison, i32 %a, 0
+  %struct.riri1 = insertvalue %struct.int4 %struct.riri0, i32 2, 1
+  %struct.riri2 = insertvalue %struct.int4 %struct.riri1, i32 %c, 2
+  %struct.riri3 = insertvalue %struct.int4 %struct.riri2, i32 4, 3
+  call void @call_v4_i32(%struct.int4 %struct.riri3)
+  ret void
+}
+define void @st_param_v4_i32_rrii(i32 %a, i32 %b) {
+; CHECK-LABEL: st_param_v4_i32_rrii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_rrii_param_0];
+; CHECK-NEXT:    ld.param.u32 %r2, [st_param_v4_i32_rrii_param_1];
+; CHECK-NEXT:    { // callseq 63, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {%r1, %r2, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 63
+; CHECK-NEXT:    ret;
+  %struct.rrii0 = insertvalue %struct.int4 poison, i32 %a, 0
+  %struct.rrii1 = insertvalue %struct.int4 %struct.rrii0, i32 %b, 1
+  %struct.rrii2 = insertvalue %struct.int4 %struct.rrii1, i32 3, 2
+  %struct.rrii3 = insertvalue %struct.int4 %struct.rrii2, i32 4, 3
+  call void @call_v4_i32(%struct.int4 %struct.rrii3)
+  ret void
+}
+define void @st_param_v4_i32_iiir(i32 %d) {
+; CHECK-LABEL: st_param_v4_i32_iiir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_iiir_param_0];
+; CHECK-NEXT:    { // callseq 64, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {1, 2, 3, %r1};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 64
+; CHECK-NEXT:    ret;
+  %struct.iiir0 = insertvalue %struct.int4 poison, i32 1, 0
+  %struct.iiir1 = insertvalue %struct.int4 %struct.iiir0, i32 2, 1
+  %struct.iiir2 = insertvalue %struct.int4 %struct.iiir1, i32 3, 2
+  %struct.iiir3 = insertvalue %struct.int4 %struct.iiir2, i32 %d, 3
+  call void @call_v4_i32(%struct.int4 %struct.iiir3)
+  ret void
+}
+define void @st_param_v4_i32_iiri(i32 %c) {
+; CHECK-LABEL: st_param_v4_i32_iiri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_iiri_param_0];
+; CHECK-NEXT:    { // callseq 65, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {1, 2, %r1, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 65
+; CHECK-NEXT:    ret;
+  %struct.iiri0 = insertvalue %struct.int4 poison, i32 1, 0
+  %struct.iiri1 = insertvalue %struct.int4 %struct.iiri0, i32 2, 1
+  %struct.iiri2 = insertvalue %struct.int4 %struct.iiri1, i32 %c, 2
+  %struct.iiri3 = insertvalue %struct.int4 %struct.iiri2, i32 4, 3
+  call void @call_v4_i32(%struct.int4 %struct.iiri3)
+  ret void
+}
+define void @st_param_v4_i32_irii(i32 %b) {
+; CHECK-LABEL: st_param_v4_i32_irii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_irii_param_0];
+; CHECK-NEXT:    { // callseq 66, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {1, %r1, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 66
+; CHECK-NEXT:    ret;
+  %struct.irii0 = insertvalue %struct.int4 poison, i32 1, 0
+  %struct.irii1 = insertvalue %struct.int4 %struct.irii0, i32 %b, 1
+  %struct.irii2 = insertvalue %struct.int4 %struct.irii1, i32 3, 2
+  %struct.irii3 = insertvalue %struct.int4 %struct.irii2, i32 4, 3
+  call void @call_v4_i32(%struct.int4 %struct.irii3)
+  ret void
+}
+define void @st_param_v4_i32_riii(i32 %a) {
+; CHECK-LABEL: st_param_v4_i32_riii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [st_param_v4_i32_riii_param_0];
+; CHECK-NEXT:    { // callseq 67, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.b32 [param0+0], {%r1, 2, 3, 4};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_i32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 67
+; CHECK-NEXT:    ret;
+  %struct.riii0 = insertvalue %struct.int4 poison, i32 %a, 0
+  %struct.riii1 = insertvalue %struct.int4 %struct.riii0, i32 2, 1
+  %struct.riii2 = insertvalue %struct.int4 %struct.riii1, i32 3, 2
+  %struct.riii3 = insertvalue %struct.int4 %struct.riii2, i32 4, 3
+  call void @call_v4_i32(%struct.int4 %struct.riii3)
+  ret void
+}
+
+define void @st_param_v4_f32_iiii() {
+; CHECK-LABEL: st_param_v4_f32_iiii(
+; CHECK:       {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    { // callseq 68, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {0f3F800000, 0f40000000, 0f40400000, 0f40800000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 68
+; CHECK-NEXT:    ret;
+  call void @call_v4_f32(%struct.float4 { float 1.0, float 2.0, float 3.0, float 4.0 })
+  ret void
+}
+define void @st_param_v4_f32_irrr(float %b, float %c, float %d) {
+; CHECK-LABEL: st_param_v4_f32_irrr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_irrr_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [st_param_v4_f32_irrr_param_1];
+; CHECK-NEXT:    ld.param.f32 %f3, [st_param_v4_f32_irrr_param_2];
+; CHECK-NEXT:    { // callseq 69, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {0f3F800000, %f1, %f2, %f3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 69
+; CHECK-NEXT:    ret;
+  %struct.irrr0 = insertvalue %struct.float4 poison, float 1.0, 0
+  %struct.irrr1 = insertvalue %struct.float4 %struct.irrr0, float %b, 1
+  %struct.irrr2 = insertvalue %struct.float4 %struct.irrr1, float %c, 2
+  %struct.irrr3 = insertvalue %struct.float4 %struct.irrr2, float %d, 3
+  call void @call_v4_f32(%struct.float4 %struct.irrr3)
+  ret void
+}
+define void @st_param_v4_f32_rirr(float %a, float %c, float %d) {
+; CHECK-LABEL: st_param_v4_f32_rirr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_rirr_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [st_param_v4_f32_rirr_param_1];
+; CHECK-NEXT:    ld.param.f32 %f3, [st_param_v4_f32_rirr_param_2];
+; CHECK-NEXT:    { // callseq 70, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {%f1, 0f40000000, %f2, %f3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 70
+; CHECK-NEXT:    ret;
+  %struct.rirr0 = insertvalue %struct.float4 poison, float %a, 0
+  %struct.rirr1 = insertvalue %struct.float4 %struct.rirr0, float 2.0, 1
+  %struct.rirr2 = insertvalue %struct.float4 %struct.rirr1, float %c, 2
+  %struct.rirr3 = insertvalue %struct.float4 %struct.rirr2, float %d, 3
+  call void @call_v4_f32(%struct.float4 %struct.rirr3)
+  ret void
+}
+define void @st_param_v4_f32_rrir(float %a, float %b, float %d) {
+; CHECK-LABEL: st_param_v4_f32_rrir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_rrir_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [st_param_v4_f32_rrir_param_1];
+; CHECK-NEXT:    ld.param.f32 %f3, [st_param_v4_f32_rrir_param_2];
+; CHECK-NEXT:    { // callseq 71, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {%f1, %f2, 0f40400000, %f3};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 71
+; CHECK-NEXT:    ret;
+  %struct.rrir0 = insertvalue %struct.float4 poison, float %a, 0
+  %struct.rrir1 = insertvalue %struct.float4 %struct.rrir0, float %b, 1
+  %struct.rrir2 = insertvalue %struct.float4 %struct.rrir1, float 3.0, 2
+  %struct.rrir3 = insertvalue %struct.float4 %struct.rrir2, float %d, 3
+  call void @call_v4_f32(%struct.float4 %struct.rrir3)
+  ret void
+}
+define void @st_param_v4_f32_rrri(float %a, float %b, float %c) {
+; CHECK-LABEL: st_param_v4_f32_rrri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_rrri_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [st_param_v4_f32_rrri_param_1];
+; CHECK-NEXT:    ld.param.f32 %f3, [st_param_v4_f32_rrri_param_2];
+; CHECK-NEXT:    { // callseq 72, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {%f1, %f2, %f3, 0f40800000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 72
+; CHECK-NEXT:    ret;
+  %struct.rrri0 = insertvalue %struct.float4 poison, float %a, 0
+  %struct.rrri1 = insertvalue %struct.float4 %struct.rrri0, float %b, 1
+  %struct.rrri2 = insertvalue %struct.float4 %struct.rrri1, float %c, 2
+  %struct.rrri3 = insertvalue %struct.float4 %struct.rrri2, float 4.0, 3
+  call void @call_v4_f32(%struct.float4 %struct.rrri3)
+  ret void
+}
+define void @st_param_v4_f32_iirr(float %c, float %d) {
+; CHECK-LABEL: st_param_v4_f32_iirr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_iirr_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [st_param_v4_f32_iirr_param_1];
+; CHECK-NEXT:    { // callseq 73, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {0f3F800000, 0f40000000, %f1, %f2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 73
+; CHECK-NEXT:    ret;
+  %struct.iirr0 = insertvalue %struct.float4 poison, float 1.0, 0
+  %struct.iirr1 = insertvalue %struct.float4 %struct.iirr0, float 2.0, 1
+  %struct.iirr2 = insertvalue %struct.float4 %struct.iirr1, float %c, 2
+  %struct.iirr3 = insertvalue %struct.float4 %struct.iirr2, float %d, 3
+  call void @call_v4_f32(%struct.float4 %struct.iirr3)
+  ret void
+}
+define void @st_param_v4_f32_irir(float %b, float %d) {
+; CHECK-LABEL: st_param_v4_f32_irir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_irir_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [st_param_v4_f32_irir_param_1];
+; CHECK-NEXT:    { // callseq 74, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {0f3F800000, %f1, 0f40400000, %f2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 74
+; CHECK-NEXT:    ret;
+  %struct.irir0 = insertvalue %struct.float4 poison, float 1.0, 0
+  %struct.irir1 = insertvalue %struct.float4 %struct.irir0, float %b, 1
+  %struct.irir2 = insertvalue %struct.float4 %struct.irir1, float 3.0, 2
+  %struct.irir3 = insertvalue %struct.float4 %struct.irir2, float %d, 3
+  call void @call_v4_f32(%struct.float4 %struct.irir3)
+  ret void
+}
+define void @st_param_v4_f32_irri(float %b, float %c) {
+; CHECK-LABEL: st_param_v4_f32_irri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_irri_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [st_param_v4_f32_irri_param_1];
+; CHECK-NEXT:    { // callseq 75, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {0f3F800000, %f1, %f2, 0f40800000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 75
+; CHECK-NEXT:    ret;
+  %struct.irri0 = insertvalue %struct.float4 poison, float 1.0, 0
+  %struct.irri1 = insertvalue %struct.float4 %struct.irri0, float %b, 1
+  %struct.irri2 = insertvalue %struct.float4 %struct.irri1, float %c, 2
+  %struct.irri3 = insertvalue %struct.float4 %struct.irri2, float 4.0, 3
+  call void @call_v4_f32(%struct.float4 %struct.irri3)
+  ret void
+}
+define void @st_param_v4_f32_riir(float %a, float %d) {
+; CHECK-LABEL: st_param_v4_f32_riir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_riir_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [st_param_v4_f32_riir_param_1];
+; CHECK-NEXT:    { // callseq 76, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {%f1, 0f40000000, 0f40400000, %f2};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 76
+; CHECK-NEXT:    ret;
+  %struct.riir0 = insertvalue %struct.float4 poison, float %a, 0
+  %struct.riir1 = insertvalue %struct.float4 %struct.riir0, float 2.0, 1
+  %struct.riir2 = insertvalue %struct.float4 %struct.riir1, float 3.0, 2
+  %struct.riir3 = insertvalue %struct.float4 %struct.riir2, float %d, 3
+  call void @call_v4_f32(%struct.float4 %struct.riir3)
+  ret void
+}
+define void @st_param_v4_f32_riri(float %a, float %c) {
+; CHECK-LABEL: st_param_v4_f32_riri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_riri_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [st_param_v4_f32_riri_param_1];
+; CHECK-NEXT:    { // callseq 77, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {%f1, 0f40000000, %f2, 0f40800000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 77
+; CHECK-NEXT:    ret;
+  %struct.riri0 = insertvalue %struct.float4 poison, float %a, 0
+  %struct.riri1 = insertvalue %struct.float4 %struct.riri0, float 2.0, 1
+  %struct.riri2 = insertvalue %struct.float4 %struct.riri1, float %c, 2
+  %struct.riri3 = insertvalue %struct.float4 %struct.riri2, float 4.0, 3
+  call void @call_v4_f32(%struct.float4 %struct.riri3)
+  ret void
+}
+define void @st_param_v4_f32_rrii(float %a, float %b) {
+; CHECK-LABEL: st_param_v4_f32_rrii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_rrii_param_0];
+; CHECK-NEXT:    ld.param.f32 %f2, [st_param_v4_f32_rrii_param_1];
+; CHECK-NEXT:    { // callseq 78, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {%f1, %f2, 0f40400000, 0f40800000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 78
+; CHECK-NEXT:    ret;
+  %struct.rrii0 = insertvalue %struct.float4 poison, float %a, 0
+  %struct.rrii1 = insertvalue %struct.float4 %struct.rrii0, float %b, 1
+  %struct.rrii2 = insertvalue %struct.float4 %struct.rrii1, float 3.0, 2
+  %struct.rrii3 = insertvalue %struct.float4 %struct.rrii2, float 4.0, 3
+  call void @call_v4_f32(%struct.float4 %struct.rrii3)
+  ret void
+}
+define void @st_param_v4_f32_iiir(float %d) {
+; CHECK-LABEL: st_param_v4_f32_iiir(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_iiir_param_0];
+; CHECK-NEXT:    { // callseq 79, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {0f3F800000, 0f40000000, 0f40400000, %f1};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 79
+; CHECK-NEXT:    ret;
+  %struct.iiir0 = insertvalue %struct.float4 poison, float 1.0, 0
+  %struct.iiir1 = insertvalue %struct.float4 %struct.iiir0, float 2.0, 1
+  %struct.iiir2 = insertvalue %struct.float4 %struct.iiir1, float 3.0, 2
+  %struct.iiir3 = insertvalue %struct.float4 %struct.iiir2, float %d, 3
+  call void @call_v4_f32(%struct.float4 %struct.iiir3)
+  ret void
+}
+define void @st_param_v4_f32_iiri(float %c) {
+; CHECK-LABEL: st_param_v4_f32_iiri(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_iiri_param_0];
+; CHECK-NEXT:    { // callseq 80, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {0f3F800000, 0f40000000, %f1, 0f40800000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 80
+; CHECK-NEXT:    ret;
+  %struct.iiri0 = insertvalue %struct.float4 poison, float 1.0, 0
+  %struct.iiri1 = insertvalue %struct.float4 %struct.iiri0, float 2.0, 1
+  %struct.iiri2 = insertvalue %struct.float4 %struct.iiri1, float %c, 2
+  %struct.iiri3 = insertvalue %struct.float4 %struct.iiri2, float 4.0, 3
+  call void @call_v4_f32(%struct.float4 %struct.iiri3)
+  ret void
+}
+define void @st_param_v4_f32_irii(float %b) {
+; CHECK-LABEL: st_param_v4_f32_irii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_irii_param_0];
+; CHECK-NEXT:    { // callseq 81, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {0f3F800000, %f1, 0f40400000, 0f40800000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 81
+; CHECK-NEXT:    ret;
+  %struct.irii0 = insertvalue %struct.float4 poison, float 1.0, 0
+  %struct.irii1 = insertvalue %struct.float4 %struct.irii0, float %b, 1
+  %struct.irii2 = insertvalue %struct.float4 %struct.irii1, float 3.0, 2
+  %struct.irii3 = insertvalue %struct.float4 %struct.irii2, float 4.0, 3
+  call void @call_v4_f32(%struct.float4 %struct.irii3)
+  ret void
+}
+define void @st_param_v4_f32_riii(float %a) {
+; CHECK-LABEL: st_param_v4_f32_riii(
+; CHECK:       {
+; CHECK-NEXT:    .reg .f32 %f<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f32 %f1, [st_param_v4_f32_riii_param_0];
+; CHECK-NEXT:    { // callseq 82, 0
+; CHECK-NEXT:    .param .align 16 .b8 param0[16];
+; CHECK-NEXT:    st.param.v4.f32 [param0+0], {%f1, 0f40000000, 0f40400000, 0f40800000};
+; CHECK-NEXT:    call.uni
+; CHECK-NEXT:    call_v4_f32,
+; CHECK-NEXT:    (
+; CHECK-NEXT:    param0
+; CHECK-NEXT:    );
+; CHECK-NEXT:    } // callseq 82
+; CHECK-NEXT:    ret;
+  %struct.riii0 = insertvalue %struct.float4 poison, float %a, 0
+  %struct.riii1 = insertvalue %struct.float4 %struct.riii0, float 2.0, 1
+  %struct.riii2 = insertvalue %struct.float4 %struct.riii1, float 3.0, 2
+  %struct.riii3 = insertvalue %struct.float4 %struct.riii2, float 4.0, 3
+  call void @call_v4_f32(%struct.float4 %struct.riii3)
+  ret void
+}
+
+declare void @call_v4_i8(%struct.char4 alignstack(4))
+declare void @call_v4_i16(%struct.short4 alignstack(8))
+declare void @call_v4_i32(%struct.int4 alignstack(16))
+declare void @call_v4_f32(%struct.float4 alignstack(16))
-- 
GitLab


From 577785c5ca78a9714584b5c99ec085f8aea0a5c0 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 18 May 2024 18:43:20 +0100
Subject: [PATCH 1198/1206] [VPlan] Remove unused removeLastOperand (NFC).

The last use of the function has been removed a while ago. Remove the
unused function.
---
 llvm/lib/Transforms/Vectorize/VPlanValue.h | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 96d04271850f..8d945f6f2b8e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -261,11 +261,6 @@ public:
     New->addUser(*this);
   }
 
-  void removeLastOperand() {
-    VPValue *Op = Operands.pop_back_val();
-    Op->removeUser(*this);
-  }
-
   typedef SmallVectorImpl<VPValue *>::iterator operand_iterator;
   typedef SmallVectorImpl<VPValue *>::const_iterator const_operand_iterator;
   typedef iterator_range<operand_iterator> operand_range;
-- 
GitLab


From 003cebdaccc4ad3a3b6f9e177ee5049c8b6a9cbb Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 18 May 2024 11:29:21 -0700
Subject: [PATCH 1199/1206] [dsymutil] Use operator==(StringRef, StringRef)
 (NFC)

---
 llvm/tools/dsymutil/MachODebugMapParser.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/tools/dsymutil/MachODebugMapParser.cpp b/llvm/tools/dsymutil/MachODebugMapParser.cpp
index 6a9f25681cdd..e28c976d6ace 100644
--- a/llvm/tools/dsymutil/MachODebugMapParser.cpp
+++ b/llvm/tools/dsymutil/MachODebugMapParser.cpp
@@ -301,7 +301,7 @@ void MachODebugMapParser::switchToNewLibDebugMapObject(
 
     if (CurrentDebugMapObject &&
         CurrentDebugMapObject->getType() == MachO::N_LIB &&
-        CurrentDebugMapObject->getObjectFilename().compare(Path.str()) == 0) {
+        CurrentDebugMapObject->getObjectFilename() == Path) {
       return;
     }
 
-- 
GitLab


From 8d3e1022c8883f2bfe9faccb82ce807c1cf4272c Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 18 May 2024 11:39:23 -0700
Subject: [PATCH 1200/1206] [DWARFLinker] Use an implicit conversion of
 SmallString to StringRef (NFC)

---
 llvm/lib/DWARFLinker/Parallel/OutputSections.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/DWARFLinker/Parallel/OutputSections.h b/llvm/lib/DWARFLinker/Parallel/OutputSections.h
index 0e1f2dae54bc..d2e4622aa764 100644
--- a/llvm/lib/DWARFLinker/Parallel/OutputSections.h
+++ b/llvm/lib/DWARFLinker/Parallel/OutputSections.h
@@ -220,7 +220,7 @@ struct SectionDescriptor : SectionDescriptorBase {
   /// Returns section content.
   StringRef getContents() override {
     if (SectionOffsetInsideAsmPrinterOutputStart == 0)
-      return StringRef(Contents.data(), Contents.size());
+      return Contents;
 
     return Contents.slice(SectionOffsetInsideAsmPrinterOutputStart,
                           SectionOffsetInsideAsmPrinterOutputEnd);
-- 
GitLab


From ba8a2ade84f4c1bfc531fe3673470377c038f31d Mon Sep 17 00:00:00 2001
From: Jessica Clarke <jrtc27@jrtc27.com>
Date: Sat, 18 May 2024 20:53:21 +0100
Subject: [PATCH 1201/1206] [DXIL] Use consistent SmallVector parameters

Fixes: 060df78cdbbf70d5a6dfff3af1d435a5a811b886
---
 llvm/lib/Target/DirectX/DXILOpLowering.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index f09e322f88e1..1329308ffec2 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -41,7 +41,7 @@ static bool isVectorArgExpansion(Function &F) {
 }
 
 static SmallVector<Value *> populateOperands(Value *Arg, IRBuilder<> &Builder) {
-  SmallVector<Value *, 4> ExtractedElements;
+  SmallVector<Value *> ExtractedElements;
   auto *VecArg = dyn_cast<FixedVectorType>(Arg->getType());
   for (unsigned I = 0; I < VecArg->getNumElements(); ++I) {
     Value *Index = ConstantInt::get(Type::getInt32Ty(Arg->getContext()), I);
-- 
GitLab


From 4c98f5b439ddd204d8ff1e423104215ebd0e1720 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Sat, 18 May 2024 22:50:19 +0100
Subject: [PATCH 1202/1206] [DAG] Use copysign in frem power-2 fold. (#91751)

As a small addition to #91148, this uses copysign to produce the correct
sign for zero when converting frem to div/trunc/mul when we do not know
that the input is positive (and we care about sign bits). The copysign
lets us get the sign of zero correct.

In testing, the only case this produced different results than fmod was:
frem -inf, 4.0 -> nan vs -nan
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 19 ++--
 llvm/test/CodeGen/AArch64/frem-power2.ll      | 92 +++++++++++++++++--
 llvm/test/CodeGen/ARM/frem-power2.ll          | 24 ++++-
 3 files changed, 117 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 2b181cd3ab1d..2b1dec8205b7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -17386,15 +17386,20 @@ SDValue DAGCombiner::visitFREM(SDNode *N) {
       TLI.isOperationLegalOrCustom(ISD::FMUL, VT) &&
       TLI.isOperationLegalOrCustom(ISD::FDIV, VT) &&
       TLI.isOperationLegalOrCustom(ISD::FTRUNC, VT) &&
-      DAG.isKnownToBeAPowerOfTwoFP(N1) &&
-      (Flags.hasNoSignedZeros() || DAG.cannotBeOrderedNegativeFP(N0))) {
+      DAG.isKnownToBeAPowerOfTwoFP(N1)) {
+    bool NeedsCopySign =
+        !Flags.hasNoSignedZeros() && !DAG.cannotBeOrderedNegativeFP(N0);
     SDValue Div = DAG.getNode(ISD::FDIV, DL, VT, N0, N1);
     SDValue Rnd = DAG.getNode(ISD::FTRUNC, DL, VT, Div);
-    if (TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT))
-      return DAG.getNode(ISD::FMA, DL, VT, DAG.getNode(ISD::FNEG, DL, VT, Rnd),
-                         N1, N0);
-    SDValue Mul = DAG.getNode(ISD::FMUL, DL, VT, Rnd, N1);
-    return DAG.getNode(ISD::FSUB, DL, VT, N0, Mul);
+    SDValue MLA;
+    if (TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
+      MLA = DAG.getNode(ISD::FMA, DL, VT, DAG.getNode(ISD::FNEG, DL, VT, Rnd),
+                        N1, N0);
+    } else {
+      SDValue Mul = DAG.getNode(ISD::FMUL, DL, VT, Rnd, N1);
+      MLA = DAG.getNode(ISD::FSUB, DL, VT, N0, Mul);
+    }
+    return NeedsCopySign ? DAG.getNode(ISD::FCOPYSIGN, DL, VT, MLA, N0) : MLA;
   }
 
   return SDValue();
diff --git a/llvm/test/CodeGen/AArch64/frem-power2.ll b/llvm/test/CodeGen/AArch64/frem-power2.ll
index 402e03c5e265..4192745abd34 100644
--- a/llvm/test/CodeGen/AArch64/frem-power2.ll
+++ b/llvm/test/CodeGen/AArch64/frem-power2.ll
@@ -3,10 +3,22 @@
 ; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define float @frem2(float %x) {
-; CHECK-LABEL: frem2:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s1, #2.00000000
-; CHECK-NEXT:    b fmodf
+; CHECK-SD-LABEL: frem2:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s1, #2.00000000
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    fdiv s2, s0, s1
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s1, s2, s1, s0
+; CHECK-SD-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem2:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    b fmodf
 entry:
   %fmod = frem float %x, 2.0
   ret float %fmod
@@ -311,6 +323,67 @@ entry:
   ret float %fmod
 }
 
+define <4 x float> @frem2_vec(<4 x float> %x) {
+; CHECK-SD-LABEL: frem2_vec:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.4s, #64, lsl #24
+; CHECK-SD-NEXT:    mov v3.16b, v0.16b
+; CHECK-SD-NEXT:    fdiv v2.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    frintz v2.4s, v2.4s
+; CHECK-SD-NEXT:    fmls v3.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    mvni v1.4s, #128, lsl #24
+; CHECK-SD-NEXT:    bit v0.16b, v3.16b, v1.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: frem2_vec:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sub sp, sp, #80
+; CHECK-GI-NEXT:    str d10, [sp, #48] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #56] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    str x30, [sp, #72] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-GI-NEXT:    .cfi_offset w30, -8
+; CHECK-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-GI-NEXT:    .cfi_offset b9, -24
+; CHECK-GI-NEXT:    .cfi_offset b10, -32
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-GI-NEXT:    mov s9, v0.s[2]
+; CHECK-GI-NEXT:    mov s10, v0.s[3]
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    fmov s0, s8
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    fmov s0, s9
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    fmov s1, #2.00000000
+; CHECK-GI-NEXT:    fmov s0, s10
+; CHECK-GI-NEXT:    bl fmodf
+; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    ldr x30, [sp, #72] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #56] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldr d10, [sp, #48] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    mov v1.s[1], v2.s[0]
+; CHECK-GI-NEXT:    ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v1.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v0.s[0]
+; CHECK-GI-NEXT:    mov v0.16b, v1.16b
+; CHECK-GI-NEXT:    add sp, sp, #80
+; CHECK-GI-NEXT:    ret
+entry:
+  %fmod = frem <4 x float> %x, <float 2.0, float 2.0, float 2.0, float 2.0>
+  ret <4 x float> %fmod
+}
+
 define <4 x float> @frem2_nsz_vec(<4 x float> %x) {
 ; CHECK-SD-LABEL: frem2_nsz_vec:
 ; CHECK-SD:       // %bb.0: // %entry
@@ -514,10 +587,15 @@ define float @frem2_constneg_sitofp(float %x, i32 %sa) {
 ; CHECK-SD-LABEL: frem2_constneg_sitofp:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    mov w8, #1 // =0x1
-; CHECK-SD-NEXT:    fmov s0, #-12.50000000
+; CHECK-SD-NEXT:    fmov s1, #-12.50000000
 ; CHECK-SD-NEXT:    lsl w8, w8, w0
-; CHECK-SD-NEXT:    scvtf s1, w8
-; CHECK-SD-NEXT:    b fmodf
+; CHECK-SD-NEXT:    scvtf s0, w8
+; CHECK-SD-NEXT:    fdiv s2, s1, s0
+; CHECK-SD-NEXT:    frintz s2, s2
+; CHECK-SD-NEXT:    fmsub s0, s2, s0, s1
+; CHECK-SD-NEXT:    fabs s0, s0
+; CHECK-SD-NEXT:    fneg s0, s0
+; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: frem2_constneg_sitofp:
 ; CHECK-GI:       // %bb.0: // %entry
diff --git a/llvm/test/CodeGen/ARM/frem-power2.ll b/llvm/test/CodeGen/ARM/frem-power2.ll
index 7f52943175ac..71c2c09c0105 100644
--- a/llvm/test/CodeGen/ARM/frem-power2.ll
+++ b/llvm/test/CodeGen/ARM/frem-power2.ll
@@ -14,13 +14,29 @@ define float @frem4(float %x) {
 ;
 ; CHECK-FP-LABEL: frem4:
 ; CHECK-FP:       @ %bb.0: @ %entry
-; CHECK-FP-NEXT:    mov.w r1, #1082130432
-; CHECK-FP-NEXT:    b fmodf
+; CHECK-FP-NEXT:    vmov.f32 s0, #4.000000e+00
+; CHECK-FP-NEXT:    vmov s2, r0
+; CHECK-FP-NEXT:    lsrs r0, r0, #31
+; CHECK-FP-NEXT:    vdiv.f32 s4, s2, s0
+; CHECK-FP-NEXT:    vrintz.f32 s4, s4
+; CHECK-FP-NEXT:    vfms.f32 s2, s4, s0
+; CHECK-FP-NEXT:    vmov r1, s2
+; CHECK-FP-NEXT:    bfi r1, r0, #31, #1
+; CHECK-FP-NEXT:    mov r0, r1
+; CHECK-FP-NEXT:    bx lr
 ;
 ; CHECK-M33-LABEL: frem4:
 ; CHECK-M33:       @ %bb.0: @ %entry
-; CHECK-M33-NEXT:    mov.w r1, #1082130432
-; CHECK-M33-NEXT:    b fmodf
+; CHECK-M33-NEXT:    vmov.f32 s0, #4.000000e+00
+; CHECK-M33-NEXT:    vmov s2, r0
+; CHECK-M33-NEXT:    lsrs r0, r0, #31
+; CHECK-M33-NEXT:    vdiv.f32 s4, s2, s0
+; CHECK-M33-NEXT:    vrintz.f32 s4, s4
+; CHECK-M33-NEXT:    vmls.f32 s2, s4, s0
+; CHECK-M33-NEXT:    vmov r1, s2
+; CHECK-M33-NEXT:    bfi r1, r0, #31, #1
+; CHECK-M33-NEXT:    mov r0, r1
+; CHECK-M33-NEXT:    bx lr
 entry:
   %fmod = frem float %x, 4.0
   ret float %fmod
-- 
GitLab


From c3677e45222a9461eed0224b99bd8ea19bc52bf6 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Sat, 18 May 2024 23:37:55 +0100
Subject: [PATCH 1203/1206] [VectorCombine] Don't transform single shuffles in
 shuffleToIdentity

This will help in later patches where the checks for operands being
instructions is removed, and might help not remove unnecessary poison lanes.
---
 llvm/lib/Transforms/Vectorize/VectorCombine.cpp          | 3 +++
 llvm/test/Transforms/VectorCombine/X86/select-shuffle.ll | 9 +++++----
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 9d43fb4ab607..15deaf908422 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -1794,6 +1794,9 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
     }
   }
 
+  if (NumVisited <= 1)
+    return false;
+
   // If we got this far, we know the shuffles are superfluous and can be
   // removed. Scan through again and generate the new tree of instructions.
   std::function<Value *(ArrayRef<InstLane>)> Generate =
diff --git a/llvm/test/Transforms/VectorCombine/X86/select-shuffle.ll b/llvm/test/Transforms/VectorCombine/X86/select-shuffle.ll
index 60a6c4b1d9b9..685d661ea6bc 100644
--- a/llvm/test/Transforms/VectorCombine/X86/select-shuffle.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/select-shuffle.ll
@@ -12,11 +12,12 @@ define <4 x double> @PR60649() {
 ; CHECK:       unreachable:
 ; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi <4 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ zeroinitializer, [[UNREACHABLE:%.*]] ]
+; CHECK-NEXT:    [[T0:%.*]] = phi <4 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ zeroinitializer, [[UNREACHABLE:%.*]] ]
 ; CHECK-NEXT:    [[T1:%.*]] = phi <4 x double> [ zeroinitializer, [[ENTRY]] ], [ zeroinitializer, [[UNREACHABLE]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> [[TMP0]], <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP2:%.*]] = fdiv <4 x double> [[TMP0]], <double 0.000000e+00, double 0.000000e+00, double undef, double undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = fmul <4 x double> [[TMP1]], <double 0.000000e+00, double 0.000000e+00, double undef, double undef>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x double> [[T0]], <4 x double> [[T0]], <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[T0]], <4 x double> [[T0]], <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = fdiv <4 x double> [[TMP1]], <double 0.000000e+00, double 0.000000e+00, double undef, double undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <4 x double> [[TMP0]], <double 0.000000e+00, double 0.000000e+00, double undef, double undef>
 ; CHECK-NEXT:    [[T5:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x double> [[T5]]
 ;
-- 
GitLab


From 597ac471cc7da97ccf957362a7e9f7a52d6910ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= <nicolai.haehnle@amd.com>
Date: Sun, 19 May 2024 01:39:47 +0200
Subject: [PATCH 1204/1206] update_test_checks: match IR basic block labels
 (#88979)

Labels are matched using a regexp of the form '^(pattern):', which
requires the addition of a "suffix" concept to NamelessValue.

Aside from that, the key challenge is that block labels are values, and
we typically capture values including the prefix '%'. However, when
labels appear at the start of a basic block, the prefix '%' is not
included, so we must capture block label values *without* the prefix
'%'.

We don't know ahead of time whether an IR value is a label or not. In
most cases, they are prefixed by the word "label" (their type), but this
isn't the case in phi nodes. We solve this issue by leveraging the
two-phase nature of variable generalization: the first pass finds all
occurences of a variable and determines whether the '%' prefix can be
included or not. The second pass does the actual substitution.

This change also unifies the generalization path for assembly with that
for IR and analysis, in the hope that any future changes avoid diverging
those cases future.

I also considered the alternative of trying to detect the phi node case
using more regular expression special cases but ultimately decided
against that because it seemed more fragile, and perhaps the approach of
keeping a tentative prefix that may later be discarded could also be
eventually applied to some metadata and attribute cases.

Note that an early version of this change was reviewed as
https://reviews.llvm.org/D142452, before version numbers were
introduced. This is a substantially updated version of that change.
---
 .../Inputs/phi-labels.ll.expected             |  36 +-
 .../update_test_checks/phi-labels.test        |   2 +-
 llvm/utils/UpdateTestChecks/asm.py            |   5 +-
 llvm/utils/UpdateTestChecks/common.py         | 925 +++++++++---------
 llvm/utils/UpdateTestChecks/isel.py           |   5 +-
 llvm/utils/update_analyze_test_checks.py      |   5 +-
 llvm/utils/update_cc_test_checks.py           |  49 +-
 llvm/utils/update_llc_test_checks.py          |   8 +-
 llvm/utils/update_test_checks.py              |  15 +-
 9 files changed, 534 insertions(+), 516 deletions(-)

diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/phi-labels.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/phi-labels.ll.expected
index 1d21ebe547f6..5e70a6c89d32 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/phi-labels.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/phi-labels.ll.expected
@@ -1,15 +1,15 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -S | FileCheck %s
 
 define i32 @phi_after_label(i1 %cc) {
 ; CHECK-LABEL: define i32 @phi_after_label(
 ; CHECK-SAME: i1 [[CC:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[CC]], label [[THEN:%.*]], label [[END:%.*]]
-; CHECK:       then:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[R:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[THEN]] ]
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[CC]], label %[[THEN:.*]], label %[[END:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br label %[[END]]
+; CHECK:       [[END]]:
+; CHECK-NEXT:    [[R:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ 1, %[[THEN]] ]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
 entry:
@@ -26,14 +26,14 @@ end:
 define void @phi_before_label(i32 %bound) {
 ; CHECK-LABEL: define void @phi_before_label(
 ; CHECK-SAME: i32 [[BOUND:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[CTR_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[CTR_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[CTR_NEXT]] = add i32 [[CTR]], 1
 ; CHECK-NEXT:    [[CC:%.*]] = icmp ult i32 [[CTR_NEXT]], [[BOUND]]
-; CHECK-NEXT:    br i1 [[CC]], label [[LOOP]], label [[END:%.*]]
-; CHECK:       end:
+; CHECK-NEXT:    br i1 [[CC]], label %[[LOOP]], label %[[END:.*]]
+; CHECK:       [[END]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -52,11 +52,11 @@ end:
 define i32 @phi_after_label_unnamed(i1 %cc) {
 ; CHECK-LABEL: define i32 @phi_after_label_unnamed(
 ; CHECK-SAME: i1 [[CC:%.*]]) {
-; CHECK-NEXT:    br i1 [[CC]], label [[TMP1:%.*]], label [[TMP2:%.*]]
-; CHECK:       1:
-; CHECK-NEXT:    br label [[TMP2]]
-; CHECK:       2:
-; CHECK-NEXT:    [[R:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ 1, [[TMP1]] ]
+; CHECK-NEXT:    br i1 [[CC]], label %[[BB1:.*]], label %[[BB2:.*]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    br label %[[BB2]]
+; CHECK:       [[BB2]]:
+; CHECK-NEXT:    [[R:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ 1, %[[BB1]] ]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
 0:
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/phi-labels.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/phi-labels.test
index 411c84de1dcb..2b0d0cb7f54b 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/phi-labels.test
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/phi-labels.test
@@ -1,4 +1,4 @@
-# RUN: cp -f %S/Inputs/phi-labels.ll %t.ll && %update_test_checks --version 4 %t.ll
+# RUN: cp -f %S/Inputs/phi-labels.ll %t.ll && %update_test_checks --version 5 %t.ll
 # RUN: diff -u %t.ll %S/Inputs/phi-labels.ll.expected
 ## Check that running the script again does not change the result:
 # RUN: %update_test_checks %t.ll
diff --git a/llvm/utils/UpdateTestChecks/asm.py b/llvm/utils/UpdateTestChecks/asm.py
index f0c456a1648d..33ede81a4160 100644
--- a/llvm/utils/UpdateTestChecks/asm.py
+++ b/llvm/utils/UpdateTestChecks/asm.py
@@ -605,6 +605,7 @@ def add_checks(
     prefix_list,
     func_dict,
     func_name,
+    ginfo: common.GeneralizerInfo,
     global_vars_seen_dict,
     is_filtered,
 ):
@@ -617,9 +618,7 @@ def add_checks(
         func_dict,
         func_name,
         check_label_format,
-        True,
-        False,
-        1,
+        ginfo,
         global_vars_seen_dict,
         is_filtered=is_filtered,
     )
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index 5595e6f41755..7da16e0f0cb2 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -30,8 +30,9 @@ Version changelog:
    in case arguments are split to a separate SAME line.
 4: --check-globals now has a third option ('smart'). The others are now called
    'none' and 'all'. 'smart' is the default.
+5: Basic block labels are matched by FileCheck expressions
 """
-DEFAULT_VERSION = 4
+DEFAULT_VERSION = 5
 
 
 SUPPORTED_ANALYSES = {
@@ -698,6 +699,7 @@ class function_body(object):
         args_and_sig,
         attrs,
         func_name_separator,
+        ginfo,
     ):
         self.scrub = string
         self.extrascrub = extra
@@ -705,24 +707,27 @@ class function_body(object):
         self.args_and_sig = args_and_sig
         self.attrs = attrs
         self.func_name_separator = func_name_separator
+        self._ginfo = ginfo
 
     def is_same_except_arg_names(
-        self, extrascrub, funcdef_attrs_and_ret, args_and_sig, attrs, is_backend
+        self, extrascrub, funcdef_attrs_and_ret, args_and_sig, attrs
     ):
         arg_names = set()
 
         def drop_arg_names(match):
-            arg_names.add(match.group(variable_group_in_ir_value_match))
-            if match.group(attribute_group_in_ir_value_match):
-                attr = match.group(attribute_group_in_ir_value_match)
+            nameless_value = self._ginfo.get_nameless_value_from_match(match)
+            if nameless_value.check_key == "%":
+                arg_names.add(self._ginfo.get_name_from_match(match))
+                substitute = ""
             else:
-                attr = ""
-            return match.group(1) + attr + match.group(match.lastindex)
+                substitute = match.group(2)
+            return match.group(1) + substitute + match.group(match.lastindex)
 
         def repl_arg_names(match):
+            nameless_value = self._ginfo.get_nameless_value_from_match(match)
             if (
-                match.group(variable_group_in_ir_value_match) is not None
-                and match.group(variable_group_in_ir_value_match) in arg_names
+                nameless_value.check_key == "%"
+                and self._ginfo.get_name_from_match(match) in arg_names
             ):
                 return match.group(1) + match.group(match.lastindex)
             return match.group(1) + match.group(2) + match.group(match.lastindex)
@@ -731,17 +736,19 @@ class function_body(object):
             return False
         if self.attrs != attrs:
             return False
-        ans0 = IR_VALUE_RE.sub(drop_arg_names, self.args_and_sig)
-        ans1 = IR_VALUE_RE.sub(drop_arg_names, args_and_sig)
+
+        regexp = self._ginfo.get_regexp()
+        ans0 = regexp.sub(drop_arg_names, self.args_and_sig)
+        ans1 = regexp.sub(drop_arg_names, args_and_sig)
         if ans0 != ans1:
             return False
-        if is_backend:
+        if self._ginfo.is_asm():
             # Check without replacements, the replacements are not applied to the
             # body for backend checks.
             return self.extrascrub == extrascrub
 
-        es0 = IR_VALUE_RE.sub(repl_arg_names, self.extrascrub)
-        es1 = IR_VALUE_RE.sub(repl_arg_names, extrascrub)
+        es0 = regexp.sub(repl_arg_names, self.extrascrub)
+        es1 = regexp.sub(repl_arg_names, extrascrub)
         es0 = SCRUB_IR_COMMENT_RE.sub(r"", es0)
         es1 = SCRUB_IR_COMMENT_RE.sub(r"", es1)
         return es0 == es1
@@ -751,7 +758,7 @@ class function_body(object):
 
 
 class FunctionTestBuilder:
-    def __init__(self, run_list, flags, scrubber_args, path):
+    def __init__(self, run_list, flags, scrubber_args, path, ginfo):
         self._verbose = flags.verbose
         self._record_args = flags.function_signature
         self._check_attributes = flags.check_attributes
@@ -770,6 +777,7 @@ class FunctionTestBuilder:
         )
         self._scrubber_args = scrubber_args
         self._path = path
+        self._ginfo = ginfo
         # Strip double-quotes if input was read by UTC_ARGS
         self._replace_value_regex = list(
             map(lambda x: x.strip('"'), flags.replace_value_regex)
@@ -804,10 +812,10 @@ class FunctionTestBuilder:
     def is_filtered(self):
         return bool(self._filters)
 
-    def process_run_line(
-        self, function_re, scrubber, raw_tool_output, prefixes, is_backend
-    ):
-        build_global_values_dictionary(self._global_var_dict, raw_tool_output, prefixes)
+    def process_run_line(self, function_re, scrubber, raw_tool_output, prefixes):
+        build_global_values_dictionary(
+            self._global_var_dict, raw_tool_output, prefixes, self._ginfo
+        )
         for m in function_re.finditer(raw_tool_output):
             if not m:
                 continue
@@ -817,7 +825,7 @@ class FunctionTestBuilder:
             # beginning of assembly function definition. In most assemblies, that is just a
             # colon: `foo:`. But, for example, in nvptx it is a brace: `foo(`. If is_backend is
             # False, just assume that separator is an empty string.
-            if is_backend:
+            if self._ginfo.is_asm():
                 # Use ':' as default separator.
                 func_name_separator = (
                     m.group("func_name_separator")
@@ -900,7 +908,6 @@ class FunctionTestBuilder:
                             funcdef_attrs_and_ret,
                             args_and_sig,
                             attrs,
-                            is_backend,
                         ):
                             self._func_dict[prefix][func].scrub = scrubbed_extra
                             self._func_dict[prefix][func].args_and_sig = args_and_sig
@@ -919,6 +926,7 @@ class FunctionTestBuilder:
                             args_and_sig,
                             attrs,
                             func_name_separator,
+                            self._ginfo,
                         )
                         self._func_order[prefix].append(func)
                     else:
@@ -959,6 +967,12 @@ SCRUB_IR_COMMENT_RE = re.compile(r"\s*;.*")
 
 
 class NamelessValue:
+    """
+    A NamelessValue object represents a type of value in the IR whose "name" we
+    generalize in the generated check lines; where the "name" could be an actual
+    name (as in e.g. `@some_global` or `%x`) or just a number (as in e.g. `%12`
+    or `!4`).
+    """
     def __init__(
         self,
         check_prefix,
@@ -971,12 +985,14 @@ class NamelessValue:
         is_number=False,
         replace_number_with_counter=False,
         match_literally=False,
-        interlaced_with_previous=False
+        interlaced_with_previous=False,
+        ir_suffix=r"",
     ):
         self.check_prefix = check_prefix
         self.check_key = check_key
         self.ir_prefix = ir_prefix
         self.ir_regexp = ir_regexp
+        self.ir_suffix = ir_suffix
         self.global_ir_rhs_regexp = global_ir_rhs_regexp
         self.is_before_functions = is_before_functions
         self.is_number = is_number
@@ -987,15 +1003,10 @@ class NamelessValue:
         self.interlaced_with_previous = interlaced_with_previous
         self.variable_mapping = {}
 
-    # Return true if this kind of IR value is "local", basically if it matches '%{{.*}}'.
+    # Return true if this kind of IR value is defined "locally" to functions,
+    # which we assume is only the case precisely for LLVM IR local values.
     def is_local_def_ir_value(self):
-        return self.ir_prefix == "%"
-
-    # Return the IR prefix and check prefix we use for this kind or IR value,
-    # e.g., (%, TMP) for locals. If the IR prefix is a regex, return the prefix
-    # used in the IR output
-    def get_ir_prefix_from_ir_value_match(self, match):
-        return re.search(self.ir_prefix, match[0])[0], self.check_prefix
+        return self.check_key == "%"
 
     # Return the IR regexp we use for this kind or IR value, e.g., [\w.-]+? for locals
     def get_ir_regex(self):
@@ -1030,205 +1041,216 @@ class NamelessValue:
         var = var.replace("-", "_")
         return var.upper()
 
-    # Create a FileCheck variable from regex.
-    def get_value_definition(self, var, match):
-        # for backwards compatibility we check locals with '.*'
-        varname = self.get_value_name(var, self.check_prefix)
-        prefix = self.get_ir_prefix_from_ir_value_match(match)[0]
-        if self.is_number:
-            regex = ""  # always capture a number in the default format
-            capture_start = "[[#"
-        else:
-            regex = self.get_ir_regex()
-            capture_start = "[["
-        if self.is_local_def_ir_value():
-            return capture_start + varname + ":" + prefix + regex + "]]"
-        return prefix + capture_start + varname + ":" + regex + "]]"
-
-    # Use a FileCheck variable.
-    def get_value_use(self, var, match, var_prefix=None):
-        if var_prefix is None:
-            var_prefix = self.check_prefix
-        capture_start = "[[#" if self.is_number else "[["
-        if self.is_local_def_ir_value():
-            return capture_start + self.get_value_name(var, var_prefix) + "]]"
-        prefix = self.get_ir_prefix_from_ir_value_match(match)[0]
-        return prefix + capture_start + self.get_value_name(var, var_prefix) + "]]"
-
-
-# Description of the different "unnamed" values we match in the IR, e.g.,
-# (local) ssa values, (debug) metadata, etc.
-ir_nameless_values = [
-    #            check_prefix   check_key  ir_prefix           ir_regexp                global_ir_rhs_regexp
-    NamelessValue(r"TMP", "%", r"%", r"[\w$.-]+?", None),
-    NamelessValue(r"ATTR", "#", r"#", r"[0-9]+", None),
-    NamelessValue(r"ATTR", "#", r"attributes #", r"[0-9]+", r"{[^}]*}"),
-    NamelessValue(r"GLOB", "@", r"@", r"[0-9]+", None),
-    NamelessValue(r"GLOB", "@", r"@", r"[0-9]+", r".+", is_before_functions=True),
-    NamelessValue(
-        r"GLOBNAMED",
-        "@",
-        r"@",
-        r"[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*",
-        r".+",
-        is_before_functions=True,
-        match_literally=True,
-        interlaced_with_previous=True,
-    ),
-    NamelessValue(r"DBG", "!", r"!dbg ", r"![0-9]+", None),
-    NamelessValue(r"DIASSIGNID", "!", r"!DIAssignID ", r"![0-9]+", None),
-    NamelessValue(r"PROF", "!", r"!prof ", r"![0-9]+", None),
-    NamelessValue(r"TBAA", "!", r"!tbaa ", r"![0-9]+", None),
-    NamelessValue(r"TBAA_STRUCT", "!", r"!tbaa.struct ", r"![0-9]+", None),
-    NamelessValue(r"RNG", "!", r"!range ", r"![0-9]+", None),
-    NamelessValue(r"LOOP", "!", r"!llvm.loop ", r"![0-9]+", None),
-    NamelessValue(r"META", "!", r"", r"![0-9]+", r"(?:distinct |)!.*"),
-    NamelessValue(r"ACC_GRP", "!", r"!llvm.access.group ", r"![0-9]+", None),
-    NamelessValue(r"META", "!", r"![a-z.]+ ", r"![0-9]+", None),
-    NamelessValue(r"META", "!", r"[, (]", r"![0-9]+", None),
-]
+    def get_affixes_from_match(self, match: re.Match):
+        prefix = re.match(self.ir_prefix, match.group(2)).group(0)
+        suffix = re.search(self.ir_suffix + "$", match.group(2)).group(0)
+        return prefix, suffix
 
-global_nameless_values = [
-    nameless_value
-    for nameless_value in ir_nameless_values
-    if nameless_value.global_ir_rhs_regexp is not None
-]
-# global variable names should be matched literally
-global_nameless_values_w_unstable_ids = [
-    nameless_value
-    for nameless_value in global_nameless_values
-    if not nameless_value.match_literally
-]
 
-asm_nameless_values = [
-    NamelessValue(
-        r"MCINST",
-        "Inst#",
-        "<MCInst #",
-        r"\d+",
-        r".+",
-        is_number=True,
-        replace_number_with_counter=True,
-    ),
-    NamelessValue(
-        r"MCREG",
-        "Reg:",
-        "<MCOperand Reg:",
-        r"\d+",
-        r".+",
-        is_number=True,
-        replace_number_with_counter=True,
-    ),
-]
+class GeneralizerInfo:
+    """
+    A GeneralizerInfo object holds information about how check lines should be generalized
+    (e.g., variable names replaced by FileCheck meta variables) as well as per-test-file
+    state (e.g. information about IR global variables).
+    """
 
-analyze_nameless_values = [
-    NamelessValue(
-        r"GRP",
-        "#",
-        r"",
-        r"0x[0-9a-f]+",
-        None,
-        replace_number_with_counter=True,
-    ),
-]
+    MODE_IR = 0
+    MODE_ASM = 1
+    MODE_ANALYZE = 2
 
+    def __init__(
+        self,
+        version,
+        mode,
+        nameless_values: List[NamelessValue],
+        regexp_prefix,
+        regexp_suffix,
+    ):
+        self._version = version
+        self._mode = mode
+        self._nameless_values = nameless_values
+
+        self._regexp_prefix = regexp_prefix
+        self._regexp_suffix = regexp_suffix
+
+        self._regexp, _ = self._build_regexp(False, False)
+        (
+            self._unstable_globals_regexp,
+            self._unstable_globals_values,
+        ) = self._build_regexp(True, True)
+
+    def _build_regexp(self, globals_only, unstable_only):
+        matches = []
+        values = []
+        for nameless_value in self._nameless_values:
+            is_global = nameless_value.global_ir_rhs_regexp is not None
+            if globals_only and not is_global:
+                continue
+            if unstable_only and nameless_value.match_literally:
+                continue
 
-def createOrRegexp(old, new):
-    if not old:
-        return new
-    if not new:
-        return old
-    return old + "|" + new
-
-
-def createPrefixMatch(prefix_str, prefix_re):
-    return "(?:" + prefix_str + "(" + prefix_re + "))"
-
-
-# Build the regexp that matches an "IR value". This can be a local variable,
-# argument, global, or metadata, anything that is "named". It is important that
-# the PREFIX and SUFFIX below only contain a single group, if that changes
-# other locations will need adjustment as well.
-IR_VALUE_REGEXP_PREFIX = r"(\s*)"
-IR_VALUE_REGEXP_STRING = r""
-for nameless_value in ir_nameless_values:
-    match = createPrefixMatch(nameless_value.ir_prefix, nameless_value.ir_regexp)
-    if nameless_value.global_ir_rhs_regexp is not None:
-        match = "^" + match
-    IR_VALUE_REGEXP_STRING = createOrRegexp(IR_VALUE_REGEXP_STRING, match)
-IR_VALUE_REGEXP_SUFFIX = r"([,\s\(\)\}]|\Z)"
-IR_VALUE_RE = re.compile(
-    IR_VALUE_REGEXP_PREFIX
-    + r"("
-    + IR_VALUE_REGEXP_STRING
-    + r")"
-    + IR_VALUE_REGEXP_SUFFIX
-)
+            match = f"(?:{nameless_value.ir_prefix}({nameless_value.ir_regexp}){nameless_value.ir_suffix})"
+            if self.is_ir() and not globals_only and is_global:
+                match = "^" + match
+            matches.append(match)
+            values.append(nameless_value)
 
-GLOBAL_VALUE_REGEXP_STRING = r""
-for nameless_value in global_nameless_values_w_unstable_ids:
-    match = createPrefixMatch(nameless_value.ir_prefix, nameless_value.ir_regexp)
-    GLOBAL_VALUE_REGEXP_STRING = createOrRegexp(GLOBAL_VALUE_REGEXP_STRING, match)
-GLOBAL_VALUE_RE = re.compile(
-    IR_VALUE_REGEXP_PREFIX
-    + r"("
-    + GLOBAL_VALUE_REGEXP_STRING
-    + r")"
-    + IR_VALUE_REGEXP_SUFFIX
-)
+        regexp_string = r"|".join(matches)
 
-# Build the regexp that matches an "ASM value" (currently only for --asm-show-inst comments).
-ASM_VALUE_REGEXP_STRING = ""
-for nameless_value in asm_nameless_values:
-    match = createPrefixMatch(nameless_value.ir_prefix, nameless_value.ir_regexp)
-    ASM_VALUE_REGEXP_STRING = createOrRegexp(ASM_VALUE_REGEXP_STRING, match)
-ASM_VALUE_REGEXP_SUFFIX = r"([>\s]|\Z)"
-ASM_VALUE_RE = re.compile(
-    r"((?:#|//)\s*)" + "(" + ASM_VALUE_REGEXP_STRING + ")" + ASM_VALUE_REGEXP_SUFFIX
-)
+        return (
+            re.compile(
+                self._regexp_prefix + r"(" + regexp_string + r")" + self._regexp_suffix
+            ),
+            values,
+        )
 
-ANALYZE_VALUE_REGEXP_PREFIX = r"(\s*)"
-ANALYZE_VALUE_REGEXP_STRING = r""
-for nameless_value in analyze_nameless_values:
-    match = createPrefixMatch(nameless_value.ir_prefix, nameless_value.ir_regexp)
-    ANALYZE_VALUE_REGEXP_STRING = createOrRegexp(ANALYZE_VALUE_REGEXP_STRING, match)
-ANALYZE_VALUE_REGEXP_SUFFIX = r"(\)?:)"
-ANALYZE_VALUE_RE = re.compile(
-    ANALYZE_VALUE_REGEXP_PREFIX
-    + r"("
-    + ANALYZE_VALUE_REGEXP_STRING
-    + r")"
-    + ANALYZE_VALUE_REGEXP_SUFFIX
-)
+    def get_version(self):
+        return self._version
 
-# The entire match is group 0, the prefix has one group (=1), the entire
-# IR_VALUE_REGEXP_STRING is one group (=2), and then the nameless values start.
-first_nameless_group_in_ir_value_match = 3
+    def is_ir(self):
+        return self._mode == GeneralizerInfo.MODE_IR
 
-# constants for the group id of special matches
-variable_group_in_ir_value_match = 3
-attribute_group_in_ir_value_match = 4
+    def is_asm(self):
+        return self._mode == GeneralizerInfo.MODE_ASM
 
+    def is_analyze(self):
+        return self._mode == GeneralizerInfo.MODE_ANALYZE
 
-# Check a match for IR_VALUE_RE and inspect it to determine if it was a local
-# value, %..., global @..., debug number !dbg !..., etc. See the PREFIXES above.
-def get_idx_from_ir_value_match(match):
-    for i in range(first_nameless_group_in_ir_value_match, match.lastindex):
-        if match.group(i) is not None:
-            return i - first_nameless_group_in_ir_value_match
-    error("Unable to identify the kind of IR value from the match!")
-    return 0
+    def get_nameless_values(self):
+        return self._nameless_values
 
+    def get_regexp(self):
+        return self._regexp
 
-# See get_idx_from_ir_value_match
-def get_name_from_ir_value_match(match):
-    return match.group(
-        get_idx_from_ir_value_match(match) + first_nameless_group_in_ir_value_match
-    )
+    def get_unstable_globals_regexp(self):
+        return self._unstable_globals_regexp
 
+    # The entire match is group 0, the prefix has one group (=1), the entire
+    # IR_VALUE_REGEXP_STRING is one group (=2), and then the nameless values start.
+    FIRST_NAMELESS_GROUP_IN_MATCH = 3
 
-def get_nameless_value_from_match(match, nameless_values) -> NamelessValue:
-    return nameless_values[get_idx_from_ir_value_match(match)]
+    def get_match_info(self, match):
+        """
+        Returns (name, nameless_value) for the given match object
+        """
+        if match.re == self._regexp:
+            values = self._nameless_values
+        else:
+            match.re == self._unstable_globals_regexp
+            values = self._unstable_globals_values
+        for i in range(len(values)):
+            g = match.group(i + GeneralizerInfo.FIRST_NAMELESS_GROUP_IN_MATCH)
+            if g is not None:
+                return g, values[i]
+        error("Unable to identify the kind of IR value from the match!")
+        return None, None
+
+    # See get_idx_from_match
+    def get_name_from_match(self, match):
+        return self.get_match_info(match)[0]
+
+    def get_nameless_value_from_match(self, match) -> NamelessValue:
+        return self.get_match_info(match)[1]
+
+
+def make_ir_generalizer(version):
+    values = []
+
+    if version >= 5:
+        values += [
+            NamelessValue(r"BB", "%", r"label %", r"[\w$.-]+?", None),
+            NamelessValue(r"BB", "%", r"^", r"[\w$.-]+?", None, ir_suffix=r":"),
+        ]
+
+    values += [
+        #            check_prefix   check_key  ir_prefix           ir_regexp                global_ir_rhs_regexp
+        NamelessValue(r"TMP", "%", r"%", r"[\w$.-]+?", None),
+        NamelessValue(r"ATTR", "#", r"#", r"[0-9]+", None),
+        NamelessValue(r"ATTR", "#", r"attributes #", r"[0-9]+", r"{[^}]*}"),
+        NamelessValue(r"GLOB", "@", r"@", r"[0-9]+", None),
+        NamelessValue(r"GLOB", "@", r"@", r"[0-9]+", r".+", is_before_functions=True),
+        NamelessValue(
+            r"GLOBNAMED",
+            "@",
+            r"@",
+            r"[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*",
+            r".+",
+            is_before_functions=True,
+            match_literally=True,
+            interlaced_with_previous=True,
+        ),
+        NamelessValue(r"DBG", "!", r"!dbg ", r"![0-9]+", None),
+        NamelessValue(r"DIASSIGNID", "!", r"!DIAssignID ", r"![0-9]+", None),
+        NamelessValue(r"PROF", "!", r"!prof ", r"![0-9]+", None),
+        NamelessValue(r"TBAA", "!", r"!tbaa ", r"![0-9]+", None),
+        NamelessValue(r"TBAA_STRUCT", "!", r"!tbaa.struct ", r"![0-9]+", None),
+        NamelessValue(r"RNG", "!", r"!range ", r"![0-9]+", None),
+        NamelessValue(r"LOOP", "!", r"!llvm.loop ", r"![0-9]+", None),
+        NamelessValue(r"META", "!", r"", r"![0-9]+", r"(?:distinct |)!.*"),
+        NamelessValue(r"ACC_GRP", "!", r"!llvm.access.group ", r"![0-9]+", None),
+        NamelessValue(r"META", "!", r"![a-z.]+ ", r"![0-9]+", None),
+        NamelessValue(r"META", "!", r"[, (]", r"![0-9]+", None),
+    ]
+
+    prefix = r"(\s*)"
+    suffix = r"([,\s\(\)\}]|\Z)"
+
+    # values = [
+    #     nameless_value
+    #     for nameless_value in IR_NAMELESS_VALUES
+    #     if not (globals_only and nameless_value.global_ir_rhs_regexp is None) and
+    #        not (unstable_ids_only and nameless_value.match_literally)
+    # ]
+
+    return GeneralizerInfo(version, GeneralizerInfo.MODE_IR, values, prefix, suffix)
+
+
+def make_asm_generalizer(version):
+    values = [
+        NamelessValue(
+            r"MCINST",
+            "Inst#",
+            "<MCInst #",
+            r"\d+",
+            r".+",
+            is_number=True,
+            replace_number_with_counter=True,
+        ),
+        NamelessValue(
+            r"MCREG",
+            "Reg:",
+            "<MCOperand Reg:",
+            r"\d+",
+            r".+",
+            is_number=True,
+            replace_number_with_counter=True,
+        ),
+    ]
+
+    prefix = r"((?:#|//)\s*)"
+    suffix = r"([>\s]|\Z)"
+
+    return GeneralizerInfo(version, GeneralizerInfo.MODE_ASM, values, prefix, suffix)
+
+
+def make_analyze_generalizer(version):
+    values = [
+        NamelessValue(
+            r"GRP",
+            "#",
+            r"",
+            r"0x[0-9a-f]+",
+            None,
+            replace_number_with_counter=True,
+        ),
+    ]
+
+    prefix = r"(\s*)"
+    suffix = r"(\)?:)"
+
+    return GeneralizerInfo(
+        version, GeneralizerInfo.MODE_ANALYZE, values, prefix, suffix
+    )
 
 
 # Return true if var clashes with the scripted FileCheck check_prefix.
@@ -1385,16 +1407,68 @@ METAVAR_RE = re.compile(r"\[\[([A-Z0-9_]+)(?::[^]]+)?\]\]")
 NUMERIC_SUFFIX_RE = re.compile(r"[0-9]*$")
 
 
+class TestVar:
+    def __init__(self, nameless_value: NamelessValue, prefix: str, suffix: str):
+        self._nameless_value = nameless_value
+
+        self._prefix = prefix
+        self._suffix = suffix
+
+    def seen(self, nameless_value: NamelessValue, prefix: str, suffix: str):
+        if prefix != self._prefix:
+            self._prefix = ""
+        if suffix != self._suffix:
+            self._suffix = ""
+
+    def get_variable_name(self, text):
+        return self._nameless_value.get_value_name(
+            text, self._nameless_value.check_prefix
+        )
+
+    def get_def(self, name, prefix, suffix):
+        if self._nameless_value.is_number:
+            return f"{prefix}[[#{name}:]]{suffix}"
+        if self._prefix:
+            assert self._prefix == prefix
+            prefix = ""
+        if self._suffix:
+            assert self._suffix == suffix
+            suffix = ""
+        return f"{prefix}[[{name}:{self._prefix}{self._nameless_value.get_ir_regex()}{self._suffix}]]{suffix}"
+
+    def get_use(self, name, prefix, suffix):
+        if self._nameless_value.is_number:
+            return f"{prefix}[[#{name}]]{suffix}"
+        if self._prefix:
+            assert self._prefix == prefix
+            prefix = ""
+        if self._suffix:
+            assert self._suffix == suffix
+            suffix = ""
+        return f"{prefix}[[{name}]]{suffix}"
+
+
 class CheckValueInfo:
     def __init__(
         self,
-        nameless_value: NamelessValue,
-        var: str,
+        key,
+        text,
+        name: str,
         prefix: str,
+        suffix: str,
     ):
-        self.nameless_value = nameless_value
-        self.var = var
+        # Key for the value, e.g. '%'
+        self.key = key
+
+        # Text to be matched by the FileCheck variable (without any prefix or suffix)
+        self.text = text
+
+        # Name of the FileCheck variable
+        self.name = name
+
+        # Prefix and suffix that were captured by the NamelessValue regular expression
         self.prefix = prefix
+        self.suffix = suffix
 
 
 # Represent a check line in a way that allows us to compare check lines while
@@ -1433,7 +1507,7 @@ def remap_metavar_names(
     new_mapping = {}
     for line in new_line_infos:
         for value in line.values:
-            new_mapping[value.var] = value.var
+            new_mapping[value.name] = value.name
 
     # Recursively commit to the identity mapping or find a better one
     def recurse(old_begin, old_end, new_begin, new_end):
@@ -1445,7 +1519,7 @@ def remap_metavar_names(
         def diffify_line(line, mapper):
             values = []
             for value in line.values:
-                mapped = mapper(value.var)
+                mapped = mapper(value.name)
                 values.append(mapped if mapped in committed_names else "?")
             return line.line.strip() + " @@@ " + " @ ".join(values)
 
@@ -1470,29 +1544,29 @@ def remap_metavar_names(
             local_commits = {}
 
             for lhs_value, rhs_value in zip(lhs_line.values, rhs_line.values):
-                if new_mapping[rhs_value.var] in committed_names:
+                if new_mapping[rhs_value.name] in committed_names:
                     # The new value has already been committed. If it was mapped
                     # to the same name as the original value, we can consider
                     # committing other values from this line. Otherwise, we
                     # should ignore this line.
-                    if new_mapping[rhs_value.var] == lhs_value.var:
+                    if new_mapping[rhs_value.name] == lhs_value.name:
                         continue
                     else:
                         break
 
-                if rhs_value.var in local_commits:
+                if rhs_value.name in local_commits:
                     # Same, but for a possible commit happening on the same line
-                    if local_commits[rhs_value.var] == lhs_value.var:
+                    if local_commits[rhs_value.name] == lhs_value.name:
                         continue
                     else:
                         break
 
-                if lhs_value.var in committed_names:
+                if lhs_value.name in committed_names:
                     # We can't map this value because the name we would map it to has already been
                     # committed for something else. Give up on this line.
                     break
 
-                local_commits[rhs_value.var] = lhs_value.var
+                local_commits[rhs_value.name] = lhs_value.name
             else:
                 # No reason not to add any commitments for this line
                 for rhs_var, lhs_var in local_commits.items():
@@ -1545,58 +1619,26 @@ def remap_metavar_names(
     return new_mapping
 
 
-def generalize_check_lines_common(
+def generalize_check_lines(
     lines,
-    is_analyze,
+    ginfo: GeneralizerInfo,
     vars_seen,
     global_vars_seen,
-    nameless_values,
-    nameless_value_regex,
-    is_asm,
-    preserve_names,
+    preserve_names=False,
     original_check_lines=None,
+    *,
+    unstable_globals_only=False,
 ):
-    # This gets called for each match that occurs in
-    # a line. We transform variables we haven't seen
-    # into defs, and variables we have seen into uses.
-    def transform_line_vars(match, transform_locals=True):
-        var = get_name_from_ir_value_match(match)
-        nameless_value = get_nameless_value_from_match(match, nameless_values)
-        if may_clash_with_default_check_prefix_name(nameless_value.check_prefix, var):
-            warn(
-                "Change IR value name '%s' or use --prefix-filecheck-ir-name to prevent possible conflict"
-                " with scripted FileCheck name." % (var,)
-            )
-        key = (var, nameless_value.check_key)
-        is_local_def = nameless_value.is_local_def_ir_value()
-        if is_local_def and not transform_locals:
-            return None
-        if is_local_def and key in vars_seen:
-            rv = nameless_value.get_value_use(var, match)
-        elif not is_local_def and key in global_vars_seen:
-            # We could have seen a different prefix for the global variables first,
-            # ensure we use that one instead of the prefix for the current match.
-            rv = nameless_value.get_value_use(var, match, global_vars_seen[key])
-        else:
-            if is_local_def:
-                vars_seen.add(key)
-            else:
-                global_vars_seen[key] = nameless_value.check_prefix
-            rv = nameless_value.get_value_definition(var, match)
-        # re.sub replaces the entire regex match
-        # with whatever you return, so we have
-        # to make sure to hand it back everything
-        # including the commas and spaces.
-        return match.group(1) + rv + match.group(match.lastindex)
-
-    def transform_non_local_line_vars(match):
-        return transform_line_vars(match, False)
+    if unstable_globals_only:
+        regexp = ginfo.get_unstable_globals_regexp()
+    else:
+        regexp = ginfo.get_regexp()
 
     multiple_braces_re = re.compile(r"({{+)|(}}+)")
     def escape_braces(match_obj):
         return '{{' + re.escape(match_obj.group(0)) + '}}'
 
-    if not is_asm and not is_analyze:
+    if ginfo.is_ir():
         for i, line in enumerate(lines):
             # An IR variable named '%.' matches the FileCheck regex string.
             line = line.replace("%.", "%dot")
@@ -1617,123 +1659,141 @@ def generalize_check_lines_common(
             lines[i] = scrubbed_line
 
     if not preserve_names:
-        if is_asm:
-            for i, _ in enumerate(lines):
-                # It can happen that two matches are back-to-back and for some reason sub
-                # will not replace both of them. For now we work around this by
-                # substituting until there is no more match.
-                changed = True
-                while changed:
-                    (lines[i], changed) = nameless_value_regex.subn(
-                        transform_line_vars, lines[i], count=1
-                    )
-        else:
-            # LLVM IR case. Start by handling global meta variables (global IR variables,
-            # metadata, attributes)
-            for i, _ in enumerate(lines):
-                start = 0
-                while True:
-                    m = nameless_value_regex.search(lines[i][start:])
-                    if m is None:
-                        break
-                    start += m.start()
-                    sub = transform_non_local_line_vars(m)
-                    if sub is not None:
-                        lines[i] = (
-                            lines[i][:start] + sub + lines[i][start + len(m.group(0)) :]
-                        )
-                    start += 1
-
-            # Collect information about new check lines and original check lines (if any)
-            new_line_infos = []
-            for line in lines:
-                filtered_line = ""
-                values = []
-                while True:
-                    m = nameless_value_regex.search(line)
-                    if m is None:
-                        filtered_line += line
-                        break
+        committed_names = set(
+            test_var.get_variable_name(name)
+            for (name, _), test_var in vars_seen.items()
+        )
+        defs = set()
 
-                    var = get_name_from_ir_value_match(m)
-                    nameless_value = get_nameless_value_from_match(m, nameless_values)
-                    var = nameless_value.get_value_name(
-                        var, nameless_value.check_prefix
-                    )
+        # Collect information about new check lines, and generalize global reference
+        new_line_infos = []
+        for line in lines:
+            filtered_line = ""
+            values = []
+            while True:
+                m = regexp.search(line)
+                if m is None:
+                    filtered_line += line
+                    break
 
-                    # Replace with a [[@@]] tag, but be sure to keep the spaces and commas.
-                    filtered_line += (
-                        line[: m.start()]
-                        + m.group(1)
-                        + VARIABLE_TAG
-                        + m.group(m.lastindex)
+                name = ginfo.get_name_from_match(m)
+                nameless_value = ginfo.get_nameless_value_from_match(m)
+                prefix, suffix = nameless_value.get_affixes_from_match(m)
+                if may_clash_with_default_check_prefix_name(
+                    nameless_value.check_prefix, name
+                ):
+                    warn(
+                        "Change IR value name '%s' or use --prefix-filecheck-ir-name to prevent possible conflict"
+                        " with scripted FileCheck name." % (name,)
                     )
-                    line = line[m.end() :]
-                    values.append(
-                        CheckValueInfo(
-                            nameless_value=nameless_value,
-                            var=var,
-                            prefix=nameless_value.get_ir_prefix_from_ir_value_match(m)[
-                                0
-                            ],
-                        )
+
+                # Record the variable as seen and (for locals) accumulate
+                # prefixes/suffixes
+                is_local_def = nameless_value.is_local_def_ir_value()
+                if is_local_def:
+                    vars_dict = vars_seen
+                else:
+                    vars_dict = global_vars_seen
+
+                key = (name, nameless_value.check_key)
+
+                if is_local_def:
+                    test_prefix = prefix
+                    test_suffix = suffix
+                else:
+                    test_prefix = ""
+                    test_suffix = ""
+
+                if key in vars_dict:
+                    vars_dict[key].seen(nameless_value, test_prefix, test_suffix)
+                else:
+                    vars_dict[key] = TestVar(nameless_value, test_prefix, test_suffix)
+                    defs.add(key)
+
+                var = vars_dict[key].get_variable_name(name)
+
+                # Replace with a [[@@]] tag, but be sure to keep the spaces and commas.
+                filtered_line += (
+                    line[: m.start()] + m.group(1) + VARIABLE_TAG + m.group(m.lastindex)
+                )
+                line = line[m.end() :]
+
+                values.append(
+                    CheckValueInfo(
+                        key=nameless_value.check_key,
+                        text=name,
+                        name=var,
+                        prefix=prefix,
+                        suffix=suffix,
                     )
-                new_line_infos.append(CheckLineInfo(filtered_line, values))
-
-            orig_line_infos = []
-            for line in original_check_lines or []:
-                filtered_line = ""
-                values = []
-                while True:
-                    m = METAVAR_RE.search(line)
-                    if m is None:
-                        filtered_line += line
-                        break
+                )
 
-                    # Replace with a [[@@]] tag, but be sure to keep the spaces and commas.
-                    filtered_line += line[: m.start()] + VARIABLE_TAG
-                    line = line[m.end() :]
-                    values.append(
-                        CheckValueInfo(
-                            nameless_value=None,
-                            var=m.group(1),
-                            prefix=None,
-                        )
+            new_line_infos.append(CheckLineInfo(filtered_line, values))
+
+        committed_names.update(
+            test_var.get_variable_name(name)
+            for (name, _), test_var in global_vars_seen.items()
+        )
+
+        # Collect information about original check lines, if any.
+        orig_line_infos = []
+        for line in original_check_lines or []:
+            filtered_line = ""
+            values = []
+            while True:
+                m = METAVAR_RE.search(line)
+                if m is None:
+                    filtered_line += line
+                    break
+
+                # Replace with a [[@@]] tag, but be sure to keep the spaces and commas.
+                filtered_line += line[: m.start()] + VARIABLE_TAG
+                line = line[m.end() :]
+                values.append(
+                    CheckValueInfo(
+                        key=None,
+                        text=None,
+                        name=m.group(1),
+                        prefix="",
+                        suffix="",
                     )
-                orig_line_infos.append(CheckLineInfo(filtered_line, values))
+                )
+            orig_line_infos.append(CheckLineInfo(filtered_line, values))
 
-            # Compute the variable name mapping
-            committed_names = set(vars_seen)
+        # Compute the variable name mapping
+        mapping = remap_metavar_names(orig_line_infos, new_line_infos, committed_names)
 
-            mapping = remap_metavar_names(
-                orig_line_infos, new_line_infos, committed_names
-            )
+        # Apply the variable name mapping
+        for i, line_info in enumerate(new_line_infos):
+            line_template = line_info.line
+            line = ""
 
-            for i, line_info in enumerate(new_line_infos):
-                line_template = line_info.line
-                line = ""
+            for value in line_info.values:
+                idx = line_template.find(VARIABLE_TAG)
+                line += line_template[:idx]
+                line_template = line_template[idx + len(VARIABLE_TAG) :]
 
-                for value in line_info.values:
-                    idx = line_template.find(VARIABLE_TAG)
-                    line += line_template[:idx]
-                    line_template = line_template[idx + len(VARIABLE_TAG) :]
+                key = (value.text, value.key)
+                if value.key == "%":
+                    vars_dict = vars_seen
+                else:
+                    vars_dict = global_vars_seen
 
-                    key = (mapping[value.var], nameless_value.check_key)
-                    is_local_def = nameless_value.is_local_def_ir_value()
-                    if is_local_def:
-                        if mapping[value.var] in vars_seen:
-                            line += f"[[{mapping[value.var]}]]"
-                        else:
-                            line += f"[[{mapping[value.var]}:{value.prefix}{value.nameless_value.get_ir_regex()}]]"
-                            vars_seen.add(mapping[value.var])
-                    else:
-                        raise RuntimeError("not implemented")
+                if key in defs:
+                    line += vars_dict[key].get_def(
+                        mapping[value.name], value.prefix, value.suffix
+                    )
+                    defs.remove(key)
+                else:
+                    line += vars_dict[key].get_use(
+                        mapping[value.name], value.prefix, value.suffix
+                    )
 
-                line += line_template
+            line += line_template
 
-                lines[i] = line
+            lines[i] = line
 
-    if is_analyze:
+    if ginfo.is_analyze():
         for i, _ in enumerate(lines):
             # Escape multiple {{ or }} as {{}} denotes a FileCheck regex.
             scrubbed_line = multiple_braces_re.sub(escape_braces, lines[i])
@@ -1742,63 +1802,6 @@ def generalize_check_lines_common(
     return lines
 
 
-# Replace IR value defs and uses with FileCheck variables.
-def generalize_check_lines(
-    lines, is_analyze, vars_seen, global_vars_seen, preserve_names, original_check_lines
-):
-    return generalize_check_lines_common(
-        lines,
-        is_analyze,
-        vars_seen,
-        global_vars_seen,
-        ir_nameless_values,
-        IR_VALUE_RE,
-        False,
-        preserve_names,
-        original_check_lines=original_check_lines,
-    )
-
-
-def generalize_global_check_line(line, preserve_names, global_vars_seen):
-    [new_line] = generalize_check_lines_common(
-        [line],
-        False,
-        set(),
-        global_vars_seen,
-        global_nameless_values_w_unstable_ids,
-        GLOBAL_VALUE_RE,
-        False,
-        preserve_names,
-    )
-    return new_line
-
-
-def generalize_asm_check_lines(lines, vars_seen, global_vars_seen):
-    return generalize_check_lines_common(
-        lines,
-        False,
-        vars_seen,
-        global_vars_seen,
-        asm_nameless_values,
-        ASM_VALUE_RE,
-        True,
-        False,
-    )
-
-
-def generalize_analyze_check_lines(lines, vars_seen, global_vars_seen):
-    return generalize_check_lines_common(
-        lines,
-        True,
-        vars_seen,
-        global_vars_seen,
-        analyze_nameless_values,
-        ANALYZE_VALUE_RE,
-        False,
-        False,
-    )
-
-
 def add_checks(
     output_lines,
     comment_marker,
@@ -1806,9 +1809,7 @@ def add_checks(
     func_dict,
     func_name,
     check_label_format,
-    is_backend,
-    is_analyze,
-    version,
+    ginfo,
     global_vars_seen_dict,
     is_filtered,
     preserve_names=False,
@@ -1853,7 +1854,7 @@ def add_checks(
 
             # Add some space between different check prefixes, but not after the last
             # check line (before the test code).
-            if is_backend:
+            if ginfo.is_asm():
                 if len(printed_prefixes) != 0:
                     output_lines.append(comment_marker)
 
@@ -1862,11 +1863,11 @@ def add_checks(
 
             global_vars_seen_before = [key for key in global_vars_seen.keys()]
 
-            vars_seen = set()
+            vars_seen = {}
             printed_prefixes.append(checkprefix)
             attrs = str(func_dict[checkprefix][func_name].attrs)
             attrs = "" if attrs == "None" else attrs
-            if version > 1:
+            if ginfo.get_version() > 1:
                 funcdef_attrs_and_ret = func_dict[checkprefix][
                     func_name
                 ].funcdef_attrs_and_ret
@@ -1881,7 +1882,7 @@ def add_checks(
             if args_and_sig:
                 args_and_sig = generalize_check_lines(
                     [args_and_sig],
-                    is_analyze,
+                    ginfo,
                     vars_seen,
                     global_vars_seen,
                     preserve_names,
@@ -1892,7 +1893,7 @@ def add_checks(
                 # Captures in label lines are not supported, thus split into a -LABEL
                 # and a separate -SAME line that contains the arguments with captures.
                 args_and_sig_prefix = ""
-                if version >= 3 and args_and_sig.startswith("("):
+                if ginfo.get_version() >= 3 and args_and_sig.startswith("("):
                     # Ensure the "(" separating function name and arguments is in the
                     # label line. This is required in case of function names that are
                     # prefixes of each other. Otherwise, the label line for "foo" might
@@ -1933,7 +1934,7 @@ def add_checks(
                 continue
 
             # For ASM output, just emit the check lines.
-            if is_backend:
+            if ginfo.is_asm():
                 body_start = 1
                 if is_filtered:
                     # For filtered output we don't add "-NEXT" so don't add extra spaces
@@ -1943,8 +1944,8 @@ def add_checks(
                     output_lines.append(
                         "%s %s:       %s" % (comment_marker, checkprefix, func_body[0])
                     )
-                func_lines = generalize_asm_check_lines(
-                    func_body[body_start:], vars_seen, global_vars_seen
+                func_lines = generalize_check_lines(
+                    func_body[body_start:], ginfo, vars_seen, global_vars_seen
                 )
                 for func_line in func_lines:
                     if func_line.strip() == "":
@@ -1963,9 +1964,9 @@ def add_checks(
                         global_vars_seen_dict[checkprefix][key] = global_vars_seen[key]
                 break
             # For analyze output, generalize the output, and emit CHECK-EMPTY lines as well.
-            elif is_analyze:
-                func_body = generalize_analyze_check_lines(
-                    func_body, vars_seen, global_vars_seen
+            elif ginfo.is_analyze():
+                func_body = generalize_check_lines(
+                    func_body, ginfo, vars_seen, global_vars_seen
                 )
                 for func_line in func_body:
                     if func_line.strip() == "":
@@ -1994,7 +1995,7 @@ def add_checks(
             else:
                 func_body = generalize_check_lines(
                     func_body,
-                    False,
+                    ginfo,
                     vars_seen,
                     global_vars_seen,
                     preserve_names,
@@ -2057,13 +2058,14 @@ def add_ir_checks(
     func_name,
     preserve_names,
     function_sig,
-    version,
+    ginfo: GeneralizerInfo,
     global_vars_seen_dict,
     is_filtered,
     original_check_lines={},
 ):
+    assert ginfo.is_ir()
     # Label format is based on IR string.
-    if function_sig and version > 1:
+    if function_sig and ginfo.get_version() > 1:
         function_def_regex = "define %s"
     elif function_sig:
         function_def_regex = "define {{[^@]+}}%s"
@@ -2079,9 +2081,7 @@ def add_ir_checks(
         func_dict,
         func_name,
         check_label_format,
-        False,
-        False,
-        version,
+        ginfo,
         global_vars_seen_dict,
         is_filtered,
         preserve_names,
@@ -2090,8 +2090,15 @@ def add_ir_checks(
 
 
 def add_analyze_checks(
-    output_lines, comment_marker, prefix_list, func_dict, func_name, is_filtered
+    output_lines,
+    comment_marker,
+    prefix_list,
+    func_dict,
+    func_name,
+    ginfo: GeneralizerInfo,
+    is_filtered,
 ):
+    assert ginfo.is_analyze()
     check_label_format = "{} %s-LABEL: '%s%s%s%s'".format(comment_marker)
     global_vars_seen_dict = {}
     return add_checks(
@@ -2101,16 +2108,14 @@ def add_analyze_checks(
         func_dict,
         func_name,
         check_label_format,
-        False,
-        True,
-        1,
+        ginfo,
         global_vars_seen_dict,
         is_filtered,
     )
 
 
-def build_global_values_dictionary(glob_val_dict, raw_tool_output, prefixes):
-    for nameless_value in itertools.chain(global_nameless_values, asm_nameless_values):
+def build_global_values_dictionary(glob_val_dict, raw_tool_output, prefixes, ginfo):
+    for nameless_value in ginfo.get_nameless_values():
         if nameless_value.global_ir_rhs_regexp is None:
             continue
 
@@ -2225,6 +2230,7 @@ def add_global_checks(
     comment_marker,
     prefix_list,
     output_lines,
+    ginfo: GeneralizerInfo,
     global_vars_seen_dict,
     preserve_names,
     is_before_functions,
@@ -2232,7 +2238,9 @@ def add_global_checks(
 ):
     printed_prefixes = set()
     output_lines_loc = {}  # Allows GLOB and GLOBNAMED to be sorted correctly
-    for nameless_value in global_nameless_values:
+    for nameless_value in ginfo.get_nameless_values():
+        if nameless_value.global_ir_rhs_regexp is None:
+            continue
         if nameless_value.is_before_functions != is_before_functions:
             continue
         for p in prefix_list:
@@ -2274,8 +2282,13 @@ def add_global_checks(
                                 break
                         if not matched:
                             continue
-                    new_line = generalize_global_check_line(
-                        line, preserve_names, global_vars_seen
+                    [new_line] = generalize_check_lines(
+                        [line],
+                        ginfo,
+                        {},
+                        global_vars_seen,
+                        preserve_names,
+                        unstable_globals_only=True,
                     )
                     new_line = filter_unstable_metadata(new_line)
                     check_line = "%s %s: %s" % (comment_marker, checkprefix, new_line)
diff --git a/llvm/utils/UpdateTestChecks/isel.py b/llvm/utils/UpdateTestChecks/isel.py
index bdb68e5815a3..855bc50b09f4 100644
--- a/llvm/utils/UpdateTestChecks/isel.py
+++ b/llvm/utils/UpdateTestChecks/isel.py
@@ -60,6 +60,7 @@ def add_checks(
     prefix_list,
     func_dict,
     func_name,
+    ginfo: common.GeneralizerInfo,
     global_vars_seen_dict,
     is_filtered,
 ):
@@ -72,9 +73,7 @@ def add_checks(
         func_dict,
         func_name,
         check_label_format,
-        True,
-        False,
-        1,
+        ginfo,
         global_vars_seen_dict,
         is_filtered=is_filtered,
     )
diff --git a/llvm/utils/update_analyze_test_checks.py b/llvm/utils/update_analyze_test_checks.py
index 03053e5447d1..47506626a0a5 100755
--- a/llvm/utils/update_analyze_test_checks.py
+++ b/llvm/utils/update_analyze_test_checks.py
@@ -96,6 +96,7 @@ def main():
             # now, we just ignore all but the last.
             prefix_list.append((check_prefixes, tool_cmd_args))
 
+        ginfo = common.make_analyze_generalizer(version=1)
         builder = common.FunctionTestBuilder(
             run_list=prefix_list,
             flags=type(
@@ -111,6 +112,7 @@ def main():
             ),
             scrubber_args=[],
             path=ti.path,
+            ginfo=ginfo,
         )
 
         for prefixes, opt_args in prefix_list:
@@ -131,7 +133,6 @@ def main():
                         common.scrub_body,
                         raw_tool_output,
                         prefixes,
-                        False,
                     )
             elif re.search(r"LV: Checking a loop in ", raw_tool_outputs) is not None:
                 # Split analysis outputs by "Printing analysis " declarations.
@@ -143,7 +144,6 @@ def main():
                         common.scrub_body,
                         raw_tool_output,
                         prefixes,
-                        False,
                     )
             else:
                 common.warn("Don't know how to deal with this output")
@@ -179,6 +179,7 @@ def main():
                         prefix_list,
                         func_dict,
                         func_name,
+                        ginfo,
                         is_filtered=builder.is_filtered(),
                     )
                 )
diff --git a/llvm/utils/update_cc_test_checks.py b/llvm/utils/update_cc_test_checks.py
index 28c6bb0409f3..3ffb07ddf6ad 100755
--- a/llvm/utils/update_cc_test_checks.py
+++ b/llvm/utils/update_cc_test_checks.py
@@ -270,7 +270,7 @@ def get_function_body(builder, args, filename, clang_args, extra_commands, prefi
             raw_tool_output = common.invoke_tool(extra_args[0], extra_args[1:], f.name)
     if "-emit-llvm" in clang_args:
         builder.process_run_line(
-            common.OPT_FUNCTION_RE, common.scrub_body, raw_tool_output, prefixes, False
+            common.OPT_FUNCTION_RE, common.scrub_body, raw_tool_output, prefixes
         )
         builder.processed_prefixes(prefixes)
     else:
@@ -360,8 +360,13 @@ def main():
 
         # Store only filechecked runlines.
         filecheck_run_list = [i for i in run_list if i[0]]
+        ginfo = common.make_ir_generalizer(version=ti.args.version)
         builder = common.FunctionTestBuilder(
-            run_list=filecheck_run_list, flags=ti.args, scrubber_args=[], path=ti.path
+            run_list=filecheck_run_list,
+            flags=ti.args,
+            scrubber_args=[],
+            path=ti.path,
+            ginfo=ginfo,
         )
 
         for prefixes, args, extra_commands, triple_in_cmd in run_list:
@@ -415,29 +420,18 @@ def main():
 
             # Now generate all the checks.
             def check_generator(my_output_lines, prefixes, func):
-                if "-emit-llvm" in clang_args:
-                    return common.add_ir_checks(
-                        my_output_lines,
-                        "//",
-                        prefixes,
-                        func_dict,
-                        func,
-                        False,
-                        ti.args.function_signature,
-                        ti.args.version,
-                        global_vars_seen_dict,
-                        is_filtered=builder.is_filtered(),
-                    )
-                else:
-                    return asm.add_checks(
-                        my_output_lines,
-                        "//",
-                        prefixes,
-                        func_dict,
-                        func,
-                        global_vars_seen_dict,
-                        is_filtered=builder.is_filtered(),
-                    )
+                return common.add_ir_checks(
+                    my_output_lines,
+                    "//",
+                    prefixes,
+                    func_dict,
+                    func,
+                    False,
+                    ti.args.function_signature,
+                    ginfo,
+                    global_vars_seen_dict,
+                    is_filtered=builder.is_filtered(),
+                )
 
             if ti.args.check_globals != 'none':
                 generated_prefixes.extend(
@@ -446,6 +440,7 @@ def main():
                         "//",
                         run_list,
                         output_lines,
+                        ginfo,
                         global_vars_seen_dict,
                         False,
                         True,
@@ -506,6 +501,7 @@ def main():
                                         "//",
                                         run_list,
                                         output_lines,
+                                        ginfo,
                                         global_vars_seen_dict,
                                         False,
                                         True,
@@ -525,7 +521,7 @@ def main():
                                     mangled,
                                     False,
                                     args.function_signature,
-                                    args.version,
+                                    ginfo,
                                     global_vars_seen_dict,
                                     is_filtered=builder.is_filtered(),
                                 )
@@ -543,6 +539,7 @@ def main():
                     "//",
                     run_list,
                     output_lines,
+                    ginfo,
                     global_vars_seen_dict,
                     False,
                     False,
diff --git a/llvm/utils/update_llc_test_checks.py b/llvm/utils/update_llc_test_checks.py
index 1ed0132781e2..c8598e74a134 100755
--- a/llvm/utils/update_llc_test_checks.py
+++ b/llvm/utils/update_llc_test_checks.py
@@ -133,6 +133,7 @@ def main():
         else:
             check_indent = ""
 
+        ginfo = common.make_asm_generalizer(version=1)
         builder = common.FunctionTestBuilder(
             run_list=run_list,
             flags=type(
@@ -148,6 +149,7 @@ def main():
             ),
             scrubber_args=[ti.args],
             path=ti.path,
+            ginfo=ginfo,
         )
 
         for (
@@ -173,9 +175,7 @@ def main():
                 triple = common.get_triple_from_march(march_in_cmd)
 
             scrubber, function_re = output_type.get_run_handler(triple)
-            builder.process_run_line(
-                function_re, scrubber, raw_tool_output, prefixes, True
-            )
+            builder.process_run_line(function_re, scrubber, raw_tool_output, prefixes)
             builder.processed_prefixes(prefixes)
 
         func_dict = builder.finish_and_get_func_dict()
@@ -218,6 +218,7 @@ def main():
                     prefixes,
                     func_dict,
                     func,
+                    ginfo,
                     global_vars_seen_dict,
                     is_filtered=builder.is_filtered(),
                 ),
@@ -243,6 +244,7 @@ def main():
                             run_list,
                             func_dict,
                             func_name,
+                            ginfo,
                             global_vars_seen_dict,
                             is_filtered=builder.is_filtered(),
                         )
diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py
index 04808ce6bb1c..16f3e618770b 100755
--- a/llvm/utils/update_test_checks.py
+++ b/llvm/utils/update_test_checks.py
@@ -147,9 +147,14 @@ def main():
             # now, we just ignore all but the last.
             prefix_list.append((check_prefixes, tool_cmd_args, preprocess_cmd))
 
+        ginfo = common.make_ir_generalizer(ti.args.version)
         global_vars_seen_dict = {}
         builder = common.FunctionTestBuilder(
-            run_list=prefix_list, flags=ti.args, scrubber_args=[], path=ti.path
+            run_list=prefix_list,
+            flags=ti.args,
+            scrubber_args=[],
+            path=ti.path,
+            ginfo=ginfo,
         )
 
         tool_binary = ti.args.tool_binary
@@ -172,7 +177,6 @@ def main():
                 common.scrub_body,
                 raw_tool_output,
                 prefixes,
-                False,
             )
             builder.processed_prefixes(prefixes)
 
@@ -217,6 +221,7 @@ def main():
                         ";",
                         prefix_list,
                         output_lines,
+                        ginfo,
                         global_vars_seen_dict,
                         args.preserve_names,
                         True,
@@ -239,7 +244,7 @@ def main():
                         func,
                         False,
                         args.function_signature,
-                        args.version,
+                        ginfo,
                         global_vars_seen_dict,
                         is_filtered=builder.is_filtered(),
                         original_check_lines=original_check_lines.get(func, {}),
@@ -271,7 +276,7 @@ def main():
                             func_name,
                             args.preserve_names,
                             args.function_signature,
-                            args.version,
+                            ginfo,
                             global_vars_seen_dict,
                             is_filtered=builder.is_filtered(),
                             original_check_lines=original_check_lines.get(
@@ -290,6 +295,7 @@ def main():
                                 ";",
                                 prefix_list,
                                 output_lines,
+                                ginfo,
                                 global_vars_seen_dict,
                                 args.preserve_names,
                                 True,
@@ -337,6 +343,7 @@ def main():
                     ";",
                     prefix_list,
                     output_lines,
+                    ginfo,
                     global_vars_seen_dict,
                     args.preserve_names,
                     False,
-- 
GitLab


From d34be649af1aa849c21a5a0570617c3a89d5f0b8 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Sat, 18 May 2024 19:39:57 -0700
Subject: [PATCH 1205/1206] [ThinLTO]Sort imported GUIDs before cache key
 update (#92622)

Add 'sort' here since it's helpful when container type
changes (for example, https://github.com/llvm/llvm-project/pull/88024
wants to change container type from `unordered_set` to `DenseMap)

@MaskRay points out `std::` doesn't randomize the iteration order of
`unordered_{set,map}`, and the iteration order for single build is
deterministic.
---
 llvm/lib/LTO/LTO.cpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 21cad1de0ced..5c603ac6ab47 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -199,13 +199,19 @@ void llvm::computeLTOCacheKey(
              [](const ImportModule &Lhs, const ImportModule &Rhs) -> bool {
                return Lhs.getHash() < Rhs.getHash();
              });
+  std::vector<uint64_t> ImportedGUIDs;
   for (const ImportModule &Entry : ImportModulesVector) {
     auto ModHash = Entry.getHash();
     Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
 
     AddUint64(Entry.getFunctions().size());
+
+    ImportedGUIDs.clear();
     for (auto &Fn : Entry.getFunctions())
-      AddUint64(Fn);
+      ImportedGUIDs.push_back(Fn);
+    llvm::sort(ImportedGUIDs);
+    for (auto &GUID : ImportedGUIDs)
+      AddUint64(GUID);
   }
 
   // Include the hash for the resolved ODR.
-- 
GitLab


From 7b977e0f644c43232732e149b03d41de321d804e Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Sat, 18 May 2024 21:51:14 -0700
Subject: [PATCH 1206/1206] [nfc][InstrFDO]Encapsulate header writes in a class
 member function (#90142)

The smaller class member are more focused and easier to maintain. This
also paves the way for partial header forward compatibility in
https://github.com/llvm/llvm-project/pull/88212

---------

Co-authored-by: Kazu Hirata <kazu@google.com>
---
 .../llvm/ProfileData/InstrProfWriter.h        |  6 ++
 llvm/lib/ProfileData/InstrProfWriter.cpp      | 70 +++++++++----------
 2 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index 1714f3b6cf3a..97f6a95ab715 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -212,6 +212,12 @@ private:
   void addTemporalProfileTrace(TemporalProfTraceTy Trace);
 
   Error writeImpl(ProfOStream &OS);
+
+  // Writes known header fields and reserves space for fields whose value are
+  // known only after payloads are written. Returns the start byte offset for
+  // back patching.
+  uint64_t writeHeader(const IndexedInstrProf::Header &header,
+                       const bool WritePrevVersion, ProfOStream &OS);
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index b5b13550b057..101992c38353 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -639,6 +639,27 @@ static Error writeMemProf(ProfOStream &OS,
               memprof::MaximumSupportedVersion));
 }
 
+uint64_t InstrProfWriter::writeHeader(const IndexedInstrProf::Header &Header,
+                                      const bool WritePrevVersion,
+                                      ProfOStream &OS) {
+  // Only write out the first four fields.
+  for (int I = 0; I < 4; I++)
+    OS.write(reinterpret_cast<const uint64_t *>(&Header)[I]);
+
+  // Remember the offset of the remaining fields to allow back patching later.
+  auto BackPatchStartOffset = OS.tell();
+
+  // Reserve the space for back patching later.
+  OS.write(0); // HashOffset
+  OS.write(0); // MemProfOffset
+  OS.write(0); // BinaryIdOffset
+  OS.write(0); // TemporalProfTracesOffset
+  if (!WritePrevVersion)
+    OS.write(0); // VTableNamesOffset
+
+  return BackPatchStartOffset;
+}
+
 Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   using namespace IndexedInstrProf;
   using namespace support;
@@ -651,7 +672,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   InfoObj->CSSummaryBuilder = &CSISB;
 
   // Populate the hash table generator.
-  SmallVector<std::pair<StringRef, const ProfilingData *>, 0> OrderedData;
+  SmallVector<std::pair<StringRef, const ProfilingData *>> OrderedData;
   for (const auto &I : FunctionData)
     if (shouldEncodeData(I.getValue()))
       OrderedData.emplace_back((I.getKey()), &I.getValue());
@@ -693,35 +714,8 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   Header.TemporalProfTracesOffset = 0;
   Header.VTableNamesOffset = 0;
 
-  // Only write out the first four fields. We need to remember the offset of the
-  // remaining fields to allow back patching later.
-  for (int I = 0; I < 4; I++)
-    OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
-
-  // Save the location of Header.HashOffset field in \c OS.
-  uint64_t HashTableStartFieldOffset = OS.tell();
-  // Reserve the space for HashOffset field.
-  OS.write(0);
-
-  // Save the location of MemProf profile data. This is stored in two parts as
-  // the schema and as a separate on-disk chained hashtable.
-  uint64_t MemProfSectionOffset = OS.tell();
-  // Reserve space for the MemProf table field to be patched later if this
-  // profile contains memory profile information.
-  OS.write(0);
-
-  // Save the location of binary ids section.
-  uint64_t BinaryIdSectionOffset = OS.tell();
-  // Reserve space for the BinaryIdOffset field to be patched later if this
-  // profile contains binary ids.
-  OS.write(0);
-
-  uint64_t TemporalProfTracesOffset = OS.tell();
-  OS.write(0);
-
-  uint64_t VTableNamesOffset = OS.tell();
-  if (!WritePrevVersion)
-    OS.write(0);
+  const uint64_t BackPatchStartOffset =
+      writeHeader(Header, WritePrevVersion, OS);
 
   // Reserve space to write profile summary data.
   uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
@@ -850,16 +844,20 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   }
   InfoObj->CSSummaryBuilder = nullptr;
 
+  const size_t MemProfOffset = BackPatchStartOffset + sizeof(uint64_t);
+  const size_t BinaryIdOffset = MemProfOffset + sizeof(uint64_t);
+  const size_t TemporalProfTracesOffset = BinaryIdOffset + sizeof(uint64_t);
+  const size_t VTableNamesOffset = TemporalProfTracesOffset + sizeof(uint64_t);
   if (!WritePrevVersion) {
     // Now do the final patch:
     PatchItem PatchItems[] = {
         // Patch the Header.HashOffset field.
-        {HashTableStartFieldOffset, &HashTableStart, 1},
+        {BackPatchStartOffset, &HashTableStart, 1},
         // Patch the Header.MemProfOffset (=0 for profiles without MemProf
         // data).
-        {MemProfSectionOffset, &MemProfSectionStart, 1},
+        {MemProfOffset, &MemProfSectionStart, 1},
         // Patch the Header.BinaryIdSectionOffset.
-        {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
+        {BinaryIdOffset, &BinaryIdSectionStart, 1},
         // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
         // traces).
         {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
@@ -875,12 +873,12 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
     // Now do the final patch:
     PatchItem PatchItems[] = {
         // Patch the Header.HashOffset field.
-        {HashTableStartFieldOffset, &HashTableStart, 1},
+        {BackPatchStartOffset, &HashTableStart, 1},
         // Patch the Header.MemProfOffset (=0 for profiles without MemProf
         // data).
-        {MemProfSectionOffset, &MemProfSectionStart, 1},
+        {MemProfOffset, &MemProfSectionStart, 1},
         // Patch the Header.BinaryIdSectionOffset.
-        {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
+        {BinaryIdOffset, &BinaryIdSectionStart, 1},
         // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
         // traces).
         {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
-- 
GitLab